diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5b19b730d854a5df64f84f21b9828c7b6ab6ef3e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_cmp.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_cmp.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cedca2d965172ec3404e3ba2faf8313ea303306c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_cmp.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_compat.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_compat.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..045ff13aeafd1db7abc240e4934e0fcf13e36518 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_compat.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_config.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..77d991dab0be79c1b9e7519a539594a9fba47eb8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_config.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_funcs.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_funcs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..954ed249fcaa21cd6aae877911edbd743aa70436 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_funcs.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_make.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_make.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..69cb792b576d137b344a423f75d4c0757fdb91fa Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_make.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_next_gen.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_next_gen.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c6c873310e0b35aa5ed4926ec5fe4b03099d1a0e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_next_gen.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_version_info.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_version_info.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bd4f81c58dd541d1d89d4ee8c0d87b867e4138cc Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/_version_info.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/converters.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/converters.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..354a680b33360f2c774ad3bbd532b0ba848d821f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/converters.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/exceptions.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1018af9ba95fa21ff09972adc662739f800c9cf3 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/exceptions.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/filters.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/filters.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3dafcd8271b9220234de1f5b5526b7a9c8c97f00 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/filters.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/setters.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/setters.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e40a247e77078b9419594f6b3cd26856f505ae61 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/setters.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/validators.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/validators.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b5cf1f81f37791799fb0e676d7cc22e66165df88 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attr/__pycache__/validators.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs-25.1.0.dist-info/licenses/LICENSE b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs-25.1.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..2bd6453d255e19b973f19b128596a8b6dd65b2c3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs-25.1.0.dist-info/licenses/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Hynek Schlawack and the attrs contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cfb701f46a3200fe2b58737b2657887cf818ac65 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/converters.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/converters.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c97ec64698bcfc5df110ba919b57d242300c52dd Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/converters.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/exceptions.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f25c3070f5918174308f3d2eb6087359c9010797 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/exceptions.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/filters.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/filters.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..220e06d776085e19c06c36c41d3894d939350f10 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/filters.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/setters.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/setters.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24e26bdc1144c44199f2f10605625d91b431d0c1 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/setters.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/validators.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/validators.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ee27671fcb57fd37f2d954a03da856b6e63b5d58 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/attrs/__pycache__/validators.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fbdc7a66a1ed328781fd542763ae7a545347be90 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/__main__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/__main__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..55c1a49a8429ee50fb8dbd94ab6877f33169e565 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/__main__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/big5freq.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/big5freq.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..740d3968a5dbe1b209591d823a879119238864c8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/big5freq.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/big5prober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/big5prober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5d47bf61b7c16ac159113847b7bf34d788002798 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/big5prober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/chardistribution.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/chardistribution.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8a2c468bdf1f51551ec0fd3fb32b73a19616bc44 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/chardistribution.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/charsetgroupprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/charsetgroupprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..175e2693d4e60a687550f8e2ef8831df4430f4c9 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/charsetgroupprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/charsetprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/charsetprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fec10f4e51e824d2eafe38cd6395f7935282f809 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/charsetprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/codingstatemachine.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/codingstatemachine.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..25f0bd3e73e687b41c570959ffb5ddfced7f0288 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/codingstatemachine.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/codingstatemachinedict.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/codingstatemachinedict.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d7523b7d05e8516a77e71b230e755cf1301272f3 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/codingstatemachinedict.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/cp949prober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/cp949prober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..024028c7102fcb9356c6c8e832b61d748a632d0c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/cp949prober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/enums.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/enums.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..552b387bca35573a42aa13ef113f509ec52c029b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/enums.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/escprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/escprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7b5854b8023dae065fe29543e9658134d60ba53e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/escprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/escsm.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/escsm.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6cc496ab3a16167776a73a152a22e2cee362f365 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/escsm.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/eucjpprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/eucjpprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fc8b6cfff111e752846f8c4845a69cc7061a11e2 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/eucjpprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/euckrfreq.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/euckrfreq.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea350d2b6a087a2929f20a556f54a53c13b7425f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/euckrfreq.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/euckrprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/euckrprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..75be8dc0d08847408f429cbc4e77d9ce05dbd8f9 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/euckrprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/euctwfreq.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/euctwfreq.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..46006c583710735b292898a582993fbd755c0d91 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/euctwfreq.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/euctwprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/euctwprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3ac787ef896ac55edefb0bf7fd0da013f07bdfdd Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/euctwprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/gb2312freq.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/gb2312freq.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3c235592182ebce6456db4d918048de1d1ba2abe Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/gb2312freq.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/gb2312prober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/gb2312prober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..503be373740faf78de3ab4ec56c5118dc7391259 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/gb2312prober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/hebrewprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/hebrewprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6611bddbb593ac22acc5a6c2a34d581c4f7a6113 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/hebrewprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/jisfreq.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/jisfreq.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0daea37c2e186238d4c2aac9456b0991ab16e6ed Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/jisfreq.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/johabfreq.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/johabfreq.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..12a8bd04948904476207c29ec606771504d1cbbb Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/johabfreq.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/johabprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/johabprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b8fa53cee4871a0de61f9c7d004130760708a9ae Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/johabprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/jpcntx.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/jpcntx.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e29d09b87d2b9470da1a5f5df9c7129737a24919 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/jpcntx.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langbulgarianmodel.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langbulgarianmodel.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8803d654710b9f7368a3470f846b2ec4ef522a52 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langbulgarianmodel.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langgreekmodel.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langgreekmodel.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5266f62c659bf17651884e53f1124747e5cd4d62 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langgreekmodel.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langhebrewmodel.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langhebrewmodel.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5f058f10ce8d59630af3f2df308942eef3ed4d52 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langhebrewmodel.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langhungarianmodel.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langhungarianmodel.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d6b86b5fa3780ca35e92de2191bc566ea3c2995 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langhungarianmodel.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langthaimodel.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langthaimodel.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f7782857532fbb6a6ef4ebb9928d4a12006410f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langthaimodel.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langturkishmodel.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langturkishmodel.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..89688171818ade9845bc1bebf8480fddb795f474 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/langturkishmodel.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/latin1prober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/latin1prober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1b51db069f784f44da6ad4503e5c07eb70a5dab Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/latin1prober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/macromanprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/macromanprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a97f85309e844992e899ccac8811856e7a71ebd7 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/macromanprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/mbcharsetprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/mbcharsetprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3b4039386db6a073a7c8d82542a04d3ddb0f0e9f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/mbcharsetprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/mbcsgroupprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/mbcsgroupprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f4289aaa2f0a764e6aa6707c7bbd29442bcfb826 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/mbcsgroupprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/mbcssm.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/mbcssm.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..87042c9ab87bc7a34a59294139ecd45b5c0e2ad9 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/mbcssm.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/resultdict.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/resultdict.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1e8141a72db674239c2d36592204e1be67d00d94 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/resultdict.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/sbcharsetprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/sbcharsetprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..438bee7175755190a3b88d9839bc5e9e85201d66 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/sbcharsetprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/sbcsgroupprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/sbcsgroupprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..05d22a85fd03e472d0c6ebe86ca300eabb2a78d9 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/sbcsgroupprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/sjisprober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/sjisprober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e81b9d6f34d11fd572098cd67a24a3fe7d16e535 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/sjisprober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/universaldetector.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/universaldetector.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea7d6c03352c608472cd0d46a5810b340cd3facf Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/universaldetector.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/utf1632prober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/utf1632prober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bfc08532749751e81c698ccdb3f3bda1c56e7076 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/utf1632prober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/utf8prober.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/utf8prober.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..654d5583185eafb9008b5ec5b6f73007c33a35f2 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/utf8prober.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/version.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2859e178fc5ced021ea5d26203034e5e48019158 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/__pycache__/version.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/cli/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/cli/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/cli/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/cli/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe411b358bd6fa43c1b90ddcb514d9d96ccf6c82 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/cli/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/cli/__pycache__/chardetect.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/cli/__pycache__/chardetect.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea78a514478f5c026bb4877c92165fb84de54597 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/cli/__pycache__/chardetect.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/cli/chardetect.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/cli/chardetect.py new file mode 100644 index 0000000000000000000000000000000000000000..43f6e144f677a113b5362dcbdfb75db4f41c2b2f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/cli/chardetect.py @@ -0,0 +1,112 @@ +""" +Script which takes one or more file paths and reports on their detected +encodings + +Example:: + + % chardetect somefile someotherfile + somefile: windows-1252 with confidence 0.5 + someotherfile: ascii with confidence 1.0 + +If no paths are provided, it takes its input from stdin. + +""" + + +import argparse +import sys +from typing import Iterable, List, Optional + +from .. import __version__ +from ..universaldetector import UniversalDetector + + +def description_of( + lines: Iterable[bytes], + name: str = "stdin", + minimal: bool = False, + should_rename_legacy: bool = False, +) -> Optional[str]: + """ + Return a string describing the probable encoding of a file or + list of strings. + + :param lines: The lines to get the encoding of. + :type lines: Iterable of bytes + :param name: Name of file or collection of lines + :type name: str + :param should_rename_legacy: Should we rename legacy encodings to + their more modern equivalents? + :type should_rename_legacy: ``bool`` + """ + u = UniversalDetector(should_rename_legacy=should_rename_legacy) + for line in lines: + line = bytearray(line) + u.feed(line) + # shortcut out of the loop to save reading further - particularly useful if we read a BOM. + if u.done: + break + u.close() + result = u.result + if minimal: + return result["encoding"] + if result["encoding"]: + return f'{name}: {result["encoding"]} with confidence {result["confidence"]}' + return f"{name}: no result" + + +def main(argv: Optional[List[str]] = None) -> None: + """ + Handles command line arguments and gets things started. + + :param argv: List of arguments, as if specified on the command-line. + If None, ``sys.argv[1:]`` is used instead. + :type argv: list of str + """ + # Get command line arguments + parser = argparse.ArgumentParser( + description=( + "Takes one or more file paths and reports their detected encodings" + ) + ) + parser.add_argument( + "input", + help="File whose encoding we would like to determine. (default: stdin)", + type=argparse.FileType("rb"), + nargs="*", + default=[sys.stdin.buffer], + ) + parser.add_argument( + "--minimal", + help="Print only the encoding to standard output", + action="store_true", + ) + parser.add_argument( + "-l", + "--legacy", + help="Rename legacy encodings to more modern ones.", + action="store_true", + ) + parser.add_argument( + "--version", action="version", version=f"%(prog)s {__version__}" + ) + args = parser.parse_args(argv) + + for f in args.input: + if f.isatty(): + print( + "You are running chardetect interactively. Press " + "CTRL-D twice at the start of a blank line to signal the " + "end of your input. If you want help, run chardetect " + "--help\n", + file=sys.stderr, + ) + print( + description_of( + f, f.name, minimal=args.minimal, should_rename_legacy=args.legacy + ) + ) + + +if __name__ == "__main__": + main() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/metadata/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/metadata/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/metadata/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/metadata/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..85d0abbf29dd05da23c9fcb69f68ed756e14d18b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/metadata/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/metadata/__pycache__/languages.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/metadata/__pycache__/languages.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38d0804b50c139ecd5e99d6ce7162b00f2b5f059 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/metadata/__pycache__/languages.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/metadata/languages.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/metadata/languages.py new file mode 100644 index 0000000000000000000000000000000000000000..eb40c5f0c8526208d434d762855d23079dc68b36 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/chardet/metadata/languages.py @@ -0,0 +1,352 @@ +""" +Metadata about languages used by our model training code for our +SingleByteCharSetProbers. Could be used for other things in the future. + +This code is based on the language metadata from the uchardet project. +""" + +from string import ascii_letters +from typing import List, Optional + +# TODO: Add Ukrainian (KOI8-U) + + +class Language: + """Metadata about a language useful for training models + + :ivar name: The human name for the language, in English. + :type name: str + :ivar iso_code: 2-letter ISO 639-1 if possible, 3-letter ISO code otherwise, + or use another catalog as a last resort. + :type iso_code: str + :ivar use_ascii: Whether or not ASCII letters should be included in trained + models. + :type use_ascii: bool + :ivar charsets: The charsets we want to support and create data for. + :type charsets: list of str + :ivar alphabet: The characters in the language's alphabet. If `use_ascii` is + `True`, you only need to add those not in the ASCII set. + :type alphabet: str + :ivar wiki_start_pages: The Wikipedia pages to start from if we're crawling + Wikipedia for training data. + :type wiki_start_pages: list of str + """ + + def __init__( + self, + name: Optional[str] = None, + iso_code: Optional[str] = None, + use_ascii: bool = True, + charsets: Optional[List[str]] = None, + alphabet: Optional[str] = None, + wiki_start_pages: Optional[List[str]] = None, + ) -> None: + super().__init__() + self.name = name + self.iso_code = iso_code + self.use_ascii = use_ascii + self.charsets = charsets + if self.use_ascii: + if alphabet: + alphabet += ascii_letters + else: + alphabet = ascii_letters + elif not alphabet: + raise ValueError("Must supply alphabet if use_ascii is False") + self.alphabet = "".join(sorted(set(alphabet))) if alphabet else None + self.wiki_start_pages = wiki_start_pages + + def __repr__(self) -> str: + param_str = ", ".join( + f"{k}={v!r}" for k, v in self.__dict__.items() if not k.startswith("_") + ) + return f"{self.__class__.__name__}({param_str})" + + +LANGUAGES = { + "Arabic": Language( + name="Arabic", + iso_code="ar", + use_ascii=False, + # We only support encodings that use isolated + # forms, because the current recommendation is + # that the rendering system handles presentation + # forms. This means we purposefully skip IBM864. + charsets=["ISO-8859-6", "WINDOWS-1256", "CP720", "CP864"], + alphabet="ءآأؤإئابةتثجحخدذرزسشصضطظعغػؼؽؾؿـفقكلمنهوىيًٌٍَُِّ", + wiki_start_pages=["الصفحة_الرئيسية"], + ), + "Belarusian": Language( + name="Belarusian", + iso_code="be", + use_ascii=False, + charsets=["ISO-8859-5", "WINDOWS-1251", "IBM866", "MacCyrillic"], + alphabet="АБВГДЕЁЖЗІЙКЛМНОПРСТУЎФХЦЧШЫЬЭЮЯабвгдеёжзійклмнопрстуўфхцчшыьэюяʼ", + wiki_start_pages=["Галоўная_старонка"], + ), + "Bulgarian": Language( + name="Bulgarian", + iso_code="bg", + use_ascii=False, + charsets=["ISO-8859-5", "WINDOWS-1251", "IBM855"], + alphabet="АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯабвгдежзийклмнопрстуфхцчшщъьюя", + wiki_start_pages=["Начална_страница"], + ), + "Czech": Language( + name="Czech", + iso_code="cz", + use_ascii=True, + charsets=["ISO-8859-2", "WINDOWS-1250"], + alphabet="áčďéěíňóřšťúůýžÁČĎÉĚÍŇÓŘŠŤÚŮÝŽ", + wiki_start_pages=["Hlavní_strana"], + ), + "Danish": Language( + name="Danish", + iso_code="da", + use_ascii=True, + charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"], + alphabet="æøåÆØÅ", + wiki_start_pages=["Forside"], + ), + "German": Language( + name="German", + iso_code="de", + use_ascii=True, + charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"], + alphabet="äöüßẞÄÖÜ", + wiki_start_pages=["Wikipedia:Hauptseite"], + ), + "Greek": Language( + name="Greek", + iso_code="el", + use_ascii=False, + charsets=["ISO-8859-7", "WINDOWS-1253"], + alphabet="αβγδεζηθικλμνξοπρσςτυφχψωάέήίόύώΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΣΤΥΦΧΨΩΆΈΉΊΌΎΏ", + wiki_start_pages=["Πύλη:Κύρια"], + ), + "English": Language( + name="English", + iso_code="en", + use_ascii=True, + charsets=["ISO-8859-1", "WINDOWS-1252", "MacRoman"], + wiki_start_pages=["Main_Page"], + ), + "Esperanto": Language( + name="Esperanto", + iso_code="eo", + # Q, W, X, and Y not used at all + use_ascii=False, + charsets=["ISO-8859-3"], + alphabet="abcĉdefgĝhĥijĵklmnoprsŝtuŭvzABCĈDEFGĜHĤIJĴKLMNOPRSŜTUŬVZ", + wiki_start_pages=["Vikipedio:Ĉefpaĝo"], + ), + "Spanish": Language( + name="Spanish", + iso_code="es", + use_ascii=True, + charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"], + alphabet="ñáéíóúüÑÁÉÍÓÚÜ", + wiki_start_pages=["Wikipedia:Portada"], + ), + "Estonian": Language( + name="Estonian", + iso_code="et", + use_ascii=False, + charsets=["ISO-8859-4", "ISO-8859-13", "WINDOWS-1257"], + # C, F, Š, Q, W, X, Y, Z, Ž are only for + # loanwords + alphabet="ABDEGHIJKLMNOPRSTUVÕÄÖÜabdeghijklmnoprstuvõäöü", + wiki_start_pages=["Esileht"], + ), + "Finnish": Language( + name="Finnish", + iso_code="fi", + use_ascii=True, + charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"], + alphabet="ÅÄÖŠŽåäöšž", + wiki_start_pages=["Wikipedia:Etusivu"], + ), + "French": Language( + name="French", + iso_code="fr", + use_ascii=True, + charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"], + alphabet="œàâçèéîïùûêŒÀÂÇÈÉÎÏÙÛÊ", + wiki_start_pages=["Wikipédia:Accueil_principal", "Bœuf (animal)"], + ), + "Hebrew": Language( + name="Hebrew", + iso_code="he", + use_ascii=False, + charsets=["ISO-8859-8", "WINDOWS-1255"], + alphabet="אבגדהוזחטיךכלםמןנסעףפץצקרשתװױײ", + wiki_start_pages=["עמוד_ראשי"], + ), + "Croatian": Language( + name="Croatian", + iso_code="hr", + # Q, W, X, Y are only used for foreign words. + use_ascii=False, + charsets=["ISO-8859-2", "WINDOWS-1250"], + alphabet="abcčćdđefghijklmnoprsštuvzžABCČĆDĐEFGHIJKLMNOPRSŠTUVZŽ", + wiki_start_pages=["Glavna_stranica"], + ), + "Hungarian": Language( + name="Hungarian", + iso_code="hu", + # Q, W, X, Y are only used for foreign words. + use_ascii=False, + charsets=["ISO-8859-2", "WINDOWS-1250"], + alphabet="abcdefghijklmnoprstuvzáéíóöőúüűABCDEFGHIJKLMNOPRSTUVZÁÉÍÓÖŐÚÜŰ", + wiki_start_pages=["Kezdőlap"], + ), + "Italian": Language( + name="Italian", + iso_code="it", + use_ascii=True, + charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"], + alphabet="ÀÈÉÌÒÓÙàèéìòóù", + wiki_start_pages=["Pagina_principale"], + ), + "Lithuanian": Language( + name="Lithuanian", + iso_code="lt", + use_ascii=False, + charsets=["ISO-8859-13", "WINDOWS-1257", "ISO-8859-4"], + # Q, W, and X not used at all + alphabet="AĄBCČDEĘĖFGHIĮYJKLMNOPRSŠTUŲŪVZŽaąbcčdeęėfghiįyjklmnoprsštuųūvzž", + wiki_start_pages=["Pagrindinis_puslapis"], + ), + "Latvian": Language( + name="Latvian", + iso_code="lv", + use_ascii=False, + charsets=["ISO-8859-13", "WINDOWS-1257", "ISO-8859-4"], + # Q, W, X, Y are only for loanwords + alphabet="AĀBCČDEĒFGĢHIĪJKĶLĻMNŅOPRSŠTUŪVZŽaābcčdeēfgģhiījkķlļmnņoprsštuūvzž", + wiki_start_pages=["Sākumlapa"], + ), + "Macedonian": Language( + name="Macedonian", + iso_code="mk", + use_ascii=False, + charsets=["ISO-8859-5", "WINDOWS-1251", "MacCyrillic", "IBM855"], + alphabet="АБВГДЃЕЖЗЅИЈКЛЉМНЊОПРСТЌУФХЦЧЏШабвгдѓежзѕијклљмнњопрстќуфхцчџш", + wiki_start_pages=["Главна_страница"], + ), + "Dutch": Language( + name="Dutch", + iso_code="nl", + use_ascii=True, + charsets=["ISO-8859-1", "WINDOWS-1252", "MacRoman"], + wiki_start_pages=["Hoofdpagina"], + ), + "Polish": Language( + name="Polish", + iso_code="pl", + # Q and X are only used for foreign words. + use_ascii=False, + charsets=["ISO-8859-2", "WINDOWS-1250"], + alphabet="AĄBCĆDEĘFGHIJKLŁMNŃOÓPRSŚTUWYZŹŻaąbcćdeęfghijklłmnńoóprsśtuwyzźż", + wiki_start_pages=["Wikipedia:Strona_główna"], + ), + "Portuguese": Language( + name="Portuguese", + iso_code="pt", + use_ascii=True, + charsets=["ISO-8859-1", "ISO-8859-15", "WINDOWS-1252", "MacRoman"], + alphabet="ÁÂÃÀÇÉÊÍÓÔÕÚáâãàçéêíóôõú", + wiki_start_pages=["Wikipédia:Página_principal"], + ), + "Romanian": Language( + name="Romanian", + iso_code="ro", + use_ascii=True, + charsets=["ISO-8859-2", "WINDOWS-1250"], + alphabet="ăâîșțĂÂÎȘȚ", + wiki_start_pages=["Pagina_principală"], + ), + "Russian": Language( + name="Russian", + iso_code="ru", + use_ascii=False, + charsets=[ + "ISO-8859-5", + "WINDOWS-1251", + "KOI8-R", + "MacCyrillic", + "IBM866", + "IBM855", + ], + alphabet="абвгдеёжзийклмнопрстуфхцчшщъыьэюяАБВГДЕЁЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯ", + wiki_start_pages=["Заглавная_страница"], + ), + "Slovak": Language( + name="Slovak", + iso_code="sk", + use_ascii=True, + charsets=["ISO-8859-2", "WINDOWS-1250"], + alphabet="áäčďéíĺľňóôŕšťúýžÁÄČĎÉÍĹĽŇÓÔŔŠŤÚÝŽ", + wiki_start_pages=["Hlavná_stránka"], + ), + "Slovene": Language( + name="Slovene", + iso_code="sl", + # Q, W, X, Y are only used for foreign words. + use_ascii=False, + charsets=["ISO-8859-2", "WINDOWS-1250"], + alphabet="abcčdefghijklmnoprsštuvzžABCČDEFGHIJKLMNOPRSŠTUVZŽ", + wiki_start_pages=["Glavna_stran"], + ), + # Serbian can be written in both Latin and Cyrillic, but there's no + # simple way to get the Latin alphabet pages from Wikipedia through + # the API, so for now we just support Cyrillic. + "Serbian": Language( + name="Serbian", + iso_code="sr", + alphabet="АБВГДЂЕЖЗИЈКЛЉМНЊОПРСТЋУФХЦЧЏШабвгдђежзијклљмнњопрстћуфхцчџш", + charsets=["ISO-8859-5", "WINDOWS-1251", "MacCyrillic", "IBM855"], + wiki_start_pages=["Главна_страна"], + ), + "Thai": Language( + name="Thai", + iso_code="th", + use_ascii=False, + charsets=["ISO-8859-11", "TIS-620", "CP874"], + alphabet="กขฃคฅฆงจฉชซฌญฎฏฐฑฒณดตถทธนบปผฝพฟภมยรฤลฦวศษสหฬอฮฯะัาำิีึืฺุู฿เแโใไๅๆ็่้๊๋์ํ๎๏๐๑๒๓๔๕๖๗๘๙๚๛", + wiki_start_pages=["หน้าหลัก"], + ), + "Turkish": Language( + name="Turkish", + iso_code="tr", + # Q, W, and X are not used by Turkish + use_ascii=False, + charsets=["ISO-8859-3", "ISO-8859-9", "WINDOWS-1254"], + alphabet="abcçdefgğhıijklmnoöprsştuüvyzâîûABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZÂÎÛ", + wiki_start_pages=["Ana_Sayfa"], + ), + "Vietnamese": Language( + name="Vietnamese", + iso_code="vi", + use_ascii=False, + # Windows-1258 is the only common 8-bit + # Vietnamese encoding supported by Python. + # From Wikipedia: + # For systems that lack support for Unicode, + # dozens of 8-bit Vietnamese code pages are + # available.[1] The most common are VISCII + # (TCVN 5712:1993), VPS, and Windows-1258.[3] + # Where ASCII is required, such as when + # ensuring readability in plain text e-mail, + # Vietnamese letters are often encoded + # according to Vietnamese Quoted-Readable + # (VIQR) or VSCII Mnemonic (VSCII-MNEM),[4] + # though usage of either variable-width + # scheme has declined dramatically following + # the adoption of Unicode on the World Wide + # Web. + charsets=["WINDOWS-1258"], + alphabet="aăâbcdđeêghiklmnoôơpqrstuưvxyAĂÂBCDĐEÊGHIKLMNOÔƠPQRSTUƯVXY", + wiki_start_pages=["Chữ_Quốc_ngữ"], + ), +} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..119b35598e880fa6968f94949723cc1303a41d0d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/ansi.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/ansi.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6f46f8bff447c1dedaa3c048d2dd0272d0ad02ee Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/ansi.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/ansitowin32.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/ansitowin32.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..faa74aeece2cf74881003bb5e8687bcc913aa901 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/ansitowin32.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/initialise.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/initialise.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f7d7329b16fc7819a6ab6b6d08025e04c29c9f8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/initialise.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/win32.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/win32.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..20a2afd9565e6bd8fd7f3376adb05c737c231eb8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/win32.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/winterm.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/winterm.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..60cd6861c7d3f657fda626a26fbedda94172f2c1 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/__pycache__/winterm.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8c5661e93a205bf4fb22404d4fc50f902cc31369 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__init__.py @@ -0,0 +1 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..857f8ad7a52cf7f135aec21039da8b200af8cd8a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/ansi_test.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/ansi_test.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0fa1a5713dbc684b808c1578b3665b7892882162 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/ansi_test.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/ansitowin32_test.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/ansitowin32_test.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..475b64bc14ea88173ca6931a9931fc8bea6c62f8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/ansitowin32_test.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/initialise_test.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/initialise_test.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aa64dfa70e12ca71bb5ac9e1dac09636b377dd68 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/initialise_test.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/isatty_test.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/isatty_test.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ab214e9d6a681561d93f8d9cc98f029e4e9bae2d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/isatty_test.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f37611cdf04953427dafaefdaa7cc67e081ca373 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/winterm_test.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/winterm_test.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff28caa23f6904a7989238fb88b09de9df11ec67 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/__pycache__/winterm_test.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/ansi_test.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/ansi_test.py new file mode 100644 index 0000000000000000000000000000000000000000..0a20c80f882066e0e1323b0c7f61e22913c32e35 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/ansi_test.py @@ -0,0 +1,76 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +import sys +from unittest import TestCase, main + +from ..ansi import Back, Fore, Style +from ..ansitowin32 import AnsiToWin32 + +stdout_orig = sys.stdout +stderr_orig = sys.stderr + + +class AnsiTest(TestCase): + + def setUp(self): + # sanity check: stdout should be a file or StringIO object. + # It will only be AnsiToWin32 if init() has previously wrapped it + self.assertNotEqual(type(sys.stdout), AnsiToWin32) + self.assertNotEqual(type(sys.stderr), AnsiToWin32) + + def tearDown(self): + sys.stdout = stdout_orig + sys.stderr = stderr_orig + + + def testForeAttributes(self): + self.assertEqual(Fore.BLACK, '\033[30m') + self.assertEqual(Fore.RED, '\033[31m') + self.assertEqual(Fore.GREEN, '\033[32m') + self.assertEqual(Fore.YELLOW, '\033[33m') + self.assertEqual(Fore.BLUE, '\033[34m') + self.assertEqual(Fore.MAGENTA, '\033[35m') + self.assertEqual(Fore.CYAN, '\033[36m') + self.assertEqual(Fore.WHITE, '\033[37m') + self.assertEqual(Fore.RESET, '\033[39m') + + # Check the light, extended versions. + self.assertEqual(Fore.LIGHTBLACK_EX, '\033[90m') + self.assertEqual(Fore.LIGHTRED_EX, '\033[91m') + self.assertEqual(Fore.LIGHTGREEN_EX, '\033[92m') + self.assertEqual(Fore.LIGHTYELLOW_EX, '\033[93m') + self.assertEqual(Fore.LIGHTBLUE_EX, '\033[94m') + self.assertEqual(Fore.LIGHTMAGENTA_EX, '\033[95m') + self.assertEqual(Fore.LIGHTCYAN_EX, '\033[96m') + self.assertEqual(Fore.LIGHTWHITE_EX, '\033[97m') + + + def testBackAttributes(self): + self.assertEqual(Back.BLACK, '\033[40m') + self.assertEqual(Back.RED, '\033[41m') + self.assertEqual(Back.GREEN, '\033[42m') + self.assertEqual(Back.YELLOW, '\033[43m') + self.assertEqual(Back.BLUE, '\033[44m') + self.assertEqual(Back.MAGENTA, '\033[45m') + self.assertEqual(Back.CYAN, '\033[46m') + self.assertEqual(Back.WHITE, '\033[47m') + self.assertEqual(Back.RESET, '\033[49m') + + # Check the light, extended versions. + self.assertEqual(Back.LIGHTBLACK_EX, '\033[100m') + self.assertEqual(Back.LIGHTRED_EX, '\033[101m') + self.assertEqual(Back.LIGHTGREEN_EX, '\033[102m') + self.assertEqual(Back.LIGHTYELLOW_EX, '\033[103m') + self.assertEqual(Back.LIGHTBLUE_EX, '\033[104m') + self.assertEqual(Back.LIGHTMAGENTA_EX, '\033[105m') + self.assertEqual(Back.LIGHTCYAN_EX, '\033[106m') + self.assertEqual(Back.LIGHTWHITE_EX, '\033[107m') + + + def testStyleAttributes(self): + self.assertEqual(Style.DIM, '\033[2m') + self.assertEqual(Style.NORMAL, '\033[22m') + self.assertEqual(Style.BRIGHT, '\033[1m') + + +if __name__ == '__main__': + main() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/ansitowin32_test.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/ansitowin32_test.py new file mode 100644 index 0000000000000000000000000000000000000000..91ca551f97b4576c680711e826a1855fb944c872 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/ansitowin32_test.py @@ -0,0 +1,294 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +from io import StringIO, TextIOWrapper +from unittest import TestCase, main +try: + from contextlib import ExitStack +except ImportError: + # python 2 + from contextlib2 import ExitStack + +try: + from unittest.mock import MagicMock, Mock, patch +except ImportError: + from mock import MagicMock, Mock, patch + +from ..ansitowin32 import AnsiToWin32, StreamWrapper +from ..win32 import ENABLE_VIRTUAL_TERMINAL_PROCESSING +from .utils import osname + + +class StreamWrapperTest(TestCase): + + def testIsAProxy(self): + mockStream = Mock() + wrapper = StreamWrapper(mockStream, None) + self.assertTrue( wrapper.random_attr is mockStream.random_attr ) + + def testDelegatesWrite(self): + mockStream = Mock() + mockConverter = Mock() + wrapper = StreamWrapper(mockStream, mockConverter) + wrapper.write('hello') + self.assertTrue(mockConverter.write.call_args, (('hello',), {})) + + def testDelegatesContext(self): + mockConverter = Mock() + s = StringIO() + with StreamWrapper(s, mockConverter) as fp: + fp.write(u'hello') + self.assertTrue(s.closed) + + def testProxyNoContextManager(self): + mockStream = MagicMock() + mockStream.__enter__.side_effect = AttributeError() + mockConverter = Mock() + with self.assertRaises(AttributeError) as excinfo: + with StreamWrapper(mockStream, mockConverter) as wrapper: + wrapper.write('hello') + + def test_closed_shouldnt_raise_on_closed_stream(self): + stream = StringIO() + stream.close() + wrapper = StreamWrapper(stream, None) + self.assertEqual(wrapper.closed, True) + + def test_closed_shouldnt_raise_on_detached_stream(self): + stream = TextIOWrapper(StringIO()) + stream.detach() + wrapper = StreamWrapper(stream, None) + self.assertEqual(wrapper.closed, True) + +class AnsiToWin32Test(TestCase): + + def testInit(self): + mockStdout = Mock() + auto = Mock() + stream = AnsiToWin32(mockStdout, autoreset=auto) + self.assertEqual(stream.wrapped, mockStdout) + self.assertEqual(stream.autoreset, auto) + + @patch('colorama.ansitowin32.winterm', None) + @patch('colorama.ansitowin32.winapi_test', lambda *_: True) + def testStripIsTrueOnWindows(self): + with osname('nt'): + mockStdout = Mock() + stream = AnsiToWin32(mockStdout) + self.assertTrue(stream.strip) + + def testStripIsFalseOffWindows(self): + with osname('posix'): + mockStdout = Mock(closed=False) + stream = AnsiToWin32(mockStdout) + self.assertFalse(stream.strip) + + def testWriteStripsAnsi(self): + mockStdout = Mock() + stream = AnsiToWin32(mockStdout) + stream.wrapped = Mock() + stream.write_and_convert = Mock() + stream.strip = True + + stream.write('abc') + + self.assertFalse(stream.wrapped.write.called) + self.assertEqual(stream.write_and_convert.call_args, (('abc',), {})) + + def testWriteDoesNotStripAnsi(self): + mockStdout = Mock() + stream = AnsiToWin32(mockStdout) + stream.wrapped = Mock() + stream.write_and_convert = Mock() + stream.strip = False + stream.convert = False + + stream.write('abc') + + self.assertFalse(stream.write_and_convert.called) + self.assertEqual(stream.wrapped.write.call_args, (('abc',), {})) + + def assert_autoresets(self, convert, autoreset=True): + stream = AnsiToWin32(Mock()) + stream.convert = convert + stream.reset_all = Mock() + stream.autoreset = autoreset + stream.winterm = Mock() + + stream.write('abc') + + self.assertEqual(stream.reset_all.called, autoreset) + + def testWriteAutoresets(self): + self.assert_autoresets(convert=True) + self.assert_autoresets(convert=False) + self.assert_autoresets(convert=True, autoreset=False) + self.assert_autoresets(convert=False, autoreset=False) + + def testWriteAndConvertWritesPlainText(self): + stream = AnsiToWin32(Mock()) + stream.write_and_convert( 'abc' ) + self.assertEqual( stream.wrapped.write.call_args, (('abc',), {}) ) + + def testWriteAndConvertStripsAllValidAnsi(self): + stream = AnsiToWin32(Mock()) + stream.call_win32 = Mock() + data = [ + 'abc\033[mdef', + 'abc\033[0mdef', + 'abc\033[2mdef', + 'abc\033[02mdef', + 'abc\033[002mdef', + 'abc\033[40mdef', + 'abc\033[040mdef', + 'abc\033[0;1mdef', + 'abc\033[40;50mdef', + 'abc\033[50;30;40mdef', + 'abc\033[Adef', + 'abc\033[0Gdef', + 'abc\033[1;20;128Hdef', + ] + for datum in data: + stream.wrapped.write.reset_mock() + stream.write_and_convert( datum ) + self.assertEqual( + [args[0] for args in stream.wrapped.write.call_args_list], + [ ('abc',), ('def',) ] + ) + + def testWriteAndConvertSkipsEmptySnippets(self): + stream = AnsiToWin32(Mock()) + stream.call_win32 = Mock() + stream.write_and_convert( '\033[40m\033[41m' ) + self.assertFalse( stream.wrapped.write.called ) + + def testWriteAndConvertCallsWin32WithParamsAndCommand(self): + stream = AnsiToWin32(Mock()) + stream.convert = True + stream.call_win32 = Mock() + stream.extract_params = Mock(return_value='params') + data = { + 'abc\033[adef': ('a', 'params'), + 'abc\033[;;bdef': ('b', 'params'), + 'abc\033[0cdef': ('c', 'params'), + 'abc\033[;;0;;Gdef': ('G', 'params'), + 'abc\033[1;20;128Hdef': ('H', 'params'), + } + for datum, expected in data.items(): + stream.call_win32.reset_mock() + stream.write_and_convert( datum ) + self.assertEqual( stream.call_win32.call_args[0], expected ) + + def test_reset_all_shouldnt_raise_on_closed_orig_stdout(self): + stream = StringIO() + converter = AnsiToWin32(stream) + stream.close() + + converter.reset_all() + + def test_wrap_shouldnt_raise_on_closed_orig_stdout(self): + stream = StringIO() + stream.close() + with \ + patch("colorama.ansitowin32.os.name", "nt"), \ + patch("colorama.ansitowin32.winapi_test", lambda: True): + converter = AnsiToWin32(stream) + self.assertTrue(converter.strip) + self.assertFalse(converter.convert) + + def test_wrap_shouldnt_raise_on_missing_closed_attr(self): + with \ + patch("colorama.ansitowin32.os.name", "nt"), \ + patch("colorama.ansitowin32.winapi_test", lambda: True): + converter = AnsiToWin32(object()) + self.assertTrue(converter.strip) + self.assertFalse(converter.convert) + + def testExtractParams(self): + stream = AnsiToWin32(Mock()) + data = { + '': (0,), + ';;': (0,), + '2': (2,), + ';;002;;': (2,), + '0;1': (0, 1), + ';;003;;456;;': (3, 456), + '11;22;33;44;55': (11, 22, 33, 44, 55), + } + for datum, expected in data.items(): + self.assertEqual(stream.extract_params('m', datum), expected) + + def testCallWin32UsesLookup(self): + listener = Mock() + stream = AnsiToWin32(listener) + stream.win32_calls = { + 1: (lambda *_, **__: listener(11),), + 2: (lambda *_, **__: listener(22),), + 3: (lambda *_, **__: listener(33),), + } + stream.call_win32('m', (3, 1, 99, 2)) + self.assertEqual( + [a[0][0] for a in listener.call_args_list], + [33, 11, 22] ) + + def test_osc_codes(self): + mockStdout = Mock() + stream = AnsiToWin32(mockStdout, convert=True) + with patch('colorama.ansitowin32.winterm') as winterm: + data = [ + '\033]0\x07', # missing arguments + '\033]0;foo\x08', # wrong OSC command + '\033]0;colorama_test_title\x07', # should work + '\033]1;colorama_test_title\x07', # wrong set command + '\033]2;colorama_test_title\x07', # should work + '\033]' + ';' * 64 + '\x08', # see issue #247 + ] + for code in data: + stream.write(code) + self.assertEqual(winterm.set_title.call_count, 2) + + def test_native_windows_ansi(self): + with ExitStack() as stack: + def p(a, b): + stack.enter_context(patch(a, b, create=True)) + # Pretend to be on Windows + p("colorama.ansitowin32.os.name", "nt") + p("colorama.ansitowin32.winapi_test", lambda: True) + p("colorama.win32.winapi_test", lambda: True) + p("colorama.winterm.win32.windll", "non-None") + p("colorama.winterm.get_osfhandle", lambda _: 1234) + + # Pretend that our mock stream has native ANSI support + p( + "colorama.winterm.win32.GetConsoleMode", + lambda _: ENABLE_VIRTUAL_TERMINAL_PROCESSING, + ) + SetConsoleMode = Mock() + p("colorama.winterm.win32.SetConsoleMode", SetConsoleMode) + + stdout = Mock() + stdout.closed = False + stdout.isatty.return_value = True + stdout.fileno.return_value = 1 + + # Our fake console says it has native vt support, so AnsiToWin32 should + # enable that support and do nothing else. + stream = AnsiToWin32(stdout) + SetConsoleMode.assert_called_with(1234, ENABLE_VIRTUAL_TERMINAL_PROCESSING) + self.assertFalse(stream.strip) + self.assertFalse(stream.convert) + self.assertFalse(stream.should_wrap()) + + # Now let's pretend we're on an old Windows console, that doesn't have + # native ANSI support. + p("colorama.winterm.win32.GetConsoleMode", lambda _: 0) + SetConsoleMode = Mock() + p("colorama.winterm.win32.SetConsoleMode", SetConsoleMode) + + stream = AnsiToWin32(stdout) + SetConsoleMode.assert_called_with(1234, ENABLE_VIRTUAL_TERMINAL_PROCESSING) + self.assertTrue(stream.strip) + self.assertTrue(stream.convert) + self.assertTrue(stream.should_wrap()) + + +if __name__ == '__main__': + main() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/initialise_test.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/initialise_test.py new file mode 100644 index 0000000000000000000000000000000000000000..89f9b07511c8fee74686d9cc434bf66345a46d6d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/initialise_test.py @@ -0,0 +1,189 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +import sys +from unittest import TestCase, main, skipUnless + +try: + from unittest.mock import patch, Mock +except ImportError: + from mock import patch, Mock + +from ..ansitowin32 import StreamWrapper +from ..initialise import init, just_fix_windows_console, _wipe_internal_state_for_tests +from .utils import osname, replace_by + +orig_stdout = sys.stdout +orig_stderr = sys.stderr + + +class InitTest(TestCase): + + @skipUnless(sys.stdout.isatty(), "sys.stdout is not a tty") + def setUp(self): + # sanity check + self.assertNotWrapped() + + def tearDown(self): + _wipe_internal_state_for_tests() + sys.stdout = orig_stdout + sys.stderr = orig_stderr + + def assertWrapped(self): + self.assertIsNot(sys.stdout, orig_stdout, 'stdout should be wrapped') + self.assertIsNot(sys.stderr, orig_stderr, 'stderr should be wrapped') + self.assertTrue(isinstance(sys.stdout, StreamWrapper), + 'bad stdout wrapper') + self.assertTrue(isinstance(sys.stderr, StreamWrapper), + 'bad stderr wrapper') + + def assertNotWrapped(self): + self.assertIs(sys.stdout, orig_stdout, 'stdout should not be wrapped') + self.assertIs(sys.stderr, orig_stderr, 'stderr should not be wrapped') + + @patch('colorama.initialise.reset_all') + @patch('colorama.ansitowin32.winapi_test', lambda *_: True) + @patch('colorama.ansitowin32.enable_vt_processing', lambda *_: False) + def testInitWrapsOnWindows(self, _): + with osname("nt"): + init() + self.assertWrapped() + + @patch('colorama.initialise.reset_all') + @patch('colorama.ansitowin32.winapi_test', lambda *_: False) + def testInitDoesntWrapOnEmulatedWindows(self, _): + with osname("nt"): + init() + self.assertNotWrapped() + + def testInitDoesntWrapOnNonWindows(self): + with osname("posix"): + init() + self.assertNotWrapped() + + def testInitDoesntWrapIfNone(self): + with replace_by(None): + init() + # We can't use assertNotWrapped here because replace_by(None) + # changes stdout/stderr already. + self.assertIsNone(sys.stdout) + self.assertIsNone(sys.stderr) + + def testInitAutoresetOnWrapsOnAllPlatforms(self): + with osname("posix"): + init(autoreset=True) + self.assertWrapped() + + def testInitWrapOffDoesntWrapOnWindows(self): + with osname("nt"): + init(wrap=False) + self.assertNotWrapped() + + def testInitWrapOffIncompatibleWithAutoresetOn(self): + self.assertRaises(ValueError, lambda: init(autoreset=True, wrap=False)) + + @patch('colorama.win32.SetConsoleTextAttribute') + @patch('colorama.initialise.AnsiToWin32') + def testAutoResetPassedOn(self, mockATW32, _): + with osname("nt"): + init(autoreset=True) + self.assertEqual(len(mockATW32.call_args_list), 2) + self.assertEqual(mockATW32.call_args_list[1][1]['autoreset'], True) + self.assertEqual(mockATW32.call_args_list[0][1]['autoreset'], True) + + @patch('colorama.initialise.AnsiToWin32') + def testAutoResetChangeable(self, mockATW32): + with osname("nt"): + init() + + init(autoreset=True) + self.assertEqual(len(mockATW32.call_args_list), 4) + self.assertEqual(mockATW32.call_args_list[2][1]['autoreset'], True) + self.assertEqual(mockATW32.call_args_list[3][1]['autoreset'], True) + + init() + self.assertEqual(len(mockATW32.call_args_list), 6) + self.assertEqual( + mockATW32.call_args_list[4][1]['autoreset'], False) + self.assertEqual( + mockATW32.call_args_list[5][1]['autoreset'], False) + + + @patch('colorama.initialise.atexit.register') + def testAtexitRegisteredOnlyOnce(self, mockRegister): + init() + self.assertTrue(mockRegister.called) + mockRegister.reset_mock() + init() + self.assertFalse(mockRegister.called) + + +class JustFixWindowsConsoleTest(TestCase): + def _reset(self): + _wipe_internal_state_for_tests() + sys.stdout = orig_stdout + sys.stderr = orig_stderr + + def tearDown(self): + self._reset() + + @patch("colorama.ansitowin32.winapi_test", lambda: True) + def testJustFixWindowsConsole(self): + if sys.platform != "win32": + # just_fix_windows_console should be a no-op + just_fix_windows_console() + self.assertIs(sys.stdout, orig_stdout) + self.assertIs(sys.stderr, orig_stderr) + else: + def fake_std(): + # Emulate stdout=not a tty, stderr=tty + # to check that we handle both cases correctly + stdout = Mock() + stdout.closed = False + stdout.isatty.return_value = False + stdout.fileno.return_value = 1 + sys.stdout = stdout + + stderr = Mock() + stderr.closed = False + stderr.isatty.return_value = True + stderr.fileno.return_value = 2 + sys.stderr = stderr + + for native_ansi in [False, True]: + with patch( + 'colorama.ansitowin32.enable_vt_processing', + lambda *_: native_ansi + ): + self._reset() + fake_std() + + # Regular single-call test + prev_stdout = sys.stdout + prev_stderr = sys.stderr + just_fix_windows_console() + self.assertIs(sys.stdout, prev_stdout) + if native_ansi: + self.assertIs(sys.stderr, prev_stderr) + else: + self.assertIsNot(sys.stderr, prev_stderr) + + # second call without resetting is always a no-op + prev_stdout = sys.stdout + prev_stderr = sys.stderr + just_fix_windows_console() + self.assertIs(sys.stdout, prev_stdout) + self.assertIs(sys.stderr, prev_stderr) + + self._reset() + fake_std() + + # If init() runs first, just_fix_windows_console should be a no-op + init() + prev_stdout = sys.stdout + prev_stderr = sys.stderr + just_fix_windows_console() + self.assertIs(prev_stdout, sys.stdout) + self.assertIs(prev_stderr, sys.stderr) + + +if __name__ == '__main__': + main() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/isatty_test.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/isatty_test.py new file mode 100644 index 0000000000000000000000000000000000000000..0f84e4befe550d4386d24264648abf1323e682ff --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/isatty_test.py @@ -0,0 +1,57 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +import sys +from unittest import TestCase, main + +from ..ansitowin32 import StreamWrapper, AnsiToWin32 +from .utils import pycharm, replace_by, replace_original_by, StreamTTY, StreamNonTTY + + +def is_a_tty(stream): + return StreamWrapper(stream, None).isatty() + +class IsattyTest(TestCase): + + def test_TTY(self): + tty = StreamTTY() + self.assertTrue(is_a_tty(tty)) + with pycharm(): + self.assertTrue(is_a_tty(tty)) + + def test_nonTTY(self): + non_tty = StreamNonTTY() + self.assertFalse(is_a_tty(non_tty)) + with pycharm(): + self.assertFalse(is_a_tty(non_tty)) + + def test_withPycharm(self): + with pycharm(): + self.assertTrue(is_a_tty(sys.stderr)) + self.assertTrue(is_a_tty(sys.stdout)) + + def test_withPycharmTTYOverride(self): + tty = StreamTTY() + with pycharm(), replace_by(tty): + self.assertTrue(is_a_tty(tty)) + + def test_withPycharmNonTTYOverride(self): + non_tty = StreamNonTTY() + with pycharm(), replace_by(non_tty): + self.assertFalse(is_a_tty(non_tty)) + + def test_withPycharmNoneOverride(self): + with pycharm(): + with replace_by(None), replace_original_by(None): + self.assertFalse(is_a_tty(None)) + self.assertFalse(is_a_tty(StreamNonTTY())) + self.assertTrue(is_a_tty(StreamTTY())) + + def test_withPycharmStreamWrapped(self): + with pycharm(): + self.assertTrue(AnsiToWin32(StreamTTY()).stream.isatty()) + self.assertFalse(AnsiToWin32(StreamNonTTY()).stream.isatty()) + self.assertTrue(AnsiToWin32(sys.stdout).stream.isatty()) + self.assertTrue(AnsiToWin32(sys.stderr).stream.isatty()) + + +if __name__ == '__main__': + main() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..472fafb4403efb9673d5cc724dafd9cf764aac5b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/utils.py @@ -0,0 +1,49 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +from contextlib import contextmanager +from io import StringIO +import sys +import os + + +class StreamTTY(StringIO): + def isatty(self): + return True + +class StreamNonTTY(StringIO): + def isatty(self): + return False + +@contextmanager +def osname(name): + orig = os.name + os.name = name + yield + os.name = orig + +@contextmanager +def replace_by(stream): + orig_stdout = sys.stdout + orig_stderr = sys.stderr + sys.stdout = stream + sys.stderr = stream + yield + sys.stdout = orig_stdout + sys.stderr = orig_stderr + +@contextmanager +def replace_original_by(stream): + orig_stdout = sys.__stdout__ + orig_stderr = sys.__stderr__ + sys.__stdout__ = stream + sys.__stderr__ = stream + yield + sys.__stdout__ = orig_stdout + sys.__stderr__ = orig_stderr + +@contextmanager +def pycharm(): + os.environ["PYCHARM_HOSTED"] = "1" + non_tty = StreamNonTTY() + with replace_by(non_tty), replace_original_by(non_tty): + yield + del os.environ["PYCHARM_HOSTED"] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/winterm_test.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/winterm_test.py new file mode 100644 index 0000000000000000000000000000000000000000..d0955f9e608377940f0d548576964f2fcf3caf48 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/colorama/tests/winterm_test.py @@ -0,0 +1,131 @@ +# Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. +import sys +from unittest import TestCase, main, skipUnless + +try: + from unittest.mock import Mock, patch +except ImportError: + from mock import Mock, patch + +from ..winterm import WinColor, WinStyle, WinTerm + + +class WinTermTest(TestCase): + + @patch('colorama.winterm.win32') + def testInit(self, mockWin32): + mockAttr = Mock() + mockAttr.wAttributes = 7 + 6 * 16 + 8 + mockWin32.GetConsoleScreenBufferInfo.return_value = mockAttr + term = WinTerm() + self.assertEqual(term._fore, 7) + self.assertEqual(term._back, 6) + self.assertEqual(term._style, 8) + + @skipUnless(sys.platform.startswith("win"), "requires Windows") + def testGetAttrs(self): + term = WinTerm() + + term._fore = 0 + term._back = 0 + term._style = 0 + self.assertEqual(term.get_attrs(), 0) + + term._fore = WinColor.YELLOW + self.assertEqual(term.get_attrs(), WinColor.YELLOW) + + term._back = WinColor.MAGENTA + self.assertEqual( + term.get_attrs(), + WinColor.YELLOW + WinColor.MAGENTA * 16) + + term._style = WinStyle.BRIGHT + self.assertEqual( + term.get_attrs(), + WinColor.YELLOW + WinColor.MAGENTA * 16 + WinStyle.BRIGHT) + + @patch('colorama.winterm.win32') + def testResetAll(self, mockWin32): + mockAttr = Mock() + mockAttr.wAttributes = 1 + 2 * 16 + 8 + mockWin32.GetConsoleScreenBufferInfo.return_value = mockAttr + term = WinTerm() + + term.set_console = Mock() + term._fore = -1 + term._back = -1 + term._style = -1 + + term.reset_all() + + self.assertEqual(term._fore, 1) + self.assertEqual(term._back, 2) + self.assertEqual(term._style, 8) + self.assertEqual(term.set_console.called, True) + + @skipUnless(sys.platform.startswith("win"), "requires Windows") + def testFore(self): + term = WinTerm() + term.set_console = Mock() + term._fore = 0 + + term.fore(5) + + self.assertEqual(term._fore, 5) + self.assertEqual(term.set_console.called, True) + + @skipUnless(sys.platform.startswith("win"), "requires Windows") + def testBack(self): + term = WinTerm() + term.set_console = Mock() + term._back = 0 + + term.back(5) + + self.assertEqual(term._back, 5) + self.assertEqual(term.set_console.called, True) + + @skipUnless(sys.platform.startswith("win"), "requires Windows") + def testStyle(self): + term = WinTerm() + term.set_console = Mock() + term._style = 0 + + term.style(22) + + self.assertEqual(term._style, 22) + self.assertEqual(term.set_console.called, True) + + @patch('colorama.winterm.win32') + def testSetConsole(self, mockWin32): + mockAttr = Mock() + mockAttr.wAttributes = 0 + mockWin32.GetConsoleScreenBufferInfo.return_value = mockAttr + term = WinTerm() + term.windll = Mock() + + term.set_console() + + self.assertEqual( + mockWin32.SetConsoleTextAttribute.call_args, + ((mockWin32.STDOUT, term.get_attrs()), {}) + ) + + @patch('colorama.winterm.win32') + def testSetConsoleOnStderr(self, mockWin32): + mockAttr = Mock() + mockAttr.wAttributes = 0 + mockWin32.GetConsoleScreenBufferInfo.return_value = mockAttr + term = WinTerm() + term.windll = Mock() + + term.set_console(on_stderr=True) + + self.assertEqual( + mockWin32.SetConsoleTextAttribute.call_args, + ((mockWin32.STDERR, term.get_attrs()), {}) + ) + + +if __name__ == '__main__': + main() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f24f8a15d86f0053a6264116576b522664f7ea01 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/__pycache__/_version.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/__pycache__/_version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..786ebb30bad629b85d308f00761a623ba89f3274 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/__pycache__/_version.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..28f6bf49484eace130ce617af86427945f6ea854 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cydriver.cpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cydriver.cpp new file mode 100644 index 0000000000000000000000000000000000000000..81303108cb9c58a2b3785f30eda9ce7841f6bfe8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cydriver.cpp @@ -0,0 +1,103006 @@ +/* Generated by Cython 3.1.5 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [ + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/cuda.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/cudaProfiler.h", + "cuda/bindings/_bindings/loader.h" + ], + "extra_compile_args": [ + "-std=c++14", + "-fpermissive", + "-Wno-deprecated-declarations", + "-fno-var-tracking-assignments", + "-O3" + ], + "include_dirs": [ + "./cuda/bindings/_bindings", + "/opt/python/cp312-cp312/include", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include" + ], + "language": "c++", + "library_dirs": [ + "/tmp/build-env-3t3_8d9r/lib/python3.12/site-packages", + "/tmp/build-env-3t3_8d9r/lib", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib64", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib" + ], + "name": "cuda.bindings._bindings.cydriver", + "sources": [ + "cuda/bindings/_bindings/cydriver.pyx", + "cuda/bindings/_bindings/loader.cpp" + ] + }, + "module_name": "cuda.bindings._bindings.cydriver" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_1_5" +#define CYTHON_HEX_VERSION 0x030105F0 +#define CYTHON_FUTURE_DIVISION 1 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef HAVE_LONG_LONG + #define HAVE_LONG_LONG +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_CLINE_IN_TRACEBACK + #define CYTHON_CLINE_IN_TRACEBACK 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1) +#endif +#endif +#define CYTHON_BACKPORT_VECTORCALL (CYTHON_METH_FASTCALL && PY_VERSION_HEX < 0x030800B1) +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME + #define CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME 100 +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + #endif + #endif + #if _MSC_VER < 1300 + #ifdef _WIN64 + typedef unsigned long long __pyx_uintptr_t; + #else + typedef unsigned int __pyx_uintptr_t; + #endif + #else + #ifdef _WIN64 + typedef unsigned __int64 __pyx_uintptr_t; + #else + typedef unsigned __int32 __pyx_uintptr_t; + #endif + #endif +#else + #include + typedef uintptr_t __pyx_uintptr_t; +#endif +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#elif CYTHON_BACKPORT_VECTORCALL + typedef PyObject *(*__pyx_vectorcallfunc)(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames); + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1)) + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(((size_t)(n)) & ~__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if __PYX_LIMITED_VERSION_HEX < 0x03090000 + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS && PY_VERSION_HEX >= 0x03080000 +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static CYTHON_INLINE PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__cuda__bindings___bindings__cydriver +#define __PYX_HAVE_API__cuda__bindings___bindings__cydriver +/* Early includes */ +#include +#include "cuda.h" +#include "cudaProfiler.h" +#include +#include "loader.h" +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value); + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "cuda/bindings/_bindings/cydriver.pyx", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* ForceInitThreads.proto */ +#ifndef __PYX_FORCE_INIT_THREADS + #define __PYX_FORCE_INIT_THREADS 0 +#endif + +/* NoFastGil.proto */ +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() + +/* Atomics.proto */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif +#if CYTHON_ATOMICS + #define __pyx_add_acquisition_count(memview)\ + __pyx_atomic_incr_relaxed(__pyx_get_slice_count_pointer(memview)) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_atomic_decr_acq_rel(__pyx_get_slice_count_pointer(memview)) +#else + #define __pyx_add_acquisition_count(memview)\ + __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_sub_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) +#endif + +/* IncludeStructmemberH.proto */ +#include + +/* CriticalSections.proto */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_Begin1(cs, arg) (void)cs +#define __Pyx_PyCriticalSection_Begin2(cs, arg1, arg2) (void)cs +#define __Pyx_PyCriticalSection_End1(cs) +#define __Pyx_PyCriticalSection_End2(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_Begin1 PyCriticalSection_Begin +#define __Pyx_PyCriticalSection_Begin2 PyCriticalSection2_Begin +#define __Pyx_PyCriticalSection_End1 PyCriticalSection_End +#define __Pyx_PyCriticalSection_End2 PyCriticalSection2_End +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* #### Code section: numeric_typedefs ### */ + +/* "cuda/bindings/cydriver.pxd":2602 + * ctypedef CUGLmap_flags_enum CUGLmap_flags + * + * ctypedef unsigned int GLenum # <<<<<<<<<<<<<< + * + * ctypedef unsigned int GLuint +*/ +typedef unsigned int __pyx_t_4cuda_8bindings_8cydriver_GLenum; + +/* "cuda/bindings/cydriver.pxd":2604 + * ctypedef unsigned int GLenum + * + * ctypedef unsigned int GLuint # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef unsigned int __pyx_t_4cuda_8bindings_8cydriver_GLuint; + +/* "cuda/bindings/cydriver.pxd":2616 + * ctypedef void* EGLStreamKHR + * + * ctypedef unsigned int EGLint # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef unsigned int __pyx_t_4cuda_8bindings_8cydriver_EGLint; + +/* "cuda/bindings/cydriver.pxd":2623 + * ctypedef void* EGLSyncKHR + * + * ctypedef uint32_t VdpDevice # <<<<<<<<<<<<<< + * + * ctypedef unsigned long long VdpGetProcAddress +*/ +typedef uint32_t __pyx_t_4cuda_8bindings_8cydriver_VdpDevice; + +/* "cuda/bindings/cydriver.pxd":2625 + * ctypedef uint32_t VdpDevice + * + * ctypedef unsigned long long VdpGetProcAddress # <<<<<<<<<<<<<< + * + * ctypedef uint32_t VdpVideoSurface +*/ +typedef unsigned PY_LONG_LONG __pyx_t_4cuda_8bindings_8cydriver_VdpGetProcAddress; + +/* "cuda/bindings/cydriver.pxd":2627 + * ctypedef unsigned long long VdpGetProcAddress + * + * ctypedef uint32_t VdpVideoSurface # <<<<<<<<<<<<<< + * + * ctypedef uint32_t VdpOutputSurface +*/ +typedef uint32_t __pyx_t_4cuda_8bindings_8cydriver_VdpVideoSurface; + +/* "cuda/bindings/cydriver.pxd":2629 + * ctypedef uint32_t VdpVideoSurface + * + * ctypedef uint32_t VdpOutputSurface # <<<<<<<<<<<<<< + * + * cdef CUresult cuGetErrorString(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil +*/ +typedef uint32_t __pyx_t_4cuda_8bindings_8cydriver_VdpOutputSurface; +/* #### Code section: complex_type_declarations ### */ +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ +union __pyx_t_4cuda_8bindings_8cydriver_anon_union15; +struct __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_st; + +/* "cuda/bindings/cydriver.pxd":2429 + * ctypedef CUoutput_mode_enum CUoutput_mode + * + * cdef enum CUeglFrameType_enum: # <<<<<<<<<<<<<< + * CU_EGL_FRAME_TYPE_ARRAY = 0 + * CU_EGL_FRAME_TYPE_PITCH = 1 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUeglFrameType_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_FRAME_TYPE_ARRAY = 0, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_FRAME_TYPE_PITCH = 1 +}; + +/* "cuda/bindings/cydriver.pxd":2435 + * ctypedef CUeglFrameType_enum CUeglFrameType + * + * cdef enum CUeglResourceLocationFlags_enum: # <<<<<<<<<<<<<< + * CU_EGL_RESOURCE_LOCATION_SYSMEM = 0 + * CU_EGL_RESOURCE_LOCATION_VIDMEM = 1 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUeglResourceLocationFlags_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_RESOURCE_LOCATION_SYSMEM = 0, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_RESOURCE_LOCATION_VIDMEM = 1 +}; + +/* "cuda/bindings/cydriver.pxd":2441 + * ctypedef CUeglResourceLocationFlags_enum CUeglResourceLocationFlags + * + * cdef enum CUeglColorFormat_enum: # <<<<<<<<<<<<<< + * CU_EGL_COLOR_FORMAT_YUV420_PLANAR = 0 + * CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR = 1 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUeglColorFormat_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR = 0, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR = 1, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_PLANAR = 2, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR = 3, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_RGB = 4, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BGR = 5, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_ARGB = 6, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_RGBA = 7, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_L = 8, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_R = 9, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_PLANAR = 10, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR = 11, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUYV_422 = 12, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_422 = 13, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_ABGR = 14, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BGRA = 15, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_A = 16, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_RG = 17, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_AYUV = 18, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR = 19, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR = 20, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR = 21, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR = 22, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR = 23, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR = 24, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR = 25, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_VYUY_ER = 26, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_ER = 27, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUYV_ER = 28, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVYU_ER = 29, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV_ER = 30, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUVA_ER = 31, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_AYUV_ER = 32, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER = 33, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER = 34, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER = 35, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER = 36, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER = 37, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER = 38, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER = 39, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER = 40, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER = 41, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER = 42, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER = 43, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER = 44, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_RGGB = 45, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_BGGR = 46, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_GRBG = 47, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_GBRG = 48, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_RGGB = 49, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_BGGR = 50, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_GRBG = 51, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_GBRG = 52, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_RGGB = 53, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_BGGR = 54, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_GRBG = 55, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_GBRG = 56, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_RGGB = 57, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_BGGR = 58, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_GRBG = 59, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_GBRG = 60, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_RGGB = 61, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_BGGR = 62, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_GRBG = 63, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_GBRG = 64, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_PLANAR = 65, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_PLANAR = 66, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR = 67, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB = 68, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR = 69, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG = 70, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG = 71, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_BCCR = 72, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_RCCB = 73, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_CRBC = 74, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_CBRC = 75, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_CCCC = 76, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_BCCR = 77, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_RCCB = 78, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CRBC = 79, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CBRC = 80, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CCCC = 81, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y = 82, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 = 83, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 = 84, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 = 85, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 = 86, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 = 87, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 = 88, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 = 89, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709 = 90, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 = 91, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 = 92, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 = 93, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR = 94, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709 = 95, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y_ER = 96, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y_709_ER = 97, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10_ER = 98, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10_709_ER = 99, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12_ER = 0x64, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12_709_ER = 0x65, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUVA = 0x66, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV = 0x67, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVYU = 0x68, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_VYUY = 0x69, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER = 0x6A, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER = 0x6B, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER = 0x6C, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER = 0x6D, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER = 0x6E, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER = 0x6F, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER = 0x70, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER = 0x71, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_709 = 0x72, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_709_ER = 0x73, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_2020 = 0x74, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_MAX = 0x75 +}; + +/* "cuda/bindings/cydriver.pxd":2588 + * ctypedef CUeglStreamConnection_st* CUeglStreamConnection + * + * cdef enum CUGLDeviceList_enum: # <<<<<<<<<<<<<< + * CU_GL_DEVICE_LIST_ALL = 1 + * CU_GL_DEVICE_LIST_CURRENT_FRAME = 2 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_DEVICE_LIST_ALL = 1, + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_DEVICE_LIST_CURRENT_FRAME = 2, + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_DEVICE_LIST_NEXT_FRAME = 3 +}; + +/* "cuda/bindings/cydriver.pxd":2595 + * ctypedef CUGLDeviceList_enum CUGLDeviceList + * + * cdef enum CUGLmap_flags_enum: # <<<<<<<<<<<<<< + * CU_GL_MAP_RESOURCE_FLAGS_NONE = 0 + * CU_GL_MAP_RESOURCE_FLAGS_READ_ONLY = 1 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUGLmap_flags_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_MAP_RESOURCE_FLAGS_NONE = 0, + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_MAP_RESOURCE_FLAGS_READ_ONLY = 1, + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_MAP_RESOURCE_FLAGS_WRITE_DISCARD = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3577 + * cdef CUresult cuGraphicsVDPAURegisterOutputSurface(CUgraphicsResource* pCudaResource, VdpOutputSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + * + * cdef enum: CUDA_VERSION = 12090 # <<<<<<<<<<<<<< + * + * cdef enum: CU_IPC_HANDLE_SIZE = 64 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_VERSION = 0x2F3A +}; + +/* "cuda/bindings/cydriver.pxd":3579 + * cdef enum: CUDA_VERSION = 12090 + * + * cdef enum: CU_IPC_HANDLE_SIZE = 64 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_LEGACY = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_IPC_HANDLE_SIZE = 64 +}; + +/* "cuda/bindings/cydriver.pxd":3581 + * cdef enum: CU_IPC_HANDLE_SIZE = 64 + * + * cdef enum: CU_STREAM_LEGACY = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_PER_THREAD = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_LEGACY = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3583 + * cdef enum: CU_STREAM_LEGACY = 1 + * + * cdef enum: CU_STREAM_PER_THREAD = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_COMPUTE_ACCELERATED_TARGET_BASE = 65536 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_PER_THREAD = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3585 + * cdef enum: CU_STREAM_PER_THREAD = 2 + * + * cdef enum: CU_COMPUTE_ACCELERATED_TARGET_BASE = 65536 # <<<<<<<<<<<<<< + * + * cdef enum: CU_COMPUTE_FAMILY_TARGET_BASE = 131072 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_COMPUTE_ACCELERATED_TARGET_BASE = 0x10000 +}; + +/* "cuda/bindings/cydriver.pxd":3587 + * cdef enum: CU_COMPUTE_ACCELERATED_TARGET_BASE = 65536 + * + * cdef enum: CU_COMPUTE_FAMILY_TARGET_BASE = 131072 # <<<<<<<<<<<<<< + * + * cdef enum: CU_GRAPH_COND_ASSIGN_DEFAULT = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_COMPUTE_FAMILY_TARGET_BASE = 0x20000 +}; + +/* "cuda/bindings/cydriver.pxd":3589 + * cdef enum: CU_COMPUTE_FAMILY_TARGET_BASE = 131072 + * + * cdef enum: CU_GRAPH_COND_ASSIGN_DEFAULT = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_DEFAULT = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GRAPH_COND_ASSIGN_DEFAULT = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3591 + * cdef enum: CU_GRAPH_COND_ASSIGN_DEFAULT = 1 + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_DEFAULT = 0 # <<<<<<<<<<<<<< + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GRAPH_KERNEL_NODE_PORT_DEFAULT = 0 +}; + +/* "cuda/bindings/cydriver.pxd":3593 + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_DEFAULT = 0 + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3595 + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC = 1 + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3597 + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER = 2 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_COOPERATIVE = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3599 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_COOPERATIVE = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_DIMENSION = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_COOPERATIVE = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3601 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_COOPERATIVE = 2 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_DIMENSION = 4 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_DIMENSION = 4 +}; + +/* "cuda/bindings/cydriver.pxd":3603 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_DIMENSION = 4 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PRIORITY = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 +}; + +/* "cuda/bindings/cydriver.pxd":3605 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PRIORITY = 8 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_PRIORITY = 8 +}; + +/* "cuda/bindings/cydriver.pxd":3607 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PRIORITY = 8 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 +}; + +/* "cuda/bindings/cydriver.pxd":3609 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 +}; + +/* "cuda/bindings/cydriver.pxd":3611 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11 +}; + +/* "cuda/bindings/cydriver.pxd":3613 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 14 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13 +}; + +/* "cuda/bindings/cydriver.pxd":3615 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 14 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 14 +}; + +/* "cuda/bindings/cydriver.pxd":3617 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 14 + * + * cdef enum: CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3619 + * cdef enum: CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 + * + * cdef enum: CU_STREAM_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_PRIORITY = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 +}; + +/* "cuda/bindings/cydriver.pxd":3621 + * cdef enum: CU_STREAM_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 + * + * cdef enum: CU_STREAM_ATTRIBUTE_PRIORITY = 8 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_PRIORITY = 8 +}; + +/* "cuda/bindings/cydriver.pxd":3623 + * cdef enum: CU_STREAM_ATTRIBUTE_PRIORITY = 8 + * + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 +}; + +/* "cuda/bindings/cydriver.pxd":3625 + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 + * + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTALLOC_PORTABLE = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 +}; + +/* "cuda/bindings/cydriver.pxd":3627 + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 + * + * cdef enum: CU_MEMHOSTALLOC_PORTABLE = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTALLOC_DEVICEMAP = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTALLOC_PORTABLE = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3629 + * cdef enum: CU_MEMHOSTALLOC_PORTABLE = 1 + * + * cdef enum: CU_MEMHOSTALLOC_DEVICEMAP = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTALLOC_WRITECOMBINED = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTALLOC_DEVICEMAP = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3631 + * cdef enum: CU_MEMHOSTALLOC_DEVICEMAP = 2 + * + * cdef enum: CU_MEMHOSTALLOC_WRITECOMBINED = 4 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTREGISTER_PORTABLE = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTALLOC_WRITECOMBINED = 4 +}; + +/* "cuda/bindings/cydriver.pxd":3633 + * cdef enum: CU_MEMHOSTALLOC_WRITECOMBINED = 4 + * + * cdef enum: CU_MEMHOSTREGISTER_PORTABLE = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTREGISTER_DEVICEMAP = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTREGISTER_PORTABLE = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3635 + * cdef enum: CU_MEMHOSTREGISTER_PORTABLE = 1 + * + * cdef enum: CU_MEMHOSTREGISTER_DEVICEMAP = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTREGISTER_IOMEMORY = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTREGISTER_DEVICEMAP = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3637 + * cdef enum: CU_MEMHOSTREGISTER_DEVICEMAP = 2 + * + * cdef enum: CU_MEMHOSTREGISTER_IOMEMORY = 4 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTREGISTER_READ_ONLY = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTREGISTER_IOMEMORY = 4 +}; + +/* "cuda/bindings/cydriver.pxd":3639 + * cdef enum: CU_MEMHOSTREGISTER_IOMEMORY = 4 + * + * cdef enum: CU_MEMHOSTREGISTER_READ_ONLY = 8 # <<<<<<<<<<<<<< + * + * cdef enum: CU_ARRAY_SPARSE_PROPERTIES_SINGLE_MIPTAIL = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTREGISTER_READ_ONLY = 8 +}; + +/* "cuda/bindings/cydriver.pxd":3641 + * cdef enum: CU_MEMHOSTREGISTER_READ_ONLY = 8 + * + * cdef enum: CU_ARRAY_SPARSE_PROPERTIES_SINGLE_MIPTAIL = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TENSOR_MAP_NUM_QWORDS = 16 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_ARRAY_SPARSE_PROPERTIES_SINGLE_MIPTAIL = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3643 + * cdef enum: CU_ARRAY_SPARSE_PROPERTIES_SINGLE_MIPTAIL = 1 + * + * cdef enum: CU_TENSOR_MAP_NUM_QWORDS = 16 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_EXTERNAL_MEMORY_DEDICATED = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TENSOR_MAP_NUM_QWORDS = 16 +}; + +/* "cuda/bindings/cydriver.pxd":3645 + * cdef enum: CU_TENSOR_MAP_NUM_QWORDS = 16 + * + * cdef enum: CUDA_EXTERNAL_MEMORY_DEDICATED = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_SIGNAL_SKIP_NVSCIBUF_MEMSYNC = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_EXTERNAL_MEMORY_DEDICATED = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3647 + * cdef enum: CUDA_EXTERNAL_MEMORY_DEDICATED = 1 + * + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_SIGNAL_SKIP_NVSCIBUF_MEMSYNC = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_WAIT_SKIP_NVSCIBUF_MEMSYNC = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_EXTERNAL_SEMAPHORE_SIGNAL_SKIP_NVSCIBUF_MEMSYNC = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3649 + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_SIGNAL_SKIP_NVSCIBUF_MEMSYNC = 1 + * + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_WAIT_SKIP_NVSCIBUF_MEMSYNC = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_NVSCISYNC_ATTR_SIGNAL = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_EXTERNAL_SEMAPHORE_WAIT_SKIP_NVSCIBUF_MEMSYNC = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3651 + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_WAIT_SKIP_NVSCIBUF_MEMSYNC = 2 + * + * cdef enum: CUDA_NVSCISYNC_ATTR_SIGNAL = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_NVSCISYNC_ATTR_WAIT = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_NVSCISYNC_ATTR_SIGNAL = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3653 + * cdef enum: CUDA_NVSCISYNC_ATTR_SIGNAL = 1 + * + * cdef enum: CUDA_NVSCISYNC_ATTR_WAIT = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEM_CREATE_USAGE_TILE_POOL = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_NVSCISYNC_ATTR_WAIT = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3655 + * cdef enum: CUDA_NVSCISYNC_ATTR_WAIT = 2 + * + * cdef enum: CU_MEM_CREATE_USAGE_TILE_POOL = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEM_CREATE_USAGE_HW_DECOMPRESS = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEM_CREATE_USAGE_TILE_POOL = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3657 + * cdef enum: CU_MEM_CREATE_USAGE_TILE_POOL = 1 + * + * cdef enum: CU_MEM_CREATE_USAGE_HW_DECOMPRESS = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEM_CREATE_USAGE_HW_DECOMPRESS = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3659 + * cdef enum: CU_MEM_CREATE_USAGE_HW_DECOMPRESS = 2 + * + * cdef enum: CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3661 + * cdef enum: CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS = 2 + * + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3663 + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC = 1 + * + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_LAYERED = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3665 + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC = 2 + * + * cdef enum: CUDA_ARRAY3D_LAYERED = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_2DARRAY = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_LAYERED = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3667 + * cdef enum: CUDA_ARRAY3D_LAYERED = 1 + * + * cdef enum: CUDA_ARRAY3D_2DARRAY = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_SURFACE_LDST = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_2DARRAY = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3669 + * cdef enum: CUDA_ARRAY3D_2DARRAY = 1 + * + * cdef enum: CUDA_ARRAY3D_SURFACE_LDST = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_CUBEMAP = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_SURFACE_LDST = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3671 + * cdef enum: CUDA_ARRAY3D_SURFACE_LDST = 2 + * + * cdef enum: CUDA_ARRAY3D_CUBEMAP = 4 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_TEXTURE_GATHER = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_CUBEMAP = 4 +}; + +/* "cuda/bindings/cydriver.pxd":3673 + * cdef enum: CUDA_ARRAY3D_CUBEMAP = 4 + * + * cdef enum: CUDA_ARRAY3D_TEXTURE_GATHER = 8 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_DEPTH_TEXTURE = 16 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_TEXTURE_GATHER = 8 +}; + +/* "cuda/bindings/cydriver.pxd":3675 + * cdef enum: CUDA_ARRAY3D_TEXTURE_GATHER = 8 + * + * cdef enum: CUDA_ARRAY3D_DEPTH_TEXTURE = 16 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_COLOR_ATTACHMENT = 32 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_DEPTH_TEXTURE = 16 +}; + +/* "cuda/bindings/cydriver.pxd":3677 + * cdef enum: CUDA_ARRAY3D_DEPTH_TEXTURE = 16 + * + * cdef enum: CUDA_ARRAY3D_COLOR_ATTACHMENT = 32 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_SPARSE = 64 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_COLOR_ATTACHMENT = 32 +}; + +/* "cuda/bindings/cydriver.pxd":3679 + * cdef enum: CUDA_ARRAY3D_COLOR_ATTACHMENT = 32 + * + * cdef enum: CUDA_ARRAY3D_SPARSE = 64 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_DEFERRED_MAPPING = 128 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_SPARSE = 64 +}; + +/* "cuda/bindings/cydriver.pxd":3681 + * cdef enum: CUDA_ARRAY3D_SPARSE = 64 + * + * cdef enum: CUDA_ARRAY3D_DEFERRED_MAPPING = 128 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_VIDEO_ENCODE_DECODE = 256 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_DEFERRED_MAPPING = 0x80 +}; + +/* "cuda/bindings/cydriver.pxd":3683 + * cdef enum: CUDA_ARRAY3D_DEFERRED_MAPPING = 128 + * + * cdef enum: CUDA_ARRAY3D_VIDEO_ENCODE_DECODE = 256 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSA_OVERRIDE_FORMAT = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_VIDEO_ENCODE_DECODE = 0x100 +}; + +/* "cuda/bindings/cydriver.pxd":3685 + * cdef enum: CUDA_ARRAY3D_VIDEO_ENCODE_DECODE = 256 + * + * cdef enum: CU_TRSA_OVERRIDE_FORMAT = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_READ_AS_INTEGER = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSA_OVERRIDE_FORMAT = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3687 + * cdef enum: CU_TRSA_OVERRIDE_FORMAT = 1 + * + * cdef enum: CU_TRSF_READ_AS_INTEGER = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_NORMALIZED_COORDINATES = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_READ_AS_INTEGER = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3689 + * cdef enum: CU_TRSF_READ_AS_INTEGER = 1 + * + * cdef enum: CU_TRSF_NORMALIZED_COORDINATES = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_SRGB = 16 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_NORMALIZED_COORDINATES = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3691 + * cdef enum: CU_TRSF_NORMALIZED_COORDINATES = 2 + * + * cdef enum: CU_TRSF_SRGB = 16 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_DISABLE_TRILINEAR_OPTIMIZATION = 32 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_SRGB = 16 +}; + +/* "cuda/bindings/cydriver.pxd":3693 + * cdef enum: CU_TRSF_SRGB = 16 + * + * cdef enum: CU_TRSF_DISABLE_TRILINEAR_OPTIMIZATION = 32 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_SEAMLESS_CUBEMAP = 64 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_DISABLE_TRILINEAR_OPTIMIZATION = 32 +}; + +/* "cuda/bindings/cydriver.pxd":3695 + * cdef enum: CU_TRSF_DISABLE_TRILINEAR_OPTIMIZATION = 32 + * + * cdef enum: CU_TRSF_SEAMLESS_CUBEMAP = 64 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_KERNEL_REQUIRED_BLOCK_DIM = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_SEAMLESS_CUBEMAP = 64 +}; + +/* "cuda/bindings/cydriver.pxd":3697 + * cdef enum: CU_TRSF_SEAMLESS_CUBEMAP = 64 + * + * cdef enum: CU_LAUNCH_KERNEL_REQUIRED_BLOCK_DIM = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_END_AS_INT = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_KERNEL_REQUIRED_BLOCK_DIM = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3699 + * cdef enum: CU_LAUNCH_KERNEL_REQUIRED_BLOCK_DIM = 1 + * + * cdef enum: CU_LAUNCH_PARAM_END_AS_INT = 0 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_END = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_END_AS_INT = 0 +}; + +/* "cuda/bindings/cydriver.pxd":3701 + * cdef enum: CU_LAUNCH_PARAM_END_AS_INT = 0 + * + * cdef enum: CU_LAUNCH_PARAM_END = 0 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER_AS_INT = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_END = 0 +}; + +/* "cuda/bindings/cydriver.pxd":3703 + * cdef enum: CU_LAUNCH_PARAM_END = 0 + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER_AS_INT = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_BUFFER_POINTER_AS_INT = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3705 + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER_AS_INT = 1 + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE_AS_INT = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_BUFFER_POINTER = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3707 + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER = 1 + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE_AS_INT = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_BUFFER_SIZE_AS_INT = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3709 + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE_AS_INT = 2 + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_PARAM_TR_DEFAULT = -1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_BUFFER_SIZE = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3711 + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE = 2 + * + * cdef enum: CU_PARAM_TR_DEFAULT = -1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_DEVICE_CPU = -1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_PARAM_TR_DEFAULT = -1L +}; + +/* "cuda/bindings/cydriver.pxd":3713 + * cdef enum: CU_PARAM_TR_DEFAULT = -1 + * + * cdef enum: CU_DEVICE_CPU = -1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_DEVICE_INVALID = -2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_DEVICE_CPU = -1L +}; + +/* "cuda/bindings/cydriver.pxd":3715 + * cdef enum: CU_DEVICE_CPU = -1 + * + * cdef enum: CU_DEVICE_INVALID = -2 # <<<<<<<<<<<<<< + * + * cdef enum: RESOURCE_ABI_VERSION = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_DEVICE_INVALID = -2L +}; + +/* "cuda/bindings/cydriver.pxd":3717 + * cdef enum: CU_DEVICE_INVALID = -2 + * + * cdef enum: RESOURCE_ABI_VERSION = 1 # <<<<<<<<<<<<<< + * + * cdef enum: RESOURCE_ABI_EXTERNAL_BYTES = 48 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_RESOURCE_ABI_VERSION = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3719 + * cdef enum: RESOURCE_ABI_VERSION = 1 + * + * cdef enum: RESOURCE_ABI_EXTERNAL_BYTES = 48 # <<<<<<<<<<<<<< + * + * cdef enum: MAX_PLANES = 3 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_RESOURCE_ABI_EXTERNAL_BYTES = 48 +}; + +/* "cuda/bindings/cydriver.pxd":3721 + * cdef enum: RESOURCE_ABI_EXTERNAL_BYTES = 48 + * + * cdef enum: MAX_PLANES = 3 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_EGL_INFINITE_TIMEOUT = 4294967295 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_MAX_PLANES = 3 +}; + +/* "cuda/bindings/cydriver.pxd":3723 + * cdef enum: MAX_PLANES = 3 + * + * cdef enum: CUDA_EGL_INFINITE_TIMEOUT = 4294967295 # <<<<<<<<<<<<<< +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_EGL_INFINITE_TIMEOUT = 0xFFFFFFFF +}; + +/* "cuda/bindings/cydriver.pxd":2433 + * CU_EGL_FRAME_TYPE_PITCH = 1 + * + * ctypedef CUeglFrameType_enum CUeglFrameType # <<<<<<<<<<<<<< + * + * cdef enum CUeglResourceLocationFlags_enum: +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUeglFrameType_enum __pyx_t_4cuda_8bindings_8cydriver_CUeglFrameType; + +/* "cuda/bindings/cydriver.pxd":2439 + * CU_EGL_RESOURCE_LOCATION_VIDMEM = 1 + * + * ctypedef CUeglResourceLocationFlags_enum CUeglResourceLocationFlags # <<<<<<<<<<<<<< + * + * cdef enum CUeglColorFormat_enum: +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUeglResourceLocationFlags_enum __pyx_t_4cuda_8bindings_8cydriver_CUeglResourceLocationFlags; + +/* "cuda/bindings/cydriver.pxd":2561 + * CU_EGL_COLOR_FORMAT_MAX = 117 + * + * ctypedef CUeglColorFormat_enum CUeglColorFormat # <<<<<<<<<<<<<< + * + * cdef union anon_union15: +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUeglColorFormat_enum __pyx_t_4cuda_8bindings_8cydriver_CUeglColorFormat; + +/* "cuda/bindings/cydriver.pxd":2563 + * ctypedef CUeglColorFormat_enum CUeglColorFormat + * + * cdef union anon_union15: # <<<<<<<<<<<<<< + * CUarray pArray[3] + * void* pPitch[3] +*/ +union __pyx_t_4cuda_8bindings_8cydriver_anon_union15 { + CUarray pArray[3]; + void *pPitch[3]; +}; + +/* "cuda/bindings/cydriver.pxd":2567 + * void* pPitch[3] + * + * cdef struct CUeglFrame_st: # <<<<<<<<<<<<<< + * anon_union15 frame + * unsigned int width +*/ +struct __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_st { + union __pyx_t_4cuda_8bindings_8cydriver_anon_union15 frame; + unsigned int width; + unsigned int height; + unsigned int depth; + unsigned int pitch; + unsigned int planeCount; + unsigned int numChannels; + __pyx_t_4cuda_8bindings_8cydriver_CUeglFrameType frameType; + __pyx_t_4cuda_8bindings_8cydriver_CUeglColorFormat eglColorFormat; + CUarray_format cuFormat; +}; + +/* "cuda/bindings/cydriver.pxd":2579 + * CUarray_format cuFormat + * + * ctypedef CUeglFrame_st CUeglFrame_v1 # <<<<<<<<<<<<<< + * + * ctypedef CUeglFrame_v1 CUeglFrame +*/ +typedef struct __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_st __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_v1; + +/* "cuda/bindings/cydriver.pxd":2581 + * ctypedef CUeglFrame_st CUeglFrame_v1 + * + * ctypedef CUeglFrame_v1 CUeglFrame # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_v1 __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame; + +/* "cuda/bindings/cydriver.pxd":2586 + * cdef struct CUeglStreamConnection_st: + * pass + * ctypedef CUeglStreamConnection_st* CUeglStreamConnection # <<<<<<<<<<<<<< + * + * cdef enum CUGLDeviceList_enum: +*/ +typedef struct CUeglStreamConnection_st *__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection; + +/* "cuda/bindings/cydriver.pxd":2593 + * CU_GL_DEVICE_LIST_NEXT_FRAME = 3 + * + * ctypedef CUGLDeviceList_enum CUGLDeviceList # <<<<<<<<<<<<<< + * + * cdef enum CUGLmap_flags_enum: +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList_enum __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList; + +/* "cuda/bindings/cydriver.pxd":2600 + * CU_GL_MAP_RESOURCE_FLAGS_WRITE_DISCARD = 2 + * + * ctypedef CUGLmap_flags_enum CUGLmap_flags # <<<<<<<<<<<<<< + * + * ctypedef unsigned int GLenum +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUGLmap_flags_enum __pyx_t_4cuda_8bindings_8cydriver_CUGLmap_flags; + +/* "cuda/bindings/cydriver.pxd":2609 + * cdef struct void: + * pass + * ctypedef void* EGLImageKHR # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef void *__pyx_t_4cuda_8bindings_8cydriver_EGLImageKHR; + +/* "cuda/bindings/cydriver.pxd":2614 + * cdef struct void: + * pass + * ctypedef void* EGLStreamKHR # <<<<<<<<<<<<<< + * + * ctypedef unsigned int EGLint +*/ +typedef void *__pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR; + +/* "cuda/bindings/cydriver.pxd":2621 + * cdef struct void: + * pass + * ctypedef void* EGLSyncKHR # <<<<<<<<<<<<<< + * + * ctypedef uint32_t VdpDevice +*/ +typedef void *__pyx_t_4cuda_8bindings_8cydriver_EGLSyncKHR; + +/* "cuda/bindings/_bindings/cydriver.pyx":489 + * + * # To make cuPythonInit reentrant + * ctypedef CUresult (*__cuGetProcAddress_v2_T)(const char*, void**, int, cuuint64_t, CUdriverProcAddressQueryResult*) except?CUDA_ERROR_NOT_FOUND nogil # <<<<<<<<<<<<<< + * cdef __cuGetProcAddress_v2_T _F_cuGetProcAddress_v2 = NULL + * +*/ +typedef CUresult (*__pyx_t_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2_T)(char const *, void **, int, cuuint64_t, CUdriverProcAddressQueryResult *); +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyErrExceptionMatches.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* PyObjectGetAttrStrNoError.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* PyObjectLookupSpecial.proto */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 0) +#define __Pyx_PyObject_LookupSpecial(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 1) +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error); +#else +#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n) +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#if !CYTHON_VECTORCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs); +#endif +#define __Pyx_BUILD_ASSERT_EXPR(cond)\ + (sizeof(char [1 - 2*!(cond)]) - 1) +#ifndef Py_MEMBER_SIZE +#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#endif +#if !CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03080000 + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets() + #define __Pyx_PyFrame_GetLocalsplus(frame) ((frame)->f_localsplus) +#else + static size_t __pyx_pyframe_localsplus_offset = 0; + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets()\ + ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ + (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) + #define __Pyx_PyFrame_GetLocalsplus(frame)\ + (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) +#endif +#endif +#endif + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* PyDictVersioning.proto */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __pyx_dict_cached_value;\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) do {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +#define __Pyx_GetModuleGlobalNameUncached(var, name) do {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + +/* PyObjectVectorCallKwBuilder.proto */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* IncludeStringH.proto */ +#include + +/* decode_c_string_utf16.proto */ +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, const char *errors) { + int byteorder = 0; + return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); +} +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16LE(const char *s, Py_ssize_t size, const char *errors) { + int byteorder = -1; + return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); +} +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16BE(const char *s, Py_ssize_t size, const char *errors) { + int byteorder = 1; + return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); +} + +/* decode_c_string.proto */ +static CYTHON_INLINE PyObject* __Pyx_decode_c_string( + const char* cstring, Py_ssize_t start, Py_ssize_t stop, + const char* encoding, const char* errors, + PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)); + +/* RaiseException.proto */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* GetTopmostException.proto */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* ErrOccurredWithGIL.proto */ +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void); + +/* TupleAndListFromArray.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +#endif +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/* BytesEquals.proto */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); + +/* UnicodeEquals.proto */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); + +/* fastcall.proto */ +#if CYTHON_AVOID_BORROWED_REFS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_PySequence_ITEM(args, i) +#elif CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_NewRef(__Pyx_PyTuple_GET_ITEM(args, i)) +#else + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_XNewRef(PyTuple_GetItem(args, i)) +#endif +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_ArgRef_FASTCALL(args, i) __Pyx_NewRef(args[i]) + #define __Pyx_NumKwargs_FASTCALL(kwds) __Pyx_PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs)) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues); + #else + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) + #endif +#else + #define __Pyx_ArgRef_FASTCALL __Pyx_ArgRef_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#if CYTHON_METH_FASTCALL || (CYTHON_COMPILING_IN_CPYTHON && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(args + start, stop - start) +#else +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + +/* DictGetItem.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key); +#define __Pyx_PyObject_Dict_GetItem(obj, name)\ + (likely(PyDict_CheckExact(obj)) ?\ + __Pyx_PyDict_GetItem(obj, name) : PyObject_GetItem(obj, name)) +#else +#define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key) +#define __Pyx_PyObject_Dict_GetItem(obj, name) PyObject_GetItem(obj, name) +#endif + +/* RaiseDoubleKeywords.proto */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywords.proto */ +static CYTHON_INLINE int __Pyx_ParseKeywords( + PyObject *kwds, PyObject *const *kwvalues, PyObject ** const argnames[], + PyObject *kwds2, PyObject *values[], + Py_ssize_t num_pos_args, Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* CallCFunction.proto */ +#define __Pyx_CallCFunction(cfunc, self, args)\ + ((PyCFunction)(void(*)(void))(cfunc)->func)(self, args) +#define __Pyx_CallCFunctionWithKeywords(cfunc, self, args, kwargs)\ + ((PyCFunctionWithKeywords)(void(*)(void))(cfunc)->func)(self, args, kwargs) +#define __Pyx_CallCFunctionFast(cfunc, self, args, nargs)\ + ((__Pyx_PyCFunctionFast)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs) +#define __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, nargs, kwnames)\ + ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs, kwnames) + +/* UnpackUnboundCMethod.proto */ +typedef struct { + PyObject *type; + PyObject **method_name; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS + __pyx_atomic_int_type initialized; +#endif + PyCFunction func; + PyObject *method; + int flag; +} __Pyx_CachedCFunction; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +static CYTHON_INLINE int __Pyx_CachedCFunction_GetAndSetInitializing(__Pyx_CachedCFunction *cfunc) { +#if !CYTHON_ATOMICS + return 1; +#else + __pyx_nonatomic_int_type expected = 0; + if (__pyx_atomic_int_cmp_exchange(&cfunc->initialized, &expected, 1)) { + return 0; + } + return expected; +#endif +} +static CYTHON_INLINE void __Pyx_CachedCFunction_SetFinishedInitializing(__Pyx_CachedCFunction *cfunc) { +#if CYTHON_ATOMICS + __pyx_atomic_store(&cfunc->initialized, 2); +#endif +} +#else +#define __Pyx_CachedCFunction_GetAndSetInitializing(cfunc) 2 +#define __Pyx_CachedCFunction_SetFinishedInitializing(cfunc) +#endif + +/* CallUnboundCMethod2.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2); +#else +#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2) +#endif + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* ArgTypeTest.proto */ +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* Import.proto */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); + +/* ImportDottedModule.proto */ +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple); +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple); + +/* LimitedApiGetTypeDict.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp); +#endif + +/* SetItemOnTypeDict.proto */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v); +#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v) + +/* FixUpExtensionType.proto */ +static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); + +/* FetchSharedCythonModule.proto */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void); + +/* dict_setdefault.proto */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, int is_safe_type); + +/* FetchCommonType.proto */ +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases); + +/* CommonTypesMetaclass.proto */ +static int __pyx_CommonTypesMetaclass_init(PyObject *module); +#define __Pyx_CommonTypesMetaclass_USED + +/* CallTypeTraverse.proto */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0 +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg); +#endif + +/* PyMethodNew.proto */ +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ); + +/* PyVectorcallFastCallDict.proto */ +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); +#endif + +/* CythonFunctionShared.proto */ +#define __Pyx_CyFunction_USED +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f)\ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj)\ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject_HEAD + PyObject *func; +#elif PY_VERSION_HEX < 0x030900B1 + PyCFunctionObject func; +#else + PyCMethodObject func; +#endif +#if CYTHON_BACKPORT_VECTORCALL ||\ + (CYTHON_COMPILING_IN_LIMITED_API && CYTHON_METH_FASTCALL) + __pyx_vectorcallfunc func_vectorcall; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_weakreflist; +#endif + PyObject *func_dict; + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_classobj; +#endif + PyObject *defaults; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; + PyObject *func_is_coroutine; +} __pyx_CyFunctionObject; +#undef __Pyx_CyOrPyCFunction_Check +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_mstate_global->__pyx_CyFunctionType) +#define __Pyx_CyOrPyCFunction_Check(obj) __Pyx_TypeCheck2(obj, __pyx_mstate_global->__pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_CyFunctionType) +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)); +#undef __Pyx_IsSameCFunction +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCyOrCFunction(func, cfunc) +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, + PyTypeObject *defaults_type); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(PyObject *module); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_BACKPORT_VECTORCALL || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif + +/* CythonFunction.proto */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); + +/* CLineInTraceback.proto */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* GCCDiagnostics.proto */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +/* GetRuntimeVersion.proto */ +static unsigned long __Pyx_get_runtime_version(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* FunctionExport.proto */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + + +/* Module declarations from "libc.stdint" */ + +/* Module declarations from "cuda.bindings.cydriver" */ + +/* Module declarations from "cuda.bindings._lib.dlfcn" */ + +/* Module declarations from "cuda.bindings._bindings.loader" */ + +/* Module declarations from "cuda.bindings._bindings.cydriver" */ +static PyObject *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___symbol_lock = 0; +static int __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPythonInit; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorString; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorName; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuInit; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDriverGetVersion; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGet; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetCount; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetName; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetLuid; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceTotalMem_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetTexture1DLinearMaxWidth; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetNvSciSyncAttributes; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetMemPool; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetMemPool; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDefaultMemPool; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetExecAffinitySupport; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFlushGPUDirectRDMAWrites; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetProperties; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceComputeCapability; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRetain; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRelease_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxSetFlags_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxGetState; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxReset_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v3; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v4; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDestroy_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPushCurrent_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPopCurrent_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCurrent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCurrent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevice; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetFlags; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetFlags; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetId; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSynchronize; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetLimit; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetLimit; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCacheConfig; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCacheConfig; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetApiVersion; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetStreamPriorityRange; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxResetPersistingL2Cache; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetExecAffinity; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxRecordEvent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxWaitEvent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxAttach; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDetach; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetSharedMemConfig; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetSharedMemConfig; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoad; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadData; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadDataEx; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadFatBinary; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleUnload; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetLoadingMode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunction; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunctionCount; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleEnumerateFunctions; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetGlobal_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkCreate_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddData_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddFile_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkComplete; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkDestroy; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetTexRef; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetSurfRef; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadData; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadFromFile; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryUnload; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernel; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernelCount; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryEnumerateKernels; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetModule; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetFunction; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetLibrary; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetGlobal; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetManaged; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetUnifiedFunction; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetCacheConfig; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetName; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetParamInfo; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetInfo_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAlloc_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocPitch_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFree_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAddressRange_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocHost_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeHost; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostAlloc; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetDevicePointer_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetFlags; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocManaged; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceRegisterAsyncNotification; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceUnregisterAsyncNotification; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetByPCIBusId; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetPCIBusId; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetEventHandle; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenEventHandle; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetMemHandle; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenMemHandle_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcCloseMemHandle; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostRegister_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostUnregister; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeer; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoD_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoH_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoD_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoA_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoD_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoA_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoH_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoA_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2D_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DUnaligned_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3D_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeer; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeerAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoDAsync_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoHAsync_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoDAsync_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoAAsync_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoHAsync_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DAsync_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DAsync_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeerAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyBatchAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DBatchAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8Async; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16Async; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32Async; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8Async; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16Async; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32Async; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayCreate_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetDescriptor_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetSparseProperties; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetSparseProperties; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetMemoryRequirements; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetMemoryRequirements; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetPlane; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayDestroy; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DCreate_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DGetDescriptor_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetLevel; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayDestroy; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetHandleForAddressRange; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemBatchDecompressAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressReserve; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressFree; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRelease; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMap; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMapArrayAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemUnmap; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemSetAccess; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAccess; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemExportToShareableHandle; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemImportFromShareableHandle; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationGranularity; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationPropertiesFromHandle; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRetainAllocationHandle; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolTrimTo; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAccess; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAccess; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolDestroy; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocFromPoolAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportToShareableHandle; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportFromShareableHandle; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportPointer; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportPointer; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastAddDevice; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindMem; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindAddr; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastUnbind; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastGetGranularity; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttributes; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerSetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttributes; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreateWithPriority; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetPriority; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetDevice; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetFlags; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetId; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitEvent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAddCallback; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCapture_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCaptureToGraph; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuThreadExchangeStreamCaptureMode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamEndCapture; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamIsCapturing; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v3; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAttachMemAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamQuery; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSynchronize; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamDestroy_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCopyAttributes; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecord; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecordWithFlags; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventQuery; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventSynchronize; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventDestroy_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalMemory; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedBuffer; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedMipmappedArray; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalMemory; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalSemaphore; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSignalExternalSemaphoresAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuWaitExternalSemaphoresAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalSemaphore; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue32_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue64_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue32_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue64_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBatchMemOp_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetCacheConfig; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetModule; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetName; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetParamInfo; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncIsLoaded; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncLoad; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernel; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernelEx; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernel; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernelMultiDevice; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchHostFunc; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetBlockShape; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedSize; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetSize; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSeti; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetf; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetv; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunch; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGrid; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGridAsync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetTexRef; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedMemConfig; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddKernelNode_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetParams_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetParams_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemcpyNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeGetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemsetNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeGetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddHostNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeGetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddChildGraphNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphChildGraphNodeGetGraph; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEmptyNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventRecordNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeGetEvent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeSetEvent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventWaitNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeGetEvent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeSetEvent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresSignalNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeGetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresWaitNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeGetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddBatchMemOpNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeGetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecBatchMemOpNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemAllocNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemAllocNodeGetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemFreeNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemFreeNodeGetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGraphMemTrim; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetGraphMemAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetGraphMemAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphClone; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeFindInClone; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetType; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetNodes; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetRootNodes; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroyNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithFlags; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecGetFlags; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecKernelNodeSetParams_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemcpyNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemsetNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecHostNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecChildGraphNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventRecordNodeSetEvent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventWaitNodeSetEvent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresSignalNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresWaitNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetEnabled; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetEnabled; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphUpload; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphLaunch; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecDestroy; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroy; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecUpdate_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeCopyAttributes; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDebugDotPrint; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRetain; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRelease; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRetainUserObject; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphReleaseUserObject; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecNodeSetParams; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphConditionalHandleCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessor; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSize; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSizeWithFlags; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyAvailableDynamicSMemPerBlock; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialClusterSize; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveClusters; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetArray; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmappedArray; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress2D_v3; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFormat; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddressMode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFilterMode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapFilterMode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelBias; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelClamp; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMaxAnisotropy; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetBorderColor; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFlags; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddress_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetArray; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmappedArray; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddressMode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFilterMode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFormat; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapFilterMode; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelBias; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelClamp; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMaxAnisotropy; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetBorderColor; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFlags; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefDestroy; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefSetArray; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefGetArray; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectDestroy; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceDesc; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetTextureDesc; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceViewDesc; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectDestroy; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectGetResourceDesc; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeTiled; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2col; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2colWide; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapReplaceAddress; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceCanAccessPeer; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxEnablePeerAccess; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDisablePeerAccess; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetP2PAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnregisterResource; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsSubResourceGetMappedArray; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedMipmappedArray; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedPointer_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceSetMapFlags_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsMapResources; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnmapResources; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttributeGlobal; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttribute; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttributeGlobal; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetExportTable; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxDestroy; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxFromGreenCtx; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDevResource; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevResource; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxGetDevResource; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevSmResourceSplitByCount; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevResourceGenerateDesc; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxRecordEvent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxWaitEvent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetGreenCtx; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxStreamCreate; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsRegisterCallback; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsUnregisterCallback; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsCurrent; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToFile; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToMemory; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetRestoreThreadId; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetState; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessLock; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessCheckpoint; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessRestore; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessUnlock; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStart; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStop; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsEGLRegisterImage; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnect; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnectWithFlags; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerDisconnect; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerAcquireFrame; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerReleaseFrame; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerConnect; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerDisconnect; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerPresentFrame; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerReturnFrame; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedEglFrame; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreateFromEGLSync; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterBuffer; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterImage; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGLGetDevices_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUGetDevice; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUCtxCreate_v2; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterVideoSurface; +static void *__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterOutputSurface; +static __pyx_t_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2_T __pyx_v_4cuda_8bindings_9_bindings_8cydriver__F_cuGetProcAddress_v2; +static PyObject *__pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs = 0; +static int __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuPythonInit(void); /*proto*/ +static CYTHON_INLINE int __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(void); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_bindings_8cydriver__inspect_function_pointers(int __pyx_skip_dispatch); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_bindings_8cydriver__inspect_function_pointer(PyObject *, int __pyx_skip_dispatch); /*proto*/ +/* #### Code section: typeinfo ### */ +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "cuda.bindings._bindings.cydriver" +extern int __pyx_module_is_main_cuda__bindings___bindings__cydriver; +int __pyx_module_is_main_cuda__bindings___bindings__cydriver = 0; + +/* Implementation of "cuda.bindings._bindings.cydriver" */ +/* #### Code section: global_var ### */ +static PyObject *__pyx_builtin_RuntimeError; +/* #### Code section: string_decls ### */ +static const char __pyx_k_[] = "?"; +static const char __pyx_k_os[] = "os"; +static const char __pyx_k_pop[] = "pop"; +static const char __pyx_k_sys[] = "sys"; +static const char __pyx_k_Lock[] = "Lock"; +static const char __pyx_k_exit[] = "__exit__"; +static const char __pyx_k_func[] = "__func__"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_name[] = "name"; +static const char __pyx_k_spec[] = "__spec__"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_enter[] = "__enter__"; +static const char __pyx_k_utf_8[] = "utf-8"; +static const char __pyx_k_cuInit[] = "__cuInit"; +static const char __pyx_k_getenv[] = "getenv"; +static const char __pyx_k_module[] = "__module__"; +static const char __pyx_k_name_2[] = "__name__"; +static const char __pyx_k_default[] = "default"; +static const char __pyx_k_maxsize[] = "maxsize"; +static const char __pyx_k_add_note[] = "add_note"; +static const char __pyx_k_cuLaunch[] = "__cuLaunch"; +static const char __pyx_k_cuMemMap[] = "__cuMemMap"; +static const char __pyx_k_cuMemcpy[] = "__cuMemcpy"; +static const char __pyx_k_encoding[] = "encoding"; +static const char __pyx_k_qualname[] = "__qualname__"; +static const char __pyx_k_set_name[] = "__set_name__"; +static const char __pyx_k_threading[] = "threading"; +static const char __pyx_k_z_A_a_9AQ[] = "\200\001\330\004\005\330\004\007\200z\220\023\220A\330\010\024\320\024.\250a\330\004\013\2109\220A\220Q"; +static const char __pyx_k_cuCtxGetId[] = "__cuCtxGetId"; +static const char __pyx_k_cuFuncLoad[] = "__cuFuncLoad"; +static const char __pyx_k_cuMemUnmap[] = "__cuMemUnmap"; +static const char __pyx_k_cuCtxAttach[] = "__cuCtxAttach"; +static const char __pyx_k_cuCtxDetach[] = "__cuCtxDetach"; +static const char __pyx_k_cuDeviceGet[] = "__cuDeviceGet"; +static const char __pyx_k_cuMemAdvise[] = "__cuMemAdvise"; +static const char __pyx_k_cuMemCreate[] = "__cuMemCreate"; +static const char __pyx_k_cuParamSetf[] = "__cuParamSetf"; +static const char __pyx_k_cuParamSeti[] = "__cuParamSeti"; +static const char __pyx_k_cuParamSetv[] = "__cuParamSetv"; +static const char __pyx_k_RuntimeError[] = "RuntimeError"; +static const char __pyx_k_cuEventQuery[] = "__cuEventQuery"; +static const char __pyx_k_cuGraphClone[] = "__cuGraphClone"; +static const char __pyx_k_cuLaunchGrid[] = "__cuLaunchGrid"; +static const char __pyx_k_cuMemFree_v2[] = "__cuMemFree_v2"; +static const char __pyx_k_cuMemRelease[] = "__cuMemRelease"; +static const char __pyx_k_cuMemcpyPeer[] = "__cuMemcpyPeer"; +static const char __pyx_k_cuModuleLoad[] = "__cuModuleLoad"; +static const char __pyx_k_initializing[] = "_initializing"; +static const char __pyx_k_is_coroutine[] = "_is_coroutine"; +static const char __pyx_k_libcuda_so_1[] = "libcuda.so.1"; +static const char __pyx_k_cuCtxGetFlags[] = "__cuCtxGetFlags"; +static const char __pyx_k_cuCtxGetLimit[] = "__cuCtxGetLimit"; +static const char __pyx_k_cuCtxSetFlags[] = "__cuCtxSetFlags"; +static const char __pyx_k_cuCtxSetLimit[] = "__cuCtxSetLimit"; +static const char __pyx_k_cuEventCreate[] = "__cuEventCreate"; +static const char __pyx_k_cuEventRecord[] = "__cuEventRecord"; +static const char __pyx_k_cuFuncGetName[] = "__cuFuncGetName"; +static const char __pyx_k_cuGraphCreate[] = "__cuGraphCreate"; +static const char __pyx_k_cuGraphLaunch[] = "__cuGraphLaunch"; +static const char __pyx_k_cuGraphUpload[] = "__cuGraphUpload"; +static const char __pyx_k_cuLinkDestroy[] = "__cuLinkDestroy"; +static const char __pyx_k_cuLogsCurrent[] = "__cuLogsCurrent"; +static const char __pyx_k_cuMemAlloc_v2[] = "__cuMemAlloc_v2"; +static const char __pyx_k_cuMemFreeHost[] = "__cuMemFreeHost"; +static const char __pyx_k_cuMemcpy2D_v2[] = "__cuMemcpy2D_v2"; +static const char __pyx_k_cuMemcpy3D_v2[] = "__cuMemcpy3D_v2"; +static const char __pyx_k_cuMemcpyAsync[] = "__cuMemcpyAsync"; +static const char __pyx_k_cuMemsetD8_v2[] = "__cuMemsetD8_v2"; +static const char __pyx_k_cuStreamGetId[] = "__cuStreamGetId"; +static const char __pyx_k_cuStreamQuery[] = "__cuStreamQuery"; +static const char __pyx_k_cuArrayDestroy[] = "__cuArrayDestroy"; +static const char __pyx_k_cuCtxCreate_v2[] = "__cuCtxCreate_v2"; +static const char __pyx_k_cuCtxCreate_v3[] = "__cuCtxCreate_v3"; +static const char __pyx_k_cuCtxCreate_v4[] = "__cuCtxCreate_v4"; +static const char __pyx_k_cuCtxGetDevice[] = "__cuCtxGetDevice"; +static const char __pyx_k_cuCtxWaitEvent[] = "__cuCtxWaitEvent"; +static const char __pyx_k_cuFuncIsLoaded[] = "__cuFuncIsLoaded"; +static const char __pyx_k_cuGetErrorName[] = "__cuGetErrorName"; +static const char __pyx_k_cuGraphAddNode[] = "__cuGraphAddNode"; +static const char __pyx_k_cuGraphDestroy[] = "__cuGraphDestroy"; +static const char __pyx_k_cuLaunchKernel[] = "__cuLaunchKernel"; +static const char __pyx_k_cuLinkComplete[] = "__cuLinkComplete"; +static const char __pyx_k_cuMemAdvise_v2[] = "__cuMemAdvise_v2"; +static const char __pyx_k_cuMemFreeAsync[] = "__cuMemFreeAsync"; +static const char __pyx_k_cuMemGetAccess[] = "__cuMemGetAccess"; +static const char __pyx_k_cuMemHostAlloc[] = "__cuMemHostAlloc"; +static const char __pyx_k_cuMemSetAccess[] = "__cuMemSetAccess"; +static const char __pyx_k_cuMemcpy3DPeer[] = "__cuMemcpy3DPeer"; +static const char __pyx_k_cuMemsetD16_v2[] = "__cuMemsetD16_v2"; +static const char __pyx_k_cuMemsetD32_v2[] = "__cuMemsetD32_v2"; +static const char __pyx_k_cuModuleUnload[] = "__cuModuleUnload"; +static const char __pyx_k_cuParamSetSize[] = "__cuParamSetSize"; +static const char __pyx_k_cuProfilerStop[] = "__cuProfilerStop"; +static const char __pyx_k_cuStreamCreate[] = "__cuStreamCreate"; +static const char __pyx_k_cuStreamGetCtx[] = "__cuStreamGetCtx"; +static const char __pyx_k_cuTexRefCreate[] = "__cuTexRefCreate"; +static const char __pyx_k_cuArrayGetPlane[] = "__cuArrayGetPlane"; +static const char __pyx_k_cuCtxDestroy_v2[] = "__cuCtxDestroy_v2"; +static const char __pyx_k_cuCtxGetCurrent[] = "__cuCtxGetCurrent"; +static const char __pyx_k_cuCtxSetCurrent[] = "__cuCtxSetCurrent"; +static const char __pyx_k_cuDeviceGetLuid[] = "__cuDeviceGetLuid"; +static const char __pyx_k_cuDeviceGetName[] = "__cuDeviceGetName"; +static const char __pyx_k_cuDeviceGetUuid[] = "__cuDeviceGetUuid"; +static const char __pyx_k_cuFuncGetModule[] = "__cuFuncGetModule"; +static const char __pyx_k_cuGraphGetEdges[] = "__cuGraphGetEdges"; +static const char __pyx_k_cuGraphGetNodes[] = "__cuGraphGetNodes"; +static const char __pyx_k_cuKernelGetName[] = "__cuKernelGetName"; +static const char __pyx_k_cuLibraryUnload[] = "__cuLibraryUnload"; +static const char __pyx_k_cuLinkCreate_v2[] = "__cuLinkCreate_v2"; +static const char __pyx_k_cuMemAllocAsync[] = "__cuMemAllocAsync"; +static const char __pyx_k_cuMemGetInfo_v2[] = "__cuMemGetInfo_v2"; +static const char __pyx_k_cuMemPoolCreate[] = "__cuMemPoolCreate"; +static const char __pyx_k_cuMemPoolTrimTo[] = "__cuMemPoolTrimTo"; +static const char __pyx_k_cuMemcpyAtoA_v2[] = "__cuMemcpyAtoA_v2"; +static const char __pyx_k_cuMemcpyAtoD_v2[] = "__cuMemcpyAtoD_v2"; +static const char __pyx_k_cuMemcpyAtoH_v2[] = "__cuMemcpyAtoH_v2"; +static const char __pyx_k_cuMemcpyDtoA_v2[] = "__cuMemcpyDtoA_v2"; +static const char __pyx_k_cuMemcpyDtoD_v2[] = "__cuMemcpyDtoD_v2"; +static const char __pyx_k_cuMemcpyDtoH_v2[] = "__cuMemcpyDtoH_v2"; +static const char __pyx_k_cuMemcpyHtoA_v2[] = "__cuMemcpyHtoA_v2"; +static const char __pyx_k_cuMemcpyHtoD_v2[] = "__cuMemcpyHtoD_v2"; +static const char __pyx_k_cuMemsetD2D8_v2[] = "__cuMemsetD2D8_v2"; +static const char __pyx_k_cuMemsetD8Async[] = "__cuMemsetD8Async"; +static const char __pyx_k_cuProfilerStart[] = "__cuProfilerStart"; +static const char __pyx_k_cuTexRefDestroy[] = "__cuTexRefDestroy"; +static const char __pyx_k_Failed_to_dlopen[] = "Failed to dlopen "; +static const char __pyx_k_cuArrayCreate_v2[] = "__cuArrayCreate_v2"; +static const char __pyx_k_cuCtxRecordEvent[] = "__cuCtxRecordEvent"; +static const char __pyx_k_cuCtxSynchronize[] = "__cuCtxSynchronize"; +static const char __pyx_k_cuDeviceGetCount[] = "__cuDeviceGetCount"; +static const char __pyx_k_cuGetErrorString[] = "__cuGetErrorString"; +static const char __pyx_k_cuGetExportTable[] = "__cuGetExportTable"; +static const char __pyx_k_cuGreenCtxCreate[] = "__cuGreenCtxCreate"; +static const char __pyx_k_cuLaunchHostFunc[] = "__cuLaunchHostFunc"; +static const char __pyx_k_cuLaunchKernelEx[] = "__cuLaunchKernelEx"; +static const char __pyx_k_cuLinkAddData_v2[] = "__cuLinkAddData_v2"; +static const char __pyx_k_cuLinkAddFile_v2[] = "__cuLinkAddFile_v2"; +static const char __pyx_k_cuLogsDumpToFile[] = "__cuLogsDumpToFile"; +static const char __pyx_k_cuMemAddressFree[] = "__cuMemAddressFree"; +static const char __pyx_k_cuMemPoolDestroy[] = "__cuMemPoolDestroy"; +static const char __pyx_k_cuMemsetD16Async[] = "__cuMemsetD16Async"; +static const char __pyx_k_cuMemsetD2D16_v2[] = "__cuMemsetD2D16_v2"; +static const char __pyx_k_cuMemsetD2D32_v2[] = "__cuMemsetD2D32_v2"; +static const char __pyx_k_cuMemsetD32Async[] = "__cuMemsetD32Async"; +static const char __pyx_k_cuModuleLoadData[] = "__cuModuleLoadData"; +static const char __pyx_k_cuParamSetTexRef[] = "__cuParamSetTexRef"; +static const char __pyx_k_cuStreamGetFlags[] = "__cuStreamGetFlags"; +static const char __pyx_k_cuTexRefGetArray[] = "__cuTexRefGetArray"; +static const char __pyx_k_cuTexRefGetFlags[] = "__cuTexRefGetFlags"; +static const char __pyx_k_cuTexRefSetArray[] = "__cuTexRefSetArray"; +static const char __pyx_k_cuTexRefSetFlags[] = "__cuTexRefSetFlags"; +static const char __pyx_k_cuVDPAUGetDevice[] = "__cuVDPAUGetDevice"; +static const char __pyx_k_cuCtxFromGreenCtx[] = "__cuCtxFromGreenCtx"; +static const char __pyx_k_cuEventDestroy_v2[] = "__cuEventDestroy_v2"; +static const char __pyx_k_cuGLGetDevices_v2[] = "__cuGLGetDevices_v2"; +static const char __pyx_k_cuGraphAddNode_v2[] = "__cuGraphAddNode_v2"; +static const char __pyx_k_cuGreenCtxDestroy[] = "__cuGreenCtxDestroy"; +static const char __pyx_k_cuIpcGetMemHandle[] = "__cuIpcGetMemHandle"; +static const char __pyx_k_cuLaunchGridAsync[] = "__cuLaunchGridAsync"; +static const char __pyx_k_cuLibraryLoadData[] = "__cuLibraryLoadData"; +static const char __pyx_k_cuMemAllocHost_v2[] = "__cuMemAllocHost_v2"; +static const char __pyx_k_cuMemAllocManaged[] = "__cuMemAllocManaged"; +static const char __pyx_k_cuMemHostGetFlags[] = "__cuMemHostGetFlags"; +static const char __pyx_k_cuMemcpyPeerAsync[] = "__cuMemcpyPeerAsync"; +static const char __pyx_k_cuMemsetD2D8Async[] = "__cuMemsetD2D8Async"; +static const char __pyx_k_cuModuleGetTexRef[] = "__cuModuleGetTexRef"; +static const char __pyx_k_cuMulticastCreate[] = "__cuMulticastCreate"; +static const char __pyx_k_cuMulticastUnbind[] = "__cuMulticastUnbind"; +static const char __pyx_k_cuStreamGetCtx_v2[] = "__cuStreamGetCtx_v2"; +static const char __pyx_k_cuStreamGetDevice[] = "__cuStreamGetDevice"; +static const char __pyx_k_cuStreamWaitEvent[] = "__cuStreamWaitEvent"; +static const char __pyx_k_cuSurfRefGetArray[] = "__cuSurfRefGetArray"; +static const char __pyx_k_cuSurfRefSetArray[] = "__cuSurfRefSetArray"; +static const char __pyx_k_cuTexObjectCreate[] = "__cuTexObjectCreate"; +static const char __pyx_k_cuTexRefGetFormat[] = "__cuTexRefGetFormat"; +static const char __pyx_k_cuTexRefSetFormat[] = "__cuTexRefSetFormat"; +static const char __pyx_k_asyncio_coroutines[] = "asyncio.coroutines"; +static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; +static const char __pyx_k_cuArray3DCreate_v2[] = "__cuArray3DCreate_v2"; +static const char __pyx_k_cuCtxGetApiVersion[] = "__cuCtxGetApiVersion"; +static const char __pyx_k_cuCtxPopCurrent_v2[] = "__cuCtxPopCurrent_v2"; +static const char __pyx_k_cuDeviceGetMemPool[] = "__cuDeviceGetMemPool"; +static const char __pyx_k_cuDeviceGetUuid_v2[] = "__cuDeviceGetUuid_v2"; +static const char __pyx_k_cuDeviceSetMemPool[] = "__cuDeviceSetMemPool"; +static const char __pyx_k_cuDriverGetVersion[] = "__cuDriverGetVersion"; +static const char __pyx_k_cuEventElapsedTime[] = "__cuEventElapsedTime"; +static const char __pyx_k_cuEventSynchronize[] = "__cuEventSynchronize"; +static const char __pyx_k_cuFuncGetAttribute[] = "__cuFuncGetAttribute"; +static const char __pyx_k_cuFuncGetParamInfo[] = "__cuFuncGetParamInfo"; +static const char __pyx_k_cuFuncSetAttribute[] = "__cuFuncSetAttribute"; +static const char __pyx_k_cuGraphAddHostNode[] = "__cuGraphAddHostNode"; +static const char __pyx_k_cuGraphDestroyNode[] = "__cuGraphDestroyNode"; +static const char __pyx_k_cuGraphExecDestroy[] = "__cuGraphExecDestroy"; +static const char __pyx_k_cuGraphGetEdges_v2[] = "__cuGraphGetEdges_v2"; +static const char __pyx_k_cuGraphNodeGetType[] = "__cuGraphNodeGetType"; +static const char __pyx_k_cuKernelGetLibrary[] = "__cuKernelGetLibrary"; +static const char __pyx_k_cuLibraryGetGlobal[] = "__cuLibraryGetGlobal"; +static const char __pyx_k_cuLibraryGetKernel[] = "__cuLibraryGetKernel"; +static const char __pyx_k_cuLibraryGetModule[] = "__cuLibraryGetModule"; +static const char __pyx_k_cuLogsDumpToMemory[] = "__cuLogsDumpToMemory"; +static const char __pyx_k_cuMemAllocPitch_v2[] = "__cuMemAllocPitch_v2"; +static const char __pyx_k_cuMemMapArrayAsync[] = "__cuMemMapArrayAsync"; +static const char __pyx_k_cuMemPoolGetAccess[] = "__cuMemPoolGetAccess"; +static const char __pyx_k_cuMemPoolSetAccess[] = "__cuMemPoolSetAccess"; +static const char __pyx_k_cuMemPrefetchAsync[] = "__cuMemPrefetchAsync"; +static const char __pyx_k_cuMemcpy2DAsync_v2[] = "__cuMemcpy2DAsync_v2"; +static const char __pyx_k_cuMemcpy3DAsync_v2[] = "__cuMemcpy3DAsync_v2"; +static const char __pyx_k_cuMemcpyBatchAsync[] = "__cuMemcpyBatchAsync"; +static const char __pyx_k_cuMemsetD2D16Async[] = "__cuMemsetD2D16Async"; +static const char __pyx_k_cuMemsetD2D32Async[] = "__cuMemsetD2D32Async"; +static const char __pyx_k_cuModuleGetSurfRef[] = "__cuModuleGetSurfRef"; +static const char __pyx_k_cuModuleLoadDataEx[] = "__cuModuleLoadDataEx"; +static const char __pyx_k_cuMulticastBindMem[] = "__cuMulticastBindMem"; +static const char __pyx_k_cuStreamDestroy_v2[] = "__cuStreamDestroy_v2"; +static const char __pyx_k_cuStreamEndCapture[] = "__cuStreamEndCapture"; +static const char __pyx_k_cuSurfObjectCreate[] = "__cuSurfObjectCreate"; +static const char __pyx_k_cuTexObjectDestroy[] = "__cuTexObjectDestroy"; +static const char __pyx_k_cuUserObjectCreate[] = "__cuUserObjectCreate"; +static const char __pyx_k_cuUserObjectRetain[] = "__cuUserObjectRetain"; +static const char __pyx_k_cuCtxGetCacheConfig[] = "__cuCtxGetCacheConfig"; +static const char __pyx_k_cuCtxGetDevResource[] = "__cuCtxGetDevResource"; +static const char __pyx_k_cuCtxPushCurrent_v2[] = "__cuCtxPushCurrent_v2"; +static const char __pyx_k_cuCtxSetCacheConfig[] = "__cuCtxSetCacheConfig"; +static const char __pyx_k_cuDeviceGetPCIBusId[] = "__cuDeviceGetPCIBusId"; +static const char __pyx_k_cuDeviceTotalMem_v2[] = "__cuDeviceTotalMem_v2"; +static const char __pyx_k_cuFuncSetBlockShape[] = "__cuFuncSetBlockShape"; +static const char __pyx_k_cuFuncSetSharedSize[] = "__cuFuncSetSharedSize"; +static const char __pyx_k_cuGetProcAddress_v2[] = "__cuGetProcAddress_v2"; +static const char __pyx_k_cuGraphAddEmptyNode[] = "__cuGraphAddEmptyNode"; +static const char __pyx_k_cuGraphExecGetFlags[] = "__cuGraphExecGetFlags"; +static const char __pyx_k_cuGraphGetRootNodes[] = "__cuGraphGetRootNodes"; +static const char __pyx_k_cuGreenCtxWaitEvent[] = "__cuGreenCtxWaitEvent"; +static const char __pyx_k_cuIpcCloseMemHandle[] = "__cuIpcCloseMemHandle"; +static const char __pyx_k_cuIpcGetEventHandle[] = "__cuIpcGetEventHandle"; +static const char __pyx_k_cuKernelGetFunction[] = "__cuKernelGetFunction"; +static const char __pyx_k_cuLibraryGetManaged[] = "__cuLibraryGetManaged"; +static const char __pyx_k_cuMemAddressReserve[] = "__cuMemAddressReserve"; +static const char __pyx_k_cuMemHostUnregister[] = "__cuMemHostUnregister"; +static const char __pyx_k_cuMemcpy3DPeerAsync[] = "__cuMemcpy3DPeerAsync"; +static const char __pyx_k_cuModuleGetFunction[] = "__cuModuleGetFunction"; +static const char __pyx_k_cuMulticastBindAddr[] = "__cuMulticastBindAddr"; +static const char __pyx_k_cuStreamAddCallback[] = "__cuStreamAddCallback"; +static const char __pyx_k_cuStreamGetGreenCtx[] = "__cuStreamGetGreenCtx"; +static const char __pyx_k_cuStreamGetPriority[] = "__cuStreamGetPriority"; +static const char __pyx_k_cuStreamIsCapturing[] = "__cuStreamIsCapturing"; +static const char __pyx_k_cuStreamSynchronize[] = "__cuStreamSynchronize"; +static const char __pyx_k_cuSurfObjectDestroy[] = "__cuSurfObjectDestroy"; +static const char __pyx_k_cuUserObjectRelease[] = "__cuUserObjectRelease"; +static const char __pyx_k_cuVDPAUCtxCreate_v2[] = "__cuVDPAUCtxCreate_v2"; +static const char __pyx_k_cuCtxGetExecAffinity[] = "__cuCtxGetExecAffinity"; +static const char __pyx_k_cuDeviceGetAttribute[] = "__cuDeviceGetAttribute"; +static const char __pyx_k_cuDeviceGraphMemTrim[] = "__cuDeviceGraphMemTrim"; +static const char __pyx_k_cuFuncSetCacheConfig[] = "__cuFuncSetCacheConfig"; +static const char __pyx_k_cuGraphAddMemcpyNode[] = "__cuGraphAddMemcpyNode"; +static const char __pyx_k_cuGraphAddMemsetNode[] = "__cuGraphAddMemsetNode"; +static const char __pyx_k_cuGraphDebugDotPrint[] = "__cuGraphDebugDotPrint"; +static const char __pyx_k_cuGraphExecUpdate_v2[] = "__cuGraphExecUpdate_v2"; +static const char __pyx_k_cuGraphNodeSetParams[] = "__cuGraphNodeSetParams"; +static const char __pyx_k_cuIpcOpenEventHandle[] = "__cuIpcOpenEventHandle"; +static const char __pyx_k_cuKernelGetAttribute[] = "__cuKernelGetAttribute"; +static const char __pyx_k_cuKernelGetParamInfo[] = "__cuKernelGetParamInfo"; +static const char __pyx_k_cuKernelSetAttribute[] = "__cuKernelSetAttribute"; +static const char __pyx_k_cuMemHostRegister_v2[] = "__cuMemHostRegister_v2"; +static const char __pyx_k_cuMemcpy3DBatchAsync[] = "__cuMemcpy3DBatchAsync"; +static const char __pyx_k_cuMemcpyAtoHAsync_v2[] = "__cuMemcpyAtoHAsync_v2"; +static const char __pyx_k_cuMemcpyDtoDAsync_v2[] = "__cuMemcpyDtoDAsync_v2"; +static const char __pyx_k_cuMemcpyDtoHAsync_v2[] = "__cuMemcpyDtoHAsync_v2"; +static const char __pyx_k_cuMemcpyHtoAAsync_v2[] = "__cuMemcpyHtoAAsync_v2"; +static const char __pyx_k_cuMemcpyHtoDAsync_v2[] = "__cuMemcpyHtoDAsync_v2"; +static const char __pyx_k_cuModuleGetGlobal_v2[] = "__cuModuleGetGlobal_v2"; +static const char __pyx_k_cuMulticastAddDevice[] = "__cuMulticastAddDevice"; +static const char __pyx_k_cuStreamGetAttribute[] = "__cuStreamGetAttribute"; +static const char __pyx_k_cuStreamSetAttribute[] = "__cuStreamSetAttribute"; +static const char __pyx_k_cuCtxEnablePeerAccess[] = "__cuCtxEnablePeerAccess"; +static const char __pyx_k_cuDeviceCanAccessPeer[] = "__cuDeviceCanAccessPeer"; +static const char __pyx_k_cuDeviceGetByPCIBusId[] = "__cuDeviceGetByPCIBusId"; +static const char __pyx_k_cuDeviceGetProperties[] = "__cuDeviceGetProperties"; +static const char __pyx_k_cuEventElapsedTime_v2[] = "__cuEventElapsedTime_v2"; +static const char __pyx_k_cuGraphAddMemFreeNode[] = "__cuGraphAddMemFreeNode"; +static const char __pyx_k_cuGraphNodeGetEnabled[] = "__cuGraphNodeGetEnabled"; +static const char __pyx_k_cuGraphNodeSetEnabled[] = "__cuGraphNodeSetEnabled"; +static const char __pyx_k_cuGreenCtxRecordEvent[] = "__cuGreenCtxRecordEvent"; +static const char __pyx_k_cuIpcOpenMemHandle_v2[] = "__cuIpcOpenMemHandle_v2"; +static const char __pyx_k_cuLibraryLoadFromFile[] = "__cuLibraryLoadFromFile"; +static const char __pyx_k_cuMemPoolGetAttribute[] = "__cuMemPoolGetAttribute"; +static const char __pyx_k_cuMemPoolSetAttribute[] = "__cuMemPoolSetAttribute"; +static const char __pyx_k_cuMemPrefetchAsync_v2[] = "__cuMemPrefetchAsync_v2"; +static const char __pyx_k_cuModuleLoadFatBinary[] = "__cuModuleLoadFatBinary"; +static const char __pyx_k_cuPointerGetAttribute[] = "__cuPointerGetAttribute"; +static const char __pyx_k_cuPointerSetAttribute[] = "__cuPointerSetAttribute"; +static const char __pyx_k_cuStreamBatchMemOp_v2[] = "__cuStreamBatchMemOp_v2"; +static const char __pyx_k_cuTexRefGetAddress_v2[] = "__cuTexRefGetAddress_v2"; +static const char __pyx_k_cuTexRefGetFilterMode[] = "__cuTexRefGetFilterMode"; +static const char __pyx_k_cuTexRefSetAddress_v2[] = "__cuTexRefSetAddress_v2"; +static const char __pyx_k_cuTexRefSetFilterMode[] = "__cuTexRefSetFilterMode"; +static const char __pyx_k_cuCoredumpGetAttribute[] = "__cuCoredumpGetAttribute"; +static const char __pyx_k_cuCoredumpSetAttribute[] = "__cuCoredumpSetAttribute"; +static const char __pyx_k_cuCtxDisablePeerAccess[] = "__cuCtxDisablePeerAccess"; +static const char __pyx_k_cuDeviceGetDevResource[] = "__cuDeviceGetDevResource"; +static const char __pyx_k_cuEventRecordWithFlags[] = "__cuEventRecordWithFlags"; +static const char __pyx_k_cuGraphAddDependencies[] = "__cuGraphAddDependencies"; +static const char __pyx_k_cuGraphAddMemAllocNode[] = "__cuGraphAddMemAllocNode"; +static const char __pyx_k_cuGraphNodeFindInClone[] = "__cuGraphNodeFindInClone"; +static const char __pyx_k_cuGraphicsMapResources[] = "__cuGraphicsMapResources"; +static const char __pyx_k_cuGreenCtxStreamCreate[] = "__cuGreenCtxStreamCreate"; +static const char __pyx_k_cuImportExternalMemory[] = "__cuImportExternalMemory"; +static const char __pyx_k_cuKernelSetCacheConfig[] = "__cuKernelSetCacheConfig"; +static const char __pyx_k_cuLogsRegisterCallback[] = "__cuLogsRegisterCallback"; +static const char __pyx_k_cuMemPoolExportPointer[] = "__cuMemPoolExportPointer"; +static const char __pyx_k_cuMemPoolImportPointer[] = "__cuMemPoolImportPointer"; +static const char __pyx_k_cuMemRangeGetAttribute[] = "__cuMemRangeGetAttribute"; +static const char __pyx_k_cuMemcpy2DUnaligned_v2[] = "__cuMemcpy2DUnaligned_v2"; +static const char __pyx_k_cuMipmappedArrayCreate[] = "__cuMipmappedArrayCreate"; +static const char __pyx_k_cuModuleGetLoadingMode[] = "__cuModuleGetLoadingMode"; +static const char __pyx_k_cuPointerGetAttributes[] = "__cuPointerGetAttributes"; +static const char __pyx_k_cuStreamAttachMemAsync[] = "__cuStreamAttachMemAsync"; +static const char __pyx_k_cuStreamCopyAttributes[] = "__cuStreamCopyAttributes"; +static const char __pyx_k_cuStreamWaitValue32_v2[] = "__cuStreamWaitValue32_v2"; +static const char __pyx_k_cuStreamWaitValue64_v2[] = "__cuStreamWaitValue64_v2"; +static const char __pyx_k_cuTensorMapEncodeTiled[] = "__cuTensorMapEncodeTiled"; +static const char __pyx_k_cuTexRefGetAddressMode[] = "__cuTexRefGetAddressMode"; +static const char __pyx_k_cuTexRefGetBorderColor[] = "__cuTexRefGetBorderColor"; +static const char __pyx_k_cuTexRefSetAddressMode[] = "__cuTexRefSetAddressMode"; +static const char __pyx_k_cuTexRefSetBorderColor[] = "__cuTexRefSetBorderColor"; +static const char __pyx_k_cuArrayGetDescriptor_v2[] = "__cuArrayGetDescriptor_v2"; +static const char __pyx_k_cuCheckpointProcessLock[] = "__cuCheckpointProcessLock"; +static const char __pyx_k_cuCtxGetSharedMemConfig[] = "__cuCtxGetSharedMemConfig"; +static const char __pyx_k_cuCtxSetSharedMemConfig[] = "__cuCtxSetSharedMemConfig"; +static const char __pyx_k_cuDestroyExternalMemory[] = "__cuDestroyExternalMemory"; +static const char __pyx_k_cuDeviceGetP2PAttribute[] = "__cuDeviceGetP2PAttribute"; +static const char __pyx_k_cuGraphAddEventWaitNode[] = "__cuGraphAddEventWaitNode"; +static const char __pyx_k_cuGraphAddKernelNode_v2[] = "__cuGraphAddKernelNode_v2"; +static const char __pyx_k_cuGraphRetainUserObject[] = "__cuGraphRetainUserObject"; +static const char __pyx_k_cuLibraryGetKernelCount[] = "__cuLibraryGetKernelCount"; +static const char __pyx_k_cuMemAllocFromPoolAsync[] = "__cuMemAllocFromPoolAsync"; +static const char __pyx_k_cuMemGetAddressRange_v2[] = "__cuMemGetAddressRange_v2"; +static const char __pyx_k_cuMemRangeGetAttributes[] = "__cuMemRangeGetAttributes"; +static const char __pyx_k_cuMipmappedArrayDestroy[] = "__cuMipmappedArrayDestroy"; +static const char __pyx_k_cuStreamBeginCapture_v2[] = "__cuStreamBeginCapture_v2"; +static const char __pyx_k_cuStreamWriteValue32_v2[] = "__cuStreamWriteValue32_v2"; +static const char __pyx_k_cuStreamWriteValue64_v2[] = "__cuStreamWriteValue64_v2"; +static const char __pyx_k_cuTensorMapEncodeIm2col[] = "__cuTensorMapEncodeIm2col"; +static const char __pyx_k_cuTexRefSetAddress2D_v3[] = "__cuTexRefSetAddress2D_v3"; +static const char __pyx_k_cuDevicePrimaryCtxRetain[] = "__cuDevicePrimaryCtxRetain"; +static const char __pyx_k_cuEventCreateFromEGLSync[] = "__cuEventCreateFromEGLSync"; +static const char __pyx_k_cuFuncSetSharedMemConfig[] = "__cuFuncSetSharedMemConfig"; +static const char __pyx_k_cuGraphAddBatchMemOpNode[] = "__cuGraphAddBatchMemOpNode"; +static const char __pyx_k_cuGraphAddChildGraphNode[] = "__cuGraphAddChildGraphNode"; +static const char __pyx_k_cuGraphExecNodeSetParams[] = "__cuGraphExecNodeSetParams"; +static const char __pyx_k_cuGraphHostNodeGetParams[] = "__cuGraphHostNodeGetParams"; +static const char __pyx_k_cuGraphHostNodeSetParams[] = "__cuGraphHostNodeSetParams"; +static const char __pyx_k_cuGraphReleaseUserObject[] = "__cuGraphReleaseUserObject"; +static const char __pyx_k_cuGraphicsUnmapResources[] = "__cuGraphicsUnmapResources"; +static const char __pyx_k_cuGreenCtxGetDevResource[] = "__cuGreenCtxGetDevResource"; +static const char __pyx_k_cuLogsUnregisterCallback[] = "__cuLogsUnregisterCallback"; +static const char __pyx_k_cuMipmappedArrayGetLevel[] = "__cuMipmappedArrayGetLevel"; +static const char __pyx_k_cuModuleGetFunctionCount[] = "__cuModuleGetFunctionCount"; +static const char __pyx_k_cuTexRefGetMaxAnisotropy[] = "__cuTexRefGetMaxAnisotropy"; +static const char __pyx_k_cuTexRefSetMaxAnisotropy[] = "__cuTexRefSetMaxAnisotropy"; +static const char __pyx_k_inspect_function_pointer[] = "_inspect_function_pointer"; +static const char __pyx_k_Function_cuInit_not_found[] = "Function \"cuInit\" not found"; +static const char __pyx_k_cuArray3DGetDescriptor_v2[] = "__cuArray3DGetDescriptor_v2"; +static const char __pyx_k_cuCheckpointProcessUnlock[] = "__cuCheckpointProcessUnlock"; +static const char __pyx_k_cuDevResourceGenerateDesc[] = "__cuDevResourceGenerateDesc"; +static const char __pyx_k_cuDeviceComputeCapability[] = "__cuDeviceComputeCapability"; +static const char __pyx_k_cuDeviceGetDefaultMemPool[] = "__cuDeviceGetDefaultMemPool"; +static const char __pyx_k_cuGraphAddDependencies_v2[] = "__cuGraphAddDependencies_v2"; +static const char __pyx_k_cuGraphAddEventRecordNode[] = "__cuGraphAddEventRecordNode"; +static const char __pyx_k_cuGraphRemoveDependencies[] = "__cuGraphRemoveDependencies"; +static const char __pyx_k_cuGraphicsGLRegisterImage[] = "__cuGraphicsGLRegisterImage"; +static const char __pyx_k_cuImportExternalSemaphore[] = "__cuImportExternalSemaphore"; +static const char __pyx_k_cuLaunchCooperativeKernel[] = "__cuLaunchCooperativeKernel"; +static const char __pyx_k_cuLibraryEnumerateKernels[] = "__cuLibraryEnumerateKernels"; +static const char __pyx_k_cuMemBatchDecompressAsync[] = "__cuMemBatchDecompressAsync"; +static const char __pyx_k_cuMulticastGetGranularity[] = "__cuMulticastGetGranularity"; +static const char __pyx_k_cuStreamGetCaptureInfo_v2[] = "__cuStreamGetCaptureInfo_v2"; +static const char __pyx_k_cuStreamGetCaptureInfo_v3[] = "__cuStreamGetCaptureInfo_v3"; +static const char __pyx_k_cuTensorMapReplaceAddress[] = "__cuTensorMapReplaceAddress"; +static const char __pyx_k_cuTexObjectGetTextureDesc[] = "__cuTexObjectGetTextureDesc"; +static const char __pyx_k_cuTexRefGetMipmappedArray[] = "__cuTexRefGetMipmappedArray"; +static const char __pyx_k_cuTexRefSetMipmappedArray[] = "__cuTexRefSetMipmappedArray"; +static const char __pyx_k_inspect_function_pointers[] = "_inspect_function_pointers"; +static const char __pyx_k_cuArrayGetSparseProperties[] = "__cuArrayGetSparseProperties"; +static const char __pyx_k_cuCheckpointProcessRestore[] = "__cuCheckpointProcessRestore"; +static const char __pyx_k_cuDestroyExternalSemaphore[] = "__cuDestroyExternalSemaphore"; +static const char __pyx_k_cuDevicePrimaryCtxGetState[] = "__cuDevicePrimaryCtxGetState"; +static const char __pyx_k_cuDevicePrimaryCtxReset_v2[] = "__cuDevicePrimaryCtxReset_v2"; +static const char __pyx_k_cuEGLStreamConsumerConnect[] = "__cuEGLStreamConsumerConnect"; +static const char __pyx_k_cuEGLStreamProducerConnect[] = "__cuEGLStreamProducerConnect"; +static const char __pyx_k_cuFlushGPUDirectRDMAWrites[] = "__cuFlushGPUDirectRDMAWrites"; +static const char __pyx_k_cuGraphMemcpyNodeGetParams[] = "__cuGraphMemcpyNodeGetParams"; +static const char __pyx_k_cuGraphMemcpyNodeSetParams[] = "__cuGraphMemcpyNodeSetParams"; +static const char __pyx_k_cuGraphMemsetNodeGetParams[] = "__cuGraphMemsetNodeGetParams"; +static const char __pyx_k_cuGraphMemsetNodeSetParams[] = "__cuGraphMemsetNodeSetParams"; +static const char __pyx_k_cuGraphNodeGetDependencies[] = "__cuGraphNodeGetDependencies"; +static const char __pyx_k_cuGraphicsEGLRegisterImage[] = "__cuGraphicsEGLRegisterImage"; +static const char __pyx_k_cuGraphicsGLRegisterBuffer[] = "__cuGraphicsGLRegisterBuffer"; +static const char __pyx_k_cuModuleEnumerateFunctions[] = "__cuModuleEnumerateFunctions"; +static const char __pyx_k_cuStreamCreateWithPriority[] = "__cuStreamCreateWithPriority"; +static const char __pyx_k_cuTexObjectGetResourceDesc[] = "__cuTexObjectGetResourceDesc"; +static const char __pyx_k_cuTexRefGetMipmapLevelBias[] = "__cuTexRefGetMipmapLevelBias"; +static const char __pyx_k_cuTexRefSetMipmapLevelBias[] = "__cuTexRefSetMipmapLevelBias"; +static const char __pyx_k_Function_cuLaunch_not_found[] = "Function \"cuLaunch\" not found"; +static const char __pyx_k_Function_cuMemMap_not_found[] = "Function \"cuMemMap\" not found"; +static const char __pyx_k_Function_cuMemcpy_not_found[] = "Function \"cuMemcpy\" not found"; +static const char __pyx_k_cuCheckpointProcessGetState[] = "__cuCheckpointProcessGetState"; +static const char __pyx_k_cuCtxGetStreamPriorityRange[] = "__cuCtxGetStreamPriorityRange"; +static const char __pyx_k_cuCtxResetPersistingL2Cache[] = "__cuCtxResetPersistingL2Cache"; +static const char __pyx_k_cuDevSmResourceSplitByCount[] = "__cuDevSmResourceSplitByCount"; +static const char __pyx_k_cuGraphInstantiateWithFlags[] = "__cuGraphInstantiateWithFlags"; +static const char __pyx_k_cuGraphMemFreeNodeGetParams[] = "__cuGraphMemFreeNodeGetParams"; +static const char __pyx_k_cuLibraryGetUnifiedFunction[] = "__cuLibraryGetUnifiedFunction"; +static const char __pyx_k_cuMemRetainAllocationHandle[] = "__cuMemRetainAllocationHandle"; +static const char __pyx_k_cuStreamBeginCaptureToGraph[] = "__cuStreamBeginCaptureToGraph"; +static const char __pyx_k_cuSurfObjectGetResourceDesc[] = "__cuSurfObjectGetResourceDesc"; +static const char __pyx_k_cuTensorMapEncodeIm2colWide[] = "__cuTensorMapEncodeIm2colWide"; +static const char __pyx_k_cuTexRefGetMipmapFilterMode[] = "__cuTexRefGetMipmapFilterMode"; +static const char __pyx_k_cuTexRefGetMipmapLevelClamp[] = "__cuTexRefGetMipmapLevelClamp"; +static const char __pyx_k_cuTexRefSetMipmapFilterMode[] = "__cuTexRefSetMipmapFilterMode"; +static const char __pyx_k_cuTexRefSetMipmapLevelClamp[] = "__cuTexRefSetMipmapLevelClamp"; +static const char __pyx_k_cuArrayGetMemoryRequirements[] = "__cuArrayGetMemoryRequirements"; +static const char __pyx_k_cuCoredumpGetAttributeGlobal[] = "__cuCoredumpGetAttributeGlobal"; +static const char __pyx_k_cuCoredumpSetAttributeGlobal[] = "__cuCoredumpSetAttributeGlobal"; +static const char __pyx_k_cuDeviceGetGraphMemAttribute[] = "__cuDeviceGetGraphMemAttribute"; +static const char __pyx_k_cuDevicePrimaryCtxRelease_v2[] = "__cuDevicePrimaryCtxRelease_v2"; +static const char __pyx_k_cuDeviceSetGraphMemAttribute[] = "__cuDeviceSetGraphMemAttribute"; +static const char __pyx_k_cuGraphEventWaitNodeGetEvent[] = "__cuGraphEventWaitNodeGetEvent"; +static const char __pyx_k_cuGraphEventWaitNodeSetEvent[] = "__cuGraphEventWaitNodeSetEvent"; +static const char __pyx_k_cuGraphExecHostNodeSetParams[] = "__cuGraphExecHostNodeSetParams"; +static const char __pyx_k_cuGraphInstantiateWithParams[] = "__cuGraphInstantiateWithParams"; +static const char __pyx_k_cuGraphMemAllocNodeGetParams[] = "__cuGraphMemAllocNodeGetParams"; +static const char __pyx_k_cuGraphNodeGetDependentNodes[] = "__cuGraphNodeGetDependentNodes"; +static const char __pyx_k_cuGraphRemoveDependencies_v2[] = "__cuGraphRemoveDependencies_v2"; +static const char __pyx_k_cuGraphicsUnregisterResource[] = "__cuGraphicsUnregisterResource"; +static const char __pyx_k_cuMemExportToShareableHandle[] = "__cuMemExportToShareableHandle"; +static const char __pyx_k_cuMemHostGetDevicePointer_v2[] = "__cuMemHostGetDevicePointer_v2"; +static const char __pyx_k_cuOccupancyMaxActiveClusters[] = "__cuOccupancyMaxActiveClusters"; +static const char __pyx_k_Function_cuCtxGetId_not_found[] = "Function \"cuCtxGetId\" not found"; +static const char __pyx_k_Function_cuFuncLoad_not_found[] = "Function \"cuFuncLoad\" not found"; +static const char __pyx_k_Function_cuMemUnmap_not_found[] = "Function \"cuMemUnmap\" not found"; +static const char __pyx_k_cuCheckpointProcessCheckpoint[] = "__cuCheckpointProcessCheckpoint"; +static const char __pyx_k_cuDevicePrimaryCtxSetFlags_v2[] = "__cuDevicePrimaryCtxSetFlags_v2"; +static const char __pyx_k_cuEGLStreamConsumerDisconnect[] = "__cuEGLStreamConsumerDisconnect"; +static const char __pyx_k_cuEGLStreamProducerDisconnect[] = "__cuEGLStreamProducerDisconnect"; +static const char __pyx_k_cuGraphChildGraphNodeGetGraph[] = "__cuGraphChildGraphNodeGetGraph"; +static const char __pyx_k_cuGraphKernelNodeGetAttribute[] = "__cuGraphKernelNodeGetAttribute"; +static const char __pyx_k_cuGraphKernelNodeGetParams_v2[] = "__cuGraphKernelNodeGetParams_v2"; +static const char __pyx_k_cuGraphKernelNodeSetAttribute[] = "__cuGraphKernelNodeSetAttribute"; +static const char __pyx_k_cuGraphKernelNodeSetParams_v2[] = "__cuGraphKernelNodeSetParams_v2"; +static const char __pyx_k_cuGraphNodeGetDependencies_v2[] = "__cuGraphNodeGetDependencies_v2"; +static const char __pyx_k_cuGraphicsResourceSetMapFlags[] = "__cuGraphicsResourceSetMapFlags_v2"; +static const char __pyx_k_cuMemGetAllocationGranularity[] = "__cuMemGetAllocationGranularity"; +static const char __pyx_k_cuMemGetHandleForAddressRange[] = "__cuMemGetHandleForAddressRange"; +static const char __pyx_k_cuWaitExternalSemaphoresAsync[] = "__cuWaitExternalSemaphoresAsync"; +static const char __pyx_k_Function_cuCtxAttach_not_found[] = "Function \"cuCtxAttach\" not found"; +static const char __pyx_k_Function_cuCtxDetach_not_found[] = "Function \"cuCtxDetach\" not found"; +static const char __pyx_k_Function_cuDeviceGet_not_found[] = "Function \"cuDeviceGet\" not found"; +static const char __pyx_k_Function_cuMemAdvise_not_found[] = "Function \"cuMemAdvise\" not found"; +static const char __pyx_k_Function_cuMemCreate_not_found[] = "Function \"cuMemCreate\" not found"; +static const char __pyx_k_Function_cuParamSetf_not_found[] = "Function \"cuParamSetf\" not found"; +static const char __pyx_k_Function_cuParamSeti_not_found[] = "Function \"cuParamSeti\" not found"; +static const char __pyx_k_Function_cuParamSetv_not_found[] = "Function \"cuParamSetv\" not found"; +static const char __pyx_k_cuCheckpointProcessGetRestoreT[] = "__cuCheckpointProcessGetRestoreThreadId"; +static const char __pyx_k_cuDeviceGetExecAffinitySupport[] = "__cuDeviceGetExecAffinitySupport"; +static const char __pyx_k_cuDeviceGetNvSciSyncAttributes[] = "__cuDeviceGetNvSciSyncAttributes"; +static const char __pyx_k_cuDeviceGetTexture1DLinearMaxW[] = "__cuDeviceGetTexture1DLinearMaxWidth"; +static const char __pyx_k_cuDeviceRegisterAsyncNotificat[] = "__cuDeviceRegisterAsyncNotification"; +static const char __pyx_k_cuDeviceUnregisterAsyncNotific[] = "__cuDeviceUnregisterAsyncNotification"; +static const char __pyx_k_cuEGLStreamConsumerAcquireFram[] = "__cuEGLStreamConsumerAcquireFrame"; +static const char __pyx_k_cuEGLStreamConsumerConnectWith[] = "__cuEGLStreamConsumerConnectWithFlags"; +static const char __pyx_k_cuEGLStreamConsumerReleaseFram[] = "__cuEGLStreamConsumerReleaseFrame"; +static const char __pyx_k_cuEGLStreamProducerPresentFram[] = "__cuEGLStreamProducerPresentFrame"; +static const char __pyx_k_cuEGLStreamProducerReturnFrame[] = "__cuEGLStreamProducerReturnFrame"; +static const char __pyx_k_cuExternalMemoryGetMappedBuffe[] = "__cuExternalMemoryGetMappedBuffer"; +static const char __pyx_k_cuExternalMemoryGetMappedMipma[] = "__cuExternalMemoryGetMappedMipmappedArray"; +static const char __pyx_k_cuGraphAddExternalSemaphoresSi[] = "__cuGraphAddExternalSemaphoresSignalNode"; +static const char __pyx_k_cuGraphAddExternalSemaphoresWa[] = "__cuGraphAddExternalSemaphoresWaitNode"; +static const char __pyx_k_cuGraphBatchMemOpNodeGetParams[] = "__cuGraphBatchMemOpNodeGetParams"; +static const char __pyx_k_cuGraphBatchMemOpNodeSetParams[] = "__cuGraphBatchMemOpNodeSetParams"; +static const char __pyx_k_cuGraphConditionalHandleCreate[] = "__cuGraphConditionalHandleCreate"; +static const char __pyx_k_cuGraphEventRecordNodeGetEvent[] = "__cuGraphEventRecordNodeGetEvent"; +static const char __pyx_k_cuGraphEventRecordNodeSetEvent[] = "__cuGraphEventRecordNodeSetEvent"; +static const char __pyx_k_cuGraphExecBatchMemOpNodeSetPa[] = "__cuGraphExecBatchMemOpNodeSetParams"; +static const char __pyx_k_cuGraphExecChildGraphNodeSetPa[] = "__cuGraphExecChildGraphNodeSetParams"; +static const char __pyx_k_cuGraphExecEventRecordNodeSetE[] = "__cuGraphExecEventRecordNodeSetEvent"; +static const char __pyx_k_cuGraphExecEventWaitNodeSetEve[] = "__cuGraphExecEventWaitNodeSetEvent"; +static const char __pyx_k_cuGraphExecExternalSemaphoresS[] = "__cuGraphExecExternalSemaphoresSignalNodeSetParams"; +static const char __pyx_k_cuGraphExecExternalSemaphoresW[] = "__cuGraphExecExternalSemaphoresWaitNodeSetParams"; +static const char __pyx_k_cuGraphExecKernelNodeSetParams[] = "__cuGraphExecKernelNodeSetParams_v2"; +static const char __pyx_k_cuGraphExecMemcpyNodeSetParams[] = "__cuGraphExecMemcpyNodeSetParams"; +static const char __pyx_k_cuGraphExecMemsetNodeSetParams[] = "__cuGraphExecMemsetNodeSetParams"; +static const char __pyx_k_cuGraphExternalSemaphoresSigna[] = "__cuGraphExternalSemaphoresSignalNodeGetParams"; +static const char __pyx_k_cuGraphExternalSemaphoresWaitN[] = "__cuGraphExternalSemaphoresWaitNodeGetParams"; +static const char __pyx_k_cuGraphKernelNodeCopyAttribute[] = "__cuGraphKernelNodeCopyAttributes"; +static const char __pyx_k_cuGraphNodeGetDependentNodes_v[] = "__cuGraphNodeGetDependentNodes_v2"; +static const char __pyx_k_cuGraphicsResourceGetMappedEgl[] = "__cuGraphicsResourceGetMappedEglFrame"; +static const char __pyx_k_cuGraphicsResourceGetMappedMip[] = "__cuGraphicsResourceGetMappedMipmappedArray"; +static const char __pyx_k_cuGraphicsResourceGetMappedPoi[] = "__cuGraphicsResourceGetMappedPointer_v2"; +static const char __pyx_k_cuGraphicsSubResourceGetMapped[] = "__cuGraphicsSubResourceGetMappedArray"; +static const char __pyx_k_cuGraphicsVDPAURegisterOutputS[] = "__cuGraphicsVDPAURegisterOutputSurface"; +static const char __pyx_k_cuGraphicsVDPAURegisterVideoSu[] = "__cuGraphicsVDPAURegisterVideoSurface"; +static const char __pyx_k_cuLaunchCooperativeKernelMulti[] = "__cuLaunchCooperativeKernelMultiDevice"; +static const char __pyx_k_cuMemGetAllocationPropertiesFr[] = "__cuMemGetAllocationPropertiesFromHandle"; +static const char __pyx_k_cuMemImportFromShareableHandle[] = "__cuMemImportFromShareableHandle"; +static const char __pyx_k_cuMemPoolExportToShareableHand[] = "__cuMemPoolExportToShareableHandle"; +static const char __pyx_k_cuMemPoolImportFromShareableHa[] = "__cuMemPoolImportFromShareableHandle"; +static const char __pyx_k_cuMipmappedArrayGetMemoryRequi[] = "__cuMipmappedArrayGetMemoryRequirements"; +static const char __pyx_k_cuMipmappedArrayGetSparsePrope[] = "__cuMipmappedArrayGetSparseProperties"; +static const char __pyx_k_cuOccupancyAvailableDynamicSMe[] = "__cuOccupancyAvailableDynamicSMemPerBlock"; +static const char __pyx_k_cuOccupancyMaxActiveBlocksPerM[] = "__cuOccupancyMaxActiveBlocksPerMultiprocessor"; +static const char __pyx_k_cuOccupancyMaxPotentialBlockSi[] = "__cuOccupancyMaxPotentialBlockSize"; +static const char __pyx_k_cuOccupancyMaxPotentialCluster[] = "__cuOccupancyMaxPotentialClusterSize"; +static const char __pyx_k_cuSignalExternalSemaphoresAsyn[] = "__cuSignalExternalSemaphoresAsync"; +static const char __pyx_k_cuStreamUpdateCaptureDependenc[] = "__cuStreamUpdateCaptureDependencies"; +static const char __pyx_k_cuTexObjectGetResourceViewDesc[] = "__cuTexObjectGetResourceViewDesc"; +static const char __pyx_k_cuThreadExchangeStreamCaptureM[] = "__cuThreadExchangeStreamCaptureMode"; +static const char __pyx_k_z_q_Q_1_z_z_Q_Ja_1_Q_Ja_3_Q_Zq[] = "\200\001\330\004\005\330\004\007\200z\220\027\230\001\330\010\017\210q\340\004\020\220\001\330\004\025\220Q\340\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\210\036\220z\240\021\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t\034\230J\240a\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t3\260:\270Q\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t/\250z\270\021\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t/\250z\270\021\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t.\250j\270\001\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t\036\230j\250\001""\330\004\005\330\004\010\210\001\320\t\033\230:\240Q\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t\034\230J\240a\330\004\005\330\004\010\210\001\320\t\034\230J\240a\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t\035\230Z\240q\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t*\250*\260A\330""\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t\035\230Z\240q\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t2\260*\270A\330\004\005\330\004\010\210\001\320\t4\260J\270a\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t\031\230\032\2401\330\004\005\330\004\010\210\001\320\t\035\230Z\240q\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t ""\240\n\250!\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t4\260J\270a\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t6\260j\300\001\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t\037\230z\250""\021\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t.\250j\270\001\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t\034\230J\240a\330\004\005\330\004\010\210\001\320\t\035\230Z\240q\330\004\005\330\004\010\210\001\320\t\031\230\032\2401\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t\033\230:\240Q\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t/\250z\270\021\330\004\005\330\004\010\210\001\320\t.\250j\270\001\330\004\005\330\004\010\210\001\320\t7\260z\300\021\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t1\260\032\2701\330\004\005\330\004\010\210\001\320\t3\260:\270Q\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t$\240J\250a\330""\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t\034\230J\240a\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t2\260*\270A\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t2\260*\270A\330\004\005\330\004\010\210\001\320\t5\260Z\270q\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330""\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t\035\230Z\240q\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t0\260\n\270!\330\004\005\330\004\010\210\001\320\t8\270\n\300!\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t0\260\n\270!\330\004\005\330\004\010\210\001\320\t.\250j\270\001\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t\033\230:\240Q\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t5\260Z\270q\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t\034\230J\240a\330\004\005\330\004\010\210\001\320\t\034\230J\240a\330\004\005\330\004\010\210\001\320\t\034\230J\240a\330\004\005\330\004\010\210\001\320\t\031\230\032\2401\330\004\005\330\004\010""\210\001\320\t\035\230Z\240q\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t.\250j\270\001\330\004\005\330\004\010\210\001\320\t.\250j\270\001\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t.\250j\270\001\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t/\250z\270\021\330\004\005\330\004\010\210\001\320\t/\250z\270\021\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t7\260z\300\021\330\004\005\330\004\010\210\001\320\t=\270Z\300q\330\004\005\330\004\010\210\001\320\t=\270Z\300q\330\004\005\330\004\010\210\001\320\t5\260Z\270q\330\004\005\330\004\010\210\001\320\t;\270:\300Q\330\004\005\330\004\010\210\001\320\t;\270:\300Q\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t/\250z\270\021\330\004\005\330\004\010\210\001\320\t/\250z\270\021\330\004\005\330\004\010\210\001\320\t3\260:\270Q\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t,\250J\260a""\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t\035\230Z\240q\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t.\250j\270\001\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t0\260\n\270!\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t2\260*\270A\330\004\005\330\004\010\210\001\320\t/\250z\270\021\330\004\005\330\004\010\210\001\320\t/\250z\270\021\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t3\260:\270Q\330\004\005\330\004\010\210\001\320\t3\260:\270Q\330\004\005\330\004\010\210\001\320\t1\260\032\2701\330\004\005\330\004\010\210\001\320\tA\300\032\3101\330\004\005\330\004\010\210\001\320\t?\270z\310\021\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t0\260\n\270!\330\004\005\330\004\010\210\001""\320\t.\250j\270\001\330\004\005\330\004\010\210\001\320\t.\250j\270\001\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t%\240Z\250q\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t/\250z\270\021\330\004\005\330\004\010\210\001\320\t<\270J\300a\330\004\005\330\004\010\210\001\320\tE\300Z\310q\330\004\005\330\004\010\210\001\320\t1\260\032\2701\330\004\005\330\004\010\210\001\320\t:\270*\300A\330\004\005\330\004\010\210\001\320\t8\270\n\300!\330\004\005\330\004\010\210\001\320\t3\260:\270Q\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004""\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t/\250z\270\021\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t4\260J\270a\330\004\005\330\004\010\210\001\320\t:\270*\300A\330\004\005\330\004\010\210\001\320\t6\260j\300\001\330\004\005\330\004\010\210\001\320\t1\260\032\2701\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t-\250Z\260q\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320""\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t&\240j\260\001\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t6\260j\300\001\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t.\250j\270\001\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t\037\230z\250\021\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t4\260J\270a\330\004\005\330\004\010\210\001\320\t.\250j\270\001\330\004\005\330\004\010\210\001\320\t0\260\n\270!\330\004\005\330\004\010\210\001\320\t0\260\n\270!\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t.\250j\270\001\330\004\005\330\004\010\210\001\320\t0\260\n\270!\330\004\005\330\004\010\210\001\320\t/\250z\270\021\330\004\005\330\004\010\210\001\320\t4\260J\270a\330\004\005\330\004\010\210\001\320\t)\250\032\2601\330\004\005\330\004\010\210\001\320\t+\250:\260Q\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005""\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t4\260J\270a\330\004\005\330\004\010\210\001\320\t5\260Z\270q\340\004\020\220\001\330\004\013\2101"; +static const char __pyx_k_Function_cuArray3DCreate_v2_not[] = "Function \"cuArray3DCreate_v2\" not found"; +static const char __pyx_k_Function_cuArray3DGetDescriptor[] = "Function \"cuArray3DGetDescriptor_v2\" not found"; +static const char __pyx_k_Function_cuCoredumpGetAttribute[] = "Function \"cuCoredumpGetAttribute\" not found"; +static const char __pyx_k_Function_cuCoredumpSetAttribute[] = "Function \"cuCoredumpSetAttribute\" not found"; +static const char __pyx_k_Function_cuCtxDisablePeerAccess[] = "Function \"cuCtxDisablePeerAccess\" not found"; +static const char __pyx_k_Function_cuCtxGetApiVersion_not[] = "Function \"cuCtxGetApiVersion\" not found"; +static const char __pyx_k_Function_cuCtxPopCurrent_v2_not[] = "Function \"cuCtxPopCurrent_v2\" not found"; +static const char __pyx_k_Function_cuDeviceGetDevResource[] = "Function \"cuDeviceGetDevResource\" not found"; +static const char __pyx_k_Function_cuDeviceGetMemPool_not[] = "Function \"cuDeviceGetMemPool\" not found"; +static const char __pyx_k_Function_cuDeviceGetUuid_v2_not[] = "Function \"cuDeviceGetUuid_v2\" not found"; +static const char __pyx_k_Function_cuDeviceSetMemPool_not[] = "Function \"cuDeviceSetMemPool\" not found"; +static const char __pyx_k_Function_cuDriverGetVersion_not[] = "Function \"cuDriverGetVersion\" not found"; +static const char __pyx_k_Function_cuEventElapsedTime_not[] = "Function \"cuEventElapsedTime\" not found"; +static const char __pyx_k_Function_cuEventQuery_not_found[] = "Function \"cuEventQuery\" not found"; +static const char __pyx_k_Function_cuEventRecordWithFlags[] = "Function \"cuEventRecordWithFlags\" not found"; +static const char __pyx_k_Function_cuEventSynchronize_not[] = "Function \"cuEventSynchronize\" not found"; +static const char __pyx_k_Function_cuFuncGetAttribute_not[] = "Function \"cuFuncGetAttribute\" not found"; +static const char __pyx_k_Function_cuFuncGetParamInfo_not[] = "Function \"cuFuncGetParamInfo\" not found"; +static const char __pyx_k_Function_cuFuncSetAttribute_not[] = "Function \"cuFuncSetAttribute\" not found"; +static const char __pyx_k_Function_cuGraphAddDependencies[] = "Function \"cuGraphAddDependencies\" not found"; +static const char __pyx_k_Function_cuGraphAddHostNode_not[] = "Function \"cuGraphAddHostNode\" not found"; +static const char __pyx_k_Function_cuGraphAddMemAllocNode[] = "Function \"cuGraphAddMemAllocNode\" not found"; +static const char __pyx_k_Function_cuGraphClone_not_found[] = "Function \"cuGraphClone\" not found"; +static const char __pyx_k_Function_cuGraphDestroyNode_not[] = "Function \"cuGraphDestroyNode\" not found"; +static const char __pyx_k_Function_cuGraphExecDestroy_not[] = "Function \"cuGraphExecDestroy\" not found"; +static const char __pyx_k_Function_cuGraphGetEdges_v2_not[] = "Function \"cuGraphGetEdges_v2\" not found"; +static const char __pyx_k_Function_cuGraphNodeFindInClone[] = "Function \"cuGraphNodeFindInClone\" not found"; +static const char __pyx_k_Function_cuGraphNodeGetType_not[] = "Function \"cuGraphNodeGetType\" not found"; +static const char __pyx_k_Function_cuGraphicsMapResources[] = "Function \"cuGraphicsMapResources\" not found"; +static const char __pyx_k_Function_cuGreenCtxStreamCreate[] = "Function \"cuGreenCtxStreamCreate\" not found"; +static const char __pyx_k_Function_cuImportExternalMemory[] = "Function \"cuImportExternalMemory\" not found"; +static const char __pyx_k_Function_cuKernelGetLibrary_not[] = "Function \"cuKernelGetLibrary\" not found"; +static const char __pyx_k_Function_cuKernelSetCacheConfig[] = "Function \"cuKernelSetCacheConfig\" not found"; +static const char __pyx_k_Function_cuLaunchGrid_not_found[] = "Function \"cuLaunchGrid\" not found"; +static const char __pyx_k_Function_cuLibraryGetGlobal_not[] = "Function \"cuLibraryGetGlobal\" not found"; +static const char __pyx_k_Function_cuLibraryGetKernel_not[] = "Function \"cuLibraryGetKernel\" not found"; +static const char __pyx_k_Function_cuLibraryGetModule_not[] = "Function \"cuLibraryGetModule\" not found"; +static const char __pyx_k_Function_cuLogsDumpToMemory_not[] = "Function \"cuLogsDumpToMemory\" not found"; +static const char __pyx_k_Function_cuLogsRegisterCallback[] = "Function \"cuLogsRegisterCallback\" not found"; +static const char __pyx_k_Function_cuMemAllocPitch_v2_not[] = "Function \"cuMemAllocPitch_v2\" not found"; +static const char __pyx_k_Function_cuMemFree_v2_not_found[] = "Function \"cuMemFree_v2\" not found"; +static const char __pyx_k_Function_cuMemMapArrayAsync_not[] = "Function \"cuMemMapArrayAsync\" not found"; +static const char __pyx_k_Function_cuMemPoolExportPointer[] = "Function \"cuMemPoolExportPointer\" not found"; +static const char __pyx_k_Function_cuMemPoolGetAccess_not[] = "Function \"cuMemPoolGetAccess\" not found"; +static const char __pyx_k_Function_cuMemPoolImportPointer[] = "Function \"cuMemPoolImportPointer\" not found"; +static const char __pyx_k_Function_cuMemPoolSetAccess_not[] = "Function \"cuMemPoolSetAccess\" not found"; +static const char __pyx_k_Function_cuMemPrefetchAsync_not[] = "Function \"cuMemPrefetchAsync\" not found"; +static const char __pyx_k_Function_cuMemRangeGetAttribute[] = "Function \"cuMemRangeGetAttribute\" not found"; +static const char __pyx_k_Function_cuMemRelease_not_found[] = "Function \"cuMemRelease\" not found"; +static const char __pyx_k_Function_cuMemcpy2DAsync_v2_not[] = "Function \"cuMemcpy2DAsync_v2\" not found"; +static const char __pyx_k_Function_cuMemcpy2DUnaligned_v2[] = "Function \"cuMemcpy2DUnaligned_v2\" not found"; +static const char __pyx_k_Function_cuMemcpy3DAsync_v2_not[] = "Function \"cuMemcpy3DAsync_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyBatchAsync_not[] = "Function \"cuMemcpyBatchAsync\" not found"; +static const char __pyx_k_Function_cuMemcpyPeer_not_found[] = "Function \"cuMemcpyPeer\" not found"; +static const char __pyx_k_Function_cuMemsetD2D16Async_not[] = "Function \"cuMemsetD2D16Async\" not found"; +static const char __pyx_k_Function_cuMemsetD2D32Async_not[] = "Function \"cuMemsetD2D32Async\" not found"; +static const char __pyx_k_Function_cuMipmappedArrayCreate[] = "Function \"cuMipmappedArrayCreate\" not found"; +static const char __pyx_k_Function_cuModuleGetLoadingMode[] = "Function \"cuModuleGetLoadingMode\" not found"; +static const char __pyx_k_Function_cuModuleGetSurfRef_not[] = "Function \"cuModuleGetSurfRef\" not found"; +static const char __pyx_k_Function_cuModuleLoadDataEx_not[] = "Function \"cuModuleLoadDataEx\" not found"; +static const char __pyx_k_Function_cuModuleLoad_not_found[] = "Function \"cuModuleLoad\" not found"; +static const char __pyx_k_Function_cuMulticastBindMem_not[] = "Function \"cuMulticastBindMem\" not found"; +static const char __pyx_k_Function_cuPointerGetAttributes[] = "Function \"cuPointerGetAttributes\" not found"; +static const char __pyx_k_Function_cuStreamAttachMemAsync[] = "Function \"cuStreamAttachMemAsync\" not found"; +static const char __pyx_k_Function_cuStreamCopyAttributes[] = "Function \"cuStreamCopyAttributes\" not found"; +static const char __pyx_k_Function_cuStreamDestroy_v2_not[] = "Function \"cuStreamDestroy_v2\" not found"; +static const char __pyx_k_Function_cuStreamEndCapture_not[] = "Function \"cuStreamEndCapture\" not found"; +static const char __pyx_k_Function_cuStreamGetCaptureInfo[] = "Function \"cuStreamGetCaptureInfo_v2\" not found"; +static const char __pyx_k_Function_cuStreamWaitValue32_v2[] = "Function \"cuStreamWaitValue32_v2\" not found"; +static const char __pyx_k_Function_cuStreamWaitValue64_v2[] = "Function \"cuStreamWaitValue64_v2\" not found"; +static const char __pyx_k_Function_cuSurfObjectCreate_not[] = "Function \"cuSurfObjectCreate\" not found"; +static const char __pyx_k_Function_cuTensorMapEncodeTiled[] = "Function \"cuTensorMapEncodeTiled\" not found"; +static const char __pyx_k_Function_cuTexObjectDestroy_not[] = "Function \"cuTexObjectDestroy\" not found"; +static const char __pyx_k_Function_cuTexRefGetAddressMode[] = "Function \"cuTexRefGetAddressMode\" not found"; +static const char __pyx_k_Function_cuTexRefGetBorderColor[] = "Function \"cuTexRefGetBorderColor\" not found"; +static const char __pyx_k_Function_cuTexRefSetAddressMode[] = "Function \"cuTexRefSetAddressMode\" not found"; +static const char __pyx_k_Function_cuTexRefSetBorderColor[] = "Function \"cuTexRefSetBorderColor\" not found"; +static const char __pyx_k_Function_cuUserObjectCreate_not[] = "Function \"cuUserObjectCreate\" not found"; +static const char __pyx_k_Function_cuUserObjectRetain_not[] = "Function \"cuUserObjectRetain\" not found"; +static const char __pyx_k_CUDA_PYTHON_CUDA_PER_THREAD_DEFA[] = "CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM"; +static const char __pyx_k_Function_cuArrayCreate_v2_not_fo[] = "Function \"cuArrayCreate_v2\" not found"; +static const char __pyx_k_Function_cuArrayDestroy_not_foun[] = "Function \"cuArrayDestroy\" not found"; +static const char __pyx_k_Function_cuArrayGetDescriptor_v2[] = "Function \"cuArrayGetDescriptor_v2\" not found"; +static const char __pyx_k_Function_cuArrayGetMemoryRequire[] = "Function \"cuArrayGetMemoryRequirements\" not found"; +static const char __pyx_k_Function_cuArrayGetPlane_not_fou[] = "Function \"cuArrayGetPlane\" not found"; +static const char __pyx_k_Function_cuArrayGetSparsePropert[] = "Function \"cuArrayGetSparseProperties\" not found"; +static const char __pyx_k_Function_cuCheckpointProcessChec[] = "Function \"cuCheckpointProcessCheckpoint\" not found"; +static const char __pyx_k_Function_cuCheckpointProcessGetR[] = "Function \"cuCheckpointProcessGetRestoreThreadId\" not found"; +static const char __pyx_k_Function_cuCheckpointProcessGetS[] = "Function \"cuCheckpointProcessGetState\" not found"; +static const char __pyx_k_Function_cuCheckpointProcessLock[] = "Function \"cuCheckpointProcessLock\" not found"; +static const char __pyx_k_Function_cuCheckpointProcessRest[] = "Function \"cuCheckpointProcessRestore\" not found"; +static const char __pyx_k_Function_cuCheckpointProcessUnlo[] = "Function \"cuCheckpointProcessUnlock\" not found"; +static const char __pyx_k_Function_cuCoredumpGetAttributeG[] = "Function \"cuCoredumpGetAttributeGlobal\" not found"; +static const char __pyx_k_Function_cuCoredumpSetAttributeG[] = "Function \"cuCoredumpSetAttributeGlobal\" not found"; +static const char __pyx_k_Function_cuCtxCreate_v2_not_foun[] = "Function \"cuCtxCreate_v2\" not found"; +static const char __pyx_k_Function_cuCtxCreate_v3_not_foun[] = "Function \"cuCtxCreate_v3\" not found"; +static const char __pyx_k_Function_cuCtxCreate_v4_not_foun[] = "Function \"cuCtxCreate_v4\" not found"; +static const char __pyx_k_Function_cuCtxDestroy_v2_not_fou[] = "Function \"cuCtxDestroy_v2\" not found"; +static const char __pyx_k_Function_cuCtxEnablePeerAccess_n[] = "Function \"cuCtxEnablePeerAccess\" not found"; +static const char __pyx_k_Function_cuCtxFromGreenCtx_not_f[] = "Function \"cuCtxFromGreenCtx\" not found"; +static const char __pyx_k_Function_cuCtxGetCacheConfig_not[] = "Function \"cuCtxGetCacheConfig\" not found"; +static const char __pyx_k_Function_cuCtxGetCurrent_not_fou[] = "Function \"cuCtxGetCurrent\" not found"; +static const char __pyx_k_Function_cuCtxGetDevResource_not[] = "Function \"cuCtxGetDevResource\" not found"; +static const char __pyx_k_Function_cuCtxGetDevice_not_foun[] = "Function \"cuCtxGetDevice\" not found"; +static const char __pyx_k_Function_cuCtxGetExecAffinity_no[] = "Function \"cuCtxGetExecAffinity\" not found"; +static const char __pyx_k_Function_cuCtxGetFlags_not_found[] = "Function \"cuCtxGetFlags\" not found"; +static const char __pyx_k_Function_cuCtxGetLimit_not_found[] = "Function \"cuCtxGetLimit\" not found"; +static const char __pyx_k_Function_cuCtxGetSharedMemConfig[] = "Function \"cuCtxGetSharedMemConfig\" not found"; +static const char __pyx_k_Function_cuCtxGetStreamPriorityR[] = "Function \"cuCtxGetStreamPriorityRange\" not found"; +static const char __pyx_k_Function_cuCtxPushCurrent_v2_not[] = "Function \"cuCtxPushCurrent_v2\" not found"; +static const char __pyx_k_Function_cuCtxRecordEvent_not_fo[] = "Function \"cuCtxRecordEvent\" not found"; +static const char __pyx_k_Function_cuCtxResetPersistingL2C[] = "Function \"cuCtxResetPersistingL2Cache\" not found"; +static const char __pyx_k_Function_cuCtxSetCacheConfig_not[] = "Function \"cuCtxSetCacheConfig\" not found"; +static const char __pyx_k_Function_cuCtxSetCurrent_not_fou[] = "Function \"cuCtxSetCurrent\" not found"; +static const char __pyx_k_Function_cuCtxSetFlags_not_found[] = "Function \"cuCtxSetFlags\" not found"; +static const char __pyx_k_Function_cuCtxSetLimit_not_found[] = "Function \"cuCtxSetLimit\" not found"; +static const char __pyx_k_Function_cuCtxSetSharedMemConfig[] = "Function \"cuCtxSetSharedMemConfig\" not found"; +static const char __pyx_k_Function_cuCtxSynchronize_not_fo[] = "Function \"cuCtxSynchronize\" not found"; +static const char __pyx_k_Function_cuCtxWaitEvent_not_foun[] = "Function \"cuCtxWaitEvent\" not found"; +static const char __pyx_k_Function_cuDestroyExternalMemory[] = "Function \"cuDestroyExternalMemory\" not found"; +static const char __pyx_k_Function_cuDestroyExternalSemaph[] = "Function \"cuDestroyExternalSemaphore\" not found"; +static const char __pyx_k_Function_cuDevResourceGenerateDe[] = "Function \"cuDevResourceGenerateDesc\" not found"; +static const char __pyx_k_Function_cuDevSmResourceSplitByC[] = "Function \"cuDevSmResourceSplitByCount\" not found"; +static const char __pyx_k_Function_cuDeviceCanAccessPeer_n[] = "Function \"cuDeviceCanAccessPeer\" not found"; +static const char __pyx_k_Function_cuDeviceComputeCapabili[] = "Function \"cuDeviceComputeCapability\" not found"; +static const char __pyx_k_Function_cuDeviceGetAttribute_no[] = "Function \"cuDeviceGetAttribute\" not found"; +static const char __pyx_k_Function_cuDeviceGetByPCIBusId_n[] = "Function \"cuDeviceGetByPCIBusId\" not found"; +static const char __pyx_k_Function_cuDeviceGetCount_not_fo[] = "Function \"cuDeviceGetCount\" not found"; +static const char __pyx_k_Function_cuDeviceGetDefaultMemPo[] = "Function \"cuDeviceGetDefaultMemPool\" not found"; +static const char __pyx_k_Function_cuDeviceGetExecAffinity[] = "Function \"cuDeviceGetExecAffinitySupport\" not found"; +static const char __pyx_k_Function_cuDeviceGetGraphMemAttr[] = "Function \"cuDeviceGetGraphMemAttribute\" not found"; +static const char __pyx_k_Function_cuDeviceGetLuid_not_fou[] = "Function \"cuDeviceGetLuid\" not found"; +static const char __pyx_k_Function_cuDeviceGetName_not_fou[] = "Function \"cuDeviceGetName\" not found"; +static const char __pyx_k_Function_cuDeviceGetNvSciSyncAtt[] = "Function \"cuDeviceGetNvSciSyncAttributes\" not found"; +static const char __pyx_k_Function_cuDeviceGetP2PAttribute[] = "Function \"cuDeviceGetP2PAttribute\" not found"; +static const char __pyx_k_Function_cuDeviceGetPCIBusId_not[] = "Function \"cuDeviceGetPCIBusId\" not found"; +static const char __pyx_k_Function_cuDeviceGetProperties_n[] = "Function \"cuDeviceGetProperties\" not found"; +static const char __pyx_k_Function_cuDeviceGetTexture1DLin[] = "Function \"cuDeviceGetTexture1DLinearMaxWidth\" not found"; +static const char __pyx_k_Function_cuDeviceGetUuid_not_fou[] = "Function \"cuDeviceGetUuid\" not found"; +static const char __pyx_k_Function_cuDeviceGraphMemTrim_no[] = "Function \"cuDeviceGraphMemTrim\" not found"; +static const char __pyx_k_Function_cuDevicePrimaryCtxGetSt[] = "Function \"cuDevicePrimaryCtxGetState\" not found"; +static const char __pyx_k_Function_cuDevicePrimaryCtxRelea[] = "Function \"cuDevicePrimaryCtxRelease_v2\" not found"; +static const char __pyx_k_Function_cuDevicePrimaryCtxReset[] = "Function \"cuDevicePrimaryCtxReset_v2\" not found"; +static const char __pyx_k_Function_cuDevicePrimaryCtxRetai[] = "Function \"cuDevicePrimaryCtxRetain\" not found"; +static const char __pyx_k_Function_cuDevicePrimaryCtxSetFl[] = "Function \"cuDevicePrimaryCtxSetFlags_v2\" not found"; +static const char __pyx_k_Function_cuDeviceRegisterAsyncNo[] = "Function \"cuDeviceRegisterAsyncNotification\" not found"; +static const char __pyx_k_Function_cuDeviceSetGraphMemAttr[] = "Function \"cuDeviceSetGraphMemAttribute\" not found"; +static const char __pyx_k_Function_cuDeviceTotalMem_v2_not[] = "Function \"cuDeviceTotalMem_v2\" not found"; +static const char __pyx_k_Function_cuDeviceUnregisterAsync[] = "Function \"cuDeviceUnregisterAsyncNotification\" not found"; +static const char __pyx_k_Function_cuEGLStreamConsumerAcqu[] = "Function \"cuEGLStreamConsumerAcquireFrame\" not found"; +static const char __pyx_k_Function_cuEGLStreamConsumerConn[] = "Function \"cuEGLStreamConsumerConnect\" not found"; +static const char __pyx_k_Function_cuEGLStreamConsumerDisc[] = "Function \"cuEGLStreamConsumerDisconnect\" not found"; +static const char __pyx_k_Function_cuEGLStreamConsumerRele[] = "Function \"cuEGLStreamConsumerReleaseFrame\" not found"; +static const char __pyx_k_Function_cuEGLStreamProducerConn[] = "Function \"cuEGLStreamProducerConnect\" not found"; +static const char __pyx_k_Function_cuEGLStreamProducerDisc[] = "Function \"cuEGLStreamProducerDisconnect\" not found"; +static const char __pyx_k_Function_cuEGLStreamProducerPres[] = "Function \"cuEGLStreamProducerPresentFrame\" not found"; +static const char __pyx_k_Function_cuEGLStreamProducerRetu[] = "Function \"cuEGLStreamProducerReturnFrame\" not found"; +static const char __pyx_k_Function_cuEventCreateFromEGLSyn[] = "Function \"cuEventCreateFromEGLSync\" not found"; +static const char __pyx_k_Function_cuEventCreate_not_found[] = "Function \"cuEventCreate\" not found"; +static const char __pyx_k_Function_cuEventDestroy_v2_not_f[] = "Function \"cuEventDestroy_v2\" not found"; +static const char __pyx_k_Function_cuEventElapsedTime_v2_n[] = "Function \"cuEventElapsedTime_v2\" not found"; +static const char __pyx_k_Function_cuEventRecord_not_found[] = "Function \"cuEventRecord\" not found"; +static const char __pyx_k_Function_cuExternalMemoryGetMapp[] = "Function \"cuExternalMemoryGetMappedBuffer\" not found"; +static const char __pyx_k_Function_cuFlushGPUDirectRDMAWri[] = "Function \"cuFlushGPUDirectRDMAWrites\" not found"; +static const char __pyx_k_Function_cuFuncGetModule_not_fou[] = "Function \"cuFuncGetModule\" not found"; +static const char __pyx_k_Function_cuFuncGetName_not_found[] = "Function \"cuFuncGetName\" not found"; +static const char __pyx_k_Function_cuFuncIsLoaded_not_foun[] = "Function \"cuFuncIsLoaded\" not found"; +static const char __pyx_k_Function_cuFuncSetBlockShape_not[] = "Function \"cuFuncSetBlockShape\" not found"; +static const char __pyx_k_Function_cuFuncSetCacheConfig_no[] = "Function \"cuFuncSetCacheConfig\" not found"; +static const char __pyx_k_Function_cuFuncSetSharedMemConfi[] = "Function \"cuFuncSetSharedMemConfig\" not found"; +static const char __pyx_k_Function_cuFuncSetSharedSize_not[] = "Function \"cuFuncSetSharedSize\" not found"; +static const char __pyx_k_Function_cuGLGetDevices_v2_not_f[] = "Function \"cuGLGetDevices_v2\" not found"; +static const char __pyx_k_Function_cuGetErrorName_not_foun[] = "Function \"cuGetErrorName\" not found"; +static const char __pyx_k_Function_cuGetErrorString_not_fo[] = "Function \"cuGetErrorString\" not found"; +static const char __pyx_k_Function_cuGetExportTable_not_fo[] = "Function \"cuGetExportTable\" not found"; +static const char __pyx_k_Function_cuGetProcAddress_v2_not[] = "Function \"cuGetProcAddress_v2\" not found"; +static const char __pyx_k_Function_cuGraphAddBatchMemOpNod[] = "Function \"cuGraphAddBatchMemOpNode\" not found"; +static const char __pyx_k_Function_cuGraphAddChildGraphNod[] = "Function \"cuGraphAddChildGraphNode\" not found"; +static const char __pyx_k_Function_cuGraphAddEmptyNode_not[] = "Function \"cuGraphAddEmptyNode\" not found"; +static const char __pyx_k_Function_cuGraphAddEventRecordNo[] = "Function \"cuGraphAddEventRecordNode\" not found"; +static const char __pyx_k_Function_cuGraphAddEventWaitNode[] = "Function \"cuGraphAddEventWaitNode\" not found"; +static const char __pyx_k_Function_cuGraphAddExternalSemap[] = "Function \"cuGraphAddExternalSemaphoresSignalNode\" not found"; +static const char __pyx_k_Function_cuGraphAddKernelNode_v2[] = "Function \"cuGraphAddKernelNode_v2\" not found"; +static const char __pyx_k_Function_cuGraphAddMemFreeNode_n[] = "Function \"cuGraphAddMemFreeNode\" not found"; +static const char __pyx_k_Function_cuGraphAddMemcpyNode_no[] = "Function \"cuGraphAddMemcpyNode\" not found"; +static const char __pyx_k_Function_cuGraphAddMemsetNode_no[] = "Function \"cuGraphAddMemsetNode\" not found"; +static const char __pyx_k_Function_cuGraphAddNode_not_foun[] = "Function \"cuGraphAddNode\" not found"; +static const char __pyx_k_Function_cuGraphAddNode_v2_not_f[] = "Function \"cuGraphAddNode_v2\" not found"; +static const char __pyx_k_Function_cuGraphBatchMemOpNodeGe[] = "Function \"cuGraphBatchMemOpNodeGetParams\" not found"; +static const char __pyx_k_Function_cuGraphBatchMemOpNodeSe[] = "Function \"cuGraphBatchMemOpNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphChildGraphNodeGe[] = "Function \"cuGraphChildGraphNodeGetGraph\" not found"; +static const char __pyx_k_Function_cuGraphConditionalHandl[] = "Function \"cuGraphConditionalHandleCreate\" not found"; +static const char __pyx_k_Function_cuGraphCreate_not_found[] = "Function \"cuGraphCreate\" not found"; +static const char __pyx_k_Function_cuGraphDebugDotPrint_no[] = "Function \"cuGraphDebugDotPrint\" not found"; +static const char __pyx_k_Function_cuGraphDestroy_not_foun[] = "Function \"cuGraphDestroy\" not found"; +static const char __pyx_k_Function_cuGraphEventRecordNodeG[] = "Function \"cuGraphEventRecordNodeGetEvent\" not found"; +static const char __pyx_k_Function_cuGraphEventRecordNodeS[] = "Function \"cuGraphEventRecordNodeSetEvent\" not found"; +static const char __pyx_k_Function_cuGraphEventWaitNodeGet[] = "Function \"cuGraphEventWaitNodeGetEvent\" not found"; +static const char __pyx_k_Function_cuGraphEventWaitNodeSet[] = "Function \"cuGraphEventWaitNodeSetEvent\" not found"; +static const char __pyx_k_Function_cuGraphExecBatchMemOpNo[] = "Function \"cuGraphExecBatchMemOpNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphExecChildGraphNo[] = "Function \"cuGraphExecChildGraphNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphExecEventRecordN[] = "Function \"cuGraphExecEventRecordNodeSetEvent\" not found"; +static const char __pyx_k_Function_cuGraphExecEventWaitNod[] = "Function \"cuGraphExecEventWaitNodeSetEvent\" not found"; +static const char __pyx_k_Function_cuGraphExecExternalSema[] = "Function \"cuGraphExecExternalSemaphoresSignalNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphExecGetFlags_not[] = "Function \"cuGraphExecGetFlags\" not found"; +static const char __pyx_k_Function_cuGraphExecHostNodeSetP[] = "Function \"cuGraphExecHostNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphExecKernelNodeSe[] = "Function \"cuGraphExecKernelNodeSetParams_v2\" not found"; +static const char __pyx_k_Function_cuGraphExecMemcpyNodeSe[] = "Function \"cuGraphExecMemcpyNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphExecMemsetNodeSe[] = "Function \"cuGraphExecMemsetNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphExecNodeSetParam[] = "Function \"cuGraphExecNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphExecUpdate_v2_no[] = "Function \"cuGraphExecUpdate_v2\" not found"; +static const char __pyx_k_Function_cuGraphExternalSemaphor[] = "Function \"cuGraphExternalSemaphoresSignalNodeGetParams\" not found"; +static const char __pyx_k_Function_cuGraphGetEdges_not_fou[] = "Function \"cuGraphGetEdges\" not found"; +static const char __pyx_k_Function_cuGraphGetNodes_not_fou[] = "Function \"cuGraphGetNodes\" not found"; +static const char __pyx_k_Function_cuGraphGetRootNodes_not[] = "Function \"cuGraphGetRootNodes\" not found"; +static const char __pyx_k_Function_cuGraphHostNodeGetParam[] = "Function \"cuGraphHostNodeGetParams\" not found"; +static const char __pyx_k_Function_cuGraphHostNodeSetParam[] = "Function \"cuGraphHostNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphInstantiateWithF[] = "Function \"cuGraphInstantiateWithFlags\" not found"; +static const char __pyx_k_Function_cuGraphInstantiateWithP[] = "Function \"cuGraphInstantiateWithParams\" not found"; +static const char __pyx_k_Function_cuGraphKernelNodeCopyAt[] = "Function \"cuGraphKernelNodeCopyAttributes\" not found"; +static const char __pyx_k_Function_cuGraphKernelNodeGetAtt[] = "Function \"cuGraphKernelNodeGetAttribute\" not found"; +static const char __pyx_k_Function_cuGraphKernelNodeGetPar[] = "Function \"cuGraphKernelNodeGetParams_v2\" not found"; +static const char __pyx_k_Function_cuGraphKernelNodeSetAtt[] = "Function \"cuGraphKernelNodeSetAttribute\" not found"; +static const char __pyx_k_Function_cuGraphKernelNodeSetPar[] = "Function \"cuGraphKernelNodeSetParams_v2\" not found"; +static const char __pyx_k_Function_cuGraphLaunch_not_found[] = "Function \"cuGraphLaunch\" not found"; +static const char __pyx_k_Function_cuGraphMemAllocNodeGetP[] = "Function \"cuGraphMemAllocNodeGetParams\" not found"; +static const char __pyx_k_Function_cuGraphMemFreeNodeGetPa[] = "Function \"cuGraphMemFreeNodeGetParams\" not found"; +static const char __pyx_k_Function_cuGraphMemcpyNodeGetPar[] = "Function \"cuGraphMemcpyNodeGetParams\" not found"; +static const char __pyx_k_Function_cuGraphMemcpyNodeSetPar[] = "Function \"cuGraphMemcpyNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphMemsetNodeGetPar[] = "Function \"cuGraphMemsetNodeGetParams\" not found"; +static const char __pyx_k_Function_cuGraphMemsetNodeSetPar[] = "Function \"cuGraphMemsetNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphNodeGetDependenc[] = "Function \"cuGraphNodeGetDependencies\" not found"; +static const char __pyx_k_Function_cuGraphNodeGetDependent[] = "Function \"cuGraphNodeGetDependentNodes\" not found"; +static const char __pyx_k_Function_cuGraphNodeGetEnabled_n[] = "Function \"cuGraphNodeGetEnabled\" not found"; +static const char __pyx_k_Function_cuGraphNodeSetEnabled_n[] = "Function \"cuGraphNodeSetEnabled\" not found"; +static const char __pyx_k_Function_cuGraphNodeSetParams_no[] = "Function \"cuGraphNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphReleaseUserObjec[] = "Function \"cuGraphReleaseUserObject\" not found"; +static const char __pyx_k_Function_cuGraphRemoveDependenci[] = "Function \"cuGraphRemoveDependencies\" not found"; +static const char __pyx_k_Function_cuGraphRetainUserObject[] = "Function \"cuGraphRetainUserObject\" not found"; +static const char __pyx_k_Function_cuGraphUpload_not_found[] = "Function \"cuGraphUpload\" not found"; +static const char __pyx_k_Function_cuGraphicsEGLRegisterIm[] = "Function \"cuGraphicsEGLRegisterImage\" not found"; +static const char __pyx_k_Function_cuGraphicsGLRegisterBuf[] = "Function \"cuGraphicsGLRegisterBuffer\" not found"; +static const char __pyx_k_Function_cuGraphicsGLRegisterIma[] = "Function \"cuGraphicsGLRegisterImage\" not found"; +static const char __pyx_k_Function_cuGraphicsResourceGetMa[] = "Function \"cuGraphicsResourceGetMappedMipmappedArray\" not found"; +static const char __pyx_k_Function_cuGraphicsResourceSetMa[] = "Function \"cuGraphicsResourceSetMapFlags_v2\" not found"; +static const char __pyx_k_Function_cuGraphicsSubResourceGe[] = "Function \"cuGraphicsSubResourceGetMappedArray\" not found"; +static const char __pyx_k_Function_cuGraphicsUnmapResource[] = "Function \"cuGraphicsUnmapResources\" not found"; +static const char __pyx_k_Function_cuGraphicsUnregisterRes[] = "Function \"cuGraphicsUnregisterResource\" not found"; +static const char __pyx_k_Function_cuGraphicsVDPAURegister[] = "Function \"cuGraphicsVDPAURegisterVideoSurface\" not found"; +static const char __pyx_k_Function_cuGreenCtxCreate_not_fo[] = "Function \"cuGreenCtxCreate\" not found"; +static const char __pyx_k_Function_cuGreenCtxDestroy_not_f[] = "Function \"cuGreenCtxDestroy\" not found"; +static const char __pyx_k_Function_cuGreenCtxGetDevResourc[] = "Function \"cuGreenCtxGetDevResource\" not found"; +static const char __pyx_k_Function_cuGreenCtxRecordEvent_n[] = "Function \"cuGreenCtxRecordEvent\" not found"; +static const char __pyx_k_Function_cuGreenCtxWaitEvent_not[] = "Function \"cuGreenCtxWaitEvent\" not found"; +static const char __pyx_k_Function_cuImportExternalSemapho[] = "Function \"cuImportExternalSemaphore\" not found"; +static const char __pyx_k_Function_cuIpcCloseMemHandle_not[] = "Function \"cuIpcCloseMemHandle\" not found"; +static const char __pyx_k_Function_cuIpcGetEventHandle_not[] = "Function \"cuIpcGetEventHandle\" not found"; +static const char __pyx_k_Function_cuIpcGetMemHandle_not_f[] = "Function \"cuIpcGetMemHandle\" not found"; +static const char __pyx_k_Function_cuIpcOpenEventHandle_no[] = "Function \"cuIpcOpenEventHandle\" not found"; +static const char __pyx_k_Function_cuIpcOpenMemHandle_v2_n[] = "Function \"cuIpcOpenMemHandle_v2\" not found"; +static const char __pyx_k_Function_cuKernelGetAttribute_no[] = "Function \"cuKernelGetAttribute\" not found"; +static const char __pyx_k_Function_cuKernelGetFunction_not[] = "Function \"cuKernelGetFunction\" not found"; +static const char __pyx_k_Function_cuKernelGetName_not_fou[] = "Function \"cuKernelGetName\" not found"; +static const char __pyx_k_Function_cuKernelGetParamInfo_no[] = "Function \"cuKernelGetParamInfo\" not found"; +static const char __pyx_k_Function_cuKernelSetAttribute_no[] = "Function \"cuKernelSetAttribute\" not found"; +static const char __pyx_k_Function_cuLaunchCooperativeKern[] = "Function \"cuLaunchCooperativeKernel\" not found"; +static const char __pyx_k_Function_cuLaunchGridAsync_not_f[] = "Function \"cuLaunchGridAsync\" not found"; +static const char __pyx_k_Function_cuLaunchHostFunc_not_fo[] = "Function \"cuLaunchHostFunc\" not found"; +static const char __pyx_k_Function_cuLaunchKernelEx_not_fo[] = "Function \"cuLaunchKernelEx\" not found"; +static const char __pyx_k_Function_cuLaunchKernel_not_foun[] = "Function \"cuLaunchKernel\" not found"; +static const char __pyx_k_Function_cuLibraryEnumerateKerne[] = "Function \"cuLibraryEnumerateKernels\" not found"; +static const char __pyx_k_Function_cuLibraryGetKernelCount[] = "Function \"cuLibraryGetKernelCount\" not found"; +static const char __pyx_k_Function_cuLibraryGetManaged_not[] = "Function \"cuLibraryGetManaged\" not found"; +static const char __pyx_k_Function_cuLibraryGetUnifiedFunc[] = "Function \"cuLibraryGetUnifiedFunction\" not found"; +static const char __pyx_k_Function_cuLibraryLoadData_not_f[] = "Function \"cuLibraryLoadData\" not found"; +static const char __pyx_k_Function_cuLibraryLoadFromFile_n[] = "Function \"cuLibraryLoadFromFile\" not found"; +static const char __pyx_k_Function_cuLibraryUnload_not_fou[] = "Function \"cuLibraryUnload\" not found"; +static const char __pyx_k_Function_cuLinkAddData_v2_not_fo[] = "Function \"cuLinkAddData_v2\" not found"; +static const char __pyx_k_Function_cuLinkAddFile_v2_not_fo[] = "Function \"cuLinkAddFile_v2\" not found"; +static const char __pyx_k_Function_cuLinkComplete_not_foun[] = "Function \"cuLinkComplete\" not found"; +static const char __pyx_k_Function_cuLinkCreate_v2_not_fou[] = "Function \"cuLinkCreate_v2\" not found"; +static const char __pyx_k_Function_cuLinkDestroy_not_found[] = "Function \"cuLinkDestroy\" not found"; +static const char __pyx_k_Function_cuLogsCurrent_not_found[] = "Function \"cuLogsCurrent\" not found"; +static const char __pyx_k_Function_cuLogsDumpToFile_not_fo[] = "Function \"cuLogsDumpToFile\" not found"; +static const char __pyx_k_Function_cuLogsUnregisterCallbac[] = "Function \"cuLogsUnregisterCallback\" not found"; +static const char __pyx_k_Function_cuMemAddressFree_not_fo[] = "Function \"cuMemAddressFree\" not found"; +static const char __pyx_k_Function_cuMemAddressReserve_not[] = "Function \"cuMemAddressReserve\" not found"; +static const char __pyx_k_Function_cuMemAdvise_v2_not_foun[] = "Function \"cuMemAdvise_v2\" not found"; +static const char __pyx_k_Function_cuMemAllocAsync_not_fou[] = "Function \"cuMemAllocAsync\" not found"; +static const char __pyx_k_Function_cuMemAllocFromPoolAsync[] = "Function \"cuMemAllocFromPoolAsync\" not found"; +static const char __pyx_k_Function_cuMemAllocHost_v2_not_f[] = "Function \"cuMemAllocHost_v2\" not found"; +static const char __pyx_k_Function_cuMemAllocManaged_not_f[] = "Function \"cuMemAllocManaged\" not found"; +static const char __pyx_k_Function_cuMemAlloc_v2_not_found[] = "Function \"cuMemAlloc_v2\" not found"; +static const char __pyx_k_Function_cuMemBatchDecompressAsy[] = "Function \"cuMemBatchDecompressAsync\" not found"; +static const char __pyx_k_Function_cuMemExportToShareableH[] = "Function \"cuMemExportToShareableHandle\" not found"; +static const char __pyx_k_Function_cuMemFreeAsync_not_foun[] = "Function \"cuMemFreeAsync\" not found"; +static const char __pyx_k_Function_cuMemFreeHost_not_found[] = "Function \"cuMemFreeHost\" not found"; +static const char __pyx_k_Function_cuMemGetAccess_not_foun[] = "Function \"cuMemGetAccess\" not found"; +static const char __pyx_k_Function_cuMemGetAddressRange_v2[] = "Function \"cuMemGetAddressRange_v2\" not found"; +static const char __pyx_k_Function_cuMemGetAllocationGranu[] = "Function \"cuMemGetAllocationGranularity\" not found"; +static const char __pyx_k_Function_cuMemGetAllocationPrope[] = "Function \"cuMemGetAllocationPropertiesFromHandle\" not found"; +static const char __pyx_k_Function_cuMemGetHandleForAddres[] = "Function \"cuMemGetHandleForAddressRange\" not found"; +static const char __pyx_k_Function_cuMemGetInfo_v2_not_fou[] = "Function \"cuMemGetInfo_v2\" not found"; +static const char __pyx_k_Function_cuMemHostAlloc_not_foun[] = "Function \"cuMemHostAlloc\" not found"; +static const char __pyx_k_Function_cuMemHostGetDevicePoint[] = "Function \"cuMemHostGetDevicePointer_v2\" not found"; +static const char __pyx_k_Function_cuMemHostGetFlags_not_f[] = "Function \"cuMemHostGetFlags\" not found"; +static const char __pyx_k_Function_cuMemHostRegister_v2_no[] = "Function \"cuMemHostRegister_v2\" not found"; +static const char __pyx_k_Function_cuMemHostUnregister_not[] = "Function \"cuMemHostUnregister\" not found"; +static const char __pyx_k_Function_cuMemImportFromShareabl[] = "Function \"cuMemImportFromShareableHandle\" not found"; +static const char __pyx_k_Function_cuMemPoolCreate_not_fou[] = "Function \"cuMemPoolCreate\" not found"; +static const char __pyx_k_Function_cuMemPoolDestroy_not_fo[] = "Function \"cuMemPoolDestroy\" not found"; +static const char __pyx_k_Function_cuMemPoolExportToSharea[] = "Function \"cuMemPoolExportToShareableHandle\" not found"; +static const char __pyx_k_Function_cuMemPoolGetAttribute_n[] = "Function \"cuMemPoolGetAttribute\" not found"; +static const char __pyx_k_Function_cuMemPoolImportFromShar[] = "Function \"cuMemPoolImportFromShareableHandle\" not found"; +static const char __pyx_k_Function_cuMemPoolSetAttribute_n[] = "Function \"cuMemPoolSetAttribute\" not found"; +static const char __pyx_k_Function_cuMemPoolTrimTo_not_fou[] = "Function \"cuMemPoolTrimTo\" not found"; +static const char __pyx_k_Function_cuMemPrefetchAsync_v2_n[] = "Function \"cuMemPrefetchAsync_v2\" not found"; +static const char __pyx_k_Function_cuMemRangeGetAttributes[] = "Function \"cuMemRangeGetAttributes\" not found"; +static const char __pyx_k_Function_cuMemRetainAllocationHa[] = "Function \"cuMemRetainAllocationHandle\" not found"; +static const char __pyx_k_Function_cuMemSetAccess_not_foun[] = "Function \"cuMemSetAccess\" not found"; +static const char __pyx_k_Function_cuMemcpy2D_v2_not_found[] = "Function \"cuMemcpy2D_v2\" not found"; +static const char __pyx_k_Function_cuMemcpy3DBatchAsync_no[] = "Function \"cuMemcpy3DBatchAsync\" not found"; +static const char __pyx_k_Function_cuMemcpy3DPeerAsync_not[] = "Function \"cuMemcpy3DPeerAsync\" not found"; +static const char __pyx_k_Function_cuMemcpy3DPeer_not_foun[] = "Function \"cuMemcpy3DPeer\" not found"; +static const char __pyx_k_Function_cuMemcpy3D_v2_not_found[] = "Function \"cuMemcpy3D_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyAsync_not_found[] = "Function \"cuMemcpyAsync\" not found"; +static const char __pyx_k_Function_cuMemcpyAtoA_v2_not_fou[] = "Function \"cuMemcpyAtoA_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyAtoD_v2_not_fou[] = "Function \"cuMemcpyAtoD_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyAtoHAsync_v2_no[] = "Function \"cuMemcpyAtoHAsync_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyAtoH_v2_not_fou[] = "Function \"cuMemcpyAtoH_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyDtoA_v2_not_fou[] = "Function \"cuMemcpyDtoA_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyDtoDAsync_v2_no[] = "Function \"cuMemcpyDtoDAsync_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyDtoD_v2_not_fou[] = "Function \"cuMemcpyDtoD_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyDtoHAsync_v2_no[] = "Function \"cuMemcpyDtoHAsync_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyDtoH_v2_not_fou[] = "Function \"cuMemcpyDtoH_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyHtoAAsync_v2_no[] = "Function \"cuMemcpyHtoAAsync_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyHtoA_v2_not_fou[] = "Function \"cuMemcpyHtoA_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyHtoDAsync_v2_no[] = "Function \"cuMemcpyHtoDAsync_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyHtoD_v2_not_fou[] = "Function \"cuMemcpyHtoD_v2\" not found"; +static const char __pyx_k_Function_cuMemcpyPeerAsync_not_f[] = "Function \"cuMemcpyPeerAsync\" not found"; +static const char __pyx_k_Function_cuMemsetD16Async_not_fo[] = "Function \"cuMemsetD16Async\" not found"; +static const char __pyx_k_Function_cuMemsetD16_v2_not_foun[] = "Function \"cuMemsetD16_v2\" not found"; +static const char __pyx_k_Function_cuMemsetD2D16_v2_not_fo[] = "Function \"cuMemsetD2D16_v2\" not found"; +static const char __pyx_k_Function_cuMemsetD2D32_v2_not_fo[] = "Function \"cuMemsetD2D32_v2\" not found"; +static const char __pyx_k_Function_cuMemsetD2D8Async_not_f[] = "Function \"cuMemsetD2D8Async\" not found"; +static const char __pyx_k_Function_cuMemsetD2D8_v2_not_fou[] = "Function \"cuMemsetD2D8_v2\" not found"; +static const char __pyx_k_Function_cuMemsetD32Async_not_fo[] = "Function \"cuMemsetD32Async\" not found"; +static const char __pyx_k_Function_cuMemsetD32_v2_not_foun[] = "Function \"cuMemsetD32_v2\" not found"; +static const char __pyx_k_Function_cuMemsetD8Async_not_fou[] = "Function \"cuMemsetD8Async\" not found"; +static const char __pyx_k_Function_cuMemsetD8_v2_not_found[] = "Function \"cuMemsetD8_v2\" not found"; +static const char __pyx_k_Function_cuMipmappedArrayDestroy[] = "Function \"cuMipmappedArrayDestroy\" not found"; +static const char __pyx_k_Function_cuMipmappedArrayGetLeve[] = "Function \"cuMipmappedArrayGetLevel\" not found"; +static const char __pyx_k_Function_cuMipmappedArrayGetMemo[] = "Function \"cuMipmappedArrayGetMemoryRequirements\" not found"; +static const char __pyx_k_Function_cuMipmappedArrayGetSpar[] = "Function \"cuMipmappedArrayGetSparseProperties\" not found"; +static const char __pyx_k_Function_cuModuleEnumerateFuncti[] = "Function \"cuModuleEnumerateFunctions\" not found"; +static const char __pyx_k_Function_cuModuleGetFunctionCoun[] = "Function \"cuModuleGetFunctionCount\" not found"; +static const char __pyx_k_Function_cuModuleGetFunction_not[] = "Function \"cuModuleGetFunction\" not found"; +static const char __pyx_k_Function_cuModuleGetGlobal_v2_no[] = "Function \"cuModuleGetGlobal_v2\" not found"; +static const char __pyx_k_Function_cuModuleGetTexRef_not_f[] = "Function \"cuModuleGetTexRef\" not found"; +static const char __pyx_k_Function_cuModuleLoadData_not_fo[] = "Function \"cuModuleLoadData\" not found"; +static const char __pyx_k_Function_cuModuleLoadFatBinary_n[] = "Function \"cuModuleLoadFatBinary\" not found"; +static const char __pyx_k_Function_cuModuleUnload_not_foun[] = "Function \"cuModuleUnload\" not found"; +static const char __pyx_k_Function_cuMulticastAddDevice_no[] = "Function \"cuMulticastAddDevice\" not found"; +static const char __pyx_k_Function_cuMulticastBindAddr_not[] = "Function \"cuMulticastBindAddr\" not found"; +static const char __pyx_k_Function_cuMulticastCreate_not_f[] = "Function \"cuMulticastCreate\" not found"; +static const char __pyx_k_Function_cuMulticastGetGranulari[] = "Function \"cuMulticastGetGranularity\" not found"; +static const char __pyx_k_Function_cuMulticastUnbind_not_f[] = "Function \"cuMulticastUnbind\" not found"; +static const char __pyx_k_Function_cuOccupancyAvailableDyn[] = "Function \"cuOccupancyAvailableDynamicSMemPerBlock\" not found"; +static const char __pyx_k_Function_cuOccupancyMaxActiveBlo[] = "Function \"cuOccupancyMaxActiveBlocksPerMultiprocessor\" not found"; +static const char __pyx_k_Function_cuOccupancyMaxActiveClu[] = "Function \"cuOccupancyMaxActiveClusters\" not found"; +static const char __pyx_k_Function_cuOccupancyMaxPotential[] = "Function \"cuOccupancyMaxPotentialBlockSize\" not found"; +static const char __pyx_k_Function_cuParamSetSize_not_foun[] = "Function \"cuParamSetSize\" not found"; +static const char __pyx_k_Function_cuParamSetTexRef_not_fo[] = "Function \"cuParamSetTexRef\" not found"; +static const char __pyx_k_Function_cuPointerGetAttribute_n[] = "Function \"cuPointerGetAttribute\" not found"; +static const char __pyx_k_Function_cuPointerSetAttribute_n[] = "Function \"cuPointerSetAttribute\" not found"; +static const char __pyx_k_Function_cuProfilerStart_not_fou[] = "Function \"cuProfilerStart\" not found"; +static const char __pyx_k_Function_cuProfilerStop_not_foun[] = "Function \"cuProfilerStop\" not found"; +static const char __pyx_k_Function_cuSignalExternalSemapho[] = "Function \"cuSignalExternalSemaphoresAsync\" not found"; +static const char __pyx_k_Function_cuStreamAddCallback_not[] = "Function \"cuStreamAddCallback\" not found"; +static const char __pyx_k_Function_cuStreamBatchMemOp_v2_n[] = "Function \"cuStreamBatchMemOp_v2\" not found"; +static const char __pyx_k_Function_cuStreamBeginCaptureToG[] = "Function \"cuStreamBeginCaptureToGraph\" not found"; +static const char __pyx_k_Function_cuStreamBeginCapture_v2[] = "Function \"cuStreamBeginCapture_v2\" not found"; +static const char __pyx_k_Function_cuStreamCreateWithPrior[] = "Function \"cuStreamCreateWithPriority\" not found"; +static const char __pyx_k_Function_cuStreamCreate_not_foun[] = "Function \"cuStreamCreate\" not found"; +static const char __pyx_k_Function_cuStreamGetAttribute_no[] = "Function \"cuStreamGetAttribute\" not found"; +static const char __pyx_k_Function_cuStreamGetCtx_not_foun[] = "Function \"cuStreamGetCtx\" not found"; +static const char __pyx_k_Function_cuStreamGetCtx_v2_not_f[] = "Function \"cuStreamGetCtx_v2\" not found"; +static const char __pyx_k_Function_cuStreamGetDevice_not_f[] = "Function \"cuStreamGetDevice\" not found"; +static const char __pyx_k_Function_cuStreamGetFlags_not_fo[] = "Function \"cuStreamGetFlags\" not found"; +static const char __pyx_k_Function_cuStreamGetGreenCtx_not[] = "Function \"cuStreamGetGreenCtx\" not found"; +static const char __pyx_k_Function_cuStreamGetId_not_found[] = "Function \"cuStreamGetId\" not found"; +static const char __pyx_k_Function_cuStreamGetPriority_not[] = "Function \"cuStreamGetPriority\" not found"; +static const char __pyx_k_Function_cuStreamIsCapturing_not[] = "Function \"cuStreamIsCapturing\" not found"; +static const char __pyx_k_Function_cuStreamQuery_not_found[] = "Function \"cuStreamQuery\" not found"; +static const char __pyx_k_Function_cuStreamSetAttribute_no[] = "Function \"cuStreamSetAttribute\" not found"; +static const char __pyx_k_Function_cuStreamSynchronize_not[] = "Function \"cuStreamSynchronize\" not found"; +static const char __pyx_k_Function_cuStreamUpdateCaptureDe[] = "Function \"cuStreamUpdateCaptureDependencies\" not found"; +static const char __pyx_k_Function_cuStreamWaitEvent_not_f[] = "Function \"cuStreamWaitEvent\" not found"; +static const char __pyx_k_Function_cuStreamWriteValue32_v2[] = "Function \"cuStreamWriteValue32_v2\" not found"; +static const char __pyx_k_Function_cuStreamWriteValue64_v2[] = "Function \"cuStreamWriteValue64_v2\" not found"; +static const char __pyx_k_Function_cuSurfObjectDestroy_not[] = "Function \"cuSurfObjectDestroy\" not found"; +static const char __pyx_k_Function_cuSurfObjectGetResource[] = "Function \"cuSurfObjectGetResourceDesc\" not found"; +static const char __pyx_k_Function_cuSurfRefGetArray_not_f[] = "Function \"cuSurfRefGetArray\" not found"; +static const char __pyx_k_Function_cuSurfRefSetArray_not_f[] = "Function \"cuSurfRefSetArray\" not found"; +static const char __pyx_k_Function_cuTensorMapEncodeIm2col[] = "Function \"cuTensorMapEncodeIm2col\" not found"; +static const char __pyx_k_Function_cuTensorMapReplaceAddre[] = "Function \"cuTensorMapReplaceAddress\" not found"; +static const char __pyx_k_Function_cuTexObjectCreate_not_f[] = "Function \"cuTexObjectCreate\" not found"; +static const char __pyx_k_Function_cuTexObjectGetResourceD[] = "Function \"cuTexObjectGetResourceDesc\" not found"; +static const char __pyx_k_Function_cuTexObjectGetResourceV[] = "Function \"cuTexObjectGetResourceViewDesc\" not found"; +static const char __pyx_k_Function_cuTexObjectGetTextureDe[] = "Function \"cuTexObjectGetTextureDesc\" not found"; +static const char __pyx_k_Function_cuTexRefCreate_not_foun[] = "Function \"cuTexRefCreate\" not found"; +static const char __pyx_k_Function_cuTexRefDestroy_not_fou[] = "Function \"cuTexRefDestroy\" not found"; +static const char __pyx_k_Function_cuTexRefGetAddress_v2_n[] = "Function \"cuTexRefGetAddress_v2\" not found"; +static const char __pyx_k_Function_cuTexRefGetArray_not_fo[] = "Function \"cuTexRefGetArray\" not found"; +static const char __pyx_k_Function_cuTexRefGetFilterMode_n[] = "Function \"cuTexRefGetFilterMode\" not found"; +static const char __pyx_k_Function_cuTexRefGetFlags_not_fo[] = "Function \"cuTexRefGetFlags\" not found"; +static const char __pyx_k_Function_cuTexRefGetFormat_not_f[] = "Function \"cuTexRefGetFormat\" not found"; +static const char __pyx_k_Function_cuTexRefGetMaxAnisotrop[] = "Function \"cuTexRefGetMaxAnisotropy\" not found"; +static const char __pyx_k_Function_cuTexRefGetMipmapFilter[] = "Function \"cuTexRefGetMipmapFilterMode\" not found"; +static const char __pyx_k_Function_cuTexRefGetMipmapLevelB[] = "Function \"cuTexRefGetMipmapLevelBias\" not found"; +static const char __pyx_k_Function_cuTexRefGetMipmapLevelC[] = "Function \"cuTexRefGetMipmapLevelClamp\" not found"; +static const char __pyx_k_Function_cuTexRefGetMipmappedArr[] = "Function \"cuTexRefGetMipmappedArray\" not found"; +static const char __pyx_k_Function_cuTexRefSetAddress2D_v3[] = "Function \"cuTexRefSetAddress2D_v3\" not found"; +static const char __pyx_k_Function_cuTexRefSetAddress_v2_n[] = "Function \"cuTexRefSetAddress_v2\" not found"; +static const char __pyx_k_Function_cuTexRefSetArray_not_fo[] = "Function \"cuTexRefSetArray\" not found"; +static const char __pyx_k_Function_cuTexRefSetFilterMode_n[] = "Function \"cuTexRefSetFilterMode\" not found"; +static const char __pyx_k_Function_cuTexRefSetFlags_not_fo[] = "Function \"cuTexRefSetFlags\" not found"; +static const char __pyx_k_Function_cuTexRefSetFormat_not_f[] = "Function \"cuTexRefSetFormat\" not found"; +static const char __pyx_k_Function_cuTexRefSetMaxAnisotrop[] = "Function \"cuTexRefSetMaxAnisotropy\" not found"; +static const char __pyx_k_Function_cuTexRefSetMipmapFilter[] = "Function \"cuTexRefSetMipmapFilterMode\" not found"; +static const char __pyx_k_Function_cuTexRefSetMipmapLevelB[] = "Function \"cuTexRefSetMipmapLevelBias\" not found"; +static const char __pyx_k_Function_cuTexRefSetMipmapLevelC[] = "Function \"cuTexRefSetMipmapLevelClamp\" not found"; +static const char __pyx_k_Function_cuTexRefSetMipmappedArr[] = "Function \"cuTexRefSetMipmappedArray\" not found"; +static const char __pyx_k_Function_cuThreadExchangeStreamC[] = "Function \"cuThreadExchangeStreamCaptureMode\" not found"; +static const char __pyx_k_Function_cuUserObjectRelease_not[] = "Function \"cuUserObjectRelease\" not found"; +static const char __pyx_k_Function_cuVDPAUCtxCreate_v2_not[] = "Function \"cuVDPAUCtxCreate_v2\" not found"; +static const char __pyx_k_Function_cuVDPAUGetDevice_not_fo[] = "Function \"cuVDPAUGetDevice\" not found"; +static const char __pyx_k_Function_cuWaitExternalSemaphore[] = "Function \"cuWaitExternalSemaphoresAsync\" not found"; +static const char __pyx_k_Note_that_Cython_is_deliberately[] = "Note that Cython is deliberately stricter than PEP-484 and rejects subclasses of builtin types. If you need to pass subclasses then set the 'annotation_typing' directive to False."; +static const char __pyx_k_cuGraphExternalSemaphoresSigna_2[] = "__cuGraphExternalSemaphoresSignalNodeSetParams"; +static const char __pyx_k_cuGraphExternalSemaphoresWaitN_2[] = "__cuGraphExternalSemaphoresWaitNodeSetParams"; +static const char __pyx_k_cuOccupancyMaxActiveBlocksPerM_2[] = "__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags"; +static const char __pyx_k_cuOccupancyMaxPotentialBlockSi_2[] = "__cuOccupancyMaxPotentialBlockSizeWithFlags"; +static const char __pyx_k_cuStreamUpdateCaptureDependenc_2[] = "__cuStreamUpdateCaptureDependencies_v2"; +static const char __pyx_k_cuda_bindings__bindings_cydriver[] = "cuda.bindings._bindings.cydriver"; +static const char __pyx_k_Function_cuGraphAddDependencies_2[] = "Function \"cuGraphAddDependencies_v2\" not found"; +static const char __pyx_k_Function_cuStreamGetCaptureInfo_2[] = "Function \"cuStreamGetCaptureInfo_v3\" not found"; +static const char __pyx_k_Function_cuEGLStreamConsumerConn_2[] = "Function \"cuEGLStreamConsumerConnectWithFlags\" not found"; +static const char __pyx_k_Function_cuExternalMemoryGetMapp_2[] = "Function \"cuExternalMemoryGetMappedMipmappedArray\" not found"; +static const char __pyx_k_Function_cuGraphAddExternalSemap_2[] = "Function \"cuGraphAddExternalSemaphoresWaitNode\" not found"; +static const char __pyx_k_Function_cuGraphExecExternalSema_2[] = "Function \"cuGraphExecExternalSemaphoresWaitNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphExternalSemaphor_2[] = "Function \"cuGraphExternalSemaphoresSignalNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphExternalSemaphor_3[] = "Function \"cuGraphExternalSemaphoresWaitNodeGetParams\" not found"; +static const char __pyx_k_Function_cuGraphExternalSemaphor_4[] = "Function \"cuGraphExternalSemaphoresWaitNodeSetParams\" not found"; +static const char __pyx_k_Function_cuGraphNodeGetDependenc_2[] = "Function \"cuGraphNodeGetDependencies_v2\" not found"; +static const char __pyx_k_Function_cuGraphNodeGetDependent_2[] = "Function \"cuGraphNodeGetDependentNodes_v2\" not found"; +static const char __pyx_k_Function_cuGraphRemoveDependenci_2[] = "Function \"cuGraphRemoveDependencies_v2\" not found"; +static const char __pyx_k_Function_cuGraphicsResourceGetMa_2[] = "Function \"cuGraphicsResourceGetMappedPointer_v2\" not found"; +static const char __pyx_k_Function_cuGraphicsResourceGetMa_3[] = "Function \"cuGraphicsResourceGetMappedEglFrame\" not found"; +static const char __pyx_k_Function_cuGraphicsVDPAURegister_2[] = "Function \"cuGraphicsVDPAURegisterOutputSurface\" not found"; +static const char __pyx_k_Function_cuLaunchCooperativeKern_2[] = "Function \"cuLaunchCooperativeKernelMultiDevice\" not found"; +static const char __pyx_k_Function_cuOccupancyMaxActiveBlo_2[] = "Function \"cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags\" not found"; +static const char __pyx_k_Function_cuOccupancyMaxPotential_2[] = "Function \"cuOccupancyMaxPotentialBlockSizeWithFlags\" not found"; +static const char __pyx_k_Function_cuOccupancyMaxPotential_3[] = "Function \"cuOccupancyMaxPotentialClusterSize\" not found"; +static const char __pyx_k_Function_cuStreamUpdateCaptureDe_2[] = "Function \"cuStreamUpdateCaptureDependencies_v2\" not found"; +static const char __pyx_k_Function_cuTensorMapEncodeIm2col_2[] = "Function \"cuTensorMapEncodeIm2colWide\" not found"; +static const char __pyx_k_cuda_bindings__bindings_cydriver_2[] = "cuda/bindings/_bindings/cydriver.pyx"; +/* #### Code section: decls ### */ +static PyObject *__pyx_pf_4cuda_8bindings_9_bindings_8cydriver__inspect_function_pointers(CYTHON_UNUSED PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_pf_4cuda_8bindings_9_bindings_8cydriver_2_inspect_function_pointer(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_name); /* proto */ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + #ifdef __Pyx_CyFunction_USED + PyTypeObject *__pyx_CyFunctionType; + #endif + #ifdef __Pyx_FusedFunction_USED + PyTypeObject *__pyx_FusedFunctionType; + #endif + #ifdef __Pyx_Generator_USED + PyTypeObject *__pyx_GeneratorType; + #endif + #ifdef __Pyx_IterableCoroutine_USED + PyTypeObject *__pyx_IterableCoroutineType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineAwaitType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineType; + #endif + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop; + PyObject *__pyx_tuple[1]; + PyObject *__pyx_codeobj_tab[2]; + PyObject *__pyx_string_tab[982]; + PyObject *__pyx_int_0; + PyObject *__pyx_int_4294967296; +/* #### Code section: module_state_contents ### */ +/* CommonTypesMetaclass.module_state_decls */ +PyTypeObject *__pyx_CommonTypesMetaclassType; + +/* CachedMethodType.module_state_decls */ +#if CYTHON_COMPILING_IN_LIMITED_API +PyObject *__Pyx_CachedMethodType; +#endif + +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_n_u_CUDA_PYTHON_CUDA_PER_THREAD_DEFA __pyx_string_tab[1] +#define __pyx_kp_u_Failed_to_dlopen __pyx_string_tab[2] +#define __pyx_kp_u_Function_cuArray3DCreate_v2_not __pyx_string_tab[3] +#define __pyx_kp_u_Function_cuArray3DGetDescriptor __pyx_string_tab[4] +#define __pyx_kp_u_Function_cuArrayCreate_v2_not_fo __pyx_string_tab[5] +#define __pyx_kp_u_Function_cuArrayDestroy_not_foun __pyx_string_tab[6] +#define __pyx_kp_u_Function_cuArrayGetDescriptor_v2 __pyx_string_tab[7] +#define __pyx_kp_u_Function_cuArrayGetMemoryRequire __pyx_string_tab[8] +#define __pyx_kp_u_Function_cuArrayGetPlane_not_fou __pyx_string_tab[9] +#define __pyx_kp_u_Function_cuArrayGetSparsePropert __pyx_string_tab[10] +#define __pyx_kp_u_Function_cuCheckpointProcessChec __pyx_string_tab[11] +#define __pyx_kp_u_Function_cuCheckpointProcessGetR __pyx_string_tab[12] +#define __pyx_kp_u_Function_cuCheckpointProcessGetS __pyx_string_tab[13] +#define __pyx_kp_u_Function_cuCheckpointProcessLock __pyx_string_tab[14] +#define __pyx_kp_u_Function_cuCheckpointProcessRest __pyx_string_tab[15] +#define __pyx_kp_u_Function_cuCheckpointProcessUnlo __pyx_string_tab[16] +#define __pyx_kp_u_Function_cuCoredumpGetAttribute __pyx_string_tab[17] +#define __pyx_kp_u_Function_cuCoredumpGetAttributeG __pyx_string_tab[18] +#define __pyx_kp_u_Function_cuCoredumpSetAttribute __pyx_string_tab[19] +#define __pyx_kp_u_Function_cuCoredumpSetAttributeG __pyx_string_tab[20] +#define __pyx_kp_u_Function_cuCtxAttach_not_found __pyx_string_tab[21] +#define __pyx_kp_u_Function_cuCtxCreate_v2_not_foun __pyx_string_tab[22] +#define __pyx_kp_u_Function_cuCtxCreate_v3_not_foun __pyx_string_tab[23] +#define __pyx_kp_u_Function_cuCtxCreate_v4_not_foun __pyx_string_tab[24] +#define __pyx_kp_u_Function_cuCtxDestroy_v2_not_fou __pyx_string_tab[25] +#define __pyx_kp_u_Function_cuCtxDetach_not_found __pyx_string_tab[26] +#define __pyx_kp_u_Function_cuCtxDisablePeerAccess __pyx_string_tab[27] +#define __pyx_kp_u_Function_cuCtxEnablePeerAccess_n __pyx_string_tab[28] +#define __pyx_kp_u_Function_cuCtxFromGreenCtx_not_f __pyx_string_tab[29] +#define __pyx_kp_u_Function_cuCtxGetApiVersion_not __pyx_string_tab[30] +#define __pyx_kp_u_Function_cuCtxGetCacheConfig_not __pyx_string_tab[31] +#define __pyx_kp_u_Function_cuCtxGetCurrent_not_fou __pyx_string_tab[32] +#define __pyx_kp_u_Function_cuCtxGetDevResource_not __pyx_string_tab[33] +#define __pyx_kp_u_Function_cuCtxGetDevice_not_foun __pyx_string_tab[34] +#define __pyx_kp_u_Function_cuCtxGetExecAffinity_no __pyx_string_tab[35] +#define __pyx_kp_u_Function_cuCtxGetFlags_not_found __pyx_string_tab[36] +#define __pyx_kp_u_Function_cuCtxGetId_not_found __pyx_string_tab[37] +#define __pyx_kp_u_Function_cuCtxGetLimit_not_found __pyx_string_tab[38] +#define __pyx_kp_u_Function_cuCtxGetSharedMemConfig __pyx_string_tab[39] +#define __pyx_kp_u_Function_cuCtxGetStreamPriorityR __pyx_string_tab[40] +#define __pyx_kp_u_Function_cuCtxPopCurrent_v2_not __pyx_string_tab[41] +#define __pyx_kp_u_Function_cuCtxPushCurrent_v2_not __pyx_string_tab[42] +#define __pyx_kp_u_Function_cuCtxRecordEvent_not_fo __pyx_string_tab[43] +#define __pyx_kp_u_Function_cuCtxResetPersistingL2C __pyx_string_tab[44] +#define __pyx_kp_u_Function_cuCtxSetCacheConfig_not __pyx_string_tab[45] +#define __pyx_kp_u_Function_cuCtxSetCurrent_not_fou __pyx_string_tab[46] +#define __pyx_kp_u_Function_cuCtxSetFlags_not_found __pyx_string_tab[47] +#define __pyx_kp_u_Function_cuCtxSetLimit_not_found __pyx_string_tab[48] +#define __pyx_kp_u_Function_cuCtxSetSharedMemConfig __pyx_string_tab[49] +#define __pyx_kp_u_Function_cuCtxSynchronize_not_fo __pyx_string_tab[50] +#define __pyx_kp_u_Function_cuCtxWaitEvent_not_foun __pyx_string_tab[51] +#define __pyx_kp_u_Function_cuDestroyExternalMemory __pyx_string_tab[52] +#define __pyx_kp_u_Function_cuDestroyExternalSemaph __pyx_string_tab[53] +#define __pyx_kp_u_Function_cuDevResourceGenerateDe __pyx_string_tab[54] +#define __pyx_kp_u_Function_cuDevSmResourceSplitByC __pyx_string_tab[55] +#define __pyx_kp_u_Function_cuDeviceCanAccessPeer_n __pyx_string_tab[56] +#define __pyx_kp_u_Function_cuDeviceComputeCapabili __pyx_string_tab[57] +#define __pyx_kp_u_Function_cuDeviceGetAttribute_no __pyx_string_tab[58] +#define __pyx_kp_u_Function_cuDeviceGetByPCIBusId_n __pyx_string_tab[59] +#define __pyx_kp_u_Function_cuDeviceGetCount_not_fo __pyx_string_tab[60] +#define __pyx_kp_u_Function_cuDeviceGetDefaultMemPo __pyx_string_tab[61] +#define __pyx_kp_u_Function_cuDeviceGetDevResource __pyx_string_tab[62] +#define __pyx_kp_u_Function_cuDeviceGetExecAffinity __pyx_string_tab[63] +#define __pyx_kp_u_Function_cuDeviceGetGraphMemAttr __pyx_string_tab[64] +#define __pyx_kp_u_Function_cuDeviceGetLuid_not_fou __pyx_string_tab[65] +#define __pyx_kp_u_Function_cuDeviceGetMemPool_not __pyx_string_tab[66] +#define __pyx_kp_u_Function_cuDeviceGetName_not_fou __pyx_string_tab[67] +#define __pyx_kp_u_Function_cuDeviceGetNvSciSyncAtt __pyx_string_tab[68] +#define __pyx_kp_u_Function_cuDeviceGetP2PAttribute __pyx_string_tab[69] +#define __pyx_kp_u_Function_cuDeviceGetPCIBusId_not __pyx_string_tab[70] +#define __pyx_kp_u_Function_cuDeviceGetProperties_n __pyx_string_tab[71] +#define __pyx_kp_u_Function_cuDeviceGetTexture1DLin __pyx_string_tab[72] +#define __pyx_kp_u_Function_cuDeviceGetUuid_not_fou __pyx_string_tab[73] +#define __pyx_kp_u_Function_cuDeviceGetUuid_v2_not __pyx_string_tab[74] +#define __pyx_kp_u_Function_cuDeviceGet_not_found __pyx_string_tab[75] +#define __pyx_kp_u_Function_cuDeviceGraphMemTrim_no __pyx_string_tab[76] +#define __pyx_kp_u_Function_cuDevicePrimaryCtxGetSt __pyx_string_tab[77] +#define __pyx_kp_u_Function_cuDevicePrimaryCtxRelea __pyx_string_tab[78] +#define __pyx_kp_u_Function_cuDevicePrimaryCtxReset __pyx_string_tab[79] +#define __pyx_kp_u_Function_cuDevicePrimaryCtxRetai __pyx_string_tab[80] +#define __pyx_kp_u_Function_cuDevicePrimaryCtxSetFl __pyx_string_tab[81] +#define __pyx_kp_u_Function_cuDeviceRegisterAsyncNo __pyx_string_tab[82] +#define __pyx_kp_u_Function_cuDeviceSetGraphMemAttr __pyx_string_tab[83] +#define __pyx_kp_u_Function_cuDeviceSetMemPool_not __pyx_string_tab[84] +#define __pyx_kp_u_Function_cuDeviceTotalMem_v2_not __pyx_string_tab[85] +#define __pyx_kp_u_Function_cuDeviceUnregisterAsync __pyx_string_tab[86] +#define __pyx_kp_u_Function_cuDriverGetVersion_not __pyx_string_tab[87] +#define __pyx_kp_u_Function_cuEGLStreamConsumerAcqu __pyx_string_tab[88] +#define __pyx_kp_u_Function_cuEGLStreamConsumerConn __pyx_string_tab[89] +#define __pyx_kp_u_Function_cuEGLStreamConsumerConn_2 __pyx_string_tab[90] +#define __pyx_kp_u_Function_cuEGLStreamConsumerDisc __pyx_string_tab[91] +#define __pyx_kp_u_Function_cuEGLStreamConsumerRele __pyx_string_tab[92] +#define __pyx_kp_u_Function_cuEGLStreamProducerConn __pyx_string_tab[93] +#define __pyx_kp_u_Function_cuEGLStreamProducerDisc __pyx_string_tab[94] +#define __pyx_kp_u_Function_cuEGLStreamProducerPres __pyx_string_tab[95] +#define __pyx_kp_u_Function_cuEGLStreamProducerRetu __pyx_string_tab[96] +#define __pyx_kp_u_Function_cuEventCreateFromEGLSyn __pyx_string_tab[97] +#define __pyx_kp_u_Function_cuEventCreate_not_found __pyx_string_tab[98] +#define __pyx_kp_u_Function_cuEventDestroy_v2_not_f __pyx_string_tab[99] +#define __pyx_kp_u_Function_cuEventElapsedTime_not __pyx_string_tab[100] +#define __pyx_kp_u_Function_cuEventElapsedTime_v2_n __pyx_string_tab[101] +#define __pyx_kp_u_Function_cuEventQuery_not_found __pyx_string_tab[102] +#define __pyx_kp_u_Function_cuEventRecordWithFlags __pyx_string_tab[103] +#define __pyx_kp_u_Function_cuEventRecord_not_found __pyx_string_tab[104] +#define __pyx_kp_u_Function_cuEventSynchronize_not __pyx_string_tab[105] +#define __pyx_kp_u_Function_cuExternalMemoryGetMapp __pyx_string_tab[106] +#define __pyx_kp_u_Function_cuExternalMemoryGetMapp_2 __pyx_string_tab[107] +#define __pyx_kp_u_Function_cuFlushGPUDirectRDMAWri __pyx_string_tab[108] +#define __pyx_kp_u_Function_cuFuncGetAttribute_not __pyx_string_tab[109] +#define __pyx_kp_u_Function_cuFuncGetModule_not_fou __pyx_string_tab[110] +#define __pyx_kp_u_Function_cuFuncGetName_not_found __pyx_string_tab[111] +#define __pyx_kp_u_Function_cuFuncGetParamInfo_not __pyx_string_tab[112] +#define __pyx_kp_u_Function_cuFuncIsLoaded_not_foun __pyx_string_tab[113] +#define __pyx_kp_u_Function_cuFuncLoad_not_found __pyx_string_tab[114] +#define __pyx_kp_u_Function_cuFuncSetAttribute_not __pyx_string_tab[115] +#define __pyx_kp_u_Function_cuFuncSetBlockShape_not __pyx_string_tab[116] +#define __pyx_kp_u_Function_cuFuncSetCacheConfig_no __pyx_string_tab[117] +#define __pyx_kp_u_Function_cuFuncSetSharedMemConfi __pyx_string_tab[118] +#define __pyx_kp_u_Function_cuFuncSetSharedSize_not __pyx_string_tab[119] +#define __pyx_kp_u_Function_cuGLGetDevices_v2_not_f __pyx_string_tab[120] +#define __pyx_kp_u_Function_cuGetErrorName_not_foun __pyx_string_tab[121] +#define __pyx_kp_u_Function_cuGetErrorString_not_fo __pyx_string_tab[122] +#define __pyx_kp_u_Function_cuGetExportTable_not_fo __pyx_string_tab[123] +#define __pyx_kp_u_Function_cuGetProcAddress_v2_not __pyx_string_tab[124] +#define __pyx_kp_u_Function_cuGraphAddBatchMemOpNod __pyx_string_tab[125] +#define __pyx_kp_u_Function_cuGraphAddChildGraphNod __pyx_string_tab[126] +#define __pyx_kp_u_Function_cuGraphAddDependencies __pyx_string_tab[127] +#define __pyx_kp_u_Function_cuGraphAddDependencies_2 __pyx_string_tab[128] +#define __pyx_kp_u_Function_cuGraphAddEmptyNode_not __pyx_string_tab[129] +#define __pyx_kp_u_Function_cuGraphAddEventRecordNo __pyx_string_tab[130] +#define __pyx_kp_u_Function_cuGraphAddEventWaitNode __pyx_string_tab[131] +#define __pyx_kp_u_Function_cuGraphAddExternalSemap __pyx_string_tab[132] +#define __pyx_kp_u_Function_cuGraphAddExternalSemap_2 __pyx_string_tab[133] +#define __pyx_kp_u_Function_cuGraphAddHostNode_not __pyx_string_tab[134] +#define __pyx_kp_u_Function_cuGraphAddKernelNode_v2 __pyx_string_tab[135] +#define __pyx_kp_u_Function_cuGraphAddMemAllocNode __pyx_string_tab[136] +#define __pyx_kp_u_Function_cuGraphAddMemFreeNode_n __pyx_string_tab[137] +#define __pyx_kp_u_Function_cuGraphAddMemcpyNode_no __pyx_string_tab[138] +#define __pyx_kp_u_Function_cuGraphAddMemsetNode_no __pyx_string_tab[139] +#define __pyx_kp_u_Function_cuGraphAddNode_not_foun __pyx_string_tab[140] +#define __pyx_kp_u_Function_cuGraphAddNode_v2_not_f __pyx_string_tab[141] +#define __pyx_kp_u_Function_cuGraphBatchMemOpNodeGe __pyx_string_tab[142] +#define __pyx_kp_u_Function_cuGraphBatchMemOpNodeSe __pyx_string_tab[143] +#define __pyx_kp_u_Function_cuGraphChildGraphNodeGe __pyx_string_tab[144] +#define __pyx_kp_u_Function_cuGraphClone_not_found __pyx_string_tab[145] +#define __pyx_kp_u_Function_cuGraphConditionalHandl __pyx_string_tab[146] +#define __pyx_kp_u_Function_cuGraphCreate_not_found __pyx_string_tab[147] +#define __pyx_kp_u_Function_cuGraphDebugDotPrint_no __pyx_string_tab[148] +#define __pyx_kp_u_Function_cuGraphDestroyNode_not __pyx_string_tab[149] +#define __pyx_kp_u_Function_cuGraphDestroy_not_foun __pyx_string_tab[150] +#define __pyx_kp_u_Function_cuGraphEventRecordNodeG __pyx_string_tab[151] +#define __pyx_kp_u_Function_cuGraphEventRecordNodeS __pyx_string_tab[152] +#define __pyx_kp_u_Function_cuGraphEventWaitNodeGet __pyx_string_tab[153] +#define __pyx_kp_u_Function_cuGraphEventWaitNodeSet __pyx_string_tab[154] +#define __pyx_kp_u_Function_cuGraphExecBatchMemOpNo __pyx_string_tab[155] +#define __pyx_kp_u_Function_cuGraphExecChildGraphNo __pyx_string_tab[156] +#define __pyx_kp_u_Function_cuGraphExecDestroy_not __pyx_string_tab[157] +#define __pyx_kp_u_Function_cuGraphExecEventRecordN __pyx_string_tab[158] +#define __pyx_kp_u_Function_cuGraphExecEventWaitNod __pyx_string_tab[159] +#define __pyx_kp_u_Function_cuGraphExecExternalSema __pyx_string_tab[160] +#define __pyx_kp_u_Function_cuGraphExecExternalSema_2 __pyx_string_tab[161] +#define __pyx_kp_u_Function_cuGraphExecGetFlags_not __pyx_string_tab[162] +#define __pyx_kp_u_Function_cuGraphExecHostNodeSetP __pyx_string_tab[163] +#define __pyx_kp_u_Function_cuGraphExecKernelNodeSe __pyx_string_tab[164] +#define __pyx_kp_u_Function_cuGraphExecMemcpyNodeSe __pyx_string_tab[165] +#define __pyx_kp_u_Function_cuGraphExecMemsetNodeSe __pyx_string_tab[166] +#define __pyx_kp_u_Function_cuGraphExecNodeSetParam __pyx_string_tab[167] +#define __pyx_kp_u_Function_cuGraphExecUpdate_v2_no __pyx_string_tab[168] +#define __pyx_kp_u_Function_cuGraphExternalSemaphor __pyx_string_tab[169] +#define __pyx_kp_u_Function_cuGraphExternalSemaphor_2 __pyx_string_tab[170] +#define __pyx_kp_u_Function_cuGraphExternalSemaphor_3 __pyx_string_tab[171] +#define __pyx_kp_u_Function_cuGraphExternalSemaphor_4 __pyx_string_tab[172] +#define __pyx_kp_u_Function_cuGraphGetEdges_not_fou __pyx_string_tab[173] +#define __pyx_kp_u_Function_cuGraphGetEdges_v2_not __pyx_string_tab[174] +#define __pyx_kp_u_Function_cuGraphGetNodes_not_fou __pyx_string_tab[175] +#define __pyx_kp_u_Function_cuGraphGetRootNodes_not __pyx_string_tab[176] +#define __pyx_kp_u_Function_cuGraphHostNodeGetParam __pyx_string_tab[177] +#define __pyx_kp_u_Function_cuGraphHostNodeSetParam __pyx_string_tab[178] +#define __pyx_kp_u_Function_cuGraphInstantiateWithF __pyx_string_tab[179] +#define __pyx_kp_u_Function_cuGraphInstantiateWithP __pyx_string_tab[180] +#define __pyx_kp_u_Function_cuGraphKernelNodeCopyAt __pyx_string_tab[181] +#define __pyx_kp_u_Function_cuGraphKernelNodeGetAtt __pyx_string_tab[182] +#define __pyx_kp_u_Function_cuGraphKernelNodeGetPar __pyx_string_tab[183] +#define __pyx_kp_u_Function_cuGraphKernelNodeSetAtt __pyx_string_tab[184] +#define __pyx_kp_u_Function_cuGraphKernelNodeSetPar __pyx_string_tab[185] +#define __pyx_kp_u_Function_cuGraphLaunch_not_found __pyx_string_tab[186] +#define __pyx_kp_u_Function_cuGraphMemAllocNodeGetP __pyx_string_tab[187] +#define __pyx_kp_u_Function_cuGraphMemFreeNodeGetPa __pyx_string_tab[188] +#define __pyx_kp_u_Function_cuGraphMemcpyNodeGetPar __pyx_string_tab[189] +#define __pyx_kp_u_Function_cuGraphMemcpyNodeSetPar __pyx_string_tab[190] +#define __pyx_kp_u_Function_cuGraphMemsetNodeGetPar __pyx_string_tab[191] +#define __pyx_kp_u_Function_cuGraphMemsetNodeSetPar __pyx_string_tab[192] +#define __pyx_kp_u_Function_cuGraphNodeFindInClone __pyx_string_tab[193] +#define __pyx_kp_u_Function_cuGraphNodeGetDependenc __pyx_string_tab[194] +#define __pyx_kp_u_Function_cuGraphNodeGetDependenc_2 __pyx_string_tab[195] +#define __pyx_kp_u_Function_cuGraphNodeGetDependent __pyx_string_tab[196] +#define __pyx_kp_u_Function_cuGraphNodeGetDependent_2 __pyx_string_tab[197] +#define __pyx_kp_u_Function_cuGraphNodeGetEnabled_n __pyx_string_tab[198] +#define __pyx_kp_u_Function_cuGraphNodeGetType_not __pyx_string_tab[199] +#define __pyx_kp_u_Function_cuGraphNodeSetEnabled_n __pyx_string_tab[200] +#define __pyx_kp_u_Function_cuGraphNodeSetParams_no __pyx_string_tab[201] +#define __pyx_kp_u_Function_cuGraphReleaseUserObjec __pyx_string_tab[202] +#define __pyx_kp_u_Function_cuGraphRemoveDependenci __pyx_string_tab[203] +#define __pyx_kp_u_Function_cuGraphRemoveDependenci_2 __pyx_string_tab[204] +#define __pyx_kp_u_Function_cuGraphRetainUserObject __pyx_string_tab[205] +#define __pyx_kp_u_Function_cuGraphUpload_not_found __pyx_string_tab[206] +#define __pyx_kp_u_Function_cuGraphicsEGLRegisterIm __pyx_string_tab[207] +#define __pyx_kp_u_Function_cuGraphicsGLRegisterBuf __pyx_string_tab[208] +#define __pyx_kp_u_Function_cuGraphicsGLRegisterIma __pyx_string_tab[209] +#define __pyx_kp_u_Function_cuGraphicsMapResources __pyx_string_tab[210] +#define __pyx_kp_u_Function_cuGraphicsResourceGetMa __pyx_string_tab[211] +#define __pyx_kp_u_Function_cuGraphicsResourceGetMa_2 __pyx_string_tab[212] +#define __pyx_kp_u_Function_cuGraphicsResourceGetMa_3 __pyx_string_tab[213] +#define __pyx_kp_u_Function_cuGraphicsResourceSetMa __pyx_string_tab[214] +#define __pyx_kp_u_Function_cuGraphicsSubResourceGe __pyx_string_tab[215] +#define __pyx_kp_u_Function_cuGraphicsUnmapResource __pyx_string_tab[216] +#define __pyx_kp_u_Function_cuGraphicsUnregisterRes __pyx_string_tab[217] +#define __pyx_kp_u_Function_cuGraphicsVDPAURegister __pyx_string_tab[218] +#define __pyx_kp_u_Function_cuGraphicsVDPAURegister_2 __pyx_string_tab[219] +#define __pyx_kp_u_Function_cuGreenCtxCreate_not_fo __pyx_string_tab[220] +#define __pyx_kp_u_Function_cuGreenCtxDestroy_not_f __pyx_string_tab[221] +#define __pyx_kp_u_Function_cuGreenCtxGetDevResourc __pyx_string_tab[222] +#define __pyx_kp_u_Function_cuGreenCtxRecordEvent_n __pyx_string_tab[223] +#define __pyx_kp_u_Function_cuGreenCtxStreamCreate __pyx_string_tab[224] +#define __pyx_kp_u_Function_cuGreenCtxWaitEvent_not __pyx_string_tab[225] +#define __pyx_kp_u_Function_cuImportExternalMemory __pyx_string_tab[226] +#define __pyx_kp_u_Function_cuImportExternalSemapho __pyx_string_tab[227] +#define __pyx_kp_u_Function_cuInit_not_found __pyx_string_tab[228] +#define __pyx_kp_u_Function_cuIpcCloseMemHandle_not __pyx_string_tab[229] +#define __pyx_kp_u_Function_cuIpcGetEventHandle_not __pyx_string_tab[230] +#define __pyx_kp_u_Function_cuIpcGetMemHandle_not_f __pyx_string_tab[231] +#define __pyx_kp_u_Function_cuIpcOpenEventHandle_no __pyx_string_tab[232] +#define __pyx_kp_u_Function_cuIpcOpenMemHandle_v2_n __pyx_string_tab[233] +#define __pyx_kp_u_Function_cuKernelGetAttribute_no __pyx_string_tab[234] +#define __pyx_kp_u_Function_cuKernelGetFunction_not __pyx_string_tab[235] +#define __pyx_kp_u_Function_cuKernelGetLibrary_not __pyx_string_tab[236] +#define __pyx_kp_u_Function_cuKernelGetName_not_fou __pyx_string_tab[237] +#define __pyx_kp_u_Function_cuKernelGetParamInfo_no __pyx_string_tab[238] +#define __pyx_kp_u_Function_cuKernelSetAttribute_no __pyx_string_tab[239] +#define __pyx_kp_u_Function_cuKernelSetCacheConfig __pyx_string_tab[240] +#define __pyx_kp_u_Function_cuLaunchCooperativeKern __pyx_string_tab[241] +#define __pyx_kp_u_Function_cuLaunchCooperativeKern_2 __pyx_string_tab[242] +#define __pyx_kp_u_Function_cuLaunchGridAsync_not_f __pyx_string_tab[243] +#define __pyx_kp_u_Function_cuLaunchGrid_not_found __pyx_string_tab[244] +#define __pyx_kp_u_Function_cuLaunchHostFunc_not_fo __pyx_string_tab[245] +#define __pyx_kp_u_Function_cuLaunchKernelEx_not_fo __pyx_string_tab[246] +#define __pyx_kp_u_Function_cuLaunchKernel_not_foun __pyx_string_tab[247] +#define __pyx_kp_u_Function_cuLaunch_not_found __pyx_string_tab[248] +#define __pyx_kp_u_Function_cuLibraryEnumerateKerne __pyx_string_tab[249] +#define __pyx_kp_u_Function_cuLibraryGetGlobal_not __pyx_string_tab[250] +#define __pyx_kp_u_Function_cuLibraryGetKernelCount __pyx_string_tab[251] +#define __pyx_kp_u_Function_cuLibraryGetKernel_not __pyx_string_tab[252] +#define __pyx_kp_u_Function_cuLibraryGetManaged_not __pyx_string_tab[253] +#define __pyx_kp_u_Function_cuLibraryGetModule_not __pyx_string_tab[254] +#define __pyx_kp_u_Function_cuLibraryGetUnifiedFunc __pyx_string_tab[255] +#define __pyx_kp_u_Function_cuLibraryLoadData_not_f __pyx_string_tab[256] +#define __pyx_kp_u_Function_cuLibraryLoadFromFile_n __pyx_string_tab[257] +#define __pyx_kp_u_Function_cuLibraryUnload_not_fou __pyx_string_tab[258] +#define __pyx_kp_u_Function_cuLinkAddData_v2_not_fo __pyx_string_tab[259] +#define __pyx_kp_u_Function_cuLinkAddFile_v2_not_fo __pyx_string_tab[260] +#define __pyx_kp_u_Function_cuLinkComplete_not_foun __pyx_string_tab[261] +#define __pyx_kp_u_Function_cuLinkCreate_v2_not_fou __pyx_string_tab[262] +#define __pyx_kp_u_Function_cuLinkDestroy_not_found __pyx_string_tab[263] +#define __pyx_kp_u_Function_cuLogsCurrent_not_found __pyx_string_tab[264] +#define __pyx_kp_u_Function_cuLogsDumpToFile_not_fo __pyx_string_tab[265] +#define __pyx_kp_u_Function_cuLogsDumpToMemory_not __pyx_string_tab[266] +#define __pyx_kp_u_Function_cuLogsRegisterCallback __pyx_string_tab[267] +#define __pyx_kp_u_Function_cuLogsUnregisterCallbac __pyx_string_tab[268] +#define __pyx_kp_u_Function_cuMemAddressFree_not_fo __pyx_string_tab[269] +#define __pyx_kp_u_Function_cuMemAddressReserve_not __pyx_string_tab[270] +#define __pyx_kp_u_Function_cuMemAdvise_not_found __pyx_string_tab[271] +#define __pyx_kp_u_Function_cuMemAdvise_v2_not_foun __pyx_string_tab[272] +#define __pyx_kp_u_Function_cuMemAllocAsync_not_fou __pyx_string_tab[273] +#define __pyx_kp_u_Function_cuMemAllocFromPoolAsync __pyx_string_tab[274] +#define __pyx_kp_u_Function_cuMemAllocHost_v2_not_f __pyx_string_tab[275] +#define __pyx_kp_u_Function_cuMemAllocManaged_not_f __pyx_string_tab[276] +#define __pyx_kp_u_Function_cuMemAllocPitch_v2_not __pyx_string_tab[277] +#define __pyx_kp_u_Function_cuMemAlloc_v2_not_found __pyx_string_tab[278] +#define __pyx_kp_u_Function_cuMemBatchDecompressAsy __pyx_string_tab[279] +#define __pyx_kp_u_Function_cuMemCreate_not_found __pyx_string_tab[280] +#define __pyx_kp_u_Function_cuMemExportToShareableH __pyx_string_tab[281] +#define __pyx_kp_u_Function_cuMemFreeAsync_not_foun __pyx_string_tab[282] +#define __pyx_kp_u_Function_cuMemFreeHost_not_found __pyx_string_tab[283] +#define __pyx_kp_u_Function_cuMemFree_v2_not_found __pyx_string_tab[284] +#define __pyx_kp_u_Function_cuMemGetAccess_not_foun __pyx_string_tab[285] +#define __pyx_kp_u_Function_cuMemGetAddressRange_v2 __pyx_string_tab[286] +#define __pyx_kp_u_Function_cuMemGetAllocationGranu __pyx_string_tab[287] +#define __pyx_kp_u_Function_cuMemGetAllocationPrope __pyx_string_tab[288] +#define __pyx_kp_u_Function_cuMemGetHandleForAddres __pyx_string_tab[289] +#define __pyx_kp_u_Function_cuMemGetInfo_v2_not_fou __pyx_string_tab[290] +#define __pyx_kp_u_Function_cuMemHostAlloc_not_foun __pyx_string_tab[291] +#define __pyx_kp_u_Function_cuMemHostGetDevicePoint __pyx_string_tab[292] +#define __pyx_kp_u_Function_cuMemHostGetFlags_not_f __pyx_string_tab[293] +#define __pyx_kp_u_Function_cuMemHostRegister_v2_no __pyx_string_tab[294] +#define __pyx_kp_u_Function_cuMemHostUnregister_not __pyx_string_tab[295] +#define __pyx_kp_u_Function_cuMemImportFromShareabl __pyx_string_tab[296] +#define __pyx_kp_u_Function_cuMemMapArrayAsync_not __pyx_string_tab[297] +#define __pyx_kp_u_Function_cuMemMap_not_found __pyx_string_tab[298] +#define __pyx_kp_u_Function_cuMemPoolCreate_not_fou __pyx_string_tab[299] +#define __pyx_kp_u_Function_cuMemPoolDestroy_not_fo __pyx_string_tab[300] +#define __pyx_kp_u_Function_cuMemPoolExportPointer __pyx_string_tab[301] +#define __pyx_kp_u_Function_cuMemPoolExportToSharea __pyx_string_tab[302] +#define __pyx_kp_u_Function_cuMemPoolGetAccess_not __pyx_string_tab[303] +#define __pyx_kp_u_Function_cuMemPoolGetAttribute_n __pyx_string_tab[304] +#define __pyx_kp_u_Function_cuMemPoolImportFromShar __pyx_string_tab[305] +#define __pyx_kp_u_Function_cuMemPoolImportPointer __pyx_string_tab[306] +#define __pyx_kp_u_Function_cuMemPoolSetAccess_not __pyx_string_tab[307] +#define __pyx_kp_u_Function_cuMemPoolSetAttribute_n __pyx_string_tab[308] +#define __pyx_kp_u_Function_cuMemPoolTrimTo_not_fou __pyx_string_tab[309] +#define __pyx_kp_u_Function_cuMemPrefetchAsync_not __pyx_string_tab[310] +#define __pyx_kp_u_Function_cuMemPrefetchAsync_v2_n __pyx_string_tab[311] +#define __pyx_kp_u_Function_cuMemRangeGetAttribute __pyx_string_tab[312] +#define __pyx_kp_u_Function_cuMemRangeGetAttributes __pyx_string_tab[313] +#define __pyx_kp_u_Function_cuMemRelease_not_found __pyx_string_tab[314] +#define __pyx_kp_u_Function_cuMemRetainAllocationHa __pyx_string_tab[315] +#define __pyx_kp_u_Function_cuMemSetAccess_not_foun __pyx_string_tab[316] +#define __pyx_kp_u_Function_cuMemUnmap_not_found __pyx_string_tab[317] +#define __pyx_kp_u_Function_cuMemcpy2DAsync_v2_not __pyx_string_tab[318] +#define __pyx_kp_u_Function_cuMemcpy2DUnaligned_v2 __pyx_string_tab[319] +#define __pyx_kp_u_Function_cuMemcpy2D_v2_not_found __pyx_string_tab[320] +#define __pyx_kp_u_Function_cuMemcpy3DAsync_v2_not __pyx_string_tab[321] +#define __pyx_kp_u_Function_cuMemcpy3DBatchAsync_no __pyx_string_tab[322] +#define __pyx_kp_u_Function_cuMemcpy3DPeerAsync_not __pyx_string_tab[323] +#define __pyx_kp_u_Function_cuMemcpy3DPeer_not_foun __pyx_string_tab[324] +#define __pyx_kp_u_Function_cuMemcpy3D_v2_not_found __pyx_string_tab[325] +#define __pyx_kp_u_Function_cuMemcpyAsync_not_found __pyx_string_tab[326] +#define __pyx_kp_u_Function_cuMemcpyAtoA_v2_not_fou __pyx_string_tab[327] +#define __pyx_kp_u_Function_cuMemcpyAtoD_v2_not_fou __pyx_string_tab[328] +#define __pyx_kp_u_Function_cuMemcpyAtoHAsync_v2_no __pyx_string_tab[329] +#define __pyx_kp_u_Function_cuMemcpyAtoH_v2_not_fou __pyx_string_tab[330] +#define __pyx_kp_u_Function_cuMemcpyBatchAsync_not __pyx_string_tab[331] +#define __pyx_kp_u_Function_cuMemcpyDtoA_v2_not_fou __pyx_string_tab[332] +#define __pyx_kp_u_Function_cuMemcpyDtoDAsync_v2_no __pyx_string_tab[333] +#define __pyx_kp_u_Function_cuMemcpyDtoD_v2_not_fou __pyx_string_tab[334] +#define __pyx_kp_u_Function_cuMemcpyDtoHAsync_v2_no __pyx_string_tab[335] +#define __pyx_kp_u_Function_cuMemcpyDtoH_v2_not_fou __pyx_string_tab[336] +#define __pyx_kp_u_Function_cuMemcpyHtoAAsync_v2_no __pyx_string_tab[337] +#define __pyx_kp_u_Function_cuMemcpyHtoA_v2_not_fou __pyx_string_tab[338] +#define __pyx_kp_u_Function_cuMemcpyHtoDAsync_v2_no __pyx_string_tab[339] +#define __pyx_kp_u_Function_cuMemcpyHtoD_v2_not_fou __pyx_string_tab[340] +#define __pyx_kp_u_Function_cuMemcpyPeerAsync_not_f __pyx_string_tab[341] +#define __pyx_kp_u_Function_cuMemcpyPeer_not_found __pyx_string_tab[342] +#define __pyx_kp_u_Function_cuMemcpy_not_found __pyx_string_tab[343] +#define __pyx_kp_u_Function_cuMemsetD16Async_not_fo __pyx_string_tab[344] +#define __pyx_kp_u_Function_cuMemsetD16_v2_not_foun __pyx_string_tab[345] +#define __pyx_kp_u_Function_cuMemsetD2D16Async_not __pyx_string_tab[346] +#define __pyx_kp_u_Function_cuMemsetD2D16_v2_not_fo __pyx_string_tab[347] +#define __pyx_kp_u_Function_cuMemsetD2D32Async_not __pyx_string_tab[348] +#define __pyx_kp_u_Function_cuMemsetD2D32_v2_not_fo __pyx_string_tab[349] +#define __pyx_kp_u_Function_cuMemsetD2D8Async_not_f __pyx_string_tab[350] +#define __pyx_kp_u_Function_cuMemsetD2D8_v2_not_fou __pyx_string_tab[351] +#define __pyx_kp_u_Function_cuMemsetD32Async_not_fo __pyx_string_tab[352] +#define __pyx_kp_u_Function_cuMemsetD32_v2_not_foun __pyx_string_tab[353] +#define __pyx_kp_u_Function_cuMemsetD8Async_not_fou __pyx_string_tab[354] +#define __pyx_kp_u_Function_cuMemsetD8_v2_not_found __pyx_string_tab[355] +#define __pyx_kp_u_Function_cuMipmappedArrayCreate __pyx_string_tab[356] +#define __pyx_kp_u_Function_cuMipmappedArrayDestroy __pyx_string_tab[357] +#define __pyx_kp_u_Function_cuMipmappedArrayGetLeve __pyx_string_tab[358] +#define __pyx_kp_u_Function_cuMipmappedArrayGetMemo __pyx_string_tab[359] +#define __pyx_kp_u_Function_cuMipmappedArrayGetSpar __pyx_string_tab[360] +#define __pyx_kp_u_Function_cuModuleEnumerateFuncti __pyx_string_tab[361] +#define __pyx_kp_u_Function_cuModuleGetFunctionCoun __pyx_string_tab[362] +#define __pyx_kp_u_Function_cuModuleGetFunction_not __pyx_string_tab[363] +#define __pyx_kp_u_Function_cuModuleGetGlobal_v2_no __pyx_string_tab[364] +#define __pyx_kp_u_Function_cuModuleGetLoadingMode __pyx_string_tab[365] +#define __pyx_kp_u_Function_cuModuleGetSurfRef_not __pyx_string_tab[366] +#define __pyx_kp_u_Function_cuModuleGetTexRef_not_f __pyx_string_tab[367] +#define __pyx_kp_u_Function_cuModuleLoadDataEx_not __pyx_string_tab[368] +#define __pyx_kp_u_Function_cuModuleLoadData_not_fo __pyx_string_tab[369] +#define __pyx_kp_u_Function_cuModuleLoadFatBinary_n __pyx_string_tab[370] +#define __pyx_kp_u_Function_cuModuleLoad_not_found __pyx_string_tab[371] +#define __pyx_kp_u_Function_cuModuleUnload_not_foun __pyx_string_tab[372] +#define __pyx_kp_u_Function_cuMulticastAddDevice_no __pyx_string_tab[373] +#define __pyx_kp_u_Function_cuMulticastBindAddr_not __pyx_string_tab[374] +#define __pyx_kp_u_Function_cuMulticastBindMem_not __pyx_string_tab[375] +#define __pyx_kp_u_Function_cuMulticastCreate_not_f __pyx_string_tab[376] +#define __pyx_kp_u_Function_cuMulticastGetGranulari __pyx_string_tab[377] +#define __pyx_kp_u_Function_cuMulticastUnbind_not_f __pyx_string_tab[378] +#define __pyx_kp_u_Function_cuOccupancyAvailableDyn __pyx_string_tab[379] +#define __pyx_kp_u_Function_cuOccupancyMaxActiveBlo __pyx_string_tab[380] +#define __pyx_kp_u_Function_cuOccupancyMaxActiveBlo_2 __pyx_string_tab[381] +#define __pyx_kp_u_Function_cuOccupancyMaxActiveClu __pyx_string_tab[382] +#define __pyx_kp_u_Function_cuOccupancyMaxPotential __pyx_string_tab[383] +#define __pyx_kp_u_Function_cuOccupancyMaxPotential_2 __pyx_string_tab[384] +#define __pyx_kp_u_Function_cuOccupancyMaxPotential_3 __pyx_string_tab[385] +#define __pyx_kp_u_Function_cuParamSetSize_not_foun __pyx_string_tab[386] +#define __pyx_kp_u_Function_cuParamSetTexRef_not_fo __pyx_string_tab[387] +#define __pyx_kp_u_Function_cuParamSetf_not_found __pyx_string_tab[388] +#define __pyx_kp_u_Function_cuParamSeti_not_found __pyx_string_tab[389] +#define __pyx_kp_u_Function_cuParamSetv_not_found __pyx_string_tab[390] +#define __pyx_kp_u_Function_cuPointerGetAttribute_n __pyx_string_tab[391] +#define __pyx_kp_u_Function_cuPointerGetAttributes __pyx_string_tab[392] +#define __pyx_kp_u_Function_cuPointerSetAttribute_n __pyx_string_tab[393] +#define __pyx_kp_u_Function_cuProfilerStart_not_fou __pyx_string_tab[394] +#define __pyx_kp_u_Function_cuProfilerStop_not_foun __pyx_string_tab[395] +#define __pyx_kp_u_Function_cuSignalExternalSemapho __pyx_string_tab[396] +#define __pyx_kp_u_Function_cuStreamAddCallback_not __pyx_string_tab[397] +#define __pyx_kp_u_Function_cuStreamAttachMemAsync __pyx_string_tab[398] +#define __pyx_kp_u_Function_cuStreamBatchMemOp_v2_n __pyx_string_tab[399] +#define __pyx_kp_u_Function_cuStreamBeginCaptureToG __pyx_string_tab[400] +#define __pyx_kp_u_Function_cuStreamBeginCapture_v2 __pyx_string_tab[401] +#define __pyx_kp_u_Function_cuStreamCopyAttributes __pyx_string_tab[402] +#define __pyx_kp_u_Function_cuStreamCreateWithPrior __pyx_string_tab[403] +#define __pyx_kp_u_Function_cuStreamCreate_not_foun __pyx_string_tab[404] +#define __pyx_kp_u_Function_cuStreamDestroy_v2_not __pyx_string_tab[405] +#define __pyx_kp_u_Function_cuStreamEndCapture_not __pyx_string_tab[406] +#define __pyx_kp_u_Function_cuStreamGetAttribute_no __pyx_string_tab[407] +#define __pyx_kp_u_Function_cuStreamGetCaptureInfo __pyx_string_tab[408] +#define __pyx_kp_u_Function_cuStreamGetCaptureInfo_2 __pyx_string_tab[409] +#define __pyx_kp_u_Function_cuStreamGetCtx_not_foun __pyx_string_tab[410] +#define __pyx_kp_u_Function_cuStreamGetCtx_v2_not_f __pyx_string_tab[411] +#define __pyx_kp_u_Function_cuStreamGetDevice_not_f __pyx_string_tab[412] +#define __pyx_kp_u_Function_cuStreamGetFlags_not_fo __pyx_string_tab[413] +#define __pyx_kp_u_Function_cuStreamGetGreenCtx_not __pyx_string_tab[414] +#define __pyx_kp_u_Function_cuStreamGetId_not_found __pyx_string_tab[415] +#define __pyx_kp_u_Function_cuStreamGetPriority_not __pyx_string_tab[416] +#define __pyx_kp_u_Function_cuStreamIsCapturing_not __pyx_string_tab[417] +#define __pyx_kp_u_Function_cuStreamQuery_not_found __pyx_string_tab[418] +#define __pyx_kp_u_Function_cuStreamSetAttribute_no __pyx_string_tab[419] +#define __pyx_kp_u_Function_cuStreamSynchronize_not __pyx_string_tab[420] +#define __pyx_kp_u_Function_cuStreamUpdateCaptureDe __pyx_string_tab[421] +#define __pyx_kp_u_Function_cuStreamUpdateCaptureDe_2 __pyx_string_tab[422] +#define __pyx_kp_u_Function_cuStreamWaitEvent_not_f __pyx_string_tab[423] +#define __pyx_kp_u_Function_cuStreamWaitValue32_v2 __pyx_string_tab[424] +#define __pyx_kp_u_Function_cuStreamWaitValue64_v2 __pyx_string_tab[425] +#define __pyx_kp_u_Function_cuStreamWriteValue32_v2 __pyx_string_tab[426] +#define __pyx_kp_u_Function_cuStreamWriteValue64_v2 __pyx_string_tab[427] +#define __pyx_kp_u_Function_cuSurfObjectCreate_not __pyx_string_tab[428] +#define __pyx_kp_u_Function_cuSurfObjectDestroy_not __pyx_string_tab[429] +#define __pyx_kp_u_Function_cuSurfObjectGetResource __pyx_string_tab[430] +#define __pyx_kp_u_Function_cuSurfRefGetArray_not_f __pyx_string_tab[431] +#define __pyx_kp_u_Function_cuSurfRefSetArray_not_f __pyx_string_tab[432] +#define __pyx_kp_u_Function_cuTensorMapEncodeIm2col __pyx_string_tab[433] +#define __pyx_kp_u_Function_cuTensorMapEncodeIm2col_2 __pyx_string_tab[434] +#define __pyx_kp_u_Function_cuTensorMapEncodeTiled __pyx_string_tab[435] +#define __pyx_kp_u_Function_cuTensorMapReplaceAddre __pyx_string_tab[436] +#define __pyx_kp_u_Function_cuTexObjectCreate_not_f __pyx_string_tab[437] +#define __pyx_kp_u_Function_cuTexObjectDestroy_not __pyx_string_tab[438] +#define __pyx_kp_u_Function_cuTexObjectGetResourceD __pyx_string_tab[439] +#define __pyx_kp_u_Function_cuTexObjectGetResourceV __pyx_string_tab[440] +#define __pyx_kp_u_Function_cuTexObjectGetTextureDe __pyx_string_tab[441] +#define __pyx_kp_u_Function_cuTexRefCreate_not_foun __pyx_string_tab[442] +#define __pyx_kp_u_Function_cuTexRefDestroy_not_fou __pyx_string_tab[443] +#define __pyx_kp_u_Function_cuTexRefGetAddressMode __pyx_string_tab[444] +#define __pyx_kp_u_Function_cuTexRefGetAddress_v2_n __pyx_string_tab[445] +#define __pyx_kp_u_Function_cuTexRefGetArray_not_fo __pyx_string_tab[446] +#define __pyx_kp_u_Function_cuTexRefGetBorderColor __pyx_string_tab[447] +#define __pyx_kp_u_Function_cuTexRefGetFilterMode_n __pyx_string_tab[448] +#define __pyx_kp_u_Function_cuTexRefGetFlags_not_fo __pyx_string_tab[449] +#define __pyx_kp_u_Function_cuTexRefGetFormat_not_f __pyx_string_tab[450] +#define __pyx_kp_u_Function_cuTexRefGetMaxAnisotrop __pyx_string_tab[451] +#define __pyx_kp_u_Function_cuTexRefGetMipmapFilter __pyx_string_tab[452] +#define __pyx_kp_u_Function_cuTexRefGetMipmapLevelB __pyx_string_tab[453] +#define __pyx_kp_u_Function_cuTexRefGetMipmapLevelC __pyx_string_tab[454] +#define __pyx_kp_u_Function_cuTexRefGetMipmappedArr __pyx_string_tab[455] +#define __pyx_kp_u_Function_cuTexRefSetAddress2D_v3 __pyx_string_tab[456] +#define __pyx_kp_u_Function_cuTexRefSetAddressMode __pyx_string_tab[457] +#define __pyx_kp_u_Function_cuTexRefSetAddress_v2_n __pyx_string_tab[458] +#define __pyx_kp_u_Function_cuTexRefSetArray_not_fo __pyx_string_tab[459] +#define __pyx_kp_u_Function_cuTexRefSetBorderColor __pyx_string_tab[460] +#define __pyx_kp_u_Function_cuTexRefSetFilterMode_n __pyx_string_tab[461] +#define __pyx_kp_u_Function_cuTexRefSetFlags_not_fo __pyx_string_tab[462] +#define __pyx_kp_u_Function_cuTexRefSetFormat_not_f __pyx_string_tab[463] +#define __pyx_kp_u_Function_cuTexRefSetMaxAnisotrop __pyx_string_tab[464] +#define __pyx_kp_u_Function_cuTexRefSetMipmapFilter __pyx_string_tab[465] +#define __pyx_kp_u_Function_cuTexRefSetMipmapLevelB __pyx_string_tab[466] +#define __pyx_kp_u_Function_cuTexRefSetMipmapLevelC __pyx_string_tab[467] +#define __pyx_kp_u_Function_cuTexRefSetMipmappedArr __pyx_string_tab[468] +#define __pyx_kp_u_Function_cuThreadExchangeStreamC __pyx_string_tab[469] +#define __pyx_kp_u_Function_cuUserObjectCreate_not __pyx_string_tab[470] +#define __pyx_kp_u_Function_cuUserObjectRelease_not __pyx_string_tab[471] +#define __pyx_kp_u_Function_cuUserObjectRetain_not __pyx_string_tab[472] +#define __pyx_kp_u_Function_cuVDPAUCtxCreate_v2_not __pyx_string_tab[473] +#define __pyx_kp_u_Function_cuVDPAUGetDevice_not_fo __pyx_string_tab[474] +#define __pyx_kp_u_Function_cuWaitExternalSemaphore __pyx_string_tab[475] +#define __pyx_n_u_Lock __pyx_string_tab[476] +#define __pyx_kp_u_Note_that_Cython_is_deliberately __pyx_string_tab[477] +#define __pyx_n_u_RuntimeError __pyx_string_tab[478] +#define __pyx_kp_u_add_note __pyx_string_tab[479] +#define __pyx_n_u_asyncio_coroutines __pyx_string_tab[480] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[481] +#define __pyx_n_u_cuArray3DCreate_v2 __pyx_string_tab[482] +#define __pyx_n_u_cuArray3DGetDescriptor_v2 __pyx_string_tab[483] +#define __pyx_n_u_cuArrayCreate_v2 __pyx_string_tab[484] +#define __pyx_n_u_cuArrayDestroy __pyx_string_tab[485] +#define __pyx_n_u_cuArrayGetDescriptor_v2 __pyx_string_tab[486] +#define __pyx_n_u_cuArrayGetMemoryRequirements __pyx_string_tab[487] +#define __pyx_n_u_cuArrayGetPlane __pyx_string_tab[488] +#define __pyx_n_u_cuArrayGetSparseProperties __pyx_string_tab[489] +#define __pyx_n_u_cuCheckpointProcessCheckpoint __pyx_string_tab[490] +#define __pyx_n_u_cuCheckpointProcessGetRestoreT __pyx_string_tab[491] +#define __pyx_n_u_cuCheckpointProcessGetState __pyx_string_tab[492] +#define __pyx_n_u_cuCheckpointProcessLock __pyx_string_tab[493] +#define __pyx_n_u_cuCheckpointProcessRestore __pyx_string_tab[494] +#define __pyx_n_u_cuCheckpointProcessUnlock __pyx_string_tab[495] +#define __pyx_n_u_cuCoredumpGetAttribute __pyx_string_tab[496] +#define __pyx_n_u_cuCoredumpGetAttributeGlobal __pyx_string_tab[497] +#define __pyx_n_u_cuCoredumpSetAttribute __pyx_string_tab[498] +#define __pyx_n_u_cuCoredumpSetAttributeGlobal __pyx_string_tab[499] +#define __pyx_n_u_cuCtxAttach __pyx_string_tab[500] +#define __pyx_n_u_cuCtxCreate_v2 __pyx_string_tab[501] +#define __pyx_n_u_cuCtxCreate_v3 __pyx_string_tab[502] +#define __pyx_n_u_cuCtxCreate_v4 __pyx_string_tab[503] +#define __pyx_n_u_cuCtxDestroy_v2 __pyx_string_tab[504] +#define __pyx_n_u_cuCtxDetach __pyx_string_tab[505] +#define __pyx_n_u_cuCtxDisablePeerAccess __pyx_string_tab[506] +#define __pyx_n_u_cuCtxEnablePeerAccess __pyx_string_tab[507] +#define __pyx_n_u_cuCtxFromGreenCtx __pyx_string_tab[508] +#define __pyx_n_u_cuCtxGetApiVersion __pyx_string_tab[509] +#define __pyx_n_u_cuCtxGetCacheConfig __pyx_string_tab[510] +#define __pyx_n_u_cuCtxGetCurrent __pyx_string_tab[511] +#define __pyx_n_u_cuCtxGetDevResource __pyx_string_tab[512] +#define __pyx_n_u_cuCtxGetDevice __pyx_string_tab[513] +#define __pyx_n_u_cuCtxGetExecAffinity __pyx_string_tab[514] +#define __pyx_n_u_cuCtxGetFlags __pyx_string_tab[515] +#define __pyx_n_u_cuCtxGetId __pyx_string_tab[516] +#define __pyx_n_u_cuCtxGetLimit __pyx_string_tab[517] +#define __pyx_n_u_cuCtxGetSharedMemConfig __pyx_string_tab[518] +#define __pyx_n_u_cuCtxGetStreamPriorityRange __pyx_string_tab[519] +#define __pyx_n_u_cuCtxPopCurrent_v2 __pyx_string_tab[520] +#define __pyx_n_u_cuCtxPushCurrent_v2 __pyx_string_tab[521] +#define __pyx_n_u_cuCtxRecordEvent __pyx_string_tab[522] +#define __pyx_n_u_cuCtxResetPersistingL2Cache __pyx_string_tab[523] +#define __pyx_n_u_cuCtxSetCacheConfig __pyx_string_tab[524] +#define __pyx_n_u_cuCtxSetCurrent __pyx_string_tab[525] +#define __pyx_n_u_cuCtxSetFlags __pyx_string_tab[526] +#define __pyx_n_u_cuCtxSetLimit __pyx_string_tab[527] +#define __pyx_n_u_cuCtxSetSharedMemConfig __pyx_string_tab[528] +#define __pyx_n_u_cuCtxSynchronize __pyx_string_tab[529] +#define __pyx_n_u_cuCtxWaitEvent __pyx_string_tab[530] +#define __pyx_n_u_cuDestroyExternalMemory __pyx_string_tab[531] +#define __pyx_n_u_cuDestroyExternalSemaphore __pyx_string_tab[532] +#define __pyx_n_u_cuDevResourceGenerateDesc __pyx_string_tab[533] +#define __pyx_n_u_cuDevSmResourceSplitByCount __pyx_string_tab[534] +#define __pyx_n_u_cuDeviceCanAccessPeer __pyx_string_tab[535] +#define __pyx_n_u_cuDeviceComputeCapability __pyx_string_tab[536] +#define __pyx_n_u_cuDeviceGet __pyx_string_tab[537] +#define __pyx_n_u_cuDeviceGetAttribute __pyx_string_tab[538] +#define __pyx_n_u_cuDeviceGetByPCIBusId __pyx_string_tab[539] +#define __pyx_n_u_cuDeviceGetCount __pyx_string_tab[540] +#define __pyx_n_u_cuDeviceGetDefaultMemPool __pyx_string_tab[541] +#define __pyx_n_u_cuDeviceGetDevResource __pyx_string_tab[542] +#define __pyx_n_u_cuDeviceGetExecAffinitySupport __pyx_string_tab[543] +#define __pyx_n_u_cuDeviceGetGraphMemAttribute __pyx_string_tab[544] +#define __pyx_n_u_cuDeviceGetLuid __pyx_string_tab[545] +#define __pyx_n_u_cuDeviceGetMemPool __pyx_string_tab[546] +#define __pyx_n_u_cuDeviceGetName __pyx_string_tab[547] +#define __pyx_n_u_cuDeviceGetNvSciSyncAttributes __pyx_string_tab[548] +#define __pyx_n_u_cuDeviceGetP2PAttribute __pyx_string_tab[549] +#define __pyx_n_u_cuDeviceGetPCIBusId __pyx_string_tab[550] +#define __pyx_n_u_cuDeviceGetProperties __pyx_string_tab[551] +#define __pyx_n_u_cuDeviceGetTexture1DLinearMaxW __pyx_string_tab[552] +#define __pyx_n_u_cuDeviceGetUuid __pyx_string_tab[553] +#define __pyx_n_u_cuDeviceGetUuid_v2 __pyx_string_tab[554] +#define __pyx_n_u_cuDeviceGraphMemTrim __pyx_string_tab[555] +#define __pyx_n_u_cuDevicePrimaryCtxGetState __pyx_string_tab[556] +#define __pyx_n_u_cuDevicePrimaryCtxRelease_v2 __pyx_string_tab[557] +#define __pyx_n_u_cuDevicePrimaryCtxReset_v2 __pyx_string_tab[558] +#define __pyx_n_u_cuDevicePrimaryCtxRetain __pyx_string_tab[559] +#define __pyx_n_u_cuDevicePrimaryCtxSetFlags_v2 __pyx_string_tab[560] +#define __pyx_n_u_cuDeviceRegisterAsyncNotificat __pyx_string_tab[561] +#define __pyx_n_u_cuDeviceSetGraphMemAttribute __pyx_string_tab[562] +#define __pyx_n_u_cuDeviceSetMemPool __pyx_string_tab[563] +#define __pyx_n_u_cuDeviceTotalMem_v2 __pyx_string_tab[564] +#define __pyx_n_u_cuDeviceUnregisterAsyncNotific __pyx_string_tab[565] +#define __pyx_n_u_cuDriverGetVersion __pyx_string_tab[566] +#define __pyx_n_u_cuEGLStreamConsumerAcquireFram __pyx_string_tab[567] +#define __pyx_n_u_cuEGLStreamConsumerConnect __pyx_string_tab[568] +#define __pyx_n_u_cuEGLStreamConsumerConnectWith __pyx_string_tab[569] +#define __pyx_n_u_cuEGLStreamConsumerDisconnect __pyx_string_tab[570] +#define __pyx_n_u_cuEGLStreamConsumerReleaseFram __pyx_string_tab[571] +#define __pyx_n_u_cuEGLStreamProducerConnect __pyx_string_tab[572] +#define __pyx_n_u_cuEGLStreamProducerDisconnect __pyx_string_tab[573] +#define __pyx_n_u_cuEGLStreamProducerPresentFram __pyx_string_tab[574] +#define __pyx_n_u_cuEGLStreamProducerReturnFrame __pyx_string_tab[575] +#define __pyx_n_u_cuEventCreate __pyx_string_tab[576] +#define __pyx_n_u_cuEventCreateFromEGLSync __pyx_string_tab[577] +#define __pyx_n_u_cuEventDestroy_v2 __pyx_string_tab[578] +#define __pyx_n_u_cuEventElapsedTime __pyx_string_tab[579] +#define __pyx_n_u_cuEventElapsedTime_v2 __pyx_string_tab[580] +#define __pyx_n_u_cuEventQuery __pyx_string_tab[581] +#define __pyx_n_u_cuEventRecord __pyx_string_tab[582] +#define __pyx_n_u_cuEventRecordWithFlags __pyx_string_tab[583] +#define __pyx_n_u_cuEventSynchronize __pyx_string_tab[584] +#define __pyx_n_u_cuExternalMemoryGetMappedBuffe __pyx_string_tab[585] +#define __pyx_n_u_cuExternalMemoryGetMappedMipma __pyx_string_tab[586] +#define __pyx_n_u_cuFlushGPUDirectRDMAWrites __pyx_string_tab[587] +#define __pyx_n_u_cuFuncGetAttribute __pyx_string_tab[588] +#define __pyx_n_u_cuFuncGetModule __pyx_string_tab[589] +#define __pyx_n_u_cuFuncGetName __pyx_string_tab[590] +#define __pyx_n_u_cuFuncGetParamInfo __pyx_string_tab[591] +#define __pyx_n_u_cuFuncIsLoaded __pyx_string_tab[592] +#define __pyx_n_u_cuFuncLoad __pyx_string_tab[593] +#define __pyx_n_u_cuFuncSetAttribute __pyx_string_tab[594] +#define __pyx_n_u_cuFuncSetBlockShape __pyx_string_tab[595] +#define __pyx_n_u_cuFuncSetCacheConfig __pyx_string_tab[596] +#define __pyx_n_u_cuFuncSetSharedMemConfig __pyx_string_tab[597] +#define __pyx_n_u_cuFuncSetSharedSize __pyx_string_tab[598] +#define __pyx_n_u_cuGLGetDevices_v2 __pyx_string_tab[599] +#define __pyx_n_u_cuGetErrorName __pyx_string_tab[600] +#define __pyx_n_u_cuGetErrorString __pyx_string_tab[601] +#define __pyx_n_u_cuGetExportTable __pyx_string_tab[602] +#define __pyx_n_u_cuGetProcAddress_v2 __pyx_string_tab[603] +#define __pyx_n_u_cuGraphAddBatchMemOpNode __pyx_string_tab[604] +#define __pyx_n_u_cuGraphAddChildGraphNode __pyx_string_tab[605] +#define __pyx_n_u_cuGraphAddDependencies __pyx_string_tab[606] +#define __pyx_n_u_cuGraphAddDependencies_v2 __pyx_string_tab[607] +#define __pyx_n_u_cuGraphAddEmptyNode __pyx_string_tab[608] +#define __pyx_n_u_cuGraphAddEventRecordNode __pyx_string_tab[609] +#define __pyx_n_u_cuGraphAddEventWaitNode __pyx_string_tab[610] +#define __pyx_n_u_cuGraphAddExternalSemaphoresSi __pyx_string_tab[611] +#define __pyx_n_u_cuGraphAddExternalSemaphoresWa __pyx_string_tab[612] +#define __pyx_n_u_cuGraphAddHostNode __pyx_string_tab[613] +#define __pyx_n_u_cuGraphAddKernelNode_v2 __pyx_string_tab[614] +#define __pyx_n_u_cuGraphAddMemAllocNode __pyx_string_tab[615] +#define __pyx_n_u_cuGraphAddMemFreeNode __pyx_string_tab[616] +#define __pyx_n_u_cuGraphAddMemcpyNode __pyx_string_tab[617] +#define __pyx_n_u_cuGraphAddMemsetNode __pyx_string_tab[618] +#define __pyx_n_u_cuGraphAddNode __pyx_string_tab[619] +#define __pyx_n_u_cuGraphAddNode_v2 __pyx_string_tab[620] +#define __pyx_n_u_cuGraphBatchMemOpNodeGetParams __pyx_string_tab[621] +#define __pyx_n_u_cuGraphBatchMemOpNodeSetParams __pyx_string_tab[622] +#define __pyx_n_u_cuGraphChildGraphNodeGetGraph __pyx_string_tab[623] +#define __pyx_n_u_cuGraphClone __pyx_string_tab[624] +#define __pyx_n_u_cuGraphConditionalHandleCreate __pyx_string_tab[625] +#define __pyx_n_u_cuGraphCreate __pyx_string_tab[626] +#define __pyx_n_u_cuGraphDebugDotPrint __pyx_string_tab[627] +#define __pyx_n_u_cuGraphDestroy __pyx_string_tab[628] +#define __pyx_n_u_cuGraphDestroyNode __pyx_string_tab[629] +#define __pyx_n_u_cuGraphEventRecordNodeGetEvent __pyx_string_tab[630] +#define __pyx_n_u_cuGraphEventRecordNodeSetEvent __pyx_string_tab[631] +#define __pyx_n_u_cuGraphEventWaitNodeGetEvent __pyx_string_tab[632] +#define __pyx_n_u_cuGraphEventWaitNodeSetEvent __pyx_string_tab[633] +#define __pyx_n_u_cuGraphExecBatchMemOpNodeSetPa __pyx_string_tab[634] +#define __pyx_n_u_cuGraphExecChildGraphNodeSetPa __pyx_string_tab[635] +#define __pyx_n_u_cuGraphExecDestroy __pyx_string_tab[636] +#define __pyx_n_u_cuGraphExecEventRecordNodeSetE __pyx_string_tab[637] +#define __pyx_n_u_cuGraphExecEventWaitNodeSetEve __pyx_string_tab[638] +#define __pyx_n_u_cuGraphExecExternalSemaphoresS __pyx_string_tab[639] +#define __pyx_n_u_cuGraphExecExternalSemaphoresW __pyx_string_tab[640] +#define __pyx_n_u_cuGraphExecGetFlags __pyx_string_tab[641] +#define __pyx_n_u_cuGraphExecHostNodeSetParams __pyx_string_tab[642] +#define __pyx_n_u_cuGraphExecKernelNodeSetParams __pyx_string_tab[643] +#define __pyx_n_u_cuGraphExecMemcpyNodeSetParams __pyx_string_tab[644] +#define __pyx_n_u_cuGraphExecMemsetNodeSetParams __pyx_string_tab[645] +#define __pyx_n_u_cuGraphExecNodeSetParams __pyx_string_tab[646] +#define __pyx_n_u_cuGraphExecUpdate_v2 __pyx_string_tab[647] +#define __pyx_n_u_cuGraphExternalSemaphoresSigna __pyx_string_tab[648] +#define __pyx_n_u_cuGraphExternalSemaphoresSigna_2 __pyx_string_tab[649] +#define __pyx_n_u_cuGraphExternalSemaphoresWaitN __pyx_string_tab[650] +#define __pyx_n_u_cuGraphExternalSemaphoresWaitN_2 __pyx_string_tab[651] +#define __pyx_n_u_cuGraphGetEdges __pyx_string_tab[652] +#define __pyx_n_u_cuGraphGetEdges_v2 __pyx_string_tab[653] +#define __pyx_n_u_cuGraphGetNodes __pyx_string_tab[654] +#define __pyx_n_u_cuGraphGetRootNodes __pyx_string_tab[655] +#define __pyx_n_u_cuGraphHostNodeGetParams __pyx_string_tab[656] +#define __pyx_n_u_cuGraphHostNodeSetParams __pyx_string_tab[657] +#define __pyx_n_u_cuGraphInstantiateWithFlags __pyx_string_tab[658] +#define __pyx_n_u_cuGraphInstantiateWithParams __pyx_string_tab[659] +#define __pyx_n_u_cuGraphKernelNodeCopyAttribute __pyx_string_tab[660] +#define __pyx_n_u_cuGraphKernelNodeGetAttribute __pyx_string_tab[661] +#define __pyx_n_u_cuGraphKernelNodeGetParams_v2 __pyx_string_tab[662] +#define __pyx_n_u_cuGraphKernelNodeSetAttribute __pyx_string_tab[663] +#define __pyx_n_u_cuGraphKernelNodeSetParams_v2 __pyx_string_tab[664] +#define __pyx_n_u_cuGraphLaunch __pyx_string_tab[665] +#define __pyx_n_u_cuGraphMemAllocNodeGetParams __pyx_string_tab[666] +#define __pyx_n_u_cuGraphMemFreeNodeGetParams __pyx_string_tab[667] +#define __pyx_n_u_cuGraphMemcpyNodeGetParams __pyx_string_tab[668] +#define __pyx_n_u_cuGraphMemcpyNodeSetParams __pyx_string_tab[669] +#define __pyx_n_u_cuGraphMemsetNodeGetParams __pyx_string_tab[670] +#define __pyx_n_u_cuGraphMemsetNodeSetParams __pyx_string_tab[671] +#define __pyx_n_u_cuGraphNodeFindInClone __pyx_string_tab[672] +#define __pyx_n_u_cuGraphNodeGetDependencies __pyx_string_tab[673] +#define __pyx_n_u_cuGraphNodeGetDependencies_v2 __pyx_string_tab[674] +#define __pyx_n_u_cuGraphNodeGetDependentNodes __pyx_string_tab[675] +#define __pyx_n_u_cuGraphNodeGetDependentNodes_v __pyx_string_tab[676] +#define __pyx_n_u_cuGraphNodeGetEnabled __pyx_string_tab[677] +#define __pyx_n_u_cuGraphNodeGetType __pyx_string_tab[678] +#define __pyx_n_u_cuGraphNodeSetEnabled __pyx_string_tab[679] +#define __pyx_n_u_cuGraphNodeSetParams __pyx_string_tab[680] +#define __pyx_n_u_cuGraphReleaseUserObject __pyx_string_tab[681] +#define __pyx_n_u_cuGraphRemoveDependencies __pyx_string_tab[682] +#define __pyx_n_u_cuGraphRemoveDependencies_v2 __pyx_string_tab[683] +#define __pyx_n_u_cuGraphRetainUserObject __pyx_string_tab[684] +#define __pyx_n_u_cuGraphUpload __pyx_string_tab[685] +#define __pyx_n_u_cuGraphicsEGLRegisterImage __pyx_string_tab[686] +#define __pyx_n_u_cuGraphicsGLRegisterBuffer __pyx_string_tab[687] +#define __pyx_n_u_cuGraphicsGLRegisterImage __pyx_string_tab[688] +#define __pyx_n_u_cuGraphicsMapResources __pyx_string_tab[689] +#define __pyx_n_u_cuGraphicsResourceGetMappedEgl __pyx_string_tab[690] +#define __pyx_n_u_cuGraphicsResourceGetMappedMip __pyx_string_tab[691] +#define __pyx_n_u_cuGraphicsResourceGetMappedPoi __pyx_string_tab[692] +#define __pyx_n_u_cuGraphicsResourceSetMapFlags __pyx_string_tab[693] +#define __pyx_n_u_cuGraphicsSubResourceGetMapped __pyx_string_tab[694] +#define __pyx_n_u_cuGraphicsUnmapResources __pyx_string_tab[695] +#define __pyx_n_u_cuGraphicsUnregisterResource __pyx_string_tab[696] +#define __pyx_n_u_cuGraphicsVDPAURegisterOutputS __pyx_string_tab[697] +#define __pyx_n_u_cuGraphicsVDPAURegisterVideoSu __pyx_string_tab[698] +#define __pyx_n_u_cuGreenCtxCreate __pyx_string_tab[699] +#define __pyx_n_u_cuGreenCtxDestroy __pyx_string_tab[700] +#define __pyx_n_u_cuGreenCtxGetDevResource __pyx_string_tab[701] +#define __pyx_n_u_cuGreenCtxRecordEvent __pyx_string_tab[702] +#define __pyx_n_u_cuGreenCtxStreamCreate __pyx_string_tab[703] +#define __pyx_n_u_cuGreenCtxWaitEvent __pyx_string_tab[704] +#define __pyx_n_u_cuImportExternalMemory __pyx_string_tab[705] +#define __pyx_n_u_cuImportExternalSemaphore __pyx_string_tab[706] +#define __pyx_n_u_cuInit __pyx_string_tab[707] +#define __pyx_n_u_cuIpcCloseMemHandle __pyx_string_tab[708] +#define __pyx_n_u_cuIpcGetEventHandle __pyx_string_tab[709] +#define __pyx_n_u_cuIpcGetMemHandle __pyx_string_tab[710] +#define __pyx_n_u_cuIpcOpenEventHandle __pyx_string_tab[711] +#define __pyx_n_u_cuIpcOpenMemHandle_v2 __pyx_string_tab[712] +#define __pyx_n_u_cuKernelGetAttribute __pyx_string_tab[713] +#define __pyx_n_u_cuKernelGetFunction __pyx_string_tab[714] +#define __pyx_n_u_cuKernelGetLibrary __pyx_string_tab[715] +#define __pyx_n_u_cuKernelGetName __pyx_string_tab[716] +#define __pyx_n_u_cuKernelGetParamInfo __pyx_string_tab[717] +#define __pyx_n_u_cuKernelSetAttribute __pyx_string_tab[718] +#define __pyx_n_u_cuKernelSetCacheConfig __pyx_string_tab[719] +#define __pyx_n_u_cuLaunch __pyx_string_tab[720] +#define __pyx_n_u_cuLaunchCooperativeKernel __pyx_string_tab[721] +#define __pyx_n_u_cuLaunchCooperativeKernelMulti __pyx_string_tab[722] +#define __pyx_n_u_cuLaunchGrid __pyx_string_tab[723] +#define __pyx_n_u_cuLaunchGridAsync __pyx_string_tab[724] +#define __pyx_n_u_cuLaunchHostFunc __pyx_string_tab[725] +#define __pyx_n_u_cuLaunchKernel __pyx_string_tab[726] +#define __pyx_n_u_cuLaunchKernelEx __pyx_string_tab[727] +#define __pyx_n_u_cuLibraryEnumerateKernels __pyx_string_tab[728] +#define __pyx_n_u_cuLibraryGetGlobal __pyx_string_tab[729] +#define __pyx_n_u_cuLibraryGetKernel __pyx_string_tab[730] +#define __pyx_n_u_cuLibraryGetKernelCount __pyx_string_tab[731] +#define __pyx_n_u_cuLibraryGetManaged __pyx_string_tab[732] +#define __pyx_n_u_cuLibraryGetModule __pyx_string_tab[733] +#define __pyx_n_u_cuLibraryGetUnifiedFunction __pyx_string_tab[734] +#define __pyx_n_u_cuLibraryLoadData __pyx_string_tab[735] +#define __pyx_n_u_cuLibraryLoadFromFile __pyx_string_tab[736] +#define __pyx_n_u_cuLibraryUnload __pyx_string_tab[737] +#define __pyx_n_u_cuLinkAddData_v2 __pyx_string_tab[738] +#define __pyx_n_u_cuLinkAddFile_v2 __pyx_string_tab[739] +#define __pyx_n_u_cuLinkComplete __pyx_string_tab[740] +#define __pyx_n_u_cuLinkCreate_v2 __pyx_string_tab[741] +#define __pyx_n_u_cuLinkDestroy __pyx_string_tab[742] +#define __pyx_n_u_cuLogsCurrent __pyx_string_tab[743] +#define __pyx_n_u_cuLogsDumpToFile __pyx_string_tab[744] +#define __pyx_n_u_cuLogsDumpToMemory __pyx_string_tab[745] +#define __pyx_n_u_cuLogsRegisterCallback __pyx_string_tab[746] +#define __pyx_n_u_cuLogsUnregisterCallback __pyx_string_tab[747] +#define __pyx_n_u_cuMemAddressFree __pyx_string_tab[748] +#define __pyx_n_u_cuMemAddressReserve __pyx_string_tab[749] +#define __pyx_n_u_cuMemAdvise __pyx_string_tab[750] +#define __pyx_n_u_cuMemAdvise_v2 __pyx_string_tab[751] +#define __pyx_n_u_cuMemAllocAsync __pyx_string_tab[752] +#define __pyx_n_u_cuMemAllocFromPoolAsync __pyx_string_tab[753] +#define __pyx_n_u_cuMemAllocHost_v2 __pyx_string_tab[754] +#define __pyx_n_u_cuMemAllocManaged __pyx_string_tab[755] +#define __pyx_n_u_cuMemAllocPitch_v2 __pyx_string_tab[756] +#define __pyx_n_u_cuMemAlloc_v2 __pyx_string_tab[757] +#define __pyx_n_u_cuMemBatchDecompressAsync __pyx_string_tab[758] +#define __pyx_n_u_cuMemCreate __pyx_string_tab[759] +#define __pyx_n_u_cuMemExportToShareableHandle __pyx_string_tab[760] +#define __pyx_n_u_cuMemFreeAsync __pyx_string_tab[761] +#define __pyx_n_u_cuMemFreeHost __pyx_string_tab[762] +#define __pyx_n_u_cuMemFree_v2 __pyx_string_tab[763] +#define __pyx_n_u_cuMemGetAccess __pyx_string_tab[764] +#define __pyx_n_u_cuMemGetAddressRange_v2 __pyx_string_tab[765] +#define __pyx_n_u_cuMemGetAllocationGranularity __pyx_string_tab[766] +#define __pyx_n_u_cuMemGetAllocationPropertiesFr __pyx_string_tab[767] +#define __pyx_n_u_cuMemGetHandleForAddressRange __pyx_string_tab[768] +#define __pyx_n_u_cuMemGetInfo_v2 __pyx_string_tab[769] +#define __pyx_n_u_cuMemHostAlloc __pyx_string_tab[770] +#define __pyx_n_u_cuMemHostGetDevicePointer_v2 __pyx_string_tab[771] +#define __pyx_n_u_cuMemHostGetFlags __pyx_string_tab[772] +#define __pyx_n_u_cuMemHostRegister_v2 __pyx_string_tab[773] +#define __pyx_n_u_cuMemHostUnregister __pyx_string_tab[774] +#define __pyx_n_u_cuMemImportFromShareableHandle __pyx_string_tab[775] +#define __pyx_n_u_cuMemMap __pyx_string_tab[776] +#define __pyx_n_u_cuMemMapArrayAsync __pyx_string_tab[777] +#define __pyx_n_u_cuMemPoolCreate __pyx_string_tab[778] +#define __pyx_n_u_cuMemPoolDestroy __pyx_string_tab[779] +#define __pyx_n_u_cuMemPoolExportPointer __pyx_string_tab[780] +#define __pyx_n_u_cuMemPoolExportToShareableHand __pyx_string_tab[781] +#define __pyx_n_u_cuMemPoolGetAccess __pyx_string_tab[782] +#define __pyx_n_u_cuMemPoolGetAttribute __pyx_string_tab[783] +#define __pyx_n_u_cuMemPoolImportFromShareableHa __pyx_string_tab[784] +#define __pyx_n_u_cuMemPoolImportPointer __pyx_string_tab[785] +#define __pyx_n_u_cuMemPoolSetAccess __pyx_string_tab[786] +#define __pyx_n_u_cuMemPoolSetAttribute __pyx_string_tab[787] +#define __pyx_n_u_cuMemPoolTrimTo __pyx_string_tab[788] +#define __pyx_n_u_cuMemPrefetchAsync __pyx_string_tab[789] +#define __pyx_n_u_cuMemPrefetchAsync_v2 __pyx_string_tab[790] +#define __pyx_n_u_cuMemRangeGetAttribute __pyx_string_tab[791] +#define __pyx_n_u_cuMemRangeGetAttributes __pyx_string_tab[792] +#define __pyx_n_u_cuMemRelease __pyx_string_tab[793] +#define __pyx_n_u_cuMemRetainAllocationHandle __pyx_string_tab[794] +#define __pyx_n_u_cuMemSetAccess __pyx_string_tab[795] +#define __pyx_n_u_cuMemUnmap __pyx_string_tab[796] +#define __pyx_n_u_cuMemcpy __pyx_string_tab[797] +#define __pyx_n_u_cuMemcpy2DAsync_v2 __pyx_string_tab[798] +#define __pyx_n_u_cuMemcpy2DUnaligned_v2 __pyx_string_tab[799] +#define __pyx_n_u_cuMemcpy2D_v2 __pyx_string_tab[800] +#define __pyx_n_u_cuMemcpy3DAsync_v2 __pyx_string_tab[801] +#define __pyx_n_u_cuMemcpy3DBatchAsync __pyx_string_tab[802] +#define __pyx_n_u_cuMemcpy3DPeer __pyx_string_tab[803] +#define __pyx_n_u_cuMemcpy3DPeerAsync __pyx_string_tab[804] +#define __pyx_n_u_cuMemcpy3D_v2 __pyx_string_tab[805] +#define __pyx_n_u_cuMemcpyAsync __pyx_string_tab[806] +#define __pyx_n_u_cuMemcpyAtoA_v2 __pyx_string_tab[807] +#define __pyx_n_u_cuMemcpyAtoD_v2 __pyx_string_tab[808] +#define __pyx_n_u_cuMemcpyAtoHAsync_v2 __pyx_string_tab[809] +#define __pyx_n_u_cuMemcpyAtoH_v2 __pyx_string_tab[810] +#define __pyx_n_u_cuMemcpyBatchAsync __pyx_string_tab[811] +#define __pyx_n_u_cuMemcpyDtoA_v2 __pyx_string_tab[812] +#define __pyx_n_u_cuMemcpyDtoDAsync_v2 __pyx_string_tab[813] +#define __pyx_n_u_cuMemcpyDtoD_v2 __pyx_string_tab[814] +#define __pyx_n_u_cuMemcpyDtoHAsync_v2 __pyx_string_tab[815] +#define __pyx_n_u_cuMemcpyDtoH_v2 __pyx_string_tab[816] +#define __pyx_n_u_cuMemcpyHtoAAsync_v2 __pyx_string_tab[817] +#define __pyx_n_u_cuMemcpyHtoA_v2 __pyx_string_tab[818] +#define __pyx_n_u_cuMemcpyHtoDAsync_v2 __pyx_string_tab[819] +#define __pyx_n_u_cuMemcpyHtoD_v2 __pyx_string_tab[820] +#define __pyx_n_u_cuMemcpyPeer __pyx_string_tab[821] +#define __pyx_n_u_cuMemcpyPeerAsync __pyx_string_tab[822] +#define __pyx_n_u_cuMemsetD16Async __pyx_string_tab[823] +#define __pyx_n_u_cuMemsetD16_v2 __pyx_string_tab[824] +#define __pyx_n_u_cuMemsetD2D16Async __pyx_string_tab[825] +#define __pyx_n_u_cuMemsetD2D16_v2 __pyx_string_tab[826] +#define __pyx_n_u_cuMemsetD2D32Async __pyx_string_tab[827] +#define __pyx_n_u_cuMemsetD2D32_v2 __pyx_string_tab[828] +#define __pyx_n_u_cuMemsetD2D8Async __pyx_string_tab[829] +#define __pyx_n_u_cuMemsetD2D8_v2 __pyx_string_tab[830] +#define __pyx_n_u_cuMemsetD32Async __pyx_string_tab[831] +#define __pyx_n_u_cuMemsetD32_v2 __pyx_string_tab[832] +#define __pyx_n_u_cuMemsetD8Async __pyx_string_tab[833] +#define __pyx_n_u_cuMemsetD8_v2 __pyx_string_tab[834] +#define __pyx_n_u_cuMipmappedArrayCreate __pyx_string_tab[835] +#define __pyx_n_u_cuMipmappedArrayDestroy __pyx_string_tab[836] +#define __pyx_n_u_cuMipmappedArrayGetLevel __pyx_string_tab[837] +#define __pyx_n_u_cuMipmappedArrayGetMemoryRequi __pyx_string_tab[838] +#define __pyx_n_u_cuMipmappedArrayGetSparsePrope __pyx_string_tab[839] +#define __pyx_n_u_cuModuleEnumerateFunctions __pyx_string_tab[840] +#define __pyx_n_u_cuModuleGetFunction __pyx_string_tab[841] +#define __pyx_n_u_cuModuleGetFunctionCount __pyx_string_tab[842] +#define __pyx_n_u_cuModuleGetGlobal_v2 __pyx_string_tab[843] +#define __pyx_n_u_cuModuleGetLoadingMode __pyx_string_tab[844] +#define __pyx_n_u_cuModuleGetSurfRef __pyx_string_tab[845] +#define __pyx_n_u_cuModuleGetTexRef __pyx_string_tab[846] +#define __pyx_n_u_cuModuleLoad __pyx_string_tab[847] +#define __pyx_n_u_cuModuleLoadData __pyx_string_tab[848] +#define __pyx_n_u_cuModuleLoadDataEx __pyx_string_tab[849] +#define __pyx_n_u_cuModuleLoadFatBinary __pyx_string_tab[850] +#define __pyx_n_u_cuModuleUnload __pyx_string_tab[851] +#define __pyx_n_u_cuMulticastAddDevice __pyx_string_tab[852] +#define __pyx_n_u_cuMulticastBindAddr __pyx_string_tab[853] +#define __pyx_n_u_cuMulticastBindMem __pyx_string_tab[854] +#define __pyx_n_u_cuMulticastCreate __pyx_string_tab[855] +#define __pyx_n_u_cuMulticastGetGranularity __pyx_string_tab[856] +#define __pyx_n_u_cuMulticastUnbind __pyx_string_tab[857] +#define __pyx_n_u_cuOccupancyAvailableDynamicSMe __pyx_string_tab[858] +#define __pyx_n_u_cuOccupancyMaxActiveBlocksPerM __pyx_string_tab[859] +#define __pyx_n_u_cuOccupancyMaxActiveBlocksPerM_2 __pyx_string_tab[860] +#define __pyx_n_u_cuOccupancyMaxActiveClusters __pyx_string_tab[861] +#define __pyx_n_u_cuOccupancyMaxPotentialBlockSi __pyx_string_tab[862] +#define __pyx_n_u_cuOccupancyMaxPotentialBlockSi_2 __pyx_string_tab[863] +#define __pyx_n_u_cuOccupancyMaxPotentialCluster __pyx_string_tab[864] +#define __pyx_n_u_cuParamSetSize __pyx_string_tab[865] +#define __pyx_n_u_cuParamSetTexRef __pyx_string_tab[866] +#define __pyx_n_u_cuParamSetf __pyx_string_tab[867] +#define __pyx_n_u_cuParamSeti __pyx_string_tab[868] +#define __pyx_n_u_cuParamSetv __pyx_string_tab[869] +#define __pyx_n_u_cuPointerGetAttribute __pyx_string_tab[870] +#define __pyx_n_u_cuPointerGetAttributes __pyx_string_tab[871] +#define __pyx_n_u_cuPointerSetAttribute __pyx_string_tab[872] +#define __pyx_n_u_cuProfilerStart __pyx_string_tab[873] +#define __pyx_n_u_cuProfilerStop __pyx_string_tab[874] +#define __pyx_n_u_cuSignalExternalSemaphoresAsyn __pyx_string_tab[875] +#define __pyx_n_u_cuStreamAddCallback __pyx_string_tab[876] +#define __pyx_n_u_cuStreamAttachMemAsync __pyx_string_tab[877] +#define __pyx_n_u_cuStreamBatchMemOp_v2 __pyx_string_tab[878] +#define __pyx_n_u_cuStreamBeginCaptureToGraph __pyx_string_tab[879] +#define __pyx_n_u_cuStreamBeginCapture_v2 __pyx_string_tab[880] +#define __pyx_n_u_cuStreamCopyAttributes __pyx_string_tab[881] +#define __pyx_n_u_cuStreamCreate __pyx_string_tab[882] +#define __pyx_n_u_cuStreamCreateWithPriority __pyx_string_tab[883] +#define __pyx_n_u_cuStreamDestroy_v2 __pyx_string_tab[884] +#define __pyx_n_u_cuStreamEndCapture __pyx_string_tab[885] +#define __pyx_n_u_cuStreamGetAttribute __pyx_string_tab[886] +#define __pyx_n_u_cuStreamGetCaptureInfo_v2 __pyx_string_tab[887] +#define __pyx_n_u_cuStreamGetCaptureInfo_v3 __pyx_string_tab[888] +#define __pyx_n_u_cuStreamGetCtx __pyx_string_tab[889] +#define __pyx_n_u_cuStreamGetCtx_v2 __pyx_string_tab[890] +#define __pyx_n_u_cuStreamGetDevice __pyx_string_tab[891] +#define __pyx_n_u_cuStreamGetFlags __pyx_string_tab[892] +#define __pyx_n_u_cuStreamGetGreenCtx __pyx_string_tab[893] +#define __pyx_n_u_cuStreamGetId __pyx_string_tab[894] +#define __pyx_n_u_cuStreamGetPriority __pyx_string_tab[895] +#define __pyx_n_u_cuStreamIsCapturing __pyx_string_tab[896] +#define __pyx_n_u_cuStreamQuery __pyx_string_tab[897] +#define __pyx_n_u_cuStreamSetAttribute __pyx_string_tab[898] +#define __pyx_n_u_cuStreamSynchronize __pyx_string_tab[899] +#define __pyx_n_u_cuStreamUpdateCaptureDependenc __pyx_string_tab[900] +#define __pyx_n_u_cuStreamUpdateCaptureDependenc_2 __pyx_string_tab[901] +#define __pyx_n_u_cuStreamWaitEvent __pyx_string_tab[902] +#define __pyx_n_u_cuStreamWaitValue32_v2 __pyx_string_tab[903] +#define __pyx_n_u_cuStreamWaitValue64_v2 __pyx_string_tab[904] +#define __pyx_n_u_cuStreamWriteValue32_v2 __pyx_string_tab[905] +#define __pyx_n_u_cuStreamWriteValue64_v2 __pyx_string_tab[906] +#define __pyx_n_u_cuSurfObjectCreate __pyx_string_tab[907] +#define __pyx_n_u_cuSurfObjectDestroy __pyx_string_tab[908] +#define __pyx_n_u_cuSurfObjectGetResourceDesc __pyx_string_tab[909] +#define __pyx_n_u_cuSurfRefGetArray __pyx_string_tab[910] +#define __pyx_n_u_cuSurfRefSetArray __pyx_string_tab[911] +#define __pyx_n_u_cuTensorMapEncodeIm2col __pyx_string_tab[912] +#define __pyx_n_u_cuTensorMapEncodeIm2colWide __pyx_string_tab[913] +#define __pyx_n_u_cuTensorMapEncodeTiled __pyx_string_tab[914] +#define __pyx_n_u_cuTensorMapReplaceAddress __pyx_string_tab[915] +#define __pyx_n_u_cuTexObjectCreate __pyx_string_tab[916] +#define __pyx_n_u_cuTexObjectDestroy __pyx_string_tab[917] +#define __pyx_n_u_cuTexObjectGetResourceDesc __pyx_string_tab[918] +#define __pyx_n_u_cuTexObjectGetResourceViewDesc __pyx_string_tab[919] +#define __pyx_n_u_cuTexObjectGetTextureDesc __pyx_string_tab[920] +#define __pyx_n_u_cuTexRefCreate __pyx_string_tab[921] +#define __pyx_n_u_cuTexRefDestroy __pyx_string_tab[922] +#define __pyx_n_u_cuTexRefGetAddressMode __pyx_string_tab[923] +#define __pyx_n_u_cuTexRefGetAddress_v2 __pyx_string_tab[924] +#define __pyx_n_u_cuTexRefGetArray __pyx_string_tab[925] +#define __pyx_n_u_cuTexRefGetBorderColor __pyx_string_tab[926] +#define __pyx_n_u_cuTexRefGetFilterMode __pyx_string_tab[927] +#define __pyx_n_u_cuTexRefGetFlags __pyx_string_tab[928] +#define __pyx_n_u_cuTexRefGetFormat __pyx_string_tab[929] +#define __pyx_n_u_cuTexRefGetMaxAnisotropy __pyx_string_tab[930] +#define __pyx_n_u_cuTexRefGetMipmapFilterMode __pyx_string_tab[931] +#define __pyx_n_u_cuTexRefGetMipmapLevelBias __pyx_string_tab[932] +#define __pyx_n_u_cuTexRefGetMipmapLevelClamp __pyx_string_tab[933] +#define __pyx_n_u_cuTexRefGetMipmappedArray __pyx_string_tab[934] +#define __pyx_n_u_cuTexRefSetAddress2D_v3 __pyx_string_tab[935] +#define __pyx_n_u_cuTexRefSetAddressMode __pyx_string_tab[936] +#define __pyx_n_u_cuTexRefSetAddress_v2 __pyx_string_tab[937] +#define __pyx_n_u_cuTexRefSetArray __pyx_string_tab[938] +#define __pyx_n_u_cuTexRefSetBorderColor __pyx_string_tab[939] +#define __pyx_n_u_cuTexRefSetFilterMode __pyx_string_tab[940] +#define __pyx_n_u_cuTexRefSetFlags __pyx_string_tab[941] +#define __pyx_n_u_cuTexRefSetFormat __pyx_string_tab[942] +#define __pyx_n_u_cuTexRefSetMaxAnisotropy __pyx_string_tab[943] +#define __pyx_n_u_cuTexRefSetMipmapFilterMode __pyx_string_tab[944] +#define __pyx_n_u_cuTexRefSetMipmapLevelBias __pyx_string_tab[945] +#define __pyx_n_u_cuTexRefSetMipmapLevelClamp __pyx_string_tab[946] +#define __pyx_n_u_cuTexRefSetMipmappedArray __pyx_string_tab[947] +#define __pyx_n_u_cuThreadExchangeStreamCaptureM __pyx_string_tab[948] +#define __pyx_n_u_cuUserObjectCreate __pyx_string_tab[949] +#define __pyx_n_u_cuUserObjectRelease __pyx_string_tab[950] +#define __pyx_n_u_cuUserObjectRetain __pyx_string_tab[951] +#define __pyx_n_u_cuVDPAUCtxCreate_v2 __pyx_string_tab[952] +#define __pyx_n_u_cuVDPAUGetDevice __pyx_string_tab[953] +#define __pyx_n_u_cuWaitExternalSemaphoresAsync __pyx_string_tab[954] +#define __pyx_n_u_cuda_bindings__bindings_cydriver __pyx_string_tab[955] +#define __pyx_kp_u_cuda_bindings__bindings_cydriver_2 __pyx_string_tab[956] +#define __pyx_n_u_default __pyx_string_tab[957] +#define __pyx_n_u_encoding __pyx_string_tab[958] +#define __pyx_n_u_enter __pyx_string_tab[959] +#define __pyx_n_u_exit __pyx_string_tab[960] +#define __pyx_n_u_func __pyx_string_tab[961] +#define __pyx_n_u_getenv __pyx_string_tab[962] +#define __pyx_n_u_initializing __pyx_string_tab[963] +#define __pyx_n_u_inspect_function_pointer __pyx_string_tab[964] +#define __pyx_n_u_inspect_function_pointers __pyx_string_tab[965] +#define __pyx_n_u_is_coroutine __pyx_string_tab[966] +#define __pyx_kp_u_libcuda_so_1 __pyx_string_tab[967] +#define __pyx_n_u_main __pyx_string_tab[968] +#define __pyx_n_u_maxsize __pyx_string_tab[969] +#define __pyx_n_u_module __pyx_string_tab[970] +#define __pyx_n_u_name __pyx_string_tab[971] +#define __pyx_n_u_name_2 __pyx_string_tab[972] +#define __pyx_n_u_os __pyx_string_tab[973] +#define __pyx_n_u_pop __pyx_string_tab[974] +#define __pyx_n_u_qualname __pyx_string_tab[975] +#define __pyx_n_u_set_name __pyx_string_tab[976] +#define __pyx_n_u_spec __pyx_string_tab[977] +#define __pyx_n_u_sys __pyx_string_tab[978] +#define __pyx_n_u_test __pyx_string_tab[979] +#define __pyx_n_u_threading __pyx_string_tab[980] +#define __pyx_kp_u_utf_8 __pyx_string_tab[981] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_CLEAR(clear_module_state->__pyx_FusedFunctionType); + #endif + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + for (int i=0; i<1; ++i) { Py_CLEAR(clear_module_state->__pyx_tuple[i]); } + for (int i=0; i<2; ++i) { Py_CLEAR(clear_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<982; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + Py_CLEAR(clear_module_state->__pyx_int_0); + Py_CLEAR(clear_module_state->__pyx_int_4294967296); + return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_VISIT(traverse_module_state->__pyx_FusedFunctionType); + #endif + for (int i=0; i<1; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_tuple[i]); } + for (int i=0; i<2; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<982; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + __Pyx_VISIT_CONST(traverse_module_state->__pyx_int_0); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_int_4294967296); + return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "cuda/bindings/_bindings/cydriver.pyx":492 + * cdef __cuGetProcAddress_v2_T _F_cuGetProcAddress_v2 = NULL + * + * cdef int _cuPythonInit() except -1 nogil: # <<<<<<<<<<<<<< + * global __cuPythonInit + * +*/ + +static int __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuPythonInit(void) { + int __pyx_v_usePTDS; + char __pyx_v_libPath[0x104]; + int __pyx_v_status; + PyObject *__pyx_v_path = NULL; + void *__pyx_v_handle; + __pyx_t_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2_T __pyx_v__F_cuGetProcAddress_v2; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + PyObject *__pyx_t_9 = NULL; + int __pyx_t_10; + int __pyx_t_11; + Py_ssize_t __pyx_t_12; + char const *__pyx_t_13; + CUresult __pyx_t_14; + PyObject *__pyx_t_15 = NULL; + int __pyx_t_16; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuPythonInit", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":498 + * cdef char libPath[260] + * + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * usePTDS = os.getenv('CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM', default=0) + * +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + /*with:*/ { + __pyx_t_1 = __Pyx_PyObject_LookupSpecial(__pyx_v_4cuda_8bindings_9_bindings_8cydriver___symbol_lock, __pyx_mstate_global->__pyx_n_u_exit); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 498, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = NULL; + __pyx_t_4 = __Pyx_PyObject_LookupSpecial(__pyx_v_4cuda_8bindings_9_bindings_8cydriver___symbol_lock, __pyx_mstate_global->__pyx_n_u_enter); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 498, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (likely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_4, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_4, __pyx_callargs+__pyx_t_5, (1-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 498, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + /*try:*/ { + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_6, &__pyx_t_7, &__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":499 + * + * with gil, __symbol_lock: + * usePTDS = os.getenv('CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM', default=0) # <<<<<<<<<<<<<< + * + * # Load library +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 499, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_getenv); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 499, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_9))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_9); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_9); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_9, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_4, __pyx_mstate_global->__pyx_n_u_CUDA_PYTHON_CUDA_PER_THREAD_DEFA}; + __pyx_t_3 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 499, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_default, __pyx_mstate_global->__pyx_int_0, __pyx_t_3, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 499, __pyx_L10_error) + __pyx_t_2 = __Pyx_Object_Vectorcall_CallFromBuilder(__pyx_t_9, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 499, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_10 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 499, __pyx_L10_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_usePTDS = __pyx_t_10; + + /* "cuda/bindings/_bindings/cydriver.pyx":502 + * + * # Load library + * libPath[0] = 0 # <<<<<<<<<<<<<< + * status = loader.getCUDALibraryPath(libPath, sys.maxsize > 2**32) + * if status == 0 and len(libPath) != 0: +*/ + (__pyx_v_libPath[0]) = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":503 + * # Load library + * libPath[0] = 0 + * status = loader.getCUDALibraryPath(libPath, sys.maxsize > 2**32) # <<<<<<<<<<<<<< + * if status == 0 and len(libPath) != 0: + * path = libPath.decode('utf-8') +*/ + __Pyx_GetModuleGlobalName(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_sys); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 503, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_9 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_maxsize); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 503, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_9); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = PyObject_RichCompare(__pyx_t_9, __pyx_mstate_global->__pyx_int_4294967296, Py_GT); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 503, __pyx_L10_error) + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __pyx_t_10 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely((__pyx_t_10 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 503, __pyx_L10_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_v_status = getCUDALibraryPath(__pyx_v_libPath, __pyx_t_10); + + /* "cuda/bindings/_bindings/cydriver.pyx":504 + * libPath[0] = 0 + * status = loader.getCUDALibraryPath(libPath, sys.maxsize > 2**32) + * if status == 0 and len(libPath) != 0: # <<<<<<<<<<<<<< + * path = libPath.decode('utf-8') + * else: +*/ + __pyx_t_11 = (__pyx_v_status == 0); + if (__pyx_t_11) { + } else { + __pyx_t_10 = __pyx_t_11; + goto __pyx_L17_bool_binop_done; + } + __pyx_t_12 = __Pyx_ssize_strlen(__pyx_v_libPath); if (unlikely(__pyx_t_12 == ((Py_ssize_t)-1))) __PYX_ERR(0, 504, __pyx_L10_error) + __pyx_t_11 = (__pyx_t_12 != 0); + __pyx_t_10 = __pyx_t_11; + __pyx_L17_bool_binop_done:; + if (__pyx_t_10) { + + /* "cuda/bindings/_bindings/cydriver.pyx":505 + * status = loader.getCUDALibraryPath(libPath, sys.maxsize > 2**32) + * if status == 0 and len(libPath) != 0: + * path = libPath.decode('utf-8') # <<<<<<<<<<<<<< + * else: + * path = 'libcuda.so.1' +*/ + __pyx_t_12 = __Pyx_ssize_strlen(__pyx_v_libPath); if (unlikely(__pyx_t_12 == ((Py_ssize_t)-1))) __PYX_ERR(0, 505, __pyx_L10_error) + __pyx_t_2 = __Pyx_decode_c_string(__pyx_v_libPath, 0, __pyx_t_12, NULL, NULL, PyUnicode_DecodeUTF8); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 505, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_v_path = __pyx_t_2; + __pyx_t_2 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":504 + * libPath[0] = 0 + * status = loader.getCUDALibraryPath(libPath, sys.maxsize > 2**32) + * if status == 0 and len(libPath) != 0: # <<<<<<<<<<<<<< + * path = libPath.decode('utf-8') + * else: +*/ + goto __pyx_L16; + } + + /* "cuda/bindings/_bindings/cydriver.pyx":507 + * path = libPath.decode('utf-8') + * else: + * path = 'libcuda.so.1' # <<<<<<<<<<<<<< + * handle = dlfcn.dlopen(bytes(path, encoding='utf-8'), dlfcn.RTLD_NOW) + * if handle == NULL: +*/ + /*else*/ { + __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_libcuda_so_1); + __pyx_v_path = __pyx_mstate_global->__pyx_kp_u_libcuda_so_1; + } + __pyx_L16:; + + /* "cuda/bindings/_bindings/cydriver.pyx":508 + * else: + * path = 'libcuda.so.1' + * handle = dlfcn.dlopen(bytes(path, encoding='utf-8'), dlfcn.RTLD_NOW) # <<<<<<<<<<<<<< + * if handle == NULL: + * raise RuntimeError('Failed to dlopen ' + path) +*/ + __pyx_t_9 = NULL; + __Pyx_INCREF((PyObject *)(&PyBytes_Type)); + __pyx_t_3 = ((PyObject *)(&PyBytes_Type)); + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_9, __pyx_v_path}; + __pyx_t_4 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 508, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_4); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_encoding, __pyx_mstate_global->__pyx_kp_u_utf_8, __pyx_t_4, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 508, __pyx_L10_error) + __pyx_t_2 = __Pyx_Object_Vectorcall_CallFromBuilder(__pyx_t_3, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_4); + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 508, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_13 = __Pyx_PyBytes_AsString(__pyx_t_2); if (unlikely((!__pyx_t_13) && PyErr_Occurred())) __PYX_ERR(0, 508, __pyx_L10_error) + __pyx_v_handle = dlopen(__pyx_t_13, RTLD_NOW); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":509 + * path = 'libcuda.so.1' + * handle = dlfcn.dlopen(bytes(path, encoding='utf-8'), dlfcn.RTLD_NOW) + * if handle == NULL: # <<<<<<<<<<<<<< + * raise RuntimeError('Failed to dlopen ' + path) + * +*/ + __pyx_t_10 = (__pyx_v_handle == NULL); + if (unlikely(__pyx_t_10)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":510 + * handle = dlfcn.dlopen(bytes(path, encoding='utf-8'), dlfcn.RTLD_NOW) + * if handle == NULL: + * raise RuntimeError('Failed to dlopen ' + path) # <<<<<<<<<<<<<< + * + * # Get latest __cuGetProcAddress_v2 +*/ + __pyx_t_3 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_4 = __pyx_builtin_RuntimeError; + __pyx_t_9 = PyNumber_Add(__pyx_mstate_global->__pyx_kp_u_Failed_to_dlopen, __pyx_v_path); if (unlikely(!__pyx_t_9)) __PYX_ERR(0, 510, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_9); + __pyx_t_5 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_t_9}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_4, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 510, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 510, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":509 + * path = 'libcuda.so.1' + * handle = dlfcn.dlopen(bytes(path, encoding='utf-8'), dlfcn.RTLD_NOW) + * if handle == NULL: # <<<<<<<<<<<<<< + * raise RuntimeError('Failed to dlopen ' + path) + * +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":514 + * # Get latest __cuGetProcAddress_v2 + * global __cuGetProcAddress_v2 + * __cuGetProcAddress_v2 = dlfcn.dlsym(handle, 'cuGetProcAddress_v2') # <<<<<<<<<<<<<< + * + * # Load using cuGetProcAddress if available +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGetProcAddress_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":517 + * + * # Load using cuGetProcAddress if available + * if __cuGetProcAddress_v2 != NULL: # <<<<<<<<<<<<<< + * _F_cuGetProcAddress_v2 = <__cuGetProcAddress_v2_T>__cuGetProcAddress_v2 + * if usePTDS: +*/ + __pyx_t_10 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2 != NULL); + if (__pyx_t_10) { + + /* "cuda/bindings/_bindings/cydriver.pyx":518 + * # Load using cuGetProcAddress if available + * if __cuGetProcAddress_v2 != NULL: + * _F_cuGetProcAddress_v2 = <__cuGetProcAddress_v2_T>__cuGetProcAddress_v2 # <<<<<<<<<<<<<< + * if usePTDS: + * # Get all PTDS version of functions +*/ + __pyx_v__F_cuGetProcAddress_v2 = ((__pyx_t_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2_T)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2); + + /* "cuda/bindings/_bindings/cydriver.pyx":519 + * if __cuGetProcAddress_v2 != NULL: + * _F_cuGetProcAddress_v2 = <__cuGetProcAddress_v2_T>__cuGetProcAddress_v2 + * if usePTDS: # <<<<<<<<<<<<<< + * # Get all PTDS version of functions + * pass +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cydriver.pyx":523 + * pass + * global __cuMemcpy + * _F_cuGetProcAddress_v2('cuMemcpy', &__cuMemcpy, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyPeer + * _F_cuGetProcAddress_v2('cuMemcpyPeer', &__cuMemcpyPeer, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 523, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":525 + * _F_cuGetProcAddress_v2('cuMemcpy', &__cuMemcpy, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyPeer + * _F_cuGetProcAddress_v2('cuMemcpyPeer', &__cuMemcpyPeer, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyHtoD_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoD', &__cuMemcpyHtoD_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyPeer"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeer), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 525, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":527 + * _F_cuGetProcAddress_v2('cuMemcpyPeer', &__cuMemcpyPeer, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyHtoD_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoD', &__cuMemcpyHtoD_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyDtoH_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoH', &__cuMemcpyDtoH_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyHtoD"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoD_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 527, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":529 + * _F_cuGetProcAddress_v2('cuMemcpyHtoD', &__cuMemcpyHtoD_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyDtoH_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoH', &__cuMemcpyDtoH_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyDtoD_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoD', &__cuMemcpyDtoD_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyDtoH"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoH_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 529, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":531 + * _F_cuGetProcAddress_v2('cuMemcpyDtoH', &__cuMemcpyDtoH_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyDtoD_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoD', &__cuMemcpyDtoD_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyDtoA_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoA', &__cuMemcpyDtoA_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyDtoD"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoD_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 531, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":533 + * _F_cuGetProcAddress_v2('cuMemcpyDtoD', &__cuMemcpyDtoD_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyDtoA_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoA', &__cuMemcpyDtoA_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyAtoD_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoD', &__cuMemcpyAtoD_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyDtoA"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoA_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 533, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":535 + * _F_cuGetProcAddress_v2('cuMemcpyDtoA', &__cuMemcpyDtoA_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyAtoD_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoD', &__cuMemcpyAtoD_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyHtoA_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoA', &__cuMemcpyHtoA_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyAtoD"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoD_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 535, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":537 + * _F_cuGetProcAddress_v2('cuMemcpyAtoD', &__cuMemcpyAtoD_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyHtoA_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoA', &__cuMemcpyHtoA_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyAtoH_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoH', &__cuMemcpyAtoH_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyHtoA"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoA_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 537, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":539 + * _F_cuGetProcAddress_v2('cuMemcpyHtoA', &__cuMemcpyHtoA_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyAtoH_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoH', &__cuMemcpyAtoH_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyAtoA_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoA', &__cuMemcpyAtoA_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyAtoH"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoH_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 539, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":541 + * _F_cuGetProcAddress_v2('cuMemcpyAtoH', &__cuMemcpyAtoH_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyAtoA_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoA', &__cuMemcpyAtoA_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy2D_v2 + * _F_cuGetProcAddress_v2('cuMemcpy2D', &__cuMemcpy2D_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyAtoA"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoA_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 541, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":543 + * _F_cuGetProcAddress_v2('cuMemcpyAtoA', &__cuMemcpyAtoA_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpy2D_v2 + * _F_cuGetProcAddress_v2('cuMemcpy2D', &__cuMemcpy2D_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy2DUnaligned_v2 + * _F_cuGetProcAddress_v2('cuMemcpy2DUnaligned', &__cuMemcpy2DUnaligned_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy2D"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2D_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 543, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":545 + * _F_cuGetProcAddress_v2('cuMemcpy2D', &__cuMemcpy2D_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpy2DUnaligned_v2 + * _F_cuGetProcAddress_v2('cuMemcpy2DUnaligned', &__cuMemcpy2DUnaligned_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy3D_v2 + * _F_cuGetProcAddress_v2('cuMemcpy3D', &__cuMemcpy3D_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy2DUnaligned"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DUnaligned_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 545, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":547 + * _F_cuGetProcAddress_v2('cuMemcpy2DUnaligned', &__cuMemcpy2DUnaligned_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpy3D_v2 + * _F_cuGetProcAddress_v2('cuMemcpy3D', &__cuMemcpy3D_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeer + * _F_cuGetProcAddress_v2('cuMemcpy3DPeer', &__cuMemcpy3DPeer, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy3D"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3D_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 547, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":549 + * _F_cuGetProcAddress_v2('cuMemcpy3D', &__cuMemcpy3D_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpy3DPeer + * _F_cuGetProcAddress_v2('cuMemcpy3DPeer', &__cuMemcpy3DPeer, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyAsync + * _F_cuGetProcAddress_v2('cuMemcpyAsync', &__cuMemcpyAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy3DPeer"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeer), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 549, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":551 + * _F_cuGetProcAddress_v2('cuMemcpy3DPeer', &__cuMemcpy3DPeer, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyAsync + * _F_cuGetProcAddress_v2('cuMemcpyAsync', &__cuMemcpyAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyPeerAsync + * _F_cuGetProcAddress_v2('cuMemcpyPeerAsync', &__cuMemcpyPeerAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAsync), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 551, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":553 + * _F_cuGetProcAddress_v2('cuMemcpyAsync', &__cuMemcpyAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyPeerAsync + * _F_cuGetProcAddress_v2('cuMemcpyPeerAsync', &__cuMemcpyPeerAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyHtoDAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoDAsync', &__cuMemcpyHtoDAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyPeerAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeerAsync), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 553, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":555 + * _F_cuGetProcAddress_v2('cuMemcpyPeerAsync', &__cuMemcpyPeerAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyHtoDAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoDAsync', &__cuMemcpyHtoDAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyDtoHAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoHAsync', &__cuMemcpyDtoHAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyHtoDAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoDAsync_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 555, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":557 + * _F_cuGetProcAddress_v2('cuMemcpyHtoDAsync', &__cuMemcpyHtoDAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyDtoHAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoHAsync', &__cuMemcpyDtoHAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyDtoDAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoDAsync', &__cuMemcpyDtoDAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyDtoHAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoHAsync_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 557, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":559 + * _F_cuGetProcAddress_v2('cuMemcpyDtoHAsync', &__cuMemcpyDtoHAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyDtoDAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoDAsync', &__cuMemcpyDtoDAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyHtoAAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoAAsync', &__cuMemcpyHtoAAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyDtoDAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoDAsync_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 559, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":561 + * _F_cuGetProcAddress_v2('cuMemcpyDtoDAsync', &__cuMemcpyDtoDAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyHtoAAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoAAsync', &__cuMemcpyHtoAAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyAtoHAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoHAsync', &__cuMemcpyAtoHAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyHtoAAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoAAsync_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 561, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":563 + * _F_cuGetProcAddress_v2('cuMemcpyHtoAAsync', &__cuMemcpyHtoAAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyAtoHAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoHAsync', &__cuMemcpyAtoHAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy2DAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpy2DAsync', &__cuMemcpy2DAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyAtoHAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoHAsync_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 563, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":565 + * _F_cuGetProcAddress_v2('cuMemcpyAtoHAsync', &__cuMemcpyAtoHAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpy2DAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpy2DAsync', &__cuMemcpy2DAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy3DAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpy3DAsync', &__cuMemcpy3DAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy2DAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DAsync_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 565, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":567 + * _F_cuGetProcAddress_v2('cuMemcpy2DAsync', &__cuMemcpy2DAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpy3DAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpy3DAsync', &__cuMemcpy3DAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeerAsync + * _F_cuGetProcAddress_v2('cuMemcpy3DPeerAsync', &__cuMemcpy3DPeerAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy3DAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DAsync_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 567, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":569 + * _F_cuGetProcAddress_v2('cuMemcpy3DAsync', &__cuMemcpy3DAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpy3DPeerAsync + * _F_cuGetProcAddress_v2('cuMemcpy3DPeerAsync', &__cuMemcpy3DPeerAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyBatchAsync + * _F_cuGetProcAddress_v2('cuMemcpyBatchAsync', &__cuMemcpyBatchAsync, 12080, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy3DPeerAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeerAsync), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 569, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":571 + * _F_cuGetProcAddress_v2('cuMemcpy3DPeerAsync', &__cuMemcpy3DPeerAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpyBatchAsync + * _F_cuGetProcAddress_v2('cuMemcpyBatchAsync', &__cuMemcpyBatchAsync, 12080, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy3DBatchAsync + * _F_cuGetProcAddress_v2('cuMemcpy3DBatchAsync', &__cuMemcpy3DBatchAsync, 12080, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyBatchAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyBatchAsync), 0x2F30, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 571, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":573 + * _F_cuGetProcAddress_v2('cuMemcpyBatchAsync', &__cuMemcpyBatchAsync, 12080, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemcpy3DBatchAsync + * _F_cuGetProcAddress_v2('cuMemcpy3DBatchAsync', &__cuMemcpy3DBatchAsync, 12080, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD8_v2 + * _F_cuGetProcAddress_v2('cuMemsetD8', &__cuMemsetD8_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy3DBatchAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DBatchAsync), 0x2F30, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 573, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":575 + * _F_cuGetProcAddress_v2('cuMemcpy3DBatchAsync', &__cuMemcpy3DBatchAsync, 12080, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemsetD8_v2 + * _F_cuGetProcAddress_v2('cuMemsetD8', &__cuMemsetD8_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD16_v2 + * _F_cuGetProcAddress_v2('cuMemsetD16', &__cuMemsetD16_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD8"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 575, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":577 + * _F_cuGetProcAddress_v2('cuMemsetD8', &__cuMemsetD8_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemsetD16_v2 + * _F_cuGetProcAddress_v2('cuMemsetD16', &__cuMemsetD16_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD32_v2 + * _F_cuGetProcAddress_v2('cuMemsetD32', &__cuMemsetD32_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD16"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 577, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":579 + * _F_cuGetProcAddress_v2('cuMemsetD16', &__cuMemsetD16_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemsetD32_v2 + * _F_cuGetProcAddress_v2('cuMemsetD32', &__cuMemsetD32_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD2D8_v2 + * _F_cuGetProcAddress_v2('cuMemsetD2D8', &__cuMemsetD2D8_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD32"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 579, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":581 + * _F_cuGetProcAddress_v2('cuMemsetD32', &__cuMemsetD32_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemsetD2D8_v2 + * _F_cuGetProcAddress_v2('cuMemsetD2D8', &__cuMemsetD2D8_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD2D16_v2 + * _F_cuGetProcAddress_v2('cuMemsetD2D16', &__cuMemsetD2D16_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD2D8"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 581, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":583 + * _F_cuGetProcAddress_v2('cuMemsetD2D8', &__cuMemsetD2D8_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemsetD2D16_v2 + * _F_cuGetProcAddress_v2('cuMemsetD2D16', &__cuMemsetD2D16_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD2D32_v2 + * _F_cuGetProcAddress_v2('cuMemsetD2D32', &__cuMemsetD2D32_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD2D16"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 583, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":585 + * _F_cuGetProcAddress_v2('cuMemsetD2D16', &__cuMemsetD2D16_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemsetD2D32_v2 + * _F_cuGetProcAddress_v2('cuMemsetD2D32', &__cuMemsetD2D32_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD8Async + * _F_cuGetProcAddress_v2('cuMemsetD8Async', &__cuMemsetD8Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD2D32"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32_v2), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 585, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":587 + * _F_cuGetProcAddress_v2('cuMemsetD2D32', &__cuMemsetD2D32_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemsetD8Async + * _F_cuGetProcAddress_v2('cuMemsetD8Async', &__cuMemsetD8Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD16Async + * _F_cuGetProcAddress_v2('cuMemsetD16Async', &__cuMemsetD16Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD8Async"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8Async), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 587, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":589 + * _F_cuGetProcAddress_v2('cuMemsetD8Async', &__cuMemsetD8Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemsetD16Async + * _F_cuGetProcAddress_v2('cuMemsetD16Async', &__cuMemsetD16Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD32Async + * _F_cuGetProcAddress_v2('cuMemsetD32Async', &__cuMemsetD32Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD16Async"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16Async), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 589, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":591 + * _F_cuGetProcAddress_v2('cuMemsetD16Async', &__cuMemsetD16Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemsetD32Async + * _F_cuGetProcAddress_v2('cuMemsetD32Async', &__cuMemsetD32Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD2D8Async + * _F_cuGetProcAddress_v2('cuMemsetD2D8Async', &__cuMemsetD2D8Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD32Async"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32Async), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 591, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":593 + * _F_cuGetProcAddress_v2('cuMemsetD32Async', &__cuMemsetD32Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemsetD2D8Async + * _F_cuGetProcAddress_v2('cuMemsetD2D8Async', &__cuMemsetD2D8Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD2D16Async + * _F_cuGetProcAddress_v2('cuMemsetD2D16Async', &__cuMemsetD2D16Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD2D8Async"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8Async), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 593, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":595 + * _F_cuGetProcAddress_v2('cuMemsetD2D8Async', &__cuMemsetD2D8Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemsetD2D16Async + * _F_cuGetProcAddress_v2('cuMemsetD2D16Async', &__cuMemsetD2D16Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD2D32Async + * _F_cuGetProcAddress_v2('cuMemsetD2D32Async', &__cuMemsetD2D32Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD2D16Async"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16Async), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 595, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":597 + * _F_cuGetProcAddress_v2('cuMemsetD2D16Async', &__cuMemsetD2D16Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemsetD2D32Async + * _F_cuGetProcAddress_v2('cuMemsetD2D32Async', &__cuMemsetD2D32Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemBatchDecompressAsync + * _F_cuGetProcAddress_v2('cuMemBatchDecompressAsync', &__cuMemBatchDecompressAsync, 12060, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD2D32Async"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32Async), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 597, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":599 + * _F_cuGetProcAddress_v2('cuMemsetD2D32Async', &__cuMemsetD2D32Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemBatchDecompressAsync + * _F_cuGetProcAddress_v2('cuMemBatchDecompressAsync', &__cuMemBatchDecompressAsync, 12060, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemMapArrayAsync + * _F_cuGetProcAddress_v2('cuMemMapArrayAsync', &__cuMemMapArrayAsync, 11010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemBatchDecompressAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemBatchDecompressAsync), 0x2F1C, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 599, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":601 + * _F_cuGetProcAddress_v2('cuMemBatchDecompressAsync', &__cuMemBatchDecompressAsync, 12060, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemMapArrayAsync + * _F_cuGetProcAddress_v2('cuMemMapArrayAsync', &__cuMemMapArrayAsync, 11010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemFreeAsync + * _F_cuGetProcAddress_v2('cuMemFreeAsync', &__cuMemFreeAsync, 11020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemMapArrayAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMapArrayAsync), 0x2B02, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 601, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":603 + * _F_cuGetProcAddress_v2('cuMemMapArrayAsync', &__cuMemMapArrayAsync, 11010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemFreeAsync + * _F_cuGetProcAddress_v2('cuMemFreeAsync', &__cuMemFreeAsync, 11020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemAllocAsync + * _F_cuGetProcAddress_v2('cuMemAllocAsync', &__cuMemAllocAsync, 11020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemFreeAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeAsync), 0x2B0C, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 603, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":605 + * _F_cuGetProcAddress_v2('cuMemFreeAsync', &__cuMemFreeAsync, 11020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemAllocAsync + * _F_cuGetProcAddress_v2('cuMemAllocAsync', &__cuMemAllocAsync, 11020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemAllocFromPoolAsync + * _F_cuGetProcAddress_v2('cuMemAllocFromPoolAsync', &__cuMemAllocFromPoolAsync, 11020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemAllocAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocAsync), 0x2B0C, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 605, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":607 + * _F_cuGetProcAddress_v2('cuMemAllocAsync', &__cuMemAllocAsync, 11020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemAllocFromPoolAsync + * _F_cuGetProcAddress_v2('cuMemAllocFromPoolAsync', &__cuMemAllocFromPoolAsync, 11020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync + * _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync, 8000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemAllocFromPoolAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocFromPoolAsync), 0x2B0C, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 607, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":609 + * _F_cuGetProcAddress_v2('cuMemAllocFromPoolAsync', &__cuMemAllocFromPoolAsync, 11020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemPrefetchAsync + * _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync, 8000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync_v2 + * _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync_v2, 12020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPrefetchAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync), 0x1F40, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 609, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":611 + * _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync, 8000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuMemPrefetchAsync_v2 + * _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync_v2, 12020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetPriority + * _F_cuGetProcAddress_v2('cuStreamGetPriority', &__cuStreamGetPriority, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPrefetchAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync_v2), 0x2EF4, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 611, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":613 + * _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync_v2, 12020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamGetPriority + * _F_cuGetProcAddress_v2('cuStreamGetPriority', &__cuStreamGetPriority, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetDevice + * _F_cuGetProcAddress_v2('cuStreamGetDevice', &__cuStreamGetDevice, 12080, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetPriority"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetPriority), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 613, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":615 + * _F_cuGetProcAddress_v2('cuStreamGetPriority', &__cuStreamGetPriority, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamGetDevice + * _F_cuGetProcAddress_v2('cuStreamGetDevice', &__cuStreamGetDevice, 12080, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetFlags + * _F_cuGetProcAddress_v2('cuStreamGetFlags', &__cuStreamGetFlags, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetDevice"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetDevice), 0x2F30, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 615, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":617 + * _F_cuGetProcAddress_v2('cuStreamGetDevice', &__cuStreamGetDevice, 12080, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamGetFlags + * _F_cuGetProcAddress_v2('cuStreamGetFlags', &__cuStreamGetFlags, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetId + * _F_cuGetProcAddress_v2('cuStreamGetId', &__cuStreamGetId, 12000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetFlags), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 617, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":619 + * _F_cuGetProcAddress_v2('cuStreamGetFlags', &__cuStreamGetFlags, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamGetId + * _F_cuGetProcAddress_v2('cuStreamGetId', &__cuStreamGetId, 12000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetCtx + * _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx, 9020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetId"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetId), 0x2EE0, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 619, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":621 + * _F_cuGetProcAddress_v2('cuStreamGetId', &__cuStreamGetId, 12000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamGetCtx + * _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx, 9020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetCtx_v2 + * _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx_v2, 12050, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetCtx"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx), 0x233C, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 621, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":623 + * _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx, 9020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamGetCtx_v2 + * _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx_v2, 12050, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamWaitEvent + * _F_cuGetProcAddress_v2('cuStreamWaitEvent', &__cuStreamWaitEvent, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetCtx"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx_v2), 0x2F12, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 623, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":625 + * _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx_v2, 12050, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamWaitEvent + * _F_cuGetProcAddress_v2('cuStreamWaitEvent', &__cuStreamWaitEvent, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamAddCallback + * _F_cuGetProcAddress_v2('cuStreamAddCallback', &__cuStreamAddCallback, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamWaitEvent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitEvent), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 625, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":627 + * _F_cuGetProcAddress_v2('cuStreamWaitEvent', &__cuStreamWaitEvent, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamAddCallback + * _F_cuGetProcAddress_v2('cuStreamAddCallback', &__cuStreamAddCallback, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamBeginCapture_v2 + * _F_cuGetProcAddress_v2('cuStreamBeginCapture', &__cuStreamBeginCapture_v2, 10010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamAddCallback"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAddCallback), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 627, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":629 + * _F_cuGetProcAddress_v2('cuStreamAddCallback', &__cuStreamAddCallback, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamBeginCapture_v2 + * _F_cuGetProcAddress_v2('cuStreamBeginCapture', &__cuStreamBeginCapture_v2, 10010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamBeginCaptureToGraph + * _F_cuGetProcAddress_v2('cuStreamBeginCaptureToGraph', &__cuStreamBeginCaptureToGraph, 12030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamBeginCapture"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCapture_v2), 0x271A, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 629, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":631 + * _F_cuGetProcAddress_v2('cuStreamBeginCapture', &__cuStreamBeginCapture_v2, 10010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamBeginCaptureToGraph + * _F_cuGetProcAddress_v2('cuStreamBeginCaptureToGraph', &__cuStreamBeginCaptureToGraph, 12030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamEndCapture + * _F_cuGetProcAddress_v2('cuStreamEndCapture', &__cuStreamEndCapture, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamBeginCaptureToGraph"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCaptureToGraph), 0x2EFE, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 631, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":633 + * _F_cuGetProcAddress_v2('cuStreamBeginCaptureToGraph', &__cuStreamBeginCaptureToGraph, 12030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamEndCapture + * _F_cuGetProcAddress_v2('cuStreamEndCapture', &__cuStreamEndCapture, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamIsCapturing + * _F_cuGetProcAddress_v2('cuStreamIsCapturing', &__cuStreamIsCapturing, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamEndCapture"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamEndCapture), 0x2710, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 633, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":635 + * _F_cuGetProcAddress_v2('cuStreamEndCapture', &__cuStreamEndCapture, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamIsCapturing + * _F_cuGetProcAddress_v2('cuStreamIsCapturing', &__cuStreamIsCapturing, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v2 + * _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v2, 11030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamIsCapturing"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamIsCapturing), 0x2710, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 635, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":637 + * _F_cuGetProcAddress_v2('cuStreamIsCapturing', &__cuStreamIsCapturing, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamGetCaptureInfo_v2 + * _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v2, 11030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v3 + * _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v3, 12030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetCaptureInfo"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v2), 0x2B16, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 637, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":639 + * _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v2, 11030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamGetCaptureInfo_v3 + * _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v3, 12030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies + * _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies, 11030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetCaptureInfo"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v3), 0x2EFE, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 639, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":641 + * _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v3, 12030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamUpdateCaptureDependencies + * _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies, 11030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies_v2 + * _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies_v2, 12030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamUpdateCaptureDependencies"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies), 0x2B16, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 641, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":643 + * _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies, 11030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamUpdateCaptureDependencies_v2 + * _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies_v2, 12030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamAttachMemAsync + * _F_cuGetProcAddress_v2('cuStreamAttachMemAsync', &__cuStreamAttachMemAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamUpdateCaptureDependencies"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies_v2), 0x2EFE, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 643, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":645 + * _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies_v2, 12030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamAttachMemAsync + * _F_cuGetProcAddress_v2('cuStreamAttachMemAsync', &__cuStreamAttachMemAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamQuery + * _F_cuGetProcAddress_v2('cuStreamQuery', &__cuStreamQuery, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamAttachMemAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAttachMemAsync), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 645, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":647 + * _F_cuGetProcAddress_v2('cuStreamAttachMemAsync', &__cuStreamAttachMemAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamQuery + * _F_cuGetProcAddress_v2('cuStreamQuery', &__cuStreamQuery, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamSynchronize + * _F_cuGetProcAddress_v2('cuStreamSynchronize', &__cuStreamSynchronize, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamQuery"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamQuery), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 647, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":649 + * _F_cuGetProcAddress_v2('cuStreamQuery', &__cuStreamQuery, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamSynchronize + * _F_cuGetProcAddress_v2('cuStreamSynchronize', &__cuStreamSynchronize, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamCopyAttributes + * _F_cuGetProcAddress_v2('cuStreamCopyAttributes', &__cuStreamCopyAttributes, 11000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamSynchronize"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSynchronize), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 649, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":651 + * _F_cuGetProcAddress_v2('cuStreamSynchronize', &__cuStreamSynchronize, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamCopyAttributes + * _F_cuGetProcAddress_v2('cuStreamCopyAttributes', &__cuStreamCopyAttributes, 11000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetAttribute + * _F_cuGetProcAddress_v2('cuStreamGetAttribute', &__cuStreamGetAttribute, 11000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamCopyAttributes"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCopyAttributes), 0x2AF8, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 651, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":653 + * _F_cuGetProcAddress_v2('cuStreamCopyAttributes', &__cuStreamCopyAttributes, 11000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamGetAttribute + * _F_cuGetProcAddress_v2('cuStreamGetAttribute', &__cuStreamGetAttribute, 11000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamSetAttribute + * _F_cuGetProcAddress_v2('cuStreamSetAttribute', &__cuStreamSetAttribute, 11000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetAttribute), 0x2AF8, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 653, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":655 + * _F_cuGetProcAddress_v2('cuStreamGetAttribute', &__cuStreamGetAttribute, 11000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamSetAttribute + * _F_cuGetProcAddress_v2('cuStreamSetAttribute', &__cuStreamSetAttribute, 11000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuEventRecord + * _F_cuGetProcAddress_v2('cuEventRecord', &__cuEventRecord, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamSetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSetAttribute), 0x2AF8, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 655, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":657 + * _F_cuGetProcAddress_v2('cuStreamSetAttribute', &__cuStreamSetAttribute, 11000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuEventRecord + * _F_cuGetProcAddress_v2('cuEventRecord', &__cuEventRecord, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuEventRecordWithFlags + * _F_cuGetProcAddress_v2('cuEventRecordWithFlags', &__cuEventRecordWithFlags, 11010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEventRecord"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecord), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 657, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":659 + * _F_cuGetProcAddress_v2('cuEventRecord', &__cuEventRecord, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuEventRecordWithFlags + * _F_cuGetProcAddress_v2('cuEventRecordWithFlags', &__cuEventRecordWithFlags, 11010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuSignalExternalSemaphoresAsync + * _F_cuGetProcAddress_v2('cuSignalExternalSemaphoresAsync', &__cuSignalExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEventRecordWithFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecordWithFlags), 0x2B02, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 659, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":661 + * _F_cuGetProcAddress_v2('cuEventRecordWithFlags', &__cuEventRecordWithFlags, 11010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuSignalExternalSemaphoresAsync + * _F_cuGetProcAddress_v2('cuSignalExternalSemaphoresAsync', &__cuSignalExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuWaitExternalSemaphoresAsync + * _F_cuGetProcAddress_v2('cuWaitExternalSemaphoresAsync', &__cuWaitExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuSignalExternalSemaphoresAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSignalExternalSemaphoresAsync), 0x2710, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 661, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":663 + * _F_cuGetProcAddress_v2('cuSignalExternalSemaphoresAsync', &__cuSignalExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuWaitExternalSemaphoresAsync + * _F_cuGetProcAddress_v2('cuWaitExternalSemaphoresAsync', &__cuWaitExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamWaitValue32_v2 + * _F_cuGetProcAddress_v2('cuStreamWaitValue32', &__cuStreamWaitValue32_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuWaitExternalSemaphoresAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuWaitExternalSemaphoresAsync), 0x2710, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 663, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":665 + * _F_cuGetProcAddress_v2('cuWaitExternalSemaphoresAsync', &__cuWaitExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamWaitValue32_v2 + * _F_cuGetProcAddress_v2('cuStreamWaitValue32', &__cuStreamWaitValue32_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamWaitValue64_v2 + * _F_cuGetProcAddress_v2('cuStreamWaitValue64', &__cuStreamWaitValue64_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamWaitValue32"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue32_v2), 0x2B3E, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 665, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":667 + * _F_cuGetProcAddress_v2('cuStreamWaitValue32', &__cuStreamWaitValue32_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamWaitValue64_v2 + * _F_cuGetProcAddress_v2('cuStreamWaitValue64', &__cuStreamWaitValue64_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamWriteValue32_v2 + * _F_cuGetProcAddress_v2('cuStreamWriteValue32', &__cuStreamWriteValue32_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamWaitValue64"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue64_v2), 0x2B3E, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 667, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":669 + * _F_cuGetProcAddress_v2('cuStreamWaitValue64', &__cuStreamWaitValue64_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamWriteValue32_v2 + * _F_cuGetProcAddress_v2('cuStreamWriteValue32', &__cuStreamWriteValue32_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamWriteValue64_v2 + * _F_cuGetProcAddress_v2('cuStreamWriteValue64', &__cuStreamWriteValue64_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamWriteValue32"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue32_v2), 0x2B3E, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 669, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":671 + * _F_cuGetProcAddress_v2('cuStreamWriteValue32', &__cuStreamWriteValue32_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamWriteValue64_v2 + * _F_cuGetProcAddress_v2('cuStreamWriteValue64', &__cuStreamWriteValue64_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuStreamBatchMemOp_v2 + * _F_cuGetProcAddress_v2('cuStreamBatchMemOp', &__cuStreamBatchMemOp_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamWriteValue64"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue64_v2), 0x2B3E, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 671, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":673 + * _F_cuGetProcAddress_v2('cuStreamWriteValue64', &__cuStreamWriteValue64_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuStreamBatchMemOp_v2 + * _F_cuGetProcAddress_v2('cuStreamBatchMemOp', &__cuStreamBatchMemOp_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuLaunchKernel + * _F_cuGetProcAddress_v2('cuLaunchKernel', &__cuLaunchKernel, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamBatchMemOp"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBatchMemOp_v2), 0x2B3E, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 673, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":675 + * _F_cuGetProcAddress_v2('cuStreamBatchMemOp', &__cuStreamBatchMemOp_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuLaunchKernel + * _F_cuGetProcAddress_v2('cuLaunchKernel', &__cuLaunchKernel, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuLaunchKernelEx + * _F_cuGetProcAddress_v2('cuLaunchKernelEx', &__cuLaunchKernelEx, 11060, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLaunchKernel"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernel), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 675, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":677 + * _F_cuGetProcAddress_v2('cuLaunchKernel', &__cuLaunchKernel, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuLaunchKernelEx + * _F_cuGetProcAddress_v2('cuLaunchKernelEx', &__cuLaunchKernelEx, 11060, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuLaunchCooperativeKernel + * _F_cuGetProcAddress_v2('cuLaunchCooperativeKernel', &__cuLaunchCooperativeKernel, 9000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLaunchKernelEx"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernelEx), 0x2B34, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 677, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":679 + * _F_cuGetProcAddress_v2('cuLaunchKernelEx', &__cuLaunchKernelEx, 11060, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuLaunchCooperativeKernel + * _F_cuGetProcAddress_v2('cuLaunchCooperativeKernel', &__cuLaunchCooperativeKernel, 9000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuLaunchHostFunc + * _F_cuGetProcAddress_v2('cuLaunchHostFunc', &__cuLaunchHostFunc, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLaunchCooperativeKernel"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernel), 0x2328, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 679, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":681 + * _F_cuGetProcAddress_v2('cuLaunchCooperativeKernel', &__cuLaunchCooperativeKernel, 9000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuLaunchHostFunc + * _F_cuGetProcAddress_v2('cuLaunchHostFunc', &__cuLaunchHostFunc, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuGraphInstantiateWithParams + * _F_cuGetProcAddress_v2('cuGraphInstantiateWithParams', &__cuGraphInstantiateWithParams, 12000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLaunchHostFunc"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchHostFunc), 0x2710, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 681, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":683 + * _F_cuGetProcAddress_v2('cuLaunchHostFunc', &__cuLaunchHostFunc, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuGraphInstantiateWithParams + * _F_cuGetProcAddress_v2('cuGraphInstantiateWithParams', &__cuGraphInstantiateWithParams, 12000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuGraphUpload + * _F_cuGetProcAddress_v2('cuGraphUpload', &__cuGraphUpload, 11010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphInstantiateWithParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithParams), 0x2EE0, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 683, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":685 + * _F_cuGetProcAddress_v2('cuGraphInstantiateWithParams', &__cuGraphInstantiateWithParams, 12000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuGraphUpload + * _F_cuGetProcAddress_v2('cuGraphUpload', &__cuGraphUpload, 11010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuGraphLaunch + * _F_cuGetProcAddress_v2('cuGraphLaunch', &__cuGraphLaunch, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphUpload"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphUpload), 0x2B02, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 685, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":687 + * _F_cuGetProcAddress_v2('cuGraphUpload', &__cuGraphUpload, 11010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuGraphLaunch + * _F_cuGetProcAddress_v2('cuGraphLaunch', &__cuGraphLaunch, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsMapResources + * _F_cuGetProcAddress_v2('cuGraphicsMapResources', &__cuGraphicsMapResources, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphLaunch"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphLaunch), 0x2710, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 687, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":689 + * _F_cuGetProcAddress_v2('cuGraphLaunch', &__cuGraphLaunch, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuGraphicsMapResources + * _F_cuGetProcAddress_v2('cuGraphicsMapResources', &__cuGraphicsMapResources, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsUnmapResources + * _F_cuGetProcAddress_v2('cuGraphicsUnmapResources', &__cuGraphicsUnmapResources, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsMapResources"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsMapResources), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 689, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":691 + * _F_cuGetProcAddress_v2('cuGraphicsMapResources', &__cuGraphicsMapResources, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + * global __cuGraphicsUnmapResources + * _F_cuGetProcAddress_v2('cuGraphicsUnmapResources', &__cuGraphicsUnmapResources, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) # <<<<<<<<<<<<<< + * else: + * # Else get the regular version +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsUnmapResources"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnmapResources), 0x1B58, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 691, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":519 + * if __cuGetProcAddress_v2 != NULL: + * _F_cuGetProcAddress_v2 = <__cuGetProcAddress_v2_T>__cuGetProcAddress_v2 + * if usePTDS: # <<<<<<<<<<<<<< + * # Get all PTDS version of functions + * pass +*/ + goto __pyx_L21; + } + + /* "cuda/bindings/_bindings/cydriver.pyx":694 + * else: + * # Else get the regular version + * pass # <<<<<<<<<<<<<< + * global __cuMemcpy + * _F_cuGetProcAddress_v2('cuMemcpy', &__cuMemcpy, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + /*else*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":696 + * pass + * global __cuMemcpy + * _F_cuGetProcAddress_v2('cuMemcpy', &__cuMemcpy, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyPeer + * _F_cuGetProcAddress_v2('cuMemcpyPeer', &__cuMemcpyPeer, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 696, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":698 + * _F_cuGetProcAddress_v2('cuMemcpy', &__cuMemcpy, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyPeer + * _F_cuGetProcAddress_v2('cuMemcpyPeer', &__cuMemcpyPeer, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyHtoD_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoD', &__cuMemcpyHtoD_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyPeer"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeer), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 698, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":700 + * _F_cuGetProcAddress_v2('cuMemcpyPeer', &__cuMemcpyPeer, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyHtoD_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoD', &__cuMemcpyHtoD_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyDtoH_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoH', &__cuMemcpyDtoH_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyHtoD"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoD_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 700, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":702 + * _F_cuGetProcAddress_v2('cuMemcpyHtoD', &__cuMemcpyHtoD_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyDtoH_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoH', &__cuMemcpyDtoH_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyDtoD_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoD', &__cuMemcpyDtoD_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyDtoH"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoH_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 702, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":704 + * _F_cuGetProcAddress_v2('cuMemcpyDtoH', &__cuMemcpyDtoH_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyDtoD_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoD', &__cuMemcpyDtoD_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyDtoA_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoA', &__cuMemcpyDtoA_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyDtoD"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoD_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 704, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":706 + * _F_cuGetProcAddress_v2('cuMemcpyDtoD', &__cuMemcpyDtoD_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyDtoA_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoA', &__cuMemcpyDtoA_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyAtoD_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoD', &__cuMemcpyAtoD_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyDtoA"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoA_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 706, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":708 + * _F_cuGetProcAddress_v2('cuMemcpyDtoA', &__cuMemcpyDtoA_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyAtoD_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoD', &__cuMemcpyAtoD_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyHtoA_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoA', &__cuMemcpyHtoA_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyAtoD"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoD_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 708, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":710 + * _F_cuGetProcAddress_v2('cuMemcpyAtoD', &__cuMemcpyAtoD_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyHtoA_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoA', &__cuMemcpyHtoA_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyAtoH_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoH', &__cuMemcpyAtoH_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyHtoA"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoA_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 710, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":712 + * _F_cuGetProcAddress_v2('cuMemcpyHtoA', &__cuMemcpyHtoA_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyAtoH_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoH', &__cuMemcpyAtoH_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyAtoA_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoA', &__cuMemcpyAtoA_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyAtoH"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoH_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 712, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":714 + * _F_cuGetProcAddress_v2('cuMemcpyAtoH', &__cuMemcpyAtoH_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyAtoA_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoA', &__cuMemcpyAtoA_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy2D_v2 + * _F_cuGetProcAddress_v2('cuMemcpy2D', &__cuMemcpy2D_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyAtoA"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoA_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 714, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":716 + * _F_cuGetProcAddress_v2('cuMemcpyAtoA', &__cuMemcpyAtoA_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpy2D_v2 + * _F_cuGetProcAddress_v2('cuMemcpy2D', &__cuMemcpy2D_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy2DUnaligned_v2 + * _F_cuGetProcAddress_v2('cuMemcpy2DUnaligned', &__cuMemcpy2DUnaligned_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy2D"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2D_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 716, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":718 + * _F_cuGetProcAddress_v2('cuMemcpy2D', &__cuMemcpy2D_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpy2DUnaligned_v2 + * _F_cuGetProcAddress_v2('cuMemcpy2DUnaligned', &__cuMemcpy2DUnaligned_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy3D_v2 + * _F_cuGetProcAddress_v2('cuMemcpy3D', &__cuMemcpy3D_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy2DUnaligned"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DUnaligned_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 718, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":720 + * _F_cuGetProcAddress_v2('cuMemcpy2DUnaligned', &__cuMemcpy2DUnaligned_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpy3D_v2 + * _F_cuGetProcAddress_v2('cuMemcpy3D', &__cuMemcpy3D_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeer + * _F_cuGetProcAddress_v2('cuMemcpy3DPeer', &__cuMemcpy3DPeer, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy3D"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3D_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 720, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":722 + * _F_cuGetProcAddress_v2('cuMemcpy3D', &__cuMemcpy3D_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpy3DPeer + * _F_cuGetProcAddress_v2('cuMemcpy3DPeer', &__cuMemcpy3DPeer, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyAsync + * _F_cuGetProcAddress_v2('cuMemcpyAsync', &__cuMemcpyAsync, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy3DPeer"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeer), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 722, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":724 + * _F_cuGetProcAddress_v2('cuMemcpy3DPeer', &__cuMemcpy3DPeer, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyAsync + * _F_cuGetProcAddress_v2('cuMemcpyAsync', &__cuMemcpyAsync, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyPeerAsync + * _F_cuGetProcAddress_v2('cuMemcpyPeerAsync', &__cuMemcpyPeerAsync, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAsync), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 724, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":726 + * _F_cuGetProcAddress_v2('cuMemcpyAsync', &__cuMemcpyAsync, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyPeerAsync + * _F_cuGetProcAddress_v2('cuMemcpyPeerAsync', &__cuMemcpyPeerAsync, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyHtoDAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoDAsync', &__cuMemcpyHtoDAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyPeerAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeerAsync), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 726, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":728 + * _F_cuGetProcAddress_v2('cuMemcpyPeerAsync', &__cuMemcpyPeerAsync, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyHtoDAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoDAsync', &__cuMemcpyHtoDAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyDtoHAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoHAsync', &__cuMemcpyDtoHAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyHtoDAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoDAsync_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 728, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":730 + * _F_cuGetProcAddress_v2('cuMemcpyHtoDAsync', &__cuMemcpyHtoDAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyDtoHAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoHAsync', &__cuMemcpyDtoHAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyDtoDAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoDAsync', &__cuMemcpyDtoDAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyDtoHAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoHAsync_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 730, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":732 + * _F_cuGetProcAddress_v2('cuMemcpyDtoHAsync', &__cuMemcpyDtoHAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyDtoDAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyDtoDAsync', &__cuMemcpyDtoDAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyHtoAAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoAAsync', &__cuMemcpyHtoAAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyDtoDAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoDAsync_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 732, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":734 + * _F_cuGetProcAddress_v2('cuMemcpyDtoDAsync', &__cuMemcpyDtoDAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyHtoAAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyHtoAAsync', &__cuMemcpyHtoAAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyAtoHAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoHAsync', &__cuMemcpyAtoHAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyHtoAAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoAAsync_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 734, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":736 + * _F_cuGetProcAddress_v2('cuMemcpyHtoAAsync', &__cuMemcpyHtoAAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyAtoHAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpyAtoHAsync', &__cuMemcpyAtoHAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy2DAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpy2DAsync', &__cuMemcpy2DAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyAtoHAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoHAsync_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 736, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":738 + * _F_cuGetProcAddress_v2('cuMemcpyAtoHAsync', &__cuMemcpyAtoHAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpy2DAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpy2DAsync', &__cuMemcpy2DAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy3DAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpy3DAsync', &__cuMemcpy3DAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy2DAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DAsync_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 738, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":740 + * _F_cuGetProcAddress_v2('cuMemcpy2DAsync', &__cuMemcpy2DAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpy3DAsync_v2 + * _F_cuGetProcAddress_v2('cuMemcpy3DAsync', &__cuMemcpy3DAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeerAsync + * _F_cuGetProcAddress_v2('cuMemcpy3DPeerAsync', &__cuMemcpy3DPeerAsync, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy3DAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DAsync_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 740, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":742 + * _F_cuGetProcAddress_v2('cuMemcpy3DAsync', &__cuMemcpy3DAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpy3DPeerAsync + * _F_cuGetProcAddress_v2('cuMemcpy3DPeerAsync', &__cuMemcpy3DPeerAsync, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpyBatchAsync + * _F_cuGetProcAddress_v2('cuMemcpyBatchAsync', &__cuMemcpyBatchAsync, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy3DPeerAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeerAsync), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 742, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":744 + * _F_cuGetProcAddress_v2('cuMemcpy3DPeerAsync', &__cuMemcpy3DPeerAsync, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpyBatchAsync + * _F_cuGetProcAddress_v2('cuMemcpyBatchAsync', &__cuMemcpyBatchAsync, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemcpy3DBatchAsync + * _F_cuGetProcAddress_v2('cuMemcpy3DBatchAsync', &__cuMemcpy3DBatchAsync, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpyBatchAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyBatchAsync), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 744, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":746 + * _F_cuGetProcAddress_v2('cuMemcpyBatchAsync', &__cuMemcpyBatchAsync, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemcpy3DBatchAsync + * _F_cuGetProcAddress_v2('cuMemcpy3DBatchAsync', &__cuMemcpy3DBatchAsync, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD8_v2 + * _F_cuGetProcAddress_v2('cuMemsetD8', &__cuMemsetD8_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemcpy3DBatchAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DBatchAsync), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 746, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":748 + * _F_cuGetProcAddress_v2('cuMemcpy3DBatchAsync', &__cuMemcpy3DBatchAsync, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemsetD8_v2 + * _F_cuGetProcAddress_v2('cuMemsetD8', &__cuMemsetD8_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD16_v2 + * _F_cuGetProcAddress_v2('cuMemsetD16', &__cuMemsetD16_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD8"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 748, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":750 + * _F_cuGetProcAddress_v2('cuMemsetD8', &__cuMemsetD8_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemsetD16_v2 + * _F_cuGetProcAddress_v2('cuMemsetD16', &__cuMemsetD16_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD32_v2 + * _F_cuGetProcAddress_v2('cuMemsetD32', &__cuMemsetD32_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD16"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 750, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":752 + * _F_cuGetProcAddress_v2('cuMemsetD16', &__cuMemsetD16_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemsetD32_v2 + * _F_cuGetProcAddress_v2('cuMemsetD32', &__cuMemsetD32_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD2D8_v2 + * _F_cuGetProcAddress_v2('cuMemsetD2D8', &__cuMemsetD2D8_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD32"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 752, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":754 + * _F_cuGetProcAddress_v2('cuMemsetD32', &__cuMemsetD32_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemsetD2D8_v2 + * _F_cuGetProcAddress_v2('cuMemsetD2D8', &__cuMemsetD2D8_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD2D16_v2 + * _F_cuGetProcAddress_v2('cuMemsetD2D16', &__cuMemsetD2D16_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD2D8"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 754, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":756 + * _F_cuGetProcAddress_v2('cuMemsetD2D8', &__cuMemsetD2D8_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemsetD2D16_v2 + * _F_cuGetProcAddress_v2('cuMemsetD2D16', &__cuMemsetD2D16_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD2D32_v2 + * _F_cuGetProcAddress_v2('cuMemsetD2D32', &__cuMemsetD2D32_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD2D16"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 756, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":758 + * _F_cuGetProcAddress_v2('cuMemsetD2D16', &__cuMemsetD2D16_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemsetD2D32_v2 + * _F_cuGetProcAddress_v2('cuMemsetD2D32', &__cuMemsetD2D32_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD8Async + * _F_cuGetProcAddress_v2('cuMemsetD8Async', &__cuMemsetD8Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD2D32"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 758, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":760 + * _F_cuGetProcAddress_v2('cuMemsetD2D32', &__cuMemsetD2D32_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemsetD8Async + * _F_cuGetProcAddress_v2('cuMemsetD8Async', &__cuMemsetD8Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD16Async + * _F_cuGetProcAddress_v2('cuMemsetD16Async', &__cuMemsetD16Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD8Async"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8Async), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 760, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":762 + * _F_cuGetProcAddress_v2('cuMemsetD8Async', &__cuMemsetD8Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemsetD16Async + * _F_cuGetProcAddress_v2('cuMemsetD16Async', &__cuMemsetD16Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD32Async + * _F_cuGetProcAddress_v2('cuMemsetD32Async', &__cuMemsetD32Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD16Async"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16Async), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 762, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":764 + * _F_cuGetProcAddress_v2('cuMemsetD16Async', &__cuMemsetD16Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemsetD32Async + * _F_cuGetProcAddress_v2('cuMemsetD32Async', &__cuMemsetD32Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD2D8Async + * _F_cuGetProcAddress_v2('cuMemsetD2D8Async', &__cuMemsetD2D8Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD32Async"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32Async), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 764, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":766 + * _F_cuGetProcAddress_v2('cuMemsetD32Async', &__cuMemsetD32Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemsetD2D8Async + * _F_cuGetProcAddress_v2('cuMemsetD2D8Async', &__cuMemsetD2D8Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD2D16Async + * _F_cuGetProcAddress_v2('cuMemsetD2D16Async', &__cuMemsetD2D16Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD2D8Async"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8Async), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 766, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":768 + * _F_cuGetProcAddress_v2('cuMemsetD2D8Async', &__cuMemsetD2D8Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemsetD2D16Async + * _F_cuGetProcAddress_v2('cuMemsetD2D16Async', &__cuMemsetD2D16Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemsetD2D32Async + * _F_cuGetProcAddress_v2('cuMemsetD2D32Async', &__cuMemsetD2D32Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD2D16Async"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16Async), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 768, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":770 + * _F_cuGetProcAddress_v2('cuMemsetD2D16Async', &__cuMemsetD2D16Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemsetD2D32Async + * _F_cuGetProcAddress_v2('cuMemsetD2D32Async', &__cuMemsetD2D32Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemBatchDecompressAsync + * _F_cuGetProcAddress_v2('cuMemBatchDecompressAsync', &__cuMemBatchDecompressAsync, 12060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemsetD2D32Async"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32Async), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 770, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":772 + * _F_cuGetProcAddress_v2('cuMemsetD2D32Async', &__cuMemsetD2D32Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemBatchDecompressAsync + * _F_cuGetProcAddress_v2('cuMemBatchDecompressAsync', &__cuMemBatchDecompressAsync, 12060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemMapArrayAsync + * _F_cuGetProcAddress_v2('cuMemMapArrayAsync', &__cuMemMapArrayAsync, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemBatchDecompressAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemBatchDecompressAsync), 0x2F1C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 772, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":774 + * _F_cuGetProcAddress_v2('cuMemBatchDecompressAsync', &__cuMemBatchDecompressAsync, 12060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemMapArrayAsync + * _F_cuGetProcAddress_v2('cuMemMapArrayAsync', &__cuMemMapArrayAsync, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemFreeAsync + * _F_cuGetProcAddress_v2('cuMemFreeAsync', &__cuMemFreeAsync, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemMapArrayAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMapArrayAsync), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 774, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":776 + * _F_cuGetProcAddress_v2('cuMemMapArrayAsync', &__cuMemMapArrayAsync, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemFreeAsync + * _F_cuGetProcAddress_v2('cuMemFreeAsync', &__cuMemFreeAsync, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemAllocAsync + * _F_cuGetProcAddress_v2('cuMemAllocAsync', &__cuMemAllocAsync, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemFreeAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeAsync), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 776, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":778 + * _F_cuGetProcAddress_v2('cuMemFreeAsync', &__cuMemFreeAsync, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemAllocAsync + * _F_cuGetProcAddress_v2('cuMemAllocAsync', &__cuMemAllocAsync, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemAllocFromPoolAsync + * _F_cuGetProcAddress_v2('cuMemAllocFromPoolAsync', &__cuMemAllocFromPoolAsync, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemAllocAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocAsync), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 778, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":780 + * _F_cuGetProcAddress_v2('cuMemAllocAsync', &__cuMemAllocAsync, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemAllocFromPoolAsync + * _F_cuGetProcAddress_v2('cuMemAllocFromPoolAsync', &__cuMemAllocFromPoolAsync, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync + * _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemAllocFromPoolAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocFromPoolAsync), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 780, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":782 + * _F_cuGetProcAddress_v2('cuMemAllocFromPoolAsync', &__cuMemAllocFromPoolAsync, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemPrefetchAsync + * _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync_v2 + * _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync_v2, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPrefetchAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync), 0x1F40, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 782, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":784 + * _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemPrefetchAsync_v2 + * _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync_v2, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetPriority + * _F_cuGetProcAddress_v2('cuStreamGetPriority', &__cuStreamGetPriority, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPrefetchAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync_v2), 0x2EF4, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 784, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":786 + * _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync_v2, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamGetPriority + * _F_cuGetProcAddress_v2('cuStreamGetPriority', &__cuStreamGetPriority, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetDevice + * _F_cuGetProcAddress_v2('cuStreamGetDevice', &__cuStreamGetDevice, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetPriority"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetPriority), 0x13BA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 786, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":788 + * _F_cuGetProcAddress_v2('cuStreamGetPriority', &__cuStreamGetPriority, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamGetDevice + * _F_cuGetProcAddress_v2('cuStreamGetDevice', &__cuStreamGetDevice, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetFlags + * _F_cuGetProcAddress_v2('cuStreamGetFlags', &__cuStreamGetFlags, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetDevice"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetDevice), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 788, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":790 + * _F_cuGetProcAddress_v2('cuStreamGetDevice', &__cuStreamGetDevice, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamGetFlags + * _F_cuGetProcAddress_v2('cuStreamGetFlags', &__cuStreamGetFlags, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetId + * _F_cuGetProcAddress_v2('cuStreamGetId', &__cuStreamGetId, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetFlags), 0x13BA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 790, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":792 + * _F_cuGetProcAddress_v2('cuStreamGetFlags', &__cuStreamGetFlags, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamGetId + * _F_cuGetProcAddress_v2('cuStreamGetId', &__cuStreamGetId, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetCtx + * _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx, 9020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetId"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetId), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 792, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":794 + * _F_cuGetProcAddress_v2('cuStreamGetId', &__cuStreamGetId, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamGetCtx + * _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx, 9020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetCtx_v2 + * _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx_v2, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetCtx"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx), 0x233C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 794, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":796 + * _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx, 9020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamGetCtx_v2 + * _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx_v2, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamWaitEvent + * _F_cuGetProcAddress_v2('cuStreamWaitEvent', &__cuStreamWaitEvent, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetCtx"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx_v2), 0x2F12, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 796, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":798 + * _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx_v2, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamWaitEvent + * _F_cuGetProcAddress_v2('cuStreamWaitEvent', &__cuStreamWaitEvent, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamAddCallback + * _F_cuGetProcAddress_v2('cuStreamAddCallback', &__cuStreamAddCallback, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamWaitEvent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitEvent), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 798, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":800 + * _F_cuGetProcAddress_v2('cuStreamWaitEvent', &__cuStreamWaitEvent, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamAddCallback + * _F_cuGetProcAddress_v2('cuStreamAddCallback', &__cuStreamAddCallback, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamBeginCapture_v2 + * _F_cuGetProcAddress_v2('cuStreamBeginCapture', &__cuStreamBeginCapture_v2, 10010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamAddCallback"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAddCallback), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 800, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":802 + * _F_cuGetProcAddress_v2('cuStreamAddCallback', &__cuStreamAddCallback, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamBeginCapture_v2 + * _F_cuGetProcAddress_v2('cuStreamBeginCapture', &__cuStreamBeginCapture_v2, 10010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamBeginCaptureToGraph + * _F_cuGetProcAddress_v2('cuStreamBeginCaptureToGraph', &__cuStreamBeginCaptureToGraph, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamBeginCapture"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCapture_v2), 0x271A, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 802, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":804 + * _F_cuGetProcAddress_v2('cuStreamBeginCapture', &__cuStreamBeginCapture_v2, 10010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamBeginCaptureToGraph + * _F_cuGetProcAddress_v2('cuStreamBeginCaptureToGraph', &__cuStreamBeginCaptureToGraph, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamEndCapture + * _F_cuGetProcAddress_v2('cuStreamEndCapture', &__cuStreamEndCapture, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamBeginCaptureToGraph"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCaptureToGraph), 0x2EFE, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 804, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":806 + * _F_cuGetProcAddress_v2('cuStreamBeginCaptureToGraph', &__cuStreamBeginCaptureToGraph, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamEndCapture + * _F_cuGetProcAddress_v2('cuStreamEndCapture', &__cuStreamEndCapture, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamIsCapturing + * _F_cuGetProcAddress_v2('cuStreamIsCapturing', &__cuStreamIsCapturing, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamEndCapture"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamEndCapture), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 806, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":808 + * _F_cuGetProcAddress_v2('cuStreamEndCapture', &__cuStreamEndCapture, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamIsCapturing + * _F_cuGetProcAddress_v2('cuStreamIsCapturing', &__cuStreamIsCapturing, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v2 + * _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v2, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamIsCapturing"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamIsCapturing), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 808, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":810 + * _F_cuGetProcAddress_v2('cuStreamIsCapturing', &__cuStreamIsCapturing, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamGetCaptureInfo_v2 + * _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v2, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v3 + * _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v3, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetCaptureInfo"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v2), 0x2B16, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 810, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":812 + * _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v2, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamGetCaptureInfo_v3 + * _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v3, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies + * _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetCaptureInfo"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v3), 0x2EFE, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 812, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":814 + * _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v3, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamUpdateCaptureDependencies + * _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies_v2 + * _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamUpdateCaptureDependencies"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies), 0x2B16, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 814, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":816 + * _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamUpdateCaptureDependencies_v2 + * _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamAttachMemAsync + * _F_cuGetProcAddress_v2('cuStreamAttachMemAsync', &__cuStreamAttachMemAsync, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamUpdateCaptureDependencies"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies_v2), 0x2EFE, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 816, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":818 + * _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamAttachMemAsync + * _F_cuGetProcAddress_v2('cuStreamAttachMemAsync', &__cuStreamAttachMemAsync, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamQuery + * _F_cuGetProcAddress_v2('cuStreamQuery', &__cuStreamQuery, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamAttachMemAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAttachMemAsync), 0x1770, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 818, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":820 + * _F_cuGetProcAddress_v2('cuStreamAttachMemAsync', &__cuStreamAttachMemAsync, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamQuery + * _F_cuGetProcAddress_v2('cuStreamQuery', &__cuStreamQuery, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamSynchronize + * _F_cuGetProcAddress_v2('cuStreamSynchronize', &__cuStreamSynchronize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamQuery"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamQuery), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 820, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":822 + * _F_cuGetProcAddress_v2('cuStreamQuery', &__cuStreamQuery, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamSynchronize + * _F_cuGetProcAddress_v2('cuStreamSynchronize', &__cuStreamSynchronize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamCopyAttributes + * _F_cuGetProcAddress_v2('cuStreamCopyAttributes', &__cuStreamCopyAttributes, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamSynchronize"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSynchronize), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 822, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":824 + * _F_cuGetProcAddress_v2('cuStreamSynchronize', &__cuStreamSynchronize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamCopyAttributes + * _F_cuGetProcAddress_v2('cuStreamCopyAttributes', &__cuStreamCopyAttributes, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetAttribute + * _F_cuGetProcAddress_v2('cuStreamGetAttribute', &__cuStreamGetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamCopyAttributes"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCopyAttributes), 0x2AF8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 824, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":826 + * _F_cuGetProcAddress_v2('cuStreamCopyAttributes', &__cuStreamCopyAttributes, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamGetAttribute + * _F_cuGetProcAddress_v2('cuStreamGetAttribute', &__cuStreamGetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamSetAttribute + * _F_cuGetProcAddress_v2('cuStreamSetAttribute', &__cuStreamSetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetAttribute), 0x2AF8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 826, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":828 + * _F_cuGetProcAddress_v2('cuStreamGetAttribute', &__cuStreamGetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamSetAttribute + * _F_cuGetProcAddress_v2('cuStreamSetAttribute', &__cuStreamSetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEventRecord + * _F_cuGetProcAddress_v2('cuEventRecord', &__cuEventRecord, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamSetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSetAttribute), 0x2AF8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 828, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":830 + * _F_cuGetProcAddress_v2('cuStreamSetAttribute', &__cuStreamSetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEventRecord + * _F_cuGetProcAddress_v2('cuEventRecord', &__cuEventRecord, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEventRecordWithFlags + * _F_cuGetProcAddress_v2('cuEventRecordWithFlags', &__cuEventRecordWithFlags, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEventRecord"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecord), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 830, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":832 + * _F_cuGetProcAddress_v2('cuEventRecord', &__cuEventRecord, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEventRecordWithFlags + * _F_cuGetProcAddress_v2('cuEventRecordWithFlags', &__cuEventRecordWithFlags, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuSignalExternalSemaphoresAsync + * _F_cuGetProcAddress_v2('cuSignalExternalSemaphoresAsync', &__cuSignalExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEventRecordWithFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecordWithFlags), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 832, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":834 + * _F_cuGetProcAddress_v2('cuEventRecordWithFlags', &__cuEventRecordWithFlags, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuSignalExternalSemaphoresAsync + * _F_cuGetProcAddress_v2('cuSignalExternalSemaphoresAsync', &__cuSignalExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuWaitExternalSemaphoresAsync + * _F_cuGetProcAddress_v2('cuWaitExternalSemaphoresAsync', &__cuWaitExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuSignalExternalSemaphoresAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSignalExternalSemaphoresAsync), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 834, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":836 + * _F_cuGetProcAddress_v2('cuSignalExternalSemaphoresAsync', &__cuSignalExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuWaitExternalSemaphoresAsync + * _F_cuGetProcAddress_v2('cuWaitExternalSemaphoresAsync', &__cuWaitExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamWaitValue32_v2 + * _F_cuGetProcAddress_v2('cuStreamWaitValue32', &__cuStreamWaitValue32_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuWaitExternalSemaphoresAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuWaitExternalSemaphoresAsync), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 836, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":838 + * _F_cuGetProcAddress_v2('cuWaitExternalSemaphoresAsync', &__cuWaitExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamWaitValue32_v2 + * _F_cuGetProcAddress_v2('cuStreamWaitValue32', &__cuStreamWaitValue32_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamWaitValue64_v2 + * _F_cuGetProcAddress_v2('cuStreamWaitValue64', &__cuStreamWaitValue64_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamWaitValue32"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue32_v2), 0x2B3E, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 838, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":840 + * _F_cuGetProcAddress_v2('cuStreamWaitValue32', &__cuStreamWaitValue32_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamWaitValue64_v2 + * _F_cuGetProcAddress_v2('cuStreamWaitValue64', &__cuStreamWaitValue64_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamWriteValue32_v2 + * _F_cuGetProcAddress_v2('cuStreamWriteValue32', &__cuStreamWriteValue32_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamWaitValue64"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue64_v2), 0x2B3E, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 840, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":842 + * _F_cuGetProcAddress_v2('cuStreamWaitValue64', &__cuStreamWaitValue64_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamWriteValue32_v2 + * _F_cuGetProcAddress_v2('cuStreamWriteValue32', &__cuStreamWriteValue32_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamWriteValue64_v2 + * _F_cuGetProcAddress_v2('cuStreamWriteValue64', &__cuStreamWriteValue64_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamWriteValue32"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue32_v2), 0x2B3E, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 842, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":844 + * _F_cuGetProcAddress_v2('cuStreamWriteValue32', &__cuStreamWriteValue32_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamWriteValue64_v2 + * _F_cuGetProcAddress_v2('cuStreamWriteValue64', &__cuStreamWriteValue64_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamBatchMemOp_v2 + * _F_cuGetProcAddress_v2('cuStreamBatchMemOp', &__cuStreamBatchMemOp_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamWriteValue64"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue64_v2), 0x2B3E, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 844, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":846 + * _F_cuGetProcAddress_v2('cuStreamWriteValue64', &__cuStreamWriteValue64_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamBatchMemOp_v2 + * _F_cuGetProcAddress_v2('cuStreamBatchMemOp', &__cuStreamBatchMemOp_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLaunchKernel + * _F_cuGetProcAddress_v2('cuLaunchKernel', &__cuLaunchKernel, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamBatchMemOp"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBatchMemOp_v2), 0x2B3E, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 846, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":848 + * _F_cuGetProcAddress_v2('cuStreamBatchMemOp', &__cuStreamBatchMemOp_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLaunchKernel + * _F_cuGetProcAddress_v2('cuLaunchKernel', &__cuLaunchKernel, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLaunchKernelEx + * _F_cuGetProcAddress_v2('cuLaunchKernelEx', &__cuLaunchKernelEx, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLaunchKernel"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernel), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 848, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":850 + * _F_cuGetProcAddress_v2('cuLaunchKernel', &__cuLaunchKernel, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLaunchKernelEx + * _F_cuGetProcAddress_v2('cuLaunchKernelEx', &__cuLaunchKernelEx, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLaunchCooperativeKernel + * _F_cuGetProcAddress_v2('cuLaunchCooperativeKernel', &__cuLaunchCooperativeKernel, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLaunchKernelEx"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernelEx), 0x2B34, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 850, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":852 + * _F_cuGetProcAddress_v2('cuLaunchKernelEx', &__cuLaunchKernelEx, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLaunchCooperativeKernel + * _F_cuGetProcAddress_v2('cuLaunchCooperativeKernel', &__cuLaunchCooperativeKernel, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLaunchHostFunc + * _F_cuGetProcAddress_v2('cuLaunchHostFunc', &__cuLaunchHostFunc, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLaunchCooperativeKernel"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernel), 0x2328, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 852, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":854 + * _F_cuGetProcAddress_v2('cuLaunchCooperativeKernel', &__cuLaunchCooperativeKernel, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLaunchHostFunc + * _F_cuGetProcAddress_v2('cuLaunchHostFunc', &__cuLaunchHostFunc, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphInstantiateWithParams + * _F_cuGetProcAddress_v2('cuGraphInstantiateWithParams', &__cuGraphInstantiateWithParams, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLaunchHostFunc"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchHostFunc), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 854, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":856 + * _F_cuGetProcAddress_v2('cuLaunchHostFunc', &__cuLaunchHostFunc, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphInstantiateWithParams + * _F_cuGetProcAddress_v2('cuGraphInstantiateWithParams', &__cuGraphInstantiateWithParams, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphUpload + * _F_cuGetProcAddress_v2('cuGraphUpload', &__cuGraphUpload, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphInstantiateWithParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithParams), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 856, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":858 + * _F_cuGetProcAddress_v2('cuGraphInstantiateWithParams', &__cuGraphInstantiateWithParams, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphUpload + * _F_cuGetProcAddress_v2('cuGraphUpload', &__cuGraphUpload, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphLaunch + * _F_cuGetProcAddress_v2('cuGraphLaunch', &__cuGraphLaunch, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphUpload"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphUpload), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 858, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":860 + * _F_cuGetProcAddress_v2('cuGraphUpload', &__cuGraphUpload, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphLaunch + * _F_cuGetProcAddress_v2('cuGraphLaunch', &__cuGraphLaunch, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsMapResources + * _F_cuGetProcAddress_v2('cuGraphicsMapResources', &__cuGraphicsMapResources, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphLaunch"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphLaunch), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 860, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":862 + * _F_cuGetProcAddress_v2('cuGraphLaunch', &__cuGraphLaunch, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphicsMapResources + * _F_cuGetProcAddress_v2('cuGraphicsMapResources', &__cuGraphicsMapResources, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsUnmapResources + * _F_cuGetProcAddress_v2('cuGraphicsUnmapResources', &__cuGraphicsUnmapResources, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsMapResources"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsMapResources), 0xBB8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 862, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":864 + * _F_cuGetProcAddress_v2('cuGraphicsMapResources', &__cuGraphicsMapResources, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphicsUnmapResources + * _F_cuGetProcAddress_v2('cuGraphicsUnmapResources', &__cuGraphicsUnmapResources, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * # Get remaining functions + * global __cuGetErrorString +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsUnmapResources"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnmapResources), 0xBB8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 864, __pyx_L10_error) + } + __pyx_L21:; + + /* "cuda/bindings/_bindings/cydriver.pyx":867 + * # Get remaining functions + * global __cuGetErrorString + * _F_cuGetProcAddress_v2('cuGetErrorString', &__cuGetErrorString, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGetErrorName + * _F_cuGetProcAddress_v2('cuGetErrorName', &__cuGetErrorName, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGetErrorString"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorString), 0x1770, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 867, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":869 + * _F_cuGetProcAddress_v2('cuGetErrorString', &__cuGetErrorString, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGetErrorName + * _F_cuGetProcAddress_v2('cuGetErrorName', &__cuGetErrorName, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuInit + * _F_cuGetProcAddress_v2('cuInit', &__cuInit, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGetErrorName"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorName), 0x1770, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 869, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":871 + * _F_cuGetProcAddress_v2('cuGetErrorName', &__cuGetErrorName, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuInit + * _F_cuGetProcAddress_v2('cuInit', &__cuInit, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDriverGetVersion + * _F_cuGetProcAddress_v2('cuDriverGetVersion', &__cuDriverGetVersion, 2020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuInit"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuInit), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 871, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":873 + * _F_cuGetProcAddress_v2('cuInit', &__cuInit, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDriverGetVersion + * _F_cuGetProcAddress_v2('cuDriverGetVersion', &__cuDriverGetVersion, 2020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGet + * _F_cuGetProcAddress_v2('cuDeviceGet', &__cuDeviceGet, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDriverGetVersion"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDriverGetVersion), 0x7E4, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 873, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":875 + * _F_cuGetProcAddress_v2('cuDriverGetVersion', &__cuDriverGetVersion, 2020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGet + * _F_cuGetProcAddress_v2('cuDeviceGet', &__cuDeviceGet, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetCount + * _F_cuGetProcAddress_v2('cuDeviceGetCount', &__cuDeviceGetCount, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGet"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGet), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 875, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":877 + * _F_cuGetProcAddress_v2('cuDeviceGet', &__cuDeviceGet, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetCount + * _F_cuGetProcAddress_v2('cuDeviceGetCount', &__cuDeviceGetCount, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetName + * _F_cuGetProcAddress_v2('cuDeviceGetName', &__cuDeviceGetName, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetCount"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetCount), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 877, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":879 + * _F_cuGetProcAddress_v2('cuDeviceGetCount', &__cuDeviceGetCount, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetName + * _F_cuGetProcAddress_v2('cuDeviceGetName', &__cuDeviceGetName, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetUuid + * _F_cuGetProcAddress_v2('cuDeviceGetUuid', &__cuDeviceGetUuid, 9020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetName"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetName), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 879, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":881 + * _F_cuGetProcAddress_v2('cuDeviceGetName', &__cuDeviceGetName, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetUuid + * _F_cuGetProcAddress_v2('cuDeviceGetUuid', &__cuDeviceGetUuid, 9020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetUuid_v2 + * _F_cuGetProcAddress_v2('cuDeviceGetUuid', &__cuDeviceGetUuid_v2, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetUuid"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid), 0x233C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 881, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":883 + * _F_cuGetProcAddress_v2('cuDeviceGetUuid', &__cuDeviceGetUuid, 9020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetUuid_v2 + * _F_cuGetProcAddress_v2('cuDeviceGetUuid', &__cuDeviceGetUuid_v2, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetLuid + * _F_cuGetProcAddress_v2('cuDeviceGetLuid', &__cuDeviceGetLuid, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetUuid"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid_v2), 0x2B20, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 883, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":885 + * _F_cuGetProcAddress_v2('cuDeviceGetUuid', &__cuDeviceGetUuid_v2, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetLuid + * _F_cuGetProcAddress_v2('cuDeviceGetLuid', &__cuDeviceGetLuid, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceTotalMem_v2 + * _F_cuGetProcAddress_v2('cuDeviceTotalMem', &__cuDeviceTotalMem_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetLuid"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetLuid), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 885, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":887 + * _F_cuGetProcAddress_v2('cuDeviceGetLuid', &__cuDeviceGetLuid, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceTotalMem_v2 + * _F_cuGetProcAddress_v2('cuDeviceTotalMem', &__cuDeviceTotalMem_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetTexture1DLinearMaxWidth + * _F_cuGetProcAddress_v2('cuDeviceGetTexture1DLinearMaxWidth', &__cuDeviceGetTexture1DLinearMaxWidth, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceTotalMem"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceTotalMem_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 887, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":889 + * _F_cuGetProcAddress_v2('cuDeviceTotalMem', &__cuDeviceTotalMem_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetTexture1DLinearMaxWidth + * _F_cuGetProcAddress_v2('cuDeviceGetTexture1DLinearMaxWidth', &__cuDeviceGetTexture1DLinearMaxWidth, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetAttribute + * _F_cuGetProcAddress_v2('cuDeviceGetAttribute', &__cuDeviceGetAttribute, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetTexture1DLinearMaxWidth"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetTexture1DLinearMaxWidth), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 889, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":891 + * _F_cuGetProcAddress_v2('cuDeviceGetTexture1DLinearMaxWidth', &__cuDeviceGetTexture1DLinearMaxWidth, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetAttribute + * _F_cuGetProcAddress_v2('cuDeviceGetAttribute', &__cuDeviceGetAttribute, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetNvSciSyncAttributes + * _F_cuGetProcAddress_v2('cuDeviceGetNvSciSyncAttributes', &__cuDeviceGetNvSciSyncAttributes, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetAttribute), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 891, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":893 + * _F_cuGetProcAddress_v2('cuDeviceGetAttribute', &__cuDeviceGetAttribute, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetNvSciSyncAttributes + * _F_cuGetProcAddress_v2('cuDeviceGetNvSciSyncAttributes', &__cuDeviceGetNvSciSyncAttributes, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceSetMemPool + * _F_cuGetProcAddress_v2('cuDeviceSetMemPool', &__cuDeviceSetMemPool, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetNvSciSyncAttributes"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetNvSciSyncAttributes), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 893, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":895 + * _F_cuGetProcAddress_v2('cuDeviceGetNvSciSyncAttributes', &__cuDeviceGetNvSciSyncAttributes, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceSetMemPool + * _F_cuGetProcAddress_v2('cuDeviceSetMemPool', &__cuDeviceSetMemPool, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetMemPool + * _F_cuGetProcAddress_v2('cuDeviceGetMemPool', &__cuDeviceGetMemPool, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceSetMemPool"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetMemPool), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 895, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":897 + * _F_cuGetProcAddress_v2('cuDeviceSetMemPool', &__cuDeviceSetMemPool, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetMemPool + * _F_cuGetProcAddress_v2('cuDeviceGetMemPool', &__cuDeviceGetMemPool, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetDefaultMemPool + * _F_cuGetProcAddress_v2('cuDeviceGetDefaultMemPool', &__cuDeviceGetDefaultMemPool, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetMemPool"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetMemPool), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 897, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":899 + * _F_cuGetProcAddress_v2('cuDeviceGetMemPool', &__cuDeviceGetMemPool, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetDefaultMemPool + * _F_cuGetProcAddress_v2('cuDeviceGetDefaultMemPool', &__cuDeviceGetDefaultMemPool, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetExecAffinitySupport + * _F_cuGetProcAddress_v2('cuDeviceGetExecAffinitySupport', &__cuDeviceGetExecAffinitySupport, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetDefaultMemPool"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDefaultMemPool), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 899, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":901 + * _F_cuGetProcAddress_v2('cuDeviceGetDefaultMemPool', &__cuDeviceGetDefaultMemPool, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetExecAffinitySupport + * _F_cuGetProcAddress_v2('cuDeviceGetExecAffinitySupport', &__cuDeviceGetExecAffinitySupport, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuFlushGPUDirectRDMAWrites + * _F_cuGetProcAddress_v2('cuFlushGPUDirectRDMAWrites', &__cuFlushGPUDirectRDMAWrites, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetExecAffinitySupport"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetExecAffinitySupport), 0x2B20, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 901, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":903 + * _F_cuGetProcAddress_v2('cuDeviceGetExecAffinitySupport', &__cuDeviceGetExecAffinitySupport, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuFlushGPUDirectRDMAWrites + * _F_cuGetProcAddress_v2('cuFlushGPUDirectRDMAWrites', &__cuFlushGPUDirectRDMAWrites, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetProperties + * _F_cuGetProcAddress_v2('cuDeviceGetProperties', &__cuDeviceGetProperties, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuFlushGPUDirectRDMAWrites"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFlushGPUDirectRDMAWrites), 0x2B16, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 903, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":905 + * _F_cuGetProcAddress_v2('cuFlushGPUDirectRDMAWrites', &__cuFlushGPUDirectRDMAWrites, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetProperties + * _F_cuGetProcAddress_v2('cuDeviceGetProperties', &__cuDeviceGetProperties, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceComputeCapability + * _F_cuGetProcAddress_v2('cuDeviceComputeCapability', &__cuDeviceComputeCapability, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetProperties"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetProperties), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 905, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":907 + * _F_cuGetProcAddress_v2('cuDeviceGetProperties', &__cuDeviceGetProperties, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceComputeCapability + * _F_cuGetProcAddress_v2('cuDeviceComputeCapability', &__cuDeviceComputeCapability, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxRetain + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxRetain', &__cuDevicePrimaryCtxRetain, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceComputeCapability"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceComputeCapability), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 907, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":909 + * _F_cuGetProcAddress_v2('cuDeviceComputeCapability', &__cuDeviceComputeCapability, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDevicePrimaryCtxRetain + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxRetain', &__cuDevicePrimaryCtxRetain, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxRelease_v2 + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxRelease', &__cuDevicePrimaryCtxRelease_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDevicePrimaryCtxRetain"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRetain), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 909, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":911 + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxRetain', &__cuDevicePrimaryCtxRetain, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDevicePrimaryCtxRelease_v2 + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxRelease', &__cuDevicePrimaryCtxRelease_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxSetFlags_v2 + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxSetFlags', &__cuDevicePrimaryCtxSetFlags_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDevicePrimaryCtxRelease"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRelease_v2), 0x2AF8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 911, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":913 + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxRelease', &__cuDevicePrimaryCtxRelease_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDevicePrimaryCtxSetFlags_v2 + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxSetFlags', &__cuDevicePrimaryCtxSetFlags_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxGetState + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxGetState', &__cuDevicePrimaryCtxGetState, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDevicePrimaryCtxSetFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxSetFlags_v2), 0x2AF8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 913, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":915 + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxSetFlags', &__cuDevicePrimaryCtxSetFlags_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDevicePrimaryCtxGetState + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxGetState', &__cuDevicePrimaryCtxGetState, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxReset_v2 + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxReset', &__cuDevicePrimaryCtxReset_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDevicePrimaryCtxGetState"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxGetState), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 915, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":917 + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxGetState', &__cuDevicePrimaryCtxGetState, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDevicePrimaryCtxReset_v2 + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxReset', &__cuDevicePrimaryCtxReset_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxCreate_v2 + * _F_cuGetProcAddress_v2('cuCtxCreate', &__cuCtxCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDevicePrimaryCtxReset"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxReset_v2), 0x2AF8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 917, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":919 + * _F_cuGetProcAddress_v2('cuDevicePrimaryCtxReset', &__cuDevicePrimaryCtxReset_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxCreate_v2 + * _F_cuGetProcAddress_v2('cuCtxCreate', &__cuCtxCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxCreate_v3 + * _F_cuGetProcAddress_v2('cuCtxCreate', &__cuCtxCreate_v3, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 919, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":921 + * _F_cuGetProcAddress_v2('cuCtxCreate', &__cuCtxCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxCreate_v3 + * _F_cuGetProcAddress_v2('cuCtxCreate', &__cuCtxCreate_v3, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxCreate_v4 + * _F_cuGetProcAddress_v2('cuCtxCreate', &__cuCtxCreate_v4, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v3), 0x2B20, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 921, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":923 + * _F_cuGetProcAddress_v2('cuCtxCreate', &__cuCtxCreate_v3, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxCreate_v4 + * _F_cuGetProcAddress_v2('cuCtxCreate', &__cuCtxCreate_v4, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxDestroy_v2 + * _F_cuGetProcAddress_v2('cuCtxDestroy', &__cuCtxDestroy_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v4), 0x2F12, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 923, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":925 + * _F_cuGetProcAddress_v2('cuCtxCreate', &__cuCtxCreate_v4, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxDestroy_v2 + * _F_cuGetProcAddress_v2('cuCtxDestroy', &__cuCtxDestroy_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxPushCurrent_v2 + * _F_cuGetProcAddress_v2('cuCtxPushCurrent', &__cuCtxPushCurrent_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxDestroy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDestroy_v2), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 925, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":927 + * _F_cuGetProcAddress_v2('cuCtxDestroy', &__cuCtxDestroy_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxPushCurrent_v2 + * _F_cuGetProcAddress_v2('cuCtxPushCurrent', &__cuCtxPushCurrent_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxPopCurrent_v2 + * _F_cuGetProcAddress_v2('cuCtxPopCurrent', &__cuCtxPopCurrent_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxPushCurrent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPushCurrent_v2), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 927, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":929 + * _F_cuGetProcAddress_v2('cuCtxPushCurrent', &__cuCtxPushCurrent_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxPopCurrent_v2 + * _F_cuGetProcAddress_v2('cuCtxPopCurrent', &__cuCtxPopCurrent_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxSetCurrent + * _F_cuGetProcAddress_v2('cuCtxSetCurrent', &__cuCtxSetCurrent, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxPopCurrent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPopCurrent_v2), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 929, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":931 + * _F_cuGetProcAddress_v2('cuCtxPopCurrent', &__cuCtxPopCurrent_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxSetCurrent + * _F_cuGetProcAddress_v2('cuCtxSetCurrent', &__cuCtxSetCurrent, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxGetCurrent + * _F_cuGetProcAddress_v2('cuCtxGetCurrent', &__cuCtxGetCurrent, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxSetCurrent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCurrent), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 931, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":933 + * _F_cuGetProcAddress_v2('cuCtxSetCurrent', &__cuCtxSetCurrent, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxGetCurrent + * _F_cuGetProcAddress_v2('cuCtxGetCurrent', &__cuCtxGetCurrent, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxGetDevice + * _F_cuGetProcAddress_v2('cuCtxGetDevice', &__cuCtxGetDevice, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxGetCurrent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCurrent), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 933, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":935 + * _F_cuGetProcAddress_v2('cuCtxGetCurrent', &__cuCtxGetCurrent, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxGetDevice + * _F_cuGetProcAddress_v2('cuCtxGetDevice', &__cuCtxGetDevice, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxGetFlags + * _F_cuGetProcAddress_v2('cuCtxGetFlags', &__cuCtxGetFlags, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxGetDevice"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevice), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 935, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":937 + * _F_cuGetProcAddress_v2('cuCtxGetDevice', &__cuCtxGetDevice, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxGetFlags + * _F_cuGetProcAddress_v2('cuCtxGetFlags', &__cuCtxGetFlags, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxSetFlags + * _F_cuGetProcAddress_v2('cuCtxSetFlags', &__cuCtxSetFlags, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxGetFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetFlags), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 937, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":939 + * _F_cuGetProcAddress_v2('cuCtxGetFlags', &__cuCtxGetFlags, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxSetFlags + * _F_cuGetProcAddress_v2('cuCtxSetFlags', &__cuCtxSetFlags, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxGetId + * _F_cuGetProcAddress_v2('cuCtxGetId', &__cuCtxGetId, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxSetFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetFlags), 0x2EEA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 939, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":941 + * _F_cuGetProcAddress_v2('cuCtxSetFlags', &__cuCtxSetFlags, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxGetId + * _F_cuGetProcAddress_v2('cuCtxGetId', &__cuCtxGetId, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxSynchronize + * _F_cuGetProcAddress_v2('cuCtxSynchronize', &__cuCtxSynchronize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxGetId"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetId), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 941, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":943 + * _F_cuGetProcAddress_v2('cuCtxGetId', &__cuCtxGetId, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxSynchronize + * _F_cuGetProcAddress_v2('cuCtxSynchronize', &__cuCtxSynchronize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxSetLimit + * _F_cuGetProcAddress_v2('cuCtxSetLimit', &__cuCtxSetLimit, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxSynchronize"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSynchronize), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 943, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":945 + * _F_cuGetProcAddress_v2('cuCtxSynchronize', &__cuCtxSynchronize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxSetLimit + * _F_cuGetProcAddress_v2('cuCtxSetLimit', &__cuCtxSetLimit, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxGetLimit + * _F_cuGetProcAddress_v2('cuCtxGetLimit', &__cuCtxGetLimit, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxSetLimit"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetLimit), 0xBC2, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 945, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":947 + * _F_cuGetProcAddress_v2('cuCtxSetLimit', &__cuCtxSetLimit, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxGetLimit + * _F_cuGetProcAddress_v2('cuCtxGetLimit', &__cuCtxGetLimit, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxGetCacheConfig + * _F_cuGetProcAddress_v2('cuCtxGetCacheConfig', &__cuCtxGetCacheConfig, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxGetLimit"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetLimit), 0xBC2, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 947, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":949 + * _F_cuGetProcAddress_v2('cuCtxGetLimit', &__cuCtxGetLimit, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxGetCacheConfig + * _F_cuGetProcAddress_v2('cuCtxGetCacheConfig', &__cuCtxGetCacheConfig, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxSetCacheConfig + * _F_cuGetProcAddress_v2('cuCtxSetCacheConfig', &__cuCtxSetCacheConfig, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxGetCacheConfig"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCacheConfig), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 949, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":951 + * _F_cuGetProcAddress_v2('cuCtxGetCacheConfig', &__cuCtxGetCacheConfig, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxSetCacheConfig + * _F_cuGetProcAddress_v2('cuCtxSetCacheConfig', &__cuCtxSetCacheConfig, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxGetApiVersion + * _F_cuGetProcAddress_v2('cuCtxGetApiVersion', &__cuCtxGetApiVersion, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxSetCacheConfig"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCacheConfig), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 951, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":953 + * _F_cuGetProcAddress_v2('cuCtxSetCacheConfig', &__cuCtxSetCacheConfig, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxGetApiVersion + * _F_cuGetProcAddress_v2('cuCtxGetApiVersion', &__cuCtxGetApiVersion, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxGetStreamPriorityRange + * _F_cuGetProcAddress_v2('cuCtxGetStreamPriorityRange', &__cuCtxGetStreamPriorityRange, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxGetApiVersion"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetApiVersion), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 953, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":955 + * _F_cuGetProcAddress_v2('cuCtxGetApiVersion', &__cuCtxGetApiVersion, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxGetStreamPriorityRange + * _F_cuGetProcAddress_v2('cuCtxGetStreamPriorityRange', &__cuCtxGetStreamPriorityRange, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxResetPersistingL2Cache + * _F_cuGetProcAddress_v2('cuCtxResetPersistingL2Cache', &__cuCtxResetPersistingL2Cache, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxGetStreamPriorityRange"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetStreamPriorityRange), 0x13BA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 955, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":957 + * _F_cuGetProcAddress_v2('cuCtxGetStreamPriorityRange', &__cuCtxGetStreamPriorityRange, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxResetPersistingL2Cache + * _F_cuGetProcAddress_v2('cuCtxResetPersistingL2Cache', &__cuCtxResetPersistingL2Cache, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxGetExecAffinity + * _F_cuGetProcAddress_v2('cuCtxGetExecAffinity', &__cuCtxGetExecAffinity, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxResetPersistingL2Cache"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxResetPersistingL2Cache), 0x2AF8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 957, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":959 + * _F_cuGetProcAddress_v2('cuCtxResetPersistingL2Cache', &__cuCtxResetPersistingL2Cache, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxGetExecAffinity + * _F_cuGetProcAddress_v2('cuCtxGetExecAffinity', &__cuCtxGetExecAffinity, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxRecordEvent + * _F_cuGetProcAddress_v2('cuCtxRecordEvent', &__cuCtxRecordEvent, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxGetExecAffinity"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetExecAffinity), 0x2B20, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 959, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":961 + * _F_cuGetProcAddress_v2('cuCtxGetExecAffinity', &__cuCtxGetExecAffinity, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxRecordEvent + * _F_cuGetProcAddress_v2('cuCtxRecordEvent', &__cuCtxRecordEvent, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxWaitEvent + * _F_cuGetProcAddress_v2('cuCtxWaitEvent', &__cuCtxWaitEvent, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxRecordEvent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxRecordEvent), 0x2F12, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 961, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":963 + * _F_cuGetProcAddress_v2('cuCtxRecordEvent', &__cuCtxRecordEvent, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxWaitEvent + * _F_cuGetProcAddress_v2('cuCtxWaitEvent', &__cuCtxWaitEvent, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxAttach + * _F_cuGetProcAddress_v2('cuCtxAttach', &__cuCtxAttach, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxWaitEvent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxWaitEvent), 0x2F12, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 963, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":965 + * _F_cuGetProcAddress_v2('cuCtxWaitEvent', &__cuCtxWaitEvent, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxAttach + * _F_cuGetProcAddress_v2('cuCtxAttach', &__cuCtxAttach, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxDetach + * _F_cuGetProcAddress_v2('cuCtxDetach', &__cuCtxDetach, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxAttach"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxAttach), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 965, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":967 + * _F_cuGetProcAddress_v2('cuCtxAttach', &__cuCtxAttach, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxDetach + * _F_cuGetProcAddress_v2('cuCtxDetach', &__cuCtxDetach, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxGetSharedMemConfig + * _F_cuGetProcAddress_v2('cuCtxGetSharedMemConfig', &__cuCtxGetSharedMemConfig, 4020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxDetach"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDetach), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 967, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":969 + * _F_cuGetProcAddress_v2('cuCtxDetach', &__cuCtxDetach, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxGetSharedMemConfig + * _F_cuGetProcAddress_v2('cuCtxGetSharedMemConfig', &__cuCtxGetSharedMemConfig, 4020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxSetSharedMemConfig + * _F_cuGetProcAddress_v2('cuCtxSetSharedMemConfig', &__cuCtxSetSharedMemConfig, 4020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxGetSharedMemConfig"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetSharedMemConfig), 0xFB4, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 969, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":971 + * _F_cuGetProcAddress_v2('cuCtxGetSharedMemConfig', &__cuCtxGetSharedMemConfig, 4020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxSetSharedMemConfig + * _F_cuGetProcAddress_v2('cuCtxSetSharedMemConfig', &__cuCtxSetSharedMemConfig, 4020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuModuleLoad + * _F_cuGetProcAddress_v2('cuModuleLoad', &__cuModuleLoad, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxSetSharedMemConfig"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetSharedMemConfig), 0xFB4, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 971, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":973 + * _F_cuGetProcAddress_v2('cuCtxSetSharedMemConfig', &__cuCtxSetSharedMemConfig, 4020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuModuleLoad + * _F_cuGetProcAddress_v2('cuModuleLoad', &__cuModuleLoad, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuModuleLoadData + * _F_cuGetProcAddress_v2('cuModuleLoadData', &__cuModuleLoadData, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuModuleLoad"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoad), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 973, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":975 + * _F_cuGetProcAddress_v2('cuModuleLoad', &__cuModuleLoad, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuModuleLoadData + * _F_cuGetProcAddress_v2('cuModuleLoadData', &__cuModuleLoadData, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuModuleLoadDataEx + * _F_cuGetProcAddress_v2('cuModuleLoadDataEx', &__cuModuleLoadDataEx, 2010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuModuleLoadData"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadData), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 975, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":977 + * _F_cuGetProcAddress_v2('cuModuleLoadData', &__cuModuleLoadData, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuModuleLoadDataEx + * _F_cuGetProcAddress_v2('cuModuleLoadDataEx', &__cuModuleLoadDataEx, 2010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuModuleLoadFatBinary + * _F_cuGetProcAddress_v2('cuModuleLoadFatBinary', &__cuModuleLoadFatBinary, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuModuleLoadDataEx"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadDataEx), 0x7DA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 977, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":979 + * _F_cuGetProcAddress_v2('cuModuleLoadDataEx', &__cuModuleLoadDataEx, 2010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuModuleLoadFatBinary + * _F_cuGetProcAddress_v2('cuModuleLoadFatBinary', &__cuModuleLoadFatBinary, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuModuleUnload + * _F_cuGetProcAddress_v2('cuModuleUnload', &__cuModuleUnload, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuModuleLoadFatBinary"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadFatBinary), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 979, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":981 + * _F_cuGetProcAddress_v2('cuModuleLoadFatBinary', &__cuModuleLoadFatBinary, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuModuleUnload + * _F_cuGetProcAddress_v2('cuModuleUnload', &__cuModuleUnload, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuModuleGetLoadingMode + * _F_cuGetProcAddress_v2('cuModuleGetLoadingMode', &__cuModuleGetLoadingMode, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuModuleUnload"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleUnload), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 981, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":983 + * _F_cuGetProcAddress_v2('cuModuleUnload', &__cuModuleUnload, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuModuleGetLoadingMode + * _F_cuGetProcAddress_v2('cuModuleGetLoadingMode', &__cuModuleGetLoadingMode, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuModuleGetFunction + * _F_cuGetProcAddress_v2('cuModuleGetFunction', &__cuModuleGetFunction, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuModuleGetLoadingMode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetLoadingMode), 0x2B3E, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 983, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":985 + * _F_cuGetProcAddress_v2('cuModuleGetLoadingMode', &__cuModuleGetLoadingMode, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuModuleGetFunction + * _F_cuGetProcAddress_v2('cuModuleGetFunction', &__cuModuleGetFunction, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuModuleGetFunctionCount + * _F_cuGetProcAddress_v2('cuModuleGetFunctionCount', &__cuModuleGetFunctionCount, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuModuleGetFunction"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunction), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 985, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":987 + * _F_cuGetProcAddress_v2('cuModuleGetFunction', &__cuModuleGetFunction, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuModuleGetFunctionCount + * _F_cuGetProcAddress_v2('cuModuleGetFunctionCount', &__cuModuleGetFunctionCount, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuModuleEnumerateFunctions + * _F_cuGetProcAddress_v2('cuModuleEnumerateFunctions', &__cuModuleEnumerateFunctions, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuModuleGetFunctionCount"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunctionCount), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 987, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":989 + * _F_cuGetProcAddress_v2('cuModuleGetFunctionCount', &__cuModuleGetFunctionCount, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuModuleEnumerateFunctions + * _F_cuGetProcAddress_v2('cuModuleEnumerateFunctions', &__cuModuleEnumerateFunctions, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuModuleGetGlobal_v2 + * _F_cuGetProcAddress_v2('cuModuleGetGlobal', &__cuModuleGetGlobal_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuModuleEnumerateFunctions"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleEnumerateFunctions), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 989, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":991 + * _F_cuGetProcAddress_v2('cuModuleEnumerateFunctions', &__cuModuleEnumerateFunctions, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuModuleGetGlobal_v2 + * _F_cuGetProcAddress_v2('cuModuleGetGlobal', &__cuModuleGetGlobal_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLinkCreate_v2 + * _F_cuGetProcAddress_v2('cuLinkCreate', &__cuLinkCreate_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuModuleGetGlobal"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetGlobal_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 991, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":993 + * _F_cuGetProcAddress_v2('cuModuleGetGlobal', &__cuModuleGetGlobal_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLinkCreate_v2 + * _F_cuGetProcAddress_v2('cuLinkCreate', &__cuLinkCreate_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLinkAddData_v2 + * _F_cuGetProcAddress_v2('cuLinkAddData', &__cuLinkAddData_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLinkCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkCreate_v2), 0x17A2, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 993, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":995 + * _F_cuGetProcAddress_v2('cuLinkCreate', &__cuLinkCreate_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLinkAddData_v2 + * _F_cuGetProcAddress_v2('cuLinkAddData', &__cuLinkAddData_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLinkAddFile_v2 + * _F_cuGetProcAddress_v2('cuLinkAddFile', &__cuLinkAddFile_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLinkAddData"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddData_v2), 0x17A2, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 995, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":997 + * _F_cuGetProcAddress_v2('cuLinkAddData', &__cuLinkAddData_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLinkAddFile_v2 + * _F_cuGetProcAddress_v2('cuLinkAddFile', &__cuLinkAddFile_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLinkComplete + * _F_cuGetProcAddress_v2('cuLinkComplete', &__cuLinkComplete, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLinkAddFile"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddFile_v2), 0x17A2, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 997, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":999 + * _F_cuGetProcAddress_v2('cuLinkAddFile', &__cuLinkAddFile_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLinkComplete + * _F_cuGetProcAddress_v2('cuLinkComplete', &__cuLinkComplete, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLinkDestroy + * _F_cuGetProcAddress_v2('cuLinkDestroy', &__cuLinkDestroy, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLinkComplete"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkComplete), 0x13BA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 999, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1001 + * _F_cuGetProcAddress_v2('cuLinkComplete', &__cuLinkComplete, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLinkDestroy + * _F_cuGetProcAddress_v2('cuLinkDestroy', &__cuLinkDestroy, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuModuleGetTexRef + * _F_cuGetProcAddress_v2('cuModuleGetTexRef', &__cuModuleGetTexRef, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLinkDestroy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkDestroy), 0x13BA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1001, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1003 + * _F_cuGetProcAddress_v2('cuLinkDestroy', &__cuLinkDestroy, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuModuleGetTexRef + * _F_cuGetProcAddress_v2('cuModuleGetTexRef', &__cuModuleGetTexRef, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuModuleGetSurfRef + * _F_cuGetProcAddress_v2('cuModuleGetSurfRef', &__cuModuleGetSurfRef, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuModuleGetTexRef"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetTexRef), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1003, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1005 + * _F_cuGetProcAddress_v2('cuModuleGetTexRef', &__cuModuleGetTexRef, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuModuleGetSurfRef + * _F_cuGetProcAddress_v2('cuModuleGetSurfRef', &__cuModuleGetSurfRef, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLibraryLoadData + * _F_cuGetProcAddress_v2('cuLibraryLoadData', &__cuLibraryLoadData, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuModuleGetSurfRef"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetSurfRef), 0xBB8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1005, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1007 + * _F_cuGetProcAddress_v2('cuModuleGetSurfRef', &__cuModuleGetSurfRef, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLibraryLoadData + * _F_cuGetProcAddress_v2('cuLibraryLoadData', &__cuLibraryLoadData, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLibraryLoadFromFile + * _F_cuGetProcAddress_v2('cuLibraryLoadFromFile', &__cuLibraryLoadFromFile, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLibraryLoadData"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadData), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1007, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1009 + * _F_cuGetProcAddress_v2('cuLibraryLoadData', &__cuLibraryLoadData, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLibraryLoadFromFile + * _F_cuGetProcAddress_v2('cuLibraryLoadFromFile', &__cuLibraryLoadFromFile, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLibraryUnload + * _F_cuGetProcAddress_v2('cuLibraryUnload', &__cuLibraryUnload, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLibraryLoadFromFile"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadFromFile), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1009, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1011 + * _F_cuGetProcAddress_v2('cuLibraryLoadFromFile', &__cuLibraryLoadFromFile, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLibraryUnload + * _F_cuGetProcAddress_v2('cuLibraryUnload', &__cuLibraryUnload, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLibraryGetKernel + * _F_cuGetProcAddress_v2('cuLibraryGetKernel', &__cuLibraryGetKernel, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLibraryUnload"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryUnload), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1011, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1013 + * _F_cuGetProcAddress_v2('cuLibraryUnload', &__cuLibraryUnload, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLibraryGetKernel + * _F_cuGetProcAddress_v2('cuLibraryGetKernel', &__cuLibraryGetKernel, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLibraryGetKernelCount + * _F_cuGetProcAddress_v2('cuLibraryGetKernelCount', &__cuLibraryGetKernelCount, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLibraryGetKernel"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernel), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1013, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1015 + * _F_cuGetProcAddress_v2('cuLibraryGetKernel', &__cuLibraryGetKernel, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLibraryGetKernelCount + * _F_cuGetProcAddress_v2('cuLibraryGetKernelCount', &__cuLibraryGetKernelCount, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLibraryEnumerateKernels + * _F_cuGetProcAddress_v2('cuLibraryEnumerateKernels', &__cuLibraryEnumerateKernels, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLibraryGetKernelCount"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernelCount), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1015, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1017 + * _F_cuGetProcAddress_v2('cuLibraryGetKernelCount', &__cuLibraryGetKernelCount, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLibraryEnumerateKernels + * _F_cuGetProcAddress_v2('cuLibraryEnumerateKernels', &__cuLibraryEnumerateKernels, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLibraryGetModule + * _F_cuGetProcAddress_v2('cuLibraryGetModule', &__cuLibraryGetModule, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLibraryEnumerateKernels"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryEnumerateKernels), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1017, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1019 + * _F_cuGetProcAddress_v2('cuLibraryEnumerateKernels', &__cuLibraryEnumerateKernels, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLibraryGetModule + * _F_cuGetProcAddress_v2('cuLibraryGetModule', &__cuLibraryGetModule, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuKernelGetFunction + * _F_cuGetProcAddress_v2('cuKernelGetFunction', &__cuKernelGetFunction, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLibraryGetModule"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetModule), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1019, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1021 + * _F_cuGetProcAddress_v2('cuLibraryGetModule', &__cuLibraryGetModule, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuKernelGetFunction + * _F_cuGetProcAddress_v2('cuKernelGetFunction', &__cuKernelGetFunction, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuKernelGetLibrary + * _F_cuGetProcAddress_v2('cuKernelGetLibrary', &__cuKernelGetLibrary, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuKernelGetFunction"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetFunction), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1021, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1023 + * _F_cuGetProcAddress_v2('cuKernelGetFunction', &__cuKernelGetFunction, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuKernelGetLibrary + * _F_cuGetProcAddress_v2('cuKernelGetLibrary', &__cuKernelGetLibrary, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLibraryGetGlobal + * _F_cuGetProcAddress_v2('cuLibraryGetGlobal', &__cuLibraryGetGlobal, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuKernelGetLibrary"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetLibrary), 0x2F12, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1023, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1025 + * _F_cuGetProcAddress_v2('cuKernelGetLibrary', &__cuKernelGetLibrary, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLibraryGetGlobal + * _F_cuGetProcAddress_v2('cuLibraryGetGlobal', &__cuLibraryGetGlobal, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLibraryGetManaged + * _F_cuGetProcAddress_v2('cuLibraryGetManaged', &__cuLibraryGetManaged, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLibraryGetGlobal"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetGlobal), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1025, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1027 + * _F_cuGetProcAddress_v2('cuLibraryGetGlobal', &__cuLibraryGetGlobal, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLibraryGetManaged + * _F_cuGetProcAddress_v2('cuLibraryGetManaged', &__cuLibraryGetManaged, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLibraryGetUnifiedFunction + * _F_cuGetProcAddress_v2('cuLibraryGetUnifiedFunction', &__cuLibraryGetUnifiedFunction, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLibraryGetManaged"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetManaged), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1027, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1029 + * _F_cuGetProcAddress_v2('cuLibraryGetManaged', &__cuLibraryGetManaged, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLibraryGetUnifiedFunction + * _F_cuGetProcAddress_v2('cuLibraryGetUnifiedFunction', &__cuLibraryGetUnifiedFunction, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuKernelGetAttribute + * _F_cuGetProcAddress_v2('cuKernelGetAttribute', &__cuKernelGetAttribute, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLibraryGetUnifiedFunction"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetUnifiedFunction), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1029, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1031 + * _F_cuGetProcAddress_v2('cuLibraryGetUnifiedFunction', &__cuLibraryGetUnifiedFunction, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuKernelGetAttribute + * _F_cuGetProcAddress_v2('cuKernelGetAttribute', &__cuKernelGetAttribute, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuKernelSetAttribute + * _F_cuGetProcAddress_v2('cuKernelSetAttribute', &__cuKernelSetAttribute, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuKernelGetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetAttribute), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1031, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1033 + * _F_cuGetProcAddress_v2('cuKernelGetAttribute', &__cuKernelGetAttribute, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuKernelSetAttribute + * _F_cuGetProcAddress_v2('cuKernelSetAttribute', &__cuKernelSetAttribute, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuKernelSetCacheConfig + * _F_cuGetProcAddress_v2('cuKernelSetCacheConfig', &__cuKernelSetCacheConfig, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuKernelSetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetAttribute), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1033, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1035 + * _F_cuGetProcAddress_v2('cuKernelSetAttribute', &__cuKernelSetAttribute, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuKernelSetCacheConfig + * _F_cuGetProcAddress_v2('cuKernelSetCacheConfig', &__cuKernelSetCacheConfig, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuKernelGetName + * _F_cuGetProcAddress_v2('cuKernelGetName', &__cuKernelGetName, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuKernelSetCacheConfig"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetCacheConfig), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1035, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1037 + * _F_cuGetProcAddress_v2('cuKernelSetCacheConfig', &__cuKernelSetCacheConfig, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuKernelGetName + * _F_cuGetProcAddress_v2('cuKernelGetName', &__cuKernelGetName, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuKernelGetParamInfo + * _F_cuGetProcAddress_v2('cuKernelGetParamInfo', &__cuKernelGetParamInfo, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuKernelGetName"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetName), 0x2EFE, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1037, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1039 + * _F_cuGetProcAddress_v2('cuKernelGetName', &__cuKernelGetName, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuKernelGetParamInfo + * _F_cuGetProcAddress_v2('cuKernelGetParamInfo', &__cuKernelGetParamInfo, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemGetInfo_v2 + * _F_cuGetProcAddress_v2('cuMemGetInfo', &__cuMemGetInfo_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuKernelGetParamInfo"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetParamInfo), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1039, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1041 + * _F_cuGetProcAddress_v2('cuKernelGetParamInfo', &__cuKernelGetParamInfo, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemGetInfo_v2 + * _F_cuGetProcAddress_v2('cuMemGetInfo', &__cuMemGetInfo_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemAlloc_v2 + * _F_cuGetProcAddress_v2('cuMemAlloc', &__cuMemAlloc_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemGetInfo"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetInfo_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1041, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1043 + * _F_cuGetProcAddress_v2('cuMemGetInfo', &__cuMemGetInfo_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemAlloc_v2 + * _F_cuGetProcAddress_v2('cuMemAlloc', &__cuMemAlloc_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemAllocPitch_v2 + * _F_cuGetProcAddress_v2('cuMemAllocPitch', &__cuMemAllocPitch_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemAlloc"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAlloc_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1043, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1045 + * _F_cuGetProcAddress_v2('cuMemAlloc', &__cuMemAlloc_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemAllocPitch_v2 + * _F_cuGetProcAddress_v2('cuMemAllocPitch', &__cuMemAllocPitch_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemFree_v2 + * _F_cuGetProcAddress_v2('cuMemFree', &__cuMemFree_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemAllocPitch"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocPitch_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1045, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1047 + * _F_cuGetProcAddress_v2('cuMemAllocPitch', &__cuMemAllocPitch_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemFree_v2 + * _F_cuGetProcAddress_v2('cuMemFree', &__cuMemFree_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemGetAddressRange_v2 + * _F_cuGetProcAddress_v2('cuMemGetAddressRange', &__cuMemGetAddressRange_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemFree"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFree_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1047, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1049 + * _F_cuGetProcAddress_v2('cuMemFree', &__cuMemFree_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemGetAddressRange_v2 + * _F_cuGetProcAddress_v2('cuMemGetAddressRange', &__cuMemGetAddressRange_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemAllocHost_v2 + * _F_cuGetProcAddress_v2('cuMemAllocHost', &__cuMemAllocHost_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemGetAddressRange"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAddressRange_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1049, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1051 + * _F_cuGetProcAddress_v2('cuMemGetAddressRange', &__cuMemGetAddressRange_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemAllocHost_v2 + * _F_cuGetProcAddress_v2('cuMemAllocHost', &__cuMemAllocHost_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemFreeHost + * _F_cuGetProcAddress_v2('cuMemFreeHost', &__cuMemFreeHost, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemAllocHost"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocHost_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1051, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1053 + * _F_cuGetProcAddress_v2('cuMemAllocHost', &__cuMemAllocHost_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemFreeHost + * _F_cuGetProcAddress_v2('cuMemFreeHost', &__cuMemFreeHost, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemHostAlloc + * _F_cuGetProcAddress_v2('cuMemHostAlloc', &__cuMemHostAlloc, 2020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemFreeHost"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeHost), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1053, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1055 + * _F_cuGetProcAddress_v2('cuMemFreeHost', &__cuMemFreeHost, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemHostAlloc + * _F_cuGetProcAddress_v2('cuMemHostAlloc', &__cuMemHostAlloc, 2020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemHostGetDevicePointer_v2 + * _F_cuGetProcAddress_v2('cuMemHostGetDevicePointer', &__cuMemHostGetDevicePointer_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemHostAlloc"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostAlloc), 0x7E4, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1055, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1057 + * _F_cuGetProcAddress_v2('cuMemHostAlloc', &__cuMemHostAlloc, 2020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemHostGetDevicePointer_v2 + * _F_cuGetProcAddress_v2('cuMemHostGetDevicePointer', &__cuMemHostGetDevicePointer_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemHostGetFlags + * _F_cuGetProcAddress_v2('cuMemHostGetFlags', &__cuMemHostGetFlags, 2030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemHostGetDevicePointer"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetDevicePointer_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1057, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1059 + * _F_cuGetProcAddress_v2('cuMemHostGetDevicePointer', &__cuMemHostGetDevicePointer_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemHostGetFlags + * _F_cuGetProcAddress_v2('cuMemHostGetFlags', &__cuMemHostGetFlags, 2030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemAllocManaged + * _F_cuGetProcAddress_v2('cuMemAllocManaged', &__cuMemAllocManaged, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemHostGetFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetFlags), 0x7EE, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1059, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1061 + * _F_cuGetProcAddress_v2('cuMemHostGetFlags', &__cuMemHostGetFlags, 2030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemAllocManaged + * _F_cuGetProcAddress_v2('cuMemAllocManaged', &__cuMemAllocManaged, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceRegisterAsyncNotification + * _F_cuGetProcAddress_v2('cuDeviceRegisterAsyncNotification', &__cuDeviceRegisterAsyncNotification, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemAllocManaged"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocManaged), 0x1770, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1061, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1063 + * _F_cuGetProcAddress_v2('cuMemAllocManaged', &__cuMemAllocManaged, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceRegisterAsyncNotification + * _F_cuGetProcAddress_v2('cuDeviceRegisterAsyncNotification', &__cuDeviceRegisterAsyncNotification, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceUnregisterAsyncNotification + * _F_cuGetProcAddress_v2('cuDeviceUnregisterAsyncNotification', &__cuDeviceUnregisterAsyncNotification, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceRegisterAsyncNotification"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceRegisterAsyncNotification), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1063, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1065 + * _F_cuGetProcAddress_v2('cuDeviceRegisterAsyncNotification', &__cuDeviceRegisterAsyncNotification, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceUnregisterAsyncNotification + * _F_cuGetProcAddress_v2('cuDeviceUnregisterAsyncNotification', &__cuDeviceUnregisterAsyncNotification, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetByPCIBusId + * _F_cuGetProcAddress_v2('cuDeviceGetByPCIBusId', &__cuDeviceGetByPCIBusId, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceUnregisterAsyncNotification"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceUnregisterAsyncNotification), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1065, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1067 + * _F_cuGetProcAddress_v2('cuDeviceUnregisterAsyncNotification', &__cuDeviceUnregisterAsyncNotification, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetByPCIBusId + * _F_cuGetProcAddress_v2('cuDeviceGetByPCIBusId', &__cuDeviceGetByPCIBusId, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetPCIBusId + * _F_cuGetProcAddress_v2('cuDeviceGetPCIBusId', &__cuDeviceGetPCIBusId, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetByPCIBusId"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetByPCIBusId), 0xFAA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1067, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1069 + * _F_cuGetProcAddress_v2('cuDeviceGetByPCIBusId', &__cuDeviceGetByPCIBusId, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetPCIBusId + * _F_cuGetProcAddress_v2('cuDeviceGetPCIBusId', &__cuDeviceGetPCIBusId, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuIpcGetEventHandle + * _F_cuGetProcAddress_v2('cuIpcGetEventHandle', &__cuIpcGetEventHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetPCIBusId"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetPCIBusId), 0xFAA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1069, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1071 + * _F_cuGetProcAddress_v2('cuDeviceGetPCIBusId', &__cuDeviceGetPCIBusId, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuIpcGetEventHandle + * _F_cuGetProcAddress_v2('cuIpcGetEventHandle', &__cuIpcGetEventHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuIpcOpenEventHandle + * _F_cuGetProcAddress_v2('cuIpcOpenEventHandle', &__cuIpcOpenEventHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuIpcGetEventHandle"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetEventHandle), 0xFAA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1071, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1073 + * _F_cuGetProcAddress_v2('cuIpcGetEventHandle', &__cuIpcGetEventHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuIpcOpenEventHandle + * _F_cuGetProcAddress_v2('cuIpcOpenEventHandle', &__cuIpcOpenEventHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuIpcGetMemHandle + * _F_cuGetProcAddress_v2('cuIpcGetMemHandle', &__cuIpcGetMemHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuIpcOpenEventHandle"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenEventHandle), 0xFAA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1073, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1075 + * _F_cuGetProcAddress_v2('cuIpcOpenEventHandle', &__cuIpcOpenEventHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuIpcGetMemHandle + * _F_cuGetProcAddress_v2('cuIpcGetMemHandle', &__cuIpcGetMemHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuIpcOpenMemHandle_v2 + * _F_cuGetProcAddress_v2('cuIpcOpenMemHandle', &__cuIpcOpenMemHandle_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuIpcGetMemHandle"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetMemHandle), 0xFAA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1075, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1077 + * _F_cuGetProcAddress_v2('cuIpcGetMemHandle', &__cuIpcGetMemHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuIpcOpenMemHandle_v2 + * _F_cuGetProcAddress_v2('cuIpcOpenMemHandle', &__cuIpcOpenMemHandle_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuIpcCloseMemHandle + * _F_cuGetProcAddress_v2('cuIpcCloseMemHandle', &__cuIpcCloseMemHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuIpcOpenMemHandle"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenMemHandle_v2), 0x2AF8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1077, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1079 + * _F_cuGetProcAddress_v2('cuIpcOpenMemHandle', &__cuIpcOpenMemHandle_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuIpcCloseMemHandle + * _F_cuGetProcAddress_v2('cuIpcCloseMemHandle', &__cuIpcCloseMemHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemHostRegister_v2 + * _F_cuGetProcAddress_v2('cuMemHostRegister', &__cuMemHostRegister_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuIpcCloseMemHandle"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcCloseMemHandle), 0xFAA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1079, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1081 + * _F_cuGetProcAddress_v2('cuIpcCloseMemHandle', &__cuIpcCloseMemHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemHostRegister_v2 + * _F_cuGetProcAddress_v2('cuMemHostRegister', &__cuMemHostRegister_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemHostUnregister + * _F_cuGetProcAddress_v2('cuMemHostUnregister', &__cuMemHostUnregister, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemHostRegister"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostRegister_v2), 0x17A2, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1081, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1083 + * _F_cuGetProcAddress_v2('cuMemHostRegister', &__cuMemHostRegister_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemHostUnregister + * _F_cuGetProcAddress_v2('cuMemHostUnregister', &__cuMemHostUnregister, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuArrayCreate_v2 + * _F_cuGetProcAddress_v2('cuArrayCreate', &__cuArrayCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemHostUnregister"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostUnregister), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1083, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1085 + * _F_cuGetProcAddress_v2('cuMemHostUnregister', &__cuMemHostUnregister, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuArrayCreate_v2 + * _F_cuGetProcAddress_v2('cuArrayCreate', &__cuArrayCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuArrayGetDescriptor_v2 + * _F_cuGetProcAddress_v2('cuArrayGetDescriptor', &__cuArrayGetDescriptor_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuArrayCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayCreate_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1085, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1087 + * _F_cuGetProcAddress_v2('cuArrayCreate', &__cuArrayCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuArrayGetDescriptor_v2 + * _F_cuGetProcAddress_v2('cuArrayGetDescriptor', &__cuArrayGetDescriptor_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuArrayGetSparseProperties + * _F_cuGetProcAddress_v2('cuArrayGetSparseProperties', &__cuArrayGetSparseProperties, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuArrayGetDescriptor"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetDescriptor_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1087, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1089 + * _F_cuGetProcAddress_v2('cuArrayGetDescriptor', &__cuArrayGetDescriptor_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuArrayGetSparseProperties + * _F_cuGetProcAddress_v2('cuArrayGetSparseProperties', &__cuArrayGetSparseProperties, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetSparseProperties + * _F_cuGetProcAddress_v2('cuMipmappedArrayGetSparseProperties', &__cuMipmappedArrayGetSparseProperties, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuArrayGetSparseProperties"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetSparseProperties), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1089, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1091 + * _F_cuGetProcAddress_v2('cuArrayGetSparseProperties', &__cuArrayGetSparseProperties, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMipmappedArrayGetSparseProperties + * _F_cuGetProcAddress_v2('cuMipmappedArrayGetSparseProperties', &__cuMipmappedArrayGetSparseProperties, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuArrayGetMemoryRequirements + * _F_cuGetProcAddress_v2('cuArrayGetMemoryRequirements', &__cuArrayGetMemoryRequirements, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMipmappedArrayGetSparseProperties"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetSparseProperties), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1091, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1093 + * _F_cuGetProcAddress_v2('cuMipmappedArrayGetSparseProperties', &__cuMipmappedArrayGetSparseProperties, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuArrayGetMemoryRequirements + * _F_cuGetProcAddress_v2('cuArrayGetMemoryRequirements', &__cuArrayGetMemoryRequirements, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetMemoryRequirements + * _F_cuGetProcAddress_v2('cuMipmappedArrayGetMemoryRequirements', &__cuMipmappedArrayGetMemoryRequirements, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuArrayGetMemoryRequirements"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetMemoryRequirements), 0x2B34, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1093, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1095 + * _F_cuGetProcAddress_v2('cuArrayGetMemoryRequirements', &__cuArrayGetMemoryRequirements, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMipmappedArrayGetMemoryRequirements + * _F_cuGetProcAddress_v2('cuMipmappedArrayGetMemoryRequirements', &__cuMipmappedArrayGetMemoryRequirements, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuArrayGetPlane + * _F_cuGetProcAddress_v2('cuArrayGetPlane', &__cuArrayGetPlane, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMipmappedArrayGetMemoryRequirements"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetMemoryRequirements), 0x2B34, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1095, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1097 + * _F_cuGetProcAddress_v2('cuMipmappedArrayGetMemoryRequirements', &__cuMipmappedArrayGetMemoryRequirements, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuArrayGetPlane + * _F_cuGetProcAddress_v2('cuArrayGetPlane', &__cuArrayGetPlane, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuArrayDestroy + * _F_cuGetProcAddress_v2('cuArrayDestroy', &__cuArrayDestroy, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuArrayGetPlane"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetPlane), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1097, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1099 + * _F_cuGetProcAddress_v2('cuArrayGetPlane', &__cuArrayGetPlane, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuArrayDestroy + * _F_cuGetProcAddress_v2('cuArrayDestroy', &__cuArrayDestroy, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuArray3DCreate_v2 + * _F_cuGetProcAddress_v2('cuArray3DCreate', &__cuArray3DCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuArrayDestroy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayDestroy), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1099, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1101 + * _F_cuGetProcAddress_v2('cuArrayDestroy', &__cuArrayDestroy, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuArray3DCreate_v2 + * _F_cuGetProcAddress_v2('cuArray3DCreate', &__cuArray3DCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuArray3DGetDescriptor_v2 + * _F_cuGetProcAddress_v2('cuArray3DGetDescriptor', &__cuArray3DGetDescriptor_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuArray3DCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DCreate_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1101, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1103 + * _F_cuGetProcAddress_v2('cuArray3DCreate', &__cuArray3DCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuArray3DGetDescriptor_v2 + * _F_cuGetProcAddress_v2('cuArray3DGetDescriptor', &__cuArray3DGetDescriptor_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMipmappedArrayCreate + * _F_cuGetProcAddress_v2('cuMipmappedArrayCreate', &__cuMipmappedArrayCreate, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuArray3DGetDescriptor"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DGetDescriptor_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1103, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1105 + * _F_cuGetProcAddress_v2('cuArray3DGetDescriptor', &__cuArray3DGetDescriptor_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMipmappedArrayCreate + * _F_cuGetProcAddress_v2('cuMipmappedArrayCreate', &__cuMipmappedArrayCreate, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetLevel + * _F_cuGetProcAddress_v2('cuMipmappedArrayGetLevel', &__cuMipmappedArrayGetLevel, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMipmappedArrayCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayCreate), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1105, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1107 + * _F_cuGetProcAddress_v2('cuMipmappedArrayCreate', &__cuMipmappedArrayCreate, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMipmappedArrayGetLevel + * _F_cuGetProcAddress_v2('cuMipmappedArrayGetLevel', &__cuMipmappedArrayGetLevel, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMipmappedArrayDestroy + * _F_cuGetProcAddress_v2('cuMipmappedArrayDestroy', &__cuMipmappedArrayDestroy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMipmappedArrayGetLevel"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetLevel), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1107, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1109 + * _F_cuGetProcAddress_v2('cuMipmappedArrayGetLevel', &__cuMipmappedArrayGetLevel, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMipmappedArrayDestroy + * _F_cuGetProcAddress_v2('cuMipmappedArrayDestroy', &__cuMipmappedArrayDestroy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemGetHandleForAddressRange + * _F_cuGetProcAddress_v2('cuMemGetHandleForAddressRange', &__cuMemGetHandleForAddressRange, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMipmappedArrayDestroy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayDestroy), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1109, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1111 + * _F_cuGetProcAddress_v2('cuMipmappedArrayDestroy', &__cuMipmappedArrayDestroy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemGetHandleForAddressRange + * _F_cuGetProcAddress_v2('cuMemGetHandleForAddressRange', &__cuMemGetHandleForAddressRange, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemAddressReserve + * _F_cuGetProcAddress_v2('cuMemAddressReserve', &__cuMemAddressReserve, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemGetHandleForAddressRange"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetHandleForAddressRange), 0x2B3E, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1111, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1113 + * _F_cuGetProcAddress_v2('cuMemGetHandleForAddressRange', &__cuMemGetHandleForAddressRange, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemAddressReserve + * _F_cuGetProcAddress_v2('cuMemAddressReserve', &__cuMemAddressReserve, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemAddressFree + * _F_cuGetProcAddress_v2('cuMemAddressFree', &__cuMemAddressFree, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemAddressReserve"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressReserve), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1113, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1115 + * _F_cuGetProcAddress_v2('cuMemAddressReserve', &__cuMemAddressReserve, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemAddressFree + * _F_cuGetProcAddress_v2('cuMemAddressFree', &__cuMemAddressFree, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemCreate + * _F_cuGetProcAddress_v2('cuMemCreate', &__cuMemCreate, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemAddressFree"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressFree), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1115, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1117 + * _F_cuGetProcAddress_v2('cuMemAddressFree', &__cuMemAddressFree, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemCreate + * _F_cuGetProcAddress_v2('cuMemCreate', &__cuMemCreate, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemRelease + * _F_cuGetProcAddress_v2('cuMemRelease', &__cuMemRelease, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemCreate), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1117, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1119 + * _F_cuGetProcAddress_v2('cuMemCreate', &__cuMemCreate, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemRelease + * _F_cuGetProcAddress_v2('cuMemRelease', &__cuMemRelease, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemMap + * _F_cuGetProcAddress_v2('cuMemMap', &__cuMemMap, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemRelease"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRelease), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1119, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1121 + * _F_cuGetProcAddress_v2('cuMemRelease', &__cuMemRelease, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemMap + * _F_cuGetProcAddress_v2('cuMemMap', &__cuMemMap, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemUnmap + * _F_cuGetProcAddress_v2('cuMemUnmap', &__cuMemUnmap, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemMap"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMap), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1121, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1123 + * _F_cuGetProcAddress_v2('cuMemMap', &__cuMemMap, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemUnmap + * _F_cuGetProcAddress_v2('cuMemUnmap', &__cuMemUnmap, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemSetAccess + * _F_cuGetProcAddress_v2('cuMemSetAccess', &__cuMemSetAccess, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemUnmap"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemUnmap), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1123, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1125 + * _F_cuGetProcAddress_v2('cuMemUnmap', &__cuMemUnmap, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemSetAccess + * _F_cuGetProcAddress_v2('cuMemSetAccess', &__cuMemSetAccess, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemGetAccess + * _F_cuGetProcAddress_v2('cuMemGetAccess', &__cuMemGetAccess, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemSetAccess"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemSetAccess), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1125, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1127 + * _F_cuGetProcAddress_v2('cuMemSetAccess', &__cuMemSetAccess, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemGetAccess + * _F_cuGetProcAddress_v2('cuMemGetAccess', &__cuMemGetAccess, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemExportToShareableHandle + * _F_cuGetProcAddress_v2('cuMemExportToShareableHandle', &__cuMemExportToShareableHandle, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemGetAccess"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAccess), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1127, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1129 + * _F_cuGetProcAddress_v2('cuMemGetAccess', &__cuMemGetAccess, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemExportToShareableHandle + * _F_cuGetProcAddress_v2('cuMemExportToShareableHandle', &__cuMemExportToShareableHandle, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemImportFromShareableHandle + * _F_cuGetProcAddress_v2('cuMemImportFromShareableHandle', &__cuMemImportFromShareableHandle, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemExportToShareableHandle"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemExportToShareableHandle), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1129, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1131 + * _F_cuGetProcAddress_v2('cuMemExportToShareableHandle', &__cuMemExportToShareableHandle, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemImportFromShareableHandle + * _F_cuGetProcAddress_v2('cuMemImportFromShareableHandle', &__cuMemImportFromShareableHandle, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemGetAllocationGranularity + * _F_cuGetProcAddress_v2('cuMemGetAllocationGranularity', &__cuMemGetAllocationGranularity, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemImportFromShareableHandle"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemImportFromShareableHandle), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1131, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1133 + * _F_cuGetProcAddress_v2('cuMemImportFromShareableHandle', &__cuMemImportFromShareableHandle, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemGetAllocationGranularity + * _F_cuGetProcAddress_v2('cuMemGetAllocationGranularity', &__cuMemGetAllocationGranularity, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemGetAllocationPropertiesFromHandle + * _F_cuGetProcAddress_v2('cuMemGetAllocationPropertiesFromHandle', &__cuMemGetAllocationPropertiesFromHandle, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemGetAllocationGranularity"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationGranularity), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1133, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1135 + * _F_cuGetProcAddress_v2('cuMemGetAllocationGranularity', &__cuMemGetAllocationGranularity, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemGetAllocationPropertiesFromHandle + * _F_cuGetProcAddress_v2('cuMemGetAllocationPropertiesFromHandle', &__cuMemGetAllocationPropertiesFromHandle, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemRetainAllocationHandle + * _F_cuGetProcAddress_v2('cuMemRetainAllocationHandle', &__cuMemRetainAllocationHandle, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemGetAllocationPropertiesFromHandle"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationPropertiesFromHandle), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1135, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1137 + * _F_cuGetProcAddress_v2('cuMemGetAllocationPropertiesFromHandle', &__cuMemGetAllocationPropertiesFromHandle, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemRetainAllocationHandle + * _F_cuGetProcAddress_v2('cuMemRetainAllocationHandle', &__cuMemRetainAllocationHandle, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemPoolTrimTo + * _F_cuGetProcAddress_v2('cuMemPoolTrimTo', &__cuMemPoolTrimTo, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemRetainAllocationHandle"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRetainAllocationHandle), 0x2AF8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1137, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1139 + * _F_cuGetProcAddress_v2('cuMemRetainAllocationHandle', &__cuMemRetainAllocationHandle, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemPoolTrimTo + * _F_cuGetProcAddress_v2('cuMemPoolTrimTo', &__cuMemPoolTrimTo, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemPoolSetAttribute + * _F_cuGetProcAddress_v2('cuMemPoolSetAttribute', &__cuMemPoolSetAttribute, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPoolTrimTo"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolTrimTo), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1139, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1141 + * _F_cuGetProcAddress_v2('cuMemPoolTrimTo', &__cuMemPoolTrimTo, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemPoolSetAttribute + * _F_cuGetProcAddress_v2('cuMemPoolSetAttribute', &__cuMemPoolSetAttribute, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemPoolGetAttribute + * _F_cuGetProcAddress_v2('cuMemPoolGetAttribute', &__cuMemPoolGetAttribute, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPoolSetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAttribute), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1141, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1143 + * _F_cuGetProcAddress_v2('cuMemPoolSetAttribute', &__cuMemPoolSetAttribute, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemPoolGetAttribute + * _F_cuGetProcAddress_v2('cuMemPoolGetAttribute', &__cuMemPoolGetAttribute, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemPoolSetAccess + * _F_cuGetProcAddress_v2('cuMemPoolSetAccess', &__cuMemPoolSetAccess, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPoolGetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAttribute), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1143, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1145 + * _F_cuGetProcAddress_v2('cuMemPoolGetAttribute', &__cuMemPoolGetAttribute, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemPoolSetAccess + * _F_cuGetProcAddress_v2('cuMemPoolSetAccess', &__cuMemPoolSetAccess, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemPoolGetAccess + * _F_cuGetProcAddress_v2('cuMemPoolGetAccess', &__cuMemPoolGetAccess, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPoolSetAccess"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAccess), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1145, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1147 + * _F_cuGetProcAddress_v2('cuMemPoolSetAccess', &__cuMemPoolSetAccess, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemPoolGetAccess + * _F_cuGetProcAddress_v2('cuMemPoolGetAccess', &__cuMemPoolGetAccess, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemPoolCreate + * _F_cuGetProcAddress_v2('cuMemPoolCreate', &__cuMemPoolCreate, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPoolGetAccess"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAccess), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1147, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1149 + * _F_cuGetProcAddress_v2('cuMemPoolGetAccess', &__cuMemPoolGetAccess, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemPoolCreate + * _F_cuGetProcAddress_v2('cuMemPoolCreate', &__cuMemPoolCreate, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemPoolDestroy + * _F_cuGetProcAddress_v2('cuMemPoolDestroy', &__cuMemPoolDestroy, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPoolCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolCreate), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1149, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1151 + * _F_cuGetProcAddress_v2('cuMemPoolCreate', &__cuMemPoolCreate, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemPoolDestroy + * _F_cuGetProcAddress_v2('cuMemPoolDestroy', &__cuMemPoolDestroy, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemPoolExportToShareableHandle + * _F_cuGetProcAddress_v2('cuMemPoolExportToShareableHandle', &__cuMemPoolExportToShareableHandle, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPoolDestroy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolDestroy), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1151, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1153 + * _F_cuGetProcAddress_v2('cuMemPoolDestroy', &__cuMemPoolDestroy, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemPoolExportToShareableHandle + * _F_cuGetProcAddress_v2('cuMemPoolExportToShareableHandle', &__cuMemPoolExportToShareableHandle, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemPoolImportFromShareableHandle + * _F_cuGetProcAddress_v2('cuMemPoolImportFromShareableHandle', &__cuMemPoolImportFromShareableHandle, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPoolExportToShareableHandle"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportToShareableHandle), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1153, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1155 + * _F_cuGetProcAddress_v2('cuMemPoolExportToShareableHandle', &__cuMemPoolExportToShareableHandle, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemPoolImportFromShareableHandle + * _F_cuGetProcAddress_v2('cuMemPoolImportFromShareableHandle', &__cuMemPoolImportFromShareableHandle, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemPoolExportPointer + * _F_cuGetProcAddress_v2('cuMemPoolExportPointer', &__cuMemPoolExportPointer, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPoolImportFromShareableHandle"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportFromShareableHandle), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1155, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1157 + * _F_cuGetProcAddress_v2('cuMemPoolImportFromShareableHandle', &__cuMemPoolImportFromShareableHandle, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemPoolExportPointer + * _F_cuGetProcAddress_v2('cuMemPoolExportPointer', &__cuMemPoolExportPointer, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemPoolImportPointer + * _F_cuGetProcAddress_v2('cuMemPoolImportPointer', &__cuMemPoolImportPointer, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPoolExportPointer"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportPointer), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1157, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1159 + * _F_cuGetProcAddress_v2('cuMemPoolExportPointer', &__cuMemPoolExportPointer, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemPoolImportPointer + * _F_cuGetProcAddress_v2('cuMemPoolImportPointer', &__cuMemPoolImportPointer, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMulticastCreate + * _F_cuGetProcAddress_v2('cuMulticastCreate', &__cuMulticastCreate, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemPoolImportPointer"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportPointer), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1159, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1161 + * _F_cuGetProcAddress_v2('cuMemPoolImportPointer', &__cuMemPoolImportPointer, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMulticastCreate + * _F_cuGetProcAddress_v2('cuMulticastCreate', &__cuMulticastCreate, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMulticastAddDevice + * _F_cuGetProcAddress_v2('cuMulticastAddDevice', &__cuMulticastAddDevice, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMulticastCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastCreate), 0x2EEA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1161, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1163 + * _F_cuGetProcAddress_v2('cuMulticastCreate', &__cuMulticastCreate, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMulticastAddDevice + * _F_cuGetProcAddress_v2('cuMulticastAddDevice', &__cuMulticastAddDevice, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMulticastBindMem + * _F_cuGetProcAddress_v2('cuMulticastBindMem', &__cuMulticastBindMem, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMulticastAddDevice"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastAddDevice), 0x2EEA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1163, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1165 + * _F_cuGetProcAddress_v2('cuMulticastAddDevice', &__cuMulticastAddDevice, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMulticastBindMem + * _F_cuGetProcAddress_v2('cuMulticastBindMem', &__cuMulticastBindMem, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMulticastBindAddr + * _F_cuGetProcAddress_v2('cuMulticastBindAddr', &__cuMulticastBindAddr, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMulticastBindMem"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindMem), 0x2EEA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1165, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1167 + * _F_cuGetProcAddress_v2('cuMulticastBindMem', &__cuMulticastBindMem, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMulticastBindAddr + * _F_cuGetProcAddress_v2('cuMulticastBindAddr', &__cuMulticastBindAddr, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMulticastUnbind + * _F_cuGetProcAddress_v2('cuMulticastUnbind', &__cuMulticastUnbind, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMulticastBindAddr"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindAddr), 0x2EEA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1167, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1169 + * _F_cuGetProcAddress_v2('cuMulticastBindAddr', &__cuMulticastBindAddr, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMulticastUnbind + * _F_cuGetProcAddress_v2('cuMulticastUnbind', &__cuMulticastUnbind, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMulticastGetGranularity + * _F_cuGetProcAddress_v2('cuMulticastGetGranularity', &__cuMulticastGetGranularity, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMulticastUnbind"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastUnbind), 0x2EEA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1169, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1171 + * _F_cuGetProcAddress_v2('cuMulticastUnbind', &__cuMulticastUnbind, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMulticastGetGranularity + * _F_cuGetProcAddress_v2('cuMulticastGetGranularity', &__cuMulticastGetGranularity, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuPointerGetAttribute + * _F_cuGetProcAddress_v2('cuPointerGetAttribute', &__cuPointerGetAttribute, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMulticastGetGranularity"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastGetGranularity), 0x2EEA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1171, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1173 + * _F_cuGetProcAddress_v2('cuMulticastGetGranularity', &__cuMulticastGetGranularity, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuPointerGetAttribute + * _F_cuGetProcAddress_v2('cuPointerGetAttribute', &__cuPointerGetAttribute, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemAdvise + * _F_cuGetProcAddress_v2('cuMemAdvise', &__cuMemAdvise, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuPointerGetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttribute), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1173, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1175 + * _F_cuGetProcAddress_v2('cuPointerGetAttribute', &__cuPointerGetAttribute, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemAdvise + * _F_cuGetProcAddress_v2('cuMemAdvise', &__cuMemAdvise, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemAdvise_v2 + * _F_cuGetProcAddress_v2('cuMemAdvise', &__cuMemAdvise_v2, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemAdvise"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise), 0x1F40, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1175, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1177 + * _F_cuGetProcAddress_v2('cuMemAdvise', &__cuMemAdvise, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemAdvise_v2 + * _F_cuGetProcAddress_v2('cuMemAdvise', &__cuMemAdvise_v2, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemRangeGetAttribute + * _F_cuGetProcAddress_v2('cuMemRangeGetAttribute', &__cuMemRangeGetAttribute, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemAdvise"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise_v2), 0x2EF4, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1177, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1179 + * _F_cuGetProcAddress_v2('cuMemAdvise', &__cuMemAdvise_v2, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemRangeGetAttribute + * _F_cuGetProcAddress_v2('cuMemRangeGetAttribute', &__cuMemRangeGetAttribute, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuMemRangeGetAttributes + * _F_cuGetProcAddress_v2('cuMemRangeGetAttributes', &__cuMemRangeGetAttributes, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemRangeGetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttribute), 0x1F40, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1179, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1181 + * _F_cuGetProcAddress_v2('cuMemRangeGetAttribute', &__cuMemRangeGetAttribute, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuMemRangeGetAttributes + * _F_cuGetProcAddress_v2('cuMemRangeGetAttributes', &__cuMemRangeGetAttributes, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuPointerSetAttribute + * _F_cuGetProcAddress_v2('cuPointerSetAttribute', &__cuPointerSetAttribute, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuMemRangeGetAttributes"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttributes), 0x1F40, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1181, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1183 + * _F_cuGetProcAddress_v2('cuMemRangeGetAttributes', &__cuMemRangeGetAttributes, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuPointerSetAttribute + * _F_cuGetProcAddress_v2('cuPointerSetAttribute', &__cuPointerSetAttribute, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuPointerGetAttributes + * _F_cuGetProcAddress_v2('cuPointerGetAttributes', &__cuPointerGetAttributes, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuPointerSetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerSetAttribute), 0x1770, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1183, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1185 + * _F_cuGetProcAddress_v2('cuPointerSetAttribute', &__cuPointerSetAttribute, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuPointerGetAttributes + * _F_cuGetProcAddress_v2('cuPointerGetAttributes', &__cuPointerGetAttributes, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamCreate + * _F_cuGetProcAddress_v2('cuStreamCreate', &__cuStreamCreate, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuPointerGetAttributes"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttributes), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1185, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1187 + * _F_cuGetProcAddress_v2('cuPointerGetAttributes', &__cuPointerGetAttributes, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamCreate + * _F_cuGetProcAddress_v2('cuStreamCreate', &__cuStreamCreate, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamCreateWithPriority + * _F_cuGetProcAddress_v2('cuStreamCreateWithPriority', &__cuStreamCreateWithPriority, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreate), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1187, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1189 + * _F_cuGetProcAddress_v2('cuStreamCreate', &__cuStreamCreate, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamCreateWithPriority + * _F_cuGetProcAddress_v2('cuStreamCreateWithPriority', &__cuStreamCreateWithPriority, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuThreadExchangeStreamCaptureMode + * _F_cuGetProcAddress_v2('cuThreadExchangeStreamCaptureMode', &__cuThreadExchangeStreamCaptureMode, 10010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamCreateWithPriority"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreateWithPriority), 0x13BA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1189, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1191 + * _F_cuGetProcAddress_v2('cuStreamCreateWithPriority', &__cuStreamCreateWithPriority, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuThreadExchangeStreamCaptureMode + * _F_cuGetProcAddress_v2('cuThreadExchangeStreamCaptureMode', &__cuThreadExchangeStreamCaptureMode, 10010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamDestroy_v2 + * _F_cuGetProcAddress_v2('cuStreamDestroy', &__cuStreamDestroy_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuThreadExchangeStreamCaptureMode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuThreadExchangeStreamCaptureMode), 0x271A, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1191, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1193 + * _F_cuGetProcAddress_v2('cuThreadExchangeStreamCaptureMode', &__cuThreadExchangeStreamCaptureMode, 10010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamDestroy_v2 + * _F_cuGetProcAddress_v2('cuStreamDestroy', &__cuStreamDestroy_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEventCreate + * _F_cuGetProcAddress_v2('cuEventCreate', &__cuEventCreate, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamDestroy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamDestroy_v2), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1193, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1195 + * _F_cuGetProcAddress_v2('cuStreamDestroy', &__cuStreamDestroy_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEventCreate + * _F_cuGetProcAddress_v2('cuEventCreate', &__cuEventCreate, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEventQuery + * _F_cuGetProcAddress_v2('cuEventQuery', &__cuEventQuery, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEventCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreate), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1195, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1197 + * _F_cuGetProcAddress_v2('cuEventCreate', &__cuEventCreate, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEventQuery + * _F_cuGetProcAddress_v2('cuEventQuery', &__cuEventQuery, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEventSynchronize + * _F_cuGetProcAddress_v2('cuEventSynchronize', &__cuEventSynchronize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEventQuery"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventQuery), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1197, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1199 + * _F_cuGetProcAddress_v2('cuEventQuery', &__cuEventQuery, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEventSynchronize + * _F_cuGetProcAddress_v2('cuEventSynchronize', &__cuEventSynchronize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEventDestroy_v2 + * _F_cuGetProcAddress_v2('cuEventDestroy', &__cuEventDestroy_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEventSynchronize"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventSynchronize), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1199, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1201 + * _F_cuGetProcAddress_v2('cuEventSynchronize', &__cuEventSynchronize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEventDestroy_v2 + * _F_cuGetProcAddress_v2('cuEventDestroy', &__cuEventDestroy_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEventElapsedTime + * _F_cuGetProcAddress_v2('cuEventElapsedTime', &__cuEventElapsedTime, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEventDestroy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventDestroy_v2), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1201, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1203 + * _F_cuGetProcAddress_v2('cuEventDestroy', &__cuEventDestroy_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEventElapsedTime + * _F_cuGetProcAddress_v2('cuEventElapsedTime', &__cuEventElapsedTime, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEventElapsedTime_v2 + * _F_cuGetProcAddress_v2('cuEventElapsedTime', &__cuEventElapsedTime_v2, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEventElapsedTime"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1203, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1205 + * _F_cuGetProcAddress_v2('cuEventElapsedTime', &__cuEventElapsedTime, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEventElapsedTime_v2 + * _F_cuGetProcAddress_v2('cuEventElapsedTime', &__cuEventElapsedTime_v2, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuImportExternalMemory + * _F_cuGetProcAddress_v2('cuImportExternalMemory', &__cuImportExternalMemory, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEventElapsedTime"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime_v2), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1205, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1207 + * _F_cuGetProcAddress_v2('cuEventElapsedTime', &__cuEventElapsedTime_v2, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuImportExternalMemory + * _F_cuGetProcAddress_v2('cuImportExternalMemory', &__cuImportExternalMemory, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuExternalMemoryGetMappedBuffer + * _F_cuGetProcAddress_v2('cuExternalMemoryGetMappedBuffer', &__cuExternalMemoryGetMappedBuffer, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuImportExternalMemory"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalMemory), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1207, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1209 + * _F_cuGetProcAddress_v2('cuImportExternalMemory', &__cuImportExternalMemory, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuExternalMemoryGetMappedBuffer + * _F_cuGetProcAddress_v2('cuExternalMemoryGetMappedBuffer', &__cuExternalMemoryGetMappedBuffer, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuExternalMemoryGetMappedMipmappedArray + * _F_cuGetProcAddress_v2('cuExternalMemoryGetMappedMipmappedArray', &__cuExternalMemoryGetMappedMipmappedArray, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuExternalMemoryGetMappedBuffer"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedBuffer), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1209, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1211 + * _F_cuGetProcAddress_v2('cuExternalMemoryGetMappedBuffer', &__cuExternalMemoryGetMappedBuffer, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuExternalMemoryGetMappedMipmappedArray + * _F_cuGetProcAddress_v2('cuExternalMemoryGetMappedMipmappedArray', &__cuExternalMemoryGetMappedMipmappedArray, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDestroyExternalMemory + * _F_cuGetProcAddress_v2('cuDestroyExternalMemory', &__cuDestroyExternalMemory, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuExternalMemoryGetMappedMipmappedArray"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedMipmappedArray), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1211, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1213 + * _F_cuGetProcAddress_v2('cuExternalMemoryGetMappedMipmappedArray', &__cuExternalMemoryGetMappedMipmappedArray, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDestroyExternalMemory + * _F_cuGetProcAddress_v2('cuDestroyExternalMemory', &__cuDestroyExternalMemory, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuImportExternalSemaphore + * _F_cuGetProcAddress_v2('cuImportExternalSemaphore', &__cuImportExternalSemaphore, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDestroyExternalMemory"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalMemory), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1213, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1215 + * _F_cuGetProcAddress_v2('cuDestroyExternalMemory', &__cuDestroyExternalMemory, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuImportExternalSemaphore + * _F_cuGetProcAddress_v2('cuImportExternalSemaphore', &__cuImportExternalSemaphore, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDestroyExternalSemaphore + * _F_cuGetProcAddress_v2('cuDestroyExternalSemaphore', &__cuDestroyExternalSemaphore, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuImportExternalSemaphore"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalSemaphore), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1215, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1217 + * _F_cuGetProcAddress_v2('cuImportExternalSemaphore', &__cuImportExternalSemaphore, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDestroyExternalSemaphore + * _F_cuGetProcAddress_v2('cuDestroyExternalSemaphore', &__cuDestroyExternalSemaphore, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuFuncGetAttribute + * _F_cuGetProcAddress_v2('cuFuncGetAttribute', &__cuFuncGetAttribute, 2020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDestroyExternalSemaphore"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalSemaphore), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1217, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1219 + * _F_cuGetProcAddress_v2('cuDestroyExternalSemaphore', &__cuDestroyExternalSemaphore, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuFuncGetAttribute + * _F_cuGetProcAddress_v2('cuFuncGetAttribute', &__cuFuncGetAttribute, 2020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuFuncSetAttribute + * _F_cuGetProcAddress_v2('cuFuncSetAttribute', &__cuFuncSetAttribute, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuFuncGetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetAttribute), 0x7E4, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1219, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1221 + * _F_cuGetProcAddress_v2('cuFuncGetAttribute', &__cuFuncGetAttribute, 2020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuFuncSetAttribute + * _F_cuGetProcAddress_v2('cuFuncSetAttribute', &__cuFuncSetAttribute, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuFuncSetCacheConfig + * _F_cuGetProcAddress_v2('cuFuncSetCacheConfig', &__cuFuncSetCacheConfig, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuFuncSetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetAttribute), 0x2328, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1221, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1223 + * _F_cuGetProcAddress_v2('cuFuncSetAttribute', &__cuFuncSetAttribute, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuFuncSetCacheConfig + * _F_cuGetProcAddress_v2('cuFuncSetCacheConfig', &__cuFuncSetCacheConfig, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuFuncGetModule + * _F_cuGetProcAddress_v2('cuFuncGetModule', &__cuFuncGetModule, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuFuncSetCacheConfig"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetCacheConfig), 0xBB8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1223, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1225 + * _F_cuGetProcAddress_v2('cuFuncSetCacheConfig', &__cuFuncSetCacheConfig, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuFuncGetModule + * _F_cuGetProcAddress_v2('cuFuncGetModule', &__cuFuncGetModule, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuFuncGetName + * _F_cuGetProcAddress_v2('cuFuncGetName', &__cuFuncGetName, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuFuncGetModule"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetModule), 0x2AF8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1225, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1227 + * _F_cuGetProcAddress_v2('cuFuncGetModule', &__cuFuncGetModule, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuFuncGetName + * _F_cuGetProcAddress_v2('cuFuncGetName', &__cuFuncGetName, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuFuncGetParamInfo + * _F_cuGetProcAddress_v2('cuFuncGetParamInfo', &__cuFuncGetParamInfo, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuFuncGetName"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetName), 0x2EFE, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1227, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1229 + * _F_cuGetProcAddress_v2('cuFuncGetName', &__cuFuncGetName, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuFuncGetParamInfo + * _F_cuGetProcAddress_v2('cuFuncGetParamInfo', &__cuFuncGetParamInfo, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuFuncIsLoaded + * _F_cuGetProcAddress_v2('cuFuncIsLoaded', &__cuFuncIsLoaded, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuFuncGetParamInfo"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetParamInfo), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1229, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1231 + * _F_cuGetProcAddress_v2('cuFuncGetParamInfo', &__cuFuncGetParamInfo, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuFuncIsLoaded + * _F_cuGetProcAddress_v2('cuFuncIsLoaded', &__cuFuncIsLoaded, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuFuncLoad + * _F_cuGetProcAddress_v2('cuFuncLoad', &__cuFuncLoad, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuFuncIsLoaded"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncIsLoaded), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1231, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1233 + * _F_cuGetProcAddress_v2('cuFuncIsLoaded', &__cuFuncIsLoaded, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuFuncLoad + * _F_cuGetProcAddress_v2('cuFuncLoad', &__cuFuncLoad, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLaunchCooperativeKernelMultiDevice + * _F_cuGetProcAddress_v2('cuLaunchCooperativeKernelMultiDevice', &__cuLaunchCooperativeKernelMultiDevice, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuFuncLoad"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncLoad), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1233, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1235 + * _F_cuGetProcAddress_v2('cuFuncLoad', &__cuFuncLoad, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLaunchCooperativeKernelMultiDevice + * _F_cuGetProcAddress_v2('cuLaunchCooperativeKernelMultiDevice', &__cuLaunchCooperativeKernelMultiDevice, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuFuncSetBlockShape + * _F_cuGetProcAddress_v2('cuFuncSetBlockShape', &__cuFuncSetBlockShape, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLaunchCooperativeKernelMultiDevice"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernelMultiDevice), 0x2328, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1235, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1237 + * _F_cuGetProcAddress_v2('cuLaunchCooperativeKernelMultiDevice', &__cuLaunchCooperativeKernelMultiDevice, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuFuncSetBlockShape + * _F_cuGetProcAddress_v2('cuFuncSetBlockShape', &__cuFuncSetBlockShape, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuFuncSetSharedSize + * _F_cuGetProcAddress_v2('cuFuncSetSharedSize', &__cuFuncSetSharedSize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuFuncSetBlockShape"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetBlockShape), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1237, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1239 + * _F_cuGetProcAddress_v2('cuFuncSetBlockShape', &__cuFuncSetBlockShape, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuFuncSetSharedSize + * _F_cuGetProcAddress_v2('cuFuncSetSharedSize', &__cuFuncSetSharedSize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuParamSetSize + * _F_cuGetProcAddress_v2('cuParamSetSize', &__cuParamSetSize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuFuncSetSharedSize"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedSize), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1239, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1241 + * _F_cuGetProcAddress_v2('cuFuncSetSharedSize', &__cuFuncSetSharedSize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuParamSetSize + * _F_cuGetProcAddress_v2('cuParamSetSize', &__cuParamSetSize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuParamSeti + * _F_cuGetProcAddress_v2('cuParamSeti', &__cuParamSeti, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuParamSetSize"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetSize), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1241, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1243 + * _F_cuGetProcAddress_v2('cuParamSetSize', &__cuParamSetSize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuParamSeti + * _F_cuGetProcAddress_v2('cuParamSeti', &__cuParamSeti, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuParamSetf + * _F_cuGetProcAddress_v2('cuParamSetf', &__cuParamSetf, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuParamSeti"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSeti), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1243, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1245 + * _F_cuGetProcAddress_v2('cuParamSeti', &__cuParamSeti, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuParamSetf + * _F_cuGetProcAddress_v2('cuParamSetf', &__cuParamSetf, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuParamSetv + * _F_cuGetProcAddress_v2('cuParamSetv', &__cuParamSetv, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuParamSetf"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetf), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1245, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1247 + * _F_cuGetProcAddress_v2('cuParamSetf', &__cuParamSetf, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuParamSetv + * _F_cuGetProcAddress_v2('cuParamSetv', &__cuParamSetv, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLaunch + * _F_cuGetProcAddress_v2('cuLaunch', &__cuLaunch, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuParamSetv"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetv), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1247, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1249 + * _F_cuGetProcAddress_v2('cuParamSetv', &__cuParamSetv, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLaunch + * _F_cuGetProcAddress_v2('cuLaunch', &__cuLaunch, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLaunchGrid + * _F_cuGetProcAddress_v2('cuLaunchGrid', &__cuLaunchGrid, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLaunch"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunch), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1249, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1251 + * _F_cuGetProcAddress_v2('cuLaunch', &__cuLaunch, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLaunchGrid + * _F_cuGetProcAddress_v2('cuLaunchGrid', &__cuLaunchGrid, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLaunchGridAsync + * _F_cuGetProcAddress_v2('cuLaunchGridAsync', &__cuLaunchGridAsync, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLaunchGrid"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGrid), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1251, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1253 + * _F_cuGetProcAddress_v2('cuLaunchGrid', &__cuLaunchGrid, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLaunchGridAsync + * _F_cuGetProcAddress_v2('cuLaunchGridAsync', &__cuLaunchGridAsync, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuParamSetTexRef + * _F_cuGetProcAddress_v2('cuParamSetTexRef', &__cuParamSetTexRef, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLaunchGridAsync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGridAsync), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1253, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1255 + * _F_cuGetProcAddress_v2('cuLaunchGridAsync', &__cuLaunchGridAsync, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuParamSetTexRef + * _F_cuGetProcAddress_v2('cuParamSetTexRef', &__cuParamSetTexRef, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuFuncSetSharedMemConfig + * _F_cuGetProcAddress_v2('cuFuncSetSharedMemConfig', &__cuFuncSetSharedMemConfig, 4020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuParamSetTexRef"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetTexRef), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1255, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1257 + * _F_cuGetProcAddress_v2('cuParamSetTexRef', &__cuParamSetTexRef, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuFuncSetSharedMemConfig + * _F_cuGetProcAddress_v2('cuFuncSetSharedMemConfig', &__cuFuncSetSharedMemConfig, 4020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphCreate + * _F_cuGetProcAddress_v2('cuGraphCreate', &__cuGraphCreate, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuFuncSetSharedMemConfig"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedMemConfig), 0xFB4, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1257, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1259 + * _F_cuGetProcAddress_v2('cuFuncSetSharedMemConfig', &__cuFuncSetSharedMemConfig, 4020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphCreate + * _F_cuGetProcAddress_v2('cuGraphCreate', &__cuGraphCreate, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddKernelNode_v2 + * _F_cuGetProcAddress_v2('cuGraphAddKernelNode', &__cuGraphAddKernelNode_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphCreate), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1259, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1261 + * _F_cuGetProcAddress_v2('cuGraphCreate', &__cuGraphCreate, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddKernelNode_v2 + * _F_cuGetProcAddress_v2('cuGraphAddKernelNode', &__cuGraphAddKernelNode_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeGetParams_v2 + * _F_cuGetProcAddress_v2('cuGraphKernelNodeGetParams', &__cuGraphKernelNodeGetParams_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddKernelNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddKernelNode_v2), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1261, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1263 + * _F_cuGetProcAddress_v2('cuGraphAddKernelNode', &__cuGraphAddKernelNode_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphKernelNodeGetParams_v2 + * _F_cuGetProcAddress_v2('cuGraphKernelNodeGetParams', &__cuGraphKernelNodeGetParams_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeSetParams_v2 + * _F_cuGetProcAddress_v2('cuGraphKernelNodeSetParams', &__cuGraphKernelNodeSetParams_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphKernelNodeGetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetParams_v2), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1263, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1265 + * _F_cuGetProcAddress_v2('cuGraphKernelNodeGetParams', &__cuGraphKernelNodeGetParams_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphKernelNodeSetParams_v2 + * _F_cuGetProcAddress_v2('cuGraphKernelNodeSetParams', &__cuGraphKernelNodeSetParams_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddMemcpyNode + * _F_cuGetProcAddress_v2('cuGraphAddMemcpyNode', &__cuGraphAddMemcpyNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphKernelNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetParams_v2), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1265, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1267 + * _F_cuGetProcAddress_v2('cuGraphKernelNodeSetParams', &__cuGraphKernelNodeSetParams_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddMemcpyNode + * _F_cuGetProcAddress_v2('cuGraphAddMemcpyNode', &__cuGraphAddMemcpyNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphMemcpyNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphMemcpyNodeGetParams', &__cuGraphMemcpyNodeGetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddMemcpyNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemcpyNode), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1267, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1269 + * _F_cuGetProcAddress_v2('cuGraphAddMemcpyNode', &__cuGraphAddMemcpyNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphMemcpyNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphMemcpyNodeGetParams', &__cuGraphMemcpyNodeGetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphMemcpyNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphMemcpyNodeSetParams', &__cuGraphMemcpyNodeSetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphMemcpyNodeGetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeGetParams), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1269, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1271 + * _F_cuGetProcAddress_v2('cuGraphMemcpyNodeGetParams', &__cuGraphMemcpyNodeGetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphMemcpyNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphMemcpyNodeSetParams', &__cuGraphMemcpyNodeSetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddMemsetNode + * _F_cuGetProcAddress_v2('cuGraphAddMemsetNode', &__cuGraphAddMemsetNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphMemcpyNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeSetParams), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1271, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1273 + * _F_cuGetProcAddress_v2('cuGraphMemcpyNodeSetParams', &__cuGraphMemcpyNodeSetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddMemsetNode + * _F_cuGetProcAddress_v2('cuGraphAddMemsetNode', &__cuGraphAddMemsetNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphMemsetNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphMemsetNodeGetParams', &__cuGraphMemsetNodeGetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddMemsetNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemsetNode), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1273, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1275 + * _F_cuGetProcAddress_v2('cuGraphAddMemsetNode', &__cuGraphAddMemsetNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphMemsetNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphMemsetNodeGetParams', &__cuGraphMemsetNodeGetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphMemsetNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphMemsetNodeSetParams', &__cuGraphMemsetNodeSetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphMemsetNodeGetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeGetParams), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1275, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1277 + * _F_cuGetProcAddress_v2('cuGraphMemsetNodeGetParams', &__cuGraphMemsetNodeGetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphMemsetNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphMemsetNodeSetParams', &__cuGraphMemsetNodeSetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddHostNode + * _F_cuGetProcAddress_v2('cuGraphAddHostNode', &__cuGraphAddHostNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphMemsetNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeSetParams), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1277, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1279 + * _F_cuGetProcAddress_v2('cuGraphMemsetNodeSetParams', &__cuGraphMemsetNodeSetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddHostNode + * _F_cuGetProcAddress_v2('cuGraphAddHostNode', &__cuGraphAddHostNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphHostNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphHostNodeGetParams', &__cuGraphHostNodeGetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddHostNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddHostNode), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1279, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1281 + * _F_cuGetProcAddress_v2('cuGraphAddHostNode', &__cuGraphAddHostNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphHostNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphHostNodeGetParams', &__cuGraphHostNodeGetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphHostNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphHostNodeSetParams', &__cuGraphHostNodeSetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphHostNodeGetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeGetParams), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1281, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1283 + * _F_cuGetProcAddress_v2('cuGraphHostNodeGetParams', &__cuGraphHostNodeGetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphHostNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphHostNodeSetParams', &__cuGraphHostNodeSetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddChildGraphNode + * _F_cuGetProcAddress_v2('cuGraphAddChildGraphNode', &__cuGraphAddChildGraphNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphHostNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeSetParams), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1283, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1285 + * _F_cuGetProcAddress_v2('cuGraphHostNodeSetParams', &__cuGraphHostNodeSetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddChildGraphNode + * _F_cuGetProcAddress_v2('cuGraphAddChildGraphNode', &__cuGraphAddChildGraphNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphChildGraphNodeGetGraph + * _F_cuGetProcAddress_v2('cuGraphChildGraphNodeGetGraph', &__cuGraphChildGraphNodeGetGraph, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddChildGraphNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddChildGraphNode), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1285, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1287 + * _F_cuGetProcAddress_v2('cuGraphAddChildGraphNode', &__cuGraphAddChildGraphNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphChildGraphNodeGetGraph + * _F_cuGetProcAddress_v2('cuGraphChildGraphNodeGetGraph', &__cuGraphChildGraphNodeGetGraph, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddEmptyNode + * _F_cuGetProcAddress_v2('cuGraphAddEmptyNode', &__cuGraphAddEmptyNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphChildGraphNodeGetGraph"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphChildGraphNodeGetGraph), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1287, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1289 + * _F_cuGetProcAddress_v2('cuGraphChildGraphNodeGetGraph', &__cuGraphChildGraphNodeGetGraph, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddEmptyNode + * _F_cuGetProcAddress_v2('cuGraphAddEmptyNode', &__cuGraphAddEmptyNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddEventRecordNode + * _F_cuGetProcAddress_v2('cuGraphAddEventRecordNode', &__cuGraphAddEventRecordNode, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddEmptyNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEmptyNode), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1289, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1291 + * _F_cuGetProcAddress_v2('cuGraphAddEmptyNode', &__cuGraphAddEmptyNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddEventRecordNode + * _F_cuGetProcAddress_v2('cuGraphAddEventRecordNode', &__cuGraphAddEventRecordNode, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphEventRecordNodeGetEvent + * _F_cuGetProcAddress_v2('cuGraphEventRecordNodeGetEvent', &__cuGraphEventRecordNodeGetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddEventRecordNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventRecordNode), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1291, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1293 + * _F_cuGetProcAddress_v2('cuGraphAddEventRecordNode', &__cuGraphAddEventRecordNode, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphEventRecordNodeGetEvent + * _F_cuGetProcAddress_v2('cuGraphEventRecordNodeGetEvent', &__cuGraphEventRecordNodeGetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphEventRecordNodeSetEvent + * _F_cuGetProcAddress_v2('cuGraphEventRecordNodeSetEvent', &__cuGraphEventRecordNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphEventRecordNodeGetEvent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeGetEvent), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1293, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1295 + * _F_cuGetProcAddress_v2('cuGraphEventRecordNodeGetEvent', &__cuGraphEventRecordNodeGetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphEventRecordNodeSetEvent + * _F_cuGetProcAddress_v2('cuGraphEventRecordNodeSetEvent', &__cuGraphEventRecordNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddEventWaitNode + * _F_cuGetProcAddress_v2('cuGraphAddEventWaitNode', &__cuGraphAddEventWaitNode, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphEventRecordNodeSetEvent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeSetEvent), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1295, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1297 + * _F_cuGetProcAddress_v2('cuGraphEventRecordNodeSetEvent', &__cuGraphEventRecordNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddEventWaitNode + * _F_cuGetProcAddress_v2('cuGraphAddEventWaitNode', &__cuGraphAddEventWaitNode, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphEventWaitNodeGetEvent + * _F_cuGetProcAddress_v2('cuGraphEventWaitNodeGetEvent', &__cuGraphEventWaitNodeGetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddEventWaitNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventWaitNode), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1297, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1299 + * _F_cuGetProcAddress_v2('cuGraphAddEventWaitNode', &__cuGraphAddEventWaitNode, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphEventWaitNodeGetEvent + * _F_cuGetProcAddress_v2('cuGraphEventWaitNodeGetEvent', &__cuGraphEventWaitNodeGetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphEventWaitNodeSetEvent + * _F_cuGetProcAddress_v2('cuGraphEventWaitNodeSetEvent', &__cuGraphEventWaitNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphEventWaitNodeGetEvent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeGetEvent), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1299, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1301 + * _F_cuGetProcAddress_v2('cuGraphEventWaitNodeGetEvent', &__cuGraphEventWaitNodeGetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphEventWaitNodeSetEvent + * _F_cuGetProcAddress_v2('cuGraphEventWaitNodeSetEvent', &__cuGraphEventWaitNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddExternalSemaphoresSignalNode + * _F_cuGetProcAddress_v2('cuGraphAddExternalSemaphoresSignalNode', &__cuGraphAddExternalSemaphoresSignalNode, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphEventWaitNodeSetEvent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeSetEvent), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1301, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1303 + * _F_cuGetProcAddress_v2('cuGraphEventWaitNodeSetEvent', &__cuGraphEventWaitNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddExternalSemaphoresSignalNode + * _F_cuGetProcAddress_v2('cuGraphAddExternalSemaphoresSignalNode', &__cuGraphAddExternalSemaphoresSignalNode, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresSignalNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresSignalNodeGetParams', &__cuGraphExternalSemaphoresSignalNodeGetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddExternalSemaphoresSignalNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresSignalNode), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1303, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1305 + * _F_cuGetProcAddress_v2('cuGraphAddExternalSemaphoresSignalNode', &__cuGraphAddExternalSemaphoresSignalNode, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExternalSemaphoresSignalNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresSignalNodeGetParams', &__cuGraphExternalSemaphoresSignalNodeGetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresSignalNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresSignalNodeSetParams', &__cuGraphExternalSemaphoresSignalNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExternalSemaphoresSignalNodeGetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeGetParams), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1305, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1307 + * _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresSignalNodeGetParams', &__cuGraphExternalSemaphoresSignalNodeGetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExternalSemaphoresSignalNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresSignalNodeSetParams', &__cuGraphExternalSemaphoresSignalNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddExternalSemaphoresWaitNode + * _F_cuGetProcAddress_v2('cuGraphAddExternalSemaphoresWaitNode', &__cuGraphAddExternalSemaphoresWaitNode, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExternalSemaphoresSignalNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeSetParams), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1307, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1309 + * _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresSignalNodeSetParams', &__cuGraphExternalSemaphoresSignalNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddExternalSemaphoresWaitNode + * _F_cuGetProcAddress_v2('cuGraphAddExternalSemaphoresWaitNode', &__cuGraphAddExternalSemaphoresWaitNode, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresWaitNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresWaitNodeGetParams', &__cuGraphExternalSemaphoresWaitNodeGetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddExternalSemaphoresWaitNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresWaitNode), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1309, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1311 + * _F_cuGetProcAddress_v2('cuGraphAddExternalSemaphoresWaitNode', &__cuGraphAddExternalSemaphoresWaitNode, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExternalSemaphoresWaitNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresWaitNodeGetParams', &__cuGraphExternalSemaphoresWaitNodeGetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresWaitNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresWaitNodeSetParams', &__cuGraphExternalSemaphoresWaitNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExternalSemaphoresWaitNodeGetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeGetParams), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1311, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1313 + * _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresWaitNodeGetParams', &__cuGraphExternalSemaphoresWaitNodeGetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExternalSemaphoresWaitNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresWaitNodeSetParams', &__cuGraphExternalSemaphoresWaitNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddBatchMemOpNode + * _F_cuGetProcAddress_v2('cuGraphAddBatchMemOpNode', &__cuGraphAddBatchMemOpNode, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExternalSemaphoresWaitNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeSetParams), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1313, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1315 + * _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresWaitNodeSetParams', &__cuGraphExternalSemaphoresWaitNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddBatchMemOpNode + * _F_cuGetProcAddress_v2('cuGraphAddBatchMemOpNode', &__cuGraphAddBatchMemOpNode, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphBatchMemOpNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphBatchMemOpNodeGetParams', &__cuGraphBatchMemOpNodeGetParams, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddBatchMemOpNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddBatchMemOpNode), 0x2B3E, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1315, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1317 + * _F_cuGetProcAddress_v2('cuGraphAddBatchMemOpNode', &__cuGraphAddBatchMemOpNode, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphBatchMemOpNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphBatchMemOpNodeGetParams', &__cuGraphBatchMemOpNodeGetParams, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphBatchMemOpNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphBatchMemOpNodeSetParams', &__cuGraphBatchMemOpNodeSetParams, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphBatchMemOpNodeGetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeGetParams), 0x2B3E, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1317, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1319 + * _F_cuGetProcAddress_v2('cuGraphBatchMemOpNodeGetParams', &__cuGraphBatchMemOpNodeGetParams, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphBatchMemOpNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphBatchMemOpNodeSetParams', &__cuGraphBatchMemOpNodeSetParams, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecBatchMemOpNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecBatchMemOpNodeSetParams', &__cuGraphExecBatchMemOpNodeSetParams, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphBatchMemOpNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeSetParams), 0x2B3E, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1319, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1321 + * _F_cuGetProcAddress_v2('cuGraphBatchMemOpNodeSetParams', &__cuGraphBatchMemOpNodeSetParams, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecBatchMemOpNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecBatchMemOpNodeSetParams', &__cuGraphExecBatchMemOpNodeSetParams, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddMemAllocNode + * _F_cuGetProcAddress_v2('cuGraphAddMemAllocNode', &__cuGraphAddMemAllocNode, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecBatchMemOpNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecBatchMemOpNodeSetParams), 0x2B3E, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1321, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1323 + * _F_cuGetProcAddress_v2('cuGraphExecBatchMemOpNodeSetParams', &__cuGraphExecBatchMemOpNodeSetParams, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddMemAllocNode + * _F_cuGetProcAddress_v2('cuGraphAddMemAllocNode', &__cuGraphAddMemAllocNode, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphMemAllocNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphMemAllocNodeGetParams', &__cuGraphMemAllocNodeGetParams, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddMemAllocNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemAllocNode), 0x2B20, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1323, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1325 + * _F_cuGetProcAddress_v2('cuGraphAddMemAllocNode', &__cuGraphAddMemAllocNode, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphMemAllocNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphMemAllocNodeGetParams', &__cuGraphMemAllocNodeGetParams, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddMemFreeNode + * _F_cuGetProcAddress_v2('cuGraphAddMemFreeNode', &__cuGraphAddMemFreeNode, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphMemAllocNodeGetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemAllocNodeGetParams), 0x2B20, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1325, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1327 + * _F_cuGetProcAddress_v2('cuGraphMemAllocNodeGetParams', &__cuGraphMemAllocNodeGetParams, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddMemFreeNode + * _F_cuGetProcAddress_v2('cuGraphAddMemFreeNode', &__cuGraphAddMemFreeNode, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphMemFreeNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphMemFreeNodeGetParams', &__cuGraphMemFreeNodeGetParams, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddMemFreeNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemFreeNode), 0x2B20, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1327, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1329 + * _F_cuGetProcAddress_v2('cuGraphAddMemFreeNode', &__cuGraphAddMemFreeNode, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphMemFreeNodeGetParams + * _F_cuGetProcAddress_v2('cuGraphMemFreeNodeGetParams', &__cuGraphMemFreeNodeGetParams, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGraphMemTrim + * _F_cuGetProcAddress_v2('cuDeviceGraphMemTrim', &__cuDeviceGraphMemTrim, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphMemFreeNodeGetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemFreeNodeGetParams), 0x2B20, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1329, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1331 + * _F_cuGetProcAddress_v2('cuGraphMemFreeNodeGetParams', &__cuGraphMemFreeNodeGetParams, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGraphMemTrim + * _F_cuGetProcAddress_v2('cuDeviceGraphMemTrim', &__cuDeviceGraphMemTrim, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetGraphMemAttribute + * _F_cuGetProcAddress_v2('cuDeviceGetGraphMemAttribute', &__cuDeviceGetGraphMemAttribute, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGraphMemTrim"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGraphMemTrim), 0x2B20, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1331, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1333 + * _F_cuGetProcAddress_v2('cuDeviceGraphMemTrim', &__cuDeviceGraphMemTrim, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetGraphMemAttribute + * _F_cuGetProcAddress_v2('cuDeviceGetGraphMemAttribute', &__cuDeviceGetGraphMemAttribute, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceSetGraphMemAttribute + * _F_cuGetProcAddress_v2('cuDeviceSetGraphMemAttribute', &__cuDeviceSetGraphMemAttribute, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetGraphMemAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetGraphMemAttribute), 0x2B20, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1333, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1335 + * _F_cuGetProcAddress_v2('cuDeviceGetGraphMemAttribute', &__cuDeviceGetGraphMemAttribute, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceSetGraphMemAttribute + * _F_cuGetProcAddress_v2('cuDeviceSetGraphMemAttribute', &__cuDeviceSetGraphMemAttribute, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphClone + * _F_cuGetProcAddress_v2('cuGraphClone', &__cuGraphClone, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceSetGraphMemAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetGraphMemAttribute), 0x2B20, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1335, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1337 + * _F_cuGetProcAddress_v2('cuDeviceSetGraphMemAttribute', &__cuDeviceSetGraphMemAttribute, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphClone + * _F_cuGetProcAddress_v2('cuGraphClone', &__cuGraphClone, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphNodeFindInClone + * _F_cuGetProcAddress_v2('cuGraphNodeFindInClone', &__cuGraphNodeFindInClone, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphClone"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphClone), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1337, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1339 + * _F_cuGetProcAddress_v2('cuGraphClone', &__cuGraphClone, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphNodeFindInClone + * _F_cuGetProcAddress_v2('cuGraphNodeFindInClone', &__cuGraphNodeFindInClone, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphNodeGetType + * _F_cuGetProcAddress_v2('cuGraphNodeGetType', &__cuGraphNodeGetType, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphNodeFindInClone"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeFindInClone), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1339, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1341 + * _F_cuGetProcAddress_v2('cuGraphNodeFindInClone', &__cuGraphNodeFindInClone, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphNodeGetType + * _F_cuGetProcAddress_v2('cuGraphNodeGetType', &__cuGraphNodeGetType, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphGetNodes + * _F_cuGetProcAddress_v2('cuGraphGetNodes', &__cuGraphGetNodes, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphNodeGetType"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetType), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1341, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1343 + * _F_cuGetProcAddress_v2('cuGraphNodeGetType', &__cuGraphNodeGetType, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphGetNodes + * _F_cuGetProcAddress_v2('cuGraphGetNodes', &__cuGraphGetNodes, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphGetRootNodes + * _F_cuGetProcAddress_v2('cuGraphGetRootNodes', &__cuGraphGetRootNodes, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphGetNodes"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetNodes), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1343, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1345 + * _F_cuGetProcAddress_v2('cuGraphGetNodes', &__cuGraphGetNodes, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphGetRootNodes + * _F_cuGetProcAddress_v2('cuGraphGetRootNodes', &__cuGraphGetRootNodes, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphGetEdges + * _F_cuGetProcAddress_v2('cuGraphGetEdges', &__cuGraphGetEdges, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphGetRootNodes"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetRootNodes), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1345, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1347 + * _F_cuGetProcAddress_v2('cuGraphGetRootNodes', &__cuGraphGetRootNodes, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphGetEdges + * _F_cuGetProcAddress_v2('cuGraphGetEdges', &__cuGraphGetEdges, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphGetEdges_v2 + * _F_cuGetProcAddress_v2('cuGraphGetEdges', &__cuGraphGetEdges_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphGetEdges"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1347, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1349 + * _F_cuGetProcAddress_v2('cuGraphGetEdges', &__cuGraphGetEdges, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphGetEdges_v2 + * _F_cuGetProcAddress_v2('cuGraphGetEdges', &__cuGraphGetEdges_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependencies + * _F_cuGetProcAddress_v2('cuGraphNodeGetDependencies', &__cuGraphNodeGetDependencies, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphGetEdges"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges_v2), 0x2EFE, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1349, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1351 + * _F_cuGetProcAddress_v2('cuGraphGetEdges', &__cuGraphGetEdges_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphNodeGetDependencies + * _F_cuGetProcAddress_v2('cuGraphNodeGetDependencies', &__cuGraphNodeGetDependencies, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependencies_v2 + * _F_cuGetProcAddress_v2('cuGraphNodeGetDependencies', &__cuGraphNodeGetDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphNodeGetDependencies"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1351, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1353 + * _F_cuGetProcAddress_v2('cuGraphNodeGetDependencies', &__cuGraphNodeGetDependencies, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphNodeGetDependencies_v2 + * _F_cuGetProcAddress_v2('cuGraphNodeGetDependencies', &__cuGraphNodeGetDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependentNodes + * _F_cuGetProcAddress_v2('cuGraphNodeGetDependentNodes', &__cuGraphNodeGetDependentNodes, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphNodeGetDependencies"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies_v2), 0x2EFE, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1353, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1355 + * _F_cuGetProcAddress_v2('cuGraphNodeGetDependencies', &__cuGraphNodeGetDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphNodeGetDependentNodes + * _F_cuGetProcAddress_v2('cuGraphNodeGetDependentNodes', &__cuGraphNodeGetDependentNodes, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependentNodes_v2 + * _F_cuGetProcAddress_v2('cuGraphNodeGetDependentNodes', &__cuGraphNodeGetDependentNodes_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphNodeGetDependentNodes"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1355, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1357 + * _F_cuGetProcAddress_v2('cuGraphNodeGetDependentNodes', &__cuGraphNodeGetDependentNodes, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphNodeGetDependentNodes_v2 + * _F_cuGetProcAddress_v2('cuGraphNodeGetDependentNodes', &__cuGraphNodeGetDependentNodes_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddDependencies + * _F_cuGetProcAddress_v2('cuGraphAddDependencies', &__cuGraphAddDependencies, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphNodeGetDependentNodes"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes_v2), 0x2EFE, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1357, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1359 + * _F_cuGetProcAddress_v2('cuGraphNodeGetDependentNodes', &__cuGraphNodeGetDependentNodes_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddDependencies + * _F_cuGetProcAddress_v2('cuGraphAddDependencies', &__cuGraphAddDependencies, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddDependencies_v2 + * _F_cuGetProcAddress_v2('cuGraphAddDependencies', &__cuGraphAddDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddDependencies"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1359, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1361 + * _F_cuGetProcAddress_v2('cuGraphAddDependencies', &__cuGraphAddDependencies, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddDependencies_v2 + * _F_cuGetProcAddress_v2('cuGraphAddDependencies', &__cuGraphAddDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphRemoveDependencies + * _F_cuGetProcAddress_v2('cuGraphRemoveDependencies', &__cuGraphRemoveDependencies, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddDependencies"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies_v2), 0x2EFE, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1361, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1363 + * _F_cuGetProcAddress_v2('cuGraphAddDependencies', &__cuGraphAddDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphRemoveDependencies + * _F_cuGetProcAddress_v2('cuGraphRemoveDependencies', &__cuGraphRemoveDependencies, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphRemoveDependencies_v2 + * _F_cuGetProcAddress_v2('cuGraphRemoveDependencies', &__cuGraphRemoveDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphRemoveDependencies"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1363, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1365 + * _F_cuGetProcAddress_v2('cuGraphRemoveDependencies', &__cuGraphRemoveDependencies, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphRemoveDependencies_v2 + * _F_cuGetProcAddress_v2('cuGraphRemoveDependencies', &__cuGraphRemoveDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphDestroyNode + * _F_cuGetProcAddress_v2('cuGraphDestroyNode', &__cuGraphDestroyNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphRemoveDependencies"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies_v2), 0x2EFE, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1365, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1367 + * _F_cuGetProcAddress_v2('cuGraphRemoveDependencies', &__cuGraphRemoveDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphDestroyNode + * _F_cuGetProcAddress_v2('cuGraphDestroyNode', &__cuGraphDestroyNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphInstantiateWithFlags + * _F_cuGetProcAddress_v2('cuGraphInstantiateWithFlags', &__cuGraphInstantiateWithFlags, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphDestroyNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroyNode), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1367, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1369 + * _F_cuGetProcAddress_v2('cuGraphDestroyNode', &__cuGraphDestroyNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphInstantiateWithFlags + * _F_cuGetProcAddress_v2('cuGraphInstantiateWithFlags', &__cuGraphInstantiateWithFlags, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecGetFlags + * _F_cuGetProcAddress_v2('cuGraphExecGetFlags', &__cuGraphExecGetFlags, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphInstantiateWithFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithFlags), 0x2B20, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1369, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1371 + * _F_cuGetProcAddress_v2('cuGraphInstantiateWithFlags', &__cuGraphInstantiateWithFlags, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecGetFlags + * _F_cuGetProcAddress_v2('cuGraphExecGetFlags', &__cuGraphExecGetFlags, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecKernelNodeSetParams_v2 + * _F_cuGetProcAddress_v2('cuGraphExecKernelNodeSetParams', &__cuGraphExecKernelNodeSetParams_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecGetFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecGetFlags), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1371, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1373 + * _F_cuGetProcAddress_v2('cuGraphExecGetFlags', &__cuGraphExecGetFlags, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecKernelNodeSetParams_v2 + * _F_cuGetProcAddress_v2('cuGraphExecKernelNodeSetParams', &__cuGraphExecKernelNodeSetParams_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecMemcpyNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecMemcpyNodeSetParams', &__cuGraphExecMemcpyNodeSetParams, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecKernelNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecKernelNodeSetParams_v2), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1373, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1375 + * _F_cuGetProcAddress_v2('cuGraphExecKernelNodeSetParams', &__cuGraphExecKernelNodeSetParams_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecMemcpyNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecMemcpyNodeSetParams', &__cuGraphExecMemcpyNodeSetParams, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecMemsetNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecMemsetNodeSetParams', &__cuGraphExecMemsetNodeSetParams, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecMemcpyNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemcpyNodeSetParams), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1375, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1377 + * _F_cuGetProcAddress_v2('cuGraphExecMemcpyNodeSetParams', &__cuGraphExecMemcpyNodeSetParams, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecMemsetNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecMemsetNodeSetParams', &__cuGraphExecMemsetNodeSetParams, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecHostNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecHostNodeSetParams', &__cuGraphExecHostNodeSetParams, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecMemsetNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemsetNodeSetParams), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1377, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1379 + * _F_cuGetProcAddress_v2('cuGraphExecMemsetNodeSetParams', &__cuGraphExecMemsetNodeSetParams, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecHostNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecHostNodeSetParams', &__cuGraphExecHostNodeSetParams, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecChildGraphNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecChildGraphNodeSetParams', &__cuGraphExecChildGraphNodeSetParams, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecHostNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecHostNodeSetParams), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1379, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1381 + * _F_cuGetProcAddress_v2('cuGraphExecHostNodeSetParams', &__cuGraphExecHostNodeSetParams, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecChildGraphNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecChildGraphNodeSetParams', &__cuGraphExecChildGraphNodeSetParams, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecEventRecordNodeSetEvent + * _F_cuGetProcAddress_v2('cuGraphExecEventRecordNodeSetEvent', &__cuGraphExecEventRecordNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecChildGraphNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecChildGraphNodeSetParams), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1381, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1383 + * _F_cuGetProcAddress_v2('cuGraphExecChildGraphNodeSetParams', &__cuGraphExecChildGraphNodeSetParams, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecEventRecordNodeSetEvent + * _F_cuGetProcAddress_v2('cuGraphExecEventRecordNodeSetEvent', &__cuGraphExecEventRecordNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecEventWaitNodeSetEvent + * _F_cuGetProcAddress_v2('cuGraphExecEventWaitNodeSetEvent', &__cuGraphExecEventWaitNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecEventRecordNodeSetEvent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventRecordNodeSetEvent), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1383, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1385 + * _F_cuGetProcAddress_v2('cuGraphExecEventRecordNodeSetEvent', &__cuGraphExecEventRecordNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecEventWaitNodeSetEvent + * _F_cuGetProcAddress_v2('cuGraphExecEventWaitNodeSetEvent', &__cuGraphExecEventWaitNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecExternalSemaphoresSignalNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecExternalSemaphoresSignalNodeSetParams', &__cuGraphExecExternalSemaphoresSignalNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecEventWaitNodeSetEvent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventWaitNodeSetEvent), 0x2B02, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1385, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1387 + * _F_cuGetProcAddress_v2('cuGraphExecEventWaitNodeSetEvent', &__cuGraphExecEventWaitNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecExternalSemaphoresSignalNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecExternalSemaphoresSignalNodeSetParams', &__cuGraphExecExternalSemaphoresSignalNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecExternalSemaphoresWaitNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecExternalSemaphoresWaitNodeSetParams', &__cuGraphExecExternalSemaphoresWaitNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecExternalSemaphoresSignalNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresSignalNodeSetParams), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1387, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1389 + * _F_cuGetProcAddress_v2('cuGraphExecExternalSemaphoresSignalNodeSetParams', &__cuGraphExecExternalSemaphoresSignalNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecExternalSemaphoresWaitNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecExternalSemaphoresWaitNodeSetParams', &__cuGraphExecExternalSemaphoresWaitNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphNodeSetEnabled + * _F_cuGetProcAddress_v2('cuGraphNodeSetEnabled', &__cuGraphNodeSetEnabled, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecExternalSemaphoresWaitNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresWaitNodeSetParams), 0x2B0C, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1389, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1391 + * _F_cuGetProcAddress_v2('cuGraphExecExternalSemaphoresWaitNodeSetParams', &__cuGraphExecExternalSemaphoresWaitNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphNodeSetEnabled + * _F_cuGetProcAddress_v2('cuGraphNodeSetEnabled', &__cuGraphNodeSetEnabled, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphNodeGetEnabled + * _F_cuGetProcAddress_v2('cuGraphNodeGetEnabled', &__cuGraphNodeGetEnabled, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphNodeSetEnabled"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetEnabled), 0x2B34, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1391, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1393 + * _F_cuGetProcAddress_v2('cuGraphNodeSetEnabled', &__cuGraphNodeSetEnabled, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphNodeGetEnabled + * _F_cuGetProcAddress_v2('cuGraphNodeGetEnabled', &__cuGraphNodeGetEnabled, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecDestroy + * _F_cuGetProcAddress_v2('cuGraphExecDestroy', &__cuGraphExecDestroy, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphNodeGetEnabled"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetEnabled), 0x2B34, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1393, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1395 + * _F_cuGetProcAddress_v2('cuGraphNodeGetEnabled', &__cuGraphNodeGetEnabled, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecDestroy + * _F_cuGetProcAddress_v2('cuGraphExecDestroy', &__cuGraphExecDestroy, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphDestroy + * _F_cuGetProcAddress_v2('cuGraphDestroy', &__cuGraphDestroy, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecDestroy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecDestroy), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1395, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1397 + * _F_cuGetProcAddress_v2('cuGraphExecDestroy', &__cuGraphExecDestroy, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphDestroy + * _F_cuGetProcAddress_v2('cuGraphDestroy', &__cuGraphDestroy, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecUpdate_v2 + * _F_cuGetProcAddress_v2('cuGraphExecUpdate', &__cuGraphExecUpdate_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphDestroy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroy), 0x2710, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1397, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1399 + * _F_cuGetProcAddress_v2('cuGraphDestroy', &__cuGraphDestroy, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecUpdate_v2 + * _F_cuGetProcAddress_v2('cuGraphExecUpdate', &__cuGraphExecUpdate_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeCopyAttributes + * _F_cuGetProcAddress_v2('cuGraphKernelNodeCopyAttributes', &__cuGraphKernelNodeCopyAttributes, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecUpdate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecUpdate_v2), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1399, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1401 + * _F_cuGetProcAddress_v2('cuGraphExecUpdate', &__cuGraphExecUpdate_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphKernelNodeCopyAttributes + * _F_cuGetProcAddress_v2('cuGraphKernelNodeCopyAttributes', &__cuGraphKernelNodeCopyAttributes, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeGetAttribute + * _F_cuGetProcAddress_v2('cuGraphKernelNodeGetAttribute', &__cuGraphKernelNodeGetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphKernelNodeCopyAttributes"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeCopyAttributes), 0x2AF8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1401, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1403 + * _F_cuGetProcAddress_v2('cuGraphKernelNodeCopyAttributes', &__cuGraphKernelNodeCopyAttributes, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphKernelNodeGetAttribute + * _F_cuGetProcAddress_v2('cuGraphKernelNodeGetAttribute', &__cuGraphKernelNodeGetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeSetAttribute + * _F_cuGetProcAddress_v2('cuGraphKernelNodeSetAttribute', &__cuGraphKernelNodeSetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphKernelNodeGetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetAttribute), 0x2AF8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1403, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1405 + * _F_cuGetProcAddress_v2('cuGraphKernelNodeGetAttribute', &__cuGraphKernelNodeGetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphKernelNodeSetAttribute + * _F_cuGetProcAddress_v2('cuGraphKernelNodeSetAttribute', &__cuGraphKernelNodeSetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphDebugDotPrint + * _F_cuGetProcAddress_v2('cuGraphDebugDotPrint', &__cuGraphDebugDotPrint, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphKernelNodeSetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetAttribute), 0x2AF8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1405, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1407 + * _F_cuGetProcAddress_v2('cuGraphKernelNodeSetAttribute', &__cuGraphKernelNodeSetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphDebugDotPrint + * _F_cuGetProcAddress_v2('cuGraphDebugDotPrint', &__cuGraphDebugDotPrint, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuUserObjectCreate + * _F_cuGetProcAddress_v2('cuUserObjectCreate', &__cuUserObjectCreate, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphDebugDotPrint"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDebugDotPrint), 0x2B16, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1407, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1409 + * _F_cuGetProcAddress_v2('cuGraphDebugDotPrint', &__cuGraphDebugDotPrint, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuUserObjectCreate + * _F_cuGetProcAddress_v2('cuUserObjectCreate', &__cuUserObjectCreate, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuUserObjectRetain + * _F_cuGetProcAddress_v2('cuUserObjectRetain', &__cuUserObjectRetain, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuUserObjectCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectCreate), 0x2B16, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1409, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1411 + * _F_cuGetProcAddress_v2('cuUserObjectCreate', &__cuUserObjectCreate, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuUserObjectRetain + * _F_cuGetProcAddress_v2('cuUserObjectRetain', &__cuUserObjectRetain, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuUserObjectRelease + * _F_cuGetProcAddress_v2('cuUserObjectRelease', &__cuUserObjectRelease, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuUserObjectRetain"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRetain), 0x2B16, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1411, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1413 + * _F_cuGetProcAddress_v2('cuUserObjectRetain', &__cuUserObjectRetain, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuUserObjectRelease + * _F_cuGetProcAddress_v2('cuUserObjectRelease', &__cuUserObjectRelease, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphRetainUserObject + * _F_cuGetProcAddress_v2('cuGraphRetainUserObject', &__cuGraphRetainUserObject, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuUserObjectRelease"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRelease), 0x2B16, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1413, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1415 + * _F_cuGetProcAddress_v2('cuUserObjectRelease', &__cuUserObjectRelease, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphRetainUserObject + * _F_cuGetProcAddress_v2('cuGraphRetainUserObject', &__cuGraphRetainUserObject, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphReleaseUserObject + * _F_cuGetProcAddress_v2('cuGraphReleaseUserObject', &__cuGraphReleaseUserObject, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphRetainUserObject"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRetainUserObject), 0x2B16, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1415, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1417 + * _F_cuGetProcAddress_v2('cuGraphRetainUserObject', &__cuGraphRetainUserObject, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphReleaseUserObject + * _F_cuGetProcAddress_v2('cuGraphReleaseUserObject', &__cuGraphReleaseUserObject, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddNode + * _F_cuGetProcAddress_v2('cuGraphAddNode', &__cuGraphAddNode, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphReleaseUserObject"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphReleaseUserObject), 0x2B16, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1417, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1419 + * _F_cuGetProcAddress_v2('cuGraphReleaseUserObject', &__cuGraphReleaseUserObject, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddNode + * _F_cuGetProcAddress_v2('cuGraphAddNode', &__cuGraphAddNode, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphAddNode_v2 + * _F_cuGetProcAddress_v2('cuGraphAddNode', &__cuGraphAddNode_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode), 0x2EF4, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1419, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1421 + * _F_cuGetProcAddress_v2('cuGraphAddNode', &__cuGraphAddNode, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphAddNode_v2 + * _F_cuGetProcAddress_v2('cuGraphAddNode', &__cuGraphAddNode_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphNodeSetParams', &__cuGraphNodeSetParams, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphAddNode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode_v2), 0x2EFE, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1421, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1423 + * _F_cuGetProcAddress_v2('cuGraphAddNode', &__cuGraphAddNode_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphNodeSetParams', &__cuGraphNodeSetParams, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphExecNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecNodeSetParams', &__cuGraphExecNodeSetParams, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetParams), 0x2EF4, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1423, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1425 + * _F_cuGetProcAddress_v2('cuGraphNodeSetParams', &__cuGraphNodeSetParams, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphExecNodeSetParams + * _F_cuGetProcAddress_v2('cuGraphExecNodeSetParams', &__cuGraphExecNodeSetParams, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphConditionalHandleCreate + * _F_cuGetProcAddress_v2('cuGraphConditionalHandleCreate', &__cuGraphConditionalHandleCreate, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphExecNodeSetParams"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecNodeSetParams), 0x2EF4, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1425, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1427 + * _F_cuGetProcAddress_v2('cuGraphExecNodeSetParams', &__cuGraphExecNodeSetParams, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphConditionalHandleCreate + * _F_cuGetProcAddress_v2('cuGraphConditionalHandleCreate', &__cuGraphConditionalHandleCreate, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveBlocksPerMultiprocessor + * _F_cuGetProcAddress_v2('cuOccupancyMaxActiveBlocksPerMultiprocessor', &__cuOccupancyMaxActiveBlocksPerMultiprocessor, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphConditionalHandleCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphConditionalHandleCreate), 0x2EFE, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1427, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1429 + * _F_cuGetProcAddress_v2('cuGraphConditionalHandleCreate', &__cuGraphConditionalHandleCreate, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuOccupancyMaxActiveBlocksPerMultiprocessor + * _F_cuGetProcAddress_v2('cuOccupancyMaxActiveBlocksPerMultiprocessor', &__cuOccupancyMaxActiveBlocksPerMultiprocessor, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + * _F_cuGetProcAddress_v2('cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags', &__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuOccupancyMaxActiveBlocksPerMultiprocessor"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessor), 0x17A2, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1429, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1431 + * _F_cuGetProcAddress_v2('cuOccupancyMaxActiveBlocksPerMultiprocessor', &__cuOccupancyMaxActiveBlocksPerMultiprocessor, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + * _F_cuGetProcAddress_v2('cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags', &__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialBlockSize + * _F_cuGetProcAddress_v2('cuOccupancyMaxPotentialBlockSize', &__cuOccupancyMaxPotentialBlockSize, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1431, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1433 + * _F_cuGetProcAddress_v2('cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags', &__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuOccupancyMaxPotentialBlockSize + * _F_cuGetProcAddress_v2('cuOccupancyMaxPotentialBlockSize', &__cuOccupancyMaxPotentialBlockSize, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialBlockSizeWithFlags + * _F_cuGetProcAddress_v2('cuOccupancyMaxPotentialBlockSizeWithFlags', &__cuOccupancyMaxPotentialBlockSizeWithFlags, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuOccupancyMaxPotentialBlockSize"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSize), 0x17A2, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1433, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1435 + * _F_cuGetProcAddress_v2('cuOccupancyMaxPotentialBlockSize', &__cuOccupancyMaxPotentialBlockSize, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuOccupancyMaxPotentialBlockSizeWithFlags + * _F_cuGetProcAddress_v2('cuOccupancyMaxPotentialBlockSizeWithFlags', &__cuOccupancyMaxPotentialBlockSizeWithFlags, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuOccupancyAvailableDynamicSMemPerBlock + * _F_cuGetProcAddress_v2('cuOccupancyAvailableDynamicSMemPerBlock', &__cuOccupancyAvailableDynamicSMemPerBlock, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuOccupancyMaxPotentialBlockSizeWithFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSizeWithFlags), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1435, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1437 + * _F_cuGetProcAddress_v2('cuOccupancyMaxPotentialBlockSizeWithFlags', &__cuOccupancyMaxPotentialBlockSizeWithFlags, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuOccupancyAvailableDynamicSMemPerBlock + * _F_cuGetProcAddress_v2('cuOccupancyAvailableDynamicSMemPerBlock', &__cuOccupancyAvailableDynamicSMemPerBlock, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialClusterSize + * _F_cuGetProcAddress_v2('cuOccupancyMaxPotentialClusterSize', &__cuOccupancyMaxPotentialClusterSize, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuOccupancyAvailableDynamicSMemPerBlock"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyAvailableDynamicSMemPerBlock), 0x2724, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1437, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1439 + * _F_cuGetProcAddress_v2('cuOccupancyAvailableDynamicSMemPerBlock', &__cuOccupancyAvailableDynamicSMemPerBlock, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuOccupancyMaxPotentialClusterSize + * _F_cuGetProcAddress_v2('cuOccupancyMaxPotentialClusterSize', &__cuOccupancyMaxPotentialClusterSize, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveClusters + * _F_cuGetProcAddress_v2('cuOccupancyMaxActiveClusters', &__cuOccupancyMaxActiveClusters, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuOccupancyMaxPotentialClusterSize"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialClusterSize), 0x2B3E, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1439, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1441 + * _F_cuGetProcAddress_v2('cuOccupancyMaxPotentialClusterSize', &__cuOccupancyMaxPotentialClusterSize, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuOccupancyMaxActiveClusters + * _F_cuGetProcAddress_v2('cuOccupancyMaxActiveClusters', &__cuOccupancyMaxActiveClusters, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefSetArray + * _F_cuGetProcAddress_v2('cuTexRefSetArray', &__cuTexRefSetArray, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuOccupancyMaxActiveClusters"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveClusters), 0x2B3E, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1441, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1443 + * _F_cuGetProcAddress_v2('cuOccupancyMaxActiveClusters', &__cuOccupancyMaxActiveClusters, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefSetArray + * _F_cuGetProcAddress_v2('cuTexRefSetArray', &__cuTexRefSetArray, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmappedArray + * _F_cuGetProcAddress_v2('cuTexRefSetMipmappedArray', &__cuTexRefSetMipmappedArray, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefSetArray"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetArray), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1443, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1445 + * _F_cuGetProcAddress_v2('cuTexRefSetArray', &__cuTexRefSetArray, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefSetMipmappedArray + * _F_cuGetProcAddress_v2('cuTexRefSetMipmappedArray', &__cuTexRefSetMipmappedArray, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefSetAddress_v2 + * _F_cuGetProcAddress_v2('cuTexRefSetAddress', &__cuTexRefSetAddress_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefSetMipmappedArray"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmappedArray), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1445, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1447 + * _F_cuGetProcAddress_v2('cuTexRefSetMipmappedArray', &__cuTexRefSetMipmappedArray, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefSetAddress_v2 + * _F_cuGetProcAddress_v2('cuTexRefSetAddress', &__cuTexRefSetAddress_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefSetAddress2D_v3 + * _F_cuGetProcAddress_v2('cuTexRefSetAddress2D', &__cuTexRefSetAddress2D_v3, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefSetAddress"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1447, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1449 + * _F_cuGetProcAddress_v2('cuTexRefSetAddress', &__cuTexRefSetAddress_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefSetAddress2D_v3 + * _F_cuGetProcAddress_v2('cuTexRefSetAddress2D', &__cuTexRefSetAddress2D_v3, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefSetFormat + * _F_cuGetProcAddress_v2('cuTexRefSetFormat', &__cuTexRefSetFormat, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefSetAddress2D"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress2D_v3), 0xFAA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1449, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1451 + * _F_cuGetProcAddress_v2('cuTexRefSetAddress2D', &__cuTexRefSetAddress2D_v3, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefSetFormat + * _F_cuGetProcAddress_v2('cuTexRefSetFormat', &__cuTexRefSetFormat, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefSetAddressMode + * _F_cuGetProcAddress_v2('cuTexRefSetAddressMode', &__cuTexRefSetAddressMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefSetFormat"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFormat), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1451, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1453 + * _F_cuGetProcAddress_v2('cuTexRefSetFormat', &__cuTexRefSetFormat, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefSetAddressMode + * _F_cuGetProcAddress_v2('cuTexRefSetAddressMode', &__cuTexRefSetAddressMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefSetFilterMode + * _F_cuGetProcAddress_v2('cuTexRefSetFilterMode', &__cuTexRefSetFilterMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefSetAddressMode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddressMode), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1453, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1455 + * _F_cuGetProcAddress_v2('cuTexRefSetAddressMode', &__cuTexRefSetAddressMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefSetFilterMode + * _F_cuGetProcAddress_v2('cuTexRefSetFilterMode', &__cuTexRefSetFilterMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapFilterMode + * _F_cuGetProcAddress_v2('cuTexRefSetMipmapFilterMode', &__cuTexRefSetMipmapFilterMode, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefSetFilterMode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFilterMode), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1455, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1457 + * _F_cuGetProcAddress_v2('cuTexRefSetFilterMode', &__cuTexRefSetFilterMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefSetMipmapFilterMode + * _F_cuGetProcAddress_v2('cuTexRefSetMipmapFilterMode', &__cuTexRefSetMipmapFilterMode, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapLevelBias + * _F_cuGetProcAddress_v2('cuTexRefSetMipmapLevelBias', &__cuTexRefSetMipmapLevelBias, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefSetMipmapFilterMode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapFilterMode), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1457, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1459 + * _F_cuGetProcAddress_v2('cuTexRefSetMipmapFilterMode', &__cuTexRefSetMipmapFilterMode, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefSetMipmapLevelBias + * _F_cuGetProcAddress_v2('cuTexRefSetMipmapLevelBias', &__cuTexRefSetMipmapLevelBias, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapLevelClamp + * _F_cuGetProcAddress_v2('cuTexRefSetMipmapLevelClamp', &__cuTexRefSetMipmapLevelClamp, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefSetMipmapLevelBias"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelBias), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1459, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1461 + * _F_cuGetProcAddress_v2('cuTexRefSetMipmapLevelBias', &__cuTexRefSetMipmapLevelBias, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefSetMipmapLevelClamp + * _F_cuGetProcAddress_v2('cuTexRefSetMipmapLevelClamp', &__cuTexRefSetMipmapLevelClamp, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefSetMaxAnisotropy + * _F_cuGetProcAddress_v2('cuTexRefSetMaxAnisotropy', &__cuTexRefSetMaxAnisotropy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefSetMipmapLevelClamp"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelClamp), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1461, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1463 + * _F_cuGetProcAddress_v2('cuTexRefSetMipmapLevelClamp', &__cuTexRefSetMipmapLevelClamp, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefSetMaxAnisotropy + * _F_cuGetProcAddress_v2('cuTexRefSetMaxAnisotropy', &__cuTexRefSetMaxAnisotropy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefSetBorderColor + * _F_cuGetProcAddress_v2('cuTexRefSetBorderColor', &__cuTexRefSetBorderColor, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefSetMaxAnisotropy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMaxAnisotropy), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1463, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1465 + * _F_cuGetProcAddress_v2('cuTexRefSetMaxAnisotropy', &__cuTexRefSetMaxAnisotropy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefSetBorderColor + * _F_cuGetProcAddress_v2('cuTexRefSetBorderColor', &__cuTexRefSetBorderColor, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefSetFlags + * _F_cuGetProcAddress_v2('cuTexRefSetFlags', &__cuTexRefSetFlags, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefSetBorderColor"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetBorderColor), 0x1F40, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1465, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1467 + * _F_cuGetProcAddress_v2('cuTexRefSetBorderColor', &__cuTexRefSetBorderColor, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefSetFlags + * _F_cuGetProcAddress_v2('cuTexRefSetFlags', &__cuTexRefSetFlags, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefGetAddress_v2 + * _F_cuGetProcAddress_v2('cuTexRefGetAddress', &__cuTexRefGetAddress_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefSetFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFlags), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1467, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1469 + * _F_cuGetProcAddress_v2('cuTexRefSetFlags', &__cuTexRefSetFlags, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefGetAddress_v2 + * _F_cuGetProcAddress_v2('cuTexRefGetAddress', &__cuTexRefGetAddress_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefGetArray + * _F_cuGetProcAddress_v2('cuTexRefGetArray', &__cuTexRefGetArray, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefGetAddress"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddress_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1469, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1471 + * _F_cuGetProcAddress_v2('cuTexRefGetAddress', &__cuTexRefGetAddress_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefGetArray + * _F_cuGetProcAddress_v2('cuTexRefGetArray', &__cuTexRefGetArray, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmappedArray + * _F_cuGetProcAddress_v2('cuTexRefGetMipmappedArray', &__cuTexRefGetMipmappedArray, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefGetArray"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetArray), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1471, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1473 + * _F_cuGetProcAddress_v2('cuTexRefGetArray', &__cuTexRefGetArray, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefGetMipmappedArray + * _F_cuGetProcAddress_v2('cuTexRefGetMipmappedArray', &__cuTexRefGetMipmappedArray, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefGetAddressMode + * _F_cuGetProcAddress_v2('cuTexRefGetAddressMode', &__cuTexRefGetAddressMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefGetMipmappedArray"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmappedArray), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1473, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1475 + * _F_cuGetProcAddress_v2('cuTexRefGetMipmappedArray', &__cuTexRefGetMipmappedArray, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefGetAddressMode + * _F_cuGetProcAddress_v2('cuTexRefGetAddressMode', &__cuTexRefGetAddressMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefGetFilterMode + * _F_cuGetProcAddress_v2('cuTexRefGetFilterMode', &__cuTexRefGetFilterMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefGetAddressMode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddressMode), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1475, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1477 + * _F_cuGetProcAddress_v2('cuTexRefGetAddressMode', &__cuTexRefGetAddressMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefGetFilterMode + * _F_cuGetProcAddress_v2('cuTexRefGetFilterMode', &__cuTexRefGetFilterMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefGetFormat + * _F_cuGetProcAddress_v2('cuTexRefGetFormat', &__cuTexRefGetFormat, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefGetFilterMode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFilterMode), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1477, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1479 + * _F_cuGetProcAddress_v2('cuTexRefGetFilterMode', &__cuTexRefGetFilterMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefGetFormat + * _F_cuGetProcAddress_v2('cuTexRefGetFormat', &__cuTexRefGetFormat, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapFilterMode + * _F_cuGetProcAddress_v2('cuTexRefGetMipmapFilterMode', &__cuTexRefGetMipmapFilterMode, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefGetFormat"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFormat), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1479, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1481 + * _F_cuGetProcAddress_v2('cuTexRefGetFormat', &__cuTexRefGetFormat, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefGetMipmapFilterMode + * _F_cuGetProcAddress_v2('cuTexRefGetMipmapFilterMode', &__cuTexRefGetMipmapFilterMode, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapLevelBias + * _F_cuGetProcAddress_v2('cuTexRefGetMipmapLevelBias', &__cuTexRefGetMipmapLevelBias, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefGetMipmapFilterMode"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapFilterMode), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1481, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1483 + * _F_cuGetProcAddress_v2('cuTexRefGetMipmapFilterMode', &__cuTexRefGetMipmapFilterMode, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefGetMipmapLevelBias + * _F_cuGetProcAddress_v2('cuTexRefGetMipmapLevelBias', &__cuTexRefGetMipmapLevelBias, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapLevelClamp + * _F_cuGetProcAddress_v2('cuTexRefGetMipmapLevelClamp', &__cuTexRefGetMipmapLevelClamp, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefGetMipmapLevelBias"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelBias), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1483, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1485 + * _F_cuGetProcAddress_v2('cuTexRefGetMipmapLevelBias', &__cuTexRefGetMipmapLevelBias, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefGetMipmapLevelClamp + * _F_cuGetProcAddress_v2('cuTexRefGetMipmapLevelClamp', &__cuTexRefGetMipmapLevelClamp, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefGetMaxAnisotropy + * _F_cuGetProcAddress_v2('cuTexRefGetMaxAnisotropy', &__cuTexRefGetMaxAnisotropy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefGetMipmapLevelClamp"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelClamp), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1485, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1487 + * _F_cuGetProcAddress_v2('cuTexRefGetMipmapLevelClamp', &__cuTexRefGetMipmapLevelClamp, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefGetMaxAnisotropy + * _F_cuGetProcAddress_v2('cuTexRefGetMaxAnisotropy', &__cuTexRefGetMaxAnisotropy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefGetBorderColor + * _F_cuGetProcAddress_v2('cuTexRefGetBorderColor', &__cuTexRefGetBorderColor, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefGetMaxAnisotropy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMaxAnisotropy), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1487, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1489 + * _F_cuGetProcAddress_v2('cuTexRefGetMaxAnisotropy', &__cuTexRefGetMaxAnisotropy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefGetBorderColor + * _F_cuGetProcAddress_v2('cuTexRefGetBorderColor', &__cuTexRefGetBorderColor, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefGetFlags + * _F_cuGetProcAddress_v2('cuTexRefGetFlags', &__cuTexRefGetFlags, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefGetBorderColor"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetBorderColor), 0x1F40, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1489, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1491 + * _F_cuGetProcAddress_v2('cuTexRefGetBorderColor', &__cuTexRefGetBorderColor, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefGetFlags + * _F_cuGetProcAddress_v2('cuTexRefGetFlags', &__cuTexRefGetFlags, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefCreate + * _F_cuGetProcAddress_v2('cuTexRefCreate', &__cuTexRefCreate, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefGetFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFlags), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1491, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1493 + * _F_cuGetProcAddress_v2('cuTexRefGetFlags', &__cuTexRefGetFlags, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefCreate + * _F_cuGetProcAddress_v2('cuTexRefCreate', &__cuTexRefCreate, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexRefDestroy + * _F_cuGetProcAddress_v2('cuTexRefDestroy', &__cuTexRefDestroy, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefCreate), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1493, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1495 + * _F_cuGetProcAddress_v2('cuTexRefCreate', &__cuTexRefCreate, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexRefDestroy + * _F_cuGetProcAddress_v2('cuTexRefDestroy', &__cuTexRefDestroy, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuSurfRefSetArray + * _F_cuGetProcAddress_v2('cuSurfRefSetArray', &__cuSurfRefSetArray, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexRefDestroy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefDestroy), 0x7D0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1495, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1497 + * _F_cuGetProcAddress_v2('cuTexRefDestroy', &__cuTexRefDestroy, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuSurfRefSetArray + * _F_cuGetProcAddress_v2('cuSurfRefSetArray', &__cuSurfRefSetArray, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuSurfRefGetArray + * _F_cuGetProcAddress_v2('cuSurfRefGetArray', &__cuSurfRefGetArray, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuSurfRefSetArray"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefSetArray), 0xBB8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1497, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1499 + * _F_cuGetProcAddress_v2('cuSurfRefSetArray', &__cuSurfRefSetArray, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuSurfRefGetArray + * _F_cuGetProcAddress_v2('cuSurfRefGetArray', &__cuSurfRefGetArray, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexObjectCreate + * _F_cuGetProcAddress_v2('cuTexObjectCreate', &__cuTexObjectCreate, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuSurfRefGetArray"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefGetArray), 0xBB8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1499, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1501 + * _F_cuGetProcAddress_v2('cuSurfRefGetArray', &__cuSurfRefGetArray, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexObjectCreate + * _F_cuGetProcAddress_v2('cuTexObjectCreate', &__cuTexObjectCreate, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexObjectDestroy + * _F_cuGetProcAddress_v2('cuTexObjectDestroy', &__cuTexObjectDestroy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexObjectCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectCreate), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1501, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1503 + * _F_cuGetProcAddress_v2('cuTexObjectCreate', &__cuTexObjectCreate, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexObjectDestroy + * _F_cuGetProcAddress_v2('cuTexObjectDestroy', &__cuTexObjectDestroy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexObjectGetResourceDesc + * _F_cuGetProcAddress_v2('cuTexObjectGetResourceDesc', &__cuTexObjectGetResourceDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexObjectDestroy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectDestroy), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1503, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1505 + * _F_cuGetProcAddress_v2('cuTexObjectDestroy', &__cuTexObjectDestroy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexObjectGetResourceDesc + * _F_cuGetProcAddress_v2('cuTexObjectGetResourceDesc', &__cuTexObjectGetResourceDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexObjectGetTextureDesc + * _F_cuGetProcAddress_v2('cuTexObjectGetTextureDesc', &__cuTexObjectGetTextureDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexObjectGetResourceDesc"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceDesc), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1505, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1507 + * _F_cuGetProcAddress_v2('cuTexObjectGetResourceDesc', &__cuTexObjectGetResourceDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexObjectGetTextureDesc + * _F_cuGetProcAddress_v2('cuTexObjectGetTextureDesc', &__cuTexObjectGetTextureDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTexObjectGetResourceViewDesc + * _F_cuGetProcAddress_v2('cuTexObjectGetResourceViewDesc', &__cuTexObjectGetResourceViewDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexObjectGetTextureDesc"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetTextureDesc), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1507, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1509 + * _F_cuGetProcAddress_v2('cuTexObjectGetTextureDesc', &__cuTexObjectGetTextureDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTexObjectGetResourceViewDesc + * _F_cuGetProcAddress_v2('cuTexObjectGetResourceViewDesc', &__cuTexObjectGetResourceViewDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuSurfObjectCreate + * _F_cuGetProcAddress_v2('cuSurfObjectCreate', &__cuSurfObjectCreate, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTexObjectGetResourceViewDesc"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceViewDesc), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1509, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1511 + * _F_cuGetProcAddress_v2('cuTexObjectGetResourceViewDesc', &__cuTexObjectGetResourceViewDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuSurfObjectCreate + * _F_cuGetProcAddress_v2('cuSurfObjectCreate', &__cuSurfObjectCreate, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuSurfObjectDestroy + * _F_cuGetProcAddress_v2('cuSurfObjectDestroy', &__cuSurfObjectDestroy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuSurfObjectCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectCreate), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1511, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1513 + * _F_cuGetProcAddress_v2('cuSurfObjectCreate', &__cuSurfObjectCreate, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuSurfObjectDestroy + * _F_cuGetProcAddress_v2('cuSurfObjectDestroy', &__cuSurfObjectDestroy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuSurfObjectGetResourceDesc + * _F_cuGetProcAddress_v2('cuSurfObjectGetResourceDesc', &__cuSurfObjectGetResourceDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuSurfObjectDestroy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectDestroy), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1513, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1515 + * _F_cuGetProcAddress_v2('cuSurfObjectDestroy', &__cuSurfObjectDestroy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuSurfObjectGetResourceDesc + * _F_cuGetProcAddress_v2('cuSurfObjectGetResourceDesc', &__cuSurfObjectGetResourceDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeTiled + * _F_cuGetProcAddress_v2('cuTensorMapEncodeTiled', &__cuTensorMapEncodeTiled, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuSurfObjectGetResourceDesc"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectGetResourceDesc), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1515, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1517 + * _F_cuGetProcAddress_v2('cuSurfObjectGetResourceDesc', &__cuSurfObjectGetResourceDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTensorMapEncodeTiled + * _F_cuGetProcAddress_v2('cuTensorMapEncodeTiled', &__cuTensorMapEncodeTiled, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeIm2col + * _F_cuGetProcAddress_v2('cuTensorMapEncodeIm2col', &__cuTensorMapEncodeIm2col, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTensorMapEncodeTiled"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeTiled), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1517, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1519 + * _F_cuGetProcAddress_v2('cuTensorMapEncodeTiled', &__cuTensorMapEncodeTiled, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTensorMapEncodeIm2col + * _F_cuGetProcAddress_v2('cuTensorMapEncodeIm2col', &__cuTensorMapEncodeIm2col, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeIm2colWide + * _F_cuGetProcAddress_v2('cuTensorMapEncodeIm2colWide', &__cuTensorMapEncodeIm2colWide, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTensorMapEncodeIm2col"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2col), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1519, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1521 + * _F_cuGetProcAddress_v2('cuTensorMapEncodeIm2col', &__cuTensorMapEncodeIm2col, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTensorMapEncodeIm2colWide + * _F_cuGetProcAddress_v2('cuTensorMapEncodeIm2colWide', &__cuTensorMapEncodeIm2colWide, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuTensorMapReplaceAddress + * _F_cuGetProcAddress_v2('cuTensorMapReplaceAddress', &__cuTensorMapReplaceAddress, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTensorMapEncodeIm2colWide"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2colWide), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1521, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1523 + * _F_cuGetProcAddress_v2('cuTensorMapEncodeIm2colWide', &__cuTensorMapEncodeIm2colWide, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuTensorMapReplaceAddress + * _F_cuGetProcAddress_v2('cuTensorMapReplaceAddress', &__cuTensorMapReplaceAddress, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceCanAccessPeer + * _F_cuGetProcAddress_v2('cuDeviceCanAccessPeer', &__cuDeviceCanAccessPeer, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuTensorMapReplaceAddress"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapReplaceAddress), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1523, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1525 + * _F_cuGetProcAddress_v2('cuTensorMapReplaceAddress', &__cuTensorMapReplaceAddress, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceCanAccessPeer + * _F_cuGetProcAddress_v2('cuDeviceCanAccessPeer', &__cuDeviceCanAccessPeer, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxEnablePeerAccess + * _F_cuGetProcAddress_v2('cuCtxEnablePeerAccess', &__cuCtxEnablePeerAccess, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceCanAccessPeer"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceCanAccessPeer), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1525, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1527 + * _F_cuGetProcAddress_v2('cuDeviceCanAccessPeer', &__cuDeviceCanAccessPeer, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxEnablePeerAccess + * _F_cuGetProcAddress_v2('cuCtxEnablePeerAccess', &__cuCtxEnablePeerAccess, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxDisablePeerAccess + * _F_cuGetProcAddress_v2('cuCtxDisablePeerAccess', &__cuCtxDisablePeerAccess, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxEnablePeerAccess"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxEnablePeerAccess), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1527, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1529 + * _F_cuGetProcAddress_v2('cuCtxEnablePeerAccess', &__cuCtxEnablePeerAccess, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxDisablePeerAccess + * _F_cuGetProcAddress_v2('cuCtxDisablePeerAccess', &__cuCtxDisablePeerAccess, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetP2PAttribute + * _F_cuGetProcAddress_v2('cuDeviceGetP2PAttribute', &__cuDeviceGetP2PAttribute, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxDisablePeerAccess"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDisablePeerAccess), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1529, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1531 + * _F_cuGetProcAddress_v2('cuCtxDisablePeerAccess', &__cuCtxDisablePeerAccess, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetP2PAttribute + * _F_cuGetProcAddress_v2('cuDeviceGetP2PAttribute', &__cuDeviceGetP2PAttribute, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsUnregisterResource + * _F_cuGetProcAddress_v2('cuGraphicsUnregisterResource', &__cuGraphicsUnregisterResource, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetP2PAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetP2PAttribute), 0x1F40, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1531, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1533 + * _F_cuGetProcAddress_v2('cuDeviceGetP2PAttribute', &__cuDeviceGetP2PAttribute, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphicsUnregisterResource + * _F_cuGetProcAddress_v2('cuGraphicsUnregisterResource', &__cuGraphicsUnregisterResource, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsSubResourceGetMappedArray + * _F_cuGetProcAddress_v2('cuGraphicsSubResourceGetMappedArray', &__cuGraphicsSubResourceGetMappedArray, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsUnregisterResource"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnregisterResource), 0xBB8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1533, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1535 + * _F_cuGetProcAddress_v2('cuGraphicsUnregisterResource', &__cuGraphicsUnregisterResource, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphicsSubResourceGetMappedArray + * _F_cuGetProcAddress_v2('cuGraphicsSubResourceGetMappedArray', &__cuGraphicsSubResourceGetMappedArray, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedMipmappedArray + * _F_cuGetProcAddress_v2('cuGraphicsResourceGetMappedMipmappedArray', &__cuGraphicsResourceGetMappedMipmappedArray, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsSubResourceGetMappedArray"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsSubResourceGetMappedArray), 0xBB8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1535, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1537 + * _F_cuGetProcAddress_v2('cuGraphicsSubResourceGetMappedArray', &__cuGraphicsSubResourceGetMappedArray, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphicsResourceGetMappedMipmappedArray + * _F_cuGetProcAddress_v2('cuGraphicsResourceGetMappedMipmappedArray', &__cuGraphicsResourceGetMappedMipmappedArray, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedPointer_v2 + * _F_cuGetProcAddress_v2('cuGraphicsResourceGetMappedPointer', &__cuGraphicsResourceGetMappedPointer_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsResourceGetMappedMipmappedArray"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedMipmappedArray), 0x1388, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1537, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1539 + * _F_cuGetProcAddress_v2('cuGraphicsResourceGetMappedMipmappedArray', &__cuGraphicsResourceGetMappedMipmappedArray, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphicsResourceGetMappedPointer_v2 + * _F_cuGetProcAddress_v2('cuGraphicsResourceGetMappedPointer', &__cuGraphicsResourceGetMappedPointer_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsResourceSetMapFlags_v2 + * _F_cuGetProcAddress_v2('cuGraphicsResourceSetMapFlags', &__cuGraphicsResourceSetMapFlags_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsResourceGetMappedPointer"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedPointer_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1539, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1541 + * _F_cuGetProcAddress_v2('cuGraphicsResourceGetMappedPointer', &__cuGraphicsResourceGetMappedPointer_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphicsResourceSetMapFlags_v2 + * _F_cuGetProcAddress_v2('cuGraphicsResourceSetMapFlags', &__cuGraphicsResourceSetMapFlags_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGetProcAddress_v2 + * _F_cuGetProcAddress_v2('cuGetProcAddress', &__cuGetProcAddress_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsResourceSetMapFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceSetMapFlags_v2), 0x17A2, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1541, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1543 + * _F_cuGetProcAddress_v2('cuGraphicsResourceSetMapFlags', &__cuGraphicsResourceSetMapFlags_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGetProcAddress_v2 + * _F_cuGetProcAddress_v2('cuGetProcAddress', &__cuGetProcAddress_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCoredumpGetAttribute + * _F_cuGetProcAddress_v2('cuCoredumpGetAttribute', &__cuCoredumpGetAttribute, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGetProcAddress"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2), 0x2EE0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1543, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1545 + * _F_cuGetProcAddress_v2('cuGetProcAddress', &__cuGetProcAddress_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCoredumpGetAttribute + * _F_cuGetProcAddress_v2('cuCoredumpGetAttribute', &__cuCoredumpGetAttribute, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCoredumpGetAttributeGlobal + * _F_cuGetProcAddress_v2('cuCoredumpGetAttributeGlobal', &__cuCoredumpGetAttributeGlobal, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCoredumpGetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttribute), 0x2EEA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1545, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1547 + * _F_cuGetProcAddress_v2('cuCoredumpGetAttribute', &__cuCoredumpGetAttribute, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCoredumpGetAttributeGlobal + * _F_cuGetProcAddress_v2('cuCoredumpGetAttributeGlobal', &__cuCoredumpGetAttributeGlobal, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCoredumpSetAttribute + * _F_cuGetProcAddress_v2('cuCoredumpSetAttribute', &__cuCoredumpSetAttribute, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCoredumpGetAttributeGlobal"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttributeGlobal), 0x2EEA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1547, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1549 + * _F_cuGetProcAddress_v2('cuCoredumpGetAttributeGlobal', &__cuCoredumpGetAttributeGlobal, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCoredumpSetAttribute + * _F_cuGetProcAddress_v2('cuCoredumpSetAttribute', &__cuCoredumpSetAttribute, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCoredumpSetAttributeGlobal + * _F_cuGetProcAddress_v2('cuCoredumpSetAttributeGlobal', &__cuCoredumpSetAttributeGlobal, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCoredumpSetAttribute"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttribute), 0x2EEA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1549, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1551 + * _F_cuGetProcAddress_v2('cuCoredumpSetAttribute', &__cuCoredumpSetAttribute, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCoredumpSetAttributeGlobal + * _F_cuGetProcAddress_v2('cuCoredumpSetAttributeGlobal', &__cuCoredumpSetAttributeGlobal, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGetExportTable + * _F_cuGetProcAddress_v2('cuGetExportTable', &__cuGetExportTable, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCoredumpSetAttributeGlobal"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttributeGlobal), 0x2EEA, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1551, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1553 + * _F_cuGetProcAddress_v2('cuCoredumpSetAttributeGlobal', &__cuCoredumpSetAttributeGlobal, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGetExportTable + * _F_cuGetProcAddress_v2('cuGetExportTable', &__cuGetExportTable, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGreenCtxCreate + * _F_cuGetProcAddress_v2('cuGreenCtxCreate', &__cuGreenCtxCreate, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGetExportTable"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetExportTable), 0xBB8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1553, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1555 + * _F_cuGetProcAddress_v2('cuGetExportTable', &__cuGetExportTable, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGreenCtxCreate + * _F_cuGetProcAddress_v2('cuGreenCtxCreate', &__cuGreenCtxCreate, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGreenCtxDestroy + * _F_cuGetProcAddress_v2('cuGreenCtxDestroy', &__cuGreenCtxDestroy, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGreenCtxCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxCreate), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1555, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1557 + * _F_cuGetProcAddress_v2('cuGreenCtxCreate', &__cuGreenCtxCreate, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGreenCtxDestroy + * _F_cuGetProcAddress_v2('cuGreenCtxDestroy', &__cuGreenCtxDestroy, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxFromGreenCtx + * _F_cuGetProcAddress_v2('cuCtxFromGreenCtx', &__cuCtxFromGreenCtx, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGreenCtxDestroy"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxDestroy), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1557, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1559 + * _F_cuGetProcAddress_v2('cuGreenCtxDestroy', &__cuGreenCtxDestroy, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxFromGreenCtx + * _F_cuGetProcAddress_v2('cuCtxFromGreenCtx', &__cuCtxFromGreenCtx, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDeviceGetDevResource + * _F_cuGetProcAddress_v2('cuDeviceGetDevResource', &__cuDeviceGetDevResource, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxFromGreenCtx"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxFromGreenCtx), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1559, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1561 + * _F_cuGetProcAddress_v2('cuCtxFromGreenCtx', &__cuCtxFromGreenCtx, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDeviceGetDevResource + * _F_cuGetProcAddress_v2('cuDeviceGetDevResource', &__cuDeviceGetDevResource, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCtxGetDevResource + * _F_cuGetProcAddress_v2('cuCtxGetDevResource', &__cuCtxGetDevResource, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDeviceGetDevResource"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDevResource), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1561, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1563 + * _F_cuGetProcAddress_v2('cuDeviceGetDevResource', &__cuDeviceGetDevResource, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCtxGetDevResource + * _F_cuGetProcAddress_v2('cuCtxGetDevResource', &__cuCtxGetDevResource, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGreenCtxGetDevResource + * _F_cuGetProcAddress_v2('cuGreenCtxGetDevResource', &__cuGreenCtxGetDevResource, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCtxGetDevResource"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevResource), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1563, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1565 + * _F_cuGetProcAddress_v2('cuCtxGetDevResource', &__cuCtxGetDevResource, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGreenCtxGetDevResource + * _F_cuGetProcAddress_v2('cuGreenCtxGetDevResource', &__cuGreenCtxGetDevResource, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDevSmResourceSplitByCount + * _F_cuGetProcAddress_v2('cuDevSmResourceSplitByCount', &__cuDevSmResourceSplitByCount, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGreenCtxGetDevResource"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxGetDevResource), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1565, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1567 + * _F_cuGetProcAddress_v2('cuGreenCtxGetDevResource', &__cuGreenCtxGetDevResource, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDevSmResourceSplitByCount + * _F_cuGetProcAddress_v2('cuDevSmResourceSplitByCount', &__cuDevSmResourceSplitByCount, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuDevResourceGenerateDesc + * _F_cuGetProcAddress_v2('cuDevResourceGenerateDesc', &__cuDevResourceGenerateDesc, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDevSmResourceSplitByCount"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevSmResourceSplitByCount), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1567, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1569 + * _F_cuGetProcAddress_v2('cuDevSmResourceSplitByCount', &__cuDevSmResourceSplitByCount, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuDevResourceGenerateDesc + * _F_cuGetProcAddress_v2('cuDevResourceGenerateDesc', &__cuDevResourceGenerateDesc, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGreenCtxRecordEvent + * _F_cuGetProcAddress_v2('cuGreenCtxRecordEvent', &__cuGreenCtxRecordEvent, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuDevResourceGenerateDesc"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevResourceGenerateDesc), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1569, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1571 + * _F_cuGetProcAddress_v2('cuDevResourceGenerateDesc', &__cuDevResourceGenerateDesc, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGreenCtxRecordEvent + * _F_cuGetProcAddress_v2('cuGreenCtxRecordEvent', &__cuGreenCtxRecordEvent, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGreenCtxWaitEvent + * _F_cuGetProcAddress_v2('cuGreenCtxWaitEvent', &__cuGreenCtxWaitEvent, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGreenCtxRecordEvent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxRecordEvent), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1571, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1573 + * _F_cuGetProcAddress_v2('cuGreenCtxRecordEvent', &__cuGreenCtxRecordEvent, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGreenCtxWaitEvent + * _F_cuGetProcAddress_v2('cuGreenCtxWaitEvent', &__cuGreenCtxWaitEvent, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuStreamGetGreenCtx + * _F_cuGetProcAddress_v2('cuStreamGetGreenCtx', &__cuStreamGetGreenCtx, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGreenCtxWaitEvent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxWaitEvent), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1573, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1575 + * _F_cuGetProcAddress_v2('cuGreenCtxWaitEvent', &__cuGreenCtxWaitEvent, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuStreamGetGreenCtx + * _F_cuGetProcAddress_v2('cuStreamGetGreenCtx', &__cuStreamGetGreenCtx, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGreenCtxStreamCreate + * _F_cuGetProcAddress_v2('cuGreenCtxStreamCreate', &__cuGreenCtxStreamCreate, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuStreamGetGreenCtx"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetGreenCtx), 0x2F08, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1575, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1577 + * _F_cuGetProcAddress_v2('cuStreamGetGreenCtx', &__cuStreamGetGreenCtx, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGreenCtxStreamCreate + * _F_cuGetProcAddress_v2('cuGreenCtxStreamCreate', &__cuGreenCtxStreamCreate, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLogsRegisterCallback + * _F_cuGetProcAddress_v2('cuLogsRegisterCallback', &__cuLogsRegisterCallback, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGreenCtxStreamCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxStreamCreate), 0x2F12, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1577, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1579 + * _F_cuGetProcAddress_v2('cuGreenCtxStreamCreate', &__cuGreenCtxStreamCreate, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLogsRegisterCallback + * _F_cuGetProcAddress_v2('cuLogsRegisterCallback', &__cuLogsRegisterCallback, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLogsUnregisterCallback + * _F_cuGetProcAddress_v2('cuLogsUnregisterCallback', &__cuLogsUnregisterCallback, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLogsRegisterCallback"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsRegisterCallback), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1579, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1581 + * _F_cuGetProcAddress_v2('cuLogsRegisterCallback', &__cuLogsRegisterCallback, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLogsUnregisterCallback + * _F_cuGetProcAddress_v2('cuLogsUnregisterCallback', &__cuLogsUnregisterCallback, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLogsCurrent + * _F_cuGetProcAddress_v2('cuLogsCurrent', &__cuLogsCurrent, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLogsUnregisterCallback"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsUnregisterCallback), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1581, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1583 + * _F_cuGetProcAddress_v2('cuLogsUnregisterCallback', &__cuLogsUnregisterCallback, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLogsCurrent + * _F_cuGetProcAddress_v2('cuLogsCurrent', &__cuLogsCurrent, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLogsDumpToFile + * _F_cuGetProcAddress_v2('cuLogsDumpToFile', &__cuLogsDumpToFile, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLogsCurrent"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsCurrent), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1583, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1585 + * _F_cuGetProcAddress_v2('cuLogsCurrent', &__cuLogsCurrent, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLogsDumpToFile + * _F_cuGetProcAddress_v2('cuLogsDumpToFile', &__cuLogsDumpToFile, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuLogsDumpToMemory + * _F_cuGetProcAddress_v2('cuLogsDumpToMemory', &__cuLogsDumpToMemory, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLogsDumpToFile"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToFile), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1585, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1587 + * _F_cuGetProcAddress_v2('cuLogsDumpToFile', &__cuLogsDumpToFile, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuLogsDumpToMemory + * _F_cuGetProcAddress_v2('cuLogsDumpToMemory', &__cuLogsDumpToMemory, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCheckpointProcessGetRestoreThreadId + * _F_cuGetProcAddress_v2('cuCheckpointProcessGetRestoreThreadId', &__cuCheckpointProcessGetRestoreThreadId, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuLogsDumpToMemory"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToMemory), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1587, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1589 + * _F_cuGetProcAddress_v2('cuLogsDumpToMemory', &__cuLogsDumpToMemory, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCheckpointProcessGetRestoreThreadId + * _F_cuGetProcAddress_v2('cuCheckpointProcessGetRestoreThreadId', &__cuCheckpointProcessGetRestoreThreadId, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCheckpointProcessGetState + * _F_cuGetProcAddress_v2('cuCheckpointProcessGetState', &__cuCheckpointProcessGetState, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCheckpointProcessGetRestoreThreadId"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetRestoreThreadId), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1589, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1591 + * _F_cuGetProcAddress_v2('cuCheckpointProcessGetRestoreThreadId', &__cuCheckpointProcessGetRestoreThreadId, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCheckpointProcessGetState + * _F_cuGetProcAddress_v2('cuCheckpointProcessGetState', &__cuCheckpointProcessGetState, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCheckpointProcessLock + * _F_cuGetProcAddress_v2('cuCheckpointProcessLock', &__cuCheckpointProcessLock, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCheckpointProcessGetState"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetState), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1591, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1593 + * _F_cuGetProcAddress_v2('cuCheckpointProcessGetState', &__cuCheckpointProcessGetState, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCheckpointProcessLock + * _F_cuGetProcAddress_v2('cuCheckpointProcessLock', &__cuCheckpointProcessLock, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCheckpointProcessCheckpoint + * _F_cuGetProcAddress_v2('cuCheckpointProcessCheckpoint', &__cuCheckpointProcessCheckpoint, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCheckpointProcessLock"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessLock), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1593, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1595 + * _F_cuGetProcAddress_v2('cuCheckpointProcessLock', &__cuCheckpointProcessLock, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCheckpointProcessCheckpoint + * _F_cuGetProcAddress_v2('cuCheckpointProcessCheckpoint', &__cuCheckpointProcessCheckpoint, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCheckpointProcessRestore + * _F_cuGetProcAddress_v2('cuCheckpointProcessRestore', &__cuCheckpointProcessRestore, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCheckpointProcessCheckpoint"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessCheckpoint), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1595, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1597 + * _F_cuGetProcAddress_v2('cuCheckpointProcessCheckpoint', &__cuCheckpointProcessCheckpoint, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCheckpointProcessRestore + * _F_cuGetProcAddress_v2('cuCheckpointProcessRestore', &__cuCheckpointProcessRestore, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuCheckpointProcessUnlock + * _F_cuGetProcAddress_v2('cuCheckpointProcessUnlock', &__cuCheckpointProcessUnlock, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCheckpointProcessRestore"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessRestore), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1597, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1599 + * _F_cuGetProcAddress_v2('cuCheckpointProcessRestore', &__cuCheckpointProcessRestore, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuCheckpointProcessUnlock + * _F_cuGetProcAddress_v2('cuCheckpointProcessUnlock', &__cuCheckpointProcessUnlock, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuProfilerStart + * _F_cuGetProcAddress_v2('cuProfilerStart', &__cuProfilerStart, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuCheckpointProcessUnlock"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessUnlock), 0x2F30, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1599, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1601 + * _F_cuGetProcAddress_v2('cuCheckpointProcessUnlock', &__cuCheckpointProcessUnlock, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuProfilerStart + * _F_cuGetProcAddress_v2('cuProfilerStart', &__cuProfilerStart, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuProfilerStop + * _F_cuGetProcAddress_v2('cuProfilerStop', &__cuProfilerStop, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuProfilerStart"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStart), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1601, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1603 + * _F_cuGetProcAddress_v2('cuProfilerStart', &__cuProfilerStart, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuProfilerStop + * _F_cuGetProcAddress_v2('cuProfilerStop', &__cuProfilerStop, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsEGLRegisterImage + * _F_cuGetProcAddress_v2('cuGraphicsEGLRegisterImage', &__cuGraphicsEGLRegisterImage, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuProfilerStop"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStop), 0xFA0, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1603, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1605 + * _F_cuGetProcAddress_v2('cuProfilerStop', &__cuProfilerStop, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphicsEGLRegisterImage + * _F_cuGetProcAddress_v2('cuGraphicsEGLRegisterImage', &__cuGraphicsEGLRegisterImage, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerConnect + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerConnect', &__cuEGLStreamConsumerConnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsEGLRegisterImage"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsEGLRegisterImage), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1605, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1607 + * _F_cuGetProcAddress_v2('cuGraphicsEGLRegisterImage', &__cuGraphicsEGLRegisterImage, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEGLStreamConsumerConnect + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerConnect', &__cuEGLStreamConsumerConnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerConnectWithFlags + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerConnectWithFlags', &__cuEGLStreamConsumerConnectWithFlags, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEGLStreamConsumerConnect"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnect), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1607, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1609 + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerConnect', &__cuEGLStreamConsumerConnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEGLStreamConsumerConnectWithFlags + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerConnectWithFlags', &__cuEGLStreamConsumerConnectWithFlags, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerDisconnect + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerDisconnect', &__cuEGLStreamConsumerDisconnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEGLStreamConsumerConnectWithFlags"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnectWithFlags), 0x1F40, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1609, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1611 + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerConnectWithFlags', &__cuEGLStreamConsumerConnectWithFlags, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEGLStreamConsumerDisconnect + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerDisconnect', &__cuEGLStreamConsumerDisconnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerAcquireFrame + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerAcquireFrame', &__cuEGLStreamConsumerAcquireFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEGLStreamConsumerDisconnect"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerDisconnect), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1611, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1613 + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerDisconnect', &__cuEGLStreamConsumerDisconnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEGLStreamConsumerAcquireFrame + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerAcquireFrame', &__cuEGLStreamConsumerAcquireFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerReleaseFrame + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerReleaseFrame', &__cuEGLStreamConsumerReleaseFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEGLStreamConsumerAcquireFrame"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerAcquireFrame), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1613, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1615 + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerAcquireFrame', &__cuEGLStreamConsumerAcquireFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEGLStreamConsumerReleaseFrame + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerReleaseFrame', &__cuEGLStreamConsumerReleaseFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerConnect + * _F_cuGetProcAddress_v2('cuEGLStreamProducerConnect', &__cuEGLStreamProducerConnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEGLStreamConsumerReleaseFrame"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerReleaseFrame), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1615, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1617 + * _F_cuGetProcAddress_v2('cuEGLStreamConsumerReleaseFrame', &__cuEGLStreamConsumerReleaseFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEGLStreamProducerConnect + * _F_cuGetProcAddress_v2('cuEGLStreamProducerConnect', &__cuEGLStreamProducerConnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerDisconnect + * _F_cuGetProcAddress_v2('cuEGLStreamProducerDisconnect', &__cuEGLStreamProducerDisconnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEGLStreamProducerConnect"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerConnect), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1617, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1619 + * _F_cuGetProcAddress_v2('cuEGLStreamProducerConnect', &__cuEGLStreamProducerConnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEGLStreamProducerDisconnect + * _F_cuGetProcAddress_v2('cuEGLStreamProducerDisconnect', &__cuEGLStreamProducerDisconnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerPresentFrame + * _F_cuGetProcAddress_v2('cuEGLStreamProducerPresentFrame', &__cuEGLStreamProducerPresentFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEGLStreamProducerDisconnect"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerDisconnect), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1619, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1621 + * _F_cuGetProcAddress_v2('cuEGLStreamProducerDisconnect', &__cuEGLStreamProducerDisconnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEGLStreamProducerPresentFrame + * _F_cuGetProcAddress_v2('cuEGLStreamProducerPresentFrame', &__cuEGLStreamProducerPresentFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerReturnFrame + * _F_cuGetProcAddress_v2('cuEGLStreamProducerReturnFrame', &__cuEGLStreamProducerReturnFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEGLStreamProducerPresentFrame"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerPresentFrame), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1621, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1623 + * _F_cuGetProcAddress_v2('cuEGLStreamProducerPresentFrame', &__cuEGLStreamProducerPresentFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEGLStreamProducerReturnFrame + * _F_cuGetProcAddress_v2('cuEGLStreamProducerReturnFrame', &__cuEGLStreamProducerReturnFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedEglFrame + * _F_cuGetProcAddress_v2('cuGraphicsResourceGetMappedEglFrame', &__cuGraphicsResourceGetMappedEglFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEGLStreamProducerReturnFrame"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerReturnFrame), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1623, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1625 + * _F_cuGetProcAddress_v2('cuEGLStreamProducerReturnFrame', &__cuEGLStreamProducerReturnFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphicsResourceGetMappedEglFrame + * _F_cuGetProcAddress_v2('cuGraphicsResourceGetMappedEglFrame', &__cuGraphicsResourceGetMappedEglFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuEventCreateFromEGLSync + * _F_cuGetProcAddress_v2('cuEventCreateFromEGLSync', &__cuEventCreateFromEGLSync, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsResourceGetMappedEglFrame"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedEglFrame), 0x1B58, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1625, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1627 + * _F_cuGetProcAddress_v2('cuGraphicsResourceGetMappedEglFrame', &__cuGraphicsResourceGetMappedEglFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuEventCreateFromEGLSync + * _F_cuGetProcAddress_v2('cuEventCreateFromEGLSync', &__cuEventCreateFromEGLSync, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsGLRegisterBuffer + * _F_cuGetProcAddress_v2('cuGraphicsGLRegisterBuffer', &__cuGraphicsGLRegisterBuffer, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuEventCreateFromEGLSync"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreateFromEGLSync), 0x2328, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1627, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1629 + * _F_cuGetProcAddress_v2('cuEventCreateFromEGLSync', &__cuEventCreateFromEGLSync, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphicsGLRegisterBuffer + * _F_cuGetProcAddress_v2('cuGraphicsGLRegisterBuffer', &__cuGraphicsGLRegisterBuffer, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsGLRegisterImage + * _F_cuGetProcAddress_v2('cuGraphicsGLRegisterImage', &__cuGraphicsGLRegisterImage, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsGLRegisterBuffer"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterBuffer), 0xBB8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1629, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1631 + * _F_cuGetProcAddress_v2('cuGraphicsGLRegisterBuffer', &__cuGraphicsGLRegisterBuffer, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphicsGLRegisterImage + * _F_cuGetProcAddress_v2('cuGraphicsGLRegisterImage', &__cuGraphicsGLRegisterImage, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGLGetDevices_v2 + * _F_cuGetProcAddress_v2('cuGLGetDevices', &__cuGLGetDevices_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsGLRegisterImage"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterImage), 0xBB8, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1631, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1633 + * _F_cuGetProcAddress_v2('cuGraphicsGLRegisterImage', &__cuGraphicsGLRegisterImage, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGLGetDevices_v2 + * _F_cuGetProcAddress_v2('cuGLGetDevices', &__cuGLGetDevices_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuVDPAUGetDevice + * _F_cuGetProcAddress_v2('cuVDPAUGetDevice', &__cuVDPAUGetDevice, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGLGetDevices"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGLGetDevices_v2), 0x17A2, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1633, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1635 + * _F_cuGetProcAddress_v2('cuGLGetDevices', &__cuGLGetDevices_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuVDPAUGetDevice + * _F_cuGetProcAddress_v2('cuVDPAUGetDevice', &__cuVDPAUGetDevice, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuVDPAUCtxCreate_v2 + * _F_cuGetProcAddress_v2('cuVDPAUCtxCreate', &__cuVDPAUCtxCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuVDPAUGetDevice"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUGetDevice), 0xBC2, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1635, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1637 + * _F_cuGetProcAddress_v2('cuVDPAUGetDevice', &__cuVDPAUGetDevice, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuVDPAUCtxCreate_v2 + * _F_cuGetProcAddress_v2('cuVDPAUCtxCreate', &__cuVDPAUCtxCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsVDPAURegisterVideoSurface + * _F_cuGetProcAddress_v2('cuGraphicsVDPAURegisterVideoSurface', &__cuGraphicsVDPAURegisterVideoSurface, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuVDPAUCtxCreate"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUCtxCreate_v2), 0xBCC, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1637, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1639 + * _F_cuGetProcAddress_v2('cuVDPAUCtxCreate', &__cuVDPAUCtxCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphicsVDPAURegisterVideoSurface + * _F_cuGetProcAddress_v2('cuGraphicsVDPAURegisterVideoSurface', &__cuGraphicsVDPAURegisterVideoSurface, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * global __cuGraphicsVDPAURegisterOutputSurface + * _F_cuGetProcAddress_v2('cuGraphicsVDPAURegisterOutputSurface', &__cuGraphicsVDPAURegisterOutputSurface, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsVDPAURegisterVideoSurface"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterVideoSurface), 0xBC2, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1639, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1641 + * _F_cuGetProcAddress_v2('cuGraphicsVDPAURegisterVideoSurface', &__cuGraphicsVDPAURegisterVideoSurface, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * global __cuGraphicsVDPAURegisterOutputSurface + * _F_cuGetProcAddress_v2('cuGraphicsVDPAURegisterOutputSurface', &__cuGraphicsVDPAURegisterOutputSurface, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) # <<<<<<<<<<<<<< + * __cuPythonInit = True + * return 0 +*/ + __pyx_t_14 = __pyx_v__F_cuGetProcAddress_v2(((char const *)"cuGraphicsVDPAURegisterOutputSurface"), (&__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterOutputSurface), 0xBC2, CU_GET_PROC_ADDRESS_DEFAULT, NULL); if (unlikely(__pyx_t_14 == ((CUresult)CUDA_ERROR_NOT_FOUND) && PyErr_Occurred())) __PYX_ERR(0, 1641, __pyx_L10_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":1642 + * global __cuGraphicsVDPAURegisterOutputSurface + * _F_cuGetProcAddress_v2('cuGraphicsVDPAURegisterOutputSurface', &__cuGraphicsVDPAURegisterOutputSurface, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * __cuPythonInit = True # <<<<<<<<<<<<<< + * return 0 + * +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPythonInit = 1; + + /* "cuda/bindings/_bindings/cydriver.pyx":1643 + * _F_cuGetProcAddress_v2('cuGraphicsVDPAURegisterOutputSurface', &__cuGraphicsVDPAURegisterOutputSurface, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + * __cuPythonInit = True + * return 0 # <<<<<<<<<<<<<< + * + * # Load using dlsym +*/ + __pyx_r = 0; + goto __pyx_L14_try_return; + + /* "cuda/bindings/_bindings/cydriver.pyx":517 + * + * # Load using cuGetProcAddress if available + * if __cuGetProcAddress_v2 != NULL: # <<<<<<<<<<<<<< + * _F_cuGetProcAddress_v2 = <__cuGetProcAddress_v2_T>__cuGetProcAddress_v2 + * if usePTDS: +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":1646 + * + * # Load using dlsym + * if usePTDS: # <<<<<<<<<<<<<< + * # Get all PTDS version of functions + * pass +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cydriver.pyx":1650 + * pass + * global __cuMemcpy + * __cuMemcpy = dlfcn.dlsym(handle, 'cuMemcpy_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpyPeer + * __cuMemcpyPeer = dlfcn.dlsym(handle, 'cuMemcpyPeer_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1652 + * __cuMemcpy = dlfcn.dlsym(handle, 'cuMemcpy_ptds') + * global __cuMemcpyPeer + * __cuMemcpyPeer = dlfcn.dlsym(handle, 'cuMemcpyPeer_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpyHtoD_v2 + * __cuMemcpyHtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoD_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeer = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyPeer_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1654 + * __cuMemcpyPeer = dlfcn.dlsym(handle, 'cuMemcpyPeer_ptds') + * global __cuMemcpyHtoD_v2 + * __cuMemcpyHtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoD_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpyDtoH_v2 + * __cuMemcpyDtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoH_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoD_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyHtoD_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1656 + * __cuMemcpyHtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoD_v2_ptds') + * global __cuMemcpyDtoH_v2 + * __cuMemcpyDtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoH_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpyDtoD_v2 + * __cuMemcpyDtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoD_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoH_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyDtoH_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1658 + * __cuMemcpyDtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoH_v2_ptds') + * global __cuMemcpyDtoD_v2 + * __cuMemcpyDtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoD_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpyDtoA_v2 + * __cuMemcpyDtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoA_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoD_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyDtoD_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1660 + * __cuMemcpyDtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoD_v2_ptds') + * global __cuMemcpyDtoA_v2 + * __cuMemcpyDtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoA_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpyAtoD_v2 + * __cuMemcpyAtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoD_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoA_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyDtoA_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1662 + * __cuMemcpyDtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoA_v2_ptds') + * global __cuMemcpyAtoD_v2 + * __cuMemcpyAtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoD_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpyHtoA_v2 + * __cuMemcpyHtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoA_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoD_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyAtoD_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1664 + * __cuMemcpyAtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoD_v2_ptds') + * global __cuMemcpyHtoA_v2 + * __cuMemcpyHtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoA_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpyAtoH_v2 + * __cuMemcpyAtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoH_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoA_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyHtoA_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1666 + * __cuMemcpyHtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoA_v2_ptds') + * global __cuMemcpyAtoH_v2 + * __cuMemcpyAtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoH_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpyAtoA_v2 + * __cuMemcpyAtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoA_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoH_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyAtoH_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1668 + * __cuMemcpyAtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoH_v2_ptds') + * global __cuMemcpyAtoA_v2 + * __cuMemcpyAtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoA_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpy2D_v2 + * __cuMemcpy2D_v2 = dlfcn.dlsym(handle, 'cuMemcpy2D_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoA_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyAtoA_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1670 + * __cuMemcpyAtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoA_v2_ptds') + * global __cuMemcpy2D_v2 + * __cuMemcpy2D_v2 = dlfcn.dlsym(handle, 'cuMemcpy2D_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpy2DUnaligned_v2 + * __cuMemcpy2DUnaligned_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DUnaligned_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2D_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy2D_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1672 + * __cuMemcpy2D_v2 = dlfcn.dlsym(handle, 'cuMemcpy2D_v2_ptds') + * global __cuMemcpy2DUnaligned_v2 + * __cuMemcpy2DUnaligned_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DUnaligned_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpy3D_v2 + * __cuMemcpy3D_v2 = dlfcn.dlsym(handle, 'cuMemcpy3D_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DUnaligned_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy2DUnaligned_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1674 + * __cuMemcpy2DUnaligned_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DUnaligned_v2_ptds') + * global __cuMemcpy3D_v2 + * __cuMemcpy3D_v2 = dlfcn.dlsym(handle, 'cuMemcpy3D_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeer + * __cuMemcpy3DPeer = dlfcn.dlsym(handle, 'cuMemcpy3DPeer_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3D_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy3D_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1676 + * __cuMemcpy3D_v2 = dlfcn.dlsym(handle, 'cuMemcpy3D_v2_ptds') + * global __cuMemcpy3DPeer + * __cuMemcpy3DPeer = dlfcn.dlsym(handle, 'cuMemcpy3DPeer_ptds') # <<<<<<<<<<<<<< + * global __cuMemcpyAsync + * __cuMemcpyAsync = dlfcn.dlsym(handle, 'cuMemcpyAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeer = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy3DPeer_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1678 + * __cuMemcpy3DPeer = dlfcn.dlsym(handle, 'cuMemcpy3DPeer_ptds') + * global __cuMemcpyAsync + * __cuMemcpyAsync = dlfcn.dlsym(handle, 'cuMemcpyAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuMemcpyPeerAsync + * __cuMemcpyPeerAsync = dlfcn.dlsym(handle, 'cuMemcpyPeerAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1680 + * __cuMemcpyAsync = dlfcn.dlsym(handle, 'cuMemcpyAsync_ptsz') + * global __cuMemcpyPeerAsync + * __cuMemcpyPeerAsync = dlfcn.dlsym(handle, 'cuMemcpyPeerAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuMemcpyHtoDAsync_v2 + * __cuMemcpyHtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoDAsync_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeerAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyPeerAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1682 + * __cuMemcpyPeerAsync = dlfcn.dlsym(handle, 'cuMemcpyPeerAsync_ptsz') + * global __cuMemcpyHtoDAsync_v2 + * __cuMemcpyHtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoDAsync_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuMemcpyDtoHAsync_v2 + * __cuMemcpyDtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoHAsync_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoDAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyHtoDAsync_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1684 + * __cuMemcpyHtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoDAsync_v2_ptsz') + * global __cuMemcpyDtoHAsync_v2 + * __cuMemcpyDtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoHAsync_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuMemcpyDtoDAsync_v2 + * __cuMemcpyDtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoDAsync_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoHAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyDtoHAsync_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1686 + * __cuMemcpyDtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoHAsync_v2_ptsz') + * global __cuMemcpyDtoDAsync_v2 + * __cuMemcpyDtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoDAsync_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuMemcpyHtoAAsync_v2 + * __cuMemcpyHtoAAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoAAsync_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoDAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyDtoDAsync_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1688 + * __cuMemcpyDtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoDAsync_v2_ptsz') + * global __cuMemcpyHtoAAsync_v2 + * __cuMemcpyHtoAAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoAAsync_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuMemcpyAtoHAsync_v2 + * __cuMemcpyAtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoHAsync_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoAAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyHtoAAsync_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1690 + * __cuMemcpyHtoAAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoAAsync_v2_ptsz') + * global __cuMemcpyAtoHAsync_v2 + * __cuMemcpyAtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoHAsync_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuMemcpy2DAsync_v2 + * __cuMemcpy2DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DAsync_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoHAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyAtoHAsync_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1692 + * __cuMemcpyAtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoHAsync_v2_ptsz') + * global __cuMemcpy2DAsync_v2 + * __cuMemcpy2DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DAsync_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuMemcpy3DAsync_v2 + * __cuMemcpy3DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy3DAsync_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy2DAsync_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1694 + * __cuMemcpy2DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DAsync_v2_ptsz') + * global __cuMemcpy3DAsync_v2 + * __cuMemcpy3DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy3DAsync_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeerAsync + * __cuMemcpy3DPeerAsync = dlfcn.dlsym(handle, 'cuMemcpy3DPeerAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy3DAsync_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1696 + * __cuMemcpy3DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy3DAsync_v2_ptsz') + * global __cuMemcpy3DPeerAsync + * __cuMemcpy3DPeerAsync = dlfcn.dlsym(handle, 'cuMemcpy3DPeerAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuMemcpyBatchAsync + * __cuMemcpyBatchAsync = dlfcn.dlsym(handle, 'cuMemcpyBatchAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeerAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy3DPeerAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1698 + * __cuMemcpy3DPeerAsync = dlfcn.dlsym(handle, 'cuMemcpy3DPeerAsync_ptsz') + * global __cuMemcpyBatchAsync + * __cuMemcpyBatchAsync = dlfcn.dlsym(handle, 'cuMemcpyBatchAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuMemcpy3DBatchAsync + * __cuMemcpy3DBatchAsync = dlfcn.dlsym(handle, 'cuMemcpy3DBatchAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyBatchAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyBatchAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1700 + * __cuMemcpyBatchAsync = dlfcn.dlsym(handle, 'cuMemcpyBatchAsync_ptsz') + * global __cuMemcpy3DBatchAsync + * __cuMemcpy3DBatchAsync = dlfcn.dlsym(handle, 'cuMemcpy3DBatchAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuMemsetD8_v2 + * __cuMemsetD8_v2 = dlfcn.dlsym(handle, 'cuMemsetD8_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DBatchAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy3DBatchAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1702 + * __cuMemcpy3DBatchAsync = dlfcn.dlsym(handle, 'cuMemcpy3DBatchAsync_ptsz') + * global __cuMemsetD8_v2 + * __cuMemsetD8_v2 = dlfcn.dlsym(handle, 'cuMemsetD8_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemsetD16_v2 + * __cuMemsetD16_v2 = dlfcn.dlsym(handle, 'cuMemsetD16_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD8_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1704 + * __cuMemsetD8_v2 = dlfcn.dlsym(handle, 'cuMemsetD8_v2_ptds') + * global __cuMemsetD16_v2 + * __cuMemsetD16_v2 = dlfcn.dlsym(handle, 'cuMemsetD16_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemsetD32_v2 + * __cuMemsetD32_v2 = dlfcn.dlsym(handle, 'cuMemsetD32_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD16_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1706 + * __cuMemsetD16_v2 = dlfcn.dlsym(handle, 'cuMemsetD16_v2_ptds') + * global __cuMemsetD32_v2 + * __cuMemsetD32_v2 = dlfcn.dlsym(handle, 'cuMemsetD32_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemsetD2D8_v2 + * __cuMemsetD2D8_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D8_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD32_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1708 + * __cuMemsetD32_v2 = dlfcn.dlsym(handle, 'cuMemsetD32_v2_ptds') + * global __cuMemsetD2D8_v2 + * __cuMemsetD2D8_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D8_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemsetD2D16_v2 + * __cuMemsetD2D16_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D16_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD2D8_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1710 + * __cuMemsetD2D8_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D8_v2_ptds') + * global __cuMemsetD2D16_v2 + * __cuMemsetD2D16_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D16_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemsetD2D32_v2 + * __cuMemsetD2D32_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D32_v2_ptds') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD2D16_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1712 + * __cuMemsetD2D16_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D16_v2_ptds') + * global __cuMemsetD2D32_v2 + * __cuMemsetD2D32_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D32_v2_ptds') # <<<<<<<<<<<<<< + * global __cuMemsetD8Async + * __cuMemsetD8Async = dlfcn.dlsym(handle, 'cuMemsetD8Async_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD2D32_v2_ptds")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1714 + * __cuMemsetD2D32_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D32_v2_ptds') + * global __cuMemsetD8Async + * __cuMemsetD8Async = dlfcn.dlsym(handle, 'cuMemsetD8Async_ptsz') # <<<<<<<<<<<<<< + * global __cuMemsetD16Async + * __cuMemsetD16Async = dlfcn.dlsym(handle, 'cuMemsetD16Async_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8Async = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD8Async_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1716 + * __cuMemsetD8Async = dlfcn.dlsym(handle, 'cuMemsetD8Async_ptsz') + * global __cuMemsetD16Async + * __cuMemsetD16Async = dlfcn.dlsym(handle, 'cuMemsetD16Async_ptsz') # <<<<<<<<<<<<<< + * global __cuMemsetD32Async + * __cuMemsetD32Async = dlfcn.dlsym(handle, 'cuMemsetD32Async_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16Async = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD16Async_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1718 + * __cuMemsetD16Async = dlfcn.dlsym(handle, 'cuMemsetD16Async_ptsz') + * global __cuMemsetD32Async + * __cuMemsetD32Async = dlfcn.dlsym(handle, 'cuMemsetD32Async_ptsz') # <<<<<<<<<<<<<< + * global __cuMemsetD2D8Async + * __cuMemsetD2D8Async = dlfcn.dlsym(handle, 'cuMemsetD2D8Async_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32Async = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD32Async_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1720 + * __cuMemsetD32Async = dlfcn.dlsym(handle, 'cuMemsetD32Async_ptsz') + * global __cuMemsetD2D8Async + * __cuMemsetD2D8Async = dlfcn.dlsym(handle, 'cuMemsetD2D8Async_ptsz') # <<<<<<<<<<<<<< + * global __cuMemsetD2D16Async + * __cuMemsetD2D16Async = dlfcn.dlsym(handle, 'cuMemsetD2D16Async_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8Async = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD2D8Async_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1722 + * __cuMemsetD2D8Async = dlfcn.dlsym(handle, 'cuMemsetD2D8Async_ptsz') + * global __cuMemsetD2D16Async + * __cuMemsetD2D16Async = dlfcn.dlsym(handle, 'cuMemsetD2D16Async_ptsz') # <<<<<<<<<<<<<< + * global __cuMemsetD2D32Async + * __cuMemsetD2D32Async = dlfcn.dlsym(handle, 'cuMemsetD2D32Async_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16Async = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD2D16Async_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1724 + * __cuMemsetD2D16Async = dlfcn.dlsym(handle, 'cuMemsetD2D16Async_ptsz') + * global __cuMemsetD2D32Async + * __cuMemsetD2D32Async = dlfcn.dlsym(handle, 'cuMemsetD2D32Async_ptsz') # <<<<<<<<<<<<<< + * global __cuMemBatchDecompressAsync + * __cuMemBatchDecompressAsync = dlfcn.dlsym(handle, 'cuMemBatchDecompressAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32Async = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD2D32Async_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1726 + * __cuMemsetD2D32Async = dlfcn.dlsym(handle, 'cuMemsetD2D32Async_ptsz') + * global __cuMemBatchDecompressAsync + * __cuMemBatchDecompressAsync = dlfcn.dlsym(handle, 'cuMemBatchDecompressAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuMemMapArrayAsync + * __cuMemMapArrayAsync = dlfcn.dlsym(handle, 'cuMemMapArrayAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemBatchDecompressAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemBatchDecompressAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1728 + * __cuMemBatchDecompressAsync = dlfcn.dlsym(handle, 'cuMemBatchDecompressAsync_ptsz') + * global __cuMemMapArrayAsync + * __cuMemMapArrayAsync = dlfcn.dlsym(handle, 'cuMemMapArrayAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuMemFreeAsync + * __cuMemFreeAsync = dlfcn.dlsym(handle, 'cuMemFreeAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMapArrayAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemMapArrayAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1730 + * __cuMemMapArrayAsync = dlfcn.dlsym(handle, 'cuMemMapArrayAsync_ptsz') + * global __cuMemFreeAsync + * __cuMemFreeAsync = dlfcn.dlsym(handle, 'cuMemFreeAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuMemAllocAsync + * __cuMemAllocAsync = dlfcn.dlsym(handle, 'cuMemAllocAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemFreeAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1732 + * __cuMemFreeAsync = dlfcn.dlsym(handle, 'cuMemFreeAsync_ptsz') + * global __cuMemAllocAsync + * __cuMemAllocAsync = dlfcn.dlsym(handle, 'cuMemAllocAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuMemAllocFromPoolAsync + * __cuMemAllocFromPoolAsync = dlfcn.dlsym(handle, 'cuMemAllocFromPoolAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemAllocAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1734 + * __cuMemAllocAsync = dlfcn.dlsym(handle, 'cuMemAllocAsync_ptsz') + * global __cuMemAllocFromPoolAsync + * __cuMemAllocFromPoolAsync = dlfcn.dlsym(handle, 'cuMemAllocFromPoolAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync + * __cuMemPrefetchAsync = dlfcn.dlsym(handle, 'cuMemPrefetchAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocFromPoolAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemAllocFromPoolAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1736 + * __cuMemAllocFromPoolAsync = dlfcn.dlsym(handle, 'cuMemAllocFromPoolAsync_ptsz') + * global __cuMemPrefetchAsync + * __cuMemPrefetchAsync = dlfcn.dlsym(handle, 'cuMemPrefetchAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync_v2 + * __cuMemPrefetchAsync_v2 = dlfcn.dlsym(handle, 'cuMemPrefetchAsync_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemPrefetchAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1738 + * __cuMemPrefetchAsync = dlfcn.dlsym(handle, 'cuMemPrefetchAsync_ptsz') + * global __cuMemPrefetchAsync_v2 + * __cuMemPrefetchAsync_v2 = dlfcn.dlsym(handle, 'cuMemPrefetchAsync_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamGetPriority + * __cuStreamGetPriority = dlfcn.dlsym(handle, 'cuStreamGetPriority_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemPrefetchAsync_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1740 + * __cuMemPrefetchAsync_v2 = dlfcn.dlsym(handle, 'cuMemPrefetchAsync_v2_ptsz') + * global __cuStreamGetPriority + * __cuStreamGetPriority = dlfcn.dlsym(handle, 'cuStreamGetPriority_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamGetDevice + * __cuStreamGetDevice = dlfcn.dlsym(handle, 'cuStreamGetDevice_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetPriority = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetPriority_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1742 + * __cuStreamGetPriority = dlfcn.dlsym(handle, 'cuStreamGetPriority_ptsz') + * global __cuStreamGetDevice + * __cuStreamGetDevice = dlfcn.dlsym(handle, 'cuStreamGetDevice_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamGetFlags + * __cuStreamGetFlags = dlfcn.dlsym(handle, 'cuStreamGetFlags_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetDevice = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetDevice_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1744 + * __cuStreamGetDevice = dlfcn.dlsym(handle, 'cuStreamGetDevice_ptsz') + * global __cuStreamGetFlags + * __cuStreamGetFlags = dlfcn.dlsym(handle, 'cuStreamGetFlags_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamGetId + * __cuStreamGetId = dlfcn.dlsym(handle, 'cuStreamGetId_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetFlags = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetFlags_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1746 + * __cuStreamGetFlags = dlfcn.dlsym(handle, 'cuStreamGetFlags_ptsz') + * global __cuStreamGetId + * __cuStreamGetId = dlfcn.dlsym(handle, 'cuStreamGetId_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamGetCtx + * __cuStreamGetCtx = dlfcn.dlsym(handle, 'cuStreamGetCtx_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetId = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetId_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1748 + * __cuStreamGetId = dlfcn.dlsym(handle, 'cuStreamGetId_ptsz') + * global __cuStreamGetCtx + * __cuStreamGetCtx = dlfcn.dlsym(handle, 'cuStreamGetCtx_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamGetCtx_v2 + * __cuStreamGetCtx_v2 = dlfcn.dlsym(handle, 'cuStreamGetCtx_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetCtx_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1750 + * __cuStreamGetCtx = dlfcn.dlsym(handle, 'cuStreamGetCtx_ptsz') + * global __cuStreamGetCtx_v2 + * __cuStreamGetCtx_v2 = dlfcn.dlsym(handle, 'cuStreamGetCtx_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamWaitEvent + * __cuStreamWaitEvent = dlfcn.dlsym(handle, 'cuStreamWaitEvent_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetCtx_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1752 + * __cuStreamGetCtx_v2 = dlfcn.dlsym(handle, 'cuStreamGetCtx_v2_ptsz') + * global __cuStreamWaitEvent + * __cuStreamWaitEvent = dlfcn.dlsym(handle, 'cuStreamWaitEvent_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamAddCallback + * __cuStreamAddCallback = dlfcn.dlsym(handle, 'cuStreamAddCallback_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitEvent = dlsym(__pyx_v_handle, ((char const *)"cuStreamWaitEvent_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1754 + * __cuStreamWaitEvent = dlfcn.dlsym(handle, 'cuStreamWaitEvent_ptsz') + * global __cuStreamAddCallback + * __cuStreamAddCallback = dlfcn.dlsym(handle, 'cuStreamAddCallback_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamBeginCapture_v2 + * __cuStreamBeginCapture_v2 = dlfcn.dlsym(handle, 'cuStreamBeginCapture_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAddCallback = dlsym(__pyx_v_handle, ((char const *)"cuStreamAddCallback_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1756 + * __cuStreamAddCallback = dlfcn.dlsym(handle, 'cuStreamAddCallback_ptsz') + * global __cuStreamBeginCapture_v2 + * __cuStreamBeginCapture_v2 = dlfcn.dlsym(handle, 'cuStreamBeginCapture_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamBeginCaptureToGraph + * __cuStreamBeginCaptureToGraph = dlfcn.dlsym(handle, 'cuStreamBeginCaptureToGraph_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCapture_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamBeginCapture_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1758 + * __cuStreamBeginCapture_v2 = dlfcn.dlsym(handle, 'cuStreamBeginCapture_v2_ptsz') + * global __cuStreamBeginCaptureToGraph + * __cuStreamBeginCaptureToGraph = dlfcn.dlsym(handle, 'cuStreamBeginCaptureToGraph_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamEndCapture + * __cuStreamEndCapture = dlfcn.dlsym(handle, 'cuStreamEndCapture_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCaptureToGraph = dlsym(__pyx_v_handle, ((char const *)"cuStreamBeginCaptureToGraph_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1760 + * __cuStreamBeginCaptureToGraph = dlfcn.dlsym(handle, 'cuStreamBeginCaptureToGraph_ptsz') + * global __cuStreamEndCapture + * __cuStreamEndCapture = dlfcn.dlsym(handle, 'cuStreamEndCapture_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamIsCapturing + * __cuStreamIsCapturing = dlfcn.dlsym(handle, 'cuStreamIsCapturing_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamEndCapture = dlsym(__pyx_v_handle, ((char const *)"cuStreamEndCapture_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1762 + * __cuStreamEndCapture = dlfcn.dlsym(handle, 'cuStreamEndCapture_ptsz') + * global __cuStreamIsCapturing + * __cuStreamIsCapturing = dlfcn.dlsym(handle, 'cuStreamIsCapturing_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v2 + * __cuStreamGetCaptureInfo_v2 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamIsCapturing = dlsym(__pyx_v_handle, ((char const *)"cuStreamIsCapturing_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1764 + * __cuStreamIsCapturing = dlfcn.dlsym(handle, 'cuStreamIsCapturing_ptsz') + * global __cuStreamGetCaptureInfo_v2 + * __cuStreamGetCaptureInfo_v2 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v3 + * __cuStreamGetCaptureInfo_v3 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v3_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetCaptureInfo_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1766 + * __cuStreamGetCaptureInfo_v2 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v2_ptsz') + * global __cuStreamGetCaptureInfo_v3 + * __cuStreamGetCaptureInfo_v3 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v3_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies + * __cuStreamUpdateCaptureDependencies = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v3 = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetCaptureInfo_v3_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1768 + * __cuStreamGetCaptureInfo_v3 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v3_ptsz') + * global __cuStreamUpdateCaptureDependencies + * __cuStreamUpdateCaptureDependencies = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies_v2 + * __cuStreamUpdateCaptureDependencies_v2 = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies = dlsym(__pyx_v_handle, ((char const *)"cuStreamUpdateCaptureDependencies_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1770 + * __cuStreamUpdateCaptureDependencies = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies_ptsz') + * global __cuStreamUpdateCaptureDependencies_v2 + * __cuStreamUpdateCaptureDependencies_v2 = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamAttachMemAsync + * __cuStreamAttachMemAsync = dlfcn.dlsym(handle, 'cuStreamAttachMemAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamUpdateCaptureDependencies_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1772 + * __cuStreamUpdateCaptureDependencies_v2 = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies_v2_ptsz') + * global __cuStreamAttachMemAsync + * __cuStreamAttachMemAsync = dlfcn.dlsym(handle, 'cuStreamAttachMemAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamQuery + * __cuStreamQuery = dlfcn.dlsym(handle, 'cuStreamQuery_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAttachMemAsync = dlsym(__pyx_v_handle, ((char const *)"cuStreamAttachMemAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1774 + * __cuStreamAttachMemAsync = dlfcn.dlsym(handle, 'cuStreamAttachMemAsync_ptsz') + * global __cuStreamQuery + * __cuStreamQuery = dlfcn.dlsym(handle, 'cuStreamQuery_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamSynchronize + * __cuStreamSynchronize = dlfcn.dlsym(handle, 'cuStreamSynchronize_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamQuery = dlsym(__pyx_v_handle, ((char const *)"cuStreamQuery_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1776 + * __cuStreamQuery = dlfcn.dlsym(handle, 'cuStreamQuery_ptsz') + * global __cuStreamSynchronize + * __cuStreamSynchronize = dlfcn.dlsym(handle, 'cuStreamSynchronize_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamCopyAttributes + * __cuStreamCopyAttributes = dlfcn.dlsym(handle, 'cuStreamCopyAttributes_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSynchronize = dlsym(__pyx_v_handle, ((char const *)"cuStreamSynchronize_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1778 + * __cuStreamSynchronize = dlfcn.dlsym(handle, 'cuStreamSynchronize_ptsz') + * global __cuStreamCopyAttributes + * __cuStreamCopyAttributes = dlfcn.dlsym(handle, 'cuStreamCopyAttributes_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamGetAttribute + * __cuStreamGetAttribute = dlfcn.dlsym(handle, 'cuStreamGetAttribute_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCopyAttributes = dlsym(__pyx_v_handle, ((char const *)"cuStreamCopyAttributes_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1780 + * __cuStreamCopyAttributes = dlfcn.dlsym(handle, 'cuStreamCopyAttributes_ptsz') + * global __cuStreamGetAttribute + * __cuStreamGetAttribute = dlfcn.dlsym(handle, 'cuStreamGetAttribute_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamSetAttribute + * __cuStreamSetAttribute = dlfcn.dlsym(handle, 'cuStreamSetAttribute_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetAttribute_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1782 + * __cuStreamGetAttribute = dlfcn.dlsym(handle, 'cuStreamGetAttribute_ptsz') + * global __cuStreamSetAttribute + * __cuStreamSetAttribute = dlfcn.dlsym(handle, 'cuStreamSetAttribute_ptsz') # <<<<<<<<<<<<<< + * global __cuEventRecord + * __cuEventRecord = dlfcn.dlsym(handle, 'cuEventRecord_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuStreamSetAttribute_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1784 + * __cuStreamSetAttribute = dlfcn.dlsym(handle, 'cuStreamSetAttribute_ptsz') + * global __cuEventRecord + * __cuEventRecord = dlfcn.dlsym(handle, 'cuEventRecord_ptsz') # <<<<<<<<<<<<<< + * global __cuEventRecordWithFlags + * __cuEventRecordWithFlags = dlfcn.dlsym(handle, 'cuEventRecordWithFlags_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecord = dlsym(__pyx_v_handle, ((char const *)"cuEventRecord_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1786 + * __cuEventRecord = dlfcn.dlsym(handle, 'cuEventRecord_ptsz') + * global __cuEventRecordWithFlags + * __cuEventRecordWithFlags = dlfcn.dlsym(handle, 'cuEventRecordWithFlags_ptsz') # <<<<<<<<<<<<<< + * global __cuSignalExternalSemaphoresAsync + * __cuSignalExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuSignalExternalSemaphoresAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecordWithFlags = dlsym(__pyx_v_handle, ((char const *)"cuEventRecordWithFlags_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1788 + * __cuEventRecordWithFlags = dlfcn.dlsym(handle, 'cuEventRecordWithFlags_ptsz') + * global __cuSignalExternalSemaphoresAsync + * __cuSignalExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuSignalExternalSemaphoresAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuWaitExternalSemaphoresAsync + * __cuWaitExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuWaitExternalSemaphoresAsync_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSignalExternalSemaphoresAsync = dlsym(__pyx_v_handle, ((char const *)"cuSignalExternalSemaphoresAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1790 + * __cuSignalExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuSignalExternalSemaphoresAsync_ptsz') + * global __cuWaitExternalSemaphoresAsync + * __cuWaitExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuWaitExternalSemaphoresAsync_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamWaitValue32_v2 + * __cuStreamWaitValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue32_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuWaitExternalSemaphoresAsync = dlsym(__pyx_v_handle, ((char const *)"cuWaitExternalSemaphoresAsync_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1792 + * __cuWaitExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuWaitExternalSemaphoresAsync_ptsz') + * global __cuStreamWaitValue32_v2 + * __cuStreamWaitValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue32_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamWaitValue64_v2 + * __cuStreamWaitValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue64_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue32_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamWaitValue32_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1794 + * __cuStreamWaitValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue32_v2_ptsz') + * global __cuStreamWaitValue64_v2 + * __cuStreamWaitValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue64_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamWriteValue32_v2 + * __cuStreamWriteValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue32_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue64_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamWaitValue64_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1796 + * __cuStreamWaitValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue64_v2_ptsz') + * global __cuStreamWriteValue32_v2 + * __cuStreamWriteValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue32_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamWriteValue64_v2 + * __cuStreamWriteValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue64_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue32_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamWriteValue32_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1798 + * __cuStreamWriteValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue32_v2_ptsz') + * global __cuStreamWriteValue64_v2 + * __cuStreamWriteValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue64_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuStreamBatchMemOp_v2 + * __cuStreamBatchMemOp_v2 = dlfcn.dlsym(handle, 'cuStreamBatchMemOp_v2_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue64_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamWriteValue64_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1800 + * __cuStreamWriteValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue64_v2_ptsz') + * global __cuStreamBatchMemOp_v2 + * __cuStreamBatchMemOp_v2 = dlfcn.dlsym(handle, 'cuStreamBatchMemOp_v2_ptsz') # <<<<<<<<<<<<<< + * global __cuLaunchKernel + * __cuLaunchKernel = dlfcn.dlsym(handle, 'cuLaunchKernel_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBatchMemOp_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamBatchMemOp_v2_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1802 + * __cuStreamBatchMemOp_v2 = dlfcn.dlsym(handle, 'cuStreamBatchMemOp_v2_ptsz') + * global __cuLaunchKernel + * __cuLaunchKernel = dlfcn.dlsym(handle, 'cuLaunchKernel_ptsz') # <<<<<<<<<<<<<< + * global __cuLaunchKernelEx + * __cuLaunchKernelEx = dlfcn.dlsym(handle, 'cuLaunchKernelEx_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernel = dlsym(__pyx_v_handle, ((char const *)"cuLaunchKernel_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1804 + * __cuLaunchKernel = dlfcn.dlsym(handle, 'cuLaunchKernel_ptsz') + * global __cuLaunchKernelEx + * __cuLaunchKernelEx = dlfcn.dlsym(handle, 'cuLaunchKernelEx_ptsz') # <<<<<<<<<<<<<< + * global __cuLaunchCooperativeKernel + * __cuLaunchCooperativeKernel = dlfcn.dlsym(handle, 'cuLaunchCooperativeKernel_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernelEx = dlsym(__pyx_v_handle, ((char const *)"cuLaunchKernelEx_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1806 + * __cuLaunchKernelEx = dlfcn.dlsym(handle, 'cuLaunchKernelEx_ptsz') + * global __cuLaunchCooperativeKernel + * __cuLaunchCooperativeKernel = dlfcn.dlsym(handle, 'cuLaunchCooperativeKernel_ptsz') # <<<<<<<<<<<<<< + * global __cuLaunchHostFunc + * __cuLaunchHostFunc = dlfcn.dlsym(handle, 'cuLaunchHostFunc_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernel = dlsym(__pyx_v_handle, ((char const *)"cuLaunchCooperativeKernel_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1808 + * __cuLaunchCooperativeKernel = dlfcn.dlsym(handle, 'cuLaunchCooperativeKernel_ptsz') + * global __cuLaunchHostFunc + * __cuLaunchHostFunc = dlfcn.dlsym(handle, 'cuLaunchHostFunc_ptsz') # <<<<<<<<<<<<<< + * global __cuGraphInstantiateWithParams + * __cuGraphInstantiateWithParams = dlfcn.dlsym(handle, 'cuGraphInstantiateWithParams_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchHostFunc = dlsym(__pyx_v_handle, ((char const *)"cuLaunchHostFunc_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1810 + * __cuLaunchHostFunc = dlfcn.dlsym(handle, 'cuLaunchHostFunc_ptsz') + * global __cuGraphInstantiateWithParams + * __cuGraphInstantiateWithParams = dlfcn.dlsym(handle, 'cuGraphInstantiateWithParams_ptsz') # <<<<<<<<<<<<<< + * global __cuGraphUpload + * __cuGraphUpload = dlfcn.dlsym(handle, 'cuGraphUpload_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphInstantiateWithParams_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1812 + * __cuGraphInstantiateWithParams = dlfcn.dlsym(handle, 'cuGraphInstantiateWithParams_ptsz') + * global __cuGraphUpload + * __cuGraphUpload = dlfcn.dlsym(handle, 'cuGraphUpload_ptsz') # <<<<<<<<<<<<<< + * global __cuGraphLaunch + * __cuGraphLaunch = dlfcn.dlsym(handle, 'cuGraphLaunch_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphUpload = dlsym(__pyx_v_handle, ((char const *)"cuGraphUpload_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1814 + * __cuGraphUpload = dlfcn.dlsym(handle, 'cuGraphUpload_ptsz') + * global __cuGraphLaunch + * __cuGraphLaunch = dlfcn.dlsym(handle, 'cuGraphLaunch_ptsz') # <<<<<<<<<<<<<< + * global __cuGraphicsMapResources + * __cuGraphicsMapResources = dlfcn.dlsym(handle, 'cuGraphicsMapResources_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphLaunch = dlsym(__pyx_v_handle, ((char const *)"cuGraphLaunch_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1816 + * __cuGraphLaunch = dlfcn.dlsym(handle, 'cuGraphLaunch_ptsz') + * global __cuGraphicsMapResources + * __cuGraphicsMapResources = dlfcn.dlsym(handle, 'cuGraphicsMapResources_ptsz') # <<<<<<<<<<<<<< + * global __cuGraphicsUnmapResources + * __cuGraphicsUnmapResources = dlfcn.dlsym(handle, 'cuGraphicsUnmapResources_ptsz') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsMapResources = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsMapResources_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1818 + * __cuGraphicsMapResources = dlfcn.dlsym(handle, 'cuGraphicsMapResources_ptsz') + * global __cuGraphicsUnmapResources + * __cuGraphicsUnmapResources = dlfcn.dlsym(handle, 'cuGraphicsUnmapResources_ptsz') # <<<<<<<<<<<<<< + * else: + * # Else get the regular version +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnmapResources = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsUnmapResources_ptsz")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1646 + * + * # Load using dlsym + * if usePTDS: # <<<<<<<<<<<<<< + * # Get all PTDS version of functions + * pass +*/ + goto __pyx_L22; + } + + /* "cuda/bindings/_bindings/cydriver.pyx":1821 + * else: + * # Else get the regular version + * pass # <<<<<<<<<<<<<< + * global __cuMemcpy + * __cuMemcpy = dlfcn.dlsym(handle, 'cuMemcpy') +*/ + /*else*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":1823 + * pass + * global __cuMemcpy + * __cuMemcpy = dlfcn.dlsym(handle, 'cuMemcpy') # <<<<<<<<<<<<<< + * global __cuMemcpyPeer + * __cuMemcpyPeer = dlfcn.dlsym(handle, 'cuMemcpyPeer') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1825 + * __cuMemcpy = dlfcn.dlsym(handle, 'cuMemcpy') + * global __cuMemcpyPeer + * __cuMemcpyPeer = dlfcn.dlsym(handle, 'cuMemcpyPeer') # <<<<<<<<<<<<<< + * global __cuMemcpyHtoD_v2 + * __cuMemcpyHtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoD_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeer = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyPeer")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1827 + * __cuMemcpyPeer = dlfcn.dlsym(handle, 'cuMemcpyPeer') + * global __cuMemcpyHtoD_v2 + * __cuMemcpyHtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoD_v2') # <<<<<<<<<<<<<< + * global __cuMemcpyDtoH_v2 + * __cuMemcpyDtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoH_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoD_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyHtoD_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1829 + * __cuMemcpyHtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoD_v2') + * global __cuMemcpyDtoH_v2 + * __cuMemcpyDtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoH_v2') # <<<<<<<<<<<<<< + * global __cuMemcpyDtoD_v2 + * __cuMemcpyDtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoD_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoH_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyDtoH_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1831 + * __cuMemcpyDtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoH_v2') + * global __cuMemcpyDtoD_v2 + * __cuMemcpyDtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoD_v2') # <<<<<<<<<<<<<< + * global __cuMemcpyDtoA_v2 + * __cuMemcpyDtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoA_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoD_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyDtoD_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1833 + * __cuMemcpyDtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoD_v2') + * global __cuMemcpyDtoA_v2 + * __cuMemcpyDtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoA_v2') # <<<<<<<<<<<<<< + * global __cuMemcpyAtoD_v2 + * __cuMemcpyAtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoD_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoA_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyDtoA_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1835 + * __cuMemcpyDtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoA_v2') + * global __cuMemcpyAtoD_v2 + * __cuMemcpyAtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoD_v2') # <<<<<<<<<<<<<< + * global __cuMemcpyHtoA_v2 + * __cuMemcpyHtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoA_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoD_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyAtoD_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1837 + * __cuMemcpyAtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoD_v2') + * global __cuMemcpyHtoA_v2 + * __cuMemcpyHtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoA_v2') # <<<<<<<<<<<<<< + * global __cuMemcpyAtoH_v2 + * __cuMemcpyAtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoH_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoA_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyHtoA_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1839 + * __cuMemcpyHtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoA_v2') + * global __cuMemcpyAtoH_v2 + * __cuMemcpyAtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoH_v2') # <<<<<<<<<<<<<< + * global __cuMemcpyAtoA_v2 + * __cuMemcpyAtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoA_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoH_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyAtoH_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1841 + * __cuMemcpyAtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoH_v2') + * global __cuMemcpyAtoA_v2 + * __cuMemcpyAtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoA_v2') # <<<<<<<<<<<<<< + * global __cuMemcpy2D_v2 + * __cuMemcpy2D_v2 = dlfcn.dlsym(handle, 'cuMemcpy2D_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoA_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyAtoA_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1843 + * __cuMemcpyAtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoA_v2') + * global __cuMemcpy2D_v2 + * __cuMemcpy2D_v2 = dlfcn.dlsym(handle, 'cuMemcpy2D_v2') # <<<<<<<<<<<<<< + * global __cuMemcpy2DUnaligned_v2 + * __cuMemcpy2DUnaligned_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DUnaligned_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2D_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy2D_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1845 + * __cuMemcpy2D_v2 = dlfcn.dlsym(handle, 'cuMemcpy2D_v2') + * global __cuMemcpy2DUnaligned_v2 + * __cuMemcpy2DUnaligned_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DUnaligned_v2') # <<<<<<<<<<<<<< + * global __cuMemcpy3D_v2 + * __cuMemcpy3D_v2 = dlfcn.dlsym(handle, 'cuMemcpy3D_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DUnaligned_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy2DUnaligned_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1847 + * __cuMemcpy2DUnaligned_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DUnaligned_v2') + * global __cuMemcpy3D_v2 + * __cuMemcpy3D_v2 = dlfcn.dlsym(handle, 'cuMemcpy3D_v2') # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeer + * __cuMemcpy3DPeer = dlfcn.dlsym(handle, 'cuMemcpy3DPeer') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3D_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy3D_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1849 + * __cuMemcpy3D_v2 = dlfcn.dlsym(handle, 'cuMemcpy3D_v2') + * global __cuMemcpy3DPeer + * __cuMemcpy3DPeer = dlfcn.dlsym(handle, 'cuMemcpy3DPeer') # <<<<<<<<<<<<<< + * global __cuMemcpyAsync + * __cuMemcpyAsync = dlfcn.dlsym(handle, 'cuMemcpyAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeer = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy3DPeer")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1851 + * __cuMemcpy3DPeer = dlfcn.dlsym(handle, 'cuMemcpy3DPeer') + * global __cuMemcpyAsync + * __cuMemcpyAsync = dlfcn.dlsym(handle, 'cuMemcpyAsync') # <<<<<<<<<<<<<< + * global __cuMemcpyPeerAsync + * __cuMemcpyPeerAsync = dlfcn.dlsym(handle, 'cuMemcpyPeerAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1853 + * __cuMemcpyAsync = dlfcn.dlsym(handle, 'cuMemcpyAsync') + * global __cuMemcpyPeerAsync + * __cuMemcpyPeerAsync = dlfcn.dlsym(handle, 'cuMemcpyPeerAsync') # <<<<<<<<<<<<<< + * global __cuMemcpyHtoDAsync_v2 + * __cuMemcpyHtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoDAsync_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeerAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyPeerAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1855 + * __cuMemcpyPeerAsync = dlfcn.dlsym(handle, 'cuMemcpyPeerAsync') + * global __cuMemcpyHtoDAsync_v2 + * __cuMemcpyHtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoDAsync_v2') # <<<<<<<<<<<<<< + * global __cuMemcpyDtoHAsync_v2 + * __cuMemcpyDtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoHAsync_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoDAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyHtoDAsync_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1857 + * __cuMemcpyHtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoDAsync_v2') + * global __cuMemcpyDtoHAsync_v2 + * __cuMemcpyDtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoHAsync_v2') # <<<<<<<<<<<<<< + * global __cuMemcpyDtoDAsync_v2 + * __cuMemcpyDtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoDAsync_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoHAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyDtoHAsync_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1859 + * __cuMemcpyDtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoHAsync_v2') + * global __cuMemcpyDtoDAsync_v2 + * __cuMemcpyDtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoDAsync_v2') # <<<<<<<<<<<<<< + * global __cuMemcpyHtoAAsync_v2 + * __cuMemcpyHtoAAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoAAsync_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoDAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyDtoDAsync_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1861 + * __cuMemcpyDtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoDAsync_v2') + * global __cuMemcpyHtoAAsync_v2 + * __cuMemcpyHtoAAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoAAsync_v2') # <<<<<<<<<<<<<< + * global __cuMemcpyAtoHAsync_v2 + * __cuMemcpyAtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoHAsync_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoAAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyHtoAAsync_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1863 + * __cuMemcpyHtoAAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoAAsync_v2') + * global __cuMemcpyAtoHAsync_v2 + * __cuMemcpyAtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoHAsync_v2') # <<<<<<<<<<<<<< + * global __cuMemcpy2DAsync_v2 + * __cuMemcpy2DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DAsync_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoHAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyAtoHAsync_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1865 + * __cuMemcpyAtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoHAsync_v2') + * global __cuMemcpy2DAsync_v2 + * __cuMemcpy2DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DAsync_v2') # <<<<<<<<<<<<<< + * global __cuMemcpy3DAsync_v2 + * __cuMemcpy3DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy3DAsync_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy2DAsync_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1867 + * __cuMemcpy2DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DAsync_v2') + * global __cuMemcpy3DAsync_v2 + * __cuMemcpy3DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy3DAsync_v2') # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeerAsync + * __cuMemcpy3DPeerAsync = dlfcn.dlsym(handle, 'cuMemcpy3DPeerAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy3DAsync_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1869 + * __cuMemcpy3DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy3DAsync_v2') + * global __cuMemcpy3DPeerAsync + * __cuMemcpy3DPeerAsync = dlfcn.dlsym(handle, 'cuMemcpy3DPeerAsync') # <<<<<<<<<<<<<< + * global __cuMemcpyBatchAsync + * __cuMemcpyBatchAsync = dlfcn.dlsym(handle, 'cuMemcpyBatchAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeerAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy3DPeerAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1871 + * __cuMemcpy3DPeerAsync = dlfcn.dlsym(handle, 'cuMemcpy3DPeerAsync') + * global __cuMemcpyBatchAsync + * __cuMemcpyBatchAsync = dlfcn.dlsym(handle, 'cuMemcpyBatchAsync') # <<<<<<<<<<<<<< + * global __cuMemcpy3DBatchAsync + * __cuMemcpy3DBatchAsync = dlfcn.dlsym(handle, 'cuMemcpy3DBatchAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyBatchAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemcpyBatchAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1873 + * __cuMemcpyBatchAsync = dlfcn.dlsym(handle, 'cuMemcpyBatchAsync') + * global __cuMemcpy3DBatchAsync + * __cuMemcpy3DBatchAsync = dlfcn.dlsym(handle, 'cuMemcpy3DBatchAsync') # <<<<<<<<<<<<<< + * global __cuMemsetD8_v2 + * __cuMemsetD8_v2 = dlfcn.dlsym(handle, 'cuMemsetD8_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DBatchAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemcpy3DBatchAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1875 + * __cuMemcpy3DBatchAsync = dlfcn.dlsym(handle, 'cuMemcpy3DBatchAsync') + * global __cuMemsetD8_v2 + * __cuMemsetD8_v2 = dlfcn.dlsym(handle, 'cuMemsetD8_v2') # <<<<<<<<<<<<<< + * global __cuMemsetD16_v2 + * __cuMemsetD16_v2 = dlfcn.dlsym(handle, 'cuMemsetD16_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD8_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1877 + * __cuMemsetD8_v2 = dlfcn.dlsym(handle, 'cuMemsetD8_v2') + * global __cuMemsetD16_v2 + * __cuMemsetD16_v2 = dlfcn.dlsym(handle, 'cuMemsetD16_v2') # <<<<<<<<<<<<<< + * global __cuMemsetD32_v2 + * __cuMemsetD32_v2 = dlfcn.dlsym(handle, 'cuMemsetD32_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD16_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1879 + * __cuMemsetD16_v2 = dlfcn.dlsym(handle, 'cuMemsetD16_v2') + * global __cuMemsetD32_v2 + * __cuMemsetD32_v2 = dlfcn.dlsym(handle, 'cuMemsetD32_v2') # <<<<<<<<<<<<<< + * global __cuMemsetD2D8_v2 + * __cuMemsetD2D8_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D8_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD32_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1881 + * __cuMemsetD32_v2 = dlfcn.dlsym(handle, 'cuMemsetD32_v2') + * global __cuMemsetD2D8_v2 + * __cuMemsetD2D8_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D8_v2') # <<<<<<<<<<<<<< + * global __cuMemsetD2D16_v2 + * __cuMemsetD2D16_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D16_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD2D8_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1883 + * __cuMemsetD2D8_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D8_v2') + * global __cuMemsetD2D16_v2 + * __cuMemsetD2D16_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D16_v2') # <<<<<<<<<<<<<< + * global __cuMemsetD2D32_v2 + * __cuMemsetD2D32_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D32_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD2D16_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1885 + * __cuMemsetD2D16_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D16_v2') + * global __cuMemsetD2D32_v2 + * __cuMemsetD2D32_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D32_v2') # <<<<<<<<<<<<<< + * global __cuMemsetD8Async + * __cuMemsetD8Async = dlfcn.dlsym(handle, 'cuMemsetD8Async') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD2D32_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1887 + * __cuMemsetD2D32_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D32_v2') + * global __cuMemsetD8Async + * __cuMemsetD8Async = dlfcn.dlsym(handle, 'cuMemsetD8Async') # <<<<<<<<<<<<<< + * global __cuMemsetD16Async + * __cuMemsetD16Async = dlfcn.dlsym(handle, 'cuMemsetD16Async') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8Async = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD8Async")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1889 + * __cuMemsetD8Async = dlfcn.dlsym(handle, 'cuMemsetD8Async') + * global __cuMemsetD16Async + * __cuMemsetD16Async = dlfcn.dlsym(handle, 'cuMemsetD16Async') # <<<<<<<<<<<<<< + * global __cuMemsetD32Async + * __cuMemsetD32Async = dlfcn.dlsym(handle, 'cuMemsetD32Async') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16Async = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD16Async")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1891 + * __cuMemsetD16Async = dlfcn.dlsym(handle, 'cuMemsetD16Async') + * global __cuMemsetD32Async + * __cuMemsetD32Async = dlfcn.dlsym(handle, 'cuMemsetD32Async') # <<<<<<<<<<<<<< + * global __cuMemsetD2D8Async + * __cuMemsetD2D8Async = dlfcn.dlsym(handle, 'cuMemsetD2D8Async') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32Async = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD32Async")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1893 + * __cuMemsetD32Async = dlfcn.dlsym(handle, 'cuMemsetD32Async') + * global __cuMemsetD2D8Async + * __cuMemsetD2D8Async = dlfcn.dlsym(handle, 'cuMemsetD2D8Async') # <<<<<<<<<<<<<< + * global __cuMemsetD2D16Async + * __cuMemsetD2D16Async = dlfcn.dlsym(handle, 'cuMemsetD2D16Async') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8Async = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD2D8Async")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1895 + * __cuMemsetD2D8Async = dlfcn.dlsym(handle, 'cuMemsetD2D8Async') + * global __cuMemsetD2D16Async + * __cuMemsetD2D16Async = dlfcn.dlsym(handle, 'cuMemsetD2D16Async') # <<<<<<<<<<<<<< + * global __cuMemsetD2D32Async + * __cuMemsetD2D32Async = dlfcn.dlsym(handle, 'cuMemsetD2D32Async') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16Async = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD2D16Async")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1897 + * __cuMemsetD2D16Async = dlfcn.dlsym(handle, 'cuMemsetD2D16Async') + * global __cuMemsetD2D32Async + * __cuMemsetD2D32Async = dlfcn.dlsym(handle, 'cuMemsetD2D32Async') # <<<<<<<<<<<<<< + * global __cuMemBatchDecompressAsync + * __cuMemBatchDecompressAsync = dlfcn.dlsym(handle, 'cuMemBatchDecompressAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32Async = dlsym(__pyx_v_handle, ((char const *)"cuMemsetD2D32Async")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1899 + * __cuMemsetD2D32Async = dlfcn.dlsym(handle, 'cuMemsetD2D32Async') + * global __cuMemBatchDecompressAsync + * __cuMemBatchDecompressAsync = dlfcn.dlsym(handle, 'cuMemBatchDecompressAsync') # <<<<<<<<<<<<<< + * global __cuMemMapArrayAsync + * __cuMemMapArrayAsync = dlfcn.dlsym(handle, 'cuMemMapArrayAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemBatchDecompressAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemBatchDecompressAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1901 + * __cuMemBatchDecompressAsync = dlfcn.dlsym(handle, 'cuMemBatchDecompressAsync') + * global __cuMemMapArrayAsync + * __cuMemMapArrayAsync = dlfcn.dlsym(handle, 'cuMemMapArrayAsync') # <<<<<<<<<<<<<< + * global __cuMemFreeAsync + * __cuMemFreeAsync = dlfcn.dlsym(handle, 'cuMemFreeAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMapArrayAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemMapArrayAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1903 + * __cuMemMapArrayAsync = dlfcn.dlsym(handle, 'cuMemMapArrayAsync') + * global __cuMemFreeAsync + * __cuMemFreeAsync = dlfcn.dlsym(handle, 'cuMemFreeAsync') # <<<<<<<<<<<<<< + * global __cuMemAllocAsync + * __cuMemAllocAsync = dlfcn.dlsym(handle, 'cuMemAllocAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemFreeAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1905 + * __cuMemFreeAsync = dlfcn.dlsym(handle, 'cuMemFreeAsync') + * global __cuMemAllocAsync + * __cuMemAllocAsync = dlfcn.dlsym(handle, 'cuMemAllocAsync') # <<<<<<<<<<<<<< + * global __cuMemAllocFromPoolAsync + * __cuMemAllocFromPoolAsync = dlfcn.dlsym(handle, 'cuMemAllocFromPoolAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemAllocAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1907 + * __cuMemAllocAsync = dlfcn.dlsym(handle, 'cuMemAllocAsync') + * global __cuMemAllocFromPoolAsync + * __cuMemAllocFromPoolAsync = dlfcn.dlsym(handle, 'cuMemAllocFromPoolAsync') # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync + * __cuMemPrefetchAsync = dlfcn.dlsym(handle, 'cuMemPrefetchAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocFromPoolAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemAllocFromPoolAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1909 + * __cuMemAllocFromPoolAsync = dlfcn.dlsym(handle, 'cuMemAllocFromPoolAsync') + * global __cuMemPrefetchAsync + * __cuMemPrefetchAsync = dlfcn.dlsym(handle, 'cuMemPrefetchAsync') # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync_v2 + * __cuMemPrefetchAsync_v2 = dlfcn.dlsym(handle, 'cuMemPrefetchAsync_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync = dlsym(__pyx_v_handle, ((char const *)"cuMemPrefetchAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1911 + * __cuMemPrefetchAsync = dlfcn.dlsym(handle, 'cuMemPrefetchAsync') + * global __cuMemPrefetchAsync_v2 + * __cuMemPrefetchAsync_v2 = dlfcn.dlsym(handle, 'cuMemPrefetchAsync_v2') # <<<<<<<<<<<<<< + * global __cuStreamGetPriority + * __cuStreamGetPriority = dlfcn.dlsym(handle, 'cuStreamGetPriority') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemPrefetchAsync_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1913 + * __cuMemPrefetchAsync_v2 = dlfcn.dlsym(handle, 'cuMemPrefetchAsync_v2') + * global __cuStreamGetPriority + * __cuStreamGetPriority = dlfcn.dlsym(handle, 'cuStreamGetPriority') # <<<<<<<<<<<<<< + * global __cuStreamGetDevice + * __cuStreamGetDevice = dlfcn.dlsym(handle, 'cuStreamGetDevice') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetPriority = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetPriority")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1915 + * __cuStreamGetPriority = dlfcn.dlsym(handle, 'cuStreamGetPriority') + * global __cuStreamGetDevice + * __cuStreamGetDevice = dlfcn.dlsym(handle, 'cuStreamGetDevice') # <<<<<<<<<<<<<< + * global __cuStreamGetFlags + * __cuStreamGetFlags = dlfcn.dlsym(handle, 'cuStreamGetFlags') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetDevice = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetDevice")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1917 + * __cuStreamGetDevice = dlfcn.dlsym(handle, 'cuStreamGetDevice') + * global __cuStreamGetFlags + * __cuStreamGetFlags = dlfcn.dlsym(handle, 'cuStreamGetFlags') # <<<<<<<<<<<<<< + * global __cuStreamGetId + * __cuStreamGetId = dlfcn.dlsym(handle, 'cuStreamGetId') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetFlags = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetFlags")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1919 + * __cuStreamGetFlags = dlfcn.dlsym(handle, 'cuStreamGetFlags') + * global __cuStreamGetId + * __cuStreamGetId = dlfcn.dlsym(handle, 'cuStreamGetId') # <<<<<<<<<<<<<< + * global __cuStreamGetCtx + * __cuStreamGetCtx = dlfcn.dlsym(handle, 'cuStreamGetCtx') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetId = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetId")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1921 + * __cuStreamGetId = dlfcn.dlsym(handle, 'cuStreamGetId') + * global __cuStreamGetCtx + * __cuStreamGetCtx = dlfcn.dlsym(handle, 'cuStreamGetCtx') # <<<<<<<<<<<<<< + * global __cuStreamGetCtx_v2 + * __cuStreamGetCtx_v2 = dlfcn.dlsym(handle, 'cuStreamGetCtx_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetCtx")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1923 + * __cuStreamGetCtx = dlfcn.dlsym(handle, 'cuStreamGetCtx') + * global __cuStreamGetCtx_v2 + * __cuStreamGetCtx_v2 = dlfcn.dlsym(handle, 'cuStreamGetCtx_v2') # <<<<<<<<<<<<<< + * global __cuStreamWaitEvent + * __cuStreamWaitEvent = dlfcn.dlsym(handle, 'cuStreamWaitEvent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetCtx_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1925 + * __cuStreamGetCtx_v2 = dlfcn.dlsym(handle, 'cuStreamGetCtx_v2') + * global __cuStreamWaitEvent + * __cuStreamWaitEvent = dlfcn.dlsym(handle, 'cuStreamWaitEvent') # <<<<<<<<<<<<<< + * global __cuStreamAddCallback + * __cuStreamAddCallback = dlfcn.dlsym(handle, 'cuStreamAddCallback') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitEvent = dlsym(__pyx_v_handle, ((char const *)"cuStreamWaitEvent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1927 + * __cuStreamWaitEvent = dlfcn.dlsym(handle, 'cuStreamWaitEvent') + * global __cuStreamAddCallback + * __cuStreamAddCallback = dlfcn.dlsym(handle, 'cuStreamAddCallback') # <<<<<<<<<<<<<< + * global __cuStreamBeginCapture_v2 + * __cuStreamBeginCapture_v2 = dlfcn.dlsym(handle, 'cuStreamBeginCapture_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAddCallback = dlsym(__pyx_v_handle, ((char const *)"cuStreamAddCallback")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1929 + * __cuStreamAddCallback = dlfcn.dlsym(handle, 'cuStreamAddCallback') + * global __cuStreamBeginCapture_v2 + * __cuStreamBeginCapture_v2 = dlfcn.dlsym(handle, 'cuStreamBeginCapture_v2') # <<<<<<<<<<<<<< + * global __cuStreamBeginCaptureToGraph + * __cuStreamBeginCaptureToGraph = dlfcn.dlsym(handle, 'cuStreamBeginCaptureToGraph') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCapture_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamBeginCapture_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1931 + * __cuStreamBeginCapture_v2 = dlfcn.dlsym(handle, 'cuStreamBeginCapture_v2') + * global __cuStreamBeginCaptureToGraph + * __cuStreamBeginCaptureToGraph = dlfcn.dlsym(handle, 'cuStreamBeginCaptureToGraph') # <<<<<<<<<<<<<< + * global __cuStreamEndCapture + * __cuStreamEndCapture = dlfcn.dlsym(handle, 'cuStreamEndCapture') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCaptureToGraph = dlsym(__pyx_v_handle, ((char const *)"cuStreamBeginCaptureToGraph")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1933 + * __cuStreamBeginCaptureToGraph = dlfcn.dlsym(handle, 'cuStreamBeginCaptureToGraph') + * global __cuStreamEndCapture + * __cuStreamEndCapture = dlfcn.dlsym(handle, 'cuStreamEndCapture') # <<<<<<<<<<<<<< + * global __cuStreamIsCapturing + * __cuStreamIsCapturing = dlfcn.dlsym(handle, 'cuStreamIsCapturing') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamEndCapture = dlsym(__pyx_v_handle, ((char const *)"cuStreamEndCapture")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1935 + * __cuStreamEndCapture = dlfcn.dlsym(handle, 'cuStreamEndCapture') + * global __cuStreamIsCapturing + * __cuStreamIsCapturing = dlfcn.dlsym(handle, 'cuStreamIsCapturing') # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v2 + * __cuStreamGetCaptureInfo_v2 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamIsCapturing = dlsym(__pyx_v_handle, ((char const *)"cuStreamIsCapturing")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1937 + * __cuStreamIsCapturing = dlfcn.dlsym(handle, 'cuStreamIsCapturing') + * global __cuStreamGetCaptureInfo_v2 + * __cuStreamGetCaptureInfo_v2 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v2') # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v3 + * __cuStreamGetCaptureInfo_v3 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v3') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetCaptureInfo_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1939 + * __cuStreamGetCaptureInfo_v2 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v2') + * global __cuStreamGetCaptureInfo_v3 + * __cuStreamGetCaptureInfo_v3 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v3') # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies + * __cuStreamUpdateCaptureDependencies = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v3 = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetCaptureInfo_v3")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1941 + * __cuStreamGetCaptureInfo_v3 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v3') + * global __cuStreamUpdateCaptureDependencies + * __cuStreamUpdateCaptureDependencies = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies') # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies_v2 + * __cuStreamUpdateCaptureDependencies_v2 = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies = dlsym(__pyx_v_handle, ((char const *)"cuStreamUpdateCaptureDependencies")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1943 + * __cuStreamUpdateCaptureDependencies = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies') + * global __cuStreamUpdateCaptureDependencies_v2 + * __cuStreamUpdateCaptureDependencies_v2 = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies_v2') # <<<<<<<<<<<<<< + * global __cuStreamAttachMemAsync + * __cuStreamAttachMemAsync = dlfcn.dlsym(handle, 'cuStreamAttachMemAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamUpdateCaptureDependencies_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1945 + * __cuStreamUpdateCaptureDependencies_v2 = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies_v2') + * global __cuStreamAttachMemAsync + * __cuStreamAttachMemAsync = dlfcn.dlsym(handle, 'cuStreamAttachMemAsync') # <<<<<<<<<<<<<< + * global __cuStreamQuery + * __cuStreamQuery = dlfcn.dlsym(handle, 'cuStreamQuery') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAttachMemAsync = dlsym(__pyx_v_handle, ((char const *)"cuStreamAttachMemAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1947 + * __cuStreamAttachMemAsync = dlfcn.dlsym(handle, 'cuStreamAttachMemAsync') + * global __cuStreamQuery + * __cuStreamQuery = dlfcn.dlsym(handle, 'cuStreamQuery') # <<<<<<<<<<<<<< + * global __cuStreamSynchronize + * __cuStreamSynchronize = dlfcn.dlsym(handle, 'cuStreamSynchronize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamQuery = dlsym(__pyx_v_handle, ((char const *)"cuStreamQuery")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1949 + * __cuStreamQuery = dlfcn.dlsym(handle, 'cuStreamQuery') + * global __cuStreamSynchronize + * __cuStreamSynchronize = dlfcn.dlsym(handle, 'cuStreamSynchronize') # <<<<<<<<<<<<<< + * global __cuStreamCopyAttributes + * __cuStreamCopyAttributes = dlfcn.dlsym(handle, 'cuStreamCopyAttributes') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSynchronize = dlsym(__pyx_v_handle, ((char const *)"cuStreamSynchronize")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1951 + * __cuStreamSynchronize = dlfcn.dlsym(handle, 'cuStreamSynchronize') + * global __cuStreamCopyAttributes + * __cuStreamCopyAttributes = dlfcn.dlsym(handle, 'cuStreamCopyAttributes') # <<<<<<<<<<<<<< + * global __cuStreamGetAttribute + * __cuStreamGetAttribute = dlfcn.dlsym(handle, 'cuStreamGetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCopyAttributes = dlsym(__pyx_v_handle, ((char const *)"cuStreamCopyAttributes")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1953 + * __cuStreamCopyAttributes = dlfcn.dlsym(handle, 'cuStreamCopyAttributes') + * global __cuStreamGetAttribute + * __cuStreamGetAttribute = dlfcn.dlsym(handle, 'cuStreamGetAttribute') # <<<<<<<<<<<<<< + * global __cuStreamSetAttribute + * __cuStreamSetAttribute = dlfcn.dlsym(handle, 'cuStreamSetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1955 + * __cuStreamGetAttribute = dlfcn.dlsym(handle, 'cuStreamGetAttribute') + * global __cuStreamSetAttribute + * __cuStreamSetAttribute = dlfcn.dlsym(handle, 'cuStreamSetAttribute') # <<<<<<<<<<<<<< + * global __cuEventRecord + * __cuEventRecord = dlfcn.dlsym(handle, 'cuEventRecord') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuStreamSetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1957 + * __cuStreamSetAttribute = dlfcn.dlsym(handle, 'cuStreamSetAttribute') + * global __cuEventRecord + * __cuEventRecord = dlfcn.dlsym(handle, 'cuEventRecord') # <<<<<<<<<<<<<< + * global __cuEventRecordWithFlags + * __cuEventRecordWithFlags = dlfcn.dlsym(handle, 'cuEventRecordWithFlags') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecord = dlsym(__pyx_v_handle, ((char const *)"cuEventRecord")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1959 + * __cuEventRecord = dlfcn.dlsym(handle, 'cuEventRecord') + * global __cuEventRecordWithFlags + * __cuEventRecordWithFlags = dlfcn.dlsym(handle, 'cuEventRecordWithFlags') # <<<<<<<<<<<<<< + * global __cuSignalExternalSemaphoresAsync + * __cuSignalExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuSignalExternalSemaphoresAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecordWithFlags = dlsym(__pyx_v_handle, ((char const *)"cuEventRecordWithFlags")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1961 + * __cuEventRecordWithFlags = dlfcn.dlsym(handle, 'cuEventRecordWithFlags') + * global __cuSignalExternalSemaphoresAsync + * __cuSignalExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuSignalExternalSemaphoresAsync') # <<<<<<<<<<<<<< + * global __cuWaitExternalSemaphoresAsync + * __cuWaitExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuWaitExternalSemaphoresAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSignalExternalSemaphoresAsync = dlsym(__pyx_v_handle, ((char const *)"cuSignalExternalSemaphoresAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1963 + * __cuSignalExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuSignalExternalSemaphoresAsync') + * global __cuWaitExternalSemaphoresAsync + * __cuWaitExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuWaitExternalSemaphoresAsync') # <<<<<<<<<<<<<< + * global __cuStreamWaitValue32_v2 + * __cuStreamWaitValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue32_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuWaitExternalSemaphoresAsync = dlsym(__pyx_v_handle, ((char const *)"cuWaitExternalSemaphoresAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1965 + * __cuWaitExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuWaitExternalSemaphoresAsync') + * global __cuStreamWaitValue32_v2 + * __cuStreamWaitValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue32_v2') # <<<<<<<<<<<<<< + * global __cuStreamWaitValue64_v2 + * __cuStreamWaitValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue64_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue32_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamWaitValue32_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1967 + * __cuStreamWaitValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue32_v2') + * global __cuStreamWaitValue64_v2 + * __cuStreamWaitValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue64_v2') # <<<<<<<<<<<<<< + * global __cuStreamWriteValue32_v2 + * __cuStreamWriteValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue32_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue64_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamWaitValue64_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1969 + * __cuStreamWaitValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue64_v2') + * global __cuStreamWriteValue32_v2 + * __cuStreamWriteValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue32_v2') # <<<<<<<<<<<<<< + * global __cuStreamWriteValue64_v2 + * __cuStreamWriteValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue64_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue32_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamWriteValue32_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1971 + * __cuStreamWriteValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue32_v2') + * global __cuStreamWriteValue64_v2 + * __cuStreamWriteValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue64_v2') # <<<<<<<<<<<<<< + * global __cuStreamBatchMemOp_v2 + * __cuStreamBatchMemOp_v2 = dlfcn.dlsym(handle, 'cuStreamBatchMemOp_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue64_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamWriteValue64_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1973 + * __cuStreamWriteValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue64_v2') + * global __cuStreamBatchMemOp_v2 + * __cuStreamBatchMemOp_v2 = dlfcn.dlsym(handle, 'cuStreamBatchMemOp_v2') # <<<<<<<<<<<<<< + * global __cuLaunchKernel + * __cuLaunchKernel = dlfcn.dlsym(handle, 'cuLaunchKernel') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBatchMemOp_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamBatchMemOp_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1975 + * __cuStreamBatchMemOp_v2 = dlfcn.dlsym(handle, 'cuStreamBatchMemOp_v2') + * global __cuLaunchKernel + * __cuLaunchKernel = dlfcn.dlsym(handle, 'cuLaunchKernel') # <<<<<<<<<<<<<< + * global __cuLaunchKernelEx + * __cuLaunchKernelEx = dlfcn.dlsym(handle, 'cuLaunchKernelEx') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernel = dlsym(__pyx_v_handle, ((char const *)"cuLaunchKernel")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1977 + * __cuLaunchKernel = dlfcn.dlsym(handle, 'cuLaunchKernel') + * global __cuLaunchKernelEx + * __cuLaunchKernelEx = dlfcn.dlsym(handle, 'cuLaunchKernelEx') # <<<<<<<<<<<<<< + * global __cuLaunchCooperativeKernel + * __cuLaunchCooperativeKernel = dlfcn.dlsym(handle, 'cuLaunchCooperativeKernel') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernelEx = dlsym(__pyx_v_handle, ((char const *)"cuLaunchKernelEx")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1979 + * __cuLaunchKernelEx = dlfcn.dlsym(handle, 'cuLaunchKernelEx') + * global __cuLaunchCooperativeKernel + * __cuLaunchCooperativeKernel = dlfcn.dlsym(handle, 'cuLaunchCooperativeKernel') # <<<<<<<<<<<<<< + * global __cuLaunchHostFunc + * __cuLaunchHostFunc = dlfcn.dlsym(handle, 'cuLaunchHostFunc') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernel = dlsym(__pyx_v_handle, ((char const *)"cuLaunchCooperativeKernel")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1981 + * __cuLaunchCooperativeKernel = dlfcn.dlsym(handle, 'cuLaunchCooperativeKernel') + * global __cuLaunchHostFunc + * __cuLaunchHostFunc = dlfcn.dlsym(handle, 'cuLaunchHostFunc') # <<<<<<<<<<<<<< + * global __cuGraphInstantiateWithParams + * __cuGraphInstantiateWithParams = dlfcn.dlsym(handle, 'cuGraphInstantiateWithParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchHostFunc = dlsym(__pyx_v_handle, ((char const *)"cuLaunchHostFunc")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1983 + * __cuLaunchHostFunc = dlfcn.dlsym(handle, 'cuLaunchHostFunc') + * global __cuGraphInstantiateWithParams + * __cuGraphInstantiateWithParams = dlfcn.dlsym(handle, 'cuGraphInstantiateWithParams') # <<<<<<<<<<<<<< + * global __cuGraphUpload + * __cuGraphUpload = dlfcn.dlsym(handle, 'cuGraphUpload') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphInstantiateWithParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1985 + * __cuGraphInstantiateWithParams = dlfcn.dlsym(handle, 'cuGraphInstantiateWithParams') + * global __cuGraphUpload + * __cuGraphUpload = dlfcn.dlsym(handle, 'cuGraphUpload') # <<<<<<<<<<<<<< + * global __cuGraphLaunch + * __cuGraphLaunch = dlfcn.dlsym(handle, 'cuGraphLaunch') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphUpload = dlsym(__pyx_v_handle, ((char const *)"cuGraphUpload")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1987 + * __cuGraphUpload = dlfcn.dlsym(handle, 'cuGraphUpload') + * global __cuGraphLaunch + * __cuGraphLaunch = dlfcn.dlsym(handle, 'cuGraphLaunch') # <<<<<<<<<<<<<< + * global __cuGraphicsMapResources + * __cuGraphicsMapResources = dlfcn.dlsym(handle, 'cuGraphicsMapResources') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphLaunch = dlsym(__pyx_v_handle, ((char const *)"cuGraphLaunch")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1989 + * __cuGraphLaunch = dlfcn.dlsym(handle, 'cuGraphLaunch') + * global __cuGraphicsMapResources + * __cuGraphicsMapResources = dlfcn.dlsym(handle, 'cuGraphicsMapResources') # <<<<<<<<<<<<<< + * global __cuGraphicsUnmapResources + * __cuGraphicsUnmapResources = dlfcn.dlsym(handle, 'cuGraphicsUnmapResources') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsMapResources = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsMapResources")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1991 + * __cuGraphicsMapResources = dlfcn.dlsym(handle, 'cuGraphicsMapResources') + * global __cuGraphicsUnmapResources + * __cuGraphicsUnmapResources = dlfcn.dlsym(handle, 'cuGraphicsUnmapResources') # <<<<<<<<<<<<<< + * # Get remaining functions + * global __cuGetErrorString +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnmapResources = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsUnmapResources")); + } + __pyx_L22:; + + /* "cuda/bindings/_bindings/cydriver.pyx":1994 + * # Get remaining functions + * global __cuGetErrorString + * __cuGetErrorString = dlfcn.dlsym(handle, 'cuGetErrorString') # <<<<<<<<<<<<<< + * global __cuGetErrorName + * __cuGetErrorName = dlfcn.dlsym(handle, 'cuGetErrorName') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorString = dlsym(__pyx_v_handle, ((char const *)"cuGetErrorString")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1996 + * __cuGetErrorString = dlfcn.dlsym(handle, 'cuGetErrorString') + * global __cuGetErrorName + * __cuGetErrorName = dlfcn.dlsym(handle, 'cuGetErrorName') # <<<<<<<<<<<<<< + * global __cuInit + * __cuInit = dlfcn.dlsym(handle, 'cuInit') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorName = dlsym(__pyx_v_handle, ((char const *)"cuGetErrorName")); + + /* "cuda/bindings/_bindings/cydriver.pyx":1998 + * __cuGetErrorName = dlfcn.dlsym(handle, 'cuGetErrorName') + * global __cuInit + * __cuInit = dlfcn.dlsym(handle, 'cuInit') # <<<<<<<<<<<<<< + * global __cuDriverGetVersion + * __cuDriverGetVersion = dlfcn.dlsym(handle, 'cuDriverGetVersion') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuInit = dlsym(__pyx_v_handle, ((char const *)"cuInit")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2000 + * __cuInit = dlfcn.dlsym(handle, 'cuInit') + * global __cuDriverGetVersion + * __cuDriverGetVersion = dlfcn.dlsym(handle, 'cuDriverGetVersion') # <<<<<<<<<<<<<< + * global __cuDeviceGet + * __cuDeviceGet = dlfcn.dlsym(handle, 'cuDeviceGet') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDriverGetVersion = dlsym(__pyx_v_handle, ((char const *)"cuDriverGetVersion")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2002 + * __cuDriverGetVersion = dlfcn.dlsym(handle, 'cuDriverGetVersion') + * global __cuDeviceGet + * __cuDeviceGet = dlfcn.dlsym(handle, 'cuDeviceGet') # <<<<<<<<<<<<<< + * global __cuDeviceGetCount + * __cuDeviceGetCount = dlfcn.dlsym(handle, 'cuDeviceGetCount') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGet = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGet")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2004 + * __cuDeviceGet = dlfcn.dlsym(handle, 'cuDeviceGet') + * global __cuDeviceGetCount + * __cuDeviceGetCount = dlfcn.dlsym(handle, 'cuDeviceGetCount') # <<<<<<<<<<<<<< + * global __cuDeviceGetName + * __cuDeviceGetName = dlfcn.dlsym(handle, 'cuDeviceGetName') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetCount = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetCount")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2006 + * __cuDeviceGetCount = dlfcn.dlsym(handle, 'cuDeviceGetCount') + * global __cuDeviceGetName + * __cuDeviceGetName = dlfcn.dlsym(handle, 'cuDeviceGetName') # <<<<<<<<<<<<<< + * global __cuDeviceGetUuid + * __cuDeviceGetUuid = dlfcn.dlsym(handle, 'cuDeviceGetUuid') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetName = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetName")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2008 + * __cuDeviceGetName = dlfcn.dlsym(handle, 'cuDeviceGetName') + * global __cuDeviceGetUuid + * __cuDeviceGetUuid = dlfcn.dlsym(handle, 'cuDeviceGetUuid') # <<<<<<<<<<<<<< + * global __cuDeviceGetUuid_v2 + * __cuDeviceGetUuid_v2 = dlfcn.dlsym(handle, 'cuDeviceGetUuid_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetUuid")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2010 + * __cuDeviceGetUuid = dlfcn.dlsym(handle, 'cuDeviceGetUuid') + * global __cuDeviceGetUuid_v2 + * __cuDeviceGetUuid_v2 = dlfcn.dlsym(handle, 'cuDeviceGetUuid_v2') # <<<<<<<<<<<<<< + * global __cuDeviceGetLuid + * __cuDeviceGetLuid = dlfcn.dlsym(handle, 'cuDeviceGetLuid') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid_v2 = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetUuid_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2012 + * __cuDeviceGetUuid_v2 = dlfcn.dlsym(handle, 'cuDeviceGetUuid_v2') + * global __cuDeviceGetLuid + * __cuDeviceGetLuid = dlfcn.dlsym(handle, 'cuDeviceGetLuid') # <<<<<<<<<<<<<< + * global __cuDeviceTotalMem_v2 + * __cuDeviceTotalMem_v2 = dlfcn.dlsym(handle, 'cuDeviceTotalMem_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetLuid = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetLuid")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2014 + * __cuDeviceGetLuid = dlfcn.dlsym(handle, 'cuDeviceGetLuid') + * global __cuDeviceTotalMem_v2 + * __cuDeviceTotalMem_v2 = dlfcn.dlsym(handle, 'cuDeviceTotalMem_v2') # <<<<<<<<<<<<<< + * global __cuDeviceGetTexture1DLinearMaxWidth + * __cuDeviceGetTexture1DLinearMaxWidth = dlfcn.dlsym(handle, 'cuDeviceGetTexture1DLinearMaxWidth') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceTotalMem_v2 = dlsym(__pyx_v_handle, ((char const *)"cuDeviceTotalMem_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2016 + * __cuDeviceTotalMem_v2 = dlfcn.dlsym(handle, 'cuDeviceTotalMem_v2') + * global __cuDeviceGetTexture1DLinearMaxWidth + * __cuDeviceGetTexture1DLinearMaxWidth = dlfcn.dlsym(handle, 'cuDeviceGetTexture1DLinearMaxWidth') # <<<<<<<<<<<<<< + * global __cuDeviceGetAttribute + * __cuDeviceGetAttribute = dlfcn.dlsym(handle, 'cuDeviceGetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetTexture1DLinearMaxWidth = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetTexture1DLinearMaxWidth")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2018 + * __cuDeviceGetTexture1DLinearMaxWidth = dlfcn.dlsym(handle, 'cuDeviceGetTexture1DLinearMaxWidth') + * global __cuDeviceGetAttribute + * __cuDeviceGetAttribute = dlfcn.dlsym(handle, 'cuDeviceGetAttribute') # <<<<<<<<<<<<<< + * global __cuDeviceGetNvSciSyncAttributes + * __cuDeviceGetNvSciSyncAttributes = dlfcn.dlsym(handle, 'cuDeviceGetNvSciSyncAttributes') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2020 + * __cuDeviceGetAttribute = dlfcn.dlsym(handle, 'cuDeviceGetAttribute') + * global __cuDeviceGetNvSciSyncAttributes + * __cuDeviceGetNvSciSyncAttributes = dlfcn.dlsym(handle, 'cuDeviceGetNvSciSyncAttributes') # <<<<<<<<<<<<<< + * global __cuDeviceSetMemPool + * __cuDeviceSetMemPool = dlfcn.dlsym(handle, 'cuDeviceSetMemPool') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetNvSciSyncAttributes = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetNvSciSyncAttributes")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2022 + * __cuDeviceGetNvSciSyncAttributes = dlfcn.dlsym(handle, 'cuDeviceGetNvSciSyncAttributes') + * global __cuDeviceSetMemPool + * __cuDeviceSetMemPool = dlfcn.dlsym(handle, 'cuDeviceSetMemPool') # <<<<<<<<<<<<<< + * global __cuDeviceGetMemPool + * __cuDeviceGetMemPool = dlfcn.dlsym(handle, 'cuDeviceGetMemPool') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetMemPool = dlsym(__pyx_v_handle, ((char const *)"cuDeviceSetMemPool")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2024 + * __cuDeviceSetMemPool = dlfcn.dlsym(handle, 'cuDeviceSetMemPool') + * global __cuDeviceGetMemPool + * __cuDeviceGetMemPool = dlfcn.dlsym(handle, 'cuDeviceGetMemPool') # <<<<<<<<<<<<<< + * global __cuDeviceGetDefaultMemPool + * __cuDeviceGetDefaultMemPool = dlfcn.dlsym(handle, 'cuDeviceGetDefaultMemPool') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetMemPool = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetMemPool")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2026 + * __cuDeviceGetMemPool = dlfcn.dlsym(handle, 'cuDeviceGetMemPool') + * global __cuDeviceGetDefaultMemPool + * __cuDeviceGetDefaultMemPool = dlfcn.dlsym(handle, 'cuDeviceGetDefaultMemPool') # <<<<<<<<<<<<<< + * global __cuDeviceGetExecAffinitySupport + * __cuDeviceGetExecAffinitySupport = dlfcn.dlsym(handle, 'cuDeviceGetExecAffinitySupport') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDefaultMemPool = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetDefaultMemPool")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2028 + * __cuDeviceGetDefaultMemPool = dlfcn.dlsym(handle, 'cuDeviceGetDefaultMemPool') + * global __cuDeviceGetExecAffinitySupport + * __cuDeviceGetExecAffinitySupport = dlfcn.dlsym(handle, 'cuDeviceGetExecAffinitySupport') # <<<<<<<<<<<<<< + * global __cuFlushGPUDirectRDMAWrites + * __cuFlushGPUDirectRDMAWrites = dlfcn.dlsym(handle, 'cuFlushGPUDirectRDMAWrites') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetExecAffinitySupport = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetExecAffinitySupport")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2030 + * __cuDeviceGetExecAffinitySupport = dlfcn.dlsym(handle, 'cuDeviceGetExecAffinitySupport') + * global __cuFlushGPUDirectRDMAWrites + * __cuFlushGPUDirectRDMAWrites = dlfcn.dlsym(handle, 'cuFlushGPUDirectRDMAWrites') # <<<<<<<<<<<<<< + * global __cuDeviceGetProperties + * __cuDeviceGetProperties = dlfcn.dlsym(handle, 'cuDeviceGetProperties') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFlushGPUDirectRDMAWrites = dlsym(__pyx_v_handle, ((char const *)"cuFlushGPUDirectRDMAWrites")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2032 + * __cuFlushGPUDirectRDMAWrites = dlfcn.dlsym(handle, 'cuFlushGPUDirectRDMAWrites') + * global __cuDeviceGetProperties + * __cuDeviceGetProperties = dlfcn.dlsym(handle, 'cuDeviceGetProperties') # <<<<<<<<<<<<<< + * global __cuDeviceComputeCapability + * __cuDeviceComputeCapability = dlfcn.dlsym(handle, 'cuDeviceComputeCapability') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetProperties = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetProperties")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2034 + * __cuDeviceGetProperties = dlfcn.dlsym(handle, 'cuDeviceGetProperties') + * global __cuDeviceComputeCapability + * __cuDeviceComputeCapability = dlfcn.dlsym(handle, 'cuDeviceComputeCapability') # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxRetain + * __cuDevicePrimaryCtxRetain = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxRetain') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceComputeCapability = dlsym(__pyx_v_handle, ((char const *)"cuDeviceComputeCapability")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2036 + * __cuDeviceComputeCapability = dlfcn.dlsym(handle, 'cuDeviceComputeCapability') + * global __cuDevicePrimaryCtxRetain + * __cuDevicePrimaryCtxRetain = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxRetain') # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxRelease_v2 + * __cuDevicePrimaryCtxRelease_v2 = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxRelease_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRetain = dlsym(__pyx_v_handle, ((char const *)"cuDevicePrimaryCtxRetain")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2038 + * __cuDevicePrimaryCtxRetain = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxRetain') + * global __cuDevicePrimaryCtxRelease_v2 + * __cuDevicePrimaryCtxRelease_v2 = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxRelease_v2') # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxSetFlags_v2 + * __cuDevicePrimaryCtxSetFlags_v2 = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxSetFlags_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRelease_v2 = dlsym(__pyx_v_handle, ((char const *)"cuDevicePrimaryCtxRelease_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2040 + * __cuDevicePrimaryCtxRelease_v2 = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxRelease_v2') + * global __cuDevicePrimaryCtxSetFlags_v2 + * __cuDevicePrimaryCtxSetFlags_v2 = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxSetFlags_v2') # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxGetState + * __cuDevicePrimaryCtxGetState = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxGetState') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxSetFlags_v2 = dlsym(__pyx_v_handle, ((char const *)"cuDevicePrimaryCtxSetFlags_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2042 + * __cuDevicePrimaryCtxSetFlags_v2 = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxSetFlags_v2') + * global __cuDevicePrimaryCtxGetState + * __cuDevicePrimaryCtxGetState = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxGetState') # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxReset_v2 + * __cuDevicePrimaryCtxReset_v2 = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxReset_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxGetState = dlsym(__pyx_v_handle, ((char const *)"cuDevicePrimaryCtxGetState")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2044 + * __cuDevicePrimaryCtxGetState = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxGetState') + * global __cuDevicePrimaryCtxReset_v2 + * __cuDevicePrimaryCtxReset_v2 = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxReset_v2') # <<<<<<<<<<<<<< + * global __cuCtxCreate_v2 + * __cuCtxCreate_v2 = dlfcn.dlsym(handle, 'cuCtxCreate_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxReset_v2 = dlsym(__pyx_v_handle, ((char const *)"cuDevicePrimaryCtxReset_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2046 + * __cuDevicePrimaryCtxReset_v2 = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxReset_v2') + * global __cuCtxCreate_v2 + * __cuCtxCreate_v2 = dlfcn.dlsym(handle, 'cuCtxCreate_v2') # <<<<<<<<<<<<<< + * global __cuCtxCreate_v3 + * __cuCtxCreate_v3 = dlfcn.dlsym(handle, 'cuCtxCreate_v3') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v2 = dlsym(__pyx_v_handle, ((char const *)"cuCtxCreate_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2048 + * __cuCtxCreate_v2 = dlfcn.dlsym(handle, 'cuCtxCreate_v2') + * global __cuCtxCreate_v3 + * __cuCtxCreate_v3 = dlfcn.dlsym(handle, 'cuCtxCreate_v3') # <<<<<<<<<<<<<< + * global __cuCtxCreate_v4 + * __cuCtxCreate_v4 = dlfcn.dlsym(handle, 'cuCtxCreate_v4') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v3 = dlsym(__pyx_v_handle, ((char const *)"cuCtxCreate_v3")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2050 + * __cuCtxCreate_v3 = dlfcn.dlsym(handle, 'cuCtxCreate_v3') + * global __cuCtxCreate_v4 + * __cuCtxCreate_v4 = dlfcn.dlsym(handle, 'cuCtxCreate_v4') # <<<<<<<<<<<<<< + * global __cuCtxDestroy_v2 + * __cuCtxDestroy_v2 = dlfcn.dlsym(handle, 'cuCtxDestroy_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v4 = dlsym(__pyx_v_handle, ((char const *)"cuCtxCreate_v4")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2052 + * __cuCtxCreate_v4 = dlfcn.dlsym(handle, 'cuCtxCreate_v4') + * global __cuCtxDestroy_v2 + * __cuCtxDestroy_v2 = dlfcn.dlsym(handle, 'cuCtxDestroy_v2') # <<<<<<<<<<<<<< + * global __cuCtxPushCurrent_v2 + * __cuCtxPushCurrent_v2 = dlfcn.dlsym(handle, 'cuCtxPushCurrent_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDestroy_v2 = dlsym(__pyx_v_handle, ((char const *)"cuCtxDestroy_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2054 + * __cuCtxDestroy_v2 = dlfcn.dlsym(handle, 'cuCtxDestroy_v2') + * global __cuCtxPushCurrent_v2 + * __cuCtxPushCurrent_v2 = dlfcn.dlsym(handle, 'cuCtxPushCurrent_v2') # <<<<<<<<<<<<<< + * global __cuCtxPopCurrent_v2 + * __cuCtxPopCurrent_v2 = dlfcn.dlsym(handle, 'cuCtxPopCurrent_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPushCurrent_v2 = dlsym(__pyx_v_handle, ((char const *)"cuCtxPushCurrent_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2056 + * __cuCtxPushCurrent_v2 = dlfcn.dlsym(handle, 'cuCtxPushCurrent_v2') + * global __cuCtxPopCurrent_v2 + * __cuCtxPopCurrent_v2 = dlfcn.dlsym(handle, 'cuCtxPopCurrent_v2') # <<<<<<<<<<<<<< + * global __cuCtxSetCurrent + * __cuCtxSetCurrent = dlfcn.dlsym(handle, 'cuCtxSetCurrent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPopCurrent_v2 = dlsym(__pyx_v_handle, ((char const *)"cuCtxPopCurrent_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2058 + * __cuCtxPopCurrent_v2 = dlfcn.dlsym(handle, 'cuCtxPopCurrent_v2') + * global __cuCtxSetCurrent + * __cuCtxSetCurrent = dlfcn.dlsym(handle, 'cuCtxSetCurrent') # <<<<<<<<<<<<<< + * global __cuCtxGetCurrent + * __cuCtxGetCurrent = dlfcn.dlsym(handle, 'cuCtxGetCurrent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCurrent = dlsym(__pyx_v_handle, ((char const *)"cuCtxSetCurrent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2060 + * __cuCtxSetCurrent = dlfcn.dlsym(handle, 'cuCtxSetCurrent') + * global __cuCtxGetCurrent + * __cuCtxGetCurrent = dlfcn.dlsym(handle, 'cuCtxGetCurrent') # <<<<<<<<<<<<<< + * global __cuCtxGetDevice + * __cuCtxGetDevice = dlfcn.dlsym(handle, 'cuCtxGetDevice') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCurrent = dlsym(__pyx_v_handle, ((char const *)"cuCtxGetCurrent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2062 + * __cuCtxGetCurrent = dlfcn.dlsym(handle, 'cuCtxGetCurrent') + * global __cuCtxGetDevice + * __cuCtxGetDevice = dlfcn.dlsym(handle, 'cuCtxGetDevice') # <<<<<<<<<<<<<< + * global __cuCtxGetFlags + * __cuCtxGetFlags = dlfcn.dlsym(handle, 'cuCtxGetFlags') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevice = dlsym(__pyx_v_handle, ((char const *)"cuCtxGetDevice")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2064 + * __cuCtxGetDevice = dlfcn.dlsym(handle, 'cuCtxGetDevice') + * global __cuCtxGetFlags + * __cuCtxGetFlags = dlfcn.dlsym(handle, 'cuCtxGetFlags') # <<<<<<<<<<<<<< + * global __cuCtxSetFlags + * __cuCtxSetFlags = dlfcn.dlsym(handle, 'cuCtxSetFlags') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetFlags = dlsym(__pyx_v_handle, ((char const *)"cuCtxGetFlags")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2066 + * __cuCtxGetFlags = dlfcn.dlsym(handle, 'cuCtxGetFlags') + * global __cuCtxSetFlags + * __cuCtxSetFlags = dlfcn.dlsym(handle, 'cuCtxSetFlags') # <<<<<<<<<<<<<< + * global __cuCtxGetId + * __cuCtxGetId = dlfcn.dlsym(handle, 'cuCtxGetId') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetFlags = dlsym(__pyx_v_handle, ((char const *)"cuCtxSetFlags")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2068 + * __cuCtxSetFlags = dlfcn.dlsym(handle, 'cuCtxSetFlags') + * global __cuCtxGetId + * __cuCtxGetId = dlfcn.dlsym(handle, 'cuCtxGetId') # <<<<<<<<<<<<<< + * global __cuCtxSynchronize + * __cuCtxSynchronize = dlfcn.dlsym(handle, 'cuCtxSynchronize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetId = dlsym(__pyx_v_handle, ((char const *)"cuCtxGetId")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2070 + * __cuCtxGetId = dlfcn.dlsym(handle, 'cuCtxGetId') + * global __cuCtxSynchronize + * __cuCtxSynchronize = dlfcn.dlsym(handle, 'cuCtxSynchronize') # <<<<<<<<<<<<<< + * global __cuCtxSetLimit + * __cuCtxSetLimit = dlfcn.dlsym(handle, 'cuCtxSetLimit') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSynchronize = dlsym(__pyx_v_handle, ((char const *)"cuCtxSynchronize")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2072 + * __cuCtxSynchronize = dlfcn.dlsym(handle, 'cuCtxSynchronize') + * global __cuCtxSetLimit + * __cuCtxSetLimit = dlfcn.dlsym(handle, 'cuCtxSetLimit') # <<<<<<<<<<<<<< + * global __cuCtxGetLimit + * __cuCtxGetLimit = dlfcn.dlsym(handle, 'cuCtxGetLimit') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetLimit = dlsym(__pyx_v_handle, ((char const *)"cuCtxSetLimit")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2074 + * __cuCtxSetLimit = dlfcn.dlsym(handle, 'cuCtxSetLimit') + * global __cuCtxGetLimit + * __cuCtxGetLimit = dlfcn.dlsym(handle, 'cuCtxGetLimit') # <<<<<<<<<<<<<< + * global __cuCtxGetCacheConfig + * __cuCtxGetCacheConfig = dlfcn.dlsym(handle, 'cuCtxGetCacheConfig') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetLimit = dlsym(__pyx_v_handle, ((char const *)"cuCtxGetLimit")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2076 + * __cuCtxGetLimit = dlfcn.dlsym(handle, 'cuCtxGetLimit') + * global __cuCtxGetCacheConfig + * __cuCtxGetCacheConfig = dlfcn.dlsym(handle, 'cuCtxGetCacheConfig') # <<<<<<<<<<<<<< + * global __cuCtxSetCacheConfig + * __cuCtxSetCacheConfig = dlfcn.dlsym(handle, 'cuCtxSetCacheConfig') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCacheConfig = dlsym(__pyx_v_handle, ((char const *)"cuCtxGetCacheConfig")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2078 + * __cuCtxGetCacheConfig = dlfcn.dlsym(handle, 'cuCtxGetCacheConfig') + * global __cuCtxSetCacheConfig + * __cuCtxSetCacheConfig = dlfcn.dlsym(handle, 'cuCtxSetCacheConfig') # <<<<<<<<<<<<<< + * global __cuCtxGetApiVersion + * __cuCtxGetApiVersion = dlfcn.dlsym(handle, 'cuCtxGetApiVersion') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCacheConfig = dlsym(__pyx_v_handle, ((char const *)"cuCtxSetCacheConfig")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2080 + * __cuCtxSetCacheConfig = dlfcn.dlsym(handle, 'cuCtxSetCacheConfig') + * global __cuCtxGetApiVersion + * __cuCtxGetApiVersion = dlfcn.dlsym(handle, 'cuCtxGetApiVersion') # <<<<<<<<<<<<<< + * global __cuCtxGetStreamPriorityRange + * __cuCtxGetStreamPriorityRange = dlfcn.dlsym(handle, 'cuCtxGetStreamPriorityRange') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetApiVersion = dlsym(__pyx_v_handle, ((char const *)"cuCtxGetApiVersion")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2082 + * __cuCtxGetApiVersion = dlfcn.dlsym(handle, 'cuCtxGetApiVersion') + * global __cuCtxGetStreamPriorityRange + * __cuCtxGetStreamPriorityRange = dlfcn.dlsym(handle, 'cuCtxGetStreamPriorityRange') # <<<<<<<<<<<<<< + * global __cuCtxResetPersistingL2Cache + * __cuCtxResetPersistingL2Cache = dlfcn.dlsym(handle, 'cuCtxResetPersistingL2Cache') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetStreamPriorityRange = dlsym(__pyx_v_handle, ((char const *)"cuCtxGetStreamPriorityRange")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2084 + * __cuCtxGetStreamPriorityRange = dlfcn.dlsym(handle, 'cuCtxGetStreamPriorityRange') + * global __cuCtxResetPersistingL2Cache + * __cuCtxResetPersistingL2Cache = dlfcn.dlsym(handle, 'cuCtxResetPersistingL2Cache') # <<<<<<<<<<<<<< + * global __cuCtxGetExecAffinity + * __cuCtxGetExecAffinity = dlfcn.dlsym(handle, 'cuCtxGetExecAffinity') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxResetPersistingL2Cache = dlsym(__pyx_v_handle, ((char const *)"cuCtxResetPersistingL2Cache")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2086 + * __cuCtxResetPersistingL2Cache = dlfcn.dlsym(handle, 'cuCtxResetPersistingL2Cache') + * global __cuCtxGetExecAffinity + * __cuCtxGetExecAffinity = dlfcn.dlsym(handle, 'cuCtxGetExecAffinity') # <<<<<<<<<<<<<< + * global __cuCtxRecordEvent + * __cuCtxRecordEvent = dlfcn.dlsym(handle, 'cuCtxRecordEvent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetExecAffinity = dlsym(__pyx_v_handle, ((char const *)"cuCtxGetExecAffinity")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2088 + * __cuCtxGetExecAffinity = dlfcn.dlsym(handle, 'cuCtxGetExecAffinity') + * global __cuCtxRecordEvent + * __cuCtxRecordEvent = dlfcn.dlsym(handle, 'cuCtxRecordEvent') # <<<<<<<<<<<<<< + * global __cuCtxWaitEvent + * __cuCtxWaitEvent = dlfcn.dlsym(handle, 'cuCtxWaitEvent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxRecordEvent = dlsym(__pyx_v_handle, ((char const *)"cuCtxRecordEvent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2090 + * __cuCtxRecordEvent = dlfcn.dlsym(handle, 'cuCtxRecordEvent') + * global __cuCtxWaitEvent + * __cuCtxWaitEvent = dlfcn.dlsym(handle, 'cuCtxWaitEvent') # <<<<<<<<<<<<<< + * global __cuCtxAttach + * __cuCtxAttach = dlfcn.dlsym(handle, 'cuCtxAttach') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxWaitEvent = dlsym(__pyx_v_handle, ((char const *)"cuCtxWaitEvent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2092 + * __cuCtxWaitEvent = dlfcn.dlsym(handle, 'cuCtxWaitEvent') + * global __cuCtxAttach + * __cuCtxAttach = dlfcn.dlsym(handle, 'cuCtxAttach') # <<<<<<<<<<<<<< + * global __cuCtxDetach + * __cuCtxDetach = dlfcn.dlsym(handle, 'cuCtxDetach') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxAttach = dlsym(__pyx_v_handle, ((char const *)"cuCtxAttach")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2094 + * __cuCtxAttach = dlfcn.dlsym(handle, 'cuCtxAttach') + * global __cuCtxDetach + * __cuCtxDetach = dlfcn.dlsym(handle, 'cuCtxDetach') # <<<<<<<<<<<<<< + * global __cuCtxGetSharedMemConfig + * __cuCtxGetSharedMemConfig = dlfcn.dlsym(handle, 'cuCtxGetSharedMemConfig') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDetach = dlsym(__pyx_v_handle, ((char const *)"cuCtxDetach")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2096 + * __cuCtxDetach = dlfcn.dlsym(handle, 'cuCtxDetach') + * global __cuCtxGetSharedMemConfig + * __cuCtxGetSharedMemConfig = dlfcn.dlsym(handle, 'cuCtxGetSharedMemConfig') # <<<<<<<<<<<<<< + * global __cuCtxSetSharedMemConfig + * __cuCtxSetSharedMemConfig = dlfcn.dlsym(handle, 'cuCtxSetSharedMemConfig') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetSharedMemConfig = dlsym(__pyx_v_handle, ((char const *)"cuCtxGetSharedMemConfig")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2098 + * __cuCtxGetSharedMemConfig = dlfcn.dlsym(handle, 'cuCtxGetSharedMemConfig') + * global __cuCtxSetSharedMemConfig + * __cuCtxSetSharedMemConfig = dlfcn.dlsym(handle, 'cuCtxSetSharedMemConfig') # <<<<<<<<<<<<<< + * global __cuModuleLoad + * __cuModuleLoad = dlfcn.dlsym(handle, 'cuModuleLoad') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetSharedMemConfig = dlsym(__pyx_v_handle, ((char const *)"cuCtxSetSharedMemConfig")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2100 + * __cuCtxSetSharedMemConfig = dlfcn.dlsym(handle, 'cuCtxSetSharedMemConfig') + * global __cuModuleLoad + * __cuModuleLoad = dlfcn.dlsym(handle, 'cuModuleLoad') # <<<<<<<<<<<<<< + * global __cuModuleLoadData + * __cuModuleLoadData = dlfcn.dlsym(handle, 'cuModuleLoadData') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoad = dlsym(__pyx_v_handle, ((char const *)"cuModuleLoad")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2102 + * __cuModuleLoad = dlfcn.dlsym(handle, 'cuModuleLoad') + * global __cuModuleLoadData + * __cuModuleLoadData = dlfcn.dlsym(handle, 'cuModuleLoadData') # <<<<<<<<<<<<<< + * global __cuModuleLoadDataEx + * __cuModuleLoadDataEx = dlfcn.dlsym(handle, 'cuModuleLoadDataEx') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadData = dlsym(__pyx_v_handle, ((char const *)"cuModuleLoadData")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2104 + * __cuModuleLoadData = dlfcn.dlsym(handle, 'cuModuleLoadData') + * global __cuModuleLoadDataEx + * __cuModuleLoadDataEx = dlfcn.dlsym(handle, 'cuModuleLoadDataEx') # <<<<<<<<<<<<<< + * global __cuModuleLoadFatBinary + * __cuModuleLoadFatBinary = dlfcn.dlsym(handle, 'cuModuleLoadFatBinary') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadDataEx = dlsym(__pyx_v_handle, ((char const *)"cuModuleLoadDataEx")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2106 + * __cuModuleLoadDataEx = dlfcn.dlsym(handle, 'cuModuleLoadDataEx') + * global __cuModuleLoadFatBinary + * __cuModuleLoadFatBinary = dlfcn.dlsym(handle, 'cuModuleLoadFatBinary') # <<<<<<<<<<<<<< + * global __cuModuleUnload + * __cuModuleUnload = dlfcn.dlsym(handle, 'cuModuleUnload') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadFatBinary = dlsym(__pyx_v_handle, ((char const *)"cuModuleLoadFatBinary")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2108 + * __cuModuleLoadFatBinary = dlfcn.dlsym(handle, 'cuModuleLoadFatBinary') + * global __cuModuleUnload + * __cuModuleUnload = dlfcn.dlsym(handle, 'cuModuleUnload') # <<<<<<<<<<<<<< + * global __cuModuleGetLoadingMode + * __cuModuleGetLoadingMode = dlfcn.dlsym(handle, 'cuModuleGetLoadingMode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleUnload = dlsym(__pyx_v_handle, ((char const *)"cuModuleUnload")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2110 + * __cuModuleUnload = dlfcn.dlsym(handle, 'cuModuleUnload') + * global __cuModuleGetLoadingMode + * __cuModuleGetLoadingMode = dlfcn.dlsym(handle, 'cuModuleGetLoadingMode') # <<<<<<<<<<<<<< + * global __cuModuleGetFunction + * __cuModuleGetFunction = dlfcn.dlsym(handle, 'cuModuleGetFunction') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetLoadingMode = dlsym(__pyx_v_handle, ((char const *)"cuModuleGetLoadingMode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2112 + * __cuModuleGetLoadingMode = dlfcn.dlsym(handle, 'cuModuleGetLoadingMode') + * global __cuModuleGetFunction + * __cuModuleGetFunction = dlfcn.dlsym(handle, 'cuModuleGetFunction') # <<<<<<<<<<<<<< + * global __cuModuleGetFunctionCount + * __cuModuleGetFunctionCount = dlfcn.dlsym(handle, 'cuModuleGetFunctionCount') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunction = dlsym(__pyx_v_handle, ((char const *)"cuModuleGetFunction")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2114 + * __cuModuleGetFunction = dlfcn.dlsym(handle, 'cuModuleGetFunction') + * global __cuModuleGetFunctionCount + * __cuModuleGetFunctionCount = dlfcn.dlsym(handle, 'cuModuleGetFunctionCount') # <<<<<<<<<<<<<< + * global __cuModuleEnumerateFunctions + * __cuModuleEnumerateFunctions = dlfcn.dlsym(handle, 'cuModuleEnumerateFunctions') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunctionCount = dlsym(__pyx_v_handle, ((char const *)"cuModuleGetFunctionCount")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2116 + * __cuModuleGetFunctionCount = dlfcn.dlsym(handle, 'cuModuleGetFunctionCount') + * global __cuModuleEnumerateFunctions + * __cuModuleEnumerateFunctions = dlfcn.dlsym(handle, 'cuModuleEnumerateFunctions') # <<<<<<<<<<<<<< + * global __cuModuleGetGlobal_v2 + * __cuModuleGetGlobal_v2 = dlfcn.dlsym(handle, 'cuModuleGetGlobal_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleEnumerateFunctions = dlsym(__pyx_v_handle, ((char const *)"cuModuleEnumerateFunctions")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2118 + * __cuModuleEnumerateFunctions = dlfcn.dlsym(handle, 'cuModuleEnumerateFunctions') + * global __cuModuleGetGlobal_v2 + * __cuModuleGetGlobal_v2 = dlfcn.dlsym(handle, 'cuModuleGetGlobal_v2') # <<<<<<<<<<<<<< + * global __cuLinkCreate_v2 + * __cuLinkCreate_v2 = dlfcn.dlsym(handle, 'cuLinkCreate_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetGlobal_v2 = dlsym(__pyx_v_handle, ((char const *)"cuModuleGetGlobal_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2120 + * __cuModuleGetGlobal_v2 = dlfcn.dlsym(handle, 'cuModuleGetGlobal_v2') + * global __cuLinkCreate_v2 + * __cuLinkCreate_v2 = dlfcn.dlsym(handle, 'cuLinkCreate_v2') # <<<<<<<<<<<<<< + * global __cuLinkAddData_v2 + * __cuLinkAddData_v2 = dlfcn.dlsym(handle, 'cuLinkAddData_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkCreate_v2 = dlsym(__pyx_v_handle, ((char const *)"cuLinkCreate_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2122 + * __cuLinkCreate_v2 = dlfcn.dlsym(handle, 'cuLinkCreate_v2') + * global __cuLinkAddData_v2 + * __cuLinkAddData_v2 = dlfcn.dlsym(handle, 'cuLinkAddData_v2') # <<<<<<<<<<<<<< + * global __cuLinkAddFile_v2 + * __cuLinkAddFile_v2 = dlfcn.dlsym(handle, 'cuLinkAddFile_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddData_v2 = dlsym(__pyx_v_handle, ((char const *)"cuLinkAddData_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2124 + * __cuLinkAddData_v2 = dlfcn.dlsym(handle, 'cuLinkAddData_v2') + * global __cuLinkAddFile_v2 + * __cuLinkAddFile_v2 = dlfcn.dlsym(handle, 'cuLinkAddFile_v2') # <<<<<<<<<<<<<< + * global __cuLinkComplete + * __cuLinkComplete = dlfcn.dlsym(handle, 'cuLinkComplete') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddFile_v2 = dlsym(__pyx_v_handle, ((char const *)"cuLinkAddFile_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2126 + * __cuLinkAddFile_v2 = dlfcn.dlsym(handle, 'cuLinkAddFile_v2') + * global __cuLinkComplete + * __cuLinkComplete = dlfcn.dlsym(handle, 'cuLinkComplete') # <<<<<<<<<<<<<< + * global __cuLinkDestroy + * __cuLinkDestroy = dlfcn.dlsym(handle, 'cuLinkDestroy') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkComplete = dlsym(__pyx_v_handle, ((char const *)"cuLinkComplete")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2128 + * __cuLinkComplete = dlfcn.dlsym(handle, 'cuLinkComplete') + * global __cuLinkDestroy + * __cuLinkDestroy = dlfcn.dlsym(handle, 'cuLinkDestroy') # <<<<<<<<<<<<<< + * global __cuModuleGetTexRef + * __cuModuleGetTexRef = dlfcn.dlsym(handle, 'cuModuleGetTexRef') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkDestroy = dlsym(__pyx_v_handle, ((char const *)"cuLinkDestroy")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2130 + * __cuLinkDestroy = dlfcn.dlsym(handle, 'cuLinkDestroy') + * global __cuModuleGetTexRef + * __cuModuleGetTexRef = dlfcn.dlsym(handle, 'cuModuleGetTexRef') # <<<<<<<<<<<<<< + * global __cuModuleGetSurfRef + * __cuModuleGetSurfRef = dlfcn.dlsym(handle, 'cuModuleGetSurfRef') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetTexRef = dlsym(__pyx_v_handle, ((char const *)"cuModuleGetTexRef")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2132 + * __cuModuleGetTexRef = dlfcn.dlsym(handle, 'cuModuleGetTexRef') + * global __cuModuleGetSurfRef + * __cuModuleGetSurfRef = dlfcn.dlsym(handle, 'cuModuleGetSurfRef') # <<<<<<<<<<<<<< + * global __cuLibraryLoadData + * __cuLibraryLoadData = dlfcn.dlsym(handle, 'cuLibraryLoadData') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetSurfRef = dlsym(__pyx_v_handle, ((char const *)"cuModuleGetSurfRef")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2134 + * __cuModuleGetSurfRef = dlfcn.dlsym(handle, 'cuModuleGetSurfRef') + * global __cuLibraryLoadData + * __cuLibraryLoadData = dlfcn.dlsym(handle, 'cuLibraryLoadData') # <<<<<<<<<<<<<< + * global __cuLibraryLoadFromFile + * __cuLibraryLoadFromFile = dlfcn.dlsym(handle, 'cuLibraryLoadFromFile') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadData = dlsym(__pyx_v_handle, ((char const *)"cuLibraryLoadData")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2136 + * __cuLibraryLoadData = dlfcn.dlsym(handle, 'cuLibraryLoadData') + * global __cuLibraryLoadFromFile + * __cuLibraryLoadFromFile = dlfcn.dlsym(handle, 'cuLibraryLoadFromFile') # <<<<<<<<<<<<<< + * global __cuLibraryUnload + * __cuLibraryUnload = dlfcn.dlsym(handle, 'cuLibraryUnload') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadFromFile = dlsym(__pyx_v_handle, ((char const *)"cuLibraryLoadFromFile")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2138 + * __cuLibraryLoadFromFile = dlfcn.dlsym(handle, 'cuLibraryLoadFromFile') + * global __cuLibraryUnload + * __cuLibraryUnload = dlfcn.dlsym(handle, 'cuLibraryUnload') # <<<<<<<<<<<<<< + * global __cuLibraryGetKernel + * __cuLibraryGetKernel = dlfcn.dlsym(handle, 'cuLibraryGetKernel') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryUnload = dlsym(__pyx_v_handle, ((char const *)"cuLibraryUnload")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2140 + * __cuLibraryUnload = dlfcn.dlsym(handle, 'cuLibraryUnload') + * global __cuLibraryGetKernel + * __cuLibraryGetKernel = dlfcn.dlsym(handle, 'cuLibraryGetKernel') # <<<<<<<<<<<<<< + * global __cuLibraryGetKernelCount + * __cuLibraryGetKernelCount = dlfcn.dlsym(handle, 'cuLibraryGetKernelCount') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernel = dlsym(__pyx_v_handle, ((char const *)"cuLibraryGetKernel")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2142 + * __cuLibraryGetKernel = dlfcn.dlsym(handle, 'cuLibraryGetKernel') + * global __cuLibraryGetKernelCount + * __cuLibraryGetKernelCount = dlfcn.dlsym(handle, 'cuLibraryGetKernelCount') # <<<<<<<<<<<<<< + * global __cuLibraryEnumerateKernels + * __cuLibraryEnumerateKernels = dlfcn.dlsym(handle, 'cuLibraryEnumerateKernels') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernelCount = dlsym(__pyx_v_handle, ((char const *)"cuLibraryGetKernelCount")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2144 + * __cuLibraryGetKernelCount = dlfcn.dlsym(handle, 'cuLibraryGetKernelCount') + * global __cuLibraryEnumerateKernels + * __cuLibraryEnumerateKernels = dlfcn.dlsym(handle, 'cuLibraryEnumerateKernels') # <<<<<<<<<<<<<< + * global __cuLibraryGetModule + * __cuLibraryGetModule = dlfcn.dlsym(handle, 'cuLibraryGetModule') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryEnumerateKernels = dlsym(__pyx_v_handle, ((char const *)"cuLibraryEnumerateKernels")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2146 + * __cuLibraryEnumerateKernels = dlfcn.dlsym(handle, 'cuLibraryEnumerateKernels') + * global __cuLibraryGetModule + * __cuLibraryGetModule = dlfcn.dlsym(handle, 'cuLibraryGetModule') # <<<<<<<<<<<<<< + * global __cuKernelGetFunction + * __cuKernelGetFunction = dlfcn.dlsym(handle, 'cuKernelGetFunction') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetModule = dlsym(__pyx_v_handle, ((char const *)"cuLibraryGetModule")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2148 + * __cuLibraryGetModule = dlfcn.dlsym(handle, 'cuLibraryGetModule') + * global __cuKernelGetFunction + * __cuKernelGetFunction = dlfcn.dlsym(handle, 'cuKernelGetFunction') # <<<<<<<<<<<<<< + * global __cuKernelGetLibrary + * __cuKernelGetLibrary = dlfcn.dlsym(handle, 'cuKernelGetLibrary') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetFunction = dlsym(__pyx_v_handle, ((char const *)"cuKernelGetFunction")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2150 + * __cuKernelGetFunction = dlfcn.dlsym(handle, 'cuKernelGetFunction') + * global __cuKernelGetLibrary + * __cuKernelGetLibrary = dlfcn.dlsym(handle, 'cuKernelGetLibrary') # <<<<<<<<<<<<<< + * global __cuLibraryGetGlobal + * __cuLibraryGetGlobal = dlfcn.dlsym(handle, 'cuLibraryGetGlobal') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetLibrary = dlsym(__pyx_v_handle, ((char const *)"cuKernelGetLibrary")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2152 + * __cuKernelGetLibrary = dlfcn.dlsym(handle, 'cuKernelGetLibrary') + * global __cuLibraryGetGlobal + * __cuLibraryGetGlobal = dlfcn.dlsym(handle, 'cuLibraryGetGlobal') # <<<<<<<<<<<<<< + * global __cuLibraryGetManaged + * __cuLibraryGetManaged = dlfcn.dlsym(handle, 'cuLibraryGetManaged') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetGlobal = dlsym(__pyx_v_handle, ((char const *)"cuLibraryGetGlobal")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2154 + * __cuLibraryGetGlobal = dlfcn.dlsym(handle, 'cuLibraryGetGlobal') + * global __cuLibraryGetManaged + * __cuLibraryGetManaged = dlfcn.dlsym(handle, 'cuLibraryGetManaged') # <<<<<<<<<<<<<< + * global __cuLibraryGetUnifiedFunction + * __cuLibraryGetUnifiedFunction = dlfcn.dlsym(handle, 'cuLibraryGetUnifiedFunction') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetManaged = dlsym(__pyx_v_handle, ((char const *)"cuLibraryGetManaged")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2156 + * __cuLibraryGetManaged = dlfcn.dlsym(handle, 'cuLibraryGetManaged') + * global __cuLibraryGetUnifiedFunction + * __cuLibraryGetUnifiedFunction = dlfcn.dlsym(handle, 'cuLibraryGetUnifiedFunction') # <<<<<<<<<<<<<< + * global __cuKernelGetAttribute + * __cuKernelGetAttribute = dlfcn.dlsym(handle, 'cuKernelGetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetUnifiedFunction = dlsym(__pyx_v_handle, ((char const *)"cuLibraryGetUnifiedFunction")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2158 + * __cuLibraryGetUnifiedFunction = dlfcn.dlsym(handle, 'cuLibraryGetUnifiedFunction') + * global __cuKernelGetAttribute + * __cuKernelGetAttribute = dlfcn.dlsym(handle, 'cuKernelGetAttribute') # <<<<<<<<<<<<<< + * global __cuKernelSetAttribute + * __cuKernelSetAttribute = dlfcn.dlsym(handle, 'cuKernelSetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuKernelGetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2160 + * __cuKernelGetAttribute = dlfcn.dlsym(handle, 'cuKernelGetAttribute') + * global __cuKernelSetAttribute + * __cuKernelSetAttribute = dlfcn.dlsym(handle, 'cuKernelSetAttribute') # <<<<<<<<<<<<<< + * global __cuKernelSetCacheConfig + * __cuKernelSetCacheConfig = dlfcn.dlsym(handle, 'cuKernelSetCacheConfig') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuKernelSetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2162 + * __cuKernelSetAttribute = dlfcn.dlsym(handle, 'cuKernelSetAttribute') + * global __cuKernelSetCacheConfig + * __cuKernelSetCacheConfig = dlfcn.dlsym(handle, 'cuKernelSetCacheConfig') # <<<<<<<<<<<<<< + * global __cuKernelGetName + * __cuKernelGetName = dlfcn.dlsym(handle, 'cuKernelGetName') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetCacheConfig = dlsym(__pyx_v_handle, ((char const *)"cuKernelSetCacheConfig")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2164 + * __cuKernelSetCacheConfig = dlfcn.dlsym(handle, 'cuKernelSetCacheConfig') + * global __cuKernelGetName + * __cuKernelGetName = dlfcn.dlsym(handle, 'cuKernelGetName') # <<<<<<<<<<<<<< + * global __cuKernelGetParamInfo + * __cuKernelGetParamInfo = dlfcn.dlsym(handle, 'cuKernelGetParamInfo') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetName = dlsym(__pyx_v_handle, ((char const *)"cuKernelGetName")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2166 + * __cuKernelGetName = dlfcn.dlsym(handle, 'cuKernelGetName') + * global __cuKernelGetParamInfo + * __cuKernelGetParamInfo = dlfcn.dlsym(handle, 'cuKernelGetParamInfo') # <<<<<<<<<<<<<< + * global __cuMemGetInfo_v2 + * __cuMemGetInfo_v2 = dlfcn.dlsym(handle, 'cuMemGetInfo_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetParamInfo = dlsym(__pyx_v_handle, ((char const *)"cuKernelGetParamInfo")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2168 + * __cuKernelGetParamInfo = dlfcn.dlsym(handle, 'cuKernelGetParamInfo') + * global __cuMemGetInfo_v2 + * __cuMemGetInfo_v2 = dlfcn.dlsym(handle, 'cuMemGetInfo_v2') # <<<<<<<<<<<<<< + * global __cuMemAlloc_v2 + * __cuMemAlloc_v2 = dlfcn.dlsym(handle, 'cuMemAlloc_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetInfo_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemGetInfo_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2170 + * __cuMemGetInfo_v2 = dlfcn.dlsym(handle, 'cuMemGetInfo_v2') + * global __cuMemAlloc_v2 + * __cuMemAlloc_v2 = dlfcn.dlsym(handle, 'cuMemAlloc_v2') # <<<<<<<<<<<<<< + * global __cuMemAllocPitch_v2 + * __cuMemAllocPitch_v2 = dlfcn.dlsym(handle, 'cuMemAllocPitch_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAlloc_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemAlloc_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2172 + * __cuMemAlloc_v2 = dlfcn.dlsym(handle, 'cuMemAlloc_v2') + * global __cuMemAllocPitch_v2 + * __cuMemAllocPitch_v2 = dlfcn.dlsym(handle, 'cuMemAllocPitch_v2') # <<<<<<<<<<<<<< + * global __cuMemFree_v2 + * __cuMemFree_v2 = dlfcn.dlsym(handle, 'cuMemFree_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocPitch_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemAllocPitch_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2174 + * __cuMemAllocPitch_v2 = dlfcn.dlsym(handle, 'cuMemAllocPitch_v2') + * global __cuMemFree_v2 + * __cuMemFree_v2 = dlfcn.dlsym(handle, 'cuMemFree_v2') # <<<<<<<<<<<<<< + * global __cuMemGetAddressRange_v2 + * __cuMemGetAddressRange_v2 = dlfcn.dlsym(handle, 'cuMemGetAddressRange_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFree_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemFree_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2176 + * __cuMemFree_v2 = dlfcn.dlsym(handle, 'cuMemFree_v2') + * global __cuMemGetAddressRange_v2 + * __cuMemGetAddressRange_v2 = dlfcn.dlsym(handle, 'cuMemGetAddressRange_v2') # <<<<<<<<<<<<<< + * global __cuMemAllocHost_v2 + * __cuMemAllocHost_v2 = dlfcn.dlsym(handle, 'cuMemAllocHost_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAddressRange_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemGetAddressRange_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2178 + * __cuMemGetAddressRange_v2 = dlfcn.dlsym(handle, 'cuMemGetAddressRange_v2') + * global __cuMemAllocHost_v2 + * __cuMemAllocHost_v2 = dlfcn.dlsym(handle, 'cuMemAllocHost_v2') # <<<<<<<<<<<<<< + * global __cuMemFreeHost + * __cuMemFreeHost = dlfcn.dlsym(handle, 'cuMemFreeHost') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocHost_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemAllocHost_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2180 + * __cuMemAllocHost_v2 = dlfcn.dlsym(handle, 'cuMemAllocHost_v2') + * global __cuMemFreeHost + * __cuMemFreeHost = dlfcn.dlsym(handle, 'cuMemFreeHost') # <<<<<<<<<<<<<< + * global __cuMemHostAlloc + * __cuMemHostAlloc = dlfcn.dlsym(handle, 'cuMemHostAlloc') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeHost = dlsym(__pyx_v_handle, ((char const *)"cuMemFreeHost")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2182 + * __cuMemFreeHost = dlfcn.dlsym(handle, 'cuMemFreeHost') + * global __cuMemHostAlloc + * __cuMemHostAlloc = dlfcn.dlsym(handle, 'cuMemHostAlloc') # <<<<<<<<<<<<<< + * global __cuMemHostGetDevicePointer_v2 + * __cuMemHostGetDevicePointer_v2 = dlfcn.dlsym(handle, 'cuMemHostGetDevicePointer_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostAlloc = dlsym(__pyx_v_handle, ((char const *)"cuMemHostAlloc")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2184 + * __cuMemHostAlloc = dlfcn.dlsym(handle, 'cuMemHostAlloc') + * global __cuMemHostGetDevicePointer_v2 + * __cuMemHostGetDevicePointer_v2 = dlfcn.dlsym(handle, 'cuMemHostGetDevicePointer_v2') # <<<<<<<<<<<<<< + * global __cuMemHostGetFlags + * __cuMemHostGetFlags = dlfcn.dlsym(handle, 'cuMemHostGetFlags') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetDevicePointer_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemHostGetDevicePointer_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2186 + * __cuMemHostGetDevicePointer_v2 = dlfcn.dlsym(handle, 'cuMemHostGetDevicePointer_v2') + * global __cuMemHostGetFlags + * __cuMemHostGetFlags = dlfcn.dlsym(handle, 'cuMemHostGetFlags') # <<<<<<<<<<<<<< + * global __cuMemAllocManaged + * __cuMemAllocManaged = dlfcn.dlsym(handle, 'cuMemAllocManaged') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetFlags = dlsym(__pyx_v_handle, ((char const *)"cuMemHostGetFlags")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2188 + * __cuMemHostGetFlags = dlfcn.dlsym(handle, 'cuMemHostGetFlags') + * global __cuMemAllocManaged + * __cuMemAllocManaged = dlfcn.dlsym(handle, 'cuMemAllocManaged') # <<<<<<<<<<<<<< + * global __cuDeviceRegisterAsyncNotification + * __cuDeviceRegisterAsyncNotification = dlfcn.dlsym(handle, 'cuDeviceRegisterAsyncNotification') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocManaged = dlsym(__pyx_v_handle, ((char const *)"cuMemAllocManaged")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2190 + * __cuMemAllocManaged = dlfcn.dlsym(handle, 'cuMemAllocManaged') + * global __cuDeviceRegisterAsyncNotification + * __cuDeviceRegisterAsyncNotification = dlfcn.dlsym(handle, 'cuDeviceRegisterAsyncNotification') # <<<<<<<<<<<<<< + * global __cuDeviceUnregisterAsyncNotification + * __cuDeviceUnregisterAsyncNotification = dlfcn.dlsym(handle, 'cuDeviceUnregisterAsyncNotification') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceRegisterAsyncNotification = dlsym(__pyx_v_handle, ((char const *)"cuDeviceRegisterAsyncNotification")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2192 + * __cuDeviceRegisterAsyncNotification = dlfcn.dlsym(handle, 'cuDeviceRegisterAsyncNotification') + * global __cuDeviceUnregisterAsyncNotification + * __cuDeviceUnregisterAsyncNotification = dlfcn.dlsym(handle, 'cuDeviceUnregisterAsyncNotification') # <<<<<<<<<<<<<< + * global __cuDeviceGetByPCIBusId + * __cuDeviceGetByPCIBusId = dlfcn.dlsym(handle, 'cuDeviceGetByPCIBusId') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceUnregisterAsyncNotification = dlsym(__pyx_v_handle, ((char const *)"cuDeviceUnregisterAsyncNotification")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2194 + * __cuDeviceUnregisterAsyncNotification = dlfcn.dlsym(handle, 'cuDeviceUnregisterAsyncNotification') + * global __cuDeviceGetByPCIBusId + * __cuDeviceGetByPCIBusId = dlfcn.dlsym(handle, 'cuDeviceGetByPCIBusId') # <<<<<<<<<<<<<< + * global __cuDeviceGetPCIBusId + * __cuDeviceGetPCIBusId = dlfcn.dlsym(handle, 'cuDeviceGetPCIBusId') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetByPCIBusId = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetByPCIBusId")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2196 + * __cuDeviceGetByPCIBusId = dlfcn.dlsym(handle, 'cuDeviceGetByPCIBusId') + * global __cuDeviceGetPCIBusId + * __cuDeviceGetPCIBusId = dlfcn.dlsym(handle, 'cuDeviceGetPCIBusId') # <<<<<<<<<<<<<< + * global __cuIpcGetEventHandle + * __cuIpcGetEventHandle = dlfcn.dlsym(handle, 'cuIpcGetEventHandle') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetPCIBusId = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetPCIBusId")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2198 + * __cuDeviceGetPCIBusId = dlfcn.dlsym(handle, 'cuDeviceGetPCIBusId') + * global __cuIpcGetEventHandle + * __cuIpcGetEventHandle = dlfcn.dlsym(handle, 'cuIpcGetEventHandle') # <<<<<<<<<<<<<< + * global __cuIpcOpenEventHandle + * __cuIpcOpenEventHandle = dlfcn.dlsym(handle, 'cuIpcOpenEventHandle') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetEventHandle = dlsym(__pyx_v_handle, ((char const *)"cuIpcGetEventHandle")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2200 + * __cuIpcGetEventHandle = dlfcn.dlsym(handle, 'cuIpcGetEventHandle') + * global __cuIpcOpenEventHandle + * __cuIpcOpenEventHandle = dlfcn.dlsym(handle, 'cuIpcOpenEventHandle') # <<<<<<<<<<<<<< + * global __cuIpcGetMemHandle + * __cuIpcGetMemHandle = dlfcn.dlsym(handle, 'cuIpcGetMemHandle') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenEventHandle = dlsym(__pyx_v_handle, ((char const *)"cuIpcOpenEventHandle")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2202 + * __cuIpcOpenEventHandle = dlfcn.dlsym(handle, 'cuIpcOpenEventHandle') + * global __cuIpcGetMemHandle + * __cuIpcGetMemHandle = dlfcn.dlsym(handle, 'cuIpcGetMemHandle') # <<<<<<<<<<<<<< + * global __cuIpcOpenMemHandle_v2 + * __cuIpcOpenMemHandle_v2 = dlfcn.dlsym(handle, 'cuIpcOpenMemHandle_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetMemHandle = dlsym(__pyx_v_handle, ((char const *)"cuIpcGetMemHandle")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2204 + * __cuIpcGetMemHandle = dlfcn.dlsym(handle, 'cuIpcGetMemHandle') + * global __cuIpcOpenMemHandle_v2 + * __cuIpcOpenMemHandle_v2 = dlfcn.dlsym(handle, 'cuIpcOpenMemHandle_v2') # <<<<<<<<<<<<<< + * global __cuIpcCloseMemHandle + * __cuIpcCloseMemHandle = dlfcn.dlsym(handle, 'cuIpcCloseMemHandle') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenMemHandle_v2 = dlsym(__pyx_v_handle, ((char const *)"cuIpcOpenMemHandle_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2206 + * __cuIpcOpenMemHandle_v2 = dlfcn.dlsym(handle, 'cuIpcOpenMemHandle_v2') + * global __cuIpcCloseMemHandle + * __cuIpcCloseMemHandle = dlfcn.dlsym(handle, 'cuIpcCloseMemHandle') # <<<<<<<<<<<<<< + * global __cuMemHostRegister_v2 + * __cuMemHostRegister_v2 = dlfcn.dlsym(handle, 'cuMemHostRegister_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcCloseMemHandle = dlsym(__pyx_v_handle, ((char const *)"cuIpcCloseMemHandle")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2208 + * __cuIpcCloseMemHandle = dlfcn.dlsym(handle, 'cuIpcCloseMemHandle') + * global __cuMemHostRegister_v2 + * __cuMemHostRegister_v2 = dlfcn.dlsym(handle, 'cuMemHostRegister_v2') # <<<<<<<<<<<<<< + * global __cuMemHostUnregister + * __cuMemHostUnregister = dlfcn.dlsym(handle, 'cuMemHostUnregister') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostRegister_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemHostRegister_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2210 + * __cuMemHostRegister_v2 = dlfcn.dlsym(handle, 'cuMemHostRegister_v2') + * global __cuMemHostUnregister + * __cuMemHostUnregister = dlfcn.dlsym(handle, 'cuMemHostUnregister') # <<<<<<<<<<<<<< + * global __cuArrayCreate_v2 + * __cuArrayCreate_v2 = dlfcn.dlsym(handle, 'cuArrayCreate_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostUnregister = dlsym(__pyx_v_handle, ((char const *)"cuMemHostUnregister")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2212 + * __cuMemHostUnregister = dlfcn.dlsym(handle, 'cuMemHostUnregister') + * global __cuArrayCreate_v2 + * __cuArrayCreate_v2 = dlfcn.dlsym(handle, 'cuArrayCreate_v2') # <<<<<<<<<<<<<< + * global __cuArrayGetDescriptor_v2 + * __cuArrayGetDescriptor_v2 = dlfcn.dlsym(handle, 'cuArrayGetDescriptor_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayCreate_v2 = dlsym(__pyx_v_handle, ((char const *)"cuArrayCreate_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2214 + * __cuArrayCreate_v2 = dlfcn.dlsym(handle, 'cuArrayCreate_v2') + * global __cuArrayGetDescriptor_v2 + * __cuArrayGetDescriptor_v2 = dlfcn.dlsym(handle, 'cuArrayGetDescriptor_v2') # <<<<<<<<<<<<<< + * global __cuArrayGetSparseProperties + * __cuArrayGetSparseProperties = dlfcn.dlsym(handle, 'cuArrayGetSparseProperties') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetDescriptor_v2 = dlsym(__pyx_v_handle, ((char const *)"cuArrayGetDescriptor_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2216 + * __cuArrayGetDescriptor_v2 = dlfcn.dlsym(handle, 'cuArrayGetDescriptor_v2') + * global __cuArrayGetSparseProperties + * __cuArrayGetSparseProperties = dlfcn.dlsym(handle, 'cuArrayGetSparseProperties') # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetSparseProperties + * __cuMipmappedArrayGetSparseProperties = dlfcn.dlsym(handle, 'cuMipmappedArrayGetSparseProperties') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetSparseProperties = dlsym(__pyx_v_handle, ((char const *)"cuArrayGetSparseProperties")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2218 + * __cuArrayGetSparseProperties = dlfcn.dlsym(handle, 'cuArrayGetSparseProperties') + * global __cuMipmappedArrayGetSparseProperties + * __cuMipmappedArrayGetSparseProperties = dlfcn.dlsym(handle, 'cuMipmappedArrayGetSparseProperties') # <<<<<<<<<<<<<< + * global __cuArrayGetMemoryRequirements + * __cuArrayGetMemoryRequirements = dlfcn.dlsym(handle, 'cuArrayGetMemoryRequirements') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetSparseProperties = dlsym(__pyx_v_handle, ((char const *)"cuMipmappedArrayGetSparseProperties")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2220 + * __cuMipmappedArrayGetSparseProperties = dlfcn.dlsym(handle, 'cuMipmappedArrayGetSparseProperties') + * global __cuArrayGetMemoryRequirements + * __cuArrayGetMemoryRequirements = dlfcn.dlsym(handle, 'cuArrayGetMemoryRequirements') # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetMemoryRequirements + * __cuMipmappedArrayGetMemoryRequirements = dlfcn.dlsym(handle, 'cuMipmappedArrayGetMemoryRequirements') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetMemoryRequirements = dlsym(__pyx_v_handle, ((char const *)"cuArrayGetMemoryRequirements")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2222 + * __cuArrayGetMemoryRequirements = dlfcn.dlsym(handle, 'cuArrayGetMemoryRequirements') + * global __cuMipmappedArrayGetMemoryRequirements + * __cuMipmappedArrayGetMemoryRequirements = dlfcn.dlsym(handle, 'cuMipmappedArrayGetMemoryRequirements') # <<<<<<<<<<<<<< + * global __cuArrayGetPlane + * __cuArrayGetPlane = dlfcn.dlsym(handle, 'cuArrayGetPlane') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetMemoryRequirements = dlsym(__pyx_v_handle, ((char const *)"cuMipmappedArrayGetMemoryRequirements")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2224 + * __cuMipmappedArrayGetMemoryRequirements = dlfcn.dlsym(handle, 'cuMipmappedArrayGetMemoryRequirements') + * global __cuArrayGetPlane + * __cuArrayGetPlane = dlfcn.dlsym(handle, 'cuArrayGetPlane') # <<<<<<<<<<<<<< + * global __cuArrayDestroy + * __cuArrayDestroy = dlfcn.dlsym(handle, 'cuArrayDestroy') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetPlane = dlsym(__pyx_v_handle, ((char const *)"cuArrayGetPlane")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2226 + * __cuArrayGetPlane = dlfcn.dlsym(handle, 'cuArrayGetPlane') + * global __cuArrayDestroy + * __cuArrayDestroy = dlfcn.dlsym(handle, 'cuArrayDestroy') # <<<<<<<<<<<<<< + * global __cuArray3DCreate_v2 + * __cuArray3DCreate_v2 = dlfcn.dlsym(handle, 'cuArray3DCreate_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayDestroy = dlsym(__pyx_v_handle, ((char const *)"cuArrayDestroy")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2228 + * __cuArrayDestroy = dlfcn.dlsym(handle, 'cuArrayDestroy') + * global __cuArray3DCreate_v2 + * __cuArray3DCreate_v2 = dlfcn.dlsym(handle, 'cuArray3DCreate_v2') # <<<<<<<<<<<<<< + * global __cuArray3DGetDescriptor_v2 + * __cuArray3DGetDescriptor_v2 = dlfcn.dlsym(handle, 'cuArray3DGetDescriptor_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DCreate_v2 = dlsym(__pyx_v_handle, ((char const *)"cuArray3DCreate_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2230 + * __cuArray3DCreate_v2 = dlfcn.dlsym(handle, 'cuArray3DCreate_v2') + * global __cuArray3DGetDescriptor_v2 + * __cuArray3DGetDescriptor_v2 = dlfcn.dlsym(handle, 'cuArray3DGetDescriptor_v2') # <<<<<<<<<<<<<< + * global __cuMipmappedArrayCreate + * __cuMipmappedArrayCreate = dlfcn.dlsym(handle, 'cuMipmappedArrayCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DGetDescriptor_v2 = dlsym(__pyx_v_handle, ((char const *)"cuArray3DGetDescriptor_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2232 + * __cuArray3DGetDescriptor_v2 = dlfcn.dlsym(handle, 'cuArray3DGetDescriptor_v2') + * global __cuMipmappedArrayCreate + * __cuMipmappedArrayCreate = dlfcn.dlsym(handle, 'cuMipmappedArrayCreate') # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetLevel + * __cuMipmappedArrayGetLevel = dlfcn.dlsym(handle, 'cuMipmappedArrayGetLevel') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayCreate = dlsym(__pyx_v_handle, ((char const *)"cuMipmappedArrayCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2234 + * __cuMipmappedArrayCreate = dlfcn.dlsym(handle, 'cuMipmappedArrayCreate') + * global __cuMipmappedArrayGetLevel + * __cuMipmappedArrayGetLevel = dlfcn.dlsym(handle, 'cuMipmappedArrayGetLevel') # <<<<<<<<<<<<<< + * global __cuMipmappedArrayDestroy + * __cuMipmappedArrayDestroy = dlfcn.dlsym(handle, 'cuMipmappedArrayDestroy') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetLevel = dlsym(__pyx_v_handle, ((char const *)"cuMipmappedArrayGetLevel")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2236 + * __cuMipmappedArrayGetLevel = dlfcn.dlsym(handle, 'cuMipmappedArrayGetLevel') + * global __cuMipmappedArrayDestroy + * __cuMipmappedArrayDestroy = dlfcn.dlsym(handle, 'cuMipmappedArrayDestroy') # <<<<<<<<<<<<<< + * global __cuMemGetHandleForAddressRange + * __cuMemGetHandleForAddressRange = dlfcn.dlsym(handle, 'cuMemGetHandleForAddressRange') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayDestroy = dlsym(__pyx_v_handle, ((char const *)"cuMipmappedArrayDestroy")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2238 + * __cuMipmappedArrayDestroy = dlfcn.dlsym(handle, 'cuMipmappedArrayDestroy') + * global __cuMemGetHandleForAddressRange + * __cuMemGetHandleForAddressRange = dlfcn.dlsym(handle, 'cuMemGetHandleForAddressRange') # <<<<<<<<<<<<<< + * global __cuMemAddressReserve + * __cuMemAddressReserve = dlfcn.dlsym(handle, 'cuMemAddressReserve') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetHandleForAddressRange = dlsym(__pyx_v_handle, ((char const *)"cuMemGetHandleForAddressRange")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2240 + * __cuMemGetHandleForAddressRange = dlfcn.dlsym(handle, 'cuMemGetHandleForAddressRange') + * global __cuMemAddressReserve + * __cuMemAddressReserve = dlfcn.dlsym(handle, 'cuMemAddressReserve') # <<<<<<<<<<<<<< + * global __cuMemAddressFree + * __cuMemAddressFree = dlfcn.dlsym(handle, 'cuMemAddressFree') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressReserve = dlsym(__pyx_v_handle, ((char const *)"cuMemAddressReserve")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2242 + * __cuMemAddressReserve = dlfcn.dlsym(handle, 'cuMemAddressReserve') + * global __cuMemAddressFree + * __cuMemAddressFree = dlfcn.dlsym(handle, 'cuMemAddressFree') # <<<<<<<<<<<<<< + * global __cuMemCreate + * __cuMemCreate = dlfcn.dlsym(handle, 'cuMemCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressFree = dlsym(__pyx_v_handle, ((char const *)"cuMemAddressFree")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2244 + * __cuMemAddressFree = dlfcn.dlsym(handle, 'cuMemAddressFree') + * global __cuMemCreate + * __cuMemCreate = dlfcn.dlsym(handle, 'cuMemCreate') # <<<<<<<<<<<<<< + * global __cuMemRelease + * __cuMemRelease = dlfcn.dlsym(handle, 'cuMemRelease') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemCreate = dlsym(__pyx_v_handle, ((char const *)"cuMemCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2246 + * __cuMemCreate = dlfcn.dlsym(handle, 'cuMemCreate') + * global __cuMemRelease + * __cuMemRelease = dlfcn.dlsym(handle, 'cuMemRelease') # <<<<<<<<<<<<<< + * global __cuMemMap + * __cuMemMap = dlfcn.dlsym(handle, 'cuMemMap') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRelease = dlsym(__pyx_v_handle, ((char const *)"cuMemRelease")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2248 + * __cuMemRelease = dlfcn.dlsym(handle, 'cuMemRelease') + * global __cuMemMap + * __cuMemMap = dlfcn.dlsym(handle, 'cuMemMap') # <<<<<<<<<<<<<< + * global __cuMemUnmap + * __cuMemUnmap = dlfcn.dlsym(handle, 'cuMemUnmap') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMap = dlsym(__pyx_v_handle, ((char const *)"cuMemMap")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2250 + * __cuMemMap = dlfcn.dlsym(handle, 'cuMemMap') + * global __cuMemUnmap + * __cuMemUnmap = dlfcn.dlsym(handle, 'cuMemUnmap') # <<<<<<<<<<<<<< + * global __cuMemSetAccess + * __cuMemSetAccess = dlfcn.dlsym(handle, 'cuMemSetAccess') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemUnmap = dlsym(__pyx_v_handle, ((char const *)"cuMemUnmap")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2252 + * __cuMemUnmap = dlfcn.dlsym(handle, 'cuMemUnmap') + * global __cuMemSetAccess + * __cuMemSetAccess = dlfcn.dlsym(handle, 'cuMemSetAccess') # <<<<<<<<<<<<<< + * global __cuMemGetAccess + * __cuMemGetAccess = dlfcn.dlsym(handle, 'cuMemGetAccess') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemSetAccess = dlsym(__pyx_v_handle, ((char const *)"cuMemSetAccess")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2254 + * __cuMemSetAccess = dlfcn.dlsym(handle, 'cuMemSetAccess') + * global __cuMemGetAccess + * __cuMemGetAccess = dlfcn.dlsym(handle, 'cuMemGetAccess') # <<<<<<<<<<<<<< + * global __cuMemExportToShareableHandle + * __cuMemExportToShareableHandle = dlfcn.dlsym(handle, 'cuMemExportToShareableHandle') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAccess = dlsym(__pyx_v_handle, ((char const *)"cuMemGetAccess")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2256 + * __cuMemGetAccess = dlfcn.dlsym(handle, 'cuMemGetAccess') + * global __cuMemExportToShareableHandle + * __cuMemExportToShareableHandle = dlfcn.dlsym(handle, 'cuMemExportToShareableHandle') # <<<<<<<<<<<<<< + * global __cuMemImportFromShareableHandle + * __cuMemImportFromShareableHandle = dlfcn.dlsym(handle, 'cuMemImportFromShareableHandle') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemExportToShareableHandle = dlsym(__pyx_v_handle, ((char const *)"cuMemExportToShareableHandle")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2258 + * __cuMemExportToShareableHandle = dlfcn.dlsym(handle, 'cuMemExportToShareableHandle') + * global __cuMemImportFromShareableHandle + * __cuMemImportFromShareableHandle = dlfcn.dlsym(handle, 'cuMemImportFromShareableHandle') # <<<<<<<<<<<<<< + * global __cuMemGetAllocationGranularity + * __cuMemGetAllocationGranularity = dlfcn.dlsym(handle, 'cuMemGetAllocationGranularity') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemImportFromShareableHandle = dlsym(__pyx_v_handle, ((char const *)"cuMemImportFromShareableHandle")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2260 + * __cuMemImportFromShareableHandle = dlfcn.dlsym(handle, 'cuMemImportFromShareableHandle') + * global __cuMemGetAllocationGranularity + * __cuMemGetAllocationGranularity = dlfcn.dlsym(handle, 'cuMemGetAllocationGranularity') # <<<<<<<<<<<<<< + * global __cuMemGetAllocationPropertiesFromHandle + * __cuMemGetAllocationPropertiesFromHandle = dlfcn.dlsym(handle, 'cuMemGetAllocationPropertiesFromHandle') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationGranularity = dlsym(__pyx_v_handle, ((char const *)"cuMemGetAllocationGranularity")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2262 + * __cuMemGetAllocationGranularity = dlfcn.dlsym(handle, 'cuMemGetAllocationGranularity') + * global __cuMemGetAllocationPropertiesFromHandle + * __cuMemGetAllocationPropertiesFromHandle = dlfcn.dlsym(handle, 'cuMemGetAllocationPropertiesFromHandle') # <<<<<<<<<<<<<< + * global __cuMemRetainAllocationHandle + * __cuMemRetainAllocationHandle = dlfcn.dlsym(handle, 'cuMemRetainAllocationHandle') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationPropertiesFromHandle = dlsym(__pyx_v_handle, ((char const *)"cuMemGetAllocationPropertiesFromHandle")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2264 + * __cuMemGetAllocationPropertiesFromHandle = dlfcn.dlsym(handle, 'cuMemGetAllocationPropertiesFromHandle') + * global __cuMemRetainAllocationHandle + * __cuMemRetainAllocationHandle = dlfcn.dlsym(handle, 'cuMemRetainAllocationHandle') # <<<<<<<<<<<<<< + * global __cuMemPoolTrimTo + * __cuMemPoolTrimTo = dlfcn.dlsym(handle, 'cuMemPoolTrimTo') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRetainAllocationHandle = dlsym(__pyx_v_handle, ((char const *)"cuMemRetainAllocationHandle")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2266 + * __cuMemRetainAllocationHandle = dlfcn.dlsym(handle, 'cuMemRetainAllocationHandle') + * global __cuMemPoolTrimTo + * __cuMemPoolTrimTo = dlfcn.dlsym(handle, 'cuMemPoolTrimTo') # <<<<<<<<<<<<<< + * global __cuMemPoolSetAttribute + * __cuMemPoolSetAttribute = dlfcn.dlsym(handle, 'cuMemPoolSetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolTrimTo = dlsym(__pyx_v_handle, ((char const *)"cuMemPoolTrimTo")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2268 + * __cuMemPoolTrimTo = dlfcn.dlsym(handle, 'cuMemPoolTrimTo') + * global __cuMemPoolSetAttribute + * __cuMemPoolSetAttribute = dlfcn.dlsym(handle, 'cuMemPoolSetAttribute') # <<<<<<<<<<<<<< + * global __cuMemPoolGetAttribute + * __cuMemPoolGetAttribute = dlfcn.dlsym(handle, 'cuMemPoolGetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuMemPoolSetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2270 + * __cuMemPoolSetAttribute = dlfcn.dlsym(handle, 'cuMemPoolSetAttribute') + * global __cuMemPoolGetAttribute + * __cuMemPoolGetAttribute = dlfcn.dlsym(handle, 'cuMemPoolGetAttribute') # <<<<<<<<<<<<<< + * global __cuMemPoolSetAccess + * __cuMemPoolSetAccess = dlfcn.dlsym(handle, 'cuMemPoolSetAccess') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuMemPoolGetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2272 + * __cuMemPoolGetAttribute = dlfcn.dlsym(handle, 'cuMemPoolGetAttribute') + * global __cuMemPoolSetAccess + * __cuMemPoolSetAccess = dlfcn.dlsym(handle, 'cuMemPoolSetAccess') # <<<<<<<<<<<<<< + * global __cuMemPoolGetAccess + * __cuMemPoolGetAccess = dlfcn.dlsym(handle, 'cuMemPoolGetAccess') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAccess = dlsym(__pyx_v_handle, ((char const *)"cuMemPoolSetAccess")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2274 + * __cuMemPoolSetAccess = dlfcn.dlsym(handle, 'cuMemPoolSetAccess') + * global __cuMemPoolGetAccess + * __cuMemPoolGetAccess = dlfcn.dlsym(handle, 'cuMemPoolGetAccess') # <<<<<<<<<<<<<< + * global __cuMemPoolCreate + * __cuMemPoolCreate = dlfcn.dlsym(handle, 'cuMemPoolCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAccess = dlsym(__pyx_v_handle, ((char const *)"cuMemPoolGetAccess")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2276 + * __cuMemPoolGetAccess = dlfcn.dlsym(handle, 'cuMemPoolGetAccess') + * global __cuMemPoolCreate + * __cuMemPoolCreate = dlfcn.dlsym(handle, 'cuMemPoolCreate') # <<<<<<<<<<<<<< + * global __cuMemPoolDestroy + * __cuMemPoolDestroy = dlfcn.dlsym(handle, 'cuMemPoolDestroy') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolCreate = dlsym(__pyx_v_handle, ((char const *)"cuMemPoolCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2278 + * __cuMemPoolCreate = dlfcn.dlsym(handle, 'cuMemPoolCreate') + * global __cuMemPoolDestroy + * __cuMemPoolDestroy = dlfcn.dlsym(handle, 'cuMemPoolDestroy') # <<<<<<<<<<<<<< + * global __cuMemPoolExportToShareableHandle + * __cuMemPoolExportToShareableHandle = dlfcn.dlsym(handle, 'cuMemPoolExportToShareableHandle') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolDestroy = dlsym(__pyx_v_handle, ((char const *)"cuMemPoolDestroy")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2280 + * __cuMemPoolDestroy = dlfcn.dlsym(handle, 'cuMemPoolDestroy') + * global __cuMemPoolExportToShareableHandle + * __cuMemPoolExportToShareableHandle = dlfcn.dlsym(handle, 'cuMemPoolExportToShareableHandle') # <<<<<<<<<<<<<< + * global __cuMemPoolImportFromShareableHandle + * __cuMemPoolImportFromShareableHandle = dlfcn.dlsym(handle, 'cuMemPoolImportFromShareableHandle') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportToShareableHandle = dlsym(__pyx_v_handle, ((char const *)"cuMemPoolExportToShareableHandle")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2282 + * __cuMemPoolExportToShareableHandle = dlfcn.dlsym(handle, 'cuMemPoolExportToShareableHandle') + * global __cuMemPoolImportFromShareableHandle + * __cuMemPoolImportFromShareableHandle = dlfcn.dlsym(handle, 'cuMemPoolImportFromShareableHandle') # <<<<<<<<<<<<<< + * global __cuMemPoolExportPointer + * __cuMemPoolExportPointer = dlfcn.dlsym(handle, 'cuMemPoolExportPointer') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportFromShareableHandle = dlsym(__pyx_v_handle, ((char const *)"cuMemPoolImportFromShareableHandle")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2284 + * __cuMemPoolImportFromShareableHandle = dlfcn.dlsym(handle, 'cuMemPoolImportFromShareableHandle') + * global __cuMemPoolExportPointer + * __cuMemPoolExportPointer = dlfcn.dlsym(handle, 'cuMemPoolExportPointer') # <<<<<<<<<<<<<< + * global __cuMemPoolImportPointer + * __cuMemPoolImportPointer = dlfcn.dlsym(handle, 'cuMemPoolImportPointer') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportPointer = dlsym(__pyx_v_handle, ((char const *)"cuMemPoolExportPointer")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2286 + * __cuMemPoolExportPointer = dlfcn.dlsym(handle, 'cuMemPoolExportPointer') + * global __cuMemPoolImportPointer + * __cuMemPoolImportPointer = dlfcn.dlsym(handle, 'cuMemPoolImportPointer') # <<<<<<<<<<<<<< + * global __cuMulticastCreate + * __cuMulticastCreate = dlfcn.dlsym(handle, 'cuMulticastCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportPointer = dlsym(__pyx_v_handle, ((char const *)"cuMemPoolImportPointer")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2288 + * __cuMemPoolImportPointer = dlfcn.dlsym(handle, 'cuMemPoolImportPointer') + * global __cuMulticastCreate + * __cuMulticastCreate = dlfcn.dlsym(handle, 'cuMulticastCreate') # <<<<<<<<<<<<<< + * global __cuMulticastAddDevice + * __cuMulticastAddDevice = dlfcn.dlsym(handle, 'cuMulticastAddDevice') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastCreate = dlsym(__pyx_v_handle, ((char const *)"cuMulticastCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2290 + * __cuMulticastCreate = dlfcn.dlsym(handle, 'cuMulticastCreate') + * global __cuMulticastAddDevice + * __cuMulticastAddDevice = dlfcn.dlsym(handle, 'cuMulticastAddDevice') # <<<<<<<<<<<<<< + * global __cuMulticastBindMem + * __cuMulticastBindMem = dlfcn.dlsym(handle, 'cuMulticastBindMem') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastAddDevice = dlsym(__pyx_v_handle, ((char const *)"cuMulticastAddDevice")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2292 + * __cuMulticastAddDevice = dlfcn.dlsym(handle, 'cuMulticastAddDevice') + * global __cuMulticastBindMem + * __cuMulticastBindMem = dlfcn.dlsym(handle, 'cuMulticastBindMem') # <<<<<<<<<<<<<< + * global __cuMulticastBindAddr + * __cuMulticastBindAddr = dlfcn.dlsym(handle, 'cuMulticastBindAddr') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindMem = dlsym(__pyx_v_handle, ((char const *)"cuMulticastBindMem")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2294 + * __cuMulticastBindMem = dlfcn.dlsym(handle, 'cuMulticastBindMem') + * global __cuMulticastBindAddr + * __cuMulticastBindAddr = dlfcn.dlsym(handle, 'cuMulticastBindAddr') # <<<<<<<<<<<<<< + * global __cuMulticastUnbind + * __cuMulticastUnbind = dlfcn.dlsym(handle, 'cuMulticastUnbind') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindAddr = dlsym(__pyx_v_handle, ((char const *)"cuMulticastBindAddr")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2296 + * __cuMulticastBindAddr = dlfcn.dlsym(handle, 'cuMulticastBindAddr') + * global __cuMulticastUnbind + * __cuMulticastUnbind = dlfcn.dlsym(handle, 'cuMulticastUnbind') # <<<<<<<<<<<<<< + * global __cuMulticastGetGranularity + * __cuMulticastGetGranularity = dlfcn.dlsym(handle, 'cuMulticastGetGranularity') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastUnbind = dlsym(__pyx_v_handle, ((char const *)"cuMulticastUnbind")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2298 + * __cuMulticastUnbind = dlfcn.dlsym(handle, 'cuMulticastUnbind') + * global __cuMulticastGetGranularity + * __cuMulticastGetGranularity = dlfcn.dlsym(handle, 'cuMulticastGetGranularity') # <<<<<<<<<<<<<< + * global __cuPointerGetAttribute + * __cuPointerGetAttribute = dlfcn.dlsym(handle, 'cuPointerGetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastGetGranularity = dlsym(__pyx_v_handle, ((char const *)"cuMulticastGetGranularity")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2300 + * __cuMulticastGetGranularity = dlfcn.dlsym(handle, 'cuMulticastGetGranularity') + * global __cuPointerGetAttribute + * __cuPointerGetAttribute = dlfcn.dlsym(handle, 'cuPointerGetAttribute') # <<<<<<<<<<<<<< + * global __cuMemAdvise + * __cuMemAdvise = dlfcn.dlsym(handle, 'cuMemAdvise') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuPointerGetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2302 + * __cuPointerGetAttribute = dlfcn.dlsym(handle, 'cuPointerGetAttribute') + * global __cuMemAdvise + * __cuMemAdvise = dlfcn.dlsym(handle, 'cuMemAdvise') # <<<<<<<<<<<<<< + * global __cuMemAdvise_v2 + * __cuMemAdvise_v2 = dlfcn.dlsym(handle, 'cuMemAdvise_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise = dlsym(__pyx_v_handle, ((char const *)"cuMemAdvise")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2304 + * __cuMemAdvise = dlfcn.dlsym(handle, 'cuMemAdvise') + * global __cuMemAdvise_v2 + * __cuMemAdvise_v2 = dlfcn.dlsym(handle, 'cuMemAdvise_v2') # <<<<<<<<<<<<<< + * global __cuMemRangeGetAttribute + * __cuMemRangeGetAttribute = dlfcn.dlsym(handle, 'cuMemRangeGetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise_v2 = dlsym(__pyx_v_handle, ((char const *)"cuMemAdvise_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2306 + * __cuMemAdvise_v2 = dlfcn.dlsym(handle, 'cuMemAdvise_v2') + * global __cuMemRangeGetAttribute + * __cuMemRangeGetAttribute = dlfcn.dlsym(handle, 'cuMemRangeGetAttribute') # <<<<<<<<<<<<<< + * global __cuMemRangeGetAttributes + * __cuMemRangeGetAttributes = dlfcn.dlsym(handle, 'cuMemRangeGetAttributes') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuMemRangeGetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2308 + * __cuMemRangeGetAttribute = dlfcn.dlsym(handle, 'cuMemRangeGetAttribute') + * global __cuMemRangeGetAttributes + * __cuMemRangeGetAttributes = dlfcn.dlsym(handle, 'cuMemRangeGetAttributes') # <<<<<<<<<<<<<< + * global __cuPointerSetAttribute + * __cuPointerSetAttribute = dlfcn.dlsym(handle, 'cuPointerSetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttributes = dlsym(__pyx_v_handle, ((char const *)"cuMemRangeGetAttributes")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2310 + * __cuMemRangeGetAttributes = dlfcn.dlsym(handle, 'cuMemRangeGetAttributes') + * global __cuPointerSetAttribute + * __cuPointerSetAttribute = dlfcn.dlsym(handle, 'cuPointerSetAttribute') # <<<<<<<<<<<<<< + * global __cuPointerGetAttributes + * __cuPointerGetAttributes = dlfcn.dlsym(handle, 'cuPointerGetAttributes') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerSetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuPointerSetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2312 + * __cuPointerSetAttribute = dlfcn.dlsym(handle, 'cuPointerSetAttribute') + * global __cuPointerGetAttributes + * __cuPointerGetAttributes = dlfcn.dlsym(handle, 'cuPointerGetAttributes') # <<<<<<<<<<<<<< + * global __cuStreamCreate + * __cuStreamCreate = dlfcn.dlsym(handle, 'cuStreamCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttributes = dlsym(__pyx_v_handle, ((char const *)"cuPointerGetAttributes")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2314 + * __cuPointerGetAttributes = dlfcn.dlsym(handle, 'cuPointerGetAttributes') + * global __cuStreamCreate + * __cuStreamCreate = dlfcn.dlsym(handle, 'cuStreamCreate') # <<<<<<<<<<<<<< + * global __cuStreamCreateWithPriority + * __cuStreamCreateWithPriority = dlfcn.dlsym(handle, 'cuStreamCreateWithPriority') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreate = dlsym(__pyx_v_handle, ((char const *)"cuStreamCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2316 + * __cuStreamCreate = dlfcn.dlsym(handle, 'cuStreamCreate') + * global __cuStreamCreateWithPriority + * __cuStreamCreateWithPriority = dlfcn.dlsym(handle, 'cuStreamCreateWithPriority') # <<<<<<<<<<<<<< + * global __cuThreadExchangeStreamCaptureMode + * __cuThreadExchangeStreamCaptureMode = dlfcn.dlsym(handle, 'cuThreadExchangeStreamCaptureMode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreateWithPriority = dlsym(__pyx_v_handle, ((char const *)"cuStreamCreateWithPriority")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2318 + * __cuStreamCreateWithPriority = dlfcn.dlsym(handle, 'cuStreamCreateWithPriority') + * global __cuThreadExchangeStreamCaptureMode + * __cuThreadExchangeStreamCaptureMode = dlfcn.dlsym(handle, 'cuThreadExchangeStreamCaptureMode') # <<<<<<<<<<<<<< + * global __cuStreamDestroy_v2 + * __cuStreamDestroy_v2 = dlfcn.dlsym(handle, 'cuStreamDestroy_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuThreadExchangeStreamCaptureMode = dlsym(__pyx_v_handle, ((char const *)"cuThreadExchangeStreamCaptureMode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2320 + * __cuThreadExchangeStreamCaptureMode = dlfcn.dlsym(handle, 'cuThreadExchangeStreamCaptureMode') + * global __cuStreamDestroy_v2 + * __cuStreamDestroy_v2 = dlfcn.dlsym(handle, 'cuStreamDestroy_v2') # <<<<<<<<<<<<<< + * global __cuEventCreate + * __cuEventCreate = dlfcn.dlsym(handle, 'cuEventCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamDestroy_v2 = dlsym(__pyx_v_handle, ((char const *)"cuStreamDestroy_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2322 + * __cuStreamDestroy_v2 = dlfcn.dlsym(handle, 'cuStreamDestroy_v2') + * global __cuEventCreate + * __cuEventCreate = dlfcn.dlsym(handle, 'cuEventCreate') # <<<<<<<<<<<<<< + * global __cuEventQuery + * __cuEventQuery = dlfcn.dlsym(handle, 'cuEventQuery') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreate = dlsym(__pyx_v_handle, ((char const *)"cuEventCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2324 + * __cuEventCreate = dlfcn.dlsym(handle, 'cuEventCreate') + * global __cuEventQuery + * __cuEventQuery = dlfcn.dlsym(handle, 'cuEventQuery') # <<<<<<<<<<<<<< + * global __cuEventSynchronize + * __cuEventSynchronize = dlfcn.dlsym(handle, 'cuEventSynchronize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventQuery = dlsym(__pyx_v_handle, ((char const *)"cuEventQuery")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2326 + * __cuEventQuery = dlfcn.dlsym(handle, 'cuEventQuery') + * global __cuEventSynchronize + * __cuEventSynchronize = dlfcn.dlsym(handle, 'cuEventSynchronize') # <<<<<<<<<<<<<< + * global __cuEventDestroy_v2 + * __cuEventDestroy_v2 = dlfcn.dlsym(handle, 'cuEventDestroy_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventSynchronize = dlsym(__pyx_v_handle, ((char const *)"cuEventSynchronize")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2328 + * __cuEventSynchronize = dlfcn.dlsym(handle, 'cuEventSynchronize') + * global __cuEventDestroy_v2 + * __cuEventDestroy_v2 = dlfcn.dlsym(handle, 'cuEventDestroy_v2') # <<<<<<<<<<<<<< + * global __cuEventElapsedTime + * __cuEventElapsedTime = dlfcn.dlsym(handle, 'cuEventElapsedTime') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventDestroy_v2 = dlsym(__pyx_v_handle, ((char const *)"cuEventDestroy_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2330 + * __cuEventDestroy_v2 = dlfcn.dlsym(handle, 'cuEventDestroy_v2') + * global __cuEventElapsedTime + * __cuEventElapsedTime = dlfcn.dlsym(handle, 'cuEventElapsedTime') # <<<<<<<<<<<<<< + * global __cuEventElapsedTime_v2 + * __cuEventElapsedTime_v2 = dlfcn.dlsym(handle, 'cuEventElapsedTime_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime = dlsym(__pyx_v_handle, ((char const *)"cuEventElapsedTime")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2332 + * __cuEventElapsedTime = dlfcn.dlsym(handle, 'cuEventElapsedTime') + * global __cuEventElapsedTime_v2 + * __cuEventElapsedTime_v2 = dlfcn.dlsym(handle, 'cuEventElapsedTime_v2') # <<<<<<<<<<<<<< + * global __cuImportExternalMemory + * __cuImportExternalMemory = dlfcn.dlsym(handle, 'cuImportExternalMemory') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime_v2 = dlsym(__pyx_v_handle, ((char const *)"cuEventElapsedTime_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2334 + * __cuEventElapsedTime_v2 = dlfcn.dlsym(handle, 'cuEventElapsedTime_v2') + * global __cuImportExternalMemory + * __cuImportExternalMemory = dlfcn.dlsym(handle, 'cuImportExternalMemory') # <<<<<<<<<<<<<< + * global __cuExternalMemoryGetMappedBuffer + * __cuExternalMemoryGetMappedBuffer = dlfcn.dlsym(handle, 'cuExternalMemoryGetMappedBuffer') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalMemory = dlsym(__pyx_v_handle, ((char const *)"cuImportExternalMemory")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2336 + * __cuImportExternalMemory = dlfcn.dlsym(handle, 'cuImportExternalMemory') + * global __cuExternalMemoryGetMappedBuffer + * __cuExternalMemoryGetMappedBuffer = dlfcn.dlsym(handle, 'cuExternalMemoryGetMappedBuffer') # <<<<<<<<<<<<<< + * global __cuExternalMemoryGetMappedMipmappedArray + * __cuExternalMemoryGetMappedMipmappedArray = dlfcn.dlsym(handle, 'cuExternalMemoryGetMappedMipmappedArray') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedBuffer = dlsym(__pyx_v_handle, ((char const *)"cuExternalMemoryGetMappedBuffer")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2338 + * __cuExternalMemoryGetMappedBuffer = dlfcn.dlsym(handle, 'cuExternalMemoryGetMappedBuffer') + * global __cuExternalMemoryGetMappedMipmappedArray + * __cuExternalMemoryGetMappedMipmappedArray = dlfcn.dlsym(handle, 'cuExternalMemoryGetMappedMipmappedArray') # <<<<<<<<<<<<<< + * global __cuDestroyExternalMemory + * __cuDestroyExternalMemory = dlfcn.dlsym(handle, 'cuDestroyExternalMemory') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedMipmappedArray = dlsym(__pyx_v_handle, ((char const *)"cuExternalMemoryGetMappedMipmappedArray")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2340 + * __cuExternalMemoryGetMappedMipmappedArray = dlfcn.dlsym(handle, 'cuExternalMemoryGetMappedMipmappedArray') + * global __cuDestroyExternalMemory + * __cuDestroyExternalMemory = dlfcn.dlsym(handle, 'cuDestroyExternalMemory') # <<<<<<<<<<<<<< + * global __cuImportExternalSemaphore + * __cuImportExternalSemaphore = dlfcn.dlsym(handle, 'cuImportExternalSemaphore') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalMemory = dlsym(__pyx_v_handle, ((char const *)"cuDestroyExternalMemory")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2342 + * __cuDestroyExternalMemory = dlfcn.dlsym(handle, 'cuDestroyExternalMemory') + * global __cuImportExternalSemaphore + * __cuImportExternalSemaphore = dlfcn.dlsym(handle, 'cuImportExternalSemaphore') # <<<<<<<<<<<<<< + * global __cuDestroyExternalSemaphore + * __cuDestroyExternalSemaphore = dlfcn.dlsym(handle, 'cuDestroyExternalSemaphore') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalSemaphore = dlsym(__pyx_v_handle, ((char const *)"cuImportExternalSemaphore")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2344 + * __cuImportExternalSemaphore = dlfcn.dlsym(handle, 'cuImportExternalSemaphore') + * global __cuDestroyExternalSemaphore + * __cuDestroyExternalSemaphore = dlfcn.dlsym(handle, 'cuDestroyExternalSemaphore') # <<<<<<<<<<<<<< + * global __cuFuncGetAttribute + * __cuFuncGetAttribute = dlfcn.dlsym(handle, 'cuFuncGetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalSemaphore = dlsym(__pyx_v_handle, ((char const *)"cuDestroyExternalSemaphore")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2346 + * __cuDestroyExternalSemaphore = dlfcn.dlsym(handle, 'cuDestroyExternalSemaphore') + * global __cuFuncGetAttribute + * __cuFuncGetAttribute = dlfcn.dlsym(handle, 'cuFuncGetAttribute') # <<<<<<<<<<<<<< + * global __cuFuncSetAttribute + * __cuFuncSetAttribute = dlfcn.dlsym(handle, 'cuFuncSetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuFuncGetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2348 + * __cuFuncGetAttribute = dlfcn.dlsym(handle, 'cuFuncGetAttribute') + * global __cuFuncSetAttribute + * __cuFuncSetAttribute = dlfcn.dlsym(handle, 'cuFuncSetAttribute') # <<<<<<<<<<<<<< + * global __cuFuncSetCacheConfig + * __cuFuncSetCacheConfig = dlfcn.dlsym(handle, 'cuFuncSetCacheConfig') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuFuncSetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2350 + * __cuFuncSetAttribute = dlfcn.dlsym(handle, 'cuFuncSetAttribute') + * global __cuFuncSetCacheConfig + * __cuFuncSetCacheConfig = dlfcn.dlsym(handle, 'cuFuncSetCacheConfig') # <<<<<<<<<<<<<< + * global __cuFuncGetModule + * __cuFuncGetModule = dlfcn.dlsym(handle, 'cuFuncGetModule') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetCacheConfig = dlsym(__pyx_v_handle, ((char const *)"cuFuncSetCacheConfig")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2352 + * __cuFuncSetCacheConfig = dlfcn.dlsym(handle, 'cuFuncSetCacheConfig') + * global __cuFuncGetModule + * __cuFuncGetModule = dlfcn.dlsym(handle, 'cuFuncGetModule') # <<<<<<<<<<<<<< + * global __cuFuncGetName + * __cuFuncGetName = dlfcn.dlsym(handle, 'cuFuncGetName') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetModule = dlsym(__pyx_v_handle, ((char const *)"cuFuncGetModule")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2354 + * __cuFuncGetModule = dlfcn.dlsym(handle, 'cuFuncGetModule') + * global __cuFuncGetName + * __cuFuncGetName = dlfcn.dlsym(handle, 'cuFuncGetName') # <<<<<<<<<<<<<< + * global __cuFuncGetParamInfo + * __cuFuncGetParamInfo = dlfcn.dlsym(handle, 'cuFuncGetParamInfo') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetName = dlsym(__pyx_v_handle, ((char const *)"cuFuncGetName")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2356 + * __cuFuncGetName = dlfcn.dlsym(handle, 'cuFuncGetName') + * global __cuFuncGetParamInfo + * __cuFuncGetParamInfo = dlfcn.dlsym(handle, 'cuFuncGetParamInfo') # <<<<<<<<<<<<<< + * global __cuFuncIsLoaded + * __cuFuncIsLoaded = dlfcn.dlsym(handle, 'cuFuncIsLoaded') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetParamInfo = dlsym(__pyx_v_handle, ((char const *)"cuFuncGetParamInfo")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2358 + * __cuFuncGetParamInfo = dlfcn.dlsym(handle, 'cuFuncGetParamInfo') + * global __cuFuncIsLoaded + * __cuFuncIsLoaded = dlfcn.dlsym(handle, 'cuFuncIsLoaded') # <<<<<<<<<<<<<< + * global __cuFuncLoad + * __cuFuncLoad = dlfcn.dlsym(handle, 'cuFuncLoad') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncIsLoaded = dlsym(__pyx_v_handle, ((char const *)"cuFuncIsLoaded")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2360 + * __cuFuncIsLoaded = dlfcn.dlsym(handle, 'cuFuncIsLoaded') + * global __cuFuncLoad + * __cuFuncLoad = dlfcn.dlsym(handle, 'cuFuncLoad') # <<<<<<<<<<<<<< + * global __cuLaunchCooperativeKernelMultiDevice + * __cuLaunchCooperativeKernelMultiDevice = dlfcn.dlsym(handle, 'cuLaunchCooperativeKernelMultiDevice') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncLoad = dlsym(__pyx_v_handle, ((char const *)"cuFuncLoad")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2362 + * __cuFuncLoad = dlfcn.dlsym(handle, 'cuFuncLoad') + * global __cuLaunchCooperativeKernelMultiDevice + * __cuLaunchCooperativeKernelMultiDevice = dlfcn.dlsym(handle, 'cuLaunchCooperativeKernelMultiDevice') # <<<<<<<<<<<<<< + * global __cuFuncSetBlockShape + * __cuFuncSetBlockShape = dlfcn.dlsym(handle, 'cuFuncSetBlockShape') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernelMultiDevice = dlsym(__pyx_v_handle, ((char const *)"cuLaunchCooperativeKernelMultiDevice")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2364 + * __cuLaunchCooperativeKernelMultiDevice = dlfcn.dlsym(handle, 'cuLaunchCooperativeKernelMultiDevice') + * global __cuFuncSetBlockShape + * __cuFuncSetBlockShape = dlfcn.dlsym(handle, 'cuFuncSetBlockShape') # <<<<<<<<<<<<<< + * global __cuFuncSetSharedSize + * __cuFuncSetSharedSize = dlfcn.dlsym(handle, 'cuFuncSetSharedSize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetBlockShape = dlsym(__pyx_v_handle, ((char const *)"cuFuncSetBlockShape")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2366 + * __cuFuncSetBlockShape = dlfcn.dlsym(handle, 'cuFuncSetBlockShape') + * global __cuFuncSetSharedSize + * __cuFuncSetSharedSize = dlfcn.dlsym(handle, 'cuFuncSetSharedSize') # <<<<<<<<<<<<<< + * global __cuParamSetSize + * __cuParamSetSize = dlfcn.dlsym(handle, 'cuParamSetSize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedSize = dlsym(__pyx_v_handle, ((char const *)"cuFuncSetSharedSize")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2368 + * __cuFuncSetSharedSize = dlfcn.dlsym(handle, 'cuFuncSetSharedSize') + * global __cuParamSetSize + * __cuParamSetSize = dlfcn.dlsym(handle, 'cuParamSetSize') # <<<<<<<<<<<<<< + * global __cuParamSeti + * __cuParamSeti = dlfcn.dlsym(handle, 'cuParamSeti') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetSize = dlsym(__pyx_v_handle, ((char const *)"cuParamSetSize")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2370 + * __cuParamSetSize = dlfcn.dlsym(handle, 'cuParamSetSize') + * global __cuParamSeti + * __cuParamSeti = dlfcn.dlsym(handle, 'cuParamSeti') # <<<<<<<<<<<<<< + * global __cuParamSetf + * __cuParamSetf = dlfcn.dlsym(handle, 'cuParamSetf') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSeti = dlsym(__pyx_v_handle, ((char const *)"cuParamSeti")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2372 + * __cuParamSeti = dlfcn.dlsym(handle, 'cuParamSeti') + * global __cuParamSetf + * __cuParamSetf = dlfcn.dlsym(handle, 'cuParamSetf') # <<<<<<<<<<<<<< + * global __cuParamSetv + * __cuParamSetv = dlfcn.dlsym(handle, 'cuParamSetv') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetf = dlsym(__pyx_v_handle, ((char const *)"cuParamSetf")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2374 + * __cuParamSetf = dlfcn.dlsym(handle, 'cuParamSetf') + * global __cuParamSetv + * __cuParamSetv = dlfcn.dlsym(handle, 'cuParamSetv') # <<<<<<<<<<<<<< + * global __cuLaunch + * __cuLaunch = dlfcn.dlsym(handle, 'cuLaunch') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetv = dlsym(__pyx_v_handle, ((char const *)"cuParamSetv")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2376 + * __cuParamSetv = dlfcn.dlsym(handle, 'cuParamSetv') + * global __cuLaunch + * __cuLaunch = dlfcn.dlsym(handle, 'cuLaunch') # <<<<<<<<<<<<<< + * global __cuLaunchGrid + * __cuLaunchGrid = dlfcn.dlsym(handle, 'cuLaunchGrid') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunch = dlsym(__pyx_v_handle, ((char const *)"cuLaunch")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2378 + * __cuLaunch = dlfcn.dlsym(handle, 'cuLaunch') + * global __cuLaunchGrid + * __cuLaunchGrid = dlfcn.dlsym(handle, 'cuLaunchGrid') # <<<<<<<<<<<<<< + * global __cuLaunchGridAsync + * __cuLaunchGridAsync = dlfcn.dlsym(handle, 'cuLaunchGridAsync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGrid = dlsym(__pyx_v_handle, ((char const *)"cuLaunchGrid")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2380 + * __cuLaunchGrid = dlfcn.dlsym(handle, 'cuLaunchGrid') + * global __cuLaunchGridAsync + * __cuLaunchGridAsync = dlfcn.dlsym(handle, 'cuLaunchGridAsync') # <<<<<<<<<<<<<< + * global __cuParamSetTexRef + * __cuParamSetTexRef = dlfcn.dlsym(handle, 'cuParamSetTexRef') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGridAsync = dlsym(__pyx_v_handle, ((char const *)"cuLaunchGridAsync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2382 + * __cuLaunchGridAsync = dlfcn.dlsym(handle, 'cuLaunchGridAsync') + * global __cuParamSetTexRef + * __cuParamSetTexRef = dlfcn.dlsym(handle, 'cuParamSetTexRef') # <<<<<<<<<<<<<< + * global __cuFuncSetSharedMemConfig + * __cuFuncSetSharedMemConfig = dlfcn.dlsym(handle, 'cuFuncSetSharedMemConfig') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetTexRef = dlsym(__pyx_v_handle, ((char const *)"cuParamSetTexRef")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2384 + * __cuParamSetTexRef = dlfcn.dlsym(handle, 'cuParamSetTexRef') + * global __cuFuncSetSharedMemConfig + * __cuFuncSetSharedMemConfig = dlfcn.dlsym(handle, 'cuFuncSetSharedMemConfig') # <<<<<<<<<<<<<< + * global __cuGraphCreate + * __cuGraphCreate = dlfcn.dlsym(handle, 'cuGraphCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedMemConfig = dlsym(__pyx_v_handle, ((char const *)"cuFuncSetSharedMemConfig")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2386 + * __cuFuncSetSharedMemConfig = dlfcn.dlsym(handle, 'cuFuncSetSharedMemConfig') + * global __cuGraphCreate + * __cuGraphCreate = dlfcn.dlsym(handle, 'cuGraphCreate') # <<<<<<<<<<<<<< + * global __cuGraphAddKernelNode_v2 + * __cuGraphAddKernelNode_v2 = dlfcn.dlsym(handle, 'cuGraphAddKernelNode_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphCreate = dlsym(__pyx_v_handle, ((char const *)"cuGraphCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2388 + * __cuGraphCreate = dlfcn.dlsym(handle, 'cuGraphCreate') + * global __cuGraphAddKernelNode_v2 + * __cuGraphAddKernelNode_v2 = dlfcn.dlsym(handle, 'cuGraphAddKernelNode_v2') # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeGetParams_v2 + * __cuGraphKernelNodeGetParams_v2 = dlfcn.dlsym(handle, 'cuGraphKernelNodeGetParams_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddKernelNode_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddKernelNode_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2390 + * __cuGraphAddKernelNode_v2 = dlfcn.dlsym(handle, 'cuGraphAddKernelNode_v2') + * global __cuGraphKernelNodeGetParams_v2 + * __cuGraphKernelNodeGetParams_v2 = dlfcn.dlsym(handle, 'cuGraphKernelNodeGetParams_v2') # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeSetParams_v2 + * __cuGraphKernelNodeSetParams_v2 = dlfcn.dlsym(handle, 'cuGraphKernelNodeSetParams_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetParams_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGraphKernelNodeGetParams_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2392 + * __cuGraphKernelNodeGetParams_v2 = dlfcn.dlsym(handle, 'cuGraphKernelNodeGetParams_v2') + * global __cuGraphKernelNodeSetParams_v2 + * __cuGraphKernelNodeSetParams_v2 = dlfcn.dlsym(handle, 'cuGraphKernelNodeSetParams_v2') # <<<<<<<<<<<<<< + * global __cuGraphAddMemcpyNode + * __cuGraphAddMemcpyNode = dlfcn.dlsym(handle, 'cuGraphAddMemcpyNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetParams_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGraphKernelNodeSetParams_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2394 + * __cuGraphKernelNodeSetParams_v2 = dlfcn.dlsym(handle, 'cuGraphKernelNodeSetParams_v2') + * global __cuGraphAddMemcpyNode + * __cuGraphAddMemcpyNode = dlfcn.dlsym(handle, 'cuGraphAddMemcpyNode') # <<<<<<<<<<<<<< + * global __cuGraphMemcpyNodeGetParams + * __cuGraphMemcpyNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemcpyNodeGetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemcpyNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddMemcpyNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2396 + * __cuGraphAddMemcpyNode = dlfcn.dlsym(handle, 'cuGraphAddMemcpyNode') + * global __cuGraphMemcpyNodeGetParams + * __cuGraphMemcpyNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemcpyNodeGetParams') # <<<<<<<<<<<<<< + * global __cuGraphMemcpyNodeSetParams + * __cuGraphMemcpyNodeSetParams = dlfcn.dlsym(handle, 'cuGraphMemcpyNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeGetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphMemcpyNodeGetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2398 + * __cuGraphMemcpyNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemcpyNodeGetParams') + * global __cuGraphMemcpyNodeSetParams + * __cuGraphMemcpyNodeSetParams = dlfcn.dlsym(handle, 'cuGraphMemcpyNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphAddMemsetNode + * __cuGraphAddMemsetNode = dlfcn.dlsym(handle, 'cuGraphAddMemsetNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphMemcpyNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2400 + * __cuGraphMemcpyNodeSetParams = dlfcn.dlsym(handle, 'cuGraphMemcpyNodeSetParams') + * global __cuGraphAddMemsetNode + * __cuGraphAddMemsetNode = dlfcn.dlsym(handle, 'cuGraphAddMemsetNode') # <<<<<<<<<<<<<< + * global __cuGraphMemsetNodeGetParams + * __cuGraphMemsetNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemsetNodeGetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemsetNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddMemsetNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2402 + * __cuGraphAddMemsetNode = dlfcn.dlsym(handle, 'cuGraphAddMemsetNode') + * global __cuGraphMemsetNodeGetParams + * __cuGraphMemsetNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemsetNodeGetParams') # <<<<<<<<<<<<<< + * global __cuGraphMemsetNodeSetParams + * __cuGraphMemsetNodeSetParams = dlfcn.dlsym(handle, 'cuGraphMemsetNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeGetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphMemsetNodeGetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2404 + * __cuGraphMemsetNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemsetNodeGetParams') + * global __cuGraphMemsetNodeSetParams + * __cuGraphMemsetNodeSetParams = dlfcn.dlsym(handle, 'cuGraphMemsetNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphAddHostNode + * __cuGraphAddHostNode = dlfcn.dlsym(handle, 'cuGraphAddHostNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphMemsetNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2406 + * __cuGraphMemsetNodeSetParams = dlfcn.dlsym(handle, 'cuGraphMemsetNodeSetParams') + * global __cuGraphAddHostNode + * __cuGraphAddHostNode = dlfcn.dlsym(handle, 'cuGraphAddHostNode') # <<<<<<<<<<<<<< + * global __cuGraphHostNodeGetParams + * __cuGraphHostNodeGetParams = dlfcn.dlsym(handle, 'cuGraphHostNodeGetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddHostNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddHostNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2408 + * __cuGraphAddHostNode = dlfcn.dlsym(handle, 'cuGraphAddHostNode') + * global __cuGraphHostNodeGetParams + * __cuGraphHostNodeGetParams = dlfcn.dlsym(handle, 'cuGraphHostNodeGetParams') # <<<<<<<<<<<<<< + * global __cuGraphHostNodeSetParams + * __cuGraphHostNodeSetParams = dlfcn.dlsym(handle, 'cuGraphHostNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeGetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphHostNodeGetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2410 + * __cuGraphHostNodeGetParams = dlfcn.dlsym(handle, 'cuGraphHostNodeGetParams') + * global __cuGraphHostNodeSetParams + * __cuGraphHostNodeSetParams = dlfcn.dlsym(handle, 'cuGraphHostNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphAddChildGraphNode + * __cuGraphAddChildGraphNode = dlfcn.dlsym(handle, 'cuGraphAddChildGraphNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphHostNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2412 + * __cuGraphHostNodeSetParams = dlfcn.dlsym(handle, 'cuGraphHostNodeSetParams') + * global __cuGraphAddChildGraphNode + * __cuGraphAddChildGraphNode = dlfcn.dlsym(handle, 'cuGraphAddChildGraphNode') # <<<<<<<<<<<<<< + * global __cuGraphChildGraphNodeGetGraph + * __cuGraphChildGraphNodeGetGraph = dlfcn.dlsym(handle, 'cuGraphChildGraphNodeGetGraph') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddChildGraphNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddChildGraphNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2414 + * __cuGraphAddChildGraphNode = dlfcn.dlsym(handle, 'cuGraphAddChildGraphNode') + * global __cuGraphChildGraphNodeGetGraph + * __cuGraphChildGraphNodeGetGraph = dlfcn.dlsym(handle, 'cuGraphChildGraphNodeGetGraph') # <<<<<<<<<<<<<< + * global __cuGraphAddEmptyNode + * __cuGraphAddEmptyNode = dlfcn.dlsym(handle, 'cuGraphAddEmptyNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphChildGraphNodeGetGraph = dlsym(__pyx_v_handle, ((char const *)"cuGraphChildGraphNodeGetGraph")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2416 + * __cuGraphChildGraphNodeGetGraph = dlfcn.dlsym(handle, 'cuGraphChildGraphNodeGetGraph') + * global __cuGraphAddEmptyNode + * __cuGraphAddEmptyNode = dlfcn.dlsym(handle, 'cuGraphAddEmptyNode') # <<<<<<<<<<<<<< + * global __cuGraphAddEventRecordNode + * __cuGraphAddEventRecordNode = dlfcn.dlsym(handle, 'cuGraphAddEventRecordNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEmptyNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddEmptyNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2418 + * __cuGraphAddEmptyNode = dlfcn.dlsym(handle, 'cuGraphAddEmptyNode') + * global __cuGraphAddEventRecordNode + * __cuGraphAddEventRecordNode = dlfcn.dlsym(handle, 'cuGraphAddEventRecordNode') # <<<<<<<<<<<<<< + * global __cuGraphEventRecordNodeGetEvent + * __cuGraphEventRecordNodeGetEvent = dlfcn.dlsym(handle, 'cuGraphEventRecordNodeGetEvent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventRecordNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddEventRecordNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2420 + * __cuGraphAddEventRecordNode = dlfcn.dlsym(handle, 'cuGraphAddEventRecordNode') + * global __cuGraphEventRecordNodeGetEvent + * __cuGraphEventRecordNodeGetEvent = dlfcn.dlsym(handle, 'cuGraphEventRecordNodeGetEvent') # <<<<<<<<<<<<<< + * global __cuGraphEventRecordNodeSetEvent + * __cuGraphEventRecordNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphEventRecordNodeSetEvent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeGetEvent = dlsym(__pyx_v_handle, ((char const *)"cuGraphEventRecordNodeGetEvent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2422 + * __cuGraphEventRecordNodeGetEvent = dlfcn.dlsym(handle, 'cuGraphEventRecordNodeGetEvent') + * global __cuGraphEventRecordNodeSetEvent + * __cuGraphEventRecordNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphEventRecordNodeSetEvent') # <<<<<<<<<<<<<< + * global __cuGraphAddEventWaitNode + * __cuGraphAddEventWaitNode = dlfcn.dlsym(handle, 'cuGraphAddEventWaitNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeSetEvent = dlsym(__pyx_v_handle, ((char const *)"cuGraphEventRecordNodeSetEvent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2424 + * __cuGraphEventRecordNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphEventRecordNodeSetEvent') + * global __cuGraphAddEventWaitNode + * __cuGraphAddEventWaitNode = dlfcn.dlsym(handle, 'cuGraphAddEventWaitNode') # <<<<<<<<<<<<<< + * global __cuGraphEventWaitNodeGetEvent + * __cuGraphEventWaitNodeGetEvent = dlfcn.dlsym(handle, 'cuGraphEventWaitNodeGetEvent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventWaitNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddEventWaitNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2426 + * __cuGraphAddEventWaitNode = dlfcn.dlsym(handle, 'cuGraphAddEventWaitNode') + * global __cuGraphEventWaitNodeGetEvent + * __cuGraphEventWaitNodeGetEvent = dlfcn.dlsym(handle, 'cuGraphEventWaitNodeGetEvent') # <<<<<<<<<<<<<< + * global __cuGraphEventWaitNodeSetEvent + * __cuGraphEventWaitNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphEventWaitNodeSetEvent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeGetEvent = dlsym(__pyx_v_handle, ((char const *)"cuGraphEventWaitNodeGetEvent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2428 + * __cuGraphEventWaitNodeGetEvent = dlfcn.dlsym(handle, 'cuGraphEventWaitNodeGetEvent') + * global __cuGraphEventWaitNodeSetEvent + * __cuGraphEventWaitNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphEventWaitNodeSetEvent') # <<<<<<<<<<<<<< + * global __cuGraphAddExternalSemaphoresSignalNode + * __cuGraphAddExternalSemaphoresSignalNode = dlfcn.dlsym(handle, 'cuGraphAddExternalSemaphoresSignalNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeSetEvent = dlsym(__pyx_v_handle, ((char const *)"cuGraphEventWaitNodeSetEvent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2430 + * __cuGraphEventWaitNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphEventWaitNodeSetEvent') + * global __cuGraphAddExternalSemaphoresSignalNode + * __cuGraphAddExternalSemaphoresSignalNode = dlfcn.dlsym(handle, 'cuGraphAddExternalSemaphoresSignalNode') # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresSignalNodeGetParams + * __cuGraphExternalSemaphoresSignalNodeGetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresSignalNodeGetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresSignalNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddExternalSemaphoresSignalNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2432 + * __cuGraphAddExternalSemaphoresSignalNode = dlfcn.dlsym(handle, 'cuGraphAddExternalSemaphoresSignalNode') + * global __cuGraphExternalSemaphoresSignalNodeGetParams + * __cuGraphExternalSemaphoresSignalNodeGetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresSignalNodeGetParams') # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresSignalNodeSetParams + * __cuGraphExternalSemaphoresSignalNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresSignalNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeGetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphExternalSemaphoresSignalNodeGetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2434 + * __cuGraphExternalSemaphoresSignalNodeGetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresSignalNodeGetParams') + * global __cuGraphExternalSemaphoresSignalNodeSetParams + * __cuGraphExternalSemaphoresSignalNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresSignalNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphAddExternalSemaphoresWaitNode + * __cuGraphAddExternalSemaphoresWaitNode = dlfcn.dlsym(handle, 'cuGraphAddExternalSemaphoresWaitNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphExternalSemaphoresSignalNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2436 + * __cuGraphExternalSemaphoresSignalNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresSignalNodeSetParams') + * global __cuGraphAddExternalSemaphoresWaitNode + * __cuGraphAddExternalSemaphoresWaitNode = dlfcn.dlsym(handle, 'cuGraphAddExternalSemaphoresWaitNode') # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresWaitNodeGetParams + * __cuGraphExternalSemaphoresWaitNodeGetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresWaitNodeGetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresWaitNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddExternalSemaphoresWaitNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2438 + * __cuGraphAddExternalSemaphoresWaitNode = dlfcn.dlsym(handle, 'cuGraphAddExternalSemaphoresWaitNode') + * global __cuGraphExternalSemaphoresWaitNodeGetParams + * __cuGraphExternalSemaphoresWaitNodeGetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresWaitNodeGetParams') # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresWaitNodeSetParams + * __cuGraphExternalSemaphoresWaitNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresWaitNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeGetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphExternalSemaphoresWaitNodeGetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2440 + * __cuGraphExternalSemaphoresWaitNodeGetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresWaitNodeGetParams') + * global __cuGraphExternalSemaphoresWaitNodeSetParams + * __cuGraphExternalSemaphoresWaitNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresWaitNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphAddBatchMemOpNode + * __cuGraphAddBatchMemOpNode = dlfcn.dlsym(handle, 'cuGraphAddBatchMemOpNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphExternalSemaphoresWaitNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2442 + * __cuGraphExternalSemaphoresWaitNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresWaitNodeSetParams') + * global __cuGraphAddBatchMemOpNode + * __cuGraphAddBatchMemOpNode = dlfcn.dlsym(handle, 'cuGraphAddBatchMemOpNode') # <<<<<<<<<<<<<< + * global __cuGraphBatchMemOpNodeGetParams + * __cuGraphBatchMemOpNodeGetParams = dlfcn.dlsym(handle, 'cuGraphBatchMemOpNodeGetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddBatchMemOpNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddBatchMemOpNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2444 + * __cuGraphAddBatchMemOpNode = dlfcn.dlsym(handle, 'cuGraphAddBatchMemOpNode') + * global __cuGraphBatchMemOpNodeGetParams + * __cuGraphBatchMemOpNodeGetParams = dlfcn.dlsym(handle, 'cuGraphBatchMemOpNodeGetParams') # <<<<<<<<<<<<<< + * global __cuGraphBatchMemOpNodeSetParams + * __cuGraphBatchMemOpNodeSetParams = dlfcn.dlsym(handle, 'cuGraphBatchMemOpNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeGetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphBatchMemOpNodeGetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2446 + * __cuGraphBatchMemOpNodeGetParams = dlfcn.dlsym(handle, 'cuGraphBatchMemOpNodeGetParams') + * global __cuGraphBatchMemOpNodeSetParams + * __cuGraphBatchMemOpNodeSetParams = dlfcn.dlsym(handle, 'cuGraphBatchMemOpNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphExecBatchMemOpNodeSetParams + * __cuGraphExecBatchMemOpNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecBatchMemOpNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphBatchMemOpNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2448 + * __cuGraphBatchMemOpNodeSetParams = dlfcn.dlsym(handle, 'cuGraphBatchMemOpNodeSetParams') + * global __cuGraphExecBatchMemOpNodeSetParams + * __cuGraphExecBatchMemOpNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecBatchMemOpNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphAddMemAllocNode + * __cuGraphAddMemAllocNode = dlfcn.dlsym(handle, 'cuGraphAddMemAllocNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecBatchMemOpNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecBatchMemOpNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2450 + * __cuGraphExecBatchMemOpNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecBatchMemOpNodeSetParams') + * global __cuGraphAddMemAllocNode + * __cuGraphAddMemAllocNode = dlfcn.dlsym(handle, 'cuGraphAddMemAllocNode') # <<<<<<<<<<<<<< + * global __cuGraphMemAllocNodeGetParams + * __cuGraphMemAllocNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemAllocNodeGetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemAllocNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddMemAllocNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2452 + * __cuGraphAddMemAllocNode = dlfcn.dlsym(handle, 'cuGraphAddMemAllocNode') + * global __cuGraphMemAllocNodeGetParams + * __cuGraphMemAllocNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemAllocNodeGetParams') # <<<<<<<<<<<<<< + * global __cuGraphAddMemFreeNode + * __cuGraphAddMemFreeNode = dlfcn.dlsym(handle, 'cuGraphAddMemFreeNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemAllocNodeGetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphMemAllocNodeGetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2454 + * __cuGraphMemAllocNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemAllocNodeGetParams') + * global __cuGraphAddMemFreeNode + * __cuGraphAddMemFreeNode = dlfcn.dlsym(handle, 'cuGraphAddMemFreeNode') # <<<<<<<<<<<<<< + * global __cuGraphMemFreeNodeGetParams + * __cuGraphMemFreeNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemFreeNodeGetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemFreeNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddMemFreeNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2456 + * __cuGraphAddMemFreeNode = dlfcn.dlsym(handle, 'cuGraphAddMemFreeNode') + * global __cuGraphMemFreeNodeGetParams + * __cuGraphMemFreeNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemFreeNodeGetParams') # <<<<<<<<<<<<<< + * global __cuDeviceGraphMemTrim + * __cuDeviceGraphMemTrim = dlfcn.dlsym(handle, 'cuDeviceGraphMemTrim') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemFreeNodeGetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphMemFreeNodeGetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2458 + * __cuGraphMemFreeNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemFreeNodeGetParams') + * global __cuDeviceGraphMemTrim + * __cuDeviceGraphMemTrim = dlfcn.dlsym(handle, 'cuDeviceGraphMemTrim') # <<<<<<<<<<<<<< + * global __cuDeviceGetGraphMemAttribute + * __cuDeviceGetGraphMemAttribute = dlfcn.dlsym(handle, 'cuDeviceGetGraphMemAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGraphMemTrim = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGraphMemTrim")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2460 + * __cuDeviceGraphMemTrim = dlfcn.dlsym(handle, 'cuDeviceGraphMemTrim') + * global __cuDeviceGetGraphMemAttribute + * __cuDeviceGetGraphMemAttribute = dlfcn.dlsym(handle, 'cuDeviceGetGraphMemAttribute') # <<<<<<<<<<<<<< + * global __cuDeviceSetGraphMemAttribute + * __cuDeviceSetGraphMemAttribute = dlfcn.dlsym(handle, 'cuDeviceSetGraphMemAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetGraphMemAttribute = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetGraphMemAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2462 + * __cuDeviceGetGraphMemAttribute = dlfcn.dlsym(handle, 'cuDeviceGetGraphMemAttribute') + * global __cuDeviceSetGraphMemAttribute + * __cuDeviceSetGraphMemAttribute = dlfcn.dlsym(handle, 'cuDeviceSetGraphMemAttribute') # <<<<<<<<<<<<<< + * global __cuGraphClone + * __cuGraphClone = dlfcn.dlsym(handle, 'cuGraphClone') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetGraphMemAttribute = dlsym(__pyx_v_handle, ((char const *)"cuDeviceSetGraphMemAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2464 + * __cuDeviceSetGraphMemAttribute = dlfcn.dlsym(handle, 'cuDeviceSetGraphMemAttribute') + * global __cuGraphClone + * __cuGraphClone = dlfcn.dlsym(handle, 'cuGraphClone') # <<<<<<<<<<<<<< + * global __cuGraphNodeFindInClone + * __cuGraphNodeFindInClone = dlfcn.dlsym(handle, 'cuGraphNodeFindInClone') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphClone = dlsym(__pyx_v_handle, ((char const *)"cuGraphClone")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2466 + * __cuGraphClone = dlfcn.dlsym(handle, 'cuGraphClone') + * global __cuGraphNodeFindInClone + * __cuGraphNodeFindInClone = dlfcn.dlsym(handle, 'cuGraphNodeFindInClone') # <<<<<<<<<<<<<< + * global __cuGraphNodeGetType + * __cuGraphNodeGetType = dlfcn.dlsym(handle, 'cuGraphNodeGetType') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeFindInClone = dlsym(__pyx_v_handle, ((char const *)"cuGraphNodeFindInClone")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2468 + * __cuGraphNodeFindInClone = dlfcn.dlsym(handle, 'cuGraphNodeFindInClone') + * global __cuGraphNodeGetType + * __cuGraphNodeGetType = dlfcn.dlsym(handle, 'cuGraphNodeGetType') # <<<<<<<<<<<<<< + * global __cuGraphGetNodes + * __cuGraphGetNodes = dlfcn.dlsym(handle, 'cuGraphGetNodes') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetType = dlsym(__pyx_v_handle, ((char const *)"cuGraphNodeGetType")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2470 + * __cuGraphNodeGetType = dlfcn.dlsym(handle, 'cuGraphNodeGetType') + * global __cuGraphGetNodes + * __cuGraphGetNodes = dlfcn.dlsym(handle, 'cuGraphGetNodes') # <<<<<<<<<<<<<< + * global __cuGraphGetRootNodes + * __cuGraphGetRootNodes = dlfcn.dlsym(handle, 'cuGraphGetRootNodes') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetNodes = dlsym(__pyx_v_handle, ((char const *)"cuGraphGetNodes")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2472 + * __cuGraphGetNodes = dlfcn.dlsym(handle, 'cuGraphGetNodes') + * global __cuGraphGetRootNodes + * __cuGraphGetRootNodes = dlfcn.dlsym(handle, 'cuGraphGetRootNodes') # <<<<<<<<<<<<<< + * global __cuGraphGetEdges + * __cuGraphGetEdges = dlfcn.dlsym(handle, 'cuGraphGetEdges') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetRootNodes = dlsym(__pyx_v_handle, ((char const *)"cuGraphGetRootNodes")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2474 + * __cuGraphGetRootNodes = dlfcn.dlsym(handle, 'cuGraphGetRootNodes') + * global __cuGraphGetEdges + * __cuGraphGetEdges = dlfcn.dlsym(handle, 'cuGraphGetEdges') # <<<<<<<<<<<<<< + * global __cuGraphGetEdges_v2 + * __cuGraphGetEdges_v2 = dlfcn.dlsym(handle, 'cuGraphGetEdges_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges = dlsym(__pyx_v_handle, ((char const *)"cuGraphGetEdges")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2476 + * __cuGraphGetEdges = dlfcn.dlsym(handle, 'cuGraphGetEdges') + * global __cuGraphGetEdges_v2 + * __cuGraphGetEdges_v2 = dlfcn.dlsym(handle, 'cuGraphGetEdges_v2') # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependencies + * __cuGraphNodeGetDependencies = dlfcn.dlsym(handle, 'cuGraphNodeGetDependencies') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGraphGetEdges_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2478 + * __cuGraphGetEdges_v2 = dlfcn.dlsym(handle, 'cuGraphGetEdges_v2') + * global __cuGraphNodeGetDependencies + * __cuGraphNodeGetDependencies = dlfcn.dlsym(handle, 'cuGraphNodeGetDependencies') # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependencies_v2 + * __cuGraphNodeGetDependencies_v2 = dlfcn.dlsym(handle, 'cuGraphNodeGetDependencies_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies = dlsym(__pyx_v_handle, ((char const *)"cuGraphNodeGetDependencies")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2480 + * __cuGraphNodeGetDependencies = dlfcn.dlsym(handle, 'cuGraphNodeGetDependencies') + * global __cuGraphNodeGetDependencies_v2 + * __cuGraphNodeGetDependencies_v2 = dlfcn.dlsym(handle, 'cuGraphNodeGetDependencies_v2') # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependentNodes + * __cuGraphNodeGetDependentNodes = dlfcn.dlsym(handle, 'cuGraphNodeGetDependentNodes') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGraphNodeGetDependencies_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2482 + * __cuGraphNodeGetDependencies_v2 = dlfcn.dlsym(handle, 'cuGraphNodeGetDependencies_v2') + * global __cuGraphNodeGetDependentNodes + * __cuGraphNodeGetDependentNodes = dlfcn.dlsym(handle, 'cuGraphNodeGetDependentNodes') # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependentNodes_v2 + * __cuGraphNodeGetDependentNodes_v2 = dlfcn.dlsym(handle, 'cuGraphNodeGetDependentNodes_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes = dlsym(__pyx_v_handle, ((char const *)"cuGraphNodeGetDependentNodes")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2484 + * __cuGraphNodeGetDependentNodes = dlfcn.dlsym(handle, 'cuGraphNodeGetDependentNodes') + * global __cuGraphNodeGetDependentNodes_v2 + * __cuGraphNodeGetDependentNodes_v2 = dlfcn.dlsym(handle, 'cuGraphNodeGetDependentNodes_v2') # <<<<<<<<<<<<<< + * global __cuGraphAddDependencies + * __cuGraphAddDependencies = dlfcn.dlsym(handle, 'cuGraphAddDependencies') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGraphNodeGetDependentNodes_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2486 + * __cuGraphNodeGetDependentNodes_v2 = dlfcn.dlsym(handle, 'cuGraphNodeGetDependentNodes_v2') + * global __cuGraphAddDependencies + * __cuGraphAddDependencies = dlfcn.dlsym(handle, 'cuGraphAddDependencies') # <<<<<<<<<<<<<< + * global __cuGraphAddDependencies_v2 + * __cuGraphAddDependencies_v2 = dlfcn.dlsym(handle, 'cuGraphAddDependencies_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddDependencies")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2488 + * __cuGraphAddDependencies = dlfcn.dlsym(handle, 'cuGraphAddDependencies') + * global __cuGraphAddDependencies_v2 + * __cuGraphAddDependencies_v2 = dlfcn.dlsym(handle, 'cuGraphAddDependencies_v2') # <<<<<<<<<<<<<< + * global __cuGraphRemoveDependencies + * __cuGraphRemoveDependencies = dlfcn.dlsym(handle, 'cuGraphRemoveDependencies') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddDependencies_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2490 + * __cuGraphAddDependencies_v2 = dlfcn.dlsym(handle, 'cuGraphAddDependencies_v2') + * global __cuGraphRemoveDependencies + * __cuGraphRemoveDependencies = dlfcn.dlsym(handle, 'cuGraphRemoveDependencies') # <<<<<<<<<<<<<< + * global __cuGraphRemoveDependencies_v2 + * __cuGraphRemoveDependencies_v2 = dlfcn.dlsym(handle, 'cuGraphRemoveDependencies_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies = dlsym(__pyx_v_handle, ((char const *)"cuGraphRemoveDependencies")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2492 + * __cuGraphRemoveDependencies = dlfcn.dlsym(handle, 'cuGraphRemoveDependencies') + * global __cuGraphRemoveDependencies_v2 + * __cuGraphRemoveDependencies_v2 = dlfcn.dlsym(handle, 'cuGraphRemoveDependencies_v2') # <<<<<<<<<<<<<< + * global __cuGraphDestroyNode + * __cuGraphDestroyNode = dlfcn.dlsym(handle, 'cuGraphDestroyNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGraphRemoveDependencies_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2494 + * __cuGraphRemoveDependencies_v2 = dlfcn.dlsym(handle, 'cuGraphRemoveDependencies_v2') + * global __cuGraphDestroyNode + * __cuGraphDestroyNode = dlfcn.dlsym(handle, 'cuGraphDestroyNode') # <<<<<<<<<<<<<< + * global __cuGraphInstantiateWithFlags + * __cuGraphInstantiateWithFlags = dlfcn.dlsym(handle, 'cuGraphInstantiateWithFlags') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroyNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphDestroyNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2496 + * __cuGraphDestroyNode = dlfcn.dlsym(handle, 'cuGraphDestroyNode') + * global __cuGraphInstantiateWithFlags + * __cuGraphInstantiateWithFlags = dlfcn.dlsym(handle, 'cuGraphInstantiateWithFlags') # <<<<<<<<<<<<<< + * global __cuGraphExecGetFlags + * __cuGraphExecGetFlags = dlfcn.dlsym(handle, 'cuGraphExecGetFlags') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithFlags = dlsym(__pyx_v_handle, ((char const *)"cuGraphInstantiateWithFlags")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2498 + * __cuGraphInstantiateWithFlags = dlfcn.dlsym(handle, 'cuGraphInstantiateWithFlags') + * global __cuGraphExecGetFlags + * __cuGraphExecGetFlags = dlfcn.dlsym(handle, 'cuGraphExecGetFlags') # <<<<<<<<<<<<<< + * global __cuGraphExecKernelNodeSetParams_v2 + * __cuGraphExecKernelNodeSetParams_v2 = dlfcn.dlsym(handle, 'cuGraphExecKernelNodeSetParams_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecGetFlags = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecGetFlags")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2500 + * __cuGraphExecGetFlags = dlfcn.dlsym(handle, 'cuGraphExecGetFlags') + * global __cuGraphExecKernelNodeSetParams_v2 + * __cuGraphExecKernelNodeSetParams_v2 = dlfcn.dlsym(handle, 'cuGraphExecKernelNodeSetParams_v2') # <<<<<<<<<<<<<< + * global __cuGraphExecMemcpyNodeSetParams + * __cuGraphExecMemcpyNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecMemcpyNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecKernelNodeSetParams_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecKernelNodeSetParams_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2502 + * __cuGraphExecKernelNodeSetParams_v2 = dlfcn.dlsym(handle, 'cuGraphExecKernelNodeSetParams_v2') + * global __cuGraphExecMemcpyNodeSetParams + * __cuGraphExecMemcpyNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecMemcpyNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphExecMemsetNodeSetParams + * __cuGraphExecMemsetNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecMemsetNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemcpyNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecMemcpyNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2504 + * __cuGraphExecMemcpyNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecMemcpyNodeSetParams') + * global __cuGraphExecMemsetNodeSetParams + * __cuGraphExecMemsetNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecMemsetNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphExecHostNodeSetParams + * __cuGraphExecHostNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecHostNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemsetNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecMemsetNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2506 + * __cuGraphExecMemsetNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecMemsetNodeSetParams') + * global __cuGraphExecHostNodeSetParams + * __cuGraphExecHostNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecHostNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphExecChildGraphNodeSetParams + * __cuGraphExecChildGraphNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecChildGraphNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecHostNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecHostNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2508 + * __cuGraphExecHostNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecHostNodeSetParams') + * global __cuGraphExecChildGraphNodeSetParams + * __cuGraphExecChildGraphNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecChildGraphNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphExecEventRecordNodeSetEvent + * __cuGraphExecEventRecordNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphExecEventRecordNodeSetEvent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecChildGraphNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecChildGraphNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2510 + * __cuGraphExecChildGraphNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecChildGraphNodeSetParams') + * global __cuGraphExecEventRecordNodeSetEvent + * __cuGraphExecEventRecordNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphExecEventRecordNodeSetEvent') # <<<<<<<<<<<<<< + * global __cuGraphExecEventWaitNodeSetEvent + * __cuGraphExecEventWaitNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphExecEventWaitNodeSetEvent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventRecordNodeSetEvent = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecEventRecordNodeSetEvent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2512 + * __cuGraphExecEventRecordNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphExecEventRecordNodeSetEvent') + * global __cuGraphExecEventWaitNodeSetEvent + * __cuGraphExecEventWaitNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphExecEventWaitNodeSetEvent') # <<<<<<<<<<<<<< + * global __cuGraphExecExternalSemaphoresSignalNodeSetParams + * __cuGraphExecExternalSemaphoresSignalNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecExternalSemaphoresSignalNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventWaitNodeSetEvent = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecEventWaitNodeSetEvent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2514 + * __cuGraphExecEventWaitNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphExecEventWaitNodeSetEvent') + * global __cuGraphExecExternalSemaphoresSignalNodeSetParams + * __cuGraphExecExternalSemaphoresSignalNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecExternalSemaphoresSignalNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphExecExternalSemaphoresWaitNodeSetParams + * __cuGraphExecExternalSemaphoresWaitNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecExternalSemaphoresWaitNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresSignalNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecExternalSemaphoresSignalNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2516 + * __cuGraphExecExternalSemaphoresSignalNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecExternalSemaphoresSignalNodeSetParams') + * global __cuGraphExecExternalSemaphoresWaitNodeSetParams + * __cuGraphExecExternalSemaphoresWaitNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecExternalSemaphoresWaitNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphNodeSetEnabled + * __cuGraphNodeSetEnabled = dlfcn.dlsym(handle, 'cuGraphNodeSetEnabled') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresWaitNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecExternalSemaphoresWaitNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2518 + * __cuGraphExecExternalSemaphoresWaitNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecExternalSemaphoresWaitNodeSetParams') + * global __cuGraphNodeSetEnabled + * __cuGraphNodeSetEnabled = dlfcn.dlsym(handle, 'cuGraphNodeSetEnabled') # <<<<<<<<<<<<<< + * global __cuGraphNodeGetEnabled + * __cuGraphNodeGetEnabled = dlfcn.dlsym(handle, 'cuGraphNodeGetEnabled') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetEnabled = dlsym(__pyx_v_handle, ((char const *)"cuGraphNodeSetEnabled")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2520 + * __cuGraphNodeSetEnabled = dlfcn.dlsym(handle, 'cuGraphNodeSetEnabled') + * global __cuGraphNodeGetEnabled + * __cuGraphNodeGetEnabled = dlfcn.dlsym(handle, 'cuGraphNodeGetEnabled') # <<<<<<<<<<<<<< + * global __cuGraphExecDestroy + * __cuGraphExecDestroy = dlfcn.dlsym(handle, 'cuGraphExecDestroy') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetEnabled = dlsym(__pyx_v_handle, ((char const *)"cuGraphNodeGetEnabled")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2522 + * __cuGraphNodeGetEnabled = dlfcn.dlsym(handle, 'cuGraphNodeGetEnabled') + * global __cuGraphExecDestroy + * __cuGraphExecDestroy = dlfcn.dlsym(handle, 'cuGraphExecDestroy') # <<<<<<<<<<<<<< + * global __cuGraphDestroy + * __cuGraphDestroy = dlfcn.dlsym(handle, 'cuGraphDestroy') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecDestroy = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecDestroy")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2524 + * __cuGraphExecDestroy = dlfcn.dlsym(handle, 'cuGraphExecDestroy') + * global __cuGraphDestroy + * __cuGraphDestroy = dlfcn.dlsym(handle, 'cuGraphDestroy') # <<<<<<<<<<<<<< + * global __cuGraphExecUpdate_v2 + * __cuGraphExecUpdate_v2 = dlfcn.dlsym(handle, 'cuGraphExecUpdate_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroy = dlsym(__pyx_v_handle, ((char const *)"cuGraphDestroy")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2526 + * __cuGraphDestroy = dlfcn.dlsym(handle, 'cuGraphDestroy') + * global __cuGraphExecUpdate_v2 + * __cuGraphExecUpdate_v2 = dlfcn.dlsym(handle, 'cuGraphExecUpdate_v2') # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeCopyAttributes + * __cuGraphKernelNodeCopyAttributes = dlfcn.dlsym(handle, 'cuGraphKernelNodeCopyAttributes') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecUpdate_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecUpdate_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2528 + * __cuGraphExecUpdate_v2 = dlfcn.dlsym(handle, 'cuGraphExecUpdate_v2') + * global __cuGraphKernelNodeCopyAttributes + * __cuGraphKernelNodeCopyAttributes = dlfcn.dlsym(handle, 'cuGraphKernelNodeCopyAttributes') # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeGetAttribute + * __cuGraphKernelNodeGetAttribute = dlfcn.dlsym(handle, 'cuGraphKernelNodeGetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeCopyAttributes = dlsym(__pyx_v_handle, ((char const *)"cuGraphKernelNodeCopyAttributes")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2530 + * __cuGraphKernelNodeCopyAttributes = dlfcn.dlsym(handle, 'cuGraphKernelNodeCopyAttributes') + * global __cuGraphKernelNodeGetAttribute + * __cuGraphKernelNodeGetAttribute = dlfcn.dlsym(handle, 'cuGraphKernelNodeGetAttribute') # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeSetAttribute + * __cuGraphKernelNodeSetAttribute = dlfcn.dlsym(handle, 'cuGraphKernelNodeSetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuGraphKernelNodeGetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2532 + * __cuGraphKernelNodeGetAttribute = dlfcn.dlsym(handle, 'cuGraphKernelNodeGetAttribute') + * global __cuGraphKernelNodeSetAttribute + * __cuGraphKernelNodeSetAttribute = dlfcn.dlsym(handle, 'cuGraphKernelNodeSetAttribute') # <<<<<<<<<<<<<< + * global __cuGraphDebugDotPrint + * __cuGraphDebugDotPrint = dlfcn.dlsym(handle, 'cuGraphDebugDotPrint') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuGraphKernelNodeSetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2534 + * __cuGraphKernelNodeSetAttribute = dlfcn.dlsym(handle, 'cuGraphKernelNodeSetAttribute') + * global __cuGraphDebugDotPrint + * __cuGraphDebugDotPrint = dlfcn.dlsym(handle, 'cuGraphDebugDotPrint') # <<<<<<<<<<<<<< + * global __cuUserObjectCreate + * __cuUserObjectCreate = dlfcn.dlsym(handle, 'cuUserObjectCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDebugDotPrint = dlsym(__pyx_v_handle, ((char const *)"cuGraphDebugDotPrint")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2536 + * __cuGraphDebugDotPrint = dlfcn.dlsym(handle, 'cuGraphDebugDotPrint') + * global __cuUserObjectCreate + * __cuUserObjectCreate = dlfcn.dlsym(handle, 'cuUserObjectCreate') # <<<<<<<<<<<<<< + * global __cuUserObjectRetain + * __cuUserObjectRetain = dlfcn.dlsym(handle, 'cuUserObjectRetain') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectCreate = dlsym(__pyx_v_handle, ((char const *)"cuUserObjectCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2538 + * __cuUserObjectCreate = dlfcn.dlsym(handle, 'cuUserObjectCreate') + * global __cuUserObjectRetain + * __cuUserObjectRetain = dlfcn.dlsym(handle, 'cuUserObjectRetain') # <<<<<<<<<<<<<< + * global __cuUserObjectRelease + * __cuUserObjectRelease = dlfcn.dlsym(handle, 'cuUserObjectRelease') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRetain = dlsym(__pyx_v_handle, ((char const *)"cuUserObjectRetain")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2540 + * __cuUserObjectRetain = dlfcn.dlsym(handle, 'cuUserObjectRetain') + * global __cuUserObjectRelease + * __cuUserObjectRelease = dlfcn.dlsym(handle, 'cuUserObjectRelease') # <<<<<<<<<<<<<< + * global __cuGraphRetainUserObject + * __cuGraphRetainUserObject = dlfcn.dlsym(handle, 'cuGraphRetainUserObject') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRelease = dlsym(__pyx_v_handle, ((char const *)"cuUserObjectRelease")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2542 + * __cuUserObjectRelease = dlfcn.dlsym(handle, 'cuUserObjectRelease') + * global __cuGraphRetainUserObject + * __cuGraphRetainUserObject = dlfcn.dlsym(handle, 'cuGraphRetainUserObject') # <<<<<<<<<<<<<< + * global __cuGraphReleaseUserObject + * __cuGraphReleaseUserObject = dlfcn.dlsym(handle, 'cuGraphReleaseUserObject') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRetainUserObject = dlsym(__pyx_v_handle, ((char const *)"cuGraphRetainUserObject")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2544 + * __cuGraphRetainUserObject = dlfcn.dlsym(handle, 'cuGraphRetainUserObject') + * global __cuGraphReleaseUserObject + * __cuGraphReleaseUserObject = dlfcn.dlsym(handle, 'cuGraphReleaseUserObject') # <<<<<<<<<<<<<< + * global __cuGraphAddNode + * __cuGraphAddNode = dlfcn.dlsym(handle, 'cuGraphAddNode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphReleaseUserObject = dlsym(__pyx_v_handle, ((char const *)"cuGraphReleaseUserObject")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2546 + * __cuGraphReleaseUserObject = dlfcn.dlsym(handle, 'cuGraphReleaseUserObject') + * global __cuGraphAddNode + * __cuGraphAddNode = dlfcn.dlsym(handle, 'cuGraphAddNode') # <<<<<<<<<<<<<< + * global __cuGraphAddNode_v2 + * __cuGraphAddNode_v2 = dlfcn.dlsym(handle, 'cuGraphAddNode_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddNode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2548 + * __cuGraphAddNode = dlfcn.dlsym(handle, 'cuGraphAddNode') + * global __cuGraphAddNode_v2 + * __cuGraphAddNode_v2 = dlfcn.dlsym(handle, 'cuGraphAddNode_v2') # <<<<<<<<<<<<<< + * global __cuGraphNodeSetParams + * __cuGraphNodeSetParams = dlfcn.dlsym(handle, 'cuGraphNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGraphAddNode_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2550 + * __cuGraphAddNode_v2 = dlfcn.dlsym(handle, 'cuGraphAddNode_v2') + * global __cuGraphNodeSetParams + * __cuGraphNodeSetParams = dlfcn.dlsym(handle, 'cuGraphNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphExecNodeSetParams + * __cuGraphExecNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecNodeSetParams') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2552 + * __cuGraphNodeSetParams = dlfcn.dlsym(handle, 'cuGraphNodeSetParams') + * global __cuGraphExecNodeSetParams + * __cuGraphExecNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecNodeSetParams') # <<<<<<<<<<<<<< + * global __cuGraphConditionalHandleCreate + * __cuGraphConditionalHandleCreate = dlfcn.dlsym(handle, 'cuGraphConditionalHandleCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecNodeSetParams = dlsym(__pyx_v_handle, ((char const *)"cuGraphExecNodeSetParams")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2554 + * __cuGraphExecNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecNodeSetParams') + * global __cuGraphConditionalHandleCreate + * __cuGraphConditionalHandleCreate = dlfcn.dlsym(handle, 'cuGraphConditionalHandleCreate') # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveBlocksPerMultiprocessor + * __cuOccupancyMaxActiveBlocksPerMultiprocessor = dlfcn.dlsym(handle, 'cuOccupancyMaxActiveBlocksPerMultiprocessor') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphConditionalHandleCreate = dlsym(__pyx_v_handle, ((char const *)"cuGraphConditionalHandleCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2556 + * __cuGraphConditionalHandleCreate = dlfcn.dlsym(handle, 'cuGraphConditionalHandleCreate') + * global __cuOccupancyMaxActiveBlocksPerMultiprocessor + * __cuOccupancyMaxActiveBlocksPerMultiprocessor = dlfcn.dlsym(handle, 'cuOccupancyMaxActiveBlocksPerMultiprocessor') # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + * __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags = dlfcn.dlsym(handle, 'cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessor = dlsym(__pyx_v_handle, ((char const *)"cuOccupancyMaxActiveBlocksPerMultiprocessor")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2558 + * __cuOccupancyMaxActiveBlocksPerMultiprocessor = dlfcn.dlsym(handle, 'cuOccupancyMaxActiveBlocksPerMultiprocessor') + * global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + * __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags = dlfcn.dlsym(handle, 'cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags') # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialBlockSize + * __cuOccupancyMaxPotentialBlockSize = dlfcn.dlsym(handle, 'cuOccupancyMaxPotentialBlockSize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags = dlsym(__pyx_v_handle, ((char const *)"cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2560 + * __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags = dlfcn.dlsym(handle, 'cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags') + * global __cuOccupancyMaxPotentialBlockSize + * __cuOccupancyMaxPotentialBlockSize = dlfcn.dlsym(handle, 'cuOccupancyMaxPotentialBlockSize') # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialBlockSizeWithFlags + * __cuOccupancyMaxPotentialBlockSizeWithFlags = dlfcn.dlsym(handle, 'cuOccupancyMaxPotentialBlockSizeWithFlags') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSize = dlsym(__pyx_v_handle, ((char const *)"cuOccupancyMaxPotentialBlockSize")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2562 + * __cuOccupancyMaxPotentialBlockSize = dlfcn.dlsym(handle, 'cuOccupancyMaxPotentialBlockSize') + * global __cuOccupancyMaxPotentialBlockSizeWithFlags + * __cuOccupancyMaxPotentialBlockSizeWithFlags = dlfcn.dlsym(handle, 'cuOccupancyMaxPotentialBlockSizeWithFlags') # <<<<<<<<<<<<<< + * global __cuOccupancyAvailableDynamicSMemPerBlock + * __cuOccupancyAvailableDynamicSMemPerBlock = dlfcn.dlsym(handle, 'cuOccupancyAvailableDynamicSMemPerBlock') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSizeWithFlags = dlsym(__pyx_v_handle, ((char const *)"cuOccupancyMaxPotentialBlockSizeWithFlags")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2564 + * __cuOccupancyMaxPotentialBlockSizeWithFlags = dlfcn.dlsym(handle, 'cuOccupancyMaxPotentialBlockSizeWithFlags') + * global __cuOccupancyAvailableDynamicSMemPerBlock + * __cuOccupancyAvailableDynamicSMemPerBlock = dlfcn.dlsym(handle, 'cuOccupancyAvailableDynamicSMemPerBlock') # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialClusterSize + * __cuOccupancyMaxPotentialClusterSize = dlfcn.dlsym(handle, 'cuOccupancyMaxPotentialClusterSize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyAvailableDynamicSMemPerBlock = dlsym(__pyx_v_handle, ((char const *)"cuOccupancyAvailableDynamicSMemPerBlock")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2566 + * __cuOccupancyAvailableDynamicSMemPerBlock = dlfcn.dlsym(handle, 'cuOccupancyAvailableDynamicSMemPerBlock') + * global __cuOccupancyMaxPotentialClusterSize + * __cuOccupancyMaxPotentialClusterSize = dlfcn.dlsym(handle, 'cuOccupancyMaxPotentialClusterSize') # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveClusters + * __cuOccupancyMaxActiveClusters = dlfcn.dlsym(handle, 'cuOccupancyMaxActiveClusters') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialClusterSize = dlsym(__pyx_v_handle, ((char const *)"cuOccupancyMaxPotentialClusterSize")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2568 + * __cuOccupancyMaxPotentialClusterSize = dlfcn.dlsym(handle, 'cuOccupancyMaxPotentialClusterSize') + * global __cuOccupancyMaxActiveClusters + * __cuOccupancyMaxActiveClusters = dlfcn.dlsym(handle, 'cuOccupancyMaxActiveClusters') # <<<<<<<<<<<<<< + * global __cuTexRefSetArray + * __cuTexRefSetArray = dlfcn.dlsym(handle, 'cuTexRefSetArray') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveClusters = dlsym(__pyx_v_handle, ((char const *)"cuOccupancyMaxActiveClusters")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2570 + * __cuOccupancyMaxActiveClusters = dlfcn.dlsym(handle, 'cuOccupancyMaxActiveClusters') + * global __cuTexRefSetArray + * __cuTexRefSetArray = dlfcn.dlsym(handle, 'cuTexRefSetArray') # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmappedArray + * __cuTexRefSetMipmappedArray = dlfcn.dlsym(handle, 'cuTexRefSetMipmappedArray') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetArray = dlsym(__pyx_v_handle, ((char const *)"cuTexRefSetArray")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2572 + * __cuTexRefSetArray = dlfcn.dlsym(handle, 'cuTexRefSetArray') + * global __cuTexRefSetMipmappedArray + * __cuTexRefSetMipmappedArray = dlfcn.dlsym(handle, 'cuTexRefSetMipmappedArray') # <<<<<<<<<<<<<< + * global __cuTexRefSetAddress_v2 + * __cuTexRefSetAddress_v2 = dlfcn.dlsym(handle, 'cuTexRefSetAddress_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmappedArray = dlsym(__pyx_v_handle, ((char const *)"cuTexRefSetMipmappedArray")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2574 + * __cuTexRefSetMipmappedArray = dlfcn.dlsym(handle, 'cuTexRefSetMipmappedArray') + * global __cuTexRefSetAddress_v2 + * __cuTexRefSetAddress_v2 = dlfcn.dlsym(handle, 'cuTexRefSetAddress_v2') # <<<<<<<<<<<<<< + * global __cuTexRefSetAddress2D_v3 + * __cuTexRefSetAddress2D_v3 = dlfcn.dlsym(handle, 'cuTexRefSetAddress2D_v3') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress_v2 = dlsym(__pyx_v_handle, ((char const *)"cuTexRefSetAddress_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2576 + * __cuTexRefSetAddress_v2 = dlfcn.dlsym(handle, 'cuTexRefSetAddress_v2') + * global __cuTexRefSetAddress2D_v3 + * __cuTexRefSetAddress2D_v3 = dlfcn.dlsym(handle, 'cuTexRefSetAddress2D_v3') # <<<<<<<<<<<<<< + * global __cuTexRefSetFormat + * __cuTexRefSetFormat = dlfcn.dlsym(handle, 'cuTexRefSetFormat') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress2D_v3 = dlsym(__pyx_v_handle, ((char const *)"cuTexRefSetAddress2D_v3")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2578 + * __cuTexRefSetAddress2D_v3 = dlfcn.dlsym(handle, 'cuTexRefSetAddress2D_v3') + * global __cuTexRefSetFormat + * __cuTexRefSetFormat = dlfcn.dlsym(handle, 'cuTexRefSetFormat') # <<<<<<<<<<<<<< + * global __cuTexRefSetAddressMode + * __cuTexRefSetAddressMode = dlfcn.dlsym(handle, 'cuTexRefSetAddressMode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFormat = dlsym(__pyx_v_handle, ((char const *)"cuTexRefSetFormat")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2580 + * __cuTexRefSetFormat = dlfcn.dlsym(handle, 'cuTexRefSetFormat') + * global __cuTexRefSetAddressMode + * __cuTexRefSetAddressMode = dlfcn.dlsym(handle, 'cuTexRefSetAddressMode') # <<<<<<<<<<<<<< + * global __cuTexRefSetFilterMode + * __cuTexRefSetFilterMode = dlfcn.dlsym(handle, 'cuTexRefSetFilterMode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddressMode = dlsym(__pyx_v_handle, ((char const *)"cuTexRefSetAddressMode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2582 + * __cuTexRefSetAddressMode = dlfcn.dlsym(handle, 'cuTexRefSetAddressMode') + * global __cuTexRefSetFilterMode + * __cuTexRefSetFilterMode = dlfcn.dlsym(handle, 'cuTexRefSetFilterMode') # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapFilterMode + * __cuTexRefSetMipmapFilterMode = dlfcn.dlsym(handle, 'cuTexRefSetMipmapFilterMode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFilterMode = dlsym(__pyx_v_handle, ((char const *)"cuTexRefSetFilterMode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2584 + * __cuTexRefSetFilterMode = dlfcn.dlsym(handle, 'cuTexRefSetFilterMode') + * global __cuTexRefSetMipmapFilterMode + * __cuTexRefSetMipmapFilterMode = dlfcn.dlsym(handle, 'cuTexRefSetMipmapFilterMode') # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapLevelBias + * __cuTexRefSetMipmapLevelBias = dlfcn.dlsym(handle, 'cuTexRefSetMipmapLevelBias') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapFilterMode = dlsym(__pyx_v_handle, ((char const *)"cuTexRefSetMipmapFilterMode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2586 + * __cuTexRefSetMipmapFilterMode = dlfcn.dlsym(handle, 'cuTexRefSetMipmapFilterMode') + * global __cuTexRefSetMipmapLevelBias + * __cuTexRefSetMipmapLevelBias = dlfcn.dlsym(handle, 'cuTexRefSetMipmapLevelBias') # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapLevelClamp + * __cuTexRefSetMipmapLevelClamp = dlfcn.dlsym(handle, 'cuTexRefSetMipmapLevelClamp') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelBias = dlsym(__pyx_v_handle, ((char const *)"cuTexRefSetMipmapLevelBias")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2588 + * __cuTexRefSetMipmapLevelBias = dlfcn.dlsym(handle, 'cuTexRefSetMipmapLevelBias') + * global __cuTexRefSetMipmapLevelClamp + * __cuTexRefSetMipmapLevelClamp = dlfcn.dlsym(handle, 'cuTexRefSetMipmapLevelClamp') # <<<<<<<<<<<<<< + * global __cuTexRefSetMaxAnisotropy + * __cuTexRefSetMaxAnisotropy = dlfcn.dlsym(handle, 'cuTexRefSetMaxAnisotropy') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelClamp = dlsym(__pyx_v_handle, ((char const *)"cuTexRefSetMipmapLevelClamp")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2590 + * __cuTexRefSetMipmapLevelClamp = dlfcn.dlsym(handle, 'cuTexRefSetMipmapLevelClamp') + * global __cuTexRefSetMaxAnisotropy + * __cuTexRefSetMaxAnisotropy = dlfcn.dlsym(handle, 'cuTexRefSetMaxAnisotropy') # <<<<<<<<<<<<<< + * global __cuTexRefSetBorderColor + * __cuTexRefSetBorderColor = dlfcn.dlsym(handle, 'cuTexRefSetBorderColor') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMaxAnisotropy = dlsym(__pyx_v_handle, ((char const *)"cuTexRefSetMaxAnisotropy")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2592 + * __cuTexRefSetMaxAnisotropy = dlfcn.dlsym(handle, 'cuTexRefSetMaxAnisotropy') + * global __cuTexRefSetBorderColor + * __cuTexRefSetBorderColor = dlfcn.dlsym(handle, 'cuTexRefSetBorderColor') # <<<<<<<<<<<<<< + * global __cuTexRefSetFlags + * __cuTexRefSetFlags = dlfcn.dlsym(handle, 'cuTexRefSetFlags') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetBorderColor = dlsym(__pyx_v_handle, ((char const *)"cuTexRefSetBorderColor")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2594 + * __cuTexRefSetBorderColor = dlfcn.dlsym(handle, 'cuTexRefSetBorderColor') + * global __cuTexRefSetFlags + * __cuTexRefSetFlags = dlfcn.dlsym(handle, 'cuTexRefSetFlags') # <<<<<<<<<<<<<< + * global __cuTexRefGetAddress_v2 + * __cuTexRefGetAddress_v2 = dlfcn.dlsym(handle, 'cuTexRefGetAddress_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFlags = dlsym(__pyx_v_handle, ((char const *)"cuTexRefSetFlags")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2596 + * __cuTexRefSetFlags = dlfcn.dlsym(handle, 'cuTexRefSetFlags') + * global __cuTexRefGetAddress_v2 + * __cuTexRefGetAddress_v2 = dlfcn.dlsym(handle, 'cuTexRefGetAddress_v2') # <<<<<<<<<<<<<< + * global __cuTexRefGetArray + * __cuTexRefGetArray = dlfcn.dlsym(handle, 'cuTexRefGetArray') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddress_v2 = dlsym(__pyx_v_handle, ((char const *)"cuTexRefGetAddress_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2598 + * __cuTexRefGetAddress_v2 = dlfcn.dlsym(handle, 'cuTexRefGetAddress_v2') + * global __cuTexRefGetArray + * __cuTexRefGetArray = dlfcn.dlsym(handle, 'cuTexRefGetArray') # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmappedArray + * __cuTexRefGetMipmappedArray = dlfcn.dlsym(handle, 'cuTexRefGetMipmappedArray') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetArray = dlsym(__pyx_v_handle, ((char const *)"cuTexRefGetArray")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2600 + * __cuTexRefGetArray = dlfcn.dlsym(handle, 'cuTexRefGetArray') + * global __cuTexRefGetMipmappedArray + * __cuTexRefGetMipmappedArray = dlfcn.dlsym(handle, 'cuTexRefGetMipmappedArray') # <<<<<<<<<<<<<< + * global __cuTexRefGetAddressMode + * __cuTexRefGetAddressMode = dlfcn.dlsym(handle, 'cuTexRefGetAddressMode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmappedArray = dlsym(__pyx_v_handle, ((char const *)"cuTexRefGetMipmappedArray")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2602 + * __cuTexRefGetMipmappedArray = dlfcn.dlsym(handle, 'cuTexRefGetMipmappedArray') + * global __cuTexRefGetAddressMode + * __cuTexRefGetAddressMode = dlfcn.dlsym(handle, 'cuTexRefGetAddressMode') # <<<<<<<<<<<<<< + * global __cuTexRefGetFilterMode + * __cuTexRefGetFilterMode = dlfcn.dlsym(handle, 'cuTexRefGetFilterMode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddressMode = dlsym(__pyx_v_handle, ((char const *)"cuTexRefGetAddressMode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2604 + * __cuTexRefGetAddressMode = dlfcn.dlsym(handle, 'cuTexRefGetAddressMode') + * global __cuTexRefGetFilterMode + * __cuTexRefGetFilterMode = dlfcn.dlsym(handle, 'cuTexRefGetFilterMode') # <<<<<<<<<<<<<< + * global __cuTexRefGetFormat + * __cuTexRefGetFormat = dlfcn.dlsym(handle, 'cuTexRefGetFormat') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFilterMode = dlsym(__pyx_v_handle, ((char const *)"cuTexRefGetFilterMode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2606 + * __cuTexRefGetFilterMode = dlfcn.dlsym(handle, 'cuTexRefGetFilterMode') + * global __cuTexRefGetFormat + * __cuTexRefGetFormat = dlfcn.dlsym(handle, 'cuTexRefGetFormat') # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapFilterMode + * __cuTexRefGetMipmapFilterMode = dlfcn.dlsym(handle, 'cuTexRefGetMipmapFilterMode') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFormat = dlsym(__pyx_v_handle, ((char const *)"cuTexRefGetFormat")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2608 + * __cuTexRefGetFormat = dlfcn.dlsym(handle, 'cuTexRefGetFormat') + * global __cuTexRefGetMipmapFilterMode + * __cuTexRefGetMipmapFilterMode = dlfcn.dlsym(handle, 'cuTexRefGetMipmapFilterMode') # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapLevelBias + * __cuTexRefGetMipmapLevelBias = dlfcn.dlsym(handle, 'cuTexRefGetMipmapLevelBias') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapFilterMode = dlsym(__pyx_v_handle, ((char const *)"cuTexRefGetMipmapFilterMode")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2610 + * __cuTexRefGetMipmapFilterMode = dlfcn.dlsym(handle, 'cuTexRefGetMipmapFilterMode') + * global __cuTexRefGetMipmapLevelBias + * __cuTexRefGetMipmapLevelBias = dlfcn.dlsym(handle, 'cuTexRefGetMipmapLevelBias') # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapLevelClamp + * __cuTexRefGetMipmapLevelClamp = dlfcn.dlsym(handle, 'cuTexRefGetMipmapLevelClamp') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelBias = dlsym(__pyx_v_handle, ((char const *)"cuTexRefGetMipmapLevelBias")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2612 + * __cuTexRefGetMipmapLevelBias = dlfcn.dlsym(handle, 'cuTexRefGetMipmapLevelBias') + * global __cuTexRefGetMipmapLevelClamp + * __cuTexRefGetMipmapLevelClamp = dlfcn.dlsym(handle, 'cuTexRefGetMipmapLevelClamp') # <<<<<<<<<<<<<< + * global __cuTexRefGetMaxAnisotropy + * __cuTexRefGetMaxAnisotropy = dlfcn.dlsym(handle, 'cuTexRefGetMaxAnisotropy') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelClamp = dlsym(__pyx_v_handle, ((char const *)"cuTexRefGetMipmapLevelClamp")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2614 + * __cuTexRefGetMipmapLevelClamp = dlfcn.dlsym(handle, 'cuTexRefGetMipmapLevelClamp') + * global __cuTexRefGetMaxAnisotropy + * __cuTexRefGetMaxAnisotropy = dlfcn.dlsym(handle, 'cuTexRefGetMaxAnisotropy') # <<<<<<<<<<<<<< + * global __cuTexRefGetBorderColor + * __cuTexRefGetBorderColor = dlfcn.dlsym(handle, 'cuTexRefGetBorderColor') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMaxAnisotropy = dlsym(__pyx_v_handle, ((char const *)"cuTexRefGetMaxAnisotropy")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2616 + * __cuTexRefGetMaxAnisotropy = dlfcn.dlsym(handle, 'cuTexRefGetMaxAnisotropy') + * global __cuTexRefGetBorderColor + * __cuTexRefGetBorderColor = dlfcn.dlsym(handle, 'cuTexRefGetBorderColor') # <<<<<<<<<<<<<< + * global __cuTexRefGetFlags + * __cuTexRefGetFlags = dlfcn.dlsym(handle, 'cuTexRefGetFlags') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetBorderColor = dlsym(__pyx_v_handle, ((char const *)"cuTexRefGetBorderColor")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2618 + * __cuTexRefGetBorderColor = dlfcn.dlsym(handle, 'cuTexRefGetBorderColor') + * global __cuTexRefGetFlags + * __cuTexRefGetFlags = dlfcn.dlsym(handle, 'cuTexRefGetFlags') # <<<<<<<<<<<<<< + * global __cuTexRefCreate + * __cuTexRefCreate = dlfcn.dlsym(handle, 'cuTexRefCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFlags = dlsym(__pyx_v_handle, ((char const *)"cuTexRefGetFlags")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2620 + * __cuTexRefGetFlags = dlfcn.dlsym(handle, 'cuTexRefGetFlags') + * global __cuTexRefCreate + * __cuTexRefCreate = dlfcn.dlsym(handle, 'cuTexRefCreate') # <<<<<<<<<<<<<< + * global __cuTexRefDestroy + * __cuTexRefDestroy = dlfcn.dlsym(handle, 'cuTexRefDestroy') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefCreate = dlsym(__pyx_v_handle, ((char const *)"cuTexRefCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2622 + * __cuTexRefCreate = dlfcn.dlsym(handle, 'cuTexRefCreate') + * global __cuTexRefDestroy + * __cuTexRefDestroy = dlfcn.dlsym(handle, 'cuTexRefDestroy') # <<<<<<<<<<<<<< + * global __cuSurfRefSetArray + * __cuSurfRefSetArray = dlfcn.dlsym(handle, 'cuSurfRefSetArray') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefDestroy = dlsym(__pyx_v_handle, ((char const *)"cuTexRefDestroy")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2624 + * __cuTexRefDestroy = dlfcn.dlsym(handle, 'cuTexRefDestroy') + * global __cuSurfRefSetArray + * __cuSurfRefSetArray = dlfcn.dlsym(handle, 'cuSurfRefSetArray') # <<<<<<<<<<<<<< + * global __cuSurfRefGetArray + * __cuSurfRefGetArray = dlfcn.dlsym(handle, 'cuSurfRefGetArray') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefSetArray = dlsym(__pyx_v_handle, ((char const *)"cuSurfRefSetArray")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2626 + * __cuSurfRefSetArray = dlfcn.dlsym(handle, 'cuSurfRefSetArray') + * global __cuSurfRefGetArray + * __cuSurfRefGetArray = dlfcn.dlsym(handle, 'cuSurfRefGetArray') # <<<<<<<<<<<<<< + * global __cuTexObjectCreate + * __cuTexObjectCreate = dlfcn.dlsym(handle, 'cuTexObjectCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefGetArray = dlsym(__pyx_v_handle, ((char const *)"cuSurfRefGetArray")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2628 + * __cuSurfRefGetArray = dlfcn.dlsym(handle, 'cuSurfRefGetArray') + * global __cuTexObjectCreate + * __cuTexObjectCreate = dlfcn.dlsym(handle, 'cuTexObjectCreate') # <<<<<<<<<<<<<< + * global __cuTexObjectDestroy + * __cuTexObjectDestroy = dlfcn.dlsym(handle, 'cuTexObjectDestroy') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectCreate = dlsym(__pyx_v_handle, ((char const *)"cuTexObjectCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2630 + * __cuTexObjectCreate = dlfcn.dlsym(handle, 'cuTexObjectCreate') + * global __cuTexObjectDestroy + * __cuTexObjectDestroy = dlfcn.dlsym(handle, 'cuTexObjectDestroy') # <<<<<<<<<<<<<< + * global __cuTexObjectGetResourceDesc + * __cuTexObjectGetResourceDesc = dlfcn.dlsym(handle, 'cuTexObjectGetResourceDesc') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectDestroy = dlsym(__pyx_v_handle, ((char const *)"cuTexObjectDestroy")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2632 + * __cuTexObjectDestroy = dlfcn.dlsym(handle, 'cuTexObjectDestroy') + * global __cuTexObjectGetResourceDesc + * __cuTexObjectGetResourceDesc = dlfcn.dlsym(handle, 'cuTexObjectGetResourceDesc') # <<<<<<<<<<<<<< + * global __cuTexObjectGetTextureDesc + * __cuTexObjectGetTextureDesc = dlfcn.dlsym(handle, 'cuTexObjectGetTextureDesc') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceDesc = dlsym(__pyx_v_handle, ((char const *)"cuTexObjectGetResourceDesc")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2634 + * __cuTexObjectGetResourceDesc = dlfcn.dlsym(handle, 'cuTexObjectGetResourceDesc') + * global __cuTexObjectGetTextureDesc + * __cuTexObjectGetTextureDesc = dlfcn.dlsym(handle, 'cuTexObjectGetTextureDesc') # <<<<<<<<<<<<<< + * global __cuTexObjectGetResourceViewDesc + * __cuTexObjectGetResourceViewDesc = dlfcn.dlsym(handle, 'cuTexObjectGetResourceViewDesc') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetTextureDesc = dlsym(__pyx_v_handle, ((char const *)"cuTexObjectGetTextureDesc")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2636 + * __cuTexObjectGetTextureDesc = dlfcn.dlsym(handle, 'cuTexObjectGetTextureDesc') + * global __cuTexObjectGetResourceViewDesc + * __cuTexObjectGetResourceViewDesc = dlfcn.dlsym(handle, 'cuTexObjectGetResourceViewDesc') # <<<<<<<<<<<<<< + * global __cuSurfObjectCreate + * __cuSurfObjectCreate = dlfcn.dlsym(handle, 'cuSurfObjectCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceViewDesc = dlsym(__pyx_v_handle, ((char const *)"cuTexObjectGetResourceViewDesc")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2638 + * __cuTexObjectGetResourceViewDesc = dlfcn.dlsym(handle, 'cuTexObjectGetResourceViewDesc') + * global __cuSurfObjectCreate + * __cuSurfObjectCreate = dlfcn.dlsym(handle, 'cuSurfObjectCreate') # <<<<<<<<<<<<<< + * global __cuSurfObjectDestroy + * __cuSurfObjectDestroy = dlfcn.dlsym(handle, 'cuSurfObjectDestroy') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectCreate = dlsym(__pyx_v_handle, ((char const *)"cuSurfObjectCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2640 + * __cuSurfObjectCreate = dlfcn.dlsym(handle, 'cuSurfObjectCreate') + * global __cuSurfObjectDestroy + * __cuSurfObjectDestroy = dlfcn.dlsym(handle, 'cuSurfObjectDestroy') # <<<<<<<<<<<<<< + * global __cuSurfObjectGetResourceDesc + * __cuSurfObjectGetResourceDesc = dlfcn.dlsym(handle, 'cuSurfObjectGetResourceDesc') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectDestroy = dlsym(__pyx_v_handle, ((char const *)"cuSurfObjectDestroy")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2642 + * __cuSurfObjectDestroy = dlfcn.dlsym(handle, 'cuSurfObjectDestroy') + * global __cuSurfObjectGetResourceDesc + * __cuSurfObjectGetResourceDesc = dlfcn.dlsym(handle, 'cuSurfObjectGetResourceDesc') # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeTiled + * __cuTensorMapEncodeTiled = dlfcn.dlsym(handle, 'cuTensorMapEncodeTiled') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectGetResourceDesc = dlsym(__pyx_v_handle, ((char const *)"cuSurfObjectGetResourceDesc")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2644 + * __cuSurfObjectGetResourceDesc = dlfcn.dlsym(handle, 'cuSurfObjectGetResourceDesc') + * global __cuTensorMapEncodeTiled + * __cuTensorMapEncodeTiled = dlfcn.dlsym(handle, 'cuTensorMapEncodeTiled') # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeIm2col + * __cuTensorMapEncodeIm2col = dlfcn.dlsym(handle, 'cuTensorMapEncodeIm2col') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeTiled = dlsym(__pyx_v_handle, ((char const *)"cuTensorMapEncodeTiled")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2646 + * __cuTensorMapEncodeTiled = dlfcn.dlsym(handle, 'cuTensorMapEncodeTiled') + * global __cuTensorMapEncodeIm2col + * __cuTensorMapEncodeIm2col = dlfcn.dlsym(handle, 'cuTensorMapEncodeIm2col') # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeIm2colWide + * __cuTensorMapEncodeIm2colWide = dlfcn.dlsym(handle, 'cuTensorMapEncodeIm2colWide') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2col = dlsym(__pyx_v_handle, ((char const *)"cuTensorMapEncodeIm2col")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2648 + * __cuTensorMapEncodeIm2col = dlfcn.dlsym(handle, 'cuTensorMapEncodeIm2col') + * global __cuTensorMapEncodeIm2colWide + * __cuTensorMapEncodeIm2colWide = dlfcn.dlsym(handle, 'cuTensorMapEncodeIm2colWide') # <<<<<<<<<<<<<< + * global __cuTensorMapReplaceAddress + * __cuTensorMapReplaceAddress = dlfcn.dlsym(handle, 'cuTensorMapReplaceAddress') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2colWide = dlsym(__pyx_v_handle, ((char const *)"cuTensorMapEncodeIm2colWide")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2650 + * __cuTensorMapEncodeIm2colWide = dlfcn.dlsym(handle, 'cuTensorMapEncodeIm2colWide') + * global __cuTensorMapReplaceAddress + * __cuTensorMapReplaceAddress = dlfcn.dlsym(handle, 'cuTensorMapReplaceAddress') # <<<<<<<<<<<<<< + * global __cuDeviceCanAccessPeer + * __cuDeviceCanAccessPeer = dlfcn.dlsym(handle, 'cuDeviceCanAccessPeer') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapReplaceAddress = dlsym(__pyx_v_handle, ((char const *)"cuTensorMapReplaceAddress")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2652 + * __cuTensorMapReplaceAddress = dlfcn.dlsym(handle, 'cuTensorMapReplaceAddress') + * global __cuDeviceCanAccessPeer + * __cuDeviceCanAccessPeer = dlfcn.dlsym(handle, 'cuDeviceCanAccessPeer') # <<<<<<<<<<<<<< + * global __cuCtxEnablePeerAccess + * __cuCtxEnablePeerAccess = dlfcn.dlsym(handle, 'cuCtxEnablePeerAccess') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceCanAccessPeer = dlsym(__pyx_v_handle, ((char const *)"cuDeviceCanAccessPeer")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2654 + * __cuDeviceCanAccessPeer = dlfcn.dlsym(handle, 'cuDeviceCanAccessPeer') + * global __cuCtxEnablePeerAccess + * __cuCtxEnablePeerAccess = dlfcn.dlsym(handle, 'cuCtxEnablePeerAccess') # <<<<<<<<<<<<<< + * global __cuCtxDisablePeerAccess + * __cuCtxDisablePeerAccess = dlfcn.dlsym(handle, 'cuCtxDisablePeerAccess') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxEnablePeerAccess = dlsym(__pyx_v_handle, ((char const *)"cuCtxEnablePeerAccess")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2656 + * __cuCtxEnablePeerAccess = dlfcn.dlsym(handle, 'cuCtxEnablePeerAccess') + * global __cuCtxDisablePeerAccess + * __cuCtxDisablePeerAccess = dlfcn.dlsym(handle, 'cuCtxDisablePeerAccess') # <<<<<<<<<<<<<< + * global __cuDeviceGetP2PAttribute + * __cuDeviceGetP2PAttribute = dlfcn.dlsym(handle, 'cuDeviceGetP2PAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDisablePeerAccess = dlsym(__pyx_v_handle, ((char const *)"cuCtxDisablePeerAccess")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2658 + * __cuCtxDisablePeerAccess = dlfcn.dlsym(handle, 'cuCtxDisablePeerAccess') + * global __cuDeviceGetP2PAttribute + * __cuDeviceGetP2PAttribute = dlfcn.dlsym(handle, 'cuDeviceGetP2PAttribute') # <<<<<<<<<<<<<< + * global __cuGraphicsUnregisterResource + * __cuGraphicsUnregisterResource = dlfcn.dlsym(handle, 'cuGraphicsUnregisterResource') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetP2PAttribute = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetP2PAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2660 + * __cuDeviceGetP2PAttribute = dlfcn.dlsym(handle, 'cuDeviceGetP2PAttribute') + * global __cuGraphicsUnregisterResource + * __cuGraphicsUnregisterResource = dlfcn.dlsym(handle, 'cuGraphicsUnregisterResource') # <<<<<<<<<<<<<< + * global __cuGraphicsSubResourceGetMappedArray + * __cuGraphicsSubResourceGetMappedArray = dlfcn.dlsym(handle, 'cuGraphicsSubResourceGetMappedArray') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnregisterResource = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsUnregisterResource")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2662 + * __cuGraphicsUnregisterResource = dlfcn.dlsym(handle, 'cuGraphicsUnregisterResource') + * global __cuGraphicsSubResourceGetMappedArray + * __cuGraphicsSubResourceGetMappedArray = dlfcn.dlsym(handle, 'cuGraphicsSubResourceGetMappedArray') # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedMipmappedArray + * __cuGraphicsResourceGetMappedMipmappedArray = dlfcn.dlsym(handle, 'cuGraphicsResourceGetMappedMipmappedArray') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsSubResourceGetMappedArray = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsSubResourceGetMappedArray")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2664 + * __cuGraphicsSubResourceGetMappedArray = dlfcn.dlsym(handle, 'cuGraphicsSubResourceGetMappedArray') + * global __cuGraphicsResourceGetMappedMipmappedArray + * __cuGraphicsResourceGetMappedMipmappedArray = dlfcn.dlsym(handle, 'cuGraphicsResourceGetMappedMipmappedArray') # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedPointer_v2 + * __cuGraphicsResourceGetMappedPointer_v2 = dlfcn.dlsym(handle, 'cuGraphicsResourceGetMappedPointer_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedMipmappedArray = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsResourceGetMappedMipmappedArray")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2666 + * __cuGraphicsResourceGetMappedMipmappedArray = dlfcn.dlsym(handle, 'cuGraphicsResourceGetMappedMipmappedArray') + * global __cuGraphicsResourceGetMappedPointer_v2 + * __cuGraphicsResourceGetMappedPointer_v2 = dlfcn.dlsym(handle, 'cuGraphicsResourceGetMappedPointer_v2') # <<<<<<<<<<<<<< + * global __cuGraphicsResourceSetMapFlags_v2 + * __cuGraphicsResourceSetMapFlags_v2 = dlfcn.dlsym(handle, 'cuGraphicsResourceSetMapFlags_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedPointer_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsResourceGetMappedPointer_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2668 + * __cuGraphicsResourceGetMappedPointer_v2 = dlfcn.dlsym(handle, 'cuGraphicsResourceGetMappedPointer_v2') + * global __cuGraphicsResourceSetMapFlags_v2 + * __cuGraphicsResourceSetMapFlags_v2 = dlfcn.dlsym(handle, 'cuGraphicsResourceSetMapFlags_v2') # <<<<<<<<<<<<<< + * global __cuGetProcAddress_v2 + * __cuGetProcAddress_v2 = dlfcn.dlsym(handle, 'cuGetProcAddress_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceSetMapFlags_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsResourceSetMapFlags_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2670 + * __cuGraphicsResourceSetMapFlags_v2 = dlfcn.dlsym(handle, 'cuGraphicsResourceSetMapFlags_v2') + * global __cuGetProcAddress_v2 + * __cuGetProcAddress_v2 = dlfcn.dlsym(handle, 'cuGetProcAddress_v2') # <<<<<<<<<<<<<< + * global __cuCoredumpGetAttribute + * __cuCoredumpGetAttribute = dlfcn.dlsym(handle, 'cuCoredumpGetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGetProcAddress_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2672 + * __cuGetProcAddress_v2 = dlfcn.dlsym(handle, 'cuGetProcAddress_v2') + * global __cuCoredumpGetAttribute + * __cuCoredumpGetAttribute = dlfcn.dlsym(handle, 'cuCoredumpGetAttribute') # <<<<<<<<<<<<<< + * global __cuCoredumpGetAttributeGlobal + * __cuCoredumpGetAttributeGlobal = dlfcn.dlsym(handle, 'cuCoredumpGetAttributeGlobal') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuCoredumpGetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2674 + * __cuCoredumpGetAttribute = dlfcn.dlsym(handle, 'cuCoredumpGetAttribute') + * global __cuCoredumpGetAttributeGlobal + * __cuCoredumpGetAttributeGlobal = dlfcn.dlsym(handle, 'cuCoredumpGetAttributeGlobal') # <<<<<<<<<<<<<< + * global __cuCoredumpSetAttribute + * __cuCoredumpSetAttribute = dlfcn.dlsym(handle, 'cuCoredumpSetAttribute') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttributeGlobal = dlsym(__pyx_v_handle, ((char const *)"cuCoredumpGetAttributeGlobal")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2676 + * __cuCoredumpGetAttributeGlobal = dlfcn.dlsym(handle, 'cuCoredumpGetAttributeGlobal') + * global __cuCoredumpSetAttribute + * __cuCoredumpSetAttribute = dlfcn.dlsym(handle, 'cuCoredumpSetAttribute') # <<<<<<<<<<<<<< + * global __cuCoredumpSetAttributeGlobal + * __cuCoredumpSetAttributeGlobal = dlfcn.dlsym(handle, 'cuCoredumpSetAttributeGlobal') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttribute = dlsym(__pyx_v_handle, ((char const *)"cuCoredumpSetAttribute")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2678 + * __cuCoredumpSetAttribute = dlfcn.dlsym(handle, 'cuCoredumpSetAttribute') + * global __cuCoredumpSetAttributeGlobal + * __cuCoredumpSetAttributeGlobal = dlfcn.dlsym(handle, 'cuCoredumpSetAttributeGlobal') # <<<<<<<<<<<<<< + * global __cuGetExportTable + * __cuGetExportTable = dlfcn.dlsym(handle, 'cuGetExportTable') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttributeGlobal = dlsym(__pyx_v_handle, ((char const *)"cuCoredumpSetAttributeGlobal")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2680 + * __cuCoredumpSetAttributeGlobal = dlfcn.dlsym(handle, 'cuCoredumpSetAttributeGlobal') + * global __cuGetExportTable + * __cuGetExportTable = dlfcn.dlsym(handle, 'cuGetExportTable') # <<<<<<<<<<<<<< + * global __cuGreenCtxCreate + * __cuGreenCtxCreate = dlfcn.dlsym(handle, 'cuGreenCtxCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetExportTable = dlsym(__pyx_v_handle, ((char const *)"cuGetExportTable")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2682 + * __cuGetExportTable = dlfcn.dlsym(handle, 'cuGetExportTable') + * global __cuGreenCtxCreate + * __cuGreenCtxCreate = dlfcn.dlsym(handle, 'cuGreenCtxCreate') # <<<<<<<<<<<<<< + * global __cuGreenCtxDestroy + * __cuGreenCtxDestroy = dlfcn.dlsym(handle, 'cuGreenCtxDestroy') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxCreate = dlsym(__pyx_v_handle, ((char const *)"cuGreenCtxCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2684 + * __cuGreenCtxCreate = dlfcn.dlsym(handle, 'cuGreenCtxCreate') + * global __cuGreenCtxDestroy + * __cuGreenCtxDestroy = dlfcn.dlsym(handle, 'cuGreenCtxDestroy') # <<<<<<<<<<<<<< + * global __cuCtxFromGreenCtx + * __cuCtxFromGreenCtx = dlfcn.dlsym(handle, 'cuCtxFromGreenCtx') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxDestroy = dlsym(__pyx_v_handle, ((char const *)"cuGreenCtxDestroy")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2686 + * __cuGreenCtxDestroy = dlfcn.dlsym(handle, 'cuGreenCtxDestroy') + * global __cuCtxFromGreenCtx + * __cuCtxFromGreenCtx = dlfcn.dlsym(handle, 'cuCtxFromGreenCtx') # <<<<<<<<<<<<<< + * global __cuDeviceGetDevResource + * __cuDeviceGetDevResource = dlfcn.dlsym(handle, 'cuDeviceGetDevResource') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxFromGreenCtx = dlsym(__pyx_v_handle, ((char const *)"cuCtxFromGreenCtx")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2688 + * __cuCtxFromGreenCtx = dlfcn.dlsym(handle, 'cuCtxFromGreenCtx') + * global __cuDeviceGetDevResource + * __cuDeviceGetDevResource = dlfcn.dlsym(handle, 'cuDeviceGetDevResource') # <<<<<<<<<<<<<< + * global __cuCtxGetDevResource + * __cuCtxGetDevResource = dlfcn.dlsym(handle, 'cuCtxGetDevResource') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDevResource = dlsym(__pyx_v_handle, ((char const *)"cuDeviceGetDevResource")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2690 + * __cuDeviceGetDevResource = dlfcn.dlsym(handle, 'cuDeviceGetDevResource') + * global __cuCtxGetDevResource + * __cuCtxGetDevResource = dlfcn.dlsym(handle, 'cuCtxGetDevResource') # <<<<<<<<<<<<<< + * global __cuGreenCtxGetDevResource + * __cuGreenCtxGetDevResource = dlfcn.dlsym(handle, 'cuGreenCtxGetDevResource') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevResource = dlsym(__pyx_v_handle, ((char const *)"cuCtxGetDevResource")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2692 + * __cuCtxGetDevResource = dlfcn.dlsym(handle, 'cuCtxGetDevResource') + * global __cuGreenCtxGetDevResource + * __cuGreenCtxGetDevResource = dlfcn.dlsym(handle, 'cuGreenCtxGetDevResource') # <<<<<<<<<<<<<< + * global __cuDevSmResourceSplitByCount + * __cuDevSmResourceSplitByCount = dlfcn.dlsym(handle, 'cuDevSmResourceSplitByCount') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxGetDevResource = dlsym(__pyx_v_handle, ((char const *)"cuGreenCtxGetDevResource")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2694 + * __cuGreenCtxGetDevResource = dlfcn.dlsym(handle, 'cuGreenCtxGetDevResource') + * global __cuDevSmResourceSplitByCount + * __cuDevSmResourceSplitByCount = dlfcn.dlsym(handle, 'cuDevSmResourceSplitByCount') # <<<<<<<<<<<<<< + * global __cuDevResourceGenerateDesc + * __cuDevResourceGenerateDesc = dlfcn.dlsym(handle, 'cuDevResourceGenerateDesc') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevSmResourceSplitByCount = dlsym(__pyx_v_handle, ((char const *)"cuDevSmResourceSplitByCount")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2696 + * __cuDevSmResourceSplitByCount = dlfcn.dlsym(handle, 'cuDevSmResourceSplitByCount') + * global __cuDevResourceGenerateDesc + * __cuDevResourceGenerateDesc = dlfcn.dlsym(handle, 'cuDevResourceGenerateDesc') # <<<<<<<<<<<<<< + * global __cuGreenCtxRecordEvent + * __cuGreenCtxRecordEvent = dlfcn.dlsym(handle, 'cuGreenCtxRecordEvent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevResourceGenerateDesc = dlsym(__pyx_v_handle, ((char const *)"cuDevResourceGenerateDesc")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2698 + * __cuDevResourceGenerateDesc = dlfcn.dlsym(handle, 'cuDevResourceGenerateDesc') + * global __cuGreenCtxRecordEvent + * __cuGreenCtxRecordEvent = dlfcn.dlsym(handle, 'cuGreenCtxRecordEvent') # <<<<<<<<<<<<<< + * global __cuGreenCtxWaitEvent + * __cuGreenCtxWaitEvent = dlfcn.dlsym(handle, 'cuGreenCtxWaitEvent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxRecordEvent = dlsym(__pyx_v_handle, ((char const *)"cuGreenCtxRecordEvent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2700 + * __cuGreenCtxRecordEvent = dlfcn.dlsym(handle, 'cuGreenCtxRecordEvent') + * global __cuGreenCtxWaitEvent + * __cuGreenCtxWaitEvent = dlfcn.dlsym(handle, 'cuGreenCtxWaitEvent') # <<<<<<<<<<<<<< + * global __cuStreamGetGreenCtx + * __cuStreamGetGreenCtx = dlfcn.dlsym(handle, 'cuStreamGetGreenCtx') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxWaitEvent = dlsym(__pyx_v_handle, ((char const *)"cuGreenCtxWaitEvent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2702 + * __cuGreenCtxWaitEvent = dlfcn.dlsym(handle, 'cuGreenCtxWaitEvent') + * global __cuStreamGetGreenCtx + * __cuStreamGetGreenCtx = dlfcn.dlsym(handle, 'cuStreamGetGreenCtx') # <<<<<<<<<<<<<< + * global __cuGreenCtxStreamCreate + * __cuGreenCtxStreamCreate = dlfcn.dlsym(handle, 'cuGreenCtxStreamCreate') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetGreenCtx = dlsym(__pyx_v_handle, ((char const *)"cuStreamGetGreenCtx")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2704 + * __cuStreamGetGreenCtx = dlfcn.dlsym(handle, 'cuStreamGetGreenCtx') + * global __cuGreenCtxStreamCreate + * __cuGreenCtxStreamCreate = dlfcn.dlsym(handle, 'cuGreenCtxStreamCreate') # <<<<<<<<<<<<<< + * global __cuLogsRegisterCallback + * __cuLogsRegisterCallback = dlfcn.dlsym(handle, 'cuLogsRegisterCallback') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxStreamCreate = dlsym(__pyx_v_handle, ((char const *)"cuGreenCtxStreamCreate")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2706 + * __cuGreenCtxStreamCreate = dlfcn.dlsym(handle, 'cuGreenCtxStreamCreate') + * global __cuLogsRegisterCallback + * __cuLogsRegisterCallback = dlfcn.dlsym(handle, 'cuLogsRegisterCallback') # <<<<<<<<<<<<<< + * global __cuLogsUnregisterCallback + * __cuLogsUnregisterCallback = dlfcn.dlsym(handle, 'cuLogsUnregisterCallback') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsRegisterCallback = dlsym(__pyx_v_handle, ((char const *)"cuLogsRegisterCallback")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2708 + * __cuLogsRegisterCallback = dlfcn.dlsym(handle, 'cuLogsRegisterCallback') + * global __cuLogsUnregisterCallback + * __cuLogsUnregisterCallback = dlfcn.dlsym(handle, 'cuLogsUnregisterCallback') # <<<<<<<<<<<<<< + * global __cuLogsCurrent + * __cuLogsCurrent = dlfcn.dlsym(handle, 'cuLogsCurrent') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsUnregisterCallback = dlsym(__pyx_v_handle, ((char const *)"cuLogsUnregisterCallback")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2710 + * __cuLogsUnregisterCallback = dlfcn.dlsym(handle, 'cuLogsUnregisterCallback') + * global __cuLogsCurrent + * __cuLogsCurrent = dlfcn.dlsym(handle, 'cuLogsCurrent') # <<<<<<<<<<<<<< + * global __cuLogsDumpToFile + * __cuLogsDumpToFile = dlfcn.dlsym(handle, 'cuLogsDumpToFile') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsCurrent = dlsym(__pyx_v_handle, ((char const *)"cuLogsCurrent")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2712 + * __cuLogsCurrent = dlfcn.dlsym(handle, 'cuLogsCurrent') + * global __cuLogsDumpToFile + * __cuLogsDumpToFile = dlfcn.dlsym(handle, 'cuLogsDumpToFile') # <<<<<<<<<<<<<< + * global __cuLogsDumpToMemory + * __cuLogsDumpToMemory = dlfcn.dlsym(handle, 'cuLogsDumpToMemory') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToFile = dlsym(__pyx_v_handle, ((char const *)"cuLogsDumpToFile")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2714 + * __cuLogsDumpToFile = dlfcn.dlsym(handle, 'cuLogsDumpToFile') + * global __cuLogsDumpToMemory + * __cuLogsDumpToMemory = dlfcn.dlsym(handle, 'cuLogsDumpToMemory') # <<<<<<<<<<<<<< + * global __cuCheckpointProcessGetRestoreThreadId + * __cuCheckpointProcessGetRestoreThreadId = dlfcn.dlsym(handle, 'cuCheckpointProcessGetRestoreThreadId') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToMemory = dlsym(__pyx_v_handle, ((char const *)"cuLogsDumpToMemory")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2716 + * __cuLogsDumpToMemory = dlfcn.dlsym(handle, 'cuLogsDumpToMemory') + * global __cuCheckpointProcessGetRestoreThreadId + * __cuCheckpointProcessGetRestoreThreadId = dlfcn.dlsym(handle, 'cuCheckpointProcessGetRestoreThreadId') # <<<<<<<<<<<<<< + * global __cuCheckpointProcessGetState + * __cuCheckpointProcessGetState = dlfcn.dlsym(handle, 'cuCheckpointProcessGetState') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetRestoreThreadId = dlsym(__pyx_v_handle, ((char const *)"cuCheckpointProcessGetRestoreThreadId")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2718 + * __cuCheckpointProcessGetRestoreThreadId = dlfcn.dlsym(handle, 'cuCheckpointProcessGetRestoreThreadId') + * global __cuCheckpointProcessGetState + * __cuCheckpointProcessGetState = dlfcn.dlsym(handle, 'cuCheckpointProcessGetState') # <<<<<<<<<<<<<< + * global __cuCheckpointProcessLock + * __cuCheckpointProcessLock = dlfcn.dlsym(handle, 'cuCheckpointProcessLock') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetState = dlsym(__pyx_v_handle, ((char const *)"cuCheckpointProcessGetState")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2720 + * __cuCheckpointProcessGetState = dlfcn.dlsym(handle, 'cuCheckpointProcessGetState') + * global __cuCheckpointProcessLock + * __cuCheckpointProcessLock = dlfcn.dlsym(handle, 'cuCheckpointProcessLock') # <<<<<<<<<<<<<< + * global __cuCheckpointProcessCheckpoint + * __cuCheckpointProcessCheckpoint = dlfcn.dlsym(handle, 'cuCheckpointProcessCheckpoint') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessLock = dlsym(__pyx_v_handle, ((char const *)"cuCheckpointProcessLock")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2722 + * __cuCheckpointProcessLock = dlfcn.dlsym(handle, 'cuCheckpointProcessLock') + * global __cuCheckpointProcessCheckpoint + * __cuCheckpointProcessCheckpoint = dlfcn.dlsym(handle, 'cuCheckpointProcessCheckpoint') # <<<<<<<<<<<<<< + * global __cuCheckpointProcessRestore + * __cuCheckpointProcessRestore = dlfcn.dlsym(handle, 'cuCheckpointProcessRestore') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessCheckpoint = dlsym(__pyx_v_handle, ((char const *)"cuCheckpointProcessCheckpoint")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2724 + * __cuCheckpointProcessCheckpoint = dlfcn.dlsym(handle, 'cuCheckpointProcessCheckpoint') + * global __cuCheckpointProcessRestore + * __cuCheckpointProcessRestore = dlfcn.dlsym(handle, 'cuCheckpointProcessRestore') # <<<<<<<<<<<<<< + * global __cuCheckpointProcessUnlock + * __cuCheckpointProcessUnlock = dlfcn.dlsym(handle, 'cuCheckpointProcessUnlock') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessRestore = dlsym(__pyx_v_handle, ((char const *)"cuCheckpointProcessRestore")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2726 + * __cuCheckpointProcessRestore = dlfcn.dlsym(handle, 'cuCheckpointProcessRestore') + * global __cuCheckpointProcessUnlock + * __cuCheckpointProcessUnlock = dlfcn.dlsym(handle, 'cuCheckpointProcessUnlock') # <<<<<<<<<<<<<< + * global __cuProfilerStart + * __cuProfilerStart = dlfcn.dlsym(handle, 'cuProfilerStart') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessUnlock = dlsym(__pyx_v_handle, ((char const *)"cuCheckpointProcessUnlock")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2728 + * __cuCheckpointProcessUnlock = dlfcn.dlsym(handle, 'cuCheckpointProcessUnlock') + * global __cuProfilerStart + * __cuProfilerStart = dlfcn.dlsym(handle, 'cuProfilerStart') # <<<<<<<<<<<<<< + * global __cuProfilerStop + * __cuProfilerStop = dlfcn.dlsym(handle, 'cuProfilerStop') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStart = dlsym(__pyx_v_handle, ((char const *)"cuProfilerStart")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2730 + * __cuProfilerStart = dlfcn.dlsym(handle, 'cuProfilerStart') + * global __cuProfilerStop + * __cuProfilerStop = dlfcn.dlsym(handle, 'cuProfilerStop') # <<<<<<<<<<<<<< + * global __cuGraphicsEGLRegisterImage + * __cuGraphicsEGLRegisterImage = dlfcn.dlsym(handle, 'cuGraphicsEGLRegisterImage') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStop = dlsym(__pyx_v_handle, ((char const *)"cuProfilerStop")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2732 + * __cuProfilerStop = dlfcn.dlsym(handle, 'cuProfilerStop') + * global __cuGraphicsEGLRegisterImage + * __cuGraphicsEGLRegisterImage = dlfcn.dlsym(handle, 'cuGraphicsEGLRegisterImage') # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerConnect + * __cuEGLStreamConsumerConnect = dlfcn.dlsym(handle, 'cuEGLStreamConsumerConnect') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsEGLRegisterImage = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsEGLRegisterImage")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2734 + * __cuGraphicsEGLRegisterImage = dlfcn.dlsym(handle, 'cuGraphicsEGLRegisterImage') + * global __cuEGLStreamConsumerConnect + * __cuEGLStreamConsumerConnect = dlfcn.dlsym(handle, 'cuEGLStreamConsumerConnect') # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerConnectWithFlags + * __cuEGLStreamConsumerConnectWithFlags = dlfcn.dlsym(handle, 'cuEGLStreamConsumerConnectWithFlags') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnect = dlsym(__pyx_v_handle, ((char const *)"cuEGLStreamConsumerConnect")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2736 + * __cuEGLStreamConsumerConnect = dlfcn.dlsym(handle, 'cuEGLStreamConsumerConnect') + * global __cuEGLStreamConsumerConnectWithFlags + * __cuEGLStreamConsumerConnectWithFlags = dlfcn.dlsym(handle, 'cuEGLStreamConsumerConnectWithFlags') # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerDisconnect + * __cuEGLStreamConsumerDisconnect = dlfcn.dlsym(handle, 'cuEGLStreamConsumerDisconnect') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnectWithFlags = dlsym(__pyx_v_handle, ((char const *)"cuEGLStreamConsumerConnectWithFlags")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2738 + * __cuEGLStreamConsumerConnectWithFlags = dlfcn.dlsym(handle, 'cuEGLStreamConsumerConnectWithFlags') + * global __cuEGLStreamConsumerDisconnect + * __cuEGLStreamConsumerDisconnect = dlfcn.dlsym(handle, 'cuEGLStreamConsumerDisconnect') # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerAcquireFrame + * __cuEGLStreamConsumerAcquireFrame = dlfcn.dlsym(handle, 'cuEGLStreamConsumerAcquireFrame') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerDisconnect = dlsym(__pyx_v_handle, ((char const *)"cuEGLStreamConsumerDisconnect")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2740 + * __cuEGLStreamConsumerDisconnect = dlfcn.dlsym(handle, 'cuEGLStreamConsumerDisconnect') + * global __cuEGLStreamConsumerAcquireFrame + * __cuEGLStreamConsumerAcquireFrame = dlfcn.dlsym(handle, 'cuEGLStreamConsumerAcquireFrame') # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerReleaseFrame + * __cuEGLStreamConsumerReleaseFrame = dlfcn.dlsym(handle, 'cuEGLStreamConsumerReleaseFrame') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerAcquireFrame = dlsym(__pyx_v_handle, ((char const *)"cuEGLStreamConsumerAcquireFrame")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2742 + * __cuEGLStreamConsumerAcquireFrame = dlfcn.dlsym(handle, 'cuEGLStreamConsumerAcquireFrame') + * global __cuEGLStreamConsumerReleaseFrame + * __cuEGLStreamConsumerReleaseFrame = dlfcn.dlsym(handle, 'cuEGLStreamConsumerReleaseFrame') # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerConnect + * __cuEGLStreamProducerConnect = dlfcn.dlsym(handle, 'cuEGLStreamProducerConnect') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerReleaseFrame = dlsym(__pyx_v_handle, ((char const *)"cuEGLStreamConsumerReleaseFrame")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2744 + * __cuEGLStreamConsumerReleaseFrame = dlfcn.dlsym(handle, 'cuEGLStreamConsumerReleaseFrame') + * global __cuEGLStreamProducerConnect + * __cuEGLStreamProducerConnect = dlfcn.dlsym(handle, 'cuEGLStreamProducerConnect') # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerDisconnect + * __cuEGLStreamProducerDisconnect = dlfcn.dlsym(handle, 'cuEGLStreamProducerDisconnect') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerConnect = dlsym(__pyx_v_handle, ((char const *)"cuEGLStreamProducerConnect")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2746 + * __cuEGLStreamProducerConnect = dlfcn.dlsym(handle, 'cuEGLStreamProducerConnect') + * global __cuEGLStreamProducerDisconnect + * __cuEGLStreamProducerDisconnect = dlfcn.dlsym(handle, 'cuEGLStreamProducerDisconnect') # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerPresentFrame + * __cuEGLStreamProducerPresentFrame = dlfcn.dlsym(handle, 'cuEGLStreamProducerPresentFrame') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerDisconnect = dlsym(__pyx_v_handle, ((char const *)"cuEGLStreamProducerDisconnect")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2748 + * __cuEGLStreamProducerDisconnect = dlfcn.dlsym(handle, 'cuEGLStreamProducerDisconnect') + * global __cuEGLStreamProducerPresentFrame + * __cuEGLStreamProducerPresentFrame = dlfcn.dlsym(handle, 'cuEGLStreamProducerPresentFrame') # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerReturnFrame + * __cuEGLStreamProducerReturnFrame = dlfcn.dlsym(handle, 'cuEGLStreamProducerReturnFrame') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerPresentFrame = dlsym(__pyx_v_handle, ((char const *)"cuEGLStreamProducerPresentFrame")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2750 + * __cuEGLStreamProducerPresentFrame = dlfcn.dlsym(handle, 'cuEGLStreamProducerPresentFrame') + * global __cuEGLStreamProducerReturnFrame + * __cuEGLStreamProducerReturnFrame = dlfcn.dlsym(handle, 'cuEGLStreamProducerReturnFrame') # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedEglFrame + * __cuGraphicsResourceGetMappedEglFrame = dlfcn.dlsym(handle, 'cuGraphicsResourceGetMappedEglFrame') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerReturnFrame = dlsym(__pyx_v_handle, ((char const *)"cuEGLStreamProducerReturnFrame")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2752 + * __cuEGLStreamProducerReturnFrame = dlfcn.dlsym(handle, 'cuEGLStreamProducerReturnFrame') + * global __cuGraphicsResourceGetMappedEglFrame + * __cuGraphicsResourceGetMappedEglFrame = dlfcn.dlsym(handle, 'cuGraphicsResourceGetMappedEglFrame') # <<<<<<<<<<<<<< + * global __cuEventCreateFromEGLSync + * __cuEventCreateFromEGLSync = dlfcn.dlsym(handle, 'cuEventCreateFromEGLSync') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedEglFrame = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsResourceGetMappedEglFrame")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2754 + * __cuGraphicsResourceGetMappedEglFrame = dlfcn.dlsym(handle, 'cuGraphicsResourceGetMappedEglFrame') + * global __cuEventCreateFromEGLSync + * __cuEventCreateFromEGLSync = dlfcn.dlsym(handle, 'cuEventCreateFromEGLSync') # <<<<<<<<<<<<<< + * global __cuGraphicsGLRegisterBuffer + * __cuGraphicsGLRegisterBuffer = dlfcn.dlsym(handle, 'cuGraphicsGLRegisterBuffer') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreateFromEGLSync = dlsym(__pyx_v_handle, ((char const *)"cuEventCreateFromEGLSync")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2756 + * __cuEventCreateFromEGLSync = dlfcn.dlsym(handle, 'cuEventCreateFromEGLSync') + * global __cuGraphicsGLRegisterBuffer + * __cuGraphicsGLRegisterBuffer = dlfcn.dlsym(handle, 'cuGraphicsGLRegisterBuffer') # <<<<<<<<<<<<<< + * global __cuGraphicsGLRegisterImage + * __cuGraphicsGLRegisterImage = dlfcn.dlsym(handle, 'cuGraphicsGLRegisterImage') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterBuffer = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsGLRegisterBuffer")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2758 + * __cuGraphicsGLRegisterBuffer = dlfcn.dlsym(handle, 'cuGraphicsGLRegisterBuffer') + * global __cuGraphicsGLRegisterImage + * __cuGraphicsGLRegisterImage = dlfcn.dlsym(handle, 'cuGraphicsGLRegisterImage') # <<<<<<<<<<<<<< + * global __cuGLGetDevices_v2 + * __cuGLGetDevices_v2 = dlfcn.dlsym(handle, 'cuGLGetDevices_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterImage = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsGLRegisterImage")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2760 + * __cuGraphicsGLRegisterImage = dlfcn.dlsym(handle, 'cuGraphicsGLRegisterImage') + * global __cuGLGetDevices_v2 + * __cuGLGetDevices_v2 = dlfcn.dlsym(handle, 'cuGLGetDevices_v2') # <<<<<<<<<<<<<< + * global __cuVDPAUGetDevice + * __cuVDPAUGetDevice = dlfcn.dlsym(handle, 'cuVDPAUGetDevice') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGLGetDevices_v2 = dlsym(__pyx_v_handle, ((char const *)"cuGLGetDevices_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2762 + * __cuGLGetDevices_v2 = dlfcn.dlsym(handle, 'cuGLGetDevices_v2') + * global __cuVDPAUGetDevice + * __cuVDPAUGetDevice = dlfcn.dlsym(handle, 'cuVDPAUGetDevice') # <<<<<<<<<<<<<< + * global __cuVDPAUCtxCreate_v2 + * __cuVDPAUCtxCreate_v2 = dlfcn.dlsym(handle, 'cuVDPAUCtxCreate_v2') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUGetDevice = dlsym(__pyx_v_handle, ((char const *)"cuVDPAUGetDevice")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2764 + * __cuVDPAUGetDevice = dlfcn.dlsym(handle, 'cuVDPAUGetDevice') + * global __cuVDPAUCtxCreate_v2 + * __cuVDPAUCtxCreate_v2 = dlfcn.dlsym(handle, 'cuVDPAUCtxCreate_v2') # <<<<<<<<<<<<<< + * global __cuGraphicsVDPAURegisterVideoSurface + * __cuGraphicsVDPAURegisterVideoSurface = dlfcn.dlsym(handle, 'cuGraphicsVDPAURegisterVideoSurface') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUCtxCreate_v2 = dlsym(__pyx_v_handle, ((char const *)"cuVDPAUCtxCreate_v2")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2766 + * __cuVDPAUCtxCreate_v2 = dlfcn.dlsym(handle, 'cuVDPAUCtxCreate_v2') + * global __cuGraphicsVDPAURegisterVideoSurface + * __cuGraphicsVDPAURegisterVideoSurface = dlfcn.dlsym(handle, 'cuGraphicsVDPAURegisterVideoSurface') # <<<<<<<<<<<<<< + * global __cuGraphicsVDPAURegisterOutputSurface + * __cuGraphicsVDPAURegisterOutputSurface = dlfcn.dlsym(handle, 'cuGraphicsVDPAURegisterOutputSurface') +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterVideoSurface = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsVDPAURegisterVideoSurface")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2768 + * __cuGraphicsVDPAURegisterVideoSurface = dlfcn.dlsym(handle, 'cuGraphicsVDPAURegisterVideoSurface') + * global __cuGraphicsVDPAURegisterOutputSurface + * __cuGraphicsVDPAURegisterOutputSurface = dlfcn.dlsym(handle, 'cuGraphicsVDPAURegisterOutputSurface') # <<<<<<<<<<<<<< + * __cuPythonInit = True + * return 0 +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterOutputSurface = dlsym(__pyx_v_handle, ((char const *)"cuGraphicsVDPAURegisterOutputSurface")); + + /* "cuda/bindings/_bindings/cydriver.pyx":2769 + * global __cuGraphicsVDPAURegisterOutputSurface + * __cuGraphicsVDPAURegisterOutputSurface = dlfcn.dlsym(handle, 'cuGraphicsVDPAURegisterOutputSurface') + * __cuPythonInit = True # <<<<<<<<<<<<<< + * return 0 + * +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPythonInit = 1; + + /* "cuda/bindings/_bindings/cydriver.pyx":2770 + * __cuGraphicsVDPAURegisterOutputSurface = dlfcn.dlsym(handle, 'cuGraphicsVDPAURegisterOutputSurface') + * __cuPythonInit = True + * return 0 # <<<<<<<<<<<<<< + * + * # Create a very small function to check whether we are init'ed, so the C +*/ + __pyx_r = 0; + goto __pyx_L14_try_return; + + /* "cuda/bindings/_bindings/cydriver.pyx":498 + * cdef char libPath[260] + * + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * usePTDS = os.getenv('CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM', default=0) + * +*/ + } + __pyx_L10_error:; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + /*except:*/ { + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuPythonInit", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_4, &__pyx_t_9) < 0) __PYX_ERR(0, 498, __pyx_L12_except_error) + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_9); + __pyx_t_3 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_4, __pyx_t_9); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 498, __pyx_L12_except_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_15 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_3, NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 498, __pyx_L12_except_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_10 = __Pyx_PyObject_IsTrue(__pyx_t_15); + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + if (__pyx_t_10 < (0)) __PYX_ERR(0, 498, __pyx_L12_except_error) + __pyx_t_11 = (!__pyx_t_10); + if (unlikely(__pyx_t_11)) { + __Pyx_GIVEREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_9); + __Pyx_ErrRestoreWithState(__pyx_t_2, __pyx_t_4, __pyx_t_9); + __pyx_t_2 = 0; __pyx_t_4 = 0; __pyx_t_9 = 0; + __PYX_ERR(0, 498, __pyx_L12_except_error) + } + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + goto __pyx_L11_exception_handled; + } + __pyx_L12_except_error:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L4_error; + __pyx_L14_try_return:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L7_return; + __pyx_L11_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + } + } + /*finally:*/ { + /*normal exit:*/{ + if (__pyx_t_1) { + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_mstate_global->__pyx_tuple[0], NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 498, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + goto __pyx_L9; + } + __pyx_L7_return: { + __pyx_t_16 = __pyx_r; + if (__pyx_t_1) { + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_mstate_global->__pyx_tuple[0], NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 498, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + __pyx_r = __pyx_t_16; + goto __pyx_L3_return; + } + __pyx_L9:; + } + goto __pyx_L26; + __pyx_L6_error:; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L4_error; + __pyx_L26:; + } + } + /*finally:*/ { + /*normal exit:*/{ + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L5; + } + __pyx_L3_return: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L0; + } + __pyx_L4_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + __pyx_L5:; + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":492 + * cdef __cuGetProcAddress_v2_T _F_cuGetProcAddress_v2 = NULL + * + * cdef int _cuPythonInit() except -1 nogil: # <<<<<<<<<<<<<< + * global __cuPythonInit + * +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_9); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuPythonInit", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_v_path); + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2774 + * # Create a very small function to check whether we are init'ed, so the C + * # compiler can inline it. + * cdef inline int cuPythonInit() except -1 nogil: # <<<<<<<<<<<<<< + * if __cuPythonInit: + * return 0 +*/ + +static CYTHON_INLINE int __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(void) { + int __pyx_r; + int __pyx_t_1; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cydriver.pyx":2775 + * # compiler can inline it. + * cdef inline int cuPythonInit() except -1 nogil: + * if __cuPythonInit: # <<<<<<<<<<<<<< + * return 0 + * return _cuPythonInit() +*/ + if (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPythonInit) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2776 + * cdef inline int cuPythonInit() except -1 nogil: + * if __cuPythonInit: + * return 0 # <<<<<<<<<<<<<< + * return _cuPythonInit() + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2775 + * # compiler can inline it. + * cdef inline int cuPythonInit() except -1 nogil: + * if __cuPythonInit: # <<<<<<<<<<<<<< + * return 0 + * return _cuPythonInit() +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2777 + * if __cuPythonInit: + * return 0 + * return _cuPythonInit() # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGetErrorString(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2777, __pyx_L1_error) + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2774 + * # Create a very small function to check whether we are init'ed, so the C + * # compiler can inline it. + * cdef inline int cuPythonInit() except -1 nogil: # <<<<<<<<<<<<<< + * if __cuPythonInit: + * return 0 +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver.cuPythonInit", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2779 + * return _cuPythonInit() + * + * cdef CUresult _cuGetErrorString(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGetErrorString + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGetErrorString(CUresult __pyx_v_error, char const **__pyx_v_pStr) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGetErrorString", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2781 + * cdef CUresult _cuGetErrorString(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGetErrorString + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGetErrorString == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2781, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2782 + * global __cuGetErrorString + * cuPythonInit() + * if __cuGetErrorString == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGetErrorString" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorString == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2783 + * cuPythonInit() + * if __cuGetErrorString == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGetErrorString" not found') + * err = ( __cuGetErrorString)(error, pStr) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2784 + * if __cuGetErrorString == NULL: + * with gil: + * raise RuntimeError('Function "cuGetErrorString" not found') # <<<<<<<<<<<<<< + * err = ( __cuGetErrorString)(error, pStr) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGetErrorString_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2784, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2784, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2783 + * cuPythonInit() + * if __cuGetErrorString == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGetErrorString" not found') + * err = ( __cuGetErrorString)(error, pStr) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2782 + * global __cuGetErrorString + * cuPythonInit() + * if __cuGetErrorString == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGetErrorString" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2785 + * with gil: + * raise RuntimeError('Function "cuGetErrorString" not found') + * err = ( __cuGetErrorString)(error, pStr) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUresult, char const **))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorString)(__pyx_v_error, __pyx_v_pStr); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2785, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2786 + * raise RuntimeError('Function "cuGetErrorString" not found') + * err = ( __cuGetErrorString)(error, pStr) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGetErrorName(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2779 + * return _cuPythonInit() + * + * cdef CUresult _cuGetErrorString(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGetErrorString + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGetErrorString", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2788 + * return err + * + * cdef CUresult _cuGetErrorName(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGetErrorName + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGetErrorName(CUresult __pyx_v_error, char const **__pyx_v_pStr) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGetErrorName", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2790 + * cdef CUresult _cuGetErrorName(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGetErrorName + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGetErrorName == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2790, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2791 + * global __cuGetErrorName + * cuPythonInit() + * if __cuGetErrorName == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGetErrorName" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorName == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2792 + * cuPythonInit() + * if __cuGetErrorName == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGetErrorName" not found') + * err = ( __cuGetErrorName)(error, pStr) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2793 + * if __cuGetErrorName == NULL: + * with gil: + * raise RuntimeError('Function "cuGetErrorName" not found') # <<<<<<<<<<<<<< + * err = ( __cuGetErrorName)(error, pStr) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGetErrorName_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2793, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2793, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2792 + * cuPythonInit() + * if __cuGetErrorName == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGetErrorName" not found') + * err = ( __cuGetErrorName)(error, pStr) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2791 + * global __cuGetErrorName + * cuPythonInit() + * if __cuGetErrorName == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGetErrorName" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2794 + * with gil: + * raise RuntimeError('Function "cuGetErrorName" not found') + * err = ( __cuGetErrorName)(error, pStr) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUresult, char const **))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorName)(__pyx_v_error, __pyx_v_pStr); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2794, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2795 + * raise RuntimeError('Function "cuGetErrorName" not found') + * err = ( __cuGetErrorName)(error, pStr) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuInit(unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2788 + * return err + * + * cdef CUresult _cuGetErrorName(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGetErrorName + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGetErrorName", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2797 + * return err + * + * cdef CUresult _cuInit(unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuInit + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuInit(unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuInit", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2799 + * cdef CUresult _cuInit(unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuInit + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuInit == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2799, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2800 + * global __cuInit + * cuPythonInit() + * if __cuInit == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuInit" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuInit == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2801 + * cuPythonInit() + * if __cuInit == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuInit" not found') + * err = ( __cuInit)(Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2802 + * if __cuInit == NULL: + * with gil: + * raise RuntimeError('Function "cuInit" not found') # <<<<<<<<<<<<<< + * err = ( __cuInit)(Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuInit_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2802, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2802, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2801 + * cuPythonInit() + * if __cuInit == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuInit" not found') + * err = ( __cuInit)(Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2800 + * global __cuInit + * cuPythonInit() + * if __cuInit == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuInit" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2803 + * with gil: + * raise RuntimeError('Function "cuInit" not found') + * err = ( __cuInit)(Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuInit)(__pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2803, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2804 + * raise RuntimeError('Function "cuInit" not found') + * err = ( __cuInit)(Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDriverGetVersion(int* driverVersion) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2797 + * return err + * + * cdef CUresult _cuInit(unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuInit + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuInit", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2806 + * return err + * + * cdef CUresult _cuDriverGetVersion(int* driverVersion) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDriverGetVersion + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDriverGetVersion(int *__pyx_v_driverVersion) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDriverGetVersion", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2808 + * cdef CUresult _cuDriverGetVersion(int* driverVersion) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDriverGetVersion + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDriverGetVersion == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2808, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2809 + * global __cuDriverGetVersion + * cuPythonInit() + * if __cuDriverGetVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDriverGetVersion" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDriverGetVersion == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2810 + * cuPythonInit() + * if __cuDriverGetVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDriverGetVersion" not found') + * err = ( __cuDriverGetVersion)(driverVersion) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2811 + * if __cuDriverGetVersion == NULL: + * with gil: + * raise RuntimeError('Function "cuDriverGetVersion" not found') # <<<<<<<<<<<<<< + * err = ( __cuDriverGetVersion)(driverVersion) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDriverGetVersion_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2811, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2811, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2810 + * cuPythonInit() + * if __cuDriverGetVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDriverGetVersion" not found') + * err = ( __cuDriverGetVersion)(driverVersion) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2809 + * global __cuDriverGetVersion + * cuPythonInit() + * if __cuDriverGetVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDriverGetVersion" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2812 + * with gil: + * raise RuntimeError('Function "cuDriverGetVersion" not found') + * err = ( __cuDriverGetVersion)(driverVersion) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDriverGetVersion)(__pyx_v_driverVersion); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2812, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2813 + * raise RuntimeError('Function "cuDriverGetVersion" not found') + * err = ( __cuDriverGetVersion)(driverVersion) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGet(CUdevice* device, int ordinal) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2806 + * return err + * + * cdef CUresult _cuDriverGetVersion(int* driverVersion) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDriverGetVersion + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDriverGetVersion", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2815 + * return err + * + * cdef CUresult _cuDeviceGet(CUdevice* device, int ordinal) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGet + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGet(CUdevice *__pyx_v_device, int __pyx_v_ordinal) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGet", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2817 + * cdef CUresult _cuDeviceGet(CUdevice* device, int ordinal) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGet + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGet == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2817, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2818 + * global __cuDeviceGet + * cuPythonInit() + * if __cuDeviceGet == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGet" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGet == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2819 + * cuPythonInit() + * if __cuDeviceGet == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGet" not found') + * err = ( __cuDeviceGet)(device, ordinal) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2820 + * if __cuDeviceGet == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGet" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGet)(device, ordinal) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGet_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2820, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2820, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2819 + * cuPythonInit() + * if __cuDeviceGet == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGet" not found') + * err = ( __cuDeviceGet)(device, ordinal) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2818 + * global __cuDeviceGet + * cuPythonInit() + * if __cuDeviceGet == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGet" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2821 + * with gil: + * raise RuntimeError('Function "cuDeviceGet" not found') + * err = ( __cuDeviceGet)(device, ordinal) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice *, int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGet)(__pyx_v_device, __pyx_v_ordinal); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2821, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2822 + * raise RuntimeError('Function "cuDeviceGet" not found') + * err = ( __cuDeviceGet)(device, ordinal) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetCount(int* count) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2815 + * return err + * + * cdef CUresult _cuDeviceGet(CUdevice* device, int ordinal) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGet + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGet", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2824 + * return err + * + * cdef CUresult _cuDeviceGetCount(int* count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetCount + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetCount(int *__pyx_v_count) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetCount", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2826 + * cdef CUresult _cuDeviceGetCount(int* count) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetCount + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetCount == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2826, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2827 + * global __cuDeviceGetCount + * cuPythonInit() + * if __cuDeviceGetCount == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetCount" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetCount == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2828 + * cuPythonInit() + * if __cuDeviceGetCount == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetCount" not found') + * err = ( __cuDeviceGetCount)(count) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2829 + * if __cuDeviceGetCount == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetCount" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetCount)(count) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetCount_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2829, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2829, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2828 + * cuPythonInit() + * if __cuDeviceGetCount == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetCount" not found') + * err = ( __cuDeviceGetCount)(count) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2827 + * global __cuDeviceGetCount + * cuPythonInit() + * if __cuDeviceGetCount == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetCount" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2830 + * with gil: + * raise RuntimeError('Function "cuDeviceGetCount" not found') + * err = ( __cuDeviceGetCount)(count) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetCount)(__pyx_v_count); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2830, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2831 + * raise RuntimeError('Function "cuDeviceGetCount" not found') + * err = ( __cuDeviceGetCount)(count) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetName(char* name, int length, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2824 + * return err + * + * cdef CUresult _cuDeviceGetCount(int* count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetCount + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetCount", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2833 + * return err + * + * cdef CUresult _cuDeviceGetName(char* name, int length, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetName + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetName(char *__pyx_v_name, int __pyx_v_length, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetName", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2835 + * cdef CUresult _cuDeviceGetName(char* name, int length, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetName + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetName == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2835, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2836 + * global __cuDeviceGetName + * cuPythonInit() + * if __cuDeviceGetName == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetName" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetName == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2837 + * cuPythonInit() + * if __cuDeviceGetName == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetName" not found') + * err = ( __cuDeviceGetName)(name, length, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2838 + * if __cuDeviceGetName == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetName" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetName)(name, length, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetName_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2838, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2838, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2837 + * cuPythonInit() + * if __cuDeviceGetName == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetName" not found') + * err = ( __cuDeviceGetName)(name, length, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2836 + * global __cuDeviceGetName + * cuPythonInit() + * if __cuDeviceGetName == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetName" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2839 + * with gil: + * raise RuntimeError('Function "cuDeviceGetName" not found') + * err = ( __cuDeviceGetName)(name, length, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(char *, int, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetName)(__pyx_v_name, __pyx_v_length, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2839, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2840 + * raise RuntimeError('Function "cuDeviceGetName" not found') + * err = ( __cuDeviceGetName)(name, length, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetUuid(CUuuid* uuid, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2833 + * return err + * + * cdef CUresult _cuDeviceGetName(char* name, int length, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetName + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetName", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2842 + * return err + * + * cdef CUresult _cuDeviceGetUuid(CUuuid* uuid, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetUuid + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetUuid(CUuuid *__pyx_v_uuid, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetUuid", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2844 + * cdef CUresult _cuDeviceGetUuid(CUuuid* uuid, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetUuid + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetUuid == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2844, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2845 + * global __cuDeviceGetUuid + * cuPythonInit() + * if __cuDeviceGetUuid == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetUuid" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2846 + * cuPythonInit() + * if __cuDeviceGetUuid == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetUuid" not found') + * err = ( __cuDeviceGetUuid)(uuid, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2847 + * if __cuDeviceGetUuid == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetUuid" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetUuid)(uuid, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetUuid_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2847, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2847, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2846 + * cuPythonInit() + * if __cuDeviceGetUuid == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetUuid" not found') + * err = ( __cuDeviceGetUuid)(uuid, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2845 + * global __cuDeviceGetUuid + * cuPythonInit() + * if __cuDeviceGetUuid == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetUuid" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2848 + * with gil: + * raise RuntimeError('Function "cuDeviceGetUuid" not found') + * err = ( __cuDeviceGetUuid)(uuid, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUuuid *, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid)(__pyx_v_uuid, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2848, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2849 + * raise RuntimeError('Function "cuDeviceGetUuid" not found') + * err = ( __cuDeviceGetUuid)(uuid, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetUuid_v2(CUuuid* uuid, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2842 + * return err + * + * cdef CUresult _cuDeviceGetUuid(CUuuid* uuid, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetUuid + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetUuid", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2851 + * return err + * + * cdef CUresult _cuDeviceGetUuid_v2(CUuuid* uuid, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetUuid_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetUuid_v2(CUuuid *__pyx_v_uuid, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetUuid_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2853 + * cdef CUresult _cuDeviceGetUuid_v2(CUuuid* uuid, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetUuid_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetUuid_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2853, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2854 + * global __cuDeviceGetUuid_v2 + * cuPythonInit() + * if __cuDeviceGetUuid_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetUuid_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2855 + * cuPythonInit() + * if __cuDeviceGetUuid_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetUuid_v2" not found') + * err = ( __cuDeviceGetUuid_v2)(uuid, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2856 + * if __cuDeviceGetUuid_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetUuid_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetUuid_v2)(uuid, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetUuid_v2_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2856, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2856, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2855 + * cuPythonInit() + * if __cuDeviceGetUuid_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetUuid_v2" not found') + * err = ( __cuDeviceGetUuid_v2)(uuid, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2854 + * global __cuDeviceGetUuid_v2 + * cuPythonInit() + * if __cuDeviceGetUuid_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetUuid_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2857 + * with gil: + * raise RuntimeError('Function "cuDeviceGetUuid_v2" not found') + * err = ( __cuDeviceGetUuid_v2)(uuid, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUuuid *, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid_v2)(__pyx_v_uuid, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2857, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2858 + * raise RuntimeError('Function "cuDeviceGetUuid_v2" not found') + * err = ( __cuDeviceGetUuid_v2)(uuid, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetLuid(char* luid, unsigned int* deviceNodeMask, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2851 + * return err + * + * cdef CUresult _cuDeviceGetUuid_v2(CUuuid* uuid, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetUuid_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetUuid_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2860 + * return err + * + * cdef CUresult _cuDeviceGetLuid(char* luid, unsigned int* deviceNodeMask, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetLuid + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetLuid(char *__pyx_v_luid, unsigned int *__pyx_v_deviceNodeMask, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetLuid", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2862 + * cdef CUresult _cuDeviceGetLuid(char* luid, unsigned int* deviceNodeMask, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetLuid + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetLuid == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2862, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2863 + * global __cuDeviceGetLuid + * cuPythonInit() + * if __cuDeviceGetLuid == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetLuid" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetLuid == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2864 + * cuPythonInit() + * if __cuDeviceGetLuid == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetLuid" not found') + * err = ( __cuDeviceGetLuid)(luid, deviceNodeMask, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2865 + * if __cuDeviceGetLuid == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetLuid" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetLuid)(luid, deviceNodeMask, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetLuid_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2865, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2865, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2864 + * cuPythonInit() + * if __cuDeviceGetLuid == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetLuid" not found') + * err = ( __cuDeviceGetLuid)(luid, deviceNodeMask, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2863 + * global __cuDeviceGetLuid + * cuPythonInit() + * if __cuDeviceGetLuid == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetLuid" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2866 + * with gil: + * raise RuntimeError('Function "cuDeviceGetLuid" not found') + * err = ( __cuDeviceGetLuid)(luid, deviceNodeMask, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(char *, unsigned int *, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetLuid)(__pyx_v_luid, __pyx_v_deviceNodeMask, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2866, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2867 + * raise RuntimeError('Function "cuDeviceGetLuid" not found') + * err = ( __cuDeviceGetLuid)(luid, deviceNodeMask, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceTotalMem_v2(size_t* numbytes, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2860 + * return err + * + * cdef CUresult _cuDeviceGetLuid(char* luid, unsigned int* deviceNodeMask, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetLuid + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetLuid", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2869 + * return err + * + * cdef CUresult _cuDeviceTotalMem_v2(size_t* numbytes, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceTotalMem_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceTotalMem_v2(size_t *__pyx_v_numbytes, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceTotalMem_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2871 + * cdef CUresult _cuDeviceTotalMem_v2(size_t* numbytes, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceTotalMem_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceTotalMem_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2871, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2872 + * global __cuDeviceTotalMem_v2 + * cuPythonInit() + * if __cuDeviceTotalMem_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceTotalMem_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceTotalMem_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2873 + * cuPythonInit() + * if __cuDeviceTotalMem_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceTotalMem_v2" not found') + * err = ( __cuDeviceTotalMem_v2)(numbytes, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2874 + * if __cuDeviceTotalMem_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceTotalMem_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceTotalMem_v2)(numbytes, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceTotalMem_v2_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2874, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2874, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2873 + * cuPythonInit() + * if __cuDeviceTotalMem_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceTotalMem_v2" not found') + * err = ( __cuDeviceTotalMem_v2)(numbytes, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2872 + * global __cuDeviceTotalMem_v2 + * cuPythonInit() + * if __cuDeviceTotalMem_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceTotalMem_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2875 + * with gil: + * raise RuntimeError('Function "cuDeviceTotalMem_v2" not found') + * err = ( __cuDeviceTotalMem_v2)(numbytes, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(size_t *, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceTotalMem_v2)(__pyx_v_numbytes, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2875, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2876 + * raise RuntimeError('Function "cuDeviceTotalMem_v2" not found') + * err = ( __cuDeviceTotalMem_v2)(numbytes, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, CUarray_format pformat, unsigned numChannels, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2869 + * return err + * + * cdef CUresult _cuDeviceTotalMem_v2(size_t* numbytes, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceTotalMem_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceTotalMem_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2878 + * return err + * + * cdef CUresult _cuDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, CUarray_format pformat, unsigned numChannels, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetTexture1DLinearMaxWidth + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetTexture1DLinearMaxWidth(size_t *__pyx_v_maxWidthInElements, CUarray_format __pyx_v_pformat, unsigned int __pyx_v_numChannels, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetTexture1DLinearMaxWidth", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2880 + * cdef CUresult _cuDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, CUarray_format pformat, unsigned numChannels, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetTexture1DLinearMaxWidth + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetTexture1DLinearMaxWidth == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2880, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2881 + * global __cuDeviceGetTexture1DLinearMaxWidth + * cuPythonInit() + * if __cuDeviceGetTexture1DLinearMaxWidth == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetTexture1DLinearMaxWidth" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetTexture1DLinearMaxWidth == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2882 + * cuPythonInit() + * if __cuDeviceGetTexture1DLinearMaxWidth == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetTexture1DLinearMaxWidth" not found') + * err = ( __cuDeviceGetTexture1DLinearMaxWidth)(maxWidthInElements, pformat, numChannels, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2883 + * if __cuDeviceGetTexture1DLinearMaxWidth == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetTexture1DLinearMaxWidth" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetTexture1DLinearMaxWidth)(maxWidthInElements, pformat, numChannels, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetTexture1DLin}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2883, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2883, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2882 + * cuPythonInit() + * if __cuDeviceGetTexture1DLinearMaxWidth == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetTexture1DLinearMaxWidth" not found') + * err = ( __cuDeviceGetTexture1DLinearMaxWidth)(maxWidthInElements, pformat, numChannels, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2881 + * global __cuDeviceGetTexture1DLinearMaxWidth + * cuPythonInit() + * if __cuDeviceGetTexture1DLinearMaxWidth == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetTexture1DLinearMaxWidth" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2884 + * with gil: + * raise RuntimeError('Function "cuDeviceGetTexture1DLinearMaxWidth" not found') + * err = ( __cuDeviceGetTexture1DLinearMaxWidth)(maxWidthInElements, pformat, numChannels, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(size_t *, CUarray_format, unsigned int, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetTexture1DLinearMaxWidth)(__pyx_v_maxWidthInElements, __pyx_v_pformat, __pyx_v_numChannels, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2884, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2885 + * raise RuntimeError('Function "cuDeviceGetTexture1DLinearMaxWidth" not found') + * err = ( __cuDeviceGetTexture1DLinearMaxWidth)(maxWidthInElements, pformat, numChannels, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetAttribute(int* pi, CUdevice_attribute attrib, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2878 + * return err + * + * cdef CUresult _cuDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, CUarray_format pformat, unsigned numChannels, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetTexture1DLinearMaxWidth + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetTexture1DLinearMaxWidth", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2887 + * return err + * + * cdef CUresult _cuDeviceGetAttribute(int* pi, CUdevice_attribute attrib, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetAttribute(int *__pyx_v_pi, CUdevice_attribute __pyx_v_attrib, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2889 + * cdef CUresult _cuDeviceGetAttribute(int* pi, CUdevice_attribute attrib, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2889, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2890 + * global __cuDeviceGetAttribute + * cuPythonInit() + * if __cuDeviceGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2891 + * cuPythonInit() + * if __cuDeviceGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetAttribute" not found') + * err = ( __cuDeviceGetAttribute)(pi, attrib, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2892 + * if __cuDeviceGetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetAttribute)(pi, attrib, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetAttribute_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2892, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2892, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2891 + * cuPythonInit() + * if __cuDeviceGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetAttribute" not found') + * err = ( __cuDeviceGetAttribute)(pi, attrib, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2890 + * global __cuDeviceGetAttribute + * cuPythonInit() + * if __cuDeviceGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2893 + * with gil: + * raise RuntimeError('Function "cuDeviceGetAttribute" not found') + * err = ( __cuDeviceGetAttribute)(pi, attrib, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, CUdevice_attribute, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetAttribute)(__pyx_v_pi, __pyx_v_attrib, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2893, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2894 + * raise RuntimeError('Function "cuDeviceGetAttribute" not found') + * err = ( __cuDeviceGetAttribute)(pi, attrib, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, CUdevice dev, int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2887 + * return err + * + * cdef CUresult _cuDeviceGetAttribute(int* pi, CUdevice_attribute attrib, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2896 + * return err + * + * cdef CUresult _cuDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, CUdevice dev, int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetNvSciSyncAttributes + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetNvSciSyncAttributes(void *__pyx_v_nvSciSyncAttrList, CUdevice __pyx_v_dev, int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetNvSciSyncAttributes", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2898 + * cdef CUresult _cuDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, CUdevice dev, int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetNvSciSyncAttributes + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetNvSciSyncAttributes == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2898, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2899 + * global __cuDeviceGetNvSciSyncAttributes + * cuPythonInit() + * if __cuDeviceGetNvSciSyncAttributes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetNvSciSyncAttributes" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetNvSciSyncAttributes == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2900 + * cuPythonInit() + * if __cuDeviceGetNvSciSyncAttributes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetNvSciSyncAttributes" not found') + * err = ( __cuDeviceGetNvSciSyncAttributes)(nvSciSyncAttrList, dev, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2901 + * if __cuDeviceGetNvSciSyncAttributes == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetNvSciSyncAttributes" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetNvSciSyncAttributes)(nvSciSyncAttrList, dev, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetNvSciSyncAtt}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2901, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2901, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2900 + * cuPythonInit() + * if __cuDeviceGetNvSciSyncAttributes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetNvSciSyncAttributes" not found') + * err = ( __cuDeviceGetNvSciSyncAttributes)(nvSciSyncAttrList, dev, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2899 + * global __cuDeviceGetNvSciSyncAttributes + * cuPythonInit() + * if __cuDeviceGetNvSciSyncAttributes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetNvSciSyncAttributes" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2902 + * with gil: + * raise RuntimeError('Function "cuDeviceGetNvSciSyncAttributes" not found') + * err = ( __cuDeviceGetNvSciSyncAttributes)(nvSciSyncAttrList, dev, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void *, CUdevice, int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetNvSciSyncAttributes)(__pyx_v_nvSciSyncAttrList, __pyx_v_dev, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2902, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2903 + * raise RuntimeError('Function "cuDeviceGetNvSciSyncAttributes" not found') + * err = ( __cuDeviceGetNvSciSyncAttributes)(nvSciSyncAttrList, dev, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceSetMemPool(CUdevice dev, CUmemoryPool pool) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2896 + * return err + * + * cdef CUresult _cuDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, CUdevice dev, int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetNvSciSyncAttributes + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetNvSciSyncAttributes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2905 + * return err + * + * cdef CUresult _cuDeviceSetMemPool(CUdevice dev, CUmemoryPool pool) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceSetMemPool + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceSetMemPool(CUdevice __pyx_v_dev, CUmemoryPool __pyx_v_pool) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceSetMemPool", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2907 + * cdef CUresult _cuDeviceSetMemPool(CUdevice dev, CUmemoryPool pool) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceSetMemPool + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceSetMemPool == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2907, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2908 + * global __cuDeviceSetMemPool + * cuPythonInit() + * if __cuDeviceSetMemPool == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceSetMemPool" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetMemPool == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2909 + * cuPythonInit() + * if __cuDeviceSetMemPool == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceSetMemPool" not found') + * err = ( __cuDeviceSetMemPool)(dev, pool) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2910 + * if __cuDeviceSetMemPool == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceSetMemPool" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceSetMemPool)(dev, pool) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceSetMemPool_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2910, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2910, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2909 + * cuPythonInit() + * if __cuDeviceSetMemPool == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceSetMemPool" not found') + * err = ( __cuDeviceSetMemPool)(dev, pool) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2908 + * global __cuDeviceSetMemPool + * cuPythonInit() + * if __cuDeviceSetMemPool == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceSetMemPool" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2911 + * with gil: + * raise RuntimeError('Function "cuDeviceSetMemPool" not found') + * err = ( __cuDeviceSetMemPool)(dev, pool) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice, CUmemoryPool))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetMemPool)(__pyx_v_dev, __pyx_v_pool); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2911, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2912 + * raise RuntimeError('Function "cuDeviceSetMemPool" not found') + * err = ( __cuDeviceSetMemPool)(dev, pool) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetMemPool(CUmemoryPool* pool, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2905 + * return err + * + * cdef CUresult _cuDeviceSetMemPool(CUdevice dev, CUmemoryPool pool) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceSetMemPool + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceSetMemPool", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2914 + * return err + * + * cdef CUresult _cuDeviceGetMemPool(CUmemoryPool* pool, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetMemPool + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetMemPool(CUmemoryPool *__pyx_v_pool, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetMemPool", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2916 + * cdef CUresult _cuDeviceGetMemPool(CUmemoryPool* pool, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetMemPool + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetMemPool == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2916, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2917 + * global __cuDeviceGetMemPool + * cuPythonInit() + * if __cuDeviceGetMemPool == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetMemPool" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetMemPool == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2918 + * cuPythonInit() + * if __cuDeviceGetMemPool == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetMemPool" not found') + * err = ( __cuDeviceGetMemPool)(pool, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2919 + * if __cuDeviceGetMemPool == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetMemPool" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetMemPool)(pool, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetMemPool_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2919, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2919, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2918 + * cuPythonInit() + * if __cuDeviceGetMemPool == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetMemPool" not found') + * err = ( __cuDeviceGetMemPool)(pool, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2917 + * global __cuDeviceGetMemPool + * cuPythonInit() + * if __cuDeviceGetMemPool == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetMemPool" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2920 + * with gil: + * raise RuntimeError('Function "cuDeviceGetMemPool" not found') + * err = ( __cuDeviceGetMemPool)(pool, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemoryPool *, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetMemPool)(__pyx_v_pool, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2920, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2921 + * raise RuntimeError('Function "cuDeviceGetMemPool" not found') + * err = ( __cuDeviceGetMemPool)(pool, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetDefaultMemPool(CUmemoryPool* pool_out, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2914 + * return err + * + * cdef CUresult _cuDeviceGetMemPool(CUmemoryPool* pool, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetMemPool + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetMemPool", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2923 + * return err + * + * cdef CUresult _cuDeviceGetDefaultMemPool(CUmemoryPool* pool_out, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetDefaultMemPool + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetDefaultMemPool(CUmemoryPool *__pyx_v_pool_out, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetDefaultMemPool", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2925 + * cdef CUresult _cuDeviceGetDefaultMemPool(CUmemoryPool* pool_out, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetDefaultMemPool + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetDefaultMemPool == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2925, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2926 + * global __cuDeviceGetDefaultMemPool + * cuPythonInit() + * if __cuDeviceGetDefaultMemPool == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetDefaultMemPool" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDefaultMemPool == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2927 + * cuPythonInit() + * if __cuDeviceGetDefaultMemPool == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetDefaultMemPool" not found') + * err = ( __cuDeviceGetDefaultMemPool)(pool_out, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2928 + * if __cuDeviceGetDefaultMemPool == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetDefaultMemPool" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetDefaultMemPool)(pool_out, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetDefaultMemPo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2928, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2928, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2927 + * cuPythonInit() + * if __cuDeviceGetDefaultMemPool == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetDefaultMemPool" not found') + * err = ( __cuDeviceGetDefaultMemPool)(pool_out, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2926 + * global __cuDeviceGetDefaultMemPool + * cuPythonInit() + * if __cuDeviceGetDefaultMemPool == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetDefaultMemPool" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2929 + * with gil: + * raise RuntimeError('Function "cuDeviceGetDefaultMemPool" not found') + * err = ( __cuDeviceGetDefaultMemPool)(pool_out, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemoryPool *, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDefaultMemPool)(__pyx_v_pool_out, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2929, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2930 + * raise RuntimeError('Function "cuDeviceGetDefaultMemPool" not found') + * err = ( __cuDeviceGetDefaultMemPool)(pool_out, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetExecAffinitySupport(int* pi, CUexecAffinityType typename, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2923 + * return err + * + * cdef CUresult _cuDeviceGetDefaultMemPool(CUmemoryPool* pool_out, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetDefaultMemPool + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetDefaultMemPool", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2932 + * return err + * + * cdef CUresult _cuDeviceGetExecAffinitySupport(int* pi, CUexecAffinityType typename, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetExecAffinitySupport + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetExecAffinitySupport(int *__pyx_v_pi, CUexecAffinityType __pyx_v_typename, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetExecAffinitySupport", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2934 + * cdef CUresult _cuDeviceGetExecAffinitySupport(int* pi, CUexecAffinityType typename, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetExecAffinitySupport + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetExecAffinitySupport == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2934, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2935 + * global __cuDeviceGetExecAffinitySupport + * cuPythonInit() + * if __cuDeviceGetExecAffinitySupport == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetExecAffinitySupport" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetExecAffinitySupport == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2936 + * cuPythonInit() + * if __cuDeviceGetExecAffinitySupport == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetExecAffinitySupport" not found') + * err = ( __cuDeviceGetExecAffinitySupport)(pi, typename, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2937 + * if __cuDeviceGetExecAffinitySupport == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetExecAffinitySupport" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetExecAffinitySupport)(pi, typename, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetExecAffinity}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2937, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2937, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2936 + * cuPythonInit() + * if __cuDeviceGetExecAffinitySupport == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetExecAffinitySupport" not found') + * err = ( __cuDeviceGetExecAffinitySupport)(pi, typename, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2935 + * global __cuDeviceGetExecAffinitySupport + * cuPythonInit() + * if __cuDeviceGetExecAffinitySupport == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetExecAffinitySupport" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2938 + * with gil: + * raise RuntimeError('Function "cuDeviceGetExecAffinitySupport" not found') + * err = ( __cuDeviceGetExecAffinitySupport)(pi, typename, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, CUexecAffinityType, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetExecAffinitySupport)(__pyx_v_pi, __pyx_v_typename, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2938, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2939 + * raise RuntimeError('Function "cuDeviceGetExecAffinitySupport" not found') + * err = ( __cuDeviceGetExecAffinitySupport)(pi, typename, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuFlushGPUDirectRDMAWrites(CUflushGPUDirectRDMAWritesTarget target, CUflushGPUDirectRDMAWritesScope scope) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2932 + * return err + * + * cdef CUresult _cuDeviceGetExecAffinitySupport(int* pi, CUexecAffinityType typename, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetExecAffinitySupport + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetExecAffinitySupport", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2941 + * return err + * + * cdef CUresult _cuFlushGPUDirectRDMAWrites(CUflushGPUDirectRDMAWritesTarget target, CUflushGPUDirectRDMAWritesScope scope) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFlushGPUDirectRDMAWrites + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFlushGPUDirectRDMAWrites(CUflushGPUDirectRDMAWritesTarget __pyx_v_target, CUflushGPUDirectRDMAWritesScope __pyx_v_scope) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFlushGPUDirectRDMAWrites", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2943 + * cdef CUresult _cuFlushGPUDirectRDMAWrites(CUflushGPUDirectRDMAWritesTarget target, CUflushGPUDirectRDMAWritesScope scope) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuFlushGPUDirectRDMAWrites + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuFlushGPUDirectRDMAWrites == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2943, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2944 + * global __cuFlushGPUDirectRDMAWrites + * cuPythonInit() + * if __cuFlushGPUDirectRDMAWrites == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFlushGPUDirectRDMAWrites" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFlushGPUDirectRDMAWrites == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2945 + * cuPythonInit() + * if __cuFlushGPUDirectRDMAWrites == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFlushGPUDirectRDMAWrites" not found') + * err = ( __cuFlushGPUDirectRDMAWrites)(target, scope) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2946 + * if __cuFlushGPUDirectRDMAWrites == NULL: + * with gil: + * raise RuntimeError('Function "cuFlushGPUDirectRDMAWrites" not found') # <<<<<<<<<<<<<< + * err = ( __cuFlushGPUDirectRDMAWrites)(target, scope) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuFlushGPUDirectRDMAWri}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2946, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2946, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2945 + * cuPythonInit() + * if __cuFlushGPUDirectRDMAWrites == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFlushGPUDirectRDMAWrites" not found') + * err = ( __cuFlushGPUDirectRDMAWrites)(target, scope) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2944 + * global __cuFlushGPUDirectRDMAWrites + * cuPythonInit() + * if __cuFlushGPUDirectRDMAWrites == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFlushGPUDirectRDMAWrites" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2947 + * with gil: + * raise RuntimeError('Function "cuFlushGPUDirectRDMAWrites" not found') + * err = ( __cuFlushGPUDirectRDMAWrites)(target, scope) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUflushGPUDirectRDMAWritesTarget, CUflushGPUDirectRDMAWritesScope))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFlushGPUDirectRDMAWrites)(__pyx_v_target, __pyx_v_scope); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2947, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2948 + * raise RuntimeError('Function "cuFlushGPUDirectRDMAWrites" not found') + * err = ( __cuFlushGPUDirectRDMAWrites)(target, scope) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetProperties(CUdevprop* prop, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2941 + * return err + * + * cdef CUresult _cuFlushGPUDirectRDMAWrites(CUflushGPUDirectRDMAWritesTarget target, CUflushGPUDirectRDMAWritesScope scope) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFlushGPUDirectRDMAWrites + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuFlushGPUDirectRDMAWrites", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2950 + * return err + * + * cdef CUresult _cuDeviceGetProperties(CUdevprop* prop, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetProperties + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetProperties(CUdevprop *__pyx_v_prop, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetProperties", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2952 + * cdef CUresult _cuDeviceGetProperties(CUdevprop* prop, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetProperties + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetProperties == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2952, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2953 + * global __cuDeviceGetProperties + * cuPythonInit() + * if __cuDeviceGetProperties == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetProperties" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetProperties == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2954 + * cuPythonInit() + * if __cuDeviceGetProperties == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetProperties" not found') + * err = ( __cuDeviceGetProperties)(prop, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2955 + * if __cuDeviceGetProperties == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetProperties" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetProperties)(prop, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetProperties_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2955, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2955, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2954 + * cuPythonInit() + * if __cuDeviceGetProperties == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetProperties" not found') + * err = ( __cuDeviceGetProperties)(prop, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2953 + * global __cuDeviceGetProperties + * cuPythonInit() + * if __cuDeviceGetProperties == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetProperties" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2956 + * with gil: + * raise RuntimeError('Function "cuDeviceGetProperties" not found') + * err = ( __cuDeviceGetProperties)(prop, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevprop *, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetProperties)(__pyx_v_prop, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2956, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2957 + * raise RuntimeError('Function "cuDeviceGetProperties" not found') + * err = ( __cuDeviceGetProperties)(prop, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceComputeCapability(int* major, int* minor, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2950 + * return err + * + * cdef CUresult _cuDeviceGetProperties(CUdevprop* prop, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetProperties + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetProperties", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2959 + * return err + * + * cdef CUresult _cuDeviceComputeCapability(int* major, int* minor, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceComputeCapability + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceComputeCapability(int *__pyx_v_major, int *__pyx_v_minor, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceComputeCapability", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2961 + * cdef CUresult _cuDeviceComputeCapability(int* major, int* minor, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceComputeCapability + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceComputeCapability == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2961, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2962 + * global __cuDeviceComputeCapability + * cuPythonInit() + * if __cuDeviceComputeCapability == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceComputeCapability" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceComputeCapability == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2963 + * cuPythonInit() + * if __cuDeviceComputeCapability == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceComputeCapability" not found') + * err = ( __cuDeviceComputeCapability)(major, minor, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2964 + * if __cuDeviceComputeCapability == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceComputeCapability" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceComputeCapability)(major, minor, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceComputeCapabili}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2964, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2964, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2963 + * cuPythonInit() + * if __cuDeviceComputeCapability == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceComputeCapability" not found') + * err = ( __cuDeviceComputeCapability)(major, minor, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2962 + * global __cuDeviceComputeCapability + * cuPythonInit() + * if __cuDeviceComputeCapability == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceComputeCapability" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2965 + * with gil: + * raise RuntimeError('Function "cuDeviceComputeCapability" not found') + * err = ( __cuDeviceComputeCapability)(major, minor, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, int *, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceComputeCapability)(__pyx_v_major, __pyx_v_minor, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2965, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2966 + * raise RuntimeError('Function "cuDeviceComputeCapability" not found') + * err = ( __cuDeviceComputeCapability)(major, minor, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDevicePrimaryCtxRetain(CUcontext* pctx, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2959 + * return err + * + * cdef CUresult _cuDeviceComputeCapability(int* major, int* minor, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceComputeCapability + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceComputeCapability", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2968 + * return err + * + * cdef CUresult _cuDevicePrimaryCtxRetain(CUcontext* pctx, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxRetain + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevicePrimaryCtxRetain(CUcontext *__pyx_v_pctx, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDevicePrimaryCtxRetain", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2970 + * cdef CUresult _cuDevicePrimaryCtxRetain(CUcontext* pctx, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDevicePrimaryCtxRetain + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDevicePrimaryCtxRetain == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2970, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2971 + * global __cuDevicePrimaryCtxRetain + * cuPythonInit() + * if __cuDevicePrimaryCtxRetain == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxRetain" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRetain == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2972 + * cuPythonInit() + * if __cuDevicePrimaryCtxRetain == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevicePrimaryCtxRetain" not found') + * err = ( __cuDevicePrimaryCtxRetain)(pctx, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2973 + * if __cuDevicePrimaryCtxRetain == NULL: + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxRetain" not found') # <<<<<<<<<<<<<< + * err = ( __cuDevicePrimaryCtxRetain)(pctx, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDevicePrimaryCtxRetai}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2973, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2973, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2972 + * cuPythonInit() + * if __cuDevicePrimaryCtxRetain == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevicePrimaryCtxRetain" not found') + * err = ( __cuDevicePrimaryCtxRetain)(pctx, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2971 + * global __cuDevicePrimaryCtxRetain + * cuPythonInit() + * if __cuDevicePrimaryCtxRetain == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxRetain" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2974 + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxRetain" not found') + * err = ( __cuDevicePrimaryCtxRetain)(pctx, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext *, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRetain)(__pyx_v_pctx, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2974, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2975 + * raise RuntimeError('Function "cuDevicePrimaryCtxRetain" not found') + * err = ( __cuDevicePrimaryCtxRetain)(pctx, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDevicePrimaryCtxRelease_v2(CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2968 + * return err + * + * cdef CUresult _cuDevicePrimaryCtxRetain(CUcontext* pctx, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxRetain + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDevicePrimaryCtxRetain", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2977 + * return err + * + * cdef CUresult _cuDevicePrimaryCtxRelease_v2(CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxRelease_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevicePrimaryCtxRelease_v2(CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDevicePrimaryCtxRelease_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2979 + * cdef CUresult _cuDevicePrimaryCtxRelease_v2(CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDevicePrimaryCtxRelease_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDevicePrimaryCtxRelease_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2979, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2980 + * global __cuDevicePrimaryCtxRelease_v2 + * cuPythonInit() + * if __cuDevicePrimaryCtxRelease_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxRelease_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRelease_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2981 + * cuPythonInit() + * if __cuDevicePrimaryCtxRelease_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevicePrimaryCtxRelease_v2" not found') + * err = ( __cuDevicePrimaryCtxRelease_v2)(dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2982 + * if __cuDevicePrimaryCtxRelease_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxRelease_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuDevicePrimaryCtxRelease_v2)(dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDevicePrimaryCtxRelea}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2982, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2982, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2981 + * cuPythonInit() + * if __cuDevicePrimaryCtxRelease_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevicePrimaryCtxRelease_v2" not found') + * err = ( __cuDevicePrimaryCtxRelease_v2)(dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2980 + * global __cuDevicePrimaryCtxRelease_v2 + * cuPythonInit() + * if __cuDevicePrimaryCtxRelease_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxRelease_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2983 + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxRelease_v2" not found') + * err = ( __cuDevicePrimaryCtxRelease_v2)(dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRelease_v2)(__pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2983, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2984 + * raise RuntimeError('Function "cuDevicePrimaryCtxRelease_v2" not found') + * err = ( __cuDevicePrimaryCtxRelease_v2)(dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDevicePrimaryCtxSetFlags_v2(CUdevice dev, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2977 + * return err + * + * cdef CUresult _cuDevicePrimaryCtxRelease_v2(CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxRelease_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDevicePrimaryCtxRelease_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2986 + * return err + * + * cdef CUresult _cuDevicePrimaryCtxSetFlags_v2(CUdevice dev, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxSetFlags_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevicePrimaryCtxSetFlags_v2(CUdevice __pyx_v_dev, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDevicePrimaryCtxSetFlags_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2988 + * cdef CUresult _cuDevicePrimaryCtxSetFlags_v2(CUdevice dev, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDevicePrimaryCtxSetFlags_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDevicePrimaryCtxSetFlags_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2988, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2989 + * global __cuDevicePrimaryCtxSetFlags_v2 + * cuPythonInit() + * if __cuDevicePrimaryCtxSetFlags_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxSetFlags_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxSetFlags_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2990 + * cuPythonInit() + * if __cuDevicePrimaryCtxSetFlags_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevicePrimaryCtxSetFlags_v2" not found') + * err = ( __cuDevicePrimaryCtxSetFlags_v2)(dev, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":2991 + * if __cuDevicePrimaryCtxSetFlags_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxSetFlags_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuDevicePrimaryCtxSetFlags_v2)(dev, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDevicePrimaryCtxSetFl}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 2991, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 2991, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2990 + * cuPythonInit() + * if __cuDevicePrimaryCtxSetFlags_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevicePrimaryCtxSetFlags_v2" not found') + * err = ( __cuDevicePrimaryCtxSetFlags_v2)(dev, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2989 + * global __cuDevicePrimaryCtxSetFlags_v2 + * cuPythonInit() + * if __cuDevicePrimaryCtxSetFlags_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxSetFlags_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2992 + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxSetFlags_v2" not found') + * err = ( __cuDevicePrimaryCtxSetFlags_v2)(dev, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxSetFlags_v2)(__pyx_v_dev, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 2992, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":2993 + * raise RuntimeError('Function "cuDevicePrimaryCtxSetFlags_v2" not found') + * err = ( __cuDevicePrimaryCtxSetFlags_v2)(dev, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDevicePrimaryCtxGetState(CUdevice dev, unsigned int* flags, int* active) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2986 + * return err + * + * cdef CUresult _cuDevicePrimaryCtxSetFlags_v2(CUdevice dev, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxSetFlags_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDevicePrimaryCtxSetFlags_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":2995 + * return err + * + * cdef CUresult _cuDevicePrimaryCtxGetState(CUdevice dev, unsigned int* flags, int* active) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxGetState + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevicePrimaryCtxGetState(CUdevice __pyx_v_dev, unsigned int *__pyx_v_flags, int *__pyx_v_active) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDevicePrimaryCtxGetState", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":2997 + * cdef CUresult _cuDevicePrimaryCtxGetState(CUdevice dev, unsigned int* flags, int* active) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDevicePrimaryCtxGetState + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDevicePrimaryCtxGetState == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 2997, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":2998 + * global __cuDevicePrimaryCtxGetState + * cuPythonInit() + * if __cuDevicePrimaryCtxGetState == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxGetState" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxGetState == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":2999 + * cuPythonInit() + * if __cuDevicePrimaryCtxGetState == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevicePrimaryCtxGetState" not found') + * err = ( __cuDevicePrimaryCtxGetState)(dev, flags, active) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3000 + * if __cuDevicePrimaryCtxGetState == NULL: + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxGetState" not found') # <<<<<<<<<<<<<< + * err = ( __cuDevicePrimaryCtxGetState)(dev, flags, active) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDevicePrimaryCtxGetSt}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3000, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3000, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2999 + * cuPythonInit() + * if __cuDevicePrimaryCtxGetState == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevicePrimaryCtxGetState" not found') + * err = ( __cuDevicePrimaryCtxGetState)(dev, flags, active) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":2998 + * global __cuDevicePrimaryCtxGetState + * cuPythonInit() + * if __cuDevicePrimaryCtxGetState == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxGetState" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3001 + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxGetState" not found') + * err = ( __cuDevicePrimaryCtxGetState)(dev, flags, active) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice, unsigned int *, int *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxGetState)(__pyx_v_dev, __pyx_v_flags, __pyx_v_active); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3001, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3002 + * raise RuntimeError('Function "cuDevicePrimaryCtxGetState" not found') + * err = ( __cuDevicePrimaryCtxGetState)(dev, flags, active) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDevicePrimaryCtxReset_v2(CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":2995 + * return err + * + * cdef CUresult _cuDevicePrimaryCtxGetState(CUdevice dev, unsigned int* flags, int* active) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxGetState + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDevicePrimaryCtxGetState", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3004 + * return err + * + * cdef CUresult _cuDevicePrimaryCtxReset_v2(CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxReset_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevicePrimaryCtxReset_v2(CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDevicePrimaryCtxReset_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3006 + * cdef CUresult _cuDevicePrimaryCtxReset_v2(CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDevicePrimaryCtxReset_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDevicePrimaryCtxReset_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3006, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3007 + * global __cuDevicePrimaryCtxReset_v2 + * cuPythonInit() + * if __cuDevicePrimaryCtxReset_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxReset_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxReset_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3008 + * cuPythonInit() + * if __cuDevicePrimaryCtxReset_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevicePrimaryCtxReset_v2" not found') + * err = ( __cuDevicePrimaryCtxReset_v2)(dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3009 + * if __cuDevicePrimaryCtxReset_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxReset_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuDevicePrimaryCtxReset_v2)(dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDevicePrimaryCtxReset}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3009, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3009, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3008 + * cuPythonInit() + * if __cuDevicePrimaryCtxReset_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevicePrimaryCtxReset_v2" not found') + * err = ( __cuDevicePrimaryCtxReset_v2)(dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3007 + * global __cuDevicePrimaryCtxReset_v2 + * cuPythonInit() + * if __cuDevicePrimaryCtxReset_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxReset_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3010 + * with gil: + * raise RuntimeError('Function "cuDevicePrimaryCtxReset_v2" not found') + * err = ( __cuDevicePrimaryCtxReset_v2)(dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxReset_v2)(__pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3010, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3011 + * raise RuntimeError('Function "cuDevicePrimaryCtxReset_v2" not found') + * err = ( __cuDevicePrimaryCtxReset_v2)(dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxCreate_v2(CUcontext* pctx, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3004 + * return err + * + * cdef CUresult _cuDevicePrimaryCtxReset_v2(CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxReset_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDevicePrimaryCtxReset_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3013 + * return err + * + * cdef CUresult _cuCtxCreate_v2(CUcontext* pctx, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxCreate_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxCreate_v2(CUcontext *__pyx_v_pctx, unsigned int __pyx_v_flags, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxCreate_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3015 + * cdef CUresult _cuCtxCreate_v2(CUcontext* pctx, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxCreate_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxCreate_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3015, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3016 + * global __cuCtxCreate_v2 + * cuPythonInit() + * if __cuCtxCreate_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxCreate_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3017 + * cuPythonInit() + * if __cuCtxCreate_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxCreate_v2" not found') + * err = ( __cuCtxCreate_v2)(pctx, flags, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3018 + * if __cuCtxCreate_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxCreate_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxCreate_v2)(pctx, flags, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxCreate_v2_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3018, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3018, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3017 + * cuPythonInit() + * if __cuCtxCreate_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxCreate_v2" not found') + * err = ( __cuCtxCreate_v2)(pctx, flags, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3016 + * global __cuCtxCreate_v2 + * cuPythonInit() + * if __cuCtxCreate_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxCreate_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3019 + * with gil: + * raise RuntimeError('Function "cuCtxCreate_v2" not found') + * err = ( __cuCtxCreate_v2)(pctx, flags, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext *, unsigned int, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v2)(__pyx_v_pctx, __pyx_v_flags, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3019, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3020 + * raise RuntimeError('Function "cuCtxCreate_v2" not found') + * err = ( __cuCtxCreate_v2)(pctx, flags, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxCreate_v3(CUcontext* pctx, CUexecAffinityParam* paramsArray, int numParams, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3013 + * return err + * + * cdef CUresult _cuCtxCreate_v2(CUcontext* pctx, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxCreate_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxCreate_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3022 + * return err + * + * cdef CUresult _cuCtxCreate_v3(CUcontext* pctx, CUexecAffinityParam* paramsArray, int numParams, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxCreate_v3 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxCreate_v3(CUcontext *__pyx_v_pctx, CUexecAffinityParam *__pyx_v_paramsArray, int __pyx_v_numParams, unsigned int __pyx_v_flags, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxCreate_v3", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3024 + * cdef CUresult _cuCtxCreate_v3(CUcontext* pctx, CUexecAffinityParam* paramsArray, int numParams, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxCreate_v3 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxCreate_v3 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3024, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3025 + * global __cuCtxCreate_v3 + * cuPythonInit() + * if __cuCtxCreate_v3 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxCreate_v3" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v3 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3026 + * cuPythonInit() + * if __cuCtxCreate_v3 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxCreate_v3" not found') + * err = ( __cuCtxCreate_v3)(pctx, paramsArray, numParams, flags, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3027 + * if __cuCtxCreate_v3 == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxCreate_v3" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxCreate_v3)(pctx, paramsArray, numParams, flags, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxCreate_v3_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3027, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3027, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3026 + * cuPythonInit() + * if __cuCtxCreate_v3 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxCreate_v3" not found') + * err = ( __cuCtxCreate_v3)(pctx, paramsArray, numParams, flags, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3025 + * global __cuCtxCreate_v3 + * cuPythonInit() + * if __cuCtxCreate_v3 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxCreate_v3" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3028 + * with gil: + * raise RuntimeError('Function "cuCtxCreate_v3" not found') + * err = ( __cuCtxCreate_v3)(pctx, paramsArray, numParams, flags, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext *, CUexecAffinityParam *, int, unsigned int, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v3)(__pyx_v_pctx, __pyx_v_paramsArray, __pyx_v_numParams, __pyx_v_flags, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3028, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3029 + * raise RuntimeError('Function "cuCtxCreate_v3" not found') + * err = ( __cuCtxCreate_v3)(pctx, paramsArray, numParams, flags, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxCreate_v4(CUcontext* pctx, CUctxCreateParams* ctxCreateParams, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3022 + * return err + * + * cdef CUresult _cuCtxCreate_v3(CUcontext* pctx, CUexecAffinityParam* paramsArray, int numParams, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxCreate_v3 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxCreate_v3", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3031 + * return err + * + * cdef CUresult _cuCtxCreate_v4(CUcontext* pctx, CUctxCreateParams* ctxCreateParams, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxCreate_v4 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxCreate_v4(CUcontext *__pyx_v_pctx, CUctxCreateParams *__pyx_v_ctxCreateParams, unsigned int __pyx_v_flags, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxCreate_v4", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3033 + * cdef CUresult _cuCtxCreate_v4(CUcontext* pctx, CUctxCreateParams* ctxCreateParams, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxCreate_v4 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxCreate_v4 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3033, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3034 + * global __cuCtxCreate_v4 + * cuPythonInit() + * if __cuCtxCreate_v4 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxCreate_v4" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v4 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3035 + * cuPythonInit() + * if __cuCtxCreate_v4 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxCreate_v4" not found') + * err = ( __cuCtxCreate_v4)(pctx, ctxCreateParams, flags, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3036 + * if __cuCtxCreate_v4 == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxCreate_v4" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxCreate_v4)(pctx, ctxCreateParams, flags, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxCreate_v4_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3036, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3036, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3035 + * cuPythonInit() + * if __cuCtxCreate_v4 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxCreate_v4" not found') + * err = ( __cuCtxCreate_v4)(pctx, ctxCreateParams, flags, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3034 + * global __cuCtxCreate_v4 + * cuPythonInit() + * if __cuCtxCreate_v4 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxCreate_v4" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3037 + * with gil: + * raise RuntimeError('Function "cuCtxCreate_v4" not found') + * err = ( __cuCtxCreate_v4)(pctx, ctxCreateParams, flags, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext *, CUctxCreateParams *, unsigned int, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v4)(__pyx_v_pctx, __pyx_v_ctxCreateParams, __pyx_v_flags, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3037, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3038 + * raise RuntimeError('Function "cuCtxCreate_v4" not found') + * err = ( __cuCtxCreate_v4)(pctx, ctxCreateParams, flags, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxDestroy_v2(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3031 + * return err + * + * cdef CUresult _cuCtxCreate_v4(CUcontext* pctx, CUctxCreateParams* ctxCreateParams, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxCreate_v4 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxCreate_v4", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3040 + * return err + * + * cdef CUresult _cuCtxDestroy_v2(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxDestroy_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxDestroy_v2(CUcontext __pyx_v_ctx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxDestroy_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3042 + * cdef CUresult _cuCtxDestroy_v2(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxDestroy_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxDestroy_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3042, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3043 + * global __cuCtxDestroy_v2 + * cuPythonInit() + * if __cuCtxDestroy_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxDestroy_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDestroy_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3044 + * cuPythonInit() + * if __cuCtxDestroy_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxDestroy_v2" not found') + * err = ( __cuCtxDestroy_v2)(ctx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3045 + * if __cuCtxDestroy_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxDestroy_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxDestroy_v2)(ctx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxDestroy_v2_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3045, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3045, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3044 + * cuPythonInit() + * if __cuCtxDestroy_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxDestroy_v2" not found') + * err = ( __cuCtxDestroy_v2)(ctx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3043 + * global __cuCtxDestroy_v2 + * cuPythonInit() + * if __cuCtxDestroy_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxDestroy_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3046 + * with gil: + * raise RuntimeError('Function "cuCtxDestroy_v2" not found') + * err = ( __cuCtxDestroy_v2)(ctx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDestroy_v2)(__pyx_v_ctx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3046, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3047 + * raise RuntimeError('Function "cuCtxDestroy_v2" not found') + * err = ( __cuCtxDestroy_v2)(ctx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxPushCurrent_v2(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3040 + * return err + * + * cdef CUresult _cuCtxDestroy_v2(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxDestroy_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxDestroy_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3049 + * return err + * + * cdef CUresult _cuCtxPushCurrent_v2(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxPushCurrent_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxPushCurrent_v2(CUcontext __pyx_v_ctx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxPushCurrent_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3051 + * cdef CUresult _cuCtxPushCurrent_v2(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxPushCurrent_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxPushCurrent_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3051, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3052 + * global __cuCtxPushCurrent_v2 + * cuPythonInit() + * if __cuCtxPushCurrent_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxPushCurrent_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPushCurrent_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3053 + * cuPythonInit() + * if __cuCtxPushCurrent_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxPushCurrent_v2" not found') + * err = ( __cuCtxPushCurrent_v2)(ctx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3054 + * if __cuCtxPushCurrent_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxPushCurrent_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxPushCurrent_v2)(ctx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxPushCurrent_v2_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3054, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3054, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3053 + * cuPythonInit() + * if __cuCtxPushCurrent_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxPushCurrent_v2" not found') + * err = ( __cuCtxPushCurrent_v2)(ctx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3052 + * global __cuCtxPushCurrent_v2 + * cuPythonInit() + * if __cuCtxPushCurrent_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxPushCurrent_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3055 + * with gil: + * raise RuntimeError('Function "cuCtxPushCurrent_v2" not found') + * err = ( __cuCtxPushCurrent_v2)(ctx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPushCurrent_v2)(__pyx_v_ctx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3055, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3056 + * raise RuntimeError('Function "cuCtxPushCurrent_v2" not found') + * err = ( __cuCtxPushCurrent_v2)(ctx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxPopCurrent_v2(CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3049 + * return err + * + * cdef CUresult _cuCtxPushCurrent_v2(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxPushCurrent_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxPushCurrent_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3058 + * return err + * + * cdef CUresult _cuCtxPopCurrent_v2(CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxPopCurrent_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxPopCurrent_v2(CUcontext *__pyx_v_pctx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxPopCurrent_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3060 + * cdef CUresult _cuCtxPopCurrent_v2(CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxPopCurrent_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxPopCurrent_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3060, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3061 + * global __cuCtxPopCurrent_v2 + * cuPythonInit() + * if __cuCtxPopCurrent_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxPopCurrent_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPopCurrent_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3062 + * cuPythonInit() + * if __cuCtxPopCurrent_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxPopCurrent_v2" not found') + * err = ( __cuCtxPopCurrent_v2)(pctx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3063 + * if __cuCtxPopCurrent_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxPopCurrent_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxPopCurrent_v2)(pctx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxPopCurrent_v2_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3063, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3063, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3062 + * cuPythonInit() + * if __cuCtxPopCurrent_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxPopCurrent_v2" not found') + * err = ( __cuCtxPopCurrent_v2)(pctx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3061 + * global __cuCtxPopCurrent_v2 + * cuPythonInit() + * if __cuCtxPopCurrent_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxPopCurrent_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3064 + * with gil: + * raise RuntimeError('Function "cuCtxPopCurrent_v2" not found') + * err = ( __cuCtxPopCurrent_v2)(pctx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPopCurrent_v2)(__pyx_v_pctx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3064, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3065 + * raise RuntimeError('Function "cuCtxPopCurrent_v2" not found') + * err = ( __cuCtxPopCurrent_v2)(pctx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxSetCurrent(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3058 + * return err + * + * cdef CUresult _cuCtxPopCurrent_v2(CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxPopCurrent_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxPopCurrent_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3067 + * return err + * + * cdef CUresult _cuCtxSetCurrent(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxSetCurrent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxSetCurrent(CUcontext __pyx_v_ctx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxSetCurrent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3069 + * cdef CUresult _cuCtxSetCurrent(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxSetCurrent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxSetCurrent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3069, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3070 + * global __cuCtxSetCurrent + * cuPythonInit() + * if __cuCtxSetCurrent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxSetCurrent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCurrent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3071 + * cuPythonInit() + * if __cuCtxSetCurrent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxSetCurrent" not found') + * err = ( __cuCtxSetCurrent)(ctx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3072 + * if __cuCtxSetCurrent == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxSetCurrent" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxSetCurrent)(ctx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxSetCurrent_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3072, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3072, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3071 + * cuPythonInit() + * if __cuCtxSetCurrent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxSetCurrent" not found') + * err = ( __cuCtxSetCurrent)(ctx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3070 + * global __cuCtxSetCurrent + * cuPythonInit() + * if __cuCtxSetCurrent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxSetCurrent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3073 + * with gil: + * raise RuntimeError('Function "cuCtxSetCurrent" not found') + * err = ( __cuCtxSetCurrent)(ctx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCurrent)(__pyx_v_ctx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3073, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3074 + * raise RuntimeError('Function "cuCtxSetCurrent" not found') + * err = ( __cuCtxSetCurrent)(ctx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxGetCurrent(CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3067 + * return err + * + * cdef CUresult _cuCtxSetCurrent(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxSetCurrent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxSetCurrent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3076 + * return err + * + * cdef CUresult _cuCtxGetCurrent(CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetCurrent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetCurrent(CUcontext *__pyx_v_pctx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxGetCurrent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3078 + * cdef CUresult _cuCtxGetCurrent(CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxGetCurrent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxGetCurrent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3078, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3079 + * global __cuCtxGetCurrent + * cuPythonInit() + * if __cuCtxGetCurrent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetCurrent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCurrent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3080 + * cuPythonInit() + * if __cuCtxGetCurrent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetCurrent" not found') + * err = ( __cuCtxGetCurrent)(pctx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3081 + * if __cuCtxGetCurrent == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxGetCurrent" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxGetCurrent)(pctx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxGetCurrent_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3081, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3081, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3080 + * cuPythonInit() + * if __cuCtxGetCurrent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetCurrent" not found') + * err = ( __cuCtxGetCurrent)(pctx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3079 + * global __cuCtxGetCurrent + * cuPythonInit() + * if __cuCtxGetCurrent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetCurrent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3082 + * with gil: + * raise RuntimeError('Function "cuCtxGetCurrent" not found') + * err = ( __cuCtxGetCurrent)(pctx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCurrent)(__pyx_v_pctx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3082, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3083 + * raise RuntimeError('Function "cuCtxGetCurrent" not found') + * err = ( __cuCtxGetCurrent)(pctx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxGetDevice(CUdevice* device) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3076 + * return err + * + * cdef CUresult _cuCtxGetCurrent(CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetCurrent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxGetCurrent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3085 + * return err + * + * cdef CUresult _cuCtxGetDevice(CUdevice* device) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetDevice + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetDevice(CUdevice *__pyx_v_device) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxGetDevice", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3087 + * cdef CUresult _cuCtxGetDevice(CUdevice* device) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxGetDevice + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxGetDevice == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3087, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3088 + * global __cuCtxGetDevice + * cuPythonInit() + * if __cuCtxGetDevice == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetDevice" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevice == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3089 + * cuPythonInit() + * if __cuCtxGetDevice == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetDevice" not found') + * err = ( __cuCtxGetDevice)(device) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3090 + * if __cuCtxGetDevice == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxGetDevice" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxGetDevice)(device) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxGetDevice_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3090, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3090, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3089 + * cuPythonInit() + * if __cuCtxGetDevice == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetDevice" not found') + * err = ( __cuCtxGetDevice)(device) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3088 + * global __cuCtxGetDevice + * cuPythonInit() + * if __cuCtxGetDevice == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetDevice" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3091 + * with gil: + * raise RuntimeError('Function "cuCtxGetDevice" not found') + * err = ( __cuCtxGetDevice)(device) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevice)(__pyx_v_device); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3091, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3092 + * raise RuntimeError('Function "cuCtxGetDevice" not found') + * err = ( __cuCtxGetDevice)(device) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxGetFlags(unsigned int* flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3085 + * return err + * + * cdef CUresult _cuCtxGetDevice(CUdevice* device) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetDevice + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxGetDevice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3094 + * return err + * + * cdef CUresult _cuCtxGetFlags(unsigned int* flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetFlags + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetFlags(unsigned int *__pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxGetFlags", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3096 + * cdef CUresult _cuCtxGetFlags(unsigned int* flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxGetFlags + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxGetFlags == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3096, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3097 + * global __cuCtxGetFlags + * cuPythonInit() + * if __cuCtxGetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetFlags" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetFlags == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3098 + * cuPythonInit() + * if __cuCtxGetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetFlags" not found') + * err = ( __cuCtxGetFlags)(flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3099 + * if __cuCtxGetFlags == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxGetFlags" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxGetFlags)(flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxGetFlags_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3099, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3099, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3098 + * cuPythonInit() + * if __cuCtxGetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetFlags" not found') + * err = ( __cuCtxGetFlags)(flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3097 + * global __cuCtxGetFlags + * cuPythonInit() + * if __cuCtxGetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetFlags" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3100 + * with gil: + * raise RuntimeError('Function "cuCtxGetFlags" not found') + * err = ( __cuCtxGetFlags)(flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(unsigned int *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetFlags)(__pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3100, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3101 + * raise RuntimeError('Function "cuCtxGetFlags" not found') + * err = ( __cuCtxGetFlags)(flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxSetFlags(unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3094 + * return err + * + * cdef CUresult _cuCtxGetFlags(unsigned int* flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetFlags + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxGetFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3103 + * return err + * + * cdef CUresult _cuCtxSetFlags(unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxSetFlags + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxSetFlags(unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxSetFlags", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3105 + * cdef CUresult _cuCtxSetFlags(unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxSetFlags + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxSetFlags == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3105, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3106 + * global __cuCtxSetFlags + * cuPythonInit() + * if __cuCtxSetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxSetFlags" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetFlags == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3107 + * cuPythonInit() + * if __cuCtxSetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxSetFlags" not found') + * err = ( __cuCtxSetFlags)(flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3108 + * if __cuCtxSetFlags == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxSetFlags" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxSetFlags)(flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxSetFlags_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3108, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3108, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3107 + * cuPythonInit() + * if __cuCtxSetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxSetFlags" not found') + * err = ( __cuCtxSetFlags)(flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3106 + * global __cuCtxSetFlags + * cuPythonInit() + * if __cuCtxSetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxSetFlags" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3109 + * with gil: + * raise RuntimeError('Function "cuCtxSetFlags" not found') + * err = ( __cuCtxSetFlags)(flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetFlags)(__pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3109, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3110 + * raise RuntimeError('Function "cuCtxSetFlags" not found') + * err = ( __cuCtxSetFlags)(flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxGetId(CUcontext ctx, unsigned long long* ctxId) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3103 + * return err + * + * cdef CUresult _cuCtxSetFlags(unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxSetFlags + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxSetFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3112 + * return err + * + * cdef CUresult _cuCtxGetId(CUcontext ctx, unsigned long long* ctxId) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetId + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetId(CUcontext __pyx_v_ctx, unsigned PY_LONG_LONG *__pyx_v_ctxId) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxGetId", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3114 + * cdef CUresult _cuCtxGetId(CUcontext ctx, unsigned long long* ctxId) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxGetId + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxGetId == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3114, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3115 + * global __cuCtxGetId + * cuPythonInit() + * if __cuCtxGetId == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetId" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetId == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3116 + * cuPythonInit() + * if __cuCtxGetId == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetId" not found') + * err = ( __cuCtxGetId)(ctx, ctxId) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3117 + * if __cuCtxGetId == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxGetId" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxGetId)(ctx, ctxId) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxGetId_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3117, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3117, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3116 + * cuPythonInit() + * if __cuCtxGetId == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetId" not found') + * err = ( __cuCtxGetId)(ctx, ctxId) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3115 + * global __cuCtxGetId + * cuPythonInit() + * if __cuCtxGetId == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetId" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3118 + * with gil: + * raise RuntimeError('Function "cuCtxGetId" not found') + * err = ( __cuCtxGetId)(ctx, ctxId) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext, unsigned PY_LONG_LONG *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetId)(__pyx_v_ctx, __pyx_v_ctxId); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3118, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3119 + * raise RuntimeError('Function "cuCtxGetId" not found') + * err = ( __cuCtxGetId)(ctx, ctxId) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxSynchronize() except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3112 + * return err + * + * cdef CUresult _cuCtxGetId(CUcontext ctx, unsigned long long* ctxId) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetId + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxGetId", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3121 + * return err + * + * cdef CUresult _cuCtxSynchronize() except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxSynchronize + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxSynchronize(void) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxSynchronize", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3123 + * cdef CUresult _cuCtxSynchronize() except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxSynchronize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxSynchronize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3123, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3124 + * global __cuCtxSynchronize + * cuPythonInit() + * if __cuCtxSynchronize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxSynchronize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSynchronize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3125 + * cuPythonInit() + * if __cuCtxSynchronize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxSynchronize" not found') + * err = ( __cuCtxSynchronize)() +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3126 + * if __cuCtxSynchronize == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxSynchronize" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxSynchronize)() + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxSynchronize_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3126, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3126, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3125 + * cuPythonInit() + * if __cuCtxSynchronize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxSynchronize" not found') + * err = ( __cuCtxSynchronize)() +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3124 + * global __cuCtxSynchronize + * cuPythonInit() + * if __cuCtxSynchronize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxSynchronize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3127 + * with gil: + * raise RuntimeError('Function "cuCtxSynchronize" not found') + * err = ( __cuCtxSynchronize)() # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSynchronize)(); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3127, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3128 + * raise RuntimeError('Function "cuCtxSynchronize" not found') + * err = ( __cuCtxSynchronize)() + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxSetLimit(CUlimit limit, size_t value) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3121 + * return err + * + * cdef CUresult _cuCtxSynchronize() except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxSynchronize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxSynchronize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3130 + * return err + * + * cdef CUresult _cuCtxSetLimit(CUlimit limit, size_t value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxSetLimit + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxSetLimit(CUlimit __pyx_v_limit, size_t __pyx_v_value) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxSetLimit", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3132 + * cdef CUresult _cuCtxSetLimit(CUlimit limit, size_t value) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxSetLimit + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxSetLimit == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3132, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3133 + * global __cuCtxSetLimit + * cuPythonInit() + * if __cuCtxSetLimit == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxSetLimit" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetLimit == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3134 + * cuPythonInit() + * if __cuCtxSetLimit == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxSetLimit" not found') + * err = ( __cuCtxSetLimit)(limit, value) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3135 + * if __cuCtxSetLimit == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxSetLimit" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxSetLimit)(limit, value) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxSetLimit_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3135, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3135, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3134 + * cuPythonInit() + * if __cuCtxSetLimit == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxSetLimit" not found') + * err = ( __cuCtxSetLimit)(limit, value) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3133 + * global __cuCtxSetLimit + * cuPythonInit() + * if __cuCtxSetLimit == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxSetLimit" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3136 + * with gil: + * raise RuntimeError('Function "cuCtxSetLimit" not found') + * err = ( __cuCtxSetLimit)(limit, value) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlimit, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetLimit)(__pyx_v_limit, __pyx_v_value); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3136, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3137 + * raise RuntimeError('Function "cuCtxSetLimit" not found') + * err = ( __cuCtxSetLimit)(limit, value) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxGetLimit(size_t* pvalue, CUlimit limit) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3130 + * return err + * + * cdef CUresult _cuCtxSetLimit(CUlimit limit, size_t value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxSetLimit + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxSetLimit", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3139 + * return err + * + * cdef CUresult _cuCtxGetLimit(size_t* pvalue, CUlimit limit) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetLimit + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetLimit(size_t *__pyx_v_pvalue, CUlimit __pyx_v_limit) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxGetLimit", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3141 + * cdef CUresult _cuCtxGetLimit(size_t* pvalue, CUlimit limit) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxGetLimit + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxGetLimit == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3141, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3142 + * global __cuCtxGetLimit + * cuPythonInit() + * if __cuCtxGetLimit == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetLimit" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetLimit == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3143 + * cuPythonInit() + * if __cuCtxGetLimit == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetLimit" not found') + * err = ( __cuCtxGetLimit)(pvalue, limit) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3144 + * if __cuCtxGetLimit == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxGetLimit" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxGetLimit)(pvalue, limit) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxGetLimit_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3144, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3144, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3143 + * cuPythonInit() + * if __cuCtxGetLimit == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetLimit" not found') + * err = ( __cuCtxGetLimit)(pvalue, limit) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3142 + * global __cuCtxGetLimit + * cuPythonInit() + * if __cuCtxGetLimit == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetLimit" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3145 + * with gil: + * raise RuntimeError('Function "cuCtxGetLimit" not found') + * err = ( __cuCtxGetLimit)(pvalue, limit) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(size_t *, CUlimit))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetLimit)(__pyx_v_pvalue, __pyx_v_limit); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3145, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3146 + * raise RuntimeError('Function "cuCtxGetLimit" not found') + * err = ( __cuCtxGetLimit)(pvalue, limit) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxGetCacheConfig(CUfunc_cache* pconfig) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3139 + * return err + * + * cdef CUresult _cuCtxGetLimit(size_t* pvalue, CUlimit limit) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetLimit + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxGetLimit", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3148 + * return err + * + * cdef CUresult _cuCtxGetCacheConfig(CUfunc_cache* pconfig) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetCacheConfig + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetCacheConfig(CUfunc_cache *__pyx_v_pconfig) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxGetCacheConfig", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3150 + * cdef CUresult _cuCtxGetCacheConfig(CUfunc_cache* pconfig) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxGetCacheConfig + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxGetCacheConfig == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3150, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3151 + * global __cuCtxGetCacheConfig + * cuPythonInit() + * if __cuCtxGetCacheConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetCacheConfig" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCacheConfig == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3152 + * cuPythonInit() + * if __cuCtxGetCacheConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetCacheConfig" not found') + * err = ( __cuCtxGetCacheConfig)(pconfig) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3153 + * if __cuCtxGetCacheConfig == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxGetCacheConfig" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxGetCacheConfig)(pconfig) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxGetCacheConfig_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3153, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3153, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3152 + * cuPythonInit() + * if __cuCtxGetCacheConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetCacheConfig" not found') + * err = ( __cuCtxGetCacheConfig)(pconfig) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3151 + * global __cuCtxGetCacheConfig + * cuPythonInit() + * if __cuCtxGetCacheConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetCacheConfig" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3154 + * with gil: + * raise RuntimeError('Function "cuCtxGetCacheConfig" not found') + * err = ( __cuCtxGetCacheConfig)(pconfig) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunc_cache *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCacheConfig)(__pyx_v_pconfig); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3154, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3155 + * raise RuntimeError('Function "cuCtxGetCacheConfig" not found') + * err = ( __cuCtxGetCacheConfig)(pconfig) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxSetCacheConfig(CUfunc_cache config) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3148 + * return err + * + * cdef CUresult _cuCtxGetCacheConfig(CUfunc_cache* pconfig) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetCacheConfig + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxGetCacheConfig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3157 + * return err + * + * cdef CUresult _cuCtxSetCacheConfig(CUfunc_cache config) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxSetCacheConfig + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxSetCacheConfig(CUfunc_cache __pyx_v_config) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxSetCacheConfig", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3159 + * cdef CUresult _cuCtxSetCacheConfig(CUfunc_cache config) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxSetCacheConfig + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxSetCacheConfig == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3159, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3160 + * global __cuCtxSetCacheConfig + * cuPythonInit() + * if __cuCtxSetCacheConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxSetCacheConfig" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCacheConfig == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3161 + * cuPythonInit() + * if __cuCtxSetCacheConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxSetCacheConfig" not found') + * err = ( __cuCtxSetCacheConfig)(config) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3162 + * if __cuCtxSetCacheConfig == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxSetCacheConfig" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxSetCacheConfig)(config) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxSetCacheConfig_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3162, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3162, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3161 + * cuPythonInit() + * if __cuCtxSetCacheConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxSetCacheConfig" not found') + * err = ( __cuCtxSetCacheConfig)(config) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3160 + * global __cuCtxSetCacheConfig + * cuPythonInit() + * if __cuCtxSetCacheConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxSetCacheConfig" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3163 + * with gil: + * raise RuntimeError('Function "cuCtxSetCacheConfig" not found') + * err = ( __cuCtxSetCacheConfig)(config) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunc_cache))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCacheConfig)(__pyx_v_config); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3163, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3164 + * raise RuntimeError('Function "cuCtxSetCacheConfig" not found') + * err = ( __cuCtxSetCacheConfig)(config) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxGetApiVersion(CUcontext ctx, unsigned int* version) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3157 + * return err + * + * cdef CUresult _cuCtxSetCacheConfig(CUfunc_cache config) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxSetCacheConfig + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxSetCacheConfig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3166 + * return err + * + * cdef CUresult _cuCtxGetApiVersion(CUcontext ctx, unsigned int* version) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetApiVersion + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetApiVersion(CUcontext __pyx_v_ctx, unsigned int *__pyx_v_version) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxGetApiVersion", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3168 + * cdef CUresult _cuCtxGetApiVersion(CUcontext ctx, unsigned int* version) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxGetApiVersion + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxGetApiVersion == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3168, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3169 + * global __cuCtxGetApiVersion + * cuPythonInit() + * if __cuCtxGetApiVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetApiVersion" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetApiVersion == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3170 + * cuPythonInit() + * if __cuCtxGetApiVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetApiVersion" not found') + * err = ( __cuCtxGetApiVersion)(ctx, version) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3171 + * if __cuCtxGetApiVersion == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxGetApiVersion" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxGetApiVersion)(ctx, version) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxGetApiVersion_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3171, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3171, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3170 + * cuPythonInit() + * if __cuCtxGetApiVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetApiVersion" not found') + * err = ( __cuCtxGetApiVersion)(ctx, version) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3169 + * global __cuCtxGetApiVersion + * cuPythonInit() + * if __cuCtxGetApiVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetApiVersion" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3172 + * with gil: + * raise RuntimeError('Function "cuCtxGetApiVersion" not found') + * err = ( __cuCtxGetApiVersion)(ctx, version) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext, unsigned int *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetApiVersion)(__pyx_v_ctx, __pyx_v_version); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3172, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3173 + * raise RuntimeError('Function "cuCtxGetApiVersion" not found') + * err = ( __cuCtxGetApiVersion)(ctx, version) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3166 + * return err + * + * cdef CUresult _cuCtxGetApiVersion(CUcontext ctx, unsigned int* version) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetApiVersion + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxGetApiVersion", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3175 + * return err + * + * cdef CUresult _cuCtxGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetStreamPriorityRange + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetStreamPriorityRange(int *__pyx_v_leastPriority, int *__pyx_v_greatestPriority) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxGetStreamPriorityRange", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3177 + * cdef CUresult _cuCtxGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxGetStreamPriorityRange + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxGetStreamPriorityRange == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3177, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3178 + * global __cuCtxGetStreamPriorityRange + * cuPythonInit() + * if __cuCtxGetStreamPriorityRange == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetStreamPriorityRange" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetStreamPriorityRange == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3179 + * cuPythonInit() + * if __cuCtxGetStreamPriorityRange == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetStreamPriorityRange" not found') + * err = ( __cuCtxGetStreamPriorityRange)(leastPriority, greatestPriority) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3180 + * if __cuCtxGetStreamPriorityRange == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxGetStreamPriorityRange" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxGetStreamPriorityRange)(leastPriority, greatestPriority) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxGetStreamPriorityR}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3180, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3180, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3179 + * cuPythonInit() + * if __cuCtxGetStreamPriorityRange == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetStreamPriorityRange" not found') + * err = ( __cuCtxGetStreamPriorityRange)(leastPriority, greatestPriority) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3178 + * global __cuCtxGetStreamPriorityRange + * cuPythonInit() + * if __cuCtxGetStreamPriorityRange == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetStreamPriorityRange" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3181 + * with gil: + * raise RuntimeError('Function "cuCtxGetStreamPriorityRange" not found') + * err = ( __cuCtxGetStreamPriorityRange)(leastPriority, greatestPriority) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, int *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetStreamPriorityRange)(__pyx_v_leastPriority, __pyx_v_greatestPriority); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3181, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3182 + * raise RuntimeError('Function "cuCtxGetStreamPriorityRange" not found') + * err = ( __cuCtxGetStreamPriorityRange)(leastPriority, greatestPriority) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxResetPersistingL2Cache() except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3175 + * return err + * + * cdef CUresult _cuCtxGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetStreamPriorityRange + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxGetStreamPriorityRange", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3184 + * return err + * + * cdef CUresult _cuCtxResetPersistingL2Cache() except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxResetPersistingL2Cache + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxResetPersistingL2Cache(void) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxResetPersistingL2Cache", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3186 + * cdef CUresult _cuCtxResetPersistingL2Cache() except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxResetPersistingL2Cache + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxResetPersistingL2Cache == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3186, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3187 + * global __cuCtxResetPersistingL2Cache + * cuPythonInit() + * if __cuCtxResetPersistingL2Cache == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxResetPersistingL2Cache" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxResetPersistingL2Cache == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3188 + * cuPythonInit() + * if __cuCtxResetPersistingL2Cache == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxResetPersistingL2Cache" not found') + * err = ( __cuCtxResetPersistingL2Cache)() +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3189 + * if __cuCtxResetPersistingL2Cache == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxResetPersistingL2Cache" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxResetPersistingL2Cache)() + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxResetPersistingL2C}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3189, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3189, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3188 + * cuPythonInit() + * if __cuCtxResetPersistingL2Cache == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxResetPersistingL2Cache" not found') + * err = ( __cuCtxResetPersistingL2Cache)() +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3187 + * global __cuCtxResetPersistingL2Cache + * cuPythonInit() + * if __cuCtxResetPersistingL2Cache == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxResetPersistingL2Cache" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3190 + * with gil: + * raise RuntimeError('Function "cuCtxResetPersistingL2Cache" not found') + * err = ( __cuCtxResetPersistingL2Cache)() # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxResetPersistingL2Cache)(); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3190, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3191 + * raise RuntimeError('Function "cuCtxResetPersistingL2Cache" not found') + * err = ( __cuCtxResetPersistingL2Cache)() + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxGetExecAffinity(CUexecAffinityParam* pExecAffinity, CUexecAffinityType typename) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3184 + * return err + * + * cdef CUresult _cuCtxResetPersistingL2Cache() except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxResetPersistingL2Cache + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxResetPersistingL2Cache", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3193 + * return err + * + * cdef CUresult _cuCtxGetExecAffinity(CUexecAffinityParam* pExecAffinity, CUexecAffinityType typename) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetExecAffinity + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetExecAffinity(CUexecAffinityParam *__pyx_v_pExecAffinity, CUexecAffinityType __pyx_v_typename) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxGetExecAffinity", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3195 + * cdef CUresult _cuCtxGetExecAffinity(CUexecAffinityParam* pExecAffinity, CUexecAffinityType typename) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxGetExecAffinity + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxGetExecAffinity == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3195, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3196 + * global __cuCtxGetExecAffinity + * cuPythonInit() + * if __cuCtxGetExecAffinity == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetExecAffinity" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetExecAffinity == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3197 + * cuPythonInit() + * if __cuCtxGetExecAffinity == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetExecAffinity" not found') + * err = ( __cuCtxGetExecAffinity)(pExecAffinity, typename) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3198 + * if __cuCtxGetExecAffinity == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxGetExecAffinity" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxGetExecAffinity)(pExecAffinity, typename) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxGetExecAffinity_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3198, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3198, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3197 + * cuPythonInit() + * if __cuCtxGetExecAffinity == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetExecAffinity" not found') + * err = ( __cuCtxGetExecAffinity)(pExecAffinity, typename) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3196 + * global __cuCtxGetExecAffinity + * cuPythonInit() + * if __cuCtxGetExecAffinity == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetExecAffinity" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3199 + * with gil: + * raise RuntimeError('Function "cuCtxGetExecAffinity" not found') + * err = ( __cuCtxGetExecAffinity)(pExecAffinity, typename) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUexecAffinityParam *, CUexecAffinityType))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetExecAffinity)(__pyx_v_pExecAffinity, __pyx_v_typename); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3199, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3200 + * raise RuntimeError('Function "cuCtxGetExecAffinity" not found') + * err = ( __cuCtxGetExecAffinity)(pExecAffinity, typename) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxRecordEvent(CUcontext hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3193 + * return err + * + * cdef CUresult _cuCtxGetExecAffinity(CUexecAffinityParam* pExecAffinity, CUexecAffinityType typename) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetExecAffinity + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxGetExecAffinity", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3202 + * return err + * + * cdef CUresult _cuCtxRecordEvent(CUcontext hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxRecordEvent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxRecordEvent(CUcontext __pyx_v_hCtx, CUevent __pyx_v_hEvent) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxRecordEvent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3204 + * cdef CUresult _cuCtxRecordEvent(CUcontext hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxRecordEvent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxRecordEvent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3204, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3205 + * global __cuCtxRecordEvent + * cuPythonInit() + * if __cuCtxRecordEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxRecordEvent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxRecordEvent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3206 + * cuPythonInit() + * if __cuCtxRecordEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxRecordEvent" not found') + * err = ( __cuCtxRecordEvent)(hCtx, hEvent) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3207 + * if __cuCtxRecordEvent == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxRecordEvent" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxRecordEvent)(hCtx, hEvent) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxRecordEvent_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3207, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3207, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3206 + * cuPythonInit() + * if __cuCtxRecordEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxRecordEvent" not found') + * err = ( __cuCtxRecordEvent)(hCtx, hEvent) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3205 + * global __cuCtxRecordEvent + * cuPythonInit() + * if __cuCtxRecordEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxRecordEvent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3208 + * with gil: + * raise RuntimeError('Function "cuCtxRecordEvent" not found') + * err = ( __cuCtxRecordEvent)(hCtx, hEvent) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext, CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxRecordEvent)(__pyx_v_hCtx, __pyx_v_hEvent); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3208, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3209 + * raise RuntimeError('Function "cuCtxRecordEvent" not found') + * err = ( __cuCtxRecordEvent)(hCtx, hEvent) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxWaitEvent(CUcontext hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3202 + * return err + * + * cdef CUresult _cuCtxRecordEvent(CUcontext hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxRecordEvent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxRecordEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3211 + * return err + * + * cdef CUresult _cuCtxWaitEvent(CUcontext hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxWaitEvent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxWaitEvent(CUcontext __pyx_v_hCtx, CUevent __pyx_v_hEvent) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxWaitEvent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3213 + * cdef CUresult _cuCtxWaitEvent(CUcontext hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxWaitEvent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxWaitEvent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3213, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3214 + * global __cuCtxWaitEvent + * cuPythonInit() + * if __cuCtxWaitEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxWaitEvent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxWaitEvent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3215 + * cuPythonInit() + * if __cuCtxWaitEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxWaitEvent" not found') + * err = ( __cuCtxWaitEvent)(hCtx, hEvent) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3216 + * if __cuCtxWaitEvent == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxWaitEvent" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxWaitEvent)(hCtx, hEvent) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxWaitEvent_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3216, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3216, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3215 + * cuPythonInit() + * if __cuCtxWaitEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxWaitEvent" not found') + * err = ( __cuCtxWaitEvent)(hCtx, hEvent) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3214 + * global __cuCtxWaitEvent + * cuPythonInit() + * if __cuCtxWaitEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxWaitEvent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3217 + * with gil: + * raise RuntimeError('Function "cuCtxWaitEvent" not found') + * err = ( __cuCtxWaitEvent)(hCtx, hEvent) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext, CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxWaitEvent)(__pyx_v_hCtx, __pyx_v_hEvent); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3217, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3218 + * raise RuntimeError('Function "cuCtxWaitEvent" not found') + * err = ( __cuCtxWaitEvent)(hCtx, hEvent) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxAttach(CUcontext* pctx, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3211 + * return err + * + * cdef CUresult _cuCtxWaitEvent(CUcontext hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxWaitEvent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxWaitEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3220 + * return err + * + * cdef CUresult _cuCtxAttach(CUcontext* pctx, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxAttach + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxAttach(CUcontext *__pyx_v_pctx, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxAttach", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3222 + * cdef CUresult _cuCtxAttach(CUcontext* pctx, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxAttach + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxAttach == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3222, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3223 + * global __cuCtxAttach + * cuPythonInit() + * if __cuCtxAttach == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxAttach" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxAttach == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3224 + * cuPythonInit() + * if __cuCtxAttach == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxAttach" not found') + * err = ( __cuCtxAttach)(pctx, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3225 + * if __cuCtxAttach == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxAttach" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxAttach)(pctx, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxAttach_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3225, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3225, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3224 + * cuPythonInit() + * if __cuCtxAttach == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxAttach" not found') + * err = ( __cuCtxAttach)(pctx, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3223 + * global __cuCtxAttach + * cuPythonInit() + * if __cuCtxAttach == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxAttach" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3226 + * with gil: + * raise RuntimeError('Function "cuCtxAttach" not found') + * err = ( __cuCtxAttach)(pctx, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxAttach)(__pyx_v_pctx, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3226, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3227 + * raise RuntimeError('Function "cuCtxAttach" not found') + * err = ( __cuCtxAttach)(pctx, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxDetach(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3220 + * return err + * + * cdef CUresult _cuCtxAttach(CUcontext* pctx, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxAttach + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxAttach", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3229 + * return err + * + * cdef CUresult _cuCtxDetach(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxDetach + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxDetach(CUcontext __pyx_v_ctx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxDetach", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3231 + * cdef CUresult _cuCtxDetach(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxDetach + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxDetach == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3231, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3232 + * global __cuCtxDetach + * cuPythonInit() + * if __cuCtxDetach == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxDetach" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDetach == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3233 + * cuPythonInit() + * if __cuCtxDetach == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxDetach" not found') + * err = ( __cuCtxDetach)(ctx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3234 + * if __cuCtxDetach == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxDetach" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxDetach)(ctx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxDetach_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3234, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3234, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3233 + * cuPythonInit() + * if __cuCtxDetach == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxDetach" not found') + * err = ( __cuCtxDetach)(ctx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3232 + * global __cuCtxDetach + * cuPythonInit() + * if __cuCtxDetach == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxDetach" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3235 + * with gil: + * raise RuntimeError('Function "cuCtxDetach" not found') + * err = ( __cuCtxDetach)(ctx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDetach)(__pyx_v_ctx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3235, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3236 + * raise RuntimeError('Function "cuCtxDetach" not found') + * err = ( __cuCtxDetach)(ctx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxGetSharedMemConfig(CUsharedconfig* pConfig) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3229 + * return err + * + * cdef CUresult _cuCtxDetach(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxDetach + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxDetach", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3238 + * return err + * + * cdef CUresult _cuCtxGetSharedMemConfig(CUsharedconfig* pConfig) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetSharedMemConfig + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetSharedMemConfig(CUsharedconfig *__pyx_v_pConfig) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxGetSharedMemConfig", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3240 + * cdef CUresult _cuCtxGetSharedMemConfig(CUsharedconfig* pConfig) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxGetSharedMemConfig + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxGetSharedMemConfig == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3240, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3241 + * global __cuCtxGetSharedMemConfig + * cuPythonInit() + * if __cuCtxGetSharedMemConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetSharedMemConfig" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetSharedMemConfig == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3242 + * cuPythonInit() + * if __cuCtxGetSharedMemConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetSharedMemConfig" not found') + * err = ( __cuCtxGetSharedMemConfig)(pConfig) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3243 + * if __cuCtxGetSharedMemConfig == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxGetSharedMemConfig" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxGetSharedMemConfig)(pConfig) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxGetSharedMemConfig}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3243, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3243, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3242 + * cuPythonInit() + * if __cuCtxGetSharedMemConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetSharedMemConfig" not found') + * err = ( __cuCtxGetSharedMemConfig)(pConfig) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3241 + * global __cuCtxGetSharedMemConfig + * cuPythonInit() + * if __cuCtxGetSharedMemConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetSharedMemConfig" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3244 + * with gil: + * raise RuntimeError('Function "cuCtxGetSharedMemConfig" not found') + * err = ( __cuCtxGetSharedMemConfig)(pConfig) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUsharedconfig *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetSharedMemConfig)(__pyx_v_pConfig); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3244, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3245 + * raise RuntimeError('Function "cuCtxGetSharedMemConfig" not found') + * err = ( __cuCtxGetSharedMemConfig)(pConfig) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxSetSharedMemConfig(CUsharedconfig config) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3238 + * return err + * + * cdef CUresult _cuCtxGetSharedMemConfig(CUsharedconfig* pConfig) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetSharedMemConfig + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxGetSharedMemConfig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3247 + * return err + * + * cdef CUresult _cuCtxSetSharedMemConfig(CUsharedconfig config) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxSetSharedMemConfig + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxSetSharedMemConfig(CUsharedconfig __pyx_v_config) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxSetSharedMemConfig", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3249 + * cdef CUresult _cuCtxSetSharedMemConfig(CUsharedconfig config) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxSetSharedMemConfig + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxSetSharedMemConfig == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3249, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3250 + * global __cuCtxSetSharedMemConfig + * cuPythonInit() + * if __cuCtxSetSharedMemConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxSetSharedMemConfig" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetSharedMemConfig == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3251 + * cuPythonInit() + * if __cuCtxSetSharedMemConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxSetSharedMemConfig" not found') + * err = ( __cuCtxSetSharedMemConfig)(config) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3252 + * if __cuCtxSetSharedMemConfig == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxSetSharedMemConfig" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxSetSharedMemConfig)(config) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxSetSharedMemConfig}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3252, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3252, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3251 + * cuPythonInit() + * if __cuCtxSetSharedMemConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxSetSharedMemConfig" not found') + * err = ( __cuCtxSetSharedMemConfig)(config) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3250 + * global __cuCtxSetSharedMemConfig + * cuPythonInit() + * if __cuCtxSetSharedMemConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxSetSharedMemConfig" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3253 + * with gil: + * raise RuntimeError('Function "cuCtxSetSharedMemConfig" not found') + * err = ( __cuCtxSetSharedMemConfig)(config) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUsharedconfig))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetSharedMemConfig)(__pyx_v_config); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3253, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3254 + * raise RuntimeError('Function "cuCtxSetSharedMemConfig" not found') + * err = ( __cuCtxSetSharedMemConfig)(config) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuModuleLoad(CUmodule* module, const char* fname) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3247 + * return err + * + * cdef CUresult _cuCtxSetSharedMemConfig(CUsharedconfig config) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxSetSharedMemConfig + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxSetSharedMemConfig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3256 + * return err + * + * cdef CUresult _cuModuleLoad(CUmodule* module, const char* fname) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleLoad + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleLoad(CUmodule *__pyx_v_module, char const *__pyx_v_fname) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuModuleLoad", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3258 + * cdef CUresult _cuModuleLoad(CUmodule* module, const char* fname) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuModuleLoad + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuModuleLoad == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3258, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3259 + * global __cuModuleLoad + * cuPythonInit() + * if __cuModuleLoad == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleLoad" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoad == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3260 + * cuPythonInit() + * if __cuModuleLoad == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleLoad" not found') + * err = ( __cuModuleLoad)(module, fname) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3261 + * if __cuModuleLoad == NULL: + * with gil: + * raise RuntimeError('Function "cuModuleLoad" not found') # <<<<<<<<<<<<<< + * err = ( __cuModuleLoad)(module, fname) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuModuleLoad_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3261, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3261, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3260 + * cuPythonInit() + * if __cuModuleLoad == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleLoad" not found') + * err = ( __cuModuleLoad)(module, fname) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3259 + * global __cuModuleLoad + * cuPythonInit() + * if __cuModuleLoad == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleLoad" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3262 + * with gil: + * raise RuntimeError('Function "cuModuleLoad" not found') + * err = ( __cuModuleLoad)(module, fname) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmodule *, char const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoad)(__pyx_v_module, __pyx_v_fname); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3262, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3263 + * raise RuntimeError('Function "cuModuleLoad" not found') + * err = ( __cuModuleLoad)(module, fname) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuModuleLoadData(CUmodule* module, const void* image) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3256 + * return err + * + * cdef CUresult _cuModuleLoad(CUmodule* module, const char* fname) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleLoad + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuModuleLoad", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3265 + * return err + * + * cdef CUresult _cuModuleLoadData(CUmodule* module, const void* image) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleLoadData + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleLoadData(CUmodule *__pyx_v_module, void const *__pyx_v_image) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuModuleLoadData", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3267 + * cdef CUresult _cuModuleLoadData(CUmodule* module, const void* image) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuModuleLoadData + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuModuleLoadData == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3267, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3268 + * global __cuModuleLoadData + * cuPythonInit() + * if __cuModuleLoadData == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleLoadData" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadData == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3269 + * cuPythonInit() + * if __cuModuleLoadData == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleLoadData" not found') + * err = ( __cuModuleLoadData)(module, image) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3270 + * if __cuModuleLoadData == NULL: + * with gil: + * raise RuntimeError('Function "cuModuleLoadData" not found') # <<<<<<<<<<<<<< + * err = ( __cuModuleLoadData)(module, image) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuModuleLoadData_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3270, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3270, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3269 + * cuPythonInit() + * if __cuModuleLoadData == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleLoadData" not found') + * err = ( __cuModuleLoadData)(module, image) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3268 + * global __cuModuleLoadData + * cuPythonInit() + * if __cuModuleLoadData == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleLoadData" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3271 + * with gil: + * raise RuntimeError('Function "cuModuleLoadData" not found') + * err = ( __cuModuleLoadData)(module, image) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmodule *, void const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadData)(__pyx_v_module, __pyx_v_image); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3271, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3272 + * raise RuntimeError('Function "cuModuleLoadData" not found') + * err = ( __cuModuleLoadData)(module, image) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuModuleLoadDataEx(CUmodule* module, const void* image, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3265 + * return err + * + * cdef CUresult _cuModuleLoadData(CUmodule* module, const void* image) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleLoadData + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuModuleLoadData", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3274 + * return err + * + * cdef CUresult _cuModuleLoadDataEx(CUmodule* module, const void* image, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleLoadDataEx + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleLoadDataEx(CUmodule *__pyx_v_module, void const *__pyx_v_image, unsigned int __pyx_v_numOptions, CUjit_option *__pyx_v_options, void **__pyx_v_optionValues) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuModuleLoadDataEx", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3276 + * cdef CUresult _cuModuleLoadDataEx(CUmodule* module, const void* image, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuModuleLoadDataEx + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuModuleLoadDataEx == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3276, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3277 + * global __cuModuleLoadDataEx + * cuPythonInit() + * if __cuModuleLoadDataEx == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleLoadDataEx" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadDataEx == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3278 + * cuPythonInit() + * if __cuModuleLoadDataEx == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleLoadDataEx" not found') + * err = ( __cuModuleLoadDataEx)(module, image, numOptions, options, optionValues) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3279 + * if __cuModuleLoadDataEx == NULL: + * with gil: + * raise RuntimeError('Function "cuModuleLoadDataEx" not found') # <<<<<<<<<<<<<< + * err = ( __cuModuleLoadDataEx)(module, image, numOptions, options, optionValues) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuModuleLoadDataEx_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3279, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3279, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3278 + * cuPythonInit() + * if __cuModuleLoadDataEx == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleLoadDataEx" not found') + * err = ( __cuModuleLoadDataEx)(module, image, numOptions, options, optionValues) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3277 + * global __cuModuleLoadDataEx + * cuPythonInit() + * if __cuModuleLoadDataEx == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleLoadDataEx" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3280 + * with gil: + * raise RuntimeError('Function "cuModuleLoadDataEx" not found') + * err = ( __cuModuleLoadDataEx)(module, image, numOptions, options, optionValues) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmodule *, void const *, unsigned int, CUjit_option *, void **))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadDataEx)(__pyx_v_module, __pyx_v_image, __pyx_v_numOptions, __pyx_v_options, __pyx_v_optionValues); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3280, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3281 + * raise RuntimeError('Function "cuModuleLoadDataEx" not found') + * err = ( __cuModuleLoadDataEx)(module, image, numOptions, options, optionValues) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuModuleLoadFatBinary(CUmodule* module, const void* fatCubin) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3274 + * return err + * + * cdef CUresult _cuModuleLoadDataEx(CUmodule* module, const void* image, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleLoadDataEx + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuModuleLoadDataEx", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3283 + * return err + * + * cdef CUresult _cuModuleLoadFatBinary(CUmodule* module, const void* fatCubin) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleLoadFatBinary + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleLoadFatBinary(CUmodule *__pyx_v_module, void const *__pyx_v_fatCubin) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuModuleLoadFatBinary", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3285 + * cdef CUresult _cuModuleLoadFatBinary(CUmodule* module, const void* fatCubin) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuModuleLoadFatBinary + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuModuleLoadFatBinary == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3285, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3286 + * global __cuModuleLoadFatBinary + * cuPythonInit() + * if __cuModuleLoadFatBinary == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleLoadFatBinary" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadFatBinary == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3287 + * cuPythonInit() + * if __cuModuleLoadFatBinary == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleLoadFatBinary" not found') + * err = ( __cuModuleLoadFatBinary)(module, fatCubin) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3288 + * if __cuModuleLoadFatBinary == NULL: + * with gil: + * raise RuntimeError('Function "cuModuleLoadFatBinary" not found') # <<<<<<<<<<<<<< + * err = ( __cuModuleLoadFatBinary)(module, fatCubin) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuModuleLoadFatBinary_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3288, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3288, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3287 + * cuPythonInit() + * if __cuModuleLoadFatBinary == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleLoadFatBinary" not found') + * err = ( __cuModuleLoadFatBinary)(module, fatCubin) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3286 + * global __cuModuleLoadFatBinary + * cuPythonInit() + * if __cuModuleLoadFatBinary == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleLoadFatBinary" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3289 + * with gil: + * raise RuntimeError('Function "cuModuleLoadFatBinary" not found') + * err = ( __cuModuleLoadFatBinary)(module, fatCubin) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmodule *, void const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadFatBinary)(__pyx_v_module, __pyx_v_fatCubin); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3289, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3290 + * raise RuntimeError('Function "cuModuleLoadFatBinary" not found') + * err = ( __cuModuleLoadFatBinary)(module, fatCubin) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuModuleUnload(CUmodule hmod) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3283 + * return err + * + * cdef CUresult _cuModuleLoadFatBinary(CUmodule* module, const void* fatCubin) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleLoadFatBinary + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuModuleLoadFatBinary", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3292 + * return err + * + * cdef CUresult _cuModuleUnload(CUmodule hmod) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleUnload + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleUnload(CUmodule __pyx_v_hmod) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuModuleUnload", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3294 + * cdef CUresult _cuModuleUnload(CUmodule hmod) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuModuleUnload + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuModuleUnload == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3294, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3295 + * global __cuModuleUnload + * cuPythonInit() + * if __cuModuleUnload == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleUnload" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleUnload == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3296 + * cuPythonInit() + * if __cuModuleUnload == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleUnload" not found') + * err = ( __cuModuleUnload)(hmod) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3297 + * if __cuModuleUnload == NULL: + * with gil: + * raise RuntimeError('Function "cuModuleUnload" not found') # <<<<<<<<<<<<<< + * err = ( __cuModuleUnload)(hmod) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuModuleUnload_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3297, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3297, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3296 + * cuPythonInit() + * if __cuModuleUnload == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleUnload" not found') + * err = ( __cuModuleUnload)(hmod) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3295 + * global __cuModuleUnload + * cuPythonInit() + * if __cuModuleUnload == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleUnload" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3298 + * with gil: + * raise RuntimeError('Function "cuModuleUnload" not found') + * err = ( __cuModuleUnload)(hmod) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmodule))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleUnload)(__pyx_v_hmod); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3298, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3299 + * raise RuntimeError('Function "cuModuleUnload" not found') + * err = ( __cuModuleUnload)(hmod) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuModuleGetLoadingMode(CUmoduleLoadingMode* mode) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3292 + * return err + * + * cdef CUresult _cuModuleUnload(CUmodule hmod) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleUnload + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuModuleUnload", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3301 + * return err + * + * cdef CUresult _cuModuleGetLoadingMode(CUmoduleLoadingMode* mode) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleGetLoadingMode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleGetLoadingMode(CUmoduleLoadingMode *__pyx_v_mode) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuModuleGetLoadingMode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3303 + * cdef CUresult _cuModuleGetLoadingMode(CUmoduleLoadingMode* mode) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuModuleGetLoadingMode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuModuleGetLoadingMode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3303, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3304 + * global __cuModuleGetLoadingMode + * cuPythonInit() + * if __cuModuleGetLoadingMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleGetLoadingMode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetLoadingMode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3305 + * cuPythonInit() + * if __cuModuleGetLoadingMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleGetLoadingMode" not found') + * err = ( __cuModuleGetLoadingMode)(mode) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3306 + * if __cuModuleGetLoadingMode == NULL: + * with gil: + * raise RuntimeError('Function "cuModuleGetLoadingMode" not found') # <<<<<<<<<<<<<< + * err = ( __cuModuleGetLoadingMode)(mode) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuModuleGetLoadingMode}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3306, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3306, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3305 + * cuPythonInit() + * if __cuModuleGetLoadingMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleGetLoadingMode" not found') + * err = ( __cuModuleGetLoadingMode)(mode) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3304 + * global __cuModuleGetLoadingMode + * cuPythonInit() + * if __cuModuleGetLoadingMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleGetLoadingMode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3307 + * with gil: + * raise RuntimeError('Function "cuModuleGetLoadingMode" not found') + * err = ( __cuModuleGetLoadingMode)(mode) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmoduleLoadingMode *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetLoadingMode)(__pyx_v_mode); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3307, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3308 + * raise RuntimeError('Function "cuModuleGetLoadingMode" not found') + * err = ( __cuModuleGetLoadingMode)(mode) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuModuleGetFunction(CUfunction* hfunc, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3301 + * return err + * + * cdef CUresult _cuModuleGetLoadingMode(CUmoduleLoadingMode* mode) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleGetLoadingMode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuModuleGetLoadingMode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3310 + * return err + * + * cdef CUresult _cuModuleGetFunction(CUfunction* hfunc, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleGetFunction + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleGetFunction(CUfunction *__pyx_v_hfunc, CUmodule __pyx_v_hmod, char const *__pyx_v_name) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuModuleGetFunction", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3312 + * cdef CUresult _cuModuleGetFunction(CUfunction* hfunc, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuModuleGetFunction + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuModuleGetFunction == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3312, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3313 + * global __cuModuleGetFunction + * cuPythonInit() + * if __cuModuleGetFunction == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleGetFunction" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunction == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3314 + * cuPythonInit() + * if __cuModuleGetFunction == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleGetFunction" not found') + * err = ( __cuModuleGetFunction)(hfunc, hmod, name) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3315 + * if __cuModuleGetFunction == NULL: + * with gil: + * raise RuntimeError('Function "cuModuleGetFunction" not found') # <<<<<<<<<<<<<< + * err = ( __cuModuleGetFunction)(hfunc, hmod, name) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuModuleGetFunction_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3315, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3315, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3314 + * cuPythonInit() + * if __cuModuleGetFunction == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleGetFunction" not found') + * err = ( __cuModuleGetFunction)(hfunc, hmod, name) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3313 + * global __cuModuleGetFunction + * cuPythonInit() + * if __cuModuleGetFunction == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleGetFunction" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3316 + * with gil: + * raise RuntimeError('Function "cuModuleGetFunction" not found') + * err = ( __cuModuleGetFunction)(hfunc, hmod, name) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction *, CUmodule, char const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunction)(__pyx_v_hfunc, __pyx_v_hmod, __pyx_v_name); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3316, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3317 + * raise RuntimeError('Function "cuModuleGetFunction" not found') + * err = ( __cuModuleGetFunction)(hfunc, hmod, name) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuModuleGetFunctionCount(unsigned int* count, CUmodule mod) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3310 + * return err + * + * cdef CUresult _cuModuleGetFunction(CUfunction* hfunc, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleGetFunction + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuModuleGetFunction", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3319 + * return err + * + * cdef CUresult _cuModuleGetFunctionCount(unsigned int* count, CUmodule mod) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleGetFunctionCount + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleGetFunctionCount(unsigned int *__pyx_v_count, CUmodule __pyx_v_mod) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuModuleGetFunctionCount", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3321 + * cdef CUresult _cuModuleGetFunctionCount(unsigned int* count, CUmodule mod) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuModuleGetFunctionCount + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuModuleGetFunctionCount == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3321, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3322 + * global __cuModuleGetFunctionCount + * cuPythonInit() + * if __cuModuleGetFunctionCount == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleGetFunctionCount" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunctionCount == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3323 + * cuPythonInit() + * if __cuModuleGetFunctionCount == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleGetFunctionCount" not found') + * err = ( __cuModuleGetFunctionCount)(count, mod) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3324 + * if __cuModuleGetFunctionCount == NULL: + * with gil: + * raise RuntimeError('Function "cuModuleGetFunctionCount" not found') # <<<<<<<<<<<<<< + * err = ( __cuModuleGetFunctionCount)(count, mod) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuModuleGetFunctionCoun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3324, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3324, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3323 + * cuPythonInit() + * if __cuModuleGetFunctionCount == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleGetFunctionCount" not found') + * err = ( __cuModuleGetFunctionCount)(count, mod) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3322 + * global __cuModuleGetFunctionCount + * cuPythonInit() + * if __cuModuleGetFunctionCount == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleGetFunctionCount" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3325 + * with gil: + * raise RuntimeError('Function "cuModuleGetFunctionCount" not found') + * err = ( __cuModuleGetFunctionCount)(count, mod) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(unsigned int *, CUmodule))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunctionCount)(__pyx_v_count, __pyx_v_mod); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3325, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3326 + * raise RuntimeError('Function "cuModuleGetFunctionCount" not found') + * err = ( __cuModuleGetFunctionCount)(count, mod) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuModuleEnumerateFunctions(CUfunction* functions, unsigned int numFunctions, CUmodule mod) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3319 + * return err + * + * cdef CUresult _cuModuleGetFunctionCount(unsigned int* count, CUmodule mod) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleGetFunctionCount + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuModuleGetFunctionCount", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3328 + * return err + * + * cdef CUresult _cuModuleEnumerateFunctions(CUfunction* functions, unsigned int numFunctions, CUmodule mod) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleEnumerateFunctions + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleEnumerateFunctions(CUfunction *__pyx_v_functions, unsigned int __pyx_v_numFunctions, CUmodule __pyx_v_mod) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuModuleEnumerateFunctions", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3330 + * cdef CUresult _cuModuleEnumerateFunctions(CUfunction* functions, unsigned int numFunctions, CUmodule mod) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuModuleEnumerateFunctions + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuModuleEnumerateFunctions == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3330, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3331 + * global __cuModuleEnumerateFunctions + * cuPythonInit() + * if __cuModuleEnumerateFunctions == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleEnumerateFunctions" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleEnumerateFunctions == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3332 + * cuPythonInit() + * if __cuModuleEnumerateFunctions == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleEnumerateFunctions" not found') + * err = ( __cuModuleEnumerateFunctions)(functions, numFunctions, mod) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3333 + * if __cuModuleEnumerateFunctions == NULL: + * with gil: + * raise RuntimeError('Function "cuModuleEnumerateFunctions" not found') # <<<<<<<<<<<<<< + * err = ( __cuModuleEnumerateFunctions)(functions, numFunctions, mod) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuModuleEnumerateFuncti}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3333, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3333, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3332 + * cuPythonInit() + * if __cuModuleEnumerateFunctions == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleEnumerateFunctions" not found') + * err = ( __cuModuleEnumerateFunctions)(functions, numFunctions, mod) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3331 + * global __cuModuleEnumerateFunctions + * cuPythonInit() + * if __cuModuleEnumerateFunctions == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleEnumerateFunctions" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3334 + * with gil: + * raise RuntimeError('Function "cuModuleEnumerateFunctions" not found') + * err = ( __cuModuleEnumerateFunctions)(functions, numFunctions, mod) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction *, unsigned int, CUmodule))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleEnumerateFunctions)(__pyx_v_functions, __pyx_v_numFunctions, __pyx_v_mod); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3334, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3335 + * raise RuntimeError('Function "cuModuleEnumerateFunctions" not found') + * err = ( __cuModuleEnumerateFunctions)(functions, numFunctions, mod) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuModuleGetGlobal_v2(CUdeviceptr* dptr, size_t* numbytes, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3328 + * return err + * + * cdef CUresult _cuModuleEnumerateFunctions(CUfunction* functions, unsigned int numFunctions, CUmodule mod) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleEnumerateFunctions + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuModuleEnumerateFunctions", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3337 + * return err + * + * cdef CUresult _cuModuleGetGlobal_v2(CUdeviceptr* dptr, size_t* numbytes, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleGetGlobal_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleGetGlobal_v2(CUdeviceptr *__pyx_v_dptr, size_t *__pyx_v_numbytes, CUmodule __pyx_v_hmod, char const *__pyx_v_name) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuModuleGetGlobal_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3339 + * cdef CUresult _cuModuleGetGlobal_v2(CUdeviceptr* dptr, size_t* numbytes, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuModuleGetGlobal_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuModuleGetGlobal_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3339, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3340 + * global __cuModuleGetGlobal_v2 + * cuPythonInit() + * if __cuModuleGetGlobal_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleGetGlobal_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetGlobal_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3341 + * cuPythonInit() + * if __cuModuleGetGlobal_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleGetGlobal_v2" not found') + * err = ( __cuModuleGetGlobal_v2)(dptr, numbytes, hmod, name) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3342 + * if __cuModuleGetGlobal_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuModuleGetGlobal_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuModuleGetGlobal_v2)(dptr, numbytes, hmod, name) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuModuleGetGlobal_v2_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3342, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3342, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3341 + * cuPythonInit() + * if __cuModuleGetGlobal_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleGetGlobal_v2" not found') + * err = ( __cuModuleGetGlobal_v2)(dptr, numbytes, hmod, name) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3340 + * global __cuModuleGetGlobal_v2 + * cuPythonInit() + * if __cuModuleGetGlobal_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleGetGlobal_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3343 + * with gil: + * raise RuntimeError('Function "cuModuleGetGlobal_v2" not found') + * err = ( __cuModuleGetGlobal_v2)(dptr, numbytes, hmod, name) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, size_t *, CUmodule, char const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetGlobal_v2)(__pyx_v_dptr, __pyx_v_numbytes, __pyx_v_hmod, __pyx_v_name); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3343, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3344 + * raise RuntimeError('Function "cuModuleGetGlobal_v2" not found') + * err = ( __cuModuleGetGlobal_v2)(dptr, numbytes, hmod, name) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLinkCreate_v2(unsigned int numOptions, CUjit_option* options, void** optionValues, CUlinkState* stateOut) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3337 + * return err + * + * cdef CUresult _cuModuleGetGlobal_v2(CUdeviceptr* dptr, size_t* numbytes, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleGetGlobal_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuModuleGetGlobal_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3346 + * return err + * + * cdef CUresult _cuLinkCreate_v2(unsigned int numOptions, CUjit_option* options, void** optionValues, CUlinkState* stateOut) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLinkCreate_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLinkCreate_v2(unsigned int __pyx_v_numOptions, CUjit_option *__pyx_v_options, void **__pyx_v_optionValues, CUlinkState *__pyx_v_stateOut) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLinkCreate_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3348 + * cdef CUresult _cuLinkCreate_v2(unsigned int numOptions, CUjit_option* options, void** optionValues, CUlinkState* stateOut) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLinkCreate_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLinkCreate_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3348, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3349 + * global __cuLinkCreate_v2 + * cuPythonInit() + * if __cuLinkCreate_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLinkCreate_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkCreate_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3350 + * cuPythonInit() + * if __cuLinkCreate_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLinkCreate_v2" not found') + * err = ( __cuLinkCreate_v2)(numOptions, options, optionValues, stateOut) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3351 + * if __cuLinkCreate_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuLinkCreate_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuLinkCreate_v2)(numOptions, options, optionValues, stateOut) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLinkCreate_v2_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3351, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3351, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3350 + * cuPythonInit() + * if __cuLinkCreate_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLinkCreate_v2" not found') + * err = ( __cuLinkCreate_v2)(numOptions, options, optionValues, stateOut) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3349 + * global __cuLinkCreate_v2 + * cuPythonInit() + * if __cuLinkCreate_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLinkCreate_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3352 + * with gil: + * raise RuntimeError('Function "cuLinkCreate_v2" not found') + * err = ( __cuLinkCreate_v2)(numOptions, options, optionValues, stateOut) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(unsigned int, CUjit_option *, void **, CUlinkState *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkCreate_v2)(__pyx_v_numOptions, __pyx_v_options, __pyx_v_optionValues, __pyx_v_stateOut); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3352, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3353 + * raise RuntimeError('Function "cuLinkCreate_v2" not found') + * err = ( __cuLinkCreate_v2)(numOptions, options, optionValues, stateOut) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLinkAddData_v2(CUlinkState state, CUjitInputType typename, void* data, size_t size, const char* name, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3346 + * return err + * + * cdef CUresult _cuLinkCreate_v2(unsigned int numOptions, CUjit_option* options, void** optionValues, CUlinkState* stateOut) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLinkCreate_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLinkCreate_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3355 + * return err + * + * cdef CUresult _cuLinkAddData_v2(CUlinkState state, CUjitInputType typename, void* data, size_t size, const char* name, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLinkAddData_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLinkAddData_v2(CUlinkState __pyx_v_state, CUjitInputType __pyx_v_typename, void *__pyx_v_data, size_t __pyx_v_size, char const *__pyx_v_name, unsigned int __pyx_v_numOptions, CUjit_option *__pyx_v_options, void **__pyx_v_optionValues) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLinkAddData_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3357 + * cdef CUresult _cuLinkAddData_v2(CUlinkState state, CUjitInputType typename, void* data, size_t size, const char* name, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLinkAddData_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLinkAddData_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3357, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3358 + * global __cuLinkAddData_v2 + * cuPythonInit() + * if __cuLinkAddData_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLinkAddData_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddData_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3359 + * cuPythonInit() + * if __cuLinkAddData_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLinkAddData_v2" not found') + * err = ( __cuLinkAddData_v2)(state, typename, data, size, name, numOptions, options, optionValues) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3360 + * if __cuLinkAddData_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuLinkAddData_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuLinkAddData_v2)(state, typename, data, size, name, numOptions, options, optionValues) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLinkAddData_v2_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3360, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3360, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3359 + * cuPythonInit() + * if __cuLinkAddData_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLinkAddData_v2" not found') + * err = ( __cuLinkAddData_v2)(state, typename, data, size, name, numOptions, options, optionValues) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3358 + * global __cuLinkAddData_v2 + * cuPythonInit() + * if __cuLinkAddData_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLinkAddData_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3361 + * with gil: + * raise RuntimeError('Function "cuLinkAddData_v2" not found') + * err = ( __cuLinkAddData_v2)(state, typename, data, size, name, numOptions, options, optionValues) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlinkState, CUjitInputType, void *, size_t, char const *, unsigned int, CUjit_option *, void **))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddData_v2)(__pyx_v_state, __pyx_v_typename, __pyx_v_data, __pyx_v_size, __pyx_v_name, __pyx_v_numOptions, __pyx_v_options, __pyx_v_optionValues); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3361, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3362 + * raise RuntimeError('Function "cuLinkAddData_v2" not found') + * err = ( __cuLinkAddData_v2)(state, typename, data, size, name, numOptions, options, optionValues) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLinkAddFile_v2(CUlinkState state, CUjitInputType typename, const char* path, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3355 + * return err + * + * cdef CUresult _cuLinkAddData_v2(CUlinkState state, CUjitInputType typename, void* data, size_t size, const char* name, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLinkAddData_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLinkAddData_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3364 + * return err + * + * cdef CUresult _cuLinkAddFile_v2(CUlinkState state, CUjitInputType typename, const char* path, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLinkAddFile_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLinkAddFile_v2(CUlinkState __pyx_v_state, CUjitInputType __pyx_v_typename, char const *__pyx_v_path, unsigned int __pyx_v_numOptions, CUjit_option *__pyx_v_options, void **__pyx_v_optionValues) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLinkAddFile_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3366 + * cdef CUresult _cuLinkAddFile_v2(CUlinkState state, CUjitInputType typename, const char* path, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLinkAddFile_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLinkAddFile_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3366, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3367 + * global __cuLinkAddFile_v2 + * cuPythonInit() + * if __cuLinkAddFile_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLinkAddFile_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddFile_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3368 + * cuPythonInit() + * if __cuLinkAddFile_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLinkAddFile_v2" not found') + * err = ( __cuLinkAddFile_v2)(state, typename, path, numOptions, options, optionValues) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3369 + * if __cuLinkAddFile_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuLinkAddFile_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuLinkAddFile_v2)(state, typename, path, numOptions, options, optionValues) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLinkAddFile_v2_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3369, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3369, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3368 + * cuPythonInit() + * if __cuLinkAddFile_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLinkAddFile_v2" not found') + * err = ( __cuLinkAddFile_v2)(state, typename, path, numOptions, options, optionValues) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3367 + * global __cuLinkAddFile_v2 + * cuPythonInit() + * if __cuLinkAddFile_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLinkAddFile_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3370 + * with gil: + * raise RuntimeError('Function "cuLinkAddFile_v2" not found') + * err = ( __cuLinkAddFile_v2)(state, typename, path, numOptions, options, optionValues) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlinkState, CUjitInputType, char const *, unsigned int, CUjit_option *, void **))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddFile_v2)(__pyx_v_state, __pyx_v_typename, __pyx_v_path, __pyx_v_numOptions, __pyx_v_options, __pyx_v_optionValues); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3370, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3371 + * raise RuntimeError('Function "cuLinkAddFile_v2" not found') + * err = ( __cuLinkAddFile_v2)(state, typename, path, numOptions, options, optionValues) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLinkComplete(CUlinkState state, void** cubinOut, size_t* sizeOut) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3364 + * return err + * + * cdef CUresult _cuLinkAddFile_v2(CUlinkState state, CUjitInputType typename, const char* path, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLinkAddFile_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLinkAddFile_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3373 + * return err + * + * cdef CUresult _cuLinkComplete(CUlinkState state, void** cubinOut, size_t* sizeOut) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLinkComplete + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLinkComplete(CUlinkState __pyx_v_state, void **__pyx_v_cubinOut, size_t *__pyx_v_sizeOut) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLinkComplete", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3375 + * cdef CUresult _cuLinkComplete(CUlinkState state, void** cubinOut, size_t* sizeOut) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLinkComplete + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLinkComplete == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3375, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3376 + * global __cuLinkComplete + * cuPythonInit() + * if __cuLinkComplete == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLinkComplete" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkComplete == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3377 + * cuPythonInit() + * if __cuLinkComplete == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLinkComplete" not found') + * err = ( __cuLinkComplete)(state, cubinOut, sizeOut) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3378 + * if __cuLinkComplete == NULL: + * with gil: + * raise RuntimeError('Function "cuLinkComplete" not found') # <<<<<<<<<<<<<< + * err = ( __cuLinkComplete)(state, cubinOut, sizeOut) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLinkComplete_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3378, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3378, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3377 + * cuPythonInit() + * if __cuLinkComplete == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLinkComplete" not found') + * err = ( __cuLinkComplete)(state, cubinOut, sizeOut) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3376 + * global __cuLinkComplete + * cuPythonInit() + * if __cuLinkComplete == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLinkComplete" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3379 + * with gil: + * raise RuntimeError('Function "cuLinkComplete" not found') + * err = ( __cuLinkComplete)(state, cubinOut, sizeOut) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlinkState, void **, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkComplete)(__pyx_v_state, __pyx_v_cubinOut, __pyx_v_sizeOut); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3379, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3380 + * raise RuntimeError('Function "cuLinkComplete" not found') + * err = ( __cuLinkComplete)(state, cubinOut, sizeOut) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLinkDestroy(CUlinkState state) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3373 + * return err + * + * cdef CUresult _cuLinkComplete(CUlinkState state, void** cubinOut, size_t* sizeOut) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLinkComplete + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLinkComplete", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3382 + * return err + * + * cdef CUresult _cuLinkDestroy(CUlinkState state) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLinkDestroy + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLinkDestroy(CUlinkState __pyx_v_state) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLinkDestroy", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3384 + * cdef CUresult _cuLinkDestroy(CUlinkState state) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLinkDestroy + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLinkDestroy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3384, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3385 + * global __cuLinkDestroy + * cuPythonInit() + * if __cuLinkDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLinkDestroy" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkDestroy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3386 + * cuPythonInit() + * if __cuLinkDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLinkDestroy" not found') + * err = ( __cuLinkDestroy)(state) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3387 + * if __cuLinkDestroy == NULL: + * with gil: + * raise RuntimeError('Function "cuLinkDestroy" not found') # <<<<<<<<<<<<<< + * err = ( __cuLinkDestroy)(state) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLinkDestroy_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3387, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3387, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3386 + * cuPythonInit() + * if __cuLinkDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLinkDestroy" not found') + * err = ( __cuLinkDestroy)(state) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3385 + * global __cuLinkDestroy + * cuPythonInit() + * if __cuLinkDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLinkDestroy" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3388 + * with gil: + * raise RuntimeError('Function "cuLinkDestroy" not found') + * err = ( __cuLinkDestroy)(state) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlinkState))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkDestroy)(__pyx_v_state); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3388, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3389 + * raise RuntimeError('Function "cuLinkDestroy" not found') + * err = ( __cuLinkDestroy)(state) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuModuleGetTexRef(CUtexref* pTexRef, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3382 + * return err + * + * cdef CUresult _cuLinkDestroy(CUlinkState state) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLinkDestroy + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLinkDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3391 + * return err + * + * cdef CUresult _cuModuleGetTexRef(CUtexref* pTexRef, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleGetTexRef + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleGetTexRef(CUtexref *__pyx_v_pTexRef, CUmodule __pyx_v_hmod, char const *__pyx_v_name) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuModuleGetTexRef", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3393 + * cdef CUresult _cuModuleGetTexRef(CUtexref* pTexRef, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuModuleGetTexRef + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuModuleGetTexRef == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3393, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3394 + * global __cuModuleGetTexRef + * cuPythonInit() + * if __cuModuleGetTexRef == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleGetTexRef" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetTexRef == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3395 + * cuPythonInit() + * if __cuModuleGetTexRef == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleGetTexRef" not found') + * err = ( __cuModuleGetTexRef)(pTexRef, hmod, name) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3396 + * if __cuModuleGetTexRef == NULL: + * with gil: + * raise RuntimeError('Function "cuModuleGetTexRef" not found') # <<<<<<<<<<<<<< + * err = ( __cuModuleGetTexRef)(pTexRef, hmod, name) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuModuleGetTexRef_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3396, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3396, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3395 + * cuPythonInit() + * if __cuModuleGetTexRef == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleGetTexRef" not found') + * err = ( __cuModuleGetTexRef)(pTexRef, hmod, name) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3394 + * global __cuModuleGetTexRef + * cuPythonInit() + * if __cuModuleGetTexRef == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleGetTexRef" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3397 + * with gil: + * raise RuntimeError('Function "cuModuleGetTexRef" not found') + * err = ( __cuModuleGetTexRef)(pTexRef, hmod, name) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref *, CUmodule, char const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetTexRef)(__pyx_v_pTexRef, __pyx_v_hmod, __pyx_v_name); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3397, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3398 + * raise RuntimeError('Function "cuModuleGetTexRef" not found') + * err = ( __cuModuleGetTexRef)(pTexRef, hmod, name) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuModuleGetSurfRef(CUsurfref* pSurfRef, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3391 + * return err + * + * cdef CUresult _cuModuleGetTexRef(CUtexref* pTexRef, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleGetTexRef + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuModuleGetTexRef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3400 + * return err + * + * cdef CUresult _cuModuleGetSurfRef(CUsurfref* pSurfRef, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleGetSurfRef + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleGetSurfRef(CUsurfref *__pyx_v_pSurfRef, CUmodule __pyx_v_hmod, char const *__pyx_v_name) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuModuleGetSurfRef", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3402 + * cdef CUresult _cuModuleGetSurfRef(CUsurfref* pSurfRef, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuModuleGetSurfRef + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuModuleGetSurfRef == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3402, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3403 + * global __cuModuleGetSurfRef + * cuPythonInit() + * if __cuModuleGetSurfRef == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleGetSurfRef" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetSurfRef == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3404 + * cuPythonInit() + * if __cuModuleGetSurfRef == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleGetSurfRef" not found') + * err = ( __cuModuleGetSurfRef)(pSurfRef, hmod, name) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3405 + * if __cuModuleGetSurfRef == NULL: + * with gil: + * raise RuntimeError('Function "cuModuleGetSurfRef" not found') # <<<<<<<<<<<<<< + * err = ( __cuModuleGetSurfRef)(pSurfRef, hmod, name) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuModuleGetSurfRef_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3405, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3405, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3404 + * cuPythonInit() + * if __cuModuleGetSurfRef == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuModuleGetSurfRef" not found') + * err = ( __cuModuleGetSurfRef)(pSurfRef, hmod, name) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3403 + * global __cuModuleGetSurfRef + * cuPythonInit() + * if __cuModuleGetSurfRef == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuModuleGetSurfRef" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3406 + * with gil: + * raise RuntimeError('Function "cuModuleGetSurfRef" not found') + * err = ( __cuModuleGetSurfRef)(pSurfRef, hmod, name) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUsurfref *, CUmodule, char const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetSurfRef)(__pyx_v_pSurfRef, __pyx_v_hmod, __pyx_v_name); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3406, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3407 + * raise RuntimeError('Function "cuModuleGetSurfRef" not found') + * err = ( __cuModuleGetSurfRef)(pSurfRef, hmod, name) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLibraryLoadData(CUlibrary* library, const void* code, CUjit_option* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, CUlibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3400 + * return err + * + * cdef CUresult _cuModuleGetSurfRef(CUsurfref* pSurfRef, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuModuleGetSurfRef + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuModuleGetSurfRef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3409 + * return err + * + * cdef CUresult _cuLibraryLoadData(CUlibrary* library, const void* code, CUjit_option* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, CUlibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryLoadData + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryLoadData(CUlibrary *__pyx_v_library, void const *__pyx_v_code, CUjit_option *__pyx_v_jitOptions, void **__pyx_v_jitOptionsValues, unsigned int __pyx_v_numJitOptions, CUlibraryOption *__pyx_v_libraryOptions, void **__pyx_v_libraryOptionValues, unsigned int __pyx_v_numLibraryOptions) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLibraryLoadData", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3411 + * cdef CUresult _cuLibraryLoadData(CUlibrary* library, const void* code, CUjit_option* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, CUlibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLibraryLoadData + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLibraryLoadData == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3411, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3412 + * global __cuLibraryLoadData + * cuPythonInit() + * if __cuLibraryLoadData == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryLoadData" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadData == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3413 + * cuPythonInit() + * if __cuLibraryLoadData == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryLoadData" not found') + * err = ( __cuLibraryLoadData)(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3414 + * if __cuLibraryLoadData == NULL: + * with gil: + * raise RuntimeError('Function "cuLibraryLoadData" not found') # <<<<<<<<<<<<<< + * err = ( __cuLibraryLoadData)(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLibraryLoadData_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3414, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3414, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3413 + * cuPythonInit() + * if __cuLibraryLoadData == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryLoadData" not found') + * err = ( __cuLibraryLoadData)(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3412 + * global __cuLibraryLoadData + * cuPythonInit() + * if __cuLibraryLoadData == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryLoadData" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3415 + * with gil: + * raise RuntimeError('Function "cuLibraryLoadData" not found') + * err = ( __cuLibraryLoadData)(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlibrary *, void const *, CUjit_option *, void **, unsigned int, CUlibraryOption *, void **, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadData)(__pyx_v_library, __pyx_v_code, __pyx_v_jitOptions, __pyx_v_jitOptionsValues, __pyx_v_numJitOptions, __pyx_v_libraryOptions, __pyx_v_libraryOptionValues, __pyx_v_numLibraryOptions); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3415, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3416 + * raise RuntimeError('Function "cuLibraryLoadData" not found') + * err = ( __cuLibraryLoadData)(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLibraryLoadFromFile(CUlibrary* library, const char* fileName, CUjit_option* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, CUlibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3409 + * return err + * + * cdef CUresult _cuLibraryLoadData(CUlibrary* library, const void* code, CUjit_option* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, CUlibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryLoadData + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLibraryLoadData", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3418 + * return err + * + * cdef CUresult _cuLibraryLoadFromFile(CUlibrary* library, const char* fileName, CUjit_option* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, CUlibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryLoadFromFile + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryLoadFromFile(CUlibrary *__pyx_v_library, char const *__pyx_v_fileName, CUjit_option *__pyx_v_jitOptions, void **__pyx_v_jitOptionsValues, unsigned int __pyx_v_numJitOptions, CUlibraryOption *__pyx_v_libraryOptions, void **__pyx_v_libraryOptionValues, unsigned int __pyx_v_numLibraryOptions) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLibraryLoadFromFile", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3420 + * cdef CUresult _cuLibraryLoadFromFile(CUlibrary* library, const char* fileName, CUjit_option* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, CUlibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLibraryLoadFromFile + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLibraryLoadFromFile == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3420, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3421 + * global __cuLibraryLoadFromFile + * cuPythonInit() + * if __cuLibraryLoadFromFile == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryLoadFromFile" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadFromFile == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3422 + * cuPythonInit() + * if __cuLibraryLoadFromFile == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryLoadFromFile" not found') + * err = ( __cuLibraryLoadFromFile)(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3423 + * if __cuLibraryLoadFromFile == NULL: + * with gil: + * raise RuntimeError('Function "cuLibraryLoadFromFile" not found') # <<<<<<<<<<<<<< + * err = ( __cuLibraryLoadFromFile)(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLibraryLoadFromFile_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3423, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3423, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3422 + * cuPythonInit() + * if __cuLibraryLoadFromFile == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryLoadFromFile" not found') + * err = ( __cuLibraryLoadFromFile)(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3421 + * global __cuLibraryLoadFromFile + * cuPythonInit() + * if __cuLibraryLoadFromFile == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryLoadFromFile" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3424 + * with gil: + * raise RuntimeError('Function "cuLibraryLoadFromFile" not found') + * err = ( __cuLibraryLoadFromFile)(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlibrary *, char const *, CUjit_option *, void **, unsigned int, CUlibraryOption *, void **, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadFromFile)(__pyx_v_library, __pyx_v_fileName, __pyx_v_jitOptions, __pyx_v_jitOptionsValues, __pyx_v_numJitOptions, __pyx_v_libraryOptions, __pyx_v_libraryOptionValues, __pyx_v_numLibraryOptions); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3424, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3425 + * raise RuntimeError('Function "cuLibraryLoadFromFile" not found') + * err = ( __cuLibraryLoadFromFile)(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLibraryUnload(CUlibrary library) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3418 + * return err + * + * cdef CUresult _cuLibraryLoadFromFile(CUlibrary* library, const char* fileName, CUjit_option* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, CUlibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryLoadFromFile + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLibraryLoadFromFile", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3427 + * return err + * + * cdef CUresult _cuLibraryUnload(CUlibrary library) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryUnload + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryUnload(CUlibrary __pyx_v_library) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLibraryUnload", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3429 + * cdef CUresult _cuLibraryUnload(CUlibrary library) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLibraryUnload + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLibraryUnload == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3429, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3430 + * global __cuLibraryUnload + * cuPythonInit() + * if __cuLibraryUnload == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryUnload" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryUnload == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3431 + * cuPythonInit() + * if __cuLibraryUnload == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryUnload" not found') + * err = ( __cuLibraryUnload)(library) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3432 + * if __cuLibraryUnload == NULL: + * with gil: + * raise RuntimeError('Function "cuLibraryUnload" not found') # <<<<<<<<<<<<<< + * err = ( __cuLibraryUnload)(library) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLibraryUnload_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3432, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3432, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3431 + * cuPythonInit() + * if __cuLibraryUnload == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryUnload" not found') + * err = ( __cuLibraryUnload)(library) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3430 + * global __cuLibraryUnload + * cuPythonInit() + * if __cuLibraryUnload == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryUnload" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3433 + * with gil: + * raise RuntimeError('Function "cuLibraryUnload" not found') + * err = ( __cuLibraryUnload)(library) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlibrary))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryUnload)(__pyx_v_library); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3433, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3434 + * raise RuntimeError('Function "cuLibraryUnload" not found') + * err = ( __cuLibraryUnload)(library) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLibraryGetKernel(CUkernel* pKernel, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3427 + * return err + * + * cdef CUresult _cuLibraryUnload(CUlibrary library) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryUnload + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLibraryUnload", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3436 + * return err + * + * cdef CUresult _cuLibraryGetKernel(CUkernel* pKernel, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryGetKernel + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryGetKernel(CUkernel *__pyx_v_pKernel, CUlibrary __pyx_v_library, char const *__pyx_v_name) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLibraryGetKernel", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3438 + * cdef CUresult _cuLibraryGetKernel(CUkernel* pKernel, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLibraryGetKernel + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLibraryGetKernel == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3438, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3439 + * global __cuLibraryGetKernel + * cuPythonInit() + * if __cuLibraryGetKernel == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryGetKernel" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernel == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3440 + * cuPythonInit() + * if __cuLibraryGetKernel == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryGetKernel" not found') + * err = ( __cuLibraryGetKernel)(pKernel, library, name) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3441 + * if __cuLibraryGetKernel == NULL: + * with gil: + * raise RuntimeError('Function "cuLibraryGetKernel" not found') # <<<<<<<<<<<<<< + * err = ( __cuLibraryGetKernel)(pKernel, library, name) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLibraryGetKernel_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3441, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3441, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3440 + * cuPythonInit() + * if __cuLibraryGetKernel == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryGetKernel" not found') + * err = ( __cuLibraryGetKernel)(pKernel, library, name) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3439 + * global __cuLibraryGetKernel + * cuPythonInit() + * if __cuLibraryGetKernel == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryGetKernel" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3442 + * with gil: + * raise RuntimeError('Function "cuLibraryGetKernel" not found') + * err = ( __cuLibraryGetKernel)(pKernel, library, name) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUkernel *, CUlibrary, char const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernel)(__pyx_v_pKernel, __pyx_v_library, __pyx_v_name); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3442, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3443 + * raise RuntimeError('Function "cuLibraryGetKernel" not found') + * err = ( __cuLibraryGetKernel)(pKernel, library, name) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLibraryGetKernelCount(unsigned int* count, CUlibrary lib) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3436 + * return err + * + * cdef CUresult _cuLibraryGetKernel(CUkernel* pKernel, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryGetKernel + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLibraryGetKernel", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3445 + * return err + * + * cdef CUresult _cuLibraryGetKernelCount(unsigned int* count, CUlibrary lib) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryGetKernelCount + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryGetKernelCount(unsigned int *__pyx_v_count, CUlibrary __pyx_v_lib) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLibraryGetKernelCount", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3447 + * cdef CUresult _cuLibraryGetKernelCount(unsigned int* count, CUlibrary lib) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLibraryGetKernelCount + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLibraryGetKernelCount == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3447, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3448 + * global __cuLibraryGetKernelCount + * cuPythonInit() + * if __cuLibraryGetKernelCount == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryGetKernelCount" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernelCount == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3449 + * cuPythonInit() + * if __cuLibraryGetKernelCount == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryGetKernelCount" not found') + * err = ( __cuLibraryGetKernelCount)(count, lib) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3450 + * if __cuLibraryGetKernelCount == NULL: + * with gil: + * raise RuntimeError('Function "cuLibraryGetKernelCount" not found') # <<<<<<<<<<<<<< + * err = ( __cuLibraryGetKernelCount)(count, lib) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLibraryGetKernelCount}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3450, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3450, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3449 + * cuPythonInit() + * if __cuLibraryGetKernelCount == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryGetKernelCount" not found') + * err = ( __cuLibraryGetKernelCount)(count, lib) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3448 + * global __cuLibraryGetKernelCount + * cuPythonInit() + * if __cuLibraryGetKernelCount == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryGetKernelCount" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3451 + * with gil: + * raise RuntimeError('Function "cuLibraryGetKernelCount" not found') + * err = ( __cuLibraryGetKernelCount)(count, lib) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(unsigned int *, CUlibrary))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernelCount)(__pyx_v_count, __pyx_v_lib); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3451, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3452 + * raise RuntimeError('Function "cuLibraryGetKernelCount" not found') + * err = ( __cuLibraryGetKernelCount)(count, lib) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLibraryEnumerateKernels(CUkernel* kernels, unsigned int numKernels, CUlibrary lib) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3445 + * return err + * + * cdef CUresult _cuLibraryGetKernelCount(unsigned int* count, CUlibrary lib) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryGetKernelCount + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLibraryGetKernelCount", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3454 + * return err + * + * cdef CUresult _cuLibraryEnumerateKernels(CUkernel* kernels, unsigned int numKernels, CUlibrary lib) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryEnumerateKernels + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryEnumerateKernels(CUkernel *__pyx_v_kernels, unsigned int __pyx_v_numKernels, CUlibrary __pyx_v_lib) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLibraryEnumerateKernels", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3456 + * cdef CUresult _cuLibraryEnumerateKernels(CUkernel* kernels, unsigned int numKernels, CUlibrary lib) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLibraryEnumerateKernels + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLibraryEnumerateKernels == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3456, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3457 + * global __cuLibraryEnumerateKernels + * cuPythonInit() + * if __cuLibraryEnumerateKernels == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryEnumerateKernels" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryEnumerateKernels == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3458 + * cuPythonInit() + * if __cuLibraryEnumerateKernels == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryEnumerateKernels" not found') + * err = ( __cuLibraryEnumerateKernels)(kernels, numKernels, lib) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3459 + * if __cuLibraryEnumerateKernels == NULL: + * with gil: + * raise RuntimeError('Function "cuLibraryEnumerateKernels" not found') # <<<<<<<<<<<<<< + * err = ( __cuLibraryEnumerateKernels)(kernels, numKernels, lib) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLibraryEnumerateKerne}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3459, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3459, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3458 + * cuPythonInit() + * if __cuLibraryEnumerateKernels == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryEnumerateKernels" not found') + * err = ( __cuLibraryEnumerateKernels)(kernels, numKernels, lib) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3457 + * global __cuLibraryEnumerateKernels + * cuPythonInit() + * if __cuLibraryEnumerateKernels == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryEnumerateKernels" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3460 + * with gil: + * raise RuntimeError('Function "cuLibraryEnumerateKernels" not found') + * err = ( __cuLibraryEnumerateKernels)(kernels, numKernels, lib) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUkernel *, unsigned int, CUlibrary))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryEnumerateKernels)(__pyx_v_kernels, __pyx_v_numKernels, __pyx_v_lib); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3460, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3461 + * raise RuntimeError('Function "cuLibraryEnumerateKernels" not found') + * err = ( __cuLibraryEnumerateKernels)(kernels, numKernels, lib) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLibraryGetModule(CUmodule* pMod, CUlibrary library) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3454 + * return err + * + * cdef CUresult _cuLibraryEnumerateKernels(CUkernel* kernels, unsigned int numKernels, CUlibrary lib) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryEnumerateKernels + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLibraryEnumerateKernels", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3463 + * return err + * + * cdef CUresult _cuLibraryGetModule(CUmodule* pMod, CUlibrary library) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryGetModule + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryGetModule(CUmodule *__pyx_v_pMod, CUlibrary __pyx_v_library) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLibraryGetModule", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3465 + * cdef CUresult _cuLibraryGetModule(CUmodule* pMod, CUlibrary library) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLibraryGetModule + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLibraryGetModule == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3465, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3466 + * global __cuLibraryGetModule + * cuPythonInit() + * if __cuLibraryGetModule == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryGetModule" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetModule == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3467 + * cuPythonInit() + * if __cuLibraryGetModule == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryGetModule" not found') + * err = ( __cuLibraryGetModule)(pMod, library) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3468 + * if __cuLibraryGetModule == NULL: + * with gil: + * raise RuntimeError('Function "cuLibraryGetModule" not found') # <<<<<<<<<<<<<< + * err = ( __cuLibraryGetModule)(pMod, library) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLibraryGetModule_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3468, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3468, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3467 + * cuPythonInit() + * if __cuLibraryGetModule == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryGetModule" not found') + * err = ( __cuLibraryGetModule)(pMod, library) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3466 + * global __cuLibraryGetModule + * cuPythonInit() + * if __cuLibraryGetModule == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryGetModule" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3469 + * with gil: + * raise RuntimeError('Function "cuLibraryGetModule" not found') + * err = ( __cuLibraryGetModule)(pMod, library) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmodule *, CUlibrary))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetModule)(__pyx_v_pMod, __pyx_v_library); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3469, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3470 + * raise RuntimeError('Function "cuLibraryGetModule" not found') + * err = ( __cuLibraryGetModule)(pMod, library) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuKernelGetFunction(CUfunction* pFunc, CUkernel kernel) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3463 + * return err + * + * cdef CUresult _cuLibraryGetModule(CUmodule* pMod, CUlibrary library) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryGetModule + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLibraryGetModule", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3472 + * return err + * + * cdef CUresult _cuKernelGetFunction(CUfunction* pFunc, CUkernel kernel) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelGetFunction + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelGetFunction(CUfunction *__pyx_v_pFunc, CUkernel __pyx_v_kernel) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuKernelGetFunction", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3474 + * cdef CUresult _cuKernelGetFunction(CUfunction* pFunc, CUkernel kernel) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuKernelGetFunction + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuKernelGetFunction == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3474, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3475 + * global __cuKernelGetFunction + * cuPythonInit() + * if __cuKernelGetFunction == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelGetFunction" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetFunction == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3476 + * cuPythonInit() + * if __cuKernelGetFunction == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelGetFunction" not found') + * err = ( __cuKernelGetFunction)(pFunc, kernel) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3477 + * if __cuKernelGetFunction == NULL: + * with gil: + * raise RuntimeError('Function "cuKernelGetFunction" not found') # <<<<<<<<<<<<<< + * err = ( __cuKernelGetFunction)(pFunc, kernel) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuKernelGetFunction_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3477, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3477, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3476 + * cuPythonInit() + * if __cuKernelGetFunction == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelGetFunction" not found') + * err = ( __cuKernelGetFunction)(pFunc, kernel) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3475 + * global __cuKernelGetFunction + * cuPythonInit() + * if __cuKernelGetFunction == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelGetFunction" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3478 + * with gil: + * raise RuntimeError('Function "cuKernelGetFunction" not found') + * err = ( __cuKernelGetFunction)(pFunc, kernel) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction *, CUkernel))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetFunction)(__pyx_v_pFunc, __pyx_v_kernel); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3478, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3479 + * raise RuntimeError('Function "cuKernelGetFunction" not found') + * err = ( __cuKernelGetFunction)(pFunc, kernel) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuKernelGetLibrary(CUlibrary* pLib, CUkernel kernel) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3472 + * return err + * + * cdef CUresult _cuKernelGetFunction(CUfunction* pFunc, CUkernel kernel) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelGetFunction + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuKernelGetFunction", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3481 + * return err + * + * cdef CUresult _cuKernelGetLibrary(CUlibrary* pLib, CUkernel kernel) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelGetLibrary + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelGetLibrary(CUlibrary *__pyx_v_pLib, CUkernel __pyx_v_kernel) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuKernelGetLibrary", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3483 + * cdef CUresult _cuKernelGetLibrary(CUlibrary* pLib, CUkernel kernel) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuKernelGetLibrary + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuKernelGetLibrary == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3483, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3484 + * global __cuKernelGetLibrary + * cuPythonInit() + * if __cuKernelGetLibrary == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelGetLibrary" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetLibrary == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3485 + * cuPythonInit() + * if __cuKernelGetLibrary == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelGetLibrary" not found') + * err = ( __cuKernelGetLibrary)(pLib, kernel) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3486 + * if __cuKernelGetLibrary == NULL: + * with gil: + * raise RuntimeError('Function "cuKernelGetLibrary" not found') # <<<<<<<<<<<<<< + * err = ( __cuKernelGetLibrary)(pLib, kernel) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuKernelGetLibrary_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3486, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3486, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3485 + * cuPythonInit() + * if __cuKernelGetLibrary == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelGetLibrary" not found') + * err = ( __cuKernelGetLibrary)(pLib, kernel) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3484 + * global __cuKernelGetLibrary + * cuPythonInit() + * if __cuKernelGetLibrary == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelGetLibrary" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3487 + * with gil: + * raise RuntimeError('Function "cuKernelGetLibrary" not found') + * err = ( __cuKernelGetLibrary)(pLib, kernel) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlibrary *, CUkernel))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetLibrary)(__pyx_v_pLib, __pyx_v_kernel); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3487, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3488 + * raise RuntimeError('Function "cuKernelGetLibrary" not found') + * err = ( __cuKernelGetLibrary)(pLib, kernel) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLibraryGetGlobal(CUdeviceptr* dptr, size_t* numbytes, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3481 + * return err + * + * cdef CUresult _cuKernelGetLibrary(CUlibrary* pLib, CUkernel kernel) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelGetLibrary + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuKernelGetLibrary", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3490 + * return err + * + * cdef CUresult _cuLibraryGetGlobal(CUdeviceptr* dptr, size_t* numbytes, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryGetGlobal + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryGetGlobal(CUdeviceptr *__pyx_v_dptr, size_t *__pyx_v_numbytes, CUlibrary __pyx_v_library, char const *__pyx_v_name) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLibraryGetGlobal", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3492 + * cdef CUresult _cuLibraryGetGlobal(CUdeviceptr* dptr, size_t* numbytes, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLibraryGetGlobal + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLibraryGetGlobal == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3492, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3493 + * global __cuLibraryGetGlobal + * cuPythonInit() + * if __cuLibraryGetGlobal == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryGetGlobal" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetGlobal == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3494 + * cuPythonInit() + * if __cuLibraryGetGlobal == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryGetGlobal" not found') + * err = ( __cuLibraryGetGlobal)(dptr, numbytes, library, name) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3495 + * if __cuLibraryGetGlobal == NULL: + * with gil: + * raise RuntimeError('Function "cuLibraryGetGlobal" not found') # <<<<<<<<<<<<<< + * err = ( __cuLibraryGetGlobal)(dptr, numbytes, library, name) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLibraryGetGlobal_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3495, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3495, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3494 + * cuPythonInit() + * if __cuLibraryGetGlobal == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryGetGlobal" not found') + * err = ( __cuLibraryGetGlobal)(dptr, numbytes, library, name) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3493 + * global __cuLibraryGetGlobal + * cuPythonInit() + * if __cuLibraryGetGlobal == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryGetGlobal" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3496 + * with gil: + * raise RuntimeError('Function "cuLibraryGetGlobal" not found') + * err = ( __cuLibraryGetGlobal)(dptr, numbytes, library, name) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, size_t *, CUlibrary, char const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetGlobal)(__pyx_v_dptr, __pyx_v_numbytes, __pyx_v_library, __pyx_v_name); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3496, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3497 + * raise RuntimeError('Function "cuLibraryGetGlobal" not found') + * err = ( __cuLibraryGetGlobal)(dptr, numbytes, library, name) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLibraryGetManaged(CUdeviceptr* dptr, size_t* numbytes, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3490 + * return err + * + * cdef CUresult _cuLibraryGetGlobal(CUdeviceptr* dptr, size_t* numbytes, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryGetGlobal + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLibraryGetGlobal", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3499 + * return err + * + * cdef CUresult _cuLibraryGetManaged(CUdeviceptr* dptr, size_t* numbytes, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryGetManaged + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryGetManaged(CUdeviceptr *__pyx_v_dptr, size_t *__pyx_v_numbytes, CUlibrary __pyx_v_library, char const *__pyx_v_name) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLibraryGetManaged", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3501 + * cdef CUresult _cuLibraryGetManaged(CUdeviceptr* dptr, size_t* numbytes, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLibraryGetManaged + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLibraryGetManaged == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3501, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3502 + * global __cuLibraryGetManaged + * cuPythonInit() + * if __cuLibraryGetManaged == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryGetManaged" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetManaged == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3503 + * cuPythonInit() + * if __cuLibraryGetManaged == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryGetManaged" not found') + * err = ( __cuLibraryGetManaged)(dptr, numbytes, library, name) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3504 + * if __cuLibraryGetManaged == NULL: + * with gil: + * raise RuntimeError('Function "cuLibraryGetManaged" not found') # <<<<<<<<<<<<<< + * err = ( __cuLibraryGetManaged)(dptr, numbytes, library, name) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLibraryGetManaged_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3504, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3504, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3503 + * cuPythonInit() + * if __cuLibraryGetManaged == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryGetManaged" not found') + * err = ( __cuLibraryGetManaged)(dptr, numbytes, library, name) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3502 + * global __cuLibraryGetManaged + * cuPythonInit() + * if __cuLibraryGetManaged == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryGetManaged" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3505 + * with gil: + * raise RuntimeError('Function "cuLibraryGetManaged" not found') + * err = ( __cuLibraryGetManaged)(dptr, numbytes, library, name) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, size_t *, CUlibrary, char const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetManaged)(__pyx_v_dptr, __pyx_v_numbytes, __pyx_v_library, __pyx_v_name); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3505, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3506 + * raise RuntimeError('Function "cuLibraryGetManaged" not found') + * err = ( __cuLibraryGetManaged)(dptr, numbytes, library, name) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLibraryGetUnifiedFunction(void** fptr, CUlibrary library, const char* symbol) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3499 + * return err + * + * cdef CUresult _cuLibraryGetManaged(CUdeviceptr* dptr, size_t* numbytes, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryGetManaged + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLibraryGetManaged", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3508 + * return err + * + * cdef CUresult _cuLibraryGetUnifiedFunction(void** fptr, CUlibrary library, const char* symbol) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryGetUnifiedFunction + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryGetUnifiedFunction(void **__pyx_v_fptr, CUlibrary __pyx_v_library, char const *__pyx_v_symbol) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLibraryGetUnifiedFunction", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3510 + * cdef CUresult _cuLibraryGetUnifiedFunction(void** fptr, CUlibrary library, const char* symbol) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLibraryGetUnifiedFunction + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLibraryGetUnifiedFunction == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3510, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3511 + * global __cuLibraryGetUnifiedFunction + * cuPythonInit() + * if __cuLibraryGetUnifiedFunction == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryGetUnifiedFunction" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetUnifiedFunction == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3512 + * cuPythonInit() + * if __cuLibraryGetUnifiedFunction == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryGetUnifiedFunction" not found') + * err = ( __cuLibraryGetUnifiedFunction)(fptr, library, symbol) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3513 + * if __cuLibraryGetUnifiedFunction == NULL: + * with gil: + * raise RuntimeError('Function "cuLibraryGetUnifiedFunction" not found') # <<<<<<<<<<<<<< + * err = ( __cuLibraryGetUnifiedFunction)(fptr, library, symbol) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLibraryGetUnifiedFunc}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3513, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3513, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3512 + * cuPythonInit() + * if __cuLibraryGetUnifiedFunction == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLibraryGetUnifiedFunction" not found') + * err = ( __cuLibraryGetUnifiedFunction)(fptr, library, symbol) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3511 + * global __cuLibraryGetUnifiedFunction + * cuPythonInit() + * if __cuLibraryGetUnifiedFunction == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLibraryGetUnifiedFunction" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3514 + * with gil: + * raise RuntimeError('Function "cuLibraryGetUnifiedFunction" not found') + * err = ( __cuLibraryGetUnifiedFunction)(fptr, library, symbol) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void **, CUlibrary, char const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetUnifiedFunction)(__pyx_v_fptr, __pyx_v_library, __pyx_v_symbol); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3514, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3515 + * raise RuntimeError('Function "cuLibraryGetUnifiedFunction" not found') + * err = ( __cuLibraryGetUnifiedFunction)(fptr, library, symbol) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuKernelGetAttribute(int* pi, CUfunction_attribute attrib, CUkernel kernel, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3508 + * return err + * + * cdef CUresult _cuLibraryGetUnifiedFunction(void** fptr, CUlibrary library, const char* symbol) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLibraryGetUnifiedFunction + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLibraryGetUnifiedFunction", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3517 + * return err + * + * cdef CUresult _cuKernelGetAttribute(int* pi, CUfunction_attribute attrib, CUkernel kernel, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelGetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelGetAttribute(int *__pyx_v_pi, CUfunction_attribute __pyx_v_attrib, CUkernel __pyx_v_kernel, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuKernelGetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3519 + * cdef CUresult _cuKernelGetAttribute(int* pi, CUfunction_attribute attrib, CUkernel kernel, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuKernelGetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuKernelGetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3519, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3520 + * global __cuKernelGetAttribute + * cuPythonInit() + * if __cuKernelGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelGetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3521 + * cuPythonInit() + * if __cuKernelGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelGetAttribute" not found') + * err = ( __cuKernelGetAttribute)(pi, attrib, kernel, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3522 + * if __cuKernelGetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuKernelGetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuKernelGetAttribute)(pi, attrib, kernel, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuKernelGetAttribute_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3522, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3522, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3521 + * cuPythonInit() + * if __cuKernelGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelGetAttribute" not found') + * err = ( __cuKernelGetAttribute)(pi, attrib, kernel, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3520 + * global __cuKernelGetAttribute + * cuPythonInit() + * if __cuKernelGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelGetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3523 + * with gil: + * raise RuntimeError('Function "cuKernelGetAttribute" not found') + * err = ( __cuKernelGetAttribute)(pi, attrib, kernel, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, CUfunction_attribute, CUkernel, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetAttribute)(__pyx_v_pi, __pyx_v_attrib, __pyx_v_kernel, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3523, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3524 + * raise RuntimeError('Function "cuKernelGetAttribute" not found') + * err = ( __cuKernelGetAttribute)(pi, attrib, kernel, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuKernelSetAttribute(CUfunction_attribute attrib, int val, CUkernel kernel, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3517 + * return err + * + * cdef CUresult _cuKernelGetAttribute(int* pi, CUfunction_attribute attrib, CUkernel kernel, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelGetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuKernelGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3526 + * return err + * + * cdef CUresult _cuKernelSetAttribute(CUfunction_attribute attrib, int val, CUkernel kernel, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelSetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelSetAttribute(CUfunction_attribute __pyx_v_attrib, int __pyx_v_val, CUkernel __pyx_v_kernel, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuKernelSetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3528 + * cdef CUresult _cuKernelSetAttribute(CUfunction_attribute attrib, int val, CUkernel kernel, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuKernelSetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuKernelSetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3528, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3529 + * global __cuKernelSetAttribute + * cuPythonInit() + * if __cuKernelSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelSetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3530 + * cuPythonInit() + * if __cuKernelSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelSetAttribute" not found') + * err = ( __cuKernelSetAttribute)(attrib, val, kernel, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3531 + * if __cuKernelSetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuKernelSetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuKernelSetAttribute)(attrib, val, kernel, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuKernelSetAttribute_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3531, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3531, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3530 + * cuPythonInit() + * if __cuKernelSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelSetAttribute" not found') + * err = ( __cuKernelSetAttribute)(attrib, val, kernel, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3529 + * global __cuKernelSetAttribute + * cuPythonInit() + * if __cuKernelSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelSetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3532 + * with gil: + * raise RuntimeError('Function "cuKernelSetAttribute" not found') + * err = ( __cuKernelSetAttribute)(attrib, val, kernel, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction_attribute, int, CUkernel, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetAttribute)(__pyx_v_attrib, __pyx_v_val, __pyx_v_kernel, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3532, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3533 + * raise RuntimeError('Function "cuKernelSetAttribute" not found') + * err = ( __cuKernelSetAttribute)(attrib, val, kernel, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuKernelSetCacheConfig(CUkernel kernel, CUfunc_cache config, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3526 + * return err + * + * cdef CUresult _cuKernelSetAttribute(CUfunction_attribute attrib, int val, CUkernel kernel, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelSetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuKernelSetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3535 + * return err + * + * cdef CUresult _cuKernelSetCacheConfig(CUkernel kernel, CUfunc_cache config, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelSetCacheConfig + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelSetCacheConfig(CUkernel __pyx_v_kernel, CUfunc_cache __pyx_v_config, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuKernelSetCacheConfig", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3537 + * cdef CUresult _cuKernelSetCacheConfig(CUkernel kernel, CUfunc_cache config, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuKernelSetCacheConfig + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuKernelSetCacheConfig == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3537, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3538 + * global __cuKernelSetCacheConfig + * cuPythonInit() + * if __cuKernelSetCacheConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelSetCacheConfig" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetCacheConfig == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3539 + * cuPythonInit() + * if __cuKernelSetCacheConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelSetCacheConfig" not found') + * err = ( __cuKernelSetCacheConfig)(kernel, config, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3540 + * if __cuKernelSetCacheConfig == NULL: + * with gil: + * raise RuntimeError('Function "cuKernelSetCacheConfig" not found') # <<<<<<<<<<<<<< + * err = ( __cuKernelSetCacheConfig)(kernel, config, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuKernelSetCacheConfig}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3540, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3540, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3539 + * cuPythonInit() + * if __cuKernelSetCacheConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelSetCacheConfig" not found') + * err = ( __cuKernelSetCacheConfig)(kernel, config, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3538 + * global __cuKernelSetCacheConfig + * cuPythonInit() + * if __cuKernelSetCacheConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelSetCacheConfig" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3541 + * with gil: + * raise RuntimeError('Function "cuKernelSetCacheConfig" not found') + * err = ( __cuKernelSetCacheConfig)(kernel, config, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUkernel, CUfunc_cache, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetCacheConfig)(__pyx_v_kernel, __pyx_v_config, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3541, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3542 + * raise RuntimeError('Function "cuKernelSetCacheConfig" not found') + * err = ( __cuKernelSetCacheConfig)(kernel, config, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuKernelGetName(const char** name, CUkernel hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3535 + * return err + * + * cdef CUresult _cuKernelSetCacheConfig(CUkernel kernel, CUfunc_cache config, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelSetCacheConfig + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuKernelSetCacheConfig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3544 + * return err + * + * cdef CUresult _cuKernelGetName(const char** name, CUkernel hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelGetName + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelGetName(char const **__pyx_v_name, CUkernel __pyx_v_hfunc) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuKernelGetName", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3546 + * cdef CUresult _cuKernelGetName(const char** name, CUkernel hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuKernelGetName + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuKernelGetName == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3546, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3547 + * global __cuKernelGetName + * cuPythonInit() + * if __cuKernelGetName == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelGetName" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetName == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3548 + * cuPythonInit() + * if __cuKernelGetName == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelGetName" not found') + * err = ( __cuKernelGetName)(name, hfunc) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3549 + * if __cuKernelGetName == NULL: + * with gil: + * raise RuntimeError('Function "cuKernelGetName" not found') # <<<<<<<<<<<<<< + * err = ( __cuKernelGetName)(name, hfunc) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuKernelGetName_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3549, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3549, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3548 + * cuPythonInit() + * if __cuKernelGetName == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelGetName" not found') + * err = ( __cuKernelGetName)(name, hfunc) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3547 + * global __cuKernelGetName + * cuPythonInit() + * if __cuKernelGetName == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelGetName" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3550 + * with gil: + * raise RuntimeError('Function "cuKernelGetName" not found') + * err = ( __cuKernelGetName)(name, hfunc) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(char const **, CUkernel))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetName)(__pyx_v_name, __pyx_v_hfunc); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3550, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3551 + * raise RuntimeError('Function "cuKernelGetName" not found') + * err = ( __cuKernelGetName)(name, hfunc) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuKernelGetParamInfo(CUkernel kernel, size_t paramIndex, size_t* paramOffset, size_t* paramSize) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3544 + * return err + * + * cdef CUresult _cuKernelGetName(const char** name, CUkernel hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelGetName + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuKernelGetName", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3553 + * return err + * + * cdef CUresult _cuKernelGetParamInfo(CUkernel kernel, size_t paramIndex, size_t* paramOffset, size_t* paramSize) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelGetParamInfo + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelGetParamInfo(CUkernel __pyx_v_kernel, size_t __pyx_v_paramIndex, size_t *__pyx_v_paramOffset, size_t *__pyx_v_paramSize) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuKernelGetParamInfo", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3555 + * cdef CUresult _cuKernelGetParamInfo(CUkernel kernel, size_t paramIndex, size_t* paramOffset, size_t* paramSize) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuKernelGetParamInfo + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuKernelGetParamInfo == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3555, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3556 + * global __cuKernelGetParamInfo + * cuPythonInit() + * if __cuKernelGetParamInfo == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelGetParamInfo" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetParamInfo == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3557 + * cuPythonInit() + * if __cuKernelGetParamInfo == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelGetParamInfo" not found') + * err = ( __cuKernelGetParamInfo)(kernel, paramIndex, paramOffset, paramSize) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3558 + * if __cuKernelGetParamInfo == NULL: + * with gil: + * raise RuntimeError('Function "cuKernelGetParamInfo" not found') # <<<<<<<<<<<<<< + * err = ( __cuKernelGetParamInfo)(kernel, paramIndex, paramOffset, paramSize) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuKernelGetParamInfo_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3558, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3558, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3557 + * cuPythonInit() + * if __cuKernelGetParamInfo == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuKernelGetParamInfo" not found') + * err = ( __cuKernelGetParamInfo)(kernel, paramIndex, paramOffset, paramSize) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3556 + * global __cuKernelGetParamInfo + * cuPythonInit() + * if __cuKernelGetParamInfo == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuKernelGetParamInfo" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3559 + * with gil: + * raise RuntimeError('Function "cuKernelGetParamInfo" not found') + * err = ( __cuKernelGetParamInfo)(kernel, paramIndex, paramOffset, paramSize) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUkernel, size_t, size_t *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetParamInfo)(__pyx_v_kernel, __pyx_v_paramIndex, __pyx_v_paramOffset, __pyx_v_paramSize); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3559, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3560 + * raise RuntimeError('Function "cuKernelGetParamInfo" not found') + * err = ( __cuKernelGetParamInfo)(kernel, paramIndex, paramOffset, paramSize) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemGetInfo_v2(size_t* free, size_t* total) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3553 + * return err + * + * cdef CUresult _cuKernelGetParamInfo(CUkernel kernel, size_t paramIndex, size_t* paramOffset, size_t* paramSize) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuKernelGetParamInfo + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuKernelGetParamInfo", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3562 + * return err + * + * cdef CUresult _cuMemGetInfo_v2(size_t* free, size_t* total) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemGetInfo_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemGetInfo_v2(size_t *__pyx_v_free, size_t *__pyx_v_total) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemGetInfo_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3564 + * cdef CUresult _cuMemGetInfo_v2(size_t* free, size_t* total) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemGetInfo_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemGetInfo_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3564, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3565 + * global __cuMemGetInfo_v2 + * cuPythonInit() + * if __cuMemGetInfo_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemGetInfo_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetInfo_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3566 + * cuPythonInit() + * if __cuMemGetInfo_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemGetInfo_v2" not found') + * err = ( __cuMemGetInfo_v2)(free, total) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3567 + * if __cuMemGetInfo_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemGetInfo_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemGetInfo_v2)(free, total) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemGetInfo_v2_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3567, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3567, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3566 + * cuPythonInit() + * if __cuMemGetInfo_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemGetInfo_v2" not found') + * err = ( __cuMemGetInfo_v2)(free, total) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3565 + * global __cuMemGetInfo_v2 + * cuPythonInit() + * if __cuMemGetInfo_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemGetInfo_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3568 + * with gil: + * raise RuntimeError('Function "cuMemGetInfo_v2" not found') + * err = ( __cuMemGetInfo_v2)(free, total) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(size_t *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetInfo_v2)(__pyx_v_free, __pyx_v_total); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3568, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3569 + * raise RuntimeError('Function "cuMemGetInfo_v2" not found') + * err = ( __cuMemGetInfo_v2)(free, total) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemAlloc_v2(CUdeviceptr* dptr, size_t bytesize) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3562 + * return err + * + * cdef CUresult _cuMemGetInfo_v2(size_t* free, size_t* total) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemGetInfo_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemGetInfo_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3571 + * return err + * + * cdef CUresult _cuMemAlloc_v2(CUdeviceptr* dptr, size_t bytesize) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAlloc_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAlloc_v2(CUdeviceptr *__pyx_v_dptr, size_t __pyx_v_bytesize) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemAlloc_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3573 + * cdef CUresult _cuMemAlloc_v2(CUdeviceptr* dptr, size_t bytesize) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemAlloc_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemAlloc_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3573, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3574 + * global __cuMemAlloc_v2 + * cuPythonInit() + * if __cuMemAlloc_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAlloc_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAlloc_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3575 + * cuPythonInit() + * if __cuMemAlloc_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAlloc_v2" not found') + * err = ( __cuMemAlloc_v2)(dptr, bytesize) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3576 + * if __cuMemAlloc_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemAlloc_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemAlloc_v2)(dptr, bytesize) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemAlloc_v2_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3576, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3576, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3575 + * cuPythonInit() + * if __cuMemAlloc_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAlloc_v2" not found') + * err = ( __cuMemAlloc_v2)(dptr, bytesize) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3574 + * global __cuMemAlloc_v2 + * cuPythonInit() + * if __cuMemAlloc_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAlloc_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3577 + * with gil: + * raise RuntimeError('Function "cuMemAlloc_v2" not found') + * err = ( __cuMemAlloc_v2)(dptr, bytesize) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAlloc_v2)(__pyx_v_dptr, __pyx_v_bytesize); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3577, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3578 + * raise RuntimeError('Function "cuMemAlloc_v2" not found') + * err = ( __cuMemAlloc_v2)(dptr, bytesize) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemAllocPitch_v2(CUdeviceptr* dptr, size_t* pPitch, size_t WidthInBytes, size_t Height, unsigned int ElementSizeBytes) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3571 + * return err + * + * cdef CUresult _cuMemAlloc_v2(CUdeviceptr* dptr, size_t bytesize) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAlloc_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemAlloc_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3580 + * return err + * + * cdef CUresult _cuMemAllocPitch_v2(CUdeviceptr* dptr, size_t* pPitch, size_t WidthInBytes, size_t Height, unsigned int ElementSizeBytes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAllocPitch_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAllocPitch_v2(CUdeviceptr *__pyx_v_dptr, size_t *__pyx_v_pPitch, size_t __pyx_v_WidthInBytes, size_t __pyx_v_Height, unsigned int __pyx_v_ElementSizeBytes) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemAllocPitch_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3582 + * cdef CUresult _cuMemAllocPitch_v2(CUdeviceptr* dptr, size_t* pPitch, size_t WidthInBytes, size_t Height, unsigned int ElementSizeBytes) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemAllocPitch_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemAllocPitch_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3582, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3583 + * global __cuMemAllocPitch_v2 + * cuPythonInit() + * if __cuMemAllocPitch_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAllocPitch_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocPitch_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3584 + * cuPythonInit() + * if __cuMemAllocPitch_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAllocPitch_v2" not found') + * err = ( __cuMemAllocPitch_v2)(dptr, pPitch, WidthInBytes, Height, ElementSizeBytes) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3585 + * if __cuMemAllocPitch_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemAllocPitch_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemAllocPitch_v2)(dptr, pPitch, WidthInBytes, Height, ElementSizeBytes) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemAllocPitch_v2_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3585, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3585, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3584 + * cuPythonInit() + * if __cuMemAllocPitch_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAllocPitch_v2" not found') + * err = ( __cuMemAllocPitch_v2)(dptr, pPitch, WidthInBytes, Height, ElementSizeBytes) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3583 + * global __cuMemAllocPitch_v2 + * cuPythonInit() + * if __cuMemAllocPitch_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAllocPitch_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3586 + * with gil: + * raise RuntimeError('Function "cuMemAllocPitch_v2" not found') + * err = ( __cuMemAllocPitch_v2)(dptr, pPitch, WidthInBytes, Height, ElementSizeBytes) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, size_t *, size_t, size_t, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocPitch_v2)(__pyx_v_dptr, __pyx_v_pPitch, __pyx_v_WidthInBytes, __pyx_v_Height, __pyx_v_ElementSizeBytes); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3586, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3587 + * raise RuntimeError('Function "cuMemAllocPitch_v2" not found') + * err = ( __cuMemAllocPitch_v2)(dptr, pPitch, WidthInBytes, Height, ElementSizeBytes) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemFree_v2(CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3580 + * return err + * + * cdef CUresult _cuMemAllocPitch_v2(CUdeviceptr* dptr, size_t* pPitch, size_t WidthInBytes, size_t Height, unsigned int ElementSizeBytes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAllocPitch_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemAllocPitch_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3589 + * return err + * + * cdef CUresult _cuMemFree_v2(CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemFree_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemFree_v2(CUdeviceptr __pyx_v_dptr) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemFree_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3591 + * cdef CUresult _cuMemFree_v2(CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemFree_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemFree_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3591, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3592 + * global __cuMemFree_v2 + * cuPythonInit() + * if __cuMemFree_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemFree_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFree_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3593 + * cuPythonInit() + * if __cuMemFree_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemFree_v2" not found') + * err = ( __cuMemFree_v2)(dptr) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3594 + * if __cuMemFree_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemFree_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemFree_v2)(dptr) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemFree_v2_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3594, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3594, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3593 + * cuPythonInit() + * if __cuMemFree_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemFree_v2" not found') + * err = ( __cuMemFree_v2)(dptr) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3592 + * global __cuMemFree_v2 + * cuPythonInit() + * if __cuMemFree_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemFree_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3595 + * with gil: + * raise RuntimeError('Function "cuMemFree_v2" not found') + * err = ( __cuMemFree_v2)(dptr) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFree_v2)(__pyx_v_dptr); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3595, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3596 + * raise RuntimeError('Function "cuMemFree_v2" not found') + * err = ( __cuMemFree_v2)(dptr) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemGetAddressRange_v2(CUdeviceptr* pbase, size_t* psize, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3589 + * return err + * + * cdef CUresult _cuMemFree_v2(CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemFree_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemFree_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3598 + * return err + * + * cdef CUresult _cuMemGetAddressRange_v2(CUdeviceptr* pbase, size_t* psize, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemGetAddressRange_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemGetAddressRange_v2(CUdeviceptr *__pyx_v_pbase, size_t *__pyx_v_psize, CUdeviceptr __pyx_v_dptr) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemGetAddressRange_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3600 + * cdef CUresult _cuMemGetAddressRange_v2(CUdeviceptr* pbase, size_t* psize, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemGetAddressRange_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemGetAddressRange_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3600, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3601 + * global __cuMemGetAddressRange_v2 + * cuPythonInit() + * if __cuMemGetAddressRange_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemGetAddressRange_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAddressRange_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3602 + * cuPythonInit() + * if __cuMemGetAddressRange_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemGetAddressRange_v2" not found') + * err = ( __cuMemGetAddressRange_v2)(pbase, psize, dptr) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3603 + * if __cuMemGetAddressRange_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemGetAddressRange_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemGetAddressRange_v2)(pbase, psize, dptr) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemGetAddressRange_v2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3603, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3603, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3602 + * cuPythonInit() + * if __cuMemGetAddressRange_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemGetAddressRange_v2" not found') + * err = ( __cuMemGetAddressRange_v2)(pbase, psize, dptr) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3601 + * global __cuMemGetAddressRange_v2 + * cuPythonInit() + * if __cuMemGetAddressRange_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemGetAddressRange_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3604 + * with gil: + * raise RuntimeError('Function "cuMemGetAddressRange_v2" not found') + * err = ( __cuMemGetAddressRange_v2)(pbase, psize, dptr) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, size_t *, CUdeviceptr))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAddressRange_v2)(__pyx_v_pbase, __pyx_v_psize, __pyx_v_dptr); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3604, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3605 + * raise RuntimeError('Function "cuMemGetAddressRange_v2" not found') + * err = ( __cuMemGetAddressRange_v2)(pbase, psize, dptr) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemAllocHost_v2(void** pp, size_t bytesize) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3598 + * return err + * + * cdef CUresult _cuMemGetAddressRange_v2(CUdeviceptr* pbase, size_t* psize, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemGetAddressRange_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemGetAddressRange_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3607 + * return err + * + * cdef CUresult _cuMemAllocHost_v2(void** pp, size_t bytesize) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAllocHost_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAllocHost_v2(void **__pyx_v_pp, size_t __pyx_v_bytesize) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemAllocHost_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3609 + * cdef CUresult _cuMemAllocHost_v2(void** pp, size_t bytesize) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemAllocHost_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemAllocHost_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3609, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3610 + * global __cuMemAllocHost_v2 + * cuPythonInit() + * if __cuMemAllocHost_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAllocHost_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocHost_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3611 + * cuPythonInit() + * if __cuMemAllocHost_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAllocHost_v2" not found') + * err = ( __cuMemAllocHost_v2)(pp, bytesize) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3612 + * if __cuMemAllocHost_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemAllocHost_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemAllocHost_v2)(pp, bytesize) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemAllocHost_v2_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3612, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3612, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3611 + * cuPythonInit() + * if __cuMemAllocHost_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAllocHost_v2" not found') + * err = ( __cuMemAllocHost_v2)(pp, bytesize) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3610 + * global __cuMemAllocHost_v2 + * cuPythonInit() + * if __cuMemAllocHost_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAllocHost_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3613 + * with gil: + * raise RuntimeError('Function "cuMemAllocHost_v2" not found') + * err = ( __cuMemAllocHost_v2)(pp, bytesize) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void **, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocHost_v2)(__pyx_v_pp, __pyx_v_bytesize); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3613, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3614 + * raise RuntimeError('Function "cuMemAllocHost_v2" not found') + * err = ( __cuMemAllocHost_v2)(pp, bytesize) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemFreeHost(void* p) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3607 + * return err + * + * cdef CUresult _cuMemAllocHost_v2(void** pp, size_t bytesize) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAllocHost_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemAllocHost_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3616 + * return err + * + * cdef CUresult _cuMemFreeHost(void* p) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemFreeHost + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemFreeHost(void *__pyx_v_p) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemFreeHost", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3618 + * cdef CUresult _cuMemFreeHost(void* p) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemFreeHost + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemFreeHost == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3618, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3619 + * global __cuMemFreeHost + * cuPythonInit() + * if __cuMemFreeHost == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemFreeHost" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeHost == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3620 + * cuPythonInit() + * if __cuMemFreeHost == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemFreeHost" not found') + * err = ( __cuMemFreeHost)(p) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3621 + * if __cuMemFreeHost == NULL: + * with gil: + * raise RuntimeError('Function "cuMemFreeHost" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemFreeHost)(p) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemFreeHost_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3621, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3621, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3620 + * cuPythonInit() + * if __cuMemFreeHost == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemFreeHost" not found') + * err = ( __cuMemFreeHost)(p) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3619 + * global __cuMemFreeHost + * cuPythonInit() + * if __cuMemFreeHost == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemFreeHost" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3622 + * with gil: + * raise RuntimeError('Function "cuMemFreeHost" not found') + * err = ( __cuMemFreeHost)(p) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeHost)(__pyx_v_p); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3622, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3623 + * raise RuntimeError('Function "cuMemFreeHost" not found') + * err = ( __cuMemFreeHost)(p) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemHostAlloc(void** pp, size_t bytesize, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3616 + * return err + * + * cdef CUresult _cuMemFreeHost(void* p) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemFreeHost + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemFreeHost", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3625 + * return err + * + * cdef CUresult _cuMemHostAlloc(void** pp, size_t bytesize, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemHostAlloc + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemHostAlloc(void **__pyx_v_pp, size_t __pyx_v_bytesize, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemHostAlloc", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3627 + * cdef CUresult _cuMemHostAlloc(void** pp, size_t bytesize, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemHostAlloc + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemHostAlloc == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3627, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3628 + * global __cuMemHostAlloc + * cuPythonInit() + * if __cuMemHostAlloc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemHostAlloc" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostAlloc == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3629 + * cuPythonInit() + * if __cuMemHostAlloc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemHostAlloc" not found') + * err = ( __cuMemHostAlloc)(pp, bytesize, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3630 + * if __cuMemHostAlloc == NULL: + * with gil: + * raise RuntimeError('Function "cuMemHostAlloc" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemHostAlloc)(pp, bytesize, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemHostAlloc_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3630, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3630, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3629 + * cuPythonInit() + * if __cuMemHostAlloc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemHostAlloc" not found') + * err = ( __cuMemHostAlloc)(pp, bytesize, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3628 + * global __cuMemHostAlloc + * cuPythonInit() + * if __cuMemHostAlloc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemHostAlloc" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3631 + * with gil: + * raise RuntimeError('Function "cuMemHostAlloc" not found') + * err = ( __cuMemHostAlloc)(pp, bytesize, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void **, size_t, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostAlloc)(__pyx_v_pp, __pyx_v_bytesize, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3631, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3632 + * raise RuntimeError('Function "cuMemHostAlloc" not found') + * err = ( __cuMemHostAlloc)(pp, bytesize, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemHostGetDevicePointer_v2(CUdeviceptr* pdptr, void* p, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3625 + * return err + * + * cdef CUresult _cuMemHostAlloc(void** pp, size_t bytesize, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemHostAlloc + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemHostAlloc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3634 + * return err + * + * cdef CUresult _cuMemHostGetDevicePointer_v2(CUdeviceptr* pdptr, void* p, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemHostGetDevicePointer_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemHostGetDevicePointer_v2(CUdeviceptr *__pyx_v_pdptr, void *__pyx_v_p, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemHostGetDevicePointer_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3636 + * cdef CUresult _cuMemHostGetDevicePointer_v2(CUdeviceptr* pdptr, void* p, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemHostGetDevicePointer_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemHostGetDevicePointer_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3636, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3637 + * global __cuMemHostGetDevicePointer_v2 + * cuPythonInit() + * if __cuMemHostGetDevicePointer_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemHostGetDevicePointer_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetDevicePointer_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3638 + * cuPythonInit() + * if __cuMemHostGetDevicePointer_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemHostGetDevicePointer_v2" not found') + * err = ( __cuMemHostGetDevicePointer_v2)(pdptr, p, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3639 + * if __cuMemHostGetDevicePointer_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemHostGetDevicePointer_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemHostGetDevicePointer_v2)(pdptr, p, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemHostGetDevicePoint}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3639, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3639, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3638 + * cuPythonInit() + * if __cuMemHostGetDevicePointer_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemHostGetDevicePointer_v2" not found') + * err = ( __cuMemHostGetDevicePointer_v2)(pdptr, p, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3637 + * global __cuMemHostGetDevicePointer_v2 + * cuPythonInit() + * if __cuMemHostGetDevicePointer_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemHostGetDevicePointer_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3640 + * with gil: + * raise RuntimeError('Function "cuMemHostGetDevicePointer_v2" not found') + * err = ( __cuMemHostGetDevicePointer_v2)(pdptr, p, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, void *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetDevicePointer_v2)(__pyx_v_pdptr, __pyx_v_p, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3640, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3641 + * raise RuntimeError('Function "cuMemHostGetDevicePointer_v2" not found') + * err = ( __cuMemHostGetDevicePointer_v2)(pdptr, p, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemHostGetFlags(unsigned int* pFlags, void* p) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3634 + * return err + * + * cdef CUresult _cuMemHostGetDevicePointer_v2(CUdeviceptr* pdptr, void* p, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemHostGetDevicePointer_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemHostGetDevicePointer_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3643 + * return err + * + * cdef CUresult _cuMemHostGetFlags(unsigned int* pFlags, void* p) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemHostGetFlags + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemHostGetFlags(unsigned int *__pyx_v_pFlags, void *__pyx_v_p) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemHostGetFlags", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3645 + * cdef CUresult _cuMemHostGetFlags(unsigned int* pFlags, void* p) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemHostGetFlags + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemHostGetFlags == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3645, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3646 + * global __cuMemHostGetFlags + * cuPythonInit() + * if __cuMemHostGetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemHostGetFlags" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetFlags == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3647 + * cuPythonInit() + * if __cuMemHostGetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemHostGetFlags" not found') + * err = ( __cuMemHostGetFlags)(pFlags, p) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3648 + * if __cuMemHostGetFlags == NULL: + * with gil: + * raise RuntimeError('Function "cuMemHostGetFlags" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemHostGetFlags)(pFlags, p) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemHostGetFlags_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3648, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3648, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3647 + * cuPythonInit() + * if __cuMemHostGetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemHostGetFlags" not found') + * err = ( __cuMemHostGetFlags)(pFlags, p) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3646 + * global __cuMemHostGetFlags + * cuPythonInit() + * if __cuMemHostGetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemHostGetFlags" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3649 + * with gil: + * raise RuntimeError('Function "cuMemHostGetFlags" not found') + * err = ( __cuMemHostGetFlags)(pFlags, p) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(unsigned int *, void *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetFlags)(__pyx_v_pFlags, __pyx_v_p); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3649, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3650 + * raise RuntimeError('Function "cuMemHostGetFlags" not found') + * err = ( __cuMemHostGetFlags)(pFlags, p) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemAllocManaged(CUdeviceptr* dptr, size_t bytesize, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3643 + * return err + * + * cdef CUresult _cuMemHostGetFlags(unsigned int* pFlags, void* p) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemHostGetFlags + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemHostGetFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3652 + * return err + * + * cdef CUresult _cuMemAllocManaged(CUdeviceptr* dptr, size_t bytesize, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAllocManaged + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAllocManaged(CUdeviceptr *__pyx_v_dptr, size_t __pyx_v_bytesize, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemAllocManaged", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3654 + * cdef CUresult _cuMemAllocManaged(CUdeviceptr* dptr, size_t bytesize, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemAllocManaged + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemAllocManaged == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3654, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3655 + * global __cuMemAllocManaged + * cuPythonInit() + * if __cuMemAllocManaged == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAllocManaged" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocManaged == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3656 + * cuPythonInit() + * if __cuMemAllocManaged == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAllocManaged" not found') + * err = ( __cuMemAllocManaged)(dptr, bytesize, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3657 + * if __cuMemAllocManaged == NULL: + * with gil: + * raise RuntimeError('Function "cuMemAllocManaged" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemAllocManaged)(dptr, bytesize, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemAllocManaged_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3657, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3657, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3656 + * cuPythonInit() + * if __cuMemAllocManaged == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAllocManaged" not found') + * err = ( __cuMemAllocManaged)(dptr, bytesize, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3655 + * global __cuMemAllocManaged + * cuPythonInit() + * if __cuMemAllocManaged == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAllocManaged" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3658 + * with gil: + * raise RuntimeError('Function "cuMemAllocManaged" not found') + * err = ( __cuMemAllocManaged)(dptr, bytesize, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, size_t, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocManaged)(__pyx_v_dptr, __pyx_v_bytesize, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3658, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3659 + * raise RuntimeError('Function "cuMemAllocManaged" not found') + * err = ( __cuMemAllocManaged)(dptr, bytesize, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceRegisterAsyncNotification(CUdevice device, CUasyncCallback callbackFunc, void* userData, CUasyncCallbackHandle* callback) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3652 + * return err + * + * cdef CUresult _cuMemAllocManaged(CUdeviceptr* dptr, size_t bytesize, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAllocManaged + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemAllocManaged", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3661 + * return err + * + * cdef CUresult _cuDeviceRegisterAsyncNotification(CUdevice device, CUasyncCallback callbackFunc, void* userData, CUasyncCallbackHandle* callback) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceRegisterAsyncNotification + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceRegisterAsyncNotification(CUdevice __pyx_v_device, CUasyncCallback __pyx_v_callbackFunc, void *__pyx_v_userData, CUasyncCallbackHandle *__pyx_v_callback) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceRegisterAsyncNotification", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3663 + * cdef CUresult _cuDeviceRegisterAsyncNotification(CUdevice device, CUasyncCallback callbackFunc, void* userData, CUasyncCallbackHandle* callback) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceRegisterAsyncNotification + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceRegisterAsyncNotification == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3663, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3664 + * global __cuDeviceRegisterAsyncNotification + * cuPythonInit() + * if __cuDeviceRegisterAsyncNotification == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceRegisterAsyncNotification" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceRegisterAsyncNotification == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3665 + * cuPythonInit() + * if __cuDeviceRegisterAsyncNotification == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceRegisterAsyncNotification" not found') + * err = ( __cuDeviceRegisterAsyncNotification)(device, callbackFunc, userData, callback) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3666 + * if __cuDeviceRegisterAsyncNotification == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceRegisterAsyncNotification" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceRegisterAsyncNotification)(device, callbackFunc, userData, callback) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceRegisterAsyncNo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3666, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3666, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3665 + * cuPythonInit() + * if __cuDeviceRegisterAsyncNotification == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceRegisterAsyncNotification" not found') + * err = ( __cuDeviceRegisterAsyncNotification)(device, callbackFunc, userData, callback) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3664 + * global __cuDeviceRegisterAsyncNotification + * cuPythonInit() + * if __cuDeviceRegisterAsyncNotification == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceRegisterAsyncNotification" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3667 + * with gil: + * raise RuntimeError('Function "cuDeviceRegisterAsyncNotification" not found') + * err = ( __cuDeviceRegisterAsyncNotification)(device, callbackFunc, userData, callback) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice, CUasyncCallback, void *, CUasyncCallbackHandle *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceRegisterAsyncNotification)(__pyx_v_device, __pyx_v_callbackFunc, __pyx_v_userData, __pyx_v_callback); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3667, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3668 + * raise RuntimeError('Function "cuDeviceRegisterAsyncNotification" not found') + * err = ( __cuDeviceRegisterAsyncNotification)(device, callbackFunc, userData, callback) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceUnregisterAsyncNotification(CUdevice device, CUasyncCallbackHandle callback) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3661 + * return err + * + * cdef CUresult _cuDeviceRegisterAsyncNotification(CUdevice device, CUasyncCallback callbackFunc, void* userData, CUasyncCallbackHandle* callback) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceRegisterAsyncNotification + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceRegisterAsyncNotification", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3670 + * return err + * + * cdef CUresult _cuDeviceUnregisterAsyncNotification(CUdevice device, CUasyncCallbackHandle callback) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceUnregisterAsyncNotification + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceUnregisterAsyncNotification(CUdevice __pyx_v_device, CUasyncCallbackHandle __pyx_v_callback) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceUnregisterAsyncNotification", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3672 + * cdef CUresult _cuDeviceUnregisterAsyncNotification(CUdevice device, CUasyncCallbackHandle callback) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceUnregisterAsyncNotification + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceUnregisterAsyncNotification == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3672, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3673 + * global __cuDeviceUnregisterAsyncNotification + * cuPythonInit() + * if __cuDeviceUnregisterAsyncNotification == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceUnregisterAsyncNotification" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceUnregisterAsyncNotification == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3674 + * cuPythonInit() + * if __cuDeviceUnregisterAsyncNotification == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceUnregisterAsyncNotification" not found') + * err = ( __cuDeviceUnregisterAsyncNotification)(device, callback) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3675 + * if __cuDeviceUnregisterAsyncNotification == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceUnregisterAsyncNotification" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceUnregisterAsyncNotification)(device, callback) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceUnregisterAsync}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3675, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3675, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3674 + * cuPythonInit() + * if __cuDeviceUnregisterAsyncNotification == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceUnregisterAsyncNotification" not found') + * err = ( __cuDeviceUnregisterAsyncNotification)(device, callback) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3673 + * global __cuDeviceUnregisterAsyncNotification + * cuPythonInit() + * if __cuDeviceUnregisterAsyncNotification == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceUnregisterAsyncNotification" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3676 + * with gil: + * raise RuntimeError('Function "cuDeviceUnregisterAsyncNotification" not found') + * err = ( __cuDeviceUnregisterAsyncNotification)(device, callback) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice, CUasyncCallbackHandle))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceUnregisterAsyncNotification)(__pyx_v_device, __pyx_v_callback); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3676, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3677 + * raise RuntimeError('Function "cuDeviceUnregisterAsyncNotification" not found') + * err = ( __cuDeviceUnregisterAsyncNotification)(device, callback) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetByPCIBusId(CUdevice* dev, const char* pciBusId) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3670 + * return err + * + * cdef CUresult _cuDeviceUnregisterAsyncNotification(CUdevice device, CUasyncCallbackHandle callback) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceUnregisterAsyncNotification + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceUnregisterAsyncNotification", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3679 + * return err + * + * cdef CUresult _cuDeviceGetByPCIBusId(CUdevice* dev, const char* pciBusId) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetByPCIBusId + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetByPCIBusId(CUdevice *__pyx_v_dev, char const *__pyx_v_pciBusId) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetByPCIBusId", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3681 + * cdef CUresult _cuDeviceGetByPCIBusId(CUdevice* dev, const char* pciBusId) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetByPCIBusId + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetByPCIBusId == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3681, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3682 + * global __cuDeviceGetByPCIBusId + * cuPythonInit() + * if __cuDeviceGetByPCIBusId == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetByPCIBusId" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetByPCIBusId == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3683 + * cuPythonInit() + * if __cuDeviceGetByPCIBusId == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetByPCIBusId" not found') + * err = ( __cuDeviceGetByPCIBusId)(dev, pciBusId) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3684 + * if __cuDeviceGetByPCIBusId == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetByPCIBusId" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetByPCIBusId)(dev, pciBusId) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetByPCIBusId_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3684, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3684, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3683 + * cuPythonInit() + * if __cuDeviceGetByPCIBusId == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetByPCIBusId" not found') + * err = ( __cuDeviceGetByPCIBusId)(dev, pciBusId) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3682 + * global __cuDeviceGetByPCIBusId + * cuPythonInit() + * if __cuDeviceGetByPCIBusId == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetByPCIBusId" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3685 + * with gil: + * raise RuntimeError('Function "cuDeviceGetByPCIBusId" not found') + * err = ( __cuDeviceGetByPCIBusId)(dev, pciBusId) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice *, char const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetByPCIBusId)(__pyx_v_dev, __pyx_v_pciBusId); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3685, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3686 + * raise RuntimeError('Function "cuDeviceGetByPCIBusId" not found') + * err = ( __cuDeviceGetByPCIBusId)(dev, pciBusId) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetPCIBusId(char* pciBusId, int length, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3679 + * return err + * + * cdef CUresult _cuDeviceGetByPCIBusId(CUdevice* dev, const char* pciBusId) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetByPCIBusId + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetByPCIBusId", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3688 + * return err + * + * cdef CUresult _cuDeviceGetPCIBusId(char* pciBusId, int length, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetPCIBusId + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetPCIBusId(char *__pyx_v_pciBusId, int __pyx_v_length, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetPCIBusId", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3690 + * cdef CUresult _cuDeviceGetPCIBusId(char* pciBusId, int length, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetPCIBusId + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetPCIBusId == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3690, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3691 + * global __cuDeviceGetPCIBusId + * cuPythonInit() + * if __cuDeviceGetPCIBusId == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetPCIBusId" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetPCIBusId == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3692 + * cuPythonInit() + * if __cuDeviceGetPCIBusId == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetPCIBusId" not found') + * err = ( __cuDeviceGetPCIBusId)(pciBusId, length, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3693 + * if __cuDeviceGetPCIBusId == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetPCIBusId" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetPCIBusId)(pciBusId, length, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetPCIBusId_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3693, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3693, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3692 + * cuPythonInit() + * if __cuDeviceGetPCIBusId == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetPCIBusId" not found') + * err = ( __cuDeviceGetPCIBusId)(pciBusId, length, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3691 + * global __cuDeviceGetPCIBusId + * cuPythonInit() + * if __cuDeviceGetPCIBusId == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetPCIBusId" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3694 + * with gil: + * raise RuntimeError('Function "cuDeviceGetPCIBusId" not found') + * err = ( __cuDeviceGetPCIBusId)(pciBusId, length, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(char *, int, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetPCIBusId)(__pyx_v_pciBusId, __pyx_v_length, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3694, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3695 + * raise RuntimeError('Function "cuDeviceGetPCIBusId" not found') + * err = ( __cuDeviceGetPCIBusId)(pciBusId, length, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuIpcGetEventHandle(CUipcEventHandle* pHandle, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3688 + * return err + * + * cdef CUresult _cuDeviceGetPCIBusId(char* pciBusId, int length, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetPCIBusId + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetPCIBusId", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3697 + * return err + * + * cdef CUresult _cuIpcGetEventHandle(CUipcEventHandle* pHandle, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuIpcGetEventHandle + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuIpcGetEventHandle(CUipcEventHandle *__pyx_v_pHandle, CUevent __pyx_v_event) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuIpcGetEventHandle", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3699 + * cdef CUresult _cuIpcGetEventHandle(CUipcEventHandle* pHandle, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuIpcGetEventHandle + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuIpcGetEventHandle == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3699, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3700 + * global __cuIpcGetEventHandle + * cuPythonInit() + * if __cuIpcGetEventHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuIpcGetEventHandle" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetEventHandle == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3701 + * cuPythonInit() + * if __cuIpcGetEventHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuIpcGetEventHandle" not found') + * err = ( __cuIpcGetEventHandle)(pHandle, event) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3702 + * if __cuIpcGetEventHandle == NULL: + * with gil: + * raise RuntimeError('Function "cuIpcGetEventHandle" not found') # <<<<<<<<<<<<<< + * err = ( __cuIpcGetEventHandle)(pHandle, event) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuIpcGetEventHandle_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3702, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3702, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3701 + * cuPythonInit() + * if __cuIpcGetEventHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuIpcGetEventHandle" not found') + * err = ( __cuIpcGetEventHandle)(pHandle, event) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3700 + * global __cuIpcGetEventHandle + * cuPythonInit() + * if __cuIpcGetEventHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuIpcGetEventHandle" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3703 + * with gil: + * raise RuntimeError('Function "cuIpcGetEventHandle" not found') + * err = ( __cuIpcGetEventHandle)(pHandle, event) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUipcEventHandle *, CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetEventHandle)(__pyx_v_pHandle, __pyx_v_event); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3703, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3704 + * raise RuntimeError('Function "cuIpcGetEventHandle" not found') + * err = ( __cuIpcGetEventHandle)(pHandle, event) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuIpcOpenEventHandle(CUevent* phEvent, CUipcEventHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3697 + * return err + * + * cdef CUresult _cuIpcGetEventHandle(CUipcEventHandle* pHandle, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuIpcGetEventHandle + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuIpcGetEventHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3706 + * return err + * + * cdef CUresult _cuIpcOpenEventHandle(CUevent* phEvent, CUipcEventHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuIpcOpenEventHandle + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuIpcOpenEventHandle(CUevent *__pyx_v_phEvent, CUipcEventHandle __pyx_v_handle) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuIpcOpenEventHandle", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3708 + * cdef CUresult _cuIpcOpenEventHandle(CUevent* phEvent, CUipcEventHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuIpcOpenEventHandle + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuIpcOpenEventHandle == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3708, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3709 + * global __cuIpcOpenEventHandle + * cuPythonInit() + * if __cuIpcOpenEventHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuIpcOpenEventHandle" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenEventHandle == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3710 + * cuPythonInit() + * if __cuIpcOpenEventHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuIpcOpenEventHandle" not found') + * err = ( __cuIpcOpenEventHandle)(phEvent, handle) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3711 + * if __cuIpcOpenEventHandle == NULL: + * with gil: + * raise RuntimeError('Function "cuIpcOpenEventHandle" not found') # <<<<<<<<<<<<<< + * err = ( __cuIpcOpenEventHandle)(phEvent, handle) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuIpcOpenEventHandle_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3711, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3711, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3710 + * cuPythonInit() + * if __cuIpcOpenEventHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuIpcOpenEventHandle" not found') + * err = ( __cuIpcOpenEventHandle)(phEvent, handle) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3709 + * global __cuIpcOpenEventHandle + * cuPythonInit() + * if __cuIpcOpenEventHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuIpcOpenEventHandle" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3712 + * with gil: + * raise RuntimeError('Function "cuIpcOpenEventHandle" not found') + * err = ( __cuIpcOpenEventHandle)(phEvent, handle) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUevent *, CUipcEventHandle))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenEventHandle)(__pyx_v_phEvent, __pyx_v_handle); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3712, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3713 + * raise RuntimeError('Function "cuIpcOpenEventHandle" not found') + * err = ( __cuIpcOpenEventHandle)(phEvent, handle) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuIpcGetMemHandle(CUipcMemHandle* pHandle, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3706 + * return err + * + * cdef CUresult _cuIpcOpenEventHandle(CUevent* phEvent, CUipcEventHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuIpcOpenEventHandle + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuIpcOpenEventHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3715 + * return err + * + * cdef CUresult _cuIpcGetMemHandle(CUipcMemHandle* pHandle, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuIpcGetMemHandle + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuIpcGetMemHandle(CUipcMemHandle *__pyx_v_pHandle, CUdeviceptr __pyx_v_dptr) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuIpcGetMemHandle", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3717 + * cdef CUresult _cuIpcGetMemHandle(CUipcMemHandle* pHandle, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuIpcGetMemHandle + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuIpcGetMemHandle == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3717, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3718 + * global __cuIpcGetMemHandle + * cuPythonInit() + * if __cuIpcGetMemHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuIpcGetMemHandle" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetMemHandle == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3719 + * cuPythonInit() + * if __cuIpcGetMemHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuIpcGetMemHandle" not found') + * err = ( __cuIpcGetMemHandle)(pHandle, dptr) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3720 + * if __cuIpcGetMemHandle == NULL: + * with gil: + * raise RuntimeError('Function "cuIpcGetMemHandle" not found') # <<<<<<<<<<<<<< + * err = ( __cuIpcGetMemHandle)(pHandle, dptr) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuIpcGetMemHandle_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3720, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3720, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3719 + * cuPythonInit() + * if __cuIpcGetMemHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuIpcGetMemHandle" not found') + * err = ( __cuIpcGetMemHandle)(pHandle, dptr) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3718 + * global __cuIpcGetMemHandle + * cuPythonInit() + * if __cuIpcGetMemHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuIpcGetMemHandle" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3721 + * with gil: + * raise RuntimeError('Function "cuIpcGetMemHandle" not found') + * err = ( __cuIpcGetMemHandle)(pHandle, dptr) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUipcMemHandle *, CUdeviceptr))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetMemHandle)(__pyx_v_pHandle, __pyx_v_dptr); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3721, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3722 + * raise RuntimeError('Function "cuIpcGetMemHandle" not found') + * err = ( __cuIpcGetMemHandle)(pHandle, dptr) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuIpcOpenMemHandle_v2(CUdeviceptr* pdptr, CUipcMemHandle handle, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3715 + * return err + * + * cdef CUresult _cuIpcGetMemHandle(CUipcMemHandle* pHandle, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuIpcGetMemHandle + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuIpcGetMemHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3724 + * return err + * + * cdef CUresult _cuIpcOpenMemHandle_v2(CUdeviceptr* pdptr, CUipcMemHandle handle, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuIpcOpenMemHandle_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuIpcOpenMemHandle_v2(CUdeviceptr *__pyx_v_pdptr, CUipcMemHandle __pyx_v_handle, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuIpcOpenMemHandle_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3726 + * cdef CUresult _cuIpcOpenMemHandle_v2(CUdeviceptr* pdptr, CUipcMemHandle handle, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuIpcOpenMemHandle_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuIpcOpenMemHandle_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3726, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3727 + * global __cuIpcOpenMemHandle_v2 + * cuPythonInit() + * if __cuIpcOpenMemHandle_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuIpcOpenMemHandle_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenMemHandle_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3728 + * cuPythonInit() + * if __cuIpcOpenMemHandle_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuIpcOpenMemHandle_v2" not found') + * err = ( __cuIpcOpenMemHandle_v2)(pdptr, handle, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3729 + * if __cuIpcOpenMemHandle_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuIpcOpenMemHandle_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuIpcOpenMemHandle_v2)(pdptr, handle, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuIpcOpenMemHandle_v2_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3729, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3729, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3728 + * cuPythonInit() + * if __cuIpcOpenMemHandle_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuIpcOpenMemHandle_v2" not found') + * err = ( __cuIpcOpenMemHandle_v2)(pdptr, handle, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3727 + * global __cuIpcOpenMemHandle_v2 + * cuPythonInit() + * if __cuIpcOpenMemHandle_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuIpcOpenMemHandle_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3730 + * with gil: + * raise RuntimeError('Function "cuIpcOpenMemHandle_v2" not found') + * err = ( __cuIpcOpenMemHandle_v2)(pdptr, handle, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, CUipcMemHandle, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenMemHandle_v2)(__pyx_v_pdptr, __pyx_v_handle, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3730, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3731 + * raise RuntimeError('Function "cuIpcOpenMemHandle_v2" not found') + * err = ( __cuIpcOpenMemHandle_v2)(pdptr, handle, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuIpcCloseMemHandle(CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3724 + * return err + * + * cdef CUresult _cuIpcOpenMemHandle_v2(CUdeviceptr* pdptr, CUipcMemHandle handle, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuIpcOpenMemHandle_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuIpcOpenMemHandle_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3733 + * return err + * + * cdef CUresult _cuIpcCloseMemHandle(CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuIpcCloseMemHandle + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuIpcCloseMemHandle(CUdeviceptr __pyx_v_dptr) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuIpcCloseMemHandle", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3735 + * cdef CUresult _cuIpcCloseMemHandle(CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuIpcCloseMemHandle + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuIpcCloseMemHandle == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3735, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3736 + * global __cuIpcCloseMemHandle + * cuPythonInit() + * if __cuIpcCloseMemHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuIpcCloseMemHandle" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcCloseMemHandle == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3737 + * cuPythonInit() + * if __cuIpcCloseMemHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuIpcCloseMemHandle" not found') + * err = ( __cuIpcCloseMemHandle)(dptr) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3738 + * if __cuIpcCloseMemHandle == NULL: + * with gil: + * raise RuntimeError('Function "cuIpcCloseMemHandle" not found') # <<<<<<<<<<<<<< + * err = ( __cuIpcCloseMemHandle)(dptr) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuIpcCloseMemHandle_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3738, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3738, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3737 + * cuPythonInit() + * if __cuIpcCloseMemHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuIpcCloseMemHandle" not found') + * err = ( __cuIpcCloseMemHandle)(dptr) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3736 + * global __cuIpcCloseMemHandle + * cuPythonInit() + * if __cuIpcCloseMemHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuIpcCloseMemHandle" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3739 + * with gil: + * raise RuntimeError('Function "cuIpcCloseMemHandle" not found') + * err = ( __cuIpcCloseMemHandle)(dptr) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcCloseMemHandle)(__pyx_v_dptr); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3739, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3740 + * raise RuntimeError('Function "cuIpcCloseMemHandle" not found') + * err = ( __cuIpcCloseMemHandle)(dptr) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemHostRegister_v2(void* p, size_t bytesize, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3733 + * return err + * + * cdef CUresult _cuIpcCloseMemHandle(CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuIpcCloseMemHandle + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuIpcCloseMemHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3742 + * return err + * + * cdef CUresult _cuMemHostRegister_v2(void* p, size_t bytesize, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemHostRegister_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemHostRegister_v2(void *__pyx_v_p, size_t __pyx_v_bytesize, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemHostRegister_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3744 + * cdef CUresult _cuMemHostRegister_v2(void* p, size_t bytesize, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemHostRegister_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemHostRegister_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3744, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3745 + * global __cuMemHostRegister_v2 + * cuPythonInit() + * if __cuMemHostRegister_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemHostRegister_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostRegister_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3746 + * cuPythonInit() + * if __cuMemHostRegister_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemHostRegister_v2" not found') + * err = ( __cuMemHostRegister_v2)(p, bytesize, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3747 + * if __cuMemHostRegister_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemHostRegister_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemHostRegister_v2)(p, bytesize, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemHostRegister_v2_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3747, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3747, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3746 + * cuPythonInit() + * if __cuMemHostRegister_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemHostRegister_v2" not found') + * err = ( __cuMemHostRegister_v2)(p, bytesize, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3745 + * global __cuMemHostRegister_v2 + * cuPythonInit() + * if __cuMemHostRegister_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemHostRegister_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3748 + * with gil: + * raise RuntimeError('Function "cuMemHostRegister_v2" not found') + * err = ( __cuMemHostRegister_v2)(p, bytesize, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void *, size_t, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostRegister_v2)(__pyx_v_p, __pyx_v_bytesize, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3748, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3749 + * raise RuntimeError('Function "cuMemHostRegister_v2" not found') + * err = ( __cuMemHostRegister_v2)(p, bytesize, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemHostUnregister(void* p) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3742 + * return err + * + * cdef CUresult _cuMemHostRegister_v2(void* p, size_t bytesize, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemHostRegister_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemHostRegister_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3751 + * return err + * + * cdef CUresult _cuMemHostUnregister(void* p) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemHostUnregister + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemHostUnregister(void *__pyx_v_p) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemHostUnregister", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3753 + * cdef CUresult _cuMemHostUnregister(void* p) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemHostUnregister + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemHostUnregister == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3753, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3754 + * global __cuMemHostUnregister + * cuPythonInit() + * if __cuMemHostUnregister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemHostUnregister" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostUnregister == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3755 + * cuPythonInit() + * if __cuMemHostUnregister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemHostUnregister" not found') + * err = ( __cuMemHostUnregister)(p) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3756 + * if __cuMemHostUnregister == NULL: + * with gil: + * raise RuntimeError('Function "cuMemHostUnregister" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemHostUnregister)(p) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemHostUnregister_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3756, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3756, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3755 + * cuPythonInit() + * if __cuMemHostUnregister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemHostUnregister" not found') + * err = ( __cuMemHostUnregister)(p) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3754 + * global __cuMemHostUnregister + * cuPythonInit() + * if __cuMemHostUnregister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemHostUnregister" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3757 + * with gil: + * raise RuntimeError('Function "cuMemHostUnregister" not found') + * err = ( __cuMemHostUnregister)(p) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostUnregister)(__pyx_v_p); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3757, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3758 + * raise RuntimeError('Function "cuMemHostUnregister" not found') + * err = ( __cuMemHostUnregister)(p) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpy(CUdeviceptr dst, CUdeviceptr src, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3751 + * return err + * + * cdef CUresult _cuMemHostUnregister(void* p) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemHostUnregister + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemHostUnregister", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3760 + * return err + * + * cdef CUresult _cuMemcpy(CUdeviceptr dst, CUdeviceptr src, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy(CUdeviceptr __pyx_v_dst, CUdeviceptr __pyx_v_src, size_t __pyx_v_ByteCount) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpy", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3762 + * cdef CUresult _cuMemcpy(CUdeviceptr dst, CUdeviceptr src, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpy + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3762, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3763 + * global __cuMemcpy + * cuPythonInit() + * if __cuMemcpy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3764 + * cuPythonInit() + * if __cuMemcpy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy" not found') + * err = ( __cuMemcpy)(dst, src, ByteCount) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3765 + * if __cuMemcpy == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpy" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpy)(dst, src, ByteCount) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpy_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3765, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3765, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3764 + * cuPythonInit() + * if __cuMemcpy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy" not found') + * err = ( __cuMemcpy)(dst, src, ByteCount) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3763 + * global __cuMemcpy + * cuPythonInit() + * if __cuMemcpy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3766 + * with gil: + * raise RuntimeError('Function "cuMemcpy" not found') + * err = ( __cuMemcpy)(dst, src, ByteCount) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, CUdeviceptr, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy)(__pyx_v_dst, __pyx_v_src, __pyx_v_ByteCount); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3766, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3767 + * raise RuntimeError('Function "cuMemcpy" not found') + * err = ( __cuMemcpy)(dst, src, ByteCount) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyPeer(CUdeviceptr dstDevice, CUcontext dstContext, CUdeviceptr srcDevice, CUcontext srcContext, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3760 + * return err + * + * cdef CUresult _cuMemcpy(CUdeviceptr dst, CUdeviceptr src, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3769 + * return err + * + * cdef CUresult _cuMemcpyPeer(CUdeviceptr dstDevice, CUcontext dstContext, CUdeviceptr srcDevice, CUcontext srcContext, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyPeer + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyPeer(CUdeviceptr __pyx_v_dstDevice, CUcontext __pyx_v_dstContext, CUdeviceptr __pyx_v_srcDevice, CUcontext __pyx_v_srcContext, size_t __pyx_v_ByteCount) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyPeer", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3771 + * cdef CUresult _cuMemcpyPeer(CUdeviceptr dstDevice, CUcontext dstContext, CUdeviceptr srcDevice, CUcontext srcContext, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyPeer + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyPeer == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3771, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3772 + * global __cuMemcpyPeer + * cuPythonInit() + * if __cuMemcpyPeer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyPeer" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeer == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3773 + * cuPythonInit() + * if __cuMemcpyPeer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyPeer" not found') + * err = ( __cuMemcpyPeer)(dstDevice, dstContext, srcDevice, srcContext, ByteCount) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3774 + * if __cuMemcpyPeer == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyPeer" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyPeer)(dstDevice, dstContext, srcDevice, srcContext, ByteCount) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyPeer_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3774, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3774, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3773 + * cuPythonInit() + * if __cuMemcpyPeer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyPeer" not found') + * err = ( __cuMemcpyPeer)(dstDevice, dstContext, srcDevice, srcContext, ByteCount) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3772 + * global __cuMemcpyPeer + * cuPythonInit() + * if __cuMemcpyPeer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyPeer" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3775 + * with gil: + * raise RuntimeError('Function "cuMemcpyPeer" not found') + * err = ( __cuMemcpyPeer)(dstDevice, dstContext, srcDevice, srcContext, ByteCount) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, CUcontext, CUdeviceptr, CUcontext, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeer)(__pyx_v_dstDevice, __pyx_v_dstContext, __pyx_v_srcDevice, __pyx_v_srcContext, __pyx_v_ByteCount); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3775, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3776 + * raise RuntimeError('Function "cuMemcpyPeer" not found') + * err = ( __cuMemcpyPeer)(dstDevice, dstContext, srcDevice, srcContext, ByteCount) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyHtoD_v2(CUdeviceptr dstDevice, const void* srcHost, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3769 + * return err + * + * cdef CUresult _cuMemcpyPeer(CUdeviceptr dstDevice, CUcontext dstContext, CUdeviceptr srcDevice, CUcontext srcContext, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyPeer + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyPeer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3778 + * return err + * + * cdef CUresult _cuMemcpyHtoD_v2(CUdeviceptr dstDevice, const void* srcHost, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyHtoD_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyHtoD_v2(CUdeviceptr __pyx_v_dstDevice, void const *__pyx_v_srcHost, size_t __pyx_v_ByteCount) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyHtoD_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3780 + * cdef CUresult _cuMemcpyHtoD_v2(CUdeviceptr dstDevice, const void* srcHost, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyHtoD_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyHtoD_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3780, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3781 + * global __cuMemcpyHtoD_v2 + * cuPythonInit() + * if __cuMemcpyHtoD_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoD_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoD_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3782 + * cuPythonInit() + * if __cuMemcpyHtoD_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyHtoD_v2" not found') + * err = ( __cuMemcpyHtoD_v2)(dstDevice, srcHost, ByteCount) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3783 + * if __cuMemcpyHtoD_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoD_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyHtoD_v2)(dstDevice, srcHost, ByteCount) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyHtoD_v2_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3783, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3783, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3782 + * cuPythonInit() + * if __cuMemcpyHtoD_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyHtoD_v2" not found') + * err = ( __cuMemcpyHtoD_v2)(dstDevice, srcHost, ByteCount) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3781 + * global __cuMemcpyHtoD_v2 + * cuPythonInit() + * if __cuMemcpyHtoD_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoD_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3784 + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoD_v2" not found') + * err = ( __cuMemcpyHtoD_v2)(dstDevice, srcHost, ByteCount) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, void const *, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoD_v2)(__pyx_v_dstDevice, __pyx_v_srcHost, __pyx_v_ByteCount); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3784, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3785 + * raise RuntimeError('Function "cuMemcpyHtoD_v2" not found') + * err = ( __cuMemcpyHtoD_v2)(dstDevice, srcHost, ByteCount) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyDtoH_v2(void* dstHost, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3778 + * return err + * + * cdef CUresult _cuMemcpyHtoD_v2(CUdeviceptr dstDevice, const void* srcHost, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyHtoD_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyHtoD_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3787 + * return err + * + * cdef CUresult _cuMemcpyDtoH_v2(void* dstHost, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyDtoH_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyDtoH_v2(void *__pyx_v_dstHost, CUdeviceptr __pyx_v_srcDevice, size_t __pyx_v_ByteCount) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyDtoH_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3789 + * cdef CUresult _cuMemcpyDtoH_v2(void* dstHost, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyDtoH_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyDtoH_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3789, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3790 + * global __cuMemcpyDtoH_v2 + * cuPythonInit() + * if __cuMemcpyDtoH_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoH_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoH_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3791 + * cuPythonInit() + * if __cuMemcpyDtoH_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyDtoH_v2" not found') + * err = ( __cuMemcpyDtoH_v2)(dstHost, srcDevice, ByteCount) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3792 + * if __cuMemcpyDtoH_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoH_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyDtoH_v2)(dstHost, srcDevice, ByteCount) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyDtoH_v2_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3792, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3792, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3791 + * cuPythonInit() + * if __cuMemcpyDtoH_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyDtoH_v2" not found') + * err = ( __cuMemcpyDtoH_v2)(dstHost, srcDevice, ByteCount) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3790 + * global __cuMemcpyDtoH_v2 + * cuPythonInit() + * if __cuMemcpyDtoH_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoH_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3793 + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoH_v2" not found') + * err = ( __cuMemcpyDtoH_v2)(dstHost, srcDevice, ByteCount) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void *, CUdeviceptr, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoH_v2)(__pyx_v_dstHost, __pyx_v_srcDevice, __pyx_v_ByteCount); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3793, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3794 + * raise RuntimeError('Function "cuMemcpyDtoH_v2" not found') + * err = ( __cuMemcpyDtoH_v2)(dstHost, srcDevice, ByteCount) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyDtoD_v2(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3787 + * return err + * + * cdef CUresult _cuMemcpyDtoH_v2(void* dstHost, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyDtoH_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyDtoH_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3796 + * return err + * + * cdef CUresult _cuMemcpyDtoD_v2(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyDtoD_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyDtoD_v2(CUdeviceptr __pyx_v_dstDevice, CUdeviceptr __pyx_v_srcDevice, size_t __pyx_v_ByteCount) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyDtoD_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3798 + * cdef CUresult _cuMemcpyDtoD_v2(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyDtoD_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyDtoD_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3798, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3799 + * global __cuMemcpyDtoD_v2 + * cuPythonInit() + * if __cuMemcpyDtoD_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoD_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoD_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3800 + * cuPythonInit() + * if __cuMemcpyDtoD_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyDtoD_v2" not found') + * err = ( __cuMemcpyDtoD_v2)(dstDevice, srcDevice, ByteCount) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3801 + * if __cuMemcpyDtoD_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoD_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyDtoD_v2)(dstDevice, srcDevice, ByteCount) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyDtoD_v2_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3801, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3801, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3800 + * cuPythonInit() + * if __cuMemcpyDtoD_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyDtoD_v2" not found') + * err = ( __cuMemcpyDtoD_v2)(dstDevice, srcDevice, ByteCount) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3799 + * global __cuMemcpyDtoD_v2 + * cuPythonInit() + * if __cuMemcpyDtoD_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoD_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3802 + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoD_v2" not found') + * err = ( __cuMemcpyDtoD_v2)(dstDevice, srcDevice, ByteCount) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, CUdeviceptr, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoD_v2)(__pyx_v_dstDevice, __pyx_v_srcDevice, __pyx_v_ByteCount); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3802, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3803 + * raise RuntimeError('Function "cuMemcpyDtoD_v2" not found') + * err = ( __cuMemcpyDtoD_v2)(dstDevice, srcDevice, ByteCount) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyDtoA_v2(CUarray dstArray, size_t dstOffset, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3796 + * return err + * + * cdef CUresult _cuMemcpyDtoD_v2(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyDtoD_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyDtoD_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3805 + * return err + * + * cdef CUresult _cuMemcpyDtoA_v2(CUarray dstArray, size_t dstOffset, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyDtoA_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyDtoA_v2(CUarray __pyx_v_dstArray, size_t __pyx_v_dstOffset, CUdeviceptr __pyx_v_srcDevice, size_t __pyx_v_ByteCount) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyDtoA_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3807 + * cdef CUresult _cuMemcpyDtoA_v2(CUarray dstArray, size_t dstOffset, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyDtoA_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyDtoA_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3807, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3808 + * global __cuMemcpyDtoA_v2 + * cuPythonInit() + * if __cuMemcpyDtoA_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoA_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoA_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3809 + * cuPythonInit() + * if __cuMemcpyDtoA_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyDtoA_v2" not found') + * err = ( __cuMemcpyDtoA_v2)(dstArray, dstOffset, srcDevice, ByteCount) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3810 + * if __cuMemcpyDtoA_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoA_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyDtoA_v2)(dstArray, dstOffset, srcDevice, ByteCount) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyDtoA_v2_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3810, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3810, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3809 + * cuPythonInit() + * if __cuMemcpyDtoA_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyDtoA_v2" not found') + * err = ( __cuMemcpyDtoA_v2)(dstArray, dstOffset, srcDevice, ByteCount) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3808 + * global __cuMemcpyDtoA_v2 + * cuPythonInit() + * if __cuMemcpyDtoA_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoA_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3811 + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoA_v2" not found') + * err = ( __cuMemcpyDtoA_v2)(dstArray, dstOffset, srcDevice, ByteCount) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarray, size_t, CUdeviceptr, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoA_v2)(__pyx_v_dstArray, __pyx_v_dstOffset, __pyx_v_srcDevice, __pyx_v_ByteCount); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3811, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3812 + * raise RuntimeError('Function "cuMemcpyDtoA_v2" not found') + * err = ( __cuMemcpyDtoA_v2)(dstArray, dstOffset, srcDevice, ByteCount) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyAtoD_v2(CUdeviceptr dstDevice, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3805 + * return err + * + * cdef CUresult _cuMemcpyDtoA_v2(CUarray dstArray, size_t dstOffset, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyDtoA_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyDtoA_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3814 + * return err + * + * cdef CUresult _cuMemcpyAtoD_v2(CUdeviceptr dstDevice, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyAtoD_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyAtoD_v2(CUdeviceptr __pyx_v_dstDevice, CUarray __pyx_v_srcArray, size_t __pyx_v_srcOffset, size_t __pyx_v_ByteCount) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyAtoD_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3816 + * cdef CUresult _cuMemcpyAtoD_v2(CUdeviceptr dstDevice, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyAtoD_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyAtoD_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3816, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3817 + * global __cuMemcpyAtoD_v2 + * cuPythonInit() + * if __cuMemcpyAtoD_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoD_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoD_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3818 + * cuPythonInit() + * if __cuMemcpyAtoD_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyAtoD_v2" not found') + * err = ( __cuMemcpyAtoD_v2)(dstDevice, srcArray, srcOffset, ByteCount) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3819 + * if __cuMemcpyAtoD_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoD_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyAtoD_v2)(dstDevice, srcArray, srcOffset, ByteCount) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyAtoD_v2_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3819, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3819, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3818 + * cuPythonInit() + * if __cuMemcpyAtoD_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyAtoD_v2" not found') + * err = ( __cuMemcpyAtoD_v2)(dstDevice, srcArray, srcOffset, ByteCount) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3817 + * global __cuMemcpyAtoD_v2 + * cuPythonInit() + * if __cuMemcpyAtoD_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoD_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3820 + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoD_v2" not found') + * err = ( __cuMemcpyAtoD_v2)(dstDevice, srcArray, srcOffset, ByteCount) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, CUarray, size_t, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoD_v2)(__pyx_v_dstDevice, __pyx_v_srcArray, __pyx_v_srcOffset, __pyx_v_ByteCount); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3820, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3821 + * raise RuntimeError('Function "cuMemcpyAtoD_v2" not found') + * err = ( __cuMemcpyAtoD_v2)(dstDevice, srcArray, srcOffset, ByteCount) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyHtoA_v2(CUarray dstArray, size_t dstOffset, const void* srcHost, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3814 + * return err + * + * cdef CUresult _cuMemcpyAtoD_v2(CUdeviceptr dstDevice, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyAtoD_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyAtoD_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3823 + * return err + * + * cdef CUresult _cuMemcpyHtoA_v2(CUarray dstArray, size_t dstOffset, const void* srcHost, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyHtoA_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyHtoA_v2(CUarray __pyx_v_dstArray, size_t __pyx_v_dstOffset, void const *__pyx_v_srcHost, size_t __pyx_v_ByteCount) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyHtoA_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3825 + * cdef CUresult _cuMemcpyHtoA_v2(CUarray dstArray, size_t dstOffset, const void* srcHost, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyHtoA_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyHtoA_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3825, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3826 + * global __cuMemcpyHtoA_v2 + * cuPythonInit() + * if __cuMemcpyHtoA_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoA_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoA_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3827 + * cuPythonInit() + * if __cuMemcpyHtoA_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyHtoA_v2" not found') + * err = ( __cuMemcpyHtoA_v2)(dstArray, dstOffset, srcHost, ByteCount) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3828 + * if __cuMemcpyHtoA_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoA_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyHtoA_v2)(dstArray, dstOffset, srcHost, ByteCount) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyHtoA_v2_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3828, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3828, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3827 + * cuPythonInit() + * if __cuMemcpyHtoA_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyHtoA_v2" not found') + * err = ( __cuMemcpyHtoA_v2)(dstArray, dstOffset, srcHost, ByteCount) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3826 + * global __cuMemcpyHtoA_v2 + * cuPythonInit() + * if __cuMemcpyHtoA_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoA_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3829 + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoA_v2" not found') + * err = ( __cuMemcpyHtoA_v2)(dstArray, dstOffset, srcHost, ByteCount) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarray, size_t, void const *, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoA_v2)(__pyx_v_dstArray, __pyx_v_dstOffset, __pyx_v_srcHost, __pyx_v_ByteCount); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3829, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3830 + * raise RuntimeError('Function "cuMemcpyHtoA_v2" not found') + * err = ( __cuMemcpyHtoA_v2)(dstArray, dstOffset, srcHost, ByteCount) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyAtoH_v2(void* dstHost, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3823 + * return err + * + * cdef CUresult _cuMemcpyHtoA_v2(CUarray dstArray, size_t dstOffset, const void* srcHost, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyHtoA_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyHtoA_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3832 + * return err + * + * cdef CUresult _cuMemcpyAtoH_v2(void* dstHost, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyAtoH_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyAtoH_v2(void *__pyx_v_dstHost, CUarray __pyx_v_srcArray, size_t __pyx_v_srcOffset, size_t __pyx_v_ByteCount) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyAtoH_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3834 + * cdef CUresult _cuMemcpyAtoH_v2(void* dstHost, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyAtoH_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyAtoH_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3834, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3835 + * global __cuMemcpyAtoH_v2 + * cuPythonInit() + * if __cuMemcpyAtoH_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoH_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoH_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3836 + * cuPythonInit() + * if __cuMemcpyAtoH_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyAtoH_v2" not found') + * err = ( __cuMemcpyAtoH_v2)(dstHost, srcArray, srcOffset, ByteCount) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3837 + * if __cuMemcpyAtoH_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoH_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyAtoH_v2)(dstHost, srcArray, srcOffset, ByteCount) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyAtoH_v2_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3837, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3837, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3836 + * cuPythonInit() + * if __cuMemcpyAtoH_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyAtoH_v2" not found') + * err = ( __cuMemcpyAtoH_v2)(dstHost, srcArray, srcOffset, ByteCount) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3835 + * global __cuMemcpyAtoH_v2 + * cuPythonInit() + * if __cuMemcpyAtoH_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoH_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3838 + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoH_v2" not found') + * err = ( __cuMemcpyAtoH_v2)(dstHost, srcArray, srcOffset, ByteCount) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void *, CUarray, size_t, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoH_v2)(__pyx_v_dstHost, __pyx_v_srcArray, __pyx_v_srcOffset, __pyx_v_ByteCount); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3838, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3839 + * raise RuntimeError('Function "cuMemcpyAtoH_v2" not found') + * err = ( __cuMemcpyAtoH_v2)(dstHost, srcArray, srcOffset, ByteCount) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyAtoA_v2(CUarray dstArray, size_t dstOffset, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3832 + * return err + * + * cdef CUresult _cuMemcpyAtoH_v2(void* dstHost, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyAtoH_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyAtoH_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3841 + * return err + * + * cdef CUresult _cuMemcpyAtoA_v2(CUarray dstArray, size_t dstOffset, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyAtoA_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyAtoA_v2(CUarray __pyx_v_dstArray, size_t __pyx_v_dstOffset, CUarray __pyx_v_srcArray, size_t __pyx_v_srcOffset, size_t __pyx_v_ByteCount) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyAtoA_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3843 + * cdef CUresult _cuMemcpyAtoA_v2(CUarray dstArray, size_t dstOffset, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyAtoA_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyAtoA_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3843, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3844 + * global __cuMemcpyAtoA_v2 + * cuPythonInit() + * if __cuMemcpyAtoA_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoA_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoA_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3845 + * cuPythonInit() + * if __cuMemcpyAtoA_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyAtoA_v2" not found') + * err = ( __cuMemcpyAtoA_v2)(dstArray, dstOffset, srcArray, srcOffset, ByteCount) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3846 + * if __cuMemcpyAtoA_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoA_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyAtoA_v2)(dstArray, dstOffset, srcArray, srcOffset, ByteCount) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyAtoA_v2_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3846, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3846, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3845 + * cuPythonInit() + * if __cuMemcpyAtoA_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyAtoA_v2" not found') + * err = ( __cuMemcpyAtoA_v2)(dstArray, dstOffset, srcArray, srcOffset, ByteCount) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3844 + * global __cuMemcpyAtoA_v2 + * cuPythonInit() + * if __cuMemcpyAtoA_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoA_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3847 + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoA_v2" not found') + * err = ( __cuMemcpyAtoA_v2)(dstArray, dstOffset, srcArray, srcOffset, ByteCount) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarray, size_t, CUarray, size_t, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoA_v2)(__pyx_v_dstArray, __pyx_v_dstOffset, __pyx_v_srcArray, __pyx_v_srcOffset, __pyx_v_ByteCount); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3847, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3848 + * raise RuntimeError('Function "cuMemcpyAtoA_v2" not found') + * err = ( __cuMemcpyAtoA_v2)(dstArray, dstOffset, srcArray, srcOffset, ByteCount) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpy2D_v2(const CUDA_MEMCPY2D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3841 + * return err + * + * cdef CUresult _cuMemcpyAtoA_v2(CUarray dstArray, size_t dstOffset, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyAtoA_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyAtoA_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3850 + * return err + * + * cdef CUresult _cuMemcpy2D_v2(const CUDA_MEMCPY2D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy2D_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy2D_v2(CUDA_MEMCPY2D const *__pyx_v_pCopy) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpy2D_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3852 + * cdef CUresult _cuMemcpy2D_v2(const CUDA_MEMCPY2D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpy2D_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpy2D_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3852, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3853 + * global __cuMemcpy2D_v2 + * cuPythonInit() + * if __cuMemcpy2D_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy2D_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2D_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3854 + * cuPythonInit() + * if __cuMemcpy2D_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy2D_v2" not found') + * err = ( __cuMemcpy2D_v2)(pCopy) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3855 + * if __cuMemcpy2D_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpy2D_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpy2D_v2)(pCopy) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpy2D_v2_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3855, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3855, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3854 + * cuPythonInit() + * if __cuMemcpy2D_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy2D_v2" not found') + * err = ( __cuMemcpy2D_v2)(pCopy) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3853 + * global __cuMemcpy2D_v2 + * cuPythonInit() + * if __cuMemcpy2D_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy2D_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3856 + * with gil: + * raise RuntimeError('Function "cuMemcpy2D_v2" not found') + * err = ( __cuMemcpy2D_v2)(pCopy) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_MEMCPY2D const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2D_v2)(__pyx_v_pCopy); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3856, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3857 + * raise RuntimeError('Function "cuMemcpy2D_v2" not found') + * err = ( __cuMemcpy2D_v2)(pCopy) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpy2DUnaligned_v2(const CUDA_MEMCPY2D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3850 + * return err + * + * cdef CUresult _cuMemcpy2D_v2(const CUDA_MEMCPY2D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy2D_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpy2D_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3859 + * return err + * + * cdef CUresult _cuMemcpy2DUnaligned_v2(const CUDA_MEMCPY2D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy2DUnaligned_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy2DUnaligned_v2(CUDA_MEMCPY2D const *__pyx_v_pCopy) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpy2DUnaligned_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3861 + * cdef CUresult _cuMemcpy2DUnaligned_v2(const CUDA_MEMCPY2D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpy2DUnaligned_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpy2DUnaligned_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3861, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3862 + * global __cuMemcpy2DUnaligned_v2 + * cuPythonInit() + * if __cuMemcpy2DUnaligned_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy2DUnaligned_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DUnaligned_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3863 + * cuPythonInit() + * if __cuMemcpy2DUnaligned_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy2DUnaligned_v2" not found') + * err = ( __cuMemcpy2DUnaligned_v2)(pCopy) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3864 + * if __cuMemcpy2DUnaligned_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpy2DUnaligned_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpy2DUnaligned_v2)(pCopy) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpy2DUnaligned_v2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3864, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3864, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3863 + * cuPythonInit() + * if __cuMemcpy2DUnaligned_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy2DUnaligned_v2" not found') + * err = ( __cuMemcpy2DUnaligned_v2)(pCopy) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3862 + * global __cuMemcpy2DUnaligned_v2 + * cuPythonInit() + * if __cuMemcpy2DUnaligned_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy2DUnaligned_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3865 + * with gil: + * raise RuntimeError('Function "cuMemcpy2DUnaligned_v2" not found') + * err = ( __cuMemcpy2DUnaligned_v2)(pCopy) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_MEMCPY2D const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DUnaligned_v2)(__pyx_v_pCopy); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3865, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3866 + * raise RuntimeError('Function "cuMemcpy2DUnaligned_v2" not found') + * err = ( __cuMemcpy2DUnaligned_v2)(pCopy) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpy3D_v2(const CUDA_MEMCPY3D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3859 + * return err + * + * cdef CUresult _cuMemcpy2DUnaligned_v2(const CUDA_MEMCPY2D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy2DUnaligned_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpy2DUnaligned_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3868 + * return err + * + * cdef CUresult _cuMemcpy3D_v2(const CUDA_MEMCPY3D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy3D_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy3D_v2(CUDA_MEMCPY3D const *__pyx_v_pCopy) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpy3D_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3870 + * cdef CUresult _cuMemcpy3D_v2(const CUDA_MEMCPY3D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpy3D_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpy3D_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3870, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3871 + * global __cuMemcpy3D_v2 + * cuPythonInit() + * if __cuMemcpy3D_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy3D_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3D_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3872 + * cuPythonInit() + * if __cuMemcpy3D_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy3D_v2" not found') + * err = ( __cuMemcpy3D_v2)(pCopy) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3873 + * if __cuMemcpy3D_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpy3D_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpy3D_v2)(pCopy) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpy3D_v2_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3873, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3873, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3872 + * cuPythonInit() + * if __cuMemcpy3D_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy3D_v2" not found') + * err = ( __cuMemcpy3D_v2)(pCopy) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3871 + * global __cuMemcpy3D_v2 + * cuPythonInit() + * if __cuMemcpy3D_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy3D_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3874 + * with gil: + * raise RuntimeError('Function "cuMemcpy3D_v2" not found') + * err = ( __cuMemcpy3D_v2)(pCopy) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_MEMCPY3D const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3D_v2)(__pyx_v_pCopy); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3874, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3875 + * raise RuntimeError('Function "cuMemcpy3D_v2" not found') + * err = ( __cuMemcpy3D_v2)(pCopy) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpy3DPeer(const CUDA_MEMCPY3D_PEER* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3868 + * return err + * + * cdef CUresult _cuMemcpy3D_v2(const CUDA_MEMCPY3D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy3D_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpy3D_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3877 + * return err + * + * cdef CUresult _cuMemcpy3DPeer(const CUDA_MEMCPY3D_PEER* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeer + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy3DPeer(CUDA_MEMCPY3D_PEER const *__pyx_v_pCopy) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpy3DPeer", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3879 + * cdef CUresult _cuMemcpy3DPeer(const CUDA_MEMCPY3D_PEER* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpy3DPeer + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpy3DPeer == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3879, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3880 + * global __cuMemcpy3DPeer + * cuPythonInit() + * if __cuMemcpy3DPeer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy3DPeer" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeer == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3881 + * cuPythonInit() + * if __cuMemcpy3DPeer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy3DPeer" not found') + * err = ( __cuMemcpy3DPeer)(pCopy) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3882 + * if __cuMemcpy3DPeer == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpy3DPeer" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpy3DPeer)(pCopy) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpy3DPeer_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3882, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3882, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3881 + * cuPythonInit() + * if __cuMemcpy3DPeer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy3DPeer" not found') + * err = ( __cuMemcpy3DPeer)(pCopy) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3880 + * global __cuMemcpy3DPeer + * cuPythonInit() + * if __cuMemcpy3DPeer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy3DPeer" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3883 + * with gil: + * raise RuntimeError('Function "cuMemcpy3DPeer" not found') + * err = ( __cuMemcpy3DPeer)(pCopy) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_MEMCPY3D_PEER const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeer)(__pyx_v_pCopy); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3883, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3884 + * raise RuntimeError('Function "cuMemcpy3DPeer" not found') + * err = ( __cuMemcpy3DPeer)(pCopy) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyAsync(CUdeviceptr dst, CUdeviceptr src, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3877 + * return err + * + * cdef CUresult _cuMemcpy3DPeer(const CUDA_MEMCPY3D_PEER* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeer + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpy3DPeer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3886 + * return err + * + * cdef CUresult _cuMemcpyAsync(CUdeviceptr dst, CUdeviceptr src, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyAsync(CUdeviceptr __pyx_v_dst, CUdeviceptr __pyx_v_src, size_t __pyx_v_ByteCount, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3888 + * cdef CUresult _cuMemcpyAsync(CUdeviceptr dst, CUdeviceptr src, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3888, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3889 + * global __cuMemcpyAsync + * cuPythonInit() + * if __cuMemcpyAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3890 + * cuPythonInit() + * if __cuMemcpyAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyAsync" not found') + * err = ( __cuMemcpyAsync)(dst, src, ByteCount, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3891 + * if __cuMemcpyAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyAsync)(dst, src, ByteCount, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyAsync_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3891, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3891, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3890 + * cuPythonInit() + * if __cuMemcpyAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyAsync" not found') + * err = ( __cuMemcpyAsync)(dst, src, ByteCount, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3889 + * global __cuMemcpyAsync + * cuPythonInit() + * if __cuMemcpyAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3892 + * with gil: + * raise RuntimeError('Function "cuMemcpyAsync" not found') + * err = ( __cuMemcpyAsync)(dst, src, ByteCount, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, CUdeviceptr, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAsync)(__pyx_v_dst, __pyx_v_src, __pyx_v_ByteCount, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3892, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3893 + * raise RuntimeError('Function "cuMemcpyAsync" not found') + * err = ( __cuMemcpyAsync)(dst, src, ByteCount, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyPeerAsync(CUdeviceptr dstDevice, CUcontext dstContext, CUdeviceptr srcDevice, CUcontext srcContext, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3886 + * return err + * + * cdef CUresult _cuMemcpyAsync(CUdeviceptr dst, CUdeviceptr src, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3895 + * return err + * + * cdef CUresult _cuMemcpyPeerAsync(CUdeviceptr dstDevice, CUcontext dstContext, CUdeviceptr srcDevice, CUcontext srcContext, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyPeerAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyPeerAsync(CUdeviceptr __pyx_v_dstDevice, CUcontext __pyx_v_dstContext, CUdeviceptr __pyx_v_srcDevice, CUcontext __pyx_v_srcContext, size_t __pyx_v_ByteCount, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyPeerAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3897 + * cdef CUresult _cuMemcpyPeerAsync(CUdeviceptr dstDevice, CUcontext dstContext, CUdeviceptr srcDevice, CUcontext srcContext, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyPeerAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyPeerAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3897, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3898 + * global __cuMemcpyPeerAsync + * cuPythonInit() + * if __cuMemcpyPeerAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyPeerAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeerAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3899 + * cuPythonInit() + * if __cuMemcpyPeerAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyPeerAsync" not found') + * err = ( __cuMemcpyPeerAsync)(dstDevice, dstContext, srcDevice, srcContext, ByteCount, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3900 + * if __cuMemcpyPeerAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyPeerAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyPeerAsync)(dstDevice, dstContext, srcDevice, srcContext, ByteCount, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyPeerAsync_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3900, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3900, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3899 + * cuPythonInit() + * if __cuMemcpyPeerAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyPeerAsync" not found') + * err = ( __cuMemcpyPeerAsync)(dstDevice, dstContext, srcDevice, srcContext, ByteCount, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3898 + * global __cuMemcpyPeerAsync + * cuPythonInit() + * if __cuMemcpyPeerAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyPeerAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3901 + * with gil: + * raise RuntimeError('Function "cuMemcpyPeerAsync" not found') + * err = ( __cuMemcpyPeerAsync)(dstDevice, dstContext, srcDevice, srcContext, ByteCount, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, CUcontext, CUdeviceptr, CUcontext, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeerAsync)(__pyx_v_dstDevice, __pyx_v_dstContext, __pyx_v_srcDevice, __pyx_v_srcContext, __pyx_v_ByteCount, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3901, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3902 + * raise RuntimeError('Function "cuMemcpyPeerAsync" not found') + * err = ( __cuMemcpyPeerAsync)(dstDevice, dstContext, srcDevice, srcContext, ByteCount, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyHtoDAsync_v2(CUdeviceptr dstDevice, const void* srcHost, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3895 + * return err + * + * cdef CUresult _cuMemcpyPeerAsync(CUdeviceptr dstDevice, CUcontext dstContext, CUdeviceptr srcDevice, CUcontext srcContext, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyPeerAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyPeerAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3904 + * return err + * + * cdef CUresult _cuMemcpyHtoDAsync_v2(CUdeviceptr dstDevice, const void* srcHost, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyHtoDAsync_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyHtoDAsync_v2(CUdeviceptr __pyx_v_dstDevice, void const *__pyx_v_srcHost, size_t __pyx_v_ByteCount, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyHtoDAsync_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3906 + * cdef CUresult _cuMemcpyHtoDAsync_v2(CUdeviceptr dstDevice, const void* srcHost, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyHtoDAsync_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyHtoDAsync_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3906, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3907 + * global __cuMemcpyHtoDAsync_v2 + * cuPythonInit() + * if __cuMemcpyHtoDAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoDAsync_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoDAsync_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3908 + * cuPythonInit() + * if __cuMemcpyHtoDAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyHtoDAsync_v2" not found') + * err = ( __cuMemcpyHtoDAsync_v2)(dstDevice, srcHost, ByteCount, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3909 + * if __cuMemcpyHtoDAsync_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoDAsync_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyHtoDAsync_v2)(dstDevice, srcHost, ByteCount, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyHtoDAsync_v2_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3909, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3909, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3908 + * cuPythonInit() + * if __cuMemcpyHtoDAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyHtoDAsync_v2" not found') + * err = ( __cuMemcpyHtoDAsync_v2)(dstDevice, srcHost, ByteCount, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3907 + * global __cuMemcpyHtoDAsync_v2 + * cuPythonInit() + * if __cuMemcpyHtoDAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoDAsync_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3910 + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoDAsync_v2" not found') + * err = ( __cuMemcpyHtoDAsync_v2)(dstDevice, srcHost, ByteCount, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, void const *, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoDAsync_v2)(__pyx_v_dstDevice, __pyx_v_srcHost, __pyx_v_ByteCount, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3910, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3911 + * raise RuntimeError('Function "cuMemcpyHtoDAsync_v2" not found') + * err = ( __cuMemcpyHtoDAsync_v2)(dstDevice, srcHost, ByteCount, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyDtoHAsync_v2(void* dstHost, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3904 + * return err + * + * cdef CUresult _cuMemcpyHtoDAsync_v2(CUdeviceptr dstDevice, const void* srcHost, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyHtoDAsync_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyHtoDAsync_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3913 + * return err + * + * cdef CUresult _cuMemcpyDtoHAsync_v2(void* dstHost, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyDtoHAsync_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyDtoHAsync_v2(void *__pyx_v_dstHost, CUdeviceptr __pyx_v_srcDevice, size_t __pyx_v_ByteCount, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyDtoHAsync_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3915 + * cdef CUresult _cuMemcpyDtoHAsync_v2(void* dstHost, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyDtoHAsync_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyDtoHAsync_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3915, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3916 + * global __cuMemcpyDtoHAsync_v2 + * cuPythonInit() + * if __cuMemcpyDtoHAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoHAsync_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoHAsync_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3917 + * cuPythonInit() + * if __cuMemcpyDtoHAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyDtoHAsync_v2" not found') + * err = ( __cuMemcpyDtoHAsync_v2)(dstHost, srcDevice, ByteCount, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3918 + * if __cuMemcpyDtoHAsync_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoHAsync_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyDtoHAsync_v2)(dstHost, srcDevice, ByteCount, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyDtoHAsync_v2_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3918, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3918, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3917 + * cuPythonInit() + * if __cuMemcpyDtoHAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyDtoHAsync_v2" not found') + * err = ( __cuMemcpyDtoHAsync_v2)(dstHost, srcDevice, ByteCount, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3916 + * global __cuMemcpyDtoHAsync_v2 + * cuPythonInit() + * if __cuMemcpyDtoHAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoHAsync_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3919 + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoHAsync_v2" not found') + * err = ( __cuMemcpyDtoHAsync_v2)(dstHost, srcDevice, ByteCount, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void *, CUdeviceptr, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoHAsync_v2)(__pyx_v_dstHost, __pyx_v_srcDevice, __pyx_v_ByteCount, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3919, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3920 + * raise RuntimeError('Function "cuMemcpyDtoHAsync_v2" not found') + * err = ( __cuMemcpyDtoHAsync_v2)(dstHost, srcDevice, ByteCount, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyDtoDAsync_v2(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3913 + * return err + * + * cdef CUresult _cuMemcpyDtoHAsync_v2(void* dstHost, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyDtoHAsync_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyDtoHAsync_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3922 + * return err + * + * cdef CUresult _cuMemcpyDtoDAsync_v2(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyDtoDAsync_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyDtoDAsync_v2(CUdeviceptr __pyx_v_dstDevice, CUdeviceptr __pyx_v_srcDevice, size_t __pyx_v_ByteCount, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyDtoDAsync_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3924 + * cdef CUresult _cuMemcpyDtoDAsync_v2(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyDtoDAsync_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyDtoDAsync_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3924, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3925 + * global __cuMemcpyDtoDAsync_v2 + * cuPythonInit() + * if __cuMemcpyDtoDAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoDAsync_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoDAsync_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3926 + * cuPythonInit() + * if __cuMemcpyDtoDAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyDtoDAsync_v2" not found') + * err = ( __cuMemcpyDtoDAsync_v2)(dstDevice, srcDevice, ByteCount, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3927 + * if __cuMemcpyDtoDAsync_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoDAsync_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyDtoDAsync_v2)(dstDevice, srcDevice, ByteCount, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyDtoDAsync_v2_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3927, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3927, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3926 + * cuPythonInit() + * if __cuMemcpyDtoDAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyDtoDAsync_v2" not found') + * err = ( __cuMemcpyDtoDAsync_v2)(dstDevice, srcDevice, ByteCount, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3925 + * global __cuMemcpyDtoDAsync_v2 + * cuPythonInit() + * if __cuMemcpyDtoDAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoDAsync_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3928 + * with gil: + * raise RuntimeError('Function "cuMemcpyDtoDAsync_v2" not found') + * err = ( __cuMemcpyDtoDAsync_v2)(dstDevice, srcDevice, ByteCount, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, CUdeviceptr, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoDAsync_v2)(__pyx_v_dstDevice, __pyx_v_srcDevice, __pyx_v_ByteCount, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3928, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3929 + * raise RuntimeError('Function "cuMemcpyDtoDAsync_v2" not found') + * err = ( __cuMemcpyDtoDAsync_v2)(dstDevice, srcDevice, ByteCount, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyHtoAAsync_v2(CUarray dstArray, size_t dstOffset, const void* srcHost, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3922 + * return err + * + * cdef CUresult _cuMemcpyDtoDAsync_v2(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyDtoDAsync_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyDtoDAsync_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3931 + * return err + * + * cdef CUresult _cuMemcpyHtoAAsync_v2(CUarray dstArray, size_t dstOffset, const void* srcHost, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyHtoAAsync_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyHtoAAsync_v2(CUarray __pyx_v_dstArray, size_t __pyx_v_dstOffset, void const *__pyx_v_srcHost, size_t __pyx_v_ByteCount, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyHtoAAsync_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3933 + * cdef CUresult _cuMemcpyHtoAAsync_v2(CUarray dstArray, size_t dstOffset, const void* srcHost, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyHtoAAsync_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyHtoAAsync_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3933, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3934 + * global __cuMemcpyHtoAAsync_v2 + * cuPythonInit() + * if __cuMemcpyHtoAAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoAAsync_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoAAsync_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3935 + * cuPythonInit() + * if __cuMemcpyHtoAAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyHtoAAsync_v2" not found') + * err = ( __cuMemcpyHtoAAsync_v2)(dstArray, dstOffset, srcHost, ByteCount, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3936 + * if __cuMemcpyHtoAAsync_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoAAsync_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyHtoAAsync_v2)(dstArray, dstOffset, srcHost, ByteCount, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyHtoAAsync_v2_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3936, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3936, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3935 + * cuPythonInit() + * if __cuMemcpyHtoAAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyHtoAAsync_v2" not found') + * err = ( __cuMemcpyHtoAAsync_v2)(dstArray, dstOffset, srcHost, ByteCount, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3934 + * global __cuMemcpyHtoAAsync_v2 + * cuPythonInit() + * if __cuMemcpyHtoAAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoAAsync_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3937 + * with gil: + * raise RuntimeError('Function "cuMemcpyHtoAAsync_v2" not found') + * err = ( __cuMemcpyHtoAAsync_v2)(dstArray, dstOffset, srcHost, ByteCount, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarray, size_t, void const *, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoAAsync_v2)(__pyx_v_dstArray, __pyx_v_dstOffset, __pyx_v_srcHost, __pyx_v_ByteCount, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3937, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3938 + * raise RuntimeError('Function "cuMemcpyHtoAAsync_v2" not found') + * err = ( __cuMemcpyHtoAAsync_v2)(dstArray, dstOffset, srcHost, ByteCount, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyAtoHAsync_v2(void* dstHost, CUarray srcArray, size_t srcOffset, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3931 + * return err + * + * cdef CUresult _cuMemcpyHtoAAsync_v2(CUarray dstArray, size_t dstOffset, const void* srcHost, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyHtoAAsync_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyHtoAAsync_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3940 + * return err + * + * cdef CUresult _cuMemcpyAtoHAsync_v2(void* dstHost, CUarray srcArray, size_t srcOffset, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyAtoHAsync_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyAtoHAsync_v2(void *__pyx_v_dstHost, CUarray __pyx_v_srcArray, size_t __pyx_v_srcOffset, size_t __pyx_v_ByteCount, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyAtoHAsync_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3942 + * cdef CUresult _cuMemcpyAtoHAsync_v2(void* dstHost, CUarray srcArray, size_t srcOffset, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyAtoHAsync_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyAtoHAsync_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3942, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3943 + * global __cuMemcpyAtoHAsync_v2 + * cuPythonInit() + * if __cuMemcpyAtoHAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoHAsync_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoHAsync_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3944 + * cuPythonInit() + * if __cuMemcpyAtoHAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyAtoHAsync_v2" not found') + * err = ( __cuMemcpyAtoHAsync_v2)(dstHost, srcArray, srcOffset, ByteCount, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3945 + * if __cuMemcpyAtoHAsync_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoHAsync_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyAtoHAsync_v2)(dstHost, srcArray, srcOffset, ByteCount, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyAtoHAsync_v2_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3945, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3945, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3944 + * cuPythonInit() + * if __cuMemcpyAtoHAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyAtoHAsync_v2" not found') + * err = ( __cuMemcpyAtoHAsync_v2)(dstHost, srcArray, srcOffset, ByteCount, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3943 + * global __cuMemcpyAtoHAsync_v2 + * cuPythonInit() + * if __cuMemcpyAtoHAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoHAsync_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3946 + * with gil: + * raise RuntimeError('Function "cuMemcpyAtoHAsync_v2" not found') + * err = ( __cuMemcpyAtoHAsync_v2)(dstHost, srcArray, srcOffset, ByteCount, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void *, CUarray, size_t, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoHAsync_v2)(__pyx_v_dstHost, __pyx_v_srcArray, __pyx_v_srcOffset, __pyx_v_ByteCount, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3946, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3947 + * raise RuntimeError('Function "cuMemcpyAtoHAsync_v2" not found') + * err = ( __cuMemcpyAtoHAsync_v2)(dstHost, srcArray, srcOffset, ByteCount, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpy2DAsync_v2(const CUDA_MEMCPY2D* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3940 + * return err + * + * cdef CUresult _cuMemcpyAtoHAsync_v2(void* dstHost, CUarray srcArray, size_t srcOffset, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyAtoHAsync_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyAtoHAsync_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3949 + * return err + * + * cdef CUresult _cuMemcpy2DAsync_v2(const CUDA_MEMCPY2D* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy2DAsync_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy2DAsync_v2(CUDA_MEMCPY2D const *__pyx_v_pCopy, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpy2DAsync_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3951 + * cdef CUresult _cuMemcpy2DAsync_v2(const CUDA_MEMCPY2D* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpy2DAsync_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpy2DAsync_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3951, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3952 + * global __cuMemcpy2DAsync_v2 + * cuPythonInit() + * if __cuMemcpy2DAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy2DAsync_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DAsync_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3953 + * cuPythonInit() + * if __cuMemcpy2DAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy2DAsync_v2" not found') + * err = ( __cuMemcpy2DAsync_v2)(pCopy, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3954 + * if __cuMemcpy2DAsync_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpy2DAsync_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpy2DAsync_v2)(pCopy, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpy2DAsync_v2_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3954, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3954, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3953 + * cuPythonInit() + * if __cuMemcpy2DAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy2DAsync_v2" not found') + * err = ( __cuMemcpy2DAsync_v2)(pCopy, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3952 + * global __cuMemcpy2DAsync_v2 + * cuPythonInit() + * if __cuMemcpy2DAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy2DAsync_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3955 + * with gil: + * raise RuntimeError('Function "cuMemcpy2DAsync_v2" not found') + * err = ( __cuMemcpy2DAsync_v2)(pCopy, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_MEMCPY2D const *, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DAsync_v2)(__pyx_v_pCopy, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3955, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3956 + * raise RuntimeError('Function "cuMemcpy2DAsync_v2" not found') + * err = ( __cuMemcpy2DAsync_v2)(pCopy, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpy3DAsync_v2(const CUDA_MEMCPY3D* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3949 + * return err + * + * cdef CUresult _cuMemcpy2DAsync_v2(const CUDA_MEMCPY2D* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy2DAsync_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpy2DAsync_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3958 + * return err + * + * cdef CUresult _cuMemcpy3DAsync_v2(const CUDA_MEMCPY3D* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy3DAsync_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy3DAsync_v2(CUDA_MEMCPY3D const *__pyx_v_pCopy, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpy3DAsync_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3960 + * cdef CUresult _cuMemcpy3DAsync_v2(const CUDA_MEMCPY3D* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpy3DAsync_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpy3DAsync_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3960, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3961 + * global __cuMemcpy3DAsync_v2 + * cuPythonInit() + * if __cuMemcpy3DAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy3DAsync_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DAsync_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3962 + * cuPythonInit() + * if __cuMemcpy3DAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy3DAsync_v2" not found') + * err = ( __cuMemcpy3DAsync_v2)(pCopy, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3963 + * if __cuMemcpy3DAsync_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpy3DAsync_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpy3DAsync_v2)(pCopy, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpy3DAsync_v2_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3963, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3963, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3962 + * cuPythonInit() + * if __cuMemcpy3DAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy3DAsync_v2" not found') + * err = ( __cuMemcpy3DAsync_v2)(pCopy, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3961 + * global __cuMemcpy3DAsync_v2 + * cuPythonInit() + * if __cuMemcpy3DAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy3DAsync_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3964 + * with gil: + * raise RuntimeError('Function "cuMemcpy3DAsync_v2" not found') + * err = ( __cuMemcpy3DAsync_v2)(pCopy, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_MEMCPY3D const *, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DAsync_v2)(__pyx_v_pCopy, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3964, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3965 + * raise RuntimeError('Function "cuMemcpy3DAsync_v2" not found') + * err = ( __cuMemcpy3DAsync_v2)(pCopy, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpy3DPeerAsync(const CUDA_MEMCPY3D_PEER* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3958 + * return err + * + * cdef CUresult _cuMemcpy3DAsync_v2(const CUDA_MEMCPY3D* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy3DAsync_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpy3DAsync_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3967 + * return err + * + * cdef CUresult _cuMemcpy3DPeerAsync(const CUDA_MEMCPY3D_PEER* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeerAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy3DPeerAsync(CUDA_MEMCPY3D_PEER const *__pyx_v_pCopy, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpy3DPeerAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3969 + * cdef CUresult _cuMemcpy3DPeerAsync(const CUDA_MEMCPY3D_PEER* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpy3DPeerAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpy3DPeerAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3969, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3970 + * global __cuMemcpy3DPeerAsync + * cuPythonInit() + * if __cuMemcpy3DPeerAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy3DPeerAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeerAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3971 + * cuPythonInit() + * if __cuMemcpy3DPeerAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy3DPeerAsync" not found') + * err = ( __cuMemcpy3DPeerAsync)(pCopy, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3972 + * if __cuMemcpy3DPeerAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpy3DPeerAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpy3DPeerAsync)(pCopy, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpy3DPeerAsync_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3972, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3972, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3971 + * cuPythonInit() + * if __cuMemcpy3DPeerAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy3DPeerAsync" not found') + * err = ( __cuMemcpy3DPeerAsync)(pCopy, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3970 + * global __cuMemcpy3DPeerAsync + * cuPythonInit() + * if __cuMemcpy3DPeerAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy3DPeerAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3973 + * with gil: + * raise RuntimeError('Function "cuMemcpy3DPeerAsync" not found') + * err = ( __cuMemcpy3DPeerAsync)(pCopy, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_MEMCPY3D_PEER const *, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeerAsync)(__pyx_v_pCopy, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3973, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3974 + * raise RuntimeError('Function "cuMemcpy3DPeerAsync" not found') + * err = ( __cuMemcpy3DPeerAsync)(pCopy, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpyBatchAsync(CUdeviceptr* dsts, CUdeviceptr* srcs, size_t* sizes, size_t count, CUmemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3967 + * return err + * + * cdef CUresult _cuMemcpy3DPeerAsync(const CUDA_MEMCPY3D_PEER* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeerAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpy3DPeerAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3976 + * return err + * + * cdef CUresult _cuMemcpyBatchAsync(CUdeviceptr* dsts, CUdeviceptr* srcs, size_t* sizes, size_t count, CUmemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyBatchAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyBatchAsync(CUdeviceptr *__pyx_v_dsts, CUdeviceptr *__pyx_v_srcs, size_t *__pyx_v_sizes, size_t __pyx_v_count, CUmemcpyAttributes *__pyx_v_attrs, size_t *__pyx_v_attrsIdxs, size_t __pyx_v_numAttrs, size_t *__pyx_v_failIdx, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpyBatchAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3978 + * cdef CUresult _cuMemcpyBatchAsync(CUdeviceptr* dsts, CUdeviceptr* srcs, size_t* sizes, size_t count, CUmemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpyBatchAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpyBatchAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3978, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3979 + * global __cuMemcpyBatchAsync + * cuPythonInit() + * if __cuMemcpyBatchAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyBatchAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyBatchAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3980 + * cuPythonInit() + * if __cuMemcpyBatchAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyBatchAsync" not found') + * err = ( __cuMemcpyBatchAsync)(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3981 + * if __cuMemcpyBatchAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpyBatchAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpyBatchAsync)(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpyBatchAsync_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3981, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3981, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3980 + * cuPythonInit() + * if __cuMemcpyBatchAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpyBatchAsync" not found') + * err = ( __cuMemcpyBatchAsync)(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3979 + * global __cuMemcpyBatchAsync + * cuPythonInit() + * if __cuMemcpyBatchAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpyBatchAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3982 + * with gil: + * raise RuntimeError('Function "cuMemcpyBatchAsync" not found') + * err = ( __cuMemcpyBatchAsync)(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, CUdeviceptr *, size_t *, size_t, CUmemcpyAttributes *, size_t *, size_t, size_t *, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyBatchAsync)(__pyx_v_dsts, __pyx_v_srcs, __pyx_v_sizes, __pyx_v_count, __pyx_v_attrs, __pyx_v_attrsIdxs, __pyx_v_numAttrs, __pyx_v_failIdx, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3982, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3983 + * raise RuntimeError('Function "cuMemcpyBatchAsync" not found') + * err = ( __cuMemcpyBatchAsync)(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemcpy3DBatchAsync(size_t numOps, CUDA_MEMCPY3D_BATCH_OP* opList, size_t* failIdx, unsigned long long flags, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3976 + * return err + * + * cdef CUresult _cuMemcpyBatchAsync(CUdeviceptr* dsts, CUdeviceptr* srcs, size_t* sizes, size_t count, CUmemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpyBatchAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpyBatchAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3985 + * return err + * + * cdef CUresult _cuMemcpy3DBatchAsync(size_t numOps, CUDA_MEMCPY3D_BATCH_OP* opList, size_t* failIdx, unsigned long long flags, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy3DBatchAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy3DBatchAsync(size_t __pyx_v_numOps, CUDA_MEMCPY3D_BATCH_OP *__pyx_v_opList, size_t *__pyx_v_failIdx, unsigned PY_LONG_LONG __pyx_v_flags, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemcpy3DBatchAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3987 + * cdef CUresult _cuMemcpy3DBatchAsync(size_t numOps, CUDA_MEMCPY3D_BATCH_OP* opList, size_t* failIdx, unsigned long long flags, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemcpy3DBatchAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemcpy3DBatchAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3987, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3988 + * global __cuMemcpy3DBatchAsync + * cuPythonInit() + * if __cuMemcpy3DBatchAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy3DBatchAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DBatchAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3989 + * cuPythonInit() + * if __cuMemcpy3DBatchAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy3DBatchAsync" not found') + * err = ( __cuMemcpy3DBatchAsync)(numOps, opList, failIdx, flags, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3990 + * if __cuMemcpy3DBatchAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuMemcpy3DBatchAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemcpy3DBatchAsync)(numOps, opList, failIdx, flags, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemcpy3DBatchAsync_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3990, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3990, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3989 + * cuPythonInit() + * if __cuMemcpy3DBatchAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemcpy3DBatchAsync" not found') + * err = ( __cuMemcpy3DBatchAsync)(numOps, opList, failIdx, flags, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3988 + * global __cuMemcpy3DBatchAsync + * cuPythonInit() + * if __cuMemcpy3DBatchAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemcpy3DBatchAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3991 + * with gil: + * raise RuntimeError('Function "cuMemcpy3DBatchAsync" not found') + * err = ( __cuMemcpy3DBatchAsync)(numOps, opList, failIdx, flags, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(size_t, CUDA_MEMCPY3D_BATCH_OP *, size_t *, unsigned PY_LONG_LONG, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DBatchAsync)(__pyx_v_numOps, __pyx_v_opList, __pyx_v_failIdx, __pyx_v_flags, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 3991, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":3992 + * raise RuntimeError('Function "cuMemcpy3DBatchAsync" not found') + * err = ( __cuMemcpy3DBatchAsync)(numOps, opList, failIdx, flags, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemsetD8_v2(CUdeviceptr dstDevice, unsigned char uc, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3985 + * return err + * + * cdef CUresult _cuMemcpy3DBatchAsync(size_t numOps, CUDA_MEMCPY3D_BATCH_OP* opList, size_t* failIdx, unsigned long long flags, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemcpy3DBatchAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemcpy3DBatchAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":3994 + * return err + * + * cdef CUresult _cuMemsetD8_v2(CUdeviceptr dstDevice, unsigned char uc, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD8_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD8_v2(CUdeviceptr __pyx_v_dstDevice, unsigned char __pyx_v_uc, size_t __pyx_v_N) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemsetD8_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":3996 + * cdef CUresult _cuMemsetD8_v2(CUdeviceptr dstDevice, unsigned char uc, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemsetD8_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemsetD8_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 3996, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":3997 + * global __cuMemsetD8_v2 + * cuPythonInit() + * if __cuMemsetD8_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD8_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":3998 + * cuPythonInit() + * if __cuMemsetD8_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD8_v2" not found') + * err = ( __cuMemsetD8_v2)(dstDevice, uc, N) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":3999 + * if __cuMemsetD8_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemsetD8_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemsetD8_v2)(dstDevice, uc, N) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemsetD8_v2_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 3999, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 3999, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3998 + * cuPythonInit() + * if __cuMemsetD8_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD8_v2" not found') + * err = ( __cuMemsetD8_v2)(dstDevice, uc, N) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":3997 + * global __cuMemsetD8_v2 + * cuPythonInit() + * if __cuMemsetD8_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD8_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4000 + * with gil: + * raise RuntimeError('Function "cuMemsetD8_v2" not found') + * err = ( __cuMemsetD8_v2)(dstDevice, uc, N) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, unsigned char, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8_v2)(__pyx_v_dstDevice, __pyx_v_uc, __pyx_v_N); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4000, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4001 + * raise RuntimeError('Function "cuMemsetD8_v2" not found') + * err = ( __cuMemsetD8_v2)(dstDevice, uc, N) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemsetD16_v2(CUdeviceptr dstDevice, unsigned short us, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":3994 + * return err + * + * cdef CUresult _cuMemsetD8_v2(CUdeviceptr dstDevice, unsigned char uc, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD8_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemsetD8_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4003 + * return err + * + * cdef CUresult _cuMemsetD16_v2(CUdeviceptr dstDevice, unsigned short us, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD16_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD16_v2(CUdeviceptr __pyx_v_dstDevice, unsigned short __pyx_v_us, size_t __pyx_v_N) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemsetD16_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4005 + * cdef CUresult _cuMemsetD16_v2(CUdeviceptr dstDevice, unsigned short us, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemsetD16_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemsetD16_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4005, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4006 + * global __cuMemsetD16_v2 + * cuPythonInit() + * if __cuMemsetD16_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD16_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4007 + * cuPythonInit() + * if __cuMemsetD16_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD16_v2" not found') + * err = ( __cuMemsetD16_v2)(dstDevice, us, N) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4008 + * if __cuMemsetD16_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemsetD16_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemsetD16_v2)(dstDevice, us, N) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemsetD16_v2_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4008, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4008, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4007 + * cuPythonInit() + * if __cuMemsetD16_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD16_v2" not found') + * err = ( __cuMemsetD16_v2)(dstDevice, us, N) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4006 + * global __cuMemsetD16_v2 + * cuPythonInit() + * if __cuMemsetD16_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD16_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4009 + * with gil: + * raise RuntimeError('Function "cuMemsetD16_v2" not found') + * err = ( __cuMemsetD16_v2)(dstDevice, us, N) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, unsigned short, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16_v2)(__pyx_v_dstDevice, __pyx_v_us, __pyx_v_N); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4009, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4010 + * raise RuntimeError('Function "cuMemsetD16_v2" not found') + * err = ( __cuMemsetD16_v2)(dstDevice, us, N) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemsetD32_v2(CUdeviceptr dstDevice, unsigned int ui, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4003 + * return err + * + * cdef CUresult _cuMemsetD16_v2(CUdeviceptr dstDevice, unsigned short us, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD16_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemsetD16_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4012 + * return err + * + * cdef CUresult _cuMemsetD32_v2(CUdeviceptr dstDevice, unsigned int ui, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD32_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD32_v2(CUdeviceptr __pyx_v_dstDevice, unsigned int __pyx_v_ui, size_t __pyx_v_N) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemsetD32_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4014 + * cdef CUresult _cuMemsetD32_v2(CUdeviceptr dstDevice, unsigned int ui, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemsetD32_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemsetD32_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4014, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4015 + * global __cuMemsetD32_v2 + * cuPythonInit() + * if __cuMemsetD32_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD32_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4016 + * cuPythonInit() + * if __cuMemsetD32_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD32_v2" not found') + * err = ( __cuMemsetD32_v2)(dstDevice, ui, N) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4017 + * if __cuMemsetD32_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemsetD32_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemsetD32_v2)(dstDevice, ui, N) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemsetD32_v2_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4017, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4017, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4016 + * cuPythonInit() + * if __cuMemsetD32_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD32_v2" not found') + * err = ( __cuMemsetD32_v2)(dstDevice, ui, N) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4015 + * global __cuMemsetD32_v2 + * cuPythonInit() + * if __cuMemsetD32_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD32_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4018 + * with gil: + * raise RuntimeError('Function "cuMemsetD32_v2" not found') + * err = ( __cuMemsetD32_v2)(dstDevice, ui, N) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, unsigned int, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32_v2)(__pyx_v_dstDevice, __pyx_v_ui, __pyx_v_N); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4018, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4019 + * raise RuntimeError('Function "cuMemsetD32_v2" not found') + * err = ( __cuMemsetD32_v2)(dstDevice, ui, N) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemsetD2D8_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned char uc, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4012 + * return err + * + * cdef CUresult _cuMemsetD32_v2(CUdeviceptr dstDevice, unsigned int ui, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD32_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemsetD32_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4021 + * return err + * + * cdef CUresult _cuMemsetD2D8_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned char uc, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD2D8_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD2D8_v2(CUdeviceptr __pyx_v_dstDevice, size_t __pyx_v_dstPitch, unsigned char __pyx_v_uc, size_t __pyx_v_Width, size_t __pyx_v_Height) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemsetD2D8_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4023 + * cdef CUresult _cuMemsetD2D8_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned char uc, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemsetD2D8_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemsetD2D8_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4023, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4024 + * global __cuMemsetD2D8_v2 + * cuPythonInit() + * if __cuMemsetD2D8_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD2D8_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4025 + * cuPythonInit() + * if __cuMemsetD2D8_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD2D8_v2" not found') + * err = ( __cuMemsetD2D8_v2)(dstDevice, dstPitch, uc, Width, Height) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4026 + * if __cuMemsetD2D8_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemsetD2D8_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemsetD2D8_v2)(dstDevice, dstPitch, uc, Width, Height) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemsetD2D8_v2_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4026, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4026, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4025 + * cuPythonInit() + * if __cuMemsetD2D8_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD2D8_v2" not found') + * err = ( __cuMemsetD2D8_v2)(dstDevice, dstPitch, uc, Width, Height) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4024 + * global __cuMemsetD2D8_v2 + * cuPythonInit() + * if __cuMemsetD2D8_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD2D8_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4027 + * with gil: + * raise RuntimeError('Function "cuMemsetD2D8_v2" not found') + * err = ( __cuMemsetD2D8_v2)(dstDevice, dstPitch, uc, Width, Height) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t, unsigned char, size_t, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8_v2)(__pyx_v_dstDevice, __pyx_v_dstPitch, __pyx_v_uc, __pyx_v_Width, __pyx_v_Height); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4027, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4028 + * raise RuntimeError('Function "cuMemsetD2D8_v2" not found') + * err = ( __cuMemsetD2D8_v2)(dstDevice, dstPitch, uc, Width, Height) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemsetD2D16_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned short us, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4021 + * return err + * + * cdef CUresult _cuMemsetD2D8_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned char uc, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD2D8_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemsetD2D8_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4030 + * return err + * + * cdef CUresult _cuMemsetD2D16_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned short us, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD2D16_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD2D16_v2(CUdeviceptr __pyx_v_dstDevice, size_t __pyx_v_dstPitch, unsigned short __pyx_v_us, size_t __pyx_v_Width, size_t __pyx_v_Height) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemsetD2D16_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4032 + * cdef CUresult _cuMemsetD2D16_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned short us, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemsetD2D16_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemsetD2D16_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4032, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4033 + * global __cuMemsetD2D16_v2 + * cuPythonInit() + * if __cuMemsetD2D16_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD2D16_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4034 + * cuPythonInit() + * if __cuMemsetD2D16_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD2D16_v2" not found') + * err = ( __cuMemsetD2D16_v2)(dstDevice, dstPitch, us, Width, Height) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4035 + * if __cuMemsetD2D16_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemsetD2D16_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemsetD2D16_v2)(dstDevice, dstPitch, us, Width, Height) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemsetD2D16_v2_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4035, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4035, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4034 + * cuPythonInit() + * if __cuMemsetD2D16_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD2D16_v2" not found') + * err = ( __cuMemsetD2D16_v2)(dstDevice, dstPitch, us, Width, Height) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4033 + * global __cuMemsetD2D16_v2 + * cuPythonInit() + * if __cuMemsetD2D16_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD2D16_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4036 + * with gil: + * raise RuntimeError('Function "cuMemsetD2D16_v2" not found') + * err = ( __cuMemsetD2D16_v2)(dstDevice, dstPitch, us, Width, Height) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t, unsigned short, size_t, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16_v2)(__pyx_v_dstDevice, __pyx_v_dstPitch, __pyx_v_us, __pyx_v_Width, __pyx_v_Height); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4036, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4037 + * raise RuntimeError('Function "cuMemsetD2D16_v2" not found') + * err = ( __cuMemsetD2D16_v2)(dstDevice, dstPitch, us, Width, Height) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemsetD2D32_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned int ui, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4030 + * return err + * + * cdef CUresult _cuMemsetD2D16_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned short us, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD2D16_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemsetD2D16_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4039 + * return err + * + * cdef CUresult _cuMemsetD2D32_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned int ui, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD2D32_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD2D32_v2(CUdeviceptr __pyx_v_dstDevice, size_t __pyx_v_dstPitch, unsigned int __pyx_v_ui, size_t __pyx_v_Width, size_t __pyx_v_Height) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemsetD2D32_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4041 + * cdef CUresult _cuMemsetD2D32_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned int ui, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemsetD2D32_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemsetD2D32_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4041, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4042 + * global __cuMemsetD2D32_v2 + * cuPythonInit() + * if __cuMemsetD2D32_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD2D32_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4043 + * cuPythonInit() + * if __cuMemsetD2D32_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD2D32_v2" not found') + * err = ( __cuMemsetD2D32_v2)(dstDevice, dstPitch, ui, Width, Height) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4044 + * if __cuMemsetD2D32_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemsetD2D32_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemsetD2D32_v2)(dstDevice, dstPitch, ui, Width, Height) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemsetD2D32_v2_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4044, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4044, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4043 + * cuPythonInit() + * if __cuMemsetD2D32_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD2D32_v2" not found') + * err = ( __cuMemsetD2D32_v2)(dstDevice, dstPitch, ui, Width, Height) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4042 + * global __cuMemsetD2D32_v2 + * cuPythonInit() + * if __cuMemsetD2D32_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD2D32_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4045 + * with gil: + * raise RuntimeError('Function "cuMemsetD2D32_v2" not found') + * err = ( __cuMemsetD2D32_v2)(dstDevice, dstPitch, ui, Width, Height) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t, unsigned int, size_t, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32_v2)(__pyx_v_dstDevice, __pyx_v_dstPitch, __pyx_v_ui, __pyx_v_Width, __pyx_v_Height); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4045, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4046 + * raise RuntimeError('Function "cuMemsetD2D32_v2" not found') + * err = ( __cuMemsetD2D32_v2)(dstDevice, dstPitch, ui, Width, Height) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemsetD8Async(CUdeviceptr dstDevice, unsigned char uc, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4039 + * return err + * + * cdef CUresult _cuMemsetD2D32_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned int ui, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD2D32_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemsetD2D32_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4048 + * return err + * + * cdef CUresult _cuMemsetD8Async(CUdeviceptr dstDevice, unsigned char uc, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD8Async + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD8Async(CUdeviceptr __pyx_v_dstDevice, unsigned char __pyx_v_uc, size_t __pyx_v_N, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemsetD8Async", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4050 + * cdef CUresult _cuMemsetD8Async(CUdeviceptr dstDevice, unsigned char uc, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemsetD8Async + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemsetD8Async == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4050, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4051 + * global __cuMemsetD8Async + * cuPythonInit() + * if __cuMemsetD8Async == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD8Async" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8Async == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4052 + * cuPythonInit() + * if __cuMemsetD8Async == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD8Async" not found') + * err = ( __cuMemsetD8Async)(dstDevice, uc, N, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4053 + * if __cuMemsetD8Async == NULL: + * with gil: + * raise RuntimeError('Function "cuMemsetD8Async" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemsetD8Async)(dstDevice, uc, N, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemsetD8Async_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4053, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4053, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4052 + * cuPythonInit() + * if __cuMemsetD8Async == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD8Async" not found') + * err = ( __cuMemsetD8Async)(dstDevice, uc, N, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4051 + * global __cuMemsetD8Async + * cuPythonInit() + * if __cuMemsetD8Async == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD8Async" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4054 + * with gil: + * raise RuntimeError('Function "cuMemsetD8Async" not found') + * err = ( __cuMemsetD8Async)(dstDevice, uc, N, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, unsigned char, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8Async)(__pyx_v_dstDevice, __pyx_v_uc, __pyx_v_N, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4054, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4055 + * raise RuntimeError('Function "cuMemsetD8Async" not found') + * err = ( __cuMemsetD8Async)(dstDevice, uc, N, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemsetD16Async(CUdeviceptr dstDevice, unsigned short us, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4048 + * return err + * + * cdef CUresult _cuMemsetD8Async(CUdeviceptr dstDevice, unsigned char uc, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD8Async + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemsetD8Async", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4057 + * return err + * + * cdef CUresult _cuMemsetD16Async(CUdeviceptr dstDevice, unsigned short us, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD16Async + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD16Async(CUdeviceptr __pyx_v_dstDevice, unsigned short __pyx_v_us, size_t __pyx_v_N, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemsetD16Async", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4059 + * cdef CUresult _cuMemsetD16Async(CUdeviceptr dstDevice, unsigned short us, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemsetD16Async + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemsetD16Async == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4059, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4060 + * global __cuMemsetD16Async + * cuPythonInit() + * if __cuMemsetD16Async == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD16Async" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16Async == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4061 + * cuPythonInit() + * if __cuMemsetD16Async == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD16Async" not found') + * err = ( __cuMemsetD16Async)(dstDevice, us, N, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4062 + * if __cuMemsetD16Async == NULL: + * with gil: + * raise RuntimeError('Function "cuMemsetD16Async" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemsetD16Async)(dstDevice, us, N, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemsetD16Async_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4062, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4062, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4061 + * cuPythonInit() + * if __cuMemsetD16Async == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD16Async" not found') + * err = ( __cuMemsetD16Async)(dstDevice, us, N, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4060 + * global __cuMemsetD16Async + * cuPythonInit() + * if __cuMemsetD16Async == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD16Async" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4063 + * with gil: + * raise RuntimeError('Function "cuMemsetD16Async" not found') + * err = ( __cuMemsetD16Async)(dstDevice, us, N, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, unsigned short, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16Async)(__pyx_v_dstDevice, __pyx_v_us, __pyx_v_N, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4063, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4064 + * raise RuntimeError('Function "cuMemsetD16Async" not found') + * err = ( __cuMemsetD16Async)(dstDevice, us, N, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemsetD32Async(CUdeviceptr dstDevice, unsigned int ui, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4057 + * return err + * + * cdef CUresult _cuMemsetD16Async(CUdeviceptr dstDevice, unsigned short us, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD16Async + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemsetD16Async", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4066 + * return err + * + * cdef CUresult _cuMemsetD32Async(CUdeviceptr dstDevice, unsigned int ui, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD32Async + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD32Async(CUdeviceptr __pyx_v_dstDevice, unsigned int __pyx_v_ui, size_t __pyx_v_N, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemsetD32Async", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4068 + * cdef CUresult _cuMemsetD32Async(CUdeviceptr dstDevice, unsigned int ui, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemsetD32Async + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemsetD32Async == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4068, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4069 + * global __cuMemsetD32Async + * cuPythonInit() + * if __cuMemsetD32Async == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD32Async" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32Async == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4070 + * cuPythonInit() + * if __cuMemsetD32Async == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD32Async" not found') + * err = ( __cuMemsetD32Async)(dstDevice, ui, N, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4071 + * if __cuMemsetD32Async == NULL: + * with gil: + * raise RuntimeError('Function "cuMemsetD32Async" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemsetD32Async)(dstDevice, ui, N, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemsetD32Async_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4071, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4071, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4070 + * cuPythonInit() + * if __cuMemsetD32Async == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD32Async" not found') + * err = ( __cuMemsetD32Async)(dstDevice, ui, N, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4069 + * global __cuMemsetD32Async + * cuPythonInit() + * if __cuMemsetD32Async == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD32Async" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4072 + * with gil: + * raise RuntimeError('Function "cuMemsetD32Async" not found') + * err = ( __cuMemsetD32Async)(dstDevice, ui, N, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, unsigned int, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32Async)(__pyx_v_dstDevice, __pyx_v_ui, __pyx_v_N, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4072, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4073 + * raise RuntimeError('Function "cuMemsetD32Async" not found') + * err = ( __cuMemsetD32Async)(dstDevice, ui, N, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemsetD2D8Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned char uc, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4066 + * return err + * + * cdef CUresult _cuMemsetD32Async(CUdeviceptr dstDevice, unsigned int ui, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD32Async + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemsetD32Async", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4075 + * return err + * + * cdef CUresult _cuMemsetD2D8Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned char uc, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD2D8Async + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD2D8Async(CUdeviceptr __pyx_v_dstDevice, size_t __pyx_v_dstPitch, unsigned char __pyx_v_uc, size_t __pyx_v_Width, size_t __pyx_v_Height, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemsetD2D8Async", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4077 + * cdef CUresult _cuMemsetD2D8Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned char uc, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemsetD2D8Async + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemsetD2D8Async == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4077, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4078 + * global __cuMemsetD2D8Async + * cuPythonInit() + * if __cuMemsetD2D8Async == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD2D8Async" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8Async == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4079 + * cuPythonInit() + * if __cuMemsetD2D8Async == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD2D8Async" not found') + * err = ( __cuMemsetD2D8Async)(dstDevice, dstPitch, uc, Width, Height, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4080 + * if __cuMemsetD2D8Async == NULL: + * with gil: + * raise RuntimeError('Function "cuMemsetD2D8Async" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemsetD2D8Async)(dstDevice, dstPitch, uc, Width, Height, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemsetD2D8Async_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4080, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4080, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4079 + * cuPythonInit() + * if __cuMemsetD2D8Async == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD2D8Async" not found') + * err = ( __cuMemsetD2D8Async)(dstDevice, dstPitch, uc, Width, Height, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4078 + * global __cuMemsetD2D8Async + * cuPythonInit() + * if __cuMemsetD2D8Async == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD2D8Async" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4081 + * with gil: + * raise RuntimeError('Function "cuMemsetD2D8Async" not found') + * err = ( __cuMemsetD2D8Async)(dstDevice, dstPitch, uc, Width, Height, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t, unsigned char, size_t, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8Async)(__pyx_v_dstDevice, __pyx_v_dstPitch, __pyx_v_uc, __pyx_v_Width, __pyx_v_Height, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4081, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4082 + * raise RuntimeError('Function "cuMemsetD2D8Async" not found') + * err = ( __cuMemsetD2D8Async)(dstDevice, dstPitch, uc, Width, Height, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemsetD2D16Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned short us, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4075 + * return err + * + * cdef CUresult _cuMemsetD2D8Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned char uc, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD2D8Async + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemsetD2D8Async", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4084 + * return err + * + * cdef CUresult _cuMemsetD2D16Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned short us, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD2D16Async + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD2D16Async(CUdeviceptr __pyx_v_dstDevice, size_t __pyx_v_dstPitch, unsigned short __pyx_v_us, size_t __pyx_v_Width, size_t __pyx_v_Height, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemsetD2D16Async", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4086 + * cdef CUresult _cuMemsetD2D16Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned short us, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemsetD2D16Async + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemsetD2D16Async == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4086, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4087 + * global __cuMemsetD2D16Async + * cuPythonInit() + * if __cuMemsetD2D16Async == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD2D16Async" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16Async == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4088 + * cuPythonInit() + * if __cuMemsetD2D16Async == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD2D16Async" not found') + * err = ( __cuMemsetD2D16Async)(dstDevice, dstPitch, us, Width, Height, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4089 + * if __cuMemsetD2D16Async == NULL: + * with gil: + * raise RuntimeError('Function "cuMemsetD2D16Async" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemsetD2D16Async)(dstDevice, dstPitch, us, Width, Height, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemsetD2D16Async_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4089, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4089, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4088 + * cuPythonInit() + * if __cuMemsetD2D16Async == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD2D16Async" not found') + * err = ( __cuMemsetD2D16Async)(dstDevice, dstPitch, us, Width, Height, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4087 + * global __cuMemsetD2D16Async + * cuPythonInit() + * if __cuMemsetD2D16Async == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD2D16Async" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4090 + * with gil: + * raise RuntimeError('Function "cuMemsetD2D16Async" not found') + * err = ( __cuMemsetD2D16Async)(dstDevice, dstPitch, us, Width, Height, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t, unsigned short, size_t, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16Async)(__pyx_v_dstDevice, __pyx_v_dstPitch, __pyx_v_us, __pyx_v_Width, __pyx_v_Height, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4090, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4091 + * raise RuntimeError('Function "cuMemsetD2D16Async" not found') + * err = ( __cuMemsetD2D16Async)(dstDevice, dstPitch, us, Width, Height, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemsetD2D32Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned int ui, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4084 + * return err + * + * cdef CUresult _cuMemsetD2D16Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned short us, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD2D16Async + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemsetD2D16Async", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4093 + * return err + * + * cdef CUresult _cuMemsetD2D32Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned int ui, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD2D32Async + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD2D32Async(CUdeviceptr __pyx_v_dstDevice, size_t __pyx_v_dstPitch, unsigned int __pyx_v_ui, size_t __pyx_v_Width, size_t __pyx_v_Height, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemsetD2D32Async", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4095 + * cdef CUresult _cuMemsetD2D32Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned int ui, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemsetD2D32Async + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemsetD2D32Async == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4095, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4096 + * global __cuMemsetD2D32Async + * cuPythonInit() + * if __cuMemsetD2D32Async == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD2D32Async" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32Async == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4097 + * cuPythonInit() + * if __cuMemsetD2D32Async == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD2D32Async" not found') + * err = ( __cuMemsetD2D32Async)(dstDevice, dstPitch, ui, Width, Height, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4098 + * if __cuMemsetD2D32Async == NULL: + * with gil: + * raise RuntimeError('Function "cuMemsetD2D32Async" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemsetD2D32Async)(dstDevice, dstPitch, ui, Width, Height, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemsetD2D32Async_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4098, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4098, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4097 + * cuPythonInit() + * if __cuMemsetD2D32Async == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemsetD2D32Async" not found') + * err = ( __cuMemsetD2D32Async)(dstDevice, dstPitch, ui, Width, Height, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4096 + * global __cuMemsetD2D32Async + * cuPythonInit() + * if __cuMemsetD2D32Async == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemsetD2D32Async" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4099 + * with gil: + * raise RuntimeError('Function "cuMemsetD2D32Async" not found') + * err = ( __cuMemsetD2D32Async)(dstDevice, dstPitch, ui, Width, Height, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t, unsigned int, size_t, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32Async)(__pyx_v_dstDevice, __pyx_v_dstPitch, __pyx_v_ui, __pyx_v_Width, __pyx_v_Height, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4099, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4100 + * raise RuntimeError('Function "cuMemsetD2D32Async" not found') + * err = ( __cuMemsetD2D32Async)(dstDevice, dstPitch, ui, Width, Height, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuArrayCreate_v2(CUarray* pHandle, const CUDA_ARRAY_DESCRIPTOR* pAllocateArray) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4093 + * return err + * + * cdef CUresult _cuMemsetD2D32Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned int ui, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemsetD2D32Async + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemsetD2D32Async", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4102 + * return err + * + * cdef CUresult _cuArrayCreate_v2(CUarray* pHandle, const CUDA_ARRAY_DESCRIPTOR* pAllocateArray) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArrayCreate_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArrayCreate_v2(CUarray *__pyx_v_pHandle, CUDA_ARRAY_DESCRIPTOR const *__pyx_v_pAllocateArray) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuArrayCreate_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4104 + * cdef CUresult _cuArrayCreate_v2(CUarray* pHandle, const CUDA_ARRAY_DESCRIPTOR* pAllocateArray) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuArrayCreate_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuArrayCreate_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4104, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4105 + * global __cuArrayCreate_v2 + * cuPythonInit() + * if __cuArrayCreate_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArrayCreate_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayCreate_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4106 + * cuPythonInit() + * if __cuArrayCreate_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArrayCreate_v2" not found') + * err = ( __cuArrayCreate_v2)(pHandle, pAllocateArray) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4107 + * if __cuArrayCreate_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuArrayCreate_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuArrayCreate_v2)(pHandle, pAllocateArray) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuArrayCreate_v2_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4107, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4107, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4106 + * cuPythonInit() + * if __cuArrayCreate_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArrayCreate_v2" not found') + * err = ( __cuArrayCreate_v2)(pHandle, pAllocateArray) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4105 + * global __cuArrayCreate_v2 + * cuPythonInit() + * if __cuArrayCreate_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArrayCreate_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4108 + * with gil: + * raise RuntimeError('Function "cuArrayCreate_v2" not found') + * err = ( __cuArrayCreate_v2)(pHandle, pAllocateArray) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarray *, CUDA_ARRAY_DESCRIPTOR const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayCreate_v2)(__pyx_v_pHandle, __pyx_v_pAllocateArray); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4108, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4109 + * raise RuntimeError('Function "cuArrayCreate_v2" not found') + * err = ( __cuArrayCreate_v2)(pHandle, pAllocateArray) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuArrayGetDescriptor_v2(CUDA_ARRAY_DESCRIPTOR* pArrayDescriptor, CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4102 + * return err + * + * cdef CUresult _cuArrayCreate_v2(CUarray* pHandle, const CUDA_ARRAY_DESCRIPTOR* pAllocateArray) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArrayCreate_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuArrayCreate_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4111 + * return err + * + * cdef CUresult _cuArrayGetDescriptor_v2(CUDA_ARRAY_DESCRIPTOR* pArrayDescriptor, CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArrayGetDescriptor_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArrayGetDescriptor_v2(CUDA_ARRAY_DESCRIPTOR *__pyx_v_pArrayDescriptor, CUarray __pyx_v_hArray) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuArrayGetDescriptor_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4113 + * cdef CUresult _cuArrayGetDescriptor_v2(CUDA_ARRAY_DESCRIPTOR* pArrayDescriptor, CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuArrayGetDescriptor_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuArrayGetDescriptor_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4113, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4114 + * global __cuArrayGetDescriptor_v2 + * cuPythonInit() + * if __cuArrayGetDescriptor_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArrayGetDescriptor_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetDescriptor_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4115 + * cuPythonInit() + * if __cuArrayGetDescriptor_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArrayGetDescriptor_v2" not found') + * err = ( __cuArrayGetDescriptor_v2)(pArrayDescriptor, hArray) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4116 + * if __cuArrayGetDescriptor_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuArrayGetDescriptor_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuArrayGetDescriptor_v2)(pArrayDescriptor, hArray) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuArrayGetDescriptor_v2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4116, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4116, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4115 + * cuPythonInit() + * if __cuArrayGetDescriptor_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArrayGetDescriptor_v2" not found') + * err = ( __cuArrayGetDescriptor_v2)(pArrayDescriptor, hArray) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4114 + * global __cuArrayGetDescriptor_v2 + * cuPythonInit() + * if __cuArrayGetDescriptor_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArrayGetDescriptor_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4117 + * with gil: + * raise RuntimeError('Function "cuArrayGetDescriptor_v2" not found') + * err = ( __cuArrayGetDescriptor_v2)(pArrayDescriptor, hArray) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_ARRAY_DESCRIPTOR *, CUarray))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetDescriptor_v2)(__pyx_v_pArrayDescriptor, __pyx_v_hArray); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4117, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4118 + * raise RuntimeError('Function "cuArrayGetDescriptor_v2" not found') + * err = ( __cuArrayGetDescriptor_v2)(pArrayDescriptor, hArray) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES* sparseProperties, CUarray array) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4111 + * return err + * + * cdef CUresult _cuArrayGetDescriptor_v2(CUDA_ARRAY_DESCRIPTOR* pArrayDescriptor, CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArrayGetDescriptor_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuArrayGetDescriptor_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4120 + * return err + * + * cdef CUresult _cuArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES* sparseProperties, CUarray array) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArrayGetSparseProperties + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES *__pyx_v_sparseProperties, CUarray __pyx_v_array) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuArrayGetSparseProperties", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4122 + * cdef CUresult _cuArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES* sparseProperties, CUarray array) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuArrayGetSparseProperties + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuArrayGetSparseProperties == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4122, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4123 + * global __cuArrayGetSparseProperties + * cuPythonInit() + * if __cuArrayGetSparseProperties == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArrayGetSparseProperties" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetSparseProperties == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4124 + * cuPythonInit() + * if __cuArrayGetSparseProperties == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArrayGetSparseProperties" not found') + * err = ( __cuArrayGetSparseProperties)(sparseProperties, array) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4125 + * if __cuArrayGetSparseProperties == NULL: + * with gil: + * raise RuntimeError('Function "cuArrayGetSparseProperties" not found') # <<<<<<<<<<<<<< + * err = ( __cuArrayGetSparseProperties)(sparseProperties, array) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuArrayGetSparsePropert}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4125, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4125, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4124 + * cuPythonInit() + * if __cuArrayGetSparseProperties == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArrayGetSparseProperties" not found') + * err = ( __cuArrayGetSparseProperties)(sparseProperties, array) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4123 + * global __cuArrayGetSparseProperties + * cuPythonInit() + * if __cuArrayGetSparseProperties == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArrayGetSparseProperties" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4126 + * with gil: + * raise RuntimeError('Function "cuArrayGetSparseProperties" not found') + * err = ( __cuArrayGetSparseProperties)(sparseProperties, array) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_ARRAY_SPARSE_PROPERTIES *, CUarray))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetSparseProperties)(__pyx_v_sparseProperties, __pyx_v_array); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4126, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4127 + * raise RuntimeError('Function "cuArrayGetSparseProperties" not found') + * err = ( __cuArrayGetSparseProperties)(sparseProperties, array) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMipmappedArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES* sparseProperties, CUmipmappedArray mipmap) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4120 + * return err + * + * cdef CUresult _cuArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES* sparseProperties, CUarray array) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArrayGetSparseProperties + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuArrayGetSparseProperties", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4129 + * return err + * + * cdef CUresult _cuMipmappedArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES* sparseProperties, CUmipmappedArray mipmap) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetSparseProperties + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMipmappedArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES *__pyx_v_sparseProperties, CUmipmappedArray __pyx_v_mipmap) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMipmappedArrayGetSparseProperties", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4131 + * cdef CUresult _cuMipmappedArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES* sparseProperties, CUmipmappedArray mipmap) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMipmappedArrayGetSparseProperties + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMipmappedArrayGetSparseProperties == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4131, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4132 + * global __cuMipmappedArrayGetSparseProperties + * cuPythonInit() + * if __cuMipmappedArrayGetSparseProperties == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayGetSparseProperties" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetSparseProperties == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4133 + * cuPythonInit() + * if __cuMipmappedArrayGetSparseProperties == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMipmappedArrayGetSparseProperties" not found') + * err = ( __cuMipmappedArrayGetSparseProperties)(sparseProperties, mipmap) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4134 + * if __cuMipmappedArrayGetSparseProperties == NULL: + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayGetSparseProperties" not found') # <<<<<<<<<<<<<< + * err = ( __cuMipmappedArrayGetSparseProperties)(sparseProperties, mipmap) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMipmappedArrayGetSpar}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4134, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4134, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4133 + * cuPythonInit() + * if __cuMipmappedArrayGetSparseProperties == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMipmappedArrayGetSparseProperties" not found') + * err = ( __cuMipmappedArrayGetSparseProperties)(sparseProperties, mipmap) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4132 + * global __cuMipmappedArrayGetSparseProperties + * cuPythonInit() + * if __cuMipmappedArrayGetSparseProperties == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayGetSparseProperties" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4135 + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayGetSparseProperties" not found') + * err = ( __cuMipmappedArrayGetSparseProperties)(sparseProperties, mipmap) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_ARRAY_SPARSE_PROPERTIES *, CUmipmappedArray))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetSparseProperties)(__pyx_v_sparseProperties, __pyx_v_mipmap); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4135, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4136 + * raise RuntimeError('Function "cuMipmappedArrayGetSparseProperties" not found') + * err = ( __cuMipmappedArrayGetSparseProperties)(sparseProperties, mipmap) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS* memoryRequirements, CUarray array, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4129 + * return err + * + * cdef CUresult _cuMipmappedArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES* sparseProperties, CUmipmappedArray mipmap) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetSparseProperties + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMipmappedArrayGetSparseProperties", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4138 + * return err + * + * cdef CUresult _cuArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS* memoryRequirements, CUarray array, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArrayGetMemoryRequirements + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS *__pyx_v_memoryRequirements, CUarray __pyx_v_array, CUdevice __pyx_v_device) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuArrayGetMemoryRequirements", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4140 + * cdef CUresult _cuArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS* memoryRequirements, CUarray array, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuArrayGetMemoryRequirements + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuArrayGetMemoryRequirements == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4140, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4141 + * global __cuArrayGetMemoryRequirements + * cuPythonInit() + * if __cuArrayGetMemoryRequirements == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArrayGetMemoryRequirements" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetMemoryRequirements == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4142 + * cuPythonInit() + * if __cuArrayGetMemoryRequirements == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArrayGetMemoryRequirements" not found') + * err = ( __cuArrayGetMemoryRequirements)(memoryRequirements, array, device) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4143 + * if __cuArrayGetMemoryRequirements == NULL: + * with gil: + * raise RuntimeError('Function "cuArrayGetMemoryRequirements" not found') # <<<<<<<<<<<<<< + * err = ( __cuArrayGetMemoryRequirements)(memoryRequirements, array, device) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuArrayGetMemoryRequire}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4143, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4143, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4142 + * cuPythonInit() + * if __cuArrayGetMemoryRequirements == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArrayGetMemoryRequirements" not found') + * err = ( __cuArrayGetMemoryRequirements)(memoryRequirements, array, device) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4141 + * global __cuArrayGetMemoryRequirements + * cuPythonInit() + * if __cuArrayGetMemoryRequirements == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArrayGetMemoryRequirements" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4144 + * with gil: + * raise RuntimeError('Function "cuArrayGetMemoryRequirements" not found') + * err = ( __cuArrayGetMemoryRequirements)(memoryRequirements, array, device) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_ARRAY_MEMORY_REQUIREMENTS *, CUarray, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetMemoryRequirements)(__pyx_v_memoryRequirements, __pyx_v_array, __pyx_v_device); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4144, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4145 + * raise RuntimeError('Function "cuArrayGetMemoryRequirements" not found') + * err = ( __cuArrayGetMemoryRequirements)(memoryRequirements, array, device) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMipmappedArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS* memoryRequirements, CUmipmappedArray mipmap, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4138 + * return err + * + * cdef CUresult _cuArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS* memoryRequirements, CUarray array, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArrayGetMemoryRequirements + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuArrayGetMemoryRequirements", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4147 + * return err + * + * cdef CUresult _cuMipmappedArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS* memoryRequirements, CUmipmappedArray mipmap, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetMemoryRequirements + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMipmappedArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS *__pyx_v_memoryRequirements, CUmipmappedArray __pyx_v_mipmap, CUdevice __pyx_v_device) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMipmappedArrayGetMemoryRequirements", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4149 + * cdef CUresult _cuMipmappedArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS* memoryRequirements, CUmipmappedArray mipmap, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMipmappedArrayGetMemoryRequirements + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMipmappedArrayGetMemoryRequirements == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4149, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4150 + * global __cuMipmappedArrayGetMemoryRequirements + * cuPythonInit() + * if __cuMipmappedArrayGetMemoryRequirements == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayGetMemoryRequirements" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetMemoryRequirements == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4151 + * cuPythonInit() + * if __cuMipmappedArrayGetMemoryRequirements == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMipmappedArrayGetMemoryRequirements" not found') + * err = ( __cuMipmappedArrayGetMemoryRequirements)(memoryRequirements, mipmap, device) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4152 + * if __cuMipmappedArrayGetMemoryRequirements == NULL: + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayGetMemoryRequirements" not found') # <<<<<<<<<<<<<< + * err = ( __cuMipmappedArrayGetMemoryRequirements)(memoryRequirements, mipmap, device) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMipmappedArrayGetMemo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4152, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4152, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4151 + * cuPythonInit() + * if __cuMipmappedArrayGetMemoryRequirements == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMipmappedArrayGetMemoryRequirements" not found') + * err = ( __cuMipmappedArrayGetMemoryRequirements)(memoryRequirements, mipmap, device) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4150 + * global __cuMipmappedArrayGetMemoryRequirements + * cuPythonInit() + * if __cuMipmappedArrayGetMemoryRequirements == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayGetMemoryRequirements" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4153 + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayGetMemoryRequirements" not found') + * err = ( __cuMipmappedArrayGetMemoryRequirements)(memoryRequirements, mipmap, device) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_ARRAY_MEMORY_REQUIREMENTS *, CUmipmappedArray, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetMemoryRequirements)(__pyx_v_memoryRequirements, __pyx_v_mipmap, __pyx_v_device); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4153, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4154 + * raise RuntimeError('Function "cuMipmappedArrayGetMemoryRequirements" not found') + * err = ( __cuMipmappedArrayGetMemoryRequirements)(memoryRequirements, mipmap, device) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuArrayGetPlane(CUarray* pPlaneArray, CUarray hArray, unsigned int planeIdx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4147 + * return err + * + * cdef CUresult _cuMipmappedArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS* memoryRequirements, CUmipmappedArray mipmap, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetMemoryRequirements + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMipmappedArrayGetMemoryRequirements", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4156 + * return err + * + * cdef CUresult _cuArrayGetPlane(CUarray* pPlaneArray, CUarray hArray, unsigned int planeIdx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArrayGetPlane + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArrayGetPlane(CUarray *__pyx_v_pPlaneArray, CUarray __pyx_v_hArray, unsigned int __pyx_v_planeIdx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuArrayGetPlane", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4158 + * cdef CUresult _cuArrayGetPlane(CUarray* pPlaneArray, CUarray hArray, unsigned int planeIdx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuArrayGetPlane + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuArrayGetPlane == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4158, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4159 + * global __cuArrayGetPlane + * cuPythonInit() + * if __cuArrayGetPlane == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArrayGetPlane" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetPlane == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4160 + * cuPythonInit() + * if __cuArrayGetPlane == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArrayGetPlane" not found') + * err = ( __cuArrayGetPlane)(pPlaneArray, hArray, planeIdx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4161 + * if __cuArrayGetPlane == NULL: + * with gil: + * raise RuntimeError('Function "cuArrayGetPlane" not found') # <<<<<<<<<<<<<< + * err = ( __cuArrayGetPlane)(pPlaneArray, hArray, planeIdx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuArrayGetPlane_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4161, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4161, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4160 + * cuPythonInit() + * if __cuArrayGetPlane == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArrayGetPlane" not found') + * err = ( __cuArrayGetPlane)(pPlaneArray, hArray, planeIdx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4159 + * global __cuArrayGetPlane + * cuPythonInit() + * if __cuArrayGetPlane == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArrayGetPlane" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4162 + * with gil: + * raise RuntimeError('Function "cuArrayGetPlane" not found') + * err = ( __cuArrayGetPlane)(pPlaneArray, hArray, planeIdx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarray *, CUarray, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetPlane)(__pyx_v_pPlaneArray, __pyx_v_hArray, __pyx_v_planeIdx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4162, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4163 + * raise RuntimeError('Function "cuArrayGetPlane" not found') + * err = ( __cuArrayGetPlane)(pPlaneArray, hArray, planeIdx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuArrayDestroy(CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4156 + * return err + * + * cdef CUresult _cuArrayGetPlane(CUarray* pPlaneArray, CUarray hArray, unsigned int planeIdx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArrayGetPlane + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuArrayGetPlane", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4165 + * return err + * + * cdef CUresult _cuArrayDestroy(CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArrayDestroy + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArrayDestroy(CUarray __pyx_v_hArray) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuArrayDestroy", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4167 + * cdef CUresult _cuArrayDestroy(CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuArrayDestroy + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuArrayDestroy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4167, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4168 + * global __cuArrayDestroy + * cuPythonInit() + * if __cuArrayDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArrayDestroy" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayDestroy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4169 + * cuPythonInit() + * if __cuArrayDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArrayDestroy" not found') + * err = ( __cuArrayDestroy)(hArray) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4170 + * if __cuArrayDestroy == NULL: + * with gil: + * raise RuntimeError('Function "cuArrayDestroy" not found') # <<<<<<<<<<<<<< + * err = ( __cuArrayDestroy)(hArray) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuArrayDestroy_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4170, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4170, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4169 + * cuPythonInit() + * if __cuArrayDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArrayDestroy" not found') + * err = ( __cuArrayDestroy)(hArray) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4168 + * global __cuArrayDestroy + * cuPythonInit() + * if __cuArrayDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArrayDestroy" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4171 + * with gil: + * raise RuntimeError('Function "cuArrayDestroy" not found') + * err = ( __cuArrayDestroy)(hArray) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarray))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayDestroy)(__pyx_v_hArray); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4171, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4172 + * raise RuntimeError('Function "cuArrayDestroy" not found') + * err = ( __cuArrayDestroy)(hArray) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuArray3DCreate_v2(CUarray* pHandle, const CUDA_ARRAY3D_DESCRIPTOR* pAllocateArray) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4165 + * return err + * + * cdef CUresult _cuArrayDestroy(CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArrayDestroy + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuArrayDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4174 + * return err + * + * cdef CUresult _cuArray3DCreate_v2(CUarray* pHandle, const CUDA_ARRAY3D_DESCRIPTOR* pAllocateArray) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArray3DCreate_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArray3DCreate_v2(CUarray *__pyx_v_pHandle, CUDA_ARRAY3D_DESCRIPTOR const *__pyx_v_pAllocateArray) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuArray3DCreate_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4176 + * cdef CUresult _cuArray3DCreate_v2(CUarray* pHandle, const CUDA_ARRAY3D_DESCRIPTOR* pAllocateArray) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuArray3DCreate_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuArray3DCreate_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4176, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4177 + * global __cuArray3DCreate_v2 + * cuPythonInit() + * if __cuArray3DCreate_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArray3DCreate_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DCreate_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4178 + * cuPythonInit() + * if __cuArray3DCreate_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArray3DCreate_v2" not found') + * err = ( __cuArray3DCreate_v2)(pHandle, pAllocateArray) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4179 + * if __cuArray3DCreate_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuArray3DCreate_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuArray3DCreate_v2)(pHandle, pAllocateArray) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuArray3DCreate_v2_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4179, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4179, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4178 + * cuPythonInit() + * if __cuArray3DCreate_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArray3DCreate_v2" not found') + * err = ( __cuArray3DCreate_v2)(pHandle, pAllocateArray) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4177 + * global __cuArray3DCreate_v2 + * cuPythonInit() + * if __cuArray3DCreate_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArray3DCreate_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4180 + * with gil: + * raise RuntimeError('Function "cuArray3DCreate_v2" not found') + * err = ( __cuArray3DCreate_v2)(pHandle, pAllocateArray) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarray *, CUDA_ARRAY3D_DESCRIPTOR const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DCreate_v2)(__pyx_v_pHandle, __pyx_v_pAllocateArray); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4180, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4181 + * raise RuntimeError('Function "cuArray3DCreate_v2" not found') + * err = ( __cuArray3DCreate_v2)(pHandle, pAllocateArray) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuArray3DGetDescriptor_v2(CUDA_ARRAY3D_DESCRIPTOR* pArrayDescriptor, CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4174 + * return err + * + * cdef CUresult _cuArray3DCreate_v2(CUarray* pHandle, const CUDA_ARRAY3D_DESCRIPTOR* pAllocateArray) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArray3DCreate_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuArray3DCreate_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4183 + * return err + * + * cdef CUresult _cuArray3DGetDescriptor_v2(CUDA_ARRAY3D_DESCRIPTOR* pArrayDescriptor, CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArray3DGetDescriptor_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArray3DGetDescriptor_v2(CUDA_ARRAY3D_DESCRIPTOR *__pyx_v_pArrayDescriptor, CUarray __pyx_v_hArray) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuArray3DGetDescriptor_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4185 + * cdef CUresult _cuArray3DGetDescriptor_v2(CUDA_ARRAY3D_DESCRIPTOR* pArrayDescriptor, CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuArray3DGetDescriptor_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuArray3DGetDescriptor_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4185, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4186 + * global __cuArray3DGetDescriptor_v2 + * cuPythonInit() + * if __cuArray3DGetDescriptor_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArray3DGetDescriptor_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DGetDescriptor_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4187 + * cuPythonInit() + * if __cuArray3DGetDescriptor_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArray3DGetDescriptor_v2" not found') + * err = ( __cuArray3DGetDescriptor_v2)(pArrayDescriptor, hArray) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4188 + * if __cuArray3DGetDescriptor_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuArray3DGetDescriptor_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuArray3DGetDescriptor_v2)(pArrayDescriptor, hArray) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuArray3DGetDescriptor}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4188, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4188, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4187 + * cuPythonInit() + * if __cuArray3DGetDescriptor_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuArray3DGetDescriptor_v2" not found') + * err = ( __cuArray3DGetDescriptor_v2)(pArrayDescriptor, hArray) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4186 + * global __cuArray3DGetDescriptor_v2 + * cuPythonInit() + * if __cuArray3DGetDescriptor_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuArray3DGetDescriptor_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4189 + * with gil: + * raise RuntimeError('Function "cuArray3DGetDescriptor_v2" not found') + * err = ( __cuArray3DGetDescriptor_v2)(pArrayDescriptor, hArray) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_ARRAY3D_DESCRIPTOR *, CUarray))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DGetDescriptor_v2)(__pyx_v_pArrayDescriptor, __pyx_v_hArray); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4189, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4190 + * raise RuntimeError('Function "cuArray3DGetDescriptor_v2" not found') + * err = ( __cuArray3DGetDescriptor_v2)(pArrayDescriptor, hArray) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMipmappedArrayCreate(CUmipmappedArray* pHandle, const CUDA_ARRAY3D_DESCRIPTOR* pMipmappedArrayDesc, unsigned int numMipmapLevels) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4183 + * return err + * + * cdef CUresult _cuArray3DGetDescriptor_v2(CUDA_ARRAY3D_DESCRIPTOR* pArrayDescriptor, CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuArray3DGetDescriptor_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuArray3DGetDescriptor_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4192 + * return err + * + * cdef CUresult _cuMipmappedArrayCreate(CUmipmappedArray* pHandle, const CUDA_ARRAY3D_DESCRIPTOR* pMipmappedArrayDesc, unsigned int numMipmapLevels) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMipmappedArrayCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMipmappedArrayCreate(CUmipmappedArray *__pyx_v_pHandle, CUDA_ARRAY3D_DESCRIPTOR const *__pyx_v_pMipmappedArrayDesc, unsigned int __pyx_v_numMipmapLevels) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMipmappedArrayCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4194 + * cdef CUresult _cuMipmappedArrayCreate(CUmipmappedArray* pHandle, const CUDA_ARRAY3D_DESCRIPTOR* pMipmappedArrayDesc, unsigned int numMipmapLevels) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMipmappedArrayCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMipmappedArrayCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4194, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4195 + * global __cuMipmappedArrayCreate + * cuPythonInit() + * if __cuMipmappedArrayCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4196 + * cuPythonInit() + * if __cuMipmappedArrayCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMipmappedArrayCreate" not found') + * err = ( __cuMipmappedArrayCreate)(pHandle, pMipmappedArrayDesc, numMipmapLevels) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4197 + * if __cuMipmappedArrayCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuMipmappedArrayCreate)(pHandle, pMipmappedArrayDesc, numMipmapLevels) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMipmappedArrayCreate}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4197, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4197, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4196 + * cuPythonInit() + * if __cuMipmappedArrayCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMipmappedArrayCreate" not found') + * err = ( __cuMipmappedArrayCreate)(pHandle, pMipmappedArrayDesc, numMipmapLevels) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4195 + * global __cuMipmappedArrayCreate + * cuPythonInit() + * if __cuMipmappedArrayCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4198 + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayCreate" not found') + * err = ( __cuMipmappedArrayCreate)(pHandle, pMipmappedArrayDesc, numMipmapLevels) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmipmappedArray *, CUDA_ARRAY3D_DESCRIPTOR const *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayCreate)(__pyx_v_pHandle, __pyx_v_pMipmappedArrayDesc, __pyx_v_numMipmapLevels); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4198, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4199 + * raise RuntimeError('Function "cuMipmappedArrayCreate" not found') + * err = ( __cuMipmappedArrayCreate)(pHandle, pMipmappedArrayDesc, numMipmapLevels) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMipmappedArrayGetLevel(CUarray* pLevelArray, CUmipmappedArray hMipmappedArray, unsigned int level) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4192 + * return err + * + * cdef CUresult _cuMipmappedArrayCreate(CUmipmappedArray* pHandle, const CUDA_ARRAY3D_DESCRIPTOR* pMipmappedArrayDesc, unsigned int numMipmapLevels) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMipmappedArrayCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMipmappedArrayCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4201 + * return err + * + * cdef CUresult _cuMipmappedArrayGetLevel(CUarray* pLevelArray, CUmipmappedArray hMipmappedArray, unsigned int level) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetLevel + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMipmappedArrayGetLevel(CUarray *__pyx_v_pLevelArray, CUmipmappedArray __pyx_v_hMipmappedArray, unsigned int __pyx_v_level) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMipmappedArrayGetLevel", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4203 + * cdef CUresult _cuMipmappedArrayGetLevel(CUarray* pLevelArray, CUmipmappedArray hMipmappedArray, unsigned int level) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMipmappedArrayGetLevel + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMipmappedArrayGetLevel == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4203, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4204 + * global __cuMipmappedArrayGetLevel + * cuPythonInit() + * if __cuMipmappedArrayGetLevel == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayGetLevel" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetLevel == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4205 + * cuPythonInit() + * if __cuMipmappedArrayGetLevel == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMipmappedArrayGetLevel" not found') + * err = ( __cuMipmappedArrayGetLevel)(pLevelArray, hMipmappedArray, level) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4206 + * if __cuMipmappedArrayGetLevel == NULL: + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayGetLevel" not found') # <<<<<<<<<<<<<< + * err = ( __cuMipmappedArrayGetLevel)(pLevelArray, hMipmappedArray, level) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMipmappedArrayGetLeve}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4206, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4206, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4205 + * cuPythonInit() + * if __cuMipmappedArrayGetLevel == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMipmappedArrayGetLevel" not found') + * err = ( __cuMipmappedArrayGetLevel)(pLevelArray, hMipmappedArray, level) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4204 + * global __cuMipmappedArrayGetLevel + * cuPythonInit() + * if __cuMipmappedArrayGetLevel == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayGetLevel" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4207 + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayGetLevel" not found') + * err = ( __cuMipmappedArrayGetLevel)(pLevelArray, hMipmappedArray, level) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarray *, CUmipmappedArray, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetLevel)(__pyx_v_pLevelArray, __pyx_v_hMipmappedArray, __pyx_v_level); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4207, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4208 + * raise RuntimeError('Function "cuMipmappedArrayGetLevel" not found') + * err = ( __cuMipmappedArrayGetLevel)(pLevelArray, hMipmappedArray, level) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMipmappedArrayDestroy(CUmipmappedArray hMipmappedArray) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4201 + * return err + * + * cdef CUresult _cuMipmappedArrayGetLevel(CUarray* pLevelArray, CUmipmappedArray hMipmappedArray, unsigned int level) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetLevel + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMipmappedArrayGetLevel", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4210 + * return err + * + * cdef CUresult _cuMipmappedArrayDestroy(CUmipmappedArray hMipmappedArray) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMipmappedArrayDestroy + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMipmappedArrayDestroy(CUmipmappedArray __pyx_v_hMipmappedArray) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMipmappedArrayDestroy", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4212 + * cdef CUresult _cuMipmappedArrayDestroy(CUmipmappedArray hMipmappedArray) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMipmappedArrayDestroy + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMipmappedArrayDestroy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4212, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4213 + * global __cuMipmappedArrayDestroy + * cuPythonInit() + * if __cuMipmappedArrayDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayDestroy" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayDestroy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4214 + * cuPythonInit() + * if __cuMipmappedArrayDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMipmappedArrayDestroy" not found') + * err = ( __cuMipmappedArrayDestroy)(hMipmappedArray) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4215 + * if __cuMipmappedArrayDestroy == NULL: + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayDestroy" not found') # <<<<<<<<<<<<<< + * err = ( __cuMipmappedArrayDestroy)(hMipmappedArray) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMipmappedArrayDestroy}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4215, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4215, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4214 + * cuPythonInit() + * if __cuMipmappedArrayDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMipmappedArrayDestroy" not found') + * err = ( __cuMipmappedArrayDestroy)(hMipmappedArray) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4213 + * global __cuMipmappedArrayDestroy + * cuPythonInit() + * if __cuMipmappedArrayDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayDestroy" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4216 + * with gil: + * raise RuntimeError('Function "cuMipmappedArrayDestroy" not found') + * err = ( __cuMipmappedArrayDestroy)(hMipmappedArray) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmipmappedArray))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayDestroy)(__pyx_v_hMipmappedArray); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4216, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4217 + * raise RuntimeError('Function "cuMipmappedArrayDestroy" not found') + * err = ( __cuMipmappedArrayDestroy)(hMipmappedArray) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemGetHandleForAddressRange(void* handle, CUdeviceptr dptr, size_t size, CUmemRangeHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4210 + * return err + * + * cdef CUresult _cuMipmappedArrayDestroy(CUmipmappedArray hMipmappedArray) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMipmappedArrayDestroy + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMipmappedArrayDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4219 + * return err + * + * cdef CUresult _cuMemGetHandleForAddressRange(void* handle, CUdeviceptr dptr, size_t size, CUmemRangeHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemGetHandleForAddressRange + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemGetHandleForAddressRange(void *__pyx_v_handle, CUdeviceptr __pyx_v_dptr, size_t __pyx_v_size, CUmemRangeHandleType __pyx_v_handleType, unsigned PY_LONG_LONG __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemGetHandleForAddressRange", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4221 + * cdef CUresult _cuMemGetHandleForAddressRange(void* handle, CUdeviceptr dptr, size_t size, CUmemRangeHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemGetHandleForAddressRange + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemGetHandleForAddressRange == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4221, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4222 + * global __cuMemGetHandleForAddressRange + * cuPythonInit() + * if __cuMemGetHandleForAddressRange == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemGetHandleForAddressRange" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetHandleForAddressRange == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4223 + * cuPythonInit() + * if __cuMemGetHandleForAddressRange == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemGetHandleForAddressRange" not found') + * err = ( __cuMemGetHandleForAddressRange)(handle, dptr, size, handleType, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4224 + * if __cuMemGetHandleForAddressRange == NULL: + * with gil: + * raise RuntimeError('Function "cuMemGetHandleForAddressRange" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemGetHandleForAddressRange)(handle, dptr, size, handleType, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemGetHandleForAddres}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4224, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4224, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4223 + * cuPythonInit() + * if __cuMemGetHandleForAddressRange == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemGetHandleForAddressRange" not found') + * err = ( __cuMemGetHandleForAddressRange)(handle, dptr, size, handleType, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4222 + * global __cuMemGetHandleForAddressRange + * cuPythonInit() + * if __cuMemGetHandleForAddressRange == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemGetHandleForAddressRange" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4225 + * with gil: + * raise RuntimeError('Function "cuMemGetHandleForAddressRange" not found') + * err = ( __cuMemGetHandleForAddressRange)(handle, dptr, size, handleType, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void *, CUdeviceptr, size_t, CUmemRangeHandleType, unsigned PY_LONG_LONG))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetHandleForAddressRange)(__pyx_v_handle, __pyx_v_dptr, __pyx_v_size, __pyx_v_handleType, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4225, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4226 + * raise RuntimeError('Function "cuMemGetHandleForAddressRange" not found') + * err = ( __cuMemGetHandleForAddressRange)(handle, dptr, size, handleType, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemBatchDecompressAsync(CUmemDecompressParams* paramsArray, size_t count, unsigned int flags, size_t* errorIndex, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4219 + * return err + * + * cdef CUresult _cuMemGetHandleForAddressRange(void* handle, CUdeviceptr dptr, size_t size, CUmemRangeHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemGetHandleForAddressRange + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemGetHandleForAddressRange", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4228 + * return err + * + * cdef CUresult _cuMemBatchDecompressAsync(CUmemDecompressParams* paramsArray, size_t count, unsigned int flags, size_t* errorIndex, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemBatchDecompressAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemBatchDecompressAsync(CUmemDecompressParams *__pyx_v_paramsArray, size_t __pyx_v_count, unsigned int __pyx_v_flags, size_t *__pyx_v_errorIndex, CUstream __pyx_v_stream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemBatchDecompressAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4230 + * cdef CUresult _cuMemBatchDecompressAsync(CUmemDecompressParams* paramsArray, size_t count, unsigned int flags, size_t* errorIndex, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemBatchDecompressAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemBatchDecompressAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4230, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4231 + * global __cuMemBatchDecompressAsync + * cuPythonInit() + * if __cuMemBatchDecompressAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemBatchDecompressAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemBatchDecompressAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4232 + * cuPythonInit() + * if __cuMemBatchDecompressAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemBatchDecompressAsync" not found') + * err = ( __cuMemBatchDecompressAsync)(paramsArray, count, flags, errorIndex, stream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4233 + * if __cuMemBatchDecompressAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuMemBatchDecompressAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemBatchDecompressAsync)(paramsArray, count, flags, errorIndex, stream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemBatchDecompressAsy}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4233, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4233, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4232 + * cuPythonInit() + * if __cuMemBatchDecompressAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemBatchDecompressAsync" not found') + * err = ( __cuMemBatchDecompressAsync)(paramsArray, count, flags, errorIndex, stream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4231 + * global __cuMemBatchDecompressAsync + * cuPythonInit() + * if __cuMemBatchDecompressAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemBatchDecompressAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4234 + * with gil: + * raise RuntimeError('Function "cuMemBatchDecompressAsync" not found') + * err = ( __cuMemBatchDecompressAsync)(paramsArray, count, flags, errorIndex, stream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemDecompressParams *, size_t, unsigned int, size_t *, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemBatchDecompressAsync)(__pyx_v_paramsArray, __pyx_v_count, __pyx_v_flags, __pyx_v_errorIndex, __pyx_v_stream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4234, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4235 + * raise RuntimeError('Function "cuMemBatchDecompressAsync" not found') + * err = ( __cuMemBatchDecompressAsync)(paramsArray, count, flags, errorIndex, stream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemAddressReserve(CUdeviceptr* ptr, size_t size, size_t alignment, CUdeviceptr addr, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4228 + * return err + * + * cdef CUresult _cuMemBatchDecompressAsync(CUmemDecompressParams* paramsArray, size_t count, unsigned int flags, size_t* errorIndex, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemBatchDecompressAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemBatchDecompressAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4237 + * return err + * + * cdef CUresult _cuMemAddressReserve(CUdeviceptr* ptr, size_t size, size_t alignment, CUdeviceptr addr, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAddressReserve + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAddressReserve(CUdeviceptr *__pyx_v_ptr, size_t __pyx_v_size, size_t __pyx_v_alignment, CUdeviceptr __pyx_v_addr, unsigned PY_LONG_LONG __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemAddressReserve", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4239 + * cdef CUresult _cuMemAddressReserve(CUdeviceptr* ptr, size_t size, size_t alignment, CUdeviceptr addr, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemAddressReserve + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemAddressReserve == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4239, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4240 + * global __cuMemAddressReserve + * cuPythonInit() + * if __cuMemAddressReserve == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAddressReserve" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressReserve == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4241 + * cuPythonInit() + * if __cuMemAddressReserve == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAddressReserve" not found') + * err = ( __cuMemAddressReserve)(ptr, size, alignment, addr, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4242 + * if __cuMemAddressReserve == NULL: + * with gil: + * raise RuntimeError('Function "cuMemAddressReserve" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemAddressReserve)(ptr, size, alignment, addr, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemAddressReserve_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4242, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4242, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4241 + * cuPythonInit() + * if __cuMemAddressReserve == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAddressReserve" not found') + * err = ( __cuMemAddressReserve)(ptr, size, alignment, addr, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4240 + * global __cuMemAddressReserve + * cuPythonInit() + * if __cuMemAddressReserve == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAddressReserve" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4243 + * with gil: + * raise RuntimeError('Function "cuMemAddressReserve" not found') + * err = ( __cuMemAddressReserve)(ptr, size, alignment, addr, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, size_t, size_t, CUdeviceptr, unsigned PY_LONG_LONG))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressReserve)(__pyx_v_ptr, __pyx_v_size, __pyx_v_alignment, __pyx_v_addr, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4243, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4244 + * raise RuntimeError('Function "cuMemAddressReserve" not found') + * err = ( __cuMemAddressReserve)(ptr, size, alignment, addr, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemAddressFree(CUdeviceptr ptr, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4237 + * return err + * + * cdef CUresult _cuMemAddressReserve(CUdeviceptr* ptr, size_t size, size_t alignment, CUdeviceptr addr, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAddressReserve + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemAddressReserve", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4246 + * return err + * + * cdef CUresult _cuMemAddressFree(CUdeviceptr ptr, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAddressFree + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAddressFree(CUdeviceptr __pyx_v_ptr, size_t __pyx_v_size) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemAddressFree", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4248 + * cdef CUresult _cuMemAddressFree(CUdeviceptr ptr, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemAddressFree + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemAddressFree == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4248, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4249 + * global __cuMemAddressFree + * cuPythonInit() + * if __cuMemAddressFree == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAddressFree" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressFree == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4250 + * cuPythonInit() + * if __cuMemAddressFree == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAddressFree" not found') + * err = ( __cuMemAddressFree)(ptr, size) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4251 + * if __cuMemAddressFree == NULL: + * with gil: + * raise RuntimeError('Function "cuMemAddressFree" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemAddressFree)(ptr, size) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemAddressFree_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4251, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4251, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4250 + * cuPythonInit() + * if __cuMemAddressFree == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAddressFree" not found') + * err = ( __cuMemAddressFree)(ptr, size) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4249 + * global __cuMemAddressFree + * cuPythonInit() + * if __cuMemAddressFree == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAddressFree" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4252 + * with gil: + * raise RuntimeError('Function "cuMemAddressFree" not found') + * err = ( __cuMemAddressFree)(ptr, size) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressFree)(__pyx_v_ptr, __pyx_v_size); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4252, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4253 + * raise RuntimeError('Function "cuMemAddressFree" not found') + * err = ( __cuMemAddressFree)(ptr, size) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemCreate(CUmemGenericAllocationHandle* handle, size_t size, const CUmemAllocationProp* prop, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4246 + * return err + * + * cdef CUresult _cuMemAddressFree(CUdeviceptr ptr, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAddressFree + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemAddressFree", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4255 + * return err + * + * cdef CUresult _cuMemCreate(CUmemGenericAllocationHandle* handle, size_t size, const CUmemAllocationProp* prop, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemCreate(CUmemGenericAllocationHandle *__pyx_v_handle, size_t __pyx_v_size, CUmemAllocationProp const *__pyx_v_prop, unsigned PY_LONG_LONG __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4257 + * cdef CUresult _cuMemCreate(CUmemGenericAllocationHandle* handle, size_t size, const CUmemAllocationProp* prop, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4257, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4258 + * global __cuMemCreate + * cuPythonInit() + * if __cuMemCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4259 + * cuPythonInit() + * if __cuMemCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemCreate" not found') + * err = ( __cuMemCreate)(handle, size, prop, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4260 + * if __cuMemCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuMemCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemCreate)(handle, size, prop, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemCreate_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4260, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4260, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4259 + * cuPythonInit() + * if __cuMemCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemCreate" not found') + * err = ( __cuMemCreate)(handle, size, prop, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4258 + * global __cuMemCreate + * cuPythonInit() + * if __cuMemCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4261 + * with gil: + * raise RuntimeError('Function "cuMemCreate" not found') + * err = ( __cuMemCreate)(handle, size, prop, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemGenericAllocationHandle *, size_t, CUmemAllocationProp const *, unsigned PY_LONG_LONG))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemCreate)(__pyx_v_handle, __pyx_v_size, __pyx_v_prop, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4261, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4262 + * raise RuntimeError('Function "cuMemCreate" not found') + * err = ( __cuMemCreate)(handle, size, prop, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemRelease(CUmemGenericAllocationHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4255 + * return err + * + * cdef CUresult _cuMemCreate(CUmemGenericAllocationHandle* handle, size_t size, const CUmemAllocationProp* prop, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4264 + * return err + * + * cdef CUresult _cuMemRelease(CUmemGenericAllocationHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemRelease + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemRelease(CUmemGenericAllocationHandle __pyx_v_handle) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemRelease", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4266 + * cdef CUresult _cuMemRelease(CUmemGenericAllocationHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemRelease + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemRelease == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4266, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4267 + * global __cuMemRelease + * cuPythonInit() + * if __cuMemRelease == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemRelease" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRelease == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4268 + * cuPythonInit() + * if __cuMemRelease == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemRelease" not found') + * err = ( __cuMemRelease)(handle) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4269 + * if __cuMemRelease == NULL: + * with gil: + * raise RuntimeError('Function "cuMemRelease" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemRelease)(handle) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemRelease_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4269, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4269, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4268 + * cuPythonInit() + * if __cuMemRelease == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemRelease" not found') + * err = ( __cuMemRelease)(handle) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4267 + * global __cuMemRelease + * cuPythonInit() + * if __cuMemRelease == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemRelease" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4270 + * with gil: + * raise RuntimeError('Function "cuMemRelease" not found') + * err = ( __cuMemRelease)(handle) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemGenericAllocationHandle))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRelease)(__pyx_v_handle); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4270, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4271 + * raise RuntimeError('Function "cuMemRelease" not found') + * err = ( __cuMemRelease)(handle) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemMap(CUdeviceptr ptr, size_t size, size_t offset, CUmemGenericAllocationHandle handle, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4264 + * return err + * + * cdef CUresult _cuMemRelease(CUmemGenericAllocationHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemRelease + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemRelease", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4273 + * return err + * + * cdef CUresult _cuMemMap(CUdeviceptr ptr, size_t size, size_t offset, CUmemGenericAllocationHandle handle, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemMap + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemMap(CUdeviceptr __pyx_v_ptr, size_t __pyx_v_size, size_t __pyx_v_offset, CUmemGenericAllocationHandle __pyx_v_handle, unsigned PY_LONG_LONG __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemMap", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4275 + * cdef CUresult _cuMemMap(CUdeviceptr ptr, size_t size, size_t offset, CUmemGenericAllocationHandle handle, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemMap + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemMap == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4275, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4276 + * global __cuMemMap + * cuPythonInit() + * if __cuMemMap == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemMap" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMap == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4277 + * cuPythonInit() + * if __cuMemMap == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemMap" not found') + * err = ( __cuMemMap)(ptr, size, offset, handle, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4278 + * if __cuMemMap == NULL: + * with gil: + * raise RuntimeError('Function "cuMemMap" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemMap)(ptr, size, offset, handle, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemMap_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4278, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4278, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4277 + * cuPythonInit() + * if __cuMemMap == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemMap" not found') + * err = ( __cuMemMap)(ptr, size, offset, handle, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4276 + * global __cuMemMap + * cuPythonInit() + * if __cuMemMap == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemMap" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4279 + * with gil: + * raise RuntimeError('Function "cuMemMap" not found') + * err = ( __cuMemMap)(ptr, size, offset, handle, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t, size_t, CUmemGenericAllocationHandle, unsigned PY_LONG_LONG))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMap)(__pyx_v_ptr, __pyx_v_size, __pyx_v_offset, __pyx_v_handle, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4279, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4280 + * raise RuntimeError('Function "cuMemMap" not found') + * err = ( __cuMemMap)(ptr, size, offset, handle, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemMapArrayAsync(CUarrayMapInfo* mapInfoList, unsigned int count, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4273 + * return err + * + * cdef CUresult _cuMemMap(CUdeviceptr ptr, size_t size, size_t offset, CUmemGenericAllocationHandle handle, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemMap + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemMap", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4282 + * return err + * + * cdef CUresult _cuMemMapArrayAsync(CUarrayMapInfo* mapInfoList, unsigned int count, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemMapArrayAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemMapArrayAsync(CUarrayMapInfo *__pyx_v_mapInfoList, unsigned int __pyx_v_count, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemMapArrayAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4284 + * cdef CUresult _cuMemMapArrayAsync(CUarrayMapInfo* mapInfoList, unsigned int count, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemMapArrayAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemMapArrayAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4284, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4285 + * global __cuMemMapArrayAsync + * cuPythonInit() + * if __cuMemMapArrayAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemMapArrayAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMapArrayAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4286 + * cuPythonInit() + * if __cuMemMapArrayAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemMapArrayAsync" not found') + * err = ( __cuMemMapArrayAsync)(mapInfoList, count, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4287 + * if __cuMemMapArrayAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuMemMapArrayAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemMapArrayAsync)(mapInfoList, count, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemMapArrayAsync_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4287, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4287, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4286 + * cuPythonInit() + * if __cuMemMapArrayAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemMapArrayAsync" not found') + * err = ( __cuMemMapArrayAsync)(mapInfoList, count, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4285 + * global __cuMemMapArrayAsync + * cuPythonInit() + * if __cuMemMapArrayAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemMapArrayAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4288 + * with gil: + * raise RuntimeError('Function "cuMemMapArrayAsync" not found') + * err = ( __cuMemMapArrayAsync)(mapInfoList, count, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarrayMapInfo *, unsigned int, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMapArrayAsync)(__pyx_v_mapInfoList, __pyx_v_count, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4288, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4289 + * raise RuntimeError('Function "cuMemMapArrayAsync" not found') + * err = ( __cuMemMapArrayAsync)(mapInfoList, count, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemUnmap(CUdeviceptr ptr, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4282 + * return err + * + * cdef CUresult _cuMemMapArrayAsync(CUarrayMapInfo* mapInfoList, unsigned int count, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemMapArrayAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemMapArrayAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4291 + * return err + * + * cdef CUresult _cuMemUnmap(CUdeviceptr ptr, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemUnmap + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemUnmap(CUdeviceptr __pyx_v_ptr, size_t __pyx_v_size) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemUnmap", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4293 + * cdef CUresult _cuMemUnmap(CUdeviceptr ptr, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemUnmap + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemUnmap == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4293, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4294 + * global __cuMemUnmap + * cuPythonInit() + * if __cuMemUnmap == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemUnmap" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemUnmap == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4295 + * cuPythonInit() + * if __cuMemUnmap == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemUnmap" not found') + * err = ( __cuMemUnmap)(ptr, size) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4296 + * if __cuMemUnmap == NULL: + * with gil: + * raise RuntimeError('Function "cuMemUnmap" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemUnmap)(ptr, size) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemUnmap_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4296, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4296, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4295 + * cuPythonInit() + * if __cuMemUnmap == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemUnmap" not found') + * err = ( __cuMemUnmap)(ptr, size) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4294 + * global __cuMemUnmap + * cuPythonInit() + * if __cuMemUnmap == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemUnmap" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4297 + * with gil: + * raise RuntimeError('Function "cuMemUnmap" not found') + * err = ( __cuMemUnmap)(ptr, size) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemUnmap)(__pyx_v_ptr, __pyx_v_size); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4297, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4298 + * raise RuntimeError('Function "cuMemUnmap" not found') + * err = ( __cuMemUnmap)(ptr, size) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemSetAccess(CUdeviceptr ptr, size_t size, const CUmemAccessDesc* desc, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4291 + * return err + * + * cdef CUresult _cuMemUnmap(CUdeviceptr ptr, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemUnmap + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemUnmap", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4300 + * return err + * + * cdef CUresult _cuMemSetAccess(CUdeviceptr ptr, size_t size, const CUmemAccessDesc* desc, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemSetAccess + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemSetAccess(CUdeviceptr __pyx_v_ptr, size_t __pyx_v_size, CUmemAccessDesc const *__pyx_v_desc, size_t __pyx_v_count) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemSetAccess", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4302 + * cdef CUresult _cuMemSetAccess(CUdeviceptr ptr, size_t size, const CUmemAccessDesc* desc, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemSetAccess + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemSetAccess == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4302, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4303 + * global __cuMemSetAccess + * cuPythonInit() + * if __cuMemSetAccess == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemSetAccess" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemSetAccess == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4304 + * cuPythonInit() + * if __cuMemSetAccess == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemSetAccess" not found') + * err = ( __cuMemSetAccess)(ptr, size, desc, count) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4305 + * if __cuMemSetAccess == NULL: + * with gil: + * raise RuntimeError('Function "cuMemSetAccess" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemSetAccess)(ptr, size, desc, count) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemSetAccess_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4305, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4305, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4304 + * cuPythonInit() + * if __cuMemSetAccess == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemSetAccess" not found') + * err = ( __cuMemSetAccess)(ptr, size, desc, count) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4303 + * global __cuMemSetAccess + * cuPythonInit() + * if __cuMemSetAccess == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemSetAccess" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4306 + * with gil: + * raise RuntimeError('Function "cuMemSetAccess" not found') + * err = ( __cuMemSetAccess)(ptr, size, desc, count) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t, CUmemAccessDesc const *, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemSetAccess)(__pyx_v_ptr, __pyx_v_size, __pyx_v_desc, __pyx_v_count); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4306, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4307 + * raise RuntimeError('Function "cuMemSetAccess" not found') + * err = ( __cuMemSetAccess)(ptr, size, desc, count) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemGetAccess(unsigned long long* flags, const CUmemLocation* location, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4300 + * return err + * + * cdef CUresult _cuMemSetAccess(CUdeviceptr ptr, size_t size, const CUmemAccessDesc* desc, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemSetAccess + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemSetAccess", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4309 + * return err + * + * cdef CUresult _cuMemGetAccess(unsigned long long* flags, const CUmemLocation* location, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemGetAccess + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemGetAccess(unsigned PY_LONG_LONG *__pyx_v_flags, CUmemLocation const *__pyx_v_location, CUdeviceptr __pyx_v_ptr) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemGetAccess", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4311 + * cdef CUresult _cuMemGetAccess(unsigned long long* flags, const CUmemLocation* location, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemGetAccess + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemGetAccess == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4311, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4312 + * global __cuMemGetAccess + * cuPythonInit() + * if __cuMemGetAccess == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemGetAccess" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAccess == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4313 + * cuPythonInit() + * if __cuMemGetAccess == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemGetAccess" not found') + * err = ( __cuMemGetAccess)(flags, location, ptr) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4314 + * if __cuMemGetAccess == NULL: + * with gil: + * raise RuntimeError('Function "cuMemGetAccess" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemGetAccess)(flags, location, ptr) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemGetAccess_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4314, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4314, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4313 + * cuPythonInit() + * if __cuMemGetAccess == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemGetAccess" not found') + * err = ( __cuMemGetAccess)(flags, location, ptr) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4312 + * global __cuMemGetAccess + * cuPythonInit() + * if __cuMemGetAccess == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemGetAccess" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4315 + * with gil: + * raise RuntimeError('Function "cuMemGetAccess" not found') + * err = ( __cuMemGetAccess)(flags, location, ptr) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(unsigned PY_LONG_LONG *, CUmemLocation const *, CUdeviceptr))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAccess)(__pyx_v_flags, __pyx_v_location, __pyx_v_ptr); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4315, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4316 + * raise RuntimeError('Function "cuMemGetAccess" not found') + * err = ( __cuMemGetAccess)(flags, location, ptr) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemExportToShareableHandle(void* shareableHandle, CUmemGenericAllocationHandle handle, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4309 + * return err + * + * cdef CUresult _cuMemGetAccess(unsigned long long* flags, const CUmemLocation* location, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemGetAccess + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemGetAccess", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4318 + * return err + * + * cdef CUresult _cuMemExportToShareableHandle(void* shareableHandle, CUmemGenericAllocationHandle handle, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemExportToShareableHandle + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemExportToShareableHandle(void *__pyx_v_shareableHandle, CUmemGenericAllocationHandle __pyx_v_handle, CUmemAllocationHandleType __pyx_v_handleType, unsigned PY_LONG_LONG __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemExportToShareableHandle", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4320 + * cdef CUresult _cuMemExportToShareableHandle(void* shareableHandle, CUmemGenericAllocationHandle handle, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemExportToShareableHandle + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemExportToShareableHandle == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4320, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4321 + * global __cuMemExportToShareableHandle + * cuPythonInit() + * if __cuMemExportToShareableHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemExportToShareableHandle" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemExportToShareableHandle == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4322 + * cuPythonInit() + * if __cuMemExportToShareableHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemExportToShareableHandle" not found') + * err = ( __cuMemExportToShareableHandle)(shareableHandle, handle, handleType, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4323 + * if __cuMemExportToShareableHandle == NULL: + * with gil: + * raise RuntimeError('Function "cuMemExportToShareableHandle" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemExportToShareableHandle)(shareableHandle, handle, handleType, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemExportToShareableH}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4323, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4323, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4322 + * cuPythonInit() + * if __cuMemExportToShareableHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemExportToShareableHandle" not found') + * err = ( __cuMemExportToShareableHandle)(shareableHandle, handle, handleType, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4321 + * global __cuMemExportToShareableHandle + * cuPythonInit() + * if __cuMemExportToShareableHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemExportToShareableHandle" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4324 + * with gil: + * raise RuntimeError('Function "cuMemExportToShareableHandle" not found') + * err = ( __cuMemExportToShareableHandle)(shareableHandle, handle, handleType, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void *, CUmemGenericAllocationHandle, CUmemAllocationHandleType, unsigned PY_LONG_LONG))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemExportToShareableHandle)(__pyx_v_shareableHandle, __pyx_v_handle, __pyx_v_handleType, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4324, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4325 + * raise RuntimeError('Function "cuMemExportToShareableHandle" not found') + * err = ( __cuMemExportToShareableHandle)(shareableHandle, handle, handleType, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemImportFromShareableHandle(CUmemGenericAllocationHandle* handle, void* osHandle, CUmemAllocationHandleType shHandleType) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4318 + * return err + * + * cdef CUresult _cuMemExportToShareableHandle(void* shareableHandle, CUmemGenericAllocationHandle handle, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemExportToShareableHandle + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemExportToShareableHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4327 + * return err + * + * cdef CUresult _cuMemImportFromShareableHandle(CUmemGenericAllocationHandle* handle, void* osHandle, CUmemAllocationHandleType shHandleType) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemImportFromShareableHandle + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemImportFromShareableHandle(CUmemGenericAllocationHandle *__pyx_v_handle, void *__pyx_v_osHandle, CUmemAllocationHandleType __pyx_v_shHandleType) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemImportFromShareableHandle", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4329 + * cdef CUresult _cuMemImportFromShareableHandle(CUmemGenericAllocationHandle* handle, void* osHandle, CUmemAllocationHandleType shHandleType) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemImportFromShareableHandle + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemImportFromShareableHandle == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4329, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4330 + * global __cuMemImportFromShareableHandle + * cuPythonInit() + * if __cuMemImportFromShareableHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemImportFromShareableHandle" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemImportFromShareableHandle == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4331 + * cuPythonInit() + * if __cuMemImportFromShareableHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemImportFromShareableHandle" not found') + * err = ( __cuMemImportFromShareableHandle)(handle, osHandle, shHandleType) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4332 + * if __cuMemImportFromShareableHandle == NULL: + * with gil: + * raise RuntimeError('Function "cuMemImportFromShareableHandle" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemImportFromShareableHandle)(handle, osHandle, shHandleType) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemImportFromShareabl}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4332, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4332, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4331 + * cuPythonInit() + * if __cuMemImportFromShareableHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemImportFromShareableHandle" not found') + * err = ( __cuMemImportFromShareableHandle)(handle, osHandle, shHandleType) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4330 + * global __cuMemImportFromShareableHandle + * cuPythonInit() + * if __cuMemImportFromShareableHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemImportFromShareableHandle" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4333 + * with gil: + * raise RuntimeError('Function "cuMemImportFromShareableHandle" not found') + * err = ( __cuMemImportFromShareableHandle)(handle, osHandle, shHandleType) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemGenericAllocationHandle *, void *, CUmemAllocationHandleType))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemImportFromShareableHandle)(__pyx_v_handle, __pyx_v_osHandle, __pyx_v_shHandleType); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4333, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4334 + * raise RuntimeError('Function "cuMemImportFromShareableHandle" not found') + * err = ( __cuMemImportFromShareableHandle)(handle, osHandle, shHandleType) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemGetAllocationGranularity(size_t* granularity, const CUmemAllocationProp* prop, CUmemAllocationGranularity_flags option) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4327 + * return err + * + * cdef CUresult _cuMemImportFromShareableHandle(CUmemGenericAllocationHandle* handle, void* osHandle, CUmemAllocationHandleType shHandleType) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemImportFromShareableHandle + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemImportFromShareableHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4336 + * return err + * + * cdef CUresult _cuMemGetAllocationGranularity(size_t* granularity, const CUmemAllocationProp* prop, CUmemAllocationGranularity_flags option) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemGetAllocationGranularity + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemGetAllocationGranularity(size_t *__pyx_v_granularity, CUmemAllocationProp const *__pyx_v_prop, CUmemAllocationGranularity_flags __pyx_v_option) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemGetAllocationGranularity", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4338 + * cdef CUresult _cuMemGetAllocationGranularity(size_t* granularity, const CUmemAllocationProp* prop, CUmemAllocationGranularity_flags option) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemGetAllocationGranularity + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemGetAllocationGranularity == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4338, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4339 + * global __cuMemGetAllocationGranularity + * cuPythonInit() + * if __cuMemGetAllocationGranularity == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemGetAllocationGranularity" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationGranularity == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4340 + * cuPythonInit() + * if __cuMemGetAllocationGranularity == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemGetAllocationGranularity" not found') + * err = ( __cuMemGetAllocationGranularity)(granularity, prop, option) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4341 + * if __cuMemGetAllocationGranularity == NULL: + * with gil: + * raise RuntimeError('Function "cuMemGetAllocationGranularity" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemGetAllocationGranularity)(granularity, prop, option) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemGetAllocationGranu}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4341, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4341, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4340 + * cuPythonInit() + * if __cuMemGetAllocationGranularity == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemGetAllocationGranularity" not found') + * err = ( __cuMemGetAllocationGranularity)(granularity, prop, option) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4339 + * global __cuMemGetAllocationGranularity + * cuPythonInit() + * if __cuMemGetAllocationGranularity == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemGetAllocationGranularity" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4342 + * with gil: + * raise RuntimeError('Function "cuMemGetAllocationGranularity" not found') + * err = ( __cuMemGetAllocationGranularity)(granularity, prop, option) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(size_t *, CUmemAllocationProp const *, CUmemAllocationGranularity_flags))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationGranularity)(__pyx_v_granularity, __pyx_v_prop, __pyx_v_option); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4342, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4343 + * raise RuntimeError('Function "cuMemGetAllocationGranularity" not found') + * err = ( __cuMemGetAllocationGranularity)(granularity, prop, option) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemGetAllocationPropertiesFromHandle(CUmemAllocationProp* prop, CUmemGenericAllocationHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4336 + * return err + * + * cdef CUresult _cuMemGetAllocationGranularity(size_t* granularity, const CUmemAllocationProp* prop, CUmemAllocationGranularity_flags option) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemGetAllocationGranularity + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemGetAllocationGranularity", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4345 + * return err + * + * cdef CUresult _cuMemGetAllocationPropertiesFromHandle(CUmemAllocationProp* prop, CUmemGenericAllocationHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemGetAllocationPropertiesFromHandle + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemGetAllocationPropertiesFromHandle(CUmemAllocationProp *__pyx_v_prop, CUmemGenericAllocationHandle __pyx_v_handle) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemGetAllocationPropertiesFromHandle", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4347 + * cdef CUresult _cuMemGetAllocationPropertiesFromHandle(CUmemAllocationProp* prop, CUmemGenericAllocationHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemGetAllocationPropertiesFromHandle + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemGetAllocationPropertiesFromHandle == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4347, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4348 + * global __cuMemGetAllocationPropertiesFromHandle + * cuPythonInit() + * if __cuMemGetAllocationPropertiesFromHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemGetAllocationPropertiesFromHandle" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationPropertiesFromHandle == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4349 + * cuPythonInit() + * if __cuMemGetAllocationPropertiesFromHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemGetAllocationPropertiesFromHandle" not found') + * err = ( __cuMemGetAllocationPropertiesFromHandle)(prop, handle) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4350 + * if __cuMemGetAllocationPropertiesFromHandle == NULL: + * with gil: + * raise RuntimeError('Function "cuMemGetAllocationPropertiesFromHandle" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemGetAllocationPropertiesFromHandle)(prop, handle) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemGetAllocationPrope}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4350, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4350, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4349 + * cuPythonInit() + * if __cuMemGetAllocationPropertiesFromHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemGetAllocationPropertiesFromHandle" not found') + * err = ( __cuMemGetAllocationPropertiesFromHandle)(prop, handle) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4348 + * global __cuMemGetAllocationPropertiesFromHandle + * cuPythonInit() + * if __cuMemGetAllocationPropertiesFromHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemGetAllocationPropertiesFromHandle" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4351 + * with gil: + * raise RuntimeError('Function "cuMemGetAllocationPropertiesFromHandle" not found') + * err = ( __cuMemGetAllocationPropertiesFromHandle)(prop, handle) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemAllocationProp *, CUmemGenericAllocationHandle))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationPropertiesFromHandle)(__pyx_v_prop, __pyx_v_handle); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4351, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4352 + * raise RuntimeError('Function "cuMemGetAllocationPropertiesFromHandle" not found') + * err = ( __cuMemGetAllocationPropertiesFromHandle)(prop, handle) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemRetainAllocationHandle(CUmemGenericAllocationHandle* handle, void* addr) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4345 + * return err + * + * cdef CUresult _cuMemGetAllocationPropertiesFromHandle(CUmemAllocationProp* prop, CUmemGenericAllocationHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemGetAllocationPropertiesFromHandle + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemGetAllocationPropertiesFromHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4354 + * return err + * + * cdef CUresult _cuMemRetainAllocationHandle(CUmemGenericAllocationHandle* handle, void* addr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemRetainAllocationHandle + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemRetainAllocationHandle(CUmemGenericAllocationHandle *__pyx_v_handle, void *__pyx_v_addr) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemRetainAllocationHandle", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4356 + * cdef CUresult _cuMemRetainAllocationHandle(CUmemGenericAllocationHandle* handle, void* addr) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemRetainAllocationHandle + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemRetainAllocationHandle == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4356, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4357 + * global __cuMemRetainAllocationHandle + * cuPythonInit() + * if __cuMemRetainAllocationHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemRetainAllocationHandle" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRetainAllocationHandle == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4358 + * cuPythonInit() + * if __cuMemRetainAllocationHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemRetainAllocationHandle" not found') + * err = ( __cuMemRetainAllocationHandle)(handle, addr) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4359 + * if __cuMemRetainAllocationHandle == NULL: + * with gil: + * raise RuntimeError('Function "cuMemRetainAllocationHandle" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemRetainAllocationHandle)(handle, addr) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemRetainAllocationHa}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4359, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4359, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4358 + * cuPythonInit() + * if __cuMemRetainAllocationHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemRetainAllocationHandle" not found') + * err = ( __cuMemRetainAllocationHandle)(handle, addr) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4357 + * global __cuMemRetainAllocationHandle + * cuPythonInit() + * if __cuMemRetainAllocationHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemRetainAllocationHandle" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4360 + * with gil: + * raise RuntimeError('Function "cuMemRetainAllocationHandle" not found') + * err = ( __cuMemRetainAllocationHandle)(handle, addr) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemGenericAllocationHandle *, void *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRetainAllocationHandle)(__pyx_v_handle, __pyx_v_addr); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4360, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4361 + * raise RuntimeError('Function "cuMemRetainAllocationHandle" not found') + * err = ( __cuMemRetainAllocationHandle)(handle, addr) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemFreeAsync(CUdeviceptr dptr, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4354 + * return err + * + * cdef CUresult _cuMemRetainAllocationHandle(CUmemGenericAllocationHandle* handle, void* addr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemRetainAllocationHandle + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemRetainAllocationHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4363 + * return err + * + * cdef CUresult _cuMemFreeAsync(CUdeviceptr dptr, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemFreeAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemFreeAsync(CUdeviceptr __pyx_v_dptr, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemFreeAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4365 + * cdef CUresult _cuMemFreeAsync(CUdeviceptr dptr, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemFreeAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemFreeAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4365, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4366 + * global __cuMemFreeAsync + * cuPythonInit() + * if __cuMemFreeAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemFreeAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4367 + * cuPythonInit() + * if __cuMemFreeAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemFreeAsync" not found') + * err = ( __cuMemFreeAsync)(dptr, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4368 + * if __cuMemFreeAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuMemFreeAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemFreeAsync)(dptr, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemFreeAsync_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4368, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4368, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4367 + * cuPythonInit() + * if __cuMemFreeAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemFreeAsync" not found') + * err = ( __cuMemFreeAsync)(dptr, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4366 + * global __cuMemFreeAsync + * cuPythonInit() + * if __cuMemFreeAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemFreeAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4369 + * with gil: + * raise RuntimeError('Function "cuMemFreeAsync" not found') + * err = ( __cuMemFreeAsync)(dptr, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeAsync)(__pyx_v_dptr, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4369, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4370 + * raise RuntimeError('Function "cuMemFreeAsync" not found') + * err = ( __cuMemFreeAsync)(dptr, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemAllocAsync(CUdeviceptr* dptr, size_t bytesize, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4363 + * return err + * + * cdef CUresult _cuMemFreeAsync(CUdeviceptr dptr, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemFreeAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemFreeAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4372 + * return err + * + * cdef CUresult _cuMemAllocAsync(CUdeviceptr* dptr, size_t bytesize, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAllocAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAllocAsync(CUdeviceptr *__pyx_v_dptr, size_t __pyx_v_bytesize, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemAllocAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4374 + * cdef CUresult _cuMemAllocAsync(CUdeviceptr* dptr, size_t bytesize, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemAllocAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemAllocAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4374, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4375 + * global __cuMemAllocAsync + * cuPythonInit() + * if __cuMemAllocAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAllocAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4376 + * cuPythonInit() + * if __cuMemAllocAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAllocAsync" not found') + * err = ( __cuMemAllocAsync)(dptr, bytesize, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4377 + * if __cuMemAllocAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuMemAllocAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemAllocAsync)(dptr, bytesize, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemAllocAsync_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4377, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4377, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4376 + * cuPythonInit() + * if __cuMemAllocAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAllocAsync" not found') + * err = ( __cuMemAllocAsync)(dptr, bytesize, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4375 + * global __cuMemAllocAsync + * cuPythonInit() + * if __cuMemAllocAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAllocAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4378 + * with gil: + * raise RuntimeError('Function "cuMemAllocAsync" not found') + * err = ( __cuMemAllocAsync)(dptr, bytesize, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, size_t, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocAsync)(__pyx_v_dptr, __pyx_v_bytesize, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4378, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4379 + * raise RuntimeError('Function "cuMemAllocAsync" not found') + * err = ( __cuMemAllocAsync)(dptr, bytesize, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemPoolTrimTo(CUmemoryPool pool, size_t minBytesToKeep) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4372 + * return err + * + * cdef CUresult _cuMemAllocAsync(CUdeviceptr* dptr, size_t bytesize, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAllocAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemAllocAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4381 + * return err + * + * cdef CUresult _cuMemPoolTrimTo(CUmemoryPool pool, size_t minBytesToKeep) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolTrimTo + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolTrimTo(CUmemoryPool __pyx_v_pool, size_t __pyx_v_minBytesToKeep) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemPoolTrimTo", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4383 + * cdef CUresult _cuMemPoolTrimTo(CUmemoryPool pool, size_t minBytesToKeep) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemPoolTrimTo + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemPoolTrimTo == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4383, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4384 + * global __cuMemPoolTrimTo + * cuPythonInit() + * if __cuMemPoolTrimTo == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolTrimTo" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolTrimTo == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4385 + * cuPythonInit() + * if __cuMemPoolTrimTo == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolTrimTo" not found') + * err = ( __cuMemPoolTrimTo)(pool, minBytesToKeep) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4386 + * if __cuMemPoolTrimTo == NULL: + * with gil: + * raise RuntimeError('Function "cuMemPoolTrimTo" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemPoolTrimTo)(pool, minBytesToKeep) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemPoolTrimTo_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4386, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4386, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4385 + * cuPythonInit() + * if __cuMemPoolTrimTo == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolTrimTo" not found') + * err = ( __cuMemPoolTrimTo)(pool, minBytesToKeep) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4384 + * global __cuMemPoolTrimTo + * cuPythonInit() + * if __cuMemPoolTrimTo == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolTrimTo" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4387 + * with gil: + * raise RuntimeError('Function "cuMemPoolTrimTo" not found') + * err = ( __cuMemPoolTrimTo)(pool, minBytesToKeep) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemoryPool, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolTrimTo)(__pyx_v_pool, __pyx_v_minBytesToKeep); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4387, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4388 + * raise RuntimeError('Function "cuMemPoolTrimTo" not found') + * err = ( __cuMemPoolTrimTo)(pool, minBytesToKeep) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemPoolSetAttribute(CUmemoryPool pool, CUmemPool_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4381 + * return err + * + * cdef CUresult _cuMemPoolTrimTo(CUmemoryPool pool, size_t minBytesToKeep) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolTrimTo + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemPoolTrimTo", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4390 + * return err + * + * cdef CUresult _cuMemPoolSetAttribute(CUmemoryPool pool, CUmemPool_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolSetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolSetAttribute(CUmemoryPool __pyx_v_pool, CUmemPool_attribute __pyx_v_attr, void *__pyx_v_value) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemPoolSetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4392 + * cdef CUresult _cuMemPoolSetAttribute(CUmemoryPool pool, CUmemPool_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemPoolSetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemPoolSetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4392, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4393 + * global __cuMemPoolSetAttribute + * cuPythonInit() + * if __cuMemPoolSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolSetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4394 + * cuPythonInit() + * if __cuMemPoolSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolSetAttribute" not found') + * err = ( __cuMemPoolSetAttribute)(pool, attr, value) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4395 + * if __cuMemPoolSetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuMemPoolSetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemPoolSetAttribute)(pool, attr, value) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemPoolSetAttribute_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4395, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4395, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4394 + * cuPythonInit() + * if __cuMemPoolSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolSetAttribute" not found') + * err = ( __cuMemPoolSetAttribute)(pool, attr, value) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4393 + * global __cuMemPoolSetAttribute + * cuPythonInit() + * if __cuMemPoolSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolSetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4396 + * with gil: + * raise RuntimeError('Function "cuMemPoolSetAttribute" not found') + * err = ( __cuMemPoolSetAttribute)(pool, attr, value) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemoryPool, CUmemPool_attribute, void *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAttribute)(__pyx_v_pool, __pyx_v_attr, __pyx_v_value); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4396, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4397 + * raise RuntimeError('Function "cuMemPoolSetAttribute" not found') + * err = ( __cuMemPoolSetAttribute)(pool, attr, value) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemPoolGetAttribute(CUmemoryPool pool, CUmemPool_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4390 + * return err + * + * cdef CUresult _cuMemPoolSetAttribute(CUmemoryPool pool, CUmemPool_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolSetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemPoolSetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4399 + * return err + * + * cdef CUresult _cuMemPoolGetAttribute(CUmemoryPool pool, CUmemPool_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolGetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolGetAttribute(CUmemoryPool __pyx_v_pool, CUmemPool_attribute __pyx_v_attr, void *__pyx_v_value) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemPoolGetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4401 + * cdef CUresult _cuMemPoolGetAttribute(CUmemoryPool pool, CUmemPool_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemPoolGetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemPoolGetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4401, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4402 + * global __cuMemPoolGetAttribute + * cuPythonInit() + * if __cuMemPoolGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolGetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4403 + * cuPythonInit() + * if __cuMemPoolGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolGetAttribute" not found') + * err = ( __cuMemPoolGetAttribute)(pool, attr, value) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4404 + * if __cuMemPoolGetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuMemPoolGetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemPoolGetAttribute)(pool, attr, value) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemPoolGetAttribute_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4404, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4404, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4403 + * cuPythonInit() + * if __cuMemPoolGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolGetAttribute" not found') + * err = ( __cuMemPoolGetAttribute)(pool, attr, value) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4402 + * global __cuMemPoolGetAttribute + * cuPythonInit() + * if __cuMemPoolGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolGetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4405 + * with gil: + * raise RuntimeError('Function "cuMemPoolGetAttribute" not found') + * err = ( __cuMemPoolGetAttribute)(pool, attr, value) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemoryPool, CUmemPool_attribute, void *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAttribute)(__pyx_v_pool, __pyx_v_attr, __pyx_v_value); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4405, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4406 + * raise RuntimeError('Function "cuMemPoolGetAttribute" not found') + * err = ( __cuMemPoolGetAttribute)(pool, attr, value) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemPoolSetAccess(CUmemoryPool pool, const CUmemAccessDesc* map, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4399 + * return err + * + * cdef CUresult _cuMemPoolGetAttribute(CUmemoryPool pool, CUmemPool_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolGetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemPoolGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4408 + * return err + * + * cdef CUresult _cuMemPoolSetAccess(CUmemoryPool pool, const CUmemAccessDesc* map, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolSetAccess + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolSetAccess(CUmemoryPool __pyx_v_pool, CUmemAccessDesc const *__pyx_v_map, size_t __pyx_v_count) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemPoolSetAccess", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4410 + * cdef CUresult _cuMemPoolSetAccess(CUmemoryPool pool, const CUmemAccessDesc* map, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemPoolSetAccess + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemPoolSetAccess == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4410, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4411 + * global __cuMemPoolSetAccess + * cuPythonInit() + * if __cuMemPoolSetAccess == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolSetAccess" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAccess == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4412 + * cuPythonInit() + * if __cuMemPoolSetAccess == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolSetAccess" not found') + * err = ( __cuMemPoolSetAccess)(pool, map, count) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4413 + * if __cuMemPoolSetAccess == NULL: + * with gil: + * raise RuntimeError('Function "cuMemPoolSetAccess" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemPoolSetAccess)(pool, map, count) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemPoolSetAccess_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4413, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4413, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4412 + * cuPythonInit() + * if __cuMemPoolSetAccess == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolSetAccess" not found') + * err = ( __cuMemPoolSetAccess)(pool, map, count) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4411 + * global __cuMemPoolSetAccess + * cuPythonInit() + * if __cuMemPoolSetAccess == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolSetAccess" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4414 + * with gil: + * raise RuntimeError('Function "cuMemPoolSetAccess" not found') + * err = ( __cuMemPoolSetAccess)(pool, map, count) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemoryPool, CUmemAccessDesc const *, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAccess)(__pyx_v_pool, __pyx_v_map, __pyx_v_count); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4414, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4415 + * raise RuntimeError('Function "cuMemPoolSetAccess" not found') + * err = ( __cuMemPoolSetAccess)(pool, map, count) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemPoolGetAccess(CUmemAccess_flags* flags, CUmemoryPool memPool, CUmemLocation* location) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4408 + * return err + * + * cdef CUresult _cuMemPoolSetAccess(CUmemoryPool pool, const CUmemAccessDesc* map, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolSetAccess + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemPoolSetAccess", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4417 + * return err + * + * cdef CUresult _cuMemPoolGetAccess(CUmemAccess_flags* flags, CUmemoryPool memPool, CUmemLocation* location) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolGetAccess + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolGetAccess(CUmemAccess_flags *__pyx_v_flags, CUmemoryPool __pyx_v_memPool, CUmemLocation *__pyx_v_location) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemPoolGetAccess", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4419 + * cdef CUresult _cuMemPoolGetAccess(CUmemAccess_flags* flags, CUmemoryPool memPool, CUmemLocation* location) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemPoolGetAccess + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemPoolGetAccess == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4419, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4420 + * global __cuMemPoolGetAccess + * cuPythonInit() + * if __cuMemPoolGetAccess == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolGetAccess" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAccess == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4421 + * cuPythonInit() + * if __cuMemPoolGetAccess == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolGetAccess" not found') + * err = ( __cuMemPoolGetAccess)(flags, memPool, location) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4422 + * if __cuMemPoolGetAccess == NULL: + * with gil: + * raise RuntimeError('Function "cuMemPoolGetAccess" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemPoolGetAccess)(flags, memPool, location) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemPoolGetAccess_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4422, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4422, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4421 + * cuPythonInit() + * if __cuMemPoolGetAccess == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolGetAccess" not found') + * err = ( __cuMemPoolGetAccess)(flags, memPool, location) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4420 + * global __cuMemPoolGetAccess + * cuPythonInit() + * if __cuMemPoolGetAccess == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolGetAccess" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4423 + * with gil: + * raise RuntimeError('Function "cuMemPoolGetAccess" not found') + * err = ( __cuMemPoolGetAccess)(flags, memPool, location) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemAccess_flags *, CUmemoryPool, CUmemLocation *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAccess)(__pyx_v_flags, __pyx_v_memPool, __pyx_v_location); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4423, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4424 + * raise RuntimeError('Function "cuMemPoolGetAccess" not found') + * err = ( __cuMemPoolGetAccess)(flags, memPool, location) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemPoolCreate(CUmemoryPool* pool, const CUmemPoolProps* poolProps) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4417 + * return err + * + * cdef CUresult _cuMemPoolGetAccess(CUmemAccess_flags* flags, CUmemoryPool memPool, CUmemLocation* location) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolGetAccess + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemPoolGetAccess", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4426 + * return err + * + * cdef CUresult _cuMemPoolCreate(CUmemoryPool* pool, const CUmemPoolProps* poolProps) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolCreate(CUmemoryPool *__pyx_v_pool, CUmemPoolProps const *__pyx_v_poolProps) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemPoolCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4428 + * cdef CUresult _cuMemPoolCreate(CUmemoryPool* pool, const CUmemPoolProps* poolProps) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemPoolCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemPoolCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4428, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4429 + * global __cuMemPoolCreate + * cuPythonInit() + * if __cuMemPoolCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4430 + * cuPythonInit() + * if __cuMemPoolCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolCreate" not found') + * err = ( __cuMemPoolCreate)(pool, poolProps) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4431 + * if __cuMemPoolCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuMemPoolCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemPoolCreate)(pool, poolProps) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemPoolCreate_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4431, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4431, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4430 + * cuPythonInit() + * if __cuMemPoolCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolCreate" not found') + * err = ( __cuMemPoolCreate)(pool, poolProps) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4429 + * global __cuMemPoolCreate + * cuPythonInit() + * if __cuMemPoolCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4432 + * with gil: + * raise RuntimeError('Function "cuMemPoolCreate" not found') + * err = ( __cuMemPoolCreate)(pool, poolProps) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemoryPool *, CUmemPoolProps const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolCreate)(__pyx_v_pool, __pyx_v_poolProps); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4432, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4433 + * raise RuntimeError('Function "cuMemPoolCreate" not found') + * err = ( __cuMemPoolCreate)(pool, poolProps) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemPoolDestroy(CUmemoryPool pool) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4426 + * return err + * + * cdef CUresult _cuMemPoolCreate(CUmemoryPool* pool, const CUmemPoolProps* poolProps) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemPoolCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4435 + * return err + * + * cdef CUresult _cuMemPoolDestroy(CUmemoryPool pool) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolDestroy + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolDestroy(CUmemoryPool __pyx_v_pool) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemPoolDestroy", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4437 + * cdef CUresult _cuMemPoolDestroy(CUmemoryPool pool) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemPoolDestroy + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemPoolDestroy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4437, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4438 + * global __cuMemPoolDestroy + * cuPythonInit() + * if __cuMemPoolDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolDestroy" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolDestroy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4439 + * cuPythonInit() + * if __cuMemPoolDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolDestroy" not found') + * err = ( __cuMemPoolDestroy)(pool) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4440 + * if __cuMemPoolDestroy == NULL: + * with gil: + * raise RuntimeError('Function "cuMemPoolDestroy" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemPoolDestroy)(pool) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemPoolDestroy_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4440, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4440, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4439 + * cuPythonInit() + * if __cuMemPoolDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolDestroy" not found') + * err = ( __cuMemPoolDestroy)(pool) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4438 + * global __cuMemPoolDestroy + * cuPythonInit() + * if __cuMemPoolDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolDestroy" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4441 + * with gil: + * raise RuntimeError('Function "cuMemPoolDestroy" not found') + * err = ( __cuMemPoolDestroy)(pool) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemoryPool))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolDestroy)(__pyx_v_pool); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4441, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4442 + * raise RuntimeError('Function "cuMemPoolDestroy" not found') + * err = ( __cuMemPoolDestroy)(pool) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemAllocFromPoolAsync(CUdeviceptr* dptr, size_t bytesize, CUmemoryPool pool, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4435 + * return err + * + * cdef CUresult _cuMemPoolDestroy(CUmemoryPool pool) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolDestroy + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemPoolDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4444 + * return err + * + * cdef CUresult _cuMemAllocFromPoolAsync(CUdeviceptr* dptr, size_t bytesize, CUmemoryPool pool, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAllocFromPoolAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAllocFromPoolAsync(CUdeviceptr *__pyx_v_dptr, size_t __pyx_v_bytesize, CUmemoryPool __pyx_v_pool, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemAllocFromPoolAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4446 + * cdef CUresult _cuMemAllocFromPoolAsync(CUdeviceptr* dptr, size_t bytesize, CUmemoryPool pool, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemAllocFromPoolAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemAllocFromPoolAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4446, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4447 + * global __cuMemAllocFromPoolAsync + * cuPythonInit() + * if __cuMemAllocFromPoolAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAllocFromPoolAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocFromPoolAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4448 + * cuPythonInit() + * if __cuMemAllocFromPoolAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAllocFromPoolAsync" not found') + * err = ( __cuMemAllocFromPoolAsync)(dptr, bytesize, pool, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4449 + * if __cuMemAllocFromPoolAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuMemAllocFromPoolAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemAllocFromPoolAsync)(dptr, bytesize, pool, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemAllocFromPoolAsync}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4449, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4449, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4448 + * cuPythonInit() + * if __cuMemAllocFromPoolAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAllocFromPoolAsync" not found') + * err = ( __cuMemAllocFromPoolAsync)(dptr, bytesize, pool, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4447 + * global __cuMemAllocFromPoolAsync + * cuPythonInit() + * if __cuMemAllocFromPoolAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAllocFromPoolAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4450 + * with gil: + * raise RuntimeError('Function "cuMemAllocFromPoolAsync" not found') + * err = ( __cuMemAllocFromPoolAsync)(dptr, bytesize, pool, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, size_t, CUmemoryPool, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocFromPoolAsync)(__pyx_v_dptr, __pyx_v_bytesize, __pyx_v_pool, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4450, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4451 + * raise RuntimeError('Function "cuMemAllocFromPoolAsync" not found') + * err = ( __cuMemAllocFromPoolAsync)(dptr, bytesize, pool, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemPoolExportToShareableHandle(void* handle_out, CUmemoryPool pool, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4444 + * return err + * + * cdef CUresult _cuMemAllocFromPoolAsync(CUdeviceptr* dptr, size_t bytesize, CUmemoryPool pool, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAllocFromPoolAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemAllocFromPoolAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4453 + * return err + * + * cdef CUresult _cuMemPoolExportToShareableHandle(void* handle_out, CUmemoryPool pool, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolExportToShareableHandle + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolExportToShareableHandle(void *__pyx_v_handle_out, CUmemoryPool __pyx_v_pool, CUmemAllocationHandleType __pyx_v_handleType, unsigned PY_LONG_LONG __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemPoolExportToShareableHandle", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4455 + * cdef CUresult _cuMemPoolExportToShareableHandle(void* handle_out, CUmemoryPool pool, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemPoolExportToShareableHandle + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemPoolExportToShareableHandle == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4455, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4456 + * global __cuMemPoolExportToShareableHandle + * cuPythonInit() + * if __cuMemPoolExportToShareableHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolExportToShareableHandle" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportToShareableHandle == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4457 + * cuPythonInit() + * if __cuMemPoolExportToShareableHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolExportToShareableHandle" not found') + * err = ( __cuMemPoolExportToShareableHandle)(handle_out, pool, handleType, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4458 + * if __cuMemPoolExportToShareableHandle == NULL: + * with gil: + * raise RuntimeError('Function "cuMemPoolExportToShareableHandle" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemPoolExportToShareableHandle)(handle_out, pool, handleType, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemPoolExportToSharea}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4458, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4458, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4457 + * cuPythonInit() + * if __cuMemPoolExportToShareableHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolExportToShareableHandle" not found') + * err = ( __cuMemPoolExportToShareableHandle)(handle_out, pool, handleType, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4456 + * global __cuMemPoolExportToShareableHandle + * cuPythonInit() + * if __cuMemPoolExportToShareableHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolExportToShareableHandle" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4459 + * with gil: + * raise RuntimeError('Function "cuMemPoolExportToShareableHandle" not found') + * err = ( __cuMemPoolExportToShareableHandle)(handle_out, pool, handleType, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void *, CUmemoryPool, CUmemAllocationHandleType, unsigned PY_LONG_LONG))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportToShareableHandle)(__pyx_v_handle_out, __pyx_v_pool, __pyx_v_handleType, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4459, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4460 + * raise RuntimeError('Function "cuMemPoolExportToShareableHandle" not found') + * err = ( __cuMemPoolExportToShareableHandle)(handle_out, pool, handleType, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemPoolImportFromShareableHandle(CUmemoryPool* pool_out, void* handle, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4453 + * return err + * + * cdef CUresult _cuMemPoolExportToShareableHandle(void* handle_out, CUmemoryPool pool, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolExportToShareableHandle + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemPoolExportToShareableHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4462 + * return err + * + * cdef CUresult _cuMemPoolImportFromShareableHandle(CUmemoryPool* pool_out, void* handle, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolImportFromShareableHandle + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolImportFromShareableHandle(CUmemoryPool *__pyx_v_pool_out, void *__pyx_v_handle, CUmemAllocationHandleType __pyx_v_handleType, unsigned PY_LONG_LONG __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemPoolImportFromShareableHandle", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4464 + * cdef CUresult _cuMemPoolImportFromShareableHandle(CUmemoryPool* pool_out, void* handle, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemPoolImportFromShareableHandle + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemPoolImportFromShareableHandle == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4464, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4465 + * global __cuMemPoolImportFromShareableHandle + * cuPythonInit() + * if __cuMemPoolImportFromShareableHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolImportFromShareableHandle" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportFromShareableHandle == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4466 + * cuPythonInit() + * if __cuMemPoolImportFromShareableHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolImportFromShareableHandle" not found') + * err = ( __cuMemPoolImportFromShareableHandle)(pool_out, handle, handleType, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4467 + * if __cuMemPoolImportFromShareableHandle == NULL: + * with gil: + * raise RuntimeError('Function "cuMemPoolImportFromShareableHandle" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemPoolImportFromShareableHandle)(pool_out, handle, handleType, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemPoolImportFromShar}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4467, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4467, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4466 + * cuPythonInit() + * if __cuMemPoolImportFromShareableHandle == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolImportFromShareableHandle" not found') + * err = ( __cuMemPoolImportFromShareableHandle)(pool_out, handle, handleType, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4465 + * global __cuMemPoolImportFromShareableHandle + * cuPythonInit() + * if __cuMemPoolImportFromShareableHandle == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolImportFromShareableHandle" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4468 + * with gil: + * raise RuntimeError('Function "cuMemPoolImportFromShareableHandle" not found') + * err = ( __cuMemPoolImportFromShareableHandle)(pool_out, handle, handleType, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemoryPool *, void *, CUmemAllocationHandleType, unsigned PY_LONG_LONG))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportFromShareableHandle)(__pyx_v_pool_out, __pyx_v_handle, __pyx_v_handleType, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4468, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4469 + * raise RuntimeError('Function "cuMemPoolImportFromShareableHandle" not found') + * err = ( __cuMemPoolImportFromShareableHandle)(pool_out, handle, handleType, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemPoolExportPointer(CUmemPoolPtrExportData* shareData_out, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4462 + * return err + * + * cdef CUresult _cuMemPoolImportFromShareableHandle(CUmemoryPool* pool_out, void* handle, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolImportFromShareableHandle + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemPoolImportFromShareableHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4471 + * return err + * + * cdef CUresult _cuMemPoolExportPointer(CUmemPoolPtrExportData* shareData_out, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolExportPointer + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolExportPointer(CUmemPoolPtrExportData *__pyx_v_shareData_out, CUdeviceptr __pyx_v_ptr) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemPoolExportPointer", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4473 + * cdef CUresult _cuMemPoolExportPointer(CUmemPoolPtrExportData* shareData_out, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemPoolExportPointer + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemPoolExportPointer == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4473, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4474 + * global __cuMemPoolExportPointer + * cuPythonInit() + * if __cuMemPoolExportPointer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolExportPointer" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportPointer == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4475 + * cuPythonInit() + * if __cuMemPoolExportPointer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolExportPointer" not found') + * err = ( __cuMemPoolExportPointer)(shareData_out, ptr) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4476 + * if __cuMemPoolExportPointer == NULL: + * with gil: + * raise RuntimeError('Function "cuMemPoolExportPointer" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemPoolExportPointer)(shareData_out, ptr) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemPoolExportPointer}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4476, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4476, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4475 + * cuPythonInit() + * if __cuMemPoolExportPointer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolExportPointer" not found') + * err = ( __cuMemPoolExportPointer)(shareData_out, ptr) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4474 + * global __cuMemPoolExportPointer + * cuPythonInit() + * if __cuMemPoolExportPointer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolExportPointer" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4477 + * with gil: + * raise RuntimeError('Function "cuMemPoolExportPointer" not found') + * err = ( __cuMemPoolExportPointer)(shareData_out, ptr) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemPoolPtrExportData *, CUdeviceptr))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportPointer)(__pyx_v_shareData_out, __pyx_v_ptr); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4477, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4478 + * raise RuntimeError('Function "cuMemPoolExportPointer" not found') + * err = ( __cuMemPoolExportPointer)(shareData_out, ptr) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemPoolImportPointer(CUdeviceptr* ptr_out, CUmemoryPool pool, CUmemPoolPtrExportData* shareData) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4471 + * return err + * + * cdef CUresult _cuMemPoolExportPointer(CUmemPoolPtrExportData* shareData_out, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolExportPointer + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemPoolExportPointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4480 + * return err + * + * cdef CUresult _cuMemPoolImportPointer(CUdeviceptr* ptr_out, CUmemoryPool pool, CUmemPoolPtrExportData* shareData) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolImportPointer + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolImportPointer(CUdeviceptr *__pyx_v_ptr_out, CUmemoryPool __pyx_v_pool, CUmemPoolPtrExportData *__pyx_v_shareData) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemPoolImportPointer", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4482 + * cdef CUresult _cuMemPoolImportPointer(CUdeviceptr* ptr_out, CUmemoryPool pool, CUmemPoolPtrExportData* shareData) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemPoolImportPointer + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemPoolImportPointer == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4482, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4483 + * global __cuMemPoolImportPointer + * cuPythonInit() + * if __cuMemPoolImportPointer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolImportPointer" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportPointer == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4484 + * cuPythonInit() + * if __cuMemPoolImportPointer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolImportPointer" not found') + * err = ( __cuMemPoolImportPointer)(ptr_out, pool, shareData) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4485 + * if __cuMemPoolImportPointer == NULL: + * with gil: + * raise RuntimeError('Function "cuMemPoolImportPointer" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemPoolImportPointer)(ptr_out, pool, shareData) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemPoolImportPointer}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4485, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4485, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4484 + * cuPythonInit() + * if __cuMemPoolImportPointer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPoolImportPointer" not found') + * err = ( __cuMemPoolImportPointer)(ptr_out, pool, shareData) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4483 + * global __cuMemPoolImportPointer + * cuPythonInit() + * if __cuMemPoolImportPointer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPoolImportPointer" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4486 + * with gil: + * raise RuntimeError('Function "cuMemPoolImportPointer" not found') + * err = ( __cuMemPoolImportPointer)(ptr_out, pool, shareData) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, CUmemoryPool, CUmemPoolPtrExportData *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportPointer)(__pyx_v_ptr_out, __pyx_v_pool, __pyx_v_shareData); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4486, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4487 + * raise RuntimeError('Function "cuMemPoolImportPointer" not found') + * err = ( __cuMemPoolImportPointer)(ptr_out, pool, shareData) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMulticastCreate(CUmemGenericAllocationHandle* mcHandle, const CUmulticastObjectProp* prop) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4480 + * return err + * + * cdef CUresult _cuMemPoolImportPointer(CUdeviceptr* ptr_out, CUmemoryPool pool, CUmemPoolPtrExportData* shareData) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPoolImportPointer + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemPoolImportPointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4489 + * return err + * + * cdef CUresult _cuMulticastCreate(CUmemGenericAllocationHandle* mcHandle, const CUmulticastObjectProp* prop) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMulticastCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMulticastCreate(CUmemGenericAllocationHandle *__pyx_v_mcHandle, CUmulticastObjectProp const *__pyx_v_prop) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMulticastCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4491 + * cdef CUresult _cuMulticastCreate(CUmemGenericAllocationHandle* mcHandle, const CUmulticastObjectProp* prop) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMulticastCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMulticastCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4491, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4492 + * global __cuMulticastCreate + * cuPythonInit() + * if __cuMulticastCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMulticastCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4493 + * cuPythonInit() + * if __cuMulticastCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMulticastCreate" not found') + * err = ( __cuMulticastCreate)(mcHandle, prop) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4494 + * if __cuMulticastCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuMulticastCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuMulticastCreate)(mcHandle, prop) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMulticastCreate_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4494, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4494, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4493 + * cuPythonInit() + * if __cuMulticastCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMulticastCreate" not found') + * err = ( __cuMulticastCreate)(mcHandle, prop) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4492 + * global __cuMulticastCreate + * cuPythonInit() + * if __cuMulticastCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMulticastCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4495 + * with gil: + * raise RuntimeError('Function "cuMulticastCreate" not found') + * err = ( __cuMulticastCreate)(mcHandle, prop) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemGenericAllocationHandle *, CUmulticastObjectProp const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastCreate)(__pyx_v_mcHandle, __pyx_v_prop); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4495, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4496 + * raise RuntimeError('Function "cuMulticastCreate" not found') + * err = ( __cuMulticastCreate)(mcHandle, prop) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMulticastAddDevice(CUmemGenericAllocationHandle mcHandle, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4489 + * return err + * + * cdef CUresult _cuMulticastCreate(CUmemGenericAllocationHandle* mcHandle, const CUmulticastObjectProp* prop) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMulticastCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMulticastCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4498 + * return err + * + * cdef CUresult _cuMulticastAddDevice(CUmemGenericAllocationHandle mcHandle, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMulticastAddDevice + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMulticastAddDevice(CUmemGenericAllocationHandle __pyx_v_mcHandle, CUdevice __pyx_v_dev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMulticastAddDevice", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4500 + * cdef CUresult _cuMulticastAddDevice(CUmemGenericAllocationHandle mcHandle, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMulticastAddDevice + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMulticastAddDevice == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4500, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4501 + * global __cuMulticastAddDevice + * cuPythonInit() + * if __cuMulticastAddDevice == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMulticastAddDevice" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastAddDevice == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4502 + * cuPythonInit() + * if __cuMulticastAddDevice == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMulticastAddDevice" not found') + * err = ( __cuMulticastAddDevice)(mcHandle, dev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4503 + * if __cuMulticastAddDevice == NULL: + * with gil: + * raise RuntimeError('Function "cuMulticastAddDevice" not found') # <<<<<<<<<<<<<< + * err = ( __cuMulticastAddDevice)(mcHandle, dev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMulticastAddDevice_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4503, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4503, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4502 + * cuPythonInit() + * if __cuMulticastAddDevice == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMulticastAddDevice" not found') + * err = ( __cuMulticastAddDevice)(mcHandle, dev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4501 + * global __cuMulticastAddDevice + * cuPythonInit() + * if __cuMulticastAddDevice == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMulticastAddDevice" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4504 + * with gil: + * raise RuntimeError('Function "cuMulticastAddDevice" not found') + * err = ( __cuMulticastAddDevice)(mcHandle, dev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemGenericAllocationHandle, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastAddDevice)(__pyx_v_mcHandle, __pyx_v_dev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4504, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4505 + * raise RuntimeError('Function "cuMulticastAddDevice" not found') + * err = ( __cuMulticastAddDevice)(mcHandle, dev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMulticastBindMem(CUmemGenericAllocationHandle mcHandle, size_t mcOffset, CUmemGenericAllocationHandle memHandle, size_t memOffset, size_t size, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4498 + * return err + * + * cdef CUresult _cuMulticastAddDevice(CUmemGenericAllocationHandle mcHandle, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMulticastAddDevice + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMulticastAddDevice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4507 + * return err + * + * cdef CUresult _cuMulticastBindMem(CUmemGenericAllocationHandle mcHandle, size_t mcOffset, CUmemGenericAllocationHandle memHandle, size_t memOffset, size_t size, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMulticastBindMem + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMulticastBindMem(CUmemGenericAllocationHandle __pyx_v_mcHandle, size_t __pyx_v_mcOffset, CUmemGenericAllocationHandle __pyx_v_memHandle, size_t __pyx_v_memOffset, size_t __pyx_v_size, unsigned PY_LONG_LONG __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMulticastBindMem", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4509 + * cdef CUresult _cuMulticastBindMem(CUmemGenericAllocationHandle mcHandle, size_t mcOffset, CUmemGenericAllocationHandle memHandle, size_t memOffset, size_t size, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMulticastBindMem + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMulticastBindMem == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4509, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4510 + * global __cuMulticastBindMem + * cuPythonInit() + * if __cuMulticastBindMem == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMulticastBindMem" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindMem == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4511 + * cuPythonInit() + * if __cuMulticastBindMem == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMulticastBindMem" not found') + * err = ( __cuMulticastBindMem)(mcHandle, mcOffset, memHandle, memOffset, size, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4512 + * if __cuMulticastBindMem == NULL: + * with gil: + * raise RuntimeError('Function "cuMulticastBindMem" not found') # <<<<<<<<<<<<<< + * err = ( __cuMulticastBindMem)(mcHandle, mcOffset, memHandle, memOffset, size, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMulticastBindMem_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4512, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4512, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4511 + * cuPythonInit() + * if __cuMulticastBindMem == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMulticastBindMem" not found') + * err = ( __cuMulticastBindMem)(mcHandle, mcOffset, memHandle, memOffset, size, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4510 + * global __cuMulticastBindMem + * cuPythonInit() + * if __cuMulticastBindMem == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMulticastBindMem" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4513 + * with gil: + * raise RuntimeError('Function "cuMulticastBindMem" not found') + * err = ( __cuMulticastBindMem)(mcHandle, mcOffset, memHandle, memOffset, size, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemGenericAllocationHandle, size_t, CUmemGenericAllocationHandle, size_t, size_t, unsigned PY_LONG_LONG))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindMem)(__pyx_v_mcHandle, __pyx_v_mcOffset, __pyx_v_memHandle, __pyx_v_memOffset, __pyx_v_size, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4513, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4514 + * raise RuntimeError('Function "cuMulticastBindMem" not found') + * err = ( __cuMulticastBindMem)(mcHandle, mcOffset, memHandle, memOffset, size, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMulticastBindAddr(CUmemGenericAllocationHandle mcHandle, size_t mcOffset, CUdeviceptr memptr, size_t size, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4507 + * return err + * + * cdef CUresult _cuMulticastBindMem(CUmemGenericAllocationHandle mcHandle, size_t mcOffset, CUmemGenericAllocationHandle memHandle, size_t memOffset, size_t size, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMulticastBindMem + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMulticastBindMem", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4516 + * return err + * + * cdef CUresult _cuMulticastBindAddr(CUmemGenericAllocationHandle mcHandle, size_t mcOffset, CUdeviceptr memptr, size_t size, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMulticastBindAddr + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMulticastBindAddr(CUmemGenericAllocationHandle __pyx_v_mcHandle, size_t __pyx_v_mcOffset, CUdeviceptr __pyx_v_memptr, size_t __pyx_v_size, unsigned PY_LONG_LONG __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMulticastBindAddr", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4518 + * cdef CUresult _cuMulticastBindAddr(CUmemGenericAllocationHandle mcHandle, size_t mcOffset, CUdeviceptr memptr, size_t size, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMulticastBindAddr + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMulticastBindAddr == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4518, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4519 + * global __cuMulticastBindAddr + * cuPythonInit() + * if __cuMulticastBindAddr == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMulticastBindAddr" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindAddr == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4520 + * cuPythonInit() + * if __cuMulticastBindAddr == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMulticastBindAddr" not found') + * err = ( __cuMulticastBindAddr)(mcHandle, mcOffset, memptr, size, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4521 + * if __cuMulticastBindAddr == NULL: + * with gil: + * raise RuntimeError('Function "cuMulticastBindAddr" not found') # <<<<<<<<<<<<<< + * err = ( __cuMulticastBindAddr)(mcHandle, mcOffset, memptr, size, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMulticastBindAddr_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4521, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4521, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4520 + * cuPythonInit() + * if __cuMulticastBindAddr == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMulticastBindAddr" not found') + * err = ( __cuMulticastBindAddr)(mcHandle, mcOffset, memptr, size, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4519 + * global __cuMulticastBindAddr + * cuPythonInit() + * if __cuMulticastBindAddr == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMulticastBindAddr" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4522 + * with gil: + * raise RuntimeError('Function "cuMulticastBindAddr" not found') + * err = ( __cuMulticastBindAddr)(mcHandle, mcOffset, memptr, size, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemGenericAllocationHandle, size_t, CUdeviceptr, size_t, unsigned PY_LONG_LONG))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindAddr)(__pyx_v_mcHandle, __pyx_v_mcOffset, __pyx_v_memptr, __pyx_v_size, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4522, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4523 + * raise RuntimeError('Function "cuMulticastBindAddr" not found') + * err = ( __cuMulticastBindAddr)(mcHandle, mcOffset, memptr, size, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMulticastUnbind(CUmemGenericAllocationHandle mcHandle, CUdevice dev, size_t mcOffset, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4516 + * return err + * + * cdef CUresult _cuMulticastBindAddr(CUmemGenericAllocationHandle mcHandle, size_t mcOffset, CUdeviceptr memptr, size_t size, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMulticastBindAddr + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMulticastBindAddr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4525 + * return err + * + * cdef CUresult _cuMulticastUnbind(CUmemGenericAllocationHandle mcHandle, CUdevice dev, size_t mcOffset, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMulticastUnbind + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMulticastUnbind(CUmemGenericAllocationHandle __pyx_v_mcHandle, CUdevice __pyx_v_dev, size_t __pyx_v_mcOffset, size_t __pyx_v_size) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMulticastUnbind", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4527 + * cdef CUresult _cuMulticastUnbind(CUmemGenericAllocationHandle mcHandle, CUdevice dev, size_t mcOffset, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMulticastUnbind + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMulticastUnbind == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4527, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4528 + * global __cuMulticastUnbind + * cuPythonInit() + * if __cuMulticastUnbind == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMulticastUnbind" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastUnbind == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4529 + * cuPythonInit() + * if __cuMulticastUnbind == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMulticastUnbind" not found') + * err = ( __cuMulticastUnbind)(mcHandle, dev, mcOffset, size) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4530 + * if __cuMulticastUnbind == NULL: + * with gil: + * raise RuntimeError('Function "cuMulticastUnbind" not found') # <<<<<<<<<<<<<< + * err = ( __cuMulticastUnbind)(mcHandle, dev, mcOffset, size) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMulticastUnbind_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4530, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4530, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4529 + * cuPythonInit() + * if __cuMulticastUnbind == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMulticastUnbind" not found') + * err = ( __cuMulticastUnbind)(mcHandle, dev, mcOffset, size) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4528 + * global __cuMulticastUnbind + * cuPythonInit() + * if __cuMulticastUnbind == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMulticastUnbind" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4531 + * with gil: + * raise RuntimeError('Function "cuMulticastUnbind" not found') + * err = ( __cuMulticastUnbind)(mcHandle, dev, mcOffset, size) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmemGenericAllocationHandle, CUdevice, size_t, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastUnbind)(__pyx_v_mcHandle, __pyx_v_dev, __pyx_v_mcOffset, __pyx_v_size); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4531, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4532 + * raise RuntimeError('Function "cuMulticastUnbind" not found') + * err = ( __cuMulticastUnbind)(mcHandle, dev, mcOffset, size) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMulticastGetGranularity(size_t* granularity, const CUmulticastObjectProp* prop, CUmulticastGranularity_flags option) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4525 + * return err + * + * cdef CUresult _cuMulticastUnbind(CUmemGenericAllocationHandle mcHandle, CUdevice dev, size_t mcOffset, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMulticastUnbind + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMulticastUnbind", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4534 + * return err + * + * cdef CUresult _cuMulticastGetGranularity(size_t* granularity, const CUmulticastObjectProp* prop, CUmulticastGranularity_flags option) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMulticastGetGranularity + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMulticastGetGranularity(size_t *__pyx_v_granularity, CUmulticastObjectProp const *__pyx_v_prop, CUmulticastGranularity_flags __pyx_v_option) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMulticastGetGranularity", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4536 + * cdef CUresult _cuMulticastGetGranularity(size_t* granularity, const CUmulticastObjectProp* prop, CUmulticastGranularity_flags option) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMulticastGetGranularity + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMulticastGetGranularity == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4536, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4537 + * global __cuMulticastGetGranularity + * cuPythonInit() + * if __cuMulticastGetGranularity == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMulticastGetGranularity" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastGetGranularity == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4538 + * cuPythonInit() + * if __cuMulticastGetGranularity == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMulticastGetGranularity" not found') + * err = ( __cuMulticastGetGranularity)(granularity, prop, option) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4539 + * if __cuMulticastGetGranularity == NULL: + * with gil: + * raise RuntimeError('Function "cuMulticastGetGranularity" not found') # <<<<<<<<<<<<<< + * err = ( __cuMulticastGetGranularity)(granularity, prop, option) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMulticastGetGranulari}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4539, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4539, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4538 + * cuPythonInit() + * if __cuMulticastGetGranularity == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMulticastGetGranularity" not found') + * err = ( __cuMulticastGetGranularity)(granularity, prop, option) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4537 + * global __cuMulticastGetGranularity + * cuPythonInit() + * if __cuMulticastGetGranularity == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMulticastGetGranularity" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4540 + * with gil: + * raise RuntimeError('Function "cuMulticastGetGranularity" not found') + * err = ( __cuMulticastGetGranularity)(granularity, prop, option) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(size_t *, CUmulticastObjectProp const *, CUmulticastGranularity_flags))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastGetGranularity)(__pyx_v_granularity, __pyx_v_prop, __pyx_v_option); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4540, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4541 + * raise RuntimeError('Function "cuMulticastGetGranularity" not found') + * err = ( __cuMulticastGetGranularity)(granularity, prop, option) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuPointerGetAttribute(void* data, CUpointer_attribute attribute, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4534 + * return err + * + * cdef CUresult _cuMulticastGetGranularity(size_t* granularity, const CUmulticastObjectProp* prop, CUmulticastGranularity_flags option) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMulticastGetGranularity + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMulticastGetGranularity", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4543 + * return err + * + * cdef CUresult _cuPointerGetAttribute(void* data, CUpointer_attribute attribute, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuPointerGetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuPointerGetAttribute(void *__pyx_v_data, CUpointer_attribute __pyx_v_attribute, CUdeviceptr __pyx_v_ptr) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuPointerGetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4545 + * cdef CUresult _cuPointerGetAttribute(void* data, CUpointer_attribute attribute, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuPointerGetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuPointerGetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4545, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4546 + * global __cuPointerGetAttribute + * cuPythonInit() + * if __cuPointerGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuPointerGetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4547 + * cuPythonInit() + * if __cuPointerGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuPointerGetAttribute" not found') + * err = ( __cuPointerGetAttribute)(data, attribute, ptr) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4548 + * if __cuPointerGetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuPointerGetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuPointerGetAttribute)(data, attribute, ptr) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuPointerGetAttribute_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4548, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4548, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4547 + * cuPythonInit() + * if __cuPointerGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuPointerGetAttribute" not found') + * err = ( __cuPointerGetAttribute)(data, attribute, ptr) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4546 + * global __cuPointerGetAttribute + * cuPythonInit() + * if __cuPointerGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuPointerGetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4549 + * with gil: + * raise RuntimeError('Function "cuPointerGetAttribute" not found') + * err = ( __cuPointerGetAttribute)(data, attribute, ptr) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void *, CUpointer_attribute, CUdeviceptr))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttribute)(__pyx_v_data, __pyx_v_attribute, __pyx_v_ptr); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4549, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4550 + * raise RuntimeError('Function "cuPointerGetAttribute" not found') + * err = ( __cuPointerGetAttribute)(data, attribute, ptr) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUdevice dstDevice, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4543 + * return err + * + * cdef CUresult _cuPointerGetAttribute(void* data, CUpointer_attribute attribute, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuPointerGetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuPointerGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4552 + * return err + * + * cdef CUresult _cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUdevice dstDevice, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPrefetchAsync(CUdeviceptr __pyx_v_devPtr, size_t __pyx_v_count, CUdevice __pyx_v_dstDevice, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemPrefetchAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4554 + * cdef CUresult _cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUdevice dstDevice, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemPrefetchAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemPrefetchAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4554, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4555 + * global __cuMemPrefetchAsync + * cuPythonInit() + * if __cuMemPrefetchAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPrefetchAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4556 + * cuPythonInit() + * if __cuMemPrefetchAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPrefetchAsync" not found') + * err = ( __cuMemPrefetchAsync)(devPtr, count, dstDevice, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4557 + * if __cuMemPrefetchAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuMemPrefetchAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemPrefetchAsync)(devPtr, count, dstDevice, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemPrefetchAsync_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4557, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4557, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4556 + * cuPythonInit() + * if __cuMemPrefetchAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPrefetchAsync" not found') + * err = ( __cuMemPrefetchAsync)(devPtr, count, dstDevice, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4555 + * global __cuMemPrefetchAsync + * cuPythonInit() + * if __cuMemPrefetchAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPrefetchAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4558 + * with gil: + * raise RuntimeError('Function "cuMemPrefetchAsync" not found') + * err = ( __cuMemPrefetchAsync)(devPtr, count, dstDevice, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t, CUdevice, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync)(__pyx_v_devPtr, __pyx_v_count, __pyx_v_dstDevice, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4558, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4559 + * raise RuntimeError('Function "cuMemPrefetchAsync" not found') + * err = ( __cuMemPrefetchAsync)(devPtr, count, dstDevice, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemPrefetchAsync_v2(CUdeviceptr devPtr, size_t count, CUmemLocation location, unsigned int flags, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4552 + * return err + * + * cdef CUresult _cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUdevice dstDevice, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemPrefetchAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4561 + * return err + * + * cdef CUresult _cuMemPrefetchAsync_v2(CUdeviceptr devPtr, size_t count, CUmemLocation location, unsigned int flags, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPrefetchAsync_v2(CUdeviceptr __pyx_v_devPtr, size_t __pyx_v_count, CUmemLocation __pyx_v_location, unsigned int __pyx_v_flags, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemPrefetchAsync_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4563 + * cdef CUresult _cuMemPrefetchAsync_v2(CUdeviceptr devPtr, size_t count, CUmemLocation location, unsigned int flags, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemPrefetchAsync_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemPrefetchAsync_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4563, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4564 + * global __cuMemPrefetchAsync_v2 + * cuPythonInit() + * if __cuMemPrefetchAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPrefetchAsync_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4565 + * cuPythonInit() + * if __cuMemPrefetchAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPrefetchAsync_v2" not found') + * err = ( __cuMemPrefetchAsync_v2)(devPtr, count, location, flags, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4566 + * if __cuMemPrefetchAsync_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemPrefetchAsync_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemPrefetchAsync_v2)(devPtr, count, location, flags, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemPrefetchAsync_v2_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4566, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4566, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4565 + * cuPythonInit() + * if __cuMemPrefetchAsync_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemPrefetchAsync_v2" not found') + * err = ( __cuMemPrefetchAsync_v2)(devPtr, count, location, flags, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4564 + * global __cuMemPrefetchAsync_v2 + * cuPythonInit() + * if __cuMemPrefetchAsync_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemPrefetchAsync_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4567 + * with gil: + * raise RuntimeError('Function "cuMemPrefetchAsync_v2" not found') + * err = ( __cuMemPrefetchAsync_v2)(devPtr, count, location, flags, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t, CUmemLocation, unsigned int, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync_v2)(__pyx_v_devPtr, __pyx_v_count, __pyx_v_location, __pyx_v_flags, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4567, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4568 + * raise RuntimeError('Function "cuMemPrefetchAsync_v2" not found') + * err = ( __cuMemPrefetchAsync_v2)(devPtr, count, location, flags, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemAdvise(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4561 + * return err + * + * cdef CUresult _cuMemPrefetchAsync_v2(CUdeviceptr devPtr, size_t count, CUmemLocation location, unsigned int flags, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemPrefetchAsync_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4570 + * return err + * + * cdef CUresult _cuMemAdvise(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAdvise + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAdvise(CUdeviceptr __pyx_v_devPtr, size_t __pyx_v_count, CUmem_advise __pyx_v_advice, CUdevice __pyx_v_device) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemAdvise", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4572 + * cdef CUresult _cuMemAdvise(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemAdvise + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemAdvise == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4572, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4573 + * global __cuMemAdvise + * cuPythonInit() + * if __cuMemAdvise == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAdvise" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4574 + * cuPythonInit() + * if __cuMemAdvise == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAdvise" not found') + * err = ( __cuMemAdvise)(devPtr, count, advice, device) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4575 + * if __cuMemAdvise == NULL: + * with gil: + * raise RuntimeError('Function "cuMemAdvise" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemAdvise)(devPtr, count, advice, device) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemAdvise_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4575, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4575, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4574 + * cuPythonInit() + * if __cuMemAdvise == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAdvise" not found') + * err = ( __cuMemAdvise)(devPtr, count, advice, device) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4573 + * global __cuMemAdvise + * cuPythonInit() + * if __cuMemAdvise == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAdvise" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4576 + * with gil: + * raise RuntimeError('Function "cuMemAdvise" not found') + * err = ( __cuMemAdvise)(devPtr, count, advice, device) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t, CUmem_advise, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise)(__pyx_v_devPtr, __pyx_v_count, __pyx_v_advice, __pyx_v_device); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4576, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4577 + * raise RuntimeError('Function "cuMemAdvise" not found') + * err = ( __cuMemAdvise)(devPtr, count, advice, device) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemAdvise_v2(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUmemLocation location) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4570 + * return err + * + * cdef CUresult _cuMemAdvise(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAdvise + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemAdvise", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4579 + * return err + * + * cdef CUresult _cuMemAdvise_v2(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUmemLocation location) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAdvise_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAdvise_v2(CUdeviceptr __pyx_v_devPtr, size_t __pyx_v_count, CUmem_advise __pyx_v_advice, CUmemLocation __pyx_v_location) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemAdvise_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4581 + * cdef CUresult _cuMemAdvise_v2(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUmemLocation location) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemAdvise_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemAdvise_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4581, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4582 + * global __cuMemAdvise_v2 + * cuPythonInit() + * if __cuMemAdvise_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAdvise_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4583 + * cuPythonInit() + * if __cuMemAdvise_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAdvise_v2" not found') + * err = ( __cuMemAdvise_v2)(devPtr, count, advice, location) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4584 + * if __cuMemAdvise_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuMemAdvise_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemAdvise_v2)(devPtr, count, advice, location) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemAdvise_v2_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4584, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4584, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4583 + * cuPythonInit() + * if __cuMemAdvise_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemAdvise_v2" not found') + * err = ( __cuMemAdvise_v2)(devPtr, count, advice, location) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4582 + * global __cuMemAdvise_v2 + * cuPythonInit() + * if __cuMemAdvise_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemAdvise_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4585 + * with gil: + * raise RuntimeError('Function "cuMemAdvise_v2" not found') + * err = ( __cuMemAdvise_v2)(devPtr, count, advice, location) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr, size_t, CUmem_advise, CUmemLocation))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise_v2)(__pyx_v_devPtr, __pyx_v_count, __pyx_v_advice, __pyx_v_location); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4585, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4586 + * raise RuntimeError('Function "cuMemAdvise_v2" not found') + * err = ( __cuMemAdvise_v2)(devPtr, count, advice, location) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemRangeGetAttribute(void* data, size_t dataSize, CUmem_range_attribute attribute, CUdeviceptr devPtr, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4579 + * return err + * + * cdef CUresult _cuMemAdvise_v2(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUmemLocation location) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemAdvise_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemAdvise_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4588 + * return err + * + * cdef CUresult _cuMemRangeGetAttribute(void* data, size_t dataSize, CUmem_range_attribute attribute, CUdeviceptr devPtr, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemRangeGetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemRangeGetAttribute(void *__pyx_v_data, size_t __pyx_v_dataSize, CUmem_range_attribute __pyx_v_attribute, CUdeviceptr __pyx_v_devPtr, size_t __pyx_v_count) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemRangeGetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4590 + * cdef CUresult _cuMemRangeGetAttribute(void* data, size_t dataSize, CUmem_range_attribute attribute, CUdeviceptr devPtr, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemRangeGetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemRangeGetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4590, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4591 + * global __cuMemRangeGetAttribute + * cuPythonInit() + * if __cuMemRangeGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemRangeGetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4592 + * cuPythonInit() + * if __cuMemRangeGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemRangeGetAttribute" not found') + * err = ( __cuMemRangeGetAttribute)(data, dataSize, attribute, devPtr, count) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4593 + * if __cuMemRangeGetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuMemRangeGetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemRangeGetAttribute)(data, dataSize, attribute, devPtr, count) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemRangeGetAttribute}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4593, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4593, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4592 + * cuPythonInit() + * if __cuMemRangeGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemRangeGetAttribute" not found') + * err = ( __cuMemRangeGetAttribute)(data, dataSize, attribute, devPtr, count) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4591 + * global __cuMemRangeGetAttribute + * cuPythonInit() + * if __cuMemRangeGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemRangeGetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4594 + * with gil: + * raise RuntimeError('Function "cuMemRangeGetAttribute" not found') + * err = ( __cuMemRangeGetAttribute)(data, dataSize, attribute, devPtr, count) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void *, size_t, CUmem_range_attribute, CUdeviceptr, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttribute)(__pyx_v_data, __pyx_v_dataSize, __pyx_v_attribute, __pyx_v_devPtr, __pyx_v_count); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4594, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4595 + * raise RuntimeError('Function "cuMemRangeGetAttribute" not found') + * err = ( __cuMemRangeGetAttribute)(data, dataSize, attribute, devPtr, count) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuMemRangeGetAttributes(void** data, size_t* dataSizes, CUmem_range_attribute* attributes, size_t numAttributes, CUdeviceptr devPtr, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4588 + * return err + * + * cdef CUresult _cuMemRangeGetAttribute(void* data, size_t dataSize, CUmem_range_attribute attribute, CUdeviceptr devPtr, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemRangeGetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemRangeGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4597 + * return err + * + * cdef CUresult _cuMemRangeGetAttributes(void** data, size_t* dataSizes, CUmem_range_attribute* attributes, size_t numAttributes, CUdeviceptr devPtr, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemRangeGetAttributes + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemRangeGetAttributes(void **__pyx_v_data, size_t *__pyx_v_dataSizes, CUmem_range_attribute *__pyx_v_attributes, size_t __pyx_v_numAttributes, CUdeviceptr __pyx_v_devPtr, size_t __pyx_v_count) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuMemRangeGetAttributes", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4599 + * cdef CUresult _cuMemRangeGetAttributes(void** data, size_t* dataSizes, CUmem_range_attribute* attributes, size_t numAttributes, CUdeviceptr devPtr, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuMemRangeGetAttributes + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuMemRangeGetAttributes == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4599, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4600 + * global __cuMemRangeGetAttributes + * cuPythonInit() + * if __cuMemRangeGetAttributes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemRangeGetAttributes" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttributes == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4601 + * cuPythonInit() + * if __cuMemRangeGetAttributes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemRangeGetAttributes" not found') + * err = ( __cuMemRangeGetAttributes)(data, dataSizes, attributes, numAttributes, devPtr, count) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4602 + * if __cuMemRangeGetAttributes == NULL: + * with gil: + * raise RuntimeError('Function "cuMemRangeGetAttributes" not found') # <<<<<<<<<<<<<< + * err = ( __cuMemRangeGetAttributes)(data, dataSizes, attributes, numAttributes, devPtr, count) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuMemRangeGetAttributes}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4602, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4602, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4601 + * cuPythonInit() + * if __cuMemRangeGetAttributes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuMemRangeGetAttributes" not found') + * err = ( __cuMemRangeGetAttributes)(data, dataSizes, attributes, numAttributes, devPtr, count) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4600 + * global __cuMemRangeGetAttributes + * cuPythonInit() + * if __cuMemRangeGetAttributes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuMemRangeGetAttributes" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4603 + * with gil: + * raise RuntimeError('Function "cuMemRangeGetAttributes" not found') + * err = ( __cuMemRangeGetAttributes)(data, dataSizes, attributes, numAttributes, devPtr, count) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void **, size_t *, CUmem_range_attribute *, size_t, CUdeviceptr, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttributes)(__pyx_v_data, __pyx_v_dataSizes, __pyx_v_attributes, __pyx_v_numAttributes, __pyx_v_devPtr, __pyx_v_count); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4603, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4604 + * raise RuntimeError('Function "cuMemRangeGetAttributes" not found') + * err = ( __cuMemRangeGetAttributes)(data, dataSizes, attributes, numAttributes, devPtr, count) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuPointerSetAttribute(const void* value, CUpointer_attribute attribute, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4597 + * return err + * + * cdef CUresult _cuMemRangeGetAttributes(void** data, size_t* dataSizes, CUmem_range_attribute* attributes, size_t numAttributes, CUdeviceptr devPtr, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuMemRangeGetAttributes + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuMemRangeGetAttributes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4606 + * return err + * + * cdef CUresult _cuPointerSetAttribute(const void* value, CUpointer_attribute attribute, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuPointerSetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuPointerSetAttribute(void const *__pyx_v_value, CUpointer_attribute __pyx_v_attribute, CUdeviceptr __pyx_v_ptr) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuPointerSetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4608 + * cdef CUresult _cuPointerSetAttribute(const void* value, CUpointer_attribute attribute, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuPointerSetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuPointerSetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4608, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4609 + * global __cuPointerSetAttribute + * cuPythonInit() + * if __cuPointerSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuPointerSetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerSetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4610 + * cuPythonInit() + * if __cuPointerSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuPointerSetAttribute" not found') + * err = ( __cuPointerSetAttribute)(value, attribute, ptr) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4611 + * if __cuPointerSetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuPointerSetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuPointerSetAttribute)(value, attribute, ptr) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuPointerSetAttribute_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4611, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4611, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4610 + * cuPythonInit() + * if __cuPointerSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuPointerSetAttribute" not found') + * err = ( __cuPointerSetAttribute)(value, attribute, ptr) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4609 + * global __cuPointerSetAttribute + * cuPythonInit() + * if __cuPointerSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuPointerSetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4612 + * with gil: + * raise RuntimeError('Function "cuPointerSetAttribute" not found') + * err = ( __cuPointerSetAttribute)(value, attribute, ptr) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void const *, CUpointer_attribute, CUdeviceptr))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerSetAttribute)(__pyx_v_value, __pyx_v_attribute, __pyx_v_ptr); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4612, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4613 + * raise RuntimeError('Function "cuPointerSetAttribute" not found') + * err = ( __cuPointerSetAttribute)(value, attribute, ptr) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuPointerGetAttributes(unsigned int numAttributes, CUpointer_attribute* attributes, void** data, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4606 + * return err + * + * cdef CUresult _cuPointerSetAttribute(const void* value, CUpointer_attribute attribute, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuPointerSetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuPointerSetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4615 + * return err + * + * cdef CUresult _cuPointerGetAttributes(unsigned int numAttributes, CUpointer_attribute* attributes, void** data, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuPointerGetAttributes + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuPointerGetAttributes(unsigned int __pyx_v_numAttributes, CUpointer_attribute *__pyx_v_attributes, void **__pyx_v_data, CUdeviceptr __pyx_v_ptr) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuPointerGetAttributes", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4617 + * cdef CUresult _cuPointerGetAttributes(unsigned int numAttributes, CUpointer_attribute* attributes, void** data, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuPointerGetAttributes + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuPointerGetAttributes == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4617, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4618 + * global __cuPointerGetAttributes + * cuPythonInit() + * if __cuPointerGetAttributes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuPointerGetAttributes" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttributes == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4619 + * cuPythonInit() + * if __cuPointerGetAttributes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuPointerGetAttributes" not found') + * err = ( __cuPointerGetAttributes)(numAttributes, attributes, data, ptr) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4620 + * if __cuPointerGetAttributes == NULL: + * with gil: + * raise RuntimeError('Function "cuPointerGetAttributes" not found') # <<<<<<<<<<<<<< + * err = ( __cuPointerGetAttributes)(numAttributes, attributes, data, ptr) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuPointerGetAttributes}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4620, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4620, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4619 + * cuPythonInit() + * if __cuPointerGetAttributes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuPointerGetAttributes" not found') + * err = ( __cuPointerGetAttributes)(numAttributes, attributes, data, ptr) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4618 + * global __cuPointerGetAttributes + * cuPythonInit() + * if __cuPointerGetAttributes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuPointerGetAttributes" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4621 + * with gil: + * raise RuntimeError('Function "cuPointerGetAttributes" not found') + * err = ( __cuPointerGetAttributes)(numAttributes, attributes, data, ptr) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(unsigned int, CUpointer_attribute *, void **, CUdeviceptr))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttributes)(__pyx_v_numAttributes, __pyx_v_attributes, __pyx_v_data, __pyx_v_ptr); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4621, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4622 + * raise RuntimeError('Function "cuPointerGetAttributes" not found') + * err = ( __cuPointerGetAttributes)(numAttributes, attributes, data, ptr) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamCreate(CUstream* phStream, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4615 + * return err + * + * cdef CUresult _cuPointerGetAttributes(unsigned int numAttributes, CUpointer_attribute* attributes, void** data, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuPointerGetAttributes + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuPointerGetAttributes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4624 + * return err + * + * cdef CUresult _cuStreamCreate(CUstream* phStream, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamCreate(CUstream *__pyx_v_phStream, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4626 + * cdef CUresult _cuStreamCreate(CUstream* phStream, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4626, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4627 + * global __cuStreamCreate + * cuPythonInit() + * if __cuStreamCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4628 + * cuPythonInit() + * if __cuStreamCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamCreate" not found') + * err = ( __cuStreamCreate)(phStream, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4629 + * if __cuStreamCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamCreate)(phStream, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamCreate_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4629, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4629, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4628 + * cuPythonInit() + * if __cuStreamCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamCreate" not found') + * err = ( __cuStreamCreate)(phStream, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4627 + * global __cuStreamCreate + * cuPythonInit() + * if __cuStreamCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4630 + * with gil: + * raise RuntimeError('Function "cuStreamCreate" not found') + * err = ( __cuStreamCreate)(phStream, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreate)(__pyx_v_phStream, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4630, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4631 + * raise RuntimeError('Function "cuStreamCreate" not found') + * err = ( __cuStreamCreate)(phStream, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamCreateWithPriority(CUstream* phStream, unsigned int flags, int priority) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4624 + * return err + * + * cdef CUresult _cuStreamCreate(CUstream* phStream, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4633 + * return err + * + * cdef CUresult _cuStreamCreateWithPriority(CUstream* phStream, unsigned int flags, int priority) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamCreateWithPriority + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamCreateWithPriority(CUstream *__pyx_v_phStream, unsigned int __pyx_v_flags, int __pyx_v_priority) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamCreateWithPriority", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4635 + * cdef CUresult _cuStreamCreateWithPriority(CUstream* phStream, unsigned int flags, int priority) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamCreateWithPriority + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamCreateWithPriority == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4635, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4636 + * global __cuStreamCreateWithPriority + * cuPythonInit() + * if __cuStreamCreateWithPriority == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamCreateWithPriority" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreateWithPriority == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4637 + * cuPythonInit() + * if __cuStreamCreateWithPriority == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamCreateWithPriority" not found') + * err = ( __cuStreamCreateWithPriority)(phStream, flags, priority) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4638 + * if __cuStreamCreateWithPriority == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamCreateWithPriority" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamCreateWithPriority)(phStream, flags, priority) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamCreateWithPrior}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4638, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4638, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4637 + * cuPythonInit() + * if __cuStreamCreateWithPriority == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamCreateWithPriority" not found') + * err = ( __cuStreamCreateWithPriority)(phStream, flags, priority) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4636 + * global __cuStreamCreateWithPriority + * cuPythonInit() + * if __cuStreamCreateWithPriority == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamCreateWithPriority" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4639 + * with gil: + * raise RuntimeError('Function "cuStreamCreateWithPriority" not found') + * err = ( __cuStreamCreateWithPriority)(phStream, flags, priority) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream *, unsigned int, int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreateWithPriority)(__pyx_v_phStream, __pyx_v_flags, __pyx_v_priority); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4639, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4640 + * raise RuntimeError('Function "cuStreamCreateWithPriority" not found') + * err = ( __cuStreamCreateWithPriority)(phStream, flags, priority) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamGetPriority(CUstream hStream, int* priority) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4633 + * return err + * + * cdef CUresult _cuStreamCreateWithPriority(CUstream* phStream, unsigned int flags, int priority) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamCreateWithPriority + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamCreateWithPriority", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4642 + * return err + * + * cdef CUresult _cuStreamGetPriority(CUstream hStream, int* priority) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetPriority + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetPriority(CUstream __pyx_v_hStream, int *__pyx_v_priority) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamGetPriority", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4644 + * cdef CUresult _cuStreamGetPriority(CUstream hStream, int* priority) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamGetPriority + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamGetPriority == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4644, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4645 + * global __cuStreamGetPriority + * cuPythonInit() + * if __cuStreamGetPriority == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetPriority" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetPriority == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4646 + * cuPythonInit() + * if __cuStreamGetPriority == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetPriority" not found') + * err = ( __cuStreamGetPriority)(hStream, priority) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4647 + * if __cuStreamGetPriority == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamGetPriority" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamGetPriority)(hStream, priority) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamGetPriority_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4647, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4647, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4646 + * cuPythonInit() + * if __cuStreamGetPriority == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetPriority" not found') + * err = ( __cuStreamGetPriority)(hStream, priority) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4645 + * global __cuStreamGetPriority + * cuPythonInit() + * if __cuStreamGetPriority == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetPriority" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4648 + * with gil: + * raise RuntimeError('Function "cuStreamGetPriority" not found') + * err = ( __cuStreamGetPriority)(hStream, priority) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, int *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetPriority)(__pyx_v_hStream, __pyx_v_priority); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4648, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4649 + * raise RuntimeError('Function "cuStreamGetPriority" not found') + * err = ( __cuStreamGetPriority)(hStream, priority) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamGetDevice(CUstream hStream, CUdevice* device) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4642 + * return err + * + * cdef CUresult _cuStreamGetPriority(CUstream hStream, int* priority) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetPriority + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamGetPriority", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4651 + * return err + * + * cdef CUresult _cuStreamGetDevice(CUstream hStream, CUdevice* device) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetDevice + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetDevice(CUstream __pyx_v_hStream, CUdevice *__pyx_v_device) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamGetDevice", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4653 + * cdef CUresult _cuStreamGetDevice(CUstream hStream, CUdevice* device) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamGetDevice + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamGetDevice == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4653, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4654 + * global __cuStreamGetDevice + * cuPythonInit() + * if __cuStreamGetDevice == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetDevice" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetDevice == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4655 + * cuPythonInit() + * if __cuStreamGetDevice == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetDevice" not found') + * err = ( __cuStreamGetDevice)(hStream, device) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4656 + * if __cuStreamGetDevice == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamGetDevice" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamGetDevice)(hStream, device) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamGetDevice_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4656, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4656, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4655 + * cuPythonInit() + * if __cuStreamGetDevice == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetDevice" not found') + * err = ( __cuStreamGetDevice)(hStream, device) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4654 + * global __cuStreamGetDevice + * cuPythonInit() + * if __cuStreamGetDevice == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetDevice" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4657 + * with gil: + * raise RuntimeError('Function "cuStreamGetDevice" not found') + * err = ( __cuStreamGetDevice)(hStream, device) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUdevice *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetDevice)(__pyx_v_hStream, __pyx_v_device); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4657, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4658 + * raise RuntimeError('Function "cuStreamGetDevice" not found') + * err = ( __cuStreamGetDevice)(hStream, device) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamGetFlags(CUstream hStream, unsigned int* flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4651 + * return err + * + * cdef CUresult _cuStreamGetDevice(CUstream hStream, CUdevice* device) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetDevice + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamGetDevice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4660 + * return err + * + * cdef CUresult _cuStreamGetFlags(CUstream hStream, unsigned int* flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetFlags + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetFlags(CUstream __pyx_v_hStream, unsigned int *__pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamGetFlags", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4662 + * cdef CUresult _cuStreamGetFlags(CUstream hStream, unsigned int* flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamGetFlags + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamGetFlags == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4662, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4663 + * global __cuStreamGetFlags + * cuPythonInit() + * if __cuStreamGetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetFlags" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetFlags == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4664 + * cuPythonInit() + * if __cuStreamGetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetFlags" not found') + * err = ( __cuStreamGetFlags)(hStream, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4665 + * if __cuStreamGetFlags == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamGetFlags" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamGetFlags)(hStream, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamGetFlags_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4665, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4665, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4664 + * cuPythonInit() + * if __cuStreamGetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetFlags" not found') + * err = ( __cuStreamGetFlags)(hStream, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4663 + * global __cuStreamGetFlags + * cuPythonInit() + * if __cuStreamGetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetFlags" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4666 + * with gil: + * raise RuntimeError('Function "cuStreamGetFlags" not found') + * err = ( __cuStreamGetFlags)(hStream, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, unsigned int *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetFlags)(__pyx_v_hStream, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4666, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4667 + * raise RuntimeError('Function "cuStreamGetFlags" not found') + * err = ( __cuStreamGetFlags)(hStream, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamGetId(CUstream hStream, unsigned long long* streamId) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4660 + * return err + * + * cdef CUresult _cuStreamGetFlags(CUstream hStream, unsigned int* flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetFlags + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamGetFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4669 + * return err + * + * cdef CUresult _cuStreamGetId(CUstream hStream, unsigned long long* streamId) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetId + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetId(CUstream __pyx_v_hStream, unsigned PY_LONG_LONG *__pyx_v_streamId) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamGetId", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4671 + * cdef CUresult _cuStreamGetId(CUstream hStream, unsigned long long* streamId) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamGetId + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamGetId == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4671, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4672 + * global __cuStreamGetId + * cuPythonInit() + * if __cuStreamGetId == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetId" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetId == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4673 + * cuPythonInit() + * if __cuStreamGetId == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetId" not found') + * err = ( __cuStreamGetId)(hStream, streamId) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4674 + * if __cuStreamGetId == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamGetId" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamGetId)(hStream, streamId) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamGetId_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4674, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4674, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4673 + * cuPythonInit() + * if __cuStreamGetId == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetId" not found') + * err = ( __cuStreamGetId)(hStream, streamId) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4672 + * global __cuStreamGetId + * cuPythonInit() + * if __cuStreamGetId == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetId" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4675 + * with gil: + * raise RuntimeError('Function "cuStreamGetId" not found') + * err = ( __cuStreamGetId)(hStream, streamId) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, unsigned PY_LONG_LONG *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetId)(__pyx_v_hStream, __pyx_v_streamId); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4675, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4676 + * raise RuntimeError('Function "cuStreamGetId" not found') + * err = ( __cuStreamGetId)(hStream, streamId) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamGetCtx(CUstream hStream, CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4669 + * return err + * + * cdef CUresult _cuStreamGetId(CUstream hStream, unsigned long long* streamId) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetId + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamGetId", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4678 + * return err + * + * cdef CUresult _cuStreamGetCtx(CUstream hStream, CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetCtx + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetCtx(CUstream __pyx_v_hStream, CUcontext *__pyx_v_pctx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamGetCtx", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4680 + * cdef CUresult _cuStreamGetCtx(CUstream hStream, CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamGetCtx + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamGetCtx == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4680, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4681 + * global __cuStreamGetCtx + * cuPythonInit() + * if __cuStreamGetCtx == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetCtx" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4682 + * cuPythonInit() + * if __cuStreamGetCtx == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetCtx" not found') + * err = ( __cuStreamGetCtx)(hStream, pctx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4683 + * if __cuStreamGetCtx == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamGetCtx" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamGetCtx)(hStream, pctx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamGetCtx_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4683, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4683, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4682 + * cuPythonInit() + * if __cuStreamGetCtx == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetCtx" not found') + * err = ( __cuStreamGetCtx)(hStream, pctx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4681 + * global __cuStreamGetCtx + * cuPythonInit() + * if __cuStreamGetCtx == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetCtx" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4684 + * with gil: + * raise RuntimeError('Function "cuStreamGetCtx" not found') + * err = ( __cuStreamGetCtx)(hStream, pctx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUcontext *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx)(__pyx_v_hStream, __pyx_v_pctx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4684, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4685 + * raise RuntimeError('Function "cuStreamGetCtx" not found') + * err = ( __cuStreamGetCtx)(hStream, pctx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamGetCtx_v2(CUstream hStream, CUcontext* pCtx, CUgreenCtx* pGreenCtx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4678 + * return err + * + * cdef CUresult _cuStreamGetCtx(CUstream hStream, CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetCtx + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamGetCtx", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4687 + * return err + * + * cdef CUresult _cuStreamGetCtx_v2(CUstream hStream, CUcontext* pCtx, CUgreenCtx* pGreenCtx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetCtx_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetCtx_v2(CUstream __pyx_v_hStream, CUcontext *__pyx_v_pCtx, CUgreenCtx *__pyx_v_pGreenCtx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamGetCtx_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4689 + * cdef CUresult _cuStreamGetCtx_v2(CUstream hStream, CUcontext* pCtx, CUgreenCtx* pGreenCtx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamGetCtx_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamGetCtx_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4689, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4690 + * global __cuStreamGetCtx_v2 + * cuPythonInit() + * if __cuStreamGetCtx_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetCtx_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4691 + * cuPythonInit() + * if __cuStreamGetCtx_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetCtx_v2" not found') + * err = ( __cuStreamGetCtx_v2)(hStream, pCtx, pGreenCtx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4692 + * if __cuStreamGetCtx_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamGetCtx_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamGetCtx_v2)(hStream, pCtx, pGreenCtx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamGetCtx_v2_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4692, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4692, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4691 + * cuPythonInit() + * if __cuStreamGetCtx_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetCtx_v2" not found') + * err = ( __cuStreamGetCtx_v2)(hStream, pCtx, pGreenCtx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4690 + * global __cuStreamGetCtx_v2 + * cuPythonInit() + * if __cuStreamGetCtx_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetCtx_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4693 + * with gil: + * raise RuntimeError('Function "cuStreamGetCtx_v2" not found') + * err = ( __cuStreamGetCtx_v2)(hStream, pCtx, pGreenCtx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUcontext *, CUgreenCtx *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx_v2)(__pyx_v_hStream, __pyx_v_pCtx, __pyx_v_pGreenCtx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4693, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4694 + * raise RuntimeError('Function "cuStreamGetCtx_v2" not found') + * err = ( __cuStreamGetCtx_v2)(hStream, pCtx, pGreenCtx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamWaitEvent(CUstream hStream, CUevent hEvent, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4687 + * return err + * + * cdef CUresult _cuStreamGetCtx_v2(CUstream hStream, CUcontext* pCtx, CUgreenCtx* pGreenCtx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetCtx_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamGetCtx_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4696 + * return err + * + * cdef CUresult _cuStreamWaitEvent(CUstream hStream, CUevent hEvent, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamWaitEvent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamWaitEvent(CUstream __pyx_v_hStream, CUevent __pyx_v_hEvent, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamWaitEvent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4698 + * cdef CUresult _cuStreamWaitEvent(CUstream hStream, CUevent hEvent, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamWaitEvent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamWaitEvent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4698, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4699 + * global __cuStreamWaitEvent + * cuPythonInit() + * if __cuStreamWaitEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamWaitEvent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitEvent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4700 + * cuPythonInit() + * if __cuStreamWaitEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamWaitEvent" not found') + * err = ( __cuStreamWaitEvent)(hStream, hEvent, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4701 + * if __cuStreamWaitEvent == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamWaitEvent" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamWaitEvent)(hStream, hEvent, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamWaitEvent_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4701, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4701, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4700 + * cuPythonInit() + * if __cuStreamWaitEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamWaitEvent" not found') + * err = ( __cuStreamWaitEvent)(hStream, hEvent, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4699 + * global __cuStreamWaitEvent + * cuPythonInit() + * if __cuStreamWaitEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamWaitEvent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4702 + * with gil: + * raise RuntimeError('Function "cuStreamWaitEvent" not found') + * err = ( __cuStreamWaitEvent)(hStream, hEvent, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUevent, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitEvent)(__pyx_v_hStream, __pyx_v_hEvent, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4702, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4703 + * raise RuntimeError('Function "cuStreamWaitEvent" not found') + * err = ( __cuStreamWaitEvent)(hStream, hEvent, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamAddCallback(CUstream hStream, CUstreamCallback callback, void* userData, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4696 + * return err + * + * cdef CUresult _cuStreamWaitEvent(CUstream hStream, CUevent hEvent, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamWaitEvent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamWaitEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4705 + * return err + * + * cdef CUresult _cuStreamAddCallback(CUstream hStream, CUstreamCallback callback, void* userData, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamAddCallback + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamAddCallback(CUstream __pyx_v_hStream, CUstreamCallback __pyx_v_callback, void *__pyx_v_userData, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamAddCallback", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4707 + * cdef CUresult _cuStreamAddCallback(CUstream hStream, CUstreamCallback callback, void* userData, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamAddCallback + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamAddCallback == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4707, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4708 + * global __cuStreamAddCallback + * cuPythonInit() + * if __cuStreamAddCallback == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamAddCallback" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAddCallback == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4709 + * cuPythonInit() + * if __cuStreamAddCallback == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamAddCallback" not found') + * err = ( __cuStreamAddCallback)(hStream, callback, userData, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4710 + * if __cuStreamAddCallback == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamAddCallback" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamAddCallback)(hStream, callback, userData, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamAddCallback_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4710, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4710, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4709 + * cuPythonInit() + * if __cuStreamAddCallback == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamAddCallback" not found') + * err = ( __cuStreamAddCallback)(hStream, callback, userData, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4708 + * global __cuStreamAddCallback + * cuPythonInit() + * if __cuStreamAddCallback == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamAddCallback" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4711 + * with gil: + * raise RuntimeError('Function "cuStreamAddCallback" not found') + * err = ( __cuStreamAddCallback)(hStream, callback, userData, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUstreamCallback, void *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAddCallback)(__pyx_v_hStream, __pyx_v_callback, __pyx_v_userData, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4711, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4712 + * raise RuntimeError('Function "cuStreamAddCallback" not found') + * err = ( __cuStreamAddCallback)(hStream, callback, userData, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamBeginCapture_v2(CUstream hStream, CUstreamCaptureMode mode) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4705 + * return err + * + * cdef CUresult _cuStreamAddCallback(CUstream hStream, CUstreamCallback callback, void* userData, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamAddCallback + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamAddCallback", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4714 + * return err + * + * cdef CUresult _cuStreamBeginCapture_v2(CUstream hStream, CUstreamCaptureMode mode) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamBeginCapture_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamBeginCapture_v2(CUstream __pyx_v_hStream, CUstreamCaptureMode __pyx_v_mode) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamBeginCapture_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4716 + * cdef CUresult _cuStreamBeginCapture_v2(CUstream hStream, CUstreamCaptureMode mode) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamBeginCapture_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamBeginCapture_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4716, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4717 + * global __cuStreamBeginCapture_v2 + * cuPythonInit() + * if __cuStreamBeginCapture_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamBeginCapture_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCapture_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4718 + * cuPythonInit() + * if __cuStreamBeginCapture_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamBeginCapture_v2" not found') + * err = ( __cuStreamBeginCapture_v2)(hStream, mode) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4719 + * if __cuStreamBeginCapture_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamBeginCapture_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamBeginCapture_v2)(hStream, mode) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamBeginCapture_v2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4719, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4719, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4718 + * cuPythonInit() + * if __cuStreamBeginCapture_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamBeginCapture_v2" not found') + * err = ( __cuStreamBeginCapture_v2)(hStream, mode) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4717 + * global __cuStreamBeginCapture_v2 + * cuPythonInit() + * if __cuStreamBeginCapture_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamBeginCapture_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4720 + * with gil: + * raise RuntimeError('Function "cuStreamBeginCapture_v2" not found') + * err = ( __cuStreamBeginCapture_v2)(hStream, mode) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUstreamCaptureMode))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCapture_v2)(__pyx_v_hStream, __pyx_v_mode); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4720, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4721 + * raise RuntimeError('Function "cuStreamBeginCapture_v2" not found') + * err = ( __cuStreamBeginCapture_v2)(hStream, mode) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamBeginCaptureToGraph(CUstream hStream, CUgraph hGraph, const CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, CUstreamCaptureMode mode) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4714 + * return err + * + * cdef CUresult _cuStreamBeginCapture_v2(CUstream hStream, CUstreamCaptureMode mode) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamBeginCapture_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamBeginCapture_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4723 + * return err + * + * cdef CUresult _cuStreamBeginCaptureToGraph(CUstream hStream, CUgraph hGraph, const CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, CUstreamCaptureMode mode) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamBeginCaptureToGraph + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamBeginCaptureToGraph(CUstream __pyx_v_hStream, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, CUgraphEdgeData const *__pyx_v_dependencyData, size_t __pyx_v_numDependencies, CUstreamCaptureMode __pyx_v_mode) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamBeginCaptureToGraph", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4725 + * cdef CUresult _cuStreamBeginCaptureToGraph(CUstream hStream, CUgraph hGraph, const CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, CUstreamCaptureMode mode) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamBeginCaptureToGraph + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamBeginCaptureToGraph == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4725, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4726 + * global __cuStreamBeginCaptureToGraph + * cuPythonInit() + * if __cuStreamBeginCaptureToGraph == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamBeginCaptureToGraph" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCaptureToGraph == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4727 + * cuPythonInit() + * if __cuStreamBeginCaptureToGraph == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamBeginCaptureToGraph" not found') + * err = ( __cuStreamBeginCaptureToGraph)(hStream, hGraph, dependencies, dependencyData, numDependencies, mode) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4728 + * if __cuStreamBeginCaptureToGraph == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamBeginCaptureToGraph" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamBeginCaptureToGraph)(hStream, hGraph, dependencies, dependencyData, numDependencies, mode) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamBeginCaptureToG}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4728, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4728, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4727 + * cuPythonInit() + * if __cuStreamBeginCaptureToGraph == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamBeginCaptureToGraph" not found') + * err = ( __cuStreamBeginCaptureToGraph)(hStream, hGraph, dependencies, dependencyData, numDependencies, mode) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4726 + * global __cuStreamBeginCaptureToGraph + * cuPythonInit() + * if __cuStreamBeginCaptureToGraph == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamBeginCaptureToGraph" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4729 + * with gil: + * raise RuntimeError('Function "cuStreamBeginCaptureToGraph" not found') + * err = ( __cuStreamBeginCaptureToGraph)(hStream, hGraph, dependencies, dependencyData, numDependencies, mode) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUgraph, CUgraphNode const *, CUgraphEdgeData const *, size_t, CUstreamCaptureMode))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCaptureToGraph)(__pyx_v_hStream, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_dependencyData, __pyx_v_numDependencies, __pyx_v_mode); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4729, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4730 + * raise RuntimeError('Function "cuStreamBeginCaptureToGraph" not found') + * err = ( __cuStreamBeginCaptureToGraph)(hStream, hGraph, dependencies, dependencyData, numDependencies, mode) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuThreadExchangeStreamCaptureMode(CUstreamCaptureMode* mode) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4723 + * return err + * + * cdef CUresult _cuStreamBeginCaptureToGraph(CUstream hStream, CUgraph hGraph, const CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, CUstreamCaptureMode mode) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamBeginCaptureToGraph + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamBeginCaptureToGraph", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4732 + * return err + * + * cdef CUresult _cuThreadExchangeStreamCaptureMode(CUstreamCaptureMode* mode) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuThreadExchangeStreamCaptureMode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuThreadExchangeStreamCaptureMode(CUstreamCaptureMode *__pyx_v_mode) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuThreadExchangeStreamCaptureMode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4734 + * cdef CUresult _cuThreadExchangeStreamCaptureMode(CUstreamCaptureMode* mode) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuThreadExchangeStreamCaptureMode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuThreadExchangeStreamCaptureMode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4734, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4735 + * global __cuThreadExchangeStreamCaptureMode + * cuPythonInit() + * if __cuThreadExchangeStreamCaptureMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuThreadExchangeStreamCaptureMode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuThreadExchangeStreamCaptureMode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4736 + * cuPythonInit() + * if __cuThreadExchangeStreamCaptureMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuThreadExchangeStreamCaptureMode" not found') + * err = ( __cuThreadExchangeStreamCaptureMode)(mode) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4737 + * if __cuThreadExchangeStreamCaptureMode == NULL: + * with gil: + * raise RuntimeError('Function "cuThreadExchangeStreamCaptureMode" not found') # <<<<<<<<<<<<<< + * err = ( __cuThreadExchangeStreamCaptureMode)(mode) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuThreadExchangeStreamC}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4737, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4737, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4736 + * cuPythonInit() + * if __cuThreadExchangeStreamCaptureMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuThreadExchangeStreamCaptureMode" not found') + * err = ( __cuThreadExchangeStreamCaptureMode)(mode) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4735 + * global __cuThreadExchangeStreamCaptureMode + * cuPythonInit() + * if __cuThreadExchangeStreamCaptureMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuThreadExchangeStreamCaptureMode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4738 + * with gil: + * raise RuntimeError('Function "cuThreadExchangeStreamCaptureMode" not found') + * err = ( __cuThreadExchangeStreamCaptureMode)(mode) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstreamCaptureMode *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuThreadExchangeStreamCaptureMode)(__pyx_v_mode); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4738, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4739 + * raise RuntimeError('Function "cuThreadExchangeStreamCaptureMode" not found') + * err = ( __cuThreadExchangeStreamCaptureMode)(mode) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamEndCapture(CUstream hStream, CUgraph* phGraph) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4732 + * return err + * + * cdef CUresult _cuThreadExchangeStreamCaptureMode(CUstreamCaptureMode* mode) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuThreadExchangeStreamCaptureMode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuThreadExchangeStreamCaptureMode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4741 + * return err + * + * cdef CUresult _cuStreamEndCapture(CUstream hStream, CUgraph* phGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamEndCapture + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamEndCapture(CUstream __pyx_v_hStream, CUgraph *__pyx_v_phGraph) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamEndCapture", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4743 + * cdef CUresult _cuStreamEndCapture(CUstream hStream, CUgraph* phGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamEndCapture + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamEndCapture == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4743, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4744 + * global __cuStreamEndCapture + * cuPythonInit() + * if __cuStreamEndCapture == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamEndCapture" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamEndCapture == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4745 + * cuPythonInit() + * if __cuStreamEndCapture == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamEndCapture" not found') + * err = ( __cuStreamEndCapture)(hStream, phGraph) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4746 + * if __cuStreamEndCapture == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamEndCapture" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamEndCapture)(hStream, phGraph) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamEndCapture_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4746, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4746, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4745 + * cuPythonInit() + * if __cuStreamEndCapture == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamEndCapture" not found') + * err = ( __cuStreamEndCapture)(hStream, phGraph) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4744 + * global __cuStreamEndCapture + * cuPythonInit() + * if __cuStreamEndCapture == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamEndCapture" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4747 + * with gil: + * raise RuntimeError('Function "cuStreamEndCapture" not found') + * err = ( __cuStreamEndCapture)(hStream, phGraph) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUgraph *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamEndCapture)(__pyx_v_hStream, __pyx_v_phGraph); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4747, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4748 + * raise RuntimeError('Function "cuStreamEndCapture" not found') + * err = ( __cuStreamEndCapture)(hStream, phGraph) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamIsCapturing(CUstream hStream, CUstreamCaptureStatus* captureStatus) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4741 + * return err + * + * cdef CUresult _cuStreamEndCapture(CUstream hStream, CUgraph* phGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamEndCapture + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamEndCapture", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4750 + * return err + * + * cdef CUresult _cuStreamIsCapturing(CUstream hStream, CUstreamCaptureStatus* captureStatus) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamIsCapturing + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamIsCapturing(CUstream __pyx_v_hStream, CUstreamCaptureStatus *__pyx_v_captureStatus) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamIsCapturing", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4752 + * cdef CUresult _cuStreamIsCapturing(CUstream hStream, CUstreamCaptureStatus* captureStatus) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamIsCapturing + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamIsCapturing == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4752, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4753 + * global __cuStreamIsCapturing + * cuPythonInit() + * if __cuStreamIsCapturing == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamIsCapturing" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamIsCapturing == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4754 + * cuPythonInit() + * if __cuStreamIsCapturing == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamIsCapturing" not found') + * err = ( __cuStreamIsCapturing)(hStream, captureStatus) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4755 + * if __cuStreamIsCapturing == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamIsCapturing" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamIsCapturing)(hStream, captureStatus) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamIsCapturing_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4755, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4755, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4754 + * cuPythonInit() + * if __cuStreamIsCapturing == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamIsCapturing" not found') + * err = ( __cuStreamIsCapturing)(hStream, captureStatus) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4753 + * global __cuStreamIsCapturing + * cuPythonInit() + * if __cuStreamIsCapturing == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamIsCapturing" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4756 + * with gil: + * raise RuntimeError('Function "cuStreamIsCapturing" not found') + * err = ( __cuStreamIsCapturing)(hStream, captureStatus) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUstreamCaptureStatus *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamIsCapturing)(__pyx_v_hStream, __pyx_v_captureStatus); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4756, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4757 + * raise RuntimeError('Function "cuStreamIsCapturing" not found') + * err = ( __cuStreamIsCapturing)(hStream, captureStatus) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamGetCaptureInfo_v2(CUstream hStream, CUstreamCaptureStatus* captureStatus_out, cuuint64_t* id_out, CUgraph* graph_out, const CUgraphNode** dependencies_out, size_t* numDependencies_out) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4750 + * return err + * + * cdef CUresult _cuStreamIsCapturing(CUstream hStream, CUstreamCaptureStatus* captureStatus) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamIsCapturing + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamIsCapturing", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4759 + * return err + * + * cdef CUresult _cuStreamGetCaptureInfo_v2(CUstream hStream, CUstreamCaptureStatus* captureStatus_out, cuuint64_t* id_out, CUgraph* graph_out, const CUgraphNode** dependencies_out, size_t* numDependencies_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetCaptureInfo_v2(CUstream __pyx_v_hStream, CUstreamCaptureStatus *__pyx_v_captureStatus_out, cuuint64_t *__pyx_v_id_out, CUgraph *__pyx_v_graph_out, CUgraphNode const **__pyx_v_dependencies_out, size_t *__pyx_v_numDependencies_out) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamGetCaptureInfo_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4761 + * cdef CUresult _cuStreamGetCaptureInfo_v2(CUstream hStream, CUstreamCaptureStatus* captureStatus_out, cuuint64_t* id_out, CUgraph* graph_out, const CUgraphNode** dependencies_out, size_t* numDependencies_out) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamGetCaptureInfo_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamGetCaptureInfo_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4761, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4762 + * global __cuStreamGetCaptureInfo_v2 + * cuPythonInit() + * if __cuStreamGetCaptureInfo_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4763 + * cuPythonInit() + * if __cuStreamGetCaptureInfo_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v2" not found') + * err = ( __cuStreamGetCaptureInfo_v2)(hStream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4764 + * if __cuStreamGetCaptureInfo_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamGetCaptureInfo_v2)(hStream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamGetCaptureInfo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4764, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4764, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4763 + * cuPythonInit() + * if __cuStreamGetCaptureInfo_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v2" not found') + * err = ( __cuStreamGetCaptureInfo_v2)(hStream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4762 + * global __cuStreamGetCaptureInfo_v2 + * cuPythonInit() + * if __cuStreamGetCaptureInfo_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4765 + * with gil: + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v2" not found') + * err = ( __cuStreamGetCaptureInfo_v2)(hStream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUstreamCaptureStatus *, cuuint64_t *, CUgraph *, CUgraphNode const **, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v2)(__pyx_v_hStream, __pyx_v_captureStatus_out, __pyx_v_id_out, __pyx_v_graph_out, __pyx_v_dependencies_out, __pyx_v_numDependencies_out); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4765, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4766 + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v2" not found') + * err = ( __cuStreamGetCaptureInfo_v2)(hStream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamGetCaptureInfo_v3(CUstream hStream, CUstreamCaptureStatus* captureStatus_out, cuuint64_t* id_out, CUgraph* graph_out, const CUgraphNode** dependencies_out, const CUgraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4759 + * return err + * + * cdef CUresult _cuStreamGetCaptureInfo_v2(CUstream hStream, CUstreamCaptureStatus* captureStatus_out, cuuint64_t* id_out, CUgraph* graph_out, const CUgraphNode** dependencies_out, size_t* numDependencies_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamGetCaptureInfo_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4768 + * return err + * + * cdef CUresult _cuStreamGetCaptureInfo_v3(CUstream hStream, CUstreamCaptureStatus* captureStatus_out, cuuint64_t* id_out, CUgraph* graph_out, const CUgraphNode** dependencies_out, const CUgraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v3 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetCaptureInfo_v3(CUstream __pyx_v_hStream, CUstreamCaptureStatus *__pyx_v_captureStatus_out, cuuint64_t *__pyx_v_id_out, CUgraph *__pyx_v_graph_out, CUgraphNode const **__pyx_v_dependencies_out, CUgraphEdgeData const **__pyx_v_edgeData_out, size_t *__pyx_v_numDependencies_out) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamGetCaptureInfo_v3", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4770 + * cdef CUresult _cuStreamGetCaptureInfo_v3(CUstream hStream, CUstreamCaptureStatus* captureStatus_out, cuuint64_t* id_out, CUgraph* graph_out, const CUgraphNode** dependencies_out, const CUgraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamGetCaptureInfo_v3 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamGetCaptureInfo_v3 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4770, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4771 + * global __cuStreamGetCaptureInfo_v3 + * cuPythonInit() + * if __cuStreamGetCaptureInfo_v3 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v3" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v3 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4772 + * cuPythonInit() + * if __cuStreamGetCaptureInfo_v3 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v3" not found') + * err = ( __cuStreamGetCaptureInfo_v3)(hStream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4773 + * if __cuStreamGetCaptureInfo_v3 == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v3" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamGetCaptureInfo_v3)(hStream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamGetCaptureInfo_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4773, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4773, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4772 + * cuPythonInit() + * if __cuStreamGetCaptureInfo_v3 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v3" not found') + * err = ( __cuStreamGetCaptureInfo_v3)(hStream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4771 + * global __cuStreamGetCaptureInfo_v3 + * cuPythonInit() + * if __cuStreamGetCaptureInfo_v3 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v3" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4774 + * with gil: + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v3" not found') + * err = ( __cuStreamGetCaptureInfo_v3)(hStream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUstreamCaptureStatus *, cuuint64_t *, CUgraph *, CUgraphNode const **, CUgraphEdgeData const **, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v3)(__pyx_v_hStream, __pyx_v_captureStatus_out, __pyx_v_id_out, __pyx_v_graph_out, __pyx_v_dependencies_out, __pyx_v_edgeData_out, __pyx_v_numDependencies_out); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4774, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4775 + * raise RuntimeError('Function "cuStreamGetCaptureInfo_v3" not found') + * err = ( __cuStreamGetCaptureInfo_v3)(hStream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamUpdateCaptureDependencies(CUstream hStream, CUgraphNode* dependencies, size_t numDependencies, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4768 + * return err + * + * cdef CUresult _cuStreamGetCaptureInfo_v3(CUstream hStream, CUstreamCaptureStatus* captureStatus_out, cuuint64_t* id_out, CUgraph* graph_out, const CUgraphNode** dependencies_out, const CUgraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v3 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamGetCaptureInfo_v3", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4777 + * return err + * + * cdef CUresult _cuStreamUpdateCaptureDependencies(CUstream hStream, CUgraphNode* dependencies, size_t numDependencies, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamUpdateCaptureDependencies(CUstream __pyx_v_hStream, CUgraphNode *__pyx_v_dependencies, size_t __pyx_v_numDependencies, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamUpdateCaptureDependencies", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4779 + * cdef CUresult _cuStreamUpdateCaptureDependencies(CUstream hStream, CUgraphNode* dependencies, size_t numDependencies, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamUpdateCaptureDependencies + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamUpdateCaptureDependencies == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4779, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4780 + * global __cuStreamUpdateCaptureDependencies + * cuPythonInit() + * if __cuStreamUpdateCaptureDependencies == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4781 + * cuPythonInit() + * if __cuStreamUpdateCaptureDependencies == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies" not found') + * err = ( __cuStreamUpdateCaptureDependencies)(hStream, dependencies, numDependencies, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4782 + * if __cuStreamUpdateCaptureDependencies == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamUpdateCaptureDependencies)(hStream, dependencies, numDependencies, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamUpdateCaptureDe}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4782, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4782, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4781 + * cuPythonInit() + * if __cuStreamUpdateCaptureDependencies == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies" not found') + * err = ( __cuStreamUpdateCaptureDependencies)(hStream, dependencies, numDependencies, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4780 + * global __cuStreamUpdateCaptureDependencies + * cuPythonInit() + * if __cuStreamUpdateCaptureDependencies == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4783 + * with gil: + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies" not found') + * err = ( __cuStreamUpdateCaptureDependencies)(hStream, dependencies, numDependencies, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUgraphNode *, size_t, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies)(__pyx_v_hStream, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4783, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4784 + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies" not found') + * err = ( __cuStreamUpdateCaptureDependencies)(hStream, dependencies, numDependencies, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamUpdateCaptureDependencies_v2(CUstream hStream, CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4777 + * return err + * + * cdef CUresult _cuStreamUpdateCaptureDependencies(CUstream hStream, CUgraphNode* dependencies, size_t numDependencies, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamUpdateCaptureDependencies", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4786 + * return err + * + * cdef CUresult _cuStreamUpdateCaptureDependencies_v2(CUstream hStream, CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamUpdateCaptureDependencies_v2(CUstream __pyx_v_hStream, CUgraphNode *__pyx_v_dependencies, CUgraphEdgeData const *__pyx_v_dependencyData, size_t __pyx_v_numDependencies, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamUpdateCaptureDependencies_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4788 + * cdef CUresult _cuStreamUpdateCaptureDependencies_v2(CUstream hStream, CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamUpdateCaptureDependencies_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamUpdateCaptureDependencies_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4788, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4789 + * global __cuStreamUpdateCaptureDependencies_v2 + * cuPythonInit() + * if __cuStreamUpdateCaptureDependencies_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4790 + * cuPythonInit() + * if __cuStreamUpdateCaptureDependencies_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies_v2" not found') + * err = ( __cuStreamUpdateCaptureDependencies_v2)(hStream, dependencies, dependencyData, numDependencies, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4791 + * if __cuStreamUpdateCaptureDependencies_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamUpdateCaptureDependencies_v2)(hStream, dependencies, dependencyData, numDependencies, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamUpdateCaptureDe_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4791, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4791, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4790 + * cuPythonInit() + * if __cuStreamUpdateCaptureDependencies_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies_v2" not found') + * err = ( __cuStreamUpdateCaptureDependencies_v2)(hStream, dependencies, dependencyData, numDependencies, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4789 + * global __cuStreamUpdateCaptureDependencies_v2 + * cuPythonInit() + * if __cuStreamUpdateCaptureDependencies_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4792 + * with gil: + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies_v2" not found') + * err = ( __cuStreamUpdateCaptureDependencies_v2)(hStream, dependencies, dependencyData, numDependencies, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUgraphNode *, CUgraphEdgeData const *, size_t, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies_v2)(__pyx_v_hStream, __pyx_v_dependencies, __pyx_v_dependencyData, __pyx_v_numDependencies, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4792, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4793 + * raise RuntimeError('Function "cuStreamUpdateCaptureDependencies_v2" not found') + * err = ( __cuStreamUpdateCaptureDependencies_v2)(hStream, dependencies, dependencyData, numDependencies, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamAttachMemAsync(CUstream hStream, CUdeviceptr dptr, size_t length, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4786 + * return err + * + * cdef CUresult _cuStreamUpdateCaptureDependencies_v2(CUstream hStream, CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamUpdateCaptureDependencies_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4795 + * return err + * + * cdef CUresult _cuStreamAttachMemAsync(CUstream hStream, CUdeviceptr dptr, size_t length, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamAttachMemAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamAttachMemAsync(CUstream __pyx_v_hStream, CUdeviceptr __pyx_v_dptr, size_t __pyx_v_length, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamAttachMemAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4797 + * cdef CUresult _cuStreamAttachMemAsync(CUstream hStream, CUdeviceptr dptr, size_t length, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamAttachMemAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamAttachMemAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4797, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4798 + * global __cuStreamAttachMemAsync + * cuPythonInit() + * if __cuStreamAttachMemAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamAttachMemAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAttachMemAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4799 + * cuPythonInit() + * if __cuStreamAttachMemAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamAttachMemAsync" not found') + * err = ( __cuStreamAttachMemAsync)(hStream, dptr, length, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4800 + * if __cuStreamAttachMemAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamAttachMemAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamAttachMemAsync)(hStream, dptr, length, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamAttachMemAsync}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4800, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4800, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4799 + * cuPythonInit() + * if __cuStreamAttachMemAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamAttachMemAsync" not found') + * err = ( __cuStreamAttachMemAsync)(hStream, dptr, length, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4798 + * global __cuStreamAttachMemAsync + * cuPythonInit() + * if __cuStreamAttachMemAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamAttachMemAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4801 + * with gil: + * raise RuntimeError('Function "cuStreamAttachMemAsync" not found') + * err = ( __cuStreamAttachMemAsync)(hStream, dptr, length, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUdeviceptr, size_t, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAttachMemAsync)(__pyx_v_hStream, __pyx_v_dptr, __pyx_v_length, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4801, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4802 + * raise RuntimeError('Function "cuStreamAttachMemAsync" not found') + * err = ( __cuStreamAttachMemAsync)(hStream, dptr, length, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamQuery(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4795 + * return err + * + * cdef CUresult _cuStreamAttachMemAsync(CUstream hStream, CUdeviceptr dptr, size_t length, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamAttachMemAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamAttachMemAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4804 + * return err + * + * cdef CUresult _cuStreamQuery(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamQuery + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamQuery(CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamQuery", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4806 + * cdef CUresult _cuStreamQuery(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamQuery + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamQuery == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4806, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4807 + * global __cuStreamQuery + * cuPythonInit() + * if __cuStreamQuery == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamQuery" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamQuery == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4808 + * cuPythonInit() + * if __cuStreamQuery == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamQuery" not found') + * err = ( __cuStreamQuery)(hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4809 + * if __cuStreamQuery == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamQuery" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamQuery)(hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamQuery_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4809, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4809, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4808 + * cuPythonInit() + * if __cuStreamQuery == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamQuery" not found') + * err = ( __cuStreamQuery)(hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4807 + * global __cuStreamQuery + * cuPythonInit() + * if __cuStreamQuery == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamQuery" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4810 + * with gil: + * raise RuntimeError('Function "cuStreamQuery" not found') + * err = ( __cuStreamQuery)(hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamQuery)(__pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4810, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4811 + * raise RuntimeError('Function "cuStreamQuery" not found') + * err = ( __cuStreamQuery)(hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamSynchronize(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4804 + * return err + * + * cdef CUresult _cuStreamQuery(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamQuery + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamQuery", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4813 + * return err + * + * cdef CUresult _cuStreamSynchronize(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamSynchronize + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamSynchronize(CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamSynchronize", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4815 + * cdef CUresult _cuStreamSynchronize(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamSynchronize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamSynchronize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4815, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4816 + * global __cuStreamSynchronize + * cuPythonInit() + * if __cuStreamSynchronize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamSynchronize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSynchronize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4817 + * cuPythonInit() + * if __cuStreamSynchronize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamSynchronize" not found') + * err = ( __cuStreamSynchronize)(hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4818 + * if __cuStreamSynchronize == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamSynchronize" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamSynchronize)(hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamSynchronize_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4818, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4818, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4817 + * cuPythonInit() + * if __cuStreamSynchronize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamSynchronize" not found') + * err = ( __cuStreamSynchronize)(hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4816 + * global __cuStreamSynchronize + * cuPythonInit() + * if __cuStreamSynchronize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamSynchronize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4819 + * with gil: + * raise RuntimeError('Function "cuStreamSynchronize" not found') + * err = ( __cuStreamSynchronize)(hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSynchronize)(__pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4819, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4820 + * raise RuntimeError('Function "cuStreamSynchronize" not found') + * err = ( __cuStreamSynchronize)(hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamDestroy_v2(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4813 + * return err + * + * cdef CUresult _cuStreamSynchronize(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamSynchronize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamSynchronize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4822 + * return err + * + * cdef CUresult _cuStreamDestroy_v2(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamDestroy_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamDestroy_v2(CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamDestroy_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4824 + * cdef CUresult _cuStreamDestroy_v2(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamDestroy_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamDestroy_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4824, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4825 + * global __cuStreamDestroy_v2 + * cuPythonInit() + * if __cuStreamDestroy_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamDestroy_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamDestroy_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4826 + * cuPythonInit() + * if __cuStreamDestroy_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamDestroy_v2" not found') + * err = ( __cuStreamDestroy_v2)(hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4827 + * if __cuStreamDestroy_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamDestroy_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamDestroy_v2)(hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamDestroy_v2_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4827, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4827, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4826 + * cuPythonInit() + * if __cuStreamDestroy_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamDestroy_v2" not found') + * err = ( __cuStreamDestroy_v2)(hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4825 + * global __cuStreamDestroy_v2 + * cuPythonInit() + * if __cuStreamDestroy_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamDestroy_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4828 + * with gil: + * raise RuntimeError('Function "cuStreamDestroy_v2" not found') + * err = ( __cuStreamDestroy_v2)(hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamDestroy_v2)(__pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4828, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4829 + * raise RuntimeError('Function "cuStreamDestroy_v2" not found') + * err = ( __cuStreamDestroy_v2)(hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamCopyAttributes(CUstream dst, CUstream src) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4822 + * return err + * + * cdef CUresult _cuStreamDestroy_v2(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamDestroy_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamDestroy_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4831 + * return err + * + * cdef CUresult _cuStreamCopyAttributes(CUstream dst, CUstream src) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamCopyAttributes + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamCopyAttributes(CUstream __pyx_v_dst, CUstream __pyx_v_src) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamCopyAttributes", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4833 + * cdef CUresult _cuStreamCopyAttributes(CUstream dst, CUstream src) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamCopyAttributes + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamCopyAttributes == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4833, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4834 + * global __cuStreamCopyAttributes + * cuPythonInit() + * if __cuStreamCopyAttributes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamCopyAttributes" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCopyAttributes == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4835 + * cuPythonInit() + * if __cuStreamCopyAttributes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamCopyAttributes" not found') + * err = ( __cuStreamCopyAttributes)(dst, src) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4836 + * if __cuStreamCopyAttributes == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamCopyAttributes" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamCopyAttributes)(dst, src) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamCopyAttributes}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4836, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4836, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4835 + * cuPythonInit() + * if __cuStreamCopyAttributes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamCopyAttributes" not found') + * err = ( __cuStreamCopyAttributes)(dst, src) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4834 + * global __cuStreamCopyAttributes + * cuPythonInit() + * if __cuStreamCopyAttributes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamCopyAttributes" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4837 + * with gil: + * raise RuntimeError('Function "cuStreamCopyAttributes" not found') + * err = ( __cuStreamCopyAttributes)(dst, src) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCopyAttributes)(__pyx_v_dst, __pyx_v_src); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4837, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4838 + * raise RuntimeError('Function "cuStreamCopyAttributes" not found') + * err = ( __cuStreamCopyAttributes)(dst, src) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamGetAttribute(CUstream hStream, CUstreamAttrID attr, CUstreamAttrValue* value_out) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4831 + * return err + * + * cdef CUresult _cuStreamCopyAttributes(CUstream dst, CUstream src) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamCopyAttributes + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamCopyAttributes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4840 + * return err + * + * cdef CUresult _cuStreamGetAttribute(CUstream hStream, CUstreamAttrID attr, CUstreamAttrValue* value_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetAttribute(CUstream __pyx_v_hStream, CUstreamAttrID __pyx_v_attr, CUstreamAttrValue *__pyx_v_value_out) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamGetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4842 + * cdef CUresult _cuStreamGetAttribute(CUstream hStream, CUstreamAttrID attr, CUstreamAttrValue* value_out) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamGetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamGetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4842, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4843 + * global __cuStreamGetAttribute + * cuPythonInit() + * if __cuStreamGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4844 + * cuPythonInit() + * if __cuStreamGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetAttribute" not found') + * err = ( __cuStreamGetAttribute)(hStream, attr, value_out) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4845 + * if __cuStreamGetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamGetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamGetAttribute)(hStream, attr, value_out) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamGetAttribute_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4845, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4845, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4844 + * cuPythonInit() + * if __cuStreamGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetAttribute" not found') + * err = ( __cuStreamGetAttribute)(hStream, attr, value_out) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4843 + * global __cuStreamGetAttribute + * cuPythonInit() + * if __cuStreamGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4846 + * with gil: + * raise RuntimeError('Function "cuStreamGetAttribute" not found') + * err = ( __cuStreamGetAttribute)(hStream, attr, value_out) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUstreamAttrID, CUstreamAttrValue *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetAttribute)(__pyx_v_hStream, __pyx_v_attr, __pyx_v_value_out); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4846, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4847 + * raise RuntimeError('Function "cuStreamGetAttribute" not found') + * err = ( __cuStreamGetAttribute)(hStream, attr, value_out) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamSetAttribute(CUstream hStream, CUstreamAttrID attr, const CUstreamAttrValue* value) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4840 + * return err + * + * cdef CUresult _cuStreamGetAttribute(CUstream hStream, CUstreamAttrID attr, CUstreamAttrValue* value_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4849 + * return err + * + * cdef CUresult _cuStreamSetAttribute(CUstream hStream, CUstreamAttrID attr, const CUstreamAttrValue* value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamSetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamSetAttribute(CUstream __pyx_v_hStream, CUstreamAttrID __pyx_v_attr, CUstreamAttrValue const *__pyx_v_value) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamSetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4851 + * cdef CUresult _cuStreamSetAttribute(CUstream hStream, CUstreamAttrID attr, const CUstreamAttrValue* value) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamSetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamSetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4851, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4852 + * global __cuStreamSetAttribute + * cuPythonInit() + * if __cuStreamSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamSetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4853 + * cuPythonInit() + * if __cuStreamSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamSetAttribute" not found') + * err = ( __cuStreamSetAttribute)(hStream, attr, value) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4854 + * if __cuStreamSetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamSetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamSetAttribute)(hStream, attr, value) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamSetAttribute_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4854, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4854, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4853 + * cuPythonInit() + * if __cuStreamSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamSetAttribute" not found') + * err = ( __cuStreamSetAttribute)(hStream, attr, value) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4852 + * global __cuStreamSetAttribute + * cuPythonInit() + * if __cuStreamSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamSetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4855 + * with gil: + * raise RuntimeError('Function "cuStreamSetAttribute" not found') + * err = ( __cuStreamSetAttribute)(hStream, attr, value) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUstreamAttrID, CUstreamAttrValue const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSetAttribute)(__pyx_v_hStream, __pyx_v_attr, __pyx_v_value); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4855, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4856 + * raise RuntimeError('Function "cuStreamSetAttribute" not found') + * err = ( __cuStreamSetAttribute)(hStream, attr, value) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEventCreate(CUevent* phEvent, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4849 + * return err + * + * cdef CUresult _cuStreamSetAttribute(CUstream hStream, CUstreamAttrID attr, const CUstreamAttrValue* value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamSetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamSetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4858 + * return err + * + * cdef CUresult _cuEventCreate(CUevent* phEvent, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventCreate(CUevent *__pyx_v_phEvent, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEventCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4860 + * cdef CUresult _cuEventCreate(CUevent* phEvent, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEventCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEventCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4860, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4861 + * global __cuEventCreate + * cuPythonInit() + * if __cuEventCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4862 + * cuPythonInit() + * if __cuEventCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventCreate" not found') + * err = ( __cuEventCreate)(phEvent, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4863 + * if __cuEventCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuEventCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuEventCreate)(phEvent, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEventCreate_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4863, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4863, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4862 + * cuPythonInit() + * if __cuEventCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventCreate" not found') + * err = ( __cuEventCreate)(phEvent, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4861 + * global __cuEventCreate + * cuPythonInit() + * if __cuEventCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4864 + * with gil: + * raise RuntimeError('Function "cuEventCreate" not found') + * err = ( __cuEventCreate)(phEvent, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUevent *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreate)(__pyx_v_phEvent, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4864, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4865 + * raise RuntimeError('Function "cuEventCreate" not found') + * err = ( __cuEventCreate)(phEvent, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEventRecord(CUevent hEvent, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4858 + * return err + * + * cdef CUresult _cuEventCreate(CUevent* phEvent, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEventCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4867 + * return err + * + * cdef CUresult _cuEventRecord(CUevent hEvent, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventRecord + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventRecord(CUevent __pyx_v_hEvent, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEventRecord", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4869 + * cdef CUresult _cuEventRecord(CUevent hEvent, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEventRecord + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEventRecord == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4869, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4870 + * global __cuEventRecord + * cuPythonInit() + * if __cuEventRecord == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventRecord" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecord == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4871 + * cuPythonInit() + * if __cuEventRecord == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventRecord" not found') + * err = ( __cuEventRecord)(hEvent, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4872 + * if __cuEventRecord == NULL: + * with gil: + * raise RuntimeError('Function "cuEventRecord" not found') # <<<<<<<<<<<<<< + * err = ( __cuEventRecord)(hEvent, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEventRecord_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4872, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4872, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4871 + * cuPythonInit() + * if __cuEventRecord == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventRecord" not found') + * err = ( __cuEventRecord)(hEvent, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4870 + * global __cuEventRecord + * cuPythonInit() + * if __cuEventRecord == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventRecord" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4873 + * with gil: + * raise RuntimeError('Function "cuEventRecord" not found') + * err = ( __cuEventRecord)(hEvent, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUevent, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecord)(__pyx_v_hEvent, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4873, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4874 + * raise RuntimeError('Function "cuEventRecord" not found') + * err = ( __cuEventRecord)(hEvent, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEventRecordWithFlags(CUevent hEvent, CUstream hStream, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4867 + * return err + * + * cdef CUresult _cuEventRecord(CUevent hEvent, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventRecord + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEventRecord", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4876 + * return err + * + * cdef CUresult _cuEventRecordWithFlags(CUevent hEvent, CUstream hStream, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventRecordWithFlags + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventRecordWithFlags(CUevent __pyx_v_hEvent, CUstream __pyx_v_hStream, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEventRecordWithFlags", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4878 + * cdef CUresult _cuEventRecordWithFlags(CUevent hEvent, CUstream hStream, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEventRecordWithFlags + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEventRecordWithFlags == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4878, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4879 + * global __cuEventRecordWithFlags + * cuPythonInit() + * if __cuEventRecordWithFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventRecordWithFlags" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecordWithFlags == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4880 + * cuPythonInit() + * if __cuEventRecordWithFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventRecordWithFlags" not found') + * err = ( __cuEventRecordWithFlags)(hEvent, hStream, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4881 + * if __cuEventRecordWithFlags == NULL: + * with gil: + * raise RuntimeError('Function "cuEventRecordWithFlags" not found') # <<<<<<<<<<<<<< + * err = ( __cuEventRecordWithFlags)(hEvent, hStream, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEventRecordWithFlags}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4881, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4881, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4880 + * cuPythonInit() + * if __cuEventRecordWithFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventRecordWithFlags" not found') + * err = ( __cuEventRecordWithFlags)(hEvent, hStream, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4879 + * global __cuEventRecordWithFlags + * cuPythonInit() + * if __cuEventRecordWithFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventRecordWithFlags" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4882 + * with gil: + * raise RuntimeError('Function "cuEventRecordWithFlags" not found') + * err = ( __cuEventRecordWithFlags)(hEvent, hStream, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUevent, CUstream, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecordWithFlags)(__pyx_v_hEvent, __pyx_v_hStream, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4882, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4883 + * raise RuntimeError('Function "cuEventRecordWithFlags" not found') + * err = ( __cuEventRecordWithFlags)(hEvent, hStream, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEventQuery(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4876 + * return err + * + * cdef CUresult _cuEventRecordWithFlags(CUevent hEvent, CUstream hStream, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventRecordWithFlags + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEventRecordWithFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4885 + * return err + * + * cdef CUresult _cuEventQuery(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventQuery + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventQuery(CUevent __pyx_v_hEvent) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEventQuery", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4887 + * cdef CUresult _cuEventQuery(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEventQuery + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEventQuery == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4887, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4888 + * global __cuEventQuery + * cuPythonInit() + * if __cuEventQuery == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventQuery" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventQuery == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4889 + * cuPythonInit() + * if __cuEventQuery == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventQuery" not found') + * err = ( __cuEventQuery)(hEvent) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4890 + * if __cuEventQuery == NULL: + * with gil: + * raise RuntimeError('Function "cuEventQuery" not found') # <<<<<<<<<<<<<< + * err = ( __cuEventQuery)(hEvent) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEventQuery_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4890, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4890, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4889 + * cuPythonInit() + * if __cuEventQuery == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventQuery" not found') + * err = ( __cuEventQuery)(hEvent) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4888 + * global __cuEventQuery + * cuPythonInit() + * if __cuEventQuery == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventQuery" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4891 + * with gil: + * raise RuntimeError('Function "cuEventQuery" not found') + * err = ( __cuEventQuery)(hEvent) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventQuery)(__pyx_v_hEvent); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4891, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4892 + * raise RuntimeError('Function "cuEventQuery" not found') + * err = ( __cuEventQuery)(hEvent) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEventSynchronize(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4885 + * return err + * + * cdef CUresult _cuEventQuery(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventQuery + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEventQuery", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4894 + * return err + * + * cdef CUresult _cuEventSynchronize(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventSynchronize + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventSynchronize(CUevent __pyx_v_hEvent) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEventSynchronize", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4896 + * cdef CUresult _cuEventSynchronize(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEventSynchronize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEventSynchronize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4896, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4897 + * global __cuEventSynchronize + * cuPythonInit() + * if __cuEventSynchronize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventSynchronize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventSynchronize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4898 + * cuPythonInit() + * if __cuEventSynchronize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventSynchronize" not found') + * err = ( __cuEventSynchronize)(hEvent) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4899 + * if __cuEventSynchronize == NULL: + * with gil: + * raise RuntimeError('Function "cuEventSynchronize" not found') # <<<<<<<<<<<<<< + * err = ( __cuEventSynchronize)(hEvent) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEventSynchronize_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4899, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4899, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4898 + * cuPythonInit() + * if __cuEventSynchronize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventSynchronize" not found') + * err = ( __cuEventSynchronize)(hEvent) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4897 + * global __cuEventSynchronize + * cuPythonInit() + * if __cuEventSynchronize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventSynchronize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4900 + * with gil: + * raise RuntimeError('Function "cuEventSynchronize" not found') + * err = ( __cuEventSynchronize)(hEvent) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventSynchronize)(__pyx_v_hEvent); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4900, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4901 + * raise RuntimeError('Function "cuEventSynchronize" not found') + * err = ( __cuEventSynchronize)(hEvent) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEventDestroy_v2(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4894 + * return err + * + * cdef CUresult _cuEventSynchronize(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventSynchronize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEventSynchronize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4903 + * return err + * + * cdef CUresult _cuEventDestroy_v2(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventDestroy_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventDestroy_v2(CUevent __pyx_v_hEvent) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEventDestroy_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4905 + * cdef CUresult _cuEventDestroy_v2(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEventDestroy_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEventDestroy_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4905, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4906 + * global __cuEventDestroy_v2 + * cuPythonInit() + * if __cuEventDestroy_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventDestroy_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventDestroy_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4907 + * cuPythonInit() + * if __cuEventDestroy_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventDestroy_v2" not found') + * err = ( __cuEventDestroy_v2)(hEvent) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4908 + * if __cuEventDestroy_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuEventDestroy_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuEventDestroy_v2)(hEvent) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEventDestroy_v2_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4908, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4908, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4907 + * cuPythonInit() + * if __cuEventDestroy_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventDestroy_v2" not found') + * err = ( __cuEventDestroy_v2)(hEvent) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4906 + * global __cuEventDestroy_v2 + * cuPythonInit() + * if __cuEventDestroy_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventDestroy_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4909 + * with gil: + * raise RuntimeError('Function "cuEventDestroy_v2" not found') + * err = ( __cuEventDestroy_v2)(hEvent) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventDestroy_v2)(__pyx_v_hEvent); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4909, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4910 + * raise RuntimeError('Function "cuEventDestroy_v2" not found') + * err = ( __cuEventDestroy_v2)(hEvent) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEventElapsedTime(float* pMilliseconds, CUevent hStart, CUevent hEnd) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4903 + * return err + * + * cdef CUresult _cuEventDestroy_v2(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventDestroy_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEventDestroy_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4912 + * return err + * + * cdef CUresult _cuEventElapsedTime(float* pMilliseconds, CUevent hStart, CUevent hEnd) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventElapsedTime + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventElapsedTime(float *__pyx_v_pMilliseconds, CUevent __pyx_v_hStart, CUevent __pyx_v_hEnd) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEventElapsedTime", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4914 + * cdef CUresult _cuEventElapsedTime(float* pMilliseconds, CUevent hStart, CUevent hEnd) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEventElapsedTime + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEventElapsedTime == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4914, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4915 + * global __cuEventElapsedTime + * cuPythonInit() + * if __cuEventElapsedTime == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventElapsedTime" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4916 + * cuPythonInit() + * if __cuEventElapsedTime == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventElapsedTime" not found') + * err = ( __cuEventElapsedTime)(pMilliseconds, hStart, hEnd) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4917 + * if __cuEventElapsedTime == NULL: + * with gil: + * raise RuntimeError('Function "cuEventElapsedTime" not found') # <<<<<<<<<<<<<< + * err = ( __cuEventElapsedTime)(pMilliseconds, hStart, hEnd) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEventElapsedTime_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4917, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4917, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4916 + * cuPythonInit() + * if __cuEventElapsedTime == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventElapsedTime" not found') + * err = ( __cuEventElapsedTime)(pMilliseconds, hStart, hEnd) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4915 + * global __cuEventElapsedTime + * cuPythonInit() + * if __cuEventElapsedTime == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventElapsedTime" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4918 + * with gil: + * raise RuntimeError('Function "cuEventElapsedTime" not found') + * err = ( __cuEventElapsedTime)(pMilliseconds, hStart, hEnd) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(float *, CUevent, CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime)(__pyx_v_pMilliseconds, __pyx_v_hStart, __pyx_v_hEnd); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4918, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4919 + * raise RuntimeError('Function "cuEventElapsedTime" not found') + * err = ( __cuEventElapsedTime)(pMilliseconds, hStart, hEnd) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEventElapsedTime_v2(float* pMilliseconds, CUevent hStart, CUevent hEnd) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4912 + * return err + * + * cdef CUresult _cuEventElapsedTime(float* pMilliseconds, CUevent hStart, CUevent hEnd) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventElapsedTime + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEventElapsedTime", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4921 + * return err + * + * cdef CUresult _cuEventElapsedTime_v2(float* pMilliseconds, CUevent hStart, CUevent hEnd) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventElapsedTime_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventElapsedTime_v2(float *__pyx_v_pMilliseconds, CUevent __pyx_v_hStart, CUevent __pyx_v_hEnd) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEventElapsedTime_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4923 + * cdef CUresult _cuEventElapsedTime_v2(float* pMilliseconds, CUevent hStart, CUevent hEnd) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEventElapsedTime_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEventElapsedTime_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4923, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4924 + * global __cuEventElapsedTime_v2 + * cuPythonInit() + * if __cuEventElapsedTime_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventElapsedTime_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4925 + * cuPythonInit() + * if __cuEventElapsedTime_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventElapsedTime_v2" not found') + * err = ( __cuEventElapsedTime_v2)(pMilliseconds, hStart, hEnd) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4926 + * if __cuEventElapsedTime_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuEventElapsedTime_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuEventElapsedTime_v2)(pMilliseconds, hStart, hEnd) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEventElapsedTime_v2_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4926, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4926, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4925 + * cuPythonInit() + * if __cuEventElapsedTime_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventElapsedTime_v2" not found') + * err = ( __cuEventElapsedTime_v2)(pMilliseconds, hStart, hEnd) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4924 + * global __cuEventElapsedTime_v2 + * cuPythonInit() + * if __cuEventElapsedTime_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventElapsedTime_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4927 + * with gil: + * raise RuntimeError('Function "cuEventElapsedTime_v2" not found') + * err = ( __cuEventElapsedTime_v2)(pMilliseconds, hStart, hEnd) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(float *, CUevent, CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime_v2)(__pyx_v_pMilliseconds, __pyx_v_hStart, __pyx_v_hEnd); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4927, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4928 + * raise RuntimeError('Function "cuEventElapsedTime_v2" not found') + * err = ( __cuEventElapsedTime_v2)(pMilliseconds, hStart, hEnd) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuImportExternalMemory(CUexternalMemory* extMem_out, const CUDA_EXTERNAL_MEMORY_HANDLE_DESC* memHandleDesc) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4921 + * return err + * + * cdef CUresult _cuEventElapsedTime_v2(float* pMilliseconds, CUevent hStart, CUevent hEnd) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventElapsedTime_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEventElapsedTime_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4930 + * return err + * + * cdef CUresult _cuImportExternalMemory(CUexternalMemory* extMem_out, const CUDA_EXTERNAL_MEMORY_HANDLE_DESC* memHandleDesc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuImportExternalMemory + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuImportExternalMemory(CUexternalMemory *__pyx_v_extMem_out, CUDA_EXTERNAL_MEMORY_HANDLE_DESC const *__pyx_v_memHandleDesc) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuImportExternalMemory", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4932 + * cdef CUresult _cuImportExternalMemory(CUexternalMemory* extMem_out, const CUDA_EXTERNAL_MEMORY_HANDLE_DESC* memHandleDesc) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuImportExternalMemory + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuImportExternalMemory == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4932, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4933 + * global __cuImportExternalMemory + * cuPythonInit() + * if __cuImportExternalMemory == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuImportExternalMemory" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalMemory == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4934 + * cuPythonInit() + * if __cuImportExternalMemory == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuImportExternalMemory" not found') + * err = ( __cuImportExternalMemory)(extMem_out, memHandleDesc) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4935 + * if __cuImportExternalMemory == NULL: + * with gil: + * raise RuntimeError('Function "cuImportExternalMemory" not found') # <<<<<<<<<<<<<< + * err = ( __cuImportExternalMemory)(extMem_out, memHandleDesc) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuImportExternalMemory}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4935, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4935, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4934 + * cuPythonInit() + * if __cuImportExternalMemory == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuImportExternalMemory" not found') + * err = ( __cuImportExternalMemory)(extMem_out, memHandleDesc) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4933 + * global __cuImportExternalMemory + * cuPythonInit() + * if __cuImportExternalMemory == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuImportExternalMemory" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4936 + * with gil: + * raise RuntimeError('Function "cuImportExternalMemory" not found') + * err = ( __cuImportExternalMemory)(extMem_out, memHandleDesc) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUexternalMemory *, CUDA_EXTERNAL_MEMORY_HANDLE_DESC const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalMemory)(__pyx_v_extMem_out, __pyx_v_memHandleDesc); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4936, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4937 + * raise RuntimeError('Function "cuImportExternalMemory" not found') + * err = ( __cuImportExternalMemory)(extMem_out, memHandleDesc) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuExternalMemoryGetMappedBuffer(CUdeviceptr* devPtr, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_BUFFER_DESC* bufferDesc) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4930 + * return err + * + * cdef CUresult _cuImportExternalMemory(CUexternalMemory* extMem_out, const CUDA_EXTERNAL_MEMORY_HANDLE_DESC* memHandleDesc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuImportExternalMemory + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuImportExternalMemory", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4939 + * return err + * + * cdef CUresult _cuExternalMemoryGetMappedBuffer(CUdeviceptr* devPtr, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_BUFFER_DESC* bufferDesc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuExternalMemoryGetMappedBuffer + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuExternalMemoryGetMappedBuffer(CUdeviceptr *__pyx_v_devPtr, CUexternalMemory __pyx_v_extMem, CUDA_EXTERNAL_MEMORY_BUFFER_DESC const *__pyx_v_bufferDesc) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuExternalMemoryGetMappedBuffer", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4941 + * cdef CUresult _cuExternalMemoryGetMappedBuffer(CUdeviceptr* devPtr, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_BUFFER_DESC* bufferDesc) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuExternalMemoryGetMappedBuffer + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuExternalMemoryGetMappedBuffer == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4941, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4942 + * global __cuExternalMemoryGetMappedBuffer + * cuPythonInit() + * if __cuExternalMemoryGetMappedBuffer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuExternalMemoryGetMappedBuffer" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedBuffer == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4943 + * cuPythonInit() + * if __cuExternalMemoryGetMappedBuffer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuExternalMemoryGetMappedBuffer" not found') + * err = ( __cuExternalMemoryGetMappedBuffer)(devPtr, extMem, bufferDesc) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4944 + * if __cuExternalMemoryGetMappedBuffer == NULL: + * with gil: + * raise RuntimeError('Function "cuExternalMemoryGetMappedBuffer" not found') # <<<<<<<<<<<<<< + * err = ( __cuExternalMemoryGetMappedBuffer)(devPtr, extMem, bufferDesc) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuExternalMemoryGetMapp}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4944, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4944, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4943 + * cuPythonInit() + * if __cuExternalMemoryGetMappedBuffer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuExternalMemoryGetMappedBuffer" not found') + * err = ( __cuExternalMemoryGetMappedBuffer)(devPtr, extMem, bufferDesc) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4942 + * global __cuExternalMemoryGetMappedBuffer + * cuPythonInit() + * if __cuExternalMemoryGetMappedBuffer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuExternalMemoryGetMappedBuffer" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4945 + * with gil: + * raise RuntimeError('Function "cuExternalMemoryGetMappedBuffer" not found') + * err = ( __cuExternalMemoryGetMappedBuffer)(devPtr, extMem, bufferDesc) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, CUexternalMemory, CUDA_EXTERNAL_MEMORY_BUFFER_DESC const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedBuffer)(__pyx_v_devPtr, __pyx_v_extMem, __pyx_v_bufferDesc); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4945, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4946 + * raise RuntimeError('Function "cuExternalMemoryGetMappedBuffer" not found') + * err = ( __cuExternalMemoryGetMappedBuffer)(devPtr, extMem, bufferDesc) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuExternalMemoryGetMappedMipmappedArray(CUmipmappedArray* mipmap, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC* mipmapDesc) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4939 + * return err + * + * cdef CUresult _cuExternalMemoryGetMappedBuffer(CUdeviceptr* devPtr, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_BUFFER_DESC* bufferDesc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuExternalMemoryGetMappedBuffer + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuExternalMemoryGetMappedBuffer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4948 + * return err + * + * cdef CUresult _cuExternalMemoryGetMappedMipmappedArray(CUmipmappedArray* mipmap, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC* mipmapDesc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuExternalMemoryGetMappedMipmappedArray + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuExternalMemoryGetMappedMipmappedArray(CUmipmappedArray *__pyx_v_mipmap, CUexternalMemory __pyx_v_extMem, CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC const *__pyx_v_mipmapDesc) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuExternalMemoryGetMappedMipmappedArray", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4950 + * cdef CUresult _cuExternalMemoryGetMappedMipmappedArray(CUmipmappedArray* mipmap, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC* mipmapDesc) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuExternalMemoryGetMappedMipmappedArray + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuExternalMemoryGetMappedMipmappedArray == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4950, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4951 + * global __cuExternalMemoryGetMappedMipmappedArray + * cuPythonInit() + * if __cuExternalMemoryGetMappedMipmappedArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuExternalMemoryGetMappedMipmappedArray" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedMipmappedArray == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4952 + * cuPythonInit() + * if __cuExternalMemoryGetMappedMipmappedArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuExternalMemoryGetMappedMipmappedArray" not found') + * err = ( __cuExternalMemoryGetMappedMipmappedArray)(mipmap, extMem, mipmapDesc) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4953 + * if __cuExternalMemoryGetMappedMipmappedArray == NULL: + * with gil: + * raise RuntimeError('Function "cuExternalMemoryGetMappedMipmappedArray" not found') # <<<<<<<<<<<<<< + * err = ( __cuExternalMemoryGetMappedMipmappedArray)(mipmap, extMem, mipmapDesc) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuExternalMemoryGetMapp_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4953, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4953, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4952 + * cuPythonInit() + * if __cuExternalMemoryGetMappedMipmappedArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuExternalMemoryGetMappedMipmappedArray" not found') + * err = ( __cuExternalMemoryGetMappedMipmappedArray)(mipmap, extMem, mipmapDesc) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4951 + * global __cuExternalMemoryGetMappedMipmappedArray + * cuPythonInit() + * if __cuExternalMemoryGetMappedMipmappedArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuExternalMemoryGetMappedMipmappedArray" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4954 + * with gil: + * raise RuntimeError('Function "cuExternalMemoryGetMappedMipmappedArray" not found') + * err = ( __cuExternalMemoryGetMappedMipmappedArray)(mipmap, extMem, mipmapDesc) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmipmappedArray *, CUexternalMemory, CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedMipmappedArray)(__pyx_v_mipmap, __pyx_v_extMem, __pyx_v_mipmapDesc); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4954, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4955 + * raise RuntimeError('Function "cuExternalMemoryGetMappedMipmappedArray" not found') + * err = ( __cuExternalMemoryGetMappedMipmappedArray)(mipmap, extMem, mipmapDesc) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDestroyExternalMemory(CUexternalMemory extMem) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4948 + * return err + * + * cdef CUresult _cuExternalMemoryGetMappedMipmappedArray(CUmipmappedArray* mipmap, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC* mipmapDesc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuExternalMemoryGetMappedMipmappedArray + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuExternalMemoryGetMappedMipmappedArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4957 + * return err + * + * cdef CUresult _cuDestroyExternalMemory(CUexternalMemory extMem) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDestroyExternalMemory + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDestroyExternalMemory(CUexternalMemory __pyx_v_extMem) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDestroyExternalMemory", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4959 + * cdef CUresult _cuDestroyExternalMemory(CUexternalMemory extMem) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDestroyExternalMemory + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDestroyExternalMemory == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4959, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4960 + * global __cuDestroyExternalMemory + * cuPythonInit() + * if __cuDestroyExternalMemory == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDestroyExternalMemory" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalMemory == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4961 + * cuPythonInit() + * if __cuDestroyExternalMemory == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDestroyExternalMemory" not found') + * err = ( __cuDestroyExternalMemory)(extMem) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4962 + * if __cuDestroyExternalMemory == NULL: + * with gil: + * raise RuntimeError('Function "cuDestroyExternalMemory" not found') # <<<<<<<<<<<<<< + * err = ( __cuDestroyExternalMemory)(extMem) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDestroyExternalMemory}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4962, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4962, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4961 + * cuPythonInit() + * if __cuDestroyExternalMemory == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDestroyExternalMemory" not found') + * err = ( __cuDestroyExternalMemory)(extMem) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4960 + * global __cuDestroyExternalMemory + * cuPythonInit() + * if __cuDestroyExternalMemory == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDestroyExternalMemory" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4963 + * with gil: + * raise RuntimeError('Function "cuDestroyExternalMemory" not found') + * err = ( __cuDestroyExternalMemory)(extMem) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUexternalMemory))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalMemory)(__pyx_v_extMem); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4963, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4964 + * raise RuntimeError('Function "cuDestroyExternalMemory" not found') + * err = ( __cuDestroyExternalMemory)(extMem) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuImportExternalSemaphore(CUexternalSemaphore* extSem_out, const CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC* semHandleDesc) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4957 + * return err + * + * cdef CUresult _cuDestroyExternalMemory(CUexternalMemory extMem) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDestroyExternalMemory + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDestroyExternalMemory", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4966 + * return err + * + * cdef CUresult _cuImportExternalSemaphore(CUexternalSemaphore* extSem_out, const CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC* semHandleDesc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuImportExternalSemaphore + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuImportExternalSemaphore(CUexternalSemaphore *__pyx_v_extSem_out, CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC const *__pyx_v_semHandleDesc) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuImportExternalSemaphore", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4968 + * cdef CUresult _cuImportExternalSemaphore(CUexternalSemaphore* extSem_out, const CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC* semHandleDesc) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuImportExternalSemaphore + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuImportExternalSemaphore == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4968, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4969 + * global __cuImportExternalSemaphore + * cuPythonInit() + * if __cuImportExternalSemaphore == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuImportExternalSemaphore" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalSemaphore == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4970 + * cuPythonInit() + * if __cuImportExternalSemaphore == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuImportExternalSemaphore" not found') + * err = ( __cuImportExternalSemaphore)(extSem_out, semHandleDesc) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4971 + * if __cuImportExternalSemaphore == NULL: + * with gil: + * raise RuntimeError('Function "cuImportExternalSemaphore" not found') # <<<<<<<<<<<<<< + * err = ( __cuImportExternalSemaphore)(extSem_out, semHandleDesc) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuImportExternalSemapho}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4971, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4971, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4970 + * cuPythonInit() + * if __cuImportExternalSemaphore == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuImportExternalSemaphore" not found') + * err = ( __cuImportExternalSemaphore)(extSem_out, semHandleDesc) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4969 + * global __cuImportExternalSemaphore + * cuPythonInit() + * if __cuImportExternalSemaphore == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuImportExternalSemaphore" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4972 + * with gil: + * raise RuntimeError('Function "cuImportExternalSemaphore" not found') + * err = ( __cuImportExternalSemaphore)(extSem_out, semHandleDesc) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUexternalSemaphore *, CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalSemaphore)(__pyx_v_extSem_out, __pyx_v_semHandleDesc); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4972, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4973 + * raise RuntimeError('Function "cuImportExternalSemaphore" not found') + * err = ( __cuImportExternalSemaphore)(extSem_out, semHandleDesc) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuSignalExternalSemaphoresAsync(const CUexternalSemaphore* extSemArray, const CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS* paramsArray, unsigned int numExtSems, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4966 + * return err + * + * cdef CUresult _cuImportExternalSemaphore(CUexternalSemaphore* extSem_out, const CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC* semHandleDesc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuImportExternalSemaphore + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuImportExternalSemaphore", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4975 + * return err + * + * cdef CUresult _cuSignalExternalSemaphoresAsync(const CUexternalSemaphore* extSemArray, const CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS* paramsArray, unsigned int numExtSems, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuSignalExternalSemaphoresAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuSignalExternalSemaphoresAsync(CUexternalSemaphore const *__pyx_v_extSemArray, CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS const *__pyx_v_paramsArray, unsigned int __pyx_v_numExtSems, CUstream __pyx_v_stream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuSignalExternalSemaphoresAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4977 + * cdef CUresult _cuSignalExternalSemaphoresAsync(const CUexternalSemaphore* extSemArray, const CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS* paramsArray, unsigned int numExtSems, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuSignalExternalSemaphoresAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuSignalExternalSemaphoresAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4977, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4978 + * global __cuSignalExternalSemaphoresAsync + * cuPythonInit() + * if __cuSignalExternalSemaphoresAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuSignalExternalSemaphoresAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSignalExternalSemaphoresAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4979 + * cuPythonInit() + * if __cuSignalExternalSemaphoresAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuSignalExternalSemaphoresAsync" not found') + * err = ( __cuSignalExternalSemaphoresAsync)(extSemArray, paramsArray, numExtSems, stream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4980 + * if __cuSignalExternalSemaphoresAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuSignalExternalSemaphoresAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuSignalExternalSemaphoresAsync)(extSemArray, paramsArray, numExtSems, stream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuSignalExternalSemapho}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4980, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4980, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4979 + * cuPythonInit() + * if __cuSignalExternalSemaphoresAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuSignalExternalSemaphoresAsync" not found') + * err = ( __cuSignalExternalSemaphoresAsync)(extSemArray, paramsArray, numExtSems, stream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4978 + * global __cuSignalExternalSemaphoresAsync + * cuPythonInit() + * if __cuSignalExternalSemaphoresAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuSignalExternalSemaphoresAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4981 + * with gil: + * raise RuntimeError('Function "cuSignalExternalSemaphoresAsync" not found') + * err = ( __cuSignalExternalSemaphoresAsync)(extSemArray, paramsArray, numExtSems, stream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUexternalSemaphore const *, CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS const *, unsigned int, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSignalExternalSemaphoresAsync)(__pyx_v_extSemArray, __pyx_v_paramsArray, __pyx_v_numExtSems, __pyx_v_stream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4981, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4982 + * raise RuntimeError('Function "cuSignalExternalSemaphoresAsync" not found') + * err = ( __cuSignalExternalSemaphoresAsync)(extSemArray, paramsArray, numExtSems, stream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuWaitExternalSemaphoresAsync(const CUexternalSemaphore* extSemArray, const CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS* paramsArray, unsigned int numExtSems, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4975 + * return err + * + * cdef CUresult _cuSignalExternalSemaphoresAsync(const CUexternalSemaphore* extSemArray, const CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS* paramsArray, unsigned int numExtSems, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuSignalExternalSemaphoresAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuSignalExternalSemaphoresAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4984 + * return err + * + * cdef CUresult _cuWaitExternalSemaphoresAsync(const CUexternalSemaphore* extSemArray, const CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS* paramsArray, unsigned int numExtSems, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuWaitExternalSemaphoresAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuWaitExternalSemaphoresAsync(CUexternalSemaphore const *__pyx_v_extSemArray, CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS const *__pyx_v_paramsArray, unsigned int __pyx_v_numExtSems, CUstream __pyx_v_stream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuWaitExternalSemaphoresAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4986 + * cdef CUresult _cuWaitExternalSemaphoresAsync(const CUexternalSemaphore* extSemArray, const CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS* paramsArray, unsigned int numExtSems, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuWaitExternalSemaphoresAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuWaitExternalSemaphoresAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4986, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4987 + * global __cuWaitExternalSemaphoresAsync + * cuPythonInit() + * if __cuWaitExternalSemaphoresAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuWaitExternalSemaphoresAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuWaitExternalSemaphoresAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4988 + * cuPythonInit() + * if __cuWaitExternalSemaphoresAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuWaitExternalSemaphoresAsync" not found') + * err = ( __cuWaitExternalSemaphoresAsync)(extSemArray, paramsArray, numExtSems, stream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4989 + * if __cuWaitExternalSemaphoresAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuWaitExternalSemaphoresAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuWaitExternalSemaphoresAsync)(extSemArray, paramsArray, numExtSems, stream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuWaitExternalSemaphore}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4989, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4989, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4988 + * cuPythonInit() + * if __cuWaitExternalSemaphoresAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuWaitExternalSemaphoresAsync" not found') + * err = ( __cuWaitExternalSemaphoresAsync)(extSemArray, paramsArray, numExtSems, stream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4987 + * global __cuWaitExternalSemaphoresAsync + * cuPythonInit() + * if __cuWaitExternalSemaphoresAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuWaitExternalSemaphoresAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4990 + * with gil: + * raise RuntimeError('Function "cuWaitExternalSemaphoresAsync" not found') + * err = ( __cuWaitExternalSemaphoresAsync)(extSemArray, paramsArray, numExtSems, stream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUexternalSemaphore const *, CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS const *, unsigned int, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuWaitExternalSemaphoresAsync)(__pyx_v_extSemArray, __pyx_v_paramsArray, __pyx_v_numExtSems, __pyx_v_stream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4990, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":4991 + * raise RuntimeError('Function "cuWaitExternalSemaphoresAsync" not found') + * err = ( __cuWaitExternalSemaphoresAsync)(extSemArray, paramsArray, numExtSems, stream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDestroyExternalSemaphore(CUexternalSemaphore extSem) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4984 + * return err + * + * cdef CUresult _cuWaitExternalSemaphoresAsync(const CUexternalSemaphore* extSemArray, const CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS* paramsArray, unsigned int numExtSems, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuWaitExternalSemaphoresAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuWaitExternalSemaphoresAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":4993 + * return err + * + * cdef CUresult _cuDestroyExternalSemaphore(CUexternalSemaphore extSem) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDestroyExternalSemaphore + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDestroyExternalSemaphore(CUexternalSemaphore __pyx_v_extSem) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDestroyExternalSemaphore", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":4995 + * cdef CUresult _cuDestroyExternalSemaphore(CUexternalSemaphore extSem) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDestroyExternalSemaphore + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDestroyExternalSemaphore == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 4995, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":4996 + * global __cuDestroyExternalSemaphore + * cuPythonInit() + * if __cuDestroyExternalSemaphore == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDestroyExternalSemaphore" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalSemaphore == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":4997 + * cuPythonInit() + * if __cuDestroyExternalSemaphore == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDestroyExternalSemaphore" not found') + * err = ( __cuDestroyExternalSemaphore)(extSem) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":4998 + * if __cuDestroyExternalSemaphore == NULL: + * with gil: + * raise RuntimeError('Function "cuDestroyExternalSemaphore" not found') # <<<<<<<<<<<<<< + * err = ( __cuDestroyExternalSemaphore)(extSem) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDestroyExternalSemaph}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 4998, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 4998, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4997 + * cuPythonInit() + * if __cuDestroyExternalSemaphore == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDestroyExternalSemaphore" not found') + * err = ( __cuDestroyExternalSemaphore)(extSem) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4996 + * global __cuDestroyExternalSemaphore + * cuPythonInit() + * if __cuDestroyExternalSemaphore == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDestroyExternalSemaphore" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":4999 + * with gil: + * raise RuntimeError('Function "cuDestroyExternalSemaphore" not found') + * err = ( __cuDestroyExternalSemaphore)(extSem) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUexternalSemaphore))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalSemaphore)(__pyx_v_extSem); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 4999, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5000 + * raise RuntimeError('Function "cuDestroyExternalSemaphore" not found') + * err = ( __cuDestroyExternalSemaphore)(extSem) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamWaitValue32_v2(CUstream stream, CUdeviceptr addr, cuuint32_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":4993 + * return err + * + * cdef CUresult _cuDestroyExternalSemaphore(CUexternalSemaphore extSem) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDestroyExternalSemaphore + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDestroyExternalSemaphore", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5002 + * return err + * + * cdef CUresult _cuStreamWaitValue32_v2(CUstream stream, CUdeviceptr addr, cuuint32_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamWaitValue32_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamWaitValue32_v2(CUstream __pyx_v_stream, CUdeviceptr __pyx_v_addr, cuuint32_t __pyx_v_value, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamWaitValue32_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5004 + * cdef CUresult _cuStreamWaitValue32_v2(CUstream stream, CUdeviceptr addr, cuuint32_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamWaitValue32_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamWaitValue32_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5004, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5005 + * global __cuStreamWaitValue32_v2 + * cuPythonInit() + * if __cuStreamWaitValue32_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamWaitValue32_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue32_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5006 + * cuPythonInit() + * if __cuStreamWaitValue32_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamWaitValue32_v2" not found') + * err = ( __cuStreamWaitValue32_v2)(stream, addr, value, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5007 + * if __cuStreamWaitValue32_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamWaitValue32_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamWaitValue32_v2)(stream, addr, value, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamWaitValue32_v2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5007, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5007, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5006 + * cuPythonInit() + * if __cuStreamWaitValue32_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamWaitValue32_v2" not found') + * err = ( __cuStreamWaitValue32_v2)(stream, addr, value, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5005 + * global __cuStreamWaitValue32_v2 + * cuPythonInit() + * if __cuStreamWaitValue32_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamWaitValue32_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5008 + * with gil: + * raise RuntimeError('Function "cuStreamWaitValue32_v2" not found') + * err = ( __cuStreamWaitValue32_v2)(stream, addr, value, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUdeviceptr, cuuint32_t, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue32_v2)(__pyx_v_stream, __pyx_v_addr, __pyx_v_value, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5008, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5009 + * raise RuntimeError('Function "cuStreamWaitValue32_v2" not found') + * err = ( __cuStreamWaitValue32_v2)(stream, addr, value, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamWaitValue64_v2(CUstream stream, CUdeviceptr addr, cuuint64_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5002 + * return err + * + * cdef CUresult _cuStreamWaitValue32_v2(CUstream stream, CUdeviceptr addr, cuuint32_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamWaitValue32_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamWaitValue32_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5011 + * return err + * + * cdef CUresult _cuStreamWaitValue64_v2(CUstream stream, CUdeviceptr addr, cuuint64_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamWaitValue64_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamWaitValue64_v2(CUstream __pyx_v_stream, CUdeviceptr __pyx_v_addr, cuuint64_t __pyx_v_value, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamWaitValue64_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5013 + * cdef CUresult _cuStreamWaitValue64_v2(CUstream stream, CUdeviceptr addr, cuuint64_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamWaitValue64_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamWaitValue64_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5013, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5014 + * global __cuStreamWaitValue64_v2 + * cuPythonInit() + * if __cuStreamWaitValue64_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamWaitValue64_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue64_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5015 + * cuPythonInit() + * if __cuStreamWaitValue64_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamWaitValue64_v2" not found') + * err = ( __cuStreamWaitValue64_v2)(stream, addr, value, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5016 + * if __cuStreamWaitValue64_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamWaitValue64_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamWaitValue64_v2)(stream, addr, value, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamWaitValue64_v2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5016, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5016, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5015 + * cuPythonInit() + * if __cuStreamWaitValue64_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamWaitValue64_v2" not found') + * err = ( __cuStreamWaitValue64_v2)(stream, addr, value, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5014 + * global __cuStreamWaitValue64_v2 + * cuPythonInit() + * if __cuStreamWaitValue64_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamWaitValue64_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5017 + * with gil: + * raise RuntimeError('Function "cuStreamWaitValue64_v2" not found') + * err = ( __cuStreamWaitValue64_v2)(stream, addr, value, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUdeviceptr, cuuint64_t, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue64_v2)(__pyx_v_stream, __pyx_v_addr, __pyx_v_value, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5017, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5018 + * raise RuntimeError('Function "cuStreamWaitValue64_v2" not found') + * err = ( __cuStreamWaitValue64_v2)(stream, addr, value, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamWriteValue32_v2(CUstream stream, CUdeviceptr addr, cuuint32_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5011 + * return err + * + * cdef CUresult _cuStreamWaitValue64_v2(CUstream stream, CUdeviceptr addr, cuuint64_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamWaitValue64_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamWaitValue64_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5020 + * return err + * + * cdef CUresult _cuStreamWriteValue32_v2(CUstream stream, CUdeviceptr addr, cuuint32_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamWriteValue32_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamWriteValue32_v2(CUstream __pyx_v_stream, CUdeviceptr __pyx_v_addr, cuuint32_t __pyx_v_value, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamWriteValue32_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5022 + * cdef CUresult _cuStreamWriteValue32_v2(CUstream stream, CUdeviceptr addr, cuuint32_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamWriteValue32_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamWriteValue32_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5022, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5023 + * global __cuStreamWriteValue32_v2 + * cuPythonInit() + * if __cuStreamWriteValue32_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamWriteValue32_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue32_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5024 + * cuPythonInit() + * if __cuStreamWriteValue32_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamWriteValue32_v2" not found') + * err = ( __cuStreamWriteValue32_v2)(stream, addr, value, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5025 + * if __cuStreamWriteValue32_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamWriteValue32_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamWriteValue32_v2)(stream, addr, value, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamWriteValue32_v2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5025, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5025, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5024 + * cuPythonInit() + * if __cuStreamWriteValue32_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamWriteValue32_v2" not found') + * err = ( __cuStreamWriteValue32_v2)(stream, addr, value, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5023 + * global __cuStreamWriteValue32_v2 + * cuPythonInit() + * if __cuStreamWriteValue32_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamWriteValue32_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5026 + * with gil: + * raise RuntimeError('Function "cuStreamWriteValue32_v2" not found') + * err = ( __cuStreamWriteValue32_v2)(stream, addr, value, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUdeviceptr, cuuint32_t, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue32_v2)(__pyx_v_stream, __pyx_v_addr, __pyx_v_value, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5026, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5027 + * raise RuntimeError('Function "cuStreamWriteValue32_v2" not found') + * err = ( __cuStreamWriteValue32_v2)(stream, addr, value, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamWriteValue64_v2(CUstream stream, CUdeviceptr addr, cuuint64_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5020 + * return err + * + * cdef CUresult _cuStreamWriteValue32_v2(CUstream stream, CUdeviceptr addr, cuuint32_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamWriteValue32_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamWriteValue32_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5029 + * return err + * + * cdef CUresult _cuStreamWriteValue64_v2(CUstream stream, CUdeviceptr addr, cuuint64_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamWriteValue64_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamWriteValue64_v2(CUstream __pyx_v_stream, CUdeviceptr __pyx_v_addr, cuuint64_t __pyx_v_value, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamWriteValue64_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5031 + * cdef CUresult _cuStreamWriteValue64_v2(CUstream stream, CUdeviceptr addr, cuuint64_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamWriteValue64_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamWriteValue64_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5031, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5032 + * global __cuStreamWriteValue64_v2 + * cuPythonInit() + * if __cuStreamWriteValue64_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamWriteValue64_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue64_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5033 + * cuPythonInit() + * if __cuStreamWriteValue64_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamWriteValue64_v2" not found') + * err = ( __cuStreamWriteValue64_v2)(stream, addr, value, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5034 + * if __cuStreamWriteValue64_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamWriteValue64_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamWriteValue64_v2)(stream, addr, value, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamWriteValue64_v2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5034, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5034, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5033 + * cuPythonInit() + * if __cuStreamWriteValue64_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamWriteValue64_v2" not found') + * err = ( __cuStreamWriteValue64_v2)(stream, addr, value, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5032 + * global __cuStreamWriteValue64_v2 + * cuPythonInit() + * if __cuStreamWriteValue64_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamWriteValue64_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5035 + * with gil: + * raise RuntimeError('Function "cuStreamWriteValue64_v2" not found') + * err = ( __cuStreamWriteValue64_v2)(stream, addr, value, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUdeviceptr, cuuint64_t, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue64_v2)(__pyx_v_stream, __pyx_v_addr, __pyx_v_value, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5035, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5036 + * raise RuntimeError('Function "cuStreamWriteValue64_v2" not found') + * err = ( __cuStreamWriteValue64_v2)(stream, addr, value, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamBatchMemOp_v2(CUstream stream, unsigned int count, CUstreamBatchMemOpParams* paramArray, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5029 + * return err + * + * cdef CUresult _cuStreamWriteValue64_v2(CUstream stream, CUdeviceptr addr, cuuint64_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamWriteValue64_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamWriteValue64_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5038 + * return err + * + * cdef CUresult _cuStreamBatchMemOp_v2(CUstream stream, unsigned int count, CUstreamBatchMemOpParams* paramArray, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamBatchMemOp_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamBatchMemOp_v2(CUstream __pyx_v_stream, unsigned int __pyx_v_count, CUstreamBatchMemOpParams *__pyx_v_paramArray, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamBatchMemOp_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5040 + * cdef CUresult _cuStreamBatchMemOp_v2(CUstream stream, unsigned int count, CUstreamBatchMemOpParams* paramArray, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamBatchMemOp_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamBatchMemOp_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5040, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5041 + * global __cuStreamBatchMemOp_v2 + * cuPythonInit() + * if __cuStreamBatchMemOp_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamBatchMemOp_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBatchMemOp_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5042 + * cuPythonInit() + * if __cuStreamBatchMemOp_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamBatchMemOp_v2" not found') + * err = ( __cuStreamBatchMemOp_v2)(stream, count, paramArray, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5043 + * if __cuStreamBatchMemOp_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamBatchMemOp_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamBatchMemOp_v2)(stream, count, paramArray, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamBatchMemOp_v2_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5043, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5043, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5042 + * cuPythonInit() + * if __cuStreamBatchMemOp_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamBatchMemOp_v2" not found') + * err = ( __cuStreamBatchMemOp_v2)(stream, count, paramArray, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5041 + * global __cuStreamBatchMemOp_v2 + * cuPythonInit() + * if __cuStreamBatchMemOp_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamBatchMemOp_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5044 + * with gil: + * raise RuntimeError('Function "cuStreamBatchMemOp_v2" not found') + * err = ( __cuStreamBatchMemOp_v2)(stream, count, paramArray, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, unsigned int, CUstreamBatchMemOpParams *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBatchMemOp_v2)(__pyx_v_stream, __pyx_v_count, __pyx_v_paramArray, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5044, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5045 + * raise RuntimeError('Function "cuStreamBatchMemOp_v2" not found') + * err = ( __cuStreamBatchMemOp_v2)(stream, count, paramArray, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuFuncGetAttribute(int* pi, CUfunction_attribute attrib, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5038 + * return err + * + * cdef CUresult _cuStreamBatchMemOp_v2(CUstream stream, unsigned int count, CUstreamBatchMemOpParams* paramArray, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamBatchMemOp_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamBatchMemOp_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5047 + * return err + * + * cdef CUresult _cuFuncGetAttribute(int* pi, CUfunction_attribute attrib, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncGetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncGetAttribute(int *__pyx_v_pi, CUfunction_attribute __pyx_v_attrib, CUfunction __pyx_v_hfunc) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFuncGetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5049 + * cdef CUresult _cuFuncGetAttribute(int* pi, CUfunction_attribute attrib, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuFuncGetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuFuncGetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5049, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5050 + * global __cuFuncGetAttribute + * cuPythonInit() + * if __cuFuncGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncGetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5051 + * cuPythonInit() + * if __cuFuncGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncGetAttribute" not found') + * err = ( __cuFuncGetAttribute)(pi, attrib, hfunc) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5052 + * if __cuFuncGetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuFuncGetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuFuncGetAttribute)(pi, attrib, hfunc) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuFuncGetAttribute_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5052, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5052, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5051 + * cuPythonInit() + * if __cuFuncGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncGetAttribute" not found') + * err = ( __cuFuncGetAttribute)(pi, attrib, hfunc) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5050 + * global __cuFuncGetAttribute + * cuPythonInit() + * if __cuFuncGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncGetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5053 + * with gil: + * raise RuntimeError('Function "cuFuncGetAttribute" not found') + * err = ( __cuFuncGetAttribute)(pi, attrib, hfunc) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, CUfunction_attribute, CUfunction))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetAttribute)(__pyx_v_pi, __pyx_v_attrib, __pyx_v_hfunc); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5053, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5054 + * raise RuntimeError('Function "cuFuncGetAttribute" not found') + * err = ( __cuFuncGetAttribute)(pi, attrib, hfunc) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuFuncSetAttribute(CUfunction hfunc, CUfunction_attribute attrib, int value) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5047 + * return err + * + * cdef CUresult _cuFuncGetAttribute(int* pi, CUfunction_attribute attrib, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncGetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuFuncGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5056 + * return err + * + * cdef CUresult _cuFuncSetAttribute(CUfunction hfunc, CUfunction_attribute attrib, int value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncSetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncSetAttribute(CUfunction __pyx_v_hfunc, CUfunction_attribute __pyx_v_attrib, int __pyx_v_value) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFuncSetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5058 + * cdef CUresult _cuFuncSetAttribute(CUfunction hfunc, CUfunction_attribute attrib, int value) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuFuncSetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuFuncSetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5058, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5059 + * global __cuFuncSetAttribute + * cuPythonInit() + * if __cuFuncSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncSetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5060 + * cuPythonInit() + * if __cuFuncSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncSetAttribute" not found') + * err = ( __cuFuncSetAttribute)(hfunc, attrib, value) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5061 + * if __cuFuncSetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuFuncSetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuFuncSetAttribute)(hfunc, attrib, value) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuFuncSetAttribute_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5061, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5061, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5060 + * cuPythonInit() + * if __cuFuncSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncSetAttribute" not found') + * err = ( __cuFuncSetAttribute)(hfunc, attrib, value) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5059 + * global __cuFuncSetAttribute + * cuPythonInit() + * if __cuFuncSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncSetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5062 + * with gil: + * raise RuntimeError('Function "cuFuncSetAttribute" not found') + * err = ( __cuFuncSetAttribute)(hfunc, attrib, value) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, CUfunction_attribute, int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetAttribute)(__pyx_v_hfunc, __pyx_v_attrib, __pyx_v_value); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5062, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5063 + * raise RuntimeError('Function "cuFuncSetAttribute" not found') + * err = ( __cuFuncSetAttribute)(hfunc, attrib, value) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuFuncSetCacheConfig(CUfunction hfunc, CUfunc_cache config) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5056 + * return err + * + * cdef CUresult _cuFuncSetAttribute(CUfunction hfunc, CUfunction_attribute attrib, int value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncSetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuFuncSetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5065 + * return err + * + * cdef CUresult _cuFuncSetCacheConfig(CUfunction hfunc, CUfunc_cache config) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncSetCacheConfig + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncSetCacheConfig(CUfunction __pyx_v_hfunc, CUfunc_cache __pyx_v_config) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFuncSetCacheConfig", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5067 + * cdef CUresult _cuFuncSetCacheConfig(CUfunction hfunc, CUfunc_cache config) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuFuncSetCacheConfig + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuFuncSetCacheConfig == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5067, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5068 + * global __cuFuncSetCacheConfig + * cuPythonInit() + * if __cuFuncSetCacheConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncSetCacheConfig" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetCacheConfig == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5069 + * cuPythonInit() + * if __cuFuncSetCacheConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncSetCacheConfig" not found') + * err = ( __cuFuncSetCacheConfig)(hfunc, config) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5070 + * if __cuFuncSetCacheConfig == NULL: + * with gil: + * raise RuntimeError('Function "cuFuncSetCacheConfig" not found') # <<<<<<<<<<<<<< + * err = ( __cuFuncSetCacheConfig)(hfunc, config) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuFuncSetCacheConfig_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5070, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5070, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5069 + * cuPythonInit() + * if __cuFuncSetCacheConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncSetCacheConfig" not found') + * err = ( __cuFuncSetCacheConfig)(hfunc, config) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5068 + * global __cuFuncSetCacheConfig + * cuPythonInit() + * if __cuFuncSetCacheConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncSetCacheConfig" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5071 + * with gil: + * raise RuntimeError('Function "cuFuncSetCacheConfig" not found') + * err = ( __cuFuncSetCacheConfig)(hfunc, config) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, CUfunc_cache))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetCacheConfig)(__pyx_v_hfunc, __pyx_v_config); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5071, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5072 + * raise RuntimeError('Function "cuFuncSetCacheConfig" not found') + * err = ( __cuFuncSetCacheConfig)(hfunc, config) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuFuncGetModule(CUmodule* hmod, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5065 + * return err + * + * cdef CUresult _cuFuncSetCacheConfig(CUfunction hfunc, CUfunc_cache config) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncSetCacheConfig + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuFuncSetCacheConfig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5074 + * return err + * + * cdef CUresult _cuFuncGetModule(CUmodule* hmod, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncGetModule + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncGetModule(CUmodule *__pyx_v_hmod, CUfunction __pyx_v_hfunc) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFuncGetModule", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5076 + * cdef CUresult _cuFuncGetModule(CUmodule* hmod, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuFuncGetModule + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuFuncGetModule == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5076, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5077 + * global __cuFuncGetModule + * cuPythonInit() + * if __cuFuncGetModule == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncGetModule" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetModule == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5078 + * cuPythonInit() + * if __cuFuncGetModule == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncGetModule" not found') + * err = ( __cuFuncGetModule)(hmod, hfunc) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5079 + * if __cuFuncGetModule == NULL: + * with gil: + * raise RuntimeError('Function "cuFuncGetModule" not found') # <<<<<<<<<<<<<< + * err = ( __cuFuncGetModule)(hmod, hfunc) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuFuncGetModule_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5079, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5079, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5078 + * cuPythonInit() + * if __cuFuncGetModule == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncGetModule" not found') + * err = ( __cuFuncGetModule)(hmod, hfunc) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5077 + * global __cuFuncGetModule + * cuPythonInit() + * if __cuFuncGetModule == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncGetModule" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5080 + * with gil: + * raise RuntimeError('Function "cuFuncGetModule" not found') + * err = ( __cuFuncGetModule)(hmod, hfunc) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmodule *, CUfunction))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetModule)(__pyx_v_hmod, __pyx_v_hfunc); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5080, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5081 + * raise RuntimeError('Function "cuFuncGetModule" not found') + * err = ( __cuFuncGetModule)(hmod, hfunc) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuFuncGetName(const char** name, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5074 + * return err + * + * cdef CUresult _cuFuncGetModule(CUmodule* hmod, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncGetModule + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuFuncGetModule", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5083 + * return err + * + * cdef CUresult _cuFuncGetName(const char** name, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncGetName + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncGetName(char const **__pyx_v_name, CUfunction __pyx_v_hfunc) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFuncGetName", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5085 + * cdef CUresult _cuFuncGetName(const char** name, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuFuncGetName + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuFuncGetName == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5085, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5086 + * global __cuFuncGetName + * cuPythonInit() + * if __cuFuncGetName == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncGetName" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetName == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5087 + * cuPythonInit() + * if __cuFuncGetName == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncGetName" not found') + * err = ( __cuFuncGetName)(name, hfunc) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5088 + * if __cuFuncGetName == NULL: + * with gil: + * raise RuntimeError('Function "cuFuncGetName" not found') # <<<<<<<<<<<<<< + * err = ( __cuFuncGetName)(name, hfunc) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuFuncGetName_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5088, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5088, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5087 + * cuPythonInit() + * if __cuFuncGetName == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncGetName" not found') + * err = ( __cuFuncGetName)(name, hfunc) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5086 + * global __cuFuncGetName + * cuPythonInit() + * if __cuFuncGetName == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncGetName" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5089 + * with gil: + * raise RuntimeError('Function "cuFuncGetName" not found') + * err = ( __cuFuncGetName)(name, hfunc) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(char const **, CUfunction))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetName)(__pyx_v_name, __pyx_v_hfunc); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5089, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5090 + * raise RuntimeError('Function "cuFuncGetName" not found') + * err = ( __cuFuncGetName)(name, hfunc) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuFuncGetParamInfo(CUfunction func, size_t paramIndex, size_t* paramOffset, size_t* paramSize) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5083 + * return err + * + * cdef CUresult _cuFuncGetName(const char** name, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncGetName + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuFuncGetName", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5092 + * return err + * + * cdef CUresult _cuFuncGetParamInfo(CUfunction func, size_t paramIndex, size_t* paramOffset, size_t* paramSize) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncGetParamInfo + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncGetParamInfo(CUfunction __pyx_v_func, size_t __pyx_v_paramIndex, size_t *__pyx_v_paramOffset, size_t *__pyx_v_paramSize) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFuncGetParamInfo", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5094 + * cdef CUresult _cuFuncGetParamInfo(CUfunction func, size_t paramIndex, size_t* paramOffset, size_t* paramSize) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuFuncGetParamInfo + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuFuncGetParamInfo == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5094, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5095 + * global __cuFuncGetParamInfo + * cuPythonInit() + * if __cuFuncGetParamInfo == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncGetParamInfo" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetParamInfo == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5096 + * cuPythonInit() + * if __cuFuncGetParamInfo == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncGetParamInfo" not found') + * err = ( __cuFuncGetParamInfo)(func, paramIndex, paramOffset, paramSize) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5097 + * if __cuFuncGetParamInfo == NULL: + * with gil: + * raise RuntimeError('Function "cuFuncGetParamInfo" not found') # <<<<<<<<<<<<<< + * err = ( __cuFuncGetParamInfo)(func, paramIndex, paramOffset, paramSize) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuFuncGetParamInfo_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5097, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5097, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5096 + * cuPythonInit() + * if __cuFuncGetParamInfo == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncGetParamInfo" not found') + * err = ( __cuFuncGetParamInfo)(func, paramIndex, paramOffset, paramSize) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5095 + * global __cuFuncGetParamInfo + * cuPythonInit() + * if __cuFuncGetParamInfo == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncGetParamInfo" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5098 + * with gil: + * raise RuntimeError('Function "cuFuncGetParamInfo" not found') + * err = ( __cuFuncGetParamInfo)(func, paramIndex, paramOffset, paramSize) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, size_t, size_t *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetParamInfo)(__pyx_v_func, __pyx_v_paramIndex, __pyx_v_paramOffset, __pyx_v_paramSize); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5098, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5099 + * raise RuntimeError('Function "cuFuncGetParamInfo" not found') + * err = ( __cuFuncGetParamInfo)(func, paramIndex, paramOffset, paramSize) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuFuncIsLoaded(CUfunctionLoadingState* state, CUfunction function) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5092 + * return err + * + * cdef CUresult _cuFuncGetParamInfo(CUfunction func, size_t paramIndex, size_t* paramOffset, size_t* paramSize) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncGetParamInfo + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuFuncGetParamInfo", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5101 + * return err + * + * cdef CUresult _cuFuncIsLoaded(CUfunctionLoadingState* state, CUfunction function) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncIsLoaded + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncIsLoaded(CUfunctionLoadingState *__pyx_v_state, CUfunction __pyx_v_function) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFuncIsLoaded", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5103 + * cdef CUresult _cuFuncIsLoaded(CUfunctionLoadingState* state, CUfunction function) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuFuncIsLoaded + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuFuncIsLoaded == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5103, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5104 + * global __cuFuncIsLoaded + * cuPythonInit() + * if __cuFuncIsLoaded == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncIsLoaded" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncIsLoaded == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5105 + * cuPythonInit() + * if __cuFuncIsLoaded == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncIsLoaded" not found') + * err = ( __cuFuncIsLoaded)(state, function) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5106 + * if __cuFuncIsLoaded == NULL: + * with gil: + * raise RuntimeError('Function "cuFuncIsLoaded" not found') # <<<<<<<<<<<<<< + * err = ( __cuFuncIsLoaded)(state, function) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuFuncIsLoaded_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5106, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5106, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5105 + * cuPythonInit() + * if __cuFuncIsLoaded == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncIsLoaded" not found') + * err = ( __cuFuncIsLoaded)(state, function) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5104 + * global __cuFuncIsLoaded + * cuPythonInit() + * if __cuFuncIsLoaded == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncIsLoaded" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5107 + * with gil: + * raise RuntimeError('Function "cuFuncIsLoaded" not found') + * err = ( __cuFuncIsLoaded)(state, function) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunctionLoadingState *, CUfunction))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncIsLoaded)(__pyx_v_state, __pyx_v_function); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5107, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5108 + * raise RuntimeError('Function "cuFuncIsLoaded" not found') + * err = ( __cuFuncIsLoaded)(state, function) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuFuncLoad(CUfunction function) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5101 + * return err + * + * cdef CUresult _cuFuncIsLoaded(CUfunctionLoadingState* state, CUfunction function) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncIsLoaded + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuFuncIsLoaded", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5110 + * return err + * + * cdef CUresult _cuFuncLoad(CUfunction function) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncLoad + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncLoad(CUfunction __pyx_v_function) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFuncLoad", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5112 + * cdef CUresult _cuFuncLoad(CUfunction function) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuFuncLoad + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuFuncLoad == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5112, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5113 + * global __cuFuncLoad + * cuPythonInit() + * if __cuFuncLoad == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncLoad" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncLoad == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5114 + * cuPythonInit() + * if __cuFuncLoad == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncLoad" not found') + * err = ( __cuFuncLoad)(function) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5115 + * if __cuFuncLoad == NULL: + * with gil: + * raise RuntimeError('Function "cuFuncLoad" not found') # <<<<<<<<<<<<<< + * err = ( __cuFuncLoad)(function) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuFuncLoad_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5115, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5115, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5114 + * cuPythonInit() + * if __cuFuncLoad == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncLoad" not found') + * err = ( __cuFuncLoad)(function) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5113 + * global __cuFuncLoad + * cuPythonInit() + * if __cuFuncLoad == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncLoad" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5116 + * with gil: + * raise RuntimeError('Function "cuFuncLoad" not found') + * err = ( __cuFuncLoad)(function) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncLoad)(__pyx_v_function); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5116, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5117 + * raise RuntimeError('Function "cuFuncLoad" not found') + * err = ( __cuFuncLoad)(function) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLaunchKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void** kernelParams, void** extra) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5110 + * return err + * + * cdef CUresult _cuFuncLoad(CUfunction function) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncLoad + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuFuncLoad", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5119 + * return err + * + * cdef CUresult _cuLaunchKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void** kernelParams, void** extra) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchKernel + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchKernel(CUfunction __pyx_v_f, unsigned int __pyx_v_gridDimX, unsigned int __pyx_v_gridDimY, unsigned int __pyx_v_gridDimZ, unsigned int __pyx_v_blockDimX, unsigned int __pyx_v_blockDimY, unsigned int __pyx_v_blockDimZ, unsigned int __pyx_v_sharedMemBytes, CUstream __pyx_v_hStream, void **__pyx_v_kernelParams, void **__pyx_v_extra) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLaunchKernel", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5121 + * cdef CUresult _cuLaunchKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void** kernelParams, void** extra) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLaunchKernel + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLaunchKernel == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5121, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5122 + * global __cuLaunchKernel + * cuPythonInit() + * if __cuLaunchKernel == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchKernel" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernel == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5123 + * cuPythonInit() + * if __cuLaunchKernel == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchKernel" not found') + * err = ( __cuLaunchKernel)(f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams, extra) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5124 + * if __cuLaunchKernel == NULL: + * with gil: + * raise RuntimeError('Function "cuLaunchKernel" not found') # <<<<<<<<<<<<<< + * err = ( __cuLaunchKernel)(f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams, extra) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLaunchKernel_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5124, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5124, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5123 + * cuPythonInit() + * if __cuLaunchKernel == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchKernel" not found') + * err = ( __cuLaunchKernel)(f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams, extra) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5122 + * global __cuLaunchKernel + * cuPythonInit() + * if __cuLaunchKernel == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchKernel" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5125 + * with gil: + * raise RuntimeError('Function "cuLaunchKernel" not found') + * err = ( __cuLaunchKernel)(f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams, extra) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, CUstream, void **, void **))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernel)(__pyx_v_f, __pyx_v_gridDimX, __pyx_v_gridDimY, __pyx_v_gridDimZ, __pyx_v_blockDimX, __pyx_v_blockDimY, __pyx_v_blockDimZ, __pyx_v_sharedMemBytes, __pyx_v_hStream, __pyx_v_kernelParams, __pyx_v_extra); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5125, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5126 + * raise RuntimeError('Function "cuLaunchKernel" not found') + * err = ( __cuLaunchKernel)(f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams, extra) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLaunchKernelEx(const CUlaunchConfig* config, CUfunction f, void** kernelParams, void** extra) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5119 + * return err + * + * cdef CUresult _cuLaunchKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void** kernelParams, void** extra) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchKernel + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLaunchKernel", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5128 + * return err + * + * cdef CUresult _cuLaunchKernelEx(const CUlaunchConfig* config, CUfunction f, void** kernelParams, void** extra) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchKernelEx + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchKernelEx(CUlaunchConfig const *__pyx_v_config, CUfunction __pyx_v_f, void **__pyx_v_kernelParams, void **__pyx_v_extra) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLaunchKernelEx", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5130 + * cdef CUresult _cuLaunchKernelEx(const CUlaunchConfig* config, CUfunction f, void** kernelParams, void** extra) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLaunchKernelEx + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLaunchKernelEx == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5130, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5131 + * global __cuLaunchKernelEx + * cuPythonInit() + * if __cuLaunchKernelEx == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchKernelEx" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernelEx == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5132 + * cuPythonInit() + * if __cuLaunchKernelEx == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchKernelEx" not found') + * err = ( __cuLaunchKernelEx)(config, f, kernelParams, extra) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5133 + * if __cuLaunchKernelEx == NULL: + * with gil: + * raise RuntimeError('Function "cuLaunchKernelEx" not found') # <<<<<<<<<<<<<< + * err = ( __cuLaunchKernelEx)(config, f, kernelParams, extra) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLaunchKernelEx_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5133, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5133, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5132 + * cuPythonInit() + * if __cuLaunchKernelEx == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchKernelEx" not found') + * err = ( __cuLaunchKernelEx)(config, f, kernelParams, extra) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5131 + * global __cuLaunchKernelEx + * cuPythonInit() + * if __cuLaunchKernelEx == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchKernelEx" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5134 + * with gil: + * raise RuntimeError('Function "cuLaunchKernelEx" not found') + * err = ( __cuLaunchKernelEx)(config, f, kernelParams, extra) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlaunchConfig const *, CUfunction, void **, void **))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernelEx)(__pyx_v_config, __pyx_v_f, __pyx_v_kernelParams, __pyx_v_extra); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5134, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5135 + * raise RuntimeError('Function "cuLaunchKernelEx" not found') + * err = ( __cuLaunchKernelEx)(config, f, kernelParams, extra) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLaunchCooperativeKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void** kernelParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5128 + * return err + * + * cdef CUresult _cuLaunchKernelEx(const CUlaunchConfig* config, CUfunction f, void** kernelParams, void** extra) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchKernelEx + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLaunchKernelEx", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5137 + * return err + * + * cdef CUresult _cuLaunchCooperativeKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void** kernelParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchCooperativeKernel + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchCooperativeKernel(CUfunction __pyx_v_f, unsigned int __pyx_v_gridDimX, unsigned int __pyx_v_gridDimY, unsigned int __pyx_v_gridDimZ, unsigned int __pyx_v_blockDimX, unsigned int __pyx_v_blockDimY, unsigned int __pyx_v_blockDimZ, unsigned int __pyx_v_sharedMemBytes, CUstream __pyx_v_hStream, void **__pyx_v_kernelParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLaunchCooperativeKernel", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5139 + * cdef CUresult _cuLaunchCooperativeKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void** kernelParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLaunchCooperativeKernel + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLaunchCooperativeKernel == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5139, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5140 + * global __cuLaunchCooperativeKernel + * cuPythonInit() + * if __cuLaunchCooperativeKernel == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchCooperativeKernel" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernel == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5141 + * cuPythonInit() + * if __cuLaunchCooperativeKernel == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchCooperativeKernel" not found') + * err = ( __cuLaunchCooperativeKernel)(f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5142 + * if __cuLaunchCooperativeKernel == NULL: + * with gil: + * raise RuntimeError('Function "cuLaunchCooperativeKernel" not found') # <<<<<<<<<<<<<< + * err = ( __cuLaunchCooperativeKernel)(f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLaunchCooperativeKern}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5142, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5142, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5141 + * cuPythonInit() + * if __cuLaunchCooperativeKernel == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchCooperativeKernel" not found') + * err = ( __cuLaunchCooperativeKernel)(f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5140 + * global __cuLaunchCooperativeKernel + * cuPythonInit() + * if __cuLaunchCooperativeKernel == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchCooperativeKernel" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5143 + * with gil: + * raise RuntimeError('Function "cuLaunchCooperativeKernel" not found') + * err = ( __cuLaunchCooperativeKernel)(f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, CUstream, void **))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernel)(__pyx_v_f, __pyx_v_gridDimX, __pyx_v_gridDimY, __pyx_v_gridDimZ, __pyx_v_blockDimX, __pyx_v_blockDimY, __pyx_v_blockDimZ, __pyx_v_sharedMemBytes, __pyx_v_hStream, __pyx_v_kernelParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5143, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5144 + * raise RuntimeError('Function "cuLaunchCooperativeKernel" not found') + * err = ( __cuLaunchCooperativeKernel)(f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLaunchCooperativeKernelMultiDevice(CUDA_LAUNCH_PARAMS* launchParamsList, unsigned int numDevices, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5137 + * return err + * + * cdef CUresult _cuLaunchCooperativeKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void** kernelParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchCooperativeKernel + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLaunchCooperativeKernel", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5146 + * return err + * + * cdef CUresult _cuLaunchCooperativeKernelMultiDevice(CUDA_LAUNCH_PARAMS* launchParamsList, unsigned int numDevices, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchCooperativeKernelMultiDevice + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchCooperativeKernelMultiDevice(CUDA_LAUNCH_PARAMS *__pyx_v_launchParamsList, unsigned int __pyx_v_numDevices, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLaunchCooperativeKernelMultiDevice", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5148 + * cdef CUresult _cuLaunchCooperativeKernelMultiDevice(CUDA_LAUNCH_PARAMS* launchParamsList, unsigned int numDevices, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLaunchCooperativeKernelMultiDevice + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLaunchCooperativeKernelMultiDevice == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5148, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5149 + * global __cuLaunchCooperativeKernelMultiDevice + * cuPythonInit() + * if __cuLaunchCooperativeKernelMultiDevice == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchCooperativeKernelMultiDevice" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernelMultiDevice == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5150 + * cuPythonInit() + * if __cuLaunchCooperativeKernelMultiDevice == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchCooperativeKernelMultiDevice" not found') + * err = ( __cuLaunchCooperativeKernelMultiDevice)(launchParamsList, numDevices, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5151 + * if __cuLaunchCooperativeKernelMultiDevice == NULL: + * with gil: + * raise RuntimeError('Function "cuLaunchCooperativeKernelMultiDevice" not found') # <<<<<<<<<<<<<< + * err = ( __cuLaunchCooperativeKernelMultiDevice)(launchParamsList, numDevices, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLaunchCooperativeKern_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5151, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5151, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5150 + * cuPythonInit() + * if __cuLaunchCooperativeKernelMultiDevice == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchCooperativeKernelMultiDevice" not found') + * err = ( __cuLaunchCooperativeKernelMultiDevice)(launchParamsList, numDevices, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5149 + * global __cuLaunchCooperativeKernelMultiDevice + * cuPythonInit() + * if __cuLaunchCooperativeKernelMultiDevice == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchCooperativeKernelMultiDevice" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5152 + * with gil: + * raise RuntimeError('Function "cuLaunchCooperativeKernelMultiDevice" not found') + * err = ( __cuLaunchCooperativeKernelMultiDevice)(launchParamsList, numDevices, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_LAUNCH_PARAMS *, unsigned int, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernelMultiDevice)(__pyx_v_launchParamsList, __pyx_v_numDevices, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5152, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5153 + * raise RuntimeError('Function "cuLaunchCooperativeKernelMultiDevice" not found') + * err = ( __cuLaunchCooperativeKernelMultiDevice)(launchParamsList, numDevices, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLaunchHostFunc(CUstream hStream, CUhostFn fn, void* userData) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5146 + * return err + * + * cdef CUresult _cuLaunchCooperativeKernelMultiDevice(CUDA_LAUNCH_PARAMS* launchParamsList, unsigned int numDevices, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchCooperativeKernelMultiDevice + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLaunchCooperativeKernelMultiDevice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5155 + * return err + * + * cdef CUresult _cuLaunchHostFunc(CUstream hStream, CUhostFn fn, void* userData) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchHostFunc + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchHostFunc(CUstream __pyx_v_hStream, CUhostFn __pyx_v_fn, void *__pyx_v_userData) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLaunchHostFunc", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5157 + * cdef CUresult _cuLaunchHostFunc(CUstream hStream, CUhostFn fn, void* userData) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLaunchHostFunc + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLaunchHostFunc == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5157, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5158 + * global __cuLaunchHostFunc + * cuPythonInit() + * if __cuLaunchHostFunc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchHostFunc" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchHostFunc == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5159 + * cuPythonInit() + * if __cuLaunchHostFunc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchHostFunc" not found') + * err = ( __cuLaunchHostFunc)(hStream, fn, userData) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5160 + * if __cuLaunchHostFunc == NULL: + * with gil: + * raise RuntimeError('Function "cuLaunchHostFunc" not found') # <<<<<<<<<<<<<< + * err = ( __cuLaunchHostFunc)(hStream, fn, userData) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLaunchHostFunc_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5160, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5160, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5159 + * cuPythonInit() + * if __cuLaunchHostFunc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchHostFunc" not found') + * err = ( __cuLaunchHostFunc)(hStream, fn, userData) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5158 + * global __cuLaunchHostFunc + * cuPythonInit() + * if __cuLaunchHostFunc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchHostFunc" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5161 + * with gil: + * raise RuntimeError('Function "cuLaunchHostFunc" not found') + * err = ( __cuLaunchHostFunc)(hStream, fn, userData) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUhostFn, void *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchHostFunc)(__pyx_v_hStream, __pyx_v_fn, __pyx_v_userData); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5161, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5162 + * raise RuntimeError('Function "cuLaunchHostFunc" not found') + * err = ( __cuLaunchHostFunc)(hStream, fn, userData) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuFuncSetBlockShape(CUfunction hfunc, int x, int y, int z) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5155 + * return err + * + * cdef CUresult _cuLaunchHostFunc(CUstream hStream, CUhostFn fn, void* userData) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchHostFunc + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLaunchHostFunc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5164 + * return err + * + * cdef CUresult _cuFuncSetBlockShape(CUfunction hfunc, int x, int y, int z) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncSetBlockShape + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncSetBlockShape(CUfunction __pyx_v_hfunc, int __pyx_v_x, int __pyx_v_y, int __pyx_v_z) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFuncSetBlockShape", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5166 + * cdef CUresult _cuFuncSetBlockShape(CUfunction hfunc, int x, int y, int z) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuFuncSetBlockShape + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuFuncSetBlockShape == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5166, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5167 + * global __cuFuncSetBlockShape + * cuPythonInit() + * if __cuFuncSetBlockShape == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncSetBlockShape" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetBlockShape == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5168 + * cuPythonInit() + * if __cuFuncSetBlockShape == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncSetBlockShape" not found') + * err = ( __cuFuncSetBlockShape)(hfunc, x, y, z) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5169 + * if __cuFuncSetBlockShape == NULL: + * with gil: + * raise RuntimeError('Function "cuFuncSetBlockShape" not found') # <<<<<<<<<<<<<< + * err = ( __cuFuncSetBlockShape)(hfunc, x, y, z) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuFuncSetBlockShape_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5169, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5169, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5168 + * cuPythonInit() + * if __cuFuncSetBlockShape == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncSetBlockShape" not found') + * err = ( __cuFuncSetBlockShape)(hfunc, x, y, z) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5167 + * global __cuFuncSetBlockShape + * cuPythonInit() + * if __cuFuncSetBlockShape == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncSetBlockShape" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5170 + * with gil: + * raise RuntimeError('Function "cuFuncSetBlockShape" not found') + * err = ( __cuFuncSetBlockShape)(hfunc, x, y, z) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, int, int, int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetBlockShape)(__pyx_v_hfunc, __pyx_v_x, __pyx_v_y, __pyx_v_z); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5170, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5171 + * raise RuntimeError('Function "cuFuncSetBlockShape" not found') + * err = ( __cuFuncSetBlockShape)(hfunc, x, y, z) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuFuncSetSharedSize(CUfunction hfunc, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5164 + * return err + * + * cdef CUresult _cuFuncSetBlockShape(CUfunction hfunc, int x, int y, int z) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncSetBlockShape + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuFuncSetBlockShape", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5173 + * return err + * + * cdef CUresult _cuFuncSetSharedSize(CUfunction hfunc, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncSetSharedSize + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncSetSharedSize(CUfunction __pyx_v_hfunc, unsigned int __pyx_v_numbytes) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFuncSetSharedSize", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5175 + * cdef CUresult _cuFuncSetSharedSize(CUfunction hfunc, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuFuncSetSharedSize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuFuncSetSharedSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5175, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5176 + * global __cuFuncSetSharedSize + * cuPythonInit() + * if __cuFuncSetSharedSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncSetSharedSize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5177 + * cuPythonInit() + * if __cuFuncSetSharedSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncSetSharedSize" not found') + * err = ( __cuFuncSetSharedSize)(hfunc, numbytes) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5178 + * if __cuFuncSetSharedSize == NULL: + * with gil: + * raise RuntimeError('Function "cuFuncSetSharedSize" not found') # <<<<<<<<<<<<<< + * err = ( __cuFuncSetSharedSize)(hfunc, numbytes) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuFuncSetSharedSize_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5178, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5178, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5177 + * cuPythonInit() + * if __cuFuncSetSharedSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncSetSharedSize" not found') + * err = ( __cuFuncSetSharedSize)(hfunc, numbytes) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5176 + * global __cuFuncSetSharedSize + * cuPythonInit() + * if __cuFuncSetSharedSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncSetSharedSize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5179 + * with gil: + * raise RuntimeError('Function "cuFuncSetSharedSize" not found') + * err = ( __cuFuncSetSharedSize)(hfunc, numbytes) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedSize)(__pyx_v_hfunc, __pyx_v_numbytes); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5179, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5180 + * raise RuntimeError('Function "cuFuncSetSharedSize" not found') + * err = ( __cuFuncSetSharedSize)(hfunc, numbytes) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuParamSetSize(CUfunction hfunc, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5173 + * return err + * + * cdef CUresult _cuFuncSetSharedSize(CUfunction hfunc, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncSetSharedSize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuFuncSetSharedSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5182 + * return err + * + * cdef CUresult _cuParamSetSize(CUfunction hfunc, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuParamSetSize + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuParamSetSize(CUfunction __pyx_v_hfunc, unsigned int __pyx_v_numbytes) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuParamSetSize", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5184 + * cdef CUresult _cuParamSetSize(CUfunction hfunc, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuParamSetSize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuParamSetSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5184, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5185 + * global __cuParamSetSize + * cuPythonInit() + * if __cuParamSetSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuParamSetSize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5186 + * cuPythonInit() + * if __cuParamSetSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuParamSetSize" not found') + * err = ( __cuParamSetSize)(hfunc, numbytes) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5187 + * if __cuParamSetSize == NULL: + * with gil: + * raise RuntimeError('Function "cuParamSetSize" not found') # <<<<<<<<<<<<<< + * err = ( __cuParamSetSize)(hfunc, numbytes) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuParamSetSize_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5187, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5187, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5186 + * cuPythonInit() + * if __cuParamSetSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuParamSetSize" not found') + * err = ( __cuParamSetSize)(hfunc, numbytes) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5185 + * global __cuParamSetSize + * cuPythonInit() + * if __cuParamSetSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuParamSetSize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5188 + * with gil: + * raise RuntimeError('Function "cuParamSetSize" not found') + * err = ( __cuParamSetSize)(hfunc, numbytes) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetSize)(__pyx_v_hfunc, __pyx_v_numbytes); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5188, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5189 + * raise RuntimeError('Function "cuParamSetSize" not found') + * err = ( __cuParamSetSize)(hfunc, numbytes) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuParamSeti(CUfunction hfunc, int offset, unsigned int value) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5182 + * return err + * + * cdef CUresult _cuParamSetSize(CUfunction hfunc, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuParamSetSize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuParamSetSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5191 + * return err + * + * cdef CUresult _cuParamSeti(CUfunction hfunc, int offset, unsigned int value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuParamSeti + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuParamSeti(CUfunction __pyx_v_hfunc, int __pyx_v_offset, unsigned int __pyx_v_value) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuParamSeti", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5193 + * cdef CUresult _cuParamSeti(CUfunction hfunc, int offset, unsigned int value) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuParamSeti + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuParamSeti == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5193, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5194 + * global __cuParamSeti + * cuPythonInit() + * if __cuParamSeti == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuParamSeti" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSeti == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5195 + * cuPythonInit() + * if __cuParamSeti == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuParamSeti" not found') + * err = ( __cuParamSeti)(hfunc, offset, value) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5196 + * if __cuParamSeti == NULL: + * with gil: + * raise RuntimeError('Function "cuParamSeti" not found') # <<<<<<<<<<<<<< + * err = ( __cuParamSeti)(hfunc, offset, value) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuParamSeti_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5196, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5196, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5195 + * cuPythonInit() + * if __cuParamSeti == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuParamSeti" not found') + * err = ( __cuParamSeti)(hfunc, offset, value) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5194 + * global __cuParamSeti + * cuPythonInit() + * if __cuParamSeti == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuParamSeti" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5197 + * with gil: + * raise RuntimeError('Function "cuParamSeti" not found') + * err = ( __cuParamSeti)(hfunc, offset, value) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, int, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSeti)(__pyx_v_hfunc, __pyx_v_offset, __pyx_v_value); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5197, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5198 + * raise RuntimeError('Function "cuParamSeti" not found') + * err = ( __cuParamSeti)(hfunc, offset, value) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuParamSetf(CUfunction hfunc, int offset, float value) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5191 + * return err + * + * cdef CUresult _cuParamSeti(CUfunction hfunc, int offset, unsigned int value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuParamSeti + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuParamSeti", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5200 + * return err + * + * cdef CUresult _cuParamSetf(CUfunction hfunc, int offset, float value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuParamSetf + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuParamSetf(CUfunction __pyx_v_hfunc, int __pyx_v_offset, float __pyx_v_value) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuParamSetf", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5202 + * cdef CUresult _cuParamSetf(CUfunction hfunc, int offset, float value) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuParamSetf + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuParamSetf == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5202, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5203 + * global __cuParamSetf + * cuPythonInit() + * if __cuParamSetf == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuParamSetf" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetf == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5204 + * cuPythonInit() + * if __cuParamSetf == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuParamSetf" not found') + * err = ( __cuParamSetf)(hfunc, offset, value) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5205 + * if __cuParamSetf == NULL: + * with gil: + * raise RuntimeError('Function "cuParamSetf" not found') # <<<<<<<<<<<<<< + * err = ( __cuParamSetf)(hfunc, offset, value) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuParamSetf_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5205, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5205, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5204 + * cuPythonInit() + * if __cuParamSetf == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuParamSetf" not found') + * err = ( __cuParamSetf)(hfunc, offset, value) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5203 + * global __cuParamSetf + * cuPythonInit() + * if __cuParamSetf == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuParamSetf" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5206 + * with gil: + * raise RuntimeError('Function "cuParamSetf" not found') + * err = ( __cuParamSetf)(hfunc, offset, value) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, int, float))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetf)(__pyx_v_hfunc, __pyx_v_offset, __pyx_v_value); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5206, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5207 + * raise RuntimeError('Function "cuParamSetf" not found') + * err = ( __cuParamSetf)(hfunc, offset, value) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuParamSetv(CUfunction hfunc, int offset, void* ptr, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5200 + * return err + * + * cdef CUresult _cuParamSetf(CUfunction hfunc, int offset, float value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuParamSetf + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuParamSetf", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5209 + * return err + * + * cdef CUresult _cuParamSetv(CUfunction hfunc, int offset, void* ptr, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuParamSetv + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuParamSetv(CUfunction __pyx_v_hfunc, int __pyx_v_offset, void *__pyx_v_ptr, unsigned int __pyx_v_numbytes) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuParamSetv", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5211 + * cdef CUresult _cuParamSetv(CUfunction hfunc, int offset, void* ptr, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuParamSetv + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuParamSetv == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5211, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5212 + * global __cuParamSetv + * cuPythonInit() + * if __cuParamSetv == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuParamSetv" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetv == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5213 + * cuPythonInit() + * if __cuParamSetv == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuParamSetv" not found') + * err = ( __cuParamSetv)(hfunc, offset, ptr, numbytes) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5214 + * if __cuParamSetv == NULL: + * with gil: + * raise RuntimeError('Function "cuParamSetv" not found') # <<<<<<<<<<<<<< + * err = ( __cuParamSetv)(hfunc, offset, ptr, numbytes) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuParamSetv_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5214, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5214, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5213 + * cuPythonInit() + * if __cuParamSetv == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuParamSetv" not found') + * err = ( __cuParamSetv)(hfunc, offset, ptr, numbytes) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5212 + * global __cuParamSetv + * cuPythonInit() + * if __cuParamSetv == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuParamSetv" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5215 + * with gil: + * raise RuntimeError('Function "cuParamSetv" not found') + * err = ( __cuParamSetv)(hfunc, offset, ptr, numbytes) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, int, void *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetv)(__pyx_v_hfunc, __pyx_v_offset, __pyx_v_ptr, __pyx_v_numbytes); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5215, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5216 + * raise RuntimeError('Function "cuParamSetv" not found') + * err = ( __cuParamSetv)(hfunc, offset, ptr, numbytes) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLaunch(CUfunction f) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5209 + * return err + * + * cdef CUresult _cuParamSetv(CUfunction hfunc, int offset, void* ptr, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuParamSetv + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuParamSetv", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5218 + * return err + * + * cdef CUresult _cuLaunch(CUfunction f) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunch + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunch(CUfunction __pyx_v_f) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLaunch", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5220 + * cdef CUresult _cuLaunch(CUfunction f) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLaunch + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLaunch == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5220, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5221 + * global __cuLaunch + * cuPythonInit() + * if __cuLaunch == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunch" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunch == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5222 + * cuPythonInit() + * if __cuLaunch == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunch" not found') + * err = ( __cuLaunch)(f) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5223 + * if __cuLaunch == NULL: + * with gil: + * raise RuntimeError('Function "cuLaunch" not found') # <<<<<<<<<<<<<< + * err = ( __cuLaunch)(f) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLaunch_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5223, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5223, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5222 + * cuPythonInit() + * if __cuLaunch == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunch" not found') + * err = ( __cuLaunch)(f) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5221 + * global __cuLaunch + * cuPythonInit() + * if __cuLaunch == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunch" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5224 + * with gil: + * raise RuntimeError('Function "cuLaunch" not found') + * err = ( __cuLaunch)(f) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunch)(__pyx_v_f); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5224, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5225 + * raise RuntimeError('Function "cuLaunch" not found') + * err = ( __cuLaunch)(f) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLaunchGrid(CUfunction f, int grid_width, int grid_height) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5218 + * return err + * + * cdef CUresult _cuLaunch(CUfunction f) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunch + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLaunch", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5227 + * return err + * + * cdef CUresult _cuLaunchGrid(CUfunction f, int grid_width, int grid_height) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchGrid + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchGrid(CUfunction __pyx_v_f, int __pyx_v_grid_width, int __pyx_v_grid_height) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLaunchGrid", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5229 + * cdef CUresult _cuLaunchGrid(CUfunction f, int grid_width, int grid_height) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLaunchGrid + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLaunchGrid == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5229, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5230 + * global __cuLaunchGrid + * cuPythonInit() + * if __cuLaunchGrid == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchGrid" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGrid == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5231 + * cuPythonInit() + * if __cuLaunchGrid == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchGrid" not found') + * err = ( __cuLaunchGrid)(f, grid_width, grid_height) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5232 + * if __cuLaunchGrid == NULL: + * with gil: + * raise RuntimeError('Function "cuLaunchGrid" not found') # <<<<<<<<<<<<<< + * err = ( __cuLaunchGrid)(f, grid_width, grid_height) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLaunchGrid_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5232, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5232, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5231 + * cuPythonInit() + * if __cuLaunchGrid == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchGrid" not found') + * err = ( __cuLaunchGrid)(f, grid_width, grid_height) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5230 + * global __cuLaunchGrid + * cuPythonInit() + * if __cuLaunchGrid == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchGrid" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5233 + * with gil: + * raise RuntimeError('Function "cuLaunchGrid" not found') + * err = ( __cuLaunchGrid)(f, grid_width, grid_height) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, int, int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGrid)(__pyx_v_f, __pyx_v_grid_width, __pyx_v_grid_height); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5233, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5234 + * raise RuntimeError('Function "cuLaunchGrid" not found') + * err = ( __cuLaunchGrid)(f, grid_width, grid_height) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLaunchGridAsync(CUfunction f, int grid_width, int grid_height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5227 + * return err + * + * cdef CUresult _cuLaunchGrid(CUfunction f, int grid_width, int grid_height) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchGrid + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLaunchGrid", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5236 + * return err + * + * cdef CUresult _cuLaunchGridAsync(CUfunction f, int grid_width, int grid_height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchGridAsync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchGridAsync(CUfunction __pyx_v_f, int __pyx_v_grid_width, int __pyx_v_grid_height, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLaunchGridAsync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5238 + * cdef CUresult _cuLaunchGridAsync(CUfunction f, int grid_width, int grid_height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLaunchGridAsync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLaunchGridAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5238, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5239 + * global __cuLaunchGridAsync + * cuPythonInit() + * if __cuLaunchGridAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchGridAsync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGridAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5240 + * cuPythonInit() + * if __cuLaunchGridAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchGridAsync" not found') + * err = ( __cuLaunchGridAsync)(f, grid_width, grid_height, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5241 + * if __cuLaunchGridAsync == NULL: + * with gil: + * raise RuntimeError('Function "cuLaunchGridAsync" not found') # <<<<<<<<<<<<<< + * err = ( __cuLaunchGridAsync)(f, grid_width, grid_height, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLaunchGridAsync_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5241, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5241, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5240 + * cuPythonInit() + * if __cuLaunchGridAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLaunchGridAsync" not found') + * err = ( __cuLaunchGridAsync)(f, grid_width, grid_height, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5239 + * global __cuLaunchGridAsync + * cuPythonInit() + * if __cuLaunchGridAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLaunchGridAsync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5242 + * with gil: + * raise RuntimeError('Function "cuLaunchGridAsync" not found') + * err = ( __cuLaunchGridAsync)(f, grid_width, grid_height, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, int, int, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGridAsync)(__pyx_v_f, __pyx_v_grid_width, __pyx_v_grid_height, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5242, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5243 + * raise RuntimeError('Function "cuLaunchGridAsync" not found') + * err = ( __cuLaunchGridAsync)(f, grid_width, grid_height, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuParamSetTexRef(CUfunction hfunc, int texunit, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5236 + * return err + * + * cdef CUresult _cuLaunchGridAsync(CUfunction f, int grid_width, int grid_height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLaunchGridAsync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLaunchGridAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5245 + * return err + * + * cdef CUresult _cuParamSetTexRef(CUfunction hfunc, int texunit, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuParamSetTexRef + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuParamSetTexRef(CUfunction __pyx_v_hfunc, int __pyx_v_texunit, CUtexref __pyx_v_hTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuParamSetTexRef", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5247 + * cdef CUresult _cuParamSetTexRef(CUfunction hfunc, int texunit, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuParamSetTexRef + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuParamSetTexRef == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5247, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5248 + * global __cuParamSetTexRef + * cuPythonInit() + * if __cuParamSetTexRef == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuParamSetTexRef" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetTexRef == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5249 + * cuPythonInit() + * if __cuParamSetTexRef == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuParamSetTexRef" not found') + * err = ( __cuParamSetTexRef)(hfunc, texunit, hTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5250 + * if __cuParamSetTexRef == NULL: + * with gil: + * raise RuntimeError('Function "cuParamSetTexRef" not found') # <<<<<<<<<<<<<< + * err = ( __cuParamSetTexRef)(hfunc, texunit, hTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuParamSetTexRef_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5250, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5250, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5249 + * cuPythonInit() + * if __cuParamSetTexRef == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuParamSetTexRef" not found') + * err = ( __cuParamSetTexRef)(hfunc, texunit, hTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5248 + * global __cuParamSetTexRef + * cuPythonInit() + * if __cuParamSetTexRef == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuParamSetTexRef" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5251 + * with gil: + * raise RuntimeError('Function "cuParamSetTexRef" not found') + * err = ( __cuParamSetTexRef)(hfunc, texunit, hTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, int, CUtexref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetTexRef)(__pyx_v_hfunc, __pyx_v_texunit, __pyx_v_hTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5251, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5252 + * raise RuntimeError('Function "cuParamSetTexRef" not found') + * err = ( __cuParamSetTexRef)(hfunc, texunit, hTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuFuncSetSharedMemConfig(CUfunction hfunc, CUsharedconfig config) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5245 + * return err + * + * cdef CUresult _cuParamSetTexRef(CUfunction hfunc, int texunit, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuParamSetTexRef + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuParamSetTexRef", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5254 + * return err + * + * cdef CUresult _cuFuncSetSharedMemConfig(CUfunction hfunc, CUsharedconfig config) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncSetSharedMemConfig + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncSetSharedMemConfig(CUfunction __pyx_v_hfunc, CUsharedconfig __pyx_v_config) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFuncSetSharedMemConfig", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5256 + * cdef CUresult _cuFuncSetSharedMemConfig(CUfunction hfunc, CUsharedconfig config) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuFuncSetSharedMemConfig + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuFuncSetSharedMemConfig == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5256, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5257 + * global __cuFuncSetSharedMemConfig + * cuPythonInit() + * if __cuFuncSetSharedMemConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncSetSharedMemConfig" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedMemConfig == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5258 + * cuPythonInit() + * if __cuFuncSetSharedMemConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncSetSharedMemConfig" not found') + * err = ( __cuFuncSetSharedMemConfig)(hfunc, config) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5259 + * if __cuFuncSetSharedMemConfig == NULL: + * with gil: + * raise RuntimeError('Function "cuFuncSetSharedMemConfig" not found') # <<<<<<<<<<<<<< + * err = ( __cuFuncSetSharedMemConfig)(hfunc, config) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuFuncSetSharedMemConfi}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5259, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5259, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5258 + * cuPythonInit() + * if __cuFuncSetSharedMemConfig == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuFuncSetSharedMemConfig" not found') + * err = ( __cuFuncSetSharedMemConfig)(hfunc, config) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5257 + * global __cuFuncSetSharedMemConfig + * cuPythonInit() + * if __cuFuncSetSharedMemConfig == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuFuncSetSharedMemConfig" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5260 + * with gil: + * raise RuntimeError('Function "cuFuncSetSharedMemConfig" not found') + * err = ( __cuFuncSetSharedMemConfig)(hfunc, config) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfunction, CUsharedconfig))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedMemConfig)(__pyx_v_hfunc, __pyx_v_config); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5260, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5261 + * raise RuntimeError('Function "cuFuncSetSharedMemConfig" not found') + * err = ( __cuFuncSetSharedMemConfig)(hfunc, config) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphCreate(CUgraph* phGraph, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5254 + * return err + * + * cdef CUresult _cuFuncSetSharedMemConfig(CUfunction hfunc, CUsharedconfig config) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuFuncSetSharedMemConfig + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuFuncSetSharedMemConfig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5263 + * return err + * + * cdef CUresult _cuGraphCreate(CUgraph* phGraph, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphCreate(CUgraph *__pyx_v_phGraph, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5265 + * cdef CUresult _cuGraphCreate(CUgraph* phGraph, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5265, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5266 + * global __cuGraphCreate + * cuPythonInit() + * if __cuGraphCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5267 + * cuPythonInit() + * if __cuGraphCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphCreate" not found') + * err = ( __cuGraphCreate)(phGraph, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5268 + * if __cuGraphCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphCreate)(phGraph, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphCreate_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5268, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5268, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5267 + * cuPythonInit() + * if __cuGraphCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphCreate" not found') + * err = ( __cuGraphCreate)(phGraph, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5266 + * global __cuGraphCreate + * cuPythonInit() + * if __cuGraphCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5269 + * with gil: + * raise RuntimeError('Function "cuGraphCreate" not found') + * err = ( __cuGraphCreate)(phGraph, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphCreate)(__pyx_v_phGraph, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5269, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5270 + * raise RuntimeError('Function "cuGraphCreate" not found') + * err = ( __cuGraphCreate)(phGraph, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddKernelNode_v2(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5263 + * return err + * + * cdef CUresult _cuGraphCreate(CUgraph* phGraph, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5272 + * return err + * + * cdef CUresult _cuGraphAddKernelNode_v2(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddKernelNode_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddKernelNode_v2(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies, CUDA_KERNEL_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddKernelNode_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5274 + * cdef CUresult _cuGraphAddKernelNode_v2(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddKernelNode_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddKernelNode_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5274, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5275 + * global __cuGraphAddKernelNode_v2 + * cuPythonInit() + * if __cuGraphAddKernelNode_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddKernelNode_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddKernelNode_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5276 + * cuPythonInit() + * if __cuGraphAddKernelNode_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddKernelNode_v2" not found') + * err = ( __cuGraphAddKernelNode_v2)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5277 + * if __cuGraphAddKernelNode_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddKernelNode_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddKernelNode_v2)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddKernelNode_v2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5277, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5277, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5276 + * cuPythonInit() + * if __cuGraphAddKernelNode_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddKernelNode_v2" not found') + * err = ( __cuGraphAddKernelNode_v2)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5275 + * global __cuGraphAddKernelNode_v2 + * cuPythonInit() + * if __cuGraphAddKernelNode_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddKernelNode_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5278 + * with gil: + * raise RuntimeError('Function "cuGraphAddKernelNode_v2" not found') + * err = ( __cuGraphAddKernelNode_v2)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_KERNEL_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddKernelNode_v2)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5278, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5279 + * raise RuntimeError('Function "cuGraphAddKernelNode_v2" not found') + * err = ( __cuGraphAddKernelNode_v2)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphKernelNodeGetParams_v2(CUgraphNode hNode, CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5272 + * return err + * + * cdef CUresult _cuGraphAddKernelNode_v2(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddKernelNode_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddKernelNode_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5281 + * return err + * + * cdef CUresult _cuGraphKernelNodeGetParams_v2(CUgraphNode hNode, CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeGetParams_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphKernelNodeGetParams_v2(CUgraphNode __pyx_v_hNode, CUDA_KERNEL_NODE_PARAMS *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphKernelNodeGetParams_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5283 + * cdef CUresult _cuGraphKernelNodeGetParams_v2(CUgraphNode hNode, CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphKernelNodeGetParams_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphKernelNodeGetParams_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5283, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5284 + * global __cuGraphKernelNodeGetParams_v2 + * cuPythonInit() + * if __cuGraphKernelNodeGetParams_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeGetParams_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetParams_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5285 + * cuPythonInit() + * if __cuGraphKernelNodeGetParams_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphKernelNodeGetParams_v2" not found') + * err = ( __cuGraphKernelNodeGetParams_v2)(hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5286 + * if __cuGraphKernelNodeGetParams_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeGetParams_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphKernelNodeGetParams_v2)(hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphKernelNodeGetPar}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5286, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5286, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5285 + * cuPythonInit() + * if __cuGraphKernelNodeGetParams_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphKernelNodeGetParams_v2" not found') + * err = ( __cuGraphKernelNodeGetParams_v2)(hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5284 + * global __cuGraphKernelNodeGetParams_v2 + * cuPythonInit() + * if __cuGraphKernelNodeGetParams_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeGetParams_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5287 + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeGetParams_v2" not found') + * err = ( __cuGraphKernelNodeGetParams_v2)(hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_KERNEL_NODE_PARAMS *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetParams_v2)(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5287, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5288 + * raise RuntimeError('Function "cuGraphKernelNodeGetParams_v2" not found') + * err = ( __cuGraphKernelNodeGetParams_v2)(hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphKernelNodeSetParams_v2(CUgraphNode hNode, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5281 + * return err + * + * cdef CUresult _cuGraphKernelNodeGetParams_v2(CUgraphNode hNode, CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeGetParams_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphKernelNodeGetParams_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5290 + * return err + * + * cdef CUresult _cuGraphKernelNodeSetParams_v2(CUgraphNode hNode, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeSetParams_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphKernelNodeSetParams_v2(CUgraphNode __pyx_v_hNode, CUDA_KERNEL_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphKernelNodeSetParams_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5292 + * cdef CUresult _cuGraphKernelNodeSetParams_v2(CUgraphNode hNode, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphKernelNodeSetParams_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphKernelNodeSetParams_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5292, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5293 + * global __cuGraphKernelNodeSetParams_v2 + * cuPythonInit() + * if __cuGraphKernelNodeSetParams_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeSetParams_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetParams_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5294 + * cuPythonInit() + * if __cuGraphKernelNodeSetParams_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphKernelNodeSetParams_v2" not found') + * err = ( __cuGraphKernelNodeSetParams_v2)(hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5295 + * if __cuGraphKernelNodeSetParams_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeSetParams_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphKernelNodeSetParams_v2)(hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphKernelNodeSetPar}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5295, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5295, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5294 + * cuPythonInit() + * if __cuGraphKernelNodeSetParams_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphKernelNodeSetParams_v2" not found') + * err = ( __cuGraphKernelNodeSetParams_v2)(hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5293 + * global __cuGraphKernelNodeSetParams_v2 + * cuPythonInit() + * if __cuGraphKernelNodeSetParams_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeSetParams_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5296 + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeSetParams_v2" not found') + * err = ( __cuGraphKernelNodeSetParams_v2)(hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_KERNEL_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetParams_v2)(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5296, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5297 + * raise RuntimeError('Function "cuGraphKernelNodeSetParams_v2" not found') + * err = ( __cuGraphKernelNodeSetParams_v2)(hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddMemcpyNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_MEMCPY3D* copyParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5290 + * return err + * + * cdef CUresult _cuGraphKernelNodeSetParams_v2(CUgraphNode hNode, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeSetParams_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphKernelNodeSetParams_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5299 + * return err + * + * cdef CUresult _cuGraphAddMemcpyNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_MEMCPY3D* copyParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddMemcpyNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddMemcpyNode(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies, CUDA_MEMCPY3D const *__pyx_v_copyParams, CUcontext __pyx_v_ctx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddMemcpyNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5301 + * cdef CUresult _cuGraphAddMemcpyNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_MEMCPY3D* copyParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddMemcpyNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddMemcpyNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5301, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5302 + * global __cuGraphAddMemcpyNode + * cuPythonInit() + * if __cuGraphAddMemcpyNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddMemcpyNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemcpyNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5303 + * cuPythonInit() + * if __cuGraphAddMemcpyNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddMemcpyNode" not found') + * err = ( __cuGraphAddMemcpyNode)(phGraphNode, hGraph, dependencies, numDependencies, copyParams, ctx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5304 + * if __cuGraphAddMemcpyNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddMemcpyNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddMemcpyNode)(phGraphNode, hGraph, dependencies, numDependencies, copyParams, ctx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddMemcpyNode_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5304, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5304, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5303 + * cuPythonInit() + * if __cuGraphAddMemcpyNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddMemcpyNode" not found') + * err = ( __cuGraphAddMemcpyNode)(phGraphNode, hGraph, dependencies, numDependencies, copyParams, ctx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5302 + * global __cuGraphAddMemcpyNode + * cuPythonInit() + * if __cuGraphAddMemcpyNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddMemcpyNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5305 + * with gil: + * raise RuntimeError('Function "cuGraphAddMemcpyNode" not found') + * err = ( __cuGraphAddMemcpyNode)(phGraphNode, hGraph, dependencies, numDependencies, copyParams, ctx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_MEMCPY3D const *, CUcontext))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemcpyNode)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_copyParams, __pyx_v_ctx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5305, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5306 + * raise RuntimeError('Function "cuGraphAddMemcpyNode" not found') + * err = ( __cuGraphAddMemcpyNode)(phGraphNode, hGraph, dependencies, numDependencies, copyParams, ctx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphMemcpyNodeGetParams(CUgraphNode hNode, CUDA_MEMCPY3D* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5299 + * return err + * + * cdef CUresult _cuGraphAddMemcpyNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_MEMCPY3D* copyParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddMemcpyNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddMemcpyNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5308 + * return err + * + * cdef CUresult _cuGraphMemcpyNodeGetParams(CUgraphNode hNode, CUDA_MEMCPY3D* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphMemcpyNodeGetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphMemcpyNodeGetParams(CUgraphNode __pyx_v_hNode, CUDA_MEMCPY3D *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphMemcpyNodeGetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5310 + * cdef CUresult _cuGraphMemcpyNodeGetParams(CUgraphNode hNode, CUDA_MEMCPY3D* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphMemcpyNodeGetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphMemcpyNodeGetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5310, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5311 + * global __cuGraphMemcpyNodeGetParams + * cuPythonInit() + * if __cuGraphMemcpyNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphMemcpyNodeGetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeGetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5312 + * cuPythonInit() + * if __cuGraphMemcpyNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphMemcpyNodeGetParams" not found') + * err = ( __cuGraphMemcpyNodeGetParams)(hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5313 + * if __cuGraphMemcpyNodeGetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphMemcpyNodeGetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphMemcpyNodeGetParams)(hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphMemcpyNodeGetPar}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5313, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5313, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5312 + * cuPythonInit() + * if __cuGraphMemcpyNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphMemcpyNodeGetParams" not found') + * err = ( __cuGraphMemcpyNodeGetParams)(hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5311 + * global __cuGraphMemcpyNodeGetParams + * cuPythonInit() + * if __cuGraphMemcpyNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphMemcpyNodeGetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5314 + * with gil: + * raise RuntimeError('Function "cuGraphMemcpyNodeGetParams" not found') + * err = ( __cuGraphMemcpyNodeGetParams)(hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_MEMCPY3D *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeGetParams)(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5314, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5315 + * raise RuntimeError('Function "cuGraphMemcpyNodeGetParams" not found') + * err = ( __cuGraphMemcpyNodeGetParams)(hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphMemcpyNodeSetParams(CUgraphNode hNode, const CUDA_MEMCPY3D* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5308 + * return err + * + * cdef CUresult _cuGraphMemcpyNodeGetParams(CUgraphNode hNode, CUDA_MEMCPY3D* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphMemcpyNodeGetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphMemcpyNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5317 + * return err + * + * cdef CUresult _cuGraphMemcpyNodeSetParams(CUgraphNode hNode, const CUDA_MEMCPY3D* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphMemcpyNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphMemcpyNodeSetParams(CUgraphNode __pyx_v_hNode, CUDA_MEMCPY3D const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphMemcpyNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5319 + * cdef CUresult _cuGraphMemcpyNodeSetParams(CUgraphNode hNode, const CUDA_MEMCPY3D* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphMemcpyNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphMemcpyNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5319, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5320 + * global __cuGraphMemcpyNodeSetParams + * cuPythonInit() + * if __cuGraphMemcpyNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphMemcpyNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5321 + * cuPythonInit() + * if __cuGraphMemcpyNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphMemcpyNodeSetParams" not found') + * err = ( __cuGraphMemcpyNodeSetParams)(hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5322 + * if __cuGraphMemcpyNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphMemcpyNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphMemcpyNodeSetParams)(hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphMemcpyNodeSetPar}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5322, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5322, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5321 + * cuPythonInit() + * if __cuGraphMemcpyNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphMemcpyNodeSetParams" not found') + * err = ( __cuGraphMemcpyNodeSetParams)(hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5320 + * global __cuGraphMemcpyNodeSetParams + * cuPythonInit() + * if __cuGraphMemcpyNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphMemcpyNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5323 + * with gil: + * raise RuntimeError('Function "cuGraphMemcpyNodeSetParams" not found') + * err = ( __cuGraphMemcpyNodeSetParams)(hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_MEMCPY3D const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeSetParams)(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5323, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5324 + * raise RuntimeError('Function "cuGraphMemcpyNodeSetParams" not found') + * err = ( __cuGraphMemcpyNodeSetParams)(hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddMemsetNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_MEMSET_NODE_PARAMS* memsetParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5317 + * return err + * + * cdef CUresult _cuGraphMemcpyNodeSetParams(CUgraphNode hNode, const CUDA_MEMCPY3D* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphMemcpyNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphMemcpyNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5326 + * return err + * + * cdef CUresult _cuGraphAddMemsetNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_MEMSET_NODE_PARAMS* memsetParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddMemsetNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddMemsetNode(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies, CUDA_MEMSET_NODE_PARAMS const *__pyx_v_memsetParams, CUcontext __pyx_v_ctx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddMemsetNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5328 + * cdef CUresult _cuGraphAddMemsetNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_MEMSET_NODE_PARAMS* memsetParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddMemsetNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddMemsetNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5328, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5329 + * global __cuGraphAddMemsetNode + * cuPythonInit() + * if __cuGraphAddMemsetNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddMemsetNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemsetNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5330 + * cuPythonInit() + * if __cuGraphAddMemsetNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddMemsetNode" not found') + * err = ( __cuGraphAddMemsetNode)(phGraphNode, hGraph, dependencies, numDependencies, memsetParams, ctx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5331 + * if __cuGraphAddMemsetNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddMemsetNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddMemsetNode)(phGraphNode, hGraph, dependencies, numDependencies, memsetParams, ctx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddMemsetNode_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5331, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5331, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5330 + * cuPythonInit() + * if __cuGraphAddMemsetNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddMemsetNode" not found') + * err = ( __cuGraphAddMemsetNode)(phGraphNode, hGraph, dependencies, numDependencies, memsetParams, ctx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5329 + * global __cuGraphAddMemsetNode + * cuPythonInit() + * if __cuGraphAddMemsetNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddMemsetNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5332 + * with gil: + * raise RuntimeError('Function "cuGraphAddMemsetNode" not found') + * err = ( __cuGraphAddMemsetNode)(phGraphNode, hGraph, dependencies, numDependencies, memsetParams, ctx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_MEMSET_NODE_PARAMS const *, CUcontext))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemsetNode)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_memsetParams, __pyx_v_ctx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5332, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5333 + * raise RuntimeError('Function "cuGraphAddMemsetNode" not found') + * err = ( __cuGraphAddMemsetNode)(phGraphNode, hGraph, dependencies, numDependencies, memsetParams, ctx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphMemsetNodeGetParams(CUgraphNode hNode, CUDA_MEMSET_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5326 + * return err + * + * cdef CUresult _cuGraphAddMemsetNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_MEMSET_NODE_PARAMS* memsetParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddMemsetNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddMemsetNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5335 + * return err + * + * cdef CUresult _cuGraphMemsetNodeGetParams(CUgraphNode hNode, CUDA_MEMSET_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphMemsetNodeGetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphMemsetNodeGetParams(CUgraphNode __pyx_v_hNode, CUDA_MEMSET_NODE_PARAMS *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphMemsetNodeGetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5337 + * cdef CUresult _cuGraphMemsetNodeGetParams(CUgraphNode hNode, CUDA_MEMSET_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphMemsetNodeGetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphMemsetNodeGetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5337, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5338 + * global __cuGraphMemsetNodeGetParams + * cuPythonInit() + * if __cuGraphMemsetNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphMemsetNodeGetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeGetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5339 + * cuPythonInit() + * if __cuGraphMemsetNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphMemsetNodeGetParams" not found') + * err = ( __cuGraphMemsetNodeGetParams)(hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5340 + * if __cuGraphMemsetNodeGetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphMemsetNodeGetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphMemsetNodeGetParams)(hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphMemsetNodeGetPar}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5340, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5340, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5339 + * cuPythonInit() + * if __cuGraphMemsetNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphMemsetNodeGetParams" not found') + * err = ( __cuGraphMemsetNodeGetParams)(hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5338 + * global __cuGraphMemsetNodeGetParams + * cuPythonInit() + * if __cuGraphMemsetNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphMemsetNodeGetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5341 + * with gil: + * raise RuntimeError('Function "cuGraphMemsetNodeGetParams" not found') + * err = ( __cuGraphMemsetNodeGetParams)(hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_MEMSET_NODE_PARAMS *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeGetParams)(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5341, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5342 + * raise RuntimeError('Function "cuGraphMemsetNodeGetParams" not found') + * err = ( __cuGraphMemsetNodeGetParams)(hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphMemsetNodeSetParams(CUgraphNode hNode, const CUDA_MEMSET_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5335 + * return err + * + * cdef CUresult _cuGraphMemsetNodeGetParams(CUgraphNode hNode, CUDA_MEMSET_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphMemsetNodeGetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphMemsetNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5344 + * return err + * + * cdef CUresult _cuGraphMemsetNodeSetParams(CUgraphNode hNode, const CUDA_MEMSET_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphMemsetNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphMemsetNodeSetParams(CUgraphNode __pyx_v_hNode, CUDA_MEMSET_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphMemsetNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5346 + * cdef CUresult _cuGraphMemsetNodeSetParams(CUgraphNode hNode, const CUDA_MEMSET_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphMemsetNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphMemsetNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5346, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5347 + * global __cuGraphMemsetNodeSetParams + * cuPythonInit() + * if __cuGraphMemsetNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphMemsetNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5348 + * cuPythonInit() + * if __cuGraphMemsetNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphMemsetNodeSetParams" not found') + * err = ( __cuGraphMemsetNodeSetParams)(hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5349 + * if __cuGraphMemsetNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphMemsetNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphMemsetNodeSetParams)(hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphMemsetNodeSetPar}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5349, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5349, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5348 + * cuPythonInit() + * if __cuGraphMemsetNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphMemsetNodeSetParams" not found') + * err = ( __cuGraphMemsetNodeSetParams)(hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5347 + * global __cuGraphMemsetNodeSetParams + * cuPythonInit() + * if __cuGraphMemsetNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphMemsetNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5350 + * with gil: + * raise RuntimeError('Function "cuGraphMemsetNodeSetParams" not found') + * err = ( __cuGraphMemsetNodeSetParams)(hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_MEMSET_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeSetParams)(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5350, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5351 + * raise RuntimeError('Function "cuGraphMemsetNodeSetParams" not found') + * err = ( __cuGraphMemsetNodeSetParams)(hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddHostNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5344 + * return err + * + * cdef CUresult _cuGraphMemsetNodeSetParams(CUgraphNode hNode, const CUDA_MEMSET_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphMemsetNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphMemsetNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5353 + * return err + * + * cdef CUresult _cuGraphAddHostNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddHostNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddHostNode(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies, CUDA_HOST_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddHostNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5355 + * cdef CUresult _cuGraphAddHostNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddHostNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddHostNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5355, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5356 + * global __cuGraphAddHostNode + * cuPythonInit() + * if __cuGraphAddHostNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddHostNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddHostNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5357 + * cuPythonInit() + * if __cuGraphAddHostNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddHostNode" not found') + * err = ( __cuGraphAddHostNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5358 + * if __cuGraphAddHostNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddHostNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddHostNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddHostNode_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5358, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5358, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5357 + * cuPythonInit() + * if __cuGraphAddHostNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddHostNode" not found') + * err = ( __cuGraphAddHostNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5356 + * global __cuGraphAddHostNode + * cuPythonInit() + * if __cuGraphAddHostNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddHostNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5359 + * with gil: + * raise RuntimeError('Function "cuGraphAddHostNode" not found') + * err = ( __cuGraphAddHostNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_HOST_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddHostNode)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5359, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5360 + * raise RuntimeError('Function "cuGraphAddHostNode" not found') + * err = ( __cuGraphAddHostNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphHostNodeGetParams(CUgraphNode hNode, CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5353 + * return err + * + * cdef CUresult _cuGraphAddHostNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddHostNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddHostNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5362 + * return err + * + * cdef CUresult _cuGraphHostNodeGetParams(CUgraphNode hNode, CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphHostNodeGetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphHostNodeGetParams(CUgraphNode __pyx_v_hNode, CUDA_HOST_NODE_PARAMS *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphHostNodeGetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5364 + * cdef CUresult _cuGraphHostNodeGetParams(CUgraphNode hNode, CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphHostNodeGetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphHostNodeGetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5364, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5365 + * global __cuGraphHostNodeGetParams + * cuPythonInit() + * if __cuGraphHostNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphHostNodeGetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeGetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5366 + * cuPythonInit() + * if __cuGraphHostNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphHostNodeGetParams" not found') + * err = ( __cuGraphHostNodeGetParams)(hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5367 + * if __cuGraphHostNodeGetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphHostNodeGetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphHostNodeGetParams)(hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphHostNodeGetParam}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5367, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5367, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5366 + * cuPythonInit() + * if __cuGraphHostNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphHostNodeGetParams" not found') + * err = ( __cuGraphHostNodeGetParams)(hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5365 + * global __cuGraphHostNodeGetParams + * cuPythonInit() + * if __cuGraphHostNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphHostNodeGetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5368 + * with gil: + * raise RuntimeError('Function "cuGraphHostNodeGetParams" not found') + * err = ( __cuGraphHostNodeGetParams)(hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_HOST_NODE_PARAMS *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeGetParams)(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5368, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5369 + * raise RuntimeError('Function "cuGraphHostNodeGetParams" not found') + * err = ( __cuGraphHostNodeGetParams)(hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphHostNodeSetParams(CUgraphNode hNode, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5362 + * return err + * + * cdef CUresult _cuGraphHostNodeGetParams(CUgraphNode hNode, CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphHostNodeGetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphHostNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5371 + * return err + * + * cdef CUresult _cuGraphHostNodeSetParams(CUgraphNode hNode, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphHostNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphHostNodeSetParams(CUgraphNode __pyx_v_hNode, CUDA_HOST_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphHostNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5373 + * cdef CUresult _cuGraphHostNodeSetParams(CUgraphNode hNode, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphHostNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphHostNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5373, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5374 + * global __cuGraphHostNodeSetParams + * cuPythonInit() + * if __cuGraphHostNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphHostNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5375 + * cuPythonInit() + * if __cuGraphHostNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphHostNodeSetParams" not found') + * err = ( __cuGraphHostNodeSetParams)(hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5376 + * if __cuGraphHostNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphHostNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphHostNodeSetParams)(hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphHostNodeSetParam}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5376, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5376, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5375 + * cuPythonInit() + * if __cuGraphHostNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphHostNodeSetParams" not found') + * err = ( __cuGraphHostNodeSetParams)(hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5374 + * global __cuGraphHostNodeSetParams + * cuPythonInit() + * if __cuGraphHostNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphHostNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5377 + * with gil: + * raise RuntimeError('Function "cuGraphHostNodeSetParams" not found') + * err = ( __cuGraphHostNodeSetParams)(hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_HOST_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeSetParams)(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5377, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5378 + * raise RuntimeError('Function "cuGraphHostNodeSetParams" not found') + * err = ( __cuGraphHostNodeSetParams)(hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddChildGraphNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUgraph childGraph) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5371 + * return err + * + * cdef CUresult _cuGraphHostNodeSetParams(CUgraphNode hNode, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphHostNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphHostNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5380 + * return err + * + * cdef CUresult _cuGraphAddChildGraphNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUgraph childGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddChildGraphNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddChildGraphNode(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies, CUgraph __pyx_v_childGraph) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddChildGraphNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5382 + * cdef CUresult _cuGraphAddChildGraphNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUgraph childGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddChildGraphNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddChildGraphNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5382, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5383 + * global __cuGraphAddChildGraphNode + * cuPythonInit() + * if __cuGraphAddChildGraphNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddChildGraphNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddChildGraphNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5384 + * cuPythonInit() + * if __cuGraphAddChildGraphNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddChildGraphNode" not found') + * err = ( __cuGraphAddChildGraphNode)(phGraphNode, hGraph, dependencies, numDependencies, childGraph) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5385 + * if __cuGraphAddChildGraphNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddChildGraphNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddChildGraphNode)(phGraphNode, hGraph, dependencies, numDependencies, childGraph) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddChildGraphNod}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5385, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5385, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5384 + * cuPythonInit() + * if __cuGraphAddChildGraphNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddChildGraphNode" not found') + * err = ( __cuGraphAddChildGraphNode)(phGraphNode, hGraph, dependencies, numDependencies, childGraph) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5383 + * global __cuGraphAddChildGraphNode + * cuPythonInit() + * if __cuGraphAddChildGraphNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddChildGraphNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5386 + * with gil: + * raise RuntimeError('Function "cuGraphAddChildGraphNode" not found') + * err = ( __cuGraphAddChildGraphNode)(phGraphNode, hGraph, dependencies, numDependencies, childGraph) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUgraph))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddChildGraphNode)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_childGraph); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5386, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5387 + * raise RuntimeError('Function "cuGraphAddChildGraphNode" not found') + * err = ( __cuGraphAddChildGraphNode)(phGraphNode, hGraph, dependencies, numDependencies, childGraph) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphChildGraphNodeGetGraph(CUgraphNode hNode, CUgraph* phGraph) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5380 + * return err + * + * cdef CUresult _cuGraphAddChildGraphNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUgraph childGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddChildGraphNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddChildGraphNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5389 + * return err + * + * cdef CUresult _cuGraphChildGraphNodeGetGraph(CUgraphNode hNode, CUgraph* phGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphChildGraphNodeGetGraph + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphChildGraphNodeGetGraph(CUgraphNode __pyx_v_hNode, CUgraph *__pyx_v_phGraph) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphChildGraphNodeGetGraph", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5391 + * cdef CUresult _cuGraphChildGraphNodeGetGraph(CUgraphNode hNode, CUgraph* phGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphChildGraphNodeGetGraph + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphChildGraphNodeGetGraph == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5391, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5392 + * global __cuGraphChildGraphNodeGetGraph + * cuPythonInit() + * if __cuGraphChildGraphNodeGetGraph == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphChildGraphNodeGetGraph" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphChildGraphNodeGetGraph == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5393 + * cuPythonInit() + * if __cuGraphChildGraphNodeGetGraph == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphChildGraphNodeGetGraph" not found') + * err = ( __cuGraphChildGraphNodeGetGraph)(hNode, phGraph) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5394 + * if __cuGraphChildGraphNodeGetGraph == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphChildGraphNodeGetGraph" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphChildGraphNodeGetGraph)(hNode, phGraph) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphChildGraphNodeGe}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5394, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5394, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5393 + * cuPythonInit() + * if __cuGraphChildGraphNodeGetGraph == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphChildGraphNodeGetGraph" not found') + * err = ( __cuGraphChildGraphNodeGetGraph)(hNode, phGraph) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5392 + * global __cuGraphChildGraphNodeGetGraph + * cuPythonInit() + * if __cuGraphChildGraphNodeGetGraph == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphChildGraphNodeGetGraph" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5395 + * with gil: + * raise RuntimeError('Function "cuGraphChildGraphNodeGetGraph" not found') + * err = ( __cuGraphChildGraphNodeGetGraph)(hNode, phGraph) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUgraph *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphChildGraphNodeGetGraph)(__pyx_v_hNode, __pyx_v_phGraph); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5395, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5396 + * raise RuntimeError('Function "cuGraphChildGraphNodeGetGraph" not found') + * err = ( __cuGraphChildGraphNodeGetGraph)(hNode, phGraph) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddEmptyNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5389 + * return err + * + * cdef CUresult _cuGraphChildGraphNodeGetGraph(CUgraphNode hNode, CUgraph* phGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphChildGraphNodeGetGraph + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphChildGraphNodeGetGraph", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5398 + * return err + * + * cdef CUresult _cuGraphAddEmptyNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddEmptyNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddEmptyNode(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddEmptyNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5400 + * cdef CUresult _cuGraphAddEmptyNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddEmptyNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddEmptyNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5400, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5401 + * global __cuGraphAddEmptyNode + * cuPythonInit() + * if __cuGraphAddEmptyNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddEmptyNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEmptyNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5402 + * cuPythonInit() + * if __cuGraphAddEmptyNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddEmptyNode" not found') + * err = ( __cuGraphAddEmptyNode)(phGraphNode, hGraph, dependencies, numDependencies) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5403 + * if __cuGraphAddEmptyNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddEmptyNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddEmptyNode)(phGraphNode, hGraph, dependencies, numDependencies) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddEmptyNode_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5403, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5403, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5402 + * cuPythonInit() + * if __cuGraphAddEmptyNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddEmptyNode" not found') + * err = ( __cuGraphAddEmptyNode)(phGraphNode, hGraph, dependencies, numDependencies) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5401 + * global __cuGraphAddEmptyNode + * cuPythonInit() + * if __cuGraphAddEmptyNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddEmptyNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5404 + * with gil: + * raise RuntimeError('Function "cuGraphAddEmptyNode" not found') + * err = ( __cuGraphAddEmptyNode)(phGraphNode, hGraph, dependencies, numDependencies) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEmptyNode)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5404, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5405 + * raise RuntimeError('Function "cuGraphAddEmptyNode" not found') + * err = ( __cuGraphAddEmptyNode)(phGraphNode, hGraph, dependencies, numDependencies) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddEventRecordNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5398 + * return err + * + * cdef CUresult _cuGraphAddEmptyNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddEmptyNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddEmptyNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5407 + * return err + * + * cdef CUresult _cuGraphAddEventRecordNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddEventRecordNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddEventRecordNode(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies, CUevent __pyx_v_event) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddEventRecordNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5409 + * cdef CUresult _cuGraphAddEventRecordNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddEventRecordNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddEventRecordNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5409, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5410 + * global __cuGraphAddEventRecordNode + * cuPythonInit() + * if __cuGraphAddEventRecordNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddEventRecordNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventRecordNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5411 + * cuPythonInit() + * if __cuGraphAddEventRecordNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddEventRecordNode" not found') + * err = ( __cuGraphAddEventRecordNode)(phGraphNode, hGraph, dependencies, numDependencies, event) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5412 + * if __cuGraphAddEventRecordNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddEventRecordNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddEventRecordNode)(phGraphNode, hGraph, dependencies, numDependencies, event) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddEventRecordNo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5412, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5412, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5411 + * cuPythonInit() + * if __cuGraphAddEventRecordNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddEventRecordNode" not found') + * err = ( __cuGraphAddEventRecordNode)(phGraphNode, hGraph, dependencies, numDependencies, event) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5410 + * global __cuGraphAddEventRecordNode + * cuPythonInit() + * if __cuGraphAddEventRecordNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddEventRecordNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5413 + * with gil: + * raise RuntimeError('Function "cuGraphAddEventRecordNode" not found') + * err = ( __cuGraphAddEventRecordNode)(phGraphNode, hGraph, dependencies, numDependencies, event) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventRecordNode)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_event); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5413, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5414 + * raise RuntimeError('Function "cuGraphAddEventRecordNode" not found') + * err = ( __cuGraphAddEventRecordNode)(phGraphNode, hGraph, dependencies, numDependencies, event) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphEventRecordNodeGetEvent(CUgraphNode hNode, CUevent* event_out) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5407 + * return err + * + * cdef CUresult _cuGraphAddEventRecordNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddEventRecordNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddEventRecordNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5416 + * return err + * + * cdef CUresult _cuGraphEventRecordNodeGetEvent(CUgraphNode hNode, CUevent* event_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphEventRecordNodeGetEvent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphEventRecordNodeGetEvent(CUgraphNode __pyx_v_hNode, CUevent *__pyx_v_event_out) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphEventRecordNodeGetEvent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5418 + * cdef CUresult _cuGraphEventRecordNodeGetEvent(CUgraphNode hNode, CUevent* event_out) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphEventRecordNodeGetEvent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphEventRecordNodeGetEvent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5418, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5419 + * global __cuGraphEventRecordNodeGetEvent + * cuPythonInit() + * if __cuGraphEventRecordNodeGetEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphEventRecordNodeGetEvent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeGetEvent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5420 + * cuPythonInit() + * if __cuGraphEventRecordNodeGetEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphEventRecordNodeGetEvent" not found') + * err = ( __cuGraphEventRecordNodeGetEvent)(hNode, event_out) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5421 + * if __cuGraphEventRecordNodeGetEvent == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphEventRecordNodeGetEvent" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphEventRecordNodeGetEvent)(hNode, event_out) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphEventRecordNodeG}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5421, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5421, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5420 + * cuPythonInit() + * if __cuGraphEventRecordNodeGetEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphEventRecordNodeGetEvent" not found') + * err = ( __cuGraphEventRecordNodeGetEvent)(hNode, event_out) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5419 + * global __cuGraphEventRecordNodeGetEvent + * cuPythonInit() + * if __cuGraphEventRecordNodeGetEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphEventRecordNodeGetEvent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5422 + * with gil: + * raise RuntimeError('Function "cuGraphEventRecordNodeGetEvent" not found') + * err = ( __cuGraphEventRecordNodeGetEvent)(hNode, event_out) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUevent *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeGetEvent)(__pyx_v_hNode, __pyx_v_event_out); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5422, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5423 + * raise RuntimeError('Function "cuGraphEventRecordNodeGetEvent" not found') + * err = ( __cuGraphEventRecordNodeGetEvent)(hNode, event_out) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphEventRecordNodeSetEvent(CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5416 + * return err + * + * cdef CUresult _cuGraphEventRecordNodeGetEvent(CUgraphNode hNode, CUevent* event_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphEventRecordNodeGetEvent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphEventRecordNodeGetEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5425 + * return err + * + * cdef CUresult _cuGraphEventRecordNodeSetEvent(CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphEventRecordNodeSetEvent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphEventRecordNodeSetEvent(CUgraphNode __pyx_v_hNode, CUevent __pyx_v_event) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphEventRecordNodeSetEvent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5427 + * cdef CUresult _cuGraphEventRecordNodeSetEvent(CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphEventRecordNodeSetEvent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphEventRecordNodeSetEvent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5427, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5428 + * global __cuGraphEventRecordNodeSetEvent + * cuPythonInit() + * if __cuGraphEventRecordNodeSetEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphEventRecordNodeSetEvent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeSetEvent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5429 + * cuPythonInit() + * if __cuGraphEventRecordNodeSetEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphEventRecordNodeSetEvent" not found') + * err = ( __cuGraphEventRecordNodeSetEvent)(hNode, event) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5430 + * if __cuGraphEventRecordNodeSetEvent == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphEventRecordNodeSetEvent" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphEventRecordNodeSetEvent)(hNode, event) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphEventRecordNodeS}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5430, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5430, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5429 + * cuPythonInit() + * if __cuGraphEventRecordNodeSetEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphEventRecordNodeSetEvent" not found') + * err = ( __cuGraphEventRecordNodeSetEvent)(hNode, event) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5428 + * global __cuGraphEventRecordNodeSetEvent + * cuPythonInit() + * if __cuGraphEventRecordNodeSetEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphEventRecordNodeSetEvent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5431 + * with gil: + * raise RuntimeError('Function "cuGraphEventRecordNodeSetEvent" not found') + * err = ( __cuGraphEventRecordNodeSetEvent)(hNode, event) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeSetEvent)(__pyx_v_hNode, __pyx_v_event); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5431, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5432 + * raise RuntimeError('Function "cuGraphEventRecordNodeSetEvent" not found') + * err = ( __cuGraphEventRecordNodeSetEvent)(hNode, event) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddEventWaitNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5425 + * return err + * + * cdef CUresult _cuGraphEventRecordNodeSetEvent(CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphEventRecordNodeSetEvent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphEventRecordNodeSetEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5434 + * return err + * + * cdef CUresult _cuGraphAddEventWaitNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddEventWaitNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddEventWaitNode(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies, CUevent __pyx_v_event) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddEventWaitNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5436 + * cdef CUresult _cuGraphAddEventWaitNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddEventWaitNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddEventWaitNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5436, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5437 + * global __cuGraphAddEventWaitNode + * cuPythonInit() + * if __cuGraphAddEventWaitNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddEventWaitNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventWaitNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5438 + * cuPythonInit() + * if __cuGraphAddEventWaitNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddEventWaitNode" not found') + * err = ( __cuGraphAddEventWaitNode)(phGraphNode, hGraph, dependencies, numDependencies, event) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5439 + * if __cuGraphAddEventWaitNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddEventWaitNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddEventWaitNode)(phGraphNode, hGraph, dependencies, numDependencies, event) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddEventWaitNode}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5439, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5439, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5438 + * cuPythonInit() + * if __cuGraphAddEventWaitNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddEventWaitNode" not found') + * err = ( __cuGraphAddEventWaitNode)(phGraphNode, hGraph, dependencies, numDependencies, event) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5437 + * global __cuGraphAddEventWaitNode + * cuPythonInit() + * if __cuGraphAddEventWaitNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddEventWaitNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5440 + * with gil: + * raise RuntimeError('Function "cuGraphAddEventWaitNode" not found') + * err = ( __cuGraphAddEventWaitNode)(phGraphNode, hGraph, dependencies, numDependencies, event) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventWaitNode)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_event); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5440, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5441 + * raise RuntimeError('Function "cuGraphAddEventWaitNode" not found') + * err = ( __cuGraphAddEventWaitNode)(phGraphNode, hGraph, dependencies, numDependencies, event) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphEventWaitNodeGetEvent(CUgraphNode hNode, CUevent* event_out) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5434 + * return err + * + * cdef CUresult _cuGraphAddEventWaitNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddEventWaitNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddEventWaitNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5443 + * return err + * + * cdef CUresult _cuGraphEventWaitNodeGetEvent(CUgraphNode hNode, CUevent* event_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphEventWaitNodeGetEvent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphEventWaitNodeGetEvent(CUgraphNode __pyx_v_hNode, CUevent *__pyx_v_event_out) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphEventWaitNodeGetEvent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5445 + * cdef CUresult _cuGraphEventWaitNodeGetEvent(CUgraphNode hNode, CUevent* event_out) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphEventWaitNodeGetEvent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphEventWaitNodeGetEvent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5445, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5446 + * global __cuGraphEventWaitNodeGetEvent + * cuPythonInit() + * if __cuGraphEventWaitNodeGetEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphEventWaitNodeGetEvent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeGetEvent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5447 + * cuPythonInit() + * if __cuGraphEventWaitNodeGetEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphEventWaitNodeGetEvent" not found') + * err = ( __cuGraphEventWaitNodeGetEvent)(hNode, event_out) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5448 + * if __cuGraphEventWaitNodeGetEvent == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphEventWaitNodeGetEvent" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphEventWaitNodeGetEvent)(hNode, event_out) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphEventWaitNodeGet}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5448, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5448, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5447 + * cuPythonInit() + * if __cuGraphEventWaitNodeGetEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphEventWaitNodeGetEvent" not found') + * err = ( __cuGraphEventWaitNodeGetEvent)(hNode, event_out) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5446 + * global __cuGraphEventWaitNodeGetEvent + * cuPythonInit() + * if __cuGraphEventWaitNodeGetEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphEventWaitNodeGetEvent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5449 + * with gil: + * raise RuntimeError('Function "cuGraphEventWaitNodeGetEvent" not found') + * err = ( __cuGraphEventWaitNodeGetEvent)(hNode, event_out) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUevent *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeGetEvent)(__pyx_v_hNode, __pyx_v_event_out); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5449, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5450 + * raise RuntimeError('Function "cuGraphEventWaitNodeGetEvent" not found') + * err = ( __cuGraphEventWaitNodeGetEvent)(hNode, event_out) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphEventWaitNodeSetEvent(CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5443 + * return err + * + * cdef CUresult _cuGraphEventWaitNodeGetEvent(CUgraphNode hNode, CUevent* event_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphEventWaitNodeGetEvent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphEventWaitNodeGetEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5452 + * return err + * + * cdef CUresult _cuGraphEventWaitNodeSetEvent(CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphEventWaitNodeSetEvent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphEventWaitNodeSetEvent(CUgraphNode __pyx_v_hNode, CUevent __pyx_v_event) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphEventWaitNodeSetEvent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5454 + * cdef CUresult _cuGraphEventWaitNodeSetEvent(CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphEventWaitNodeSetEvent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphEventWaitNodeSetEvent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5454, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5455 + * global __cuGraphEventWaitNodeSetEvent + * cuPythonInit() + * if __cuGraphEventWaitNodeSetEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphEventWaitNodeSetEvent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeSetEvent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5456 + * cuPythonInit() + * if __cuGraphEventWaitNodeSetEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphEventWaitNodeSetEvent" not found') + * err = ( __cuGraphEventWaitNodeSetEvent)(hNode, event) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5457 + * if __cuGraphEventWaitNodeSetEvent == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphEventWaitNodeSetEvent" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphEventWaitNodeSetEvent)(hNode, event) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphEventWaitNodeSet}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5457, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5457, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5456 + * cuPythonInit() + * if __cuGraphEventWaitNodeSetEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphEventWaitNodeSetEvent" not found') + * err = ( __cuGraphEventWaitNodeSetEvent)(hNode, event) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5455 + * global __cuGraphEventWaitNodeSetEvent + * cuPythonInit() + * if __cuGraphEventWaitNodeSetEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphEventWaitNodeSetEvent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5458 + * with gil: + * raise RuntimeError('Function "cuGraphEventWaitNodeSetEvent" not found') + * err = ( __cuGraphEventWaitNodeSetEvent)(hNode, event) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeSetEvent)(__pyx_v_hNode, __pyx_v_event); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5458, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5459 + * raise RuntimeError('Function "cuGraphEventWaitNodeSetEvent" not found') + * err = ( __cuGraphEventWaitNodeSetEvent)(hNode, event) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddExternalSemaphoresSignalNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5452 + * return err + * + * cdef CUresult _cuGraphEventWaitNodeSetEvent(CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphEventWaitNodeSetEvent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphEventWaitNodeSetEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5461 + * return err + * + * cdef CUresult _cuGraphAddExternalSemaphoresSignalNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddExternalSemaphoresSignalNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddExternalSemaphoresSignalNode(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddExternalSemaphoresSignalNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5463 + * cdef CUresult _cuGraphAddExternalSemaphoresSignalNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddExternalSemaphoresSignalNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddExternalSemaphoresSignalNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5463, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5464 + * global __cuGraphAddExternalSemaphoresSignalNode + * cuPythonInit() + * if __cuGraphAddExternalSemaphoresSignalNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresSignalNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresSignalNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5465 + * cuPythonInit() + * if __cuGraphAddExternalSemaphoresSignalNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresSignalNode" not found') + * err = ( __cuGraphAddExternalSemaphoresSignalNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5466 + * if __cuGraphAddExternalSemaphoresSignalNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresSignalNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddExternalSemaphoresSignalNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddExternalSemap}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5466, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5466, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5465 + * cuPythonInit() + * if __cuGraphAddExternalSemaphoresSignalNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresSignalNode" not found') + * err = ( __cuGraphAddExternalSemaphoresSignalNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5464 + * global __cuGraphAddExternalSemaphoresSignalNode + * cuPythonInit() + * if __cuGraphAddExternalSemaphoresSignalNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresSignalNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5467 + * with gil: + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresSignalNode" not found') + * err = ( __cuGraphAddExternalSemaphoresSignalNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresSignalNode)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5467, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5468 + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresSignalNode" not found') + * err = ( __cuGraphAddExternalSemaphoresSignalNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExternalSemaphoresSignalNodeGetParams(CUgraphNode hNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5461 + * return err + * + * cdef CUresult _cuGraphAddExternalSemaphoresSignalNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddExternalSemaphoresSignalNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddExternalSemaphoresSignalNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5470 + * return err + * + * cdef CUresult _cuGraphExternalSemaphoresSignalNodeGetParams(CUgraphNode hNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresSignalNodeGetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExternalSemaphoresSignalNodeGetParams(CUgraphNode __pyx_v_hNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS *__pyx_v_params_out) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExternalSemaphoresSignalNodeGetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5472 + * cdef CUresult _cuGraphExternalSemaphoresSignalNodeGetParams(CUgraphNode hNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExternalSemaphoresSignalNodeGetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExternalSemaphoresSignalNodeGetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5472, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5473 + * global __cuGraphExternalSemaphoresSignalNodeGetParams + * cuPythonInit() + * if __cuGraphExternalSemaphoresSignalNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeGetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeGetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5474 + * cuPythonInit() + * if __cuGraphExternalSemaphoresSignalNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeGetParams" not found') + * err = ( __cuGraphExternalSemaphoresSignalNodeGetParams)(hNode, params_out) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5475 + * if __cuGraphExternalSemaphoresSignalNodeGetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeGetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExternalSemaphoresSignalNodeGetParams)(hNode, params_out) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExternalSemaphor}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5475, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5475, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5474 + * cuPythonInit() + * if __cuGraphExternalSemaphoresSignalNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeGetParams" not found') + * err = ( __cuGraphExternalSemaphoresSignalNodeGetParams)(hNode, params_out) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5473 + * global __cuGraphExternalSemaphoresSignalNodeGetParams + * cuPythonInit() + * if __cuGraphExternalSemaphoresSignalNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeGetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5476 + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeGetParams" not found') + * err = ( __cuGraphExternalSemaphoresSignalNodeGetParams)(hNode, params_out) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeGetParams)(__pyx_v_hNode, __pyx_v_params_out); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5476, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5477 + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeGetParams" not found') + * err = ( __cuGraphExternalSemaphoresSignalNodeGetParams)(hNode, params_out) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExternalSemaphoresSignalNodeSetParams(CUgraphNode hNode, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5470 + * return err + * + * cdef CUresult _cuGraphExternalSemaphoresSignalNodeGetParams(CUgraphNode hNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresSignalNodeGetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExternalSemaphoresSignalNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5479 + * return err + * + * cdef CUresult _cuGraphExternalSemaphoresSignalNodeSetParams(CUgraphNode hNode, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresSignalNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExternalSemaphoresSignalNodeSetParams(CUgraphNode __pyx_v_hNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExternalSemaphoresSignalNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5481 + * cdef CUresult _cuGraphExternalSemaphoresSignalNodeSetParams(CUgraphNode hNode, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExternalSemaphoresSignalNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExternalSemaphoresSignalNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5481, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5482 + * global __cuGraphExternalSemaphoresSignalNodeSetParams + * cuPythonInit() + * if __cuGraphExternalSemaphoresSignalNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5483 + * cuPythonInit() + * if __cuGraphExternalSemaphoresSignalNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeSetParams" not found') + * err = ( __cuGraphExternalSemaphoresSignalNodeSetParams)(hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5484 + * if __cuGraphExternalSemaphoresSignalNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExternalSemaphoresSignalNodeSetParams)(hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExternalSemaphor_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5484, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5484, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5483 + * cuPythonInit() + * if __cuGraphExternalSemaphoresSignalNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeSetParams" not found') + * err = ( __cuGraphExternalSemaphoresSignalNodeSetParams)(hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5482 + * global __cuGraphExternalSemaphoresSignalNodeSetParams + * cuPythonInit() + * if __cuGraphExternalSemaphoresSignalNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5485 + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeSetParams" not found') + * err = ( __cuGraphExternalSemaphoresSignalNodeSetParams)(hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeSetParams)(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5485, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5486 + * raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeSetParams" not found') + * err = ( __cuGraphExternalSemaphoresSignalNodeSetParams)(hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddExternalSemaphoresWaitNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5479 + * return err + * + * cdef CUresult _cuGraphExternalSemaphoresSignalNodeSetParams(CUgraphNode hNode, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresSignalNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExternalSemaphoresSignalNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5488 + * return err + * + * cdef CUresult _cuGraphAddExternalSemaphoresWaitNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddExternalSemaphoresWaitNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddExternalSemaphoresWaitNode(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies, CUDA_EXT_SEM_WAIT_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddExternalSemaphoresWaitNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5490 + * cdef CUresult _cuGraphAddExternalSemaphoresWaitNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddExternalSemaphoresWaitNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddExternalSemaphoresWaitNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5490, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5491 + * global __cuGraphAddExternalSemaphoresWaitNode + * cuPythonInit() + * if __cuGraphAddExternalSemaphoresWaitNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresWaitNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresWaitNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5492 + * cuPythonInit() + * if __cuGraphAddExternalSemaphoresWaitNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresWaitNode" not found') + * err = ( __cuGraphAddExternalSemaphoresWaitNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5493 + * if __cuGraphAddExternalSemaphoresWaitNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresWaitNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddExternalSemaphoresWaitNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddExternalSemap_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5493, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5493, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5492 + * cuPythonInit() + * if __cuGraphAddExternalSemaphoresWaitNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresWaitNode" not found') + * err = ( __cuGraphAddExternalSemaphoresWaitNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5491 + * global __cuGraphAddExternalSemaphoresWaitNode + * cuPythonInit() + * if __cuGraphAddExternalSemaphoresWaitNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresWaitNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5494 + * with gil: + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresWaitNode" not found') + * err = ( __cuGraphAddExternalSemaphoresWaitNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_EXT_SEM_WAIT_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresWaitNode)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5494, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5495 + * raise RuntimeError('Function "cuGraphAddExternalSemaphoresWaitNode" not found') + * err = ( __cuGraphAddExternalSemaphoresWaitNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExternalSemaphoresWaitNodeGetParams(CUgraphNode hNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5488 + * return err + * + * cdef CUresult _cuGraphAddExternalSemaphoresWaitNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddExternalSemaphoresWaitNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddExternalSemaphoresWaitNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5497 + * return err + * + * cdef CUresult _cuGraphExternalSemaphoresWaitNodeGetParams(CUgraphNode hNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresWaitNodeGetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExternalSemaphoresWaitNodeGetParams(CUgraphNode __pyx_v_hNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS *__pyx_v_params_out) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExternalSemaphoresWaitNodeGetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5499 + * cdef CUresult _cuGraphExternalSemaphoresWaitNodeGetParams(CUgraphNode hNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExternalSemaphoresWaitNodeGetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExternalSemaphoresWaitNodeGetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5499, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5500 + * global __cuGraphExternalSemaphoresWaitNodeGetParams + * cuPythonInit() + * if __cuGraphExternalSemaphoresWaitNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeGetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeGetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5501 + * cuPythonInit() + * if __cuGraphExternalSemaphoresWaitNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeGetParams" not found') + * err = ( __cuGraphExternalSemaphoresWaitNodeGetParams)(hNode, params_out) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5502 + * if __cuGraphExternalSemaphoresWaitNodeGetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeGetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExternalSemaphoresWaitNodeGetParams)(hNode, params_out) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExternalSemaphor_3}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5502, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5502, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5501 + * cuPythonInit() + * if __cuGraphExternalSemaphoresWaitNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeGetParams" not found') + * err = ( __cuGraphExternalSemaphoresWaitNodeGetParams)(hNode, params_out) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5500 + * global __cuGraphExternalSemaphoresWaitNodeGetParams + * cuPythonInit() + * if __cuGraphExternalSemaphoresWaitNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeGetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5503 + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeGetParams" not found') + * err = ( __cuGraphExternalSemaphoresWaitNodeGetParams)(hNode, params_out) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeGetParams)(__pyx_v_hNode, __pyx_v_params_out); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5503, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5504 + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeGetParams" not found') + * err = ( __cuGraphExternalSemaphoresWaitNodeGetParams)(hNode, params_out) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExternalSemaphoresWaitNodeSetParams(CUgraphNode hNode, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5497 + * return err + * + * cdef CUresult _cuGraphExternalSemaphoresWaitNodeGetParams(CUgraphNode hNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresWaitNodeGetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExternalSemaphoresWaitNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5506 + * return err + * + * cdef CUresult _cuGraphExternalSemaphoresWaitNodeSetParams(CUgraphNode hNode, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresWaitNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExternalSemaphoresWaitNodeSetParams(CUgraphNode __pyx_v_hNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExternalSemaphoresWaitNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5508 + * cdef CUresult _cuGraphExternalSemaphoresWaitNodeSetParams(CUgraphNode hNode, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExternalSemaphoresWaitNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExternalSemaphoresWaitNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5508, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5509 + * global __cuGraphExternalSemaphoresWaitNodeSetParams + * cuPythonInit() + * if __cuGraphExternalSemaphoresWaitNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5510 + * cuPythonInit() + * if __cuGraphExternalSemaphoresWaitNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeSetParams" not found') + * err = ( __cuGraphExternalSemaphoresWaitNodeSetParams)(hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5511 + * if __cuGraphExternalSemaphoresWaitNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExternalSemaphoresWaitNodeSetParams)(hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExternalSemaphor_4}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5511, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5511, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5510 + * cuPythonInit() + * if __cuGraphExternalSemaphoresWaitNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeSetParams" not found') + * err = ( __cuGraphExternalSemaphoresWaitNodeSetParams)(hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5509 + * global __cuGraphExternalSemaphoresWaitNodeSetParams + * cuPythonInit() + * if __cuGraphExternalSemaphoresWaitNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5512 + * with gil: + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeSetParams" not found') + * err = ( __cuGraphExternalSemaphoresWaitNodeSetParams)(hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeSetParams)(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5512, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5513 + * raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeSetParams" not found') + * err = ( __cuGraphExternalSemaphoresWaitNodeSetParams)(hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddBatchMemOpNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5506 + * return err + * + * cdef CUresult _cuGraphExternalSemaphoresWaitNodeSetParams(CUgraphNode hNode, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresWaitNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExternalSemaphoresWaitNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5515 + * return err + * + * cdef CUresult _cuGraphAddBatchMemOpNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddBatchMemOpNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddBatchMemOpNode(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies, CUDA_BATCH_MEM_OP_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddBatchMemOpNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5517 + * cdef CUresult _cuGraphAddBatchMemOpNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddBatchMemOpNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddBatchMemOpNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5517, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5518 + * global __cuGraphAddBatchMemOpNode + * cuPythonInit() + * if __cuGraphAddBatchMemOpNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddBatchMemOpNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddBatchMemOpNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5519 + * cuPythonInit() + * if __cuGraphAddBatchMemOpNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddBatchMemOpNode" not found') + * err = ( __cuGraphAddBatchMemOpNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5520 + * if __cuGraphAddBatchMemOpNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddBatchMemOpNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddBatchMemOpNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddBatchMemOpNod}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5520, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5520, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5519 + * cuPythonInit() + * if __cuGraphAddBatchMemOpNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddBatchMemOpNode" not found') + * err = ( __cuGraphAddBatchMemOpNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5518 + * global __cuGraphAddBatchMemOpNode + * cuPythonInit() + * if __cuGraphAddBatchMemOpNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddBatchMemOpNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5521 + * with gil: + * raise RuntimeError('Function "cuGraphAddBatchMemOpNode" not found') + * err = ( __cuGraphAddBatchMemOpNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_BATCH_MEM_OP_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddBatchMemOpNode)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5521, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5522 + * raise RuntimeError('Function "cuGraphAddBatchMemOpNode" not found') + * err = ( __cuGraphAddBatchMemOpNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphBatchMemOpNodeGetParams(CUgraphNode hNode, CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams_out) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5515 + * return err + * + * cdef CUresult _cuGraphAddBatchMemOpNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddBatchMemOpNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddBatchMemOpNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5524 + * return err + * + * cdef CUresult _cuGraphBatchMemOpNodeGetParams(CUgraphNode hNode, CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphBatchMemOpNodeGetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphBatchMemOpNodeGetParams(CUgraphNode __pyx_v_hNode, CUDA_BATCH_MEM_OP_NODE_PARAMS *__pyx_v_nodeParams_out) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphBatchMemOpNodeGetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5526 + * cdef CUresult _cuGraphBatchMemOpNodeGetParams(CUgraphNode hNode, CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams_out) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphBatchMemOpNodeGetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphBatchMemOpNodeGetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5526, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5527 + * global __cuGraphBatchMemOpNodeGetParams + * cuPythonInit() + * if __cuGraphBatchMemOpNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphBatchMemOpNodeGetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeGetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5528 + * cuPythonInit() + * if __cuGraphBatchMemOpNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphBatchMemOpNodeGetParams" not found') + * err = ( __cuGraphBatchMemOpNodeGetParams)(hNode, nodeParams_out) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5529 + * if __cuGraphBatchMemOpNodeGetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphBatchMemOpNodeGetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphBatchMemOpNodeGetParams)(hNode, nodeParams_out) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphBatchMemOpNodeGe}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5529, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5529, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5528 + * cuPythonInit() + * if __cuGraphBatchMemOpNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphBatchMemOpNodeGetParams" not found') + * err = ( __cuGraphBatchMemOpNodeGetParams)(hNode, nodeParams_out) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5527 + * global __cuGraphBatchMemOpNodeGetParams + * cuPythonInit() + * if __cuGraphBatchMemOpNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphBatchMemOpNodeGetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5530 + * with gil: + * raise RuntimeError('Function "cuGraphBatchMemOpNodeGetParams" not found') + * err = ( __cuGraphBatchMemOpNodeGetParams)(hNode, nodeParams_out) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_BATCH_MEM_OP_NODE_PARAMS *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeGetParams)(__pyx_v_hNode, __pyx_v_nodeParams_out); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5530, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5531 + * raise RuntimeError('Function "cuGraphBatchMemOpNodeGetParams" not found') + * err = ( __cuGraphBatchMemOpNodeGetParams)(hNode, nodeParams_out) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphBatchMemOpNodeSetParams(CUgraphNode hNode, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5524 + * return err + * + * cdef CUresult _cuGraphBatchMemOpNodeGetParams(CUgraphNode hNode, CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphBatchMemOpNodeGetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphBatchMemOpNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5533 + * return err + * + * cdef CUresult _cuGraphBatchMemOpNodeSetParams(CUgraphNode hNode, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphBatchMemOpNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphBatchMemOpNodeSetParams(CUgraphNode __pyx_v_hNode, CUDA_BATCH_MEM_OP_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphBatchMemOpNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5535 + * cdef CUresult _cuGraphBatchMemOpNodeSetParams(CUgraphNode hNode, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphBatchMemOpNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphBatchMemOpNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5535, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5536 + * global __cuGraphBatchMemOpNodeSetParams + * cuPythonInit() + * if __cuGraphBatchMemOpNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphBatchMemOpNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5537 + * cuPythonInit() + * if __cuGraphBatchMemOpNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphBatchMemOpNodeSetParams" not found') + * err = ( __cuGraphBatchMemOpNodeSetParams)(hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5538 + * if __cuGraphBatchMemOpNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphBatchMemOpNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphBatchMemOpNodeSetParams)(hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphBatchMemOpNodeSe}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5538, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5538, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5537 + * cuPythonInit() + * if __cuGraphBatchMemOpNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphBatchMemOpNodeSetParams" not found') + * err = ( __cuGraphBatchMemOpNodeSetParams)(hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5536 + * global __cuGraphBatchMemOpNodeSetParams + * cuPythonInit() + * if __cuGraphBatchMemOpNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphBatchMemOpNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5539 + * with gil: + * raise RuntimeError('Function "cuGraphBatchMemOpNodeSetParams" not found') + * err = ( __cuGraphBatchMemOpNodeSetParams)(hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_BATCH_MEM_OP_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeSetParams)(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5539, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5540 + * raise RuntimeError('Function "cuGraphBatchMemOpNodeSetParams" not found') + * err = ( __cuGraphBatchMemOpNodeSetParams)(hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecBatchMemOpNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5533 + * return err + * + * cdef CUresult _cuGraphBatchMemOpNodeSetParams(CUgraphNode hNode, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphBatchMemOpNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphBatchMemOpNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5542 + * return err + * + * cdef CUresult _cuGraphExecBatchMemOpNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecBatchMemOpNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecBatchMemOpNodeSetParams(CUgraphExec __pyx_v_hGraphExec, CUgraphNode __pyx_v_hNode, CUDA_BATCH_MEM_OP_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecBatchMemOpNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5544 + * cdef CUresult _cuGraphExecBatchMemOpNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecBatchMemOpNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecBatchMemOpNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5544, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5545 + * global __cuGraphExecBatchMemOpNodeSetParams + * cuPythonInit() + * if __cuGraphExecBatchMemOpNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecBatchMemOpNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecBatchMemOpNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5546 + * cuPythonInit() + * if __cuGraphExecBatchMemOpNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecBatchMemOpNodeSetParams" not found') + * err = ( __cuGraphExecBatchMemOpNodeSetParams)(hGraphExec, hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5547 + * if __cuGraphExecBatchMemOpNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecBatchMemOpNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecBatchMemOpNodeSetParams)(hGraphExec, hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecBatchMemOpNo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5547, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5547, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5546 + * cuPythonInit() + * if __cuGraphExecBatchMemOpNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecBatchMemOpNodeSetParams" not found') + * err = ( __cuGraphExecBatchMemOpNodeSetParams)(hGraphExec, hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5545 + * global __cuGraphExecBatchMemOpNodeSetParams + * cuPythonInit() + * if __cuGraphExecBatchMemOpNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecBatchMemOpNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5548 + * with gil: + * raise RuntimeError('Function "cuGraphExecBatchMemOpNodeSetParams" not found') + * err = ( __cuGraphExecBatchMemOpNodeSetParams)(hGraphExec, hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraphNode, CUDA_BATCH_MEM_OP_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecBatchMemOpNodeSetParams)(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5548, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5549 + * raise RuntimeError('Function "cuGraphExecBatchMemOpNodeSetParams" not found') + * err = ( __cuGraphExecBatchMemOpNodeSetParams)(hGraphExec, hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddMemAllocNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUDA_MEM_ALLOC_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5542 + * return err + * + * cdef CUresult _cuGraphExecBatchMemOpNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecBatchMemOpNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecBatchMemOpNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5551 + * return err + * + * cdef CUresult _cuGraphAddMemAllocNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUDA_MEM_ALLOC_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddMemAllocNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddMemAllocNode(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies, CUDA_MEM_ALLOC_NODE_PARAMS *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddMemAllocNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5553 + * cdef CUresult _cuGraphAddMemAllocNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUDA_MEM_ALLOC_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddMemAllocNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddMemAllocNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5553, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5554 + * global __cuGraphAddMemAllocNode + * cuPythonInit() + * if __cuGraphAddMemAllocNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddMemAllocNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemAllocNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5555 + * cuPythonInit() + * if __cuGraphAddMemAllocNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddMemAllocNode" not found') + * err = ( __cuGraphAddMemAllocNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5556 + * if __cuGraphAddMemAllocNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddMemAllocNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddMemAllocNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddMemAllocNode}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5556, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5556, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5555 + * cuPythonInit() + * if __cuGraphAddMemAllocNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddMemAllocNode" not found') + * err = ( __cuGraphAddMemAllocNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5554 + * global __cuGraphAddMemAllocNode + * cuPythonInit() + * if __cuGraphAddMemAllocNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddMemAllocNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5557 + * with gil: + * raise RuntimeError('Function "cuGraphAddMemAllocNode" not found') + * err = ( __cuGraphAddMemAllocNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_MEM_ALLOC_NODE_PARAMS *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemAllocNode)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5557, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5558 + * raise RuntimeError('Function "cuGraphAddMemAllocNode" not found') + * err = ( __cuGraphAddMemAllocNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphMemAllocNodeGetParams(CUgraphNode hNode, CUDA_MEM_ALLOC_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5551 + * return err + * + * cdef CUresult _cuGraphAddMemAllocNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUDA_MEM_ALLOC_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddMemAllocNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddMemAllocNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5560 + * return err + * + * cdef CUresult _cuGraphMemAllocNodeGetParams(CUgraphNode hNode, CUDA_MEM_ALLOC_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphMemAllocNodeGetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphMemAllocNodeGetParams(CUgraphNode __pyx_v_hNode, CUDA_MEM_ALLOC_NODE_PARAMS *__pyx_v_params_out) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphMemAllocNodeGetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5562 + * cdef CUresult _cuGraphMemAllocNodeGetParams(CUgraphNode hNode, CUDA_MEM_ALLOC_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphMemAllocNodeGetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphMemAllocNodeGetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5562, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5563 + * global __cuGraphMemAllocNodeGetParams + * cuPythonInit() + * if __cuGraphMemAllocNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphMemAllocNodeGetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemAllocNodeGetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5564 + * cuPythonInit() + * if __cuGraphMemAllocNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphMemAllocNodeGetParams" not found') + * err = ( __cuGraphMemAllocNodeGetParams)(hNode, params_out) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5565 + * if __cuGraphMemAllocNodeGetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphMemAllocNodeGetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphMemAllocNodeGetParams)(hNode, params_out) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphMemAllocNodeGetP}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5565, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5565, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5564 + * cuPythonInit() + * if __cuGraphMemAllocNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphMemAllocNodeGetParams" not found') + * err = ( __cuGraphMemAllocNodeGetParams)(hNode, params_out) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5563 + * global __cuGraphMemAllocNodeGetParams + * cuPythonInit() + * if __cuGraphMemAllocNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphMemAllocNodeGetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5566 + * with gil: + * raise RuntimeError('Function "cuGraphMemAllocNodeGetParams" not found') + * err = ( __cuGraphMemAllocNodeGetParams)(hNode, params_out) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUDA_MEM_ALLOC_NODE_PARAMS *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemAllocNodeGetParams)(__pyx_v_hNode, __pyx_v_params_out); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5566, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5567 + * raise RuntimeError('Function "cuGraphMemAllocNodeGetParams" not found') + * err = ( __cuGraphMemAllocNodeGetParams)(hNode, params_out) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddMemFreeNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5560 + * return err + * + * cdef CUresult _cuGraphMemAllocNodeGetParams(CUgraphNode hNode, CUDA_MEM_ALLOC_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphMemAllocNodeGetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphMemAllocNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5569 + * return err + * + * cdef CUresult _cuGraphAddMemFreeNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddMemFreeNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddMemFreeNode(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies, CUdeviceptr __pyx_v_dptr) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddMemFreeNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5571 + * cdef CUresult _cuGraphAddMemFreeNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddMemFreeNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddMemFreeNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5571, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5572 + * global __cuGraphAddMemFreeNode + * cuPythonInit() + * if __cuGraphAddMemFreeNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddMemFreeNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemFreeNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5573 + * cuPythonInit() + * if __cuGraphAddMemFreeNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddMemFreeNode" not found') + * err = ( __cuGraphAddMemFreeNode)(phGraphNode, hGraph, dependencies, numDependencies, dptr) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5574 + * if __cuGraphAddMemFreeNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddMemFreeNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddMemFreeNode)(phGraphNode, hGraph, dependencies, numDependencies, dptr) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddMemFreeNode_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5574, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5574, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5573 + * cuPythonInit() + * if __cuGraphAddMemFreeNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddMemFreeNode" not found') + * err = ( __cuGraphAddMemFreeNode)(phGraphNode, hGraph, dependencies, numDependencies, dptr) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5572 + * global __cuGraphAddMemFreeNode + * cuPythonInit() + * if __cuGraphAddMemFreeNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddMemFreeNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5575 + * with gil: + * raise RuntimeError('Function "cuGraphAddMemFreeNode" not found') + * err = ( __cuGraphAddMemFreeNode)(phGraphNode, hGraph, dependencies, numDependencies, dptr) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUdeviceptr))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemFreeNode)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_dptr); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5575, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5576 + * raise RuntimeError('Function "cuGraphAddMemFreeNode" not found') + * err = ( __cuGraphAddMemFreeNode)(phGraphNode, hGraph, dependencies, numDependencies, dptr) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphMemFreeNodeGetParams(CUgraphNode hNode, CUdeviceptr* dptr_out) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5569 + * return err + * + * cdef CUresult _cuGraphAddMemFreeNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddMemFreeNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddMemFreeNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5578 + * return err + * + * cdef CUresult _cuGraphMemFreeNodeGetParams(CUgraphNode hNode, CUdeviceptr* dptr_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphMemFreeNodeGetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphMemFreeNodeGetParams(CUgraphNode __pyx_v_hNode, CUdeviceptr *__pyx_v_dptr_out) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphMemFreeNodeGetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5580 + * cdef CUresult _cuGraphMemFreeNodeGetParams(CUgraphNode hNode, CUdeviceptr* dptr_out) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphMemFreeNodeGetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphMemFreeNodeGetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5580, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5581 + * global __cuGraphMemFreeNodeGetParams + * cuPythonInit() + * if __cuGraphMemFreeNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphMemFreeNodeGetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemFreeNodeGetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5582 + * cuPythonInit() + * if __cuGraphMemFreeNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphMemFreeNodeGetParams" not found') + * err = ( __cuGraphMemFreeNodeGetParams)(hNode, dptr_out) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5583 + * if __cuGraphMemFreeNodeGetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphMemFreeNodeGetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphMemFreeNodeGetParams)(hNode, dptr_out) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphMemFreeNodeGetPa}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5583, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5583, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5582 + * cuPythonInit() + * if __cuGraphMemFreeNodeGetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphMemFreeNodeGetParams" not found') + * err = ( __cuGraphMemFreeNodeGetParams)(hNode, dptr_out) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5581 + * global __cuGraphMemFreeNodeGetParams + * cuPythonInit() + * if __cuGraphMemFreeNodeGetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphMemFreeNodeGetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5584 + * with gil: + * raise RuntimeError('Function "cuGraphMemFreeNodeGetParams" not found') + * err = ( __cuGraphMemFreeNodeGetParams)(hNode, dptr_out) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUdeviceptr *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemFreeNodeGetParams)(__pyx_v_hNode, __pyx_v_dptr_out); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5584, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5585 + * raise RuntimeError('Function "cuGraphMemFreeNodeGetParams" not found') + * err = ( __cuGraphMemFreeNodeGetParams)(hNode, dptr_out) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGraphMemTrim(CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5578 + * return err + * + * cdef CUresult _cuGraphMemFreeNodeGetParams(CUgraphNode hNode, CUdeviceptr* dptr_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphMemFreeNodeGetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphMemFreeNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5587 + * return err + * + * cdef CUresult _cuDeviceGraphMemTrim(CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGraphMemTrim + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGraphMemTrim(CUdevice __pyx_v_device) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGraphMemTrim", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5589 + * cdef CUresult _cuDeviceGraphMemTrim(CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGraphMemTrim + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGraphMemTrim == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5589, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5590 + * global __cuDeviceGraphMemTrim + * cuPythonInit() + * if __cuDeviceGraphMemTrim == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGraphMemTrim" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGraphMemTrim == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5591 + * cuPythonInit() + * if __cuDeviceGraphMemTrim == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGraphMemTrim" not found') + * err = ( __cuDeviceGraphMemTrim)(device) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5592 + * if __cuDeviceGraphMemTrim == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGraphMemTrim" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGraphMemTrim)(device) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGraphMemTrim_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5592, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5592, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5591 + * cuPythonInit() + * if __cuDeviceGraphMemTrim == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGraphMemTrim" not found') + * err = ( __cuDeviceGraphMemTrim)(device) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5590 + * global __cuDeviceGraphMemTrim + * cuPythonInit() + * if __cuDeviceGraphMemTrim == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGraphMemTrim" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5593 + * with gil: + * raise RuntimeError('Function "cuDeviceGraphMemTrim" not found') + * err = ( __cuDeviceGraphMemTrim)(device) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGraphMemTrim)(__pyx_v_device); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5593, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5594 + * raise RuntimeError('Function "cuDeviceGraphMemTrim" not found') + * err = ( __cuDeviceGraphMemTrim)(device) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetGraphMemAttribute(CUdevice device, CUgraphMem_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5587 + * return err + * + * cdef CUresult _cuDeviceGraphMemTrim(CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGraphMemTrim + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGraphMemTrim", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5596 + * return err + * + * cdef CUresult _cuDeviceGetGraphMemAttribute(CUdevice device, CUgraphMem_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetGraphMemAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetGraphMemAttribute(CUdevice __pyx_v_device, CUgraphMem_attribute __pyx_v_attr, void *__pyx_v_value) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetGraphMemAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5598 + * cdef CUresult _cuDeviceGetGraphMemAttribute(CUdevice device, CUgraphMem_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetGraphMemAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetGraphMemAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5598, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5599 + * global __cuDeviceGetGraphMemAttribute + * cuPythonInit() + * if __cuDeviceGetGraphMemAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetGraphMemAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetGraphMemAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5600 + * cuPythonInit() + * if __cuDeviceGetGraphMemAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetGraphMemAttribute" not found') + * err = ( __cuDeviceGetGraphMemAttribute)(device, attr, value) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5601 + * if __cuDeviceGetGraphMemAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetGraphMemAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetGraphMemAttribute)(device, attr, value) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetGraphMemAttr}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5601, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5601, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5600 + * cuPythonInit() + * if __cuDeviceGetGraphMemAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetGraphMemAttribute" not found') + * err = ( __cuDeviceGetGraphMemAttribute)(device, attr, value) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5599 + * global __cuDeviceGetGraphMemAttribute + * cuPythonInit() + * if __cuDeviceGetGraphMemAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetGraphMemAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5602 + * with gil: + * raise RuntimeError('Function "cuDeviceGetGraphMemAttribute" not found') + * err = ( __cuDeviceGetGraphMemAttribute)(device, attr, value) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice, CUgraphMem_attribute, void *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetGraphMemAttribute)(__pyx_v_device, __pyx_v_attr, __pyx_v_value); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5602, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5603 + * raise RuntimeError('Function "cuDeviceGetGraphMemAttribute" not found') + * err = ( __cuDeviceGetGraphMemAttribute)(device, attr, value) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceSetGraphMemAttribute(CUdevice device, CUgraphMem_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5596 + * return err + * + * cdef CUresult _cuDeviceGetGraphMemAttribute(CUdevice device, CUgraphMem_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetGraphMemAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetGraphMemAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5605 + * return err + * + * cdef CUresult _cuDeviceSetGraphMemAttribute(CUdevice device, CUgraphMem_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceSetGraphMemAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceSetGraphMemAttribute(CUdevice __pyx_v_device, CUgraphMem_attribute __pyx_v_attr, void *__pyx_v_value) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceSetGraphMemAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5607 + * cdef CUresult _cuDeviceSetGraphMemAttribute(CUdevice device, CUgraphMem_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceSetGraphMemAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceSetGraphMemAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5607, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5608 + * global __cuDeviceSetGraphMemAttribute + * cuPythonInit() + * if __cuDeviceSetGraphMemAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceSetGraphMemAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetGraphMemAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5609 + * cuPythonInit() + * if __cuDeviceSetGraphMemAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceSetGraphMemAttribute" not found') + * err = ( __cuDeviceSetGraphMemAttribute)(device, attr, value) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5610 + * if __cuDeviceSetGraphMemAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceSetGraphMemAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceSetGraphMemAttribute)(device, attr, value) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceSetGraphMemAttr}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5610, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5610, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5609 + * cuPythonInit() + * if __cuDeviceSetGraphMemAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceSetGraphMemAttribute" not found') + * err = ( __cuDeviceSetGraphMemAttribute)(device, attr, value) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5608 + * global __cuDeviceSetGraphMemAttribute + * cuPythonInit() + * if __cuDeviceSetGraphMemAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceSetGraphMemAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5611 + * with gil: + * raise RuntimeError('Function "cuDeviceSetGraphMemAttribute" not found') + * err = ( __cuDeviceSetGraphMemAttribute)(device, attr, value) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice, CUgraphMem_attribute, void *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetGraphMemAttribute)(__pyx_v_device, __pyx_v_attr, __pyx_v_value); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5611, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5612 + * raise RuntimeError('Function "cuDeviceSetGraphMemAttribute" not found') + * err = ( __cuDeviceSetGraphMemAttribute)(device, attr, value) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphClone(CUgraph* phGraphClone, CUgraph originalGraph) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5605 + * return err + * + * cdef CUresult _cuDeviceSetGraphMemAttribute(CUdevice device, CUgraphMem_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceSetGraphMemAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceSetGraphMemAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5614 + * return err + * + * cdef CUresult _cuGraphClone(CUgraph* phGraphClone, CUgraph originalGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphClone + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphClone(CUgraph *__pyx_v_phGraphClone, CUgraph __pyx_v_originalGraph) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphClone", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5616 + * cdef CUresult _cuGraphClone(CUgraph* phGraphClone, CUgraph originalGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphClone + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphClone == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5616, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5617 + * global __cuGraphClone + * cuPythonInit() + * if __cuGraphClone == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphClone" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphClone == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5618 + * cuPythonInit() + * if __cuGraphClone == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphClone" not found') + * err = ( __cuGraphClone)(phGraphClone, originalGraph) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5619 + * if __cuGraphClone == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphClone" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphClone)(phGraphClone, originalGraph) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphClone_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5619, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5619, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5618 + * cuPythonInit() + * if __cuGraphClone == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphClone" not found') + * err = ( __cuGraphClone)(phGraphClone, originalGraph) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5617 + * global __cuGraphClone + * cuPythonInit() + * if __cuGraphClone == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphClone" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5620 + * with gil: + * raise RuntimeError('Function "cuGraphClone" not found') + * err = ( __cuGraphClone)(phGraphClone, originalGraph) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph *, CUgraph))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphClone)(__pyx_v_phGraphClone, __pyx_v_originalGraph); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5620, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5621 + * raise RuntimeError('Function "cuGraphClone" not found') + * err = ( __cuGraphClone)(phGraphClone, originalGraph) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphNodeFindInClone(CUgraphNode* phNode, CUgraphNode hOriginalNode, CUgraph hClonedGraph) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5614 + * return err + * + * cdef CUresult _cuGraphClone(CUgraph* phGraphClone, CUgraph originalGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphClone + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphClone", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5623 + * return err + * + * cdef CUresult _cuGraphNodeFindInClone(CUgraphNode* phNode, CUgraphNode hOriginalNode, CUgraph hClonedGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeFindInClone + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeFindInClone(CUgraphNode *__pyx_v_phNode, CUgraphNode __pyx_v_hOriginalNode, CUgraph __pyx_v_hClonedGraph) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphNodeFindInClone", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5625 + * cdef CUresult _cuGraphNodeFindInClone(CUgraphNode* phNode, CUgraphNode hOriginalNode, CUgraph hClonedGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphNodeFindInClone + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphNodeFindInClone == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5625, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5626 + * global __cuGraphNodeFindInClone + * cuPythonInit() + * if __cuGraphNodeFindInClone == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeFindInClone" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeFindInClone == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5627 + * cuPythonInit() + * if __cuGraphNodeFindInClone == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeFindInClone" not found') + * err = ( __cuGraphNodeFindInClone)(phNode, hOriginalNode, hClonedGraph) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5628 + * if __cuGraphNodeFindInClone == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphNodeFindInClone" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphNodeFindInClone)(phNode, hOriginalNode, hClonedGraph) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphNodeFindInClone}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5628, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5628, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5627 + * cuPythonInit() + * if __cuGraphNodeFindInClone == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeFindInClone" not found') + * err = ( __cuGraphNodeFindInClone)(phNode, hOriginalNode, hClonedGraph) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5626 + * global __cuGraphNodeFindInClone + * cuPythonInit() + * if __cuGraphNodeFindInClone == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeFindInClone" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5629 + * with gil: + * raise RuntimeError('Function "cuGraphNodeFindInClone" not found') + * err = ( __cuGraphNodeFindInClone)(phNode, hOriginalNode, hClonedGraph) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraphNode, CUgraph))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeFindInClone)(__pyx_v_phNode, __pyx_v_hOriginalNode, __pyx_v_hClonedGraph); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5629, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5630 + * raise RuntimeError('Function "cuGraphNodeFindInClone" not found') + * err = ( __cuGraphNodeFindInClone)(phNode, hOriginalNode, hClonedGraph) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphNodeGetType(CUgraphNode hNode, CUgraphNodeType* typename) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5623 + * return err + * + * cdef CUresult _cuGraphNodeFindInClone(CUgraphNode* phNode, CUgraphNode hOriginalNode, CUgraph hClonedGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeFindInClone + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphNodeFindInClone", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5632 + * return err + * + * cdef CUresult _cuGraphNodeGetType(CUgraphNode hNode, CUgraphNodeType* typename) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeGetType + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeGetType(CUgraphNode __pyx_v_hNode, CUgraphNodeType *__pyx_v_typename) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphNodeGetType", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5634 + * cdef CUresult _cuGraphNodeGetType(CUgraphNode hNode, CUgraphNodeType* typename) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphNodeGetType + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphNodeGetType == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5634, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5635 + * global __cuGraphNodeGetType + * cuPythonInit() + * if __cuGraphNodeGetType == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetType" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetType == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5636 + * cuPythonInit() + * if __cuGraphNodeGetType == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeGetType" not found') + * err = ( __cuGraphNodeGetType)(hNode, typename) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5637 + * if __cuGraphNodeGetType == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetType" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphNodeGetType)(hNode, typename) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphNodeGetType_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5637, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5637, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5636 + * cuPythonInit() + * if __cuGraphNodeGetType == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeGetType" not found') + * err = ( __cuGraphNodeGetType)(hNode, typename) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5635 + * global __cuGraphNodeGetType + * cuPythonInit() + * if __cuGraphNodeGetType == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetType" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5638 + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetType" not found') + * err = ( __cuGraphNodeGetType)(hNode, typename) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUgraphNodeType *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetType)(__pyx_v_hNode, __pyx_v_typename); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5638, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5639 + * raise RuntimeError('Function "cuGraphNodeGetType" not found') + * err = ( __cuGraphNodeGetType)(hNode, typename) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphGetNodes(CUgraph hGraph, CUgraphNode* nodes, size_t* numNodes) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5632 + * return err + * + * cdef CUresult _cuGraphNodeGetType(CUgraphNode hNode, CUgraphNodeType* typename) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeGetType + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphNodeGetType", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5641 + * return err + * + * cdef CUresult _cuGraphGetNodes(CUgraph hGraph, CUgraphNode* nodes, size_t* numNodes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphGetNodes + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphGetNodes(CUgraph __pyx_v_hGraph, CUgraphNode *__pyx_v_nodes, size_t *__pyx_v_numNodes) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphGetNodes", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5643 + * cdef CUresult _cuGraphGetNodes(CUgraph hGraph, CUgraphNode* nodes, size_t* numNodes) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphGetNodes + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphGetNodes == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5643, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5644 + * global __cuGraphGetNodes + * cuPythonInit() + * if __cuGraphGetNodes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphGetNodes" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetNodes == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5645 + * cuPythonInit() + * if __cuGraphGetNodes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphGetNodes" not found') + * err = ( __cuGraphGetNodes)(hGraph, nodes, numNodes) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5646 + * if __cuGraphGetNodes == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphGetNodes" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphGetNodes)(hGraph, nodes, numNodes) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphGetNodes_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5646, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5646, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5645 + * cuPythonInit() + * if __cuGraphGetNodes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphGetNodes" not found') + * err = ( __cuGraphGetNodes)(hGraph, nodes, numNodes) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5644 + * global __cuGraphGetNodes + * cuPythonInit() + * if __cuGraphGetNodes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphGetNodes" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5647 + * with gil: + * raise RuntimeError('Function "cuGraphGetNodes" not found') + * err = ( __cuGraphGetNodes)(hGraph, nodes, numNodes) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph, CUgraphNode *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetNodes)(__pyx_v_hGraph, __pyx_v_nodes, __pyx_v_numNodes); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5647, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5648 + * raise RuntimeError('Function "cuGraphGetNodes" not found') + * err = ( __cuGraphGetNodes)(hGraph, nodes, numNodes) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphGetRootNodes(CUgraph hGraph, CUgraphNode* rootNodes, size_t* numRootNodes) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5641 + * return err + * + * cdef CUresult _cuGraphGetNodes(CUgraph hGraph, CUgraphNode* nodes, size_t* numNodes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphGetNodes + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphGetNodes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5650 + * return err + * + * cdef CUresult _cuGraphGetRootNodes(CUgraph hGraph, CUgraphNode* rootNodes, size_t* numRootNodes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphGetRootNodes + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphGetRootNodes(CUgraph __pyx_v_hGraph, CUgraphNode *__pyx_v_rootNodes, size_t *__pyx_v_numRootNodes) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphGetRootNodes", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5652 + * cdef CUresult _cuGraphGetRootNodes(CUgraph hGraph, CUgraphNode* rootNodes, size_t* numRootNodes) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphGetRootNodes + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphGetRootNodes == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5652, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5653 + * global __cuGraphGetRootNodes + * cuPythonInit() + * if __cuGraphGetRootNodes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphGetRootNodes" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetRootNodes == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5654 + * cuPythonInit() + * if __cuGraphGetRootNodes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphGetRootNodes" not found') + * err = ( __cuGraphGetRootNodes)(hGraph, rootNodes, numRootNodes) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5655 + * if __cuGraphGetRootNodes == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphGetRootNodes" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphGetRootNodes)(hGraph, rootNodes, numRootNodes) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphGetRootNodes_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5655, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5655, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5654 + * cuPythonInit() + * if __cuGraphGetRootNodes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphGetRootNodes" not found') + * err = ( __cuGraphGetRootNodes)(hGraph, rootNodes, numRootNodes) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5653 + * global __cuGraphGetRootNodes + * cuPythonInit() + * if __cuGraphGetRootNodes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphGetRootNodes" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5656 + * with gil: + * raise RuntimeError('Function "cuGraphGetRootNodes" not found') + * err = ( __cuGraphGetRootNodes)(hGraph, rootNodes, numRootNodes) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph, CUgraphNode *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetRootNodes)(__pyx_v_hGraph, __pyx_v_rootNodes, __pyx_v_numRootNodes); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5656, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5657 + * raise RuntimeError('Function "cuGraphGetRootNodes" not found') + * err = ( __cuGraphGetRootNodes)(hGraph, rootNodes, numRootNodes) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphGetEdges(CUgraph hGraph, CUgraphNode* from_, CUgraphNode* to, size_t* numEdges) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5650 + * return err + * + * cdef CUresult _cuGraphGetRootNodes(CUgraph hGraph, CUgraphNode* rootNodes, size_t* numRootNodes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphGetRootNodes + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphGetRootNodes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5659 + * return err + * + * cdef CUresult _cuGraphGetEdges(CUgraph hGraph, CUgraphNode* from_, CUgraphNode* to, size_t* numEdges) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphGetEdges + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphGetEdges(CUgraph __pyx_v_hGraph, CUgraphNode *__pyx_v_from_, CUgraphNode *__pyx_v_to, size_t *__pyx_v_numEdges) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphGetEdges", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5661 + * cdef CUresult _cuGraphGetEdges(CUgraph hGraph, CUgraphNode* from_, CUgraphNode* to, size_t* numEdges) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphGetEdges + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphGetEdges == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5661, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5662 + * global __cuGraphGetEdges + * cuPythonInit() + * if __cuGraphGetEdges == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphGetEdges" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5663 + * cuPythonInit() + * if __cuGraphGetEdges == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphGetEdges" not found') + * err = ( __cuGraphGetEdges)(hGraph, from_, to, numEdges) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5664 + * if __cuGraphGetEdges == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphGetEdges" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphGetEdges)(hGraph, from_, to, numEdges) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphGetEdges_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5664, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5664, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5663 + * cuPythonInit() + * if __cuGraphGetEdges == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphGetEdges" not found') + * err = ( __cuGraphGetEdges)(hGraph, from_, to, numEdges) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5662 + * global __cuGraphGetEdges + * cuPythonInit() + * if __cuGraphGetEdges == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphGetEdges" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5665 + * with gil: + * raise RuntimeError('Function "cuGraphGetEdges" not found') + * err = ( __cuGraphGetEdges)(hGraph, from_, to, numEdges) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph, CUgraphNode *, CUgraphNode *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges)(__pyx_v_hGraph, __pyx_v_from_, __pyx_v_to, __pyx_v_numEdges); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5665, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5666 + * raise RuntimeError('Function "cuGraphGetEdges" not found') + * err = ( __cuGraphGetEdges)(hGraph, from_, to, numEdges) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphGetEdges_v2(CUgraph hGraph, CUgraphNode* from_, CUgraphNode* to, CUgraphEdgeData* edgeData, size_t* numEdges) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5659 + * return err + * + * cdef CUresult _cuGraphGetEdges(CUgraph hGraph, CUgraphNode* from_, CUgraphNode* to, size_t* numEdges) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphGetEdges + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphGetEdges", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5668 + * return err + * + * cdef CUresult _cuGraphGetEdges_v2(CUgraph hGraph, CUgraphNode* from_, CUgraphNode* to, CUgraphEdgeData* edgeData, size_t* numEdges) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphGetEdges_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphGetEdges_v2(CUgraph __pyx_v_hGraph, CUgraphNode *__pyx_v_from_, CUgraphNode *__pyx_v_to, CUgraphEdgeData *__pyx_v_edgeData, size_t *__pyx_v_numEdges) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphGetEdges_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5670 + * cdef CUresult _cuGraphGetEdges_v2(CUgraph hGraph, CUgraphNode* from_, CUgraphNode* to, CUgraphEdgeData* edgeData, size_t* numEdges) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphGetEdges_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphGetEdges_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5670, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5671 + * global __cuGraphGetEdges_v2 + * cuPythonInit() + * if __cuGraphGetEdges_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphGetEdges_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5672 + * cuPythonInit() + * if __cuGraphGetEdges_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphGetEdges_v2" not found') + * err = ( __cuGraphGetEdges_v2)(hGraph, from_, to, edgeData, numEdges) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5673 + * if __cuGraphGetEdges_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphGetEdges_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphGetEdges_v2)(hGraph, from_, to, edgeData, numEdges) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphGetEdges_v2_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5673, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5673, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5672 + * cuPythonInit() + * if __cuGraphGetEdges_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphGetEdges_v2" not found') + * err = ( __cuGraphGetEdges_v2)(hGraph, from_, to, edgeData, numEdges) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5671 + * global __cuGraphGetEdges_v2 + * cuPythonInit() + * if __cuGraphGetEdges_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphGetEdges_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5674 + * with gil: + * raise RuntimeError('Function "cuGraphGetEdges_v2" not found') + * err = ( __cuGraphGetEdges_v2)(hGraph, from_, to, edgeData, numEdges) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph, CUgraphNode *, CUgraphNode *, CUgraphEdgeData *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges_v2)(__pyx_v_hGraph, __pyx_v_from_, __pyx_v_to, __pyx_v_edgeData, __pyx_v_numEdges); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5674, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5675 + * raise RuntimeError('Function "cuGraphGetEdges_v2" not found') + * err = ( __cuGraphGetEdges_v2)(hGraph, from_, to, edgeData, numEdges) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphNodeGetDependencies(CUgraphNode hNode, CUgraphNode* dependencies, size_t* numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5668 + * return err + * + * cdef CUresult _cuGraphGetEdges_v2(CUgraph hGraph, CUgraphNode* from_, CUgraphNode* to, CUgraphEdgeData* edgeData, size_t* numEdges) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphGetEdges_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphGetEdges_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5677 + * return err + * + * cdef CUresult _cuGraphNodeGetDependencies(CUgraphNode hNode, CUgraphNode* dependencies, size_t* numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependencies + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeGetDependencies(CUgraphNode __pyx_v_hNode, CUgraphNode *__pyx_v_dependencies, size_t *__pyx_v_numDependencies) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphNodeGetDependencies", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5679 + * cdef CUresult _cuGraphNodeGetDependencies(CUgraphNode hNode, CUgraphNode* dependencies, size_t* numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphNodeGetDependencies + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphNodeGetDependencies == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5679, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5680 + * global __cuGraphNodeGetDependencies + * cuPythonInit() + * if __cuGraphNodeGetDependencies == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependencies" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5681 + * cuPythonInit() + * if __cuGraphNodeGetDependencies == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeGetDependencies" not found') + * err = ( __cuGraphNodeGetDependencies)(hNode, dependencies, numDependencies) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5682 + * if __cuGraphNodeGetDependencies == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependencies" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphNodeGetDependencies)(hNode, dependencies, numDependencies) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphNodeGetDependenc}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5682, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5682, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5681 + * cuPythonInit() + * if __cuGraphNodeGetDependencies == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeGetDependencies" not found') + * err = ( __cuGraphNodeGetDependencies)(hNode, dependencies, numDependencies) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5680 + * global __cuGraphNodeGetDependencies + * cuPythonInit() + * if __cuGraphNodeGetDependencies == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependencies" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5683 + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependencies" not found') + * err = ( __cuGraphNodeGetDependencies)(hNode, dependencies, numDependencies) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUgraphNode *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies)(__pyx_v_hNode, __pyx_v_dependencies, __pyx_v_numDependencies); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5683, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5684 + * raise RuntimeError('Function "cuGraphNodeGetDependencies" not found') + * err = ( __cuGraphNodeGetDependencies)(hNode, dependencies, numDependencies) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphNodeGetDependencies_v2(CUgraphNode hNode, CUgraphNode* dependencies, CUgraphEdgeData* edgeData, size_t* numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5677 + * return err + * + * cdef CUresult _cuGraphNodeGetDependencies(CUgraphNode hNode, CUgraphNode* dependencies, size_t* numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependencies + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphNodeGetDependencies", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5686 + * return err + * + * cdef CUresult _cuGraphNodeGetDependencies_v2(CUgraphNode hNode, CUgraphNode* dependencies, CUgraphEdgeData* edgeData, size_t* numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependencies_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeGetDependencies_v2(CUgraphNode __pyx_v_hNode, CUgraphNode *__pyx_v_dependencies, CUgraphEdgeData *__pyx_v_edgeData, size_t *__pyx_v_numDependencies) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphNodeGetDependencies_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5688 + * cdef CUresult _cuGraphNodeGetDependencies_v2(CUgraphNode hNode, CUgraphNode* dependencies, CUgraphEdgeData* edgeData, size_t* numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphNodeGetDependencies_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphNodeGetDependencies_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5688, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5689 + * global __cuGraphNodeGetDependencies_v2 + * cuPythonInit() + * if __cuGraphNodeGetDependencies_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependencies_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5690 + * cuPythonInit() + * if __cuGraphNodeGetDependencies_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeGetDependencies_v2" not found') + * err = ( __cuGraphNodeGetDependencies_v2)(hNode, dependencies, edgeData, numDependencies) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5691 + * if __cuGraphNodeGetDependencies_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependencies_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphNodeGetDependencies_v2)(hNode, dependencies, edgeData, numDependencies) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphNodeGetDependenc_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5691, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5691, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5690 + * cuPythonInit() + * if __cuGraphNodeGetDependencies_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeGetDependencies_v2" not found') + * err = ( __cuGraphNodeGetDependencies_v2)(hNode, dependencies, edgeData, numDependencies) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5689 + * global __cuGraphNodeGetDependencies_v2 + * cuPythonInit() + * if __cuGraphNodeGetDependencies_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependencies_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5692 + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependencies_v2" not found') + * err = ( __cuGraphNodeGetDependencies_v2)(hNode, dependencies, edgeData, numDependencies) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUgraphNode *, CUgraphEdgeData *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies_v2)(__pyx_v_hNode, __pyx_v_dependencies, __pyx_v_edgeData, __pyx_v_numDependencies); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5692, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5693 + * raise RuntimeError('Function "cuGraphNodeGetDependencies_v2" not found') + * err = ( __cuGraphNodeGetDependencies_v2)(hNode, dependencies, edgeData, numDependencies) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphNodeGetDependentNodes(CUgraphNode hNode, CUgraphNode* dependentNodes, size_t* numDependentNodes) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5686 + * return err + * + * cdef CUresult _cuGraphNodeGetDependencies_v2(CUgraphNode hNode, CUgraphNode* dependencies, CUgraphEdgeData* edgeData, size_t* numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependencies_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphNodeGetDependencies_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5695 + * return err + * + * cdef CUresult _cuGraphNodeGetDependentNodes(CUgraphNode hNode, CUgraphNode* dependentNodes, size_t* numDependentNodes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependentNodes + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeGetDependentNodes(CUgraphNode __pyx_v_hNode, CUgraphNode *__pyx_v_dependentNodes, size_t *__pyx_v_numDependentNodes) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphNodeGetDependentNodes", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5697 + * cdef CUresult _cuGraphNodeGetDependentNodes(CUgraphNode hNode, CUgraphNode* dependentNodes, size_t* numDependentNodes) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphNodeGetDependentNodes + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphNodeGetDependentNodes == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5697, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5698 + * global __cuGraphNodeGetDependentNodes + * cuPythonInit() + * if __cuGraphNodeGetDependentNodes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5699 + * cuPythonInit() + * if __cuGraphNodeGetDependentNodes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes" not found') + * err = ( __cuGraphNodeGetDependentNodes)(hNode, dependentNodes, numDependentNodes) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5700 + * if __cuGraphNodeGetDependentNodes == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphNodeGetDependentNodes)(hNode, dependentNodes, numDependentNodes) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphNodeGetDependent}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5700, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5700, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5699 + * cuPythonInit() + * if __cuGraphNodeGetDependentNodes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes" not found') + * err = ( __cuGraphNodeGetDependentNodes)(hNode, dependentNodes, numDependentNodes) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5698 + * global __cuGraphNodeGetDependentNodes + * cuPythonInit() + * if __cuGraphNodeGetDependentNodes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5701 + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes" not found') + * err = ( __cuGraphNodeGetDependentNodes)(hNode, dependentNodes, numDependentNodes) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUgraphNode *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes)(__pyx_v_hNode, __pyx_v_dependentNodes, __pyx_v_numDependentNodes); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5701, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5702 + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes" not found') + * err = ( __cuGraphNodeGetDependentNodes)(hNode, dependentNodes, numDependentNodes) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphNodeGetDependentNodes_v2(CUgraphNode hNode, CUgraphNode* dependentNodes, CUgraphEdgeData* edgeData, size_t* numDependentNodes) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5695 + * return err + * + * cdef CUresult _cuGraphNodeGetDependentNodes(CUgraphNode hNode, CUgraphNode* dependentNodes, size_t* numDependentNodes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependentNodes + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphNodeGetDependentNodes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5704 + * return err + * + * cdef CUresult _cuGraphNodeGetDependentNodes_v2(CUgraphNode hNode, CUgraphNode* dependentNodes, CUgraphEdgeData* edgeData, size_t* numDependentNodes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependentNodes_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeGetDependentNodes_v2(CUgraphNode __pyx_v_hNode, CUgraphNode *__pyx_v_dependentNodes, CUgraphEdgeData *__pyx_v_edgeData, size_t *__pyx_v_numDependentNodes) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphNodeGetDependentNodes_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5706 + * cdef CUresult _cuGraphNodeGetDependentNodes_v2(CUgraphNode hNode, CUgraphNode* dependentNodes, CUgraphEdgeData* edgeData, size_t* numDependentNodes) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphNodeGetDependentNodes_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphNodeGetDependentNodes_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5706, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5707 + * global __cuGraphNodeGetDependentNodes_v2 + * cuPythonInit() + * if __cuGraphNodeGetDependentNodes_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5708 + * cuPythonInit() + * if __cuGraphNodeGetDependentNodes_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes_v2" not found') + * err = ( __cuGraphNodeGetDependentNodes_v2)(hNode, dependentNodes, edgeData, numDependentNodes) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5709 + * if __cuGraphNodeGetDependentNodes_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphNodeGetDependentNodes_v2)(hNode, dependentNodes, edgeData, numDependentNodes) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphNodeGetDependent_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5709, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5709, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5708 + * cuPythonInit() + * if __cuGraphNodeGetDependentNodes_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes_v2" not found') + * err = ( __cuGraphNodeGetDependentNodes_v2)(hNode, dependentNodes, edgeData, numDependentNodes) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5707 + * global __cuGraphNodeGetDependentNodes_v2 + * cuPythonInit() + * if __cuGraphNodeGetDependentNodes_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5710 + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes_v2" not found') + * err = ( __cuGraphNodeGetDependentNodes_v2)(hNode, dependentNodes, edgeData, numDependentNodes) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUgraphNode *, CUgraphEdgeData *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes_v2)(__pyx_v_hNode, __pyx_v_dependentNodes, __pyx_v_edgeData, __pyx_v_numDependentNodes); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5710, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5711 + * raise RuntimeError('Function "cuGraphNodeGetDependentNodes_v2" not found') + * err = ( __cuGraphNodeGetDependentNodes_v2)(hNode, dependentNodes, edgeData, numDependentNodes) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddDependencies(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5704 + * return err + * + * cdef CUresult _cuGraphNodeGetDependentNodes_v2(CUgraphNode hNode, CUgraphNode* dependentNodes, CUgraphEdgeData* edgeData, size_t* numDependentNodes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependentNodes_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphNodeGetDependentNodes_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5713 + * return err + * + * cdef CUresult _cuGraphAddDependencies(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddDependencies + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddDependencies(CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_from_, CUgraphNode const *__pyx_v_to, size_t __pyx_v_numDependencies) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddDependencies", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5715 + * cdef CUresult _cuGraphAddDependencies(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddDependencies + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddDependencies == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5715, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5716 + * global __cuGraphAddDependencies + * cuPythonInit() + * if __cuGraphAddDependencies == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddDependencies" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5717 + * cuPythonInit() + * if __cuGraphAddDependencies == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddDependencies" not found') + * err = ( __cuGraphAddDependencies)(hGraph, from_, to, numDependencies) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5718 + * if __cuGraphAddDependencies == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddDependencies" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddDependencies)(hGraph, from_, to, numDependencies) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddDependencies}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5718, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5718, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5717 + * cuPythonInit() + * if __cuGraphAddDependencies == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddDependencies" not found') + * err = ( __cuGraphAddDependencies)(hGraph, from_, to, numDependencies) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5716 + * global __cuGraphAddDependencies + * cuPythonInit() + * if __cuGraphAddDependencies == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddDependencies" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5719 + * with gil: + * raise RuntimeError('Function "cuGraphAddDependencies" not found') + * err = ( __cuGraphAddDependencies)(hGraph, from_, to, numDependencies) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph, CUgraphNode const *, CUgraphNode const *, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies)(__pyx_v_hGraph, __pyx_v_from_, __pyx_v_to, __pyx_v_numDependencies); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5719, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5720 + * raise RuntimeError('Function "cuGraphAddDependencies" not found') + * err = ( __cuGraphAddDependencies)(hGraph, from_, to, numDependencies) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddDependencies_v2(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, const CUgraphEdgeData* edgeData, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5713 + * return err + * + * cdef CUresult _cuGraphAddDependencies(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddDependencies + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddDependencies", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5722 + * return err + * + * cdef CUresult _cuGraphAddDependencies_v2(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, const CUgraphEdgeData* edgeData, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddDependencies_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddDependencies_v2(CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_from_, CUgraphNode const *__pyx_v_to, CUgraphEdgeData const *__pyx_v_edgeData, size_t __pyx_v_numDependencies) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddDependencies_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5724 + * cdef CUresult _cuGraphAddDependencies_v2(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, const CUgraphEdgeData* edgeData, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddDependencies_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddDependencies_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5724, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5725 + * global __cuGraphAddDependencies_v2 + * cuPythonInit() + * if __cuGraphAddDependencies_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddDependencies_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5726 + * cuPythonInit() + * if __cuGraphAddDependencies_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddDependencies_v2" not found') + * err = ( __cuGraphAddDependencies_v2)(hGraph, from_, to, edgeData, numDependencies) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5727 + * if __cuGraphAddDependencies_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddDependencies_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddDependencies_v2)(hGraph, from_, to, edgeData, numDependencies) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddDependencies_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5727, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5727, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5726 + * cuPythonInit() + * if __cuGraphAddDependencies_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddDependencies_v2" not found') + * err = ( __cuGraphAddDependencies_v2)(hGraph, from_, to, edgeData, numDependencies) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5725 + * global __cuGraphAddDependencies_v2 + * cuPythonInit() + * if __cuGraphAddDependencies_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddDependencies_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5728 + * with gil: + * raise RuntimeError('Function "cuGraphAddDependencies_v2" not found') + * err = ( __cuGraphAddDependencies_v2)(hGraph, from_, to, edgeData, numDependencies) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph, CUgraphNode const *, CUgraphNode const *, CUgraphEdgeData const *, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies_v2)(__pyx_v_hGraph, __pyx_v_from_, __pyx_v_to, __pyx_v_edgeData, __pyx_v_numDependencies); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5728, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5729 + * raise RuntimeError('Function "cuGraphAddDependencies_v2" not found') + * err = ( __cuGraphAddDependencies_v2)(hGraph, from_, to, edgeData, numDependencies) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphRemoveDependencies(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5722 + * return err + * + * cdef CUresult _cuGraphAddDependencies_v2(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, const CUgraphEdgeData* edgeData, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddDependencies_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddDependencies_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5731 + * return err + * + * cdef CUresult _cuGraphRemoveDependencies(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphRemoveDependencies + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphRemoveDependencies(CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_from_, CUgraphNode const *__pyx_v_to, size_t __pyx_v_numDependencies) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphRemoveDependencies", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5733 + * cdef CUresult _cuGraphRemoveDependencies(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphRemoveDependencies + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphRemoveDependencies == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5733, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5734 + * global __cuGraphRemoveDependencies + * cuPythonInit() + * if __cuGraphRemoveDependencies == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphRemoveDependencies" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5735 + * cuPythonInit() + * if __cuGraphRemoveDependencies == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphRemoveDependencies" not found') + * err = ( __cuGraphRemoveDependencies)(hGraph, from_, to, numDependencies) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5736 + * if __cuGraphRemoveDependencies == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphRemoveDependencies" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphRemoveDependencies)(hGraph, from_, to, numDependencies) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphRemoveDependenci}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5736, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5736, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5735 + * cuPythonInit() + * if __cuGraphRemoveDependencies == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphRemoveDependencies" not found') + * err = ( __cuGraphRemoveDependencies)(hGraph, from_, to, numDependencies) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5734 + * global __cuGraphRemoveDependencies + * cuPythonInit() + * if __cuGraphRemoveDependencies == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphRemoveDependencies" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5737 + * with gil: + * raise RuntimeError('Function "cuGraphRemoveDependencies" not found') + * err = ( __cuGraphRemoveDependencies)(hGraph, from_, to, numDependencies) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph, CUgraphNode const *, CUgraphNode const *, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies)(__pyx_v_hGraph, __pyx_v_from_, __pyx_v_to, __pyx_v_numDependencies); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5737, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5738 + * raise RuntimeError('Function "cuGraphRemoveDependencies" not found') + * err = ( __cuGraphRemoveDependencies)(hGraph, from_, to, numDependencies) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphRemoveDependencies_v2(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, const CUgraphEdgeData* edgeData, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5731 + * return err + * + * cdef CUresult _cuGraphRemoveDependencies(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphRemoveDependencies + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphRemoveDependencies", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5740 + * return err + * + * cdef CUresult _cuGraphRemoveDependencies_v2(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, const CUgraphEdgeData* edgeData, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphRemoveDependencies_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphRemoveDependencies_v2(CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_from_, CUgraphNode const *__pyx_v_to, CUgraphEdgeData const *__pyx_v_edgeData, size_t __pyx_v_numDependencies) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphRemoveDependencies_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5742 + * cdef CUresult _cuGraphRemoveDependencies_v2(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, const CUgraphEdgeData* edgeData, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphRemoveDependencies_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphRemoveDependencies_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5742, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5743 + * global __cuGraphRemoveDependencies_v2 + * cuPythonInit() + * if __cuGraphRemoveDependencies_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphRemoveDependencies_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5744 + * cuPythonInit() + * if __cuGraphRemoveDependencies_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphRemoveDependencies_v2" not found') + * err = ( __cuGraphRemoveDependencies_v2)(hGraph, from_, to, edgeData, numDependencies) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5745 + * if __cuGraphRemoveDependencies_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphRemoveDependencies_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphRemoveDependencies_v2)(hGraph, from_, to, edgeData, numDependencies) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphRemoveDependenci_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5745, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5745, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5744 + * cuPythonInit() + * if __cuGraphRemoveDependencies_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphRemoveDependencies_v2" not found') + * err = ( __cuGraphRemoveDependencies_v2)(hGraph, from_, to, edgeData, numDependencies) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5743 + * global __cuGraphRemoveDependencies_v2 + * cuPythonInit() + * if __cuGraphRemoveDependencies_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphRemoveDependencies_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5746 + * with gil: + * raise RuntimeError('Function "cuGraphRemoveDependencies_v2" not found') + * err = ( __cuGraphRemoveDependencies_v2)(hGraph, from_, to, edgeData, numDependencies) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph, CUgraphNode const *, CUgraphNode const *, CUgraphEdgeData const *, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies_v2)(__pyx_v_hGraph, __pyx_v_from_, __pyx_v_to, __pyx_v_edgeData, __pyx_v_numDependencies); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5746, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5747 + * raise RuntimeError('Function "cuGraphRemoveDependencies_v2" not found') + * err = ( __cuGraphRemoveDependencies_v2)(hGraph, from_, to, edgeData, numDependencies) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphDestroyNode(CUgraphNode hNode) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5740 + * return err + * + * cdef CUresult _cuGraphRemoveDependencies_v2(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, const CUgraphEdgeData* edgeData, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphRemoveDependencies_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphRemoveDependencies_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5749 + * return err + * + * cdef CUresult _cuGraphDestroyNode(CUgraphNode hNode) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphDestroyNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphDestroyNode(CUgraphNode __pyx_v_hNode) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphDestroyNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5751 + * cdef CUresult _cuGraphDestroyNode(CUgraphNode hNode) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphDestroyNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphDestroyNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5751, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5752 + * global __cuGraphDestroyNode + * cuPythonInit() + * if __cuGraphDestroyNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphDestroyNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroyNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5753 + * cuPythonInit() + * if __cuGraphDestroyNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphDestroyNode" not found') + * err = ( __cuGraphDestroyNode)(hNode) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5754 + * if __cuGraphDestroyNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphDestroyNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphDestroyNode)(hNode) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphDestroyNode_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5754, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5754, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5753 + * cuPythonInit() + * if __cuGraphDestroyNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphDestroyNode" not found') + * err = ( __cuGraphDestroyNode)(hNode) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5752 + * global __cuGraphDestroyNode + * cuPythonInit() + * if __cuGraphDestroyNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphDestroyNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5755 + * with gil: + * raise RuntimeError('Function "cuGraphDestroyNode" not found') + * err = ( __cuGraphDestroyNode)(hNode) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroyNode)(__pyx_v_hNode); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5755, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5756 + * raise RuntimeError('Function "cuGraphDestroyNode" not found') + * err = ( __cuGraphDestroyNode)(hNode) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphInstantiateWithFlags(CUgraphExec* phGraphExec, CUgraph hGraph, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5749 + * return err + * + * cdef CUresult _cuGraphDestroyNode(CUgraphNode hNode) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphDestroyNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphDestroyNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5758 + * return err + * + * cdef CUresult _cuGraphInstantiateWithFlags(CUgraphExec* phGraphExec, CUgraph hGraph, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphInstantiateWithFlags + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphInstantiateWithFlags(CUgraphExec *__pyx_v_phGraphExec, CUgraph __pyx_v_hGraph, unsigned PY_LONG_LONG __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphInstantiateWithFlags", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5760 + * cdef CUresult _cuGraphInstantiateWithFlags(CUgraphExec* phGraphExec, CUgraph hGraph, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphInstantiateWithFlags + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphInstantiateWithFlags == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5760, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5761 + * global __cuGraphInstantiateWithFlags + * cuPythonInit() + * if __cuGraphInstantiateWithFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphInstantiateWithFlags" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithFlags == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5762 + * cuPythonInit() + * if __cuGraphInstantiateWithFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphInstantiateWithFlags" not found') + * err = ( __cuGraphInstantiateWithFlags)(phGraphExec, hGraph, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5763 + * if __cuGraphInstantiateWithFlags == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphInstantiateWithFlags" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphInstantiateWithFlags)(phGraphExec, hGraph, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphInstantiateWithF}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5763, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5763, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5762 + * cuPythonInit() + * if __cuGraphInstantiateWithFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphInstantiateWithFlags" not found') + * err = ( __cuGraphInstantiateWithFlags)(phGraphExec, hGraph, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5761 + * global __cuGraphInstantiateWithFlags + * cuPythonInit() + * if __cuGraphInstantiateWithFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphInstantiateWithFlags" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5764 + * with gil: + * raise RuntimeError('Function "cuGraphInstantiateWithFlags" not found') + * err = ( __cuGraphInstantiateWithFlags)(phGraphExec, hGraph, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec *, CUgraph, unsigned PY_LONG_LONG))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithFlags)(__pyx_v_phGraphExec, __pyx_v_hGraph, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5764, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5765 + * raise RuntimeError('Function "cuGraphInstantiateWithFlags" not found') + * err = ( __cuGraphInstantiateWithFlags)(phGraphExec, hGraph, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphInstantiateWithParams(CUgraphExec* phGraphExec, CUgraph hGraph, CUDA_GRAPH_INSTANTIATE_PARAMS* instantiateParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5758 + * return err + * + * cdef CUresult _cuGraphInstantiateWithFlags(CUgraphExec* phGraphExec, CUgraph hGraph, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphInstantiateWithFlags + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphInstantiateWithFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5767 + * return err + * + * cdef CUresult _cuGraphInstantiateWithParams(CUgraphExec* phGraphExec, CUgraph hGraph, CUDA_GRAPH_INSTANTIATE_PARAMS* instantiateParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphInstantiateWithParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphInstantiateWithParams(CUgraphExec *__pyx_v_phGraphExec, CUgraph __pyx_v_hGraph, CUDA_GRAPH_INSTANTIATE_PARAMS *__pyx_v_instantiateParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphInstantiateWithParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5769 + * cdef CUresult _cuGraphInstantiateWithParams(CUgraphExec* phGraphExec, CUgraph hGraph, CUDA_GRAPH_INSTANTIATE_PARAMS* instantiateParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphInstantiateWithParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphInstantiateWithParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5769, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5770 + * global __cuGraphInstantiateWithParams + * cuPythonInit() + * if __cuGraphInstantiateWithParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphInstantiateWithParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5771 + * cuPythonInit() + * if __cuGraphInstantiateWithParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphInstantiateWithParams" not found') + * err = ( __cuGraphInstantiateWithParams)(phGraphExec, hGraph, instantiateParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5772 + * if __cuGraphInstantiateWithParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphInstantiateWithParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphInstantiateWithParams)(phGraphExec, hGraph, instantiateParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphInstantiateWithP}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5772, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5772, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5771 + * cuPythonInit() + * if __cuGraphInstantiateWithParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphInstantiateWithParams" not found') + * err = ( __cuGraphInstantiateWithParams)(phGraphExec, hGraph, instantiateParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5770 + * global __cuGraphInstantiateWithParams + * cuPythonInit() + * if __cuGraphInstantiateWithParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphInstantiateWithParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5773 + * with gil: + * raise RuntimeError('Function "cuGraphInstantiateWithParams" not found') + * err = ( __cuGraphInstantiateWithParams)(phGraphExec, hGraph, instantiateParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec *, CUgraph, CUDA_GRAPH_INSTANTIATE_PARAMS *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithParams)(__pyx_v_phGraphExec, __pyx_v_hGraph, __pyx_v_instantiateParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5773, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5774 + * raise RuntimeError('Function "cuGraphInstantiateWithParams" not found') + * err = ( __cuGraphInstantiateWithParams)(phGraphExec, hGraph, instantiateParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecGetFlags(CUgraphExec hGraphExec, cuuint64_t* flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5767 + * return err + * + * cdef CUresult _cuGraphInstantiateWithParams(CUgraphExec* phGraphExec, CUgraph hGraph, CUDA_GRAPH_INSTANTIATE_PARAMS* instantiateParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphInstantiateWithParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphInstantiateWithParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5776 + * return err + * + * cdef CUresult _cuGraphExecGetFlags(CUgraphExec hGraphExec, cuuint64_t* flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecGetFlags + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecGetFlags(CUgraphExec __pyx_v_hGraphExec, cuuint64_t *__pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecGetFlags", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5778 + * cdef CUresult _cuGraphExecGetFlags(CUgraphExec hGraphExec, cuuint64_t* flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecGetFlags + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecGetFlags == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5778, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5779 + * global __cuGraphExecGetFlags + * cuPythonInit() + * if __cuGraphExecGetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecGetFlags" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecGetFlags == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5780 + * cuPythonInit() + * if __cuGraphExecGetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecGetFlags" not found') + * err = ( __cuGraphExecGetFlags)(hGraphExec, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5781 + * if __cuGraphExecGetFlags == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecGetFlags" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecGetFlags)(hGraphExec, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecGetFlags_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5781, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5781, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5780 + * cuPythonInit() + * if __cuGraphExecGetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecGetFlags" not found') + * err = ( __cuGraphExecGetFlags)(hGraphExec, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5779 + * global __cuGraphExecGetFlags + * cuPythonInit() + * if __cuGraphExecGetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecGetFlags" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5782 + * with gil: + * raise RuntimeError('Function "cuGraphExecGetFlags" not found') + * err = ( __cuGraphExecGetFlags)(hGraphExec, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, cuuint64_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecGetFlags)(__pyx_v_hGraphExec, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5782, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5783 + * raise RuntimeError('Function "cuGraphExecGetFlags" not found') + * err = ( __cuGraphExecGetFlags)(hGraphExec, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecKernelNodeSetParams_v2(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5776 + * return err + * + * cdef CUresult _cuGraphExecGetFlags(CUgraphExec hGraphExec, cuuint64_t* flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecGetFlags + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecGetFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5785 + * return err + * + * cdef CUresult _cuGraphExecKernelNodeSetParams_v2(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecKernelNodeSetParams_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecKernelNodeSetParams_v2(CUgraphExec __pyx_v_hGraphExec, CUgraphNode __pyx_v_hNode, CUDA_KERNEL_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecKernelNodeSetParams_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5787 + * cdef CUresult _cuGraphExecKernelNodeSetParams_v2(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecKernelNodeSetParams_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecKernelNodeSetParams_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5787, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5788 + * global __cuGraphExecKernelNodeSetParams_v2 + * cuPythonInit() + * if __cuGraphExecKernelNodeSetParams_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecKernelNodeSetParams_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecKernelNodeSetParams_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5789 + * cuPythonInit() + * if __cuGraphExecKernelNodeSetParams_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecKernelNodeSetParams_v2" not found') + * err = ( __cuGraphExecKernelNodeSetParams_v2)(hGraphExec, hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5790 + * if __cuGraphExecKernelNodeSetParams_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecKernelNodeSetParams_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecKernelNodeSetParams_v2)(hGraphExec, hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecKernelNodeSe}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5790, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5790, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5789 + * cuPythonInit() + * if __cuGraphExecKernelNodeSetParams_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecKernelNodeSetParams_v2" not found') + * err = ( __cuGraphExecKernelNodeSetParams_v2)(hGraphExec, hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5788 + * global __cuGraphExecKernelNodeSetParams_v2 + * cuPythonInit() + * if __cuGraphExecKernelNodeSetParams_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecKernelNodeSetParams_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5791 + * with gil: + * raise RuntimeError('Function "cuGraphExecKernelNodeSetParams_v2" not found') + * err = ( __cuGraphExecKernelNodeSetParams_v2)(hGraphExec, hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraphNode, CUDA_KERNEL_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecKernelNodeSetParams_v2)(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5791, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5792 + * raise RuntimeError('Function "cuGraphExecKernelNodeSetParams_v2" not found') + * err = ( __cuGraphExecKernelNodeSetParams_v2)(hGraphExec, hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecMemcpyNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_MEMCPY3D* copyParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5785 + * return err + * + * cdef CUresult _cuGraphExecKernelNodeSetParams_v2(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecKernelNodeSetParams_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecKernelNodeSetParams_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5794 + * return err + * + * cdef CUresult _cuGraphExecMemcpyNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_MEMCPY3D* copyParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecMemcpyNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecMemcpyNodeSetParams(CUgraphExec __pyx_v_hGraphExec, CUgraphNode __pyx_v_hNode, CUDA_MEMCPY3D const *__pyx_v_copyParams, CUcontext __pyx_v_ctx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecMemcpyNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5796 + * cdef CUresult _cuGraphExecMemcpyNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_MEMCPY3D* copyParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecMemcpyNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecMemcpyNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5796, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5797 + * global __cuGraphExecMemcpyNodeSetParams + * cuPythonInit() + * if __cuGraphExecMemcpyNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecMemcpyNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemcpyNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5798 + * cuPythonInit() + * if __cuGraphExecMemcpyNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecMemcpyNodeSetParams" not found') + * err = ( __cuGraphExecMemcpyNodeSetParams)(hGraphExec, hNode, copyParams, ctx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5799 + * if __cuGraphExecMemcpyNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecMemcpyNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecMemcpyNodeSetParams)(hGraphExec, hNode, copyParams, ctx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecMemcpyNodeSe}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5799, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5799, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5798 + * cuPythonInit() + * if __cuGraphExecMemcpyNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecMemcpyNodeSetParams" not found') + * err = ( __cuGraphExecMemcpyNodeSetParams)(hGraphExec, hNode, copyParams, ctx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5797 + * global __cuGraphExecMemcpyNodeSetParams + * cuPythonInit() + * if __cuGraphExecMemcpyNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecMemcpyNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5800 + * with gil: + * raise RuntimeError('Function "cuGraphExecMemcpyNodeSetParams" not found') + * err = ( __cuGraphExecMemcpyNodeSetParams)(hGraphExec, hNode, copyParams, ctx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraphNode, CUDA_MEMCPY3D const *, CUcontext))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemcpyNodeSetParams)(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_copyParams, __pyx_v_ctx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5800, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5801 + * raise RuntimeError('Function "cuGraphExecMemcpyNodeSetParams" not found') + * err = ( __cuGraphExecMemcpyNodeSetParams)(hGraphExec, hNode, copyParams, ctx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecMemsetNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_MEMSET_NODE_PARAMS* memsetParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5794 + * return err + * + * cdef CUresult _cuGraphExecMemcpyNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_MEMCPY3D* copyParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecMemcpyNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecMemcpyNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5803 + * return err + * + * cdef CUresult _cuGraphExecMemsetNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_MEMSET_NODE_PARAMS* memsetParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecMemsetNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecMemsetNodeSetParams(CUgraphExec __pyx_v_hGraphExec, CUgraphNode __pyx_v_hNode, CUDA_MEMSET_NODE_PARAMS const *__pyx_v_memsetParams, CUcontext __pyx_v_ctx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecMemsetNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5805 + * cdef CUresult _cuGraphExecMemsetNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_MEMSET_NODE_PARAMS* memsetParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecMemsetNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecMemsetNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5805, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5806 + * global __cuGraphExecMemsetNodeSetParams + * cuPythonInit() + * if __cuGraphExecMemsetNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecMemsetNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemsetNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5807 + * cuPythonInit() + * if __cuGraphExecMemsetNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecMemsetNodeSetParams" not found') + * err = ( __cuGraphExecMemsetNodeSetParams)(hGraphExec, hNode, memsetParams, ctx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5808 + * if __cuGraphExecMemsetNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecMemsetNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecMemsetNodeSetParams)(hGraphExec, hNode, memsetParams, ctx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecMemsetNodeSe}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5808, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5808, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5807 + * cuPythonInit() + * if __cuGraphExecMemsetNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecMemsetNodeSetParams" not found') + * err = ( __cuGraphExecMemsetNodeSetParams)(hGraphExec, hNode, memsetParams, ctx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5806 + * global __cuGraphExecMemsetNodeSetParams + * cuPythonInit() + * if __cuGraphExecMemsetNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecMemsetNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5809 + * with gil: + * raise RuntimeError('Function "cuGraphExecMemsetNodeSetParams" not found') + * err = ( __cuGraphExecMemsetNodeSetParams)(hGraphExec, hNode, memsetParams, ctx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraphNode, CUDA_MEMSET_NODE_PARAMS const *, CUcontext))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemsetNodeSetParams)(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_memsetParams, __pyx_v_ctx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5809, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5810 + * raise RuntimeError('Function "cuGraphExecMemsetNodeSetParams" not found') + * err = ( __cuGraphExecMemsetNodeSetParams)(hGraphExec, hNode, memsetParams, ctx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecHostNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5803 + * return err + * + * cdef CUresult _cuGraphExecMemsetNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_MEMSET_NODE_PARAMS* memsetParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecMemsetNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecMemsetNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5812 + * return err + * + * cdef CUresult _cuGraphExecHostNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecHostNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecHostNodeSetParams(CUgraphExec __pyx_v_hGraphExec, CUgraphNode __pyx_v_hNode, CUDA_HOST_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecHostNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5814 + * cdef CUresult _cuGraphExecHostNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecHostNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecHostNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5814, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5815 + * global __cuGraphExecHostNodeSetParams + * cuPythonInit() + * if __cuGraphExecHostNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecHostNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecHostNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5816 + * cuPythonInit() + * if __cuGraphExecHostNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecHostNodeSetParams" not found') + * err = ( __cuGraphExecHostNodeSetParams)(hGraphExec, hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5817 + * if __cuGraphExecHostNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecHostNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecHostNodeSetParams)(hGraphExec, hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecHostNodeSetP}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5817, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5817, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5816 + * cuPythonInit() + * if __cuGraphExecHostNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecHostNodeSetParams" not found') + * err = ( __cuGraphExecHostNodeSetParams)(hGraphExec, hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5815 + * global __cuGraphExecHostNodeSetParams + * cuPythonInit() + * if __cuGraphExecHostNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecHostNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5818 + * with gil: + * raise RuntimeError('Function "cuGraphExecHostNodeSetParams" not found') + * err = ( __cuGraphExecHostNodeSetParams)(hGraphExec, hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraphNode, CUDA_HOST_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecHostNodeSetParams)(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5818, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5819 + * raise RuntimeError('Function "cuGraphExecHostNodeSetParams" not found') + * err = ( __cuGraphExecHostNodeSetParams)(hGraphExec, hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecChildGraphNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraph childGraph) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5812 + * return err + * + * cdef CUresult _cuGraphExecHostNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecHostNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecHostNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5821 + * return err + * + * cdef CUresult _cuGraphExecChildGraphNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraph childGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecChildGraphNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecChildGraphNodeSetParams(CUgraphExec __pyx_v_hGraphExec, CUgraphNode __pyx_v_hNode, CUgraph __pyx_v_childGraph) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecChildGraphNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5823 + * cdef CUresult _cuGraphExecChildGraphNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraph childGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecChildGraphNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecChildGraphNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5823, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5824 + * global __cuGraphExecChildGraphNodeSetParams + * cuPythonInit() + * if __cuGraphExecChildGraphNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecChildGraphNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecChildGraphNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5825 + * cuPythonInit() + * if __cuGraphExecChildGraphNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecChildGraphNodeSetParams" not found') + * err = ( __cuGraphExecChildGraphNodeSetParams)(hGraphExec, hNode, childGraph) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5826 + * if __cuGraphExecChildGraphNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecChildGraphNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecChildGraphNodeSetParams)(hGraphExec, hNode, childGraph) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecChildGraphNo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5826, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5826, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5825 + * cuPythonInit() + * if __cuGraphExecChildGraphNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecChildGraphNodeSetParams" not found') + * err = ( __cuGraphExecChildGraphNodeSetParams)(hGraphExec, hNode, childGraph) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5824 + * global __cuGraphExecChildGraphNodeSetParams + * cuPythonInit() + * if __cuGraphExecChildGraphNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecChildGraphNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5827 + * with gil: + * raise RuntimeError('Function "cuGraphExecChildGraphNodeSetParams" not found') + * err = ( __cuGraphExecChildGraphNodeSetParams)(hGraphExec, hNode, childGraph) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraphNode, CUgraph))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecChildGraphNodeSetParams)(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_childGraph); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5827, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5828 + * raise RuntimeError('Function "cuGraphExecChildGraphNodeSetParams" not found') + * err = ( __cuGraphExecChildGraphNodeSetParams)(hGraphExec, hNode, childGraph) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecEventRecordNodeSetEvent(CUgraphExec hGraphExec, CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5821 + * return err + * + * cdef CUresult _cuGraphExecChildGraphNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraph childGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecChildGraphNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecChildGraphNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5830 + * return err + * + * cdef CUresult _cuGraphExecEventRecordNodeSetEvent(CUgraphExec hGraphExec, CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecEventRecordNodeSetEvent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecEventRecordNodeSetEvent(CUgraphExec __pyx_v_hGraphExec, CUgraphNode __pyx_v_hNode, CUevent __pyx_v_event) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecEventRecordNodeSetEvent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5832 + * cdef CUresult _cuGraphExecEventRecordNodeSetEvent(CUgraphExec hGraphExec, CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecEventRecordNodeSetEvent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecEventRecordNodeSetEvent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5832, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5833 + * global __cuGraphExecEventRecordNodeSetEvent + * cuPythonInit() + * if __cuGraphExecEventRecordNodeSetEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecEventRecordNodeSetEvent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventRecordNodeSetEvent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5834 + * cuPythonInit() + * if __cuGraphExecEventRecordNodeSetEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecEventRecordNodeSetEvent" not found') + * err = ( __cuGraphExecEventRecordNodeSetEvent)(hGraphExec, hNode, event) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5835 + * if __cuGraphExecEventRecordNodeSetEvent == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecEventRecordNodeSetEvent" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecEventRecordNodeSetEvent)(hGraphExec, hNode, event) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecEventRecordN}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5835, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5835, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5834 + * cuPythonInit() + * if __cuGraphExecEventRecordNodeSetEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecEventRecordNodeSetEvent" not found') + * err = ( __cuGraphExecEventRecordNodeSetEvent)(hGraphExec, hNode, event) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5833 + * global __cuGraphExecEventRecordNodeSetEvent + * cuPythonInit() + * if __cuGraphExecEventRecordNodeSetEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecEventRecordNodeSetEvent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5836 + * with gil: + * raise RuntimeError('Function "cuGraphExecEventRecordNodeSetEvent" not found') + * err = ( __cuGraphExecEventRecordNodeSetEvent)(hGraphExec, hNode, event) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraphNode, CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventRecordNodeSetEvent)(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_event); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5836, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5837 + * raise RuntimeError('Function "cuGraphExecEventRecordNodeSetEvent" not found') + * err = ( __cuGraphExecEventRecordNodeSetEvent)(hGraphExec, hNode, event) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecEventWaitNodeSetEvent(CUgraphExec hGraphExec, CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5830 + * return err + * + * cdef CUresult _cuGraphExecEventRecordNodeSetEvent(CUgraphExec hGraphExec, CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecEventRecordNodeSetEvent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecEventRecordNodeSetEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5839 + * return err + * + * cdef CUresult _cuGraphExecEventWaitNodeSetEvent(CUgraphExec hGraphExec, CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecEventWaitNodeSetEvent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecEventWaitNodeSetEvent(CUgraphExec __pyx_v_hGraphExec, CUgraphNode __pyx_v_hNode, CUevent __pyx_v_event) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecEventWaitNodeSetEvent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5841 + * cdef CUresult _cuGraphExecEventWaitNodeSetEvent(CUgraphExec hGraphExec, CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecEventWaitNodeSetEvent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecEventWaitNodeSetEvent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5841, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5842 + * global __cuGraphExecEventWaitNodeSetEvent + * cuPythonInit() + * if __cuGraphExecEventWaitNodeSetEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecEventWaitNodeSetEvent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventWaitNodeSetEvent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5843 + * cuPythonInit() + * if __cuGraphExecEventWaitNodeSetEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecEventWaitNodeSetEvent" not found') + * err = ( __cuGraphExecEventWaitNodeSetEvent)(hGraphExec, hNode, event) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5844 + * if __cuGraphExecEventWaitNodeSetEvent == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecEventWaitNodeSetEvent" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecEventWaitNodeSetEvent)(hGraphExec, hNode, event) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecEventWaitNod}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5844, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5844, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5843 + * cuPythonInit() + * if __cuGraphExecEventWaitNodeSetEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecEventWaitNodeSetEvent" not found') + * err = ( __cuGraphExecEventWaitNodeSetEvent)(hGraphExec, hNode, event) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5842 + * global __cuGraphExecEventWaitNodeSetEvent + * cuPythonInit() + * if __cuGraphExecEventWaitNodeSetEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecEventWaitNodeSetEvent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5845 + * with gil: + * raise RuntimeError('Function "cuGraphExecEventWaitNodeSetEvent" not found') + * err = ( __cuGraphExecEventWaitNodeSetEvent)(hGraphExec, hNode, event) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraphNode, CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventWaitNodeSetEvent)(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_event); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5845, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5846 + * raise RuntimeError('Function "cuGraphExecEventWaitNodeSetEvent" not found') + * err = ( __cuGraphExecEventWaitNodeSetEvent)(hGraphExec, hNode, event) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecExternalSemaphoresSignalNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5839 + * return err + * + * cdef CUresult _cuGraphExecEventWaitNodeSetEvent(CUgraphExec hGraphExec, CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecEventWaitNodeSetEvent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecEventWaitNodeSetEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5848 + * return err + * + * cdef CUresult _cuGraphExecExternalSemaphoresSignalNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecExternalSemaphoresSignalNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecExternalSemaphoresSignalNodeSetParams(CUgraphExec __pyx_v_hGraphExec, CUgraphNode __pyx_v_hNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecExternalSemaphoresSignalNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5850 + * cdef CUresult _cuGraphExecExternalSemaphoresSignalNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecExternalSemaphoresSignalNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecExternalSemaphoresSignalNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5850, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5851 + * global __cuGraphExecExternalSemaphoresSignalNodeSetParams + * cuPythonInit() + * if __cuGraphExecExternalSemaphoresSignalNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresSignalNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresSignalNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5852 + * cuPythonInit() + * if __cuGraphExecExternalSemaphoresSignalNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresSignalNodeSetParams" not found') + * err = ( __cuGraphExecExternalSemaphoresSignalNodeSetParams)(hGraphExec, hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5853 + * if __cuGraphExecExternalSemaphoresSignalNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresSignalNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecExternalSemaphoresSignalNodeSetParams)(hGraphExec, hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecExternalSema}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5853, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5853, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5852 + * cuPythonInit() + * if __cuGraphExecExternalSemaphoresSignalNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresSignalNodeSetParams" not found') + * err = ( __cuGraphExecExternalSemaphoresSignalNodeSetParams)(hGraphExec, hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5851 + * global __cuGraphExecExternalSemaphoresSignalNodeSetParams + * cuPythonInit() + * if __cuGraphExecExternalSemaphoresSignalNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresSignalNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5854 + * with gil: + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresSignalNodeSetParams" not found') + * err = ( __cuGraphExecExternalSemaphoresSignalNodeSetParams)(hGraphExec, hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraphNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresSignalNodeSetParams)(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5854, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5855 + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresSignalNodeSetParams" not found') + * err = ( __cuGraphExecExternalSemaphoresSignalNodeSetParams)(hGraphExec, hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecExternalSemaphoresWaitNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5848 + * return err + * + * cdef CUresult _cuGraphExecExternalSemaphoresSignalNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecExternalSemaphoresSignalNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecExternalSemaphoresSignalNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5857 + * return err + * + * cdef CUresult _cuGraphExecExternalSemaphoresWaitNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecExternalSemaphoresWaitNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecExternalSemaphoresWaitNodeSetParams(CUgraphExec __pyx_v_hGraphExec, CUgraphNode __pyx_v_hNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS const *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecExternalSemaphoresWaitNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5859 + * cdef CUresult _cuGraphExecExternalSemaphoresWaitNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecExternalSemaphoresWaitNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecExternalSemaphoresWaitNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5859, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5860 + * global __cuGraphExecExternalSemaphoresWaitNodeSetParams + * cuPythonInit() + * if __cuGraphExecExternalSemaphoresWaitNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresWaitNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresWaitNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5861 + * cuPythonInit() + * if __cuGraphExecExternalSemaphoresWaitNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresWaitNodeSetParams" not found') + * err = ( __cuGraphExecExternalSemaphoresWaitNodeSetParams)(hGraphExec, hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5862 + * if __cuGraphExecExternalSemaphoresWaitNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresWaitNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecExternalSemaphoresWaitNodeSetParams)(hGraphExec, hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecExternalSema_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5862, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5862, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5861 + * cuPythonInit() + * if __cuGraphExecExternalSemaphoresWaitNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresWaitNodeSetParams" not found') + * err = ( __cuGraphExecExternalSemaphoresWaitNodeSetParams)(hGraphExec, hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5860 + * global __cuGraphExecExternalSemaphoresWaitNodeSetParams + * cuPythonInit() + * if __cuGraphExecExternalSemaphoresWaitNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresWaitNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5863 + * with gil: + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresWaitNodeSetParams" not found') + * err = ( __cuGraphExecExternalSemaphoresWaitNodeSetParams)(hGraphExec, hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraphNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresWaitNodeSetParams)(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5863, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5864 + * raise RuntimeError('Function "cuGraphExecExternalSemaphoresWaitNodeSetParams" not found') + * err = ( __cuGraphExecExternalSemaphoresWaitNodeSetParams)(hGraphExec, hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphNodeSetEnabled(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int isEnabled) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5857 + * return err + * + * cdef CUresult _cuGraphExecExternalSemaphoresWaitNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecExternalSemaphoresWaitNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecExternalSemaphoresWaitNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5866 + * return err + * + * cdef CUresult _cuGraphNodeSetEnabled(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int isEnabled) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeSetEnabled + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeSetEnabled(CUgraphExec __pyx_v_hGraphExec, CUgraphNode __pyx_v_hNode, unsigned int __pyx_v_isEnabled) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphNodeSetEnabled", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5868 + * cdef CUresult _cuGraphNodeSetEnabled(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int isEnabled) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphNodeSetEnabled + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphNodeSetEnabled == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5868, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5869 + * global __cuGraphNodeSetEnabled + * cuPythonInit() + * if __cuGraphNodeSetEnabled == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeSetEnabled" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetEnabled == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5870 + * cuPythonInit() + * if __cuGraphNodeSetEnabled == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeSetEnabled" not found') + * err = ( __cuGraphNodeSetEnabled)(hGraphExec, hNode, isEnabled) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5871 + * if __cuGraphNodeSetEnabled == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphNodeSetEnabled" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphNodeSetEnabled)(hGraphExec, hNode, isEnabled) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphNodeSetEnabled_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5871, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5871, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5870 + * cuPythonInit() + * if __cuGraphNodeSetEnabled == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeSetEnabled" not found') + * err = ( __cuGraphNodeSetEnabled)(hGraphExec, hNode, isEnabled) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5869 + * global __cuGraphNodeSetEnabled + * cuPythonInit() + * if __cuGraphNodeSetEnabled == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeSetEnabled" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5872 + * with gil: + * raise RuntimeError('Function "cuGraphNodeSetEnabled" not found') + * err = ( __cuGraphNodeSetEnabled)(hGraphExec, hNode, isEnabled) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraphNode, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetEnabled)(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_isEnabled); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5872, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5873 + * raise RuntimeError('Function "cuGraphNodeSetEnabled" not found') + * err = ( __cuGraphNodeSetEnabled)(hGraphExec, hNode, isEnabled) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphNodeGetEnabled(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int* isEnabled) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5866 + * return err + * + * cdef CUresult _cuGraphNodeSetEnabled(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int isEnabled) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeSetEnabled + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphNodeSetEnabled", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5875 + * return err + * + * cdef CUresult _cuGraphNodeGetEnabled(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int* isEnabled) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeGetEnabled + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeGetEnabled(CUgraphExec __pyx_v_hGraphExec, CUgraphNode __pyx_v_hNode, unsigned int *__pyx_v_isEnabled) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphNodeGetEnabled", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5877 + * cdef CUresult _cuGraphNodeGetEnabled(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int* isEnabled) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphNodeGetEnabled + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphNodeGetEnabled == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5877, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5878 + * global __cuGraphNodeGetEnabled + * cuPythonInit() + * if __cuGraphNodeGetEnabled == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetEnabled" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetEnabled == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5879 + * cuPythonInit() + * if __cuGraphNodeGetEnabled == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeGetEnabled" not found') + * err = ( __cuGraphNodeGetEnabled)(hGraphExec, hNode, isEnabled) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5880 + * if __cuGraphNodeGetEnabled == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetEnabled" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphNodeGetEnabled)(hGraphExec, hNode, isEnabled) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphNodeGetEnabled_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5880, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5880, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5879 + * cuPythonInit() + * if __cuGraphNodeGetEnabled == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeGetEnabled" not found') + * err = ( __cuGraphNodeGetEnabled)(hGraphExec, hNode, isEnabled) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5878 + * global __cuGraphNodeGetEnabled + * cuPythonInit() + * if __cuGraphNodeGetEnabled == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetEnabled" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5881 + * with gil: + * raise RuntimeError('Function "cuGraphNodeGetEnabled" not found') + * err = ( __cuGraphNodeGetEnabled)(hGraphExec, hNode, isEnabled) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraphNode, unsigned int *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetEnabled)(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_isEnabled); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5881, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5882 + * raise RuntimeError('Function "cuGraphNodeGetEnabled" not found') + * err = ( __cuGraphNodeGetEnabled)(hGraphExec, hNode, isEnabled) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphUpload(CUgraphExec hGraphExec, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5875 + * return err + * + * cdef CUresult _cuGraphNodeGetEnabled(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int* isEnabled) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeGetEnabled + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphNodeGetEnabled", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5884 + * return err + * + * cdef CUresult _cuGraphUpload(CUgraphExec hGraphExec, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphUpload + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphUpload(CUgraphExec __pyx_v_hGraphExec, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphUpload", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5886 + * cdef CUresult _cuGraphUpload(CUgraphExec hGraphExec, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphUpload + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphUpload == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5886, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5887 + * global __cuGraphUpload + * cuPythonInit() + * if __cuGraphUpload == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphUpload" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphUpload == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5888 + * cuPythonInit() + * if __cuGraphUpload == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphUpload" not found') + * err = ( __cuGraphUpload)(hGraphExec, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5889 + * if __cuGraphUpload == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphUpload" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphUpload)(hGraphExec, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphUpload_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5889, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5889, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5888 + * cuPythonInit() + * if __cuGraphUpload == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphUpload" not found') + * err = ( __cuGraphUpload)(hGraphExec, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5887 + * global __cuGraphUpload + * cuPythonInit() + * if __cuGraphUpload == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphUpload" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5890 + * with gil: + * raise RuntimeError('Function "cuGraphUpload" not found') + * err = ( __cuGraphUpload)(hGraphExec, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphUpload)(__pyx_v_hGraphExec, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5890, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5891 + * raise RuntimeError('Function "cuGraphUpload" not found') + * err = ( __cuGraphUpload)(hGraphExec, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphLaunch(CUgraphExec hGraphExec, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5884 + * return err + * + * cdef CUresult _cuGraphUpload(CUgraphExec hGraphExec, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphUpload + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphUpload", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5893 + * return err + * + * cdef CUresult _cuGraphLaunch(CUgraphExec hGraphExec, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphLaunch + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphLaunch(CUgraphExec __pyx_v_hGraphExec, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphLaunch", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5895 + * cdef CUresult _cuGraphLaunch(CUgraphExec hGraphExec, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphLaunch + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphLaunch == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5895, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5896 + * global __cuGraphLaunch + * cuPythonInit() + * if __cuGraphLaunch == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphLaunch" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphLaunch == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5897 + * cuPythonInit() + * if __cuGraphLaunch == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphLaunch" not found') + * err = ( __cuGraphLaunch)(hGraphExec, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5898 + * if __cuGraphLaunch == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphLaunch" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphLaunch)(hGraphExec, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphLaunch_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5898, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5898, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5897 + * cuPythonInit() + * if __cuGraphLaunch == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphLaunch" not found') + * err = ( __cuGraphLaunch)(hGraphExec, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5896 + * global __cuGraphLaunch + * cuPythonInit() + * if __cuGraphLaunch == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphLaunch" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5899 + * with gil: + * raise RuntimeError('Function "cuGraphLaunch" not found') + * err = ( __cuGraphLaunch)(hGraphExec, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphLaunch)(__pyx_v_hGraphExec, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5899, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5900 + * raise RuntimeError('Function "cuGraphLaunch" not found') + * err = ( __cuGraphLaunch)(hGraphExec, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecDestroy(CUgraphExec hGraphExec) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5893 + * return err + * + * cdef CUresult _cuGraphLaunch(CUgraphExec hGraphExec, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphLaunch + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphLaunch", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5902 + * return err + * + * cdef CUresult _cuGraphExecDestroy(CUgraphExec hGraphExec) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecDestroy + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecDestroy(CUgraphExec __pyx_v_hGraphExec) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecDestroy", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5904 + * cdef CUresult _cuGraphExecDestroy(CUgraphExec hGraphExec) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecDestroy + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecDestroy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5904, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5905 + * global __cuGraphExecDestroy + * cuPythonInit() + * if __cuGraphExecDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecDestroy" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecDestroy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5906 + * cuPythonInit() + * if __cuGraphExecDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecDestroy" not found') + * err = ( __cuGraphExecDestroy)(hGraphExec) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5907 + * if __cuGraphExecDestroy == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecDestroy" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecDestroy)(hGraphExec) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecDestroy_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5907, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5907, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5906 + * cuPythonInit() + * if __cuGraphExecDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecDestroy" not found') + * err = ( __cuGraphExecDestroy)(hGraphExec) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5905 + * global __cuGraphExecDestroy + * cuPythonInit() + * if __cuGraphExecDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecDestroy" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5908 + * with gil: + * raise RuntimeError('Function "cuGraphExecDestroy" not found') + * err = ( __cuGraphExecDestroy)(hGraphExec) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecDestroy)(__pyx_v_hGraphExec); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5908, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5909 + * raise RuntimeError('Function "cuGraphExecDestroy" not found') + * err = ( __cuGraphExecDestroy)(hGraphExec) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphDestroy(CUgraph hGraph) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5902 + * return err + * + * cdef CUresult _cuGraphExecDestroy(CUgraphExec hGraphExec) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecDestroy + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5911 + * return err + * + * cdef CUresult _cuGraphDestroy(CUgraph hGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphDestroy + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphDestroy(CUgraph __pyx_v_hGraph) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphDestroy", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5913 + * cdef CUresult _cuGraphDestroy(CUgraph hGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphDestroy + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphDestroy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5913, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5914 + * global __cuGraphDestroy + * cuPythonInit() + * if __cuGraphDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphDestroy" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5915 + * cuPythonInit() + * if __cuGraphDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphDestroy" not found') + * err = ( __cuGraphDestroy)(hGraph) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5916 + * if __cuGraphDestroy == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphDestroy" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphDestroy)(hGraph) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphDestroy_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5916, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5916, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5915 + * cuPythonInit() + * if __cuGraphDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphDestroy" not found') + * err = ( __cuGraphDestroy)(hGraph) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5914 + * global __cuGraphDestroy + * cuPythonInit() + * if __cuGraphDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphDestroy" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5917 + * with gil: + * raise RuntimeError('Function "cuGraphDestroy" not found') + * err = ( __cuGraphDestroy)(hGraph) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroy)(__pyx_v_hGraph); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5917, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5918 + * raise RuntimeError('Function "cuGraphDestroy" not found') + * err = ( __cuGraphDestroy)(hGraph) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecUpdate_v2(CUgraphExec hGraphExec, CUgraph hGraph, CUgraphExecUpdateResultInfo* resultInfo) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5911 + * return err + * + * cdef CUresult _cuGraphDestroy(CUgraph hGraph) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphDestroy + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5920 + * return err + * + * cdef CUresult _cuGraphExecUpdate_v2(CUgraphExec hGraphExec, CUgraph hGraph, CUgraphExecUpdateResultInfo* resultInfo) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecUpdate_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecUpdate_v2(CUgraphExec __pyx_v_hGraphExec, CUgraph __pyx_v_hGraph, CUgraphExecUpdateResultInfo *__pyx_v_resultInfo) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecUpdate_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5922 + * cdef CUresult _cuGraphExecUpdate_v2(CUgraphExec hGraphExec, CUgraph hGraph, CUgraphExecUpdateResultInfo* resultInfo) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecUpdate_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecUpdate_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5922, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5923 + * global __cuGraphExecUpdate_v2 + * cuPythonInit() + * if __cuGraphExecUpdate_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecUpdate_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecUpdate_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5924 + * cuPythonInit() + * if __cuGraphExecUpdate_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecUpdate_v2" not found') + * err = ( __cuGraphExecUpdate_v2)(hGraphExec, hGraph, resultInfo) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5925 + * if __cuGraphExecUpdate_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecUpdate_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecUpdate_v2)(hGraphExec, hGraph, resultInfo) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecUpdate_v2_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5925, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5925, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5924 + * cuPythonInit() + * if __cuGraphExecUpdate_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecUpdate_v2" not found') + * err = ( __cuGraphExecUpdate_v2)(hGraphExec, hGraph, resultInfo) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5923 + * global __cuGraphExecUpdate_v2 + * cuPythonInit() + * if __cuGraphExecUpdate_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecUpdate_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5926 + * with gil: + * raise RuntimeError('Function "cuGraphExecUpdate_v2" not found') + * err = ( __cuGraphExecUpdate_v2)(hGraphExec, hGraph, resultInfo) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraph, CUgraphExecUpdateResultInfo *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecUpdate_v2)(__pyx_v_hGraphExec, __pyx_v_hGraph, __pyx_v_resultInfo); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5926, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5927 + * raise RuntimeError('Function "cuGraphExecUpdate_v2" not found') + * err = ( __cuGraphExecUpdate_v2)(hGraphExec, hGraph, resultInfo) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphKernelNodeCopyAttributes(CUgraphNode dst, CUgraphNode src) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5920 + * return err + * + * cdef CUresult _cuGraphExecUpdate_v2(CUgraphExec hGraphExec, CUgraph hGraph, CUgraphExecUpdateResultInfo* resultInfo) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecUpdate_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecUpdate_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5929 + * return err + * + * cdef CUresult _cuGraphKernelNodeCopyAttributes(CUgraphNode dst, CUgraphNode src) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeCopyAttributes + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphKernelNodeCopyAttributes(CUgraphNode __pyx_v_dst, CUgraphNode __pyx_v_src) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphKernelNodeCopyAttributes", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5931 + * cdef CUresult _cuGraphKernelNodeCopyAttributes(CUgraphNode dst, CUgraphNode src) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphKernelNodeCopyAttributes + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphKernelNodeCopyAttributes == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5931, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5932 + * global __cuGraphKernelNodeCopyAttributes + * cuPythonInit() + * if __cuGraphKernelNodeCopyAttributes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeCopyAttributes" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeCopyAttributes == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5933 + * cuPythonInit() + * if __cuGraphKernelNodeCopyAttributes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphKernelNodeCopyAttributes" not found') + * err = ( __cuGraphKernelNodeCopyAttributes)(dst, src) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5934 + * if __cuGraphKernelNodeCopyAttributes == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeCopyAttributes" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphKernelNodeCopyAttributes)(dst, src) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphKernelNodeCopyAt}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5934, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5934, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5933 + * cuPythonInit() + * if __cuGraphKernelNodeCopyAttributes == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphKernelNodeCopyAttributes" not found') + * err = ( __cuGraphKernelNodeCopyAttributes)(dst, src) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5932 + * global __cuGraphKernelNodeCopyAttributes + * cuPythonInit() + * if __cuGraphKernelNodeCopyAttributes == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeCopyAttributes" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5935 + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeCopyAttributes" not found') + * err = ( __cuGraphKernelNodeCopyAttributes)(dst, src) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUgraphNode))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeCopyAttributes)(__pyx_v_dst, __pyx_v_src); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5935, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5936 + * raise RuntimeError('Function "cuGraphKernelNodeCopyAttributes" not found') + * err = ( __cuGraphKernelNodeCopyAttributes)(dst, src) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphKernelNodeGetAttribute(CUgraphNode hNode, CUkernelNodeAttrID attr, CUkernelNodeAttrValue* value_out) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5929 + * return err + * + * cdef CUresult _cuGraphKernelNodeCopyAttributes(CUgraphNode dst, CUgraphNode src) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeCopyAttributes + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphKernelNodeCopyAttributes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5938 + * return err + * + * cdef CUresult _cuGraphKernelNodeGetAttribute(CUgraphNode hNode, CUkernelNodeAttrID attr, CUkernelNodeAttrValue* value_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeGetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphKernelNodeGetAttribute(CUgraphNode __pyx_v_hNode, CUkernelNodeAttrID __pyx_v_attr, CUkernelNodeAttrValue *__pyx_v_value_out) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphKernelNodeGetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5940 + * cdef CUresult _cuGraphKernelNodeGetAttribute(CUgraphNode hNode, CUkernelNodeAttrID attr, CUkernelNodeAttrValue* value_out) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphKernelNodeGetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphKernelNodeGetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5940, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5941 + * global __cuGraphKernelNodeGetAttribute + * cuPythonInit() + * if __cuGraphKernelNodeGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeGetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5942 + * cuPythonInit() + * if __cuGraphKernelNodeGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphKernelNodeGetAttribute" not found') + * err = ( __cuGraphKernelNodeGetAttribute)(hNode, attr, value_out) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5943 + * if __cuGraphKernelNodeGetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeGetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphKernelNodeGetAttribute)(hNode, attr, value_out) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphKernelNodeGetAtt}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5943, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5943, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5942 + * cuPythonInit() + * if __cuGraphKernelNodeGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphKernelNodeGetAttribute" not found') + * err = ( __cuGraphKernelNodeGetAttribute)(hNode, attr, value_out) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5941 + * global __cuGraphKernelNodeGetAttribute + * cuPythonInit() + * if __cuGraphKernelNodeGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeGetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5944 + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeGetAttribute" not found') + * err = ( __cuGraphKernelNodeGetAttribute)(hNode, attr, value_out) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUkernelNodeAttrID, CUkernelNodeAttrValue *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetAttribute)(__pyx_v_hNode, __pyx_v_attr, __pyx_v_value_out); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5944, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5945 + * raise RuntimeError('Function "cuGraphKernelNodeGetAttribute" not found') + * err = ( __cuGraphKernelNodeGetAttribute)(hNode, attr, value_out) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphKernelNodeSetAttribute(CUgraphNode hNode, CUkernelNodeAttrID attr, const CUkernelNodeAttrValue* value) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5938 + * return err + * + * cdef CUresult _cuGraphKernelNodeGetAttribute(CUgraphNode hNode, CUkernelNodeAttrID attr, CUkernelNodeAttrValue* value_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeGetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphKernelNodeGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5947 + * return err + * + * cdef CUresult _cuGraphKernelNodeSetAttribute(CUgraphNode hNode, CUkernelNodeAttrID attr, const CUkernelNodeAttrValue* value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeSetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphKernelNodeSetAttribute(CUgraphNode __pyx_v_hNode, CUkernelNodeAttrID __pyx_v_attr, CUkernelNodeAttrValue const *__pyx_v_value) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphKernelNodeSetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5949 + * cdef CUresult _cuGraphKernelNodeSetAttribute(CUgraphNode hNode, CUkernelNodeAttrID attr, const CUkernelNodeAttrValue* value) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphKernelNodeSetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphKernelNodeSetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5949, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5950 + * global __cuGraphKernelNodeSetAttribute + * cuPythonInit() + * if __cuGraphKernelNodeSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeSetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5951 + * cuPythonInit() + * if __cuGraphKernelNodeSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphKernelNodeSetAttribute" not found') + * err = ( __cuGraphKernelNodeSetAttribute)(hNode, attr, value) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5952 + * if __cuGraphKernelNodeSetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeSetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphKernelNodeSetAttribute)(hNode, attr, value) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphKernelNodeSetAtt}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5952, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5952, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5951 + * cuPythonInit() + * if __cuGraphKernelNodeSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphKernelNodeSetAttribute" not found') + * err = ( __cuGraphKernelNodeSetAttribute)(hNode, attr, value) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5950 + * global __cuGraphKernelNodeSetAttribute + * cuPythonInit() + * if __cuGraphKernelNodeSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeSetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5953 + * with gil: + * raise RuntimeError('Function "cuGraphKernelNodeSetAttribute" not found') + * err = ( __cuGraphKernelNodeSetAttribute)(hNode, attr, value) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUkernelNodeAttrID, CUkernelNodeAttrValue const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetAttribute)(__pyx_v_hNode, __pyx_v_attr, __pyx_v_value); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5953, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5954 + * raise RuntimeError('Function "cuGraphKernelNodeSetAttribute" not found') + * err = ( __cuGraphKernelNodeSetAttribute)(hNode, attr, value) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphDebugDotPrint(CUgraph hGraph, const char* path, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5947 + * return err + * + * cdef CUresult _cuGraphKernelNodeSetAttribute(CUgraphNode hNode, CUkernelNodeAttrID attr, const CUkernelNodeAttrValue* value) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeSetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphKernelNodeSetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5956 + * return err + * + * cdef CUresult _cuGraphDebugDotPrint(CUgraph hGraph, const char* path, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphDebugDotPrint + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphDebugDotPrint(CUgraph __pyx_v_hGraph, char const *__pyx_v_path, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphDebugDotPrint", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5958 + * cdef CUresult _cuGraphDebugDotPrint(CUgraph hGraph, const char* path, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphDebugDotPrint + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphDebugDotPrint == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5958, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5959 + * global __cuGraphDebugDotPrint + * cuPythonInit() + * if __cuGraphDebugDotPrint == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphDebugDotPrint" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDebugDotPrint == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5960 + * cuPythonInit() + * if __cuGraphDebugDotPrint == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphDebugDotPrint" not found') + * err = ( __cuGraphDebugDotPrint)(hGraph, path, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5961 + * if __cuGraphDebugDotPrint == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphDebugDotPrint" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphDebugDotPrint)(hGraph, path, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphDebugDotPrint_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5961, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5961, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5960 + * cuPythonInit() + * if __cuGraphDebugDotPrint == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphDebugDotPrint" not found') + * err = ( __cuGraphDebugDotPrint)(hGraph, path, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5959 + * global __cuGraphDebugDotPrint + * cuPythonInit() + * if __cuGraphDebugDotPrint == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphDebugDotPrint" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5962 + * with gil: + * raise RuntimeError('Function "cuGraphDebugDotPrint" not found') + * err = ( __cuGraphDebugDotPrint)(hGraph, path, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph, char const *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDebugDotPrint)(__pyx_v_hGraph, __pyx_v_path, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5962, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5963 + * raise RuntimeError('Function "cuGraphDebugDotPrint" not found') + * err = ( __cuGraphDebugDotPrint)(hGraph, path, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuUserObjectCreate(CUuserObject* object_out, void* ptr, CUhostFn destroy, unsigned int initialRefcount, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5956 + * return err + * + * cdef CUresult _cuGraphDebugDotPrint(CUgraph hGraph, const char* path, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphDebugDotPrint + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphDebugDotPrint", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5965 + * return err + * + * cdef CUresult _cuUserObjectCreate(CUuserObject* object_out, void* ptr, CUhostFn destroy, unsigned int initialRefcount, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuUserObjectCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuUserObjectCreate(CUuserObject *__pyx_v_object_out, void *__pyx_v_ptr, CUhostFn __pyx_v_destroy, unsigned int __pyx_v_initialRefcount, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuUserObjectCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5967 + * cdef CUresult _cuUserObjectCreate(CUuserObject* object_out, void* ptr, CUhostFn destroy, unsigned int initialRefcount, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuUserObjectCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuUserObjectCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5967, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5968 + * global __cuUserObjectCreate + * cuPythonInit() + * if __cuUserObjectCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuUserObjectCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5969 + * cuPythonInit() + * if __cuUserObjectCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuUserObjectCreate" not found') + * err = ( __cuUserObjectCreate)(object_out, ptr, destroy, initialRefcount, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5970 + * if __cuUserObjectCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuUserObjectCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuUserObjectCreate)(object_out, ptr, destroy, initialRefcount, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuUserObjectCreate_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5970, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5970, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5969 + * cuPythonInit() + * if __cuUserObjectCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuUserObjectCreate" not found') + * err = ( __cuUserObjectCreate)(object_out, ptr, destroy, initialRefcount, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5968 + * global __cuUserObjectCreate + * cuPythonInit() + * if __cuUserObjectCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuUserObjectCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5971 + * with gil: + * raise RuntimeError('Function "cuUserObjectCreate" not found') + * err = ( __cuUserObjectCreate)(object_out, ptr, destroy, initialRefcount, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUuserObject *, void *, CUhostFn, unsigned int, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectCreate)(__pyx_v_object_out, __pyx_v_ptr, __pyx_v_destroy, __pyx_v_initialRefcount, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5971, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5972 + * raise RuntimeError('Function "cuUserObjectCreate" not found') + * err = ( __cuUserObjectCreate)(object_out, ptr, destroy, initialRefcount, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuUserObjectRetain(CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5965 + * return err + * + * cdef CUresult _cuUserObjectCreate(CUuserObject* object_out, void* ptr, CUhostFn destroy, unsigned int initialRefcount, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuUserObjectCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuUserObjectCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5974 + * return err + * + * cdef CUresult _cuUserObjectRetain(CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuUserObjectRetain + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuUserObjectRetain(CUuserObject __pyx_v_object, unsigned int __pyx_v_count) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuUserObjectRetain", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5976 + * cdef CUresult _cuUserObjectRetain(CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuUserObjectRetain + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuUserObjectRetain == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5976, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5977 + * global __cuUserObjectRetain + * cuPythonInit() + * if __cuUserObjectRetain == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuUserObjectRetain" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRetain == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5978 + * cuPythonInit() + * if __cuUserObjectRetain == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuUserObjectRetain" not found') + * err = ( __cuUserObjectRetain)(object, count) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5979 + * if __cuUserObjectRetain == NULL: + * with gil: + * raise RuntimeError('Function "cuUserObjectRetain" not found') # <<<<<<<<<<<<<< + * err = ( __cuUserObjectRetain)(object, count) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuUserObjectRetain_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5979, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5979, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5978 + * cuPythonInit() + * if __cuUserObjectRetain == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuUserObjectRetain" not found') + * err = ( __cuUserObjectRetain)(object, count) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5977 + * global __cuUserObjectRetain + * cuPythonInit() + * if __cuUserObjectRetain == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuUserObjectRetain" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5980 + * with gil: + * raise RuntimeError('Function "cuUserObjectRetain" not found') + * err = ( __cuUserObjectRetain)(object, count) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUuserObject, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRetain)(__pyx_v_object, __pyx_v_count); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5980, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5981 + * raise RuntimeError('Function "cuUserObjectRetain" not found') + * err = ( __cuUserObjectRetain)(object, count) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuUserObjectRelease(CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5974 + * return err + * + * cdef CUresult _cuUserObjectRetain(CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuUserObjectRetain + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuUserObjectRetain", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5983 + * return err + * + * cdef CUresult _cuUserObjectRelease(CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuUserObjectRelease + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuUserObjectRelease(CUuserObject __pyx_v_object, unsigned int __pyx_v_count) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuUserObjectRelease", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5985 + * cdef CUresult _cuUserObjectRelease(CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuUserObjectRelease + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuUserObjectRelease == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5985, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5986 + * global __cuUserObjectRelease + * cuPythonInit() + * if __cuUserObjectRelease == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuUserObjectRelease" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRelease == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5987 + * cuPythonInit() + * if __cuUserObjectRelease == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuUserObjectRelease" not found') + * err = ( __cuUserObjectRelease)(object, count) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5988 + * if __cuUserObjectRelease == NULL: + * with gil: + * raise RuntimeError('Function "cuUserObjectRelease" not found') # <<<<<<<<<<<<<< + * err = ( __cuUserObjectRelease)(object, count) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuUserObjectRelease_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5988, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5988, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5987 + * cuPythonInit() + * if __cuUserObjectRelease == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuUserObjectRelease" not found') + * err = ( __cuUserObjectRelease)(object, count) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5986 + * global __cuUserObjectRelease + * cuPythonInit() + * if __cuUserObjectRelease == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuUserObjectRelease" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5989 + * with gil: + * raise RuntimeError('Function "cuUserObjectRelease" not found') + * err = ( __cuUserObjectRelease)(object, count) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUuserObject, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRelease)(__pyx_v_object, __pyx_v_count); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5989, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5990 + * raise RuntimeError('Function "cuUserObjectRelease" not found') + * err = ( __cuUserObjectRelease)(object, count) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphRetainUserObject(CUgraph graph, CUuserObject object, unsigned int count, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5983 + * return err + * + * cdef CUresult _cuUserObjectRelease(CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuUserObjectRelease + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuUserObjectRelease", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":5992 + * return err + * + * cdef CUresult _cuGraphRetainUserObject(CUgraph graph, CUuserObject object, unsigned int count, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphRetainUserObject + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphRetainUserObject(CUgraph __pyx_v_graph, CUuserObject __pyx_v_object, unsigned int __pyx_v_count, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphRetainUserObject", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":5994 + * cdef CUresult _cuGraphRetainUserObject(CUgraph graph, CUuserObject object, unsigned int count, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphRetainUserObject + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphRetainUserObject == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 5994, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":5995 + * global __cuGraphRetainUserObject + * cuPythonInit() + * if __cuGraphRetainUserObject == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphRetainUserObject" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRetainUserObject == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":5996 + * cuPythonInit() + * if __cuGraphRetainUserObject == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphRetainUserObject" not found') + * err = ( __cuGraphRetainUserObject)(graph, object, count, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":5997 + * if __cuGraphRetainUserObject == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphRetainUserObject" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphRetainUserObject)(graph, object, count, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphRetainUserObject}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 5997, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 5997, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5996 + * cuPythonInit() + * if __cuGraphRetainUserObject == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphRetainUserObject" not found') + * err = ( __cuGraphRetainUserObject)(graph, object, count, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5995 + * global __cuGraphRetainUserObject + * cuPythonInit() + * if __cuGraphRetainUserObject == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphRetainUserObject" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":5998 + * with gil: + * raise RuntimeError('Function "cuGraphRetainUserObject" not found') + * err = ( __cuGraphRetainUserObject)(graph, object, count, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph, CUuserObject, unsigned int, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRetainUserObject)(__pyx_v_graph, __pyx_v_object, __pyx_v_count, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 5998, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":5999 + * raise RuntimeError('Function "cuGraphRetainUserObject" not found') + * err = ( __cuGraphRetainUserObject)(graph, object, count, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphReleaseUserObject(CUgraph graph, CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":5992 + * return err + * + * cdef CUresult _cuGraphRetainUserObject(CUgraph graph, CUuserObject object, unsigned int count, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphRetainUserObject + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphRetainUserObject", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6001 + * return err + * + * cdef CUresult _cuGraphReleaseUserObject(CUgraph graph, CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphReleaseUserObject + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphReleaseUserObject(CUgraph __pyx_v_graph, CUuserObject __pyx_v_object, unsigned int __pyx_v_count) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphReleaseUserObject", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6003 + * cdef CUresult _cuGraphReleaseUserObject(CUgraph graph, CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphReleaseUserObject + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphReleaseUserObject == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6003, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6004 + * global __cuGraphReleaseUserObject + * cuPythonInit() + * if __cuGraphReleaseUserObject == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphReleaseUserObject" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphReleaseUserObject == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6005 + * cuPythonInit() + * if __cuGraphReleaseUserObject == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphReleaseUserObject" not found') + * err = ( __cuGraphReleaseUserObject)(graph, object, count) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6006 + * if __cuGraphReleaseUserObject == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphReleaseUserObject" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphReleaseUserObject)(graph, object, count) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphReleaseUserObjec}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6006, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6006, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6005 + * cuPythonInit() + * if __cuGraphReleaseUserObject == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphReleaseUserObject" not found') + * err = ( __cuGraphReleaseUserObject)(graph, object, count) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6004 + * global __cuGraphReleaseUserObject + * cuPythonInit() + * if __cuGraphReleaseUserObject == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphReleaseUserObject" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6007 + * with gil: + * raise RuntimeError('Function "cuGraphReleaseUserObject" not found') + * err = ( __cuGraphReleaseUserObject)(graph, object, count) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraph, CUuserObject, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphReleaseUserObject)(__pyx_v_graph, __pyx_v_object, __pyx_v_count); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6007, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6008 + * raise RuntimeError('Function "cuGraphReleaseUserObject" not found') + * err = ( __cuGraphReleaseUserObject)(graph, object, count) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6001 + * return err + * + * cdef CUresult _cuGraphReleaseUserObject(CUgraph graph, CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphReleaseUserObject + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphReleaseUserObject", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6010 + * return err + * + * cdef CUresult _cuGraphAddNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddNode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddNode(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, size_t __pyx_v_numDependencies, CUgraphNodeParams *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddNode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6012 + * cdef CUresult _cuGraphAddNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddNode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddNode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6012, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6013 + * global __cuGraphAddNode + * cuPythonInit() + * if __cuGraphAddNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddNode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6014 + * cuPythonInit() + * if __cuGraphAddNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddNode" not found') + * err = ( __cuGraphAddNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6015 + * if __cuGraphAddNode == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddNode" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddNode_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6015, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6015, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6014 + * cuPythonInit() + * if __cuGraphAddNode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddNode" not found') + * err = ( __cuGraphAddNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6013 + * global __cuGraphAddNode + * cuPythonInit() + * if __cuGraphAddNode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddNode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6016 + * with gil: + * raise RuntimeError('Function "cuGraphAddNode" not found') + * err = ( __cuGraphAddNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUgraphNodeParams *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6016, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6017 + * raise RuntimeError('Function "cuGraphAddNode" not found') + * err = ( __cuGraphAddNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphAddNode_v2(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6010 + * return err + * + * cdef CUresult _cuGraphAddNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddNode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6019 + * return err + * + * cdef CUresult _cuGraphAddNode_v2(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddNode_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddNode_v2(CUgraphNode *__pyx_v_phGraphNode, CUgraph __pyx_v_hGraph, CUgraphNode const *__pyx_v_dependencies, CUgraphEdgeData const *__pyx_v_dependencyData, size_t __pyx_v_numDependencies, CUgraphNodeParams *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphAddNode_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6021 + * cdef CUresult _cuGraphAddNode_v2(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphAddNode_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphAddNode_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6021, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6022 + * global __cuGraphAddNode_v2 + * cuPythonInit() + * if __cuGraphAddNode_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddNode_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6023 + * cuPythonInit() + * if __cuGraphAddNode_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddNode_v2" not found') + * err = ( __cuGraphAddNode_v2)(phGraphNode, hGraph, dependencies, dependencyData, numDependencies, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6024 + * if __cuGraphAddNode_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphAddNode_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphAddNode_v2)(phGraphNode, hGraph, dependencies, dependencyData, numDependencies, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphAddNode_v2_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6024, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6024, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6023 + * cuPythonInit() + * if __cuGraphAddNode_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphAddNode_v2" not found') + * err = ( __cuGraphAddNode_v2)(phGraphNode, hGraph, dependencies, dependencyData, numDependencies, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6022 + * global __cuGraphAddNode_v2 + * cuPythonInit() + * if __cuGraphAddNode_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphAddNode_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6025 + * with gil: + * raise RuntimeError('Function "cuGraphAddNode_v2" not found') + * err = ( __cuGraphAddNode_v2)(phGraphNode, hGraph, dependencies, dependencyData, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode *, CUgraph, CUgraphNode const *, CUgraphEdgeData const *, size_t, CUgraphNodeParams *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode_v2)(__pyx_v_phGraphNode, __pyx_v_hGraph, __pyx_v_dependencies, __pyx_v_dependencyData, __pyx_v_numDependencies, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6025, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6026 + * raise RuntimeError('Function "cuGraphAddNode_v2" not found') + * err = ( __cuGraphAddNode_v2)(phGraphNode, hGraph, dependencies, dependencyData, numDependencies, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphNodeSetParams(CUgraphNode hNode, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6019 + * return err + * + * cdef CUresult _cuGraphAddNode_v2(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphAddNode_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphAddNode_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6028 + * return err + * + * cdef CUresult _cuGraphNodeSetParams(CUgraphNode hNode, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeSetParams(CUgraphNode __pyx_v_hNode, CUgraphNodeParams *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6030 + * cdef CUresult _cuGraphNodeSetParams(CUgraphNode hNode, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6030, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6031 + * global __cuGraphNodeSetParams + * cuPythonInit() + * if __cuGraphNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6032 + * cuPythonInit() + * if __cuGraphNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeSetParams" not found') + * err = ( __cuGraphNodeSetParams)(hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6033 + * if __cuGraphNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphNodeSetParams)(hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphNodeSetParams_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6033, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6033, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6032 + * cuPythonInit() + * if __cuGraphNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphNodeSetParams" not found') + * err = ( __cuGraphNodeSetParams)(hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6031 + * global __cuGraphNodeSetParams + * cuPythonInit() + * if __cuGraphNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6034 + * with gil: + * raise RuntimeError('Function "cuGraphNodeSetParams" not found') + * err = ( __cuGraphNodeSetParams)(hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphNode, CUgraphNodeParams *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetParams)(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6034, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6035 + * raise RuntimeError('Function "cuGraphNodeSetParams" not found') + * err = ( __cuGraphNodeSetParams)(hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphExecNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6028 + * return err + * + * cdef CUresult _cuGraphNodeSetParams(CUgraphNode hNode, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6037 + * return err + * + * cdef CUresult _cuGraphExecNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecNodeSetParams + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecNodeSetParams(CUgraphExec __pyx_v_hGraphExec, CUgraphNode __pyx_v_hNode, CUgraphNodeParams *__pyx_v_nodeParams) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphExecNodeSetParams", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6039 + * cdef CUresult _cuGraphExecNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphExecNodeSetParams + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphExecNodeSetParams == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6039, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6040 + * global __cuGraphExecNodeSetParams + * cuPythonInit() + * if __cuGraphExecNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecNodeSetParams" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecNodeSetParams == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6041 + * cuPythonInit() + * if __cuGraphExecNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecNodeSetParams" not found') + * err = ( __cuGraphExecNodeSetParams)(hGraphExec, hNode, nodeParams) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6042 + * if __cuGraphExecNodeSetParams == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphExecNodeSetParams" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphExecNodeSetParams)(hGraphExec, hNode, nodeParams) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphExecNodeSetParam}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6042, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6042, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6041 + * cuPythonInit() + * if __cuGraphExecNodeSetParams == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphExecNodeSetParams" not found') + * err = ( __cuGraphExecNodeSetParams)(hGraphExec, hNode, nodeParams) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6040 + * global __cuGraphExecNodeSetParams + * cuPythonInit() + * if __cuGraphExecNodeSetParams == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphExecNodeSetParams" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6043 + * with gil: + * raise RuntimeError('Function "cuGraphExecNodeSetParams" not found') + * err = ( __cuGraphExecNodeSetParams)(hGraphExec, hNode, nodeParams) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphExec, CUgraphNode, CUgraphNodeParams *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecNodeSetParams)(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6043, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6044 + * raise RuntimeError('Function "cuGraphExecNodeSetParams" not found') + * err = ( __cuGraphExecNodeSetParams)(hGraphExec, hNode, nodeParams) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphConditionalHandleCreate(CUgraphConditionalHandle* pHandle_out, CUgraph hGraph, CUcontext ctx, unsigned int defaultLaunchValue, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6037 + * return err + * + * cdef CUresult _cuGraphExecNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphExecNodeSetParams + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphExecNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6046 + * return err + * + * cdef CUresult _cuGraphConditionalHandleCreate(CUgraphConditionalHandle* pHandle_out, CUgraph hGraph, CUcontext ctx, unsigned int defaultLaunchValue, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphConditionalHandleCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphConditionalHandleCreate(CUgraphConditionalHandle *__pyx_v_pHandle_out, CUgraph __pyx_v_hGraph, CUcontext __pyx_v_ctx, unsigned int __pyx_v_defaultLaunchValue, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphConditionalHandleCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6048 + * cdef CUresult _cuGraphConditionalHandleCreate(CUgraphConditionalHandle* pHandle_out, CUgraph hGraph, CUcontext ctx, unsigned int defaultLaunchValue, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphConditionalHandleCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphConditionalHandleCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6048, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6049 + * global __cuGraphConditionalHandleCreate + * cuPythonInit() + * if __cuGraphConditionalHandleCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphConditionalHandleCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphConditionalHandleCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6050 + * cuPythonInit() + * if __cuGraphConditionalHandleCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphConditionalHandleCreate" not found') + * err = ( __cuGraphConditionalHandleCreate)(pHandle_out, hGraph, ctx, defaultLaunchValue, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6051 + * if __cuGraphConditionalHandleCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphConditionalHandleCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphConditionalHandleCreate)(pHandle_out, hGraph, ctx, defaultLaunchValue, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphConditionalHandl}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6051, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6051, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6050 + * cuPythonInit() + * if __cuGraphConditionalHandleCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphConditionalHandleCreate" not found') + * err = ( __cuGraphConditionalHandleCreate)(pHandle_out, hGraph, ctx, defaultLaunchValue, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6049 + * global __cuGraphConditionalHandleCreate + * cuPythonInit() + * if __cuGraphConditionalHandleCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphConditionalHandleCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6052 + * with gil: + * raise RuntimeError('Function "cuGraphConditionalHandleCreate" not found') + * err = ( __cuGraphConditionalHandleCreate)(pHandle_out, hGraph, ctx, defaultLaunchValue, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphConditionalHandle *, CUgraph, CUcontext, unsigned int, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphConditionalHandleCreate)(__pyx_v_pHandle_out, __pyx_v_hGraph, __pyx_v_ctx, __pyx_v_defaultLaunchValue, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6052, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6053 + * raise RuntimeError('Function "cuGraphConditionalHandleCreate" not found') + * err = ( __cuGraphConditionalHandleCreate)(pHandle_out, hGraph, ctx, defaultLaunchValue, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6046 + * return err + * + * cdef CUresult _cuGraphConditionalHandleCreate(CUgraphConditionalHandle* pHandle_out, CUgraph hGraph, CUcontext ctx, unsigned int defaultLaunchValue, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphConditionalHandleCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphConditionalHandleCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6055 + * return err + * + * cdef CUresult _cuOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveBlocksPerMultiprocessor + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyMaxActiveBlocksPerMultiprocessor(int *__pyx_v_numBlocks, CUfunction __pyx_v_func, int __pyx_v_blockSize, size_t __pyx_v_dynamicSMemSize) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuOccupancyMaxActiveBlocksPerMultiprocessor", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6057 + * cdef CUresult _cuOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuOccupancyMaxActiveBlocksPerMultiprocessor + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuOccupancyMaxActiveBlocksPerMultiprocessor == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6057, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6058 + * global __cuOccupancyMaxActiveBlocksPerMultiprocessor + * cuPythonInit() + * if __cuOccupancyMaxActiveBlocksPerMultiprocessor == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessor" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessor == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6059 + * cuPythonInit() + * if __cuOccupancyMaxActiveBlocksPerMultiprocessor == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessor" not found') + * err = ( __cuOccupancyMaxActiveBlocksPerMultiprocessor)(numBlocks, func, blockSize, dynamicSMemSize) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6060 + * if __cuOccupancyMaxActiveBlocksPerMultiprocessor == NULL: + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessor" not found') # <<<<<<<<<<<<<< + * err = ( __cuOccupancyMaxActiveBlocksPerMultiprocessor)(numBlocks, func, blockSize, dynamicSMemSize) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuOccupancyMaxActiveBlo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6060, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6060, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6059 + * cuPythonInit() + * if __cuOccupancyMaxActiveBlocksPerMultiprocessor == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessor" not found') + * err = ( __cuOccupancyMaxActiveBlocksPerMultiprocessor)(numBlocks, func, blockSize, dynamicSMemSize) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6058 + * global __cuOccupancyMaxActiveBlocksPerMultiprocessor + * cuPythonInit() + * if __cuOccupancyMaxActiveBlocksPerMultiprocessor == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessor" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6061 + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessor" not found') + * err = ( __cuOccupancyMaxActiveBlocksPerMultiprocessor)(numBlocks, func, blockSize, dynamicSMemSize) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, CUfunction, int, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessor)(__pyx_v_numBlocks, __pyx_v_func, __pyx_v_blockSize, __pyx_v_dynamicSMemSize); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6061, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6062 + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessor" not found') + * err = ( __cuOccupancyMaxActiveBlocksPerMultiprocessor)(numBlocks, func, blockSize, dynamicSMemSize) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6055 + * return err + * + * cdef CUresult _cuOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveBlocksPerMultiprocessor + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuOccupancyMaxActiveBlocksPerMultiprocessor", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6064 + * return err + * + * cdef CUresult _cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int *__pyx_v_numBlocks, CUfunction __pyx_v_func, int __pyx_v_blockSize, size_t __pyx_v_dynamicSMemSize, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6066 + * cdef CUresult _cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6066, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6067 + * global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + * cuPythonInit() + * if __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6068 + * cuPythonInit() + * if __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags" not found') + * err = ( __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags)(numBlocks, func, blockSize, dynamicSMemSize, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6069 + * if __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags == NULL: + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags" not found') # <<<<<<<<<<<<<< + * err = ( __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags)(numBlocks, func, blockSize, dynamicSMemSize, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuOccupancyMaxActiveBlo_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6069, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6069, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6068 + * cuPythonInit() + * if __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags" not found') + * err = ( __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags)(numBlocks, func, blockSize, dynamicSMemSize, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6067 + * global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + * cuPythonInit() + * if __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6070 + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags" not found') + * err = ( __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags)(numBlocks, func, blockSize, dynamicSMemSize, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, CUfunction, int, size_t, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags)(__pyx_v_numBlocks, __pyx_v_func, __pyx_v_blockSize, __pyx_v_dynamicSMemSize, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6070, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6071 + * raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags" not found') + * err = ( __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags)(numBlocks, func, blockSize, dynamicSMemSize, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuOccupancyMaxPotentialBlockSize(int* minGridSize, int* blockSize, CUfunction func, CUoccupancyB2DSize blockSizeToDynamicSMemSize, size_t dynamicSMemSize, int blockSizeLimit) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6064 + * return err + * + * cdef CUresult _cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6073 + * return err + * + * cdef CUresult _cuOccupancyMaxPotentialBlockSize(int* minGridSize, int* blockSize, CUfunction func, CUoccupancyB2DSize blockSizeToDynamicSMemSize, size_t dynamicSMemSize, int blockSizeLimit) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialBlockSize + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyMaxPotentialBlockSize(int *__pyx_v_minGridSize, int *__pyx_v_blockSize, CUfunction __pyx_v_func, CUoccupancyB2DSize __pyx_v_blockSizeToDynamicSMemSize, size_t __pyx_v_dynamicSMemSize, int __pyx_v_blockSizeLimit) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuOccupancyMaxPotentialBlockSize", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6075 + * cdef CUresult _cuOccupancyMaxPotentialBlockSize(int* minGridSize, int* blockSize, CUfunction func, CUoccupancyB2DSize blockSizeToDynamicSMemSize, size_t dynamicSMemSize, int blockSizeLimit) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuOccupancyMaxPotentialBlockSize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuOccupancyMaxPotentialBlockSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6075, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6076 + * global __cuOccupancyMaxPotentialBlockSize + * cuPythonInit() + * if __cuOccupancyMaxPotentialBlockSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6077 + * cuPythonInit() + * if __cuOccupancyMaxPotentialBlockSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSize" not found') + * err = ( __cuOccupancyMaxPotentialBlockSize)(minGridSize, blockSize, func, blockSizeToDynamicSMemSize, dynamicSMemSize, blockSizeLimit) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6078 + * if __cuOccupancyMaxPotentialBlockSize == NULL: + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSize" not found') # <<<<<<<<<<<<<< + * err = ( __cuOccupancyMaxPotentialBlockSize)(minGridSize, blockSize, func, blockSizeToDynamicSMemSize, dynamicSMemSize, blockSizeLimit) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuOccupancyMaxPotential}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6078, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6078, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6077 + * cuPythonInit() + * if __cuOccupancyMaxPotentialBlockSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSize" not found') + * err = ( __cuOccupancyMaxPotentialBlockSize)(minGridSize, blockSize, func, blockSizeToDynamicSMemSize, dynamicSMemSize, blockSizeLimit) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6076 + * global __cuOccupancyMaxPotentialBlockSize + * cuPythonInit() + * if __cuOccupancyMaxPotentialBlockSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6079 + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSize" not found') + * err = ( __cuOccupancyMaxPotentialBlockSize)(minGridSize, blockSize, func, blockSizeToDynamicSMemSize, dynamicSMemSize, blockSizeLimit) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, int *, CUfunction, CUoccupancyB2DSize, size_t, int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSize)(__pyx_v_minGridSize, __pyx_v_blockSize, __pyx_v_func, __pyx_v_blockSizeToDynamicSMemSize, __pyx_v_dynamicSMemSize, __pyx_v_blockSizeLimit); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6079, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6080 + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSize" not found') + * err = ( __cuOccupancyMaxPotentialBlockSize)(minGridSize, blockSize, func, blockSizeToDynamicSMemSize, dynamicSMemSize, blockSizeLimit) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuOccupancyMaxPotentialBlockSizeWithFlags(int* minGridSize, int* blockSize, CUfunction func, CUoccupancyB2DSize blockSizeToDynamicSMemSize, size_t dynamicSMemSize, int blockSizeLimit, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6073 + * return err + * + * cdef CUresult _cuOccupancyMaxPotentialBlockSize(int* minGridSize, int* blockSize, CUfunction func, CUoccupancyB2DSize blockSizeToDynamicSMemSize, size_t dynamicSMemSize, int blockSizeLimit) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialBlockSize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuOccupancyMaxPotentialBlockSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6082 + * return err + * + * cdef CUresult _cuOccupancyMaxPotentialBlockSizeWithFlags(int* minGridSize, int* blockSize, CUfunction func, CUoccupancyB2DSize blockSizeToDynamicSMemSize, size_t dynamicSMemSize, int blockSizeLimit, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialBlockSizeWithFlags + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyMaxPotentialBlockSizeWithFlags(int *__pyx_v_minGridSize, int *__pyx_v_blockSize, CUfunction __pyx_v_func, CUoccupancyB2DSize __pyx_v_blockSizeToDynamicSMemSize, size_t __pyx_v_dynamicSMemSize, int __pyx_v_blockSizeLimit, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuOccupancyMaxPotentialBlockSizeWithFlags", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6084 + * cdef CUresult _cuOccupancyMaxPotentialBlockSizeWithFlags(int* minGridSize, int* blockSize, CUfunction func, CUoccupancyB2DSize blockSizeToDynamicSMemSize, size_t dynamicSMemSize, int blockSizeLimit, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuOccupancyMaxPotentialBlockSizeWithFlags + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuOccupancyMaxPotentialBlockSizeWithFlags == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6084, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6085 + * global __cuOccupancyMaxPotentialBlockSizeWithFlags + * cuPythonInit() + * if __cuOccupancyMaxPotentialBlockSizeWithFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSizeWithFlags" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSizeWithFlags == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6086 + * cuPythonInit() + * if __cuOccupancyMaxPotentialBlockSizeWithFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSizeWithFlags" not found') + * err = ( __cuOccupancyMaxPotentialBlockSizeWithFlags)(minGridSize, blockSize, func, blockSizeToDynamicSMemSize, dynamicSMemSize, blockSizeLimit, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6087 + * if __cuOccupancyMaxPotentialBlockSizeWithFlags == NULL: + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSizeWithFlags" not found') # <<<<<<<<<<<<<< + * err = ( __cuOccupancyMaxPotentialBlockSizeWithFlags)(minGridSize, blockSize, func, blockSizeToDynamicSMemSize, dynamicSMemSize, blockSizeLimit, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuOccupancyMaxPotential_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6087, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6087, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6086 + * cuPythonInit() + * if __cuOccupancyMaxPotentialBlockSizeWithFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSizeWithFlags" not found') + * err = ( __cuOccupancyMaxPotentialBlockSizeWithFlags)(minGridSize, blockSize, func, blockSizeToDynamicSMemSize, dynamicSMemSize, blockSizeLimit, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6085 + * global __cuOccupancyMaxPotentialBlockSizeWithFlags + * cuPythonInit() + * if __cuOccupancyMaxPotentialBlockSizeWithFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSizeWithFlags" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6088 + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSizeWithFlags" not found') + * err = ( __cuOccupancyMaxPotentialBlockSizeWithFlags)(minGridSize, blockSize, func, blockSizeToDynamicSMemSize, dynamicSMemSize, blockSizeLimit, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, int *, CUfunction, CUoccupancyB2DSize, size_t, int, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSizeWithFlags)(__pyx_v_minGridSize, __pyx_v_blockSize, __pyx_v_func, __pyx_v_blockSizeToDynamicSMemSize, __pyx_v_dynamicSMemSize, __pyx_v_blockSizeLimit, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6088, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6089 + * raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSizeWithFlags" not found') + * err = ( __cuOccupancyMaxPotentialBlockSizeWithFlags)(minGridSize, blockSize, func, blockSizeToDynamicSMemSize, dynamicSMemSize, blockSizeLimit, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, CUfunction func, int numBlocks, int blockSize) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6082 + * return err + * + * cdef CUresult _cuOccupancyMaxPotentialBlockSizeWithFlags(int* minGridSize, int* blockSize, CUfunction func, CUoccupancyB2DSize blockSizeToDynamicSMemSize, size_t dynamicSMemSize, int blockSizeLimit, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialBlockSizeWithFlags + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuOccupancyMaxPotentialBlockSizeWithFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6091 + * return err + * + * cdef CUresult _cuOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, CUfunction func, int numBlocks, int blockSize) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyAvailableDynamicSMemPerBlock + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyAvailableDynamicSMemPerBlock(size_t *__pyx_v_dynamicSmemSize, CUfunction __pyx_v_func, int __pyx_v_numBlocks, int __pyx_v_blockSize) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuOccupancyAvailableDynamicSMemPerBlock", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6093 + * cdef CUresult _cuOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, CUfunction func, int numBlocks, int blockSize) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuOccupancyAvailableDynamicSMemPerBlock + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuOccupancyAvailableDynamicSMemPerBlock == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6093, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6094 + * global __cuOccupancyAvailableDynamicSMemPerBlock + * cuPythonInit() + * if __cuOccupancyAvailableDynamicSMemPerBlock == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyAvailableDynamicSMemPerBlock" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyAvailableDynamicSMemPerBlock == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6095 + * cuPythonInit() + * if __cuOccupancyAvailableDynamicSMemPerBlock == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyAvailableDynamicSMemPerBlock" not found') + * err = ( __cuOccupancyAvailableDynamicSMemPerBlock)(dynamicSmemSize, func, numBlocks, blockSize) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6096 + * if __cuOccupancyAvailableDynamicSMemPerBlock == NULL: + * with gil: + * raise RuntimeError('Function "cuOccupancyAvailableDynamicSMemPerBlock" not found') # <<<<<<<<<<<<<< + * err = ( __cuOccupancyAvailableDynamicSMemPerBlock)(dynamicSmemSize, func, numBlocks, blockSize) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuOccupancyAvailableDyn}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6096, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6096, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6095 + * cuPythonInit() + * if __cuOccupancyAvailableDynamicSMemPerBlock == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyAvailableDynamicSMemPerBlock" not found') + * err = ( __cuOccupancyAvailableDynamicSMemPerBlock)(dynamicSmemSize, func, numBlocks, blockSize) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6094 + * global __cuOccupancyAvailableDynamicSMemPerBlock + * cuPythonInit() + * if __cuOccupancyAvailableDynamicSMemPerBlock == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyAvailableDynamicSMemPerBlock" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6097 + * with gil: + * raise RuntimeError('Function "cuOccupancyAvailableDynamicSMemPerBlock" not found') + * err = ( __cuOccupancyAvailableDynamicSMemPerBlock)(dynamicSmemSize, func, numBlocks, blockSize) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(size_t *, CUfunction, int, int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyAvailableDynamicSMemPerBlock)(__pyx_v_dynamicSmemSize, __pyx_v_func, __pyx_v_numBlocks, __pyx_v_blockSize); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6097, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6098 + * raise RuntimeError('Function "cuOccupancyAvailableDynamicSMemPerBlock" not found') + * err = ( __cuOccupancyAvailableDynamicSMemPerBlock)(dynamicSmemSize, func, numBlocks, blockSize) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuOccupancyMaxPotentialClusterSize(int* clusterSize, CUfunction func, const CUlaunchConfig* config) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6091 + * return err + * + * cdef CUresult _cuOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, CUfunction func, int numBlocks, int blockSize) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyAvailableDynamicSMemPerBlock + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuOccupancyAvailableDynamicSMemPerBlock", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6100 + * return err + * + * cdef CUresult _cuOccupancyMaxPotentialClusterSize(int* clusterSize, CUfunction func, const CUlaunchConfig* config) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialClusterSize + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyMaxPotentialClusterSize(int *__pyx_v_clusterSize, CUfunction __pyx_v_func, CUlaunchConfig const *__pyx_v_config) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuOccupancyMaxPotentialClusterSize", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6102 + * cdef CUresult _cuOccupancyMaxPotentialClusterSize(int* clusterSize, CUfunction func, const CUlaunchConfig* config) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuOccupancyMaxPotentialClusterSize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuOccupancyMaxPotentialClusterSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6102, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6103 + * global __cuOccupancyMaxPotentialClusterSize + * cuPythonInit() + * if __cuOccupancyMaxPotentialClusterSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxPotentialClusterSize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialClusterSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6104 + * cuPythonInit() + * if __cuOccupancyMaxPotentialClusterSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyMaxPotentialClusterSize" not found') + * err = ( __cuOccupancyMaxPotentialClusterSize)(clusterSize, func, config) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6105 + * if __cuOccupancyMaxPotentialClusterSize == NULL: + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxPotentialClusterSize" not found') # <<<<<<<<<<<<<< + * err = ( __cuOccupancyMaxPotentialClusterSize)(clusterSize, func, config) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuOccupancyMaxPotential_3}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6105, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6105, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6104 + * cuPythonInit() + * if __cuOccupancyMaxPotentialClusterSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyMaxPotentialClusterSize" not found') + * err = ( __cuOccupancyMaxPotentialClusterSize)(clusterSize, func, config) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6103 + * global __cuOccupancyMaxPotentialClusterSize + * cuPythonInit() + * if __cuOccupancyMaxPotentialClusterSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxPotentialClusterSize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6106 + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxPotentialClusterSize" not found') + * err = ( __cuOccupancyMaxPotentialClusterSize)(clusterSize, func, config) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, CUfunction, CUlaunchConfig const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialClusterSize)(__pyx_v_clusterSize, __pyx_v_func, __pyx_v_config); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6106, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6107 + * raise RuntimeError('Function "cuOccupancyMaxPotentialClusterSize" not found') + * err = ( __cuOccupancyMaxPotentialClusterSize)(clusterSize, func, config) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuOccupancyMaxActiveClusters(int* numClusters, CUfunction func, const CUlaunchConfig* config) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6100 + * return err + * + * cdef CUresult _cuOccupancyMaxPotentialClusterSize(int* clusterSize, CUfunction func, const CUlaunchConfig* config) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialClusterSize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuOccupancyMaxPotentialClusterSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6109 + * return err + * + * cdef CUresult _cuOccupancyMaxActiveClusters(int* numClusters, CUfunction func, const CUlaunchConfig* config) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveClusters + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyMaxActiveClusters(int *__pyx_v_numClusters, CUfunction __pyx_v_func, CUlaunchConfig const *__pyx_v_config) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuOccupancyMaxActiveClusters", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6111 + * cdef CUresult _cuOccupancyMaxActiveClusters(int* numClusters, CUfunction func, const CUlaunchConfig* config) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuOccupancyMaxActiveClusters + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuOccupancyMaxActiveClusters == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6111, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6112 + * global __cuOccupancyMaxActiveClusters + * cuPythonInit() + * if __cuOccupancyMaxActiveClusters == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxActiveClusters" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveClusters == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6113 + * cuPythonInit() + * if __cuOccupancyMaxActiveClusters == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyMaxActiveClusters" not found') + * err = ( __cuOccupancyMaxActiveClusters)(numClusters, func, config) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6114 + * if __cuOccupancyMaxActiveClusters == NULL: + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxActiveClusters" not found') # <<<<<<<<<<<<<< + * err = ( __cuOccupancyMaxActiveClusters)(numClusters, func, config) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuOccupancyMaxActiveClu}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6114, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6114, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6113 + * cuPythonInit() + * if __cuOccupancyMaxActiveClusters == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuOccupancyMaxActiveClusters" not found') + * err = ( __cuOccupancyMaxActiveClusters)(numClusters, func, config) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6112 + * global __cuOccupancyMaxActiveClusters + * cuPythonInit() + * if __cuOccupancyMaxActiveClusters == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxActiveClusters" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6115 + * with gil: + * raise RuntimeError('Function "cuOccupancyMaxActiveClusters" not found') + * err = ( __cuOccupancyMaxActiveClusters)(numClusters, func, config) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, CUfunction, CUlaunchConfig const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveClusters)(__pyx_v_numClusters, __pyx_v_func, __pyx_v_config); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6115, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6116 + * raise RuntimeError('Function "cuOccupancyMaxActiveClusters" not found') + * err = ( __cuOccupancyMaxActiveClusters)(numClusters, func, config) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefSetArray(CUtexref hTexRef, CUarray hArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6109 + * return err + * + * cdef CUresult _cuOccupancyMaxActiveClusters(int* numClusters, CUfunction func, const CUlaunchConfig* config) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveClusters + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuOccupancyMaxActiveClusters", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6118 + * return err + * + * cdef CUresult _cuTexRefSetArray(CUtexref hTexRef, CUarray hArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetArray + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetArray(CUtexref __pyx_v_hTexRef, CUarray __pyx_v_hArray, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefSetArray", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6120 + * cdef CUresult _cuTexRefSetArray(CUtexref hTexRef, CUarray hArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefSetArray + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefSetArray == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6120, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6121 + * global __cuTexRefSetArray + * cuPythonInit() + * if __cuTexRefSetArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetArray" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetArray == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6122 + * cuPythonInit() + * if __cuTexRefSetArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetArray" not found') + * err = ( __cuTexRefSetArray)(hTexRef, hArray, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6123 + * if __cuTexRefSetArray == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefSetArray" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefSetArray)(hTexRef, hArray, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefSetArray_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6123, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6123, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6122 + * cuPythonInit() + * if __cuTexRefSetArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetArray" not found') + * err = ( __cuTexRefSetArray)(hTexRef, hArray, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6121 + * global __cuTexRefSetArray + * cuPythonInit() + * if __cuTexRefSetArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetArray" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6124 + * with gil: + * raise RuntimeError('Function "cuTexRefSetArray" not found') + * err = ( __cuTexRefSetArray)(hTexRef, hArray, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref, CUarray, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetArray)(__pyx_v_hTexRef, __pyx_v_hArray, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6124, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6125 + * raise RuntimeError('Function "cuTexRefSetArray" not found') + * err = ( __cuTexRefSetArray)(hTexRef, hArray, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefSetMipmappedArray(CUtexref hTexRef, CUmipmappedArray hMipmappedArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6118 + * return err + * + * cdef CUresult _cuTexRefSetArray(CUtexref hTexRef, CUarray hArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetArray + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefSetArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6127 + * return err + * + * cdef CUresult _cuTexRefSetMipmappedArray(CUtexref hTexRef, CUmipmappedArray hMipmappedArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmappedArray + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetMipmappedArray(CUtexref __pyx_v_hTexRef, CUmipmappedArray __pyx_v_hMipmappedArray, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefSetMipmappedArray", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6129 + * cdef CUresult _cuTexRefSetMipmappedArray(CUtexref hTexRef, CUmipmappedArray hMipmappedArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefSetMipmappedArray + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefSetMipmappedArray == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6129, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6130 + * global __cuTexRefSetMipmappedArray + * cuPythonInit() + * if __cuTexRefSetMipmappedArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmappedArray" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmappedArray == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6131 + * cuPythonInit() + * if __cuTexRefSetMipmappedArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetMipmappedArray" not found') + * err = ( __cuTexRefSetMipmappedArray)(hTexRef, hMipmappedArray, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6132 + * if __cuTexRefSetMipmappedArray == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmappedArray" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefSetMipmappedArray)(hTexRef, hMipmappedArray, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefSetMipmappedArr}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6132, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6132, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6131 + * cuPythonInit() + * if __cuTexRefSetMipmappedArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetMipmappedArray" not found') + * err = ( __cuTexRefSetMipmappedArray)(hTexRef, hMipmappedArray, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6130 + * global __cuTexRefSetMipmappedArray + * cuPythonInit() + * if __cuTexRefSetMipmappedArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmappedArray" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6133 + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmappedArray" not found') + * err = ( __cuTexRefSetMipmappedArray)(hTexRef, hMipmappedArray, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref, CUmipmappedArray, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmappedArray)(__pyx_v_hTexRef, __pyx_v_hMipmappedArray, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6133, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6134 + * raise RuntimeError('Function "cuTexRefSetMipmappedArray" not found') + * err = ( __cuTexRefSetMipmappedArray)(hTexRef, hMipmappedArray, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefSetAddress_v2(size_t* ByteOffset, CUtexref hTexRef, CUdeviceptr dptr, size_t numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6127 + * return err + * + * cdef CUresult _cuTexRefSetMipmappedArray(CUtexref hTexRef, CUmipmappedArray hMipmappedArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmappedArray + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefSetMipmappedArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6136 + * return err + * + * cdef CUresult _cuTexRefSetAddress_v2(size_t* ByteOffset, CUtexref hTexRef, CUdeviceptr dptr, size_t numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetAddress_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetAddress_v2(size_t *__pyx_v_ByteOffset, CUtexref __pyx_v_hTexRef, CUdeviceptr __pyx_v_dptr, size_t __pyx_v_numbytes) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefSetAddress_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6138 + * cdef CUresult _cuTexRefSetAddress_v2(size_t* ByteOffset, CUtexref hTexRef, CUdeviceptr dptr, size_t numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefSetAddress_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefSetAddress_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6138, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6139 + * global __cuTexRefSetAddress_v2 + * cuPythonInit() + * if __cuTexRefSetAddress_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetAddress_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6140 + * cuPythonInit() + * if __cuTexRefSetAddress_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetAddress_v2" not found') + * err = ( __cuTexRefSetAddress_v2)(ByteOffset, hTexRef, dptr, numbytes) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6141 + * if __cuTexRefSetAddress_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefSetAddress_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefSetAddress_v2)(ByteOffset, hTexRef, dptr, numbytes) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefSetAddress_v2_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6141, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6141, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6140 + * cuPythonInit() + * if __cuTexRefSetAddress_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetAddress_v2" not found') + * err = ( __cuTexRefSetAddress_v2)(ByteOffset, hTexRef, dptr, numbytes) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6139 + * global __cuTexRefSetAddress_v2 + * cuPythonInit() + * if __cuTexRefSetAddress_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetAddress_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6142 + * with gil: + * raise RuntimeError('Function "cuTexRefSetAddress_v2" not found') + * err = ( __cuTexRefSetAddress_v2)(ByteOffset, hTexRef, dptr, numbytes) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(size_t *, CUtexref, CUdeviceptr, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress_v2)(__pyx_v_ByteOffset, __pyx_v_hTexRef, __pyx_v_dptr, __pyx_v_numbytes); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6142, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6143 + * raise RuntimeError('Function "cuTexRefSetAddress_v2" not found') + * err = ( __cuTexRefSetAddress_v2)(ByteOffset, hTexRef, dptr, numbytes) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefSetAddress2D_v3(CUtexref hTexRef, const CUDA_ARRAY_DESCRIPTOR* desc, CUdeviceptr dptr, size_t Pitch) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6136 + * return err + * + * cdef CUresult _cuTexRefSetAddress_v2(size_t* ByteOffset, CUtexref hTexRef, CUdeviceptr dptr, size_t numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetAddress_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefSetAddress_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6145 + * return err + * + * cdef CUresult _cuTexRefSetAddress2D_v3(CUtexref hTexRef, const CUDA_ARRAY_DESCRIPTOR* desc, CUdeviceptr dptr, size_t Pitch) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetAddress2D_v3 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetAddress2D_v3(CUtexref __pyx_v_hTexRef, CUDA_ARRAY_DESCRIPTOR const *__pyx_v_desc, CUdeviceptr __pyx_v_dptr, size_t __pyx_v_Pitch) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefSetAddress2D_v3", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6147 + * cdef CUresult _cuTexRefSetAddress2D_v3(CUtexref hTexRef, const CUDA_ARRAY_DESCRIPTOR* desc, CUdeviceptr dptr, size_t Pitch) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefSetAddress2D_v3 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefSetAddress2D_v3 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6147, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6148 + * global __cuTexRefSetAddress2D_v3 + * cuPythonInit() + * if __cuTexRefSetAddress2D_v3 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetAddress2D_v3" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress2D_v3 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6149 + * cuPythonInit() + * if __cuTexRefSetAddress2D_v3 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetAddress2D_v3" not found') + * err = ( __cuTexRefSetAddress2D_v3)(hTexRef, desc, dptr, Pitch) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6150 + * if __cuTexRefSetAddress2D_v3 == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefSetAddress2D_v3" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefSetAddress2D_v3)(hTexRef, desc, dptr, Pitch) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefSetAddress2D_v3}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6150, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6150, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6149 + * cuPythonInit() + * if __cuTexRefSetAddress2D_v3 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetAddress2D_v3" not found') + * err = ( __cuTexRefSetAddress2D_v3)(hTexRef, desc, dptr, Pitch) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6148 + * global __cuTexRefSetAddress2D_v3 + * cuPythonInit() + * if __cuTexRefSetAddress2D_v3 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetAddress2D_v3" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6151 + * with gil: + * raise RuntimeError('Function "cuTexRefSetAddress2D_v3" not found') + * err = ( __cuTexRefSetAddress2D_v3)(hTexRef, desc, dptr, Pitch) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref, CUDA_ARRAY_DESCRIPTOR const *, CUdeviceptr, size_t))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress2D_v3)(__pyx_v_hTexRef, __pyx_v_desc, __pyx_v_dptr, __pyx_v_Pitch); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6151, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6152 + * raise RuntimeError('Function "cuTexRefSetAddress2D_v3" not found') + * err = ( __cuTexRefSetAddress2D_v3)(hTexRef, desc, dptr, Pitch) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefSetFormat(CUtexref hTexRef, CUarray_format fmt, int NumPackedComponents) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6145 + * return err + * + * cdef CUresult _cuTexRefSetAddress2D_v3(CUtexref hTexRef, const CUDA_ARRAY_DESCRIPTOR* desc, CUdeviceptr dptr, size_t Pitch) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetAddress2D_v3 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefSetAddress2D_v3", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6154 + * return err + * + * cdef CUresult _cuTexRefSetFormat(CUtexref hTexRef, CUarray_format fmt, int NumPackedComponents) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetFormat + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetFormat(CUtexref __pyx_v_hTexRef, CUarray_format __pyx_v_fmt, int __pyx_v_NumPackedComponents) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefSetFormat", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6156 + * cdef CUresult _cuTexRefSetFormat(CUtexref hTexRef, CUarray_format fmt, int NumPackedComponents) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefSetFormat + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefSetFormat == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6156, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6157 + * global __cuTexRefSetFormat + * cuPythonInit() + * if __cuTexRefSetFormat == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetFormat" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFormat == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6158 + * cuPythonInit() + * if __cuTexRefSetFormat == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetFormat" not found') + * err = ( __cuTexRefSetFormat)(hTexRef, fmt, NumPackedComponents) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6159 + * if __cuTexRefSetFormat == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefSetFormat" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefSetFormat)(hTexRef, fmt, NumPackedComponents) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefSetFormat_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6159, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6159, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6158 + * cuPythonInit() + * if __cuTexRefSetFormat == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetFormat" not found') + * err = ( __cuTexRefSetFormat)(hTexRef, fmt, NumPackedComponents) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6157 + * global __cuTexRefSetFormat + * cuPythonInit() + * if __cuTexRefSetFormat == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetFormat" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6160 + * with gil: + * raise RuntimeError('Function "cuTexRefSetFormat" not found') + * err = ( __cuTexRefSetFormat)(hTexRef, fmt, NumPackedComponents) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref, CUarray_format, int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFormat)(__pyx_v_hTexRef, __pyx_v_fmt, __pyx_v_NumPackedComponents); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6160, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6161 + * raise RuntimeError('Function "cuTexRefSetFormat" not found') + * err = ( __cuTexRefSetFormat)(hTexRef, fmt, NumPackedComponents) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefSetAddressMode(CUtexref hTexRef, int dim, CUaddress_mode am) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6154 + * return err + * + * cdef CUresult _cuTexRefSetFormat(CUtexref hTexRef, CUarray_format fmt, int NumPackedComponents) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetFormat + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefSetFormat", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6163 + * return err + * + * cdef CUresult _cuTexRefSetAddressMode(CUtexref hTexRef, int dim, CUaddress_mode am) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetAddressMode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetAddressMode(CUtexref __pyx_v_hTexRef, int __pyx_v_dim, CUaddress_mode __pyx_v_am) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefSetAddressMode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6165 + * cdef CUresult _cuTexRefSetAddressMode(CUtexref hTexRef, int dim, CUaddress_mode am) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefSetAddressMode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefSetAddressMode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6165, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6166 + * global __cuTexRefSetAddressMode + * cuPythonInit() + * if __cuTexRefSetAddressMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetAddressMode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddressMode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6167 + * cuPythonInit() + * if __cuTexRefSetAddressMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetAddressMode" not found') + * err = ( __cuTexRefSetAddressMode)(hTexRef, dim, am) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6168 + * if __cuTexRefSetAddressMode == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefSetAddressMode" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefSetAddressMode)(hTexRef, dim, am) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefSetAddressMode}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6168, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6168, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6167 + * cuPythonInit() + * if __cuTexRefSetAddressMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetAddressMode" not found') + * err = ( __cuTexRefSetAddressMode)(hTexRef, dim, am) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6166 + * global __cuTexRefSetAddressMode + * cuPythonInit() + * if __cuTexRefSetAddressMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetAddressMode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6169 + * with gil: + * raise RuntimeError('Function "cuTexRefSetAddressMode" not found') + * err = ( __cuTexRefSetAddressMode)(hTexRef, dim, am) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref, int, CUaddress_mode))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddressMode)(__pyx_v_hTexRef, __pyx_v_dim, __pyx_v_am); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6169, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6170 + * raise RuntimeError('Function "cuTexRefSetAddressMode" not found') + * err = ( __cuTexRefSetAddressMode)(hTexRef, dim, am) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefSetFilterMode(CUtexref hTexRef, CUfilter_mode fm) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6163 + * return err + * + * cdef CUresult _cuTexRefSetAddressMode(CUtexref hTexRef, int dim, CUaddress_mode am) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetAddressMode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefSetAddressMode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6172 + * return err + * + * cdef CUresult _cuTexRefSetFilterMode(CUtexref hTexRef, CUfilter_mode fm) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetFilterMode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetFilterMode(CUtexref __pyx_v_hTexRef, CUfilter_mode __pyx_v_fm) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefSetFilterMode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6174 + * cdef CUresult _cuTexRefSetFilterMode(CUtexref hTexRef, CUfilter_mode fm) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefSetFilterMode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefSetFilterMode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6174, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6175 + * global __cuTexRefSetFilterMode + * cuPythonInit() + * if __cuTexRefSetFilterMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetFilterMode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFilterMode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6176 + * cuPythonInit() + * if __cuTexRefSetFilterMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetFilterMode" not found') + * err = ( __cuTexRefSetFilterMode)(hTexRef, fm) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6177 + * if __cuTexRefSetFilterMode == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefSetFilterMode" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefSetFilterMode)(hTexRef, fm) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefSetFilterMode_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6177, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6177, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6176 + * cuPythonInit() + * if __cuTexRefSetFilterMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetFilterMode" not found') + * err = ( __cuTexRefSetFilterMode)(hTexRef, fm) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6175 + * global __cuTexRefSetFilterMode + * cuPythonInit() + * if __cuTexRefSetFilterMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetFilterMode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6178 + * with gil: + * raise RuntimeError('Function "cuTexRefSetFilterMode" not found') + * err = ( __cuTexRefSetFilterMode)(hTexRef, fm) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref, CUfilter_mode))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFilterMode)(__pyx_v_hTexRef, __pyx_v_fm); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6178, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6179 + * raise RuntimeError('Function "cuTexRefSetFilterMode" not found') + * err = ( __cuTexRefSetFilterMode)(hTexRef, fm) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefSetMipmapFilterMode(CUtexref hTexRef, CUfilter_mode fm) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6172 + * return err + * + * cdef CUresult _cuTexRefSetFilterMode(CUtexref hTexRef, CUfilter_mode fm) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetFilterMode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefSetFilterMode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6181 + * return err + * + * cdef CUresult _cuTexRefSetMipmapFilterMode(CUtexref hTexRef, CUfilter_mode fm) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapFilterMode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetMipmapFilterMode(CUtexref __pyx_v_hTexRef, CUfilter_mode __pyx_v_fm) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefSetMipmapFilterMode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6183 + * cdef CUresult _cuTexRefSetMipmapFilterMode(CUtexref hTexRef, CUfilter_mode fm) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefSetMipmapFilterMode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefSetMipmapFilterMode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6183, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6184 + * global __cuTexRefSetMipmapFilterMode + * cuPythonInit() + * if __cuTexRefSetMipmapFilterMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmapFilterMode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapFilterMode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6185 + * cuPythonInit() + * if __cuTexRefSetMipmapFilterMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetMipmapFilterMode" not found') + * err = ( __cuTexRefSetMipmapFilterMode)(hTexRef, fm) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6186 + * if __cuTexRefSetMipmapFilterMode == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmapFilterMode" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefSetMipmapFilterMode)(hTexRef, fm) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefSetMipmapFilter}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6186, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6186, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6185 + * cuPythonInit() + * if __cuTexRefSetMipmapFilterMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetMipmapFilterMode" not found') + * err = ( __cuTexRefSetMipmapFilterMode)(hTexRef, fm) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6184 + * global __cuTexRefSetMipmapFilterMode + * cuPythonInit() + * if __cuTexRefSetMipmapFilterMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmapFilterMode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6187 + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmapFilterMode" not found') + * err = ( __cuTexRefSetMipmapFilterMode)(hTexRef, fm) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref, CUfilter_mode))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapFilterMode)(__pyx_v_hTexRef, __pyx_v_fm); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6187, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6188 + * raise RuntimeError('Function "cuTexRefSetMipmapFilterMode" not found') + * err = ( __cuTexRefSetMipmapFilterMode)(hTexRef, fm) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefSetMipmapLevelBias(CUtexref hTexRef, float bias) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6181 + * return err + * + * cdef CUresult _cuTexRefSetMipmapFilterMode(CUtexref hTexRef, CUfilter_mode fm) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapFilterMode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefSetMipmapFilterMode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6190 + * return err + * + * cdef CUresult _cuTexRefSetMipmapLevelBias(CUtexref hTexRef, float bias) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapLevelBias + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetMipmapLevelBias(CUtexref __pyx_v_hTexRef, float __pyx_v_bias) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefSetMipmapLevelBias", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6192 + * cdef CUresult _cuTexRefSetMipmapLevelBias(CUtexref hTexRef, float bias) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefSetMipmapLevelBias + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefSetMipmapLevelBias == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6192, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6193 + * global __cuTexRefSetMipmapLevelBias + * cuPythonInit() + * if __cuTexRefSetMipmapLevelBias == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmapLevelBias" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelBias == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6194 + * cuPythonInit() + * if __cuTexRefSetMipmapLevelBias == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetMipmapLevelBias" not found') + * err = ( __cuTexRefSetMipmapLevelBias)(hTexRef, bias) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6195 + * if __cuTexRefSetMipmapLevelBias == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmapLevelBias" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefSetMipmapLevelBias)(hTexRef, bias) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefSetMipmapLevelB}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6195, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6195, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6194 + * cuPythonInit() + * if __cuTexRefSetMipmapLevelBias == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetMipmapLevelBias" not found') + * err = ( __cuTexRefSetMipmapLevelBias)(hTexRef, bias) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6193 + * global __cuTexRefSetMipmapLevelBias + * cuPythonInit() + * if __cuTexRefSetMipmapLevelBias == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmapLevelBias" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6196 + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmapLevelBias" not found') + * err = ( __cuTexRefSetMipmapLevelBias)(hTexRef, bias) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref, float))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelBias)(__pyx_v_hTexRef, __pyx_v_bias); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6196, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6197 + * raise RuntimeError('Function "cuTexRefSetMipmapLevelBias" not found') + * err = ( __cuTexRefSetMipmapLevelBias)(hTexRef, bias) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefSetMipmapLevelClamp(CUtexref hTexRef, float minMipmapLevelClamp, float maxMipmapLevelClamp) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6190 + * return err + * + * cdef CUresult _cuTexRefSetMipmapLevelBias(CUtexref hTexRef, float bias) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapLevelBias + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefSetMipmapLevelBias", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6199 + * return err + * + * cdef CUresult _cuTexRefSetMipmapLevelClamp(CUtexref hTexRef, float minMipmapLevelClamp, float maxMipmapLevelClamp) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapLevelClamp + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetMipmapLevelClamp(CUtexref __pyx_v_hTexRef, float __pyx_v_minMipmapLevelClamp, float __pyx_v_maxMipmapLevelClamp) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefSetMipmapLevelClamp", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6201 + * cdef CUresult _cuTexRefSetMipmapLevelClamp(CUtexref hTexRef, float minMipmapLevelClamp, float maxMipmapLevelClamp) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefSetMipmapLevelClamp + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefSetMipmapLevelClamp == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6201, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6202 + * global __cuTexRefSetMipmapLevelClamp + * cuPythonInit() + * if __cuTexRefSetMipmapLevelClamp == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmapLevelClamp" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelClamp == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6203 + * cuPythonInit() + * if __cuTexRefSetMipmapLevelClamp == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetMipmapLevelClamp" not found') + * err = ( __cuTexRefSetMipmapLevelClamp)(hTexRef, minMipmapLevelClamp, maxMipmapLevelClamp) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6204 + * if __cuTexRefSetMipmapLevelClamp == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmapLevelClamp" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefSetMipmapLevelClamp)(hTexRef, minMipmapLevelClamp, maxMipmapLevelClamp) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefSetMipmapLevelC}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6204, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6204, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6203 + * cuPythonInit() + * if __cuTexRefSetMipmapLevelClamp == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetMipmapLevelClamp" not found') + * err = ( __cuTexRefSetMipmapLevelClamp)(hTexRef, minMipmapLevelClamp, maxMipmapLevelClamp) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6202 + * global __cuTexRefSetMipmapLevelClamp + * cuPythonInit() + * if __cuTexRefSetMipmapLevelClamp == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmapLevelClamp" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6205 + * with gil: + * raise RuntimeError('Function "cuTexRefSetMipmapLevelClamp" not found') + * err = ( __cuTexRefSetMipmapLevelClamp)(hTexRef, minMipmapLevelClamp, maxMipmapLevelClamp) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref, float, float))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelClamp)(__pyx_v_hTexRef, __pyx_v_minMipmapLevelClamp, __pyx_v_maxMipmapLevelClamp); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6205, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6206 + * raise RuntimeError('Function "cuTexRefSetMipmapLevelClamp" not found') + * err = ( __cuTexRefSetMipmapLevelClamp)(hTexRef, minMipmapLevelClamp, maxMipmapLevelClamp) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefSetMaxAnisotropy(CUtexref hTexRef, unsigned int maxAniso) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6199 + * return err + * + * cdef CUresult _cuTexRefSetMipmapLevelClamp(CUtexref hTexRef, float minMipmapLevelClamp, float maxMipmapLevelClamp) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapLevelClamp + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefSetMipmapLevelClamp", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6208 + * return err + * + * cdef CUresult _cuTexRefSetMaxAnisotropy(CUtexref hTexRef, unsigned int maxAniso) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetMaxAnisotropy + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetMaxAnisotropy(CUtexref __pyx_v_hTexRef, unsigned int __pyx_v_maxAniso) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefSetMaxAnisotropy", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6210 + * cdef CUresult _cuTexRefSetMaxAnisotropy(CUtexref hTexRef, unsigned int maxAniso) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefSetMaxAnisotropy + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefSetMaxAnisotropy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6210, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6211 + * global __cuTexRefSetMaxAnisotropy + * cuPythonInit() + * if __cuTexRefSetMaxAnisotropy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetMaxAnisotropy" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMaxAnisotropy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6212 + * cuPythonInit() + * if __cuTexRefSetMaxAnisotropy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetMaxAnisotropy" not found') + * err = ( __cuTexRefSetMaxAnisotropy)(hTexRef, maxAniso) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6213 + * if __cuTexRefSetMaxAnisotropy == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefSetMaxAnisotropy" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefSetMaxAnisotropy)(hTexRef, maxAniso) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefSetMaxAnisotrop}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6213, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6213, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6212 + * cuPythonInit() + * if __cuTexRefSetMaxAnisotropy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetMaxAnisotropy" not found') + * err = ( __cuTexRefSetMaxAnisotropy)(hTexRef, maxAniso) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6211 + * global __cuTexRefSetMaxAnisotropy + * cuPythonInit() + * if __cuTexRefSetMaxAnisotropy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetMaxAnisotropy" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6214 + * with gil: + * raise RuntimeError('Function "cuTexRefSetMaxAnisotropy" not found') + * err = ( __cuTexRefSetMaxAnisotropy)(hTexRef, maxAniso) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMaxAnisotropy)(__pyx_v_hTexRef, __pyx_v_maxAniso); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6214, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6215 + * raise RuntimeError('Function "cuTexRefSetMaxAnisotropy" not found') + * err = ( __cuTexRefSetMaxAnisotropy)(hTexRef, maxAniso) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefSetBorderColor(CUtexref hTexRef, float* pBorderColor) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6208 + * return err + * + * cdef CUresult _cuTexRefSetMaxAnisotropy(CUtexref hTexRef, unsigned int maxAniso) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetMaxAnisotropy + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefSetMaxAnisotropy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6217 + * return err + * + * cdef CUresult _cuTexRefSetBorderColor(CUtexref hTexRef, float* pBorderColor) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetBorderColor + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetBorderColor(CUtexref __pyx_v_hTexRef, float *__pyx_v_pBorderColor) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefSetBorderColor", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6219 + * cdef CUresult _cuTexRefSetBorderColor(CUtexref hTexRef, float* pBorderColor) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefSetBorderColor + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefSetBorderColor == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6219, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6220 + * global __cuTexRefSetBorderColor + * cuPythonInit() + * if __cuTexRefSetBorderColor == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetBorderColor" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetBorderColor == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6221 + * cuPythonInit() + * if __cuTexRefSetBorderColor == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetBorderColor" not found') + * err = ( __cuTexRefSetBorderColor)(hTexRef, pBorderColor) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6222 + * if __cuTexRefSetBorderColor == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefSetBorderColor" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefSetBorderColor)(hTexRef, pBorderColor) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefSetBorderColor}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6222, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6222, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6221 + * cuPythonInit() + * if __cuTexRefSetBorderColor == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetBorderColor" not found') + * err = ( __cuTexRefSetBorderColor)(hTexRef, pBorderColor) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6220 + * global __cuTexRefSetBorderColor + * cuPythonInit() + * if __cuTexRefSetBorderColor == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetBorderColor" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6223 + * with gil: + * raise RuntimeError('Function "cuTexRefSetBorderColor" not found') + * err = ( __cuTexRefSetBorderColor)(hTexRef, pBorderColor) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref, float *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetBorderColor)(__pyx_v_hTexRef, __pyx_v_pBorderColor); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6223, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6224 + * raise RuntimeError('Function "cuTexRefSetBorderColor" not found') + * err = ( __cuTexRefSetBorderColor)(hTexRef, pBorderColor) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefSetFlags(CUtexref hTexRef, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6217 + * return err + * + * cdef CUresult _cuTexRefSetBorderColor(CUtexref hTexRef, float* pBorderColor) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetBorderColor + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefSetBorderColor", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6226 + * return err + * + * cdef CUresult _cuTexRefSetFlags(CUtexref hTexRef, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetFlags + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetFlags(CUtexref __pyx_v_hTexRef, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefSetFlags", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6228 + * cdef CUresult _cuTexRefSetFlags(CUtexref hTexRef, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefSetFlags + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefSetFlags == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6228, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6229 + * global __cuTexRefSetFlags + * cuPythonInit() + * if __cuTexRefSetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetFlags" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFlags == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6230 + * cuPythonInit() + * if __cuTexRefSetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetFlags" not found') + * err = ( __cuTexRefSetFlags)(hTexRef, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6231 + * if __cuTexRefSetFlags == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefSetFlags" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefSetFlags)(hTexRef, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefSetFlags_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6231, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6231, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6230 + * cuPythonInit() + * if __cuTexRefSetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefSetFlags" not found') + * err = ( __cuTexRefSetFlags)(hTexRef, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6229 + * global __cuTexRefSetFlags + * cuPythonInit() + * if __cuTexRefSetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefSetFlags" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6232 + * with gil: + * raise RuntimeError('Function "cuTexRefSetFlags" not found') + * err = ( __cuTexRefSetFlags)(hTexRef, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFlags)(__pyx_v_hTexRef, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6232, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6233 + * raise RuntimeError('Function "cuTexRefSetFlags" not found') + * err = ( __cuTexRefSetFlags)(hTexRef, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefGetAddress_v2(CUdeviceptr* pdptr, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6226 + * return err + * + * cdef CUresult _cuTexRefSetFlags(CUtexref hTexRef, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefSetFlags + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefSetFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6235 + * return err + * + * cdef CUresult _cuTexRefGetAddress_v2(CUdeviceptr* pdptr, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetAddress_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetAddress_v2(CUdeviceptr *__pyx_v_pdptr, CUtexref __pyx_v_hTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefGetAddress_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6237 + * cdef CUresult _cuTexRefGetAddress_v2(CUdeviceptr* pdptr, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefGetAddress_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefGetAddress_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6237, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6238 + * global __cuTexRefGetAddress_v2 + * cuPythonInit() + * if __cuTexRefGetAddress_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetAddress_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddress_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6239 + * cuPythonInit() + * if __cuTexRefGetAddress_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetAddress_v2" not found') + * err = ( __cuTexRefGetAddress_v2)(pdptr, hTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6240 + * if __cuTexRefGetAddress_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefGetAddress_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefGetAddress_v2)(pdptr, hTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefGetAddress_v2_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6240, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6240, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6239 + * cuPythonInit() + * if __cuTexRefGetAddress_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetAddress_v2" not found') + * err = ( __cuTexRefGetAddress_v2)(pdptr, hTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6238 + * global __cuTexRefGetAddress_v2 + * cuPythonInit() + * if __cuTexRefGetAddress_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetAddress_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6241 + * with gil: + * raise RuntimeError('Function "cuTexRefGetAddress_v2" not found') + * err = ( __cuTexRefGetAddress_v2)(pdptr, hTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, CUtexref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddress_v2)(__pyx_v_pdptr, __pyx_v_hTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6241, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6242 + * raise RuntimeError('Function "cuTexRefGetAddress_v2" not found') + * err = ( __cuTexRefGetAddress_v2)(pdptr, hTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefGetArray(CUarray* phArray, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6235 + * return err + * + * cdef CUresult _cuTexRefGetAddress_v2(CUdeviceptr* pdptr, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetAddress_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefGetAddress_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6244 + * return err + * + * cdef CUresult _cuTexRefGetArray(CUarray* phArray, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetArray + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetArray(CUarray *__pyx_v_phArray, CUtexref __pyx_v_hTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefGetArray", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6246 + * cdef CUresult _cuTexRefGetArray(CUarray* phArray, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefGetArray + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefGetArray == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6246, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6247 + * global __cuTexRefGetArray + * cuPythonInit() + * if __cuTexRefGetArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetArray" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetArray == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6248 + * cuPythonInit() + * if __cuTexRefGetArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetArray" not found') + * err = ( __cuTexRefGetArray)(phArray, hTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6249 + * if __cuTexRefGetArray == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefGetArray" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefGetArray)(phArray, hTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefGetArray_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6249, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6249, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6248 + * cuPythonInit() + * if __cuTexRefGetArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetArray" not found') + * err = ( __cuTexRefGetArray)(phArray, hTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6247 + * global __cuTexRefGetArray + * cuPythonInit() + * if __cuTexRefGetArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetArray" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6250 + * with gil: + * raise RuntimeError('Function "cuTexRefGetArray" not found') + * err = ( __cuTexRefGetArray)(phArray, hTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarray *, CUtexref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetArray)(__pyx_v_phArray, __pyx_v_hTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6250, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6251 + * raise RuntimeError('Function "cuTexRefGetArray" not found') + * err = ( __cuTexRefGetArray)(phArray, hTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefGetMipmappedArray(CUmipmappedArray* phMipmappedArray, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6244 + * return err + * + * cdef CUresult _cuTexRefGetArray(CUarray* phArray, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetArray + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefGetArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6253 + * return err + * + * cdef CUresult _cuTexRefGetMipmappedArray(CUmipmappedArray* phMipmappedArray, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmappedArray + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetMipmappedArray(CUmipmappedArray *__pyx_v_phMipmappedArray, CUtexref __pyx_v_hTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefGetMipmappedArray", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6255 + * cdef CUresult _cuTexRefGetMipmappedArray(CUmipmappedArray* phMipmappedArray, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefGetMipmappedArray + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefGetMipmappedArray == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6255, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6256 + * global __cuTexRefGetMipmappedArray + * cuPythonInit() + * if __cuTexRefGetMipmappedArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmappedArray" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmappedArray == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6257 + * cuPythonInit() + * if __cuTexRefGetMipmappedArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetMipmappedArray" not found') + * err = ( __cuTexRefGetMipmappedArray)(phMipmappedArray, hTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6258 + * if __cuTexRefGetMipmappedArray == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmappedArray" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefGetMipmappedArray)(phMipmappedArray, hTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefGetMipmappedArr}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6258, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6258, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6257 + * cuPythonInit() + * if __cuTexRefGetMipmappedArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetMipmappedArray" not found') + * err = ( __cuTexRefGetMipmappedArray)(phMipmappedArray, hTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6256 + * global __cuTexRefGetMipmappedArray + * cuPythonInit() + * if __cuTexRefGetMipmappedArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmappedArray" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6259 + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmappedArray" not found') + * err = ( __cuTexRefGetMipmappedArray)(phMipmappedArray, hTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmipmappedArray *, CUtexref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmappedArray)(__pyx_v_phMipmappedArray, __pyx_v_hTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6259, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6260 + * raise RuntimeError('Function "cuTexRefGetMipmappedArray" not found') + * err = ( __cuTexRefGetMipmappedArray)(phMipmappedArray, hTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefGetAddressMode(CUaddress_mode* pam, CUtexref hTexRef, int dim) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6253 + * return err + * + * cdef CUresult _cuTexRefGetMipmappedArray(CUmipmappedArray* phMipmappedArray, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmappedArray + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefGetMipmappedArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6262 + * return err + * + * cdef CUresult _cuTexRefGetAddressMode(CUaddress_mode* pam, CUtexref hTexRef, int dim) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetAddressMode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetAddressMode(CUaddress_mode *__pyx_v_pam, CUtexref __pyx_v_hTexRef, int __pyx_v_dim) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefGetAddressMode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6264 + * cdef CUresult _cuTexRefGetAddressMode(CUaddress_mode* pam, CUtexref hTexRef, int dim) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefGetAddressMode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefGetAddressMode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6264, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6265 + * global __cuTexRefGetAddressMode + * cuPythonInit() + * if __cuTexRefGetAddressMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetAddressMode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddressMode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6266 + * cuPythonInit() + * if __cuTexRefGetAddressMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetAddressMode" not found') + * err = ( __cuTexRefGetAddressMode)(pam, hTexRef, dim) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6267 + * if __cuTexRefGetAddressMode == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefGetAddressMode" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefGetAddressMode)(pam, hTexRef, dim) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefGetAddressMode}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6267, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6267, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6266 + * cuPythonInit() + * if __cuTexRefGetAddressMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetAddressMode" not found') + * err = ( __cuTexRefGetAddressMode)(pam, hTexRef, dim) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6265 + * global __cuTexRefGetAddressMode + * cuPythonInit() + * if __cuTexRefGetAddressMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetAddressMode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6268 + * with gil: + * raise RuntimeError('Function "cuTexRefGetAddressMode" not found') + * err = ( __cuTexRefGetAddressMode)(pam, hTexRef, dim) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUaddress_mode *, CUtexref, int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddressMode)(__pyx_v_pam, __pyx_v_hTexRef, __pyx_v_dim); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6268, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6269 + * raise RuntimeError('Function "cuTexRefGetAddressMode" not found') + * err = ( __cuTexRefGetAddressMode)(pam, hTexRef, dim) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefGetFilterMode(CUfilter_mode* pfm, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6262 + * return err + * + * cdef CUresult _cuTexRefGetAddressMode(CUaddress_mode* pam, CUtexref hTexRef, int dim) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetAddressMode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefGetAddressMode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6271 + * return err + * + * cdef CUresult _cuTexRefGetFilterMode(CUfilter_mode* pfm, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetFilterMode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetFilterMode(CUfilter_mode *__pyx_v_pfm, CUtexref __pyx_v_hTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefGetFilterMode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6273 + * cdef CUresult _cuTexRefGetFilterMode(CUfilter_mode* pfm, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefGetFilterMode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefGetFilterMode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6273, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6274 + * global __cuTexRefGetFilterMode + * cuPythonInit() + * if __cuTexRefGetFilterMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetFilterMode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFilterMode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6275 + * cuPythonInit() + * if __cuTexRefGetFilterMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetFilterMode" not found') + * err = ( __cuTexRefGetFilterMode)(pfm, hTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6276 + * if __cuTexRefGetFilterMode == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefGetFilterMode" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefGetFilterMode)(pfm, hTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefGetFilterMode_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6276, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6276, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6275 + * cuPythonInit() + * if __cuTexRefGetFilterMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetFilterMode" not found') + * err = ( __cuTexRefGetFilterMode)(pfm, hTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6274 + * global __cuTexRefGetFilterMode + * cuPythonInit() + * if __cuTexRefGetFilterMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetFilterMode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6277 + * with gil: + * raise RuntimeError('Function "cuTexRefGetFilterMode" not found') + * err = ( __cuTexRefGetFilterMode)(pfm, hTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfilter_mode *, CUtexref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFilterMode)(__pyx_v_pfm, __pyx_v_hTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6277, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6278 + * raise RuntimeError('Function "cuTexRefGetFilterMode" not found') + * err = ( __cuTexRefGetFilterMode)(pfm, hTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefGetFormat(CUarray_format* pFormat, int* pNumChannels, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6271 + * return err + * + * cdef CUresult _cuTexRefGetFilterMode(CUfilter_mode* pfm, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetFilterMode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefGetFilterMode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6280 + * return err + * + * cdef CUresult _cuTexRefGetFormat(CUarray_format* pFormat, int* pNumChannels, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetFormat + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetFormat(CUarray_format *__pyx_v_pFormat, int *__pyx_v_pNumChannels, CUtexref __pyx_v_hTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefGetFormat", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6282 + * cdef CUresult _cuTexRefGetFormat(CUarray_format* pFormat, int* pNumChannels, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefGetFormat + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefGetFormat == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6282, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6283 + * global __cuTexRefGetFormat + * cuPythonInit() + * if __cuTexRefGetFormat == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetFormat" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFormat == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6284 + * cuPythonInit() + * if __cuTexRefGetFormat == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetFormat" not found') + * err = ( __cuTexRefGetFormat)(pFormat, pNumChannels, hTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6285 + * if __cuTexRefGetFormat == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefGetFormat" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefGetFormat)(pFormat, pNumChannels, hTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefGetFormat_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6285, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6285, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6284 + * cuPythonInit() + * if __cuTexRefGetFormat == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetFormat" not found') + * err = ( __cuTexRefGetFormat)(pFormat, pNumChannels, hTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6283 + * global __cuTexRefGetFormat + * cuPythonInit() + * if __cuTexRefGetFormat == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetFormat" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6286 + * with gil: + * raise RuntimeError('Function "cuTexRefGetFormat" not found') + * err = ( __cuTexRefGetFormat)(pFormat, pNumChannels, hTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarray_format *, int *, CUtexref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFormat)(__pyx_v_pFormat, __pyx_v_pNumChannels, __pyx_v_hTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6286, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6287 + * raise RuntimeError('Function "cuTexRefGetFormat" not found') + * err = ( __cuTexRefGetFormat)(pFormat, pNumChannels, hTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefGetMipmapFilterMode(CUfilter_mode* pfm, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6280 + * return err + * + * cdef CUresult _cuTexRefGetFormat(CUarray_format* pFormat, int* pNumChannels, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetFormat + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefGetFormat", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6289 + * return err + * + * cdef CUresult _cuTexRefGetMipmapFilterMode(CUfilter_mode* pfm, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapFilterMode + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetMipmapFilterMode(CUfilter_mode *__pyx_v_pfm, CUtexref __pyx_v_hTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefGetMipmapFilterMode", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6291 + * cdef CUresult _cuTexRefGetMipmapFilterMode(CUfilter_mode* pfm, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefGetMipmapFilterMode + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefGetMipmapFilterMode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6291, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6292 + * global __cuTexRefGetMipmapFilterMode + * cuPythonInit() + * if __cuTexRefGetMipmapFilterMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmapFilterMode" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapFilterMode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6293 + * cuPythonInit() + * if __cuTexRefGetMipmapFilterMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetMipmapFilterMode" not found') + * err = ( __cuTexRefGetMipmapFilterMode)(pfm, hTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6294 + * if __cuTexRefGetMipmapFilterMode == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmapFilterMode" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefGetMipmapFilterMode)(pfm, hTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefGetMipmapFilter}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6294, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6294, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6293 + * cuPythonInit() + * if __cuTexRefGetMipmapFilterMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetMipmapFilterMode" not found') + * err = ( __cuTexRefGetMipmapFilterMode)(pfm, hTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6292 + * global __cuTexRefGetMipmapFilterMode + * cuPythonInit() + * if __cuTexRefGetMipmapFilterMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmapFilterMode" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6295 + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmapFilterMode" not found') + * err = ( __cuTexRefGetMipmapFilterMode)(pfm, hTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUfilter_mode *, CUtexref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapFilterMode)(__pyx_v_pfm, __pyx_v_hTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6295, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6296 + * raise RuntimeError('Function "cuTexRefGetMipmapFilterMode" not found') + * err = ( __cuTexRefGetMipmapFilterMode)(pfm, hTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefGetMipmapLevelBias(float* pbias, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6289 + * return err + * + * cdef CUresult _cuTexRefGetMipmapFilterMode(CUfilter_mode* pfm, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapFilterMode + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefGetMipmapFilterMode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6298 + * return err + * + * cdef CUresult _cuTexRefGetMipmapLevelBias(float* pbias, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapLevelBias + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetMipmapLevelBias(float *__pyx_v_pbias, CUtexref __pyx_v_hTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefGetMipmapLevelBias", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6300 + * cdef CUresult _cuTexRefGetMipmapLevelBias(float* pbias, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefGetMipmapLevelBias + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefGetMipmapLevelBias == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6300, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6301 + * global __cuTexRefGetMipmapLevelBias + * cuPythonInit() + * if __cuTexRefGetMipmapLevelBias == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmapLevelBias" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelBias == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6302 + * cuPythonInit() + * if __cuTexRefGetMipmapLevelBias == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetMipmapLevelBias" not found') + * err = ( __cuTexRefGetMipmapLevelBias)(pbias, hTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6303 + * if __cuTexRefGetMipmapLevelBias == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmapLevelBias" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefGetMipmapLevelBias)(pbias, hTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefGetMipmapLevelB}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6303, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6303, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6302 + * cuPythonInit() + * if __cuTexRefGetMipmapLevelBias == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetMipmapLevelBias" not found') + * err = ( __cuTexRefGetMipmapLevelBias)(pbias, hTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6301 + * global __cuTexRefGetMipmapLevelBias + * cuPythonInit() + * if __cuTexRefGetMipmapLevelBias == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmapLevelBias" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6304 + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmapLevelBias" not found') + * err = ( __cuTexRefGetMipmapLevelBias)(pbias, hTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(float *, CUtexref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelBias)(__pyx_v_pbias, __pyx_v_hTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6304, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6305 + * raise RuntimeError('Function "cuTexRefGetMipmapLevelBias" not found') + * err = ( __cuTexRefGetMipmapLevelBias)(pbias, hTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefGetMipmapLevelClamp(float* pminMipmapLevelClamp, float* pmaxMipmapLevelClamp, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6298 + * return err + * + * cdef CUresult _cuTexRefGetMipmapLevelBias(float* pbias, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapLevelBias + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefGetMipmapLevelBias", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6307 + * return err + * + * cdef CUresult _cuTexRefGetMipmapLevelClamp(float* pminMipmapLevelClamp, float* pmaxMipmapLevelClamp, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapLevelClamp + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetMipmapLevelClamp(float *__pyx_v_pminMipmapLevelClamp, float *__pyx_v_pmaxMipmapLevelClamp, CUtexref __pyx_v_hTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefGetMipmapLevelClamp", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6309 + * cdef CUresult _cuTexRefGetMipmapLevelClamp(float* pminMipmapLevelClamp, float* pmaxMipmapLevelClamp, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefGetMipmapLevelClamp + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefGetMipmapLevelClamp == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6309, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6310 + * global __cuTexRefGetMipmapLevelClamp + * cuPythonInit() + * if __cuTexRefGetMipmapLevelClamp == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmapLevelClamp" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelClamp == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6311 + * cuPythonInit() + * if __cuTexRefGetMipmapLevelClamp == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetMipmapLevelClamp" not found') + * err = ( __cuTexRefGetMipmapLevelClamp)(pminMipmapLevelClamp, pmaxMipmapLevelClamp, hTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6312 + * if __cuTexRefGetMipmapLevelClamp == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmapLevelClamp" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefGetMipmapLevelClamp)(pminMipmapLevelClamp, pmaxMipmapLevelClamp, hTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefGetMipmapLevelC}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6312, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6312, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6311 + * cuPythonInit() + * if __cuTexRefGetMipmapLevelClamp == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetMipmapLevelClamp" not found') + * err = ( __cuTexRefGetMipmapLevelClamp)(pminMipmapLevelClamp, pmaxMipmapLevelClamp, hTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6310 + * global __cuTexRefGetMipmapLevelClamp + * cuPythonInit() + * if __cuTexRefGetMipmapLevelClamp == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmapLevelClamp" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6313 + * with gil: + * raise RuntimeError('Function "cuTexRefGetMipmapLevelClamp" not found') + * err = ( __cuTexRefGetMipmapLevelClamp)(pminMipmapLevelClamp, pmaxMipmapLevelClamp, hTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(float *, float *, CUtexref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelClamp)(__pyx_v_pminMipmapLevelClamp, __pyx_v_pmaxMipmapLevelClamp, __pyx_v_hTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6313, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6314 + * raise RuntimeError('Function "cuTexRefGetMipmapLevelClamp" not found') + * err = ( __cuTexRefGetMipmapLevelClamp)(pminMipmapLevelClamp, pmaxMipmapLevelClamp, hTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefGetMaxAnisotropy(int* pmaxAniso, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6307 + * return err + * + * cdef CUresult _cuTexRefGetMipmapLevelClamp(float* pminMipmapLevelClamp, float* pmaxMipmapLevelClamp, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapLevelClamp + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefGetMipmapLevelClamp", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6316 + * return err + * + * cdef CUresult _cuTexRefGetMaxAnisotropy(int* pmaxAniso, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetMaxAnisotropy + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetMaxAnisotropy(int *__pyx_v_pmaxAniso, CUtexref __pyx_v_hTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefGetMaxAnisotropy", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6318 + * cdef CUresult _cuTexRefGetMaxAnisotropy(int* pmaxAniso, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefGetMaxAnisotropy + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefGetMaxAnisotropy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6318, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6319 + * global __cuTexRefGetMaxAnisotropy + * cuPythonInit() + * if __cuTexRefGetMaxAnisotropy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetMaxAnisotropy" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMaxAnisotropy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6320 + * cuPythonInit() + * if __cuTexRefGetMaxAnisotropy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetMaxAnisotropy" not found') + * err = ( __cuTexRefGetMaxAnisotropy)(pmaxAniso, hTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6321 + * if __cuTexRefGetMaxAnisotropy == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefGetMaxAnisotropy" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefGetMaxAnisotropy)(pmaxAniso, hTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefGetMaxAnisotrop}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6321, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6321, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6320 + * cuPythonInit() + * if __cuTexRefGetMaxAnisotropy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetMaxAnisotropy" not found') + * err = ( __cuTexRefGetMaxAnisotropy)(pmaxAniso, hTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6319 + * global __cuTexRefGetMaxAnisotropy + * cuPythonInit() + * if __cuTexRefGetMaxAnisotropy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetMaxAnisotropy" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6322 + * with gil: + * raise RuntimeError('Function "cuTexRefGetMaxAnisotropy" not found') + * err = ( __cuTexRefGetMaxAnisotropy)(pmaxAniso, hTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, CUtexref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMaxAnisotropy)(__pyx_v_pmaxAniso, __pyx_v_hTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6322, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6323 + * raise RuntimeError('Function "cuTexRefGetMaxAnisotropy" not found') + * err = ( __cuTexRefGetMaxAnisotropy)(pmaxAniso, hTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefGetBorderColor(float* pBorderColor, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6316 + * return err + * + * cdef CUresult _cuTexRefGetMaxAnisotropy(int* pmaxAniso, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetMaxAnisotropy + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefGetMaxAnisotropy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6325 + * return err + * + * cdef CUresult _cuTexRefGetBorderColor(float* pBorderColor, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetBorderColor + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetBorderColor(float *__pyx_v_pBorderColor, CUtexref __pyx_v_hTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefGetBorderColor", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6327 + * cdef CUresult _cuTexRefGetBorderColor(float* pBorderColor, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefGetBorderColor + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefGetBorderColor == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6327, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6328 + * global __cuTexRefGetBorderColor + * cuPythonInit() + * if __cuTexRefGetBorderColor == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetBorderColor" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetBorderColor == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6329 + * cuPythonInit() + * if __cuTexRefGetBorderColor == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetBorderColor" not found') + * err = ( __cuTexRefGetBorderColor)(pBorderColor, hTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6330 + * if __cuTexRefGetBorderColor == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefGetBorderColor" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefGetBorderColor)(pBorderColor, hTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefGetBorderColor}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6330, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6330, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6329 + * cuPythonInit() + * if __cuTexRefGetBorderColor == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetBorderColor" not found') + * err = ( __cuTexRefGetBorderColor)(pBorderColor, hTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6328 + * global __cuTexRefGetBorderColor + * cuPythonInit() + * if __cuTexRefGetBorderColor == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetBorderColor" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6331 + * with gil: + * raise RuntimeError('Function "cuTexRefGetBorderColor" not found') + * err = ( __cuTexRefGetBorderColor)(pBorderColor, hTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(float *, CUtexref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetBorderColor)(__pyx_v_pBorderColor, __pyx_v_hTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6331, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6332 + * raise RuntimeError('Function "cuTexRefGetBorderColor" not found') + * err = ( __cuTexRefGetBorderColor)(pBorderColor, hTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefGetFlags(unsigned int* pFlags, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6325 + * return err + * + * cdef CUresult _cuTexRefGetBorderColor(float* pBorderColor, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetBorderColor + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefGetBorderColor", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6334 + * return err + * + * cdef CUresult _cuTexRefGetFlags(unsigned int* pFlags, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetFlags + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetFlags(unsigned int *__pyx_v_pFlags, CUtexref __pyx_v_hTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefGetFlags", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6336 + * cdef CUresult _cuTexRefGetFlags(unsigned int* pFlags, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefGetFlags + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefGetFlags == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6336, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6337 + * global __cuTexRefGetFlags + * cuPythonInit() + * if __cuTexRefGetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetFlags" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFlags == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6338 + * cuPythonInit() + * if __cuTexRefGetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetFlags" not found') + * err = ( __cuTexRefGetFlags)(pFlags, hTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6339 + * if __cuTexRefGetFlags == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefGetFlags" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefGetFlags)(pFlags, hTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefGetFlags_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6339, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6339, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6338 + * cuPythonInit() + * if __cuTexRefGetFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefGetFlags" not found') + * err = ( __cuTexRefGetFlags)(pFlags, hTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6337 + * global __cuTexRefGetFlags + * cuPythonInit() + * if __cuTexRefGetFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefGetFlags" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6340 + * with gil: + * raise RuntimeError('Function "cuTexRefGetFlags" not found') + * err = ( __cuTexRefGetFlags)(pFlags, hTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(unsigned int *, CUtexref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFlags)(__pyx_v_pFlags, __pyx_v_hTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6340, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6341 + * raise RuntimeError('Function "cuTexRefGetFlags" not found') + * err = ( __cuTexRefGetFlags)(pFlags, hTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefCreate(CUtexref* pTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6334 + * return err + * + * cdef CUresult _cuTexRefGetFlags(unsigned int* pFlags, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefGetFlags + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefGetFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6343 + * return err + * + * cdef CUresult _cuTexRefCreate(CUtexref* pTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefCreate(CUtexref *__pyx_v_pTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6345 + * cdef CUresult _cuTexRefCreate(CUtexref* pTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6345, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6346 + * global __cuTexRefCreate + * cuPythonInit() + * if __cuTexRefCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6347 + * cuPythonInit() + * if __cuTexRefCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefCreate" not found') + * err = ( __cuTexRefCreate)(pTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6348 + * if __cuTexRefCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefCreate)(pTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefCreate_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6348, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6348, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6347 + * cuPythonInit() + * if __cuTexRefCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefCreate" not found') + * err = ( __cuTexRefCreate)(pTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6346 + * global __cuTexRefCreate + * cuPythonInit() + * if __cuTexRefCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6349 + * with gil: + * raise RuntimeError('Function "cuTexRefCreate" not found') + * err = ( __cuTexRefCreate)(pTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefCreate)(__pyx_v_pTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6349, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6350 + * raise RuntimeError('Function "cuTexRefCreate" not found') + * err = ( __cuTexRefCreate)(pTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexRefDestroy(CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6343 + * return err + * + * cdef CUresult _cuTexRefCreate(CUtexref* pTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6352 + * return err + * + * cdef CUresult _cuTexRefDestroy(CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefDestroy + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefDestroy(CUtexref __pyx_v_hTexRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexRefDestroy", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6354 + * cdef CUresult _cuTexRefDestroy(CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexRefDestroy + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexRefDestroy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6354, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6355 + * global __cuTexRefDestroy + * cuPythonInit() + * if __cuTexRefDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefDestroy" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefDestroy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6356 + * cuPythonInit() + * if __cuTexRefDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefDestroy" not found') + * err = ( __cuTexRefDestroy)(hTexRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6357 + * if __cuTexRefDestroy == NULL: + * with gil: + * raise RuntimeError('Function "cuTexRefDestroy" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexRefDestroy)(hTexRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexRefDestroy_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6357, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6357, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6356 + * cuPythonInit() + * if __cuTexRefDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexRefDestroy" not found') + * err = ( __cuTexRefDestroy)(hTexRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6355 + * global __cuTexRefDestroy + * cuPythonInit() + * if __cuTexRefDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexRefDestroy" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6358 + * with gil: + * raise RuntimeError('Function "cuTexRefDestroy" not found') + * err = ( __cuTexRefDestroy)(hTexRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefDestroy)(__pyx_v_hTexRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6358, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6359 + * raise RuntimeError('Function "cuTexRefDestroy" not found') + * err = ( __cuTexRefDestroy)(hTexRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuSurfRefSetArray(CUsurfref hSurfRef, CUarray hArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6352 + * return err + * + * cdef CUresult _cuTexRefDestroy(CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexRefDestroy + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexRefDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6361 + * return err + * + * cdef CUresult _cuSurfRefSetArray(CUsurfref hSurfRef, CUarray hArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuSurfRefSetArray + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuSurfRefSetArray(CUsurfref __pyx_v_hSurfRef, CUarray __pyx_v_hArray, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuSurfRefSetArray", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6363 + * cdef CUresult _cuSurfRefSetArray(CUsurfref hSurfRef, CUarray hArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuSurfRefSetArray + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuSurfRefSetArray == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6363, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6364 + * global __cuSurfRefSetArray + * cuPythonInit() + * if __cuSurfRefSetArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuSurfRefSetArray" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefSetArray == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6365 + * cuPythonInit() + * if __cuSurfRefSetArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuSurfRefSetArray" not found') + * err = ( __cuSurfRefSetArray)(hSurfRef, hArray, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6366 + * if __cuSurfRefSetArray == NULL: + * with gil: + * raise RuntimeError('Function "cuSurfRefSetArray" not found') # <<<<<<<<<<<<<< + * err = ( __cuSurfRefSetArray)(hSurfRef, hArray, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuSurfRefSetArray_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6366, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6366, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6365 + * cuPythonInit() + * if __cuSurfRefSetArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuSurfRefSetArray" not found') + * err = ( __cuSurfRefSetArray)(hSurfRef, hArray, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6364 + * global __cuSurfRefSetArray + * cuPythonInit() + * if __cuSurfRefSetArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuSurfRefSetArray" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6367 + * with gil: + * raise RuntimeError('Function "cuSurfRefSetArray" not found') + * err = ( __cuSurfRefSetArray)(hSurfRef, hArray, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUsurfref, CUarray, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefSetArray)(__pyx_v_hSurfRef, __pyx_v_hArray, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6367, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6368 + * raise RuntimeError('Function "cuSurfRefSetArray" not found') + * err = ( __cuSurfRefSetArray)(hSurfRef, hArray, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuSurfRefGetArray(CUarray* phArray, CUsurfref hSurfRef) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6361 + * return err + * + * cdef CUresult _cuSurfRefSetArray(CUsurfref hSurfRef, CUarray hArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuSurfRefSetArray + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuSurfRefSetArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6370 + * return err + * + * cdef CUresult _cuSurfRefGetArray(CUarray* phArray, CUsurfref hSurfRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuSurfRefGetArray + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuSurfRefGetArray(CUarray *__pyx_v_phArray, CUsurfref __pyx_v_hSurfRef) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuSurfRefGetArray", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6372 + * cdef CUresult _cuSurfRefGetArray(CUarray* phArray, CUsurfref hSurfRef) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuSurfRefGetArray + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuSurfRefGetArray == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6372, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6373 + * global __cuSurfRefGetArray + * cuPythonInit() + * if __cuSurfRefGetArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuSurfRefGetArray" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefGetArray == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6374 + * cuPythonInit() + * if __cuSurfRefGetArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuSurfRefGetArray" not found') + * err = ( __cuSurfRefGetArray)(phArray, hSurfRef) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6375 + * if __cuSurfRefGetArray == NULL: + * with gil: + * raise RuntimeError('Function "cuSurfRefGetArray" not found') # <<<<<<<<<<<<<< + * err = ( __cuSurfRefGetArray)(phArray, hSurfRef) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuSurfRefGetArray_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6375, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6375, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6374 + * cuPythonInit() + * if __cuSurfRefGetArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuSurfRefGetArray" not found') + * err = ( __cuSurfRefGetArray)(phArray, hSurfRef) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6373 + * global __cuSurfRefGetArray + * cuPythonInit() + * if __cuSurfRefGetArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuSurfRefGetArray" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6376 + * with gil: + * raise RuntimeError('Function "cuSurfRefGetArray" not found') + * err = ( __cuSurfRefGetArray)(phArray, hSurfRef) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarray *, CUsurfref))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefGetArray)(__pyx_v_phArray, __pyx_v_hSurfRef); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6376, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6377 + * raise RuntimeError('Function "cuSurfRefGetArray" not found') + * err = ( __cuSurfRefGetArray)(phArray, hSurfRef) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexObjectCreate(CUtexObject* pTexObject, const CUDA_RESOURCE_DESC* pResDesc, const CUDA_TEXTURE_DESC* pTexDesc, const CUDA_RESOURCE_VIEW_DESC* pResViewDesc) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6370 + * return err + * + * cdef CUresult _cuSurfRefGetArray(CUarray* phArray, CUsurfref hSurfRef) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuSurfRefGetArray + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuSurfRefGetArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6379 + * return err + * + * cdef CUresult _cuTexObjectCreate(CUtexObject* pTexObject, const CUDA_RESOURCE_DESC* pResDesc, const CUDA_TEXTURE_DESC* pTexDesc, const CUDA_RESOURCE_VIEW_DESC* pResViewDesc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexObjectCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexObjectCreate(CUtexObject *__pyx_v_pTexObject, CUDA_RESOURCE_DESC const *__pyx_v_pResDesc, CUDA_TEXTURE_DESC const *__pyx_v_pTexDesc, CUDA_RESOURCE_VIEW_DESC const *__pyx_v_pResViewDesc) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexObjectCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6381 + * cdef CUresult _cuTexObjectCreate(CUtexObject* pTexObject, const CUDA_RESOURCE_DESC* pResDesc, const CUDA_TEXTURE_DESC* pTexDesc, const CUDA_RESOURCE_VIEW_DESC* pResViewDesc) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexObjectCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexObjectCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6381, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6382 + * global __cuTexObjectCreate + * cuPythonInit() + * if __cuTexObjectCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexObjectCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6383 + * cuPythonInit() + * if __cuTexObjectCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexObjectCreate" not found') + * err = ( __cuTexObjectCreate)(pTexObject, pResDesc, pTexDesc, pResViewDesc) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6384 + * if __cuTexObjectCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuTexObjectCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexObjectCreate)(pTexObject, pResDesc, pTexDesc, pResViewDesc) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexObjectCreate_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6384, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6384, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6383 + * cuPythonInit() + * if __cuTexObjectCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexObjectCreate" not found') + * err = ( __cuTexObjectCreate)(pTexObject, pResDesc, pTexDesc, pResViewDesc) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6382 + * global __cuTexObjectCreate + * cuPythonInit() + * if __cuTexObjectCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexObjectCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6385 + * with gil: + * raise RuntimeError('Function "cuTexObjectCreate" not found') + * err = ( __cuTexObjectCreate)(pTexObject, pResDesc, pTexDesc, pResViewDesc) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexObject *, CUDA_RESOURCE_DESC const *, CUDA_TEXTURE_DESC const *, CUDA_RESOURCE_VIEW_DESC const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectCreate)(__pyx_v_pTexObject, __pyx_v_pResDesc, __pyx_v_pTexDesc, __pyx_v_pResViewDesc); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6385, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6386 + * raise RuntimeError('Function "cuTexObjectCreate" not found') + * err = ( __cuTexObjectCreate)(pTexObject, pResDesc, pTexDesc, pResViewDesc) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexObjectDestroy(CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6379 + * return err + * + * cdef CUresult _cuTexObjectCreate(CUtexObject* pTexObject, const CUDA_RESOURCE_DESC* pResDesc, const CUDA_TEXTURE_DESC* pTexDesc, const CUDA_RESOURCE_VIEW_DESC* pResViewDesc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexObjectCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexObjectCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6388 + * return err + * + * cdef CUresult _cuTexObjectDestroy(CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexObjectDestroy + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexObjectDestroy(CUtexObject __pyx_v_texObject) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexObjectDestroy", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6390 + * cdef CUresult _cuTexObjectDestroy(CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexObjectDestroy + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexObjectDestroy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6390, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6391 + * global __cuTexObjectDestroy + * cuPythonInit() + * if __cuTexObjectDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexObjectDestroy" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectDestroy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6392 + * cuPythonInit() + * if __cuTexObjectDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexObjectDestroy" not found') + * err = ( __cuTexObjectDestroy)(texObject) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6393 + * if __cuTexObjectDestroy == NULL: + * with gil: + * raise RuntimeError('Function "cuTexObjectDestroy" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexObjectDestroy)(texObject) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexObjectDestroy_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6393, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6393, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6392 + * cuPythonInit() + * if __cuTexObjectDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexObjectDestroy" not found') + * err = ( __cuTexObjectDestroy)(texObject) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6391 + * global __cuTexObjectDestroy + * cuPythonInit() + * if __cuTexObjectDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexObjectDestroy" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6394 + * with gil: + * raise RuntimeError('Function "cuTexObjectDestroy" not found') + * err = ( __cuTexObjectDestroy)(texObject) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtexObject))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectDestroy)(__pyx_v_texObject); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6394, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6395 + * raise RuntimeError('Function "cuTexObjectDestroy" not found') + * err = ( __cuTexObjectDestroy)(texObject) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexObjectGetResourceDesc(CUDA_RESOURCE_DESC* pResDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6388 + * return err + * + * cdef CUresult _cuTexObjectDestroy(CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexObjectDestroy + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexObjectDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6397 + * return err + * + * cdef CUresult _cuTexObjectGetResourceDesc(CUDA_RESOURCE_DESC* pResDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexObjectGetResourceDesc + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexObjectGetResourceDesc(CUDA_RESOURCE_DESC *__pyx_v_pResDesc, CUtexObject __pyx_v_texObject) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexObjectGetResourceDesc", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6399 + * cdef CUresult _cuTexObjectGetResourceDesc(CUDA_RESOURCE_DESC* pResDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexObjectGetResourceDesc + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexObjectGetResourceDesc == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6399, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6400 + * global __cuTexObjectGetResourceDesc + * cuPythonInit() + * if __cuTexObjectGetResourceDesc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexObjectGetResourceDesc" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceDesc == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6401 + * cuPythonInit() + * if __cuTexObjectGetResourceDesc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexObjectGetResourceDesc" not found') + * err = ( __cuTexObjectGetResourceDesc)(pResDesc, texObject) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6402 + * if __cuTexObjectGetResourceDesc == NULL: + * with gil: + * raise RuntimeError('Function "cuTexObjectGetResourceDesc" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexObjectGetResourceDesc)(pResDesc, texObject) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexObjectGetResourceD}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6402, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6402, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6401 + * cuPythonInit() + * if __cuTexObjectGetResourceDesc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexObjectGetResourceDesc" not found') + * err = ( __cuTexObjectGetResourceDesc)(pResDesc, texObject) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6400 + * global __cuTexObjectGetResourceDesc + * cuPythonInit() + * if __cuTexObjectGetResourceDesc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexObjectGetResourceDesc" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6403 + * with gil: + * raise RuntimeError('Function "cuTexObjectGetResourceDesc" not found') + * err = ( __cuTexObjectGetResourceDesc)(pResDesc, texObject) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_RESOURCE_DESC *, CUtexObject))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceDesc)(__pyx_v_pResDesc, __pyx_v_texObject); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6403, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6404 + * raise RuntimeError('Function "cuTexObjectGetResourceDesc" not found') + * err = ( __cuTexObjectGetResourceDesc)(pResDesc, texObject) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexObjectGetTextureDesc(CUDA_TEXTURE_DESC* pTexDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6397 + * return err + * + * cdef CUresult _cuTexObjectGetResourceDesc(CUDA_RESOURCE_DESC* pResDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexObjectGetResourceDesc + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexObjectGetResourceDesc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6406 + * return err + * + * cdef CUresult _cuTexObjectGetTextureDesc(CUDA_TEXTURE_DESC* pTexDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexObjectGetTextureDesc + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexObjectGetTextureDesc(CUDA_TEXTURE_DESC *__pyx_v_pTexDesc, CUtexObject __pyx_v_texObject) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexObjectGetTextureDesc", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6408 + * cdef CUresult _cuTexObjectGetTextureDesc(CUDA_TEXTURE_DESC* pTexDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexObjectGetTextureDesc + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexObjectGetTextureDesc == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6408, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6409 + * global __cuTexObjectGetTextureDesc + * cuPythonInit() + * if __cuTexObjectGetTextureDesc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexObjectGetTextureDesc" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetTextureDesc == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6410 + * cuPythonInit() + * if __cuTexObjectGetTextureDesc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexObjectGetTextureDesc" not found') + * err = ( __cuTexObjectGetTextureDesc)(pTexDesc, texObject) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6411 + * if __cuTexObjectGetTextureDesc == NULL: + * with gil: + * raise RuntimeError('Function "cuTexObjectGetTextureDesc" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexObjectGetTextureDesc)(pTexDesc, texObject) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexObjectGetTextureDe}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6411, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6411, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6410 + * cuPythonInit() + * if __cuTexObjectGetTextureDesc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexObjectGetTextureDesc" not found') + * err = ( __cuTexObjectGetTextureDesc)(pTexDesc, texObject) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6409 + * global __cuTexObjectGetTextureDesc + * cuPythonInit() + * if __cuTexObjectGetTextureDesc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexObjectGetTextureDesc" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6412 + * with gil: + * raise RuntimeError('Function "cuTexObjectGetTextureDesc" not found') + * err = ( __cuTexObjectGetTextureDesc)(pTexDesc, texObject) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_TEXTURE_DESC *, CUtexObject))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetTextureDesc)(__pyx_v_pTexDesc, __pyx_v_texObject); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6412, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6413 + * raise RuntimeError('Function "cuTexObjectGetTextureDesc" not found') + * err = ( __cuTexObjectGetTextureDesc)(pTexDesc, texObject) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTexObjectGetResourceViewDesc(CUDA_RESOURCE_VIEW_DESC* pResViewDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6406 + * return err + * + * cdef CUresult _cuTexObjectGetTextureDesc(CUDA_TEXTURE_DESC* pTexDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexObjectGetTextureDesc + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexObjectGetTextureDesc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6415 + * return err + * + * cdef CUresult _cuTexObjectGetResourceViewDesc(CUDA_RESOURCE_VIEW_DESC* pResViewDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexObjectGetResourceViewDesc + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexObjectGetResourceViewDesc(CUDA_RESOURCE_VIEW_DESC *__pyx_v_pResViewDesc, CUtexObject __pyx_v_texObject) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTexObjectGetResourceViewDesc", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6417 + * cdef CUresult _cuTexObjectGetResourceViewDesc(CUDA_RESOURCE_VIEW_DESC* pResViewDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTexObjectGetResourceViewDesc + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTexObjectGetResourceViewDesc == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6417, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6418 + * global __cuTexObjectGetResourceViewDesc + * cuPythonInit() + * if __cuTexObjectGetResourceViewDesc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexObjectGetResourceViewDesc" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceViewDesc == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6419 + * cuPythonInit() + * if __cuTexObjectGetResourceViewDesc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexObjectGetResourceViewDesc" not found') + * err = ( __cuTexObjectGetResourceViewDesc)(pResViewDesc, texObject) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6420 + * if __cuTexObjectGetResourceViewDesc == NULL: + * with gil: + * raise RuntimeError('Function "cuTexObjectGetResourceViewDesc" not found') # <<<<<<<<<<<<<< + * err = ( __cuTexObjectGetResourceViewDesc)(pResViewDesc, texObject) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTexObjectGetResourceV}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6420, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6420, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6419 + * cuPythonInit() + * if __cuTexObjectGetResourceViewDesc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTexObjectGetResourceViewDesc" not found') + * err = ( __cuTexObjectGetResourceViewDesc)(pResViewDesc, texObject) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6418 + * global __cuTexObjectGetResourceViewDesc + * cuPythonInit() + * if __cuTexObjectGetResourceViewDesc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTexObjectGetResourceViewDesc" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6421 + * with gil: + * raise RuntimeError('Function "cuTexObjectGetResourceViewDesc" not found') + * err = ( __cuTexObjectGetResourceViewDesc)(pResViewDesc, texObject) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_RESOURCE_VIEW_DESC *, CUtexObject))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceViewDesc)(__pyx_v_pResViewDesc, __pyx_v_texObject); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6421, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6422 + * raise RuntimeError('Function "cuTexObjectGetResourceViewDesc" not found') + * err = ( __cuTexObjectGetResourceViewDesc)(pResViewDesc, texObject) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuSurfObjectCreate(CUsurfObject* pSurfObject, const CUDA_RESOURCE_DESC* pResDesc) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6415 + * return err + * + * cdef CUresult _cuTexObjectGetResourceViewDesc(CUDA_RESOURCE_VIEW_DESC* pResViewDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTexObjectGetResourceViewDesc + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTexObjectGetResourceViewDesc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6424 + * return err + * + * cdef CUresult _cuSurfObjectCreate(CUsurfObject* pSurfObject, const CUDA_RESOURCE_DESC* pResDesc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuSurfObjectCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuSurfObjectCreate(CUsurfObject *__pyx_v_pSurfObject, CUDA_RESOURCE_DESC const *__pyx_v_pResDesc) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuSurfObjectCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6426 + * cdef CUresult _cuSurfObjectCreate(CUsurfObject* pSurfObject, const CUDA_RESOURCE_DESC* pResDesc) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuSurfObjectCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuSurfObjectCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6426, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6427 + * global __cuSurfObjectCreate + * cuPythonInit() + * if __cuSurfObjectCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuSurfObjectCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6428 + * cuPythonInit() + * if __cuSurfObjectCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuSurfObjectCreate" not found') + * err = ( __cuSurfObjectCreate)(pSurfObject, pResDesc) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6429 + * if __cuSurfObjectCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuSurfObjectCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuSurfObjectCreate)(pSurfObject, pResDesc) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuSurfObjectCreate_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6429, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6429, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6428 + * cuPythonInit() + * if __cuSurfObjectCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuSurfObjectCreate" not found') + * err = ( __cuSurfObjectCreate)(pSurfObject, pResDesc) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6427 + * global __cuSurfObjectCreate + * cuPythonInit() + * if __cuSurfObjectCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuSurfObjectCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6430 + * with gil: + * raise RuntimeError('Function "cuSurfObjectCreate" not found') + * err = ( __cuSurfObjectCreate)(pSurfObject, pResDesc) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUsurfObject *, CUDA_RESOURCE_DESC const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectCreate)(__pyx_v_pSurfObject, __pyx_v_pResDesc); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6430, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6431 + * raise RuntimeError('Function "cuSurfObjectCreate" not found') + * err = ( __cuSurfObjectCreate)(pSurfObject, pResDesc) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuSurfObjectDestroy(CUsurfObject surfObject) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6424 + * return err + * + * cdef CUresult _cuSurfObjectCreate(CUsurfObject* pSurfObject, const CUDA_RESOURCE_DESC* pResDesc) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuSurfObjectCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuSurfObjectCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6433 + * return err + * + * cdef CUresult _cuSurfObjectDestroy(CUsurfObject surfObject) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuSurfObjectDestroy + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuSurfObjectDestroy(CUsurfObject __pyx_v_surfObject) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuSurfObjectDestroy", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6435 + * cdef CUresult _cuSurfObjectDestroy(CUsurfObject surfObject) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuSurfObjectDestroy + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuSurfObjectDestroy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6435, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6436 + * global __cuSurfObjectDestroy + * cuPythonInit() + * if __cuSurfObjectDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuSurfObjectDestroy" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectDestroy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6437 + * cuPythonInit() + * if __cuSurfObjectDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuSurfObjectDestroy" not found') + * err = ( __cuSurfObjectDestroy)(surfObject) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6438 + * if __cuSurfObjectDestroy == NULL: + * with gil: + * raise RuntimeError('Function "cuSurfObjectDestroy" not found') # <<<<<<<<<<<<<< + * err = ( __cuSurfObjectDestroy)(surfObject) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuSurfObjectDestroy_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6438, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6438, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6437 + * cuPythonInit() + * if __cuSurfObjectDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuSurfObjectDestroy" not found') + * err = ( __cuSurfObjectDestroy)(surfObject) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6436 + * global __cuSurfObjectDestroy + * cuPythonInit() + * if __cuSurfObjectDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuSurfObjectDestroy" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6439 + * with gil: + * raise RuntimeError('Function "cuSurfObjectDestroy" not found') + * err = ( __cuSurfObjectDestroy)(surfObject) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUsurfObject))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectDestroy)(__pyx_v_surfObject); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6439, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6440 + * raise RuntimeError('Function "cuSurfObjectDestroy" not found') + * err = ( __cuSurfObjectDestroy)(surfObject) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuSurfObjectGetResourceDesc(CUDA_RESOURCE_DESC* pResDesc, CUsurfObject surfObject) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6433 + * return err + * + * cdef CUresult _cuSurfObjectDestroy(CUsurfObject surfObject) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuSurfObjectDestroy + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuSurfObjectDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6442 + * return err + * + * cdef CUresult _cuSurfObjectGetResourceDesc(CUDA_RESOURCE_DESC* pResDesc, CUsurfObject surfObject) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuSurfObjectGetResourceDesc + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuSurfObjectGetResourceDesc(CUDA_RESOURCE_DESC *__pyx_v_pResDesc, CUsurfObject __pyx_v_surfObject) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuSurfObjectGetResourceDesc", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6444 + * cdef CUresult _cuSurfObjectGetResourceDesc(CUDA_RESOURCE_DESC* pResDesc, CUsurfObject surfObject) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuSurfObjectGetResourceDesc + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuSurfObjectGetResourceDesc == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6444, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6445 + * global __cuSurfObjectGetResourceDesc + * cuPythonInit() + * if __cuSurfObjectGetResourceDesc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuSurfObjectGetResourceDesc" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectGetResourceDesc == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6446 + * cuPythonInit() + * if __cuSurfObjectGetResourceDesc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuSurfObjectGetResourceDesc" not found') + * err = ( __cuSurfObjectGetResourceDesc)(pResDesc, surfObject) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6447 + * if __cuSurfObjectGetResourceDesc == NULL: + * with gil: + * raise RuntimeError('Function "cuSurfObjectGetResourceDesc" not found') # <<<<<<<<<<<<<< + * err = ( __cuSurfObjectGetResourceDesc)(pResDesc, surfObject) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuSurfObjectGetResource}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6447, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6447, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6446 + * cuPythonInit() + * if __cuSurfObjectGetResourceDesc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuSurfObjectGetResourceDesc" not found') + * err = ( __cuSurfObjectGetResourceDesc)(pResDesc, surfObject) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6445 + * global __cuSurfObjectGetResourceDesc + * cuPythonInit() + * if __cuSurfObjectGetResourceDesc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuSurfObjectGetResourceDesc" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6448 + * with gil: + * raise RuntimeError('Function "cuSurfObjectGetResourceDesc" not found') + * err = ( __cuSurfObjectGetResourceDesc)(pResDesc, surfObject) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUDA_RESOURCE_DESC *, CUsurfObject))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectGetResourceDesc)(__pyx_v_pResDesc, __pyx_v_surfObject); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6448, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6449 + * raise RuntimeError('Function "cuSurfObjectGetResourceDesc" not found') + * err = ( __cuSurfObjectGetResourceDesc)(pResDesc, surfObject) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTensorMapEncodeTiled(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, const cuuint32_t* boxDim, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6442 + * return err + * + * cdef CUresult _cuSurfObjectGetResourceDesc(CUDA_RESOURCE_DESC* pResDesc, CUsurfObject surfObject) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuSurfObjectGetResourceDesc + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuSurfObjectGetResourceDesc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6451 + * return err + * + * cdef CUresult _cuTensorMapEncodeTiled(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, const cuuint32_t* boxDim, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeTiled + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTensorMapEncodeTiled(CUtensorMap *__pyx_v_tensorMap, CUtensorMapDataType __pyx_v_tensorDataType, cuuint32_t __pyx_v_tensorRank, void *__pyx_v_globalAddress, cuuint64_t const *__pyx_v_globalDim, cuuint64_t const *__pyx_v_globalStrides, cuuint32_t const *__pyx_v_boxDim, cuuint32_t const *__pyx_v_elementStrides, CUtensorMapInterleave __pyx_v_interleave, CUtensorMapSwizzle __pyx_v_swizzle, CUtensorMapL2promotion __pyx_v_l2Promotion, CUtensorMapFloatOOBfill __pyx_v_oobFill) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTensorMapEncodeTiled", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6453 + * cdef CUresult _cuTensorMapEncodeTiled(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, const cuuint32_t* boxDim, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTensorMapEncodeTiled + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTensorMapEncodeTiled == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6453, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6454 + * global __cuTensorMapEncodeTiled + * cuPythonInit() + * if __cuTensorMapEncodeTiled == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTensorMapEncodeTiled" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeTiled == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6455 + * cuPythonInit() + * if __cuTensorMapEncodeTiled == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTensorMapEncodeTiled" not found') + * err = ( __cuTensorMapEncodeTiled)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, boxDim, elementStrides, interleave, swizzle, l2Promotion, oobFill) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6456 + * if __cuTensorMapEncodeTiled == NULL: + * with gil: + * raise RuntimeError('Function "cuTensorMapEncodeTiled" not found') # <<<<<<<<<<<<<< + * err = ( __cuTensorMapEncodeTiled)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, boxDim, elementStrides, interleave, swizzle, l2Promotion, oobFill) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTensorMapEncodeTiled}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6456, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6456, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6455 + * cuPythonInit() + * if __cuTensorMapEncodeTiled == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTensorMapEncodeTiled" not found') + * err = ( __cuTensorMapEncodeTiled)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, boxDim, elementStrides, interleave, swizzle, l2Promotion, oobFill) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6454 + * global __cuTensorMapEncodeTiled + * cuPythonInit() + * if __cuTensorMapEncodeTiled == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTensorMapEncodeTiled" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6457 + * with gil: + * raise RuntimeError('Function "cuTensorMapEncodeTiled" not found') + * err = ( __cuTensorMapEncodeTiled)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, boxDim, elementStrides, interleave, swizzle, l2Promotion, oobFill) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtensorMap *, CUtensorMapDataType, cuuint32_t, void *, cuuint64_t const *, cuuint64_t const *, cuuint32_t const *, cuuint32_t const *, CUtensorMapInterleave, CUtensorMapSwizzle, CUtensorMapL2promotion, CUtensorMapFloatOOBfill))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeTiled)(__pyx_v_tensorMap, __pyx_v_tensorDataType, __pyx_v_tensorRank, __pyx_v_globalAddress, __pyx_v_globalDim, __pyx_v_globalStrides, __pyx_v_boxDim, __pyx_v_elementStrides, __pyx_v_interleave, __pyx_v_swizzle, __pyx_v_l2Promotion, __pyx_v_oobFill); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6457, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6458 + * raise RuntimeError('Function "cuTensorMapEncodeTiled" not found') + * err = ( __cuTensorMapEncodeTiled)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, boxDim, elementStrides, interleave, swizzle, l2Promotion, oobFill) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTensorMapEncodeIm2col(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, const int* pixelBoxLowerCorner, const int* pixelBoxUpperCorner, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6451 + * return err + * + * cdef CUresult _cuTensorMapEncodeTiled(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, const cuuint32_t* boxDim, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeTiled + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTensorMapEncodeTiled", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6460 + * return err + * + * cdef CUresult _cuTensorMapEncodeIm2col(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, const int* pixelBoxLowerCorner, const int* pixelBoxUpperCorner, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeIm2col + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTensorMapEncodeIm2col(CUtensorMap *__pyx_v_tensorMap, CUtensorMapDataType __pyx_v_tensorDataType, cuuint32_t __pyx_v_tensorRank, void *__pyx_v_globalAddress, cuuint64_t const *__pyx_v_globalDim, cuuint64_t const *__pyx_v_globalStrides, int const *__pyx_v_pixelBoxLowerCorner, int const *__pyx_v_pixelBoxUpperCorner, cuuint32_t __pyx_v_channelsPerPixel, cuuint32_t __pyx_v_pixelsPerColumn, cuuint32_t const *__pyx_v_elementStrides, CUtensorMapInterleave __pyx_v_interleave, CUtensorMapSwizzle __pyx_v_swizzle, CUtensorMapL2promotion __pyx_v_l2Promotion, CUtensorMapFloatOOBfill __pyx_v_oobFill) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTensorMapEncodeIm2col", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6462 + * cdef CUresult _cuTensorMapEncodeIm2col(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, const int* pixelBoxLowerCorner, const int* pixelBoxUpperCorner, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTensorMapEncodeIm2col + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTensorMapEncodeIm2col == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6462, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6463 + * global __cuTensorMapEncodeIm2col + * cuPythonInit() + * if __cuTensorMapEncodeIm2col == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTensorMapEncodeIm2col" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2col == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6464 + * cuPythonInit() + * if __cuTensorMapEncodeIm2col == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTensorMapEncodeIm2col" not found') + * err = ( __cuTensorMapEncodeIm2col)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCorner, pixelBoxUpperCorner, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, swizzle, l2Promotion, oobFill) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6465 + * if __cuTensorMapEncodeIm2col == NULL: + * with gil: + * raise RuntimeError('Function "cuTensorMapEncodeIm2col" not found') # <<<<<<<<<<<<<< + * err = ( __cuTensorMapEncodeIm2col)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCorner, pixelBoxUpperCorner, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, swizzle, l2Promotion, oobFill) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTensorMapEncodeIm2col}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6465, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6465, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6464 + * cuPythonInit() + * if __cuTensorMapEncodeIm2col == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTensorMapEncodeIm2col" not found') + * err = ( __cuTensorMapEncodeIm2col)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCorner, pixelBoxUpperCorner, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, swizzle, l2Promotion, oobFill) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6463 + * global __cuTensorMapEncodeIm2col + * cuPythonInit() + * if __cuTensorMapEncodeIm2col == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTensorMapEncodeIm2col" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6466 + * with gil: + * raise RuntimeError('Function "cuTensorMapEncodeIm2col" not found') + * err = ( __cuTensorMapEncodeIm2col)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCorner, pixelBoxUpperCorner, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, swizzle, l2Promotion, oobFill) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtensorMap *, CUtensorMapDataType, cuuint32_t, void *, cuuint64_t const *, cuuint64_t const *, int const *, int const *, cuuint32_t, cuuint32_t, cuuint32_t const *, CUtensorMapInterleave, CUtensorMapSwizzle, CUtensorMapL2promotion, CUtensorMapFloatOOBfill))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2col)(__pyx_v_tensorMap, __pyx_v_tensorDataType, __pyx_v_tensorRank, __pyx_v_globalAddress, __pyx_v_globalDim, __pyx_v_globalStrides, __pyx_v_pixelBoxLowerCorner, __pyx_v_pixelBoxUpperCorner, __pyx_v_channelsPerPixel, __pyx_v_pixelsPerColumn, __pyx_v_elementStrides, __pyx_v_interleave, __pyx_v_swizzle, __pyx_v_l2Promotion, __pyx_v_oobFill); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6466, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6467 + * raise RuntimeError('Function "cuTensorMapEncodeIm2col" not found') + * err = ( __cuTensorMapEncodeIm2col)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCorner, pixelBoxUpperCorner, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, swizzle, l2Promotion, oobFill) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTensorMapEncodeIm2colWide(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, int pixelBoxLowerCornerWidth, int pixelBoxUpperCornerWidth, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapIm2ColWideMode mode, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6460 + * return err + * + * cdef CUresult _cuTensorMapEncodeIm2col(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, const int* pixelBoxLowerCorner, const int* pixelBoxUpperCorner, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeIm2col + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTensorMapEncodeIm2col", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6469 + * return err + * + * cdef CUresult _cuTensorMapEncodeIm2colWide(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, int pixelBoxLowerCornerWidth, int pixelBoxUpperCornerWidth, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapIm2ColWideMode mode, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeIm2colWide + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTensorMapEncodeIm2colWide(CUtensorMap *__pyx_v_tensorMap, CUtensorMapDataType __pyx_v_tensorDataType, cuuint32_t __pyx_v_tensorRank, void *__pyx_v_globalAddress, cuuint64_t const *__pyx_v_globalDim, cuuint64_t const *__pyx_v_globalStrides, int __pyx_v_pixelBoxLowerCornerWidth, int __pyx_v_pixelBoxUpperCornerWidth, cuuint32_t __pyx_v_channelsPerPixel, cuuint32_t __pyx_v_pixelsPerColumn, cuuint32_t const *__pyx_v_elementStrides, CUtensorMapInterleave __pyx_v_interleave, CUtensorMapIm2ColWideMode __pyx_v_mode, CUtensorMapSwizzle __pyx_v_swizzle, CUtensorMapL2promotion __pyx_v_l2Promotion, CUtensorMapFloatOOBfill __pyx_v_oobFill) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTensorMapEncodeIm2colWide", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6471 + * cdef CUresult _cuTensorMapEncodeIm2colWide(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, int pixelBoxLowerCornerWidth, int pixelBoxUpperCornerWidth, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapIm2ColWideMode mode, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTensorMapEncodeIm2colWide + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTensorMapEncodeIm2colWide == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6471, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6472 + * global __cuTensorMapEncodeIm2colWide + * cuPythonInit() + * if __cuTensorMapEncodeIm2colWide == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTensorMapEncodeIm2colWide" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2colWide == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6473 + * cuPythonInit() + * if __cuTensorMapEncodeIm2colWide == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTensorMapEncodeIm2colWide" not found') + * err = ( __cuTensorMapEncodeIm2colWide)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCornerWidth, pixelBoxUpperCornerWidth, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, mode, swizzle, l2Promotion, oobFill) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6474 + * if __cuTensorMapEncodeIm2colWide == NULL: + * with gil: + * raise RuntimeError('Function "cuTensorMapEncodeIm2colWide" not found') # <<<<<<<<<<<<<< + * err = ( __cuTensorMapEncodeIm2colWide)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCornerWidth, pixelBoxUpperCornerWidth, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, mode, swizzle, l2Promotion, oobFill) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTensorMapEncodeIm2col_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6474, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6474, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6473 + * cuPythonInit() + * if __cuTensorMapEncodeIm2colWide == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTensorMapEncodeIm2colWide" not found') + * err = ( __cuTensorMapEncodeIm2colWide)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCornerWidth, pixelBoxUpperCornerWidth, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, mode, swizzle, l2Promotion, oobFill) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6472 + * global __cuTensorMapEncodeIm2colWide + * cuPythonInit() + * if __cuTensorMapEncodeIm2colWide == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTensorMapEncodeIm2colWide" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6475 + * with gil: + * raise RuntimeError('Function "cuTensorMapEncodeIm2colWide" not found') + * err = ( __cuTensorMapEncodeIm2colWide)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCornerWidth, pixelBoxUpperCornerWidth, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, mode, swizzle, l2Promotion, oobFill) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtensorMap *, CUtensorMapDataType, cuuint32_t, void *, cuuint64_t const *, cuuint64_t const *, int, int, cuuint32_t, cuuint32_t, cuuint32_t const *, CUtensorMapInterleave, CUtensorMapIm2ColWideMode, CUtensorMapSwizzle, CUtensorMapL2promotion, CUtensorMapFloatOOBfill))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2colWide)(__pyx_v_tensorMap, __pyx_v_tensorDataType, __pyx_v_tensorRank, __pyx_v_globalAddress, __pyx_v_globalDim, __pyx_v_globalStrides, __pyx_v_pixelBoxLowerCornerWidth, __pyx_v_pixelBoxUpperCornerWidth, __pyx_v_channelsPerPixel, __pyx_v_pixelsPerColumn, __pyx_v_elementStrides, __pyx_v_interleave, __pyx_v_mode, __pyx_v_swizzle, __pyx_v_l2Promotion, __pyx_v_oobFill); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6475, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6476 + * raise RuntimeError('Function "cuTensorMapEncodeIm2colWide" not found') + * err = ( __cuTensorMapEncodeIm2colWide)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCornerWidth, pixelBoxUpperCornerWidth, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, mode, swizzle, l2Promotion, oobFill) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuTensorMapReplaceAddress(CUtensorMap* tensorMap, void* globalAddress) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6469 + * return err + * + * cdef CUresult _cuTensorMapEncodeIm2colWide(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, int pixelBoxLowerCornerWidth, int pixelBoxUpperCornerWidth, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapIm2ColWideMode mode, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeIm2colWide + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTensorMapEncodeIm2colWide", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6478 + * return err + * + * cdef CUresult _cuTensorMapReplaceAddress(CUtensorMap* tensorMap, void* globalAddress) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTensorMapReplaceAddress + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTensorMapReplaceAddress(CUtensorMap *__pyx_v_tensorMap, void *__pyx_v_globalAddress) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuTensorMapReplaceAddress", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6480 + * cdef CUresult _cuTensorMapReplaceAddress(CUtensorMap* tensorMap, void* globalAddress) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuTensorMapReplaceAddress + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuTensorMapReplaceAddress == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6480, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6481 + * global __cuTensorMapReplaceAddress + * cuPythonInit() + * if __cuTensorMapReplaceAddress == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTensorMapReplaceAddress" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapReplaceAddress == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6482 + * cuPythonInit() + * if __cuTensorMapReplaceAddress == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTensorMapReplaceAddress" not found') + * err = ( __cuTensorMapReplaceAddress)(tensorMap, globalAddress) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6483 + * if __cuTensorMapReplaceAddress == NULL: + * with gil: + * raise RuntimeError('Function "cuTensorMapReplaceAddress" not found') # <<<<<<<<<<<<<< + * err = ( __cuTensorMapReplaceAddress)(tensorMap, globalAddress) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuTensorMapReplaceAddre}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6483, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6483, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6482 + * cuPythonInit() + * if __cuTensorMapReplaceAddress == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuTensorMapReplaceAddress" not found') + * err = ( __cuTensorMapReplaceAddress)(tensorMap, globalAddress) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6481 + * global __cuTensorMapReplaceAddress + * cuPythonInit() + * if __cuTensorMapReplaceAddress == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuTensorMapReplaceAddress" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6484 + * with gil: + * raise RuntimeError('Function "cuTensorMapReplaceAddress" not found') + * err = ( __cuTensorMapReplaceAddress)(tensorMap, globalAddress) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUtensorMap *, void *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapReplaceAddress)(__pyx_v_tensorMap, __pyx_v_globalAddress); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6484, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6485 + * raise RuntimeError('Function "cuTensorMapReplaceAddress" not found') + * err = ( __cuTensorMapReplaceAddress)(tensorMap, globalAddress) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceCanAccessPeer(int* canAccessPeer, CUdevice dev, CUdevice peerDev) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6478 + * return err + * + * cdef CUresult _cuTensorMapReplaceAddress(CUtensorMap* tensorMap, void* globalAddress) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuTensorMapReplaceAddress + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuTensorMapReplaceAddress", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6487 + * return err + * + * cdef CUresult _cuDeviceCanAccessPeer(int* canAccessPeer, CUdevice dev, CUdevice peerDev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceCanAccessPeer + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceCanAccessPeer(int *__pyx_v_canAccessPeer, CUdevice __pyx_v_dev, CUdevice __pyx_v_peerDev) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceCanAccessPeer", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6489 + * cdef CUresult _cuDeviceCanAccessPeer(int* canAccessPeer, CUdevice dev, CUdevice peerDev) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceCanAccessPeer + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceCanAccessPeer == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6489, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6490 + * global __cuDeviceCanAccessPeer + * cuPythonInit() + * if __cuDeviceCanAccessPeer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceCanAccessPeer" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceCanAccessPeer == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6491 + * cuPythonInit() + * if __cuDeviceCanAccessPeer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceCanAccessPeer" not found') + * err = ( __cuDeviceCanAccessPeer)(canAccessPeer, dev, peerDev) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6492 + * if __cuDeviceCanAccessPeer == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceCanAccessPeer" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceCanAccessPeer)(canAccessPeer, dev, peerDev) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceCanAccessPeer_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6492, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6492, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6491 + * cuPythonInit() + * if __cuDeviceCanAccessPeer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceCanAccessPeer" not found') + * err = ( __cuDeviceCanAccessPeer)(canAccessPeer, dev, peerDev) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6490 + * global __cuDeviceCanAccessPeer + * cuPythonInit() + * if __cuDeviceCanAccessPeer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceCanAccessPeer" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6493 + * with gil: + * raise RuntimeError('Function "cuDeviceCanAccessPeer" not found') + * err = ( __cuDeviceCanAccessPeer)(canAccessPeer, dev, peerDev) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, CUdevice, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceCanAccessPeer)(__pyx_v_canAccessPeer, __pyx_v_dev, __pyx_v_peerDev); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6493, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6494 + * raise RuntimeError('Function "cuDeviceCanAccessPeer" not found') + * err = ( __cuDeviceCanAccessPeer)(canAccessPeer, dev, peerDev) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxEnablePeerAccess(CUcontext peerContext, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6487 + * return err + * + * cdef CUresult _cuDeviceCanAccessPeer(int* canAccessPeer, CUdevice dev, CUdevice peerDev) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceCanAccessPeer + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceCanAccessPeer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6496 + * return err + * + * cdef CUresult _cuCtxEnablePeerAccess(CUcontext peerContext, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxEnablePeerAccess + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxEnablePeerAccess(CUcontext __pyx_v_peerContext, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxEnablePeerAccess", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6498 + * cdef CUresult _cuCtxEnablePeerAccess(CUcontext peerContext, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxEnablePeerAccess + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxEnablePeerAccess == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6498, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6499 + * global __cuCtxEnablePeerAccess + * cuPythonInit() + * if __cuCtxEnablePeerAccess == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxEnablePeerAccess" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxEnablePeerAccess == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6500 + * cuPythonInit() + * if __cuCtxEnablePeerAccess == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxEnablePeerAccess" not found') + * err = ( __cuCtxEnablePeerAccess)(peerContext, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6501 + * if __cuCtxEnablePeerAccess == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxEnablePeerAccess" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxEnablePeerAccess)(peerContext, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxEnablePeerAccess_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6501, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6501, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6500 + * cuPythonInit() + * if __cuCtxEnablePeerAccess == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxEnablePeerAccess" not found') + * err = ( __cuCtxEnablePeerAccess)(peerContext, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6499 + * global __cuCtxEnablePeerAccess + * cuPythonInit() + * if __cuCtxEnablePeerAccess == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxEnablePeerAccess" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6502 + * with gil: + * raise RuntimeError('Function "cuCtxEnablePeerAccess" not found') + * err = ( __cuCtxEnablePeerAccess)(peerContext, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxEnablePeerAccess)(__pyx_v_peerContext, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6502, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6503 + * raise RuntimeError('Function "cuCtxEnablePeerAccess" not found') + * err = ( __cuCtxEnablePeerAccess)(peerContext, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxDisablePeerAccess(CUcontext peerContext) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6496 + * return err + * + * cdef CUresult _cuCtxEnablePeerAccess(CUcontext peerContext, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxEnablePeerAccess + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxEnablePeerAccess", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6505 + * return err + * + * cdef CUresult _cuCtxDisablePeerAccess(CUcontext peerContext) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxDisablePeerAccess + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxDisablePeerAccess(CUcontext __pyx_v_peerContext) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxDisablePeerAccess", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6507 + * cdef CUresult _cuCtxDisablePeerAccess(CUcontext peerContext) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxDisablePeerAccess + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxDisablePeerAccess == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6507, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6508 + * global __cuCtxDisablePeerAccess + * cuPythonInit() + * if __cuCtxDisablePeerAccess == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxDisablePeerAccess" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDisablePeerAccess == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6509 + * cuPythonInit() + * if __cuCtxDisablePeerAccess == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxDisablePeerAccess" not found') + * err = ( __cuCtxDisablePeerAccess)(peerContext) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6510 + * if __cuCtxDisablePeerAccess == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxDisablePeerAccess" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxDisablePeerAccess)(peerContext) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxDisablePeerAccess}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6510, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6510, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6509 + * cuPythonInit() + * if __cuCtxDisablePeerAccess == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxDisablePeerAccess" not found') + * err = ( __cuCtxDisablePeerAccess)(peerContext) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6508 + * global __cuCtxDisablePeerAccess + * cuPythonInit() + * if __cuCtxDisablePeerAccess == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxDisablePeerAccess" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6511 + * with gil: + * raise RuntimeError('Function "cuCtxDisablePeerAccess" not found') + * err = ( __cuCtxDisablePeerAccess)(peerContext) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDisablePeerAccess)(__pyx_v_peerContext); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6511, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6512 + * raise RuntimeError('Function "cuCtxDisablePeerAccess" not found') + * err = ( __cuCtxDisablePeerAccess)(peerContext) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetP2PAttribute(int* value, CUdevice_P2PAttribute attrib, CUdevice srcDevice, CUdevice dstDevice) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6505 + * return err + * + * cdef CUresult _cuCtxDisablePeerAccess(CUcontext peerContext) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxDisablePeerAccess + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxDisablePeerAccess", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6514 + * return err + * + * cdef CUresult _cuDeviceGetP2PAttribute(int* value, CUdevice_P2PAttribute attrib, CUdevice srcDevice, CUdevice dstDevice) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetP2PAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetP2PAttribute(int *__pyx_v_value, CUdevice_P2PAttribute __pyx_v_attrib, CUdevice __pyx_v_srcDevice, CUdevice __pyx_v_dstDevice) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetP2PAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6516 + * cdef CUresult _cuDeviceGetP2PAttribute(int* value, CUdevice_P2PAttribute attrib, CUdevice srcDevice, CUdevice dstDevice) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetP2PAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetP2PAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6516, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6517 + * global __cuDeviceGetP2PAttribute + * cuPythonInit() + * if __cuDeviceGetP2PAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetP2PAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetP2PAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6518 + * cuPythonInit() + * if __cuDeviceGetP2PAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetP2PAttribute" not found') + * err = ( __cuDeviceGetP2PAttribute)(value, attrib, srcDevice, dstDevice) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6519 + * if __cuDeviceGetP2PAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetP2PAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetP2PAttribute)(value, attrib, srcDevice, dstDevice) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetP2PAttribute}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6519, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6519, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6518 + * cuPythonInit() + * if __cuDeviceGetP2PAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetP2PAttribute" not found') + * err = ( __cuDeviceGetP2PAttribute)(value, attrib, srcDevice, dstDevice) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6517 + * global __cuDeviceGetP2PAttribute + * cuPythonInit() + * if __cuDeviceGetP2PAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetP2PAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6520 + * with gil: + * raise RuntimeError('Function "cuDeviceGetP2PAttribute" not found') + * err = ( __cuDeviceGetP2PAttribute)(value, attrib, srcDevice, dstDevice) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int *, CUdevice_P2PAttribute, CUdevice, CUdevice))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetP2PAttribute)(__pyx_v_value, __pyx_v_attrib, __pyx_v_srcDevice, __pyx_v_dstDevice); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6520, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6521 + * raise RuntimeError('Function "cuDeviceGetP2PAttribute" not found') + * err = ( __cuDeviceGetP2PAttribute)(value, attrib, srcDevice, dstDevice) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphicsUnregisterResource(CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6514 + * return err + * + * cdef CUresult _cuDeviceGetP2PAttribute(int* value, CUdevice_P2PAttribute attrib, CUdevice srcDevice, CUdevice dstDevice) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetP2PAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetP2PAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6523 + * return err + * + * cdef CUresult _cuGraphicsUnregisterResource(CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsUnregisterResource + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsUnregisterResource(CUgraphicsResource __pyx_v_resource) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphicsUnregisterResource", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6525 + * cdef CUresult _cuGraphicsUnregisterResource(CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphicsUnregisterResource + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphicsUnregisterResource == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6525, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6526 + * global __cuGraphicsUnregisterResource + * cuPythonInit() + * if __cuGraphicsUnregisterResource == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsUnregisterResource" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnregisterResource == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6527 + * cuPythonInit() + * if __cuGraphicsUnregisterResource == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsUnregisterResource" not found') + * err = ( __cuGraphicsUnregisterResource)(resource) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6528 + * if __cuGraphicsUnregisterResource == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphicsUnregisterResource" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphicsUnregisterResource)(resource) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphicsUnregisterRes}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6528, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6528, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6527 + * cuPythonInit() + * if __cuGraphicsUnregisterResource == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsUnregisterResource" not found') + * err = ( __cuGraphicsUnregisterResource)(resource) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6526 + * global __cuGraphicsUnregisterResource + * cuPythonInit() + * if __cuGraphicsUnregisterResource == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsUnregisterResource" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6529 + * with gil: + * raise RuntimeError('Function "cuGraphicsUnregisterResource" not found') + * err = ( __cuGraphicsUnregisterResource)(resource) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphicsResource))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnregisterResource)(__pyx_v_resource); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6529, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6530 + * raise RuntimeError('Function "cuGraphicsUnregisterResource" not found') + * err = ( __cuGraphicsUnregisterResource)(resource) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphicsSubResourceGetMappedArray(CUarray* pArray, CUgraphicsResource resource, unsigned int arrayIndex, unsigned int mipLevel) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6523 + * return err + * + * cdef CUresult _cuGraphicsUnregisterResource(CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsUnregisterResource + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphicsUnregisterResource", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6532 + * return err + * + * cdef CUresult _cuGraphicsSubResourceGetMappedArray(CUarray* pArray, CUgraphicsResource resource, unsigned int arrayIndex, unsigned int mipLevel) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsSubResourceGetMappedArray + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsSubResourceGetMappedArray(CUarray *__pyx_v_pArray, CUgraphicsResource __pyx_v_resource, unsigned int __pyx_v_arrayIndex, unsigned int __pyx_v_mipLevel) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphicsSubResourceGetMappedArray", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6534 + * cdef CUresult _cuGraphicsSubResourceGetMappedArray(CUarray* pArray, CUgraphicsResource resource, unsigned int arrayIndex, unsigned int mipLevel) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphicsSubResourceGetMappedArray + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphicsSubResourceGetMappedArray == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6534, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6535 + * global __cuGraphicsSubResourceGetMappedArray + * cuPythonInit() + * if __cuGraphicsSubResourceGetMappedArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsSubResourceGetMappedArray" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsSubResourceGetMappedArray == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6536 + * cuPythonInit() + * if __cuGraphicsSubResourceGetMappedArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsSubResourceGetMappedArray" not found') + * err = ( __cuGraphicsSubResourceGetMappedArray)(pArray, resource, arrayIndex, mipLevel) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6537 + * if __cuGraphicsSubResourceGetMappedArray == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphicsSubResourceGetMappedArray" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphicsSubResourceGetMappedArray)(pArray, resource, arrayIndex, mipLevel) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphicsSubResourceGe}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6537, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6537, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6536 + * cuPythonInit() + * if __cuGraphicsSubResourceGetMappedArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsSubResourceGetMappedArray" not found') + * err = ( __cuGraphicsSubResourceGetMappedArray)(pArray, resource, arrayIndex, mipLevel) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6535 + * global __cuGraphicsSubResourceGetMappedArray + * cuPythonInit() + * if __cuGraphicsSubResourceGetMappedArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsSubResourceGetMappedArray" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6538 + * with gil: + * raise RuntimeError('Function "cuGraphicsSubResourceGetMappedArray" not found') + * err = ( __cuGraphicsSubResourceGetMappedArray)(pArray, resource, arrayIndex, mipLevel) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUarray *, CUgraphicsResource, unsigned int, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsSubResourceGetMappedArray)(__pyx_v_pArray, __pyx_v_resource, __pyx_v_arrayIndex, __pyx_v_mipLevel); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6538, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6539 + * raise RuntimeError('Function "cuGraphicsSubResourceGetMappedArray" not found') + * err = ( __cuGraphicsSubResourceGetMappedArray)(pArray, resource, arrayIndex, mipLevel) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphicsResourceGetMappedMipmappedArray(CUmipmappedArray* pMipmappedArray, CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6532 + * return err + * + * cdef CUresult _cuGraphicsSubResourceGetMappedArray(CUarray* pArray, CUgraphicsResource resource, unsigned int arrayIndex, unsigned int mipLevel) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsSubResourceGetMappedArray + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphicsSubResourceGetMappedArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6541 + * return err + * + * cdef CUresult _cuGraphicsResourceGetMappedMipmappedArray(CUmipmappedArray* pMipmappedArray, CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedMipmappedArray + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsResourceGetMappedMipmappedArray(CUmipmappedArray *__pyx_v_pMipmappedArray, CUgraphicsResource __pyx_v_resource) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphicsResourceGetMappedMipmappedArray", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6543 + * cdef CUresult _cuGraphicsResourceGetMappedMipmappedArray(CUmipmappedArray* pMipmappedArray, CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphicsResourceGetMappedMipmappedArray + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphicsResourceGetMappedMipmappedArray == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6543, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6544 + * global __cuGraphicsResourceGetMappedMipmappedArray + * cuPythonInit() + * if __cuGraphicsResourceGetMappedMipmappedArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceGetMappedMipmappedArray" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedMipmappedArray == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6545 + * cuPythonInit() + * if __cuGraphicsResourceGetMappedMipmappedArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsResourceGetMappedMipmappedArray" not found') + * err = ( __cuGraphicsResourceGetMappedMipmappedArray)(pMipmappedArray, resource) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6546 + * if __cuGraphicsResourceGetMappedMipmappedArray == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceGetMappedMipmappedArray" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphicsResourceGetMappedMipmappedArray)(pMipmappedArray, resource) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphicsResourceGetMa}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6546, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6546, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6545 + * cuPythonInit() + * if __cuGraphicsResourceGetMappedMipmappedArray == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsResourceGetMappedMipmappedArray" not found') + * err = ( __cuGraphicsResourceGetMappedMipmappedArray)(pMipmappedArray, resource) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6544 + * global __cuGraphicsResourceGetMappedMipmappedArray + * cuPythonInit() + * if __cuGraphicsResourceGetMappedMipmappedArray == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceGetMappedMipmappedArray" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6547 + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceGetMappedMipmappedArray" not found') + * err = ( __cuGraphicsResourceGetMappedMipmappedArray)(pMipmappedArray, resource) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUmipmappedArray *, CUgraphicsResource))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedMipmappedArray)(__pyx_v_pMipmappedArray, __pyx_v_resource); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6547, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6548 + * raise RuntimeError('Function "cuGraphicsResourceGetMappedMipmappedArray" not found') + * err = ( __cuGraphicsResourceGetMappedMipmappedArray)(pMipmappedArray, resource) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphicsResourceGetMappedPointer_v2(CUdeviceptr* pDevPtr, size_t* pSize, CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6541 + * return err + * + * cdef CUresult _cuGraphicsResourceGetMappedMipmappedArray(CUmipmappedArray* pMipmappedArray, CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedMipmappedArray + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphicsResourceGetMappedMipmappedArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6550 + * return err + * + * cdef CUresult _cuGraphicsResourceGetMappedPointer_v2(CUdeviceptr* pDevPtr, size_t* pSize, CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedPointer_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsResourceGetMappedPointer_v2(CUdeviceptr *__pyx_v_pDevPtr, size_t *__pyx_v_pSize, CUgraphicsResource __pyx_v_resource) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphicsResourceGetMappedPointer_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6552 + * cdef CUresult _cuGraphicsResourceGetMappedPointer_v2(CUdeviceptr* pDevPtr, size_t* pSize, CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphicsResourceGetMappedPointer_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphicsResourceGetMappedPointer_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6552, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6553 + * global __cuGraphicsResourceGetMappedPointer_v2 + * cuPythonInit() + * if __cuGraphicsResourceGetMappedPointer_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceGetMappedPointer_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedPointer_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6554 + * cuPythonInit() + * if __cuGraphicsResourceGetMappedPointer_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsResourceGetMappedPointer_v2" not found') + * err = ( __cuGraphicsResourceGetMappedPointer_v2)(pDevPtr, pSize, resource) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6555 + * if __cuGraphicsResourceGetMappedPointer_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceGetMappedPointer_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphicsResourceGetMappedPointer_v2)(pDevPtr, pSize, resource) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphicsResourceGetMa_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6555, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6555, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6554 + * cuPythonInit() + * if __cuGraphicsResourceGetMappedPointer_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsResourceGetMappedPointer_v2" not found') + * err = ( __cuGraphicsResourceGetMappedPointer_v2)(pDevPtr, pSize, resource) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6553 + * global __cuGraphicsResourceGetMappedPointer_v2 + * cuPythonInit() + * if __cuGraphicsResourceGetMappedPointer_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceGetMappedPointer_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6556 + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceGetMappedPointer_v2" not found') + * err = ( __cuGraphicsResourceGetMappedPointer_v2)(pDevPtr, pSize, resource) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdeviceptr *, size_t *, CUgraphicsResource))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedPointer_v2)(__pyx_v_pDevPtr, __pyx_v_pSize, __pyx_v_resource); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6556, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6557 + * raise RuntimeError('Function "cuGraphicsResourceGetMappedPointer_v2" not found') + * err = ( __cuGraphicsResourceGetMappedPointer_v2)(pDevPtr, pSize, resource) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphicsResourceSetMapFlags_v2(CUgraphicsResource resource, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6550 + * return err + * + * cdef CUresult _cuGraphicsResourceGetMappedPointer_v2(CUdeviceptr* pDevPtr, size_t* pSize, CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedPointer_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphicsResourceGetMappedPointer_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6559 + * return err + * + * cdef CUresult _cuGraphicsResourceSetMapFlags_v2(CUgraphicsResource resource, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsResourceSetMapFlags_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsResourceSetMapFlags_v2(CUgraphicsResource __pyx_v_resource, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphicsResourceSetMapFlags_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6561 + * cdef CUresult _cuGraphicsResourceSetMapFlags_v2(CUgraphicsResource resource, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphicsResourceSetMapFlags_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphicsResourceSetMapFlags_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6561, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6562 + * global __cuGraphicsResourceSetMapFlags_v2 + * cuPythonInit() + * if __cuGraphicsResourceSetMapFlags_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceSetMapFlags_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceSetMapFlags_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6563 + * cuPythonInit() + * if __cuGraphicsResourceSetMapFlags_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsResourceSetMapFlags_v2" not found') + * err = ( __cuGraphicsResourceSetMapFlags_v2)(resource, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6564 + * if __cuGraphicsResourceSetMapFlags_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceSetMapFlags_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphicsResourceSetMapFlags_v2)(resource, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphicsResourceSetMa}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6564, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6564, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6563 + * cuPythonInit() + * if __cuGraphicsResourceSetMapFlags_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsResourceSetMapFlags_v2" not found') + * err = ( __cuGraphicsResourceSetMapFlags_v2)(resource, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6562 + * global __cuGraphicsResourceSetMapFlags_v2 + * cuPythonInit() + * if __cuGraphicsResourceSetMapFlags_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceSetMapFlags_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6565 + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceSetMapFlags_v2" not found') + * err = ( __cuGraphicsResourceSetMapFlags_v2)(resource, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphicsResource, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceSetMapFlags_v2)(__pyx_v_resource, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6565, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6566 + * raise RuntimeError('Function "cuGraphicsResourceSetMapFlags_v2" not found') + * err = ( __cuGraphicsResourceSetMapFlags_v2)(resource, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphicsMapResources(unsigned int count, CUgraphicsResource* resources, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6559 + * return err + * + * cdef CUresult _cuGraphicsResourceSetMapFlags_v2(CUgraphicsResource resource, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsResourceSetMapFlags_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphicsResourceSetMapFlags_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6568 + * return err + * + * cdef CUresult _cuGraphicsMapResources(unsigned int count, CUgraphicsResource* resources, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsMapResources + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsMapResources(unsigned int __pyx_v_count, CUgraphicsResource *__pyx_v_resources, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphicsMapResources", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6570 + * cdef CUresult _cuGraphicsMapResources(unsigned int count, CUgraphicsResource* resources, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphicsMapResources + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphicsMapResources == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6570, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6571 + * global __cuGraphicsMapResources + * cuPythonInit() + * if __cuGraphicsMapResources == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsMapResources" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsMapResources == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6572 + * cuPythonInit() + * if __cuGraphicsMapResources == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsMapResources" not found') + * err = ( __cuGraphicsMapResources)(count, resources, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6573 + * if __cuGraphicsMapResources == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphicsMapResources" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphicsMapResources)(count, resources, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphicsMapResources}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6573, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6573, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6572 + * cuPythonInit() + * if __cuGraphicsMapResources == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsMapResources" not found') + * err = ( __cuGraphicsMapResources)(count, resources, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6571 + * global __cuGraphicsMapResources + * cuPythonInit() + * if __cuGraphicsMapResources == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsMapResources" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6574 + * with gil: + * raise RuntimeError('Function "cuGraphicsMapResources" not found') + * err = ( __cuGraphicsMapResources)(count, resources, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(unsigned int, CUgraphicsResource *, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsMapResources)(__pyx_v_count, __pyx_v_resources, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6574, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6575 + * raise RuntimeError('Function "cuGraphicsMapResources" not found') + * err = ( __cuGraphicsMapResources)(count, resources, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphicsUnmapResources(unsigned int count, CUgraphicsResource* resources, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6568 + * return err + * + * cdef CUresult _cuGraphicsMapResources(unsigned int count, CUgraphicsResource* resources, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsMapResources + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphicsMapResources", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6577 + * return err + * + * cdef CUresult _cuGraphicsUnmapResources(unsigned int count, CUgraphicsResource* resources, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsUnmapResources + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsUnmapResources(unsigned int __pyx_v_count, CUgraphicsResource *__pyx_v_resources, CUstream __pyx_v_hStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphicsUnmapResources", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6579 + * cdef CUresult _cuGraphicsUnmapResources(unsigned int count, CUgraphicsResource* resources, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphicsUnmapResources + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphicsUnmapResources == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6579, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6580 + * global __cuGraphicsUnmapResources + * cuPythonInit() + * if __cuGraphicsUnmapResources == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsUnmapResources" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnmapResources == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6581 + * cuPythonInit() + * if __cuGraphicsUnmapResources == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsUnmapResources" not found') + * err = ( __cuGraphicsUnmapResources)(count, resources, hStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6582 + * if __cuGraphicsUnmapResources == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphicsUnmapResources" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphicsUnmapResources)(count, resources, hStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphicsUnmapResource}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6582, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6582, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6581 + * cuPythonInit() + * if __cuGraphicsUnmapResources == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsUnmapResources" not found') + * err = ( __cuGraphicsUnmapResources)(count, resources, hStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6580 + * global __cuGraphicsUnmapResources + * cuPythonInit() + * if __cuGraphicsUnmapResources == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsUnmapResources" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6583 + * with gil: + * raise RuntimeError('Function "cuGraphicsUnmapResources" not found') + * err = ( __cuGraphicsUnmapResources)(count, resources, hStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(unsigned int, CUgraphicsResource *, CUstream))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnmapResources)(__pyx_v_count, __pyx_v_resources, __pyx_v_hStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6583, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6584 + * raise RuntimeError('Function "cuGraphicsUnmapResources" not found') + * err = ( __cuGraphicsUnmapResources)(count, resources, hStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGetProcAddress_v2(const char* symbol, void** pfn, int cudaVersion, cuuint64_t flags, CUdriverProcAddressQueryResult* symbolStatus) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6577 + * return err + * + * cdef CUresult _cuGraphicsUnmapResources(unsigned int count, CUgraphicsResource* resources, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsUnmapResources + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphicsUnmapResources", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6586 + * return err + * + * cdef CUresult _cuGetProcAddress_v2(const char* symbol, void** pfn, int cudaVersion, cuuint64_t flags, CUdriverProcAddressQueryResult* symbolStatus) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGetProcAddress_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGetProcAddress_v2(char const *__pyx_v_symbol, void **__pyx_v_pfn, int __pyx_v_cudaVersion, cuuint64_t __pyx_v_flags, CUdriverProcAddressQueryResult *__pyx_v_symbolStatus) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGetProcAddress_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6588 + * cdef CUresult _cuGetProcAddress_v2(const char* symbol, void** pfn, int cudaVersion, cuuint64_t flags, CUdriverProcAddressQueryResult* symbolStatus) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGetProcAddress_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGetProcAddress_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6588, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6589 + * global __cuGetProcAddress_v2 + * cuPythonInit() + * if __cuGetProcAddress_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGetProcAddress_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6590 + * cuPythonInit() + * if __cuGetProcAddress_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGetProcAddress_v2" not found') + * err = ( __cuGetProcAddress_v2)(symbol, pfn, cudaVersion, flags, symbolStatus) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6591 + * if __cuGetProcAddress_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGetProcAddress_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGetProcAddress_v2)(symbol, pfn, cudaVersion, flags, symbolStatus) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGetProcAddress_v2_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6591, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6591, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6590 + * cuPythonInit() + * if __cuGetProcAddress_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGetProcAddress_v2" not found') + * err = ( __cuGetProcAddress_v2)(symbol, pfn, cudaVersion, flags, symbolStatus) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6589 + * global __cuGetProcAddress_v2 + * cuPythonInit() + * if __cuGetProcAddress_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGetProcAddress_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6592 + * with gil: + * raise RuntimeError('Function "cuGetProcAddress_v2" not found') + * err = ( __cuGetProcAddress_v2)(symbol, pfn, cudaVersion, flags, symbolStatus) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(char const *, void **, int, cuuint64_t, CUdriverProcAddressQueryResult *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2)(__pyx_v_symbol, __pyx_v_pfn, __pyx_v_cudaVersion, __pyx_v_flags, __pyx_v_symbolStatus); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6592, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6593 + * raise RuntimeError('Function "cuGetProcAddress_v2" not found') + * err = ( __cuGetProcAddress_v2)(symbol, pfn, cudaVersion, flags, symbolStatus) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCoredumpGetAttribute(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6586 + * return err + * + * cdef CUresult _cuGetProcAddress_v2(const char* symbol, void** pfn, int cudaVersion, cuuint64_t flags, CUdriverProcAddressQueryResult* symbolStatus) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGetProcAddress_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGetProcAddress_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6595 + * return err + * + * cdef CUresult _cuCoredumpGetAttribute(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCoredumpGetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCoredumpGetAttribute(CUcoredumpSettings __pyx_v_attrib, void *__pyx_v_value, size_t *__pyx_v_size) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCoredumpGetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6597 + * cdef CUresult _cuCoredumpGetAttribute(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCoredumpGetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCoredumpGetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6597, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6598 + * global __cuCoredumpGetAttribute + * cuPythonInit() + * if __cuCoredumpGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCoredumpGetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6599 + * cuPythonInit() + * if __cuCoredumpGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCoredumpGetAttribute" not found') + * err = ( __cuCoredumpGetAttribute)(attrib, value, size) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6600 + * if __cuCoredumpGetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuCoredumpGetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuCoredumpGetAttribute)(attrib, value, size) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCoredumpGetAttribute}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6600, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6600, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6599 + * cuPythonInit() + * if __cuCoredumpGetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCoredumpGetAttribute" not found') + * err = ( __cuCoredumpGetAttribute)(attrib, value, size) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6598 + * global __cuCoredumpGetAttribute + * cuPythonInit() + * if __cuCoredumpGetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCoredumpGetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6601 + * with gil: + * raise RuntimeError('Function "cuCoredumpGetAttribute" not found') + * err = ( __cuCoredumpGetAttribute)(attrib, value, size) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcoredumpSettings, void *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttribute)(__pyx_v_attrib, __pyx_v_value, __pyx_v_size); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6601, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6602 + * raise RuntimeError('Function "cuCoredumpGetAttribute" not found') + * err = ( __cuCoredumpGetAttribute)(attrib, value, size) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCoredumpGetAttributeGlobal(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6595 + * return err + * + * cdef CUresult _cuCoredumpGetAttribute(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCoredumpGetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCoredumpGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6604 + * return err + * + * cdef CUresult _cuCoredumpGetAttributeGlobal(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCoredumpGetAttributeGlobal + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCoredumpGetAttributeGlobal(CUcoredumpSettings __pyx_v_attrib, void *__pyx_v_value, size_t *__pyx_v_size) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCoredumpGetAttributeGlobal", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6606 + * cdef CUresult _cuCoredumpGetAttributeGlobal(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCoredumpGetAttributeGlobal + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCoredumpGetAttributeGlobal == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6606, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6607 + * global __cuCoredumpGetAttributeGlobal + * cuPythonInit() + * if __cuCoredumpGetAttributeGlobal == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCoredumpGetAttributeGlobal" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttributeGlobal == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6608 + * cuPythonInit() + * if __cuCoredumpGetAttributeGlobal == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCoredumpGetAttributeGlobal" not found') + * err = ( __cuCoredumpGetAttributeGlobal)(attrib, value, size) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6609 + * if __cuCoredumpGetAttributeGlobal == NULL: + * with gil: + * raise RuntimeError('Function "cuCoredumpGetAttributeGlobal" not found') # <<<<<<<<<<<<<< + * err = ( __cuCoredumpGetAttributeGlobal)(attrib, value, size) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCoredumpGetAttributeG}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6609, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6609, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6608 + * cuPythonInit() + * if __cuCoredumpGetAttributeGlobal == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCoredumpGetAttributeGlobal" not found') + * err = ( __cuCoredumpGetAttributeGlobal)(attrib, value, size) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6607 + * global __cuCoredumpGetAttributeGlobal + * cuPythonInit() + * if __cuCoredumpGetAttributeGlobal == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCoredumpGetAttributeGlobal" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6610 + * with gil: + * raise RuntimeError('Function "cuCoredumpGetAttributeGlobal" not found') + * err = ( __cuCoredumpGetAttributeGlobal)(attrib, value, size) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcoredumpSettings, void *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttributeGlobal)(__pyx_v_attrib, __pyx_v_value, __pyx_v_size); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6610, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6611 + * raise RuntimeError('Function "cuCoredumpGetAttributeGlobal" not found') + * err = ( __cuCoredumpGetAttributeGlobal)(attrib, value, size) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCoredumpSetAttribute(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6604 + * return err + * + * cdef CUresult _cuCoredumpGetAttributeGlobal(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCoredumpGetAttributeGlobal + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCoredumpGetAttributeGlobal", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6613 + * return err + * + * cdef CUresult _cuCoredumpSetAttribute(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCoredumpSetAttribute + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCoredumpSetAttribute(CUcoredumpSettings __pyx_v_attrib, void *__pyx_v_value, size_t *__pyx_v_size) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCoredumpSetAttribute", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6615 + * cdef CUresult _cuCoredumpSetAttribute(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCoredumpSetAttribute + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCoredumpSetAttribute == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6615, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6616 + * global __cuCoredumpSetAttribute + * cuPythonInit() + * if __cuCoredumpSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCoredumpSetAttribute" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttribute == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6617 + * cuPythonInit() + * if __cuCoredumpSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCoredumpSetAttribute" not found') + * err = ( __cuCoredumpSetAttribute)(attrib, value, size) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6618 + * if __cuCoredumpSetAttribute == NULL: + * with gil: + * raise RuntimeError('Function "cuCoredumpSetAttribute" not found') # <<<<<<<<<<<<<< + * err = ( __cuCoredumpSetAttribute)(attrib, value, size) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCoredumpSetAttribute}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6618, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6618, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6617 + * cuPythonInit() + * if __cuCoredumpSetAttribute == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCoredumpSetAttribute" not found') + * err = ( __cuCoredumpSetAttribute)(attrib, value, size) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6616 + * global __cuCoredumpSetAttribute + * cuPythonInit() + * if __cuCoredumpSetAttribute == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCoredumpSetAttribute" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6619 + * with gil: + * raise RuntimeError('Function "cuCoredumpSetAttribute" not found') + * err = ( __cuCoredumpSetAttribute)(attrib, value, size) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcoredumpSettings, void *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttribute)(__pyx_v_attrib, __pyx_v_value, __pyx_v_size); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6619, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6620 + * raise RuntimeError('Function "cuCoredumpSetAttribute" not found') + * err = ( __cuCoredumpSetAttribute)(attrib, value, size) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCoredumpSetAttributeGlobal(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6613 + * return err + * + * cdef CUresult _cuCoredumpSetAttribute(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCoredumpSetAttribute + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCoredumpSetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6622 + * return err + * + * cdef CUresult _cuCoredumpSetAttributeGlobal(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCoredumpSetAttributeGlobal + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCoredumpSetAttributeGlobal(CUcoredumpSettings __pyx_v_attrib, void *__pyx_v_value, size_t *__pyx_v_size) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCoredumpSetAttributeGlobal", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6624 + * cdef CUresult _cuCoredumpSetAttributeGlobal(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCoredumpSetAttributeGlobal + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCoredumpSetAttributeGlobal == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6624, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6625 + * global __cuCoredumpSetAttributeGlobal + * cuPythonInit() + * if __cuCoredumpSetAttributeGlobal == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCoredumpSetAttributeGlobal" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttributeGlobal == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6626 + * cuPythonInit() + * if __cuCoredumpSetAttributeGlobal == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCoredumpSetAttributeGlobal" not found') + * err = ( __cuCoredumpSetAttributeGlobal)(attrib, value, size) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6627 + * if __cuCoredumpSetAttributeGlobal == NULL: + * with gil: + * raise RuntimeError('Function "cuCoredumpSetAttributeGlobal" not found') # <<<<<<<<<<<<<< + * err = ( __cuCoredumpSetAttributeGlobal)(attrib, value, size) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCoredumpSetAttributeG}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6627, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6627, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6626 + * cuPythonInit() + * if __cuCoredumpSetAttributeGlobal == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCoredumpSetAttributeGlobal" not found') + * err = ( __cuCoredumpSetAttributeGlobal)(attrib, value, size) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6625 + * global __cuCoredumpSetAttributeGlobal + * cuPythonInit() + * if __cuCoredumpSetAttributeGlobal == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCoredumpSetAttributeGlobal" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6628 + * with gil: + * raise RuntimeError('Function "cuCoredumpSetAttributeGlobal" not found') + * err = ( __cuCoredumpSetAttributeGlobal)(attrib, value, size) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcoredumpSettings, void *, size_t *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttributeGlobal)(__pyx_v_attrib, __pyx_v_value, __pyx_v_size); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6628, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6629 + * raise RuntimeError('Function "cuCoredumpSetAttributeGlobal" not found') + * err = ( __cuCoredumpSetAttributeGlobal)(attrib, value, size) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGetExportTable(const void** ppExportTable, const CUuuid* pExportTableId) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6622 + * return err + * + * cdef CUresult _cuCoredumpSetAttributeGlobal(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCoredumpSetAttributeGlobal + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCoredumpSetAttributeGlobal", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6631 + * return err + * + * cdef CUresult _cuGetExportTable(const void** ppExportTable, const CUuuid* pExportTableId) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGetExportTable + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGetExportTable(void const **__pyx_v_ppExportTable, CUuuid const *__pyx_v_pExportTableId) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGetExportTable", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6633 + * cdef CUresult _cuGetExportTable(const void** ppExportTable, const CUuuid* pExportTableId) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGetExportTable + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGetExportTable == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6633, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6634 + * global __cuGetExportTable + * cuPythonInit() + * if __cuGetExportTable == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGetExportTable" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetExportTable == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6635 + * cuPythonInit() + * if __cuGetExportTable == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGetExportTable" not found') + * err = ( __cuGetExportTable)(ppExportTable, pExportTableId) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6636 + * if __cuGetExportTable == NULL: + * with gil: + * raise RuntimeError('Function "cuGetExportTable" not found') # <<<<<<<<<<<<<< + * err = ( __cuGetExportTable)(ppExportTable, pExportTableId) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGetExportTable_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6636, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6636, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6635 + * cuPythonInit() + * if __cuGetExportTable == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGetExportTable" not found') + * err = ( __cuGetExportTable)(ppExportTable, pExportTableId) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6634 + * global __cuGetExportTable + * cuPythonInit() + * if __cuGetExportTable == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGetExportTable" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6637 + * with gil: + * raise RuntimeError('Function "cuGetExportTable" not found') + * err = ( __cuGetExportTable)(ppExportTable, pExportTableId) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void const **, CUuuid const *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetExportTable)(__pyx_v_ppExportTable, __pyx_v_pExportTableId); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6637, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6638 + * raise RuntimeError('Function "cuGetExportTable" not found') + * err = ( __cuGetExportTable)(ppExportTable, pExportTableId) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGreenCtxCreate(CUgreenCtx* phCtx, CUdevResourceDesc desc, CUdevice dev, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6631 + * return err + * + * cdef CUresult _cuGetExportTable(const void** ppExportTable, const CUuuid* pExportTableId) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGetExportTable + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGetExportTable", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6640 + * return err + * + * cdef CUresult _cuGreenCtxCreate(CUgreenCtx* phCtx, CUdevResourceDesc desc, CUdevice dev, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGreenCtxCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGreenCtxCreate(CUgreenCtx *__pyx_v_phCtx, CUdevResourceDesc __pyx_v_desc, CUdevice __pyx_v_dev, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGreenCtxCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6642 + * cdef CUresult _cuGreenCtxCreate(CUgreenCtx* phCtx, CUdevResourceDesc desc, CUdevice dev, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGreenCtxCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGreenCtxCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6642, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6643 + * global __cuGreenCtxCreate + * cuPythonInit() + * if __cuGreenCtxCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGreenCtxCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6644 + * cuPythonInit() + * if __cuGreenCtxCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGreenCtxCreate" not found') + * err = ( __cuGreenCtxCreate)(phCtx, desc, dev, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6645 + * if __cuGreenCtxCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuGreenCtxCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuGreenCtxCreate)(phCtx, desc, dev, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGreenCtxCreate_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6645, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6645, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6644 + * cuPythonInit() + * if __cuGreenCtxCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGreenCtxCreate" not found') + * err = ( __cuGreenCtxCreate)(phCtx, desc, dev, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6643 + * global __cuGreenCtxCreate + * cuPythonInit() + * if __cuGreenCtxCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGreenCtxCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6646 + * with gil: + * raise RuntimeError('Function "cuGreenCtxCreate" not found') + * err = ( __cuGreenCtxCreate)(phCtx, desc, dev, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgreenCtx *, CUdevResourceDesc, CUdevice, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxCreate)(__pyx_v_phCtx, __pyx_v_desc, __pyx_v_dev, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6646, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6647 + * raise RuntimeError('Function "cuGreenCtxCreate" not found') + * err = ( __cuGreenCtxCreate)(phCtx, desc, dev, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGreenCtxDestroy(CUgreenCtx hCtx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6640 + * return err + * + * cdef CUresult _cuGreenCtxCreate(CUgreenCtx* phCtx, CUdevResourceDesc desc, CUdevice dev, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGreenCtxCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGreenCtxCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6649 + * return err + * + * cdef CUresult _cuGreenCtxDestroy(CUgreenCtx hCtx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGreenCtxDestroy + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGreenCtxDestroy(CUgreenCtx __pyx_v_hCtx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGreenCtxDestroy", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6651 + * cdef CUresult _cuGreenCtxDestroy(CUgreenCtx hCtx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGreenCtxDestroy + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGreenCtxDestroy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6651, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6652 + * global __cuGreenCtxDestroy + * cuPythonInit() + * if __cuGreenCtxDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGreenCtxDestroy" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxDestroy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6653 + * cuPythonInit() + * if __cuGreenCtxDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGreenCtxDestroy" not found') + * err = ( __cuGreenCtxDestroy)(hCtx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6654 + * if __cuGreenCtxDestroy == NULL: + * with gil: + * raise RuntimeError('Function "cuGreenCtxDestroy" not found') # <<<<<<<<<<<<<< + * err = ( __cuGreenCtxDestroy)(hCtx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGreenCtxDestroy_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6654, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6654, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6653 + * cuPythonInit() + * if __cuGreenCtxDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGreenCtxDestroy" not found') + * err = ( __cuGreenCtxDestroy)(hCtx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6652 + * global __cuGreenCtxDestroy + * cuPythonInit() + * if __cuGreenCtxDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGreenCtxDestroy" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6655 + * with gil: + * raise RuntimeError('Function "cuGreenCtxDestroy" not found') + * err = ( __cuGreenCtxDestroy)(hCtx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgreenCtx))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxDestroy)(__pyx_v_hCtx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6655, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6656 + * raise RuntimeError('Function "cuGreenCtxDestroy" not found') + * err = ( __cuGreenCtxDestroy)(hCtx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxFromGreenCtx(CUcontext* pContext, CUgreenCtx hCtx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6649 + * return err + * + * cdef CUresult _cuGreenCtxDestroy(CUgreenCtx hCtx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGreenCtxDestroy + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGreenCtxDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6658 + * return err + * + * cdef CUresult _cuCtxFromGreenCtx(CUcontext* pContext, CUgreenCtx hCtx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxFromGreenCtx + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxFromGreenCtx(CUcontext *__pyx_v_pContext, CUgreenCtx __pyx_v_hCtx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxFromGreenCtx", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6660 + * cdef CUresult _cuCtxFromGreenCtx(CUcontext* pContext, CUgreenCtx hCtx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxFromGreenCtx + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxFromGreenCtx == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6660, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6661 + * global __cuCtxFromGreenCtx + * cuPythonInit() + * if __cuCtxFromGreenCtx == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxFromGreenCtx" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxFromGreenCtx == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6662 + * cuPythonInit() + * if __cuCtxFromGreenCtx == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxFromGreenCtx" not found') + * err = ( __cuCtxFromGreenCtx)(pContext, hCtx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6663 + * if __cuCtxFromGreenCtx == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxFromGreenCtx" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxFromGreenCtx)(pContext, hCtx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxFromGreenCtx_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6663, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6663, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6662 + * cuPythonInit() + * if __cuCtxFromGreenCtx == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxFromGreenCtx" not found') + * err = ( __cuCtxFromGreenCtx)(pContext, hCtx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6661 + * global __cuCtxFromGreenCtx + * cuPythonInit() + * if __cuCtxFromGreenCtx == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxFromGreenCtx" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6664 + * with gil: + * raise RuntimeError('Function "cuCtxFromGreenCtx" not found') + * err = ( __cuCtxFromGreenCtx)(pContext, hCtx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext *, CUgreenCtx))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxFromGreenCtx)(__pyx_v_pContext, __pyx_v_hCtx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6664, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6665 + * raise RuntimeError('Function "cuCtxFromGreenCtx" not found') + * err = ( __cuCtxFromGreenCtx)(pContext, hCtx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDeviceGetDevResource(CUdevice device, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6658 + * return err + * + * cdef CUresult _cuCtxFromGreenCtx(CUcontext* pContext, CUgreenCtx hCtx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxFromGreenCtx + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxFromGreenCtx", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6667 + * return err + * + * cdef CUresult _cuDeviceGetDevResource(CUdevice device, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetDevResource + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetDevResource(CUdevice __pyx_v_device, CUdevResource *__pyx_v_resource, CUdevResourceType __pyx_v_typename) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDeviceGetDevResource", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6669 + * cdef CUresult _cuDeviceGetDevResource(CUdevice device, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDeviceGetDevResource + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDeviceGetDevResource == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6669, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6670 + * global __cuDeviceGetDevResource + * cuPythonInit() + * if __cuDeviceGetDevResource == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetDevResource" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDevResource == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6671 + * cuPythonInit() + * if __cuDeviceGetDevResource == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetDevResource" not found') + * err = ( __cuDeviceGetDevResource)(device, resource, typename) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6672 + * if __cuDeviceGetDevResource == NULL: + * with gil: + * raise RuntimeError('Function "cuDeviceGetDevResource" not found') # <<<<<<<<<<<<<< + * err = ( __cuDeviceGetDevResource)(device, resource, typename) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDeviceGetDevResource}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6672, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6672, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6671 + * cuPythonInit() + * if __cuDeviceGetDevResource == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDeviceGetDevResource" not found') + * err = ( __cuDeviceGetDevResource)(device, resource, typename) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6670 + * global __cuDeviceGetDevResource + * cuPythonInit() + * if __cuDeviceGetDevResource == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDeviceGetDevResource" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6673 + * with gil: + * raise RuntimeError('Function "cuDeviceGetDevResource" not found') + * err = ( __cuDeviceGetDevResource)(device, resource, typename) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice, CUdevResource *, CUdevResourceType))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDevResource)(__pyx_v_device, __pyx_v_resource, __pyx_v_typename); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6673, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6674 + * raise RuntimeError('Function "cuDeviceGetDevResource" not found') + * err = ( __cuDeviceGetDevResource)(device, resource, typename) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCtxGetDevResource(CUcontext hCtx, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6667 + * return err + * + * cdef CUresult _cuDeviceGetDevResource(CUdevice device, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDeviceGetDevResource + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDeviceGetDevResource", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6676 + * return err + * + * cdef CUresult _cuCtxGetDevResource(CUcontext hCtx, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetDevResource + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetDevResource(CUcontext __pyx_v_hCtx, CUdevResource *__pyx_v_resource, CUdevResourceType __pyx_v_typename) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCtxGetDevResource", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6678 + * cdef CUresult _cuCtxGetDevResource(CUcontext hCtx, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCtxGetDevResource + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCtxGetDevResource == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6678, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6679 + * global __cuCtxGetDevResource + * cuPythonInit() + * if __cuCtxGetDevResource == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetDevResource" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevResource == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6680 + * cuPythonInit() + * if __cuCtxGetDevResource == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetDevResource" not found') + * err = ( __cuCtxGetDevResource)(hCtx, resource, typename) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6681 + * if __cuCtxGetDevResource == NULL: + * with gil: + * raise RuntimeError('Function "cuCtxGetDevResource" not found') # <<<<<<<<<<<<<< + * err = ( __cuCtxGetDevResource)(hCtx, resource, typename) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCtxGetDevResource_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6681, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6681, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6680 + * cuPythonInit() + * if __cuCtxGetDevResource == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCtxGetDevResource" not found') + * err = ( __cuCtxGetDevResource)(hCtx, resource, typename) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6679 + * global __cuCtxGetDevResource + * cuPythonInit() + * if __cuCtxGetDevResource == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCtxGetDevResource" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6682 + * with gil: + * raise RuntimeError('Function "cuCtxGetDevResource" not found') + * err = ( __cuCtxGetDevResource)(hCtx, resource, typename) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext, CUdevResource *, CUdevResourceType))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevResource)(__pyx_v_hCtx, __pyx_v_resource, __pyx_v_typename); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6682, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6683 + * raise RuntimeError('Function "cuCtxGetDevResource" not found') + * err = ( __cuCtxGetDevResource)(hCtx, resource, typename) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGreenCtxGetDevResource(CUgreenCtx hCtx, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6676 + * return err + * + * cdef CUresult _cuCtxGetDevResource(CUcontext hCtx, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCtxGetDevResource + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCtxGetDevResource", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6685 + * return err + * + * cdef CUresult _cuGreenCtxGetDevResource(CUgreenCtx hCtx, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGreenCtxGetDevResource + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGreenCtxGetDevResource(CUgreenCtx __pyx_v_hCtx, CUdevResource *__pyx_v_resource, CUdevResourceType __pyx_v_typename) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGreenCtxGetDevResource", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6687 + * cdef CUresult _cuGreenCtxGetDevResource(CUgreenCtx hCtx, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGreenCtxGetDevResource + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGreenCtxGetDevResource == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6687, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6688 + * global __cuGreenCtxGetDevResource + * cuPythonInit() + * if __cuGreenCtxGetDevResource == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGreenCtxGetDevResource" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxGetDevResource == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6689 + * cuPythonInit() + * if __cuGreenCtxGetDevResource == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGreenCtxGetDevResource" not found') + * err = ( __cuGreenCtxGetDevResource)(hCtx, resource, typename) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6690 + * if __cuGreenCtxGetDevResource == NULL: + * with gil: + * raise RuntimeError('Function "cuGreenCtxGetDevResource" not found') # <<<<<<<<<<<<<< + * err = ( __cuGreenCtxGetDevResource)(hCtx, resource, typename) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGreenCtxGetDevResourc}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6690, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6690, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6689 + * cuPythonInit() + * if __cuGreenCtxGetDevResource == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGreenCtxGetDevResource" not found') + * err = ( __cuGreenCtxGetDevResource)(hCtx, resource, typename) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6688 + * global __cuGreenCtxGetDevResource + * cuPythonInit() + * if __cuGreenCtxGetDevResource == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGreenCtxGetDevResource" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6691 + * with gil: + * raise RuntimeError('Function "cuGreenCtxGetDevResource" not found') + * err = ( __cuGreenCtxGetDevResource)(hCtx, resource, typename) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgreenCtx, CUdevResource *, CUdevResourceType))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxGetDevResource)(__pyx_v_hCtx, __pyx_v_resource, __pyx_v_typename); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6691, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6692 + * raise RuntimeError('Function "cuGreenCtxGetDevResource" not found') + * err = ( __cuGreenCtxGetDevResource)(hCtx, resource, typename) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDevSmResourceSplitByCount(CUdevResource* result, unsigned int* nbGroups, const CUdevResource* input, CUdevResource* remaining, unsigned int useFlags, unsigned int minCount) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6685 + * return err + * + * cdef CUresult _cuGreenCtxGetDevResource(CUgreenCtx hCtx, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGreenCtxGetDevResource + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGreenCtxGetDevResource", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6694 + * return err + * + * cdef CUresult _cuDevSmResourceSplitByCount(CUdevResource* result, unsigned int* nbGroups, const CUdevResource* input, CUdevResource* remaining, unsigned int useFlags, unsigned int minCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevSmResourceSplitByCount + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevSmResourceSplitByCount(CUdevResource *__pyx_v_result, unsigned int *__pyx_v_nbGroups, CUdevResource const *__pyx_v_input, CUdevResource *__pyx_v_remaining, unsigned int __pyx_v_useFlags, unsigned int __pyx_v_minCount) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDevSmResourceSplitByCount", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6696 + * cdef CUresult _cuDevSmResourceSplitByCount(CUdevResource* result, unsigned int* nbGroups, const CUdevResource* input, CUdevResource* remaining, unsigned int useFlags, unsigned int minCount) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDevSmResourceSplitByCount + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDevSmResourceSplitByCount == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6696, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6697 + * global __cuDevSmResourceSplitByCount + * cuPythonInit() + * if __cuDevSmResourceSplitByCount == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevSmResourceSplitByCount" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevSmResourceSplitByCount == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6698 + * cuPythonInit() + * if __cuDevSmResourceSplitByCount == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevSmResourceSplitByCount" not found') + * err = ( __cuDevSmResourceSplitByCount)(result, nbGroups, input, remaining, useFlags, minCount) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6699 + * if __cuDevSmResourceSplitByCount == NULL: + * with gil: + * raise RuntimeError('Function "cuDevSmResourceSplitByCount" not found') # <<<<<<<<<<<<<< + * err = ( __cuDevSmResourceSplitByCount)(result, nbGroups, input, remaining, useFlags, minCount) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDevSmResourceSplitByC}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6699, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6699, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6698 + * cuPythonInit() + * if __cuDevSmResourceSplitByCount == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevSmResourceSplitByCount" not found') + * err = ( __cuDevSmResourceSplitByCount)(result, nbGroups, input, remaining, useFlags, minCount) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6697 + * global __cuDevSmResourceSplitByCount + * cuPythonInit() + * if __cuDevSmResourceSplitByCount == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevSmResourceSplitByCount" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6700 + * with gil: + * raise RuntimeError('Function "cuDevSmResourceSplitByCount" not found') + * err = ( __cuDevSmResourceSplitByCount)(result, nbGroups, input, remaining, useFlags, minCount) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevResource *, unsigned int *, CUdevResource const *, CUdevResource *, unsigned int, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevSmResourceSplitByCount)(__pyx_v_result, __pyx_v_nbGroups, __pyx_v_input, __pyx_v_remaining, __pyx_v_useFlags, __pyx_v_minCount); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6700, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6701 + * raise RuntimeError('Function "cuDevSmResourceSplitByCount" not found') + * err = ( __cuDevSmResourceSplitByCount)(result, nbGroups, input, remaining, useFlags, minCount) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuDevResourceGenerateDesc(CUdevResourceDesc* phDesc, CUdevResource* resources, unsigned int nbResources) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6694 + * return err + * + * cdef CUresult _cuDevSmResourceSplitByCount(CUdevResource* result, unsigned int* nbGroups, const CUdevResource* input, CUdevResource* remaining, unsigned int useFlags, unsigned int minCount) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevSmResourceSplitByCount + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDevSmResourceSplitByCount", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6703 + * return err + * + * cdef CUresult _cuDevResourceGenerateDesc(CUdevResourceDesc* phDesc, CUdevResource* resources, unsigned int nbResources) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevResourceGenerateDesc + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevResourceGenerateDesc(CUdevResourceDesc *__pyx_v_phDesc, CUdevResource *__pyx_v_resources, unsigned int __pyx_v_nbResources) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuDevResourceGenerateDesc", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6705 + * cdef CUresult _cuDevResourceGenerateDesc(CUdevResourceDesc* phDesc, CUdevResource* resources, unsigned int nbResources) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuDevResourceGenerateDesc + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuDevResourceGenerateDesc == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6705, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6706 + * global __cuDevResourceGenerateDesc + * cuPythonInit() + * if __cuDevResourceGenerateDesc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevResourceGenerateDesc" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevResourceGenerateDesc == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6707 + * cuPythonInit() + * if __cuDevResourceGenerateDesc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevResourceGenerateDesc" not found') + * err = ( __cuDevResourceGenerateDesc)(phDesc, resources, nbResources) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6708 + * if __cuDevResourceGenerateDesc == NULL: + * with gil: + * raise RuntimeError('Function "cuDevResourceGenerateDesc" not found') # <<<<<<<<<<<<<< + * err = ( __cuDevResourceGenerateDesc)(phDesc, resources, nbResources) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuDevResourceGenerateDe}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6708, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6708, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6707 + * cuPythonInit() + * if __cuDevResourceGenerateDesc == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuDevResourceGenerateDesc" not found') + * err = ( __cuDevResourceGenerateDesc)(phDesc, resources, nbResources) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6706 + * global __cuDevResourceGenerateDesc + * cuPythonInit() + * if __cuDevResourceGenerateDesc == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuDevResourceGenerateDesc" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6709 + * with gil: + * raise RuntimeError('Function "cuDevResourceGenerateDesc" not found') + * err = ( __cuDevResourceGenerateDesc)(phDesc, resources, nbResources) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevResourceDesc *, CUdevResource *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevResourceGenerateDesc)(__pyx_v_phDesc, __pyx_v_resources, __pyx_v_nbResources); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6709, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6710 + * raise RuntimeError('Function "cuDevResourceGenerateDesc" not found') + * err = ( __cuDevResourceGenerateDesc)(phDesc, resources, nbResources) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGreenCtxRecordEvent(CUgreenCtx hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6703 + * return err + * + * cdef CUresult _cuDevResourceGenerateDesc(CUdevResourceDesc* phDesc, CUdevResource* resources, unsigned int nbResources) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuDevResourceGenerateDesc + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuDevResourceGenerateDesc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6712 + * return err + * + * cdef CUresult _cuGreenCtxRecordEvent(CUgreenCtx hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGreenCtxRecordEvent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGreenCtxRecordEvent(CUgreenCtx __pyx_v_hCtx, CUevent __pyx_v_hEvent) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGreenCtxRecordEvent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6714 + * cdef CUresult _cuGreenCtxRecordEvent(CUgreenCtx hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGreenCtxRecordEvent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGreenCtxRecordEvent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6714, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6715 + * global __cuGreenCtxRecordEvent + * cuPythonInit() + * if __cuGreenCtxRecordEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGreenCtxRecordEvent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxRecordEvent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6716 + * cuPythonInit() + * if __cuGreenCtxRecordEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGreenCtxRecordEvent" not found') + * err = ( __cuGreenCtxRecordEvent)(hCtx, hEvent) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6717 + * if __cuGreenCtxRecordEvent == NULL: + * with gil: + * raise RuntimeError('Function "cuGreenCtxRecordEvent" not found') # <<<<<<<<<<<<<< + * err = ( __cuGreenCtxRecordEvent)(hCtx, hEvent) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGreenCtxRecordEvent_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6717, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6717, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6716 + * cuPythonInit() + * if __cuGreenCtxRecordEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGreenCtxRecordEvent" not found') + * err = ( __cuGreenCtxRecordEvent)(hCtx, hEvent) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6715 + * global __cuGreenCtxRecordEvent + * cuPythonInit() + * if __cuGreenCtxRecordEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGreenCtxRecordEvent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6718 + * with gil: + * raise RuntimeError('Function "cuGreenCtxRecordEvent" not found') + * err = ( __cuGreenCtxRecordEvent)(hCtx, hEvent) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgreenCtx, CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxRecordEvent)(__pyx_v_hCtx, __pyx_v_hEvent); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6718, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6719 + * raise RuntimeError('Function "cuGreenCtxRecordEvent" not found') + * err = ( __cuGreenCtxRecordEvent)(hCtx, hEvent) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGreenCtxWaitEvent(CUgreenCtx hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6712 + * return err + * + * cdef CUresult _cuGreenCtxRecordEvent(CUgreenCtx hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGreenCtxRecordEvent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGreenCtxRecordEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6721 + * return err + * + * cdef CUresult _cuGreenCtxWaitEvent(CUgreenCtx hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGreenCtxWaitEvent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGreenCtxWaitEvent(CUgreenCtx __pyx_v_hCtx, CUevent __pyx_v_hEvent) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGreenCtxWaitEvent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6723 + * cdef CUresult _cuGreenCtxWaitEvent(CUgreenCtx hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGreenCtxWaitEvent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGreenCtxWaitEvent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6723, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6724 + * global __cuGreenCtxWaitEvent + * cuPythonInit() + * if __cuGreenCtxWaitEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGreenCtxWaitEvent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxWaitEvent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6725 + * cuPythonInit() + * if __cuGreenCtxWaitEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGreenCtxWaitEvent" not found') + * err = ( __cuGreenCtxWaitEvent)(hCtx, hEvent) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6726 + * if __cuGreenCtxWaitEvent == NULL: + * with gil: + * raise RuntimeError('Function "cuGreenCtxWaitEvent" not found') # <<<<<<<<<<<<<< + * err = ( __cuGreenCtxWaitEvent)(hCtx, hEvent) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGreenCtxWaitEvent_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6726, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6726, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6725 + * cuPythonInit() + * if __cuGreenCtxWaitEvent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGreenCtxWaitEvent" not found') + * err = ( __cuGreenCtxWaitEvent)(hCtx, hEvent) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6724 + * global __cuGreenCtxWaitEvent + * cuPythonInit() + * if __cuGreenCtxWaitEvent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGreenCtxWaitEvent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6727 + * with gil: + * raise RuntimeError('Function "cuGreenCtxWaitEvent" not found') + * err = ( __cuGreenCtxWaitEvent)(hCtx, hEvent) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgreenCtx, CUevent))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxWaitEvent)(__pyx_v_hCtx, __pyx_v_hEvent); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6727, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6728 + * raise RuntimeError('Function "cuGreenCtxWaitEvent" not found') + * err = ( __cuGreenCtxWaitEvent)(hCtx, hEvent) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuStreamGetGreenCtx(CUstream hStream, CUgreenCtx* phCtx) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6721 + * return err + * + * cdef CUresult _cuGreenCtxWaitEvent(CUgreenCtx hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGreenCtxWaitEvent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGreenCtxWaitEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6730 + * return err + * + * cdef CUresult _cuStreamGetGreenCtx(CUstream hStream, CUgreenCtx* phCtx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetGreenCtx + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetGreenCtx(CUstream __pyx_v_hStream, CUgreenCtx *__pyx_v_phCtx) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuStreamGetGreenCtx", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6732 + * cdef CUresult _cuStreamGetGreenCtx(CUstream hStream, CUgreenCtx* phCtx) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuStreamGetGreenCtx + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuStreamGetGreenCtx == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6732, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6733 + * global __cuStreamGetGreenCtx + * cuPythonInit() + * if __cuStreamGetGreenCtx == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetGreenCtx" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetGreenCtx == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6734 + * cuPythonInit() + * if __cuStreamGetGreenCtx == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetGreenCtx" not found') + * err = ( __cuStreamGetGreenCtx)(hStream, phCtx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6735 + * if __cuStreamGetGreenCtx == NULL: + * with gil: + * raise RuntimeError('Function "cuStreamGetGreenCtx" not found') # <<<<<<<<<<<<<< + * err = ( __cuStreamGetGreenCtx)(hStream, phCtx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuStreamGetGreenCtx_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6735, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6735, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6734 + * cuPythonInit() + * if __cuStreamGetGreenCtx == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuStreamGetGreenCtx" not found') + * err = ( __cuStreamGetGreenCtx)(hStream, phCtx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6733 + * global __cuStreamGetGreenCtx + * cuPythonInit() + * if __cuStreamGetGreenCtx == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuStreamGetGreenCtx" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6736 + * with gil: + * raise RuntimeError('Function "cuStreamGetGreenCtx" not found') + * err = ( __cuStreamGetGreenCtx)(hStream, phCtx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream, CUgreenCtx *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetGreenCtx)(__pyx_v_hStream, __pyx_v_phCtx); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6736, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6737 + * raise RuntimeError('Function "cuStreamGetGreenCtx" not found') + * err = ( __cuStreamGetGreenCtx)(hStream, phCtx) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGreenCtxStreamCreate(CUstream* phStream, CUgreenCtx greenCtx, unsigned int flags, int priority) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6730 + * return err + * + * cdef CUresult _cuStreamGetGreenCtx(CUstream hStream, CUgreenCtx* phCtx) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuStreamGetGreenCtx + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuStreamGetGreenCtx", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6739 + * return err + * + * cdef CUresult _cuGreenCtxStreamCreate(CUstream* phStream, CUgreenCtx greenCtx, unsigned int flags, int priority) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGreenCtxStreamCreate + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGreenCtxStreamCreate(CUstream *__pyx_v_phStream, CUgreenCtx __pyx_v_greenCtx, unsigned int __pyx_v_flags, int __pyx_v_priority) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGreenCtxStreamCreate", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6741 + * cdef CUresult _cuGreenCtxStreamCreate(CUstream* phStream, CUgreenCtx greenCtx, unsigned int flags, int priority) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGreenCtxStreamCreate + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGreenCtxStreamCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6741, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6742 + * global __cuGreenCtxStreamCreate + * cuPythonInit() + * if __cuGreenCtxStreamCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGreenCtxStreamCreate" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxStreamCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6743 + * cuPythonInit() + * if __cuGreenCtxStreamCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGreenCtxStreamCreate" not found') + * err = ( __cuGreenCtxStreamCreate)(phStream, greenCtx, flags, priority) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6744 + * if __cuGreenCtxStreamCreate == NULL: + * with gil: + * raise RuntimeError('Function "cuGreenCtxStreamCreate" not found') # <<<<<<<<<<<<<< + * err = ( __cuGreenCtxStreamCreate)(phStream, greenCtx, flags, priority) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGreenCtxStreamCreate}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6744, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6744, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6743 + * cuPythonInit() + * if __cuGreenCtxStreamCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGreenCtxStreamCreate" not found') + * err = ( __cuGreenCtxStreamCreate)(phStream, greenCtx, flags, priority) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6742 + * global __cuGreenCtxStreamCreate + * cuPythonInit() + * if __cuGreenCtxStreamCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGreenCtxStreamCreate" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6745 + * with gil: + * raise RuntimeError('Function "cuGreenCtxStreamCreate" not found') + * err = ( __cuGreenCtxStreamCreate)(phStream, greenCtx, flags, priority) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUstream *, CUgreenCtx, unsigned int, int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxStreamCreate)(__pyx_v_phStream, __pyx_v_greenCtx, __pyx_v_flags, __pyx_v_priority); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6745, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6746 + * raise RuntimeError('Function "cuGreenCtxStreamCreate" not found') + * err = ( __cuGreenCtxStreamCreate)(phStream, greenCtx, flags, priority) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLogsRegisterCallback(CUlogsCallback callbackFunc, void* userData, CUlogsCallbackHandle* callback_out) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6739 + * return err + * + * cdef CUresult _cuGreenCtxStreamCreate(CUstream* phStream, CUgreenCtx greenCtx, unsigned int flags, int priority) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGreenCtxStreamCreate + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGreenCtxStreamCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6748 + * return err + * + * cdef CUresult _cuLogsRegisterCallback(CUlogsCallback callbackFunc, void* userData, CUlogsCallbackHandle* callback_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLogsRegisterCallback + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLogsRegisterCallback(CUlogsCallback __pyx_v_callbackFunc, void *__pyx_v_userData, CUlogsCallbackHandle *__pyx_v_callback_out) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLogsRegisterCallback", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6750 + * cdef CUresult _cuLogsRegisterCallback(CUlogsCallback callbackFunc, void* userData, CUlogsCallbackHandle* callback_out) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLogsRegisterCallback + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLogsRegisterCallback == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6750, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6751 + * global __cuLogsRegisterCallback + * cuPythonInit() + * if __cuLogsRegisterCallback == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLogsRegisterCallback" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsRegisterCallback == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6752 + * cuPythonInit() + * if __cuLogsRegisterCallback == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLogsRegisterCallback" not found') + * err = ( __cuLogsRegisterCallback)(callbackFunc, userData, callback_out) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6753 + * if __cuLogsRegisterCallback == NULL: + * with gil: + * raise RuntimeError('Function "cuLogsRegisterCallback" not found') # <<<<<<<<<<<<<< + * err = ( __cuLogsRegisterCallback)(callbackFunc, userData, callback_out) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLogsRegisterCallback}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6753, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6753, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6752 + * cuPythonInit() + * if __cuLogsRegisterCallback == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLogsRegisterCallback" not found') + * err = ( __cuLogsRegisterCallback)(callbackFunc, userData, callback_out) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6751 + * global __cuLogsRegisterCallback + * cuPythonInit() + * if __cuLogsRegisterCallback == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLogsRegisterCallback" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6754 + * with gil: + * raise RuntimeError('Function "cuLogsRegisterCallback" not found') + * err = ( __cuLogsRegisterCallback)(callbackFunc, userData, callback_out) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlogsCallback, void *, CUlogsCallbackHandle *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsRegisterCallback)(__pyx_v_callbackFunc, __pyx_v_userData, __pyx_v_callback_out); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6754, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6755 + * raise RuntimeError('Function "cuLogsRegisterCallback" not found') + * err = ( __cuLogsRegisterCallback)(callbackFunc, userData, callback_out) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLogsUnregisterCallback(CUlogsCallbackHandle callback) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6748 + * return err + * + * cdef CUresult _cuLogsRegisterCallback(CUlogsCallback callbackFunc, void* userData, CUlogsCallbackHandle* callback_out) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLogsRegisterCallback + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLogsRegisterCallback", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6757 + * return err + * + * cdef CUresult _cuLogsUnregisterCallback(CUlogsCallbackHandle callback) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLogsUnregisterCallback + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLogsUnregisterCallback(CUlogsCallbackHandle __pyx_v_callback) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLogsUnregisterCallback", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6759 + * cdef CUresult _cuLogsUnregisterCallback(CUlogsCallbackHandle callback) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLogsUnregisterCallback + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLogsUnregisterCallback == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6759, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6760 + * global __cuLogsUnregisterCallback + * cuPythonInit() + * if __cuLogsUnregisterCallback == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLogsUnregisterCallback" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsUnregisterCallback == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6761 + * cuPythonInit() + * if __cuLogsUnregisterCallback == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLogsUnregisterCallback" not found') + * err = ( __cuLogsUnregisterCallback)(callback) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6762 + * if __cuLogsUnregisterCallback == NULL: + * with gil: + * raise RuntimeError('Function "cuLogsUnregisterCallback" not found') # <<<<<<<<<<<<<< + * err = ( __cuLogsUnregisterCallback)(callback) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLogsUnregisterCallbac}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6762, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6762, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6761 + * cuPythonInit() + * if __cuLogsUnregisterCallback == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLogsUnregisterCallback" not found') + * err = ( __cuLogsUnregisterCallback)(callback) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6760 + * global __cuLogsUnregisterCallback + * cuPythonInit() + * if __cuLogsUnregisterCallback == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLogsUnregisterCallback" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6763 + * with gil: + * raise RuntimeError('Function "cuLogsUnregisterCallback" not found') + * err = ( __cuLogsUnregisterCallback)(callback) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlogsCallbackHandle))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsUnregisterCallback)(__pyx_v_callback); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6763, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6764 + * raise RuntimeError('Function "cuLogsUnregisterCallback" not found') + * err = ( __cuLogsUnregisterCallback)(callback) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLogsCurrent(CUlogIterator* iterator_out, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6757 + * return err + * + * cdef CUresult _cuLogsUnregisterCallback(CUlogsCallbackHandle callback) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLogsUnregisterCallback + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLogsUnregisterCallback", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6766 + * return err + * + * cdef CUresult _cuLogsCurrent(CUlogIterator* iterator_out, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLogsCurrent + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLogsCurrent(CUlogIterator *__pyx_v_iterator_out, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLogsCurrent", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6768 + * cdef CUresult _cuLogsCurrent(CUlogIterator* iterator_out, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLogsCurrent + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLogsCurrent == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6768, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6769 + * global __cuLogsCurrent + * cuPythonInit() + * if __cuLogsCurrent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLogsCurrent" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsCurrent == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6770 + * cuPythonInit() + * if __cuLogsCurrent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLogsCurrent" not found') + * err = ( __cuLogsCurrent)(iterator_out, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6771 + * if __cuLogsCurrent == NULL: + * with gil: + * raise RuntimeError('Function "cuLogsCurrent" not found') # <<<<<<<<<<<<<< + * err = ( __cuLogsCurrent)(iterator_out, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLogsCurrent_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6771, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6771, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6770 + * cuPythonInit() + * if __cuLogsCurrent == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLogsCurrent" not found') + * err = ( __cuLogsCurrent)(iterator_out, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6769 + * global __cuLogsCurrent + * cuPythonInit() + * if __cuLogsCurrent == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLogsCurrent" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6772 + * with gil: + * raise RuntimeError('Function "cuLogsCurrent" not found') + * err = ( __cuLogsCurrent)(iterator_out, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlogIterator *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsCurrent)(__pyx_v_iterator_out, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6772, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6773 + * raise RuntimeError('Function "cuLogsCurrent" not found') + * err = ( __cuLogsCurrent)(iterator_out, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLogsDumpToFile(CUlogIterator* iterator, const char* pathToFile, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6766 + * return err + * + * cdef CUresult _cuLogsCurrent(CUlogIterator* iterator_out, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLogsCurrent + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLogsCurrent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6775 + * return err + * + * cdef CUresult _cuLogsDumpToFile(CUlogIterator* iterator, const char* pathToFile, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLogsDumpToFile + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLogsDumpToFile(CUlogIterator *__pyx_v_iterator, char const *__pyx_v_pathToFile, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLogsDumpToFile", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6777 + * cdef CUresult _cuLogsDumpToFile(CUlogIterator* iterator, const char* pathToFile, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLogsDumpToFile + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLogsDumpToFile == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6777, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6778 + * global __cuLogsDumpToFile + * cuPythonInit() + * if __cuLogsDumpToFile == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLogsDumpToFile" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToFile == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6779 + * cuPythonInit() + * if __cuLogsDumpToFile == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLogsDumpToFile" not found') + * err = ( __cuLogsDumpToFile)(iterator, pathToFile, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6780 + * if __cuLogsDumpToFile == NULL: + * with gil: + * raise RuntimeError('Function "cuLogsDumpToFile" not found') # <<<<<<<<<<<<<< + * err = ( __cuLogsDumpToFile)(iterator, pathToFile, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLogsDumpToFile_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6780, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6780, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6779 + * cuPythonInit() + * if __cuLogsDumpToFile == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLogsDumpToFile" not found') + * err = ( __cuLogsDumpToFile)(iterator, pathToFile, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6778 + * global __cuLogsDumpToFile + * cuPythonInit() + * if __cuLogsDumpToFile == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLogsDumpToFile" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6781 + * with gil: + * raise RuntimeError('Function "cuLogsDumpToFile" not found') + * err = ( __cuLogsDumpToFile)(iterator, pathToFile, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlogIterator *, char const *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToFile)(__pyx_v_iterator, __pyx_v_pathToFile, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6781, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6782 + * raise RuntimeError('Function "cuLogsDumpToFile" not found') + * err = ( __cuLogsDumpToFile)(iterator, pathToFile, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuLogsDumpToMemory(CUlogIterator* iterator, char* buffer, size_t* size, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6775 + * return err + * + * cdef CUresult _cuLogsDumpToFile(CUlogIterator* iterator, const char* pathToFile, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLogsDumpToFile + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLogsDumpToFile", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6784 + * return err + * + * cdef CUresult _cuLogsDumpToMemory(CUlogIterator* iterator, char* buffer, size_t* size, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLogsDumpToMemory + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLogsDumpToMemory(CUlogIterator *__pyx_v_iterator, char *__pyx_v_buffer, size_t *__pyx_v_size, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuLogsDumpToMemory", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6786 + * cdef CUresult _cuLogsDumpToMemory(CUlogIterator* iterator, char* buffer, size_t* size, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuLogsDumpToMemory + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuLogsDumpToMemory == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6786, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6787 + * global __cuLogsDumpToMemory + * cuPythonInit() + * if __cuLogsDumpToMemory == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLogsDumpToMemory" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToMemory == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6788 + * cuPythonInit() + * if __cuLogsDumpToMemory == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLogsDumpToMemory" not found') + * err = ( __cuLogsDumpToMemory)(iterator, buffer, size, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6789 + * if __cuLogsDumpToMemory == NULL: + * with gil: + * raise RuntimeError('Function "cuLogsDumpToMemory" not found') # <<<<<<<<<<<<<< + * err = ( __cuLogsDumpToMemory)(iterator, buffer, size, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuLogsDumpToMemory_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6789, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6789, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6788 + * cuPythonInit() + * if __cuLogsDumpToMemory == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuLogsDumpToMemory" not found') + * err = ( __cuLogsDumpToMemory)(iterator, buffer, size, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6787 + * global __cuLogsDumpToMemory + * cuPythonInit() + * if __cuLogsDumpToMemory == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuLogsDumpToMemory" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6790 + * with gil: + * raise RuntimeError('Function "cuLogsDumpToMemory" not found') + * err = ( __cuLogsDumpToMemory)(iterator, buffer, size, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUlogIterator *, char *, size_t *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToMemory)(__pyx_v_iterator, __pyx_v_buffer, __pyx_v_size, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6790, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6791 + * raise RuntimeError('Function "cuLogsDumpToMemory" not found') + * err = ( __cuLogsDumpToMemory)(iterator, buffer, size, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCheckpointProcessGetRestoreThreadId(int pid, int* tid) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6784 + * return err + * + * cdef CUresult _cuLogsDumpToMemory(CUlogIterator* iterator, char* buffer, size_t* size, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuLogsDumpToMemory + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuLogsDumpToMemory", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6793 + * return err + * + * cdef CUresult _cuCheckpointProcessGetRestoreThreadId(int pid, int* tid) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCheckpointProcessGetRestoreThreadId + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCheckpointProcessGetRestoreThreadId(int __pyx_v_pid, int *__pyx_v_tid) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCheckpointProcessGetRestoreThreadId", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6795 + * cdef CUresult _cuCheckpointProcessGetRestoreThreadId(int pid, int* tid) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCheckpointProcessGetRestoreThreadId + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCheckpointProcessGetRestoreThreadId == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6795, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6796 + * global __cuCheckpointProcessGetRestoreThreadId + * cuPythonInit() + * if __cuCheckpointProcessGetRestoreThreadId == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessGetRestoreThreadId" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetRestoreThreadId == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6797 + * cuPythonInit() + * if __cuCheckpointProcessGetRestoreThreadId == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCheckpointProcessGetRestoreThreadId" not found') + * err = ( __cuCheckpointProcessGetRestoreThreadId)(pid, tid) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6798 + * if __cuCheckpointProcessGetRestoreThreadId == NULL: + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessGetRestoreThreadId" not found') # <<<<<<<<<<<<<< + * err = ( __cuCheckpointProcessGetRestoreThreadId)(pid, tid) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCheckpointProcessGetR}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6798, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6798, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6797 + * cuPythonInit() + * if __cuCheckpointProcessGetRestoreThreadId == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCheckpointProcessGetRestoreThreadId" not found') + * err = ( __cuCheckpointProcessGetRestoreThreadId)(pid, tid) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6796 + * global __cuCheckpointProcessGetRestoreThreadId + * cuPythonInit() + * if __cuCheckpointProcessGetRestoreThreadId == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessGetRestoreThreadId" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6799 + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessGetRestoreThreadId" not found') + * err = ( __cuCheckpointProcessGetRestoreThreadId)(pid, tid) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int, int *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetRestoreThreadId)(__pyx_v_pid, __pyx_v_tid); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6799, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6800 + * raise RuntimeError('Function "cuCheckpointProcessGetRestoreThreadId" not found') + * err = ( __cuCheckpointProcessGetRestoreThreadId)(pid, tid) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCheckpointProcessGetState(int pid, CUprocessState* state) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6793 + * return err + * + * cdef CUresult _cuCheckpointProcessGetRestoreThreadId(int pid, int* tid) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCheckpointProcessGetRestoreThreadId + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCheckpointProcessGetRestoreThreadId", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6802 + * return err + * + * cdef CUresult _cuCheckpointProcessGetState(int pid, CUprocessState* state) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCheckpointProcessGetState + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCheckpointProcessGetState(int __pyx_v_pid, CUprocessState *__pyx_v_state) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCheckpointProcessGetState", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6804 + * cdef CUresult _cuCheckpointProcessGetState(int pid, CUprocessState* state) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCheckpointProcessGetState + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCheckpointProcessGetState == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6804, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6805 + * global __cuCheckpointProcessGetState + * cuPythonInit() + * if __cuCheckpointProcessGetState == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessGetState" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetState == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6806 + * cuPythonInit() + * if __cuCheckpointProcessGetState == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCheckpointProcessGetState" not found') + * err = ( __cuCheckpointProcessGetState)(pid, state) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6807 + * if __cuCheckpointProcessGetState == NULL: + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessGetState" not found') # <<<<<<<<<<<<<< + * err = ( __cuCheckpointProcessGetState)(pid, state) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCheckpointProcessGetS}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6807, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6807, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6806 + * cuPythonInit() + * if __cuCheckpointProcessGetState == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCheckpointProcessGetState" not found') + * err = ( __cuCheckpointProcessGetState)(pid, state) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6805 + * global __cuCheckpointProcessGetState + * cuPythonInit() + * if __cuCheckpointProcessGetState == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessGetState" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6808 + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessGetState" not found') + * err = ( __cuCheckpointProcessGetState)(pid, state) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int, CUprocessState *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetState)(__pyx_v_pid, __pyx_v_state); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6808, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6809 + * raise RuntimeError('Function "cuCheckpointProcessGetState" not found') + * err = ( __cuCheckpointProcessGetState)(pid, state) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCheckpointProcessLock(int pid, CUcheckpointLockArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6802 + * return err + * + * cdef CUresult _cuCheckpointProcessGetState(int pid, CUprocessState* state) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCheckpointProcessGetState + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCheckpointProcessGetState", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6811 + * return err + * + * cdef CUresult _cuCheckpointProcessLock(int pid, CUcheckpointLockArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCheckpointProcessLock + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCheckpointProcessLock(int __pyx_v_pid, CUcheckpointLockArgs *__pyx_v_args) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCheckpointProcessLock", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6813 + * cdef CUresult _cuCheckpointProcessLock(int pid, CUcheckpointLockArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCheckpointProcessLock + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCheckpointProcessLock == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6813, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6814 + * global __cuCheckpointProcessLock + * cuPythonInit() + * if __cuCheckpointProcessLock == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessLock" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessLock == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6815 + * cuPythonInit() + * if __cuCheckpointProcessLock == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCheckpointProcessLock" not found') + * err = ( __cuCheckpointProcessLock)(pid, args) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6816 + * if __cuCheckpointProcessLock == NULL: + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessLock" not found') # <<<<<<<<<<<<<< + * err = ( __cuCheckpointProcessLock)(pid, args) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCheckpointProcessLock}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6816, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6816, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6815 + * cuPythonInit() + * if __cuCheckpointProcessLock == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCheckpointProcessLock" not found') + * err = ( __cuCheckpointProcessLock)(pid, args) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6814 + * global __cuCheckpointProcessLock + * cuPythonInit() + * if __cuCheckpointProcessLock == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessLock" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6817 + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessLock" not found') + * err = ( __cuCheckpointProcessLock)(pid, args) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int, CUcheckpointLockArgs *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessLock)(__pyx_v_pid, __pyx_v_args); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6817, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6818 + * raise RuntimeError('Function "cuCheckpointProcessLock" not found') + * err = ( __cuCheckpointProcessLock)(pid, args) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCheckpointProcessCheckpoint(int pid, CUcheckpointCheckpointArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6811 + * return err + * + * cdef CUresult _cuCheckpointProcessLock(int pid, CUcheckpointLockArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCheckpointProcessLock + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCheckpointProcessLock", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6820 + * return err + * + * cdef CUresult _cuCheckpointProcessCheckpoint(int pid, CUcheckpointCheckpointArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCheckpointProcessCheckpoint + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCheckpointProcessCheckpoint(int __pyx_v_pid, CUcheckpointCheckpointArgs *__pyx_v_args) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCheckpointProcessCheckpoint", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6822 + * cdef CUresult _cuCheckpointProcessCheckpoint(int pid, CUcheckpointCheckpointArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCheckpointProcessCheckpoint + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCheckpointProcessCheckpoint == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6822, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6823 + * global __cuCheckpointProcessCheckpoint + * cuPythonInit() + * if __cuCheckpointProcessCheckpoint == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessCheckpoint" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessCheckpoint == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6824 + * cuPythonInit() + * if __cuCheckpointProcessCheckpoint == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCheckpointProcessCheckpoint" not found') + * err = ( __cuCheckpointProcessCheckpoint)(pid, args) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6825 + * if __cuCheckpointProcessCheckpoint == NULL: + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessCheckpoint" not found') # <<<<<<<<<<<<<< + * err = ( __cuCheckpointProcessCheckpoint)(pid, args) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCheckpointProcessChec}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6825, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6825, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6824 + * cuPythonInit() + * if __cuCheckpointProcessCheckpoint == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCheckpointProcessCheckpoint" not found') + * err = ( __cuCheckpointProcessCheckpoint)(pid, args) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6823 + * global __cuCheckpointProcessCheckpoint + * cuPythonInit() + * if __cuCheckpointProcessCheckpoint == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessCheckpoint" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6826 + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessCheckpoint" not found') + * err = ( __cuCheckpointProcessCheckpoint)(pid, args) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int, CUcheckpointCheckpointArgs *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessCheckpoint)(__pyx_v_pid, __pyx_v_args); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6826, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6827 + * raise RuntimeError('Function "cuCheckpointProcessCheckpoint" not found') + * err = ( __cuCheckpointProcessCheckpoint)(pid, args) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCheckpointProcessRestore(int pid, CUcheckpointRestoreArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6820 + * return err + * + * cdef CUresult _cuCheckpointProcessCheckpoint(int pid, CUcheckpointCheckpointArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCheckpointProcessCheckpoint + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCheckpointProcessCheckpoint", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6829 + * return err + * + * cdef CUresult _cuCheckpointProcessRestore(int pid, CUcheckpointRestoreArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCheckpointProcessRestore + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCheckpointProcessRestore(int __pyx_v_pid, CUcheckpointRestoreArgs *__pyx_v_args) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCheckpointProcessRestore", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6831 + * cdef CUresult _cuCheckpointProcessRestore(int pid, CUcheckpointRestoreArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCheckpointProcessRestore + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCheckpointProcessRestore == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6831, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6832 + * global __cuCheckpointProcessRestore + * cuPythonInit() + * if __cuCheckpointProcessRestore == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessRestore" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessRestore == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6833 + * cuPythonInit() + * if __cuCheckpointProcessRestore == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCheckpointProcessRestore" not found') + * err = ( __cuCheckpointProcessRestore)(pid, args) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6834 + * if __cuCheckpointProcessRestore == NULL: + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessRestore" not found') # <<<<<<<<<<<<<< + * err = ( __cuCheckpointProcessRestore)(pid, args) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCheckpointProcessRest}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6834, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6834, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6833 + * cuPythonInit() + * if __cuCheckpointProcessRestore == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCheckpointProcessRestore" not found') + * err = ( __cuCheckpointProcessRestore)(pid, args) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6832 + * global __cuCheckpointProcessRestore + * cuPythonInit() + * if __cuCheckpointProcessRestore == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessRestore" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6835 + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessRestore" not found') + * err = ( __cuCheckpointProcessRestore)(pid, args) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int, CUcheckpointRestoreArgs *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessRestore)(__pyx_v_pid, __pyx_v_args); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6835, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6836 + * raise RuntimeError('Function "cuCheckpointProcessRestore" not found') + * err = ( __cuCheckpointProcessRestore)(pid, args) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuCheckpointProcessUnlock(int pid, CUcheckpointUnlockArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6829 + * return err + * + * cdef CUresult _cuCheckpointProcessRestore(int pid, CUcheckpointRestoreArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCheckpointProcessRestore + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCheckpointProcessRestore", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6838 + * return err + * + * cdef CUresult _cuCheckpointProcessUnlock(int pid, CUcheckpointUnlockArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCheckpointProcessUnlock + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCheckpointProcessUnlock(int __pyx_v_pid, CUcheckpointUnlockArgs *__pyx_v_args) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuCheckpointProcessUnlock", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6840 + * cdef CUresult _cuCheckpointProcessUnlock(int pid, CUcheckpointUnlockArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuCheckpointProcessUnlock + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuCheckpointProcessUnlock == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6840, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6841 + * global __cuCheckpointProcessUnlock + * cuPythonInit() + * if __cuCheckpointProcessUnlock == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessUnlock" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessUnlock == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6842 + * cuPythonInit() + * if __cuCheckpointProcessUnlock == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCheckpointProcessUnlock" not found') + * err = ( __cuCheckpointProcessUnlock)(pid, args) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6843 + * if __cuCheckpointProcessUnlock == NULL: + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessUnlock" not found') # <<<<<<<<<<<<<< + * err = ( __cuCheckpointProcessUnlock)(pid, args) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuCheckpointProcessUnlo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6843, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6843, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6842 + * cuPythonInit() + * if __cuCheckpointProcessUnlock == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuCheckpointProcessUnlock" not found') + * err = ( __cuCheckpointProcessUnlock)(pid, args) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6841 + * global __cuCheckpointProcessUnlock + * cuPythonInit() + * if __cuCheckpointProcessUnlock == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessUnlock" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6844 + * with gil: + * raise RuntimeError('Function "cuCheckpointProcessUnlock" not found') + * err = ( __cuCheckpointProcessUnlock)(pid, args) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(int, CUcheckpointUnlockArgs *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessUnlock)(__pyx_v_pid, __pyx_v_args); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6844, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6845 + * raise RuntimeError('Function "cuCheckpointProcessUnlock" not found') + * err = ( __cuCheckpointProcessUnlock)(pid, args) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuProfilerStart() except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6838 + * return err + * + * cdef CUresult _cuCheckpointProcessUnlock(int pid, CUcheckpointUnlockArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuCheckpointProcessUnlock + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuCheckpointProcessUnlock", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6847 + * return err + * + * cdef CUresult _cuProfilerStart() except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuProfilerStart + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuProfilerStart(void) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuProfilerStart", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6849 + * cdef CUresult _cuProfilerStart() except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuProfilerStart + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuProfilerStart == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6849, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6850 + * global __cuProfilerStart + * cuPythonInit() + * if __cuProfilerStart == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuProfilerStart" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStart == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6851 + * cuPythonInit() + * if __cuProfilerStart == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuProfilerStart" not found') + * err = ( __cuProfilerStart)() +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6852 + * if __cuProfilerStart == NULL: + * with gil: + * raise RuntimeError('Function "cuProfilerStart" not found') # <<<<<<<<<<<<<< + * err = ( __cuProfilerStart)() + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuProfilerStart_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6852, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6852, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6851 + * cuPythonInit() + * if __cuProfilerStart == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuProfilerStart" not found') + * err = ( __cuProfilerStart)() +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6850 + * global __cuProfilerStart + * cuPythonInit() + * if __cuProfilerStart == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuProfilerStart" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6853 + * with gil: + * raise RuntimeError('Function "cuProfilerStart" not found') + * err = ( __cuProfilerStart)() # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStart)(); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6853, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6854 + * raise RuntimeError('Function "cuProfilerStart" not found') + * err = ( __cuProfilerStart)() + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuProfilerStop() except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6847 + * return err + * + * cdef CUresult _cuProfilerStart() except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuProfilerStart + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuProfilerStart", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6856 + * return err + * + * cdef CUresult _cuProfilerStop() except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuProfilerStop + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuProfilerStop(void) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuProfilerStop", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6858 + * cdef CUresult _cuProfilerStop() except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuProfilerStop + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuProfilerStop == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6858, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6859 + * global __cuProfilerStop + * cuPythonInit() + * if __cuProfilerStop == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuProfilerStop" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStop == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6860 + * cuPythonInit() + * if __cuProfilerStop == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuProfilerStop" not found') + * err = ( __cuProfilerStop)() +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6861 + * if __cuProfilerStop == NULL: + * with gil: + * raise RuntimeError('Function "cuProfilerStop" not found') # <<<<<<<<<<<<<< + * err = ( __cuProfilerStop)() + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuProfilerStop_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6861, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6861, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6860 + * cuPythonInit() + * if __cuProfilerStop == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuProfilerStop" not found') + * err = ( __cuProfilerStop)() +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6859 + * global __cuProfilerStop + * cuPythonInit() + * if __cuProfilerStop == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuProfilerStop" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6862 + * with gil: + * raise RuntimeError('Function "cuProfilerStop" not found') + * err = ( __cuProfilerStop)() # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(void))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStop)(); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6862, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6863 + * raise RuntimeError('Function "cuProfilerStop" not found') + * err = ( __cuProfilerStop)() + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphicsEGLRegisterImage(CUgraphicsResource* pCudaResource, EGLImageKHR image, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6856 + * return err + * + * cdef CUresult _cuProfilerStop() except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuProfilerStop + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuProfilerStop", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6865 + * return err + * + * cdef CUresult _cuGraphicsEGLRegisterImage(CUgraphicsResource* pCudaResource, EGLImageKHR image, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsEGLRegisterImage + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsEGLRegisterImage(CUgraphicsResource *__pyx_v_pCudaResource, __pyx_t_4cuda_8bindings_8cydriver_EGLImageKHR __pyx_v_image, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphicsEGLRegisterImage", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6867 + * cdef CUresult _cuGraphicsEGLRegisterImage(CUgraphicsResource* pCudaResource, EGLImageKHR image, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphicsEGLRegisterImage + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphicsEGLRegisterImage == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6867, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6868 + * global __cuGraphicsEGLRegisterImage + * cuPythonInit() + * if __cuGraphicsEGLRegisterImage == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsEGLRegisterImage" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsEGLRegisterImage == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6869 + * cuPythonInit() + * if __cuGraphicsEGLRegisterImage == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsEGLRegisterImage" not found') + * err = ( __cuGraphicsEGLRegisterImage)(pCudaResource, image, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6870 + * if __cuGraphicsEGLRegisterImage == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphicsEGLRegisterImage" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphicsEGLRegisterImage)(pCudaResource, image, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphicsEGLRegisterIm}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6870, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6870, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6869 + * cuPythonInit() + * if __cuGraphicsEGLRegisterImage == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsEGLRegisterImage" not found') + * err = ( __cuGraphicsEGLRegisterImage)(pCudaResource, image, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6868 + * global __cuGraphicsEGLRegisterImage + * cuPythonInit() + * if __cuGraphicsEGLRegisterImage == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsEGLRegisterImage" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6871 + * with gil: + * raise RuntimeError('Function "cuGraphicsEGLRegisterImage" not found') + * err = ( __cuGraphicsEGLRegisterImage)(pCudaResource, image, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_EGLImageKHR, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsEGLRegisterImage)(__pyx_v_pCudaResource, __pyx_v_image, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6871, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6872 + * raise RuntimeError('Function "cuGraphicsEGLRegisterImage" not found') + * err = ( __cuGraphicsEGLRegisterImage)(pCudaResource, image, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEGLStreamConsumerConnect(CUeglStreamConnection* conn, EGLStreamKHR stream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6865 + * return err + * + * cdef CUresult _cuGraphicsEGLRegisterImage(CUgraphicsResource* pCudaResource, EGLImageKHR image, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsEGLRegisterImage + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphicsEGLRegisterImage", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6874 + * return err + * + * cdef CUresult _cuEGLStreamConsumerConnect(CUeglStreamConnection* conn, EGLStreamKHR stream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerConnect + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerConnect(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *__pyx_v_conn, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR __pyx_v_stream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEGLStreamConsumerConnect", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6876 + * cdef CUresult _cuEGLStreamConsumerConnect(CUeglStreamConnection* conn, EGLStreamKHR stream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEGLStreamConsumerConnect + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEGLStreamConsumerConnect == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6876, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6877 + * global __cuEGLStreamConsumerConnect + * cuPythonInit() + * if __cuEGLStreamConsumerConnect == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerConnect" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnect == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6878 + * cuPythonInit() + * if __cuEGLStreamConsumerConnect == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamConsumerConnect" not found') + * err = ( __cuEGLStreamConsumerConnect)(conn, stream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6879 + * if __cuEGLStreamConsumerConnect == NULL: + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerConnect" not found') # <<<<<<<<<<<<<< + * err = ( __cuEGLStreamConsumerConnect)(conn, stream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEGLStreamConsumerConn}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6879, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6879, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6878 + * cuPythonInit() + * if __cuEGLStreamConsumerConnect == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamConsumerConnect" not found') + * err = ( __cuEGLStreamConsumerConnect)(conn, stream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6877 + * global __cuEGLStreamConsumerConnect + * cuPythonInit() + * if __cuEGLStreamConsumerConnect == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerConnect" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6880 + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerConnect" not found') + * err = ( __cuEGLStreamConsumerConnect)(conn, stream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnect)(__pyx_v_conn, __pyx_v_stream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6880, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6881 + * raise RuntimeError('Function "cuEGLStreamConsumerConnect" not found') + * err = ( __cuEGLStreamConsumerConnect)(conn, stream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEGLStreamConsumerConnectWithFlags(CUeglStreamConnection* conn, EGLStreamKHR stream, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6874 + * return err + * + * cdef CUresult _cuEGLStreamConsumerConnect(CUeglStreamConnection* conn, EGLStreamKHR stream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerConnect + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEGLStreamConsumerConnect", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6883 + * return err + * + * cdef CUresult _cuEGLStreamConsumerConnectWithFlags(CUeglStreamConnection* conn, EGLStreamKHR stream, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerConnectWithFlags + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerConnectWithFlags(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *__pyx_v_conn, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR __pyx_v_stream, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEGLStreamConsumerConnectWithFlags", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6885 + * cdef CUresult _cuEGLStreamConsumerConnectWithFlags(CUeglStreamConnection* conn, EGLStreamKHR stream, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEGLStreamConsumerConnectWithFlags + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEGLStreamConsumerConnectWithFlags == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6885, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6886 + * global __cuEGLStreamConsumerConnectWithFlags + * cuPythonInit() + * if __cuEGLStreamConsumerConnectWithFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerConnectWithFlags" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnectWithFlags == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6887 + * cuPythonInit() + * if __cuEGLStreamConsumerConnectWithFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamConsumerConnectWithFlags" not found') + * err = ( __cuEGLStreamConsumerConnectWithFlags)(conn, stream, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6888 + * if __cuEGLStreamConsumerConnectWithFlags == NULL: + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerConnectWithFlags" not found') # <<<<<<<<<<<<<< + * err = ( __cuEGLStreamConsumerConnectWithFlags)(conn, stream, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEGLStreamConsumerConn_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6888, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6888, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6887 + * cuPythonInit() + * if __cuEGLStreamConsumerConnectWithFlags == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamConsumerConnectWithFlags" not found') + * err = ( __cuEGLStreamConsumerConnectWithFlags)(conn, stream, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6886 + * global __cuEGLStreamConsumerConnectWithFlags + * cuPythonInit() + * if __cuEGLStreamConsumerConnectWithFlags == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerConnectWithFlags" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6889 + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerConnectWithFlags" not found') + * err = ( __cuEGLStreamConsumerConnectWithFlags)(conn, stream, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnectWithFlags)(__pyx_v_conn, __pyx_v_stream, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6889, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6890 + * raise RuntimeError('Function "cuEGLStreamConsumerConnectWithFlags" not found') + * err = ( __cuEGLStreamConsumerConnectWithFlags)(conn, stream, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEGLStreamConsumerDisconnect(CUeglStreamConnection* conn) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6883 + * return err + * + * cdef CUresult _cuEGLStreamConsumerConnectWithFlags(CUeglStreamConnection* conn, EGLStreamKHR stream, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerConnectWithFlags + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEGLStreamConsumerConnectWithFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6892 + * return err + * + * cdef CUresult _cuEGLStreamConsumerDisconnect(CUeglStreamConnection* conn) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerDisconnect + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerDisconnect(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *__pyx_v_conn) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEGLStreamConsumerDisconnect", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6894 + * cdef CUresult _cuEGLStreamConsumerDisconnect(CUeglStreamConnection* conn) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEGLStreamConsumerDisconnect + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEGLStreamConsumerDisconnect == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6894, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6895 + * global __cuEGLStreamConsumerDisconnect + * cuPythonInit() + * if __cuEGLStreamConsumerDisconnect == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerDisconnect" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerDisconnect == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6896 + * cuPythonInit() + * if __cuEGLStreamConsumerDisconnect == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamConsumerDisconnect" not found') + * err = ( __cuEGLStreamConsumerDisconnect)(conn) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6897 + * if __cuEGLStreamConsumerDisconnect == NULL: + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerDisconnect" not found') # <<<<<<<<<<<<<< + * err = ( __cuEGLStreamConsumerDisconnect)(conn) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEGLStreamConsumerDisc}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6897, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6897, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6896 + * cuPythonInit() + * if __cuEGLStreamConsumerDisconnect == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamConsumerDisconnect" not found') + * err = ( __cuEGLStreamConsumerDisconnect)(conn) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6895 + * global __cuEGLStreamConsumerDisconnect + * cuPythonInit() + * if __cuEGLStreamConsumerDisconnect == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerDisconnect" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6898 + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerDisconnect" not found') + * err = ( __cuEGLStreamConsumerDisconnect)(conn) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerDisconnect)(__pyx_v_conn); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6898, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6899 + * raise RuntimeError('Function "cuEGLStreamConsumerDisconnect" not found') + * err = ( __cuEGLStreamConsumerDisconnect)(conn) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEGLStreamConsumerAcquireFrame(CUeglStreamConnection* conn, CUgraphicsResource* pCudaResource, CUstream* pStream, unsigned int timeout) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6892 + * return err + * + * cdef CUresult _cuEGLStreamConsumerDisconnect(CUeglStreamConnection* conn) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerDisconnect + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEGLStreamConsumerDisconnect", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6901 + * return err + * + * cdef CUresult _cuEGLStreamConsumerAcquireFrame(CUeglStreamConnection* conn, CUgraphicsResource* pCudaResource, CUstream* pStream, unsigned int timeout) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerAcquireFrame + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerAcquireFrame(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *__pyx_v_conn, CUgraphicsResource *__pyx_v_pCudaResource, CUstream *__pyx_v_pStream, unsigned int __pyx_v_timeout) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEGLStreamConsumerAcquireFrame", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6903 + * cdef CUresult _cuEGLStreamConsumerAcquireFrame(CUeglStreamConnection* conn, CUgraphicsResource* pCudaResource, CUstream* pStream, unsigned int timeout) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEGLStreamConsumerAcquireFrame + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEGLStreamConsumerAcquireFrame == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6903, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6904 + * global __cuEGLStreamConsumerAcquireFrame + * cuPythonInit() + * if __cuEGLStreamConsumerAcquireFrame == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerAcquireFrame" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerAcquireFrame == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6905 + * cuPythonInit() + * if __cuEGLStreamConsumerAcquireFrame == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamConsumerAcquireFrame" not found') + * err = ( __cuEGLStreamConsumerAcquireFrame)(conn, pCudaResource, pStream, timeout) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6906 + * if __cuEGLStreamConsumerAcquireFrame == NULL: + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerAcquireFrame" not found') # <<<<<<<<<<<<<< + * err = ( __cuEGLStreamConsumerAcquireFrame)(conn, pCudaResource, pStream, timeout) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEGLStreamConsumerAcqu}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6906, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6906, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6905 + * cuPythonInit() + * if __cuEGLStreamConsumerAcquireFrame == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamConsumerAcquireFrame" not found') + * err = ( __cuEGLStreamConsumerAcquireFrame)(conn, pCudaResource, pStream, timeout) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6904 + * global __cuEGLStreamConsumerAcquireFrame + * cuPythonInit() + * if __cuEGLStreamConsumerAcquireFrame == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerAcquireFrame" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6907 + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerAcquireFrame" not found') + * err = ( __cuEGLStreamConsumerAcquireFrame)(conn, pCudaResource, pStream, timeout) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, CUgraphicsResource *, CUstream *, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerAcquireFrame)(__pyx_v_conn, __pyx_v_pCudaResource, __pyx_v_pStream, __pyx_v_timeout); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6907, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6908 + * raise RuntimeError('Function "cuEGLStreamConsumerAcquireFrame" not found') + * err = ( __cuEGLStreamConsumerAcquireFrame)(conn, pCudaResource, pStream, timeout) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEGLStreamConsumerReleaseFrame(CUeglStreamConnection* conn, CUgraphicsResource pCudaResource, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6901 + * return err + * + * cdef CUresult _cuEGLStreamConsumerAcquireFrame(CUeglStreamConnection* conn, CUgraphicsResource* pCudaResource, CUstream* pStream, unsigned int timeout) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerAcquireFrame + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEGLStreamConsumerAcquireFrame", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6910 + * return err + * + * cdef CUresult _cuEGLStreamConsumerReleaseFrame(CUeglStreamConnection* conn, CUgraphicsResource pCudaResource, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerReleaseFrame + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerReleaseFrame(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *__pyx_v_conn, CUgraphicsResource __pyx_v_pCudaResource, CUstream *__pyx_v_pStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEGLStreamConsumerReleaseFrame", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6912 + * cdef CUresult _cuEGLStreamConsumerReleaseFrame(CUeglStreamConnection* conn, CUgraphicsResource pCudaResource, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEGLStreamConsumerReleaseFrame + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEGLStreamConsumerReleaseFrame == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6912, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6913 + * global __cuEGLStreamConsumerReleaseFrame + * cuPythonInit() + * if __cuEGLStreamConsumerReleaseFrame == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerReleaseFrame" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerReleaseFrame == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6914 + * cuPythonInit() + * if __cuEGLStreamConsumerReleaseFrame == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamConsumerReleaseFrame" not found') + * err = ( __cuEGLStreamConsumerReleaseFrame)(conn, pCudaResource, pStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6915 + * if __cuEGLStreamConsumerReleaseFrame == NULL: + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerReleaseFrame" not found') # <<<<<<<<<<<<<< + * err = ( __cuEGLStreamConsumerReleaseFrame)(conn, pCudaResource, pStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEGLStreamConsumerRele}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6915, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6915, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6914 + * cuPythonInit() + * if __cuEGLStreamConsumerReleaseFrame == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamConsumerReleaseFrame" not found') + * err = ( __cuEGLStreamConsumerReleaseFrame)(conn, pCudaResource, pStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6913 + * global __cuEGLStreamConsumerReleaseFrame + * cuPythonInit() + * if __cuEGLStreamConsumerReleaseFrame == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerReleaseFrame" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6916 + * with gil: + * raise RuntimeError('Function "cuEGLStreamConsumerReleaseFrame" not found') + * err = ( __cuEGLStreamConsumerReleaseFrame)(conn, pCudaResource, pStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, CUgraphicsResource, CUstream *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerReleaseFrame)(__pyx_v_conn, __pyx_v_pCudaResource, __pyx_v_pStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6916, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6917 + * raise RuntimeError('Function "cuEGLStreamConsumerReleaseFrame" not found') + * err = ( __cuEGLStreamConsumerReleaseFrame)(conn, pCudaResource, pStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEGLStreamProducerConnect(CUeglStreamConnection* conn, EGLStreamKHR stream, EGLint width, EGLint height) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6910 + * return err + * + * cdef CUresult _cuEGLStreamConsumerReleaseFrame(CUeglStreamConnection* conn, CUgraphicsResource pCudaResource, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerReleaseFrame + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEGLStreamConsumerReleaseFrame", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6919 + * return err + * + * cdef CUresult _cuEGLStreamProducerConnect(CUeglStreamConnection* conn, EGLStreamKHR stream, EGLint width, EGLint height) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerConnect + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerConnect(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *__pyx_v_conn, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR __pyx_v_stream, __pyx_t_4cuda_8bindings_8cydriver_EGLint __pyx_v_width, __pyx_t_4cuda_8bindings_8cydriver_EGLint __pyx_v_height) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEGLStreamProducerConnect", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6921 + * cdef CUresult _cuEGLStreamProducerConnect(CUeglStreamConnection* conn, EGLStreamKHR stream, EGLint width, EGLint height) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEGLStreamProducerConnect + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEGLStreamProducerConnect == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6921, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6922 + * global __cuEGLStreamProducerConnect + * cuPythonInit() + * if __cuEGLStreamProducerConnect == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerConnect" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerConnect == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6923 + * cuPythonInit() + * if __cuEGLStreamProducerConnect == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamProducerConnect" not found') + * err = ( __cuEGLStreamProducerConnect)(conn, stream, width, height) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6924 + * if __cuEGLStreamProducerConnect == NULL: + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerConnect" not found') # <<<<<<<<<<<<<< + * err = ( __cuEGLStreamProducerConnect)(conn, stream, width, height) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEGLStreamProducerConn}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6924, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6924, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6923 + * cuPythonInit() + * if __cuEGLStreamProducerConnect == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamProducerConnect" not found') + * err = ( __cuEGLStreamProducerConnect)(conn, stream, width, height) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6922 + * global __cuEGLStreamProducerConnect + * cuPythonInit() + * if __cuEGLStreamProducerConnect == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerConnect" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6925 + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerConnect" not found') + * err = ( __cuEGLStreamProducerConnect)(conn, stream, width, height) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR, __pyx_t_4cuda_8bindings_8cydriver_EGLint, __pyx_t_4cuda_8bindings_8cydriver_EGLint))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerConnect)(__pyx_v_conn, __pyx_v_stream, __pyx_v_width, __pyx_v_height); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6925, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6926 + * raise RuntimeError('Function "cuEGLStreamProducerConnect" not found') + * err = ( __cuEGLStreamProducerConnect)(conn, stream, width, height) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEGLStreamProducerDisconnect(CUeglStreamConnection* conn) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6919 + * return err + * + * cdef CUresult _cuEGLStreamProducerConnect(CUeglStreamConnection* conn, EGLStreamKHR stream, EGLint width, EGLint height) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerConnect + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEGLStreamProducerConnect", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6928 + * return err + * + * cdef CUresult _cuEGLStreamProducerDisconnect(CUeglStreamConnection* conn) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerDisconnect + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerDisconnect(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *__pyx_v_conn) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEGLStreamProducerDisconnect", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6930 + * cdef CUresult _cuEGLStreamProducerDisconnect(CUeglStreamConnection* conn) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEGLStreamProducerDisconnect + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEGLStreamProducerDisconnect == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6930, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6931 + * global __cuEGLStreamProducerDisconnect + * cuPythonInit() + * if __cuEGLStreamProducerDisconnect == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerDisconnect" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerDisconnect == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6932 + * cuPythonInit() + * if __cuEGLStreamProducerDisconnect == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamProducerDisconnect" not found') + * err = ( __cuEGLStreamProducerDisconnect)(conn) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6933 + * if __cuEGLStreamProducerDisconnect == NULL: + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerDisconnect" not found') # <<<<<<<<<<<<<< + * err = ( __cuEGLStreamProducerDisconnect)(conn) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEGLStreamProducerDisc}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6933, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6933, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6932 + * cuPythonInit() + * if __cuEGLStreamProducerDisconnect == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamProducerDisconnect" not found') + * err = ( __cuEGLStreamProducerDisconnect)(conn) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6931 + * global __cuEGLStreamProducerDisconnect + * cuPythonInit() + * if __cuEGLStreamProducerDisconnect == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerDisconnect" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6934 + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerDisconnect" not found') + * err = ( __cuEGLStreamProducerDisconnect)(conn) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerDisconnect)(__pyx_v_conn); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6934, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6935 + * raise RuntimeError('Function "cuEGLStreamProducerDisconnect" not found') + * err = ( __cuEGLStreamProducerDisconnect)(conn) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEGLStreamProducerPresentFrame(CUeglStreamConnection* conn, CUeglFrame eglframe, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6928 + * return err + * + * cdef CUresult _cuEGLStreamProducerDisconnect(CUeglStreamConnection* conn) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerDisconnect + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEGLStreamProducerDisconnect", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6937 + * return err + * + * cdef CUresult _cuEGLStreamProducerPresentFrame(CUeglStreamConnection* conn, CUeglFrame eglframe, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerPresentFrame + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerPresentFrame(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *__pyx_v_conn, __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame __pyx_v_eglframe, CUstream *__pyx_v_pStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEGLStreamProducerPresentFrame", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6939 + * cdef CUresult _cuEGLStreamProducerPresentFrame(CUeglStreamConnection* conn, CUeglFrame eglframe, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEGLStreamProducerPresentFrame + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEGLStreamProducerPresentFrame == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6939, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6940 + * global __cuEGLStreamProducerPresentFrame + * cuPythonInit() + * if __cuEGLStreamProducerPresentFrame == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerPresentFrame" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerPresentFrame == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6941 + * cuPythonInit() + * if __cuEGLStreamProducerPresentFrame == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamProducerPresentFrame" not found') + * err = ( __cuEGLStreamProducerPresentFrame)(conn, eglframe, pStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6942 + * if __cuEGLStreamProducerPresentFrame == NULL: + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerPresentFrame" not found') # <<<<<<<<<<<<<< + * err = ( __cuEGLStreamProducerPresentFrame)(conn, eglframe, pStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEGLStreamProducerPres}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6942, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6942, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6941 + * cuPythonInit() + * if __cuEGLStreamProducerPresentFrame == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamProducerPresentFrame" not found') + * err = ( __cuEGLStreamProducerPresentFrame)(conn, eglframe, pStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6940 + * global __cuEGLStreamProducerPresentFrame + * cuPythonInit() + * if __cuEGLStreamProducerPresentFrame == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerPresentFrame" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6943 + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerPresentFrame" not found') + * err = ( __cuEGLStreamProducerPresentFrame)(conn, eglframe, pStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame, CUstream *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerPresentFrame)(__pyx_v_conn, __pyx_v_eglframe, __pyx_v_pStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6943, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6944 + * raise RuntimeError('Function "cuEGLStreamProducerPresentFrame" not found') + * err = ( __cuEGLStreamProducerPresentFrame)(conn, eglframe, pStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEGLStreamProducerReturnFrame(CUeglStreamConnection* conn, CUeglFrame* eglframe, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6937 + * return err + * + * cdef CUresult _cuEGLStreamProducerPresentFrame(CUeglStreamConnection* conn, CUeglFrame eglframe, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerPresentFrame + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEGLStreamProducerPresentFrame", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6946 + * return err + * + * cdef CUresult _cuEGLStreamProducerReturnFrame(CUeglStreamConnection* conn, CUeglFrame* eglframe, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerReturnFrame + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerReturnFrame(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *__pyx_v_conn, __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame *__pyx_v_eglframe, CUstream *__pyx_v_pStream) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEGLStreamProducerReturnFrame", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6948 + * cdef CUresult _cuEGLStreamProducerReturnFrame(CUeglStreamConnection* conn, CUeglFrame* eglframe, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEGLStreamProducerReturnFrame + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEGLStreamProducerReturnFrame == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6948, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6949 + * global __cuEGLStreamProducerReturnFrame + * cuPythonInit() + * if __cuEGLStreamProducerReturnFrame == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerReturnFrame" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerReturnFrame == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6950 + * cuPythonInit() + * if __cuEGLStreamProducerReturnFrame == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamProducerReturnFrame" not found') + * err = ( __cuEGLStreamProducerReturnFrame)(conn, eglframe, pStream) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6951 + * if __cuEGLStreamProducerReturnFrame == NULL: + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerReturnFrame" not found') # <<<<<<<<<<<<<< + * err = ( __cuEGLStreamProducerReturnFrame)(conn, eglframe, pStream) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEGLStreamProducerRetu}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6951, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6951, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6950 + * cuPythonInit() + * if __cuEGLStreamProducerReturnFrame == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEGLStreamProducerReturnFrame" not found') + * err = ( __cuEGLStreamProducerReturnFrame)(conn, eglframe, pStream) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6949 + * global __cuEGLStreamProducerReturnFrame + * cuPythonInit() + * if __cuEGLStreamProducerReturnFrame == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerReturnFrame" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6952 + * with gil: + * raise RuntimeError('Function "cuEGLStreamProducerReturnFrame" not found') + * err = ( __cuEGLStreamProducerReturnFrame)(conn, eglframe, pStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame *, CUstream *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerReturnFrame)(__pyx_v_conn, __pyx_v_eglframe, __pyx_v_pStream); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6952, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6953 + * raise RuntimeError('Function "cuEGLStreamProducerReturnFrame" not found') + * err = ( __cuEGLStreamProducerReturnFrame)(conn, eglframe, pStream) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphicsResourceGetMappedEglFrame(CUeglFrame* eglFrame, CUgraphicsResource resource, unsigned int index, unsigned int mipLevel) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6946 + * return err + * + * cdef CUresult _cuEGLStreamProducerReturnFrame(CUeglStreamConnection* conn, CUeglFrame* eglframe, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerReturnFrame + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEGLStreamProducerReturnFrame", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6955 + * return err + * + * cdef CUresult _cuGraphicsResourceGetMappedEglFrame(CUeglFrame* eglFrame, CUgraphicsResource resource, unsigned int index, unsigned int mipLevel) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedEglFrame + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsResourceGetMappedEglFrame(__pyx_t_4cuda_8bindings_8cydriver_CUeglFrame *__pyx_v_eglFrame, CUgraphicsResource __pyx_v_resource, unsigned int __pyx_v_index, unsigned int __pyx_v_mipLevel) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphicsResourceGetMappedEglFrame", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6957 + * cdef CUresult _cuGraphicsResourceGetMappedEglFrame(CUeglFrame* eglFrame, CUgraphicsResource resource, unsigned int index, unsigned int mipLevel) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphicsResourceGetMappedEglFrame + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphicsResourceGetMappedEglFrame == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6957, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6958 + * global __cuGraphicsResourceGetMappedEglFrame + * cuPythonInit() + * if __cuGraphicsResourceGetMappedEglFrame == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceGetMappedEglFrame" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedEglFrame == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6959 + * cuPythonInit() + * if __cuGraphicsResourceGetMappedEglFrame == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsResourceGetMappedEglFrame" not found') + * err = ( __cuGraphicsResourceGetMappedEglFrame)(eglFrame, resource, index, mipLevel) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6960 + * if __cuGraphicsResourceGetMappedEglFrame == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceGetMappedEglFrame" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphicsResourceGetMappedEglFrame)(eglFrame, resource, index, mipLevel) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphicsResourceGetMa_3}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6960, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6960, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6959 + * cuPythonInit() + * if __cuGraphicsResourceGetMappedEglFrame == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsResourceGetMappedEglFrame" not found') + * err = ( __cuGraphicsResourceGetMappedEglFrame)(eglFrame, resource, index, mipLevel) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6958 + * global __cuGraphicsResourceGetMappedEglFrame + * cuPythonInit() + * if __cuGraphicsResourceGetMappedEglFrame == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceGetMappedEglFrame" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6961 + * with gil: + * raise RuntimeError('Function "cuGraphicsResourceGetMappedEglFrame" not found') + * err = ( __cuGraphicsResourceGetMappedEglFrame)(eglFrame, resource, index, mipLevel) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(__pyx_t_4cuda_8bindings_8cydriver_CUeglFrame *, CUgraphicsResource, unsigned int, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedEglFrame)(__pyx_v_eglFrame, __pyx_v_resource, __pyx_v_index, __pyx_v_mipLevel); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6961, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6962 + * raise RuntimeError('Function "cuGraphicsResourceGetMappedEglFrame" not found') + * err = ( __cuGraphicsResourceGetMappedEglFrame)(eglFrame, resource, index, mipLevel) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuEventCreateFromEGLSync(CUevent* phEvent, EGLSyncKHR eglSync, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6955 + * return err + * + * cdef CUresult _cuGraphicsResourceGetMappedEglFrame(CUeglFrame* eglFrame, CUgraphicsResource resource, unsigned int index, unsigned int mipLevel) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedEglFrame + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphicsResourceGetMappedEglFrame", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6964 + * return err + * + * cdef CUresult _cuEventCreateFromEGLSync(CUevent* phEvent, EGLSyncKHR eglSync, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventCreateFromEGLSync + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventCreateFromEGLSync(CUevent *__pyx_v_phEvent, __pyx_t_4cuda_8bindings_8cydriver_EGLSyncKHR __pyx_v_eglSync, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuEventCreateFromEGLSync", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6966 + * cdef CUresult _cuEventCreateFromEGLSync(CUevent* phEvent, EGLSyncKHR eglSync, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuEventCreateFromEGLSync + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuEventCreateFromEGLSync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6966, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6967 + * global __cuEventCreateFromEGLSync + * cuPythonInit() + * if __cuEventCreateFromEGLSync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventCreateFromEGLSync" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreateFromEGLSync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6968 + * cuPythonInit() + * if __cuEventCreateFromEGLSync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventCreateFromEGLSync" not found') + * err = ( __cuEventCreateFromEGLSync)(phEvent, eglSync, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6969 + * if __cuEventCreateFromEGLSync == NULL: + * with gil: + * raise RuntimeError('Function "cuEventCreateFromEGLSync" not found') # <<<<<<<<<<<<<< + * err = ( __cuEventCreateFromEGLSync)(phEvent, eglSync, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuEventCreateFromEGLSyn}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6969, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6969, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6968 + * cuPythonInit() + * if __cuEventCreateFromEGLSync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuEventCreateFromEGLSync" not found') + * err = ( __cuEventCreateFromEGLSync)(phEvent, eglSync, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6967 + * global __cuEventCreateFromEGLSync + * cuPythonInit() + * if __cuEventCreateFromEGLSync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuEventCreateFromEGLSync" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6970 + * with gil: + * raise RuntimeError('Function "cuEventCreateFromEGLSync" not found') + * err = ( __cuEventCreateFromEGLSync)(phEvent, eglSync, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUevent *, __pyx_t_4cuda_8bindings_8cydriver_EGLSyncKHR, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreateFromEGLSync)(__pyx_v_phEvent, __pyx_v_eglSync, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6970, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6971 + * raise RuntimeError('Function "cuEventCreateFromEGLSync" not found') + * err = ( __cuEventCreateFromEGLSync)(phEvent, eglSync, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphicsGLRegisterBuffer(CUgraphicsResource* pCudaResource, GLuint buffer, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6964 + * return err + * + * cdef CUresult _cuEventCreateFromEGLSync(CUevent* phEvent, EGLSyncKHR eglSync, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuEventCreateFromEGLSync + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuEventCreateFromEGLSync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6973 + * return err + * + * cdef CUresult _cuGraphicsGLRegisterBuffer(CUgraphicsResource* pCudaResource, GLuint buffer, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsGLRegisterBuffer + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsGLRegisterBuffer(CUgraphicsResource *__pyx_v_pCudaResource, __pyx_t_4cuda_8bindings_8cydriver_GLuint __pyx_v_buffer, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphicsGLRegisterBuffer", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6975 + * cdef CUresult _cuGraphicsGLRegisterBuffer(CUgraphicsResource* pCudaResource, GLuint buffer, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphicsGLRegisterBuffer + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphicsGLRegisterBuffer == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6975, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6976 + * global __cuGraphicsGLRegisterBuffer + * cuPythonInit() + * if __cuGraphicsGLRegisterBuffer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsGLRegisterBuffer" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterBuffer == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6977 + * cuPythonInit() + * if __cuGraphicsGLRegisterBuffer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsGLRegisterBuffer" not found') + * err = ( __cuGraphicsGLRegisterBuffer)(pCudaResource, buffer, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6978 + * if __cuGraphicsGLRegisterBuffer == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphicsGLRegisterBuffer" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphicsGLRegisterBuffer)(pCudaResource, buffer, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphicsGLRegisterBuf}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6978, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6978, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6977 + * cuPythonInit() + * if __cuGraphicsGLRegisterBuffer == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsGLRegisterBuffer" not found') + * err = ( __cuGraphicsGLRegisterBuffer)(pCudaResource, buffer, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6976 + * global __cuGraphicsGLRegisterBuffer + * cuPythonInit() + * if __cuGraphicsGLRegisterBuffer == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsGLRegisterBuffer" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6979 + * with gil: + * raise RuntimeError('Function "cuGraphicsGLRegisterBuffer" not found') + * err = ( __cuGraphicsGLRegisterBuffer)(pCudaResource, buffer, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_GLuint, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterBuffer)(__pyx_v_pCudaResource, __pyx_v_buffer, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6979, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6980 + * raise RuntimeError('Function "cuGraphicsGLRegisterBuffer" not found') + * err = ( __cuGraphicsGLRegisterBuffer)(pCudaResource, buffer, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphicsGLRegisterImage(CUgraphicsResource* pCudaResource, GLuint image, GLenum target, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6973 + * return err + * + * cdef CUresult _cuGraphicsGLRegisterBuffer(CUgraphicsResource* pCudaResource, GLuint buffer, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsGLRegisterBuffer + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphicsGLRegisterBuffer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6982 + * return err + * + * cdef CUresult _cuGraphicsGLRegisterImage(CUgraphicsResource* pCudaResource, GLuint image, GLenum target, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsGLRegisterImage + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsGLRegisterImage(CUgraphicsResource *__pyx_v_pCudaResource, __pyx_t_4cuda_8bindings_8cydriver_GLuint __pyx_v_image, __pyx_t_4cuda_8bindings_8cydriver_GLenum __pyx_v_target, unsigned int __pyx_v_Flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphicsGLRegisterImage", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6984 + * cdef CUresult _cuGraphicsGLRegisterImage(CUgraphicsResource* pCudaResource, GLuint image, GLenum target, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphicsGLRegisterImage + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphicsGLRegisterImage == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6984, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6985 + * global __cuGraphicsGLRegisterImage + * cuPythonInit() + * if __cuGraphicsGLRegisterImage == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsGLRegisterImage" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterImage == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6986 + * cuPythonInit() + * if __cuGraphicsGLRegisterImage == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsGLRegisterImage" not found') + * err = ( __cuGraphicsGLRegisterImage)(pCudaResource, image, target, Flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6987 + * if __cuGraphicsGLRegisterImage == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphicsGLRegisterImage" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphicsGLRegisterImage)(pCudaResource, image, target, Flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphicsGLRegisterIma}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6987, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6987, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6986 + * cuPythonInit() + * if __cuGraphicsGLRegisterImage == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsGLRegisterImage" not found') + * err = ( __cuGraphicsGLRegisterImage)(pCudaResource, image, target, Flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6985 + * global __cuGraphicsGLRegisterImage + * cuPythonInit() + * if __cuGraphicsGLRegisterImage == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsGLRegisterImage" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6988 + * with gil: + * raise RuntimeError('Function "cuGraphicsGLRegisterImage" not found') + * err = ( __cuGraphicsGLRegisterImage)(pCudaResource, image, target, Flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_GLuint, __pyx_t_4cuda_8bindings_8cydriver_GLenum, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterImage)(__pyx_v_pCudaResource, __pyx_v_image, __pyx_v_target, __pyx_v_Flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6988, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6989 + * raise RuntimeError('Function "cuGraphicsGLRegisterImage" not found') + * err = ( __cuGraphicsGLRegisterImage)(pCudaResource, image, target, Flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGLGetDevices_v2(unsigned int* pCudaDeviceCount, CUdevice* pCudaDevices, unsigned int cudaDeviceCount, CUGLDeviceList deviceList) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6982 + * return err + * + * cdef CUresult _cuGraphicsGLRegisterImage(CUgraphicsResource* pCudaResource, GLuint image, GLenum target, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsGLRegisterImage + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphicsGLRegisterImage", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":6991 + * return err + * + * cdef CUresult _cuGLGetDevices_v2(unsigned int* pCudaDeviceCount, CUdevice* pCudaDevices, unsigned int cudaDeviceCount, CUGLDeviceList deviceList) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGLGetDevices_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGLGetDevices_v2(unsigned int *__pyx_v_pCudaDeviceCount, CUdevice *__pyx_v_pCudaDevices, unsigned int __pyx_v_cudaDeviceCount, __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList __pyx_v_deviceList) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGLGetDevices_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":6993 + * cdef CUresult _cuGLGetDevices_v2(unsigned int* pCudaDeviceCount, CUdevice* pCudaDevices, unsigned int cudaDeviceCount, CUGLDeviceList deviceList) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGLGetDevices_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGLGetDevices_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 6993, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":6994 + * global __cuGLGetDevices_v2 + * cuPythonInit() + * if __cuGLGetDevices_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGLGetDevices_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGLGetDevices_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":6995 + * cuPythonInit() + * if __cuGLGetDevices_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGLGetDevices_v2" not found') + * err = ( __cuGLGetDevices_v2)(pCudaDeviceCount, pCudaDevices, cudaDeviceCount, deviceList) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":6996 + * if __cuGLGetDevices_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuGLGetDevices_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuGLGetDevices_v2)(pCudaDeviceCount, pCudaDevices, cudaDeviceCount, deviceList) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGLGetDevices_v2_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6996, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 6996, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6995 + * cuPythonInit() + * if __cuGLGetDevices_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGLGetDevices_v2" not found') + * err = ( __cuGLGetDevices_v2)(pCudaDeviceCount, pCudaDevices, cudaDeviceCount, deviceList) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6994 + * global __cuGLGetDevices_v2 + * cuPythonInit() + * if __cuGLGetDevices_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGLGetDevices_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":6997 + * with gil: + * raise RuntimeError('Function "cuGLGetDevices_v2" not found') + * err = ( __cuGLGetDevices_v2)(pCudaDeviceCount, pCudaDevices, cudaDeviceCount, deviceList) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(unsigned int *, CUdevice *, unsigned int, __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGLGetDevices_v2)(__pyx_v_pCudaDeviceCount, __pyx_v_pCudaDevices, __pyx_v_cudaDeviceCount, __pyx_v_deviceList); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 6997, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":6998 + * raise RuntimeError('Function "cuGLGetDevices_v2" not found') + * err = ( __cuGLGetDevices_v2)(pCudaDeviceCount, pCudaDevices, cudaDeviceCount, deviceList) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuVDPAUGetDevice(CUdevice* pDevice, VdpDevice vdpDevice, VdpGetProcAddress* vdpGetProcAddress) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":6991 + * return err + * + * cdef CUresult _cuGLGetDevices_v2(unsigned int* pCudaDeviceCount, CUdevice* pCudaDevices, unsigned int cudaDeviceCount, CUGLDeviceList deviceList) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGLGetDevices_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGLGetDevices_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":7000 + * return err + * + * cdef CUresult _cuVDPAUGetDevice(CUdevice* pDevice, VdpDevice vdpDevice, VdpGetProcAddress* vdpGetProcAddress) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuVDPAUGetDevice + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuVDPAUGetDevice(CUdevice *__pyx_v_pDevice, __pyx_t_4cuda_8bindings_8cydriver_VdpDevice __pyx_v_vdpDevice, __pyx_t_4cuda_8bindings_8cydriver_VdpGetProcAddress *__pyx_v_vdpGetProcAddress) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuVDPAUGetDevice", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":7002 + * cdef CUresult _cuVDPAUGetDevice(CUdevice* pDevice, VdpDevice vdpDevice, VdpGetProcAddress* vdpGetProcAddress) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuVDPAUGetDevice + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuVDPAUGetDevice == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 7002, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":7003 + * global __cuVDPAUGetDevice + * cuPythonInit() + * if __cuVDPAUGetDevice == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuVDPAUGetDevice" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUGetDevice == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":7004 + * cuPythonInit() + * if __cuVDPAUGetDevice == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuVDPAUGetDevice" not found') + * err = ( __cuVDPAUGetDevice)(pDevice, vdpDevice, vdpGetProcAddress) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":7005 + * if __cuVDPAUGetDevice == NULL: + * with gil: + * raise RuntimeError('Function "cuVDPAUGetDevice" not found') # <<<<<<<<<<<<<< + * err = ( __cuVDPAUGetDevice)(pDevice, vdpDevice, vdpGetProcAddress) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuVDPAUGetDevice_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7005, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 7005, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":7004 + * cuPythonInit() + * if __cuVDPAUGetDevice == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuVDPAUGetDevice" not found') + * err = ( __cuVDPAUGetDevice)(pDevice, vdpDevice, vdpGetProcAddress) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":7003 + * global __cuVDPAUGetDevice + * cuPythonInit() + * if __cuVDPAUGetDevice == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuVDPAUGetDevice" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":7006 + * with gil: + * raise RuntimeError('Function "cuVDPAUGetDevice" not found') + * err = ( __cuVDPAUGetDevice)(pDevice, vdpDevice, vdpGetProcAddress) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUdevice *, __pyx_t_4cuda_8bindings_8cydriver_VdpDevice, __pyx_t_4cuda_8bindings_8cydriver_VdpGetProcAddress *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUGetDevice)(__pyx_v_pDevice, __pyx_v_vdpDevice, __pyx_v_vdpGetProcAddress); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 7006, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":7007 + * raise RuntimeError('Function "cuVDPAUGetDevice" not found') + * err = ( __cuVDPAUGetDevice)(pDevice, vdpDevice, vdpGetProcAddress) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuVDPAUCtxCreate_v2(CUcontext* pCtx, unsigned int flags, CUdevice device, VdpDevice vdpDevice, VdpGetProcAddress* vdpGetProcAddress) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7000 + * return err + * + * cdef CUresult _cuVDPAUGetDevice(CUdevice* pDevice, VdpDevice vdpDevice, VdpGetProcAddress* vdpGetProcAddress) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuVDPAUGetDevice + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuVDPAUGetDevice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":7009 + * return err + * + * cdef CUresult _cuVDPAUCtxCreate_v2(CUcontext* pCtx, unsigned int flags, CUdevice device, VdpDevice vdpDevice, VdpGetProcAddress* vdpGetProcAddress) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuVDPAUCtxCreate_v2 + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuVDPAUCtxCreate_v2(CUcontext *__pyx_v_pCtx, unsigned int __pyx_v_flags, CUdevice __pyx_v_device, __pyx_t_4cuda_8bindings_8cydriver_VdpDevice __pyx_v_vdpDevice, __pyx_t_4cuda_8bindings_8cydriver_VdpGetProcAddress *__pyx_v_vdpGetProcAddress) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuVDPAUCtxCreate_v2", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":7011 + * cdef CUresult _cuVDPAUCtxCreate_v2(CUcontext* pCtx, unsigned int flags, CUdevice device, VdpDevice vdpDevice, VdpGetProcAddress* vdpGetProcAddress) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuVDPAUCtxCreate_v2 + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuVDPAUCtxCreate_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 7011, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":7012 + * global __cuVDPAUCtxCreate_v2 + * cuPythonInit() + * if __cuVDPAUCtxCreate_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuVDPAUCtxCreate_v2" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUCtxCreate_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":7013 + * cuPythonInit() + * if __cuVDPAUCtxCreate_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuVDPAUCtxCreate_v2" not found') + * err = ( __cuVDPAUCtxCreate_v2)(pCtx, flags, device, vdpDevice, vdpGetProcAddress) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":7014 + * if __cuVDPAUCtxCreate_v2 == NULL: + * with gil: + * raise RuntimeError('Function "cuVDPAUCtxCreate_v2" not found') # <<<<<<<<<<<<<< + * err = ( __cuVDPAUCtxCreate_v2)(pCtx, flags, device, vdpDevice, vdpGetProcAddress) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuVDPAUCtxCreate_v2_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7014, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 7014, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":7013 + * cuPythonInit() + * if __cuVDPAUCtxCreate_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuVDPAUCtxCreate_v2" not found') + * err = ( __cuVDPAUCtxCreate_v2)(pCtx, flags, device, vdpDevice, vdpGetProcAddress) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":7012 + * global __cuVDPAUCtxCreate_v2 + * cuPythonInit() + * if __cuVDPAUCtxCreate_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuVDPAUCtxCreate_v2" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":7015 + * with gil: + * raise RuntimeError('Function "cuVDPAUCtxCreate_v2" not found') + * err = ( __cuVDPAUCtxCreate_v2)(pCtx, flags, device, vdpDevice, vdpGetProcAddress) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUcontext *, unsigned int, CUdevice, __pyx_t_4cuda_8bindings_8cydriver_VdpDevice, __pyx_t_4cuda_8bindings_8cydriver_VdpGetProcAddress *))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUCtxCreate_v2)(__pyx_v_pCtx, __pyx_v_flags, __pyx_v_device, __pyx_v_vdpDevice, __pyx_v_vdpGetProcAddress); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 7015, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":7016 + * raise RuntimeError('Function "cuVDPAUCtxCreate_v2" not found') + * err = ( __cuVDPAUCtxCreate_v2)(pCtx, flags, device, vdpDevice, vdpGetProcAddress) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphicsVDPAURegisterVideoSurface(CUgraphicsResource* pCudaResource, VdpVideoSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7009 + * return err + * + * cdef CUresult _cuVDPAUCtxCreate_v2(CUcontext* pCtx, unsigned int flags, CUdevice device, VdpDevice vdpDevice, VdpGetProcAddress* vdpGetProcAddress) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuVDPAUCtxCreate_v2 + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuVDPAUCtxCreate_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":7018 + * return err + * + * cdef CUresult _cuGraphicsVDPAURegisterVideoSurface(CUgraphicsResource* pCudaResource, VdpVideoSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsVDPAURegisterVideoSurface + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsVDPAURegisterVideoSurface(CUgraphicsResource *__pyx_v_pCudaResource, __pyx_t_4cuda_8bindings_8cydriver_VdpVideoSurface __pyx_v_vdpSurface, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphicsVDPAURegisterVideoSurface", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":7020 + * cdef CUresult _cuGraphicsVDPAURegisterVideoSurface(CUgraphicsResource* pCudaResource, VdpVideoSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphicsVDPAURegisterVideoSurface + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphicsVDPAURegisterVideoSurface == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 7020, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":7021 + * global __cuGraphicsVDPAURegisterVideoSurface + * cuPythonInit() + * if __cuGraphicsVDPAURegisterVideoSurface == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsVDPAURegisterVideoSurface" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterVideoSurface == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":7022 + * cuPythonInit() + * if __cuGraphicsVDPAURegisterVideoSurface == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsVDPAURegisterVideoSurface" not found') + * err = ( __cuGraphicsVDPAURegisterVideoSurface)(pCudaResource, vdpSurface, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":7023 + * if __cuGraphicsVDPAURegisterVideoSurface == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphicsVDPAURegisterVideoSurface" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphicsVDPAURegisterVideoSurface)(pCudaResource, vdpSurface, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphicsVDPAURegister}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7023, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 7023, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":7022 + * cuPythonInit() + * if __cuGraphicsVDPAURegisterVideoSurface == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsVDPAURegisterVideoSurface" not found') + * err = ( __cuGraphicsVDPAURegisterVideoSurface)(pCudaResource, vdpSurface, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":7021 + * global __cuGraphicsVDPAURegisterVideoSurface + * cuPythonInit() + * if __cuGraphicsVDPAURegisterVideoSurface == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsVDPAURegisterVideoSurface" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":7024 + * with gil: + * raise RuntimeError('Function "cuGraphicsVDPAURegisterVideoSurface" not found') + * err = ( __cuGraphicsVDPAURegisterVideoSurface)(pCudaResource, vdpSurface, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_VdpVideoSurface, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterVideoSurface)(__pyx_v_pCudaResource, __pyx_v_vdpSurface, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 7024, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":7025 + * raise RuntimeError('Function "cuGraphicsVDPAURegisterVideoSurface" not found') + * err = ( __cuGraphicsVDPAURegisterVideoSurface)(pCudaResource, vdpSurface, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef CUresult _cuGraphicsVDPAURegisterOutputSurface(CUgraphicsResource* pCudaResource, VdpOutputSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7018 + * return err + * + * cdef CUresult _cuGraphicsVDPAURegisterVideoSurface(CUgraphicsResource* pCudaResource, VdpVideoSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsVDPAURegisterVideoSurface + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphicsVDPAURegisterVideoSurface", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":7027 + * return err + * + * cdef CUresult _cuGraphicsVDPAURegisterOutputSurface(CUgraphicsResource* pCudaResource, VdpOutputSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsVDPAURegisterOutputSurface + * cuPythonInit() +*/ + +static CUresult __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsVDPAURegisterOutputSurface(CUgraphicsResource *__pyx_v_pCudaResource, __pyx_t_4cuda_8bindings_8cydriver_VdpOutputSurface __pyx_v_vdpSurface, unsigned int __pyx_v_flags) { + CUresult __pyx_v_err; + CUresult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + CUresult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuGraphicsVDPAURegisterOutputSurface", 1); + + /* "cuda/bindings/_bindings/cydriver.pyx":7029 + * cdef CUresult _cuGraphicsVDPAURegisterOutputSurface(CUgraphicsResource* pCudaResource, VdpOutputSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + * global __cuGraphicsVDPAURegisterOutputSurface + * cuPythonInit() # <<<<<<<<<<<<<< + * if __cuGraphicsVDPAURegisterOutputSurface == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 7029, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":7030 + * global __cuGraphicsVDPAURegisterOutputSurface + * cuPythonInit() + * if __cuGraphicsVDPAURegisterOutputSurface == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsVDPAURegisterOutputSurface" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterOutputSurface == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cydriver.pyx":7031 + * cuPythonInit() + * if __cuGraphicsVDPAURegisterOutputSurface == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsVDPAURegisterOutputSurface" not found') + * err = ( __cuGraphicsVDPAURegisterOutputSurface)(pCudaResource, vdpSurface, flags) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cydriver.pyx":7032 + * if __cuGraphicsVDPAURegisterOutputSurface == NULL: + * with gil: + * raise RuntimeError('Function "cuGraphicsVDPAURegisterOutputSurface" not found') # <<<<<<<<<<<<<< + * err = ( __cuGraphicsVDPAURegisterOutputSurface)(pCudaResource, vdpSurface, flags) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_cuGraphicsVDPAURegister_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7032, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 7032, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cydriver.pyx":7031 + * cuPythonInit() + * if __cuGraphicsVDPAURegisterOutputSurface == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "cuGraphicsVDPAURegisterOutputSurface" not found') + * err = ( __cuGraphicsVDPAURegisterOutputSurface)(pCudaResource, vdpSurface, flags) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cydriver.pyx":7030 + * global __cuGraphicsVDPAURegisterOutputSurface + * cuPythonInit() + * if __cuGraphicsVDPAURegisterOutputSurface == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "cuGraphicsVDPAURegisterOutputSurface" not found') +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":7033 + * with gil: + * raise RuntimeError('Function "cuGraphicsVDPAURegisterOutputSurface" not found') + * err = ( __cuGraphicsVDPAURegisterOutputSurface)(pCudaResource, vdpSurface, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((CUresult (*)(CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_VdpOutputSurface, unsigned int))__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterOutputSurface)(__pyx_v_pCudaResource, __pyx_v_vdpSurface, __pyx_v_flags); if (unlikely(__pyx_t_7 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 7033, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cydriver.pyx":7034 + * raise RuntimeError('Function "cuGraphicsVDPAURegisterOutputSurface" not found') + * err = ( __cuGraphicsVDPAURegisterOutputSurface)(pCudaResource, vdpSurface, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef dict func_ptrs = None +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7027 + * return err + * + * cdef CUresult _cuGraphicsVDPAURegisterOutputSurface(CUgraphicsResource* pCudaResource, VdpOutputSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: # <<<<<<<<<<<<<< + * global __cuGraphicsVDPAURegisterOutputSurface + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._cuGraphicsVDPAURegisterOutputSurface", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUDA_ERROR_NOT_FOUND; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":7038 + * cdef dict func_ptrs = None + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + +static PyObject *__pyx_pw_4cuda_8bindings_9_bindings_8cydriver_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_bindings_8cydriver__inspect_function_pointers(CYTHON_UNUSED int __pyx_skip_dispatch) { + PyObject *__pyx_v_data = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointers", 0); + + /* "cuda/bindings/_bindings/cydriver.pyx":7040 + * cpdef dict _inspect_function_pointers(): + * global func_ptrs + * if func_ptrs is not None: # <<<<<<<<<<<<<< + * return func_ptrs + * +*/ + __pyx_t_1 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs != ((PyObject*)Py_None)); + if (__pyx_t_1) { + + /* "cuda/bindings/_bindings/cydriver.pyx":7041 + * global func_ptrs + * if func_ptrs is not None: + * return func_ptrs # <<<<<<<<<<<<<< + * + * cuPythonInit() +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs); + __pyx_r = __pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7040 + * cpdef dict _inspect_function_pointers(): + * global func_ptrs + * if func_ptrs is not None: # <<<<<<<<<<<<<< + * return func_ptrs + * +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":7043 + * return func_ptrs + * + * cuPythonInit() # <<<<<<<<<<<<<< + * cdef dict data = {} + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver_cuPythonInit(); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(0, 7043, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cydriver.pyx":7044 + * + * cuPythonInit() + * cdef dict data = {} # <<<<<<<<<<<<<< + * + * global __cuGetErrorString +*/ + __pyx_t_3 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7044, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_v_data = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7047 + * + * global __cuGetErrorString + * data["__cuGetErrorString"] = __cuGetErrorString # <<<<<<<<<<<<<< + * global __cuGetErrorName + * data["__cuGetErrorName"] = __cuGetErrorName +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorString)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7047, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGetErrorString, __pyx_t_3) < 0))) __PYX_ERR(0, 7047, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7049 + * data["__cuGetErrorString"] = __cuGetErrorString + * global __cuGetErrorName + * data["__cuGetErrorName"] = __cuGetErrorName # <<<<<<<<<<<<<< + * global __cuInit + * data["__cuInit"] = __cuInit +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorName)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7049, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGetErrorName, __pyx_t_3) < 0))) __PYX_ERR(0, 7049, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7051 + * data["__cuGetErrorName"] = __cuGetErrorName + * global __cuInit + * data["__cuInit"] = __cuInit # <<<<<<<<<<<<<< + * global __cuDriverGetVersion + * data["__cuDriverGetVersion"] = __cuDriverGetVersion +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuInit)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7051, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuInit, __pyx_t_3) < 0))) __PYX_ERR(0, 7051, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7053 + * data["__cuInit"] = __cuInit + * global __cuDriverGetVersion + * data["__cuDriverGetVersion"] = __cuDriverGetVersion # <<<<<<<<<<<<<< + * global __cuDeviceGet + * data["__cuDeviceGet"] = __cuDeviceGet +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDriverGetVersion)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7053, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDriverGetVersion, __pyx_t_3) < 0))) __PYX_ERR(0, 7053, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7055 + * data["__cuDriverGetVersion"] = __cuDriverGetVersion + * global __cuDeviceGet + * data["__cuDeviceGet"] = __cuDeviceGet # <<<<<<<<<<<<<< + * global __cuDeviceGetCount + * data["__cuDeviceGetCount"] = __cuDeviceGetCount +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGet)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7055, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGet, __pyx_t_3) < 0))) __PYX_ERR(0, 7055, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7057 + * data["__cuDeviceGet"] = __cuDeviceGet + * global __cuDeviceGetCount + * data["__cuDeviceGetCount"] = __cuDeviceGetCount # <<<<<<<<<<<<<< + * global __cuDeviceGetName + * data["__cuDeviceGetName"] = __cuDeviceGetName +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetCount)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7057, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetCount, __pyx_t_3) < 0))) __PYX_ERR(0, 7057, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7059 + * data["__cuDeviceGetCount"] = __cuDeviceGetCount + * global __cuDeviceGetName + * data["__cuDeviceGetName"] = __cuDeviceGetName # <<<<<<<<<<<<<< + * global __cuDeviceGetUuid + * data["__cuDeviceGetUuid"] = __cuDeviceGetUuid +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetName)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7059, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetName, __pyx_t_3) < 0))) __PYX_ERR(0, 7059, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7061 + * data["__cuDeviceGetName"] = __cuDeviceGetName + * global __cuDeviceGetUuid + * data["__cuDeviceGetUuid"] = __cuDeviceGetUuid # <<<<<<<<<<<<<< + * global __cuDeviceGetUuid_v2 + * data["__cuDeviceGetUuid_v2"] = __cuDeviceGetUuid_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7061, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetUuid, __pyx_t_3) < 0))) __PYX_ERR(0, 7061, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7063 + * data["__cuDeviceGetUuid"] = __cuDeviceGetUuid + * global __cuDeviceGetUuid_v2 + * data["__cuDeviceGetUuid_v2"] = __cuDeviceGetUuid_v2 # <<<<<<<<<<<<<< + * global __cuDeviceGetLuid + * data["__cuDeviceGetLuid"] = __cuDeviceGetLuid +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7063, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetUuid_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7063, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7065 + * data["__cuDeviceGetUuid_v2"] = __cuDeviceGetUuid_v2 + * global __cuDeviceGetLuid + * data["__cuDeviceGetLuid"] = __cuDeviceGetLuid # <<<<<<<<<<<<<< + * global __cuDeviceTotalMem_v2 + * data["__cuDeviceTotalMem_v2"] = __cuDeviceTotalMem_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetLuid)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7065, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetLuid, __pyx_t_3) < 0))) __PYX_ERR(0, 7065, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7067 + * data["__cuDeviceGetLuid"] = __cuDeviceGetLuid + * global __cuDeviceTotalMem_v2 + * data["__cuDeviceTotalMem_v2"] = __cuDeviceTotalMem_v2 # <<<<<<<<<<<<<< + * global __cuDeviceGetTexture1DLinearMaxWidth + * data["__cuDeviceGetTexture1DLinearMaxWidth"] = __cuDeviceGetTexture1DLinearMaxWidth +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceTotalMem_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7067, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceTotalMem_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7067, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7069 + * data["__cuDeviceTotalMem_v2"] = __cuDeviceTotalMem_v2 + * global __cuDeviceGetTexture1DLinearMaxWidth + * data["__cuDeviceGetTexture1DLinearMaxWidth"] = __cuDeviceGetTexture1DLinearMaxWidth # <<<<<<<<<<<<<< + * global __cuDeviceGetAttribute + * data["__cuDeviceGetAttribute"] = __cuDeviceGetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetTexture1DLinearMaxWidth)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7069, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetTexture1DLinearMaxW, __pyx_t_3) < 0))) __PYX_ERR(0, 7069, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7071 + * data["__cuDeviceGetTexture1DLinearMaxWidth"] = __cuDeviceGetTexture1DLinearMaxWidth + * global __cuDeviceGetAttribute + * data["__cuDeviceGetAttribute"] = __cuDeviceGetAttribute # <<<<<<<<<<<<<< + * global __cuDeviceGetNvSciSyncAttributes + * data["__cuDeviceGetNvSciSyncAttributes"] = __cuDeviceGetNvSciSyncAttributes +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7071, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7071, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7073 + * data["__cuDeviceGetAttribute"] = __cuDeviceGetAttribute + * global __cuDeviceGetNvSciSyncAttributes + * data["__cuDeviceGetNvSciSyncAttributes"] = __cuDeviceGetNvSciSyncAttributes # <<<<<<<<<<<<<< + * global __cuDeviceSetMemPool + * data["__cuDeviceSetMemPool"] = __cuDeviceSetMemPool +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetNvSciSyncAttributes)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7073, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetNvSciSyncAttributes, __pyx_t_3) < 0))) __PYX_ERR(0, 7073, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7075 + * data["__cuDeviceGetNvSciSyncAttributes"] = __cuDeviceGetNvSciSyncAttributes + * global __cuDeviceSetMemPool + * data["__cuDeviceSetMemPool"] = __cuDeviceSetMemPool # <<<<<<<<<<<<<< + * global __cuDeviceGetMemPool + * data["__cuDeviceGetMemPool"] = __cuDeviceGetMemPool +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetMemPool)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7075, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceSetMemPool, __pyx_t_3) < 0))) __PYX_ERR(0, 7075, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7077 + * data["__cuDeviceSetMemPool"] = __cuDeviceSetMemPool + * global __cuDeviceGetMemPool + * data["__cuDeviceGetMemPool"] = __cuDeviceGetMemPool # <<<<<<<<<<<<<< + * global __cuDeviceGetDefaultMemPool + * data["__cuDeviceGetDefaultMemPool"] = __cuDeviceGetDefaultMemPool +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetMemPool)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7077, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetMemPool, __pyx_t_3) < 0))) __PYX_ERR(0, 7077, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7079 + * data["__cuDeviceGetMemPool"] = __cuDeviceGetMemPool + * global __cuDeviceGetDefaultMemPool + * data["__cuDeviceGetDefaultMemPool"] = __cuDeviceGetDefaultMemPool # <<<<<<<<<<<<<< + * global __cuDeviceGetExecAffinitySupport + * data["__cuDeviceGetExecAffinitySupport"] = __cuDeviceGetExecAffinitySupport +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDefaultMemPool)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7079, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetDefaultMemPool, __pyx_t_3) < 0))) __PYX_ERR(0, 7079, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7081 + * data["__cuDeviceGetDefaultMemPool"] = __cuDeviceGetDefaultMemPool + * global __cuDeviceGetExecAffinitySupport + * data["__cuDeviceGetExecAffinitySupport"] = __cuDeviceGetExecAffinitySupport # <<<<<<<<<<<<<< + * global __cuFlushGPUDirectRDMAWrites + * data["__cuFlushGPUDirectRDMAWrites"] = __cuFlushGPUDirectRDMAWrites +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetExecAffinitySupport)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7081, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetExecAffinitySupport, __pyx_t_3) < 0))) __PYX_ERR(0, 7081, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7083 + * data["__cuDeviceGetExecAffinitySupport"] = __cuDeviceGetExecAffinitySupport + * global __cuFlushGPUDirectRDMAWrites + * data["__cuFlushGPUDirectRDMAWrites"] = __cuFlushGPUDirectRDMAWrites # <<<<<<<<<<<<<< + * global __cuDeviceGetProperties + * data["__cuDeviceGetProperties"] = __cuDeviceGetProperties +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFlushGPUDirectRDMAWrites)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7083, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFlushGPUDirectRDMAWrites, __pyx_t_3) < 0))) __PYX_ERR(0, 7083, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7085 + * data["__cuFlushGPUDirectRDMAWrites"] = __cuFlushGPUDirectRDMAWrites + * global __cuDeviceGetProperties + * data["__cuDeviceGetProperties"] = __cuDeviceGetProperties # <<<<<<<<<<<<<< + * global __cuDeviceComputeCapability + * data["__cuDeviceComputeCapability"] = __cuDeviceComputeCapability +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetProperties)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7085, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetProperties, __pyx_t_3) < 0))) __PYX_ERR(0, 7085, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7087 + * data["__cuDeviceGetProperties"] = __cuDeviceGetProperties + * global __cuDeviceComputeCapability + * data["__cuDeviceComputeCapability"] = __cuDeviceComputeCapability # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxRetain + * data["__cuDevicePrimaryCtxRetain"] = __cuDevicePrimaryCtxRetain +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceComputeCapability)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7087, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceComputeCapability, __pyx_t_3) < 0))) __PYX_ERR(0, 7087, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7089 + * data["__cuDeviceComputeCapability"] = __cuDeviceComputeCapability + * global __cuDevicePrimaryCtxRetain + * data["__cuDevicePrimaryCtxRetain"] = __cuDevicePrimaryCtxRetain # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxRelease_v2 + * data["__cuDevicePrimaryCtxRelease_v2"] = __cuDevicePrimaryCtxRelease_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRetain)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7089, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDevicePrimaryCtxRetain, __pyx_t_3) < 0))) __PYX_ERR(0, 7089, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7091 + * data["__cuDevicePrimaryCtxRetain"] = __cuDevicePrimaryCtxRetain + * global __cuDevicePrimaryCtxRelease_v2 + * data["__cuDevicePrimaryCtxRelease_v2"] = __cuDevicePrimaryCtxRelease_v2 # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxSetFlags_v2 + * data["__cuDevicePrimaryCtxSetFlags_v2"] = __cuDevicePrimaryCtxSetFlags_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRelease_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7091, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDevicePrimaryCtxRelease_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7091, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7093 + * data["__cuDevicePrimaryCtxRelease_v2"] = __cuDevicePrimaryCtxRelease_v2 + * global __cuDevicePrimaryCtxSetFlags_v2 + * data["__cuDevicePrimaryCtxSetFlags_v2"] = __cuDevicePrimaryCtxSetFlags_v2 # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxGetState + * data["__cuDevicePrimaryCtxGetState"] = __cuDevicePrimaryCtxGetState +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxSetFlags_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7093, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDevicePrimaryCtxSetFlags_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7093, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7095 + * data["__cuDevicePrimaryCtxSetFlags_v2"] = __cuDevicePrimaryCtxSetFlags_v2 + * global __cuDevicePrimaryCtxGetState + * data["__cuDevicePrimaryCtxGetState"] = __cuDevicePrimaryCtxGetState # <<<<<<<<<<<<<< + * global __cuDevicePrimaryCtxReset_v2 + * data["__cuDevicePrimaryCtxReset_v2"] = __cuDevicePrimaryCtxReset_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxGetState)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7095, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDevicePrimaryCtxGetState, __pyx_t_3) < 0))) __PYX_ERR(0, 7095, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7097 + * data["__cuDevicePrimaryCtxGetState"] = __cuDevicePrimaryCtxGetState + * global __cuDevicePrimaryCtxReset_v2 + * data["__cuDevicePrimaryCtxReset_v2"] = __cuDevicePrimaryCtxReset_v2 # <<<<<<<<<<<<<< + * global __cuCtxCreate_v2 + * data["__cuCtxCreate_v2"] = __cuCtxCreate_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxReset_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7097, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDevicePrimaryCtxReset_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7097, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7099 + * data["__cuDevicePrimaryCtxReset_v2"] = __cuDevicePrimaryCtxReset_v2 + * global __cuCtxCreate_v2 + * data["__cuCtxCreate_v2"] = __cuCtxCreate_v2 # <<<<<<<<<<<<<< + * global __cuCtxCreate_v3 + * data["__cuCtxCreate_v3"] = __cuCtxCreate_v3 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7099, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxCreate_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7099, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7101 + * data["__cuCtxCreate_v2"] = __cuCtxCreate_v2 + * global __cuCtxCreate_v3 + * data["__cuCtxCreate_v3"] = __cuCtxCreate_v3 # <<<<<<<<<<<<<< + * global __cuCtxCreate_v4 + * data["__cuCtxCreate_v4"] = __cuCtxCreate_v4 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v3)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7101, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxCreate_v3, __pyx_t_3) < 0))) __PYX_ERR(0, 7101, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7103 + * data["__cuCtxCreate_v3"] = __cuCtxCreate_v3 + * global __cuCtxCreate_v4 + * data["__cuCtxCreate_v4"] = __cuCtxCreate_v4 # <<<<<<<<<<<<<< + * global __cuCtxDestroy_v2 + * data["__cuCtxDestroy_v2"] = __cuCtxDestroy_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v4)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7103, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxCreate_v4, __pyx_t_3) < 0))) __PYX_ERR(0, 7103, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7105 + * data["__cuCtxCreate_v4"] = __cuCtxCreate_v4 + * global __cuCtxDestroy_v2 + * data["__cuCtxDestroy_v2"] = __cuCtxDestroy_v2 # <<<<<<<<<<<<<< + * global __cuCtxPushCurrent_v2 + * data["__cuCtxPushCurrent_v2"] = __cuCtxPushCurrent_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDestroy_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7105, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxDestroy_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7105, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7107 + * data["__cuCtxDestroy_v2"] = __cuCtxDestroy_v2 + * global __cuCtxPushCurrent_v2 + * data["__cuCtxPushCurrent_v2"] = __cuCtxPushCurrent_v2 # <<<<<<<<<<<<<< + * global __cuCtxPopCurrent_v2 + * data["__cuCtxPopCurrent_v2"] = __cuCtxPopCurrent_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPushCurrent_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7107, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxPushCurrent_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7107, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7109 + * data["__cuCtxPushCurrent_v2"] = __cuCtxPushCurrent_v2 + * global __cuCtxPopCurrent_v2 + * data["__cuCtxPopCurrent_v2"] = __cuCtxPopCurrent_v2 # <<<<<<<<<<<<<< + * global __cuCtxSetCurrent + * data["__cuCtxSetCurrent"] = __cuCtxSetCurrent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPopCurrent_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7109, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxPopCurrent_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7109, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7111 + * data["__cuCtxPopCurrent_v2"] = __cuCtxPopCurrent_v2 + * global __cuCtxSetCurrent + * data["__cuCtxSetCurrent"] = __cuCtxSetCurrent # <<<<<<<<<<<<<< + * global __cuCtxGetCurrent + * data["__cuCtxGetCurrent"] = __cuCtxGetCurrent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCurrent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7111, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxSetCurrent, __pyx_t_3) < 0))) __PYX_ERR(0, 7111, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7113 + * data["__cuCtxSetCurrent"] = __cuCtxSetCurrent + * global __cuCtxGetCurrent + * data["__cuCtxGetCurrent"] = __cuCtxGetCurrent # <<<<<<<<<<<<<< + * global __cuCtxGetDevice + * data["__cuCtxGetDevice"] = __cuCtxGetDevice +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCurrent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7113, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxGetCurrent, __pyx_t_3) < 0))) __PYX_ERR(0, 7113, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7115 + * data["__cuCtxGetCurrent"] = __cuCtxGetCurrent + * global __cuCtxGetDevice + * data["__cuCtxGetDevice"] = __cuCtxGetDevice # <<<<<<<<<<<<<< + * global __cuCtxGetFlags + * data["__cuCtxGetFlags"] = __cuCtxGetFlags +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevice)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7115, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxGetDevice, __pyx_t_3) < 0))) __PYX_ERR(0, 7115, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7117 + * data["__cuCtxGetDevice"] = __cuCtxGetDevice + * global __cuCtxGetFlags + * data["__cuCtxGetFlags"] = __cuCtxGetFlags # <<<<<<<<<<<<<< + * global __cuCtxSetFlags + * data["__cuCtxSetFlags"] = __cuCtxSetFlags +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetFlags)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7117, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxGetFlags, __pyx_t_3) < 0))) __PYX_ERR(0, 7117, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7119 + * data["__cuCtxGetFlags"] = __cuCtxGetFlags + * global __cuCtxSetFlags + * data["__cuCtxSetFlags"] = __cuCtxSetFlags # <<<<<<<<<<<<<< + * global __cuCtxGetId + * data["__cuCtxGetId"] = __cuCtxGetId +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetFlags)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7119, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxSetFlags, __pyx_t_3) < 0))) __PYX_ERR(0, 7119, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7121 + * data["__cuCtxSetFlags"] = __cuCtxSetFlags + * global __cuCtxGetId + * data["__cuCtxGetId"] = __cuCtxGetId # <<<<<<<<<<<<<< + * global __cuCtxSynchronize + * data["__cuCtxSynchronize"] = __cuCtxSynchronize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetId)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7121, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxGetId, __pyx_t_3) < 0))) __PYX_ERR(0, 7121, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7123 + * data["__cuCtxGetId"] = __cuCtxGetId + * global __cuCtxSynchronize + * data["__cuCtxSynchronize"] = __cuCtxSynchronize # <<<<<<<<<<<<<< + * global __cuCtxSetLimit + * data["__cuCtxSetLimit"] = __cuCtxSetLimit +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSynchronize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7123, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxSynchronize, __pyx_t_3) < 0))) __PYX_ERR(0, 7123, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7125 + * data["__cuCtxSynchronize"] = __cuCtxSynchronize + * global __cuCtxSetLimit + * data["__cuCtxSetLimit"] = __cuCtxSetLimit # <<<<<<<<<<<<<< + * global __cuCtxGetLimit + * data["__cuCtxGetLimit"] = __cuCtxGetLimit +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetLimit)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7125, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxSetLimit, __pyx_t_3) < 0))) __PYX_ERR(0, 7125, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7127 + * data["__cuCtxSetLimit"] = __cuCtxSetLimit + * global __cuCtxGetLimit + * data["__cuCtxGetLimit"] = __cuCtxGetLimit # <<<<<<<<<<<<<< + * global __cuCtxGetCacheConfig + * data["__cuCtxGetCacheConfig"] = __cuCtxGetCacheConfig +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetLimit)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7127, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxGetLimit, __pyx_t_3) < 0))) __PYX_ERR(0, 7127, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7129 + * data["__cuCtxGetLimit"] = __cuCtxGetLimit + * global __cuCtxGetCacheConfig + * data["__cuCtxGetCacheConfig"] = __cuCtxGetCacheConfig # <<<<<<<<<<<<<< + * global __cuCtxSetCacheConfig + * data["__cuCtxSetCacheConfig"] = __cuCtxSetCacheConfig +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCacheConfig)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7129, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxGetCacheConfig, __pyx_t_3) < 0))) __PYX_ERR(0, 7129, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7131 + * data["__cuCtxGetCacheConfig"] = __cuCtxGetCacheConfig + * global __cuCtxSetCacheConfig + * data["__cuCtxSetCacheConfig"] = __cuCtxSetCacheConfig # <<<<<<<<<<<<<< + * global __cuCtxGetApiVersion + * data["__cuCtxGetApiVersion"] = __cuCtxGetApiVersion +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCacheConfig)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7131, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxSetCacheConfig, __pyx_t_3) < 0))) __PYX_ERR(0, 7131, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7133 + * data["__cuCtxSetCacheConfig"] = __cuCtxSetCacheConfig + * global __cuCtxGetApiVersion + * data["__cuCtxGetApiVersion"] = __cuCtxGetApiVersion # <<<<<<<<<<<<<< + * global __cuCtxGetStreamPriorityRange + * data["__cuCtxGetStreamPriorityRange"] = __cuCtxGetStreamPriorityRange +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetApiVersion)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7133, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxGetApiVersion, __pyx_t_3) < 0))) __PYX_ERR(0, 7133, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7135 + * data["__cuCtxGetApiVersion"] = __cuCtxGetApiVersion + * global __cuCtxGetStreamPriorityRange + * data["__cuCtxGetStreamPriorityRange"] = __cuCtxGetStreamPriorityRange # <<<<<<<<<<<<<< + * global __cuCtxResetPersistingL2Cache + * data["__cuCtxResetPersistingL2Cache"] = __cuCtxResetPersistingL2Cache +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetStreamPriorityRange)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7135, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxGetStreamPriorityRange, __pyx_t_3) < 0))) __PYX_ERR(0, 7135, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7137 + * data["__cuCtxGetStreamPriorityRange"] = __cuCtxGetStreamPriorityRange + * global __cuCtxResetPersistingL2Cache + * data["__cuCtxResetPersistingL2Cache"] = __cuCtxResetPersistingL2Cache # <<<<<<<<<<<<<< + * global __cuCtxGetExecAffinity + * data["__cuCtxGetExecAffinity"] = __cuCtxGetExecAffinity +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxResetPersistingL2Cache)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7137, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxResetPersistingL2Cache, __pyx_t_3) < 0))) __PYX_ERR(0, 7137, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7139 + * data["__cuCtxResetPersistingL2Cache"] = __cuCtxResetPersistingL2Cache + * global __cuCtxGetExecAffinity + * data["__cuCtxGetExecAffinity"] = __cuCtxGetExecAffinity # <<<<<<<<<<<<<< + * global __cuCtxRecordEvent + * data["__cuCtxRecordEvent"] = __cuCtxRecordEvent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetExecAffinity)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7139, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxGetExecAffinity, __pyx_t_3) < 0))) __PYX_ERR(0, 7139, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7141 + * data["__cuCtxGetExecAffinity"] = __cuCtxGetExecAffinity + * global __cuCtxRecordEvent + * data["__cuCtxRecordEvent"] = __cuCtxRecordEvent # <<<<<<<<<<<<<< + * global __cuCtxWaitEvent + * data["__cuCtxWaitEvent"] = __cuCtxWaitEvent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxRecordEvent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7141, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxRecordEvent, __pyx_t_3) < 0))) __PYX_ERR(0, 7141, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7143 + * data["__cuCtxRecordEvent"] = __cuCtxRecordEvent + * global __cuCtxWaitEvent + * data["__cuCtxWaitEvent"] = __cuCtxWaitEvent # <<<<<<<<<<<<<< + * global __cuCtxAttach + * data["__cuCtxAttach"] = __cuCtxAttach +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxWaitEvent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7143, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxWaitEvent, __pyx_t_3) < 0))) __PYX_ERR(0, 7143, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7145 + * data["__cuCtxWaitEvent"] = __cuCtxWaitEvent + * global __cuCtxAttach + * data["__cuCtxAttach"] = __cuCtxAttach # <<<<<<<<<<<<<< + * global __cuCtxDetach + * data["__cuCtxDetach"] = __cuCtxDetach +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxAttach)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7145, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxAttach, __pyx_t_3) < 0))) __PYX_ERR(0, 7145, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7147 + * data["__cuCtxAttach"] = __cuCtxAttach + * global __cuCtxDetach + * data["__cuCtxDetach"] = __cuCtxDetach # <<<<<<<<<<<<<< + * global __cuCtxGetSharedMemConfig + * data["__cuCtxGetSharedMemConfig"] = __cuCtxGetSharedMemConfig +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDetach)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7147, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxDetach, __pyx_t_3) < 0))) __PYX_ERR(0, 7147, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7149 + * data["__cuCtxDetach"] = __cuCtxDetach + * global __cuCtxGetSharedMemConfig + * data["__cuCtxGetSharedMemConfig"] = __cuCtxGetSharedMemConfig # <<<<<<<<<<<<<< + * global __cuCtxSetSharedMemConfig + * data["__cuCtxSetSharedMemConfig"] = __cuCtxSetSharedMemConfig +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetSharedMemConfig)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7149, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxGetSharedMemConfig, __pyx_t_3) < 0))) __PYX_ERR(0, 7149, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7151 + * data["__cuCtxGetSharedMemConfig"] = __cuCtxGetSharedMemConfig + * global __cuCtxSetSharedMemConfig + * data["__cuCtxSetSharedMemConfig"] = __cuCtxSetSharedMemConfig # <<<<<<<<<<<<<< + * global __cuModuleLoad + * data["__cuModuleLoad"] = __cuModuleLoad +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetSharedMemConfig)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7151, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxSetSharedMemConfig, __pyx_t_3) < 0))) __PYX_ERR(0, 7151, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7153 + * data["__cuCtxSetSharedMemConfig"] = __cuCtxSetSharedMemConfig + * global __cuModuleLoad + * data["__cuModuleLoad"] = __cuModuleLoad # <<<<<<<<<<<<<< + * global __cuModuleLoadData + * data["__cuModuleLoadData"] = __cuModuleLoadData +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoad)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7153, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuModuleLoad, __pyx_t_3) < 0))) __PYX_ERR(0, 7153, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7155 + * data["__cuModuleLoad"] = __cuModuleLoad + * global __cuModuleLoadData + * data["__cuModuleLoadData"] = __cuModuleLoadData # <<<<<<<<<<<<<< + * global __cuModuleLoadDataEx + * data["__cuModuleLoadDataEx"] = __cuModuleLoadDataEx +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadData)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7155, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuModuleLoadData, __pyx_t_3) < 0))) __PYX_ERR(0, 7155, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7157 + * data["__cuModuleLoadData"] = __cuModuleLoadData + * global __cuModuleLoadDataEx + * data["__cuModuleLoadDataEx"] = __cuModuleLoadDataEx # <<<<<<<<<<<<<< + * global __cuModuleLoadFatBinary + * data["__cuModuleLoadFatBinary"] = __cuModuleLoadFatBinary +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadDataEx)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7157, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuModuleLoadDataEx, __pyx_t_3) < 0))) __PYX_ERR(0, 7157, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7159 + * data["__cuModuleLoadDataEx"] = __cuModuleLoadDataEx + * global __cuModuleLoadFatBinary + * data["__cuModuleLoadFatBinary"] = __cuModuleLoadFatBinary # <<<<<<<<<<<<<< + * global __cuModuleUnload + * data["__cuModuleUnload"] = __cuModuleUnload +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadFatBinary)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7159, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuModuleLoadFatBinary, __pyx_t_3) < 0))) __PYX_ERR(0, 7159, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7161 + * data["__cuModuleLoadFatBinary"] = __cuModuleLoadFatBinary + * global __cuModuleUnload + * data["__cuModuleUnload"] = __cuModuleUnload # <<<<<<<<<<<<<< + * global __cuModuleGetLoadingMode + * data["__cuModuleGetLoadingMode"] = __cuModuleGetLoadingMode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleUnload)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7161, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuModuleUnload, __pyx_t_3) < 0))) __PYX_ERR(0, 7161, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7163 + * data["__cuModuleUnload"] = __cuModuleUnload + * global __cuModuleGetLoadingMode + * data["__cuModuleGetLoadingMode"] = __cuModuleGetLoadingMode # <<<<<<<<<<<<<< + * global __cuModuleGetFunction + * data["__cuModuleGetFunction"] = __cuModuleGetFunction +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetLoadingMode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7163, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuModuleGetLoadingMode, __pyx_t_3) < 0))) __PYX_ERR(0, 7163, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7165 + * data["__cuModuleGetLoadingMode"] = __cuModuleGetLoadingMode + * global __cuModuleGetFunction + * data["__cuModuleGetFunction"] = __cuModuleGetFunction # <<<<<<<<<<<<<< + * global __cuModuleGetFunctionCount + * data["__cuModuleGetFunctionCount"] = __cuModuleGetFunctionCount +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunction)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7165, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuModuleGetFunction, __pyx_t_3) < 0))) __PYX_ERR(0, 7165, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7167 + * data["__cuModuleGetFunction"] = __cuModuleGetFunction + * global __cuModuleGetFunctionCount + * data["__cuModuleGetFunctionCount"] = __cuModuleGetFunctionCount # <<<<<<<<<<<<<< + * global __cuModuleEnumerateFunctions + * data["__cuModuleEnumerateFunctions"] = __cuModuleEnumerateFunctions +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunctionCount)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7167, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuModuleGetFunctionCount, __pyx_t_3) < 0))) __PYX_ERR(0, 7167, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7169 + * data["__cuModuleGetFunctionCount"] = __cuModuleGetFunctionCount + * global __cuModuleEnumerateFunctions + * data["__cuModuleEnumerateFunctions"] = __cuModuleEnumerateFunctions # <<<<<<<<<<<<<< + * global __cuModuleGetGlobal_v2 + * data["__cuModuleGetGlobal_v2"] = __cuModuleGetGlobal_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleEnumerateFunctions)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7169, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuModuleEnumerateFunctions, __pyx_t_3) < 0))) __PYX_ERR(0, 7169, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7171 + * data["__cuModuleEnumerateFunctions"] = __cuModuleEnumerateFunctions + * global __cuModuleGetGlobal_v2 + * data["__cuModuleGetGlobal_v2"] = __cuModuleGetGlobal_v2 # <<<<<<<<<<<<<< + * global __cuLinkCreate_v2 + * data["__cuLinkCreate_v2"] = __cuLinkCreate_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetGlobal_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7171, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuModuleGetGlobal_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7171, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7173 + * data["__cuModuleGetGlobal_v2"] = __cuModuleGetGlobal_v2 + * global __cuLinkCreate_v2 + * data["__cuLinkCreate_v2"] = __cuLinkCreate_v2 # <<<<<<<<<<<<<< + * global __cuLinkAddData_v2 + * data["__cuLinkAddData_v2"] = __cuLinkAddData_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkCreate_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7173, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLinkCreate_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7173, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7175 + * data["__cuLinkCreate_v2"] = __cuLinkCreate_v2 + * global __cuLinkAddData_v2 + * data["__cuLinkAddData_v2"] = __cuLinkAddData_v2 # <<<<<<<<<<<<<< + * global __cuLinkAddFile_v2 + * data["__cuLinkAddFile_v2"] = __cuLinkAddFile_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddData_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7175, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLinkAddData_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7175, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7177 + * data["__cuLinkAddData_v2"] = __cuLinkAddData_v2 + * global __cuLinkAddFile_v2 + * data["__cuLinkAddFile_v2"] = __cuLinkAddFile_v2 # <<<<<<<<<<<<<< + * global __cuLinkComplete + * data["__cuLinkComplete"] = __cuLinkComplete +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddFile_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7177, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLinkAddFile_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7177, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7179 + * data["__cuLinkAddFile_v2"] = __cuLinkAddFile_v2 + * global __cuLinkComplete + * data["__cuLinkComplete"] = __cuLinkComplete # <<<<<<<<<<<<<< + * global __cuLinkDestroy + * data["__cuLinkDestroy"] = __cuLinkDestroy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkComplete)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7179, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLinkComplete, __pyx_t_3) < 0))) __PYX_ERR(0, 7179, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7181 + * data["__cuLinkComplete"] = __cuLinkComplete + * global __cuLinkDestroy + * data["__cuLinkDestroy"] = __cuLinkDestroy # <<<<<<<<<<<<<< + * global __cuModuleGetTexRef + * data["__cuModuleGetTexRef"] = __cuModuleGetTexRef +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkDestroy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7181, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLinkDestroy, __pyx_t_3) < 0))) __PYX_ERR(0, 7181, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7183 + * data["__cuLinkDestroy"] = __cuLinkDestroy + * global __cuModuleGetTexRef + * data["__cuModuleGetTexRef"] = __cuModuleGetTexRef # <<<<<<<<<<<<<< + * global __cuModuleGetSurfRef + * data["__cuModuleGetSurfRef"] = __cuModuleGetSurfRef +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetTexRef)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7183, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuModuleGetTexRef, __pyx_t_3) < 0))) __PYX_ERR(0, 7183, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7185 + * data["__cuModuleGetTexRef"] = __cuModuleGetTexRef + * global __cuModuleGetSurfRef + * data["__cuModuleGetSurfRef"] = __cuModuleGetSurfRef # <<<<<<<<<<<<<< + * global __cuLibraryLoadData + * data["__cuLibraryLoadData"] = __cuLibraryLoadData +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetSurfRef)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7185, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuModuleGetSurfRef, __pyx_t_3) < 0))) __PYX_ERR(0, 7185, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7187 + * data["__cuModuleGetSurfRef"] = __cuModuleGetSurfRef + * global __cuLibraryLoadData + * data["__cuLibraryLoadData"] = __cuLibraryLoadData # <<<<<<<<<<<<<< + * global __cuLibraryLoadFromFile + * data["__cuLibraryLoadFromFile"] = __cuLibraryLoadFromFile +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadData)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7187, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLibraryLoadData, __pyx_t_3) < 0))) __PYX_ERR(0, 7187, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7189 + * data["__cuLibraryLoadData"] = __cuLibraryLoadData + * global __cuLibraryLoadFromFile + * data["__cuLibraryLoadFromFile"] = __cuLibraryLoadFromFile # <<<<<<<<<<<<<< + * global __cuLibraryUnload + * data["__cuLibraryUnload"] = __cuLibraryUnload +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadFromFile)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7189, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLibraryLoadFromFile, __pyx_t_3) < 0))) __PYX_ERR(0, 7189, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7191 + * data["__cuLibraryLoadFromFile"] = __cuLibraryLoadFromFile + * global __cuLibraryUnload + * data["__cuLibraryUnload"] = __cuLibraryUnload # <<<<<<<<<<<<<< + * global __cuLibraryGetKernel + * data["__cuLibraryGetKernel"] = __cuLibraryGetKernel +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryUnload)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7191, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLibraryUnload, __pyx_t_3) < 0))) __PYX_ERR(0, 7191, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7193 + * data["__cuLibraryUnload"] = __cuLibraryUnload + * global __cuLibraryGetKernel + * data["__cuLibraryGetKernel"] = __cuLibraryGetKernel # <<<<<<<<<<<<<< + * global __cuLibraryGetKernelCount + * data["__cuLibraryGetKernelCount"] = __cuLibraryGetKernelCount +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernel)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLibraryGetKernel, __pyx_t_3) < 0))) __PYX_ERR(0, 7193, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7195 + * data["__cuLibraryGetKernel"] = __cuLibraryGetKernel + * global __cuLibraryGetKernelCount + * data["__cuLibraryGetKernelCount"] = __cuLibraryGetKernelCount # <<<<<<<<<<<<<< + * global __cuLibraryEnumerateKernels + * data["__cuLibraryEnumerateKernels"] = __cuLibraryEnumerateKernels +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernelCount)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7195, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLibraryGetKernelCount, __pyx_t_3) < 0))) __PYX_ERR(0, 7195, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7197 + * data["__cuLibraryGetKernelCount"] = __cuLibraryGetKernelCount + * global __cuLibraryEnumerateKernels + * data["__cuLibraryEnumerateKernels"] = __cuLibraryEnumerateKernels # <<<<<<<<<<<<<< + * global __cuLibraryGetModule + * data["__cuLibraryGetModule"] = __cuLibraryGetModule +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryEnumerateKernels)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7197, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLibraryEnumerateKernels, __pyx_t_3) < 0))) __PYX_ERR(0, 7197, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7199 + * data["__cuLibraryEnumerateKernels"] = __cuLibraryEnumerateKernels + * global __cuLibraryGetModule + * data["__cuLibraryGetModule"] = __cuLibraryGetModule # <<<<<<<<<<<<<< + * global __cuKernelGetFunction + * data["__cuKernelGetFunction"] = __cuKernelGetFunction +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetModule)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7199, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLibraryGetModule, __pyx_t_3) < 0))) __PYX_ERR(0, 7199, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7201 + * data["__cuLibraryGetModule"] = __cuLibraryGetModule + * global __cuKernelGetFunction + * data["__cuKernelGetFunction"] = __cuKernelGetFunction # <<<<<<<<<<<<<< + * global __cuKernelGetLibrary + * data["__cuKernelGetLibrary"] = __cuKernelGetLibrary +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetFunction)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7201, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuKernelGetFunction, __pyx_t_3) < 0))) __PYX_ERR(0, 7201, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7203 + * data["__cuKernelGetFunction"] = __cuKernelGetFunction + * global __cuKernelGetLibrary + * data["__cuKernelGetLibrary"] = __cuKernelGetLibrary # <<<<<<<<<<<<<< + * global __cuLibraryGetGlobal + * data["__cuLibraryGetGlobal"] = __cuLibraryGetGlobal +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetLibrary)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7203, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuKernelGetLibrary, __pyx_t_3) < 0))) __PYX_ERR(0, 7203, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7205 + * data["__cuKernelGetLibrary"] = __cuKernelGetLibrary + * global __cuLibraryGetGlobal + * data["__cuLibraryGetGlobal"] = __cuLibraryGetGlobal # <<<<<<<<<<<<<< + * global __cuLibraryGetManaged + * data["__cuLibraryGetManaged"] = __cuLibraryGetManaged +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetGlobal)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7205, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLibraryGetGlobal, __pyx_t_3) < 0))) __PYX_ERR(0, 7205, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7207 + * data["__cuLibraryGetGlobal"] = __cuLibraryGetGlobal + * global __cuLibraryGetManaged + * data["__cuLibraryGetManaged"] = __cuLibraryGetManaged # <<<<<<<<<<<<<< + * global __cuLibraryGetUnifiedFunction + * data["__cuLibraryGetUnifiedFunction"] = __cuLibraryGetUnifiedFunction +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetManaged)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7207, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLibraryGetManaged, __pyx_t_3) < 0))) __PYX_ERR(0, 7207, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7209 + * data["__cuLibraryGetManaged"] = __cuLibraryGetManaged + * global __cuLibraryGetUnifiedFunction + * data["__cuLibraryGetUnifiedFunction"] = __cuLibraryGetUnifiedFunction # <<<<<<<<<<<<<< + * global __cuKernelGetAttribute + * data["__cuKernelGetAttribute"] = __cuKernelGetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetUnifiedFunction)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7209, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLibraryGetUnifiedFunction, __pyx_t_3) < 0))) __PYX_ERR(0, 7209, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7211 + * data["__cuLibraryGetUnifiedFunction"] = __cuLibraryGetUnifiedFunction + * global __cuKernelGetAttribute + * data["__cuKernelGetAttribute"] = __cuKernelGetAttribute # <<<<<<<<<<<<<< + * global __cuKernelSetAttribute + * data["__cuKernelSetAttribute"] = __cuKernelSetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7211, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuKernelGetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7211, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7213 + * data["__cuKernelGetAttribute"] = __cuKernelGetAttribute + * global __cuKernelSetAttribute + * data["__cuKernelSetAttribute"] = __cuKernelSetAttribute # <<<<<<<<<<<<<< + * global __cuKernelSetCacheConfig + * data["__cuKernelSetCacheConfig"] = __cuKernelSetCacheConfig +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7213, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuKernelSetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7213, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7215 + * data["__cuKernelSetAttribute"] = __cuKernelSetAttribute + * global __cuKernelSetCacheConfig + * data["__cuKernelSetCacheConfig"] = __cuKernelSetCacheConfig # <<<<<<<<<<<<<< + * global __cuKernelGetName + * data["__cuKernelGetName"] = __cuKernelGetName +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetCacheConfig)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7215, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuKernelSetCacheConfig, __pyx_t_3) < 0))) __PYX_ERR(0, 7215, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7217 + * data["__cuKernelSetCacheConfig"] = __cuKernelSetCacheConfig + * global __cuKernelGetName + * data["__cuKernelGetName"] = __cuKernelGetName # <<<<<<<<<<<<<< + * global __cuKernelGetParamInfo + * data["__cuKernelGetParamInfo"] = __cuKernelGetParamInfo +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetName)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7217, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuKernelGetName, __pyx_t_3) < 0))) __PYX_ERR(0, 7217, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7219 + * data["__cuKernelGetName"] = __cuKernelGetName + * global __cuKernelGetParamInfo + * data["__cuKernelGetParamInfo"] = __cuKernelGetParamInfo # <<<<<<<<<<<<<< + * global __cuMemGetInfo_v2 + * data["__cuMemGetInfo_v2"] = __cuMemGetInfo_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetParamInfo)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7219, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuKernelGetParamInfo, __pyx_t_3) < 0))) __PYX_ERR(0, 7219, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7221 + * data["__cuKernelGetParamInfo"] = __cuKernelGetParamInfo + * global __cuMemGetInfo_v2 + * data["__cuMemGetInfo_v2"] = __cuMemGetInfo_v2 # <<<<<<<<<<<<<< + * global __cuMemAlloc_v2 + * data["__cuMemAlloc_v2"] = __cuMemAlloc_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetInfo_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7221, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemGetInfo_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7221, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7223 + * data["__cuMemGetInfo_v2"] = __cuMemGetInfo_v2 + * global __cuMemAlloc_v2 + * data["__cuMemAlloc_v2"] = __cuMemAlloc_v2 # <<<<<<<<<<<<<< + * global __cuMemAllocPitch_v2 + * data["__cuMemAllocPitch_v2"] = __cuMemAllocPitch_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAlloc_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7223, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemAlloc_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7223, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7225 + * data["__cuMemAlloc_v2"] = __cuMemAlloc_v2 + * global __cuMemAllocPitch_v2 + * data["__cuMemAllocPitch_v2"] = __cuMemAllocPitch_v2 # <<<<<<<<<<<<<< + * global __cuMemFree_v2 + * data["__cuMemFree_v2"] = __cuMemFree_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocPitch_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7225, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemAllocPitch_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7225, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7227 + * data["__cuMemAllocPitch_v2"] = __cuMemAllocPitch_v2 + * global __cuMemFree_v2 + * data["__cuMemFree_v2"] = __cuMemFree_v2 # <<<<<<<<<<<<<< + * global __cuMemGetAddressRange_v2 + * data["__cuMemGetAddressRange_v2"] = __cuMemGetAddressRange_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFree_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7227, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemFree_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7227, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7229 + * data["__cuMemFree_v2"] = __cuMemFree_v2 + * global __cuMemGetAddressRange_v2 + * data["__cuMemGetAddressRange_v2"] = __cuMemGetAddressRange_v2 # <<<<<<<<<<<<<< + * global __cuMemAllocHost_v2 + * data["__cuMemAllocHost_v2"] = __cuMemAllocHost_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAddressRange_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7229, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemGetAddressRange_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7229, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7231 + * data["__cuMemGetAddressRange_v2"] = __cuMemGetAddressRange_v2 + * global __cuMemAllocHost_v2 + * data["__cuMemAllocHost_v2"] = __cuMemAllocHost_v2 # <<<<<<<<<<<<<< + * global __cuMemFreeHost + * data["__cuMemFreeHost"] = __cuMemFreeHost +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocHost_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7231, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemAllocHost_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7231, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7233 + * data["__cuMemAllocHost_v2"] = __cuMemAllocHost_v2 + * global __cuMemFreeHost + * data["__cuMemFreeHost"] = __cuMemFreeHost # <<<<<<<<<<<<<< + * global __cuMemHostAlloc + * data["__cuMemHostAlloc"] = __cuMemHostAlloc +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeHost)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7233, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemFreeHost, __pyx_t_3) < 0))) __PYX_ERR(0, 7233, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7235 + * data["__cuMemFreeHost"] = __cuMemFreeHost + * global __cuMemHostAlloc + * data["__cuMemHostAlloc"] = __cuMemHostAlloc # <<<<<<<<<<<<<< + * global __cuMemHostGetDevicePointer_v2 + * data["__cuMemHostGetDevicePointer_v2"] = __cuMemHostGetDevicePointer_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostAlloc)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7235, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemHostAlloc, __pyx_t_3) < 0))) __PYX_ERR(0, 7235, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7237 + * data["__cuMemHostAlloc"] = __cuMemHostAlloc + * global __cuMemHostGetDevicePointer_v2 + * data["__cuMemHostGetDevicePointer_v2"] = __cuMemHostGetDevicePointer_v2 # <<<<<<<<<<<<<< + * global __cuMemHostGetFlags + * data["__cuMemHostGetFlags"] = __cuMemHostGetFlags +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetDevicePointer_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7237, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemHostGetDevicePointer_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7237, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7239 + * data["__cuMemHostGetDevicePointer_v2"] = __cuMemHostGetDevicePointer_v2 + * global __cuMemHostGetFlags + * data["__cuMemHostGetFlags"] = __cuMemHostGetFlags # <<<<<<<<<<<<<< + * global __cuMemAllocManaged + * data["__cuMemAllocManaged"] = __cuMemAllocManaged +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetFlags)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7239, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemHostGetFlags, __pyx_t_3) < 0))) __PYX_ERR(0, 7239, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7241 + * data["__cuMemHostGetFlags"] = __cuMemHostGetFlags + * global __cuMemAllocManaged + * data["__cuMemAllocManaged"] = __cuMemAllocManaged # <<<<<<<<<<<<<< + * global __cuDeviceRegisterAsyncNotification + * data["__cuDeviceRegisterAsyncNotification"] = __cuDeviceRegisterAsyncNotification +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocManaged)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7241, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemAllocManaged, __pyx_t_3) < 0))) __PYX_ERR(0, 7241, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7243 + * data["__cuMemAllocManaged"] = __cuMemAllocManaged + * global __cuDeviceRegisterAsyncNotification + * data["__cuDeviceRegisterAsyncNotification"] = __cuDeviceRegisterAsyncNotification # <<<<<<<<<<<<<< + * global __cuDeviceUnregisterAsyncNotification + * data["__cuDeviceUnregisterAsyncNotification"] = __cuDeviceUnregisterAsyncNotification +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceRegisterAsyncNotification)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7243, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceRegisterAsyncNotificat, __pyx_t_3) < 0))) __PYX_ERR(0, 7243, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7245 + * data["__cuDeviceRegisterAsyncNotification"] = __cuDeviceRegisterAsyncNotification + * global __cuDeviceUnregisterAsyncNotification + * data["__cuDeviceUnregisterAsyncNotification"] = __cuDeviceUnregisterAsyncNotification # <<<<<<<<<<<<<< + * global __cuDeviceGetByPCIBusId + * data["__cuDeviceGetByPCIBusId"] = __cuDeviceGetByPCIBusId +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceUnregisterAsyncNotification)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7245, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceUnregisterAsyncNotific, __pyx_t_3) < 0))) __PYX_ERR(0, 7245, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7247 + * data["__cuDeviceUnregisterAsyncNotification"] = __cuDeviceUnregisterAsyncNotification + * global __cuDeviceGetByPCIBusId + * data["__cuDeviceGetByPCIBusId"] = __cuDeviceGetByPCIBusId # <<<<<<<<<<<<<< + * global __cuDeviceGetPCIBusId + * data["__cuDeviceGetPCIBusId"] = __cuDeviceGetPCIBusId +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetByPCIBusId)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7247, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetByPCIBusId, __pyx_t_3) < 0))) __PYX_ERR(0, 7247, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7249 + * data["__cuDeviceGetByPCIBusId"] = __cuDeviceGetByPCIBusId + * global __cuDeviceGetPCIBusId + * data["__cuDeviceGetPCIBusId"] = __cuDeviceGetPCIBusId # <<<<<<<<<<<<<< + * global __cuIpcGetEventHandle + * data["__cuIpcGetEventHandle"] = __cuIpcGetEventHandle +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetPCIBusId)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7249, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetPCIBusId, __pyx_t_3) < 0))) __PYX_ERR(0, 7249, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7251 + * data["__cuDeviceGetPCIBusId"] = __cuDeviceGetPCIBusId + * global __cuIpcGetEventHandle + * data["__cuIpcGetEventHandle"] = __cuIpcGetEventHandle # <<<<<<<<<<<<<< + * global __cuIpcOpenEventHandle + * data["__cuIpcOpenEventHandle"] = __cuIpcOpenEventHandle +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetEventHandle)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7251, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuIpcGetEventHandle, __pyx_t_3) < 0))) __PYX_ERR(0, 7251, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7253 + * data["__cuIpcGetEventHandle"] = __cuIpcGetEventHandle + * global __cuIpcOpenEventHandle + * data["__cuIpcOpenEventHandle"] = __cuIpcOpenEventHandle # <<<<<<<<<<<<<< + * global __cuIpcGetMemHandle + * data["__cuIpcGetMemHandle"] = __cuIpcGetMemHandle +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenEventHandle)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7253, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuIpcOpenEventHandle, __pyx_t_3) < 0))) __PYX_ERR(0, 7253, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7255 + * data["__cuIpcOpenEventHandle"] = __cuIpcOpenEventHandle + * global __cuIpcGetMemHandle + * data["__cuIpcGetMemHandle"] = __cuIpcGetMemHandle # <<<<<<<<<<<<<< + * global __cuIpcOpenMemHandle_v2 + * data["__cuIpcOpenMemHandle_v2"] = __cuIpcOpenMemHandle_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetMemHandle)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7255, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuIpcGetMemHandle, __pyx_t_3) < 0))) __PYX_ERR(0, 7255, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7257 + * data["__cuIpcGetMemHandle"] = __cuIpcGetMemHandle + * global __cuIpcOpenMemHandle_v2 + * data["__cuIpcOpenMemHandle_v2"] = __cuIpcOpenMemHandle_v2 # <<<<<<<<<<<<<< + * global __cuIpcCloseMemHandle + * data["__cuIpcCloseMemHandle"] = __cuIpcCloseMemHandle +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenMemHandle_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7257, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuIpcOpenMemHandle_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7257, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7259 + * data["__cuIpcOpenMemHandle_v2"] = __cuIpcOpenMemHandle_v2 + * global __cuIpcCloseMemHandle + * data["__cuIpcCloseMemHandle"] = __cuIpcCloseMemHandle # <<<<<<<<<<<<<< + * global __cuMemHostRegister_v2 + * data["__cuMemHostRegister_v2"] = __cuMemHostRegister_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcCloseMemHandle)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7259, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuIpcCloseMemHandle, __pyx_t_3) < 0))) __PYX_ERR(0, 7259, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7261 + * data["__cuIpcCloseMemHandle"] = __cuIpcCloseMemHandle + * global __cuMemHostRegister_v2 + * data["__cuMemHostRegister_v2"] = __cuMemHostRegister_v2 # <<<<<<<<<<<<<< + * global __cuMemHostUnregister + * data["__cuMemHostUnregister"] = __cuMemHostUnregister +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostRegister_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7261, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemHostRegister_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7261, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7263 + * data["__cuMemHostRegister_v2"] = __cuMemHostRegister_v2 + * global __cuMemHostUnregister + * data["__cuMemHostUnregister"] = __cuMemHostUnregister # <<<<<<<<<<<<<< + * global __cuMemcpy + * data["__cuMemcpy"] = __cuMemcpy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostUnregister)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7263, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemHostUnregister, __pyx_t_3) < 0))) __PYX_ERR(0, 7263, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7265 + * data["__cuMemHostUnregister"] = __cuMemHostUnregister + * global __cuMemcpy + * data["__cuMemcpy"] = __cuMemcpy # <<<<<<<<<<<<<< + * global __cuMemcpyPeer + * data["__cuMemcpyPeer"] = __cuMemcpyPeer +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7265, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpy, __pyx_t_3) < 0))) __PYX_ERR(0, 7265, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7267 + * data["__cuMemcpy"] = __cuMemcpy + * global __cuMemcpyPeer + * data["__cuMemcpyPeer"] = __cuMemcpyPeer # <<<<<<<<<<<<<< + * global __cuMemcpyHtoD_v2 + * data["__cuMemcpyHtoD_v2"] = __cuMemcpyHtoD_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeer)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7267, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyPeer, __pyx_t_3) < 0))) __PYX_ERR(0, 7267, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7269 + * data["__cuMemcpyPeer"] = __cuMemcpyPeer + * global __cuMemcpyHtoD_v2 + * data["__cuMemcpyHtoD_v2"] = __cuMemcpyHtoD_v2 # <<<<<<<<<<<<<< + * global __cuMemcpyDtoH_v2 + * data["__cuMemcpyDtoH_v2"] = __cuMemcpyDtoH_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoD_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7269, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyHtoD_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7269, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7271 + * data["__cuMemcpyHtoD_v2"] = __cuMemcpyHtoD_v2 + * global __cuMemcpyDtoH_v2 + * data["__cuMemcpyDtoH_v2"] = __cuMemcpyDtoH_v2 # <<<<<<<<<<<<<< + * global __cuMemcpyDtoD_v2 + * data["__cuMemcpyDtoD_v2"] = __cuMemcpyDtoD_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoH_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7271, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyDtoH_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7271, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7273 + * data["__cuMemcpyDtoH_v2"] = __cuMemcpyDtoH_v2 + * global __cuMemcpyDtoD_v2 + * data["__cuMemcpyDtoD_v2"] = __cuMemcpyDtoD_v2 # <<<<<<<<<<<<<< + * global __cuMemcpyDtoA_v2 + * data["__cuMemcpyDtoA_v2"] = __cuMemcpyDtoA_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoD_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7273, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyDtoD_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7273, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7275 + * data["__cuMemcpyDtoD_v2"] = __cuMemcpyDtoD_v2 + * global __cuMemcpyDtoA_v2 + * data["__cuMemcpyDtoA_v2"] = __cuMemcpyDtoA_v2 # <<<<<<<<<<<<<< + * global __cuMemcpyAtoD_v2 + * data["__cuMemcpyAtoD_v2"] = __cuMemcpyAtoD_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoA_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7275, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyDtoA_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7275, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7277 + * data["__cuMemcpyDtoA_v2"] = __cuMemcpyDtoA_v2 + * global __cuMemcpyAtoD_v2 + * data["__cuMemcpyAtoD_v2"] = __cuMemcpyAtoD_v2 # <<<<<<<<<<<<<< + * global __cuMemcpyHtoA_v2 + * data["__cuMemcpyHtoA_v2"] = __cuMemcpyHtoA_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoD_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7277, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyAtoD_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7277, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7279 + * data["__cuMemcpyAtoD_v2"] = __cuMemcpyAtoD_v2 + * global __cuMemcpyHtoA_v2 + * data["__cuMemcpyHtoA_v2"] = __cuMemcpyHtoA_v2 # <<<<<<<<<<<<<< + * global __cuMemcpyAtoH_v2 + * data["__cuMemcpyAtoH_v2"] = __cuMemcpyAtoH_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoA_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7279, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyHtoA_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7279, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7281 + * data["__cuMemcpyHtoA_v2"] = __cuMemcpyHtoA_v2 + * global __cuMemcpyAtoH_v2 + * data["__cuMemcpyAtoH_v2"] = __cuMemcpyAtoH_v2 # <<<<<<<<<<<<<< + * global __cuMemcpyAtoA_v2 + * data["__cuMemcpyAtoA_v2"] = __cuMemcpyAtoA_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoH_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7281, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyAtoH_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7281, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7283 + * data["__cuMemcpyAtoH_v2"] = __cuMemcpyAtoH_v2 + * global __cuMemcpyAtoA_v2 + * data["__cuMemcpyAtoA_v2"] = __cuMemcpyAtoA_v2 # <<<<<<<<<<<<<< + * global __cuMemcpy2D_v2 + * data["__cuMemcpy2D_v2"] = __cuMemcpy2D_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoA_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7283, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyAtoA_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7283, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7285 + * data["__cuMemcpyAtoA_v2"] = __cuMemcpyAtoA_v2 + * global __cuMemcpy2D_v2 + * data["__cuMemcpy2D_v2"] = __cuMemcpy2D_v2 # <<<<<<<<<<<<<< + * global __cuMemcpy2DUnaligned_v2 + * data["__cuMemcpy2DUnaligned_v2"] = __cuMemcpy2DUnaligned_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2D_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7285, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpy2D_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7285, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7287 + * data["__cuMemcpy2D_v2"] = __cuMemcpy2D_v2 + * global __cuMemcpy2DUnaligned_v2 + * data["__cuMemcpy2DUnaligned_v2"] = __cuMemcpy2DUnaligned_v2 # <<<<<<<<<<<<<< + * global __cuMemcpy3D_v2 + * data["__cuMemcpy3D_v2"] = __cuMemcpy3D_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DUnaligned_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7287, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpy2DUnaligned_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7287, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7289 + * data["__cuMemcpy2DUnaligned_v2"] = __cuMemcpy2DUnaligned_v2 + * global __cuMemcpy3D_v2 + * data["__cuMemcpy3D_v2"] = __cuMemcpy3D_v2 # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeer + * data["__cuMemcpy3DPeer"] = __cuMemcpy3DPeer +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3D_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7289, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpy3D_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7289, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7291 + * data["__cuMemcpy3D_v2"] = __cuMemcpy3D_v2 + * global __cuMemcpy3DPeer + * data["__cuMemcpy3DPeer"] = __cuMemcpy3DPeer # <<<<<<<<<<<<<< + * global __cuMemcpyAsync + * data["__cuMemcpyAsync"] = __cuMemcpyAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeer)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7291, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpy3DPeer, __pyx_t_3) < 0))) __PYX_ERR(0, 7291, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7293 + * data["__cuMemcpy3DPeer"] = __cuMemcpy3DPeer + * global __cuMemcpyAsync + * data["__cuMemcpyAsync"] = __cuMemcpyAsync # <<<<<<<<<<<<<< + * global __cuMemcpyPeerAsync + * data["__cuMemcpyPeerAsync"] = __cuMemcpyPeerAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7293, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7293, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7295 + * data["__cuMemcpyAsync"] = __cuMemcpyAsync + * global __cuMemcpyPeerAsync + * data["__cuMemcpyPeerAsync"] = __cuMemcpyPeerAsync # <<<<<<<<<<<<<< + * global __cuMemcpyHtoDAsync_v2 + * data["__cuMemcpyHtoDAsync_v2"] = __cuMemcpyHtoDAsync_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeerAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7295, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyPeerAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7295, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7297 + * data["__cuMemcpyPeerAsync"] = __cuMemcpyPeerAsync + * global __cuMemcpyHtoDAsync_v2 + * data["__cuMemcpyHtoDAsync_v2"] = __cuMemcpyHtoDAsync_v2 # <<<<<<<<<<<<<< + * global __cuMemcpyDtoHAsync_v2 + * data["__cuMemcpyDtoHAsync_v2"] = __cuMemcpyDtoHAsync_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoDAsync_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7297, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyHtoDAsync_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7297, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7299 + * data["__cuMemcpyHtoDAsync_v2"] = __cuMemcpyHtoDAsync_v2 + * global __cuMemcpyDtoHAsync_v2 + * data["__cuMemcpyDtoHAsync_v2"] = __cuMemcpyDtoHAsync_v2 # <<<<<<<<<<<<<< + * global __cuMemcpyDtoDAsync_v2 + * data["__cuMemcpyDtoDAsync_v2"] = __cuMemcpyDtoDAsync_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoHAsync_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7299, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyDtoHAsync_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7299, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7301 + * data["__cuMemcpyDtoHAsync_v2"] = __cuMemcpyDtoHAsync_v2 + * global __cuMemcpyDtoDAsync_v2 + * data["__cuMemcpyDtoDAsync_v2"] = __cuMemcpyDtoDAsync_v2 # <<<<<<<<<<<<<< + * global __cuMemcpyHtoAAsync_v2 + * data["__cuMemcpyHtoAAsync_v2"] = __cuMemcpyHtoAAsync_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoDAsync_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7301, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyDtoDAsync_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7301, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7303 + * data["__cuMemcpyDtoDAsync_v2"] = __cuMemcpyDtoDAsync_v2 + * global __cuMemcpyHtoAAsync_v2 + * data["__cuMemcpyHtoAAsync_v2"] = __cuMemcpyHtoAAsync_v2 # <<<<<<<<<<<<<< + * global __cuMemcpyAtoHAsync_v2 + * data["__cuMemcpyAtoHAsync_v2"] = __cuMemcpyAtoHAsync_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoAAsync_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7303, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyHtoAAsync_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7303, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7305 + * data["__cuMemcpyHtoAAsync_v2"] = __cuMemcpyHtoAAsync_v2 + * global __cuMemcpyAtoHAsync_v2 + * data["__cuMemcpyAtoHAsync_v2"] = __cuMemcpyAtoHAsync_v2 # <<<<<<<<<<<<<< + * global __cuMemcpy2DAsync_v2 + * data["__cuMemcpy2DAsync_v2"] = __cuMemcpy2DAsync_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoHAsync_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7305, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyAtoHAsync_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7305, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7307 + * data["__cuMemcpyAtoHAsync_v2"] = __cuMemcpyAtoHAsync_v2 + * global __cuMemcpy2DAsync_v2 + * data["__cuMemcpy2DAsync_v2"] = __cuMemcpy2DAsync_v2 # <<<<<<<<<<<<<< + * global __cuMemcpy3DAsync_v2 + * data["__cuMemcpy3DAsync_v2"] = __cuMemcpy3DAsync_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DAsync_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7307, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpy2DAsync_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7307, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7309 + * data["__cuMemcpy2DAsync_v2"] = __cuMemcpy2DAsync_v2 + * global __cuMemcpy3DAsync_v2 + * data["__cuMemcpy3DAsync_v2"] = __cuMemcpy3DAsync_v2 # <<<<<<<<<<<<<< + * global __cuMemcpy3DPeerAsync + * data["__cuMemcpy3DPeerAsync"] = __cuMemcpy3DPeerAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DAsync_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7309, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpy3DAsync_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7309, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7311 + * data["__cuMemcpy3DAsync_v2"] = __cuMemcpy3DAsync_v2 + * global __cuMemcpy3DPeerAsync + * data["__cuMemcpy3DPeerAsync"] = __cuMemcpy3DPeerAsync # <<<<<<<<<<<<<< + * global __cuMemcpyBatchAsync + * data["__cuMemcpyBatchAsync"] = __cuMemcpyBatchAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeerAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7311, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpy3DPeerAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7311, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7313 + * data["__cuMemcpy3DPeerAsync"] = __cuMemcpy3DPeerAsync + * global __cuMemcpyBatchAsync + * data["__cuMemcpyBatchAsync"] = __cuMemcpyBatchAsync # <<<<<<<<<<<<<< + * global __cuMemcpy3DBatchAsync + * data["__cuMemcpy3DBatchAsync"] = __cuMemcpy3DBatchAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyBatchAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7313, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpyBatchAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7313, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7315 + * data["__cuMemcpyBatchAsync"] = __cuMemcpyBatchAsync + * global __cuMemcpy3DBatchAsync + * data["__cuMemcpy3DBatchAsync"] = __cuMemcpy3DBatchAsync # <<<<<<<<<<<<<< + * global __cuMemsetD8_v2 + * data["__cuMemsetD8_v2"] = __cuMemsetD8_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DBatchAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7315, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemcpy3DBatchAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7315, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7317 + * data["__cuMemcpy3DBatchAsync"] = __cuMemcpy3DBatchAsync + * global __cuMemsetD8_v2 + * data["__cuMemsetD8_v2"] = __cuMemsetD8_v2 # <<<<<<<<<<<<<< + * global __cuMemsetD16_v2 + * data["__cuMemsetD16_v2"] = __cuMemsetD16_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7317, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemsetD8_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7317, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7319 + * data["__cuMemsetD8_v2"] = __cuMemsetD8_v2 + * global __cuMemsetD16_v2 + * data["__cuMemsetD16_v2"] = __cuMemsetD16_v2 # <<<<<<<<<<<<<< + * global __cuMemsetD32_v2 + * data["__cuMemsetD32_v2"] = __cuMemsetD32_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7319, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemsetD16_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7319, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7321 + * data["__cuMemsetD16_v2"] = __cuMemsetD16_v2 + * global __cuMemsetD32_v2 + * data["__cuMemsetD32_v2"] = __cuMemsetD32_v2 # <<<<<<<<<<<<<< + * global __cuMemsetD2D8_v2 + * data["__cuMemsetD2D8_v2"] = __cuMemsetD2D8_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7321, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemsetD32_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7321, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7323 + * data["__cuMemsetD32_v2"] = __cuMemsetD32_v2 + * global __cuMemsetD2D8_v2 + * data["__cuMemsetD2D8_v2"] = __cuMemsetD2D8_v2 # <<<<<<<<<<<<<< + * global __cuMemsetD2D16_v2 + * data["__cuMemsetD2D16_v2"] = __cuMemsetD2D16_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7323, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemsetD2D8_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7323, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7325 + * data["__cuMemsetD2D8_v2"] = __cuMemsetD2D8_v2 + * global __cuMemsetD2D16_v2 + * data["__cuMemsetD2D16_v2"] = __cuMemsetD2D16_v2 # <<<<<<<<<<<<<< + * global __cuMemsetD2D32_v2 + * data["__cuMemsetD2D32_v2"] = __cuMemsetD2D32_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7325, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemsetD2D16_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7325, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7327 + * data["__cuMemsetD2D16_v2"] = __cuMemsetD2D16_v2 + * global __cuMemsetD2D32_v2 + * data["__cuMemsetD2D32_v2"] = __cuMemsetD2D32_v2 # <<<<<<<<<<<<<< + * global __cuMemsetD8Async + * data["__cuMemsetD8Async"] = __cuMemsetD8Async +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7327, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemsetD2D32_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7327, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7329 + * data["__cuMemsetD2D32_v2"] = __cuMemsetD2D32_v2 + * global __cuMemsetD8Async + * data["__cuMemsetD8Async"] = __cuMemsetD8Async # <<<<<<<<<<<<<< + * global __cuMemsetD16Async + * data["__cuMemsetD16Async"] = __cuMemsetD16Async +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8Async)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7329, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemsetD8Async, __pyx_t_3) < 0))) __PYX_ERR(0, 7329, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7331 + * data["__cuMemsetD8Async"] = __cuMemsetD8Async + * global __cuMemsetD16Async + * data["__cuMemsetD16Async"] = __cuMemsetD16Async # <<<<<<<<<<<<<< + * global __cuMemsetD32Async + * data["__cuMemsetD32Async"] = __cuMemsetD32Async +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16Async)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7331, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemsetD16Async, __pyx_t_3) < 0))) __PYX_ERR(0, 7331, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7333 + * data["__cuMemsetD16Async"] = __cuMemsetD16Async + * global __cuMemsetD32Async + * data["__cuMemsetD32Async"] = __cuMemsetD32Async # <<<<<<<<<<<<<< + * global __cuMemsetD2D8Async + * data["__cuMemsetD2D8Async"] = __cuMemsetD2D8Async +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32Async)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7333, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemsetD32Async, __pyx_t_3) < 0))) __PYX_ERR(0, 7333, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7335 + * data["__cuMemsetD32Async"] = __cuMemsetD32Async + * global __cuMemsetD2D8Async + * data["__cuMemsetD2D8Async"] = __cuMemsetD2D8Async # <<<<<<<<<<<<<< + * global __cuMemsetD2D16Async + * data["__cuMemsetD2D16Async"] = __cuMemsetD2D16Async +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8Async)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemsetD2D8Async, __pyx_t_3) < 0))) __PYX_ERR(0, 7335, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7337 + * data["__cuMemsetD2D8Async"] = __cuMemsetD2D8Async + * global __cuMemsetD2D16Async + * data["__cuMemsetD2D16Async"] = __cuMemsetD2D16Async # <<<<<<<<<<<<<< + * global __cuMemsetD2D32Async + * data["__cuMemsetD2D32Async"] = __cuMemsetD2D32Async +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16Async)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7337, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemsetD2D16Async, __pyx_t_3) < 0))) __PYX_ERR(0, 7337, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7339 + * data["__cuMemsetD2D16Async"] = __cuMemsetD2D16Async + * global __cuMemsetD2D32Async + * data["__cuMemsetD2D32Async"] = __cuMemsetD2D32Async # <<<<<<<<<<<<<< + * global __cuArrayCreate_v2 + * data["__cuArrayCreate_v2"] = __cuArrayCreate_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32Async)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7339, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemsetD2D32Async, __pyx_t_3) < 0))) __PYX_ERR(0, 7339, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7341 + * data["__cuMemsetD2D32Async"] = __cuMemsetD2D32Async + * global __cuArrayCreate_v2 + * data["__cuArrayCreate_v2"] = __cuArrayCreate_v2 # <<<<<<<<<<<<<< + * global __cuArrayGetDescriptor_v2 + * data["__cuArrayGetDescriptor_v2"] = __cuArrayGetDescriptor_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayCreate_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7341, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuArrayCreate_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7341, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7343 + * data["__cuArrayCreate_v2"] = __cuArrayCreate_v2 + * global __cuArrayGetDescriptor_v2 + * data["__cuArrayGetDescriptor_v2"] = __cuArrayGetDescriptor_v2 # <<<<<<<<<<<<<< + * global __cuArrayGetSparseProperties + * data["__cuArrayGetSparseProperties"] = __cuArrayGetSparseProperties +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetDescriptor_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7343, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuArrayGetDescriptor_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7343, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7345 + * data["__cuArrayGetDescriptor_v2"] = __cuArrayGetDescriptor_v2 + * global __cuArrayGetSparseProperties + * data["__cuArrayGetSparseProperties"] = __cuArrayGetSparseProperties # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetSparseProperties + * data["__cuMipmappedArrayGetSparseProperties"] = __cuMipmappedArrayGetSparseProperties +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetSparseProperties)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7345, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuArrayGetSparseProperties, __pyx_t_3) < 0))) __PYX_ERR(0, 7345, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7347 + * data["__cuArrayGetSparseProperties"] = __cuArrayGetSparseProperties + * global __cuMipmappedArrayGetSparseProperties + * data["__cuMipmappedArrayGetSparseProperties"] = __cuMipmappedArrayGetSparseProperties # <<<<<<<<<<<<<< + * global __cuArrayGetMemoryRequirements + * data["__cuArrayGetMemoryRequirements"] = __cuArrayGetMemoryRequirements +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetSparseProperties)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMipmappedArrayGetSparsePrope, __pyx_t_3) < 0))) __PYX_ERR(0, 7347, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7349 + * data["__cuMipmappedArrayGetSparseProperties"] = __cuMipmappedArrayGetSparseProperties + * global __cuArrayGetMemoryRequirements + * data["__cuArrayGetMemoryRequirements"] = __cuArrayGetMemoryRequirements # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetMemoryRequirements + * data["__cuMipmappedArrayGetMemoryRequirements"] = __cuMipmappedArrayGetMemoryRequirements +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetMemoryRequirements)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7349, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuArrayGetMemoryRequirements, __pyx_t_3) < 0))) __PYX_ERR(0, 7349, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7351 + * data["__cuArrayGetMemoryRequirements"] = __cuArrayGetMemoryRequirements + * global __cuMipmappedArrayGetMemoryRequirements + * data["__cuMipmappedArrayGetMemoryRequirements"] = __cuMipmappedArrayGetMemoryRequirements # <<<<<<<<<<<<<< + * global __cuArrayGetPlane + * data["__cuArrayGetPlane"] = __cuArrayGetPlane +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetMemoryRequirements)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7351, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMipmappedArrayGetMemoryRequi, __pyx_t_3) < 0))) __PYX_ERR(0, 7351, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7353 + * data["__cuMipmappedArrayGetMemoryRequirements"] = __cuMipmappedArrayGetMemoryRequirements + * global __cuArrayGetPlane + * data["__cuArrayGetPlane"] = __cuArrayGetPlane # <<<<<<<<<<<<<< + * global __cuArrayDestroy + * data["__cuArrayDestroy"] = __cuArrayDestroy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetPlane)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuArrayGetPlane, __pyx_t_3) < 0))) __PYX_ERR(0, 7353, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7355 + * data["__cuArrayGetPlane"] = __cuArrayGetPlane + * global __cuArrayDestroy + * data["__cuArrayDestroy"] = __cuArrayDestroy # <<<<<<<<<<<<<< + * global __cuArray3DCreate_v2 + * data["__cuArray3DCreate_v2"] = __cuArray3DCreate_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayDestroy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7355, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuArrayDestroy, __pyx_t_3) < 0))) __PYX_ERR(0, 7355, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7357 + * data["__cuArrayDestroy"] = __cuArrayDestroy + * global __cuArray3DCreate_v2 + * data["__cuArray3DCreate_v2"] = __cuArray3DCreate_v2 # <<<<<<<<<<<<<< + * global __cuArray3DGetDescriptor_v2 + * data["__cuArray3DGetDescriptor_v2"] = __cuArray3DGetDescriptor_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DCreate_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7357, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuArray3DCreate_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7357, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7359 + * data["__cuArray3DCreate_v2"] = __cuArray3DCreate_v2 + * global __cuArray3DGetDescriptor_v2 + * data["__cuArray3DGetDescriptor_v2"] = __cuArray3DGetDescriptor_v2 # <<<<<<<<<<<<<< + * global __cuMipmappedArrayCreate + * data["__cuMipmappedArrayCreate"] = __cuMipmappedArrayCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DGetDescriptor_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7359, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuArray3DGetDescriptor_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7359, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7361 + * data["__cuArray3DGetDescriptor_v2"] = __cuArray3DGetDescriptor_v2 + * global __cuMipmappedArrayCreate + * data["__cuMipmappedArrayCreate"] = __cuMipmappedArrayCreate # <<<<<<<<<<<<<< + * global __cuMipmappedArrayGetLevel + * data["__cuMipmappedArrayGetLevel"] = __cuMipmappedArrayGetLevel +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7361, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMipmappedArrayCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7361, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7363 + * data["__cuMipmappedArrayCreate"] = __cuMipmappedArrayCreate + * global __cuMipmappedArrayGetLevel + * data["__cuMipmappedArrayGetLevel"] = __cuMipmappedArrayGetLevel # <<<<<<<<<<<<<< + * global __cuMipmappedArrayDestroy + * data["__cuMipmappedArrayDestroy"] = __cuMipmappedArrayDestroy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetLevel)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7363, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMipmappedArrayGetLevel, __pyx_t_3) < 0))) __PYX_ERR(0, 7363, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7365 + * data["__cuMipmappedArrayGetLevel"] = __cuMipmappedArrayGetLevel + * global __cuMipmappedArrayDestroy + * data["__cuMipmappedArrayDestroy"] = __cuMipmappedArrayDestroy # <<<<<<<<<<<<<< + * global __cuMemGetHandleForAddressRange + * data["__cuMemGetHandleForAddressRange"] = __cuMemGetHandleForAddressRange +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayDestroy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7365, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMipmappedArrayDestroy, __pyx_t_3) < 0))) __PYX_ERR(0, 7365, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7367 + * data["__cuMipmappedArrayDestroy"] = __cuMipmappedArrayDestroy + * global __cuMemGetHandleForAddressRange + * data["__cuMemGetHandleForAddressRange"] = __cuMemGetHandleForAddressRange # <<<<<<<<<<<<<< + * global __cuMemBatchDecompressAsync + * data["__cuMemBatchDecompressAsync"] = __cuMemBatchDecompressAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetHandleForAddressRange)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7367, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemGetHandleForAddressRange, __pyx_t_3) < 0))) __PYX_ERR(0, 7367, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7369 + * data["__cuMemGetHandleForAddressRange"] = __cuMemGetHandleForAddressRange + * global __cuMemBatchDecompressAsync + * data["__cuMemBatchDecompressAsync"] = __cuMemBatchDecompressAsync # <<<<<<<<<<<<<< + * global __cuMemAddressReserve + * data["__cuMemAddressReserve"] = __cuMemAddressReserve +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemBatchDecompressAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7369, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemBatchDecompressAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7369, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7371 + * data["__cuMemBatchDecompressAsync"] = __cuMemBatchDecompressAsync + * global __cuMemAddressReserve + * data["__cuMemAddressReserve"] = __cuMemAddressReserve # <<<<<<<<<<<<<< + * global __cuMemAddressFree + * data["__cuMemAddressFree"] = __cuMemAddressFree +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressReserve)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7371, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemAddressReserve, __pyx_t_3) < 0))) __PYX_ERR(0, 7371, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7373 + * data["__cuMemAddressReserve"] = __cuMemAddressReserve + * global __cuMemAddressFree + * data["__cuMemAddressFree"] = __cuMemAddressFree # <<<<<<<<<<<<<< + * global __cuMemCreate + * data["__cuMemCreate"] = __cuMemCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressFree)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7373, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemAddressFree, __pyx_t_3) < 0))) __PYX_ERR(0, 7373, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7375 + * data["__cuMemAddressFree"] = __cuMemAddressFree + * global __cuMemCreate + * data["__cuMemCreate"] = __cuMemCreate # <<<<<<<<<<<<<< + * global __cuMemRelease + * data["__cuMemRelease"] = __cuMemRelease +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7375, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7375, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7377 + * data["__cuMemCreate"] = __cuMemCreate + * global __cuMemRelease + * data["__cuMemRelease"] = __cuMemRelease # <<<<<<<<<<<<<< + * global __cuMemMap + * data["__cuMemMap"] = __cuMemMap +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRelease)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7377, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemRelease, __pyx_t_3) < 0))) __PYX_ERR(0, 7377, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7379 + * data["__cuMemRelease"] = __cuMemRelease + * global __cuMemMap + * data["__cuMemMap"] = __cuMemMap # <<<<<<<<<<<<<< + * global __cuMemMapArrayAsync + * data["__cuMemMapArrayAsync"] = __cuMemMapArrayAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMap)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7379, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemMap, __pyx_t_3) < 0))) __PYX_ERR(0, 7379, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7381 + * data["__cuMemMap"] = __cuMemMap + * global __cuMemMapArrayAsync + * data["__cuMemMapArrayAsync"] = __cuMemMapArrayAsync # <<<<<<<<<<<<<< + * global __cuMemUnmap + * data["__cuMemUnmap"] = __cuMemUnmap +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMapArrayAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7381, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemMapArrayAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7381, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7383 + * data["__cuMemMapArrayAsync"] = __cuMemMapArrayAsync + * global __cuMemUnmap + * data["__cuMemUnmap"] = __cuMemUnmap # <<<<<<<<<<<<<< + * global __cuMemSetAccess + * data["__cuMemSetAccess"] = __cuMemSetAccess +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemUnmap)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7383, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemUnmap, __pyx_t_3) < 0))) __PYX_ERR(0, 7383, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7385 + * data["__cuMemUnmap"] = __cuMemUnmap + * global __cuMemSetAccess + * data["__cuMemSetAccess"] = __cuMemSetAccess # <<<<<<<<<<<<<< + * global __cuMemGetAccess + * data["__cuMemGetAccess"] = __cuMemGetAccess +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemSetAccess)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7385, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemSetAccess, __pyx_t_3) < 0))) __PYX_ERR(0, 7385, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7387 + * data["__cuMemSetAccess"] = __cuMemSetAccess + * global __cuMemGetAccess + * data["__cuMemGetAccess"] = __cuMemGetAccess # <<<<<<<<<<<<<< + * global __cuMemExportToShareableHandle + * data["__cuMemExportToShareableHandle"] = __cuMemExportToShareableHandle +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAccess)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7387, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemGetAccess, __pyx_t_3) < 0))) __PYX_ERR(0, 7387, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7389 + * data["__cuMemGetAccess"] = __cuMemGetAccess + * global __cuMemExportToShareableHandle + * data["__cuMemExportToShareableHandle"] = __cuMemExportToShareableHandle # <<<<<<<<<<<<<< + * global __cuMemImportFromShareableHandle + * data["__cuMemImportFromShareableHandle"] = __cuMemImportFromShareableHandle +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemExportToShareableHandle)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7389, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemExportToShareableHandle, __pyx_t_3) < 0))) __PYX_ERR(0, 7389, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7391 + * data["__cuMemExportToShareableHandle"] = __cuMemExportToShareableHandle + * global __cuMemImportFromShareableHandle + * data["__cuMemImportFromShareableHandle"] = __cuMemImportFromShareableHandle # <<<<<<<<<<<<<< + * global __cuMemGetAllocationGranularity + * data["__cuMemGetAllocationGranularity"] = __cuMemGetAllocationGranularity +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemImportFromShareableHandle)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7391, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemImportFromShareableHandle, __pyx_t_3) < 0))) __PYX_ERR(0, 7391, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7393 + * data["__cuMemImportFromShareableHandle"] = __cuMemImportFromShareableHandle + * global __cuMemGetAllocationGranularity + * data["__cuMemGetAllocationGranularity"] = __cuMemGetAllocationGranularity # <<<<<<<<<<<<<< + * global __cuMemGetAllocationPropertiesFromHandle + * data["__cuMemGetAllocationPropertiesFromHandle"] = __cuMemGetAllocationPropertiesFromHandle +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationGranularity)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7393, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemGetAllocationGranularity, __pyx_t_3) < 0))) __PYX_ERR(0, 7393, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7395 + * data["__cuMemGetAllocationGranularity"] = __cuMemGetAllocationGranularity + * global __cuMemGetAllocationPropertiesFromHandle + * data["__cuMemGetAllocationPropertiesFromHandle"] = __cuMemGetAllocationPropertiesFromHandle # <<<<<<<<<<<<<< + * global __cuMemRetainAllocationHandle + * data["__cuMemRetainAllocationHandle"] = __cuMemRetainAllocationHandle +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationPropertiesFromHandle)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7395, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemGetAllocationPropertiesFr, __pyx_t_3) < 0))) __PYX_ERR(0, 7395, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7397 + * data["__cuMemGetAllocationPropertiesFromHandle"] = __cuMemGetAllocationPropertiesFromHandle + * global __cuMemRetainAllocationHandle + * data["__cuMemRetainAllocationHandle"] = __cuMemRetainAllocationHandle # <<<<<<<<<<<<<< + * global __cuMemFreeAsync + * data["__cuMemFreeAsync"] = __cuMemFreeAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRetainAllocationHandle)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7397, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemRetainAllocationHandle, __pyx_t_3) < 0))) __PYX_ERR(0, 7397, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7399 + * data["__cuMemRetainAllocationHandle"] = __cuMemRetainAllocationHandle + * global __cuMemFreeAsync + * data["__cuMemFreeAsync"] = __cuMemFreeAsync # <<<<<<<<<<<<<< + * global __cuMemAllocAsync + * data["__cuMemAllocAsync"] = __cuMemAllocAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7399, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemFreeAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7399, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7401 + * data["__cuMemFreeAsync"] = __cuMemFreeAsync + * global __cuMemAllocAsync + * data["__cuMemAllocAsync"] = __cuMemAllocAsync # <<<<<<<<<<<<<< + * global __cuMemPoolTrimTo + * data["__cuMemPoolTrimTo"] = __cuMemPoolTrimTo +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7401, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemAllocAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7401, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7403 + * data["__cuMemAllocAsync"] = __cuMemAllocAsync + * global __cuMemPoolTrimTo + * data["__cuMemPoolTrimTo"] = __cuMemPoolTrimTo # <<<<<<<<<<<<<< + * global __cuMemPoolSetAttribute + * data["__cuMemPoolSetAttribute"] = __cuMemPoolSetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolTrimTo)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7403, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemPoolTrimTo, __pyx_t_3) < 0))) __PYX_ERR(0, 7403, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7405 + * data["__cuMemPoolTrimTo"] = __cuMemPoolTrimTo + * global __cuMemPoolSetAttribute + * data["__cuMemPoolSetAttribute"] = __cuMemPoolSetAttribute # <<<<<<<<<<<<<< + * global __cuMemPoolGetAttribute + * data["__cuMemPoolGetAttribute"] = __cuMemPoolGetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7405, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemPoolSetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7405, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7407 + * data["__cuMemPoolSetAttribute"] = __cuMemPoolSetAttribute + * global __cuMemPoolGetAttribute + * data["__cuMemPoolGetAttribute"] = __cuMemPoolGetAttribute # <<<<<<<<<<<<<< + * global __cuMemPoolSetAccess + * data["__cuMemPoolSetAccess"] = __cuMemPoolSetAccess +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7407, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemPoolGetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7407, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7409 + * data["__cuMemPoolGetAttribute"] = __cuMemPoolGetAttribute + * global __cuMemPoolSetAccess + * data["__cuMemPoolSetAccess"] = __cuMemPoolSetAccess # <<<<<<<<<<<<<< + * global __cuMemPoolGetAccess + * data["__cuMemPoolGetAccess"] = __cuMemPoolGetAccess +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAccess)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7409, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemPoolSetAccess, __pyx_t_3) < 0))) __PYX_ERR(0, 7409, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7411 + * data["__cuMemPoolSetAccess"] = __cuMemPoolSetAccess + * global __cuMemPoolGetAccess + * data["__cuMemPoolGetAccess"] = __cuMemPoolGetAccess # <<<<<<<<<<<<<< + * global __cuMemPoolCreate + * data["__cuMemPoolCreate"] = __cuMemPoolCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAccess)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7411, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemPoolGetAccess, __pyx_t_3) < 0))) __PYX_ERR(0, 7411, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7413 + * data["__cuMemPoolGetAccess"] = __cuMemPoolGetAccess + * global __cuMemPoolCreate + * data["__cuMemPoolCreate"] = __cuMemPoolCreate # <<<<<<<<<<<<<< + * global __cuMemPoolDestroy + * data["__cuMemPoolDestroy"] = __cuMemPoolDestroy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7413, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemPoolCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7413, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7415 + * data["__cuMemPoolCreate"] = __cuMemPoolCreate + * global __cuMemPoolDestroy + * data["__cuMemPoolDestroy"] = __cuMemPoolDestroy # <<<<<<<<<<<<<< + * global __cuMemAllocFromPoolAsync + * data["__cuMemAllocFromPoolAsync"] = __cuMemAllocFromPoolAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolDestroy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7415, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemPoolDestroy, __pyx_t_3) < 0))) __PYX_ERR(0, 7415, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7417 + * data["__cuMemPoolDestroy"] = __cuMemPoolDestroy + * global __cuMemAllocFromPoolAsync + * data["__cuMemAllocFromPoolAsync"] = __cuMemAllocFromPoolAsync # <<<<<<<<<<<<<< + * global __cuMemPoolExportToShareableHandle + * data["__cuMemPoolExportToShareableHandle"] = __cuMemPoolExportToShareableHandle +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocFromPoolAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7417, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemAllocFromPoolAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7417, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7419 + * data["__cuMemAllocFromPoolAsync"] = __cuMemAllocFromPoolAsync + * global __cuMemPoolExportToShareableHandle + * data["__cuMemPoolExportToShareableHandle"] = __cuMemPoolExportToShareableHandle # <<<<<<<<<<<<<< + * global __cuMemPoolImportFromShareableHandle + * data["__cuMemPoolImportFromShareableHandle"] = __cuMemPoolImportFromShareableHandle +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportToShareableHandle)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7419, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemPoolExportToShareableHand, __pyx_t_3) < 0))) __PYX_ERR(0, 7419, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7421 + * data["__cuMemPoolExportToShareableHandle"] = __cuMemPoolExportToShareableHandle + * global __cuMemPoolImportFromShareableHandle + * data["__cuMemPoolImportFromShareableHandle"] = __cuMemPoolImportFromShareableHandle # <<<<<<<<<<<<<< + * global __cuMemPoolExportPointer + * data["__cuMemPoolExportPointer"] = __cuMemPoolExportPointer +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportFromShareableHandle)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7421, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemPoolImportFromShareableHa, __pyx_t_3) < 0))) __PYX_ERR(0, 7421, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7423 + * data["__cuMemPoolImportFromShareableHandle"] = __cuMemPoolImportFromShareableHandle + * global __cuMemPoolExportPointer + * data["__cuMemPoolExportPointer"] = __cuMemPoolExportPointer # <<<<<<<<<<<<<< + * global __cuMemPoolImportPointer + * data["__cuMemPoolImportPointer"] = __cuMemPoolImportPointer +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportPointer)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7423, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemPoolExportPointer, __pyx_t_3) < 0))) __PYX_ERR(0, 7423, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7425 + * data["__cuMemPoolExportPointer"] = __cuMemPoolExportPointer + * global __cuMemPoolImportPointer + * data["__cuMemPoolImportPointer"] = __cuMemPoolImportPointer # <<<<<<<<<<<<<< + * global __cuMulticastCreate + * data["__cuMulticastCreate"] = __cuMulticastCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportPointer)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemPoolImportPointer, __pyx_t_3) < 0))) __PYX_ERR(0, 7425, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7427 + * data["__cuMemPoolImportPointer"] = __cuMemPoolImportPointer + * global __cuMulticastCreate + * data["__cuMulticastCreate"] = __cuMulticastCreate # <<<<<<<<<<<<<< + * global __cuMulticastAddDevice + * data["__cuMulticastAddDevice"] = __cuMulticastAddDevice +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7427, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMulticastCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7427, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7429 + * data["__cuMulticastCreate"] = __cuMulticastCreate + * global __cuMulticastAddDevice + * data["__cuMulticastAddDevice"] = __cuMulticastAddDevice # <<<<<<<<<<<<<< + * global __cuMulticastBindMem + * data["__cuMulticastBindMem"] = __cuMulticastBindMem +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastAddDevice)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7429, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMulticastAddDevice, __pyx_t_3) < 0))) __PYX_ERR(0, 7429, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7431 + * data["__cuMulticastAddDevice"] = __cuMulticastAddDevice + * global __cuMulticastBindMem + * data["__cuMulticastBindMem"] = __cuMulticastBindMem # <<<<<<<<<<<<<< + * global __cuMulticastBindAddr + * data["__cuMulticastBindAddr"] = __cuMulticastBindAddr +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindMem)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7431, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMulticastBindMem, __pyx_t_3) < 0))) __PYX_ERR(0, 7431, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7433 + * data["__cuMulticastBindMem"] = __cuMulticastBindMem + * global __cuMulticastBindAddr + * data["__cuMulticastBindAddr"] = __cuMulticastBindAddr # <<<<<<<<<<<<<< + * global __cuMulticastUnbind + * data["__cuMulticastUnbind"] = __cuMulticastUnbind +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindAddr)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7433, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMulticastBindAddr, __pyx_t_3) < 0))) __PYX_ERR(0, 7433, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7435 + * data["__cuMulticastBindAddr"] = __cuMulticastBindAddr + * global __cuMulticastUnbind + * data["__cuMulticastUnbind"] = __cuMulticastUnbind # <<<<<<<<<<<<<< + * global __cuMulticastGetGranularity + * data["__cuMulticastGetGranularity"] = __cuMulticastGetGranularity +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastUnbind)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7435, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMulticastUnbind, __pyx_t_3) < 0))) __PYX_ERR(0, 7435, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7437 + * data["__cuMulticastUnbind"] = __cuMulticastUnbind + * global __cuMulticastGetGranularity + * data["__cuMulticastGetGranularity"] = __cuMulticastGetGranularity # <<<<<<<<<<<<<< + * global __cuPointerGetAttribute + * data["__cuPointerGetAttribute"] = __cuPointerGetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastGetGranularity)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7437, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMulticastGetGranularity, __pyx_t_3) < 0))) __PYX_ERR(0, 7437, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7439 + * data["__cuMulticastGetGranularity"] = __cuMulticastGetGranularity + * global __cuPointerGetAttribute + * data["__cuPointerGetAttribute"] = __cuPointerGetAttribute # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync + * data["__cuMemPrefetchAsync"] = __cuMemPrefetchAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7439, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuPointerGetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7439, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7441 + * data["__cuPointerGetAttribute"] = __cuPointerGetAttribute + * global __cuMemPrefetchAsync + * data["__cuMemPrefetchAsync"] = __cuMemPrefetchAsync # <<<<<<<<<<<<<< + * global __cuMemPrefetchAsync_v2 + * data["__cuMemPrefetchAsync_v2"] = __cuMemPrefetchAsync_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7441, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemPrefetchAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7441, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7443 + * data["__cuMemPrefetchAsync"] = __cuMemPrefetchAsync + * global __cuMemPrefetchAsync_v2 + * data["__cuMemPrefetchAsync_v2"] = __cuMemPrefetchAsync_v2 # <<<<<<<<<<<<<< + * global __cuMemAdvise + * data["__cuMemAdvise"] = __cuMemAdvise +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7443, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemPrefetchAsync_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7443, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7445 + * data["__cuMemPrefetchAsync_v2"] = __cuMemPrefetchAsync_v2 + * global __cuMemAdvise + * data["__cuMemAdvise"] = __cuMemAdvise # <<<<<<<<<<<<<< + * global __cuMemAdvise_v2 + * data["__cuMemAdvise_v2"] = __cuMemAdvise_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7445, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemAdvise, __pyx_t_3) < 0))) __PYX_ERR(0, 7445, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7447 + * data["__cuMemAdvise"] = __cuMemAdvise + * global __cuMemAdvise_v2 + * data["__cuMemAdvise_v2"] = __cuMemAdvise_v2 # <<<<<<<<<<<<<< + * global __cuMemRangeGetAttribute + * data["__cuMemRangeGetAttribute"] = __cuMemRangeGetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7447, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemAdvise_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7447, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7449 + * data["__cuMemAdvise_v2"] = __cuMemAdvise_v2 + * global __cuMemRangeGetAttribute + * data["__cuMemRangeGetAttribute"] = __cuMemRangeGetAttribute # <<<<<<<<<<<<<< + * global __cuMemRangeGetAttributes + * data["__cuMemRangeGetAttributes"] = __cuMemRangeGetAttributes +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7449, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemRangeGetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7449, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7451 + * data["__cuMemRangeGetAttribute"] = __cuMemRangeGetAttribute + * global __cuMemRangeGetAttributes + * data["__cuMemRangeGetAttributes"] = __cuMemRangeGetAttributes # <<<<<<<<<<<<<< + * global __cuPointerSetAttribute + * data["__cuPointerSetAttribute"] = __cuPointerSetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttributes)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7451, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuMemRangeGetAttributes, __pyx_t_3) < 0))) __PYX_ERR(0, 7451, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7453 + * data["__cuMemRangeGetAttributes"] = __cuMemRangeGetAttributes + * global __cuPointerSetAttribute + * data["__cuPointerSetAttribute"] = __cuPointerSetAttribute # <<<<<<<<<<<<<< + * global __cuPointerGetAttributes + * data["__cuPointerGetAttributes"] = __cuPointerGetAttributes +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerSetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7453, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuPointerSetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7453, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7455 + * data["__cuPointerSetAttribute"] = __cuPointerSetAttribute + * global __cuPointerGetAttributes + * data["__cuPointerGetAttributes"] = __cuPointerGetAttributes # <<<<<<<<<<<<<< + * global __cuStreamCreate + * data["__cuStreamCreate"] = __cuStreamCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttributes)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7455, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuPointerGetAttributes, __pyx_t_3) < 0))) __PYX_ERR(0, 7455, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7457 + * data["__cuPointerGetAttributes"] = __cuPointerGetAttributes + * global __cuStreamCreate + * data["__cuStreamCreate"] = __cuStreamCreate # <<<<<<<<<<<<<< + * global __cuStreamCreateWithPriority + * data["__cuStreamCreateWithPriority"] = __cuStreamCreateWithPriority +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7457, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7457, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7459 + * data["__cuStreamCreate"] = __cuStreamCreate + * global __cuStreamCreateWithPriority + * data["__cuStreamCreateWithPriority"] = __cuStreamCreateWithPriority # <<<<<<<<<<<<<< + * global __cuStreamGetPriority + * data["__cuStreamGetPriority"] = __cuStreamGetPriority +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreateWithPriority)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7459, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamCreateWithPriority, __pyx_t_3) < 0))) __PYX_ERR(0, 7459, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7461 + * data["__cuStreamCreateWithPriority"] = __cuStreamCreateWithPriority + * global __cuStreamGetPriority + * data["__cuStreamGetPriority"] = __cuStreamGetPriority # <<<<<<<<<<<<<< + * global __cuStreamGetDevice + * data["__cuStreamGetDevice"] = __cuStreamGetDevice +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetPriority)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7461, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamGetPriority, __pyx_t_3) < 0))) __PYX_ERR(0, 7461, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7463 + * data["__cuStreamGetPriority"] = __cuStreamGetPriority + * global __cuStreamGetDevice + * data["__cuStreamGetDevice"] = __cuStreamGetDevice # <<<<<<<<<<<<<< + * global __cuStreamGetFlags + * data["__cuStreamGetFlags"] = __cuStreamGetFlags +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetDevice)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7463, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamGetDevice, __pyx_t_3) < 0))) __PYX_ERR(0, 7463, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7465 + * data["__cuStreamGetDevice"] = __cuStreamGetDevice + * global __cuStreamGetFlags + * data["__cuStreamGetFlags"] = __cuStreamGetFlags # <<<<<<<<<<<<<< + * global __cuStreamGetId + * data["__cuStreamGetId"] = __cuStreamGetId +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetFlags)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7465, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamGetFlags, __pyx_t_3) < 0))) __PYX_ERR(0, 7465, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7467 + * data["__cuStreamGetFlags"] = __cuStreamGetFlags + * global __cuStreamGetId + * data["__cuStreamGetId"] = __cuStreamGetId # <<<<<<<<<<<<<< + * global __cuStreamGetCtx + * data["__cuStreamGetCtx"] = __cuStreamGetCtx +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetId)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7467, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamGetId, __pyx_t_3) < 0))) __PYX_ERR(0, 7467, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7469 + * data["__cuStreamGetId"] = __cuStreamGetId + * global __cuStreamGetCtx + * data["__cuStreamGetCtx"] = __cuStreamGetCtx # <<<<<<<<<<<<<< + * global __cuStreamGetCtx_v2 + * data["__cuStreamGetCtx_v2"] = __cuStreamGetCtx_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7469, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamGetCtx, __pyx_t_3) < 0))) __PYX_ERR(0, 7469, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7471 + * data["__cuStreamGetCtx"] = __cuStreamGetCtx + * global __cuStreamGetCtx_v2 + * data["__cuStreamGetCtx_v2"] = __cuStreamGetCtx_v2 # <<<<<<<<<<<<<< + * global __cuStreamWaitEvent + * data["__cuStreamWaitEvent"] = __cuStreamWaitEvent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7471, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamGetCtx_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7471, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7473 + * data["__cuStreamGetCtx_v2"] = __cuStreamGetCtx_v2 + * global __cuStreamWaitEvent + * data["__cuStreamWaitEvent"] = __cuStreamWaitEvent # <<<<<<<<<<<<<< + * global __cuStreamAddCallback + * data["__cuStreamAddCallback"] = __cuStreamAddCallback +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitEvent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7473, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamWaitEvent, __pyx_t_3) < 0))) __PYX_ERR(0, 7473, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7475 + * data["__cuStreamWaitEvent"] = __cuStreamWaitEvent + * global __cuStreamAddCallback + * data["__cuStreamAddCallback"] = __cuStreamAddCallback # <<<<<<<<<<<<<< + * global __cuStreamBeginCapture_v2 + * data["__cuStreamBeginCapture_v2"] = __cuStreamBeginCapture_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAddCallback)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7475, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamAddCallback, __pyx_t_3) < 0))) __PYX_ERR(0, 7475, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7477 + * data["__cuStreamAddCallback"] = __cuStreamAddCallback + * global __cuStreamBeginCapture_v2 + * data["__cuStreamBeginCapture_v2"] = __cuStreamBeginCapture_v2 # <<<<<<<<<<<<<< + * global __cuStreamBeginCaptureToGraph + * data["__cuStreamBeginCaptureToGraph"] = __cuStreamBeginCaptureToGraph +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCapture_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7477, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamBeginCapture_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7477, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7479 + * data["__cuStreamBeginCapture_v2"] = __cuStreamBeginCapture_v2 + * global __cuStreamBeginCaptureToGraph + * data["__cuStreamBeginCaptureToGraph"] = __cuStreamBeginCaptureToGraph # <<<<<<<<<<<<<< + * global __cuThreadExchangeStreamCaptureMode + * data["__cuThreadExchangeStreamCaptureMode"] = __cuThreadExchangeStreamCaptureMode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCaptureToGraph)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7479, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamBeginCaptureToGraph, __pyx_t_3) < 0))) __PYX_ERR(0, 7479, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7481 + * data["__cuStreamBeginCaptureToGraph"] = __cuStreamBeginCaptureToGraph + * global __cuThreadExchangeStreamCaptureMode + * data["__cuThreadExchangeStreamCaptureMode"] = __cuThreadExchangeStreamCaptureMode # <<<<<<<<<<<<<< + * global __cuStreamEndCapture + * data["__cuStreamEndCapture"] = __cuStreamEndCapture +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuThreadExchangeStreamCaptureMode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7481, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuThreadExchangeStreamCaptureM, __pyx_t_3) < 0))) __PYX_ERR(0, 7481, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7483 + * data["__cuThreadExchangeStreamCaptureMode"] = __cuThreadExchangeStreamCaptureMode + * global __cuStreamEndCapture + * data["__cuStreamEndCapture"] = __cuStreamEndCapture # <<<<<<<<<<<<<< + * global __cuStreamIsCapturing + * data["__cuStreamIsCapturing"] = __cuStreamIsCapturing +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamEndCapture)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7483, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamEndCapture, __pyx_t_3) < 0))) __PYX_ERR(0, 7483, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7485 + * data["__cuStreamEndCapture"] = __cuStreamEndCapture + * global __cuStreamIsCapturing + * data["__cuStreamIsCapturing"] = __cuStreamIsCapturing # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v2 + * data["__cuStreamGetCaptureInfo_v2"] = __cuStreamGetCaptureInfo_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamIsCapturing)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7485, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamIsCapturing, __pyx_t_3) < 0))) __PYX_ERR(0, 7485, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7487 + * data["__cuStreamIsCapturing"] = __cuStreamIsCapturing + * global __cuStreamGetCaptureInfo_v2 + * data["__cuStreamGetCaptureInfo_v2"] = __cuStreamGetCaptureInfo_v2 # <<<<<<<<<<<<<< + * global __cuStreamGetCaptureInfo_v3 + * data["__cuStreamGetCaptureInfo_v3"] = __cuStreamGetCaptureInfo_v3 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7487, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamGetCaptureInfo_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7487, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7489 + * data["__cuStreamGetCaptureInfo_v2"] = __cuStreamGetCaptureInfo_v2 + * global __cuStreamGetCaptureInfo_v3 + * data["__cuStreamGetCaptureInfo_v3"] = __cuStreamGetCaptureInfo_v3 # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies + * data["__cuStreamUpdateCaptureDependencies"] = __cuStreamUpdateCaptureDependencies +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v3)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7489, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamGetCaptureInfo_v3, __pyx_t_3) < 0))) __PYX_ERR(0, 7489, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7491 + * data["__cuStreamGetCaptureInfo_v3"] = __cuStreamGetCaptureInfo_v3 + * global __cuStreamUpdateCaptureDependencies + * data["__cuStreamUpdateCaptureDependencies"] = __cuStreamUpdateCaptureDependencies # <<<<<<<<<<<<<< + * global __cuStreamUpdateCaptureDependencies_v2 + * data["__cuStreamUpdateCaptureDependencies_v2"] = __cuStreamUpdateCaptureDependencies_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7491, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamUpdateCaptureDependenc, __pyx_t_3) < 0))) __PYX_ERR(0, 7491, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7493 + * data["__cuStreamUpdateCaptureDependencies"] = __cuStreamUpdateCaptureDependencies + * global __cuStreamUpdateCaptureDependencies_v2 + * data["__cuStreamUpdateCaptureDependencies_v2"] = __cuStreamUpdateCaptureDependencies_v2 # <<<<<<<<<<<<<< + * global __cuStreamAttachMemAsync + * data["__cuStreamAttachMemAsync"] = __cuStreamAttachMemAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7493, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamUpdateCaptureDependenc_2, __pyx_t_3) < 0))) __PYX_ERR(0, 7493, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7495 + * data["__cuStreamUpdateCaptureDependencies_v2"] = __cuStreamUpdateCaptureDependencies_v2 + * global __cuStreamAttachMemAsync + * data["__cuStreamAttachMemAsync"] = __cuStreamAttachMemAsync # <<<<<<<<<<<<<< + * global __cuStreamQuery + * data["__cuStreamQuery"] = __cuStreamQuery +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAttachMemAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7495, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamAttachMemAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7495, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7497 + * data["__cuStreamAttachMemAsync"] = __cuStreamAttachMemAsync + * global __cuStreamQuery + * data["__cuStreamQuery"] = __cuStreamQuery # <<<<<<<<<<<<<< + * global __cuStreamSynchronize + * data["__cuStreamSynchronize"] = __cuStreamSynchronize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamQuery)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7497, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamQuery, __pyx_t_3) < 0))) __PYX_ERR(0, 7497, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7499 + * data["__cuStreamQuery"] = __cuStreamQuery + * global __cuStreamSynchronize + * data["__cuStreamSynchronize"] = __cuStreamSynchronize # <<<<<<<<<<<<<< + * global __cuStreamDestroy_v2 + * data["__cuStreamDestroy_v2"] = __cuStreamDestroy_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSynchronize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7499, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamSynchronize, __pyx_t_3) < 0))) __PYX_ERR(0, 7499, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7501 + * data["__cuStreamSynchronize"] = __cuStreamSynchronize + * global __cuStreamDestroy_v2 + * data["__cuStreamDestroy_v2"] = __cuStreamDestroy_v2 # <<<<<<<<<<<<<< + * global __cuStreamCopyAttributes + * data["__cuStreamCopyAttributes"] = __cuStreamCopyAttributes +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamDestroy_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7501, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamDestroy_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7501, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7503 + * data["__cuStreamDestroy_v2"] = __cuStreamDestroy_v2 + * global __cuStreamCopyAttributes + * data["__cuStreamCopyAttributes"] = __cuStreamCopyAttributes # <<<<<<<<<<<<<< + * global __cuStreamGetAttribute + * data["__cuStreamGetAttribute"] = __cuStreamGetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCopyAttributes)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7503, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamCopyAttributes, __pyx_t_3) < 0))) __PYX_ERR(0, 7503, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7505 + * data["__cuStreamCopyAttributes"] = __cuStreamCopyAttributes + * global __cuStreamGetAttribute + * data["__cuStreamGetAttribute"] = __cuStreamGetAttribute # <<<<<<<<<<<<<< + * global __cuStreamSetAttribute + * data["__cuStreamSetAttribute"] = __cuStreamSetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7505, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamGetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7505, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7507 + * data["__cuStreamGetAttribute"] = __cuStreamGetAttribute + * global __cuStreamSetAttribute + * data["__cuStreamSetAttribute"] = __cuStreamSetAttribute # <<<<<<<<<<<<<< + * global __cuEventCreate + * data["__cuEventCreate"] = __cuEventCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7507, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamSetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7507, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7509 + * data["__cuStreamSetAttribute"] = __cuStreamSetAttribute + * global __cuEventCreate + * data["__cuEventCreate"] = __cuEventCreate # <<<<<<<<<<<<<< + * global __cuEventRecord + * data["__cuEventRecord"] = __cuEventRecord +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7509, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEventCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7509, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7511 + * data["__cuEventCreate"] = __cuEventCreate + * global __cuEventRecord + * data["__cuEventRecord"] = __cuEventRecord # <<<<<<<<<<<<<< + * global __cuEventRecordWithFlags + * data["__cuEventRecordWithFlags"] = __cuEventRecordWithFlags +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecord)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7511, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEventRecord, __pyx_t_3) < 0))) __PYX_ERR(0, 7511, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7513 + * data["__cuEventRecord"] = __cuEventRecord + * global __cuEventRecordWithFlags + * data["__cuEventRecordWithFlags"] = __cuEventRecordWithFlags # <<<<<<<<<<<<<< + * global __cuEventQuery + * data["__cuEventQuery"] = __cuEventQuery +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecordWithFlags)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7513, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEventRecordWithFlags, __pyx_t_3) < 0))) __PYX_ERR(0, 7513, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7515 + * data["__cuEventRecordWithFlags"] = __cuEventRecordWithFlags + * global __cuEventQuery + * data["__cuEventQuery"] = __cuEventQuery # <<<<<<<<<<<<<< + * global __cuEventSynchronize + * data["__cuEventSynchronize"] = __cuEventSynchronize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventQuery)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7515, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEventQuery, __pyx_t_3) < 0))) __PYX_ERR(0, 7515, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7517 + * data["__cuEventQuery"] = __cuEventQuery + * global __cuEventSynchronize + * data["__cuEventSynchronize"] = __cuEventSynchronize # <<<<<<<<<<<<<< + * global __cuEventDestroy_v2 + * data["__cuEventDestroy_v2"] = __cuEventDestroy_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventSynchronize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7517, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEventSynchronize, __pyx_t_3) < 0))) __PYX_ERR(0, 7517, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7519 + * data["__cuEventSynchronize"] = __cuEventSynchronize + * global __cuEventDestroy_v2 + * data["__cuEventDestroy_v2"] = __cuEventDestroy_v2 # <<<<<<<<<<<<<< + * global __cuEventElapsedTime + * data["__cuEventElapsedTime"] = __cuEventElapsedTime +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventDestroy_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7519, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEventDestroy_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7519, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7521 + * data["__cuEventDestroy_v2"] = __cuEventDestroy_v2 + * global __cuEventElapsedTime + * data["__cuEventElapsedTime"] = __cuEventElapsedTime # <<<<<<<<<<<<<< + * global __cuEventElapsedTime_v2 + * data["__cuEventElapsedTime_v2"] = __cuEventElapsedTime_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7521, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEventElapsedTime, __pyx_t_3) < 0))) __PYX_ERR(0, 7521, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7523 + * data["__cuEventElapsedTime"] = __cuEventElapsedTime + * global __cuEventElapsedTime_v2 + * data["__cuEventElapsedTime_v2"] = __cuEventElapsedTime_v2 # <<<<<<<<<<<<<< + * global __cuImportExternalMemory + * data["__cuImportExternalMemory"] = __cuImportExternalMemory +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7523, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEventElapsedTime_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7523, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7525 + * data["__cuEventElapsedTime_v2"] = __cuEventElapsedTime_v2 + * global __cuImportExternalMemory + * data["__cuImportExternalMemory"] = __cuImportExternalMemory # <<<<<<<<<<<<<< + * global __cuExternalMemoryGetMappedBuffer + * data["__cuExternalMemoryGetMappedBuffer"] = __cuExternalMemoryGetMappedBuffer +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalMemory)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7525, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuImportExternalMemory, __pyx_t_3) < 0))) __PYX_ERR(0, 7525, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7527 + * data["__cuImportExternalMemory"] = __cuImportExternalMemory + * global __cuExternalMemoryGetMappedBuffer + * data["__cuExternalMemoryGetMappedBuffer"] = __cuExternalMemoryGetMappedBuffer # <<<<<<<<<<<<<< + * global __cuExternalMemoryGetMappedMipmappedArray + * data["__cuExternalMemoryGetMappedMipmappedArray"] = __cuExternalMemoryGetMappedMipmappedArray +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedBuffer)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7527, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuExternalMemoryGetMappedBuffe, __pyx_t_3) < 0))) __PYX_ERR(0, 7527, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7529 + * data["__cuExternalMemoryGetMappedBuffer"] = __cuExternalMemoryGetMappedBuffer + * global __cuExternalMemoryGetMappedMipmappedArray + * data["__cuExternalMemoryGetMappedMipmappedArray"] = __cuExternalMemoryGetMappedMipmappedArray # <<<<<<<<<<<<<< + * global __cuDestroyExternalMemory + * data["__cuDestroyExternalMemory"] = __cuDestroyExternalMemory +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedMipmappedArray)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7529, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuExternalMemoryGetMappedMipma, __pyx_t_3) < 0))) __PYX_ERR(0, 7529, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7531 + * data["__cuExternalMemoryGetMappedMipmappedArray"] = __cuExternalMemoryGetMappedMipmappedArray + * global __cuDestroyExternalMemory + * data["__cuDestroyExternalMemory"] = __cuDestroyExternalMemory # <<<<<<<<<<<<<< + * global __cuImportExternalSemaphore + * data["__cuImportExternalSemaphore"] = __cuImportExternalSemaphore +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalMemory)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7531, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDestroyExternalMemory, __pyx_t_3) < 0))) __PYX_ERR(0, 7531, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7533 + * data["__cuDestroyExternalMemory"] = __cuDestroyExternalMemory + * global __cuImportExternalSemaphore + * data["__cuImportExternalSemaphore"] = __cuImportExternalSemaphore # <<<<<<<<<<<<<< + * global __cuSignalExternalSemaphoresAsync + * data["__cuSignalExternalSemaphoresAsync"] = __cuSignalExternalSemaphoresAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalSemaphore)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7533, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuImportExternalSemaphore, __pyx_t_3) < 0))) __PYX_ERR(0, 7533, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7535 + * data["__cuImportExternalSemaphore"] = __cuImportExternalSemaphore + * global __cuSignalExternalSemaphoresAsync + * data["__cuSignalExternalSemaphoresAsync"] = __cuSignalExternalSemaphoresAsync # <<<<<<<<<<<<<< + * global __cuWaitExternalSemaphoresAsync + * data["__cuWaitExternalSemaphoresAsync"] = __cuWaitExternalSemaphoresAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSignalExternalSemaphoresAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7535, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuSignalExternalSemaphoresAsyn, __pyx_t_3) < 0))) __PYX_ERR(0, 7535, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7537 + * data["__cuSignalExternalSemaphoresAsync"] = __cuSignalExternalSemaphoresAsync + * global __cuWaitExternalSemaphoresAsync + * data["__cuWaitExternalSemaphoresAsync"] = __cuWaitExternalSemaphoresAsync # <<<<<<<<<<<<<< + * global __cuDestroyExternalSemaphore + * data["__cuDestroyExternalSemaphore"] = __cuDestroyExternalSemaphore +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuWaitExternalSemaphoresAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7537, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuWaitExternalSemaphoresAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7537, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7539 + * data["__cuWaitExternalSemaphoresAsync"] = __cuWaitExternalSemaphoresAsync + * global __cuDestroyExternalSemaphore + * data["__cuDestroyExternalSemaphore"] = __cuDestroyExternalSemaphore # <<<<<<<<<<<<<< + * global __cuStreamWaitValue32_v2 + * data["__cuStreamWaitValue32_v2"] = __cuStreamWaitValue32_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalSemaphore)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7539, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDestroyExternalSemaphore, __pyx_t_3) < 0))) __PYX_ERR(0, 7539, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7541 + * data["__cuDestroyExternalSemaphore"] = __cuDestroyExternalSemaphore + * global __cuStreamWaitValue32_v2 + * data["__cuStreamWaitValue32_v2"] = __cuStreamWaitValue32_v2 # <<<<<<<<<<<<<< + * global __cuStreamWaitValue64_v2 + * data["__cuStreamWaitValue64_v2"] = __cuStreamWaitValue64_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue32_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7541, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamWaitValue32_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7541, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7543 + * data["__cuStreamWaitValue32_v2"] = __cuStreamWaitValue32_v2 + * global __cuStreamWaitValue64_v2 + * data["__cuStreamWaitValue64_v2"] = __cuStreamWaitValue64_v2 # <<<<<<<<<<<<<< + * global __cuStreamWriteValue32_v2 + * data["__cuStreamWriteValue32_v2"] = __cuStreamWriteValue32_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue64_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7543, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamWaitValue64_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7543, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7545 + * data["__cuStreamWaitValue64_v2"] = __cuStreamWaitValue64_v2 + * global __cuStreamWriteValue32_v2 + * data["__cuStreamWriteValue32_v2"] = __cuStreamWriteValue32_v2 # <<<<<<<<<<<<<< + * global __cuStreamWriteValue64_v2 + * data["__cuStreamWriteValue64_v2"] = __cuStreamWriteValue64_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue32_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7545, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamWriteValue32_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7545, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7547 + * data["__cuStreamWriteValue32_v2"] = __cuStreamWriteValue32_v2 + * global __cuStreamWriteValue64_v2 + * data["__cuStreamWriteValue64_v2"] = __cuStreamWriteValue64_v2 # <<<<<<<<<<<<<< + * global __cuStreamBatchMemOp_v2 + * data["__cuStreamBatchMemOp_v2"] = __cuStreamBatchMemOp_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue64_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7547, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamWriteValue64_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7547, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7549 + * data["__cuStreamWriteValue64_v2"] = __cuStreamWriteValue64_v2 + * global __cuStreamBatchMemOp_v2 + * data["__cuStreamBatchMemOp_v2"] = __cuStreamBatchMemOp_v2 # <<<<<<<<<<<<<< + * global __cuFuncGetAttribute + * data["__cuFuncGetAttribute"] = __cuFuncGetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBatchMemOp_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7549, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamBatchMemOp_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7549, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7551 + * data["__cuStreamBatchMemOp_v2"] = __cuStreamBatchMemOp_v2 + * global __cuFuncGetAttribute + * data["__cuFuncGetAttribute"] = __cuFuncGetAttribute # <<<<<<<<<<<<<< + * global __cuFuncSetAttribute + * data["__cuFuncSetAttribute"] = __cuFuncSetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7551, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFuncGetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7551, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7553 + * data["__cuFuncGetAttribute"] = __cuFuncGetAttribute + * global __cuFuncSetAttribute + * data["__cuFuncSetAttribute"] = __cuFuncSetAttribute # <<<<<<<<<<<<<< + * global __cuFuncSetCacheConfig + * data["__cuFuncSetCacheConfig"] = __cuFuncSetCacheConfig +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7553, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFuncSetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7553, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7555 + * data["__cuFuncSetAttribute"] = __cuFuncSetAttribute + * global __cuFuncSetCacheConfig + * data["__cuFuncSetCacheConfig"] = __cuFuncSetCacheConfig # <<<<<<<<<<<<<< + * global __cuFuncGetModule + * data["__cuFuncGetModule"] = __cuFuncGetModule +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetCacheConfig)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7555, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFuncSetCacheConfig, __pyx_t_3) < 0))) __PYX_ERR(0, 7555, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7557 + * data["__cuFuncSetCacheConfig"] = __cuFuncSetCacheConfig + * global __cuFuncGetModule + * data["__cuFuncGetModule"] = __cuFuncGetModule # <<<<<<<<<<<<<< + * global __cuFuncGetName + * data["__cuFuncGetName"] = __cuFuncGetName +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetModule)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7557, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFuncGetModule, __pyx_t_3) < 0))) __PYX_ERR(0, 7557, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7559 + * data["__cuFuncGetModule"] = __cuFuncGetModule + * global __cuFuncGetName + * data["__cuFuncGetName"] = __cuFuncGetName # <<<<<<<<<<<<<< + * global __cuFuncGetParamInfo + * data["__cuFuncGetParamInfo"] = __cuFuncGetParamInfo +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetName)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7559, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFuncGetName, __pyx_t_3) < 0))) __PYX_ERR(0, 7559, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7561 + * data["__cuFuncGetName"] = __cuFuncGetName + * global __cuFuncGetParamInfo + * data["__cuFuncGetParamInfo"] = __cuFuncGetParamInfo # <<<<<<<<<<<<<< + * global __cuFuncIsLoaded + * data["__cuFuncIsLoaded"] = __cuFuncIsLoaded +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetParamInfo)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7561, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFuncGetParamInfo, __pyx_t_3) < 0))) __PYX_ERR(0, 7561, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7563 + * data["__cuFuncGetParamInfo"] = __cuFuncGetParamInfo + * global __cuFuncIsLoaded + * data["__cuFuncIsLoaded"] = __cuFuncIsLoaded # <<<<<<<<<<<<<< + * global __cuFuncLoad + * data["__cuFuncLoad"] = __cuFuncLoad +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncIsLoaded)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7563, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFuncIsLoaded, __pyx_t_3) < 0))) __PYX_ERR(0, 7563, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7565 + * data["__cuFuncIsLoaded"] = __cuFuncIsLoaded + * global __cuFuncLoad + * data["__cuFuncLoad"] = __cuFuncLoad # <<<<<<<<<<<<<< + * global __cuLaunchKernel + * data["__cuLaunchKernel"] = __cuLaunchKernel +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncLoad)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7565, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFuncLoad, __pyx_t_3) < 0))) __PYX_ERR(0, 7565, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7567 + * data["__cuFuncLoad"] = __cuFuncLoad + * global __cuLaunchKernel + * data["__cuLaunchKernel"] = __cuLaunchKernel # <<<<<<<<<<<<<< + * global __cuLaunchKernelEx + * data["__cuLaunchKernelEx"] = __cuLaunchKernelEx +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernel)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7567, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLaunchKernel, __pyx_t_3) < 0))) __PYX_ERR(0, 7567, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7569 + * data["__cuLaunchKernel"] = __cuLaunchKernel + * global __cuLaunchKernelEx + * data["__cuLaunchKernelEx"] = __cuLaunchKernelEx # <<<<<<<<<<<<<< + * global __cuLaunchCooperativeKernel + * data["__cuLaunchCooperativeKernel"] = __cuLaunchCooperativeKernel +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernelEx)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7569, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLaunchKernelEx, __pyx_t_3) < 0))) __PYX_ERR(0, 7569, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7571 + * data["__cuLaunchKernelEx"] = __cuLaunchKernelEx + * global __cuLaunchCooperativeKernel + * data["__cuLaunchCooperativeKernel"] = __cuLaunchCooperativeKernel # <<<<<<<<<<<<<< + * global __cuLaunchCooperativeKernelMultiDevice + * data["__cuLaunchCooperativeKernelMultiDevice"] = __cuLaunchCooperativeKernelMultiDevice +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernel)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7571, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLaunchCooperativeKernel, __pyx_t_3) < 0))) __PYX_ERR(0, 7571, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7573 + * data["__cuLaunchCooperativeKernel"] = __cuLaunchCooperativeKernel + * global __cuLaunchCooperativeKernelMultiDevice + * data["__cuLaunchCooperativeKernelMultiDevice"] = __cuLaunchCooperativeKernelMultiDevice # <<<<<<<<<<<<<< + * global __cuLaunchHostFunc + * data["__cuLaunchHostFunc"] = __cuLaunchHostFunc +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernelMultiDevice)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7573, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLaunchCooperativeKernelMulti, __pyx_t_3) < 0))) __PYX_ERR(0, 7573, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7575 + * data["__cuLaunchCooperativeKernelMultiDevice"] = __cuLaunchCooperativeKernelMultiDevice + * global __cuLaunchHostFunc + * data["__cuLaunchHostFunc"] = __cuLaunchHostFunc # <<<<<<<<<<<<<< + * global __cuFuncSetBlockShape + * data["__cuFuncSetBlockShape"] = __cuFuncSetBlockShape +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchHostFunc)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7575, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLaunchHostFunc, __pyx_t_3) < 0))) __PYX_ERR(0, 7575, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7577 + * data["__cuLaunchHostFunc"] = __cuLaunchHostFunc + * global __cuFuncSetBlockShape + * data["__cuFuncSetBlockShape"] = __cuFuncSetBlockShape # <<<<<<<<<<<<<< + * global __cuFuncSetSharedSize + * data["__cuFuncSetSharedSize"] = __cuFuncSetSharedSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetBlockShape)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7577, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFuncSetBlockShape, __pyx_t_3) < 0))) __PYX_ERR(0, 7577, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7579 + * data["__cuFuncSetBlockShape"] = __cuFuncSetBlockShape + * global __cuFuncSetSharedSize + * data["__cuFuncSetSharedSize"] = __cuFuncSetSharedSize # <<<<<<<<<<<<<< + * global __cuParamSetSize + * data["__cuParamSetSize"] = __cuParamSetSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7579, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFuncSetSharedSize, __pyx_t_3) < 0))) __PYX_ERR(0, 7579, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7581 + * data["__cuFuncSetSharedSize"] = __cuFuncSetSharedSize + * global __cuParamSetSize + * data["__cuParamSetSize"] = __cuParamSetSize # <<<<<<<<<<<<<< + * global __cuParamSeti + * data["__cuParamSeti"] = __cuParamSeti +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7581, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuParamSetSize, __pyx_t_3) < 0))) __PYX_ERR(0, 7581, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7583 + * data["__cuParamSetSize"] = __cuParamSetSize + * global __cuParamSeti + * data["__cuParamSeti"] = __cuParamSeti # <<<<<<<<<<<<<< + * global __cuParamSetf + * data["__cuParamSetf"] = __cuParamSetf +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSeti)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7583, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuParamSeti, __pyx_t_3) < 0))) __PYX_ERR(0, 7583, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7585 + * data["__cuParamSeti"] = __cuParamSeti + * global __cuParamSetf + * data["__cuParamSetf"] = __cuParamSetf # <<<<<<<<<<<<<< + * global __cuParamSetv + * data["__cuParamSetv"] = __cuParamSetv +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetf)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7585, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuParamSetf, __pyx_t_3) < 0))) __PYX_ERR(0, 7585, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7587 + * data["__cuParamSetf"] = __cuParamSetf + * global __cuParamSetv + * data["__cuParamSetv"] = __cuParamSetv # <<<<<<<<<<<<<< + * global __cuLaunch + * data["__cuLaunch"] = __cuLaunch +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetv)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7587, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuParamSetv, __pyx_t_3) < 0))) __PYX_ERR(0, 7587, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7589 + * data["__cuParamSetv"] = __cuParamSetv + * global __cuLaunch + * data["__cuLaunch"] = __cuLaunch # <<<<<<<<<<<<<< + * global __cuLaunchGrid + * data["__cuLaunchGrid"] = __cuLaunchGrid +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunch)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7589, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLaunch, __pyx_t_3) < 0))) __PYX_ERR(0, 7589, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7591 + * data["__cuLaunch"] = __cuLaunch + * global __cuLaunchGrid + * data["__cuLaunchGrid"] = __cuLaunchGrid # <<<<<<<<<<<<<< + * global __cuLaunchGridAsync + * data["__cuLaunchGridAsync"] = __cuLaunchGridAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGrid)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7591, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLaunchGrid, __pyx_t_3) < 0))) __PYX_ERR(0, 7591, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7593 + * data["__cuLaunchGrid"] = __cuLaunchGrid + * global __cuLaunchGridAsync + * data["__cuLaunchGridAsync"] = __cuLaunchGridAsync # <<<<<<<<<<<<<< + * global __cuParamSetTexRef + * data["__cuParamSetTexRef"] = __cuParamSetTexRef +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGridAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7593, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLaunchGridAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 7593, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7595 + * data["__cuLaunchGridAsync"] = __cuLaunchGridAsync + * global __cuParamSetTexRef + * data["__cuParamSetTexRef"] = __cuParamSetTexRef # <<<<<<<<<<<<<< + * global __cuFuncSetSharedMemConfig + * data["__cuFuncSetSharedMemConfig"] = __cuFuncSetSharedMemConfig +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetTexRef)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7595, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuParamSetTexRef, __pyx_t_3) < 0))) __PYX_ERR(0, 7595, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7597 + * data["__cuParamSetTexRef"] = __cuParamSetTexRef + * global __cuFuncSetSharedMemConfig + * data["__cuFuncSetSharedMemConfig"] = __cuFuncSetSharedMemConfig # <<<<<<<<<<<<<< + * global __cuGraphCreate + * data["__cuGraphCreate"] = __cuGraphCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedMemConfig)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7597, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFuncSetSharedMemConfig, __pyx_t_3) < 0))) __PYX_ERR(0, 7597, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7599 + * data["__cuFuncSetSharedMemConfig"] = __cuFuncSetSharedMemConfig + * global __cuGraphCreate + * data["__cuGraphCreate"] = __cuGraphCreate # <<<<<<<<<<<<<< + * global __cuGraphAddKernelNode_v2 + * data["__cuGraphAddKernelNode_v2"] = __cuGraphAddKernelNode_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7599, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7599, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7601 + * data["__cuGraphCreate"] = __cuGraphCreate + * global __cuGraphAddKernelNode_v2 + * data["__cuGraphAddKernelNode_v2"] = __cuGraphAddKernelNode_v2 # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeGetParams_v2 + * data["__cuGraphKernelNodeGetParams_v2"] = __cuGraphKernelNodeGetParams_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddKernelNode_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7601, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddKernelNode_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7601, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7603 + * data["__cuGraphAddKernelNode_v2"] = __cuGraphAddKernelNode_v2 + * global __cuGraphKernelNodeGetParams_v2 + * data["__cuGraphKernelNodeGetParams_v2"] = __cuGraphKernelNodeGetParams_v2 # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeSetParams_v2 + * data["__cuGraphKernelNodeSetParams_v2"] = __cuGraphKernelNodeSetParams_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetParams_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7603, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphKernelNodeGetParams_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7603, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7605 + * data["__cuGraphKernelNodeGetParams_v2"] = __cuGraphKernelNodeGetParams_v2 + * global __cuGraphKernelNodeSetParams_v2 + * data["__cuGraphKernelNodeSetParams_v2"] = __cuGraphKernelNodeSetParams_v2 # <<<<<<<<<<<<<< + * global __cuGraphAddMemcpyNode + * data["__cuGraphAddMemcpyNode"] = __cuGraphAddMemcpyNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetParams_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7605, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphKernelNodeSetParams_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7605, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7607 + * data["__cuGraphKernelNodeSetParams_v2"] = __cuGraphKernelNodeSetParams_v2 + * global __cuGraphAddMemcpyNode + * data["__cuGraphAddMemcpyNode"] = __cuGraphAddMemcpyNode # <<<<<<<<<<<<<< + * global __cuGraphMemcpyNodeGetParams + * data["__cuGraphMemcpyNodeGetParams"] = __cuGraphMemcpyNodeGetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemcpyNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7607, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddMemcpyNode, __pyx_t_3) < 0))) __PYX_ERR(0, 7607, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7609 + * data["__cuGraphAddMemcpyNode"] = __cuGraphAddMemcpyNode + * global __cuGraphMemcpyNodeGetParams + * data["__cuGraphMemcpyNodeGetParams"] = __cuGraphMemcpyNodeGetParams # <<<<<<<<<<<<<< + * global __cuGraphMemcpyNodeSetParams + * data["__cuGraphMemcpyNodeSetParams"] = __cuGraphMemcpyNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeGetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7609, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphMemcpyNodeGetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7609, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7611 + * data["__cuGraphMemcpyNodeGetParams"] = __cuGraphMemcpyNodeGetParams + * global __cuGraphMemcpyNodeSetParams + * data["__cuGraphMemcpyNodeSetParams"] = __cuGraphMemcpyNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphAddMemsetNode + * data["__cuGraphAddMemsetNode"] = __cuGraphAddMemsetNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7611, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphMemcpyNodeSetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7611, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7613 + * data["__cuGraphMemcpyNodeSetParams"] = __cuGraphMemcpyNodeSetParams + * global __cuGraphAddMemsetNode + * data["__cuGraphAddMemsetNode"] = __cuGraphAddMemsetNode # <<<<<<<<<<<<<< + * global __cuGraphMemsetNodeGetParams + * data["__cuGraphMemsetNodeGetParams"] = __cuGraphMemsetNodeGetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemsetNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7613, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddMemsetNode, __pyx_t_3) < 0))) __PYX_ERR(0, 7613, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7615 + * data["__cuGraphAddMemsetNode"] = __cuGraphAddMemsetNode + * global __cuGraphMemsetNodeGetParams + * data["__cuGraphMemsetNodeGetParams"] = __cuGraphMemsetNodeGetParams # <<<<<<<<<<<<<< + * global __cuGraphMemsetNodeSetParams + * data["__cuGraphMemsetNodeSetParams"] = __cuGraphMemsetNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeGetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7615, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphMemsetNodeGetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7615, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7617 + * data["__cuGraphMemsetNodeGetParams"] = __cuGraphMemsetNodeGetParams + * global __cuGraphMemsetNodeSetParams + * data["__cuGraphMemsetNodeSetParams"] = __cuGraphMemsetNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphAddHostNode + * data["__cuGraphAddHostNode"] = __cuGraphAddHostNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7617, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphMemsetNodeSetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7617, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7619 + * data["__cuGraphMemsetNodeSetParams"] = __cuGraphMemsetNodeSetParams + * global __cuGraphAddHostNode + * data["__cuGraphAddHostNode"] = __cuGraphAddHostNode # <<<<<<<<<<<<<< + * global __cuGraphHostNodeGetParams + * data["__cuGraphHostNodeGetParams"] = __cuGraphHostNodeGetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddHostNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7619, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddHostNode, __pyx_t_3) < 0))) __PYX_ERR(0, 7619, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7621 + * data["__cuGraphAddHostNode"] = __cuGraphAddHostNode + * global __cuGraphHostNodeGetParams + * data["__cuGraphHostNodeGetParams"] = __cuGraphHostNodeGetParams # <<<<<<<<<<<<<< + * global __cuGraphHostNodeSetParams + * data["__cuGraphHostNodeSetParams"] = __cuGraphHostNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeGetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7621, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphHostNodeGetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7621, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7623 + * data["__cuGraphHostNodeGetParams"] = __cuGraphHostNodeGetParams + * global __cuGraphHostNodeSetParams + * data["__cuGraphHostNodeSetParams"] = __cuGraphHostNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphAddChildGraphNode + * data["__cuGraphAddChildGraphNode"] = __cuGraphAddChildGraphNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7623, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphHostNodeSetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7623, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7625 + * data["__cuGraphHostNodeSetParams"] = __cuGraphHostNodeSetParams + * global __cuGraphAddChildGraphNode + * data["__cuGraphAddChildGraphNode"] = __cuGraphAddChildGraphNode # <<<<<<<<<<<<<< + * global __cuGraphChildGraphNodeGetGraph + * data["__cuGraphChildGraphNodeGetGraph"] = __cuGraphChildGraphNodeGetGraph +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddChildGraphNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7625, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddChildGraphNode, __pyx_t_3) < 0))) __PYX_ERR(0, 7625, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7627 + * data["__cuGraphAddChildGraphNode"] = __cuGraphAddChildGraphNode + * global __cuGraphChildGraphNodeGetGraph + * data["__cuGraphChildGraphNodeGetGraph"] = __cuGraphChildGraphNodeGetGraph # <<<<<<<<<<<<<< + * global __cuGraphAddEmptyNode + * data["__cuGraphAddEmptyNode"] = __cuGraphAddEmptyNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphChildGraphNodeGetGraph)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7627, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphChildGraphNodeGetGraph, __pyx_t_3) < 0))) __PYX_ERR(0, 7627, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7629 + * data["__cuGraphChildGraphNodeGetGraph"] = __cuGraphChildGraphNodeGetGraph + * global __cuGraphAddEmptyNode + * data["__cuGraphAddEmptyNode"] = __cuGraphAddEmptyNode # <<<<<<<<<<<<<< + * global __cuGraphAddEventRecordNode + * data["__cuGraphAddEventRecordNode"] = __cuGraphAddEventRecordNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEmptyNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7629, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddEmptyNode, __pyx_t_3) < 0))) __PYX_ERR(0, 7629, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7631 + * data["__cuGraphAddEmptyNode"] = __cuGraphAddEmptyNode + * global __cuGraphAddEventRecordNode + * data["__cuGraphAddEventRecordNode"] = __cuGraphAddEventRecordNode # <<<<<<<<<<<<<< + * global __cuGraphEventRecordNodeGetEvent + * data["__cuGraphEventRecordNodeGetEvent"] = __cuGraphEventRecordNodeGetEvent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventRecordNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7631, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddEventRecordNode, __pyx_t_3) < 0))) __PYX_ERR(0, 7631, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7633 + * data["__cuGraphAddEventRecordNode"] = __cuGraphAddEventRecordNode + * global __cuGraphEventRecordNodeGetEvent + * data["__cuGraphEventRecordNodeGetEvent"] = __cuGraphEventRecordNodeGetEvent # <<<<<<<<<<<<<< + * global __cuGraphEventRecordNodeSetEvent + * data["__cuGraphEventRecordNodeSetEvent"] = __cuGraphEventRecordNodeSetEvent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeGetEvent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7633, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphEventRecordNodeGetEvent, __pyx_t_3) < 0))) __PYX_ERR(0, 7633, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7635 + * data["__cuGraphEventRecordNodeGetEvent"] = __cuGraphEventRecordNodeGetEvent + * global __cuGraphEventRecordNodeSetEvent + * data["__cuGraphEventRecordNodeSetEvent"] = __cuGraphEventRecordNodeSetEvent # <<<<<<<<<<<<<< + * global __cuGraphAddEventWaitNode + * data["__cuGraphAddEventWaitNode"] = __cuGraphAddEventWaitNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeSetEvent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7635, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphEventRecordNodeSetEvent, __pyx_t_3) < 0))) __PYX_ERR(0, 7635, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7637 + * data["__cuGraphEventRecordNodeSetEvent"] = __cuGraphEventRecordNodeSetEvent + * global __cuGraphAddEventWaitNode + * data["__cuGraphAddEventWaitNode"] = __cuGraphAddEventWaitNode # <<<<<<<<<<<<<< + * global __cuGraphEventWaitNodeGetEvent + * data["__cuGraphEventWaitNodeGetEvent"] = __cuGraphEventWaitNodeGetEvent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventWaitNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7637, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddEventWaitNode, __pyx_t_3) < 0))) __PYX_ERR(0, 7637, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7639 + * data["__cuGraphAddEventWaitNode"] = __cuGraphAddEventWaitNode + * global __cuGraphEventWaitNodeGetEvent + * data["__cuGraphEventWaitNodeGetEvent"] = __cuGraphEventWaitNodeGetEvent # <<<<<<<<<<<<<< + * global __cuGraphEventWaitNodeSetEvent + * data["__cuGraphEventWaitNodeSetEvent"] = __cuGraphEventWaitNodeSetEvent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeGetEvent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7639, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphEventWaitNodeGetEvent, __pyx_t_3) < 0))) __PYX_ERR(0, 7639, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7641 + * data["__cuGraphEventWaitNodeGetEvent"] = __cuGraphEventWaitNodeGetEvent + * global __cuGraphEventWaitNodeSetEvent + * data["__cuGraphEventWaitNodeSetEvent"] = __cuGraphEventWaitNodeSetEvent # <<<<<<<<<<<<<< + * global __cuGraphAddExternalSemaphoresSignalNode + * data["__cuGraphAddExternalSemaphoresSignalNode"] = __cuGraphAddExternalSemaphoresSignalNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeSetEvent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7641, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphEventWaitNodeSetEvent, __pyx_t_3) < 0))) __PYX_ERR(0, 7641, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7643 + * data["__cuGraphEventWaitNodeSetEvent"] = __cuGraphEventWaitNodeSetEvent + * global __cuGraphAddExternalSemaphoresSignalNode + * data["__cuGraphAddExternalSemaphoresSignalNode"] = __cuGraphAddExternalSemaphoresSignalNode # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresSignalNodeGetParams + * data["__cuGraphExternalSemaphoresSignalNodeGetParams"] = __cuGraphExternalSemaphoresSignalNodeGetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresSignalNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7643, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddExternalSemaphoresSi, __pyx_t_3) < 0))) __PYX_ERR(0, 7643, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7645 + * data["__cuGraphAddExternalSemaphoresSignalNode"] = __cuGraphAddExternalSemaphoresSignalNode + * global __cuGraphExternalSemaphoresSignalNodeGetParams + * data["__cuGraphExternalSemaphoresSignalNodeGetParams"] = __cuGraphExternalSemaphoresSignalNodeGetParams # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresSignalNodeSetParams + * data["__cuGraphExternalSemaphoresSignalNodeSetParams"] = __cuGraphExternalSemaphoresSignalNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeGetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7645, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExternalSemaphoresSigna, __pyx_t_3) < 0))) __PYX_ERR(0, 7645, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7647 + * data["__cuGraphExternalSemaphoresSignalNodeGetParams"] = __cuGraphExternalSemaphoresSignalNodeGetParams + * global __cuGraphExternalSemaphoresSignalNodeSetParams + * data["__cuGraphExternalSemaphoresSignalNodeSetParams"] = __cuGraphExternalSemaphoresSignalNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphAddExternalSemaphoresWaitNode + * data["__cuGraphAddExternalSemaphoresWaitNode"] = __cuGraphAddExternalSemaphoresWaitNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7647, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExternalSemaphoresSigna_2, __pyx_t_3) < 0))) __PYX_ERR(0, 7647, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7649 + * data["__cuGraphExternalSemaphoresSignalNodeSetParams"] = __cuGraphExternalSemaphoresSignalNodeSetParams + * global __cuGraphAddExternalSemaphoresWaitNode + * data["__cuGraphAddExternalSemaphoresWaitNode"] = __cuGraphAddExternalSemaphoresWaitNode # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresWaitNodeGetParams + * data["__cuGraphExternalSemaphoresWaitNodeGetParams"] = __cuGraphExternalSemaphoresWaitNodeGetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresWaitNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7649, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddExternalSemaphoresWa, __pyx_t_3) < 0))) __PYX_ERR(0, 7649, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7651 + * data["__cuGraphAddExternalSemaphoresWaitNode"] = __cuGraphAddExternalSemaphoresWaitNode + * global __cuGraphExternalSemaphoresWaitNodeGetParams + * data["__cuGraphExternalSemaphoresWaitNodeGetParams"] = __cuGraphExternalSemaphoresWaitNodeGetParams # <<<<<<<<<<<<<< + * global __cuGraphExternalSemaphoresWaitNodeSetParams + * data["__cuGraphExternalSemaphoresWaitNodeSetParams"] = __cuGraphExternalSemaphoresWaitNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeGetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7651, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExternalSemaphoresWaitN, __pyx_t_3) < 0))) __PYX_ERR(0, 7651, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7653 + * data["__cuGraphExternalSemaphoresWaitNodeGetParams"] = __cuGraphExternalSemaphoresWaitNodeGetParams + * global __cuGraphExternalSemaphoresWaitNodeSetParams + * data["__cuGraphExternalSemaphoresWaitNodeSetParams"] = __cuGraphExternalSemaphoresWaitNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphAddBatchMemOpNode + * data["__cuGraphAddBatchMemOpNode"] = __cuGraphAddBatchMemOpNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7653, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExternalSemaphoresWaitN_2, __pyx_t_3) < 0))) __PYX_ERR(0, 7653, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7655 + * data["__cuGraphExternalSemaphoresWaitNodeSetParams"] = __cuGraphExternalSemaphoresWaitNodeSetParams + * global __cuGraphAddBatchMemOpNode + * data["__cuGraphAddBatchMemOpNode"] = __cuGraphAddBatchMemOpNode # <<<<<<<<<<<<<< + * global __cuGraphBatchMemOpNodeGetParams + * data["__cuGraphBatchMemOpNodeGetParams"] = __cuGraphBatchMemOpNodeGetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddBatchMemOpNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7655, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddBatchMemOpNode, __pyx_t_3) < 0))) __PYX_ERR(0, 7655, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7657 + * data["__cuGraphAddBatchMemOpNode"] = __cuGraphAddBatchMemOpNode + * global __cuGraphBatchMemOpNodeGetParams + * data["__cuGraphBatchMemOpNodeGetParams"] = __cuGraphBatchMemOpNodeGetParams # <<<<<<<<<<<<<< + * global __cuGraphBatchMemOpNodeSetParams + * data["__cuGraphBatchMemOpNodeSetParams"] = __cuGraphBatchMemOpNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeGetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7657, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphBatchMemOpNodeGetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7657, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7659 + * data["__cuGraphBatchMemOpNodeGetParams"] = __cuGraphBatchMemOpNodeGetParams + * global __cuGraphBatchMemOpNodeSetParams + * data["__cuGraphBatchMemOpNodeSetParams"] = __cuGraphBatchMemOpNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphExecBatchMemOpNodeSetParams + * data["__cuGraphExecBatchMemOpNodeSetParams"] = __cuGraphExecBatchMemOpNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7659, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphBatchMemOpNodeSetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7659, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7661 + * data["__cuGraphBatchMemOpNodeSetParams"] = __cuGraphBatchMemOpNodeSetParams + * global __cuGraphExecBatchMemOpNodeSetParams + * data["__cuGraphExecBatchMemOpNodeSetParams"] = __cuGraphExecBatchMemOpNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphAddMemAllocNode + * data["__cuGraphAddMemAllocNode"] = __cuGraphAddMemAllocNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecBatchMemOpNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7661, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecBatchMemOpNodeSetPa, __pyx_t_3) < 0))) __PYX_ERR(0, 7661, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7663 + * data["__cuGraphExecBatchMemOpNodeSetParams"] = __cuGraphExecBatchMemOpNodeSetParams + * global __cuGraphAddMemAllocNode + * data["__cuGraphAddMemAllocNode"] = __cuGraphAddMemAllocNode # <<<<<<<<<<<<<< + * global __cuGraphMemAllocNodeGetParams + * data["__cuGraphMemAllocNodeGetParams"] = __cuGraphMemAllocNodeGetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemAllocNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7663, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddMemAllocNode, __pyx_t_3) < 0))) __PYX_ERR(0, 7663, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7665 + * data["__cuGraphAddMemAllocNode"] = __cuGraphAddMemAllocNode + * global __cuGraphMemAllocNodeGetParams + * data["__cuGraphMemAllocNodeGetParams"] = __cuGraphMemAllocNodeGetParams # <<<<<<<<<<<<<< + * global __cuGraphAddMemFreeNode + * data["__cuGraphAddMemFreeNode"] = __cuGraphAddMemFreeNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemAllocNodeGetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7665, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphMemAllocNodeGetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7665, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7667 + * data["__cuGraphMemAllocNodeGetParams"] = __cuGraphMemAllocNodeGetParams + * global __cuGraphAddMemFreeNode + * data["__cuGraphAddMemFreeNode"] = __cuGraphAddMemFreeNode # <<<<<<<<<<<<<< + * global __cuGraphMemFreeNodeGetParams + * data["__cuGraphMemFreeNodeGetParams"] = __cuGraphMemFreeNodeGetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemFreeNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7667, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddMemFreeNode, __pyx_t_3) < 0))) __PYX_ERR(0, 7667, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7669 + * data["__cuGraphAddMemFreeNode"] = __cuGraphAddMemFreeNode + * global __cuGraphMemFreeNodeGetParams + * data["__cuGraphMemFreeNodeGetParams"] = __cuGraphMemFreeNodeGetParams # <<<<<<<<<<<<<< + * global __cuDeviceGraphMemTrim + * data["__cuDeviceGraphMemTrim"] = __cuDeviceGraphMemTrim +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemFreeNodeGetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7669, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphMemFreeNodeGetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7669, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7671 + * data["__cuGraphMemFreeNodeGetParams"] = __cuGraphMemFreeNodeGetParams + * global __cuDeviceGraphMemTrim + * data["__cuDeviceGraphMemTrim"] = __cuDeviceGraphMemTrim # <<<<<<<<<<<<<< + * global __cuDeviceGetGraphMemAttribute + * data["__cuDeviceGetGraphMemAttribute"] = __cuDeviceGetGraphMemAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGraphMemTrim)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7671, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGraphMemTrim, __pyx_t_3) < 0))) __PYX_ERR(0, 7671, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7673 + * data["__cuDeviceGraphMemTrim"] = __cuDeviceGraphMemTrim + * global __cuDeviceGetGraphMemAttribute + * data["__cuDeviceGetGraphMemAttribute"] = __cuDeviceGetGraphMemAttribute # <<<<<<<<<<<<<< + * global __cuDeviceSetGraphMemAttribute + * data["__cuDeviceSetGraphMemAttribute"] = __cuDeviceSetGraphMemAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetGraphMemAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7673, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetGraphMemAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7673, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7675 + * data["__cuDeviceGetGraphMemAttribute"] = __cuDeviceGetGraphMemAttribute + * global __cuDeviceSetGraphMemAttribute + * data["__cuDeviceSetGraphMemAttribute"] = __cuDeviceSetGraphMemAttribute # <<<<<<<<<<<<<< + * global __cuGraphClone + * data["__cuGraphClone"] = __cuGraphClone +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetGraphMemAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7675, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceSetGraphMemAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7675, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7677 + * data["__cuDeviceSetGraphMemAttribute"] = __cuDeviceSetGraphMemAttribute + * global __cuGraphClone + * data["__cuGraphClone"] = __cuGraphClone # <<<<<<<<<<<<<< + * global __cuGraphNodeFindInClone + * data["__cuGraphNodeFindInClone"] = __cuGraphNodeFindInClone +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphClone)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7677, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphClone, __pyx_t_3) < 0))) __PYX_ERR(0, 7677, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7679 + * data["__cuGraphClone"] = __cuGraphClone + * global __cuGraphNodeFindInClone + * data["__cuGraphNodeFindInClone"] = __cuGraphNodeFindInClone # <<<<<<<<<<<<<< + * global __cuGraphNodeGetType + * data["__cuGraphNodeGetType"] = __cuGraphNodeGetType +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeFindInClone)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7679, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphNodeFindInClone, __pyx_t_3) < 0))) __PYX_ERR(0, 7679, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7681 + * data["__cuGraphNodeFindInClone"] = __cuGraphNodeFindInClone + * global __cuGraphNodeGetType + * data["__cuGraphNodeGetType"] = __cuGraphNodeGetType # <<<<<<<<<<<<<< + * global __cuGraphGetNodes + * data["__cuGraphGetNodes"] = __cuGraphGetNodes +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetType)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7681, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphNodeGetType, __pyx_t_3) < 0))) __PYX_ERR(0, 7681, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7683 + * data["__cuGraphNodeGetType"] = __cuGraphNodeGetType + * global __cuGraphGetNodes + * data["__cuGraphGetNodes"] = __cuGraphGetNodes # <<<<<<<<<<<<<< + * global __cuGraphGetRootNodes + * data["__cuGraphGetRootNodes"] = __cuGraphGetRootNodes +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetNodes)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7683, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphGetNodes, __pyx_t_3) < 0))) __PYX_ERR(0, 7683, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7685 + * data["__cuGraphGetNodes"] = __cuGraphGetNodes + * global __cuGraphGetRootNodes + * data["__cuGraphGetRootNodes"] = __cuGraphGetRootNodes # <<<<<<<<<<<<<< + * global __cuGraphGetEdges + * data["__cuGraphGetEdges"] = __cuGraphGetEdges +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetRootNodes)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7685, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphGetRootNodes, __pyx_t_3) < 0))) __PYX_ERR(0, 7685, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7687 + * data["__cuGraphGetRootNodes"] = __cuGraphGetRootNodes + * global __cuGraphGetEdges + * data["__cuGraphGetEdges"] = __cuGraphGetEdges # <<<<<<<<<<<<<< + * global __cuGraphGetEdges_v2 + * data["__cuGraphGetEdges_v2"] = __cuGraphGetEdges_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7687, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphGetEdges, __pyx_t_3) < 0))) __PYX_ERR(0, 7687, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7689 + * data["__cuGraphGetEdges"] = __cuGraphGetEdges + * global __cuGraphGetEdges_v2 + * data["__cuGraphGetEdges_v2"] = __cuGraphGetEdges_v2 # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependencies + * data["__cuGraphNodeGetDependencies"] = __cuGraphNodeGetDependencies +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7689, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphGetEdges_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7689, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7691 + * data["__cuGraphGetEdges_v2"] = __cuGraphGetEdges_v2 + * global __cuGraphNodeGetDependencies + * data["__cuGraphNodeGetDependencies"] = __cuGraphNodeGetDependencies # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependencies_v2 + * data["__cuGraphNodeGetDependencies_v2"] = __cuGraphNodeGetDependencies_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7691, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphNodeGetDependencies, __pyx_t_3) < 0))) __PYX_ERR(0, 7691, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7693 + * data["__cuGraphNodeGetDependencies"] = __cuGraphNodeGetDependencies + * global __cuGraphNodeGetDependencies_v2 + * data["__cuGraphNodeGetDependencies_v2"] = __cuGraphNodeGetDependencies_v2 # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependentNodes + * data["__cuGraphNodeGetDependentNodes"] = __cuGraphNodeGetDependentNodes +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7693, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphNodeGetDependencies_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7693, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7695 + * data["__cuGraphNodeGetDependencies_v2"] = __cuGraphNodeGetDependencies_v2 + * global __cuGraphNodeGetDependentNodes + * data["__cuGraphNodeGetDependentNodes"] = __cuGraphNodeGetDependentNodes # <<<<<<<<<<<<<< + * global __cuGraphNodeGetDependentNodes_v2 + * data["__cuGraphNodeGetDependentNodes_v2"] = __cuGraphNodeGetDependentNodes_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7695, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphNodeGetDependentNodes, __pyx_t_3) < 0))) __PYX_ERR(0, 7695, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7697 + * data["__cuGraphNodeGetDependentNodes"] = __cuGraphNodeGetDependentNodes + * global __cuGraphNodeGetDependentNodes_v2 + * data["__cuGraphNodeGetDependentNodes_v2"] = __cuGraphNodeGetDependentNodes_v2 # <<<<<<<<<<<<<< + * global __cuGraphAddDependencies + * data["__cuGraphAddDependencies"] = __cuGraphAddDependencies +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7697, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphNodeGetDependentNodes_v, __pyx_t_3) < 0))) __PYX_ERR(0, 7697, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7699 + * data["__cuGraphNodeGetDependentNodes_v2"] = __cuGraphNodeGetDependentNodes_v2 + * global __cuGraphAddDependencies + * data["__cuGraphAddDependencies"] = __cuGraphAddDependencies # <<<<<<<<<<<<<< + * global __cuGraphAddDependencies_v2 + * data["__cuGraphAddDependencies_v2"] = __cuGraphAddDependencies_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7699, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddDependencies, __pyx_t_3) < 0))) __PYX_ERR(0, 7699, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7701 + * data["__cuGraphAddDependencies"] = __cuGraphAddDependencies + * global __cuGraphAddDependencies_v2 + * data["__cuGraphAddDependencies_v2"] = __cuGraphAddDependencies_v2 # <<<<<<<<<<<<<< + * global __cuGraphRemoveDependencies + * data["__cuGraphRemoveDependencies"] = __cuGraphRemoveDependencies +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7701, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddDependencies_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7701, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7703 + * data["__cuGraphAddDependencies_v2"] = __cuGraphAddDependencies_v2 + * global __cuGraphRemoveDependencies + * data["__cuGraphRemoveDependencies"] = __cuGraphRemoveDependencies # <<<<<<<<<<<<<< + * global __cuGraphRemoveDependencies_v2 + * data["__cuGraphRemoveDependencies_v2"] = __cuGraphRemoveDependencies_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7703, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphRemoveDependencies, __pyx_t_3) < 0))) __PYX_ERR(0, 7703, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7705 + * data["__cuGraphRemoveDependencies"] = __cuGraphRemoveDependencies + * global __cuGraphRemoveDependencies_v2 + * data["__cuGraphRemoveDependencies_v2"] = __cuGraphRemoveDependencies_v2 # <<<<<<<<<<<<<< + * global __cuGraphDestroyNode + * data["__cuGraphDestroyNode"] = __cuGraphDestroyNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7705, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphRemoveDependencies_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7705, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7707 + * data["__cuGraphRemoveDependencies_v2"] = __cuGraphRemoveDependencies_v2 + * global __cuGraphDestroyNode + * data["__cuGraphDestroyNode"] = __cuGraphDestroyNode # <<<<<<<<<<<<<< + * global __cuGraphInstantiateWithFlags + * data["__cuGraphInstantiateWithFlags"] = __cuGraphInstantiateWithFlags +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroyNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7707, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphDestroyNode, __pyx_t_3) < 0))) __PYX_ERR(0, 7707, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7709 + * data["__cuGraphDestroyNode"] = __cuGraphDestroyNode + * global __cuGraphInstantiateWithFlags + * data["__cuGraphInstantiateWithFlags"] = __cuGraphInstantiateWithFlags # <<<<<<<<<<<<<< + * global __cuGraphInstantiateWithParams + * data["__cuGraphInstantiateWithParams"] = __cuGraphInstantiateWithParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithFlags)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7709, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphInstantiateWithFlags, __pyx_t_3) < 0))) __PYX_ERR(0, 7709, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7711 + * data["__cuGraphInstantiateWithFlags"] = __cuGraphInstantiateWithFlags + * global __cuGraphInstantiateWithParams + * data["__cuGraphInstantiateWithParams"] = __cuGraphInstantiateWithParams # <<<<<<<<<<<<<< + * global __cuGraphExecGetFlags + * data["__cuGraphExecGetFlags"] = __cuGraphExecGetFlags +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7711, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphInstantiateWithParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7711, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7713 + * data["__cuGraphInstantiateWithParams"] = __cuGraphInstantiateWithParams + * global __cuGraphExecGetFlags + * data["__cuGraphExecGetFlags"] = __cuGraphExecGetFlags # <<<<<<<<<<<<<< + * global __cuGraphExecKernelNodeSetParams_v2 + * data["__cuGraphExecKernelNodeSetParams_v2"] = __cuGraphExecKernelNodeSetParams_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecGetFlags)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7713, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecGetFlags, __pyx_t_3) < 0))) __PYX_ERR(0, 7713, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7715 + * data["__cuGraphExecGetFlags"] = __cuGraphExecGetFlags + * global __cuGraphExecKernelNodeSetParams_v2 + * data["__cuGraphExecKernelNodeSetParams_v2"] = __cuGraphExecKernelNodeSetParams_v2 # <<<<<<<<<<<<<< + * global __cuGraphExecMemcpyNodeSetParams + * data["__cuGraphExecMemcpyNodeSetParams"] = __cuGraphExecMemcpyNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecKernelNodeSetParams_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7715, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecKernelNodeSetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7715, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7717 + * data["__cuGraphExecKernelNodeSetParams_v2"] = __cuGraphExecKernelNodeSetParams_v2 + * global __cuGraphExecMemcpyNodeSetParams + * data["__cuGraphExecMemcpyNodeSetParams"] = __cuGraphExecMemcpyNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphExecMemsetNodeSetParams + * data["__cuGraphExecMemsetNodeSetParams"] = __cuGraphExecMemsetNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemcpyNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7717, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecMemcpyNodeSetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7717, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7719 + * data["__cuGraphExecMemcpyNodeSetParams"] = __cuGraphExecMemcpyNodeSetParams + * global __cuGraphExecMemsetNodeSetParams + * data["__cuGraphExecMemsetNodeSetParams"] = __cuGraphExecMemsetNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphExecHostNodeSetParams + * data["__cuGraphExecHostNodeSetParams"] = __cuGraphExecHostNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemsetNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7719, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecMemsetNodeSetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7719, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7721 + * data["__cuGraphExecMemsetNodeSetParams"] = __cuGraphExecMemsetNodeSetParams + * global __cuGraphExecHostNodeSetParams + * data["__cuGraphExecHostNodeSetParams"] = __cuGraphExecHostNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphExecChildGraphNodeSetParams + * data["__cuGraphExecChildGraphNodeSetParams"] = __cuGraphExecChildGraphNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecHostNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7721, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecHostNodeSetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7721, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7723 + * data["__cuGraphExecHostNodeSetParams"] = __cuGraphExecHostNodeSetParams + * global __cuGraphExecChildGraphNodeSetParams + * data["__cuGraphExecChildGraphNodeSetParams"] = __cuGraphExecChildGraphNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphExecEventRecordNodeSetEvent + * data["__cuGraphExecEventRecordNodeSetEvent"] = __cuGraphExecEventRecordNodeSetEvent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecChildGraphNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7723, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecChildGraphNodeSetPa, __pyx_t_3) < 0))) __PYX_ERR(0, 7723, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7725 + * data["__cuGraphExecChildGraphNodeSetParams"] = __cuGraphExecChildGraphNodeSetParams + * global __cuGraphExecEventRecordNodeSetEvent + * data["__cuGraphExecEventRecordNodeSetEvent"] = __cuGraphExecEventRecordNodeSetEvent # <<<<<<<<<<<<<< + * global __cuGraphExecEventWaitNodeSetEvent + * data["__cuGraphExecEventWaitNodeSetEvent"] = __cuGraphExecEventWaitNodeSetEvent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventRecordNodeSetEvent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7725, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecEventRecordNodeSetE, __pyx_t_3) < 0))) __PYX_ERR(0, 7725, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7727 + * data["__cuGraphExecEventRecordNodeSetEvent"] = __cuGraphExecEventRecordNodeSetEvent + * global __cuGraphExecEventWaitNodeSetEvent + * data["__cuGraphExecEventWaitNodeSetEvent"] = __cuGraphExecEventWaitNodeSetEvent # <<<<<<<<<<<<<< + * global __cuGraphExecExternalSemaphoresSignalNodeSetParams + * data["__cuGraphExecExternalSemaphoresSignalNodeSetParams"] = __cuGraphExecExternalSemaphoresSignalNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventWaitNodeSetEvent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7727, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecEventWaitNodeSetEve, __pyx_t_3) < 0))) __PYX_ERR(0, 7727, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7729 + * data["__cuGraphExecEventWaitNodeSetEvent"] = __cuGraphExecEventWaitNodeSetEvent + * global __cuGraphExecExternalSemaphoresSignalNodeSetParams + * data["__cuGraphExecExternalSemaphoresSignalNodeSetParams"] = __cuGraphExecExternalSemaphoresSignalNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphExecExternalSemaphoresWaitNodeSetParams + * data["__cuGraphExecExternalSemaphoresWaitNodeSetParams"] = __cuGraphExecExternalSemaphoresWaitNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresSignalNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7729, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecExternalSemaphoresS, __pyx_t_3) < 0))) __PYX_ERR(0, 7729, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7731 + * data["__cuGraphExecExternalSemaphoresSignalNodeSetParams"] = __cuGraphExecExternalSemaphoresSignalNodeSetParams + * global __cuGraphExecExternalSemaphoresWaitNodeSetParams + * data["__cuGraphExecExternalSemaphoresWaitNodeSetParams"] = __cuGraphExecExternalSemaphoresWaitNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphNodeSetEnabled + * data["__cuGraphNodeSetEnabled"] = __cuGraphNodeSetEnabled +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresWaitNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7731, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecExternalSemaphoresW, __pyx_t_3) < 0))) __PYX_ERR(0, 7731, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7733 + * data["__cuGraphExecExternalSemaphoresWaitNodeSetParams"] = __cuGraphExecExternalSemaphoresWaitNodeSetParams + * global __cuGraphNodeSetEnabled + * data["__cuGraphNodeSetEnabled"] = __cuGraphNodeSetEnabled # <<<<<<<<<<<<<< + * global __cuGraphNodeGetEnabled + * data["__cuGraphNodeGetEnabled"] = __cuGraphNodeGetEnabled +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetEnabled)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7733, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphNodeSetEnabled, __pyx_t_3) < 0))) __PYX_ERR(0, 7733, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7735 + * data["__cuGraphNodeSetEnabled"] = __cuGraphNodeSetEnabled + * global __cuGraphNodeGetEnabled + * data["__cuGraphNodeGetEnabled"] = __cuGraphNodeGetEnabled # <<<<<<<<<<<<<< + * global __cuGraphUpload + * data["__cuGraphUpload"] = __cuGraphUpload +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetEnabled)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7735, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphNodeGetEnabled, __pyx_t_3) < 0))) __PYX_ERR(0, 7735, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7737 + * data["__cuGraphNodeGetEnabled"] = __cuGraphNodeGetEnabled + * global __cuGraphUpload + * data["__cuGraphUpload"] = __cuGraphUpload # <<<<<<<<<<<<<< + * global __cuGraphLaunch + * data["__cuGraphLaunch"] = __cuGraphLaunch +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphUpload)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7737, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphUpload, __pyx_t_3) < 0))) __PYX_ERR(0, 7737, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7739 + * data["__cuGraphUpload"] = __cuGraphUpload + * global __cuGraphLaunch + * data["__cuGraphLaunch"] = __cuGraphLaunch # <<<<<<<<<<<<<< + * global __cuGraphExecDestroy + * data["__cuGraphExecDestroy"] = __cuGraphExecDestroy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphLaunch)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7739, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphLaunch, __pyx_t_3) < 0))) __PYX_ERR(0, 7739, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7741 + * data["__cuGraphLaunch"] = __cuGraphLaunch + * global __cuGraphExecDestroy + * data["__cuGraphExecDestroy"] = __cuGraphExecDestroy # <<<<<<<<<<<<<< + * global __cuGraphDestroy + * data["__cuGraphDestroy"] = __cuGraphDestroy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecDestroy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7741, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecDestroy, __pyx_t_3) < 0))) __PYX_ERR(0, 7741, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7743 + * data["__cuGraphExecDestroy"] = __cuGraphExecDestroy + * global __cuGraphDestroy + * data["__cuGraphDestroy"] = __cuGraphDestroy # <<<<<<<<<<<<<< + * global __cuGraphExecUpdate_v2 + * data["__cuGraphExecUpdate_v2"] = __cuGraphExecUpdate_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7743, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphDestroy, __pyx_t_3) < 0))) __PYX_ERR(0, 7743, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7745 + * data["__cuGraphDestroy"] = __cuGraphDestroy + * global __cuGraphExecUpdate_v2 + * data["__cuGraphExecUpdate_v2"] = __cuGraphExecUpdate_v2 # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeCopyAttributes + * data["__cuGraphKernelNodeCopyAttributes"] = __cuGraphKernelNodeCopyAttributes +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecUpdate_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7745, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecUpdate_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7745, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7747 + * data["__cuGraphExecUpdate_v2"] = __cuGraphExecUpdate_v2 + * global __cuGraphKernelNodeCopyAttributes + * data["__cuGraphKernelNodeCopyAttributes"] = __cuGraphKernelNodeCopyAttributes # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeGetAttribute + * data["__cuGraphKernelNodeGetAttribute"] = __cuGraphKernelNodeGetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeCopyAttributes)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7747, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphKernelNodeCopyAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7747, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7749 + * data["__cuGraphKernelNodeCopyAttributes"] = __cuGraphKernelNodeCopyAttributes + * global __cuGraphKernelNodeGetAttribute + * data["__cuGraphKernelNodeGetAttribute"] = __cuGraphKernelNodeGetAttribute # <<<<<<<<<<<<<< + * global __cuGraphKernelNodeSetAttribute + * data["__cuGraphKernelNodeSetAttribute"] = __cuGraphKernelNodeSetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7749, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphKernelNodeGetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7749, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7751 + * data["__cuGraphKernelNodeGetAttribute"] = __cuGraphKernelNodeGetAttribute + * global __cuGraphKernelNodeSetAttribute + * data["__cuGraphKernelNodeSetAttribute"] = __cuGraphKernelNodeSetAttribute # <<<<<<<<<<<<<< + * global __cuGraphDebugDotPrint + * data["__cuGraphDebugDotPrint"] = __cuGraphDebugDotPrint +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7751, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphKernelNodeSetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7751, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7753 + * data["__cuGraphKernelNodeSetAttribute"] = __cuGraphKernelNodeSetAttribute + * global __cuGraphDebugDotPrint + * data["__cuGraphDebugDotPrint"] = __cuGraphDebugDotPrint # <<<<<<<<<<<<<< + * global __cuUserObjectCreate + * data["__cuUserObjectCreate"] = __cuUserObjectCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDebugDotPrint)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7753, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphDebugDotPrint, __pyx_t_3) < 0))) __PYX_ERR(0, 7753, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7755 + * data["__cuGraphDebugDotPrint"] = __cuGraphDebugDotPrint + * global __cuUserObjectCreate + * data["__cuUserObjectCreate"] = __cuUserObjectCreate # <<<<<<<<<<<<<< + * global __cuUserObjectRetain + * data["__cuUserObjectRetain"] = __cuUserObjectRetain +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7755, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuUserObjectCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7755, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7757 + * data["__cuUserObjectCreate"] = __cuUserObjectCreate + * global __cuUserObjectRetain + * data["__cuUserObjectRetain"] = __cuUserObjectRetain # <<<<<<<<<<<<<< + * global __cuUserObjectRelease + * data["__cuUserObjectRelease"] = __cuUserObjectRelease +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRetain)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7757, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuUserObjectRetain, __pyx_t_3) < 0))) __PYX_ERR(0, 7757, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7759 + * data["__cuUserObjectRetain"] = __cuUserObjectRetain + * global __cuUserObjectRelease + * data["__cuUserObjectRelease"] = __cuUserObjectRelease # <<<<<<<<<<<<<< + * global __cuGraphRetainUserObject + * data["__cuGraphRetainUserObject"] = __cuGraphRetainUserObject +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRelease)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7759, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuUserObjectRelease, __pyx_t_3) < 0))) __PYX_ERR(0, 7759, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7761 + * data["__cuUserObjectRelease"] = __cuUserObjectRelease + * global __cuGraphRetainUserObject + * data["__cuGraphRetainUserObject"] = __cuGraphRetainUserObject # <<<<<<<<<<<<<< + * global __cuGraphReleaseUserObject + * data["__cuGraphReleaseUserObject"] = __cuGraphReleaseUserObject +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRetainUserObject)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7761, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphRetainUserObject, __pyx_t_3) < 0))) __PYX_ERR(0, 7761, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7763 + * data["__cuGraphRetainUserObject"] = __cuGraphRetainUserObject + * global __cuGraphReleaseUserObject + * data["__cuGraphReleaseUserObject"] = __cuGraphReleaseUserObject # <<<<<<<<<<<<<< + * global __cuGraphAddNode + * data["__cuGraphAddNode"] = __cuGraphAddNode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphReleaseUserObject)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7763, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphReleaseUserObject, __pyx_t_3) < 0))) __PYX_ERR(0, 7763, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7765 + * data["__cuGraphReleaseUserObject"] = __cuGraphReleaseUserObject + * global __cuGraphAddNode + * data["__cuGraphAddNode"] = __cuGraphAddNode # <<<<<<<<<<<<<< + * global __cuGraphAddNode_v2 + * data["__cuGraphAddNode_v2"] = __cuGraphAddNode_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7765, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddNode, __pyx_t_3) < 0))) __PYX_ERR(0, 7765, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7767 + * data["__cuGraphAddNode"] = __cuGraphAddNode + * global __cuGraphAddNode_v2 + * data["__cuGraphAddNode_v2"] = __cuGraphAddNode_v2 # <<<<<<<<<<<<<< + * global __cuGraphNodeSetParams + * data["__cuGraphNodeSetParams"] = __cuGraphNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7767, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphAddNode_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7767, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7769 + * data["__cuGraphAddNode_v2"] = __cuGraphAddNode_v2 + * global __cuGraphNodeSetParams + * data["__cuGraphNodeSetParams"] = __cuGraphNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphExecNodeSetParams + * data["__cuGraphExecNodeSetParams"] = __cuGraphExecNodeSetParams +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7769, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphNodeSetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7769, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7771 + * data["__cuGraphNodeSetParams"] = __cuGraphNodeSetParams + * global __cuGraphExecNodeSetParams + * data["__cuGraphExecNodeSetParams"] = __cuGraphExecNodeSetParams # <<<<<<<<<<<<<< + * global __cuGraphConditionalHandleCreate + * data["__cuGraphConditionalHandleCreate"] = __cuGraphConditionalHandleCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecNodeSetParams)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7771, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphExecNodeSetParams, __pyx_t_3) < 0))) __PYX_ERR(0, 7771, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7773 + * data["__cuGraphExecNodeSetParams"] = __cuGraphExecNodeSetParams + * global __cuGraphConditionalHandleCreate + * data["__cuGraphConditionalHandleCreate"] = __cuGraphConditionalHandleCreate # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveBlocksPerMultiprocessor + * data["__cuOccupancyMaxActiveBlocksPerMultiprocessor"] = __cuOccupancyMaxActiveBlocksPerMultiprocessor +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphConditionalHandleCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7773, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphConditionalHandleCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7773, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7775 + * data["__cuGraphConditionalHandleCreate"] = __cuGraphConditionalHandleCreate + * global __cuOccupancyMaxActiveBlocksPerMultiprocessor + * data["__cuOccupancyMaxActiveBlocksPerMultiprocessor"] = __cuOccupancyMaxActiveBlocksPerMultiprocessor # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + * data["__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags"] = __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessor)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7775, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuOccupancyMaxActiveBlocksPerM, __pyx_t_3) < 0))) __PYX_ERR(0, 7775, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7777 + * data["__cuOccupancyMaxActiveBlocksPerMultiprocessor"] = __cuOccupancyMaxActiveBlocksPerMultiprocessor + * global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + * data["__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags"] = __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialBlockSize + * data["__cuOccupancyMaxPotentialBlockSize"] = __cuOccupancyMaxPotentialBlockSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7777, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuOccupancyMaxActiveBlocksPerM_2, __pyx_t_3) < 0))) __PYX_ERR(0, 7777, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7779 + * data["__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags"] = __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + * global __cuOccupancyMaxPotentialBlockSize + * data["__cuOccupancyMaxPotentialBlockSize"] = __cuOccupancyMaxPotentialBlockSize # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialBlockSizeWithFlags + * data["__cuOccupancyMaxPotentialBlockSizeWithFlags"] = __cuOccupancyMaxPotentialBlockSizeWithFlags +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7779, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuOccupancyMaxPotentialBlockSi, __pyx_t_3) < 0))) __PYX_ERR(0, 7779, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7781 + * data["__cuOccupancyMaxPotentialBlockSize"] = __cuOccupancyMaxPotentialBlockSize + * global __cuOccupancyMaxPotentialBlockSizeWithFlags + * data["__cuOccupancyMaxPotentialBlockSizeWithFlags"] = __cuOccupancyMaxPotentialBlockSizeWithFlags # <<<<<<<<<<<<<< + * global __cuOccupancyAvailableDynamicSMemPerBlock + * data["__cuOccupancyAvailableDynamicSMemPerBlock"] = __cuOccupancyAvailableDynamicSMemPerBlock +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSizeWithFlags)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7781, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuOccupancyMaxPotentialBlockSi_2, __pyx_t_3) < 0))) __PYX_ERR(0, 7781, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7783 + * data["__cuOccupancyMaxPotentialBlockSizeWithFlags"] = __cuOccupancyMaxPotentialBlockSizeWithFlags + * global __cuOccupancyAvailableDynamicSMemPerBlock + * data["__cuOccupancyAvailableDynamicSMemPerBlock"] = __cuOccupancyAvailableDynamicSMemPerBlock # <<<<<<<<<<<<<< + * global __cuOccupancyMaxPotentialClusterSize + * data["__cuOccupancyMaxPotentialClusterSize"] = __cuOccupancyMaxPotentialClusterSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyAvailableDynamicSMemPerBlock)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7783, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuOccupancyAvailableDynamicSMe, __pyx_t_3) < 0))) __PYX_ERR(0, 7783, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7785 + * data["__cuOccupancyAvailableDynamicSMemPerBlock"] = __cuOccupancyAvailableDynamicSMemPerBlock + * global __cuOccupancyMaxPotentialClusterSize + * data["__cuOccupancyMaxPotentialClusterSize"] = __cuOccupancyMaxPotentialClusterSize # <<<<<<<<<<<<<< + * global __cuOccupancyMaxActiveClusters + * data["__cuOccupancyMaxActiveClusters"] = __cuOccupancyMaxActiveClusters +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialClusterSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7785, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuOccupancyMaxPotentialCluster, __pyx_t_3) < 0))) __PYX_ERR(0, 7785, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7787 + * data["__cuOccupancyMaxPotentialClusterSize"] = __cuOccupancyMaxPotentialClusterSize + * global __cuOccupancyMaxActiveClusters + * data["__cuOccupancyMaxActiveClusters"] = __cuOccupancyMaxActiveClusters # <<<<<<<<<<<<<< + * global __cuTexRefSetArray + * data["__cuTexRefSetArray"] = __cuTexRefSetArray +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveClusters)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7787, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuOccupancyMaxActiveClusters, __pyx_t_3) < 0))) __PYX_ERR(0, 7787, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7789 + * data["__cuOccupancyMaxActiveClusters"] = __cuOccupancyMaxActiveClusters + * global __cuTexRefSetArray + * data["__cuTexRefSetArray"] = __cuTexRefSetArray # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmappedArray + * data["__cuTexRefSetMipmappedArray"] = __cuTexRefSetMipmappedArray +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetArray)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7789, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefSetArray, __pyx_t_3) < 0))) __PYX_ERR(0, 7789, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7791 + * data["__cuTexRefSetArray"] = __cuTexRefSetArray + * global __cuTexRefSetMipmappedArray + * data["__cuTexRefSetMipmappedArray"] = __cuTexRefSetMipmappedArray # <<<<<<<<<<<<<< + * global __cuTexRefSetAddress_v2 + * data["__cuTexRefSetAddress_v2"] = __cuTexRefSetAddress_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmappedArray)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7791, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefSetMipmappedArray, __pyx_t_3) < 0))) __PYX_ERR(0, 7791, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7793 + * data["__cuTexRefSetMipmappedArray"] = __cuTexRefSetMipmappedArray + * global __cuTexRefSetAddress_v2 + * data["__cuTexRefSetAddress_v2"] = __cuTexRefSetAddress_v2 # <<<<<<<<<<<<<< + * global __cuTexRefSetAddress2D_v3 + * data["__cuTexRefSetAddress2D_v3"] = __cuTexRefSetAddress2D_v3 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7793, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefSetAddress_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7793, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7795 + * data["__cuTexRefSetAddress_v2"] = __cuTexRefSetAddress_v2 + * global __cuTexRefSetAddress2D_v3 + * data["__cuTexRefSetAddress2D_v3"] = __cuTexRefSetAddress2D_v3 # <<<<<<<<<<<<<< + * global __cuTexRefSetFormat + * data["__cuTexRefSetFormat"] = __cuTexRefSetFormat +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress2D_v3)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7795, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefSetAddress2D_v3, __pyx_t_3) < 0))) __PYX_ERR(0, 7795, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7797 + * data["__cuTexRefSetAddress2D_v3"] = __cuTexRefSetAddress2D_v3 + * global __cuTexRefSetFormat + * data["__cuTexRefSetFormat"] = __cuTexRefSetFormat # <<<<<<<<<<<<<< + * global __cuTexRefSetAddressMode + * data["__cuTexRefSetAddressMode"] = __cuTexRefSetAddressMode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFormat)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7797, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefSetFormat, __pyx_t_3) < 0))) __PYX_ERR(0, 7797, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7799 + * data["__cuTexRefSetFormat"] = __cuTexRefSetFormat + * global __cuTexRefSetAddressMode + * data["__cuTexRefSetAddressMode"] = __cuTexRefSetAddressMode # <<<<<<<<<<<<<< + * global __cuTexRefSetFilterMode + * data["__cuTexRefSetFilterMode"] = __cuTexRefSetFilterMode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddressMode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7799, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefSetAddressMode, __pyx_t_3) < 0))) __PYX_ERR(0, 7799, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7801 + * data["__cuTexRefSetAddressMode"] = __cuTexRefSetAddressMode + * global __cuTexRefSetFilterMode + * data["__cuTexRefSetFilterMode"] = __cuTexRefSetFilterMode # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapFilterMode + * data["__cuTexRefSetMipmapFilterMode"] = __cuTexRefSetMipmapFilterMode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFilterMode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7801, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefSetFilterMode, __pyx_t_3) < 0))) __PYX_ERR(0, 7801, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7803 + * data["__cuTexRefSetFilterMode"] = __cuTexRefSetFilterMode + * global __cuTexRefSetMipmapFilterMode + * data["__cuTexRefSetMipmapFilterMode"] = __cuTexRefSetMipmapFilterMode # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapLevelBias + * data["__cuTexRefSetMipmapLevelBias"] = __cuTexRefSetMipmapLevelBias +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapFilterMode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7803, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefSetMipmapFilterMode, __pyx_t_3) < 0))) __PYX_ERR(0, 7803, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7805 + * data["__cuTexRefSetMipmapFilterMode"] = __cuTexRefSetMipmapFilterMode + * global __cuTexRefSetMipmapLevelBias + * data["__cuTexRefSetMipmapLevelBias"] = __cuTexRefSetMipmapLevelBias # <<<<<<<<<<<<<< + * global __cuTexRefSetMipmapLevelClamp + * data["__cuTexRefSetMipmapLevelClamp"] = __cuTexRefSetMipmapLevelClamp +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelBias)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7805, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefSetMipmapLevelBias, __pyx_t_3) < 0))) __PYX_ERR(0, 7805, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7807 + * data["__cuTexRefSetMipmapLevelBias"] = __cuTexRefSetMipmapLevelBias + * global __cuTexRefSetMipmapLevelClamp + * data["__cuTexRefSetMipmapLevelClamp"] = __cuTexRefSetMipmapLevelClamp # <<<<<<<<<<<<<< + * global __cuTexRefSetMaxAnisotropy + * data["__cuTexRefSetMaxAnisotropy"] = __cuTexRefSetMaxAnisotropy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelClamp)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7807, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefSetMipmapLevelClamp, __pyx_t_3) < 0))) __PYX_ERR(0, 7807, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7809 + * data["__cuTexRefSetMipmapLevelClamp"] = __cuTexRefSetMipmapLevelClamp + * global __cuTexRefSetMaxAnisotropy + * data["__cuTexRefSetMaxAnisotropy"] = __cuTexRefSetMaxAnisotropy # <<<<<<<<<<<<<< + * global __cuTexRefSetBorderColor + * data["__cuTexRefSetBorderColor"] = __cuTexRefSetBorderColor +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMaxAnisotropy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7809, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefSetMaxAnisotropy, __pyx_t_3) < 0))) __PYX_ERR(0, 7809, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7811 + * data["__cuTexRefSetMaxAnisotropy"] = __cuTexRefSetMaxAnisotropy + * global __cuTexRefSetBorderColor + * data["__cuTexRefSetBorderColor"] = __cuTexRefSetBorderColor # <<<<<<<<<<<<<< + * global __cuTexRefSetFlags + * data["__cuTexRefSetFlags"] = __cuTexRefSetFlags +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetBorderColor)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7811, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefSetBorderColor, __pyx_t_3) < 0))) __PYX_ERR(0, 7811, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7813 + * data["__cuTexRefSetBorderColor"] = __cuTexRefSetBorderColor + * global __cuTexRefSetFlags + * data["__cuTexRefSetFlags"] = __cuTexRefSetFlags # <<<<<<<<<<<<<< + * global __cuTexRefGetAddress_v2 + * data["__cuTexRefGetAddress_v2"] = __cuTexRefGetAddress_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFlags)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7813, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefSetFlags, __pyx_t_3) < 0))) __PYX_ERR(0, 7813, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7815 + * data["__cuTexRefSetFlags"] = __cuTexRefSetFlags + * global __cuTexRefGetAddress_v2 + * data["__cuTexRefGetAddress_v2"] = __cuTexRefGetAddress_v2 # <<<<<<<<<<<<<< + * global __cuTexRefGetArray + * data["__cuTexRefGetArray"] = __cuTexRefGetArray +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddress_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7815, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefGetAddress_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7815, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7817 + * data["__cuTexRefGetAddress_v2"] = __cuTexRefGetAddress_v2 + * global __cuTexRefGetArray + * data["__cuTexRefGetArray"] = __cuTexRefGetArray # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmappedArray + * data["__cuTexRefGetMipmappedArray"] = __cuTexRefGetMipmappedArray +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetArray)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7817, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefGetArray, __pyx_t_3) < 0))) __PYX_ERR(0, 7817, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7819 + * data["__cuTexRefGetArray"] = __cuTexRefGetArray + * global __cuTexRefGetMipmappedArray + * data["__cuTexRefGetMipmappedArray"] = __cuTexRefGetMipmappedArray # <<<<<<<<<<<<<< + * global __cuTexRefGetAddressMode + * data["__cuTexRefGetAddressMode"] = __cuTexRefGetAddressMode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmappedArray)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7819, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefGetMipmappedArray, __pyx_t_3) < 0))) __PYX_ERR(0, 7819, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7821 + * data["__cuTexRefGetMipmappedArray"] = __cuTexRefGetMipmappedArray + * global __cuTexRefGetAddressMode + * data["__cuTexRefGetAddressMode"] = __cuTexRefGetAddressMode # <<<<<<<<<<<<<< + * global __cuTexRefGetFilterMode + * data["__cuTexRefGetFilterMode"] = __cuTexRefGetFilterMode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddressMode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7821, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefGetAddressMode, __pyx_t_3) < 0))) __PYX_ERR(0, 7821, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7823 + * data["__cuTexRefGetAddressMode"] = __cuTexRefGetAddressMode + * global __cuTexRefGetFilterMode + * data["__cuTexRefGetFilterMode"] = __cuTexRefGetFilterMode # <<<<<<<<<<<<<< + * global __cuTexRefGetFormat + * data["__cuTexRefGetFormat"] = __cuTexRefGetFormat +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFilterMode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7823, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefGetFilterMode, __pyx_t_3) < 0))) __PYX_ERR(0, 7823, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7825 + * data["__cuTexRefGetFilterMode"] = __cuTexRefGetFilterMode + * global __cuTexRefGetFormat + * data["__cuTexRefGetFormat"] = __cuTexRefGetFormat # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapFilterMode + * data["__cuTexRefGetMipmapFilterMode"] = __cuTexRefGetMipmapFilterMode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFormat)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7825, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefGetFormat, __pyx_t_3) < 0))) __PYX_ERR(0, 7825, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7827 + * data["__cuTexRefGetFormat"] = __cuTexRefGetFormat + * global __cuTexRefGetMipmapFilterMode + * data["__cuTexRefGetMipmapFilterMode"] = __cuTexRefGetMipmapFilterMode # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapLevelBias + * data["__cuTexRefGetMipmapLevelBias"] = __cuTexRefGetMipmapLevelBias +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapFilterMode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7827, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefGetMipmapFilterMode, __pyx_t_3) < 0))) __PYX_ERR(0, 7827, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7829 + * data["__cuTexRefGetMipmapFilterMode"] = __cuTexRefGetMipmapFilterMode + * global __cuTexRefGetMipmapLevelBias + * data["__cuTexRefGetMipmapLevelBias"] = __cuTexRefGetMipmapLevelBias # <<<<<<<<<<<<<< + * global __cuTexRefGetMipmapLevelClamp + * data["__cuTexRefGetMipmapLevelClamp"] = __cuTexRefGetMipmapLevelClamp +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelBias)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7829, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefGetMipmapLevelBias, __pyx_t_3) < 0))) __PYX_ERR(0, 7829, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7831 + * data["__cuTexRefGetMipmapLevelBias"] = __cuTexRefGetMipmapLevelBias + * global __cuTexRefGetMipmapLevelClamp + * data["__cuTexRefGetMipmapLevelClamp"] = __cuTexRefGetMipmapLevelClamp # <<<<<<<<<<<<<< + * global __cuTexRefGetMaxAnisotropy + * data["__cuTexRefGetMaxAnisotropy"] = __cuTexRefGetMaxAnisotropy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelClamp)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7831, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefGetMipmapLevelClamp, __pyx_t_3) < 0))) __PYX_ERR(0, 7831, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7833 + * data["__cuTexRefGetMipmapLevelClamp"] = __cuTexRefGetMipmapLevelClamp + * global __cuTexRefGetMaxAnisotropy + * data["__cuTexRefGetMaxAnisotropy"] = __cuTexRefGetMaxAnisotropy # <<<<<<<<<<<<<< + * global __cuTexRefGetBorderColor + * data["__cuTexRefGetBorderColor"] = __cuTexRefGetBorderColor +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMaxAnisotropy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7833, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefGetMaxAnisotropy, __pyx_t_3) < 0))) __PYX_ERR(0, 7833, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7835 + * data["__cuTexRefGetMaxAnisotropy"] = __cuTexRefGetMaxAnisotropy + * global __cuTexRefGetBorderColor + * data["__cuTexRefGetBorderColor"] = __cuTexRefGetBorderColor # <<<<<<<<<<<<<< + * global __cuTexRefGetFlags + * data["__cuTexRefGetFlags"] = __cuTexRefGetFlags +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetBorderColor)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7835, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefGetBorderColor, __pyx_t_3) < 0))) __PYX_ERR(0, 7835, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7837 + * data["__cuTexRefGetBorderColor"] = __cuTexRefGetBorderColor + * global __cuTexRefGetFlags + * data["__cuTexRefGetFlags"] = __cuTexRefGetFlags # <<<<<<<<<<<<<< + * global __cuTexRefCreate + * data["__cuTexRefCreate"] = __cuTexRefCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFlags)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7837, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefGetFlags, __pyx_t_3) < 0))) __PYX_ERR(0, 7837, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7839 + * data["__cuTexRefGetFlags"] = __cuTexRefGetFlags + * global __cuTexRefCreate + * data["__cuTexRefCreate"] = __cuTexRefCreate # <<<<<<<<<<<<<< + * global __cuTexRefDestroy + * data["__cuTexRefDestroy"] = __cuTexRefDestroy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7839, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7839, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7841 + * data["__cuTexRefCreate"] = __cuTexRefCreate + * global __cuTexRefDestroy + * data["__cuTexRefDestroy"] = __cuTexRefDestroy # <<<<<<<<<<<<<< + * global __cuSurfRefSetArray + * data["__cuSurfRefSetArray"] = __cuSurfRefSetArray +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefDestroy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7841, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexRefDestroy, __pyx_t_3) < 0))) __PYX_ERR(0, 7841, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7843 + * data["__cuTexRefDestroy"] = __cuTexRefDestroy + * global __cuSurfRefSetArray + * data["__cuSurfRefSetArray"] = __cuSurfRefSetArray # <<<<<<<<<<<<<< + * global __cuSurfRefGetArray + * data["__cuSurfRefGetArray"] = __cuSurfRefGetArray +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefSetArray)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7843, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuSurfRefSetArray, __pyx_t_3) < 0))) __PYX_ERR(0, 7843, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7845 + * data["__cuSurfRefSetArray"] = __cuSurfRefSetArray + * global __cuSurfRefGetArray + * data["__cuSurfRefGetArray"] = __cuSurfRefGetArray # <<<<<<<<<<<<<< + * global __cuTexObjectCreate + * data["__cuTexObjectCreate"] = __cuTexObjectCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefGetArray)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7845, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuSurfRefGetArray, __pyx_t_3) < 0))) __PYX_ERR(0, 7845, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7847 + * data["__cuSurfRefGetArray"] = __cuSurfRefGetArray + * global __cuTexObjectCreate + * data["__cuTexObjectCreate"] = __cuTexObjectCreate # <<<<<<<<<<<<<< + * global __cuTexObjectDestroy + * data["__cuTexObjectDestroy"] = __cuTexObjectDestroy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7847, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexObjectCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7847, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7849 + * data["__cuTexObjectCreate"] = __cuTexObjectCreate + * global __cuTexObjectDestroy + * data["__cuTexObjectDestroy"] = __cuTexObjectDestroy # <<<<<<<<<<<<<< + * global __cuTexObjectGetResourceDesc + * data["__cuTexObjectGetResourceDesc"] = __cuTexObjectGetResourceDesc +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectDestroy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7849, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexObjectDestroy, __pyx_t_3) < 0))) __PYX_ERR(0, 7849, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7851 + * data["__cuTexObjectDestroy"] = __cuTexObjectDestroy + * global __cuTexObjectGetResourceDesc + * data["__cuTexObjectGetResourceDesc"] = __cuTexObjectGetResourceDesc # <<<<<<<<<<<<<< + * global __cuTexObjectGetTextureDesc + * data["__cuTexObjectGetTextureDesc"] = __cuTexObjectGetTextureDesc +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceDesc)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7851, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexObjectGetResourceDesc, __pyx_t_3) < 0))) __PYX_ERR(0, 7851, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7853 + * data["__cuTexObjectGetResourceDesc"] = __cuTexObjectGetResourceDesc + * global __cuTexObjectGetTextureDesc + * data["__cuTexObjectGetTextureDesc"] = __cuTexObjectGetTextureDesc # <<<<<<<<<<<<<< + * global __cuTexObjectGetResourceViewDesc + * data["__cuTexObjectGetResourceViewDesc"] = __cuTexObjectGetResourceViewDesc +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetTextureDesc)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7853, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexObjectGetTextureDesc, __pyx_t_3) < 0))) __PYX_ERR(0, 7853, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7855 + * data["__cuTexObjectGetTextureDesc"] = __cuTexObjectGetTextureDesc + * global __cuTexObjectGetResourceViewDesc + * data["__cuTexObjectGetResourceViewDesc"] = __cuTexObjectGetResourceViewDesc # <<<<<<<<<<<<<< + * global __cuSurfObjectCreate + * data["__cuSurfObjectCreate"] = __cuSurfObjectCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceViewDesc)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7855, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTexObjectGetResourceViewDesc, __pyx_t_3) < 0))) __PYX_ERR(0, 7855, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7857 + * data["__cuTexObjectGetResourceViewDesc"] = __cuTexObjectGetResourceViewDesc + * global __cuSurfObjectCreate + * data["__cuSurfObjectCreate"] = __cuSurfObjectCreate # <<<<<<<<<<<<<< + * global __cuSurfObjectDestroy + * data["__cuSurfObjectDestroy"] = __cuSurfObjectDestroy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7857, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuSurfObjectCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7857, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7859 + * data["__cuSurfObjectCreate"] = __cuSurfObjectCreate + * global __cuSurfObjectDestroy + * data["__cuSurfObjectDestroy"] = __cuSurfObjectDestroy # <<<<<<<<<<<<<< + * global __cuSurfObjectGetResourceDesc + * data["__cuSurfObjectGetResourceDesc"] = __cuSurfObjectGetResourceDesc +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectDestroy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7859, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuSurfObjectDestroy, __pyx_t_3) < 0))) __PYX_ERR(0, 7859, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7861 + * data["__cuSurfObjectDestroy"] = __cuSurfObjectDestroy + * global __cuSurfObjectGetResourceDesc + * data["__cuSurfObjectGetResourceDesc"] = __cuSurfObjectGetResourceDesc # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeTiled + * data["__cuTensorMapEncodeTiled"] = __cuTensorMapEncodeTiled +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectGetResourceDesc)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7861, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuSurfObjectGetResourceDesc, __pyx_t_3) < 0))) __PYX_ERR(0, 7861, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7863 + * data["__cuSurfObjectGetResourceDesc"] = __cuSurfObjectGetResourceDesc + * global __cuTensorMapEncodeTiled + * data["__cuTensorMapEncodeTiled"] = __cuTensorMapEncodeTiled # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeIm2col + * data["__cuTensorMapEncodeIm2col"] = __cuTensorMapEncodeIm2col +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeTiled)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7863, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTensorMapEncodeTiled, __pyx_t_3) < 0))) __PYX_ERR(0, 7863, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7865 + * data["__cuTensorMapEncodeTiled"] = __cuTensorMapEncodeTiled + * global __cuTensorMapEncodeIm2col + * data["__cuTensorMapEncodeIm2col"] = __cuTensorMapEncodeIm2col # <<<<<<<<<<<<<< + * global __cuTensorMapEncodeIm2colWide + * data["__cuTensorMapEncodeIm2colWide"] = __cuTensorMapEncodeIm2colWide +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2col)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7865, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTensorMapEncodeIm2col, __pyx_t_3) < 0))) __PYX_ERR(0, 7865, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7867 + * data["__cuTensorMapEncodeIm2col"] = __cuTensorMapEncodeIm2col + * global __cuTensorMapEncodeIm2colWide + * data["__cuTensorMapEncodeIm2colWide"] = __cuTensorMapEncodeIm2colWide # <<<<<<<<<<<<<< + * global __cuTensorMapReplaceAddress + * data["__cuTensorMapReplaceAddress"] = __cuTensorMapReplaceAddress +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2colWide)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7867, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTensorMapEncodeIm2colWide, __pyx_t_3) < 0))) __PYX_ERR(0, 7867, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7869 + * data["__cuTensorMapEncodeIm2colWide"] = __cuTensorMapEncodeIm2colWide + * global __cuTensorMapReplaceAddress + * data["__cuTensorMapReplaceAddress"] = __cuTensorMapReplaceAddress # <<<<<<<<<<<<<< + * global __cuDeviceCanAccessPeer + * data["__cuDeviceCanAccessPeer"] = __cuDeviceCanAccessPeer +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapReplaceAddress)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7869, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuTensorMapReplaceAddress, __pyx_t_3) < 0))) __PYX_ERR(0, 7869, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7871 + * data["__cuTensorMapReplaceAddress"] = __cuTensorMapReplaceAddress + * global __cuDeviceCanAccessPeer + * data["__cuDeviceCanAccessPeer"] = __cuDeviceCanAccessPeer # <<<<<<<<<<<<<< + * global __cuCtxEnablePeerAccess + * data["__cuCtxEnablePeerAccess"] = __cuCtxEnablePeerAccess +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceCanAccessPeer)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7871, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceCanAccessPeer, __pyx_t_3) < 0))) __PYX_ERR(0, 7871, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7873 + * data["__cuDeviceCanAccessPeer"] = __cuDeviceCanAccessPeer + * global __cuCtxEnablePeerAccess + * data["__cuCtxEnablePeerAccess"] = __cuCtxEnablePeerAccess # <<<<<<<<<<<<<< + * global __cuCtxDisablePeerAccess + * data["__cuCtxDisablePeerAccess"] = __cuCtxDisablePeerAccess +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxEnablePeerAccess)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7873, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxEnablePeerAccess, __pyx_t_3) < 0))) __PYX_ERR(0, 7873, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7875 + * data["__cuCtxEnablePeerAccess"] = __cuCtxEnablePeerAccess + * global __cuCtxDisablePeerAccess + * data["__cuCtxDisablePeerAccess"] = __cuCtxDisablePeerAccess # <<<<<<<<<<<<<< + * global __cuDeviceGetP2PAttribute + * data["__cuDeviceGetP2PAttribute"] = __cuDeviceGetP2PAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDisablePeerAccess)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7875, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxDisablePeerAccess, __pyx_t_3) < 0))) __PYX_ERR(0, 7875, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7877 + * data["__cuCtxDisablePeerAccess"] = __cuCtxDisablePeerAccess + * global __cuDeviceGetP2PAttribute + * data["__cuDeviceGetP2PAttribute"] = __cuDeviceGetP2PAttribute # <<<<<<<<<<<<<< + * global __cuGraphicsUnregisterResource + * data["__cuGraphicsUnregisterResource"] = __cuGraphicsUnregisterResource +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetP2PAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7877, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetP2PAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7877, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7879 + * data["__cuDeviceGetP2PAttribute"] = __cuDeviceGetP2PAttribute + * global __cuGraphicsUnregisterResource + * data["__cuGraphicsUnregisterResource"] = __cuGraphicsUnregisterResource # <<<<<<<<<<<<<< + * global __cuGraphicsSubResourceGetMappedArray + * data["__cuGraphicsSubResourceGetMappedArray"] = __cuGraphicsSubResourceGetMappedArray +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnregisterResource)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7879, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphicsUnregisterResource, __pyx_t_3) < 0))) __PYX_ERR(0, 7879, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7881 + * data["__cuGraphicsUnregisterResource"] = __cuGraphicsUnregisterResource + * global __cuGraphicsSubResourceGetMappedArray + * data["__cuGraphicsSubResourceGetMappedArray"] = __cuGraphicsSubResourceGetMappedArray # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedMipmappedArray + * data["__cuGraphicsResourceGetMappedMipmappedArray"] = __cuGraphicsResourceGetMappedMipmappedArray +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsSubResourceGetMappedArray)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7881, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphicsSubResourceGetMapped, __pyx_t_3) < 0))) __PYX_ERR(0, 7881, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7883 + * data["__cuGraphicsSubResourceGetMappedArray"] = __cuGraphicsSubResourceGetMappedArray + * global __cuGraphicsResourceGetMappedMipmappedArray + * data["__cuGraphicsResourceGetMappedMipmappedArray"] = __cuGraphicsResourceGetMappedMipmappedArray # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedPointer_v2 + * data["__cuGraphicsResourceGetMappedPointer_v2"] = __cuGraphicsResourceGetMappedPointer_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedMipmappedArray)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7883, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphicsResourceGetMappedMip, __pyx_t_3) < 0))) __PYX_ERR(0, 7883, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7885 + * data["__cuGraphicsResourceGetMappedMipmappedArray"] = __cuGraphicsResourceGetMappedMipmappedArray + * global __cuGraphicsResourceGetMappedPointer_v2 + * data["__cuGraphicsResourceGetMappedPointer_v2"] = __cuGraphicsResourceGetMappedPointer_v2 # <<<<<<<<<<<<<< + * global __cuGraphicsResourceSetMapFlags_v2 + * data["__cuGraphicsResourceSetMapFlags_v2"] = __cuGraphicsResourceSetMapFlags_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedPointer_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7885, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphicsResourceGetMappedPoi, __pyx_t_3) < 0))) __PYX_ERR(0, 7885, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7887 + * data["__cuGraphicsResourceGetMappedPointer_v2"] = __cuGraphicsResourceGetMappedPointer_v2 + * global __cuGraphicsResourceSetMapFlags_v2 + * data["__cuGraphicsResourceSetMapFlags_v2"] = __cuGraphicsResourceSetMapFlags_v2 # <<<<<<<<<<<<<< + * global __cuGraphicsMapResources + * data["__cuGraphicsMapResources"] = __cuGraphicsMapResources +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceSetMapFlags_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7887, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphicsResourceSetMapFlags, __pyx_t_3) < 0))) __PYX_ERR(0, 7887, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7889 + * data["__cuGraphicsResourceSetMapFlags_v2"] = __cuGraphicsResourceSetMapFlags_v2 + * global __cuGraphicsMapResources + * data["__cuGraphicsMapResources"] = __cuGraphicsMapResources # <<<<<<<<<<<<<< + * global __cuGraphicsUnmapResources + * data["__cuGraphicsUnmapResources"] = __cuGraphicsUnmapResources +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsMapResources)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7889, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphicsMapResources, __pyx_t_3) < 0))) __PYX_ERR(0, 7889, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7891 + * data["__cuGraphicsMapResources"] = __cuGraphicsMapResources + * global __cuGraphicsUnmapResources + * data["__cuGraphicsUnmapResources"] = __cuGraphicsUnmapResources # <<<<<<<<<<<<<< + * global __cuGetProcAddress_v2 + * data["__cuGetProcAddress_v2"] = __cuGetProcAddress_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnmapResources)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7891, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphicsUnmapResources, __pyx_t_3) < 0))) __PYX_ERR(0, 7891, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7893 + * data["__cuGraphicsUnmapResources"] = __cuGraphicsUnmapResources + * global __cuGetProcAddress_v2 + * data["__cuGetProcAddress_v2"] = __cuGetProcAddress_v2 # <<<<<<<<<<<<<< + * global __cuCoredumpGetAttribute + * data["__cuCoredumpGetAttribute"] = __cuCoredumpGetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7893, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGetProcAddress_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7893, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7895 + * data["__cuGetProcAddress_v2"] = __cuGetProcAddress_v2 + * global __cuCoredumpGetAttribute + * data["__cuCoredumpGetAttribute"] = __cuCoredumpGetAttribute # <<<<<<<<<<<<<< + * global __cuCoredumpGetAttributeGlobal + * data["__cuCoredumpGetAttributeGlobal"] = __cuCoredumpGetAttributeGlobal +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7895, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCoredumpGetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7895, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7897 + * data["__cuCoredumpGetAttribute"] = __cuCoredumpGetAttribute + * global __cuCoredumpGetAttributeGlobal + * data["__cuCoredumpGetAttributeGlobal"] = __cuCoredumpGetAttributeGlobal # <<<<<<<<<<<<<< + * global __cuCoredumpSetAttribute + * data["__cuCoredumpSetAttribute"] = __cuCoredumpSetAttribute +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttributeGlobal)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7897, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCoredumpGetAttributeGlobal, __pyx_t_3) < 0))) __PYX_ERR(0, 7897, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7899 + * data["__cuCoredumpGetAttributeGlobal"] = __cuCoredumpGetAttributeGlobal + * global __cuCoredumpSetAttribute + * data["__cuCoredumpSetAttribute"] = __cuCoredumpSetAttribute # <<<<<<<<<<<<<< + * global __cuCoredumpSetAttributeGlobal + * data["__cuCoredumpSetAttributeGlobal"] = __cuCoredumpSetAttributeGlobal +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttribute)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7899, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCoredumpSetAttribute, __pyx_t_3) < 0))) __PYX_ERR(0, 7899, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7901 + * data["__cuCoredumpSetAttribute"] = __cuCoredumpSetAttribute + * global __cuCoredumpSetAttributeGlobal + * data["__cuCoredumpSetAttributeGlobal"] = __cuCoredumpSetAttributeGlobal # <<<<<<<<<<<<<< + * global __cuGetExportTable + * data["__cuGetExportTable"] = __cuGetExportTable +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttributeGlobal)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7901, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCoredumpSetAttributeGlobal, __pyx_t_3) < 0))) __PYX_ERR(0, 7901, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7903 + * data["__cuCoredumpSetAttributeGlobal"] = __cuCoredumpSetAttributeGlobal + * global __cuGetExportTable + * data["__cuGetExportTable"] = __cuGetExportTable # <<<<<<<<<<<<<< + * global __cuGreenCtxCreate + * data["__cuGreenCtxCreate"] = __cuGreenCtxCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetExportTable)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7903, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGetExportTable, __pyx_t_3) < 0))) __PYX_ERR(0, 7903, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7905 + * data["__cuGetExportTable"] = __cuGetExportTable + * global __cuGreenCtxCreate + * data["__cuGreenCtxCreate"] = __cuGreenCtxCreate # <<<<<<<<<<<<<< + * global __cuGreenCtxDestroy + * data["__cuGreenCtxDestroy"] = __cuGreenCtxDestroy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7905, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGreenCtxCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7905, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7907 + * data["__cuGreenCtxCreate"] = __cuGreenCtxCreate + * global __cuGreenCtxDestroy + * data["__cuGreenCtxDestroy"] = __cuGreenCtxDestroy # <<<<<<<<<<<<<< + * global __cuCtxFromGreenCtx + * data["__cuCtxFromGreenCtx"] = __cuCtxFromGreenCtx +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxDestroy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7907, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGreenCtxDestroy, __pyx_t_3) < 0))) __PYX_ERR(0, 7907, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7909 + * data["__cuGreenCtxDestroy"] = __cuGreenCtxDestroy + * global __cuCtxFromGreenCtx + * data["__cuCtxFromGreenCtx"] = __cuCtxFromGreenCtx # <<<<<<<<<<<<<< + * global __cuDeviceGetDevResource + * data["__cuDeviceGetDevResource"] = __cuDeviceGetDevResource +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxFromGreenCtx)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7909, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxFromGreenCtx, __pyx_t_3) < 0))) __PYX_ERR(0, 7909, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7911 + * data["__cuCtxFromGreenCtx"] = __cuCtxFromGreenCtx + * global __cuDeviceGetDevResource + * data["__cuDeviceGetDevResource"] = __cuDeviceGetDevResource # <<<<<<<<<<<<<< + * global __cuCtxGetDevResource + * data["__cuCtxGetDevResource"] = __cuCtxGetDevResource +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDevResource)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7911, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDeviceGetDevResource, __pyx_t_3) < 0))) __PYX_ERR(0, 7911, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7913 + * data["__cuDeviceGetDevResource"] = __cuDeviceGetDevResource + * global __cuCtxGetDevResource + * data["__cuCtxGetDevResource"] = __cuCtxGetDevResource # <<<<<<<<<<<<<< + * global __cuGreenCtxGetDevResource + * data["__cuGreenCtxGetDevResource"] = __cuGreenCtxGetDevResource +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevResource)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7913, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCtxGetDevResource, __pyx_t_3) < 0))) __PYX_ERR(0, 7913, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7915 + * data["__cuCtxGetDevResource"] = __cuCtxGetDevResource + * global __cuGreenCtxGetDevResource + * data["__cuGreenCtxGetDevResource"] = __cuGreenCtxGetDevResource # <<<<<<<<<<<<<< + * global __cuDevSmResourceSplitByCount + * data["__cuDevSmResourceSplitByCount"] = __cuDevSmResourceSplitByCount +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxGetDevResource)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7915, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGreenCtxGetDevResource, __pyx_t_3) < 0))) __PYX_ERR(0, 7915, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7917 + * data["__cuGreenCtxGetDevResource"] = __cuGreenCtxGetDevResource + * global __cuDevSmResourceSplitByCount + * data["__cuDevSmResourceSplitByCount"] = __cuDevSmResourceSplitByCount # <<<<<<<<<<<<<< + * global __cuDevResourceGenerateDesc + * data["__cuDevResourceGenerateDesc"] = __cuDevResourceGenerateDesc +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevSmResourceSplitByCount)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7917, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDevSmResourceSplitByCount, __pyx_t_3) < 0))) __PYX_ERR(0, 7917, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7919 + * data["__cuDevSmResourceSplitByCount"] = __cuDevSmResourceSplitByCount + * global __cuDevResourceGenerateDesc + * data["__cuDevResourceGenerateDesc"] = __cuDevResourceGenerateDesc # <<<<<<<<<<<<<< + * global __cuGreenCtxRecordEvent + * data["__cuGreenCtxRecordEvent"] = __cuGreenCtxRecordEvent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevResourceGenerateDesc)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7919, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuDevResourceGenerateDesc, __pyx_t_3) < 0))) __PYX_ERR(0, 7919, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7921 + * data["__cuDevResourceGenerateDesc"] = __cuDevResourceGenerateDesc + * global __cuGreenCtxRecordEvent + * data["__cuGreenCtxRecordEvent"] = __cuGreenCtxRecordEvent # <<<<<<<<<<<<<< + * global __cuGreenCtxWaitEvent + * data["__cuGreenCtxWaitEvent"] = __cuGreenCtxWaitEvent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxRecordEvent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7921, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGreenCtxRecordEvent, __pyx_t_3) < 0))) __PYX_ERR(0, 7921, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7923 + * data["__cuGreenCtxRecordEvent"] = __cuGreenCtxRecordEvent + * global __cuGreenCtxWaitEvent + * data["__cuGreenCtxWaitEvent"] = __cuGreenCtxWaitEvent # <<<<<<<<<<<<<< + * global __cuStreamGetGreenCtx + * data["__cuStreamGetGreenCtx"] = __cuStreamGetGreenCtx +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxWaitEvent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7923, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGreenCtxWaitEvent, __pyx_t_3) < 0))) __PYX_ERR(0, 7923, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7925 + * data["__cuGreenCtxWaitEvent"] = __cuGreenCtxWaitEvent + * global __cuStreamGetGreenCtx + * data["__cuStreamGetGreenCtx"] = __cuStreamGetGreenCtx # <<<<<<<<<<<<<< + * global __cuGreenCtxStreamCreate + * data["__cuGreenCtxStreamCreate"] = __cuGreenCtxStreamCreate +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetGreenCtx)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7925, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuStreamGetGreenCtx, __pyx_t_3) < 0))) __PYX_ERR(0, 7925, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7927 + * data["__cuStreamGetGreenCtx"] = __cuStreamGetGreenCtx + * global __cuGreenCtxStreamCreate + * data["__cuGreenCtxStreamCreate"] = __cuGreenCtxStreamCreate # <<<<<<<<<<<<<< + * global __cuLogsRegisterCallback + * data["__cuLogsRegisterCallback"] = __cuLogsRegisterCallback +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxStreamCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7927, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGreenCtxStreamCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 7927, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7929 + * data["__cuGreenCtxStreamCreate"] = __cuGreenCtxStreamCreate + * global __cuLogsRegisterCallback + * data["__cuLogsRegisterCallback"] = __cuLogsRegisterCallback # <<<<<<<<<<<<<< + * global __cuLogsUnregisterCallback + * data["__cuLogsUnregisterCallback"] = __cuLogsUnregisterCallback +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsRegisterCallback)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7929, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLogsRegisterCallback, __pyx_t_3) < 0))) __PYX_ERR(0, 7929, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7931 + * data["__cuLogsRegisterCallback"] = __cuLogsRegisterCallback + * global __cuLogsUnregisterCallback + * data["__cuLogsUnregisterCallback"] = __cuLogsUnregisterCallback # <<<<<<<<<<<<<< + * global __cuLogsCurrent + * data["__cuLogsCurrent"] = __cuLogsCurrent +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsUnregisterCallback)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7931, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLogsUnregisterCallback, __pyx_t_3) < 0))) __PYX_ERR(0, 7931, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7933 + * data["__cuLogsUnregisterCallback"] = __cuLogsUnregisterCallback + * global __cuLogsCurrent + * data["__cuLogsCurrent"] = __cuLogsCurrent # <<<<<<<<<<<<<< + * global __cuLogsDumpToFile + * data["__cuLogsDumpToFile"] = __cuLogsDumpToFile +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsCurrent)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7933, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLogsCurrent, __pyx_t_3) < 0))) __PYX_ERR(0, 7933, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7935 + * data["__cuLogsCurrent"] = __cuLogsCurrent + * global __cuLogsDumpToFile + * data["__cuLogsDumpToFile"] = __cuLogsDumpToFile # <<<<<<<<<<<<<< + * global __cuLogsDumpToMemory + * data["__cuLogsDumpToMemory"] = __cuLogsDumpToMemory +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToFile)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7935, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLogsDumpToFile, __pyx_t_3) < 0))) __PYX_ERR(0, 7935, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7937 + * data["__cuLogsDumpToFile"] = __cuLogsDumpToFile + * global __cuLogsDumpToMemory + * data["__cuLogsDumpToMemory"] = __cuLogsDumpToMemory # <<<<<<<<<<<<<< + * global __cuCheckpointProcessGetRestoreThreadId + * data["__cuCheckpointProcessGetRestoreThreadId"] = __cuCheckpointProcessGetRestoreThreadId +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToMemory)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7937, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuLogsDumpToMemory, __pyx_t_3) < 0))) __PYX_ERR(0, 7937, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7939 + * data["__cuLogsDumpToMemory"] = __cuLogsDumpToMemory + * global __cuCheckpointProcessGetRestoreThreadId + * data["__cuCheckpointProcessGetRestoreThreadId"] = __cuCheckpointProcessGetRestoreThreadId # <<<<<<<<<<<<<< + * global __cuCheckpointProcessGetState + * data["__cuCheckpointProcessGetState"] = __cuCheckpointProcessGetState +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetRestoreThreadId)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7939, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCheckpointProcessGetRestoreT, __pyx_t_3) < 0))) __PYX_ERR(0, 7939, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7941 + * data["__cuCheckpointProcessGetRestoreThreadId"] = __cuCheckpointProcessGetRestoreThreadId + * global __cuCheckpointProcessGetState + * data["__cuCheckpointProcessGetState"] = __cuCheckpointProcessGetState # <<<<<<<<<<<<<< + * global __cuCheckpointProcessLock + * data["__cuCheckpointProcessLock"] = __cuCheckpointProcessLock +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetState)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7941, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCheckpointProcessGetState, __pyx_t_3) < 0))) __PYX_ERR(0, 7941, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7943 + * data["__cuCheckpointProcessGetState"] = __cuCheckpointProcessGetState + * global __cuCheckpointProcessLock + * data["__cuCheckpointProcessLock"] = __cuCheckpointProcessLock # <<<<<<<<<<<<<< + * global __cuCheckpointProcessCheckpoint + * data["__cuCheckpointProcessCheckpoint"] = __cuCheckpointProcessCheckpoint +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessLock)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7943, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCheckpointProcessLock, __pyx_t_3) < 0))) __PYX_ERR(0, 7943, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7945 + * data["__cuCheckpointProcessLock"] = __cuCheckpointProcessLock + * global __cuCheckpointProcessCheckpoint + * data["__cuCheckpointProcessCheckpoint"] = __cuCheckpointProcessCheckpoint # <<<<<<<<<<<<<< + * global __cuCheckpointProcessRestore + * data["__cuCheckpointProcessRestore"] = __cuCheckpointProcessRestore +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessCheckpoint)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7945, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCheckpointProcessCheckpoint, __pyx_t_3) < 0))) __PYX_ERR(0, 7945, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7947 + * data["__cuCheckpointProcessCheckpoint"] = __cuCheckpointProcessCheckpoint + * global __cuCheckpointProcessRestore + * data["__cuCheckpointProcessRestore"] = __cuCheckpointProcessRestore # <<<<<<<<<<<<<< + * global __cuCheckpointProcessUnlock + * data["__cuCheckpointProcessUnlock"] = __cuCheckpointProcessUnlock +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessRestore)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7947, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCheckpointProcessRestore, __pyx_t_3) < 0))) __PYX_ERR(0, 7947, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7949 + * data["__cuCheckpointProcessRestore"] = __cuCheckpointProcessRestore + * global __cuCheckpointProcessUnlock + * data["__cuCheckpointProcessUnlock"] = __cuCheckpointProcessUnlock # <<<<<<<<<<<<<< + * global __cuProfilerStart + * data["__cuProfilerStart"] = __cuProfilerStart +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessUnlock)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7949, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuCheckpointProcessUnlock, __pyx_t_3) < 0))) __PYX_ERR(0, 7949, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7951 + * data["__cuCheckpointProcessUnlock"] = __cuCheckpointProcessUnlock + * global __cuProfilerStart + * data["__cuProfilerStart"] = __cuProfilerStart # <<<<<<<<<<<<<< + * global __cuProfilerStop + * data["__cuProfilerStop"] = __cuProfilerStop +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStart)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7951, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuProfilerStart, __pyx_t_3) < 0))) __PYX_ERR(0, 7951, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7953 + * data["__cuProfilerStart"] = __cuProfilerStart + * global __cuProfilerStop + * data["__cuProfilerStop"] = __cuProfilerStop # <<<<<<<<<<<<<< + * global __cuGraphicsEGLRegisterImage + * data["__cuGraphicsEGLRegisterImage"] = __cuGraphicsEGLRegisterImage +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStop)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7953, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuProfilerStop, __pyx_t_3) < 0))) __PYX_ERR(0, 7953, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7955 + * data["__cuProfilerStop"] = __cuProfilerStop + * global __cuGraphicsEGLRegisterImage + * data["__cuGraphicsEGLRegisterImage"] = __cuGraphicsEGLRegisterImage # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerConnect + * data["__cuEGLStreamConsumerConnect"] = __cuEGLStreamConsumerConnect +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsEGLRegisterImage)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7955, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphicsEGLRegisterImage, __pyx_t_3) < 0))) __PYX_ERR(0, 7955, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7957 + * data["__cuGraphicsEGLRegisterImage"] = __cuGraphicsEGLRegisterImage + * global __cuEGLStreamConsumerConnect + * data["__cuEGLStreamConsumerConnect"] = __cuEGLStreamConsumerConnect # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerConnectWithFlags + * data["__cuEGLStreamConsumerConnectWithFlags"] = __cuEGLStreamConsumerConnectWithFlags +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnect)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7957, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEGLStreamConsumerConnect, __pyx_t_3) < 0))) __PYX_ERR(0, 7957, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7959 + * data["__cuEGLStreamConsumerConnect"] = __cuEGLStreamConsumerConnect + * global __cuEGLStreamConsumerConnectWithFlags + * data["__cuEGLStreamConsumerConnectWithFlags"] = __cuEGLStreamConsumerConnectWithFlags # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerDisconnect + * data["__cuEGLStreamConsumerDisconnect"] = __cuEGLStreamConsumerDisconnect +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnectWithFlags)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7959, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEGLStreamConsumerConnectWith, __pyx_t_3) < 0))) __PYX_ERR(0, 7959, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7961 + * data["__cuEGLStreamConsumerConnectWithFlags"] = __cuEGLStreamConsumerConnectWithFlags + * global __cuEGLStreamConsumerDisconnect + * data["__cuEGLStreamConsumerDisconnect"] = __cuEGLStreamConsumerDisconnect # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerAcquireFrame + * data["__cuEGLStreamConsumerAcquireFrame"] = __cuEGLStreamConsumerAcquireFrame +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerDisconnect)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7961, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEGLStreamConsumerDisconnect, __pyx_t_3) < 0))) __PYX_ERR(0, 7961, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7963 + * data["__cuEGLStreamConsumerDisconnect"] = __cuEGLStreamConsumerDisconnect + * global __cuEGLStreamConsumerAcquireFrame + * data["__cuEGLStreamConsumerAcquireFrame"] = __cuEGLStreamConsumerAcquireFrame # <<<<<<<<<<<<<< + * global __cuEGLStreamConsumerReleaseFrame + * data["__cuEGLStreamConsumerReleaseFrame"] = __cuEGLStreamConsumerReleaseFrame +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerAcquireFrame)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7963, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEGLStreamConsumerAcquireFram, __pyx_t_3) < 0))) __PYX_ERR(0, 7963, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7965 + * data["__cuEGLStreamConsumerAcquireFrame"] = __cuEGLStreamConsumerAcquireFrame + * global __cuEGLStreamConsumerReleaseFrame + * data["__cuEGLStreamConsumerReleaseFrame"] = __cuEGLStreamConsumerReleaseFrame # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerConnect + * data["__cuEGLStreamProducerConnect"] = __cuEGLStreamProducerConnect +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerReleaseFrame)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7965, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEGLStreamConsumerReleaseFram, __pyx_t_3) < 0))) __PYX_ERR(0, 7965, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7967 + * data["__cuEGLStreamConsumerReleaseFrame"] = __cuEGLStreamConsumerReleaseFrame + * global __cuEGLStreamProducerConnect + * data["__cuEGLStreamProducerConnect"] = __cuEGLStreamProducerConnect # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerDisconnect + * data["__cuEGLStreamProducerDisconnect"] = __cuEGLStreamProducerDisconnect +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerConnect)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7967, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEGLStreamProducerConnect, __pyx_t_3) < 0))) __PYX_ERR(0, 7967, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7969 + * data["__cuEGLStreamProducerConnect"] = __cuEGLStreamProducerConnect + * global __cuEGLStreamProducerDisconnect + * data["__cuEGLStreamProducerDisconnect"] = __cuEGLStreamProducerDisconnect # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerPresentFrame + * data["__cuEGLStreamProducerPresentFrame"] = __cuEGLStreamProducerPresentFrame +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerDisconnect)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7969, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEGLStreamProducerDisconnect, __pyx_t_3) < 0))) __PYX_ERR(0, 7969, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7971 + * data["__cuEGLStreamProducerDisconnect"] = __cuEGLStreamProducerDisconnect + * global __cuEGLStreamProducerPresentFrame + * data["__cuEGLStreamProducerPresentFrame"] = __cuEGLStreamProducerPresentFrame # <<<<<<<<<<<<<< + * global __cuEGLStreamProducerReturnFrame + * data["__cuEGLStreamProducerReturnFrame"] = __cuEGLStreamProducerReturnFrame +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerPresentFrame)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7971, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEGLStreamProducerPresentFram, __pyx_t_3) < 0))) __PYX_ERR(0, 7971, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7973 + * data["__cuEGLStreamProducerPresentFrame"] = __cuEGLStreamProducerPresentFrame + * global __cuEGLStreamProducerReturnFrame + * data["__cuEGLStreamProducerReturnFrame"] = __cuEGLStreamProducerReturnFrame # <<<<<<<<<<<<<< + * global __cuGraphicsResourceGetMappedEglFrame + * data["__cuGraphicsResourceGetMappedEglFrame"] = __cuGraphicsResourceGetMappedEglFrame +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerReturnFrame)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7973, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEGLStreamProducerReturnFrame, __pyx_t_3) < 0))) __PYX_ERR(0, 7973, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7975 + * data["__cuEGLStreamProducerReturnFrame"] = __cuEGLStreamProducerReturnFrame + * global __cuGraphicsResourceGetMappedEglFrame + * data["__cuGraphicsResourceGetMappedEglFrame"] = __cuGraphicsResourceGetMappedEglFrame # <<<<<<<<<<<<<< + * global __cuEventCreateFromEGLSync + * data["__cuEventCreateFromEGLSync"] = __cuEventCreateFromEGLSync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedEglFrame)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7975, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphicsResourceGetMappedEgl, __pyx_t_3) < 0))) __PYX_ERR(0, 7975, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7977 + * data["__cuGraphicsResourceGetMappedEglFrame"] = __cuGraphicsResourceGetMappedEglFrame + * global __cuEventCreateFromEGLSync + * data["__cuEventCreateFromEGLSync"] = __cuEventCreateFromEGLSync # <<<<<<<<<<<<<< + * global __cuGraphicsGLRegisterBuffer + * data["__cuGraphicsGLRegisterBuffer"] = __cuGraphicsGLRegisterBuffer +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreateFromEGLSync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7977, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuEventCreateFromEGLSync, __pyx_t_3) < 0))) __PYX_ERR(0, 7977, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7979 + * data["__cuEventCreateFromEGLSync"] = __cuEventCreateFromEGLSync + * global __cuGraphicsGLRegisterBuffer + * data["__cuGraphicsGLRegisterBuffer"] = __cuGraphicsGLRegisterBuffer # <<<<<<<<<<<<<< + * global __cuGraphicsGLRegisterImage + * data["__cuGraphicsGLRegisterImage"] = __cuGraphicsGLRegisterImage +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterBuffer)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7979, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphicsGLRegisterBuffer, __pyx_t_3) < 0))) __PYX_ERR(0, 7979, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7981 + * data["__cuGraphicsGLRegisterBuffer"] = __cuGraphicsGLRegisterBuffer + * global __cuGraphicsGLRegisterImage + * data["__cuGraphicsGLRegisterImage"] = __cuGraphicsGLRegisterImage # <<<<<<<<<<<<<< + * global __cuGLGetDevices_v2 + * data["__cuGLGetDevices_v2"] = __cuGLGetDevices_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterImage)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7981, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphicsGLRegisterImage, __pyx_t_3) < 0))) __PYX_ERR(0, 7981, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7983 + * data["__cuGraphicsGLRegisterImage"] = __cuGraphicsGLRegisterImage + * global __cuGLGetDevices_v2 + * data["__cuGLGetDevices_v2"] = __cuGLGetDevices_v2 # <<<<<<<<<<<<<< + * global __cuVDPAUGetDevice + * data["__cuVDPAUGetDevice"] = __cuVDPAUGetDevice +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGLGetDevices_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7983, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGLGetDevices_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7983, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7985 + * data["__cuGLGetDevices_v2"] = __cuGLGetDevices_v2 + * global __cuVDPAUGetDevice + * data["__cuVDPAUGetDevice"] = __cuVDPAUGetDevice # <<<<<<<<<<<<<< + * global __cuVDPAUCtxCreate_v2 + * data["__cuVDPAUCtxCreate_v2"] = __cuVDPAUCtxCreate_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUGetDevice)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7985, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuVDPAUGetDevice, __pyx_t_3) < 0))) __PYX_ERR(0, 7985, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7987 + * data["__cuVDPAUGetDevice"] = __cuVDPAUGetDevice + * global __cuVDPAUCtxCreate_v2 + * data["__cuVDPAUCtxCreate_v2"] = __cuVDPAUCtxCreate_v2 # <<<<<<<<<<<<<< + * global __cuGraphicsVDPAURegisterVideoSurface + * data["__cuGraphicsVDPAURegisterVideoSurface"] = __cuGraphicsVDPAURegisterVideoSurface +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUCtxCreate_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7987, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuVDPAUCtxCreate_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 7987, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7989 + * data["__cuVDPAUCtxCreate_v2"] = __cuVDPAUCtxCreate_v2 + * global __cuGraphicsVDPAURegisterVideoSurface + * data["__cuGraphicsVDPAURegisterVideoSurface"] = __cuGraphicsVDPAURegisterVideoSurface # <<<<<<<<<<<<<< + * global __cuGraphicsVDPAURegisterOutputSurface + * data["__cuGraphicsVDPAURegisterOutputSurface"] = __cuGraphicsVDPAURegisterOutputSurface +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterVideoSurface)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7989, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphicsVDPAURegisterVideoSu, __pyx_t_3) < 0))) __PYX_ERR(0, 7989, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7991 + * data["__cuGraphicsVDPAURegisterVideoSurface"] = __cuGraphicsVDPAURegisterVideoSurface + * global __cuGraphicsVDPAURegisterOutputSurface + * data["__cuGraphicsVDPAURegisterOutputSurface"] = __cuGraphicsVDPAURegisterOutputSurface # <<<<<<<<<<<<<< + * + * func_ptrs = data +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterOutputSurface)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 7991, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuGraphicsVDPAURegisterOutputS, __pyx_t_3) < 0))) __PYX_ERR(0, 7991, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7993 + * data["__cuGraphicsVDPAURegisterOutputSurface"] = __cuGraphicsVDPAURegisterOutputSurface + * + * func_ptrs = data # <<<<<<<<<<<<<< + * return data + * +*/ + __Pyx_INCREF(__pyx_v_data); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs, __pyx_v_data); + __Pyx_GIVEREF(__pyx_v_data); + + /* "cuda/bindings/_bindings/cydriver.pyx":7994 + * + * func_ptrs = data + * return data # <<<<<<<<<<<<<< + * + * cpdef _inspect_function_pointer(str name): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_data); + __pyx_r = __pyx_v_data; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7038 + * cdef dict func_ptrs = None + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._inspect_function_pointers", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_data); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_4cuda_8bindings_9_bindings_8cydriver_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +PyDoc_STRVAR(__pyx_doc_4cuda_8bindings_9_bindings_8cydriver__inspect_function_pointers, "_inspect_function_pointers() -> dict"); +static PyMethodDef __pyx_mdef_4cuda_8bindings_9_bindings_8cydriver_1_inspect_function_pointers = {"_inspect_function_pointers", (PyCFunction)__pyx_pw_4cuda_8bindings_9_bindings_8cydriver_1_inspect_function_pointers, METH_NOARGS, __pyx_doc_4cuda_8bindings_9_bindings_8cydriver__inspect_function_pointers}; +static PyObject *__pyx_pw_4cuda_8bindings_9_bindings_8cydriver_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_inspect_function_pointers (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_4cuda_8bindings_9_bindings_8cydriver__inspect_function_pointers(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_4cuda_8bindings_9_bindings_8cydriver__inspect_function_pointers(CYTHON_UNUSED PyObject *__pyx_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointers", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__inspect_function_pointers(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 7038, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._inspect_function_pointers", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cydriver.pyx":7996 + * return data + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + +static PyObject *__pyx_pw_4cuda_8bindings_9_bindings_8cydriver_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_bindings_8cydriver__inspect_function_pointer(PyObject *__pyx_v_name, CYTHON_UNUSED int __pyx_skip_dispatch) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointer", 0); + + /* "cuda/bindings/_bindings/cydriver.pyx":7998 + * cpdef _inspect_function_pointer(str name): + * global func_ptrs + * if func_ptrs is None: # <<<<<<<<<<<<<< + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] +*/ + __pyx_t_1 = (__pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs == ((PyObject*)Py_None)); + if (__pyx_t_1) { + + /* "cuda/bindings/_bindings/cydriver.pyx":7999 + * global func_ptrs + * if func_ptrs is None: + * func_ptrs = _inspect_function_pointers() # <<<<<<<<<<<<<< + * return func_ptrs[name] +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__inspect_function_pointers(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 7999, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs, ((PyObject*)__pyx_t_2)); + __Pyx_GIVEREF(__pyx_t_2); + __pyx_t_2 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7998 + * cpdef _inspect_function_pointer(str name): + * global func_ptrs + * if func_ptrs is None: # <<<<<<<<<<<<<< + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] +*/ + } + + /* "cuda/bindings/_bindings/cydriver.pyx":8000 + * if func_ptrs is None: + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] # <<<<<<<<<<<<<< +*/ + __Pyx_XDECREF(__pyx_r); + if (unlikely(__pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 8000, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_PyDict_GetItem(__pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs, __pyx_v_name); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 8000, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7996 + * return data + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_4cuda_8bindings_9_bindings_8cydriver_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_4cuda_8bindings_9_bindings_8cydriver_2_inspect_function_pointer, "_inspect_function_pointer(str name)"); +static PyMethodDef __pyx_mdef_4cuda_8bindings_9_bindings_8cydriver_3_inspect_function_pointer = {"_inspect_function_pointer", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cuda_8bindings_9_bindings_8cydriver_3_inspect_function_pointer, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cuda_8bindings_9_bindings_8cydriver_2_inspect_function_pointer}; +static PyObject *__pyx_pw_4cuda_8bindings_9_bindings_8cydriver_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_name = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_inspect_function_pointer (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_name,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 7996, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 7996, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_inspect_function_pointer", 0) < (0)) __PYX_ERR(0, 7996, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_inspect_function_pointer", 1, 1, 1, i); __PYX_ERR(0, 7996, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 7996, __pyx_L3_error) + } + __pyx_v_name = ((PyObject*)values[0]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_inspect_function_pointer", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 7996, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_name), (&PyUnicode_Type), 1, "name", 1))) __PYX_ERR(0, 7996, __pyx_L1_error) + __pyx_r = __pyx_pf_4cuda_8bindings_9_bindings_8cydriver_2_inspect_function_pointer(__pyx_self, __pyx_v_name); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_4cuda_8bindings_9_bindings_8cydriver_2_inspect_function_pointer(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_name) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointer", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__inspect_function_pointer(__pyx_v_name, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 7996, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cuda.bindings._bindings.cydriver._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +/* #### Code section: module_exttypes ### */ + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___symbol_lock = Py_None; Py_INCREF(Py_None); + __pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs = ((PyObject*)Py_None); Py_INCREF(Py_None); + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + if (__Pyx_ExportFunction("_cuGetErrorString", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGetErrorString, "CUresult (CUresult, char const **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGetErrorName", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGetErrorName, "CUresult (CUresult, char const **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuInit", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuInit, "CUresult (unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDriverGetVersion", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDriverGetVersion, "CUresult (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGet", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGet, "CUresult (CUdevice *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetCount", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetCount, "CUresult (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetName", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetName, "CUresult (char *, int, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetUuid", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetUuid, "CUresult (CUuuid *, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetUuid_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetUuid_v2, "CUresult (CUuuid *, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetLuid", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetLuid, "CUresult (char *, unsigned int *, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceTotalMem_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceTotalMem_v2, "CUresult (size_t *, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetTexture1DLinearMaxWidth", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetTexture1DLinearMaxWidth, "CUresult (size_t *, CUarray_format, unsigned int, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetAttribute, "CUresult (int *, CUdevice_attribute, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetNvSciSyncAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetNvSciSyncAttributes, "CUresult (void *, CUdevice, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceSetMemPool", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceSetMemPool, "CUresult (CUdevice, CUmemoryPool)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetMemPool", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetMemPool, "CUresult (CUmemoryPool *, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetDefaultMemPool", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetDefaultMemPool, "CUresult (CUmemoryPool *, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetExecAffinitySupport", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetExecAffinitySupport, "CUresult (int *, CUexecAffinityType, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFlushGPUDirectRDMAWrites", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFlushGPUDirectRDMAWrites, "CUresult (CUflushGPUDirectRDMAWritesTarget, CUflushGPUDirectRDMAWritesScope)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetProperties", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetProperties, "CUresult (CUdevprop *, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceComputeCapability", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceComputeCapability, "CUresult (int *, int *, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDevicePrimaryCtxRetain", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevicePrimaryCtxRetain, "CUresult (CUcontext *, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDevicePrimaryCtxRelease_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevicePrimaryCtxRelease_v2, "CUresult (CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDevicePrimaryCtxSetFlags_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevicePrimaryCtxSetFlags_v2, "CUresult (CUdevice, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDevicePrimaryCtxGetState", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevicePrimaryCtxGetState, "CUresult (CUdevice, unsigned int *, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDevicePrimaryCtxReset_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevicePrimaryCtxReset_v2, "CUresult (CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxCreate_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxCreate_v2, "CUresult (CUcontext *, unsigned int, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxCreate_v3", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxCreate_v3, "CUresult (CUcontext *, CUexecAffinityParam *, int, unsigned int, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxCreate_v4", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxCreate_v4, "CUresult (CUcontext *, CUctxCreateParams *, unsigned int, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxDestroy_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxDestroy_v2, "CUresult (CUcontext)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxPushCurrent_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxPushCurrent_v2, "CUresult (CUcontext)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxPopCurrent_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxPopCurrent_v2, "CUresult (CUcontext *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxSetCurrent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxSetCurrent, "CUresult (CUcontext)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxGetCurrent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetCurrent, "CUresult (CUcontext *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxGetDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetDevice, "CUresult (CUdevice *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxGetFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetFlags, "CUresult (unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxSetFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxSetFlags, "CUresult (unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxGetId", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetId, "CUresult (CUcontext, unsigned PY_LONG_LONG *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxSynchronize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxSynchronize, "CUresult (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxSetLimit", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxSetLimit, "CUresult (CUlimit, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxGetLimit", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetLimit, "CUresult (size_t *, CUlimit)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxGetCacheConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetCacheConfig, "CUresult (CUfunc_cache *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxSetCacheConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxSetCacheConfig, "CUresult (CUfunc_cache)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxGetApiVersion", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetApiVersion, "CUresult (CUcontext, unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxGetStreamPriorityRange", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetStreamPriorityRange, "CUresult (int *, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxResetPersistingL2Cache", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxResetPersistingL2Cache, "CUresult (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxGetExecAffinity", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetExecAffinity, "CUresult (CUexecAffinityParam *, CUexecAffinityType)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxRecordEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxRecordEvent, "CUresult (CUcontext, CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxWaitEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxWaitEvent, "CUresult (CUcontext, CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxAttach", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxAttach, "CUresult (CUcontext *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxDetach", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxDetach, "CUresult (CUcontext)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxGetSharedMemConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetSharedMemConfig, "CUresult (CUsharedconfig *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxSetSharedMemConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxSetSharedMemConfig, "CUresult (CUsharedconfig)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuModuleLoad", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleLoad, "CUresult (CUmodule *, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuModuleLoadData", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleLoadData, "CUresult (CUmodule *, void const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuModuleLoadDataEx", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleLoadDataEx, "CUresult (CUmodule *, void const *, unsigned int, CUjit_option *, void **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuModuleLoadFatBinary", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleLoadFatBinary, "CUresult (CUmodule *, void const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuModuleUnload", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleUnload, "CUresult (CUmodule)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuModuleGetLoadingMode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleGetLoadingMode, "CUresult (CUmoduleLoadingMode *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuModuleGetFunction", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleGetFunction, "CUresult (CUfunction *, CUmodule, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuModuleGetFunctionCount", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleGetFunctionCount, "CUresult (unsigned int *, CUmodule)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuModuleEnumerateFunctions", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleEnumerateFunctions, "CUresult (CUfunction *, unsigned int, CUmodule)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuModuleGetGlobal_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleGetGlobal_v2, "CUresult (CUdeviceptr *, size_t *, CUmodule, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLinkCreate_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLinkCreate_v2, "CUresult (unsigned int, CUjit_option *, void **, CUlinkState *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLinkAddData_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLinkAddData_v2, "CUresult (CUlinkState, CUjitInputType, void *, size_t, char const *, unsigned int, CUjit_option *, void **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLinkAddFile_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLinkAddFile_v2, "CUresult (CUlinkState, CUjitInputType, char const *, unsigned int, CUjit_option *, void **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLinkComplete", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLinkComplete, "CUresult (CUlinkState, void **, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLinkDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLinkDestroy, "CUresult (CUlinkState)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuModuleGetTexRef", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleGetTexRef, "CUresult (CUtexref *, CUmodule, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuModuleGetSurfRef", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuModuleGetSurfRef, "CUresult (CUsurfref *, CUmodule, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLibraryLoadData", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryLoadData, "CUresult (CUlibrary *, void const *, CUjit_option *, void **, unsigned int, CUlibraryOption *, void **, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLibraryLoadFromFile", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryLoadFromFile, "CUresult (CUlibrary *, char const *, CUjit_option *, void **, unsigned int, CUlibraryOption *, void **, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLibraryUnload", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryUnload, "CUresult (CUlibrary)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLibraryGetKernel", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryGetKernel, "CUresult (CUkernel *, CUlibrary, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLibraryGetKernelCount", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryGetKernelCount, "CUresult (unsigned int *, CUlibrary)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLibraryEnumerateKernels", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryEnumerateKernels, "CUresult (CUkernel *, unsigned int, CUlibrary)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLibraryGetModule", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryGetModule, "CUresult (CUmodule *, CUlibrary)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuKernelGetFunction", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelGetFunction, "CUresult (CUfunction *, CUkernel)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuKernelGetLibrary", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelGetLibrary, "CUresult (CUlibrary *, CUkernel)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLibraryGetGlobal", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryGetGlobal, "CUresult (CUdeviceptr *, size_t *, CUlibrary, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLibraryGetManaged", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryGetManaged, "CUresult (CUdeviceptr *, size_t *, CUlibrary, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLibraryGetUnifiedFunction", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLibraryGetUnifiedFunction, "CUresult (void **, CUlibrary, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuKernelGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelGetAttribute, "CUresult (int *, CUfunction_attribute, CUkernel, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuKernelSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelSetAttribute, "CUresult (CUfunction_attribute, int, CUkernel, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuKernelSetCacheConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelSetCacheConfig, "CUresult (CUkernel, CUfunc_cache, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuKernelGetName", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelGetName, "CUresult (char const **, CUkernel)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuKernelGetParamInfo", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuKernelGetParamInfo, "CUresult (CUkernel, size_t, size_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemGetInfo_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemGetInfo_v2, "CUresult (size_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemAlloc_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAlloc_v2, "CUresult (CUdeviceptr *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemAllocPitch_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAllocPitch_v2, "CUresult (CUdeviceptr *, size_t *, size_t, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemFree_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemFree_v2, "CUresult (CUdeviceptr)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemGetAddressRange_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemGetAddressRange_v2, "CUresult (CUdeviceptr *, size_t *, CUdeviceptr)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemAllocHost_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAllocHost_v2, "CUresult (void **, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemFreeHost", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemFreeHost, "CUresult (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemHostAlloc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemHostAlloc, "CUresult (void **, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemHostGetDevicePointer_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemHostGetDevicePointer_v2, "CUresult (CUdeviceptr *, void *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemHostGetFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemHostGetFlags, "CUresult (unsigned int *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemAllocManaged", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAllocManaged, "CUresult (CUdeviceptr *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceRegisterAsyncNotification", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceRegisterAsyncNotification, "CUresult (CUdevice, CUasyncCallback, void *, CUasyncCallbackHandle *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceUnregisterAsyncNotification", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceUnregisterAsyncNotification, "CUresult (CUdevice, CUasyncCallbackHandle)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetByPCIBusId", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetByPCIBusId, "CUresult (CUdevice *, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetPCIBusId", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetPCIBusId, "CUresult (char *, int, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuIpcGetEventHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuIpcGetEventHandle, "CUresult (CUipcEventHandle *, CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuIpcOpenEventHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuIpcOpenEventHandle, "CUresult (CUevent *, CUipcEventHandle)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuIpcGetMemHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuIpcGetMemHandle, "CUresult (CUipcMemHandle *, CUdeviceptr)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuIpcOpenMemHandle_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuIpcOpenMemHandle_v2, "CUresult (CUdeviceptr *, CUipcMemHandle, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuIpcCloseMemHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuIpcCloseMemHandle, "CUresult (CUdeviceptr)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemHostRegister_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemHostRegister_v2, "CUresult (void *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemHostUnregister", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemHostUnregister, "CUresult (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy, "CUresult (CUdeviceptr, CUdeviceptr, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyPeer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyPeer, "CUresult (CUdeviceptr, CUcontext, CUdeviceptr, CUcontext, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyHtoD_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyHtoD_v2, "CUresult (CUdeviceptr, void const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyDtoH_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyDtoH_v2, "CUresult (void *, CUdeviceptr, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyDtoD_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyDtoD_v2, "CUresult (CUdeviceptr, CUdeviceptr, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyDtoA_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyDtoA_v2, "CUresult (CUarray, size_t, CUdeviceptr, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyAtoD_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyAtoD_v2, "CUresult (CUdeviceptr, CUarray, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyHtoA_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyHtoA_v2, "CUresult (CUarray, size_t, void const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyAtoH_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyAtoH_v2, "CUresult (void *, CUarray, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyAtoA_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyAtoA_v2, "CUresult (CUarray, size_t, CUarray, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpy2D_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy2D_v2, "CUresult (CUDA_MEMCPY2D const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpy2DUnaligned_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy2DUnaligned_v2, "CUresult (CUDA_MEMCPY2D const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpy3D_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy3D_v2, "CUresult (CUDA_MEMCPY3D const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpy3DPeer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy3DPeer, "CUresult (CUDA_MEMCPY3D_PEER const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyAsync, "CUresult (CUdeviceptr, CUdeviceptr, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyPeerAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyPeerAsync, "CUresult (CUdeviceptr, CUcontext, CUdeviceptr, CUcontext, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyHtoDAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyHtoDAsync_v2, "CUresult (CUdeviceptr, void const *, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyDtoHAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyDtoHAsync_v2, "CUresult (void *, CUdeviceptr, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyDtoDAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyDtoDAsync_v2, "CUresult (CUdeviceptr, CUdeviceptr, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyHtoAAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyHtoAAsync_v2, "CUresult (CUarray, size_t, void const *, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyAtoHAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyAtoHAsync_v2, "CUresult (void *, CUarray, size_t, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpy2DAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy2DAsync_v2, "CUresult (CUDA_MEMCPY2D const *, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpy3DAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy3DAsync_v2, "CUresult (CUDA_MEMCPY3D const *, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpy3DPeerAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy3DPeerAsync, "CUresult (CUDA_MEMCPY3D_PEER const *, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpyBatchAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpyBatchAsync, "CUresult (CUdeviceptr *, CUdeviceptr *, size_t *, size_t, CUmemcpyAttributes *, size_t *, size_t, size_t *, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemcpy3DBatchAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemcpy3DBatchAsync, "CUresult (size_t, CUDA_MEMCPY3D_BATCH_OP *, size_t *, unsigned PY_LONG_LONG, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemsetD8_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD8_v2, "CUresult (CUdeviceptr, unsigned char, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemsetD16_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD16_v2, "CUresult (CUdeviceptr, unsigned short, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemsetD32_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD32_v2, "CUresult (CUdeviceptr, unsigned int, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemsetD2D8_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD2D8_v2, "CUresult (CUdeviceptr, size_t, unsigned char, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemsetD2D16_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD2D16_v2, "CUresult (CUdeviceptr, size_t, unsigned short, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemsetD2D32_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD2D32_v2, "CUresult (CUdeviceptr, size_t, unsigned int, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemsetD8Async", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD8Async, "CUresult (CUdeviceptr, unsigned char, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemsetD16Async", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD16Async, "CUresult (CUdeviceptr, unsigned short, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemsetD32Async", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD32Async, "CUresult (CUdeviceptr, unsigned int, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemsetD2D8Async", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD2D8Async, "CUresult (CUdeviceptr, size_t, unsigned char, size_t, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemsetD2D16Async", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD2D16Async, "CUresult (CUdeviceptr, size_t, unsigned short, size_t, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemsetD2D32Async", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemsetD2D32Async, "CUresult (CUdeviceptr, size_t, unsigned int, size_t, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuArrayCreate_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArrayCreate_v2, "CUresult (CUarray *, CUDA_ARRAY_DESCRIPTOR const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuArrayGetDescriptor_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArrayGetDescriptor_v2, "CUresult (CUDA_ARRAY_DESCRIPTOR *, CUarray)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuArrayGetSparseProperties", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArrayGetSparseProperties, "CUresult (CUDA_ARRAY_SPARSE_PROPERTIES *, CUarray)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMipmappedArrayGetSparseProperties", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMipmappedArrayGetSparseProperties, "CUresult (CUDA_ARRAY_SPARSE_PROPERTIES *, CUmipmappedArray)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuArrayGetMemoryRequirements", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArrayGetMemoryRequirements, "CUresult (CUDA_ARRAY_MEMORY_REQUIREMENTS *, CUarray, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMipmappedArrayGetMemoryRequirements", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMipmappedArrayGetMemoryRequirements, "CUresult (CUDA_ARRAY_MEMORY_REQUIREMENTS *, CUmipmappedArray, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuArrayGetPlane", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArrayGetPlane, "CUresult (CUarray *, CUarray, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuArrayDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArrayDestroy, "CUresult (CUarray)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuArray3DCreate_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArray3DCreate_v2, "CUresult (CUarray *, CUDA_ARRAY3D_DESCRIPTOR const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuArray3DGetDescriptor_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuArray3DGetDescriptor_v2, "CUresult (CUDA_ARRAY3D_DESCRIPTOR *, CUarray)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMipmappedArrayCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMipmappedArrayCreate, "CUresult (CUmipmappedArray *, CUDA_ARRAY3D_DESCRIPTOR const *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMipmappedArrayGetLevel", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMipmappedArrayGetLevel, "CUresult (CUarray *, CUmipmappedArray, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMipmappedArrayDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMipmappedArrayDestroy, "CUresult (CUmipmappedArray)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemGetHandleForAddressRange", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemGetHandleForAddressRange, "CUresult (void *, CUdeviceptr, size_t, CUmemRangeHandleType, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemBatchDecompressAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemBatchDecompressAsync, "CUresult (CUmemDecompressParams *, size_t, unsigned int, size_t *, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemAddressReserve", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAddressReserve, "CUresult (CUdeviceptr *, size_t, size_t, CUdeviceptr, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemAddressFree", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAddressFree, "CUresult (CUdeviceptr, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemCreate, "CUresult (CUmemGenericAllocationHandle *, size_t, CUmemAllocationProp const *, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemRelease", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemRelease, "CUresult (CUmemGenericAllocationHandle)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemMap", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemMap, "CUresult (CUdeviceptr, size_t, size_t, CUmemGenericAllocationHandle, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemMapArrayAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemMapArrayAsync, "CUresult (CUarrayMapInfo *, unsigned int, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemUnmap", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemUnmap, "CUresult (CUdeviceptr, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemSetAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemSetAccess, "CUresult (CUdeviceptr, size_t, CUmemAccessDesc const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemGetAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemGetAccess, "CUresult (unsigned PY_LONG_LONG *, CUmemLocation const *, CUdeviceptr)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemExportToShareableHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemExportToShareableHandle, "CUresult (void *, CUmemGenericAllocationHandle, CUmemAllocationHandleType, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemImportFromShareableHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemImportFromShareableHandle, "CUresult (CUmemGenericAllocationHandle *, void *, CUmemAllocationHandleType)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemGetAllocationGranularity", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemGetAllocationGranularity, "CUresult (size_t *, CUmemAllocationProp const *, CUmemAllocationGranularity_flags)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemGetAllocationPropertiesFromHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemGetAllocationPropertiesFromHandle, "CUresult (CUmemAllocationProp *, CUmemGenericAllocationHandle)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemRetainAllocationHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemRetainAllocationHandle, "CUresult (CUmemGenericAllocationHandle *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemFreeAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemFreeAsync, "CUresult (CUdeviceptr, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemAllocAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAllocAsync, "CUresult (CUdeviceptr *, size_t, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemPoolTrimTo", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolTrimTo, "CUresult (CUmemoryPool, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemPoolSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolSetAttribute, "CUresult (CUmemoryPool, CUmemPool_attribute, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemPoolGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolGetAttribute, "CUresult (CUmemoryPool, CUmemPool_attribute, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemPoolSetAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolSetAccess, "CUresult (CUmemoryPool, CUmemAccessDesc const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemPoolGetAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolGetAccess, "CUresult (CUmemAccess_flags *, CUmemoryPool, CUmemLocation *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemPoolCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolCreate, "CUresult (CUmemoryPool *, CUmemPoolProps const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemPoolDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolDestroy, "CUresult (CUmemoryPool)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemAllocFromPoolAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAllocFromPoolAsync, "CUresult (CUdeviceptr *, size_t, CUmemoryPool, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemPoolExportToShareableHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolExportToShareableHandle, "CUresult (void *, CUmemoryPool, CUmemAllocationHandleType, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemPoolImportFromShareableHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolImportFromShareableHandle, "CUresult (CUmemoryPool *, void *, CUmemAllocationHandleType, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemPoolExportPointer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolExportPointer, "CUresult (CUmemPoolPtrExportData *, CUdeviceptr)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemPoolImportPointer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPoolImportPointer, "CUresult (CUdeviceptr *, CUmemoryPool, CUmemPoolPtrExportData *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMulticastCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMulticastCreate, "CUresult (CUmemGenericAllocationHandle *, CUmulticastObjectProp const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMulticastAddDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMulticastAddDevice, "CUresult (CUmemGenericAllocationHandle, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMulticastBindMem", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMulticastBindMem, "CUresult (CUmemGenericAllocationHandle, size_t, CUmemGenericAllocationHandle, size_t, size_t, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMulticastBindAddr", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMulticastBindAddr, "CUresult (CUmemGenericAllocationHandle, size_t, CUdeviceptr, size_t, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMulticastUnbind", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMulticastUnbind, "CUresult (CUmemGenericAllocationHandle, CUdevice, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMulticastGetGranularity", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMulticastGetGranularity, "CUresult (size_t *, CUmulticastObjectProp const *, CUmulticastGranularity_flags)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuPointerGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuPointerGetAttribute, "CUresult (void *, CUpointer_attribute, CUdeviceptr)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemPrefetchAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPrefetchAsync, "CUresult (CUdeviceptr, size_t, CUdevice, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemPrefetchAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemPrefetchAsync_v2, "CUresult (CUdeviceptr, size_t, CUmemLocation, unsigned int, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemAdvise", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAdvise, "CUresult (CUdeviceptr, size_t, CUmem_advise, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemAdvise_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemAdvise_v2, "CUresult (CUdeviceptr, size_t, CUmem_advise, CUmemLocation)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemRangeGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemRangeGetAttribute, "CUresult (void *, size_t, CUmem_range_attribute, CUdeviceptr, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuMemRangeGetAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuMemRangeGetAttributes, "CUresult (void **, size_t *, CUmem_range_attribute *, size_t, CUdeviceptr, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuPointerSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuPointerSetAttribute, "CUresult (void const *, CUpointer_attribute, CUdeviceptr)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuPointerGetAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuPointerGetAttributes, "CUresult (unsigned int, CUpointer_attribute *, void **, CUdeviceptr)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamCreate, "CUresult (CUstream *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamCreateWithPriority", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamCreateWithPriority, "CUresult (CUstream *, unsigned int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamGetPriority", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetPriority, "CUresult (CUstream, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamGetDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetDevice, "CUresult (CUstream, CUdevice *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamGetFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetFlags, "CUresult (CUstream, unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamGetId", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetId, "CUresult (CUstream, unsigned PY_LONG_LONG *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamGetCtx", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetCtx, "CUresult (CUstream, CUcontext *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamGetCtx_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetCtx_v2, "CUresult (CUstream, CUcontext *, CUgreenCtx *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamWaitEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamWaitEvent, "CUresult (CUstream, CUevent, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamAddCallback", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamAddCallback, "CUresult (CUstream, CUstreamCallback, void *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamBeginCapture_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamBeginCapture_v2, "CUresult (CUstream, CUstreamCaptureMode)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamBeginCaptureToGraph", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamBeginCaptureToGraph, "CUresult (CUstream, CUgraph, CUgraphNode const *, CUgraphEdgeData const *, size_t, CUstreamCaptureMode)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuThreadExchangeStreamCaptureMode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuThreadExchangeStreamCaptureMode, "CUresult (CUstreamCaptureMode *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamEndCapture", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamEndCapture, "CUresult (CUstream, CUgraph *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamIsCapturing", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamIsCapturing, "CUresult (CUstream, CUstreamCaptureStatus *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamGetCaptureInfo_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetCaptureInfo_v2, "CUresult (CUstream, CUstreamCaptureStatus *, cuuint64_t *, CUgraph *, CUgraphNode const **, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamGetCaptureInfo_v3", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetCaptureInfo_v3, "CUresult (CUstream, CUstreamCaptureStatus *, cuuint64_t *, CUgraph *, CUgraphNode const **, CUgraphEdgeData const **, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamUpdateCaptureDependencies", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamUpdateCaptureDependencies, "CUresult (CUstream, CUgraphNode *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamUpdateCaptureDependencies_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamUpdateCaptureDependencies_v2, "CUresult (CUstream, CUgraphNode *, CUgraphEdgeData const *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamAttachMemAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamAttachMemAsync, "CUresult (CUstream, CUdeviceptr, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamQuery", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamQuery, "CUresult (CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamSynchronize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamSynchronize, "CUresult (CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamDestroy_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamDestroy_v2, "CUresult (CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamCopyAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamCopyAttributes, "CUresult (CUstream, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetAttribute, "CUresult (CUstream, CUstreamAttrID, CUstreamAttrValue *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamSetAttribute, "CUresult (CUstream, CUstreamAttrID, CUstreamAttrValue const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEventCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventCreate, "CUresult (CUevent *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEventRecord", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventRecord, "CUresult (CUevent, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEventRecordWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventRecordWithFlags, "CUresult (CUevent, CUstream, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEventQuery", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventQuery, "CUresult (CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEventSynchronize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventSynchronize, "CUresult (CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEventDestroy_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventDestroy_v2, "CUresult (CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEventElapsedTime", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventElapsedTime, "CUresult (float *, CUevent, CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEventElapsedTime_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventElapsedTime_v2, "CUresult (float *, CUevent, CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuImportExternalMemory", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuImportExternalMemory, "CUresult (CUexternalMemory *, CUDA_EXTERNAL_MEMORY_HANDLE_DESC const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuExternalMemoryGetMappedBuffer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuExternalMemoryGetMappedBuffer, "CUresult (CUdeviceptr *, CUexternalMemory, CUDA_EXTERNAL_MEMORY_BUFFER_DESC const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuExternalMemoryGetMappedMipmappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuExternalMemoryGetMappedMipmappedArray, "CUresult (CUmipmappedArray *, CUexternalMemory, CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDestroyExternalMemory", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDestroyExternalMemory, "CUresult (CUexternalMemory)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuImportExternalSemaphore", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuImportExternalSemaphore, "CUresult (CUexternalSemaphore *, CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuSignalExternalSemaphoresAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuSignalExternalSemaphoresAsync, "CUresult (CUexternalSemaphore const *, CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS const *, unsigned int, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuWaitExternalSemaphoresAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuWaitExternalSemaphoresAsync, "CUresult (CUexternalSemaphore const *, CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS const *, unsigned int, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDestroyExternalSemaphore", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDestroyExternalSemaphore, "CUresult (CUexternalSemaphore)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamWaitValue32_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamWaitValue32_v2, "CUresult (CUstream, CUdeviceptr, cuuint32_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamWaitValue64_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamWaitValue64_v2, "CUresult (CUstream, CUdeviceptr, cuuint64_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamWriteValue32_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamWriteValue32_v2, "CUresult (CUstream, CUdeviceptr, cuuint32_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamWriteValue64_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamWriteValue64_v2, "CUresult (CUstream, CUdeviceptr, cuuint64_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamBatchMemOp_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamBatchMemOp_v2, "CUresult (CUstream, unsigned int, CUstreamBatchMemOpParams *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFuncGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncGetAttribute, "CUresult (int *, CUfunction_attribute, CUfunction)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFuncSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncSetAttribute, "CUresult (CUfunction, CUfunction_attribute, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFuncSetCacheConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncSetCacheConfig, "CUresult (CUfunction, CUfunc_cache)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFuncGetModule", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncGetModule, "CUresult (CUmodule *, CUfunction)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFuncGetName", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncGetName, "CUresult (char const **, CUfunction)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFuncGetParamInfo", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncGetParamInfo, "CUresult (CUfunction, size_t, size_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFuncIsLoaded", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncIsLoaded, "CUresult (CUfunctionLoadingState *, CUfunction)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFuncLoad", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncLoad, "CUresult (CUfunction)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLaunchKernel", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchKernel, "CUresult (CUfunction, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, CUstream, void **, void **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLaunchKernelEx", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchKernelEx, "CUresult (CUlaunchConfig const *, CUfunction, void **, void **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLaunchCooperativeKernel", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchCooperativeKernel, "CUresult (CUfunction, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, unsigned int, CUstream, void **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLaunchCooperativeKernelMultiDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchCooperativeKernelMultiDevice, "CUresult (CUDA_LAUNCH_PARAMS *, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLaunchHostFunc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchHostFunc, "CUresult (CUstream, CUhostFn, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFuncSetBlockShape", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncSetBlockShape, "CUresult (CUfunction, int, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFuncSetSharedSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncSetSharedSize, "CUresult (CUfunction, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuParamSetSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuParamSetSize, "CUresult (CUfunction, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuParamSeti", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuParamSeti, "CUresult (CUfunction, int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuParamSetf", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuParamSetf, "CUresult (CUfunction, int, float)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuParamSetv", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuParamSetv, "CUresult (CUfunction, int, void *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLaunch", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunch, "CUresult (CUfunction)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLaunchGrid", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchGrid, "CUresult (CUfunction, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLaunchGridAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLaunchGridAsync, "CUresult (CUfunction, int, int, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuParamSetTexRef", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuParamSetTexRef, "CUresult (CUfunction, int, CUtexref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFuncSetSharedMemConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuFuncSetSharedMemConfig, "CUresult (CUfunction, CUsharedconfig)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphCreate, "CUresult (CUgraph *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddKernelNode_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddKernelNode_v2, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_KERNEL_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphKernelNodeGetParams_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphKernelNodeGetParams_v2, "CUresult (CUgraphNode, CUDA_KERNEL_NODE_PARAMS *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphKernelNodeSetParams_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphKernelNodeSetParams_v2, "CUresult (CUgraphNode, CUDA_KERNEL_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddMemcpyNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddMemcpyNode, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_MEMCPY3D const *, CUcontext)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphMemcpyNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphMemcpyNodeGetParams, "CUresult (CUgraphNode, CUDA_MEMCPY3D *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphMemcpyNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphMemcpyNodeSetParams, "CUresult (CUgraphNode, CUDA_MEMCPY3D const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddMemsetNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddMemsetNode, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_MEMSET_NODE_PARAMS const *, CUcontext)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphMemsetNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphMemsetNodeGetParams, "CUresult (CUgraphNode, CUDA_MEMSET_NODE_PARAMS *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphMemsetNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphMemsetNodeSetParams, "CUresult (CUgraphNode, CUDA_MEMSET_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddHostNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddHostNode, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_HOST_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphHostNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphHostNodeGetParams, "CUresult (CUgraphNode, CUDA_HOST_NODE_PARAMS *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphHostNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphHostNodeSetParams, "CUresult (CUgraphNode, CUDA_HOST_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddChildGraphNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddChildGraphNode, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUgraph)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphChildGraphNodeGetGraph", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphChildGraphNodeGetGraph, "CUresult (CUgraphNode, CUgraph *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddEmptyNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddEmptyNode, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddEventRecordNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddEventRecordNode, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphEventRecordNodeGetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphEventRecordNodeGetEvent, "CUresult (CUgraphNode, CUevent *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphEventRecordNodeSetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphEventRecordNodeSetEvent, "CUresult (CUgraphNode, CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddEventWaitNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddEventWaitNode, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphEventWaitNodeGetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphEventWaitNodeGetEvent, "CUresult (CUgraphNode, CUevent *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphEventWaitNodeSetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphEventWaitNodeSetEvent, "CUresult (CUgraphNode, CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddExternalSemaphoresSignalNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddExternalSemaphoresSignalNode, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExternalSemaphoresSignalNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExternalSemaphoresSignalNodeGetParams, "CUresult (CUgraphNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExternalSemaphoresSignalNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExternalSemaphoresSignalNodeSetParams, "CUresult (CUgraphNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddExternalSemaphoresWaitNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddExternalSemaphoresWaitNode, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_EXT_SEM_WAIT_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExternalSemaphoresWaitNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExternalSemaphoresWaitNodeGetParams, "CUresult (CUgraphNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExternalSemaphoresWaitNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExternalSemaphoresWaitNodeSetParams, "CUresult (CUgraphNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddBatchMemOpNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddBatchMemOpNode, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_BATCH_MEM_OP_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphBatchMemOpNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphBatchMemOpNodeGetParams, "CUresult (CUgraphNode, CUDA_BATCH_MEM_OP_NODE_PARAMS *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphBatchMemOpNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphBatchMemOpNodeSetParams, "CUresult (CUgraphNode, CUDA_BATCH_MEM_OP_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecBatchMemOpNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecBatchMemOpNodeSetParams, "CUresult (CUgraphExec, CUgraphNode, CUDA_BATCH_MEM_OP_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddMemAllocNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddMemAllocNode, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUDA_MEM_ALLOC_NODE_PARAMS *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphMemAllocNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphMemAllocNodeGetParams, "CUresult (CUgraphNode, CUDA_MEM_ALLOC_NODE_PARAMS *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddMemFreeNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddMemFreeNode, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUdeviceptr)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphMemFreeNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphMemFreeNodeGetParams, "CUresult (CUgraphNode, CUdeviceptr *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGraphMemTrim", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGraphMemTrim, "CUresult (CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetGraphMemAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetGraphMemAttribute, "CUresult (CUdevice, CUgraphMem_attribute, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceSetGraphMemAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceSetGraphMemAttribute, "CUresult (CUdevice, CUgraphMem_attribute, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphClone", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphClone, "CUresult (CUgraph *, CUgraph)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphNodeFindInClone", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeFindInClone, "CUresult (CUgraphNode *, CUgraphNode, CUgraph)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphNodeGetType", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeGetType, "CUresult (CUgraphNode, CUgraphNodeType *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphGetNodes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphGetNodes, "CUresult (CUgraph, CUgraphNode *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphGetRootNodes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphGetRootNodes, "CUresult (CUgraph, CUgraphNode *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphGetEdges", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphGetEdges, "CUresult (CUgraph, CUgraphNode *, CUgraphNode *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphGetEdges_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphGetEdges_v2, "CUresult (CUgraph, CUgraphNode *, CUgraphNode *, CUgraphEdgeData *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphNodeGetDependencies", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeGetDependencies, "CUresult (CUgraphNode, CUgraphNode *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphNodeGetDependencies_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeGetDependencies_v2, "CUresult (CUgraphNode, CUgraphNode *, CUgraphEdgeData *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphNodeGetDependentNodes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeGetDependentNodes, "CUresult (CUgraphNode, CUgraphNode *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphNodeGetDependentNodes_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeGetDependentNodes_v2, "CUresult (CUgraphNode, CUgraphNode *, CUgraphEdgeData *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddDependencies", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddDependencies, "CUresult (CUgraph, CUgraphNode const *, CUgraphNode const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddDependencies_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddDependencies_v2, "CUresult (CUgraph, CUgraphNode const *, CUgraphNode const *, CUgraphEdgeData const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphRemoveDependencies", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphRemoveDependencies, "CUresult (CUgraph, CUgraphNode const *, CUgraphNode const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphRemoveDependencies_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphRemoveDependencies_v2, "CUresult (CUgraph, CUgraphNode const *, CUgraphNode const *, CUgraphEdgeData const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphDestroyNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphDestroyNode, "CUresult (CUgraphNode)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphInstantiateWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphInstantiateWithFlags, "CUresult (CUgraphExec *, CUgraph, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphInstantiateWithParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphInstantiateWithParams, "CUresult (CUgraphExec *, CUgraph, CUDA_GRAPH_INSTANTIATE_PARAMS *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecGetFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecGetFlags, "CUresult (CUgraphExec, cuuint64_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecKernelNodeSetParams_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecKernelNodeSetParams_v2, "CUresult (CUgraphExec, CUgraphNode, CUDA_KERNEL_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecMemcpyNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecMemcpyNodeSetParams, "CUresult (CUgraphExec, CUgraphNode, CUDA_MEMCPY3D const *, CUcontext)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecMemsetNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecMemsetNodeSetParams, "CUresult (CUgraphExec, CUgraphNode, CUDA_MEMSET_NODE_PARAMS const *, CUcontext)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecHostNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecHostNodeSetParams, "CUresult (CUgraphExec, CUgraphNode, CUDA_HOST_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecChildGraphNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecChildGraphNodeSetParams, "CUresult (CUgraphExec, CUgraphNode, CUgraph)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecEventRecordNodeSetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecEventRecordNodeSetEvent, "CUresult (CUgraphExec, CUgraphNode, CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecEventWaitNodeSetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecEventWaitNodeSetEvent, "CUresult (CUgraphExec, CUgraphNode, CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecExternalSemaphoresSignalNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecExternalSemaphoresSignalNodeSetParams, "CUresult (CUgraphExec, CUgraphNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecExternalSemaphoresWaitNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecExternalSemaphoresWaitNodeSetParams, "CUresult (CUgraphExec, CUgraphNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphNodeSetEnabled", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeSetEnabled, "CUresult (CUgraphExec, CUgraphNode, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphNodeGetEnabled", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeGetEnabled, "CUresult (CUgraphExec, CUgraphNode, unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphUpload", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphUpload, "CUresult (CUgraphExec, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphLaunch", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphLaunch, "CUresult (CUgraphExec, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecDestroy, "CUresult (CUgraphExec)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphDestroy, "CUresult (CUgraph)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecUpdate_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecUpdate_v2, "CUresult (CUgraphExec, CUgraph, CUgraphExecUpdateResultInfo *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphKernelNodeCopyAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphKernelNodeCopyAttributes, "CUresult (CUgraphNode, CUgraphNode)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphKernelNodeGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphKernelNodeGetAttribute, "CUresult (CUgraphNode, CUkernelNodeAttrID, CUkernelNodeAttrValue *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphKernelNodeSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphKernelNodeSetAttribute, "CUresult (CUgraphNode, CUkernelNodeAttrID, CUkernelNodeAttrValue const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphDebugDotPrint", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphDebugDotPrint, "CUresult (CUgraph, char const *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuUserObjectCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuUserObjectCreate, "CUresult (CUuserObject *, void *, CUhostFn, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuUserObjectRetain", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuUserObjectRetain, "CUresult (CUuserObject, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuUserObjectRelease", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuUserObjectRelease, "CUresult (CUuserObject, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphRetainUserObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphRetainUserObject, "CUresult (CUgraph, CUuserObject, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphReleaseUserObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphReleaseUserObject, "CUresult (CUgraph, CUuserObject, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddNode, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, size_t, CUgraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphAddNode_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphAddNode_v2, "CUresult (CUgraphNode *, CUgraph, CUgraphNode const *, CUgraphEdgeData const *, size_t, CUgraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphNodeSetParams, "CUresult (CUgraphNode, CUgraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphExecNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphExecNodeSetParams, "CUresult (CUgraphExec, CUgraphNode, CUgraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphConditionalHandleCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphConditionalHandleCreate, "CUresult (CUgraphConditionalHandle *, CUgraph, CUcontext, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuOccupancyMaxActiveBlocksPerMultiprocessor", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyMaxActiveBlocksPerMultiprocessor, "CUresult (int *, CUfunction, int, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, "CUresult (int *, CUfunction, int, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuOccupancyMaxPotentialBlockSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyMaxPotentialBlockSize, "CUresult (int *, int *, CUfunction, CUoccupancyB2DSize, size_t, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuOccupancyMaxPotentialBlockSizeWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyMaxPotentialBlockSizeWithFlags, "CUresult (int *, int *, CUfunction, CUoccupancyB2DSize, size_t, int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuOccupancyAvailableDynamicSMemPerBlock", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyAvailableDynamicSMemPerBlock, "CUresult (size_t *, CUfunction, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuOccupancyMaxPotentialClusterSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyMaxPotentialClusterSize, "CUresult (int *, CUfunction, CUlaunchConfig const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuOccupancyMaxActiveClusters", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuOccupancyMaxActiveClusters, "CUresult (int *, CUfunction, CUlaunchConfig const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefSetArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetArray, "CUresult (CUtexref, CUarray, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefSetMipmappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetMipmappedArray, "CUresult (CUtexref, CUmipmappedArray, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefSetAddress_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetAddress_v2, "CUresult (size_t *, CUtexref, CUdeviceptr, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefSetAddress2D_v3", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetAddress2D_v3, "CUresult (CUtexref, CUDA_ARRAY_DESCRIPTOR const *, CUdeviceptr, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefSetFormat", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetFormat, "CUresult (CUtexref, CUarray_format, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefSetAddressMode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetAddressMode, "CUresult (CUtexref, int, CUaddress_mode)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefSetFilterMode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetFilterMode, "CUresult (CUtexref, CUfilter_mode)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefSetMipmapFilterMode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetMipmapFilterMode, "CUresult (CUtexref, CUfilter_mode)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefSetMipmapLevelBias", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetMipmapLevelBias, "CUresult (CUtexref, float)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefSetMipmapLevelClamp", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetMipmapLevelClamp, "CUresult (CUtexref, float, float)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefSetMaxAnisotropy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetMaxAnisotropy, "CUresult (CUtexref, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefSetBorderColor", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetBorderColor, "CUresult (CUtexref, float *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefSetFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefSetFlags, "CUresult (CUtexref, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefGetAddress_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetAddress_v2, "CUresult (CUdeviceptr *, CUtexref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefGetArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetArray, "CUresult (CUarray *, CUtexref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefGetMipmappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetMipmappedArray, "CUresult (CUmipmappedArray *, CUtexref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefGetAddressMode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetAddressMode, "CUresult (CUaddress_mode *, CUtexref, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefGetFilterMode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetFilterMode, "CUresult (CUfilter_mode *, CUtexref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefGetFormat", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetFormat, "CUresult (CUarray_format *, int *, CUtexref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefGetMipmapFilterMode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetMipmapFilterMode, "CUresult (CUfilter_mode *, CUtexref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefGetMipmapLevelBias", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetMipmapLevelBias, "CUresult (float *, CUtexref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefGetMipmapLevelClamp", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetMipmapLevelClamp, "CUresult (float *, float *, CUtexref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefGetMaxAnisotropy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetMaxAnisotropy, "CUresult (int *, CUtexref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefGetBorderColor", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetBorderColor, "CUresult (float *, CUtexref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefGetFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefGetFlags, "CUresult (unsigned int *, CUtexref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefCreate, "CUresult (CUtexref *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexRefDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexRefDestroy, "CUresult (CUtexref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuSurfRefSetArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuSurfRefSetArray, "CUresult (CUsurfref, CUarray, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuSurfRefGetArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuSurfRefGetArray, "CUresult (CUarray *, CUsurfref)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexObjectCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexObjectCreate, "CUresult (CUtexObject *, CUDA_RESOURCE_DESC const *, CUDA_TEXTURE_DESC const *, CUDA_RESOURCE_VIEW_DESC const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexObjectDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexObjectDestroy, "CUresult (CUtexObject)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexObjectGetResourceDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexObjectGetResourceDesc, "CUresult (CUDA_RESOURCE_DESC *, CUtexObject)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexObjectGetTextureDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexObjectGetTextureDesc, "CUresult (CUDA_TEXTURE_DESC *, CUtexObject)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTexObjectGetResourceViewDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTexObjectGetResourceViewDesc, "CUresult (CUDA_RESOURCE_VIEW_DESC *, CUtexObject)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuSurfObjectCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuSurfObjectCreate, "CUresult (CUsurfObject *, CUDA_RESOURCE_DESC const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuSurfObjectDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuSurfObjectDestroy, "CUresult (CUsurfObject)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuSurfObjectGetResourceDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuSurfObjectGetResourceDesc, "CUresult (CUDA_RESOURCE_DESC *, CUsurfObject)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTensorMapEncodeTiled", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTensorMapEncodeTiled, "CUresult (CUtensorMap *, CUtensorMapDataType, cuuint32_t, void *, cuuint64_t const *, cuuint64_t const *, cuuint32_t const *, cuuint32_t const *, CUtensorMapInterleave, CUtensorMapSwizzle, CUtensorMapL2promotion, CUtensorMapFloatOOBfill)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTensorMapEncodeIm2col", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTensorMapEncodeIm2col, "CUresult (CUtensorMap *, CUtensorMapDataType, cuuint32_t, void *, cuuint64_t const *, cuuint64_t const *, int const *, int const *, cuuint32_t, cuuint32_t, cuuint32_t const *, CUtensorMapInterleave, CUtensorMapSwizzle, CUtensorMapL2promotion, CUtensorMapFloatOOBfill)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTensorMapEncodeIm2colWide", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTensorMapEncodeIm2colWide, "CUresult (CUtensorMap *, CUtensorMapDataType, cuuint32_t, void *, cuuint64_t const *, cuuint64_t const *, int, int, cuuint32_t, cuuint32_t, cuuint32_t const *, CUtensorMapInterleave, CUtensorMapIm2ColWideMode, CUtensorMapSwizzle, CUtensorMapL2promotion, CUtensorMapFloatOOBfill)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuTensorMapReplaceAddress", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuTensorMapReplaceAddress, "CUresult (CUtensorMap *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceCanAccessPeer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceCanAccessPeer, "CUresult (int *, CUdevice, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxEnablePeerAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxEnablePeerAccess, "CUresult (CUcontext, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxDisablePeerAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxDisablePeerAccess, "CUresult (CUcontext)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetP2PAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetP2PAttribute, "CUresult (int *, CUdevice_P2PAttribute, CUdevice, CUdevice)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphicsUnregisterResource", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsUnregisterResource, "CUresult (CUgraphicsResource)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphicsSubResourceGetMappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsSubResourceGetMappedArray, "CUresult (CUarray *, CUgraphicsResource, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphicsResourceGetMappedMipmappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsResourceGetMappedMipmappedArray, "CUresult (CUmipmappedArray *, CUgraphicsResource)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphicsResourceGetMappedPointer_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsResourceGetMappedPointer_v2, "CUresult (CUdeviceptr *, size_t *, CUgraphicsResource)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphicsResourceSetMapFlags_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsResourceSetMapFlags_v2, "CUresult (CUgraphicsResource, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphicsMapResources", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsMapResources, "CUresult (unsigned int, CUgraphicsResource *, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphicsUnmapResources", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsUnmapResources, "CUresult (unsigned int, CUgraphicsResource *, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGetProcAddress_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGetProcAddress_v2, "CUresult (char const *, void **, int, cuuint64_t, CUdriverProcAddressQueryResult *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCoredumpGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCoredumpGetAttribute, "CUresult (CUcoredumpSettings, void *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCoredumpGetAttributeGlobal", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCoredumpGetAttributeGlobal, "CUresult (CUcoredumpSettings, void *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCoredumpSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCoredumpSetAttribute, "CUresult (CUcoredumpSettings, void *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCoredumpSetAttributeGlobal", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCoredumpSetAttributeGlobal, "CUresult (CUcoredumpSettings, void *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGetExportTable", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGetExportTable, "CUresult (void const **, CUuuid const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGreenCtxCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGreenCtxCreate, "CUresult (CUgreenCtx *, CUdevResourceDesc, CUdevice, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGreenCtxDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGreenCtxDestroy, "CUresult (CUgreenCtx)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxFromGreenCtx", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxFromGreenCtx, "CUresult (CUcontext *, CUgreenCtx)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDeviceGetDevResource", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDeviceGetDevResource, "CUresult (CUdevice, CUdevResource *, CUdevResourceType)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCtxGetDevResource", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCtxGetDevResource, "CUresult (CUcontext, CUdevResource *, CUdevResourceType)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGreenCtxGetDevResource", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGreenCtxGetDevResource, "CUresult (CUgreenCtx, CUdevResource *, CUdevResourceType)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDevSmResourceSplitByCount", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevSmResourceSplitByCount, "CUresult (CUdevResource *, unsigned int *, CUdevResource const *, CUdevResource *, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuDevResourceGenerateDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuDevResourceGenerateDesc, "CUresult (CUdevResourceDesc *, CUdevResource *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGreenCtxRecordEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGreenCtxRecordEvent, "CUresult (CUgreenCtx, CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGreenCtxWaitEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGreenCtxWaitEvent, "CUresult (CUgreenCtx, CUevent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuStreamGetGreenCtx", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuStreamGetGreenCtx, "CUresult (CUstream, CUgreenCtx *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGreenCtxStreamCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGreenCtxStreamCreate, "CUresult (CUstream *, CUgreenCtx, unsigned int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLogsRegisterCallback", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLogsRegisterCallback, "CUresult (CUlogsCallback, void *, CUlogsCallbackHandle *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLogsUnregisterCallback", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLogsUnregisterCallback, "CUresult (CUlogsCallbackHandle)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLogsCurrent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLogsCurrent, "CUresult (CUlogIterator *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLogsDumpToFile", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLogsDumpToFile, "CUresult (CUlogIterator *, char const *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuLogsDumpToMemory", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuLogsDumpToMemory, "CUresult (CUlogIterator *, char *, size_t *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCheckpointProcessGetRestoreThreadId", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCheckpointProcessGetRestoreThreadId, "CUresult (int, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCheckpointProcessGetState", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCheckpointProcessGetState, "CUresult (int, CUprocessState *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCheckpointProcessLock", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCheckpointProcessLock, "CUresult (int, CUcheckpointLockArgs *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCheckpointProcessCheckpoint", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCheckpointProcessCheckpoint, "CUresult (int, CUcheckpointCheckpointArgs *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCheckpointProcessRestore", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCheckpointProcessRestore, "CUresult (int, CUcheckpointRestoreArgs *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuCheckpointProcessUnlock", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuCheckpointProcessUnlock, "CUresult (int, CUcheckpointUnlockArgs *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuProfilerStart", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuProfilerStart, "CUresult (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuProfilerStop", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuProfilerStop, "CUresult (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphicsEGLRegisterImage", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsEGLRegisterImage, "CUresult (CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_EGLImageKHR, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEGLStreamConsumerConnect", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerConnect, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEGLStreamConsumerConnectWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerConnectWithFlags, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEGLStreamConsumerDisconnect", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerDisconnect, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEGLStreamConsumerAcquireFrame", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerAcquireFrame, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, CUgraphicsResource *, CUstream *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEGLStreamConsumerReleaseFrame", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerReleaseFrame, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, CUgraphicsResource, CUstream *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEGLStreamProducerConnect", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerConnect, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR, __pyx_t_4cuda_8bindings_8cydriver_EGLint, __pyx_t_4cuda_8bindings_8cydriver_EGLint)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEGLStreamProducerDisconnect", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerDisconnect, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEGLStreamProducerPresentFrame", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerPresentFrame, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame, CUstream *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEGLStreamProducerReturnFrame", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerReturnFrame, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame *, CUstream *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphicsResourceGetMappedEglFrame", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsResourceGetMappedEglFrame, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglFrame *, CUgraphicsResource, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuEventCreateFromEGLSync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventCreateFromEGLSync, "CUresult (CUevent *, __pyx_t_4cuda_8bindings_8cydriver_EGLSyncKHR, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphicsGLRegisterBuffer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsGLRegisterBuffer, "CUresult (CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_GLuint, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphicsGLRegisterImage", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsGLRegisterImage, "CUresult (CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_GLuint, __pyx_t_4cuda_8bindings_8cydriver_GLenum, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGLGetDevices_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGLGetDevices_v2, "CUresult (unsigned int *, CUdevice *, unsigned int, __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuVDPAUGetDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuVDPAUGetDevice, "CUresult (CUdevice *, __pyx_t_4cuda_8bindings_8cydriver_VdpDevice, __pyx_t_4cuda_8bindings_8cydriver_VdpGetProcAddress *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuVDPAUCtxCreate_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuVDPAUCtxCreate_v2, "CUresult (CUcontext *, unsigned int, CUdevice, __pyx_t_4cuda_8bindings_8cydriver_VdpDevice, __pyx_t_4cuda_8bindings_8cydriver_VdpGetProcAddress *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphicsVDPAURegisterVideoSurface", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsVDPAURegisterVideoSurface, "CUresult (CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_VdpVideoSurface, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuGraphicsVDPAURegisterOutputSurface", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsVDPAURegisterOutputSurface, "CUresult (CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_VdpOutputSurface, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_cydriver(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_cydriver}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "cydriver", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_cydriver(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_cydriver(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x03090000 + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_cydriver(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'cydriver' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "cydriver" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_NOT_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY +__Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); +if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); +} +#endif + +__Pyx_RefNannySetupContext("PyInit_cydriver", 0); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #ifdef __Pxy_PyFrame_Initialize_Offsets + __Pxy_PyFrame_Initialize_Offsets(); + #endif + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #if 0 || defined(__Pyx_CyFunction_USED) || defined(__Pyx_FusedFunction_USED) || defined(__Pyx_Coroutine_USED) || defined(__Pyx_Generator_USED) || defined(__Pyx_AsyncGen_USED) + if (__pyx_CommonTypesMetaclass_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_AsyncGen_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + if (__pyx_module_is_main_cuda__bindings___bindings__cydriver) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name_2, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "cuda.bindings._bindings.cydriver")) { + if (unlikely((PyDict_SetItemString(modules, "cuda.bindings._bindings.cydriver", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_function_export_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_type_init_code(__pyx_mstate); + (void)__Pyx_modinit_type_import_code(__pyx_mstate); + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + (void)__Pyx_modinit_function_import_code(__pyx_mstate); + /*--- Execution code ---*/ + + /* "cuda/bindings/_bindings/cydriver.pyx":7 + * cimport cuda.bindings._lib.dlfcn as dlfcn + * from libc.stdint cimport intptr_t, uintptr_t + * import os # <<<<<<<<<<<<<< + * import sys + * cimport cuda.bindings._bindings.loader as loader +*/ + __pyx_t_2 = __Pyx_ImportDottedModule(__pyx_mstate_global->__pyx_n_u_os, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_os, __pyx_t_2) < (0)) __PYX_ERR(0, 7, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":8 + * from libc.stdint cimport intptr_t, uintptr_t + * import os + * import sys # <<<<<<<<<<<<<< + * cimport cuda.bindings._bindings.loader as loader + * import threading +*/ + __pyx_t_2 = __Pyx_ImportDottedModule(__pyx_mstate_global->__pyx_n_u_sys, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_sys, __pyx_t_2) < (0)) __PYX_ERR(0, 8, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":10 + * import sys + * cimport cuda.bindings._bindings.loader as loader + * import threading # <<<<<<<<<<<<<< + * + * cdef object __symbol_lock = threading.Lock() +*/ + __pyx_t_2 = __Pyx_ImportDottedModule(__pyx_mstate_global->__pyx_n_u_threading, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_threading, __pyx_t_2) < (0)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":12 + * import threading + * + * cdef object __symbol_lock = threading.Lock() # <<<<<<<<<<<<<< + * cdef bint __cuPythonInit = False + * cdef void *__cuGetErrorString = NULL +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_threading); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_Lock); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_bindings_8cydriver___symbol_lock); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_bindings_8cydriver___symbol_lock, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_2); + __pyx_t_2 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":13 + * + * cdef object __symbol_lock = threading.Lock() + * cdef bint __cuPythonInit = False # <<<<<<<<<<<<<< + * cdef void *__cuGetErrorString = NULL + * cdef void *__cuGetErrorName = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPythonInit = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":14 + * cdef object __symbol_lock = threading.Lock() + * cdef bint __cuPythonInit = False + * cdef void *__cuGetErrorString = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGetErrorName = NULL + * cdef void *__cuInit = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorString = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":15 + * cdef bint __cuPythonInit = False + * cdef void *__cuGetErrorString = NULL + * cdef void *__cuGetErrorName = NULL # <<<<<<<<<<<<<< + * cdef void *__cuInit = NULL + * cdef void *__cuDriverGetVersion = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetErrorName = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":16 + * cdef void *__cuGetErrorString = NULL + * cdef void *__cuGetErrorName = NULL + * cdef void *__cuInit = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDriverGetVersion = NULL + * cdef void *__cuDeviceGet = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuInit = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":17 + * cdef void *__cuGetErrorName = NULL + * cdef void *__cuInit = NULL + * cdef void *__cuDriverGetVersion = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGet = NULL + * cdef void *__cuDeviceGetCount = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDriverGetVersion = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":18 + * cdef void *__cuInit = NULL + * cdef void *__cuDriverGetVersion = NULL + * cdef void *__cuDeviceGet = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetCount = NULL + * cdef void *__cuDeviceGetName = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGet = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":19 + * cdef void *__cuDriverGetVersion = NULL + * cdef void *__cuDeviceGet = NULL + * cdef void *__cuDeviceGetCount = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetName = NULL + * cdef void *__cuDeviceGetUuid = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetCount = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":20 + * cdef void *__cuDeviceGet = NULL + * cdef void *__cuDeviceGetCount = NULL + * cdef void *__cuDeviceGetName = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetUuid = NULL + * cdef void *__cuDeviceGetUuid_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetName = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":21 + * cdef void *__cuDeviceGetCount = NULL + * cdef void *__cuDeviceGetName = NULL + * cdef void *__cuDeviceGetUuid = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetUuid_v2 = NULL + * cdef void *__cuDeviceGetLuid = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":22 + * cdef void *__cuDeviceGetName = NULL + * cdef void *__cuDeviceGetUuid = NULL + * cdef void *__cuDeviceGetUuid_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetLuid = NULL + * cdef void *__cuDeviceTotalMem_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetUuid_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":23 + * cdef void *__cuDeviceGetUuid = NULL + * cdef void *__cuDeviceGetUuid_v2 = NULL + * cdef void *__cuDeviceGetLuid = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceTotalMem_v2 = NULL + * cdef void *__cuDeviceGetTexture1DLinearMaxWidth = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetLuid = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":24 + * cdef void *__cuDeviceGetUuid_v2 = NULL + * cdef void *__cuDeviceGetLuid = NULL + * cdef void *__cuDeviceTotalMem_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetTexture1DLinearMaxWidth = NULL + * cdef void *__cuDeviceGetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceTotalMem_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":25 + * cdef void *__cuDeviceGetLuid = NULL + * cdef void *__cuDeviceTotalMem_v2 = NULL + * cdef void *__cuDeviceGetTexture1DLinearMaxWidth = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetAttribute = NULL + * cdef void *__cuDeviceGetNvSciSyncAttributes = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetTexture1DLinearMaxWidth = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":26 + * cdef void *__cuDeviceTotalMem_v2 = NULL + * cdef void *__cuDeviceGetTexture1DLinearMaxWidth = NULL + * cdef void *__cuDeviceGetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetNvSciSyncAttributes = NULL + * cdef void *__cuDeviceSetMemPool = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":27 + * cdef void *__cuDeviceGetTexture1DLinearMaxWidth = NULL + * cdef void *__cuDeviceGetAttribute = NULL + * cdef void *__cuDeviceGetNvSciSyncAttributes = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceSetMemPool = NULL + * cdef void *__cuDeviceGetMemPool = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetNvSciSyncAttributes = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":28 + * cdef void *__cuDeviceGetAttribute = NULL + * cdef void *__cuDeviceGetNvSciSyncAttributes = NULL + * cdef void *__cuDeviceSetMemPool = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetMemPool = NULL + * cdef void *__cuDeviceGetDefaultMemPool = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetMemPool = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":29 + * cdef void *__cuDeviceGetNvSciSyncAttributes = NULL + * cdef void *__cuDeviceSetMemPool = NULL + * cdef void *__cuDeviceGetMemPool = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetDefaultMemPool = NULL + * cdef void *__cuDeviceGetExecAffinitySupport = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetMemPool = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":30 + * cdef void *__cuDeviceSetMemPool = NULL + * cdef void *__cuDeviceGetMemPool = NULL + * cdef void *__cuDeviceGetDefaultMemPool = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetExecAffinitySupport = NULL + * cdef void *__cuFlushGPUDirectRDMAWrites = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDefaultMemPool = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":31 + * cdef void *__cuDeviceGetMemPool = NULL + * cdef void *__cuDeviceGetDefaultMemPool = NULL + * cdef void *__cuDeviceGetExecAffinitySupport = NULL # <<<<<<<<<<<<<< + * cdef void *__cuFlushGPUDirectRDMAWrites = NULL + * cdef void *__cuDeviceGetProperties = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetExecAffinitySupport = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":32 + * cdef void *__cuDeviceGetDefaultMemPool = NULL + * cdef void *__cuDeviceGetExecAffinitySupport = NULL + * cdef void *__cuFlushGPUDirectRDMAWrites = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetProperties = NULL + * cdef void *__cuDeviceComputeCapability = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFlushGPUDirectRDMAWrites = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":33 + * cdef void *__cuDeviceGetExecAffinitySupport = NULL + * cdef void *__cuFlushGPUDirectRDMAWrites = NULL + * cdef void *__cuDeviceGetProperties = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceComputeCapability = NULL + * cdef void *__cuDevicePrimaryCtxRetain = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetProperties = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":34 + * cdef void *__cuFlushGPUDirectRDMAWrites = NULL + * cdef void *__cuDeviceGetProperties = NULL + * cdef void *__cuDeviceComputeCapability = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDevicePrimaryCtxRetain = NULL + * cdef void *__cuDevicePrimaryCtxRelease_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceComputeCapability = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":35 + * cdef void *__cuDeviceGetProperties = NULL + * cdef void *__cuDeviceComputeCapability = NULL + * cdef void *__cuDevicePrimaryCtxRetain = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDevicePrimaryCtxRelease_v2 = NULL + * cdef void *__cuDevicePrimaryCtxSetFlags_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRetain = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":36 + * cdef void *__cuDeviceComputeCapability = NULL + * cdef void *__cuDevicePrimaryCtxRetain = NULL + * cdef void *__cuDevicePrimaryCtxRelease_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDevicePrimaryCtxSetFlags_v2 = NULL + * cdef void *__cuDevicePrimaryCtxGetState = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxRelease_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":37 + * cdef void *__cuDevicePrimaryCtxRetain = NULL + * cdef void *__cuDevicePrimaryCtxRelease_v2 = NULL + * cdef void *__cuDevicePrimaryCtxSetFlags_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDevicePrimaryCtxGetState = NULL + * cdef void *__cuDevicePrimaryCtxReset_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxSetFlags_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":38 + * cdef void *__cuDevicePrimaryCtxRelease_v2 = NULL + * cdef void *__cuDevicePrimaryCtxSetFlags_v2 = NULL + * cdef void *__cuDevicePrimaryCtxGetState = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDevicePrimaryCtxReset_v2 = NULL + * cdef void *__cuCtxCreate_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxGetState = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":39 + * cdef void *__cuDevicePrimaryCtxSetFlags_v2 = NULL + * cdef void *__cuDevicePrimaryCtxGetState = NULL + * cdef void *__cuDevicePrimaryCtxReset_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxCreate_v2 = NULL + * cdef void *__cuCtxCreate_v3 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevicePrimaryCtxReset_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":40 + * cdef void *__cuDevicePrimaryCtxGetState = NULL + * cdef void *__cuDevicePrimaryCtxReset_v2 = NULL + * cdef void *__cuCtxCreate_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxCreate_v3 = NULL + * cdef void *__cuCtxCreate_v4 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":41 + * cdef void *__cuDevicePrimaryCtxReset_v2 = NULL + * cdef void *__cuCtxCreate_v2 = NULL + * cdef void *__cuCtxCreate_v3 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxCreate_v4 = NULL + * cdef void *__cuCtxDestroy_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v3 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":42 + * cdef void *__cuCtxCreate_v2 = NULL + * cdef void *__cuCtxCreate_v3 = NULL + * cdef void *__cuCtxCreate_v4 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxDestroy_v2 = NULL + * cdef void *__cuCtxPushCurrent_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxCreate_v4 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":43 + * cdef void *__cuCtxCreate_v3 = NULL + * cdef void *__cuCtxCreate_v4 = NULL + * cdef void *__cuCtxDestroy_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxPushCurrent_v2 = NULL + * cdef void *__cuCtxPopCurrent_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDestroy_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":44 + * cdef void *__cuCtxCreate_v4 = NULL + * cdef void *__cuCtxDestroy_v2 = NULL + * cdef void *__cuCtxPushCurrent_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxPopCurrent_v2 = NULL + * cdef void *__cuCtxSetCurrent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPushCurrent_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":45 + * cdef void *__cuCtxDestroy_v2 = NULL + * cdef void *__cuCtxPushCurrent_v2 = NULL + * cdef void *__cuCtxPopCurrent_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxSetCurrent = NULL + * cdef void *__cuCtxGetCurrent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxPopCurrent_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":46 + * cdef void *__cuCtxPushCurrent_v2 = NULL + * cdef void *__cuCtxPopCurrent_v2 = NULL + * cdef void *__cuCtxSetCurrent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxGetCurrent = NULL + * cdef void *__cuCtxGetDevice = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCurrent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":47 + * cdef void *__cuCtxPopCurrent_v2 = NULL + * cdef void *__cuCtxSetCurrent = NULL + * cdef void *__cuCtxGetCurrent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxGetDevice = NULL + * cdef void *__cuCtxGetFlags = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCurrent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":48 + * cdef void *__cuCtxSetCurrent = NULL + * cdef void *__cuCtxGetCurrent = NULL + * cdef void *__cuCtxGetDevice = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxGetFlags = NULL + * cdef void *__cuCtxSetFlags = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevice = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":49 + * cdef void *__cuCtxGetCurrent = NULL + * cdef void *__cuCtxGetDevice = NULL + * cdef void *__cuCtxGetFlags = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxSetFlags = NULL + * cdef void *__cuCtxGetId = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetFlags = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":50 + * cdef void *__cuCtxGetDevice = NULL + * cdef void *__cuCtxGetFlags = NULL + * cdef void *__cuCtxSetFlags = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxGetId = NULL + * cdef void *__cuCtxSynchronize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetFlags = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":51 + * cdef void *__cuCtxGetFlags = NULL + * cdef void *__cuCtxSetFlags = NULL + * cdef void *__cuCtxGetId = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxSynchronize = NULL + * cdef void *__cuCtxSetLimit = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetId = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":52 + * cdef void *__cuCtxSetFlags = NULL + * cdef void *__cuCtxGetId = NULL + * cdef void *__cuCtxSynchronize = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxSetLimit = NULL + * cdef void *__cuCtxGetLimit = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSynchronize = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":53 + * cdef void *__cuCtxGetId = NULL + * cdef void *__cuCtxSynchronize = NULL + * cdef void *__cuCtxSetLimit = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxGetLimit = NULL + * cdef void *__cuCtxGetCacheConfig = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetLimit = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":54 + * cdef void *__cuCtxSynchronize = NULL + * cdef void *__cuCtxSetLimit = NULL + * cdef void *__cuCtxGetLimit = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxGetCacheConfig = NULL + * cdef void *__cuCtxSetCacheConfig = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetLimit = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":55 + * cdef void *__cuCtxSetLimit = NULL + * cdef void *__cuCtxGetLimit = NULL + * cdef void *__cuCtxGetCacheConfig = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxSetCacheConfig = NULL + * cdef void *__cuCtxGetApiVersion = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetCacheConfig = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":56 + * cdef void *__cuCtxGetLimit = NULL + * cdef void *__cuCtxGetCacheConfig = NULL + * cdef void *__cuCtxSetCacheConfig = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxGetApiVersion = NULL + * cdef void *__cuCtxGetStreamPriorityRange = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetCacheConfig = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":57 + * cdef void *__cuCtxGetCacheConfig = NULL + * cdef void *__cuCtxSetCacheConfig = NULL + * cdef void *__cuCtxGetApiVersion = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxGetStreamPriorityRange = NULL + * cdef void *__cuCtxResetPersistingL2Cache = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetApiVersion = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":58 + * cdef void *__cuCtxSetCacheConfig = NULL + * cdef void *__cuCtxGetApiVersion = NULL + * cdef void *__cuCtxGetStreamPriorityRange = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxResetPersistingL2Cache = NULL + * cdef void *__cuCtxGetExecAffinity = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetStreamPriorityRange = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":59 + * cdef void *__cuCtxGetApiVersion = NULL + * cdef void *__cuCtxGetStreamPriorityRange = NULL + * cdef void *__cuCtxResetPersistingL2Cache = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxGetExecAffinity = NULL + * cdef void *__cuCtxRecordEvent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxResetPersistingL2Cache = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":60 + * cdef void *__cuCtxGetStreamPriorityRange = NULL + * cdef void *__cuCtxResetPersistingL2Cache = NULL + * cdef void *__cuCtxGetExecAffinity = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxRecordEvent = NULL + * cdef void *__cuCtxWaitEvent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetExecAffinity = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":61 + * cdef void *__cuCtxResetPersistingL2Cache = NULL + * cdef void *__cuCtxGetExecAffinity = NULL + * cdef void *__cuCtxRecordEvent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxWaitEvent = NULL + * cdef void *__cuCtxAttach = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxRecordEvent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":62 + * cdef void *__cuCtxGetExecAffinity = NULL + * cdef void *__cuCtxRecordEvent = NULL + * cdef void *__cuCtxWaitEvent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxAttach = NULL + * cdef void *__cuCtxDetach = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxWaitEvent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":63 + * cdef void *__cuCtxRecordEvent = NULL + * cdef void *__cuCtxWaitEvent = NULL + * cdef void *__cuCtxAttach = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxDetach = NULL + * cdef void *__cuCtxGetSharedMemConfig = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxAttach = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":64 + * cdef void *__cuCtxWaitEvent = NULL + * cdef void *__cuCtxAttach = NULL + * cdef void *__cuCtxDetach = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxGetSharedMemConfig = NULL + * cdef void *__cuCtxSetSharedMemConfig = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDetach = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":65 + * cdef void *__cuCtxAttach = NULL + * cdef void *__cuCtxDetach = NULL + * cdef void *__cuCtxGetSharedMemConfig = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxSetSharedMemConfig = NULL + * cdef void *__cuModuleLoad = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetSharedMemConfig = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":66 + * cdef void *__cuCtxDetach = NULL + * cdef void *__cuCtxGetSharedMemConfig = NULL + * cdef void *__cuCtxSetSharedMemConfig = NULL # <<<<<<<<<<<<<< + * cdef void *__cuModuleLoad = NULL + * cdef void *__cuModuleLoadData = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxSetSharedMemConfig = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":67 + * cdef void *__cuCtxGetSharedMemConfig = NULL + * cdef void *__cuCtxSetSharedMemConfig = NULL + * cdef void *__cuModuleLoad = NULL # <<<<<<<<<<<<<< + * cdef void *__cuModuleLoadData = NULL + * cdef void *__cuModuleLoadDataEx = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoad = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":68 + * cdef void *__cuCtxSetSharedMemConfig = NULL + * cdef void *__cuModuleLoad = NULL + * cdef void *__cuModuleLoadData = NULL # <<<<<<<<<<<<<< + * cdef void *__cuModuleLoadDataEx = NULL + * cdef void *__cuModuleLoadFatBinary = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadData = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":69 + * cdef void *__cuModuleLoad = NULL + * cdef void *__cuModuleLoadData = NULL + * cdef void *__cuModuleLoadDataEx = NULL # <<<<<<<<<<<<<< + * cdef void *__cuModuleLoadFatBinary = NULL + * cdef void *__cuModuleUnload = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadDataEx = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":70 + * cdef void *__cuModuleLoadData = NULL + * cdef void *__cuModuleLoadDataEx = NULL + * cdef void *__cuModuleLoadFatBinary = NULL # <<<<<<<<<<<<<< + * cdef void *__cuModuleUnload = NULL + * cdef void *__cuModuleGetLoadingMode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleLoadFatBinary = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":71 + * cdef void *__cuModuleLoadDataEx = NULL + * cdef void *__cuModuleLoadFatBinary = NULL + * cdef void *__cuModuleUnload = NULL # <<<<<<<<<<<<<< + * cdef void *__cuModuleGetLoadingMode = NULL + * cdef void *__cuModuleGetFunction = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleUnload = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":72 + * cdef void *__cuModuleLoadFatBinary = NULL + * cdef void *__cuModuleUnload = NULL + * cdef void *__cuModuleGetLoadingMode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuModuleGetFunction = NULL + * cdef void *__cuModuleGetFunctionCount = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetLoadingMode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":73 + * cdef void *__cuModuleUnload = NULL + * cdef void *__cuModuleGetLoadingMode = NULL + * cdef void *__cuModuleGetFunction = NULL # <<<<<<<<<<<<<< + * cdef void *__cuModuleGetFunctionCount = NULL + * cdef void *__cuModuleEnumerateFunctions = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunction = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":74 + * cdef void *__cuModuleGetLoadingMode = NULL + * cdef void *__cuModuleGetFunction = NULL + * cdef void *__cuModuleGetFunctionCount = NULL # <<<<<<<<<<<<<< + * cdef void *__cuModuleEnumerateFunctions = NULL + * cdef void *__cuModuleGetGlobal_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetFunctionCount = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":75 + * cdef void *__cuModuleGetFunction = NULL + * cdef void *__cuModuleGetFunctionCount = NULL + * cdef void *__cuModuleEnumerateFunctions = NULL # <<<<<<<<<<<<<< + * cdef void *__cuModuleGetGlobal_v2 = NULL + * cdef void *__cuLinkCreate_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleEnumerateFunctions = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":76 + * cdef void *__cuModuleGetFunctionCount = NULL + * cdef void *__cuModuleEnumerateFunctions = NULL + * cdef void *__cuModuleGetGlobal_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLinkCreate_v2 = NULL + * cdef void *__cuLinkAddData_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetGlobal_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":77 + * cdef void *__cuModuleEnumerateFunctions = NULL + * cdef void *__cuModuleGetGlobal_v2 = NULL + * cdef void *__cuLinkCreate_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLinkAddData_v2 = NULL + * cdef void *__cuLinkAddFile_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkCreate_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":78 + * cdef void *__cuModuleGetGlobal_v2 = NULL + * cdef void *__cuLinkCreate_v2 = NULL + * cdef void *__cuLinkAddData_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLinkAddFile_v2 = NULL + * cdef void *__cuLinkComplete = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddData_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":79 + * cdef void *__cuLinkCreate_v2 = NULL + * cdef void *__cuLinkAddData_v2 = NULL + * cdef void *__cuLinkAddFile_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLinkComplete = NULL + * cdef void *__cuLinkDestroy = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkAddFile_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":80 + * cdef void *__cuLinkAddData_v2 = NULL + * cdef void *__cuLinkAddFile_v2 = NULL + * cdef void *__cuLinkComplete = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLinkDestroy = NULL + * cdef void *__cuModuleGetTexRef = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkComplete = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":81 + * cdef void *__cuLinkAddFile_v2 = NULL + * cdef void *__cuLinkComplete = NULL + * cdef void *__cuLinkDestroy = NULL # <<<<<<<<<<<<<< + * cdef void *__cuModuleGetTexRef = NULL + * cdef void *__cuModuleGetSurfRef = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLinkDestroy = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":82 + * cdef void *__cuLinkComplete = NULL + * cdef void *__cuLinkDestroy = NULL + * cdef void *__cuModuleGetTexRef = NULL # <<<<<<<<<<<<<< + * cdef void *__cuModuleGetSurfRef = NULL + * cdef void *__cuLibraryLoadData = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetTexRef = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":83 + * cdef void *__cuLinkDestroy = NULL + * cdef void *__cuModuleGetTexRef = NULL + * cdef void *__cuModuleGetSurfRef = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLibraryLoadData = NULL + * cdef void *__cuLibraryLoadFromFile = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuModuleGetSurfRef = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":84 + * cdef void *__cuModuleGetTexRef = NULL + * cdef void *__cuModuleGetSurfRef = NULL + * cdef void *__cuLibraryLoadData = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLibraryLoadFromFile = NULL + * cdef void *__cuLibraryUnload = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadData = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":85 + * cdef void *__cuModuleGetSurfRef = NULL + * cdef void *__cuLibraryLoadData = NULL + * cdef void *__cuLibraryLoadFromFile = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLibraryUnload = NULL + * cdef void *__cuLibraryGetKernel = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryLoadFromFile = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":86 + * cdef void *__cuLibraryLoadData = NULL + * cdef void *__cuLibraryLoadFromFile = NULL + * cdef void *__cuLibraryUnload = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLibraryGetKernel = NULL + * cdef void *__cuLibraryGetKernelCount = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryUnload = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":87 + * cdef void *__cuLibraryLoadFromFile = NULL + * cdef void *__cuLibraryUnload = NULL + * cdef void *__cuLibraryGetKernel = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLibraryGetKernelCount = NULL + * cdef void *__cuLibraryEnumerateKernels = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernel = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":88 + * cdef void *__cuLibraryUnload = NULL + * cdef void *__cuLibraryGetKernel = NULL + * cdef void *__cuLibraryGetKernelCount = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLibraryEnumerateKernels = NULL + * cdef void *__cuLibraryGetModule = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetKernelCount = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":89 + * cdef void *__cuLibraryGetKernel = NULL + * cdef void *__cuLibraryGetKernelCount = NULL + * cdef void *__cuLibraryEnumerateKernels = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLibraryGetModule = NULL + * cdef void *__cuKernelGetFunction = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryEnumerateKernels = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":90 + * cdef void *__cuLibraryGetKernelCount = NULL + * cdef void *__cuLibraryEnumerateKernels = NULL + * cdef void *__cuLibraryGetModule = NULL # <<<<<<<<<<<<<< + * cdef void *__cuKernelGetFunction = NULL + * cdef void *__cuKernelGetLibrary = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetModule = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":91 + * cdef void *__cuLibraryEnumerateKernels = NULL + * cdef void *__cuLibraryGetModule = NULL + * cdef void *__cuKernelGetFunction = NULL # <<<<<<<<<<<<<< + * cdef void *__cuKernelGetLibrary = NULL + * cdef void *__cuLibraryGetGlobal = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetFunction = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":92 + * cdef void *__cuLibraryGetModule = NULL + * cdef void *__cuKernelGetFunction = NULL + * cdef void *__cuKernelGetLibrary = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLibraryGetGlobal = NULL + * cdef void *__cuLibraryGetManaged = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetLibrary = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":93 + * cdef void *__cuKernelGetFunction = NULL + * cdef void *__cuKernelGetLibrary = NULL + * cdef void *__cuLibraryGetGlobal = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLibraryGetManaged = NULL + * cdef void *__cuLibraryGetUnifiedFunction = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetGlobal = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":94 + * cdef void *__cuKernelGetLibrary = NULL + * cdef void *__cuLibraryGetGlobal = NULL + * cdef void *__cuLibraryGetManaged = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLibraryGetUnifiedFunction = NULL + * cdef void *__cuKernelGetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetManaged = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":95 + * cdef void *__cuLibraryGetGlobal = NULL + * cdef void *__cuLibraryGetManaged = NULL + * cdef void *__cuLibraryGetUnifiedFunction = NULL # <<<<<<<<<<<<<< + * cdef void *__cuKernelGetAttribute = NULL + * cdef void *__cuKernelSetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLibraryGetUnifiedFunction = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":96 + * cdef void *__cuLibraryGetManaged = NULL + * cdef void *__cuLibraryGetUnifiedFunction = NULL + * cdef void *__cuKernelGetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuKernelSetAttribute = NULL + * cdef void *__cuKernelSetCacheConfig = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":97 + * cdef void *__cuLibraryGetUnifiedFunction = NULL + * cdef void *__cuKernelGetAttribute = NULL + * cdef void *__cuKernelSetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuKernelSetCacheConfig = NULL + * cdef void *__cuKernelGetName = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":98 + * cdef void *__cuKernelGetAttribute = NULL + * cdef void *__cuKernelSetAttribute = NULL + * cdef void *__cuKernelSetCacheConfig = NULL # <<<<<<<<<<<<<< + * cdef void *__cuKernelGetName = NULL + * cdef void *__cuKernelGetParamInfo = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelSetCacheConfig = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":99 + * cdef void *__cuKernelSetAttribute = NULL + * cdef void *__cuKernelSetCacheConfig = NULL + * cdef void *__cuKernelGetName = NULL # <<<<<<<<<<<<<< + * cdef void *__cuKernelGetParamInfo = NULL + * cdef void *__cuMemGetInfo_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetName = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":100 + * cdef void *__cuKernelSetCacheConfig = NULL + * cdef void *__cuKernelGetName = NULL + * cdef void *__cuKernelGetParamInfo = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemGetInfo_v2 = NULL + * cdef void *__cuMemAlloc_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuKernelGetParamInfo = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":101 + * cdef void *__cuKernelGetName = NULL + * cdef void *__cuKernelGetParamInfo = NULL + * cdef void *__cuMemGetInfo_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemAlloc_v2 = NULL + * cdef void *__cuMemAllocPitch_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetInfo_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":102 + * cdef void *__cuKernelGetParamInfo = NULL + * cdef void *__cuMemGetInfo_v2 = NULL + * cdef void *__cuMemAlloc_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemAllocPitch_v2 = NULL + * cdef void *__cuMemFree_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAlloc_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":103 + * cdef void *__cuMemGetInfo_v2 = NULL + * cdef void *__cuMemAlloc_v2 = NULL + * cdef void *__cuMemAllocPitch_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemFree_v2 = NULL + * cdef void *__cuMemGetAddressRange_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocPitch_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":104 + * cdef void *__cuMemAlloc_v2 = NULL + * cdef void *__cuMemAllocPitch_v2 = NULL + * cdef void *__cuMemFree_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemGetAddressRange_v2 = NULL + * cdef void *__cuMemAllocHost_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFree_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":105 + * cdef void *__cuMemAllocPitch_v2 = NULL + * cdef void *__cuMemFree_v2 = NULL + * cdef void *__cuMemGetAddressRange_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemAllocHost_v2 = NULL + * cdef void *__cuMemFreeHost = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAddressRange_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":106 + * cdef void *__cuMemFree_v2 = NULL + * cdef void *__cuMemGetAddressRange_v2 = NULL + * cdef void *__cuMemAllocHost_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemFreeHost = NULL + * cdef void *__cuMemHostAlloc = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocHost_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":107 + * cdef void *__cuMemGetAddressRange_v2 = NULL + * cdef void *__cuMemAllocHost_v2 = NULL + * cdef void *__cuMemFreeHost = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemHostAlloc = NULL + * cdef void *__cuMemHostGetDevicePointer_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeHost = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":108 + * cdef void *__cuMemAllocHost_v2 = NULL + * cdef void *__cuMemFreeHost = NULL + * cdef void *__cuMemHostAlloc = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemHostGetDevicePointer_v2 = NULL + * cdef void *__cuMemHostGetFlags = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostAlloc = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":109 + * cdef void *__cuMemFreeHost = NULL + * cdef void *__cuMemHostAlloc = NULL + * cdef void *__cuMemHostGetDevicePointer_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemHostGetFlags = NULL + * cdef void *__cuMemAllocManaged = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetDevicePointer_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":110 + * cdef void *__cuMemHostAlloc = NULL + * cdef void *__cuMemHostGetDevicePointer_v2 = NULL + * cdef void *__cuMemHostGetFlags = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemAllocManaged = NULL + * cdef void *__cuDeviceRegisterAsyncNotification = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostGetFlags = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":111 + * cdef void *__cuMemHostGetDevicePointer_v2 = NULL + * cdef void *__cuMemHostGetFlags = NULL + * cdef void *__cuMemAllocManaged = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceRegisterAsyncNotification = NULL + * cdef void *__cuDeviceUnregisterAsyncNotification = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocManaged = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":112 + * cdef void *__cuMemHostGetFlags = NULL + * cdef void *__cuMemAllocManaged = NULL + * cdef void *__cuDeviceRegisterAsyncNotification = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceUnregisterAsyncNotification = NULL + * cdef void *__cuDeviceGetByPCIBusId = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceRegisterAsyncNotification = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":113 + * cdef void *__cuMemAllocManaged = NULL + * cdef void *__cuDeviceRegisterAsyncNotification = NULL + * cdef void *__cuDeviceUnregisterAsyncNotification = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetByPCIBusId = NULL + * cdef void *__cuDeviceGetPCIBusId = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceUnregisterAsyncNotification = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":114 + * cdef void *__cuDeviceRegisterAsyncNotification = NULL + * cdef void *__cuDeviceUnregisterAsyncNotification = NULL + * cdef void *__cuDeviceGetByPCIBusId = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetPCIBusId = NULL + * cdef void *__cuIpcGetEventHandle = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetByPCIBusId = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":115 + * cdef void *__cuDeviceUnregisterAsyncNotification = NULL + * cdef void *__cuDeviceGetByPCIBusId = NULL + * cdef void *__cuDeviceGetPCIBusId = NULL # <<<<<<<<<<<<<< + * cdef void *__cuIpcGetEventHandle = NULL + * cdef void *__cuIpcOpenEventHandle = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetPCIBusId = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":116 + * cdef void *__cuDeviceGetByPCIBusId = NULL + * cdef void *__cuDeviceGetPCIBusId = NULL + * cdef void *__cuIpcGetEventHandle = NULL # <<<<<<<<<<<<<< + * cdef void *__cuIpcOpenEventHandle = NULL + * cdef void *__cuIpcGetMemHandle = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetEventHandle = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":117 + * cdef void *__cuDeviceGetPCIBusId = NULL + * cdef void *__cuIpcGetEventHandle = NULL + * cdef void *__cuIpcOpenEventHandle = NULL # <<<<<<<<<<<<<< + * cdef void *__cuIpcGetMemHandle = NULL + * cdef void *__cuIpcOpenMemHandle_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenEventHandle = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":118 + * cdef void *__cuIpcGetEventHandle = NULL + * cdef void *__cuIpcOpenEventHandle = NULL + * cdef void *__cuIpcGetMemHandle = NULL # <<<<<<<<<<<<<< + * cdef void *__cuIpcOpenMemHandle_v2 = NULL + * cdef void *__cuIpcCloseMemHandle = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcGetMemHandle = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":119 + * cdef void *__cuIpcOpenEventHandle = NULL + * cdef void *__cuIpcGetMemHandle = NULL + * cdef void *__cuIpcOpenMemHandle_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuIpcCloseMemHandle = NULL + * cdef void *__cuMemHostRegister_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcOpenMemHandle_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":120 + * cdef void *__cuIpcGetMemHandle = NULL + * cdef void *__cuIpcOpenMemHandle_v2 = NULL + * cdef void *__cuIpcCloseMemHandle = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemHostRegister_v2 = NULL + * cdef void *__cuMemHostUnregister = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuIpcCloseMemHandle = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":121 + * cdef void *__cuIpcOpenMemHandle_v2 = NULL + * cdef void *__cuIpcCloseMemHandle = NULL + * cdef void *__cuMemHostRegister_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemHostUnregister = NULL + * cdef void *__cuMemcpy = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostRegister_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":122 + * cdef void *__cuIpcCloseMemHandle = NULL + * cdef void *__cuMemHostRegister_v2 = NULL + * cdef void *__cuMemHostUnregister = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpy = NULL + * cdef void *__cuMemcpyPeer = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemHostUnregister = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":123 + * cdef void *__cuMemHostRegister_v2 = NULL + * cdef void *__cuMemHostUnregister = NULL + * cdef void *__cuMemcpy = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyPeer = NULL + * cdef void *__cuMemcpyHtoD_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":124 + * cdef void *__cuMemHostUnregister = NULL + * cdef void *__cuMemcpy = NULL + * cdef void *__cuMemcpyPeer = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyHtoD_v2 = NULL + * cdef void *__cuMemcpyDtoH_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeer = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":125 + * cdef void *__cuMemcpy = NULL + * cdef void *__cuMemcpyPeer = NULL + * cdef void *__cuMemcpyHtoD_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyDtoH_v2 = NULL + * cdef void *__cuMemcpyDtoD_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoD_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":126 + * cdef void *__cuMemcpyPeer = NULL + * cdef void *__cuMemcpyHtoD_v2 = NULL + * cdef void *__cuMemcpyDtoH_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyDtoD_v2 = NULL + * cdef void *__cuMemcpyDtoA_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoH_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":127 + * cdef void *__cuMemcpyHtoD_v2 = NULL + * cdef void *__cuMemcpyDtoH_v2 = NULL + * cdef void *__cuMemcpyDtoD_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyDtoA_v2 = NULL + * cdef void *__cuMemcpyAtoD_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoD_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":128 + * cdef void *__cuMemcpyDtoH_v2 = NULL + * cdef void *__cuMemcpyDtoD_v2 = NULL + * cdef void *__cuMemcpyDtoA_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyAtoD_v2 = NULL + * cdef void *__cuMemcpyHtoA_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoA_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":129 + * cdef void *__cuMemcpyDtoD_v2 = NULL + * cdef void *__cuMemcpyDtoA_v2 = NULL + * cdef void *__cuMemcpyAtoD_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyHtoA_v2 = NULL + * cdef void *__cuMemcpyAtoH_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoD_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":130 + * cdef void *__cuMemcpyDtoA_v2 = NULL + * cdef void *__cuMemcpyAtoD_v2 = NULL + * cdef void *__cuMemcpyHtoA_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyAtoH_v2 = NULL + * cdef void *__cuMemcpyAtoA_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoA_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":131 + * cdef void *__cuMemcpyAtoD_v2 = NULL + * cdef void *__cuMemcpyHtoA_v2 = NULL + * cdef void *__cuMemcpyAtoH_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyAtoA_v2 = NULL + * cdef void *__cuMemcpy2D_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoH_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":132 + * cdef void *__cuMemcpyHtoA_v2 = NULL + * cdef void *__cuMemcpyAtoH_v2 = NULL + * cdef void *__cuMemcpyAtoA_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpy2D_v2 = NULL + * cdef void *__cuMemcpy2DUnaligned_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoA_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":133 + * cdef void *__cuMemcpyAtoH_v2 = NULL + * cdef void *__cuMemcpyAtoA_v2 = NULL + * cdef void *__cuMemcpy2D_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpy2DUnaligned_v2 = NULL + * cdef void *__cuMemcpy3D_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2D_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":134 + * cdef void *__cuMemcpyAtoA_v2 = NULL + * cdef void *__cuMemcpy2D_v2 = NULL + * cdef void *__cuMemcpy2DUnaligned_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpy3D_v2 = NULL + * cdef void *__cuMemcpy3DPeer = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DUnaligned_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":135 + * cdef void *__cuMemcpy2D_v2 = NULL + * cdef void *__cuMemcpy2DUnaligned_v2 = NULL + * cdef void *__cuMemcpy3D_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpy3DPeer = NULL + * cdef void *__cuMemcpyAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3D_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":136 + * cdef void *__cuMemcpy2DUnaligned_v2 = NULL + * cdef void *__cuMemcpy3D_v2 = NULL + * cdef void *__cuMemcpy3DPeer = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyAsync = NULL + * cdef void *__cuMemcpyPeerAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeer = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":137 + * cdef void *__cuMemcpy3D_v2 = NULL + * cdef void *__cuMemcpy3DPeer = NULL + * cdef void *__cuMemcpyAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyPeerAsync = NULL + * cdef void *__cuMemcpyHtoDAsync_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":138 + * cdef void *__cuMemcpy3DPeer = NULL + * cdef void *__cuMemcpyAsync = NULL + * cdef void *__cuMemcpyPeerAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyHtoDAsync_v2 = NULL + * cdef void *__cuMemcpyDtoHAsync_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyPeerAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":139 + * cdef void *__cuMemcpyAsync = NULL + * cdef void *__cuMemcpyPeerAsync = NULL + * cdef void *__cuMemcpyHtoDAsync_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyDtoHAsync_v2 = NULL + * cdef void *__cuMemcpyDtoDAsync_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoDAsync_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":140 + * cdef void *__cuMemcpyPeerAsync = NULL + * cdef void *__cuMemcpyHtoDAsync_v2 = NULL + * cdef void *__cuMemcpyDtoHAsync_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyDtoDAsync_v2 = NULL + * cdef void *__cuMemcpyHtoAAsync_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoHAsync_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":141 + * cdef void *__cuMemcpyHtoDAsync_v2 = NULL + * cdef void *__cuMemcpyDtoHAsync_v2 = NULL + * cdef void *__cuMemcpyDtoDAsync_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyHtoAAsync_v2 = NULL + * cdef void *__cuMemcpyAtoHAsync_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyDtoDAsync_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":142 + * cdef void *__cuMemcpyDtoHAsync_v2 = NULL + * cdef void *__cuMemcpyDtoDAsync_v2 = NULL + * cdef void *__cuMemcpyHtoAAsync_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyAtoHAsync_v2 = NULL + * cdef void *__cuMemcpy2DAsync_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyHtoAAsync_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":143 + * cdef void *__cuMemcpyDtoDAsync_v2 = NULL + * cdef void *__cuMemcpyHtoAAsync_v2 = NULL + * cdef void *__cuMemcpyAtoHAsync_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpy2DAsync_v2 = NULL + * cdef void *__cuMemcpy3DAsync_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyAtoHAsync_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":144 + * cdef void *__cuMemcpyHtoAAsync_v2 = NULL + * cdef void *__cuMemcpyAtoHAsync_v2 = NULL + * cdef void *__cuMemcpy2DAsync_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpy3DAsync_v2 = NULL + * cdef void *__cuMemcpy3DPeerAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy2DAsync_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":145 + * cdef void *__cuMemcpyAtoHAsync_v2 = NULL + * cdef void *__cuMemcpy2DAsync_v2 = NULL + * cdef void *__cuMemcpy3DAsync_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpy3DPeerAsync = NULL + * cdef void *__cuMemcpyBatchAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DAsync_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":146 + * cdef void *__cuMemcpy2DAsync_v2 = NULL + * cdef void *__cuMemcpy3DAsync_v2 = NULL + * cdef void *__cuMemcpy3DPeerAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpyBatchAsync = NULL + * cdef void *__cuMemcpy3DBatchAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DPeerAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":147 + * cdef void *__cuMemcpy3DAsync_v2 = NULL + * cdef void *__cuMemcpy3DPeerAsync = NULL + * cdef void *__cuMemcpyBatchAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemcpy3DBatchAsync = NULL + * cdef void *__cuMemsetD8_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpyBatchAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":148 + * cdef void *__cuMemcpy3DPeerAsync = NULL + * cdef void *__cuMemcpyBatchAsync = NULL + * cdef void *__cuMemcpy3DBatchAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemsetD8_v2 = NULL + * cdef void *__cuMemsetD16_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemcpy3DBatchAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":149 + * cdef void *__cuMemcpyBatchAsync = NULL + * cdef void *__cuMemcpy3DBatchAsync = NULL + * cdef void *__cuMemsetD8_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemsetD16_v2 = NULL + * cdef void *__cuMemsetD32_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":150 + * cdef void *__cuMemcpy3DBatchAsync = NULL + * cdef void *__cuMemsetD8_v2 = NULL + * cdef void *__cuMemsetD16_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemsetD32_v2 = NULL + * cdef void *__cuMemsetD2D8_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":151 + * cdef void *__cuMemsetD8_v2 = NULL + * cdef void *__cuMemsetD16_v2 = NULL + * cdef void *__cuMemsetD32_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemsetD2D8_v2 = NULL + * cdef void *__cuMemsetD2D16_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":152 + * cdef void *__cuMemsetD16_v2 = NULL + * cdef void *__cuMemsetD32_v2 = NULL + * cdef void *__cuMemsetD2D8_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemsetD2D16_v2 = NULL + * cdef void *__cuMemsetD2D32_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":153 + * cdef void *__cuMemsetD32_v2 = NULL + * cdef void *__cuMemsetD2D8_v2 = NULL + * cdef void *__cuMemsetD2D16_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemsetD2D32_v2 = NULL + * cdef void *__cuMemsetD8Async = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":154 + * cdef void *__cuMemsetD2D8_v2 = NULL + * cdef void *__cuMemsetD2D16_v2 = NULL + * cdef void *__cuMemsetD2D32_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemsetD8Async = NULL + * cdef void *__cuMemsetD16Async = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":155 + * cdef void *__cuMemsetD2D16_v2 = NULL + * cdef void *__cuMemsetD2D32_v2 = NULL + * cdef void *__cuMemsetD8Async = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemsetD16Async = NULL + * cdef void *__cuMemsetD32Async = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD8Async = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":156 + * cdef void *__cuMemsetD2D32_v2 = NULL + * cdef void *__cuMemsetD8Async = NULL + * cdef void *__cuMemsetD16Async = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemsetD32Async = NULL + * cdef void *__cuMemsetD2D8Async = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD16Async = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":157 + * cdef void *__cuMemsetD8Async = NULL + * cdef void *__cuMemsetD16Async = NULL + * cdef void *__cuMemsetD32Async = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemsetD2D8Async = NULL + * cdef void *__cuMemsetD2D16Async = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD32Async = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":158 + * cdef void *__cuMemsetD16Async = NULL + * cdef void *__cuMemsetD32Async = NULL + * cdef void *__cuMemsetD2D8Async = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemsetD2D16Async = NULL + * cdef void *__cuMemsetD2D32Async = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D8Async = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":159 + * cdef void *__cuMemsetD32Async = NULL + * cdef void *__cuMemsetD2D8Async = NULL + * cdef void *__cuMemsetD2D16Async = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemsetD2D32Async = NULL + * cdef void *__cuArrayCreate_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D16Async = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":160 + * cdef void *__cuMemsetD2D8Async = NULL + * cdef void *__cuMemsetD2D16Async = NULL + * cdef void *__cuMemsetD2D32Async = NULL # <<<<<<<<<<<<<< + * cdef void *__cuArrayCreate_v2 = NULL + * cdef void *__cuArrayGetDescriptor_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemsetD2D32Async = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":161 + * cdef void *__cuMemsetD2D16Async = NULL + * cdef void *__cuMemsetD2D32Async = NULL + * cdef void *__cuArrayCreate_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuArrayGetDescriptor_v2 = NULL + * cdef void *__cuArrayGetSparseProperties = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayCreate_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":162 + * cdef void *__cuMemsetD2D32Async = NULL + * cdef void *__cuArrayCreate_v2 = NULL + * cdef void *__cuArrayGetDescriptor_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuArrayGetSparseProperties = NULL + * cdef void *__cuMipmappedArrayGetSparseProperties = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetDescriptor_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":163 + * cdef void *__cuArrayCreate_v2 = NULL + * cdef void *__cuArrayGetDescriptor_v2 = NULL + * cdef void *__cuArrayGetSparseProperties = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMipmappedArrayGetSparseProperties = NULL + * cdef void *__cuArrayGetMemoryRequirements = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetSparseProperties = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":164 + * cdef void *__cuArrayGetDescriptor_v2 = NULL + * cdef void *__cuArrayGetSparseProperties = NULL + * cdef void *__cuMipmappedArrayGetSparseProperties = NULL # <<<<<<<<<<<<<< + * cdef void *__cuArrayGetMemoryRequirements = NULL + * cdef void *__cuMipmappedArrayGetMemoryRequirements = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetSparseProperties = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":165 + * cdef void *__cuArrayGetSparseProperties = NULL + * cdef void *__cuMipmappedArrayGetSparseProperties = NULL + * cdef void *__cuArrayGetMemoryRequirements = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMipmappedArrayGetMemoryRequirements = NULL + * cdef void *__cuArrayGetPlane = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetMemoryRequirements = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":166 + * cdef void *__cuMipmappedArrayGetSparseProperties = NULL + * cdef void *__cuArrayGetMemoryRequirements = NULL + * cdef void *__cuMipmappedArrayGetMemoryRequirements = NULL # <<<<<<<<<<<<<< + * cdef void *__cuArrayGetPlane = NULL + * cdef void *__cuArrayDestroy = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetMemoryRequirements = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":167 + * cdef void *__cuArrayGetMemoryRequirements = NULL + * cdef void *__cuMipmappedArrayGetMemoryRequirements = NULL + * cdef void *__cuArrayGetPlane = NULL # <<<<<<<<<<<<<< + * cdef void *__cuArrayDestroy = NULL + * cdef void *__cuArray3DCreate_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayGetPlane = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":168 + * cdef void *__cuMipmappedArrayGetMemoryRequirements = NULL + * cdef void *__cuArrayGetPlane = NULL + * cdef void *__cuArrayDestroy = NULL # <<<<<<<<<<<<<< + * cdef void *__cuArray3DCreate_v2 = NULL + * cdef void *__cuArray3DGetDescriptor_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArrayDestroy = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":169 + * cdef void *__cuArrayGetPlane = NULL + * cdef void *__cuArrayDestroy = NULL + * cdef void *__cuArray3DCreate_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuArray3DGetDescriptor_v2 = NULL + * cdef void *__cuMipmappedArrayCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DCreate_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":170 + * cdef void *__cuArrayDestroy = NULL + * cdef void *__cuArray3DCreate_v2 = NULL + * cdef void *__cuArray3DGetDescriptor_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMipmappedArrayCreate = NULL + * cdef void *__cuMipmappedArrayGetLevel = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuArray3DGetDescriptor_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":171 + * cdef void *__cuArray3DCreate_v2 = NULL + * cdef void *__cuArray3DGetDescriptor_v2 = NULL + * cdef void *__cuMipmappedArrayCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMipmappedArrayGetLevel = NULL + * cdef void *__cuMipmappedArrayDestroy = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":172 + * cdef void *__cuArray3DGetDescriptor_v2 = NULL + * cdef void *__cuMipmappedArrayCreate = NULL + * cdef void *__cuMipmappedArrayGetLevel = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMipmappedArrayDestroy = NULL + * cdef void *__cuMemGetHandleForAddressRange = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayGetLevel = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":173 + * cdef void *__cuMipmappedArrayCreate = NULL + * cdef void *__cuMipmappedArrayGetLevel = NULL + * cdef void *__cuMipmappedArrayDestroy = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemGetHandleForAddressRange = NULL + * cdef void *__cuMemBatchDecompressAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMipmappedArrayDestroy = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":174 + * cdef void *__cuMipmappedArrayGetLevel = NULL + * cdef void *__cuMipmappedArrayDestroy = NULL + * cdef void *__cuMemGetHandleForAddressRange = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemBatchDecompressAsync = NULL + * cdef void *__cuMemAddressReserve = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetHandleForAddressRange = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":175 + * cdef void *__cuMipmappedArrayDestroy = NULL + * cdef void *__cuMemGetHandleForAddressRange = NULL + * cdef void *__cuMemBatchDecompressAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemAddressReserve = NULL + * cdef void *__cuMemAddressFree = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemBatchDecompressAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":176 + * cdef void *__cuMemGetHandleForAddressRange = NULL + * cdef void *__cuMemBatchDecompressAsync = NULL + * cdef void *__cuMemAddressReserve = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemAddressFree = NULL + * cdef void *__cuMemCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressReserve = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":177 + * cdef void *__cuMemBatchDecompressAsync = NULL + * cdef void *__cuMemAddressReserve = NULL + * cdef void *__cuMemAddressFree = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemCreate = NULL + * cdef void *__cuMemRelease = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAddressFree = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":178 + * cdef void *__cuMemAddressReserve = NULL + * cdef void *__cuMemAddressFree = NULL + * cdef void *__cuMemCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemRelease = NULL + * cdef void *__cuMemMap = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":179 + * cdef void *__cuMemAddressFree = NULL + * cdef void *__cuMemCreate = NULL + * cdef void *__cuMemRelease = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemMap = NULL + * cdef void *__cuMemMapArrayAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRelease = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":180 + * cdef void *__cuMemCreate = NULL + * cdef void *__cuMemRelease = NULL + * cdef void *__cuMemMap = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemMapArrayAsync = NULL + * cdef void *__cuMemUnmap = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMap = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":181 + * cdef void *__cuMemRelease = NULL + * cdef void *__cuMemMap = NULL + * cdef void *__cuMemMapArrayAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemUnmap = NULL + * cdef void *__cuMemSetAccess = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemMapArrayAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":182 + * cdef void *__cuMemMap = NULL + * cdef void *__cuMemMapArrayAsync = NULL + * cdef void *__cuMemUnmap = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemSetAccess = NULL + * cdef void *__cuMemGetAccess = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemUnmap = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":183 + * cdef void *__cuMemMapArrayAsync = NULL + * cdef void *__cuMemUnmap = NULL + * cdef void *__cuMemSetAccess = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemGetAccess = NULL + * cdef void *__cuMemExportToShareableHandle = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemSetAccess = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":184 + * cdef void *__cuMemUnmap = NULL + * cdef void *__cuMemSetAccess = NULL + * cdef void *__cuMemGetAccess = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemExportToShareableHandle = NULL + * cdef void *__cuMemImportFromShareableHandle = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAccess = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":185 + * cdef void *__cuMemSetAccess = NULL + * cdef void *__cuMemGetAccess = NULL + * cdef void *__cuMemExportToShareableHandle = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemImportFromShareableHandle = NULL + * cdef void *__cuMemGetAllocationGranularity = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemExportToShareableHandle = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":186 + * cdef void *__cuMemGetAccess = NULL + * cdef void *__cuMemExportToShareableHandle = NULL + * cdef void *__cuMemImportFromShareableHandle = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemGetAllocationGranularity = NULL + * cdef void *__cuMemGetAllocationPropertiesFromHandle = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemImportFromShareableHandle = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":187 + * cdef void *__cuMemExportToShareableHandle = NULL + * cdef void *__cuMemImportFromShareableHandle = NULL + * cdef void *__cuMemGetAllocationGranularity = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemGetAllocationPropertiesFromHandle = NULL + * cdef void *__cuMemRetainAllocationHandle = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationGranularity = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":188 + * cdef void *__cuMemImportFromShareableHandle = NULL + * cdef void *__cuMemGetAllocationGranularity = NULL + * cdef void *__cuMemGetAllocationPropertiesFromHandle = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemRetainAllocationHandle = NULL + * cdef void *__cuMemFreeAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemGetAllocationPropertiesFromHandle = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":189 + * cdef void *__cuMemGetAllocationGranularity = NULL + * cdef void *__cuMemGetAllocationPropertiesFromHandle = NULL + * cdef void *__cuMemRetainAllocationHandle = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemFreeAsync = NULL + * cdef void *__cuMemAllocAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRetainAllocationHandle = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":190 + * cdef void *__cuMemGetAllocationPropertiesFromHandle = NULL + * cdef void *__cuMemRetainAllocationHandle = NULL + * cdef void *__cuMemFreeAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemAllocAsync = NULL + * cdef void *__cuMemPoolTrimTo = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemFreeAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":191 + * cdef void *__cuMemRetainAllocationHandle = NULL + * cdef void *__cuMemFreeAsync = NULL + * cdef void *__cuMemAllocAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemPoolTrimTo = NULL + * cdef void *__cuMemPoolSetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":192 + * cdef void *__cuMemFreeAsync = NULL + * cdef void *__cuMemAllocAsync = NULL + * cdef void *__cuMemPoolTrimTo = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemPoolSetAttribute = NULL + * cdef void *__cuMemPoolGetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolTrimTo = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":193 + * cdef void *__cuMemAllocAsync = NULL + * cdef void *__cuMemPoolTrimTo = NULL + * cdef void *__cuMemPoolSetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemPoolGetAttribute = NULL + * cdef void *__cuMemPoolSetAccess = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":194 + * cdef void *__cuMemPoolTrimTo = NULL + * cdef void *__cuMemPoolSetAttribute = NULL + * cdef void *__cuMemPoolGetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemPoolSetAccess = NULL + * cdef void *__cuMemPoolGetAccess = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":195 + * cdef void *__cuMemPoolSetAttribute = NULL + * cdef void *__cuMemPoolGetAttribute = NULL + * cdef void *__cuMemPoolSetAccess = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemPoolGetAccess = NULL + * cdef void *__cuMemPoolCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolSetAccess = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":196 + * cdef void *__cuMemPoolGetAttribute = NULL + * cdef void *__cuMemPoolSetAccess = NULL + * cdef void *__cuMemPoolGetAccess = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemPoolCreate = NULL + * cdef void *__cuMemPoolDestroy = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolGetAccess = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":197 + * cdef void *__cuMemPoolSetAccess = NULL + * cdef void *__cuMemPoolGetAccess = NULL + * cdef void *__cuMemPoolCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemPoolDestroy = NULL + * cdef void *__cuMemAllocFromPoolAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":198 + * cdef void *__cuMemPoolGetAccess = NULL + * cdef void *__cuMemPoolCreate = NULL + * cdef void *__cuMemPoolDestroy = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemAllocFromPoolAsync = NULL + * cdef void *__cuMemPoolExportToShareableHandle = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolDestroy = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":199 + * cdef void *__cuMemPoolCreate = NULL + * cdef void *__cuMemPoolDestroy = NULL + * cdef void *__cuMemAllocFromPoolAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemPoolExportToShareableHandle = NULL + * cdef void *__cuMemPoolImportFromShareableHandle = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAllocFromPoolAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":200 + * cdef void *__cuMemPoolDestroy = NULL + * cdef void *__cuMemAllocFromPoolAsync = NULL + * cdef void *__cuMemPoolExportToShareableHandle = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemPoolImportFromShareableHandle = NULL + * cdef void *__cuMemPoolExportPointer = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportToShareableHandle = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":201 + * cdef void *__cuMemAllocFromPoolAsync = NULL + * cdef void *__cuMemPoolExportToShareableHandle = NULL + * cdef void *__cuMemPoolImportFromShareableHandle = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemPoolExportPointer = NULL + * cdef void *__cuMemPoolImportPointer = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportFromShareableHandle = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":202 + * cdef void *__cuMemPoolExportToShareableHandle = NULL + * cdef void *__cuMemPoolImportFromShareableHandle = NULL + * cdef void *__cuMemPoolExportPointer = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemPoolImportPointer = NULL + * cdef void *__cuMulticastCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolExportPointer = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":203 + * cdef void *__cuMemPoolImportFromShareableHandle = NULL + * cdef void *__cuMemPoolExportPointer = NULL + * cdef void *__cuMemPoolImportPointer = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMulticastCreate = NULL + * cdef void *__cuMulticastAddDevice = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPoolImportPointer = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":204 + * cdef void *__cuMemPoolExportPointer = NULL + * cdef void *__cuMemPoolImportPointer = NULL + * cdef void *__cuMulticastCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMulticastAddDevice = NULL + * cdef void *__cuMulticastBindMem = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":205 + * cdef void *__cuMemPoolImportPointer = NULL + * cdef void *__cuMulticastCreate = NULL + * cdef void *__cuMulticastAddDevice = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMulticastBindMem = NULL + * cdef void *__cuMulticastBindAddr = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastAddDevice = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":206 + * cdef void *__cuMulticastCreate = NULL + * cdef void *__cuMulticastAddDevice = NULL + * cdef void *__cuMulticastBindMem = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMulticastBindAddr = NULL + * cdef void *__cuMulticastUnbind = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindMem = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":207 + * cdef void *__cuMulticastAddDevice = NULL + * cdef void *__cuMulticastBindMem = NULL + * cdef void *__cuMulticastBindAddr = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMulticastUnbind = NULL + * cdef void *__cuMulticastGetGranularity = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastBindAddr = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":208 + * cdef void *__cuMulticastBindMem = NULL + * cdef void *__cuMulticastBindAddr = NULL + * cdef void *__cuMulticastUnbind = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMulticastGetGranularity = NULL + * cdef void *__cuPointerGetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastUnbind = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":209 + * cdef void *__cuMulticastBindAddr = NULL + * cdef void *__cuMulticastUnbind = NULL + * cdef void *__cuMulticastGetGranularity = NULL # <<<<<<<<<<<<<< + * cdef void *__cuPointerGetAttribute = NULL + * cdef void *__cuMemPrefetchAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMulticastGetGranularity = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":210 + * cdef void *__cuMulticastUnbind = NULL + * cdef void *__cuMulticastGetGranularity = NULL + * cdef void *__cuPointerGetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemPrefetchAsync = NULL + * cdef void *__cuMemPrefetchAsync_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":211 + * cdef void *__cuMulticastGetGranularity = NULL + * cdef void *__cuPointerGetAttribute = NULL + * cdef void *__cuMemPrefetchAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemPrefetchAsync_v2 = NULL + * cdef void *__cuMemAdvise = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":212 + * cdef void *__cuPointerGetAttribute = NULL + * cdef void *__cuMemPrefetchAsync = NULL + * cdef void *__cuMemPrefetchAsync_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemAdvise = NULL + * cdef void *__cuMemAdvise_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemPrefetchAsync_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":213 + * cdef void *__cuMemPrefetchAsync = NULL + * cdef void *__cuMemPrefetchAsync_v2 = NULL + * cdef void *__cuMemAdvise = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemAdvise_v2 = NULL + * cdef void *__cuMemRangeGetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":214 + * cdef void *__cuMemPrefetchAsync_v2 = NULL + * cdef void *__cuMemAdvise = NULL + * cdef void *__cuMemAdvise_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemRangeGetAttribute = NULL + * cdef void *__cuMemRangeGetAttributes = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemAdvise_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":215 + * cdef void *__cuMemAdvise = NULL + * cdef void *__cuMemAdvise_v2 = NULL + * cdef void *__cuMemRangeGetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuMemRangeGetAttributes = NULL + * cdef void *__cuPointerSetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":216 + * cdef void *__cuMemAdvise_v2 = NULL + * cdef void *__cuMemRangeGetAttribute = NULL + * cdef void *__cuMemRangeGetAttributes = NULL # <<<<<<<<<<<<<< + * cdef void *__cuPointerSetAttribute = NULL + * cdef void *__cuPointerGetAttributes = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuMemRangeGetAttributes = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":217 + * cdef void *__cuMemRangeGetAttribute = NULL + * cdef void *__cuMemRangeGetAttributes = NULL + * cdef void *__cuPointerSetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuPointerGetAttributes = NULL + * cdef void *__cuStreamCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerSetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":218 + * cdef void *__cuMemRangeGetAttributes = NULL + * cdef void *__cuPointerSetAttribute = NULL + * cdef void *__cuPointerGetAttributes = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamCreate = NULL + * cdef void *__cuStreamCreateWithPriority = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuPointerGetAttributes = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":219 + * cdef void *__cuPointerSetAttribute = NULL + * cdef void *__cuPointerGetAttributes = NULL + * cdef void *__cuStreamCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamCreateWithPriority = NULL + * cdef void *__cuStreamGetPriority = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":220 + * cdef void *__cuPointerGetAttributes = NULL + * cdef void *__cuStreamCreate = NULL + * cdef void *__cuStreamCreateWithPriority = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamGetPriority = NULL + * cdef void *__cuStreamGetDevice = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCreateWithPriority = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":221 + * cdef void *__cuStreamCreate = NULL + * cdef void *__cuStreamCreateWithPriority = NULL + * cdef void *__cuStreamGetPriority = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamGetDevice = NULL + * cdef void *__cuStreamGetFlags = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetPriority = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":222 + * cdef void *__cuStreamCreateWithPriority = NULL + * cdef void *__cuStreamGetPriority = NULL + * cdef void *__cuStreamGetDevice = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamGetFlags = NULL + * cdef void *__cuStreamGetId = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetDevice = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":223 + * cdef void *__cuStreamGetPriority = NULL + * cdef void *__cuStreamGetDevice = NULL + * cdef void *__cuStreamGetFlags = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamGetId = NULL + * cdef void *__cuStreamGetCtx = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetFlags = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":224 + * cdef void *__cuStreamGetDevice = NULL + * cdef void *__cuStreamGetFlags = NULL + * cdef void *__cuStreamGetId = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamGetCtx = NULL + * cdef void *__cuStreamGetCtx_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetId = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":225 + * cdef void *__cuStreamGetFlags = NULL + * cdef void *__cuStreamGetId = NULL + * cdef void *__cuStreamGetCtx = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamGetCtx_v2 = NULL + * cdef void *__cuStreamWaitEvent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":226 + * cdef void *__cuStreamGetId = NULL + * cdef void *__cuStreamGetCtx = NULL + * cdef void *__cuStreamGetCtx_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamWaitEvent = NULL + * cdef void *__cuStreamAddCallback = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCtx_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":227 + * cdef void *__cuStreamGetCtx = NULL + * cdef void *__cuStreamGetCtx_v2 = NULL + * cdef void *__cuStreamWaitEvent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamAddCallback = NULL + * cdef void *__cuStreamBeginCapture_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitEvent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":228 + * cdef void *__cuStreamGetCtx_v2 = NULL + * cdef void *__cuStreamWaitEvent = NULL + * cdef void *__cuStreamAddCallback = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamBeginCapture_v2 = NULL + * cdef void *__cuStreamBeginCaptureToGraph = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAddCallback = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":229 + * cdef void *__cuStreamWaitEvent = NULL + * cdef void *__cuStreamAddCallback = NULL + * cdef void *__cuStreamBeginCapture_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamBeginCaptureToGraph = NULL + * cdef void *__cuThreadExchangeStreamCaptureMode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCapture_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":230 + * cdef void *__cuStreamAddCallback = NULL + * cdef void *__cuStreamBeginCapture_v2 = NULL + * cdef void *__cuStreamBeginCaptureToGraph = NULL # <<<<<<<<<<<<<< + * cdef void *__cuThreadExchangeStreamCaptureMode = NULL + * cdef void *__cuStreamEndCapture = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBeginCaptureToGraph = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":231 + * cdef void *__cuStreamBeginCapture_v2 = NULL + * cdef void *__cuStreamBeginCaptureToGraph = NULL + * cdef void *__cuThreadExchangeStreamCaptureMode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamEndCapture = NULL + * cdef void *__cuStreamIsCapturing = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuThreadExchangeStreamCaptureMode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":232 + * cdef void *__cuStreamBeginCaptureToGraph = NULL + * cdef void *__cuThreadExchangeStreamCaptureMode = NULL + * cdef void *__cuStreamEndCapture = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamIsCapturing = NULL + * cdef void *__cuStreamGetCaptureInfo_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamEndCapture = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":233 + * cdef void *__cuThreadExchangeStreamCaptureMode = NULL + * cdef void *__cuStreamEndCapture = NULL + * cdef void *__cuStreamIsCapturing = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamGetCaptureInfo_v2 = NULL + * cdef void *__cuStreamGetCaptureInfo_v3 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamIsCapturing = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":234 + * cdef void *__cuStreamEndCapture = NULL + * cdef void *__cuStreamIsCapturing = NULL + * cdef void *__cuStreamGetCaptureInfo_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamGetCaptureInfo_v3 = NULL + * cdef void *__cuStreamUpdateCaptureDependencies = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":235 + * cdef void *__cuStreamIsCapturing = NULL + * cdef void *__cuStreamGetCaptureInfo_v2 = NULL + * cdef void *__cuStreamGetCaptureInfo_v3 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamUpdateCaptureDependencies = NULL + * cdef void *__cuStreamUpdateCaptureDependencies_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetCaptureInfo_v3 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":236 + * cdef void *__cuStreamGetCaptureInfo_v2 = NULL + * cdef void *__cuStreamGetCaptureInfo_v3 = NULL + * cdef void *__cuStreamUpdateCaptureDependencies = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamUpdateCaptureDependencies_v2 = NULL + * cdef void *__cuStreamAttachMemAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":237 + * cdef void *__cuStreamGetCaptureInfo_v3 = NULL + * cdef void *__cuStreamUpdateCaptureDependencies = NULL + * cdef void *__cuStreamUpdateCaptureDependencies_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamAttachMemAsync = NULL + * cdef void *__cuStreamQuery = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamUpdateCaptureDependencies_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":238 + * cdef void *__cuStreamUpdateCaptureDependencies = NULL + * cdef void *__cuStreamUpdateCaptureDependencies_v2 = NULL + * cdef void *__cuStreamAttachMemAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamQuery = NULL + * cdef void *__cuStreamSynchronize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamAttachMemAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":239 + * cdef void *__cuStreamUpdateCaptureDependencies_v2 = NULL + * cdef void *__cuStreamAttachMemAsync = NULL + * cdef void *__cuStreamQuery = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamSynchronize = NULL + * cdef void *__cuStreamDestroy_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamQuery = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":240 + * cdef void *__cuStreamAttachMemAsync = NULL + * cdef void *__cuStreamQuery = NULL + * cdef void *__cuStreamSynchronize = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamDestroy_v2 = NULL + * cdef void *__cuStreamCopyAttributes = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSynchronize = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":241 + * cdef void *__cuStreamQuery = NULL + * cdef void *__cuStreamSynchronize = NULL + * cdef void *__cuStreamDestroy_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamCopyAttributes = NULL + * cdef void *__cuStreamGetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamDestroy_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":242 + * cdef void *__cuStreamSynchronize = NULL + * cdef void *__cuStreamDestroy_v2 = NULL + * cdef void *__cuStreamCopyAttributes = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamGetAttribute = NULL + * cdef void *__cuStreamSetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamCopyAttributes = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":243 + * cdef void *__cuStreamDestroy_v2 = NULL + * cdef void *__cuStreamCopyAttributes = NULL + * cdef void *__cuStreamGetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamSetAttribute = NULL + * cdef void *__cuEventCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":244 + * cdef void *__cuStreamCopyAttributes = NULL + * cdef void *__cuStreamGetAttribute = NULL + * cdef void *__cuStreamSetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEventCreate = NULL + * cdef void *__cuEventRecord = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamSetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":245 + * cdef void *__cuStreamGetAttribute = NULL + * cdef void *__cuStreamSetAttribute = NULL + * cdef void *__cuEventCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEventRecord = NULL + * cdef void *__cuEventRecordWithFlags = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":246 + * cdef void *__cuStreamSetAttribute = NULL + * cdef void *__cuEventCreate = NULL + * cdef void *__cuEventRecord = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEventRecordWithFlags = NULL + * cdef void *__cuEventQuery = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecord = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":247 + * cdef void *__cuEventCreate = NULL + * cdef void *__cuEventRecord = NULL + * cdef void *__cuEventRecordWithFlags = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEventQuery = NULL + * cdef void *__cuEventSynchronize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventRecordWithFlags = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":248 + * cdef void *__cuEventRecord = NULL + * cdef void *__cuEventRecordWithFlags = NULL + * cdef void *__cuEventQuery = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEventSynchronize = NULL + * cdef void *__cuEventDestroy_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventQuery = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":249 + * cdef void *__cuEventRecordWithFlags = NULL + * cdef void *__cuEventQuery = NULL + * cdef void *__cuEventSynchronize = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEventDestroy_v2 = NULL + * cdef void *__cuEventElapsedTime = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventSynchronize = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":250 + * cdef void *__cuEventQuery = NULL + * cdef void *__cuEventSynchronize = NULL + * cdef void *__cuEventDestroy_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEventElapsedTime = NULL + * cdef void *__cuEventElapsedTime_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventDestroy_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":251 + * cdef void *__cuEventSynchronize = NULL + * cdef void *__cuEventDestroy_v2 = NULL + * cdef void *__cuEventElapsedTime = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEventElapsedTime_v2 = NULL + * cdef void *__cuImportExternalMemory = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":252 + * cdef void *__cuEventDestroy_v2 = NULL + * cdef void *__cuEventElapsedTime = NULL + * cdef void *__cuEventElapsedTime_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuImportExternalMemory = NULL + * cdef void *__cuExternalMemoryGetMappedBuffer = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventElapsedTime_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":253 + * cdef void *__cuEventElapsedTime = NULL + * cdef void *__cuEventElapsedTime_v2 = NULL + * cdef void *__cuImportExternalMemory = NULL # <<<<<<<<<<<<<< + * cdef void *__cuExternalMemoryGetMappedBuffer = NULL + * cdef void *__cuExternalMemoryGetMappedMipmappedArray = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalMemory = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":254 + * cdef void *__cuEventElapsedTime_v2 = NULL + * cdef void *__cuImportExternalMemory = NULL + * cdef void *__cuExternalMemoryGetMappedBuffer = NULL # <<<<<<<<<<<<<< + * cdef void *__cuExternalMemoryGetMappedMipmappedArray = NULL + * cdef void *__cuDestroyExternalMemory = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedBuffer = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":255 + * cdef void *__cuImportExternalMemory = NULL + * cdef void *__cuExternalMemoryGetMappedBuffer = NULL + * cdef void *__cuExternalMemoryGetMappedMipmappedArray = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDestroyExternalMemory = NULL + * cdef void *__cuImportExternalSemaphore = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuExternalMemoryGetMappedMipmappedArray = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":256 + * cdef void *__cuExternalMemoryGetMappedBuffer = NULL + * cdef void *__cuExternalMemoryGetMappedMipmappedArray = NULL + * cdef void *__cuDestroyExternalMemory = NULL # <<<<<<<<<<<<<< + * cdef void *__cuImportExternalSemaphore = NULL + * cdef void *__cuSignalExternalSemaphoresAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalMemory = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":257 + * cdef void *__cuExternalMemoryGetMappedMipmappedArray = NULL + * cdef void *__cuDestroyExternalMemory = NULL + * cdef void *__cuImportExternalSemaphore = NULL # <<<<<<<<<<<<<< + * cdef void *__cuSignalExternalSemaphoresAsync = NULL + * cdef void *__cuWaitExternalSemaphoresAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuImportExternalSemaphore = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":258 + * cdef void *__cuDestroyExternalMemory = NULL + * cdef void *__cuImportExternalSemaphore = NULL + * cdef void *__cuSignalExternalSemaphoresAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuWaitExternalSemaphoresAsync = NULL + * cdef void *__cuDestroyExternalSemaphore = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSignalExternalSemaphoresAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":259 + * cdef void *__cuImportExternalSemaphore = NULL + * cdef void *__cuSignalExternalSemaphoresAsync = NULL + * cdef void *__cuWaitExternalSemaphoresAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDestroyExternalSemaphore = NULL + * cdef void *__cuStreamWaitValue32_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuWaitExternalSemaphoresAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":260 + * cdef void *__cuSignalExternalSemaphoresAsync = NULL + * cdef void *__cuWaitExternalSemaphoresAsync = NULL + * cdef void *__cuDestroyExternalSemaphore = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamWaitValue32_v2 = NULL + * cdef void *__cuStreamWaitValue64_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDestroyExternalSemaphore = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":261 + * cdef void *__cuWaitExternalSemaphoresAsync = NULL + * cdef void *__cuDestroyExternalSemaphore = NULL + * cdef void *__cuStreamWaitValue32_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamWaitValue64_v2 = NULL + * cdef void *__cuStreamWriteValue32_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue32_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":262 + * cdef void *__cuDestroyExternalSemaphore = NULL + * cdef void *__cuStreamWaitValue32_v2 = NULL + * cdef void *__cuStreamWaitValue64_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamWriteValue32_v2 = NULL + * cdef void *__cuStreamWriteValue64_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWaitValue64_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":263 + * cdef void *__cuStreamWaitValue32_v2 = NULL + * cdef void *__cuStreamWaitValue64_v2 = NULL + * cdef void *__cuStreamWriteValue32_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamWriteValue64_v2 = NULL + * cdef void *__cuStreamBatchMemOp_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue32_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":264 + * cdef void *__cuStreamWaitValue64_v2 = NULL + * cdef void *__cuStreamWriteValue32_v2 = NULL + * cdef void *__cuStreamWriteValue64_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamBatchMemOp_v2 = NULL + * cdef void *__cuFuncGetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamWriteValue64_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":265 + * cdef void *__cuStreamWriteValue32_v2 = NULL + * cdef void *__cuStreamWriteValue64_v2 = NULL + * cdef void *__cuStreamBatchMemOp_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuFuncGetAttribute = NULL + * cdef void *__cuFuncSetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamBatchMemOp_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":266 + * cdef void *__cuStreamWriteValue64_v2 = NULL + * cdef void *__cuStreamBatchMemOp_v2 = NULL + * cdef void *__cuFuncGetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuFuncSetAttribute = NULL + * cdef void *__cuFuncSetCacheConfig = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":267 + * cdef void *__cuStreamBatchMemOp_v2 = NULL + * cdef void *__cuFuncGetAttribute = NULL + * cdef void *__cuFuncSetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuFuncSetCacheConfig = NULL + * cdef void *__cuFuncGetModule = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":268 + * cdef void *__cuFuncGetAttribute = NULL + * cdef void *__cuFuncSetAttribute = NULL + * cdef void *__cuFuncSetCacheConfig = NULL # <<<<<<<<<<<<<< + * cdef void *__cuFuncGetModule = NULL + * cdef void *__cuFuncGetName = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetCacheConfig = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":269 + * cdef void *__cuFuncSetAttribute = NULL + * cdef void *__cuFuncSetCacheConfig = NULL + * cdef void *__cuFuncGetModule = NULL # <<<<<<<<<<<<<< + * cdef void *__cuFuncGetName = NULL + * cdef void *__cuFuncGetParamInfo = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetModule = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":270 + * cdef void *__cuFuncSetCacheConfig = NULL + * cdef void *__cuFuncGetModule = NULL + * cdef void *__cuFuncGetName = NULL # <<<<<<<<<<<<<< + * cdef void *__cuFuncGetParamInfo = NULL + * cdef void *__cuFuncIsLoaded = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetName = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":271 + * cdef void *__cuFuncGetModule = NULL + * cdef void *__cuFuncGetName = NULL + * cdef void *__cuFuncGetParamInfo = NULL # <<<<<<<<<<<<<< + * cdef void *__cuFuncIsLoaded = NULL + * cdef void *__cuFuncLoad = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncGetParamInfo = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":272 + * cdef void *__cuFuncGetName = NULL + * cdef void *__cuFuncGetParamInfo = NULL + * cdef void *__cuFuncIsLoaded = NULL # <<<<<<<<<<<<<< + * cdef void *__cuFuncLoad = NULL + * cdef void *__cuLaunchKernel = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncIsLoaded = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":273 + * cdef void *__cuFuncGetParamInfo = NULL + * cdef void *__cuFuncIsLoaded = NULL + * cdef void *__cuFuncLoad = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLaunchKernel = NULL + * cdef void *__cuLaunchKernelEx = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncLoad = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":274 + * cdef void *__cuFuncIsLoaded = NULL + * cdef void *__cuFuncLoad = NULL + * cdef void *__cuLaunchKernel = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLaunchKernelEx = NULL + * cdef void *__cuLaunchCooperativeKernel = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernel = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":275 + * cdef void *__cuFuncLoad = NULL + * cdef void *__cuLaunchKernel = NULL + * cdef void *__cuLaunchKernelEx = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLaunchCooperativeKernel = NULL + * cdef void *__cuLaunchCooperativeKernelMultiDevice = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchKernelEx = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":276 + * cdef void *__cuLaunchKernel = NULL + * cdef void *__cuLaunchKernelEx = NULL + * cdef void *__cuLaunchCooperativeKernel = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLaunchCooperativeKernelMultiDevice = NULL + * cdef void *__cuLaunchHostFunc = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernel = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":277 + * cdef void *__cuLaunchKernelEx = NULL + * cdef void *__cuLaunchCooperativeKernel = NULL + * cdef void *__cuLaunchCooperativeKernelMultiDevice = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLaunchHostFunc = NULL + * cdef void *__cuFuncSetBlockShape = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchCooperativeKernelMultiDevice = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":278 + * cdef void *__cuLaunchCooperativeKernel = NULL + * cdef void *__cuLaunchCooperativeKernelMultiDevice = NULL + * cdef void *__cuLaunchHostFunc = NULL # <<<<<<<<<<<<<< + * cdef void *__cuFuncSetBlockShape = NULL + * cdef void *__cuFuncSetSharedSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchHostFunc = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":279 + * cdef void *__cuLaunchCooperativeKernelMultiDevice = NULL + * cdef void *__cuLaunchHostFunc = NULL + * cdef void *__cuFuncSetBlockShape = NULL # <<<<<<<<<<<<<< + * cdef void *__cuFuncSetSharedSize = NULL + * cdef void *__cuParamSetSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetBlockShape = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":280 + * cdef void *__cuLaunchHostFunc = NULL + * cdef void *__cuFuncSetBlockShape = NULL + * cdef void *__cuFuncSetSharedSize = NULL # <<<<<<<<<<<<<< + * cdef void *__cuParamSetSize = NULL + * cdef void *__cuParamSeti = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedSize = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":281 + * cdef void *__cuFuncSetBlockShape = NULL + * cdef void *__cuFuncSetSharedSize = NULL + * cdef void *__cuParamSetSize = NULL # <<<<<<<<<<<<<< + * cdef void *__cuParamSeti = NULL + * cdef void *__cuParamSetf = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetSize = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":282 + * cdef void *__cuFuncSetSharedSize = NULL + * cdef void *__cuParamSetSize = NULL + * cdef void *__cuParamSeti = NULL # <<<<<<<<<<<<<< + * cdef void *__cuParamSetf = NULL + * cdef void *__cuParamSetv = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSeti = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":283 + * cdef void *__cuParamSetSize = NULL + * cdef void *__cuParamSeti = NULL + * cdef void *__cuParamSetf = NULL # <<<<<<<<<<<<<< + * cdef void *__cuParamSetv = NULL + * cdef void *__cuLaunch = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetf = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":284 + * cdef void *__cuParamSeti = NULL + * cdef void *__cuParamSetf = NULL + * cdef void *__cuParamSetv = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLaunch = NULL + * cdef void *__cuLaunchGrid = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetv = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":285 + * cdef void *__cuParamSetf = NULL + * cdef void *__cuParamSetv = NULL + * cdef void *__cuLaunch = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLaunchGrid = NULL + * cdef void *__cuLaunchGridAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunch = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":286 + * cdef void *__cuParamSetv = NULL + * cdef void *__cuLaunch = NULL + * cdef void *__cuLaunchGrid = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLaunchGridAsync = NULL + * cdef void *__cuParamSetTexRef = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGrid = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":287 + * cdef void *__cuLaunch = NULL + * cdef void *__cuLaunchGrid = NULL + * cdef void *__cuLaunchGridAsync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuParamSetTexRef = NULL + * cdef void *__cuFuncSetSharedMemConfig = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLaunchGridAsync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":288 + * cdef void *__cuLaunchGrid = NULL + * cdef void *__cuLaunchGridAsync = NULL + * cdef void *__cuParamSetTexRef = NULL # <<<<<<<<<<<<<< + * cdef void *__cuFuncSetSharedMemConfig = NULL + * cdef void *__cuGraphCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuParamSetTexRef = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":289 + * cdef void *__cuLaunchGridAsync = NULL + * cdef void *__cuParamSetTexRef = NULL + * cdef void *__cuFuncSetSharedMemConfig = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphCreate = NULL + * cdef void *__cuGraphAddKernelNode_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuFuncSetSharedMemConfig = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":290 + * cdef void *__cuParamSetTexRef = NULL + * cdef void *__cuFuncSetSharedMemConfig = NULL + * cdef void *__cuGraphCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddKernelNode_v2 = NULL + * cdef void *__cuGraphKernelNodeGetParams_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":291 + * cdef void *__cuFuncSetSharedMemConfig = NULL + * cdef void *__cuGraphCreate = NULL + * cdef void *__cuGraphAddKernelNode_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphKernelNodeGetParams_v2 = NULL + * cdef void *__cuGraphKernelNodeSetParams_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddKernelNode_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":292 + * cdef void *__cuGraphCreate = NULL + * cdef void *__cuGraphAddKernelNode_v2 = NULL + * cdef void *__cuGraphKernelNodeGetParams_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphKernelNodeSetParams_v2 = NULL + * cdef void *__cuGraphAddMemcpyNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetParams_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":293 + * cdef void *__cuGraphAddKernelNode_v2 = NULL + * cdef void *__cuGraphKernelNodeGetParams_v2 = NULL + * cdef void *__cuGraphKernelNodeSetParams_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddMemcpyNode = NULL + * cdef void *__cuGraphMemcpyNodeGetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetParams_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":294 + * cdef void *__cuGraphKernelNodeGetParams_v2 = NULL + * cdef void *__cuGraphKernelNodeSetParams_v2 = NULL + * cdef void *__cuGraphAddMemcpyNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphMemcpyNodeGetParams = NULL + * cdef void *__cuGraphMemcpyNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemcpyNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":295 + * cdef void *__cuGraphKernelNodeSetParams_v2 = NULL + * cdef void *__cuGraphAddMemcpyNode = NULL + * cdef void *__cuGraphMemcpyNodeGetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphMemcpyNodeSetParams = NULL + * cdef void *__cuGraphAddMemsetNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeGetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":296 + * cdef void *__cuGraphAddMemcpyNode = NULL + * cdef void *__cuGraphMemcpyNodeGetParams = NULL + * cdef void *__cuGraphMemcpyNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddMemsetNode = NULL + * cdef void *__cuGraphMemsetNodeGetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemcpyNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":297 + * cdef void *__cuGraphMemcpyNodeGetParams = NULL + * cdef void *__cuGraphMemcpyNodeSetParams = NULL + * cdef void *__cuGraphAddMemsetNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphMemsetNodeGetParams = NULL + * cdef void *__cuGraphMemsetNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemsetNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":298 + * cdef void *__cuGraphMemcpyNodeSetParams = NULL + * cdef void *__cuGraphAddMemsetNode = NULL + * cdef void *__cuGraphMemsetNodeGetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphMemsetNodeSetParams = NULL + * cdef void *__cuGraphAddHostNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeGetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":299 + * cdef void *__cuGraphAddMemsetNode = NULL + * cdef void *__cuGraphMemsetNodeGetParams = NULL + * cdef void *__cuGraphMemsetNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddHostNode = NULL + * cdef void *__cuGraphHostNodeGetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemsetNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":300 + * cdef void *__cuGraphMemsetNodeGetParams = NULL + * cdef void *__cuGraphMemsetNodeSetParams = NULL + * cdef void *__cuGraphAddHostNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphHostNodeGetParams = NULL + * cdef void *__cuGraphHostNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddHostNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":301 + * cdef void *__cuGraphMemsetNodeSetParams = NULL + * cdef void *__cuGraphAddHostNode = NULL + * cdef void *__cuGraphHostNodeGetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphHostNodeSetParams = NULL + * cdef void *__cuGraphAddChildGraphNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeGetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":302 + * cdef void *__cuGraphAddHostNode = NULL + * cdef void *__cuGraphHostNodeGetParams = NULL + * cdef void *__cuGraphHostNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddChildGraphNode = NULL + * cdef void *__cuGraphChildGraphNodeGetGraph = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphHostNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":303 + * cdef void *__cuGraphHostNodeGetParams = NULL + * cdef void *__cuGraphHostNodeSetParams = NULL + * cdef void *__cuGraphAddChildGraphNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphChildGraphNodeGetGraph = NULL + * cdef void *__cuGraphAddEmptyNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddChildGraphNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":304 + * cdef void *__cuGraphHostNodeSetParams = NULL + * cdef void *__cuGraphAddChildGraphNode = NULL + * cdef void *__cuGraphChildGraphNodeGetGraph = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddEmptyNode = NULL + * cdef void *__cuGraphAddEventRecordNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphChildGraphNodeGetGraph = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":305 + * cdef void *__cuGraphAddChildGraphNode = NULL + * cdef void *__cuGraphChildGraphNodeGetGraph = NULL + * cdef void *__cuGraphAddEmptyNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddEventRecordNode = NULL + * cdef void *__cuGraphEventRecordNodeGetEvent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEmptyNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":306 + * cdef void *__cuGraphChildGraphNodeGetGraph = NULL + * cdef void *__cuGraphAddEmptyNode = NULL + * cdef void *__cuGraphAddEventRecordNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphEventRecordNodeGetEvent = NULL + * cdef void *__cuGraphEventRecordNodeSetEvent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventRecordNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":307 + * cdef void *__cuGraphAddEmptyNode = NULL + * cdef void *__cuGraphAddEventRecordNode = NULL + * cdef void *__cuGraphEventRecordNodeGetEvent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphEventRecordNodeSetEvent = NULL + * cdef void *__cuGraphAddEventWaitNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeGetEvent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":308 + * cdef void *__cuGraphAddEventRecordNode = NULL + * cdef void *__cuGraphEventRecordNodeGetEvent = NULL + * cdef void *__cuGraphEventRecordNodeSetEvent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddEventWaitNode = NULL + * cdef void *__cuGraphEventWaitNodeGetEvent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventRecordNodeSetEvent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":309 + * cdef void *__cuGraphEventRecordNodeGetEvent = NULL + * cdef void *__cuGraphEventRecordNodeSetEvent = NULL + * cdef void *__cuGraphAddEventWaitNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphEventWaitNodeGetEvent = NULL + * cdef void *__cuGraphEventWaitNodeSetEvent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddEventWaitNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":310 + * cdef void *__cuGraphEventRecordNodeSetEvent = NULL + * cdef void *__cuGraphAddEventWaitNode = NULL + * cdef void *__cuGraphEventWaitNodeGetEvent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphEventWaitNodeSetEvent = NULL + * cdef void *__cuGraphAddExternalSemaphoresSignalNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeGetEvent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":311 + * cdef void *__cuGraphAddEventWaitNode = NULL + * cdef void *__cuGraphEventWaitNodeGetEvent = NULL + * cdef void *__cuGraphEventWaitNodeSetEvent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddExternalSemaphoresSignalNode = NULL + * cdef void *__cuGraphExternalSemaphoresSignalNodeGetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphEventWaitNodeSetEvent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":312 + * cdef void *__cuGraphEventWaitNodeGetEvent = NULL + * cdef void *__cuGraphEventWaitNodeSetEvent = NULL + * cdef void *__cuGraphAddExternalSemaphoresSignalNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExternalSemaphoresSignalNodeGetParams = NULL + * cdef void *__cuGraphExternalSemaphoresSignalNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresSignalNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":313 + * cdef void *__cuGraphEventWaitNodeSetEvent = NULL + * cdef void *__cuGraphAddExternalSemaphoresSignalNode = NULL + * cdef void *__cuGraphExternalSemaphoresSignalNodeGetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExternalSemaphoresSignalNodeSetParams = NULL + * cdef void *__cuGraphAddExternalSemaphoresWaitNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeGetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":314 + * cdef void *__cuGraphAddExternalSemaphoresSignalNode = NULL + * cdef void *__cuGraphExternalSemaphoresSignalNodeGetParams = NULL + * cdef void *__cuGraphExternalSemaphoresSignalNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddExternalSemaphoresWaitNode = NULL + * cdef void *__cuGraphExternalSemaphoresWaitNodeGetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresSignalNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":315 + * cdef void *__cuGraphExternalSemaphoresSignalNodeGetParams = NULL + * cdef void *__cuGraphExternalSemaphoresSignalNodeSetParams = NULL + * cdef void *__cuGraphAddExternalSemaphoresWaitNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExternalSemaphoresWaitNodeGetParams = NULL + * cdef void *__cuGraphExternalSemaphoresWaitNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddExternalSemaphoresWaitNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":316 + * cdef void *__cuGraphExternalSemaphoresSignalNodeSetParams = NULL + * cdef void *__cuGraphAddExternalSemaphoresWaitNode = NULL + * cdef void *__cuGraphExternalSemaphoresWaitNodeGetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExternalSemaphoresWaitNodeSetParams = NULL + * cdef void *__cuGraphAddBatchMemOpNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeGetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":317 + * cdef void *__cuGraphAddExternalSemaphoresWaitNode = NULL + * cdef void *__cuGraphExternalSemaphoresWaitNodeGetParams = NULL + * cdef void *__cuGraphExternalSemaphoresWaitNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddBatchMemOpNode = NULL + * cdef void *__cuGraphBatchMemOpNodeGetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExternalSemaphoresWaitNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":318 + * cdef void *__cuGraphExternalSemaphoresWaitNodeGetParams = NULL + * cdef void *__cuGraphExternalSemaphoresWaitNodeSetParams = NULL + * cdef void *__cuGraphAddBatchMemOpNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphBatchMemOpNodeGetParams = NULL + * cdef void *__cuGraphBatchMemOpNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddBatchMemOpNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":319 + * cdef void *__cuGraphExternalSemaphoresWaitNodeSetParams = NULL + * cdef void *__cuGraphAddBatchMemOpNode = NULL + * cdef void *__cuGraphBatchMemOpNodeGetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphBatchMemOpNodeSetParams = NULL + * cdef void *__cuGraphExecBatchMemOpNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeGetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":320 + * cdef void *__cuGraphAddBatchMemOpNode = NULL + * cdef void *__cuGraphBatchMemOpNodeGetParams = NULL + * cdef void *__cuGraphBatchMemOpNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecBatchMemOpNodeSetParams = NULL + * cdef void *__cuGraphAddMemAllocNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphBatchMemOpNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":321 + * cdef void *__cuGraphBatchMemOpNodeGetParams = NULL + * cdef void *__cuGraphBatchMemOpNodeSetParams = NULL + * cdef void *__cuGraphExecBatchMemOpNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddMemAllocNode = NULL + * cdef void *__cuGraphMemAllocNodeGetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecBatchMemOpNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":322 + * cdef void *__cuGraphBatchMemOpNodeSetParams = NULL + * cdef void *__cuGraphExecBatchMemOpNodeSetParams = NULL + * cdef void *__cuGraphAddMemAllocNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphMemAllocNodeGetParams = NULL + * cdef void *__cuGraphAddMemFreeNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemAllocNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":323 + * cdef void *__cuGraphExecBatchMemOpNodeSetParams = NULL + * cdef void *__cuGraphAddMemAllocNode = NULL + * cdef void *__cuGraphMemAllocNodeGetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddMemFreeNode = NULL + * cdef void *__cuGraphMemFreeNodeGetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemAllocNodeGetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":324 + * cdef void *__cuGraphAddMemAllocNode = NULL + * cdef void *__cuGraphMemAllocNodeGetParams = NULL + * cdef void *__cuGraphAddMemFreeNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphMemFreeNodeGetParams = NULL + * cdef void *__cuDeviceGraphMemTrim = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddMemFreeNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":325 + * cdef void *__cuGraphMemAllocNodeGetParams = NULL + * cdef void *__cuGraphAddMemFreeNode = NULL + * cdef void *__cuGraphMemFreeNodeGetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGraphMemTrim = NULL + * cdef void *__cuDeviceGetGraphMemAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphMemFreeNodeGetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":326 + * cdef void *__cuGraphAddMemFreeNode = NULL + * cdef void *__cuGraphMemFreeNodeGetParams = NULL + * cdef void *__cuDeviceGraphMemTrim = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetGraphMemAttribute = NULL + * cdef void *__cuDeviceSetGraphMemAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGraphMemTrim = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":327 + * cdef void *__cuGraphMemFreeNodeGetParams = NULL + * cdef void *__cuDeviceGraphMemTrim = NULL + * cdef void *__cuDeviceGetGraphMemAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceSetGraphMemAttribute = NULL + * cdef void *__cuGraphClone = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetGraphMemAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":328 + * cdef void *__cuDeviceGraphMemTrim = NULL + * cdef void *__cuDeviceGetGraphMemAttribute = NULL + * cdef void *__cuDeviceSetGraphMemAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphClone = NULL + * cdef void *__cuGraphNodeFindInClone = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceSetGraphMemAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":329 + * cdef void *__cuDeviceGetGraphMemAttribute = NULL + * cdef void *__cuDeviceSetGraphMemAttribute = NULL + * cdef void *__cuGraphClone = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphNodeFindInClone = NULL + * cdef void *__cuGraphNodeGetType = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphClone = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":330 + * cdef void *__cuDeviceSetGraphMemAttribute = NULL + * cdef void *__cuGraphClone = NULL + * cdef void *__cuGraphNodeFindInClone = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphNodeGetType = NULL + * cdef void *__cuGraphGetNodes = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeFindInClone = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":331 + * cdef void *__cuGraphClone = NULL + * cdef void *__cuGraphNodeFindInClone = NULL + * cdef void *__cuGraphNodeGetType = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphGetNodes = NULL + * cdef void *__cuGraphGetRootNodes = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetType = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":332 + * cdef void *__cuGraphNodeFindInClone = NULL + * cdef void *__cuGraphNodeGetType = NULL + * cdef void *__cuGraphGetNodes = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphGetRootNodes = NULL + * cdef void *__cuGraphGetEdges = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetNodes = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":333 + * cdef void *__cuGraphNodeGetType = NULL + * cdef void *__cuGraphGetNodes = NULL + * cdef void *__cuGraphGetRootNodes = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphGetEdges = NULL + * cdef void *__cuGraphGetEdges_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetRootNodes = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":334 + * cdef void *__cuGraphGetNodes = NULL + * cdef void *__cuGraphGetRootNodes = NULL + * cdef void *__cuGraphGetEdges = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphGetEdges_v2 = NULL + * cdef void *__cuGraphNodeGetDependencies = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":335 + * cdef void *__cuGraphGetRootNodes = NULL + * cdef void *__cuGraphGetEdges = NULL + * cdef void *__cuGraphGetEdges_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphNodeGetDependencies = NULL + * cdef void *__cuGraphNodeGetDependencies_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphGetEdges_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":336 + * cdef void *__cuGraphGetEdges = NULL + * cdef void *__cuGraphGetEdges_v2 = NULL + * cdef void *__cuGraphNodeGetDependencies = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphNodeGetDependencies_v2 = NULL + * cdef void *__cuGraphNodeGetDependentNodes = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":337 + * cdef void *__cuGraphGetEdges_v2 = NULL + * cdef void *__cuGraphNodeGetDependencies = NULL + * cdef void *__cuGraphNodeGetDependencies_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphNodeGetDependentNodes = NULL + * cdef void *__cuGraphNodeGetDependentNodes_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependencies_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":338 + * cdef void *__cuGraphNodeGetDependencies = NULL + * cdef void *__cuGraphNodeGetDependencies_v2 = NULL + * cdef void *__cuGraphNodeGetDependentNodes = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphNodeGetDependentNodes_v2 = NULL + * cdef void *__cuGraphAddDependencies = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":339 + * cdef void *__cuGraphNodeGetDependencies_v2 = NULL + * cdef void *__cuGraphNodeGetDependentNodes = NULL + * cdef void *__cuGraphNodeGetDependentNodes_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddDependencies = NULL + * cdef void *__cuGraphAddDependencies_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetDependentNodes_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":340 + * cdef void *__cuGraphNodeGetDependentNodes = NULL + * cdef void *__cuGraphNodeGetDependentNodes_v2 = NULL + * cdef void *__cuGraphAddDependencies = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddDependencies_v2 = NULL + * cdef void *__cuGraphRemoveDependencies = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":341 + * cdef void *__cuGraphNodeGetDependentNodes_v2 = NULL + * cdef void *__cuGraphAddDependencies = NULL + * cdef void *__cuGraphAddDependencies_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphRemoveDependencies = NULL + * cdef void *__cuGraphRemoveDependencies_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddDependencies_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":342 + * cdef void *__cuGraphAddDependencies = NULL + * cdef void *__cuGraphAddDependencies_v2 = NULL + * cdef void *__cuGraphRemoveDependencies = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphRemoveDependencies_v2 = NULL + * cdef void *__cuGraphDestroyNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":343 + * cdef void *__cuGraphAddDependencies_v2 = NULL + * cdef void *__cuGraphRemoveDependencies = NULL + * cdef void *__cuGraphRemoveDependencies_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphDestroyNode = NULL + * cdef void *__cuGraphInstantiateWithFlags = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRemoveDependencies_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":344 + * cdef void *__cuGraphRemoveDependencies = NULL + * cdef void *__cuGraphRemoveDependencies_v2 = NULL + * cdef void *__cuGraphDestroyNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphInstantiateWithFlags = NULL + * cdef void *__cuGraphInstantiateWithParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroyNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":345 + * cdef void *__cuGraphRemoveDependencies_v2 = NULL + * cdef void *__cuGraphDestroyNode = NULL + * cdef void *__cuGraphInstantiateWithFlags = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphInstantiateWithParams = NULL + * cdef void *__cuGraphExecGetFlags = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithFlags = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":346 + * cdef void *__cuGraphDestroyNode = NULL + * cdef void *__cuGraphInstantiateWithFlags = NULL + * cdef void *__cuGraphInstantiateWithParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecGetFlags = NULL + * cdef void *__cuGraphExecKernelNodeSetParams_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphInstantiateWithParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":347 + * cdef void *__cuGraphInstantiateWithFlags = NULL + * cdef void *__cuGraphInstantiateWithParams = NULL + * cdef void *__cuGraphExecGetFlags = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecKernelNodeSetParams_v2 = NULL + * cdef void *__cuGraphExecMemcpyNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecGetFlags = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":348 + * cdef void *__cuGraphInstantiateWithParams = NULL + * cdef void *__cuGraphExecGetFlags = NULL + * cdef void *__cuGraphExecKernelNodeSetParams_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecMemcpyNodeSetParams = NULL + * cdef void *__cuGraphExecMemsetNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecKernelNodeSetParams_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":349 + * cdef void *__cuGraphExecGetFlags = NULL + * cdef void *__cuGraphExecKernelNodeSetParams_v2 = NULL + * cdef void *__cuGraphExecMemcpyNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecMemsetNodeSetParams = NULL + * cdef void *__cuGraphExecHostNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemcpyNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":350 + * cdef void *__cuGraphExecKernelNodeSetParams_v2 = NULL + * cdef void *__cuGraphExecMemcpyNodeSetParams = NULL + * cdef void *__cuGraphExecMemsetNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecHostNodeSetParams = NULL + * cdef void *__cuGraphExecChildGraphNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecMemsetNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":351 + * cdef void *__cuGraphExecMemcpyNodeSetParams = NULL + * cdef void *__cuGraphExecMemsetNodeSetParams = NULL + * cdef void *__cuGraphExecHostNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecChildGraphNodeSetParams = NULL + * cdef void *__cuGraphExecEventRecordNodeSetEvent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecHostNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":352 + * cdef void *__cuGraphExecMemsetNodeSetParams = NULL + * cdef void *__cuGraphExecHostNodeSetParams = NULL + * cdef void *__cuGraphExecChildGraphNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecEventRecordNodeSetEvent = NULL + * cdef void *__cuGraphExecEventWaitNodeSetEvent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecChildGraphNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":353 + * cdef void *__cuGraphExecHostNodeSetParams = NULL + * cdef void *__cuGraphExecChildGraphNodeSetParams = NULL + * cdef void *__cuGraphExecEventRecordNodeSetEvent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecEventWaitNodeSetEvent = NULL + * cdef void *__cuGraphExecExternalSemaphoresSignalNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventRecordNodeSetEvent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":354 + * cdef void *__cuGraphExecChildGraphNodeSetParams = NULL + * cdef void *__cuGraphExecEventRecordNodeSetEvent = NULL + * cdef void *__cuGraphExecEventWaitNodeSetEvent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecExternalSemaphoresSignalNodeSetParams = NULL + * cdef void *__cuGraphExecExternalSemaphoresWaitNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecEventWaitNodeSetEvent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":355 + * cdef void *__cuGraphExecEventRecordNodeSetEvent = NULL + * cdef void *__cuGraphExecEventWaitNodeSetEvent = NULL + * cdef void *__cuGraphExecExternalSemaphoresSignalNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecExternalSemaphoresWaitNodeSetParams = NULL + * cdef void *__cuGraphNodeSetEnabled = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresSignalNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":356 + * cdef void *__cuGraphExecEventWaitNodeSetEvent = NULL + * cdef void *__cuGraphExecExternalSemaphoresSignalNodeSetParams = NULL + * cdef void *__cuGraphExecExternalSemaphoresWaitNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphNodeSetEnabled = NULL + * cdef void *__cuGraphNodeGetEnabled = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecExternalSemaphoresWaitNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":357 + * cdef void *__cuGraphExecExternalSemaphoresSignalNodeSetParams = NULL + * cdef void *__cuGraphExecExternalSemaphoresWaitNodeSetParams = NULL + * cdef void *__cuGraphNodeSetEnabled = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphNodeGetEnabled = NULL + * cdef void *__cuGraphUpload = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetEnabled = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":358 + * cdef void *__cuGraphExecExternalSemaphoresWaitNodeSetParams = NULL + * cdef void *__cuGraphNodeSetEnabled = NULL + * cdef void *__cuGraphNodeGetEnabled = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphUpload = NULL + * cdef void *__cuGraphLaunch = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeGetEnabled = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":359 + * cdef void *__cuGraphNodeSetEnabled = NULL + * cdef void *__cuGraphNodeGetEnabled = NULL + * cdef void *__cuGraphUpload = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphLaunch = NULL + * cdef void *__cuGraphExecDestroy = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphUpload = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":360 + * cdef void *__cuGraphNodeGetEnabled = NULL + * cdef void *__cuGraphUpload = NULL + * cdef void *__cuGraphLaunch = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecDestroy = NULL + * cdef void *__cuGraphDestroy = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphLaunch = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":361 + * cdef void *__cuGraphUpload = NULL + * cdef void *__cuGraphLaunch = NULL + * cdef void *__cuGraphExecDestroy = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphDestroy = NULL + * cdef void *__cuGraphExecUpdate_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecDestroy = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":362 + * cdef void *__cuGraphLaunch = NULL + * cdef void *__cuGraphExecDestroy = NULL + * cdef void *__cuGraphDestroy = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecUpdate_v2 = NULL + * cdef void *__cuGraphKernelNodeCopyAttributes = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDestroy = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":363 + * cdef void *__cuGraphExecDestroy = NULL + * cdef void *__cuGraphDestroy = NULL + * cdef void *__cuGraphExecUpdate_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphKernelNodeCopyAttributes = NULL + * cdef void *__cuGraphKernelNodeGetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecUpdate_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":364 + * cdef void *__cuGraphDestroy = NULL + * cdef void *__cuGraphExecUpdate_v2 = NULL + * cdef void *__cuGraphKernelNodeCopyAttributes = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphKernelNodeGetAttribute = NULL + * cdef void *__cuGraphKernelNodeSetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeCopyAttributes = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":365 + * cdef void *__cuGraphExecUpdate_v2 = NULL + * cdef void *__cuGraphKernelNodeCopyAttributes = NULL + * cdef void *__cuGraphKernelNodeGetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphKernelNodeSetAttribute = NULL + * cdef void *__cuGraphDebugDotPrint = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeGetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":366 + * cdef void *__cuGraphKernelNodeCopyAttributes = NULL + * cdef void *__cuGraphKernelNodeGetAttribute = NULL + * cdef void *__cuGraphKernelNodeSetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphDebugDotPrint = NULL + * cdef void *__cuUserObjectCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphKernelNodeSetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":367 + * cdef void *__cuGraphKernelNodeGetAttribute = NULL + * cdef void *__cuGraphKernelNodeSetAttribute = NULL + * cdef void *__cuGraphDebugDotPrint = NULL # <<<<<<<<<<<<<< + * cdef void *__cuUserObjectCreate = NULL + * cdef void *__cuUserObjectRetain = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphDebugDotPrint = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":368 + * cdef void *__cuGraphKernelNodeSetAttribute = NULL + * cdef void *__cuGraphDebugDotPrint = NULL + * cdef void *__cuUserObjectCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuUserObjectRetain = NULL + * cdef void *__cuUserObjectRelease = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":369 + * cdef void *__cuGraphDebugDotPrint = NULL + * cdef void *__cuUserObjectCreate = NULL + * cdef void *__cuUserObjectRetain = NULL # <<<<<<<<<<<<<< + * cdef void *__cuUserObjectRelease = NULL + * cdef void *__cuGraphRetainUserObject = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRetain = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":370 + * cdef void *__cuUserObjectCreate = NULL + * cdef void *__cuUserObjectRetain = NULL + * cdef void *__cuUserObjectRelease = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphRetainUserObject = NULL + * cdef void *__cuGraphReleaseUserObject = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuUserObjectRelease = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":371 + * cdef void *__cuUserObjectRetain = NULL + * cdef void *__cuUserObjectRelease = NULL + * cdef void *__cuGraphRetainUserObject = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphReleaseUserObject = NULL + * cdef void *__cuGraphAddNode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphRetainUserObject = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":372 + * cdef void *__cuUserObjectRelease = NULL + * cdef void *__cuGraphRetainUserObject = NULL + * cdef void *__cuGraphReleaseUserObject = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddNode = NULL + * cdef void *__cuGraphAddNode_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphReleaseUserObject = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":373 + * cdef void *__cuGraphRetainUserObject = NULL + * cdef void *__cuGraphReleaseUserObject = NULL + * cdef void *__cuGraphAddNode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphAddNode_v2 = NULL + * cdef void *__cuGraphNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":374 + * cdef void *__cuGraphReleaseUserObject = NULL + * cdef void *__cuGraphAddNode = NULL + * cdef void *__cuGraphAddNode_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphNodeSetParams = NULL + * cdef void *__cuGraphExecNodeSetParams = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphAddNode_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":375 + * cdef void *__cuGraphAddNode = NULL + * cdef void *__cuGraphAddNode_v2 = NULL + * cdef void *__cuGraphNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphExecNodeSetParams = NULL + * cdef void *__cuGraphConditionalHandleCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":376 + * cdef void *__cuGraphAddNode_v2 = NULL + * cdef void *__cuGraphNodeSetParams = NULL + * cdef void *__cuGraphExecNodeSetParams = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphConditionalHandleCreate = NULL + * cdef void *__cuOccupancyMaxActiveBlocksPerMultiprocessor = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphExecNodeSetParams = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":377 + * cdef void *__cuGraphNodeSetParams = NULL + * cdef void *__cuGraphExecNodeSetParams = NULL + * cdef void *__cuGraphConditionalHandleCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuOccupancyMaxActiveBlocksPerMultiprocessor = NULL + * cdef void *__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphConditionalHandleCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":378 + * cdef void *__cuGraphExecNodeSetParams = NULL + * cdef void *__cuGraphConditionalHandleCreate = NULL + * cdef void *__cuOccupancyMaxActiveBlocksPerMultiprocessor = NULL # <<<<<<<<<<<<<< + * cdef void *__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags = NULL + * cdef void *__cuOccupancyMaxPotentialBlockSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessor = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":379 + * cdef void *__cuGraphConditionalHandleCreate = NULL + * cdef void *__cuOccupancyMaxActiveBlocksPerMultiprocessor = NULL + * cdef void *__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags = NULL # <<<<<<<<<<<<<< + * cdef void *__cuOccupancyMaxPotentialBlockSize = NULL + * cdef void *__cuOccupancyMaxPotentialBlockSizeWithFlags = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":380 + * cdef void *__cuOccupancyMaxActiveBlocksPerMultiprocessor = NULL + * cdef void *__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags = NULL + * cdef void *__cuOccupancyMaxPotentialBlockSize = NULL # <<<<<<<<<<<<<< + * cdef void *__cuOccupancyMaxPotentialBlockSizeWithFlags = NULL + * cdef void *__cuOccupancyAvailableDynamicSMemPerBlock = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSize = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":381 + * cdef void *__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags = NULL + * cdef void *__cuOccupancyMaxPotentialBlockSize = NULL + * cdef void *__cuOccupancyMaxPotentialBlockSizeWithFlags = NULL # <<<<<<<<<<<<<< + * cdef void *__cuOccupancyAvailableDynamicSMemPerBlock = NULL + * cdef void *__cuOccupancyMaxPotentialClusterSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialBlockSizeWithFlags = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":382 + * cdef void *__cuOccupancyMaxPotentialBlockSize = NULL + * cdef void *__cuOccupancyMaxPotentialBlockSizeWithFlags = NULL + * cdef void *__cuOccupancyAvailableDynamicSMemPerBlock = NULL # <<<<<<<<<<<<<< + * cdef void *__cuOccupancyMaxPotentialClusterSize = NULL + * cdef void *__cuOccupancyMaxActiveClusters = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyAvailableDynamicSMemPerBlock = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":383 + * cdef void *__cuOccupancyMaxPotentialBlockSizeWithFlags = NULL + * cdef void *__cuOccupancyAvailableDynamicSMemPerBlock = NULL + * cdef void *__cuOccupancyMaxPotentialClusterSize = NULL # <<<<<<<<<<<<<< + * cdef void *__cuOccupancyMaxActiveClusters = NULL + * cdef void *__cuTexRefSetArray = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxPotentialClusterSize = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":384 + * cdef void *__cuOccupancyAvailableDynamicSMemPerBlock = NULL + * cdef void *__cuOccupancyMaxPotentialClusterSize = NULL + * cdef void *__cuOccupancyMaxActiveClusters = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefSetArray = NULL + * cdef void *__cuTexRefSetMipmappedArray = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuOccupancyMaxActiveClusters = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":385 + * cdef void *__cuOccupancyMaxPotentialClusterSize = NULL + * cdef void *__cuOccupancyMaxActiveClusters = NULL + * cdef void *__cuTexRefSetArray = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefSetMipmappedArray = NULL + * cdef void *__cuTexRefSetAddress_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetArray = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":386 + * cdef void *__cuOccupancyMaxActiveClusters = NULL + * cdef void *__cuTexRefSetArray = NULL + * cdef void *__cuTexRefSetMipmappedArray = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefSetAddress_v2 = NULL + * cdef void *__cuTexRefSetAddress2D_v3 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmappedArray = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":387 + * cdef void *__cuTexRefSetArray = NULL + * cdef void *__cuTexRefSetMipmappedArray = NULL + * cdef void *__cuTexRefSetAddress_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefSetAddress2D_v3 = NULL + * cdef void *__cuTexRefSetFormat = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":388 + * cdef void *__cuTexRefSetMipmappedArray = NULL + * cdef void *__cuTexRefSetAddress_v2 = NULL + * cdef void *__cuTexRefSetAddress2D_v3 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefSetFormat = NULL + * cdef void *__cuTexRefSetAddressMode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddress2D_v3 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":389 + * cdef void *__cuTexRefSetAddress_v2 = NULL + * cdef void *__cuTexRefSetAddress2D_v3 = NULL + * cdef void *__cuTexRefSetFormat = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefSetAddressMode = NULL + * cdef void *__cuTexRefSetFilterMode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFormat = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":390 + * cdef void *__cuTexRefSetAddress2D_v3 = NULL + * cdef void *__cuTexRefSetFormat = NULL + * cdef void *__cuTexRefSetAddressMode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefSetFilterMode = NULL + * cdef void *__cuTexRefSetMipmapFilterMode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetAddressMode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":391 + * cdef void *__cuTexRefSetFormat = NULL + * cdef void *__cuTexRefSetAddressMode = NULL + * cdef void *__cuTexRefSetFilterMode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefSetMipmapFilterMode = NULL + * cdef void *__cuTexRefSetMipmapLevelBias = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFilterMode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":392 + * cdef void *__cuTexRefSetAddressMode = NULL + * cdef void *__cuTexRefSetFilterMode = NULL + * cdef void *__cuTexRefSetMipmapFilterMode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefSetMipmapLevelBias = NULL + * cdef void *__cuTexRefSetMipmapLevelClamp = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapFilterMode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":393 + * cdef void *__cuTexRefSetFilterMode = NULL + * cdef void *__cuTexRefSetMipmapFilterMode = NULL + * cdef void *__cuTexRefSetMipmapLevelBias = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefSetMipmapLevelClamp = NULL + * cdef void *__cuTexRefSetMaxAnisotropy = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelBias = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":394 + * cdef void *__cuTexRefSetMipmapFilterMode = NULL + * cdef void *__cuTexRefSetMipmapLevelBias = NULL + * cdef void *__cuTexRefSetMipmapLevelClamp = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefSetMaxAnisotropy = NULL + * cdef void *__cuTexRefSetBorderColor = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMipmapLevelClamp = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":395 + * cdef void *__cuTexRefSetMipmapLevelBias = NULL + * cdef void *__cuTexRefSetMipmapLevelClamp = NULL + * cdef void *__cuTexRefSetMaxAnisotropy = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefSetBorderColor = NULL + * cdef void *__cuTexRefSetFlags = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetMaxAnisotropy = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":396 + * cdef void *__cuTexRefSetMipmapLevelClamp = NULL + * cdef void *__cuTexRefSetMaxAnisotropy = NULL + * cdef void *__cuTexRefSetBorderColor = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefSetFlags = NULL + * cdef void *__cuTexRefGetAddress_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetBorderColor = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":397 + * cdef void *__cuTexRefSetMaxAnisotropy = NULL + * cdef void *__cuTexRefSetBorderColor = NULL + * cdef void *__cuTexRefSetFlags = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefGetAddress_v2 = NULL + * cdef void *__cuTexRefGetArray = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefSetFlags = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":398 + * cdef void *__cuTexRefSetBorderColor = NULL + * cdef void *__cuTexRefSetFlags = NULL + * cdef void *__cuTexRefGetAddress_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefGetArray = NULL + * cdef void *__cuTexRefGetMipmappedArray = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddress_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":399 + * cdef void *__cuTexRefSetFlags = NULL + * cdef void *__cuTexRefGetAddress_v2 = NULL + * cdef void *__cuTexRefGetArray = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefGetMipmappedArray = NULL + * cdef void *__cuTexRefGetAddressMode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetArray = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":400 + * cdef void *__cuTexRefGetAddress_v2 = NULL + * cdef void *__cuTexRefGetArray = NULL + * cdef void *__cuTexRefGetMipmappedArray = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefGetAddressMode = NULL + * cdef void *__cuTexRefGetFilterMode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmappedArray = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":401 + * cdef void *__cuTexRefGetArray = NULL + * cdef void *__cuTexRefGetMipmappedArray = NULL + * cdef void *__cuTexRefGetAddressMode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefGetFilterMode = NULL + * cdef void *__cuTexRefGetFormat = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetAddressMode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":402 + * cdef void *__cuTexRefGetMipmappedArray = NULL + * cdef void *__cuTexRefGetAddressMode = NULL + * cdef void *__cuTexRefGetFilterMode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefGetFormat = NULL + * cdef void *__cuTexRefGetMipmapFilterMode = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFilterMode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":403 + * cdef void *__cuTexRefGetAddressMode = NULL + * cdef void *__cuTexRefGetFilterMode = NULL + * cdef void *__cuTexRefGetFormat = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefGetMipmapFilterMode = NULL + * cdef void *__cuTexRefGetMipmapLevelBias = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFormat = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":404 + * cdef void *__cuTexRefGetFilterMode = NULL + * cdef void *__cuTexRefGetFormat = NULL + * cdef void *__cuTexRefGetMipmapFilterMode = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefGetMipmapLevelBias = NULL + * cdef void *__cuTexRefGetMipmapLevelClamp = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapFilterMode = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":405 + * cdef void *__cuTexRefGetFormat = NULL + * cdef void *__cuTexRefGetMipmapFilterMode = NULL + * cdef void *__cuTexRefGetMipmapLevelBias = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefGetMipmapLevelClamp = NULL + * cdef void *__cuTexRefGetMaxAnisotropy = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelBias = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":406 + * cdef void *__cuTexRefGetMipmapFilterMode = NULL + * cdef void *__cuTexRefGetMipmapLevelBias = NULL + * cdef void *__cuTexRefGetMipmapLevelClamp = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefGetMaxAnisotropy = NULL + * cdef void *__cuTexRefGetBorderColor = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMipmapLevelClamp = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":407 + * cdef void *__cuTexRefGetMipmapLevelBias = NULL + * cdef void *__cuTexRefGetMipmapLevelClamp = NULL + * cdef void *__cuTexRefGetMaxAnisotropy = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefGetBorderColor = NULL + * cdef void *__cuTexRefGetFlags = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetMaxAnisotropy = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":408 + * cdef void *__cuTexRefGetMipmapLevelClamp = NULL + * cdef void *__cuTexRefGetMaxAnisotropy = NULL + * cdef void *__cuTexRefGetBorderColor = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefGetFlags = NULL + * cdef void *__cuTexRefCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetBorderColor = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":409 + * cdef void *__cuTexRefGetMaxAnisotropy = NULL + * cdef void *__cuTexRefGetBorderColor = NULL + * cdef void *__cuTexRefGetFlags = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefCreate = NULL + * cdef void *__cuTexRefDestroy = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefGetFlags = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":410 + * cdef void *__cuTexRefGetBorderColor = NULL + * cdef void *__cuTexRefGetFlags = NULL + * cdef void *__cuTexRefCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexRefDestroy = NULL + * cdef void *__cuSurfRefSetArray = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":411 + * cdef void *__cuTexRefGetFlags = NULL + * cdef void *__cuTexRefCreate = NULL + * cdef void *__cuTexRefDestroy = NULL # <<<<<<<<<<<<<< + * cdef void *__cuSurfRefSetArray = NULL + * cdef void *__cuSurfRefGetArray = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexRefDestroy = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":412 + * cdef void *__cuTexRefCreate = NULL + * cdef void *__cuTexRefDestroy = NULL + * cdef void *__cuSurfRefSetArray = NULL # <<<<<<<<<<<<<< + * cdef void *__cuSurfRefGetArray = NULL + * cdef void *__cuTexObjectCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefSetArray = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":413 + * cdef void *__cuTexRefDestroy = NULL + * cdef void *__cuSurfRefSetArray = NULL + * cdef void *__cuSurfRefGetArray = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexObjectCreate = NULL + * cdef void *__cuTexObjectDestroy = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfRefGetArray = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":414 + * cdef void *__cuSurfRefSetArray = NULL + * cdef void *__cuSurfRefGetArray = NULL + * cdef void *__cuTexObjectCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexObjectDestroy = NULL + * cdef void *__cuTexObjectGetResourceDesc = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":415 + * cdef void *__cuSurfRefGetArray = NULL + * cdef void *__cuTexObjectCreate = NULL + * cdef void *__cuTexObjectDestroy = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexObjectGetResourceDesc = NULL + * cdef void *__cuTexObjectGetTextureDesc = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectDestroy = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":416 + * cdef void *__cuTexObjectCreate = NULL + * cdef void *__cuTexObjectDestroy = NULL + * cdef void *__cuTexObjectGetResourceDesc = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexObjectGetTextureDesc = NULL + * cdef void *__cuTexObjectGetResourceViewDesc = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceDesc = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":417 + * cdef void *__cuTexObjectDestroy = NULL + * cdef void *__cuTexObjectGetResourceDesc = NULL + * cdef void *__cuTexObjectGetTextureDesc = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTexObjectGetResourceViewDesc = NULL + * cdef void *__cuSurfObjectCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetTextureDesc = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":418 + * cdef void *__cuTexObjectGetResourceDesc = NULL + * cdef void *__cuTexObjectGetTextureDesc = NULL + * cdef void *__cuTexObjectGetResourceViewDesc = NULL # <<<<<<<<<<<<<< + * cdef void *__cuSurfObjectCreate = NULL + * cdef void *__cuSurfObjectDestroy = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTexObjectGetResourceViewDesc = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":419 + * cdef void *__cuTexObjectGetTextureDesc = NULL + * cdef void *__cuTexObjectGetResourceViewDesc = NULL + * cdef void *__cuSurfObjectCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuSurfObjectDestroy = NULL + * cdef void *__cuSurfObjectGetResourceDesc = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":420 + * cdef void *__cuTexObjectGetResourceViewDesc = NULL + * cdef void *__cuSurfObjectCreate = NULL + * cdef void *__cuSurfObjectDestroy = NULL # <<<<<<<<<<<<<< + * cdef void *__cuSurfObjectGetResourceDesc = NULL + * cdef void *__cuTensorMapEncodeTiled = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectDestroy = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":421 + * cdef void *__cuSurfObjectCreate = NULL + * cdef void *__cuSurfObjectDestroy = NULL + * cdef void *__cuSurfObjectGetResourceDesc = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTensorMapEncodeTiled = NULL + * cdef void *__cuTensorMapEncodeIm2col = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuSurfObjectGetResourceDesc = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":422 + * cdef void *__cuSurfObjectDestroy = NULL + * cdef void *__cuSurfObjectGetResourceDesc = NULL + * cdef void *__cuTensorMapEncodeTiled = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTensorMapEncodeIm2col = NULL + * cdef void *__cuTensorMapEncodeIm2colWide = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeTiled = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":423 + * cdef void *__cuSurfObjectGetResourceDesc = NULL + * cdef void *__cuTensorMapEncodeTiled = NULL + * cdef void *__cuTensorMapEncodeIm2col = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTensorMapEncodeIm2colWide = NULL + * cdef void *__cuTensorMapReplaceAddress = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2col = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":424 + * cdef void *__cuTensorMapEncodeTiled = NULL + * cdef void *__cuTensorMapEncodeIm2col = NULL + * cdef void *__cuTensorMapEncodeIm2colWide = NULL # <<<<<<<<<<<<<< + * cdef void *__cuTensorMapReplaceAddress = NULL + * cdef void *__cuDeviceCanAccessPeer = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapEncodeIm2colWide = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":425 + * cdef void *__cuTensorMapEncodeIm2col = NULL + * cdef void *__cuTensorMapEncodeIm2colWide = NULL + * cdef void *__cuTensorMapReplaceAddress = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceCanAccessPeer = NULL + * cdef void *__cuCtxEnablePeerAccess = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuTensorMapReplaceAddress = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":426 + * cdef void *__cuTensorMapEncodeIm2colWide = NULL + * cdef void *__cuTensorMapReplaceAddress = NULL + * cdef void *__cuDeviceCanAccessPeer = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxEnablePeerAccess = NULL + * cdef void *__cuCtxDisablePeerAccess = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceCanAccessPeer = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":427 + * cdef void *__cuTensorMapReplaceAddress = NULL + * cdef void *__cuDeviceCanAccessPeer = NULL + * cdef void *__cuCtxEnablePeerAccess = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxDisablePeerAccess = NULL + * cdef void *__cuDeviceGetP2PAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxEnablePeerAccess = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":428 + * cdef void *__cuDeviceCanAccessPeer = NULL + * cdef void *__cuCtxEnablePeerAccess = NULL + * cdef void *__cuCtxDisablePeerAccess = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetP2PAttribute = NULL + * cdef void *__cuGraphicsUnregisterResource = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxDisablePeerAccess = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":429 + * cdef void *__cuCtxEnablePeerAccess = NULL + * cdef void *__cuCtxDisablePeerAccess = NULL + * cdef void *__cuDeviceGetP2PAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphicsUnregisterResource = NULL + * cdef void *__cuGraphicsSubResourceGetMappedArray = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetP2PAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":430 + * cdef void *__cuCtxDisablePeerAccess = NULL + * cdef void *__cuDeviceGetP2PAttribute = NULL + * cdef void *__cuGraphicsUnregisterResource = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphicsSubResourceGetMappedArray = NULL + * cdef void *__cuGraphicsResourceGetMappedMipmappedArray = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnregisterResource = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":431 + * cdef void *__cuDeviceGetP2PAttribute = NULL + * cdef void *__cuGraphicsUnregisterResource = NULL + * cdef void *__cuGraphicsSubResourceGetMappedArray = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphicsResourceGetMappedMipmappedArray = NULL + * cdef void *__cuGraphicsResourceGetMappedPointer_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsSubResourceGetMappedArray = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":432 + * cdef void *__cuGraphicsUnregisterResource = NULL + * cdef void *__cuGraphicsSubResourceGetMappedArray = NULL + * cdef void *__cuGraphicsResourceGetMappedMipmappedArray = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphicsResourceGetMappedPointer_v2 = NULL + * cdef void *__cuGraphicsResourceSetMapFlags_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedMipmappedArray = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":433 + * cdef void *__cuGraphicsSubResourceGetMappedArray = NULL + * cdef void *__cuGraphicsResourceGetMappedMipmappedArray = NULL + * cdef void *__cuGraphicsResourceGetMappedPointer_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphicsResourceSetMapFlags_v2 = NULL + * cdef void *__cuGraphicsMapResources = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedPointer_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":434 + * cdef void *__cuGraphicsResourceGetMappedMipmappedArray = NULL + * cdef void *__cuGraphicsResourceGetMappedPointer_v2 = NULL + * cdef void *__cuGraphicsResourceSetMapFlags_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphicsMapResources = NULL + * cdef void *__cuGraphicsUnmapResources = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceSetMapFlags_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":435 + * cdef void *__cuGraphicsResourceGetMappedPointer_v2 = NULL + * cdef void *__cuGraphicsResourceSetMapFlags_v2 = NULL + * cdef void *__cuGraphicsMapResources = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphicsUnmapResources = NULL + * cdef void *__cuGetProcAddress_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsMapResources = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":436 + * cdef void *__cuGraphicsResourceSetMapFlags_v2 = NULL + * cdef void *__cuGraphicsMapResources = NULL + * cdef void *__cuGraphicsUnmapResources = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGetProcAddress_v2 = NULL + * cdef void *__cuCoredumpGetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsUnmapResources = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":437 + * cdef void *__cuGraphicsMapResources = NULL + * cdef void *__cuGraphicsUnmapResources = NULL + * cdef void *__cuGetProcAddress_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCoredumpGetAttribute = NULL + * cdef void *__cuCoredumpGetAttributeGlobal = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetProcAddress_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":438 + * cdef void *__cuGraphicsUnmapResources = NULL + * cdef void *__cuGetProcAddress_v2 = NULL + * cdef void *__cuCoredumpGetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCoredumpGetAttributeGlobal = NULL + * cdef void *__cuCoredumpSetAttribute = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":439 + * cdef void *__cuGetProcAddress_v2 = NULL + * cdef void *__cuCoredumpGetAttribute = NULL + * cdef void *__cuCoredumpGetAttributeGlobal = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCoredumpSetAttribute = NULL + * cdef void *__cuCoredumpSetAttributeGlobal = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpGetAttributeGlobal = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":440 + * cdef void *__cuCoredumpGetAttribute = NULL + * cdef void *__cuCoredumpGetAttributeGlobal = NULL + * cdef void *__cuCoredumpSetAttribute = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCoredumpSetAttributeGlobal = NULL + * cdef void *__cuGetExportTable = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttribute = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":441 + * cdef void *__cuCoredumpGetAttributeGlobal = NULL + * cdef void *__cuCoredumpSetAttribute = NULL + * cdef void *__cuCoredumpSetAttributeGlobal = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGetExportTable = NULL + * cdef void *__cuGreenCtxCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCoredumpSetAttributeGlobal = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":442 + * cdef void *__cuCoredumpSetAttribute = NULL + * cdef void *__cuCoredumpSetAttributeGlobal = NULL + * cdef void *__cuGetExportTable = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGreenCtxCreate = NULL + * cdef void *__cuGreenCtxDestroy = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGetExportTable = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":443 + * cdef void *__cuCoredumpSetAttributeGlobal = NULL + * cdef void *__cuGetExportTable = NULL + * cdef void *__cuGreenCtxCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGreenCtxDestroy = NULL + * cdef void *__cuCtxFromGreenCtx = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":444 + * cdef void *__cuGetExportTable = NULL + * cdef void *__cuGreenCtxCreate = NULL + * cdef void *__cuGreenCtxDestroy = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxFromGreenCtx = NULL + * cdef void *__cuDeviceGetDevResource = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxDestroy = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":445 + * cdef void *__cuGreenCtxCreate = NULL + * cdef void *__cuGreenCtxDestroy = NULL + * cdef void *__cuCtxFromGreenCtx = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDeviceGetDevResource = NULL + * cdef void *__cuCtxGetDevResource = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxFromGreenCtx = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":446 + * cdef void *__cuGreenCtxDestroy = NULL + * cdef void *__cuCtxFromGreenCtx = NULL + * cdef void *__cuDeviceGetDevResource = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCtxGetDevResource = NULL + * cdef void *__cuGreenCtxGetDevResource = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDeviceGetDevResource = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":447 + * cdef void *__cuCtxFromGreenCtx = NULL + * cdef void *__cuDeviceGetDevResource = NULL + * cdef void *__cuCtxGetDevResource = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGreenCtxGetDevResource = NULL + * cdef void *__cuDevSmResourceSplitByCount = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCtxGetDevResource = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":448 + * cdef void *__cuDeviceGetDevResource = NULL + * cdef void *__cuCtxGetDevResource = NULL + * cdef void *__cuGreenCtxGetDevResource = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDevSmResourceSplitByCount = NULL + * cdef void *__cuDevResourceGenerateDesc = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxGetDevResource = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":449 + * cdef void *__cuCtxGetDevResource = NULL + * cdef void *__cuGreenCtxGetDevResource = NULL + * cdef void *__cuDevSmResourceSplitByCount = NULL # <<<<<<<<<<<<<< + * cdef void *__cuDevResourceGenerateDesc = NULL + * cdef void *__cuGreenCtxRecordEvent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevSmResourceSplitByCount = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":450 + * cdef void *__cuGreenCtxGetDevResource = NULL + * cdef void *__cuDevSmResourceSplitByCount = NULL + * cdef void *__cuDevResourceGenerateDesc = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGreenCtxRecordEvent = NULL + * cdef void *__cuGreenCtxWaitEvent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuDevResourceGenerateDesc = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":451 + * cdef void *__cuDevSmResourceSplitByCount = NULL + * cdef void *__cuDevResourceGenerateDesc = NULL + * cdef void *__cuGreenCtxRecordEvent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGreenCtxWaitEvent = NULL + * cdef void *__cuStreamGetGreenCtx = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxRecordEvent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":452 + * cdef void *__cuDevResourceGenerateDesc = NULL + * cdef void *__cuGreenCtxRecordEvent = NULL + * cdef void *__cuGreenCtxWaitEvent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuStreamGetGreenCtx = NULL + * cdef void *__cuGreenCtxStreamCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxWaitEvent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":453 + * cdef void *__cuGreenCtxRecordEvent = NULL + * cdef void *__cuGreenCtxWaitEvent = NULL + * cdef void *__cuStreamGetGreenCtx = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGreenCtxStreamCreate = NULL + * cdef void *__cuLogsRegisterCallback = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuStreamGetGreenCtx = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":454 + * cdef void *__cuGreenCtxWaitEvent = NULL + * cdef void *__cuStreamGetGreenCtx = NULL + * cdef void *__cuGreenCtxStreamCreate = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLogsRegisterCallback = NULL + * cdef void *__cuLogsUnregisterCallback = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGreenCtxStreamCreate = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":455 + * cdef void *__cuStreamGetGreenCtx = NULL + * cdef void *__cuGreenCtxStreamCreate = NULL + * cdef void *__cuLogsRegisterCallback = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLogsUnregisterCallback = NULL + * cdef void *__cuLogsCurrent = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsRegisterCallback = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":456 + * cdef void *__cuGreenCtxStreamCreate = NULL + * cdef void *__cuLogsRegisterCallback = NULL + * cdef void *__cuLogsUnregisterCallback = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLogsCurrent = NULL + * cdef void *__cuLogsDumpToFile = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsUnregisterCallback = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":457 + * cdef void *__cuLogsRegisterCallback = NULL + * cdef void *__cuLogsUnregisterCallback = NULL + * cdef void *__cuLogsCurrent = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLogsDumpToFile = NULL + * cdef void *__cuLogsDumpToMemory = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsCurrent = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":458 + * cdef void *__cuLogsUnregisterCallback = NULL + * cdef void *__cuLogsCurrent = NULL + * cdef void *__cuLogsDumpToFile = NULL # <<<<<<<<<<<<<< + * cdef void *__cuLogsDumpToMemory = NULL + * cdef void *__cuCheckpointProcessGetRestoreThreadId = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToFile = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":459 + * cdef void *__cuLogsCurrent = NULL + * cdef void *__cuLogsDumpToFile = NULL + * cdef void *__cuLogsDumpToMemory = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCheckpointProcessGetRestoreThreadId = NULL + * cdef void *__cuCheckpointProcessGetState = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuLogsDumpToMemory = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":460 + * cdef void *__cuLogsDumpToFile = NULL + * cdef void *__cuLogsDumpToMemory = NULL + * cdef void *__cuCheckpointProcessGetRestoreThreadId = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCheckpointProcessGetState = NULL + * cdef void *__cuCheckpointProcessLock = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetRestoreThreadId = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":461 + * cdef void *__cuLogsDumpToMemory = NULL + * cdef void *__cuCheckpointProcessGetRestoreThreadId = NULL + * cdef void *__cuCheckpointProcessGetState = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCheckpointProcessLock = NULL + * cdef void *__cuCheckpointProcessCheckpoint = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessGetState = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":462 + * cdef void *__cuCheckpointProcessGetRestoreThreadId = NULL + * cdef void *__cuCheckpointProcessGetState = NULL + * cdef void *__cuCheckpointProcessLock = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCheckpointProcessCheckpoint = NULL + * cdef void *__cuCheckpointProcessRestore = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessLock = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":463 + * cdef void *__cuCheckpointProcessGetState = NULL + * cdef void *__cuCheckpointProcessLock = NULL + * cdef void *__cuCheckpointProcessCheckpoint = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCheckpointProcessRestore = NULL + * cdef void *__cuCheckpointProcessUnlock = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessCheckpoint = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":464 + * cdef void *__cuCheckpointProcessLock = NULL + * cdef void *__cuCheckpointProcessCheckpoint = NULL + * cdef void *__cuCheckpointProcessRestore = NULL # <<<<<<<<<<<<<< + * cdef void *__cuCheckpointProcessUnlock = NULL + * cdef void *__cuProfilerStart = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessRestore = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":465 + * cdef void *__cuCheckpointProcessCheckpoint = NULL + * cdef void *__cuCheckpointProcessRestore = NULL + * cdef void *__cuCheckpointProcessUnlock = NULL # <<<<<<<<<<<<<< + * cdef void *__cuProfilerStart = NULL + * cdef void *__cuProfilerStop = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuCheckpointProcessUnlock = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":466 + * cdef void *__cuCheckpointProcessRestore = NULL + * cdef void *__cuCheckpointProcessUnlock = NULL + * cdef void *__cuProfilerStart = NULL # <<<<<<<<<<<<<< + * cdef void *__cuProfilerStop = NULL + * cdef void *__cuGraphicsEGLRegisterImage = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStart = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":467 + * cdef void *__cuCheckpointProcessUnlock = NULL + * cdef void *__cuProfilerStart = NULL + * cdef void *__cuProfilerStop = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphicsEGLRegisterImage = NULL + * cdef void *__cuEGLStreamConsumerConnect = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuProfilerStop = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":468 + * cdef void *__cuProfilerStart = NULL + * cdef void *__cuProfilerStop = NULL + * cdef void *__cuGraphicsEGLRegisterImage = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEGLStreamConsumerConnect = NULL + * cdef void *__cuEGLStreamConsumerConnectWithFlags = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsEGLRegisterImage = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":469 + * cdef void *__cuProfilerStop = NULL + * cdef void *__cuGraphicsEGLRegisterImage = NULL + * cdef void *__cuEGLStreamConsumerConnect = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEGLStreamConsumerConnectWithFlags = NULL + * cdef void *__cuEGLStreamConsumerDisconnect = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnect = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":470 + * cdef void *__cuGraphicsEGLRegisterImage = NULL + * cdef void *__cuEGLStreamConsumerConnect = NULL + * cdef void *__cuEGLStreamConsumerConnectWithFlags = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEGLStreamConsumerDisconnect = NULL + * cdef void *__cuEGLStreamConsumerAcquireFrame = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerConnectWithFlags = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":471 + * cdef void *__cuEGLStreamConsumerConnect = NULL + * cdef void *__cuEGLStreamConsumerConnectWithFlags = NULL + * cdef void *__cuEGLStreamConsumerDisconnect = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEGLStreamConsumerAcquireFrame = NULL + * cdef void *__cuEGLStreamConsumerReleaseFrame = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerDisconnect = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":472 + * cdef void *__cuEGLStreamConsumerConnectWithFlags = NULL + * cdef void *__cuEGLStreamConsumerDisconnect = NULL + * cdef void *__cuEGLStreamConsumerAcquireFrame = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEGLStreamConsumerReleaseFrame = NULL + * cdef void *__cuEGLStreamProducerConnect = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerAcquireFrame = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":473 + * cdef void *__cuEGLStreamConsumerDisconnect = NULL + * cdef void *__cuEGLStreamConsumerAcquireFrame = NULL + * cdef void *__cuEGLStreamConsumerReleaseFrame = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEGLStreamProducerConnect = NULL + * cdef void *__cuEGLStreamProducerDisconnect = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamConsumerReleaseFrame = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":474 + * cdef void *__cuEGLStreamConsumerAcquireFrame = NULL + * cdef void *__cuEGLStreamConsumerReleaseFrame = NULL + * cdef void *__cuEGLStreamProducerConnect = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEGLStreamProducerDisconnect = NULL + * cdef void *__cuEGLStreamProducerPresentFrame = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerConnect = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":475 + * cdef void *__cuEGLStreamConsumerReleaseFrame = NULL + * cdef void *__cuEGLStreamProducerConnect = NULL + * cdef void *__cuEGLStreamProducerDisconnect = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEGLStreamProducerPresentFrame = NULL + * cdef void *__cuEGLStreamProducerReturnFrame = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerDisconnect = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":476 + * cdef void *__cuEGLStreamProducerConnect = NULL + * cdef void *__cuEGLStreamProducerDisconnect = NULL + * cdef void *__cuEGLStreamProducerPresentFrame = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEGLStreamProducerReturnFrame = NULL + * cdef void *__cuGraphicsResourceGetMappedEglFrame = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerPresentFrame = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":477 + * cdef void *__cuEGLStreamProducerDisconnect = NULL + * cdef void *__cuEGLStreamProducerPresentFrame = NULL + * cdef void *__cuEGLStreamProducerReturnFrame = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphicsResourceGetMappedEglFrame = NULL + * cdef void *__cuEventCreateFromEGLSync = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEGLStreamProducerReturnFrame = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":478 + * cdef void *__cuEGLStreamProducerPresentFrame = NULL + * cdef void *__cuEGLStreamProducerReturnFrame = NULL + * cdef void *__cuGraphicsResourceGetMappedEglFrame = NULL # <<<<<<<<<<<<<< + * cdef void *__cuEventCreateFromEGLSync = NULL + * cdef void *__cuGraphicsGLRegisterBuffer = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsResourceGetMappedEglFrame = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":479 + * cdef void *__cuEGLStreamProducerReturnFrame = NULL + * cdef void *__cuGraphicsResourceGetMappedEglFrame = NULL + * cdef void *__cuEventCreateFromEGLSync = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphicsGLRegisterBuffer = NULL + * cdef void *__cuGraphicsGLRegisterImage = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuEventCreateFromEGLSync = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":480 + * cdef void *__cuGraphicsResourceGetMappedEglFrame = NULL + * cdef void *__cuEventCreateFromEGLSync = NULL + * cdef void *__cuGraphicsGLRegisterBuffer = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphicsGLRegisterImage = NULL + * cdef void *__cuGLGetDevices_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterBuffer = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":481 + * cdef void *__cuEventCreateFromEGLSync = NULL + * cdef void *__cuGraphicsGLRegisterBuffer = NULL + * cdef void *__cuGraphicsGLRegisterImage = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGLGetDevices_v2 = NULL + * cdef void *__cuVDPAUGetDevice = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsGLRegisterImage = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":482 + * cdef void *__cuGraphicsGLRegisterBuffer = NULL + * cdef void *__cuGraphicsGLRegisterImage = NULL + * cdef void *__cuGLGetDevices_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuVDPAUGetDevice = NULL + * cdef void *__cuVDPAUCtxCreate_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGLGetDevices_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":483 + * cdef void *__cuGraphicsGLRegisterImage = NULL + * cdef void *__cuGLGetDevices_v2 = NULL + * cdef void *__cuVDPAUGetDevice = NULL # <<<<<<<<<<<<<< + * cdef void *__cuVDPAUCtxCreate_v2 = NULL + * cdef void *__cuGraphicsVDPAURegisterVideoSurface = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUGetDevice = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":484 + * cdef void *__cuGLGetDevices_v2 = NULL + * cdef void *__cuVDPAUGetDevice = NULL + * cdef void *__cuVDPAUCtxCreate_v2 = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphicsVDPAURegisterVideoSurface = NULL + * cdef void *__cuGraphicsVDPAURegisterOutputSurface = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuVDPAUCtxCreate_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":485 + * cdef void *__cuVDPAUGetDevice = NULL + * cdef void *__cuVDPAUCtxCreate_v2 = NULL + * cdef void *__cuGraphicsVDPAURegisterVideoSurface = NULL # <<<<<<<<<<<<<< + * cdef void *__cuGraphicsVDPAURegisterOutputSurface = NULL + * +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterVideoSurface = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":486 + * cdef void *__cuVDPAUCtxCreate_v2 = NULL + * cdef void *__cuGraphicsVDPAURegisterVideoSurface = NULL + * cdef void *__cuGraphicsVDPAURegisterOutputSurface = NULL # <<<<<<<<<<<<<< + * + * # To make cuPythonInit reentrant +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver___cuGraphicsVDPAURegisterOutputSurface = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":490 + * # To make cuPythonInit reentrant + * ctypedef CUresult (*__cuGetProcAddress_v2_T)(const char*, void**, int, cuuint64_t, CUdriverProcAddressQueryResult*) except?CUDA_ERROR_NOT_FOUND nogil + * cdef __cuGetProcAddress_v2_T _F_cuGetProcAddress_v2 = NULL # <<<<<<<<<<<<<< + * + * cdef int _cuPythonInit() except -1 nogil: +*/ + __pyx_v_4cuda_8bindings_9_bindings_8cydriver__F_cuGetProcAddress_v2 = NULL; + + /* "cuda/bindings/_bindings/cydriver.pyx":7036 + * return err + * + * cdef dict func_ptrs = None # <<<<<<<<<<<<<< + * + * cpdef dict _inspect_function_pointers(): +*/ + __Pyx_INCREF(Py_None); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_bindings_8cydriver_func_ptrs, ((PyObject*)Py_None)); + __Pyx_GIVEREF(Py_None); + + /* "cuda/bindings/_bindings/cydriver.pyx":7038 + * cdef dict func_ptrs = None + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_4cuda_8bindings_9_bindings_8cydriver_1_inspect_function_pointers, 0, __pyx_mstate_global->__pyx_n_u_inspect_function_pointers, NULL, __pyx_mstate_global->__pyx_n_u_cuda_bindings__bindings_cydriver, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 7038, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_inspect_function_pointers, __pyx_t_2) < (0)) __PYX_ERR(0, 7038, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":7996 + * return data + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_4cuda_8bindings_9_bindings_8cydriver_3_inspect_function_pointer, 0, __pyx_mstate_global->__pyx_n_u_inspect_function_pointer, NULL, __pyx_mstate_global->__pyx_n_u_cuda_bindings__bindings_cydriver, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[1])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 7996, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_inspect_function_pointer, __pyx_t_2) < (0)) __PYX_ERR(0, 7996, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_bindings/cydriver.pyx":1 + * # SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # <<<<<<<<<<<<<< + * # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + * +*/ + __pyx_t_2 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_2) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init cuda.bindings._bindings.cydriver", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init cuda.bindings._bindings.cydriver"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ + +typedef struct { + const char *s; +#if 179 <= 65535 + const unsigned short n; +#elif 179 / 2 < INT_MAX + const unsigned int n; +#elif 179 / 2 < LONG_MAX + const unsigned long n; +#else + const Py_ssize_t n; +#endif +#if 1 <= 31 + const unsigned int encoding : 5; +#elif 1 <= 255 + const unsigned char encoding; +#elif 1 <= 65535 + const unsigned short encoding; +#else + const Py_ssize_t encoding; +#endif + const unsigned int is_unicode : 1; + const unsigned int intern : 1; +} __Pyx_StringTabEntry; +static const char * const __pyx_string_tab_encodings[] = { 0 }; +static const __Pyx_StringTabEntry __pyx_string_tab[] = { + {__pyx_k_, sizeof(__pyx_k_), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_ */ + {__pyx_k_CUDA_PYTHON_CUDA_PER_THREAD_DEFA, sizeof(__pyx_k_CUDA_PYTHON_CUDA_PER_THREAD_DEFA), 0, 1, 1}, /* PyObject cname: __pyx_n_u_CUDA_PYTHON_CUDA_PER_THREAD_DEFA */ + {__pyx_k_Failed_to_dlopen, sizeof(__pyx_k_Failed_to_dlopen), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Failed_to_dlopen */ + {__pyx_k_Function_cuArray3DCreate_v2_not, sizeof(__pyx_k_Function_cuArray3DCreate_v2_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuArray3DCreate_v2_not */ + {__pyx_k_Function_cuArray3DGetDescriptor, sizeof(__pyx_k_Function_cuArray3DGetDescriptor), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuArray3DGetDescriptor */ + {__pyx_k_Function_cuArrayCreate_v2_not_fo, sizeof(__pyx_k_Function_cuArrayCreate_v2_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuArrayCreate_v2_not_fo */ + {__pyx_k_Function_cuArrayDestroy_not_foun, sizeof(__pyx_k_Function_cuArrayDestroy_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuArrayDestroy_not_foun */ + {__pyx_k_Function_cuArrayGetDescriptor_v2, sizeof(__pyx_k_Function_cuArrayGetDescriptor_v2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuArrayGetDescriptor_v2 */ + {__pyx_k_Function_cuArrayGetMemoryRequire, sizeof(__pyx_k_Function_cuArrayGetMemoryRequire), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuArrayGetMemoryRequire */ + {__pyx_k_Function_cuArrayGetPlane_not_fou, sizeof(__pyx_k_Function_cuArrayGetPlane_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuArrayGetPlane_not_fou */ + {__pyx_k_Function_cuArrayGetSparsePropert, sizeof(__pyx_k_Function_cuArrayGetSparsePropert), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuArrayGetSparsePropert */ + {__pyx_k_Function_cuCheckpointProcessChec, sizeof(__pyx_k_Function_cuCheckpointProcessChec), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCheckpointProcessChec */ + {__pyx_k_Function_cuCheckpointProcessGetR, sizeof(__pyx_k_Function_cuCheckpointProcessGetR), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCheckpointProcessGetR */ + {__pyx_k_Function_cuCheckpointProcessGetS, sizeof(__pyx_k_Function_cuCheckpointProcessGetS), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCheckpointProcessGetS */ + {__pyx_k_Function_cuCheckpointProcessLock, sizeof(__pyx_k_Function_cuCheckpointProcessLock), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCheckpointProcessLock */ + {__pyx_k_Function_cuCheckpointProcessRest, sizeof(__pyx_k_Function_cuCheckpointProcessRest), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCheckpointProcessRest */ + {__pyx_k_Function_cuCheckpointProcessUnlo, sizeof(__pyx_k_Function_cuCheckpointProcessUnlo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCheckpointProcessUnlo */ + {__pyx_k_Function_cuCoredumpGetAttribute, sizeof(__pyx_k_Function_cuCoredumpGetAttribute), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCoredumpGetAttribute */ + {__pyx_k_Function_cuCoredumpGetAttributeG, sizeof(__pyx_k_Function_cuCoredumpGetAttributeG), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCoredumpGetAttributeG */ + {__pyx_k_Function_cuCoredumpSetAttribute, sizeof(__pyx_k_Function_cuCoredumpSetAttribute), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCoredumpSetAttribute */ + {__pyx_k_Function_cuCoredumpSetAttributeG, sizeof(__pyx_k_Function_cuCoredumpSetAttributeG), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCoredumpSetAttributeG */ + {__pyx_k_Function_cuCtxAttach_not_found, sizeof(__pyx_k_Function_cuCtxAttach_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxAttach_not_found */ + {__pyx_k_Function_cuCtxCreate_v2_not_foun, sizeof(__pyx_k_Function_cuCtxCreate_v2_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxCreate_v2_not_foun */ + {__pyx_k_Function_cuCtxCreate_v3_not_foun, sizeof(__pyx_k_Function_cuCtxCreate_v3_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxCreate_v3_not_foun */ + {__pyx_k_Function_cuCtxCreate_v4_not_foun, sizeof(__pyx_k_Function_cuCtxCreate_v4_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxCreate_v4_not_foun */ + {__pyx_k_Function_cuCtxDestroy_v2_not_fou, sizeof(__pyx_k_Function_cuCtxDestroy_v2_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxDestroy_v2_not_fou */ + {__pyx_k_Function_cuCtxDetach_not_found, sizeof(__pyx_k_Function_cuCtxDetach_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxDetach_not_found */ + {__pyx_k_Function_cuCtxDisablePeerAccess, sizeof(__pyx_k_Function_cuCtxDisablePeerAccess), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxDisablePeerAccess */ + {__pyx_k_Function_cuCtxEnablePeerAccess_n, sizeof(__pyx_k_Function_cuCtxEnablePeerAccess_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxEnablePeerAccess_n */ + {__pyx_k_Function_cuCtxFromGreenCtx_not_f, sizeof(__pyx_k_Function_cuCtxFromGreenCtx_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxFromGreenCtx_not_f */ + {__pyx_k_Function_cuCtxGetApiVersion_not, sizeof(__pyx_k_Function_cuCtxGetApiVersion_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxGetApiVersion_not */ + {__pyx_k_Function_cuCtxGetCacheConfig_not, sizeof(__pyx_k_Function_cuCtxGetCacheConfig_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxGetCacheConfig_not */ + {__pyx_k_Function_cuCtxGetCurrent_not_fou, sizeof(__pyx_k_Function_cuCtxGetCurrent_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxGetCurrent_not_fou */ + {__pyx_k_Function_cuCtxGetDevResource_not, sizeof(__pyx_k_Function_cuCtxGetDevResource_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxGetDevResource_not */ + {__pyx_k_Function_cuCtxGetDevice_not_foun, sizeof(__pyx_k_Function_cuCtxGetDevice_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxGetDevice_not_foun */ + {__pyx_k_Function_cuCtxGetExecAffinity_no, sizeof(__pyx_k_Function_cuCtxGetExecAffinity_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxGetExecAffinity_no */ + {__pyx_k_Function_cuCtxGetFlags_not_found, sizeof(__pyx_k_Function_cuCtxGetFlags_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxGetFlags_not_found */ + {__pyx_k_Function_cuCtxGetId_not_found, sizeof(__pyx_k_Function_cuCtxGetId_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxGetId_not_found */ + {__pyx_k_Function_cuCtxGetLimit_not_found, sizeof(__pyx_k_Function_cuCtxGetLimit_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxGetLimit_not_found */ + {__pyx_k_Function_cuCtxGetSharedMemConfig, sizeof(__pyx_k_Function_cuCtxGetSharedMemConfig), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxGetSharedMemConfig */ + {__pyx_k_Function_cuCtxGetStreamPriorityR, sizeof(__pyx_k_Function_cuCtxGetStreamPriorityR), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxGetStreamPriorityR */ + {__pyx_k_Function_cuCtxPopCurrent_v2_not, sizeof(__pyx_k_Function_cuCtxPopCurrent_v2_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxPopCurrent_v2_not */ + {__pyx_k_Function_cuCtxPushCurrent_v2_not, sizeof(__pyx_k_Function_cuCtxPushCurrent_v2_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxPushCurrent_v2_not */ + {__pyx_k_Function_cuCtxRecordEvent_not_fo, sizeof(__pyx_k_Function_cuCtxRecordEvent_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxRecordEvent_not_fo */ + {__pyx_k_Function_cuCtxResetPersistingL2C, sizeof(__pyx_k_Function_cuCtxResetPersistingL2C), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxResetPersistingL2C */ + {__pyx_k_Function_cuCtxSetCacheConfig_not, sizeof(__pyx_k_Function_cuCtxSetCacheConfig_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxSetCacheConfig_not */ + {__pyx_k_Function_cuCtxSetCurrent_not_fou, sizeof(__pyx_k_Function_cuCtxSetCurrent_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxSetCurrent_not_fou */ + {__pyx_k_Function_cuCtxSetFlags_not_found, sizeof(__pyx_k_Function_cuCtxSetFlags_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxSetFlags_not_found */ + {__pyx_k_Function_cuCtxSetLimit_not_found, sizeof(__pyx_k_Function_cuCtxSetLimit_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxSetLimit_not_found */ + {__pyx_k_Function_cuCtxSetSharedMemConfig, sizeof(__pyx_k_Function_cuCtxSetSharedMemConfig), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxSetSharedMemConfig */ + {__pyx_k_Function_cuCtxSynchronize_not_fo, sizeof(__pyx_k_Function_cuCtxSynchronize_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxSynchronize_not_fo */ + {__pyx_k_Function_cuCtxWaitEvent_not_foun, sizeof(__pyx_k_Function_cuCtxWaitEvent_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuCtxWaitEvent_not_foun */ + {__pyx_k_Function_cuDestroyExternalMemory, sizeof(__pyx_k_Function_cuDestroyExternalMemory), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDestroyExternalMemory */ + {__pyx_k_Function_cuDestroyExternalSemaph, sizeof(__pyx_k_Function_cuDestroyExternalSemaph), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDestroyExternalSemaph */ + {__pyx_k_Function_cuDevResourceGenerateDe, sizeof(__pyx_k_Function_cuDevResourceGenerateDe), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDevResourceGenerateDe */ + {__pyx_k_Function_cuDevSmResourceSplitByC, sizeof(__pyx_k_Function_cuDevSmResourceSplitByC), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDevSmResourceSplitByC */ + {__pyx_k_Function_cuDeviceCanAccessPeer_n, sizeof(__pyx_k_Function_cuDeviceCanAccessPeer_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceCanAccessPeer_n */ + {__pyx_k_Function_cuDeviceComputeCapabili, sizeof(__pyx_k_Function_cuDeviceComputeCapabili), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceComputeCapabili */ + {__pyx_k_Function_cuDeviceGetAttribute_no, sizeof(__pyx_k_Function_cuDeviceGetAttribute_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetAttribute_no */ + {__pyx_k_Function_cuDeviceGetByPCIBusId_n, sizeof(__pyx_k_Function_cuDeviceGetByPCIBusId_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetByPCIBusId_n */ + {__pyx_k_Function_cuDeviceGetCount_not_fo, sizeof(__pyx_k_Function_cuDeviceGetCount_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetCount_not_fo */ + {__pyx_k_Function_cuDeviceGetDefaultMemPo, sizeof(__pyx_k_Function_cuDeviceGetDefaultMemPo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetDefaultMemPo */ + {__pyx_k_Function_cuDeviceGetDevResource, sizeof(__pyx_k_Function_cuDeviceGetDevResource), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetDevResource */ + {__pyx_k_Function_cuDeviceGetExecAffinity, sizeof(__pyx_k_Function_cuDeviceGetExecAffinity), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetExecAffinity */ + {__pyx_k_Function_cuDeviceGetGraphMemAttr, sizeof(__pyx_k_Function_cuDeviceGetGraphMemAttr), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetGraphMemAttr */ + {__pyx_k_Function_cuDeviceGetLuid_not_fou, sizeof(__pyx_k_Function_cuDeviceGetLuid_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetLuid_not_fou */ + {__pyx_k_Function_cuDeviceGetMemPool_not, sizeof(__pyx_k_Function_cuDeviceGetMemPool_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetMemPool_not */ + {__pyx_k_Function_cuDeviceGetName_not_fou, sizeof(__pyx_k_Function_cuDeviceGetName_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetName_not_fou */ + {__pyx_k_Function_cuDeviceGetNvSciSyncAtt, sizeof(__pyx_k_Function_cuDeviceGetNvSciSyncAtt), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetNvSciSyncAtt */ + {__pyx_k_Function_cuDeviceGetP2PAttribute, sizeof(__pyx_k_Function_cuDeviceGetP2PAttribute), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetP2PAttribute */ + {__pyx_k_Function_cuDeviceGetPCIBusId_not, sizeof(__pyx_k_Function_cuDeviceGetPCIBusId_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetPCIBusId_not */ + {__pyx_k_Function_cuDeviceGetProperties_n, sizeof(__pyx_k_Function_cuDeviceGetProperties_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetProperties_n */ + {__pyx_k_Function_cuDeviceGetTexture1DLin, sizeof(__pyx_k_Function_cuDeviceGetTexture1DLin), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetTexture1DLin */ + {__pyx_k_Function_cuDeviceGetUuid_not_fou, sizeof(__pyx_k_Function_cuDeviceGetUuid_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetUuid_not_fou */ + {__pyx_k_Function_cuDeviceGetUuid_v2_not, sizeof(__pyx_k_Function_cuDeviceGetUuid_v2_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGetUuid_v2_not */ + {__pyx_k_Function_cuDeviceGet_not_found, sizeof(__pyx_k_Function_cuDeviceGet_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGet_not_found */ + {__pyx_k_Function_cuDeviceGraphMemTrim_no, sizeof(__pyx_k_Function_cuDeviceGraphMemTrim_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceGraphMemTrim_no */ + {__pyx_k_Function_cuDevicePrimaryCtxGetSt, sizeof(__pyx_k_Function_cuDevicePrimaryCtxGetSt), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDevicePrimaryCtxGetSt */ + {__pyx_k_Function_cuDevicePrimaryCtxRelea, sizeof(__pyx_k_Function_cuDevicePrimaryCtxRelea), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDevicePrimaryCtxRelea */ + {__pyx_k_Function_cuDevicePrimaryCtxReset, sizeof(__pyx_k_Function_cuDevicePrimaryCtxReset), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDevicePrimaryCtxReset */ + {__pyx_k_Function_cuDevicePrimaryCtxRetai, sizeof(__pyx_k_Function_cuDevicePrimaryCtxRetai), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDevicePrimaryCtxRetai */ + {__pyx_k_Function_cuDevicePrimaryCtxSetFl, sizeof(__pyx_k_Function_cuDevicePrimaryCtxSetFl), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDevicePrimaryCtxSetFl */ + {__pyx_k_Function_cuDeviceRegisterAsyncNo, sizeof(__pyx_k_Function_cuDeviceRegisterAsyncNo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceRegisterAsyncNo */ + {__pyx_k_Function_cuDeviceSetGraphMemAttr, sizeof(__pyx_k_Function_cuDeviceSetGraphMemAttr), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceSetGraphMemAttr */ + {__pyx_k_Function_cuDeviceSetMemPool_not, sizeof(__pyx_k_Function_cuDeviceSetMemPool_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceSetMemPool_not */ + {__pyx_k_Function_cuDeviceTotalMem_v2_not, sizeof(__pyx_k_Function_cuDeviceTotalMem_v2_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceTotalMem_v2_not */ + {__pyx_k_Function_cuDeviceUnregisterAsync, sizeof(__pyx_k_Function_cuDeviceUnregisterAsync), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDeviceUnregisterAsync */ + {__pyx_k_Function_cuDriverGetVersion_not, sizeof(__pyx_k_Function_cuDriverGetVersion_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuDriverGetVersion_not */ + {__pyx_k_Function_cuEGLStreamConsumerAcqu, sizeof(__pyx_k_Function_cuEGLStreamConsumerAcqu), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEGLStreamConsumerAcqu */ + {__pyx_k_Function_cuEGLStreamConsumerConn, sizeof(__pyx_k_Function_cuEGLStreamConsumerConn), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEGLStreamConsumerConn */ + {__pyx_k_Function_cuEGLStreamConsumerConn_2, sizeof(__pyx_k_Function_cuEGLStreamConsumerConn_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEGLStreamConsumerConn_2 */ + {__pyx_k_Function_cuEGLStreamConsumerDisc, sizeof(__pyx_k_Function_cuEGLStreamConsumerDisc), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEGLStreamConsumerDisc */ + {__pyx_k_Function_cuEGLStreamConsumerRele, sizeof(__pyx_k_Function_cuEGLStreamConsumerRele), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEGLStreamConsumerRele */ + {__pyx_k_Function_cuEGLStreamProducerConn, sizeof(__pyx_k_Function_cuEGLStreamProducerConn), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEGLStreamProducerConn */ + {__pyx_k_Function_cuEGLStreamProducerDisc, sizeof(__pyx_k_Function_cuEGLStreamProducerDisc), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEGLStreamProducerDisc */ + {__pyx_k_Function_cuEGLStreamProducerPres, sizeof(__pyx_k_Function_cuEGLStreamProducerPres), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEGLStreamProducerPres */ + {__pyx_k_Function_cuEGLStreamProducerRetu, sizeof(__pyx_k_Function_cuEGLStreamProducerRetu), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEGLStreamProducerRetu */ + {__pyx_k_Function_cuEventCreateFromEGLSyn, sizeof(__pyx_k_Function_cuEventCreateFromEGLSyn), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEventCreateFromEGLSyn */ + {__pyx_k_Function_cuEventCreate_not_found, sizeof(__pyx_k_Function_cuEventCreate_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEventCreate_not_found */ + {__pyx_k_Function_cuEventDestroy_v2_not_f, sizeof(__pyx_k_Function_cuEventDestroy_v2_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEventDestroy_v2_not_f */ + {__pyx_k_Function_cuEventElapsedTime_not, sizeof(__pyx_k_Function_cuEventElapsedTime_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEventElapsedTime_not */ + {__pyx_k_Function_cuEventElapsedTime_v2_n, sizeof(__pyx_k_Function_cuEventElapsedTime_v2_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEventElapsedTime_v2_n */ + {__pyx_k_Function_cuEventQuery_not_found, sizeof(__pyx_k_Function_cuEventQuery_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEventQuery_not_found */ + {__pyx_k_Function_cuEventRecordWithFlags, sizeof(__pyx_k_Function_cuEventRecordWithFlags), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEventRecordWithFlags */ + {__pyx_k_Function_cuEventRecord_not_found, sizeof(__pyx_k_Function_cuEventRecord_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEventRecord_not_found */ + {__pyx_k_Function_cuEventSynchronize_not, sizeof(__pyx_k_Function_cuEventSynchronize_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuEventSynchronize_not */ + {__pyx_k_Function_cuExternalMemoryGetMapp, sizeof(__pyx_k_Function_cuExternalMemoryGetMapp), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuExternalMemoryGetMapp */ + {__pyx_k_Function_cuExternalMemoryGetMapp_2, sizeof(__pyx_k_Function_cuExternalMemoryGetMapp_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuExternalMemoryGetMapp_2 */ + {__pyx_k_Function_cuFlushGPUDirectRDMAWri, sizeof(__pyx_k_Function_cuFlushGPUDirectRDMAWri), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuFlushGPUDirectRDMAWri */ + {__pyx_k_Function_cuFuncGetAttribute_not, sizeof(__pyx_k_Function_cuFuncGetAttribute_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuFuncGetAttribute_not */ + {__pyx_k_Function_cuFuncGetModule_not_fou, sizeof(__pyx_k_Function_cuFuncGetModule_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuFuncGetModule_not_fou */ + {__pyx_k_Function_cuFuncGetName_not_found, sizeof(__pyx_k_Function_cuFuncGetName_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuFuncGetName_not_found */ + {__pyx_k_Function_cuFuncGetParamInfo_not, sizeof(__pyx_k_Function_cuFuncGetParamInfo_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuFuncGetParamInfo_not */ + {__pyx_k_Function_cuFuncIsLoaded_not_foun, sizeof(__pyx_k_Function_cuFuncIsLoaded_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuFuncIsLoaded_not_foun */ + {__pyx_k_Function_cuFuncLoad_not_found, sizeof(__pyx_k_Function_cuFuncLoad_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuFuncLoad_not_found */ + {__pyx_k_Function_cuFuncSetAttribute_not, sizeof(__pyx_k_Function_cuFuncSetAttribute_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuFuncSetAttribute_not */ + {__pyx_k_Function_cuFuncSetBlockShape_not, sizeof(__pyx_k_Function_cuFuncSetBlockShape_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuFuncSetBlockShape_not */ + {__pyx_k_Function_cuFuncSetCacheConfig_no, sizeof(__pyx_k_Function_cuFuncSetCacheConfig_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuFuncSetCacheConfig_no */ + {__pyx_k_Function_cuFuncSetSharedMemConfi, sizeof(__pyx_k_Function_cuFuncSetSharedMemConfi), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuFuncSetSharedMemConfi */ + {__pyx_k_Function_cuFuncSetSharedSize_not, sizeof(__pyx_k_Function_cuFuncSetSharedSize_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuFuncSetSharedSize_not */ + {__pyx_k_Function_cuGLGetDevices_v2_not_f, sizeof(__pyx_k_Function_cuGLGetDevices_v2_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGLGetDevices_v2_not_f */ + {__pyx_k_Function_cuGetErrorName_not_foun, sizeof(__pyx_k_Function_cuGetErrorName_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGetErrorName_not_foun */ + {__pyx_k_Function_cuGetErrorString_not_fo, sizeof(__pyx_k_Function_cuGetErrorString_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGetErrorString_not_fo */ + {__pyx_k_Function_cuGetExportTable_not_fo, sizeof(__pyx_k_Function_cuGetExportTable_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGetExportTable_not_fo */ + {__pyx_k_Function_cuGetProcAddress_v2_not, sizeof(__pyx_k_Function_cuGetProcAddress_v2_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGetProcAddress_v2_not */ + {__pyx_k_Function_cuGraphAddBatchMemOpNod, sizeof(__pyx_k_Function_cuGraphAddBatchMemOpNod), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddBatchMemOpNod */ + {__pyx_k_Function_cuGraphAddChildGraphNod, sizeof(__pyx_k_Function_cuGraphAddChildGraphNod), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddChildGraphNod */ + {__pyx_k_Function_cuGraphAddDependencies, sizeof(__pyx_k_Function_cuGraphAddDependencies), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddDependencies */ + {__pyx_k_Function_cuGraphAddDependencies_2, sizeof(__pyx_k_Function_cuGraphAddDependencies_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddDependencies_2 */ + {__pyx_k_Function_cuGraphAddEmptyNode_not, sizeof(__pyx_k_Function_cuGraphAddEmptyNode_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddEmptyNode_not */ + {__pyx_k_Function_cuGraphAddEventRecordNo, sizeof(__pyx_k_Function_cuGraphAddEventRecordNo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddEventRecordNo */ + {__pyx_k_Function_cuGraphAddEventWaitNode, sizeof(__pyx_k_Function_cuGraphAddEventWaitNode), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddEventWaitNode */ + {__pyx_k_Function_cuGraphAddExternalSemap, sizeof(__pyx_k_Function_cuGraphAddExternalSemap), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddExternalSemap */ + {__pyx_k_Function_cuGraphAddExternalSemap_2, sizeof(__pyx_k_Function_cuGraphAddExternalSemap_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddExternalSemap_2 */ + {__pyx_k_Function_cuGraphAddHostNode_not, sizeof(__pyx_k_Function_cuGraphAddHostNode_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddHostNode_not */ + {__pyx_k_Function_cuGraphAddKernelNode_v2, sizeof(__pyx_k_Function_cuGraphAddKernelNode_v2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddKernelNode_v2 */ + {__pyx_k_Function_cuGraphAddMemAllocNode, sizeof(__pyx_k_Function_cuGraphAddMemAllocNode), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddMemAllocNode */ + {__pyx_k_Function_cuGraphAddMemFreeNode_n, sizeof(__pyx_k_Function_cuGraphAddMemFreeNode_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddMemFreeNode_n */ + {__pyx_k_Function_cuGraphAddMemcpyNode_no, sizeof(__pyx_k_Function_cuGraphAddMemcpyNode_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddMemcpyNode_no */ + {__pyx_k_Function_cuGraphAddMemsetNode_no, sizeof(__pyx_k_Function_cuGraphAddMemsetNode_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddMemsetNode_no */ + {__pyx_k_Function_cuGraphAddNode_not_foun, sizeof(__pyx_k_Function_cuGraphAddNode_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddNode_not_foun */ + {__pyx_k_Function_cuGraphAddNode_v2_not_f, sizeof(__pyx_k_Function_cuGraphAddNode_v2_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphAddNode_v2_not_f */ + {__pyx_k_Function_cuGraphBatchMemOpNodeGe, sizeof(__pyx_k_Function_cuGraphBatchMemOpNodeGe), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphBatchMemOpNodeGe */ + {__pyx_k_Function_cuGraphBatchMemOpNodeSe, sizeof(__pyx_k_Function_cuGraphBatchMemOpNodeSe), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphBatchMemOpNodeSe */ + {__pyx_k_Function_cuGraphChildGraphNodeGe, sizeof(__pyx_k_Function_cuGraphChildGraphNodeGe), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphChildGraphNodeGe */ + {__pyx_k_Function_cuGraphClone_not_found, sizeof(__pyx_k_Function_cuGraphClone_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphClone_not_found */ + {__pyx_k_Function_cuGraphConditionalHandl, sizeof(__pyx_k_Function_cuGraphConditionalHandl), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphConditionalHandl */ + {__pyx_k_Function_cuGraphCreate_not_found, sizeof(__pyx_k_Function_cuGraphCreate_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphCreate_not_found */ + {__pyx_k_Function_cuGraphDebugDotPrint_no, sizeof(__pyx_k_Function_cuGraphDebugDotPrint_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphDebugDotPrint_no */ + {__pyx_k_Function_cuGraphDestroyNode_not, sizeof(__pyx_k_Function_cuGraphDestroyNode_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphDestroyNode_not */ + {__pyx_k_Function_cuGraphDestroy_not_foun, sizeof(__pyx_k_Function_cuGraphDestroy_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphDestroy_not_foun */ + {__pyx_k_Function_cuGraphEventRecordNodeG, sizeof(__pyx_k_Function_cuGraphEventRecordNodeG), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphEventRecordNodeG */ + {__pyx_k_Function_cuGraphEventRecordNodeS, sizeof(__pyx_k_Function_cuGraphEventRecordNodeS), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphEventRecordNodeS */ + {__pyx_k_Function_cuGraphEventWaitNodeGet, sizeof(__pyx_k_Function_cuGraphEventWaitNodeGet), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphEventWaitNodeGet */ + {__pyx_k_Function_cuGraphEventWaitNodeSet, sizeof(__pyx_k_Function_cuGraphEventWaitNodeSet), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphEventWaitNodeSet */ + {__pyx_k_Function_cuGraphExecBatchMemOpNo, sizeof(__pyx_k_Function_cuGraphExecBatchMemOpNo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecBatchMemOpNo */ + {__pyx_k_Function_cuGraphExecChildGraphNo, sizeof(__pyx_k_Function_cuGraphExecChildGraphNo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecChildGraphNo */ + {__pyx_k_Function_cuGraphExecDestroy_not, sizeof(__pyx_k_Function_cuGraphExecDestroy_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecDestroy_not */ + {__pyx_k_Function_cuGraphExecEventRecordN, sizeof(__pyx_k_Function_cuGraphExecEventRecordN), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecEventRecordN */ + {__pyx_k_Function_cuGraphExecEventWaitNod, sizeof(__pyx_k_Function_cuGraphExecEventWaitNod), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecEventWaitNod */ + {__pyx_k_Function_cuGraphExecExternalSema, sizeof(__pyx_k_Function_cuGraphExecExternalSema), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecExternalSema */ + {__pyx_k_Function_cuGraphExecExternalSema_2, sizeof(__pyx_k_Function_cuGraphExecExternalSema_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecExternalSema_2 */ + {__pyx_k_Function_cuGraphExecGetFlags_not, sizeof(__pyx_k_Function_cuGraphExecGetFlags_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecGetFlags_not */ + {__pyx_k_Function_cuGraphExecHostNodeSetP, sizeof(__pyx_k_Function_cuGraphExecHostNodeSetP), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecHostNodeSetP */ + {__pyx_k_Function_cuGraphExecKernelNodeSe, sizeof(__pyx_k_Function_cuGraphExecKernelNodeSe), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecKernelNodeSe */ + {__pyx_k_Function_cuGraphExecMemcpyNodeSe, sizeof(__pyx_k_Function_cuGraphExecMemcpyNodeSe), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecMemcpyNodeSe */ + {__pyx_k_Function_cuGraphExecMemsetNodeSe, sizeof(__pyx_k_Function_cuGraphExecMemsetNodeSe), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecMemsetNodeSe */ + {__pyx_k_Function_cuGraphExecNodeSetParam, sizeof(__pyx_k_Function_cuGraphExecNodeSetParam), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecNodeSetParam */ + {__pyx_k_Function_cuGraphExecUpdate_v2_no, sizeof(__pyx_k_Function_cuGraphExecUpdate_v2_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExecUpdate_v2_no */ + {__pyx_k_Function_cuGraphExternalSemaphor, sizeof(__pyx_k_Function_cuGraphExternalSemaphor), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExternalSemaphor */ + {__pyx_k_Function_cuGraphExternalSemaphor_2, sizeof(__pyx_k_Function_cuGraphExternalSemaphor_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExternalSemaphor_2 */ + {__pyx_k_Function_cuGraphExternalSemaphor_3, sizeof(__pyx_k_Function_cuGraphExternalSemaphor_3), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExternalSemaphor_3 */ + {__pyx_k_Function_cuGraphExternalSemaphor_4, sizeof(__pyx_k_Function_cuGraphExternalSemaphor_4), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphExternalSemaphor_4 */ + {__pyx_k_Function_cuGraphGetEdges_not_fou, sizeof(__pyx_k_Function_cuGraphGetEdges_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphGetEdges_not_fou */ + {__pyx_k_Function_cuGraphGetEdges_v2_not, sizeof(__pyx_k_Function_cuGraphGetEdges_v2_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphGetEdges_v2_not */ + {__pyx_k_Function_cuGraphGetNodes_not_fou, sizeof(__pyx_k_Function_cuGraphGetNodes_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphGetNodes_not_fou */ + {__pyx_k_Function_cuGraphGetRootNodes_not, sizeof(__pyx_k_Function_cuGraphGetRootNodes_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphGetRootNodes_not */ + {__pyx_k_Function_cuGraphHostNodeGetParam, sizeof(__pyx_k_Function_cuGraphHostNodeGetParam), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphHostNodeGetParam */ + {__pyx_k_Function_cuGraphHostNodeSetParam, sizeof(__pyx_k_Function_cuGraphHostNodeSetParam), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphHostNodeSetParam */ + {__pyx_k_Function_cuGraphInstantiateWithF, sizeof(__pyx_k_Function_cuGraphInstantiateWithF), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphInstantiateWithF */ + {__pyx_k_Function_cuGraphInstantiateWithP, sizeof(__pyx_k_Function_cuGraphInstantiateWithP), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphInstantiateWithP */ + {__pyx_k_Function_cuGraphKernelNodeCopyAt, sizeof(__pyx_k_Function_cuGraphKernelNodeCopyAt), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphKernelNodeCopyAt */ + {__pyx_k_Function_cuGraphKernelNodeGetAtt, sizeof(__pyx_k_Function_cuGraphKernelNodeGetAtt), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphKernelNodeGetAtt */ + {__pyx_k_Function_cuGraphKernelNodeGetPar, sizeof(__pyx_k_Function_cuGraphKernelNodeGetPar), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphKernelNodeGetPar */ + {__pyx_k_Function_cuGraphKernelNodeSetAtt, sizeof(__pyx_k_Function_cuGraphKernelNodeSetAtt), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphKernelNodeSetAtt */ + {__pyx_k_Function_cuGraphKernelNodeSetPar, sizeof(__pyx_k_Function_cuGraphKernelNodeSetPar), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphKernelNodeSetPar */ + {__pyx_k_Function_cuGraphLaunch_not_found, sizeof(__pyx_k_Function_cuGraphLaunch_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphLaunch_not_found */ + {__pyx_k_Function_cuGraphMemAllocNodeGetP, sizeof(__pyx_k_Function_cuGraphMemAllocNodeGetP), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphMemAllocNodeGetP */ + {__pyx_k_Function_cuGraphMemFreeNodeGetPa, sizeof(__pyx_k_Function_cuGraphMemFreeNodeGetPa), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphMemFreeNodeGetPa */ + {__pyx_k_Function_cuGraphMemcpyNodeGetPar, sizeof(__pyx_k_Function_cuGraphMemcpyNodeGetPar), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphMemcpyNodeGetPar */ + {__pyx_k_Function_cuGraphMemcpyNodeSetPar, sizeof(__pyx_k_Function_cuGraphMemcpyNodeSetPar), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphMemcpyNodeSetPar */ + {__pyx_k_Function_cuGraphMemsetNodeGetPar, sizeof(__pyx_k_Function_cuGraphMemsetNodeGetPar), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphMemsetNodeGetPar */ + {__pyx_k_Function_cuGraphMemsetNodeSetPar, sizeof(__pyx_k_Function_cuGraphMemsetNodeSetPar), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphMemsetNodeSetPar */ + {__pyx_k_Function_cuGraphNodeFindInClone, sizeof(__pyx_k_Function_cuGraphNodeFindInClone), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphNodeFindInClone */ + {__pyx_k_Function_cuGraphNodeGetDependenc, sizeof(__pyx_k_Function_cuGraphNodeGetDependenc), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphNodeGetDependenc */ + {__pyx_k_Function_cuGraphNodeGetDependenc_2, sizeof(__pyx_k_Function_cuGraphNodeGetDependenc_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphNodeGetDependenc_2 */ + {__pyx_k_Function_cuGraphNodeGetDependent, sizeof(__pyx_k_Function_cuGraphNodeGetDependent), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphNodeGetDependent */ + {__pyx_k_Function_cuGraphNodeGetDependent_2, sizeof(__pyx_k_Function_cuGraphNodeGetDependent_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphNodeGetDependent_2 */ + {__pyx_k_Function_cuGraphNodeGetEnabled_n, sizeof(__pyx_k_Function_cuGraphNodeGetEnabled_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphNodeGetEnabled_n */ + {__pyx_k_Function_cuGraphNodeGetType_not, sizeof(__pyx_k_Function_cuGraphNodeGetType_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphNodeGetType_not */ + {__pyx_k_Function_cuGraphNodeSetEnabled_n, sizeof(__pyx_k_Function_cuGraphNodeSetEnabled_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphNodeSetEnabled_n */ + {__pyx_k_Function_cuGraphNodeSetParams_no, sizeof(__pyx_k_Function_cuGraphNodeSetParams_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphNodeSetParams_no */ + {__pyx_k_Function_cuGraphReleaseUserObjec, sizeof(__pyx_k_Function_cuGraphReleaseUserObjec), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphReleaseUserObjec */ + {__pyx_k_Function_cuGraphRemoveDependenci, sizeof(__pyx_k_Function_cuGraphRemoveDependenci), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphRemoveDependenci */ + {__pyx_k_Function_cuGraphRemoveDependenci_2, sizeof(__pyx_k_Function_cuGraphRemoveDependenci_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphRemoveDependenci_2 */ + {__pyx_k_Function_cuGraphRetainUserObject, sizeof(__pyx_k_Function_cuGraphRetainUserObject), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphRetainUserObject */ + {__pyx_k_Function_cuGraphUpload_not_found, sizeof(__pyx_k_Function_cuGraphUpload_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphUpload_not_found */ + {__pyx_k_Function_cuGraphicsEGLRegisterIm, sizeof(__pyx_k_Function_cuGraphicsEGLRegisterIm), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphicsEGLRegisterIm */ + {__pyx_k_Function_cuGraphicsGLRegisterBuf, sizeof(__pyx_k_Function_cuGraphicsGLRegisterBuf), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphicsGLRegisterBuf */ + {__pyx_k_Function_cuGraphicsGLRegisterIma, sizeof(__pyx_k_Function_cuGraphicsGLRegisterIma), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphicsGLRegisterIma */ + {__pyx_k_Function_cuGraphicsMapResources, sizeof(__pyx_k_Function_cuGraphicsMapResources), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphicsMapResources */ + {__pyx_k_Function_cuGraphicsResourceGetMa, sizeof(__pyx_k_Function_cuGraphicsResourceGetMa), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphicsResourceGetMa */ + {__pyx_k_Function_cuGraphicsResourceGetMa_2, sizeof(__pyx_k_Function_cuGraphicsResourceGetMa_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphicsResourceGetMa_2 */ + {__pyx_k_Function_cuGraphicsResourceGetMa_3, sizeof(__pyx_k_Function_cuGraphicsResourceGetMa_3), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphicsResourceGetMa_3 */ + {__pyx_k_Function_cuGraphicsResourceSetMa, sizeof(__pyx_k_Function_cuGraphicsResourceSetMa), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphicsResourceSetMa */ + {__pyx_k_Function_cuGraphicsSubResourceGe, sizeof(__pyx_k_Function_cuGraphicsSubResourceGe), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphicsSubResourceGe */ + {__pyx_k_Function_cuGraphicsUnmapResource, sizeof(__pyx_k_Function_cuGraphicsUnmapResource), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphicsUnmapResource */ + {__pyx_k_Function_cuGraphicsUnregisterRes, sizeof(__pyx_k_Function_cuGraphicsUnregisterRes), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphicsUnregisterRes */ + {__pyx_k_Function_cuGraphicsVDPAURegister, sizeof(__pyx_k_Function_cuGraphicsVDPAURegister), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphicsVDPAURegister */ + {__pyx_k_Function_cuGraphicsVDPAURegister_2, sizeof(__pyx_k_Function_cuGraphicsVDPAURegister_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGraphicsVDPAURegister_2 */ + {__pyx_k_Function_cuGreenCtxCreate_not_fo, sizeof(__pyx_k_Function_cuGreenCtxCreate_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGreenCtxCreate_not_fo */ + {__pyx_k_Function_cuGreenCtxDestroy_not_f, sizeof(__pyx_k_Function_cuGreenCtxDestroy_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGreenCtxDestroy_not_f */ + {__pyx_k_Function_cuGreenCtxGetDevResourc, sizeof(__pyx_k_Function_cuGreenCtxGetDevResourc), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGreenCtxGetDevResourc */ + {__pyx_k_Function_cuGreenCtxRecordEvent_n, sizeof(__pyx_k_Function_cuGreenCtxRecordEvent_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGreenCtxRecordEvent_n */ + {__pyx_k_Function_cuGreenCtxStreamCreate, sizeof(__pyx_k_Function_cuGreenCtxStreamCreate), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGreenCtxStreamCreate */ + {__pyx_k_Function_cuGreenCtxWaitEvent_not, sizeof(__pyx_k_Function_cuGreenCtxWaitEvent_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuGreenCtxWaitEvent_not */ + {__pyx_k_Function_cuImportExternalMemory, sizeof(__pyx_k_Function_cuImportExternalMemory), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuImportExternalMemory */ + {__pyx_k_Function_cuImportExternalSemapho, sizeof(__pyx_k_Function_cuImportExternalSemapho), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuImportExternalSemapho */ + {__pyx_k_Function_cuInit_not_found, sizeof(__pyx_k_Function_cuInit_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuInit_not_found */ + {__pyx_k_Function_cuIpcCloseMemHandle_not, sizeof(__pyx_k_Function_cuIpcCloseMemHandle_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuIpcCloseMemHandle_not */ + {__pyx_k_Function_cuIpcGetEventHandle_not, sizeof(__pyx_k_Function_cuIpcGetEventHandle_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuIpcGetEventHandle_not */ + {__pyx_k_Function_cuIpcGetMemHandle_not_f, sizeof(__pyx_k_Function_cuIpcGetMemHandle_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuIpcGetMemHandle_not_f */ + {__pyx_k_Function_cuIpcOpenEventHandle_no, sizeof(__pyx_k_Function_cuIpcOpenEventHandle_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuIpcOpenEventHandle_no */ + {__pyx_k_Function_cuIpcOpenMemHandle_v2_n, sizeof(__pyx_k_Function_cuIpcOpenMemHandle_v2_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuIpcOpenMemHandle_v2_n */ + {__pyx_k_Function_cuKernelGetAttribute_no, sizeof(__pyx_k_Function_cuKernelGetAttribute_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuKernelGetAttribute_no */ + {__pyx_k_Function_cuKernelGetFunction_not, sizeof(__pyx_k_Function_cuKernelGetFunction_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuKernelGetFunction_not */ + {__pyx_k_Function_cuKernelGetLibrary_not, sizeof(__pyx_k_Function_cuKernelGetLibrary_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuKernelGetLibrary_not */ + {__pyx_k_Function_cuKernelGetName_not_fou, sizeof(__pyx_k_Function_cuKernelGetName_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuKernelGetName_not_fou */ + {__pyx_k_Function_cuKernelGetParamInfo_no, sizeof(__pyx_k_Function_cuKernelGetParamInfo_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuKernelGetParamInfo_no */ + {__pyx_k_Function_cuKernelSetAttribute_no, sizeof(__pyx_k_Function_cuKernelSetAttribute_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuKernelSetAttribute_no */ + {__pyx_k_Function_cuKernelSetCacheConfig, sizeof(__pyx_k_Function_cuKernelSetCacheConfig), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuKernelSetCacheConfig */ + {__pyx_k_Function_cuLaunchCooperativeKern, sizeof(__pyx_k_Function_cuLaunchCooperativeKern), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLaunchCooperativeKern */ + {__pyx_k_Function_cuLaunchCooperativeKern_2, sizeof(__pyx_k_Function_cuLaunchCooperativeKern_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLaunchCooperativeKern_2 */ + {__pyx_k_Function_cuLaunchGridAsync_not_f, sizeof(__pyx_k_Function_cuLaunchGridAsync_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLaunchGridAsync_not_f */ + {__pyx_k_Function_cuLaunchGrid_not_found, sizeof(__pyx_k_Function_cuLaunchGrid_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLaunchGrid_not_found */ + {__pyx_k_Function_cuLaunchHostFunc_not_fo, sizeof(__pyx_k_Function_cuLaunchHostFunc_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLaunchHostFunc_not_fo */ + {__pyx_k_Function_cuLaunchKernelEx_not_fo, sizeof(__pyx_k_Function_cuLaunchKernelEx_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLaunchKernelEx_not_fo */ + {__pyx_k_Function_cuLaunchKernel_not_foun, sizeof(__pyx_k_Function_cuLaunchKernel_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLaunchKernel_not_foun */ + {__pyx_k_Function_cuLaunch_not_found, sizeof(__pyx_k_Function_cuLaunch_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLaunch_not_found */ + {__pyx_k_Function_cuLibraryEnumerateKerne, sizeof(__pyx_k_Function_cuLibraryEnumerateKerne), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLibraryEnumerateKerne */ + {__pyx_k_Function_cuLibraryGetGlobal_not, sizeof(__pyx_k_Function_cuLibraryGetGlobal_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLibraryGetGlobal_not */ + {__pyx_k_Function_cuLibraryGetKernelCount, sizeof(__pyx_k_Function_cuLibraryGetKernelCount), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLibraryGetKernelCount */ + {__pyx_k_Function_cuLibraryGetKernel_not, sizeof(__pyx_k_Function_cuLibraryGetKernel_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLibraryGetKernel_not */ + {__pyx_k_Function_cuLibraryGetManaged_not, sizeof(__pyx_k_Function_cuLibraryGetManaged_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLibraryGetManaged_not */ + {__pyx_k_Function_cuLibraryGetModule_not, sizeof(__pyx_k_Function_cuLibraryGetModule_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLibraryGetModule_not */ + {__pyx_k_Function_cuLibraryGetUnifiedFunc, sizeof(__pyx_k_Function_cuLibraryGetUnifiedFunc), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLibraryGetUnifiedFunc */ + {__pyx_k_Function_cuLibraryLoadData_not_f, sizeof(__pyx_k_Function_cuLibraryLoadData_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLibraryLoadData_not_f */ + {__pyx_k_Function_cuLibraryLoadFromFile_n, sizeof(__pyx_k_Function_cuLibraryLoadFromFile_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLibraryLoadFromFile_n */ + {__pyx_k_Function_cuLibraryUnload_not_fou, sizeof(__pyx_k_Function_cuLibraryUnload_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLibraryUnload_not_fou */ + {__pyx_k_Function_cuLinkAddData_v2_not_fo, sizeof(__pyx_k_Function_cuLinkAddData_v2_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLinkAddData_v2_not_fo */ + {__pyx_k_Function_cuLinkAddFile_v2_not_fo, sizeof(__pyx_k_Function_cuLinkAddFile_v2_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLinkAddFile_v2_not_fo */ + {__pyx_k_Function_cuLinkComplete_not_foun, sizeof(__pyx_k_Function_cuLinkComplete_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLinkComplete_not_foun */ + {__pyx_k_Function_cuLinkCreate_v2_not_fou, sizeof(__pyx_k_Function_cuLinkCreate_v2_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLinkCreate_v2_not_fou */ + {__pyx_k_Function_cuLinkDestroy_not_found, sizeof(__pyx_k_Function_cuLinkDestroy_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLinkDestroy_not_found */ + {__pyx_k_Function_cuLogsCurrent_not_found, sizeof(__pyx_k_Function_cuLogsCurrent_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLogsCurrent_not_found */ + {__pyx_k_Function_cuLogsDumpToFile_not_fo, sizeof(__pyx_k_Function_cuLogsDumpToFile_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLogsDumpToFile_not_fo */ + {__pyx_k_Function_cuLogsDumpToMemory_not, sizeof(__pyx_k_Function_cuLogsDumpToMemory_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLogsDumpToMemory_not */ + {__pyx_k_Function_cuLogsRegisterCallback, sizeof(__pyx_k_Function_cuLogsRegisterCallback), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLogsRegisterCallback */ + {__pyx_k_Function_cuLogsUnregisterCallbac, sizeof(__pyx_k_Function_cuLogsUnregisterCallbac), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuLogsUnregisterCallbac */ + {__pyx_k_Function_cuMemAddressFree_not_fo, sizeof(__pyx_k_Function_cuMemAddressFree_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemAddressFree_not_fo */ + {__pyx_k_Function_cuMemAddressReserve_not, sizeof(__pyx_k_Function_cuMemAddressReserve_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemAddressReserve_not */ + {__pyx_k_Function_cuMemAdvise_not_found, sizeof(__pyx_k_Function_cuMemAdvise_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemAdvise_not_found */ + {__pyx_k_Function_cuMemAdvise_v2_not_foun, sizeof(__pyx_k_Function_cuMemAdvise_v2_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemAdvise_v2_not_foun */ + {__pyx_k_Function_cuMemAllocAsync_not_fou, sizeof(__pyx_k_Function_cuMemAllocAsync_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemAllocAsync_not_fou */ + {__pyx_k_Function_cuMemAllocFromPoolAsync, sizeof(__pyx_k_Function_cuMemAllocFromPoolAsync), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemAllocFromPoolAsync */ + {__pyx_k_Function_cuMemAllocHost_v2_not_f, sizeof(__pyx_k_Function_cuMemAllocHost_v2_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemAllocHost_v2_not_f */ + {__pyx_k_Function_cuMemAllocManaged_not_f, sizeof(__pyx_k_Function_cuMemAllocManaged_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemAllocManaged_not_f */ + {__pyx_k_Function_cuMemAllocPitch_v2_not, sizeof(__pyx_k_Function_cuMemAllocPitch_v2_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemAllocPitch_v2_not */ + {__pyx_k_Function_cuMemAlloc_v2_not_found, sizeof(__pyx_k_Function_cuMemAlloc_v2_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemAlloc_v2_not_found */ + {__pyx_k_Function_cuMemBatchDecompressAsy, sizeof(__pyx_k_Function_cuMemBatchDecompressAsy), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemBatchDecompressAsy */ + {__pyx_k_Function_cuMemCreate_not_found, sizeof(__pyx_k_Function_cuMemCreate_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemCreate_not_found */ + {__pyx_k_Function_cuMemExportToShareableH, sizeof(__pyx_k_Function_cuMemExportToShareableH), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemExportToShareableH */ + {__pyx_k_Function_cuMemFreeAsync_not_foun, sizeof(__pyx_k_Function_cuMemFreeAsync_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemFreeAsync_not_foun */ + {__pyx_k_Function_cuMemFreeHost_not_found, sizeof(__pyx_k_Function_cuMemFreeHost_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemFreeHost_not_found */ + {__pyx_k_Function_cuMemFree_v2_not_found, sizeof(__pyx_k_Function_cuMemFree_v2_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemFree_v2_not_found */ + {__pyx_k_Function_cuMemGetAccess_not_foun, sizeof(__pyx_k_Function_cuMemGetAccess_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemGetAccess_not_foun */ + {__pyx_k_Function_cuMemGetAddressRange_v2, sizeof(__pyx_k_Function_cuMemGetAddressRange_v2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemGetAddressRange_v2 */ + {__pyx_k_Function_cuMemGetAllocationGranu, sizeof(__pyx_k_Function_cuMemGetAllocationGranu), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemGetAllocationGranu */ + {__pyx_k_Function_cuMemGetAllocationPrope, sizeof(__pyx_k_Function_cuMemGetAllocationPrope), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemGetAllocationPrope */ + {__pyx_k_Function_cuMemGetHandleForAddres, sizeof(__pyx_k_Function_cuMemGetHandleForAddres), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemGetHandleForAddres */ + {__pyx_k_Function_cuMemGetInfo_v2_not_fou, sizeof(__pyx_k_Function_cuMemGetInfo_v2_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemGetInfo_v2_not_fou */ + {__pyx_k_Function_cuMemHostAlloc_not_foun, sizeof(__pyx_k_Function_cuMemHostAlloc_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemHostAlloc_not_foun */ + {__pyx_k_Function_cuMemHostGetDevicePoint, sizeof(__pyx_k_Function_cuMemHostGetDevicePoint), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemHostGetDevicePoint */ + {__pyx_k_Function_cuMemHostGetFlags_not_f, sizeof(__pyx_k_Function_cuMemHostGetFlags_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemHostGetFlags_not_f */ + {__pyx_k_Function_cuMemHostRegister_v2_no, sizeof(__pyx_k_Function_cuMemHostRegister_v2_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemHostRegister_v2_no */ + {__pyx_k_Function_cuMemHostUnregister_not, sizeof(__pyx_k_Function_cuMemHostUnregister_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemHostUnregister_not */ + {__pyx_k_Function_cuMemImportFromShareabl, sizeof(__pyx_k_Function_cuMemImportFromShareabl), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemImportFromShareabl */ + {__pyx_k_Function_cuMemMapArrayAsync_not, sizeof(__pyx_k_Function_cuMemMapArrayAsync_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemMapArrayAsync_not */ + {__pyx_k_Function_cuMemMap_not_found, sizeof(__pyx_k_Function_cuMemMap_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemMap_not_found */ + {__pyx_k_Function_cuMemPoolCreate_not_fou, sizeof(__pyx_k_Function_cuMemPoolCreate_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemPoolCreate_not_fou */ + {__pyx_k_Function_cuMemPoolDestroy_not_fo, sizeof(__pyx_k_Function_cuMemPoolDestroy_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemPoolDestroy_not_fo */ + {__pyx_k_Function_cuMemPoolExportPointer, sizeof(__pyx_k_Function_cuMemPoolExportPointer), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemPoolExportPointer */ + {__pyx_k_Function_cuMemPoolExportToSharea, sizeof(__pyx_k_Function_cuMemPoolExportToSharea), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemPoolExportToSharea */ + {__pyx_k_Function_cuMemPoolGetAccess_not, sizeof(__pyx_k_Function_cuMemPoolGetAccess_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemPoolGetAccess_not */ + {__pyx_k_Function_cuMemPoolGetAttribute_n, sizeof(__pyx_k_Function_cuMemPoolGetAttribute_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemPoolGetAttribute_n */ + {__pyx_k_Function_cuMemPoolImportFromShar, sizeof(__pyx_k_Function_cuMemPoolImportFromShar), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemPoolImportFromShar */ + {__pyx_k_Function_cuMemPoolImportPointer, sizeof(__pyx_k_Function_cuMemPoolImportPointer), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemPoolImportPointer */ + {__pyx_k_Function_cuMemPoolSetAccess_not, sizeof(__pyx_k_Function_cuMemPoolSetAccess_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemPoolSetAccess_not */ + {__pyx_k_Function_cuMemPoolSetAttribute_n, sizeof(__pyx_k_Function_cuMemPoolSetAttribute_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemPoolSetAttribute_n */ + {__pyx_k_Function_cuMemPoolTrimTo_not_fou, sizeof(__pyx_k_Function_cuMemPoolTrimTo_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemPoolTrimTo_not_fou */ + {__pyx_k_Function_cuMemPrefetchAsync_not, sizeof(__pyx_k_Function_cuMemPrefetchAsync_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemPrefetchAsync_not */ + {__pyx_k_Function_cuMemPrefetchAsync_v2_n, sizeof(__pyx_k_Function_cuMemPrefetchAsync_v2_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemPrefetchAsync_v2_n */ + {__pyx_k_Function_cuMemRangeGetAttribute, sizeof(__pyx_k_Function_cuMemRangeGetAttribute), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemRangeGetAttribute */ + {__pyx_k_Function_cuMemRangeGetAttributes, sizeof(__pyx_k_Function_cuMemRangeGetAttributes), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemRangeGetAttributes */ + {__pyx_k_Function_cuMemRelease_not_found, sizeof(__pyx_k_Function_cuMemRelease_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemRelease_not_found */ + {__pyx_k_Function_cuMemRetainAllocationHa, sizeof(__pyx_k_Function_cuMemRetainAllocationHa), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemRetainAllocationHa */ + {__pyx_k_Function_cuMemSetAccess_not_foun, sizeof(__pyx_k_Function_cuMemSetAccess_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemSetAccess_not_foun */ + {__pyx_k_Function_cuMemUnmap_not_found, sizeof(__pyx_k_Function_cuMemUnmap_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemUnmap_not_found */ + {__pyx_k_Function_cuMemcpy2DAsync_v2_not, sizeof(__pyx_k_Function_cuMemcpy2DAsync_v2_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpy2DAsync_v2_not */ + {__pyx_k_Function_cuMemcpy2DUnaligned_v2, sizeof(__pyx_k_Function_cuMemcpy2DUnaligned_v2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpy2DUnaligned_v2 */ + {__pyx_k_Function_cuMemcpy2D_v2_not_found, sizeof(__pyx_k_Function_cuMemcpy2D_v2_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpy2D_v2_not_found */ + {__pyx_k_Function_cuMemcpy3DAsync_v2_not, sizeof(__pyx_k_Function_cuMemcpy3DAsync_v2_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpy3DAsync_v2_not */ + {__pyx_k_Function_cuMemcpy3DBatchAsync_no, sizeof(__pyx_k_Function_cuMemcpy3DBatchAsync_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpy3DBatchAsync_no */ + {__pyx_k_Function_cuMemcpy3DPeerAsync_not, sizeof(__pyx_k_Function_cuMemcpy3DPeerAsync_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpy3DPeerAsync_not */ + {__pyx_k_Function_cuMemcpy3DPeer_not_foun, sizeof(__pyx_k_Function_cuMemcpy3DPeer_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpy3DPeer_not_foun */ + {__pyx_k_Function_cuMemcpy3D_v2_not_found, sizeof(__pyx_k_Function_cuMemcpy3D_v2_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpy3D_v2_not_found */ + {__pyx_k_Function_cuMemcpyAsync_not_found, sizeof(__pyx_k_Function_cuMemcpyAsync_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyAsync_not_found */ + {__pyx_k_Function_cuMemcpyAtoA_v2_not_fou, sizeof(__pyx_k_Function_cuMemcpyAtoA_v2_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyAtoA_v2_not_fou */ + {__pyx_k_Function_cuMemcpyAtoD_v2_not_fou, sizeof(__pyx_k_Function_cuMemcpyAtoD_v2_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyAtoD_v2_not_fou */ + {__pyx_k_Function_cuMemcpyAtoHAsync_v2_no, sizeof(__pyx_k_Function_cuMemcpyAtoHAsync_v2_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyAtoHAsync_v2_no */ + {__pyx_k_Function_cuMemcpyAtoH_v2_not_fou, sizeof(__pyx_k_Function_cuMemcpyAtoH_v2_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyAtoH_v2_not_fou */ + {__pyx_k_Function_cuMemcpyBatchAsync_not, sizeof(__pyx_k_Function_cuMemcpyBatchAsync_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyBatchAsync_not */ + {__pyx_k_Function_cuMemcpyDtoA_v2_not_fou, sizeof(__pyx_k_Function_cuMemcpyDtoA_v2_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyDtoA_v2_not_fou */ + {__pyx_k_Function_cuMemcpyDtoDAsync_v2_no, sizeof(__pyx_k_Function_cuMemcpyDtoDAsync_v2_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyDtoDAsync_v2_no */ + {__pyx_k_Function_cuMemcpyDtoD_v2_not_fou, sizeof(__pyx_k_Function_cuMemcpyDtoD_v2_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyDtoD_v2_not_fou */ + {__pyx_k_Function_cuMemcpyDtoHAsync_v2_no, sizeof(__pyx_k_Function_cuMemcpyDtoHAsync_v2_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyDtoHAsync_v2_no */ + {__pyx_k_Function_cuMemcpyDtoH_v2_not_fou, sizeof(__pyx_k_Function_cuMemcpyDtoH_v2_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyDtoH_v2_not_fou */ + {__pyx_k_Function_cuMemcpyHtoAAsync_v2_no, sizeof(__pyx_k_Function_cuMemcpyHtoAAsync_v2_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyHtoAAsync_v2_no */ + {__pyx_k_Function_cuMemcpyHtoA_v2_not_fou, sizeof(__pyx_k_Function_cuMemcpyHtoA_v2_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyHtoA_v2_not_fou */ + {__pyx_k_Function_cuMemcpyHtoDAsync_v2_no, sizeof(__pyx_k_Function_cuMemcpyHtoDAsync_v2_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyHtoDAsync_v2_no */ + {__pyx_k_Function_cuMemcpyHtoD_v2_not_fou, sizeof(__pyx_k_Function_cuMemcpyHtoD_v2_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyHtoD_v2_not_fou */ + {__pyx_k_Function_cuMemcpyPeerAsync_not_f, sizeof(__pyx_k_Function_cuMemcpyPeerAsync_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyPeerAsync_not_f */ + {__pyx_k_Function_cuMemcpyPeer_not_found, sizeof(__pyx_k_Function_cuMemcpyPeer_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpyPeer_not_found */ + {__pyx_k_Function_cuMemcpy_not_found, sizeof(__pyx_k_Function_cuMemcpy_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemcpy_not_found */ + {__pyx_k_Function_cuMemsetD16Async_not_fo, sizeof(__pyx_k_Function_cuMemsetD16Async_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemsetD16Async_not_fo */ + {__pyx_k_Function_cuMemsetD16_v2_not_foun, sizeof(__pyx_k_Function_cuMemsetD16_v2_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemsetD16_v2_not_foun */ + {__pyx_k_Function_cuMemsetD2D16Async_not, sizeof(__pyx_k_Function_cuMemsetD2D16Async_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemsetD2D16Async_not */ + {__pyx_k_Function_cuMemsetD2D16_v2_not_fo, sizeof(__pyx_k_Function_cuMemsetD2D16_v2_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemsetD2D16_v2_not_fo */ + {__pyx_k_Function_cuMemsetD2D32Async_not, sizeof(__pyx_k_Function_cuMemsetD2D32Async_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemsetD2D32Async_not */ + {__pyx_k_Function_cuMemsetD2D32_v2_not_fo, sizeof(__pyx_k_Function_cuMemsetD2D32_v2_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemsetD2D32_v2_not_fo */ + {__pyx_k_Function_cuMemsetD2D8Async_not_f, sizeof(__pyx_k_Function_cuMemsetD2D8Async_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemsetD2D8Async_not_f */ + {__pyx_k_Function_cuMemsetD2D8_v2_not_fou, sizeof(__pyx_k_Function_cuMemsetD2D8_v2_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemsetD2D8_v2_not_fou */ + {__pyx_k_Function_cuMemsetD32Async_not_fo, sizeof(__pyx_k_Function_cuMemsetD32Async_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemsetD32Async_not_fo */ + {__pyx_k_Function_cuMemsetD32_v2_not_foun, sizeof(__pyx_k_Function_cuMemsetD32_v2_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemsetD32_v2_not_foun */ + {__pyx_k_Function_cuMemsetD8Async_not_fou, sizeof(__pyx_k_Function_cuMemsetD8Async_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemsetD8Async_not_fou */ + {__pyx_k_Function_cuMemsetD8_v2_not_found, sizeof(__pyx_k_Function_cuMemsetD8_v2_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMemsetD8_v2_not_found */ + {__pyx_k_Function_cuMipmappedArrayCreate, sizeof(__pyx_k_Function_cuMipmappedArrayCreate), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMipmappedArrayCreate */ + {__pyx_k_Function_cuMipmappedArrayDestroy, sizeof(__pyx_k_Function_cuMipmappedArrayDestroy), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMipmappedArrayDestroy */ + {__pyx_k_Function_cuMipmappedArrayGetLeve, sizeof(__pyx_k_Function_cuMipmappedArrayGetLeve), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMipmappedArrayGetLeve */ + {__pyx_k_Function_cuMipmappedArrayGetMemo, sizeof(__pyx_k_Function_cuMipmappedArrayGetMemo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMipmappedArrayGetMemo */ + {__pyx_k_Function_cuMipmappedArrayGetSpar, sizeof(__pyx_k_Function_cuMipmappedArrayGetSpar), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMipmappedArrayGetSpar */ + {__pyx_k_Function_cuModuleEnumerateFuncti, sizeof(__pyx_k_Function_cuModuleEnumerateFuncti), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuModuleEnumerateFuncti */ + {__pyx_k_Function_cuModuleGetFunctionCoun, sizeof(__pyx_k_Function_cuModuleGetFunctionCoun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuModuleGetFunctionCoun */ + {__pyx_k_Function_cuModuleGetFunction_not, sizeof(__pyx_k_Function_cuModuleGetFunction_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuModuleGetFunction_not */ + {__pyx_k_Function_cuModuleGetGlobal_v2_no, sizeof(__pyx_k_Function_cuModuleGetGlobal_v2_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuModuleGetGlobal_v2_no */ + {__pyx_k_Function_cuModuleGetLoadingMode, sizeof(__pyx_k_Function_cuModuleGetLoadingMode), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuModuleGetLoadingMode */ + {__pyx_k_Function_cuModuleGetSurfRef_not, sizeof(__pyx_k_Function_cuModuleGetSurfRef_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuModuleGetSurfRef_not */ + {__pyx_k_Function_cuModuleGetTexRef_not_f, sizeof(__pyx_k_Function_cuModuleGetTexRef_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuModuleGetTexRef_not_f */ + {__pyx_k_Function_cuModuleLoadDataEx_not, sizeof(__pyx_k_Function_cuModuleLoadDataEx_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuModuleLoadDataEx_not */ + {__pyx_k_Function_cuModuleLoadData_not_fo, sizeof(__pyx_k_Function_cuModuleLoadData_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuModuleLoadData_not_fo */ + {__pyx_k_Function_cuModuleLoadFatBinary_n, sizeof(__pyx_k_Function_cuModuleLoadFatBinary_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuModuleLoadFatBinary_n */ + {__pyx_k_Function_cuModuleLoad_not_found, sizeof(__pyx_k_Function_cuModuleLoad_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuModuleLoad_not_found */ + {__pyx_k_Function_cuModuleUnload_not_foun, sizeof(__pyx_k_Function_cuModuleUnload_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuModuleUnload_not_foun */ + {__pyx_k_Function_cuMulticastAddDevice_no, sizeof(__pyx_k_Function_cuMulticastAddDevice_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMulticastAddDevice_no */ + {__pyx_k_Function_cuMulticastBindAddr_not, sizeof(__pyx_k_Function_cuMulticastBindAddr_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMulticastBindAddr_not */ + {__pyx_k_Function_cuMulticastBindMem_not, sizeof(__pyx_k_Function_cuMulticastBindMem_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMulticastBindMem_not */ + {__pyx_k_Function_cuMulticastCreate_not_f, sizeof(__pyx_k_Function_cuMulticastCreate_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMulticastCreate_not_f */ + {__pyx_k_Function_cuMulticastGetGranulari, sizeof(__pyx_k_Function_cuMulticastGetGranulari), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMulticastGetGranulari */ + {__pyx_k_Function_cuMulticastUnbind_not_f, sizeof(__pyx_k_Function_cuMulticastUnbind_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuMulticastUnbind_not_f */ + {__pyx_k_Function_cuOccupancyAvailableDyn, sizeof(__pyx_k_Function_cuOccupancyAvailableDyn), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuOccupancyAvailableDyn */ + {__pyx_k_Function_cuOccupancyMaxActiveBlo, sizeof(__pyx_k_Function_cuOccupancyMaxActiveBlo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuOccupancyMaxActiveBlo */ + {__pyx_k_Function_cuOccupancyMaxActiveBlo_2, sizeof(__pyx_k_Function_cuOccupancyMaxActiveBlo_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuOccupancyMaxActiveBlo_2 */ + {__pyx_k_Function_cuOccupancyMaxActiveClu, sizeof(__pyx_k_Function_cuOccupancyMaxActiveClu), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuOccupancyMaxActiveClu */ + {__pyx_k_Function_cuOccupancyMaxPotential, sizeof(__pyx_k_Function_cuOccupancyMaxPotential), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuOccupancyMaxPotential */ + {__pyx_k_Function_cuOccupancyMaxPotential_2, sizeof(__pyx_k_Function_cuOccupancyMaxPotential_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuOccupancyMaxPotential_2 */ + {__pyx_k_Function_cuOccupancyMaxPotential_3, sizeof(__pyx_k_Function_cuOccupancyMaxPotential_3), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuOccupancyMaxPotential_3 */ + {__pyx_k_Function_cuParamSetSize_not_foun, sizeof(__pyx_k_Function_cuParamSetSize_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuParamSetSize_not_foun */ + {__pyx_k_Function_cuParamSetTexRef_not_fo, sizeof(__pyx_k_Function_cuParamSetTexRef_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuParamSetTexRef_not_fo */ + {__pyx_k_Function_cuParamSetf_not_found, sizeof(__pyx_k_Function_cuParamSetf_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuParamSetf_not_found */ + {__pyx_k_Function_cuParamSeti_not_found, sizeof(__pyx_k_Function_cuParamSeti_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuParamSeti_not_found */ + {__pyx_k_Function_cuParamSetv_not_found, sizeof(__pyx_k_Function_cuParamSetv_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuParamSetv_not_found */ + {__pyx_k_Function_cuPointerGetAttribute_n, sizeof(__pyx_k_Function_cuPointerGetAttribute_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuPointerGetAttribute_n */ + {__pyx_k_Function_cuPointerGetAttributes, sizeof(__pyx_k_Function_cuPointerGetAttributes), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuPointerGetAttributes */ + {__pyx_k_Function_cuPointerSetAttribute_n, sizeof(__pyx_k_Function_cuPointerSetAttribute_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuPointerSetAttribute_n */ + {__pyx_k_Function_cuProfilerStart_not_fou, sizeof(__pyx_k_Function_cuProfilerStart_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuProfilerStart_not_fou */ + {__pyx_k_Function_cuProfilerStop_not_foun, sizeof(__pyx_k_Function_cuProfilerStop_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuProfilerStop_not_foun */ + {__pyx_k_Function_cuSignalExternalSemapho, sizeof(__pyx_k_Function_cuSignalExternalSemapho), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuSignalExternalSemapho */ + {__pyx_k_Function_cuStreamAddCallback_not, sizeof(__pyx_k_Function_cuStreamAddCallback_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamAddCallback_not */ + {__pyx_k_Function_cuStreamAttachMemAsync, sizeof(__pyx_k_Function_cuStreamAttachMemAsync), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamAttachMemAsync */ + {__pyx_k_Function_cuStreamBatchMemOp_v2_n, sizeof(__pyx_k_Function_cuStreamBatchMemOp_v2_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamBatchMemOp_v2_n */ + {__pyx_k_Function_cuStreamBeginCaptureToG, sizeof(__pyx_k_Function_cuStreamBeginCaptureToG), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamBeginCaptureToG */ + {__pyx_k_Function_cuStreamBeginCapture_v2, sizeof(__pyx_k_Function_cuStreamBeginCapture_v2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamBeginCapture_v2 */ + {__pyx_k_Function_cuStreamCopyAttributes, sizeof(__pyx_k_Function_cuStreamCopyAttributes), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamCopyAttributes */ + {__pyx_k_Function_cuStreamCreateWithPrior, sizeof(__pyx_k_Function_cuStreamCreateWithPrior), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamCreateWithPrior */ + {__pyx_k_Function_cuStreamCreate_not_foun, sizeof(__pyx_k_Function_cuStreamCreate_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamCreate_not_foun */ + {__pyx_k_Function_cuStreamDestroy_v2_not, sizeof(__pyx_k_Function_cuStreamDestroy_v2_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamDestroy_v2_not */ + {__pyx_k_Function_cuStreamEndCapture_not, sizeof(__pyx_k_Function_cuStreamEndCapture_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamEndCapture_not */ + {__pyx_k_Function_cuStreamGetAttribute_no, sizeof(__pyx_k_Function_cuStreamGetAttribute_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamGetAttribute_no */ + {__pyx_k_Function_cuStreamGetCaptureInfo, sizeof(__pyx_k_Function_cuStreamGetCaptureInfo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamGetCaptureInfo */ + {__pyx_k_Function_cuStreamGetCaptureInfo_2, sizeof(__pyx_k_Function_cuStreamGetCaptureInfo_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamGetCaptureInfo_2 */ + {__pyx_k_Function_cuStreamGetCtx_not_foun, sizeof(__pyx_k_Function_cuStreamGetCtx_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamGetCtx_not_foun */ + {__pyx_k_Function_cuStreamGetCtx_v2_not_f, sizeof(__pyx_k_Function_cuStreamGetCtx_v2_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamGetCtx_v2_not_f */ + {__pyx_k_Function_cuStreamGetDevice_not_f, sizeof(__pyx_k_Function_cuStreamGetDevice_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamGetDevice_not_f */ + {__pyx_k_Function_cuStreamGetFlags_not_fo, sizeof(__pyx_k_Function_cuStreamGetFlags_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamGetFlags_not_fo */ + {__pyx_k_Function_cuStreamGetGreenCtx_not, sizeof(__pyx_k_Function_cuStreamGetGreenCtx_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamGetGreenCtx_not */ + {__pyx_k_Function_cuStreamGetId_not_found, sizeof(__pyx_k_Function_cuStreamGetId_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamGetId_not_found */ + {__pyx_k_Function_cuStreamGetPriority_not, sizeof(__pyx_k_Function_cuStreamGetPriority_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamGetPriority_not */ + {__pyx_k_Function_cuStreamIsCapturing_not, sizeof(__pyx_k_Function_cuStreamIsCapturing_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamIsCapturing_not */ + {__pyx_k_Function_cuStreamQuery_not_found, sizeof(__pyx_k_Function_cuStreamQuery_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamQuery_not_found */ + {__pyx_k_Function_cuStreamSetAttribute_no, sizeof(__pyx_k_Function_cuStreamSetAttribute_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamSetAttribute_no */ + {__pyx_k_Function_cuStreamSynchronize_not, sizeof(__pyx_k_Function_cuStreamSynchronize_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamSynchronize_not */ + {__pyx_k_Function_cuStreamUpdateCaptureDe, sizeof(__pyx_k_Function_cuStreamUpdateCaptureDe), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamUpdateCaptureDe */ + {__pyx_k_Function_cuStreamUpdateCaptureDe_2, sizeof(__pyx_k_Function_cuStreamUpdateCaptureDe_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamUpdateCaptureDe_2 */ + {__pyx_k_Function_cuStreamWaitEvent_not_f, sizeof(__pyx_k_Function_cuStreamWaitEvent_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamWaitEvent_not_f */ + {__pyx_k_Function_cuStreamWaitValue32_v2, sizeof(__pyx_k_Function_cuStreamWaitValue32_v2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamWaitValue32_v2 */ + {__pyx_k_Function_cuStreamWaitValue64_v2, sizeof(__pyx_k_Function_cuStreamWaitValue64_v2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamWaitValue64_v2 */ + {__pyx_k_Function_cuStreamWriteValue32_v2, sizeof(__pyx_k_Function_cuStreamWriteValue32_v2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamWriteValue32_v2 */ + {__pyx_k_Function_cuStreamWriteValue64_v2, sizeof(__pyx_k_Function_cuStreamWriteValue64_v2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuStreamWriteValue64_v2 */ + {__pyx_k_Function_cuSurfObjectCreate_not, sizeof(__pyx_k_Function_cuSurfObjectCreate_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuSurfObjectCreate_not */ + {__pyx_k_Function_cuSurfObjectDestroy_not, sizeof(__pyx_k_Function_cuSurfObjectDestroy_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuSurfObjectDestroy_not */ + {__pyx_k_Function_cuSurfObjectGetResource, sizeof(__pyx_k_Function_cuSurfObjectGetResource), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuSurfObjectGetResource */ + {__pyx_k_Function_cuSurfRefGetArray_not_f, sizeof(__pyx_k_Function_cuSurfRefGetArray_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuSurfRefGetArray_not_f */ + {__pyx_k_Function_cuSurfRefSetArray_not_f, sizeof(__pyx_k_Function_cuSurfRefSetArray_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuSurfRefSetArray_not_f */ + {__pyx_k_Function_cuTensorMapEncodeIm2col, sizeof(__pyx_k_Function_cuTensorMapEncodeIm2col), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTensorMapEncodeIm2col */ + {__pyx_k_Function_cuTensorMapEncodeIm2col_2, sizeof(__pyx_k_Function_cuTensorMapEncodeIm2col_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTensorMapEncodeIm2col_2 */ + {__pyx_k_Function_cuTensorMapEncodeTiled, sizeof(__pyx_k_Function_cuTensorMapEncodeTiled), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTensorMapEncodeTiled */ + {__pyx_k_Function_cuTensorMapReplaceAddre, sizeof(__pyx_k_Function_cuTensorMapReplaceAddre), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTensorMapReplaceAddre */ + {__pyx_k_Function_cuTexObjectCreate_not_f, sizeof(__pyx_k_Function_cuTexObjectCreate_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexObjectCreate_not_f */ + {__pyx_k_Function_cuTexObjectDestroy_not, sizeof(__pyx_k_Function_cuTexObjectDestroy_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexObjectDestroy_not */ + {__pyx_k_Function_cuTexObjectGetResourceD, sizeof(__pyx_k_Function_cuTexObjectGetResourceD), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexObjectGetResourceD */ + {__pyx_k_Function_cuTexObjectGetResourceV, sizeof(__pyx_k_Function_cuTexObjectGetResourceV), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexObjectGetResourceV */ + {__pyx_k_Function_cuTexObjectGetTextureDe, sizeof(__pyx_k_Function_cuTexObjectGetTextureDe), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexObjectGetTextureDe */ + {__pyx_k_Function_cuTexRefCreate_not_foun, sizeof(__pyx_k_Function_cuTexRefCreate_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefCreate_not_foun */ + {__pyx_k_Function_cuTexRefDestroy_not_fou, sizeof(__pyx_k_Function_cuTexRefDestroy_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefDestroy_not_fou */ + {__pyx_k_Function_cuTexRefGetAddressMode, sizeof(__pyx_k_Function_cuTexRefGetAddressMode), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefGetAddressMode */ + {__pyx_k_Function_cuTexRefGetAddress_v2_n, sizeof(__pyx_k_Function_cuTexRefGetAddress_v2_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefGetAddress_v2_n */ + {__pyx_k_Function_cuTexRefGetArray_not_fo, sizeof(__pyx_k_Function_cuTexRefGetArray_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefGetArray_not_fo */ + {__pyx_k_Function_cuTexRefGetBorderColor, sizeof(__pyx_k_Function_cuTexRefGetBorderColor), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefGetBorderColor */ + {__pyx_k_Function_cuTexRefGetFilterMode_n, sizeof(__pyx_k_Function_cuTexRefGetFilterMode_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefGetFilterMode_n */ + {__pyx_k_Function_cuTexRefGetFlags_not_fo, sizeof(__pyx_k_Function_cuTexRefGetFlags_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefGetFlags_not_fo */ + {__pyx_k_Function_cuTexRefGetFormat_not_f, sizeof(__pyx_k_Function_cuTexRefGetFormat_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefGetFormat_not_f */ + {__pyx_k_Function_cuTexRefGetMaxAnisotrop, sizeof(__pyx_k_Function_cuTexRefGetMaxAnisotrop), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefGetMaxAnisotrop */ + {__pyx_k_Function_cuTexRefGetMipmapFilter, sizeof(__pyx_k_Function_cuTexRefGetMipmapFilter), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefGetMipmapFilter */ + {__pyx_k_Function_cuTexRefGetMipmapLevelB, sizeof(__pyx_k_Function_cuTexRefGetMipmapLevelB), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefGetMipmapLevelB */ + {__pyx_k_Function_cuTexRefGetMipmapLevelC, sizeof(__pyx_k_Function_cuTexRefGetMipmapLevelC), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefGetMipmapLevelC */ + {__pyx_k_Function_cuTexRefGetMipmappedArr, sizeof(__pyx_k_Function_cuTexRefGetMipmappedArr), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefGetMipmappedArr */ + {__pyx_k_Function_cuTexRefSetAddress2D_v3, sizeof(__pyx_k_Function_cuTexRefSetAddress2D_v3), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefSetAddress2D_v3 */ + {__pyx_k_Function_cuTexRefSetAddressMode, sizeof(__pyx_k_Function_cuTexRefSetAddressMode), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefSetAddressMode */ + {__pyx_k_Function_cuTexRefSetAddress_v2_n, sizeof(__pyx_k_Function_cuTexRefSetAddress_v2_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefSetAddress_v2_n */ + {__pyx_k_Function_cuTexRefSetArray_not_fo, sizeof(__pyx_k_Function_cuTexRefSetArray_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefSetArray_not_fo */ + {__pyx_k_Function_cuTexRefSetBorderColor, sizeof(__pyx_k_Function_cuTexRefSetBorderColor), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefSetBorderColor */ + {__pyx_k_Function_cuTexRefSetFilterMode_n, sizeof(__pyx_k_Function_cuTexRefSetFilterMode_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefSetFilterMode_n */ + {__pyx_k_Function_cuTexRefSetFlags_not_fo, sizeof(__pyx_k_Function_cuTexRefSetFlags_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefSetFlags_not_fo */ + {__pyx_k_Function_cuTexRefSetFormat_not_f, sizeof(__pyx_k_Function_cuTexRefSetFormat_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefSetFormat_not_f */ + {__pyx_k_Function_cuTexRefSetMaxAnisotrop, sizeof(__pyx_k_Function_cuTexRefSetMaxAnisotrop), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefSetMaxAnisotrop */ + {__pyx_k_Function_cuTexRefSetMipmapFilter, sizeof(__pyx_k_Function_cuTexRefSetMipmapFilter), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefSetMipmapFilter */ + {__pyx_k_Function_cuTexRefSetMipmapLevelB, sizeof(__pyx_k_Function_cuTexRefSetMipmapLevelB), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefSetMipmapLevelB */ + {__pyx_k_Function_cuTexRefSetMipmapLevelC, sizeof(__pyx_k_Function_cuTexRefSetMipmapLevelC), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefSetMipmapLevelC */ + {__pyx_k_Function_cuTexRefSetMipmappedArr, sizeof(__pyx_k_Function_cuTexRefSetMipmappedArr), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuTexRefSetMipmappedArr */ + {__pyx_k_Function_cuThreadExchangeStreamC, sizeof(__pyx_k_Function_cuThreadExchangeStreamC), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuThreadExchangeStreamC */ + {__pyx_k_Function_cuUserObjectCreate_not, sizeof(__pyx_k_Function_cuUserObjectCreate_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuUserObjectCreate_not */ + {__pyx_k_Function_cuUserObjectRelease_not, sizeof(__pyx_k_Function_cuUserObjectRelease_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuUserObjectRelease_not */ + {__pyx_k_Function_cuUserObjectRetain_not, sizeof(__pyx_k_Function_cuUserObjectRetain_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuUserObjectRetain_not */ + {__pyx_k_Function_cuVDPAUCtxCreate_v2_not, sizeof(__pyx_k_Function_cuVDPAUCtxCreate_v2_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuVDPAUCtxCreate_v2_not */ + {__pyx_k_Function_cuVDPAUGetDevice_not_fo, sizeof(__pyx_k_Function_cuVDPAUGetDevice_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuVDPAUGetDevice_not_fo */ + {__pyx_k_Function_cuWaitExternalSemaphore, sizeof(__pyx_k_Function_cuWaitExternalSemaphore), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_cuWaitExternalSemaphore */ + {__pyx_k_Lock, sizeof(__pyx_k_Lock), 0, 1, 1}, /* PyObject cname: __pyx_n_u_Lock */ + {__pyx_k_Note_that_Cython_is_deliberately, sizeof(__pyx_k_Note_that_Cython_is_deliberately), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Note_that_Cython_is_deliberately */ + {__pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_RuntimeError */ + {__pyx_k_add_note, sizeof(__pyx_k_add_note), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_add_note */ + {__pyx_k_asyncio_coroutines, sizeof(__pyx_k_asyncio_coroutines), 0, 1, 1}, /* PyObject cname: __pyx_n_u_asyncio_coroutines */ + {__pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cline_in_traceback */ + {__pyx_k_cuArray3DCreate_v2, sizeof(__pyx_k_cuArray3DCreate_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuArray3DCreate_v2 */ + {__pyx_k_cuArray3DGetDescriptor_v2, sizeof(__pyx_k_cuArray3DGetDescriptor_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuArray3DGetDescriptor_v2 */ + {__pyx_k_cuArrayCreate_v2, sizeof(__pyx_k_cuArrayCreate_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuArrayCreate_v2 */ + {__pyx_k_cuArrayDestroy, sizeof(__pyx_k_cuArrayDestroy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuArrayDestroy */ + {__pyx_k_cuArrayGetDescriptor_v2, sizeof(__pyx_k_cuArrayGetDescriptor_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuArrayGetDescriptor_v2 */ + {__pyx_k_cuArrayGetMemoryRequirements, sizeof(__pyx_k_cuArrayGetMemoryRequirements), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuArrayGetMemoryRequirements */ + {__pyx_k_cuArrayGetPlane, sizeof(__pyx_k_cuArrayGetPlane), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuArrayGetPlane */ + {__pyx_k_cuArrayGetSparseProperties, sizeof(__pyx_k_cuArrayGetSparseProperties), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuArrayGetSparseProperties */ + {__pyx_k_cuCheckpointProcessCheckpoint, sizeof(__pyx_k_cuCheckpointProcessCheckpoint), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCheckpointProcessCheckpoint */ + {__pyx_k_cuCheckpointProcessGetRestoreT, sizeof(__pyx_k_cuCheckpointProcessGetRestoreT), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCheckpointProcessGetRestoreT */ + {__pyx_k_cuCheckpointProcessGetState, sizeof(__pyx_k_cuCheckpointProcessGetState), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCheckpointProcessGetState */ + {__pyx_k_cuCheckpointProcessLock, sizeof(__pyx_k_cuCheckpointProcessLock), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCheckpointProcessLock */ + {__pyx_k_cuCheckpointProcessRestore, sizeof(__pyx_k_cuCheckpointProcessRestore), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCheckpointProcessRestore */ + {__pyx_k_cuCheckpointProcessUnlock, sizeof(__pyx_k_cuCheckpointProcessUnlock), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCheckpointProcessUnlock */ + {__pyx_k_cuCoredumpGetAttribute, sizeof(__pyx_k_cuCoredumpGetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCoredumpGetAttribute */ + {__pyx_k_cuCoredumpGetAttributeGlobal, sizeof(__pyx_k_cuCoredumpGetAttributeGlobal), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCoredumpGetAttributeGlobal */ + {__pyx_k_cuCoredumpSetAttribute, sizeof(__pyx_k_cuCoredumpSetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCoredumpSetAttribute */ + {__pyx_k_cuCoredumpSetAttributeGlobal, sizeof(__pyx_k_cuCoredumpSetAttributeGlobal), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCoredumpSetAttributeGlobal */ + {__pyx_k_cuCtxAttach, sizeof(__pyx_k_cuCtxAttach), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxAttach */ + {__pyx_k_cuCtxCreate_v2, sizeof(__pyx_k_cuCtxCreate_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxCreate_v2 */ + {__pyx_k_cuCtxCreate_v3, sizeof(__pyx_k_cuCtxCreate_v3), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxCreate_v3 */ + {__pyx_k_cuCtxCreate_v4, sizeof(__pyx_k_cuCtxCreate_v4), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxCreate_v4 */ + {__pyx_k_cuCtxDestroy_v2, sizeof(__pyx_k_cuCtxDestroy_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxDestroy_v2 */ + {__pyx_k_cuCtxDetach, sizeof(__pyx_k_cuCtxDetach), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxDetach */ + {__pyx_k_cuCtxDisablePeerAccess, sizeof(__pyx_k_cuCtxDisablePeerAccess), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxDisablePeerAccess */ + {__pyx_k_cuCtxEnablePeerAccess, sizeof(__pyx_k_cuCtxEnablePeerAccess), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxEnablePeerAccess */ + {__pyx_k_cuCtxFromGreenCtx, sizeof(__pyx_k_cuCtxFromGreenCtx), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxFromGreenCtx */ + {__pyx_k_cuCtxGetApiVersion, sizeof(__pyx_k_cuCtxGetApiVersion), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxGetApiVersion */ + {__pyx_k_cuCtxGetCacheConfig, sizeof(__pyx_k_cuCtxGetCacheConfig), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxGetCacheConfig */ + {__pyx_k_cuCtxGetCurrent, sizeof(__pyx_k_cuCtxGetCurrent), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxGetCurrent */ + {__pyx_k_cuCtxGetDevResource, sizeof(__pyx_k_cuCtxGetDevResource), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxGetDevResource */ + {__pyx_k_cuCtxGetDevice, sizeof(__pyx_k_cuCtxGetDevice), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxGetDevice */ + {__pyx_k_cuCtxGetExecAffinity, sizeof(__pyx_k_cuCtxGetExecAffinity), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxGetExecAffinity */ + {__pyx_k_cuCtxGetFlags, sizeof(__pyx_k_cuCtxGetFlags), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxGetFlags */ + {__pyx_k_cuCtxGetId, sizeof(__pyx_k_cuCtxGetId), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxGetId */ + {__pyx_k_cuCtxGetLimit, sizeof(__pyx_k_cuCtxGetLimit), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxGetLimit */ + {__pyx_k_cuCtxGetSharedMemConfig, sizeof(__pyx_k_cuCtxGetSharedMemConfig), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxGetSharedMemConfig */ + {__pyx_k_cuCtxGetStreamPriorityRange, sizeof(__pyx_k_cuCtxGetStreamPriorityRange), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxGetStreamPriorityRange */ + {__pyx_k_cuCtxPopCurrent_v2, sizeof(__pyx_k_cuCtxPopCurrent_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxPopCurrent_v2 */ + {__pyx_k_cuCtxPushCurrent_v2, sizeof(__pyx_k_cuCtxPushCurrent_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxPushCurrent_v2 */ + {__pyx_k_cuCtxRecordEvent, sizeof(__pyx_k_cuCtxRecordEvent), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxRecordEvent */ + {__pyx_k_cuCtxResetPersistingL2Cache, sizeof(__pyx_k_cuCtxResetPersistingL2Cache), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxResetPersistingL2Cache */ + {__pyx_k_cuCtxSetCacheConfig, sizeof(__pyx_k_cuCtxSetCacheConfig), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxSetCacheConfig */ + {__pyx_k_cuCtxSetCurrent, sizeof(__pyx_k_cuCtxSetCurrent), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxSetCurrent */ + {__pyx_k_cuCtxSetFlags, sizeof(__pyx_k_cuCtxSetFlags), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxSetFlags */ + {__pyx_k_cuCtxSetLimit, sizeof(__pyx_k_cuCtxSetLimit), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxSetLimit */ + {__pyx_k_cuCtxSetSharedMemConfig, sizeof(__pyx_k_cuCtxSetSharedMemConfig), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxSetSharedMemConfig */ + {__pyx_k_cuCtxSynchronize, sizeof(__pyx_k_cuCtxSynchronize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxSynchronize */ + {__pyx_k_cuCtxWaitEvent, sizeof(__pyx_k_cuCtxWaitEvent), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuCtxWaitEvent */ + {__pyx_k_cuDestroyExternalMemory, sizeof(__pyx_k_cuDestroyExternalMemory), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDestroyExternalMemory */ + {__pyx_k_cuDestroyExternalSemaphore, sizeof(__pyx_k_cuDestroyExternalSemaphore), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDestroyExternalSemaphore */ + {__pyx_k_cuDevResourceGenerateDesc, sizeof(__pyx_k_cuDevResourceGenerateDesc), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDevResourceGenerateDesc */ + {__pyx_k_cuDevSmResourceSplitByCount, sizeof(__pyx_k_cuDevSmResourceSplitByCount), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDevSmResourceSplitByCount */ + {__pyx_k_cuDeviceCanAccessPeer, sizeof(__pyx_k_cuDeviceCanAccessPeer), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceCanAccessPeer */ + {__pyx_k_cuDeviceComputeCapability, sizeof(__pyx_k_cuDeviceComputeCapability), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceComputeCapability */ + {__pyx_k_cuDeviceGet, sizeof(__pyx_k_cuDeviceGet), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGet */ + {__pyx_k_cuDeviceGetAttribute, sizeof(__pyx_k_cuDeviceGetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetAttribute */ + {__pyx_k_cuDeviceGetByPCIBusId, sizeof(__pyx_k_cuDeviceGetByPCIBusId), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetByPCIBusId */ + {__pyx_k_cuDeviceGetCount, sizeof(__pyx_k_cuDeviceGetCount), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetCount */ + {__pyx_k_cuDeviceGetDefaultMemPool, sizeof(__pyx_k_cuDeviceGetDefaultMemPool), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetDefaultMemPool */ + {__pyx_k_cuDeviceGetDevResource, sizeof(__pyx_k_cuDeviceGetDevResource), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetDevResource */ + {__pyx_k_cuDeviceGetExecAffinitySupport, sizeof(__pyx_k_cuDeviceGetExecAffinitySupport), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetExecAffinitySupport */ + {__pyx_k_cuDeviceGetGraphMemAttribute, sizeof(__pyx_k_cuDeviceGetGraphMemAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetGraphMemAttribute */ + {__pyx_k_cuDeviceGetLuid, sizeof(__pyx_k_cuDeviceGetLuid), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetLuid */ + {__pyx_k_cuDeviceGetMemPool, sizeof(__pyx_k_cuDeviceGetMemPool), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetMemPool */ + {__pyx_k_cuDeviceGetName, sizeof(__pyx_k_cuDeviceGetName), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetName */ + {__pyx_k_cuDeviceGetNvSciSyncAttributes, sizeof(__pyx_k_cuDeviceGetNvSciSyncAttributes), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetNvSciSyncAttributes */ + {__pyx_k_cuDeviceGetP2PAttribute, sizeof(__pyx_k_cuDeviceGetP2PAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetP2PAttribute */ + {__pyx_k_cuDeviceGetPCIBusId, sizeof(__pyx_k_cuDeviceGetPCIBusId), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetPCIBusId */ + {__pyx_k_cuDeviceGetProperties, sizeof(__pyx_k_cuDeviceGetProperties), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetProperties */ + {__pyx_k_cuDeviceGetTexture1DLinearMaxW, sizeof(__pyx_k_cuDeviceGetTexture1DLinearMaxW), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetTexture1DLinearMaxW */ + {__pyx_k_cuDeviceGetUuid, sizeof(__pyx_k_cuDeviceGetUuid), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetUuid */ + {__pyx_k_cuDeviceGetUuid_v2, sizeof(__pyx_k_cuDeviceGetUuid_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGetUuid_v2 */ + {__pyx_k_cuDeviceGraphMemTrim, sizeof(__pyx_k_cuDeviceGraphMemTrim), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceGraphMemTrim */ + {__pyx_k_cuDevicePrimaryCtxGetState, sizeof(__pyx_k_cuDevicePrimaryCtxGetState), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDevicePrimaryCtxGetState */ + {__pyx_k_cuDevicePrimaryCtxRelease_v2, sizeof(__pyx_k_cuDevicePrimaryCtxRelease_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDevicePrimaryCtxRelease_v2 */ + {__pyx_k_cuDevicePrimaryCtxReset_v2, sizeof(__pyx_k_cuDevicePrimaryCtxReset_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDevicePrimaryCtxReset_v2 */ + {__pyx_k_cuDevicePrimaryCtxRetain, sizeof(__pyx_k_cuDevicePrimaryCtxRetain), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDevicePrimaryCtxRetain */ + {__pyx_k_cuDevicePrimaryCtxSetFlags_v2, sizeof(__pyx_k_cuDevicePrimaryCtxSetFlags_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDevicePrimaryCtxSetFlags_v2 */ + {__pyx_k_cuDeviceRegisterAsyncNotificat, sizeof(__pyx_k_cuDeviceRegisterAsyncNotificat), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceRegisterAsyncNotificat */ + {__pyx_k_cuDeviceSetGraphMemAttribute, sizeof(__pyx_k_cuDeviceSetGraphMemAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceSetGraphMemAttribute */ + {__pyx_k_cuDeviceSetMemPool, sizeof(__pyx_k_cuDeviceSetMemPool), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceSetMemPool */ + {__pyx_k_cuDeviceTotalMem_v2, sizeof(__pyx_k_cuDeviceTotalMem_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceTotalMem_v2 */ + {__pyx_k_cuDeviceUnregisterAsyncNotific, sizeof(__pyx_k_cuDeviceUnregisterAsyncNotific), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDeviceUnregisterAsyncNotific */ + {__pyx_k_cuDriverGetVersion, sizeof(__pyx_k_cuDriverGetVersion), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuDriverGetVersion */ + {__pyx_k_cuEGLStreamConsumerAcquireFram, sizeof(__pyx_k_cuEGLStreamConsumerAcquireFram), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEGLStreamConsumerAcquireFram */ + {__pyx_k_cuEGLStreamConsumerConnect, sizeof(__pyx_k_cuEGLStreamConsumerConnect), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEGLStreamConsumerConnect */ + {__pyx_k_cuEGLStreamConsumerConnectWith, sizeof(__pyx_k_cuEGLStreamConsumerConnectWith), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEGLStreamConsumerConnectWith */ + {__pyx_k_cuEGLStreamConsumerDisconnect, sizeof(__pyx_k_cuEGLStreamConsumerDisconnect), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEGLStreamConsumerDisconnect */ + {__pyx_k_cuEGLStreamConsumerReleaseFram, sizeof(__pyx_k_cuEGLStreamConsumerReleaseFram), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEGLStreamConsumerReleaseFram */ + {__pyx_k_cuEGLStreamProducerConnect, sizeof(__pyx_k_cuEGLStreamProducerConnect), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEGLStreamProducerConnect */ + {__pyx_k_cuEGLStreamProducerDisconnect, sizeof(__pyx_k_cuEGLStreamProducerDisconnect), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEGLStreamProducerDisconnect */ + {__pyx_k_cuEGLStreamProducerPresentFram, sizeof(__pyx_k_cuEGLStreamProducerPresentFram), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEGLStreamProducerPresentFram */ + {__pyx_k_cuEGLStreamProducerReturnFrame, sizeof(__pyx_k_cuEGLStreamProducerReturnFrame), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEGLStreamProducerReturnFrame */ + {__pyx_k_cuEventCreate, sizeof(__pyx_k_cuEventCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEventCreate */ + {__pyx_k_cuEventCreateFromEGLSync, sizeof(__pyx_k_cuEventCreateFromEGLSync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEventCreateFromEGLSync */ + {__pyx_k_cuEventDestroy_v2, sizeof(__pyx_k_cuEventDestroy_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEventDestroy_v2 */ + {__pyx_k_cuEventElapsedTime, sizeof(__pyx_k_cuEventElapsedTime), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEventElapsedTime */ + {__pyx_k_cuEventElapsedTime_v2, sizeof(__pyx_k_cuEventElapsedTime_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEventElapsedTime_v2 */ + {__pyx_k_cuEventQuery, sizeof(__pyx_k_cuEventQuery), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEventQuery */ + {__pyx_k_cuEventRecord, sizeof(__pyx_k_cuEventRecord), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEventRecord */ + {__pyx_k_cuEventRecordWithFlags, sizeof(__pyx_k_cuEventRecordWithFlags), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEventRecordWithFlags */ + {__pyx_k_cuEventSynchronize, sizeof(__pyx_k_cuEventSynchronize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuEventSynchronize */ + {__pyx_k_cuExternalMemoryGetMappedBuffe, sizeof(__pyx_k_cuExternalMemoryGetMappedBuffe), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuExternalMemoryGetMappedBuffe */ + {__pyx_k_cuExternalMemoryGetMappedMipma, sizeof(__pyx_k_cuExternalMemoryGetMappedMipma), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuExternalMemoryGetMappedMipma */ + {__pyx_k_cuFlushGPUDirectRDMAWrites, sizeof(__pyx_k_cuFlushGPUDirectRDMAWrites), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFlushGPUDirectRDMAWrites */ + {__pyx_k_cuFuncGetAttribute, sizeof(__pyx_k_cuFuncGetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFuncGetAttribute */ + {__pyx_k_cuFuncGetModule, sizeof(__pyx_k_cuFuncGetModule), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFuncGetModule */ + {__pyx_k_cuFuncGetName, sizeof(__pyx_k_cuFuncGetName), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFuncGetName */ + {__pyx_k_cuFuncGetParamInfo, sizeof(__pyx_k_cuFuncGetParamInfo), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFuncGetParamInfo */ + {__pyx_k_cuFuncIsLoaded, sizeof(__pyx_k_cuFuncIsLoaded), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFuncIsLoaded */ + {__pyx_k_cuFuncLoad, sizeof(__pyx_k_cuFuncLoad), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFuncLoad */ + {__pyx_k_cuFuncSetAttribute, sizeof(__pyx_k_cuFuncSetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFuncSetAttribute */ + {__pyx_k_cuFuncSetBlockShape, sizeof(__pyx_k_cuFuncSetBlockShape), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFuncSetBlockShape */ + {__pyx_k_cuFuncSetCacheConfig, sizeof(__pyx_k_cuFuncSetCacheConfig), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFuncSetCacheConfig */ + {__pyx_k_cuFuncSetSharedMemConfig, sizeof(__pyx_k_cuFuncSetSharedMemConfig), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFuncSetSharedMemConfig */ + {__pyx_k_cuFuncSetSharedSize, sizeof(__pyx_k_cuFuncSetSharedSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFuncSetSharedSize */ + {__pyx_k_cuGLGetDevices_v2, sizeof(__pyx_k_cuGLGetDevices_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGLGetDevices_v2 */ + {__pyx_k_cuGetErrorName, sizeof(__pyx_k_cuGetErrorName), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGetErrorName */ + {__pyx_k_cuGetErrorString, sizeof(__pyx_k_cuGetErrorString), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGetErrorString */ + {__pyx_k_cuGetExportTable, sizeof(__pyx_k_cuGetExportTable), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGetExportTable */ + {__pyx_k_cuGetProcAddress_v2, sizeof(__pyx_k_cuGetProcAddress_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGetProcAddress_v2 */ + {__pyx_k_cuGraphAddBatchMemOpNode, sizeof(__pyx_k_cuGraphAddBatchMemOpNode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddBatchMemOpNode */ + {__pyx_k_cuGraphAddChildGraphNode, sizeof(__pyx_k_cuGraphAddChildGraphNode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddChildGraphNode */ + {__pyx_k_cuGraphAddDependencies, sizeof(__pyx_k_cuGraphAddDependencies), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddDependencies */ + {__pyx_k_cuGraphAddDependencies_v2, sizeof(__pyx_k_cuGraphAddDependencies_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddDependencies_v2 */ + {__pyx_k_cuGraphAddEmptyNode, sizeof(__pyx_k_cuGraphAddEmptyNode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddEmptyNode */ + {__pyx_k_cuGraphAddEventRecordNode, sizeof(__pyx_k_cuGraphAddEventRecordNode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddEventRecordNode */ + {__pyx_k_cuGraphAddEventWaitNode, sizeof(__pyx_k_cuGraphAddEventWaitNode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddEventWaitNode */ + {__pyx_k_cuGraphAddExternalSemaphoresSi, sizeof(__pyx_k_cuGraphAddExternalSemaphoresSi), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddExternalSemaphoresSi */ + {__pyx_k_cuGraphAddExternalSemaphoresWa, sizeof(__pyx_k_cuGraphAddExternalSemaphoresWa), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddExternalSemaphoresWa */ + {__pyx_k_cuGraphAddHostNode, sizeof(__pyx_k_cuGraphAddHostNode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddHostNode */ + {__pyx_k_cuGraphAddKernelNode_v2, sizeof(__pyx_k_cuGraphAddKernelNode_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddKernelNode_v2 */ + {__pyx_k_cuGraphAddMemAllocNode, sizeof(__pyx_k_cuGraphAddMemAllocNode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddMemAllocNode */ + {__pyx_k_cuGraphAddMemFreeNode, sizeof(__pyx_k_cuGraphAddMemFreeNode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddMemFreeNode */ + {__pyx_k_cuGraphAddMemcpyNode, sizeof(__pyx_k_cuGraphAddMemcpyNode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddMemcpyNode */ + {__pyx_k_cuGraphAddMemsetNode, sizeof(__pyx_k_cuGraphAddMemsetNode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddMemsetNode */ + {__pyx_k_cuGraphAddNode, sizeof(__pyx_k_cuGraphAddNode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddNode */ + {__pyx_k_cuGraphAddNode_v2, sizeof(__pyx_k_cuGraphAddNode_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphAddNode_v2 */ + {__pyx_k_cuGraphBatchMemOpNodeGetParams, sizeof(__pyx_k_cuGraphBatchMemOpNodeGetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphBatchMemOpNodeGetParams */ + {__pyx_k_cuGraphBatchMemOpNodeSetParams, sizeof(__pyx_k_cuGraphBatchMemOpNodeSetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphBatchMemOpNodeSetParams */ + {__pyx_k_cuGraphChildGraphNodeGetGraph, sizeof(__pyx_k_cuGraphChildGraphNodeGetGraph), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphChildGraphNodeGetGraph */ + {__pyx_k_cuGraphClone, sizeof(__pyx_k_cuGraphClone), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphClone */ + {__pyx_k_cuGraphConditionalHandleCreate, sizeof(__pyx_k_cuGraphConditionalHandleCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphConditionalHandleCreate */ + {__pyx_k_cuGraphCreate, sizeof(__pyx_k_cuGraphCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphCreate */ + {__pyx_k_cuGraphDebugDotPrint, sizeof(__pyx_k_cuGraphDebugDotPrint), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphDebugDotPrint */ + {__pyx_k_cuGraphDestroy, sizeof(__pyx_k_cuGraphDestroy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphDestroy */ + {__pyx_k_cuGraphDestroyNode, sizeof(__pyx_k_cuGraphDestroyNode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphDestroyNode */ + {__pyx_k_cuGraphEventRecordNodeGetEvent, sizeof(__pyx_k_cuGraphEventRecordNodeGetEvent), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphEventRecordNodeGetEvent */ + {__pyx_k_cuGraphEventRecordNodeSetEvent, sizeof(__pyx_k_cuGraphEventRecordNodeSetEvent), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphEventRecordNodeSetEvent */ + {__pyx_k_cuGraphEventWaitNodeGetEvent, sizeof(__pyx_k_cuGraphEventWaitNodeGetEvent), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphEventWaitNodeGetEvent */ + {__pyx_k_cuGraphEventWaitNodeSetEvent, sizeof(__pyx_k_cuGraphEventWaitNodeSetEvent), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphEventWaitNodeSetEvent */ + {__pyx_k_cuGraphExecBatchMemOpNodeSetPa, sizeof(__pyx_k_cuGraphExecBatchMemOpNodeSetPa), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecBatchMemOpNodeSetPa */ + {__pyx_k_cuGraphExecChildGraphNodeSetPa, sizeof(__pyx_k_cuGraphExecChildGraphNodeSetPa), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecChildGraphNodeSetPa */ + {__pyx_k_cuGraphExecDestroy, sizeof(__pyx_k_cuGraphExecDestroy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecDestroy */ + {__pyx_k_cuGraphExecEventRecordNodeSetE, sizeof(__pyx_k_cuGraphExecEventRecordNodeSetE), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecEventRecordNodeSetE */ + {__pyx_k_cuGraphExecEventWaitNodeSetEve, sizeof(__pyx_k_cuGraphExecEventWaitNodeSetEve), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecEventWaitNodeSetEve */ + {__pyx_k_cuGraphExecExternalSemaphoresS, sizeof(__pyx_k_cuGraphExecExternalSemaphoresS), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecExternalSemaphoresS */ + {__pyx_k_cuGraphExecExternalSemaphoresW, sizeof(__pyx_k_cuGraphExecExternalSemaphoresW), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecExternalSemaphoresW */ + {__pyx_k_cuGraphExecGetFlags, sizeof(__pyx_k_cuGraphExecGetFlags), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecGetFlags */ + {__pyx_k_cuGraphExecHostNodeSetParams, sizeof(__pyx_k_cuGraphExecHostNodeSetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecHostNodeSetParams */ + {__pyx_k_cuGraphExecKernelNodeSetParams, sizeof(__pyx_k_cuGraphExecKernelNodeSetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecKernelNodeSetParams */ + {__pyx_k_cuGraphExecMemcpyNodeSetParams, sizeof(__pyx_k_cuGraphExecMemcpyNodeSetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecMemcpyNodeSetParams */ + {__pyx_k_cuGraphExecMemsetNodeSetParams, sizeof(__pyx_k_cuGraphExecMemsetNodeSetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecMemsetNodeSetParams */ + {__pyx_k_cuGraphExecNodeSetParams, sizeof(__pyx_k_cuGraphExecNodeSetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecNodeSetParams */ + {__pyx_k_cuGraphExecUpdate_v2, sizeof(__pyx_k_cuGraphExecUpdate_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExecUpdate_v2 */ + {__pyx_k_cuGraphExternalSemaphoresSigna, sizeof(__pyx_k_cuGraphExternalSemaphoresSigna), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExternalSemaphoresSigna */ + {__pyx_k_cuGraphExternalSemaphoresSigna_2, sizeof(__pyx_k_cuGraphExternalSemaphoresSigna_2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExternalSemaphoresSigna_2 */ + {__pyx_k_cuGraphExternalSemaphoresWaitN, sizeof(__pyx_k_cuGraphExternalSemaphoresWaitN), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExternalSemaphoresWaitN */ + {__pyx_k_cuGraphExternalSemaphoresWaitN_2, sizeof(__pyx_k_cuGraphExternalSemaphoresWaitN_2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphExternalSemaphoresWaitN_2 */ + {__pyx_k_cuGraphGetEdges, sizeof(__pyx_k_cuGraphGetEdges), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphGetEdges */ + {__pyx_k_cuGraphGetEdges_v2, sizeof(__pyx_k_cuGraphGetEdges_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphGetEdges_v2 */ + {__pyx_k_cuGraphGetNodes, sizeof(__pyx_k_cuGraphGetNodes), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphGetNodes */ + {__pyx_k_cuGraphGetRootNodes, sizeof(__pyx_k_cuGraphGetRootNodes), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphGetRootNodes */ + {__pyx_k_cuGraphHostNodeGetParams, sizeof(__pyx_k_cuGraphHostNodeGetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphHostNodeGetParams */ + {__pyx_k_cuGraphHostNodeSetParams, sizeof(__pyx_k_cuGraphHostNodeSetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphHostNodeSetParams */ + {__pyx_k_cuGraphInstantiateWithFlags, sizeof(__pyx_k_cuGraphInstantiateWithFlags), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphInstantiateWithFlags */ + {__pyx_k_cuGraphInstantiateWithParams, sizeof(__pyx_k_cuGraphInstantiateWithParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphInstantiateWithParams */ + {__pyx_k_cuGraphKernelNodeCopyAttribute, sizeof(__pyx_k_cuGraphKernelNodeCopyAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphKernelNodeCopyAttribute */ + {__pyx_k_cuGraphKernelNodeGetAttribute, sizeof(__pyx_k_cuGraphKernelNodeGetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphKernelNodeGetAttribute */ + {__pyx_k_cuGraphKernelNodeGetParams_v2, sizeof(__pyx_k_cuGraphKernelNodeGetParams_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphKernelNodeGetParams_v2 */ + {__pyx_k_cuGraphKernelNodeSetAttribute, sizeof(__pyx_k_cuGraphKernelNodeSetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphKernelNodeSetAttribute */ + {__pyx_k_cuGraphKernelNodeSetParams_v2, sizeof(__pyx_k_cuGraphKernelNodeSetParams_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphKernelNodeSetParams_v2 */ + {__pyx_k_cuGraphLaunch, sizeof(__pyx_k_cuGraphLaunch), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphLaunch */ + {__pyx_k_cuGraphMemAllocNodeGetParams, sizeof(__pyx_k_cuGraphMemAllocNodeGetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphMemAllocNodeGetParams */ + {__pyx_k_cuGraphMemFreeNodeGetParams, sizeof(__pyx_k_cuGraphMemFreeNodeGetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphMemFreeNodeGetParams */ + {__pyx_k_cuGraphMemcpyNodeGetParams, sizeof(__pyx_k_cuGraphMemcpyNodeGetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphMemcpyNodeGetParams */ + {__pyx_k_cuGraphMemcpyNodeSetParams, sizeof(__pyx_k_cuGraphMemcpyNodeSetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphMemcpyNodeSetParams */ + {__pyx_k_cuGraphMemsetNodeGetParams, sizeof(__pyx_k_cuGraphMemsetNodeGetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphMemsetNodeGetParams */ + {__pyx_k_cuGraphMemsetNodeSetParams, sizeof(__pyx_k_cuGraphMemsetNodeSetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphMemsetNodeSetParams */ + {__pyx_k_cuGraphNodeFindInClone, sizeof(__pyx_k_cuGraphNodeFindInClone), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphNodeFindInClone */ + {__pyx_k_cuGraphNodeGetDependencies, sizeof(__pyx_k_cuGraphNodeGetDependencies), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphNodeGetDependencies */ + {__pyx_k_cuGraphNodeGetDependencies_v2, sizeof(__pyx_k_cuGraphNodeGetDependencies_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphNodeGetDependencies_v2 */ + {__pyx_k_cuGraphNodeGetDependentNodes, sizeof(__pyx_k_cuGraphNodeGetDependentNodes), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphNodeGetDependentNodes */ + {__pyx_k_cuGraphNodeGetDependentNodes_v, sizeof(__pyx_k_cuGraphNodeGetDependentNodes_v), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphNodeGetDependentNodes_v */ + {__pyx_k_cuGraphNodeGetEnabled, sizeof(__pyx_k_cuGraphNodeGetEnabled), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphNodeGetEnabled */ + {__pyx_k_cuGraphNodeGetType, sizeof(__pyx_k_cuGraphNodeGetType), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphNodeGetType */ + {__pyx_k_cuGraphNodeSetEnabled, sizeof(__pyx_k_cuGraphNodeSetEnabled), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphNodeSetEnabled */ + {__pyx_k_cuGraphNodeSetParams, sizeof(__pyx_k_cuGraphNodeSetParams), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphNodeSetParams */ + {__pyx_k_cuGraphReleaseUserObject, sizeof(__pyx_k_cuGraphReleaseUserObject), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphReleaseUserObject */ + {__pyx_k_cuGraphRemoveDependencies, sizeof(__pyx_k_cuGraphRemoveDependencies), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphRemoveDependencies */ + {__pyx_k_cuGraphRemoveDependencies_v2, sizeof(__pyx_k_cuGraphRemoveDependencies_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphRemoveDependencies_v2 */ + {__pyx_k_cuGraphRetainUserObject, sizeof(__pyx_k_cuGraphRetainUserObject), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphRetainUserObject */ + {__pyx_k_cuGraphUpload, sizeof(__pyx_k_cuGraphUpload), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphUpload */ + {__pyx_k_cuGraphicsEGLRegisterImage, sizeof(__pyx_k_cuGraphicsEGLRegisterImage), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphicsEGLRegisterImage */ + {__pyx_k_cuGraphicsGLRegisterBuffer, sizeof(__pyx_k_cuGraphicsGLRegisterBuffer), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphicsGLRegisterBuffer */ + {__pyx_k_cuGraphicsGLRegisterImage, sizeof(__pyx_k_cuGraphicsGLRegisterImage), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphicsGLRegisterImage */ + {__pyx_k_cuGraphicsMapResources, sizeof(__pyx_k_cuGraphicsMapResources), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphicsMapResources */ + {__pyx_k_cuGraphicsResourceGetMappedEgl, sizeof(__pyx_k_cuGraphicsResourceGetMappedEgl), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphicsResourceGetMappedEgl */ + {__pyx_k_cuGraphicsResourceGetMappedMip, sizeof(__pyx_k_cuGraphicsResourceGetMappedMip), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphicsResourceGetMappedMip */ + {__pyx_k_cuGraphicsResourceGetMappedPoi, sizeof(__pyx_k_cuGraphicsResourceGetMappedPoi), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphicsResourceGetMappedPoi */ + {__pyx_k_cuGraphicsResourceSetMapFlags, sizeof(__pyx_k_cuGraphicsResourceSetMapFlags), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphicsResourceSetMapFlags */ + {__pyx_k_cuGraphicsSubResourceGetMapped, sizeof(__pyx_k_cuGraphicsSubResourceGetMapped), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphicsSubResourceGetMapped */ + {__pyx_k_cuGraphicsUnmapResources, sizeof(__pyx_k_cuGraphicsUnmapResources), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphicsUnmapResources */ + {__pyx_k_cuGraphicsUnregisterResource, sizeof(__pyx_k_cuGraphicsUnregisterResource), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphicsUnregisterResource */ + {__pyx_k_cuGraphicsVDPAURegisterOutputS, sizeof(__pyx_k_cuGraphicsVDPAURegisterOutputS), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphicsVDPAURegisterOutputS */ + {__pyx_k_cuGraphicsVDPAURegisterVideoSu, sizeof(__pyx_k_cuGraphicsVDPAURegisterVideoSu), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGraphicsVDPAURegisterVideoSu */ + {__pyx_k_cuGreenCtxCreate, sizeof(__pyx_k_cuGreenCtxCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGreenCtxCreate */ + {__pyx_k_cuGreenCtxDestroy, sizeof(__pyx_k_cuGreenCtxDestroy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGreenCtxDestroy */ + {__pyx_k_cuGreenCtxGetDevResource, sizeof(__pyx_k_cuGreenCtxGetDevResource), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGreenCtxGetDevResource */ + {__pyx_k_cuGreenCtxRecordEvent, sizeof(__pyx_k_cuGreenCtxRecordEvent), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGreenCtxRecordEvent */ + {__pyx_k_cuGreenCtxStreamCreate, sizeof(__pyx_k_cuGreenCtxStreamCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGreenCtxStreamCreate */ + {__pyx_k_cuGreenCtxWaitEvent, sizeof(__pyx_k_cuGreenCtxWaitEvent), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuGreenCtxWaitEvent */ + {__pyx_k_cuImportExternalMemory, sizeof(__pyx_k_cuImportExternalMemory), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuImportExternalMemory */ + {__pyx_k_cuImportExternalSemaphore, sizeof(__pyx_k_cuImportExternalSemaphore), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuImportExternalSemaphore */ + {__pyx_k_cuInit, sizeof(__pyx_k_cuInit), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuInit */ + {__pyx_k_cuIpcCloseMemHandle, sizeof(__pyx_k_cuIpcCloseMemHandle), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuIpcCloseMemHandle */ + {__pyx_k_cuIpcGetEventHandle, sizeof(__pyx_k_cuIpcGetEventHandle), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuIpcGetEventHandle */ + {__pyx_k_cuIpcGetMemHandle, sizeof(__pyx_k_cuIpcGetMemHandle), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuIpcGetMemHandle */ + {__pyx_k_cuIpcOpenEventHandle, sizeof(__pyx_k_cuIpcOpenEventHandle), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuIpcOpenEventHandle */ + {__pyx_k_cuIpcOpenMemHandle_v2, sizeof(__pyx_k_cuIpcOpenMemHandle_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuIpcOpenMemHandle_v2 */ + {__pyx_k_cuKernelGetAttribute, sizeof(__pyx_k_cuKernelGetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuKernelGetAttribute */ + {__pyx_k_cuKernelGetFunction, sizeof(__pyx_k_cuKernelGetFunction), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuKernelGetFunction */ + {__pyx_k_cuKernelGetLibrary, sizeof(__pyx_k_cuKernelGetLibrary), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuKernelGetLibrary */ + {__pyx_k_cuKernelGetName, sizeof(__pyx_k_cuKernelGetName), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuKernelGetName */ + {__pyx_k_cuKernelGetParamInfo, sizeof(__pyx_k_cuKernelGetParamInfo), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuKernelGetParamInfo */ + {__pyx_k_cuKernelSetAttribute, sizeof(__pyx_k_cuKernelSetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuKernelSetAttribute */ + {__pyx_k_cuKernelSetCacheConfig, sizeof(__pyx_k_cuKernelSetCacheConfig), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuKernelSetCacheConfig */ + {__pyx_k_cuLaunch, sizeof(__pyx_k_cuLaunch), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLaunch */ + {__pyx_k_cuLaunchCooperativeKernel, sizeof(__pyx_k_cuLaunchCooperativeKernel), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLaunchCooperativeKernel */ + {__pyx_k_cuLaunchCooperativeKernelMulti, sizeof(__pyx_k_cuLaunchCooperativeKernelMulti), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLaunchCooperativeKernelMulti */ + {__pyx_k_cuLaunchGrid, sizeof(__pyx_k_cuLaunchGrid), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLaunchGrid */ + {__pyx_k_cuLaunchGridAsync, sizeof(__pyx_k_cuLaunchGridAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLaunchGridAsync */ + {__pyx_k_cuLaunchHostFunc, sizeof(__pyx_k_cuLaunchHostFunc), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLaunchHostFunc */ + {__pyx_k_cuLaunchKernel, sizeof(__pyx_k_cuLaunchKernel), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLaunchKernel */ + {__pyx_k_cuLaunchKernelEx, sizeof(__pyx_k_cuLaunchKernelEx), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLaunchKernelEx */ + {__pyx_k_cuLibraryEnumerateKernels, sizeof(__pyx_k_cuLibraryEnumerateKernels), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLibraryEnumerateKernels */ + {__pyx_k_cuLibraryGetGlobal, sizeof(__pyx_k_cuLibraryGetGlobal), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLibraryGetGlobal */ + {__pyx_k_cuLibraryGetKernel, sizeof(__pyx_k_cuLibraryGetKernel), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLibraryGetKernel */ + {__pyx_k_cuLibraryGetKernelCount, sizeof(__pyx_k_cuLibraryGetKernelCount), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLibraryGetKernelCount */ + {__pyx_k_cuLibraryGetManaged, sizeof(__pyx_k_cuLibraryGetManaged), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLibraryGetManaged */ + {__pyx_k_cuLibraryGetModule, sizeof(__pyx_k_cuLibraryGetModule), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLibraryGetModule */ + {__pyx_k_cuLibraryGetUnifiedFunction, sizeof(__pyx_k_cuLibraryGetUnifiedFunction), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLibraryGetUnifiedFunction */ + {__pyx_k_cuLibraryLoadData, sizeof(__pyx_k_cuLibraryLoadData), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLibraryLoadData */ + {__pyx_k_cuLibraryLoadFromFile, sizeof(__pyx_k_cuLibraryLoadFromFile), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLibraryLoadFromFile */ + {__pyx_k_cuLibraryUnload, sizeof(__pyx_k_cuLibraryUnload), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLibraryUnload */ + {__pyx_k_cuLinkAddData_v2, sizeof(__pyx_k_cuLinkAddData_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLinkAddData_v2 */ + {__pyx_k_cuLinkAddFile_v2, sizeof(__pyx_k_cuLinkAddFile_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLinkAddFile_v2 */ + {__pyx_k_cuLinkComplete, sizeof(__pyx_k_cuLinkComplete), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLinkComplete */ + {__pyx_k_cuLinkCreate_v2, sizeof(__pyx_k_cuLinkCreate_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLinkCreate_v2 */ + {__pyx_k_cuLinkDestroy, sizeof(__pyx_k_cuLinkDestroy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLinkDestroy */ + {__pyx_k_cuLogsCurrent, sizeof(__pyx_k_cuLogsCurrent), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLogsCurrent */ + {__pyx_k_cuLogsDumpToFile, sizeof(__pyx_k_cuLogsDumpToFile), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLogsDumpToFile */ + {__pyx_k_cuLogsDumpToMemory, sizeof(__pyx_k_cuLogsDumpToMemory), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLogsDumpToMemory */ + {__pyx_k_cuLogsRegisterCallback, sizeof(__pyx_k_cuLogsRegisterCallback), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLogsRegisterCallback */ + {__pyx_k_cuLogsUnregisterCallback, sizeof(__pyx_k_cuLogsUnregisterCallback), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuLogsUnregisterCallback */ + {__pyx_k_cuMemAddressFree, sizeof(__pyx_k_cuMemAddressFree), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemAddressFree */ + {__pyx_k_cuMemAddressReserve, sizeof(__pyx_k_cuMemAddressReserve), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemAddressReserve */ + {__pyx_k_cuMemAdvise, sizeof(__pyx_k_cuMemAdvise), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemAdvise */ + {__pyx_k_cuMemAdvise_v2, sizeof(__pyx_k_cuMemAdvise_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemAdvise_v2 */ + {__pyx_k_cuMemAllocAsync, sizeof(__pyx_k_cuMemAllocAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemAllocAsync */ + {__pyx_k_cuMemAllocFromPoolAsync, sizeof(__pyx_k_cuMemAllocFromPoolAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemAllocFromPoolAsync */ + {__pyx_k_cuMemAllocHost_v2, sizeof(__pyx_k_cuMemAllocHost_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemAllocHost_v2 */ + {__pyx_k_cuMemAllocManaged, sizeof(__pyx_k_cuMemAllocManaged), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemAllocManaged */ + {__pyx_k_cuMemAllocPitch_v2, sizeof(__pyx_k_cuMemAllocPitch_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemAllocPitch_v2 */ + {__pyx_k_cuMemAlloc_v2, sizeof(__pyx_k_cuMemAlloc_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemAlloc_v2 */ + {__pyx_k_cuMemBatchDecompressAsync, sizeof(__pyx_k_cuMemBatchDecompressAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemBatchDecompressAsync */ + {__pyx_k_cuMemCreate, sizeof(__pyx_k_cuMemCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemCreate */ + {__pyx_k_cuMemExportToShareableHandle, sizeof(__pyx_k_cuMemExportToShareableHandle), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemExportToShareableHandle */ + {__pyx_k_cuMemFreeAsync, sizeof(__pyx_k_cuMemFreeAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemFreeAsync */ + {__pyx_k_cuMemFreeHost, sizeof(__pyx_k_cuMemFreeHost), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemFreeHost */ + {__pyx_k_cuMemFree_v2, sizeof(__pyx_k_cuMemFree_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemFree_v2 */ + {__pyx_k_cuMemGetAccess, sizeof(__pyx_k_cuMemGetAccess), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemGetAccess */ + {__pyx_k_cuMemGetAddressRange_v2, sizeof(__pyx_k_cuMemGetAddressRange_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemGetAddressRange_v2 */ + {__pyx_k_cuMemGetAllocationGranularity, sizeof(__pyx_k_cuMemGetAllocationGranularity), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemGetAllocationGranularity */ + {__pyx_k_cuMemGetAllocationPropertiesFr, sizeof(__pyx_k_cuMemGetAllocationPropertiesFr), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemGetAllocationPropertiesFr */ + {__pyx_k_cuMemGetHandleForAddressRange, sizeof(__pyx_k_cuMemGetHandleForAddressRange), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemGetHandleForAddressRange */ + {__pyx_k_cuMemGetInfo_v2, sizeof(__pyx_k_cuMemGetInfo_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemGetInfo_v2 */ + {__pyx_k_cuMemHostAlloc, sizeof(__pyx_k_cuMemHostAlloc), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemHostAlloc */ + {__pyx_k_cuMemHostGetDevicePointer_v2, sizeof(__pyx_k_cuMemHostGetDevicePointer_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemHostGetDevicePointer_v2 */ + {__pyx_k_cuMemHostGetFlags, sizeof(__pyx_k_cuMemHostGetFlags), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemHostGetFlags */ + {__pyx_k_cuMemHostRegister_v2, sizeof(__pyx_k_cuMemHostRegister_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemHostRegister_v2 */ + {__pyx_k_cuMemHostUnregister, sizeof(__pyx_k_cuMemHostUnregister), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemHostUnregister */ + {__pyx_k_cuMemImportFromShareableHandle, sizeof(__pyx_k_cuMemImportFromShareableHandle), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemImportFromShareableHandle */ + {__pyx_k_cuMemMap, sizeof(__pyx_k_cuMemMap), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemMap */ + {__pyx_k_cuMemMapArrayAsync, sizeof(__pyx_k_cuMemMapArrayAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemMapArrayAsync */ + {__pyx_k_cuMemPoolCreate, sizeof(__pyx_k_cuMemPoolCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemPoolCreate */ + {__pyx_k_cuMemPoolDestroy, sizeof(__pyx_k_cuMemPoolDestroy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemPoolDestroy */ + {__pyx_k_cuMemPoolExportPointer, sizeof(__pyx_k_cuMemPoolExportPointer), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemPoolExportPointer */ + {__pyx_k_cuMemPoolExportToShareableHand, sizeof(__pyx_k_cuMemPoolExportToShareableHand), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemPoolExportToShareableHand */ + {__pyx_k_cuMemPoolGetAccess, sizeof(__pyx_k_cuMemPoolGetAccess), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemPoolGetAccess */ + {__pyx_k_cuMemPoolGetAttribute, sizeof(__pyx_k_cuMemPoolGetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemPoolGetAttribute */ + {__pyx_k_cuMemPoolImportFromShareableHa, sizeof(__pyx_k_cuMemPoolImportFromShareableHa), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemPoolImportFromShareableHa */ + {__pyx_k_cuMemPoolImportPointer, sizeof(__pyx_k_cuMemPoolImportPointer), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemPoolImportPointer */ + {__pyx_k_cuMemPoolSetAccess, sizeof(__pyx_k_cuMemPoolSetAccess), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemPoolSetAccess */ + {__pyx_k_cuMemPoolSetAttribute, sizeof(__pyx_k_cuMemPoolSetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemPoolSetAttribute */ + {__pyx_k_cuMemPoolTrimTo, sizeof(__pyx_k_cuMemPoolTrimTo), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemPoolTrimTo */ + {__pyx_k_cuMemPrefetchAsync, sizeof(__pyx_k_cuMemPrefetchAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemPrefetchAsync */ + {__pyx_k_cuMemPrefetchAsync_v2, sizeof(__pyx_k_cuMemPrefetchAsync_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemPrefetchAsync_v2 */ + {__pyx_k_cuMemRangeGetAttribute, sizeof(__pyx_k_cuMemRangeGetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemRangeGetAttribute */ + {__pyx_k_cuMemRangeGetAttributes, sizeof(__pyx_k_cuMemRangeGetAttributes), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemRangeGetAttributes */ + {__pyx_k_cuMemRelease, sizeof(__pyx_k_cuMemRelease), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemRelease */ + {__pyx_k_cuMemRetainAllocationHandle, sizeof(__pyx_k_cuMemRetainAllocationHandle), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemRetainAllocationHandle */ + {__pyx_k_cuMemSetAccess, sizeof(__pyx_k_cuMemSetAccess), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemSetAccess */ + {__pyx_k_cuMemUnmap, sizeof(__pyx_k_cuMemUnmap), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemUnmap */ + {__pyx_k_cuMemcpy, sizeof(__pyx_k_cuMemcpy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpy */ + {__pyx_k_cuMemcpy2DAsync_v2, sizeof(__pyx_k_cuMemcpy2DAsync_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpy2DAsync_v2 */ + {__pyx_k_cuMemcpy2DUnaligned_v2, sizeof(__pyx_k_cuMemcpy2DUnaligned_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpy2DUnaligned_v2 */ + {__pyx_k_cuMemcpy2D_v2, sizeof(__pyx_k_cuMemcpy2D_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpy2D_v2 */ + {__pyx_k_cuMemcpy3DAsync_v2, sizeof(__pyx_k_cuMemcpy3DAsync_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpy3DAsync_v2 */ + {__pyx_k_cuMemcpy3DBatchAsync, sizeof(__pyx_k_cuMemcpy3DBatchAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpy3DBatchAsync */ + {__pyx_k_cuMemcpy3DPeer, sizeof(__pyx_k_cuMemcpy3DPeer), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpy3DPeer */ + {__pyx_k_cuMemcpy3DPeerAsync, sizeof(__pyx_k_cuMemcpy3DPeerAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpy3DPeerAsync */ + {__pyx_k_cuMemcpy3D_v2, sizeof(__pyx_k_cuMemcpy3D_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpy3D_v2 */ + {__pyx_k_cuMemcpyAsync, sizeof(__pyx_k_cuMemcpyAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyAsync */ + {__pyx_k_cuMemcpyAtoA_v2, sizeof(__pyx_k_cuMemcpyAtoA_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyAtoA_v2 */ + {__pyx_k_cuMemcpyAtoD_v2, sizeof(__pyx_k_cuMemcpyAtoD_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyAtoD_v2 */ + {__pyx_k_cuMemcpyAtoHAsync_v2, sizeof(__pyx_k_cuMemcpyAtoHAsync_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyAtoHAsync_v2 */ + {__pyx_k_cuMemcpyAtoH_v2, sizeof(__pyx_k_cuMemcpyAtoH_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyAtoH_v2 */ + {__pyx_k_cuMemcpyBatchAsync, sizeof(__pyx_k_cuMemcpyBatchAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyBatchAsync */ + {__pyx_k_cuMemcpyDtoA_v2, sizeof(__pyx_k_cuMemcpyDtoA_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyDtoA_v2 */ + {__pyx_k_cuMemcpyDtoDAsync_v2, sizeof(__pyx_k_cuMemcpyDtoDAsync_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyDtoDAsync_v2 */ + {__pyx_k_cuMemcpyDtoD_v2, sizeof(__pyx_k_cuMemcpyDtoD_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyDtoD_v2 */ + {__pyx_k_cuMemcpyDtoHAsync_v2, sizeof(__pyx_k_cuMemcpyDtoHAsync_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyDtoHAsync_v2 */ + {__pyx_k_cuMemcpyDtoH_v2, sizeof(__pyx_k_cuMemcpyDtoH_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyDtoH_v2 */ + {__pyx_k_cuMemcpyHtoAAsync_v2, sizeof(__pyx_k_cuMemcpyHtoAAsync_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyHtoAAsync_v2 */ + {__pyx_k_cuMemcpyHtoA_v2, sizeof(__pyx_k_cuMemcpyHtoA_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyHtoA_v2 */ + {__pyx_k_cuMemcpyHtoDAsync_v2, sizeof(__pyx_k_cuMemcpyHtoDAsync_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyHtoDAsync_v2 */ + {__pyx_k_cuMemcpyHtoD_v2, sizeof(__pyx_k_cuMemcpyHtoD_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyHtoD_v2 */ + {__pyx_k_cuMemcpyPeer, sizeof(__pyx_k_cuMemcpyPeer), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyPeer */ + {__pyx_k_cuMemcpyPeerAsync, sizeof(__pyx_k_cuMemcpyPeerAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemcpyPeerAsync */ + {__pyx_k_cuMemsetD16Async, sizeof(__pyx_k_cuMemsetD16Async), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemsetD16Async */ + {__pyx_k_cuMemsetD16_v2, sizeof(__pyx_k_cuMemsetD16_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemsetD16_v2 */ + {__pyx_k_cuMemsetD2D16Async, sizeof(__pyx_k_cuMemsetD2D16Async), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemsetD2D16Async */ + {__pyx_k_cuMemsetD2D16_v2, sizeof(__pyx_k_cuMemsetD2D16_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemsetD2D16_v2 */ + {__pyx_k_cuMemsetD2D32Async, sizeof(__pyx_k_cuMemsetD2D32Async), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemsetD2D32Async */ + {__pyx_k_cuMemsetD2D32_v2, sizeof(__pyx_k_cuMemsetD2D32_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemsetD2D32_v2 */ + {__pyx_k_cuMemsetD2D8Async, sizeof(__pyx_k_cuMemsetD2D8Async), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemsetD2D8Async */ + {__pyx_k_cuMemsetD2D8_v2, sizeof(__pyx_k_cuMemsetD2D8_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemsetD2D8_v2 */ + {__pyx_k_cuMemsetD32Async, sizeof(__pyx_k_cuMemsetD32Async), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemsetD32Async */ + {__pyx_k_cuMemsetD32_v2, sizeof(__pyx_k_cuMemsetD32_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemsetD32_v2 */ + {__pyx_k_cuMemsetD8Async, sizeof(__pyx_k_cuMemsetD8Async), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemsetD8Async */ + {__pyx_k_cuMemsetD8_v2, sizeof(__pyx_k_cuMemsetD8_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMemsetD8_v2 */ + {__pyx_k_cuMipmappedArrayCreate, sizeof(__pyx_k_cuMipmappedArrayCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMipmappedArrayCreate */ + {__pyx_k_cuMipmappedArrayDestroy, sizeof(__pyx_k_cuMipmappedArrayDestroy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMipmappedArrayDestroy */ + {__pyx_k_cuMipmappedArrayGetLevel, sizeof(__pyx_k_cuMipmappedArrayGetLevel), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMipmappedArrayGetLevel */ + {__pyx_k_cuMipmappedArrayGetMemoryRequi, sizeof(__pyx_k_cuMipmappedArrayGetMemoryRequi), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMipmappedArrayGetMemoryRequi */ + {__pyx_k_cuMipmappedArrayGetSparsePrope, sizeof(__pyx_k_cuMipmappedArrayGetSparsePrope), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMipmappedArrayGetSparsePrope */ + {__pyx_k_cuModuleEnumerateFunctions, sizeof(__pyx_k_cuModuleEnumerateFunctions), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuModuleEnumerateFunctions */ + {__pyx_k_cuModuleGetFunction, sizeof(__pyx_k_cuModuleGetFunction), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuModuleGetFunction */ + {__pyx_k_cuModuleGetFunctionCount, sizeof(__pyx_k_cuModuleGetFunctionCount), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuModuleGetFunctionCount */ + {__pyx_k_cuModuleGetGlobal_v2, sizeof(__pyx_k_cuModuleGetGlobal_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuModuleGetGlobal_v2 */ + {__pyx_k_cuModuleGetLoadingMode, sizeof(__pyx_k_cuModuleGetLoadingMode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuModuleGetLoadingMode */ + {__pyx_k_cuModuleGetSurfRef, sizeof(__pyx_k_cuModuleGetSurfRef), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuModuleGetSurfRef */ + {__pyx_k_cuModuleGetTexRef, sizeof(__pyx_k_cuModuleGetTexRef), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuModuleGetTexRef */ + {__pyx_k_cuModuleLoad, sizeof(__pyx_k_cuModuleLoad), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuModuleLoad */ + {__pyx_k_cuModuleLoadData, sizeof(__pyx_k_cuModuleLoadData), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuModuleLoadData */ + {__pyx_k_cuModuleLoadDataEx, sizeof(__pyx_k_cuModuleLoadDataEx), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuModuleLoadDataEx */ + {__pyx_k_cuModuleLoadFatBinary, sizeof(__pyx_k_cuModuleLoadFatBinary), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuModuleLoadFatBinary */ + {__pyx_k_cuModuleUnload, sizeof(__pyx_k_cuModuleUnload), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuModuleUnload */ + {__pyx_k_cuMulticastAddDevice, sizeof(__pyx_k_cuMulticastAddDevice), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMulticastAddDevice */ + {__pyx_k_cuMulticastBindAddr, sizeof(__pyx_k_cuMulticastBindAddr), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMulticastBindAddr */ + {__pyx_k_cuMulticastBindMem, sizeof(__pyx_k_cuMulticastBindMem), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMulticastBindMem */ + {__pyx_k_cuMulticastCreate, sizeof(__pyx_k_cuMulticastCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMulticastCreate */ + {__pyx_k_cuMulticastGetGranularity, sizeof(__pyx_k_cuMulticastGetGranularity), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMulticastGetGranularity */ + {__pyx_k_cuMulticastUnbind, sizeof(__pyx_k_cuMulticastUnbind), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuMulticastUnbind */ + {__pyx_k_cuOccupancyAvailableDynamicSMe, sizeof(__pyx_k_cuOccupancyAvailableDynamicSMe), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuOccupancyAvailableDynamicSMe */ + {__pyx_k_cuOccupancyMaxActiveBlocksPerM, sizeof(__pyx_k_cuOccupancyMaxActiveBlocksPerM), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuOccupancyMaxActiveBlocksPerM */ + {__pyx_k_cuOccupancyMaxActiveBlocksPerM_2, sizeof(__pyx_k_cuOccupancyMaxActiveBlocksPerM_2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuOccupancyMaxActiveBlocksPerM_2 */ + {__pyx_k_cuOccupancyMaxActiveClusters, sizeof(__pyx_k_cuOccupancyMaxActiveClusters), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuOccupancyMaxActiveClusters */ + {__pyx_k_cuOccupancyMaxPotentialBlockSi, sizeof(__pyx_k_cuOccupancyMaxPotentialBlockSi), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuOccupancyMaxPotentialBlockSi */ + {__pyx_k_cuOccupancyMaxPotentialBlockSi_2, sizeof(__pyx_k_cuOccupancyMaxPotentialBlockSi_2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuOccupancyMaxPotentialBlockSi_2 */ + {__pyx_k_cuOccupancyMaxPotentialCluster, sizeof(__pyx_k_cuOccupancyMaxPotentialCluster), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuOccupancyMaxPotentialCluster */ + {__pyx_k_cuParamSetSize, sizeof(__pyx_k_cuParamSetSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuParamSetSize */ + {__pyx_k_cuParamSetTexRef, sizeof(__pyx_k_cuParamSetTexRef), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuParamSetTexRef */ + {__pyx_k_cuParamSetf, sizeof(__pyx_k_cuParamSetf), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuParamSetf */ + {__pyx_k_cuParamSeti, sizeof(__pyx_k_cuParamSeti), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuParamSeti */ + {__pyx_k_cuParamSetv, sizeof(__pyx_k_cuParamSetv), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuParamSetv */ + {__pyx_k_cuPointerGetAttribute, sizeof(__pyx_k_cuPointerGetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuPointerGetAttribute */ + {__pyx_k_cuPointerGetAttributes, sizeof(__pyx_k_cuPointerGetAttributes), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuPointerGetAttributes */ + {__pyx_k_cuPointerSetAttribute, sizeof(__pyx_k_cuPointerSetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuPointerSetAttribute */ + {__pyx_k_cuProfilerStart, sizeof(__pyx_k_cuProfilerStart), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuProfilerStart */ + {__pyx_k_cuProfilerStop, sizeof(__pyx_k_cuProfilerStop), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuProfilerStop */ + {__pyx_k_cuSignalExternalSemaphoresAsyn, sizeof(__pyx_k_cuSignalExternalSemaphoresAsyn), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuSignalExternalSemaphoresAsyn */ + {__pyx_k_cuStreamAddCallback, sizeof(__pyx_k_cuStreamAddCallback), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamAddCallback */ + {__pyx_k_cuStreamAttachMemAsync, sizeof(__pyx_k_cuStreamAttachMemAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamAttachMemAsync */ + {__pyx_k_cuStreamBatchMemOp_v2, sizeof(__pyx_k_cuStreamBatchMemOp_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamBatchMemOp_v2 */ + {__pyx_k_cuStreamBeginCaptureToGraph, sizeof(__pyx_k_cuStreamBeginCaptureToGraph), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamBeginCaptureToGraph */ + {__pyx_k_cuStreamBeginCapture_v2, sizeof(__pyx_k_cuStreamBeginCapture_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamBeginCapture_v2 */ + {__pyx_k_cuStreamCopyAttributes, sizeof(__pyx_k_cuStreamCopyAttributes), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamCopyAttributes */ + {__pyx_k_cuStreamCreate, sizeof(__pyx_k_cuStreamCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamCreate */ + {__pyx_k_cuStreamCreateWithPriority, sizeof(__pyx_k_cuStreamCreateWithPriority), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamCreateWithPriority */ + {__pyx_k_cuStreamDestroy_v2, sizeof(__pyx_k_cuStreamDestroy_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamDestroy_v2 */ + {__pyx_k_cuStreamEndCapture, sizeof(__pyx_k_cuStreamEndCapture), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamEndCapture */ + {__pyx_k_cuStreamGetAttribute, sizeof(__pyx_k_cuStreamGetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamGetAttribute */ + {__pyx_k_cuStreamGetCaptureInfo_v2, sizeof(__pyx_k_cuStreamGetCaptureInfo_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamGetCaptureInfo_v2 */ + {__pyx_k_cuStreamGetCaptureInfo_v3, sizeof(__pyx_k_cuStreamGetCaptureInfo_v3), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamGetCaptureInfo_v3 */ + {__pyx_k_cuStreamGetCtx, sizeof(__pyx_k_cuStreamGetCtx), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamGetCtx */ + {__pyx_k_cuStreamGetCtx_v2, sizeof(__pyx_k_cuStreamGetCtx_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamGetCtx_v2 */ + {__pyx_k_cuStreamGetDevice, sizeof(__pyx_k_cuStreamGetDevice), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamGetDevice */ + {__pyx_k_cuStreamGetFlags, sizeof(__pyx_k_cuStreamGetFlags), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamGetFlags */ + {__pyx_k_cuStreamGetGreenCtx, sizeof(__pyx_k_cuStreamGetGreenCtx), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamGetGreenCtx */ + {__pyx_k_cuStreamGetId, sizeof(__pyx_k_cuStreamGetId), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamGetId */ + {__pyx_k_cuStreamGetPriority, sizeof(__pyx_k_cuStreamGetPriority), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamGetPriority */ + {__pyx_k_cuStreamIsCapturing, sizeof(__pyx_k_cuStreamIsCapturing), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamIsCapturing */ + {__pyx_k_cuStreamQuery, sizeof(__pyx_k_cuStreamQuery), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamQuery */ + {__pyx_k_cuStreamSetAttribute, sizeof(__pyx_k_cuStreamSetAttribute), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamSetAttribute */ + {__pyx_k_cuStreamSynchronize, sizeof(__pyx_k_cuStreamSynchronize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamSynchronize */ + {__pyx_k_cuStreamUpdateCaptureDependenc, sizeof(__pyx_k_cuStreamUpdateCaptureDependenc), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamUpdateCaptureDependenc */ + {__pyx_k_cuStreamUpdateCaptureDependenc_2, sizeof(__pyx_k_cuStreamUpdateCaptureDependenc_2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamUpdateCaptureDependenc_2 */ + {__pyx_k_cuStreamWaitEvent, sizeof(__pyx_k_cuStreamWaitEvent), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamWaitEvent */ + {__pyx_k_cuStreamWaitValue32_v2, sizeof(__pyx_k_cuStreamWaitValue32_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamWaitValue32_v2 */ + {__pyx_k_cuStreamWaitValue64_v2, sizeof(__pyx_k_cuStreamWaitValue64_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamWaitValue64_v2 */ + {__pyx_k_cuStreamWriteValue32_v2, sizeof(__pyx_k_cuStreamWriteValue32_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamWriteValue32_v2 */ + {__pyx_k_cuStreamWriteValue64_v2, sizeof(__pyx_k_cuStreamWriteValue64_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuStreamWriteValue64_v2 */ + {__pyx_k_cuSurfObjectCreate, sizeof(__pyx_k_cuSurfObjectCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuSurfObjectCreate */ + {__pyx_k_cuSurfObjectDestroy, sizeof(__pyx_k_cuSurfObjectDestroy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuSurfObjectDestroy */ + {__pyx_k_cuSurfObjectGetResourceDesc, sizeof(__pyx_k_cuSurfObjectGetResourceDesc), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuSurfObjectGetResourceDesc */ + {__pyx_k_cuSurfRefGetArray, sizeof(__pyx_k_cuSurfRefGetArray), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuSurfRefGetArray */ + {__pyx_k_cuSurfRefSetArray, sizeof(__pyx_k_cuSurfRefSetArray), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuSurfRefSetArray */ + {__pyx_k_cuTensorMapEncodeIm2col, sizeof(__pyx_k_cuTensorMapEncodeIm2col), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTensorMapEncodeIm2col */ + {__pyx_k_cuTensorMapEncodeIm2colWide, sizeof(__pyx_k_cuTensorMapEncodeIm2colWide), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTensorMapEncodeIm2colWide */ + {__pyx_k_cuTensorMapEncodeTiled, sizeof(__pyx_k_cuTensorMapEncodeTiled), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTensorMapEncodeTiled */ + {__pyx_k_cuTensorMapReplaceAddress, sizeof(__pyx_k_cuTensorMapReplaceAddress), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTensorMapReplaceAddress */ + {__pyx_k_cuTexObjectCreate, sizeof(__pyx_k_cuTexObjectCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexObjectCreate */ + {__pyx_k_cuTexObjectDestroy, sizeof(__pyx_k_cuTexObjectDestroy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexObjectDestroy */ + {__pyx_k_cuTexObjectGetResourceDesc, sizeof(__pyx_k_cuTexObjectGetResourceDesc), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexObjectGetResourceDesc */ + {__pyx_k_cuTexObjectGetResourceViewDesc, sizeof(__pyx_k_cuTexObjectGetResourceViewDesc), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexObjectGetResourceViewDesc */ + {__pyx_k_cuTexObjectGetTextureDesc, sizeof(__pyx_k_cuTexObjectGetTextureDesc), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexObjectGetTextureDesc */ + {__pyx_k_cuTexRefCreate, sizeof(__pyx_k_cuTexRefCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefCreate */ + {__pyx_k_cuTexRefDestroy, sizeof(__pyx_k_cuTexRefDestroy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefDestroy */ + {__pyx_k_cuTexRefGetAddressMode, sizeof(__pyx_k_cuTexRefGetAddressMode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefGetAddressMode */ + {__pyx_k_cuTexRefGetAddress_v2, sizeof(__pyx_k_cuTexRefGetAddress_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefGetAddress_v2 */ + {__pyx_k_cuTexRefGetArray, sizeof(__pyx_k_cuTexRefGetArray), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefGetArray */ + {__pyx_k_cuTexRefGetBorderColor, sizeof(__pyx_k_cuTexRefGetBorderColor), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefGetBorderColor */ + {__pyx_k_cuTexRefGetFilterMode, sizeof(__pyx_k_cuTexRefGetFilterMode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefGetFilterMode */ + {__pyx_k_cuTexRefGetFlags, sizeof(__pyx_k_cuTexRefGetFlags), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefGetFlags */ + {__pyx_k_cuTexRefGetFormat, sizeof(__pyx_k_cuTexRefGetFormat), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefGetFormat */ + {__pyx_k_cuTexRefGetMaxAnisotropy, sizeof(__pyx_k_cuTexRefGetMaxAnisotropy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefGetMaxAnisotropy */ + {__pyx_k_cuTexRefGetMipmapFilterMode, sizeof(__pyx_k_cuTexRefGetMipmapFilterMode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefGetMipmapFilterMode */ + {__pyx_k_cuTexRefGetMipmapLevelBias, sizeof(__pyx_k_cuTexRefGetMipmapLevelBias), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefGetMipmapLevelBias */ + {__pyx_k_cuTexRefGetMipmapLevelClamp, sizeof(__pyx_k_cuTexRefGetMipmapLevelClamp), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefGetMipmapLevelClamp */ + {__pyx_k_cuTexRefGetMipmappedArray, sizeof(__pyx_k_cuTexRefGetMipmappedArray), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefGetMipmappedArray */ + {__pyx_k_cuTexRefSetAddress2D_v3, sizeof(__pyx_k_cuTexRefSetAddress2D_v3), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefSetAddress2D_v3 */ + {__pyx_k_cuTexRefSetAddressMode, sizeof(__pyx_k_cuTexRefSetAddressMode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefSetAddressMode */ + {__pyx_k_cuTexRefSetAddress_v2, sizeof(__pyx_k_cuTexRefSetAddress_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefSetAddress_v2 */ + {__pyx_k_cuTexRefSetArray, sizeof(__pyx_k_cuTexRefSetArray), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefSetArray */ + {__pyx_k_cuTexRefSetBorderColor, sizeof(__pyx_k_cuTexRefSetBorderColor), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefSetBorderColor */ + {__pyx_k_cuTexRefSetFilterMode, sizeof(__pyx_k_cuTexRefSetFilterMode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefSetFilterMode */ + {__pyx_k_cuTexRefSetFlags, sizeof(__pyx_k_cuTexRefSetFlags), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefSetFlags */ + {__pyx_k_cuTexRefSetFormat, sizeof(__pyx_k_cuTexRefSetFormat), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefSetFormat */ + {__pyx_k_cuTexRefSetMaxAnisotropy, sizeof(__pyx_k_cuTexRefSetMaxAnisotropy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefSetMaxAnisotropy */ + {__pyx_k_cuTexRefSetMipmapFilterMode, sizeof(__pyx_k_cuTexRefSetMipmapFilterMode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefSetMipmapFilterMode */ + {__pyx_k_cuTexRefSetMipmapLevelBias, sizeof(__pyx_k_cuTexRefSetMipmapLevelBias), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefSetMipmapLevelBias */ + {__pyx_k_cuTexRefSetMipmapLevelClamp, sizeof(__pyx_k_cuTexRefSetMipmapLevelClamp), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefSetMipmapLevelClamp */ + {__pyx_k_cuTexRefSetMipmappedArray, sizeof(__pyx_k_cuTexRefSetMipmappedArray), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuTexRefSetMipmappedArray */ + {__pyx_k_cuThreadExchangeStreamCaptureM, sizeof(__pyx_k_cuThreadExchangeStreamCaptureM), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuThreadExchangeStreamCaptureM */ + {__pyx_k_cuUserObjectCreate, sizeof(__pyx_k_cuUserObjectCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuUserObjectCreate */ + {__pyx_k_cuUserObjectRelease, sizeof(__pyx_k_cuUserObjectRelease), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuUserObjectRelease */ + {__pyx_k_cuUserObjectRetain, sizeof(__pyx_k_cuUserObjectRetain), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuUserObjectRetain */ + {__pyx_k_cuVDPAUCtxCreate_v2, sizeof(__pyx_k_cuVDPAUCtxCreate_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuVDPAUCtxCreate_v2 */ + {__pyx_k_cuVDPAUGetDevice, sizeof(__pyx_k_cuVDPAUGetDevice), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuVDPAUGetDevice */ + {__pyx_k_cuWaitExternalSemaphoresAsync, sizeof(__pyx_k_cuWaitExternalSemaphoresAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuWaitExternalSemaphoresAsync */ + {__pyx_k_cuda_bindings__bindings_cydriver, sizeof(__pyx_k_cuda_bindings__bindings_cydriver), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuda_bindings__bindings_cydriver */ + {__pyx_k_cuda_bindings__bindings_cydriver_2, sizeof(__pyx_k_cuda_bindings__bindings_cydriver_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_cuda_bindings__bindings_cydriver_2 */ + {__pyx_k_default, sizeof(__pyx_k_default), 0, 1, 1}, /* PyObject cname: __pyx_n_u_default */ + {__pyx_k_encoding, sizeof(__pyx_k_encoding), 0, 1, 1}, /* PyObject cname: __pyx_n_u_encoding */ + {__pyx_k_enter, sizeof(__pyx_k_enter), 0, 1, 1}, /* PyObject cname: __pyx_n_u_enter */ + {__pyx_k_exit, sizeof(__pyx_k_exit), 0, 1, 1}, /* PyObject cname: __pyx_n_u_exit */ + {__pyx_k_func, sizeof(__pyx_k_func), 0, 1, 1}, /* PyObject cname: __pyx_n_u_func */ + {__pyx_k_getenv, sizeof(__pyx_k_getenv), 0, 1, 1}, /* PyObject cname: __pyx_n_u_getenv */ + {__pyx_k_initializing, sizeof(__pyx_k_initializing), 0, 1, 1}, /* PyObject cname: __pyx_n_u_initializing */ + {__pyx_k_inspect_function_pointer, sizeof(__pyx_k_inspect_function_pointer), 0, 1, 1}, /* PyObject cname: __pyx_n_u_inspect_function_pointer */ + {__pyx_k_inspect_function_pointers, sizeof(__pyx_k_inspect_function_pointers), 0, 1, 1}, /* PyObject cname: __pyx_n_u_inspect_function_pointers */ + {__pyx_k_is_coroutine, sizeof(__pyx_k_is_coroutine), 0, 1, 1}, /* PyObject cname: __pyx_n_u_is_coroutine */ + {__pyx_k_libcuda_so_1, sizeof(__pyx_k_libcuda_so_1), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_libcuda_so_1 */ + {__pyx_k_main, sizeof(__pyx_k_main), 0, 1, 1}, /* PyObject cname: __pyx_n_u_main */ + {__pyx_k_maxsize, sizeof(__pyx_k_maxsize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_maxsize */ + {__pyx_k_module, sizeof(__pyx_k_module), 0, 1, 1}, /* PyObject cname: __pyx_n_u_module */ + {__pyx_k_name, sizeof(__pyx_k_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name */ + {__pyx_k_name_2, sizeof(__pyx_k_name_2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name_2 */ + {__pyx_k_os, sizeof(__pyx_k_os), 0, 1, 1}, /* PyObject cname: __pyx_n_u_os */ + {__pyx_k_pop, sizeof(__pyx_k_pop), 0, 1, 1}, /* PyObject cname: __pyx_n_u_pop */ + {__pyx_k_qualname, sizeof(__pyx_k_qualname), 0, 1, 1}, /* PyObject cname: __pyx_n_u_qualname */ + {__pyx_k_set_name, sizeof(__pyx_k_set_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_set_name */ + {__pyx_k_spec, sizeof(__pyx_k_spec), 0, 1, 1}, /* PyObject cname: __pyx_n_u_spec */ + {__pyx_k_sys, sizeof(__pyx_k_sys), 0, 1, 1}, /* PyObject cname: __pyx_n_u_sys */ + {__pyx_k_test, sizeof(__pyx_k_test), 0, 1, 1}, /* PyObject cname: __pyx_n_u_test */ + {__pyx_k_threading, sizeof(__pyx_k_threading), 0, 1, 1}, /* PyObject cname: __pyx_n_u_threading */ + {__pyx_k_utf_8, sizeof(__pyx_k_utf_8), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_utf_8 */ + {0, 0, 0, 0, 0} +}; +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names); + +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(0, 510, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "cuda/bindings/_bindings/cydriver.pyx":498 + * cdef char libPath[260] + * + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * usePTDS = os.getenv('CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM', default=0) + * +*/ + __pyx_mstate_global->__pyx_tuple[0] = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_tuple[0])) __PYX_ERR(0, 498, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.method_name = &__pyx_mstate->__pyx_n_u_pop; + if (__Pyx_InitStrings(__pyx_string_tab, __pyx_mstate->__pyx_string_tab, __pyx_string_tab_encodings) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + __pyx_mstate->__pyx_int_0 = PyLong_FromLong(0); if (unlikely(!__pyx_mstate->__pyx_int_0)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_int_4294967296 = PyLong_FromString("4294967296", 0, 0); if (unlikely(!__pyx_mstate->__pyx_int_4294967296)) __PYX_ERR(0, 1, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ +\ + typedef struct { + unsigned int argcount : 1; + unsigned int num_posonly_args : 1; + unsigned int num_kwonly_args : 1; + unsigned int nlocals : 1; + unsigned int flags : 10; + unsigned int first_line : 13; + unsigned int line_table_length : 18; + } __Pyx_PyCode_New_function_description; +/* NewCodeObj.proto */ +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + const char *line_table, + PyObject *tuple_dedup_map +); + + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + PyObject* tuple_dedup_map = PyDict_New(); + if (unlikely(!tuple_dedup_map)) return -1; + { + const __Pyx_PyCode_New_function_description descr = {0, 0, 0, 0, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 7038, 7133}; + PyObject* const varnames[] = {0}; + __pyx_mstate_global->__pyx_codeobj_tab[0] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_cuda_bindings__bindings_cydriver_2, __pyx_mstate->__pyx_n_u_inspect_function_pointers, __pyx_k_z_q_Q_1_z_z_Q_Ja_1_Q_Ja_3_Q_Zq, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[0])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 1, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 7996, 31}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_name}; + __pyx_mstate_global->__pyx_codeobj_tab[1] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_cuda_bindings__bindings_cydriver_2, __pyx_mstate->__pyx_n_u_inspect_function_pointer, __pyx_k_z_A_a_9AQ, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[1])) goto bad; + } + Py_DECREF(tuple_dedup_map); + return 0; + bad: + Py_DECREF(tuple_dedup_map); + return -1; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CachedMethodType.init */ + #if CYTHON_COMPILING_IN_LIMITED_API +{ + PyObject *typesModule=NULL; + typesModule = PyImport_ImportModule("types"); + if (typesModule) { + __pyx_mstate_global->__Pyx_CachedMethodType = PyObject_GetAttrString(typesModule, "MethodType"); + Py_DECREF(typesModule); + } +} // error handling follows +#endif + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* PyErrExceptionMatches */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStr */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* PyObjectGetAttrStrNoError */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name); + if (unlikely(!result) && !PyErr_Occurred()) { + PyErr_Format(PyExc_NameError, + "name '%U' is not defined", name); + } + return result; +} + +/* PyObjectLookupSpecial */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) { + PyObject *res; + PyTypeObject *tp = Py_TYPE(obj); + res = _PyType_Lookup(tp, attr_name); + if (likely(res)) { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (!f) { + Py_INCREF(res); + } else { + res = f(res, obj, (PyObject *)tp); + } + } else if (with_error) { + PyErr_SetObject(PyExc_AttributeError, attr_name); + } + return res; +} +#endif + +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject *const *args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = __Pyx_PyFrame_GetLocalsplus(f); + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; + PyObject *kwdefs; + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) { + return NULL; + } + if ( + co->co_kwonlyargcount == 0 && + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, (int)nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif + +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = Py_TYPE(func)->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + #if PY_VERSION_HEX < 0x030800B1 + #if CYTHON_FAST_PYCCALL + if (PyCFunction_Check(func)) { + if (kwargs) { + return _PyCFunction_FastCallDict(func, args, nargs, kwargs); + } else { + return _PyCFunction_FastCallKeywords(func, args, nargs, NULL); + } + } + if (!kwargs && __Pyx_IS_TYPE(func, &PyMethodDescr_Type)) { + return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL); + } + #endif + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs); + } + #endif + #endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #elif defined(__Pyx_CyFunction_USED) && CYTHON_BACKPORT_VECTORCALL + if (__Pyx_CyFunction_CheckExact(func)) { + __pyx_vectorcallfunc f = __Pyx_CyFunction_func_vectorcall(func); + if (f) return f(func, args, _nargs, NULL); + } + #elif CYTHON_COMPILING_IN_LIMITED_API && CYTHON_VECTORCALL + return PyObject_Vectorcall(func, args, _nargs, NULL); + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* PyDictVersioning */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!__pyx_m)) { + if (!PyErr_Occurred()) + PyErr_SetNone(PyExc_NameError); + return NULL; + } + result = PyObject_GetAttr(__pyx_m, name); + if (likely(result)) { + return result; + } + PyErr_Clear(); +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear(); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return result; + } +#else + result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + +/* PyObjectVectorCallKwBuilder */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* decode_c_string */ +static CYTHON_INLINE PyObject* __Pyx_decode_c_string( + const char* cstring, Py_ssize_t start, Py_ssize_t stop, + const char* encoding, const char* errors, + PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)) { + Py_ssize_t length; + if (unlikely((start < 0) | (stop < 0))) { + size_t slen = strlen(cstring); + if (unlikely(slen > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, + "c-string too long to convert to Python"); + return NULL; + } + length = (Py_ssize_t) slen; + if (start < 0) { + start += length; + if (start < 0) + start = 0; + } + if (stop < 0) + stop += length; + } + if (unlikely(stop <= start)) + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_unicode); + length = stop - start; + cstring += start; + if (decode_func) { + return decode_func(cstring, length, errors); + } else { + return PyUnicode_Decode(cstring, length, encoding, errors); + } +} + +/* RaiseException */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyException_SetTraceback(value, tb); +#elif CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} + +/* GetTopmostException */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + +/* SaveResetException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #endif +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); + #else + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); + #endif +} +#endif + +/* GetException */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) +#endif +{ + PyObject *local_type = NULL, *local_value, *local_tb = NULL; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030C0000 + local_value = tstate->current_exception; + tstate->current_exception = 0; + #else + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + #endif +#elif __PYX_LIMITED_VERSION_HEX > 0x030C0000 + local_value = PyErr_GetRaisedException(); +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif +#if __PYX_LIMITED_VERSION_HEX > 0x030C0000 + if (likely(local_value)) { + local_type = (PyObject*) Py_TYPE(local_value); + Py_INCREF(local_type); + local_tb = PyException_GetTraceback(local_value); + } +#else + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } +#endif // __PYX_LIMITED_VERSION_HEX > 0x030C0000 + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if CYTHON_USE_EXC_INFO_STACK + { + _PyErr_StackItem *exc_info = tstate->exc_info; + #if PY_VERSION_HEX >= 0x030B00a4 + tmp_value = exc_info->exc_value; + exc_info->exc_value = local_value; + tmp_type = NULL; + tmp_tb = NULL; + Py_XDECREF(local_type); + Py_XDECREF(local_tb); + #else + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; + #endif + } + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#elif __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + PyErr_SetHandledException(local_value); + Py_XDECREF(local_value); + Py_XDECREF(local_type); + Py_XDECREF(local_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +#if __PYX_LIMITED_VERSION_HEX <= 0x030C0000 +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +#endif +} + +/* ErrOccurredWithGIL */ +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void) { + int err; + PyGILState_STATE _save = PyGILState_Ensure(); + err = !!PyErr_Occurred(); + PyGILState_Release(_save); + return err; +} + +/* TupleAndListFromArray */ +#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + Py_ssize_t i; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + for (i = 0; i < n; i++) { + if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < (0))) { + Py_DECREF(res); + return NULL; + } + Py_INCREF(src[i]); + } + return res; +} +#elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + +/* BytesEquals */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL ||\ + !(CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) + return PyObject_RichCompareBool(s1, s2, equals); +#else + if (s1 == s2) { + return (equals == Py_EQ); + } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { + const char *ps1, *ps2; + Py_ssize_t length = PyBytes_GET_SIZE(s1); + if (length != PyBytes_GET_SIZE(s2)) + return (equals == Py_NE); + ps1 = PyBytes_AS_STRING(s1); + ps2 = PyBytes_AS_STRING(s2); + if (ps1[0] != ps2[0]) { + return (equals == Py_NE); + } else if (length == 1) { + return (equals == Py_EQ); + } else { + int result; +#if CYTHON_USE_UNICODE_INTERNALS && (PY_VERSION_HEX < 0x030B0000) + Py_hash_t hash1, hash2; + hash1 = ((PyBytesObject*)s1)->ob_shash; + hash2 = ((PyBytesObject*)s2)->ob_shash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + return (equals == Py_NE); + } +#endif + result = memcmp(ps1, ps2, (size_t)length); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { + return (equals == Py_NE); + } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) { + return (equals == Py_NE); + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +#endif +} + +/* UnicodeEquals */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL + return PyObject_RichCompareBool(s1, s2, equals); +#else + int s1_is_unicode, s2_is_unicode; + if (s1 == s2) { + goto return_eq; + } + s1_is_unicode = PyUnicode_CheckExact(s1); + s2_is_unicode = PyUnicode_CheckExact(s2); + if (s1_is_unicode & s2_is_unicode) { + Py_ssize_t length, length2; + int kind; + void *data1, *data2; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0)) + return -1; + #endif + length = __Pyx_PyUnicode_GET_LENGTH(s1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length < 0)) return -1; + #endif + length2 = __Pyx_PyUnicode_GET_LENGTH(s2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length2 < 0)) return -1; + #endif + if (length != length2) { + goto return_ne; + } +#if CYTHON_USE_UNICODE_INTERNALS + { + Py_hash_t hash1, hash2; + hash1 = ((PyASCIIObject*)s1)->hash; + hash2 = ((PyASCIIObject*)s2)->hash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + goto return_ne; + } + } +#endif + kind = __Pyx_PyUnicode_KIND(s1); + if (kind != __Pyx_PyUnicode_KIND(s2)) { + goto return_ne; + } + data1 = __Pyx_PyUnicode_DATA(s1); + data2 = __Pyx_PyUnicode_DATA(s2); + if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) { + goto return_ne; + } else if (length == 1) { + goto return_eq; + } else { + int result = memcmp(data1, data2, (size_t)(length * kind)); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & s2_is_unicode) { + goto return_ne; + } else if ((s2 == Py_None) & s1_is_unicode) { + goto return_ne; + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +return_eq: + return (equals == Py_EQ); +return_ne: + return (equals == Py_NE); +#endif +} + +/* fastcall */ +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + Py_ssize_t i, n = __Pyx_PyTuple_GET_SIZE(kwnames); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(n == -1)) return NULL; + #endif + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + if (s == namei) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + int eq = __Pyx_PyUnicode_Equals(s, namei, Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; + return kwvalues[i]; + } + } + return NULL; +} +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues) { + Py_ssize_t i, nkwargs; + PyObject *dict; +#if !CYTHON_ASSUME_SAFE_SIZE + nkwargs = PyTuple_Size(kwnames); + if (unlikely(nkwargs < 0)) return NULL; +#else + nkwargs = PyTuple_GET_SIZE(kwnames); +#endif + dict = PyDict_New(); + if (unlikely(!dict)) + return NULL; + for (i=0; itype, *target->method_name); + if (unlikely(!method)) + return -1; + result = method; +#if CYTHON_COMPILING_IN_CPYTHON + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + { + PyMethodDescrObject *descr = (PyMethodDescrObject*) method; + target->func = descr->d_method->ml_meth; + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); + } else +#endif +#if CYTHON_COMPILING_IN_PYPY +#else + if (PyCFunction_Check(method)) +#endif + { + PyObject *self; + int self_found; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + self = PyObject_GetAttrString(method, "__self__"); + if (!self) { + PyErr_Clear(); + } +#else + self = PyCFunction_GET_SELF(method); +#endif + self_found = (self && self != Py_None); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + Py_XDECREF(self); +#endif + if (self_found) { + PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method); + if (unlikely(!unbound_method)) return -1; + Py_DECREF(method); + result = unbound_method; + } + } +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + if (unlikely(target->method)) { + Py_DECREF(result); + } else +#endif + target->method = result; + return 0; +} + +/* CallUnboundCMethod2 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + PyObject *args[2] = {arg1, arg2}; + if (cfunc->flag == METH_FASTCALL) { + return __Pyx_CallCFunctionFast(cfunc, self, args, 2); + } + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, 2, NULL); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod2(&tmp_cfunc, self, arg1, arg2); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){ + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + PyObject *result = NULL; + PyObject *args = PyTuple_New(2); + if (unlikely(!args)) return NULL; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + if (cfunc->flag & METH_KEYWORDS) + result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL); + else + result = __Pyx_CallCFunction(cfunc, self, args); + Py_DECREF(args); + return result; + } +#endif + { + PyObject *args[4] = {NULL, self, arg1, arg2}; + return __Pyx_PyObject_FastCall(cfunc->method, args+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } +} + +/* ParseKeywords */ +static int __Pyx_ValidateDuplicatePosArgs( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char* function_name) +{ + PyObject ** const *name = argnames; + while (name != first_kw_arg) { + PyObject *key = **name; + int found = PyDict_Contains(kwds, key); + if (unlikely(found)) { + if (found == 1) __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; + } + name++; + } + return 0; +bad: + return -1; +} +#if CYTHON_USE_UNICODE_INTERNALS +static CYTHON_INLINE int __Pyx_UnicodeKeywordsEqual(PyObject *s1, PyObject *s2) { + int kind; + Py_ssize_t len = PyUnicode_GET_LENGTH(s1); + if (len != PyUnicode_GET_LENGTH(s2)) return 0; + kind = PyUnicode_KIND(s1); + if (kind != PyUnicode_KIND(s2)) return 0; + const void *data1 = PyUnicode_DATA(s1); + const void *data2 = PyUnicode_DATA(s2); + return (memcmp(data1, data2, (size_t) len * (size_t) kind) == 0); +} +#endif +static int __Pyx_MatchKeywordArg_str( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + #if CYTHON_USE_UNICODE_INTERNALS + Py_hash_t key_hash = ((PyASCIIObject*)key)->hash; + if (unlikely(key_hash == -1)) { + key_hash = PyObject_Hash(key); + if (unlikely(key_hash == -1)) + goto bad; + } + #endif + name = first_kw_arg; + while (*name) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (key_hash == ((PyASCIIObject*)name_str)->hash && __Pyx_UnicodeKeywordsEqual(name_str, key)) { + *index_found = (size_t) (name - argnames); + return 1; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + *index_found = (size_t) (name - argnames); + return 1; + } + } + #endif + name++; + } + name = argnames; + while (name != first_kw_arg) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (unlikely(key_hash == ((PyASCIIObject*)name_str)->hash)) { + if (__Pyx_UnicodeKeywordsEqual(name_str, key)) + goto arg_passed_twice; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + if (unlikely(name_str == key)) goto arg_passed_twice; + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + } + #endif + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +bad: + return -1; +} +static int __Pyx_MatchKeywordArg_nostr( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; + name = first_kw_arg; + while (*name) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (cmp == 1) { + *index_found = (size_t) (name - argnames); + return 1; + } + if (unlikely(cmp == -1)) goto bad; + name++; + } + name = argnames; + while (name != first_kw_arg) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (unlikely(cmp != 0)) { + if (cmp == 1) goto arg_passed_twice; + else goto bad; + } + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +bad: + return -1; +} +static CYTHON_INLINE int __Pyx_MatchKeywordArg( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + return likely(PyUnicode_CheckExact(key)) ? + __Pyx_MatchKeywordArg_str(key, argnames, first_kw_arg, index_found, function_name) : + __Pyx_MatchKeywordArg_nostr(key, argnames, first_kw_arg, index_found, function_name); +} +static void __Pyx_RejectUnknownKeyword( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char *function_name) +{ + Py_ssize_t pos = 0; + PyObject *key = NULL; + __Pyx_BEGIN_CRITICAL_SECTION(kwds); + while (PyDict_Next(kwds, &pos, &key, NULL)) { + PyObject** const *name = first_kw_arg; + while (*name && (**name != key)) name++; + if (!*name) { + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(key); + #endif + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp != 1) { + if (cmp == 0) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + break; + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + } + } + __Pyx_END_CRITICAL_SECTION(); + assert(PyErr_Occurred()); +} +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t extracted = 0; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + name = first_kw_arg; + while (*name && num_kwargs > extracted) { + PyObject * key = **name; + PyObject *value; + int found = 0; + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + found = PyDict_GetItemRef(kwds, key, &value); + #else + value = PyDict_GetItemWithError(kwds, key); + if (value) { + Py_INCREF(value); + found = 1; + } else { + if (unlikely(PyErr_Occurred())) goto bad; + } + #endif + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + extracted++; + } + name++; + } + if (num_kwargs > extracted) { + if (ignore_unknown_kwargs) { + if (unlikely(__Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name) == -1)) + goto bad; + } else { + __Pyx_RejectUnknownKeyword(kwds, argnames, first_kw_arg, function_name); + goto bad; + } + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t len; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + if (PyDict_Update(kwds2, kwds) < 0) goto bad; + name = first_kw_arg; + while (*name) { + PyObject *key = **name; + PyObject *value; +#if !CYTHON_COMPILING_IN_LIMITED_API && (PY_VERSION_HEX >= 0x030d00A2 || defined(PyDict_Pop)) + int found = PyDict_Pop(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int found = PyDict_GetItemRef(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + if (unlikely(PyDict_DelItem(kwds2, key) < 0)) goto bad; + } +#else + #if CYTHON_COMPILING_IN_CPYTHON + value = _PyDict_Pop(kwds2, key, kwds2); + #else + value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_pop, kwds2, key, kwds2); + #endif + if (value == kwds2) { + Py_DECREF(value); + } else { + if (unlikely(!value)) goto bad; + values[name-argnames] = value; + } +#endif + name++; + } + len = PyDict_Size(kwds2); + if (len > 0) { + return __Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name); + } else if (unlikely(len == -1)) { + goto bad; + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject *key = NULL; + PyObject** const * name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + for (Py_ssize_t pos = 0; pos < num_kwargs; pos++) { +#if CYTHON_AVOID_BORROWED_REFS + key = __Pyx_PySequence_ITEM(kwds, pos); +#else + key = __Pyx_PyTuple_GET_ITEM(kwds, pos); +#endif +#if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!key)) goto bad; +#endif + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + PyObject *value = kwvalues[pos]; + values[name-argnames] = __Pyx_NewRef(value); + } else { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp == 1) { + PyObject *value = kwvalues[pos]; + values[index_found] = __Pyx_NewRef(value); + } else { + if (unlikely(cmp == -1)) goto bad; + if (kwds2) { + PyObject *value = kwvalues[pos]; + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else if (!ignore_unknown_kwargs) { + goto invalid_keyword; + } + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + key = NULL; + #endif + } + return 0; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + goto bad; +bad: + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(key); + #endif + return -1; +} +static int __Pyx_ParseKeywords( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) + return __Pyx_ParseKeywordsTuple(kwds, kwvalues, argnames, kwds2, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); + else if (kwds2) + return __Pyx_ParseKeywordDictToDict(kwds, argnames, kwds2, values, num_pos_args, function_name); + else + return __Pyx_ParseKeywordDict(kwds, argnames, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* ArgTypeTest */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +{ + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; + PyObject *extra_info = __pyx_mstate_global->__pyx_empty_unicode; + int from_annotation_subclass = 0; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (!exact) { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } else if (exact == 2) { + if (__Pyx_TypeCheck(obj, type)) { + from_annotation_subclass = 1; + extra_info = __pyx_mstate_global->__pyx_kp_u_Note_that_Cython_is_deliberately; + } + } + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")" +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + "%s%U" +#endif + , name, type_name, obj_type_name +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + , (from_annotation_subclass ? ". " : ""), extra_info +#endif + ); +#if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + if (exact == 2 && from_annotation_subclass) { + PyObject *res; + PyObject *vargs[2]; + vargs[0] = PyErr_GetRaisedException(); + vargs[1] = extra_info; + res = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_kp_u_add_note, vargs, 2, NULL); + Py_XDECREF(res); + PyErr_SetRaisedException(vargs[0]); + } +#endif + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* Import */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { + PyObject *module = 0; + PyObject *empty_dict = 0; + PyObject *empty_list = 0; + empty_dict = PyDict_New(); + if (unlikely(!empty_dict)) + goto bad; + if (level == -1) { + const char* package_sep = strchr(__Pyx_MODULE_NAME, '.'); + if (package_sep != (0)) { + module = PyImport_ImportModuleLevelObject( + name, __pyx_mstate_global->__pyx_d, empty_dict, from_list, 1); + if (unlikely(!module)) { + if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError))) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } + if (!module) { + module = PyImport_ImportModuleLevelObject( + name, __pyx_mstate_global->__pyx_d, empty_dict, from_list, level); + } +bad: + Py_XDECREF(empty_dict); + Py_XDECREF(empty_list); + return module; +} + +/* ImportDottedModule */ +static PyObject *__Pyx__ImportDottedModule_Error(PyObject *name, PyObject *parts_tuple, Py_ssize_t count) { + PyObject *partial_name = NULL, *slice = NULL, *sep = NULL; + Py_ssize_t size; + if (unlikely(PyErr_Occurred())) { + PyErr_Clear(); + } +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(parts_tuple); +#else + size = PyTuple_Size(parts_tuple); + if (size < 0) goto bad; +#endif + if (likely(size == count)) { + partial_name = name; + } else { + slice = PySequence_GetSlice(parts_tuple, 0, count); + if (unlikely(!slice)) + goto bad; + sep = PyUnicode_FromStringAndSize(".", 1); + if (unlikely(!sep)) + goto bad; + partial_name = PyUnicode_Join(sep, slice); + } + PyErr_Format( + PyExc_ModuleNotFoundError, + "No module named '%U'", partial_name); +bad: + Py_XDECREF(sep); + Py_XDECREF(slice); + Py_XDECREF(partial_name); + return NULL; +} +static PyObject *__Pyx__ImportDottedModule_Lookup(PyObject *name) { + PyObject *imported_module; +#if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + return NULL; + imported_module = __Pyx_PyDict_GetItemStr(modules, name); + Py_XINCREF(imported_module); +#else + imported_module = PyImport_GetModule(name); +#endif + return imported_module; +} +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple) { + Py_ssize_t i, nparts; +#if CYTHON_ASSUME_SAFE_SIZE + nparts = PyTuple_GET_SIZE(parts_tuple); +#else + nparts = PyTuple_Size(parts_tuple); + if (nparts < 0) return NULL; +#endif + for (i=1; i < nparts && module; i++) { + PyObject *part, *submodule; +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + part = PyTuple_GET_ITEM(parts_tuple, i); +#else + part = __Pyx_PySequence_ITEM(parts_tuple, i); + if (!part) return NULL; +#endif + submodule = __Pyx_PyObject_GetAttrStrNoError(module, part); +#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(part); +#endif + Py_DECREF(module); + module = submodule; + } + if (unlikely(!module)) { + return __Pyx__ImportDottedModule_Error(name, parts_tuple, i); + } + return module; +} +static PyObject *__Pyx__ImportDottedModule(PyObject *name, PyObject *parts_tuple) { + PyObject *imported_module; + PyObject *module = __Pyx_Import(name, NULL, 0); + if (!parts_tuple || unlikely(!module)) + return module; + imported_module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(imported_module)) { + Py_DECREF(module); + return imported_module; + } + PyErr_Clear(); + return __Pyx_ImportDottedModule_WalkParts(module, name, parts_tuple); +} +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple) { +#if CYTHON_COMPILING_IN_CPYTHON + PyObject *module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(module)) { + PyObject *spec = __Pyx_PyObject_GetAttrStrNoError(module, __pyx_mstate_global->__pyx_n_u_spec); + if (likely(spec)) { + PyObject *unsafe = __Pyx_PyObject_GetAttrStrNoError(spec, __pyx_mstate_global->__pyx_n_u_initializing); + if (likely(!unsafe || !__Pyx_PyObject_IsTrue(unsafe))) { + Py_DECREF(spec); + spec = NULL; + } + Py_XDECREF(unsafe); + } + if (likely(!spec)) { + PyErr_Clear(); + return module; + } + Py_DECREF(spec); + Py_DECREF(module); + } else if (PyErr_Occurred()) { + PyErr_Clear(); + } +#endif + return __Pyx__ImportDottedModule(name, parts_tuple); +} + +/* LimitedApiGetTypeDict */ +#if CYTHON_COMPILING_IN_LIMITED_API +static Py_ssize_t __Pyx_GetTypeDictOffset(void) { + PyObject *tp_dictoffset_o; + Py_ssize_t tp_dictoffset; + tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__"); + if (unlikely(!tp_dictoffset_o)) return -1; + tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o); + Py_DECREF(tp_dictoffset_o); + if (unlikely(tp_dictoffset == 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' doesn't have a dictoffset"); + return -1; + } else if (unlikely(tp_dictoffset < 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' has an unexpected negative dictoffset. " + "Please report this as Cython bug"); + return -1; + } + return tp_dictoffset; +} +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) { + static Py_ssize_t tp_dictoffset = 0; + if (unlikely(tp_dictoffset == 0)) { + tp_dictoffset = __Pyx_GetTypeDictOffset(); + if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) { + tp_dictoffset = 0; // try again next time? + return NULL; + } + } + return *(PyObject**)((char*)tp + tp_dictoffset); +} +#endif + +/* SetItemOnTypeDict */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_SetItem(tp_dict, k, v); + if (likely(!result)) { + PyType_Modified(tp); + if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) { + PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name, (PyObject *) tp, k, NULL); + if (!setNameResult) return -1; + Py_DECREF(setNameResult); + } + } + return result; +} + +/* FixUpExtensionType */ +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if __PYX_LIMITED_VERSION_HEX > 0x030900B1 + CYTHON_UNUSED_VAR(spec); + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#else + const PyType_Slot *slot = spec->slots; + int changed = 0; +#if !CYTHON_COMPILING_IN_LIMITED_API + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { +#if !CYTHON_COMPILING_IN_CPYTHON + const +#endif // !CYTHON_COMPILING_IN_CPYTHON) + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); +#if PY_VERSION_HEX >= 0x030800b4 + type->tp_vectorcall_offset = memb->offset; +#else + type->tp_print = (printfunc) memb->offset; +#endif + changed = 1; + } +#endif // CYTHON_METH_FASTCALL +#if !CYTHON_COMPILING_IN_PYPY + else if (strcmp(memb->name, "__module__") == 0) { + PyObject *descr; + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } +#endif // !CYTHON_COMPILING_IN_PYPY + } + memb++; + } + } +#endif // !CYTHON_COMPILING_IN_LIMITED_API +#if !CYTHON_COMPILING_IN_PYPY + slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_getset) + slot++; + if (slot && slot->slot == Py_tp_getset) { + PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc; + while (getset && getset->name) { + if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) { + PyObject *descr = PyDescr_NewGetSet(type, getset); + if (unlikely(!descr)) + return -1; + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *pyname = PyUnicode_FromString(getset->name); + if (unlikely(!pyname)) { + Py_DECREF(descr); + return -1; + } + int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr); + Py_DECREF(pyname); + #else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + #endif + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } + ++getset; + } + } +#endif // !CYTHON_COMPILING_IN_PYPY + if (changed) + PyType_Modified(type); +#endif // PY_VERSION_HEX > 0x030900B1 + return 0; +} + +/* FetchSharedCythonModule */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + return __Pyx_PyImport_AddModuleRef(__PYX_ABI_MODULE_NAME); +} + +/* dict_setdefault */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, + int is_safe_type) { + PyObject* value; + CYTHON_MAYBE_UNUSED_VAR(is_safe_type); +#if CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_CallMethod(d, "setdefault", "OO", key, default_value); +#elif PY_VERSION_HEX >= 0x030d0000 + PyDict_SetDefaultRef(d, key, default_value, &value); +#else + value = PyDict_SetDefault(d, key, default_value); + if (unlikely(!value)) return NULL; + Py_INCREF(value); +#endif + return value; +} + +/* FetchCommonType */ +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject* __Pyx_PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *result = __Pyx_PyType_FromModuleAndSpec(module, spec, bases); + if (result && metaclass) { + PyObject *old_tp = (PyObject*)Py_TYPE(result); + Py_INCREF((PyObject*)metaclass); +#if __PYX_LIMITED_VERSION_HEX >= 0x03090000 + Py_SET_TYPE(result, metaclass); +#else + result->ob_type = metaclass; +#endif + Py_DECREF(old_tp); + } + return result; +} +#else +#define __Pyx_PyType_FromMetaclass(me, mo, s, b) PyType_FromMetaclass(me, mo, s, b) +#endif +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t expected_basicsize) { + Py_ssize_t basicsize; + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (expected_basicsize == 0) { + return 0; // size is inherited, nothing useful to check + } +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) return -1; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = NULL; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; +#else + basicsize = ((PyTypeObject*) cached_type)->tp_basicsize; +#endif + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module = NULL, *cached_type = NULL, *abi_module_dict, *new_cached_type, *py_object_name; + int get_item_ref_result; + const char* object_name = strrchr(spec->name, '.'); + object_name = object_name ? object_name+1 : spec->name; + py_object_name = PyUnicode_FromString(object_name); + if (!py_object_name) return NULL; + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) goto done; + abi_module_dict = PyModule_GetDict(abi_module); + if (!abi_module_dict) goto done; + get_item_ref_result = __Pyx_PyDict_GetItemRef(abi_module_dict, py_object_name, &cached_type); + if (get_item_ref_result == 1) { + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else if (unlikely(get_item_ref_result == -1)) { + goto bad; + } + CYTHON_UNUSED_VAR(module); + cached_type = __Pyx_PyType_FromMetaclass(metaclass, abi_module, spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + new_cached_type = __Pyx_PyDict_SetDefault(abi_module_dict, py_object_name, cached_type, 1); + if (unlikely(new_cached_type != cached_type)) { + if (unlikely(!new_cached_type)) goto bad; + Py_DECREF(cached_type); + cached_type = new_cached_type; + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else { + Py_DECREF(new_cached_type); + } +done: + Py_XDECREF(abi_module); + Py_DECREF(py_object_name); + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* CommonTypesMetaclass */ +static PyObject* __pyx_CommonTypesMetaclass_get_module(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED void* context) { + return PyUnicode_FromString(__PYX_ABI_MODULE_NAME); +} +static PyGetSetDef __pyx_CommonTypesMetaclass_getset[] = { + {"__module__", __pyx_CommonTypesMetaclass_get_module, NULL, NULL, NULL}, + {0, 0, 0, 0, 0} +}; +static PyType_Slot __pyx_CommonTypesMetaclass_slots[] = { + {Py_tp_getset, (void *)__pyx_CommonTypesMetaclass_getset}, + {0, 0} +}; +static PyType_Spec __pyx_CommonTypesMetaclass_spec = { + __PYX_TYPE_MODULE_PREFIX "_common_types_metatype", + 0, + 0, +#if PY_VERSION_HEX >= 0x030A0000 + Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | +#endif + Py_TPFLAGS_DEFAULT, + __pyx_CommonTypesMetaclass_slots +}; +static int __pyx_CommonTypesMetaclass_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + PyObject *bases = PyTuple_Pack(1, &PyType_Type); + if (unlikely(!bases)) { + return -1; + } + mstate->__pyx_CommonTypesMetaclassType = __Pyx_FetchCommonTypeFromSpec(NULL, module, &__pyx_CommonTypesMetaclass_spec, bases); + Py_DECREF(bases); + if (unlikely(mstate->__pyx_CommonTypesMetaclassType == NULL)) { + return -1; + } + return 0; +} + +/* CallTypeTraverse */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) { + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 + if (__Pyx_get_runtime_version() < 0x03090000) return 0; + #endif + if (!always_call) { + PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*); + unsigned long flags = PyType_GetFlags(base); + if (flags & Py_TPFLAGS_HEAPTYPE) { + return 0; + } + } + Py_VISIT((PyObject*)Py_TYPE(o)); + return 0; +} +#endif + +/* PyMethodNew */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + PyObject *result; + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + #if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + { + PyObject *args[] = {func, self}; + result = PyObject_Vectorcall(__pyx_mstate_global->__Pyx_CachedMethodType, args, 2, NULL); + } + #else + result = PyObject_CallFunctionObjArgs(__pyx_mstate_global->__Pyx_CachedMethodType, func, self, NULL); + #endif + return result; +} +#else +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#endif + +/* PyVectorcallFastCallDict */ +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i, pos; + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + #if !CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t nkw = PyDict_Size(kw); + if (unlikely(nkw == -1)) return NULL; + #else + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + #endif + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; + } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (PyDict_Next(kw, &pos, &key, &value)) { + keys_are_strings &= + #if CYTHON_COMPILING_IN_LIMITED_API + PyType_GetFlags(Py_TYPE(key)); + #else + Py_TYPE(key)->tp_flags; + #endif + Py_INCREF(key); + Py_INCREF(value); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(PyTuple_SetItem(kwnames, i, key) < 0)) goto cleanup; + #else + PyTuple_SET_ITEM(kwnames, i, key); + #endif + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + res = vc(func, newargs, nargs, kwnames); +cleanup: + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + Py_ssize_t kw_size = + likely(kw == NULL) ? + 0 : +#if !CYTHON_ASSUME_SAFE_SIZE + PyDict_Size(kw); +#else + PyDict_GET_SIZE(kw); +#endif + if (kw_size == 0) { + return vc(func, args, nargs, NULL); + } +#if !CYTHON_ASSUME_SAFE_SIZE + else if (unlikely(kw_size == -1)) { + return NULL; + } +#endif + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); +} +#endif + +/* CythonFunctionShared */ +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunctionNoMethod(PyObject *func, void (*cfunc)(void)) { + if (__Pyx_CyFunction_Check(func)) { + return PyCFunction_GetFunction(((__pyx_CyFunctionObject*)func)->func) == (PyCFunction) cfunc; + } else if (PyCFunction_Check(func)) { + return PyCFunction_GetFunction(func) == (PyCFunction) cfunc; + } + return 0; +} +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if ((PyObject*)Py_TYPE(func) == __pyx_mstate_global->__Pyx_CachedMethodType) { + int result; + PyObject *newFunc = PyObject_GetAttr(func, __pyx_mstate_global->__pyx_n_u_func); + if (unlikely(!newFunc)) { + PyErr_Clear(); // It's only an optimization, so don't throw an error + return 0; + } + result = __Pyx__IsSameCyOrCFunctionNoMethod(newFunc, cfunc); + Py_DECREF(newFunc); + return result; + } + return __Pyx__IsSameCyOrCFunctionNoMethod(func, cfunc); +} +#else +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if (PyMethod_Check(func)) { + func = PyMethod_GET_FUNCTION(func); + } + return __Pyx_CyOrPyCFunction_Check(func) && __Pyx_CyOrPyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +} +#endif +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#else + __Pyx_Py_XDECREF_SET( + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#endif +} +static PyObject * +__Pyx_CyFunction_get_doc_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_doc == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_doc = PyObject_GetAttrString(op->func, "__doc__"); + if (unlikely(!op->func_doc)) return NULL; +#else + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } +#endif + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + PyObject *result; + CYTHON_UNUSED_VAR(closure); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_doc_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_doc, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_name == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_name = PyObject_GetAttrString(op->func, "__name__"); +#else + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_name_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_name, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + Py_INCREF(op->func_qualname); + result = op->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_dict_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_dict_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL)) { + PyErr_SetString(PyExc_TypeError, + "function's dictionary may not be deleted"); + return -1; + } + if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "setting function's dictionary to a non-dict"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_dict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = __Pyx_PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = __Pyx_PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_defaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_kwdefaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value || value == Py_None) { + value = NULL; + } else if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) { + int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; + if (is_coroutine) { + PyObject *is_coroutine_value, *module, *fromlist, *marker = __pyx_mstate_global->__pyx_n_u_is_coroutine; + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); +#if CYTHON_ASSUME_SAFE_MACROS + PyList_SET_ITEM(fromlist, 0, marker); +#else + if (unlikely(PyList_SetItem(fromlist, 0, marker) < 0)) { + Py_DECREF(marker); + Py_DECREF(fromlist); + return NULL; + } +#endif + module = PyImport_ImportModuleLevelObject(__pyx_mstate_global->__pyx_n_u_asyncio_coroutines, NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + is_coroutine_value = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(is_coroutine_value)) { + return is_coroutine_value; + } +ignore: + PyErr_Clear(); + } + return __Pyx_PyBool_FromLong(is_coroutine); +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + result = __Pyx_CyFunction_get_is_coroutine_value(op); + if (unlikely(!result)) + return NULL; + __Pyx_BEGIN_CRITICAL_SECTION(op); + if (op->func_is_coroutine) { + Py_DECREF(result); + result = __Pyx_NewRef(op->func_is_coroutine); + } else { + op->func_is_coroutine = __Pyx_NewRef(result); + } + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static void __Pyx_CyFunction_raise_argument_count_error(__pyx_CyFunctionObject *func, const char* message, Py_ssize_t size) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + py_name, message, size); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + name, message, size); +#endif +} +static void __Pyx_CyFunction_raise_type_error(__pyx_CyFunctionObject *func, const char* message) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s", + py_name, message); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s", + name, message); +#endif +} +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject * +__Pyx_CyFunction_get_module(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_GetAttrString(op->func, "__module__"); +} +static int +__Pyx_CyFunction_set_module(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_SetAttrString(op->func, "__module__", value); +} +#endif +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {"func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, + {"func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {"__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {"func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API + {"__module__", (getter)__Pyx_CyFunction_get_module, (setter)__Pyx_CyFunction_set_module, 0, 0}, +#endif + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { +#if !CYTHON_COMPILING_IN_LIMITED_API + {"__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, +#endif + {"__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#if CYTHON_METH_FASTCALL +#if CYTHON_BACKPORT_VECTORCALL || CYTHON_COMPILING_IN_LIMITED_API + {"__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, +#else + {"__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {"__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(args); + __Pyx_BEGIN_CRITICAL_SECTION(m); + Py_INCREF(m->func_qualname); + result = m->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { +#if !CYTHON_COMPILING_IN_LIMITED_API + PyCFunctionObject *cf = (PyCFunctionObject*) op; +#endif + if (unlikely(op == NULL)) + return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + op->func = PyCFunction_NewEx(ml, (PyObject*)op, module); + if (unlikely(!op->func)) return NULL; +#endif + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; +#if !CYTHON_COMPILING_IN_LIMITED_API + cf->m_ml = ml; + cf->m_self = (PyObject *) op; +#endif + Py_XINCREF(closure); + op->func_closure = closure; +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_XINCREF(module); + cf->m_module = module; +#endif + op->func_dict = NULL; + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + op->func_classobj = NULL; +#else + ((PyCMethodObject*)op)->mm_class = NULL; +#endif + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func); +#else + Py_CLEAR(((PyCFunctionObject*)m)->m_module); +#endif + Py_CLEAR(m->func_dict); + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API +#if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); +#else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } +#endif +#endif + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); + Py_CLEAR(m->defaults); + return 0; +} +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + __Pyx_PyHeapTypeObject_GC_Del(m); +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + { + int e = __Pyx_call_type_traverse((PyObject*)m, 1, visit, arg); + if (e) return e; + } + Py_VISIT(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func); +#else + Py_VISIT(((PyCFunctionObject*)m)->m_module); +#endif + Py_VISIT(m->func_dict); + __Pyx_VISIT_CONST(m->func_name); + __Pyx_VISIT_CONST(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + __Pyx_VISIT_CONST(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); +#endif + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); + Py_VISIT(m->defaults); + return 0; +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ + PyObject *repr; + __Pyx_BEGIN_CRITICAL_SECTION(op); + repr = PyUnicode_FromFormat("", + op->func_qualname, (void *)op); + __Pyx_END_CRITICAL_SECTION(); + return repr; +} +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *f = ((__pyx_CyFunctionObject*)func)->func; + PyCFunction meth; + int flags; + meth = PyCFunction_GetFunction(f); + if (unlikely(!meth)) return NULL; + flags = PyCFunction_GetFlags(f); + if (unlikely(flags < 0)) return NULL; +#else + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + int flags = f->m_ml->ml_flags; +#endif + Py_ssize_t size; + switch (flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 0)) + return (*meth)(self, NULL); + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes no arguments", size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 1)) { + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = __Pyx_PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif + result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(arg0); + #endif + return result; + } + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes exactly one argument", size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + return NULL; + } + __Pyx_CyFunction_raise_type_error( + (__pyx_CyFunctionObject*)func, "takes no keyword arguments"); + return NULL; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *self, *result; +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)func)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)func)->m_self; +#endif + result = __Pyx_CyFunction_CallMethod(func, self, arg, kw); + return result; +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; +#if CYTHON_ASSUME_SAFE_SIZE + argc = PyTuple_GET_SIZE(args); +#else + argc = PyTuple_Size(args); + if (unlikely(argc < 0)) return NULL; +#endif + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + PyErr_Format(PyExc_TypeError, + "unbound method %.200S() needs an argument", + cyfunc->func_qualname); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; +} +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "needs an argument"); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(__Pyx_PyTuple_GET_SIZE(kwnames))) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "takes no keyword arguments"); + return -1; + } + return ret; +} +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 0)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes no arguments", nargs); + return NULL; + } + return meth(self, NULL); +} +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 1)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes exactly one argument", nargs); + return NULL; + } + return meth(self, args[0]); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))meth)(self, args, nargs, kwnames); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCMethod)(void(*)(void))meth)(self, cls, args, (size_t)nargs, kwnames); +} +#endif +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#if CYTHON_METH_FASTCALL +#if defined(Py_TPFLAGS_HAVE_VECTORCALL) + Py_TPFLAGS_HAVE_VECTORCALL | +#elif defined(_Py_TPFLAGS_HAVE_VECTORCALL) + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif +#endif // CYTHON_METH_FASTCALL +#if PY_VERSION_HEX >= 0x030A0000 + Py_TPFLAGS_IMMUTABLETYPE | +#endif + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, + __pyx_CyFunctionType_slots +}; +static int __pyx_CyFunction_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + mstate->__pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_CyFunctionType_spec, NULL); + if (unlikely(mstate->__pyx_CyFunctionType == NULL)) { + return -1; + } + return 0; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, PyTypeObject *defaults_type) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_CallObject((PyObject*)defaults_type, NULL); // _PyObject_New(defaults_type); + if (unlikely(!m->defaults)) + return NULL; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +/* CythonFunction */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyObject *op = __Pyx_CyFunction_Init( + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_mstate_global->__pyx_CyFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + PyObject_GC_Track(op); + } + return op; +} + +/* CLineInTraceback */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline; + PyObject *ptype, *pvalue, *ptraceback; +#if CYTHON_COMPILING_IN_CPYTHON + PyObject **cython_runtime_dict; +#endif + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); +#if CYTHON_COMPILING_IN_CPYTHON + cython_runtime_dict = _PyObject_GetDictPtr(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __Pyx_BEGIN_CRITICAL_SECTION(*cython_runtime_dict); + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, *cython_runtime_dict, + __Pyx_PyDict_GetItemStr(*cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback)) + Py_XINCREF(use_cline); + __Pyx_END_CRITICAL_SECTION(); + } else +#endif + { + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback); + if (use_cline_obj) { + use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; + Py_INCREF(use_cline); + Py_DECREF(use_cline_obj); + } else { + PyErr_Clear(); + use_cline = NULL; + } + } + if (!use_cline) { + c_line = 0; + (void) PyObject_SetAttr(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False); + } + else if (use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + (void) __pyx_cfilenm; + (void) __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + py_funcname = PyUnicode_FromString(funcname); + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* FormatTypeName */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static __Pyx_TypeName +__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) +{ + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u_); + } + goto done; +} +#endif + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if defined(HAVE_LONG_LONG) && !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPyVerify */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntFromPy */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + long val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (long) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + int val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030b0000 + return Py_Version & ~0xFFUL; +#else + static unsigned long __Pyx_cached_runtime_version = 0; + if (__Pyx_cached_runtime_version == 0) { + const char* rt_version = Py_GetVersion(); + unsigned long version = 0; + unsigned long factor = 0x01000000UL; + unsigned int digit = 0; + int i = 0; + while (factor) { + while ('0' <= rt_version[i] && rt_version[i] <= '9') { + digit = digit * 10 + (unsigned int) (rt_version[i] - '0'); + ++i; + } + version += factor * digit; + if (rt_version[i] != '.') + break; + digit = 0; + factor >>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } + return __Pyx_cached_runtime_version; +#endif +} + +/* CheckBinaryVersion */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } +} + +/* FunctionExport */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig) { + PyObject *d = 0; + PyObject *cobj = 0; + union { + void (*fp)(void); + void *p; + } tmp; + d = PyObject_GetAttrString(__pyx_m, "__pyx_capi__"); + if (!d) { + PyErr_Clear(); + d = PyDict_New(); + if (!d) + goto bad; + Py_INCREF(d); + if (PyModule_AddObject(__pyx_m, "__pyx_capi__", d) < 0) + goto bad; + } + tmp.fp = f; + cobj = PyCapsule_New(tmp.p, sig, 0); + if (!cobj) + goto bad; + if (PyDict_SetItemString(d, name, cobj) < 0) + goto bad; + Py_DECREF(cobj); + Py_DECREF(d); + return 0; +bad: + Py_XDECREF(cobj); + Py_XDECREF(d); + return -1; +} + +/* NewCodeObj */ +#if CYTHON_COMPILING_IN_LIMITED_API + static PyObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyObject *exception_table = NULL; + PyObject *types_module=NULL, *code_type=NULL, *result=NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + PyObject *version_info; + PyObject *py_minor_version = NULL; + #endif + long minor_version = 0; + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + minor_version = 11; + #else + if (!(version_info = PySys_GetObject("version_info"))) goto end; + if (!(py_minor_version = PySequence_GetItem(version_info, 1))) goto end; + minor_version = PyLong_AsLong(py_minor_version); + Py_DECREF(py_minor_version); + if (minor_version == -1 && PyErr_Occurred()) goto end; + #endif + if (!(types_module = PyImport_ImportModule("types"))) goto end; + if (!(code_type = PyObject_GetAttrString(types_module, "CodeType"))) goto end; + if (minor_version <= 7) { + (void)p; + result = PyObject_CallFunction(code_type, "iiiiiOOOOOOiOOO", a, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else if (minor_version <= 10) { + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOiOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else { + if (!(exception_table = PyBytes_FromStringAndSize(NULL, 0))) goto end; + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOOiOOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, name, fline, lnos, exception_table, fv, cell); + } + end: + Py_XDECREF(code_type); + Py_XDECREF(exception_table); + Py_XDECREF(types_module); + if (type) { + PyErr_Restore(type, value, traceback); + } + return result; + } +#elif PY_VERSION_HEX >= 0x030B0000 + static PyCodeObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyCodeObject *result; + result = + #if PY_VERSION_HEX >= 0x030C0000 + PyUnstable_Code_NewWithPosOnlyArgs + #else + PyCode_NewWithPosOnlyArgs + #endif + (a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, name, fline, lnos, __pyx_mstate_global->__pyx_empty_bytes); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030c00A1 + if (likely(result)) + result->_co_firsttraceable = 0; + #endif + return result; + } +#elif PY_VERSION_HEX >= 0x030800B2 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#else + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#endif +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + const char *line_table, + PyObject *tuple_dedup_map +) { + PyObject *code_obj = NULL, *varnames_tuple_dedup = NULL, *code_bytes = NULL, *line_table_bytes = NULL; + Py_ssize_t var_count = (Py_ssize_t) descr.nlocals; + PyObject *varnames_tuple = PyTuple_New(var_count); + if (unlikely(!varnames_tuple)) return NULL; + for (Py_ssize_t i=0; i < var_count; i++) { + Py_INCREF(varnames[i]); + if (__Pyx_PyTuple_SET_ITEM(varnames_tuple, i, varnames[i]) != (0)) goto done; + } + #if CYTHON_COMPILING_IN_LIMITED_API + varnames_tuple_dedup = PyDict_GetItem(tuple_dedup_map, varnames_tuple); + if (!varnames_tuple_dedup) { + if (unlikely(PyDict_SetItem(tuple_dedup_map, varnames_tuple, varnames_tuple) < 0)) goto done; + varnames_tuple_dedup = varnames_tuple; + } + #else + varnames_tuple_dedup = PyDict_SetDefault(tuple_dedup_map, varnames_tuple, varnames_tuple); + if (unlikely(!varnames_tuple_dedup)) goto done; + #endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(varnames_tuple_dedup); + #endif + if (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table != NULL + && !CYTHON_COMPILING_IN_GRAAL) { + line_table_bytes = PyBytes_FromStringAndSize(line_table, descr.line_table_length); + if (unlikely(!line_table_bytes)) goto done; + Py_ssize_t code_len = (descr.line_table_length * 2 + 4) & ~3; + code_bytes = PyBytes_FromStringAndSize(NULL, code_len); + if (unlikely(!code_bytes)) goto done; + char* c_code_bytes = PyBytes_AsString(code_bytes); + if (unlikely(!c_code_bytes)) goto done; + memset(c_code_bytes, 0, (size_t) code_len); + } + code_obj = (PyObject*) __Pyx__PyCode_New( + (int) descr.argcount, + (int) descr.num_posonly_args, + (int) descr.num_kwonly_args, + (int) descr.nlocals, + 0, + (int) descr.flags, + code_bytes ? code_bytes : __pyx_mstate_global->__pyx_empty_bytes, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + varnames_tuple_dedup, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + filename, + funcname, + (int) descr.first_line, + (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table_bytes) ? line_table_bytes : __pyx_mstate_global->__pyx_empty_bytes + ); +done: + Py_XDECREF(code_bytes); + Py_XDECREF(line_table_bytes); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(varnames_tuple_dedup); + #endif + Py_DECREF(varnames_tuple); + return code_obj; +} + +/* InitStrings */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names) { + while (t->s) { + PyObject *str; + if (t->is_unicode) { + if (t->intern) { + str = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + str = PyUnicode_Decode(t->s, t->n - 1, encoding_names[t->encoding], NULL); + } else { + str = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + str = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + if (!str) + return -1; + *target = str; + if (PyObject_Hash(str) == -1) + return -1; + ++t; + ++target; + } + return 0; +} + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} +#if CYTHON_USE_PYLONG_INTERNALS +static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value) { + if (unlikely(!__Pyx_PyLong_IsCompact(x))) + return 0; + Py_ssize_t value = __Pyx_PyLong_CompactValue(x); + if ((sizeof(long) < sizeof(Py_ssize_t)) && unlikely(value != (long) value)) + return 0; + *return_value = (long) value; + return 1; +} +#endif + + +/* MultiPhaseInitModuleState */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 +#else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 +#endif +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS +#error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" +#endif +#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#define __Pyx_ModuleStateLookup_Lock() +#define __Pyx_ModuleStateLookup_Unlock() +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 +static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; +#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#include +static std::mutex __Pyx_ModuleStateLookup_mutex; +#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() +#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) +#include +static mtx_t __Pyx_ModuleStateLookup_mutex; +static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; +static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); +} +#define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(HAVE_PTHREAD_H) +#include +static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; +#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(_WIN32) +#include // synchapi.h on its own doesn't work +static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; +#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#else +#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." +#endif +typedef struct { + int64_t id; + PyObject *module; +} __Pyx_InterpreterIdAndModule; +typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; +} __Pyx_ModuleStateLookupData; +#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; +#else +static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; +#endif +static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; +} +static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; +#else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); +#endif + return result; + } +} +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); +} +#else +#define __Pyx_ModuleStateLookup_wait_until_no_readers() +#endif +static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; +} +static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; +} +static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); +#else + __Pyx_ModuleStateLookup_data = new_data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return result; +} +static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; +#endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); +#else + __Pyx_ModuleStateLookup_data = data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; +} +#endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cydriver.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cydriver.pxd new file mode 100644 index 0000000000000000000000000000000000000000..8dfd6f87f05364a6836ee91ce11d0732e0944412 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cydriver.pxd @@ -0,0 +1,951 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +# This code was automatically generated with version 12.9.0. Do not modify it directly. +from cuda.bindings.cydriver cimport * + +cdef CUresult _cuGetErrorString(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGetErrorName(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuInit(unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDriverGetVersion(int* driverVersion) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGet(CUdevice* device, int ordinal) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetCount(int* count) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetName(char* name, int length, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetUuid(CUuuid* uuid, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetUuid_v2(CUuuid* uuid, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetLuid(char* luid, unsigned int* deviceNodeMask, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceTotalMem_v2(size_t* numbytes, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, CUarray_format pformat, unsigned numChannels, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetAttribute(int* pi, CUdevice_attribute attrib, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, CUdevice dev, int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceSetMemPool(CUdevice dev, CUmemoryPool pool) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetMemPool(CUmemoryPool* pool, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetDefaultMemPool(CUmemoryPool* pool_out, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetExecAffinitySupport(int* pi, CUexecAffinityType typename, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuFlushGPUDirectRDMAWrites(CUflushGPUDirectRDMAWritesTarget target, CUflushGPUDirectRDMAWritesScope scope) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetProperties(CUdevprop* prop, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceComputeCapability(int* major, int* minor, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDevicePrimaryCtxRetain(CUcontext* pctx, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDevicePrimaryCtxRelease_v2(CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDevicePrimaryCtxSetFlags_v2(CUdevice dev, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDevicePrimaryCtxGetState(CUdevice dev, unsigned int* flags, int* active) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDevicePrimaryCtxReset_v2(CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxCreate_v2(CUcontext* pctx, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxCreate_v3(CUcontext* pctx, CUexecAffinityParam* paramsArray, int numParams, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxCreate_v4(CUcontext* pctx, CUctxCreateParams* ctxCreateParams, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxDestroy_v2(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxPushCurrent_v2(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxPopCurrent_v2(CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxSetCurrent(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxGetCurrent(CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxGetDevice(CUdevice* device) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxGetFlags(unsigned int* flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxSetFlags(unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxGetId(CUcontext ctx, unsigned long long* ctxId) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxSynchronize() except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxSetLimit(CUlimit limit, size_t value) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxGetLimit(size_t* pvalue, CUlimit limit) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxGetCacheConfig(CUfunc_cache* pconfig) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxSetCacheConfig(CUfunc_cache config) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxGetApiVersion(CUcontext ctx, unsigned int* version) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxResetPersistingL2Cache() except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxGetExecAffinity(CUexecAffinityParam* pExecAffinity, CUexecAffinityType typename) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxRecordEvent(CUcontext hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxWaitEvent(CUcontext hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxAttach(CUcontext* pctx, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxDetach(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxGetSharedMemConfig(CUsharedconfig* pConfig) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxSetSharedMemConfig(CUsharedconfig config) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuModuleLoad(CUmodule* module, const char* fname) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuModuleLoadData(CUmodule* module, const void* image) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuModuleLoadDataEx(CUmodule* module, const void* image, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuModuleLoadFatBinary(CUmodule* module, const void* fatCubin) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuModuleUnload(CUmodule hmod) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuModuleGetLoadingMode(CUmoduleLoadingMode* mode) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuModuleGetFunction(CUfunction* hfunc, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuModuleGetFunctionCount(unsigned int* count, CUmodule mod) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuModuleEnumerateFunctions(CUfunction* functions, unsigned int numFunctions, CUmodule mod) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuModuleGetGlobal_v2(CUdeviceptr* dptr, size_t* numbytes, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLinkCreate_v2(unsigned int numOptions, CUjit_option* options, void** optionValues, CUlinkState* stateOut) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLinkAddData_v2(CUlinkState state, CUjitInputType typename, void* data, size_t size, const char* name, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLinkAddFile_v2(CUlinkState state, CUjitInputType typename, const char* path, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLinkComplete(CUlinkState state, void** cubinOut, size_t* sizeOut) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLinkDestroy(CUlinkState state) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuModuleGetTexRef(CUtexref* pTexRef, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuModuleGetSurfRef(CUsurfref* pSurfRef, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLibraryLoadData(CUlibrary* library, const void* code, CUjit_option* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, CUlibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLibraryLoadFromFile(CUlibrary* library, const char* fileName, CUjit_option* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, CUlibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLibraryUnload(CUlibrary library) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLibraryGetKernel(CUkernel* pKernel, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLibraryGetKernelCount(unsigned int* count, CUlibrary lib) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLibraryEnumerateKernels(CUkernel* kernels, unsigned int numKernels, CUlibrary lib) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLibraryGetModule(CUmodule* pMod, CUlibrary library) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuKernelGetFunction(CUfunction* pFunc, CUkernel kernel) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuKernelGetLibrary(CUlibrary* pLib, CUkernel kernel) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLibraryGetGlobal(CUdeviceptr* dptr, size_t* numbytes, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLibraryGetManaged(CUdeviceptr* dptr, size_t* numbytes, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLibraryGetUnifiedFunction(void** fptr, CUlibrary library, const char* symbol) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuKernelGetAttribute(int* pi, CUfunction_attribute attrib, CUkernel kernel, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuKernelSetAttribute(CUfunction_attribute attrib, int val, CUkernel kernel, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuKernelSetCacheConfig(CUkernel kernel, CUfunc_cache config, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuKernelGetName(const char** name, CUkernel hfunc) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuKernelGetParamInfo(CUkernel kernel, size_t paramIndex, size_t* paramOffset, size_t* paramSize) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemGetInfo_v2(size_t* free, size_t* total) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemAlloc_v2(CUdeviceptr* dptr, size_t bytesize) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemAllocPitch_v2(CUdeviceptr* dptr, size_t* pPitch, size_t WidthInBytes, size_t Height, unsigned int ElementSizeBytes) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemFree_v2(CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemGetAddressRange_v2(CUdeviceptr* pbase, size_t* psize, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemAllocHost_v2(void** pp, size_t bytesize) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemFreeHost(void* p) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemHostAlloc(void** pp, size_t bytesize, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemHostGetDevicePointer_v2(CUdeviceptr* pdptr, void* p, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemHostGetFlags(unsigned int* pFlags, void* p) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemAllocManaged(CUdeviceptr* dptr, size_t bytesize, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceRegisterAsyncNotification(CUdevice device, CUasyncCallback callbackFunc, void* userData, CUasyncCallbackHandle* callback) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceUnregisterAsyncNotification(CUdevice device, CUasyncCallbackHandle callback) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetByPCIBusId(CUdevice* dev, const char* pciBusId) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetPCIBusId(char* pciBusId, int length, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuIpcGetEventHandle(CUipcEventHandle* pHandle, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuIpcOpenEventHandle(CUevent* phEvent, CUipcEventHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuIpcGetMemHandle(CUipcMemHandle* pHandle, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuIpcOpenMemHandle_v2(CUdeviceptr* pdptr, CUipcMemHandle handle, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuIpcCloseMemHandle(CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemHostRegister_v2(void* p, size_t bytesize, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemHostUnregister(void* p) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpy(CUdeviceptr dst, CUdeviceptr src, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyPeer(CUdeviceptr dstDevice, CUcontext dstContext, CUdeviceptr srcDevice, CUcontext srcContext, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyHtoD_v2(CUdeviceptr dstDevice, const void* srcHost, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyDtoH_v2(void* dstHost, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyDtoD_v2(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyDtoA_v2(CUarray dstArray, size_t dstOffset, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyAtoD_v2(CUdeviceptr dstDevice, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyHtoA_v2(CUarray dstArray, size_t dstOffset, const void* srcHost, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyAtoH_v2(void* dstHost, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyAtoA_v2(CUarray dstArray, size_t dstOffset, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpy2D_v2(const CUDA_MEMCPY2D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpy2DUnaligned_v2(const CUDA_MEMCPY2D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpy3D_v2(const CUDA_MEMCPY3D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpy3DPeer(const CUDA_MEMCPY3D_PEER* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyAsync(CUdeviceptr dst, CUdeviceptr src, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyPeerAsync(CUdeviceptr dstDevice, CUcontext dstContext, CUdeviceptr srcDevice, CUcontext srcContext, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyHtoDAsync_v2(CUdeviceptr dstDevice, const void* srcHost, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyDtoHAsync_v2(void* dstHost, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyDtoDAsync_v2(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyHtoAAsync_v2(CUarray dstArray, size_t dstOffset, const void* srcHost, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyAtoHAsync_v2(void* dstHost, CUarray srcArray, size_t srcOffset, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpy2DAsync_v2(const CUDA_MEMCPY2D* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpy3DAsync_v2(const CUDA_MEMCPY3D* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpy3DPeerAsync(const CUDA_MEMCPY3D_PEER* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpyBatchAsync(CUdeviceptr* dsts, CUdeviceptr* srcs, size_t* sizes, size_t count, CUmemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemcpy3DBatchAsync(size_t numOps, CUDA_MEMCPY3D_BATCH_OP* opList, size_t* failIdx, unsigned long long flags, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemsetD8_v2(CUdeviceptr dstDevice, unsigned char uc, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemsetD16_v2(CUdeviceptr dstDevice, unsigned short us, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemsetD32_v2(CUdeviceptr dstDevice, unsigned int ui, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemsetD2D8_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned char uc, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemsetD2D16_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned short us, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemsetD2D32_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned int ui, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemsetD8Async(CUdeviceptr dstDevice, unsigned char uc, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemsetD16Async(CUdeviceptr dstDevice, unsigned short us, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemsetD32Async(CUdeviceptr dstDevice, unsigned int ui, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemsetD2D8Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned char uc, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemsetD2D16Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned short us, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemsetD2D32Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned int ui, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuArrayCreate_v2(CUarray* pHandle, const CUDA_ARRAY_DESCRIPTOR* pAllocateArray) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuArrayGetDescriptor_v2(CUDA_ARRAY_DESCRIPTOR* pArrayDescriptor, CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES* sparseProperties, CUarray array) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMipmappedArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES* sparseProperties, CUmipmappedArray mipmap) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS* memoryRequirements, CUarray array, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMipmappedArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS* memoryRequirements, CUmipmappedArray mipmap, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuArrayGetPlane(CUarray* pPlaneArray, CUarray hArray, unsigned int planeIdx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuArrayDestroy(CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuArray3DCreate_v2(CUarray* pHandle, const CUDA_ARRAY3D_DESCRIPTOR* pAllocateArray) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuArray3DGetDescriptor_v2(CUDA_ARRAY3D_DESCRIPTOR* pArrayDescriptor, CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMipmappedArrayCreate(CUmipmappedArray* pHandle, const CUDA_ARRAY3D_DESCRIPTOR* pMipmappedArrayDesc, unsigned int numMipmapLevels) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMipmappedArrayGetLevel(CUarray* pLevelArray, CUmipmappedArray hMipmappedArray, unsigned int level) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMipmappedArrayDestroy(CUmipmappedArray hMipmappedArray) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemGetHandleForAddressRange(void* handle, CUdeviceptr dptr, size_t size, CUmemRangeHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemBatchDecompressAsync(CUmemDecompressParams* paramsArray, size_t count, unsigned int flags, size_t* errorIndex, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemAddressReserve(CUdeviceptr* ptr, size_t size, size_t alignment, CUdeviceptr addr, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemAddressFree(CUdeviceptr ptr, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemCreate(CUmemGenericAllocationHandle* handle, size_t size, const CUmemAllocationProp* prop, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemRelease(CUmemGenericAllocationHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemMap(CUdeviceptr ptr, size_t size, size_t offset, CUmemGenericAllocationHandle handle, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemMapArrayAsync(CUarrayMapInfo* mapInfoList, unsigned int count, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemUnmap(CUdeviceptr ptr, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemSetAccess(CUdeviceptr ptr, size_t size, const CUmemAccessDesc* desc, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemGetAccess(unsigned long long* flags, const CUmemLocation* location, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemExportToShareableHandle(void* shareableHandle, CUmemGenericAllocationHandle handle, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemImportFromShareableHandle(CUmemGenericAllocationHandle* handle, void* osHandle, CUmemAllocationHandleType shHandleType) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemGetAllocationGranularity(size_t* granularity, const CUmemAllocationProp* prop, CUmemAllocationGranularity_flags option) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemGetAllocationPropertiesFromHandle(CUmemAllocationProp* prop, CUmemGenericAllocationHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemRetainAllocationHandle(CUmemGenericAllocationHandle* handle, void* addr) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemFreeAsync(CUdeviceptr dptr, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemAllocAsync(CUdeviceptr* dptr, size_t bytesize, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemPoolTrimTo(CUmemoryPool pool, size_t minBytesToKeep) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemPoolSetAttribute(CUmemoryPool pool, CUmemPool_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemPoolGetAttribute(CUmemoryPool pool, CUmemPool_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemPoolSetAccess(CUmemoryPool pool, const CUmemAccessDesc* map, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemPoolGetAccess(CUmemAccess_flags* flags, CUmemoryPool memPool, CUmemLocation* location) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemPoolCreate(CUmemoryPool* pool, const CUmemPoolProps* poolProps) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemPoolDestroy(CUmemoryPool pool) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemAllocFromPoolAsync(CUdeviceptr* dptr, size_t bytesize, CUmemoryPool pool, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemPoolExportToShareableHandle(void* handle_out, CUmemoryPool pool, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemPoolImportFromShareableHandle(CUmemoryPool* pool_out, void* handle, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemPoolExportPointer(CUmemPoolPtrExportData* shareData_out, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemPoolImportPointer(CUdeviceptr* ptr_out, CUmemoryPool pool, CUmemPoolPtrExportData* shareData) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMulticastCreate(CUmemGenericAllocationHandle* mcHandle, const CUmulticastObjectProp* prop) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMulticastAddDevice(CUmemGenericAllocationHandle mcHandle, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMulticastBindMem(CUmemGenericAllocationHandle mcHandle, size_t mcOffset, CUmemGenericAllocationHandle memHandle, size_t memOffset, size_t size, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMulticastBindAddr(CUmemGenericAllocationHandle mcHandle, size_t mcOffset, CUdeviceptr memptr, size_t size, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMulticastUnbind(CUmemGenericAllocationHandle mcHandle, CUdevice dev, size_t mcOffset, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMulticastGetGranularity(size_t* granularity, const CUmulticastObjectProp* prop, CUmulticastGranularity_flags option) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuPointerGetAttribute(void* data, CUpointer_attribute attribute, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUdevice dstDevice, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemPrefetchAsync_v2(CUdeviceptr devPtr, size_t count, CUmemLocation location, unsigned int flags, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemAdvise(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemAdvise_v2(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUmemLocation location) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemRangeGetAttribute(void* data, size_t dataSize, CUmem_range_attribute attribute, CUdeviceptr devPtr, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuMemRangeGetAttributes(void** data, size_t* dataSizes, CUmem_range_attribute* attributes, size_t numAttributes, CUdeviceptr devPtr, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuPointerSetAttribute(const void* value, CUpointer_attribute attribute, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuPointerGetAttributes(unsigned int numAttributes, CUpointer_attribute* attributes, void** data, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamCreate(CUstream* phStream, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamCreateWithPriority(CUstream* phStream, unsigned int flags, int priority) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamGetPriority(CUstream hStream, int* priority) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamGetDevice(CUstream hStream, CUdevice* device) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamGetFlags(CUstream hStream, unsigned int* flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamGetId(CUstream hStream, unsigned long long* streamId) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamGetCtx(CUstream hStream, CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamGetCtx_v2(CUstream hStream, CUcontext* pCtx, CUgreenCtx* pGreenCtx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamWaitEvent(CUstream hStream, CUevent hEvent, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamAddCallback(CUstream hStream, CUstreamCallback callback, void* userData, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamBeginCapture_v2(CUstream hStream, CUstreamCaptureMode mode) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamBeginCaptureToGraph(CUstream hStream, CUgraph hGraph, const CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, CUstreamCaptureMode mode) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuThreadExchangeStreamCaptureMode(CUstreamCaptureMode* mode) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamEndCapture(CUstream hStream, CUgraph* phGraph) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamIsCapturing(CUstream hStream, CUstreamCaptureStatus* captureStatus) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamGetCaptureInfo_v2(CUstream hStream, CUstreamCaptureStatus* captureStatus_out, cuuint64_t* id_out, CUgraph* graph_out, const CUgraphNode** dependencies_out, size_t* numDependencies_out) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamGetCaptureInfo_v3(CUstream hStream, CUstreamCaptureStatus* captureStatus_out, cuuint64_t* id_out, CUgraph* graph_out, const CUgraphNode** dependencies_out, const CUgraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamUpdateCaptureDependencies(CUstream hStream, CUgraphNode* dependencies, size_t numDependencies, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamUpdateCaptureDependencies_v2(CUstream hStream, CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamAttachMemAsync(CUstream hStream, CUdeviceptr dptr, size_t length, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamQuery(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamSynchronize(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamDestroy_v2(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamCopyAttributes(CUstream dst, CUstream src) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamGetAttribute(CUstream hStream, CUstreamAttrID attr, CUstreamAttrValue* value_out) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamSetAttribute(CUstream hStream, CUstreamAttrID attr, const CUstreamAttrValue* value) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEventCreate(CUevent* phEvent, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEventRecord(CUevent hEvent, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEventRecordWithFlags(CUevent hEvent, CUstream hStream, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEventQuery(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEventSynchronize(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEventDestroy_v2(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEventElapsedTime(float* pMilliseconds, CUevent hStart, CUevent hEnd) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEventElapsedTime_v2(float* pMilliseconds, CUevent hStart, CUevent hEnd) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuImportExternalMemory(CUexternalMemory* extMem_out, const CUDA_EXTERNAL_MEMORY_HANDLE_DESC* memHandleDesc) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuExternalMemoryGetMappedBuffer(CUdeviceptr* devPtr, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_BUFFER_DESC* bufferDesc) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuExternalMemoryGetMappedMipmappedArray(CUmipmappedArray* mipmap, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC* mipmapDesc) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDestroyExternalMemory(CUexternalMemory extMem) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuImportExternalSemaphore(CUexternalSemaphore* extSem_out, const CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC* semHandleDesc) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuSignalExternalSemaphoresAsync(const CUexternalSemaphore* extSemArray, const CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS* paramsArray, unsigned int numExtSems, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuWaitExternalSemaphoresAsync(const CUexternalSemaphore* extSemArray, const CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS* paramsArray, unsigned int numExtSems, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDestroyExternalSemaphore(CUexternalSemaphore extSem) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamWaitValue32_v2(CUstream stream, CUdeviceptr addr, cuuint32_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamWaitValue64_v2(CUstream stream, CUdeviceptr addr, cuuint64_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamWriteValue32_v2(CUstream stream, CUdeviceptr addr, cuuint32_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamWriteValue64_v2(CUstream stream, CUdeviceptr addr, cuuint64_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamBatchMemOp_v2(CUstream stream, unsigned int count, CUstreamBatchMemOpParams* paramArray, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuFuncGetAttribute(int* pi, CUfunction_attribute attrib, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuFuncSetAttribute(CUfunction hfunc, CUfunction_attribute attrib, int value) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuFuncSetCacheConfig(CUfunction hfunc, CUfunc_cache config) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuFuncGetModule(CUmodule* hmod, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuFuncGetName(const char** name, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuFuncGetParamInfo(CUfunction func, size_t paramIndex, size_t* paramOffset, size_t* paramSize) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuFuncIsLoaded(CUfunctionLoadingState* state, CUfunction function) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuFuncLoad(CUfunction function) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLaunchKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void** kernelParams, void** extra) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLaunchKernelEx(const CUlaunchConfig* config, CUfunction f, void** kernelParams, void** extra) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLaunchCooperativeKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void** kernelParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLaunchCooperativeKernelMultiDevice(CUDA_LAUNCH_PARAMS* launchParamsList, unsigned int numDevices, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLaunchHostFunc(CUstream hStream, CUhostFn fn, void* userData) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuFuncSetBlockShape(CUfunction hfunc, int x, int y, int z) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuFuncSetSharedSize(CUfunction hfunc, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuParamSetSize(CUfunction hfunc, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuParamSeti(CUfunction hfunc, int offset, unsigned int value) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuParamSetf(CUfunction hfunc, int offset, float value) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuParamSetv(CUfunction hfunc, int offset, void* ptr, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLaunch(CUfunction f) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLaunchGrid(CUfunction f, int grid_width, int grid_height) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLaunchGridAsync(CUfunction f, int grid_width, int grid_height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuParamSetTexRef(CUfunction hfunc, int texunit, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuFuncSetSharedMemConfig(CUfunction hfunc, CUsharedconfig config) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphCreate(CUgraph* phGraph, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddKernelNode_v2(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphKernelNodeGetParams_v2(CUgraphNode hNode, CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphKernelNodeSetParams_v2(CUgraphNode hNode, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddMemcpyNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_MEMCPY3D* copyParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphMemcpyNodeGetParams(CUgraphNode hNode, CUDA_MEMCPY3D* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphMemcpyNodeSetParams(CUgraphNode hNode, const CUDA_MEMCPY3D* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddMemsetNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_MEMSET_NODE_PARAMS* memsetParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphMemsetNodeGetParams(CUgraphNode hNode, CUDA_MEMSET_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphMemsetNodeSetParams(CUgraphNode hNode, const CUDA_MEMSET_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddHostNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphHostNodeGetParams(CUgraphNode hNode, CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphHostNodeSetParams(CUgraphNode hNode, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddChildGraphNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUgraph childGraph) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphChildGraphNodeGetGraph(CUgraphNode hNode, CUgraph* phGraph) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddEmptyNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddEventRecordNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphEventRecordNodeGetEvent(CUgraphNode hNode, CUevent* event_out) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphEventRecordNodeSetEvent(CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddEventWaitNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphEventWaitNodeGetEvent(CUgraphNode hNode, CUevent* event_out) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphEventWaitNodeSetEvent(CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddExternalSemaphoresSignalNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExternalSemaphoresSignalNodeGetParams(CUgraphNode hNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExternalSemaphoresSignalNodeSetParams(CUgraphNode hNode, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddExternalSemaphoresWaitNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExternalSemaphoresWaitNodeGetParams(CUgraphNode hNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExternalSemaphoresWaitNodeSetParams(CUgraphNode hNode, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddBatchMemOpNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphBatchMemOpNodeGetParams(CUgraphNode hNode, CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams_out) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphBatchMemOpNodeSetParams(CUgraphNode hNode, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecBatchMemOpNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddMemAllocNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUDA_MEM_ALLOC_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphMemAllocNodeGetParams(CUgraphNode hNode, CUDA_MEM_ALLOC_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddMemFreeNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphMemFreeNodeGetParams(CUgraphNode hNode, CUdeviceptr* dptr_out) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGraphMemTrim(CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetGraphMemAttribute(CUdevice device, CUgraphMem_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceSetGraphMemAttribute(CUdevice device, CUgraphMem_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphClone(CUgraph* phGraphClone, CUgraph originalGraph) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphNodeFindInClone(CUgraphNode* phNode, CUgraphNode hOriginalNode, CUgraph hClonedGraph) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphNodeGetType(CUgraphNode hNode, CUgraphNodeType* typename) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphGetNodes(CUgraph hGraph, CUgraphNode* nodes, size_t* numNodes) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphGetRootNodes(CUgraph hGraph, CUgraphNode* rootNodes, size_t* numRootNodes) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphGetEdges(CUgraph hGraph, CUgraphNode* from_, CUgraphNode* to, size_t* numEdges) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphGetEdges_v2(CUgraph hGraph, CUgraphNode* from_, CUgraphNode* to, CUgraphEdgeData* edgeData, size_t* numEdges) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphNodeGetDependencies(CUgraphNode hNode, CUgraphNode* dependencies, size_t* numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphNodeGetDependencies_v2(CUgraphNode hNode, CUgraphNode* dependencies, CUgraphEdgeData* edgeData, size_t* numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphNodeGetDependentNodes(CUgraphNode hNode, CUgraphNode* dependentNodes, size_t* numDependentNodes) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphNodeGetDependentNodes_v2(CUgraphNode hNode, CUgraphNode* dependentNodes, CUgraphEdgeData* edgeData, size_t* numDependentNodes) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddDependencies(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddDependencies_v2(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, const CUgraphEdgeData* edgeData, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphRemoveDependencies(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphRemoveDependencies_v2(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, const CUgraphEdgeData* edgeData, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphDestroyNode(CUgraphNode hNode) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphInstantiateWithFlags(CUgraphExec* phGraphExec, CUgraph hGraph, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphInstantiateWithParams(CUgraphExec* phGraphExec, CUgraph hGraph, CUDA_GRAPH_INSTANTIATE_PARAMS* instantiateParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecGetFlags(CUgraphExec hGraphExec, cuuint64_t* flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecKernelNodeSetParams_v2(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecMemcpyNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_MEMCPY3D* copyParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecMemsetNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_MEMSET_NODE_PARAMS* memsetParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecHostNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecChildGraphNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraph childGraph) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecEventRecordNodeSetEvent(CUgraphExec hGraphExec, CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecEventWaitNodeSetEvent(CUgraphExec hGraphExec, CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecExternalSemaphoresSignalNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecExternalSemaphoresWaitNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphNodeSetEnabled(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int isEnabled) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphNodeGetEnabled(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int* isEnabled) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphUpload(CUgraphExec hGraphExec, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphLaunch(CUgraphExec hGraphExec, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecDestroy(CUgraphExec hGraphExec) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphDestroy(CUgraph hGraph) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecUpdate_v2(CUgraphExec hGraphExec, CUgraph hGraph, CUgraphExecUpdateResultInfo* resultInfo) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphKernelNodeCopyAttributes(CUgraphNode dst, CUgraphNode src) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphKernelNodeGetAttribute(CUgraphNode hNode, CUkernelNodeAttrID attr, CUkernelNodeAttrValue* value_out) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphKernelNodeSetAttribute(CUgraphNode hNode, CUkernelNodeAttrID attr, const CUkernelNodeAttrValue* value) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphDebugDotPrint(CUgraph hGraph, const char* path, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuUserObjectCreate(CUuserObject* object_out, void* ptr, CUhostFn destroy, unsigned int initialRefcount, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuUserObjectRetain(CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuUserObjectRelease(CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphRetainUserObject(CUgraph graph, CUuserObject object, unsigned int count, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphReleaseUserObject(CUgraph graph, CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphAddNode_v2(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphNodeSetParams(CUgraphNode hNode, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphExecNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphConditionalHandleCreate(CUgraphConditionalHandle* pHandle_out, CUgraph hGraph, CUcontext ctx, unsigned int defaultLaunchValue, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuOccupancyMaxPotentialBlockSize(int* minGridSize, int* blockSize, CUfunction func, CUoccupancyB2DSize blockSizeToDynamicSMemSize, size_t dynamicSMemSize, int blockSizeLimit) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuOccupancyMaxPotentialBlockSizeWithFlags(int* minGridSize, int* blockSize, CUfunction func, CUoccupancyB2DSize blockSizeToDynamicSMemSize, size_t dynamicSMemSize, int blockSizeLimit, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, CUfunction func, int numBlocks, int blockSize) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuOccupancyMaxPotentialClusterSize(int* clusterSize, CUfunction func, const CUlaunchConfig* config) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuOccupancyMaxActiveClusters(int* numClusters, CUfunction func, const CUlaunchConfig* config) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefSetArray(CUtexref hTexRef, CUarray hArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefSetMipmappedArray(CUtexref hTexRef, CUmipmappedArray hMipmappedArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefSetAddress_v2(size_t* ByteOffset, CUtexref hTexRef, CUdeviceptr dptr, size_t numbytes) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefSetAddress2D_v3(CUtexref hTexRef, const CUDA_ARRAY_DESCRIPTOR* desc, CUdeviceptr dptr, size_t Pitch) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefSetFormat(CUtexref hTexRef, CUarray_format fmt, int NumPackedComponents) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefSetAddressMode(CUtexref hTexRef, int dim, CUaddress_mode am) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefSetFilterMode(CUtexref hTexRef, CUfilter_mode fm) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefSetMipmapFilterMode(CUtexref hTexRef, CUfilter_mode fm) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefSetMipmapLevelBias(CUtexref hTexRef, float bias) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefSetMipmapLevelClamp(CUtexref hTexRef, float minMipmapLevelClamp, float maxMipmapLevelClamp) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefSetMaxAnisotropy(CUtexref hTexRef, unsigned int maxAniso) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefSetBorderColor(CUtexref hTexRef, float* pBorderColor) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefSetFlags(CUtexref hTexRef, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefGetAddress_v2(CUdeviceptr* pdptr, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefGetArray(CUarray* phArray, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefGetMipmappedArray(CUmipmappedArray* phMipmappedArray, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefGetAddressMode(CUaddress_mode* pam, CUtexref hTexRef, int dim) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefGetFilterMode(CUfilter_mode* pfm, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefGetFormat(CUarray_format* pFormat, int* pNumChannels, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefGetMipmapFilterMode(CUfilter_mode* pfm, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefGetMipmapLevelBias(float* pbias, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefGetMipmapLevelClamp(float* pminMipmapLevelClamp, float* pmaxMipmapLevelClamp, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefGetMaxAnisotropy(int* pmaxAniso, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefGetBorderColor(float* pBorderColor, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefGetFlags(unsigned int* pFlags, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefCreate(CUtexref* pTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexRefDestroy(CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuSurfRefSetArray(CUsurfref hSurfRef, CUarray hArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuSurfRefGetArray(CUarray* phArray, CUsurfref hSurfRef) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexObjectCreate(CUtexObject* pTexObject, const CUDA_RESOURCE_DESC* pResDesc, const CUDA_TEXTURE_DESC* pTexDesc, const CUDA_RESOURCE_VIEW_DESC* pResViewDesc) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexObjectDestroy(CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexObjectGetResourceDesc(CUDA_RESOURCE_DESC* pResDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexObjectGetTextureDesc(CUDA_TEXTURE_DESC* pTexDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTexObjectGetResourceViewDesc(CUDA_RESOURCE_VIEW_DESC* pResViewDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuSurfObjectCreate(CUsurfObject* pSurfObject, const CUDA_RESOURCE_DESC* pResDesc) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuSurfObjectDestroy(CUsurfObject surfObject) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuSurfObjectGetResourceDesc(CUDA_RESOURCE_DESC* pResDesc, CUsurfObject surfObject) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTensorMapEncodeTiled(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, const cuuint32_t* boxDim, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTensorMapEncodeIm2col(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, const int* pixelBoxLowerCorner, const int* pixelBoxUpperCorner, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTensorMapEncodeIm2colWide(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, int pixelBoxLowerCornerWidth, int pixelBoxUpperCornerWidth, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapIm2ColWideMode mode, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuTensorMapReplaceAddress(CUtensorMap* tensorMap, void* globalAddress) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceCanAccessPeer(int* canAccessPeer, CUdevice dev, CUdevice peerDev) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxEnablePeerAccess(CUcontext peerContext, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxDisablePeerAccess(CUcontext peerContext) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetP2PAttribute(int* value, CUdevice_P2PAttribute attrib, CUdevice srcDevice, CUdevice dstDevice) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphicsUnregisterResource(CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphicsSubResourceGetMappedArray(CUarray* pArray, CUgraphicsResource resource, unsigned int arrayIndex, unsigned int mipLevel) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphicsResourceGetMappedMipmappedArray(CUmipmappedArray* pMipmappedArray, CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphicsResourceGetMappedPointer_v2(CUdeviceptr* pDevPtr, size_t* pSize, CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphicsResourceSetMapFlags_v2(CUgraphicsResource resource, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphicsMapResources(unsigned int count, CUgraphicsResource* resources, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphicsUnmapResources(unsigned int count, CUgraphicsResource* resources, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGetProcAddress_v2(const char* symbol, void** pfn, int cudaVersion, cuuint64_t flags, CUdriverProcAddressQueryResult* symbolStatus) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCoredumpGetAttribute(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCoredumpGetAttributeGlobal(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCoredumpSetAttribute(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCoredumpSetAttributeGlobal(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGetExportTable(const void** ppExportTable, const CUuuid* pExportTableId) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGreenCtxCreate(CUgreenCtx* phCtx, CUdevResourceDesc desc, CUdevice dev, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGreenCtxDestroy(CUgreenCtx hCtx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxFromGreenCtx(CUcontext* pContext, CUgreenCtx hCtx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDeviceGetDevResource(CUdevice device, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCtxGetDevResource(CUcontext hCtx, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGreenCtxGetDevResource(CUgreenCtx hCtx, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDevSmResourceSplitByCount(CUdevResource* result, unsigned int* nbGroups, const CUdevResource* input, CUdevResource* remaining, unsigned int useFlags, unsigned int minCount) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuDevResourceGenerateDesc(CUdevResourceDesc* phDesc, CUdevResource* resources, unsigned int nbResources) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGreenCtxRecordEvent(CUgreenCtx hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGreenCtxWaitEvent(CUgreenCtx hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuStreamGetGreenCtx(CUstream hStream, CUgreenCtx* phCtx) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGreenCtxStreamCreate(CUstream* phStream, CUgreenCtx greenCtx, unsigned int flags, int priority) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLogsRegisterCallback(CUlogsCallback callbackFunc, void* userData, CUlogsCallbackHandle* callback_out) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLogsUnregisterCallback(CUlogsCallbackHandle callback) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLogsCurrent(CUlogIterator* iterator_out, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLogsDumpToFile(CUlogIterator* iterator, const char* pathToFile, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuLogsDumpToMemory(CUlogIterator* iterator, char* buffer, size_t* size, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCheckpointProcessGetRestoreThreadId(int pid, int* tid) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCheckpointProcessGetState(int pid, CUprocessState* state) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCheckpointProcessLock(int pid, CUcheckpointLockArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCheckpointProcessCheckpoint(int pid, CUcheckpointCheckpointArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCheckpointProcessRestore(int pid, CUcheckpointRestoreArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuCheckpointProcessUnlock(int pid, CUcheckpointUnlockArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuProfilerStart() except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuProfilerStop() except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphicsEGLRegisterImage(CUgraphicsResource* pCudaResource, EGLImageKHR image, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEGLStreamConsumerConnect(CUeglStreamConnection* conn, EGLStreamKHR stream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEGLStreamConsumerConnectWithFlags(CUeglStreamConnection* conn, EGLStreamKHR stream, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEGLStreamConsumerDisconnect(CUeglStreamConnection* conn) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEGLStreamConsumerAcquireFrame(CUeglStreamConnection* conn, CUgraphicsResource* pCudaResource, CUstream* pStream, unsigned int timeout) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEGLStreamConsumerReleaseFrame(CUeglStreamConnection* conn, CUgraphicsResource pCudaResource, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEGLStreamProducerConnect(CUeglStreamConnection* conn, EGLStreamKHR stream, EGLint width, EGLint height) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEGLStreamProducerDisconnect(CUeglStreamConnection* conn) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEGLStreamProducerPresentFrame(CUeglStreamConnection* conn, CUeglFrame eglframe, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEGLStreamProducerReturnFrame(CUeglStreamConnection* conn, CUeglFrame* eglframe, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphicsResourceGetMappedEglFrame(CUeglFrame* eglFrame, CUgraphicsResource resource, unsigned int index, unsigned int mipLevel) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuEventCreateFromEGLSync(CUevent* phEvent, EGLSyncKHR eglSync, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphicsGLRegisterBuffer(CUgraphicsResource* pCudaResource, GLuint buffer, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphicsGLRegisterImage(CUgraphicsResource* pCudaResource, GLuint image, GLenum target, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGLGetDevices_v2(unsigned int* pCudaDeviceCount, CUdevice* pCudaDevices, unsigned int cudaDeviceCount, CUGLDeviceList deviceList) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuVDPAUGetDevice(CUdevice* pDevice, VdpDevice vdpDevice, VdpGetProcAddress* vdpGetProcAddress) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuVDPAUCtxCreate_v2(CUcontext* pCtx, unsigned int flags, CUdevice device, VdpDevice vdpDevice, VdpGetProcAddress* vdpGetProcAddress) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphicsVDPAURegisterVideoSurface(CUgraphicsResource* pCudaResource, VdpVideoSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + +cdef CUresult _cuGraphicsVDPAURegisterOutputSurface(CUgraphicsResource* pCudaResource, VdpOutputSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cydriver.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cydriver.pyx new file mode 100644 index 0000000000000000000000000000000000000000..f82bbea49d242ed5ea93311fd604ec522e8ee391 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cydriver.pyx @@ -0,0 +1,8000 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +# This code was automatically generated with version 12.9.0. Do not modify it directly. +cimport cuda.bindings._lib.dlfcn as dlfcn +from libc.stdint cimport intptr_t, uintptr_t +import os +import sys +cimport cuda.bindings._bindings.loader as loader +import threading + +cdef object __symbol_lock = threading.Lock() +cdef bint __cuPythonInit = False +cdef void *__cuGetErrorString = NULL +cdef void *__cuGetErrorName = NULL +cdef void *__cuInit = NULL +cdef void *__cuDriverGetVersion = NULL +cdef void *__cuDeviceGet = NULL +cdef void *__cuDeviceGetCount = NULL +cdef void *__cuDeviceGetName = NULL +cdef void *__cuDeviceGetUuid = NULL +cdef void *__cuDeviceGetUuid_v2 = NULL +cdef void *__cuDeviceGetLuid = NULL +cdef void *__cuDeviceTotalMem_v2 = NULL +cdef void *__cuDeviceGetTexture1DLinearMaxWidth = NULL +cdef void *__cuDeviceGetAttribute = NULL +cdef void *__cuDeviceGetNvSciSyncAttributes = NULL +cdef void *__cuDeviceSetMemPool = NULL +cdef void *__cuDeviceGetMemPool = NULL +cdef void *__cuDeviceGetDefaultMemPool = NULL +cdef void *__cuDeviceGetExecAffinitySupport = NULL +cdef void *__cuFlushGPUDirectRDMAWrites = NULL +cdef void *__cuDeviceGetProperties = NULL +cdef void *__cuDeviceComputeCapability = NULL +cdef void *__cuDevicePrimaryCtxRetain = NULL +cdef void *__cuDevicePrimaryCtxRelease_v2 = NULL +cdef void *__cuDevicePrimaryCtxSetFlags_v2 = NULL +cdef void *__cuDevicePrimaryCtxGetState = NULL +cdef void *__cuDevicePrimaryCtxReset_v2 = NULL +cdef void *__cuCtxCreate_v2 = NULL +cdef void *__cuCtxCreate_v3 = NULL +cdef void *__cuCtxCreate_v4 = NULL +cdef void *__cuCtxDestroy_v2 = NULL +cdef void *__cuCtxPushCurrent_v2 = NULL +cdef void *__cuCtxPopCurrent_v2 = NULL +cdef void *__cuCtxSetCurrent = NULL +cdef void *__cuCtxGetCurrent = NULL +cdef void *__cuCtxGetDevice = NULL +cdef void *__cuCtxGetFlags = NULL +cdef void *__cuCtxSetFlags = NULL +cdef void *__cuCtxGetId = NULL +cdef void *__cuCtxSynchronize = NULL +cdef void *__cuCtxSetLimit = NULL +cdef void *__cuCtxGetLimit = NULL +cdef void *__cuCtxGetCacheConfig = NULL +cdef void *__cuCtxSetCacheConfig = NULL +cdef void *__cuCtxGetApiVersion = NULL +cdef void *__cuCtxGetStreamPriorityRange = NULL +cdef void *__cuCtxResetPersistingL2Cache = NULL +cdef void *__cuCtxGetExecAffinity = NULL +cdef void *__cuCtxRecordEvent = NULL +cdef void *__cuCtxWaitEvent = NULL +cdef void *__cuCtxAttach = NULL +cdef void *__cuCtxDetach = NULL +cdef void *__cuCtxGetSharedMemConfig = NULL +cdef void *__cuCtxSetSharedMemConfig = NULL +cdef void *__cuModuleLoad = NULL +cdef void *__cuModuleLoadData = NULL +cdef void *__cuModuleLoadDataEx = NULL +cdef void *__cuModuleLoadFatBinary = NULL +cdef void *__cuModuleUnload = NULL +cdef void *__cuModuleGetLoadingMode = NULL +cdef void *__cuModuleGetFunction = NULL +cdef void *__cuModuleGetFunctionCount = NULL +cdef void *__cuModuleEnumerateFunctions = NULL +cdef void *__cuModuleGetGlobal_v2 = NULL +cdef void *__cuLinkCreate_v2 = NULL +cdef void *__cuLinkAddData_v2 = NULL +cdef void *__cuLinkAddFile_v2 = NULL +cdef void *__cuLinkComplete = NULL +cdef void *__cuLinkDestroy = NULL +cdef void *__cuModuleGetTexRef = NULL +cdef void *__cuModuleGetSurfRef = NULL +cdef void *__cuLibraryLoadData = NULL +cdef void *__cuLibraryLoadFromFile = NULL +cdef void *__cuLibraryUnload = NULL +cdef void *__cuLibraryGetKernel = NULL +cdef void *__cuLibraryGetKernelCount = NULL +cdef void *__cuLibraryEnumerateKernels = NULL +cdef void *__cuLibraryGetModule = NULL +cdef void *__cuKernelGetFunction = NULL +cdef void *__cuKernelGetLibrary = NULL +cdef void *__cuLibraryGetGlobal = NULL +cdef void *__cuLibraryGetManaged = NULL +cdef void *__cuLibraryGetUnifiedFunction = NULL +cdef void *__cuKernelGetAttribute = NULL +cdef void *__cuKernelSetAttribute = NULL +cdef void *__cuKernelSetCacheConfig = NULL +cdef void *__cuKernelGetName = NULL +cdef void *__cuKernelGetParamInfo = NULL +cdef void *__cuMemGetInfo_v2 = NULL +cdef void *__cuMemAlloc_v2 = NULL +cdef void *__cuMemAllocPitch_v2 = NULL +cdef void *__cuMemFree_v2 = NULL +cdef void *__cuMemGetAddressRange_v2 = NULL +cdef void *__cuMemAllocHost_v2 = NULL +cdef void *__cuMemFreeHost = NULL +cdef void *__cuMemHostAlloc = NULL +cdef void *__cuMemHostGetDevicePointer_v2 = NULL +cdef void *__cuMemHostGetFlags = NULL +cdef void *__cuMemAllocManaged = NULL +cdef void *__cuDeviceRegisterAsyncNotification = NULL +cdef void *__cuDeviceUnregisterAsyncNotification = NULL +cdef void *__cuDeviceGetByPCIBusId = NULL +cdef void *__cuDeviceGetPCIBusId = NULL +cdef void *__cuIpcGetEventHandle = NULL +cdef void *__cuIpcOpenEventHandle = NULL +cdef void *__cuIpcGetMemHandle = NULL +cdef void *__cuIpcOpenMemHandle_v2 = NULL +cdef void *__cuIpcCloseMemHandle = NULL +cdef void *__cuMemHostRegister_v2 = NULL +cdef void *__cuMemHostUnregister = NULL +cdef void *__cuMemcpy = NULL +cdef void *__cuMemcpyPeer = NULL +cdef void *__cuMemcpyHtoD_v2 = NULL +cdef void *__cuMemcpyDtoH_v2 = NULL +cdef void *__cuMemcpyDtoD_v2 = NULL +cdef void *__cuMemcpyDtoA_v2 = NULL +cdef void *__cuMemcpyAtoD_v2 = NULL +cdef void *__cuMemcpyHtoA_v2 = NULL +cdef void *__cuMemcpyAtoH_v2 = NULL +cdef void *__cuMemcpyAtoA_v2 = NULL +cdef void *__cuMemcpy2D_v2 = NULL +cdef void *__cuMemcpy2DUnaligned_v2 = NULL +cdef void *__cuMemcpy3D_v2 = NULL +cdef void *__cuMemcpy3DPeer = NULL +cdef void *__cuMemcpyAsync = NULL +cdef void *__cuMemcpyPeerAsync = NULL +cdef void *__cuMemcpyHtoDAsync_v2 = NULL +cdef void *__cuMemcpyDtoHAsync_v2 = NULL +cdef void *__cuMemcpyDtoDAsync_v2 = NULL +cdef void *__cuMemcpyHtoAAsync_v2 = NULL +cdef void *__cuMemcpyAtoHAsync_v2 = NULL +cdef void *__cuMemcpy2DAsync_v2 = NULL +cdef void *__cuMemcpy3DAsync_v2 = NULL +cdef void *__cuMemcpy3DPeerAsync = NULL +cdef void *__cuMemcpyBatchAsync = NULL +cdef void *__cuMemcpy3DBatchAsync = NULL +cdef void *__cuMemsetD8_v2 = NULL +cdef void *__cuMemsetD16_v2 = NULL +cdef void *__cuMemsetD32_v2 = NULL +cdef void *__cuMemsetD2D8_v2 = NULL +cdef void *__cuMemsetD2D16_v2 = NULL +cdef void *__cuMemsetD2D32_v2 = NULL +cdef void *__cuMemsetD8Async = NULL +cdef void *__cuMemsetD16Async = NULL +cdef void *__cuMemsetD32Async = NULL +cdef void *__cuMemsetD2D8Async = NULL +cdef void *__cuMemsetD2D16Async = NULL +cdef void *__cuMemsetD2D32Async = NULL +cdef void *__cuArrayCreate_v2 = NULL +cdef void *__cuArrayGetDescriptor_v2 = NULL +cdef void *__cuArrayGetSparseProperties = NULL +cdef void *__cuMipmappedArrayGetSparseProperties = NULL +cdef void *__cuArrayGetMemoryRequirements = NULL +cdef void *__cuMipmappedArrayGetMemoryRequirements = NULL +cdef void *__cuArrayGetPlane = NULL +cdef void *__cuArrayDestroy = NULL +cdef void *__cuArray3DCreate_v2 = NULL +cdef void *__cuArray3DGetDescriptor_v2 = NULL +cdef void *__cuMipmappedArrayCreate = NULL +cdef void *__cuMipmappedArrayGetLevel = NULL +cdef void *__cuMipmappedArrayDestroy = NULL +cdef void *__cuMemGetHandleForAddressRange = NULL +cdef void *__cuMemBatchDecompressAsync = NULL +cdef void *__cuMemAddressReserve = NULL +cdef void *__cuMemAddressFree = NULL +cdef void *__cuMemCreate = NULL +cdef void *__cuMemRelease = NULL +cdef void *__cuMemMap = NULL +cdef void *__cuMemMapArrayAsync = NULL +cdef void *__cuMemUnmap = NULL +cdef void *__cuMemSetAccess = NULL +cdef void *__cuMemGetAccess = NULL +cdef void *__cuMemExportToShareableHandle = NULL +cdef void *__cuMemImportFromShareableHandle = NULL +cdef void *__cuMemGetAllocationGranularity = NULL +cdef void *__cuMemGetAllocationPropertiesFromHandle = NULL +cdef void *__cuMemRetainAllocationHandle = NULL +cdef void *__cuMemFreeAsync = NULL +cdef void *__cuMemAllocAsync = NULL +cdef void *__cuMemPoolTrimTo = NULL +cdef void *__cuMemPoolSetAttribute = NULL +cdef void *__cuMemPoolGetAttribute = NULL +cdef void *__cuMemPoolSetAccess = NULL +cdef void *__cuMemPoolGetAccess = NULL +cdef void *__cuMemPoolCreate = NULL +cdef void *__cuMemPoolDestroy = NULL +cdef void *__cuMemAllocFromPoolAsync = NULL +cdef void *__cuMemPoolExportToShareableHandle = NULL +cdef void *__cuMemPoolImportFromShareableHandle = NULL +cdef void *__cuMemPoolExportPointer = NULL +cdef void *__cuMemPoolImportPointer = NULL +cdef void *__cuMulticastCreate = NULL +cdef void *__cuMulticastAddDevice = NULL +cdef void *__cuMulticastBindMem = NULL +cdef void *__cuMulticastBindAddr = NULL +cdef void *__cuMulticastUnbind = NULL +cdef void *__cuMulticastGetGranularity = NULL +cdef void *__cuPointerGetAttribute = NULL +cdef void *__cuMemPrefetchAsync = NULL +cdef void *__cuMemPrefetchAsync_v2 = NULL +cdef void *__cuMemAdvise = NULL +cdef void *__cuMemAdvise_v2 = NULL +cdef void *__cuMemRangeGetAttribute = NULL +cdef void *__cuMemRangeGetAttributes = NULL +cdef void *__cuPointerSetAttribute = NULL +cdef void *__cuPointerGetAttributes = NULL +cdef void *__cuStreamCreate = NULL +cdef void *__cuStreamCreateWithPriority = NULL +cdef void *__cuStreamGetPriority = NULL +cdef void *__cuStreamGetDevice = NULL +cdef void *__cuStreamGetFlags = NULL +cdef void *__cuStreamGetId = NULL +cdef void *__cuStreamGetCtx = NULL +cdef void *__cuStreamGetCtx_v2 = NULL +cdef void *__cuStreamWaitEvent = NULL +cdef void *__cuStreamAddCallback = NULL +cdef void *__cuStreamBeginCapture_v2 = NULL +cdef void *__cuStreamBeginCaptureToGraph = NULL +cdef void *__cuThreadExchangeStreamCaptureMode = NULL +cdef void *__cuStreamEndCapture = NULL +cdef void *__cuStreamIsCapturing = NULL +cdef void *__cuStreamGetCaptureInfo_v2 = NULL +cdef void *__cuStreamGetCaptureInfo_v3 = NULL +cdef void *__cuStreamUpdateCaptureDependencies = NULL +cdef void *__cuStreamUpdateCaptureDependencies_v2 = NULL +cdef void *__cuStreamAttachMemAsync = NULL +cdef void *__cuStreamQuery = NULL +cdef void *__cuStreamSynchronize = NULL +cdef void *__cuStreamDestroy_v2 = NULL +cdef void *__cuStreamCopyAttributes = NULL +cdef void *__cuStreamGetAttribute = NULL +cdef void *__cuStreamSetAttribute = NULL +cdef void *__cuEventCreate = NULL +cdef void *__cuEventRecord = NULL +cdef void *__cuEventRecordWithFlags = NULL +cdef void *__cuEventQuery = NULL +cdef void *__cuEventSynchronize = NULL +cdef void *__cuEventDestroy_v2 = NULL +cdef void *__cuEventElapsedTime = NULL +cdef void *__cuEventElapsedTime_v2 = NULL +cdef void *__cuImportExternalMemory = NULL +cdef void *__cuExternalMemoryGetMappedBuffer = NULL +cdef void *__cuExternalMemoryGetMappedMipmappedArray = NULL +cdef void *__cuDestroyExternalMemory = NULL +cdef void *__cuImportExternalSemaphore = NULL +cdef void *__cuSignalExternalSemaphoresAsync = NULL +cdef void *__cuWaitExternalSemaphoresAsync = NULL +cdef void *__cuDestroyExternalSemaphore = NULL +cdef void *__cuStreamWaitValue32_v2 = NULL +cdef void *__cuStreamWaitValue64_v2 = NULL +cdef void *__cuStreamWriteValue32_v2 = NULL +cdef void *__cuStreamWriteValue64_v2 = NULL +cdef void *__cuStreamBatchMemOp_v2 = NULL +cdef void *__cuFuncGetAttribute = NULL +cdef void *__cuFuncSetAttribute = NULL +cdef void *__cuFuncSetCacheConfig = NULL +cdef void *__cuFuncGetModule = NULL +cdef void *__cuFuncGetName = NULL +cdef void *__cuFuncGetParamInfo = NULL +cdef void *__cuFuncIsLoaded = NULL +cdef void *__cuFuncLoad = NULL +cdef void *__cuLaunchKernel = NULL +cdef void *__cuLaunchKernelEx = NULL +cdef void *__cuLaunchCooperativeKernel = NULL +cdef void *__cuLaunchCooperativeKernelMultiDevice = NULL +cdef void *__cuLaunchHostFunc = NULL +cdef void *__cuFuncSetBlockShape = NULL +cdef void *__cuFuncSetSharedSize = NULL +cdef void *__cuParamSetSize = NULL +cdef void *__cuParamSeti = NULL +cdef void *__cuParamSetf = NULL +cdef void *__cuParamSetv = NULL +cdef void *__cuLaunch = NULL +cdef void *__cuLaunchGrid = NULL +cdef void *__cuLaunchGridAsync = NULL +cdef void *__cuParamSetTexRef = NULL +cdef void *__cuFuncSetSharedMemConfig = NULL +cdef void *__cuGraphCreate = NULL +cdef void *__cuGraphAddKernelNode_v2 = NULL +cdef void *__cuGraphKernelNodeGetParams_v2 = NULL +cdef void *__cuGraphKernelNodeSetParams_v2 = NULL +cdef void *__cuGraphAddMemcpyNode = NULL +cdef void *__cuGraphMemcpyNodeGetParams = NULL +cdef void *__cuGraphMemcpyNodeSetParams = NULL +cdef void *__cuGraphAddMemsetNode = NULL +cdef void *__cuGraphMemsetNodeGetParams = NULL +cdef void *__cuGraphMemsetNodeSetParams = NULL +cdef void *__cuGraphAddHostNode = NULL +cdef void *__cuGraphHostNodeGetParams = NULL +cdef void *__cuGraphHostNodeSetParams = NULL +cdef void *__cuGraphAddChildGraphNode = NULL +cdef void *__cuGraphChildGraphNodeGetGraph = NULL +cdef void *__cuGraphAddEmptyNode = NULL +cdef void *__cuGraphAddEventRecordNode = NULL +cdef void *__cuGraphEventRecordNodeGetEvent = NULL +cdef void *__cuGraphEventRecordNodeSetEvent = NULL +cdef void *__cuGraphAddEventWaitNode = NULL +cdef void *__cuGraphEventWaitNodeGetEvent = NULL +cdef void *__cuGraphEventWaitNodeSetEvent = NULL +cdef void *__cuGraphAddExternalSemaphoresSignalNode = NULL +cdef void *__cuGraphExternalSemaphoresSignalNodeGetParams = NULL +cdef void *__cuGraphExternalSemaphoresSignalNodeSetParams = NULL +cdef void *__cuGraphAddExternalSemaphoresWaitNode = NULL +cdef void *__cuGraphExternalSemaphoresWaitNodeGetParams = NULL +cdef void *__cuGraphExternalSemaphoresWaitNodeSetParams = NULL +cdef void *__cuGraphAddBatchMemOpNode = NULL +cdef void *__cuGraphBatchMemOpNodeGetParams = NULL +cdef void *__cuGraphBatchMemOpNodeSetParams = NULL +cdef void *__cuGraphExecBatchMemOpNodeSetParams = NULL +cdef void *__cuGraphAddMemAllocNode = NULL +cdef void *__cuGraphMemAllocNodeGetParams = NULL +cdef void *__cuGraphAddMemFreeNode = NULL +cdef void *__cuGraphMemFreeNodeGetParams = NULL +cdef void *__cuDeviceGraphMemTrim = NULL +cdef void *__cuDeviceGetGraphMemAttribute = NULL +cdef void *__cuDeviceSetGraphMemAttribute = NULL +cdef void *__cuGraphClone = NULL +cdef void *__cuGraphNodeFindInClone = NULL +cdef void *__cuGraphNodeGetType = NULL +cdef void *__cuGraphGetNodes = NULL +cdef void *__cuGraphGetRootNodes = NULL +cdef void *__cuGraphGetEdges = NULL +cdef void *__cuGraphGetEdges_v2 = NULL +cdef void *__cuGraphNodeGetDependencies = NULL +cdef void *__cuGraphNodeGetDependencies_v2 = NULL +cdef void *__cuGraphNodeGetDependentNodes = NULL +cdef void *__cuGraphNodeGetDependentNodes_v2 = NULL +cdef void *__cuGraphAddDependencies = NULL +cdef void *__cuGraphAddDependencies_v2 = NULL +cdef void *__cuGraphRemoveDependencies = NULL +cdef void *__cuGraphRemoveDependencies_v2 = NULL +cdef void *__cuGraphDestroyNode = NULL +cdef void *__cuGraphInstantiateWithFlags = NULL +cdef void *__cuGraphInstantiateWithParams = NULL +cdef void *__cuGraphExecGetFlags = NULL +cdef void *__cuGraphExecKernelNodeSetParams_v2 = NULL +cdef void *__cuGraphExecMemcpyNodeSetParams = NULL +cdef void *__cuGraphExecMemsetNodeSetParams = NULL +cdef void *__cuGraphExecHostNodeSetParams = NULL +cdef void *__cuGraphExecChildGraphNodeSetParams = NULL +cdef void *__cuGraphExecEventRecordNodeSetEvent = NULL +cdef void *__cuGraphExecEventWaitNodeSetEvent = NULL +cdef void *__cuGraphExecExternalSemaphoresSignalNodeSetParams = NULL +cdef void *__cuGraphExecExternalSemaphoresWaitNodeSetParams = NULL +cdef void *__cuGraphNodeSetEnabled = NULL +cdef void *__cuGraphNodeGetEnabled = NULL +cdef void *__cuGraphUpload = NULL +cdef void *__cuGraphLaunch = NULL +cdef void *__cuGraphExecDestroy = NULL +cdef void *__cuGraphDestroy = NULL +cdef void *__cuGraphExecUpdate_v2 = NULL +cdef void *__cuGraphKernelNodeCopyAttributes = NULL +cdef void *__cuGraphKernelNodeGetAttribute = NULL +cdef void *__cuGraphKernelNodeSetAttribute = NULL +cdef void *__cuGraphDebugDotPrint = NULL +cdef void *__cuUserObjectCreate = NULL +cdef void *__cuUserObjectRetain = NULL +cdef void *__cuUserObjectRelease = NULL +cdef void *__cuGraphRetainUserObject = NULL +cdef void *__cuGraphReleaseUserObject = NULL +cdef void *__cuGraphAddNode = NULL +cdef void *__cuGraphAddNode_v2 = NULL +cdef void *__cuGraphNodeSetParams = NULL +cdef void *__cuGraphExecNodeSetParams = NULL +cdef void *__cuGraphConditionalHandleCreate = NULL +cdef void *__cuOccupancyMaxActiveBlocksPerMultiprocessor = NULL +cdef void *__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags = NULL +cdef void *__cuOccupancyMaxPotentialBlockSize = NULL +cdef void *__cuOccupancyMaxPotentialBlockSizeWithFlags = NULL +cdef void *__cuOccupancyAvailableDynamicSMemPerBlock = NULL +cdef void *__cuOccupancyMaxPotentialClusterSize = NULL +cdef void *__cuOccupancyMaxActiveClusters = NULL +cdef void *__cuTexRefSetArray = NULL +cdef void *__cuTexRefSetMipmappedArray = NULL +cdef void *__cuTexRefSetAddress_v2 = NULL +cdef void *__cuTexRefSetAddress2D_v3 = NULL +cdef void *__cuTexRefSetFormat = NULL +cdef void *__cuTexRefSetAddressMode = NULL +cdef void *__cuTexRefSetFilterMode = NULL +cdef void *__cuTexRefSetMipmapFilterMode = NULL +cdef void *__cuTexRefSetMipmapLevelBias = NULL +cdef void *__cuTexRefSetMipmapLevelClamp = NULL +cdef void *__cuTexRefSetMaxAnisotropy = NULL +cdef void *__cuTexRefSetBorderColor = NULL +cdef void *__cuTexRefSetFlags = NULL +cdef void *__cuTexRefGetAddress_v2 = NULL +cdef void *__cuTexRefGetArray = NULL +cdef void *__cuTexRefGetMipmappedArray = NULL +cdef void *__cuTexRefGetAddressMode = NULL +cdef void *__cuTexRefGetFilterMode = NULL +cdef void *__cuTexRefGetFormat = NULL +cdef void *__cuTexRefGetMipmapFilterMode = NULL +cdef void *__cuTexRefGetMipmapLevelBias = NULL +cdef void *__cuTexRefGetMipmapLevelClamp = NULL +cdef void *__cuTexRefGetMaxAnisotropy = NULL +cdef void *__cuTexRefGetBorderColor = NULL +cdef void *__cuTexRefGetFlags = NULL +cdef void *__cuTexRefCreate = NULL +cdef void *__cuTexRefDestroy = NULL +cdef void *__cuSurfRefSetArray = NULL +cdef void *__cuSurfRefGetArray = NULL +cdef void *__cuTexObjectCreate = NULL +cdef void *__cuTexObjectDestroy = NULL +cdef void *__cuTexObjectGetResourceDesc = NULL +cdef void *__cuTexObjectGetTextureDesc = NULL +cdef void *__cuTexObjectGetResourceViewDesc = NULL +cdef void *__cuSurfObjectCreate = NULL +cdef void *__cuSurfObjectDestroy = NULL +cdef void *__cuSurfObjectGetResourceDesc = NULL +cdef void *__cuTensorMapEncodeTiled = NULL +cdef void *__cuTensorMapEncodeIm2col = NULL +cdef void *__cuTensorMapEncodeIm2colWide = NULL +cdef void *__cuTensorMapReplaceAddress = NULL +cdef void *__cuDeviceCanAccessPeer = NULL +cdef void *__cuCtxEnablePeerAccess = NULL +cdef void *__cuCtxDisablePeerAccess = NULL +cdef void *__cuDeviceGetP2PAttribute = NULL +cdef void *__cuGraphicsUnregisterResource = NULL +cdef void *__cuGraphicsSubResourceGetMappedArray = NULL +cdef void *__cuGraphicsResourceGetMappedMipmappedArray = NULL +cdef void *__cuGraphicsResourceGetMappedPointer_v2 = NULL +cdef void *__cuGraphicsResourceSetMapFlags_v2 = NULL +cdef void *__cuGraphicsMapResources = NULL +cdef void *__cuGraphicsUnmapResources = NULL +cdef void *__cuGetProcAddress_v2 = NULL +cdef void *__cuCoredumpGetAttribute = NULL +cdef void *__cuCoredumpGetAttributeGlobal = NULL +cdef void *__cuCoredumpSetAttribute = NULL +cdef void *__cuCoredumpSetAttributeGlobal = NULL +cdef void *__cuGetExportTable = NULL +cdef void *__cuGreenCtxCreate = NULL +cdef void *__cuGreenCtxDestroy = NULL +cdef void *__cuCtxFromGreenCtx = NULL +cdef void *__cuDeviceGetDevResource = NULL +cdef void *__cuCtxGetDevResource = NULL +cdef void *__cuGreenCtxGetDevResource = NULL +cdef void *__cuDevSmResourceSplitByCount = NULL +cdef void *__cuDevResourceGenerateDesc = NULL +cdef void *__cuGreenCtxRecordEvent = NULL +cdef void *__cuGreenCtxWaitEvent = NULL +cdef void *__cuStreamGetGreenCtx = NULL +cdef void *__cuGreenCtxStreamCreate = NULL +cdef void *__cuLogsRegisterCallback = NULL +cdef void *__cuLogsUnregisterCallback = NULL +cdef void *__cuLogsCurrent = NULL +cdef void *__cuLogsDumpToFile = NULL +cdef void *__cuLogsDumpToMemory = NULL +cdef void *__cuCheckpointProcessGetRestoreThreadId = NULL +cdef void *__cuCheckpointProcessGetState = NULL +cdef void *__cuCheckpointProcessLock = NULL +cdef void *__cuCheckpointProcessCheckpoint = NULL +cdef void *__cuCheckpointProcessRestore = NULL +cdef void *__cuCheckpointProcessUnlock = NULL +cdef void *__cuProfilerStart = NULL +cdef void *__cuProfilerStop = NULL +cdef void *__cuGraphicsEGLRegisterImage = NULL +cdef void *__cuEGLStreamConsumerConnect = NULL +cdef void *__cuEGLStreamConsumerConnectWithFlags = NULL +cdef void *__cuEGLStreamConsumerDisconnect = NULL +cdef void *__cuEGLStreamConsumerAcquireFrame = NULL +cdef void *__cuEGLStreamConsumerReleaseFrame = NULL +cdef void *__cuEGLStreamProducerConnect = NULL +cdef void *__cuEGLStreamProducerDisconnect = NULL +cdef void *__cuEGLStreamProducerPresentFrame = NULL +cdef void *__cuEGLStreamProducerReturnFrame = NULL +cdef void *__cuGraphicsResourceGetMappedEglFrame = NULL +cdef void *__cuEventCreateFromEGLSync = NULL +cdef void *__cuGraphicsGLRegisterBuffer = NULL +cdef void *__cuGraphicsGLRegisterImage = NULL +cdef void *__cuGLGetDevices_v2 = NULL +cdef void *__cuVDPAUGetDevice = NULL +cdef void *__cuVDPAUCtxCreate_v2 = NULL +cdef void *__cuGraphicsVDPAURegisterVideoSurface = NULL +cdef void *__cuGraphicsVDPAURegisterOutputSurface = NULL + +# To make cuPythonInit reentrant +ctypedef CUresult (*__cuGetProcAddress_v2_T)(const char*, void**, int, cuuint64_t, CUdriverProcAddressQueryResult*) except?CUDA_ERROR_NOT_FOUND nogil +cdef __cuGetProcAddress_v2_T _F_cuGetProcAddress_v2 = NULL + +cdef int _cuPythonInit() except -1 nogil: + global __cuPythonInit + + cdef bint usePTDS + cdef char libPath[260] + + with gil, __symbol_lock: + usePTDS = os.getenv('CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM', default=0) + + # Load library + libPath[0] = 0 + status = loader.getCUDALibraryPath(libPath, sys.maxsize > 2**32) + if status == 0 and len(libPath) != 0: + path = libPath.decode('utf-8') + else: + path = 'libcuda.so.1' + handle = dlfcn.dlopen(bytes(path, encoding='utf-8'), dlfcn.RTLD_NOW) + if handle == NULL: + raise RuntimeError('Failed to dlopen ' + path) + + # Get latest __cuGetProcAddress_v2 + global __cuGetProcAddress_v2 + __cuGetProcAddress_v2 = dlfcn.dlsym(handle, 'cuGetProcAddress_v2') + + # Load using cuGetProcAddress if available + if __cuGetProcAddress_v2 != NULL: + _F_cuGetProcAddress_v2 = <__cuGetProcAddress_v2_T>__cuGetProcAddress_v2 + if usePTDS: + # Get all PTDS version of functions + pass + global __cuMemcpy + _F_cuGetProcAddress_v2('cuMemcpy', &__cuMemcpy, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyPeer + _F_cuGetProcAddress_v2('cuMemcpyPeer', &__cuMemcpyPeer, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyHtoD_v2 + _F_cuGetProcAddress_v2('cuMemcpyHtoD', &__cuMemcpyHtoD_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyDtoH_v2 + _F_cuGetProcAddress_v2('cuMemcpyDtoH', &__cuMemcpyDtoH_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyDtoD_v2 + _F_cuGetProcAddress_v2('cuMemcpyDtoD', &__cuMemcpyDtoD_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyDtoA_v2 + _F_cuGetProcAddress_v2('cuMemcpyDtoA', &__cuMemcpyDtoA_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyAtoD_v2 + _F_cuGetProcAddress_v2('cuMemcpyAtoD', &__cuMemcpyAtoD_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyHtoA_v2 + _F_cuGetProcAddress_v2('cuMemcpyHtoA', &__cuMemcpyHtoA_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyAtoH_v2 + _F_cuGetProcAddress_v2('cuMemcpyAtoH', &__cuMemcpyAtoH_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyAtoA_v2 + _F_cuGetProcAddress_v2('cuMemcpyAtoA', &__cuMemcpyAtoA_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpy2D_v2 + _F_cuGetProcAddress_v2('cuMemcpy2D', &__cuMemcpy2D_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpy2DUnaligned_v2 + _F_cuGetProcAddress_v2('cuMemcpy2DUnaligned', &__cuMemcpy2DUnaligned_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpy3D_v2 + _F_cuGetProcAddress_v2('cuMemcpy3D', &__cuMemcpy3D_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpy3DPeer + _F_cuGetProcAddress_v2('cuMemcpy3DPeer', &__cuMemcpy3DPeer, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyAsync + _F_cuGetProcAddress_v2('cuMemcpyAsync', &__cuMemcpyAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyPeerAsync + _F_cuGetProcAddress_v2('cuMemcpyPeerAsync', &__cuMemcpyPeerAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyHtoDAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpyHtoDAsync', &__cuMemcpyHtoDAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyDtoHAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpyDtoHAsync', &__cuMemcpyDtoHAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyDtoDAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpyDtoDAsync', &__cuMemcpyDtoDAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyHtoAAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpyHtoAAsync', &__cuMemcpyHtoAAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyAtoHAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpyAtoHAsync', &__cuMemcpyAtoHAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpy2DAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpy2DAsync', &__cuMemcpy2DAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpy3DAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpy3DAsync', &__cuMemcpy3DAsync_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpy3DPeerAsync + _F_cuGetProcAddress_v2('cuMemcpy3DPeerAsync', &__cuMemcpy3DPeerAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpyBatchAsync + _F_cuGetProcAddress_v2('cuMemcpyBatchAsync', &__cuMemcpyBatchAsync, 12080, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemcpy3DBatchAsync + _F_cuGetProcAddress_v2('cuMemcpy3DBatchAsync', &__cuMemcpy3DBatchAsync, 12080, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemsetD8_v2 + _F_cuGetProcAddress_v2('cuMemsetD8', &__cuMemsetD8_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemsetD16_v2 + _F_cuGetProcAddress_v2('cuMemsetD16', &__cuMemsetD16_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemsetD32_v2 + _F_cuGetProcAddress_v2('cuMemsetD32', &__cuMemsetD32_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemsetD2D8_v2 + _F_cuGetProcAddress_v2('cuMemsetD2D8', &__cuMemsetD2D8_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemsetD2D16_v2 + _F_cuGetProcAddress_v2('cuMemsetD2D16', &__cuMemsetD2D16_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemsetD2D32_v2 + _F_cuGetProcAddress_v2('cuMemsetD2D32', &__cuMemsetD2D32_v2, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemsetD8Async + _F_cuGetProcAddress_v2('cuMemsetD8Async', &__cuMemsetD8Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemsetD16Async + _F_cuGetProcAddress_v2('cuMemsetD16Async', &__cuMemsetD16Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemsetD32Async + _F_cuGetProcAddress_v2('cuMemsetD32Async', &__cuMemsetD32Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemsetD2D8Async + _F_cuGetProcAddress_v2('cuMemsetD2D8Async', &__cuMemsetD2D8Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemsetD2D16Async + _F_cuGetProcAddress_v2('cuMemsetD2D16Async', &__cuMemsetD2D16Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemsetD2D32Async + _F_cuGetProcAddress_v2('cuMemsetD2D32Async', &__cuMemsetD2D32Async, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemBatchDecompressAsync + _F_cuGetProcAddress_v2('cuMemBatchDecompressAsync', &__cuMemBatchDecompressAsync, 12060, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemMapArrayAsync + _F_cuGetProcAddress_v2('cuMemMapArrayAsync', &__cuMemMapArrayAsync, 11010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemFreeAsync + _F_cuGetProcAddress_v2('cuMemFreeAsync', &__cuMemFreeAsync, 11020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemAllocAsync + _F_cuGetProcAddress_v2('cuMemAllocAsync', &__cuMemAllocAsync, 11020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemAllocFromPoolAsync + _F_cuGetProcAddress_v2('cuMemAllocFromPoolAsync', &__cuMemAllocFromPoolAsync, 11020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemPrefetchAsync + _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync, 8000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuMemPrefetchAsync_v2 + _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync_v2, 12020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamGetPriority + _F_cuGetProcAddress_v2('cuStreamGetPriority', &__cuStreamGetPriority, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamGetDevice + _F_cuGetProcAddress_v2('cuStreamGetDevice', &__cuStreamGetDevice, 12080, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamGetFlags + _F_cuGetProcAddress_v2('cuStreamGetFlags', &__cuStreamGetFlags, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamGetId + _F_cuGetProcAddress_v2('cuStreamGetId', &__cuStreamGetId, 12000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamGetCtx + _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx, 9020, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamGetCtx_v2 + _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx_v2, 12050, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamWaitEvent + _F_cuGetProcAddress_v2('cuStreamWaitEvent', &__cuStreamWaitEvent, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamAddCallback + _F_cuGetProcAddress_v2('cuStreamAddCallback', &__cuStreamAddCallback, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamBeginCapture_v2 + _F_cuGetProcAddress_v2('cuStreamBeginCapture', &__cuStreamBeginCapture_v2, 10010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamBeginCaptureToGraph + _F_cuGetProcAddress_v2('cuStreamBeginCaptureToGraph', &__cuStreamBeginCaptureToGraph, 12030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamEndCapture + _F_cuGetProcAddress_v2('cuStreamEndCapture', &__cuStreamEndCapture, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamIsCapturing + _F_cuGetProcAddress_v2('cuStreamIsCapturing', &__cuStreamIsCapturing, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamGetCaptureInfo_v2 + _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v2, 11030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamGetCaptureInfo_v3 + _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v3, 12030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamUpdateCaptureDependencies + _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies, 11030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamUpdateCaptureDependencies_v2 + _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies_v2, 12030, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamAttachMemAsync + _F_cuGetProcAddress_v2('cuStreamAttachMemAsync', &__cuStreamAttachMemAsync, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamQuery + _F_cuGetProcAddress_v2('cuStreamQuery', &__cuStreamQuery, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamSynchronize + _F_cuGetProcAddress_v2('cuStreamSynchronize', &__cuStreamSynchronize, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamCopyAttributes + _F_cuGetProcAddress_v2('cuStreamCopyAttributes', &__cuStreamCopyAttributes, 11000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamGetAttribute + _F_cuGetProcAddress_v2('cuStreamGetAttribute', &__cuStreamGetAttribute, 11000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamSetAttribute + _F_cuGetProcAddress_v2('cuStreamSetAttribute', &__cuStreamSetAttribute, 11000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuEventRecord + _F_cuGetProcAddress_v2('cuEventRecord', &__cuEventRecord, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuEventRecordWithFlags + _F_cuGetProcAddress_v2('cuEventRecordWithFlags', &__cuEventRecordWithFlags, 11010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuSignalExternalSemaphoresAsync + _F_cuGetProcAddress_v2('cuSignalExternalSemaphoresAsync', &__cuSignalExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuWaitExternalSemaphoresAsync + _F_cuGetProcAddress_v2('cuWaitExternalSemaphoresAsync', &__cuWaitExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamWaitValue32_v2 + _F_cuGetProcAddress_v2('cuStreamWaitValue32', &__cuStreamWaitValue32_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamWaitValue64_v2 + _F_cuGetProcAddress_v2('cuStreamWaitValue64', &__cuStreamWaitValue64_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamWriteValue32_v2 + _F_cuGetProcAddress_v2('cuStreamWriteValue32', &__cuStreamWriteValue32_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamWriteValue64_v2 + _F_cuGetProcAddress_v2('cuStreamWriteValue64', &__cuStreamWriteValue64_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuStreamBatchMemOp_v2 + _F_cuGetProcAddress_v2('cuStreamBatchMemOp', &__cuStreamBatchMemOp_v2, 11070, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuLaunchKernel + _F_cuGetProcAddress_v2('cuLaunchKernel', &__cuLaunchKernel, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuLaunchKernelEx + _F_cuGetProcAddress_v2('cuLaunchKernelEx', &__cuLaunchKernelEx, 11060, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuLaunchCooperativeKernel + _F_cuGetProcAddress_v2('cuLaunchCooperativeKernel', &__cuLaunchCooperativeKernel, 9000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuLaunchHostFunc + _F_cuGetProcAddress_v2('cuLaunchHostFunc', &__cuLaunchHostFunc, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuGraphInstantiateWithParams + _F_cuGetProcAddress_v2('cuGraphInstantiateWithParams', &__cuGraphInstantiateWithParams, 12000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuGraphUpload + _F_cuGetProcAddress_v2('cuGraphUpload', &__cuGraphUpload, 11010, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuGraphLaunch + _F_cuGetProcAddress_v2('cuGraphLaunch', &__cuGraphLaunch, 10000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuGraphicsMapResources + _F_cuGetProcAddress_v2('cuGraphicsMapResources', &__cuGraphicsMapResources, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + global __cuGraphicsUnmapResources + _F_cuGetProcAddress_v2('cuGraphicsUnmapResources', &__cuGraphicsUnmapResources, 7000, CU_GET_PROC_ADDRESS_PER_THREAD_DEFAULT_STREAM, NULL) + else: + # Else get the regular version + pass + global __cuMemcpy + _F_cuGetProcAddress_v2('cuMemcpy', &__cuMemcpy, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyPeer + _F_cuGetProcAddress_v2('cuMemcpyPeer', &__cuMemcpyPeer, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyHtoD_v2 + _F_cuGetProcAddress_v2('cuMemcpyHtoD', &__cuMemcpyHtoD_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyDtoH_v2 + _F_cuGetProcAddress_v2('cuMemcpyDtoH', &__cuMemcpyDtoH_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyDtoD_v2 + _F_cuGetProcAddress_v2('cuMemcpyDtoD', &__cuMemcpyDtoD_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyDtoA_v2 + _F_cuGetProcAddress_v2('cuMemcpyDtoA', &__cuMemcpyDtoA_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyAtoD_v2 + _F_cuGetProcAddress_v2('cuMemcpyAtoD', &__cuMemcpyAtoD_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyHtoA_v2 + _F_cuGetProcAddress_v2('cuMemcpyHtoA', &__cuMemcpyHtoA_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyAtoH_v2 + _F_cuGetProcAddress_v2('cuMemcpyAtoH', &__cuMemcpyAtoH_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyAtoA_v2 + _F_cuGetProcAddress_v2('cuMemcpyAtoA', &__cuMemcpyAtoA_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpy2D_v2 + _F_cuGetProcAddress_v2('cuMemcpy2D', &__cuMemcpy2D_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpy2DUnaligned_v2 + _F_cuGetProcAddress_v2('cuMemcpy2DUnaligned', &__cuMemcpy2DUnaligned_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpy3D_v2 + _F_cuGetProcAddress_v2('cuMemcpy3D', &__cuMemcpy3D_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpy3DPeer + _F_cuGetProcAddress_v2('cuMemcpy3DPeer', &__cuMemcpy3DPeer, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyAsync + _F_cuGetProcAddress_v2('cuMemcpyAsync', &__cuMemcpyAsync, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyPeerAsync + _F_cuGetProcAddress_v2('cuMemcpyPeerAsync', &__cuMemcpyPeerAsync, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyHtoDAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpyHtoDAsync', &__cuMemcpyHtoDAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyDtoHAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpyDtoHAsync', &__cuMemcpyDtoHAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyDtoDAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpyDtoDAsync', &__cuMemcpyDtoDAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyHtoAAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpyHtoAAsync', &__cuMemcpyHtoAAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyAtoHAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpyAtoHAsync', &__cuMemcpyAtoHAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpy2DAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpy2DAsync', &__cuMemcpy2DAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpy3DAsync_v2 + _F_cuGetProcAddress_v2('cuMemcpy3DAsync', &__cuMemcpy3DAsync_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpy3DPeerAsync + _F_cuGetProcAddress_v2('cuMemcpy3DPeerAsync', &__cuMemcpy3DPeerAsync, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpyBatchAsync + _F_cuGetProcAddress_v2('cuMemcpyBatchAsync', &__cuMemcpyBatchAsync, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemcpy3DBatchAsync + _F_cuGetProcAddress_v2('cuMemcpy3DBatchAsync', &__cuMemcpy3DBatchAsync, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemsetD8_v2 + _F_cuGetProcAddress_v2('cuMemsetD8', &__cuMemsetD8_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemsetD16_v2 + _F_cuGetProcAddress_v2('cuMemsetD16', &__cuMemsetD16_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemsetD32_v2 + _F_cuGetProcAddress_v2('cuMemsetD32', &__cuMemsetD32_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemsetD2D8_v2 + _F_cuGetProcAddress_v2('cuMemsetD2D8', &__cuMemsetD2D8_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemsetD2D16_v2 + _F_cuGetProcAddress_v2('cuMemsetD2D16', &__cuMemsetD2D16_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemsetD2D32_v2 + _F_cuGetProcAddress_v2('cuMemsetD2D32', &__cuMemsetD2D32_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemsetD8Async + _F_cuGetProcAddress_v2('cuMemsetD8Async', &__cuMemsetD8Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemsetD16Async + _F_cuGetProcAddress_v2('cuMemsetD16Async', &__cuMemsetD16Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemsetD32Async + _F_cuGetProcAddress_v2('cuMemsetD32Async', &__cuMemsetD32Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemsetD2D8Async + _F_cuGetProcAddress_v2('cuMemsetD2D8Async', &__cuMemsetD2D8Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemsetD2D16Async + _F_cuGetProcAddress_v2('cuMemsetD2D16Async', &__cuMemsetD2D16Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemsetD2D32Async + _F_cuGetProcAddress_v2('cuMemsetD2D32Async', &__cuMemsetD2D32Async, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemBatchDecompressAsync + _F_cuGetProcAddress_v2('cuMemBatchDecompressAsync', &__cuMemBatchDecompressAsync, 12060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemMapArrayAsync + _F_cuGetProcAddress_v2('cuMemMapArrayAsync', &__cuMemMapArrayAsync, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemFreeAsync + _F_cuGetProcAddress_v2('cuMemFreeAsync', &__cuMemFreeAsync, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemAllocAsync + _F_cuGetProcAddress_v2('cuMemAllocAsync', &__cuMemAllocAsync, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemAllocFromPoolAsync + _F_cuGetProcAddress_v2('cuMemAllocFromPoolAsync', &__cuMemAllocFromPoolAsync, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemPrefetchAsync + _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemPrefetchAsync_v2 + _F_cuGetProcAddress_v2('cuMemPrefetchAsync', &__cuMemPrefetchAsync_v2, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamGetPriority + _F_cuGetProcAddress_v2('cuStreamGetPriority', &__cuStreamGetPriority, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamGetDevice + _F_cuGetProcAddress_v2('cuStreamGetDevice', &__cuStreamGetDevice, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamGetFlags + _F_cuGetProcAddress_v2('cuStreamGetFlags', &__cuStreamGetFlags, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamGetId + _F_cuGetProcAddress_v2('cuStreamGetId', &__cuStreamGetId, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamGetCtx + _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx, 9020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamGetCtx_v2 + _F_cuGetProcAddress_v2('cuStreamGetCtx', &__cuStreamGetCtx_v2, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamWaitEvent + _F_cuGetProcAddress_v2('cuStreamWaitEvent', &__cuStreamWaitEvent, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamAddCallback + _F_cuGetProcAddress_v2('cuStreamAddCallback', &__cuStreamAddCallback, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamBeginCapture_v2 + _F_cuGetProcAddress_v2('cuStreamBeginCapture', &__cuStreamBeginCapture_v2, 10010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamBeginCaptureToGraph + _F_cuGetProcAddress_v2('cuStreamBeginCaptureToGraph', &__cuStreamBeginCaptureToGraph, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamEndCapture + _F_cuGetProcAddress_v2('cuStreamEndCapture', &__cuStreamEndCapture, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamIsCapturing + _F_cuGetProcAddress_v2('cuStreamIsCapturing', &__cuStreamIsCapturing, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamGetCaptureInfo_v2 + _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v2, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamGetCaptureInfo_v3 + _F_cuGetProcAddress_v2('cuStreamGetCaptureInfo', &__cuStreamGetCaptureInfo_v3, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamUpdateCaptureDependencies + _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamUpdateCaptureDependencies_v2 + _F_cuGetProcAddress_v2('cuStreamUpdateCaptureDependencies', &__cuStreamUpdateCaptureDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamAttachMemAsync + _F_cuGetProcAddress_v2('cuStreamAttachMemAsync', &__cuStreamAttachMemAsync, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamQuery + _F_cuGetProcAddress_v2('cuStreamQuery', &__cuStreamQuery, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamSynchronize + _F_cuGetProcAddress_v2('cuStreamSynchronize', &__cuStreamSynchronize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamCopyAttributes + _F_cuGetProcAddress_v2('cuStreamCopyAttributes', &__cuStreamCopyAttributes, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamGetAttribute + _F_cuGetProcAddress_v2('cuStreamGetAttribute', &__cuStreamGetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamSetAttribute + _F_cuGetProcAddress_v2('cuStreamSetAttribute', &__cuStreamSetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEventRecord + _F_cuGetProcAddress_v2('cuEventRecord', &__cuEventRecord, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEventRecordWithFlags + _F_cuGetProcAddress_v2('cuEventRecordWithFlags', &__cuEventRecordWithFlags, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuSignalExternalSemaphoresAsync + _F_cuGetProcAddress_v2('cuSignalExternalSemaphoresAsync', &__cuSignalExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuWaitExternalSemaphoresAsync + _F_cuGetProcAddress_v2('cuWaitExternalSemaphoresAsync', &__cuWaitExternalSemaphoresAsync, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamWaitValue32_v2 + _F_cuGetProcAddress_v2('cuStreamWaitValue32', &__cuStreamWaitValue32_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamWaitValue64_v2 + _F_cuGetProcAddress_v2('cuStreamWaitValue64', &__cuStreamWaitValue64_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamWriteValue32_v2 + _F_cuGetProcAddress_v2('cuStreamWriteValue32', &__cuStreamWriteValue32_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamWriteValue64_v2 + _F_cuGetProcAddress_v2('cuStreamWriteValue64', &__cuStreamWriteValue64_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamBatchMemOp_v2 + _F_cuGetProcAddress_v2('cuStreamBatchMemOp', &__cuStreamBatchMemOp_v2, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLaunchKernel + _F_cuGetProcAddress_v2('cuLaunchKernel', &__cuLaunchKernel, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLaunchKernelEx + _F_cuGetProcAddress_v2('cuLaunchKernelEx', &__cuLaunchKernelEx, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLaunchCooperativeKernel + _F_cuGetProcAddress_v2('cuLaunchCooperativeKernel', &__cuLaunchCooperativeKernel, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLaunchHostFunc + _F_cuGetProcAddress_v2('cuLaunchHostFunc', &__cuLaunchHostFunc, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphInstantiateWithParams + _F_cuGetProcAddress_v2('cuGraphInstantiateWithParams', &__cuGraphInstantiateWithParams, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphUpload + _F_cuGetProcAddress_v2('cuGraphUpload', &__cuGraphUpload, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphLaunch + _F_cuGetProcAddress_v2('cuGraphLaunch', &__cuGraphLaunch, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphicsMapResources + _F_cuGetProcAddress_v2('cuGraphicsMapResources', &__cuGraphicsMapResources, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphicsUnmapResources + _F_cuGetProcAddress_v2('cuGraphicsUnmapResources', &__cuGraphicsUnmapResources, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + # Get remaining functions + global __cuGetErrorString + _F_cuGetProcAddress_v2('cuGetErrorString', &__cuGetErrorString, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGetErrorName + _F_cuGetProcAddress_v2('cuGetErrorName', &__cuGetErrorName, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuInit + _F_cuGetProcAddress_v2('cuInit', &__cuInit, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDriverGetVersion + _F_cuGetProcAddress_v2('cuDriverGetVersion', &__cuDriverGetVersion, 2020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGet + _F_cuGetProcAddress_v2('cuDeviceGet', &__cuDeviceGet, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetCount + _F_cuGetProcAddress_v2('cuDeviceGetCount', &__cuDeviceGetCount, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetName + _F_cuGetProcAddress_v2('cuDeviceGetName', &__cuDeviceGetName, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetUuid + _F_cuGetProcAddress_v2('cuDeviceGetUuid', &__cuDeviceGetUuid, 9020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetUuid_v2 + _F_cuGetProcAddress_v2('cuDeviceGetUuid', &__cuDeviceGetUuid_v2, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetLuid + _F_cuGetProcAddress_v2('cuDeviceGetLuid', &__cuDeviceGetLuid, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceTotalMem_v2 + _F_cuGetProcAddress_v2('cuDeviceTotalMem', &__cuDeviceTotalMem_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetTexture1DLinearMaxWidth + _F_cuGetProcAddress_v2('cuDeviceGetTexture1DLinearMaxWidth', &__cuDeviceGetTexture1DLinearMaxWidth, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetAttribute + _F_cuGetProcAddress_v2('cuDeviceGetAttribute', &__cuDeviceGetAttribute, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetNvSciSyncAttributes + _F_cuGetProcAddress_v2('cuDeviceGetNvSciSyncAttributes', &__cuDeviceGetNvSciSyncAttributes, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceSetMemPool + _F_cuGetProcAddress_v2('cuDeviceSetMemPool', &__cuDeviceSetMemPool, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetMemPool + _F_cuGetProcAddress_v2('cuDeviceGetMemPool', &__cuDeviceGetMemPool, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetDefaultMemPool + _F_cuGetProcAddress_v2('cuDeviceGetDefaultMemPool', &__cuDeviceGetDefaultMemPool, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetExecAffinitySupport + _F_cuGetProcAddress_v2('cuDeviceGetExecAffinitySupport', &__cuDeviceGetExecAffinitySupport, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuFlushGPUDirectRDMAWrites + _F_cuGetProcAddress_v2('cuFlushGPUDirectRDMAWrites', &__cuFlushGPUDirectRDMAWrites, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetProperties + _F_cuGetProcAddress_v2('cuDeviceGetProperties', &__cuDeviceGetProperties, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceComputeCapability + _F_cuGetProcAddress_v2('cuDeviceComputeCapability', &__cuDeviceComputeCapability, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDevicePrimaryCtxRetain + _F_cuGetProcAddress_v2('cuDevicePrimaryCtxRetain', &__cuDevicePrimaryCtxRetain, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDevicePrimaryCtxRelease_v2 + _F_cuGetProcAddress_v2('cuDevicePrimaryCtxRelease', &__cuDevicePrimaryCtxRelease_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDevicePrimaryCtxSetFlags_v2 + _F_cuGetProcAddress_v2('cuDevicePrimaryCtxSetFlags', &__cuDevicePrimaryCtxSetFlags_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDevicePrimaryCtxGetState + _F_cuGetProcAddress_v2('cuDevicePrimaryCtxGetState', &__cuDevicePrimaryCtxGetState, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDevicePrimaryCtxReset_v2 + _F_cuGetProcAddress_v2('cuDevicePrimaryCtxReset', &__cuDevicePrimaryCtxReset_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxCreate_v2 + _F_cuGetProcAddress_v2('cuCtxCreate', &__cuCtxCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxCreate_v3 + _F_cuGetProcAddress_v2('cuCtxCreate', &__cuCtxCreate_v3, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxCreate_v4 + _F_cuGetProcAddress_v2('cuCtxCreate', &__cuCtxCreate_v4, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxDestroy_v2 + _F_cuGetProcAddress_v2('cuCtxDestroy', &__cuCtxDestroy_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxPushCurrent_v2 + _F_cuGetProcAddress_v2('cuCtxPushCurrent', &__cuCtxPushCurrent_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxPopCurrent_v2 + _F_cuGetProcAddress_v2('cuCtxPopCurrent', &__cuCtxPopCurrent_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxSetCurrent + _F_cuGetProcAddress_v2('cuCtxSetCurrent', &__cuCtxSetCurrent, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxGetCurrent + _F_cuGetProcAddress_v2('cuCtxGetCurrent', &__cuCtxGetCurrent, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxGetDevice + _F_cuGetProcAddress_v2('cuCtxGetDevice', &__cuCtxGetDevice, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxGetFlags + _F_cuGetProcAddress_v2('cuCtxGetFlags', &__cuCtxGetFlags, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxSetFlags + _F_cuGetProcAddress_v2('cuCtxSetFlags', &__cuCtxSetFlags, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxGetId + _F_cuGetProcAddress_v2('cuCtxGetId', &__cuCtxGetId, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxSynchronize + _F_cuGetProcAddress_v2('cuCtxSynchronize', &__cuCtxSynchronize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxSetLimit + _F_cuGetProcAddress_v2('cuCtxSetLimit', &__cuCtxSetLimit, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxGetLimit + _F_cuGetProcAddress_v2('cuCtxGetLimit', &__cuCtxGetLimit, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxGetCacheConfig + _F_cuGetProcAddress_v2('cuCtxGetCacheConfig', &__cuCtxGetCacheConfig, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxSetCacheConfig + _F_cuGetProcAddress_v2('cuCtxSetCacheConfig', &__cuCtxSetCacheConfig, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxGetApiVersion + _F_cuGetProcAddress_v2('cuCtxGetApiVersion', &__cuCtxGetApiVersion, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxGetStreamPriorityRange + _F_cuGetProcAddress_v2('cuCtxGetStreamPriorityRange', &__cuCtxGetStreamPriorityRange, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxResetPersistingL2Cache + _F_cuGetProcAddress_v2('cuCtxResetPersistingL2Cache', &__cuCtxResetPersistingL2Cache, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxGetExecAffinity + _F_cuGetProcAddress_v2('cuCtxGetExecAffinity', &__cuCtxGetExecAffinity, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxRecordEvent + _F_cuGetProcAddress_v2('cuCtxRecordEvent', &__cuCtxRecordEvent, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxWaitEvent + _F_cuGetProcAddress_v2('cuCtxWaitEvent', &__cuCtxWaitEvent, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxAttach + _F_cuGetProcAddress_v2('cuCtxAttach', &__cuCtxAttach, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxDetach + _F_cuGetProcAddress_v2('cuCtxDetach', &__cuCtxDetach, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxGetSharedMemConfig + _F_cuGetProcAddress_v2('cuCtxGetSharedMemConfig', &__cuCtxGetSharedMemConfig, 4020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxSetSharedMemConfig + _F_cuGetProcAddress_v2('cuCtxSetSharedMemConfig', &__cuCtxSetSharedMemConfig, 4020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuModuleLoad + _F_cuGetProcAddress_v2('cuModuleLoad', &__cuModuleLoad, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuModuleLoadData + _F_cuGetProcAddress_v2('cuModuleLoadData', &__cuModuleLoadData, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuModuleLoadDataEx + _F_cuGetProcAddress_v2('cuModuleLoadDataEx', &__cuModuleLoadDataEx, 2010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuModuleLoadFatBinary + _F_cuGetProcAddress_v2('cuModuleLoadFatBinary', &__cuModuleLoadFatBinary, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuModuleUnload + _F_cuGetProcAddress_v2('cuModuleUnload', &__cuModuleUnload, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuModuleGetLoadingMode + _F_cuGetProcAddress_v2('cuModuleGetLoadingMode', &__cuModuleGetLoadingMode, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuModuleGetFunction + _F_cuGetProcAddress_v2('cuModuleGetFunction', &__cuModuleGetFunction, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuModuleGetFunctionCount + _F_cuGetProcAddress_v2('cuModuleGetFunctionCount', &__cuModuleGetFunctionCount, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuModuleEnumerateFunctions + _F_cuGetProcAddress_v2('cuModuleEnumerateFunctions', &__cuModuleEnumerateFunctions, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuModuleGetGlobal_v2 + _F_cuGetProcAddress_v2('cuModuleGetGlobal', &__cuModuleGetGlobal_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLinkCreate_v2 + _F_cuGetProcAddress_v2('cuLinkCreate', &__cuLinkCreate_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLinkAddData_v2 + _F_cuGetProcAddress_v2('cuLinkAddData', &__cuLinkAddData_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLinkAddFile_v2 + _F_cuGetProcAddress_v2('cuLinkAddFile', &__cuLinkAddFile_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLinkComplete + _F_cuGetProcAddress_v2('cuLinkComplete', &__cuLinkComplete, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLinkDestroy + _F_cuGetProcAddress_v2('cuLinkDestroy', &__cuLinkDestroy, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuModuleGetTexRef + _F_cuGetProcAddress_v2('cuModuleGetTexRef', &__cuModuleGetTexRef, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuModuleGetSurfRef + _F_cuGetProcAddress_v2('cuModuleGetSurfRef', &__cuModuleGetSurfRef, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLibraryLoadData + _F_cuGetProcAddress_v2('cuLibraryLoadData', &__cuLibraryLoadData, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLibraryLoadFromFile + _F_cuGetProcAddress_v2('cuLibraryLoadFromFile', &__cuLibraryLoadFromFile, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLibraryUnload + _F_cuGetProcAddress_v2('cuLibraryUnload', &__cuLibraryUnload, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLibraryGetKernel + _F_cuGetProcAddress_v2('cuLibraryGetKernel', &__cuLibraryGetKernel, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLibraryGetKernelCount + _F_cuGetProcAddress_v2('cuLibraryGetKernelCount', &__cuLibraryGetKernelCount, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLibraryEnumerateKernels + _F_cuGetProcAddress_v2('cuLibraryEnumerateKernels', &__cuLibraryEnumerateKernels, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLibraryGetModule + _F_cuGetProcAddress_v2('cuLibraryGetModule', &__cuLibraryGetModule, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuKernelGetFunction + _F_cuGetProcAddress_v2('cuKernelGetFunction', &__cuKernelGetFunction, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuKernelGetLibrary + _F_cuGetProcAddress_v2('cuKernelGetLibrary', &__cuKernelGetLibrary, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLibraryGetGlobal + _F_cuGetProcAddress_v2('cuLibraryGetGlobal', &__cuLibraryGetGlobal, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLibraryGetManaged + _F_cuGetProcAddress_v2('cuLibraryGetManaged', &__cuLibraryGetManaged, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLibraryGetUnifiedFunction + _F_cuGetProcAddress_v2('cuLibraryGetUnifiedFunction', &__cuLibraryGetUnifiedFunction, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuKernelGetAttribute + _F_cuGetProcAddress_v2('cuKernelGetAttribute', &__cuKernelGetAttribute, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuKernelSetAttribute + _F_cuGetProcAddress_v2('cuKernelSetAttribute', &__cuKernelSetAttribute, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuKernelSetCacheConfig + _F_cuGetProcAddress_v2('cuKernelSetCacheConfig', &__cuKernelSetCacheConfig, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuKernelGetName + _F_cuGetProcAddress_v2('cuKernelGetName', &__cuKernelGetName, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuKernelGetParamInfo + _F_cuGetProcAddress_v2('cuKernelGetParamInfo', &__cuKernelGetParamInfo, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemGetInfo_v2 + _F_cuGetProcAddress_v2('cuMemGetInfo', &__cuMemGetInfo_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemAlloc_v2 + _F_cuGetProcAddress_v2('cuMemAlloc', &__cuMemAlloc_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemAllocPitch_v2 + _F_cuGetProcAddress_v2('cuMemAllocPitch', &__cuMemAllocPitch_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemFree_v2 + _F_cuGetProcAddress_v2('cuMemFree', &__cuMemFree_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemGetAddressRange_v2 + _F_cuGetProcAddress_v2('cuMemGetAddressRange', &__cuMemGetAddressRange_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemAllocHost_v2 + _F_cuGetProcAddress_v2('cuMemAllocHost', &__cuMemAllocHost_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemFreeHost + _F_cuGetProcAddress_v2('cuMemFreeHost', &__cuMemFreeHost, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemHostAlloc + _F_cuGetProcAddress_v2('cuMemHostAlloc', &__cuMemHostAlloc, 2020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemHostGetDevicePointer_v2 + _F_cuGetProcAddress_v2('cuMemHostGetDevicePointer', &__cuMemHostGetDevicePointer_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemHostGetFlags + _F_cuGetProcAddress_v2('cuMemHostGetFlags', &__cuMemHostGetFlags, 2030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemAllocManaged + _F_cuGetProcAddress_v2('cuMemAllocManaged', &__cuMemAllocManaged, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceRegisterAsyncNotification + _F_cuGetProcAddress_v2('cuDeviceRegisterAsyncNotification', &__cuDeviceRegisterAsyncNotification, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceUnregisterAsyncNotification + _F_cuGetProcAddress_v2('cuDeviceUnregisterAsyncNotification', &__cuDeviceUnregisterAsyncNotification, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetByPCIBusId + _F_cuGetProcAddress_v2('cuDeviceGetByPCIBusId', &__cuDeviceGetByPCIBusId, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetPCIBusId + _F_cuGetProcAddress_v2('cuDeviceGetPCIBusId', &__cuDeviceGetPCIBusId, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuIpcGetEventHandle + _F_cuGetProcAddress_v2('cuIpcGetEventHandle', &__cuIpcGetEventHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuIpcOpenEventHandle + _F_cuGetProcAddress_v2('cuIpcOpenEventHandle', &__cuIpcOpenEventHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuIpcGetMemHandle + _F_cuGetProcAddress_v2('cuIpcGetMemHandle', &__cuIpcGetMemHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuIpcOpenMemHandle_v2 + _F_cuGetProcAddress_v2('cuIpcOpenMemHandle', &__cuIpcOpenMemHandle_v2, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuIpcCloseMemHandle + _F_cuGetProcAddress_v2('cuIpcCloseMemHandle', &__cuIpcCloseMemHandle, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemHostRegister_v2 + _F_cuGetProcAddress_v2('cuMemHostRegister', &__cuMemHostRegister_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemHostUnregister + _F_cuGetProcAddress_v2('cuMemHostUnregister', &__cuMemHostUnregister, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuArrayCreate_v2 + _F_cuGetProcAddress_v2('cuArrayCreate', &__cuArrayCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuArrayGetDescriptor_v2 + _F_cuGetProcAddress_v2('cuArrayGetDescriptor', &__cuArrayGetDescriptor_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuArrayGetSparseProperties + _F_cuGetProcAddress_v2('cuArrayGetSparseProperties', &__cuArrayGetSparseProperties, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMipmappedArrayGetSparseProperties + _F_cuGetProcAddress_v2('cuMipmappedArrayGetSparseProperties', &__cuMipmappedArrayGetSparseProperties, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuArrayGetMemoryRequirements + _F_cuGetProcAddress_v2('cuArrayGetMemoryRequirements', &__cuArrayGetMemoryRequirements, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMipmappedArrayGetMemoryRequirements + _F_cuGetProcAddress_v2('cuMipmappedArrayGetMemoryRequirements', &__cuMipmappedArrayGetMemoryRequirements, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuArrayGetPlane + _F_cuGetProcAddress_v2('cuArrayGetPlane', &__cuArrayGetPlane, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuArrayDestroy + _F_cuGetProcAddress_v2('cuArrayDestroy', &__cuArrayDestroy, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuArray3DCreate_v2 + _F_cuGetProcAddress_v2('cuArray3DCreate', &__cuArray3DCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuArray3DGetDescriptor_v2 + _F_cuGetProcAddress_v2('cuArray3DGetDescriptor', &__cuArray3DGetDescriptor_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMipmappedArrayCreate + _F_cuGetProcAddress_v2('cuMipmappedArrayCreate', &__cuMipmappedArrayCreate, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMipmappedArrayGetLevel + _F_cuGetProcAddress_v2('cuMipmappedArrayGetLevel', &__cuMipmappedArrayGetLevel, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMipmappedArrayDestroy + _F_cuGetProcAddress_v2('cuMipmappedArrayDestroy', &__cuMipmappedArrayDestroy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemGetHandleForAddressRange + _F_cuGetProcAddress_v2('cuMemGetHandleForAddressRange', &__cuMemGetHandleForAddressRange, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemAddressReserve + _F_cuGetProcAddress_v2('cuMemAddressReserve', &__cuMemAddressReserve, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemAddressFree + _F_cuGetProcAddress_v2('cuMemAddressFree', &__cuMemAddressFree, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemCreate + _F_cuGetProcAddress_v2('cuMemCreate', &__cuMemCreate, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemRelease + _F_cuGetProcAddress_v2('cuMemRelease', &__cuMemRelease, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemMap + _F_cuGetProcAddress_v2('cuMemMap', &__cuMemMap, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemUnmap + _F_cuGetProcAddress_v2('cuMemUnmap', &__cuMemUnmap, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemSetAccess + _F_cuGetProcAddress_v2('cuMemSetAccess', &__cuMemSetAccess, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemGetAccess + _F_cuGetProcAddress_v2('cuMemGetAccess', &__cuMemGetAccess, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemExportToShareableHandle + _F_cuGetProcAddress_v2('cuMemExportToShareableHandle', &__cuMemExportToShareableHandle, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemImportFromShareableHandle + _F_cuGetProcAddress_v2('cuMemImportFromShareableHandle', &__cuMemImportFromShareableHandle, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemGetAllocationGranularity + _F_cuGetProcAddress_v2('cuMemGetAllocationGranularity', &__cuMemGetAllocationGranularity, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemGetAllocationPropertiesFromHandle + _F_cuGetProcAddress_v2('cuMemGetAllocationPropertiesFromHandle', &__cuMemGetAllocationPropertiesFromHandle, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemRetainAllocationHandle + _F_cuGetProcAddress_v2('cuMemRetainAllocationHandle', &__cuMemRetainAllocationHandle, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemPoolTrimTo + _F_cuGetProcAddress_v2('cuMemPoolTrimTo', &__cuMemPoolTrimTo, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemPoolSetAttribute + _F_cuGetProcAddress_v2('cuMemPoolSetAttribute', &__cuMemPoolSetAttribute, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemPoolGetAttribute + _F_cuGetProcAddress_v2('cuMemPoolGetAttribute', &__cuMemPoolGetAttribute, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemPoolSetAccess + _F_cuGetProcAddress_v2('cuMemPoolSetAccess', &__cuMemPoolSetAccess, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemPoolGetAccess + _F_cuGetProcAddress_v2('cuMemPoolGetAccess', &__cuMemPoolGetAccess, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemPoolCreate + _F_cuGetProcAddress_v2('cuMemPoolCreate', &__cuMemPoolCreate, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemPoolDestroy + _F_cuGetProcAddress_v2('cuMemPoolDestroy', &__cuMemPoolDestroy, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemPoolExportToShareableHandle + _F_cuGetProcAddress_v2('cuMemPoolExportToShareableHandle', &__cuMemPoolExportToShareableHandle, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemPoolImportFromShareableHandle + _F_cuGetProcAddress_v2('cuMemPoolImportFromShareableHandle', &__cuMemPoolImportFromShareableHandle, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemPoolExportPointer + _F_cuGetProcAddress_v2('cuMemPoolExportPointer', &__cuMemPoolExportPointer, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemPoolImportPointer + _F_cuGetProcAddress_v2('cuMemPoolImportPointer', &__cuMemPoolImportPointer, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMulticastCreate + _F_cuGetProcAddress_v2('cuMulticastCreate', &__cuMulticastCreate, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMulticastAddDevice + _F_cuGetProcAddress_v2('cuMulticastAddDevice', &__cuMulticastAddDevice, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMulticastBindMem + _F_cuGetProcAddress_v2('cuMulticastBindMem', &__cuMulticastBindMem, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMulticastBindAddr + _F_cuGetProcAddress_v2('cuMulticastBindAddr', &__cuMulticastBindAddr, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMulticastUnbind + _F_cuGetProcAddress_v2('cuMulticastUnbind', &__cuMulticastUnbind, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMulticastGetGranularity + _F_cuGetProcAddress_v2('cuMulticastGetGranularity', &__cuMulticastGetGranularity, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuPointerGetAttribute + _F_cuGetProcAddress_v2('cuPointerGetAttribute', &__cuPointerGetAttribute, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemAdvise + _F_cuGetProcAddress_v2('cuMemAdvise', &__cuMemAdvise, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemAdvise_v2 + _F_cuGetProcAddress_v2('cuMemAdvise', &__cuMemAdvise_v2, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemRangeGetAttribute + _F_cuGetProcAddress_v2('cuMemRangeGetAttribute', &__cuMemRangeGetAttribute, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuMemRangeGetAttributes + _F_cuGetProcAddress_v2('cuMemRangeGetAttributes', &__cuMemRangeGetAttributes, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuPointerSetAttribute + _F_cuGetProcAddress_v2('cuPointerSetAttribute', &__cuPointerSetAttribute, 6000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuPointerGetAttributes + _F_cuGetProcAddress_v2('cuPointerGetAttributes', &__cuPointerGetAttributes, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamCreate + _F_cuGetProcAddress_v2('cuStreamCreate', &__cuStreamCreate, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamCreateWithPriority + _F_cuGetProcAddress_v2('cuStreamCreateWithPriority', &__cuStreamCreateWithPriority, 5050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuThreadExchangeStreamCaptureMode + _F_cuGetProcAddress_v2('cuThreadExchangeStreamCaptureMode', &__cuThreadExchangeStreamCaptureMode, 10010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamDestroy_v2 + _F_cuGetProcAddress_v2('cuStreamDestroy', &__cuStreamDestroy_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEventCreate + _F_cuGetProcAddress_v2('cuEventCreate', &__cuEventCreate, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEventQuery + _F_cuGetProcAddress_v2('cuEventQuery', &__cuEventQuery, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEventSynchronize + _F_cuGetProcAddress_v2('cuEventSynchronize', &__cuEventSynchronize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEventDestroy_v2 + _F_cuGetProcAddress_v2('cuEventDestroy', &__cuEventDestroy_v2, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEventElapsedTime + _F_cuGetProcAddress_v2('cuEventElapsedTime', &__cuEventElapsedTime, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEventElapsedTime_v2 + _F_cuGetProcAddress_v2('cuEventElapsedTime', &__cuEventElapsedTime_v2, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuImportExternalMemory + _F_cuGetProcAddress_v2('cuImportExternalMemory', &__cuImportExternalMemory, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuExternalMemoryGetMappedBuffer + _F_cuGetProcAddress_v2('cuExternalMemoryGetMappedBuffer', &__cuExternalMemoryGetMappedBuffer, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuExternalMemoryGetMappedMipmappedArray + _F_cuGetProcAddress_v2('cuExternalMemoryGetMappedMipmappedArray', &__cuExternalMemoryGetMappedMipmappedArray, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDestroyExternalMemory + _F_cuGetProcAddress_v2('cuDestroyExternalMemory', &__cuDestroyExternalMemory, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuImportExternalSemaphore + _F_cuGetProcAddress_v2('cuImportExternalSemaphore', &__cuImportExternalSemaphore, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDestroyExternalSemaphore + _F_cuGetProcAddress_v2('cuDestroyExternalSemaphore', &__cuDestroyExternalSemaphore, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuFuncGetAttribute + _F_cuGetProcAddress_v2('cuFuncGetAttribute', &__cuFuncGetAttribute, 2020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuFuncSetAttribute + _F_cuGetProcAddress_v2('cuFuncSetAttribute', &__cuFuncSetAttribute, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuFuncSetCacheConfig + _F_cuGetProcAddress_v2('cuFuncSetCacheConfig', &__cuFuncSetCacheConfig, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuFuncGetModule + _F_cuGetProcAddress_v2('cuFuncGetModule', &__cuFuncGetModule, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuFuncGetName + _F_cuGetProcAddress_v2('cuFuncGetName', &__cuFuncGetName, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuFuncGetParamInfo + _F_cuGetProcAddress_v2('cuFuncGetParamInfo', &__cuFuncGetParamInfo, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuFuncIsLoaded + _F_cuGetProcAddress_v2('cuFuncIsLoaded', &__cuFuncIsLoaded, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuFuncLoad + _F_cuGetProcAddress_v2('cuFuncLoad', &__cuFuncLoad, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLaunchCooperativeKernelMultiDevice + _F_cuGetProcAddress_v2('cuLaunchCooperativeKernelMultiDevice', &__cuLaunchCooperativeKernelMultiDevice, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuFuncSetBlockShape + _F_cuGetProcAddress_v2('cuFuncSetBlockShape', &__cuFuncSetBlockShape, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuFuncSetSharedSize + _F_cuGetProcAddress_v2('cuFuncSetSharedSize', &__cuFuncSetSharedSize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuParamSetSize + _F_cuGetProcAddress_v2('cuParamSetSize', &__cuParamSetSize, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuParamSeti + _F_cuGetProcAddress_v2('cuParamSeti', &__cuParamSeti, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuParamSetf + _F_cuGetProcAddress_v2('cuParamSetf', &__cuParamSetf, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuParamSetv + _F_cuGetProcAddress_v2('cuParamSetv', &__cuParamSetv, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLaunch + _F_cuGetProcAddress_v2('cuLaunch', &__cuLaunch, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLaunchGrid + _F_cuGetProcAddress_v2('cuLaunchGrid', &__cuLaunchGrid, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLaunchGridAsync + _F_cuGetProcAddress_v2('cuLaunchGridAsync', &__cuLaunchGridAsync, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuParamSetTexRef + _F_cuGetProcAddress_v2('cuParamSetTexRef', &__cuParamSetTexRef, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuFuncSetSharedMemConfig + _F_cuGetProcAddress_v2('cuFuncSetSharedMemConfig', &__cuFuncSetSharedMemConfig, 4020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphCreate + _F_cuGetProcAddress_v2('cuGraphCreate', &__cuGraphCreate, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddKernelNode_v2 + _F_cuGetProcAddress_v2('cuGraphAddKernelNode', &__cuGraphAddKernelNode_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphKernelNodeGetParams_v2 + _F_cuGetProcAddress_v2('cuGraphKernelNodeGetParams', &__cuGraphKernelNodeGetParams_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphKernelNodeSetParams_v2 + _F_cuGetProcAddress_v2('cuGraphKernelNodeSetParams', &__cuGraphKernelNodeSetParams_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddMemcpyNode + _F_cuGetProcAddress_v2('cuGraphAddMemcpyNode', &__cuGraphAddMemcpyNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphMemcpyNodeGetParams + _F_cuGetProcAddress_v2('cuGraphMemcpyNodeGetParams', &__cuGraphMemcpyNodeGetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphMemcpyNodeSetParams + _F_cuGetProcAddress_v2('cuGraphMemcpyNodeSetParams', &__cuGraphMemcpyNodeSetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddMemsetNode + _F_cuGetProcAddress_v2('cuGraphAddMemsetNode', &__cuGraphAddMemsetNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphMemsetNodeGetParams + _F_cuGetProcAddress_v2('cuGraphMemsetNodeGetParams', &__cuGraphMemsetNodeGetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphMemsetNodeSetParams + _F_cuGetProcAddress_v2('cuGraphMemsetNodeSetParams', &__cuGraphMemsetNodeSetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddHostNode + _F_cuGetProcAddress_v2('cuGraphAddHostNode', &__cuGraphAddHostNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphHostNodeGetParams + _F_cuGetProcAddress_v2('cuGraphHostNodeGetParams', &__cuGraphHostNodeGetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphHostNodeSetParams + _F_cuGetProcAddress_v2('cuGraphHostNodeSetParams', &__cuGraphHostNodeSetParams, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddChildGraphNode + _F_cuGetProcAddress_v2('cuGraphAddChildGraphNode', &__cuGraphAddChildGraphNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphChildGraphNodeGetGraph + _F_cuGetProcAddress_v2('cuGraphChildGraphNodeGetGraph', &__cuGraphChildGraphNodeGetGraph, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddEmptyNode + _F_cuGetProcAddress_v2('cuGraphAddEmptyNode', &__cuGraphAddEmptyNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddEventRecordNode + _F_cuGetProcAddress_v2('cuGraphAddEventRecordNode', &__cuGraphAddEventRecordNode, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphEventRecordNodeGetEvent + _F_cuGetProcAddress_v2('cuGraphEventRecordNodeGetEvent', &__cuGraphEventRecordNodeGetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphEventRecordNodeSetEvent + _F_cuGetProcAddress_v2('cuGraphEventRecordNodeSetEvent', &__cuGraphEventRecordNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddEventWaitNode + _F_cuGetProcAddress_v2('cuGraphAddEventWaitNode', &__cuGraphAddEventWaitNode, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphEventWaitNodeGetEvent + _F_cuGetProcAddress_v2('cuGraphEventWaitNodeGetEvent', &__cuGraphEventWaitNodeGetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphEventWaitNodeSetEvent + _F_cuGetProcAddress_v2('cuGraphEventWaitNodeSetEvent', &__cuGraphEventWaitNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddExternalSemaphoresSignalNode + _F_cuGetProcAddress_v2('cuGraphAddExternalSemaphoresSignalNode', &__cuGraphAddExternalSemaphoresSignalNode, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExternalSemaphoresSignalNodeGetParams + _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresSignalNodeGetParams', &__cuGraphExternalSemaphoresSignalNodeGetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExternalSemaphoresSignalNodeSetParams + _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresSignalNodeSetParams', &__cuGraphExternalSemaphoresSignalNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddExternalSemaphoresWaitNode + _F_cuGetProcAddress_v2('cuGraphAddExternalSemaphoresWaitNode', &__cuGraphAddExternalSemaphoresWaitNode, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExternalSemaphoresWaitNodeGetParams + _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresWaitNodeGetParams', &__cuGraphExternalSemaphoresWaitNodeGetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExternalSemaphoresWaitNodeSetParams + _F_cuGetProcAddress_v2('cuGraphExternalSemaphoresWaitNodeSetParams', &__cuGraphExternalSemaphoresWaitNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddBatchMemOpNode + _F_cuGetProcAddress_v2('cuGraphAddBatchMemOpNode', &__cuGraphAddBatchMemOpNode, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphBatchMemOpNodeGetParams + _F_cuGetProcAddress_v2('cuGraphBatchMemOpNodeGetParams', &__cuGraphBatchMemOpNodeGetParams, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphBatchMemOpNodeSetParams + _F_cuGetProcAddress_v2('cuGraphBatchMemOpNodeSetParams', &__cuGraphBatchMemOpNodeSetParams, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecBatchMemOpNodeSetParams + _F_cuGetProcAddress_v2('cuGraphExecBatchMemOpNodeSetParams', &__cuGraphExecBatchMemOpNodeSetParams, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddMemAllocNode + _F_cuGetProcAddress_v2('cuGraphAddMemAllocNode', &__cuGraphAddMemAllocNode, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphMemAllocNodeGetParams + _F_cuGetProcAddress_v2('cuGraphMemAllocNodeGetParams', &__cuGraphMemAllocNodeGetParams, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddMemFreeNode + _F_cuGetProcAddress_v2('cuGraphAddMemFreeNode', &__cuGraphAddMemFreeNode, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphMemFreeNodeGetParams + _F_cuGetProcAddress_v2('cuGraphMemFreeNodeGetParams', &__cuGraphMemFreeNodeGetParams, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGraphMemTrim + _F_cuGetProcAddress_v2('cuDeviceGraphMemTrim', &__cuDeviceGraphMemTrim, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetGraphMemAttribute + _F_cuGetProcAddress_v2('cuDeviceGetGraphMemAttribute', &__cuDeviceGetGraphMemAttribute, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceSetGraphMemAttribute + _F_cuGetProcAddress_v2('cuDeviceSetGraphMemAttribute', &__cuDeviceSetGraphMemAttribute, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphClone + _F_cuGetProcAddress_v2('cuGraphClone', &__cuGraphClone, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphNodeFindInClone + _F_cuGetProcAddress_v2('cuGraphNodeFindInClone', &__cuGraphNodeFindInClone, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphNodeGetType + _F_cuGetProcAddress_v2('cuGraphNodeGetType', &__cuGraphNodeGetType, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphGetNodes + _F_cuGetProcAddress_v2('cuGraphGetNodes', &__cuGraphGetNodes, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphGetRootNodes + _F_cuGetProcAddress_v2('cuGraphGetRootNodes', &__cuGraphGetRootNodes, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphGetEdges + _F_cuGetProcAddress_v2('cuGraphGetEdges', &__cuGraphGetEdges, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphGetEdges_v2 + _F_cuGetProcAddress_v2('cuGraphGetEdges', &__cuGraphGetEdges_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphNodeGetDependencies + _F_cuGetProcAddress_v2('cuGraphNodeGetDependencies', &__cuGraphNodeGetDependencies, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphNodeGetDependencies_v2 + _F_cuGetProcAddress_v2('cuGraphNodeGetDependencies', &__cuGraphNodeGetDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphNodeGetDependentNodes + _F_cuGetProcAddress_v2('cuGraphNodeGetDependentNodes', &__cuGraphNodeGetDependentNodes, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphNodeGetDependentNodes_v2 + _F_cuGetProcAddress_v2('cuGraphNodeGetDependentNodes', &__cuGraphNodeGetDependentNodes_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddDependencies + _F_cuGetProcAddress_v2('cuGraphAddDependencies', &__cuGraphAddDependencies, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddDependencies_v2 + _F_cuGetProcAddress_v2('cuGraphAddDependencies', &__cuGraphAddDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphRemoveDependencies + _F_cuGetProcAddress_v2('cuGraphRemoveDependencies', &__cuGraphRemoveDependencies, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphRemoveDependencies_v2 + _F_cuGetProcAddress_v2('cuGraphRemoveDependencies', &__cuGraphRemoveDependencies_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphDestroyNode + _F_cuGetProcAddress_v2('cuGraphDestroyNode', &__cuGraphDestroyNode, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphInstantiateWithFlags + _F_cuGetProcAddress_v2('cuGraphInstantiateWithFlags', &__cuGraphInstantiateWithFlags, 11040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecGetFlags + _F_cuGetProcAddress_v2('cuGraphExecGetFlags', &__cuGraphExecGetFlags, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecKernelNodeSetParams_v2 + _F_cuGetProcAddress_v2('cuGraphExecKernelNodeSetParams', &__cuGraphExecKernelNodeSetParams_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecMemcpyNodeSetParams + _F_cuGetProcAddress_v2('cuGraphExecMemcpyNodeSetParams', &__cuGraphExecMemcpyNodeSetParams, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecMemsetNodeSetParams + _F_cuGetProcAddress_v2('cuGraphExecMemsetNodeSetParams', &__cuGraphExecMemsetNodeSetParams, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecHostNodeSetParams + _F_cuGetProcAddress_v2('cuGraphExecHostNodeSetParams', &__cuGraphExecHostNodeSetParams, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecChildGraphNodeSetParams + _F_cuGetProcAddress_v2('cuGraphExecChildGraphNodeSetParams', &__cuGraphExecChildGraphNodeSetParams, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecEventRecordNodeSetEvent + _F_cuGetProcAddress_v2('cuGraphExecEventRecordNodeSetEvent', &__cuGraphExecEventRecordNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecEventWaitNodeSetEvent + _F_cuGetProcAddress_v2('cuGraphExecEventWaitNodeSetEvent', &__cuGraphExecEventWaitNodeSetEvent, 11010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecExternalSemaphoresSignalNodeSetParams + _F_cuGetProcAddress_v2('cuGraphExecExternalSemaphoresSignalNodeSetParams', &__cuGraphExecExternalSemaphoresSignalNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecExternalSemaphoresWaitNodeSetParams + _F_cuGetProcAddress_v2('cuGraphExecExternalSemaphoresWaitNodeSetParams', &__cuGraphExecExternalSemaphoresWaitNodeSetParams, 11020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphNodeSetEnabled + _F_cuGetProcAddress_v2('cuGraphNodeSetEnabled', &__cuGraphNodeSetEnabled, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphNodeGetEnabled + _F_cuGetProcAddress_v2('cuGraphNodeGetEnabled', &__cuGraphNodeGetEnabled, 11060, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecDestroy + _F_cuGetProcAddress_v2('cuGraphExecDestroy', &__cuGraphExecDestroy, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphDestroy + _F_cuGetProcAddress_v2('cuGraphDestroy', &__cuGraphDestroy, 10000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecUpdate_v2 + _F_cuGetProcAddress_v2('cuGraphExecUpdate', &__cuGraphExecUpdate_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphKernelNodeCopyAttributes + _F_cuGetProcAddress_v2('cuGraphKernelNodeCopyAttributes', &__cuGraphKernelNodeCopyAttributes, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphKernelNodeGetAttribute + _F_cuGetProcAddress_v2('cuGraphKernelNodeGetAttribute', &__cuGraphKernelNodeGetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphKernelNodeSetAttribute + _F_cuGetProcAddress_v2('cuGraphKernelNodeSetAttribute', &__cuGraphKernelNodeSetAttribute, 11000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphDebugDotPrint + _F_cuGetProcAddress_v2('cuGraphDebugDotPrint', &__cuGraphDebugDotPrint, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuUserObjectCreate + _F_cuGetProcAddress_v2('cuUserObjectCreate', &__cuUserObjectCreate, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuUserObjectRetain + _F_cuGetProcAddress_v2('cuUserObjectRetain', &__cuUserObjectRetain, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuUserObjectRelease + _F_cuGetProcAddress_v2('cuUserObjectRelease', &__cuUserObjectRelease, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphRetainUserObject + _F_cuGetProcAddress_v2('cuGraphRetainUserObject', &__cuGraphRetainUserObject, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphReleaseUserObject + _F_cuGetProcAddress_v2('cuGraphReleaseUserObject', &__cuGraphReleaseUserObject, 11030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddNode + _F_cuGetProcAddress_v2('cuGraphAddNode', &__cuGraphAddNode, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphAddNode_v2 + _F_cuGetProcAddress_v2('cuGraphAddNode', &__cuGraphAddNode_v2, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphNodeSetParams + _F_cuGetProcAddress_v2('cuGraphNodeSetParams', &__cuGraphNodeSetParams, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphExecNodeSetParams + _F_cuGetProcAddress_v2('cuGraphExecNodeSetParams', &__cuGraphExecNodeSetParams, 12020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphConditionalHandleCreate + _F_cuGetProcAddress_v2('cuGraphConditionalHandleCreate', &__cuGraphConditionalHandleCreate, 12030, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuOccupancyMaxActiveBlocksPerMultiprocessor + _F_cuGetProcAddress_v2('cuOccupancyMaxActiveBlocksPerMultiprocessor', &__cuOccupancyMaxActiveBlocksPerMultiprocessor, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + _F_cuGetProcAddress_v2('cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags', &__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuOccupancyMaxPotentialBlockSize + _F_cuGetProcAddress_v2('cuOccupancyMaxPotentialBlockSize', &__cuOccupancyMaxPotentialBlockSize, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuOccupancyMaxPotentialBlockSizeWithFlags + _F_cuGetProcAddress_v2('cuOccupancyMaxPotentialBlockSizeWithFlags', &__cuOccupancyMaxPotentialBlockSizeWithFlags, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuOccupancyAvailableDynamicSMemPerBlock + _F_cuGetProcAddress_v2('cuOccupancyAvailableDynamicSMemPerBlock', &__cuOccupancyAvailableDynamicSMemPerBlock, 10020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuOccupancyMaxPotentialClusterSize + _F_cuGetProcAddress_v2('cuOccupancyMaxPotentialClusterSize', &__cuOccupancyMaxPotentialClusterSize, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuOccupancyMaxActiveClusters + _F_cuGetProcAddress_v2('cuOccupancyMaxActiveClusters', &__cuOccupancyMaxActiveClusters, 11070, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefSetArray + _F_cuGetProcAddress_v2('cuTexRefSetArray', &__cuTexRefSetArray, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefSetMipmappedArray + _F_cuGetProcAddress_v2('cuTexRefSetMipmappedArray', &__cuTexRefSetMipmappedArray, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefSetAddress_v2 + _F_cuGetProcAddress_v2('cuTexRefSetAddress', &__cuTexRefSetAddress_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefSetAddress2D_v3 + _F_cuGetProcAddress_v2('cuTexRefSetAddress2D', &__cuTexRefSetAddress2D_v3, 4010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefSetFormat + _F_cuGetProcAddress_v2('cuTexRefSetFormat', &__cuTexRefSetFormat, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefSetAddressMode + _F_cuGetProcAddress_v2('cuTexRefSetAddressMode', &__cuTexRefSetAddressMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefSetFilterMode + _F_cuGetProcAddress_v2('cuTexRefSetFilterMode', &__cuTexRefSetFilterMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefSetMipmapFilterMode + _F_cuGetProcAddress_v2('cuTexRefSetMipmapFilterMode', &__cuTexRefSetMipmapFilterMode, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefSetMipmapLevelBias + _F_cuGetProcAddress_v2('cuTexRefSetMipmapLevelBias', &__cuTexRefSetMipmapLevelBias, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefSetMipmapLevelClamp + _F_cuGetProcAddress_v2('cuTexRefSetMipmapLevelClamp', &__cuTexRefSetMipmapLevelClamp, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefSetMaxAnisotropy + _F_cuGetProcAddress_v2('cuTexRefSetMaxAnisotropy', &__cuTexRefSetMaxAnisotropy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefSetBorderColor + _F_cuGetProcAddress_v2('cuTexRefSetBorderColor', &__cuTexRefSetBorderColor, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefSetFlags + _F_cuGetProcAddress_v2('cuTexRefSetFlags', &__cuTexRefSetFlags, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefGetAddress_v2 + _F_cuGetProcAddress_v2('cuTexRefGetAddress', &__cuTexRefGetAddress_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefGetArray + _F_cuGetProcAddress_v2('cuTexRefGetArray', &__cuTexRefGetArray, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefGetMipmappedArray + _F_cuGetProcAddress_v2('cuTexRefGetMipmappedArray', &__cuTexRefGetMipmappedArray, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefGetAddressMode + _F_cuGetProcAddress_v2('cuTexRefGetAddressMode', &__cuTexRefGetAddressMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefGetFilterMode + _F_cuGetProcAddress_v2('cuTexRefGetFilterMode', &__cuTexRefGetFilterMode, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefGetFormat + _F_cuGetProcAddress_v2('cuTexRefGetFormat', &__cuTexRefGetFormat, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefGetMipmapFilterMode + _F_cuGetProcAddress_v2('cuTexRefGetMipmapFilterMode', &__cuTexRefGetMipmapFilterMode, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefGetMipmapLevelBias + _F_cuGetProcAddress_v2('cuTexRefGetMipmapLevelBias', &__cuTexRefGetMipmapLevelBias, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefGetMipmapLevelClamp + _F_cuGetProcAddress_v2('cuTexRefGetMipmapLevelClamp', &__cuTexRefGetMipmapLevelClamp, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefGetMaxAnisotropy + _F_cuGetProcAddress_v2('cuTexRefGetMaxAnisotropy', &__cuTexRefGetMaxAnisotropy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefGetBorderColor + _F_cuGetProcAddress_v2('cuTexRefGetBorderColor', &__cuTexRefGetBorderColor, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefGetFlags + _F_cuGetProcAddress_v2('cuTexRefGetFlags', &__cuTexRefGetFlags, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefCreate + _F_cuGetProcAddress_v2('cuTexRefCreate', &__cuTexRefCreate, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexRefDestroy + _F_cuGetProcAddress_v2('cuTexRefDestroy', &__cuTexRefDestroy, 2000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuSurfRefSetArray + _F_cuGetProcAddress_v2('cuSurfRefSetArray', &__cuSurfRefSetArray, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuSurfRefGetArray + _F_cuGetProcAddress_v2('cuSurfRefGetArray', &__cuSurfRefGetArray, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexObjectCreate + _F_cuGetProcAddress_v2('cuTexObjectCreate', &__cuTexObjectCreate, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexObjectDestroy + _F_cuGetProcAddress_v2('cuTexObjectDestroy', &__cuTexObjectDestroy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexObjectGetResourceDesc + _F_cuGetProcAddress_v2('cuTexObjectGetResourceDesc', &__cuTexObjectGetResourceDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexObjectGetTextureDesc + _F_cuGetProcAddress_v2('cuTexObjectGetTextureDesc', &__cuTexObjectGetTextureDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTexObjectGetResourceViewDesc + _F_cuGetProcAddress_v2('cuTexObjectGetResourceViewDesc', &__cuTexObjectGetResourceViewDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuSurfObjectCreate + _F_cuGetProcAddress_v2('cuSurfObjectCreate', &__cuSurfObjectCreate, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuSurfObjectDestroy + _F_cuGetProcAddress_v2('cuSurfObjectDestroy', &__cuSurfObjectDestroy, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuSurfObjectGetResourceDesc + _F_cuGetProcAddress_v2('cuSurfObjectGetResourceDesc', &__cuSurfObjectGetResourceDesc, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTensorMapEncodeTiled + _F_cuGetProcAddress_v2('cuTensorMapEncodeTiled', &__cuTensorMapEncodeTiled, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTensorMapEncodeIm2col + _F_cuGetProcAddress_v2('cuTensorMapEncodeIm2col', &__cuTensorMapEncodeIm2col, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTensorMapEncodeIm2colWide + _F_cuGetProcAddress_v2('cuTensorMapEncodeIm2colWide', &__cuTensorMapEncodeIm2colWide, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuTensorMapReplaceAddress + _F_cuGetProcAddress_v2('cuTensorMapReplaceAddress', &__cuTensorMapReplaceAddress, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceCanAccessPeer + _F_cuGetProcAddress_v2('cuDeviceCanAccessPeer', &__cuDeviceCanAccessPeer, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxEnablePeerAccess + _F_cuGetProcAddress_v2('cuCtxEnablePeerAccess', &__cuCtxEnablePeerAccess, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxDisablePeerAccess + _F_cuGetProcAddress_v2('cuCtxDisablePeerAccess', &__cuCtxDisablePeerAccess, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetP2PAttribute + _F_cuGetProcAddress_v2('cuDeviceGetP2PAttribute', &__cuDeviceGetP2PAttribute, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphicsUnregisterResource + _F_cuGetProcAddress_v2('cuGraphicsUnregisterResource', &__cuGraphicsUnregisterResource, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphicsSubResourceGetMappedArray + _F_cuGetProcAddress_v2('cuGraphicsSubResourceGetMappedArray', &__cuGraphicsSubResourceGetMappedArray, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphicsResourceGetMappedMipmappedArray + _F_cuGetProcAddress_v2('cuGraphicsResourceGetMappedMipmappedArray', &__cuGraphicsResourceGetMappedMipmappedArray, 5000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphicsResourceGetMappedPointer_v2 + _F_cuGetProcAddress_v2('cuGraphicsResourceGetMappedPointer', &__cuGraphicsResourceGetMappedPointer_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphicsResourceSetMapFlags_v2 + _F_cuGetProcAddress_v2('cuGraphicsResourceSetMapFlags', &__cuGraphicsResourceSetMapFlags_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGetProcAddress_v2 + _F_cuGetProcAddress_v2('cuGetProcAddress', &__cuGetProcAddress_v2, 12000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCoredumpGetAttribute + _F_cuGetProcAddress_v2('cuCoredumpGetAttribute', &__cuCoredumpGetAttribute, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCoredumpGetAttributeGlobal + _F_cuGetProcAddress_v2('cuCoredumpGetAttributeGlobal', &__cuCoredumpGetAttributeGlobal, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCoredumpSetAttribute + _F_cuGetProcAddress_v2('cuCoredumpSetAttribute', &__cuCoredumpSetAttribute, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCoredumpSetAttributeGlobal + _F_cuGetProcAddress_v2('cuCoredumpSetAttributeGlobal', &__cuCoredumpSetAttributeGlobal, 12010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGetExportTable + _F_cuGetProcAddress_v2('cuGetExportTable', &__cuGetExportTable, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGreenCtxCreate + _F_cuGetProcAddress_v2('cuGreenCtxCreate', &__cuGreenCtxCreate, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGreenCtxDestroy + _F_cuGetProcAddress_v2('cuGreenCtxDestroy', &__cuGreenCtxDestroy, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxFromGreenCtx + _F_cuGetProcAddress_v2('cuCtxFromGreenCtx', &__cuCtxFromGreenCtx, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDeviceGetDevResource + _F_cuGetProcAddress_v2('cuDeviceGetDevResource', &__cuDeviceGetDevResource, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCtxGetDevResource + _F_cuGetProcAddress_v2('cuCtxGetDevResource', &__cuCtxGetDevResource, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGreenCtxGetDevResource + _F_cuGetProcAddress_v2('cuGreenCtxGetDevResource', &__cuGreenCtxGetDevResource, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDevSmResourceSplitByCount + _F_cuGetProcAddress_v2('cuDevSmResourceSplitByCount', &__cuDevSmResourceSplitByCount, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuDevResourceGenerateDesc + _F_cuGetProcAddress_v2('cuDevResourceGenerateDesc', &__cuDevResourceGenerateDesc, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGreenCtxRecordEvent + _F_cuGetProcAddress_v2('cuGreenCtxRecordEvent', &__cuGreenCtxRecordEvent, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGreenCtxWaitEvent + _F_cuGetProcAddress_v2('cuGreenCtxWaitEvent', &__cuGreenCtxWaitEvent, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuStreamGetGreenCtx + _F_cuGetProcAddress_v2('cuStreamGetGreenCtx', &__cuStreamGetGreenCtx, 12040, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGreenCtxStreamCreate + _F_cuGetProcAddress_v2('cuGreenCtxStreamCreate', &__cuGreenCtxStreamCreate, 12050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLogsRegisterCallback + _F_cuGetProcAddress_v2('cuLogsRegisterCallback', &__cuLogsRegisterCallback, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLogsUnregisterCallback + _F_cuGetProcAddress_v2('cuLogsUnregisterCallback', &__cuLogsUnregisterCallback, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLogsCurrent + _F_cuGetProcAddress_v2('cuLogsCurrent', &__cuLogsCurrent, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLogsDumpToFile + _F_cuGetProcAddress_v2('cuLogsDumpToFile', &__cuLogsDumpToFile, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuLogsDumpToMemory + _F_cuGetProcAddress_v2('cuLogsDumpToMemory', &__cuLogsDumpToMemory, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCheckpointProcessGetRestoreThreadId + _F_cuGetProcAddress_v2('cuCheckpointProcessGetRestoreThreadId', &__cuCheckpointProcessGetRestoreThreadId, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCheckpointProcessGetState + _F_cuGetProcAddress_v2('cuCheckpointProcessGetState', &__cuCheckpointProcessGetState, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCheckpointProcessLock + _F_cuGetProcAddress_v2('cuCheckpointProcessLock', &__cuCheckpointProcessLock, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCheckpointProcessCheckpoint + _F_cuGetProcAddress_v2('cuCheckpointProcessCheckpoint', &__cuCheckpointProcessCheckpoint, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCheckpointProcessRestore + _F_cuGetProcAddress_v2('cuCheckpointProcessRestore', &__cuCheckpointProcessRestore, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuCheckpointProcessUnlock + _F_cuGetProcAddress_v2('cuCheckpointProcessUnlock', &__cuCheckpointProcessUnlock, 12080, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuProfilerStart + _F_cuGetProcAddress_v2('cuProfilerStart', &__cuProfilerStart, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuProfilerStop + _F_cuGetProcAddress_v2('cuProfilerStop', &__cuProfilerStop, 4000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphicsEGLRegisterImage + _F_cuGetProcAddress_v2('cuGraphicsEGLRegisterImage', &__cuGraphicsEGLRegisterImage, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEGLStreamConsumerConnect + _F_cuGetProcAddress_v2('cuEGLStreamConsumerConnect', &__cuEGLStreamConsumerConnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEGLStreamConsumerConnectWithFlags + _F_cuGetProcAddress_v2('cuEGLStreamConsumerConnectWithFlags', &__cuEGLStreamConsumerConnectWithFlags, 8000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEGLStreamConsumerDisconnect + _F_cuGetProcAddress_v2('cuEGLStreamConsumerDisconnect', &__cuEGLStreamConsumerDisconnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEGLStreamConsumerAcquireFrame + _F_cuGetProcAddress_v2('cuEGLStreamConsumerAcquireFrame', &__cuEGLStreamConsumerAcquireFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEGLStreamConsumerReleaseFrame + _F_cuGetProcAddress_v2('cuEGLStreamConsumerReleaseFrame', &__cuEGLStreamConsumerReleaseFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEGLStreamProducerConnect + _F_cuGetProcAddress_v2('cuEGLStreamProducerConnect', &__cuEGLStreamProducerConnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEGLStreamProducerDisconnect + _F_cuGetProcAddress_v2('cuEGLStreamProducerDisconnect', &__cuEGLStreamProducerDisconnect, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEGLStreamProducerPresentFrame + _F_cuGetProcAddress_v2('cuEGLStreamProducerPresentFrame', &__cuEGLStreamProducerPresentFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEGLStreamProducerReturnFrame + _F_cuGetProcAddress_v2('cuEGLStreamProducerReturnFrame', &__cuEGLStreamProducerReturnFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphicsResourceGetMappedEglFrame + _F_cuGetProcAddress_v2('cuGraphicsResourceGetMappedEglFrame', &__cuGraphicsResourceGetMappedEglFrame, 7000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuEventCreateFromEGLSync + _F_cuGetProcAddress_v2('cuEventCreateFromEGLSync', &__cuEventCreateFromEGLSync, 9000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphicsGLRegisterBuffer + _F_cuGetProcAddress_v2('cuGraphicsGLRegisterBuffer', &__cuGraphicsGLRegisterBuffer, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphicsGLRegisterImage + _F_cuGetProcAddress_v2('cuGraphicsGLRegisterImage', &__cuGraphicsGLRegisterImage, 3000, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGLGetDevices_v2 + _F_cuGetProcAddress_v2('cuGLGetDevices', &__cuGLGetDevices_v2, 6050, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuVDPAUGetDevice + _F_cuGetProcAddress_v2('cuVDPAUGetDevice', &__cuVDPAUGetDevice, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuVDPAUCtxCreate_v2 + _F_cuGetProcAddress_v2('cuVDPAUCtxCreate', &__cuVDPAUCtxCreate_v2, 3020, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphicsVDPAURegisterVideoSurface + _F_cuGetProcAddress_v2('cuGraphicsVDPAURegisterVideoSurface', &__cuGraphicsVDPAURegisterVideoSurface, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + global __cuGraphicsVDPAURegisterOutputSurface + _F_cuGetProcAddress_v2('cuGraphicsVDPAURegisterOutputSurface', &__cuGraphicsVDPAURegisterOutputSurface, 3010, CU_GET_PROC_ADDRESS_DEFAULT, NULL) + __cuPythonInit = True + return 0 + + # Load using dlsym + if usePTDS: + # Get all PTDS version of functions + pass + global __cuMemcpy + __cuMemcpy = dlfcn.dlsym(handle, 'cuMemcpy_ptds') + global __cuMemcpyPeer + __cuMemcpyPeer = dlfcn.dlsym(handle, 'cuMemcpyPeer_ptds') + global __cuMemcpyHtoD_v2 + __cuMemcpyHtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoD_v2_ptds') + global __cuMemcpyDtoH_v2 + __cuMemcpyDtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoH_v2_ptds') + global __cuMemcpyDtoD_v2 + __cuMemcpyDtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoD_v2_ptds') + global __cuMemcpyDtoA_v2 + __cuMemcpyDtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoA_v2_ptds') + global __cuMemcpyAtoD_v2 + __cuMemcpyAtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoD_v2_ptds') + global __cuMemcpyHtoA_v2 + __cuMemcpyHtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoA_v2_ptds') + global __cuMemcpyAtoH_v2 + __cuMemcpyAtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoH_v2_ptds') + global __cuMemcpyAtoA_v2 + __cuMemcpyAtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoA_v2_ptds') + global __cuMemcpy2D_v2 + __cuMemcpy2D_v2 = dlfcn.dlsym(handle, 'cuMemcpy2D_v2_ptds') + global __cuMemcpy2DUnaligned_v2 + __cuMemcpy2DUnaligned_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DUnaligned_v2_ptds') + global __cuMemcpy3D_v2 + __cuMemcpy3D_v2 = dlfcn.dlsym(handle, 'cuMemcpy3D_v2_ptds') + global __cuMemcpy3DPeer + __cuMemcpy3DPeer = dlfcn.dlsym(handle, 'cuMemcpy3DPeer_ptds') + global __cuMemcpyAsync + __cuMemcpyAsync = dlfcn.dlsym(handle, 'cuMemcpyAsync_ptsz') + global __cuMemcpyPeerAsync + __cuMemcpyPeerAsync = dlfcn.dlsym(handle, 'cuMemcpyPeerAsync_ptsz') + global __cuMemcpyHtoDAsync_v2 + __cuMemcpyHtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoDAsync_v2_ptsz') + global __cuMemcpyDtoHAsync_v2 + __cuMemcpyDtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoHAsync_v2_ptsz') + global __cuMemcpyDtoDAsync_v2 + __cuMemcpyDtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoDAsync_v2_ptsz') + global __cuMemcpyHtoAAsync_v2 + __cuMemcpyHtoAAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoAAsync_v2_ptsz') + global __cuMemcpyAtoHAsync_v2 + __cuMemcpyAtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoHAsync_v2_ptsz') + global __cuMemcpy2DAsync_v2 + __cuMemcpy2DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DAsync_v2_ptsz') + global __cuMemcpy3DAsync_v2 + __cuMemcpy3DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy3DAsync_v2_ptsz') + global __cuMemcpy3DPeerAsync + __cuMemcpy3DPeerAsync = dlfcn.dlsym(handle, 'cuMemcpy3DPeerAsync_ptsz') + global __cuMemcpyBatchAsync + __cuMemcpyBatchAsync = dlfcn.dlsym(handle, 'cuMemcpyBatchAsync_ptsz') + global __cuMemcpy3DBatchAsync + __cuMemcpy3DBatchAsync = dlfcn.dlsym(handle, 'cuMemcpy3DBatchAsync_ptsz') + global __cuMemsetD8_v2 + __cuMemsetD8_v2 = dlfcn.dlsym(handle, 'cuMemsetD8_v2_ptds') + global __cuMemsetD16_v2 + __cuMemsetD16_v2 = dlfcn.dlsym(handle, 'cuMemsetD16_v2_ptds') + global __cuMemsetD32_v2 + __cuMemsetD32_v2 = dlfcn.dlsym(handle, 'cuMemsetD32_v2_ptds') + global __cuMemsetD2D8_v2 + __cuMemsetD2D8_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D8_v2_ptds') + global __cuMemsetD2D16_v2 + __cuMemsetD2D16_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D16_v2_ptds') + global __cuMemsetD2D32_v2 + __cuMemsetD2D32_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D32_v2_ptds') + global __cuMemsetD8Async + __cuMemsetD8Async = dlfcn.dlsym(handle, 'cuMemsetD8Async_ptsz') + global __cuMemsetD16Async + __cuMemsetD16Async = dlfcn.dlsym(handle, 'cuMemsetD16Async_ptsz') + global __cuMemsetD32Async + __cuMemsetD32Async = dlfcn.dlsym(handle, 'cuMemsetD32Async_ptsz') + global __cuMemsetD2D8Async + __cuMemsetD2D8Async = dlfcn.dlsym(handle, 'cuMemsetD2D8Async_ptsz') + global __cuMemsetD2D16Async + __cuMemsetD2D16Async = dlfcn.dlsym(handle, 'cuMemsetD2D16Async_ptsz') + global __cuMemsetD2D32Async + __cuMemsetD2D32Async = dlfcn.dlsym(handle, 'cuMemsetD2D32Async_ptsz') + global __cuMemBatchDecompressAsync + __cuMemBatchDecompressAsync = dlfcn.dlsym(handle, 'cuMemBatchDecompressAsync_ptsz') + global __cuMemMapArrayAsync + __cuMemMapArrayAsync = dlfcn.dlsym(handle, 'cuMemMapArrayAsync_ptsz') + global __cuMemFreeAsync + __cuMemFreeAsync = dlfcn.dlsym(handle, 'cuMemFreeAsync_ptsz') + global __cuMemAllocAsync + __cuMemAllocAsync = dlfcn.dlsym(handle, 'cuMemAllocAsync_ptsz') + global __cuMemAllocFromPoolAsync + __cuMemAllocFromPoolAsync = dlfcn.dlsym(handle, 'cuMemAllocFromPoolAsync_ptsz') + global __cuMemPrefetchAsync + __cuMemPrefetchAsync = dlfcn.dlsym(handle, 'cuMemPrefetchAsync_ptsz') + global __cuMemPrefetchAsync_v2 + __cuMemPrefetchAsync_v2 = dlfcn.dlsym(handle, 'cuMemPrefetchAsync_v2_ptsz') + global __cuStreamGetPriority + __cuStreamGetPriority = dlfcn.dlsym(handle, 'cuStreamGetPriority_ptsz') + global __cuStreamGetDevice + __cuStreamGetDevice = dlfcn.dlsym(handle, 'cuStreamGetDevice_ptsz') + global __cuStreamGetFlags + __cuStreamGetFlags = dlfcn.dlsym(handle, 'cuStreamGetFlags_ptsz') + global __cuStreamGetId + __cuStreamGetId = dlfcn.dlsym(handle, 'cuStreamGetId_ptsz') + global __cuStreamGetCtx + __cuStreamGetCtx = dlfcn.dlsym(handle, 'cuStreamGetCtx_ptsz') + global __cuStreamGetCtx_v2 + __cuStreamGetCtx_v2 = dlfcn.dlsym(handle, 'cuStreamGetCtx_v2_ptsz') + global __cuStreamWaitEvent + __cuStreamWaitEvent = dlfcn.dlsym(handle, 'cuStreamWaitEvent_ptsz') + global __cuStreamAddCallback + __cuStreamAddCallback = dlfcn.dlsym(handle, 'cuStreamAddCallback_ptsz') + global __cuStreamBeginCapture_v2 + __cuStreamBeginCapture_v2 = dlfcn.dlsym(handle, 'cuStreamBeginCapture_v2_ptsz') + global __cuStreamBeginCaptureToGraph + __cuStreamBeginCaptureToGraph = dlfcn.dlsym(handle, 'cuStreamBeginCaptureToGraph_ptsz') + global __cuStreamEndCapture + __cuStreamEndCapture = dlfcn.dlsym(handle, 'cuStreamEndCapture_ptsz') + global __cuStreamIsCapturing + __cuStreamIsCapturing = dlfcn.dlsym(handle, 'cuStreamIsCapturing_ptsz') + global __cuStreamGetCaptureInfo_v2 + __cuStreamGetCaptureInfo_v2 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v2_ptsz') + global __cuStreamGetCaptureInfo_v3 + __cuStreamGetCaptureInfo_v3 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v3_ptsz') + global __cuStreamUpdateCaptureDependencies + __cuStreamUpdateCaptureDependencies = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies_ptsz') + global __cuStreamUpdateCaptureDependencies_v2 + __cuStreamUpdateCaptureDependencies_v2 = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies_v2_ptsz') + global __cuStreamAttachMemAsync + __cuStreamAttachMemAsync = dlfcn.dlsym(handle, 'cuStreamAttachMemAsync_ptsz') + global __cuStreamQuery + __cuStreamQuery = dlfcn.dlsym(handle, 'cuStreamQuery_ptsz') + global __cuStreamSynchronize + __cuStreamSynchronize = dlfcn.dlsym(handle, 'cuStreamSynchronize_ptsz') + global __cuStreamCopyAttributes + __cuStreamCopyAttributes = dlfcn.dlsym(handle, 'cuStreamCopyAttributes_ptsz') + global __cuStreamGetAttribute + __cuStreamGetAttribute = dlfcn.dlsym(handle, 'cuStreamGetAttribute_ptsz') + global __cuStreamSetAttribute + __cuStreamSetAttribute = dlfcn.dlsym(handle, 'cuStreamSetAttribute_ptsz') + global __cuEventRecord + __cuEventRecord = dlfcn.dlsym(handle, 'cuEventRecord_ptsz') + global __cuEventRecordWithFlags + __cuEventRecordWithFlags = dlfcn.dlsym(handle, 'cuEventRecordWithFlags_ptsz') + global __cuSignalExternalSemaphoresAsync + __cuSignalExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuSignalExternalSemaphoresAsync_ptsz') + global __cuWaitExternalSemaphoresAsync + __cuWaitExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuWaitExternalSemaphoresAsync_ptsz') + global __cuStreamWaitValue32_v2 + __cuStreamWaitValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue32_v2_ptsz') + global __cuStreamWaitValue64_v2 + __cuStreamWaitValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue64_v2_ptsz') + global __cuStreamWriteValue32_v2 + __cuStreamWriteValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue32_v2_ptsz') + global __cuStreamWriteValue64_v2 + __cuStreamWriteValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue64_v2_ptsz') + global __cuStreamBatchMemOp_v2 + __cuStreamBatchMemOp_v2 = dlfcn.dlsym(handle, 'cuStreamBatchMemOp_v2_ptsz') + global __cuLaunchKernel + __cuLaunchKernel = dlfcn.dlsym(handle, 'cuLaunchKernel_ptsz') + global __cuLaunchKernelEx + __cuLaunchKernelEx = dlfcn.dlsym(handle, 'cuLaunchKernelEx_ptsz') + global __cuLaunchCooperativeKernel + __cuLaunchCooperativeKernel = dlfcn.dlsym(handle, 'cuLaunchCooperativeKernel_ptsz') + global __cuLaunchHostFunc + __cuLaunchHostFunc = dlfcn.dlsym(handle, 'cuLaunchHostFunc_ptsz') + global __cuGraphInstantiateWithParams + __cuGraphInstantiateWithParams = dlfcn.dlsym(handle, 'cuGraphInstantiateWithParams_ptsz') + global __cuGraphUpload + __cuGraphUpload = dlfcn.dlsym(handle, 'cuGraphUpload_ptsz') + global __cuGraphLaunch + __cuGraphLaunch = dlfcn.dlsym(handle, 'cuGraphLaunch_ptsz') + global __cuGraphicsMapResources + __cuGraphicsMapResources = dlfcn.dlsym(handle, 'cuGraphicsMapResources_ptsz') + global __cuGraphicsUnmapResources + __cuGraphicsUnmapResources = dlfcn.dlsym(handle, 'cuGraphicsUnmapResources_ptsz') + else: + # Else get the regular version + pass + global __cuMemcpy + __cuMemcpy = dlfcn.dlsym(handle, 'cuMemcpy') + global __cuMemcpyPeer + __cuMemcpyPeer = dlfcn.dlsym(handle, 'cuMemcpyPeer') + global __cuMemcpyHtoD_v2 + __cuMemcpyHtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoD_v2') + global __cuMemcpyDtoH_v2 + __cuMemcpyDtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoH_v2') + global __cuMemcpyDtoD_v2 + __cuMemcpyDtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoD_v2') + global __cuMemcpyDtoA_v2 + __cuMemcpyDtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoA_v2') + global __cuMemcpyAtoD_v2 + __cuMemcpyAtoD_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoD_v2') + global __cuMemcpyHtoA_v2 + __cuMemcpyHtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoA_v2') + global __cuMemcpyAtoH_v2 + __cuMemcpyAtoH_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoH_v2') + global __cuMemcpyAtoA_v2 + __cuMemcpyAtoA_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoA_v2') + global __cuMemcpy2D_v2 + __cuMemcpy2D_v2 = dlfcn.dlsym(handle, 'cuMemcpy2D_v2') + global __cuMemcpy2DUnaligned_v2 + __cuMemcpy2DUnaligned_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DUnaligned_v2') + global __cuMemcpy3D_v2 + __cuMemcpy3D_v2 = dlfcn.dlsym(handle, 'cuMemcpy3D_v2') + global __cuMemcpy3DPeer + __cuMemcpy3DPeer = dlfcn.dlsym(handle, 'cuMemcpy3DPeer') + global __cuMemcpyAsync + __cuMemcpyAsync = dlfcn.dlsym(handle, 'cuMemcpyAsync') + global __cuMemcpyPeerAsync + __cuMemcpyPeerAsync = dlfcn.dlsym(handle, 'cuMemcpyPeerAsync') + global __cuMemcpyHtoDAsync_v2 + __cuMemcpyHtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoDAsync_v2') + global __cuMemcpyDtoHAsync_v2 + __cuMemcpyDtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoHAsync_v2') + global __cuMemcpyDtoDAsync_v2 + __cuMemcpyDtoDAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyDtoDAsync_v2') + global __cuMemcpyHtoAAsync_v2 + __cuMemcpyHtoAAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyHtoAAsync_v2') + global __cuMemcpyAtoHAsync_v2 + __cuMemcpyAtoHAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpyAtoHAsync_v2') + global __cuMemcpy2DAsync_v2 + __cuMemcpy2DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy2DAsync_v2') + global __cuMemcpy3DAsync_v2 + __cuMemcpy3DAsync_v2 = dlfcn.dlsym(handle, 'cuMemcpy3DAsync_v2') + global __cuMemcpy3DPeerAsync + __cuMemcpy3DPeerAsync = dlfcn.dlsym(handle, 'cuMemcpy3DPeerAsync') + global __cuMemcpyBatchAsync + __cuMemcpyBatchAsync = dlfcn.dlsym(handle, 'cuMemcpyBatchAsync') + global __cuMemcpy3DBatchAsync + __cuMemcpy3DBatchAsync = dlfcn.dlsym(handle, 'cuMemcpy3DBatchAsync') + global __cuMemsetD8_v2 + __cuMemsetD8_v2 = dlfcn.dlsym(handle, 'cuMemsetD8_v2') + global __cuMemsetD16_v2 + __cuMemsetD16_v2 = dlfcn.dlsym(handle, 'cuMemsetD16_v2') + global __cuMemsetD32_v2 + __cuMemsetD32_v2 = dlfcn.dlsym(handle, 'cuMemsetD32_v2') + global __cuMemsetD2D8_v2 + __cuMemsetD2D8_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D8_v2') + global __cuMemsetD2D16_v2 + __cuMemsetD2D16_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D16_v2') + global __cuMemsetD2D32_v2 + __cuMemsetD2D32_v2 = dlfcn.dlsym(handle, 'cuMemsetD2D32_v2') + global __cuMemsetD8Async + __cuMemsetD8Async = dlfcn.dlsym(handle, 'cuMemsetD8Async') + global __cuMemsetD16Async + __cuMemsetD16Async = dlfcn.dlsym(handle, 'cuMemsetD16Async') + global __cuMemsetD32Async + __cuMemsetD32Async = dlfcn.dlsym(handle, 'cuMemsetD32Async') + global __cuMemsetD2D8Async + __cuMemsetD2D8Async = dlfcn.dlsym(handle, 'cuMemsetD2D8Async') + global __cuMemsetD2D16Async + __cuMemsetD2D16Async = dlfcn.dlsym(handle, 'cuMemsetD2D16Async') + global __cuMemsetD2D32Async + __cuMemsetD2D32Async = dlfcn.dlsym(handle, 'cuMemsetD2D32Async') + global __cuMemBatchDecompressAsync + __cuMemBatchDecompressAsync = dlfcn.dlsym(handle, 'cuMemBatchDecompressAsync') + global __cuMemMapArrayAsync + __cuMemMapArrayAsync = dlfcn.dlsym(handle, 'cuMemMapArrayAsync') + global __cuMemFreeAsync + __cuMemFreeAsync = dlfcn.dlsym(handle, 'cuMemFreeAsync') + global __cuMemAllocAsync + __cuMemAllocAsync = dlfcn.dlsym(handle, 'cuMemAllocAsync') + global __cuMemAllocFromPoolAsync + __cuMemAllocFromPoolAsync = dlfcn.dlsym(handle, 'cuMemAllocFromPoolAsync') + global __cuMemPrefetchAsync + __cuMemPrefetchAsync = dlfcn.dlsym(handle, 'cuMemPrefetchAsync') + global __cuMemPrefetchAsync_v2 + __cuMemPrefetchAsync_v2 = dlfcn.dlsym(handle, 'cuMemPrefetchAsync_v2') + global __cuStreamGetPriority + __cuStreamGetPriority = dlfcn.dlsym(handle, 'cuStreamGetPriority') + global __cuStreamGetDevice + __cuStreamGetDevice = dlfcn.dlsym(handle, 'cuStreamGetDevice') + global __cuStreamGetFlags + __cuStreamGetFlags = dlfcn.dlsym(handle, 'cuStreamGetFlags') + global __cuStreamGetId + __cuStreamGetId = dlfcn.dlsym(handle, 'cuStreamGetId') + global __cuStreamGetCtx + __cuStreamGetCtx = dlfcn.dlsym(handle, 'cuStreamGetCtx') + global __cuStreamGetCtx_v2 + __cuStreamGetCtx_v2 = dlfcn.dlsym(handle, 'cuStreamGetCtx_v2') + global __cuStreamWaitEvent + __cuStreamWaitEvent = dlfcn.dlsym(handle, 'cuStreamWaitEvent') + global __cuStreamAddCallback + __cuStreamAddCallback = dlfcn.dlsym(handle, 'cuStreamAddCallback') + global __cuStreamBeginCapture_v2 + __cuStreamBeginCapture_v2 = dlfcn.dlsym(handle, 'cuStreamBeginCapture_v2') + global __cuStreamBeginCaptureToGraph + __cuStreamBeginCaptureToGraph = dlfcn.dlsym(handle, 'cuStreamBeginCaptureToGraph') + global __cuStreamEndCapture + __cuStreamEndCapture = dlfcn.dlsym(handle, 'cuStreamEndCapture') + global __cuStreamIsCapturing + __cuStreamIsCapturing = dlfcn.dlsym(handle, 'cuStreamIsCapturing') + global __cuStreamGetCaptureInfo_v2 + __cuStreamGetCaptureInfo_v2 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v2') + global __cuStreamGetCaptureInfo_v3 + __cuStreamGetCaptureInfo_v3 = dlfcn.dlsym(handle, 'cuStreamGetCaptureInfo_v3') + global __cuStreamUpdateCaptureDependencies + __cuStreamUpdateCaptureDependencies = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies') + global __cuStreamUpdateCaptureDependencies_v2 + __cuStreamUpdateCaptureDependencies_v2 = dlfcn.dlsym(handle, 'cuStreamUpdateCaptureDependencies_v2') + global __cuStreamAttachMemAsync + __cuStreamAttachMemAsync = dlfcn.dlsym(handle, 'cuStreamAttachMemAsync') + global __cuStreamQuery + __cuStreamQuery = dlfcn.dlsym(handle, 'cuStreamQuery') + global __cuStreamSynchronize + __cuStreamSynchronize = dlfcn.dlsym(handle, 'cuStreamSynchronize') + global __cuStreamCopyAttributes + __cuStreamCopyAttributes = dlfcn.dlsym(handle, 'cuStreamCopyAttributes') + global __cuStreamGetAttribute + __cuStreamGetAttribute = dlfcn.dlsym(handle, 'cuStreamGetAttribute') + global __cuStreamSetAttribute + __cuStreamSetAttribute = dlfcn.dlsym(handle, 'cuStreamSetAttribute') + global __cuEventRecord + __cuEventRecord = dlfcn.dlsym(handle, 'cuEventRecord') + global __cuEventRecordWithFlags + __cuEventRecordWithFlags = dlfcn.dlsym(handle, 'cuEventRecordWithFlags') + global __cuSignalExternalSemaphoresAsync + __cuSignalExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuSignalExternalSemaphoresAsync') + global __cuWaitExternalSemaphoresAsync + __cuWaitExternalSemaphoresAsync = dlfcn.dlsym(handle, 'cuWaitExternalSemaphoresAsync') + global __cuStreamWaitValue32_v2 + __cuStreamWaitValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue32_v2') + global __cuStreamWaitValue64_v2 + __cuStreamWaitValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWaitValue64_v2') + global __cuStreamWriteValue32_v2 + __cuStreamWriteValue32_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue32_v2') + global __cuStreamWriteValue64_v2 + __cuStreamWriteValue64_v2 = dlfcn.dlsym(handle, 'cuStreamWriteValue64_v2') + global __cuStreamBatchMemOp_v2 + __cuStreamBatchMemOp_v2 = dlfcn.dlsym(handle, 'cuStreamBatchMemOp_v2') + global __cuLaunchKernel + __cuLaunchKernel = dlfcn.dlsym(handle, 'cuLaunchKernel') + global __cuLaunchKernelEx + __cuLaunchKernelEx = dlfcn.dlsym(handle, 'cuLaunchKernelEx') + global __cuLaunchCooperativeKernel + __cuLaunchCooperativeKernel = dlfcn.dlsym(handle, 'cuLaunchCooperativeKernel') + global __cuLaunchHostFunc + __cuLaunchHostFunc = dlfcn.dlsym(handle, 'cuLaunchHostFunc') + global __cuGraphInstantiateWithParams + __cuGraphInstantiateWithParams = dlfcn.dlsym(handle, 'cuGraphInstantiateWithParams') + global __cuGraphUpload + __cuGraphUpload = dlfcn.dlsym(handle, 'cuGraphUpload') + global __cuGraphLaunch + __cuGraphLaunch = dlfcn.dlsym(handle, 'cuGraphLaunch') + global __cuGraphicsMapResources + __cuGraphicsMapResources = dlfcn.dlsym(handle, 'cuGraphicsMapResources') + global __cuGraphicsUnmapResources + __cuGraphicsUnmapResources = dlfcn.dlsym(handle, 'cuGraphicsUnmapResources') + # Get remaining functions + global __cuGetErrorString + __cuGetErrorString = dlfcn.dlsym(handle, 'cuGetErrorString') + global __cuGetErrorName + __cuGetErrorName = dlfcn.dlsym(handle, 'cuGetErrorName') + global __cuInit + __cuInit = dlfcn.dlsym(handle, 'cuInit') + global __cuDriverGetVersion + __cuDriverGetVersion = dlfcn.dlsym(handle, 'cuDriverGetVersion') + global __cuDeviceGet + __cuDeviceGet = dlfcn.dlsym(handle, 'cuDeviceGet') + global __cuDeviceGetCount + __cuDeviceGetCount = dlfcn.dlsym(handle, 'cuDeviceGetCount') + global __cuDeviceGetName + __cuDeviceGetName = dlfcn.dlsym(handle, 'cuDeviceGetName') + global __cuDeviceGetUuid + __cuDeviceGetUuid = dlfcn.dlsym(handle, 'cuDeviceGetUuid') + global __cuDeviceGetUuid_v2 + __cuDeviceGetUuid_v2 = dlfcn.dlsym(handle, 'cuDeviceGetUuid_v2') + global __cuDeviceGetLuid + __cuDeviceGetLuid = dlfcn.dlsym(handle, 'cuDeviceGetLuid') + global __cuDeviceTotalMem_v2 + __cuDeviceTotalMem_v2 = dlfcn.dlsym(handle, 'cuDeviceTotalMem_v2') + global __cuDeviceGetTexture1DLinearMaxWidth + __cuDeviceGetTexture1DLinearMaxWidth = dlfcn.dlsym(handle, 'cuDeviceGetTexture1DLinearMaxWidth') + global __cuDeviceGetAttribute + __cuDeviceGetAttribute = dlfcn.dlsym(handle, 'cuDeviceGetAttribute') + global __cuDeviceGetNvSciSyncAttributes + __cuDeviceGetNvSciSyncAttributes = dlfcn.dlsym(handle, 'cuDeviceGetNvSciSyncAttributes') + global __cuDeviceSetMemPool + __cuDeviceSetMemPool = dlfcn.dlsym(handle, 'cuDeviceSetMemPool') + global __cuDeviceGetMemPool + __cuDeviceGetMemPool = dlfcn.dlsym(handle, 'cuDeviceGetMemPool') + global __cuDeviceGetDefaultMemPool + __cuDeviceGetDefaultMemPool = dlfcn.dlsym(handle, 'cuDeviceGetDefaultMemPool') + global __cuDeviceGetExecAffinitySupport + __cuDeviceGetExecAffinitySupport = dlfcn.dlsym(handle, 'cuDeviceGetExecAffinitySupport') + global __cuFlushGPUDirectRDMAWrites + __cuFlushGPUDirectRDMAWrites = dlfcn.dlsym(handle, 'cuFlushGPUDirectRDMAWrites') + global __cuDeviceGetProperties + __cuDeviceGetProperties = dlfcn.dlsym(handle, 'cuDeviceGetProperties') + global __cuDeviceComputeCapability + __cuDeviceComputeCapability = dlfcn.dlsym(handle, 'cuDeviceComputeCapability') + global __cuDevicePrimaryCtxRetain + __cuDevicePrimaryCtxRetain = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxRetain') + global __cuDevicePrimaryCtxRelease_v2 + __cuDevicePrimaryCtxRelease_v2 = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxRelease_v2') + global __cuDevicePrimaryCtxSetFlags_v2 + __cuDevicePrimaryCtxSetFlags_v2 = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxSetFlags_v2') + global __cuDevicePrimaryCtxGetState + __cuDevicePrimaryCtxGetState = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxGetState') + global __cuDevicePrimaryCtxReset_v2 + __cuDevicePrimaryCtxReset_v2 = dlfcn.dlsym(handle, 'cuDevicePrimaryCtxReset_v2') + global __cuCtxCreate_v2 + __cuCtxCreate_v2 = dlfcn.dlsym(handle, 'cuCtxCreate_v2') + global __cuCtxCreate_v3 + __cuCtxCreate_v3 = dlfcn.dlsym(handle, 'cuCtxCreate_v3') + global __cuCtxCreate_v4 + __cuCtxCreate_v4 = dlfcn.dlsym(handle, 'cuCtxCreate_v4') + global __cuCtxDestroy_v2 + __cuCtxDestroy_v2 = dlfcn.dlsym(handle, 'cuCtxDestroy_v2') + global __cuCtxPushCurrent_v2 + __cuCtxPushCurrent_v2 = dlfcn.dlsym(handle, 'cuCtxPushCurrent_v2') + global __cuCtxPopCurrent_v2 + __cuCtxPopCurrent_v2 = dlfcn.dlsym(handle, 'cuCtxPopCurrent_v2') + global __cuCtxSetCurrent + __cuCtxSetCurrent = dlfcn.dlsym(handle, 'cuCtxSetCurrent') + global __cuCtxGetCurrent + __cuCtxGetCurrent = dlfcn.dlsym(handle, 'cuCtxGetCurrent') + global __cuCtxGetDevice + __cuCtxGetDevice = dlfcn.dlsym(handle, 'cuCtxGetDevice') + global __cuCtxGetFlags + __cuCtxGetFlags = dlfcn.dlsym(handle, 'cuCtxGetFlags') + global __cuCtxSetFlags + __cuCtxSetFlags = dlfcn.dlsym(handle, 'cuCtxSetFlags') + global __cuCtxGetId + __cuCtxGetId = dlfcn.dlsym(handle, 'cuCtxGetId') + global __cuCtxSynchronize + __cuCtxSynchronize = dlfcn.dlsym(handle, 'cuCtxSynchronize') + global __cuCtxSetLimit + __cuCtxSetLimit = dlfcn.dlsym(handle, 'cuCtxSetLimit') + global __cuCtxGetLimit + __cuCtxGetLimit = dlfcn.dlsym(handle, 'cuCtxGetLimit') + global __cuCtxGetCacheConfig + __cuCtxGetCacheConfig = dlfcn.dlsym(handle, 'cuCtxGetCacheConfig') + global __cuCtxSetCacheConfig + __cuCtxSetCacheConfig = dlfcn.dlsym(handle, 'cuCtxSetCacheConfig') + global __cuCtxGetApiVersion + __cuCtxGetApiVersion = dlfcn.dlsym(handle, 'cuCtxGetApiVersion') + global __cuCtxGetStreamPriorityRange + __cuCtxGetStreamPriorityRange = dlfcn.dlsym(handle, 'cuCtxGetStreamPriorityRange') + global __cuCtxResetPersistingL2Cache + __cuCtxResetPersistingL2Cache = dlfcn.dlsym(handle, 'cuCtxResetPersistingL2Cache') + global __cuCtxGetExecAffinity + __cuCtxGetExecAffinity = dlfcn.dlsym(handle, 'cuCtxGetExecAffinity') + global __cuCtxRecordEvent + __cuCtxRecordEvent = dlfcn.dlsym(handle, 'cuCtxRecordEvent') + global __cuCtxWaitEvent + __cuCtxWaitEvent = dlfcn.dlsym(handle, 'cuCtxWaitEvent') + global __cuCtxAttach + __cuCtxAttach = dlfcn.dlsym(handle, 'cuCtxAttach') + global __cuCtxDetach + __cuCtxDetach = dlfcn.dlsym(handle, 'cuCtxDetach') + global __cuCtxGetSharedMemConfig + __cuCtxGetSharedMemConfig = dlfcn.dlsym(handle, 'cuCtxGetSharedMemConfig') + global __cuCtxSetSharedMemConfig + __cuCtxSetSharedMemConfig = dlfcn.dlsym(handle, 'cuCtxSetSharedMemConfig') + global __cuModuleLoad + __cuModuleLoad = dlfcn.dlsym(handle, 'cuModuleLoad') + global __cuModuleLoadData + __cuModuleLoadData = dlfcn.dlsym(handle, 'cuModuleLoadData') + global __cuModuleLoadDataEx + __cuModuleLoadDataEx = dlfcn.dlsym(handle, 'cuModuleLoadDataEx') + global __cuModuleLoadFatBinary + __cuModuleLoadFatBinary = dlfcn.dlsym(handle, 'cuModuleLoadFatBinary') + global __cuModuleUnload + __cuModuleUnload = dlfcn.dlsym(handle, 'cuModuleUnload') + global __cuModuleGetLoadingMode + __cuModuleGetLoadingMode = dlfcn.dlsym(handle, 'cuModuleGetLoadingMode') + global __cuModuleGetFunction + __cuModuleGetFunction = dlfcn.dlsym(handle, 'cuModuleGetFunction') + global __cuModuleGetFunctionCount + __cuModuleGetFunctionCount = dlfcn.dlsym(handle, 'cuModuleGetFunctionCount') + global __cuModuleEnumerateFunctions + __cuModuleEnumerateFunctions = dlfcn.dlsym(handle, 'cuModuleEnumerateFunctions') + global __cuModuleGetGlobal_v2 + __cuModuleGetGlobal_v2 = dlfcn.dlsym(handle, 'cuModuleGetGlobal_v2') + global __cuLinkCreate_v2 + __cuLinkCreate_v2 = dlfcn.dlsym(handle, 'cuLinkCreate_v2') + global __cuLinkAddData_v2 + __cuLinkAddData_v2 = dlfcn.dlsym(handle, 'cuLinkAddData_v2') + global __cuLinkAddFile_v2 + __cuLinkAddFile_v2 = dlfcn.dlsym(handle, 'cuLinkAddFile_v2') + global __cuLinkComplete + __cuLinkComplete = dlfcn.dlsym(handle, 'cuLinkComplete') + global __cuLinkDestroy + __cuLinkDestroy = dlfcn.dlsym(handle, 'cuLinkDestroy') + global __cuModuleGetTexRef + __cuModuleGetTexRef = dlfcn.dlsym(handle, 'cuModuleGetTexRef') + global __cuModuleGetSurfRef + __cuModuleGetSurfRef = dlfcn.dlsym(handle, 'cuModuleGetSurfRef') + global __cuLibraryLoadData + __cuLibraryLoadData = dlfcn.dlsym(handle, 'cuLibraryLoadData') + global __cuLibraryLoadFromFile + __cuLibraryLoadFromFile = dlfcn.dlsym(handle, 'cuLibraryLoadFromFile') + global __cuLibraryUnload + __cuLibraryUnload = dlfcn.dlsym(handle, 'cuLibraryUnload') + global __cuLibraryGetKernel + __cuLibraryGetKernel = dlfcn.dlsym(handle, 'cuLibraryGetKernel') + global __cuLibraryGetKernelCount + __cuLibraryGetKernelCount = dlfcn.dlsym(handle, 'cuLibraryGetKernelCount') + global __cuLibraryEnumerateKernels + __cuLibraryEnumerateKernels = dlfcn.dlsym(handle, 'cuLibraryEnumerateKernels') + global __cuLibraryGetModule + __cuLibraryGetModule = dlfcn.dlsym(handle, 'cuLibraryGetModule') + global __cuKernelGetFunction + __cuKernelGetFunction = dlfcn.dlsym(handle, 'cuKernelGetFunction') + global __cuKernelGetLibrary + __cuKernelGetLibrary = dlfcn.dlsym(handle, 'cuKernelGetLibrary') + global __cuLibraryGetGlobal + __cuLibraryGetGlobal = dlfcn.dlsym(handle, 'cuLibraryGetGlobal') + global __cuLibraryGetManaged + __cuLibraryGetManaged = dlfcn.dlsym(handle, 'cuLibraryGetManaged') + global __cuLibraryGetUnifiedFunction + __cuLibraryGetUnifiedFunction = dlfcn.dlsym(handle, 'cuLibraryGetUnifiedFunction') + global __cuKernelGetAttribute + __cuKernelGetAttribute = dlfcn.dlsym(handle, 'cuKernelGetAttribute') + global __cuKernelSetAttribute + __cuKernelSetAttribute = dlfcn.dlsym(handle, 'cuKernelSetAttribute') + global __cuKernelSetCacheConfig + __cuKernelSetCacheConfig = dlfcn.dlsym(handle, 'cuKernelSetCacheConfig') + global __cuKernelGetName + __cuKernelGetName = dlfcn.dlsym(handle, 'cuKernelGetName') + global __cuKernelGetParamInfo + __cuKernelGetParamInfo = dlfcn.dlsym(handle, 'cuKernelGetParamInfo') + global __cuMemGetInfo_v2 + __cuMemGetInfo_v2 = dlfcn.dlsym(handle, 'cuMemGetInfo_v2') + global __cuMemAlloc_v2 + __cuMemAlloc_v2 = dlfcn.dlsym(handle, 'cuMemAlloc_v2') + global __cuMemAllocPitch_v2 + __cuMemAllocPitch_v2 = dlfcn.dlsym(handle, 'cuMemAllocPitch_v2') + global __cuMemFree_v2 + __cuMemFree_v2 = dlfcn.dlsym(handle, 'cuMemFree_v2') + global __cuMemGetAddressRange_v2 + __cuMemGetAddressRange_v2 = dlfcn.dlsym(handle, 'cuMemGetAddressRange_v2') + global __cuMemAllocHost_v2 + __cuMemAllocHost_v2 = dlfcn.dlsym(handle, 'cuMemAllocHost_v2') + global __cuMemFreeHost + __cuMemFreeHost = dlfcn.dlsym(handle, 'cuMemFreeHost') + global __cuMemHostAlloc + __cuMemHostAlloc = dlfcn.dlsym(handle, 'cuMemHostAlloc') + global __cuMemHostGetDevicePointer_v2 + __cuMemHostGetDevicePointer_v2 = dlfcn.dlsym(handle, 'cuMemHostGetDevicePointer_v2') + global __cuMemHostGetFlags + __cuMemHostGetFlags = dlfcn.dlsym(handle, 'cuMemHostGetFlags') + global __cuMemAllocManaged + __cuMemAllocManaged = dlfcn.dlsym(handle, 'cuMemAllocManaged') + global __cuDeviceRegisterAsyncNotification + __cuDeviceRegisterAsyncNotification = dlfcn.dlsym(handle, 'cuDeviceRegisterAsyncNotification') + global __cuDeviceUnregisterAsyncNotification + __cuDeviceUnregisterAsyncNotification = dlfcn.dlsym(handle, 'cuDeviceUnregisterAsyncNotification') + global __cuDeviceGetByPCIBusId + __cuDeviceGetByPCIBusId = dlfcn.dlsym(handle, 'cuDeviceGetByPCIBusId') + global __cuDeviceGetPCIBusId + __cuDeviceGetPCIBusId = dlfcn.dlsym(handle, 'cuDeviceGetPCIBusId') + global __cuIpcGetEventHandle + __cuIpcGetEventHandle = dlfcn.dlsym(handle, 'cuIpcGetEventHandle') + global __cuIpcOpenEventHandle + __cuIpcOpenEventHandle = dlfcn.dlsym(handle, 'cuIpcOpenEventHandle') + global __cuIpcGetMemHandle + __cuIpcGetMemHandle = dlfcn.dlsym(handle, 'cuIpcGetMemHandle') + global __cuIpcOpenMemHandle_v2 + __cuIpcOpenMemHandle_v2 = dlfcn.dlsym(handle, 'cuIpcOpenMemHandle_v2') + global __cuIpcCloseMemHandle + __cuIpcCloseMemHandle = dlfcn.dlsym(handle, 'cuIpcCloseMemHandle') + global __cuMemHostRegister_v2 + __cuMemHostRegister_v2 = dlfcn.dlsym(handle, 'cuMemHostRegister_v2') + global __cuMemHostUnregister + __cuMemHostUnregister = dlfcn.dlsym(handle, 'cuMemHostUnregister') + global __cuArrayCreate_v2 + __cuArrayCreate_v2 = dlfcn.dlsym(handle, 'cuArrayCreate_v2') + global __cuArrayGetDescriptor_v2 + __cuArrayGetDescriptor_v2 = dlfcn.dlsym(handle, 'cuArrayGetDescriptor_v2') + global __cuArrayGetSparseProperties + __cuArrayGetSparseProperties = dlfcn.dlsym(handle, 'cuArrayGetSparseProperties') + global __cuMipmappedArrayGetSparseProperties + __cuMipmappedArrayGetSparseProperties = dlfcn.dlsym(handle, 'cuMipmappedArrayGetSparseProperties') + global __cuArrayGetMemoryRequirements + __cuArrayGetMemoryRequirements = dlfcn.dlsym(handle, 'cuArrayGetMemoryRequirements') + global __cuMipmappedArrayGetMemoryRequirements + __cuMipmappedArrayGetMemoryRequirements = dlfcn.dlsym(handle, 'cuMipmappedArrayGetMemoryRequirements') + global __cuArrayGetPlane + __cuArrayGetPlane = dlfcn.dlsym(handle, 'cuArrayGetPlane') + global __cuArrayDestroy + __cuArrayDestroy = dlfcn.dlsym(handle, 'cuArrayDestroy') + global __cuArray3DCreate_v2 + __cuArray3DCreate_v2 = dlfcn.dlsym(handle, 'cuArray3DCreate_v2') + global __cuArray3DGetDescriptor_v2 + __cuArray3DGetDescriptor_v2 = dlfcn.dlsym(handle, 'cuArray3DGetDescriptor_v2') + global __cuMipmappedArrayCreate + __cuMipmappedArrayCreate = dlfcn.dlsym(handle, 'cuMipmappedArrayCreate') + global __cuMipmappedArrayGetLevel + __cuMipmappedArrayGetLevel = dlfcn.dlsym(handle, 'cuMipmappedArrayGetLevel') + global __cuMipmappedArrayDestroy + __cuMipmappedArrayDestroy = dlfcn.dlsym(handle, 'cuMipmappedArrayDestroy') + global __cuMemGetHandleForAddressRange + __cuMemGetHandleForAddressRange = dlfcn.dlsym(handle, 'cuMemGetHandleForAddressRange') + global __cuMemAddressReserve + __cuMemAddressReserve = dlfcn.dlsym(handle, 'cuMemAddressReserve') + global __cuMemAddressFree + __cuMemAddressFree = dlfcn.dlsym(handle, 'cuMemAddressFree') + global __cuMemCreate + __cuMemCreate = dlfcn.dlsym(handle, 'cuMemCreate') + global __cuMemRelease + __cuMemRelease = dlfcn.dlsym(handle, 'cuMemRelease') + global __cuMemMap + __cuMemMap = dlfcn.dlsym(handle, 'cuMemMap') + global __cuMemUnmap + __cuMemUnmap = dlfcn.dlsym(handle, 'cuMemUnmap') + global __cuMemSetAccess + __cuMemSetAccess = dlfcn.dlsym(handle, 'cuMemSetAccess') + global __cuMemGetAccess + __cuMemGetAccess = dlfcn.dlsym(handle, 'cuMemGetAccess') + global __cuMemExportToShareableHandle + __cuMemExportToShareableHandle = dlfcn.dlsym(handle, 'cuMemExportToShareableHandle') + global __cuMemImportFromShareableHandle + __cuMemImportFromShareableHandle = dlfcn.dlsym(handle, 'cuMemImportFromShareableHandle') + global __cuMemGetAllocationGranularity + __cuMemGetAllocationGranularity = dlfcn.dlsym(handle, 'cuMemGetAllocationGranularity') + global __cuMemGetAllocationPropertiesFromHandle + __cuMemGetAllocationPropertiesFromHandle = dlfcn.dlsym(handle, 'cuMemGetAllocationPropertiesFromHandle') + global __cuMemRetainAllocationHandle + __cuMemRetainAllocationHandle = dlfcn.dlsym(handle, 'cuMemRetainAllocationHandle') + global __cuMemPoolTrimTo + __cuMemPoolTrimTo = dlfcn.dlsym(handle, 'cuMemPoolTrimTo') + global __cuMemPoolSetAttribute + __cuMemPoolSetAttribute = dlfcn.dlsym(handle, 'cuMemPoolSetAttribute') + global __cuMemPoolGetAttribute + __cuMemPoolGetAttribute = dlfcn.dlsym(handle, 'cuMemPoolGetAttribute') + global __cuMemPoolSetAccess + __cuMemPoolSetAccess = dlfcn.dlsym(handle, 'cuMemPoolSetAccess') + global __cuMemPoolGetAccess + __cuMemPoolGetAccess = dlfcn.dlsym(handle, 'cuMemPoolGetAccess') + global __cuMemPoolCreate + __cuMemPoolCreate = dlfcn.dlsym(handle, 'cuMemPoolCreate') + global __cuMemPoolDestroy + __cuMemPoolDestroy = dlfcn.dlsym(handle, 'cuMemPoolDestroy') + global __cuMemPoolExportToShareableHandle + __cuMemPoolExportToShareableHandle = dlfcn.dlsym(handle, 'cuMemPoolExportToShareableHandle') + global __cuMemPoolImportFromShareableHandle + __cuMemPoolImportFromShareableHandle = dlfcn.dlsym(handle, 'cuMemPoolImportFromShareableHandle') + global __cuMemPoolExportPointer + __cuMemPoolExportPointer = dlfcn.dlsym(handle, 'cuMemPoolExportPointer') + global __cuMemPoolImportPointer + __cuMemPoolImportPointer = dlfcn.dlsym(handle, 'cuMemPoolImportPointer') + global __cuMulticastCreate + __cuMulticastCreate = dlfcn.dlsym(handle, 'cuMulticastCreate') + global __cuMulticastAddDevice + __cuMulticastAddDevice = dlfcn.dlsym(handle, 'cuMulticastAddDevice') + global __cuMulticastBindMem + __cuMulticastBindMem = dlfcn.dlsym(handle, 'cuMulticastBindMem') + global __cuMulticastBindAddr + __cuMulticastBindAddr = dlfcn.dlsym(handle, 'cuMulticastBindAddr') + global __cuMulticastUnbind + __cuMulticastUnbind = dlfcn.dlsym(handle, 'cuMulticastUnbind') + global __cuMulticastGetGranularity + __cuMulticastGetGranularity = dlfcn.dlsym(handle, 'cuMulticastGetGranularity') + global __cuPointerGetAttribute + __cuPointerGetAttribute = dlfcn.dlsym(handle, 'cuPointerGetAttribute') + global __cuMemAdvise + __cuMemAdvise = dlfcn.dlsym(handle, 'cuMemAdvise') + global __cuMemAdvise_v2 + __cuMemAdvise_v2 = dlfcn.dlsym(handle, 'cuMemAdvise_v2') + global __cuMemRangeGetAttribute + __cuMemRangeGetAttribute = dlfcn.dlsym(handle, 'cuMemRangeGetAttribute') + global __cuMemRangeGetAttributes + __cuMemRangeGetAttributes = dlfcn.dlsym(handle, 'cuMemRangeGetAttributes') + global __cuPointerSetAttribute + __cuPointerSetAttribute = dlfcn.dlsym(handle, 'cuPointerSetAttribute') + global __cuPointerGetAttributes + __cuPointerGetAttributes = dlfcn.dlsym(handle, 'cuPointerGetAttributes') + global __cuStreamCreate + __cuStreamCreate = dlfcn.dlsym(handle, 'cuStreamCreate') + global __cuStreamCreateWithPriority + __cuStreamCreateWithPriority = dlfcn.dlsym(handle, 'cuStreamCreateWithPriority') + global __cuThreadExchangeStreamCaptureMode + __cuThreadExchangeStreamCaptureMode = dlfcn.dlsym(handle, 'cuThreadExchangeStreamCaptureMode') + global __cuStreamDestroy_v2 + __cuStreamDestroy_v2 = dlfcn.dlsym(handle, 'cuStreamDestroy_v2') + global __cuEventCreate + __cuEventCreate = dlfcn.dlsym(handle, 'cuEventCreate') + global __cuEventQuery + __cuEventQuery = dlfcn.dlsym(handle, 'cuEventQuery') + global __cuEventSynchronize + __cuEventSynchronize = dlfcn.dlsym(handle, 'cuEventSynchronize') + global __cuEventDestroy_v2 + __cuEventDestroy_v2 = dlfcn.dlsym(handle, 'cuEventDestroy_v2') + global __cuEventElapsedTime + __cuEventElapsedTime = dlfcn.dlsym(handle, 'cuEventElapsedTime') + global __cuEventElapsedTime_v2 + __cuEventElapsedTime_v2 = dlfcn.dlsym(handle, 'cuEventElapsedTime_v2') + global __cuImportExternalMemory + __cuImportExternalMemory = dlfcn.dlsym(handle, 'cuImportExternalMemory') + global __cuExternalMemoryGetMappedBuffer + __cuExternalMemoryGetMappedBuffer = dlfcn.dlsym(handle, 'cuExternalMemoryGetMappedBuffer') + global __cuExternalMemoryGetMappedMipmappedArray + __cuExternalMemoryGetMappedMipmappedArray = dlfcn.dlsym(handle, 'cuExternalMemoryGetMappedMipmappedArray') + global __cuDestroyExternalMemory + __cuDestroyExternalMemory = dlfcn.dlsym(handle, 'cuDestroyExternalMemory') + global __cuImportExternalSemaphore + __cuImportExternalSemaphore = dlfcn.dlsym(handle, 'cuImportExternalSemaphore') + global __cuDestroyExternalSemaphore + __cuDestroyExternalSemaphore = dlfcn.dlsym(handle, 'cuDestroyExternalSemaphore') + global __cuFuncGetAttribute + __cuFuncGetAttribute = dlfcn.dlsym(handle, 'cuFuncGetAttribute') + global __cuFuncSetAttribute + __cuFuncSetAttribute = dlfcn.dlsym(handle, 'cuFuncSetAttribute') + global __cuFuncSetCacheConfig + __cuFuncSetCacheConfig = dlfcn.dlsym(handle, 'cuFuncSetCacheConfig') + global __cuFuncGetModule + __cuFuncGetModule = dlfcn.dlsym(handle, 'cuFuncGetModule') + global __cuFuncGetName + __cuFuncGetName = dlfcn.dlsym(handle, 'cuFuncGetName') + global __cuFuncGetParamInfo + __cuFuncGetParamInfo = dlfcn.dlsym(handle, 'cuFuncGetParamInfo') + global __cuFuncIsLoaded + __cuFuncIsLoaded = dlfcn.dlsym(handle, 'cuFuncIsLoaded') + global __cuFuncLoad + __cuFuncLoad = dlfcn.dlsym(handle, 'cuFuncLoad') + global __cuLaunchCooperativeKernelMultiDevice + __cuLaunchCooperativeKernelMultiDevice = dlfcn.dlsym(handle, 'cuLaunchCooperativeKernelMultiDevice') + global __cuFuncSetBlockShape + __cuFuncSetBlockShape = dlfcn.dlsym(handle, 'cuFuncSetBlockShape') + global __cuFuncSetSharedSize + __cuFuncSetSharedSize = dlfcn.dlsym(handle, 'cuFuncSetSharedSize') + global __cuParamSetSize + __cuParamSetSize = dlfcn.dlsym(handle, 'cuParamSetSize') + global __cuParamSeti + __cuParamSeti = dlfcn.dlsym(handle, 'cuParamSeti') + global __cuParamSetf + __cuParamSetf = dlfcn.dlsym(handle, 'cuParamSetf') + global __cuParamSetv + __cuParamSetv = dlfcn.dlsym(handle, 'cuParamSetv') + global __cuLaunch + __cuLaunch = dlfcn.dlsym(handle, 'cuLaunch') + global __cuLaunchGrid + __cuLaunchGrid = dlfcn.dlsym(handle, 'cuLaunchGrid') + global __cuLaunchGridAsync + __cuLaunchGridAsync = dlfcn.dlsym(handle, 'cuLaunchGridAsync') + global __cuParamSetTexRef + __cuParamSetTexRef = dlfcn.dlsym(handle, 'cuParamSetTexRef') + global __cuFuncSetSharedMemConfig + __cuFuncSetSharedMemConfig = dlfcn.dlsym(handle, 'cuFuncSetSharedMemConfig') + global __cuGraphCreate + __cuGraphCreate = dlfcn.dlsym(handle, 'cuGraphCreate') + global __cuGraphAddKernelNode_v2 + __cuGraphAddKernelNode_v2 = dlfcn.dlsym(handle, 'cuGraphAddKernelNode_v2') + global __cuGraphKernelNodeGetParams_v2 + __cuGraphKernelNodeGetParams_v2 = dlfcn.dlsym(handle, 'cuGraphKernelNodeGetParams_v2') + global __cuGraphKernelNodeSetParams_v2 + __cuGraphKernelNodeSetParams_v2 = dlfcn.dlsym(handle, 'cuGraphKernelNodeSetParams_v2') + global __cuGraphAddMemcpyNode + __cuGraphAddMemcpyNode = dlfcn.dlsym(handle, 'cuGraphAddMemcpyNode') + global __cuGraphMemcpyNodeGetParams + __cuGraphMemcpyNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemcpyNodeGetParams') + global __cuGraphMemcpyNodeSetParams + __cuGraphMemcpyNodeSetParams = dlfcn.dlsym(handle, 'cuGraphMemcpyNodeSetParams') + global __cuGraphAddMemsetNode + __cuGraphAddMemsetNode = dlfcn.dlsym(handle, 'cuGraphAddMemsetNode') + global __cuGraphMemsetNodeGetParams + __cuGraphMemsetNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemsetNodeGetParams') + global __cuGraphMemsetNodeSetParams + __cuGraphMemsetNodeSetParams = dlfcn.dlsym(handle, 'cuGraphMemsetNodeSetParams') + global __cuGraphAddHostNode + __cuGraphAddHostNode = dlfcn.dlsym(handle, 'cuGraphAddHostNode') + global __cuGraphHostNodeGetParams + __cuGraphHostNodeGetParams = dlfcn.dlsym(handle, 'cuGraphHostNodeGetParams') + global __cuGraphHostNodeSetParams + __cuGraphHostNodeSetParams = dlfcn.dlsym(handle, 'cuGraphHostNodeSetParams') + global __cuGraphAddChildGraphNode + __cuGraphAddChildGraphNode = dlfcn.dlsym(handle, 'cuGraphAddChildGraphNode') + global __cuGraphChildGraphNodeGetGraph + __cuGraphChildGraphNodeGetGraph = dlfcn.dlsym(handle, 'cuGraphChildGraphNodeGetGraph') + global __cuGraphAddEmptyNode + __cuGraphAddEmptyNode = dlfcn.dlsym(handle, 'cuGraphAddEmptyNode') + global __cuGraphAddEventRecordNode + __cuGraphAddEventRecordNode = dlfcn.dlsym(handle, 'cuGraphAddEventRecordNode') + global __cuGraphEventRecordNodeGetEvent + __cuGraphEventRecordNodeGetEvent = dlfcn.dlsym(handle, 'cuGraphEventRecordNodeGetEvent') + global __cuGraphEventRecordNodeSetEvent + __cuGraphEventRecordNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphEventRecordNodeSetEvent') + global __cuGraphAddEventWaitNode + __cuGraphAddEventWaitNode = dlfcn.dlsym(handle, 'cuGraphAddEventWaitNode') + global __cuGraphEventWaitNodeGetEvent + __cuGraphEventWaitNodeGetEvent = dlfcn.dlsym(handle, 'cuGraphEventWaitNodeGetEvent') + global __cuGraphEventWaitNodeSetEvent + __cuGraphEventWaitNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphEventWaitNodeSetEvent') + global __cuGraphAddExternalSemaphoresSignalNode + __cuGraphAddExternalSemaphoresSignalNode = dlfcn.dlsym(handle, 'cuGraphAddExternalSemaphoresSignalNode') + global __cuGraphExternalSemaphoresSignalNodeGetParams + __cuGraphExternalSemaphoresSignalNodeGetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresSignalNodeGetParams') + global __cuGraphExternalSemaphoresSignalNodeSetParams + __cuGraphExternalSemaphoresSignalNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresSignalNodeSetParams') + global __cuGraphAddExternalSemaphoresWaitNode + __cuGraphAddExternalSemaphoresWaitNode = dlfcn.dlsym(handle, 'cuGraphAddExternalSemaphoresWaitNode') + global __cuGraphExternalSemaphoresWaitNodeGetParams + __cuGraphExternalSemaphoresWaitNodeGetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresWaitNodeGetParams') + global __cuGraphExternalSemaphoresWaitNodeSetParams + __cuGraphExternalSemaphoresWaitNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExternalSemaphoresWaitNodeSetParams') + global __cuGraphAddBatchMemOpNode + __cuGraphAddBatchMemOpNode = dlfcn.dlsym(handle, 'cuGraphAddBatchMemOpNode') + global __cuGraphBatchMemOpNodeGetParams + __cuGraphBatchMemOpNodeGetParams = dlfcn.dlsym(handle, 'cuGraphBatchMemOpNodeGetParams') + global __cuGraphBatchMemOpNodeSetParams + __cuGraphBatchMemOpNodeSetParams = dlfcn.dlsym(handle, 'cuGraphBatchMemOpNodeSetParams') + global __cuGraphExecBatchMemOpNodeSetParams + __cuGraphExecBatchMemOpNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecBatchMemOpNodeSetParams') + global __cuGraphAddMemAllocNode + __cuGraphAddMemAllocNode = dlfcn.dlsym(handle, 'cuGraphAddMemAllocNode') + global __cuGraphMemAllocNodeGetParams + __cuGraphMemAllocNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemAllocNodeGetParams') + global __cuGraphAddMemFreeNode + __cuGraphAddMemFreeNode = dlfcn.dlsym(handle, 'cuGraphAddMemFreeNode') + global __cuGraphMemFreeNodeGetParams + __cuGraphMemFreeNodeGetParams = dlfcn.dlsym(handle, 'cuGraphMemFreeNodeGetParams') + global __cuDeviceGraphMemTrim + __cuDeviceGraphMemTrim = dlfcn.dlsym(handle, 'cuDeviceGraphMemTrim') + global __cuDeviceGetGraphMemAttribute + __cuDeviceGetGraphMemAttribute = dlfcn.dlsym(handle, 'cuDeviceGetGraphMemAttribute') + global __cuDeviceSetGraphMemAttribute + __cuDeviceSetGraphMemAttribute = dlfcn.dlsym(handle, 'cuDeviceSetGraphMemAttribute') + global __cuGraphClone + __cuGraphClone = dlfcn.dlsym(handle, 'cuGraphClone') + global __cuGraphNodeFindInClone + __cuGraphNodeFindInClone = dlfcn.dlsym(handle, 'cuGraphNodeFindInClone') + global __cuGraphNodeGetType + __cuGraphNodeGetType = dlfcn.dlsym(handle, 'cuGraphNodeGetType') + global __cuGraphGetNodes + __cuGraphGetNodes = dlfcn.dlsym(handle, 'cuGraphGetNodes') + global __cuGraphGetRootNodes + __cuGraphGetRootNodes = dlfcn.dlsym(handle, 'cuGraphGetRootNodes') + global __cuGraphGetEdges + __cuGraphGetEdges = dlfcn.dlsym(handle, 'cuGraphGetEdges') + global __cuGraphGetEdges_v2 + __cuGraphGetEdges_v2 = dlfcn.dlsym(handle, 'cuGraphGetEdges_v2') + global __cuGraphNodeGetDependencies + __cuGraphNodeGetDependencies = dlfcn.dlsym(handle, 'cuGraphNodeGetDependencies') + global __cuGraphNodeGetDependencies_v2 + __cuGraphNodeGetDependencies_v2 = dlfcn.dlsym(handle, 'cuGraphNodeGetDependencies_v2') + global __cuGraphNodeGetDependentNodes + __cuGraphNodeGetDependentNodes = dlfcn.dlsym(handle, 'cuGraphNodeGetDependentNodes') + global __cuGraphNodeGetDependentNodes_v2 + __cuGraphNodeGetDependentNodes_v2 = dlfcn.dlsym(handle, 'cuGraphNodeGetDependentNodes_v2') + global __cuGraphAddDependencies + __cuGraphAddDependencies = dlfcn.dlsym(handle, 'cuGraphAddDependencies') + global __cuGraphAddDependencies_v2 + __cuGraphAddDependencies_v2 = dlfcn.dlsym(handle, 'cuGraphAddDependencies_v2') + global __cuGraphRemoveDependencies + __cuGraphRemoveDependencies = dlfcn.dlsym(handle, 'cuGraphRemoveDependencies') + global __cuGraphRemoveDependencies_v2 + __cuGraphRemoveDependencies_v2 = dlfcn.dlsym(handle, 'cuGraphRemoveDependencies_v2') + global __cuGraphDestroyNode + __cuGraphDestroyNode = dlfcn.dlsym(handle, 'cuGraphDestroyNode') + global __cuGraphInstantiateWithFlags + __cuGraphInstantiateWithFlags = dlfcn.dlsym(handle, 'cuGraphInstantiateWithFlags') + global __cuGraphExecGetFlags + __cuGraphExecGetFlags = dlfcn.dlsym(handle, 'cuGraphExecGetFlags') + global __cuGraphExecKernelNodeSetParams_v2 + __cuGraphExecKernelNodeSetParams_v2 = dlfcn.dlsym(handle, 'cuGraphExecKernelNodeSetParams_v2') + global __cuGraphExecMemcpyNodeSetParams + __cuGraphExecMemcpyNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecMemcpyNodeSetParams') + global __cuGraphExecMemsetNodeSetParams + __cuGraphExecMemsetNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecMemsetNodeSetParams') + global __cuGraphExecHostNodeSetParams + __cuGraphExecHostNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecHostNodeSetParams') + global __cuGraphExecChildGraphNodeSetParams + __cuGraphExecChildGraphNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecChildGraphNodeSetParams') + global __cuGraphExecEventRecordNodeSetEvent + __cuGraphExecEventRecordNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphExecEventRecordNodeSetEvent') + global __cuGraphExecEventWaitNodeSetEvent + __cuGraphExecEventWaitNodeSetEvent = dlfcn.dlsym(handle, 'cuGraphExecEventWaitNodeSetEvent') + global __cuGraphExecExternalSemaphoresSignalNodeSetParams + __cuGraphExecExternalSemaphoresSignalNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecExternalSemaphoresSignalNodeSetParams') + global __cuGraphExecExternalSemaphoresWaitNodeSetParams + __cuGraphExecExternalSemaphoresWaitNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecExternalSemaphoresWaitNodeSetParams') + global __cuGraphNodeSetEnabled + __cuGraphNodeSetEnabled = dlfcn.dlsym(handle, 'cuGraphNodeSetEnabled') + global __cuGraphNodeGetEnabled + __cuGraphNodeGetEnabled = dlfcn.dlsym(handle, 'cuGraphNodeGetEnabled') + global __cuGraphExecDestroy + __cuGraphExecDestroy = dlfcn.dlsym(handle, 'cuGraphExecDestroy') + global __cuGraphDestroy + __cuGraphDestroy = dlfcn.dlsym(handle, 'cuGraphDestroy') + global __cuGraphExecUpdate_v2 + __cuGraphExecUpdate_v2 = dlfcn.dlsym(handle, 'cuGraphExecUpdate_v2') + global __cuGraphKernelNodeCopyAttributes + __cuGraphKernelNodeCopyAttributes = dlfcn.dlsym(handle, 'cuGraphKernelNodeCopyAttributes') + global __cuGraphKernelNodeGetAttribute + __cuGraphKernelNodeGetAttribute = dlfcn.dlsym(handle, 'cuGraphKernelNodeGetAttribute') + global __cuGraphKernelNodeSetAttribute + __cuGraphKernelNodeSetAttribute = dlfcn.dlsym(handle, 'cuGraphKernelNodeSetAttribute') + global __cuGraphDebugDotPrint + __cuGraphDebugDotPrint = dlfcn.dlsym(handle, 'cuGraphDebugDotPrint') + global __cuUserObjectCreate + __cuUserObjectCreate = dlfcn.dlsym(handle, 'cuUserObjectCreate') + global __cuUserObjectRetain + __cuUserObjectRetain = dlfcn.dlsym(handle, 'cuUserObjectRetain') + global __cuUserObjectRelease + __cuUserObjectRelease = dlfcn.dlsym(handle, 'cuUserObjectRelease') + global __cuGraphRetainUserObject + __cuGraphRetainUserObject = dlfcn.dlsym(handle, 'cuGraphRetainUserObject') + global __cuGraphReleaseUserObject + __cuGraphReleaseUserObject = dlfcn.dlsym(handle, 'cuGraphReleaseUserObject') + global __cuGraphAddNode + __cuGraphAddNode = dlfcn.dlsym(handle, 'cuGraphAddNode') + global __cuGraphAddNode_v2 + __cuGraphAddNode_v2 = dlfcn.dlsym(handle, 'cuGraphAddNode_v2') + global __cuGraphNodeSetParams + __cuGraphNodeSetParams = dlfcn.dlsym(handle, 'cuGraphNodeSetParams') + global __cuGraphExecNodeSetParams + __cuGraphExecNodeSetParams = dlfcn.dlsym(handle, 'cuGraphExecNodeSetParams') + global __cuGraphConditionalHandleCreate + __cuGraphConditionalHandleCreate = dlfcn.dlsym(handle, 'cuGraphConditionalHandleCreate') + global __cuOccupancyMaxActiveBlocksPerMultiprocessor + __cuOccupancyMaxActiveBlocksPerMultiprocessor = dlfcn.dlsym(handle, 'cuOccupancyMaxActiveBlocksPerMultiprocessor') + global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags = dlfcn.dlsym(handle, 'cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags') + global __cuOccupancyMaxPotentialBlockSize + __cuOccupancyMaxPotentialBlockSize = dlfcn.dlsym(handle, 'cuOccupancyMaxPotentialBlockSize') + global __cuOccupancyMaxPotentialBlockSizeWithFlags + __cuOccupancyMaxPotentialBlockSizeWithFlags = dlfcn.dlsym(handle, 'cuOccupancyMaxPotentialBlockSizeWithFlags') + global __cuOccupancyAvailableDynamicSMemPerBlock + __cuOccupancyAvailableDynamicSMemPerBlock = dlfcn.dlsym(handle, 'cuOccupancyAvailableDynamicSMemPerBlock') + global __cuOccupancyMaxPotentialClusterSize + __cuOccupancyMaxPotentialClusterSize = dlfcn.dlsym(handle, 'cuOccupancyMaxPotentialClusterSize') + global __cuOccupancyMaxActiveClusters + __cuOccupancyMaxActiveClusters = dlfcn.dlsym(handle, 'cuOccupancyMaxActiveClusters') + global __cuTexRefSetArray + __cuTexRefSetArray = dlfcn.dlsym(handle, 'cuTexRefSetArray') + global __cuTexRefSetMipmappedArray + __cuTexRefSetMipmappedArray = dlfcn.dlsym(handle, 'cuTexRefSetMipmappedArray') + global __cuTexRefSetAddress_v2 + __cuTexRefSetAddress_v2 = dlfcn.dlsym(handle, 'cuTexRefSetAddress_v2') + global __cuTexRefSetAddress2D_v3 + __cuTexRefSetAddress2D_v3 = dlfcn.dlsym(handle, 'cuTexRefSetAddress2D_v3') + global __cuTexRefSetFormat + __cuTexRefSetFormat = dlfcn.dlsym(handle, 'cuTexRefSetFormat') + global __cuTexRefSetAddressMode + __cuTexRefSetAddressMode = dlfcn.dlsym(handle, 'cuTexRefSetAddressMode') + global __cuTexRefSetFilterMode + __cuTexRefSetFilterMode = dlfcn.dlsym(handle, 'cuTexRefSetFilterMode') + global __cuTexRefSetMipmapFilterMode + __cuTexRefSetMipmapFilterMode = dlfcn.dlsym(handle, 'cuTexRefSetMipmapFilterMode') + global __cuTexRefSetMipmapLevelBias + __cuTexRefSetMipmapLevelBias = dlfcn.dlsym(handle, 'cuTexRefSetMipmapLevelBias') + global __cuTexRefSetMipmapLevelClamp + __cuTexRefSetMipmapLevelClamp = dlfcn.dlsym(handle, 'cuTexRefSetMipmapLevelClamp') + global __cuTexRefSetMaxAnisotropy + __cuTexRefSetMaxAnisotropy = dlfcn.dlsym(handle, 'cuTexRefSetMaxAnisotropy') + global __cuTexRefSetBorderColor + __cuTexRefSetBorderColor = dlfcn.dlsym(handle, 'cuTexRefSetBorderColor') + global __cuTexRefSetFlags + __cuTexRefSetFlags = dlfcn.dlsym(handle, 'cuTexRefSetFlags') + global __cuTexRefGetAddress_v2 + __cuTexRefGetAddress_v2 = dlfcn.dlsym(handle, 'cuTexRefGetAddress_v2') + global __cuTexRefGetArray + __cuTexRefGetArray = dlfcn.dlsym(handle, 'cuTexRefGetArray') + global __cuTexRefGetMipmappedArray + __cuTexRefGetMipmappedArray = dlfcn.dlsym(handle, 'cuTexRefGetMipmappedArray') + global __cuTexRefGetAddressMode + __cuTexRefGetAddressMode = dlfcn.dlsym(handle, 'cuTexRefGetAddressMode') + global __cuTexRefGetFilterMode + __cuTexRefGetFilterMode = dlfcn.dlsym(handle, 'cuTexRefGetFilterMode') + global __cuTexRefGetFormat + __cuTexRefGetFormat = dlfcn.dlsym(handle, 'cuTexRefGetFormat') + global __cuTexRefGetMipmapFilterMode + __cuTexRefGetMipmapFilterMode = dlfcn.dlsym(handle, 'cuTexRefGetMipmapFilterMode') + global __cuTexRefGetMipmapLevelBias + __cuTexRefGetMipmapLevelBias = dlfcn.dlsym(handle, 'cuTexRefGetMipmapLevelBias') + global __cuTexRefGetMipmapLevelClamp + __cuTexRefGetMipmapLevelClamp = dlfcn.dlsym(handle, 'cuTexRefGetMipmapLevelClamp') + global __cuTexRefGetMaxAnisotropy + __cuTexRefGetMaxAnisotropy = dlfcn.dlsym(handle, 'cuTexRefGetMaxAnisotropy') + global __cuTexRefGetBorderColor + __cuTexRefGetBorderColor = dlfcn.dlsym(handle, 'cuTexRefGetBorderColor') + global __cuTexRefGetFlags + __cuTexRefGetFlags = dlfcn.dlsym(handle, 'cuTexRefGetFlags') + global __cuTexRefCreate + __cuTexRefCreate = dlfcn.dlsym(handle, 'cuTexRefCreate') + global __cuTexRefDestroy + __cuTexRefDestroy = dlfcn.dlsym(handle, 'cuTexRefDestroy') + global __cuSurfRefSetArray + __cuSurfRefSetArray = dlfcn.dlsym(handle, 'cuSurfRefSetArray') + global __cuSurfRefGetArray + __cuSurfRefGetArray = dlfcn.dlsym(handle, 'cuSurfRefGetArray') + global __cuTexObjectCreate + __cuTexObjectCreate = dlfcn.dlsym(handle, 'cuTexObjectCreate') + global __cuTexObjectDestroy + __cuTexObjectDestroy = dlfcn.dlsym(handle, 'cuTexObjectDestroy') + global __cuTexObjectGetResourceDesc + __cuTexObjectGetResourceDesc = dlfcn.dlsym(handle, 'cuTexObjectGetResourceDesc') + global __cuTexObjectGetTextureDesc + __cuTexObjectGetTextureDesc = dlfcn.dlsym(handle, 'cuTexObjectGetTextureDesc') + global __cuTexObjectGetResourceViewDesc + __cuTexObjectGetResourceViewDesc = dlfcn.dlsym(handle, 'cuTexObjectGetResourceViewDesc') + global __cuSurfObjectCreate + __cuSurfObjectCreate = dlfcn.dlsym(handle, 'cuSurfObjectCreate') + global __cuSurfObjectDestroy + __cuSurfObjectDestroy = dlfcn.dlsym(handle, 'cuSurfObjectDestroy') + global __cuSurfObjectGetResourceDesc + __cuSurfObjectGetResourceDesc = dlfcn.dlsym(handle, 'cuSurfObjectGetResourceDesc') + global __cuTensorMapEncodeTiled + __cuTensorMapEncodeTiled = dlfcn.dlsym(handle, 'cuTensorMapEncodeTiled') + global __cuTensorMapEncodeIm2col + __cuTensorMapEncodeIm2col = dlfcn.dlsym(handle, 'cuTensorMapEncodeIm2col') + global __cuTensorMapEncodeIm2colWide + __cuTensorMapEncodeIm2colWide = dlfcn.dlsym(handle, 'cuTensorMapEncodeIm2colWide') + global __cuTensorMapReplaceAddress + __cuTensorMapReplaceAddress = dlfcn.dlsym(handle, 'cuTensorMapReplaceAddress') + global __cuDeviceCanAccessPeer + __cuDeviceCanAccessPeer = dlfcn.dlsym(handle, 'cuDeviceCanAccessPeer') + global __cuCtxEnablePeerAccess + __cuCtxEnablePeerAccess = dlfcn.dlsym(handle, 'cuCtxEnablePeerAccess') + global __cuCtxDisablePeerAccess + __cuCtxDisablePeerAccess = dlfcn.dlsym(handle, 'cuCtxDisablePeerAccess') + global __cuDeviceGetP2PAttribute + __cuDeviceGetP2PAttribute = dlfcn.dlsym(handle, 'cuDeviceGetP2PAttribute') + global __cuGraphicsUnregisterResource + __cuGraphicsUnregisterResource = dlfcn.dlsym(handle, 'cuGraphicsUnregisterResource') + global __cuGraphicsSubResourceGetMappedArray + __cuGraphicsSubResourceGetMappedArray = dlfcn.dlsym(handle, 'cuGraphicsSubResourceGetMappedArray') + global __cuGraphicsResourceGetMappedMipmappedArray + __cuGraphicsResourceGetMappedMipmappedArray = dlfcn.dlsym(handle, 'cuGraphicsResourceGetMappedMipmappedArray') + global __cuGraphicsResourceGetMappedPointer_v2 + __cuGraphicsResourceGetMappedPointer_v2 = dlfcn.dlsym(handle, 'cuGraphicsResourceGetMappedPointer_v2') + global __cuGraphicsResourceSetMapFlags_v2 + __cuGraphicsResourceSetMapFlags_v2 = dlfcn.dlsym(handle, 'cuGraphicsResourceSetMapFlags_v2') + global __cuGetProcAddress_v2 + __cuGetProcAddress_v2 = dlfcn.dlsym(handle, 'cuGetProcAddress_v2') + global __cuCoredumpGetAttribute + __cuCoredumpGetAttribute = dlfcn.dlsym(handle, 'cuCoredumpGetAttribute') + global __cuCoredumpGetAttributeGlobal + __cuCoredumpGetAttributeGlobal = dlfcn.dlsym(handle, 'cuCoredumpGetAttributeGlobal') + global __cuCoredumpSetAttribute + __cuCoredumpSetAttribute = dlfcn.dlsym(handle, 'cuCoredumpSetAttribute') + global __cuCoredumpSetAttributeGlobal + __cuCoredumpSetAttributeGlobal = dlfcn.dlsym(handle, 'cuCoredumpSetAttributeGlobal') + global __cuGetExportTable + __cuGetExportTable = dlfcn.dlsym(handle, 'cuGetExportTable') + global __cuGreenCtxCreate + __cuGreenCtxCreate = dlfcn.dlsym(handle, 'cuGreenCtxCreate') + global __cuGreenCtxDestroy + __cuGreenCtxDestroy = dlfcn.dlsym(handle, 'cuGreenCtxDestroy') + global __cuCtxFromGreenCtx + __cuCtxFromGreenCtx = dlfcn.dlsym(handle, 'cuCtxFromGreenCtx') + global __cuDeviceGetDevResource + __cuDeviceGetDevResource = dlfcn.dlsym(handle, 'cuDeviceGetDevResource') + global __cuCtxGetDevResource + __cuCtxGetDevResource = dlfcn.dlsym(handle, 'cuCtxGetDevResource') + global __cuGreenCtxGetDevResource + __cuGreenCtxGetDevResource = dlfcn.dlsym(handle, 'cuGreenCtxGetDevResource') + global __cuDevSmResourceSplitByCount + __cuDevSmResourceSplitByCount = dlfcn.dlsym(handle, 'cuDevSmResourceSplitByCount') + global __cuDevResourceGenerateDesc + __cuDevResourceGenerateDesc = dlfcn.dlsym(handle, 'cuDevResourceGenerateDesc') + global __cuGreenCtxRecordEvent + __cuGreenCtxRecordEvent = dlfcn.dlsym(handle, 'cuGreenCtxRecordEvent') + global __cuGreenCtxWaitEvent + __cuGreenCtxWaitEvent = dlfcn.dlsym(handle, 'cuGreenCtxWaitEvent') + global __cuStreamGetGreenCtx + __cuStreamGetGreenCtx = dlfcn.dlsym(handle, 'cuStreamGetGreenCtx') + global __cuGreenCtxStreamCreate + __cuGreenCtxStreamCreate = dlfcn.dlsym(handle, 'cuGreenCtxStreamCreate') + global __cuLogsRegisterCallback + __cuLogsRegisterCallback = dlfcn.dlsym(handle, 'cuLogsRegisterCallback') + global __cuLogsUnregisterCallback + __cuLogsUnregisterCallback = dlfcn.dlsym(handle, 'cuLogsUnregisterCallback') + global __cuLogsCurrent + __cuLogsCurrent = dlfcn.dlsym(handle, 'cuLogsCurrent') + global __cuLogsDumpToFile + __cuLogsDumpToFile = dlfcn.dlsym(handle, 'cuLogsDumpToFile') + global __cuLogsDumpToMemory + __cuLogsDumpToMemory = dlfcn.dlsym(handle, 'cuLogsDumpToMemory') + global __cuCheckpointProcessGetRestoreThreadId + __cuCheckpointProcessGetRestoreThreadId = dlfcn.dlsym(handle, 'cuCheckpointProcessGetRestoreThreadId') + global __cuCheckpointProcessGetState + __cuCheckpointProcessGetState = dlfcn.dlsym(handle, 'cuCheckpointProcessGetState') + global __cuCheckpointProcessLock + __cuCheckpointProcessLock = dlfcn.dlsym(handle, 'cuCheckpointProcessLock') + global __cuCheckpointProcessCheckpoint + __cuCheckpointProcessCheckpoint = dlfcn.dlsym(handle, 'cuCheckpointProcessCheckpoint') + global __cuCheckpointProcessRestore + __cuCheckpointProcessRestore = dlfcn.dlsym(handle, 'cuCheckpointProcessRestore') + global __cuCheckpointProcessUnlock + __cuCheckpointProcessUnlock = dlfcn.dlsym(handle, 'cuCheckpointProcessUnlock') + global __cuProfilerStart + __cuProfilerStart = dlfcn.dlsym(handle, 'cuProfilerStart') + global __cuProfilerStop + __cuProfilerStop = dlfcn.dlsym(handle, 'cuProfilerStop') + global __cuGraphicsEGLRegisterImage + __cuGraphicsEGLRegisterImage = dlfcn.dlsym(handle, 'cuGraphicsEGLRegisterImage') + global __cuEGLStreamConsumerConnect + __cuEGLStreamConsumerConnect = dlfcn.dlsym(handle, 'cuEGLStreamConsumerConnect') + global __cuEGLStreamConsumerConnectWithFlags + __cuEGLStreamConsumerConnectWithFlags = dlfcn.dlsym(handle, 'cuEGLStreamConsumerConnectWithFlags') + global __cuEGLStreamConsumerDisconnect + __cuEGLStreamConsumerDisconnect = dlfcn.dlsym(handle, 'cuEGLStreamConsumerDisconnect') + global __cuEGLStreamConsumerAcquireFrame + __cuEGLStreamConsumerAcquireFrame = dlfcn.dlsym(handle, 'cuEGLStreamConsumerAcquireFrame') + global __cuEGLStreamConsumerReleaseFrame + __cuEGLStreamConsumerReleaseFrame = dlfcn.dlsym(handle, 'cuEGLStreamConsumerReleaseFrame') + global __cuEGLStreamProducerConnect + __cuEGLStreamProducerConnect = dlfcn.dlsym(handle, 'cuEGLStreamProducerConnect') + global __cuEGLStreamProducerDisconnect + __cuEGLStreamProducerDisconnect = dlfcn.dlsym(handle, 'cuEGLStreamProducerDisconnect') + global __cuEGLStreamProducerPresentFrame + __cuEGLStreamProducerPresentFrame = dlfcn.dlsym(handle, 'cuEGLStreamProducerPresentFrame') + global __cuEGLStreamProducerReturnFrame + __cuEGLStreamProducerReturnFrame = dlfcn.dlsym(handle, 'cuEGLStreamProducerReturnFrame') + global __cuGraphicsResourceGetMappedEglFrame + __cuGraphicsResourceGetMappedEglFrame = dlfcn.dlsym(handle, 'cuGraphicsResourceGetMappedEglFrame') + global __cuEventCreateFromEGLSync + __cuEventCreateFromEGLSync = dlfcn.dlsym(handle, 'cuEventCreateFromEGLSync') + global __cuGraphicsGLRegisterBuffer + __cuGraphicsGLRegisterBuffer = dlfcn.dlsym(handle, 'cuGraphicsGLRegisterBuffer') + global __cuGraphicsGLRegisterImage + __cuGraphicsGLRegisterImage = dlfcn.dlsym(handle, 'cuGraphicsGLRegisterImage') + global __cuGLGetDevices_v2 + __cuGLGetDevices_v2 = dlfcn.dlsym(handle, 'cuGLGetDevices_v2') + global __cuVDPAUGetDevice + __cuVDPAUGetDevice = dlfcn.dlsym(handle, 'cuVDPAUGetDevice') + global __cuVDPAUCtxCreate_v2 + __cuVDPAUCtxCreate_v2 = dlfcn.dlsym(handle, 'cuVDPAUCtxCreate_v2') + global __cuGraphicsVDPAURegisterVideoSurface + __cuGraphicsVDPAURegisterVideoSurface = dlfcn.dlsym(handle, 'cuGraphicsVDPAURegisterVideoSurface') + global __cuGraphicsVDPAURegisterOutputSurface + __cuGraphicsVDPAURegisterOutputSurface = dlfcn.dlsym(handle, 'cuGraphicsVDPAURegisterOutputSurface') + __cuPythonInit = True + return 0 + +# Create a very small function to check whether we are init'ed, so the C +# compiler can inline it. +cdef inline int cuPythonInit() except -1 nogil: + if __cuPythonInit: + return 0 + return _cuPythonInit() + +cdef CUresult _cuGetErrorString(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGetErrorString + cuPythonInit() + if __cuGetErrorString == NULL: + with gil: + raise RuntimeError('Function "cuGetErrorString" not found') + err = ( __cuGetErrorString)(error, pStr) + return err + +cdef CUresult _cuGetErrorName(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGetErrorName + cuPythonInit() + if __cuGetErrorName == NULL: + with gil: + raise RuntimeError('Function "cuGetErrorName" not found') + err = ( __cuGetErrorName)(error, pStr) + return err + +cdef CUresult _cuInit(unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuInit + cuPythonInit() + if __cuInit == NULL: + with gil: + raise RuntimeError('Function "cuInit" not found') + err = ( __cuInit)(Flags) + return err + +cdef CUresult _cuDriverGetVersion(int* driverVersion) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDriverGetVersion + cuPythonInit() + if __cuDriverGetVersion == NULL: + with gil: + raise RuntimeError('Function "cuDriverGetVersion" not found') + err = ( __cuDriverGetVersion)(driverVersion) + return err + +cdef CUresult _cuDeviceGet(CUdevice* device, int ordinal) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGet + cuPythonInit() + if __cuDeviceGet == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGet" not found') + err = ( __cuDeviceGet)(device, ordinal) + return err + +cdef CUresult _cuDeviceGetCount(int* count) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetCount + cuPythonInit() + if __cuDeviceGetCount == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetCount" not found') + err = ( __cuDeviceGetCount)(count) + return err + +cdef CUresult _cuDeviceGetName(char* name, int length, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetName + cuPythonInit() + if __cuDeviceGetName == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetName" not found') + err = ( __cuDeviceGetName)(name, length, dev) + return err + +cdef CUresult _cuDeviceGetUuid(CUuuid* uuid, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetUuid + cuPythonInit() + if __cuDeviceGetUuid == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetUuid" not found') + err = ( __cuDeviceGetUuid)(uuid, dev) + return err + +cdef CUresult _cuDeviceGetUuid_v2(CUuuid* uuid, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetUuid_v2 + cuPythonInit() + if __cuDeviceGetUuid_v2 == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetUuid_v2" not found') + err = ( __cuDeviceGetUuid_v2)(uuid, dev) + return err + +cdef CUresult _cuDeviceGetLuid(char* luid, unsigned int* deviceNodeMask, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetLuid + cuPythonInit() + if __cuDeviceGetLuid == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetLuid" not found') + err = ( __cuDeviceGetLuid)(luid, deviceNodeMask, dev) + return err + +cdef CUresult _cuDeviceTotalMem_v2(size_t* numbytes, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceTotalMem_v2 + cuPythonInit() + if __cuDeviceTotalMem_v2 == NULL: + with gil: + raise RuntimeError('Function "cuDeviceTotalMem_v2" not found') + err = ( __cuDeviceTotalMem_v2)(numbytes, dev) + return err + +cdef CUresult _cuDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, CUarray_format pformat, unsigned numChannels, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetTexture1DLinearMaxWidth + cuPythonInit() + if __cuDeviceGetTexture1DLinearMaxWidth == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetTexture1DLinearMaxWidth" not found') + err = ( __cuDeviceGetTexture1DLinearMaxWidth)(maxWidthInElements, pformat, numChannels, dev) + return err + +cdef CUresult _cuDeviceGetAttribute(int* pi, CUdevice_attribute attrib, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetAttribute + cuPythonInit() + if __cuDeviceGetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetAttribute" not found') + err = ( __cuDeviceGetAttribute)(pi, attrib, dev) + return err + +cdef CUresult _cuDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, CUdevice dev, int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetNvSciSyncAttributes + cuPythonInit() + if __cuDeviceGetNvSciSyncAttributes == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetNvSciSyncAttributes" not found') + err = ( __cuDeviceGetNvSciSyncAttributes)(nvSciSyncAttrList, dev, flags) + return err + +cdef CUresult _cuDeviceSetMemPool(CUdevice dev, CUmemoryPool pool) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceSetMemPool + cuPythonInit() + if __cuDeviceSetMemPool == NULL: + with gil: + raise RuntimeError('Function "cuDeviceSetMemPool" not found') + err = ( __cuDeviceSetMemPool)(dev, pool) + return err + +cdef CUresult _cuDeviceGetMemPool(CUmemoryPool* pool, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetMemPool + cuPythonInit() + if __cuDeviceGetMemPool == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetMemPool" not found') + err = ( __cuDeviceGetMemPool)(pool, dev) + return err + +cdef CUresult _cuDeviceGetDefaultMemPool(CUmemoryPool* pool_out, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetDefaultMemPool + cuPythonInit() + if __cuDeviceGetDefaultMemPool == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetDefaultMemPool" not found') + err = ( __cuDeviceGetDefaultMemPool)(pool_out, dev) + return err + +cdef CUresult _cuDeviceGetExecAffinitySupport(int* pi, CUexecAffinityType typename, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetExecAffinitySupport + cuPythonInit() + if __cuDeviceGetExecAffinitySupport == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetExecAffinitySupport" not found') + err = ( __cuDeviceGetExecAffinitySupport)(pi, typename, dev) + return err + +cdef CUresult _cuFlushGPUDirectRDMAWrites(CUflushGPUDirectRDMAWritesTarget target, CUflushGPUDirectRDMAWritesScope scope) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuFlushGPUDirectRDMAWrites + cuPythonInit() + if __cuFlushGPUDirectRDMAWrites == NULL: + with gil: + raise RuntimeError('Function "cuFlushGPUDirectRDMAWrites" not found') + err = ( __cuFlushGPUDirectRDMAWrites)(target, scope) + return err + +cdef CUresult _cuDeviceGetProperties(CUdevprop* prop, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetProperties + cuPythonInit() + if __cuDeviceGetProperties == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetProperties" not found') + err = ( __cuDeviceGetProperties)(prop, dev) + return err + +cdef CUresult _cuDeviceComputeCapability(int* major, int* minor, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceComputeCapability + cuPythonInit() + if __cuDeviceComputeCapability == NULL: + with gil: + raise RuntimeError('Function "cuDeviceComputeCapability" not found') + err = ( __cuDeviceComputeCapability)(major, minor, dev) + return err + +cdef CUresult _cuDevicePrimaryCtxRetain(CUcontext* pctx, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDevicePrimaryCtxRetain + cuPythonInit() + if __cuDevicePrimaryCtxRetain == NULL: + with gil: + raise RuntimeError('Function "cuDevicePrimaryCtxRetain" not found') + err = ( __cuDevicePrimaryCtxRetain)(pctx, dev) + return err + +cdef CUresult _cuDevicePrimaryCtxRelease_v2(CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDevicePrimaryCtxRelease_v2 + cuPythonInit() + if __cuDevicePrimaryCtxRelease_v2 == NULL: + with gil: + raise RuntimeError('Function "cuDevicePrimaryCtxRelease_v2" not found') + err = ( __cuDevicePrimaryCtxRelease_v2)(dev) + return err + +cdef CUresult _cuDevicePrimaryCtxSetFlags_v2(CUdevice dev, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDevicePrimaryCtxSetFlags_v2 + cuPythonInit() + if __cuDevicePrimaryCtxSetFlags_v2 == NULL: + with gil: + raise RuntimeError('Function "cuDevicePrimaryCtxSetFlags_v2" not found') + err = ( __cuDevicePrimaryCtxSetFlags_v2)(dev, flags) + return err + +cdef CUresult _cuDevicePrimaryCtxGetState(CUdevice dev, unsigned int* flags, int* active) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDevicePrimaryCtxGetState + cuPythonInit() + if __cuDevicePrimaryCtxGetState == NULL: + with gil: + raise RuntimeError('Function "cuDevicePrimaryCtxGetState" not found') + err = ( __cuDevicePrimaryCtxGetState)(dev, flags, active) + return err + +cdef CUresult _cuDevicePrimaryCtxReset_v2(CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDevicePrimaryCtxReset_v2 + cuPythonInit() + if __cuDevicePrimaryCtxReset_v2 == NULL: + with gil: + raise RuntimeError('Function "cuDevicePrimaryCtxReset_v2" not found') + err = ( __cuDevicePrimaryCtxReset_v2)(dev) + return err + +cdef CUresult _cuCtxCreate_v2(CUcontext* pctx, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxCreate_v2 + cuPythonInit() + if __cuCtxCreate_v2 == NULL: + with gil: + raise RuntimeError('Function "cuCtxCreate_v2" not found') + err = ( __cuCtxCreate_v2)(pctx, flags, dev) + return err + +cdef CUresult _cuCtxCreate_v3(CUcontext* pctx, CUexecAffinityParam* paramsArray, int numParams, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxCreate_v3 + cuPythonInit() + if __cuCtxCreate_v3 == NULL: + with gil: + raise RuntimeError('Function "cuCtxCreate_v3" not found') + err = ( __cuCtxCreate_v3)(pctx, paramsArray, numParams, flags, dev) + return err + +cdef CUresult _cuCtxCreate_v4(CUcontext* pctx, CUctxCreateParams* ctxCreateParams, unsigned int flags, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxCreate_v4 + cuPythonInit() + if __cuCtxCreate_v4 == NULL: + with gil: + raise RuntimeError('Function "cuCtxCreate_v4" not found') + err = ( __cuCtxCreate_v4)(pctx, ctxCreateParams, flags, dev) + return err + +cdef CUresult _cuCtxDestroy_v2(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxDestroy_v2 + cuPythonInit() + if __cuCtxDestroy_v2 == NULL: + with gil: + raise RuntimeError('Function "cuCtxDestroy_v2" not found') + err = ( __cuCtxDestroy_v2)(ctx) + return err + +cdef CUresult _cuCtxPushCurrent_v2(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxPushCurrent_v2 + cuPythonInit() + if __cuCtxPushCurrent_v2 == NULL: + with gil: + raise RuntimeError('Function "cuCtxPushCurrent_v2" not found') + err = ( __cuCtxPushCurrent_v2)(ctx) + return err + +cdef CUresult _cuCtxPopCurrent_v2(CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxPopCurrent_v2 + cuPythonInit() + if __cuCtxPopCurrent_v2 == NULL: + with gil: + raise RuntimeError('Function "cuCtxPopCurrent_v2" not found') + err = ( __cuCtxPopCurrent_v2)(pctx) + return err + +cdef CUresult _cuCtxSetCurrent(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxSetCurrent + cuPythonInit() + if __cuCtxSetCurrent == NULL: + with gil: + raise RuntimeError('Function "cuCtxSetCurrent" not found') + err = ( __cuCtxSetCurrent)(ctx) + return err + +cdef CUresult _cuCtxGetCurrent(CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxGetCurrent + cuPythonInit() + if __cuCtxGetCurrent == NULL: + with gil: + raise RuntimeError('Function "cuCtxGetCurrent" not found') + err = ( __cuCtxGetCurrent)(pctx) + return err + +cdef CUresult _cuCtxGetDevice(CUdevice* device) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxGetDevice + cuPythonInit() + if __cuCtxGetDevice == NULL: + with gil: + raise RuntimeError('Function "cuCtxGetDevice" not found') + err = ( __cuCtxGetDevice)(device) + return err + +cdef CUresult _cuCtxGetFlags(unsigned int* flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxGetFlags + cuPythonInit() + if __cuCtxGetFlags == NULL: + with gil: + raise RuntimeError('Function "cuCtxGetFlags" not found') + err = ( __cuCtxGetFlags)(flags) + return err + +cdef CUresult _cuCtxSetFlags(unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxSetFlags + cuPythonInit() + if __cuCtxSetFlags == NULL: + with gil: + raise RuntimeError('Function "cuCtxSetFlags" not found') + err = ( __cuCtxSetFlags)(flags) + return err + +cdef CUresult _cuCtxGetId(CUcontext ctx, unsigned long long* ctxId) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxGetId + cuPythonInit() + if __cuCtxGetId == NULL: + with gil: + raise RuntimeError('Function "cuCtxGetId" not found') + err = ( __cuCtxGetId)(ctx, ctxId) + return err + +cdef CUresult _cuCtxSynchronize() except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxSynchronize + cuPythonInit() + if __cuCtxSynchronize == NULL: + with gil: + raise RuntimeError('Function "cuCtxSynchronize" not found') + err = ( __cuCtxSynchronize)() + return err + +cdef CUresult _cuCtxSetLimit(CUlimit limit, size_t value) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxSetLimit + cuPythonInit() + if __cuCtxSetLimit == NULL: + with gil: + raise RuntimeError('Function "cuCtxSetLimit" not found') + err = ( __cuCtxSetLimit)(limit, value) + return err + +cdef CUresult _cuCtxGetLimit(size_t* pvalue, CUlimit limit) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxGetLimit + cuPythonInit() + if __cuCtxGetLimit == NULL: + with gil: + raise RuntimeError('Function "cuCtxGetLimit" not found') + err = ( __cuCtxGetLimit)(pvalue, limit) + return err + +cdef CUresult _cuCtxGetCacheConfig(CUfunc_cache* pconfig) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxGetCacheConfig + cuPythonInit() + if __cuCtxGetCacheConfig == NULL: + with gil: + raise RuntimeError('Function "cuCtxGetCacheConfig" not found') + err = ( __cuCtxGetCacheConfig)(pconfig) + return err + +cdef CUresult _cuCtxSetCacheConfig(CUfunc_cache config) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxSetCacheConfig + cuPythonInit() + if __cuCtxSetCacheConfig == NULL: + with gil: + raise RuntimeError('Function "cuCtxSetCacheConfig" not found') + err = ( __cuCtxSetCacheConfig)(config) + return err + +cdef CUresult _cuCtxGetApiVersion(CUcontext ctx, unsigned int* version) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxGetApiVersion + cuPythonInit() + if __cuCtxGetApiVersion == NULL: + with gil: + raise RuntimeError('Function "cuCtxGetApiVersion" not found') + err = ( __cuCtxGetApiVersion)(ctx, version) + return err + +cdef CUresult _cuCtxGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxGetStreamPriorityRange + cuPythonInit() + if __cuCtxGetStreamPriorityRange == NULL: + with gil: + raise RuntimeError('Function "cuCtxGetStreamPriorityRange" not found') + err = ( __cuCtxGetStreamPriorityRange)(leastPriority, greatestPriority) + return err + +cdef CUresult _cuCtxResetPersistingL2Cache() except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxResetPersistingL2Cache + cuPythonInit() + if __cuCtxResetPersistingL2Cache == NULL: + with gil: + raise RuntimeError('Function "cuCtxResetPersistingL2Cache" not found') + err = ( __cuCtxResetPersistingL2Cache)() + return err + +cdef CUresult _cuCtxGetExecAffinity(CUexecAffinityParam* pExecAffinity, CUexecAffinityType typename) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxGetExecAffinity + cuPythonInit() + if __cuCtxGetExecAffinity == NULL: + with gil: + raise RuntimeError('Function "cuCtxGetExecAffinity" not found') + err = ( __cuCtxGetExecAffinity)(pExecAffinity, typename) + return err + +cdef CUresult _cuCtxRecordEvent(CUcontext hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxRecordEvent + cuPythonInit() + if __cuCtxRecordEvent == NULL: + with gil: + raise RuntimeError('Function "cuCtxRecordEvent" not found') + err = ( __cuCtxRecordEvent)(hCtx, hEvent) + return err + +cdef CUresult _cuCtxWaitEvent(CUcontext hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxWaitEvent + cuPythonInit() + if __cuCtxWaitEvent == NULL: + with gil: + raise RuntimeError('Function "cuCtxWaitEvent" not found') + err = ( __cuCtxWaitEvent)(hCtx, hEvent) + return err + +cdef CUresult _cuCtxAttach(CUcontext* pctx, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxAttach + cuPythonInit() + if __cuCtxAttach == NULL: + with gil: + raise RuntimeError('Function "cuCtxAttach" not found') + err = ( __cuCtxAttach)(pctx, flags) + return err + +cdef CUresult _cuCtxDetach(CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxDetach + cuPythonInit() + if __cuCtxDetach == NULL: + with gil: + raise RuntimeError('Function "cuCtxDetach" not found') + err = ( __cuCtxDetach)(ctx) + return err + +cdef CUresult _cuCtxGetSharedMemConfig(CUsharedconfig* pConfig) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxGetSharedMemConfig + cuPythonInit() + if __cuCtxGetSharedMemConfig == NULL: + with gil: + raise RuntimeError('Function "cuCtxGetSharedMemConfig" not found') + err = ( __cuCtxGetSharedMemConfig)(pConfig) + return err + +cdef CUresult _cuCtxSetSharedMemConfig(CUsharedconfig config) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxSetSharedMemConfig + cuPythonInit() + if __cuCtxSetSharedMemConfig == NULL: + with gil: + raise RuntimeError('Function "cuCtxSetSharedMemConfig" not found') + err = ( __cuCtxSetSharedMemConfig)(config) + return err + +cdef CUresult _cuModuleLoad(CUmodule* module, const char* fname) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuModuleLoad + cuPythonInit() + if __cuModuleLoad == NULL: + with gil: + raise RuntimeError('Function "cuModuleLoad" not found') + err = ( __cuModuleLoad)(module, fname) + return err + +cdef CUresult _cuModuleLoadData(CUmodule* module, const void* image) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuModuleLoadData + cuPythonInit() + if __cuModuleLoadData == NULL: + with gil: + raise RuntimeError('Function "cuModuleLoadData" not found') + err = ( __cuModuleLoadData)(module, image) + return err + +cdef CUresult _cuModuleLoadDataEx(CUmodule* module, const void* image, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuModuleLoadDataEx + cuPythonInit() + if __cuModuleLoadDataEx == NULL: + with gil: + raise RuntimeError('Function "cuModuleLoadDataEx" not found') + err = ( __cuModuleLoadDataEx)(module, image, numOptions, options, optionValues) + return err + +cdef CUresult _cuModuleLoadFatBinary(CUmodule* module, const void* fatCubin) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuModuleLoadFatBinary + cuPythonInit() + if __cuModuleLoadFatBinary == NULL: + with gil: + raise RuntimeError('Function "cuModuleLoadFatBinary" not found') + err = ( __cuModuleLoadFatBinary)(module, fatCubin) + return err + +cdef CUresult _cuModuleUnload(CUmodule hmod) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuModuleUnload + cuPythonInit() + if __cuModuleUnload == NULL: + with gil: + raise RuntimeError('Function "cuModuleUnload" not found') + err = ( __cuModuleUnload)(hmod) + return err + +cdef CUresult _cuModuleGetLoadingMode(CUmoduleLoadingMode* mode) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuModuleGetLoadingMode + cuPythonInit() + if __cuModuleGetLoadingMode == NULL: + with gil: + raise RuntimeError('Function "cuModuleGetLoadingMode" not found') + err = ( __cuModuleGetLoadingMode)(mode) + return err + +cdef CUresult _cuModuleGetFunction(CUfunction* hfunc, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuModuleGetFunction + cuPythonInit() + if __cuModuleGetFunction == NULL: + with gil: + raise RuntimeError('Function "cuModuleGetFunction" not found') + err = ( __cuModuleGetFunction)(hfunc, hmod, name) + return err + +cdef CUresult _cuModuleGetFunctionCount(unsigned int* count, CUmodule mod) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuModuleGetFunctionCount + cuPythonInit() + if __cuModuleGetFunctionCount == NULL: + with gil: + raise RuntimeError('Function "cuModuleGetFunctionCount" not found') + err = ( __cuModuleGetFunctionCount)(count, mod) + return err + +cdef CUresult _cuModuleEnumerateFunctions(CUfunction* functions, unsigned int numFunctions, CUmodule mod) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuModuleEnumerateFunctions + cuPythonInit() + if __cuModuleEnumerateFunctions == NULL: + with gil: + raise RuntimeError('Function "cuModuleEnumerateFunctions" not found') + err = ( __cuModuleEnumerateFunctions)(functions, numFunctions, mod) + return err + +cdef CUresult _cuModuleGetGlobal_v2(CUdeviceptr* dptr, size_t* numbytes, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuModuleGetGlobal_v2 + cuPythonInit() + if __cuModuleGetGlobal_v2 == NULL: + with gil: + raise RuntimeError('Function "cuModuleGetGlobal_v2" not found') + err = ( __cuModuleGetGlobal_v2)(dptr, numbytes, hmod, name) + return err + +cdef CUresult _cuLinkCreate_v2(unsigned int numOptions, CUjit_option* options, void** optionValues, CUlinkState* stateOut) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLinkCreate_v2 + cuPythonInit() + if __cuLinkCreate_v2 == NULL: + with gil: + raise RuntimeError('Function "cuLinkCreate_v2" not found') + err = ( __cuLinkCreate_v2)(numOptions, options, optionValues, stateOut) + return err + +cdef CUresult _cuLinkAddData_v2(CUlinkState state, CUjitInputType typename, void* data, size_t size, const char* name, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLinkAddData_v2 + cuPythonInit() + if __cuLinkAddData_v2 == NULL: + with gil: + raise RuntimeError('Function "cuLinkAddData_v2" not found') + err = ( __cuLinkAddData_v2)(state, typename, data, size, name, numOptions, options, optionValues) + return err + +cdef CUresult _cuLinkAddFile_v2(CUlinkState state, CUjitInputType typename, const char* path, unsigned int numOptions, CUjit_option* options, void** optionValues) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLinkAddFile_v2 + cuPythonInit() + if __cuLinkAddFile_v2 == NULL: + with gil: + raise RuntimeError('Function "cuLinkAddFile_v2" not found') + err = ( __cuLinkAddFile_v2)(state, typename, path, numOptions, options, optionValues) + return err + +cdef CUresult _cuLinkComplete(CUlinkState state, void** cubinOut, size_t* sizeOut) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLinkComplete + cuPythonInit() + if __cuLinkComplete == NULL: + with gil: + raise RuntimeError('Function "cuLinkComplete" not found') + err = ( __cuLinkComplete)(state, cubinOut, sizeOut) + return err + +cdef CUresult _cuLinkDestroy(CUlinkState state) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLinkDestroy + cuPythonInit() + if __cuLinkDestroy == NULL: + with gil: + raise RuntimeError('Function "cuLinkDestroy" not found') + err = ( __cuLinkDestroy)(state) + return err + +cdef CUresult _cuModuleGetTexRef(CUtexref* pTexRef, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuModuleGetTexRef + cuPythonInit() + if __cuModuleGetTexRef == NULL: + with gil: + raise RuntimeError('Function "cuModuleGetTexRef" not found') + err = ( __cuModuleGetTexRef)(pTexRef, hmod, name) + return err + +cdef CUresult _cuModuleGetSurfRef(CUsurfref* pSurfRef, CUmodule hmod, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuModuleGetSurfRef + cuPythonInit() + if __cuModuleGetSurfRef == NULL: + with gil: + raise RuntimeError('Function "cuModuleGetSurfRef" not found') + err = ( __cuModuleGetSurfRef)(pSurfRef, hmod, name) + return err + +cdef CUresult _cuLibraryLoadData(CUlibrary* library, const void* code, CUjit_option* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, CUlibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLibraryLoadData + cuPythonInit() + if __cuLibraryLoadData == NULL: + with gil: + raise RuntimeError('Function "cuLibraryLoadData" not found') + err = ( __cuLibraryLoadData)(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + return err + +cdef CUresult _cuLibraryLoadFromFile(CUlibrary* library, const char* fileName, CUjit_option* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, CUlibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLibraryLoadFromFile + cuPythonInit() + if __cuLibraryLoadFromFile == NULL: + with gil: + raise RuntimeError('Function "cuLibraryLoadFromFile" not found') + err = ( __cuLibraryLoadFromFile)(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + return err + +cdef CUresult _cuLibraryUnload(CUlibrary library) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLibraryUnload + cuPythonInit() + if __cuLibraryUnload == NULL: + with gil: + raise RuntimeError('Function "cuLibraryUnload" not found') + err = ( __cuLibraryUnload)(library) + return err + +cdef CUresult _cuLibraryGetKernel(CUkernel* pKernel, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLibraryGetKernel + cuPythonInit() + if __cuLibraryGetKernel == NULL: + with gil: + raise RuntimeError('Function "cuLibraryGetKernel" not found') + err = ( __cuLibraryGetKernel)(pKernel, library, name) + return err + +cdef CUresult _cuLibraryGetKernelCount(unsigned int* count, CUlibrary lib) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLibraryGetKernelCount + cuPythonInit() + if __cuLibraryGetKernelCount == NULL: + with gil: + raise RuntimeError('Function "cuLibraryGetKernelCount" not found') + err = ( __cuLibraryGetKernelCount)(count, lib) + return err + +cdef CUresult _cuLibraryEnumerateKernels(CUkernel* kernels, unsigned int numKernels, CUlibrary lib) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLibraryEnumerateKernels + cuPythonInit() + if __cuLibraryEnumerateKernels == NULL: + with gil: + raise RuntimeError('Function "cuLibraryEnumerateKernels" not found') + err = ( __cuLibraryEnumerateKernels)(kernels, numKernels, lib) + return err + +cdef CUresult _cuLibraryGetModule(CUmodule* pMod, CUlibrary library) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLibraryGetModule + cuPythonInit() + if __cuLibraryGetModule == NULL: + with gil: + raise RuntimeError('Function "cuLibraryGetModule" not found') + err = ( __cuLibraryGetModule)(pMod, library) + return err + +cdef CUresult _cuKernelGetFunction(CUfunction* pFunc, CUkernel kernel) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuKernelGetFunction + cuPythonInit() + if __cuKernelGetFunction == NULL: + with gil: + raise RuntimeError('Function "cuKernelGetFunction" not found') + err = ( __cuKernelGetFunction)(pFunc, kernel) + return err + +cdef CUresult _cuKernelGetLibrary(CUlibrary* pLib, CUkernel kernel) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuKernelGetLibrary + cuPythonInit() + if __cuKernelGetLibrary == NULL: + with gil: + raise RuntimeError('Function "cuKernelGetLibrary" not found') + err = ( __cuKernelGetLibrary)(pLib, kernel) + return err + +cdef CUresult _cuLibraryGetGlobal(CUdeviceptr* dptr, size_t* numbytes, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLibraryGetGlobal + cuPythonInit() + if __cuLibraryGetGlobal == NULL: + with gil: + raise RuntimeError('Function "cuLibraryGetGlobal" not found') + err = ( __cuLibraryGetGlobal)(dptr, numbytes, library, name) + return err + +cdef CUresult _cuLibraryGetManaged(CUdeviceptr* dptr, size_t* numbytes, CUlibrary library, const char* name) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLibraryGetManaged + cuPythonInit() + if __cuLibraryGetManaged == NULL: + with gil: + raise RuntimeError('Function "cuLibraryGetManaged" not found') + err = ( __cuLibraryGetManaged)(dptr, numbytes, library, name) + return err + +cdef CUresult _cuLibraryGetUnifiedFunction(void** fptr, CUlibrary library, const char* symbol) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLibraryGetUnifiedFunction + cuPythonInit() + if __cuLibraryGetUnifiedFunction == NULL: + with gil: + raise RuntimeError('Function "cuLibraryGetUnifiedFunction" not found') + err = ( __cuLibraryGetUnifiedFunction)(fptr, library, symbol) + return err + +cdef CUresult _cuKernelGetAttribute(int* pi, CUfunction_attribute attrib, CUkernel kernel, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuKernelGetAttribute + cuPythonInit() + if __cuKernelGetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuKernelGetAttribute" not found') + err = ( __cuKernelGetAttribute)(pi, attrib, kernel, dev) + return err + +cdef CUresult _cuKernelSetAttribute(CUfunction_attribute attrib, int val, CUkernel kernel, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuKernelSetAttribute + cuPythonInit() + if __cuKernelSetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuKernelSetAttribute" not found') + err = ( __cuKernelSetAttribute)(attrib, val, kernel, dev) + return err + +cdef CUresult _cuKernelSetCacheConfig(CUkernel kernel, CUfunc_cache config, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuKernelSetCacheConfig + cuPythonInit() + if __cuKernelSetCacheConfig == NULL: + with gil: + raise RuntimeError('Function "cuKernelSetCacheConfig" not found') + err = ( __cuKernelSetCacheConfig)(kernel, config, dev) + return err + +cdef CUresult _cuKernelGetName(const char** name, CUkernel hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuKernelGetName + cuPythonInit() + if __cuKernelGetName == NULL: + with gil: + raise RuntimeError('Function "cuKernelGetName" not found') + err = ( __cuKernelGetName)(name, hfunc) + return err + +cdef CUresult _cuKernelGetParamInfo(CUkernel kernel, size_t paramIndex, size_t* paramOffset, size_t* paramSize) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuKernelGetParamInfo + cuPythonInit() + if __cuKernelGetParamInfo == NULL: + with gil: + raise RuntimeError('Function "cuKernelGetParamInfo" not found') + err = ( __cuKernelGetParamInfo)(kernel, paramIndex, paramOffset, paramSize) + return err + +cdef CUresult _cuMemGetInfo_v2(size_t* free, size_t* total) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemGetInfo_v2 + cuPythonInit() + if __cuMemGetInfo_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemGetInfo_v2" not found') + err = ( __cuMemGetInfo_v2)(free, total) + return err + +cdef CUresult _cuMemAlloc_v2(CUdeviceptr* dptr, size_t bytesize) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemAlloc_v2 + cuPythonInit() + if __cuMemAlloc_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemAlloc_v2" not found') + err = ( __cuMemAlloc_v2)(dptr, bytesize) + return err + +cdef CUresult _cuMemAllocPitch_v2(CUdeviceptr* dptr, size_t* pPitch, size_t WidthInBytes, size_t Height, unsigned int ElementSizeBytes) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemAllocPitch_v2 + cuPythonInit() + if __cuMemAllocPitch_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemAllocPitch_v2" not found') + err = ( __cuMemAllocPitch_v2)(dptr, pPitch, WidthInBytes, Height, ElementSizeBytes) + return err + +cdef CUresult _cuMemFree_v2(CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemFree_v2 + cuPythonInit() + if __cuMemFree_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemFree_v2" not found') + err = ( __cuMemFree_v2)(dptr) + return err + +cdef CUresult _cuMemGetAddressRange_v2(CUdeviceptr* pbase, size_t* psize, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemGetAddressRange_v2 + cuPythonInit() + if __cuMemGetAddressRange_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemGetAddressRange_v2" not found') + err = ( __cuMemGetAddressRange_v2)(pbase, psize, dptr) + return err + +cdef CUresult _cuMemAllocHost_v2(void** pp, size_t bytesize) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemAllocHost_v2 + cuPythonInit() + if __cuMemAllocHost_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemAllocHost_v2" not found') + err = ( __cuMemAllocHost_v2)(pp, bytesize) + return err + +cdef CUresult _cuMemFreeHost(void* p) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemFreeHost + cuPythonInit() + if __cuMemFreeHost == NULL: + with gil: + raise RuntimeError('Function "cuMemFreeHost" not found') + err = ( __cuMemFreeHost)(p) + return err + +cdef CUresult _cuMemHostAlloc(void** pp, size_t bytesize, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemHostAlloc + cuPythonInit() + if __cuMemHostAlloc == NULL: + with gil: + raise RuntimeError('Function "cuMemHostAlloc" not found') + err = ( __cuMemHostAlloc)(pp, bytesize, Flags) + return err + +cdef CUresult _cuMemHostGetDevicePointer_v2(CUdeviceptr* pdptr, void* p, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemHostGetDevicePointer_v2 + cuPythonInit() + if __cuMemHostGetDevicePointer_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemHostGetDevicePointer_v2" not found') + err = ( __cuMemHostGetDevicePointer_v2)(pdptr, p, Flags) + return err + +cdef CUresult _cuMemHostGetFlags(unsigned int* pFlags, void* p) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemHostGetFlags + cuPythonInit() + if __cuMemHostGetFlags == NULL: + with gil: + raise RuntimeError('Function "cuMemHostGetFlags" not found') + err = ( __cuMemHostGetFlags)(pFlags, p) + return err + +cdef CUresult _cuMemAllocManaged(CUdeviceptr* dptr, size_t bytesize, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemAllocManaged + cuPythonInit() + if __cuMemAllocManaged == NULL: + with gil: + raise RuntimeError('Function "cuMemAllocManaged" not found') + err = ( __cuMemAllocManaged)(dptr, bytesize, flags) + return err + +cdef CUresult _cuDeviceRegisterAsyncNotification(CUdevice device, CUasyncCallback callbackFunc, void* userData, CUasyncCallbackHandle* callback) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceRegisterAsyncNotification + cuPythonInit() + if __cuDeviceRegisterAsyncNotification == NULL: + with gil: + raise RuntimeError('Function "cuDeviceRegisterAsyncNotification" not found') + err = ( __cuDeviceRegisterAsyncNotification)(device, callbackFunc, userData, callback) + return err + +cdef CUresult _cuDeviceUnregisterAsyncNotification(CUdevice device, CUasyncCallbackHandle callback) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceUnregisterAsyncNotification + cuPythonInit() + if __cuDeviceUnregisterAsyncNotification == NULL: + with gil: + raise RuntimeError('Function "cuDeviceUnregisterAsyncNotification" not found') + err = ( __cuDeviceUnregisterAsyncNotification)(device, callback) + return err + +cdef CUresult _cuDeviceGetByPCIBusId(CUdevice* dev, const char* pciBusId) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetByPCIBusId + cuPythonInit() + if __cuDeviceGetByPCIBusId == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetByPCIBusId" not found') + err = ( __cuDeviceGetByPCIBusId)(dev, pciBusId) + return err + +cdef CUresult _cuDeviceGetPCIBusId(char* pciBusId, int length, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetPCIBusId + cuPythonInit() + if __cuDeviceGetPCIBusId == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetPCIBusId" not found') + err = ( __cuDeviceGetPCIBusId)(pciBusId, length, dev) + return err + +cdef CUresult _cuIpcGetEventHandle(CUipcEventHandle* pHandle, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuIpcGetEventHandle + cuPythonInit() + if __cuIpcGetEventHandle == NULL: + with gil: + raise RuntimeError('Function "cuIpcGetEventHandle" not found') + err = ( __cuIpcGetEventHandle)(pHandle, event) + return err + +cdef CUresult _cuIpcOpenEventHandle(CUevent* phEvent, CUipcEventHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuIpcOpenEventHandle + cuPythonInit() + if __cuIpcOpenEventHandle == NULL: + with gil: + raise RuntimeError('Function "cuIpcOpenEventHandle" not found') + err = ( __cuIpcOpenEventHandle)(phEvent, handle) + return err + +cdef CUresult _cuIpcGetMemHandle(CUipcMemHandle* pHandle, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuIpcGetMemHandle + cuPythonInit() + if __cuIpcGetMemHandle == NULL: + with gil: + raise RuntimeError('Function "cuIpcGetMemHandle" not found') + err = ( __cuIpcGetMemHandle)(pHandle, dptr) + return err + +cdef CUresult _cuIpcOpenMemHandle_v2(CUdeviceptr* pdptr, CUipcMemHandle handle, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuIpcOpenMemHandle_v2 + cuPythonInit() + if __cuIpcOpenMemHandle_v2 == NULL: + with gil: + raise RuntimeError('Function "cuIpcOpenMemHandle_v2" not found') + err = ( __cuIpcOpenMemHandle_v2)(pdptr, handle, Flags) + return err + +cdef CUresult _cuIpcCloseMemHandle(CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuIpcCloseMemHandle + cuPythonInit() + if __cuIpcCloseMemHandle == NULL: + with gil: + raise RuntimeError('Function "cuIpcCloseMemHandle" not found') + err = ( __cuIpcCloseMemHandle)(dptr) + return err + +cdef CUresult _cuMemHostRegister_v2(void* p, size_t bytesize, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemHostRegister_v2 + cuPythonInit() + if __cuMemHostRegister_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemHostRegister_v2" not found') + err = ( __cuMemHostRegister_v2)(p, bytesize, Flags) + return err + +cdef CUresult _cuMemHostUnregister(void* p) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemHostUnregister + cuPythonInit() + if __cuMemHostUnregister == NULL: + with gil: + raise RuntimeError('Function "cuMemHostUnregister" not found') + err = ( __cuMemHostUnregister)(p) + return err + +cdef CUresult _cuMemcpy(CUdeviceptr dst, CUdeviceptr src, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpy + cuPythonInit() + if __cuMemcpy == NULL: + with gil: + raise RuntimeError('Function "cuMemcpy" not found') + err = ( __cuMemcpy)(dst, src, ByteCount) + return err + +cdef CUresult _cuMemcpyPeer(CUdeviceptr dstDevice, CUcontext dstContext, CUdeviceptr srcDevice, CUcontext srcContext, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyPeer + cuPythonInit() + if __cuMemcpyPeer == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyPeer" not found') + err = ( __cuMemcpyPeer)(dstDevice, dstContext, srcDevice, srcContext, ByteCount) + return err + +cdef CUresult _cuMemcpyHtoD_v2(CUdeviceptr dstDevice, const void* srcHost, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyHtoD_v2 + cuPythonInit() + if __cuMemcpyHtoD_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyHtoD_v2" not found') + err = ( __cuMemcpyHtoD_v2)(dstDevice, srcHost, ByteCount) + return err + +cdef CUresult _cuMemcpyDtoH_v2(void* dstHost, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyDtoH_v2 + cuPythonInit() + if __cuMemcpyDtoH_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyDtoH_v2" not found') + err = ( __cuMemcpyDtoH_v2)(dstHost, srcDevice, ByteCount) + return err + +cdef CUresult _cuMemcpyDtoD_v2(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyDtoD_v2 + cuPythonInit() + if __cuMemcpyDtoD_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyDtoD_v2" not found') + err = ( __cuMemcpyDtoD_v2)(dstDevice, srcDevice, ByteCount) + return err + +cdef CUresult _cuMemcpyDtoA_v2(CUarray dstArray, size_t dstOffset, CUdeviceptr srcDevice, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyDtoA_v2 + cuPythonInit() + if __cuMemcpyDtoA_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyDtoA_v2" not found') + err = ( __cuMemcpyDtoA_v2)(dstArray, dstOffset, srcDevice, ByteCount) + return err + +cdef CUresult _cuMemcpyAtoD_v2(CUdeviceptr dstDevice, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyAtoD_v2 + cuPythonInit() + if __cuMemcpyAtoD_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyAtoD_v2" not found') + err = ( __cuMemcpyAtoD_v2)(dstDevice, srcArray, srcOffset, ByteCount) + return err + +cdef CUresult _cuMemcpyHtoA_v2(CUarray dstArray, size_t dstOffset, const void* srcHost, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyHtoA_v2 + cuPythonInit() + if __cuMemcpyHtoA_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyHtoA_v2" not found') + err = ( __cuMemcpyHtoA_v2)(dstArray, dstOffset, srcHost, ByteCount) + return err + +cdef CUresult _cuMemcpyAtoH_v2(void* dstHost, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyAtoH_v2 + cuPythonInit() + if __cuMemcpyAtoH_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyAtoH_v2" not found') + err = ( __cuMemcpyAtoH_v2)(dstHost, srcArray, srcOffset, ByteCount) + return err + +cdef CUresult _cuMemcpyAtoA_v2(CUarray dstArray, size_t dstOffset, CUarray srcArray, size_t srcOffset, size_t ByteCount) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyAtoA_v2 + cuPythonInit() + if __cuMemcpyAtoA_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyAtoA_v2" not found') + err = ( __cuMemcpyAtoA_v2)(dstArray, dstOffset, srcArray, srcOffset, ByteCount) + return err + +cdef CUresult _cuMemcpy2D_v2(const CUDA_MEMCPY2D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpy2D_v2 + cuPythonInit() + if __cuMemcpy2D_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpy2D_v2" not found') + err = ( __cuMemcpy2D_v2)(pCopy) + return err + +cdef CUresult _cuMemcpy2DUnaligned_v2(const CUDA_MEMCPY2D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpy2DUnaligned_v2 + cuPythonInit() + if __cuMemcpy2DUnaligned_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpy2DUnaligned_v2" not found') + err = ( __cuMemcpy2DUnaligned_v2)(pCopy) + return err + +cdef CUresult _cuMemcpy3D_v2(const CUDA_MEMCPY3D* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpy3D_v2 + cuPythonInit() + if __cuMemcpy3D_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpy3D_v2" not found') + err = ( __cuMemcpy3D_v2)(pCopy) + return err + +cdef CUresult _cuMemcpy3DPeer(const CUDA_MEMCPY3D_PEER* pCopy) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpy3DPeer + cuPythonInit() + if __cuMemcpy3DPeer == NULL: + with gil: + raise RuntimeError('Function "cuMemcpy3DPeer" not found') + err = ( __cuMemcpy3DPeer)(pCopy) + return err + +cdef CUresult _cuMemcpyAsync(CUdeviceptr dst, CUdeviceptr src, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyAsync + cuPythonInit() + if __cuMemcpyAsync == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyAsync" not found') + err = ( __cuMemcpyAsync)(dst, src, ByteCount, hStream) + return err + +cdef CUresult _cuMemcpyPeerAsync(CUdeviceptr dstDevice, CUcontext dstContext, CUdeviceptr srcDevice, CUcontext srcContext, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyPeerAsync + cuPythonInit() + if __cuMemcpyPeerAsync == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyPeerAsync" not found') + err = ( __cuMemcpyPeerAsync)(dstDevice, dstContext, srcDevice, srcContext, ByteCount, hStream) + return err + +cdef CUresult _cuMemcpyHtoDAsync_v2(CUdeviceptr dstDevice, const void* srcHost, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyHtoDAsync_v2 + cuPythonInit() + if __cuMemcpyHtoDAsync_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyHtoDAsync_v2" not found') + err = ( __cuMemcpyHtoDAsync_v2)(dstDevice, srcHost, ByteCount, hStream) + return err + +cdef CUresult _cuMemcpyDtoHAsync_v2(void* dstHost, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyDtoHAsync_v2 + cuPythonInit() + if __cuMemcpyDtoHAsync_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyDtoHAsync_v2" not found') + err = ( __cuMemcpyDtoHAsync_v2)(dstHost, srcDevice, ByteCount, hStream) + return err + +cdef CUresult _cuMemcpyDtoDAsync_v2(CUdeviceptr dstDevice, CUdeviceptr srcDevice, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyDtoDAsync_v2 + cuPythonInit() + if __cuMemcpyDtoDAsync_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyDtoDAsync_v2" not found') + err = ( __cuMemcpyDtoDAsync_v2)(dstDevice, srcDevice, ByteCount, hStream) + return err + +cdef CUresult _cuMemcpyHtoAAsync_v2(CUarray dstArray, size_t dstOffset, const void* srcHost, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyHtoAAsync_v2 + cuPythonInit() + if __cuMemcpyHtoAAsync_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyHtoAAsync_v2" not found') + err = ( __cuMemcpyHtoAAsync_v2)(dstArray, dstOffset, srcHost, ByteCount, hStream) + return err + +cdef CUresult _cuMemcpyAtoHAsync_v2(void* dstHost, CUarray srcArray, size_t srcOffset, size_t ByteCount, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyAtoHAsync_v2 + cuPythonInit() + if __cuMemcpyAtoHAsync_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyAtoHAsync_v2" not found') + err = ( __cuMemcpyAtoHAsync_v2)(dstHost, srcArray, srcOffset, ByteCount, hStream) + return err + +cdef CUresult _cuMemcpy2DAsync_v2(const CUDA_MEMCPY2D* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpy2DAsync_v2 + cuPythonInit() + if __cuMemcpy2DAsync_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpy2DAsync_v2" not found') + err = ( __cuMemcpy2DAsync_v2)(pCopy, hStream) + return err + +cdef CUresult _cuMemcpy3DAsync_v2(const CUDA_MEMCPY3D* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpy3DAsync_v2 + cuPythonInit() + if __cuMemcpy3DAsync_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemcpy3DAsync_v2" not found') + err = ( __cuMemcpy3DAsync_v2)(pCopy, hStream) + return err + +cdef CUresult _cuMemcpy3DPeerAsync(const CUDA_MEMCPY3D_PEER* pCopy, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpy3DPeerAsync + cuPythonInit() + if __cuMemcpy3DPeerAsync == NULL: + with gil: + raise RuntimeError('Function "cuMemcpy3DPeerAsync" not found') + err = ( __cuMemcpy3DPeerAsync)(pCopy, hStream) + return err + +cdef CUresult _cuMemcpyBatchAsync(CUdeviceptr* dsts, CUdeviceptr* srcs, size_t* sizes, size_t count, CUmemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpyBatchAsync + cuPythonInit() + if __cuMemcpyBatchAsync == NULL: + with gil: + raise RuntimeError('Function "cuMemcpyBatchAsync" not found') + err = ( __cuMemcpyBatchAsync)(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, hStream) + return err + +cdef CUresult _cuMemcpy3DBatchAsync(size_t numOps, CUDA_MEMCPY3D_BATCH_OP* opList, size_t* failIdx, unsigned long long flags, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemcpy3DBatchAsync + cuPythonInit() + if __cuMemcpy3DBatchAsync == NULL: + with gil: + raise RuntimeError('Function "cuMemcpy3DBatchAsync" not found') + err = ( __cuMemcpy3DBatchAsync)(numOps, opList, failIdx, flags, hStream) + return err + +cdef CUresult _cuMemsetD8_v2(CUdeviceptr dstDevice, unsigned char uc, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemsetD8_v2 + cuPythonInit() + if __cuMemsetD8_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemsetD8_v2" not found') + err = ( __cuMemsetD8_v2)(dstDevice, uc, N) + return err + +cdef CUresult _cuMemsetD16_v2(CUdeviceptr dstDevice, unsigned short us, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemsetD16_v2 + cuPythonInit() + if __cuMemsetD16_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemsetD16_v2" not found') + err = ( __cuMemsetD16_v2)(dstDevice, us, N) + return err + +cdef CUresult _cuMemsetD32_v2(CUdeviceptr dstDevice, unsigned int ui, size_t N) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemsetD32_v2 + cuPythonInit() + if __cuMemsetD32_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemsetD32_v2" not found') + err = ( __cuMemsetD32_v2)(dstDevice, ui, N) + return err + +cdef CUresult _cuMemsetD2D8_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned char uc, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemsetD2D8_v2 + cuPythonInit() + if __cuMemsetD2D8_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemsetD2D8_v2" not found') + err = ( __cuMemsetD2D8_v2)(dstDevice, dstPitch, uc, Width, Height) + return err + +cdef CUresult _cuMemsetD2D16_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned short us, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemsetD2D16_v2 + cuPythonInit() + if __cuMemsetD2D16_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemsetD2D16_v2" not found') + err = ( __cuMemsetD2D16_v2)(dstDevice, dstPitch, us, Width, Height) + return err + +cdef CUresult _cuMemsetD2D32_v2(CUdeviceptr dstDevice, size_t dstPitch, unsigned int ui, size_t Width, size_t Height) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemsetD2D32_v2 + cuPythonInit() + if __cuMemsetD2D32_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemsetD2D32_v2" not found') + err = ( __cuMemsetD2D32_v2)(dstDevice, dstPitch, ui, Width, Height) + return err + +cdef CUresult _cuMemsetD8Async(CUdeviceptr dstDevice, unsigned char uc, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemsetD8Async + cuPythonInit() + if __cuMemsetD8Async == NULL: + with gil: + raise RuntimeError('Function "cuMemsetD8Async" not found') + err = ( __cuMemsetD8Async)(dstDevice, uc, N, hStream) + return err + +cdef CUresult _cuMemsetD16Async(CUdeviceptr dstDevice, unsigned short us, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemsetD16Async + cuPythonInit() + if __cuMemsetD16Async == NULL: + with gil: + raise RuntimeError('Function "cuMemsetD16Async" not found') + err = ( __cuMemsetD16Async)(dstDevice, us, N, hStream) + return err + +cdef CUresult _cuMemsetD32Async(CUdeviceptr dstDevice, unsigned int ui, size_t N, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemsetD32Async + cuPythonInit() + if __cuMemsetD32Async == NULL: + with gil: + raise RuntimeError('Function "cuMemsetD32Async" not found') + err = ( __cuMemsetD32Async)(dstDevice, ui, N, hStream) + return err + +cdef CUresult _cuMemsetD2D8Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned char uc, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemsetD2D8Async + cuPythonInit() + if __cuMemsetD2D8Async == NULL: + with gil: + raise RuntimeError('Function "cuMemsetD2D8Async" not found') + err = ( __cuMemsetD2D8Async)(dstDevice, dstPitch, uc, Width, Height, hStream) + return err + +cdef CUresult _cuMemsetD2D16Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned short us, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemsetD2D16Async + cuPythonInit() + if __cuMemsetD2D16Async == NULL: + with gil: + raise RuntimeError('Function "cuMemsetD2D16Async" not found') + err = ( __cuMemsetD2D16Async)(dstDevice, dstPitch, us, Width, Height, hStream) + return err + +cdef CUresult _cuMemsetD2D32Async(CUdeviceptr dstDevice, size_t dstPitch, unsigned int ui, size_t Width, size_t Height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemsetD2D32Async + cuPythonInit() + if __cuMemsetD2D32Async == NULL: + with gil: + raise RuntimeError('Function "cuMemsetD2D32Async" not found') + err = ( __cuMemsetD2D32Async)(dstDevice, dstPitch, ui, Width, Height, hStream) + return err + +cdef CUresult _cuArrayCreate_v2(CUarray* pHandle, const CUDA_ARRAY_DESCRIPTOR* pAllocateArray) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuArrayCreate_v2 + cuPythonInit() + if __cuArrayCreate_v2 == NULL: + with gil: + raise RuntimeError('Function "cuArrayCreate_v2" not found') + err = ( __cuArrayCreate_v2)(pHandle, pAllocateArray) + return err + +cdef CUresult _cuArrayGetDescriptor_v2(CUDA_ARRAY_DESCRIPTOR* pArrayDescriptor, CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuArrayGetDescriptor_v2 + cuPythonInit() + if __cuArrayGetDescriptor_v2 == NULL: + with gil: + raise RuntimeError('Function "cuArrayGetDescriptor_v2" not found') + err = ( __cuArrayGetDescriptor_v2)(pArrayDescriptor, hArray) + return err + +cdef CUresult _cuArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES* sparseProperties, CUarray array) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuArrayGetSparseProperties + cuPythonInit() + if __cuArrayGetSparseProperties == NULL: + with gil: + raise RuntimeError('Function "cuArrayGetSparseProperties" not found') + err = ( __cuArrayGetSparseProperties)(sparseProperties, array) + return err + +cdef CUresult _cuMipmappedArrayGetSparseProperties(CUDA_ARRAY_SPARSE_PROPERTIES* sparseProperties, CUmipmappedArray mipmap) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMipmappedArrayGetSparseProperties + cuPythonInit() + if __cuMipmappedArrayGetSparseProperties == NULL: + with gil: + raise RuntimeError('Function "cuMipmappedArrayGetSparseProperties" not found') + err = ( __cuMipmappedArrayGetSparseProperties)(sparseProperties, mipmap) + return err + +cdef CUresult _cuArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS* memoryRequirements, CUarray array, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuArrayGetMemoryRequirements + cuPythonInit() + if __cuArrayGetMemoryRequirements == NULL: + with gil: + raise RuntimeError('Function "cuArrayGetMemoryRequirements" not found') + err = ( __cuArrayGetMemoryRequirements)(memoryRequirements, array, device) + return err + +cdef CUresult _cuMipmappedArrayGetMemoryRequirements(CUDA_ARRAY_MEMORY_REQUIREMENTS* memoryRequirements, CUmipmappedArray mipmap, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMipmappedArrayGetMemoryRequirements + cuPythonInit() + if __cuMipmappedArrayGetMemoryRequirements == NULL: + with gil: + raise RuntimeError('Function "cuMipmappedArrayGetMemoryRequirements" not found') + err = ( __cuMipmappedArrayGetMemoryRequirements)(memoryRequirements, mipmap, device) + return err + +cdef CUresult _cuArrayGetPlane(CUarray* pPlaneArray, CUarray hArray, unsigned int planeIdx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuArrayGetPlane + cuPythonInit() + if __cuArrayGetPlane == NULL: + with gil: + raise RuntimeError('Function "cuArrayGetPlane" not found') + err = ( __cuArrayGetPlane)(pPlaneArray, hArray, planeIdx) + return err + +cdef CUresult _cuArrayDestroy(CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuArrayDestroy + cuPythonInit() + if __cuArrayDestroy == NULL: + with gil: + raise RuntimeError('Function "cuArrayDestroy" not found') + err = ( __cuArrayDestroy)(hArray) + return err + +cdef CUresult _cuArray3DCreate_v2(CUarray* pHandle, const CUDA_ARRAY3D_DESCRIPTOR* pAllocateArray) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuArray3DCreate_v2 + cuPythonInit() + if __cuArray3DCreate_v2 == NULL: + with gil: + raise RuntimeError('Function "cuArray3DCreate_v2" not found') + err = ( __cuArray3DCreate_v2)(pHandle, pAllocateArray) + return err + +cdef CUresult _cuArray3DGetDescriptor_v2(CUDA_ARRAY3D_DESCRIPTOR* pArrayDescriptor, CUarray hArray) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuArray3DGetDescriptor_v2 + cuPythonInit() + if __cuArray3DGetDescriptor_v2 == NULL: + with gil: + raise RuntimeError('Function "cuArray3DGetDescriptor_v2" not found') + err = ( __cuArray3DGetDescriptor_v2)(pArrayDescriptor, hArray) + return err + +cdef CUresult _cuMipmappedArrayCreate(CUmipmappedArray* pHandle, const CUDA_ARRAY3D_DESCRIPTOR* pMipmappedArrayDesc, unsigned int numMipmapLevels) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMipmappedArrayCreate + cuPythonInit() + if __cuMipmappedArrayCreate == NULL: + with gil: + raise RuntimeError('Function "cuMipmappedArrayCreate" not found') + err = ( __cuMipmappedArrayCreate)(pHandle, pMipmappedArrayDesc, numMipmapLevels) + return err + +cdef CUresult _cuMipmappedArrayGetLevel(CUarray* pLevelArray, CUmipmappedArray hMipmappedArray, unsigned int level) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMipmappedArrayGetLevel + cuPythonInit() + if __cuMipmappedArrayGetLevel == NULL: + with gil: + raise RuntimeError('Function "cuMipmappedArrayGetLevel" not found') + err = ( __cuMipmappedArrayGetLevel)(pLevelArray, hMipmappedArray, level) + return err + +cdef CUresult _cuMipmappedArrayDestroy(CUmipmappedArray hMipmappedArray) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMipmappedArrayDestroy + cuPythonInit() + if __cuMipmappedArrayDestroy == NULL: + with gil: + raise RuntimeError('Function "cuMipmappedArrayDestroy" not found') + err = ( __cuMipmappedArrayDestroy)(hMipmappedArray) + return err + +cdef CUresult _cuMemGetHandleForAddressRange(void* handle, CUdeviceptr dptr, size_t size, CUmemRangeHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemGetHandleForAddressRange + cuPythonInit() + if __cuMemGetHandleForAddressRange == NULL: + with gil: + raise RuntimeError('Function "cuMemGetHandleForAddressRange" not found') + err = ( __cuMemGetHandleForAddressRange)(handle, dptr, size, handleType, flags) + return err + +cdef CUresult _cuMemBatchDecompressAsync(CUmemDecompressParams* paramsArray, size_t count, unsigned int flags, size_t* errorIndex, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemBatchDecompressAsync + cuPythonInit() + if __cuMemBatchDecompressAsync == NULL: + with gil: + raise RuntimeError('Function "cuMemBatchDecompressAsync" not found') + err = ( __cuMemBatchDecompressAsync)(paramsArray, count, flags, errorIndex, stream) + return err + +cdef CUresult _cuMemAddressReserve(CUdeviceptr* ptr, size_t size, size_t alignment, CUdeviceptr addr, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemAddressReserve + cuPythonInit() + if __cuMemAddressReserve == NULL: + with gil: + raise RuntimeError('Function "cuMemAddressReserve" not found') + err = ( __cuMemAddressReserve)(ptr, size, alignment, addr, flags) + return err + +cdef CUresult _cuMemAddressFree(CUdeviceptr ptr, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemAddressFree + cuPythonInit() + if __cuMemAddressFree == NULL: + with gil: + raise RuntimeError('Function "cuMemAddressFree" not found') + err = ( __cuMemAddressFree)(ptr, size) + return err + +cdef CUresult _cuMemCreate(CUmemGenericAllocationHandle* handle, size_t size, const CUmemAllocationProp* prop, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemCreate + cuPythonInit() + if __cuMemCreate == NULL: + with gil: + raise RuntimeError('Function "cuMemCreate" not found') + err = ( __cuMemCreate)(handle, size, prop, flags) + return err + +cdef CUresult _cuMemRelease(CUmemGenericAllocationHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemRelease + cuPythonInit() + if __cuMemRelease == NULL: + with gil: + raise RuntimeError('Function "cuMemRelease" not found') + err = ( __cuMemRelease)(handle) + return err + +cdef CUresult _cuMemMap(CUdeviceptr ptr, size_t size, size_t offset, CUmemGenericAllocationHandle handle, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemMap + cuPythonInit() + if __cuMemMap == NULL: + with gil: + raise RuntimeError('Function "cuMemMap" not found') + err = ( __cuMemMap)(ptr, size, offset, handle, flags) + return err + +cdef CUresult _cuMemMapArrayAsync(CUarrayMapInfo* mapInfoList, unsigned int count, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemMapArrayAsync + cuPythonInit() + if __cuMemMapArrayAsync == NULL: + with gil: + raise RuntimeError('Function "cuMemMapArrayAsync" not found') + err = ( __cuMemMapArrayAsync)(mapInfoList, count, hStream) + return err + +cdef CUresult _cuMemUnmap(CUdeviceptr ptr, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemUnmap + cuPythonInit() + if __cuMemUnmap == NULL: + with gil: + raise RuntimeError('Function "cuMemUnmap" not found') + err = ( __cuMemUnmap)(ptr, size) + return err + +cdef CUresult _cuMemSetAccess(CUdeviceptr ptr, size_t size, const CUmemAccessDesc* desc, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemSetAccess + cuPythonInit() + if __cuMemSetAccess == NULL: + with gil: + raise RuntimeError('Function "cuMemSetAccess" not found') + err = ( __cuMemSetAccess)(ptr, size, desc, count) + return err + +cdef CUresult _cuMemGetAccess(unsigned long long* flags, const CUmemLocation* location, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemGetAccess + cuPythonInit() + if __cuMemGetAccess == NULL: + with gil: + raise RuntimeError('Function "cuMemGetAccess" not found') + err = ( __cuMemGetAccess)(flags, location, ptr) + return err + +cdef CUresult _cuMemExportToShareableHandle(void* shareableHandle, CUmemGenericAllocationHandle handle, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemExportToShareableHandle + cuPythonInit() + if __cuMemExportToShareableHandle == NULL: + with gil: + raise RuntimeError('Function "cuMemExportToShareableHandle" not found') + err = ( __cuMemExportToShareableHandle)(shareableHandle, handle, handleType, flags) + return err + +cdef CUresult _cuMemImportFromShareableHandle(CUmemGenericAllocationHandle* handle, void* osHandle, CUmemAllocationHandleType shHandleType) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemImportFromShareableHandle + cuPythonInit() + if __cuMemImportFromShareableHandle == NULL: + with gil: + raise RuntimeError('Function "cuMemImportFromShareableHandle" not found') + err = ( __cuMemImportFromShareableHandle)(handle, osHandle, shHandleType) + return err + +cdef CUresult _cuMemGetAllocationGranularity(size_t* granularity, const CUmemAllocationProp* prop, CUmemAllocationGranularity_flags option) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemGetAllocationGranularity + cuPythonInit() + if __cuMemGetAllocationGranularity == NULL: + with gil: + raise RuntimeError('Function "cuMemGetAllocationGranularity" not found') + err = ( __cuMemGetAllocationGranularity)(granularity, prop, option) + return err + +cdef CUresult _cuMemGetAllocationPropertiesFromHandle(CUmemAllocationProp* prop, CUmemGenericAllocationHandle handle) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemGetAllocationPropertiesFromHandle + cuPythonInit() + if __cuMemGetAllocationPropertiesFromHandle == NULL: + with gil: + raise RuntimeError('Function "cuMemGetAllocationPropertiesFromHandle" not found') + err = ( __cuMemGetAllocationPropertiesFromHandle)(prop, handle) + return err + +cdef CUresult _cuMemRetainAllocationHandle(CUmemGenericAllocationHandle* handle, void* addr) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemRetainAllocationHandle + cuPythonInit() + if __cuMemRetainAllocationHandle == NULL: + with gil: + raise RuntimeError('Function "cuMemRetainAllocationHandle" not found') + err = ( __cuMemRetainAllocationHandle)(handle, addr) + return err + +cdef CUresult _cuMemFreeAsync(CUdeviceptr dptr, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemFreeAsync + cuPythonInit() + if __cuMemFreeAsync == NULL: + with gil: + raise RuntimeError('Function "cuMemFreeAsync" not found') + err = ( __cuMemFreeAsync)(dptr, hStream) + return err + +cdef CUresult _cuMemAllocAsync(CUdeviceptr* dptr, size_t bytesize, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemAllocAsync + cuPythonInit() + if __cuMemAllocAsync == NULL: + with gil: + raise RuntimeError('Function "cuMemAllocAsync" not found') + err = ( __cuMemAllocAsync)(dptr, bytesize, hStream) + return err + +cdef CUresult _cuMemPoolTrimTo(CUmemoryPool pool, size_t minBytesToKeep) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemPoolTrimTo + cuPythonInit() + if __cuMemPoolTrimTo == NULL: + with gil: + raise RuntimeError('Function "cuMemPoolTrimTo" not found') + err = ( __cuMemPoolTrimTo)(pool, minBytesToKeep) + return err + +cdef CUresult _cuMemPoolSetAttribute(CUmemoryPool pool, CUmemPool_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemPoolSetAttribute + cuPythonInit() + if __cuMemPoolSetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuMemPoolSetAttribute" not found') + err = ( __cuMemPoolSetAttribute)(pool, attr, value) + return err + +cdef CUresult _cuMemPoolGetAttribute(CUmemoryPool pool, CUmemPool_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemPoolGetAttribute + cuPythonInit() + if __cuMemPoolGetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuMemPoolGetAttribute" not found') + err = ( __cuMemPoolGetAttribute)(pool, attr, value) + return err + +cdef CUresult _cuMemPoolSetAccess(CUmemoryPool pool, const CUmemAccessDesc* map, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemPoolSetAccess + cuPythonInit() + if __cuMemPoolSetAccess == NULL: + with gil: + raise RuntimeError('Function "cuMemPoolSetAccess" not found') + err = ( __cuMemPoolSetAccess)(pool, map, count) + return err + +cdef CUresult _cuMemPoolGetAccess(CUmemAccess_flags* flags, CUmemoryPool memPool, CUmemLocation* location) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemPoolGetAccess + cuPythonInit() + if __cuMemPoolGetAccess == NULL: + with gil: + raise RuntimeError('Function "cuMemPoolGetAccess" not found') + err = ( __cuMemPoolGetAccess)(flags, memPool, location) + return err + +cdef CUresult _cuMemPoolCreate(CUmemoryPool* pool, const CUmemPoolProps* poolProps) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemPoolCreate + cuPythonInit() + if __cuMemPoolCreate == NULL: + with gil: + raise RuntimeError('Function "cuMemPoolCreate" not found') + err = ( __cuMemPoolCreate)(pool, poolProps) + return err + +cdef CUresult _cuMemPoolDestroy(CUmemoryPool pool) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemPoolDestroy + cuPythonInit() + if __cuMemPoolDestroy == NULL: + with gil: + raise RuntimeError('Function "cuMemPoolDestroy" not found') + err = ( __cuMemPoolDestroy)(pool) + return err + +cdef CUresult _cuMemAllocFromPoolAsync(CUdeviceptr* dptr, size_t bytesize, CUmemoryPool pool, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemAllocFromPoolAsync + cuPythonInit() + if __cuMemAllocFromPoolAsync == NULL: + with gil: + raise RuntimeError('Function "cuMemAllocFromPoolAsync" not found') + err = ( __cuMemAllocFromPoolAsync)(dptr, bytesize, pool, hStream) + return err + +cdef CUresult _cuMemPoolExportToShareableHandle(void* handle_out, CUmemoryPool pool, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemPoolExportToShareableHandle + cuPythonInit() + if __cuMemPoolExportToShareableHandle == NULL: + with gil: + raise RuntimeError('Function "cuMemPoolExportToShareableHandle" not found') + err = ( __cuMemPoolExportToShareableHandle)(handle_out, pool, handleType, flags) + return err + +cdef CUresult _cuMemPoolImportFromShareableHandle(CUmemoryPool* pool_out, void* handle, CUmemAllocationHandleType handleType, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemPoolImportFromShareableHandle + cuPythonInit() + if __cuMemPoolImportFromShareableHandle == NULL: + with gil: + raise RuntimeError('Function "cuMemPoolImportFromShareableHandle" not found') + err = ( __cuMemPoolImportFromShareableHandle)(pool_out, handle, handleType, flags) + return err + +cdef CUresult _cuMemPoolExportPointer(CUmemPoolPtrExportData* shareData_out, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemPoolExportPointer + cuPythonInit() + if __cuMemPoolExportPointer == NULL: + with gil: + raise RuntimeError('Function "cuMemPoolExportPointer" not found') + err = ( __cuMemPoolExportPointer)(shareData_out, ptr) + return err + +cdef CUresult _cuMemPoolImportPointer(CUdeviceptr* ptr_out, CUmemoryPool pool, CUmemPoolPtrExportData* shareData) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemPoolImportPointer + cuPythonInit() + if __cuMemPoolImportPointer == NULL: + with gil: + raise RuntimeError('Function "cuMemPoolImportPointer" not found') + err = ( __cuMemPoolImportPointer)(ptr_out, pool, shareData) + return err + +cdef CUresult _cuMulticastCreate(CUmemGenericAllocationHandle* mcHandle, const CUmulticastObjectProp* prop) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMulticastCreate + cuPythonInit() + if __cuMulticastCreate == NULL: + with gil: + raise RuntimeError('Function "cuMulticastCreate" not found') + err = ( __cuMulticastCreate)(mcHandle, prop) + return err + +cdef CUresult _cuMulticastAddDevice(CUmemGenericAllocationHandle mcHandle, CUdevice dev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMulticastAddDevice + cuPythonInit() + if __cuMulticastAddDevice == NULL: + with gil: + raise RuntimeError('Function "cuMulticastAddDevice" not found') + err = ( __cuMulticastAddDevice)(mcHandle, dev) + return err + +cdef CUresult _cuMulticastBindMem(CUmemGenericAllocationHandle mcHandle, size_t mcOffset, CUmemGenericAllocationHandle memHandle, size_t memOffset, size_t size, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMulticastBindMem + cuPythonInit() + if __cuMulticastBindMem == NULL: + with gil: + raise RuntimeError('Function "cuMulticastBindMem" not found') + err = ( __cuMulticastBindMem)(mcHandle, mcOffset, memHandle, memOffset, size, flags) + return err + +cdef CUresult _cuMulticastBindAddr(CUmemGenericAllocationHandle mcHandle, size_t mcOffset, CUdeviceptr memptr, size_t size, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMulticastBindAddr + cuPythonInit() + if __cuMulticastBindAddr == NULL: + with gil: + raise RuntimeError('Function "cuMulticastBindAddr" not found') + err = ( __cuMulticastBindAddr)(mcHandle, mcOffset, memptr, size, flags) + return err + +cdef CUresult _cuMulticastUnbind(CUmemGenericAllocationHandle mcHandle, CUdevice dev, size_t mcOffset, size_t size) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMulticastUnbind + cuPythonInit() + if __cuMulticastUnbind == NULL: + with gil: + raise RuntimeError('Function "cuMulticastUnbind" not found') + err = ( __cuMulticastUnbind)(mcHandle, dev, mcOffset, size) + return err + +cdef CUresult _cuMulticastGetGranularity(size_t* granularity, const CUmulticastObjectProp* prop, CUmulticastGranularity_flags option) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMulticastGetGranularity + cuPythonInit() + if __cuMulticastGetGranularity == NULL: + with gil: + raise RuntimeError('Function "cuMulticastGetGranularity" not found') + err = ( __cuMulticastGetGranularity)(granularity, prop, option) + return err + +cdef CUresult _cuPointerGetAttribute(void* data, CUpointer_attribute attribute, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuPointerGetAttribute + cuPythonInit() + if __cuPointerGetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuPointerGetAttribute" not found') + err = ( __cuPointerGetAttribute)(data, attribute, ptr) + return err + +cdef CUresult _cuMemPrefetchAsync(CUdeviceptr devPtr, size_t count, CUdevice dstDevice, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemPrefetchAsync + cuPythonInit() + if __cuMemPrefetchAsync == NULL: + with gil: + raise RuntimeError('Function "cuMemPrefetchAsync" not found') + err = ( __cuMemPrefetchAsync)(devPtr, count, dstDevice, hStream) + return err + +cdef CUresult _cuMemPrefetchAsync_v2(CUdeviceptr devPtr, size_t count, CUmemLocation location, unsigned int flags, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemPrefetchAsync_v2 + cuPythonInit() + if __cuMemPrefetchAsync_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemPrefetchAsync_v2" not found') + err = ( __cuMemPrefetchAsync_v2)(devPtr, count, location, flags, hStream) + return err + +cdef CUresult _cuMemAdvise(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemAdvise + cuPythonInit() + if __cuMemAdvise == NULL: + with gil: + raise RuntimeError('Function "cuMemAdvise" not found') + err = ( __cuMemAdvise)(devPtr, count, advice, device) + return err + +cdef CUresult _cuMemAdvise_v2(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUmemLocation location) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemAdvise_v2 + cuPythonInit() + if __cuMemAdvise_v2 == NULL: + with gil: + raise RuntimeError('Function "cuMemAdvise_v2" not found') + err = ( __cuMemAdvise_v2)(devPtr, count, advice, location) + return err + +cdef CUresult _cuMemRangeGetAttribute(void* data, size_t dataSize, CUmem_range_attribute attribute, CUdeviceptr devPtr, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemRangeGetAttribute + cuPythonInit() + if __cuMemRangeGetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuMemRangeGetAttribute" not found') + err = ( __cuMemRangeGetAttribute)(data, dataSize, attribute, devPtr, count) + return err + +cdef CUresult _cuMemRangeGetAttributes(void** data, size_t* dataSizes, CUmem_range_attribute* attributes, size_t numAttributes, CUdeviceptr devPtr, size_t count) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuMemRangeGetAttributes + cuPythonInit() + if __cuMemRangeGetAttributes == NULL: + with gil: + raise RuntimeError('Function "cuMemRangeGetAttributes" not found') + err = ( __cuMemRangeGetAttributes)(data, dataSizes, attributes, numAttributes, devPtr, count) + return err + +cdef CUresult _cuPointerSetAttribute(const void* value, CUpointer_attribute attribute, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuPointerSetAttribute + cuPythonInit() + if __cuPointerSetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuPointerSetAttribute" not found') + err = ( __cuPointerSetAttribute)(value, attribute, ptr) + return err + +cdef CUresult _cuPointerGetAttributes(unsigned int numAttributes, CUpointer_attribute* attributes, void** data, CUdeviceptr ptr) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuPointerGetAttributes + cuPythonInit() + if __cuPointerGetAttributes == NULL: + with gil: + raise RuntimeError('Function "cuPointerGetAttributes" not found') + err = ( __cuPointerGetAttributes)(numAttributes, attributes, data, ptr) + return err + +cdef CUresult _cuStreamCreate(CUstream* phStream, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamCreate + cuPythonInit() + if __cuStreamCreate == NULL: + with gil: + raise RuntimeError('Function "cuStreamCreate" not found') + err = ( __cuStreamCreate)(phStream, Flags) + return err + +cdef CUresult _cuStreamCreateWithPriority(CUstream* phStream, unsigned int flags, int priority) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamCreateWithPriority + cuPythonInit() + if __cuStreamCreateWithPriority == NULL: + with gil: + raise RuntimeError('Function "cuStreamCreateWithPriority" not found') + err = ( __cuStreamCreateWithPriority)(phStream, flags, priority) + return err + +cdef CUresult _cuStreamGetPriority(CUstream hStream, int* priority) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamGetPriority + cuPythonInit() + if __cuStreamGetPriority == NULL: + with gil: + raise RuntimeError('Function "cuStreamGetPriority" not found') + err = ( __cuStreamGetPriority)(hStream, priority) + return err + +cdef CUresult _cuStreamGetDevice(CUstream hStream, CUdevice* device) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamGetDevice + cuPythonInit() + if __cuStreamGetDevice == NULL: + with gil: + raise RuntimeError('Function "cuStreamGetDevice" not found') + err = ( __cuStreamGetDevice)(hStream, device) + return err + +cdef CUresult _cuStreamGetFlags(CUstream hStream, unsigned int* flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamGetFlags + cuPythonInit() + if __cuStreamGetFlags == NULL: + with gil: + raise RuntimeError('Function "cuStreamGetFlags" not found') + err = ( __cuStreamGetFlags)(hStream, flags) + return err + +cdef CUresult _cuStreamGetId(CUstream hStream, unsigned long long* streamId) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamGetId + cuPythonInit() + if __cuStreamGetId == NULL: + with gil: + raise RuntimeError('Function "cuStreamGetId" not found') + err = ( __cuStreamGetId)(hStream, streamId) + return err + +cdef CUresult _cuStreamGetCtx(CUstream hStream, CUcontext* pctx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamGetCtx + cuPythonInit() + if __cuStreamGetCtx == NULL: + with gil: + raise RuntimeError('Function "cuStreamGetCtx" not found') + err = ( __cuStreamGetCtx)(hStream, pctx) + return err + +cdef CUresult _cuStreamGetCtx_v2(CUstream hStream, CUcontext* pCtx, CUgreenCtx* pGreenCtx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamGetCtx_v2 + cuPythonInit() + if __cuStreamGetCtx_v2 == NULL: + with gil: + raise RuntimeError('Function "cuStreamGetCtx_v2" not found') + err = ( __cuStreamGetCtx_v2)(hStream, pCtx, pGreenCtx) + return err + +cdef CUresult _cuStreamWaitEvent(CUstream hStream, CUevent hEvent, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamWaitEvent + cuPythonInit() + if __cuStreamWaitEvent == NULL: + with gil: + raise RuntimeError('Function "cuStreamWaitEvent" not found') + err = ( __cuStreamWaitEvent)(hStream, hEvent, Flags) + return err + +cdef CUresult _cuStreamAddCallback(CUstream hStream, CUstreamCallback callback, void* userData, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamAddCallback + cuPythonInit() + if __cuStreamAddCallback == NULL: + with gil: + raise RuntimeError('Function "cuStreamAddCallback" not found') + err = ( __cuStreamAddCallback)(hStream, callback, userData, flags) + return err + +cdef CUresult _cuStreamBeginCapture_v2(CUstream hStream, CUstreamCaptureMode mode) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamBeginCapture_v2 + cuPythonInit() + if __cuStreamBeginCapture_v2 == NULL: + with gil: + raise RuntimeError('Function "cuStreamBeginCapture_v2" not found') + err = ( __cuStreamBeginCapture_v2)(hStream, mode) + return err + +cdef CUresult _cuStreamBeginCaptureToGraph(CUstream hStream, CUgraph hGraph, const CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, CUstreamCaptureMode mode) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamBeginCaptureToGraph + cuPythonInit() + if __cuStreamBeginCaptureToGraph == NULL: + with gil: + raise RuntimeError('Function "cuStreamBeginCaptureToGraph" not found') + err = ( __cuStreamBeginCaptureToGraph)(hStream, hGraph, dependencies, dependencyData, numDependencies, mode) + return err + +cdef CUresult _cuThreadExchangeStreamCaptureMode(CUstreamCaptureMode* mode) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuThreadExchangeStreamCaptureMode + cuPythonInit() + if __cuThreadExchangeStreamCaptureMode == NULL: + with gil: + raise RuntimeError('Function "cuThreadExchangeStreamCaptureMode" not found') + err = ( __cuThreadExchangeStreamCaptureMode)(mode) + return err + +cdef CUresult _cuStreamEndCapture(CUstream hStream, CUgraph* phGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamEndCapture + cuPythonInit() + if __cuStreamEndCapture == NULL: + with gil: + raise RuntimeError('Function "cuStreamEndCapture" not found') + err = ( __cuStreamEndCapture)(hStream, phGraph) + return err + +cdef CUresult _cuStreamIsCapturing(CUstream hStream, CUstreamCaptureStatus* captureStatus) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamIsCapturing + cuPythonInit() + if __cuStreamIsCapturing == NULL: + with gil: + raise RuntimeError('Function "cuStreamIsCapturing" not found') + err = ( __cuStreamIsCapturing)(hStream, captureStatus) + return err + +cdef CUresult _cuStreamGetCaptureInfo_v2(CUstream hStream, CUstreamCaptureStatus* captureStatus_out, cuuint64_t* id_out, CUgraph* graph_out, const CUgraphNode** dependencies_out, size_t* numDependencies_out) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamGetCaptureInfo_v2 + cuPythonInit() + if __cuStreamGetCaptureInfo_v2 == NULL: + with gil: + raise RuntimeError('Function "cuStreamGetCaptureInfo_v2" not found') + err = ( __cuStreamGetCaptureInfo_v2)(hStream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + return err + +cdef CUresult _cuStreamGetCaptureInfo_v3(CUstream hStream, CUstreamCaptureStatus* captureStatus_out, cuuint64_t* id_out, CUgraph* graph_out, const CUgraphNode** dependencies_out, const CUgraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamGetCaptureInfo_v3 + cuPythonInit() + if __cuStreamGetCaptureInfo_v3 == NULL: + with gil: + raise RuntimeError('Function "cuStreamGetCaptureInfo_v3" not found') + err = ( __cuStreamGetCaptureInfo_v3)(hStream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + return err + +cdef CUresult _cuStreamUpdateCaptureDependencies(CUstream hStream, CUgraphNode* dependencies, size_t numDependencies, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamUpdateCaptureDependencies + cuPythonInit() + if __cuStreamUpdateCaptureDependencies == NULL: + with gil: + raise RuntimeError('Function "cuStreamUpdateCaptureDependencies" not found') + err = ( __cuStreamUpdateCaptureDependencies)(hStream, dependencies, numDependencies, flags) + return err + +cdef CUresult _cuStreamUpdateCaptureDependencies_v2(CUstream hStream, CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamUpdateCaptureDependencies_v2 + cuPythonInit() + if __cuStreamUpdateCaptureDependencies_v2 == NULL: + with gil: + raise RuntimeError('Function "cuStreamUpdateCaptureDependencies_v2" not found') + err = ( __cuStreamUpdateCaptureDependencies_v2)(hStream, dependencies, dependencyData, numDependencies, flags) + return err + +cdef CUresult _cuStreamAttachMemAsync(CUstream hStream, CUdeviceptr dptr, size_t length, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamAttachMemAsync + cuPythonInit() + if __cuStreamAttachMemAsync == NULL: + with gil: + raise RuntimeError('Function "cuStreamAttachMemAsync" not found') + err = ( __cuStreamAttachMemAsync)(hStream, dptr, length, flags) + return err + +cdef CUresult _cuStreamQuery(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamQuery + cuPythonInit() + if __cuStreamQuery == NULL: + with gil: + raise RuntimeError('Function "cuStreamQuery" not found') + err = ( __cuStreamQuery)(hStream) + return err + +cdef CUresult _cuStreamSynchronize(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamSynchronize + cuPythonInit() + if __cuStreamSynchronize == NULL: + with gil: + raise RuntimeError('Function "cuStreamSynchronize" not found') + err = ( __cuStreamSynchronize)(hStream) + return err + +cdef CUresult _cuStreamDestroy_v2(CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamDestroy_v2 + cuPythonInit() + if __cuStreamDestroy_v2 == NULL: + with gil: + raise RuntimeError('Function "cuStreamDestroy_v2" not found') + err = ( __cuStreamDestroy_v2)(hStream) + return err + +cdef CUresult _cuStreamCopyAttributes(CUstream dst, CUstream src) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamCopyAttributes + cuPythonInit() + if __cuStreamCopyAttributes == NULL: + with gil: + raise RuntimeError('Function "cuStreamCopyAttributes" not found') + err = ( __cuStreamCopyAttributes)(dst, src) + return err + +cdef CUresult _cuStreamGetAttribute(CUstream hStream, CUstreamAttrID attr, CUstreamAttrValue* value_out) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamGetAttribute + cuPythonInit() + if __cuStreamGetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuStreamGetAttribute" not found') + err = ( __cuStreamGetAttribute)(hStream, attr, value_out) + return err + +cdef CUresult _cuStreamSetAttribute(CUstream hStream, CUstreamAttrID attr, const CUstreamAttrValue* value) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamSetAttribute + cuPythonInit() + if __cuStreamSetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuStreamSetAttribute" not found') + err = ( __cuStreamSetAttribute)(hStream, attr, value) + return err + +cdef CUresult _cuEventCreate(CUevent* phEvent, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEventCreate + cuPythonInit() + if __cuEventCreate == NULL: + with gil: + raise RuntimeError('Function "cuEventCreate" not found') + err = ( __cuEventCreate)(phEvent, Flags) + return err + +cdef CUresult _cuEventRecord(CUevent hEvent, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEventRecord + cuPythonInit() + if __cuEventRecord == NULL: + with gil: + raise RuntimeError('Function "cuEventRecord" not found') + err = ( __cuEventRecord)(hEvent, hStream) + return err + +cdef CUresult _cuEventRecordWithFlags(CUevent hEvent, CUstream hStream, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEventRecordWithFlags + cuPythonInit() + if __cuEventRecordWithFlags == NULL: + with gil: + raise RuntimeError('Function "cuEventRecordWithFlags" not found') + err = ( __cuEventRecordWithFlags)(hEvent, hStream, flags) + return err + +cdef CUresult _cuEventQuery(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEventQuery + cuPythonInit() + if __cuEventQuery == NULL: + with gil: + raise RuntimeError('Function "cuEventQuery" not found') + err = ( __cuEventQuery)(hEvent) + return err + +cdef CUresult _cuEventSynchronize(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEventSynchronize + cuPythonInit() + if __cuEventSynchronize == NULL: + with gil: + raise RuntimeError('Function "cuEventSynchronize" not found') + err = ( __cuEventSynchronize)(hEvent) + return err + +cdef CUresult _cuEventDestroy_v2(CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEventDestroy_v2 + cuPythonInit() + if __cuEventDestroy_v2 == NULL: + with gil: + raise RuntimeError('Function "cuEventDestroy_v2" not found') + err = ( __cuEventDestroy_v2)(hEvent) + return err + +cdef CUresult _cuEventElapsedTime(float* pMilliseconds, CUevent hStart, CUevent hEnd) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEventElapsedTime + cuPythonInit() + if __cuEventElapsedTime == NULL: + with gil: + raise RuntimeError('Function "cuEventElapsedTime" not found') + err = ( __cuEventElapsedTime)(pMilliseconds, hStart, hEnd) + return err + +cdef CUresult _cuEventElapsedTime_v2(float* pMilliseconds, CUevent hStart, CUevent hEnd) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEventElapsedTime_v2 + cuPythonInit() + if __cuEventElapsedTime_v2 == NULL: + with gil: + raise RuntimeError('Function "cuEventElapsedTime_v2" not found') + err = ( __cuEventElapsedTime_v2)(pMilliseconds, hStart, hEnd) + return err + +cdef CUresult _cuImportExternalMemory(CUexternalMemory* extMem_out, const CUDA_EXTERNAL_MEMORY_HANDLE_DESC* memHandleDesc) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuImportExternalMemory + cuPythonInit() + if __cuImportExternalMemory == NULL: + with gil: + raise RuntimeError('Function "cuImportExternalMemory" not found') + err = ( __cuImportExternalMemory)(extMem_out, memHandleDesc) + return err + +cdef CUresult _cuExternalMemoryGetMappedBuffer(CUdeviceptr* devPtr, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_BUFFER_DESC* bufferDesc) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuExternalMemoryGetMappedBuffer + cuPythonInit() + if __cuExternalMemoryGetMappedBuffer == NULL: + with gil: + raise RuntimeError('Function "cuExternalMemoryGetMappedBuffer" not found') + err = ( __cuExternalMemoryGetMappedBuffer)(devPtr, extMem, bufferDesc) + return err + +cdef CUresult _cuExternalMemoryGetMappedMipmappedArray(CUmipmappedArray* mipmap, CUexternalMemory extMem, const CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC* mipmapDesc) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuExternalMemoryGetMappedMipmappedArray + cuPythonInit() + if __cuExternalMemoryGetMappedMipmappedArray == NULL: + with gil: + raise RuntimeError('Function "cuExternalMemoryGetMappedMipmappedArray" not found') + err = ( __cuExternalMemoryGetMappedMipmappedArray)(mipmap, extMem, mipmapDesc) + return err + +cdef CUresult _cuDestroyExternalMemory(CUexternalMemory extMem) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDestroyExternalMemory + cuPythonInit() + if __cuDestroyExternalMemory == NULL: + with gil: + raise RuntimeError('Function "cuDestroyExternalMemory" not found') + err = ( __cuDestroyExternalMemory)(extMem) + return err + +cdef CUresult _cuImportExternalSemaphore(CUexternalSemaphore* extSem_out, const CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC* semHandleDesc) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuImportExternalSemaphore + cuPythonInit() + if __cuImportExternalSemaphore == NULL: + with gil: + raise RuntimeError('Function "cuImportExternalSemaphore" not found') + err = ( __cuImportExternalSemaphore)(extSem_out, semHandleDesc) + return err + +cdef CUresult _cuSignalExternalSemaphoresAsync(const CUexternalSemaphore* extSemArray, const CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS* paramsArray, unsigned int numExtSems, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuSignalExternalSemaphoresAsync + cuPythonInit() + if __cuSignalExternalSemaphoresAsync == NULL: + with gil: + raise RuntimeError('Function "cuSignalExternalSemaphoresAsync" not found') + err = ( __cuSignalExternalSemaphoresAsync)(extSemArray, paramsArray, numExtSems, stream) + return err + +cdef CUresult _cuWaitExternalSemaphoresAsync(const CUexternalSemaphore* extSemArray, const CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS* paramsArray, unsigned int numExtSems, CUstream stream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuWaitExternalSemaphoresAsync + cuPythonInit() + if __cuWaitExternalSemaphoresAsync == NULL: + with gil: + raise RuntimeError('Function "cuWaitExternalSemaphoresAsync" not found') + err = ( __cuWaitExternalSemaphoresAsync)(extSemArray, paramsArray, numExtSems, stream) + return err + +cdef CUresult _cuDestroyExternalSemaphore(CUexternalSemaphore extSem) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDestroyExternalSemaphore + cuPythonInit() + if __cuDestroyExternalSemaphore == NULL: + with gil: + raise RuntimeError('Function "cuDestroyExternalSemaphore" not found') + err = ( __cuDestroyExternalSemaphore)(extSem) + return err + +cdef CUresult _cuStreamWaitValue32_v2(CUstream stream, CUdeviceptr addr, cuuint32_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamWaitValue32_v2 + cuPythonInit() + if __cuStreamWaitValue32_v2 == NULL: + with gil: + raise RuntimeError('Function "cuStreamWaitValue32_v2" not found') + err = ( __cuStreamWaitValue32_v2)(stream, addr, value, flags) + return err + +cdef CUresult _cuStreamWaitValue64_v2(CUstream stream, CUdeviceptr addr, cuuint64_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamWaitValue64_v2 + cuPythonInit() + if __cuStreamWaitValue64_v2 == NULL: + with gil: + raise RuntimeError('Function "cuStreamWaitValue64_v2" not found') + err = ( __cuStreamWaitValue64_v2)(stream, addr, value, flags) + return err + +cdef CUresult _cuStreamWriteValue32_v2(CUstream stream, CUdeviceptr addr, cuuint32_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamWriteValue32_v2 + cuPythonInit() + if __cuStreamWriteValue32_v2 == NULL: + with gil: + raise RuntimeError('Function "cuStreamWriteValue32_v2" not found') + err = ( __cuStreamWriteValue32_v2)(stream, addr, value, flags) + return err + +cdef CUresult _cuStreamWriteValue64_v2(CUstream stream, CUdeviceptr addr, cuuint64_t value, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamWriteValue64_v2 + cuPythonInit() + if __cuStreamWriteValue64_v2 == NULL: + with gil: + raise RuntimeError('Function "cuStreamWriteValue64_v2" not found') + err = ( __cuStreamWriteValue64_v2)(stream, addr, value, flags) + return err + +cdef CUresult _cuStreamBatchMemOp_v2(CUstream stream, unsigned int count, CUstreamBatchMemOpParams* paramArray, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamBatchMemOp_v2 + cuPythonInit() + if __cuStreamBatchMemOp_v2 == NULL: + with gil: + raise RuntimeError('Function "cuStreamBatchMemOp_v2" not found') + err = ( __cuStreamBatchMemOp_v2)(stream, count, paramArray, flags) + return err + +cdef CUresult _cuFuncGetAttribute(int* pi, CUfunction_attribute attrib, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuFuncGetAttribute + cuPythonInit() + if __cuFuncGetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuFuncGetAttribute" not found') + err = ( __cuFuncGetAttribute)(pi, attrib, hfunc) + return err + +cdef CUresult _cuFuncSetAttribute(CUfunction hfunc, CUfunction_attribute attrib, int value) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuFuncSetAttribute + cuPythonInit() + if __cuFuncSetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuFuncSetAttribute" not found') + err = ( __cuFuncSetAttribute)(hfunc, attrib, value) + return err + +cdef CUresult _cuFuncSetCacheConfig(CUfunction hfunc, CUfunc_cache config) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuFuncSetCacheConfig + cuPythonInit() + if __cuFuncSetCacheConfig == NULL: + with gil: + raise RuntimeError('Function "cuFuncSetCacheConfig" not found') + err = ( __cuFuncSetCacheConfig)(hfunc, config) + return err + +cdef CUresult _cuFuncGetModule(CUmodule* hmod, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuFuncGetModule + cuPythonInit() + if __cuFuncGetModule == NULL: + with gil: + raise RuntimeError('Function "cuFuncGetModule" not found') + err = ( __cuFuncGetModule)(hmod, hfunc) + return err + +cdef CUresult _cuFuncGetName(const char** name, CUfunction hfunc) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuFuncGetName + cuPythonInit() + if __cuFuncGetName == NULL: + with gil: + raise RuntimeError('Function "cuFuncGetName" not found') + err = ( __cuFuncGetName)(name, hfunc) + return err + +cdef CUresult _cuFuncGetParamInfo(CUfunction func, size_t paramIndex, size_t* paramOffset, size_t* paramSize) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuFuncGetParamInfo + cuPythonInit() + if __cuFuncGetParamInfo == NULL: + with gil: + raise RuntimeError('Function "cuFuncGetParamInfo" not found') + err = ( __cuFuncGetParamInfo)(func, paramIndex, paramOffset, paramSize) + return err + +cdef CUresult _cuFuncIsLoaded(CUfunctionLoadingState* state, CUfunction function) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuFuncIsLoaded + cuPythonInit() + if __cuFuncIsLoaded == NULL: + with gil: + raise RuntimeError('Function "cuFuncIsLoaded" not found') + err = ( __cuFuncIsLoaded)(state, function) + return err + +cdef CUresult _cuFuncLoad(CUfunction function) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuFuncLoad + cuPythonInit() + if __cuFuncLoad == NULL: + with gil: + raise RuntimeError('Function "cuFuncLoad" not found') + err = ( __cuFuncLoad)(function) + return err + +cdef CUresult _cuLaunchKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void** kernelParams, void** extra) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLaunchKernel + cuPythonInit() + if __cuLaunchKernel == NULL: + with gil: + raise RuntimeError('Function "cuLaunchKernel" not found') + err = ( __cuLaunchKernel)(f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams, extra) + return err + +cdef CUresult _cuLaunchKernelEx(const CUlaunchConfig* config, CUfunction f, void** kernelParams, void** extra) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLaunchKernelEx + cuPythonInit() + if __cuLaunchKernelEx == NULL: + with gil: + raise RuntimeError('Function "cuLaunchKernelEx" not found') + err = ( __cuLaunchKernelEx)(config, f, kernelParams, extra) + return err + +cdef CUresult _cuLaunchCooperativeKernel(CUfunction f, unsigned int gridDimX, unsigned int gridDimY, unsigned int gridDimZ, unsigned int blockDimX, unsigned int blockDimY, unsigned int blockDimZ, unsigned int sharedMemBytes, CUstream hStream, void** kernelParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLaunchCooperativeKernel + cuPythonInit() + if __cuLaunchCooperativeKernel == NULL: + with gil: + raise RuntimeError('Function "cuLaunchCooperativeKernel" not found') + err = ( __cuLaunchCooperativeKernel)(f, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, sharedMemBytes, hStream, kernelParams) + return err + +cdef CUresult _cuLaunchCooperativeKernelMultiDevice(CUDA_LAUNCH_PARAMS* launchParamsList, unsigned int numDevices, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLaunchCooperativeKernelMultiDevice + cuPythonInit() + if __cuLaunchCooperativeKernelMultiDevice == NULL: + with gil: + raise RuntimeError('Function "cuLaunchCooperativeKernelMultiDevice" not found') + err = ( __cuLaunchCooperativeKernelMultiDevice)(launchParamsList, numDevices, flags) + return err + +cdef CUresult _cuLaunchHostFunc(CUstream hStream, CUhostFn fn, void* userData) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLaunchHostFunc + cuPythonInit() + if __cuLaunchHostFunc == NULL: + with gil: + raise RuntimeError('Function "cuLaunchHostFunc" not found') + err = ( __cuLaunchHostFunc)(hStream, fn, userData) + return err + +cdef CUresult _cuFuncSetBlockShape(CUfunction hfunc, int x, int y, int z) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuFuncSetBlockShape + cuPythonInit() + if __cuFuncSetBlockShape == NULL: + with gil: + raise RuntimeError('Function "cuFuncSetBlockShape" not found') + err = ( __cuFuncSetBlockShape)(hfunc, x, y, z) + return err + +cdef CUresult _cuFuncSetSharedSize(CUfunction hfunc, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuFuncSetSharedSize + cuPythonInit() + if __cuFuncSetSharedSize == NULL: + with gil: + raise RuntimeError('Function "cuFuncSetSharedSize" not found') + err = ( __cuFuncSetSharedSize)(hfunc, numbytes) + return err + +cdef CUresult _cuParamSetSize(CUfunction hfunc, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuParamSetSize + cuPythonInit() + if __cuParamSetSize == NULL: + with gil: + raise RuntimeError('Function "cuParamSetSize" not found') + err = ( __cuParamSetSize)(hfunc, numbytes) + return err + +cdef CUresult _cuParamSeti(CUfunction hfunc, int offset, unsigned int value) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuParamSeti + cuPythonInit() + if __cuParamSeti == NULL: + with gil: + raise RuntimeError('Function "cuParamSeti" not found') + err = ( __cuParamSeti)(hfunc, offset, value) + return err + +cdef CUresult _cuParamSetf(CUfunction hfunc, int offset, float value) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuParamSetf + cuPythonInit() + if __cuParamSetf == NULL: + with gil: + raise RuntimeError('Function "cuParamSetf" not found') + err = ( __cuParamSetf)(hfunc, offset, value) + return err + +cdef CUresult _cuParamSetv(CUfunction hfunc, int offset, void* ptr, unsigned int numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuParamSetv + cuPythonInit() + if __cuParamSetv == NULL: + with gil: + raise RuntimeError('Function "cuParamSetv" not found') + err = ( __cuParamSetv)(hfunc, offset, ptr, numbytes) + return err + +cdef CUresult _cuLaunch(CUfunction f) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLaunch + cuPythonInit() + if __cuLaunch == NULL: + with gil: + raise RuntimeError('Function "cuLaunch" not found') + err = ( __cuLaunch)(f) + return err + +cdef CUresult _cuLaunchGrid(CUfunction f, int grid_width, int grid_height) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLaunchGrid + cuPythonInit() + if __cuLaunchGrid == NULL: + with gil: + raise RuntimeError('Function "cuLaunchGrid" not found') + err = ( __cuLaunchGrid)(f, grid_width, grid_height) + return err + +cdef CUresult _cuLaunchGridAsync(CUfunction f, int grid_width, int grid_height, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLaunchGridAsync + cuPythonInit() + if __cuLaunchGridAsync == NULL: + with gil: + raise RuntimeError('Function "cuLaunchGridAsync" not found') + err = ( __cuLaunchGridAsync)(f, grid_width, grid_height, hStream) + return err + +cdef CUresult _cuParamSetTexRef(CUfunction hfunc, int texunit, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuParamSetTexRef + cuPythonInit() + if __cuParamSetTexRef == NULL: + with gil: + raise RuntimeError('Function "cuParamSetTexRef" not found') + err = ( __cuParamSetTexRef)(hfunc, texunit, hTexRef) + return err + +cdef CUresult _cuFuncSetSharedMemConfig(CUfunction hfunc, CUsharedconfig config) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuFuncSetSharedMemConfig + cuPythonInit() + if __cuFuncSetSharedMemConfig == NULL: + with gil: + raise RuntimeError('Function "cuFuncSetSharedMemConfig" not found') + err = ( __cuFuncSetSharedMemConfig)(hfunc, config) + return err + +cdef CUresult _cuGraphCreate(CUgraph* phGraph, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphCreate + cuPythonInit() + if __cuGraphCreate == NULL: + with gil: + raise RuntimeError('Function "cuGraphCreate" not found') + err = ( __cuGraphCreate)(phGraph, flags) + return err + +cdef CUresult _cuGraphAddKernelNode_v2(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddKernelNode_v2 + cuPythonInit() + if __cuGraphAddKernelNode_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddKernelNode_v2" not found') + err = ( __cuGraphAddKernelNode_v2)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + return err + +cdef CUresult _cuGraphKernelNodeGetParams_v2(CUgraphNode hNode, CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphKernelNodeGetParams_v2 + cuPythonInit() + if __cuGraphKernelNodeGetParams_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGraphKernelNodeGetParams_v2" not found') + err = ( __cuGraphKernelNodeGetParams_v2)(hNode, nodeParams) + return err + +cdef CUresult _cuGraphKernelNodeSetParams_v2(CUgraphNode hNode, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphKernelNodeSetParams_v2 + cuPythonInit() + if __cuGraphKernelNodeSetParams_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGraphKernelNodeSetParams_v2" not found') + err = ( __cuGraphKernelNodeSetParams_v2)(hNode, nodeParams) + return err + +cdef CUresult _cuGraphAddMemcpyNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_MEMCPY3D* copyParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddMemcpyNode + cuPythonInit() + if __cuGraphAddMemcpyNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddMemcpyNode" not found') + err = ( __cuGraphAddMemcpyNode)(phGraphNode, hGraph, dependencies, numDependencies, copyParams, ctx) + return err + +cdef CUresult _cuGraphMemcpyNodeGetParams(CUgraphNode hNode, CUDA_MEMCPY3D* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphMemcpyNodeGetParams + cuPythonInit() + if __cuGraphMemcpyNodeGetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphMemcpyNodeGetParams" not found') + err = ( __cuGraphMemcpyNodeGetParams)(hNode, nodeParams) + return err + +cdef CUresult _cuGraphMemcpyNodeSetParams(CUgraphNode hNode, const CUDA_MEMCPY3D* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphMemcpyNodeSetParams + cuPythonInit() + if __cuGraphMemcpyNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphMemcpyNodeSetParams" not found') + err = ( __cuGraphMemcpyNodeSetParams)(hNode, nodeParams) + return err + +cdef CUresult _cuGraphAddMemsetNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_MEMSET_NODE_PARAMS* memsetParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddMemsetNode + cuPythonInit() + if __cuGraphAddMemsetNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddMemsetNode" not found') + err = ( __cuGraphAddMemsetNode)(phGraphNode, hGraph, dependencies, numDependencies, memsetParams, ctx) + return err + +cdef CUresult _cuGraphMemsetNodeGetParams(CUgraphNode hNode, CUDA_MEMSET_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphMemsetNodeGetParams + cuPythonInit() + if __cuGraphMemsetNodeGetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphMemsetNodeGetParams" not found') + err = ( __cuGraphMemsetNodeGetParams)(hNode, nodeParams) + return err + +cdef CUresult _cuGraphMemsetNodeSetParams(CUgraphNode hNode, const CUDA_MEMSET_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphMemsetNodeSetParams + cuPythonInit() + if __cuGraphMemsetNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphMemsetNodeSetParams" not found') + err = ( __cuGraphMemsetNodeSetParams)(hNode, nodeParams) + return err + +cdef CUresult _cuGraphAddHostNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddHostNode + cuPythonInit() + if __cuGraphAddHostNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddHostNode" not found') + err = ( __cuGraphAddHostNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + return err + +cdef CUresult _cuGraphHostNodeGetParams(CUgraphNode hNode, CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphHostNodeGetParams + cuPythonInit() + if __cuGraphHostNodeGetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphHostNodeGetParams" not found') + err = ( __cuGraphHostNodeGetParams)(hNode, nodeParams) + return err + +cdef CUresult _cuGraphHostNodeSetParams(CUgraphNode hNode, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphHostNodeSetParams + cuPythonInit() + if __cuGraphHostNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphHostNodeSetParams" not found') + err = ( __cuGraphHostNodeSetParams)(hNode, nodeParams) + return err + +cdef CUresult _cuGraphAddChildGraphNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUgraph childGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddChildGraphNode + cuPythonInit() + if __cuGraphAddChildGraphNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddChildGraphNode" not found') + err = ( __cuGraphAddChildGraphNode)(phGraphNode, hGraph, dependencies, numDependencies, childGraph) + return err + +cdef CUresult _cuGraphChildGraphNodeGetGraph(CUgraphNode hNode, CUgraph* phGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphChildGraphNodeGetGraph + cuPythonInit() + if __cuGraphChildGraphNodeGetGraph == NULL: + with gil: + raise RuntimeError('Function "cuGraphChildGraphNodeGetGraph" not found') + err = ( __cuGraphChildGraphNodeGetGraph)(hNode, phGraph) + return err + +cdef CUresult _cuGraphAddEmptyNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddEmptyNode + cuPythonInit() + if __cuGraphAddEmptyNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddEmptyNode" not found') + err = ( __cuGraphAddEmptyNode)(phGraphNode, hGraph, dependencies, numDependencies) + return err + +cdef CUresult _cuGraphAddEventRecordNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddEventRecordNode + cuPythonInit() + if __cuGraphAddEventRecordNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddEventRecordNode" not found') + err = ( __cuGraphAddEventRecordNode)(phGraphNode, hGraph, dependencies, numDependencies, event) + return err + +cdef CUresult _cuGraphEventRecordNodeGetEvent(CUgraphNode hNode, CUevent* event_out) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphEventRecordNodeGetEvent + cuPythonInit() + if __cuGraphEventRecordNodeGetEvent == NULL: + with gil: + raise RuntimeError('Function "cuGraphEventRecordNodeGetEvent" not found') + err = ( __cuGraphEventRecordNodeGetEvent)(hNode, event_out) + return err + +cdef CUresult _cuGraphEventRecordNodeSetEvent(CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphEventRecordNodeSetEvent + cuPythonInit() + if __cuGraphEventRecordNodeSetEvent == NULL: + with gil: + raise RuntimeError('Function "cuGraphEventRecordNodeSetEvent" not found') + err = ( __cuGraphEventRecordNodeSetEvent)(hNode, event) + return err + +cdef CUresult _cuGraphAddEventWaitNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddEventWaitNode + cuPythonInit() + if __cuGraphAddEventWaitNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddEventWaitNode" not found') + err = ( __cuGraphAddEventWaitNode)(phGraphNode, hGraph, dependencies, numDependencies, event) + return err + +cdef CUresult _cuGraphEventWaitNodeGetEvent(CUgraphNode hNode, CUevent* event_out) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphEventWaitNodeGetEvent + cuPythonInit() + if __cuGraphEventWaitNodeGetEvent == NULL: + with gil: + raise RuntimeError('Function "cuGraphEventWaitNodeGetEvent" not found') + err = ( __cuGraphEventWaitNodeGetEvent)(hNode, event_out) + return err + +cdef CUresult _cuGraphEventWaitNodeSetEvent(CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphEventWaitNodeSetEvent + cuPythonInit() + if __cuGraphEventWaitNodeSetEvent == NULL: + with gil: + raise RuntimeError('Function "cuGraphEventWaitNodeSetEvent" not found') + err = ( __cuGraphEventWaitNodeSetEvent)(hNode, event) + return err + +cdef CUresult _cuGraphAddExternalSemaphoresSignalNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddExternalSemaphoresSignalNode + cuPythonInit() + if __cuGraphAddExternalSemaphoresSignalNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddExternalSemaphoresSignalNode" not found') + err = ( __cuGraphAddExternalSemaphoresSignalNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + return err + +cdef CUresult _cuGraphExternalSemaphoresSignalNodeGetParams(CUgraphNode hNode, CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExternalSemaphoresSignalNodeGetParams + cuPythonInit() + if __cuGraphExternalSemaphoresSignalNodeGetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeGetParams" not found') + err = ( __cuGraphExternalSemaphoresSignalNodeGetParams)(hNode, params_out) + return err + +cdef CUresult _cuGraphExternalSemaphoresSignalNodeSetParams(CUgraphNode hNode, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExternalSemaphoresSignalNodeSetParams + cuPythonInit() + if __cuGraphExternalSemaphoresSignalNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphExternalSemaphoresSignalNodeSetParams" not found') + err = ( __cuGraphExternalSemaphoresSignalNodeSetParams)(hNode, nodeParams) + return err + +cdef CUresult _cuGraphAddExternalSemaphoresWaitNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddExternalSemaphoresWaitNode + cuPythonInit() + if __cuGraphAddExternalSemaphoresWaitNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddExternalSemaphoresWaitNode" not found') + err = ( __cuGraphAddExternalSemaphoresWaitNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + return err + +cdef CUresult _cuGraphExternalSemaphoresWaitNodeGetParams(CUgraphNode hNode, CUDA_EXT_SEM_WAIT_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExternalSemaphoresWaitNodeGetParams + cuPythonInit() + if __cuGraphExternalSemaphoresWaitNodeGetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeGetParams" not found') + err = ( __cuGraphExternalSemaphoresWaitNodeGetParams)(hNode, params_out) + return err + +cdef CUresult _cuGraphExternalSemaphoresWaitNodeSetParams(CUgraphNode hNode, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExternalSemaphoresWaitNodeSetParams + cuPythonInit() + if __cuGraphExternalSemaphoresWaitNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphExternalSemaphoresWaitNodeSetParams" not found') + err = ( __cuGraphExternalSemaphoresWaitNodeSetParams)(hNode, nodeParams) + return err + +cdef CUresult _cuGraphAddBatchMemOpNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddBatchMemOpNode + cuPythonInit() + if __cuGraphAddBatchMemOpNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddBatchMemOpNode" not found') + err = ( __cuGraphAddBatchMemOpNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + return err + +cdef CUresult _cuGraphBatchMemOpNodeGetParams(CUgraphNode hNode, CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams_out) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphBatchMemOpNodeGetParams + cuPythonInit() + if __cuGraphBatchMemOpNodeGetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphBatchMemOpNodeGetParams" not found') + err = ( __cuGraphBatchMemOpNodeGetParams)(hNode, nodeParams_out) + return err + +cdef CUresult _cuGraphBatchMemOpNodeSetParams(CUgraphNode hNode, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphBatchMemOpNodeSetParams + cuPythonInit() + if __cuGraphBatchMemOpNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphBatchMemOpNodeSetParams" not found') + err = ( __cuGraphBatchMemOpNodeSetParams)(hNode, nodeParams) + return err + +cdef CUresult _cuGraphExecBatchMemOpNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_BATCH_MEM_OP_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecBatchMemOpNodeSetParams + cuPythonInit() + if __cuGraphExecBatchMemOpNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecBatchMemOpNodeSetParams" not found') + err = ( __cuGraphExecBatchMemOpNodeSetParams)(hGraphExec, hNode, nodeParams) + return err + +cdef CUresult _cuGraphAddMemAllocNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUDA_MEM_ALLOC_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddMemAllocNode + cuPythonInit() + if __cuGraphAddMemAllocNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddMemAllocNode" not found') + err = ( __cuGraphAddMemAllocNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + return err + +cdef CUresult _cuGraphMemAllocNodeGetParams(CUgraphNode hNode, CUDA_MEM_ALLOC_NODE_PARAMS* params_out) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphMemAllocNodeGetParams + cuPythonInit() + if __cuGraphMemAllocNodeGetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphMemAllocNodeGetParams" not found') + err = ( __cuGraphMemAllocNodeGetParams)(hNode, params_out) + return err + +cdef CUresult _cuGraphAddMemFreeNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUdeviceptr dptr) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddMemFreeNode + cuPythonInit() + if __cuGraphAddMemFreeNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddMemFreeNode" not found') + err = ( __cuGraphAddMemFreeNode)(phGraphNode, hGraph, dependencies, numDependencies, dptr) + return err + +cdef CUresult _cuGraphMemFreeNodeGetParams(CUgraphNode hNode, CUdeviceptr* dptr_out) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphMemFreeNodeGetParams + cuPythonInit() + if __cuGraphMemFreeNodeGetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphMemFreeNodeGetParams" not found') + err = ( __cuGraphMemFreeNodeGetParams)(hNode, dptr_out) + return err + +cdef CUresult _cuDeviceGraphMemTrim(CUdevice device) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGraphMemTrim + cuPythonInit() + if __cuDeviceGraphMemTrim == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGraphMemTrim" not found') + err = ( __cuDeviceGraphMemTrim)(device) + return err + +cdef CUresult _cuDeviceGetGraphMemAttribute(CUdevice device, CUgraphMem_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetGraphMemAttribute + cuPythonInit() + if __cuDeviceGetGraphMemAttribute == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetGraphMemAttribute" not found') + err = ( __cuDeviceGetGraphMemAttribute)(device, attr, value) + return err + +cdef CUresult _cuDeviceSetGraphMemAttribute(CUdevice device, CUgraphMem_attribute attr, void* value) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceSetGraphMemAttribute + cuPythonInit() + if __cuDeviceSetGraphMemAttribute == NULL: + with gil: + raise RuntimeError('Function "cuDeviceSetGraphMemAttribute" not found') + err = ( __cuDeviceSetGraphMemAttribute)(device, attr, value) + return err + +cdef CUresult _cuGraphClone(CUgraph* phGraphClone, CUgraph originalGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphClone + cuPythonInit() + if __cuGraphClone == NULL: + with gil: + raise RuntimeError('Function "cuGraphClone" not found') + err = ( __cuGraphClone)(phGraphClone, originalGraph) + return err + +cdef CUresult _cuGraphNodeFindInClone(CUgraphNode* phNode, CUgraphNode hOriginalNode, CUgraph hClonedGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphNodeFindInClone + cuPythonInit() + if __cuGraphNodeFindInClone == NULL: + with gil: + raise RuntimeError('Function "cuGraphNodeFindInClone" not found') + err = ( __cuGraphNodeFindInClone)(phNode, hOriginalNode, hClonedGraph) + return err + +cdef CUresult _cuGraphNodeGetType(CUgraphNode hNode, CUgraphNodeType* typename) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphNodeGetType + cuPythonInit() + if __cuGraphNodeGetType == NULL: + with gil: + raise RuntimeError('Function "cuGraphNodeGetType" not found') + err = ( __cuGraphNodeGetType)(hNode, typename) + return err + +cdef CUresult _cuGraphGetNodes(CUgraph hGraph, CUgraphNode* nodes, size_t* numNodes) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphGetNodes + cuPythonInit() + if __cuGraphGetNodes == NULL: + with gil: + raise RuntimeError('Function "cuGraphGetNodes" not found') + err = ( __cuGraphGetNodes)(hGraph, nodes, numNodes) + return err + +cdef CUresult _cuGraphGetRootNodes(CUgraph hGraph, CUgraphNode* rootNodes, size_t* numRootNodes) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphGetRootNodes + cuPythonInit() + if __cuGraphGetRootNodes == NULL: + with gil: + raise RuntimeError('Function "cuGraphGetRootNodes" not found') + err = ( __cuGraphGetRootNodes)(hGraph, rootNodes, numRootNodes) + return err + +cdef CUresult _cuGraphGetEdges(CUgraph hGraph, CUgraphNode* from_, CUgraphNode* to, size_t* numEdges) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphGetEdges + cuPythonInit() + if __cuGraphGetEdges == NULL: + with gil: + raise RuntimeError('Function "cuGraphGetEdges" not found') + err = ( __cuGraphGetEdges)(hGraph, from_, to, numEdges) + return err + +cdef CUresult _cuGraphGetEdges_v2(CUgraph hGraph, CUgraphNode* from_, CUgraphNode* to, CUgraphEdgeData* edgeData, size_t* numEdges) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphGetEdges_v2 + cuPythonInit() + if __cuGraphGetEdges_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGraphGetEdges_v2" not found') + err = ( __cuGraphGetEdges_v2)(hGraph, from_, to, edgeData, numEdges) + return err + +cdef CUresult _cuGraphNodeGetDependencies(CUgraphNode hNode, CUgraphNode* dependencies, size_t* numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphNodeGetDependencies + cuPythonInit() + if __cuGraphNodeGetDependencies == NULL: + with gil: + raise RuntimeError('Function "cuGraphNodeGetDependencies" not found') + err = ( __cuGraphNodeGetDependencies)(hNode, dependencies, numDependencies) + return err + +cdef CUresult _cuGraphNodeGetDependencies_v2(CUgraphNode hNode, CUgraphNode* dependencies, CUgraphEdgeData* edgeData, size_t* numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphNodeGetDependencies_v2 + cuPythonInit() + if __cuGraphNodeGetDependencies_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGraphNodeGetDependencies_v2" not found') + err = ( __cuGraphNodeGetDependencies_v2)(hNode, dependencies, edgeData, numDependencies) + return err + +cdef CUresult _cuGraphNodeGetDependentNodes(CUgraphNode hNode, CUgraphNode* dependentNodes, size_t* numDependentNodes) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphNodeGetDependentNodes + cuPythonInit() + if __cuGraphNodeGetDependentNodes == NULL: + with gil: + raise RuntimeError('Function "cuGraphNodeGetDependentNodes" not found') + err = ( __cuGraphNodeGetDependentNodes)(hNode, dependentNodes, numDependentNodes) + return err + +cdef CUresult _cuGraphNodeGetDependentNodes_v2(CUgraphNode hNode, CUgraphNode* dependentNodes, CUgraphEdgeData* edgeData, size_t* numDependentNodes) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphNodeGetDependentNodes_v2 + cuPythonInit() + if __cuGraphNodeGetDependentNodes_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGraphNodeGetDependentNodes_v2" not found') + err = ( __cuGraphNodeGetDependentNodes_v2)(hNode, dependentNodes, edgeData, numDependentNodes) + return err + +cdef CUresult _cuGraphAddDependencies(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddDependencies + cuPythonInit() + if __cuGraphAddDependencies == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddDependencies" not found') + err = ( __cuGraphAddDependencies)(hGraph, from_, to, numDependencies) + return err + +cdef CUresult _cuGraphAddDependencies_v2(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, const CUgraphEdgeData* edgeData, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddDependencies_v2 + cuPythonInit() + if __cuGraphAddDependencies_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddDependencies_v2" not found') + err = ( __cuGraphAddDependencies_v2)(hGraph, from_, to, edgeData, numDependencies) + return err + +cdef CUresult _cuGraphRemoveDependencies(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphRemoveDependencies + cuPythonInit() + if __cuGraphRemoveDependencies == NULL: + with gil: + raise RuntimeError('Function "cuGraphRemoveDependencies" not found') + err = ( __cuGraphRemoveDependencies)(hGraph, from_, to, numDependencies) + return err + +cdef CUresult _cuGraphRemoveDependencies_v2(CUgraph hGraph, const CUgraphNode* from_, const CUgraphNode* to, const CUgraphEdgeData* edgeData, size_t numDependencies) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphRemoveDependencies_v2 + cuPythonInit() + if __cuGraphRemoveDependencies_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGraphRemoveDependencies_v2" not found') + err = ( __cuGraphRemoveDependencies_v2)(hGraph, from_, to, edgeData, numDependencies) + return err + +cdef CUresult _cuGraphDestroyNode(CUgraphNode hNode) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphDestroyNode + cuPythonInit() + if __cuGraphDestroyNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphDestroyNode" not found') + err = ( __cuGraphDestroyNode)(hNode) + return err + +cdef CUresult _cuGraphInstantiateWithFlags(CUgraphExec* phGraphExec, CUgraph hGraph, unsigned long long flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphInstantiateWithFlags + cuPythonInit() + if __cuGraphInstantiateWithFlags == NULL: + with gil: + raise RuntimeError('Function "cuGraphInstantiateWithFlags" not found') + err = ( __cuGraphInstantiateWithFlags)(phGraphExec, hGraph, flags) + return err + +cdef CUresult _cuGraphInstantiateWithParams(CUgraphExec* phGraphExec, CUgraph hGraph, CUDA_GRAPH_INSTANTIATE_PARAMS* instantiateParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphInstantiateWithParams + cuPythonInit() + if __cuGraphInstantiateWithParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphInstantiateWithParams" not found') + err = ( __cuGraphInstantiateWithParams)(phGraphExec, hGraph, instantiateParams) + return err + +cdef CUresult _cuGraphExecGetFlags(CUgraphExec hGraphExec, cuuint64_t* flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecGetFlags + cuPythonInit() + if __cuGraphExecGetFlags == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecGetFlags" not found') + err = ( __cuGraphExecGetFlags)(hGraphExec, flags) + return err + +cdef CUresult _cuGraphExecKernelNodeSetParams_v2(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_KERNEL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecKernelNodeSetParams_v2 + cuPythonInit() + if __cuGraphExecKernelNodeSetParams_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecKernelNodeSetParams_v2" not found') + err = ( __cuGraphExecKernelNodeSetParams_v2)(hGraphExec, hNode, nodeParams) + return err + +cdef CUresult _cuGraphExecMemcpyNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_MEMCPY3D* copyParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecMemcpyNodeSetParams + cuPythonInit() + if __cuGraphExecMemcpyNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecMemcpyNodeSetParams" not found') + err = ( __cuGraphExecMemcpyNodeSetParams)(hGraphExec, hNode, copyParams, ctx) + return err + +cdef CUresult _cuGraphExecMemsetNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_MEMSET_NODE_PARAMS* memsetParams, CUcontext ctx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecMemsetNodeSetParams + cuPythonInit() + if __cuGraphExecMemsetNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecMemsetNodeSetParams" not found') + err = ( __cuGraphExecMemsetNodeSetParams)(hGraphExec, hNode, memsetParams, ctx) + return err + +cdef CUresult _cuGraphExecHostNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_HOST_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecHostNodeSetParams + cuPythonInit() + if __cuGraphExecHostNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecHostNodeSetParams" not found') + err = ( __cuGraphExecHostNodeSetParams)(hGraphExec, hNode, nodeParams) + return err + +cdef CUresult _cuGraphExecChildGraphNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraph childGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecChildGraphNodeSetParams + cuPythonInit() + if __cuGraphExecChildGraphNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecChildGraphNodeSetParams" not found') + err = ( __cuGraphExecChildGraphNodeSetParams)(hGraphExec, hNode, childGraph) + return err + +cdef CUresult _cuGraphExecEventRecordNodeSetEvent(CUgraphExec hGraphExec, CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecEventRecordNodeSetEvent + cuPythonInit() + if __cuGraphExecEventRecordNodeSetEvent == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecEventRecordNodeSetEvent" not found') + err = ( __cuGraphExecEventRecordNodeSetEvent)(hGraphExec, hNode, event) + return err + +cdef CUresult _cuGraphExecEventWaitNodeSetEvent(CUgraphExec hGraphExec, CUgraphNode hNode, CUevent event) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecEventWaitNodeSetEvent + cuPythonInit() + if __cuGraphExecEventWaitNodeSetEvent == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecEventWaitNodeSetEvent" not found') + err = ( __cuGraphExecEventWaitNodeSetEvent)(hGraphExec, hNode, event) + return err + +cdef CUresult _cuGraphExecExternalSemaphoresSignalNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_EXT_SEM_SIGNAL_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecExternalSemaphoresSignalNodeSetParams + cuPythonInit() + if __cuGraphExecExternalSemaphoresSignalNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecExternalSemaphoresSignalNodeSetParams" not found') + err = ( __cuGraphExecExternalSemaphoresSignalNodeSetParams)(hGraphExec, hNode, nodeParams) + return err + +cdef CUresult _cuGraphExecExternalSemaphoresWaitNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, const CUDA_EXT_SEM_WAIT_NODE_PARAMS* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecExternalSemaphoresWaitNodeSetParams + cuPythonInit() + if __cuGraphExecExternalSemaphoresWaitNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecExternalSemaphoresWaitNodeSetParams" not found') + err = ( __cuGraphExecExternalSemaphoresWaitNodeSetParams)(hGraphExec, hNode, nodeParams) + return err + +cdef CUresult _cuGraphNodeSetEnabled(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int isEnabled) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphNodeSetEnabled + cuPythonInit() + if __cuGraphNodeSetEnabled == NULL: + with gil: + raise RuntimeError('Function "cuGraphNodeSetEnabled" not found') + err = ( __cuGraphNodeSetEnabled)(hGraphExec, hNode, isEnabled) + return err + +cdef CUresult _cuGraphNodeGetEnabled(CUgraphExec hGraphExec, CUgraphNode hNode, unsigned int* isEnabled) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphNodeGetEnabled + cuPythonInit() + if __cuGraphNodeGetEnabled == NULL: + with gil: + raise RuntimeError('Function "cuGraphNodeGetEnabled" not found') + err = ( __cuGraphNodeGetEnabled)(hGraphExec, hNode, isEnabled) + return err + +cdef CUresult _cuGraphUpload(CUgraphExec hGraphExec, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphUpload + cuPythonInit() + if __cuGraphUpload == NULL: + with gil: + raise RuntimeError('Function "cuGraphUpload" not found') + err = ( __cuGraphUpload)(hGraphExec, hStream) + return err + +cdef CUresult _cuGraphLaunch(CUgraphExec hGraphExec, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphLaunch + cuPythonInit() + if __cuGraphLaunch == NULL: + with gil: + raise RuntimeError('Function "cuGraphLaunch" not found') + err = ( __cuGraphLaunch)(hGraphExec, hStream) + return err + +cdef CUresult _cuGraphExecDestroy(CUgraphExec hGraphExec) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecDestroy + cuPythonInit() + if __cuGraphExecDestroy == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecDestroy" not found') + err = ( __cuGraphExecDestroy)(hGraphExec) + return err + +cdef CUresult _cuGraphDestroy(CUgraph hGraph) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphDestroy + cuPythonInit() + if __cuGraphDestroy == NULL: + with gil: + raise RuntimeError('Function "cuGraphDestroy" not found') + err = ( __cuGraphDestroy)(hGraph) + return err + +cdef CUresult _cuGraphExecUpdate_v2(CUgraphExec hGraphExec, CUgraph hGraph, CUgraphExecUpdateResultInfo* resultInfo) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecUpdate_v2 + cuPythonInit() + if __cuGraphExecUpdate_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecUpdate_v2" not found') + err = ( __cuGraphExecUpdate_v2)(hGraphExec, hGraph, resultInfo) + return err + +cdef CUresult _cuGraphKernelNodeCopyAttributes(CUgraphNode dst, CUgraphNode src) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphKernelNodeCopyAttributes + cuPythonInit() + if __cuGraphKernelNodeCopyAttributes == NULL: + with gil: + raise RuntimeError('Function "cuGraphKernelNodeCopyAttributes" not found') + err = ( __cuGraphKernelNodeCopyAttributes)(dst, src) + return err + +cdef CUresult _cuGraphKernelNodeGetAttribute(CUgraphNode hNode, CUkernelNodeAttrID attr, CUkernelNodeAttrValue* value_out) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphKernelNodeGetAttribute + cuPythonInit() + if __cuGraphKernelNodeGetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuGraphKernelNodeGetAttribute" not found') + err = ( __cuGraphKernelNodeGetAttribute)(hNode, attr, value_out) + return err + +cdef CUresult _cuGraphKernelNodeSetAttribute(CUgraphNode hNode, CUkernelNodeAttrID attr, const CUkernelNodeAttrValue* value) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphKernelNodeSetAttribute + cuPythonInit() + if __cuGraphKernelNodeSetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuGraphKernelNodeSetAttribute" not found') + err = ( __cuGraphKernelNodeSetAttribute)(hNode, attr, value) + return err + +cdef CUresult _cuGraphDebugDotPrint(CUgraph hGraph, const char* path, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphDebugDotPrint + cuPythonInit() + if __cuGraphDebugDotPrint == NULL: + with gil: + raise RuntimeError('Function "cuGraphDebugDotPrint" not found') + err = ( __cuGraphDebugDotPrint)(hGraph, path, flags) + return err + +cdef CUresult _cuUserObjectCreate(CUuserObject* object_out, void* ptr, CUhostFn destroy, unsigned int initialRefcount, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuUserObjectCreate + cuPythonInit() + if __cuUserObjectCreate == NULL: + with gil: + raise RuntimeError('Function "cuUserObjectCreate" not found') + err = ( __cuUserObjectCreate)(object_out, ptr, destroy, initialRefcount, flags) + return err + +cdef CUresult _cuUserObjectRetain(CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuUserObjectRetain + cuPythonInit() + if __cuUserObjectRetain == NULL: + with gil: + raise RuntimeError('Function "cuUserObjectRetain" not found') + err = ( __cuUserObjectRetain)(object, count) + return err + +cdef CUresult _cuUserObjectRelease(CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuUserObjectRelease + cuPythonInit() + if __cuUserObjectRelease == NULL: + with gil: + raise RuntimeError('Function "cuUserObjectRelease" not found') + err = ( __cuUserObjectRelease)(object, count) + return err + +cdef CUresult _cuGraphRetainUserObject(CUgraph graph, CUuserObject object, unsigned int count, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphRetainUserObject + cuPythonInit() + if __cuGraphRetainUserObject == NULL: + with gil: + raise RuntimeError('Function "cuGraphRetainUserObject" not found') + err = ( __cuGraphRetainUserObject)(graph, object, count, flags) + return err + +cdef CUresult _cuGraphReleaseUserObject(CUgraph graph, CUuserObject object, unsigned int count) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphReleaseUserObject + cuPythonInit() + if __cuGraphReleaseUserObject == NULL: + with gil: + raise RuntimeError('Function "cuGraphReleaseUserObject" not found') + err = ( __cuGraphReleaseUserObject)(graph, object, count) + return err + +cdef CUresult _cuGraphAddNode(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, size_t numDependencies, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddNode + cuPythonInit() + if __cuGraphAddNode == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddNode" not found') + err = ( __cuGraphAddNode)(phGraphNode, hGraph, dependencies, numDependencies, nodeParams) + return err + +cdef CUresult _cuGraphAddNode_v2(CUgraphNode* phGraphNode, CUgraph hGraph, const CUgraphNode* dependencies, const CUgraphEdgeData* dependencyData, size_t numDependencies, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphAddNode_v2 + cuPythonInit() + if __cuGraphAddNode_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGraphAddNode_v2" not found') + err = ( __cuGraphAddNode_v2)(phGraphNode, hGraph, dependencies, dependencyData, numDependencies, nodeParams) + return err + +cdef CUresult _cuGraphNodeSetParams(CUgraphNode hNode, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphNodeSetParams + cuPythonInit() + if __cuGraphNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphNodeSetParams" not found') + err = ( __cuGraphNodeSetParams)(hNode, nodeParams) + return err + +cdef CUresult _cuGraphExecNodeSetParams(CUgraphExec hGraphExec, CUgraphNode hNode, CUgraphNodeParams* nodeParams) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphExecNodeSetParams + cuPythonInit() + if __cuGraphExecNodeSetParams == NULL: + with gil: + raise RuntimeError('Function "cuGraphExecNodeSetParams" not found') + err = ( __cuGraphExecNodeSetParams)(hGraphExec, hNode, nodeParams) + return err + +cdef CUresult _cuGraphConditionalHandleCreate(CUgraphConditionalHandle* pHandle_out, CUgraph hGraph, CUcontext ctx, unsigned int defaultLaunchValue, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphConditionalHandleCreate + cuPythonInit() + if __cuGraphConditionalHandleCreate == NULL: + with gil: + raise RuntimeError('Function "cuGraphConditionalHandleCreate" not found') + err = ( __cuGraphConditionalHandleCreate)(pHandle_out, hGraph, ctx, defaultLaunchValue, flags) + return err + +cdef CUresult _cuOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuOccupancyMaxActiveBlocksPerMultiprocessor + cuPythonInit() + if __cuOccupancyMaxActiveBlocksPerMultiprocessor == NULL: + with gil: + raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessor" not found') + err = ( __cuOccupancyMaxActiveBlocksPerMultiprocessor)(numBlocks, func, blockSize, dynamicSMemSize) + return err + +cdef CUresult _cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, CUfunction func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + cuPythonInit() + if __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags == NULL: + with gil: + raise RuntimeError('Function "cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags" not found') + err = ( __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags)(numBlocks, func, blockSize, dynamicSMemSize, flags) + return err + +cdef CUresult _cuOccupancyMaxPotentialBlockSize(int* minGridSize, int* blockSize, CUfunction func, CUoccupancyB2DSize blockSizeToDynamicSMemSize, size_t dynamicSMemSize, int blockSizeLimit) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuOccupancyMaxPotentialBlockSize + cuPythonInit() + if __cuOccupancyMaxPotentialBlockSize == NULL: + with gil: + raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSize" not found') + err = ( __cuOccupancyMaxPotentialBlockSize)(minGridSize, blockSize, func, blockSizeToDynamicSMemSize, dynamicSMemSize, blockSizeLimit) + return err + +cdef CUresult _cuOccupancyMaxPotentialBlockSizeWithFlags(int* minGridSize, int* blockSize, CUfunction func, CUoccupancyB2DSize blockSizeToDynamicSMemSize, size_t dynamicSMemSize, int blockSizeLimit, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuOccupancyMaxPotentialBlockSizeWithFlags + cuPythonInit() + if __cuOccupancyMaxPotentialBlockSizeWithFlags == NULL: + with gil: + raise RuntimeError('Function "cuOccupancyMaxPotentialBlockSizeWithFlags" not found') + err = ( __cuOccupancyMaxPotentialBlockSizeWithFlags)(minGridSize, blockSize, func, blockSizeToDynamicSMemSize, dynamicSMemSize, blockSizeLimit, flags) + return err + +cdef CUresult _cuOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, CUfunction func, int numBlocks, int blockSize) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuOccupancyAvailableDynamicSMemPerBlock + cuPythonInit() + if __cuOccupancyAvailableDynamicSMemPerBlock == NULL: + with gil: + raise RuntimeError('Function "cuOccupancyAvailableDynamicSMemPerBlock" not found') + err = ( __cuOccupancyAvailableDynamicSMemPerBlock)(dynamicSmemSize, func, numBlocks, blockSize) + return err + +cdef CUresult _cuOccupancyMaxPotentialClusterSize(int* clusterSize, CUfunction func, const CUlaunchConfig* config) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuOccupancyMaxPotentialClusterSize + cuPythonInit() + if __cuOccupancyMaxPotentialClusterSize == NULL: + with gil: + raise RuntimeError('Function "cuOccupancyMaxPotentialClusterSize" not found') + err = ( __cuOccupancyMaxPotentialClusterSize)(clusterSize, func, config) + return err + +cdef CUresult _cuOccupancyMaxActiveClusters(int* numClusters, CUfunction func, const CUlaunchConfig* config) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuOccupancyMaxActiveClusters + cuPythonInit() + if __cuOccupancyMaxActiveClusters == NULL: + with gil: + raise RuntimeError('Function "cuOccupancyMaxActiveClusters" not found') + err = ( __cuOccupancyMaxActiveClusters)(numClusters, func, config) + return err + +cdef CUresult _cuTexRefSetArray(CUtexref hTexRef, CUarray hArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefSetArray + cuPythonInit() + if __cuTexRefSetArray == NULL: + with gil: + raise RuntimeError('Function "cuTexRefSetArray" not found') + err = ( __cuTexRefSetArray)(hTexRef, hArray, Flags) + return err + +cdef CUresult _cuTexRefSetMipmappedArray(CUtexref hTexRef, CUmipmappedArray hMipmappedArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefSetMipmappedArray + cuPythonInit() + if __cuTexRefSetMipmappedArray == NULL: + with gil: + raise RuntimeError('Function "cuTexRefSetMipmappedArray" not found') + err = ( __cuTexRefSetMipmappedArray)(hTexRef, hMipmappedArray, Flags) + return err + +cdef CUresult _cuTexRefSetAddress_v2(size_t* ByteOffset, CUtexref hTexRef, CUdeviceptr dptr, size_t numbytes) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefSetAddress_v2 + cuPythonInit() + if __cuTexRefSetAddress_v2 == NULL: + with gil: + raise RuntimeError('Function "cuTexRefSetAddress_v2" not found') + err = ( __cuTexRefSetAddress_v2)(ByteOffset, hTexRef, dptr, numbytes) + return err + +cdef CUresult _cuTexRefSetAddress2D_v3(CUtexref hTexRef, const CUDA_ARRAY_DESCRIPTOR* desc, CUdeviceptr dptr, size_t Pitch) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefSetAddress2D_v3 + cuPythonInit() + if __cuTexRefSetAddress2D_v3 == NULL: + with gil: + raise RuntimeError('Function "cuTexRefSetAddress2D_v3" not found') + err = ( __cuTexRefSetAddress2D_v3)(hTexRef, desc, dptr, Pitch) + return err + +cdef CUresult _cuTexRefSetFormat(CUtexref hTexRef, CUarray_format fmt, int NumPackedComponents) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefSetFormat + cuPythonInit() + if __cuTexRefSetFormat == NULL: + with gil: + raise RuntimeError('Function "cuTexRefSetFormat" not found') + err = ( __cuTexRefSetFormat)(hTexRef, fmt, NumPackedComponents) + return err + +cdef CUresult _cuTexRefSetAddressMode(CUtexref hTexRef, int dim, CUaddress_mode am) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefSetAddressMode + cuPythonInit() + if __cuTexRefSetAddressMode == NULL: + with gil: + raise RuntimeError('Function "cuTexRefSetAddressMode" not found') + err = ( __cuTexRefSetAddressMode)(hTexRef, dim, am) + return err + +cdef CUresult _cuTexRefSetFilterMode(CUtexref hTexRef, CUfilter_mode fm) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefSetFilterMode + cuPythonInit() + if __cuTexRefSetFilterMode == NULL: + with gil: + raise RuntimeError('Function "cuTexRefSetFilterMode" not found') + err = ( __cuTexRefSetFilterMode)(hTexRef, fm) + return err + +cdef CUresult _cuTexRefSetMipmapFilterMode(CUtexref hTexRef, CUfilter_mode fm) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefSetMipmapFilterMode + cuPythonInit() + if __cuTexRefSetMipmapFilterMode == NULL: + with gil: + raise RuntimeError('Function "cuTexRefSetMipmapFilterMode" not found') + err = ( __cuTexRefSetMipmapFilterMode)(hTexRef, fm) + return err + +cdef CUresult _cuTexRefSetMipmapLevelBias(CUtexref hTexRef, float bias) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefSetMipmapLevelBias + cuPythonInit() + if __cuTexRefSetMipmapLevelBias == NULL: + with gil: + raise RuntimeError('Function "cuTexRefSetMipmapLevelBias" not found') + err = ( __cuTexRefSetMipmapLevelBias)(hTexRef, bias) + return err + +cdef CUresult _cuTexRefSetMipmapLevelClamp(CUtexref hTexRef, float minMipmapLevelClamp, float maxMipmapLevelClamp) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefSetMipmapLevelClamp + cuPythonInit() + if __cuTexRefSetMipmapLevelClamp == NULL: + with gil: + raise RuntimeError('Function "cuTexRefSetMipmapLevelClamp" not found') + err = ( __cuTexRefSetMipmapLevelClamp)(hTexRef, minMipmapLevelClamp, maxMipmapLevelClamp) + return err + +cdef CUresult _cuTexRefSetMaxAnisotropy(CUtexref hTexRef, unsigned int maxAniso) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefSetMaxAnisotropy + cuPythonInit() + if __cuTexRefSetMaxAnisotropy == NULL: + with gil: + raise RuntimeError('Function "cuTexRefSetMaxAnisotropy" not found') + err = ( __cuTexRefSetMaxAnisotropy)(hTexRef, maxAniso) + return err + +cdef CUresult _cuTexRefSetBorderColor(CUtexref hTexRef, float* pBorderColor) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefSetBorderColor + cuPythonInit() + if __cuTexRefSetBorderColor == NULL: + with gil: + raise RuntimeError('Function "cuTexRefSetBorderColor" not found') + err = ( __cuTexRefSetBorderColor)(hTexRef, pBorderColor) + return err + +cdef CUresult _cuTexRefSetFlags(CUtexref hTexRef, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefSetFlags + cuPythonInit() + if __cuTexRefSetFlags == NULL: + with gil: + raise RuntimeError('Function "cuTexRefSetFlags" not found') + err = ( __cuTexRefSetFlags)(hTexRef, Flags) + return err + +cdef CUresult _cuTexRefGetAddress_v2(CUdeviceptr* pdptr, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefGetAddress_v2 + cuPythonInit() + if __cuTexRefGetAddress_v2 == NULL: + with gil: + raise RuntimeError('Function "cuTexRefGetAddress_v2" not found') + err = ( __cuTexRefGetAddress_v2)(pdptr, hTexRef) + return err + +cdef CUresult _cuTexRefGetArray(CUarray* phArray, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefGetArray + cuPythonInit() + if __cuTexRefGetArray == NULL: + with gil: + raise RuntimeError('Function "cuTexRefGetArray" not found') + err = ( __cuTexRefGetArray)(phArray, hTexRef) + return err + +cdef CUresult _cuTexRefGetMipmappedArray(CUmipmappedArray* phMipmappedArray, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefGetMipmappedArray + cuPythonInit() + if __cuTexRefGetMipmappedArray == NULL: + with gil: + raise RuntimeError('Function "cuTexRefGetMipmappedArray" not found') + err = ( __cuTexRefGetMipmappedArray)(phMipmappedArray, hTexRef) + return err + +cdef CUresult _cuTexRefGetAddressMode(CUaddress_mode* pam, CUtexref hTexRef, int dim) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefGetAddressMode + cuPythonInit() + if __cuTexRefGetAddressMode == NULL: + with gil: + raise RuntimeError('Function "cuTexRefGetAddressMode" not found') + err = ( __cuTexRefGetAddressMode)(pam, hTexRef, dim) + return err + +cdef CUresult _cuTexRefGetFilterMode(CUfilter_mode* pfm, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefGetFilterMode + cuPythonInit() + if __cuTexRefGetFilterMode == NULL: + with gil: + raise RuntimeError('Function "cuTexRefGetFilterMode" not found') + err = ( __cuTexRefGetFilterMode)(pfm, hTexRef) + return err + +cdef CUresult _cuTexRefGetFormat(CUarray_format* pFormat, int* pNumChannels, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefGetFormat + cuPythonInit() + if __cuTexRefGetFormat == NULL: + with gil: + raise RuntimeError('Function "cuTexRefGetFormat" not found') + err = ( __cuTexRefGetFormat)(pFormat, pNumChannels, hTexRef) + return err + +cdef CUresult _cuTexRefGetMipmapFilterMode(CUfilter_mode* pfm, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefGetMipmapFilterMode + cuPythonInit() + if __cuTexRefGetMipmapFilterMode == NULL: + with gil: + raise RuntimeError('Function "cuTexRefGetMipmapFilterMode" not found') + err = ( __cuTexRefGetMipmapFilterMode)(pfm, hTexRef) + return err + +cdef CUresult _cuTexRefGetMipmapLevelBias(float* pbias, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefGetMipmapLevelBias + cuPythonInit() + if __cuTexRefGetMipmapLevelBias == NULL: + with gil: + raise RuntimeError('Function "cuTexRefGetMipmapLevelBias" not found') + err = ( __cuTexRefGetMipmapLevelBias)(pbias, hTexRef) + return err + +cdef CUresult _cuTexRefGetMipmapLevelClamp(float* pminMipmapLevelClamp, float* pmaxMipmapLevelClamp, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefGetMipmapLevelClamp + cuPythonInit() + if __cuTexRefGetMipmapLevelClamp == NULL: + with gil: + raise RuntimeError('Function "cuTexRefGetMipmapLevelClamp" not found') + err = ( __cuTexRefGetMipmapLevelClamp)(pminMipmapLevelClamp, pmaxMipmapLevelClamp, hTexRef) + return err + +cdef CUresult _cuTexRefGetMaxAnisotropy(int* pmaxAniso, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefGetMaxAnisotropy + cuPythonInit() + if __cuTexRefGetMaxAnisotropy == NULL: + with gil: + raise RuntimeError('Function "cuTexRefGetMaxAnisotropy" not found') + err = ( __cuTexRefGetMaxAnisotropy)(pmaxAniso, hTexRef) + return err + +cdef CUresult _cuTexRefGetBorderColor(float* pBorderColor, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefGetBorderColor + cuPythonInit() + if __cuTexRefGetBorderColor == NULL: + with gil: + raise RuntimeError('Function "cuTexRefGetBorderColor" not found') + err = ( __cuTexRefGetBorderColor)(pBorderColor, hTexRef) + return err + +cdef CUresult _cuTexRefGetFlags(unsigned int* pFlags, CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefGetFlags + cuPythonInit() + if __cuTexRefGetFlags == NULL: + with gil: + raise RuntimeError('Function "cuTexRefGetFlags" not found') + err = ( __cuTexRefGetFlags)(pFlags, hTexRef) + return err + +cdef CUresult _cuTexRefCreate(CUtexref* pTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefCreate + cuPythonInit() + if __cuTexRefCreate == NULL: + with gil: + raise RuntimeError('Function "cuTexRefCreate" not found') + err = ( __cuTexRefCreate)(pTexRef) + return err + +cdef CUresult _cuTexRefDestroy(CUtexref hTexRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexRefDestroy + cuPythonInit() + if __cuTexRefDestroy == NULL: + with gil: + raise RuntimeError('Function "cuTexRefDestroy" not found') + err = ( __cuTexRefDestroy)(hTexRef) + return err + +cdef CUresult _cuSurfRefSetArray(CUsurfref hSurfRef, CUarray hArray, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuSurfRefSetArray + cuPythonInit() + if __cuSurfRefSetArray == NULL: + with gil: + raise RuntimeError('Function "cuSurfRefSetArray" not found') + err = ( __cuSurfRefSetArray)(hSurfRef, hArray, Flags) + return err + +cdef CUresult _cuSurfRefGetArray(CUarray* phArray, CUsurfref hSurfRef) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuSurfRefGetArray + cuPythonInit() + if __cuSurfRefGetArray == NULL: + with gil: + raise RuntimeError('Function "cuSurfRefGetArray" not found') + err = ( __cuSurfRefGetArray)(phArray, hSurfRef) + return err + +cdef CUresult _cuTexObjectCreate(CUtexObject* pTexObject, const CUDA_RESOURCE_DESC* pResDesc, const CUDA_TEXTURE_DESC* pTexDesc, const CUDA_RESOURCE_VIEW_DESC* pResViewDesc) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexObjectCreate + cuPythonInit() + if __cuTexObjectCreate == NULL: + with gil: + raise RuntimeError('Function "cuTexObjectCreate" not found') + err = ( __cuTexObjectCreate)(pTexObject, pResDesc, pTexDesc, pResViewDesc) + return err + +cdef CUresult _cuTexObjectDestroy(CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexObjectDestroy + cuPythonInit() + if __cuTexObjectDestroy == NULL: + with gil: + raise RuntimeError('Function "cuTexObjectDestroy" not found') + err = ( __cuTexObjectDestroy)(texObject) + return err + +cdef CUresult _cuTexObjectGetResourceDesc(CUDA_RESOURCE_DESC* pResDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexObjectGetResourceDesc + cuPythonInit() + if __cuTexObjectGetResourceDesc == NULL: + with gil: + raise RuntimeError('Function "cuTexObjectGetResourceDesc" not found') + err = ( __cuTexObjectGetResourceDesc)(pResDesc, texObject) + return err + +cdef CUresult _cuTexObjectGetTextureDesc(CUDA_TEXTURE_DESC* pTexDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexObjectGetTextureDesc + cuPythonInit() + if __cuTexObjectGetTextureDesc == NULL: + with gil: + raise RuntimeError('Function "cuTexObjectGetTextureDesc" not found') + err = ( __cuTexObjectGetTextureDesc)(pTexDesc, texObject) + return err + +cdef CUresult _cuTexObjectGetResourceViewDesc(CUDA_RESOURCE_VIEW_DESC* pResViewDesc, CUtexObject texObject) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTexObjectGetResourceViewDesc + cuPythonInit() + if __cuTexObjectGetResourceViewDesc == NULL: + with gil: + raise RuntimeError('Function "cuTexObjectGetResourceViewDesc" not found') + err = ( __cuTexObjectGetResourceViewDesc)(pResViewDesc, texObject) + return err + +cdef CUresult _cuSurfObjectCreate(CUsurfObject* pSurfObject, const CUDA_RESOURCE_DESC* pResDesc) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuSurfObjectCreate + cuPythonInit() + if __cuSurfObjectCreate == NULL: + with gil: + raise RuntimeError('Function "cuSurfObjectCreate" not found') + err = ( __cuSurfObjectCreate)(pSurfObject, pResDesc) + return err + +cdef CUresult _cuSurfObjectDestroy(CUsurfObject surfObject) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuSurfObjectDestroy + cuPythonInit() + if __cuSurfObjectDestroy == NULL: + with gil: + raise RuntimeError('Function "cuSurfObjectDestroy" not found') + err = ( __cuSurfObjectDestroy)(surfObject) + return err + +cdef CUresult _cuSurfObjectGetResourceDesc(CUDA_RESOURCE_DESC* pResDesc, CUsurfObject surfObject) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuSurfObjectGetResourceDesc + cuPythonInit() + if __cuSurfObjectGetResourceDesc == NULL: + with gil: + raise RuntimeError('Function "cuSurfObjectGetResourceDesc" not found') + err = ( __cuSurfObjectGetResourceDesc)(pResDesc, surfObject) + return err + +cdef CUresult _cuTensorMapEncodeTiled(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, const cuuint32_t* boxDim, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTensorMapEncodeTiled + cuPythonInit() + if __cuTensorMapEncodeTiled == NULL: + with gil: + raise RuntimeError('Function "cuTensorMapEncodeTiled" not found') + err = ( __cuTensorMapEncodeTiled)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, boxDim, elementStrides, interleave, swizzle, l2Promotion, oobFill) + return err + +cdef CUresult _cuTensorMapEncodeIm2col(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, const int* pixelBoxLowerCorner, const int* pixelBoxUpperCorner, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTensorMapEncodeIm2col + cuPythonInit() + if __cuTensorMapEncodeIm2col == NULL: + with gil: + raise RuntimeError('Function "cuTensorMapEncodeIm2col" not found') + err = ( __cuTensorMapEncodeIm2col)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCorner, pixelBoxUpperCorner, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, swizzle, l2Promotion, oobFill) + return err + +cdef CUresult _cuTensorMapEncodeIm2colWide(CUtensorMap* tensorMap, CUtensorMapDataType tensorDataType, cuuint32_t tensorRank, void* globalAddress, const cuuint64_t* globalDim, const cuuint64_t* globalStrides, int pixelBoxLowerCornerWidth, int pixelBoxUpperCornerWidth, cuuint32_t channelsPerPixel, cuuint32_t pixelsPerColumn, const cuuint32_t* elementStrides, CUtensorMapInterleave interleave, CUtensorMapIm2ColWideMode mode, CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, CUtensorMapFloatOOBfill oobFill) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTensorMapEncodeIm2colWide + cuPythonInit() + if __cuTensorMapEncodeIm2colWide == NULL: + with gil: + raise RuntimeError('Function "cuTensorMapEncodeIm2colWide" not found') + err = ( __cuTensorMapEncodeIm2colWide)(tensorMap, tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCornerWidth, pixelBoxUpperCornerWidth, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, mode, swizzle, l2Promotion, oobFill) + return err + +cdef CUresult _cuTensorMapReplaceAddress(CUtensorMap* tensorMap, void* globalAddress) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuTensorMapReplaceAddress + cuPythonInit() + if __cuTensorMapReplaceAddress == NULL: + with gil: + raise RuntimeError('Function "cuTensorMapReplaceAddress" not found') + err = ( __cuTensorMapReplaceAddress)(tensorMap, globalAddress) + return err + +cdef CUresult _cuDeviceCanAccessPeer(int* canAccessPeer, CUdevice dev, CUdevice peerDev) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceCanAccessPeer + cuPythonInit() + if __cuDeviceCanAccessPeer == NULL: + with gil: + raise RuntimeError('Function "cuDeviceCanAccessPeer" not found') + err = ( __cuDeviceCanAccessPeer)(canAccessPeer, dev, peerDev) + return err + +cdef CUresult _cuCtxEnablePeerAccess(CUcontext peerContext, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxEnablePeerAccess + cuPythonInit() + if __cuCtxEnablePeerAccess == NULL: + with gil: + raise RuntimeError('Function "cuCtxEnablePeerAccess" not found') + err = ( __cuCtxEnablePeerAccess)(peerContext, Flags) + return err + +cdef CUresult _cuCtxDisablePeerAccess(CUcontext peerContext) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxDisablePeerAccess + cuPythonInit() + if __cuCtxDisablePeerAccess == NULL: + with gil: + raise RuntimeError('Function "cuCtxDisablePeerAccess" not found') + err = ( __cuCtxDisablePeerAccess)(peerContext) + return err + +cdef CUresult _cuDeviceGetP2PAttribute(int* value, CUdevice_P2PAttribute attrib, CUdevice srcDevice, CUdevice dstDevice) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetP2PAttribute + cuPythonInit() + if __cuDeviceGetP2PAttribute == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetP2PAttribute" not found') + err = ( __cuDeviceGetP2PAttribute)(value, attrib, srcDevice, dstDevice) + return err + +cdef CUresult _cuGraphicsUnregisterResource(CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphicsUnregisterResource + cuPythonInit() + if __cuGraphicsUnregisterResource == NULL: + with gil: + raise RuntimeError('Function "cuGraphicsUnregisterResource" not found') + err = ( __cuGraphicsUnregisterResource)(resource) + return err + +cdef CUresult _cuGraphicsSubResourceGetMappedArray(CUarray* pArray, CUgraphicsResource resource, unsigned int arrayIndex, unsigned int mipLevel) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphicsSubResourceGetMappedArray + cuPythonInit() + if __cuGraphicsSubResourceGetMappedArray == NULL: + with gil: + raise RuntimeError('Function "cuGraphicsSubResourceGetMappedArray" not found') + err = ( __cuGraphicsSubResourceGetMappedArray)(pArray, resource, arrayIndex, mipLevel) + return err + +cdef CUresult _cuGraphicsResourceGetMappedMipmappedArray(CUmipmappedArray* pMipmappedArray, CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphicsResourceGetMappedMipmappedArray + cuPythonInit() + if __cuGraphicsResourceGetMappedMipmappedArray == NULL: + with gil: + raise RuntimeError('Function "cuGraphicsResourceGetMappedMipmappedArray" not found') + err = ( __cuGraphicsResourceGetMappedMipmappedArray)(pMipmappedArray, resource) + return err + +cdef CUresult _cuGraphicsResourceGetMappedPointer_v2(CUdeviceptr* pDevPtr, size_t* pSize, CUgraphicsResource resource) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphicsResourceGetMappedPointer_v2 + cuPythonInit() + if __cuGraphicsResourceGetMappedPointer_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGraphicsResourceGetMappedPointer_v2" not found') + err = ( __cuGraphicsResourceGetMappedPointer_v2)(pDevPtr, pSize, resource) + return err + +cdef CUresult _cuGraphicsResourceSetMapFlags_v2(CUgraphicsResource resource, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphicsResourceSetMapFlags_v2 + cuPythonInit() + if __cuGraphicsResourceSetMapFlags_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGraphicsResourceSetMapFlags_v2" not found') + err = ( __cuGraphicsResourceSetMapFlags_v2)(resource, flags) + return err + +cdef CUresult _cuGraphicsMapResources(unsigned int count, CUgraphicsResource* resources, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphicsMapResources + cuPythonInit() + if __cuGraphicsMapResources == NULL: + with gil: + raise RuntimeError('Function "cuGraphicsMapResources" not found') + err = ( __cuGraphicsMapResources)(count, resources, hStream) + return err + +cdef CUresult _cuGraphicsUnmapResources(unsigned int count, CUgraphicsResource* resources, CUstream hStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphicsUnmapResources + cuPythonInit() + if __cuGraphicsUnmapResources == NULL: + with gil: + raise RuntimeError('Function "cuGraphicsUnmapResources" not found') + err = ( __cuGraphicsUnmapResources)(count, resources, hStream) + return err + +cdef CUresult _cuGetProcAddress_v2(const char* symbol, void** pfn, int cudaVersion, cuuint64_t flags, CUdriverProcAddressQueryResult* symbolStatus) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGetProcAddress_v2 + cuPythonInit() + if __cuGetProcAddress_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGetProcAddress_v2" not found') + err = ( __cuGetProcAddress_v2)(symbol, pfn, cudaVersion, flags, symbolStatus) + return err + +cdef CUresult _cuCoredumpGetAttribute(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCoredumpGetAttribute + cuPythonInit() + if __cuCoredumpGetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuCoredumpGetAttribute" not found') + err = ( __cuCoredumpGetAttribute)(attrib, value, size) + return err + +cdef CUresult _cuCoredumpGetAttributeGlobal(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCoredumpGetAttributeGlobal + cuPythonInit() + if __cuCoredumpGetAttributeGlobal == NULL: + with gil: + raise RuntimeError('Function "cuCoredumpGetAttributeGlobal" not found') + err = ( __cuCoredumpGetAttributeGlobal)(attrib, value, size) + return err + +cdef CUresult _cuCoredumpSetAttribute(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCoredumpSetAttribute + cuPythonInit() + if __cuCoredumpSetAttribute == NULL: + with gil: + raise RuntimeError('Function "cuCoredumpSetAttribute" not found') + err = ( __cuCoredumpSetAttribute)(attrib, value, size) + return err + +cdef CUresult _cuCoredumpSetAttributeGlobal(CUcoredumpSettings attrib, void* value, size_t* size) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCoredumpSetAttributeGlobal + cuPythonInit() + if __cuCoredumpSetAttributeGlobal == NULL: + with gil: + raise RuntimeError('Function "cuCoredumpSetAttributeGlobal" not found') + err = ( __cuCoredumpSetAttributeGlobal)(attrib, value, size) + return err + +cdef CUresult _cuGetExportTable(const void** ppExportTable, const CUuuid* pExportTableId) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGetExportTable + cuPythonInit() + if __cuGetExportTable == NULL: + with gil: + raise RuntimeError('Function "cuGetExportTable" not found') + err = ( __cuGetExportTable)(ppExportTable, pExportTableId) + return err + +cdef CUresult _cuGreenCtxCreate(CUgreenCtx* phCtx, CUdevResourceDesc desc, CUdevice dev, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGreenCtxCreate + cuPythonInit() + if __cuGreenCtxCreate == NULL: + with gil: + raise RuntimeError('Function "cuGreenCtxCreate" not found') + err = ( __cuGreenCtxCreate)(phCtx, desc, dev, flags) + return err + +cdef CUresult _cuGreenCtxDestroy(CUgreenCtx hCtx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGreenCtxDestroy + cuPythonInit() + if __cuGreenCtxDestroy == NULL: + with gil: + raise RuntimeError('Function "cuGreenCtxDestroy" not found') + err = ( __cuGreenCtxDestroy)(hCtx) + return err + +cdef CUresult _cuCtxFromGreenCtx(CUcontext* pContext, CUgreenCtx hCtx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxFromGreenCtx + cuPythonInit() + if __cuCtxFromGreenCtx == NULL: + with gil: + raise RuntimeError('Function "cuCtxFromGreenCtx" not found') + err = ( __cuCtxFromGreenCtx)(pContext, hCtx) + return err + +cdef CUresult _cuDeviceGetDevResource(CUdevice device, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDeviceGetDevResource + cuPythonInit() + if __cuDeviceGetDevResource == NULL: + with gil: + raise RuntimeError('Function "cuDeviceGetDevResource" not found') + err = ( __cuDeviceGetDevResource)(device, resource, typename) + return err + +cdef CUresult _cuCtxGetDevResource(CUcontext hCtx, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCtxGetDevResource + cuPythonInit() + if __cuCtxGetDevResource == NULL: + with gil: + raise RuntimeError('Function "cuCtxGetDevResource" not found') + err = ( __cuCtxGetDevResource)(hCtx, resource, typename) + return err + +cdef CUresult _cuGreenCtxGetDevResource(CUgreenCtx hCtx, CUdevResource* resource, CUdevResourceType typename) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGreenCtxGetDevResource + cuPythonInit() + if __cuGreenCtxGetDevResource == NULL: + with gil: + raise RuntimeError('Function "cuGreenCtxGetDevResource" not found') + err = ( __cuGreenCtxGetDevResource)(hCtx, resource, typename) + return err + +cdef CUresult _cuDevSmResourceSplitByCount(CUdevResource* result, unsigned int* nbGroups, const CUdevResource* input, CUdevResource* remaining, unsigned int useFlags, unsigned int minCount) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDevSmResourceSplitByCount + cuPythonInit() + if __cuDevSmResourceSplitByCount == NULL: + with gil: + raise RuntimeError('Function "cuDevSmResourceSplitByCount" not found') + err = ( __cuDevSmResourceSplitByCount)(result, nbGroups, input, remaining, useFlags, minCount) + return err + +cdef CUresult _cuDevResourceGenerateDesc(CUdevResourceDesc* phDesc, CUdevResource* resources, unsigned int nbResources) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuDevResourceGenerateDesc + cuPythonInit() + if __cuDevResourceGenerateDesc == NULL: + with gil: + raise RuntimeError('Function "cuDevResourceGenerateDesc" not found') + err = ( __cuDevResourceGenerateDesc)(phDesc, resources, nbResources) + return err + +cdef CUresult _cuGreenCtxRecordEvent(CUgreenCtx hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGreenCtxRecordEvent + cuPythonInit() + if __cuGreenCtxRecordEvent == NULL: + with gil: + raise RuntimeError('Function "cuGreenCtxRecordEvent" not found') + err = ( __cuGreenCtxRecordEvent)(hCtx, hEvent) + return err + +cdef CUresult _cuGreenCtxWaitEvent(CUgreenCtx hCtx, CUevent hEvent) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGreenCtxWaitEvent + cuPythonInit() + if __cuGreenCtxWaitEvent == NULL: + with gil: + raise RuntimeError('Function "cuGreenCtxWaitEvent" not found') + err = ( __cuGreenCtxWaitEvent)(hCtx, hEvent) + return err + +cdef CUresult _cuStreamGetGreenCtx(CUstream hStream, CUgreenCtx* phCtx) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuStreamGetGreenCtx + cuPythonInit() + if __cuStreamGetGreenCtx == NULL: + with gil: + raise RuntimeError('Function "cuStreamGetGreenCtx" not found') + err = ( __cuStreamGetGreenCtx)(hStream, phCtx) + return err + +cdef CUresult _cuGreenCtxStreamCreate(CUstream* phStream, CUgreenCtx greenCtx, unsigned int flags, int priority) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGreenCtxStreamCreate + cuPythonInit() + if __cuGreenCtxStreamCreate == NULL: + with gil: + raise RuntimeError('Function "cuGreenCtxStreamCreate" not found') + err = ( __cuGreenCtxStreamCreate)(phStream, greenCtx, flags, priority) + return err + +cdef CUresult _cuLogsRegisterCallback(CUlogsCallback callbackFunc, void* userData, CUlogsCallbackHandle* callback_out) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLogsRegisterCallback + cuPythonInit() + if __cuLogsRegisterCallback == NULL: + with gil: + raise RuntimeError('Function "cuLogsRegisterCallback" not found') + err = ( __cuLogsRegisterCallback)(callbackFunc, userData, callback_out) + return err + +cdef CUresult _cuLogsUnregisterCallback(CUlogsCallbackHandle callback) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLogsUnregisterCallback + cuPythonInit() + if __cuLogsUnregisterCallback == NULL: + with gil: + raise RuntimeError('Function "cuLogsUnregisterCallback" not found') + err = ( __cuLogsUnregisterCallback)(callback) + return err + +cdef CUresult _cuLogsCurrent(CUlogIterator* iterator_out, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLogsCurrent + cuPythonInit() + if __cuLogsCurrent == NULL: + with gil: + raise RuntimeError('Function "cuLogsCurrent" not found') + err = ( __cuLogsCurrent)(iterator_out, flags) + return err + +cdef CUresult _cuLogsDumpToFile(CUlogIterator* iterator, const char* pathToFile, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLogsDumpToFile + cuPythonInit() + if __cuLogsDumpToFile == NULL: + with gil: + raise RuntimeError('Function "cuLogsDumpToFile" not found') + err = ( __cuLogsDumpToFile)(iterator, pathToFile, flags) + return err + +cdef CUresult _cuLogsDumpToMemory(CUlogIterator* iterator, char* buffer, size_t* size, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuLogsDumpToMemory + cuPythonInit() + if __cuLogsDumpToMemory == NULL: + with gil: + raise RuntimeError('Function "cuLogsDumpToMemory" not found') + err = ( __cuLogsDumpToMemory)(iterator, buffer, size, flags) + return err + +cdef CUresult _cuCheckpointProcessGetRestoreThreadId(int pid, int* tid) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCheckpointProcessGetRestoreThreadId + cuPythonInit() + if __cuCheckpointProcessGetRestoreThreadId == NULL: + with gil: + raise RuntimeError('Function "cuCheckpointProcessGetRestoreThreadId" not found') + err = ( __cuCheckpointProcessGetRestoreThreadId)(pid, tid) + return err + +cdef CUresult _cuCheckpointProcessGetState(int pid, CUprocessState* state) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCheckpointProcessGetState + cuPythonInit() + if __cuCheckpointProcessGetState == NULL: + with gil: + raise RuntimeError('Function "cuCheckpointProcessGetState" not found') + err = ( __cuCheckpointProcessGetState)(pid, state) + return err + +cdef CUresult _cuCheckpointProcessLock(int pid, CUcheckpointLockArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCheckpointProcessLock + cuPythonInit() + if __cuCheckpointProcessLock == NULL: + with gil: + raise RuntimeError('Function "cuCheckpointProcessLock" not found') + err = ( __cuCheckpointProcessLock)(pid, args) + return err + +cdef CUresult _cuCheckpointProcessCheckpoint(int pid, CUcheckpointCheckpointArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCheckpointProcessCheckpoint + cuPythonInit() + if __cuCheckpointProcessCheckpoint == NULL: + with gil: + raise RuntimeError('Function "cuCheckpointProcessCheckpoint" not found') + err = ( __cuCheckpointProcessCheckpoint)(pid, args) + return err + +cdef CUresult _cuCheckpointProcessRestore(int pid, CUcheckpointRestoreArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCheckpointProcessRestore + cuPythonInit() + if __cuCheckpointProcessRestore == NULL: + with gil: + raise RuntimeError('Function "cuCheckpointProcessRestore" not found') + err = ( __cuCheckpointProcessRestore)(pid, args) + return err + +cdef CUresult _cuCheckpointProcessUnlock(int pid, CUcheckpointUnlockArgs* args) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuCheckpointProcessUnlock + cuPythonInit() + if __cuCheckpointProcessUnlock == NULL: + with gil: + raise RuntimeError('Function "cuCheckpointProcessUnlock" not found') + err = ( __cuCheckpointProcessUnlock)(pid, args) + return err + +cdef CUresult _cuProfilerStart() except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuProfilerStart + cuPythonInit() + if __cuProfilerStart == NULL: + with gil: + raise RuntimeError('Function "cuProfilerStart" not found') + err = ( __cuProfilerStart)() + return err + +cdef CUresult _cuProfilerStop() except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuProfilerStop + cuPythonInit() + if __cuProfilerStop == NULL: + with gil: + raise RuntimeError('Function "cuProfilerStop" not found') + err = ( __cuProfilerStop)() + return err + +cdef CUresult _cuGraphicsEGLRegisterImage(CUgraphicsResource* pCudaResource, EGLImageKHR image, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphicsEGLRegisterImage + cuPythonInit() + if __cuGraphicsEGLRegisterImage == NULL: + with gil: + raise RuntimeError('Function "cuGraphicsEGLRegisterImage" not found') + err = ( __cuGraphicsEGLRegisterImage)(pCudaResource, image, flags) + return err + +cdef CUresult _cuEGLStreamConsumerConnect(CUeglStreamConnection* conn, EGLStreamKHR stream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEGLStreamConsumerConnect + cuPythonInit() + if __cuEGLStreamConsumerConnect == NULL: + with gil: + raise RuntimeError('Function "cuEGLStreamConsumerConnect" not found') + err = ( __cuEGLStreamConsumerConnect)(conn, stream) + return err + +cdef CUresult _cuEGLStreamConsumerConnectWithFlags(CUeglStreamConnection* conn, EGLStreamKHR stream, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEGLStreamConsumerConnectWithFlags + cuPythonInit() + if __cuEGLStreamConsumerConnectWithFlags == NULL: + with gil: + raise RuntimeError('Function "cuEGLStreamConsumerConnectWithFlags" not found') + err = ( __cuEGLStreamConsumerConnectWithFlags)(conn, stream, flags) + return err + +cdef CUresult _cuEGLStreamConsumerDisconnect(CUeglStreamConnection* conn) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEGLStreamConsumerDisconnect + cuPythonInit() + if __cuEGLStreamConsumerDisconnect == NULL: + with gil: + raise RuntimeError('Function "cuEGLStreamConsumerDisconnect" not found') + err = ( __cuEGLStreamConsumerDisconnect)(conn) + return err + +cdef CUresult _cuEGLStreamConsumerAcquireFrame(CUeglStreamConnection* conn, CUgraphicsResource* pCudaResource, CUstream* pStream, unsigned int timeout) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEGLStreamConsumerAcquireFrame + cuPythonInit() + if __cuEGLStreamConsumerAcquireFrame == NULL: + with gil: + raise RuntimeError('Function "cuEGLStreamConsumerAcquireFrame" not found') + err = ( __cuEGLStreamConsumerAcquireFrame)(conn, pCudaResource, pStream, timeout) + return err + +cdef CUresult _cuEGLStreamConsumerReleaseFrame(CUeglStreamConnection* conn, CUgraphicsResource pCudaResource, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEGLStreamConsumerReleaseFrame + cuPythonInit() + if __cuEGLStreamConsumerReleaseFrame == NULL: + with gil: + raise RuntimeError('Function "cuEGLStreamConsumerReleaseFrame" not found') + err = ( __cuEGLStreamConsumerReleaseFrame)(conn, pCudaResource, pStream) + return err + +cdef CUresult _cuEGLStreamProducerConnect(CUeglStreamConnection* conn, EGLStreamKHR stream, EGLint width, EGLint height) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEGLStreamProducerConnect + cuPythonInit() + if __cuEGLStreamProducerConnect == NULL: + with gil: + raise RuntimeError('Function "cuEGLStreamProducerConnect" not found') + err = ( __cuEGLStreamProducerConnect)(conn, stream, width, height) + return err + +cdef CUresult _cuEGLStreamProducerDisconnect(CUeglStreamConnection* conn) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEGLStreamProducerDisconnect + cuPythonInit() + if __cuEGLStreamProducerDisconnect == NULL: + with gil: + raise RuntimeError('Function "cuEGLStreamProducerDisconnect" not found') + err = ( __cuEGLStreamProducerDisconnect)(conn) + return err + +cdef CUresult _cuEGLStreamProducerPresentFrame(CUeglStreamConnection* conn, CUeglFrame eglframe, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEGLStreamProducerPresentFrame + cuPythonInit() + if __cuEGLStreamProducerPresentFrame == NULL: + with gil: + raise RuntimeError('Function "cuEGLStreamProducerPresentFrame" not found') + err = ( __cuEGLStreamProducerPresentFrame)(conn, eglframe, pStream) + return err + +cdef CUresult _cuEGLStreamProducerReturnFrame(CUeglStreamConnection* conn, CUeglFrame* eglframe, CUstream* pStream) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEGLStreamProducerReturnFrame + cuPythonInit() + if __cuEGLStreamProducerReturnFrame == NULL: + with gil: + raise RuntimeError('Function "cuEGLStreamProducerReturnFrame" not found') + err = ( __cuEGLStreamProducerReturnFrame)(conn, eglframe, pStream) + return err + +cdef CUresult _cuGraphicsResourceGetMappedEglFrame(CUeglFrame* eglFrame, CUgraphicsResource resource, unsigned int index, unsigned int mipLevel) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphicsResourceGetMappedEglFrame + cuPythonInit() + if __cuGraphicsResourceGetMappedEglFrame == NULL: + with gil: + raise RuntimeError('Function "cuGraphicsResourceGetMappedEglFrame" not found') + err = ( __cuGraphicsResourceGetMappedEglFrame)(eglFrame, resource, index, mipLevel) + return err + +cdef CUresult _cuEventCreateFromEGLSync(CUevent* phEvent, EGLSyncKHR eglSync, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuEventCreateFromEGLSync + cuPythonInit() + if __cuEventCreateFromEGLSync == NULL: + with gil: + raise RuntimeError('Function "cuEventCreateFromEGLSync" not found') + err = ( __cuEventCreateFromEGLSync)(phEvent, eglSync, flags) + return err + +cdef CUresult _cuGraphicsGLRegisterBuffer(CUgraphicsResource* pCudaResource, GLuint buffer, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphicsGLRegisterBuffer + cuPythonInit() + if __cuGraphicsGLRegisterBuffer == NULL: + with gil: + raise RuntimeError('Function "cuGraphicsGLRegisterBuffer" not found') + err = ( __cuGraphicsGLRegisterBuffer)(pCudaResource, buffer, Flags) + return err + +cdef CUresult _cuGraphicsGLRegisterImage(CUgraphicsResource* pCudaResource, GLuint image, GLenum target, unsigned int Flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphicsGLRegisterImage + cuPythonInit() + if __cuGraphicsGLRegisterImage == NULL: + with gil: + raise RuntimeError('Function "cuGraphicsGLRegisterImage" not found') + err = ( __cuGraphicsGLRegisterImage)(pCudaResource, image, target, Flags) + return err + +cdef CUresult _cuGLGetDevices_v2(unsigned int* pCudaDeviceCount, CUdevice* pCudaDevices, unsigned int cudaDeviceCount, CUGLDeviceList deviceList) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGLGetDevices_v2 + cuPythonInit() + if __cuGLGetDevices_v2 == NULL: + with gil: + raise RuntimeError('Function "cuGLGetDevices_v2" not found') + err = ( __cuGLGetDevices_v2)(pCudaDeviceCount, pCudaDevices, cudaDeviceCount, deviceList) + return err + +cdef CUresult _cuVDPAUGetDevice(CUdevice* pDevice, VdpDevice vdpDevice, VdpGetProcAddress* vdpGetProcAddress) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuVDPAUGetDevice + cuPythonInit() + if __cuVDPAUGetDevice == NULL: + with gil: + raise RuntimeError('Function "cuVDPAUGetDevice" not found') + err = ( __cuVDPAUGetDevice)(pDevice, vdpDevice, vdpGetProcAddress) + return err + +cdef CUresult _cuVDPAUCtxCreate_v2(CUcontext* pCtx, unsigned int flags, CUdevice device, VdpDevice vdpDevice, VdpGetProcAddress* vdpGetProcAddress) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuVDPAUCtxCreate_v2 + cuPythonInit() + if __cuVDPAUCtxCreate_v2 == NULL: + with gil: + raise RuntimeError('Function "cuVDPAUCtxCreate_v2" not found') + err = ( __cuVDPAUCtxCreate_v2)(pCtx, flags, device, vdpDevice, vdpGetProcAddress) + return err + +cdef CUresult _cuGraphicsVDPAURegisterVideoSurface(CUgraphicsResource* pCudaResource, VdpVideoSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphicsVDPAURegisterVideoSurface + cuPythonInit() + if __cuGraphicsVDPAURegisterVideoSurface == NULL: + with gil: + raise RuntimeError('Function "cuGraphicsVDPAURegisterVideoSurface" not found') + err = ( __cuGraphicsVDPAURegisterVideoSurface)(pCudaResource, vdpSurface, flags) + return err + +cdef CUresult _cuGraphicsVDPAURegisterOutputSurface(CUgraphicsResource* pCudaResource, VdpOutputSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil: + global __cuGraphicsVDPAURegisterOutputSurface + cuPythonInit() + if __cuGraphicsVDPAURegisterOutputSurface == NULL: + with gil: + raise RuntimeError('Function "cuGraphicsVDPAURegisterOutputSurface" not found') + err = ( __cuGraphicsVDPAURegisterOutputSurface)(pCudaResource, vdpSurface, flags) + return err + +cdef dict func_ptrs = None + +cpdef dict _inspect_function_pointers(): + global func_ptrs + if func_ptrs is not None: + return func_ptrs + + cuPythonInit() + cdef dict data = {} + + global __cuGetErrorString + data["__cuGetErrorString"] = __cuGetErrorString + global __cuGetErrorName + data["__cuGetErrorName"] = __cuGetErrorName + global __cuInit + data["__cuInit"] = __cuInit + global __cuDriverGetVersion + data["__cuDriverGetVersion"] = __cuDriverGetVersion + global __cuDeviceGet + data["__cuDeviceGet"] = __cuDeviceGet + global __cuDeviceGetCount + data["__cuDeviceGetCount"] = __cuDeviceGetCount + global __cuDeviceGetName + data["__cuDeviceGetName"] = __cuDeviceGetName + global __cuDeviceGetUuid + data["__cuDeviceGetUuid"] = __cuDeviceGetUuid + global __cuDeviceGetUuid_v2 + data["__cuDeviceGetUuid_v2"] = __cuDeviceGetUuid_v2 + global __cuDeviceGetLuid + data["__cuDeviceGetLuid"] = __cuDeviceGetLuid + global __cuDeviceTotalMem_v2 + data["__cuDeviceTotalMem_v2"] = __cuDeviceTotalMem_v2 + global __cuDeviceGetTexture1DLinearMaxWidth + data["__cuDeviceGetTexture1DLinearMaxWidth"] = __cuDeviceGetTexture1DLinearMaxWidth + global __cuDeviceGetAttribute + data["__cuDeviceGetAttribute"] = __cuDeviceGetAttribute + global __cuDeviceGetNvSciSyncAttributes + data["__cuDeviceGetNvSciSyncAttributes"] = __cuDeviceGetNvSciSyncAttributes + global __cuDeviceSetMemPool + data["__cuDeviceSetMemPool"] = __cuDeviceSetMemPool + global __cuDeviceGetMemPool + data["__cuDeviceGetMemPool"] = __cuDeviceGetMemPool + global __cuDeviceGetDefaultMemPool + data["__cuDeviceGetDefaultMemPool"] = __cuDeviceGetDefaultMemPool + global __cuDeviceGetExecAffinitySupport + data["__cuDeviceGetExecAffinitySupport"] = __cuDeviceGetExecAffinitySupport + global __cuFlushGPUDirectRDMAWrites + data["__cuFlushGPUDirectRDMAWrites"] = __cuFlushGPUDirectRDMAWrites + global __cuDeviceGetProperties + data["__cuDeviceGetProperties"] = __cuDeviceGetProperties + global __cuDeviceComputeCapability + data["__cuDeviceComputeCapability"] = __cuDeviceComputeCapability + global __cuDevicePrimaryCtxRetain + data["__cuDevicePrimaryCtxRetain"] = __cuDevicePrimaryCtxRetain + global __cuDevicePrimaryCtxRelease_v2 + data["__cuDevicePrimaryCtxRelease_v2"] = __cuDevicePrimaryCtxRelease_v2 + global __cuDevicePrimaryCtxSetFlags_v2 + data["__cuDevicePrimaryCtxSetFlags_v2"] = __cuDevicePrimaryCtxSetFlags_v2 + global __cuDevicePrimaryCtxGetState + data["__cuDevicePrimaryCtxGetState"] = __cuDevicePrimaryCtxGetState + global __cuDevicePrimaryCtxReset_v2 + data["__cuDevicePrimaryCtxReset_v2"] = __cuDevicePrimaryCtxReset_v2 + global __cuCtxCreate_v2 + data["__cuCtxCreate_v2"] = __cuCtxCreate_v2 + global __cuCtxCreate_v3 + data["__cuCtxCreate_v3"] = __cuCtxCreate_v3 + global __cuCtxCreate_v4 + data["__cuCtxCreate_v4"] = __cuCtxCreate_v4 + global __cuCtxDestroy_v2 + data["__cuCtxDestroy_v2"] = __cuCtxDestroy_v2 + global __cuCtxPushCurrent_v2 + data["__cuCtxPushCurrent_v2"] = __cuCtxPushCurrent_v2 + global __cuCtxPopCurrent_v2 + data["__cuCtxPopCurrent_v2"] = __cuCtxPopCurrent_v2 + global __cuCtxSetCurrent + data["__cuCtxSetCurrent"] = __cuCtxSetCurrent + global __cuCtxGetCurrent + data["__cuCtxGetCurrent"] = __cuCtxGetCurrent + global __cuCtxGetDevice + data["__cuCtxGetDevice"] = __cuCtxGetDevice + global __cuCtxGetFlags + data["__cuCtxGetFlags"] = __cuCtxGetFlags + global __cuCtxSetFlags + data["__cuCtxSetFlags"] = __cuCtxSetFlags + global __cuCtxGetId + data["__cuCtxGetId"] = __cuCtxGetId + global __cuCtxSynchronize + data["__cuCtxSynchronize"] = __cuCtxSynchronize + global __cuCtxSetLimit + data["__cuCtxSetLimit"] = __cuCtxSetLimit + global __cuCtxGetLimit + data["__cuCtxGetLimit"] = __cuCtxGetLimit + global __cuCtxGetCacheConfig + data["__cuCtxGetCacheConfig"] = __cuCtxGetCacheConfig + global __cuCtxSetCacheConfig + data["__cuCtxSetCacheConfig"] = __cuCtxSetCacheConfig + global __cuCtxGetApiVersion + data["__cuCtxGetApiVersion"] = __cuCtxGetApiVersion + global __cuCtxGetStreamPriorityRange + data["__cuCtxGetStreamPriorityRange"] = __cuCtxGetStreamPriorityRange + global __cuCtxResetPersistingL2Cache + data["__cuCtxResetPersistingL2Cache"] = __cuCtxResetPersistingL2Cache + global __cuCtxGetExecAffinity + data["__cuCtxGetExecAffinity"] = __cuCtxGetExecAffinity + global __cuCtxRecordEvent + data["__cuCtxRecordEvent"] = __cuCtxRecordEvent + global __cuCtxWaitEvent + data["__cuCtxWaitEvent"] = __cuCtxWaitEvent + global __cuCtxAttach + data["__cuCtxAttach"] = __cuCtxAttach + global __cuCtxDetach + data["__cuCtxDetach"] = __cuCtxDetach + global __cuCtxGetSharedMemConfig + data["__cuCtxGetSharedMemConfig"] = __cuCtxGetSharedMemConfig + global __cuCtxSetSharedMemConfig + data["__cuCtxSetSharedMemConfig"] = __cuCtxSetSharedMemConfig + global __cuModuleLoad + data["__cuModuleLoad"] = __cuModuleLoad + global __cuModuleLoadData + data["__cuModuleLoadData"] = __cuModuleLoadData + global __cuModuleLoadDataEx + data["__cuModuleLoadDataEx"] = __cuModuleLoadDataEx + global __cuModuleLoadFatBinary + data["__cuModuleLoadFatBinary"] = __cuModuleLoadFatBinary + global __cuModuleUnload + data["__cuModuleUnload"] = __cuModuleUnload + global __cuModuleGetLoadingMode + data["__cuModuleGetLoadingMode"] = __cuModuleGetLoadingMode + global __cuModuleGetFunction + data["__cuModuleGetFunction"] = __cuModuleGetFunction + global __cuModuleGetFunctionCount + data["__cuModuleGetFunctionCount"] = __cuModuleGetFunctionCount + global __cuModuleEnumerateFunctions + data["__cuModuleEnumerateFunctions"] = __cuModuleEnumerateFunctions + global __cuModuleGetGlobal_v2 + data["__cuModuleGetGlobal_v2"] = __cuModuleGetGlobal_v2 + global __cuLinkCreate_v2 + data["__cuLinkCreate_v2"] = __cuLinkCreate_v2 + global __cuLinkAddData_v2 + data["__cuLinkAddData_v2"] = __cuLinkAddData_v2 + global __cuLinkAddFile_v2 + data["__cuLinkAddFile_v2"] = __cuLinkAddFile_v2 + global __cuLinkComplete + data["__cuLinkComplete"] = __cuLinkComplete + global __cuLinkDestroy + data["__cuLinkDestroy"] = __cuLinkDestroy + global __cuModuleGetTexRef + data["__cuModuleGetTexRef"] = __cuModuleGetTexRef + global __cuModuleGetSurfRef + data["__cuModuleGetSurfRef"] = __cuModuleGetSurfRef + global __cuLibraryLoadData + data["__cuLibraryLoadData"] = __cuLibraryLoadData + global __cuLibraryLoadFromFile + data["__cuLibraryLoadFromFile"] = __cuLibraryLoadFromFile + global __cuLibraryUnload + data["__cuLibraryUnload"] = __cuLibraryUnload + global __cuLibraryGetKernel + data["__cuLibraryGetKernel"] = __cuLibraryGetKernel + global __cuLibraryGetKernelCount + data["__cuLibraryGetKernelCount"] = __cuLibraryGetKernelCount + global __cuLibraryEnumerateKernels + data["__cuLibraryEnumerateKernels"] = __cuLibraryEnumerateKernels + global __cuLibraryGetModule + data["__cuLibraryGetModule"] = __cuLibraryGetModule + global __cuKernelGetFunction + data["__cuKernelGetFunction"] = __cuKernelGetFunction + global __cuKernelGetLibrary + data["__cuKernelGetLibrary"] = __cuKernelGetLibrary + global __cuLibraryGetGlobal + data["__cuLibraryGetGlobal"] = __cuLibraryGetGlobal + global __cuLibraryGetManaged + data["__cuLibraryGetManaged"] = __cuLibraryGetManaged + global __cuLibraryGetUnifiedFunction + data["__cuLibraryGetUnifiedFunction"] = __cuLibraryGetUnifiedFunction + global __cuKernelGetAttribute + data["__cuKernelGetAttribute"] = __cuKernelGetAttribute + global __cuKernelSetAttribute + data["__cuKernelSetAttribute"] = __cuKernelSetAttribute + global __cuKernelSetCacheConfig + data["__cuKernelSetCacheConfig"] = __cuKernelSetCacheConfig + global __cuKernelGetName + data["__cuKernelGetName"] = __cuKernelGetName + global __cuKernelGetParamInfo + data["__cuKernelGetParamInfo"] = __cuKernelGetParamInfo + global __cuMemGetInfo_v2 + data["__cuMemGetInfo_v2"] = __cuMemGetInfo_v2 + global __cuMemAlloc_v2 + data["__cuMemAlloc_v2"] = __cuMemAlloc_v2 + global __cuMemAllocPitch_v2 + data["__cuMemAllocPitch_v2"] = __cuMemAllocPitch_v2 + global __cuMemFree_v2 + data["__cuMemFree_v2"] = __cuMemFree_v2 + global __cuMemGetAddressRange_v2 + data["__cuMemGetAddressRange_v2"] = __cuMemGetAddressRange_v2 + global __cuMemAllocHost_v2 + data["__cuMemAllocHost_v2"] = __cuMemAllocHost_v2 + global __cuMemFreeHost + data["__cuMemFreeHost"] = __cuMemFreeHost + global __cuMemHostAlloc + data["__cuMemHostAlloc"] = __cuMemHostAlloc + global __cuMemHostGetDevicePointer_v2 + data["__cuMemHostGetDevicePointer_v2"] = __cuMemHostGetDevicePointer_v2 + global __cuMemHostGetFlags + data["__cuMemHostGetFlags"] = __cuMemHostGetFlags + global __cuMemAllocManaged + data["__cuMemAllocManaged"] = __cuMemAllocManaged + global __cuDeviceRegisterAsyncNotification + data["__cuDeviceRegisterAsyncNotification"] = __cuDeviceRegisterAsyncNotification + global __cuDeviceUnregisterAsyncNotification + data["__cuDeviceUnregisterAsyncNotification"] = __cuDeviceUnregisterAsyncNotification + global __cuDeviceGetByPCIBusId + data["__cuDeviceGetByPCIBusId"] = __cuDeviceGetByPCIBusId + global __cuDeviceGetPCIBusId + data["__cuDeviceGetPCIBusId"] = __cuDeviceGetPCIBusId + global __cuIpcGetEventHandle + data["__cuIpcGetEventHandle"] = __cuIpcGetEventHandle + global __cuIpcOpenEventHandle + data["__cuIpcOpenEventHandle"] = __cuIpcOpenEventHandle + global __cuIpcGetMemHandle + data["__cuIpcGetMemHandle"] = __cuIpcGetMemHandle + global __cuIpcOpenMemHandle_v2 + data["__cuIpcOpenMemHandle_v2"] = __cuIpcOpenMemHandle_v2 + global __cuIpcCloseMemHandle + data["__cuIpcCloseMemHandle"] = __cuIpcCloseMemHandle + global __cuMemHostRegister_v2 + data["__cuMemHostRegister_v2"] = __cuMemHostRegister_v2 + global __cuMemHostUnregister + data["__cuMemHostUnregister"] = __cuMemHostUnregister + global __cuMemcpy + data["__cuMemcpy"] = __cuMemcpy + global __cuMemcpyPeer + data["__cuMemcpyPeer"] = __cuMemcpyPeer + global __cuMemcpyHtoD_v2 + data["__cuMemcpyHtoD_v2"] = __cuMemcpyHtoD_v2 + global __cuMemcpyDtoH_v2 + data["__cuMemcpyDtoH_v2"] = __cuMemcpyDtoH_v2 + global __cuMemcpyDtoD_v2 + data["__cuMemcpyDtoD_v2"] = __cuMemcpyDtoD_v2 + global __cuMemcpyDtoA_v2 + data["__cuMemcpyDtoA_v2"] = __cuMemcpyDtoA_v2 + global __cuMemcpyAtoD_v2 + data["__cuMemcpyAtoD_v2"] = __cuMemcpyAtoD_v2 + global __cuMemcpyHtoA_v2 + data["__cuMemcpyHtoA_v2"] = __cuMemcpyHtoA_v2 + global __cuMemcpyAtoH_v2 + data["__cuMemcpyAtoH_v2"] = __cuMemcpyAtoH_v2 + global __cuMemcpyAtoA_v2 + data["__cuMemcpyAtoA_v2"] = __cuMemcpyAtoA_v2 + global __cuMemcpy2D_v2 + data["__cuMemcpy2D_v2"] = __cuMemcpy2D_v2 + global __cuMemcpy2DUnaligned_v2 + data["__cuMemcpy2DUnaligned_v2"] = __cuMemcpy2DUnaligned_v2 + global __cuMemcpy3D_v2 + data["__cuMemcpy3D_v2"] = __cuMemcpy3D_v2 + global __cuMemcpy3DPeer + data["__cuMemcpy3DPeer"] = __cuMemcpy3DPeer + global __cuMemcpyAsync + data["__cuMemcpyAsync"] = __cuMemcpyAsync + global __cuMemcpyPeerAsync + data["__cuMemcpyPeerAsync"] = __cuMemcpyPeerAsync + global __cuMemcpyHtoDAsync_v2 + data["__cuMemcpyHtoDAsync_v2"] = __cuMemcpyHtoDAsync_v2 + global __cuMemcpyDtoHAsync_v2 + data["__cuMemcpyDtoHAsync_v2"] = __cuMemcpyDtoHAsync_v2 + global __cuMemcpyDtoDAsync_v2 + data["__cuMemcpyDtoDAsync_v2"] = __cuMemcpyDtoDAsync_v2 + global __cuMemcpyHtoAAsync_v2 + data["__cuMemcpyHtoAAsync_v2"] = __cuMemcpyHtoAAsync_v2 + global __cuMemcpyAtoHAsync_v2 + data["__cuMemcpyAtoHAsync_v2"] = __cuMemcpyAtoHAsync_v2 + global __cuMemcpy2DAsync_v2 + data["__cuMemcpy2DAsync_v2"] = __cuMemcpy2DAsync_v2 + global __cuMemcpy3DAsync_v2 + data["__cuMemcpy3DAsync_v2"] = __cuMemcpy3DAsync_v2 + global __cuMemcpy3DPeerAsync + data["__cuMemcpy3DPeerAsync"] = __cuMemcpy3DPeerAsync + global __cuMemcpyBatchAsync + data["__cuMemcpyBatchAsync"] = __cuMemcpyBatchAsync + global __cuMemcpy3DBatchAsync + data["__cuMemcpy3DBatchAsync"] = __cuMemcpy3DBatchAsync + global __cuMemsetD8_v2 + data["__cuMemsetD8_v2"] = __cuMemsetD8_v2 + global __cuMemsetD16_v2 + data["__cuMemsetD16_v2"] = __cuMemsetD16_v2 + global __cuMemsetD32_v2 + data["__cuMemsetD32_v2"] = __cuMemsetD32_v2 + global __cuMemsetD2D8_v2 + data["__cuMemsetD2D8_v2"] = __cuMemsetD2D8_v2 + global __cuMemsetD2D16_v2 + data["__cuMemsetD2D16_v2"] = __cuMemsetD2D16_v2 + global __cuMemsetD2D32_v2 + data["__cuMemsetD2D32_v2"] = __cuMemsetD2D32_v2 + global __cuMemsetD8Async + data["__cuMemsetD8Async"] = __cuMemsetD8Async + global __cuMemsetD16Async + data["__cuMemsetD16Async"] = __cuMemsetD16Async + global __cuMemsetD32Async + data["__cuMemsetD32Async"] = __cuMemsetD32Async + global __cuMemsetD2D8Async + data["__cuMemsetD2D8Async"] = __cuMemsetD2D8Async + global __cuMemsetD2D16Async + data["__cuMemsetD2D16Async"] = __cuMemsetD2D16Async + global __cuMemsetD2D32Async + data["__cuMemsetD2D32Async"] = __cuMemsetD2D32Async + global __cuArrayCreate_v2 + data["__cuArrayCreate_v2"] = __cuArrayCreate_v2 + global __cuArrayGetDescriptor_v2 + data["__cuArrayGetDescriptor_v2"] = __cuArrayGetDescriptor_v2 + global __cuArrayGetSparseProperties + data["__cuArrayGetSparseProperties"] = __cuArrayGetSparseProperties + global __cuMipmappedArrayGetSparseProperties + data["__cuMipmappedArrayGetSparseProperties"] = __cuMipmappedArrayGetSparseProperties + global __cuArrayGetMemoryRequirements + data["__cuArrayGetMemoryRequirements"] = __cuArrayGetMemoryRequirements + global __cuMipmappedArrayGetMemoryRequirements + data["__cuMipmappedArrayGetMemoryRequirements"] = __cuMipmappedArrayGetMemoryRequirements + global __cuArrayGetPlane + data["__cuArrayGetPlane"] = __cuArrayGetPlane + global __cuArrayDestroy + data["__cuArrayDestroy"] = __cuArrayDestroy + global __cuArray3DCreate_v2 + data["__cuArray3DCreate_v2"] = __cuArray3DCreate_v2 + global __cuArray3DGetDescriptor_v2 + data["__cuArray3DGetDescriptor_v2"] = __cuArray3DGetDescriptor_v2 + global __cuMipmappedArrayCreate + data["__cuMipmappedArrayCreate"] = __cuMipmappedArrayCreate + global __cuMipmappedArrayGetLevel + data["__cuMipmappedArrayGetLevel"] = __cuMipmappedArrayGetLevel + global __cuMipmappedArrayDestroy + data["__cuMipmappedArrayDestroy"] = __cuMipmappedArrayDestroy + global __cuMemGetHandleForAddressRange + data["__cuMemGetHandleForAddressRange"] = __cuMemGetHandleForAddressRange + global __cuMemBatchDecompressAsync + data["__cuMemBatchDecompressAsync"] = __cuMemBatchDecompressAsync + global __cuMemAddressReserve + data["__cuMemAddressReserve"] = __cuMemAddressReserve + global __cuMemAddressFree + data["__cuMemAddressFree"] = __cuMemAddressFree + global __cuMemCreate + data["__cuMemCreate"] = __cuMemCreate + global __cuMemRelease + data["__cuMemRelease"] = __cuMemRelease + global __cuMemMap + data["__cuMemMap"] = __cuMemMap + global __cuMemMapArrayAsync + data["__cuMemMapArrayAsync"] = __cuMemMapArrayAsync + global __cuMemUnmap + data["__cuMemUnmap"] = __cuMemUnmap + global __cuMemSetAccess + data["__cuMemSetAccess"] = __cuMemSetAccess + global __cuMemGetAccess + data["__cuMemGetAccess"] = __cuMemGetAccess + global __cuMemExportToShareableHandle + data["__cuMemExportToShareableHandle"] = __cuMemExportToShareableHandle + global __cuMemImportFromShareableHandle + data["__cuMemImportFromShareableHandle"] = __cuMemImportFromShareableHandle + global __cuMemGetAllocationGranularity + data["__cuMemGetAllocationGranularity"] = __cuMemGetAllocationGranularity + global __cuMemGetAllocationPropertiesFromHandle + data["__cuMemGetAllocationPropertiesFromHandle"] = __cuMemGetAllocationPropertiesFromHandle + global __cuMemRetainAllocationHandle + data["__cuMemRetainAllocationHandle"] = __cuMemRetainAllocationHandle + global __cuMemFreeAsync + data["__cuMemFreeAsync"] = __cuMemFreeAsync + global __cuMemAllocAsync + data["__cuMemAllocAsync"] = __cuMemAllocAsync + global __cuMemPoolTrimTo + data["__cuMemPoolTrimTo"] = __cuMemPoolTrimTo + global __cuMemPoolSetAttribute + data["__cuMemPoolSetAttribute"] = __cuMemPoolSetAttribute + global __cuMemPoolGetAttribute + data["__cuMemPoolGetAttribute"] = __cuMemPoolGetAttribute + global __cuMemPoolSetAccess + data["__cuMemPoolSetAccess"] = __cuMemPoolSetAccess + global __cuMemPoolGetAccess + data["__cuMemPoolGetAccess"] = __cuMemPoolGetAccess + global __cuMemPoolCreate + data["__cuMemPoolCreate"] = __cuMemPoolCreate + global __cuMemPoolDestroy + data["__cuMemPoolDestroy"] = __cuMemPoolDestroy + global __cuMemAllocFromPoolAsync + data["__cuMemAllocFromPoolAsync"] = __cuMemAllocFromPoolAsync + global __cuMemPoolExportToShareableHandle + data["__cuMemPoolExportToShareableHandle"] = __cuMemPoolExportToShareableHandle + global __cuMemPoolImportFromShareableHandle + data["__cuMemPoolImportFromShareableHandle"] = __cuMemPoolImportFromShareableHandle + global __cuMemPoolExportPointer + data["__cuMemPoolExportPointer"] = __cuMemPoolExportPointer + global __cuMemPoolImportPointer + data["__cuMemPoolImportPointer"] = __cuMemPoolImportPointer + global __cuMulticastCreate + data["__cuMulticastCreate"] = __cuMulticastCreate + global __cuMulticastAddDevice + data["__cuMulticastAddDevice"] = __cuMulticastAddDevice + global __cuMulticastBindMem + data["__cuMulticastBindMem"] = __cuMulticastBindMem + global __cuMulticastBindAddr + data["__cuMulticastBindAddr"] = __cuMulticastBindAddr + global __cuMulticastUnbind + data["__cuMulticastUnbind"] = __cuMulticastUnbind + global __cuMulticastGetGranularity + data["__cuMulticastGetGranularity"] = __cuMulticastGetGranularity + global __cuPointerGetAttribute + data["__cuPointerGetAttribute"] = __cuPointerGetAttribute + global __cuMemPrefetchAsync + data["__cuMemPrefetchAsync"] = __cuMemPrefetchAsync + global __cuMemPrefetchAsync_v2 + data["__cuMemPrefetchAsync_v2"] = __cuMemPrefetchAsync_v2 + global __cuMemAdvise + data["__cuMemAdvise"] = __cuMemAdvise + global __cuMemAdvise_v2 + data["__cuMemAdvise_v2"] = __cuMemAdvise_v2 + global __cuMemRangeGetAttribute + data["__cuMemRangeGetAttribute"] = __cuMemRangeGetAttribute + global __cuMemRangeGetAttributes + data["__cuMemRangeGetAttributes"] = __cuMemRangeGetAttributes + global __cuPointerSetAttribute + data["__cuPointerSetAttribute"] = __cuPointerSetAttribute + global __cuPointerGetAttributes + data["__cuPointerGetAttributes"] = __cuPointerGetAttributes + global __cuStreamCreate + data["__cuStreamCreate"] = __cuStreamCreate + global __cuStreamCreateWithPriority + data["__cuStreamCreateWithPriority"] = __cuStreamCreateWithPriority + global __cuStreamGetPriority + data["__cuStreamGetPriority"] = __cuStreamGetPriority + global __cuStreamGetDevice + data["__cuStreamGetDevice"] = __cuStreamGetDevice + global __cuStreamGetFlags + data["__cuStreamGetFlags"] = __cuStreamGetFlags + global __cuStreamGetId + data["__cuStreamGetId"] = __cuStreamGetId + global __cuStreamGetCtx + data["__cuStreamGetCtx"] = __cuStreamGetCtx + global __cuStreamGetCtx_v2 + data["__cuStreamGetCtx_v2"] = __cuStreamGetCtx_v2 + global __cuStreamWaitEvent + data["__cuStreamWaitEvent"] = __cuStreamWaitEvent + global __cuStreamAddCallback + data["__cuStreamAddCallback"] = __cuStreamAddCallback + global __cuStreamBeginCapture_v2 + data["__cuStreamBeginCapture_v2"] = __cuStreamBeginCapture_v2 + global __cuStreamBeginCaptureToGraph + data["__cuStreamBeginCaptureToGraph"] = __cuStreamBeginCaptureToGraph + global __cuThreadExchangeStreamCaptureMode + data["__cuThreadExchangeStreamCaptureMode"] = __cuThreadExchangeStreamCaptureMode + global __cuStreamEndCapture + data["__cuStreamEndCapture"] = __cuStreamEndCapture + global __cuStreamIsCapturing + data["__cuStreamIsCapturing"] = __cuStreamIsCapturing + global __cuStreamGetCaptureInfo_v2 + data["__cuStreamGetCaptureInfo_v2"] = __cuStreamGetCaptureInfo_v2 + global __cuStreamGetCaptureInfo_v3 + data["__cuStreamGetCaptureInfo_v3"] = __cuStreamGetCaptureInfo_v3 + global __cuStreamUpdateCaptureDependencies + data["__cuStreamUpdateCaptureDependencies"] = __cuStreamUpdateCaptureDependencies + global __cuStreamUpdateCaptureDependencies_v2 + data["__cuStreamUpdateCaptureDependencies_v2"] = __cuStreamUpdateCaptureDependencies_v2 + global __cuStreamAttachMemAsync + data["__cuStreamAttachMemAsync"] = __cuStreamAttachMemAsync + global __cuStreamQuery + data["__cuStreamQuery"] = __cuStreamQuery + global __cuStreamSynchronize + data["__cuStreamSynchronize"] = __cuStreamSynchronize + global __cuStreamDestroy_v2 + data["__cuStreamDestroy_v2"] = __cuStreamDestroy_v2 + global __cuStreamCopyAttributes + data["__cuStreamCopyAttributes"] = __cuStreamCopyAttributes + global __cuStreamGetAttribute + data["__cuStreamGetAttribute"] = __cuStreamGetAttribute + global __cuStreamSetAttribute + data["__cuStreamSetAttribute"] = __cuStreamSetAttribute + global __cuEventCreate + data["__cuEventCreate"] = __cuEventCreate + global __cuEventRecord + data["__cuEventRecord"] = __cuEventRecord + global __cuEventRecordWithFlags + data["__cuEventRecordWithFlags"] = __cuEventRecordWithFlags + global __cuEventQuery + data["__cuEventQuery"] = __cuEventQuery + global __cuEventSynchronize + data["__cuEventSynchronize"] = __cuEventSynchronize + global __cuEventDestroy_v2 + data["__cuEventDestroy_v2"] = __cuEventDestroy_v2 + global __cuEventElapsedTime + data["__cuEventElapsedTime"] = __cuEventElapsedTime + global __cuEventElapsedTime_v2 + data["__cuEventElapsedTime_v2"] = __cuEventElapsedTime_v2 + global __cuImportExternalMemory + data["__cuImportExternalMemory"] = __cuImportExternalMemory + global __cuExternalMemoryGetMappedBuffer + data["__cuExternalMemoryGetMappedBuffer"] = __cuExternalMemoryGetMappedBuffer + global __cuExternalMemoryGetMappedMipmappedArray + data["__cuExternalMemoryGetMappedMipmappedArray"] = __cuExternalMemoryGetMappedMipmappedArray + global __cuDestroyExternalMemory + data["__cuDestroyExternalMemory"] = __cuDestroyExternalMemory + global __cuImportExternalSemaphore + data["__cuImportExternalSemaphore"] = __cuImportExternalSemaphore + global __cuSignalExternalSemaphoresAsync + data["__cuSignalExternalSemaphoresAsync"] = __cuSignalExternalSemaphoresAsync + global __cuWaitExternalSemaphoresAsync + data["__cuWaitExternalSemaphoresAsync"] = __cuWaitExternalSemaphoresAsync + global __cuDestroyExternalSemaphore + data["__cuDestroyExternalSemaphore"] = __cuDestroyExternalSemaphore + global __cuStreamWaitValue32_v2 + data["__cuStreamWaitValue32_v2"] = __cuStreamWaitValue32_v2 + global __cuStreamWaitValue64_v2 + data["__cuStreamWaitValue64_v2"] = __cuStreamWaitValue64_v2 + global __cuStreamWriteValue32_v2 + data["__cuStreamWriteValue32_v2"] = __cuStreamWriteValue32_v2 + global __cuStreamWriteValue64_v2 + data["__cuStreamWriteValue64_v2"] = __cuStreamWriteValue64_v2 + global __cuStreamBatchMemOp_v2 + data["__cuStreamBatchMemOp_v2"] = __cuStreamBatchMemOp_v2 + global __cuFuncGetAttribute + data["__cuFuncGetAttribute"] = __cuFuncGetAttribute + global __cuFuncSetAttribute + data["__cuFuncSetAttribute"] = __cuFuncSetAttribute + global __cuFuncSetCacheConfig + data["__cuFuncSetCacheConfig"] = __cuFuncSetCacheConfig + global __cuFuncGetModule + data["__cuFuncGetModule"] = __cuFuncGetModule + global __cuFuncGetName + data["__cuFuncGetName"] = __cuFuncGetName + global __cuFuncGetParamInfo + data["__cuFuncGetParamInfo"] = __cuFuncGetParamInfo + global __cuFuncIsLoaded + data["__cuFuncIsLoaded"] = __cuFuncIsLoaded + global __cuFuncLoad + data["__cuFuncLoad"] = __cuFuncLoad + global __cuLaunchKernel + data["__cuLaunchKernel"] = __cuLaunchKernel + global __cuLaunchKernelEx + data["__cuLaunchKernelEx"] = __cuLaunchKernelEx + global __cuLaunchCooperativeKernel + data["__cuLaunchCooperativeKernel"] = __cuLaunchCooperativeKernel + global __cuLaunchCooperativeKernelMultiDevice + data["__cuLaunchCooperativeKernelMultiDevice"] = __cuLaunchCooperativeKernelMultiDevice + global __cuLaunchHostFunc + data["__cuLaunchHostFunc"] = __cuLaunchHostFunc + global __cuFuncSetBlockShape + data["__cuFuncSetBlockShape"] = __cuFuncSetBlockShape + global __cuFuncSetSharedSize + data["__cuFuncSetSharedSize"] = __cuFuncSetSharedSize + global __cuParamSetSize + data["__cuParamSetSize"] = __cuParamSetSize + global __cuParamSeti + data["__cuParamSeti"] = __cuParamSeti + global __cuParamSetf + data["__cuParamSetf"] = __cuParamSetf + global __cuParamSetv + data["__cuParamSetv"] = __cuParamSetv + global __cuLaunch + data["__cuLaunch"] = __cuLaunch + global __cuLaunchGrid + data["__cuLaunchGrid"] = __cuLaunchGrid + global __cuLaunchGridAsync + data["__cuLaunchGridAsync"] = __cuLaunchGridAsync + global __cuParamSetTexRef + data["__cuParamSetTexRef"] = __cuParamSetTexRef + global __cuFuncSetSharedMemConfig + data["__cuFuncSetSharedMemConfig"] = __cuFuncSetSharedMemConfig + global __cuGraphCreate + data["__cuGraphCreate"] = __cuGraphCreate + global __cuGraphAddKernelNode_v2 + data["__cuGraphAddKernelNode_v2"] = __cuGraphAddKernelNode_v2 + global __cuGraphKernelNodeGetParams_v2 + data["__cuGraphKernelNodeGetParams_v2"] = __cuGraphKernelNodeGetParams_v2 + global __cuGraphKernelNodeSetParams_v2 + data["__cuGraphKernelNodeSetParams_v2"] = __cuGraphKernelNodeSetParams_v2 + global __cuGraphAddMemcpyNode + data["__cuGraphAddMemcpyNode"] = __cuGraphAddMemcpyNode + global __cuGraphMemcpyNodeGetParams + data["__cuGraphMemcpyNodeGetParams"] = __cuGraphMemcpyNodeGetParams + global __cuGraphMemcpyNodeSetParams + data["__cuGraphMemcpyNodeSetParams"] = __cuGraphMemcpyNodeSetParams + global __cuGraphAddMemsetNode + data["__cuGraphAddMemsetNode"] = __cuGraphAddMemsetNode + global __cuGraphMemsetNodeGetParams + data["__cuGraphMemsetNodeGetParams"] = __cuGraphMemsetNodeGetParams + global __cuGraphMemsetNodeSetParams + data["__cuGraphMemsetNodeSetParams"] = __cuGraphMemsetNodeSetParams + global __cuGraphAddHostNode + data["__cuGraphAddHostNode"] = __cuGraphAddHostNode + global __cuGraphHostNodeGetParams + data["__cuGraphHostNodeGetParams"] = __cuGraphHostNodeGetParams + global __cuGraphHostNodeSetParams + data["__cuGraphHostNodeSetParams"] = __cuGraphHostNodeSetParams + global __cuGraphAddChildGraphNode + data["__cuGraphAddChildGraphNode"] = __cuGraphAddChildGraphNode + global __cuGraphChildGraphNodeGetGraph + data["__cuGraphChildGraphNodeGetGraph"] = __cuGraphChildGraphNodeGetGraph + global __cuGraphAddEmptyNode + data["__cuGraphAddEmptyNode"] = __cuGraphAddEmptyNode + global __cuGraphAddEventRecordNode + data["__cuGraphAddEventRecordNode"] = __cuGraphAddEventRecordNode + global __cuGraphEventRecordNodeGetEvent + data["__cuGraphEventRecordNodeGetEvent"] = __cuGraphEventRecordNodeGetEvent + global __cuGraphEventRecordNodeSetEvent + data["__cuGraphEventRecordNodeSetEvent"] = __cuGraphEventRecordNodeSetEvent + global __cuGraphAddEventWaitNode + data["__cuGraphAddEventWaitNode"] = __cuGraphAddEventWaitNode + global __cuGraphEventWaitNodeGetEvent + data["__cuGraphEventWaitNodeGetEvent"] = __cuGraphEventWaitNodeGetEvent + global __cuGraphEventWaitNodeSetEvent + data["__cuGraphEventWaitNodeSetEvent"] = __cuGraphEventWaitNodeSetEvent + global __cuGraphAddExternalSemaphoresSignalNode + data["__cuGraphAddExternalSemaphoresSignalNode"] = __cuGraphAddExternalSemaphoresSignalNode + global __cuGraphExternalSemaphoresSignalNodeGetParams + data["__cuGraphExternalSemaphoresSignalNodeGetParams"] = __cuGraphExternalSemaphoresSignalNodeGetParams + global __cuGraphExternalSemaphoresSignalNodeSetParams + data["__cuGraphExternalSemaphoresSignalNodeSetParams"] = __cuGraphExternalSemaphoresSignalNodeSetParams + global __cuGraphAddExternalSemaphoresWaitNode + data["__cuGraphAddExternalSemaphoresWaitNode"] = __cuGraphAddExternalSemaphoresWaitNode + global __cuGraphExternalSemaphoresWaitNodeGetParams + data["__cuGraphExternalSemaphoresWaitNodeGetParams"] = __cuGraphExternalSemaphoresWaitNodeGetParams + global __cuGraphExternalSemaphoresWaitNodeSetParams + data["__cuGraphExternalSemaphoresWaitNodeSetParams"] = __cuGraphExternalSemaphoresWaitNodeSetParams + global __cuGraphAddBatchMemOpNode + data["__cuGraphAddBatchMemOpNode"] = __cuGraphAddBatchMemOpNode + global __cuGraphBatchMemOpNodeGetParams + data["__cuGraphBatchMemOpNodeGetParams"] = __cuGraphBatchMemOpNodeGetParams + global __cuGraphBatchMemOpNodeSetParams + data["__cuGraphBatchMemOpNodeSetParams"] = __cuGraphBatchMemOpNodeSetParams + global __cuGraphExecBatchMemOpNodeSetParams + data["__cuGraphExecBatchMemOpNodeSetParams"] = __cuGraphExecBatchMemOpNodeSetParams + global __cuGraphAddMemAllocNode + data["__cuGraphAddMemAllocNode"] = __cuGraphAddMemAllocNode + global __cuGraphMemAllocNodeGetParams + data["__cuGraphMemAllocNodeGetParams"] = __cuGraphMemAllocNodeGetParams + global __cuGraphAddMemFreeNode + data["__cuGraphAddMemFreeNode"] = __cuGraphAddMemFreeNode + global __cuGraphMemFreeNodeGetParams + data["__cuGraphMemFreeNodeGetParams"] = __cuGraphMemFreeNodeGetParams + global __cuDeviceGraphMemTrim + data["__cuDeviceGraphMemTrim"] = __cuDeviceGraphMemTrim + global __cuDeviceGetGraphMemAttribute + data["__cuDeviceGetGraphMemAttribute"] = __cuDeviceGetGraphMemAttribute + global __cuDeviceSetGraphMemAttribute + data["__cuDeviceSetGraphMemAttribute"] = __cuDeviceSetGraphMemAttribute + global __cuGraphClone + data["__cuGraphClone"] = __cuGraphClone + global __cuGraphNodeFindInClone + data["__cuGraphNodeFindInClone"] = __cuGraphNodeFindInClone + global __cuGraphNodeGetType + data["__cuGraphNodeGetType"] = __cuGraphNodeGetType + global __cuGraphGetNodes + data["__cuGraphGetNodes"] = __cuGraphGetNodes + global __cuGraphGetRootNodes + data["__cuGraphGetRootNodes"] = __cuGraphGetRootNodes + global __cuGraphGetEdges + data["__cuGraphGetEdges"] = __cuGraphGetEdges + global __cuGraphGetEdges_v2 + data["__cuGraphGetEdges_v2"] = __cuGraphGetEdges_v2 + global __cuGraphNodeGetDependencies + data["__cuGraphNodeGetDependencies"] = __cuGraphNodeGetDependencies + global __cuGraphNodeGetDependencies_v2 + data["__cuGraphNodeGetDependencies_v2"] = __cuGraphNodeGetDependencies_v2 + global __cuGraphNodeGetDependentNodes + data["__cuGraphNodeGetDependentNodes"] = __cuGraphNodeGetDependentNodes + global __cuGraphNodeGetDependentNodes_v2 + data["__cuGraphNodeGetDependentNodes_v2"] = __cuGraphNodeGetDependentNodes_v2 + global __cuGraphAddDependencies + data["__cuGraphAddDependencies"] = __cuGraphAddDependencies + global __cuGraphAddDependencies_v2 + data["__cuGraphAddDependencies_v2"] = __cuGraphAddDependencies_v2 + global __cuGraphRemoveDependencies + data["__cuGraphRemoveDependencies"] = __cuGraphRemoveDependencies + global __cuGraphRemoveDependencies_v2 + data["__cuGraphRemoveDependencies_v2"] = __cuGraphRemoveDependencies_v2 + global __cuGraphDestroyNode + data["__cuGraphDestroyNode"] = __cuGraphDestroyNode + global __cuGraphInstantiateWithFlags + data["__cuGraphInstantiateWithFlags"] = __cuGraphInstantiateWithFlags + global __cuGraphInstantiateWithParams + data["__cuGraphInstantiateWithParams"] = __cuGraphInstantiateWithParams + global __cuGraphExecGetFlags + data["__cuGraphExecGetFlags"] = __cuGraphExecGetFlags + global __cuGraphExecKernelNodeSetParams_v2 + data["__cuGraphExecKernelNodeSetParams_v2"] = __cuGraphExecKernelNodeSetParams_v2 + global __cuGraphExecMemcpyNodeSetParams + data["__cuGraphExecMemcpyNodeSetParams"] = __cuGraphExecMemcpyNodeSetParams + global __cuGraphExecMemsetNodeSetParams + data["__cuGraphExecMemsetNodeSetParams"] = __cuGraphExecMemsetNodeSetParams + global __cuGraphExecHostNodeSetParams + data["__cuGraphExecHostNodeSetParams"] = __cuGraphExecHostNodeSetParams + global __cuGraphExecChildGraphNodeSetParams + data["__cuGraphExecChildGraphNodeSetParams"] = __cuGraphExecChildGraphNodeSetParams + global __cuGraphExecEventRecordNodeSetEvent + data["__cuGraphExecEventRecordNodeSetEvent"] = __cuGraphExecEventRecordNodeSetEvent + global __cuGraphExecEventWaitNodeSetEvent + data["__cuGraphExecEventWaitNodeSetEvent"] = __cuGraphExecEventWaitNodeSetEvent + global __cuGraphExecExternalSemaphoresSignalNodeSetParams + data["__cuGraphExecExternalSemaphoresSignalNodeSetParams"] = __cuGraphExecExternalSemaphoresSignalNodeSetParams + global __cuGraphExecExternalSemaphoresWaitNodeSetParams + data["__cuGraphExecExternalSemaphoresWaitNodeSetParams"] = __cuGraphExecExternalSemaphoresWaitNodeSetParams + global __cuGraphNodeSetEnabled + data["__cuGraphNodeSetEnabled"] = __cuGraphNodeSetEnabled + global __cuGraphNodeGetEnabled + data["__cuGraphNodeGetEnabled"] = __cuGraphNodeGetEnabled + global __cuGraphUpload + data["__cuGraphUpload"] = __cuGraphUpload + global __cuGraphLaunch + data["__cuGraphLaunch"] = __cuGraphLaunch + global __cuGraphExecDestroy + data["__cuGraphExecDestroy"] = __cuGraphExecDestroy + global __cuGraphDestroy + data["__cuGraphDestroy"] = __cuGraphDestroy + global __cuGraphExecUpdate_v2 + data["__cuGraphExecUpdate_v2"] = __cuGraphExecUpdate_v2 + global __cuGraphKernelNodeCopyAttributes + data["__cuGraphKernelNodeCopyAttributes"] = __cuGraphKernelNodeCopyAttributes + global __cuGraphKernelNodeGetAttribute + data["__cuGraphKernelNodeGetAttribute"] = __cuGraphKernelNodeGetAttribute + global __cuGraphKernelNodeSetAttribute + data["__cuGraphKernelNodeSetAttribute"] = __cuGraphKernelNodeSetAttribute + global __cuGraphDebugDotPrint + data["__cuGraphDebugDotPrint"] = __cuGraphDebugDotPrint + global __cuUserObjectCreate + data["__cuUserObjectCreate"] = __cuUserObjectCreate + global __cuUserObjectRetain + data["__cuUserObjectRetain"] = __cuUserObjectRetain + global __cuUserObjectRelease + data["__cuUserObjectRelease"] = __cuUserObjectRelease + global __cuGraphRetainUserObject + data["__cuGraphRetainUserObject"] = __cuGraphRetainUserObject + global __cuGraphReleaseUserObject + data["__cuGraphReleaseUserObject"] = __cuGraphReleaseUserObject + global __cuGraphAddNode + data["__cuGraphAddNode"] = __cuGraphAddNode + global __cuGraphAddNode_v2 + data["__cuGraphAddNode_v2"] = __cuGraphAddNode_v2 + global __cuGraphNodeSetParams + data["__cuGraphNodeSetParams"] = __cuGraphNodeSetParams + global __cuGraphExecNodeSetParams + data["__cuGraphExecNodeSetParams"] = __cuGraphExecNodeSetParams + global __cuGraphConditionalHandleCreate + data["__cuGraphConditionalHandleCreate"] = __cuGraphConditionalHandleCreate + global __cuOccupancyMaxActiveBlocksPerMultiprocessor + data["__cuOccupancyMaxActiveBlocksPerMultiprocessor"] = __cuOccupancyMaxActiveBlocksPerMultiprocessor + global __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + data["__cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags"] = __cuOccupancyMaxActiveBlocksPerMultiprocessorWithFlags + global __cuOccupancyMaxPotentialBlockSize + data["__cuOccupancyMaxPotentialBlockSize"] = __cuOccupancyMaxPotentialBlockSize + global __cuOccupancyMaxPotentialBlockSizeWithFlags + data["__cuOccupancyMaxPotentialBlockSizeWithFlags"] = __cuOccupancyMaxPotentialBlockSizeWithFlags + global __cuOccupancyAvailableDynamicSMemPerBlock + data["__cuOccupancyAvailableDynamicSMemPerBlock"] = __cuOccupancyAvailableDynamicSMemPerBlock + global __cuOccupancyMaxPotentialClusterSize + data["__cuOccupancyMaxPotentialClusterSize"] = __cuOccupancyMaxPotentialClusterSize + global __cuOccupancyMaxActiveClusters + data["__cuOccupancyMaxActiveClusters"] = __cuOccupancyMaxActiveClusters + global __cuTexRefSetArray + data["__cuTexRefSetArray"] = __cuTexRefSetArray + global __cuTexRefSetMipmappedArray + data["__cuTexRefSetMipmappedArray"] = __cuTexRefSetMipmappedArray + global __cuTexRefSetAddress_v2 + data["__cuTexRefSetAddress_v2"] = __cuTexRefSetAddress_v2 + global __cuTexRefSetAddress2D_v3 + data["__cuTexRefSetAddress2D_v3"] = __cuTexRefSetAddress2D_v3 + global __cuTexRefSetFormat + data["__cuTexRefSetFormat"] = __cuTexRefSetFormat + global __cuTexRefSetAddressMode + data["__cuTexRefSetAddressMode"] = __cuTexRefSetAddressMode + global __cuTexRefSetFilterMode + data["__cuTexRefSetFilterMode"] = __cuTexRefSetFilterMode + global __cuTexRefSetMipmapFilterMode + data["__cuTexRefSetMipmapFilterMode"] = __cuTexRefSetMipmapFilterMode + global __cuTexRefSetMipmapLevelBias + data["__cuTexRefSetMipmapLevelBias"] = __cuTexRefSetMipmapLevelBias + global __cuTexRefSetMipmapLevelClamp + data["__cuTexRefSetMipmapLevelClamp"] = __cuTexRefSetMipmapLevelClamp + global __cuTexRefSetMaxAnisotropy + data["__cuTexRefSetMaxAnisotropy"] = __cuTexRefSetMaxAnisotropy + global __cuTexRefSetBorderColor + data["__cuTexRefSetBorderColor"] = __cuTexRefSetBorderColor + global __cuTexRefSetFlags + data["__cuTexRefSetFlags"] = __cuTexRefSetFlags + global __cuTexRefGetAddress_v2 + data["__cuTexRefGetAddress_v2"] = __cuTexRefGetAddress_v2 + global __cuTexRefGetArray + data["__cuTexRefGetArray"] = __cuTexRefGetArray + global __cuTexRefGetMipmappedArray + data["__cuTexRefGetMipmappedArray"] = __cuTexRefGetMipmappedArray + global __cuTexRefGetAddressMode + data["__cuTexRefGetAddressMode"] = __cuTexRefGetAddressMode + global __cuTexRefGetFilterMode + data["__cuTexRefGetFilterMode"] = __cuTexRefGetFilterMode + global __cuTexRefGetFormat + data["__cuTexRefGetFormat"] = __cuTexRefGetFormat + global __cuTexRefGetMipmapFilterMode + data["__cuTexRefGetMipmapFilterMode"] = __cuTexRefGetMipmapFilterMode + global __cuTexRefGetMipmapLevelBias + data["__cuTexRefGetMipmapLevelBias"] = __cuTexRefGetMipmapLevelBias + global __cuTexRefGetMipmapLevelClamp + data["__cuTexRefGetMipmapLevelClamp"] = __cuTexRefGetMipmapLevelClamp + global __cuTexRefGetMaxAnisotropy + data["__cuTexRefGetMaxAnisotropy"] = __cuTexRefGetMaxAnisotropy + global __cuTexRefGetBorderColor + data["__cuTexRefGetBorderColor"] = __cuTexRefGetBorderColor + global __cuTexRefGetFlags + data["__cuTexRefGetFlags"] = __cuTexRefGetFlags + global __cuTexRefCreate + data["__cuTexRefCreate"] = __cuTexRefCreate + global __cuTexRefDestroy + data["__cuTexRefDestroy"] = __cuTexRefDestroy + global __cuSurfRefSetArray + data["__cuSurfRefSetArray"] = __cuSurfRefSetArray + global __cuSurfRefGetArray + data["__cuSurfRefGetArray"] = __cuSurfRefGetArray + global __cuTexObjectCreate + data["__cuTexObjectCreate"] = __cuTexObjectCreate + global __cuTexObjectDestroy + data["__cuTexObjectDestroy"] = __cuTexObjectDestroy + global __cuTexObjectGetResourceDesc + data["__cuTexObjectGetResourceDesc"] = __cuTexObjectGetResourceDesc + global __cuTexObjectGetTextureDesc + data["__cuTexObjectGetTextureDesc"] = __cuTexObjectGetTextureDesc + global __cuTexObjectGetResourceViewDesc + data["__cuTexObjectGetResourceViewDesc"] = __cuTexObjectGetResourceViewDesc + global __cuSurfObjectCreate + data["__cuSurfObjectCreate"] = __cuSurfObjectCreate + global __cuSurfObjectDestroy + data["__cuSurfObjectDestroy"] = __cuSurfObjectDestroy + global __cuSurfObjectGetResourceDesc + data["__cuSurfObjectGetResourceDesc"] = __cuSurfObjectGetResourceDesc + global __cuTensorMapEncodeTiled + data["__cuTensorMapEncodeTiled"] = __cuTensorMapEncodeTiled + global __cuTensorMapEncodeIm2col + data["__cuTensorMapEncodeIm2col"] = __cuTensorMapEncodeIm2col + global __cuTensorMapEncodeIm2colWide + data["__cuTensorMapEncodeIm2colWide"] = __cuTensorMapEncodeIm2colWide + global __cuTensorMapReplaceAddress + data["__cuTensorMapReplaceAddress"] = __cuTensorMapReplaceAddress + global __cuDeviceCanAccessPeer + data["__cuDeviceCanAccessPeer"] = __cuDeviceCanAccessPeer + global __cuCtxEnablePeerAccess + data["__cuCtxEnablePeerAccess"] = __cuCtxEnablePeerAccess + global __cuCtxDisablePeerAccess + data["__cuCtxDisablePeerAccess"] = __cuCtxDisablePeerAccess + global __cuDeviceGetP2PAttribute + data["__cuDeviceGetP2PAttribute"] = __cuDeviceGetP2PAttribute + global __cuGraphicsUnregisterResource + data["__cuGraphicsUnregisterResource"] = __cuGraphicsUnregisterResource + global __cuGraphicsSubResourceGetMappedArray + data["__cuGraphicsSubResourceGetMappedArray"] = __cuGraphicsSubResourceGetMappedArray + global __cuGraphicsResourceGetMappedMipmappedArray + data["__cuGraphicsResourceGetMappedMipmappedArray"] = __cuGraphicsResourceGetMappedMipmappedArray + global __cuGraphicsResourceGetMappedPointer_v2 + data["__cuGraphicsResourceGetMappedPointer_v2"] = __cuGraphicsResourceGetMappedPointer_v2 + global __cuGraphicsResourceSetMapFlags_v2 + data["__cuGraphicsResourceSetMapFlags_v2"] = __cuGraphicsResourceSetMapFlags_v2 + global __cuGraphicsMapResources + data["__cuGraphicsMapResources"] = __cuGraphicsMapResources + global __cuGraphicsUnmapResources + data["__cuGraphicsUnmapResources"] = __cuGraphicsUnmapResources + global __cuGetProcAddress_v2 + data["__cuGetProcAddress_v2"] = __cuGetProcAddress_v2 + global __cuCoredumpGetAttribute + data["__cuCoredumpGetAttribute"] = __cuCoredumpGetAttribute + global __cuCoredumpGetAttributeGlobal + data["__cuCoredumpGetAttributeGlobal"] = __cuCoredumpGetAttributeGlobal + global __cuCoredumpSetAttribute + data["__cuCoredumpSetAttribute"] = __cuCoredumpSetAttribute + global __cuCoredumpSetAttributeGlobal + data["__cuCoredumpSetAttributeGlobal"] = __cuCoredumpSetAttributeGlobal + global __cuGetExportTable + data["__cuGetExportTable"] = __cuGetExportTable + global __cuGreenCtxCreate + data["__cuGreenCtxCreate"] = __cuGreenCtxCreate + global __cuGreenCtxDestroy + data["__cuGreenCtxDestroy"] = __cuGreenCtxDestroy + global __cuCtxFromGreenCtx + data["__cuCtxFromGreenCtx"] = __cuCtxFromGreenCtx + global __cuDeviceGetDevResource + data["__cuDeviceGetDevResource"] = __cuDeviceGetDevResource + global __cuCtxGetDevResource + data["__cuCtxGetDevResource"] = __cuCtxGetDevResource + global __cuGreenCtxGetDevResource + data["__cuGreenCtxGetDevResource"] = __cuGreenCtxGetDevResource + global __cuDevSmResourceSplitByCount + data["__cuDevSmResourceSplitByCount"] = __cuDevSmResourceSplitByCount + global __cuDevResourceGenerateDesc + data["__cuDevResourceGenerateDesc"] = __cuDevResourceGenerateDesc + global __cuGreenCtxRecordEvent + data["__cuGreenCtxRecordEvent"] = __cuGreenCtxRecordEvent + global __cuGreenCtxWaitEvent + data["__cuGreenCtxWaitEvent"] = __cuGreenCtxWaitEvent + global __cuStreamGetGreenCtx + data["__cuStreamGetGreenCtx"] = __cuStreamGetGreenCtx + global __cuGreenCtxStreamCreate + data["__cuGreenCtxStreamCreate"] = __cuGreenCtxStreamCreate + global __cuLogsRegisterCallback + data["__cuLogsRegisterCallback"] = __cuLogsRegisterCallback + global __cuLogsUnregisterCallback + data["__cuLogsUnregisterCallback"] = __cuLogsUnregisterCallback + global __cuLogsCurrent + data["__cuLogsCurrent"] = __cuLogsCurrent + global __cuLogsDumpToFile + data["__cuLogsDumpToFile"] = __cuLogsDumpToFile + global __cuLogsDumpToMemory + data["__cuLogsDumpToMemory"] = __cuLogsDumpToMemory + global __cuCheckpointProcessGetRestoreThreadId + data["__cuCheckpointProcessGetRestoreThreadId"] = __cuCheckpointProcessGetRestoreThreadId + global __cuCheckpointProcessGetState + data["__cuCheckpointProcessGetState"] = __cuCheckpointProcessGetState + global __cuCheckpointProcessLock + data["__cuCheckpointProcessLock"] = __cuCheckpointProcessLock + global __cuCheckpointProcessCheckpoint + data["__cuCheckpointProcessCheckpoint"] = __cuCheckpointProcessCheckpoint + global __cuCheckpointProcessRestore + data["__cuCheckpointProcessRestore"] = __cuCheckpointProcessRestore + global __cuCheckpointProcessUnlock + data["__cuCheckpointProcessUnlock"] = __cuCheckpointProcessUnlock + global __cuProfilerStart + data["__cuProfilerStart"] = __cuProfilerStart + global __cuProfilerStop + data["__cuProfilerStop"] = __cuProfilerStop + global __cuGraphicsEGLRegisterImage + data["__cuGraphicsEGLRegisterImage"] = __cuGraphicsEGLRegisterImage + global __cuEGLStreamConsumerConnect + data["__cuEGLStreamConsumerConnect"] = __cuEGLStreamConsumerConnect + global __cuEGLStreamConsumerConnectWithFlags + data["__cuEGLStreamConsumerConnectWithFlags"] = __cuEGLStreamConsumerConnectWithFlags + global __cuEGLStreamConsumerDisconnect + data["__cuEGLStreamConsumerDisconnect"] = __cuEGLStreamConsumerDisconnect + global __cuEGLStreamConsumerAcquireFrame + data["__cuEGLStreamConsumerAcquireFrame"] = __cuEGLStreamConsumerAcquireFrame + global __cuEGLStreamConsumerReleaseFrame + data["__cuEGLStreamConsumerReleaseFrame"] = __cuEGLStreamConsumerReleaseFrame + global __cuEGLStreamProducerConnect + data["__cuEGLStreamProducerConnect"] = __cuEGLStreamProducerConnect + global __cuEGLStreamProducerDisconnect + data["__cuEGLStreamProducerDisconnect"] = __cuEGLStreamProducerDisconnect + global __cuEGLStreamProducerPresentFrame + data["__cuEGLStreamProducerPresentFrame"] = __cuEGLStreamProducerPresentFrame + global __cuEGLStreamProducerReturnFrame + data["__cuEGLStreamProducerReturnFrame"] = __cuEGLStreamProducerReturnFrame + global __cuGraphicsResourceGetMappedEglFrame + data["__cuGraphicsResourceGetMappedEglFrame"] = __cuGraphicsResourceGetMappedEglFrame + global __cuEventCreateFromEGLSync + data["__cuEventCreateFromEGLSync"] = __cuEventCreateFromEGLSync + global __cuGraphicsGLRegisterBuffer + data["__cuGraphicsGLRegisterBuffer"] = __cuGraphicsGLRegisterBuffer + global __cuGraphicsGLRegisterImage + data["__cuGraphicsGLRegisterImage"] = __cuGraphicsGLRegisterImage + global __cuGLGetDevices_v2 + data["__cuGLGetDevices_v2"] = __cuGLGetDevices_v2 + global __cuVDPAUGetDevice + data["__cuVDPAUGetDevice"] = __cuVDPAUGetDevice + global __cuVDPAUCtxCreate_v2 + data["__cuVDPAUCtxCreate_v2"] = __cuVDPAUCtxCreate_v2 + global __cuGraphicsVDPAURegisterVideoSurface + data["__cuGraphicsVDPAURegisterVideoSurface"] = __cuGraphicsVDPAURegisterVideoSurface + global __cuGraphicsVDPAURegisterOutputSurface + data["__cuGraphicsVDPAURegisterOutputSurface"] = __cuGraphicsVDPAURegisterOutputSurface + + func_ptrs = data + return data + +cpdef _inspect_function_pointer(str name): + global func_ptrs + if func_ptrs is None: + func_ptrs = _inspect_function_pointers() + return func_ptrs[name] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cynvrtc.cpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cynvrtc.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e52312d880b00cc34bf83fdc89021299885064dc --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cynvrtc.cpp @@ -0,0 +1,14159 @@ +/* Generated by Cython 3.1.5 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [ + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/nvrtc.h" + ], + "extra_compile_args": [ + "-std=c++14", + "-fpermissive", + "-Wno-deprecated-declarations", + "-fno-var-tracking-assignments", + "-O3" + ], + "include_dirs": [ + "/opt/python/cp312-cp312/include", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include" + ], + "language": "c++", + "library_dirs": [ + "/tmp/build-env-3t3_8d9r/lib/python3.12/site-packages", + "/tmp/build-env-3t3_8d9r/lib", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib64", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib" + ], + "name": "cuda.bindings._bindings.cynvrtc", + "sources": [ + "cuda/bindings/_bindings/cynvrtc.pyx" + ] + }, + "module_name": "cuda.bindings._bindings.cynvrtc" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_1_5" +#define CYTHON_HEX_VERSION 0x030105F0 +#define CYTHON_FUTURE_DIVISION 1 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef HAVE_LONG_LONG + #define HAVE_LONG_LONG +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_CLINE_IN_TRACEBACK + #define CYTHON_CLINE_IN_TRACEBACK 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1) +#endif +#endif +#define CYTHON_BACKPORT_VECTORCALL (CYTHON_METH_FASTCALL && PY_VERSION_HEX < 0x030800B1) +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME + #define CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME 100 +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + #endif + #endif + #if _MSC_VER < 1300 + #ifdef _WIN64 + typedef unsigned long long __pyx_uintptr_t; + #else + typedef unsigned int __pyx_uintptr_t; + #endif + #else + #ifdef _WIN64 + typedef unsigned __int64 __pyx_uintptr_t; + #else + typedef unsigned __int32 __pyx_uintptr_t; + #endif + #endif +#else + #include + typedef uintptr_t __pyx_uintptr_t; +#endif +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#elif CYTHON_BACKPORT_VECTORCALL + typedef PyObject *(*__pyx_vectorcallfunc)(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames); + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1)) + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(((size_t)(n)) & ~__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if __PYX_LIMITED_VERSION_HEX < 0x03090000 + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS && PY_VERSION_HEX >= 0x03080000 +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static CYTHON_INLINE PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__cuda__bindings___bindings__cynvrtc +#define __PYX_HAVE_API__cuda__bindings___bindings__cynvrtc +/* Early includes */ +#include +#include "nvrtc.h" +#include +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value); + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "cuda/bindings/_bindings/cynvrtc.pyx", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* ForceInitThreads.proto */ +#ifndef __PYX_FORCE_INIT_THREADS + #define __PYX_FORCE_INIT_THREADS 0 +#endif + +/* NoFastGil.proto */ +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() + +/* Atomics.proto */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif +#if CYTHON_ATOMICS + #define __pyx_add_acquisition_count(memview)\ + __pyx_atomic_incr_relaxed(__pyx_get_slice_count_pointer(memview)) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_atomic_decr_acq_rel(__pyx_get_slice_count_pointer(memview)) +#else + #define __pyx_add_acquisition_count(memview)\ + __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_sub_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) +#endif + +/* IncludeStructmemberH.proto */ +#include + +/* CriticalSections.proto */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_Begin1(cs, arg) (void)cs +#define __Pyx_PyCriticalSection_Begin2(cs, arg1, arg2) (void)cs +#define __Pyx_PyCriticalSection_End1(cs) +#define __Pyx_PyCriticalSection_End2(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_Begin1 PyCriticalSection_Begin +#define __Pyx_PyCriticalSection_Begin2 PyCriticalSection2_Begin +#define __Pyx_PyCriticalSection_End1 PyCriticalSection_End +#define __Pyx_PyCriticalSection_End2 PyCriticalSection2_End +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* #### Code section: numeric_typedefs ### */ +/* #### Code section: complex_type_declarations ### */ +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyErrExceptionMatches.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* PyObjectGetAttrStrNoError.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* PyObjectLookupSpecial.proto */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 0) +#define __Pyx_PyObject_LookupSpecial(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 1) +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error); +#else +#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n) +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#if !CYTHON_VECTORCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs); +#endif +#define __Pyx_BUILD_ASSERT_EXPR(cond)\ + (sizeof(char [1 - 2*!(cond)]) - 1) +#ifndef Py_MEMBER_SIZE +#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#endif +#if !CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03080000 + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets() + #define __Pyx_PyFrame_GetLocalsplus(frame) ((frame)->f_localsplus) +#else + static size_t __pyx_pyframe_localsplus_offset = 0; + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets()\ + ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ + (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) + #define __Pyx_PyFrame_GetLocalsplus(frame)\ + (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) +#endif +#endif +#endif + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* PyDictVersioning.proto */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __pyx_dict_cached_value;\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) do {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +#define __Pyx_GetModuleGlobalNameUncached(var, name) do {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + +/* GetTopmostException.proto */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* RaiseException.proto */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* ErrOccurredWithGIL.proto */ +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void); + +/* TupleAndListFromArray.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +#endif +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/* IncludeStringH.proto */ +#include + +/* BytesEquals.proto */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); + +/* UnicodeEquals.proto */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); + +/* fastcall.proto */ +#if CYTHON_AVOID_BORROWED_REFS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_PySequence_ITEM(args, i) +#elif CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_NewRef(__Pyx_PyTuple_GET_ITEM(args, i)) +#else + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_XNewRef(PyTuple_GetItem(args, i)) +#endif +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_ArgRef_FASTCALL(args, i) __Pyx_NewRef(args[i]) + #define __Pyx_NumKwargs_FASTCALL(kwds) __Pyx_PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs)) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues); + #else + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) + #endif +#else + #define __Pyx_ArgRef_FASTCALL __Pyx_ArgRef_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#if CYTHON_METH_FASTCALL || (CYTHON_COMPILING_IN_CPYTHON && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(args + start, stop - start) +#else +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + +/* DictGetItem.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key); +#define __Pyx_PyObject_Dict_GetItem(obj, name)\ + (likely(PyDict_CheckExact(obj)) ?\ + __Pyx_PyDict_GetItem(obj, name) : PyObject_GetItem(obj, name)) +#else +#define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key) +#define __Pyx_PyObject_Dict_GetItem(obj, name) PyObject_GetItem(obj, name) +#endif + +/* RaiseDoubleKeywords.proto */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywords.proto */ +static CYTHON_INLINE int __Pyx_ParseKeywords( + PyObject *kwds, PyObject *const *kwvalues, PyObject ** const argnames[], + PyObject *kwds2, PyObject *values[], + Py_ssize_t num_pos_args, Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* CallCFunction.proto */ +#define __Pyx_CallCFunction(cfunc, self, args)\ + ((PyCFunction)(void(*)(void))(cfunc)->func)(self, args) +#define __Pyx_CallCFunctionWithKeywords(cfunc, self, args, kwargs)\ + ((PyCFunctionWithKeywords)(void(*)(void))(cfunc)->func)(self, args, kwargs) +#define __Pyx_CallCFunctionFast(cfunc, self, args, nargs)\ + ((__Pyx_PyCFunctionFast)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs) +#define __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, nargs, kwnames)\ + ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs, kwnames) + +/* UnpackUnboundCMethod.proto */ +typedef struct { + PyObject *type; + PyObject **method_name; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS + __pyx_atomic_int_type initialized; +#endif + PyCFunction func; + PyObject *method; + int flag; +} __Pyx_CachedCFunction; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +static CYTHON_INLINE int __Pyx_CachedCFunction_GetAndSetInitializing(__Pyx_CachedCFunction *cfunc) { +#if !CYTHON_ATOMICS + return 1; +#else + __pyx_nonatomic_int_type expected = 0; + if (__pyx_atomic_int_cmp_exchange(&cfunc->initialized, &expected, 1)) { + return 0; + } + return expected; +#endif +} +static CYTHON_INLINE void __Pyx_CachedCFunction_SetFinishedInitializing(__Pyx_CachedCFunction *cfunc) { +#if CYTHON_ATOMICS + __pyx_atomic_store(&cfunc->initialized, 2); +#endif +} +#else +#define __Pyx_CachedCFunction_GetAndSetInitializing(cfunc) 2 +#define __Pyx_CachedCFunction_SetFinishedInitializing(cfunc) +#endif + +/* CallUnboundCMethod2.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2); +#else +#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2) +#endif + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* ArgTypeTest.proto */ +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* ListPack.proto */ +static PyObject *__Pyx_PyList_Pack(Py_ssize_t n, ...); + +/* Import.proto */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); + +/* ImportFrom.proto */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +/* ImportDottedModule.proto */ +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple); +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple); + +/* LimitedApiGetTypeDict.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp); +#endif + +/* SetItemOnTypeDict.proto */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v); +#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v) + +/* FixUpExtensionType.proto */ +static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); + +/* FetchSharedCythonModule.proto */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void); + +/* dict_setdefault.proto */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, int is_safe_type); + +/* FetchCommonType.proto */ +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases); + +/* CommonTypesMetaclass.proto */ +static int __pyx_CommonTypesMetaclass_init(PyObject *module); +#define __Pyx_CommonTypesMetaclass_USED + +/* CallTypeTraverse.proto */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0 +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg); +#endif + +/* PyMethodNew.proto */ +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ); + +/* PyVectorcallFastCallDict.proto */ +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); +#endif + +/* CythonFunctionShared.proto */ +#define __Pyx_CyFunction_USED +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f)\ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj)\ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject_HEAD + PyObject *func; +#elif PY_VERSION_HEX < 0x030900B1 + PyCFunctionObject func; +#else + PyCMethodObject func; +#endif +#if CYTHON_BACKPORT_VECTORCALL ||\ + (CYTHON_COMPILING_IN_LIMITED_API && CYTHON_METH_FASTCALL) + __pyx_vectorcallfunc func_vectorcall; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_weakreflist; +#endif + PyObject *func_dict; + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_classobj; +#endif + PyObject *defaults; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; + PyObject *func_is_coroutine; +} __pyx_CyFunctionObject; +#undef __Pyx_CyOrPyCFunction_Check +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_mstate_global->__pyx_CyFunctionType) +#define __Pyx_CyOrPyCFunction_Check(obj) __Pyx_TypeCheck2(obj, __pyx_mstate_global->__pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_CyFunctionType) +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)); +#undef __Pyx_IsSameCFunction +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCyOrCFunction(func, cfunc) +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, + PyTypeObject *defaults_type); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(PyObject *module); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_BACKPORT_VECTORCALL || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif + +/* CythonFunction.proto */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); + +/* CLineInTraceback.proto */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* GCCDiagnostics.proto */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* CIntFromPy.proto */ +static CYTHON_INLINE size_t __Pyx_PyLong_As_size_t(PyObject *); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* PyObjectVectorCallKwBuilder.proto */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +/* GetRuntimeVersion.proto */ +static unsigned long __Pyx_get_runtime_version(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* FunctionExport.proto */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + + +/* Module declarations from "libc.stdint" */ + +/* Module declarations from "cuda.bindings.cynvrtc" */ + +/* Module declarations from "cuda.bindings._lib.dlfcn" */ + +/* Module declarations from "cuda.bindings._bindings.cynvrtc" */ +static PyObject *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___symbol_lock = 0; +static int __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___cuPythonInit; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetErrorString; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcVersion; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNumSupportedArchs; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetSupportedArchs; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcCreateProgram; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcDestroyProgram; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcCompileProgram; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPTXSize; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPTX; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetCUBINSize; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetCUBIN; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNVVMSize; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNVVM; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLTOIRSize; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLTOIR; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetOptiXIRSize; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetOptiXIR; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetProgramLogSize; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetProgramLog; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcAddNameExpression; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLoweredName; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHHeapSize; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcSetPCHHeapSize; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHCreateStatus; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHHeapSizeRequired; +static void *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcSetFlowCallback; +static PyObject *__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs = 0; +static int __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__cuPythonInit(void); /*proto*/ +static CYTHON_INLINE int __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(void); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__inspect_function_pointers(int __pyx_skip_dispatch); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__inspect_function_pointer(PyObject *, int __pyx_skip_dispatch); /*proto*/ +/* #### Code section: typeinfo ### */ +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "cuda.bindings._bindings.cynvrtc" +extern int __pyx_module_is_main_cuda__bindings___bindings__cynvrtc; +int __pyx_module_is_main_cuda__bindings___bindings__cynvrtc = 0; + +/* Implementation of "cuda.bindings._bindings.cynvrtc" */ +/* #### Code section: global_var ### */ +static PyObject *__pyx_builtin_RuntimeError; +/* #### Code section: string_decls ### */ +static const char __pyx_k_[] = "."; +static const char __pyx_k__2[] = "?"; +static const char __pyx_k_pop[] = "pop"; +static const char __pyx_k_Lock[] = "Lock"; +static const char __pyx_k_exit[] = "__exit__"; +static const char __pyx_k_func[] = "__func__"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_name[] = "name"; +static const char __pyx_k_spec[] = "__spec__"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_enter[] = "__enter__"; +static const char __pyx_k_nvrtc[] = "nvrtc"; +static const char __pyx_k_module[] = "__module__"; +static const char __pyx_k_name_2[] = "__name__"; +static const char __pyx_k_add_note[] = "add_note"; +static const char __pyx_k_qualname[] = "__qualname__"; +static const char __pyx_k_set_name[] = "__set_name__"; +static const char __pyx_k_threading[] = "threading"; +static const char __pyx_k_z_A_a_9AQ[] = "\200\001\330\004\005\330\004\007\200z\220\023\220A\330\010\024\320\024.\250a\330\004\013\2109\220A\220Q"; +static const char __pyx_k_handle_uint[] = "_handle_uint"; +static const char __pyx_k_nvrtcGetPTX[] = "__nvrtcGetPTX"; +static const char __pyx_k_RuntimeError[] = "RuntimeError"; +static const char __pyx_k_initializing[] = "_initializing"; +static const char __pyx_k_is_coroutine[] = "_is_coroutine"; +static const char __pyx_k_nvrtcGetNVVM[] = "__nvrtcGetNVVM"; +static const char __pyx_k_nvrtcVersion[] = "__nvrtcVersion"; +static const char __pyx_k_nvrtcGetCUBIN[] = "__nvrtcGetCUBIN"; +static const char __pyx_k_nvrtcGetLTOIR[] = "__nvrtcGetLTOIR"; +static const char __pyx_k_cuda_pathfinder[] = "cuda.pathfinder"; +static const char __pyx_k_nvrtcGetOptiXIR[] = "__nvrtcGetOptiXIR"; +static const char __pyx_k_nvrtcGetPTXSize[] = "__nvrtcGetPTXSize"; +static const char __pyx_k_nvrtcGetNVVMSize[] = "__nvrtcGetNVVMSize"; +static const char __pyx_k_nvrtcGetCUBINSize[] = "__nvrtcGetCUBINSize"; +static const char __pyx_k_nvrtcGetLTOIRSize[] = "__nvrtcGetLTOIRSize"; +static const char __pyx_k_asyncio_coroutines[] = "asyncio.coroutines"; +static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; +static const char __pyx_k_nvrtcCreateProgram[] = "__nvrtcCreateProgram"; +static const char __pyx_k_nvrtcGetProgramLog[] = "__nvrtcGetProgramLog"; +static const char __pyx_k_nvrtcCompileProgram[] = "__nvrtcCompileProgram"; +static const char __pyx_k_nvrtcDestroyProgram[] = "__nvrtcDestroyProgram"; +static const char __pyx_k_nvrtcGetErrorString[] = "__nvrtcGetErrorString"; +static const char __pyx_k_nvrtcGetLoweredName[] = "__nvrtcGetLoweredName"; +static const char __pyx_k_nvrtcGetOptiXIRSize[] = "__nvrtcGetOptiXIRSize"; +static const char __pyx_k_nvrtcGetPCHHeapSize[] = "__nvrtcGetPCHHeapSize"; +static const char __pyx_k_nvrtcSetPCHHeapSize[] = "__nvrtcSetPCHHeapSize"; +static const char __pyx_k_nvrtcSetFlowCallback[] = "__nvrtcSetFlowCallback"; +static const char __pyx_k_nvrtcAddNameExpression[] = "__nvrtcAddNameExpression"; +static const char __pyx_k_nvrtcGetProgramLogSize[] = "__nvrtcGetProgramLogSize"; +static const char __pyx_k_nvrtcGetSupportedArchs[] = "__nvrtcGetSupportedArchs"; +static const char __pyx_k_load_nvidia_dynamic_lib[] = "load_nvidia_dynamic_lib"; +static const char __pyx_k_nvrtcGetPCHCreateStatus[] = "__nvrtcGetPCHCreateStatus"; +static const char __pyx_k_inspect_function_pointer[] = "_inspect_function_pointer"; +static const char __pyx_k_inspect_function_pointers[] = "_inspect_function_pointers"; +static const char __pyx_k_nvrtcGetNumSupportedArchs[] = "__nvrtcGetNumSupportedArchs"; +static const char __pyx_k_nvrtcGetPCHHeapSizeRequired[] = "__nvrtcGetPCHHeapSizeRequired"; +static const char __pyx_k_Function_nvrtcGetPTX_not_found[] = "Function \"nvrtcGetPTX\" not found"; +static const char __pyx_k_z_q_Q_Ja_Zq_A_z_Q_Ja_Ja_Ja_A_j[] = "\200\001\330\004\005\330\004\007\200z\220\027\230\001\330\010\017\210q\340\004\020\220\001\330\004\025\220Q\340\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t\035\230Z\240q\330\004\005\330\004\010\210\001\320\t*\250*\260A\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t\034\230J\240a\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t!\240\032\2501\330\004\005\330\004\010\210\001\320\t\035\230Z\240q\330\004\005\330\004\010\210\001\320\t\"\240*\250A\330\004\005\330\004\010\210\001\320\t\036\230j\250\001\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t \240\n\250!\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t#\240:\250Q\330\004\005\330\004\010\210\001\320\t'\240z\260\021\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t$\240J\250a\330\004\005\330\004\010\210\001\320\t(\250\n\260!\330\004\005\330\004\010\210\001\320\t,\250J\260a\330\004\005\330\004\010\210\001\320\t%\240Z\250q\340\004\020\220\001\330\004\013\2101"; +static const char __pyx_k_Function_nvrtcAddNameExpression[] = "Function \"nvrtcAddNameExpression\" not found"; +static const char __pyx_k_Function_nvrtcCreateProgram_not[] = "Function \"nvrtcCreateProgram\" not found"; +static const char __pyx_k_Function_nvrtcGetNVVM_not_found[] = "Function \"nvrtcGetNVVM\" not found"; +static const char __pyx_k_Function_nvrtcGetProgramLogSize[] = "Function \"nvrtcGetProgramLogSize\" not found"; +static const char __pyx_k_Function_nvrtcGetProgramLog_not[] = "Function \"nvrtcGetProgramLog\" not found"; +static const char __pyx_k_Function_nvrtcGetSupportedArchs[] = "Function \"nvrtcGetSupportedArchs\" not found"; +static const char __pyx_k_Function_nvrtcVersion_not_found[] = "Function \"nvrtcVersion\" not found"; +static const char __pyx_k_cuda_bindings__bindings_cynvrtc[] = "cuda.bindings._bindings.cynvrtc"; +static const char __pyx_k_Function_nvrtcCompileProgram_not[] = "Function \"nvrtcCompileProgram\" not found"; +static const char __pyx_k_Function_nvrtcDestroyProgram_not[] = "Function \"nvrtcDestroyProgram\" not found"; +static const char __pyx_k_Function_nvrtcGetCUBINSize_not_f[] = "Function \"nvrtcGetCUBINSize\" not found"; +static const char __pyx_k_Function_nvrtcGetCUBIN_not_found[] = "Function \"nvrtcGetCUBIN\" not found"; +static const char __pyx_k_Function_nvrtcGetErrorString_not[] = "Function \"nvrtcGetErrorString\" not found"; +static const char __pyx_k_Function_nvrtcGetLTOIRSize_not_f[] = "Function \"nvrtcGetLTOIRSize\" not found"; +static const char __pyx_k_Function_nvrtcGetLTOIR_not_found[] = "Function \"nvrtcGetLTOIR\" not found"; +static const char __pyx_k_Function_nvrtcGetLoweredName_not[] = "Function \"nvrtcGetLoweredName\" not found"; +static const char __pyx_k_Function_nvrtcGetNVVMSize_not_fo[] = "Function \"nvrtcGetNVVMSize\" not found"; +static const char __pyx_k_Function_nvrtcGetNumSupportedArc[] = "Function \"nvrtcGetNumSupportedArchs\" not found"; +static const char __pyx_k_Function_nvrtcGetOptiXIRSize_not[] = "Function \"nvrtcGetOptiXIRSize\" not found"; +static const char __pyx_k_Function_nvrtcGetOptiXIR_not_fou[] = "Function \"nvrtcGetOptiXIR\" not found"; +static const char __pyx_k_Function_nvrtcGetPCHCreateStatus[] = "Function \"nvrtcGetPCHCreateStatus\" not found"; +static const char __pyx_k_Function_nvrtcGetPCHHeapSizeRequ[] = "Function \"nvrtcGetPCHHeapSizeRequired\" not found"; +static const char __pyx_k_Function_nvrtcGetPCHHeapSize_not[] = "Function \"nvrtcGetPCHHeapSize\" not found"; +static const char __pyx_k_Function_nvrtcGetPTXSize_not_fou[] = "Function \"nvrtcGetPTXSize\" not found"; +static const char __pyx_k_Function_nvrtcSetFlowCallback_no[] = "Function \"nvrtcSetFlowCallback\" not found"; +static const char __pyx_k_Function_nvrtcSetPCHHeapSize_not[] = "Function \"nvrtcSetPCHHeapSize\" not found"; +static const char __pyx_k_Note_that_Cython_is_deliberately[] = "Note that Cython is deliberately stricter than PEP-484 and rejects subclasses of builtin types. If you need to pass subclasses then set the 'annotation_typing' directive to False."; +static const char __pyx_k_cuda_bindings__bindings_cynvrtc_2[] = "cuda/bindings/_bindings/cynvrtc.pyx"; +/* #### Code section: decls ### */ +static PyObject *__pyx_pf_4cuda_8bindings_9_bindings_7cynvrtc__inspect_function_pointers(CYTHON_UNUSED PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_pf_4cuda_8bindings_9_bindings_7cynvrtc_2_inspect_function_pointer(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_name); /* proto */ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + #ifdef __Pyx_CyFunction_USED + PyTypeObject *__pyx_CyFunctionType; + #endif + #ifdef __Pyx_FusedFunction_USED + PyTypeObject *__pyx_FusedFunctionType; + #endif + #ifdef __Pyx_Generator_USED + PyTypeObject *__pyx_GeneratorType; + #endif + #ifdef __Pyx_IterableCoroutine_USED + PyTypeObject *__pyx_IterableCoroutineType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineAwaitType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineType; + #endif + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop; + PyObject *__pyx_tuple[1]; + PyObject *__pyx_codeobj_tab[2]; + PyObject *__pyx_string_tab[83]; +/* #### Code section: module_state_contents ### */ +/* CommonTypesMetaclass.module_state_decls */ +PyTypeObject *__pyx_CommonTypesMetaclassType; + +/* CachedMethodType.module_state_decls */ +#if CYTHON_COMPILING_IN_LIMITED_API +PyObject *__Pyx_CachedMethodType; +#endif + +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_kp_u_Function_nvrtcAddNameExpression __pyx_string_tab[1] +#define __pyx_kp_u_Function_nvrtcCompileProgram_not __pyx_string_tab[2] +#define __pyx_kp_u_Function_nvrtcCreateProgram_not __pyx_string_tab[3] +#define __pyx_kp_u_Function_nvrtcDestroyProgram_not __pyx_string_tab[4] +#define __pyx_kp_u_Function_nvrtcGetCUBINSize_not_f __pyx_string_tab[5] +#define __pyx_kp_u_Function_nvrtcGetCUBIN_not_found __pyx_string_tab[6] +#define __pyx_kp_u_Function_nvrtcGetErrorString_not __pyx_string_tab[7] +#define __pyx_kp_u_Function_nvrtcGetLTOIRSize_not_f __pyx_string_tab[8] +#define __pyx_kp_u_Function_nvrtcGetLTOIR_not_found __pyx_string_tab[9] +#define __pyx_kp_u_Function_nvrtcGetLoweredName_not __pyx_string_tab[10] +#define __pyx_kp_u_Function_nvrtcGetNVVMSize_not_fo __pyx_string_tab[11] +#define __pyx_kp_u_Function_nvrtcGetNVVM_not_found __pyx_string_tab[12] +#define __pyx_kp_u_Function_nvrtcGetNumSupportedArc __pyx_string_tab[13] +#define __pyx_kp_u_Function_nvrtcGetOptiXIRSize_not __pyx_string_tab[14] +#define __pyx_kp_u_Function_nvrtcGetOptiXIR_not_fou __pyx_string_tab[15] +#define __pyx_kp_u_Function_nvrtcGetPCHCreateStatus __pyx_string_tab[16] +#define __pyx_kp_u_Function_nvrtcGetPCHHeapSizeRequ __pyx_string_tab[17] +#define __pyx_kp_u_Function_nvrtcGetPCHHeapSize_not __pyx_string_tab[18] +#define __pyx_kp_u_Function_nvrtcGetPTXSize_not_fou __pyx_string_tab[19] +#define __pyx_kp_u_Function_nvrtcGetPTX_not_found __pyx_string_tab[20] +#define __pyx_kp_u_Function_nvrtcGetProgramLogSize __pyx_string_tab[21] +#define __pyx_kp_u_Function_nvrtcGetProgramLog_not __pyx_string_tab[22] +#define __pyx_kp_u_Function_nvrtcGetSupportedArchs __pyx_string_tab[23] +#define __pyx_kp_u_Function_nvrtcSetFlowCallback_no __pyx_string_tab[24] +#define __pyx_kp_u_Function_nvrtcSetPCHHeapSize_not __pyx_string_tab[25] +#define __pyx_kp_u_Function_nvrtcVersion_not_found __pyx_string_tab[26] +#define __pyx_n_u_Lock __pyx_string_tab[27] +#define __pyx_kp_u_Note_that_Cython_is_deliberately __pyx_string_tab[28] +#define __pyx_n_u_RuntimeError __pyx_string_tab[29] +#define __pyx_kp_u__2 __pyx_string_tab[30] +#define __pyx_kp_u_add_note __pyx_string_tab[31] +#define __pyx_n_u_asyncio_coroutines __pyx_string_tab[32] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[33] +#define __pyx_n_u_cuda_bindings__bindings_cynvrtc __pyx_string_tab[34] +#define __pyx_kp_u_cuda_bindings__bindings_cynvrtc_2 __pyx_string_tab[35] +#define __pyx_n_u_cuda_pathfinder __pyx_string_tab[36] +#define __pyx_n_u_enter __pyx_string_tab[37] +#define __pyx_n_u_exit __pyx_string_tab[38] +#define __pyx_n_u_func __pyx_string_tab[39] +#define __pyx_n_u_handle_uint __pyx_string_tab[40] +#define __pyx_n_u_initializing __pyx_string_tab[41] +#define __pyx_n_u_inspect_function_pointer __pyx_string_tab[42] +#define __pyx_n_u_inspect_function_pointers __pyx_string_tab[43] +#define __pyx_n_u_is_coroutine __pyx_string_tab[44] +#define __pyx_n_u_load_nvidia_dynamic_lib __pyx_string_tab[45] +#define __pyx_n_u_main __pyx_string_tab[46] +#define __pyx_n_u_module __pyx_string_tab[47] +#define __pyx_n_u_name __pyx_string_tab[48] +#define __pyx_n_u_name_2 __pyx_string_tab[49] +#define __pyx_n_u_nvrtc __pyx_string_tab[50] +#define __pyx_n_u_nvrtcAddNameExpression __pyx_string_tab[51] +#define __pyx_n_u_nvrtcCompileProgram __pyx_string_tab[52] +#define __pyx_n_u_nvrtcCreateProgram __pyx_string_tab[53] +#define __pyx_n_u_nvrtcDestroyProgram __pyx_string_tab[54] +#define __pyx_n_u_nvrtcGetCUBIN __pyx_string_tab[55] +#define __pyx_n_u_nvrtcGetCUBINSize __pyx_string_tab[56] +#define __pyx_n_u_nvrtcGetErrorString __pyx_string_tab[57] +#define __pyx_n_u_nvrtcGetLTOIR __pyx_string_tab[58] +#define __pyx_n_u_nvrtcGetLTOIRSize __pyx_string_tab[59] +#define __pyx_n_u_nvrtcGetLoweredName __pyx_string_tab[60] +#define __pyx_n_u_nvrtcGetNVVM __pyx_string_tab[61] +#define __pyx_n_u_nvrtcGetNVVMSize __pyx_string_tab[62] +#define __pyx_n_u_nvrtcGetNumSupportedArchs __pyx_string_tab[63] +#define __pyx_n_u_nvrtcGetOptiXIR __pyx_string_tab[64] +#define __pyx_n_u_nvrtcGetOptiXIRSize __pyx_string_tab[65] +#define __pyx_n_u_nvrtcGetPCHCreateStatus __pyx_string_tab[66] +#define __pyx_n_u_nvrtcGetPCHHeapSize __pyx_string_tab[67] +#define __pyx_n_u_nvrtcGetPCHHeapSizeRequired __pyx_string_tab[68] +#define __pyx_n_u_nvrtcGetPTX __pyx_string_tab[69] +#define __pyx_n_u_nvrtcGetPTXSize __pyx_string_tab[70] +#define __pyx_n_u_nvrtcGetProgramLog __pyx_string_tab[71] +#define __pyx_n_u_nvrtcGetProgramLogSize __pyx_string_tab[72] +#define __pyx_n_u_nvrtcGetSupportedArchs __pyx_string_tab[73] +#define __pyx_n_u_nvrtcSetFlowCallback __pyx_string_tab[74] +#define __pyx_n_u_nvrtcSetPCHHeapSize __pyx_string_tab[75] +#define __pyx_n_u_nvrtcVersion __pyx_string_tab[76] +#define __pyx_n_u_pop __pyx_string_tab[77] +#define __pyx_n_u_qualname __pyx_string_tab[78] +#define __pyx_n_u_set_name __pyx_string_tab[79] +#define __pyx_n_u_spec __pyx_string_tab[80] +#define __pyx_n_u_test __pyx_string_tab[81] +#define __pyx_n_u_threading __pyx_string_tab[82] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_CLEAR(clear_module_state->__pyx_FusedFunctionType); + #endif + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + for (int i=0; i<1; ++i) { Py_CLEAR(clear_module_state->__pyx_tuple[i]); } + for (int i=0; i<2; ++i) { Py_CLEAR(clear_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<83; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_VISIT(traverse_module_state->__pyx_FusedFunctionType); + #endif + for (int i=0; i<1; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_tuple[i]); } + for (int i=0; i<2; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<83; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "cuda/bindings/_bindings/cynvrtc.pyx":39 + * cdef void *__nvrtcSetFlowCallback = NULL + * + * cdef int _cuPythonInit() except -1 nogil: # <<<<<<<<<<<<<< + * global __cuPythonInit + * +*/ + +static int __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__cuPythonInit(void) { + void *__pyx_v_handle; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + uintptr_t __pyx_t_9; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + int __pyx_t_12; + int __pyx_t_13; + int __pyx_t_14; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuPythonInit", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":43 + * + * # Load library + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * handle = (load_nvidia_dynamic_lib("nvrtc")._handle_uint) + * +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + /*with:*/ { + __pyx_t_1 = __Pyx_PyObject_LookupSpecial(__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___symbol_lock, __pyx_mstate_global->__pyx_n_u_exit); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 43, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = NULL; + __pyx_t_4 = __Pyx_PyObject_LookupSpecial(__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___symbol_lock, __pyx_mstate_global->__pyx_n_u_enter); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 43, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (likely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_4, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_4, __pyx_callargs+__pyx_t_5, (1-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 43, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + /*try:*/ { + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_6, &__pyx_t_7, &__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":44 + * # Load library + * with gil, __symbol_lock: + * handle = (load_nvidia_dynamic_lib("nvrtc")._handle_uint) # <<<<<<<<<<<<<< + * + * # Load function +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 44, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_n_u_nvrtc}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_3, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 44, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_2); + } + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_handle_uint); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 44, __pyx_L10_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_9 = __Pyx_PyLong_As_size_t(__pyx_t_3); if (unlikely((__pyx_t_9 == ((uintptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 44, __pyx_L10_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_handle = ((void *)((uintptr_t)__pyx_t_9)); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":48 + * # Load function + * global __nvrtcGetErrorString + * __nvrtcGetErrorString = dlfcn.dlsym(handle, 'nvrtcGetErrorString') # <<<<<<<<<<<<<< + * global __nvrtcVersion + * __nvrtcVersion = dlfcn.dlsym(handle, 'nvrtcVersion') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetErrorString = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetErrorString")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":50 + * __nvrtcGetErrorString = dlfcn.dlsym(handle, 'nvrtcGetErrorString') + * global __nvrtcVersion + * __nvrtcVersion = dlfcn.dlsym(handle, 'nvrtcVersion') # <<<<<<<<<<<<<< + * global __nvrtcGetNumSupportedArchs + * __nvrtcGetNumSupportedArchs = dlfcn.dlsym(handle, 'nvrtcGetNumSupportedArchs') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcVersion = dlsym(__pyx_v_handle, ((char const *)"nvrtcVersion")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":52 + * __nvrtcVersion = dlfcn.dlsym(handle, 'nvrtcVersion') + * global __nvrtcGetNumSupportedArchs + * __nvrtcGetNumSupportedArchs = dlfcn.dlsym(handle, 'nvrtcGetNumSupportedArchs') # <<<<<<<<<<<<<< + * global __nvrtcGetSupportedArchs + * __nvrtcGetSupportedArchs = dlfcn.dlsym(handle, 'nvrtcGetSupportedArchs') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNumSupportedArchs = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetNumSupportedArchs")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":54 + * __nvrtcGetNumSupportedArchs = dlfcn.dlsym(handle, 'nvrtcGetNumSupportedArchs') + * global __nvrtcGetSupportedArchs + * __nvrtcGetSupportedArchs = dlfcn.dlsym(handle, 'nvrtcGetSupportedArchs') # <<<<<<<<<<<<<< + * global __nvrtcCreateProgram + * __nvrtcCreateProgram = dlfcn.dlsym(handle, 'nvrtcCreateProgram') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetSupportedArchs = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetSupportedArchs")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":56 + * __nvrtcGetSupportedArchs = dlfcn.dlsym(handle, 'nvrtcGetSupportedArchs') + * global __nvrtcCreateProgram + * __nvrtcCreateProgram = dlfcn.dlsym(handle, 'nvrtcCreateProgram') # <<<<<<<<<<<<<< + * global __nvrtcDestroyProgram + * __nvrtcDestroyProgram = dlfcn.dlsym(handle, 'nvrtcDestroyProgram') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcCreateProgram = dlsym(__pyx_v_handle, ((char const *)"nvrtcCreateProgram")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":58 + * __nvrtcCreateProgram = dlfcn.dlsym(handle, 'nvrtcCreateProgram') + * global __nvrtcDestroyProgram + * __nvrtcDestroyProgram = dlfcn.dlsym(handle, 'nvrtcDestroyProgram') # <<<<<<<<<<<<<< + * global __nvrtcCompileProgram + * __nvrtcCompileProgram = dlfcn.dlsym(handle, 'nvrtcCompileProgram') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcDestroyProgram = dlsym(__pyx_v_handle, ((char const *)"nvrtcDestroyProgram")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":60 + * __nvrtcDestroyProgram = dlfcn.dlsym(handle, 'nvrtcDestroyProgram') + * global __nvrtcCompileProgram + * __nvrtcCompileProgram = dlfcn.dlsym(handle, 'nvrtcCompileProgram') # <<<<<<<<<<<<<< + * global __nvrtcGetPTXSize + * __nvrtcGetPTXSize = dlfcn.dlsym(handle, 'nvrtcGetPTXSize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcCompileProgram = dlsym(__pyx_v_handle, ((char const *)"nvrtcCompileProgram")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":62 + * __nvrtcCompileProgram = dlfcn.dlsym(handle, 'nvrtcCompileProgram') + * global __nvrtcGetPTXSize + * __nvrtcGetPTXSize = dlfcn.dlsym(handle, 'nvrtcGetPTXSize') # <<<<<<<<<<<<<< + * global __nvrtcGetPTX + * __nvrtcGetPTX = dlfcn.dlsym(handle, 'nvrtcGetPTX') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPTXSize = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetPTXSize")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":64 + * __nvrtcGetPTXSize = dlfcn.dlsym(handle, 'nvrtcGetPTXSize') + * global __nvrtcGetPTX + * __nvrtcGetPTX = dlfcn.dlsym(handle, 'nvrtcGetPTX') # <<<<<<<<<<<<<< + * global __nvrtcGetCUBINSize + * __nvrtcGetCUBINSize = dlfcn.dlsym(handle, 'nvrtcGetCUBINSize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPTX = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetPTX")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":66 + * __nvrtcGetPTX = dlfcn.dlsym(handle, 'nvrtcGetPTX') + * global __nvrtcGetCUBINSize + * __nvrtcGetCUBINSize = dlfcn.dlsym(handle, 'nvrtcGetCUBINSize') # <<<<<<<<<<<<<< + * global __nvrtcGetCUBIN + * __nvrtcGetCUBIN = dlfcn.dlsym(handle, 'nvrtcGetCUBIN') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetCUBINSize = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetCUBINSize")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":68 + * __nvrtcGetCUBINSize = dlfcn.dlsym(handle, 'nvrtcGetCUBINSize') + * global __nvrtcGetCUBIN + * __nvrtcGetCUBIN = dlfcn.dlsym(handle, 'nvrtcGetCUBIN') # <<<<<<<<<<<<<< + * global __nvrtcGetNVVMSize + * __nvrtcGetNVVMSize = dlfcn.dlsym(handle, 'nvrtcGetNVVMSize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetCUBIN = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetCUBIN")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":70 + * __nvrtcGetCUBIN = dlfcn.dlsym(handle, 'nvrtcGetCUBIN') + * global __nvrtcGetNVVMSize + * __nvrtcGetNVVMSize = dlfcn.dlsym(handle, 'nvrtcGetNVVMSize') # <<<<<<<<<<<<<< + * global __nvrtcGetNVVM + * __nvrtcGetNVVM = dlfcn.dlsym(handle, 'nvrtcGetNVVM') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNVVMSize = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetNVVMSize")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":72 + * __nvrtcGetNVVMSize = dlfcn.dlsym(handle, 'nvrtcGetNVVMSize') + * global __nvrtcGetNVVM + * __nvrtcGetNVVM = dlfcn.dlsym(handle, 'nvrtcGetNVVM') # <<<<<<<<<<<<<< + * global __nvrtcGetLTOIRSize + * __nvrtcGetLTOIRSize = dlfcn.dlsym(handle, 'nvrtcGetLTOIRSize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNVVM = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetNVVM")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":74 + * __nvrtcGetNVVM = dlfcn.dlsym(handle, 'nvrtcGetNVVM') + * global __nvrtcGetLTOIRSize + * __nvrtcGetLTOIRSize = dlfcn.dlsym(handle, 'nvrtcGetLTOIRSize') # <<<<<<<<<<<<<< + * global __nvrtcGetLTOIR + * __nvrtcGetLTOIR = dlfcn.dlsym(handle, 'nvrtcGetLTOIR') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLTOIRSize = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetLTOIRSize")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":76 + * __nvrtcGetLTOIRSize = dlfcn.dlsym(handle, 'nvrtcGetLTOIRSize') + * global __nvrtcGetLTOIR + * __nvrtcGetLTOIR = dlfcn.dlsym(handle, 'nvrtcGetLTOIR') # <<<<<<<<<<<<<< + * global __nvrtcGetOptiXIRSize + * __nvrtcGetOptiXIRSize = dlfcn.dlsym(handle, 'nvrtcGetOptiXIRSize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLTOIR = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetLTOIR")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":78 + * __nvrtcGetLTOIR = dlfcn.dlsym(handle, 'nvrtcGetLTOIR') + * global __nvrtcGetOptiXIRSize + * __nvrtcGetOptiXIRSize = dlfcn.dlsym(handle, 'nvrtcGetOptiXIRSize') # <<<<<<<<<<<<<< + * global __nvrtcGetOptiXIR + * __nvrtcGetOptiXIR = dlfcn.dlsym(handle, 'nvrtcGetOptiXIR') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetOptiXIRSize = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetOptiXIRSize")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":80 + * __nvrtcGetOptiXIRSize = dlfcn.dlsym(handle, 'nvrtcGetOptiXIRSize') + * global __nvrtcGetOptiXIR + * __nvrtcGetOptiXIR = dlfcn.dlsym(handle, 'nvrtcGetOptiXIR') # <<<<<<<<<<<<<< + * global __nvrtcGetProgramLogSize + * __nvrtcGetProgramLogSize = dlfcn.dlsym(handle, 'nvrtcGetProgramLogSize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetOptiXIR = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetOptiXIR")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":82 + * __nvrtcGetOptiXIR = dlfcn.dlsym(handle, 'nvrtcGetOptiXIR') + * global __nvrtcGetProgramLogSize + * __nvrtcGetProgramLogSize = dlfcn.dlsym(handle, 'nvrtcGetProgramLogSize') # <<<<<<<<<<<<<< + * global __nvrtcGetProgramLog + * __nvrtcGetProgramLog = dlfcn.dlsym(handle, 'nvrtcGetProgramLog') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetProgramLogSize = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetProgramLogSize")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":84 + * __nvrtcGetProgramLogSize = dlfcn.dlsym(handle, 'nvrtcGetProgramLogSize') + * global __nvrtcGetProgramLog + * __nvrtcGetProgramLog = dlfcn.dlsym(handle, 'nvrtcGetProgramLog') # <<<<<<<<<<<<<< + * global __nvrtcAddNameExpression + * __nvrtcAddNameExpression = dlfcn.dlsym(handle, 'nvrtcAddNameExpression') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetProgramLog = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetProgramLog")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":86 + * __nvrtcGetProgramLog = dlfcn.dlsym(handle, 'nvrtcGetProgramLog') + * global __nvrtcAddNameExpression + * __nvrtcAddNameExpression = dlfcn.dlsym(handle, 'nvrtcAddNameExpression') # <<<<<<<<<<<<<< + * global __nvrtcGetLoweredName + * __nvrtcGetLoweredName = dlfcn.dlsym(handle, 'nvrtcGetLoweredName') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcAddNameExpression = dlsym(__pyx_v_handle, ((char const *)"nvrtcAddNameExpression")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":88 + * __nvrtcAddNameExpression = dlfcn.dlsym(handle, 'nvrtcAddNameExpression') + * global __nvrtcGetLoweredName + * __nvrtcGetLoweredName = dlfcn.dlsym(handle, 'nvrtcGetLoweredName') # <<<<<<<<<<<<<< + * global __nvrtcGetPCHHeapSize + * __nvrtcGetPCHHeapSize = dlfcn.dlsym(handle, 'nvrtcGetPCHHeapSize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLoweredName = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetLoweredName")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":90 + * __nvrtcGetLoweredName = dlfcn.dlsym(handle, 'nvrtcGetLoweredName') + * global __nvrtcGetPCHHeapSize + * __nvrtcGetPCHHeapSize = dlfcn.dlsym(handle, 'nvrtcGetPCHHeapSize') # <<<<<<<<<<<<<< + * global __nvrtcSetPCHHeapSize + * __nvrtcSetPCHHeapSize = dlfcn.dlsym(handle, 'nvrtcSetPCHHeapSize') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHHeapSize = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetPCHHeapSize")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":92 + * __nvrtcGetPCHHeapSize = dlfcn.dlsym(handle, 'nvrtcGetPCHHeapSize') + * global __nvrtcSetPCHHeapSize + * __nvrtcSetPCHHeapSize = dlfcn.dlsym(handle, 'nvrtcSetPCHHeapSize') # <<<<<<<<<<<<<< + * global __nvrtcGetPCHCreateStatus + * __nvrtcGetPCHCreateStatus = dlfcn.dlsym(handle, 'nvrtcGetPCHCreateStatus') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcSetPCHHeapSize = dlsym(__pyx_v_handle, ((char const *)"nvrtcSetPCHHeapSize")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":94 + * __nvrtcSetPCHHeapSize = dlfcn.dlsym(handle, 'nvrtcSetPCHHeapSize') + * global __nvrtcGetPCHCreateStatus + * __nvrtcGetPCHCreateStatus = dlfcn.dlsym(handle, 'nvrtcGetPCHCreateStatus') # <<<<<<<<<<<<<< + * global __nvrtcGetPCHHeapSizeRequired + * __nvrtcGetPCHHeapSizeRequired = dlfcn.dlsym(handle, 'nvrtcGetPCHHeapSizeRequired') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHCreateStatus = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetPCHCreateStatus")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":96 + * __nvrtcGetPCHCreateStatus = dlfcn.dlsym(handle, 'nvrtcGetPCHCreateStatus') + * global __nvrtcGetPCHHeapSizeRequired + * __nvrtcGetPCHHeapSizeRequired = dlfcn.dlsym(handle, 'nvrtcGetPCHHeapSizeRequired') # <<<<<<<<<<<<<< + * global __nvrtcSetFlowCallback + * __nvrtcSetFlowCallback = dlfcn.dlsym(handle, 'nvrtcSetFlowCallback') +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHHeapSizeRequired = dlsym(__pyx_v_handle, ((char const *)"nvrtcGetPCHHeapSizeRequired")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":98 + * __nvrtcGetPCHHeapSizeRequired = dlfcn.dlsym(handle, 'nvrtcGetPCHHeapSizeRequired') + * global __nvrtcSetFlowCallback + * __nvrtcSetFlowCallback = dlfcn.dlsym(handle, 'nvrtcSetFlowCallback') # <<<<<<<<<<<<<< + * __cuPythonInit = True + * return 0 +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcSetFlowCallback = dlsym(__pyx_v_handle, ((char const *)"nvrtcSetFlowCallback")); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":99 + * global __nvrtcSetFlowCallback + * __nvrtcSetFlowCallback = dlfcn.dlsym(handle, 'nvrtcSetFlowCallback') + * __cuPythonInit = True # <<<<<<<<<<<<<< + * return 0 + * +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___cuPythonInit = 1; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":100 + * __nvrtcSetFlowCallback = dlfcn.dlsym(handle, 'nvrtcSetFlowCallback') + * __cuPythonInit = True + * return 0 # <<<<<<<<<<<<<< + * + * # Create a very small function to check whether we are init'ed, so the C +*/ + __pyx_r = 0; + goto __pyx_L14_try_return; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":43 + * + * # Load library + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * handle = (load_nvidia_dynamic_lib("nvrtc")._handle_uint) + * +*/ + } + __pyx_L10_error:; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + /*except:*/ { + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._cuPythonInit", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_3, &__pyx_t_2, &__pyx_t_4) < 0) __PYX_ERR(0, 43, __pyx_L12_except_error) + __Pyx_XGOTREF(__pyx_t_3); + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_4); + __pyx_t_10 = PyTuple_Pack(3, __pyx_t_3, __pyx_t_2, __pyx_t_4); if (unlikely(!__pyx_t_10)) __PYX_ERR(0, 43, __pyx_L12_except_error) + __Pyx_GOTREF(__pyx_t_10); + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_10, NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_10); __pyx_t_10 = 0; + if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 43, __pyx_L12_except_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_12 = __Pyx_PyObject_IsTrue(__pyx_t_11); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (__pyx_t_12 < (0)) __PYX_ERR(0, 43, __pyx_L12_except_error) + __pyx_t_13 = (!__pyx_t_12); + if (unlikely(__pyx_t_13)) { + __Pyx_GIVEREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_2); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_ErrRestoreWithState(__pyx_t_3, __pyx_t_2, __pyx_t_4); + __pyx_t_3 = 0; __pyx_t_2 = 0; __pyx_t_4 = 0; + __PYX_ERR(0, 43, __pyx_L12_except_error) + } + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + goto __pyx_L11_exception_handled; + } + __pyx_L12_except_error:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L4_error; + __pyx_L14_try_return:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L7_return; + __pyx_L11_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + } + } + /*finally:*/ { + /*normal exit:*/{ + if (__pyx_t_1) { + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_mstate_global->__pyx_tuple[0], NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 43, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + goto __pyx_L9; + } + __pyx_L7_return: { + __pyx_t_14 = __pyx_r; + if (__pyx_t_1) { + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_mstate_global->__pyx_tuple[0], NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 43, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + __pyx_r = __pyx_t_14; + goto __pyx_L3_return; + } + __pyx_L9:; + } + goto __pyx_L19; + __pyx_L6_error:; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L4_error; + __pyx_L19:; + } + } + /*finally:*/ { + /*normal exit:*/{ + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L5; + } + __pyx_L3_return: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L0; + } + __pyx_L4_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + __pyx_L5:; + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":39 + * cdef void *__nvrtcSetFlowCallback = NULL + * + * cdef int _cuPythonInit() except -1 nogil: # <<<<<<<<<<<<<< + * global __cuPythonInit + * +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._cuPythonInit", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":104 + * # Create a very small function to check whether we are init'ed, so the C + * # compiler can inline it. + * cdef inline int cuPythonInit() except -1 nogil: # <<<<<<<<<<<<<< + * if __cuPythonInit: + * return 0 +*/ + +static CYTHON_INLINE int __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(void) { + int __pyx_r; + int __pyx_t_1; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":105 + * # compiler can inline it. + * cdef inline int cuPythonInit() except -1 nogil: + * if __cuPythonInit: # <<<<<<<<<<<<<< + * return 0 + * return _cuPythonInit() +*/ + if (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___cuPythonInit) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":106 + * cdef inline int cuPythonInit() except -1 nogil: + * if __cuPythonInit: + * return 0 # <<<<<<<<<<<<<< + * return _cuPythonInit() + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":105 + * # compiler can inline it. + * cdef inline int cuPythonInit() except -1 nogil: + * if __cuPythonInit: # <<<<<<<<<<<<<< + * return 0 + * return _cuPythonInit() +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":107 + * if __cuPythonInit: + * return 0 + * return _cuPythonInit() # <<<<<<<<<<<<<< + * + * cdef const char* _nvrtcGetErrorString(nvrtcResult result) except ?NULL nogil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 107, __pyx_L1_error) + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":104 + * # Create a very small function to check whether we are init'ed, so the C + * # compiler can inline it. + * cdef inline int cuPythonInit() except -1 nogil: # <<<<<<<<<<<<<< + * if __cuPythonInit: + * return 0 +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc.cuPythonInit", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":109 + * return _cuPythonInit() + * + * cdef const char* _nvrtcGetErrorString(nvrtcResult result) except ?NULL nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetErrorString + * cuPythonInit() +*/ + +static char const *__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetErrorString(nvrtcResult __pyx_v_result) { + char const *__pyx_v_err; + char const *__pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + char const *__pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetErrorString", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":111 + * cdef const char* _nvrtcGetErrorString(nvrtcResult result) except ?NULL nogil: + * global __nvrtcGetErrorString + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetErrorString == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 111, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":112 + * global __nvrtcGetErrorString + * cuPythonInit() + * if __nvrtcGetErrorString == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetErrorString" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetErrorString == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":113 + * cuPythonInit() + * if __nvrtcGetErrorString == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetErrorString" not found') + * err = ( __nvrtcGetErrorString)(result) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":114 + * if __nvrtcGetErrorString == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetErrorString" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetErrorString)(result) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetErrorString_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 114, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 114, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":113 + * cuPythonInit() + * if __nvrtcGetErrorString == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetErrorString" not found') + * err = ( __nvrtcGetErrorString)(result) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":112 + * global __nvrtcGetErrorString + * cuPythonInit() + * if __nvrtcGetErrorString == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetErrorString" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":115 + * with gil: + * raise RuntimeError('Function "nvrtcGetErrorString" not found') + * err = ( __nvrtcGetErrorString)(result) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((char const *(*)(nvrtcResult))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetErrorString)(__pyx_v_result); if (unlikely(__pyx_t_7 == ((char const *)0) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 115, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":116 + * raise RuntimeError('Function "nvrtcGetErrorString" not found') + * err = ( __nvrtcGetErrorString)(result) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcVersion(int* major, int* minor) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":109 + * return _cuPythonInit() + * + * cdef const char* _nvrtcGetErrorString(nvrtcResult result) except ?NULL nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetErrorString + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetErrorString", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":118 + * return err + * + * cdef nvrtcResult _nvrtcVersion(int* major, int* minor) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcVersion + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcVersion(int *__pyx_v_major, int *__pyx_v_minor) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcVersion", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":120 + * cdef nvrtcResult _nvrtcVersion(int* major, int* minor) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcVersion + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcVersion == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 120, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":121 + * global __nvrtcVersion + * cuPythonInit() + * if __nvrtcVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcVersion" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcVersion == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":122 + * cuPythonInit() + * if __nvrtcVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcVersion" not found') + * err = ( __nvrtcVersion)(major, minor) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":123 + * if __nvrtcVersion == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcVersion" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcVersion)(major, minor) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcVersion_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 123, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 123, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":122 + * cuPythonInit() + * if __nvrtcVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcVersion" not found') + * err = ( __nvrtcVersion)(major, minor) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":121 + * global __nvrtcVersion + * cuPythonInit() + * if __nvrtcVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcVersion" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":124 + * with gil: + * raise RuntimeError('Function "nvrtcVersion" not found') + * err = ( __nvrtcVersion)(major, minor) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(int *, int *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcVersion)(__pyx_v_major, __pyx_v_minor); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 124, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":125 + * raise RuntimeError('Function "nvrtcVersion" not found') + * err = ( __nvrtcVersion)(major, minor) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetNumSupportedArchs(int* numArchs) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":118 + * return err + * + * cdef nvrtcResult _nvrtcVersion(int* major, int* minor) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcVersion + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcVersion", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":127 + * return err + * + * cdef nvrtcResult _nvrtcGetNumSupportedArchs(int* numArchs) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetNumSupportedArchs + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetNumSupportedArchs(int *__pyx_v_numArchs) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetNumSupportedArchs", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":129 + * cdef nvrtcResult _nvrtcGetNumSupportedArchs(int* numArchs) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetNumSupportedArchs + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetNumSupportedArchs == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 129, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":130 + * global __nvrtcGetNumSupportedArchs + * cuPythonInit() + * if __nvrtcGetNumSupportedArchs == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetNumSupportedArchs" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNumSupportedArchs == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":131 + * cuPythonInit() + * if __nvrtcGetNumSupportedArchs == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetNumSupportedArchs" not found') + * err = ( __nvrtcGetNumSupportedArchs)(numArchs) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":132 + * if __nvrtcGetNumSupportedArchs == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetNumSupportedArchs" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetNumSupportedArchs)(numArchs) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetNumSupportedArc}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 132, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 132, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":131 + * cuPythonInit() + * if __nvrtcGetNumSupportedArchs == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetNumSupportedArchs" not found') + * err = ( __nvrtcGetNumSupportedArchs)(numArchs) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":130 + * global __nvrtcGetNumSupportedArchs + * cuPythonInit() + * if __nvrtcGetNumSupportedArchs == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetNumSupportedArchs" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":133 + * with gil: + * raise RuntimeError('Function "nvrtcGetNumSupportedArchs" not found') + * err = ( __nvrtcGetNumSupportedArchs)(numArchs) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(int *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNumSupportedArchs)(__pyx_v_numArchs); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 133, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":134 + * raise RuntimeError('Function "nvrtcGetNumSupportedArchs" not found') + * err = ( __nvrtcGetNumSupportedArchs)(numArchs) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetSupportedArchs(int* supportedArchs) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":127 + * return err + * + * cdef nvrtcResult _nvrtcGetNumSupportedArchs(int* numArchs) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetNumSupportedArchs + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetNumSupportedArchs", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":136 + * return err + * + * cdef nvrtcResult _nvrtcGetSupportedArchs(int* supportedArchs) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetSupportedArchs + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetSupportedArchs(int *__pyx_v_supportedArchs) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetSupportedArchs", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":138 + * cdef nvrtcResult _nvrtcGetSupportedArchs(int* supportedArchs) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetSupportedArchs + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetSupportedArchs == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 138, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":139 + * global __nvrtcGetSupportedArchs + * cuPythonInit() + * if __nvrtcGetSupportedArchs == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetSupportedArchs" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetSupportedArchs == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":140 + * cuPythonInit() + * if __nvrtcGetSupportedArchs == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetSupportedArchs" not found') + * err = ( __nvrtcGetSupportedArchs)(supportedArchs) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":141 + * if __nvrtcGetSupportedArchs == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetSupportedArchs" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetSupportedArchs)(supportedArchs) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetSupportedArchs}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 141, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 141, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":140 + * cuPythonInit() + * if __nvrtcGetSupportedArchs == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetSupportedArchs" not found') + * err = ( __nvrtcGetSupportedArchs)(supportedArchs) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":139 + * global __nvrtcGetSupportedArchs + * cuPythonInit() + * if __nvrtcGetSupportedArchs == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetSupportedArchs" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":142 + * with gil: + * raise RuntimeError('Function "nvrtcGetSupportedArchs" not found') + * err = ( __nvrtcGetSupportedArchs)(supportedArchs) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(int *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetSupportedArchs)(__pyx_v_supportedArchs); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 142, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":143 + * raise RuntimeError('Function "nvrtcGetSupportedArchs" not found') + * err = ( __nvrtcGetSupportedArchs)(supportedArchs) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcCreateProgram(nvrtcProgram* prog, const char* src, const char* name, int numHeaders, const char** headers, const char** includeNames) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":136 + * return err + * + * cdef nvrtcResult _nvrtcGetSupportedArchs(int* supportedArchs) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetSupportedArchs + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetSupportedArchs", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":145 + * return err + * + * cdef nvrtcResult _nvrtcCreateProgram(nvrtcProgram* prog, const char* src, const char* name, int numHeaders, const char** headers, const char** includeNames) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcCreateProgram + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcCreateProgram(nvrtcProgram *__pyx_v_prog, char const *__pyx_v_src, char const *__pyx_v_name, int __pyx_v_numHeaders, char const **__pyx_v_headers, char const **__pyx_v_includeNames) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcCreateProgram", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":147 + * cdef nvrtcResult _nvrtcCreateProgram(nvrtcProgram* prog, const char* src, const char* name, int numHeaders, const char** headers, const char** includeNames) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcCreateProgram + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcCreateProgram == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 147, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":148 + * global __nvrtcCreateProgram + * cuPythonInit() + * if __nvrtcCreateProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcCreateProgram" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcCreateProgram == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":149 + * cuPythonInit() + * if __nvrtcCreateProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcCreateProgram" not found') + * err = ( __nvrtcCreateProgram)(prog, src, name, numHeaders, headers, includeNames) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":150 + * if __nvrtcCreateProgram == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcCreateProgram" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcCreateProgram)(prog, src, name, numHeaders, headers, includeNames) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcCreateProgram_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 150, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 150, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":149 + * cuPythonInit() + * if __nvrtcCreateProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcCreateProgram" not found') + * err = ( __nvrtcCreateProgram)(prog, src, name, numHeaders, headers, includeNames) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":148 + * global __nvrtcCreateProgram + * cuPythonInit() + * if __nvrtcCreateProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcCreateProgram" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":151 + * with gil: + * raise RuntimeError('Function "nvrtcCreateProgram" not found') + * err = ( __nvrtcCreateProgram)(prog, src, name, numHeaders, headers, includeNames) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram *, char const *, char const *, int, char const **, char const **))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcCreateProgram)(__pyx_v_prog, __pyx_v_src, __pyx_v_name, __pyx_v_numHeaders, __pyx_v_headers, __pyx_v_includeNames); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 151, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":152 + * raise RuntimeError('Function "nvrtcCreateProgram" not found') + * err = ( __nvrtcCreateProgram)(prog, src, name, numHeaders, headers, includeNames) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcDestroyProgram(nvrtcProgram* prog) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":145 + * return err + * + * cdef nvrtcResult _nvrtcCreateProgram(nvrtcProgram* prog, const char* src, const char* name, int numHeaders, const char** headers, const char** includeNames) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcCreateProgram + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcCreateProgram", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":154 + * return err + * + * cdef nvrtcResult _nvrtcDestroyProgram(nvrtcProgram* prog) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcDestroyProgram + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcDestroyProgram(nvrtcProgram *__pyx_v_prog) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcDestroyProgram", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":156 + * cdef nvrtcResult _nvrtcDestroyProgram(nvrtcProgram* prog) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcDestroyProgram + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcDestroyProgram == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 156, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":157 + * global __nvrtcDestroyProgram + * cuPythonInit() + * if __nvrtcDestroyProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcDestroyProgram" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcDestroyProgram == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":158 + * cuPythonInit() + * if __nvrtcDestroyProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcDestroyProgram" not found') + * err = ( __nvrtcDestroyProgram)(prog) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":159 + * if __nvrtcDestroyProgram == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcDestroyProgram" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcDestroyProgram)(prog) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcDestroyProgram_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 159, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 159, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":158 + * cuPythonInit() + * if __nvrtcDestroyProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcDestroyProgram" not found') + * err = ( __nvrtcDestroyProgram)(prog) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":157 + * global __nvrtcDestroyProgram + * cuPythonInit() + * if __nvrtcDestroyProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcDestroyProgram" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":160 + * with gil: + * raise RuntimeError('Function "nvrtcDestroyProgram" not found') + * err = ( __nvrtcDestroyProgram)(prog) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcDestroyProgram)(__pyx_v_prog); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 160, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":161 + * raise RuntimeError('Function "nvrtcDestroyProgram" not found') + * err = ( __nvrtcDestroyProgram)(prog) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcCompileProgram(nvrtcProgram prog, int numOptions, const char** options) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":154 + * return err + * + * cdef nvrtcResult _nvrtcDestroyProgram(nvrtcProgram* prog) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcDestroyProgram + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcDestroyProgram", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":163 + * return err + * + * cdef nvrtcResult _nvrtcCompileProgram(nvrtcProgram prog, int numOptions, const char** options) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcCompileProgram + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcCompileProgram(nvrtcProgram __pyx_v_prog, int __pyx_v_numOptions, char const **__pyx_v_options) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcCompileProgram", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":165 + * cdef nvrtcResult _nvrtcCompileProgram(nvrtcProgram prog, int numOptions, const char** options) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcCompileProgram + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcCompileProgram == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 165, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":166 + * global __nvrtcCompileProgram + * cuPythonInit() + * if __nvrtcCompileProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcCompileProgram" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcCompileProgram == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":167 + * cuPythonInit() + * if __nvrtcCompileProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcCompileProgram" not found') + * err = ( __nvrtcCompileProgram)(prog, numOptions, options) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":168 + * if __nvrtcCompileProgram == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcCompileProgram" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcCompileProgram)(prog, numOptions, options) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcCompileProgram_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 168, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 168, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":167 + * cuPythonInit() + * if __nvrtcCompileProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcCompileProgram" not found') + * err = ( __nvrtcCompileProgram)(prog, numOptions, options) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":166 + * global __nvrtcCompileProgram + * cuPythonInit() + * if __nvrtcCompileProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcCompileProgram" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":169 + * with gil: + * raise RuntimeError('Function "nvrtcCompileProgram" not found') + * err = ( __nvrtcCompileProgram)(prog, numOptions, options) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, int, char const **))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcCompileProgram)(__pyx_v_prog, __pyx_v_numOptions, __pyx_v_options); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 169, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":170 + * raise RuntimeError('Function "nvrtcCompileProgram" not found') + * err = ( __nvrtcCompileProgram)(prog, numOptions, options) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetPTXSize(nvrtcProgram prog, size_t* ptxSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":163 + * return err + * + * cdef nvrtcResult _nvrtcCompileProgram(nvrtcProgram prog, int numOptions, const char** options) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcCompileProgram + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcCompileProgram", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":172 + * return err + * + * cdef nvrtcResult _nvrtcGetPTXSize(nvrtcProgram prog, size_t* ptxSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetPTXSize + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetPTXSize(nvrtcProgram __pyx_v_prog, size_t *__pyx_v_ptxSizeRet) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetPTXSize", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":174 + * cdef nvrtcResult _nvrtcGetPTXSize(nvrtcProgram prog, size_t* ptxSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetPTXSize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetPTXSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 174, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":175 + * global __nvrtcGetPTXSize + * cuPythonInit() + * if __nvrtcGetPTXSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetPTXSize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPTXSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":176 + * cuPythonInit() + * if __nvrtcGetPTXSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetPTXSize" not found') + * err = ( __nvrtcGetPTXSize)(prog, ptxSizeRet) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":177 + * if __nvrtcGetPTXSize == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetPTXSize" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetPTXSize)(prog, ptxSizeRet) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetPTXSize_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 177, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 177, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":176 + * cuPythonInit() + * if __nvrtcGetPTXSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetPTXSize" not found') + * err = ( __nvrtcGetPTXSize)(prog, ptxSizeRet) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":175 + * global __nvrtcGetPTXSize + * cuPythonInit() + * if __nvrtcGetPTXSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetPTXSize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":178 + * with gil: + * raise RuntimeError('Function "nvrtcGetPTXSize" not found') + * err = ( __nvrtcGetPTXSize)(prog, ptxSizeRet) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, size_t *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPTXSize)(__pyx_v_prog, __pyx_v_ptxSizeRet); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 178, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":179 + * raise RuntimeError('Function "nvrtcGetPTXSize" not found') + * err = ( __nvrtcGetPTXSize)(prog, ptxSizeRet) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetPTX(nvrtcProgram prog, char* ptx) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":172 + * return err + * + * cdef nvrtcResult _nvrtcGetPTXSize(nvrtcProgram prog, size_t* ptxSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetPTXSize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetPTXSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":181 + * return err + * + * cdef nvrtcResult _nvrtcGetPTX(nvrtcProgram prog, char* ptx) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetPTX + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetPTX(nvrtcProgram __pyx_v_prog, char *__pyx_v_ptx) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetPTX", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":183 + * cdef nvrtcResult _nvrtcGetPTX(nvrtcProgram prog, char* ptx) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetPTX + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetPTX == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 183, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":184 + * global __nvrtcGetPTX + * cuPythonInit() + * if __nvrtcGetPTX == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetPTX" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPTX == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":185 + * cuPythonInit() + * if __nvrtcGetPTX == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetPTX" not found') + * err = ( __nvrtcGetPTX)(prog, ptx) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":186 + * if __nvrtcGetPTX == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetPTX" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetPTX)(prog, ptx) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetPTX_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 186, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 186, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":185 + * cuPythonInit() + * if __nvrtcGetPTX == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetPTX" not found') + * err = ( __nvrtcGetPTX)(prog, ptx) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":184 + * global __nvrtcGetPTX + * cuPythonInit() + * if __nvrtcGetPTX == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetPTX" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":187 + * with gil: + * raise RuntimeError('Function "nvrtcGetPTX" not found') + * err = ( __nvrtcGetPTX)(prog, ptx) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, char *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPTX)(__pyx_v_prog, __pyx_v_ptx); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 187, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":188 + * raise RuntimeError('Function "nvrtcGetPTX" not found') + * err = ( __nvrtcGetPTX)(prog, ptx) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetCUBINSize(nvrtcProgram prog, size_t* cubinSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":181 + * return err + * + * cdef nvrtcResult _nvrtcGetPTX(nvrtcProgram prog, char* ptx) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetPTX + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetPTX", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":190 + * return err + * + * cdef nvrtcResult _nvrtcGetCUBINSize(nvrtcProgram prog, size_t* cubinSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetCUBINSize + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetCUBINSize(nvrtcProgram __pyx_v_prog, size_t *__pyx_v_cubinSizeRet) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetCUBINSize", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":192 + * cdef nvrtcResult _nvrtcGetCUBINSize(nvrtcProgram prog, size_t* cubinSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetCUBINSize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetCUBINSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 192, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":193 + * global __nvrtcGetCUBINSize + * cuPythonInit() + * if __nvrtcGetCUBINSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetCUBINSize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetCUBINSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":194 + * cuPythonInit() + * if __nvrtcGetCUBINSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetCUBINSize" not found') + * err = ( __nvrtcGetCUBINSize)(prog, cubinSizeRet) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":195 + * if __nvrtcGetCUBINSize == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetCUBINSize" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetCUBINSize)(prog, cubinSizeRet) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetCUBINSize_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 195, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 195, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":194 + * cuPythonInit() + * if __nvrtcGetCUBINSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetCUBINSize" not found') + * err = ( __nvrtcGetCUBINSize)(prog, cubinSizeRet) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":193 + * global __nvrtcGetCUBINSize + * cuPythonInit() + * if __nvrtcGetCUBINSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetCUBINSize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":196 + * with gil: + * raise RuntimeError('Function "nvrtcGetCUBINSize" not found') + * err = ( __nvrtcGetCUBINSize)(prog, cubinSizeRet) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, size_t *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetCUBINSize)(__pyx_v_prog, __pyx_v_cubinSizeRet); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 196, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":197 + * raise RuntimeError('Function "nvrtcGetCUBINSize" not found') + * err = ( __nvrtcGetCUBINSize)(prog, cubinSizeRet) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetCUBIN(nvrtcProgram prog, char* cubin) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":190 + * return err + * + * cdef nvrtcResult _nvrtcGetCUBINSize(nvrtcProgram prog, size_t* cubinSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetCUBINSize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetCUBINSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":199 + * return err + * + * cdef nvrtcResult _nvrtcGetCUBIN(nvrtcProgram prog, char* cubin) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetCUBIN + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetCUBIN(nvrtcProgram __pyx_v_prog, char *__pyx_v_cubin) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetCUBIN", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":201 + * cdef nvrtcResult _nvrtcGetCUBIN(nvrtcProgram prog, char* cubin) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetCUBIN + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetCUBIN == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 201, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":202 + * global __nvrtcGetCUBIN + * cuPythonInit() + * if __nvrtcGetCUBIN == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetCUBIN" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetCUBIN == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":203 + * cuPythonInit() + * if __nvrtcGetCUBIN == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetCUBIN" not found') + * err = ( __nvrtcGetCUBIN)(prog, cubin) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":204 + * if __nvrtcGetCUBIN == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetCUBIN" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetCUBIN)(prog, cubin) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetCUBIN_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 204, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 204, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":203 + * cuPythonInit() + * if __nvrtcGetCUBIN == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetCUBIN" not found') + * err = ( __nvrtcGetCUBIN)(prog, cubin) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":202 + * global __nvrtcGetCUBIN + * cuPythonInit() + * if __nvrtcGetCUBIN == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetCUBIN" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":205 + * with gil: + * raise RuntimeError('Function "nvrtcGetCUBIN" not found') + * err = ( __nvrtcGetCUBIN)(prog, cubin) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, char *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetCUBIN)(__pyx_v_prog, __pyx_v_cubin); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 205, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":206 + * raise RuntimeError('Function "nvrtcGetCUBIN" not found') + * err = ( __nvrtcGetCUBIN)(prog, cubin) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetNVVMSize(nvrtcProgram prog, size_t* nvvmSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":199 + * return err + * + * cdef nvrtcResult _nvrtcGetCUBIN(nvrtcProgram prog, char* cubin) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetCUBIN + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetCUBIN", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":208 + * return err + * + * cdef nvrtcResult _nvrtcGetNVVMSize(nvrtcProgram prog, size_t* nvvmSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetNVVMSize + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetNVVMSize(nvrtcProgram __pyx_v_prog, size_t *__pyx_v_nvvmSizeRet) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetNVVMSize", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":210 + * cdef nvrtcResult _nvrtcGetNVVMSize(nvrtcProgram prog, size_t* nvvmSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetNVVMSize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetNVVMSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 210, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":211 + * global __nvrtcGetNVVMSize + * cuPythonInit() + * if __nvrtcGetNVVMSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetNVVMSize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNVVMSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":212 + * cuPythonInit() + * if __nvrtcGetNVVMSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetNVVMSize" not found') + * err = ( __nvrtcGetNVVMSize)(prog, nvvmSizeRet) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":213 + * if __nvrtcGetNVVMSize == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetNVVMSize" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetNVVMSize)(prog, nvvmSizeRet) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetNVVMSize_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 213, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 213, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":212 + * cuPythonInit() + * if __nvrtcGetNVVMSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetNVVMSize" not found') + * err = ( __nvrtcGetNVVMSize)(prog, nvvmSizeRet) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":211 + * global __nvrtcGetNVVMSize + * cuPythonInit() + * if __nvrtcGetNVVMSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetNVVMSize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":214 + * with gil: + * raise RuntimeError('Function "nvrtcGetNVVMSize" not found') + * err = ( __nvrtcGetNVVMSize)(prog, nvvmSizeRet) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, size_t *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNVVMSize)(__pyx_v_prog, __pyx_v_nvvmSizeRet); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 214, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":215 + * raise RuntimeError('Function "nvrtcGetNVVMSize" not found') + * err = ( __nvrtcGetNVVMSize)(prog, nvvmSizeRet) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetNVVM(nvrtcProgram prog, char* nvvm) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":208 + * return err + * + * cdef nvrtcResult _nvrtcGetNVVMSize(nvrtcProgram prog, size_t* nvvmSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetNVVMSize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetNVVMSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":217 + * return err + * + * cdef nvrtcResult _nvrtcGetNVVM(nvrtcProgram prog, char* nvvm) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetNVVM + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetNVVM(nvrtcProgram __pyx_v_prog, char *__pyx_v_nvvm) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetNVVM", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":219 + * cdef nvrtcResult _nvrtcGetNVVM(nvrtcProgram prog, char* nvvm) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetNVVM + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetNVVM == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 219, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":220 + * global __nvrtcGetNVVM + * cuPythonInit() + * if __nvrtcGetNVVM == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetNVVM" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNVVM == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":221 + * cuPythonInit() + * if __nvrtcGetNVVM == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetNVVM" not found') + * err = ( __nvrtcGetNVVM)(prog, nvvm) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":222 + * if __nvrtcGetNVVM == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetNVVM" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetNVVM)(prog, nvvm) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetNVVM_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 222, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 222, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":221 + * cuPythonInit() + * if __nvrtcGetNVVM == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetNVVM" not found') + * err = ( __nvrtcGetNVVM)(prog, nvvm) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":220 + * global __nvrtcGetNVVM + * cuPythonInit() + * if __nvrtcGetNVVM == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetNVVM" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":223 + * with gil: + * raise RuntimeError('Function "nvrtcGetNVVM" not found') + * err = ( __nvrtcGetNVVM)(prog, nvvm) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, char *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNVVM)(__pyx_v_prog, __pyx_v_nvvm); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 223, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":224 + * raise RuntimeError('Function "nvrtcGetNVVM" not found') + * err = ( __nvrtcGetNVVM)(prog, nvvm) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetLTOIRSize(nvrtcProgram prog, size_t* LTOIRSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":217 + * return err + * + * cdef nvrtcResult _nvrtcGetNVVM(nvrtcProgram prog, char* nvvm) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetNVVM + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetNVVM", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":226 + * return err + * + * cdef nvrtcResult _nvrtcGetLTOIRSize(nvrtcProgram prog, size_t* LTOIRSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetLTOIRSize + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetLTOIRSize(nvrtcProgram __pyx_v_prog, size_t *__pyx_v_LTOIRSizeRet) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetLTOIRSize", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":228 + * cdef nvrtcResult _nvrtcGetLTOIRSize(nvrtcProgram prog, size_t* LTOIRSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetLTOIRSize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetLTOIRSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 228, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":229 + * global __nvrtcGetLTOIRSize + * cuPythonInit() + * if __nvrtcGetLTOIRSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetLTOIRSize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLTOIRSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":230 + * cuPythonInit() + * if __nvrtcGetLTOIRSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetLTOIRSize" not found') + * err = ( __nvrtcGetLTOIRSize)(prog, LTOIRSizeRet) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":231 + * if __nvrtcGetLTOIRSize == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetLTOIRSize" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetLTOIRSize)(prog, LTOIRSizeRet) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetLTOIRSize_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 231, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 231, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":230 + * cuPythonInit() + * if __nvrtcGetLTOIRSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetLTOIRSize" not found') + * err = ( __nvrtcGetLTOIRSize)(prog, LTOIRSizeRet) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":229 + * global __nvrtcGetLTOIRSize + * cuPythonInit() + * if __nvrtcGetLTOIRSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetLTOIRSize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":232 + * with gil: + * raise RuntimeError('Function "nvrtcGetLTOIRSize" not found') + * err = ( __nvrtcGetLTOIRSize)(prog, LTOIRSizeRet) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, size_t *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLTOIRSize)(__pyx_v_prog, __pyx_v_LTOIRSizeRet); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 232, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":233 + * raise RuntimeError('Function "nvrtcGetLTOIRSize" not found') + * err = ( __nvrtcGetLTOIRSize)(prog, LTOIRSizeRet) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetLTOIR(nvrtcProgram prog, char* LTOIR) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":226 + * return err + * + * cdef nvrtcResult _nvrtcGetLTOIRSize(nvrtcProgram prog, size_t* LTOIRSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetLTOIRSize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetLTOIRSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":235 + * return err + * + * cdef nvrtcResult _nvrtcGetLTOIR(nvrtcProgram prog, char* LTOIR) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetLTOIR + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetLTOIR(nvrtcProgram __pyx_v_prog, char *__pyx_v_LTOIR) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetLTOIR", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":237 + * cdef nvrtcResult _nvrtcGetLTOIR(nvrtcProgram prog, char* LTOIR) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetLTOIR + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetLTOIR == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 237, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":238 + * global __nvrtcGetLTOIR + * cuPythonInit() + * if __nvrtcGetLTOIR == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetLTOIR" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLTOIR == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":239 + * cuPythonInit() + * if __nvrtcGetLTOIR == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetLTOIR" not found') + * err = ( __nvrtcGetLTOIR)(prog, LTOIR) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":240 + * if __nvrtcGetLTOIR == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetLTOIR" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetLTOIR)(prog, LTOIR) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetLTOIR_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 240, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 240, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":239 + * cuPythonInit() + * if __nvrtcGetLTOIR == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetLTOIR" not found') + * err = ( __nvrtcGetLTOIR)(prog, LTOIR) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":238 + * global __nvrtcGetLTOIR + * cuPythonInit() + * if __nvrtcGetLTOIR == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetLTOIR" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":241 + * with gil: + * raise RuntimeError('Function "nvrtcGetLTOIR" not found') + * err = ( __nvrtcGetLTOIR)(prog, LTOIR) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, char *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLTOIR)(__pyx_v_prog, __pyx_v_LTOIR); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 241, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":242 + * raise RuntimeError('Function "nvrtcGetLTOIR" not found') + * err = ( __nvrtcGetLTOIR)(prog, LTOIR) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetOptiXIRSize(nvrtcProgram prog, size_t* optixirSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":235 + * return err + * + * cdef nvrtcResult _nvrtcGetLTOIR(nvrtcProgram prog, char* LTOIR) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetLTOIR + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetLTOIR", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":244 + * return err + * + * cdef nvrtcResult _nvrtcGetOptiXIRSize(nvrtcProgram prog, size_t* optixirSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetOptiXIRSize + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetOptiXIRSize(nvrtcProgram __pyx_v_prog, size_t *__pyx_v_optixirSizeRet) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetOptiXIRSize", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":246 + * cdef nvrtcResult _nvrtcGetOptiXIRSize(nvrtcProgram prog, size_t* optixirSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetOptiXIRSize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetOptiXIRSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 246, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":247 + * global __nvrtcGetOptiXIRSize + * cuPythonInit() + * if __nvrtcGetOptiXIRSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetOptiXIRSize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetOptiXIRSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":248 + * cuPythonInit() + * if __nvrtcGetOptiXIRSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetOptiXIRSize" not found') + * err = ( __nvrtcGetOptiXIRSize)(prog, optixirSizeRet) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":249 + * if __nvrtcGetOptiXIRSize == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetOptiXIRSize" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetOptiXIRSize)(prog, optixirSizeRet) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetOptiXIRSize_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 249, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 249, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":248 + * cuPythonInit() + * if __nvrtcGetOptiXIRSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetOptiXIRSize" not found') + * err = ( __nvrtcGetOptiXIRSize)(prog, optixirSizeRet) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":247 + * global __nvrtcGetOptiXIRSize + * cuPythonInit() + * if __nvrtcGetOptiXIRSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetOptiXIRSize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":250 + * with gil: + * raise RuntimeError('Function "nvrtcGetOptiXIRSize" not found') + * err = ( __nvrtcGetOptiXIRSize)(prog, optixirSizeRet) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, size_t *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetOptiXIRSize)(__pyx_v_prog, __pyx_v_optixirSizeRet); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 250, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":251 + * raise RuntimeError('Function "nvrtcGetOptiXIRSize" not found') + * err = ( __nvrtcGetOptiXIRSize)(prog, optixirSizeRet) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetOptiXIR(nvrtcProgram prog, char* optixir) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":244 + * return err + * + * cdef nvrtcResult _nvrtcGetOptiXIRSize(nvrtcProgram prog, size_t* optixirSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetOptiXIRSize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetOptiXIRSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":253 + * return err + * + * cdef nvrtcResult _nvrtcGetOptiXIR(nvrtcProgram prog, char* optixir) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetOptiXIR + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetOptiXIR(nvrtcProgram __pyx_v_prog, char *__pyx_v_optixir) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetOptiXIR", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":255 + * cdef nvrtcResult _nvrtcGetOptiXIR(nvrtcProgram prog, char* optixir) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetOptiXIR + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetOptiXIR == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 255, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":256 + * global __nvrtcGetOptiXIR + * cuPythonInit() + * if __nvrtcGetOptiXIR == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetOptiXIR" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetOptiXIR == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":257 + * cuPythonInit() + * if __nvrtcGetOptiXIR == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetOptiXIR" not found') + * err = ( __nvrtcGetOptiXIR)(prog, optixir) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":258 + * if __nvrtcGetOptiXIR == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetOptiXIR" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetOptiXIR)(prog, optixir) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetOptiXIR_not_fou}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 258, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 258, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":257 + * cuPythonInit() + * if __nvrtcGetOptiXIR == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetOptiXIR" not found') + * err = ( __nvrtcGetOptiXIR)(prog, optixir) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":256 + * global __nvrtcGetOptiXIR + * cuPythonInit() + * if __nvrtcGetOptiXIR == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetOptiXIR" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":259 + * with gil: + * raise RuntimeError('Function "nvrtcGetOptiXIR" not found') + * err = ( __nvrtcGetOptiXIR)(prog, optixir) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, char *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetOptiXIR)(__pyx_v_prog, __pyx_v_optixir); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 259, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":260 + * raise RuntimeError('Function "nvrtcGetOptiXIR" not found') + * err = ( __nvrtcGetOptiXIR)(prog, optixir) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetProgramLogSize(nvrtcProgram prog, size_t* logSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":253 + * return err + * + * cdef nvrtcResult _nvrtcGetOptiXIR(nvrtcProgram prog, char* optixir) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetOptiXIR + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetOptiXIR", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":262 + * return err + * + * cdef nvrtcResult _nvrtcGetProgramLogSize(nvrtcProgram prog, size_t* logSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetProgramLogSize + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetProgramLogSize(nvrtcProgram __pyx_v_prog, size_t *__pyx_v_logSizeRet) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetProgramLogSize", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":264 + * cdef nvrtcResult _nvrtcGetProgramLogSize(nvrtcProgram prog, size_t* logSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetProgramLogSize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetProgramLogSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 264, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":265 + * global __nvrtcGetProgramLogSize + * cuPythonInit() + * if __nvrtcGetProgramLogSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetProgramLogSize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetProgramLogSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":266 + * cuPythonInit() + * if __nvrtcGetProgramLogSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetProgramLogSize" not found') + * err = ( __nvrtcGetProgramLogSize)(prog, logSizeRet) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":267 + * if __nvrtcGetProgramLogSize == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetProgramLogSize" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetProgramLogSize)(prog, logSizeRet) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetProgramLogSize}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 267, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 267, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":266 + * cuPythonInit() + * if __nvrtcGetProgramLogSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetProgramLogSize" not found') + * err = ( __nvrtcGetProgramLogSize)(prog, logSizeRet) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":265 + * global __nvrtcGetProgramLogSize + * cuPythonInit() + * if __nvrtcGetProgramLogSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetProgramLogSize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":268 + * with gil: + * raise RuntimeError('Function "nvrtcGetProgramLogSize" not found') + * err = ( __nvrtcGetProgramLogSize)(prog, logSizeRet) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, size_t *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetProgramLogSize)(__pyx_v_prog, __pyx_v_logSizeRet); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 268, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":269 + * raise RuntimeError('Function "nvrtcGetProgramLogSize" not found') + * err = ( __nvrtcGetProgramLogSize)(prog, logSizeRet) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetProgramLog(nvrtcProgram prog, char* log) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":262 + * return err + * + * cdef nvrtcResult _nvrtcGetProgramLogSize(nvrtcProgram prog, size_t* logSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetProgramLogSize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetProgramLogSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":271 + * return err + * + * cdef nvrtcResult _nvrtcGetProgramLog(nvrtcProgram prog, char* log) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetProgramLog + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetProgramLog(nvrtcProgram __pyx_v_prog, char *__pyx_v_log) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetProgramLog", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":273 + * cdef nvrtcResult _nvrtcGetProgramLog(nvrtcProgram prog, char* log) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetProgramLog + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetProgramLog == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 273, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":274 + * global __nvrtcGetProgramLog + * cuPythonInit() + * if __nvrtcGetProgramLog == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetProgramLog" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetProgramLog == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":275 + * cuPythonInit() + * if __nvrtcGetProgramLog == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetProgramLog" not found') + * err = ( __nvrtcGetProgramLog)(prog, log) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":276 + * if __nvrtcGetProgramLog == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetProgramLog" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetProgramLog)(prog, log) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetProgramLog_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 276, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 276, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":275 + * cuPythonInit() + * if __nvrtcGetProgramLog == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetProgramLog" not found') + * err = ( __nvrtcGetProgramLog)(prog, log) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":274 + * global __nvrtcGetProgramLog + * cuPythonInit() + * if __nvrtcGetProgramLog == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetProgramLog" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":277 + * with gil: + * raise RuntimeError('Function "nvrtcGetProgramLog" not found') + * err = ( __nvrtcGetProgramLog)(prog, log) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, char *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetProgramLog)(__pyx_v_prog, __pyx_v_log); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 277, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":278 + * raise RuntimeError('Function "nvrtcGetProgramLog" not found') + * err = ( __nvrtcGetProgramLog)(prog, log) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcAddNameExpression(nvrtcProgram prog, const char* name_expression) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":271 + * return err + * + * cdef nvrtcResult _nvrtcGetProgramLog(nvrtcProgram prog, char* log) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetProgramLog + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetProgramLog", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":280 + * return err + * + * cdef nvrtcResult _nvrtcAddNameExpression(nvrtcProgram prog, const char* name_expression) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcAddNameExpression + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcAddNameExpression(nvrtcProgram __pyx_v_prog, char const *__pyx_v_name_expression) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcAddNameExpression", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":282 + * cdef nvrtcResult _nvrtcAddNameExpression(nvrtcProgram prog, const char* name_expression) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcAddNameExpression + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcAddNameExpression == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 282, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":283 + * global __nvrtcAddNameExpression + * cuPythonInit() + * if __nvrtcAddNameExpression == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcAddNameExpression" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcAddNameExpression == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":284 + * cuPythonInit() + * if __nvrtcAddNameExpression == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcAddNameExpression" not found') + * err = ( __nvrtcAddNameExpression)(prog, name_expression) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":285 + * if __nvrtcAddNameExpression == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcAddNameExpression" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcAddNameExpression)(prog, name_expression) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcAddNameExpression}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 285, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 285, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":284 + * cuPythonInit() + * if __nvrtcAddNameExpression == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcAddNameExpression" not found') + * err = ( __nvrtcAddNameExpression)(prog, name_expression) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":283 + * global __nvrtcAddNameExpression + * cuPythonInit() + * if __nvrtcAddNameExpression == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcAddNameExpression" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":286 + * with gil: + * raise RuntimeError('Function "nvrtcAddNameExpression" not found') + * err = ( __nvrtcAddNameExpression)(prog, name_expression) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, char const *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcAddNameExpression)(__pyx_v_prog, __pyx_v_name_expression); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 286, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":287 + * raise RuntimeError('Function "nvrtcAddNameExpression" not found') + * err = ( __nvrtcAddNameExpression)(prog, name_expression) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetLoweredName(nvrtcProgram prog, const char* name_expression, const char** lowered_name) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":280 + * return err + * + * cdef nvrtcResult _nvrtcAddNameExpression(nvrtcProgram prog, const char* name_expression) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcAddNameExpression + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcAddNameExpression", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":289 + * return err + * + * cdef nvrtcResult _nvrtcGetLoweredName(nvrtcProgram prog, const char* name_expression, const char** lowered_name) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetLoweredName + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetLoweredName(nvrtcProgram __pyx_v_prog, char const *__pyx_v_name_expression, char const **__pyx_v_lowered_name) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetLoweredName", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":291 + * cdef nvrtcResult _nvrtcGetLoweredName(nvrtcProgram prog, const char* name_expression, const char** lowered_name) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetLoweredName + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetLoweredName == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 291, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":292 + * global __nvrtcGetLoweredName + * cuPythonInit() + * if __nvrtcGetLoweredName == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetLoweredName" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLoweredName == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":293 + * cuPythonInit() + * if __nvrtcGetLoweredName == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetLoweredName" not found') + * err = ( __nvrtcGetLoweredName)(prog, name_expression, lowered_name) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":294 + * if __nvrtcGetLoweredName == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetLoweredName" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetLoweredName)(prog, name_expression, lowered_name) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetLoweredName_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 294, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 294, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":293 + * cuPythonInit() + * if __nvrtcGetLoweredName == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetLoweredName" not found') + * err = ( __nvrtcGetLoweredName)(prog, name_expression, lowered_name) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":292 + * global __nvrtcGetLoweredName + * cuPythonInit() + * if __nvrtcGetLoweredName == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetLoweredName" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":295 + * with gil: + * raise RuntimeError('Function "nvrtcGetLoweredName" not found') + * err = ( __nvrtcGetLoweredName)(prog, name_expression, lowered_name) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, char const *, char const **))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLoweredName)(__pyx_v_prog, __pyx_v_name_expression, __pyx_v_lowered_name); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 295, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":296 + * raise RuntimeError('Function "nvrtcGetLoweredName" not found') + * err = ( __nvrtcGetLoweredName)(prog, name_expression, lowered_name) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetPCHHeapSize(size_t* ret) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":289 + * return err + * + * cdef nvrtcResult _nvrtcGetLoweredName(nvrtcProgram prog, const char* name_expression, const char** lowered_name) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetLoweredName + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetLoweredName", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":298 + * return err + * + * cdef nvrtcResult _nvrtcGetPCHHeapSize(size_t* ret) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetPCHHeapSize + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetPCHHeapSize(size_t *__pyx_v_ret) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetPCHHeapSize", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":300 + * cdef nvrtcResult _nvrtcGetPCHHeapSize(size_t* ret) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetPCHHeapSize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetPCHHeapSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 300, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":301 + * global __nvrtcGetPCHHeapSize + * cuPythonInit() + * if __nvrtcGetPCHHeapSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetPCHHeapSize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHHeapSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":302 + * cuPythonInit() + * if __nvrtcGetPCHHeapSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetPCHHeapSize" not found') + * err = ( __nvrtcGetPCHHeapSize)(ret) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":303 + * if __nvrtcGetPCHHeapSize == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetPCHHeapSize" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetPCHHeapSize)(ret) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetPCHHeapSize_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 303, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 303, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":302 + * cuPythonInit() + * if __nvrtcGetPCHHeapSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetPCHHeapSize" not found') + * err = ( __nvrtcGetPCHHeapSize)(ret) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":301 + * global __nvrtcGetPCHHeapSize + * cuPythonInit() + * if __nvrtcGetPCHHeapSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetPCHHeapSize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":304 + * with gil: + * raise RuntimeError('Function "nvrtcGetPCHHeapSize" not found') + * err = ( __nvrtcGetPCHHeapSize)(ret) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(size_t *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHHeapSize)(__pyx_v_ret); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 304, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":305 + * raise RuntimeError('Function "nvrtcGetPCHHeapSize" not found') + * err = ( __nvrtcGetPCHHeapSize)(ret) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcSetPCHHeapSize(size_t size) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":298 + * return err + * + * cdef nvrtcResult _nvrtcGetPCHHeapSize(size_t* ret) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetPCHHeapSize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetPCHHeapSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":307 + * return err + * + * cdef nvrtcResult _nvrtcSetPCHHeapSize(size_t size) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcSetPCHHeapSize + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcSetPCHHeapSize(size_t __pyx_v_size) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcSetPCHHeapSize", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":309 + * cdef nvrtcResult _nvrtcSetPCHHeapSize(size_t size) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcSetPCHHeapSize + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcSetPCHHeapSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 309, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":310 + * global __nvrtcSetPCHHeapSize + * cuPythonInit() + * if __nvrtcSetPCHHeapSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcSetPCHHeapSize" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcSetPCHHeapSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":311 + * cuPythonInit() + * if __nvrtcSetPCHHeapSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcSetPCHHeapSize" not found') + * err = ( __nvrtcSetPCHHeapSize)(size) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":312 + * if __nvrtcSetPCHHeapSize == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcSetPCHHeapSize" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcSetPCHHeapSize)(size) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcSetPCHHeapSize_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 312, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 312, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":311 + * cuPythonInit() + * if __nvrtcSetPCHHeapSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcSetPCHHeapSize" not found') + * err = ( __nvrtcSetPCHHeapSize)(size) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":310 + * global __nvrtcSetPCHHeapSize + * cuPythonInit() + * if __nvrtcSetPCHHeapSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcSetPCHHeapSize" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":313 + * with gil: + * raise RuntimeError('Function "nvrtcSetPCHHeapSize" not found') + * err = ( __nvrtcSetPCHHeapSize)(size) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(size_t))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcSetPCHHeapSize)(__pyx_v_size); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 313, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":314 + * raise RuntimeError('Function "nvrtcSetPCHHeapSize" not found') + * err = ( __nvrtcSetPCHHeapSize)(size) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetPCHCreateStatus(nvrtcProgram prog) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":307 + * return err + * + * cdef nvrtcResult _nvrtcSetPCHHeapSize(size_t size) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcSetPCHHeapSize + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcSetPCHHeapSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":316 + * return err + * + * cdef nvrtcResult _nvrtcGetPCHCreateStatus(nvrtcProgram prog) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetPCHCreateStatus + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetPCHCreateStatus(nvrtcProgram __pyx_v_prog) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetPCHCreateStatus", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":318 + * cdef nvrtcResult _nvrtcGetPCHCreateStatus(nvrtcProgram prog) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetPCHCreateStatus + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetPCHCreateStatus == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 318, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":319 + * global __nvrtcGetPCHCreateStatus + * cuPythonInit() + * if __nvrtcGetPCHCreateStatus == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetPCHCreateStatus" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHCreateStatus == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":320 + * cuPythonInit() + * if __nvrtcGetPCHCreateStatus == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetPCHCreateStatus" not found') + * err = ( __nvrtcGetPCHCreateStatus)(prog) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":321 + * if __nvrtcGetPCHCreateStatus == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetPCHCreateStatus" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetPCHCreateStatus)(prog) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetPCHCreateStatus}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 321, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 321, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":320 + * cuPythonInit() + * if __nvrtcGetPCHCreateStatus == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetPCHCreateStatus" not found') + * err = ( __nvrtcGetPCHCreateStatus)(prog) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":319 + * global __nvrtcGetPCHCreateStatus + * cuPythonInit() + * if __nvrtcGetPCHCreateStatus == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetPCHCreateStatus" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":322 + * with gil: + * raise RuntimeError('Function "nvrtcGetPCHCreateStatus" not found') + * err = ( __nvrtcGetPCHCreateStatus)(prog) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHCreateStatus)(__pyx_v_prog); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 322, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":323 + * raise RuntimeError('Function "nvrtcGetPCHCreateStatus" not found') + * err = ( __nvrtcGetPCHCreateStatus)(prog) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcGetPCHHeapSizeRequired(nvrtcProgram prog, size_t* size) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":316 + * return err + * + * cdef nvrtcResult _nvrtcGetPCHCreateStatus(nvrtcProgram prog) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetPCHCreateStatus + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetPCHCreateStatus", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":325 + * return err + * + * cdef nvrtcResult _nvrtcGetPCHHeapSizeRequired(nvrtcProgram prog, size_t* size) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetPCHHeapSizeRequired + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetPCHHeapSizeRequired(nvrtcProgram __pyx_v_prog, size_t *__pyx_v_size) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcGetPCHHeapSizeRequired", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":327 + * cdef nvrtcResult _nvrtcGetPCHHeapSizeRequired(nvrtcProgram prog, size_t* size) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcGetPCHHeapSizeRequired + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcGetPCHHeapSizeRequired == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 327, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":328 + * global __nvrtcGetPCHHeapSizeRequired + * cuPythonInit() + * if __nvrtcGetPCHHeapSizeRequired == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetPCHHeapSizeRequired" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHHeapSizeRequired == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":329 + * cuPythonInit() + * if __nvrtcGetPCHHeapSizeRequired == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetPCHHeapSizeRequired" not found') + * err = ( __nvrtcGetPCHHeapSizeRequired)(prog, size) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":330 + * if __nvrtcGetPCHHeapSizeRequired == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcGetPCHHeapSizeRequired" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcGetPCHHeapSizeRequired)(prog, size) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcGetPCHHeapSizeRequ}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 330, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 330, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":329 + * cuPythonInit() + * if __nvrtcGetPCHHeapSizeRequired == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcGetPCHHeapSizeRequired" not found') + * err = ( __nvrtcGetPCHHeapSizeRequired)(prog, size) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":328 + * global __nvrtcGetPCHHeapSizeRequired + * cuPythonInit() + * if __nvrtcGetPCHHeapSizeRequired == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcGetPCHHeapSizeRequired" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":331 + * with gil: + * raise RuntimeError('Function "nvrtcGetPCHHeapSizeRequired" not found') + * err = ( __nvrtcGetPCHHeapSizeRequired)(prog, size) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, size_t *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHHeapSizeRequired)(__pyx_v_prog, __pyx_v_size); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 331, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":332 + * raise RuntimeError('Function "nvrtcGetPCHHeapSizeRequired" not found') + * err = ( __nvrtcGetPCHHeapSizeRequired)(prog, size) + * return err # <<<<<<<<<<<<<< + * + * cdef nvrtcResult _nvrtcSetFlowCallback(nvrtcProgram prog, void* callback, void* payload) except ?NVRTC_ERROR_INVALID_INPUT nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":325 + * return err + * + * cdef nvrtcResult _nvrtcGetPCHHeapSizeRequired(nvrtcProgram prog, size_t* size) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcGetPCHHeapSizeRequired + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcGetPCHHeapSizeRequired", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":334 + * return err + * + * cdef nvrtcResult _nvrtcSetFlowCallback(nvrtcProgram prog, void* callback, void* payload) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcSetFlowCallback + * cuPythonInit() +*/ + +static nvrtcResult __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcSetFlowCallback(nvrtcProgram __pyx_v_prog, void *__pyx_v_callback, void *__pyx_v_payload) { + nvrtcResult __pyx_v_err; + nvrtcResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + nvrtcResult __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvrtcSetFlowCallback", 1); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":336 + * cdef nvrtcResult _nvrtcSetFlowCallback(nvrtcProgram prog, void* callback, void* payload) except ?NVRTC_ERROR_INVALID_INPUT nogil: + * global __nvrtcSetFlowCallback + * cuPythonInit() # <<<<<<<<<<<<<< + * if __nvrtcSetFlowCallback == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 336, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":337 + * global __nvrtcSetFlowCallback + * cuPythonInit() + * if __nvrtcSetFlowCallback == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcSetFlowCallback" not found') +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcSetFlowCallback == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":338 + * cuPythonInit() + * if __nvrtcSetFlowCallback == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcSetFlowCallback" not found') + * err = ( __nvrtcSetFlowCallback)(prog, callback, payload) +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":339 + * if __nvrtcSetFlowCallback == NULL: + * with gil: + * raise RuntimeError('Function "nvrtcSetFlowCallback" not found') # <<<<<<<<<<<<<< + * err = ( __nvrtcSetFlowCallback)(prog, callback, payload) + * return err +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_5 = __pyx_builtin_RuntimeError; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Function_nvrtcSetFlowCallback_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 339, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 339, __pyx_L5_error) + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":338 + * cuPythonInit() + * if __nvrtcSetFlowCallback == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise RuntimeError('Function "nvrtcSetFlowCallback" not found') + * err = ( __nvrtcSetFlowCallback)(prog, callback, payload) +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":337 + * global __nvrtcSetFlowCallback + * cuPythonInit() + * if __nvrtcSetFlowCallback == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise RuntimeError('Function "nvrtcSetFlowCallback" not found') +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":340 + * with gil: + * raise RuntimeError('Function "nvrtcSetFlowCallback" not found') + * err = ( __nvrtcSetFlowCallback)(prog, callback, payload) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = ((nvrtcResult (*)(nvrtcProgram, void *, void *))__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcSetFlowCallback)(__pyx_v_prog, __pyx_v_callback, __pyx_v_payload); if (unlikely(__pyx_t_7 == ((nvrtcResult)NVRTC_ERROR_INVALID_INPUT) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 340, __pyx_L1_error) + __pyx_v_err = __pyx_t_7; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":341 + * raise RuntimeError('Function "nvrtcSetFlowCallback" not found') + * err = ( __nvrtcSetFlowCallback)(prog, callback, payload) + * return err # <<<<<<<<<<<<<< + * + * cdef dict func_ptrs = None +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":334 + * return err + * + * cdef nvrtcResult _nvrtcSetFlowCallback(nvrtcProgram prog, void* callback, void* payload) except ?NVRTC_ERROR_INVALID_INPUT nogil: # <<<<<<<<<<<<<< + * global __nvrtcSetFlowCallback + * cuPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._nvrtcSetFlowCallback", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NVRTC_ERROR_INVALID_INPUT; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":345 + * cdef dict func_ptrs = None + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + +static PyObject *__pyx_pw_4cuda_8bindings_9_bindings_7cynvrtc_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__inspect_function_pointers(CYTHON_UNUSED int __pyx_skip_dispatch) { + PyObject *__pyx_v_data = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointers", 0); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":347 + * cpdef dict _inspect_function_pointers(): + * global func_ptrs + * if func_ptrs is not None: # <<<<<<<<<<<<<< + * return func_ptrs + * +*/ + __pyx_t_1 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs != ((PyObject*)Py_None)); + if (__pyx_t_1) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":348 + * global func_ptrs + * if func_ptrs is not None: + * return func_ptrs # <<<<<<<<<<<<<< + * + * cuPythonInit() +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs); + __pyx_r = __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":347 + * cpdef dict _inspect_function_pointers(): + * global func_ptrs + * if func_ptrs is not None: # <<<<<<<<<<<<<< + * return func_ptrs + * +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":350 + * return func_ptrs + * + * cuPythonInit() # <<<<<<<<<<<<<< + * cdef dict data = {} + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc_cuPythonInit(); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(0, 350, __pyx_L1_error) + + /* "cuda/bindings/_bindings/cynvrtc.pyx":351 + * + * cuPythonInit() + * cdef dict data = {} # <<<<<<<<<<<<<< + * + * global __nvrtcGetErrorString +*/ + __pyx_t_3 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 351, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_v_data = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":354 + * + * global __nvrtcGetErrorString + * data["__nvrtcGetErrorString"] = __nvrtcGetErrorString # <<<<<<<<<<<<<< + * global __nvrtcVersion + * data["__nvrtcVersion"] = __nvrtcVersion +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetErrorString)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 354, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetErrorString, __pyx_t_3) < 0))) __PYX_ERR(0, 354, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":356 + * data["__nvrtcGetErrorString"] = __nvrtcGetErrorString + * global __nvrtcVersion + * data["__nvrtcVersion"] = __nvrtcVersion # <<<<<<<<<<<<<< + * global __nvrtcGetNumSupportedArchs + * data["__nvrtcGetNumSupportedArchs"] = __nvrtcGetNumSupportedArchs +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcVersion)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcVersion, __pyx_t_3) < 0))) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":358 + * data["__nvrtcVersion"] = __nvrtcVersion + * global __nvrtcGetNumSupportedArchs + * data["__nvrtcGetNumSupportedArchs"] = __nvrtcGetNumSupportedArchs # <<<<<<<<<<<<<< + * global __nvrtcGetSupportedArchs + * data["__nvrtcGetSupportedArchs"] = __nvrtcGetSupportedArchs +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNumSupportedArchs)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 358, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetNumSupportedArchs, __pyx_t_3) < 0))) __PYX_ERR(0, 358, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":360 + * data["__nvrtcGetNumSupportedArchs"] = __nvrtcGetNumSupportedArchs + * global __nvrtcGetSupportedArchs + * data["__nvrtcGetSupportedArchs"] = __nvrtcGetSupportedArchs # <<<<<<<<<<<<<< + * global __nvrtcCreateProgram + * data["__nvrtcCreateProgram"] = __nvrtcCreateProgram +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetSupportedArchs)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 360, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetSupportedArchs, __pyx_t_3) < 0))) __PYX_ERR(0, 360, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":362 + * data["__nvrtcGetSupportedArchs"] = __nvrtcGetSupportedArchs + * global __nvrtcCreateProgram + * data["__nvrtcCreateProgram"] = __nvrtcCreateProgram # <<<<<<<<<<<<<< + * global __nvrtcDestroyProgram + * data["__nvrtcDestroyProgram"] = __nvrtcDestroyProgram +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcCreateProgram)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 362, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcCreateProgram, __pyx_t_3) < 0))) __PYX_ERR(0, 362, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":364 + * data["__nvrtcCreateProgram"] = __nvrtcCreateProgram + * global __nvrtcDestroyProgram + * data["__nvrtcDestroyProgram"] = __nvrtcDestroyProgram # <<<<<<<<<<<<<< + * global __nvrtcCompileProgram + * data["__nvrtcCompileProgram"] = __nvrtcCompileProgram +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcDestroyProgram)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 364, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcDestroyProgram, __pyx_t_3) < 0))) __PYX_ERR(0, 364, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":366 + * data["__nvrtcDestroyProgram"] = __nvrtcDestroyProgram + * global __nvrtcCompileProgram + * data["__nvrtcCompileProgram"] = __nvrtcCompileProgram # <<<<<<<<<<<<<< + * global __nvrtcGetPTXSize + * data["__nvrtcGetPTXSize"] = __nvrtcGetPTXSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcCompileProgram)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 366, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcCompileProgram, __pyx_t_3) < 0))) __PYX_ERR(0, 366, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":368 + * data["__nvrtcCompileProgram"] = __nvrtcCompileProgram + * global __nvrtcGetPTXSize + * data["__nvrtcGetPTXSize"] = __nvrtcGetPTXSize # <<<<<<<<<<<<<< + * global __nvrtcGetPTX + * data["__nvrtcGetPTX"] = __nvrtcGetPTX +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPTXSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 368, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetPTXSize, __pyx_t_3) < 0))) __PYX_ERR(0, 368, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":370 + * data["__nvrtcGetPTXSize"] = __nvrtcGetPTXSize + * global __nvrtcGetPTX + * data["__nvrtcGetPTX"] = __nvrtcGetPTX # <<<<<<<<<<<<<< + * global __nvrtcGetCUBINSize + * data["__nvrtcGetCUBINSize"] = __nvrtcGetCUBINSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPTX)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 370, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetPTX, __pyx_t_3) < 0))) __PYX_ERR(0, 370, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":372 + * data["__nvrtcGetPTX"] = __nvrtcGetPTX + * global __nvrtcGetCUBINSize + * data["__nvrtcGetCUBINSize"] = __nvrtcGetCUBINSize # <<<<<<<<<<<<<< + * global __nvrtcGetCUBIN + * data["__nvrtcGetCUBIN"] = __nvrtcGetCUBIN +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetCUBINSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 372, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetCUBINSize, __pyx_t_3) < 0))) __PYX_ERR(0, 372, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":374 + * data["__nvrtcGetCUBINSize"] = __nvrtcGetCUBINSize + * global __nvrtcGetCUBIN + * data["__nvrtcGetCUBIN"] = __nvrtcGetCUBIN # <<<<<<<<<<<<<< + * global __nvrtcGetNVVMSize + * data["__nvrtcGetNVVMSize"] = __nvrtcGetNVVMSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetCUBIN)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetCUBIN, __pyx_t_3) < 0))) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":376 + * data["__nvrtcGetCUBIN"] = __nvrtcGetCUBIN + * global __nvrtcGetNVVMSize + * data["__nvrtcGetNVVMSize"] = __nvrtcGetNVVMSize # <<<<<<<<<<<<<< + * global __nvrtcGetNVVM + * data["__nvrtcGetNVVM"] = __nvrtcGetNVVM +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNVVMSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 376, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetNVVMSize, __pyx_t_3) < 0))) __PYX_ERR(0, 376, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":378 + * data["__nvrtcGetNVVMSize"] = __nvrtcGetNVVMSize + * global __nvrtcGetNVVM + * data["__nvrtcGetNVVM"] = __nvrtcGetNVVM # <<<<<<<<<<<<<< + * global __nvrtcGetLTOIRSize + * data["__nvrtcGetLTOIRSize"] = __nvrtcGetLTOIRSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNVVM)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 378, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetNVVM, __pyx_t_3) < 0))) __PYX_ERR(0, 378, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":380 + * data["__nvrtcGetNVVM"] = __nvrtcGetNVVM + * global __nvrtcGetLTOIRSize + * data["__nvrtcGetLTOIRSize"] = __nvrtcGetLTOIRSize # <<<<<<<<<<<<<< + * global __nvrtcGetLTOIR + * data["__nvrtcGetLTOIR"] = __nvrtcGetLTOIR +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLTOIRSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 380, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetLTOIRSize, __pyx_t_3) < 0))) __PYX_ERR(0, 380, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":382 + * data["__nvrtcGetLTOIRSize"] = __nvrtcGetLTOIRSize + * global __nvrtcGetLTOIR + * data["__nvrtcGetLTOIR"] = __nvrtcGetLTOIR # <<<<<<<<<<<<<< + * global __nvrtcGetOptiXIRSize + * data["__nvrtcGetOptiXIRSize"] = __nvrtcGetOptiXIRSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLTOIR)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetLTOIR, __pyx_t_3) < 0))) __PYX_ERR(0, 382, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":384 + * data["__nvrtcGetLTOIR"] = __nvrtcGetLTOIR + * global __nvrtcGetOptiXIRSize + * data["__nvrtcGetOptiXIRSize"] = __nvrtcGetOptiXIRSize # <<<<<<<<<<<<<< + * global __nvrtcGetOptiXIR + * data["__nvrtcGetOptiXIR"] = __nvrtcGetOptiXIR +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetOptiXIRSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 384, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetOptiXIRSize, __pyx_t_3) < 0))) __PYX_ERR(0, 384, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":386 + * data["__nvrtcGetOptiXIRSize"] = __nvrtcGetOptiXIRSize + * global __nvrtcGetOptiXIR + * data["__nvrtcGetOptiXIR"] = __nvrtcGetOptiXIR # <<<<<<<<<<<<<< + * global __nvrtcGetProgramLogSize + * data["__nvrtcGetProgramLogSize"] = __nvrtcGetProgramLogSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetOptiXIR)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 386, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetOptiXIR, __pyx_t_3) < 0))) __PYX_ERR(0, 386, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":388 + * data["__nvrtcGetOptiXIR"] = __nvrtcGetOptiXIR + * global __nvrtcGetProgramLogSize + * data["__nvrtcGetProgramLogSize"] = __nvrtcGetProgramLogSize # <<<<<<<<<<<<<< + * global __nvrtcGetProgramLog + * data["__nvrtcGetProgramLog"] = __nvrtcGetProgramLog +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetProgramLogSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 388, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetProgramLogSize, __pyx_t_3) < 0))) __PYX_ERR(0, 388, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":390 + * data["__nvrtcGetProgramLogSize"] = __nvrtcGetProgramLogSize + * global __nvrtcGetProgramLog + * data["__nvrtcGetProgramLog"] = __nvrtcGetProgramLog # <<<<<<<<<<<<<< + * global __nvrtcAddNameExpression + * data["__nvrtcAddNameExpression"] = __nvrtcAddNameExpression +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetProgramLog)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 390, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetProgramLog, __pyx_t_3) < 0))) __PYX_ERR(0, 390, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":392 + * data["__nvrtcGetProgramLog"] = __nvrtcGetProgramLog + * global __nvrtcAddNameExpression + * data["__nvrtcAddNameExpression"] = __nvrtcAddNameExpression # <<<<<<<<<<<<<< + * global __nvrtcGetLoweredName + * data["__nvrtcGetLoweredName"] = __nvrtcGetLoweredName +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcAddNameExpression)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 392, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcAddNameExpression, __pyx_t_3) < 0))) __PYX_ERR(0, 392, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":394 + * data["__nvrtcAddNameExpression"] = __nvrtcAddNameExpression + * global __nvrtcGetLoweredName + * data["__nvrtcGetLoweredName"] = __nvrtcGetLoweredName # <<<<<<<<<<<<<< + * global __nvrtcGetPCHHeapSize + * data["__nvrtcGetPCHHeapSize"] = __nvrtcGetPCHHeapSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLoweredName)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 394, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetLoweredName, __pyx_t_3) < 0))) __PYX_ERR(0, 394, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":396 + * data["__nvrtcGetLoweredName"] = __nvrtcGetLoweredName + * global __nvrtcGetPCHHeapSize + * data["__nvrtcGetPCHHeapSize"] = __nvrtcGetPCHHeapSize # <<<<<<<<<<<<<< + * global __nvrtcSetPCHHeapSize + * data["__nvrtcSetPCHHeapSize"] = __nvrtcSetPCHHeapSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHHeapSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 396, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetPCHHeapSize, __pyx_t_3) < 0))) __PYX_ERR(0, 396, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":398 + * data["__nvrtcGetPCHHeapSize"] = __nvrtcGetPCHHeapSize + * global __nvrtcSetPCHHeapSize + * data["__nvrtcSetPCHHeapSize"] = __nvrtcSetPCHHeapSize # <<<<<<<<<<<<<< + * global __nvrtcGetPCHCreateStatus + * data["__nvrtcGetPCHCreateStatus"] = __nvrtcGetPCHCreateStatus +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcSetPCHHeapSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 398, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcSetPCHHeapSize, __pyx_t_3) < 0))) __PYX_ERR(0, 398, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":400 + * data["__nvrtcSetPCHHeapSize"] = __nvrtcSetPCHHeapSize + * global __nvrtcGetPCHCreateStatus + * data["__nvrtcGetPCHCreateStatus"] = __nvrtcGetPCHCreateStatus # <<<<<<<<<<<<<< + * global __nvrtcGetPCHHeapSizeRequired + * data["__nvrtcGetPCHHeapSizeRequired"] = __nvrtcGetPCHHeapSizeRequired +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHCreateStatus)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 400, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetPCHCreateStatus, __pyx_t_3) < 0))) __PYX_ERR(0, 400, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":402 + * data["__nvrtcGetPCHCreateStatus"] = __nvrtcGetPCHCreateStatus + * global __nvrtcGetPCHHeapSizeRequired + * data["__nvrtcGetPCHHeapSizeRequired"] = __nvrtcGetPCHHeapSizeRequired # <<<<<<<<<<<<<< + * global __nvrtcSetFlowCallback + * data["__nvrtcSetFlowCallback"] = __nvrtcSetFlowCallback +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHHeapSizeRequired)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 402, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcGetPCHHeapSizeRequired, __pyx_t_3) < 0))) __PYX_ERR(0, 402, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":404 + * data["__nvrtcGetPCHHeapSizeRequired"] = __nvrtcGetPCHHeapSizeRequired + * global __nvrtcSetFlowCallback + * data["__nvrtcSetFlowCallback"] = __nvrtcSetFlowCallback # <<<<<<<<<<<<<< + * + * func_ptrs = data +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcSetFlowCallback)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 404, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvrtcSetFlowCallback, __pyx_t_3) < 0))) __PYX_ERR(0, 404, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":406 + * data["__nvrtcSetFlowCallback"] = __nvrtcSetFlowCallback + * + * func_ptrs = data # <<<<<<<<<<<<<< + * return data + * +*/ + __Pyx_INCREF(__pyx_v_data); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs, __pyx_v_data); + __Pyx_GIVEREF(__pyx_v_data); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":407 + * + * func_ptrs = data + * return data # <<<<<<<<<<<<<< + * + * cpdef _inspect_function_pointer(str name): +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_data); + __pyx_r = __pyx_v_data; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":345 + * cdef dict func_ptrs = None + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._inspect_function_pointers", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_data); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_4cuda_8bindings_9_bindings_7cynvrtc_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +PyDoc_STRVAR(__pyx_doc_4cuda_8bindings_9_bindings_7cynvrtc__inspect_function_pointers, "_inspect_function_pointers() -> dict"); +static PyMethodDef __pyx_mdef_4cuda_8bindings_9_bindings_7cynvrtc_1_inspect_function_pointers = {"_inspect_function_pointers", (PyCFunction)__pyx_pw_4cuda_8bindings_9_bindings_7cynvrtc_1_inspect_function_pointers, METH_NOARGS, __pyx_doc_4cuda_8bindings_9_bindings_7cynvrtc__inspect_function_pointers}; +static PyObject *__pyx_pw_4cuda_8bindings_9_bindings_7cynvrtc_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_inspect_function_pointers (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_4cuda_8bindings_9_bindings_7cynvrtc__inspect_function_pointers(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_4cuda_8bindings_9_bindings_7cynvrtc__inspect_function_pointers(CYTHON_UNUSED PyObject *__pyx_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointers", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__inspect_function_pointers(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 345, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._inspect_function_pointers", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cynvrtc.pyx":409 + * return data + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + +static PyObject *__pyx_pw_4cuda_8bindings_9_bindings_7cynvrtc_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__inspect_function_pointer(PyObject *__pyx_v_name, CYTHON_UNUSED int __pyx_skip_dispatch) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointer", 0); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":411 + * cpdef _inspect_function_pointer(str name): + * global func_ptrs + * if func_ptrs is None: # <<<<<<<<<<<<<< + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] +*/ + __pyx_t_1 = (__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs == ((PyObject*)Py_None)); + if (__pyx_t_1) { + + /* "cuda/bindings/_bindings/cynvrtc.pyx":412 + * global func_ptrs + * if func_ptrs is None: + * func_ptrs = _inspect_function_pointers() # <<<<<<<<<<<<<< + * return func_ptrs[name] +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__inspect_function_pointers(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 412, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs, ((PyObject*)__pyx_t_2)); + __Pyx_GIVEREF(__pyx_t_2); + __pyx_t_2 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":411 + * cpdef _inspect_function_pointer(str name): + * global func_ptrs + * if func_ptrs is None: # <<<<<<<<<<<<<< + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] +*/ + } + + /* "cuda/bindings/_bindings/cynvrtc.pyx":413 + * if func_ptrs is None: + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] # <<<<<<<<<<<<<< +*/ + __Pyx_XDECREF(__pyx_r); + if (unlikely(__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 413, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_PyDict_GetItem(__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs, __pyx_v_name); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 413, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":409 + * return data + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_4cuda_8bindings_9_bindings_7cynvrtc_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_4cuda_8bindings_9_bindings_7cynvrtc_2_inspect_function_pointer, "_inspect_function_pointer(str name)"); +static PyMethodDef __pyx_mdef_4cuda_8bindings_9_bindings_7cynvrtc_3_inspect_function_pointer = {"_inspect_function_pointer", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cuda_8bindings_9_bindings_7cynvrtc_3_inspect_function_pointer, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cuda_8bindings_9_bindings_7cynvrtc_2_inspect_function_pointer}; +static PyObject *__pyx_pw_4cuda_8bindings_9_bindings_7cynvrtc_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_name = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_inspect_function_pointer (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_name,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 409, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 409, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_inspect_function_pointer", 0) < (0)) __PYX_ERR(0, 409, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_inspect_function_pointer", 1, 1, 1, i); __PYX_ERR(0, 409, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 409, __pyx_L3_error) + } + __pyx_v_name = ((PyObject*)values[0]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_inspect_function_pointer", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 409, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_name), (&PyUnicode_Type), 1, "name", 1))) __PYX_ERR(0, 409, __pyx_L1_error) + __pyx_r = __pyx_pf_4cuda_8bindings_9_bindings_7cynvrtc_2_inspect_function_pointer(__pyx_self, __pyx_v_name); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_4cuda_8bindings_9_bindings_7cynvrtc_2_inspect_function_pointer(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_name) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointer", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__inspect_function_pointer(__pyx_v_name, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 409, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cuda.bindings._bindings.cynvrtc._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +/* #### Code section: module_exttypes ### */ + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___symbol_lock = Py_None; Py_INCREF(Py_None); + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs = ((PyObject*)Py_None); Py_INCREF(Py_None); + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + if (__Pyx_ExportFunction("_nvrtcGetErrorString", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetErrorString, "char const *(nvrtcResult)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcVersion", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcVersion, "nvrtcResult (int *, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetNumSupportedArchs", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetNumSupportedArchs, "nvrtcResult (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetSupportedArchs", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetSupportedArchs, "nvrtcResult (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcCreateProgram", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcCreateProgram, "nvrtcResult (nvrtcProgram *, char const *, char const *, int, char const **, char const **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcDestroyProgram", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcDestroyProgram, "nvrtcResult (nvrtcProgram *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcCompileProgram", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcCompileProgram, "nvrtcResult (nvrtcProgram, int, char const **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetPTXSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetPTXSize, "nvrtcResult (nvrtcProgram, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetPTX", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetPTX, "nvrtcResult (nvrtcProgram, char *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetCUBINSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetCUBINSize, "nvrtcResult (nvrtcProgram, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetCUBIN", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetCUBIN, "nvrtcResult (nvrtcProgram, char *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetNVVMSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetNVVMSize, "nvrtcResult (nvrtcProgram, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetNVVM", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetNVVM, "nvrtcResult (nvrtcProgram, char *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetLTOIRSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetLTOIRSize, "nvrtcResult (nvrtcProgram, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetLTOIR", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetLTOIR, "nvrtcResult (nvrtcProgram, char *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetOptiXIRSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetOptiXIRSize, "nvrtcResult (nvrtcProgram, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetOptiXIR", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetOptiXIR, "nvrtcResult (nvrtcProgram, char *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetProgramLogSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetProgramLogSize, "nvrtcResult (nvrtcProgram, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetProgramLog", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetProgramLog, "nvrtcResult (nvrtcProgram, char *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcAddNameExpression", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcAddNameExpression, "nvrtcResult (nvrtcProgram, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetLoweredName", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetLoweredName, "nvrtcResult (nvrtcProgram, char const *, char const **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetPCHHeapSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetPCHHeapSize, "nvrtcResult (size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcSetPCHHeapSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcSetPCHHeapSize, "nvrtcResult (size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetPCHCreateStatus", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetPCHCreateStatus, "nvrtcResult (nvrtcProgram)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcGetPCHHeapSizeRequired", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcGetPCHHeapSizeRequired, "nvrtcResult (nvrtcProgram, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvrtcSetFlowCallback", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_7cynvrtc__nvrtcSetFlowCallback, "nvrtcResult (nvrtcProgram, void *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_cynvrtc(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_cynvrtc}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "cynvrtc", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_cynvrtc(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_cynvrtc(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x03090000 + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_cynvrtc(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'cynvrtc' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "cynvrtc" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_NOT_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY +__Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); +if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); +} +#endif + +__Pyx_RefNannySetupContext("PyInit_cynvrtc", 0); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #ifdef __Pxy_PyFrame_Initialize_Offsets + __Pxy_PyFrame_Initialize_Offsets(); + #endif + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #if 0 || defined(__Pyx_CyFunction_USED) || defined(__Pyx_FusedFunction_USED) || defined(__Pyx_Coroutine_USED) || defined(__Pyx_Generator_USED) || defined(__Pyx_AsyncGen_USED) + if (__pyx_CommonTypesMetaclass_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_AsyncGen_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + if (__pyx_module_is_main_cuda__bindings___bindings__cynvrtc) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name_2, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "cuda.bindings._bindings.cynvrtc")) { + if (unlikely((PyDict_SetItemString(modules, "cuda.bindings._bindings.cynvrtc", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_function_export_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_type_init_code(__pyx_mstate); + (void)__Pyx_modinit_type_import_code(__pyx_mstate); + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + (void)__Pyx_modinit_function_import_code(__pyx_mstate); + /*--- Execution code ---*/ + + /* "cuda/bindings/_bindings/cynvrtc.pyx":6 + * # This code was automatically generated with version 12.9.0. Do not modify it directly. + * cimport cuda.bindings._lib.dlfcn as dlfcn + * from cuda.pathfinder import load_nvidia_dynamic_lib # <<<<<<<<<<<<<< + * from libc.stdint cimport intptr_t, uintptr_t + * import threading +*/ + __pyx_t_2 = __Pyx_PyList_Pack(1, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cuda_pathfinder, __pyx_t_2, 0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 6, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib, __pyx_t_2) < (0)) __PYX_ERR(0, 6, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":8 + * from cuda.pathfinder import load_nvidia_dynamic_lib + * from libc.stdint cimport intptr_t, uintptr_t + * import threading # <<<<<<<<<<<<<< + * + * cdef object __symbol_lock = threading.Lock() +*/ + __pyx_t_3 = __Pyx_ImportDottedModule(__pyx_mstate_global->__pyx_n_u_threading, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_threading, __pyx_t_3) < (0)) __PYX_ERR(0, 8, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":10 + * import threading + * + * cdef object __symbol_lock = threading.Lock() # <<<<<<<<<<<<<< + * cdef bint __cuPythonInit = False + * cdef void *__nvrtcGetErrorString = NULL +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_threading); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_Lock); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, NULL}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___symbol_lock); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___symbol_lock, __pyx_t_3); + __Pyx_GIVEREF(__pyx_t_3); + __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":11 + * + * cdef object __symbol_lock = threading.Lock() + * cdef bint __cuPythonInit = False # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetErrorString = NULL + * cdef void *__nvrtcVersion = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___cuPythonInit = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":12 + * cdef object __symbol_lock = threading.Lock() + * cdef bint __cuPythonInit = False + * cdef void *__nvrtcGetErrorString = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcVersion = NULL + * cdef void *__nvrtcGetNumSupportedArchs = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetErrorString = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":13 + * cdef bint __cuPythonInit = False + * cdef void *__nvrtcGetErrorString = NULL + * cdef void *__nvrtcVersion = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetNumSupportedArchs = NULL + * cdef void *__nvrtcGetSupportedArchs = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcVersion = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":14 + * cdef void *__nvrtcGetErrorString = NULL + * cdef void *__nvrtcVersion = NULL + * cdef void *__nvrtcGetNumSupportedArchs = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetSupportedArchs = NULL + * cdef void *__nvrtcCreateProgram = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNumSupportedArchs = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":15 + * cdef void *__nvrtcVersion = NULL + * cdef void *__nvrtcGetNumSupportedArchs = NULL + * cdef void *__nvrtcGetSupportedArchs = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcCreateProgram = NULL + * cdef void *__nvrtcDestroyProgram = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetSupportedArchs = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":16 + * cdef void *__nvrtcGetNumSupportedArchs = NULL + * cdef void *__nvrtcGetSupportedArchs = NULL + * cdef void *__nvrtcCreateProgram = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcDestroyProgram = NULL + * cdef void *__nvrtcCompileProgram = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcCreateProgram = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":17 + * cdef void *__nvrtcGetSupportedArchs = NULL + * cdef void *__nvrtcCreateProgram = NULL + * cdef void *__nvrtcDestroyProgram = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcCompileProgram = NULL + * cdef void *__nvrtcGetPTXSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcDestroyProgram = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":18 + * cdef void *__nvrtcCreateProgram = NULL + * cdef void *__nvrtcDestroyProgram = NULL + * cdef void *__nvrtcCompileProgram = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetPTXSize = NULL + * cdef void *__nvrtcGetPTX = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcCompileProgram = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":19 + * cdef void *__nvrtcDestroyProgram = NULL + * cdef void *__nvrtcCompileProgram = NULL + * cdef void *__nvrtcGetPTXSize = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetPTX = NULL + * cdef void *__nvrtcGetCUBINSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPTXSize = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":20 + * cdef void *__nvrtcCompileProgram = NULL + * cdef void *__nvrtcGetPTXSize = NULL + * cdef void *__nvrtcGetPTX = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetCUBINSize = NULL + * cdef void *__nvrtcGetCUBIN = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPTX = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":21 + * cdef void *__nvrtcGetPTXSize = NULL + * cdef void *__nvrtcGetPTX = NULL + * cdef void *__nvrtcGetCUBINSize = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetCUBIN = NULL + * cdef void *__nvrtcGetNVVMSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetCUBINSize = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":22 + * cdef void *__nvrtcGetPTX = NULL + * cdef void *__nvrtcGetCUBINSize = NULL + * cdef void *__nvrtcGetCUBIN = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetNVVMSize = NULL + * cdef void *__nvrtcGetNVVM = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetCUBIN = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":23 + * cdef void *__nvrtcGetCUBINSize = NULL + * cdef void *__nvrtcGetCUBIN = NULL + * cdef void *__nvrtcGetNVVMSize = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetNVVM = NULL + * cdef void *__nvrtcGetLTOIRSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNVVMSize = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":24 + * cdef void *__nvrtcGetCUBIN = NULL + * cdef void *__nvrtcGetNVVMSize = NULL + * cdef void *__nvrtcGetNVVM = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetLTOIRSize = NULL + * cdef void *__nvrtcGetLTOIR = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetNVVM = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":25 + * cdef void *__nvrtcGetNVVMSize = NULL + * cdef void *__nvrtcGetNVVM = NULL + * cdef void *__nvrtcGetLTOIRSize = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetLTOIR = NULL + * cdef void *__nvrtcGetOptiXIRSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLTOIRSize = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":26 + * cdef void *__nvrtcGetNVVM = NULL + * cdef void *__nvrtcGetLTOIRSize = NULL + * cdef void *__nvrtcGetLTOIR = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetOptiXIRSize = NULL + * cdef void *__nvrtcGetOptiXIR = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLTOIR = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":27 + * cdef void *__nvrtcGetLTOIRSize = NULL + * cdef void *__nvrtcGetLTOIR = NULL + * cdef void *__nvrtcGetOptiXIRSize = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetOptiXIR = NULL + * cdef void *__nvrtcGetProgramLogSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetOptiXIRSize = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":28 + * cdef void *__nvrtcGetLTOIR = NULL + * cdef void *__nvrtcGetOptiXIRSize = NULL + * cdef void *__nvrtcGetOptiXIR = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetProgramLogSize = NULL + * cdef void *__nvrtcGetProgramLog = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetOptiXIR = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":29 + * cdef void *__nvrtcGetOptiXIRSize = NULL + * cdef void *__nvrtcGetOptiXIR = NULL + * cdef void *__nvrtcGetProgramLogSize = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetProgramLog = NULL + * cdef void *__nvrtcAddNameExpression = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetProgramLogSize = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":30 + * cdef void *__nvrtcGetOptiXIR = NULL + * cdef void *__nvrtcGetProgramLogSize = NULL + * cdef void *__nvrtcGetProgramLog = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcAddNameExpression = NULL + * cdef void *__nvrtcGetLoweredName = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetProgramLog = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":31 + * cdef void *__nvrtcGetProgramLogSize = NULL + * cdef void *__nvrtcGetProgramLog = NULL + * cdef void *__nvrtcAddNameExpression = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetLoweredName = NULL + * cdef void *__nvrtcGetPCHHeapSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcAddNameExpression = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":32 + * cdef void *__nvrtcGetProgramLog = NULL + * cdef void *__nvrtcAddNameExpression = NULL + * cdef void *__nvrtcGetLoweredName = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetPCHHeapSize = NULL + * cdef void *__nvrtcSetPCHHeapSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetLoweredName = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":33 + * cdef void *__nvrtcAddNameExpression = NULL + * cdef void *__nvrtcGetLoweredName = NULL + * cdef void *__nvrtcGetPCHHeapSize = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcSetPCHHeapSize = NULL + * cdef void *__nvrtcGetPCHCreateStatus = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHHeapSize = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":34 + * cdef void *__nvrtcGetLoweredName = NULL + * cdef void *__nvrtcGetPCHHeapSize = NULL + * cdef void *__nvrtcSetPCHHeapSize = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetPCHCreateStatus = NULL + * cdef void *__nvrtcGetPCHHeapSizeRequired = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcSetPCHHeapSize = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":35 + * cdef void *__nvrtcGetPCHHeapSize = NULL + * cdef void *__nvrtcSetPCHHeapSize = NULL + * cdef void *__nvrtcGetPCHCreateStatus = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcGetPCHHeapSizeRequired = NULL + * cdef void *__nvrtcSetFlowCallback = NULL +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHCreateStatus = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":36 + * cdef void *__nvrtcSetPCHHeapSize = NULL + * cdef void *__nvrtcGetPCHCreateStatus = NULL + * cdef void *__nvrtcGetPCHHeapSizeRequired = NULL # <<<<<<<<<<<<<< + * cdef void *__nvrtcSetFlowCallback = NULL + * +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcGetPCHHeapSizeRequired = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":37 + * cdef void *__nvrtcGetPCHCreateStatus = NULL + * cdef void *__nvrtcGetPCHHeapSizeRequired = NULL + * cdef void *__nvrtcSetFlowCallback = NULL # <<<<<<<<<<<<<< + * + * cdef int _cuPythonInit() except -1 nogil: +*/ + __pyx_v_4cuda_8bindings_9_bindings_7cynvrtc___nvrtcSetFlowCallback = NULL; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":343 + * return err + * + * cdef dict func_ptrs = None # <<<<<<<<<<<<<< + * + * cpdef dict _inspect_function_pointers(): +*/ + __Pyx_INCREF(Py_None); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_bindings_7cynvrtc_func_ptrs, ((PyObject*)Py_None)); + __Pyx_GIVEREF(Py_None); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":345 + * cdef dict func_ptrs = None + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + __pyx_t_3 = __Pyx_CyFunction_New(&__pyx_mdef_4cuda_8bindings_9_bindings_7cynvrtc_1_inspect_function_pointers, 0, __pyx_mstate_global->__pyx_n_u_inspect_function_pointers, NULL, __pyx_mstate_global->__pyx_n_u_cuda_bindings__bindings_cynvrtc, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0])); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 345, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_inspect_function_pointers, __pyx_t_3) < (0)) __PYX_ERR(0, 345, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":409 + * return data + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + __pyx_t_3 = __Pyx_CyFunction_New(&__pyx_mdef_4cuda_8bindings_9_bindings_7cynvrtc_3_inspect_function_pointer, 0, __pyx_mstate_global->__pyx_n_u_inspect_function_pointer, NULL, __pyx_mstate_global->__pyx_n_u_cuda_bindings__bindings_cynvrtc, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[1])); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 409, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_inspect_function_pointer, __pyx_t_3) < (0)) __PYX_ERR(0, 409, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_bindings/cynvrtc.pyx":1 + * # SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # <<<<<<<<<<<<<< + * # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + * +*/ + __pyx_t_3 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_3) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init cuda.bindings._bindings.cynvrtc", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init cuda.bindings._bindings.cynvrtc"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ + +typedef struct { + const char *s; +#if 179 <= 65535 + const unsigned short n; +#elif 179 / 2 < INT_MAX + const unsigned int n; +#elif 179 / 2 < LONG_MAX + const unsigned long n; +#else + const Py_ssize_t n; +#endif +#if 1 <= 31 + const unsigned int encoding : 5; +#elif 1 <= 255 + const unsigned char encoding; +#elif 1 <= 65535 + const unsigned short encoding; +#else + const Py_ssize_t encoding; +#endif + const unsigned int is_unicode : 1; + const unsigned int intern : 1; +} __Pyx_StringTabEntry; +static const char * const __pyx_string_tab_encodings[] = { 0 }; +static const __Pyx_StringTabEntry __pyx_string_tab[] = { + {__pyx_k_, sizeof(__pyx_k_), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_ */ + {__pyx_k_Function_nvrtcAddNameExpression, sizeof(__pyx_k_Function_nvrtcAddNameExpression), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcAddNameExpression */ + {__pyx_k_Function_nvrtcCompileProgram_not, sizeof(__pyx_k_Function_nvrtcCompileProgram_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcCompileProgram_not */ + {__pyx_k_Function_nvrtcCreateProgram_not, sizeof(__pyx_k_Function_nvrtcCreateProgram_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcCreateProgram_not */ + {__pyx_k_Function_nvrtcDestroyProgram_not, sizeof(__pyx_k_Function_nvrtcDestroyProgram_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcDestroyProgram_not */ + {__pyx_k_Function_nvrtcGetCUBINSize_not_f, sizeof(__pyx_k_Function_nvrtcGetCUBINSize_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetCUBINSize_not_f */ + {__pyx_k_Function_nvrtcGetCUBIN_not_found, sizeof(__pyx_k_Function_nvrtcGetCUBIN_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetCUBIN_not_found */ + {__pyx_k_Function_nvrtcGetErrorString_not, sizeof(__pyx_k_Function_nvrtcGetErrorString_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetErrorString_not */ + {__pyx_k_Function_nvrtcGetLTOIRSize_not_f, sizeof(__pyx_k_Function_nvrtcGetLTOIRSize_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetLTOIRSize_not_f */ + {__pyx_k_Function_nvrtcGetLTOIR_not_found, sizeof(__pyx_k_Function_nvrtcGetLTOIR_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetLTOIR_not_found */ + {__pyx_k_Function_nvrtcGetLoweredName_not, sizeof(__pyx_k_Function_nvrtcGetLoweredName_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetLoweredName_not */ + {__pyx_k_Function_nvrtcGetNVVMSize_not_fo, sizeof(__pyx_k_Function_nvrtcGetNVVMSize_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetNVVMSize_not_fo */ + {__pyx_k_Function_nvrtcGetNVVM_not_found, sizeof(__pyx_k_Function_nvrtcGetNVVM_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetNVVM_not_found */ + {__pyx_k_Function_nvrtcGetNumSupportedArc, sizeof(__pyx_k_Function_nvrtcGetNumSupportedArc), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetNumSupportedArc */ + {__pyx_k_Function_nvrtcGetOptiXIRSize_not, sizeof(__pyx_k_Function_nvrtcGetOptiXIRSize_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetOptiXIRSize_not */ + {__pyx_k_Function_nvrtcGetOptiXIR_not_fou, sizeof(__pyx_k_Function_nvrtcGetOptiXIR_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetOptiXIR_not_fou */ + {__pyx_k_Function_nvrtcGetPCHCreateStatus, sizeof(__pyx_k_Function_nvrtcGetPCHCreateStatus), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetPCHCreateStatus */ + {__pyx_k_Function_nvrtcGetPCHHeapSizeRequ, sizeof(__pyx_k_Function_nvrtcGetPCHHeapSizeRequ), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetPCHHeapSizeRequ */ + {__pyx_k_Function_nvrtcGetPCHHeapSize_not, sizeof(__pyx_k_Function_nvrtcGetPCHHeapSize_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetPCHHeapSize_not */ + {__pyx_k_Function_nvrtcGetPTXSize_not_fou, sizeof(__pyx_k_Function_nvrtcGetPTXSize_not_fou), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetPTXSize_not_fou */ + {__pyx_k_Function_nvrtcGetPTX_not_found, sizeof(__pyx_k_Function_nvrtcGetPTX_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetPTX_not_found */ + {__pyx_k_Function_nvrtcGetProgramLogSize, sizeof(__pyx_k_Function_nvrtcGetProgramLogSize), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetProgramLogSize */ + {__pyx_k_Function_nvrtcGetProgramLog_not, sizeof(__pyx_k_Function_nvrtcGetProgramLog_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetProgramLog_not */ + {__pyx_k_Function_nvrtcGetSupportedArchs, sizeof(__pyx_k_Function_nvrtcGetSupportedArchs), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcGetSupportedArchs */ + {__pyx_k_Function_nvrtcSetFlowCallback_no, sizeof(__pyx_k_Function_nvrtcSetFlowCallback_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcSetFlowCallback_no */ + {__pyx_k_Function_nvrtcSetPCHHeapSize_not, sizeof(__pyx_k_Function_nvrtcSetPCHHeapSize_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcSetPCHHeapSize_not */ + {__pyx_k_Function_nvrtcVersion_not_found, sizeof(__pyx_k_Function_nvrtcVersion_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Function_nvrtcVersion_not_found */ + {__pyx_k_Lock, sizeof(__pyx_k_Lock), 0, 1, 1}, /* PyObject cname: __pyx_n_u_Lock */ + {__pyx_k_Note_that_Cython_is_deliberately, sizeof(__pyx_k_Note_that_Cython_is_deliberately), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Note_that_Cython_is_deliberately */ + {__pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_RuntimeError */ + {__pyx_k__2, sizeof(__pyx_k__2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u__2 */ + {__pyx_k_add_note, sizeof(__pyx_k_add_note), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_add_note */ + {__pyx_k_asyncio_coroutines, sizeof(__pyx_k_asyncio_coroutines), 0, 1, 1}, /* PyObject cname: __pyx_n_u_asyncio_coroutines */ + {__pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cline_in_traceback */ + {__pyx_k_cuda_bindings__bindings_cynvrtc, sizeof(__pyx_k_cuda_bindings__bindings_cynvrtc), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuda_bindings__bindings_cynvrtc */ + {__pyx_k_cuda_bindings__bindings_cynvrtc_2, sizeof(__pyx_k_cuda_bindings__bindings_cynvrtc_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_cuda_bindings__bindings_cynvrtc_2 */ + {__pyx_k_cuda_pathfinder, sizeof(__pyx_k_cuda_pathfinder), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuda_pathfinder */ + {__pyx_k_enter, sizeof(__pyx_k_enter), 0, 1, 1}, /* PyObject cname: __pyx_n_u_enter */ + {__pyx_k_exit, sizeof(__pyx_k_exit), 0, 1, 1}, /* PyObject cname: __pyx_n_u_exit */ + {__pyx_k_func, sizeof(__pyx_k_func), 0, 1, 1}, /* PyObject cname: __pyx_n_u_func */ + {__pyx_k_handle_uint, sizeof(__pyx_k_handle_uint), 0, 1, 1}, /* PyObject cname: __pyx_n_u_handle_uint */ + {__pyx_k_initializing, sizeof(__pyx_k_initializing), 0, 1, 1}, /* PyObject cname: __pyx_n_u_initializing */ + {__pyx_k_inspect_function_pointer, sizeof(__pyx_k_inspect_function_pointer), 0, 1, 1}, /* PyObject cname: __pyx_n_u_inspect_function_pointer */ + {__pyx_k_inspect_function_pointers, sizeof(__pyx_k_inspect_function_pointers), 0, 1, 1}, /* PyObject cname: __pyx_n_u_inspect_function_pointers */ + {__pyx_k_is_coroutine, sizeof(__pyx_k_is_coroutine), 0, 1, 1}, /* PyObject cname: __pyx_n_u_is_coroutine */ + {__pyx_k_load_nvidia_dynamic_lib, sizeof(__pyx_k_load_nvidia_dynamic_lib), 0, 1, 1}, /* PyObject cname: __pyx_n_u_load_nvidia_dynamic_lib */ + {__pyx_k_main, sizeof(__pyx_k_main), 0, 1, 1}, /* PyObject cname: __pyx_n_u_main */ + {__pyx_k_module, sizeof(__pyx_k_module), 0, 1, 1}, /* PyObject cname: __pyx_n_u_module */ + {__pyx_k_name, sizeof(__pyx_k_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name */ + {__pyx_k_name_2, sizeof(__pyx_k_name_2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name_2 */ + {__pyx_k_nvrtc, sizeof(__pyx_k_nvrtc), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtc */ + {__pyx_k_nvrtcAddNameExpression, sizeof(__pyx_k_nvrtcAddNameExpression), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcAddNameExpression */ + {__pyx_k_nvrtcCompileProgram, sizeof(__pyx_k_nvrtcCompileProgram), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcCompileProgram */ + {__pyx_k_nvrtcCreateProgram, sizeof(__pyx_k_nvrtcCreateProgram), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcCreateProgram */ + {__pyx_k_nvrtcDestroyProgram, sizeof(__pyx_k_nvrtcDestroyProgram), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcDestroyProgram */ + {__pyx_k_nvrtcGetCUBIN, sizeof(__pyx_k_nvrtcGetCUBIN), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetCUBIN */ + {__pyx_k_nvrtcGetCUBINSize, sizeof(__pyx_k_nvrtcGetCUBINSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetCUBINSize */ + {__pyx_k_nvrtcGetErrorString, sizeof(__pyx_k_nvrtcGetErrorString), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetErrorString */ + {__pyx_k_nvrtcGetLTOIR, sizeof(__pyx_k_nvrtcGetLTOIR), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetLTOIR */ + {__pyx_k_nvrtcGetLTOIRSize, sizeof(__pyx_k_nvrtcGetLTOIRSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetLTOIRSize */ + {__pyx_k_nvrtcGetLoweredName, sizeof(__pyx_k_nvrtcGetLoweredName), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetLoweredName */ + {__pyx_k_nvrtcGetNVVM, sizeof(__pyx_k_nvrtcGetNVVM), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetNVVM */ + {__pyx_k_nvrtcGetNVVMSize, sizeof(__pyx_k_nvrtcGetNVVMSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetNVVMSize */ + {__pyx_k_nvrtcGetNumSupportedArchs, sizeof(__pyx_k_nvrtcGetNumSupportedArchs), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetNumSupportedArchs */ + {__pyx_k_nvrtcGetOptiXIR, sizeof(__pyx_k_nvrtcGetOptiXIR), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetOptiXIR */ + {__pyx_k_nvrtcGetOptiXIRSize, sizeof(__pyx_k_nvrtcGetOptiXIRSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetOptiXIRSize */ + {__pyx_k_nvrtcGetPCHCreateStatus, sizeof(__pyx_k_nvrtcGetPCHCreateStatus), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetPCHCreateStatus */ + {__pyx_k_nvrtcGetPCHHeapSize, sizeof(__pyx_k_nvrtcGetPCHHeapSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetPCHHeapSize */ + {__pyx_k_nvrtcGetPCHHeapSizeRequired, sizeof(__pyx_k_nvrtcGetPCHHeapSizeRequired), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetPCHHeapSizeRequired */ + {__pyx_k_nvrtcGetPTX, sizeof(__pyx_k_nvrtcGetPTX), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetPTX */ + {__pyx_k_nvrtcGetPTXSize, sizeof(__pyx_k_nvrtcGetPTXSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetPTXSize */ + {__pyx_k_nvrtcGetProgramLog, sizeof(__pyx_k_nvrtcGetProgramLog), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetProgramLog */ + {__pyx_k_nvrtcGetProgramLogSize, sizeof(__pyx_k_nvrtcGetProgramLogSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetProgramLogSize */ + {__pyx_k_nvrtcGetSupportedArchs, sizeof(__pyx_k_nvrtcGetSupportedArchs), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcGetSupportedArchs */ + {__pyx_k_nvrtcSetFlowCallback, sizeof(__pyx_k_nvrtcSetFlowCallback), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcSetFlowCallback */ + {__pyx_k_nvrtcSetPCHHeapSize, sizeof(__pyx_k_nvrtcSetPCHHeapSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcSetPCHHeapSize */ + {__pyx_k_nvrtcVersion, sizeof(__pyx_k_nvrtcVersion), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvrtcVersion */ + {__pyx_k_pop, sizeof(__pyx_k_pop), 0, 1, 1}, /* PyObject cname: __pyx_n_u_pop */ + {__pyx_k_qualname, sizeof(__pyx_k_qualname), 0, 1, 1}, /* PyObject cname: __pyx_n_u_qualname */ + {__pyx_k_set_name, sizeof(__pyx_k_set_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_set_name */ + {__pyx_k_spec, sizeof(__pyx_k_spec), 0, 1, 1}, /* PyObject cname: __pyx_n_u_spec */ + {__pyx_k_test, sizeof(__pyx_k_test), 0, 1, 1}, /* PyObject cname: __pyx_n_u_test */ + {__pyx_k_threading, sizeof(__pyx_k_threading), 0, 1, 1}, /* PyObject cname: __pyx_n_u_threading */ + {0, 0, 0, 0, 0} +}; +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names); + +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(0, 114, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "cuda/bindings/_bindings/cynvrtc.pyx":43 + * + * # Load library + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * handle = (load_nvidia_dynamic_lib("nvrtc")._handle_uint) + * +*/ + __pyx_mstate_global->__pyx_tuple[0] = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_tuple[0])) __PYX_ERR(0, 43, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.method_name = &__pyx_mstate->__pyx_n_u_pop; + if (__Pyx_InitStrings(__pyx_string_tab, __pyx_mstate->__pyx_string_tab, __pyx_string_tab_encodings) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ +\ + typedef struct { + unsigned int argcount : 1; + unsigned int num_posonly_args : 1; + unsigned int num_kwonly_args : 1; + unsigned int nlocals : 1; + unsigned int flags : 10; + unsigned int first_line : 9; + unsigned int line_table_length : 13; + } __Pyx_PyCode_New_function_description; +/* NewCodeObj.proto */ +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + const char *line_table, + PyObject *tuple_dedup_map +); + + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + PyObject* tuple_dedup_map = PyDict_New(); + if (unlikely(!tuple_dedup_map)) return -1; + { + const __Pyx_PyCode_New_function_description descr = {0, 0, 0, 0, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 345, 429}; + PyObject* const varnames[] = {0}; + __pyx_mstate_global->__pyx_codeobj_tab[0] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_cuda_bindings__bindings_cynvrtc_2, __pyx_mstate->__pyx_n_u_inspect_function_pointers, __pyx_k_z_q_Q_Ja_Zq_A_z_Q_Ja_Ja_Ja_A_j, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[0])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 1, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 409, 31}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_name}; + __pyx_mstate_global->__pyx_codeobj_tab[1] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_cuda_bindings__bindings_cynvrtc_2, __pyx_mstate->__pyx_n_u_inspect_function_pointer, __pyx_k_z_A_a_9AQ, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[1])) goto bad; + } + Py_DECREF(tuple_dedup_map); + return 0; + bad: + Py_DECREF(tuple_dedup_map); + return -1; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CachedMethodType.init */ + #if CYTHON_COMPILING_IN_LIMITED_API +{ + PyObject *typesModule=NULL; + typesModule = PyImport_ImportModule("types"); + if (typesModule) { + __pyx_mstate_global->__Pyx_CachedMethodType = PyObject_GetAttrString(typesModule, "MethodType"); + Py_DECREF(typesModule); + } +} // error handling follows +#endif + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* PyErrExceptionMatches */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStr */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* PyObjectGetAttrStrNoError */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name); + if (unlikely(!result) && !PyErr_Occurred()) { + PyErr_Format(PyExc_NameError, + "name '%U' is not defined", name); + } + return result; +} + +/* PyObjectLookupSpecial */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) { + PyObject *res; + PyTypeObject *tp = Py_TYPE(obj); + res = _PyType_Lookup(tp, attr_name); + if (likely(res)) { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (!f) { + Py_INCREF(res); + } else { + res = f(res, obj, (PyObject *)tp); + } + } else if (with_error) { + PyErr_SetObject(PyExc_AttributeError, attr_name); + } + return res; +} +#endif + +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject *const *args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = __Pyx_PyFrame_GetLocalsplus(f); + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; + PyObject *kwdefs; + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) { + return NULL; + } + if ( + co->co_kwonlyargcount == 0 && + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, (int)nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif + +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = Py_TYPE(func)->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + #if PY_VERSION_HEX < 0x030800B1 + #if CYTHON_FAST_PYCCALL + if (PyCFunction_Check(func)) { + if (kwargs) { + return _PyCFunction_FastCallDict(func, args, nargs, kwargs); + } else { + return _PyCFunction_FastCallKeywords(func, args, nargs, NULL); + } + } + if (!kwargs && __Pyx_IS_TYPE(func, &PyMethodDescr_Type)) { + return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL); + } + #endif + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs); + } + #endif + #endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #elif defined(__Pyx_CyFunction_USED) && CYTHON_BACKPORT_VECTORCALL + if (__Pyx_CyFunction_CheckExact(func)) { + __pyx_vectorcallfunc f = __Pyx_CyFunction_func_vectorcall(func); + if (f) return f(func, args, _nargs, NULL); + } + #elif CYTHON_COMPILING_IN_LIMITED_API && CYTHON_VECTORCALL + return PyObject_Vectorcall(func, args, _nargs, NULL); + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* PyDictVersioning */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!__pyx_m)) { + if (!PyErr_Occurred()) + PyErr_SetNone(PyExc_NameError); + return NULL; + } + result = PyObject_GetAttr(__pyx_m, name); + if (likely(result)) { + return result; + } + PyErr_Clear(); +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear(); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return result; + } +#else + result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + +/* GetTopmostException */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + +/* SaveResetException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #endif +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); + #else + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); + #endif +} +#endif + +/* GetException */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) +#endif +{ + PyObject *local_type = NULL, *local_value, *local_tb = NULL; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030C0000 + local_value = tstate->current_exception; + tstate->current_exception = 0; + #else + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + #endif +#elif __PYX_LIMITED_VERSION_HEX > 0x030C0000 + local_value = PyErr_GetRaisedException(); +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif +#if __PYX_LIMITED_VERSION_HEX > 0x030C0000 + if (likely(local_value)) { + local_type = (PyObject*) Py_TYPE(local_value); + Py_INCREF(local_type); + local_tb = PyException_GetTraceback(local_value); + } +#else + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } +#endif // __PYX_LIMITED_VERSION_HEX > 0x030C0000 + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if CYTHON_USE_EXC_INFO_STACK + { + _PyErr_StackItem *exc_info = tstate->exc_info; + #if PY_VERSION_HEX >= 0x030B00a4 + tmp_value = exc_info->exc_value; + exc_info->exc_value = local_value; + tmp_type = NULL; + tmp_tb = NULL; + Py_XDECREF(local_type); + Py_XDECREF(local_tb); + #else + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; + #endif + } + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#elif __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + PyErr_SetHandledException(local_value); + Py_XDECREF(local_value); + Py_XDECREF(local_type); + Py_XDECREF(local_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +#if __PYX_LIMITED_VERSION_HEX <= 0x030C0000 +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +#endif +} + +/* RaiseException */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyException_SetTraceback(value, tb); +#elif CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} + +/* ErrOccurredWithGIL */ +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void) { + int err; + PyGILState_STATE _save = PyGILState_Ensure(); + err = !!PyErr_Occurred(); + PyGILState_Release(_save); + return err; +} + +/* TupleAndListFromArray */ +#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + Py_ssize_t i; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + for (i = 0; i < n; i++) { + if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < (0))) { + Py_DECREF(res); + return NULL; + } + Py_INCREF(src[i]); + } + return res; +} +#elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + +/* BytesEquals */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL ||\ + !(CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) + return PyObject_RichCompareBool(s1, s2, equals); +#else + if (s1 == s2) { + return (equals == Py_EQ); + } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { + const char *ps1, *ps2; + Py_ssize_t length = PyBytes_GET_SIZE(s1); + if (length != PyBytes_GET_SIZE(s2)) + return (equals == Py_NE); + ps1 = PyBytes_AS_STRING(s1); + ps2 = PyBytes_AS_STRING(s2); + if (ps1[0] != ps2[0]) { + return (equals == Py_NE); + } else if (length == 1) { + return (equals == Py_EQ); + } else { + int result; +#if CYTHON_USE_UNICODE_INTERNALS && (PY_VERSION_HEX < 0x030B0000) + Py_hash_t hash1, hash2; + hash1 = ((PyBytesObject*)s1)->ob_shash; + hash2 = ((PyBytesObject*)s2)->ob_shash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + return (equals == Py_NE); + } +#endif + result = memcmp(ps1, ps2, (size_t)length); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { + return (equals == Py_NE); + } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) { + return (equals == Py_NE); + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +#endif +} + +/* UnicodeEquals */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL + return PyObject_RichCompareBool(s1, s2, equals); +#else + int s1_is_unicode, s2_is_unicode; + if (s1 == s2) { + goto return_eq; + } + s1_is_unicode = PyUnicode_CheckExact(s1); + s2_is_unicode = PyUnicode_CheckExact(s2); + if (s1_is_unicode & s2_is_unicode) { + Py_ssize_t length, length2; + int kind; + void *data1, *data2; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0)) + return -1; + #endif + length = __Pyx_PyUnicode_GET_LENGTH(s1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length < 0)) return -1; + #endif + length2 = __Pyx_PyUnicode_GET_LENGTH(s2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length2 < 0)) return -1; + #endif + if (length != length2) { + goto return_ne; + } +#if CYTHON_USE_UNICODE_INTERNALS + { + Py_hash_t hash1, hash2; + hash1 = ((PyASCIIObject*)s1)->hash; + hash2 = ((PyASCIIObject*)s2)->hash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + goto return_ne; + } + } +#endif + kind = __Pyx_PyUnicode_KIND(s1); + if (kind != __Pyx_PyUnicode_KIND(s2)) { + goto return_ne; + } + data1 = __Pyx_PyUnicode_DATA(s1); + data2 = __Pyx_PyUnicode_DATA(s2); + if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) { + goto return_ne; + } else if (length == 1) { + goto return_eq; + } else { + int result = memcmp(data1, data2, (size_t)(length * kind)); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & s2_is_unicode) { + goto return_ne; + } else if ((s2 == Py_None) & s1_is_unicode) { + goto return_ne; + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +return_eq: + return (equals == Py_EQ); +return_ne: + return (equals == Py_NE); +#endif +} + +/* fastcall */ +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + Py_ssize_t i, n = __Pyx_PyTuple_GET_SIZE(kwnames); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(n == -1)) return NULL; + #endif + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + if (s == namei) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + int eq = __Pyx_PyUnicode_Equals(s, namei, Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; + return kwvalues[i]; + } + } + return NULL; +} +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues) { + Py_ssize_t i, nkwargs; + PyObject *dict; +#if !CYTHON_ASSUME_SAFE_SIZE + nkwargs = PyTuple_Size(kwnames); + if (unlikely(nkwargs < 0)) return NULL; +#else + nkwargs = PyTuple_GET_SIZE(kwnames); +#endif + dict = PyDict_New(); + if (unlikely(!dict)) + return NULL; + for (i=0; itype, *target->method_name); + if (unlikely(!method)) + return -1; + result = method; +#if CYTHON_COMPILING_IN_CPYTHON + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + { + PyMethodDescrObject *descr = (PyMethodDescrObject*) method; + target->func = descr->d_method->ml_meth; + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); + } else +#endif +#if CYTHON_COMPILING_IN_PYPY +#else + if (PyCFunction_Check(method)) +#endif + { + PyObject *self; + int self_found; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + self = PyObject_GetAttrString(method, "__self__"); + if (!self) { + PyErr_Clear(); + } +#else + self = PyCFunction_GET_SELF(method); +#endif + self_found = (self && self != Py_None); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + Py_XDECREF(self); +#endif + if (self_found) { + PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method); + if (unlikely(!unbound_method)) return -1; + Py_DECREF(method); + result = unbound_method; + } + } +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + if (unlikely(target->method)) { + Py_DECREF(result); + } else +#endif + target->method = result; + return 0; +} + +/* CallUnboundCMethod2 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + PyObject *args[2] = {arg1, arg2}; + if (cfunc->flag == METH_FASTCALL) { + return __Pyx_CallCFunctionFast(cfunc, self, args, 2); + } + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, 2, NULL); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod2(&tmp_cfunc, self, arg1, arg2); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){ + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + PyObject *result = NULL; + PyObject *args = PyTuple_New(2); + if (unlikely(!args)) return NULL; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + if (cfunc->flag & METH_KEYWORDS) + result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL); + else + result = __Pyx_CallCFunction(cfunc, self, args); + Py_DECREF(args); + return result; + } +#endif + { + PyObject *args[4] = {NULL, self, arg1, arg2}; + return __Pyx_PyObject_FastCall(cfunc->method, args+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } +} + +/* ParseKeywords */ +static int __Pyx_ValidateDuplicatePosArgs( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char* function_name) +{ + PyObject ** const *name = argnames; + while (name != first_kw_arg) { + PyObject *key = **name; + int found = PyDict_Contains(kwds, key); + if (unlikely(found)) { + if (found == 1) __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; + } + name++; + } + return 0; +bad: + return -1; +} +#if CYTHON_USE_UNICODE_INTERNALS +static CYTHON_INLINE int __Pyx_UnicodeKeywordsEqual(PyObject *s1, PyObject *s2) { + int kind; + Py_ssize_t len = PyUnicode_GET_LENGTH(s1); + if (len != PyUnicode_GET_LENGTH(s2)) return 0; + kind = PyUnicode_KIND(s1); + if (kind != PyUnicode_KIND(s2)) return 0; + const void *data1 = PyUnicode_DATA(s1); + const void *data2 = PyUnicode_DATA(s2); + return (memcmp(data1, data2, (size_t) len * (size_t) kind) == 0); +} +#endif +static int __Pyx_MatchKeywordArg_str( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + #if CYTHON_USE_UNICODE_INTERNALS + Py_hash_t key_hash = ((PyASCIIObject*)key)->hash; + if (unlikely(key_hash == -1)) { + key_hash = PyObject_Hash(key); + if (unlikely(key_hash == -1)) + goto bad; + } + #endif + name = first_kw_arg; + while (*name) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (key_hash == ((PyASCIIObject*)name_str)->hash && __Pyx_UnicodeKeywordsEqual(name_str, key)) { + *index_found = (size_t) (name - argnames); + return 1; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + *index_found = (size_t) (name - argnames); + return 1; + } + } + #endif + name++; + } + name = argnames; + while (name != first_kw_arg) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (unlikely(key_hash == ((PyASCIIObject*)name_str)->hash)) { + if (__Pyx_UnicodeKeywordsEqual(name_str, key)) + goto arg_passed_twice; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + if (unlikely(name_str == key)) goto arg_passed_twice; + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + } + #endif + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +bad: + return -1; +} +static int __Pyx_MatchKeywordArg_nostr( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; + name = first_kw_arg; + while (*name) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (cmp == 1) { + *index_found = (size_t) (name - argnames); + return 1; + } + if (unlikely(cmp == -1)) goto bad; + name++; + } + name = argnames; + while (name != first_kw_arg) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (unlikely(cmp != 0)) { + if (cmp == 1) goto arg_passed_twice; + else goto bad; + } + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +bad: + return -1; +} +static CYTHON_INLINE int __Pyx_MatchKeywordArg( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + return likely(PyUnicode_CheckExact(key)) ? + __Pyx_MatchKeywordArg_str(key, argnames, first_kw_arg, index_found, function_name) : + __Pyx_MatchKeywordArg_nostr(key, argnames, first_kw_arg, index_found, function_name); +} +static void __Pyx_RejectUnknownKeyword( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char *function_name) +{ + Py_ssize_t pos = 0; + PyObject *key = NULL; + __Pyx_BEGIN_CRITICAL_SECTION(kwds); + while (PyDict_Next(kwds, &pos, &key, NULL)) { + PyObject** const *name = first_kw_arg; + while (*name && (**name != key)) name++; + if (!*name) { + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(key); + #endif + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp != 1) { + if (cmp == 0) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + break; + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + } + } + __Pyx_END_CRITICAL_SECTION(); + assert(PyErr_Occurred()); +} +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t extracted = 0; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + name = first_kw_arg; + while (*name && num_kwargs > extracted) { + PyObject * key = **name; + PyObject *value; + int found = 0; + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + found = PyDict_GetItemRef(kwds, key, &value); + #else + value = PyDict_GetItemWithError(kwds, key); + if (value) { + Py_INCREF(value); + found = 1; + } else { + if (unlikely(PyErr_Occurred())) goto bad; + } + #endif + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + extracted++; + } + name++; + } + if (num_kwargs > extracted) { + if (ignore_unknown_kwargs) { + if (unlikely(__Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name) == -1)) + goto bad; + } else { + __Pyx_RejectUnknownKeyword(kwds, argnames, first_kw_arg, function_name); + goto bad; + } + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t len; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + if (PyDict_Update(kwds2, kwds) < 0) goto bad; + name = first_kw_arg; + while (*name) { + PyObject *key = **name; + PyObject *value; +#if !CYTHON_COMPILING_IN_LIMITED_API && (PY_VERSION_HEX >= 0x030d00A2 || defined(PyDict_Pop)) + int found = PyDict_Pop(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int found = PyDict_GetItemRef(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + if (unlikely(PyDict_DelItem(kwds2, key) < 0)) goto bad; + } +#else + #if CYTHON_COMPILING_IN_CPYTHON + value = _PyDict_Pop(kwds2, key, kwds2); + #else + value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_pop, kwds2, key, kwds2); + #endif + if (value == kwds2) { + Py_DECREF(value); + } else { + if (unlikely(!value)) goto bad; + values[name-argnames] = value; + } +#endif + name++; + } + len = PyDict_Size(kwds2); + if (len > 0) { + return __Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name); + } else if (unlikely(len == -1)) { + goto bad; + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject *key = NULL; + PyObject** const * name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + for (Py_ssize_t pos = 0; pos < num_kwargs; pos++) { +#if CYTHON_AVOID_BORROWED_REFS + key = __Pyx_PySequence_ITEM(kwds, pos); +#else + key = __Pyx_PyTuple_GET_ITEM(kwds, pos); +#endif +#if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!key)) goto bad; +#endif + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + PyObject *value = kwvalues[pos]; + values[name-argnames] = __Pyx_NewRef(value); + } else { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp == 1) { + PyObject *value = kwvalues[pos]; + values[index_found] = __Pyx_NewRef(value); + } else { + if (unlikely(cmp == -1)) goto bad; + if (kwds2) { + PyObject *value = kwvalues[pos]; + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else if (!ignore_unknown_kwargs) { + goto invalid_keyword; + } + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + key = NULL; + #endif + } + return 0; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + goto bad; +bad: + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(key); + #endif + return -1; +} +static int __Pyx_ParseKeywords( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) + return __Pyx_ParseKeywordsTuple(kwds, kwvalues, argnames, kwds2, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); + else if (kwds2) + return __Pyx_ParseKeywordDictToDict(kwds, argnames, kwds2, values, num_pos_args, function_name); + else + return __Pyx_ParseKeywordDict(kwds, argnames, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* ArgTypeTest */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +{ + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; + PyObject *extra_info = __pyx_mstate_global->__pyx_empty_unicode; + int from_annotation_subclass = 0; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (!exact) { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } else if (exact == 2) { + if (__Pyx_TypeCheck(obj, type)) { + from_annotation_subclass = 1; + extra_info = __pyx_mstate_global->__pyx_kp_u_Note_that_Cython_is_deliberately; + } + } + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")" +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + "%s%U" +#endif + , name, type_name, obj_type_name +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + , (from_annotation_subclass ? ". " : ""), extra_info +#endif + ); +#if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + if (exact == 2 && from_annotation_subclass) { + PyObject *res; + PyObject *vargs[2]; + vargs[0] = PyErr_GetRaisedException(); + vargs[1] = extra_info; + res = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_kp_u_add_note, vargs, 2, NULL); + Py_XDECREF(res); + PyErr_SetRaisedException(vargs[0]); + } +#endif + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* ListPack */ +static PyObject *__Pyx_PyList_Pack(Py_ssize_t n, ...) { + va_list va; + PyObject *l = PyList_New(n); + va_start(va, n); + if (unlikely(!l)) goto end; + for (Py_ssize_t i=0; i__pyx_d, empty_dict, from_list, 1); + if (unlikely(!module)) { + if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError))) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } + if (!module) { + module = PyImport_ImportModuleLevelObject( + name, __pyx_mstate_global->__pyx_d, empty_dict, from_list, level); + } +bad: + Py_XDECREF(empty_dict); + Py_XDECREF(empty_list); + return module; +} + +/* ImportFrom */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { + PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); + if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { + const char* module_name_str = 0; + PyObject* module_name = 0; + PyObject* module_dot = 0; + PyObject* full_name = 0; + PyErr_Clear(); + module_name_str = PyModule_GetName(module); + if (unlikely(!module_name_str)) { goto modbad; } + module_name = PyUnicode_FromString(module_name_str); + if (unlikely(!module_name)) { goto modbad; } + module_dot = PyUnicode_Concat(module_name, __pyx_mstate_global->__pyx_kp_u_); + if (unlikely(!module_dot)) { goto modbad; } + full_name = PyUnicode_Concat(module_dot, name); + if (unlikely(!full_name)) { goto modbad; } + #if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + { + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + goto modbad; + value = PyObject_GetItem(modules, full_name); + } + #else + value = PyImport_GetModule(full_name); + #endif + modbad: + Py_XDECREF(full_name); + Py_XDECREF(module_dot); + Py_XDECREF(module_name); + } + if (unlikely(!value)) { + PyErr_Format(PyExc_ImportError, "cannot import name %S", name); + } + return value; +} + +/* ImportDottedModule */ +static PyObject *__Pyx__ImportDottedModule_Error(PyObject *name, PyObject *parts_tuple, Py_ssize_t count) { + PyObject *partial_name = NULL, *slice = NULL, *sep = NULL; + Py_ssize_t size; + if (unlikely(PyErr_Occurred())) { + PyErr_Clear(); + } +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(parts_tuple); +#else + size = PyTuple_Size(parts_tuple); + if (size < 0) goto bad; +#endif + if (likely(size == count)) { + partial_name = name; + } else { + slice = PySequence_GetSlice(parts_tuple, 0, count); + if (unlikely(!slice)) + goto bad; + sep = PyUnicode_FromStringAndSize(".", 1); + if (unlikely(!sep)) + goto bad; + partial_name = PyUnicode_Join(sep, slice); + } + PyErr_Format( + PyExc_ModuleNotFoundError, + "No module named '%U'", partial_name); +bad: + Py_XDECREF(sep); + Py_XDECREF(slice); + Py_XDECREF(partial_name); + return NULL; +} +static PyObject *__Pyx__ImportDottedModule_Lookup(PyObject *name) { + PyObject *imported_module; +#if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + return NULL; + imported_module = __Pyx_PyDict_GetItemStr(modules, name); + Py_XINCREF(imported_module); +#else + imported_module = PyImport_GetModule(name); +#endif + return imported_module; +} +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple) { + Py_ssize_t i, nparts; +#if CYTHON_ASSUME_SAFE_SIZE + nparts = PyTuple_GET_SIZE(parts_tuple); +#else + nparts = PyTuple_Size(parts_tuple); + if (nparts < 0) return NULL; +#endif + for (i=1; i < nparts && module; i++) { + PyObject *part, *submodule; +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + part = PyTuple_GET_ITEM(parts_tuple, i); +#else + part = __Pyx_PySequence_ITEM(parts_tuple, i); + if (!part) return NULL; +#endif + submodule = __Pyx_PyObject_GetAttrStrNoError(module, part); +#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(part); +#endif + Py_DECREF(module); + module = submodule; + } + if (unlikely(!module)) { + return __Pyx__ImportDottedModule_Error(name, parts_tuple, i); + } + return module; +} +static PyObject *__Pyx__ImportDottedModule(PyObject *name, PyObject *parts_tuple) { + PyObject *imported_module; + PyObject *module = __Pyx_Import(name, NULL, 0); + if (!parts_tuple || unlikely(!module)) + return module; + imported_module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(imported_module)) { + Py_DECREF(module); + return imported_module; + } + PyErr_Clear(); + return __Pyx_ImportDottedModule_WalkParts(module, name, parts_tuple); +} +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple) { +#if CYTHON_COMPILING_IN_CPYTHON + PyObject *module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(module)) { + PyObject *spec = __Pyx_PyObject_GetAttrStrNoError(module, __pyx_mstate_global->__pyx_n_u_spec); + if (likely(spec)) { + PyObject *unsafe = __Pyx_PyObject_GetAttrStrNoError(spec, __pyx_mstate_global->__pyx_n_u_initializing); + if (likely(!unsafe || !__Pyx_PyObject_IsTrue(unsafe))) { + Py_DECREF(spec); + spec = NULL; + } + Py_XDECREF(unsafe); + } + if (likely(!spec)) { + PyErr_Clear(); + return module; + } + Py_DECREF(spec); + Py_DECREF(module); + } else if (PyErr_Occurred()) { + PyErr_Clear(); + } +#endif + return __Pyx__ImportDottedModule(name, parts_tuple); +} + +/* LimitedApiGetTypeDict */ +#if CYTHON_COMPILING_IN_LIMITED_API +static Py_ssize_t __Pyx_GetTypeDictOffset(void) { + PyObject *tp_dictoffset_o; + Py_ssize_t tp_dictoffset; + tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__"); + if (unlikely(!tp_dictoffset_o)) return -1; + tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o); + Py_DECREF(tp_dictoffset_o); + if (unlikely(tp_dictoffset == 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' doesn't have a dictoffset"); + return -1; + } else if (unlikely(tp_dictoffset < 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' has an unexpected negative dictoffset. " + "Please report this as Cython bug"); + return -1; + } + return tp_dictoffset; +} +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) { + static Py_ssize_t tp_dictoffset = 0; + if (unlikely(tp_dictoffset == 0)) { + tp_dictoffset = __Pyx_GetTypeDictOffset(); + if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) { + tp_dictoffset = 0; // try again next time? + return NULL; + } + } + return *(PyObject**)((char*)tp + tp_dictoffset); +} +#endif + +/* SetItemOnTypeDict */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_SetItem(tp_dict, k, v); + if (likely(!result)) { + PyType_Modified(tp); + if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) { + PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name, (PyObject *) tp, k, NULL); + if (!setNameResult) return -1; + Py_DECREF(setNameResult); + } + } + return result; +} + +/* FixUpExtensionType */ +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if __PYX_LIMITED_VERSION_HEX > 0x030900B1 + CYTHON_UNUSED_VAR(spec); + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#else + const PyType_Slot *slot = spec->slots; + int changed = 0; +#if !CYTHON_COMPILING_IN_LIMITED_API + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { +#if !CYTHON_COMPILING_IN_CPYTHON + const +#endif // !CYTHON_COMPILING_IN_CPYTHON) + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); +#if PY_VERSION_HEX >= 0x030800b4 + type->tp_vectorcall_offset = memb->offset; +#else + type->tp_print = (printfunc) memb->offset; +#endif + changed = 1; + } +#endif // CYTHON_METH_FASTCALL +#if !CYTHON_COMPILING_IN_PYPY + else if (strcmp(memb->name, "__module__") == 0) { + PyObject *descr; + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } +#endif // !CYTHON_COMPILING_IN_PYPY + } + memb++; + } + } +#endif // !CYTHON_COMPILING_IN_LIMITED_API +#if !CYTHON_COMPILING_IN_PYPY + slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_getset) + slot++; + if (slot && slot->slot == Py_tp_getset) { + PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc; + while (getset && getset->name) { + if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) { + PyObject *descr = PyDescr_NewGetSet(type, getset); + if (unlikely(!descr)) + return -1; + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *pyname = PyUnicode_FromString(getset->name); + if (unlikely(!pyname)) { + Py_DECREF(descr); + return -1; + } + int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr); + Py_DECREF(pyname); + #else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + #endif + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } + ++getset; + } + } +#endif // !CYTHON_COMPILING_IN_PYPY + if (changed) + PyType_Modified(type); +#endif // PY_VERSION_HEX > 0x030900B1 + return 0; +} + +/* FetchSharedCythonModule */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + return __Pyx_PyImport_AddModuleRef(__PYX_ABI_MODULE_NAME); +} + +/* dict_setdefault */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, + int is_safe_type) { + PyObject* value; + CYTHON_MAYBE_UNUSED_VAR(is_safe_type); +#if CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_CallMethod(d, "setdefault", "OO", key, default_value); +#elif PY_VERSION_HEX >= 0x030d0000 + PyDict_SetDefaultRef(d, key, default_value, &value); +#else + value = PyDict_SetDefault(d, key, default_value); + if (unlikely(!value)) return NULL; + Py_INCREF(value); +#endif + return value; +} + +/* FetchCommonType */ +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject* __Pyx_PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *result = __Pyx_PyType_FromModuleAndSpec(module, spec, bases); + if (result && metaclass) { + PyObject *old_tp = (PyObject*)Py_TYPE(result); + Py_INCREF((PyObject*)metaclass); +#if __PYX_LIMITED_VERSION_HEX >= 0x03090000 + Py_SET_TYPE(result, metaclass); +#else + result->ob_type = metaclass; +#endif + Py_DECREF(old_tp); + } + return result; +} +#else +#define __Pyx_PyType_FromMetaclass(me, mo, s, b) PyType_FromMetaclass(me, mo, s, b) +#endif +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t expected_basicsize) { + Py_ssize_t basicsize; + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (expected_basicsize == 0) { + return 0; // size is inherited, nothing useful to check + } +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) return -1; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = NULL; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; +#else + basicsize = ((PyTypeObject*) cached_type)->tp_basicsize; +#endif + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module = NULL, *cached_type = NULL, *abi_module_dict, *new_cached_type, *py_object_name; + int get_item_ref_result; + const char* object_name = strrchr(spec->name, '.'); + object_name = object_name ? object_name+1 : spec->name; + py_object_name = PyUnicode_FromString(object_name); + if (!py_object_name) return NULL; + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) goto done; + abi_module_dict = PyModule_GetDict(abi_module); + if (!abi_module_dict) goto done; + get_item_ref_result = __Pyx_PyDict_GetItemRef(abi_module_dict, py_object_name, &cached_type); + if (get_item_ref_result == 1) { + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else if (unlikely(get_item_ref_result == -1)) { + goto bad; + } + CYTHON_UNUSED_VAR(module); + cached_type = __Pyx_PyType_FromMetaclass(metaclass, abi_module, spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + new_cached_type = __Pyx_PyDict_SetDefault(abi_module_dict, py_object_name, cached_type, 1); + if (unlikely(new_cached_type != cached_type)) { + if (unlikely(!new_cached_type)) goto bad; + Py_DECREF(cached_type); + cached_type = new_cached_type; + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else { + Py_DECREF(new_cached_type); + } +done: + Py_XDECREF(abi_module); + Py_DECREF(py_object_name); + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* CommonTypesMetaclass */ +static PyObject* __pyx_CommonTypesMetaclass_get_module(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED void* context) { + return PyUnicode_FromString(__PYX_ABI_MODULE_NAME); +} +static PyGetSetDef __pyx_CommonTypesMetaclass_getset[] = { + {"__module__", __pyx_CommonTypesMetaclass_get_module, NULL, NULL, NULL}, + {0, 0, 0, 0, 0} +}; +static PyType_Slot __pyx_CommonTypesMetaclass_slots[] = { + {Py_tp_getset, (void *)__pyx_CommonTypesMetaclass_getset}, + {0, 0} +}; +static PyType_Spec __pyx_CommonTypesMetaclass_spec = { + __PYX_TYPE_MODULE_PREFIX "_common_types_metatype", + 0, + 0, +#if PY_VERSION_HEX >= 0x030A0000 + Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | +#endif + Py_TPFLAGS_DEFAULT, + __pyx_CommonTypesMetaclass_slots +}; +static int __pyx_CommonTypesMetaclass_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + PyObject *bases = PyTuple_Pack(1, &PyType_Type); + if (unlikely(!bases)) { + return -1; + } + mstate->__pyx_CommonTypesMetaclassType = __Pyx_FetchCommonTypeFromSpec(NULL, module, &__pyx_CommonTypesMetaclass_spec, bases); + Py_DECREF(bases); + if (unlikely(mstate->__pyx_CommonTypesMetaclassType == NULL)) { + return -1; + } + return 0; +} + +/* CallTypeTraverse */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) { + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 + if (__Pyx_get_runtime_version() < 0x03090000) return 0; + #endif + if (!always_call) { + PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*); + unsigned long flags = PyType_GetFlags(base); + if (flags & Py_TPFLAGS_HEAPTYPE) { + return 0; + } + } + Py_VISIT((PyObject*)Py_TYPE(o)); + return 0; +} +#endif + +/* PyMethodNew */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + PyObject *result; + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + #if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + { + PyObject *args[] = {func, self}; + result = PyObject_Vectorcall(__pyx_mstate_global->__Pyx_CachedMethodType, args, 2, NULL); + } + #else + result = PyObject_CallFunctionObjArgs(__pyx_mstate_global->__Pyx_CachedMethodType, func, self, NULL); + #endif + return result; +} +#else +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#endif + +/* PyVectorcallFastCallDict */ +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i, pos; + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + #if !CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t nkw = PyDict_Size(kw); + if (unlikely(nkw == -1)) return NULL; + #else + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + #endif + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; + } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (PyDict_Next(kw, &pos, &key, &value)) { + keys_are_strings &= + #if CYTHON_COMPILING_IN_LIMITED_API + PyType_GetFlags(Py_TYPE(key)); + #else + Py_TYPE(key)->tp_flags; + #endif + Py_INCREF(key); + Py_INCREF(value); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(PyTuple_SetItem(kwnames, i, key) < 0)) goto cleanup; + #else + PyTuple_SET_ITEM(kwnames, i, key); + #endif + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + res = vc(func, newargs, nargs, kwnames); +cleanup: + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + Py_ssize_t kw_size = + likely(kw == NULL) ? + 0 : +#if !CYTHON_ASSUME_SAFE_SIZE + PyDict_Size(kw); +#else + PyDict_GET_SIZE(kw); +#endif + if (kw_size == 0) { + return vc(func, args, nargs, NULL); + } +#if !CYTHON_ASSUME_SAFE_SIZE + else if (unlikely(kw_size == -1)) { + return NULL; + } +#endif + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); +} +#endif + +/* CythonFunctionShared */ +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunctionNoMethod(PyObject *func, void (*cfunc)(void)) { + if (__Pyx_CyFunction_Check(func)) { + return PyCFunction_GetFunction(((__pyx_CyFunctionObject*)func)->func) == (PyCFunction) cfunc; + } else if (PyCFunction_Check(func)) { + return PyCFunction_GetFunction(func) == (PyCFunction) cfunc; + } + return 0; +} +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if ((PyObject*)Py_TYPE(func) == __pyx_mstate_global->__Pyx_CachedMethodType) { + int result; + PyObject *newFunc = PyObject_GetAttr(func, __pyx_mstate_global->__pyx_n_u_func); + if (unlikely(!newFunc)) { + PyErr_Clear(); // It's only an optimization, so don't throw an error + return 0; + } + result = __Pyx__IsSameCyOrCFunctionNoMethod(newFunc, cfunc); + Py_DECREF(newFunc); + return result; + } + return __Pyx__IsSameCyOrCFunctionNoMethod(func, cfunc); +} +#else +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if (PyMethod_Check(func)) { + func = PyMethod_GET_FUNCTION(func); + } + return __Pyx_CyOrPyCFunction_Check(func) && __Pyx_CyOrPyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +} +#endif +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#else + __Pyx_Py_XDECREF_SET( + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#endif +} +static PyObject * +__Pyx_CyFunction_get_doc_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_doc == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_doc = PyObject_GetAttrString(op->func, "__doc__"); + if (unlikely(!op->func_doc)) return NULL; +#else + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } +#endif + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + PyObject *result; + CYTHON_UNUSED_VAR(closure); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_doc_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_doc, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_name == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_name = PyObject_GetAttrString(op->func, "__name__"); +#else + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_name_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_name, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + Py_INCREF(op->func_qualname); + result = op->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_dict_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_dict_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL)) { + PyErr_SetString(PyExc_TypeError, + "function's dictionary may not be deleted"); + return -1; + } + if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "setting function's dictionary to a non-dict"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_dict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = __Pyx_PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = __Pyx_PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_defaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_kwdefaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value || value == Py_None) { + value = NULL; + } else if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) { + int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; + if (is_coroutine) { + PyObject *is_coroutine_value, *module, *fromlist, *marker = __pyx_mstate_global->__pyx_n_u_is_coroutine; + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); +#if CYTHON_ASSUME_SAFE_MACROS + PyList_SET_ITEM(fromlist, 0, marker); +#else + if (unlikely(PyList_SetItem(fromlist, 0, marker) < 0)) { + Py_DECREF(marker); + Py_DECREF(fromlist); + return NULL; + } +#endif + module = PyImport_ImportModuleLevelObject(__pyx_mstate_global->__pyx_n_u_asyncio_coroutines, NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + is_coroutine_value = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(is_coroutine_value)) { + return is_coroutine_value; + } +ignore: + PyErr_Clear(); + } + return __Pyx_PyBool_FromLong(is_coroutine); +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + result = __Pyx_CyFunction_get_is_coroutine_value(op); + if (unlikely(!result)) + return NULL; + __Pyx_BEGIN_CRITICAL_SECTION(op); + if (op->func_is_coroutine) { + Py_DECREF(result); + result = __Pyx_NewRef(op->func_is_coroutine); + } else { + op->func_is_coroutine = __Pyx_NewRef(result); + } + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static void __Pyx_CyFunction_raise_argument_count_error(__pyx_CyFunctionObject *func, const char* message, Py_ssize_t size) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + py_name, message, size); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + name, message, size); +#endif +} +static void __Pyx_CyFunction_raise_type_error(__pyx_CyFunctionObject *func, const char* message) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s", + py_name, message); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s", + name, message); +#endif +} +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject * +__Pyx_CyFunction_get_module(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_GetAttrString(op->func, "__module__"); +} +static int +__Pyx_CyFunction_set_module(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_SetAttrString(op->func, "__module__", value); +} +#endif +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {"func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, + {"func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {"__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {"func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API + {"__module__", (getter)__Pyx_CyFunction_get_module, (setter)__Pyx_CyFunction_set_module, 0, 0}, +#endif + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { +#if !CYTHON_COMPILING_IN_LIMITED_API + {"__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, +#endif + {"__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#if CYTHON_METH_FASTCALL +#if CYTHON_BACKPORT_VECTORCALL || CYTHON_COMPILING_IN_LIMITED_API + {"__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, +#else + {"__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {"__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(args); + __Pyx_BEGIN_CRITICAL_SECTION(m); + Py_INCREF(m->func_qualname); + result = m->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { +#if !CYTHON_COMPILING_IN_LIMITED_API + PyCFunctionObject *cf = (PyCFunctionObject*) op; +#endif + if (unlikely(op == NULL)) + return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + op->func = PyCFunction_NewEx(ml, (PyObject*)op, module); + if (unlikely(!op->func)) return NULL; +#endif + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; +#if !CYTHON_COMPILING_IN_LIMITED_API + cf->m_ml = ml; + cf->m_self = (PyObject *) op; +#endif + Py_XINCREF(closure); + op->func_closure = closure; +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_XINCREF(module); + cf->m_module = module; +#endif + op->func_dict = NULL; + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + op->func_classobj = NULL; +#else + ((PyCMethodObject*)op)->mm_class = NULL; +#endif + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func); +#else + Py_CLEAR(((PyCFunctionObject*)m)->m_module); +#endif + Py_CLEAR(m->func_dict); + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API +#if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); +#else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } +#endif +#endif + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); + Py_CLEAR(m->defaults); + return 0; +} +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + __Pyx_PyHeapTypeObject_GC_Del(m); +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + { + int e = __Pyx_call_type_traverse((PyObject*)m, 1, visit, arg); + if (e) return e; + } + Py_VISIT(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func); +#else + Py_VISIT(((PyCFunctionObject*)m)->m_module); +#endif + Py_VISIT(m->func_dict); + __Pyx_VISIT_CONST(m->func_name); + __Pyx_VISIT_CONST(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + __Pyx_VISIT_CONST(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); +#endif + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); + Py_VISIT(m->defaults); + return 0; +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ + PyObject *repr; + __Pyx_BEGIN_CRITICAL_SECTION(op); + repr = PyUnicode_FromFormat("", + op->func_qualname, (void *)op); + __Pyx_END_CRITICAL_SECTION(); + return repr; +} +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *f = ((__pyx_CyFunctionObject*)func)->func; + PyCFunction meth; + int flags; + meth = PyCFunction_GetFunction(f); + if (unlikely(!meth)) return NULL; + flags = PyCFunction_GetFlags(f); + if (unlikely(flags < 0)) return NULL; +#else + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + int flags = f->m_ml->ml_flags; +#endif + Py_ssize_t size; + switch (flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 0)) + return (*meth)(self, NULL); + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes no arguments", size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 1)) { + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = __Pyx_PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif + result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(arg0); + #endif + return result; + } + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes exactly one argument", size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + return NULL; + } + __Pyx_CyFunction_raise_type_error( + (__pyx_CyFunctionObject*)func, "takes no keyword arguments"); + return NULL; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *self, *result; +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)func)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)func)->m_self; +#endif + result = __Pyx_CyFunction_CallMethod(func, self, arg, kw); + return result; +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; +#if CYTHON_ASSUME_SAFE_SIZE + argc = PyTuple_GET_SIZE(args); +#else + argc = PyTuple_Size(args); + if (unlikely(argc < 0)) return NULL; +#endif + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + PyErr_Format(PyExc_TypeError, + "unbound method %.200S() needs an argument", + cyfunc->func_qualname); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; +} +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "needs an argument"); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(__Pyx_PyTuple_GET_SIZE(kwnames))) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "takes no keyword arguments"); + return -1; + } + return ret; +} +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 0)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes no arguments", nargs); + return NULL; + } + return meth(self, NULL); +} +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 1)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes exactly one argument", nargs); + return NULL; + } + return meth(self, args[0]); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))meth)(self, args, nargs, kwnames); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCMethod)(void(*)(void))meth)(self, cls, args, (size_t)nargs, kwnames); +} +#endif +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#if CYTHON_METH_FASTCALL +#if defined(Py_TPFLAGS_HAVE_VECTORCALL) + Py_TPFLAGS_HAVE_VECTORCALL | +#elif defined(_Py_TPFLAGS_HAVE_VECTORCALL) + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif +#endif // CYTHON_METH_FASTCALL +#if PY_VERSION_HEX >= 0x030A0000 + Py_TPFLAGS_IMMUTABLETYPE | +#endif + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, + __pyx_CyFunctionType_slots +}; +static int __pyx_CyFunction_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + mstate->__pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_CyFunctionType_spec, NULL); + if (unlikely(mstate->__pyx_CyFunctionType == NULL)) { + return -1; + } + return 0; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, PyTypeObject *defaults_type) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_CallObject((PyObject*)defaults_type, NULL); // _PyObject_New(defaults_type); + if (unlikely(!m->defaults)) + return NULL; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +/* CythonFunction */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyObject *op = __Pyx_CyFunction_Init( + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_mstate_global->__pyx_CyFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + PyObject_GC_Track(op); + } + return op; +} + +/* CLineInTraceback */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline; + PyObject *ptype, *pvalue, *ptraceback; +#if CYTHON_COMPILING_IN_CPYTHON + PyObject **cython_runtime_dict; +#endif + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); +#if CYTHON_COMPILING_IN_CPYTHON + cython_runtime_dict = _PyObject_GetDictPtr(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __Pyx_BEGIN_CRITICAL_SECTION(*cython_runtime_dict); + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, *cython_runtime_dict, + __Pyx_PyDict_GetItemStr(*cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback)) + Py_XINCREF(use_cline); + __Pyx_END_CRITICAL_SECTION(); + } else +#endif + { + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback); + if (use_cline_obj) { + use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; + Py_INCREF(use_cline); + Py_DECREF(use_cline_obj); + } else { + PyErr_Clear(); + use_cline = NULL; + } + } + if (!use_cline) { + c_line = 0; + (void) PyObject_SetAttr(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False); + } + else if (use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + (void) __pyx_cfilenm; + (void) __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + py_funcname = PyUnicode_FromString(funcname); + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* CIntFromPyVerify */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntFromPy */ +static CYTHON_INLINE size_t __Pyx_PyLong_As_size_t(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + size_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (size_t) -1; + val = __Pyx_PyLong_As_size_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(size_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(size_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 2 * PyLong_SHIFT)) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(size_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 3 * PyLong_SHIFT)) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(size_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 4 * PyLong_SHIFT)) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(size_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(size_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(size_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(size_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(size_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT)) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(size_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(size_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + size_t val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (size_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (size_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (size_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (size_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(size_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((size_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(size_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((size_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((size_t) 1) << (sizeof(size_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (size_t) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to size_t"); + return (size_t) -1; +} + +/* FormatTypeName */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static __Pyx_TypeName +__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) +{ + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u__2); + } + goto done; +} +#endif + +/* PyObjectVectorCallKwBuilder */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if defined(HAVE_LONG_LONG) && !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPy */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + long val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (long) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + int val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030b0000 + return Py_Version & ~0xFFUL; +#else + static unsigned long __Pyx_cached_runtime_version = 0; + if (__Pyx_cached_runtime_version == 0) { + const char* rt_version = Py_GetVersion(); + unsigned long version = 0; + unsigned long factor = 0x01000000UL; + unsigned int digit = 0; + int i = 0; + while (factor) { + while ('0' <= rt_version[i] && rt_version[i] <= '9') { + digit = digit * 10 + (unsigned int) (rt_version[i] - '0'); + ++i; + } + version += factor * digit; + if (rt_version[i] != '.') + break; + digit = 0; + factor >>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } + return __Pyx_cached_runtime_version; +#endif +} + +/* CheckBinaryVersion */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } +} + +/* FunctionExport */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig) { + PyObject *d = 0; + PyObject *cobj = 0; + union { + void (*fp)(void); + void *p; + } tmp; + d = PyObject_GetAttrString(__pyx_m, "__pyx_capi__"); + if (!d) { + PyErr_Clear(); + d = PyDict_New(); + if (!d) + goto bad; + Py_INCREF(d); + if (PyModule_AddObject(__pyx_m, "__pyx_capi__", d) < 0) + goto bad; + } + tmp.fp = f; + cobj = PyCapsule_New(tmp.p, sig, 0); + if (!cobj) + goto bad; + if (PyDict_SetItemString(d, name, cobj) < 0) + goto bad; + Py_DECREF(cobj); + Py_DECREF(d); + return 0; +bad: + Py_XDECREF(cobj); + Py_XDECREF(d); + return -1; +} + +/* NewCodeObj */ +#if CYTHON_COMPILING_IN_LIMITED_API + static PyObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyObject *exception_table = NULL; + PyObject *types_module=NULL, *code_type=NULL, *result=NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + PyObject *version_info; + PyObject *py_minor_version = NULL; + #endif + long minor_version = 0; + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + minor_version = 11; + #else + if (!(version_info = PySys_GetObject("version_info"))) goto end; + if (!(py_minor_version = PySequence_GetItem(version_info, 1))) goto end; + minor_version = PyLong_AsLong(py_minor_version); + Py_DECREF(py_minor_version); + if (minor_version == -1 && PyErr_Occurred()) goto end; + #endif + if (!(types_module = PyImport_ImportModule("types"))) goto end; + if (!(code_type = PyObject_GetAttrString(types_module, "CodeType"))) goto end; + if (minor_version <= 7) { + (void)p; + result = PyObject_CallFunction(code_type, "iiiiiOOOOOOiOOO", a, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else if (minor_version <= 10) { + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOiOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else { + if (!(exception_table = PyBytes_FromStringAndSize(NULL, 0))) goto end; + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOOiOOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, name, fline, lnos, exception_table, fv, cell); + } + end: + Py_XDECREF(code_type); + Py_XDECREF(exception_table); + Py_XDECREF(types_module); + if (type) { + PyErr_Restore(type, value, traceback); + } + return result; + } +#elif PY_VERSION_HEX >= 0x030B0000 + static PyCodeObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyCodeObject *result; + result = + #if PY_VERSION_HEX >= 0x030C0000 + PyUnstable_Code_NewWithPosOnlyArgs + #else + PyCode_NewWithPosOnlyArgs + #endif + (a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, name, fline, lnos, __pyx_mstate_global->__pyx_empty_bytes); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030c00A1 + if (likely(result)) + result->_co_firsttraceable = 0; + #endif + return result; + } +#elif PY_VERSION_HEX >= 0x030800B2 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#else + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#endif +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + const char *line_table, + PyObject *tuple_dedup_map +) { + PyObject *code_obj = NULL, *varnames_tuple_dedup = NULL, *code_bytes = NULL, *line_table_bytes = NULL; + Py_ssize_t var_count = (Py_ssize_t) descr.nlocals; + PyObject *varnames_tuple = PyTuple_New(var_count); + if (unlikely(!varnames_tuple)) return NULL; + for (Py_ssize_t i=0; i < var_count; i++) { + Py_INCREF(varnames[i]); + if (__Pyx_PyTuple_SET_ITEM(varnames_tuple, i, varnames[i]) != (0)) goto done; + } + #if CYTHON_COMPILING_IN_LIMITED_API + varnames_tuple_dedup = PyDict_GetItem(tuple_dedup_map, varnames_tuple); + if (!varnames_tuple_dedup) { + if (unlikely(PyDict_SetItem(tuple_dedup_map, varnames_tuple, varnames_tuple) < 0)) goto done; + varnames_tuple_dedup = varnames_tuple; + } + #else + varnames_tuple_dedup = PyDict_SetDefault(tuple_dedup_map, varnames_tuple, varnames_tuple); + if (unlikely(!varnames_tuple_dedup)) goto done; + #endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(varnames_tuple_dedup); + #endif + if (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table != NULL + && !CYTHON_COMPILING_IN_GRAAL) { + line_table_bytes = PyBytes_FromStringAndSize(line_table, descr.line_table_length); + if (unlikely(!line_table_bytes)) goto done; + Py_ssize_t code_len = (descr.line_table_length * 2 + 4) & ~3; + code_bytes = PyBytes_FromStringAndSize(NULL, code_len); + if (unlikely(!code_bytes)) goto done; + char* c_code_bytes = PyBytes_AsString(code_bytes); + if (unlikely(!c_code_bytes)) goto done; + memset(c_code_bytes, 0, (size_t) code_len); + } + code_obj = (PyObject*) __Pyx__PyCode_New( + (int) descr.argcount, + (int) descr.num_posonly_args, + (int) descr.num_kwonly_args, + (int) descr.nlocals, + 0, + (int) descr.flags, + code_bytes ? code_bytes : __pyx_mstate_global->__pyx_empty_bytes, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + varnames_tuple_dedup, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + filename, + funcname, + (int) descr.first_line, + (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table_bytes) ? line_table_bytes : __pyx_mstate_global->__pyx_empty_bytes + ); +done: + Py_XDECREF(code_bytes); + Py_XDECREF(line_table_bytes); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(varnames_tuple_dedup); + #endif + Py_DECREF(varnames_tuple); + return code_obj; +} + +/* InitStrings */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names) { + while (t->s) { + PyObject *str; + if (t->is_unicode) { + if (t->intern) { + str = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + str = PyUnicode_Decode(t->s, t->n - 1, encoding_names[t->encoding], NULL); + } else { + str = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + str = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + if (!str) + return -1; + *target = str; + if (PyObject_Hash(str) == -1) + return -1; + ++t; + ++target; + } + return 0; +} + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} +#if CYTHON_USE_PYLONG_INTERNALS +static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value) { + if (unlikely(!__Pyx_PyLong_IsCompact(x))) + return 0; + Py_ssize_t value = __Pyx_PyLong_CompactValue(x); + if ((sizeof(long) < sizeof(Py_ssize_t)) && unlikely(value != (long) value)) + return 0; + *return_value = (long) value; + return 1; +} +#endif + + +/* MultiPhaseInitModuleState */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 +#else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 +#endif +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS +#error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" +#endif +#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#define __Pyx_ModuleStateLookup_Lock() +#define __Pyx_ModuleStateLookup_Unlock() +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 +static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; +#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#include +static std::mutex __Pyx_ModuleStateLookup_mutex; +#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() +#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) +#include +static mtx_t __Pyx_ModuleStateLookup_mutex; +static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; +static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); +} +#define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(HAVE_PTHREAD_H) +#include +static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; +#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(_WIN32) +#include // synchapi.h on its own doesn't work +static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; +#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#else +#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." +#endif +typedef struct { + int64_t id; + PyObject *module; +} __Pyx_InterpreterIdAndModule; +typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; +} __Pyx_ModuleStateLookupData; +#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; +#else +static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; +#endif +static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; +} +static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; +#else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); +#endif + return result; + } +} +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); +} +#else +#define __Pyx_ModuleStateLookup_wait_until_no_readers() +#endif +static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; +} +static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; +} +static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); +#else + __Pyx_ModuleStateLookup_data = new_data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return result; +} +static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; +#endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); +#else + __Pyx_ModuleStateLookup_data = data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; +} +#endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cynvrtc.cpython-312-x86_64-linux-gnu.so b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cynvrtc.cpython-312-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..03b1e95274a60ba626ac7508b0387496ff1e4db9 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cynvrtc.cpython-312-x86_64-linux-gnu.so differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cynvrtc.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cynvrtc.pxd new file mode 100644 index 0000000000000000000000000000000000000000..a592a3b4cb090b93a8cd5e2d94392eec5c21de64 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cynvrtc.pxd @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +# This code was automatically generated with version 12.9.0. Do not modify it directly. +from cuda.bindings.cynvrtc cimport * + +cdef const char* _nvrtcGetErrorString(nvrtcResult result) except ?NULL nogil + +cdef nvrtcResult _nvrtcVersion(int* major, int* minor) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetNumSupportedArchs(int* numArchs) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetSupportedArchs(int* supportedArchs) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcCreateProgram(nvrtcProgram* prog, const char* src, const char* name, int numHeaders, const char** headers, const char** includeNames) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcDestroyProgram(nvrtcProgram* prog) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcCompileProgram(nvrtcProgram prog, int numOptions, const char** options) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetPTXSize(nvrtcProgram prog, size_t* ptxSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetPTX(nvrtcProgram prog, char* ptx) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetCUBINSize(nvrtcProgram prog, size_t* cubinSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetCUBIN(nvrtcProgram prog, char* cubin) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetNVVMSize(nvrtcProgram prog, size_t* nvvmSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetNVVM(nvrtcProgram prog, char* nvvm) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetLTOIRSize(nvrtcProgram prog, size_t* LTOIRSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetLTOIR(nvrtcProgram prog, char* LTOIR) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetOptiXIRSize(nvrtcProgram prog, size_t* optixirSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetOptiXIR(nvrtcProgram prog, char* optixir) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetProgramLogSize(nvrtcProgram prog, size_t* logSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetProgramLog(nvrtcProgram prog, char* log) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcAddNameExpression(nvrtcProgram prog, const char* name_expression) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetLoweredName(nvrtcProgram prog, const char* name_expression, const char** lowered_name) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetPCHHeapSize(size_t* ret) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcSetPCHHeapSize(size_t size) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetPCHCreateStatus(nvrtcProgram prog) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcGetPCHHeapSizeRequired(nvrtcProgram prog, size_t* size) except ?NVRTC_ERROR_INVALID_INPUT nogil + +cdef nvrtcResult _nvrtcSetFlowCallback(nvrtcProgram prog, void* callback, void* payload) except ?NVRTC_ERROR_INVALID_INPUT nogil diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cynvrtc.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cynvrtc.pyx new file mode 100644 index 0000000000000000000000000000000000000000..af07a2b4daaaae854171555a7a87952299b1e052 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cynvrtc.pyx @@ -0,0 +1,413 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +# This code was automatically generated with version 12.9.0. Do not modify it directly. +cimport cuda.bindings._lib.dlfcn as dlfcn +from cuda.pathfinder import load_nvidia_dynamic_lib +from libc.stdint cimport intptr_t, uintptr_t +import threading + +cdef object __symbol_lock = threading.Lock() +cdef bint __cuPythonInit = False +cdef void *__nvrtcGetErrorString = NULL +cdef void *__nvrtcVersion = NULL +cdef void *__nvrtcGetNumSupportedArchs = NULL +cdef void *__nvrtcGetSupportedArchs = NULL +cdef void *__nvrtcCreateProgram = NULL +cdef void *__nvrtcDestroyProgram = NULL +cdef void *__nvrtcCompileProgram = NULL +cdef void *__nvrtcGetPTXSize = NULL +cdef void *__nvrtcGetPTX = NULL +cdef void *__nvrtcGetCUBINSize = NULL +cdef void *__nvrtcGetCUBIN = NULL +cdef void *__nvrtcGetNVVMSize = NULL +cdef void *__nvrtcGetNVVM = NULL +cdef void *__nvrtcGetLTOIRSize = NULL +cdef void *__nvrtcGetLTOIR = NULL +cdef void *__nvrtcGetOptiXIRSize = NULL +cdef void *__nvrtcGetOptiXIR = NULL +cdef void *__nvrtcGetProgramLogSize = NULL +cdef void *__nvrtcGetProgramLog = NULL +cdef void *__nvrtcAddNameExpression = NULL +cdef void *__nvrtcGetLoweredName = NULL +cdef void *__nvrtcGetPCHHeapSize = NULL +cdef void *__nvrtcSetPCHHeapSize = NULL +cdef void *__nvrtcGetPCHCreateStatus = NULL +cdef void *__nvrtcGetPCHHeapSizeRequired = NULL +cdef void *__nvrtcSetFlowCallback = NULL + +cdef int _cuPythonInit() except -1 nogil: + global __cuPythonInit + + # Load library + with gil, __symbol_lock: + handle = (load_nvidia_dynamic_lib("nvrtc")._handle_uint) + + # Load function + global __nvrtcGetErrorString + __nvrtcGetErrorString = dlfcn.dlsym(handle, 'nvrtcGetErrorString') + global __nvrtcVersion + __nvrtcVersion = dlfcn.dlsym(handle, 'nvrtcVersion') + global __nvrtcGetNumSupportedArchs + __nvrtcGetNumSupportedArchs = dlfcn.dlsym(handle, 'nvrtcGetNumSupportedArchs') + global __nvrtcGetSupportedArchs + __nvrtcGetSupportedArchs = dlfcn.dlsym(handle, 'nvrtcGetSupportedArchs') + global __nvrtcCreateProgram + __nvrtcCreateProgram = dlfcn.dlsym(handle, 'nvrtcCreateProgram') + global __nvrtcDestroyProgram + __nvrtcDestroyProgram = dlfcn.dlsym(handle, 'nvrtcDestroyProgram') + global __nvrtcCompileProgram + __nvrtcCompileProgram = dlfcn.dlsym(handle, 'nvrtcCompileProgram') + global __nvrtcGetPTXSize + __nvrtcGetPTXSize = dlfcn.dlsym(handle, 'nvrtcGetPTXSize') + global __nvrtcGetPTX + __nvrtcGetPTX = dlfcn.dlsym(handle, 'nvrtcGetPTX') + global __nvrtcGetCUBINSize + __nvrtcGetCUBINSize = dlfcn.dlsym(handle, 'nvrtcGetCUBINSize') + global __nvrtcGetCUBIN + __nvrtcGetCUBIN = dlfcn.dlsym(handle, 'nvrtcGetCUBIN') + global __nvrtcGetNVVMSize + __nvrtcGetNVVMSize = dlfcn.dlsym(handle, 'nvrtcGetNVVMSize') + global __nvrtcGetNVVM + __nvrtcGetNVVM = dlfcn.dlsym(handle, 'nvrtcGetNVVM') + global __nvrtcGetLTOIRSize + __nvrtcGetLTOIRSize = dlfcn.dlsym(handle, 'nvrtcGetLTOIRSize') + global __nvrtcGetLTOIR + __nvrtcGetLTOIR = dlfcn.dlsym(handle, 'nvrtcGetLTOIR') + global __nvrtcGetOptiXIRSize + __nvrtcGetOptiXIRSize = dlfcn.dlsym(handle, 'nvrtcGetOptiXIRSize') + global __nvrtcGetOptiXIR + __nvrtcGetOptiXIR = dlfcn.dlsym(handle, 'nvrtcGetOptiXIR') + global __nvrtcGetProgramLogSize + __nvrtcGetProgramLogSize = dlfcn.dlsym(handle, 'nvrtcGetProgramLogSize') + global __nvrtcGetProgramLog + __nvrtcGetProgramLog = dlfcn.dlsym(handle, 'nvrtcGetProgramLog') + global __nvrtcAddNameExpression + __nvrtcAddNameExpression = dlfcn.dlsym(handle, 'nvrtcAddNameExpression') + global __nvrtcGetLoweredName + __nvrtcGetLoweredName = dlfcn.dlsym(handle, 'nvrtcGetLoweredName') + global __nvrtcGetPCHHeapSize + __nvrtcGetPCHHeapSize = dlfcn.dlsym(handle, 'nvrtcGetPCHHeapSize') + global __nvrtcSetPCHHeapSize + __nvrtcSetPCHHeapSize = dlfcn.dlsym(handle, 'nvrtcSetPCHHeapSize') + global __nvrtcGetPCHCreateStatus + __nvrtcGetPCHCreateStatus = dlfcn.dlsym(handle, 'nvrtcGetPCHCreateStatus') + global __nvrtcGetPCHHeapSizeRequired + __nvrtcGetPCHHeapSizeRequired = dlfcn.dlsym(handle, 'nvrtcGetPCHHeapSizeRequired') + global __nvrtcSetFlowCallback + __nvrtcSetFlowCallback = dlfcn.dlsym(handle, 'nvrtcSetFlowCallback') + __cuPythonInit = True + return 0 + +# Create a very small function to check whether we are init'ed, so the C +# compiler can inline it. +cdef inline int cuPythonInit() except -1 nogil: + if __cuPythonInit: + return 0 + return _cuPythonInit() + +cdef const char* _nvrtcGetErrorString(nvrtcResult result) except ?NULL nogil: + global __nvrtcGetErrorString + cuPythonInit() + if __nvrtcGetErrorString == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetErrorString" not found') + err = ( __nvrtcGetErrorString)(result) + return err + +cdef nvrtcResult _nvrtcVersion(int* major, int* minor) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcVersion + cuPythonInit() + if __nvrtcVersion == NULL: + with gil: + raise RuntimeError('Function "nvrtcVersion" not found') + err = ( __nvrtcVersion)(major, minor) + return err + +cdef nvrtcResult _nvrtcGetNumSupportedArchs(int* numArchs) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetNumSupportedArchs + cuPythonInit() + if __nvrtcGetNumSupportedArchs == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetNumSupportedArchs" not found') + err = ( __nvrtcGetNumSupportedArchs)(numArchs) + return err + +cdef nvrtcResult _nvrtcGetSupportedArchs(int* supportedArchs) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetSupportedArchs + cuPythonInit() + if __nvrtcGetSupportedArchs == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetSupportedArchs" not found') + err = ( __nvrtcGetSupportedArchs)(supportedArchs) + return err + +cdef nvrtcResult _nvrtcCreateProgram(nvrtcProgram* prog, const char* src, const char* name, int numHeaders, const char** headers, const char** includeNames) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcCreateProgram + cuPythonInit() + if __nvrtcCreateProgram == NULL: + with gil: + raise RuntimeError('Function "nvrtcCreateProgram" not found') + err = ( __nvrtcCreateProgram)(prog, src, name, numHeaders, headers, includeNames) + return err + +cdef nvrtcResult _nvrtcDestroyProgram(nvrtcProgram* prog) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcDestroyProgram + cuPythonInit() + if __nvrtcDestroyProgram == NULL: + with gil: + raise RuntimeError('Function "nvrtcDestroyProgram" not found') + err = ( __nvrtcDestroyProgram)(prog) + return err + +cdef nvrtcResult _nvrtcCompileProgram(nvrtcProgram prog, int numOptions, const char** options) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcCompileProgram + cuPythonInit() + if __nvrtcCompileProgram == NULL: + with gil: + raise RuntimeError('Function "nvrtcCompileProgram" not found') + err = ( __nvrtcCompileProgram)(prog, numOptions, options) + return err + +cdef nvrtcResult _nvrtcGetPTXSize(nvrtcProgram prog, size_t* ptxSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetPTXSize + cuPythonInit() + if __nvrtcGetPTXSize == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetPTXSize" not found') + err = ( __nvrtcGetPTXSize)(prog, ptxSizeRet) + return err + +cdef nvrtcResult _nvrtcGetPTX(nvrtcProgram prog, char* ptx) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetPTX + cuPythonInit() + if __nvrtcGetPTX == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetPTX" not found') + err = ( __nvrtcGetPTX)(prog, ptx) + return err + +cdef nvrtcResult _nvrtcGetCUBINSize(nvrtcProgram prog, size_t* cubinSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetCUBINSize + cuPythonInit() + if __nvrtcGetCUBINSize == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetCUBINSize" not found') + err = ( __nvrtcGetCUBINSize)(prog, cubinSizeRet) + return err + +cdef nvrtcResult _nvrtcGetCUBIN(nvrtcProgram prog, char* cubin) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetCUBIN + cuPythonInit() + if __nvrtcGetCUBIN == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetCUBIN" not found') + err = ( __nvrtcGetCUBIN)(prog, cubin) + return err + +cdef nvrtcResult _nvrtcGetNVVMSize(nvrtcProgram prog, size_t* nvvmSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetNVVMSize + cuPythonInit() + if __nvrtcGetNVVMSize == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetNVVMSize" not found') + err = ( __nvrtcGetNVVMSize)(prog, nvvmSizeRet) + return err + +cdef nvrtcResult _nvrtcGetNVVM(nvrtcProgram prog, char* nvvm) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetNVVM + cuPythonInit() + if __nvrtcGetNVVM == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetNVVM" not found') + err = ( __nvrtcGetNVVM)(prog, nvvm) + return err + +cdef nvrtcResult _nvrtcGetLTOIRSize(nvrtcProgram prog, size_t* LTOIRSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetLTOIRSize + cuPythonInit() + if __nvrtcGetLTOIRSize == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetLTOIRSize" not found') + err = ( __nvrtcGetLTOIRSize)(prog, LTOIRSizeRet) + return err + +cdef nvrtcResult _nvrtcGetLTOIR(nvrtcProgram prog, char* LTOIR) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetLTOIR + cuPythonInit() + if __nvrtcGetLTOIR == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetLTOIR" not found') + err = ( __nvrtcGetLTOIR)(prog, LTOIR) + return err + +cdef nvrtcResult _nvrtcGetOptiXIRSize(nvrtcProgram prog, size_t* optixirSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetOptiXIRSize + cuPythonInit() + if __nvrtcGetOptiXIRSize == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetOptiXIRSize" not found') + err = ( __nvrtcGetOptiXIRSize)(prog, optixirSizeRet) + return err + +cdef nvrtcResult _nvrtcGetOptiXIR(nvrtcProgram prog, char* optixir) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetOptiXIR + cuPythonInit() + if __nvrtcGetOptiXIR == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetOptiXIR" not found') + err = ( __nvrtcGetOptiXIR)(prog, optixir) + return err + +cdef nvrtcResult _nvrtcGetProgramLogSize(nvrtcProgram prog, size_t* logSizeRet) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetProgramLogSize + cuPythonInit() + if __nvrtcGetProgramLogSize == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetProgramLogSize" not found') + err = ( __nvrtcGetProgramLogSize)(prog, logSizeRet) + return err + +cdef nvrtcResult _nvrtcGetProgramLog(nvrtcProgram prog, char* log) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetProgramLog + cuPythonInit() + if __nvrtcGetProgramLog == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetProgramLog" not found') + err = ( __nvrtcGetProgramLog)(prog, log) + return err + +cdef nvrtcResult _nvrtcAddNameExpression(nvrtcProgram prog, const char* name_expression) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcAddNameExpression + cuPythonInit() + if __nvrtcAddNameExpression == NULL: + with gil: + raise RuntimeError('Function "nvrtcAddNameExpression" not found') + err = ( __nvrtcAddNameExpression)(prog, name_expression) + return err + +cdef nvrtcResult _nvrtcGetLoweredName(nvrtcProgram prog, const char* name_expression, const char** lowered_name) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetLoweredName + cuPythonInit() + if __nvrtcGetLoweredName == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetLoweredName" not found') + err = ( __nvrtcGetLoweredName)(prog, name_expression, lowered_name) + return err + +cdef nvrtcResult _nvrtcGetPCHHeapSize(size_t* ret) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetPCHHeapSize + cuPythonInit() + if __nvrtcGetPCHHeapSize == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetPCHHeapSize" not found') + err = ( __nvrtcGetPCHHeapSize)(ret) + return err + +cdef nvrtcResult _nvrtcSetPCHHeapSize(size_t size) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcSetPCHHeapSize + cuPythonInit() + if __nvrtcSetPCHHeapSize == NULL: + with gil: + raise RuntimeError('Function "nvrtcSetPCHHeapSize" not found') + err = ( __nvrtcSetPCHHeapSize)(size) + return err + +cdef nvrtcResult _nvrtcGetPCHCreateStatus(nvrtcProgram prog) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetPCHCreateStatus + cuPythonInit() + if __nvrtcGetPCHCreateStatus == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetPCHCreateStatus" not found') + err = ( __nvrtcGetPCHCreateStatus)(prog) + return err + +cdef nvrtcResult _nvrtcGetPCHHeapSizeRequired(nvrtcProgram prog, size_t* size) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcGetPCHHeapSizeRequired + cuPythonInit() + if __nvrtcGetPCHHeapSizeRequired == NULL: + with gil: + raise RuntimeError('Function "nvrtcGetPCHHeapSizeRequired" not found') + err = ( __nvrtcGetPCHHeapSizeRequired)(prog, size) + return err + +cdef nvrtcResult _nvrtcSetFlowCallback(nvrtcProgram prog, void* callback, void* payload) except ?NVRTC_ERROR_INVALID_INPUT nogil: + global __nvrtcSetFlowCallback + cuPythonInit() + if __nvrtcSetFlowCallback == NULL: + with gil: + raise RuntimeError('Function "nvrtcSetFlowCallback" not found') + err = ( __nvrtcSetFlowCallback)(prog, callback, payload) + return err + +cdef dict func_ptrs = None + +cpdef dict _inspect_function_pointers(): + global func_ptrs + if func_ptrs is not None: + return func_ptrs + + cuPythonInit() + cdef dict data = {} + + global __nvrtcGetErrorString + data["__nvrtcGetErrorString"] = __nvrtcGetErrorString + global __nvrtcVersion + data["__nvrtcVersion"] = __nvrtcVersion + global __nvrtcGetNumSupportedArchs + data["__nvrtcGetNumSupportedArchs"] = __nvrtcGetNumSupportedArchs + global __nvrtcGetSupportedArchs + data["__nvrtcGetSupportedArchs"] = __nvrtcGetSupportedArchs + global __nvrtcCreateProgram + data["__nvrtcCreateProgram"] = __nvrtcCreateProgram + global __nvrtcDestroyProgram + data["__nvrtcDestroyProgram"] = __nvrtcDestroyProgram + global __nvrtcCompileProgram + data["__nvrtcCompileProgram"] = __nvrtcCompileProgram + global __nvrtcGetPTXSize + data["__nvrtcGetPTXSize"] = __nvrtcGetPTXSize + global __nvrtcGetPTX + data["__nvrtcGetPTX"] = __nvrtcGetPTX + global __nvrtcGetCUBINSize + data["__nvrtcGetCUBINSize"] = __nvrtcGetCUBINSize + global __nvrtcGetCUBIN + data["__nvrtcGetCUBIN"] = __nvrtcGetCUBIN + global __nvrtcGetNVVMSize + data["__nvrtcGetNVVMSize"] = __nvrtcGetNVVMSize + global __nvrtcGetNVVM + data["__nvrtcGetNVVM"] = __nvrtcGetNVVM + global __nvrtcGetLTOIRSize + data["__nvrtcGetLTOIRSize"] = __nvrtcGetLTOIRSize + global __nvrtcGetLTOIR + data["__nvrtcGetLTOIR"] = __nvrtcGetLTOIR + global __nvrtcGetOptiXIRSize + data["__nvrtcGetOptiXIRSize"] = __nvrtcGetOptiXIRSize + global __nvrtcGetOptiXIR + data["__nvrtcGetOptiXIR"] = __nvrtcGetOptiXIR + global __nvrtcGetProgramLogSize + data["__nvrtcGetProgramLogSize"] = __nvrtcGetProgramLogSize + global __nvrtcGetProgramLog + data["__nvrtcGetProgramLog"] = __nvrtcGetProgramLog + global __nvrtcAddNameExpression + data["__nvrtcAddNameExpression"] = __nvrtcAddNameExpression + global __nvrtcGetLoweredName + data["__nvrtcGetLoweredName"] = __nvrtcGetLoweredName + global __nvrtcGetPCHHeapSize + data["__nvrtcGetPCHHeapSize"] = __nvrtcGetPCHHeapSize + global __nvrtcSetPCHHeapSize + data["__nvrtcSetPCHHeapSize"] = __nvrtcSetPCHHeapSize + global __nvrtcGetPCHCreateStatus + data["__nvrtcGetPCHCreateStatus"] = __nvrtcGetPCHCreateStatus + global __nvrtcGetPCHHeapSizeRequired + data["__nvrtcGetPCHHeapSizeRequired"] = __nvrtcGetPCHHeapSizeRequired + global __nvrtcSetFlowCallback + data["__nvrtcSetFlowCallback"] = __nvrtcSetFlowCallback + + func_ptrs = data + return data + +cpdef _inspect_function_pointer(str name): + global func_ptrs + if func_ptrs is None: + func_ptrs = _inspect_function_pointers() + return func_ptrs[name] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime.cpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7f5aa9732026b67c9143c603c2c58f66f85db9aa --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime.cpp @@ -0,0 +1,46963 @@ +/* Generated by Cython 3.1.5 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [ + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/cuda.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/cudaProfiler.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/cuda_profiler_api.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/cuda_runtime.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/cuda_runtime_api.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/device_types.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/driver_types.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/library_types.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/surface_types.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/texture_types.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/vector_types.h" + ], + "extra_compile_args": [ + "-std=c++14", + "-fpermissive", + "-Wno-deprecated-declarations", + "-fno-var-tracking-assignments", + "-O3" + ], + "include_dirs": [ + "/opt/python/cp312-cp312/include", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include" + ], + "language": "c++", + "libraries": [ + "cudart_static", + "rt" + ], + "library_dirs": [ + "/tmp/build-env-3t3_8d9r/lib/python3.12/site-packages", + "/tmp/build-env-3t3_8d9r/lib", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib64", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib" + ], + "name": "cuda.bindings._bindings.cyruntime", + "sources": [ + "cuda/bindings/_bindings/cyruntime.pyx" + ] + }, + "module_name": "cuda.bindings._bindings.cyruntime" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_1_5" +#define CYTHON_HEX_VERSION 0x030105F0 +#define CYTHON_FUTURE_DIVISION 1 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef HAVE_LONG_LONG + #define HAVE_LONG_LONG +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_CLINE_IN_TRACEBACK + #define CYTHON_CLINE_IN_TRACEBACK 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1) +#endif +#endif +#define CYTHON_BACKPORT_VECTORCALL (CYTHON_METH_FASTCALL && PY_VERSION_HEX < 0x030800B1) +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME + #define CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME 100 +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + #endif + #endif + #if _MSC_VER < 1300 + #ifdef _WIN64 + typedef unsigned long long __pyx_uintptr_t; + #else + typedef unsigned int __pyx_uintptr_t; + #endif + #else + #ifdef _WIN64 + typedef unsigned __int64 __pyx_uintptr_t; + #else + typedef unsigned __int32 __pyx_uintptr_t; + #endif + #endif +#else + #include + typedef uintptr_t __pyx_uintptr_t; +#endif +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#elif CYTHON_BACKPORT_VECTORCALL + typedef PyObject *(*__pyx_vectorcallfunc)(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames); + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1)) + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(((size_t)(n)) & ~__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if __PYX_LIMITED_VERSION_HEX < 0x03090000 + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS && PY_VERSION_HEX >= 0x03080000 +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static CYTHON_INLINE PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__cuda__bindings___bindings__cyruntime +#define __PYX_HAVE_API__cuda__bindings___bindings__cyruntime +/* Early includes */ +#include "vector_types.h" +#include "driver_types.h" +#include "surface_types.h" +#include "texture_types.h" +#include "library_types.h" +#include "cuda_runtime_api.h" +#include "device_types.h" +#include +#include "cuda.h" +#include "cudaProfiler.h" +#include "cuda_runtime.h" +#include "cuda_profiler_api.h" + + #define CUDA_API_PER_THREAD_DEFAULT_STREAM + +#include +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value); + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "cuda/bindings/_bindings/cyruntime.pyx", + "cuda/bindings/_bindings/../_lib/cyruntime/cyruntime.pxi", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* ForceInitThreads.proto */ +#ifndef __PYX_FORCE_INIT_THREADS + #define __PYX_FORCE_INIT_THREADS 0 +#endif + +/* NoFastGil.proto */ +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() + +/* CriticalSections.proto */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_Begin1(cs, arg) (void)cs +#define __Pyx_PyCriticalSection_Begin2(cs, arg1, arg2) (void)cs +#define __Pyx_PyCriticalSection_End1(cs) +#define __Pyx_PyCriticalSection_End2(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_Begin1 PyCriticalSection_Begin +#define __Pyx_PyCriticalSection_Begin2 PyCriticalSection2_Begin +#define __Pyx_PyCriticalSection_End1 PyCriticalSection_End +#define __Pyx_PyCriticalSection_End2 PyCriticalSection2_End +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* Atomics.proto */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif +#if CYTHON_ATOMICS + #define __pyx_add_acquisition_count(memview)\ + __pyx_atomic_incr_relaxed(__pyx_get_slice_count_pointer(memview)) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_atomic_decr_acq_rel(__pyx_get_slice_count_pointer(memview)) +#else + #define __pyx_add_acquisition_count(memview)\ + __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_sub_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) +#endif + +/* #### Code section: numeric_typedefs ### */ + +/* "cuda/bindings/cyruntime.pxd":10 + * include "cyruntime_types.pxi" + * + * ctypedef unsigned int GLenum # <<<<<<<<<<<<<< + * + * ctypedef unsigned int GLuint +*/ +typedef unsigned int __pyx_t_4cuda_8bindings_9cyruntime_GLenum; + +/* "cuda/bindings/cyruntime.pxd":12 + * ctypedef unsigned int GLenum + * + * ctypedef unsigned int GLuint # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef unsigned int __pyx_t_4cuda_8bindings_9cyruntime_GLuint; + +/* "cuda/bindings/cyruntime.pxd":24 + * ctypedef void* EGLStreamKHR + * + * ctypedef unsigned int EGLint # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef unsigned int __pyx_t_4cuda_8bindings_9cyruntime_EGLint; + +/* "cuda/bindings/cyruntime.pxd":31 + * ctypedef void* EGLSyncKHR + * + * ctypedef uint32_t VdpDevice # <<<<<<<<<<<<<< + * + * ctypedef unsigned long long VdpGetProcAddress +*/ +typedef uint32_t __pyx_t_4cuda_8bindings_9cyruntime_VdpDevice; + +/* "cuda/bindings/cyruntime.pxd":33 + * ctypedef uint32_t VdpDevice + * + * ctypedef unsigned long long VdpGetProcAddress # <<<<<<<<<<<<<< + * + * ctypedef uint32_t VdpVideoSurface +*/ +typedef unsigned PY_LONG_LONG __pyx_t_4cuda_8bindings_9cyruntime_VdpGetProcAddress; + +/* "cuda/bindings/cyruntime.pxd":35 + * ctypedef unsigned long long VdpGetProcAddress + * + * ctypedef uint32_t VdpVideoSurface # <<<<<<<<<<<<<< + * + * ctypedef uint32_t VdpOutputSurface +*/ +typedef uint32_t __pyx_t_4cuda_8bindings_9cyruntime_VdpVideoSurface; + +/* "cuda/bindings/cyruntime.pxd":37 + * ctypedef uint32_t VdpVideoSurface + * + * ctypedef uint32_t VdpOutputSurface # <<<<<<<<<<<<<< + * + * cdef enum cudaEglFrameType_enum: +*/ +typedef uint32_t __pyx_t_4cuda_8bindings_9cyruntime_VdpOutputSurface; + +/* "cuda/bindings/cydriver.pxd":2602 + * ctypedef CUGLmap_flags_enum CUGLmap_flags + * + * ctypedef unsigned int GLenum # <<<<<<<<<<<<<< + * + * ctypedef unsigned int GLuint +*/ +typedef unsigned int __pyx_t_4cuda_8bindings_8cydriver_GLenum; + +/* "cuda/bindings/cydriver.pxd":2604 + * ctypedef unsigned int GLenum + * + * ctypedef unsigned int GLuint # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef unsigned int __pyx_t_4cuda_8bindings_8cydriver_GLuint; + +/* "cuda/bindings/cydriver.pxd":2616 + * ctypedef void* EGLStreamKHR + * + * ctypedef unsigned int EGLint # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef unsigned int __pyx_t_4cuda_8bindings_8cydriver_EGLint; + +/* "cuda/bindings/cydriver.pxd":2623 + * ctypedef void* EGLSyncKHR + * + * ctypedef uint32_t VdpDevice # <<<<<<<<<<<<<< + * + * ctypedef unsigned long long VdpGetProcAddress +*/ +typedef uint32_t __pyx_t_4cuda_8bindings_8cydriver_VdpDevice; + +/* "cuda/bindings/cydriver.pxd":2625 + * ctypedef uint32_t VdpDevice + * + * ctypedef unsigned long long VdpGetProcAddress # <<<<<<<<<<<<<< + * + * ctypedef uint32_t VdpVideoSurface +*/ +typedef unsigned PY_LONG_LONG __pyx_t_4cuda_8bindings_8cydriver_VdpGetProcAddress; + +/* "cuda/bindings/cydriver.pxd":2627 + * ctypedef unsigned long long VdpGetProcAddress + * + * ctypedef uint32_t VdpVideoSurface # <<<<<<<<<<<<<< + * + * ctypedef uint32_t VdpOutputSurface +*/ +typedef uint32_t __pyx_t_4cuda_8bindings_8cydriver_VdpVideoSurface; + +/* "cuda/bindings/cydriver.pxd":2629 + * ctypedef uint32_t VdpVideoSurface + * + * ctypedef uint32_t VdpOutputSurface # <<<<<<<<<<<<<< + * + * cdef CUresult cuGetErrorString(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil +*/ +typedef uint32_t __pyx_t_4cuda_8bindings_8cydriver_VdpOutputSurface; +/* #### Code section: complex_type_declarations ### */ +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ +struct __pyx_t_4cuda_8bindings_9cyruntime_cudaEglPlaneDesc_st; +union __pyx_t_4cuda_8bindings_9cyruntime_anon_union11; +struct __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame_st; + +/* "cuda/bindings/cyruntime.pxd":39 + * ctypedef uint32_t VdpOutputSurface + * + * cdef enum cudaEglFrameType_enum: # <<<<<<<<<<<<<< + * cudaEglFrameTypeArray = 0 + * cudaEglFrameTypePitch = 1 +*/ +enum __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrameType_enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglFrameTypeArray = 0, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglFrameTypePitch = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":45 + * ctypedef cudaEglFrameType_enum cudaEglFrameType + * + * cdef enum cudaEglResourceLocationFlags_enum: # <<<<<<<<<<<<<< + * cudaEglResourceLocationSysmem = 0 + * cudaEglResourceLocationVidmem = 1 +*/ +enum __pyx_t_4cuda_8bindings_9cyruntime_cudaEglResourceLocationFlags_enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglResourceLocationSysmem = 0, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglResourceLocationVidmem = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":51 + * ctypedef cudaEglResourceLocationFlags_enum cudaEglResourceLocationFlags + * + * cdef enum cudaEglColorFormat_enum: # <<<<<<<<<<<<<< + * cudaEglColorFormatYUV420Planar = 0 + * cudaEglColorFormatYUV420SemiPlanar = 1 +*/ +enum __pyx_t_4cuda_8bindings_9cyruntime_cudaEglColorFormat_enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420Planar = 0, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420SemiPlanar = 1, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV422Planar = 2, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV422SemiPlanar = 3, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatARGB = 6, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatRGBA = 7, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatL = 8, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatR = 9, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV444Planar = 10, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV444SemiPlanar = 11, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUYV422 = 12, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY422 = 13, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatABGR = 14, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBGRA = 15, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatA = 16, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatRG = 17, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatAYUV = 18, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU444SemiPlanar = 19, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU422SemiPlanar = 20, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420SemiPlanar = 21, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_444SemiPlanar = 22, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar = 23, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_444SemiPlanar = 24, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_420SemiPlanar = 25, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatVYUY_ER = 26, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY_ER = 27, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUYV_ER = 28, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVYU_ER = 29, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUVA_ER = 31, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatAYUV_ER = 32, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV444Planar_ER = 33, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV422Planar_ER = 34, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420Planar_ER = 35, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV444SemiPlanar_ER = 36, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV422SemiPlanar_ER = 37, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420SemiPlanar_ER = 38, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU444Planar_ER = 39, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU422Planar_ER = 40, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420Planar_ER = 41, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU444SemiPlanar_ER = 42, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU422SemiPlanar_ER = 43, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420SemiPlanar_ER = 44, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerRGGB = 45, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerBGGR = 46, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerGRBG = 47, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerGBRG = 48, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10RGGB = 49, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10BGGR = 50, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10GRBG = 51, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10GBRG = 52, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12RGGB = 53, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12BGGR = 54, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12GRBG = 55, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12GBRG = 56, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer14RGGB = 57, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer14BGGR = 58, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer14GRBG = 59, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer14GBRG = 60, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer20RGGB = 61, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer20BGGR = 62, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer20GRBG = 63, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer20GBRG = 64, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU444Planar = 65, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU422Planar = 66, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420Planar = 67, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerIspRGGB = 68, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerIspBGGR = 69, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerIspGRBG = 70, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerIspGBRG = 71, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerBCCR = 72, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerRCCB = 73, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerCRBC = 74, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerCBRC = 75, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10CCCC = 76, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12BCCR = 77, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12RCCB = 78, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12CRBC = 79, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12CBRC = 80, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12CCCC = 81, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY = 82, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420SemiPlanar_2020 = 83, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420SemiPlanar_2020 = 84, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420Planar_2020 = 85, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420Planar_2020 = 86, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420SemiPlanar_709 = 87, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420SemiPlanar_709 = 88, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420Planar_709 = 89, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420Planar_709 = 90, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar_709 = 91, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar_2020 = 92, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_422SemiPlanar_2020 = 93, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_422SemiPlanar = 94, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_422SemiPlanar_709 = 95, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY_ER = 96, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY_709_ER = 97, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10_ER = 98, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10_709_ER = 99, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12_ER = 0x64, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12_709_ER = 0x65, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUVA = 0x66, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVYU = 0x68, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatVYUY = 0x69, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar_ER = 0x6A, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER = 0x6B, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_444SemiPlanar_ER = 0x6C, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER = 0x6D, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_420SemiPlanar_ER = 0x6E, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER = 0x6F, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_444SemiPlanar_ER = 0x70, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_444SemiPlanar_709_ER = 0x71, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY709 = 0x72, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY709_ER = 0x73, + __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY2020 = 0x74 +}; + +/* "cuda/bindings/cyruntime.pxd":197 + * ctypedef CUeglStreamConnection_st* cudaEglStreamConnection + * + * cdef enum cudaGLDeviceList: # <<<<<<<<<<<<<< + * cudaGLDeviceListAll = 1 + * cudaGLDeviceListCurrentFrame = 2 +*/ +enum __pyx_t_4cuda_8bindings_9cyruntime_cudaGLDeviceList { + __pyx_e_4cuda_8bindings_9cyruntime_cudaGLDeviceListAll = 1, + __pyx_e_4cuda_8bindings_9cyruntime_cudaGLDeviceListCurrentFrame = 2, + __pyx_e_4cuda_8bindings_9cyruntime_cudaGLDeviceListNextFrame = 3 +}; + +/* "cuda/bindings/cyruntime.pxd":202 + * cudaGLDeviceListNextFrame = 3 + * + * cdef enum cudaGLMapFlags: # <<<<<<<<<<<<<< + * cudaGLMapFlagsNone = 0 + * cudaGLMapFlagsReadOnly = 1 +*/ +enum __pyx_t_4cuda_8bindings_9cyruntime_cudaGLMapFlags { + __pyx_e_4cuda_8bindings_9cyruntime_cudaGLMapFlagsNone = 0, + __pyx_e_4cuda_8bindings_9cyruntime_cudaGLMapFlagsReadOnly = 1, + __pyx_e_4cuda_8bindings_9cyruntime_cudaGLMapFlagsWriteDiscard = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":831 + * cdef cudaError_t getLocalRuntimeVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil + * + * cdef enum: cudaHostAllocDefault = 0 # <<<<<<<<<<<<<< + * + * cdef enum: cudaHostAllocPortable = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaHostAllocDefault = 0 +}; + +/* "cuda/bindings/cyruntime.pxd":833 + * cdef enum: cudaHostAllocDefault = 0 + * + * cdef enum: cudaHostAllocPortable = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaHostAllocMapped = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaHostAllocPortable = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":835 + * cdef enum: cudaHostAllocPortable = 1 + * + * cdef enum: cudaHostAllocMapped = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaHostAllocWriteCombined = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaHostAllocMapped = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":837 + * cdef enum: cudaHostAllocMapped = 2 + * + * cdef enum: cudaHostAllocWriteCombined = 4 # <<<<<<<<<<<<<< + * + * cdef enum: cudaHostRegisterDefault = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaHostAllocWriteCombined = 4 +}; + +/* "cuda/bindings/cyruntime.pxd":839 + * cdef enum: cudaHostAllocWriteCombined = 4 + * + * cdef enum: cudaHostRegisterDefault = 0 # <<<<<<<<<<<<<< + * + * cdef enum: cudaHostRegisterPortable = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaHostRegisterDefault = 0 +}; + +/* "cuda/bindings/cyruntime.pxd":841 + * cdef enum: cudaHostRegisterDefault = 0 + * + * cdef enum: cudaHostRegisterPortable = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaHostRegisterMapped = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaHostRegisterPortable = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":843 + * cdef enum: cudaHostRegisterPortable = 1 + * + * cdef enum: cudaHostRegisterMapped = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaHostRegisterIoMemory = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaHostRegisterMapped = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":845 + * cdef enum: cudaHostRegisterMapped = 2 + * + * cdef enum: cudaHostRegisterIoMemory = 4 # <<<<<<<<<<<<<< + * + * cdef enum: cudaHostRegisterReadOnly = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaHostRegisterIoMemory = 4 +}; + +/* "cuda/bindings/cyruntime.pxd":847 + * cdef enum: cudaHostRegisterIoMemory = 4 + * + * cdef enum: cudaHostRegisterReadOnly = 8 # <<<<<<<<<<<<<< + * + * cdef enum: cudaPeerAccessDefault = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaHostRegisterReadOnly = 8 +}; + +/* "cuda/bindings/cyruntime.pxd":849 + * cdef enum: cudaHostRegisterReadOnly = 8 + * + * cdef enum: cudaPeerAccessDefault = 0 # <<<<<<<<<<<<<< + * + * cdef enum: cudaStreamDefault = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaPeerAccessDefault = 0 +}; + +/* "cuda/bindings/cyruntime.pxd":851 + * cdef enum: cudaPeerAccessDefault = 0 + * + * cdef enum: cudaStreamDefault = 0 # <<<<<<<<<<<<<< + * + * cdef enum: cudaStreamNonBlocking = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaStreamDefault = 0 +}; + +/* "cuda/bindings/cyruntime.pxd":853 + * cdef enum: cudaStreamDefault = 0 + * + * cdef enum: cudaStreamNonBlocking = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaStreamLegacy = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaStreamNonBlocking = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":855 + * cdef enum: cudaStreamNonBlocking = 1 + * + * cdef enum: cudaStreamLegacy = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaStreamPerThread = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaStreamLegacy = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":857 + * cdef enum: cudaStreamLegacy = 1 + * + * cdef enum: cudaStreamPerThread = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaEventDefault = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaStreamPerThread = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":859 + * cdef enum: cudaStreamPerThread = 2 + * + * cdef enum: cudaEventDefault = 0 # <<<<<<<<<<<<<< + * + * cdef enum: cudaEventBlockingSync = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaEventDefault = 0 +}; + +/* "cuda/bindings/cyruntime.pxd":861 + * cdef enum: cudaEventDefault = 0 + * + * cdef enum: cudaEventBlockingSync = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaEventDisableTiming = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaEventBlockingSync = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":863 + * cdef enum: cudaEventBlockingSync = 1 + * + * cdef enum: cudaEventDisableTiming = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaEventInterprocess = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaEventDisableTiming = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":865 + * cdef enum: cudaEventDisableTiming = 2 + * + * cdef enum: cudaEventInterprocess = 4 # <<<<<<<<<<<<<< + * + * cdef enum: cudaEventRecordDefault = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaEventInterprocess = 4 +}; + +/* "cuda/bindings/cyruntime.pxd":867 + * cdef enum: cudaEventInterprocess = 4 + * + * cdef enum: cudaEventRecordDefault = 0 # <<<<<<<<<<<<<< + * + * cdef enum: cudaEventRecordExternal = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaEventRecordDefault = 0 +}; + +/* "cuda/bindings/cyruntime.pxd":869 + * cdef enum: cudaEventRecordDefault = 0 + * + * cdef enum: cudaEventRecordExternal = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaEventWaitDefault = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaEventRecordExternal = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":871 + * cdef enum: cudaEventRecordExternal = 1 + * + * cdef enum: cudaEventWaitDefault = 0 # <<<<<<<<<<<<<< + * + * cdef enum: cudaEventWaitExternal = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaEventWaitDefault = 0 +}; + +/* "cuda/bindings/cyruntime.pxd":873 + * cdef enum: cudaEventWaitDefault = 0 + * + * cdef enum: cudaEventWaitExternal = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaDeviceScheduleAuto = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaEventWaitExternal = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":875 + * cdef enum: cudaEventWaitExternal = 1 + * + * cdef enum: cudaDeviceScheduleAuto = 0 # <<<<<<<<<<<<<< + * + * cdef enum: cudaDeviceScheduleSpin = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaDeviceScheduleAuto = 0 +}; + +/* "cuda/bindings/cyruntime.pxd":877 + * cdef enum: cudaDeviceScheduleAuto = 0 + * + * cdef enum: cudaDeviceScheduleSpin = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaDeviceScheduleYield = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaDeviceScheduleSpin = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":879 + * cdef enum: cudaDeviceScheduleSpin = 1 + * + * cdef enum: cudaDeviceScheduleYield = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaDeviceScheduleBlockingSync = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaDeviceScheduleYield = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":881 + * cdef enum: cudaDeviceScheduleYield = 2 + * + * cdef enum: cudaDeviceScheduleBlockingSync = 4 # <<<<<<<<<<<<<< + * + * cdef enum: cudaDeviceBlockingSync = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaDeviceScheduleBlockingSync = 4 +}; + +/* "cuda/bindings/cyruntime.pxd":883 + * cdef enum: cudaDeviceScheduleBlockingSync = 4 + * + * cdef enum: cudaDeviceBlockingSync = 4 # <<<<<<<<<<<<<< + * + * cdef enum: cudaDeviceScheduleMask = 7 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaDeviceBlockingSync = 4 +}; + +/* "cuda/bindings/cyruntime.pxd":885 + * cdef enum: cudaDeviceBlockingSync = 4 + * + * cdef enum: cudaDeviceScheduleMask = 7 # <<<<<<<<<<<<<< + * + * cdef enum: cudaDeviceMapHost = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaDeviceScheduleMask = 7 +}; + +/* "cuda/bindings/cyruntime.pxd":887 + * cdef enum: cudaDeviceScheduleMask = 7 + * + * cdef enum: cudaDeviceMapHost = 8 # <<<<<<<<<<<<<< + * + * cdef enum: cudaDeviceLmemResizeToMax = 16 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaDeviceMapHost = 8 +}; + +/* "cuda/bindings/cyruntime.pxd":889 + * cdef enum: cudaDeviceMapHost = 8 + * + * cdef enum: cudaDeviceLmemResizeToMax = 16 # <<<<<<<<<<<<<< + * + * cdef enum: cudaDeviceSyncMemops = 128 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaDeviceLmemResizeToMax = 16 +}; + +/* "cuda/bindings/cyruntime.pxd":891 + * cdef enum: cudaDeviceLmemResizeToMax = 16 + * + * cdef enum: cudaDeviceSyncMemops = 128 # <<<<<<<<<<<<<< + * + * cdef enum: cudaDeviceMask = 255 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaDeviceSyncMemops = 0x80 +}; + +/* "cuda/bindings/cyruntime.pxd":893 + * cdef enum: cudaDeviceSyncMemops = 128 + * + * cdef enum: cudaDeviceMask = 255 # <<<<<<<<<<<<<< + * + * cdef enum: cudaArrayDefault = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaDeviceMask = 0xFF +}; + +/* "cuda/bindings/cyruntime.pxd":895 + * cdef enum: cudaDeviceMask = 255 + * + * cdef enum: cudaArrayDefault = 0 # <<<<<<<<<<<<<< + * + * cdef enum: cudaArrayLayered = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaArrayDefault = 0 +}; + +/* "cuda/bindings/cyruntime.pxd":897 + * cdef enum: cudaArrayDefault = 0 + * + * cdef enum: cudaArrayLayered = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaArraySurfaceLoadStore = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaArrayLayered = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":899 + * cdef enum: cudaArrayLayered = 1 + * + * cdef enum: cudaArraySurfaceLoadStore = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaArrayCubemap = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaArraySurfaceLoadStore = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":901 + * cdef enum: cudaArraySurfaceLoadStore = 2 + * + * cdef enum: cudaArrayCubemap = 4 # <<<<<<<<<<<<<< + * + * cdef enum: cudaArrayTextureGather = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaArrayCubemap = 4 +}; + +/* "cuda/bindings/cyruntime.pxd":903 + * cdef enum: cudaArrayCubemap = 4 + * + * cdef enum: cudaArrayTextureGather = 8 # <<<<<<<<<<<<<< + * + * cdef enum: cudaArrayColorAttachment = 32 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaArrayTextureGather = 8 +}; + +/* "cuda/bindings/cyruntime.pxd":905 + * cdef enum: cudaArrayTextureGather = 8 + * + * cdef enum: cudaArrayColorAttachment = 32 # <<<<<<<<<<<<<< + * + * cdef enum: cudaArraySparse = 64 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaArrayColorAttachment = 32 +}; + +/* "cuda/bindings/cyruntime.pxd":907 + * cdef enum: cudaArrayColorAttachment = 32 + * + * cdef enum: cudaArraySparse = 64 # <<<<<<<<<<<<<< + * + * cdef enum: cudaArrayDeferredMapping = 128 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaArraySparse = 64 +}; + +/* "cuda/bindings/cyruntime.pxd":909 + * cdef enum: cudaArraySparse = 64 + * + * cdef enum: cudaArrayDeferredMapping = 128 # <<<<<<<<<<<<<< + * + * cdef enum: cudaIpcMemLazyEnablePeerAccess = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaArrayDeferredMapping = 0x80 +}; + +/* "cuda/bindings/cyruntime.pxd":911 + * cdef enum: cudaArrayDeferredMapping = 128 + * + * cdef enum: cudaIpcMemLazyEnablePeerAccess = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaMemAttachGlobal = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaIpcMemLazyEnablePeerAccess = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":913 + * cdef enum: cudaIpcMemLazyEnablePeerAccess = 1 + * + * cdef enum: cudaMemAttachGlobal = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaMemAttachHost = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaMemAttachGlobal = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":915 + * cdef enum: cudaMemAttachGlobal = 1 + * + * cdef enum: cudaMemAttachHost = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaMemAttachSingle = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaMemAttachHost = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":917 + * cdef enum: cudaMemAttachHost = 2 + * + * cdef enum: cudaMemAttachSingle = 4 # <<<<<<<<<<<<<< + * + * cdef enum: cudaOccupancyDefault = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaMemAttachSingle = 4 +}; + +/* "cuda/bindings/cyruntime.pxd":919 + * cdef enum: cudaMemAttachSingle = 4 + * + * cdef enum: cudaOccupancyDefault = 0 # <<<<<<<<<<<<<< + * + * cdef enum: cudaOccupancyDisableCachingOverride = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaOccupancyDefault = 0 +}; + +/* "cuda/bindings/cyruntime.pxd":921 + * cdef enum: cudaOccupancyDefault = 0 + * + * cdef enum: cudaOccupancyDisableCachingOverride = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaCpuDeviceId = -1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaOccupancyDisableCachingOverride = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":923 + * cdef enum: cudaOccupancyDisableCachingOverride = 1 + * + * cdef enum: cudaCpuDeviceId = -1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaInvalidDeviceId = -2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaCpuDeviceId = -1L +}; + +/* "cuda/bindings/cyruntime.pxd":925 + * cdef enum: cudaCpuDeviceId = -1 + * + * cdef enum: cudaInvalidDeviceId = -2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaInitDeviceFlagsAreValid = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaInvalidDeviceId = -2L +}; + +/* "cuda/bindings/cyruntime.pxd":927 + * cdef enum: cudaInvalidDeviceId = -2 + * + * cdef enum: cudaInitDeviceFlagsAreValid = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaCooperativeLaunchMultiDeviceNoPreSync = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaInitDeviceFlagsAreValid = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":929 + * cdef enum: cudaInitDeviceFlagsAreValid = 1 + * + * cdef enum: cudaCooperativeLaunchMultiDeviceNoPreSync = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaCooperativeLaunchMultiDeviceNoPostSync = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaCooperativeLaunchMultiDeviceNoPreSync = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":931 + * cdef enum: cudaCooperativeLaunchMultiDeviceNoPreSync = 1 + * + * cdef enum: cudaCooperativeLaunchMultiDeviceNoPostSync = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaArraySparsePropertiesSingleMipTail = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaCooperativeLaunchMultiDeviceNoPostSync = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":933 + * cdef enum: cudaCooperativeLaunchMultiDeviceNoPostSync = 2 + * + * cdef enum: cudaArraySparsePropertiesSingleMipTail = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaMemPoolCreateUsageHwDecompress = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaArraySparsePropertiesSingleMipTail = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":935 + * cdef enum: cudaArraySparsePropertiesSingleMipTail = 1 + * + * cdef enum: cudaMemPoolCreateUsageHwDecompress = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_IPC_HANDLE_SIZE = 64 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaMemPoolCreateUsageHwDecompress = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":937 + * cdef enum: cudaMemPoolCreateUsageHwDecompress = 2 + * + * cdef enum: CUDA_IPC_HANDLE_SIZE = 64 # <<<<<<<<<<<<<< + * + * cdef enum: cudaExternalMemoryDedicated = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_CUDA_IPC_HANDLE_SIZE = 64 +}; + +/* "cuda/bindings/cyruntime.pxd":939 + * cdef enum: CUDA_IPC_HANDLE_SIZE = 64 + * + * cdef enum: cudaExternalMemoryDedicated = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaExternalSemaphoreSignalSkipNvSciBufMemSync = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaExternalMemoryDedicated = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":941 + * cdef enum: cudaExternalMemoryDedicated = 1 + * + * cdef enum: cudaExternalSemaphoreSignalSkipNvSciBufMemSync = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaExternalSemaphoreWaitSkipNvSciBufMemSync = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaExternalSemaphoreSignalSkipNvSciBufMemSync = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":943 + * cdef enum: cudaExternalSemaphoreSignalSkipNvSciBufMemSync = 1 + * + * cdef enum: cudaExternalSemaphoreWaitSkipNvSciBufMemSync = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaNvSciSyncAttrSignal = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaExternalSemaphoreWaitSkipNvSciBufMemSync = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":945 + * cdef enum: cudaExternalSemaphoreWaitSkipNvSciBufMemSync = 2 + * + * cdef enum: cudaNvSciSyncAttrSignal = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaNvSciSyncAttrWait = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaNvSciSyncAttrSignal = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":947 + * cdef enum: cudaNvSciSyncAttrSignal = 1 + * + * cdef enum: cudaNvSciSyncAttrWait = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaGraphKernelNodePortDefault = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaNvSciSyncAttrWait = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":949 + * cdef enum: cudaNvSciSyncAttrWait = 2 + * + * cdef enum: cudaGraphKernelNodePortDefault = 0 # <<<<<<<<<<<<<< + * + * cdef enum: cudaGraphKernelNodePortProgrammatic = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaGraphKernelNodePortDefault = 0 +}; + +/* "cuda/bindings/cyruntime.pxd":951 + * cdef enum: cudaGraphKernelNodePortDefault = 0 + * + * cdef enum: cudaGraphKernelNodePortProgrammatic = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaGraphKernelNodePortLaunchCompletion = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaGraphKernelNodePortProgrammatic = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":953 + * cdef enum: cudaGraphKernelNodePortProgrammatic = 1 + * + * cdef enum: cudaGraphKernelNodePortLaunchCompletion = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaStreamAttributeAccessPolicyWindow = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaGraphKernelNodePortLaunchCompletion = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":955 + * cdef enum: cudaGraphKernelNodePortLaunchCompletion = 2 + * + * cdef enum: cudaStreamAttributeAccessPolicyWindow = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaStreamAttributeSynchronizationPolicy = 3 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaStreamAttributeAccessPolicyWindow = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":957 + * cdef enum: cudaStreamAttributeAccessPolicyWindow = 1 + * + * cdef enum: cudaStreamAttributeSynchronizationPolicy = 3 # <<<<<<<<<<<<<< + * + * cdef enum: cudaStreamAttributeMemSyncDomainMap = 9 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaStreamAttributeSynchronizationPolicy = 3 +}; + +/* "cuda/bindings/cyruntime.pxd":959 + * cdef enum: cudaStreamAttributeSynchronizationPolicy = 3 + * + * cdef enum: cudaStreamAttributeMemSyncDomainMap = 9 # <<<<<<<<<<<<<< + * + * cdef enum: cudaStreamAttributeMemSyncDomain = 10 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaStreamAttributeMemSyncDomainMap = 9 +}; + +/* "cuda/bindings/cyruntime.pxd":961 + * cdef enum: cudaStreamAttributeMemSyncDomainMap = 9 + * + * cdef enum: cudaStreamAttributeMemSyncDomain = 10 # <<<<<<<<<<<<<< + * + * cdef enum: cudaStreamAttributePriority = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaStreamAttributeMemSyncDomain = 10 +}; + +/* "cuda/bindings/cyruntime.pxd":963 + * cdef enum: cudaStreamAttributeMemSyncDomain = 10 + * + * cdef enum: cudaStreamAttributePriority = 8 # <<<<<<<<<<<<<< + * + * cdef enum: cudaKernelNodeAttributeAccessPolicyWindow = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaStreamAttributePriority = 8 +}; + +/* "cuda/bindings/cyruntime.pxd":965 + * cdef enum: cudaStreamAttributePriority = 8 + * + * cdef enum: cudaKernelNodeAttributeAccessPolicyWindow = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaKernelNodeAttributeCooperative = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaKernelNodeAttributeAccessPolicyWindow = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":967 + * cdef enum: cudaKernelNodeAttributeAccessPolicyWindow = 1 + * + * cdef enum: cudaKernelNodeAttributeCooperative = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaKernelNodeAttributePriority = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaKernelNodeAttributeCooperative = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":969 + * cdef enum: cudaKernelNodeAttributeCooperative = 2 + * + * cdef enum: cudaKernelNodeAttributePriority = 8 # <<<<<<<<<<<<<< + * + * cdef enum: cudaKernelNodeAttributeClusterDimension = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaKernelNodeAttributePriority = 8 +}; + +/* "cuda/bindings/cyruntime.pxd":971 + * cdef enum: cudaKernelNodeAttributePriority = 8 + * + * cdef enum: cudaKernelNodeAttributeClusterDimension = 4 # <<<<<<<<<<<<<< + * + * cdef enum: cudaKernelNodeAttributeClusterSchedulingPolicyPreference = 5 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaKernelNodeAttributeClusterDimension = 4 +}; + +/* "cuda/bindings/cyruntime.pxd":973 + * cdef enum: cudaKernelNodeAttributeClusterDimension = 4 + * + * cdef enum: cudaKernelNodeAttributeClusterSchedulingPolicyPreference = 5 # <<<<<<<<<<<<<< + * + * cdef enum: cudaKernelNodeAttributeMemSyncDomainMap = 9 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaKernelNodeAttributeClusterSchedulingPolicyPreference = 5 +}; + +/* "cuda/bindings/cyruntime.pxd":975 + * cdef enum: cudaKernelNodeAttributeClusterSchedulingPolicyPreference = 5 + * + * cdef enum: cudaKernelNodeAttributeMemSyncDomainMap = 9 # <<<<<<<<<<<<<< + * + * cdef enum: cudaKernelNodeAttributeMemSyncDomain = 10 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaKernelNodeAttributeMemSyncDomainMap = 9 +}; + +/* "cuda/bindings/cyruntime.pxd":977 + * cdef enum: cudaKernelNodeAttributeMemSyncDomainMap = 9 + * + * cdef enum: cudaKernelNodeAttributeMemSyncDomain = 10 # <<<<<<<<<<<<<< + * + * cdef enum: cudaKernelNodeAttributePreferredSharedMemoryCarveout = 14 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaKernelNodeAttributeMemSyncDomain = 10 +}; + +/* "cuda/bindings/cyruntime.pxd":979 + * cdef enum: cudaKernelNodeAttributeMemSyncDomain = 10 + * + * cdef enum: cudaKernelNodeAttributePreferredSharedMemoryCarveout = 14 # <<<<<<<<<<<<<< + * + * cdef enum: cudaKernelNodeAttributeDeviceUpdatableKernelNode = 13 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaKernelNodeAttributePreferredSharedMemoryCarveout = 14 +}; + +/* "cuda/bindings/cyruntime.pxd":981 + * cdef enum: cudaKernelNodeAttributePreferredSharedMemoryCarveout = 14 + * + * cdef enum: cudaKernelNodeAttributeDeviceUpdatableKernelNode = 13 # <<<<<<<<<<<<<< + * + * cdef enum: cudaSurfaceType1D = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaKernelNodeAttributeDeviceUpdatableKernelNode = 13 +}; + +/* "cuda/bindings/cyruntime.pxd":983 + * cdef enum: cudaKernelNodeAttributeDeviceUpdatableKernelNode = 13 + * + * cdef enum: cudaSurfaceType1D = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaSurfaceType2D = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaSurfaceType1D = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":985 + * cdef enum: cudaSurfaceType1D = 1 + * + * cdef enum: cudaSurfaceType2D = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaSurfaceType3D = 3 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaSurfaceType2D = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":987 + * cdef enum: cudaSurfaceType2D = 2 + * + * cdef enum: cudaSurfaceType3D = 3 # <<<<<<<<<<<<<< + * + * cdef enum: cudaSurfaceTypeCubemap = 12 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaSurfaceType3D = 3 +}; + +/* "cuda/bindings/cyruntime.pxd":989 + * cdef enum: cudaSurfaceType3D = 3 + * + * cdef enum: cudaSurfaceTypeCubemap = 12 # <<<<<<<<<<<<<< + * + * cdef enum: cudaSurfaceType1DLayered = 241 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaSurfaceTypeCubemap = 12 +}; + +/* "cuda/bindings/cyruntime.pxd":991 + * cdef enum: cudaSurfaceTypeCubemap = 12 + * + * cdef enum: cudaSurfaceType1DLayered = 241 # <<<<<<<<<<<<<< + * + * cdef enum: cudaSurfaceType2DLayered = 242 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaSurfaceType1DLayered = 0xF1 +}; + +/* "cuda/bindings/cyruntime.pxd":993 + * cdef enum: cudaSurfaceType1DLayered = 241 + * + * cdef enum: cudaSurfaceType2DLayered = 242 # <<<<<<<<<<<<<< + * + * cdef enum: cudaSurfaceTypeCubemapLayered = 252 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaSurfaceType2DLayered = 0xF2 +}; + +/* "cuda/bindings/cyruntime.pxd":995 + * cdef enum: cudaSurfaceType2DLayered = 242 + * + * cdef enum: cudaSurfaceTypeCubemapLayered = 252 # <<<<<<<<<<<<<< + * + * cdef enum: cudaTextureType1D = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaSurfaceTypeCubemapLayered = 0xFC +}; + +/* "cuda/bindings/cyruntime.pxd":997 + * cdef enum: cudaSurfaceTypeCubemapLayered = 252 + * + * cdef enum: cudaTextureType1D = 1 # <<<<<<<<<<<<<< + * + * cdef enum: cudaTextureType2D = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaTextureType1D = 1 +}; + +/* "cuda/bindings/cyruntime.pxd":999 + * cdef enum: cudaTextureType1D = 1 + * + * cdef enum: cudaTextureType2D = 2 # <<<<<<<<<<<<<< + * + * cdef enum: cudaTextureType3D = 3 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaTextureType2D = 2 +}; + +/* "cuda/bindings/cyruntime.pxd":1001 + * cdef enum: cudaTextureType2D = 2 + * + * cdef enum: cudaTextureType3D = 3 # <<<<<<<<<<<<<< + * + * cdef enum: cudaTextureTypeCubemap = 12 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaTextureType3D = 3 +}; + +/* "cuda/bindings/cyruntime.pxd":1003 + * cdef enum: cudaTextureType3D = 3 + * + * cdef enum: cudaTextureTypeCubemap = 12 # <<<<<<<<<<<<<< + * + * cdef enum: cudaTextureType1DLayered = 241 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaTextureTypeCubemap = 12 +}; + +/* "cuda/bindings/cyruntime.pxd":1005 + * cdef enum: cudaTextureTypeCubemap = 12 + * + * cdef enum: cudaTextureType1DLayered = 241 # <<<<<<<<<<<<<< + * + * cdef enum: cudaTextureType2DLayered = 242 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaTextureType1DLayered = 0xF1 +}; + +/* "cuda/bindings/cyruntime.pxd":1007 + * cdef enum: cudaTextureType1DLayered = 241 + * + * cdef enum: cudaTextureType2DLayered = 242 # <<<<<<<<<<<<<< + * + * cdef enum: cudaTextureTypeCubemapLayered = 252 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaTextureType2DLayered = 0xF2 +}; + +/* "cuda/bindings/cyruntime.pxd":1009 + * cdef enum: cudaTextureType2DLayered = 242 + * + * cdef enum: cudaTextureTypeCubemapLayered = 252 # <<<<<<<<<<<<<< + * + * cdef enum: CUDART_VERSION = 12090 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_cudaTextureTypeCubemapLayered = 0xFC +}; + +/* "cuda/bindings/cyruntime.pxd":1011 + * cdef enum: cudaTextureTypeCubemapLayered = 252 + * + * cdef enum: CUDART_VERSION = 12090 # <<<<<<<<<<<<<< + * + * cdef enum: __CUDART_API_VERSION = 12090 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_CUDART_VERSION = 0x2F3A +}; + +/* "cuda/bindings/cyruntime.pxd":1013 + * cdef enum: CUDART_VERSION = 12090 + * + * cdef enum: __CUDART_API_VERSION = 12090 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_EGL_MAX_PLANES = 3 +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime___CUDART_API_VERSION = 0x2F3A +}; + +/* "cuda/bindings/cyruntime.pxd":1015 + * cdef enum: __CUDART_API_VERSION = 12090 + * + * cdef enum: CUDA_EGL_MAX_PLANES = 3 # <<<<<<<<<<<<<< +*/ +enum { + __pyx_e_4cuda_8bindings_9cyruntime_CUDA_EGL_MAX_PLANES = 3 +}; + +/* "cuda/bindings/cyruntime_types.pxi":1553 + * cudaRoundMinInf = 3 + * + * ctypedef cudaLaunchAttributeID cudaStreamAttrID # <<<<<<<<<<<<<< + * + * ctypedef cudaLaunchAttributeID cudaKernelNodeAttrID +*/ +typedef enum cudaLaunchAttributeID __pyx_t_4cuda_8bindings_9cyruntime_cudaStreamAttrID; + +/* "cuda/bindings/cyruntime_types.pxi":1555 + * ctypedef cudaLaunchAttributeID cudaStreamAttrID + * + * ctypedef cudaLaunchAttributeID cudaKernelNodeAttrID # <<<<<<<<<<<<<< + * + * ctypedef cudaLaunchAttributeValue cudaStreamAttrValue +*/ +typedef enum cudaLaunchAttributeID __pyx_t_4cuda_8bindings_9cyruntime_cudaKernelNodeAttrID; + +/* "cuda/bindings/cyruntime_types.pxi":1557 + * ctypedef cudaLaunchAttributeID cudaKernelNodeAttrID + * + * ctypedef cudaLaunchAttributeValue cudaStreamAttrValue # <<<<<<<<<<<<<< + * + * ctypedef cudaLaunchAttributeValue cudaKernelNodeAttrValue +*/ +typedef union cudaLaunchAttributeValue __pyx_t_4cuda_8bindings_9cyruntime_cudaStreamAttrValue; + +/* "cuda/bindings/cyruntime_types.pxi":1559 + * ctypedef cudaLaunchAttributeValue cudaStreamAttrValue + * + * ctypedef cudaLaunchAttributeValue cudaKernelNodeAttrValue # <<<<<<<<<<<<<< +*/ +typedef union cudaLaunchAttributeValue __pyx_t_4cuda_8bindings_9cyruntime_cudaKernelNodeAttrValue; + +/* "cuda/bindings/cyruntime.pxd":17 + * cdef struct void: + * pass + * ctypedef void* EGLImageKHR # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef void *__pyx_t_4cuda_8bindings_9cyruntime_EGLImageKHR; + +/* "cuda/bindings/cyruntime.pxd":22 + * cdef struct void: + * pass + * ctypedef void* EGLStreamKHR # <<<<<<<<<<<<<< + * + * ctypedef unsigned int EGLint +*/ +typedef void *__pyx_t_4cuda_8bindings_9cyruntime_EGLStreamKHR; + +/* "cuda/bindings/cyruntime.pxd":29 + * cdef struct void: + * pass + * ctypedef void* EGLSyncKHR # <<<<<<<<<<<<<< + * + * ctypedef uint32_t VdpDevice +*/ +typedef void *__pyx_t_4cuda_8bindings_9cyruntime_EGLSyncKHR; + +/* "cuda/bindings/cyruntime.pxd":43 + * cudaEglFrameTypePitch = 1 + * + * ctypedef cudaEglFrameType_enum cudaEglFrameType # <<<<<<<<<<<<<< + * + * cdef enum cudaEglResourceLocationFlags_enum: +*/ +typedef enum __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrameType_enum __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrameType; + +/* "cuda/bindings/cyruntime.pxd":49 + * cudaEglResourceLocationVidmem = 1 + * + * ctypedef cudaEglResourceLocationFlags_enum cudaEglResourceLocationFlags # <<<<<<<<<<<<<< + * + * cdef enum cudaEglColorFormat_enum: +*/ +typedef enum __pyx_t_4cuda_8bindings_9cyruntime_cudaEglResourceLocationFlags_enum __pyx_t_4cuda_8bindings_9cyruntime_cudaEglResourceLocationFlags; + +/* "cuda/bindings/cyruntime.pxd":166 + * cudaEglColorFormatUYVY2020 = 116 + * + * ctypedef cudaEglColorFormat_enum cudaEglColorFormat # <<<<<<<<<<<<<< + * + * cdef struct cudaEglPlaneDesc_st: +*/ +typedef enum __pyx_t_4cuda_8bindings_9cyruntime_cudaEglColorFormat_enum __pyx_t_4cuda_8bindings_9cyruntime_cudaEglColorFormat; + +/* "cuda/bindings/cyruntime.pxd":168 + * ctypedef cudaEglColorFormat_enum cudaEglColorFormat + * + * cdef struct cudaEglPlaneDesc_st: # <<<<<<<<<<<<<< + * unsigned int width + * unsigned int height +*/ +struct __pyx_t_4cuda_8bindings_9cyruntime_cudaEglPlaneDesc_st { + unsigned int width; + unsigned int height; + unsigned int depth; + unsigned int pitch; + unsigned int numChannels; + struct cudaChannelFormatDesc channelDesc; + unsigned int reserved[4]; +}; + +/* "cuda/bindings/cyruntime.pxd":177 + * unsigned int reserved[4] + * + * ctypedef cudaEglPlaneDesc_st cudaEglPlaneDesc # <<<<<<<<<<<<<< + * + * cdef union anon_union11: +*/ +typedef struct __pyx_t_4cuda_8bindings_9cyruntime_cudaEglPlaneDesc_st __pyx_t_4cuda_8bindings_9cyruntime_cudaEglPlaneDesc; + +/* "cuda/bindings/cyruntime.pxd":179 + * ctypedef cudaEglPlaneDesc_st cudaEglPlaneDesc + * + * cdef union anon_union11: # <<<<<<<<<<<<<< + * cudaArray_t pArray[3] + * cudaPitchedPtr pPitch[3] +*/ +union __pyx_t_4cuda_8bindings_9cyruntime_anon_union11 { + cudaArray_t pArray[3]; + struct cudaPitchedPtr pPitch[3]; +}; + +/* "cuda/bindings/cyruntime.pxd":183 + * cudaPitchedPtr pPitch[3] + * + * cdef struct cudaEglFrame_st: # <<<<<<<<<<<<<< + * anon_union11 frame + * cudaEglPlaneDesc planeDesc[3] +*/ +struct __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame_st { + union __pyx_t_4cuda_8bindings_9cyruntime_anon_union11 frame; + __pyx_t_4cuda_8bindings_9cyruntime_cudaEglPlaneDesc planeDesc[3]; + unsigned int planeCount; + __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrameType frameType; + __pyx_t_4cuda_8bindings_9cyruntime_cudaEglColorFormat eglColorFormat; +}; + +/* "cuda/bindings/cyruntime.pxd":190 + * cudaEglColorFormat eglColorFormat + * + * ctypedef cudaEglFrame_st cudaEglFrame # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef struct __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame_st __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame; + +/* "cuda/bindings/cyruntime.pxd":195 + * cdef struct CUeglStreamConnection_st: + * pass + * ctypedef CUeglStreamConnection_st* cudaEglStreamConnection # <<<<<<<<<<<<<< + * + * cdef enum cudaGLDeviceList: +*/ +typedef struct CUeglStreamConnection_st *__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection; +union __pyx_t_4cuda_8bindings_8cydriver_anon_union15; +struct __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_st; + +/* "cuda/bindings/cydriver.pxd":2429 + * ctypedef CUoutput_mode_enum CUoutput_mode + * + * cdef enum CUeglFrameType_enum: # <<<<<<<<<<<<<< + * CU_EGL_FRAME_TYPE_ARRAY = 0 + * CU_EGL_FRAME_TYPE_PITCH = 1 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUeglFrameType_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_FRAME_TYPE_ARRAY = 0, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_FRAME_TYPE_PITCH = 1 +}; + +/* "cuda/bindings/cydriver.pxd":2435 + * ctypedef CUeglFrameType_enum CUeglFrameType + * + * cdef enum CUeglResourceLocationFlags_enum: # <<<<<<<<<<<<<< + * CU_EGL_RESOURCE_LOCATION_SYSMEM = 0 + * CU_EGL_RESOURCE_LOCATION_VIDMEM = 1 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUeglResourceLocationFlags_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_RESOURCE_LOCATION_SYSMEM = 0, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_RESOURCE_LOCATION_VIDMEM = 1 +}; + +/* "cuda/bindings/cydriver.pxd":2441 + * ctypedef CUeglResourceLocationFlags_enum CUeglResourceLocationFlags + * + * cdef enum CUeglColorFormat_enum: # <<<<<<<<<<<<<< + * CU_EGL_COLOR_FORMAT_YUV420_PLANAR = 0 + * CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR = 1 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUeglColorFormat_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR = 0, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR = 1, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_PLANAR = 2, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR = 3, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_RGB = 4, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BGR = 5, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_ARGB = 6, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_RGBA = 7, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_L = 8, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_R = 9, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_PLANAR = 10, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR = 11, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUYV_422 = 12, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_422 = 13, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_ABGR = 14, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BGRA = 15, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_A = 16, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_RG = 17, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_AYUV = 18, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR = 19, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR = 20, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR = 21, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR = 22, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR = 23, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR = 24, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR = 25, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_VYUY_ER = 26, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_ER = 27, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUYV_ER = 28, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVYU_ER = 29, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV_ER = 30, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUVA_ER = 31, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_AYUV_ER = 32, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER = 33, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER = 34, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER = 35, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER = 36, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER = 37, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER = 38, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER = 39, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER = 40, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER = 41, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER = 42, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER = 43, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER = 44, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_RGGB = 45, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_BGGR = 46, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_GRBG = 47, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_GBRG = 48, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_RGGB = 49, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_BGGR = 50, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_GRBG = 51, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_GBRG = 52, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_RGGB = 53, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_BGGR = 54, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_GRBG = 55, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_GBRG = 56, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_RGGB = 57, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_BGGR = 58, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_GRBG = 59, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_GBRG = 60, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_RGGB = 61, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_BGGR = 62, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_GRBG = 63, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_GBRG = 64, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_PLANAR = 65, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_PLANAR = 66, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR = 67, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB = 68, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR = 69, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG = 70, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG = 71, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_BCCR = 72, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_RCCB = 73, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_CRBC = 74, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_CBRC = 75, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_CCCC = 76, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_BCCR = 77, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_RCCB = 78, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CRBC = 79, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CBRC = 80, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CCCC = 81, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y = 82, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 = 83, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 = 84, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 = 85, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 = 86, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 = 87, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 = 88, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 = 89, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709 = 90, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 = 91, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 = 92, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 = 93, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR = 94, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709 = 95, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y_ER = 96, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y_709_ER = 97, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10_ER = 98, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10_709_ER = 99, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12_ER = 0x64, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12_709_ER = 0x65, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUVA = 0x66, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV = 0x67, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVYU = 0x68, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_VYUY = 0x69, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER = 0x6A, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER = 0x6B, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER = 0x6C, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER = 0x6D, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER = 0x6E, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER = 0x6F, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER = 0x70, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER = 0x71, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_709 = 0x72, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_709_ER = 0x73, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_2020 = 0x74, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_MAX = 0x75 +}; + +/* "cuda/bindings/cydriver.pxd":2588 + * ctypedef CUeglStreamConnection_st* CUeglStreamConnection + * + * cdef enum CUGLDeviceList_enum: # <<<<<<<<<<<<<< + * CU_GL_DEVICE_LIST_ALL = 1 + * CU_GL_DEVICE_LIST_CURRENT_FRAME = 2 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_DEVICE_LIST_ALL = 1, + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_DEVICE_LIST_CURRENT_FRAME = 2, + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_DEVICE_LIST_NEXT_FRAME = 3 +}; + +/* "cuda/bindings/cydriver.pxd":2595 + * ctypedef CUGLDeviceList_enum CUGLDeviceList + * + * cdef enum CUGLmap_flags_enum: # <<<<<<<<<<<<<< + * CU_GL_MAP_RESOURCE_FLAGS_NONE = 0 + * CU_GL_MAP_RESOURCE_FLAGS_READ_ONLY = 1 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUGLmap_flags_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_MAP_RESOURCE_FLAGS_NONE = 0, + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_MAP_RESOURCE_FLAGS_READ_ONLY = 1, + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_MAP_RESOURCE_FLAGS_WRITE_DISCARD = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3577 + * cdef CUresult cuGraphicsVDPAURegisterOutputSurface(CUgraphicsResource* pCudaResource, VdpOutputSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + * + * cdef enum: CUDA_VERSION = 12090 # <<<<<<<<<<<<<< + * + * cdef enum: CU_IPC_HANDLE_SIZE = 64 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_VERSION = 0x2F3A +}; + +/* "cuda/bindings/cydriver.pxd":3579 + * cdef enum: CUDA_VERSION = 12090 + * + * cdef enum: CU_IPC_HANDLE_SIZE = 64 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_LEGACY = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_IPC_HANDLE_SIZE = 64 +}; + +/* "cuda/bindings/cydriver.pxd":3581 + * cdef enum: CU_IPC_HANDLE_SIZE = 64 + * + * cdef enum: CU_STREAM_LEGACY = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_PER_THREAD = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_LEGACY = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3583 + * cdef enum: CU_STREAM_LEGACY = 1 + * + * cdef enum: CU_STREAM_PER_THREAD = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_COMPUTE_ACCELERATED_TARGET_BASE = 65536 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_PER_THREAD = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3585 + * cdef enum: CU_STREAM_PER_THREAD = 2 + * + * cdef enum: CU_COMPUTE_ACCELERATED_TARGET_BASE = 65536 # <<<<<<<<<<<<<< + * + * cdef enum: CU_COMPUTE_FAMILY_TARGET_BASE = 131072 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_COMPUTE_ACCELERATED_TARGET_BASE = 0x10000 +}; + +/* "cuda/bindings/cydriver.pxd":3587 + * cdef enum: CU_COMPUTE_ACCELERATED_TARGET_BASE = 65536 + * + * cdef enum: CU_COMPUTE_FAMILY_TARGET_BASE = 131072 # <<<<<<<<<<<<<< + * + * cdef enum: CU_GRAPH_COND_ASSIGN_DEFAULT = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_COMPUTE_FAMILY_TARGET_BASE = 0x20000 +}; + +/* "cuda/bindings/cydriver.pxd":3589 + * cdef enum: CU_COMPUTE_FAMILY_TARGET_BASE = 131072 + * + * cdef enum: CU_GRAPH_COND_ASSIGN_DEFAULT = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_DEFAULT = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GRAPH_COND_ASSIGN_DEFAULT = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3591 + * cdef enum: CU_GRAPH_COND_ASSIGN_DEFAULT = 1 + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_DEFAULT = 0 # <<<<<<<<<<<<<< + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GRAPH_KERNEL_NODE_PORT_DEFAULT = 0 +}; + +/* "cuda/bindings/cydriver.pxd":3593 + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_DEFAULT = 0 + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3595 + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC = 1 + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3597 + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER = 2 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_COOPERATIVE = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3599 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_COOPERATIVE = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_DIMENSION = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_COOPERATIVE = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3601 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_COOPERATIVE = 2 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_DIMENSION = 4 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_DIMENSION = 4 +}; + +/* "cuda/bindings/cydriver.pxd":3603 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_DIMENSION = 4 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PRIORITY = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 +}; + +/* "cuda/bindings/cydriver.pxd":3605 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PRIORITY = 8 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_PRIORITY = 8 +}; + +/* "cuda/bindings/cydriver.pxd":3607 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PRIORITY = 8 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 +}; + +/* "cuda/bindings/cydriver.pxd":3609 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 +}; + +/* "cuda/bindings/cydriver.pxd":3611 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11 +}; + +/* "cuda/bindings/cydriver.pxd":3613 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 14 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13 +}; + +/* "cuda/bindings/cydriver.pxd":3615 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 14 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 14 +}; + +/* "cuda/bindings/cydriver.pxd":3617 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 14 + * + * cdef enum: CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3619 + * cdef enum: CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 + * + * cdef enum: CU_STREAM_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_PRIORITY = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 +}; + +/* "cuda/bindings/cydriver.pxd":3621 + * cdef enum: CU_STREAM_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 + * + * cdef enum: CU_STREAM_ATTRIBUTE_PRIORITY = 8 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_PRIORITY = 8 +}; + +/* "cuda/bindings/cydriver.pxd":3623 + * cdef enum: CU_STREAM_ATTRIBUTE_PRIORITY = 8 + * + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 +}; + +/* "cuda/bindings/cydriver.pxd":3625 + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 + * + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTALLOC_PORTABLE = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 +}; + +/* "cuda/bindings/cydriver.pxd":3627 + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 + * + * cdef enum: CU_MEMHOSTALLOC_PORTABLE = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTALLOC_DEVICEMAP = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTALLOC_PORTABLE = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3629 + * cdef enum: CU_MEMHOSTALLOC_PORTABLE = 1 + * + * cdef enum: CU_MEMHOSTALLOC_DEVICEMAP = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTALLOC_WRITECOMBINED = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTALLOC_DEVICEMAP = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3631 + * cdef enum: CU_MEMHOSTALLOC_DEVICEMAP = 2 + * + * cdef enum: CU_MEMHOSTALLOC_WRITECOMBINED = 4 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTREGISTER_PORTABLE = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTALLOC_WRITECOMBINED = 4 +}; + +/* "cuda/bindings/cydriver.pxd":3633 + * cdef enum: CU_MEMHOSTALLOC_WRITECOMBINED = 4 + * + * cdef enum: CU_MEMHOSTREGISTER_PORTABLE = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTREGISTER_DEVICEMAP = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTREGISTER_PORTABLE = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3635 + * cdef enum: CU_MEMHOSTREGISTER_PORTABLE = 1 + * + * cdef enum: CU_MEMHOSTREGISTER_DEVICEMAP = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTREGISTER_IOMEMORY = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTREGISTER_DEVICEMAP = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3637 + * cdef enum: CU_MEMHOSTREGISTER_DEVICEMAP = 2 + * + * cdef enum: CU_MEMHOSTREGISTER_IOMEMORY = 4 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTREGISTER_READ_ONLY = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTREGISTER_IOMEMORY = 4 +}; + +/* "cuda/bindings/cydriver.pxd":3639 + * cdef enum: CU_MEMHOSTREGISTER_IOMEMORY = 4 + * + * cdef enum: CU_MEMHOSTREGISTER_READ_ONLY = 8 # <<<<<<<<<<<<<< + * + * cdef enum: CU_ARRAY_SPARSE_PROPERTIES_SINGLE_MIPTAIL = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTREGISTER_READ_ONLY = 8 +}; + +/* "cuda/bindings/cydriver.pxd":3641 + * cdef enum: CU_MEMHOSTREGISTER_READ_ONLY = 8 + * + * cdef enum: CU_ARRAY_SPARSE_PROPERTIES_SINGLE_MIPTAIL = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TENSOR_MAP_NUM_QWORDS = 16 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_ARRAY_SPARSE_PROPERTIES_SINGLE_MIPTAIL = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3643 + * cdef enum: CU_ARRAY_SPARSE_PROPERTIES_SINGLE_MIPTAIL = 1 + * + * cdef enum: CU_TENSOR_MAP_NUM_QWORDS = 16 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_EXTERNAL_MEMORY_DEDICATED = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TENSOR_MAP_NUM_QWORDS = 16 +}; + +/* "cuda/bindings/cydriver.pxd":3645 + * cdef enum: CU_TENSOR_MAP_NUM_QWORDS = 16 + * + * cdef enum: CUDA_EXTERNAL_MEMORY_DEDICATED = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_SIGNAL_SKIP_NVSCIBUF_MEMSYNC = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_EXTERNAL_MEMORY_DEDICATED = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3647 + * cdef enum: CUDA_EXTERNAL_MEMORY_DEDICATED = 1 + * + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_SIGNAL_SKIP_NVSCIBUF_MEMSYNC = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_WAIT_SKIP_NVSCIBUF_MEMSYNC = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_EXTERNAL_SEMAPHORE_SIGNAL_SKIP_NVSCIBUF_MEMSYNC = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3649 + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_SIGNAL_SKIP_NVSCIBUF_MEMSYNC = 1 + * + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_WAIT_SKIP_NVSCIBUF_MEMSYNC = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_NVSCISYNC_ATTR_SIGNAL = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_EXTERNAL_SEMAPHORE_WAIT_SKIP_NVSCIBUF_MEMSYNC = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3651 + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_WAIT_SKIP_NVSCIBUF_MEMSYNC = 2 + * + * cdef enum: CUDA_NVSCISYNC_ATTR_SIGNAL = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_NVSCISYNC_ATTR_WAIT = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_NVSCISYNC_ATTR_SIGNAL = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3653 + * cdef enum: CUDA_NVSCISYNC_ATTR_SIGNAL = 1 + * + * cdef enum: CUDA_NVSCISYNC_ATTR_WAIT = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEM_CREATE_USAGE_TILE_POOL = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_NVSCISYNC_ATTR_WAIT = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3655 + * cdef enum: CUDA_NVSCISYNC_ATTR_WAIT = 2 + * + * cdef enum: CU_MEM_CREATE_USAGE_TILE_POOL = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEM_CREATE_USAGE_HW_DECOMPRESS = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEM_CREATE_USAGE_TILE_POOL = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3657 + * cdef enum: CU_MEM_CREATE_USAGE_TILE_POOL = 1 + * + * cdef enum: CU_MEM_CREATE_USAGE_HW_DECOMPRESS = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEM_CREATE_USAGE_HW_DECOMPRESS = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3659 + * cdef enum: CU_MEM_CREATE_USAGE_HW_DECOMPRESS = 2 + * + * cdef enum: CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3661 + * cdef enum: CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS = 2 + * + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3663 + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC = 1 + * + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_LAYERED = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3665 + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC = 2 + * + * cdef enum: CUDA_ARRAY3D_LAYERED = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_2DARRAY = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_LAYERED = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3667 + * cdef enum: CUDA_ARRAY3D_LAYERED = 1 + * + * cdef enum: CUDA_ARRAY3D_2DARRAY = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_SURFACE_LDST = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_2DARRAY = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3669 + * cdef enum: CUDA_ARRAY3D_2DARRAY = 1 + * + * cdef enum: CUDA_ARRAY3D_SURFACE_LDST = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_CUBEMAP = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_SURFACE_LDST = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3671 + * cdef enum: CUDA_ARRAY3D_SURFACE_LDST = 2 + * + * cdef enum: CUDA_ARRAY3D_CUBEMAP = 4 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_TEXTURE_GATHER = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_CUBEMAP = 4 +}; + +/* "cuda/bindings/cydriver.pxd":3673 + * cdef enum: CUDA_ARRAY3D_CUBEMAP = 4 + * + * cdef enum: CUDA_ARRAY3D_TEXTURE_GATHER = 8 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_DEPTH_TEXTURE = 16 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_TEXTURE_GATHER = 8 +}; + +/* "cuda/bindings/cydriver.pxd":3675 + * cdef enum: CUDA_ARRAY3D_TEXTURE_GATHER = 8 + * + * cdef enum: CUDA_ARRAY3D_DEPTH_TEXTURE = 16 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_COLOR_ATTACHMENT = 32 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_DEPTH_TEXTURE = 16 +}; + +/* "cuda/bindings/cydriver.pxd":3677 + * cdef enum: CUDA_ARRAY3D_DEPTH_TEXTURE = 16 + * + * cdef enum: CUDA_ARRAY3D_COLOR_ATTACHMENT = 32 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_SPARSE = 64 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_COLOR_ATTACHMENT = 32 +}; + +/* "cuda/bindings/cydriver.pxd":3679 + * cdef enum: CUDA_ARRAY3D_COLOR_ATTACHMENT = 32 + * + * cdef enum: CUDA_ARRAY3D_SPARSE = 64 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_DEFERRED_MAPPING = 128 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_SPARSE = 64 +}; + +/* "cuda/bindings/cydriver.pxd":3681 + * cdef enum: CUDA_ARRAY3D_SPARSE = 64 + * + * cdef enum: CUDA_ARRAY3D_DEFERRED_MAPPING = 128 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_VIDEO_ENCODE_DECODE = 256 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_DEFERRED_MAPPING = 0x80 +}; + +/* "cuda/bindings/cydriver.pxd":3683 + * cdef enum: CUDA_ARRAY3D_DEFERRED_MAPPING = 128 + * + * cdef enum: CUDA_ARRAY3D_VIDEO_ENCODE_DECODE = 256 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSA_OVERRIDE_FORMAT = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_VIDEO_ENCODE_DECODE = 0x100 +}; + +/* "cuda/bindings/cydriver.pxd":3685 + * cdef enum: CUDA_ARRAY3D_VIDEO_ENCODE_DECODE = 256 + * + * cdef enum: CU_TRSA_OVERRIDE_FORMAT = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_READ_AS_INTEGER = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSA_OVERRIDE_FORMAT = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3687 + * cdef enum: CU_TRSA_OVERRIDE_FORMAT = 1 + * + * cdef enum: CU_TRSF_READ_AS_INTEGER = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_NORMALIZED_COORDINATES = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_READ_AS_INTEGER = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3689 + * cdef enum: CU_TRSF_READ_AS_INTEGER = 1 + * + * cdef enum: CU_TRSF_NORMALIZED_COORDINATES = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_SRGB = 16 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_NORMALIZED_COORDINATES = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3691 + * cdef enum: CU_TRSF_NORMALIZED_COORDINATES = 2 + * + * cdef enum: CU_TRSF_SRGB = 16 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_DISABLE_TRILINEAR_OPTIMIZATION = 32 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_SRGB = 16 +}; + +/* "cuda/bindings/cydriver.pxd":3693 + * cdef enum: CU_TRSF_SRGB = 16 + * + * cdef enum: CU_TRSF_DISABLE_TRILINEAR_OPTIMIZATION = 32 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_SEAMLESS_CUBEMAP = 64 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_DISABLE_TRILINEAR_OPTIMIZATION = 32 +}; + +/* "cuda/bindings/cydriver.pxd":3695 + * cdef enum: CU_TRSF_DISABLE_TRILINEAR_OPTIMIZATION = 32 + * + * cdef enum: CU_TRSF_SEAMLESS_CUBEMAP = 64 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_KERNEL_REQUIRED_BLOCK_DIM = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_SEAMLESS_CUBEMAP = 64 +}; + +/* "cuda/bindings/cydriver.pxd":3697 + * cdef enum: CU_TRSF_SEAMLESS_CUBEMAP = 64 + * + * cdef enum: CU_LAUNCH_KERNEL_REQUIRED_BLOCK_DIM = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_END_AS_INT = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_KERNEL_REQUIRED_BLOCK_DIM = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3699 + * cdef enum: CU_LAUNCH_KERNEL_REQUIRED_BLOCK_DIM = 1 + * + * cdef enum: CU_LAUNCH_PARAM_END_AS_INT = 0 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_END = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_END_AS_INT = 0 +}; + +/* "cuda/bindings/cydriver.pxd":3701 + * cdef enum: CU_LAUNCH_PARAM_END_AS_INT = 0 + * + * cdef enum: CU_LAUNCH_PARAM_END = 0 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER_AS_INT = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_END = 0 +}; + +/* "cuda/bindings/cydriver.pxd":3703 + * cdef enum: CU_LAUNCH_PARAM_END = 0 + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER_AS_INT = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_BUFFER_POINTER_AS_INT = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3705 + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER_AS_INT = 1 + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE_AS_INT = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_BUFFER_POINTER = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3707 + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER = 1 + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE_AS_INT = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_BUFFER_SIZE_AS_INT = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3709 + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE_AS_INT = 2 + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_PARAM_TR_DEFAULT = -1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_BUFFER_SIZE = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3711 + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE = 2 + * + * cdef enum: CU_PARAM_TR_DEFAULT = -1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_DEVICE_CPU = -1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_PARAM_TR_DEFAULT = -1L +}; + +/* "cuda/bindings/cydriver.pxd":3713 + * cdef enum: CU_PARAM_TR_DEFAULT = -1 + * + * cdef enum: CU_DEVICE_CPU = -1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_DEVICE_INVALID = -2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_DEVICE_CPU = -1L +}; + +/* "cuda/bindings/cydriver.pxd":3715 + * cdef enum: CU_DEVICE_CPU = -1 + * + * cdef enum: CU_DEVICE_INVALID = -2 # <<<<<<<<<<<<<< + * + * cdef enum: RESOURCE_ABI_VERSION = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_DEVICE_INVALID = -2L +}; + +/* "cuda/bindings/cydriver.pxd":3717 + * cdef enum: CU_DEVICE_INVALID = -2 + * + * cdef enum: RESOURCE_ABI_VERSION = 1 # <<<<<<<<<<<<<< + * + * cdef enum: RESOURCE_ABI_EXTERNAL_BYTES = 48 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_RESOURCE_ABI_VERSION = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3719 + * cdef enum: RESOURCE_ABI_VERSION = 1 + * + * cdef enum: RESOURCE_ABI_EXTERNAL_BYTES = 48 # <<<<<<<<<<<<<< + * + * cdef enum: MAX_PLANES = 3 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_RESOURCE_ABI_EXTERNAL_BYTES = 48 +}; + +/* "cuda/bindings/cydriver.pxd":3721 + * cdef enum: RESOURCE_ABI_EXTERNAL_BYTES = 48 + * + * cdef enum: MAX_PLANES = 3 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_EGL_INFINITE_TIMEOUT = 4294967295 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_MAX_PLANES = 3 +}; + +/* "cuda/bindings/cydriver.pxd":3723 + * cdef enum: MAX_PLANES = 3 + * + * cdef enum: CUDA_EGL_INFINITE_TIMEOUT = 4294967295 # <<<<<<<<<<<<<< +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_EGL_INFINITE_TIMEOUT = 0xFFFFFFFF +}; + +/* "cuda/bindings/cydriver.pxd":2433 + * CU_EGL_FRAME_TYPE_PITCH = 1 + * + * ctypedef CUeglFrameType_enum CUeglFrameType # <<<<<<<<<<<<<< + * + * cdef enum CUeglResourceLocationFlags_enum: +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUeglFrameType_enum __pyx_t_4cuda_8bindings_8cydriver_CUeglFrameType; + +/* "cuda/bindings/cydriver.pxd":2439 + * CU_EGL_RESOURCE_LOCATION_VIDMEM = 1 + * + * ctypedef CUeglResourceLocationFlags_enum CUeglResourceLocationFlags # <<<<<<<<<<<<<< + * + * cdef enum CUeglColorFormat_enum: +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUeglResourceLocationFlags_enum __pyx_t_4cuda_8bindings_8cydriver_CUeglResourceLocationFlags; + +/* "cuda/bindings/cydriver.pxd":2561 + * CU_EGL_COLOR_FORMAT_MAX = 117 + * + * ctypedef CUeglColorFormat_enum CUeglColorFormat # <<<<<<<<<<<<<< + * + * cdef union anon_union15: +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUeglColorFormat_enum __pyx_t_4cuda_8bindings_8cydriver_CUeglColorFormat; + +/* "cuda/bindings/cydriver.pxd":2563 + * ctypedef CUeglColorFormat_enum CUeglColorFormat + * + * cdef union anon_union15: # <<<<<<<<<<<<<< + * CUarray pArray[3] + * void* pPitch[3] +*/ +union __pyx_t_4cuda_8bindings_8cydriver_anon_union15 { + CUarray pArray[3]; + void *pPitch[3]; +}; + +/* "cuda/bindings/cydriver.pxd":2567 + * void* pPitch[3] + * + * cdef struct CUeglFrame_st: # <<<<<<<<<<<<<< + * anon_union15 frame + * unsigned int width +*/ +struct __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_st { + union __pyx_t_4cuda_8bindings_8cydriver_anon_union15 frame; + unsigned int width; + unsigned int height; + unsigned int depth; + unsigned int pitch; + unsigned int planeCount; + unsigned int numChannels; + __pyx_t_4cuda_8bindings_8cydriver_CUeglFrameType frameType; + __pyx_t_4cuda_8bindings_8cydriver_CUeglColorFormat eglColorFormat; + CUarray_format cuFormat; +}; + +/* "cuda/bindings/cydriver.pxd":2579 + * CUarray_format cuFormat + * + * ctypedef CUeglFrame_st CUeglFrame_v1 # <<<<<<<<<<<<<< + * + * ctypedef CUeglFrame_v1 CUeglFrame +*/ +typedef struct __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_st __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_v1; + +/* "cuda/bindings/cydriver.pxd":2581 + * ctypedef CUeglFrame_st CUeglFrame_v1 + * + * ctypedef CUeglFrame_v1 CUeglFrame # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_v1 __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame; + +/* "cuda/bindings/cydriver.pxd":2586 + * cdef struct CUeglStreamConnection_st: + * pass + * ctypedef CUeglStreamConnection_st* CUeglStreamConnection # <<<<<<<<<<<<<< + * + * cdef enum CUGLDeviceList_enum: +*/ +typedef struct CUeglStreamConnection_st *__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection; + +/* "cuda/bindings/cydriver.pxd":2593 + * CU_GL_DEVICE_LIST_NEXT_FRAME = 3 + * + * ctypedef CUGLDeviceList_enum CUGLDeviceList # <<<<<<<<<<<<<< + * + * cdef enum CUGLmap_flags_enum: +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList_enum __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList; + +/* "cuda/bindings/cydriver.pxd":2600 + * CU_GL_MAP_RESOURCE_FLAGS_WRITE_DISCARD = 2 + * + * ctypedef CUGLmap_flags_enum CUGLmap_flags # <<<<<<<<<<<<<< + * + * ctypedef unsigned int GLenum +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUGLmap_flags_enum __pyx_t_4cuda_8bindings_8cydriver_CUGLmap_flags; + +/* "cuda/bindings/cydriver.pxd":2609 + * cdef struct void: + * pass + * ctypedef void* EGLImageKHR # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef void *__pyx_t_4cuda_8bindings_8cydriver_EGLImageKHR; + +/* "cuda/bindings/cydriver.pxd":2614 + * cdef struct void: + * pass + * ctypedef void* EGLStreamKHR # <<<<<<<<<<<<<< + * + * ctypedef unsigned int EGLint +*/ +typedef void *__pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR; + +/* "cuda/bindings/cydriver.pxd":2621 + * cdef struct void: + * pass + * ctypedef void* EGLSyncKHR # <<<<<<<<<<<<<< + * + * ctypedef uint32_t VdpDevice +*/ +typedef void *__pyx_t_4cuda_8bindings_8cydriver_EGLSyncKHR; + +/* "cuda/bindings/cyruntime_types.pxi":1553 + * cudaRoundMinInf = 3 + * + * ctypedef cudaLaunchAttributeID cudaStreamAttrID # <<<<<<<<<<<<<< + * + * ctypedef cudaLaunchAttributeID cudaKernelNodeAttrID +*/ +typedef enum cudaLaunchAttributeID __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrID; + +/* "cuda/bindings/cyruntime_types.pxi":1555 + * ctypedef cudaLaunchAttributeID cudaStreamAttrID + * + * ctypedef cudaLaunchAttributeID cudaKernelNodeAttrID # <<<<<<<<<<<<<< + * + * ctypedef cudaLaunchAttributeValue cudaStreamAttrValue +*/ +typedef enum cudaLaunchAttributeID __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrID; + +/* "cuda/bindings/cyruntime_types.pxi":1557 + * ctypedef cudaLaunchAttributeID cudaKernelNodeAttrID + * + * ctypedef cudaLaunchAttributeValue cudaStreamAttrValue # <<<<<<<<<<<<<< + * + * ctypedef cudaLaunchAttributeValue cudaKernelNodeAttrValue +*/ +typedef union cudaLaunchAttributeValue __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrValue; + +/* "cuda/bindings/cyruntime_types.pxi":1559 + * ctypedef cudaLaunchAttributeValue cudaStreamAttrValue + * + * ctypedef cudaLaunchAttributeValue cudaKernelNodeAttrValue # <<<<<<<<<<<<<< +*/ +typedef union cudaLaunchAttributeValue __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrValue; + +/* "cuda/bindings/cyruntime_types.pxi":1553 + * cudaRoundMinInf = 3 + * + * ctypedef cudaLaunchAttributeID cudaStreamAttrID # <<<<<<<<<<<<<< + * + * ctypedef cudaLaunchAttributeID cudaKernelNodeAttrID +*/ +typedef enum cudaLaunchAttributeID __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaStreamAttrID; + +/* "cuda/bindings/cyruntime_types.pxi":1555 + * ctypedef cudaLaunchAttributeID cudaStreamAttrID + * + * ctypedef cudaLaunchAttributeID cudaKernelNodeAttrID # <<<<<<<<<<<<<< + * + * ctypedef cudaLaunchAttributeValue cudaStreamAttrValue +*/ +typedef enum cudaLaunchAttributeID __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaKernelNodeAttrID; + +/* "cuda/bindings/cyruntime_types.pxi":1557 + * ctypedef cudaLaunchAttributeID cudaKernelNodeAttrID + * + * ctypedef cudaLaunchAttributeValue cudaStreamAttrValue # <<<<<<<<<<<<<< + * + * ctypedef cudaLaunchAttributeValue cudaKernelNodeAttrValue +*/ +typedef union cudaLaunchAttributeValue __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaStreamAttrValue; + +/* "cuda/bindings/cyruntime_types.pxi":1559 + * ctypedef cudaLaunchAttributeValue cudaStreamAttrValue + * + * ctypedef cudaLaunchAttributeValue cudaKernelNodeAttrValue # <<<<<<<<<<<<<< +*/ +typedef union cudaLaunchAttributeValue __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaKernelNodeAttrValue; +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyErrExceptionMatches.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* PyObjectGetAttrStrNoError.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* PyDictVersioning.proto */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __pyx_dict_cached_value;\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) do {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +#define __Pyx_GetModuleGlobalNameUncached(var, name) do {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#if !CYTHON_VECTORCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs); +#endif +#define __Pyx_BUILD_ASSERT_EXPR(cond)\ + (sizeof(char [1 - 2*!(cond)]) - 1) +#ifndef Py_MEMBER_SIZE +#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#endif +#if !CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03080000 + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets() + #define __Pyx_PyFrame_GetLocalsplus(frame) ((frame)->f_localsplus) +#else + static size_t __pyx_pyframe_localsplus_offset = 0; + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets()\ + ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ + (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) + #define __Pyx_PyFrame_GetLocalsplus(frame)\ + (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) +#endif +#endif +#endif + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* PyObjectVectorCallKwBuilder.proto */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* ErrOccurredWithGIL.proto */ +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void); + +/* IncludeStringH.proto */ +#include + +/* Import.proto */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); + +/* ImportDottedModule.proto */ +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple); +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple); + +/* CLineInTraceback.proto */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* GCCDiagnostics.proto */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_unsigned_int(unsigned int value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE unsigned int __Pyx_PyLong_As_unsigned_int(PyObject *); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +/* GetRuntimeVersion.proto */ +static unsigned long __Pyx_get_runtime_version(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* FunctionExport.proto */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig); + +/* FunctionImport.proto */ +static int __Pyx_ImportFunction_3_1_5(PyObject *module, const char *funcname, void (**f)(void), const char *sig); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + + +/* Module declarations from "libc.stdint" */ + +/* Module declarations from "cuda.bindings.cyruntime" */ + +/* Module declarations from "cuda.bindings.cydriver" */ + +/* Module declarations from "cuda.bindings._bindings.cydriver" */ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsEGLRegisterImage)(CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_EGLImageKHR, unsigned int); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerConnect)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerConnectWithFlags)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR, unsigned int); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerDisconnect)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerAcquireFrame)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, CUgraphicsResource *, CUstream *, unsigned int); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerReleaseFrame)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, CUgraphicsResource, CUstream *); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerConnect)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR, __pyx_t_4cuda_8bindings_8cydriver_EGLint, __pyx_t_4cuda_8bindings_8cydriver_EGLint); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerDisconnect)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerPresentFrame)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame, CUstream *); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerReturnFrame)(__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame *, CUstream *); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsResourceGetMappedEglFrame)(__pyx_t_4cuda_8bindings_8cydriver_CUeglFrame *, CUgraphicsResource, unsigned int, unsigned int); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventCreateFromEGLSync)(CUevent *, __pyx_t_4cuda_8bindings_8cydriver_EGLSyncKHR, unsigned int); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsGLRegisterBuffer)(CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_GLuint, unsigned int); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsGLRegisterImage)(CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_GLuint, __pyx_t_4cuda_8bindings_8cydriver_GLenum, unsigned int); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGLGetDevices_v2)(unsigned int *, CUdevice *, unsigned int, __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuVDPAUGetDevice)(CUdevice *, __pyx_t_4cuda_8bindings_8cydriver_VdpDevice, __pyx_t_4cuda_8bindings_8cydriver_VdpGetProcAddress *); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsVDPAURegisterVideoSurface)(CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_VdpVideoSurface, unsigned int); /*proto*/ +static CUresult (*__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsVDPAURegisterOutputSurface)(CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_VdpOutputSurface, unsigned int); /*proto*/ + +/* Module declarations from "cuda.bindings._bindings.cyruntime_ptds" */ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceReset)(void); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSynchronize)(void); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetLimit)(enum cudaLimit, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetLimit)(size_t *, enum cudaLimit); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetTexture1DLinearMaxWidth)(size_t *, struct cudaChannelFormatDesc const *, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetCacheConfig)(enum cudaFuncCache *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetStreamPriorityRange)(int *, int *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetCacheConfig)(enum cudaFuncCache); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetByPCIBusId)(int *, char const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetPCIBusId)(char *, int, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcGetEventHandle)(cudaIpcEventHandle_t *, cudaEvent_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcOpenEventHandle)(cudaEvent_t *, cudaIpcEventHandle_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcGetMemHandle)(cudaIpcMemHandle_t *, void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcOpenMemHandle)(void **, cudaIpcMemHandle_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcCloseMemHandle)(void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceFlushGPUDirectRDMAWrites)(enum cudaFlushGPUDirectRDMAWritesTarget, enum cudaFlushGPUDirectRDMAWritesScope); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceRegisterAsyncNotification)(int, cudaAsyncCallback, void *, cudaAsyncCallbackHandle_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceUnregisterAsyncNotification)(int, cudaAsyncCallbackHandle_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetSharedMemConfig)(enum cudaSharedMemConfig *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetSharedMemConfig)(enum cudaSharedMemConfig); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetLastError)(void); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaPeekAtLastError)(void); /*proto*/ +static char const *(*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetErrorName)(cudaError_t); /*proto*/ +static char const *(*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetErrorString)(cudaError_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceCount)(int *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceProperties_v2)(struct cudaDeviceProp *, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetAttribute)(int *, enum cudaDeviceAttr, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetDefaultMemPool)(cudaMemPool_t *, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetMemPool)(int, cudaMemPool_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetMemPool)(cudaMemPool_t *, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetNvSciSyncAttributes)(void *, int, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetP2PAttribute)(int *, enum cudaDeviceP2PAttr, int, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaChooseDevice)(int *, struct cudaDeviceProp const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaInitDevice)(int, unsigned int, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSetDevice)(int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDevice)(int *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSetDeviceFlags)(unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceFlags)(unsigned int *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreate)(cudaStream_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreateWithFlags)(cudaStream_t *, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreateWithPriority)(cudaStream_t *, unsigned int, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetPriority)(cudaStream_t, int *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetFlags)(cudaStream_t, unsigned int *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetId)(cudaStream_t, unsigned PY_LONG_LONG *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetDevice)(cudaStream_t, int *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCtxResetPersistingL2Cache)(void); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCopyAttributes)(cudaStream_t, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetAttribute)(cudaStream_t, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrID, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrValue *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamSetAttribute)(cudaStream_t, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrID, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrValue const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamDestroy)(cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamWaitEvent)(cudaStream_t, cudaEvent_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamAddCallback)(cudaStream_t, cudaStreamCallback_t, void *, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamSynchronize)(cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamQuery)(cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamAttachMemAsync)(cudaStream_t, void *, size_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamBeginCapture)(cudaStream_t, enum cudaStreamCaptureMode); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamBeginCaptureToGraph)(cudaStream_t, cudaGraph_t, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t, enum cudaStreamCaptureMode); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaThreadExchangeStreamCaptureMode)(enum cudaStreamCaptureMode *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamEndCapture)(cudaStream_t, cudaGraph_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamIsCapturing)(cudaStream_t, enum cudaStreamCaptureStatus *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetCaptureInfo_v2)(cudaStream_t, enum cudaStreamCaptureStatus *, unsigned PY_LONG_LONG *, cudaGraph_t *, cudaGraphNode_t const **, size_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetCaptureInfo_v3)(cudaStream_t, enum cudaStreamCaptureStatus *, unsigned PY_LONG_LONG *, cudaGraph_t *, cudaGraphNode_t const **, cudaGraphEdgeData const **, size_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamUpdateCaptureDependencies)(cudaStream_t, cudaGraphNode_t *, size_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamUpdateCaptureDependencies_v2)(cudaStream_t, cudaGraphNode_t *, cudaGraphEdgeData const *, size_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventCreate)(cudaEvent_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventCreateWithFlags)(cudaEvent_t *, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventRecord)(cudaEvent_t, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventRecordWithFlags)(cudaEvent_t, cudaStream_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventQuery)(cudaEvent_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventSynchronize)(cudaEvent_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventDestroy)(cudaEvent_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventElapsedTime)(float *, cudaEvent_t, cudaEvent_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventElapsedTime_v2)(float *, cudaEvent_t, cudaEvent_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaImportExternalMemory)(cudaExternalMemory_t *, struct cudaExternalMemoryHandleDesc const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaExternalMemoryGetMappedBuffer)(void **, cudaExternalMemory_t, struct cudaExternalMemoryBufferDesc const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaExternalMemoryGetMappedMipmappedArray)(cudaMipmappedArray_t *, cudaExternalMemory_t, struct cudaExternalMemoryMipmappedArrayDesc const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyExternalMemory)(cudaExternalMemory_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaImportExternalSemaphore)(cudaExternalSemaphore_t *, struct cudaExternalSemaphoreHandleDesc const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSignalExternalSemaphoresAsync_v2)(cudaExternalSemaphore_t const *, struct cudaExternalSemaphoreSignalParams const *, unsigned int, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaWaitExternalSemaphoresAsync_v2)(cudaExternalSemaphore_t const *, struct cudaExternalSemaphoreWaitParams const *, unsigned int, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyExternalSemaphore)(cudaExternalSemaphore_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetCacheConfig)(void const *, enum cudaFuncCache); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncGetAttributes)(struct cudaFuncAttributes *, void const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetAttribute)(void const *, enum cudaFuncAttribute, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLaunchHostFunc)(cudaStream_t, cudaHostFn_t, void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetSharedMemConfig)(void const *, enum cudaSharedMemConfig); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyMaxActiveBlocksPerMultiprocessor)(int *, void const *, int, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyAvailableDynamicSMemPerBlock)(size_t *, void const *, int, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags)(int *, void const *, int, size_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocManaged)(void **, size_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc)(void **, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocHost)(void **, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocPitch)(void **, size_t *, size_t, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocArray)(cudaArray_t *, struct cudaChannelFormatDesc const *, size_t, size_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFree)(void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeHost)(void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeArray)(cudaArray_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeMipmappedArray)(cudaMipmappedArray_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostAlloc)(void **, size_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostRegister)(void *, size_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostUnregister)(void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostGetDevicePointer)(void **, void *, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostGetFlags)(unsigned int *, void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc3D)(struct cudaPitchedPtr *, struct cudaExtent); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc3DArray)(cudaArray_t *, struct cudaChannelFormatDesc const *, struct cudaExtent, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocMipmappedArray)(cudaMipmappedArray_t *, struct cudaChannelFormatDesc const *, struct cudaExtent, unsigned int, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetMipmappedArrayLevel)(cudaArray_t *, cudaMipmappedArray_const_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3D)(struct cudaMemcpy3DParms const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DPeer)(struct cudaMemcpy3DPeerParms const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DAsync)(struct cudaMemcpy3DParms const *, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DPeerAsync)(struct cudaMemcpy3DPeerParms const *, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemGetInfo)(size_t *, size_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetInfo)(struct cudaChannelFormatDesc *, struct cudaExtent *, unsigned int *, cudaArray_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetPlane)(cudaArray_t *, cudaArray_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetMemoryRequirements)(struct cudaArrayMemoryRequirements *, cudaArray_t, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMipmappedArrayGetMemoryRequirements)(struct cudaArrayMemoryRequirements *, cudaMipmappedArray_t, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetSparseProperties)(struct cudaArraySparseProperties *, cudaArray_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMipmappedArrayGetSparseProperties)(struct cudaArraySparseProperties *, cudaMipmappedArray_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy)(void *, void const *, size_t, enum cudaMemcpyKind); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyPeer)(void *, int, void const *, int, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2D)(void *, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DToArray)(cudaArray_t, size_t, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DFromArray)(void *, size_t, cudaArray_const_t, size_t, size_t, size_t, size_t, enum cudaMemcpyKind); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DArrayToArray)(cudaArray_t, size_t, size_t, cudaArray_const_t, size_t, size_t, size_t, size_t, enum cudaMemcpyKind); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyAsync)(void *, void const *, size_t, enum cudaMemcpyKind, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyPeerAsync)(void *, int, void const *, int, size_t, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyBatchAsync)(void **, void **, size_t *, size_t, struct cudaMemcpyAttributes *, size_t *, size_t, size_t *, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DBatchAsync)(size_t, struct cudaMemcpy3DBatchOp *, size_t *, unsigned PY_LONG_LONG, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DAsync)(void *, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DToArrayAsync)(cudaArray_t, size_t, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DFromArrayAsync)(void *, size_t, cudaArray_const_t, size_t, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset)(void *, int, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset2D)(void *, size_t, int, size_t, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset3D)(struct cudaPitchedPtr, int, struct cudaExtent); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemsetAsync)(void *, int, size_t, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset2DAsync)(void *, size_t, int, size_t, size_t, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset3DAsync)(struct cudaPitchedPtr, int, struct cudaExtent, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPrefetchAsync)(void const *, size_t, int, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPrefetchAsync_v2)(void const *, size_t, struct cudaMemLocation, unsigned int, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemAdvise)(void const *, size_t, enum cudaMemoryAdvise, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemAdvise_v2)(void const *, size_t, enum cudaMemoryAdvise, struct cudaMemLocation); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemRangeGetAttribute)(void *, size_t, enum cudaMemRangeAttribute, void const *, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemRangeGetAttributes)(void **, size_t *, enum cudaMemRangeAttribute *, size_t, void const *, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyToArray)(cudaArray_t, size_t, size_t, void const *, size_t, enum cudaMemcpyKind); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyFromArray)(void *, cudaArray_const_t, size_t, size_t, size_t, enum cudaMemcpyKind); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyArrayToArray)(cudaArray_t, size_t, size_t, cudaArray_const_t, size_t, size_t, size_t, enum cudaMemcpyKind); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyToArrayAsync)(cudaArray_t, size_t, size_t, void const *, size_t, enum cudaMemcpyKind, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyFromArrayAsync)(void *, cudaArray_const_t, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocAsync)(void **, size_t, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeAsync)(void *, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolTrimTo)(cudaMemPool_t, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolSetAttribute)(cudaMemPool_t, enum cudaMemPoolAttr, void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolGetAttribute)(cudaMemPool_t, enum cudaMemPoolAttr, void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolSetAccess)(cudaMemPool_t, struct cudaMemAccessDesc const *, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolGetAccess)(enum cudaMemAccessFlags *, cudaMemPool_t, struct cudaMemLocation *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolCreate)(cudaMemPool_t *, struct cudaMemPoolProps const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolDestroy)(cudaMemPool_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocFromPoolAsync)(void **, size_t, cudaMemPool_t, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolExportToShareableHandle)(void *, cudaMemPool_t, enum cudaMemAllocationHandleType, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolImportFromShareableHandle)(cudaMemPool_t *, void *, enum cudaMemAllocationHandleType, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolExportPointer)(struct cudaMemPoolPtrExportData *, void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolImportPointer)(void **, cudaMemPool_t, struct cudaMemPoolPtrExportData *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaPointerGetAttributes)(struct cudaPointerAttributes *, void const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceCanAccessPeer)(int *, int, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceEnablePeerAccess)(int, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceDisablePeerAccess)(int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsUnregisterResource)(cudaGraphicsResource_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceSetMapFlags)(cudaGraphicsResource_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsMapResources)(int, cudaGraphicsResource_t *, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsUnmapResources)(int, cudaGraphicsResource_t *, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceGetMappedPointer)(void **, size_t *, cudaGraphicsResource_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsSubResourceGetMappedArray)(cudaArray_t *, cudaGraphicsResource_t, unsigned int, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceGetMappedMipmappedArray)(cudaMipmappedArray_t *, cudaGraphicsResource_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetChannelDesc)(struct cudaChannelFormatDesc *, cudaArray_const_t); /*proto*/ +static struct cudaChannelFormatDesc (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateChannelDesc)(int, int, int, int, enum cudaChannelFormatKind); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateTextureObject)(cudaTextureObject_t *, struct cudaResourceDesc const *, struct cudaTextureDesc const *, struct cudaResourceViewDesc const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyTextureObject)(cudaTextureObject_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectResourceDesc)(struct cudaResourceDesc *, cudaTextureObject_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectTextureDesc)(struct cudaTextureDesc *, cudaTextureObject_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectResourceViewDesc)(struct cudaResourceViewDesc *, cudaTextureObject_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateSurfaceObject)(cudaSurfaceObject_t *, struct cudaResourceDesc const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroySurfaceObject)(cudaSurfaceObject_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetSurfaceObjectResourceDesc)(struct cudaResourceDesc *, cudaSurfaceObject_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDriverGetVersion)(int *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaRuntimeGetVersion)(int *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphCreate)(cudaGraph_t *, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddKernelNode)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaKernelNodeParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeGetParams)(cudaGraphNode_t, struct cudaKernelNodeParams *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeSetParams)(cudaGraphNode_t, struct cudaKernelNodeParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeCopyAttributes)(cudaGraphNode_t, cudaGraphNode_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeGetAttribute)(cudaGraphNode_t, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrID, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrValue *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeSetAttribute)(cudaGraphNode_t, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrID, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrValue const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemcpyNode)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaMemcpy3DParms const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemcpyNode1D)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, void *, void const *, size_t, enum cudaMemcpyKind); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeGetParams)(cudaGraphNode_t, struct cudaMemcpy3DParms *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeSetParams)(cudaGraphNode_t, struct cudaMemcpy3DParms const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeSetParams1D)(cudaGraphNode_t, void *, void const *, size_t, enum cudaMemcpyKind); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemsetNode)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaMemsetParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemsetNodeGetParams)(cudaGraphNode_t, struct cudaMemsetParams *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemsetNodeSetParams)(cudaGraphNode_t, struct cudaMemsetParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddHostNode)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaHostNodeParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphHostNodeGetParams)(cudaGraphNode_t, struct cudaHostNodeParams *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphHostNodeSetParams)(cudaGraphNode_t, struct cudaHostNodeParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddChildGraphNode)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, cudaGraph_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphChildGraphNodeGetGraph)(cudaGraphNode_t, cudaGraph_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEmptyNode)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEventRecordNode)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, cudaEvent_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventRecordNodeGetEvent)(cudaGraphNode_t, cudaEvent_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventRecordNodeSetEvent)(cudaGraphNode_t, cudaEvent_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEventWaitNode)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, cudaEvent_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventWaitNodeGetEvent)(cudaGraphNode_t, cudaEvent_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventWaitNodeSetEvent)(cudaGraphNode_t, cudaEvent_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddExternalSemaphoresSignalNode)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaExternalSemaphoreSignalNodeParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresSignalNodeGetParams)(cudaGraphNode_t, struct cudaExternalSemaphoreSignalNodeParams *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresSignalNodeSetParams)(cudaGraphNode_t, struct cudaExternalSemaphoreSignalNodeParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddExternalSemaphoresWaitNode)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaExternalSemaphoreWaitNodeParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresWaitNodeGetParams)(cudaGraphNode_t, struct cudaExternalSemaphoreWaitNodeParams *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresWaitNodeSetParams)(cudaGraphNode_t, struct cudaExternalSemaphoreWaitNodeParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemAllocNode)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaMemAllocNodeParams *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemAllocNodeGetParams)(cudaGraphNode_t, struct cudaMemAllocNodeParams *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemFreeNode)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemFreeNodeGetParams)(cudaGraphNode_t, void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGraphMemTrim)(int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetGraphMemAttribute)(int, enum cudaGraphMemAttributeType, void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetGraphMemAttribute)(int, enum cudaGraphMemAttributeType, void *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphClone)(cudaGraph_t *, cudaGraph_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeFindInClone)(cudaGraphNode_t *, cudaGraphNode_t, cudaGraph_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetType)(cudaGraphNode_t, enum cudaGraphNodeType *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetNodes)(cudaGraph_t, cudaGraphNode_t *, size_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetRootNodes)(cudaGraph_t, cudaGraphNode_t *, size_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetEdges)(cudaGraph_t, cudaGraphNode_t *, cudaGraphNode_t *, size_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetEdges_v2)(cudaGraph_t, cudaGraphNode_t *, cudaGraphNode_t *, cudaGraphEdgeData *, size_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependencies)(cudaGraphNode_t, cudaGraphNode_t *, size_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependencies_v2)(cudaGraphNode_t, cudaGraphNode_t *, cudaGraphEdgeData *, size_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependentNodes)(cudaGraphNode_t, cudaGraphNode_t *, size_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependentNodes_v2)(cudaGraphNode_t, cudaGraphNode_t *, cudaGraphEdgeData *, size_t *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddDependencies)(cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddDependencies_v2)(cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRemoveDependencies)(cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRemoveDependencies_v2)(cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDestroyNode)(cudaGraphNode_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiate)(cudaGraphExec_t *, cudaGraph_t, unsigned PY_LONG_LONG); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiateWithFlags)(cudaGraphExec_t *, cudaGraph_t, unsigned PY_LONG_LONG); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiateWithParams)(cudaGraphExec_t *, cudaGraph_t, cudaGraphInstantiateParams *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecGetFlags)(cudaGraphExec_t, unsigned PY_LONG_LONG *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecKernelNodeSetParams)(cudaGraphExec_t, cudaGraphNode_t, struct cudaKernelNodeParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemcpyNodeSetParams)(cudaGraphExec_t, cudaGraphNode_t, struct cudaMemcpy3DParms const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemcpyNodeSetParams1D)(cudaGraphExec_t, cudaGraphNode_t, void *, void const *, size_t, enum cudaMemcpyKind); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemsetNodeSetParams)(cudaGraphExec_t, cudaGraphNode_t, struct cudaMemsetParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecHostNodeSetParams)(cudaGraphExec_t, cudaGraphNode_t, struct cudaHostNodeParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecChildGraphNodeSetParams)(cudaGraphExec_t, cudaGraphNode_t, cudaGraph_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecEventRecordNodeSetEvent)(cudaGraphExec_t, cudaGraphNode_t, cudaEvent_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecEventWaitNodeSetEvent)(cudaGraphExec_t, cudaGraphNode_t, cudaEvent_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecExternalSemaphoresSignalNodeSetParams)(cudaGraphExec_t, cudaGraphNode_t, struct cudaExternalSemaphoreSignalNodeParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecExternalSemaphoresWaitNodeSetParams)(cudaGraphExec_t, cudaGraphNode_t, struct cudaExternalSemaphoreWaitNodeParams const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeSetEnabled)(cudaGraphExec_t, cudaGraphNode_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetEnabled)(cudaGraphExec_t, cudaGraphNode_t, unsigned int *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecUpdate)(cudaGraphExec_t, cudaGraph_t, cudaGraphExecUpdateResultInfo *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphUpload)(cudaGraphExec_t, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphLaunch)(cudaGraphExec_t, cudaStream_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecDestroy)(cudaGraphExec_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDestroy)(cudaGraph_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDebugDotPrint)(cudaGraph_t, char const *, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectCreate)(cudaUserObject_t *, void *, cudaHostFn_t, unsigned int, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectRetain)(cudaUserObject_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectRelease)(cudaUserObject_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRetainUserObject)(cudaGraph_t, cudaUserObject_t, unsigned int, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphReleaseUserObject)(cudaGraph_t, cudaUserObject_t, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddNode)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaGraphNodeParams *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddNode_v2)(cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t, struct cudaGraphNodeParams *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeSetParams)(cudaGraphNode_t, struct cudaGraphNodeParams *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecNodeSetParams)(cudaGraphExec_t, cudaGraphNode_t, struct cudaGraphNodeParams *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphConditionalHandleCreate)(cudaGraphConditionalHandle *, cudaGraph_t, unsigned int, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDriverEntryPoint)(char const *, void **, unsigned PY_LONG_LONG, enum cudaDriverEntryPointQueryResult *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDriverEntryPointByVersion)(char const *, void **, unsigned int, unsigned PY_LONG_LONG, enum cudaDriverEntryPointQueryResult *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryLoadData)(cudaLibrary_t *, void const *, enum cudaJitOption *, void **, unsigned int, enum cudaLibraryOption *, void **, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryLoadFromFile)(cudaLibrary_t *, char const *, enum cudaJitOption *, void **, unsigned int, enum cudaLibraryOption *, void **, unsigned int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryUnload)(cudaLibrary_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetKernel)(cudaKernel_t *, cudaLibrary_t, char const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetGlobal)(void **, size_t *, cudaLibrary_t, char const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetManaged)(void **, size_t *, cudaLibrary_t, char const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetUnifiedFunction)(void **, cudaLibrary_t, char const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetKernelCount)(unsigned int *, cudaLibrary_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryEnumerateKernels)(cudaKernel_t *, unsigned int, cudaLibrary_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaKernelSetAttributeForDevice)(cudaKernel_t, enum cudaFuncAttribute, int, int); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetExportTable)(void const **, cudaUUID_t const *); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetKernel)(cudaKernel_t *, void const *); /*proto*/ +static struct cudaPitchedPtr (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaPitchedPtr)(void *, size_t, size_t, size_t); /*proto*/ +static struct cudaPos (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaPos)(size_t, size_t, size_t); /*proto*/ +static struct cudaExtent (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaExtent)(size_t, size_t, size_t); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaProfilerStart)(void); /*proto*/ +static cudaError_t (*__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaProfilerStop)(void); /*proto*/ + +/* Module declarations from "cython" */ + +/* Module declarations from "libc.string" */ + +/* Module declarations from "cuda.bindings._bindings.cyruntime" */ +static int __pyx_v_4cuda_8bindings_9_bindings_9cyruntime___cudaPythonInit; +static int __pyx_v_4cuda_8bindings_9_bindings_9cyruntime___usePTDS; +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getDriverEglFrame(__pyx_t_4cuda_8bindings_8cydriver_CUeglFrame *, __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame); /*proto*/ +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getRuntimeEglFrame(__pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame *, __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame); /*proto*/ +static int __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaPythonInit(void); /*proto*/ +static CYTHON_INLINE int __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(void); /*proto*/ +static int __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(struct cudaChannelFormatDesc const *, int, int, int, int, int); /*proto*/ +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getDescInfo(struct cudaChannelFormatDesc const *, int *, CUarray_format *); /*proto*/ +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getChannelFormatDescFromDriverDesc(struct cudaChannelFormatDesc *, size_t *, size_t *, size_t *, CUDA_ARRAY3D_DESCRIPTOR_v2 const *); /*proto*/ +/* #### Code section: typeinfo ### */ +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "cuda.bindings._bindings.cyruntime" +extern int __pyx_module_is_main_cuda__bindings___bindings__cyruntime; +int __pyx_module_is_main_cuda__bindings___bindings__cyruntime = 0; + +/* Implementation of "cuda.bindings._bindings.cyruntime" */ +/* #### Code section: global_var ### */ +static PyObject *__pyx_builtin_range; +/* #### Code section: string_decls ### */ +static const char __pyx_k_[] = "?"; +static const char __pyx_k_os[] = "os"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_name[] = "__name__"; +static const char __pyx_k_spec[] = "__spec__"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_range[] = "range"; +static const char __pyx_k_getenv[] = "getenv"; +static const char __pyx_k_module[] = "__module__"; +static const char __pyx_k_default[] = "default"; +static const char __pyx_k_qualname[] = "__qualname__"; +static const char __pyx_k_initializing[] = "_initializing"; +static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; +static const char __pyx_k_CUDA_PYTHON_CUDA_PER_THREAD_DEFA[] = "CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM"; +/* #### Code section: decls ### */ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + #ifdef __Pyx_CyFunction_USED + PyTypeObject *__pyx_CyFunctionType; + #endif + #ifdef __Pyx_FusedFunction_USED + PyTypeObject *__pyx_FusedFunctionType; + #endif + #ifdef __Pyx_Generator_USED + PyTypeObject *__pyx_GeneratorType; + #endif + #ifdef __Pyx_IterableCoroutine_USED + PyTypeObject *__pyx_IterableCoroutineType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineAwaitType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineType; + #endif + PyObject *__pyx_string_tab[14]; +/* #### Code section: module_state_contents ### */ +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_n_u_CUDA_PYTHON_CUDA_PER_THREAD_DEFA __pyx_string_tab[1] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[2] +#define __pyx_n_u_default __pyx_string_tab[3] +#define __pyx_n_u_getenv __pyx_string_tab[4] +#define __pyx_n_u_initializing __pyx_string_tab[5] +#define __pyx_n_u_main __pyx_string_tab[6] +#define __pyx_n_u_module __pyx_string_tab[7] +#define __pyx_n_u_name __pyx_string_tab[8] +#define __pyx_n_u_os __pyx_string_tab[9] +#define __pyx_n_u_qualname __pyx_string_tab[10] +#define __pyx_n_u_range __pyx_string_tab[11] +#define __pyx_n_u_spec __pyx_string_tab[12] +#define __pyx_n_u_test __pyx_string_tab[13] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_CLEAR(clear_module_state->__pyx_FusedFunctionType); + #endif + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + for (int i=0; i<14; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_VISIT(traverse_module_state->__pyx_FusedFunctionType); + #endif + for (int i=0; i<14; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "cuda/bindings/_bindings/cyruntime.pyx":13 + * cdef bint __cudaPythonInit = False + * cdef bint __usePTDS = False + * cdef int _cudaPythonInit() except -1 nogil: # <<<<<<<<<<<<<< + * global __cudaPythonInit + * global __usePTDS +*/ + +static int __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaPythonInit(void) { + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + int __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cudaPythonInit", 1); + + /* "cuda/bindings/_bindings/cyruntime.pyx":17 + * global __usePTDS + * + * with gil: # <<<<<<<<<<<<<< + * __usePTDS = os.getenv('CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM', default=False) + * __cudaPythonInit = True +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_bindings/cyruntime.pyx":18 + * + * with gil: + * __usePTDS = os.getenv('CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM', default=False) # <<<<<<<<<<<<<< + * __cudaPythonInit = True + * return __usePTDS +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_os); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 18, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_PyObject_GetAttrStr(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_getenv); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 18, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_4); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_4, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[2 + ((CYTHON_VECTORCALL) ? 1 : 0)] = {__pyx_t_2, __pyx_mstate_global->__pyx_n_u_CUDA_PYTHON_CUDA_PER_THREAD_DEFA}; + __pyx_t_3 = __Pyx_MakeVectorcallBuilderKwds(1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 18, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_3); + if (__Pyx_VectorcallBuilder_AddArg(__pyx_mstate_global->__pyx_n_u_default, Py_False, __pyx_t_3, __pyx_callargs+2, 0) < (0)) __PYX_ERR(0, 18, __pyx_L4_error) + __pyx_t_1 = __Pyx_Object_Vectorcall_CallFromBuilder(__pyx_t_4, __pyx_callargs+__pyx_t_5, (2-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET), __pyx_t_3); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 18, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_t_6 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely((__pyx_t_6 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 18, __pyx_L4_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_v_4cuda_8bindings_9_bindings_9cyruntime___usePTDS = __pyx_t_6; + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":17 + * global __usePTDS + * + * with gil: # <<<<<<<<<<<<<< + * __usePTDS = os.getenv('CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM', default=False) + * __cudaPythonInit = True +*/ + /*finally:*/ { + /*normal exit:*/{ + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L5; + } + __pyx_L4_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + __pyx_L5:; + } + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":19 + * with gil: + * __usePTDS = os.getenv('CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM', default=False) + * __cudaPythonInit = True # <<<<<<<<<<<<<< + * return __usePTDS + * +*/ + __pyx_v_4cuda_8bindings_9_bindings_9cyruntime___cudaPythonInit = 1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":20 + * __usePTDS = os.getenv('CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM', default=False) + * __cudaPythonInit = True + * return __usePTDS # <<<<<<<<<<<<<< + * + * # Create a very small function to check whether we are init'ed, so the C +*/ + __pyx_r = __pyx_v_4cuda_8bindings_9_bindings_9cyruntime___usePTDS; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":13 + * cdef bint __cudaPythonInit = False + * cdef bint __usePTDS = False + * cdef int _cudaPythonInit() except -1 nogil: # <<<<<<<<<<<<<< + * global __cudaPythonInit + * global __usePTDS +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaPythonInit", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":24 + * # Create a very small function to check whether we are init'ed, so the C + * # compiler can inline it. + * cdef inline int cudaPythonInit() except -1 nogil: # <<<<<<<<<<<<<< + * if __cudaPythonInit: + * return __usePTDS +*/ + +static CYTHON_INLINE int __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(void) { + int __pyx_r; + int __pyx_t_1; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":25 + * # compiler can inline it. + * cdef inline int cudaPythonInit() except -1 nogil: + * if __cudaPythonInit: # <<<<<<<<<<<<<< + * return __usePTDS + * return _cudaPythonInit() +*/ + if (__pyx_v_4cuda_8bindings_9_bindings_9cyruntime___cudaPythonInit) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":26 + * cdef inline int cudaPythonInit() except -1 nogil: + * if __cudaPythonInit: + * return __usePTDS # <<<<<<<<<<<<<< + * return _cudaPythonInit() + * +*/ + __pyx_r = __pyx_v_4cuda_8bindings_9_bindings_9cyruntime___usePTDS; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":25 + * # compiler can inline it. + * cdef inline int cudaPythonInit() except -1 nogil: + * if __cudaPythonInit: # <<<<<<<<<<<<<< + * return __usePTDS + * return _cudaPythonInit() +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":27 + * if __cudaPythonInit: + * return __usePTDS + * return _cudaPythonInit() # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceReset() except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 27, __pyx_L1_error) + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":24 + * # Create a very small function to check whether we are init'ed, so the C + * # compiler can inline it. + * cdef inline int cudaPythonInit() except -1 nogil: # <<<<<<<<<<<<<< + * if __cudaPythonInit: + * return __usePTDS +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime.cudaPythonInit", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":29 + * return _cudaPythonInit() + * + * cdef cudaError_t _cudaDeviceReset() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceReset(void) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":30 + * + * cdef cudaError_t _cudaDeviceReset() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceReset() +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 30, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":31 + * cdef cudaError_t _cudaDeviceReset() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceReset() + * return cudaDeviceReset() +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":32 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceReset() # <<<<<<<<<<<<<< + * return cudaDeviceReset() + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceReset(); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 32, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":31 + * cdef cudaError_t _cudaDeviceReset() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceReset() + * return cudaDeviceReset() +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":33 + * if usePTDS: + * return ptds._cudaDeviceReset() + * return cudaDeviceReset() # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceReset(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":29 + * return _cudaPythonInit() + * + * cdef cudaError_t _cudaDeviceReset() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceReset", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":35 + * return cudaDeviceReset() + * + * cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceSynchronize(void) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":36 + * + * cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceSynchronize() +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 36, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":37 + * cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceSynchronize() + * return cudaDeviceSynchronize() +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":38 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceSynchronize() # <<<<<<<<<<<<<< + * return cudaDeviceSynchronize() + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSynchronize(); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 38, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":37 + * cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceSynchronize() + * return cudaDeviceSynchronize() +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":39 + * if usePTDS: + * return ptds._cudaDeviceSynchronize() + * return cudaDeviceSynchronize() # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceSynchronize(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":35 + * return cudaDeviceReset() + * + * cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceSynchronize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":41 + * return cudaDeviceSynchronize() + * + * cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceSetLimit(enum cudaLimit __pyx_v_limit, size_t __pyx_v_value) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":42 + * + * cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceSetLimit(limit, value) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 42, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":43 + * cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceSetLimit(limit, value) + * return cudaDeviceSetLimit(limit, value) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":44 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceSetLimit(limit, value) # <<<<<<<<<<<<<< + * return cudaDeviceSetLimit(limit, value) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetLimit(((enum cudaLimit)__pyx_v_limit), __pyx_v_value); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 44, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":43 + * cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceSetLimit(limit, value) + * return cudaDeviceSetLimit(limit, value) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":45 + * if usePTDS: + * return ptds._cudaDeviceSetLimit(limit, value) + * return cudaDeviceSetLimit(limit, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceSetLimit(__pyx_v_limit, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":41 + * return cudaDeviceSynchronize() + * + * cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceSetLimit", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":47 + * return cudaDeviceSetLimit(limit, value) + * + * cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetLimit(size_t *__pyx_v_pValue, enum cudaLimit __pyx_v_limit) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":48 + * + * cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGetLimit(pValue, limit) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 48, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":49 + * cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetLimit(pValue, limit) + * return cudaDeviceGetLimit(pValue, limit) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":50 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGetLimit(pValue, limit) # <<<<<<<<<<<<<< + * return cudaDeviceGetLimit(pValue, limit) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetLimit(__pyx_v_pValue, ((enum cudaLimit)__pyx_v_limit)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 50, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":49 + * cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetLimit(pValue, limit) + * return cudaDeviceGetLimit(pValue, limit) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":51 + * if usePTDS: + * return ptds._cudaDeviceGetLimit(pValue, limit) + * return cudaDeviceGetLimit(pValue, limit) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetLimit(__pyx_v_pValue, __pyx_v_limit); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":47 + * return cudaDeviceSetLimit(limit, value) + * + * cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGetLimit", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":53 + * return cudaDeviceGetLimit(pValue, limit) + * + * cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetTexture1DLinearMaxWidth(size_t *__pyx_v_maxWidthInElements, struct cudaChannelFormatDesc const *__pyx_v_fmtDesc, int __pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":54 + * + * cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 54, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":55 + * cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) + * return cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":56 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) # <<<<<<<<<<<<<< + * return cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetTexture1DLinearMaxWidth(__pyx_v_maxWidthInElements, __pyx_v_fmtDesc, __pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 56, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":55 + * cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) + * return cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":57 + * if usePTDS: + * return ptds._cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) + * return cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetTexture1DLinearMaxWidth(__pyx_v_maxWidthInElements, __pyx_v_fmtDesc, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":53 + * return cudaDeviceGetLimit(pValue, limit) + * + * cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGetTexture1DLinearMaxWidth", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":59 + * return cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) + * + * cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetCacheConfig(enum cudaFuncCache *__pyx_v_pCacheConfig) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":60 + * + * cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGetCacheConfig(pCacheConfig) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 60, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":61 + * cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetCacheConfig(pCacheConfig) + * return cudaDeviceGetCacheConfig(pCacheConfig) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":62 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGetCacheConfig(pCacheConfig) # <<<<<<<<<<<<<< + * return cudaDeviceGetCacheConfig(pCacheConfig) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetCacheConfig(((enum cudaFuncCache *)__pyx_v_pCacheConfig)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 62, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":61 + * cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetCacheConfig(pCacheConfig) + * return cudaDeviceGetCacheConfig(pCacheConfig) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":63 + * if usePTDS: + * return ptds._cudaDeviceGetCacheConfig(pCacheConfig) + * return cudaDeviceGetCacheConfig(pCacheConfig) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetCacheConfig(__pyx_v_pCacheConfig); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":59 + * return cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) + * + * cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGetCacheConfig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":65 + * return cudaDeviceGetCacheConfig(pCacheConfig) + * + * cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetStreamPriorityRange(int *__pyx_v_leastPriority, int *__pyx_v_greatestPriority) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":66 + * + * cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 66, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":67 + * cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) + * return cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":68 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) # <<<<<<<<<<<<<< + * return cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetStreamPriorityRange(__pyx_v_leastPriority, __pyx_v_greatestPriority); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 68, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":67 + * cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) + * return cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":69 + * if usePTDS: + * return ptds._cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) + * return cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetStreamPriorityRange(__pyx_v_leastPriority, __pyx_v_greatestPriority); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":65 + * return cudaDeviceGetCacheConfig(pCacheConfig) + * + * cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGetStreamPriorityRange", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":71 + * return cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) + * + * cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceSetCacheConfig(enum cudaFuncCache __pyx_v_cacheConfig) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":72 + * + * cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceSetCacheConfig(cacheConfig) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 72, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":73 + * cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceSetCacheConfig(cacheConfig) + * return cudaDeviceSetCacheConfig(cacheConfig) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":74 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceSetCacheConfig(cacheConfig) # <<<<<<<<<<<<<< + * return cudaDeviceSetCacheConfig(cacheConfig) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetCacheConfig(((enum cudaFuncCache)__pyx_v_cacheConfig)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 74, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":73 + * cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceSetCacheConfig(cacheConfig) + * return cudaDeviceSetCacheConfig(cacheConfig) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":75 + * if usePTDS: + * return ptds._cudaDeviceSetCacheConfig(cacheConfig) + * return cudaDeviceSetCacheConfig(cacheConfig) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceSetCacheConfig(__pyx_v_cacheConfig); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":71 + * return cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) + * + * cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceSetCacheConfig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":77 + * return cudaDeviceSetCacheConfig(cacheConfig) + * + * cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetByPCIBusId(int *__pyx_v_device, char const *__pyx_v_pciBusId) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":78 + * + * cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGetByPCIBusId(device, pciBusId) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 78, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":79 + * cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetByPCIBusId(device, pciBusId) + * return cudaDeviceGetByPCIBusId(device, pciBusId) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":80 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGetByPCIBusId(device, pciBusId) # <<<<<<<<<<<<<< + * return cudaDeviceGetByPCIBusId(device, pciBusId) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetByPCIBusId(__pyx_v_device, __pyx_v_pciBusId); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 80, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":79 + * cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetByPCIBusId(device, pciBusId) + * return cudaDeviceGetByPCIBusId(device, pciBusId) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":81 + * if usePTDS: + * return ptds._cudaDeviceGetByPCIBusId(device, pciBusId) + * return cudaDeviceGetByPCIBusId(device, pciBusId) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetByPCIBusId(__pyx_v_device, __pyx_v_pciBusId); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":77 + * return cudaDeviceSetCacheConfig(cacheConfig) + * + * cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGetByPCIBusId", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":83 + * return cudaDeviceGetByPCIBusId(device, pciBusId) + * + * cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetPCIBusId(char *__pyx_v_pciBusId, int __pyx_v_length, int __pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":84 + * + * cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGetPCIBusId(pciBusId, length, device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 84, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":85 + * cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetPCIBusId(pciBusId, length, device) + * return cudaDeviceGetPCIBusId(pciBusId, length, device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":86 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGetPCIBusId(pciBusId, length, device) # <<<<<<<<<<<<<< + * return cudaDeviceGetPCIBusId(pciBusId, length, device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetPCIBusId(__pyx_v_pciBusId, __pyx_v_length, __pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 86, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":85 + * cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetPCIBusId(pciBusId, length, device) + * return cudaDeviceGetPCIBusId(pciBusId, length, device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":87 + * if usePTDS: + * return ptds._cudaDeviceGetPCIBusId(pciBusId, length, device) + * return cudaDeviceGetPCIBusId(pciBusId, length, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetPCIBusId(__pyx_v_pciBusId, __pyx_v_length, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":83 + * return cudaDeviceGetByPCIBusId(device, pciBusId) + * + * cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGetPCIBusId", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":89 + * return cudaDeviceGetPCIBusId(pciBusId, length, device) + * + * cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaIpcGetEventHandle(cudaIpcEventHandle_t *__pyx_v_handle, cudaEvent_t __pyx_v_event) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":90 + * + * cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaIpcGetEventHandle(handle, event) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 90, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":91 + * cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaIpcGetEventHandle(handle, event) + * return cudaIpcGetEventHandle(handle, event) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":92 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaIpcGetEventHandle(handle, event) # <<<<<<<<<<<<<< + * return cudaIpcGetEventHandle(handle, event) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcGetEventHandle(__pyx_v_handle, __pyx_v_event); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":91 + * cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaIpcGetEventHandle(handle, event) + * return cudaIpcGetEventHandle(handle, event) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":93 + * if usePTDS: + * return ptds._cudaIpcGetEventHandle(handle, event) + * return cudaIpcGetEventHandle(handle, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaIpcGetEventHandle(__pyx_v_handle, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":89 + * return cudaDeviceGetPCIBusId(pciBusId, length, device) + * + * cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaIpcGetEventHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":95 + * return cudaIpcGetEventHandle(handle, event) + * + * cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaIpcOpenEventHandle(cudaEvent_t *__pyx_v_event, cudaIpcEventHandle_t __pyx_v_handle) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":96 + * + * cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaIpcOpenEventHandle(event, handle) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 96, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":97 + * cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaIpcOpenEventHandle(event, handle) + * return cudaIpcOpenEventHandle(event, handle) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":98 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaIpcOpenEventHandle(event, handle) # <<<<<<<<<<<<<< + * return cudaIpcOpenEventHandle(event, handle) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcOpenEventHandle(__pyx_v_event, __pyx_v_handle); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 98, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":97 + * cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaIpcOpenEventHandle(event, handle) + * return cudaIpcOpenEventHandle(event, handle) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":99 + * if usePTDS: + * return ptds._cudaIpcOpenEventHandle(event, handle) + * return cudaIpcOpenEventHandle(event, handle) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaIpcOpenEventHandle(__pyx_v_event, __pyx_v_handle); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":95 + * return cudaIpcGetEventHandle(handle, event) + * + * cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaIpcOpenEventHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":101 + * return cudaIpcOpenEventHandle(event, handle) + * + * cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaIpcGetMemHandle(cudaIpcMemHandle_t *__pyx_v_handle, void *__pyx_v_devPtr) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":102 + * + * cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaIpcGetMemHandle(handle, devPtr) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 102, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":103 + * cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaIpcGetMemHandle(handle, devPtr) + * return cudaIpcGetMemHandle(handle, devPtr) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":104 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaIpcGetMemHandle(handle, devPtr) # <<<<<<<<<<<<<< + * return cudaIpcGetMemHandle(handle, devPtr) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcGetMemHandle(__pyx_v_handle, __pyx_v_devPtr); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 104, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":103 + * cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaIpcGetMemHandle(handle, devPtr) + * return cudaIpcGetMemHandle(handle, devPtr) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":105 + * if usePTDS: + * return ptds._cudaIpcGetMemHandle(handle, devPtr) + * return cudaIpcGetMemHandle(handle, devPtr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaIpcGetMemHandle(__pyx_v_handle, __pyx_v_devPtr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":101 + * return cudaIpcOpenEventHandle(event, handle) + * + * cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaIpcGetMemHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":107 + * return cudaIpcGetMemHandle(handle, devPtr) + * + * cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaIpcOpenMemHandle(void **__pyx_v_devPtr, cudaIpcMemHandle_t __pyx_v_handle, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":108 + * + * cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaIpcOpenMemHandle(devPtr, handle, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 108, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":109 + * cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaIpcOpenMemHandle(devPtr, handle, flags) + * return cudaIpcOpenMemHandle(devPtr, handle, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":110 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaIpcOpenMemHandle(devPtr, handle, flags) # <<<<<<<<<<<<<< + * return cudaIpcOpenMemHandle(devPtr, handle, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcOpenMemHandle(__pyx_v_devPtr, __pyx_v_handle, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 110, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":109 + * cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaIpcOpenMemHandle(devPtr, handle, flags) + * return cudaIpcOpenMemHandle(devPtr, handle, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":111 + * if usePTDS: + * return ptds._cudaIpcOpenMemHandle(devPtr, handle, flags) + * return cudaIpcOpenMemHandle(devPtr, handle, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaIpcOpenMemHandle(__pyx_v_devPtr, __pyx_v_handle, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":107 + * return cudaIpcGetMemHandle(handle, devPtr) + * + * cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaIpcOpenMemHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":113 + * return cudaIpcOpenMemHandle(devPtr, handle, flags) + * + * cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaIpcCloseMemHandle(void *__pyx_v_devPtr) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":114 + * + * cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaIpcCloseMemHandle(devPtr) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 114, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":115 + * cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaIpcCloseMemHandle(devPtr) + * return cudaIpcCloseMemHandle(devPtr) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":116 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaIpcCloseMemHandle(devPtr) # <<<<<<<<<<<<<< + * return cudaIpcCloseMemHandle(devPtr) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcCloseMemHandle(__pyx_v_devPtr); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 116, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":115 + * cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaIpcCloseMemHandle(devPtr) + * return cudaIpcCloseMemHandle(devPtr) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":117 + * if usePTDS: + * return ptds._cudaIpcCloseMemHandle(devPtr) + * return cudaIpcCloseMemHandle(devPtr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaIpcCloseMemHandle(__pyx_v_devPtr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":113 + * return cudaIpcOpenMemHandle(devPtr, handle, flags) + * + * cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaIpcCloseMemHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":119 + * return cudaIpcCloseMemHandle(devPtr) + * + * cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceFlushGPUDirectRDMAWrites(enum cudaFlushGPUDirectRDMAWritesTarget __pyx_v_target, enum cudaFlushGPUDirectRDMAWritesScope __pyx_v_scope) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":120 + * + * cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceFlushGPUDirectRDMAWrites(target, scope) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 120, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":121 + * cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceFlushGPUDirectRDMAWrites(target, scope) + * return cudaDeviceFlushGPUDirectRDMAWrites(target, scope) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":122 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceFlushGPUDirectRDMAWrites(target, scope) # <<<<<<<<<<<<<< + * return cudaDeviceFlushGPUDirectRDMAWrites(target, scope) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceFlushGPUDirectRDMAWrites(((enum cudaFlushGPUDirectRDMAWritesTarget)__pyx_v_target), ((enum cudaFlushGPUDirectRDMAWritesScope)__pyx_v_scope)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 122, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":121 + * cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceFlushGPUDirectRDMAWrites(target, scope) + * return cudaDeviceFlushGPUDirectRDMAWrites(target, scope) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":123 + * if usePTDS: + * return ptds._cudaDeviceFlushGPUDirectRDMAWrites(target, scope) + * return cudaDeviceFlushGPUDirectRDMAWrites(target, scope) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceFlushGPUDirectRDMAWrites(__pyx_v_target, __pyx_v_scope); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":119 + * return cudaIpcCloseMemHandle(devPtr) + * + * cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceFlushGPUDirectRDMAWrites", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":125 + * return cudaDeviceFlushGPUDirectRDMAWrites(target, scope) + * + * cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceRegisterAsyncNotification(int __pyx_v_device, cudaAsyncCallback __pyx_v_callbackFunc, void *__pyx_v_userData, cudaAsyncCallbackHandle_t *__pyx_v_callback) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":126 + * + * cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 126, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":127 + * cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) + * return cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":128 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) # <<<<<<<<<<<<<< + * return cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceRegisterAsyncNotification(__pyx_v_device, __pyx_v_callbackFunc, __pyx_v_userData, __pyx_v_callback); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 128, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":127 + * cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) + * return cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":129 + * if usePTDS: + * return ptds._cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) + * return cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceRegisterAsyncNotification(__pyx_v_device, __pyx_v_callbackFunc, __pyx_v_userData, __pyx_v_callback); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":125 + * return cudaDeviceFlushGPUDirectRDMAWrites(target, scope) + * + * cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceRegisterAsyncNotification", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":131 + * return cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) + * + * cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceUnregisterAsyncNotification(int __pyx_v_device, cudaAsyncCallbackHandle_t __pyx_v_callback) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":132 + * + * cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceUnregisterAsyncNotification(device, callback) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 132, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":133 + * cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceUnregisterAsyncNotification(device, callback) + * return cudaDeviceUnregisterAsyncNotification(device, callback) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":134 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceUnregisterAsyncNotification(device, callback) # <<<<<<<<<<<<<< + * return cudaDeviceUnregisterAsyncNotification(device, callback) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceUnregisterAsyncNotification(__pyx_v_device, __pyx_v_callback); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 134, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":133 + * cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceUnregisterAsyncNotification(device, callback) + * return cudaDeviceUnregisterAsyncNotification(device, callback) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":135 + * if usePTDS: + * return ptds._cudaDeviceUnregisterAsyncNotification(device, callback) + * return cudaDeviceUnregisterAsyncNotification(device, callback) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceUnregisterAsyncNotification(__pyx_v_device, __pyx_v_callback); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":131 + * return cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) + * + * cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceUnregisterAsyncNotification", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":137 + * return cudaDeviceUnregisterAsyncNotification(device, callback) + * + * cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetSharedMemConfig(enum cudaSharedMemConfig *__pyx_v_pConfig) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":138 + * + * cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGetSharedMemConfig(pConfig) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 138, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":139 + * cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetSharedMemConfig(pConfig) + * return cudaDeviceGetSharedMemConfig(pConfig) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":140 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGetSharedMemConfig(pConfig) # <<<<<<<<<<<<<< + * return cudaDeviceGetSharedMemConfig(pConfig) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetSharedMemConfig(((enum cudaSharedMemConfig *)__pyx_v_pConfig)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 140, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":139 + * cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetSharedMemConfig(pConfig) + * return cudaDeviceGetSharedMemConfig(pConfig) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":141 + * if usePTDS: + * return ptds._cudaDeviceGetSharedMemConfig(pConfig) + * return cudaDeviceGetSharedMemConfig(pConfig) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetSharedMemConfig(__pyx_v_pConfig); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":137 + * return cudaDeviceUnregisterAsyncNotification(device, callback) + * + * cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGetSharedMemConfig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":143 + * return cudaDeviceGetSharedMemConfig(pConfig) + * + * cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceSetSharedMemConfig(enum cudaSharedMemConfig __pyx_v_config) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":144 + * + * cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceSetSharedMemConfig(config) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 144, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":145 + * cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceSetSharedMemConfig(config) + * return cudaDeviceSetSharedMemConfig(config) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":146 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceSetSharedMemConfig(config) # <<<<<<<<<<<<<< + * return cudaDeviceSetSharedMemConfig(config) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetSharedMemConfig(((enum cudaSharedMemConfig)__pyx_v_config)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 146, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":145 + * cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceSetSharedMemConfig(config) + * return cudaDeviceSetSharedMemConfig(config) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":147 + * if usePTDS: + * return ptds._cudaDeviceSetSharedMemConfig(config) + * return cudaDeviceSetSharedMemConfig(config) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceSetSharedMemConfig(__pyx_v_config); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":143 + * return cudaDeviceGetSharedMemConfig(pConfig) + * + * cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceSetSharedMemConfig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":149 + * return cudaDeviceSetSharedMemConfig(config) + * + * cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetLastError(void) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":150 + * + * cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetLastError() +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 150, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":151 + * cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetLastError() + * return cudaGetLastError() +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":152 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetLastError() # <<<<<<<<<<<<<< + * return cudaGetLastError() + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetLastError(); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 152, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":151 + * cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetLastError() + * return cudaGetLastError() +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":153 + * if usePTDS: + * return ptds._cudaGetLastError() + * return cudaGetLastError() # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetLastError(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":149 + * return cudaDeviceSetSharedMemConfig(config) + * + * cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetLastError", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":155 + * return cudaGetLastError() + * + * cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaPeekAtLastError(void) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":156 + * + * cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaPeekAtLastError() +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 156, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":157 + * cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaPeekAtLastError() + * return cudaPeekAtLastError() +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":158 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaPeekAtLastError() # <<<<<<<<<<<<<< + * return cudaPeekAtLastError() + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaPeekAtLastError(); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 158, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":157 + * cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaPeekAtLastError() + * return cudaPeekAtLastError() +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":159 + * if usePTDS: + * return ptds._cudaPeekAtLastError() + * return cudaPeekAtLastError() # <<<<<<<<<<<<<< + * + * cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil: +*/ + __pyx_r = cudaPeekAtLastError(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":155 + * return cudaGetLastError() + * + * cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaPeekAtLastError", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":161 + * return cudaPeekAtLastError() + * + * cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static char const *__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetErrorName(cudaError_t __pyx_v_error) { + int __pyx_v_usePTDS; + char const *__pyx_r; + int __pyx_t_1; + char const *__pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":162 + * + * cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetErrorName(error) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 162, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":163 + * cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetErrorName(error) + * return cudaGetErrorName(error) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":164 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetErrorName(error) # <<<<<<<<<<<<<< + * return cudaGetErrorName(error) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetErrorName(((cudaError_t)__pyx_v_error)); if (unlikely(__pyx_t_2 == ((char const *)0) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 164, __pyx_L1_error) + __pyx_r = __pyx_t_2; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":163 + * cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetErrorName(error) + * return cudaGetErrorName(error) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":165 + * if usePTDS: + * return ptds._cudaGetErrorName(error) + * return cudaGetErrorName(error) # <<<<<<<<<<<<<< + * + * cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil: +*/ + __pyx_r = cudaGetErrorName(__pyx_v_error); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":161 + * return cudaPeekAtLastError() + * + * cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetErrorName", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":167 + * return cudaGetErrorName(error) + * + * cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static char const *__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetErrorString(cudaError_t __pyx_v_error) { + int __pyx_v_usePTDS; + char const *__pyx_r; + int __pyx_t_1; + char const *__pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":168 + * + * cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetErrorString(error) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 168, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":169 + * cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetErrorString(error) + * return cudaGetErrorString(error) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":170 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetErrorString(error) # <<<<<<<<<<<<<< + * return cudaGetErrorString(error) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetErrorString(((cudaError_t)__pyx_v_error)); if (unlikely(__pyx_t_2 == ((char const *)0) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 170, __pyx_L1_error) + __pyx_r = __pyx_t_2; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":169 + * cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetErrorString(error) + * return cudaGetErrorString(error) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":171 + * if usePTDS: + * return ptds._cudaGetErrorString(error) + * return cudaGetErrorString(error) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetErrorString(__pyx_v_error); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":167 + * return cudaGetErrorName(error) + * + * cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetErrorString", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":173 + * return cudaGetErrorString(error) + * + * cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetDeviceCount(int *__pyx_v_count) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":174 + * + * cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetDeviceCount(count) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 174, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":175 + * cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetDeviceCount(count) + * return cudaGetDeviceCount(count) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":176 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetDeviceCount(count) # <<<<<<<<<<<<<< + * return cudaGetDeviceCount(count) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceCount(__pyx_v_count); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 176, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":175 + * cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetDeviceCount(count) + * return cudaGetDeviceCount(count) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":177 + * if usePTDS: + * return ptds._cudaGetDeviceCount(count) + * return cudaGetDeviceCount(count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetDeviceCount(__pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":173 + * return cudaGetErrorString(error) + * + * cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetDeviceCount", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":179 + * return cudaGetDeviceCount(count) + * + * cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetDeviceProperties_v2(struct cudaDeviceProp *__pyx_v_prop, int __pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":180 + * + * cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetDeviceProperties_v2(prop, device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 180, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":181 + * cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetDeviceProperties_v2(prop, device) + * return cudaGetDeviceProperties_v2(prop, device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":182 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetDeviceProperties_v2(prop, device) # <<<<<<<<<<<<<< + * return cudaGetDeviceProperties_v2(prop, device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceProperties_v2(__pyx_v_prop, __pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 182, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":181 + * cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetDeviceProperties_v2(prop, device) + * return cudaGetDeviceProperties_v2(prop, device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":183 + * if usePTDS: + * return ptds._cudaGetDeviceProperties_v2(prop, device) + * return cudaGetDeviceProperties_v2(prop, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetDeviceProperties_v2(__pyx_v_prop, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":179 + * return cudaGetDeviceCount(count) + * + * cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetDeviceProperties_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":185 + * return cudaGetDeviceProperties_v2(prop, device) + * + * cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetAttribute(int *__pyx_v_value, enum cudaDeviceAttr __pyx_v_attr, int __pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":186 + * + * cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGetAttribute(value, attr, device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 186, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":187 + * cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetAttribute(value, attr, device) + * return cudaDeviceGetAttribute(value, attr, device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":188 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGetAttribute(value, attr, device) # <<<<<<<<<<<<<< + * return cudaDeviceGetAttribute(value, attr, device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetAttribute(__pyx_v_value, ((enum cudaDeviceAttr)__pyx_v_attr), __pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 188, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":187 + * cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetAttribute(value, attr, device) + * return cudaDeviceGetAttribute(value, attr, device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":189 + * if usePTDS: + * return ptds._cudaDeviceGetAttribute(value, attr, device) + * return cudaDeviceGetAttribute(value, attr, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetAttribute(__pyx_v_value, __pyx_v_attr, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":185 + * return cudaGetDeviceProperties_v2(prop, device) + * + * cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":191 + * return cudaDeviceGetAttribute(value, attr, device) + * + * cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetDefaultMemPool(cudaMemPool_t *__pyx_v_memPool, int __pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":192 + * + * cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGetDefaultMemPool(memPool, device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 192, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":193 + * cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetDefaultMemPool(memPool, device) + * return cudaDeviceGetDefaultMemPool(memPool, device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":194 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGetDefaultMemPool(memPool, device) # <<<<<<<<<<<<<< + * return cudaDeviceGetDefaultMemPool(memPool, device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetDefaultMemPool(__pyx_v_memPool, __pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 194, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":193 + * cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetDefaultMemPool(memPool, device) + * return cudaDeviceGetDefaultMemPool(memPool, device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":195 + * if usePTDS: + * return ptds._cudaDeviceGetDefaultMemPool(memPool, device) + * return cudaDeviceGetDefaultMemPool(memPool, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetDefaultMemPool(__pyx_v_memPool, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":191 + * return cudaDeviceGetAttribute(value, attr, device) + * + * cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGetDefaultMemPool", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":197 + * return cudaDeviceGetDefaultMemPool(memPool, device) + * + * cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceSetMemPool(int __pyx_v_device, cudaMemPool_t __pyx_v_memPool) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":198 + * + * cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceSetMemPool(device, memPool) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 198, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":199 + * cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceSetMemPool(device, memPool) + * return cudaDeviceSetMemPool(device, memPool) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":200 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceSetMemPool(device, memPool) # <<<<<<<<<<<<<< + * return cudaDeviceSetMemPool(device, memPool) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetMemPool(__pyx_v_device, __pyx_v_memPool); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 200, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":199 + * cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceSetMemPool(device, memPool) + * return cudaDeviceSetMemPool(device, memPool) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":201 + * if usePTDS: + * return ptds._cudaDeviceSetMemPool(device, memPool) + * return cudaDeviceSetMemPool(device, memPool) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceSetMemPool(__pyx_v_device, __pyx_v_memPool); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":197 + * return cudaDeviceGetDefaultMemPool(memPool, device) + * + * cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceSetMemPool", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":203 + * return cudaDeviceSetMemPool(device, memPool) + * + * cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetMemPool(cudaMemPool_t *__pyx_v_memPool, int __pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":204 + * + * cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGetMemPool(memPool, device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 204, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":205 + * cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetMemPool(memPool, device) + * return cudaDeviceGetMemPool(memPool, device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":206 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGetMemPool(memPool, device) # <<<<<<<<<<<<<< + * return cudaDeviceGetMemPool(memPool, device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetMemPool(__pyx_v_memPool, __pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 206, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":205 + * cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetMemPool(memPool, device) + * return cudaDeviceGetMemPool(memPool, device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":207 + * if usePTDS: + * return ptds._cudaDeviceGetMemPool(memPool, device) + * return cudaDeviceGetMemPool(memPool, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetMemPool(__pyx_v_memPool, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":203 + * return cudaDeviceSetMemPool(device, memPool) + * + * cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGetMemPool", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":209 + * return cudaDeviceGetMemPool(memPool, device) + * + * cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetNvSciSyncAttributes(void *__pyx_v_nvSciSyncAttrList, int __pyx_v_device, int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":210 + * + * cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 210, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":211 + * cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) + * return cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":212 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) # <<<<<<<<<<<<<< + * return cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetNvSciSyncAttributes(__pyx_v_nvSciSyncAttrList, __pyx_v_device, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 212, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":211 + * cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) + * return cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":213 + * if usePTDS: + * return ptds._cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) + * return cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetNvSciSyncAttributes(__pyx_v_nvSciSyncAttrList, __pyx_v_device, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":209 + * return cudaDeviceGetMemPool(memPool, device) + * + * cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGetNvSciSyncAttributes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":215 + * return cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) + * + * cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetP2PAttribute(int *__pyx_v_value, enum cudaDeviceP2PAttr __pyx_v_attr, int __pyx_v_srcDevice, int __pyx_v_dstDevice) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":216 + * + * cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 216, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":217 + * cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) + * return cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":218 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) # <<<<<<<<<<<<<< + * return cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetP2PAttribute(__pyx_v_value, ((enum cudaDeviceP2PAttr)__pyx_v_attr), __pyx_v_srcDevice, __pyx_v_dstDevice); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 218, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":217 + * cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) + * return cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":219 + * if usePTDS: + * return ptds._cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) + * return cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetP2PAttribute(__pyx_v_value, __pyx_v_attr, __pyx_v_srcDevice, __pyx_v_dstDevice); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":215 + * return cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) + * + * cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGetP2PAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":221 + * return cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) + * + * cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaChooseDevice(int *__pyx_v_device, struct cudaDeviceProp const *__pyx_v_prop) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":222 + * + * cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaChooseDevice(device, prop) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 222, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":223 + * cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaChooseDevice(device, prop) + * return cudaChooseDevice(device, prop) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":224 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaChooseDevice(device, prop) # <<<<<<<<<<<<<< + * return cudaChooseDevice(device, prop) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaChooseDevice(__pyx_v_device, __pyx_v_prop); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 224, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":223 + * cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaChooseDevice(device, prop) + * return cudaChooseDevice(device, prop) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":225 + * if usePTDS: + * return ptds._cudaChooseDevice(device, prop) + * return cudaChooseDevice(device, prop) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaChooseDevice(__pyx_v_device, __pyx_v_prop); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":221 + * return cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) + * + * cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaChooseDevice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":227 + * return cudaChooseDevice(device, prop) + * + * cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaInitDevice(int __pyx_v_device, unsigned int __pyx_v_deviceFlags, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":228 + * + * cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaInitDevice(device, deviceFlags, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 228, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":229 + * cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaInitDevice(device, deviceFlags, flags) + * return cudaInitDevice(device, deviceFlags, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":230 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaInitDevice(device, deviceFlags, flags) # <<<<<<<<<<<<<< + * return cudaInitDevice(device, deviceFlags, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaInitDevice(__pyx_v_device, __pyx_v_deviceFlags, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 230, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":229 + * cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaInitDevice(device, deviceFlags, flags) + * return cudaInitDevice(device, deviceFlags, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":231 + * if usePTDS: + * return ptds._cudaInitDevice(device, deviceFlags, flags) + * return cudaInitDevice(device, deviceFlags, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaInitDevice(__pyx_v_device, __pyx_v_deviceFlags, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":227 + * return cudaChooseDevice(device, prop) + * + * cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaInitDevice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":233 + * return cudaInitDevice(device, deviceFlags, flags) + * + * cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaSetDevice(int __pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":234 + * + * cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaSetDevice(device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 234, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":235 + * cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaSetDevice(device) + * return cudaSetDevice(device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":236 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaSetDevice(device) # <<<<<<<<<<<<<< + * return cudaSetDevice(device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSetDevice(__pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 236, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":235 + * cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaSetDevice(device) + * return cudaSetDevice(device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":237 + * if usePTDS: + * return ptds._cudaSetDevice(device) + * return cudaSetDevice(device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaSetDevice(__pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":233 + * return cudaInitDevice(device, deviceFlags, flags) + * + * cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaSetDevice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":239 + * return cudaSetDevice(device) + * + * cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetDevice(int *__pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":240 + * + * cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetDevice(device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 240, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":241 + * cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetDevice(device) + * return cudaGetDevice(device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":242 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetDevice(device) # <<<<<<<<<<<<<< + * return cudaGetDevice(device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDevice(__pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 242, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":241 + * cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetDevice(device) + * return cudaGetDevice(device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":243 + * if usePTDS: + * return ptds._cudaGetDevice(device) + * return cudaGetDevice(device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetDevice(__pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":239 + * return cudaSetDevice(device) + * + * cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetDevice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":245 + * return cudaGetDevice(device) + * + * cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaSetDeviceFlags(unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":246 + * + * cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaSetDeviceFlags(flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 246, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":247 + * cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaSetDeviceFlags(flags) + * return cudaSetDeviceFlags(flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":248 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaSetDeviceFlags(flags) # <<<<<<<<<<<<<< + * return cudaSetDeviceFlags(flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSetDeviceFlags(__pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 248, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":247 + * cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaSetDeviceFlags(flags) + * return cudaSetDeviceFlags(flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":249 + * if usePTDS: + * return ptds._cudaSetDeviceFlags(flags) + * return cudaSetDeviceFlags(flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaSetDeviceFlags(__pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":245 + * return cudaGetDevice(device) + * + * cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaSetDeviceFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":251 + * return cudaSetDeviceFlags(flags) + * + * cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetDeviceFlags(unsigned int *__pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":252 + * + * cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetDeviceFlags(flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 252, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":253 + * cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetDeviceFlags(flags) + * return cudaGetDeviceFlags(flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":254 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetDeviceFlags(flags) # <<<<<<<<<<<<<< + * return cudaGetDeviceFlags(flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceFlags(__pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 254, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":253 + * cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetDeviceFlags(flags) + * return cudaGetDeviceFlags(flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":255 + * if usePTDS: + * return ptds._cudaGetDeviceFlags(flags) + * return cudaGetDeviceFlags(flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetDeviceFlags(__pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":251 + * return cudaSetDeviceFlags(flags) + * + * cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetDeviceFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":257 + * return cudaGetDeviceFlags(flags) + * + * cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamCreate(cudaStream_t *__pyx_v_pStream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":258 + * + * cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamCreate(pStream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 258, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":259 + * cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamCreate(pStream) + * return cudaStreamCreate(pStream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":260 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamCreate(pStream) # <<<<<<<<<<<<<< + * return cudaStreamCreate(pStream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreate(__pyx_v_pStream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 260, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":259 + * cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamCreate(pStream) + * return cudaStreamCreate(pStream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":261 + * if usePTDS: + * return ptds._cudaStreamCreate(pStream) + * return cudaStreamCreate(pStream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamCreate(__pyx_v_pStream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":257 + * return cudaGetDeviceFlags(flags) + * + * cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":263 + * return cudaStreamCreate(pStream) + * + * cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamCreateWithFlags(cudaStream_t *__pyx_v_pStream, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":264 + * + * cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamCreateWithFlags(pStream, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 264, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":265 + * cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamCreateWithFlags(pStream, flags) + * return cudaStreamCreateWithFlags(pStream, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":266 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamCreateWithFlags(pStream, flags) # <<<<<<<<<<<<<< + * return cudaStreamCreateWithFlags(pStream, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreateWithFlags(__pyx_v_pStream, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 266, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":265 + * cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamCreateWithFlags(pStream, flags) + * return cudaStreamCreateWithFlags(pStream, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":267 + * if usePTDS: + * return ptds._cudaStreamCreateWithFlags(pStream, flags) + * return cudaStreamCreateWithFlags(pStream, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamCreateWithFlags(__pyx_v_pStream, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":263 + * return cudaStreamCreate(pStream) + * + * cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamCreateWithFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":269 + * return cudaStreamCreateWithFlags(pStream, flags) + * + * cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamCreateWithPriority(cudaStream_t *__pyx_v_pStream, unsigned int __pyx_v_flags, int __pyx_v_priority) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":270 + * + * cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamCreateWithPriority(pStream, flags, priority) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 270, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":271 + * cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamCreateWithPriority(pStream, flags, priority) + * return cudaStreamCreateWithPriority(pStream, flags, priority) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":272 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamCreateWithPriority(pStream, flags, priority) # <<<<<<<<<<<<<< + * return cudaStreamCreateWithPriority(pStream, flags, priority) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreateWithPriority(__pyx_v_pStream, __pyx_v_flags, __pyx_v_priority); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 272, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":271 + * cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamCreateWithPriority(pStream, flags, priority) + * return cudaStreamCreateWithPriority(pStream, flags, priority) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":273 + * if usePTDS: + * return ptds._cudaStreamCreateWithPriority(pStream, flags, priority) + * return cudaStreamCreateWithPriority(pStream, flags, priority) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamCreateWithPriority(__pyx_v_pStream, __pyx_v_flags, __pyx_v_priority); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":269 + * return cudaStreamCreateWithFlags(pStream, flags) + * + * cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamCreateWithPriority", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":275 + * return cudaStreamCreateWithPriority(pStream, flags, priority) + * + * cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetPriority(cudaStream_t __pyx_v_hStream, int *__pyx_v_priority) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":276 + * + * cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamGetPriority(hStream, priority) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 276, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":277 + * cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetPriority(hStream, priority) + * return cudaStreamGetPriority(hStream, priority) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":278 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamGetPriority(hStream, priority) # <<<<<<<<<<<<<< + * return cudaStreamGetPriority(hStream, priority) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetPriority(__pyx_v_hStream, __pyx_v_priority); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 278, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":277 + * cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetPriority(hStream, priority) + * return cudaStreamGetPriority(hStream, priority) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":279 + * if usePTDS: + * return ptds._cudaStreamGetPriority(hStream, priority) + * return cudaStreamGetPriority(hStream, priority) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetPriority(__pyx_v_hStream, __pyx_v_priority); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":275 + * return cudaStreamCreateWithPriority(pStream, flags, priority) + * + * cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamGetPriority", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":281 + * return cudaStreamGetPriority(hStream, priority) + * + * cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetFlags(cudaStream_t __pyx_v_hStream, unsigned int *__pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":282 + * + * cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamGetFlags(hStream, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 282, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":283 + * cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetFlags(hStream, flags) + * return cudaStreamGetFlags(hStream, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":284 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamGetFlags(hStream, flags) # <<<<<<<<<<<<<< + * return cudaStreamGetFlags(hStream, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetFlags(__pyx_v_hStream, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 284, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":283 + * cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetFlags(hStream, flags) + * return cudaStreamGetFlags(hStream, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":285 + * if usePTDS: + * return ptds._cudaStreamGetFlags(hStream, flags) + * return cudaStreamGetFlags(hStream, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetFlags(__pyx_v_hStream, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":281 + * return cudaStreamGetPriority(hStream, priority) + * + * cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamGetFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":287 + * return cudaStreamGetFlags(hStream, flags) + * + * cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetId(cudaStream_t __pyx_v_hStream, unsigned PY_LONG_LONG *__pyx_v_streamId) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":288 + * + * cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamGetId(hStream, streamId) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 288, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":289 + * cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetId(hStream, streamId) + * return cudaStreamGetId(hStream, streamId) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":290 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamGetId(hStream, streamId) # <<<<<<<<<<<<<< + * return cudaStreamGetId(hStream, streamId) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetId(__pyx_v_hStream, __pyx_v_streamId); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 290, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":289 + * cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetId(hStream, streamId) + * return cudaStreamGetId(hStream, streamId) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":291 + * if usePTDS: + * return ptds._cudaStreamGetId(hStream, streamId) + * return cudaStreamGetId(hStream, streamId) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetId(__pyx_v_hStream, __pyx_v_streamId); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":287 + * return cudaStreamGetFlags(hStream, flags) + * + * cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamGetId", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":293 + * return cudaStreamGetId(hStream, streamId) + * + * cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetDevice(cudaStream_t __pyx_v_hStream, int *__pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":294 + * + * cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamGetDevice(hStream, device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 294, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":295 + * cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetDevice(hStream, device) + * return cudaStreamGetDevice(hStream, device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":296 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamGetDevice(hStream, device) # <<<<<<<<<<<<<< + * return cudaStreamGetDevice(hStream, device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetDevice(__pyx_v_hStream, __pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 296, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":295 + * cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetDevice(hStream, device) + * return cudaStreamGetDevice(hStream, device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":297 + * if usePTDS: + * return ptds._cudaStreamGetDevice(hStream, device) + * return cudaStreamGetDevice(hStream, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetDevice(__pyx_v_hStream, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":293 + * return cudaStreamGetId(hStream, streamId) + * + * cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamGetDevice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":299 + * return cudaStreamGetDevice(hStream, device) + * + * cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaCtxResetPersistingL2Cache(void) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":300 + * + * cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaCtxResetPersistingL2Cache() +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 300, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":301 + * cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaCtxResetPersistingL2Cache() + * return cudaCtxResetPersistingL2Cache() +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":302 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaCtxResetPersistingL2Cache() # <<<<<<<<<<<<<< + * return cudaCtxResetPersistingL2Cache() + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCtxResetPersistingL2Cache(); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 302, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":301 + * cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaCtxResetPersistingL2Cache() + * return cudaCtxResetPersistingL2Cache() +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":303 + * if usePTDS: + * return ptds._cudaCtxResetPersistingL2Cache() + * return cudaCtxResetPersistingL2Cache() # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaCtxResetPersistingL2Cache(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":299 + * return cudaStreamGetDevice(hStream, device) + * + * cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaCtxResetPersistingL2Cache", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":305 + * return cudaCtxResetPersistingL2Cache() + * + * cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamCopyAttributes(cudaStream_t __pyx_v_dst, cudaStream_t __pyx_v_src) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":306 + * + * cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamCopyAttributes(dst, src) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 306, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":307 + * cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamCopyAttributes(dst, src) + * return cudaStreamCopyAttributes(dst, src) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":308 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamCopyAttributes(dst, src) # <<<<<<<<<<<<<< + * return cudaStreamCopyAttributes(dst, src) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCopyAttributes(__pyx_v_dst, __pyx_v_src); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 308, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":307 + * cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamCopyAttributes(dst, src) + * return cudaStreamCopyAttributes(dst, src) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":309 + * if usePTDS: + * return ptds._cudaStreamCopyAttributes(dst, src) + * return cudaStreamCopyAttributes(dst, src) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamCopyAttributes(__pyx_v_dst, __pyx_v_src); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":305 + * return cudaCtxResetPersistingL2Cache() + * + * cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamCopyAttributes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":311 + * return cudaStreamCopyAttributes(dst, src) + * + * cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetAttribute(cudaStream_t __pyx_v_hStream, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaStreamAttrID __pyx_v_attr, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaStreamAttrValue *__pyx_v_value_out) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":312 + * + * cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamGetAttribute(hStream, attr, value_out) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 312, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":313 + * cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetAttribute(hStream, attr, value_out) + * return cudaStreamGetAttribute(hStream, attr, value_out) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":314 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamGetAttribute(hStream, attr, value_out) # <<<<<<<<<<<<<< + * return cudaStreamGetAttribute(hStream, attr, value_out) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetAttribute(__pyx_v_hStream, ((__pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrID)__pyx_v_attr), __pyx_v_value_out); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 314, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":313 + * cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetAttribute(hStream, attr, value_out) + * return cudaStreamGetAttribute(hStream, attr, value_out) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":315 + * if usePTDS: + * return ptds._cudaStreamGetAttribute(hStream, attr, value_out) + * return cudaStreamGetAttribute(hStream, attr, value_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetAttribute(__pyx_v_hStream, __pyx_v_attr, __pyx_v_value_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":311 + * return cudaStreamCopyAttributes(dst, src) + * + * cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":317 + * return cudaStreamGetAttribute(hStream, attr, value_out) + * + * cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamSetAttribute(cudaStream_t __pyx_v_hStream, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaStreamAttrID __pyx_v_attr, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaStreamAttrValue const *__pyx_v_value) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":318 + * + * cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamSetAttribute(hStream, attr, value) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 318, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":319 + * cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamSetAttribute(hStream, attr, value) + * return cudaStreamSetAttribute(hStream, attr, value) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":320 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamSetAttribute(hStream, attr, value) # <<<<<<<<<<<<<< + * return cudaStreamSetAttribute(hStream, attr, value) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamSetAttribute(__pyx_v_hStream, ((__pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrID)__pyx_v_attr), __pyx_v_value); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 320, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":319 + * cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamSetAttribute(hStream, attr, value) + * return cudaStreamSetAttribute(hStream, attr, value) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":321 + * if usePTDS: + * return ptds._cudaStreamSetAttribute(hStream, attr, value) + * return cudaStreamSetAttribute(hStream, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamSetAttribute(__pyx_v_hStream, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":317 + * return cudaStreamGetAttribute(hStream, attr, value_out) + * + * cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamSetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":323 + * return cudaStreamSetAttribute(hStream, attr, value) + * + * cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamDestroy(cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":324 + * + * cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamDestroy(stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 324, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":325 + * cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamDestroy(stream) + * return cudaStreamDestroy(stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":326 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamDestroy(stream) # <<<<<<<<<<<<<< + * return cudaStreamDestroy(stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamDestroy(__pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 326, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":325 + * cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamDestroy(stream) + * return cudaStreamDestroy(stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":327 + * if usePTDS: + * return ptds._cudaStreamDestroy(stream) + * return cudaStreamDestroy(stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamDestroy(__pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":323 + * return cudaStreamSetAttribute(hStream, attr, value) + * + * cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":329 + * return cudaStreamDestroy(stream) + * + * cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamWaitEvent(cudaStream_t __pyx_v_stream, cudaEvent_t __pyx_v_event, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":330 + * + * cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamWaitEvent(stream, event, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 330, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":331 + * cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamWaitEvent(stream, event, flags) + * return cudaStreamWaitEvent(stream, event, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":332 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamWaitEvent(stream, event, flags) # <<<<<<<<<<<<<< + * return cudaStreamWaitEvent(stream, event, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamWaitEvent(__pyx_v_stream, __pyx_v_event, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 332, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":331 + * cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamWaitEvent(stream, event, flags) + * return cudaStreamWaitEvent(stream, event, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":333 + * if usePTDS: + * return ptds._cudaStreamWaitEvent(stream, event, flags) + * return cudaStreamWaitEvent(stream, event, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamWaitEvent(__pyx_v_stream, __pyx_v_event, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":329 + * return cudaStreamDestroy(stream) + * + * cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamWaitEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":335 + * return cudaStreamWaitEvent(stream, event, flags) + * + * cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamAddCallback(cudaStream_t __pyx_v_stream, cudaStreamCallback_t __pyx_v_callback, void *__pyx_v_userData, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":336 + * + * cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamAddCallback(stream, callback, userData, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 336, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":337 + * cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamAddCallback(stream, callback, userData, flags) + * return cudaStreamAddCallback(stream, callback, userData, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":338 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamAddCallback(stream, callback, userData, flags) # <<<<<<<<<<<<<< + * return cudaStreamAddCallback(stream, callback, userData, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamAddCallback(__pyx_v_stream, ((cudaStreamCallback_t)__pyx_v_callback), __pyx_v_userData, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 338, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":337 + * cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamAddCallback(stream, callback, userData, flags) + * return cudaStreamAddCallback(stream, callback, userData, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":339 + * if usePTDS: + * return ptds._cudaStreamAddCallback(stream, callback, userData, flags) + * return cudaStreamAddCallback(stream, callback, userData, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamAddCallback(__pyx_v_stream, __pyx_v_callback, __pyx_v_userData, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":335 + * return cudaStreamWaitEvent(stream, event, flags) + * + * cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamAddCallback", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":341 + * return cudaStreamAddCallback(stream, callback, userData, flags) + * + * cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamSynchronize(cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":342 + * + * cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamSynchronize(stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 342, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":343 + * cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamSynchronize(stream) + * return cudaStreamSynchronize(stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":344 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamSynchronize(stream) # <<<<<<<<<<<<<< + * return cudaStreamSynchronize(stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamSynchronize(__pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 344, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":343 + * cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamSynchronize(stream) + * return cudaStreamSynchronize(stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":345 + * if usePTDS: + * return ptds._cudaStreamSynchronize(stream) + * return cudaStreamSynchronize(stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamSynchronize(__pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":341 + * return cudaStreamAddCallback(stream, callback, userData, flags) + * + * cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamSynchronize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":347 + * return cudaStreamSynchronize(stream) + * + * cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamQuery(cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":348 + * + * cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamQuery(stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 348, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":349 + * cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamQuery(stream) + * return cudaStreamQuery(stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":350 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamQuery(stream) # <<<<<<<<<<<<<< + * return cudaStreamQuery(stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamQuery(__pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 350, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":349 + * cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamQuery(stream) + * return cudaStreamQuery(stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":351 + * if usePTDS: + * return ptds._cudaStreamQuery(stream) + * return cudaStreamQuery(stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamQuery(__pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":347 + * return cudaStreamSynchronize(stream) + * + * cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamQuery", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":353 + * return cudaStreamQuery(stream) + * + * cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamAttachMemAsync(cudaStream_t __pyx_v_stream, void *__pyx_v_devPtr, size_t __pyx_v_length, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":354 + * + * cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamAttachMemAsync(stream, devPtr, length, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 354, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":355 + * cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamAttachMemAsync(stream, devPtr, length, flags) + * return cudaStreamAttachMemAsync(stream, devPtr, length, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":356 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamAttachMemAsync(stream, devPtr, length, flags) # <<<<<<<<<<<<<< + * return cudaStreamAttachMemAsync(stream, devPtr, length, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamAttachMemAsync(__pyx_v_stream, __pyx_v_devPtr, __pyx_v_length, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 356, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":355 + * cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamAttachMemAsync(stream, devPtr, length, flags) + * return cudaStreamAttachMemAsync(stream, devPtr, length, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":357 + * if usePTDS: + * return ptds._cudaStreamAttachMemAsync(stream, devPtr, length, flags) + * return cudaStreamAttachMemAsync(stream, devPtr, length, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamAttachMemAsync(__pyx_v_stream, __pyx_v_devPtr, __pyx_v_length, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":353 + * return cudaStreamQuery(stream) + * + * cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamAttachMemAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":359 + * return cudaStreamAttachMemAsync(stream, devPtr, length, flags) + * + * cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamBeginCapture(cudaStream_t __pyx_v_stream, enum cudaStreamCaptureMode __pyx_v_mode) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":360 + * + * cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamBeginCapture(stream, mode) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 360, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":361 + * cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamBeginCapture(stream, mode) + * return cudaStreamBeginCapture(stream, mode) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":362 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamBeginCapture(stream, mode) # <<<<<<<<<<<<<< + * return cudaStreamBeginCapture(stream, mode) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamBeginCapture(__pyx_v_stream, ((enum cudaStreamCaptureMode)__pyx_v_mode)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 362, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":361 + * cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamBeginCapture(stream, mode) + * return cudaStreamBeginCapture(stream, mode) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":363 + * if usePTDS: + * return ptds._cudaStreamBeginCapture(stream, mode) + * return cudaStreamBeginCapture(stream, mode) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamBeginCapture(__pyx_v_stream, __pyx_v_mode); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":359 + * return cudaStreamAttachMemAsync(stream, devPtr, length, flags) + * + * cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamBeginCapture", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":365 + * return cudaStreamBeginCapture(stream, mode) + * + * cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamBeginCaptureToGraph(cudaStream_t __pyx_v_stream, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_dependencies, cudaGraphEdgeData const *__pyx_v_dependencyData, size_t __pyx_v_numDependencies, enum cudaStreamCaptureMode __pyx_v_mode) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":366 + * + * cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 366, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":367 + * cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) + * return cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":368 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) # <<<<<<<<<<<<<< + * return cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamBeginCaptureToGraph(__pyx_v_stream, __pyx_v_graph, __pyx_v_dependencies, __pyx_v_dependencyData, __pyx_v_numDependencies, ((enum cudaStreamCaptureMode)__pyx_v_mode)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 368, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":367 + * cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) + * return cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":369 + * if usePTDS: + * return ptds._cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) + * return cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamBeginCaptureToGraph(__pyx_v_stream, __pyx_v_graph, __pyx_v_dependencies, __pyx_v_dependencyData, __pyx_v_numDependencies, __pyx_v_mode); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":365 + * return cudaStreamBeginCapture(stream, mode) + * + * cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamBeginCaptureToGraph", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":371 + * return cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) + * + * cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaThreadExchangeStreamCaptureMode(enum cudaStreamCaptureMode *__pyx_v_mode) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":372 + * + * cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaThreadExchangeStreamCaptureMode(mode) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 372, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":373 + * cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaThreadExchangeStreamCaptureMode(mode) + * return cudaThreadExchangeStreamCaptureMode(mode) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":374 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaThreadExchangeStreamCaptureMode(mode) # <<<<<<<<<<<<<< + * return cudaThreadExchangeStreamCaptureMode(mode) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaThreadExchangeStreamCaptureMode(((enum cudaStreamCaptureMode *)__pyx_v_mode)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 374, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":373 + * cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaThreadExchangeStreamCaptureMode(mode) + * return cudaThreadExchangeStreamCaptureMode(mode) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":375 + * if usePTDS: + * return ptds._cudaThreadExchangeStreamCaptureMode(mode) + * return cudaThreadExchangeStreamCaptureMode(mode) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaThreadExchangeStreamCaptureMode(__pyx_v_mode); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":371 + * return cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) + * + * cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaThreadExchangeStreamCaptureMode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":377 + * return cudaThreadExchangeStreamCaptureMode(mode) + * + * cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamEndCapture(cudaStream_t __pyx_v_stream, cudaGraph_t *__pyx_v_pGraph) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":378 + * + * cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamEndCapture(stream, pGraph) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 378, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":379 + * cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamEndCapture(stream, pGraph) + * return cudaStreamEndCapture(stream, pGraph) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":380 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamEndCapture(stream, pGraph) # <<<<<<<<<<<<<< + * return cudaStreamEndCapture(stream, pGraph) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamEndCapture(__pyx_v_stream, __pyx_v_pGraph); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 380, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":379 + * cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamEndCapture(stream, pGraph) + * return cudaStreamEndCapture(stream, pGraph) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":381 + * if usePTDS: + * return ptds._cudaStreamEndCapture(stream, pGraph) + * return cudaStreamEndCapture(stream, pGraph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamEndCapture(__pyx_v_stream, __pyx_v_pGraph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":377 + * return cudaThreadExchangeStreamCaptureMode(mode) + * + * cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamEndCapture", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":383 + * return cudaStreamEndCapture(stream, pGraph) + * + * cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamIsCapturing(cudaStream_t __pyx_v_stream, enum cudaStreamCaptureStatus *__pyx_v_pCaptureStatus) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":384 + * + * cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamIsCapturing(stream, pCaptureStatus) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 384, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":385 + * cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamIsCapturing(stream, pCaptureStatus) + * return cudaStreamIsCapturing(stream, pCaptureStatus) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":386 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamIsCapturing(stream, pCaptureStatus) # <<<<<<<<<<<<<< + * return cudaStreamIsCapturing(stream, pCaptureStatus) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamIsCapturing(__pyx_v_stream, ((enum cudaStreamCaptureStatus *)__pyx_v_pCaptureStatus)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 386, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":385 + * cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamIsCapturing(stream, pCaptureStatus) + * return cudaStreamIsCapturing(stream, pCaptureStatus) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":387 + * if usePTDS: + * return ptds._cudaStreamIsCapturing(stream, pCaptureStatus) + * return cudaStreamIsCapturing(stream, pCaptureStatus) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamIsCapturing(__pyx_v_stream, __pyx_v_pCaptureStatus); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":383 + * return cudaStreamEndCapture(stream, pGraph) + * + * cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamIsCapturing", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":389 + * return cudaStreamIsCapturing(stream, pCaptureStatus) + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetCaptureInfo_v2(cudaStream_t __pyx_v_stream, enum cudaStreamCaptureStatus *__pyx_v_captureStatus_out, unsigned PY_LONG_LONG *__pyx_v_id_out, cudaGraph_t *__pyx_v_graph_out, cudaGraphNode_t const **__pyx_v_dependencies_out, size_t *__pyx_v_numDependencies_out) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":390 + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamGetCaptureInfo_v2(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 390, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":391 + * cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetCaptureInfo_v2(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + * return cudaStreamGetCaptureInfo_v2(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":392 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamGetCaptureInfo_v2(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) # <<<<<<<<<<<<<< + * return cudaStreamGetCaptureInfo_v2(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetCaptureInfo_v2(__pyx_v_stream, ((enum cudaStreamCaptureStatus *)__pyx_v_captureStatus_out), __pyx_v_id_out, __pyx_v_graph_out, __pyx_v_dependencies_out, __pyx_v_numDependencies_out); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 392, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":391 + * cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetCaptureInfo_v2(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + * return cudaStreamGetCaptureInfo_v2(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":393 + * if usePTDS: + * return ptds._cudaStreamGetCaptureInfo_v2(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + * return cudaStreamGetCaptureInfo_v2(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetCaptureInfo_v2(__pyx_v_stream, __pyx_v_captureStatus_out, __pyx_v_id_out, __pyx_v_graph_out, __pyx_v_dependencies_out, __pyx_v_numDependencies_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":389 + * return cudaStreamIsCapturing(stream, pCaptureStatus) + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamGetCaptureInfo_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":395 + * return cudaStreamGetCaptureInfo_v2(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetCaptureInfo_v3(cudaStream_t __pyx_v_stream, enum cudaStreamCaptureStatus *__pyx_v_captureStatus_out, unsigned PY_LONG_LONG *__pyx_v_id_out, cudaGraph_t *__pyx_v_graph_out, cudaGraphNode_t const **__pyx_v_dependencies_out, cudaGraphEdgeData const **__pyx_v_edgeData_out, size_t *__pyx_v_numDependencies_out) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":396 + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 396, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":397 + * cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + * return cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":398 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) # <<<<<<<<<<<<<< + * return cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetCaptureInfo_v3(__pyx_v_stream, ((enum cudaStreamCaptureStatus *)__pyx_v_captureStatus_out), __pyx_v_id_out, __pyx_v_graph_out, __pyx_v_dependencies_out, __pyx_v_edgeData_out, __pyx_v_numDependencies_out); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 398, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":397 + * cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + * return cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":399 + * if usePTDS: + * return ptds._cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + * return cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetCaptureInfo_v3(__pyx_v_stream, __pyx_v_captureStatus_out, __pyx_v_id_out, __pyx_v_graph_out, __pyx_v_dependencies_out, __pyx_v_edgeData_out, __pyx_v_numDependencies_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":395 + * return cudaStreamGetCaptureInfo_v2(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamGetCaptureInfo_v3", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":401 + * return cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamUpdateCaptureDependencies(cudaStream_t __pyx_v_stream, cudaGraphNode_t *__pyx_v_dependencies, size_t __pyx_v_numDependencies, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":402 + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 402, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":403 + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) + * return cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":404 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) # <<<<<<<<<<<<<< + * return cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamUpdateCaptureDependencies(__pyx_v_stream, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 404, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":403 + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) + * return cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":405 + * if usePTDS: + * return ptds._cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) + * return cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamUpdateCaptureDependencies(__pyx_v_stream, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":401 + * return cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamUpdateCaptureDependencies", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":407 + * return cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamUpdateCaptureDependencies_v2(cudaStream_t __pyx_v_stream, cudaGraphNode_t *__pyx_v_dependencies, cudaGraphEdgeData const *__pyx_v_dependencyData, size_t __pyx_v_numDependencies, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":408 + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 408, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":409 + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) + * return cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":410 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) # <<<<<<<<<<<<<< + * return cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamUpdateCaptureDependencies_v2(__pyx_v_stream, __pyx_v_dependencies, __pyx_v_dependencyData, __pyx_v_numDependencies, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 410, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":409 + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) + * return cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":411 + * if usePTDS: + * return ptds._cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) + * return cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamUpdateCaptureDependencies_v2(__pyx_v_stream, __pyx_v_dependencies, __pyx_v_dependencyData, __pyx_v_numDependencies, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":407 + * return cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaStreamUpdateCaptureDependencies_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":413 + * return cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) + * + * cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventCreate(cudaEvent_t *__pyx_v_event) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":414 + * + * cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaEventCreate(event) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 414, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":415 + * cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventCreate(event) + * return cudaEventCreate(event) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":416 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaEventCreate(event) # <<<<<<<<<<<<<< + * return cudaEventCreate(event) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventCreate(__pyx_v_event); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 416, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":415 + * cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventCreate(event) + * return cudaEventCreate(event) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":417 + * if usePTDS: + * return ptds._cudaEventCreate(event) + * return cudaEventCreate(event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventCreate(__pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":413 + * return cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) + * + * cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEventCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":419 + * return cudaEventCreate(event) + * + * cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventCreateWithFlags(cudaEvent_t *__pyx_v_event, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":420 + * + * cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaEventCreateWithFlags(event, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 420, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":421 + * cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventCreateWithFlags(event, flags) + * return cudaEventCreateWithFlags(event, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":422 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaEventCreateWithFlags(event, flags) # <<<<<<<<<<<<<< + * return cudaEventCreateWithFlags(event, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventCreateWithFlags(__pyx_v_event, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 422, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":421 + * cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventCreateWithFlags(event, flags) + * return cudaEventCreateWithFlags(event, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":423 + * if usePTDS: + * return ptds._cudaEventCreateWithFlags(event, flags) + * return cudaEventCreateWithFlags(event, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventCreateWithFlags(__pyx_v_event, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":419 + * return cudaEventCreate(event) + * + * cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEventCreateWithFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":425 + * return cudaEventCreateWithFlags(event, flags) + * + * cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventRecord(cudaEvent_t __pyx_v_event, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":426 + * + * cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaEventRecord(event, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 426, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":427 + * cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventRecord(event, stream) + * return cudaEventRecord(event, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":428 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaEventRecord(event, stream) # <<<<<<<<<<<<<< + * return cudaEventRecord(event, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventRecord(__pyx_v_event, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 428, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":427 + * cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventRecord(event, stream) + * return cudaEventRecord(event, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":429 + * if usePTDS: + * return ptds._cudaEventRecord(event, stream) + * return cudaEventRecord(event, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventRecord(__pyx_v_event, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":425 + * return cudaEventCreateWithFlags(event, flags) + * + * cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEventRecord", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":431 + * return cudaEventRecord(event, stream) + * + * cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventRecordWithFlags(cudaEvent_t __pyx_v_event, cudaStream_t __pyx_v_stream, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":432 + * + * cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaEventRecordWithFlags(event, stream, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 432, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":433 + * cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventRecordWithFlags(event, stream, flags) + * return cudaEventRecordWithFlags(event, stream, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":434 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaEventRecordWithFlags(event, stream, flags) # <<<<<<<<<<<<<< + * return cudaEventRecordWithFlags(event, stream, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventRecordWithFlags(__pyx_v_event, __pyx_v_stream, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 434, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":433 + * cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventRecordWithFlags(event, stream, flags) + * return cudaEventRecordWithFlags(event, stream, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":435 + * if usePTDS: + * return ptds._cudaEventRecordWithFlags(event, stream, flags) + * return cudaEventRecordWithFlags(event, stream, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventRecordWithFlags(__pyx_v_event, __pyx_v_stream, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":431 + * return cudaEventRecord(event, stream) + * + * cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEventRecordWithFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":437 + * return cudaEventRecordWithFlags(event, stream, flags) + * + * cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventQuery(cudaEvent_t __pyx_v_event) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":438 + * + * cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaEventQuery(event) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 438, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":439 + * cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventQuery(event) + * return cudaEventQuery(event) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":440 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaEventQuery(event) # <<<<<<<<<<<<<< + * return cudaEventQuery(event) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventQuery(__pyx_v_event); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 440, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":439 + * cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventQuery(event) + * return cudaEventQuery(event) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":441 + * if usePTDS: + * return ptds._cudaEventQuery(event) + * return cudaEventQuery(event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventQuery(__pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":437 + * return cudaEventRecordWithFlags(event, stream, flags) + * + * cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEventQuery", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":443 + * return cudaEventQuery(event) + * + * cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventSynchronize(cudaEvent_t __pyx_v_event) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":444 + * + * cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaEventSynchronize(event) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 444, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":445 + * cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventSynchronize(event) + * return cudaEventSynchronize(event) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":446 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaEventSynchronize(event) # <<<<<<<<<<<<<< + * return cudaEventSynchronize(event) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventSynchronize(__pyx_v_event); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 446, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":445 + * cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventSynchronize(event) + * return cudaEventSynchronize(event) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":447 + * if usePTDS: + * return ptds._cudaEventSynchronize(event) + * return cudaEventSynchronize(event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventSynchronize(__pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":443 + * return cudaEventQuery(event) + * + * cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEventSynchronize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":449 + * return cudaEventSynchronize(event) + * + * cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventDestroy(cudaEvent_t __pyx_v_event) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":450 + * + * cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaEventDestroy(event) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 450, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":451 + * cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventDestroy(event) + * return cudaEventDestroy(event) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":452 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaEventDestroy(event) # <<<<<<<<<<<<<< + * return cudaEventDestroy(event) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventDestroy(__pyx_v_event); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 452, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":451 + * cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventDestroy(event) + * return cudaEventDestroy(event) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":453 + * if usePTDS: + * return ptds._cudaEventDestroy(event) + * return cudaEventDestroy(event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventDestroy(__pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":449 + * return cudaEventSynchronize(event) + * + * cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEventDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":455 + * return cudaEventDestroy(event) + * + * cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventElapsedTime(float *__pyx_v_ms, cudaEvent_t __pyx_v_start, cudaEvent_t __pyx_v_end) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":456 + * + * cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaEventElapsedTime(ms, start, end) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 456, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":457 + * cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventElapsedTime(ms, start, end) + * return cudaEventElapsedTime(ms, start, end) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":458 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaEventElapsedTime(ms, start, end) # <<<<<<<<<<<<<< + * return cudaEventElapsedTime(ms, start, end) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventElapsedTime(__pyx_v_ms, __pyx_v_start, __pyx_v_end); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 458, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":457 + * cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventElapsedTime(ms, start, end) + * return cudaEventElapsedTime(ms, start, end) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":459 + * if usePTDS: + * return ptds._cudaEventElapsedTime(ms, start, end) + * return cudaEventElapsedTime(ms, start, end) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventElapsedTime(__pyx_v_ms, __pyx_v_start, __pyx_v_end); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":455 + * return cudaEventDestroy(event) + * + * cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEventElapsedTime", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":461 + * return cudaEventElapsedTime(ms, start, end) + * + * cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventElapsedTime_v2(float *__pyx_v_ms, cudaEvent_t __pyx_v_start, cudaEvent_t __pyx_v_end) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":462 + * + * cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaEventElapsedTime_v2(ms, start, end) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 462, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":463 + * cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventElapsedTime_v2(ms, start, end) + * return cudaEventElapsedTime_v2(ms, start, end) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":464 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaEventElapsedTime_v2(ms, start, end) # <<<<<<<<<<<<<< + * return cudaEventElapsedTime_v2(ms, start, end) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventElapsedTime_v2(__pyx_v_ms, __pyx_v_start, __pyx_v_end); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 464, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":463 + * cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaEventElapsedTime_v2(ms, start, end) + * return cudaEventElapsedTime_v2(ms, start, end) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":465 + * if usePTDS: + * return ptds._cudaEventElapsedTime_v2(ms, start, end) + * return cudaEventElapsedTime_v2(ms, start, end) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventElapsedTime_v2(__pyx_v_ms, __pyx_v_start, __pyx_v_end); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":461 + * return cudaEventElapsedTime(ms, start, end) + * + * cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEventElapsedTime_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":467 + * return cudaEventElapsedTime_v2(ms, start, end) + * + * cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaImportExternalMemory(cudaExternalMemory_t *__pyx_v_extMem_out, struct cudaExternalMemoryHandleDesc const *__pyx_v_memHandleDesc) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":468 + * + * cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaImportExternalMemory(extMem_out, memHandleDesc) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 468, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":469 + * cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaImportExternalMemory(extMem_out, memHandleDesc) + * return cudaImportExternalMemory(extMem_out, memHandleDesc) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":470 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaImportExternalMemory(extMem_out, memHandleDesc) # <<<<<<<<<<<<<< + * return cudaImportExternalMemory(extMem_out, memHandleDesc) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaImportExternalMemory(__pyx_v_extMem_out, __pyx_v_memHandleDesc); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 470, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":469 + * cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaImportExternalMemory(extMem_out, memHandleDesc) + * return cudaImportExternalMemory(extMem_out, memHandleDesc) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":471 + * if usePTDS: + * return ptds._cudaImportExternalMemory(extMem_out, memHandleDesc) + * return cudaImportExternalMemory(extMem_out, memHandleDesc) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaImportExternalMemory(__pyx_v_extMem_out, __pyx_v_memHandleDesc); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":467 + * return cudaEventElapsedTime_v2(ms, start, end) + * + * cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaImportExternalMemory", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":473 + * return cudaImportExternalMemory(extMem_out, memHandleDesc) + * + * cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaExternalMemoryGetMappedBuffer(void **__pyx_v_devPtr, cudaExternalMemory_t __pyx_v_extMem, struct cudaExternalMemoryBufferDesc const *__pyx_v_bufferDesc) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":474 + * + * cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 474, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":475 + * cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) + * return cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":476 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) # <<<<<<<<<<<<<< + * return cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaExternalMemoryGetMappedBuffer(__pyx_v_devPtr, __pyx_v_extMem, __pyx_v_bufferDesc); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 476, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":475 + * cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) + * return cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":477 + * if usePTDS: + * return ptds._cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) + * return cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaExternalMemoryGetMappedBuffer(__pyx_v_devPtr, __pyx_v_extMem, __pyx_v_bufferDesc); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":473 + * return cudaImportExternalMemory(extMem_out, memHandleDesc) + * + * cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaExternalMemoryGetMappedBuffer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":479 + * return cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) + * + * cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t *__pyx_v_mipmap, cudaExternalMemory_t __pyx_v_extMem, struct cudaExternalMemoryMipmappedArrayDesc const *__pyx_v_mipmapDesc) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":480 + * + * cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 480, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":481 + * cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) + * return cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":482 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) # <<<<<<<<<<<<<< + * return cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaExternalMemoryGetMappedMipmappedArray(__pyx_v_mipmap, __pyx_v_extMem, __pyx_v_mipmapDesc); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 482, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":481 + * cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) + * return cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":483 + * if usePTDS: + * return ptds._cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) + * return cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaExternalMemoryGetMappedMipmappedArray(__pyx_v_mipmap, __pyx_v_extMem, __pyx_v_mipmapDesc); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":479 + * return cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) + * + * cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaExternalMemoryGetMappedMipmappedArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":485 + * return cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) + * + * cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDestroyExternalMemory(cudaExternalMemory_t __pyx_v_extMem) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":486 + * + * cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDestroyExternalMemory(extMem) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 486, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":487 + * cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDestroyExternalMemory(extMem) + * return cudaDestroyExternalMemory(extMem) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":488 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDestroyExternalMemory(extMem) # <<<<<<<<<<<<<< + * return cudaDestroyExternalMemory(extMem) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyExternalMemory(__pyx_v_extMem); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 488, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":487 + * cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDestroyExternalMemory(extMem) + * return cudaDestroyExternalMemory(extMem) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":489 + * if usePTDS: + * return ptds._cudaDestroyExternalMemory(extMem) + * return cudaDestroyExternalMemory(extMem) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDestroyExternalMemory(__pyx_v_extMem); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":485 + * return cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) + * + * cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDestroyExternalMemory", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":491 + * return cudaDestroyExternalMemory(extMem) + * + * cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaImportExternalSemaphore(cudaExternalSemaphore_t *__pyx_v_extSem_out, struct cudaExternalSemaphoreHandleDesc const *__pyx_v_semHandleDesc) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":492 + * + * cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaImportExternalSemaphore(extSem_out, semHandleDesc) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 492, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":493 + * cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaImportExternalSemaphore(extSem_out, semHandleDesc) + * return cudaImportExternalSemaphore(extSem_out, semHandleDesc) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":494 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaImportExternalSemaphore(extSem_out, semHandleDesc) # <<<<<<<<<<<<<< + * return cudaImportExternalSemaphore(extSem_out, semHandleDesc) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaImportExternalSemaphore(__pyx_v_extSem_out, __pyx_v_semHandleDesc); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 494, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":493 + * cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaImportExternalSemaphore(extSem_out, semHandleDesc) + * return cudaImportExternalSemaphore(extSem_out, semHandleDesc) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":495 + * if usePTDS: + * return ptds._cudaImportExternalSemaphore(extSem_out, semHandleDesc) + * return cudaImportExternalSemaphore(extSem_out, semHandleDesc) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaImportExternalSemaphore(__pyx_v_extSem_out, __pyx_v_semHandleDesc); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":491 + * return cudaDestroyExternalMemory(extMem) + * + * cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaImportExternalSemaphore", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":497 + * return cudaImportExternalSemaphore(extSem_out, semHandleDesc) + * + * cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaSignalExternalSemaphoresAsync_v2(cudaExternalSemaphore_t const *__pyx_v_extSemArray, struct cudaExternalSemaphoreSignalParams const *__pyx_v_paramsArray, unsigned int __pyx_v_numExtSems, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":498 + * + * cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaSignalExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 498, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":499 + * cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaSignalExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + * return cudaSignalExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":500 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaSignalExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) # <<<<<<<<<<<<<< + * return cudaSignalExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSignalExternalSemaphoresAsync_v2(__pyx_v_extSemArray, __pyx_v_paramsArray, __pyx_v_numExtSems, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 500, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":499 + * cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaSignalExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + * return cudaSignalExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":501 + * if usePTDS: + * return ptds._cudaSignalExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + * return cudaSignalExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaSignalExternalSemaphoresAsync_v2(__pyx_v_extSemArray, __pyx_v_paramsArray, __pyx_v_numExtSems, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":497 + * return cudaImportExternalSemaphore(extSem_out, semHandleDesc) + * + * cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaSignalExternalSemaphoresAsync_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":503 + * return cudaSignalExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + * + * cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaWaitExternalSemaphoresAsync_v2(cudaExternalSemaphore_t const *__pyx_v_extSemArray, struct cudaExternalSemaphoreWaitParams const *__pyx_v_paramsArray, unsigned int __pyx_v_numExtSems, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":504 + * + * cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaWaitExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 504, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":505 + * cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaWaitExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + * return cudaWaitExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":506 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaWaitExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) # <<<<<<<<<<<<<< + * return cudaWaitExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaWaitExternalSemaphoresAsync_v2(__pyx_v_extSemArray, __pyx_v_paramsArray, __pyx_v_numExtSems, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 506, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":505 + * cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaWaitExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + * return cudaWaitExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":507 + * if usePTDS: + * return ptds._cudaWaitExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + * return cudaWaitExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaWaitExternalSemaphoresAsync_v2(__pyx_v_extSemArray, __pyx_v_paramsArray, __pyx_v_numExtSems, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":503 + * return cudaSignalExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + * + * cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaWaitExternalSemaphoresAsync_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":509 + * return cudaWaitExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + * + * cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDestroyExternalSemaphore(cudaExternalSemaphore_t __pyx_v_extSem) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":510 + * + * cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDestroyExternalSemaphore(extSem) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 510, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":511 + * cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDestroyExternalSemaphore(extSem) + * return cudaDestroyExternalSemaphore(extSem) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":512 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDestroyExternalSemaphore(extSem) # <<<<<<<<<<<<<< + * return cudaDestroyExternalSemaphore(extSem) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyExternalSemaphore(__pyx_v_extSem); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 512, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":511 + * cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDestroyExternalSemaphore(extSem) + * return cudaDestroyExternalSemaphore(extSem) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":513 + * if usePTDS: + * return ptds._cudaDestroyExternalSemaphore(extSem) + * return cudaDestroyExternalSemaphore(extSem) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDestroyExternalSemaphore(__pyx_v_extSem); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":509 + * return cudaWaitExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + * + * cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDestroyExternalSemaphore", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":515 + * return cudaDestroyExternalSemaphore(extSem) + * + * cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFuncSetCacheConfig(void const *__pyx_v_func, enum cudaFuncCache __pyx_v_cacheConfig) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":516 + * + * cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaFuncSetCacheConfig(func, cacheConfig) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 516, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":517 + * cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFuncSetCacheConfig(func, cacheConfig) + * return cudaFuncSetCacheConfig(func, cacheConfig) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":518 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaFuncSetCacheConfig(func, cacheConfig) # <<<<<<<<<<<<<< + * return cudaFuncSetCacheConfig(func, cacheConfig) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetCacheConfig(__pyx_v_func, ((enum cudaFuncCache)__pyx_v_cacheConfig)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 518, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":517 + * cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFuncSetCacheConfig(func, cacheConfig) + * return cudaFuncSetCacheConfig(func, cacheConfig) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":519 + * if usePTDS: + * return ptds._cudaFuncSetCacheConfig(func, cacheConfig) + * return cudaFuncSetCacheConfig(func, cacheConfig) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFuncSetCacheConfig(__pyx_v_func, __pyx_v_cacheConfig); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":515 + * return cudaDestroyExternalSemaphore(extSem) + * + * cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaFuncSetCacheConfig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":521 + * return cudaFuncSetCacheConfig(func, cacheConfig) + * + * cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFuncGetAttributes(struct cudaFuncAttributes *__pyx_v_attr, void const *__pyx_v_func) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":522 + * + * cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaFuncGetAttributes(attr, func) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 522, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":523 + * cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFuncGetAttributes(attr, func) + * return cudaFuncGetAttributes(attr, func) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":524 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaFuncGetAttributes(attr, func) # <<<<<<<<<<<<<< + * return cudaFuncGetAttributes(attr, func) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncGetAttributes(__pyx_v_attr, __pyx_v_func); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 524, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":523 + * cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFuncGetAttributes(attr, func) + * return cudaFuncGetAttributes(attr, func) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":525 + * if usePTDS: + * return ptds._cudaFuncGetAttributes(attr, func) + * return cudaFuncGetAttributes(attr, func) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFuncGetAttributes(__pyx_v_attr, __pyx_v_func); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":521 + * return cudaFuncSetCacheConfig(func, cacheConfig) + * + * cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaFuncGetAttributes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":527 + * return cudaFuncGetAttributes(attr, func) + * + * cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFuncSetAttribute(void const *__pyx_v_func, enum cudaFuncAttribute __pyx_v_attr, int __pyx_v_value) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":528 + * + * cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaFuncSetAttribute(func, attr, value) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 528, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":529 + * cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFuncSetAttribute(func, attr, value) + * return cudaFuncSetAttribute(func, attr, value) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":530 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaFuncSetAttribute(func, attr, value) # <<<<<<<<<<<<<< + * return cudaFuncSetAttribute(func, attr, value) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetAttribute(__pyx_v_func, ((enum cudaFuncAttribute)__pyx_v_attr), __pyx_v_value); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 530, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":529 + * cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFuncSetAttribute(func, attr, value) + * return cudaFuncSetAttribute(func, attr, value) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":531 + * if usePTDS: + * return ptds._cudaFuncSetAttribute(func, attr, value) + * return cudaFuncSetAttribute(func, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFuncSetAttribute(__pyx_v_func, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":527 + * return cudaFuncGetAttributes(attr, func) + * + * cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaFuncSetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":533 + * return cudaFuncSetAttribute(func, attr, value) + * + * cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLaunchHostFunc(cudaStream_t __pyx_v_stream, cudaHostFn_t __pyx_v_fn, void *__pyx_v_userData) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":534 + * + * cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaLaunchHostFunc(stream, fn, userData) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 534, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":535 + * cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLaunchHostFunc(stream, fn, userData) + * return cudaLaunchHostFunc(stream, fn, userData) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":536 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaLaunchHostFunc(stream, fn, userData) # <<<<<<<<<<<<<< + * return cudaLaunchHostFunc(stream, fn, userData) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLaunchHostFunc(__pyx_v_stream, __pyx_v_fn, __pyx_v_userData); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 536, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":535 + * cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLaunchHostFunc(stream, fn, userData) + * return cudaLaunchHostFunc(stream, fn, userData) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":537 + * if usePTDS: + * return ptds._cudaLaunchHostFunc(stream, fn, userData) + * return cudaLaunchHostFunc(stream, fn, userData) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLaunchHostFunc(__pyx_v_stream, __pyx_v_fn, __pyx_v_userData); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":533 + * return cudaFuncSetAttribute(func, attr, value) + * + * cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaLaunchHostFunc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":539 + * return cudaLaunchHostFunc(stream, fn, userData) + * + * cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFuncSetSharedMemConfig(void const *__pyx_v_func, enum cudaSharedMemConfig __pyx_v_config) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":540 + * + * cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaFuncSetSharedMemConfig(func, config) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 540, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":541 + * cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFuncSetSharedMemConfig(func, config) + * return cudaFuncSetSharedMemConfig(func, config) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":542 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaFuncSetSharedMemConfig(func, config) # <<<<<<<<<<<<<< + * return cudaFuncSetSharedMemConfig(func, config) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetSharedMemConfig(__pyx_v_func, ((enum cudaSharedMemConfig)__pyx_v_config)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 542, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":541 + * cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFuncSetSharedMemConfig(func, config) + * return cudaFuncSetSharedMemConfig(func, config) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":543 + * if usePTDS: + * return ptds._cudaFuncSetSharedMemConfig(func, config) + * return cudaFuncSetSharedMemConfig(func, config) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFuncSetSharedMemConfig(__pyx_v_func, __pyx_v_config); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":539 + * return cudaLaunchHostFunc(stream, fn, userData) + * + * cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaFuncSetSharedMemConfig", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":545 + * return cudaFuncSetSharedMemConfig(func, config) + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaOccupancyMaxActiveBlocksPerMultiprocessor(int *__pyx_v_numBlocks, void const *__pyx_v_func, int __pyx_v_blockSize, size_t __pyx_v_dynamicSMemSize) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":546 + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 546, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":547 + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) + * return cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":548 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) # <<<<<<<<<<<<<< + * return cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyMaxActiveBlocksPerMultiprocessor(__pyx_v_numBlocks, __pyx_v_func, __pyx_v_blockSize, __pyx_v_dynamicSMemSize); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 548, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":547 + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) + * return cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":549 + * if usePTDS: + * return ptds._cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) + * return cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaOccupancyMaxActiveBlocksPerMultiprocessor(__pyx_v_numBlocks, __pyx_v_func, __pyx_v_blockSize, __pyx_v_dynamicSMemSize); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":545 + * return cudaFuncSetSharedMemConfig(func, config) + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaOccupancyMaxActiveBlocksPerMultiprocessor", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":551 + * return cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) + * + * cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaOccupancyAvailableDynamicSMemPerBlock(size_t *__pyx_v_dynamicSmemSize, void const *__pyx_v_func, int __pyx_v_numBlocks, int __pyx_v_blockSize) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":552 + * + * cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 552, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":553 + * cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) + * return cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":554 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) # <<<<<<<<<<<<<< + * return cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyAvailableDynamicSMemPerBlock(__pyx_v_dynamicSmemSize, __pyx_v_func, __pyx_v_numBlocks, __pyx_v_blockSize); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 554, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":553 + * cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) + * return cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":555 + * if usePTDS: + * return ptds._cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) + * return cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaOccupancyAvailableDynamicSMemPerBlock(__pyx_v_dynamicSmemSize, __pyx_v_func, __pyx_v_numBlocks, __pyx_v_blockSize); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":551 + * return cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) + * + * cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaOccupancyAvailableDynamicSMemPerBlock", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":557 + * return cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int *__pyx_v_numBlocks, void const *__pyx_v_func, int __pyx_v_blockSize, size_t __pyx_v_dynamicSMemSize, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":558 + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 558, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":559 + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) + * return cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":560 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) # <<<<<<<<<<<<<< + * return cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(__pyx_v_numBlocks, __pyx_v_func, __pyx_v_blockSize, __pyx_v_dynamicSMemSize, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 560, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":559 + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) + * return cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":561 + * if usePTDS: + * return ptds._cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) + * return cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(__pyx_v_numBlocks, __pyx_v_func, __pyx_v_blockSize, __pyx_v_dynamicSMemSize, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":557 + * return cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":563 + * return cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) + * + * cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocManaged(void **__pyx_v_devPtr, size_t __pyx_v_size, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":564 + * + * cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMallocManaged(devPtr, size, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 564, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":565 + * cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocManaged(devPtr, size, flags) + * return cudaMallocManaged(devPtr, size, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":566 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMallocManaged(devPtr, size, flags) # <<<<<<<<<<<<<< + * return cudaMallocManaged(devPtr, size, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocManaged(__pyx_v_devPtr, __pyx_v_size, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 566, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":565 + * cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocManaged(devPtr, size, flags) + * return cudaMallocManaged(devPtr, size, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":567 + * if usePTDS: + * return ptds._cudaMallocManaged(devPtr, size, flags) + * return cudaMallocManaged(devPtr, size, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocManaged(__pyx_v_devPtr, __pyx_v_size, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":563 + * return cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) + * + * cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMallocManaged", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":569 + * return cudaMallocManaged(devPtr, size, flags) + * + * cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMalloc(void **__pyx_v_devPtr, size_t __pyx_v_size) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":570 + * + * cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMalloc(devPtr, size) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 570, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":571 + * cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMalloc(devPtr, size) + * return cudaMalloc(devPtr, size) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":572 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMalloc(devPtr, size) # <<<<<<<<<<<<<< + * return cudaMalloc(devPtr, size) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc(__pyx_v_devPtr, __pyx_v_size); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 572, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":571 + * cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMalloc(devPtr, size) + * return cudaMalloc(devPtr, size) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":573 + * if usePTDS: + * return ptds._cudaMalloc(devPtr, size) + * return cudaMalloc(devPtr, size) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMalloc(__pyx_v_devPtr, __pyx_v_size); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":569 + * return cudaMallocManaged(devPtr, size, flags) + * + * cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMalloc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":575 + * return cudaMalloc(devPtr, size) + * + * cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocHost(void **__pyx_v_ptr, size_t __pyx_v_size) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":576 + * + * cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMallocHost(ptr, size) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 576, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":577 + * cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocHost(ptr, size) + * return cudaMallocHost(ptr, size) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":578 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMallocHost(ptr, size) # <<<<<<<<<<<<<< + * return cudaMallocHost(ptr, size) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocHost(__pyx_v_ptr, __pyx_v_size); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 578, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":577 + * cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocHost(ptr, size) + * return cudaMallocHost(ptr, size) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":579 + * if usePTDS: + * return ptds._cudaMallocHost(ptr, size) + * return cudaMallocHost(ptr, size) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocHost(__pyx_v_ptr, __pyx_v_size); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":575 + * return cudaMalloc(devPtr, size) + * + * cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMallocHost", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":581 + * return cudaMallocHost(ptr, size) + * + * cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocPitch(void **__pyx_v_devPtr, size_t *__pyx_v_pitch, size_t __pyx_v_width, size_t __pyx_v_height) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":582 + * + * cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMallocPitch(devPtr, pitch, width, height) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 582, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":583 + * cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocPitch(devPtr, pitch, width, height) + * return cudaMallocPitch(devPtr, pitch, width, height) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":584 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMallocPitch(devPtr, pitch, width, height) # <<<<<<<<<<<<<< + * return cudaMallocPitch(devPtr, pitch, width, height) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocPitch(__pyx_v_devPtr, __pyx_v_pitch, __pyx_v_width, __pyx_v_height); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 584, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":583 + * cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocPitch(devPtr, pitch, width, height) + * return cudaMallocPitch(devPtr, pitch, width, height) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":585 + * if usePTDS: + * return ptds._cudaMallocPitch(devPtr, pitch, width, height) + * return cudaMallocPitch(devPtr, pitch, width, height) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocPitch(__pyx_v_devPtr, __pyx_v_pitch, __pyx_v_width, __pyx_v_height); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":581 + * return cudaMallocHost(ptr, size) + * + * cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMallocPitch", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":587 + * return cudaMallocPitch(devPtr, pitch, width, height) + * + * cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocArray(cudaArray_t *__pyx_v_array, struct cudaChannelFormatDesc const *__pyx_v_desc, size_t __pyx_v_width, size_t __pyx_v_height, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":588 + * + * cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMallocArray(array, desc, width, height, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 588, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":589 + * cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocArray(array, desc, width, height, flags) + * return cudaMallocArray(array, desc, width, height, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":590 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMallocArray(array, desc, width, height, flags) # <<<<<<<<<<<<<< + * return cudaMallocArray(array, desc, width, height, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocArray(__pyx_v_array, __pyx_v_desc, __pyx_v_width, __pyx_v_height, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 590, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":589 + * cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocArray(array, desc, width, height, flags) + * return cudaMallocArray(array, desc, width, height, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":591 + * if usePTDS: + * return ptds._cudaMallocArray(array, desc, width, height, flags) + * return cudaMallocArray(array, desc, width, height, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocArray(__pyx_v_array, __pyx_v_desc, __pyx_v_width, __pyx_v_height, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":587 + * return cudaMallocPitch(devPtr, pitch, width, height) + * + * cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMallocArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":593 + * return cudaMallocArray(array, desc, width, height, flags) + * + * cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFree(void *__pyx_v_devPtr) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":594 + * + * cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaFree(devPtr) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 594, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":595 + * cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFree(devPtr) + * return cudaFree(devPtr) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":596 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaFree(devPtr) # <<<<<<<<<<<<<< + * return cudaFree(devPtr) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFree(__pyx_v_devPtr); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 596, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":595 + * cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFree(devPtr) + * return cudaFree(devPtr) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":597 + * if usePTDS: + * return ptds._cudaFree(devPtr) + * return cudaFree(devPtr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFree(__pyx_v_devPtr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":593 + * return cudaMallocArray(array, desc, width, height, flags) + * + * cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaFree", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":599 + * return cudaFree(devPtr) + * + * cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFreeHost(void *__pyx_v_ptr) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":600 + * + * cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaFreeHost(ptr) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 600, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":601 + * cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFreeHost(ptr) + * return cudaFreeHost(ptr) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":602 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaFreeHost(ptr) # <<<<<<<<<<<<<< + * return cudaFreeHost(ptr) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeHost(__pyx_v_ptr); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 602, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":601 + * cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFreeHost(ptr) + * return cudaFreeHost(ptr) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":603 + * if usePTDS: + * return ptds._cudaFreeHost(ptr) + * return cudaFreeHost(ptr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFreeHost(__pyx_v_ptr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":599 + * return cudaFree(devPtr) + * + * cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaFreeHost", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":605 + * return cudaFreeHost(ptr) + * + * cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFreeArray(cudaArray_t __pyx_v_array) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":606 + * + * cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaFreeArray(array) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 606, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":607 + * cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFreeArray(array) + * return cudaFreeArray(array) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":608 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaFreeArray(array) # <<<<<<<<<<<<<< + * return cudaFreeArray(array) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeArray(__pyx_v_array); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 608, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":607 + * cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFreeArray(array) + * return cudaFreeArray(array) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":609 + * if usePTDS: + * return ptds._cudaFreeArray(array) + * return cudaFreeArray(array) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFreeArray(__pyx_v_array); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":605 + * return cudaFreeHost(ptr) + * + * cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaFreeArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":611 + * return cudaFreeArray(array) + * + * cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFreeMipmappedArray(cudaMipmappedArray_t __pyx_v_mipmappedArray) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":612 + * + * cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaFreeMipmappedArray(mipmappedArray) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 612, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":613 + * cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFreeMipmappedArray(mipmappedArray) + * return cudaFreeMipmappedArray(mipmappedArray) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":614 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaFreeMipmappedArray(mipmappedArray) # <<<<<<<<<<<<<< + * return cudaFreeMipmappedArray(mipmappedArray) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeMipmappedArray(__pyx_v_mipmappedArray); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 614, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":613 + * cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFreeMipmappedArray(mipmappedArray) + * return cudaFreeMipmappedArray(mipmappedArray) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":615 + * if usePTDS: + * return ptds._cudaFreeMipmappedArray(mipmappedArray) + * return cudaFreeMipmappedArray(mipmappedArray) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFreeMipmappedArray(__pyx_v_mipmappedArray); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":611 + * return cudaFreeArray(array) + * + * cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaFreeMipmappedArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":617 + * return cudaFreeMipmappedArray(mipmappedArray) + * + * cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaHostAlloc(void **__pyx_v_pHost, size_t __pyx_v_size, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":618 + * + * cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaHostAlloc(pHost, size, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 618, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":619 + * cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaHostAlloc(pHost, size, flags) + * return cudaHostAlloc(pHost, size, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":620 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaHostAlloc(pHost, size, flags) # <<<<<<<<<<<<<< + * return cudaHostAlloc(pHost, size, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostAlloc(__pyx_v_pHost, __pyx_v_size, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 620, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":619 + * cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaHostAlloc(pHost, size, flags) + * return cudaHostAlloc(pHost, size, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":621 + * if usePTDS: + * return ptds._cudaHostAlloc(pHost, size, flags) + * return cudaHostAlloc(pHost, size, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaHostAlloc(__pyx_v_pHost, __pyx_v_size, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":617 + * return cudaFreeMipmappedArray(mipmappedArray) + * + * cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaHostAlloc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":623 + * return cudaHostAlloc(pHost, size, flags) + * + * cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaHostRegister(void *__pyx_v_ptr, size_t __pyx_v_size, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":624 + * + * cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaHostRegister(ptr, size, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 624, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":625 + * cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaHostRegister(ptr, size, flags) + * return cudaHostRegister(ptr, size, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":626 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaHostRegister(ptr, size, flags) # <<<<<<<<<<<<<< + * return cudaHostRegister(ptr, size, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostRegister(__pyx_v_ptr, __pyx_v_size, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 626, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":625 + * cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaHostRegister(ptr, size, flags) + * return cudaHostRegister(ptr, size, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":627 + * if usePTDS: + * return ptds._cudaHostRegister(ptr, size, flags) + * return cudaHostRegister(ptr, size, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaHostRegister(__pyx_v_ptr, __pyx_v_size, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":623 + * return cudaHostAlloc(pHost, size, flags) + * + * cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaHostRegister", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":629 + * return cudaHostRegister(ptr, size, flags) + * + * cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaHostUnregister(void *__pyx_v_ptr) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":630 + * + * cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaHostUnregister(ptr) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 630, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":631 + * cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaHostUnregister(ptr) + * return cudaHostUnregister(ptr) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":632 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaHostUnregister(ptr) # <<<<<<<<<<<<<< + * return cudaHostUnregister(ptr) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostUnregister(__pyx_v_ptr); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 632, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":631 + * cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaHostUnregister(ptr) + * return cudaHostUnregister(ptr) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":633 + * if usePTDS: + * return ptds._cudaHostUnregister(ptr) + * return cudaHostUnregister(ptr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaHostUnregister(__pyx_v_ptr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":629 + * return cudaHostRegister(ptr, size, flags) + * + * cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaHostUnregister", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":635 + * return cudaHostUnregister(ptr) + * + * cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaHostGetDevicePointer(void **__pyx_v_pDevice, void *__pyx_v_pHost, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":636 + * + * cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaHostGetDevicePointer(pDevice, pHost, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 636, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":637 + * cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaHostGetDevicePointer(pDevice, pHost, flags) + * return cudaHostGetDevicePointer(pDevice, pHost, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":638 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaHostGetDevicePointer(pDevice, pHost, flags) # <<<<<<<<<<<<<< + * return cudaHostGetDevicePointer(pDevice, pHost, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostGetDevicePointer(__pyx_v_pDevice, __pyx_v_pHost, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 638, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":637 + * cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaHostGetDevicePointer(pDevice, pHost, flags) + * return cudaHostGetDevicePointer(pDevice, pHost, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":639 + * if usePTDS: + * return ptds._cudaHostGetDevicePointer(pDevice, pHost, flags) + * return cudaHostGetDevicePointer(pDevice, pHost, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaHostGetDevicePointer(__pyx_v_pDevice, __pyx_v_pHost, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":635 + * return cudaHostUnregister(ptr) + * + * cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaHostGetDevicePointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":641 + * return cudaHostGetDevicePointer(pDevice, pHost, flags) + * + * cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaHostGetFlags(unsigned int *__pyx_v_pFlags, void *__pyx_v_pHost) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":642 + * + * cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaHostGetFlags(pFlags, pHost) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 642, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":643 + * cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaHostGetFlags(pFlags, pHost) + * return cudaHostGetFlags(pFlags, pHost) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":644 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaHostGetFlags(pFlags, pHost) # <<<<<<<<<<<<<< + * return cudaHostGetFlags(pFlags, pHost) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostGetFlags(__pyx_v_pFlags, __pyx_v_pHost); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 644, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":643 + * cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaHostGetFlags(pFlags, pHost) + * return cudaHostGetFlags(pFlags, pHost) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":645 + * if usePTDS: + * return ptds._cudaHostGetFlags(pFlags, pHost) + * return cudaHostGetFlags(pFlags, pHost) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaHostGetFlags(__pyx_v_pFlags, __pyx_v_pHost); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":641 + * return cudaHostGetDevicePointer(pDevice, pHost, flags) + * + * cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaHostGetFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":647 + * return cudaHostGetFlags(pFlags, pHost) + * + * cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMalloc3D(struct cudaPitchedPtr *__pyx_v_pitchedDevPtr, struct cudaExtent __pyx_v_extent) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":648 + * + * cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMalloc3D(pitchedDevPtr, extent) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 648, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":649 + * cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMalloc3D(pitchedDevPtr, extent) + * return cudaMalloc3D(pitchedDevPtr, extent) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":650 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMalloc3D(pitchedDevPtr, extent) # <<<<<<<<<<<<<< + * return cudaMalloc3D(pitchedDevPtr, extent) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc3D(__pyx_v_pitchedDevPtr, __pyx_v_extent); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 650, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":649 + * cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMalloc3D(pitchedDevPtr, extent) + * return cudaMalloc3D(pitchedDevPtr, extent) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":651 + * if usePTDS: + * return ptds._cudaMalloc3D(pitchedDevPtr, extent) + * return cudaMalloc3D(pitchedDevPtr, extent) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMalloc3D(__pyx_v_pitchedDevPtr, __pyx_v_extent); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":647 + * return cudaHostGetFlags(pFlags, pHost) + * + * cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMalloc3D", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":653 + * return cudaMalloc3D(pitchedDevPtr, extent) + * + * cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMalloc3DArray(cudaArray_t *__pyx_v_array, struct cudaChannelFormatDesc const *__pyx_v_desc, struct cudaExtent __pyx_v_extent, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":654 + * + * cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMalloc3DArray(array, desc, extent, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 654, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":655 + * cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMalloc3DArray(array, desc, extent, flags) + * return cudaMalloc3DArray(array, desc, extent, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":656 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMalloc3DArray(array, desc, extent, flags) # <<<<<<<<<<<<<< + * return cudaMalloc3DArray(array, desc, extent, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc3DArray(__pyx_v_array, __pyx_v_desc, __pyx_v_extent, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 656, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":655 + * cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMalloc3DArray(array, desc, extent, flags) + * return cudaMalloc3DArray(array, desc, extent, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":657 + * if usePTDS: + * return ptds._cudaMalloc3DArray(array, desc, extent, flags) + * return cudaMalloc3DArray(array, desc, extent, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMalloc3DArray(__pyx_v_array, __pyx_v_desc, __pyx_v_extent, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":653 + * return cudaMalloc3D(pitchedDevPtr, extent) + * + * cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMalloc3DArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":659 + * return cudaMalloc3DArray(array, desc, extent, flags) + * + * cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocMipmappedArray(cudaMipmappedArray_t *__pyx_v_mipmappedArray, struct cudaChannelFormatDesc const *__pyx_v_desc, struct cudaExtent __pyx_v_extent, unsigned int __pyx_v_numLevels, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":660 + * + * cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 660, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":661 + * cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) + * return cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":662 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) # <<<<<<<<<<<<<< + * return cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocMipmappedArray(__pyx_v_mipmappedArray, __pyx_v_desc, __pyx_v_extent, __pyx_v_numLevels, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 662, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":661 + * cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) + * return cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":663 + * if usePTDS: + * return ptds._cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) + * return cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocMipmappedArray(__pyx_v_mipmappedArray, __pyx_v_desc, __pyx_v_extent, __pyx_v_numLevels, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":659 + * return cudaMalloc3DArray(array, desc, extent, flags) + * + * cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMallocMipmappedArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":665 + * return cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) + * + * cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetMipmappedArrayLevel(cudaArray_t *__pyx_v_levelArray, cudaMipmappedArray_const_t __pyx_v_mipmappedArray, unsigned int __pyx_v_level) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":666 + * + * cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 666, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":667 + * cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) + * return cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":668 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) # <<<<<<<<<<<<<< + * return cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetMipmappedArrayLevel(__pyx_v_levelArray, __pyx_v_mipmappedArray, __pyx_v_level); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 668, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":667 + * cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) + * return cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":669 + * if usePTDS: + * return ptds._cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) + * return cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetMipmappedArrayLevel(__pyx_v_levelArray, __pyx_v_mipmappedArray, __pyx_v_level); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":665 + * return cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) + * + * cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetMipmappedArrayLevel", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":671 + * return cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) + * + * cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy3D(struct cudaMemcpy3DParms const *__pyx_v_p) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":672 + * + * cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpy3D(p) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 672, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":673 + * cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy3D(p) + * return cudaMemcpy3D(p) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":674 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpy3D(p) # <<<<<<<<<<<<<< + * return cudaMemcpy3D(p) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3D(__pyx_v_p); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 674, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":673 + * cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy3D(p) + * return cudaMemcpy3D(p) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":675 + * if usePTDS: + * return ptds._cudaMemcpy3D(p) + * return cudaMemcpy3D(p) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy3D(__pyx_v_p); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":671 + * return cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) + * + * cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpy3D", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":677 + * return cudaMemcpy3D(p) + * + * cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy3DPeer(struct cudaMemcpy3DPeerParms const *__pyx_v_p) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":678 + * + * cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpy3DPeer(p) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 678, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":679 + * cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy3DPeer(p) + * return cudaMemcpy3DPeer(p) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":680 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpy3DPeer(p) # <<<<<<<<<<<<<< + * return cudaMemcpy3DPeer(p) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DPeer(__pyx_v_p); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 680, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":679 + * cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy3DPeer(p) + * return cudaMemcpy3DPeer(p) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":681 + * if usePTDS: + * return ptds._cudaMemcpy3DPeer(p) + * return cudaMemcpy3DPeer(p) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy3DPeer(__pyx_v_p); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":677 + * return cudaMemcpy3D(p) + * + * cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpy3DPeer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":683 + * return cudaMemcpy3DPeer(p) + * + * cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy3DAsync(struct cudaMemcpy3DParms const *__pyx_v_p, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":684 + * + * cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpy3DAsync(p, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 684, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":685 + * cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy3DAsync(p, stream) + * return cudaMemcpy3DAsync(p, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":686 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpy3DAsync(p, stream) # <<<<<<<<<<<<<< + * return cudaMemcpy3DAsync(p, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DAsync(__pyx_v_p, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 686, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":685 + * cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy3DAsync(p, stream) + * return cudaMemcpy3DAsync(p, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":687 + * if usePTDS: + * return ptds._cudaMemcpy3DAsync(p, stream) + * return cudaMemcpy3DAsync(p, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy3DAsync(__pyx_v_p, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":683 + * return cudaMemcpy3DPeer(p) + * + * cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpy3DAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":689 + * return cudaMemcpy3DAsync(p, stream) + * + * cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy3DPeerAsync(struct cudaMemcpy3DPeerParms const *__pyx_v_p, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":690 + * + * cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpy3DPeerAsync(p, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 690, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":691 + * cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy3DPeerAsync(p, stream) + * return cudaMemcpy3DPeerAsync(p, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":692 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpy3DPeerAsync(p, stream) # <<<<<<<<<<<<<< + * return cudaMemcpy3DPeerAsync(p, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DPeerAsync(__pyx_v_p, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 692, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":691 + * cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy3DPeerAsync(p, stream) + * return cudaMemcpy3DPeerAsync(p, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":693 + * if usePTDS: + * return ptds._cudaMemcpy3DPeerAsync(p, stream) + * return cudaMemcpy3DPeerAsync(p, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy3DPeerAsync(__pyx_v_p, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":689 + * return cudaMemcpy3DAsync(p, stream) + * + * cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpy3DPeerAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":695 + * return cudaMemcpy3DPeerAsync(p, stream) + * + * cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemGetInfo(size_t *__pyx_v_free, size_t *__pyx_v_total) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":696 + * + * cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemGetInfo(free, total) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 696, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":697 + * cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemGetInfo(free, total) + * return cudaMemGetInfo(free, total) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":698 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemGetInfo(free, total) # <<<<<<<<<<<<<< + * return cudaMemGetInfo(free, total) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemGetInfo(__pyx_v_free, __pyx_v_total); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 698, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":697 + * cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemGetInfo(free, total) + * return cudaMemGetInfo(free, total) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":699 + * if usePTDS: + * return ptds._cudaMemGetInfo(free, total) + * return cudaMemGetInfo(free, total) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemGetInfo(__pyx_v_free, __pyx_v_total); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":695 + * return cudaMemcpy3DPeerAsync(p, stream) + * + * cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemGetInfo", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":701 + * return cudaMemGetInfo(free, total) + * + * cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaArrayGetInfo(struct cudaChannelFormatDesc *__pyx_v_desc, struct cudaExtent *__pyx_v_extent, unsigned int *__pyx_v_flags, cudaArray_t __pyx_v_array) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":702 + * + * cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaArrayGetInfo(desc, extent, flags, array) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 702, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":703 + * cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaArrayGetInfo(desc, extent, flags, array) + * return cudaArrayGetInfo(desc, extent, flags, array) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":704 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaArrayGetInfo(desc, extent, flags, array) # <<<<<<<<<<<<<< + * return cudaArrayGetInfo(desc, extent, flags, array) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetInfo(__pyx_v_desc, __pyx_v_extent, __pyx_v_flags, __pyx_v_array); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 704, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":703 + * cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaArrayGetInfo(desc, extent, flags, array) + * return cudaArrayGetInfo(desc, extent, flags, array) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":705 + * if usePTDS: + * return ptds._cudaArrayGetInfo(desc, extent, flags, array) + * return cudaArrayGetInfo(desc, extent, flags, array) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaArrayGetInfo(__pyx_v_desc, __pyx_v_extent, __pyx_v_flags, __pyx_v_array); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":701 + * return cudaMemGetInfo(free, total) + * + * cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaArrayGetInfo", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":707 + * return cudaArrayGetInfo(desc, extent, flags, array) + * + * cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaArrayGetPlane(cudaArray_t *__pyx_v_pPlaneArray, cudaArray_t __pyx_v_hArray, unsigned int __pyx_v_planeIdx) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":708 + * + * cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 708, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":709 + * cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) + * return cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":710 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) # <<<<<<<<<<<<<< + * return cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetPlane(__pyx_v_pPlaneArray, __pyx_v_hArray, __pyx_v_planeIdx); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 710, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":709 + * cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) + * return cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":711 + * if usePTDS: + * return ptds._cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) + * return cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaArrayGetPlane(__pyx_v_pPlaneArray, __pyx_v_hArray, __pyx_v_planeIdx); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":707 + * return cudaArrayGetInfo(desc, extent, flags, array) + * + * cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaArrayGetPlane", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":713 + * return cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) + * + * cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaArrayGetMemoryRequirements(struct cudaArrayMemoryRequirements *__pyx_v_memoryRequirements, cudaArray_t __pyx_v_array, int __pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":714 + * + * cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaArrayGetMemoryRequirements(memoryRequirements, array, device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 714, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":715 + * cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaArrayGetMemoryRequirements(memoryRequirements, array, device) + * return cudaArrayGetMemoryRequirements(memoryRequirements, array, device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":716 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaArrayGetMemoryRequirements(memoryRequirements, array, device) # <<<<<<<<<<<<<< + * return cudaArrayGetMemoryRequirements(memoryRequirements, array, device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetMemoryRequirements(__pyx_v_memoryRequirements, __pyx_v_array, __pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 716, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":715 + * cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaArrayGetMemoryRequirements(memoryRequirements, array, device) + * return cudaArrayGetMemoryRequirements(memoryRequirements, array, device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":717 + * if usePTDS: + * return ptds._cudaArrayGetMemoryRequirements(memoryRequirements, array, device) + * return cudaArrayGetMemoryRequirements(memoryRequirements, array, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaArrayGetMemoryRequirements(__pyx_v_memoryRequirements, __pyx_v_array, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":713 + * return cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) + * + * cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaArrayGetMemoryRequirements", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":719 + * return cudaArrayGetMemoryRequirements(memoryRequirements, array, device) + * + * cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMipmappedArrayGetMemoryRequirements(struct cudaArrayMemoryRequirements *__pyx_v_memoryRequirements, cudaMipmappedArray_t __pyx_v_mipmap, int __pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":720 + * + * cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 720, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":721 + * cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) + * return cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":722 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) # <<<<<<<<<<<<<< + * return cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMipmappedArrayGetMemoryRequirements(__pyx_v_memoryRequirements, __pyx_v_mipmap, __pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 722, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":721 + * cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) + * return cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":723 + * if usePTDS: + * return ptds._cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) + * return cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMipmappedArrayGetMemoryRequirements(__pyx_v_memoryRequirements, __pyx_v_mipmap, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":719 + * return cudaArrayGetMemoryRequirements(memoryRequirements, array, device) + * + * cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMipmappedArrayGetMemoryRequirements", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":725 + * return cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) + * + * cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaArrayGetSparseProperties(struct cudaArraySparseProperties *__pyx_v_sparseProperties, cudaArray_t __pyx_v_array) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":726 + * + * cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaArrayGetSparseProperties(sparseProperties, array) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 726, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":727 + * cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaArrayGetSparseProperties(sparseProperties, array) + * return cudaArrayGetSparseProperties(sparseProperties, array) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":728 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaArrayGetSparseProperties(sparseProperties, array) # <<<<<<<<<<<<<< + * return cudaArrayGetSparseProperties(sparseProperties, array) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetSparseProperties(__pyx_v_sparseProperties, __pyx_v_array); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 728, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":727 + * cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaArrayGetSparseProperties(sparseProperties, array) + * return cudaArrayGetSparseProperties(sparseProperties, array) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":729 + * if usePTDS: + * return ptds._cudaArrayGetSparseProperties(sparseProperties, array) + * return cudaArrayGetSparseProperties(sparseProperties, array) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaArrayGetSparseProperties(__pyx_v_sparseProperties, __pyx_v_array); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":725 + * return cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) + * + * cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaArrayGetSparseProperties", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":731 + * return cudaArrayGetSparseProperties(sparseProperties, array) + * + * cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMipmappedArrayGetSparseProperties(struct cudaArraySparseProperties *__pyx_v_sparseProperties, cudaMipmappedArray_t __pyx_v_mipmap) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":732 + * + * cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 732, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":733 + * cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) + * return cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":734 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) # <<<<<<<<<<<<<< + * return cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMipmappedArrayGetSparseProperties(__pyx_v_sparseProperties, __pyx_v_mipmap); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 734, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":733 + * cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) + * return cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":735 + * if usePTDS: + * return ptds._cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) + * return cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMipmappedArrayGetSparseProperties(__pyx_v_sparseProperties, __pyx_v_mipmap); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":731 + * return cudaArrayGetSparseProperties(sparseProperties, array) + * + * cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMipmappedArrayGetSparseProperties", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":737 + * return cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) + * + * cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy(void *__pyx_v_dst, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":738 + * + * cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpy(dst, src, count, kind) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 738, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":739 + * cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy(dst, src, count, kind) + * return cudaMemcpy(dst, src, count, kind) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":740 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpy(dst, src, count, kind) # <<<<<<<<<<<<<< + * return cudaMemcpy(dst, src, count, kind) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy(__pyx_v_dst, __pyx_v_src, __pyx_v_count, ((enum cudaMemcpyKind)__pyx_v_kind)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 740, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":739 + * cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy(dst, src, count, kind) + * return cudaMemcpy(dst, src, count, kind) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":741 + * if usePTDS: + * return ptds._cudaMemcpy(dst, src, count, kind) + * return cudaMemcpy(dst, src, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy(__pyx_v_dst, __pyx_v_src, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":737 + * return cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) + * + * cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":743 + * return cudaMemcpy(dst, src, count, kind) + * + * cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyPeer(void *__pyx_v_dst, int __pyx_v_dstDevice, void const *__pyx_v_src, int __pyx_v_srcDevice, size_t __pyx_v_count) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":744 + * + * cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 744, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":745 + * cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) + * return cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":746 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) # <<<<<<<<<<<<<< + * return cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyPeer(__pyx_v_dst, __pyx_v_dstDevice, __pyx_v_src, __pyx_v_srcDevice, __pyx_v_count); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 746, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":745 + * cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) + * return cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":747 + * if usePTDS: + * return ptds._cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) + * return cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyPeer(__pyx_v_dst, __pyx_v_dstDevice, __pyx_v_src, __pyx_v_srcDevice, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":743 + * return cudaMemcpy(dst, src, count, kind) + * + * cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpyPeer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":749 + * return cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) + * + * cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2D(void *__pyx_v_dst, size_t __pyx_v_dpitch, void const *__pyx_v_src, size_t __pyx_v_spitch, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":750 + * + * cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 750, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":751 + * cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) + * return cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":752 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) # <<<<<<<<<<<<<< + * return cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2D(__pyx_v_dst, __pyx_v_dpitch, __pyx_v_src, __pyx_v_spitch, __pyx_v_width, __pyx_v_height, ((enum cudaMemcpyKind)__pyx_v_kind)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 752, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":751 + * cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) + * return cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":753 + * if usePTDS: + * return ptds._cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) + * return cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2D(__pyx_v_dst, __pyx_v_dpitch, __pyx_v_src, __pyx_v_spitch, __pyx_v_width, __pyx_v_height, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":749 + * return cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) + * + * cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpy2D", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":755 + * return cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) + * + * cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2DToArray(cudaArray_t __pyx_v_dst, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, void const *__pyx_v_src, size_t __pyx_v_spitch, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":756 + * + * cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 756, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":757 + * cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) + * return cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":758 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) # <<<<<<<<<<<<<< + * return cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DToArray(__pyx_v_dst, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_src, __pyx_v_spitch, __pyx_v_width, __pyx_v_height, ((enum cudaMemcpyKind)__pyx_v_kind)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 758, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":757 + * cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) + * return cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":759 + * if usePTDS: + * return ptds._cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) + * return cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2DToArray(__pyx_v_dst, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_src, __pyx_v_spitch, __pyx_v_width, __pyx_v_height, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":755 + * return cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) + * + * cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpy2DToArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":761 + * return cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) + * + * cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2DFromArray(void *__pyx_v_dst, size_t __pyx_v_dpitch, cudaArray_const_t __pyx_v_src, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":762 + * + * cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 762, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":763 + * cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) + * return cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":764 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) # <<<<<<<<<<<<<< + * return cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DFromArray(__pyx_v_dst, __pyx_v_dpitch, __pyx_v_src, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_width, __pyx_v_height, ((enum cudaMemcpyKind)__pyx_v_kind)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 764, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":763 + * cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) + * return cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":765 + * if usePTDS: + * return ptds._cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) + * return cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2DFromArray(__pyx_v_dst, __pyx_v_dpitch, __pyx_v_src, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_width, __pyx_v_height, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":761 + * return cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) + * + * cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpy2DFromArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":767 + * return cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) + * + * cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2DArrayToArray(cudaArray_t __pyx_v_dst, size_t __pyx_v_wOffsetDst, size_t __pyx_v_hOffsetDst, cudaArray_const_t __pyx_v_src, size_t __pyx_v_wOffsetSrc, size_t __pyx_v_hOffsetSrc, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":768 + * + * cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 768, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":769 + * cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) + * return cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":770 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) # <<<<<<<<<<<<<< + * return cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DArrayToArray(__pyx_v_dst, __pyx_v_wOffsetDst, __pyx_v_hOffsetDst, __pyx_v_src, __pyx_v_wOffsetSrc, __pyx_v_hOffsetSrc, __pyx_v_width, __pyx_v_height, ((enum cudaMemcpyKind)__pyx_v_kind)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 770, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":769 + * cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) + * return cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":771 + * if usePTDS: + * return ptds._cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) + * return cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2DArrayToArray(__pyx_v_dst, __pyx_v_wOffsetDst, __pyx_v_hOffsetDst, __pyx_v_src, __pyx_v_wOffsetSrc, __pyx_v_hOffsetSrc, __pyx_v_width, __pyx_v_height, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":767 + * return cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) + * + * cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpy2DArrayToArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":773 + * return cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) + * + * cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyAsync(void *__pyx_v_dst, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":774 + * + * cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpyAsync(dst, src, count, kind, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 774, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":775 + * cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyAsync(dst, src, count, kind, stream) + * return cudaMemcpyAsync(dst, src, count, kind, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":776 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpyAsync(dst, src, count, kind, stream) # <<<<<<<<<<<<<< + * return cudaMemcpyAsync(dst, src, count, kind, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyAsync(__pyx_v_dst, __pyx_v_src, __pyx_v_count, ((enum cudaMemcpyKind)__pyx_v_kind), __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 776, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":775 + * cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyAsync(dst, src, count, kind, stream) + * return cudaMemcpyAsync(dst, src, count, kind, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":777 + * if usePTDS: + * return ptds._cudaMemcpyAsync(dst, src, count, kind, stream) + * return cudaMemcpyAsync(dst, src, count, kind, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyAsync(__pyx_v_dst, __pyx_v_src, __pyx_v_count, __pyx_v_kind, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":773 + * return cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) + * + * cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpyAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":779 + * return cudaMemcpyAsync(dst, src, count, kind, stream) + * + * cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyPeerAsync(void *__pyx_v_dst, int __pyx_v_dstDevice, void const *__pyx_v_src, int __pyx_v_srcDevice, size_t __pyx_v_count, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":780 + * + * cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 780, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":781 + * cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) + * return cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":782 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) # <<<<<<<<<<<<<< + * return cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyPeerAsync(__pyx_v_dst, __pyx_v_dstDevice, __pyx_v_src, __pyx_v_srcDevice, __pyx_v_count, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 782, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":781 + * cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) + * return cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":783 + * if usePTDS: + * return ptds._cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) + * return cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyPeerAsync(__pyx_v_dst, __pyx_v_dstDevice, __pyx_v_src, __pyx_v_srcDevice, __pyx_v_count, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":779 + * return cudaMemcpyAsync(dst, src, count, kind, stream) + * + * cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpyPeerAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":785 + * return cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) + * + * cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyBatchAsync(void **__pyx_v_dsts, void **__pyx_v_srcs, size_t *__pyx_v_sizes, size_t __pyx_v_count, struct cudaMemcpyAttributes *__pyx_v_attrs, size_t *__pyx_v_attrsIdxs, size_t __pyx_v_numAttrs, size_t *__pyx_v_failIdx, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":786 + * + * cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 786, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":787 + * cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) + * return cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":788 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) # <<<<<<<<<<<<<< + * return cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyBatchAsync(__pyx_v_dsts, __pyx_v_srcs, __pyx_v_sizes, __pyx_v_count, __pyx_v_attrs, __pyx_v_attrsIdxs, __pyx_v_numAttrs, __pyx_v_failIdx, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 788, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":787 + * cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) + * return cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":789 + * if usePTDS: + * return ptds._cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) + * return cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyBatchAsync(__pyx_v_dsts, __pyx_v_srcs, __pyx_v_sizes, __pyx_v_count, __pyx_v_attrs, __pyx_v_attrsIdxs, __pyx_v_numAttrs, __pyx_v_failIdx, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":785 + * return cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) + * + * cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpyBatchAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":791 + * return cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) + * + * cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy3DBatchAsync(size_t __pyx_v_numOps, struct cudaMemcpy3DBatchOp *__pyx_v_opList, size_t *__pyx_v_failIdx, unsigned PY_LONG_LONG __pyx_v_flags, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":792 + * + * cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 792, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":793 + * cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) + * return cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":794 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) # <<<<<<<<<<<<<< + * return cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DBatchAsync(__pyx_v_numOps, __pyx_v_opList, __pyx_v_failIdx, __pyx_v_flags, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 794, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":793 + * cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) + * return cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":795 + * if usePTDS: + * return ptds._cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) + * return cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy3DBatchAsync(__pyx_v_numOps, __pyx_v_opList, __pyx_v_failIdx, __pyx_v_flags, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":791 + * return cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) + * + * cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpy3DBatchAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":797 + * return cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) + * + * cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2DAsync(void *__pyx_v_dst, size_t __pyx_v_dpitch, void const *__pyx_v_src, size_t __pyx_v_spitch, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":798 + * + * cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 798, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":799 + * cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) + * return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":800 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) # <<<<<<<<<<<<<< + * return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DAsync(__pyx_v_dst, __pyx_v_dpitch, __pyx_v_src, __pyx_v_spitch, __pyx_v_width, __pyx_v_height, ((enum cudaMemcpyKind)__pyx_v_kind), __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 800, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":799 + * cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) + * return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":801 + * if usePTDS: + * return ptds._cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) + * return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2DAsync(__pyx_v_dst, __pyx_v_dpitch, __pyx_v_src, __pyx_v_spitch, __pyx_v_width, __pyx_v_height, __pyx_v_kind, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":797 + * return cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) + * + * cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpy2DAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":803 + * return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) + * + * cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2DToArrayAsync(cudaArray_t __pyx_v_dst, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, void const *__pyx_v_src, size_t __pyx_v_spitch, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":804 + * + * cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 804, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":805 + * cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) + * return cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":806 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) # <<<<<<<<<<<<<< + * return cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DToArrayAsync(__pyx_v_dst, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_src, __pyx_v_spitch, __pyx_v_width, __pyx_v_height, ((enum cudaMemcpyKind)__pyx_v_kind), __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 806, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":805 + * cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) + * return cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":807 + * if usePTDS: + * return ptds._cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) + * return cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2DToArrayAsync(__pyx_v_dst, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_src, __pyx_v_spitch, __pyx_v_width, __pyx_v_height, __pyx_v_kind, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":803 + * return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) + * + * cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpy2DToArrayAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":809 + * return cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) + * + * cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2DFromArrayAsync(void *__pyx_v_dst, size_t __pyx_v_dpitch, cudaArray_const_t __pyx_v_src, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":810 + * + * cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 810, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":811 + * cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) + * return cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":812 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) # <<<<<<<<<<<<<< + * return cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DFromArrayAsync(__pyx_v_dst, __pyx_v_dpitch, __pyx_v_src, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_width, __pyx_v_height, ((enum cudaMemcpyKind)__pyx_v_kind), __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 812, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":811 + * cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) + * return cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":813 + * if usePTDS: + * return ptds._cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) + * return cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2DFromArrayAsync(__pyx_v_dst, __pyx_v_dpitch, __pyx_v_src, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_width, __pyx_v_height, __pyx_v_kind, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":809 + * return cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) + * + * cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpy2DFromArrayAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":815 + * return cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) + * + * cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemset(void *__pyx_v_devPtr, int __pyx_v_value, size_t __pyx_v_count) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":816 + * + * cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemset(devPtr, value, count) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 816, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":817 + * cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemset(devPtr, value, count) + * return cudaMemset(devPtr, value, count) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":818 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemset(devPtr, value, count) # <<<<<<<<<<<<<< + * return cudaMemset(devPtr, value, count) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset(__pyx_v_devPtr, __pyx_v_value, __pyx_v_count); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 818, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":817 + * cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemset(devPtr, value, count) + * return cudaMemset(devPtr, value, count) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":819 + * if usePTDS: + * return ptds._cudaMemset(devPtr, value, count) + * return cudaMemset(devPtr, value, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemset(__pyx_v_devPtr, __pyx_v_value, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":815 + * return cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) + * + * cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemset", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":821 + * return cudaMemset(devPtr, value, count) + * + * cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemset2D(void *__pyx_v_devPtr, size_t __pyx_v_pitch, int __pyx_v_value, size_t __pyx_v_width, size_t __pyx_v_height) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":822 + * + * cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemset2D(devPtr, pitch, value, width, height) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 822, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":823 + * cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemset2D(devPtr, pitch, value, width, height) + * return cudaMemset2D(devPtr, pitch, value, width, height) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":824 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemset2D(devPtr, pitch, value, width, height) # <<<<<<<<<<<<<< + * return cudaMemset2D(devPtr, pitch, value, width, height) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset2D(__pyx_v_devPtr, __pyx_v_pitch, __pyx_v_value, __pyx_v_width, __pyx_v_height); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 824, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":823 + * cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemset2D(devPtr, pitch, value, width, height) + * return cudaMemset2D(devPtr, pitch, value, width, height) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":825 + * if usePTDS: + * return ptds._cudaMemset2D(devPtr, pitch, value, width, height) + * return cudaMemset2D(devPtr, pitch, value, width, height) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemset2D(__pyx_v_devPtr, __pyx_v_pitch, __pyx_v_value, __pyx_v_width, __pyx_v_height); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":821 + * return cudaMemset(devPtr, value, count) + * + * cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemset2D", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":827 + * return cudaMemset2D(devPtr, pitch, value, width, height) + * + * cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemset3D(struct cudaPitchedPtr __pyx_v_pitchedDevPtr, int __pyx_v_value, struct cudaExtent __pyx_v_extent) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":828 + * + * cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemset3D(pitchedDevPtr, value, extent) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 828, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":829 + * cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemset3D(pitchedDevPtr, value, extent) + * return cudaMemset3D(pitchedDevPtr, value, extent) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":830 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemset3D(pitchedDevPtr, value, extent) # <<<<<<<<<<<<<< + * return cudaMemset3D(pitchedDevPtr, value, extent) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset3D(__pyx_v_pitchedDevPtr, __pyx_v_value, __pyx_v_extent); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 830, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":829 + * cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemset3D(pitchedDevPtr, value, extent) + * return cudaMemset3D(pitchedDevPtr, value, extent) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":831 + * if usePTDS: + * return ptds._cudaMemset3D(pitchedDevPtr, value, extent) + * return cudaMemset3D(pitchedDevPtr, value, extent) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemset3D(__pyx_v_pitchedDevPtr, __pyx_v_value, __pyx_v_extent); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":827 + * return cudaMemset2D(devPtr, pitch, value, width, height) + * + * cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemset3D", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":833 + * return cudaMemset3D(pitchedDevPtr, value, extent) + * + * cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemsetAsync(void *__pyx_v_devPtr, int __pyx_v_value, size_t __pyx_v_count, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":834 + * + * cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemsetAsync(devPtr, value, count, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 834, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":835 + * cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemsetAsync(devPtr, value, count, stream) + * return cudaMemsetAsync(devPtr, value, count, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":836 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemsetAsync(devPtr, value, count, stream) # <<<<<<<<<<<<<< + * return cudaMemsetAsync(devPtr, value, count, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemsetAsync(__pyx_v_devPtr, __pyx_v_value, __pyx_v_count, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 836, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":835 + * cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemsetAsync(devPtr, value, count, stream) + * return cudaMemsetAsync(devPtr, value, count, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":837 + * if usePTDS: + * return ptds._cudaMemsetAsync(devPtr, value, count, stream) + * return cudaMemsetAsync(devPtr, value, count, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemsetAsync(__pyx_v_devPtr, __pyx_v_value, __pyx_v_count, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":833 + * return cudaMemset3D(pitchedDevPtr, value, extent) + * + * cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemsetAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":839 + * return cudaMemsetAsync(devPtr, value, count, stream) + * + * cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemset2DAsync(void *__pyx_v_devPtr, size_t __pyx_v_pitch, int __pyx_v_value, size_t __pyx_v_width, size_t __pyx_v_height, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":840 + * + * cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 840, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":841 + * cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) + * return cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":842 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) # <<<<<<<<<<<<<< + * return cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset2DAsync(__pyx_v_devPtr, __pyx_v_pitch, __pyx_v_value, __pyx_v_width, __pyx_v_height, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 842, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":841 + * cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) + * return cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":843 + * if usePTDS: + * return ptds._cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) + * return cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemset2DAsync(__pyx_v_devPtr, __pyx_v_pitch, __pyx_v_value, __pyx_v_width, __pyx_v_height, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":839 + * return cudaMemsetAsync(devPtr, value, count, stream) + * + * cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemset2DAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":845 + * return cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) + * + * cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemset3DAsync(struct cudaPitchedPtr __pyx_v_pitchedDevPtr, int __pyx_v_value, struct cudaExtent __pyx_v_extent, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":846 + * + * cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 846, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":847 + * cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) + * return cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":848 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) # <<<<<<<<<<<<<< + * return cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset3DAsync(__pyx_v_pitchedDevPtr, __pyx_v_value, __pyx_v_extent, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 848, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":847 + * cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) + * return cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":849 + * if usePTDS: + * return ptds._cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) + * return cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemset3DAsync(__pyx_v_pitchedDevPtr, __pyx_v_value, __pyx_v_extent, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":845 + * return cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) + * + * cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemset3DAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":851 + * return cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) + * + * cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPrefetchAsync(void const *__pyx_v_devPtr, size_t __pyx_v_count, int __pyx_v_dstDevice, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":852 + * + * cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 852, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":853 + * cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) + * return cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":854 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) # <<<<<<<<<<<<<< + * return cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPrefetchAsync(__pyx_v_devPtr, __pyx_v_count, __pyx_v_dstDevice, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 854, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":853 + * cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) + * return cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":855 + * if usePTDS: + * return ptds._cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) + * return cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPrefetchAsync(__pyx_v_devPtr, __pyx_v_count, __pyx_v_dstDevice, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":851 + * return cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) + * + * cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemPrefetchAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":857 + * return cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) + * + * cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPrefetchAsync_v2(void const *__pyx_v_devPtr, size_t __pyx_v_count, struct cudaMemLocation __pyx_v_location, unsigned int __pyx_v_flags, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":858 + * + * cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 858, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":859 + * cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) + * return cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":860 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) # <<<<<<<<<<<<<< + * return cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPrefetchAsync_v2(__pyx_v_devPtr, __pyx_v_count, __pyx_v_location, __pyx_v_flags, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 860, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":859 + * cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) + * return cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":861 + * if usePTDS: + * return ptds._cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) + * return cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPrefetchAsync_v2(__pyx_v_devPtr, __pyx_v_count, __pyx_v_location, __pyx_v_flags, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":857 + * return cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) + * + * cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemPrefetchAsync_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":863 + * return cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) + * + * cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemAdvise(void const *__pyx_v_devPtr, size_t __pyx_v_count, enum cudaMemoryAdvise __pyx_v_advice, int __pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":864 + * + * cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemAdvise(devPtr, count, advice, device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 864, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":865 + * cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemAdvise(devPtr, count, advice, device) + * return cudaMemAdvise(devPtr, count, advice, device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":866 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemAdvise(devPtr, count, advice, device) # <<<<<<<<<<<<<< + * return cudaMemAdvise(devPtr, count, advice, device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemAdvise(__pyx_v_devPtr, __pyx_v_count, ((enum cudaMemoryAdvise)__pyx_v_advice), __pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 866, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":865 + * cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemAdvise(devPtr, count, advice, device) + * return cudaMemAdvise(devPtr, count, advice, device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":867 + * if usePTDS: + * return ptds._cudaMemAdvise(devPtr, count, advice, device) + * return cudaMemAdvise(devPtr, count, advice, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemAdvise(__pyx_v_devPtr, __pyx_v_count, __pyx_v_advice, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":863 + * return cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) + * + * cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemAdvise", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":869 + * return cudaMemAdvise(devPtr, count, advice, device) + * + * cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemAdvise_v2(void const *__pyx_v_devPtr, size_t __pyx_v_count, enum cudaMemoryAdvise __pyx_v_advice, struct cudaMemLocation __pyx_v_location) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":870 + * + * cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemAdvise_v2(devPtr, count, advice, location) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 870, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":871 + * cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemAdvise_v2(devPtr, count, advice, location) + * return cudaMemAdvise_v2(devPtr, count, advice, location) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":872 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemAdvise_v2(devPtr, count, advice, location) # <<<<<<<<<<<<<< + * return cudaMemAdvise_v2(devPtr, count, advice, location) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemAdvise_v2(__pyx_v_devPtr, __pyx_v_count, ((enum cudaMemoryAdvise)__pyx_v_advice), __pyx_v_location); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 872, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":871 + * cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemAdvise_v2(devPtr, count, advice, location) + * return cudaMemAdvise_v2(devPtr, count, advice, location) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":873 + * if usePTDS: + * return ptds._cudaMemAdvise_v2(devPtr, count, advice, location) + * return cudaMemAdvise_v2(devPtr, count, advice, location) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemAdvise_v2(__pyx_v_devPtr, __pyx_v_count, __pyx_v_advice, __pyx_v_location); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":869 + * return cudaMemAdvise(devPtr, count, advice, device) + * + * cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemAdvise_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":875 + * return cudaMemAdvise_v2(devPtr, count, advice, location) + * + * cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemRangeGetAttribute(void *__pyx_v_data, size_t __pyx_v_dataSize, enum cudaMemRangeAttribute __pyx_v_attribute, void const *__pyx_v_devPtr, size_t __pyx_v_count) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":876 + * + * cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 876, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":877 + * cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) + * return cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":878 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) # <<<<<<<<<<<<<< + * return cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemRangeGetAttribute(__pyx_v_data, __pyx_v_dataSize, ((enum cudaMemRangeAttribute)__pyx_v_attribute), __pyx_v_devPtr, __pyx_v_count); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 878, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":877 + * cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) + * return cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":879 + * if usePTDS: + * return ptds._cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) + * return cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemRangeGetAttribute(__pyx_v_data, __pyx_v_dataSize, __pyx_v_attribute, __pyx_v_devPtr, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":875 + * return cudaMemAdvise_v2(devPtr, count, advice, location) + * + * cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemRangeGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":881 + * return cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) + * + * cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemRangeGetAttributes(void **__pyx_v_data, size_t *__pyx_v_dataSizes, enum cudaMemRangeAttribute *__pyx_v_attributes, size_t __pyx_v_numAttributes, void const *__pyx_v_devPtr, size_t __pyx_v_count) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":882 + * + * cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 882, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":883 + * cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) + * return cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":884 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) # <<<<<<<<<<<<<< + * return cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemRangeGetAttributes(__pyx_v_data, __pyx_v_dataSizes, ((enum cudaMemRangeAttribute *)__pyx_v_attributes), __pyx_v_numAttributes, __pyx_v_devPtr, __pyx_v_count); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 884, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":883 + * cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) + * return cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":885 + * if usePTDS: + * return ptds._cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) + * return cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemRangeGetAttributes(__pyx_v_data, __pyx_v_dataSizes, __pyx_v_attributes, __pyx_v_numAttributes, __pyx_v_devPtr, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":881 + * return cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) + * + * cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemRangeGetAttributes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":887 + * return cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) + * + * cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyToArray(cudaArray_t __pyx_v_dst, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":888 + * + * cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 888, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":889 + * cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) + * return cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":890 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) # <<<<<<<<<<<<<< + * return cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyToArray(__pyx_v_dst, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_src, __pyx_v_count, ((enum cudaMemcpyKind)__pyx_v_kind)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 890, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":889 + * cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) + * return cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":891 + * if usePTDS: + * return ptds._cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) + * return cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyToArray(__pyx_v_dst, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_src, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":887 + * return cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) + * + * cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpyToArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":893 + * return cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) + * + * cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyFromArray(void *__pyx_v_dst, cudaArray_const_t __pyx_v_src, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":894 + * + * cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 894, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":895 + * cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) + * return cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":896 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) # <<<<<<<<<<<<<< + * return cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyFromArray(__pyx_v_dst, __pyx_v_src, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_count, ((enum cudaMemcpyKind)__pyx_v_kind)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 896, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":895 + * cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) + * return cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":897 + * if usePTDS: + * return ptds._cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) + * return cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyFromArray(__pyx_v_dst, __pyx_v_src, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":893 + * return cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) + * + * cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpyFromArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":899 + * return cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) + * + * cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyArrayToArray(cudaArray_t __pyx_v_dst, size_t __pyx_v_wOffsetDst, size_t __pyx_v_hOffsetDst, cudaArray_const_t __pyx_v_src, size_t __pyx_v_wOffsetSrc, size_t __pyx_v_hOffsetSrc, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":900 + * + * cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 900, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":901 + * cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) + * return cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":902 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) # <<<<<<<<<<<<<< + * return cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyArrayToArray(__pyx_v_dst, __pyx_v_wOffsetDst, __pyx_v_hOffsetDst, __pyx_v_src, __pyx_v_wOffsetSrc, __pyx_v_hOffsetSrc, __pyx_v_count, ((enum cudaMemcpyKind)__pyx_v_kind)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 902, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":901 + * cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) + * return cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":903 + * if usePTDS: + * return ptds._cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) + * return cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyArrayToArray(__pyx_v_dst, __pyx_v_wOffsetDst, __pyx_v_hOffsetDst, __pyx_v_src, __pyx_v_wOffsetSrc, __pyx_v_hOffsetSrc, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":899 + * return cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) + * + * cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpyArrayToArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":905 + * return cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) + * + * cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyToArrayAsync(cudaArray_t __pyx_v_dst, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":906 + * + * cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 906, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":907 + * cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) + * return cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":908 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) # <<<<<<<<<<<<<< + * return cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyToArrayAsync(__pyx_v_dst, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_src, __pyx_v_count, ((enum cudaMemcpyKind)__pyx_v_kind), __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 908, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":907 + * cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) + * return cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":909 + * if usePTDS: + * return ptds._cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) + * return cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyToArrayAsync(__pyx_v_dst, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_src, __pyx_v_count, __pyx_v_kind, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":905 + * return cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) + * + * cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpyToArrayAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":911 + * return cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) + * + * cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyFromArrayAsync(void *__pyx_v_dst, cudaArray_const_t __pyx_v_src, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":912 + * + * cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 912, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":913 + * cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) + * return cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":914 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) # <<<<<<<<<<<<<< + * return cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyFromArrayAsync(__pyx_v_dst, __pyx_v_src, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_count, ((enum cudaMemcpyKind)__pyx_v_kind), __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 914, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":913 + * cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) + * return cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":915 + * if usePTDS: + * return ptds._cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) + * return cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyFromArrayAsync(__pyx_v_dst, __pyx_v_src, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_count, __pyx_v_kind, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":911 + * return cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) + * + * cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemcpyFromArrayAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":917 + * return cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) + * + * cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocAsync(void **__pyx_v_devPtr, size_t __pyx_v_size, cudaStream_t __pyx_v_hStream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":918 + * + * cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMallocAsync(devPtr, size, hStream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 918, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":919 + * cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocAsync(devPtr, size, hStream) + * return cudaMallocAsync(devPtr, size, hStream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":920 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMallocAsync(devPtr, size, hStream) # <<<<<<<<<<<<<< + * return cudaMallocAsync(devPtr, size, hStream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocAsync(__pyx_v_devPtr, __pyx_v_size, __pyx_v_hStream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 920, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":919 + * cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocAsync(devPtr, size, hStream) + * return cudaMallocAsync(devPtr, size, hStream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":921 + * if usePTDS: + * return ptds._cudaMallocAsync(devPtr, size, hStream) + * return cudaMallocAsync(devPtr, size, hStream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocAsync(__pyx_v_devPtr, __pyx_v_size, __pyx_v_hStream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":917 + * return cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) + * + * cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMallocAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":923 + * return cudaMallocAsync(devPtr, size, hStream) + * + * cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFreeAsync(void *__pyx_v_devPtr, cudaStream_t __pyx_v_hStream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":924 + * + * cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaFreeAsync(devPtr, hStream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 924, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":925 + * cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFreeAsync(devPtr, hStream) + * return cudaFreeAsync(devPtr, hStream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":926 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaFreeAsync(devPtr, hStream) # <<<<<<<<<<<<<< + * return cudaFreeAsync(devPtr, hStream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeAsync(__pyx_v_devPtr, __pyx_v_hStream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 926, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":925 + * cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaFreeAsync(devPtr, hStream) + * return cudaFreeAsync(devPtr, hStream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":927 + * if usePTDS: + * return ptds._cudaFreeAsync(devPtr, hStream) + * return cudaFreeAsync(devPtr, hStream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFreeAsync(__pyx_v_devPtr, __pyx_v_hStream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":923 + * return cudaMallocAsync(devPtr, size, hStream) + * + * cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaFreeAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":929 + * return cudaFreeAsync(devPtr, hStream) + * + * cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolTrimTo(cudaMemPool_t __pyx_v_memPool, size_t __pyx_v_minBytesToKeep) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":930 + * + * cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemPoolTrimTo(memPool, minBytesToKeep) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 930, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":931 + * cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolTrimTo(memPool, minBytesToKeep) + * return cudaMemPoolTrimTo(memPool, minBytesToKeep) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":932 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemPoolTrimTo(memPool, minBytesToKeep) # <<<<<<<<<<<<<< + * return cudaMemPoolTrimTo(memPool, minBytesToKeep) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolTrimTo(__pyx_v_memPool, __pyx_v_minBytesToKeep); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 932, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":931 + * cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolTrimTo(memPool, minBytesToKeep) + * return cudaMemPoolTrimTo(memPool, minBytesToKeep) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":933 + * if usePTDS: + * return ptds._cudaMemPoolTrimTo(memPool, minBytesToKeep) + * return cudaMemPoolTrimTo(memPool, minBytesToKeep) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolTrimTo(__pyx_v_memPool, __pyx_v_minBytesToKeep); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":929 + * return cudaFreeAsync(devPtr, hStream) + * + * cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemPoolTrimTo", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":935 + * return cudaMemPoolTrimTo(memPool, minBytesToKeep) + * + * cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolSetAttribute(cudaMemPool_t __pyx_v_memPool, enum cudaMemPoolAttr __pyx_v_attr, void *__pyx_v_value) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":936 + * + * cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemPoolSetAttribute(memPool, attr, value) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 936, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":937 + * cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolSetAttribute(memPool, attr, value) + * return cudaMemPoolSetAttribute(memPool, attr, value) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":938 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemPoolSetAttribute(memPool, attr, value) # <<<<<<<<<<<<<< + * return cudaMemPoolSetAttribute(memPool, attr, value) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolSetAttribute(__pyx_v_memPool, ((enum cudaMemPoolAttr)__pyx_v_attr), __pyx_v_value); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 938, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":937 + * cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolSetAttribute(memPool, attr, value) + * return cudaMemPoolSetAttribute(memPool, attr, value) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":939 + * if usePTDS: + * return ptds._cudaMemPoolSetAttribute(memPool, attr, value) + * return cudaMemPoolSetAttribute(memPool, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolSetAttribute(__pyx_v_memPool, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":935 + * return cudaMemPoolTrimTo(memPool, minBytesToKeep) + * + * cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemPoolSetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":941 + * return cudaMemPoolSetAttribute(memPool, attr, value) + * + * cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolGetAttribute(cudaMemPool_t __pyx_v_memPool, enum cudaMemPoolAttr __pyx_v_attr, void *__pyx_v_value) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":942 + * + * cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemPoolGetAttribute(memPool, attr, value) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 942, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":943 + * cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolGetAttribute(memPool, attr, value) + * return cudaMemPoolGetAttribute(memPool, attr, value) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":944 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemPoolGetAttribute(memPool, attr, value) # <<<<<<<<<<<<<< + * return cudaMemPoolGetAttribute(memPool, attr, value) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolGetAttribute(__pyx_v_memPool, ((enum cudaMemPoolAttr)__pyx_v_attr), __pyx_v_value); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 944, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":943 + * cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolGetAttribute(memPool, attr, value) + * return cudaMemPoolGetAttribute(memPool, attr, value) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":945 + * if usePTDS: + * return ptds._cudaMemPoolGetAttribute(memPool, attr, value) + * return cudaMemPoolGetAttribute(memPool, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolGetAttribute(__pyx_v_memPool, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":941 + * return cudaMemPoolSetAttribute(memPool, attr, value) + * + * cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemPoolGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":947 + * return cudaMemPoolGetAttribute(memPool, attr, value) + * + * cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolSetAccess(cudaMemPool_t __pyx_v_memPool, struct cudaMemAccessDesc const *__pyx_v_descList, size_t __pyx_v_count) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":948 + * + * cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemPoolSetAccess(memPool, descList, count) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 948, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":949 + * cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolSetAccess(memPool, descList, count) + * return cudaMemPoolSetAccess(memPool, descList, count) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":950 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemPoolSetAccess(memPool, descList, count) # <<<<<<<<<<<<<< + * return cudaMemPoolSetAccess(memPool, descList, count) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolSetAccess(__pyx_v_memPool, __pyx_v_descList, __pyx_v_count); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 950, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":949 + * cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolSetAccess(memPool, descList, count) + * return cudaMemPoolSetAccess(memPool, descList, count) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":951 + * if usePTDS: + * return ptds._cudaMemPoolSetAccess(memPool, descList, count) + * return cudaMemPoolSetAccess(memPool, descList, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolSetAccess(__pyx_v_memPool, __pyx_v_descList, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":947 + * return cudaMemPoolGetAttribute(memPool, attr, value) + * + * cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemPoolSetAccess", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":953 + * return cudaMemPoolSetAccess(memPool, descList, count) + * + * cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolGetAccess(enum cudaMemAccessFlags *__pyx_v_flags, cudaMemPool_t __pyx_v_memPool, struct cudaMemLocation *__pyx_v_location) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":954 + * + * cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemPoolGetAccess(flags, memPool, location) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 954, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":955 + * cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolGetAccess(flags, memPool, location) + * return cudaMemPoolGetAccess(flags, memPool, location) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":956 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemPoolGetAccess(flags, memPool, location) # <<<<<<<<<<<<<< + * return cudaMemPoolGetAccess(flags, memPool, location) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolGetAccess(((enum cudaMemAccessFlags *)__pyx_v_flags), __pyx_v_memPool, __pyx_v_location); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 956, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":955 + * cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolGetAccess(flags, memPool, location) + * return cudaMemPoolGetAccess(flags, memPool, location) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":957 + * if usePTDS: + * return ptds._cudaMemPoolGetAccess(flags, memPool, location) + * return cudaMemPoolGetAccess(flags, memPool, location) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolGetAccess(__pyx_v_flags, __pyx_v_memPool, __pyx_v_location); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":953 + * return cudaMemPoolSetAccess(memPool, descList, count) + * + * cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemPoolGetAccess", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":959 + * return cudaMemPoolGetAccess(flags, memPool, location) + * + * cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolCreate(cudaMemPool_t *__pyx_v_memPool, struct cudaMemPoolProps const *__pyx_v_poolProps) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":960 + * + * cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemPoolCreate(memPool, poolProps) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 960, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":961 + * cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolCreate(memPool, poolProps) + * return cudaMemPoolCreate(memPool, poolProps) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":962 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemPoolCreate(memPool, poolProps) # <<<<<<<<<<<<<< + * return cudaMemPoolCreate(memPool, poolProps) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolCreate(__pyx_v_memPool, __pyx_v_poolProps); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 962, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":961 + * cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolCreate(memPool, poolProps) + * return cudaMemPoolCreate(memPool, poolProps) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":963 + * if usePTDS: + * return ptds._cudaMemPoolCreate(memPool, poolProps) + * return cudaMemPoolCreate(memPool, poolProps) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolCreate(__pyx_v_memPool, __pyx_v_poolProps); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":959 + * return cudaMemPoolGetAccess(flags, memPool, location) + * + * cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemPoolCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":965 + * return cudaMemPoolCreate(memPool, poolProps) + * + * cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolDestroy(cudaMemPool_t __pyx_v_memPool) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":966 + * + * cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemPoolDestroy(memPool) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 966, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":967 + * cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolDestroy(memPool) + * return cudaMemPoolDestroy(memPool) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":968 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemPoolDestroy(memPool) # <<<<<<<<<<<<<< + * return cudaMemPoolDestroy(memPool) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolDestroy(__pyx_v_memPool); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 968, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":967 + * cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolDestroy(memPool) + * return cudaMemPoolDestroy(memPool) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":969 + * if usePTDS: + * return ptds._cudaMemPoolDestroy(memPool) + * return cudaMemPoolDestroy(memPool) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolDestroy(__pyx_v_memPool); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":965 + * return cudaMemPoolCreate(memPool, poolProps) + * + * cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemPoolDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":971 + * return cudaMemPoolDestroy(memPool) + * + * cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocFromPoolAsync(void **__pyx_v_ptr, size_t __pyx_v_size, cudaMemPool_t __pyx_v_memPool, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":972 + * + * cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMallocFromPoolAsync(ptr, size, memPool, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 972, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":973 + * cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocFromPoolAsync(ptr, size, memPool, stream) + * return cudaMallocFromPoolAsync(ptr, size, memPool, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":974 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMallocFromPoolAsync(ptr, size, memPool, stream) # <<<<<<<<<<<<<< + * return cudaMallocFromPoolAsync(ptr, size, memPool, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocFromPoolAsync(__pyx_v_ptr, __pyx_v_size, __pyx_v_memPool, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 974, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":973 + * cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMallocFromPoolAsync(ptr, size, memPool, stream) + * return cudaMallocFromPoolAsync(ptr, size, memPool, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":975 + * if usePTDS: + * return ptds._cudaMallocFromPoolAsync(ptr, size, memPool, stream) + * return cudaMallocFromPoolAsync(ptr, size, memPool, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocFromPoolAsync(__pyx_v_ptr, __pyx_v_size, __pyx_v_memPool, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":971 + * return cudaMemPoolDestroy(memPool) + * + * cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMallocFromPoolAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":977 + * return cudaMallocFromPoolAsync(ptr, size, memPool, stream) + * + * cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolExportToShareableHandle(void *__pyx_v_shareableHandle, cudaMemPool_t __pyx_v_memPool, enum cudaMemAllocationHandleType __pyx_v_handleType, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":978 + * + * cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 978, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":979 + * cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) + * return cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":980 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) # <<<<<<<<<<<<<< + * return cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolExportToShareableHandle(__pyx_v_shareableHandle, __pyx_v_memPool, ((enum cudaMemAllocationHandleType)__pyx_v_handleType), __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 980, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":979 + * cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) + * return cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":981 + * if usePTDS: + * return ptds._cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) + * return cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolExportToShareableHandle(__pyx_v_shareableHandle, __pyx_v_memPool, __pyx_v_handleType, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":977 + * return cudaMallocFromPoolAsync(ptr, size, memPool, stream) + * + * cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemPoolExportToShareableHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":983 + * return cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) + * + * cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolImportFromShareableHandle(cudaMemPool_t *__pyx_v_memPool, void *__pyx_v_shareableHandle, enum cudaMemAllocationHandleType __pyx_v_handleType, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":984 + * + * cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 984, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":985 + * cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) + * return cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":986 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) # <<<<<<<<<<<<<< + * return cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolImportFromShareableHandle(__pyx_v_memPool, __pyx_v_shareableHandle, ((enum cudaMemAllocationHandleType)__pyx_v_handleType), __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 986, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":985 + * cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) + * return cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":987 + * if usePTDS: + * return ptds._cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) + * return cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolImportFromShareableHandle(__pyx_v_memPool, __pyx_v_shareableHandle, __pyx_v_handleType, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":983 + * return cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) + * + * cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemPoolImportFromShareableHandle", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":989 + * return cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) + * + * cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolExportPointer(struct cudaMemPoolPtrExportData *__pyx_v_exportData, void *__pyx_v_ptr) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":990 + * + * cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemPoolExportPointer(exportData, ptr) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 990, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":991 + * cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolExportPointer(exportData, ptr) + * return cudaMemPoolExportPointer(exportData, ptr) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":992 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemPoolExportPointer(exportData, ptr) # <<<<<<<<<<<<<< + * return cudaMemPoolExportPointer(exportData, ptr) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolExportPointer(__pyx_v_exportData, __pyx_v_ptr); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 992, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":991 + * cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolExportPointer(exportData, ptr) + * return cudaMemPoolExportPointer(exportData, ptr) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":993 + * if usePTDS: + * return ptds._cudaMemPoolExportPointer(exportData, ptr) + * return cudaMemPoolExportPointer(exportData, ptr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolExportPointer(__pyx_v_exportData, __pyx_v_ptr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":989 + * return cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) + * + * cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemPoolExportPointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":995 + * return cudaMemPoolExportPointer(exportData, ptr) + * + * cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolImportPointer(void **__pyx_v_ptr, cudaMemPool_t __pyx_v_memPool, struct cudaMemPoolPtrExportData *__pyx_v_exportData) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":996 + * + * cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaMemPoolImportPointer(ptr, memPool, exportData) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 996, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":997 + * cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolImportPointer(ptr, memPool, exportData) + * return cudaMemPoolImportPointer(ptr, memPool, exportData) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":998 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaMemPoolImportPointer(ptr, memPool, exportData) # <<<<<<<<<<<<<< + * return cudaMemPoolImportPointer(ptr, memPool, exportData) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolImportPointer(__pyx_v_ptr, __pyx_v_memPool, __pyx_v_exportData); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 998, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":997 + * cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaMemPoolImportPointer(ptr, memPool, exportData) + * return cudaMemPoolImportPointer(ptr, memPool, exportData) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":999 + * if usePTDS: + * return ptds._cudaMemPoolImportPointer(ptr, memPool, exportData) + * return cudaMemPoolImportPointer(ptr, memPool, exportData) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolImportPointer(__pyx_v_ptr, __pyx_v_memPool, __pyx_v_exportData); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":995 + * return cudaMemPoolExportPointer(exportData, ptr) + * + * cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaMemPoolImportPointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1001 + * return cudaMemPoolImportPointer(ptr, memPool, exportData) + * + * cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaPointerGetAttributes(struct cudaPointerAttributes *__pyx_v_attributes, void const *__pyx_v_ptr) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1002 + * + * cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaPointerGetAttributes(attributes, ptr) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1002, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1003 + * cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaPointerGetAttributes(attributes, ptr) + * return cudaPointerGetAttributes(attributes, ptr) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1004 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaPointerGetAttributes(attributes, ptr) # <<<<<<<<<<<<<< + * return cudaPointerGetAttributes(attributes, ptr) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaPointerGetAttributes(__pyx_v_attributes, __pyx_v_ptr); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1004, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1003 + * cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaPointerGetAttributes(attributes, ptr) + * return cudaPointerGetAttributes(attributes, ptr) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1005 + * if usePTDS: + * return ptds._cudaPointerGetAttributes(attributes, ptr) + * return cudaPointerGetAttributes(attributes, ptr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaPointerGetAttributes(__pyx_v_attributes, __pyx_v_ptr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1001 + * return cudaMemPoolImportPointer(ptr, memPool, exportData) + * + * cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaPointerGetAttributes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1007 + * return cudaPointerGetAttributes(attributes, ptr) + * + * cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceCanAccessPeer(int *__pyx_v_canAccessPeer, int __pyx_v_device, int __pyx_v_peerDevice) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1008 + * + * cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1008, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1009 + * cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) + * return cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1010 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) # <<<<<<<<<<<<<< + * return cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceCanAccessPeer(__pyx_v_canAccessPeer, __pyx_v_device, __pyx_v_peerDevice); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1010, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1009 + * cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) + * return cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1011 + * if usePTDS: + * return ptds._cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) + * return cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceCanAccessPeer(__pyx_v_canAccessPeer, __pyx_v_device, __pyx_v_peerDevice); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1007 + * return cudaPointerGetAttributes(attributes, ptr) + * + * cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceCanAccessPeer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1013 + * return cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) + * + * cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceEnablePeerAccess(int __pyx_v_peerDevice, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1014 + * + * cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceEnablePeerAccess(peerDevice, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1014, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1015 + * cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceEnablePeerAccess(peerDevice, flags) + * return cudaDeviceEnablePeerAccess(peerDevice, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1016 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceEnablePeerAccess(peerDevice, flags) # <<<<<<<<<<<<<< + * return cudaDeviceEnablePeerAccess(peerDevice, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceEnablePeerAccess(__pyx_v_peerDevice, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1016, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1015 + * cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceEnablePeerAccess(peerDevice, flags) + * return cudaDeviceEnablePeerAccess(peerDevice, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1017 + * if usePTDS: + * return ptds._cudaDeviceEnablePeerAccess(peerDevice, flags) + * return cudaDeviceEnablePeerAccess(peerDevice, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceEnablePeerAccess(__pyx_v_peerDevice, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1013 + * return cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) + * + * cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceEnablePeerAccess", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1019 + * return cudaDeviceEnablePeerAccess(peerDevice, flags) + * + * cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceDisablePeerAccess(int __pyx_v_peerDevice) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1020 + * + * cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceDisablePeerAccess(peerDevice) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1020, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1021 + * cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceDisablePeerAccess(peerDevice) + * return cudaDeviceDisablePeerAccess(peerDevice) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1022 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceDisablePeerAccess(peerDevice) # <<<<<<<<<<<<<< + * return cudaDeviceDisablePeerAccess(peerDevice) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceDisablePeerAccess(__pyx_v_peerDevice); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1022, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1021 + * cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceDisablePeerAccess(peerDevice) + * return cudaDeviceDisablePeerAccess(peerDevice) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1023 + * if usePTDS: + * return ptds._cudaDeviceDisablePeerAccess(peerDevice) + * return cudaDeviceDisablePeerAccess(peerDevice) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceDisablePeerAccess(__pyx_v_peerDevice); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1019 + * return cudaDeviceEnablePeerAccess(peerDevice, flags) + * + * cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceDisablePeerAccess", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1025 + * return cudaDeviceDisablePeerAccess(peerDevice) + * + * cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsUnregisterResource(cudaGraphicsResource_t __pyx_v_resource) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1026 + * + * cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphicsUnregisterResource(resource) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1026, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1027 + * cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsUnregisterResource(resource) + * return cudaGraphicsUnregisterResource(resource) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1028 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphicsUnregisterResource(resource) # <<<<<<<<<<<<<< + * return cudaGraphicsUnregisterResource(resource) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsUnregisterResource(__pyx_v_resource); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1028, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1027 + * cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsUnregisterResource(resource) + * return cudaGraphicsUnregisterResource(resource) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1029 + * if usePTDS: + * return ptds._cudaGraphicsUnregisterResource(resource) + * return cudaGraphicsUnregisterResource(resource) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsUnregisterResource(__pyx_v_resource); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1025 + * return cudaDeviceDisablePeerAccess(peerDevice) + * + * cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphicsUnregisterResource", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1031 + * return cudaGraphicsUnregisterResource(resource) + * + * cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t __pyx_v_resource, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1032 + * + * cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphicsResourceSetMapFlags(resource, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1032, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1033 + * cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsResourceSetMapFlags(resource, flags) + * return cudaGraphicsResourceSetMapFlags(resource, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1034 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphicsResourceSetMapFlags(resource, flags) # <<<<<<<<<<<<<< + * return cudaGraphicsResourceSetMapFlags(resource, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceSetMapFlags(__pyx_v_resource, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1034, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1033 + * cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsResourceSetMapFlags(resource, flags) + * return cudaGraphicsResourceSetMapFlags(resource, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1035 + * if usePTDS: + * return ptds._cudaGraphicsResourceSetMapFlags(resource, flags) + * return cudaGraphicsResourceSetMapFlags(resource, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsResourceSetMapFlags(__pyx_v_resource, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1031 + * return cudaGraphicsUnregisterResource(resource) + * + * cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphicsResourceSetMapFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1037 + * return cudaGraphicsResourceSetMapFlags(resource, flags) + * + * cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsMapResources(int __pyx_v_count, cudaGraphicsResource_t *__pyx_v_resources, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1038 + * + * cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphicsMapResources(count, resources, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1038, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1039 + * cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsMapResources(count, resources, stream) + * return cudaGraphicsMapResources(count, resources, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1040 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphicsMapResources(count, resources, stream) # <<<<<<<<<<<<<< + * return cudaGraphicsMapResources(count, resources, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsMapResources(__pyx_v_count, __pyx_v_resources, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1040, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1039 + * cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsMapResources(count, resources, stream) + * return cudaGraphicsMapResources(count, resources, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1041 + * if usePTDS: + * return ptds._cudaGraphicsMapResources(count, resources, stream) + * return cudaGraphicsMapResources(count, resources, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsMapResources(__pyx_v_count, __pyx_v_resources, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1037 + * return cudaGraphicsResourceSetMapFlags(resource, flags) + * + * cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphicsMapResources", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1043 + * return cudaGraphicsMapResources(count, resources, stream) + * + * cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsUnmapResources(int __pyx_v_count, cudaGraphicsResource_t *__pyx_v_resources, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1044 + * + * cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphicsUnmapResources(count, resources, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1044, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1045 + * cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsUnmapResources(count, resources, stream) + * return cudaGraphicsUnmapResources(count, resources, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1046 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphicsUnmapResources(count, resources, stream) # <<<<<<<<<<<<<< + * return cudaGraphicsUnmapResources(count, resources, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsUnmapResources(__pyx_v_count, __pyx_v_resources, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1046, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1045 + * cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsUnmapResources(count, resources, stream) + * return cudaGraphicsUnmapResources(count, resources, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1047 + * if usePTDS: + * return ptds._cudaGraphicsUnmapResources(count, resources, stream) + * return cudaGraphicsUnmapResources(count, resources, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsUnmapResources(__pyx_v_count, __pyx_v_resources, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1043 + * return cudaGraphicsMapResources(count, resources, stream) + * + * cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphicsUnmapResources", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1049 + * return cudaGraphicsUnmapResources(count, resources, stream) + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsResourceGetMappedPointer(void **__pyx_v_devPtr, size_t *__pyx_v_size, cudaGraphicsResource_t __pyx_v_resource) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1050 + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1050, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1051 + * cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) + * return cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1052 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) # <<<<<<<<<<<<<< + * return cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceGetMappedPointer(__pyx_v_devPtr, __pyx_v_size, __pyx_v_resource); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1052, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1051 + * cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) + * return cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1053 + * if usePTDS: + * return ptds._cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) + * return cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsResourceGetMappedPointer(__pyx_v_devPtr, __pyx_v_size, __pyx_v_resource); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1049 + * return cudaGraphicsUnmapResources(count, resources, stream) + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphicsResourceGetMappedPointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1055 + * return cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) + * + * cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsSubResourceGetMappedArray(cudaArray_t *__pyx_v_array, cudaGraphicsResource_t __pyx_v_resource, unsigned int __pyx_v_arrayIndex, unsigned int __pyx_v_mipLevel) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1056 + * + * cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1056, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1057 + * cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) + * return cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1058 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) # <<<<<<<<<<<<<< + * return cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsSubResourceGetMappedArray(__pyx_v_array, __pyx_v_resource, __pyx_v_arrayIndex, __pyx_v_mipLevel); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1058, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1057 + * cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) + * return cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1059 + * if usePTDS: + * return ptds._cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) + * return cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsSubResourceGetMappedArray(__pyx_v_array, __pyx_v_resource, __pyx_v_arrayIndex, __pyx_v_mipLevel); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1055 + * return cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) + * + * cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphicsSubResourceGetMappedArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1061 + * return cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t *__pyx_v_mipmappedArray, cudaGraphicsResource_t __pyx_v_resource) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1062 + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1062, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1063 + * cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) + * return cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1064 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) # <<<<<<<<<<<<<< + * return cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceGetMappedMipmappedArray(__pyx_v_mipmappedArray, __pyx_v_resource); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1064, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1063 + * cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) + * return cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1065 + * if usePTDS: + * return ptds._cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) + * return cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsResourceGetMappedMipmappedArray(__pyx_v_mipmappedArray, __pyx_v_resource); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1061 + * return cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphicsResourceGetMappedMipmappedArray", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1067 + * return cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) + * + * cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetChannelDesc(struct cudaChannelFormatDesc *__pyx_v_desc, cudaArray_const_t __pyx_v_array) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1068 + * + * cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetChannelDesc(desc, array) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1068, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1069 + * cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetChannelDesc(desc, array) + * return cudaGetChannelDesc(desc, array) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1070 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetChannelDesc(desc, array) # <<<<<<<<<<<<<< + * return cudaGetChannelDesc(desc, array) + * @cython.show_performance_hints(False) +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetChannelDesc(__pyx_v_desc, __pyx_v_array); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1070, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1069 + * cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetChannelDesc(desc, array) + * return cudaGetChannelDesc(desc, array) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1071 + * if usePTDS: + * return ptds._cudaGetChannelDesc(desc, array) + * return cudaGetChannelDesc(desc, array) # <<<<<<<<<<<<<< + * @cython.show_performance_hints(False) + * cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil: +*/ + __pyx_r = cudaGetChannelDesc(__pyx_v_desc, __pyx_v_array); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1067 + * return cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) + * + * cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetChannelDesc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1072 + * return ptds._cudaGetChannelDesc(desc, array) + * return cudaGetChannelDesc(desc, array) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil: + * cdef bint usePTDS = cudaPythonInit() +*/ + +static struct cudaChannelFormatDesc __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaCreateChannelDesc(int __pyx_v_x, int __pyx_v_y, int __pyx_v_z, int __pyx_v_w, enum cudaChannelFormatKind __pyx_v_f) { + int __pyx_v_usePTDS; + struct cudaChannelFormatDesc __pyx_r; + int __pyx_t_1; + struct cudaChannelFormatDesc __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1074 + * @cython.show_performance_hints(False) + * cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaCreateChannelDesc(x, y, z, w, f) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1074, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1075 + * cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaCreateChannelDesc(x, y, z, w, f) + * return cudaCreateChannelDesc(x, y, z, w, f) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1076 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaCreateChannelDesc(x, y, z, w, f) # <<<<<<<<<<<<<< + * return cudaCreateChannelDesc(x, y, z, w, f) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateChannelDesc(__pyx_v_x, __pyx_v_y, __pyx_v_z, __pyx_v_w, ((enum cudaChannelFormatKind)__pyx_v_f)); if (unlikely(__Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1076, __pyx_L1_error) + __pyx_r = __pyx_t_2; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1075 + * cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaCreateChannelDesc(x, y, z, w, f) + * return cudaCreateChannelDesc(x, y, z, w, f) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1077 + * if usePTDS: + * return ptds._cudaCreateChannelDesc(x, y, z, w, f) + * return cudaCreateChannelDesc(x, y, z, w, f) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaCreateChannelDesc(__pyx_v_x, __pyx_v_y, __pyx_v_z, __pyx_v_w, __pyx_v_f); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1072 + * return ptds._cudaGetChannelDesc(desc, array) + * return cudaGetChannelDesc(desc, array) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil: + * cdef bint usePTDS = cudaPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaCreateChannelDesc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_pretend_to_initialize(&__pyx_r); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1079 + * return cudaCreateChannelDesc(x, y, z, w, f) + * + * cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaCreateTextureObject(cudaTextureObject_t *__pyx_v_pTexObject, struct cudaResourceDesc const *__pyx_v_pResDesc, struct cudaTextureDesc const *__pyx_v_pTexDesc, struct cudaResourceViewDesc const *__pyx_v_pResViewDesc) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1080 + * + * cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1080, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1081 + * cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) + * return cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1082 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) # <<<<<<<<<<<<<< + * return cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateTextureObject(__pyx_v_pTexObject, __pyx_v_pResDesc, __pyx_v_pTexDesc, __pyx_v_pResViewDesc); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1082, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1081 + * cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) + * return cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1083 + * if usePTDS: + * return ptds._cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) + * return cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaCreateTextureObject(__pyx_v_pTexObject, __pyx_v_pResDesc, __pyx_v_pTexDesc, __pyx_v_pResViewDesc); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1079 + * return cudaCreateChannelDesc(x, y, z, w, f) + * + * cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaCreateTextureObject", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1085 + * return cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) + * + * cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDestroyTextureObject(cudaTextureObject_t __pyx_v_texObject) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1086 + * + * cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDestroyTextureObject(texObject) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1086, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1087 + * cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDestroyTextureObject(texObject) + * return cudaDestroyTextureObject(texObject) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1088 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDestroyTextureObject(texObject) # <<<<<<<<<<<<<< + * return cudaDestroyTextureObject(texObject) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyTextureObject(__pyx_v_texObject); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1088, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1087 + * cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDestroyTextureObject(texObject) + * return cudaDestroyTextureObject(texObject) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1089 + * if usePTDS: + * return ptds._cudaDestroyTextureObject(texObject) + * return cudaDestroyTextureObject(texObject) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDestroyTextureObject(__pyx_v_texObject); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1085 + * return cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) + * + * cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDestroyTextureObject", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1091 + * return cudaDestroyTextureObject(texObject) + * + * cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetTextureObjectResourceDesc(struct cudaResourceDesc *__pyx_v_pResDesc, cudaTextureObject_t __pyx_v_texObject) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1092 + * + * cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetTextureObjectResourceDesc(pResDesc, texObject) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1092, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1093 + * cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetTextureObjectResourceDesc(pResDesc, texObject) + * return cudaGetTextureObjectResourceDesc(pResDesc, texObject) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1094 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetTextureObjectResourceDesc(pResDesc, texObject) # <<<<<<<<<<<<<< + * return cudaGetTextureObjectResourceDesc(pResDesc, texObject) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectResourceDesc(__pyx_v_pResDesc, __pyx_v_texObject); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1094, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1093 + * cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetTextureObjectResourceDesc(pResDesc, texObject) + * return cudaGetTextureObjectResourceDesc(pResDesc, texObject) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1095 + * if usePTDS: + * return ptds._cudaGetTextureObjectResourceDesc(pResDesc, texObject) + * return cudaGetTextureObjectResourceDesc(pResDesc, texObject) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetTextureObjectResourceDesc(__pyx_v_pResDesc, __pyx_v_texObject); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1091 + * return cudaDestroyTextureObject(texObject) + * + * cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetTextureObjectResourceDesc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1097 + * return cudaGetTextureObjectResourceDesc(pResDesc, texObject) + * + * cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetTextureObjectTextureDesc(struct cudaTextureDesc *__pyx_v_pTexDesc, cudaTextureObject_t __pyx_v_texObject) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1098 + * + * cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetTextureObjectTextureDesc(pTexDesc, texObject) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1098, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1099 + * cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetTextureObjectTextureDesc(pTexDesc, texObject) + * return cudaGetTextureObjectTextureDesc(pTexDesc, texObject) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1100 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetTextureObjectTextureDesc(pTexDesc, texObject) # <<<<<<<<<<<<<< + * return cudaGetTextureObjectTextureDesc(pTexDesc, texObject) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectTextureDesc(__pyx_v_pTexDesc, __pyx_v_texObject); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1100, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1099 + * cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetTextureObjectTextureDesc(pTexDesc, texObject) + * return cudaGetTextureObjectTextureDesc(pTexDesc, texObject) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1101 + * if usePTDS: + * return ptds._cudaGetTextureObjectTextureDesc(pTexDesc, texObject) + * return cudaGetTextureObjectTextureDesc(pTexDesc, texObject) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetTextureObjectTextureDesc(__pyx_v_pTexDesc, __pyx_v_texObject); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1097 + * return cudaGetTextureObjectResourceDesc(pResDesc, texObject) + * + * cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetTextureObjectTextureDesc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1103 + * return cudaGetTextureObjectTextureDesc(pTexDesc, texObject) + * + * cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetTextureObjectResourceViewDesc(struct cudaResourceViewDesc *__pyx_v_pResViewDesc, cudaTextureObject_t __pyx_v_texObject) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1104 + * + * cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1104, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1105 + * cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) + * return cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1106 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) # <<<<<<<<<<<<<< + * return cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectResourceViewDesc(__pyx_v_pResViewDesc, __pyx_v_texObject); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1106, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1105 + * cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) + * return cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1107 + * if usePTDS: + * return ptds._cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) + * return cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetTextureObjectResourceViewDesc(__pyx_v_pResViewDesc, __pyx_v_texObject); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1103 + * return cudaGetTextureObjectTextureDesc(pTexDesc, texObject) + * + * cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetTextureObjectResourceViewDesc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1109 + * return cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) + * + * cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaCreateSurfaceObject(cudaSurfaceObject_t *__pyx_v_pSurfObject, struct cudaResourceDesc const *__pyx_v_pResDesc) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1110 + * + * cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaCreateSurfaceObject(pSurfObject, pResDesc) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1110, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1111 + * cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaCreateSurfaceObject(pSurfObject, pResDesc) + * return cudaCreateSurfaceObject(pSurfObject, pResDesc) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1112 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaCreateSurfaceObject(pSurfObject, pResDesc) # <<<<<<<<<<<<<< + * return cudaCreateSurfaceObject(pSurfObject, pResDesc) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateSurfaceObject(__pyx_v_pSurfObject, __pyx_v_pResDesc); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1112, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1111 + * cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaCreateSurfaceObject(pSurfObject, pResDesc) + * return cudaCreateSurfaceObject(pSurfObject, pResDesc) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1113 + * if usePTDS: + * return ptds._cudaCreateSurfaceObject(pSurfObject, pResDesc) + * return cudaCreateSurfaceObject(pSurfObject, pResDesc) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaCreateSurfaceObject(__pyx_v_pSurfObject, __pyx_v_pResDesc); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1109 + * return cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) + * + * cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaCreateSurfaceObject", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1115 + * return cudaCreateSurfaceObject(pSurfObject, pResDesc) + * + * cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDestroySurfaceObject(cudaSurfaceObject_t __pyx_v_surfObject) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1116 + * + * cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDestroySurfaceObject(surfObject) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1116, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1117 + * cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDestroySurfaceObject(surfObject) + * return cudaDestroySurfaceObject(surfObject) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1118 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDestroySurfaceObject(surfObject) # <<<<<<<<<<<<<< + * return cudaDestroySurfaceObject(surfObject) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroySurfaceObject(__pyx_v_surfObject); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1118, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1117 + * cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDestroySurfaceObject(surfObject) + * return cudaDestroySurfaceObject(surfObject) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1119 + * if usePTDS: + * return ptds._cudaDestroySurfaceObject(surfObject) + * return cudaDestroySurfaceObject(surfObject) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDestroySurfaceObject(__pyx_v_surfObject); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1115 + * return cudaCreateSurfaceObject(pSurfObject, pResDesc) + * + * cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDestroySurfaceObject", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1121 + * return cudaDestroySurfaceObject(surfObject) + * + * cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetSurfaceObjectResourceDesc(struct cudaResourceDesc *__pyx_v_pResDesc, cudaSurfaceObject_t __pyx_v_surfObject) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1122 + * + * cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1122, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1123 + * cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) + * return cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1124 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) # <<<<<<<<<<<<<< + * return cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetSurfaceObjectResourceDesc(__pyx_v_pResDesc, __pyx_v_surfObject); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1124, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1123 + * cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) + * return cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1125 + * if usePTDS: + * return ptds._cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) + * return cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetSurfaceObjectResourceDesc(__pyx_v_pResDesc, __pyx_v_surfObject); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1121 + * return cudaDestroySurfaceObject(surfObject) + * + * cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetSurfaceObjectResourceDesc", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1127 + * return cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) + * + * cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDriverGetVersion(int *__pyx_v_driverVersion) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1128 + * + * cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDriverGetVersion(driverVersion) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1128, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1129 + * cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDriverGetVersion(driverVersion) + * return cudaDriverGetVersion(driverVersion) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1130 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDriverGetVersion(driverVersion) # <<<<<<<<<<<<<< + * return cudaDriverGetVersion(driverVersion) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDriverGetVersion(__pyx_v_driverVersion); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1130, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1129 + * cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDriverGetVersion(driverVersion) + * return cudaDriverGetVersion(driverVersion) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1131 + * if usePTDS: + * return ptds._cudaDriverGetVersion(driverVersion) + * return cudaDriverGetVersion(driverVersion) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDriverGetVersion(__pyx_v_driverVersion); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1127 + * return cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) + * + * cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDriverGetVersion", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1133 + * return cudaDriverGetVersion(driverVersion) + * + * cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaRuntimeGetVersion(int *__pyx_v_runtimeVersion) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1134 + * + * cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaRuntimeGetVersion(runtimeVersion) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1134, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1135 + * cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaRuntimeGetVersion(runtimeVersion) + * return cudaRuntimeGetVersion(runtimeVersion) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1136 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaRuntimeGetVersion(runtimeVersion) # <<<<<<<<<<<<<< + * return cudaRuntimeGetVersion(runtimeVersion) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaRuntimeGetVersion(__pyx_v_runtimeVersion); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1136, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1135 + * cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaRuntimeGetVersion(runtimeVersion) + * return cudaRuntimeGetVersion(runtimeVersion) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1137 + * if usePTDS: + * return ptds._cudaRuntimeGetVersion(runtimeVersion) + * return cudaRuntimeGetVersion(runtimeVersion) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaRuntimeGetVersion(__pyx_v_runtimeVersion); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1133 + * return cudaDriverGetVersion(driverVersion) + * + * cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaRuntimeGetVersion", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1139 + * return cudaRuntimeGetVersion(runtimeVersion) + * + * cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphCreate(cudaGraph_t *__pyx_v_pGraph, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1140 + * + * cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphCreate(pGraph, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1140, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1141 + * cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphCreate(pGraph, flags) + * return cudaGraphCreate(pGraph, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1142 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphCreate(pGraph, flags) # <<<<<<<<<<<<<< + * return cudaGraphCreate(pGraph, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphCreate(__pyx_v_pGraph, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1142, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1141 + * cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphCreate(pGraph, flags) + * return cudaGraphCreate(pGraph, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1143 + * if usePTDS: + * return ptds._cudaGraphCreate(pGraph, flags) + * return cudaGraphCreate(pGraph, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphCreate(__pyx_v_pGraph, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1139 + * return cudaRuntimeGetVersion(runtimeVersion) + * + * cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1145 + * return cudaGraphCreate(pGraph, flags) + * + * cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddKernelNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaKernelNodeParams const *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1146 + * + * cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1146, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1147 + * cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * return cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1148 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddKernelNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1148, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1147 + * cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * return cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1149 + * if usePTDS: + * return ptds._cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * return cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddKernelNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1145 + * return cudaGraphCreate(pGraph, flags) + * + * cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddKernelNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1151 + * return cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * + * cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphKernelNodeGetParams(cudaGraphNode_t __pyx_v_node, struct cudaKernelNodeParams *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1152 + * + * cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphKernelNodeGetParams(node, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1152, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1153 + * cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphKernelNodeGetParams(node, pNodeParams) + * return cudaGraphKernelNodeGetParams(node, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1154 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphKernelNodeGetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeGetParams(node, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeGetParams(__pyx_v_node, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1154, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1153 + * cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphKernelNodeGetParams(node, pNodeParams) + * return cudaGraphKernelNodeGetParams(node, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1155 + * if usePTDS: + * return ptds._cudaGraphKernelNodeGetParams(node, pNodeParams) + * return cudaGraphKernelNodeGetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphKernelNodeGetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1151 + * return cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * + * cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphKernelNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1157 + * return cudaGraphKernelNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphKernelNodeSetParams(cudaGraphNode_t __pyx_v_node, struct cudaKernelNodeParams const *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1158 + * + * cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphKernelNodeSetParams(node, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1158, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1159 + * cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphKernelNodeSetParams(node, pNodeParams) + * return cudaGraphKernelNodeSetParams(node, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1160 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphKernelNodeSetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeSetParams(node, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeSetParams(__pyx_v_node, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1160, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1159 + * cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphKernelNodeSetParams(node, pNodeParams) + * return cudaGraphKernelNodeSetParams(node, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1161 + * if usePTDS: + * return ptds._cudaGraphKernelNodeSetParams(node, pNodeParams) + * return cudaGraphKernelNodeSetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphKernelNodeSetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1157 + * return cudaGraphKernelNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphKernelNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1163 + * return cudaGraphKernelNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t __pyx_v_hSrc, cudaGraphNode_t __pyx_v_hDst) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1164 + * + * cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphKernelNodeCopyAttributes(hSrc, hDst) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1164, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1165 + * cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphKernelNodeCopyAttributes(hSrc, hDst) + * return cudaGraphKernelNodeCopyAttributes(hSrc, hDst) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1166 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphKernelNodeCopyAttributes(hSrc, hDst) # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeCopyAttributes(hSrc, hDst) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeCopyAttributes(__pyx_v_hSrc, __pyx_v_hDst); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1166, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1165 + * cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphKernelNodeCopyAttributes(hSrc, hDst) + * return cudaGraphKernelNodeCopyAttributes(hSrc, hDst) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1167 + * if usePTDS: + * return ptds._cudaGraphKernelNodeCopyAttributes(hSrc, hDst) + * return cudaGraphKernelNodeCopyAttributes(hSrc, hDst) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphKernelNodeCopyAttributes(__pyx_v_hSrc, __pyx_v_hDst); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1163 + * return cudaGraphKernelNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphKernelNodeCopyAttributes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1169 + * return cudaGraphKernelNodeCopyAttributes(hSrc, hDst) + * + * cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphKernelNodeGetAttribute(cudaGraphNode_t __pyx_v_hNode, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaKernelNodeAttrID __pyx_v_attr, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaKernelNodeAttrValue *__pyx_v_value_out) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1170 + * + * cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1170, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1171 + * cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) + * return cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1172 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeGetAttribute(__pyx_v_hNode, ((__pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrID)__pyx_v_attr), __pyx_v_value_out); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1172, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1171 + * cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) + * return cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1173 + * if usePTDS: + * return ptds._cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) + * return cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphKernelNodeGetAttribute(__pyx_v_hNode, __pyx_v_attr, __pyx_v_value_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1169 + * return cudaGraphKernelNodeCopyAttributes(hSrc, hDst) + * + * cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphKernelNodeGetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1175 + * return cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) + * + * cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphKernelNodeSetAttribute(cudaGraphNode_t __pyx_v_hNode, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaKernelNodeAttrID __pyx_v_attr, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaKernelNodeAttrValue const *__pyx_v_value) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1176 + * + * cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphKernelNodeSetAttribute(hNode, attr, value) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1176, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1177 + * cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphKernelNodeSetAttribute(hNode, attr, value) + * return cudaGraphKernelNodeSetAttribute(hNode, attr, value) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1178 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphKernelNodeSetAttribute(hNode, attr, value) # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeSetAttribute(hNode, attr, value) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeSetAttribute(__pyx_v_hNode, ((__pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrID)__pyx_v_attr), __pyx_v_value); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1178, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1177 + * cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphKernelNodeSetAttribute(hNode, attr, value) + * return cudaGraphKernelNodeSetAttribute(hNode, attr, value) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1179 + * if usePTDS: + * return ptds._cudaGraphKernelNodeSetAttribute(hNode, attr, value) + * return cudaGraphKernelNodeSetAttribute(hNode, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphKernelNodeSetAttribute(__pyx_v_hNode, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1175 + * return cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) + * + * cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphKernelNodeSetAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1181 + * return cudaGraphKernelNodeSetAttribute(hNode, attr, value) + * + * cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddMemcpyNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaMemcpy3DParms const *__pyx_v_pCopyParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1182 + * + * cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1182, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1183 + * cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) + * return cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1184 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) # <<<<<<<<<<<<<< + * return cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemcpyNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_pCopyParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1184, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1183 + * cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) + * return cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1185 + * if usePTDS: + * return ptds._cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) + * return cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddMemcpyNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_pCopyParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1181 + * return cudaGraphKernelNodeSetAttribute(hNode, attr, value) + * + * cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddMemcpyNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1187 + * return cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) + * + * cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddMemcpyNode1D(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, void *__pyx_v_dst, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1188 + * + * cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1188, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1189 + * cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) + * return cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1190 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) # <<<<<<<<<<<<<< + * return cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemcpyNode1D(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_dst, __pyx_v_src, __pyx_v_count, ((enum cudaMemcpyKind)__pyx_v_kind)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1190, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1189 + * cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) + * return cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1191 + * if usePTDS: + * return ptds._cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) + * return cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddMemcpyNode1D(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_dst, __pyx_v_src, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1187 + * return cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) + * + * cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddMemcpyNode1D", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1193 + * return cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) + * + * cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemcpyNodeGetParams(cudaGraphNode_t __pyx_v_node, struct cudaMemcpy3DParms *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1194 + * + * cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphMemcpyNodeGetParams(node, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1194, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1195 + * cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemcpyNodeGetParams(node, pNodeParams) + * return cudaGraphMemcpyNodeGetParams(node, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1196 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphMemcpyNodeGetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphMemcpyNodeGetParams(node, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeGetParams(__pyx_v_node, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1196, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1195 + * cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemcpyNodeGetParams(node, pNodeParams) + * return cudaGraphMemcpyNodeGetParams(node, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1197 + * if usePTDS: + * return ptds._cudaGraphMemcpyNodeGetParams(node, pNodeParams) + * return cudaGraphMemcpyNodeGetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemcpyNodeGetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1193 + * return cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) + * + * cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphMemcpyNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1199 + * return cudaGraphMemcpyNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemcpyNodeSetParams(cudaGraphNode_t __pyx_v_node, struct cudaMemcpy3DParms const *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1200 + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphMemcpyNodeSetParams(node, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1200, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1201 + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemcpyNodeSetParams(node, pNodeParams) + * return cudaGraphMemcpyNodeSetParams(node, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1202 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphMemcpyNodeSetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphMemcpyNodeSetParams(node, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeSetParams(__pyx_v_node, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1202, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1201 + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemcpyNodeSetParams(node, pNodeParams) + * return cudaGraphMemcpyNodeSetParams(node, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1203 + * if usePTDS: + * return ptds._cudaGraphMemcpyNodeSetParams(node, pNodeParams) + * return cudaGraphMemcpyNodeSetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemcpyNodeSetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1199 + * return cudaGraphMemcpyNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphMemcpyNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1205 + * return cudaGraphMemcpyNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t __pyx_v_node, void *__pyx_v_dst, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1206 + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1206, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1207 + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) + * return cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1208 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) # <<<<<<<<<<<<<< + * return cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeSetParams1D(__pyx_v_node, __pyx_v_dst, __pyx_v_src, __pyx_v_count, ((enum cudaMemcpyKind)__pyx_v_kind)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1208, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1207 + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) + * return cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1209 + * if usePTDS: + * return ptds._cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) + * return cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemcpyNodeSetParams1D(__pyx_v_node, __pyx_v_dst, __pyx_v_src, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1205 + * return cudaGraphMemcpyNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphMemcpyNodeSetParams1D", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1211 + * return cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) + * + * cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddMemsetNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaMemsetParams const *__pyx_v_pMemsetParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1212 + * + * cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1212, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1213 + * cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) + * return cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1214 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) # <<<<<<<<<<<<<< + * return cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemsetNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_pMemsetParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1214, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1213 + * cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) + * return cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1215 + * if usePTDS: + * return ptds._cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) + * return cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddMemsetNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_pMemsetParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1211 + * return cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) + * + * cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddMemsetNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1217 + * return cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) + * + * cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemsetNodeGetParams(cudaGraphNode_t __pyx_v_node, struct cudaMemsetParams *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1218 + * + * cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphMemsetNodeGetParams(node, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1218, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1219 + * cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemsetNodeGetParams(node, pNodeParams) + * return cudaGraphMemsetNodeGetParams(node, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1220 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphMemsetNodeGetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphMemsetNodeGetParams(node, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemsetNodeGetParams(__pyx_v_node, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1220, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1219 + * cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemsetNodeGetParams(node, pNodeParams) + * return cudaGraphMemsetNodeGetParams(node, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1221 + * if usePTDS: + * return ptds._cudaGraphMemsetNodeGetParams(node, pNodeParams) + * return cudaGraphMemsetNodeGetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemsetNodeGetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1217 + * return cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) + * + * cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphMemsetNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1223 + * return cudaGraphMemsetNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemsetNodeSetParams(cudaGraphNode_t __pyx_v_node, struct cudaMemsetParams const *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1224 + * + * cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphMemsetNodeSetParams(node, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1224, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1225 + * cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemsetNodeSetParams(node, pNodeParams) + * return cudaGraphMemsetNodeSetParams(node, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1226 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphMemsetNodeSetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphMemsetNodeSetParams(node, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemsetNodeSetParams(__pyx_v_node, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1226, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1225 + * cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemsetNodeSetParams(node, pNodeParams) + * return cudaGraphMemsetNodeSetParams(node, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1227 + * if usePTDS: + * return ptds._cudaGraphMemsetNodeSetParams(node, pNodeParams) + * return cudaGraphMemsetNodeSetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemsetNodeSetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1223 + * return cudaGraphMemsetNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphMemsetNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1229 + * return cudaGraphMemsetNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddHostNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaHostNodeParams const *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1230 + * + * cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1230, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1231 + * cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * return cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1232 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddHostNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1232, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1231 + * cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * return cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1233 + * if usePTDS: + * return ptds._cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * return cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddHostNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1229 + * return cudaGraphMemsetNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddHostNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1235 + * return cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * + * cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphHostNodeGetParams(cudaGraphNode_t __pyx_v_node, struct cudaHostNodeParams *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1236 + * + * cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphHostNodeGetParams(node, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1236, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1237 + * cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphHostNodeGetParams(node, pNodeParams) + * return cudaGraphHostNodeGetParams(node, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1238 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphHostNodeGetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphHostNodeGetParams(node, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphHostNodeGetParams(__pyx_v_node, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1238, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1237 + * cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphHostNodeGetParams(node, pNodeParams) + * return cudaGraphHostNodeGetParams(node, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1239 + * if usePTDS: + * return ptds._cudaGraphHostNodeGetParams(node, pNodeParams) + * return cudaGraphHostNodeGetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphHostNodeGetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1235 + * return cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * + * cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphHostNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1241 + * return cudaGraphHostNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphHostNodeSetParams(cudaGraphNode_t __pyx_v_node, struct cudaHostNodeParams const *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1242 + * + * cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphHostNodeSetParams(node, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1242, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1243 + * cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphHostNodeSetParams(node, pNodeParams) + * return cudaGraphHostNodeSetParams(node, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1244 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphHostNodeSetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphHostNodeSetParams(node, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphHostNodeSetParams(__pyx_v_node, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1244, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1243 + * cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphHostNodeSetParams(node, pNodeParams) + * return cudaGraphHostNodeSetParams(node, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1245 + * if usePTDS: + * return ptds._cudaGraphHostNodeSetParams(node, pNodeParams) + * return cudaGraphHostNodeSetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphHostNodeSetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1241 + * return cudaGraphHostNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphHostNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1247 + * return cudaGraphHostNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddChildGraphNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, cudaGraph_t __pyx_v_childGraph) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1248 + * + * cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1248, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1249 + * cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) + * return cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1250 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) # <<<<<<<<<<<<<< + * return cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddChildGraphNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_childGraph); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1250, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1249 + * cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) + * return cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1251 + * if usePTDS: + * return ptds._cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) + * return cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddChildGraphNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_childGraph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1247 + * return cudaGraphHostNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddChildGraphNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1253 + * return cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) + * + * cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t __pyx_v_node, cudaGraph_t *__pyx_v_pGraph) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1254 + * + * cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphChildGraphNodeGetGraph(node, pGraph) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1254, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1255 + * cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphChildGraphNodeGetGraph(node, pGraph) + * return cudaGraphChildGraphNodeGetGraph(node, pGraph) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1256 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphChildGraphNodeGetGraph(node, pGraph) # <<<<<<<<<<<<<< + * return cudaGraphChildGraphNodeGetGraph(node, pGraph) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphChildGraphNodeGetGraph(__pyx_v_node, __pyx_v_pGraph); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1256, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1255 + * cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphChildGraphNodeGetGraph(node, pGraph) + * return cudaGraphChildGraphNodeGetGraph(node, pGraph) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1257 + * if usePTDS: + * return ptds._cudaGraphChildGraphNodeGetGraph(node, pGraph) + * return cudaGraphChildGraphNodeGetGraph(node, pGraph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphChildGraphNodeGetGraph(__pyx_v_node, __pyx_v_pGraph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1253 + * return cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) + * + * cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphChildGraphNodeGetGraph", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1259 + * return cudaGraphChildGraphNodeGetGraph(node, pGraph) + * + * cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddEmptyNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1260 + * + * cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1260, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1261 + * cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) + * return cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1262 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) # <<<<<<<<<<<<<< + * return cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEmptyNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1262, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1261 + * cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) + * return cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1263 + * if usePTDS: + * return ptds._cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) + * return cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddEmptyNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1259 + * return cudaGraphChildGraphNodeGetGraph(node, pGraph) + * + * cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddEmptyNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1265 + * return cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) + * + * cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddEventRecordNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, cudaEvent_t __pyx_v_event) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1266 + * + * cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1266, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1267 + * cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) + * return cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1268 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) # <<<<<<<<<<<<<< + * return cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEventRecordNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_event); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1268, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1267 + * cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) + * return cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1269 + * if usePTDS: + * return ptds._cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) + * return cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddEventRecordNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1265 + * return cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) + * + * cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddEventRecordNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1271 + * return cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) + * + * cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t __pyx_v_node, cudaEvent_t *__pyx_v_event_out) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1272 + * + * cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphEventRecordNodeGetEvent(node, event_out) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1272, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1273 + * cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphEventRecordNodeGetEvent(node, event_out) + * return cudaGraphEventRecordNodeGetEvent(node, event_out) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1274 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphEventRecordNodeGetEvent(node, event_out) # <<<<<<<<<<<<<< + * return cudaGraphEventRecordNodeGetEvent(node, event_out) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventRecordNodeGetEvent(__pyx_v_node, __pyx_v_event_out); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1274, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1273 + * cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphEventRecordNodeGetEvent(node, event_out) + * return cudaGraphEventRecordNodeGetEvent(node, event_out) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1275 + * if usePTDS: + * return ptds._cudaGraphEventRecordNodeGetEvent(node, event_out) + * return cudaGraphEventRecordNodeGetEvent(node, event_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphEventRecordNodeGetEvent(__pyx_v_node, __pyx_v_event_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1271 + * return cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) + * + * cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphEventRecordNodeGetEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1277 + * return cudaGraphEventRecordNodeGetEvent(node, event_out) + * + * cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t __pyx_v_node, cudaEvent_t __pyx_v_event) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1278 + * + * cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphEventRecordNodeSetEvent(node, event) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1278, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1279 + * cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphEventRecordNodeSetEvent(node, event) + * return cudaGraphEventRecordNodeSetEvent(node, event) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1280 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphEventRecordNodeSetEvent(node, event) # <<<<<<<<<<<<<< + * return cudaGraphEventRecordNodeSetEvent(node, event) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventRecordNodeSetEvent(__pyx_v_node, __pyx_v_event); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1280, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1279 + * cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphEventRecordNodeSetEvent(node, event) + * return cudaGraphEventRecordNodeSetEvent(node, event) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1281 + * if usePTDS: + * return ptds._cudaGraphEventRecordNodeSetEvent(node, event) + * return cudaGraphEventRecordNodeSetEvent(node, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphEventRecordNodeSetEvent(__pyx_v_node, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1277 + * return cudaGraphEventRecordNodeGetEvent(node, event_out) + * + * cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphEventRecordNodeSetEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1283 + * return cudaGraphEventRecordNodeSetEvent(node, event) + * + * cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddEventWaitNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, cudaEvent_t __pyx_v_event) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1284 + * + * cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1284, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1285 + * cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) + * return cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1286 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) # <<<<<<<<<<<<<< + * return cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEventWaitNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_event); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1286, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1285 + * cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) + * return cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1287 + * if usePTDS: + * return ptds._cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) + * return cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddEventWaitNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1283 + * return cudaGraphEventRecordNodeSetEvent(node, event) + * + * cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddEventWaitNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1289 + * return cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) + * + * cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t __pyx_v_node, cudaEvent_t *__pyx_v_event_out) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1290 + * + * cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphEventWaitNodeGetEvent(node, event_out) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1290, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1291 + * cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphEventWaitNodeGetEvent(node, event_out) + * return cudaGraphEventWaitNodeGetEvent(node, event_out) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1292 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphEventWaitNodeGetEvent(node, event_out) # <<<<<<<<<<<<<< + * return cudaGraphEventWaitNodeGetEvent(node, event_out) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventWaitNodeGetEvent(__pyx_v_node, __pyx_v_event_out); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1292, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1291 + * cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphEventWaitNodeGetEvent(node, event_out) + * return cudaGraphEventWaitNodeGetEvent(node, event_out) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1293 + * if usePTDS: + * return ptds._cudaGraphEventWaitNodeGetEvent(node, event_out) + * return cudaGraphEventWaitNodeGetEvent(node, event_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphEventWaitNodeGetEvent(__pyx_v_node, __pyx_v_event_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1289 + * return cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) + * + * cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphEventWaitNodeGetEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1295 + * return cudaGraphEventWaitNodeGetEvent(node, event_out) + * + * cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t __pyx_v_node, cudaEvent_t __pyx_v_event) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1296 + * + * cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphEventWaitNodeSetEvent(node, event) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1296, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1297 + * cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphEventWaitNodeSetEvent(node, event) + * return cudaGraphEventWaitNodeSetEvent(node, event) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1298 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphEventWaitNodeSetEvent(node, event) # <<<<<<<<<<<<<< + * return cudaGraphEventWaitNodeSetEvent(node, event) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventWaitNodeSetEvent(__pyx_v_node, __pyx_v_event); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1298, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1297 + * cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphEventWaitNodeSetEvent(node, event) + * return cudaGraphEventWaitNodeSetEvent(node, event) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1299 + * if usePTDS: + * return ptds._cudaGraphEventWaitNodeSetEvent(node, event) + * return cudaGraphEventWaitNodeSetEvent(node, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphEventWaitNodeSetEvent(__pyx_v_node, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1295 + * return cudaGraphEventWaitNodeGetEvent(node, event_out) + * + * cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphEventWaitNodeSetEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1301 + * return cudaGraphEventWaitNodeSetEvent(node, event) + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaExternalSemaphoreSignalNodeParams const *__pyx_v_nodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1302 + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1302, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1303 + * cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * return cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1304 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * return cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddExternalSemaphoresSignalNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1304, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1303 + * cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * return cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1305 + * if usePTDS: + * return ptds._cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * return cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddExternalSemaphoresSignalNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1301 + * return cudaGraphEventWaitNodeSetEvent(node, event) + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddExternalSemaphoresSignalNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1307 + * return cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t __pyx_v_hNode, struct cudaExternalSemaphoreSignalNodeParams *__pyx_v_params_out) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1308 + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1308, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1309 + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) + * return cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1310 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) # <<<<<<<<<<<<<< + * return cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresSignalNodeGetParams(__pyx_v_hNode, __pyx_v_params_out); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1310, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1309 + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) + * return cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1311 + * if usePTDS: + * return ptds._cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) + * return cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExternalSemaphoresSignalNodeGetParams(__pyx_v_hNode, __pyx_v_params_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1307 + * return cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExternalSemaphoresSignalNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1313 + * return cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t __pyx_v_hNode, struct cudaExternalSemaphoreSignalNodeParams const *__pyx_v_nodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1314 + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1314, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1315 + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) + * return cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1316 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) # <<<<<<<<<<<<<< + * return cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresSignalNodeSetParams(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1316, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1315 + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) + * return cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1317 + * if usePTDS: + * return ptds._cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) + * return cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExternalSemaphoresSignalNodeSetParams(__pyx_v_hNode, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1313 + * return cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExternalSemaphoresSignalNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1319 + * return cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaExternalSemaphoreWaitNodeParams const *__pyx_v_nodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1320 + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1320, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1321 + * cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * return cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1322 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * return cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddExternalSemaphoresWaitNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1322, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1321 + * cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * return cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1323 + * if usePTDS: + * return ptds._cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * return cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddExternalSemaphoresWaitNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1319 + * return cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddExternalSemaphoresWaitNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1325 + * return cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t __pyx_v_hNode, struct cudaExternalSemaphoreWaitNodeParams *__pyx_v_params_out) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1326 + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1326, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1327 + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) + * return cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1328 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) # <<<<<<<<<<<<<< + * return cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresWaitNodeGetParams(__pyx_v_hNode, __pyx_v_params_out); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1328, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1327 + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) + * return cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1329 + * if usePTDS: + * return ptds._cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) + * return cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExternalSemaphoresWaitNodeGetParams(__pyx_v_hNode, __pyx_v_params_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1325 + * return cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExternalSemaphoresWaitNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1331 + * return cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t __pyx_v_hNode, struct cudaExternalSemaphoreWaitNodeParams const *__pyx_v_nodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1332 + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1332, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1333 + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) + * return cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1334 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) # <<<<<<<<<<<<<< + * return cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresWaitNodeSetParams(__pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1334, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1333 + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) + * return cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1335 + * if usePTDS: + * return ptds._cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) + * return cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExternalSemaphoresWaitNodeSetParams(__pyx_v_hNode, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1331 + * return cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExternalSemaphoresWaitNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1337 + * return cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddMemAllocNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaMemAllocNodeParams *__pyx_v_nodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1338 + * + * cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1338, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1339 + * cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * return cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1340 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * return cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemAllocNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1340, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1339 + * cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * return cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1341 + * if usePTDS: + * return ptds._cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * return cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddMemAllocNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1337 + * return cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddMemAllocNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1343 + * return cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemAllocNodeGetParams(cudaGraphNode_t __pyx_v_node, struct cudaMemAllocNodeParams *__pyx_v_params_out) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1344 + * + * cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphMemAllocNodeGetParams(node, params_out) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1344, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1345 + * cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemAllocNodeGetParams(node, params_out) + * return cudaGraphMemAllocNodeGetParams(node, params_out) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1346 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphMemAllocNodeGetParams(node, params_out) # <<<<<<<<<<<<<< + * return cudaGraphMemAllocNodeGetParams(node, params_out) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemAllocNodeGetParams(__pyx_v_node, __pyx_v_params_out); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1346, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1345 + * cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemAllocNodeGetParams(node, params_out) + * return cudaGraphMemAllocNodeGetParams(node, params_out) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1347 + * if usePTDS: + * return ptds._cudaGraphMemAllocNodeGetParams(node, params_out) + * return cudaGraphMemAllocNodeGetParams(node, params_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemAllocNodeGetParams(__pyx_v_node, __pyx_v_params_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1343 + * return cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphMemAllocNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1349 + * return cudaGraphMemAllocNodeGetParams(node, params_out) + * + * cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddMemFreeNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, void *__pyx_v_dptr) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1350 + * + * cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1350, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1351 + * cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) + * return cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1352 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) # <<<<<<<<<<<<<< + * return cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemFreeNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_dptr); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1352, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1351 + * cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) + * return cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1353 + * if usePTDS: + * return ptds._cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) + * return cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddMemFreeNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_dptr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1349 + * return cudaGraphMemAllocNodeGetParams(node, params_out) + * + * cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddMemFreeNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1355 + * return cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) + * + * cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemFreeNodeGetParams(cudaGraphNode_t __pyx_v_node, void *__pyx_v_dptr_out) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1356 + * + * cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphMemFreeNodeGetParams(node, dptr_out) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1356, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1357 + * cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemFreeNodeGetParams(node, dptr_out) + * return cudaGraphMemFreeNodeGetParams(node, dptr_out) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1358 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphMemFreeNodeGetParams(node, dptr_out) # <<<<<<<<<<<<<< + * return cudaGraphMemFreeNodeGetParams(node, dptr_out) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemFreeNodeGetParams(__pyx_v_node, __pyx_v_dptr_out); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1358, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1357 + * cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphMemFreeNodeGetParams(node, dptr_out) + * return cudaGraphMemFreeNodeGetParams(node, dptr_out) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1359 + * if usePTDS: + * return ptds._cudaGraphMemFreeNodeGetParams(node, dptr_out) + * return cudaGraphMemFreeNodeGetParams(node, dptr_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemFreeNodeGetParams(__pyx_v_node, __pyx_v_dptr_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1355 + * return cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) + * + * cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphMemFreeNodeGetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1361 + * return cudaGraphMemFreeNodeGetParams(node, dptr_out) + * + * cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGraphMemTrim(int __pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1362 + * + * cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGraphMemTrim(device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1362, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1363 + * cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGraphMemTrim(device) + * return cudaDeviceGraphMemTrim(device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1364 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGraphMemTrim(device) # <<<<<<<<<<<<<< + * return cudaDeviceGraphMemTrim(device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGraphMemTrim(__pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1364, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1363 + * cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGraphMemTrim(device) + * return cudaDeviceGraphMemTrim(device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1365 + * if usePTDS: + * return ptds._cudaDeviceGraphMemTrim(device) + * return cudaDeviceGraphMemTrim(device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGraphMemTrim(__pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1361 + * return cudaGraphMemFreeNodeGetParams(node, dptr_out) + * + * cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGraphMemTrim", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1367 + * return cudaDeviceGraphMemTrim(device) + * + * cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetGraphMemAttribute(int __pyx_v_device, enum cudaGraphMemAttributeType __pyx_v_attr, void *__pyx_v_value) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1368 + * + * cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceGetGraphMemAttribute(device, attr, value) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1368, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1369 + * cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetGraphMemAttribute(device, attr, value) + * return cudaDeviceGetGraphMemAttribute(device, attr, value) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1370 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceGetGraphMemAttribute(device, attr, value) # <<<<<<<<<<<<<< + * return cudaDeviceGetGraphMemAttribute(device, attr, value) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetGraphMemAttribute(__pyx_v_device, ((enum cudaGraphMemAttributeType)__pyx_v_attr), __pyx_v_value); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1370, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1369 + * cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceGetGraphMemAttribute(device, attr, value) + * return cudaDeviceGetGraphMemAttribute(device, attr, value) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1371 + * if usePTDS: + * return ptds._cudaDeviceGetGraphMemAttribute(device, attr, value) + * return cudaDeviceGetGraphMemAttribute(device, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetGraphMemAttribute(__pyx_v_device, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1367 + * return cudaDeviceGraphMemTrim(device) + * + * cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceGetGraphMemAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1373 + * return cudaDeviceGetGraphMemAttribute(device, attr, value) + * + * cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceSetGraphMemAttribute(int __pyx_v_device, enum cudaGraphMemAttributeType __pyx_v_attr, void *__pyx_v_value) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1374 + * + * cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaDeviceSetGraphMemAttribute(device, attr, value) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1374, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1375 + * cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceSetGraphMemAttribute(device, attr, value) + * return cudaDeviceSetGraphMemAttribute(device, attr, value) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1376 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaDeviceSetGraphMemAttribute(device, attr, value) # <<<<<<<<<<<<<< + * return cudaDeviceSetGraphMemAttribute(device, attr, value) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetGraphMemAttribute(__pyx_v_device, ((enum cudaGraphMemAttributeType)__pyx_v_attr), __pyx_v_value); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1376, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1375 + * cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaDeviceSetGraphMemAttribute(device, attr, value) + * return cudaDeviceSetGraphMemAttribute(device, attr, value) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1377 + * if usePTDS: + * return ptds._cudaDeviceSetGraphMemAttribute(device, attr, value) + * return cudaDeviceSetGraphMemAttribute(device, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceSetGraphMemAttribute(__pyx_v_device, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1373 + * return cudaDeviceGetGraphMemAttribute(device, attr, value) + * + * cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaDeviceSetGraphMemAttribute", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1379 + * return cudaDeviceSetGraphMemAttribute(device, attr, value) + * + * cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphClone(cudaGraph_t *__pyx_v_pGraphClone, cudaGraph_t __pyx_v_originalGraph) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1380 + * + * cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphClone(pGraphClone, originalGraph) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1380, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1381 + * cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphClone(pGraphClone, originalGraph) + * return cudaGraphClone(pGraphClone, originalGraph) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1382 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphClone(pGraphClone, originalGraph) # <<<<<<<<<<<<<< + * return cudaGraphClone(pGraphClone, originalGraph) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphClone(__pyx_v_pGraphClone, __pyx_v_originalGraph); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1382, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1381 + * cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphClone(pGraphClone, originalGraph) + * return cudaGraphClone(pGraphClone, originalGraph) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1383 + * if usePTDS: + * return ptds._cudaGraphClone(pGraphClone, originalGraph) + * return cudaGraphClone(pGraphClone, originalGraph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphClone(__pyx_v_pGraphClone, __pyx_v_originalGraph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1379 + * return cudaDeviceSetGraphMemAttribute(device, attr, value) + * + * cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphClone", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1385 + * return cudaGraphClone(pGraphClone, originalGraph) + * + * cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeFindInClone(cudaGraphNode_t *__pyx_v_pNode, cudaGraphNode_t __pyx_v_originalNode, cudaGraph_t __pyx_v_clonedGraph) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1386 + * + * cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1386, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1387 + * cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) + * return cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1388 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) # <<<<<<<<<<<<<< + * return cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeFindInClone(__pyx_v_pNode, __pyx_v_originalNode, __pyx_v_clonedGraph); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1388, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1387 + * cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) + * return cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1389 + * if usePTDS: + * return ptds._cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) + * return cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeFindInClone(__pyx_v_pNode, __pyx_v_originalNode, __pyx_v_clonedGraph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1385 + * return cudaGraphClone(pGraphClone, originalGraph) + * + * cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphNodeFindInClone", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1391 + * return cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) + * + * cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeGetType(cudaGraphNode_t __pyx_v_node, enum cudaGraphNodeType *__pyx_v_pType) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1392 + * + * cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphNodeGetType(node, pType) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1392, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1393 + * cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeGetType(node, pType) + * return cudaGraphNodeGetType(node, pType) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1394 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphNodeGetType(node, pType) # <<<<<<<<<<<<<< + * return cudaGraphNodeGetType(node, pType) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetType(__pyx_v_node, ((enum cudaGraphNodeType *)__pyx_v_pType)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1394, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1393 + * cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeGetType(node, pType) + * return cudaGraphNodeGetType(node, pType) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1395 + * if usePTDS: + * return ptds._cudaGraphNodeGetType(node, pType) + * return cudaGraphNodeGetType(node, pType) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeGetType(__pyx_v_node, __pyx_v_pType); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1391 + * return cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) + * + * cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphNodeGetType", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1397 + * return cudaGraphNodeGetType(node, pType) + * + * cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphGetNodes(cudaGraph_t __pyx_v_graph, cudaGraphNode_t *__pyx_v_nodes, size_t *__pyx_v_numNodes) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1398 + * + * cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphGetNodes(graph, nodes, numNodes) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1398, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1399 + * cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphGetNodes(graph, nodes, numNodes) + * return cudaGraphGetNodes(graph, nodes, numNodes) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1400 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphGetNodes(graph, nodes, numNodes) # <<<<<<<<<<<<<< + * return cudaGraphGetNodes(graph, nodes, numNodes) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetNodes(__pyx_v_graph, __pyx_v_nodes, __pyx_v_numNodes); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1400, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1399 + * cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphGetNodes(graph, nodes, numNodes) + * return cudaGraphGetNodes(graph, nodes, numNodes) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1401 + * if usePTDS: + * return ptds._cudaGraphGetNodes(graph, nodes, numNodes) + * return cudaGraphGetNodes(graph, nodes, numNodes) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphGetNodes(__pyx_v_graph, __pyx_v_nodes, __pyx_v_numNodes); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1397 + * return cudaGraphNodeGetType(node, pType) + * + * cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphGetNodes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1403 + * return cudaGraphGetNodes(graph, nodes, numNodes) + * + * cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphGetRootNodes(cudaGraph_t __pyx_v_graph, cudaGraphNode_t *__pyx_v_pRootNodes, size_t *__pyx_v_pNumRootNodes) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1404 + * + * cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1404, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1405 + * cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) + * return cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1406 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) # <<<<<<<<<<<<<< + * return cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetRootNodes(__pyx_v_graph, __pyx_v_pRootNodes, __pyx_v_pNumRootNodes); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1406, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1405 + * cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) + * return cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1407 + * if usePTDS: + * return ptds._cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) + * return cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphGetRootNodes(__pyx_v_graph, __pyx_v_pRootNodes, __pyx_v_pNumRootNodes); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1403 + * return cudaGraphGetNodes(graph, nodes, numNodes) + * + * cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphGetRootNodes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1409 + * return cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) + * + * cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphGetEdges(cudaGraph_t __pyx_v_graph, cudaGraphNode_t *__pyx_v_from_, cudaGraphNode_t *__pyx_v_to, size_t *__pyx_v_numEdges) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1410 + * + * cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphGetEdges(graph, from_, to, numEdges) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1410, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1411 + * cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphGetEdges(graph, from_, to, numEdges) + * return cudaGraphGetEdges(graph, from_, to, numEdges) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1412 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphGetEdges(graph, from_, to, numEdges) # <<<<<<<<<<<<<< + * return cudaGraphGetEdges(graph, from_, to, numEdges) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetEdges(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_numEdges); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1412, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1411 + * cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphGetEdges(graph, from_, to, numEdges) + * return cudaGraphGetEdges(graph, from_, to, numEdges) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1413 + * if usePTDS: + * return ptds._cudaGraphGetEdges(graph, from_, to, numEdges) + * return cudaGraphGetEdges(graph, from_, to, numEdges) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphGetEdges(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_numEdges); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1409 + * return cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) + * + * cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphGetEdges", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1415 + * return cudaGraphGetEdges(graph, from_, to, numEdges) + * + * cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphGetEdges_v2(cudaGraph_t __pyx_v_graph, cudaGraphNode_t *__pyx_v_from_, cudaGraphNode_t *__pyx_v_to, cudaGraphEdgeData *__pyx_v_edgeData, size_t *__pyx_v_numEdges) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1416 + * + * cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1416, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1417 + * cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) + * return cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1418 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) # <<<<<<<<<<<<<< + * return cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetEdges_v2(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_edgeData, __pyx_v_numEdges); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1418, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1417 + * cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) + * return cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1419 + * if usePTDS: + * return ptds._cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) + * return cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphGetEdges_v2(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_edgeData, __pyx_v_numEdges); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1415 + * return cudaGraphGetEdges(graph, from_, to, numEdges) + * + * cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphGetEdges_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1421 + * return cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) + * + * cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeGetDependencies(cudaGraphNode_t __pyx_v_node, cudaGraphNode_t *__pyx_v_pDependencies, size_t *__pyx_v_pNumDependencies) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1422 + * + * cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1422, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1423 + * cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) + * return cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1424 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) # <<<<<<<<<<<<<< + * return cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependencies(__pyx_v_node, __pyx_v_pDependencies, __pyx_v_pNumDependencies); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1424, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1423 + * cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) + * return cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1425 + * if usePTDS: + * return ptds._cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) + * return cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeGetDependencies(__pyx_v_node, __pyx_v_pDependencies, __pyx_v_pNumDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1421 + * return cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) + * + * cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphNodeGetDependencies", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1427 + * return cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) + * + * cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeGetDependencies_v2(cudaGraphNode_t __pyx_v_node, cudaGraphNode_t *__pyx_v_pDependencies, cudaGraphEdgeData *__pyx_v_edgeData, size_t *__pyx_v_pNumDependencies) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1428 + * + * cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1428, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1429 + * cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) + * return cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1430 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) # <<<<<<<<<<<<<< + * return cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependencies_v2(__pyx_v_node, __pyx_v_pDependencies, __pyx_v_edgeData, __pyx_v_pNumDependencies); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1430, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1429 + * cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) + * return cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1431 + * if usePTDS: + * return ptds._cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) + * return cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeGetDependencies_v2(__pyx_v_node, __pyx_v_pDependencies, __pyx_v_edgeData, __pyx_v_pNumDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1427 + * return cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) + * + * cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphNodeGetDependencies_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1433 + * return cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeGetDependentNodes(cudaGraphNode_t __pyx_v_node, cudaGraphNode_t *__pyx_v_pDependentNodes, size_t *__pyx_v_pNumDependentNodes) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1434 + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1434, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1435 + * cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) + * return cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1436 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) # <<<<<<<<<<<<<< + * return cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependentNodes(__pyx_v_node, __pyx_v_pDependentNodes, __pyx_v_pNumDependentNodes); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1436, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1435 + * cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) + * return cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1437 + * if usePTDS: + * return ptds._cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) + * return cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeGetDependentNodes(__pyx_v_node, __pyx_v_pDependentNodes, __pyx_v_pNumDependentNodes); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1433 + * return cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphNodeGetDependentNodes", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1439 + * return cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t __pyx_v_node, cudaGraphNode_t *__pyx_v_pDependentNodes, cudaGraphEdgeData *__pyx_v_edgeData, size_t *__pyx_v_pNumDependentNodes) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1440 + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1440, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1441 + * cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) + * return cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1442 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) # <<<<<<<<<<<<<< + * return cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependentNodes_v2(__pyx_v_node, __pyx_v_pDependentNodes, __pyx_v_edgeData, __pyx_v_pNumDependentNodes); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1442, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1441 + * cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) + * return cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1443 + * if usePTDS: + * return ptds._cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) + * return cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeGetDependentNodes_v2(__pyx_v_node, __pyx_v_pDependentNodes, __pyx_v_edgeData, __pyx_v_pNumDependentNodes); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1439 + * return cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphNodeGetDependentNodes_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1445 + * return cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) + * + * cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddDependencies(cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_from_, cudaGraphNode_t const *__pyx_v_to, size_t __pyx_v_numDependencies) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1446 + * + * cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddDependencies(graph, from_, to, numDependencies) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1446, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1447 + * cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddDependencies(graph, from_, to, numDependencies) + * return cudaGraphAddDependencies(graph, from_, to, numDependencies) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1448 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddDependencies(graph, from_, to, numDependencies) # <<<<<<<<<<<<<< + * return cudaGraphAddDependencies(graph, from_, to, numDependencies) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddDependencies(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_numDependencies); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1448, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1447 + * cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddDependencies(graph, from_, to, numDependencies) + * return cudaGraphAddDependencies(graph, from_, to, numDependencies) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1449 + * if usePTDS: + * return ptds._cudaGraphAddDependencies(graph, from_, to, numDependencies) + * return cudaGraphAddDependencies(graph, from_, to, numDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddDependencies(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_numDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1445 + * return cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) + * + * cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddDependencies", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1451 + * return cudaGraphAddDependencies(graph, from_, to, numDependencies) + * + * cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddDependencies_v2(cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_from_, cudaGraphNode_t const *__pyx_v_to, cudaGraphEdgeData const *__pyx_v_edgeData, size_t __pyx_v_numDependencies) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1452 + * + * cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1452, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1453 + * cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) + * return cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1454 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) # <<<<<<<<<<<<<< + * return cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddDependencies_v2(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_edgeData, __pyx_v_numDependencies); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1454, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1453 + * cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) + * return cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1455 + * if usePTDS: + * return ptds._cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) + * return cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddDependencies_v2(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_edgeData, __pyx_v_numDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1451 + * return cudaGraphAddDependencies(graph, from_, to, numDependencies) + * + * cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddDependencies_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1457 + * return cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) + * + * cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphRemoveDependencies(cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_from_, cudaGraphNode_t const *__pyx_v_to, size_t __pyx_v_numDependencies) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1458 + * + * cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphRemoveDependencies(graph, from_, to, numDependencies) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1458, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1459 + * cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphRemoveDependencies(graph, from_, to, numDependencies) + * return cudaGraphRemoveDependencies(graph, from_, to, numDependencies) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1460 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphRemoveDependencies(graph, from_, to, numDependencies) # <<<<<<<<<<<<<< + * return cudaGraphRemoveDependencies(graph, from_, to, numDependencies) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRemoveDependencies(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_numDependencies); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1460, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1459 + * cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphRemoveDependencies(graph, from_, to, numDependencies) + * return cudaGraphRemoveDependencies(graph, from_, to, numDependencies) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1461 + * if usePTDS: + * return ptds._cudaGraphRemoveDependencies(graph, from_, to, numDependencies) + * return cudaGraphRemoveDependencies(graph, from_, to, numDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphRemoveDependencies(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_numDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1457 + * return cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) + * + * cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphRemoveDependencies", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1463 + * return cudaGraphRemoveDependencies(graph, from_, to, numDependencies) + * + * cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphRemoveDependencies_v2(cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_from_, cudaGraphNode_t const *__pyx_v_to, cudaGraphEdgeData const *__pyx_v_edgeData, size_t __pyx_v_numDependencies) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1464 + * + * cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1464, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1465 + * cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) + * return cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1466 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) # <<<<<<<<<<<<<< + * return cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRemoveDependencies_v2(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_edgeData, __pyx_v_numDependencies); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1466, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1465 + * cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) + * return cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1467 + * if usePTDS: + * return ptds._cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) + * return cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphRemoveDependencies_v2(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_edgeData, __pyx_v_numDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1463 + * return cudaGraphRemoveDependencies(graph, from_, to, numDependencies) + * + * cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphRemoveDependencies_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1469 + * return cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) + * + * cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphDestroyNode(cudaGraphNode_t __pyx_v_node) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1470 + * + * cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphDestroyNode(node) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1470, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1471 + * cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphDestroyNode(node) + * return cudaGraphDestroyNode(node) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1472 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphDestroyNode(node) # <<<<<<<<<<<<<< + * return cudaGraphDestroyNode(node) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDestroyNode(__pyx_v_node); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1472, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1471 + * cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphDestroyNode(node) + * return cudaGraphDestroyNode(node) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1473 + * if usePTDS: + * return ptds._cudaGraphDestroyNode(node) + * return cudaGraphDestroyNode(node) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphDestroyNode(__pyx_v_node); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1469 + * return cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) + * + * cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphDestroyNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1475 + * return cudaGraphDestroyNode(node) + * + * cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphInstantiate(cudaGraphExec_t *__pyx_v_pGraphExec, cudaGraph_t __pyx_v_graph, unsigned PY_LONG_LONG __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1476 + * + * cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphInstantiate(pGraphExec, graph, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1476, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1477 + * cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphInstantiate(pGraphExec, graph, flags) + * return cudaGraphInstantiate(pGraphExec, graph, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1478 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphInstantiate(pGraphExec, graph, flags) # <<<<<<<<<<<<<< + * return cudaGraphInstantiate(pGraphExec, graph, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiate(__pyx_v_pGraphExec, __pyx_v_graph, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1478, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1477 + * cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphInstantiate(pGraphExec, graph, flags) + * return cudaGraphInstantiate(pGraphExec, graph, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1479 + * if usePTDS: + * return ptds._cudaGraphInstantiate(pGraphExec, graph, flags) + * return cudaGraphInstantiate(pGraphExec, graph, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphInstantiate(__pyx_v_pGraphExec, __pyx_v_graph, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1475 + * return cudaGraphDestroyNode(node) + * + * cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphInstantiate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1481 + * return cudaGraphInstantiate(pGraphExec, graph, flags) + * + * cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphInstantiateWithFlags(cudaGraphExec_t *__pyx_v_pGraphExec, cudaGraph_t __pyx_v_graph, unsigned PY_LONG_LONG __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1482 + * + * cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1482, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1483 + * cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) + * return cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1484 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) # <<<<<<<<<<<<<< + * return cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiateWithFlags(__pyx_v_pGraphExec, __pyx_v_graph, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1484, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1483 + * cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) + * return cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1485 + * if usePTDS: + * return ptds._cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) + * return cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphInstantiateWithFlags(__pyx_v_pGraphExec, __pyx_v_graph, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1481 + * return cudaGraphInstantiate(pGraphExec, graph, flags) + * + * cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphInstantiateWithFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1487 + * return cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) + * + * cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphInstantiateWithParams(cudaGraphExec_t *__pyx_v_pGraphExec, cudaGraph_t __pyx_v_graph, cudaGraphInstantiateParams *__pyx_v_instantiateParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1488 + * + * cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1488, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1489 + * cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) + * return cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1490 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) # <<<<<<<<<<<<<< + * return cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiateWithParams(__pyx_v_pGraphExec, __pyx_v_graph, __pyx_v_instantiateParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1490, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1489 + * cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) + * return cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1491 + * if usePTDS: + * return ptds._cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) + * return cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphInstantiateWithParams(__pyx_v_pGraphExec, __pyx_v_graph, __pyx_v_instantiateParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1487 + * return cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) + * + * cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphInstantiateWithParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1493 + * return cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) + * + * cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecGetFlags(cudaGraphExec_t __pyx_v_graphExec, unsigned PY_LONG_LONG *__pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1494 + * + * cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecGetFlags(graphExec, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1494, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1495 + * cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecGetFlags(graphExec, flags) + * return cudaGraphExecGetFlags(graphExec, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1496 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecGetFlags(graphExec, flags) # <<<<<<<<<<<<<< + * return cudaGraphExecGetFlags(graphExec, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecGetFlags(__pyx_v_graphExec, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1496, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1495 + * cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecGetFlags(graphExec, flags) + * return cudaGraphExecGetFlags(graphExec, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1497 + * if usePTDS: + * return ptds._cudaGraphExecGetFlags(graphExec, flags) + * return cudaGraphExecGetFlags(graphExec, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecGetFlags(__pyx_v_graphExec, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1493 + * return cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) + * + * cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecGetFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1499 + * return cudaGraphExecGetFlags(graphExec, flags) + * + * cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecKernelNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_node, struct cudaKernelNodeParams const *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1500 + * + * cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1500, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1501 + * cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) + * return cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1502 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecKernelNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1502, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1501 + * cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) + * return cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1503 + * if usePTDS: + * return ptds._cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) + * return cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecKernelNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1499 + * return cudaGraphExecGetFlags(graphExec, flags) + * + * cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecKernelNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1505 + * return cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_node, struct cudaMemcpy3DParms const *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1506 + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1506, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1507 + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) + * return cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1508 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemcpyNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1508, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1507 + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) + * return cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1509 + * if usePTDS: + * return ptds._cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) + * return cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecMemcpyNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1505 + * return cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecMemcpyNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1511 + * return cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_node, void *__pyx_v_dst, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1512 + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1512, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1513 + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) + * return cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1514 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) # <<<<<<<<<<<<<< + * return cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemcpyNodeSetParams1D(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_dst, __pyx_v_src, __pyx_v_count, ((enum cudaMemcpyKind)__pyx_v_kind)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1514, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1513 + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) + * return cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1515 + * if usePTDS: + * return ptds._cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) + * return cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecMemcpyNodeSetParams1D(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_dst, __pyx_v_src, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1511 + * return cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecMemcpyNodeSetParams1D", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1517 + * return cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) + * + * cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_node, struct cudaMemsetParams const *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1518 + * + * cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1518, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1519 + * cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) + * return cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1520 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemsetNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1520, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1519 + * cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) + * return cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1521 + * if usePTDS: + * return ptds._cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) + * return cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecMemsetNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1517 + * return cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) + * + * cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecMemsetNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1523 + * return cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecHostNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_node, struct cudaHostNodeParams const *__pyx_v_pNodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1524 + * + * cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1524, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1525 + * cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) + * return cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1526 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) # <<<<<<<<<<<<<< + * return cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecHostNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_pNodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1526, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1525 + * cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) + * return cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1527 + * if usePTDS: + * return ptds._cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) + * return cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecHostNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1523 + * return cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecHostNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1529 + * return cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_node, cudaGraph_t __pyx_v_childGraph) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1530 + * + * cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1530, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1531 + * cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) + * return cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1532 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) # <<<<<<<<<<<<<< + * return cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecChildGraphNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_childGraph); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1532, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1531 + * cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) + * return cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1533 + * if usePTDS: + * return ptds._cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) + * return cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecChildGraphNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_childGraph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1529 + * return cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecChildGraphNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1535 + * return cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) + * + * cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_hNode, cudaEvent_t __pyx_v_event) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1536 + * + * cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1536, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1537 + * cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) + * return cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1538 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) # <<<<<<<<<<<<<< + * return cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecEventRecordNodeSetEvent(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_event); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1538, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1537 + * cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) + * return cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1539 + * if usePTDS: + * return ptds._cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) + * return cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecEventRecordNodeSetEvent(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1535 + * return cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) + * + * cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecEventRecordNodeSetEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1541 + * return cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) + * + * cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_hNode, cudaEvent_t __pyx_v_event) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1542 + * + * cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1542, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1543 + * cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) + * return cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1544 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) # <<<<<<<<<<<<<< + * return cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecEventWaitNodeSetEvent(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_event); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1544, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1543 + * cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) + * return cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1545 + * if usePTDS: + * return ptds._cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) + * return cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecEventWaitNodeSetEvent(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1541 + * return cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) + * + * cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecEventWaitNodeSetEvent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1547 + * return cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_hNode, struct cudaExternalSemaphoreSignalNodeParams const *__pyx_v_nodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1548 + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1548, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1549 + * cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) + * return cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1550 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) # <<<<<<<<<<<<<< + * return cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecExternalSemaphoresSignalNodeSetParams(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1550, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1549 + * cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) + * return cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1551 + * if usePTDS: + * return ptds._cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) + * return cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecExternalSemaphoresSignalNodeSetParams(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1547 + * return cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecExternalSemaphoresSignalNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1553 + * return cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_hNode, struct cudaExternalSemaphoreWaitNodeParams const *__pyx_v_nodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1554 + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1554, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1555 + * cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) + * return cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1556 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) # <<<<<<<<<<<<<< + * return cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecExternalSemaphoresWaitNodeSetParams(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_nodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1556, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1555 + * cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) + * return cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1557 + * if usePTDS: + * return ptds._cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) + * return cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecExternalSemaphoresWaitNodeSetParams(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1553 + * return cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecExternalSemaphoresWaitNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1559 + * return cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeSetEnabled(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_hNode, unsigned int __pyx_v_isEnabled) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1560 + * + * cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1560, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1561 + * cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) + * return cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1562 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) # <<<<<<<<<<<<<< + * return cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeSetEnabled(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_isEnabled); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1562, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1561 + * cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) + * return cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1563 + * if usePTDS: + * return ptds._cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) + * return cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeSetEnabled(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_isEnabled); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1559 + * return cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphNodeSetEnabled", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1565 + * return cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) + * + * cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeGetEnabled(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_hNode, unsigned int *__pyx_v_isEnabled) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1566 + * + * cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1566, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1567 + * cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) + * return cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1568 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) # <<<<<<<<<<<<<< + * return cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetEnabled(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_isEnabled); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1568, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1567 + * cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) + * return cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1569 + * if usePTDS: + * return ptds._cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) + * return cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeGetEnabled(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_isEnabled); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1565 + * return cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) + * + * cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphNodeGetEnabled", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1571 + * return cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) + * + * cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecUpdate(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraph_t __pyx_v_hGraph, cudaGraphExecUpdateResultInfo *__pyx_v_resultInfo) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1572 + * + * cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1572, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1573 + * cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) + * return cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1574 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) # <<<<<<<<<<<<<< + * return cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecUpdate(__pyx_v_hGraphExec, __pyx_v_hGraph, __pyx_v_resultInfo); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1574, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1573 + * cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) + * return cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1575 + * if usePTDS: + * return ptds._cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) + * return cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecUpdate(__pyx_v_hGraphExec, __pyx_v_hGraph, __pyx_v_resultInfo); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1571 + * return cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) + * + * cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecUpdate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1577 + * return cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) + * + * cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphUpload(cudaGraphExec_t __pyx_v_graphExec, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1578 + * + * cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphUpload(graphExec, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1578, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1579 + * cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphUpload(graphExec, stream) + * return cudaGraphUpload(graphExec, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1580 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphUpload(graphExec, stream) # <<<<<<<<<<<<<< + * return cudaGraphUpload(graphExec, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphUpload(__pyx_v_graphExec, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1580, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1579 + * cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphUpload(graphExec, stream) + * return cudaGraphUpload(graphExec, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1581 + * if usePTDS: + * return ptds._cudaGraphUpload(graphExec, stream) + * return cudaGraphUpload(graphExec, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphUpload(__pyx_v_graphExec, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1577 + * return cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) + * + * cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphUpload", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1583 + * return cudaGraphUpload(graphExec, stream) + * + * cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphLaunch(cudaGraphExec_t __pyx_v_graphExec, cudaStream_t __pyx_v_stream) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1584 + * + * cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphLaunch(graphExec, stream) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1584, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1585 + * cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphLaunch(graphExec, stream) + * return cudaGraphLaunch(graphExec, stream) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1586 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphLaunch(graphExec, stream) # <<<<<<<<<<<<<< + * return cudaGraphLaunch(graphExec, stream) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphLaunch(__pyx_v_graphExec, __pyx_v_stream); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1586, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1585 + * cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphLaunch(graphExec, stream) + * return cudaGraphLaunch(graphExec, stream) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1587 + * if usePTDS: + * return ptds._cudaGraphLaunch(graphExec, stream) + * return cudaGraphLaunch(graphExec, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphLaunch(__pyx_v_graphExec, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1583 + * return cudaGraphUpload(graphExec, stream) + * + * cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphLaunch", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1589 + * return cudaGraphLaunch(graphExec, stream) + * + * cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecDestroy(cudaGraphExec_t __pyx_v_graphExec) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1590 + * + * cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecDestroy(graphExec) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1590, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1591 + * cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecDestroy(graphExec) + * return cudaGraphExecDestroy(graphExec) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1592 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecDestroy(graphExec) # <<<<<<<<<<<<<< + * return cudaGraphExecDestroy(graphExec) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecDestroy(__pyx_v_graphExec); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1592, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1591 + * cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecDestroy(graphExec) + * return cudaGraphExecDestroy(graphExec) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1593 + * if usePTDS: + * return ptds._cudaGraphExecDestroy(graphExec) + * return cudaGraphExecDestroy(graphExec) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecDestroy(__pyx_v_graphExec); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1589 + * return cudaGraphLaunch(graphExec, stream) + * + * cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1595 + * return cudaGraphExecDestroy(graphExec) + * + * cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphDestroy(cudaGraph_t __pyx_v_graph) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1596 + * + * cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphDestroy(graph) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1596, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1597 + * cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphDestroy(graph) + * return cudaGraphDestroy(graph) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1598 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphDestroy(graph) # <<<<<<<<<<<<<< + * return cudaGraphDestroy(graph) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDestroy(__pyx_v_graph); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1598, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1597 + * cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphDestroy(graph) + * return cudaGraphDestroy(graph) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1599 + * if usePTDS: + * return ptds._cudaGraphDestroy(graph) + * return cudaGraphDestroy(graph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphDestroy(__pyx_v_graph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1595 + * return cudaGraphExecDestroy(graphExec) + * + * cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1601 + * return cudaGraphDestroy(graph) + * + * cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphDebugDotPrint(cudaGraph_t __pyx_v_graph, char const *__pyx_v_path, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1602 + * + * cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphDebugDotPrint(graph, path, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1602, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1603 + * cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphDebugDotPrint(graph, path, flags) + * return cudaGraphDebugDotPrint(graph, path, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1604 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphDebugDotPrint(graph, path, flags) # <<<<<<<<<<<<<< + * return cudaGraphDebugDotPrint(graph, path, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDebugDotPrint(__pyx_v_graph, __pyx_v_path, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1604, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1603 + * cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphDebugDotPrint(graph, path, flags) + * return cudaGraphDebugDotPrint(graph, path, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1605 + * if usePTDS: + * return ptds._cudaGraphDebugDotPrint(graph, path, flags) + * return cudaGraphDebugDotPrint(graph, path, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphDebugDotPrint(__pyx_v_graph, __pyx_v_path, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1601 + * return cudaGraphDestroy(graph) + * + * cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphDebugDotPrint", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1607 + * return cudaGraphDebugDotPrint(graph, path, flags) + * + * cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaUserObjectCreate(cudaUserObject_t *__pyx_v_object_out, void *__pyx_v_ptr, cudaHostFn_t __pyx_v_destroy, unsigned int __pyx_v_initialRefcount, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1608 + * + * cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1608, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1609 + * cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) + * return cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1610 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) # <<<<<<<<<<<<<< + * return cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectCreate(__pyx_v_object_out, __pyx_v_ptr, __pyx_v_destroy, __pyx_v_initialRefcount, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1610, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1609 + * cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) + * return cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1611 + * if usePTDS: + * return ptds._cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) + * return cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaUserObjectCreate(__pyx_v_object_out, __pyx_v_ptr, __pyx_v_destroy, __pyx_v_initialRefcount, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1607 + * return cudaGraphDebugDotPrint(graph, path, flags) + * + * cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaUserObjectCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1613 + * return cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) + * + * cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaUserObjectRetain(cudaUserObject_t __pyx_v_object, unsigned int __pyx_v_count) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1614 + * + * cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaUserObjectRetain(object, count) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1614, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1615 + * cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaUserObjectRetain(object, count) + * return cudaUserObjectRetain(object, count) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1616 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaUserObjectRetain(object, count) # <<<<<<<<<<<<<< + * return cudaUserObjectRetain(object, count) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectRetain(__pyx_v_object, __pyx_v_count); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1616, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1615 + * cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaUserObjectRetain(object, count) + * return cudaUserObjectRetain(object, count) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1617 + * if usePTDS: + * return ptds._cudaUserObjectRetain(object, count) + * return cudaUserObjectRetain(object, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaUserObjectRetain(__pyx_v_object, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1613 + * return cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) + * + * cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaUserObjectRetain", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1619 + * return cudaUserObjectRetain(object, count) + * + * cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaUserObjectRelease(cudaUserObject_t __pyx_v_object, unsigned int __pyx_v_count) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1620 + * + * cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaUserObjectRelease(object, count) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1620, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1621 + * cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaUserObjectRelease(object, count) + * return cudaUserObjectRelease(object, count) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1622 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaUserObjectRelease(object, count) # <<<<<<<<<<<<<< + * return cudaUserObjectRelease(object, count) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectRelease(__pyx_v_object, __pyx_v_count); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1622, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1621 + * cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaUserObjectRelease(object, count) + * return cudaUserObjectRelease(object, count) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1623 + * if usePTDS: + * return ptds._cudaUserObjectRelease(object, count) + * return cudaUserObjectRelease(object, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaUserObjectRelease(__pyx_v_object, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1619 + * return cudaUserObjectRetain(object, count) + * + * cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaUserObjectRelease", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1625 + * return cudaUserObjectRelease(object, count) + * + * cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphRetainUserObject(cudaGraph_t __pyx_v_graph, cudaUserObject_t __pyx_v_object, unsigned int __pyx_v_count, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1626 + * + * cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphRetainUserObject(graph, object, count, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1626, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1627 + * cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphRetainUserObject(graph, object, count, flags) + * return cudaGraphRetainUserObject(graph, object, count, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1628 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphRetainUserObject(graph, object, count, flags) # <<<<<<<<<<<<<< + * return cudaGraphRetainUserObject(graph, object, count, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRetainUserObject(__pyx_v_graph, __pyx_v_object, __pyx_v_count, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1628, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1627 + * cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphRetainUserObject(graph, object, count, flags) + * return cudaGraphRetainUserObject(graph, object, count, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1629 + * if usePTDS: + * return ptds._cudaGraphRetainUserObject(graph, object, count, flags) + * return cudaGraphRetainUserObject(graph, object, count, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphRetainUserObject(__pyx_v_graph, __pyx_v_object, __pyx_v_count, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1625 + * return cudaUserObjectRelease(object, count) + * + * cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphRetainUserObject", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1631 + * return cudaGraphRetainUserObject(graph, object, count, flags) + * + * cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphReleaseUserObject(cudaGraph_t __pyx_v_graph, cudaUserObject_t __pyx_v_object, unsigned int __pyx_v_count) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1632 + * + * cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphReleaseUserObject(graph, object, count) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1632, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1633 + * cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphReleaseUserObject(graph, object, count) + * return cudaGraphReleaseUserObject(graph, object, count) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1634 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphReleaseUserObject(graph, object, count) # <<<<<<<<<<<<<< + * return cudaGraphReleaseUserObject(graph, object, count) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphReleaseUserObject(__pyx_v_graph, __pyx_v_object, __pyx_v_count); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1634, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1633 + * cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphReleaseUserObject(graph, object, count) + * return cudaGraphReleaseUserObject(graph, object, count) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1635 + * if usePTDS: + * return ptds._cudaGraphReleaseUserObject(graph, object, count) + * return cudaGraphReleaseUserObject(graph, object, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphReleaseUserObject(__pyx_v_graph, __pyx_v_object, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1631 + * return cudaGraphRetainUserObject(graph, object, count, flags) + * + * cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphReleaseUserObject", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1637 + * return cudaGraphReleaseUserObject(graph, object, count) + * + * cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaGraphNodeParams *__pyx_v_nodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1638 + * + * cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1638, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1639 + * cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * return cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1640 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * return cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1640, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1639 + * cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * return cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1641 + * if usePTDS: + * return ptds._cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * return cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1637 + * return cudaGraphReleaseUserObject(graph, object, count) + * + * cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddNode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1643 + * return cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddNode_v2(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, cudaGraphEdgeData const *__pyx_v_dependencyData, size_t __pyx_v_numDependencies, struct cudaGraphNodeParams *__pyx_v_nodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1644 + * + * cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1644, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1645 + * cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) + * return cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1646 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * return cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddNode_v2(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_dependencyData, __pyx_v_numDependencies, __pyx_v_nodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1646, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1645 + * cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) + * return cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1647 + * if usePTDS: + * return ptds._cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) + * return cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddNode_v2(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_dependencyData, __pyx_v_numDependencies, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1643 + * return cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphAddNode_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1649 + * return cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeSetParams(cudaGraphNode_t __pyx_v_node, struct cudaGraphNodeParams *__pyx_v_nodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1650 + * + * cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphNodeSetParams(node, nodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1650, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1651 + * cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeSetParams(node, nodeParams) + * return cudaGraphNodeSetParams(node, nodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1652 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphNodeSetParams(node, nodeParams) # <<<<<<<<<<<<<< + * return cudaGraphNodeSetParams(node, nodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeSetParams(__pyx_v_node, __pyx_v_nodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1652, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1651 + * cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphNodeSetParams(node, nodeParams) + * return cudaGraphNodeSetParams(node, nodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1653 + * if usePTDS: + * return ptds._cudaGraphNodeSetParams(node, nodeParams) + * return cudaGraphNodeSetParams(node, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeSetParams(__pyx_v_node, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1649 + * return cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1655 + * return cudaGraphNodeSetParams(node, nodeParams) + * + * cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecNodeSetParams(cudaGraphExec_t __pyx_v_graphExec, cudaGraphNode_t __pyx_v_node, struct cudaGraphNodeParams *__pyx_v_nodeParams) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1656 + * + * cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphExecNodeSetParams(graphExec, node, nodeParams) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1656, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1657 + * cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecNodeSetParams(graphExec, node, nodeParams) + * return cudaGraphExecNodeSetParams(graphExec, node, nodeParams) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1658 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphExecNodeSetParams(graphExec, node, nodeParams) # <<<<<<<<<<<<<< + * return cudaGraphExecNodeSetParams(graphExec, node, nodeParams) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecNodeSetParams(__pyx_v_graphExec, __pyx_v_node, __pyx_v_nodeParams); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1658, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1657 + * cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphExecNodeSetParams(graphExec, node, nodeParams) + * return cudaGraphExecNodeSetParams(graphExec, node, nodeParams) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1659 + * if usePTDS: + * return ptds._cudaGraphExecNodeSetParams(graphExec, node, nodeParams) + * return cudaGraphExecNodeSetParams(graphExec, node, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecNodeSetParams(__pyx_v_graphExec, __pyx_v_node, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1655 + * return cudaGraphNodeSetParams(node, nodeParams) + * + * cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphExecNodeSetParams", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1661 + * return cudaGraphExecNodeSetParams(graphExec, node, nodeParams) + * + * cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle *__pyx_v_pHandle_out, cudaGraph_t __pyx_v_graph, unsigned int __pyx_v_defaultLaunchValue, unsigned int __pyx_v_flags) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1662 + * + * cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1662, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1663 + * cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) + * return cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1664 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) # <<<<<<<<<<<<<< + * return cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphConditionalHandleCreate(__pyx_v_pHandle_out, __pyx_v_graph, __pyx_v_defaultLaunchValue, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1664, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1663 + * cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) + * return cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1665 + * if usePTDS: + * return ptds._cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) + * return cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphConditionalHandleCreate(__pyx_v_pHandle_out, __pyx_v_graph, __pyx_v_defaultLaunchValue, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1661 + * return cudaGraphExecNodeSetParams(graphExec, node, nodeParams) + * + * cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphConditionalHandleCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1667 + * return cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) + * + * cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetDriverEntryPoint(char const *__pyx_v_symbol, void **__pyx_v_funcPtr, unsigned PY_LONG_LONG __pyx_v_flags, enum cudaDriverEntryPointQueryResult *__pyx_v_driverStatus) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1668 + * + * cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1668, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1669 + * cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) + * return cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1670 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) # <<<<<<<<<<<<<< + * return cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDriverEntryPoint(__pyx_v_symbol, __pyx_v_funcPtr, __pyx_v_flags, ((enum cudaDriverEntryPointQueryResult *)__pyx_v_driverStatus)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1670, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1669 + * cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) + * return cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1671 + * if usePTDS: + * return ptds._cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) + * return cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetDriverEntryPoint(__pyx_v_symbol, __pyx_v_funcPtr, __pyx_v_flags, __pyx_v_driverStatus); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1667 + * return cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) + * + * cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetDriverEntryPoint", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1673 + * return cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) + * + * cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetDriverEntryPointByVersion(char const *__pyx_v_symbol, void **__pyx_v_funcPtr, unsigned int __pyx_v_cudaVersion, unsigned PY_LONG_LONG __pyx_v_flags, enum cudaDriverEntryPointQueryResult *__pyx_v_driverStatus) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1674 + * + * cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1674, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1675 + * cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) + * return cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1676 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) # <<<<<<<<<<<<<< + * return cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDriverEntryPointByVersion(__pyx_v_symbol, __pyx_v_funcPtr, __pyx_v_cudaVersion, __pyx_v_flags, ((enum cudaDriverEntryPointQueryResult *)__pyx_v_driverStatus)); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1676, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1675 + * cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) + * return cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1677 + * if usePTDS: + * return ptds._cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) + * return cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetDriverEntryPointByVersion(__pyx_v_symbol, __pyx_v_funcPtr, __pyx_v_cudaVersion, __pyx_v_flags, __pyx_v_driverStatus); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1673 + * return cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) + * + * cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetDriverEntryPointByVersion", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1679 + * return cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) + * + * cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryLoadData(cudaLibrary_t *__pyx_v_library, void const *__pyx_v_code, enum cudaJitOption *__pyx_v_jitOptions, void **__pyx_v_jitOptionsValues, unsigned int __pyx_v_numJitOptions, enum cudaLibraryOption *__pyx_v_libraryOptions, void **__pyx_v_libraryOptionValues, unsigned int __pyx_v_numLibraryOptions) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1680 + * + * cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1680, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1681 + * cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * return cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1682 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) # <<<<<<<<<<<<<< + * return cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryLoadData(__pyx_v_library, __pyx_v_code, ((enum cudaJitOption *)__pyx_v_jitOptions), __pyx_v_jitOptionsValues, __pyx_v_numJitOptions, ((enum cudaLibraryOption *)__pyx_v_libraryOptions), __pyx_v_libraryOptionValues, __pyx_v_numLibraryOptions); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1682, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1681 + * cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * return cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1683 + * if usePTDS: + * return ptds._cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * return cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryLoadData(__pyx_v_library, __pyx_v_code, __pyx_v_jitOptions, __pyx_v_jitOptionsValues, __pyx_v_numJitOptions, __pyx_v_libraryOptions, __pyx_v_libraryOptionValues, __pyx_v_numLibraryOptions); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1679 + * return cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) + * + * cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaLibraryLoadData", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1685 + * return cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * + * cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryLoadFromFile(cudaLibrary_t *__pyx_v_library, char const *__pyx_v_fileName, enum cudaJitOption *__pyx_v_jitOptions, void **__pyx_v_jitOptionsValues, unsigned int __pyx_v_numJitOptions, enum cudaLibraryOption *__pyx_v_libraryOptions, void **__pyx_v_libraryOptionValues, unsigned int __pyx_v_numLibraryOptions) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1686 + * + * cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1686, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1687 + * cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * return cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1688 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) # <<<<<<<<<<<<<< + * return cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryLoadFromFile(__pyx_v_library, __pyx_v_fileName, ((enum cudaJitOption *)__pyx_v_jitOptions), __pyx_v_jitOptionsValues, __pyx_v_numJitOptions, ((enum cudaLibraryOption *)__pyx_v_libraryOptions), __pyx_v_libraryOptionValues, __pyx_v_numLibraryOptions); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1688, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1687 + * cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * return cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1689 + * if usePTDS: + * return ptds._cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * return cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryLoadFromFile(__pyx_v_library, __pyx_v_fileName, __pyx_v_jitOptions, __pyx_v_jitOptionsValues, __pyx_v_numJitOptions, __pyx_v_libraryOptions, __pyx_v_libraryOptionValues, __pyx_v_numLibraryOptions); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1685 + * return cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * + * cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaLibraryLoadFromFile", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1691 + * return cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * + * cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryUnload(cudaLibrary_t __pyx_v_library) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1692 + * + * cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaLibraryUnload(library) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1692, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1693 + * cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryUnload(library) + * return cudaLibraryUnload(library) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1694 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaLibraryUnload(library) # <<<<<<<<<<<<<< + * return cudaLibraryUnload(library) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryUnload(__pyx_v_library); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1694, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1693 + * cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryUnload(library) + * return cudaLibraryUnload(library) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1695 + * if usePTDS: + * return ptds._cudaLibraryUnload(library) + * return cudaLibraryUnload(library) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryUnload(__pyx_v_library); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1691 + * return cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * + * cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaLibraryUnload", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1697 + * return cudaLibraryUnload(library) + * + * cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryGetKernel(cudaKernel_t *__pyx_v_pKernel, cudaLibrary_t __pyx_v_library, char const *__pyx_v_name) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1698 + * + * cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaLibraryGetKernel(pKernel, library, name) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1698, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1699 + * cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryGetKernel(pKernel, library, name) + * return cudaLibraryGetKernel(pKernel, library, name) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1700 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaLibraryGetKernel(pKernel, library, name) # <<<<<<<<<<<<<< + * return cudaLibraryGetKernel(pKernel, library, name) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetKernel(__pyx_v_pKernel, __pyx_v_library, __pyx_v_name); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1700, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1699 + * cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryGetKernel(pKernel, library, name) + * return cudaLibraryGetKernel(pKernel, library, name) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1701 + * if usePTDS: + * return ptds._cudaLibraryGetKernel(pKernel, library, name) + * return cudaLibraryGetKernel(pKernel, library, name) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryGetKernel(__pyx_v_pKernel, __pyx_v_library, __pyx_v_name); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1697 + * return cudaLibraryUnload(library) + * + * cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaLibraryGetKernel", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1703 + * return cudaLibraryGetKernel(pKernel, library, name) + * + * cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryGetGlobal(void **__pyx_v_dptr, size_t *__pyx_v_numbytes, cudaLibrary_t __pyx_v_library, char const *__pyx_v_name) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1704 + * + * cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaLibraryGetGlobal(dptr, numbytes, library, name) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1704, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1705 + * cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryGetGlobal(dptr, numbytes, library, name) + * return cudaLibraryGetGlobal(dptr, numbytes, library, name) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1706 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaLibraryGetGlobal(dptr, numbytes, library, name) # <<<<<<<<<<<<<< + * return cudaLibraryGetGlobal(dptr, numbytes, library, name) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetGlobal(__pyx_v_dptr, __pyx_v_numbytes, __pyx_v_library, __pyx_v_name); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1706, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1705 + * cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryGetGlobal(dptr, numbytes, library, name) + * return cudaLibraryGetGlobal(dptr, numbytes, library, name) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1707 + * if usePTDS: + * return ptds._cudaLibraryGetGlobal(dptr, numbytes, library, name) + * return cudaLibraryGetGlobal(dptr, numbytes, library, name) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryGetGlobal(__pyx_v_dptr, __pyx_v_numbytes, __pyx_v_library, __pyx_v_name); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1703 + * return cudaLibraryGetKernel(pKernel, library, name) + * + * cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaLibraryGetGlobal", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1709 + * return cudaLibraryGetGlobal(dptr, numbytes, library, name) + * + * cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryGetManaged(void **__pyx_v_dptr, size_t *__pyx_v_numbytes, cudaLibrary_t __pyx_v_library, char const *__pyx_v_name) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1710 + * + * cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaLibraryGetManaged(dptr, numbytes, library, name) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1710, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1711 + * cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryGetManaged(dptr, numbytes, library, name) + * return cudaLibraryGetManaged(dptr, numbytes, library, name) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1712 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaLibraryGetManaged(dptr, numbytes, library, name) # <<<<<<<<<<<<<< + * return cudaLibraryGetManaged(dptr, numbytes, library, name) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetManaged(__pyx_v_dptr, __pyx_v_numbytes, __pyx_v_library, __pyx_v_name); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1712, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1711 + * cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryGetManaged(dptr, numbytes, library, name) + * return cudaLibraryGetManaged(dptr, numbytes, library, name) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1713 + * if usePTDS: + * return ptds._cudaLibraryGetManaged(dptr, numbytes, library, name) + * return cudaLibraryGetManaged(dptr, numbytes, library, name) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryGetManaged(__pyx_v_dptr, __pyx_v_numbytes, __pyx_v_library, __pyx_v_name); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1709 + * return cudaLibraryGetGlobal(dptr, numbytes, library, name) + * + * cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaLibraryGetManaged", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1715 + * return cudaLibraryGetManaged(dptr, numbytes, library, name) + * + * cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryGetUnifiedFunction(void **__pyx_v_fptr, cudaLibrary_t __pyx_v_library, char const *__pyx_v_symbol) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1716 + * + * cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaLibraryGetUnifiedFunction(fptr, library, symbol) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1716, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1717 + * cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryGetUnifiedFunction(fptr, library, symbol) + * return cudaLibraryGetUnifiedFunction(fptr, library, symbol) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1718 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaLibraryGetUnifiedFunction(fptr, library, symbol) # <<<<<<<<<<<<<< + * return cudaLibraryGetUnifiedFunction(fptr, library, symbol) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetUnifiedFunction(__pyx_v_fptr, __pyx_v_library, __pyx_v_symbol); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1718, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1717 + * cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryGetUnifiedFunction(fptr, library, symbol) + * return cudaLibraryGetUnifiedFunction(fptr, library, symbol) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1719 + * if usePTDS: + * return ptds._cudaLibraryGetUnifiedFunction(fptr, library, symbol) + * return cudaLibraryGetUnifiedFunction(fptr, library, symbol) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryGetUnifiedFunction(__pyx_v_fptr, __pyx_v_library, __pyx_v_symbol); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1715 + * return cudaLibraryGetManaged(dptr, numbytes, library, name) + * + * cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaLibraryGetUnifiedFunction", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1721 + * return cudaLibraryGetUnifiedFunction(fptr, library, symbol) + * + * cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryGetKernelCount(unsigned int *__pyx_v_count, cudaLibrary_t __pyx_v_lib) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1722 + * + * cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaLibraryGetKernelCount(count, lib) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1722, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1723 + * cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryGetKernelCount(count, lib) + * return cudaLibraryGetKernelCount(count, lib) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1724 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaLibraryGetKernelCount(count, lib) # <<<<<<<<<<<<<< + * return cudaLibraryGetKernelCount(count, lib) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetKernelCount(__pyx_v_count, __pyx_v_lib); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1724, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1723 + * cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryGetKernelCount(count, lib) + * return cudaLibraryGetKernelCount(count, lib) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1725 + * if usePTDS: + * return ptds._cudaLibraryGetKernelCount(count, lib) + * return cudaLibraryGetKernelCount(count, lib) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryGetKernelCount(__pyx_v_count, __pyx_v_lib); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1721 + * return cudaLibraryGetUnifiedFunction(fptr, library, symbol) + * + * cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaLibraryGetKernelCount", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1727 + * return cudaLibraryGetKernelCount(count, lib) + * + * cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryEnumerateKernels(cudaKernel_t *__pyx_v_kernels, unsigned int __pyx_v_numKernels, cudaLibrary_t __pyx_v_lib) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1728 + * + * cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaLibraryEnumerateKernels(kernels, numKernels, lib) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1728, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1729 + * cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryEnumerateKernels(kernels, numKernels, lib) + * return cudaLibraryEnumerateKernels(kernels, numKernels, lib) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1730 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaLibraryEnumerateKernels(kernels, numKernels, lib) # <<<<<<<<<<<<<< + * return cudaLibraryEnumerateKernels(kernels, numKernels, lib) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryEnumerateKernels(__pyx_v_kernels, __pyx_v_numKernels, __pyx_v_lib); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1730, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1729 + * cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaLibraryEnumerateKernels(kernels, numKernels, lib) + * return cudaLibraryEnumerateKernels(kernels, numKernels, lib) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1731 + * if usePTDS: + * return ptds._cudaLibraryEnumerateKernels(kernels, numKernels, lib) + * return cudaLibraryEnumerateKernels(kernels, numKernels, lib) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryEnumerateKernels(__pyx_v_kernels, __pyx_v_numKernels, __pyx_v_lib); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1727 + * return cudaLibraryGetKernelCount(count, lib) + * + * cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaLibraryEnumerateKernels", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1733 + * return cudaLibraryEnumerateKernels(kernels, numKernels, lib) + * + * cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaKernelSetAttributeForDevice(cudaKernel_t __pyx_v_kernel, enum cudaFuncAttribute __pyx_v_attr, int __pyx_v_value, int __pyx_v_device) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1734 + * + * cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaKernelSetAttributeForDevice(kernel, attr, value, device) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1734, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1735 + * cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaKernelSetAttributeForDevice(kernel, attr, value, device) + * return cudaKernelSetAttributeForDevice(kernel, attr, value, device) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1736 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaKernelSetAttributeForDevice(kernel, attr, value, device) # <<<<<<<<<<<<<< + * return cudaKernelSetAttributeForDevice(kernel, attr, value, device) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaKernelSetAttributeForDevice(__pyx_v_kernel, ((enum cudaFuncAttribute)__pyx_v_attr), __pyx_v_value, __pyx_v_device); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1736, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1735 + * cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaKernelSetAttributeForDevice(kernel, attr, value, device) + * return cudaKernelSetAttributeForDevice(kernel, attr, value, device) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1737 + * if usePTDS: + * return ptds._cudaKernelSetAttributeForDevice(kernel, attr, value, device) + * return cudaKernelSetAttributeForDevice(kernel, attr, value, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaKernelSetAttributeForDevice(__pyx_v_kernel, __pyx_v_attr, __pyx_v_value, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1733 + * return cudaLibraryEnumerateKernels(kernels, numKernels, lib) + * + * cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaKernelSetAttributeForDevice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1739 + * return cudaKernelSetAttributeForDevice(kernel, attr, value, device) + * + * cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetExportTable(void const **__pyx_v_ppExportTable, cudaUUID_t const *__pyx_v_pExportTableId) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1740 + * + * cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetExportTable(ppExportTable, pExportTableId) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1740, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1741 + * cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetExportTable(ppExportTable, pExportTableId) + * return cudaGetExportTable(ppExportTable, pExportTableId) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1742 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetExportTable(ppExportTable, pExportTableId) # <<<<<<<<<<<<<< + * return cudaGetExportTable(ppExportTable, pExportTableId) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetExportTable(__pyx_v_ppExportTable, __pyx_v_pExportTableId); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1742, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1741 + * cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetExportTable(ppExportTable, pExportTableId) + * return cudaGetExportTable(ppExportTable, pExportTableId) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1743 + * if usePTDS: + * return ptds._cudaGetExportTable(ppExportTable, pExportTableId) + * return cudaGetExportTable(ppExportTable, pExportTableId) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetExportTable(__pyx_v_ppExportTable, __pyx_v_pExportTableId); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1739 + * return cudaKernelSetAttributeForDevice(kernel, attr, value, device) + * + * cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetExportTable", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1745 + * return cudaGetExportTable(ppExportTable, pExportTableId) + * + * cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetKernel(cudaKernel_t *__pyx_v_kernelPtr, void const *__pyx_v_entryFuncAddr) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1746 + * + * cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaGetKernel(kernelPtr, entryFuncAddr) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1746, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1747 + * cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetKernel(kernelPtr, entryFuncAddr) + * return cudaGetKernel(kernelPtr, entryFuncAddr) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1748 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaGetKernel(kernelPtr, entryFuncAddr) # <<<<<<<<<<<<<< + * return cudaGetKernel(kernelPtr, entryFuncAddr) + * @cython.show_performance_hints(False) +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetKernel(__pyx_v_kernelPtr, __pyx_v_entryFuncAddr); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1748, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1747 + * cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaGetKernel(kernelPtr, entryFuncAddr) + * return cudaGetKernel(kernelPtr, entryFuncAddr) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1749 + * if usePTDS: + * return ptds._cudaGetKernel(kernelPtr, entryFuncAddr) + * return cudaGetKernel(kernelPtr, entryFuncAddr) # <<<<<<<<<<<<<< + * @cython.show_performance_hints(False) + * cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: +*/ + __pyx_r = cudaGetKernel(__pyx_v_kernelPtr, __pyx_v_entryFuncAddr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1745 + * return cudaGetExportTable(ppExportTable, pExportTableId) + * + * cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGetKernel", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1750 + * return ptds._cudaGetKernel(kernelPtr, entryFuncAddr) + * return cudaGetKernel(kernelPtr, entryFuncAddr) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: + * cdef bint usePTDS = cudaPythonInit() +*/ + +static struct cudaPitchedPtr __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__make_cudaPitchedPtr(void *__pyx_v_d, size_t __pyx_v_p, size_t __pyx_v_xsz, size_t __pyx_v_ysz) { + int __pyx_v_usePTDS; + struct cudaPitchedPtr __pyx_r; + int __pyx_t_1; + struct cudaPitchedPtr __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1752 + * @cython.show_performance_hints(False) + * cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._make_cudaPitchedPtr(d, p, xsz, ysz) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1752, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1753 + * cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._make_cudaPitchedPtr(d, p, xsz, ysz) + * return make_cudaPitchedPtr(d, p, xsz, ysz) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1754 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._make_cudaPitchedPtr(d, p, xsz, ysz) # <<<<<<<<<<<<<< + * return make_cudaPitchedPtr(d, p, xsz, ysz) + * @cython.show_performance_hints(False) +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaPitchedPtr(__pyx_v_d, __pyx_v_p, __pyx_v_xsz, __pyx_v_ysz); if (unlikely(__Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1754, __pyx_L1_error) + __pyx_r = __pyx_t_2; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1753 + * cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._make_cudaPitchedPtr(d, p, xsz, ysz) + * return make_cudaPitchedPtr(d, p, xsz, ysz) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1755 + * if usePTDS: + * return ptds._make_cudaPitchedPtr(d, p, xsz, ysz) + * return make_cudaPitchedPtr(d, p, xsz, ysz) # <<<<<<<<<<<<<< + * @cython.show_performance_hints(False) + * cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: +*/ + __pyx_r = make_cudaPitchedPtr(__pyx_v_d, __pyx_v_p, __pyx_v_xsz, __pyx_v_ysz); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1750 + * return ptds._cudaGetKernel(kernelPtr, entryFuncAddr) + * return cudaGetKernel(kernelPtr, entryFuncAddr) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: + * cdef bint usePTDS = cudaPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._make_cudaPitchedPtr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_pretend_to_initialize(&__pyx_r); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1756 + * return ptds._make_cudaPitchedPtr(d, p, xsz, ysz) + * return make_cudaPitchedPtr(d, p, xsz, ysz) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: + * cdef bint usePTDS = cudaPythonInit() +*/ + +static struct cudaPos __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__make_cudaPos(size_t __pyx_v_x, size_t __pyx_v_y, size_t __pyx_v_z) { + int __pyx_v_usePTDS; + struct cudaPos __pyx_r; + int __pyx_t_1; + struct cudaPos __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1758 + * @cython.show_performance_hints(False) + * cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._make_cudaPos(x, y, z) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1758, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1759 + * cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._make_cudaPos(x, y, z) + * return make_cudaPos(x, y, z) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1760 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._make_cudaPos(x, y, z) # <<<<<<<<<<<<<< + * return make_cudaPos(x, y, z) + * @cython.show_performance_hints(False) +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaPos(__pyx_v_x, __pyx_v_y, __pyx_v_z); if (unlikely(__Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1760, __pyx_L1_error) + __pyx_r = __pyx_t_2; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1759 + * cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._make_cudaPos(x, y, z) + * return make_cudaPos(x, y, z) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1761 + * if usePTDS: + * return ptds._make_cudaPos(x, y, z) + * return make_cudaPos(x, y, z) # <<<<<<<<<<<<<< + * @cython.show_performance_hints(False) + * cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil: +*/ + __pyx_r = make_cudaPos(__pyx_v_x, __pyx_v_y, __pyx_v_z); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1756 + * return ptds._make_cudaPitchedPtr(d, p, xsz, ysz) + * return make_cudaPitchedPtr(d, p, xsz, ysz) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: + * cdef bint usePTDS = cudaPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._make_cudaPos", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_pretend_to_initialize(&__pyx_r); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1762 + * return ptds._make_cudaPos(x, y, z) + * return make_cudaPos(x, y, z) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil: + * cdef bint usePTDS = cudaPythonInit() +*/ + +static struct cudaExtent __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__make_cudaExtent(size_t __pyx_v_w, size_t __pyx_v_h, size_t __pyx_v_d) { + int __pyx_v_usePTDS; + struct cudaExtent __pyx_r; + int __pyx_t_1; + struct cudaExtent __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1764 + * @cython.show_performance_hints(False) + * cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._make_cudaExtent(w, h, d) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1764, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1765 + * cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._make_cudaExtent(w, h, d) + * return make_cudaExtent(w, h, d) +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1766 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._make_cudaExtent(w, h, d) # <<<<<<<<<<<<<< + * return make_cudaExtent(w, h, d) + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaExtent(__pyx_v_w, __pyx_v_h, __pyx_v_d); if (unlikely(__Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1766, __pyx_L1_error) + __pyx_r = __pyx_t_2; + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1765 + * cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._make_cudaExtent(w, h, d) + * return make_cudaExtent(w, h, d) +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1767 + * if usePTDS: + * return ptds._make_cudaExtent(w, h, d) + * return make_cudaExtent(w, h, d) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = make_cudaExtent(__pyx_v_w, __pyx_v_h, __pyx_v_d); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1762 + * return ptds._make_cudaPos(x, y, z) + * return make_cudaPos(x, y, z) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil: + * cdef bint usePTDS = cudaPythonInit() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._make_cudaExtent", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_pretend_to_initialize(&__pyx_r); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1769 + * return make_cudaExtent(w, h, d) + * + * cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaProfilerStart(void) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1770 + * + * cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaProfilerStart() +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1770, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1771 + * cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaProfilerStart() + * return cudaProfilerStart() +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1772 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaProfilerStart() # <<<<<<<<<<<<<< + * return cudaProfilerStart() + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaProfilerStart(); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1772, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1771 + * cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaProfilerStart() + * return cudaProfilerStart() +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1773 + * if usePTDS: + * return ptds._cudaProfilerStart() + * return cudaProfilerStart() # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaProfilerStart(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1769 + * return make_cudaExtent(w, h, d) + * + * cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaProfilerStart", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime.pyx":1775 + * return cudaProfilerStart() + * + * cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaProfilerStop(void) { + int __pyx_v_usePTDS; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1776 + * + * cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() # <<<<<<<<<<<<<< + * if usePTDS: + * return ptds._cudaProfilerStop() +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_cudaPythonInit(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 1776, __pyx_L1_error) + __pyx_v_usePTDS = __pyx_t_1; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1777 + * cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaProfilerStop() + * return cudaProfilerStop() +*/ + if (__pyx_v_usePTDS) { + + /* "cuda/bindings/_bindings/cyruntime.pyx":1778 + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: + * return ptds._cudaProfilerStop() # <<<<<<<<<<<<<< + * return cudaProfilerStop() + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaProfilerStop(); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(0, 1778, __pyx_L1_error) + __pyx_r = ((cudaError_t)__pyx_t_2); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1777 + * cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: # <<<<<<<<<<<<<< + * return ptds._cudaProfilerStop() + * return cudaProfilerStop() +*/ + } + + /* "cuda/bindings/_bindings/cyruntime.pyx":1779 + * if usePTDS: + * return ptds._cudaProfilerStop() + * return cudaProfilerStop() # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = cudaProfilerStop(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1775 + * return cudaProfilerStart() + * + * cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef bint usePTDS = cudaPythonInit() + * if usePTDS: +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaProfilerStop", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":18 + * cimport cuda.bindings._bindings.cydriver as cydriver + * + * cdef cudaError_t _cudaEGLStreamProducerPresentFrame(cyruntime.cudaEglStreamConnection* conn, cyruntime.cudaEglFrame eglframe, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamProducerPresentFrame(__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *__pyx_v_conn, __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame __pyx_v_eglframe, cudaStream_t *__pyx_v_pStream) { + cudaError_t __pyx_v_err; + __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame __pyx_v_cueglFrame; + cudaError_t __pyx_r; + int __pyx_t_1; + cudaError_t __pyx_t_2; + CUresult __pyx_t_3; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":19 + * + * cdef cudaError_t _cudaEGLStreamProducerPresentFrame(cyruntime.cudaEglStreamConnection* conn, cyruntime.cudaEglFrame eglframe, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":21 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":22 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * cdef cydriver.CUeglFrame cueglFrame +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":23 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * cdef cydriver.CUeglFrame cueglFrame + * err = getDriverEglFrame(&cueglFrame, eglframe) +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":22 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * cdef cydriver.CUeglFrame cueglFrame +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":25 + * return err + * cdef cydriver.CUeglFrame cueglFrame + * err = getDriverEglFrame(&cueglFrame, eglframe) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getDriverEglFrame((&__pyx_v_cueglFrame), __pyx_v_eglframe); if (unlikely(__pyx_t_2 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 25, __pyx_L1_error) + __pyx_v_err = __pyx_t_2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":26 + * cdef cydriver.CUeglFrame cueglFrame + * err = getDriverEglFrame(&cueglFrame, eglframe) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamProducerPresentFrame(conn, cueglFrame, pStream) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":27 + * err = getDriverEglFrame(&cueglFrame, eglframe) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuEGLStreamProducerPresentFrame(conn, cueglFrame, pStream) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":26 + * cdef cydriver.CUeglFrame cueglFrame + * err = getDriverEglFrame(&cueglFrame, eglframe) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamProducerPresentFrame(conn, cueglFrame, pStream) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":28 + * if err != cudaSuccess: + * return err + * err = cydriver._cuEGLStreamProducerPresentFrame(conn, cueglFrame, pStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_3 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerPresentFrame(((__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *)__pyx_v_conn), __pyx_v_cueglFrame, __pyx_v_pStream); if (unlikely(__pyx_t_3 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 28, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_3); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":29 + * return err + * err = cydriver._cuEGLStreamProducerPresentFrame(conn, cueglFrame, pStream) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEGLStreamProducerReturnFrame(cyruntime.cudaEglStreamConnection* conn, cyruntime.cudaEglFrame* eglframe, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":18 + * cimport cuda.bindings._bindings.cydriver as cydriver + * + * cdef cudaError_t _cudaEGLStreamProducerPresentFrame(cyruntime.cudaEglStreamConnection* conn, cyruntime.cudaEglFrame eglframe, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEGLStreamProducerPresentFrame", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":31 + * return err + * + * cdef cudaError_t _cudaEGLStreamProducerReturnFrame(cyruntime.cudaEglStreamConnection* conn, cyruntime.cudaEglFrame* eglframe, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamProducerReturnFrame(__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *__pyx_v_conn, __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame *__pyx_v_eglframe, cudaStream_t *__pyx_v_pStream) { + cudaError_t __pyx_v_err; + __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame __pyx_v_cueglFrame; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + cudaError_t __pyx_t_3; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":32 + * + * cdef cudaError_t _cudaEGLStreamProducerReturnFrame(cyruntime.cudaEglStreamConnection* conn, cyruntime.cudaEglFrame* eglframe, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":34 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":35 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * if eglframe == NULL: +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":36 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * if eglframe == NULL: + * err = cudaErrorInvalidResourceHandle +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":35 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * if eglframe == NULL: +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":37 + * if err != cudaSuccess: + * return err + * if eglframe == NULL: # <<<<<<<<<<<<<< + * err = cudaErrorInvalidResourceHandle + * return err +*/ + __pyx_t_1 = (__pyx_v_eglframe == NULL); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":38 + * return err + * if eglframe == NULL: + * err = cudaErrorInvalidResourceHandle # <<<<<<<<<<<<<< + * return err + * cdef cydriver.CUeglFrame cueglFrame +*/ + __pyx_v_err = cudaErrorInvalidResourceHandle; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":39 + * if eglframe == NULL: + * err = cudaErrorInvalidResourceHandle + * return err # <<<<<<<<<<<<<< + * cdef cydriver.CUeglFrame cueglFrame + * err = cydriver._cuEGLStreamProducerReturnFrame(conn, &cueglFrame, pStream) +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":37 + * if err != cudaSuccess: + * return err + * if eglframe == NULL: # <<<<<<<<<<<<<< + * err = cudaErrorInvalidResourceHandle + * return err +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":41 + * return err + * cdef cydriver.CUeglFrame cueglFrame + * err = cydriver._cuEGLStreamProducerReturnFrame(conn, &cueglFrame, pStream) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerReturnFrame(((__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *)__pyx_v_conn), (&__pyx_v_cueglFrame), __pyx_v_pStream); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 41, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":42 + * cdef cydriver.CUeglFrame cueglFrame + * err = cydriver._cuEGLStreamProducerReturnFrame(conn, &cueglFrame, pStream) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = getRuntimeEglFrame(eglframe, cueglFrame) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":43 + * err = cydriver._cuEGLStreamProducerReturnFrame(conn, &cueglFrame, pStream) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = getRuntimeEglFrame(eglframe, cueglFrame) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":42 + * cdef cydriver.CUeglFrame cueglFrame + * err = cydriver._cuEGLStreamProducerReturnFrame(conn, &cueglFrame, pStream) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = getRuntimeEglFrame(eglframe, cueglFrame) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":44 + * if err != cudaSuccess: + * return err + * err = getRuntimeEglFrame(eglframe, cueglFrame) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_3 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getRuntimeEglFrame(__pyx_v_eglframe, __pyx_v_cueglFrame); if (unlikely(__pyx_t_3 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 44, __pyx_L1_error) + __pyx_v_err = __pyx_t_3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":45 + * return err + * err = getRuntimeEglFrame(eglframe, cueglFrame) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedEglFrame(cyruntime.cudaEglFrame* eglFrame, cudaGraphicsResource_t resource, unsigned int index, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":31 + * return err + * + * cdef cudaError_t _cudaEGLStreamProducerReturnFrame(cyruntime.cudaEglStreamConnection* conn, cyruntime.cudaEglFrame* eglframe, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEGLStreamProducerReturnFrame", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":47 + * return err + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedEglFrame(cyruntime.cudaEglFrame* eglFrame, cudaGraphicsResource_t resource, unsigned int index, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsResourceGetMappedEglFrame(__pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame *__pyx_v_eglFrame, cudaGraphicsResource_t __pyx_v_resource, unsigned int __pyx_v_index, unsigned int __pyx_v_mipLevel) { + cudaError_t __pyx_v_err; + __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame __pyx_v_cueglFrame; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + cudaError_t __pyx_t_3; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":48 + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedEglFrame(cyruntime.cudaEglFrame* eglFrame, cudaGraphicsResource_t resource, unsigned int index, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":50 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":51 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * cdef cydriver.CUeglFrame cueglFrame +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":52 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * cdef cydriver.CUeglFrame cueglFrame + * memset(&cueglFrame, 0, sizeof(cueglFrame)) +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":51 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * cdef cydriver.CUeglFrame cueglFrame +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":54 + * return err + * cdef cydriver.CUeglFrame cueglFrame + * memset(&cueglFrame, 0, sizeof(cueglFrame)) # <<<<<<<<<<<<<< + * err = cydriver._cuGraphicsResourceGetMappedEglFrame(&cueglFrame, resource, index, mipLevel) + * if err != cudaSuccess: +*/ + (void)(memset((&__pyx_v_cueglFrame), 0, (sizeof(__pyx_v_cueglFrame)))); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":55 + * cdef cydriver.CUeglFrame cueglFrame + * memset(&cueglFrame, 0, sizeof(cueglFrame)) + * err = cydriver._cuGraphicsResourceGetMappedEglFrame(&cueglFrame, resource, index, mipLevel) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsResourceGetMappedEglFrame((&__pyx_v_cueglFrame), ((CUgraphicsResource)__pyx_v_resource), __pyx_v_index, __pyx_v_mipLevel); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 55, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":56 + * memset(&cueglFrame, 0, sizeof(cueglFrame)) + * err = cydriver._cuGraphicsResourceGetMappedEglFrame(&cueglFrame, resource, index, mipLevel) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = getRuntimeEglFrame(eglFrame, cueglFrame) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":57 + * err = cydriver._cuGraphicsResourceGetMappedEglFrame(&cueglFrame, resource, index, mipLevel) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = getRuntimeEglFrame(eglFrame, cueglFrame) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":56 + * memset(&cueglFrame, 0, sizeof(cueglFrame)) + * err = cydriver._cuGraphicsResourceGetMappedEglFrame(&cueglFrame, resource, index, mipLevel) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = getRuntimeEglFrame(eglFrame, cueglFrame) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":58 + * if err != cudaSuccess: + * return err + * err = getRuntimeEglFrame(eglFrame, cueglFrame) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_3 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getRuntimeEglFrame(__pyx_v_eglFrame, __pyx_v_cueglFrame); if (unlikely(__pyx_t_3 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 58, __pyx_L1_error) + __pyx_v_err = __pyx_t_3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":59 + * return err + * err = getRuntimeEglFrame(eglFrame, cueglFrame) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaVDPAUSetVDPAUDevice(int device, cyruntime.VdpDevice vdpDevice, cyruntime.VdpGetProcAddress* vdpGetProcAddress) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":47 + * return err + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedEglFrame(cyruntime.cudaEglFrame* eglFrame, cudaGraphicsResource_t resource, unsigned int index, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphicsResourceGetMappedEglFrame", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":61 + * return err + * + * cdef cudaError_t _cudaVDPAUSetVDPAUDevice(int device, cyruntime.VdpDevice vdpDevice, cyruntime.VdpGetProcAddress* vdpGetProcAddress) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaErrorNotSupported + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaVDPAUSetVDPAUDevice(CYTHON_UNUSED int __pyx_v_device, CYTHON_UNUSED __pyx_t_4cuda_8bindings_9cyruntime_VdpDevice __pyx_v_vdpDevice, CYTHON_UNUSED __pyx_t_4cuda_8bindings_9cyruntime_VdpGetProcAddress *__pyx_v_vdpGetProcAddress) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":62 + * + * cdef cudaError_t _cudaVDPAUSetVDPAUDevice(int device, cyruntime.VdpDevice vdpDevice, cyruntime.VdpGetProcAddress* vdpGetProcAddress) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaErrorNotSupported # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaVDPAUGetDevice(int* device, cyruntime.VdpDevice vdpDevice, cyruntime.VdpGetProcAddress* vdpGetProcAddress) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaErrorNotSupported; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":61 + * return err + * + * cdef cudaError_t _cudaVDPAUSetVDPAUDevice(int device, cyruntime.VdpDevice vdpDevice, cyruntime.VdpGetProcAddress* vdpGetProcAddress) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaErrorNotSupported + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":64 + * return cudaErrorNotSupported + * + * cdef cudaError_t _cudaVDPAUGetDevice(int* device, cyruntime.VdpDevice vdpDevice, cyruntime.VdpGetProcAddress* vdpGetProcAddress) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaVDPAUGetDevice(int *__pyx_v_device, __pyx_t_4cuda_8bindings_9cyruntime_VdpDevice __pyx_v_vdpDevice, __pyx_t_4cuda_8bindings_9cyruntime_VdpGetProcAddress *__pyx_v_vdpGetProcAddress) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":65 + * + * cdef cudaError_t _cudaVDPAUGetDevice(int* device, cyruntime.VdpDevice vdpDevice, cyruntime.VdpGetProcAddress* vdpGetProcAddress) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":67 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":68 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuVDPAUGetDevice(device, vdpDevice, vdpGetProcAddress) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":69 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuVDPAUGetDevice(device, vdpDevice, vdpGetProcAddress) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":68 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuVDPAUGetDevice(device, vdpDevice, vdpGetProcAddress) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":70 + * if err != cudaSuccess: + * return err + * err = cydriver._cuVDPAUGetDevice(device, vdpDevice, vdpGetProcAddress) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuVDPAUGetDevice(((CUdevice *)__pyx_v_device), __pyx_v_vdpDevice, __pyx_v_vdpGetProcAddress); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 70, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":71 + * return err + * err = cydriver._cuVDPAUGetDevice(device, vdpDevice, vdpGetProcAddress) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsVDPAURegisterVideoSurface(cudaGraphicsResource** resource, cyruntime.VdpVideoSurface vdpSurface, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":64 + * return cudaErrorNotSupported + * + * cdef cudaError_t _cudaVDPAUGetDevice(int* device, cyruntime.VdpDevice vdpDevice, cyruntime.VdpGetProcAddress* vdpGetProcAddress) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaVDPAUGetDevice", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":73 + * return err + * + * cdef cudaError_t _cudaGraphicsVDPAURegisterVideoSurface(cudaGraphicsResource** resource, cyruntime.VdpVideoSurface vdpSurface, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsVDPAURegisterVideoSurface(struct cudaGraphicsResource **__pyx_v_resource, __pyx_t_4cuda_8bindings_9cyruntime_VdpVideoSurface __pyx_v_vdpSurface, unsigned int __pyx_v_flags) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":74 + * + * cdef cudaError_t _cudaGraphicsVDPAURegisterVideoSurface(cudaGraphicsResource** resource, cyruntime.VdpVideoSurface vdpSurface, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":76 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":77 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuGraphicsVDPAURegisterVideoSurface(resource, vdpSurface, flags) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":78 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuGraphicsVDPAURegisterVideoSurface(resource, vdpSurface, flags) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":77 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuGraphicsVDPAURegisterVideoSurface(resource, vdpSurface, flags) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":79 + * if err != cudaSuccess: + * return err + * err = cydriver._cuGraphicsVDPAURegisterVideoSurface(resource, vdpSurface, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsVDPAURegisterVideoSurface(((CUgraphicsResource *)__pyx_v_resource), __pyx_v_vdpSurface, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 79, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":80 + * return err + * err = cydriver._cuGraphicsVDPAURegisterVideoSurface(resource, vdpSurface, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsVDPAURegisterOutputSurface(cudaGraphicsResource** resource, cyruntime.VdpOutputSurface vdpSurface, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":73 + * return err + * + * cdef cudaError_t _cudaGraphicsVDPAURegisterVideoSurface(cudaGraphicsResource** resource, cyruntime.VdpVideoSurface vdpSurface, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphicsVDPAURegisterVideoSurface", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":82 + * return err + * + * cdef cudaError_t _cudaGraphicsVDPAURegisterOutputSurface(cudaGraphicsResource** resource, cyruntime.VdpOutputSurface vdpSurface, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsVDPAURegisterOutputSurface(struct cudaGraphicsResource **__pyx_v_resource, __pyx_t_4cuda_8bindings_9cyruntime_VdpOutputSurface __pyx_v_vdpSurface, unsigned int __pyx_v_flags) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":83 + * + * cdef cudaError_t _cudaGraphicsVDPAURegisterOutputSurface(cudaGraphicsResource** resource, cyruntime.VdpOutputSurface vdpSurface, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":85 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":86 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuGraphicsVDPAURegisterOutputSurface(resource, vdpSurface, flags) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":87 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuGraphicsVDPAURegisterOutputSurface(resource, vdpSurface, flags) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":86 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuGraphicsVDPAURegisterOutputSurface(resource, vdpSurface, flags) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":88 + * if err != cudaSuccess: + * return err + * err = cydriver._cuGraphicsVDPAURegisterOutputSurface(resource, vdpSurface, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsVDPAURegisterOutputSurface(((CUgraphicsResource *)__pyx_v_resource), __pyx_v_vdpSurface, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 88, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":89 + * return err + * err = cydriver._cuGraphicsVDPAURegisterOutputSurface(resource, vdpSurface, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGLGetDevices(unsigned int* pCudaDeviceCount, int* pCudaDevices, unsigned int cudaDeviceCount, cyruntime.cudaGLDeviceList deviceList) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":82 + * return err + * + * cdef cudaError_t _cudaGraphicsVDPAURegisterOutputSurface(cudaGraphicsResource** resource, cyruntime.VdpOutputSurface vdpSurface, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphicsVDPAURegisterOutputSurface", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":91 + * return err + * + * cdef cudaError_t _cudaGLGetDevices(unsigned int* pCudaDeviceCount, int* pCudaDevices, unsigned int cudaDeviceCount, cyruntime.cudaGLDeviceList deviceList) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGLGetDevices(unsigned int *__pyx_v_pCudaDeviceCount, int *__pyx_v_pCudaDevices, unsigned int __pyx_v_cudaDeviceCount, enum __pyx_t_4cuda_8bindings_9cyruntime_cudaGLDeviceList __pyx_v_deviceList) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":92 + * + * cdef cudaError_t _cudaGLGetDevices(unsigned int* pCudaDeviceCount, int* pCudaDevices, unsigned int cudaDeviceCount, cyruntime.cudaGLDeviceList deviceList) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":94 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":95 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuGLGetDevices_v2(pCudaDeviceCount, pCudaDevices, cudaDeviceCount, deviceList) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":96 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuGLGetDevices_v2(pCudaDeviceCount, pCudaDevices, cudaDeviceCount, deviceList) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":95 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuGLGetDevices_v2(pCudaDeviceCount, pCudaDevices, cudaDeviceCount, deviceList) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":97 + * if err != cudaSuccess: + * return err + * err = cydriver._cuGLGetDevices_v2(pCudaDeviceCount, pCudaDevices, cudaDeviceCount, deviceList) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGLGetDevices_v2(__pyx_v_pCudaDeviceCount, ((CUdevice *)__pyx_v_pCudaDevices), __pyx_v_cudaDeviceCount, ((__pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList)__pyx_v_deviceList)); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 97, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":98 + * return err + * err = cydriver._cuGLGetDevices_v2(pCudaDeviceCount, pCudaDevices, cudaDeviceCount, deviceList) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsGLRegisterImage(cudaGraphicsResource** resource, cyruntime.GLuint image, cyruntime.GLenum target, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":91 + * return err + * + * cdef cudaError_t _cudaGLGetDevices(unsigned int* pCudaDeviceCount, int* pCudaDevices, unsigned int cudaDeviceCount, cyruntime.cudaGLDeviceList deviceList) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGLGetDevices", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":100 + * return err + * + * cdef cudaError_t _cudaGraphicsGLRegisterImage(cudaGraphicsResource** resource, cyruntime.GLuint image, cyruntime.GLenum target, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsGLRegisterImage(struct cudaGraphicsResource **__pyx_v_resource, __pyx_t_4cuda_8bindings_9cyruntime_GLuint __pyx_v_image, __pyx_t_4cuda_8bindings_9cyruntime_GLenum __pyx_v_target, unsigned int __pyx_v_flags) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":101 + * + * cdef cudaError_t _cudaGraphicsGLRegisterImage(cudaGraphicsResource** resource, cyruntime.GLuint image, cyruntime.GLenum target, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":103 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":104 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuGraphicsGLRegisterImage(resource, image, target, flags) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":105 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuGraphicsGLRegisterImage(resource, image, target, flags) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":104 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuGraphicsGLRegisterImage(resource, image, target, flags) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":106 + * if err != cudaSuccess: + * return err + * err = cydriver._cuGraphicsGLRegisterImage(resource, image, target, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsGLRegisterImage(((CUgraphicsResource *)__pyx_v_resource), __pyx_v_image, __pyx_v_target, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 106, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":107 + * return err + * err = cydriver._cuGraphicsGLRegisterImage(resource, image, target, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsGLRegisterBuffer(cudaGraphicsResource** resource, cyruntime.GLuint buffer, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":100 + * return err + * + * cdef cudaError_t _cudaGraphicsGLRegisterImage(cudaGraphicsResource** resource, cyruntime.GLuint image, cyruntime.GLenum target, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphicsGLRegisterImage", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":109 + * return err + * + * cdef cudaError_t _cudaGraphicsGLRegisterBuffer(cudaGraphicsResource** resource, cyruntime.GLuint buffer, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsGLRegisterBuffer(struct cudaGraphicsResource **__pyx_v_resource, __pyx_t_4cuda_8bindings_9cyruntime_GLuint __pyx_v_buffer, unsigned int __pyx_v_flags) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":110 + * + * cdef cudaError_t _cudaGraphicsGLRegisterBuffer(cudaGraphicsResource** resource, cyruntime.GLuint buffer, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":112 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":113 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuGraphicsGLRegisterBuffer(resource, buffer, flags) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":114 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuGraphicsGLRegisterBuffer(resource, buffer, flags) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":113 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuGraphicsGLRegisterBuffer(resource, buffer, flags) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":115 + * if err != cudaSuccess: + * return err + * err = cydriver._cuGraphicsGLRegisterBuffer(resource, buffer, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsGLRegisterBuffer(((CUgraphicsResource *)__pyx_v_resource), __pyx_v_buffer, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 115, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":116 + * return err + * err = cydriver._cuGraphicsGLRegisterBuffer(resource, buffer, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsEGLRegisterImage(cudaGraphicsResource_t* pCudaResource, cyruntime.EGLImageKHR image, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":109 + * return err + * + * cdef cudaError_t _cudaGraphicsGLRegisterBuffer(cudaGraphicsResource** resource, cyruntime.GLuint buffer, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphicsGLRegisterBuffer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":118 + * return err + * + * cdef cudaError_t _cudaGraphicsEGLRegisterImage(cudaGraphicsResource_t* pCudaResource, cyruntime.EGLImageKHR image, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsEGLRegisterImage(cudaGraphicsResource_t *__pyx_v_pCudaResource, __pyx_t_4cuda_8bindings_9cyruntime_EGLImageKHR __pyx_v_image, unsigned int __pyx_v_flags) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":119 + * + * cdef cudaError_t _cudaGraphicsEGLRegisterImage(cudaGraphicsResource_t* pCudaResource, cyruntime.EGLImageKHR image, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":121 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":122 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuGraphicsEGLRegisterImage(pCudaResource, image, flags) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":123 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuGraphicsEGLRegisterImage(pCudaResource, image, flags) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":122 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuGraphicsEGLRegisterImage(pCudaResource, image, flags) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":124 + * if err != cudaSuccess: + * return err + * err = cydriver._cuGraphicsEGLRegisterImage(pCudaResource, image, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsEGLRegisterImage(((CUgraphicsResource *)__pyx_v_pCudaResource), __pyx_v_image, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 124, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":125 + * return err + * err = cydriver._cuGraphicsEGLRegisterImage(pCudaResource, image, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEGLStreamConsumerConnect(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":118 + * return err + * + * cdef cudaError_t _cudaGraphicsEGLRegisterImage(cudaGraphicsResource_t* pCudaResource, cyruntime.EGLImageKHR image, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaGraphicsEGLRegisterImage", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":127 + * return err + * + * cdef cudaError_t _cudaEGLStreamConsumerConnect(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamConsumerConnect(__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *__pyx_v_conn, __pyx_t_4cuda_8bindings_9cyruntime_EGLStreamKHR __pyx_v_eglStream) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":128 + * + * cdef cudaError_t _cudaEGLStreamConsumerConnect(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":130 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":131 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamConsumerConnect(conn, eglStream) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":132 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuEGLStreamConsumerConnect(conn, eglStream) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":131 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamConsumerConnect(conn, eglStream) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":133 + * if err != cudaSuccess: + * return err + * err = cydriver._cuEGLStreamConsumerConnect(conn, eglStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerConnect(((__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *)__pyx_v_conn), __pyx_v_eglStream); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 133, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":134 + * return err + * err = cydriver._cuEGLStreamConsumerConnect(conn, eglStream) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEGLStreamConsumerConnectWithFlags(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":127 + * return err + * + * cdef cudaError_t _cudaEGLStreamConsumerConnect(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEGLStreamConsumerConnect", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":136 + * return err + * + * cdef cudaError_t _cudaEGLStreamConsumerConnectWithFlags(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamConsumerConnectWithFlags(__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *__pyx_v_conn, __pyx_t_4cuda_8bindings_9cyruntime_EGLStreamKHR __pyx_v_eglStream, unsigned int __pyx_v_flags) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":137 + * + * cdef cudaError_t _cudaEGLStreamConsumerConnectWithFlags(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":139 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":140 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamConsumerConnectWithFlags(conn, eglStream, flags) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":141 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuEGLStreamConsumerConnectWithFlags(conn, eglStream, flags) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":140 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamConsumerConnectWithFlags(conn, eglStream, flags) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":142 + * if err != cudaSuccess: + * return err + * err = cydriver._cuEGLStreamConsumerConnectWithFlags(conn, eglStream, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerConnectWithFlags(((__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *)__pyx_v_conn), __pyx_v_eglStream, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 142, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":143 + * return err + * err = cydriver._cuEGLStreamConsumerConnectWithFlags(conn, eglStream, flags) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEGLStreamConsumerDisconnect(cyruntime.cudaEglStreamConnection* conn) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":136 + * return err + * + * cdef cudaError_t _cudaEGLStreamConsumerConnectWithFlags(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEGLStreamConsumerConnectWithFlags", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":145 + * return err + * + * cdef cudaError_t _cudaEGLStreamConsumerDisconnect(cyruntime.cudaEglStreamConnection* conn) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamConsumerDisconnect(__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *__pyx_v_conn) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":146 + * + * cdef cudaError_t _cudaEGLStreamConsumerDisconnect(cyruntime.cudaEglStreamConnection* conn) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":148 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":149 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamConsumerDisconnect(conn) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":150 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuEGLStreamConsumerDisconnect(conn) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":149 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamConsumerDisconnect(conn) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":151 + * if err != cudaSuccess: + * return err + * err = cydriver._cuEGLStreamConsumerDisconnect(conn) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerDisconnect(((__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *)__pyx_v_conn)); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 151, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":152 + * return err + * err = cydriver._cuEGLStreamConsumerDisconnect(conn) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEGLStreamConsumerAcquireFrame(cyruntime.cudaEglStreamConnection* conn, cudaGraphicsResource_t* pCudaResource, cudaStream_t* pStream, unsigned int timeout) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":145 + * return err + * + * cdef cudaError_t _cudaEGLStreamConsumerDisconnect(cyruntime.cudaEglStreamConnection* conn) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEGLStreamConsumerDisconnect", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":154 + * return err + * + * cdef cudaError_t _cudaEGLStreamConsumerAcquireFrame(cyruntime.cudaEglStreamConnection* conn, cudaGraphicsResource_t* pCudaResource, cudaStream_t* pStream, unsigned int timeout) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamConsumerAcquireFrame(__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *__pyx_v_conn, cudaGraphicsResource_t *__pyx_v_pCudaResource, cudaStream_t *__pyx_v_pStream, unsigned int __pyx_v_timeout) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":155 + * + * cdef cudaError_t _cudaEGLStreamConsumerAcquireFrame(cyruntime.cudaEglStreamConnection* conn, cudaGraphicsResource_t* pCudaResource, cudaStream_t* pStream, unsigned int timeout) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":157 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":158 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamConsumerAcquireFrame(conn, pCudaResource, pStream, timeout) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":159 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuEGLStreamConsumerAcquireFrame(conn, pCudaResource, pStream, timeout) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":158 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamConsumerAcquireFrame(conn, pCudaResource, pStream, timeout) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":160 + * if err != cudaSuccess: + * return err + * err = cydriver._cuEGLStreamConsumerAcquireFrame(conn, pCudaResource, pStream, timeout) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerAcquireFrame(((__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *)__pyx_v_conn), ((CUgraphicsResource *)__pyx_v_pCudaResource), ((CUstream *)__pyx_v_pStream), __pyx_v_timeout); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 160, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":161 + * return err + * err = cydriver._cuEGLStreamConsumerAcquireFrame(conn, pCudaResource, pStream, timeout) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEGLStreamConsumerReleaseFrame(cyruntime.cudaEglStreamConnection* conn, cudaGraphicsResource_t pCudaResource, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":154 + * return err + * + * cdef cudaError_t _cudaEGLStreamConsumerAcquireFrame(cyruntime.cudaEglStreamConnection* conn, cudaGraphicsResource_t* pCudaResource, cudaStream_t* pStream, unsigned int timeout) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEGLStreamConsumerAcquireFrame", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":163 + * return err + * + * cdef cudaError_t _cudaEGLStreamConsumerReleaseFrame(cyruntime.cudaEglStreamConnection* conn, cudaGraphicsResource_t pCudaResource, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamConsumerReleaseFrame(__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *__pyx_v_conn, cudaGraphicsResource_t __pyx_v_pCudaResource, cudaStream_t *__pyx_v_pStream) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":164 + * + * cdef cudaError_t _cudaEGLStreamConsumerReleaseFrame(cyruntime.cudaEglStreamConnection* conn, cudaGraphicsResource_t pCudaResource, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":166 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":167 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamConsumerReleaseFrame(conn, pCudaResource, pStream) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":168 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuEGLStreamConsumerReleaseFrame(conn, pCudaResource, pStream) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":167 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamConsumerReleaseFrame(conn, pCudaResource, pStream) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":169 + * if err != cudaSuccess: + * return err + * err = cydriver._cuEGLStreamConsumerReleaseFrame(conn, pCudaResource, pStream) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerReleaseFrame(((__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *)__pyx_v_conn), ((CUgraphicsResource)__pyx_v_pCudaResource), ((CUstream *)__pyx_v_pStream)); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 169, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":170 + * return err + * err = cydriver._cuEGLStreamConsumerReleaseFrame(conn, pCudaResource, pStream) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEGLStreamProducerConnect(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream, cyruntime.EGLint width, cyruntime.EGLint height) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":163 + * return err + * + * cdef cudaError_t _cudaEGLStreamConsumerReleaseFrame(cyruntime.cudaEglStreamConnection* conn, cudaGraphicsResource_t pCudaResource, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEGLStreamConsumerReleaseFrame", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":172 + * return err + * + * cdef cudaError_t _cudaEGLStreamProducerConnect(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream, cyruntime.EGLint width, cyruntime.EGLint height) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamProducerConnect(__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *__pyx_v_conn, __pyx_t_4cuda_8bindings_9cyruntime_EGLStreamKHR __pyx_v_eglStream, __pyx_t_4cuda_8bindings_9cyruntime_EGLint __pyx_v_width, __pyx_t_4cuda_8bindings_9cyruntime_EGLint __pyx_v_height) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":173 + * + * cdef cudaError_t _cudaEGLStreamProducerConnect(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream, cyruntime.EGLint width, cyruntime.EGLint height) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":175 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":176 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamProducerConnect(conn, eglStream, width, height) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":177 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuEGLStreamProducerConnect(conn, eglStream, width, height) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":176 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamProducerConnect(conn, eglStream, width, height) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":178 + * if err != cudaSuccess: + * return err + * err = cydriver._cuEGLStreamProducerConnect(conn, eglStream, width, height) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerConnect(((__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *)__pyx_v_conn), __pyx_v_eglStream, __pyx_v_width, __pyx_v_height); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 178, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":179 + * return err + * err = cydriver._cuEGLStreamProducerConnect(conn, eglStream, width, height) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEGLStreamProducerDisconnect(cyruntime.cudaEglStreamConnection* conn) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":172 + * return err + * + * cdef cudaError_t _cudaEGLStreamProducerConnect(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream, cyruntime.EGLint width, cyruntime.EGLint height) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEGLStreamProducerConnect", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":181 + * return err + * + * cdef cudaError_t _cudaEGLStreamProducerDisconnect(cyruntime.cudaEglStreamConnection* conn) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamProducerDisconnect(__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *__pyx_v_conn) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":182 + * + * cdef cudaError_t _cudaEGLStreamProducerDisconnect(cyruntime.cudaEglStreamConnection* conn) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":184 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":185 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamProducerDisconnect(conn) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":186 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuEGLStreamProducerDisconnect(conn) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":185 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEGLStreamProducerDisconnect(conn) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":187 + * if err != cudaSuccess: + * return err + * err = cydriver._cuEGLStreamProducerDisconnect(conn) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerDisconnect(((__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *)__pyx_v_conn)); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 187, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":188 + * return err + * err = cydriver._cuEGLStreamProducerDisconnect(conn) + * return err # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventCreateFromEGLSync(cudaEvent_t* phEvent, cyruntime.EGLSyncKHR eglSync, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":181 + * return err + * + * cdef cudaError_t _cudaEGLStreamProducerDisconnect(cyruntime.cudaEglStreamConnection* conn) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEGLStreamProducerDisconnect", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":190 + * return err + * + * cdef cudaError_t _cudaEventCreateFromEGLSync(cudaEvent_t* phEvent, cyruntime.EGLSyncKHR eglSync, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventCreateFromEGLSync(cudaEvent_t *__pyx_v_phEvent, __pyx_t_4cuda_8bindings_9cyruntime_EGLSyncKHR __pyx_v_eglSync, unsigned int __pyx_v_flags) { + cudaError_t __pyx_v_err; + cudaError_t __pyx_r; + int __pyx_t_1; + CUresult __pyx_t_2; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":191 + * + * cdef cudaError_t _cudaEventCreateFromEGLSync(cudaEvent_t* phEvent, cyruntime.EGLSyncKHR eglSync, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":193 + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_v_err = cudaFree(((void *)0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":194 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEventCreateFromEGLSync(phEvent, eglSync, flags) +*/ + __pyx_t_1 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":195 + * err = cudaFree(0) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * err = cydriver._cuEventCreateFromEGLSync(phEvent, eglSync, flags) + * return err +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":194 + * # cudaFree(0) is a NOP operations that initializes the context state + * err = cudaFree(0) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * err = cydriver._cuEventCreateFromEGLSync(phEvent, eglSync, flags) +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":196 + * if err != cudaSuccess: + * return err + * err = cydriver._cuEventCreateFromEGLSync(phEvent, eglSync, flags) # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventCreateFromEGLSync(((CUevent *)__pyx_v_phEvent), __pyx_v_eglSync, __pyx_v_flags); if (unlikely(__pyx_t_2 == ((CUresult)CUDA_ERROR_NOT_FOUND) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 196, __pyx_L1_error) + __pyx_v_err = ((cudaError_t)__pyx_t_2); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":197 + * return err + * err = cydriver._cuEventCreateFromEGLSync(phEvent, eglSync, flags) + * return err # <<<<<<<<<<<<<< + * + * ## utility functions +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":190 + * return err + * + * cdef cudaError_t _cudaEventCreateFromEGLSync(cudaEvent_t* phEvent, cyruntime.EGLSyncKHR eglSync, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * # cudaFree(0) is a NOP operations that initializes the context state +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime._cudaEventCreateFromEGLSync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":201 + * ## utility functions + * + * cdef int case_desc(const cudaChannelFormatDesc* d, int x, int y, int z, int w, int f) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return d[0].x == x and d[0].y == y and d[0].z == z and d[0].w == w and d[0].f == f + * +*/ + +static int __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(struct cudaChannelFormatDesc const *__pyx_v_d, int __pyx_v_x, int __pyx_v_y, int __pyx_v_z, int __pyx_v_w, int __pyx_v_f) { + int __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":202 + * + * cdef int case_desc(const cudaChannelFormatDesc* d, int x, int y, int z, int w, int f) except ?cudaErrorCallRequiresNewerDriver nogil: + * return d[0].x == x and d[0].y == y and d[0].z == z and d[0].w == w and d[0].f == f # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_2 = ((__pyx_v_d[0]).x == __pyx_v_x); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L3_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).y == __pyx_v_y); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L3_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).z == __pyx_v_z); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L3_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).w == __pyx_v_w); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L3_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).f == __pyx_v_f); + __pyx_t_1 = __pyx_t_2; + __pyx_L3_bool_binop_done:; + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":201 + * ## utility functions + * + * cdef int case_desc(const cudaChannelFormatDesc* d, int x, int y, int z, int w, int f) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return d[0].x == x and d[0].y == y and d[0].z == z and d[0].w == w and d[0].f == f + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":205 + * + * + * cdef cudaError_t getDescInfo(const cudaChannelFormatDesc* d, int *numberOfChannels, cydriver.CUarray_format *format) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * # Check validity + * if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSigned, +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getDescInfo(struct cudaChannelFormatDesc const *__pyx_v_d, int *__pyx_v_numberOfChannels, CUarray_format *__pyx_v_format) { + cudaError_t __pyx_r; + enum cudaChannelFormatKind __pyx_t_1; + int __pyx_t_2; + int __pyx_t_3; + int __pyx_t_4; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":207 + * cdef cudaError_t getDescInfo(const cudaChannelFormatDesc* d, int *numberOfChannels, cydriver.CUarray_format *format) except ?cudaErrorCallRequiresNewerDriver nogil: + * # Check validity + * if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSigned, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * if (d[0].x != 8) and (d[0].x != 16) and (d[0].x != 32): +*/ + __pyx_t_1 = (__pyx_v_d[0]).f; + __pyx_t_3 = (__pyx_t_1 == cudaChannelFormatKindSigned); + if (!__pyx_t_3) { + } else { + __pyx_t_2 = __pyx_t_3; + goto __pyx_L4_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":208 + * # Check validity + * if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSigned, + * cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * if (d[0].x != 8) and (d[0].x != 16) and (d[0].x != 32): + * return cudaErrorInvalidChannelDescriptor +*/ + __pyx_t_3 = (__pyx_t_1 == cudaChannelFormatKindUnsigned); + __pyx_t_2 = __pyx_t_3; + __pyx_L4_bool_binop_done:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":207 + * cdef cudaError_t getDescInfo(const cudaChannelFormatDesc* d, int *numberOfChannels, cydriver.CUarray_format *format) except ?cudaErrorCallRequiresNewerDriver nogil: + * # Check validity + * if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSigned, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * if (d[0].x != 8) and (d[0].x != 16) and (d[0].x != 32): +*/ + __pyx_t_3 = __pyx_t_2; + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":209 + * if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSigned, + * cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * if (d[0].x != 8) and (d[0].x != 16) and (d[0].x != 32): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindFloat,): +*/ + __pyx_t_2 = ((__pyx_v_d[0]).x != 8); + if (__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L7_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).x != 16); + if (__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L7_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).x != 32); + __pyx_t_3 = __pyx_t_2; + __pyx_L7_bool_binop_done:; + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":210 + * cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * if (d[0].x != 8) and (d[0].x != 16) and (d[0].x != 32): + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindFloat,): + * if (d[0].x != 16) and (d[0].x != 32): +*/ + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":209 + * if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSigned, + * cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * if (d[0].x != 8) and (d[0].x != 16) and (d[0].x != 32): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindFloat,): +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":207 + * cdef cudaError_t getDescInfo(const cudaChannelFormatDesc* d, int *numberOfChannels, cydriver.CUarray_format *format) except ?cudaErrorCallRequiresNewerDriver nogil: + * # Check validity + * if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSigned, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * if (d[0].x != 8) and (d[0].x != 16) and (d[0].x != 32): +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":211 + * if (d[0].x != 8) and (d[0].x != 16) and (d[0].x != 32): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindFloat,): # <<<<<<<<<<<<<< + * if (d[0].x != 16) and (d[0].x != 32): + * return cudaErrorInvalidChannelDescriptor +*/ + __pyx_t_1 = (__pyx_v_d[0]).f; + __pyx_t_3 = (__pyx_t_1 == cudaChannelFormatKindFloat); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":212 + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindFloat,): + * if (d[0].x != 16) and (d[0].x != 32): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12,): +*/ + __pyx_t_2 = ((__pyx_v_d[0]).x != 16); + if (__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L11_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).x != 32); + __pyx_t_3 = __pyx_t_2; + __pyx_L11_bool_binop_done:; + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":213 + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindFloat,): + * if (d[0].x != 16) and (d[0].x != 32): + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12,): + * if (d[0].x != 8) or (d[0].y != 8) or (d[0].z != 8) or (d[0].w != 0): +*/ + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":212 + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindFloat,): + * if (d[0].x != 16) and (d[0].x != 32): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12,): +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":211 + * if (d[0].x != 8) and (d[0].x != 16) and (d[0].x != 32): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindFloat,): # <<<<<<<<<<<<<< + * if (d[0].x != 16) and (d[0].x != 32): + * return cudaErrorInvalidChannelDescriptor +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":214 + * if (d[0].x != 16) and (d[0].x != 32): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12,): # <<<<<<<<<<<<<< + * if (d[0].x != 8) or (d[0].y != 8) or (d[0].z != 8) or (d[0].w != 0): + * return cudaErrorInvalidChannelDescriptor +*/ + __pyx_t_1 = (__pyx_v_d[0]).f; + __pyx_t_3 = (__pyx_t_1 == cudaChannelFormatKindNV12); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":215 + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12,): + * if (d[0].x != 8) or (d[0].y != 8) or (d[0].z != 8) or (d[0].w != 0): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1, +*/ + __pyx_t_2 = ((__pyx_v_d[0]).x != 8); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L14_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).y != 8); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L14_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).z != 8); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L14_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).w != 0); + __pyx_t_3 = __pyx_t_2; + __pyx_L14_bool_binop_done:; + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":216 + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12,): + * if (d[0].x != 8) or (d[0].y != 8) or (d[0].z != 8) or (d[0].w != 0): + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2, +*/ + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":215 + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12,): + * if (d[0].x != 8) or (d[0].y != 8) or (d[0].z != 8) or (d[0].w != 0): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1, +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":214 + * if (d[0].x != 16) and (d[0].x != 32): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12,): # <<<<<<<<<<<<<< + * if (d[0].x != 8) or (d[0].y != 8) or (d[0].z != 8) or (d[0].w != 0): + * return cudaErrorInvalidChannelDescriptor +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":217 + * if (d[0].x != 8) or (d[0].y != 8) or (d[0].z != 8) or (d[0].w != 0): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4, +*/ + __pyx_t_1 = (__pyx_v_d[0]).f; + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindSignedNormalized8X1); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L18_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":218 + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X1, +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindSignedNormalized8X2); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L18_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":217 + * if (d[0].x != 8) or (d[0].y != 8) or (d[0].z != 8) or (d[0].w != 0): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4, +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindSignedNormalized8X4); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L18_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":220 + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X2, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4,): +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedNormalized8X1); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L18_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":217 + * if (d[0].x != 8) or (d[0].y != 8) or (d[0].z != 8) or (d[0].w != 0): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4, +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedNormalized8X2); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L18_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":222 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X1, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X2, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4,): # <<<<<<<<<<<<<< + * if (d[0].x != 8): + * return cudaErrorInvalidChannelDescriptor +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedNormalized8X4); + __pyx_t_3 = __pyx_t_2; + __pyx_L18_bool_binop_done:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":217 + * if (d[0].x != 8) or (d[0].y != 8) or (d[0].z != 8) or (d[0].w != 0): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4, +*/ + __pyx_t_2 = __pyx_t_3; + if (__pyx_t_2) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":223 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X2, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4,): + * if (d[0].x != 8): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1, +*/ + __pyx_t_2 = ((__pyx_v_d[0]).x != 8); + if (__pyx_t_2) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":224 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4,): + * if (d[0].x != 8): + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2, +*/ + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":223 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X2, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4,): + * if (d[0].x != 8): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1, +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":217 + * if (d[0].x != 8) or (d[0].y != 8) or (d[0].z != 8) or (d[0].w != 0): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4, +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":225 + * if (d[0].x != 8): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4, +*/ + __pyx_t_1 = (__pyx_v_d[0]).f; + __pyx_t_3 = (__pyx_t_1 == cudaChannelFormatKindSignedNormalized16X1); + if (!__pyx_t_3) { + } else { + __pyx_t_2 = __pyx_t_3; + goto __pyx_L25_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":226 + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X1, +*/ + __pyx_t_3 = (__pyx_t_1 == cudaChannelFormatKindSignedNormalized16X2); + if (!__pyx_t_3) { + } else { + __pyx_t_2 = __pyx_t_3; + goto __pyx_L25_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":225 + * if (d[0].x != 8): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4, +*/ + __pyx_t_3 = (__pyx_t_1 == cudaChannelFormatKindSignedNormalized16X4); + if (!__pyx_t_3) { + } else { + __pyx_t_2 = __pyx_t_3; + goto __pyx_L25_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":228 + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X2, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4,): +*/ + __pyx_t_3 = (__pyx_t_1 == cudaChannelFormatKindUnsignedNormalized16X1); + if (!__pyx_t_3) { + } else { + __pyx_t_2 = __pyx_t_3; + goto __pyx_L25_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":225 + * if (d[0].x != 8): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4, +*/ + __pyx_t_3 = (__pyx_t_1 == cudaChannelFormatKindUnsignedNormalized16X2); + if (!__pyx_t_3) { + } else { + __pyx_t_2 = __pyx_t_3; + goto __pyx_L25_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":230 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X1, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X2, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4,): # <<<<<<<<<<<<<< + * if (d[0].x != 16): + * return cudaErrorInvalidChannelDescriptor +*/ + __pyx_t_3 = (__pyx_t_1 == cudaChannelFormatKindUnsignedNormalized16X4); + __pyx_t_2 = __pyx_t_3; + __pyx_L25_bool_binop_done:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":225 + * if (d[0].x != 8): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4, +*/ + __pyx_t_3 = __pyx_t_2; + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":231 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X2, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4,): + * if (d[0].x != 16): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1, +*/ + __pyx_t_3 = ((__pyx_v_d[0]).x != 16); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":232 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4,): + * if (d[0].x != 16): + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB, +*/ + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":231 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X2, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4,): + * if (d[0].x != 16): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1, +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":225 + * if (d[0].x != 8): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2, + * cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4, +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":233 + * if (d[0].x != 16): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2, +*/ + __pyx_t_1 = (__pyx_v_d[0]).f; + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedBlockCompressed1); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L32_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":234 + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB, +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedBlockCompressed1SRGB); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L32_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":233 + * if (d[0].x != 16): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2, +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedBlockCompressed2); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L32_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":236 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB, +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedBlockCompressed2SRGB); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L32_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":233 + * if (d[0].x != 16): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2, +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedBlockCompressed3); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L32_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":238 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4, + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4, +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedBlockCompressed3SRGB); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L32_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":233 + * if (d[0].x != 16): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2, +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedBlockCompressed4); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L32_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":240 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4, + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5, + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5, +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindSignedBlockCompressed4); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L32_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":233 + * if (d[0].x != 16): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2, +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedBlockCompressed5); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L32_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":242 + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5, + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB,): +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindSignedBlockCompressed5); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L32_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":233 + * if (d[0].x != 16): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2, +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedBlockCompressed7); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L32_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":244 + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB,): # <<<<<<<<<<<<<< + * if (d[0].x != 8): + * return cudaErrorInvalidChannelDescriptor +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedBlockCompressed7SRGB); + __pyx_t_3 = __pyx_t_2; + __pyx_L32_bool_binop_done:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":233 + * if (d[0].x != 16): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2, +*/ + __pyx_t_2 = __pyx_t_3; + if (__pyx_t_2) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":245 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB,): + * if (d[0].x != 8): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, +*/ + __pyx_t_2 = ((__pyx_v_d[0]).x != 8); + if (__pyx_t_2) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":246 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB,): + * if (d[0].x != 8): + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): +*/ + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":245 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB,): + * if (d[0].x != 8): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":233 + * if (d[0].x != 16): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2, +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":247 + * if (d[0].x != 8): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): + * if (d[0].x != 16) or (d[0].y != 16) or (d[0].z != 16) or (d[0].w != 0): +*/ + __pyx_t_1 = (__pyx_v_d[0]).f; + __pyx_t_3 = (__pyx_t_1 == cudaChannelFormatKindUnsignedBlockCompressed6H); + if (!__pyx_t_3) { + } else { + __pyx_t_2 = __pyx_t_3; + goto __pyx_L45_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":248 + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): # <<<<<<<<<<<<<< + * if (d[0].x != 16) or (d[0].y != 16) or (d[0].z != 16) or (d[0].w != 0): + * return cudaErrorInvalidChannelDescriptor +*/ + __pyx_t_3 = (__pyx_t_1 == cudaChannelFormatKindSignedBlockCompressed6H); + __pyx_t_2 = __pyx_t_3; + __pyx_L45_bool_binop_done:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":247 + * if (d[0].x != 8): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): + * if (d[0].x != 16) or (d[0].y != 16) or (d[0].z != 16) or (d[0].w != 0): +*/ + __pyx_t_3 = __pyx_t_2; + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":249 + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): + * if (d[0].x != 16) or (d[0].y != 16) or (d[0].z != 16) or (d[0].w != 0): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102,): +*/ + __pyx_t_2 = ((__pyx_v_d[0]).x != 16); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L48_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).y != 16); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L48_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).z != 16); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L48_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).w != 0); + __pyx_t_3 = __pyx_t_2; + __pyx_L48_bool_binop_done:; + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":250 + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): + * if (d[0].x != 16) or (d[0].y != 16) or (d[0].z != 16) or (d[0].w != 0): + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102,): + * if (d[0].x != 10) or (d[0].y != 10) or (d[0].z != 10) or (d[0].w != 2): +*/ + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":249 + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): + * if (d[0].x != 16) or (d[0].y != 16) or (d[0].z != 16) or (d[0].w != 0): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102,): +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":247 + * if (d[0].x != 8): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): + * if (d[0].x != 16) or (d[0].y != 16) or (d[0].z != 16) or (d[0].w != 0): +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":251 + * if (d[0].x != 16) or (d[0].y != 16) or (d[0].z != 16) or (d[0].w != 0): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102,): # <<<<<<<<<<<<<< + * if (d[0].x != 10) or (d[0].y != 10) or (d[0].z != 10) or (d[0].w != 2): + * return cudaErrorInvalidChannelDescriptor +*/ + __pyx_t_1 = (__pyx_v_d[0]).f; + __pyx_t_3 = (__pyx_t_1 == cudaChannelFormatKindUnsignedNormalized1010102); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":252 + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102,): + * if (d[0].x != 10) or (d[0].y != 10) or (d[0].z != 10) or (d[0].w != 2): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * else: +*/ + __pyx_t_2 = ((__pyx_v_d[0]).x != 10); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L53_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).y != 10); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L53_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).z != 10); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L53_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).w != 2); + __pyx_t_3 = __pyx_t_2; + __pyx_L53_bool_binop_done:; + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":253 + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102,): + * if (d[0].x != 10) or (d[0].y != 10) or (d[0].z != 10) or (d[0].w != 2): + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * else: + * return cudaErrorInvalidChannelDescriptor +*/ + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":252 + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102,): + * if (d[0].x != 10) or (d[0].y != 10) or (d[0].z != 10) or (d[0].w != 2): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * else: +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":251 + * if (d[0].x != 16) or (d[0].y != 16) or (d[0].z != 16) or (d[0].w != 0): + * return cudaErrorInvalidChannelDescriptor + * elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102,): # <<<<<<<<<<<<<< + * if (d[0].x != 10) or (d[0].y != 10) or (d[0].z != 10) or (d[0].w != 2): + * return cudaErrorInvalidChannelDescriptor +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":255 + * return cudaErrorInvalidChannelDescriptor + * else: + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * + * # If Y is non-zero, it must match X +*/ + /*else*/ { + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + } + __pyx_L3:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":260 + * # If Z is non-zero, it must match Y + * # If W is non-zero, it must match Z + * if (((d[0].y != 0) and (d[0].y != d[0].x)) or # <<<<<<<<<<<<<< + * ((d[0].z != 0) and (d[0].z != d[0].y)) or + * ((d[0].w != 0) and (d[0].w != d[0].z))): +*/ + __pyx_t_2 = ((__pyx_v_d[0]).y != 0); + if (!__pyx_t_2) { + goto __pyx_L59_next_or; + } else { + } + __pyx_t_2 = ((__pyx_v_d[0]).y != (__pyx_v_d[0]).x); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L58_bool_binop_done; + } + __pyx_L59_next_or:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":261 + * # If W is non-zero, it must match Z + * if (((d[0].y != 0) and (d[0].y != d[0].x)) or + * ((d[0].z != 0) and (d[0].z != d[0].y)) or # <<<<<<<<<<<<<< + * ((d[0].w != 0) and (d[0].w != d[0].z))): + * if d[0].f != cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102: +*/ + __pyx_t_2 = ((__pyx_v_d[0]).z != 0); + if (!__pyx_t_2) { + goto __pyx_L61_next_or; + } else { + } + __pyx_t_2 = ((__pyx_v_d[0]).z != (__pyx_v_d[0]).y); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L58_bool_binop_done; + } + __pyx_L61_next_or:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":262 + * if (((d[0].y != 0) and (d[0].y != d[0].x)) or + * ((d[0].z != 0) and (d[0].z != d[0].y)) or + * ((d[0].w != 0) and (d[0].w != d[0].z))): # <<<<<<<<<<<<<< + * if d[0].f != cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102: + * return cudaErrorInvalidChannelDescriptor +*/ + __pyx_t_2 = ((__pyx_v_d[0]).w != 0); + if (__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L58_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_d[0]).w != (__pyx_v_d[0]).z); + __pyx_t_3 = __pyx_t_2; + __pyx_L58_bool_binop_done:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":260 + * # If Z is non-zero, it must match Y + * # If W is non-zero, it must match Z + * if (((d[0].y != 0) and (d[0].y != d[0].x)) or # <<<<<<<<<<<<<< + * ((d[0].z != 0) and (d[0].z != d[0].y)) or + * ((d[0].w != 0) and (d[0].w != d[0].z))): +*/ + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":263 + * ((d[0].z != 0) and (d[0].z != d[0].y)) or + * ((d[0].w != 0) and (d[0].w != d[0].z))): + * if d[0].f != cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102: # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * if case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): +*/ + __pyx_t_3 = ((__pyx_v_d[0]).f != cudaChannelFormatKindUnsignedNormalized1010102); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":264 + * ((d[0].w != 0) and (d[0].w != d[0].z))): + * if d[0].f != cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102: + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * if case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 1 +*/ + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":263 + * ((d[0].z != 0) and (d[0].z != d[0].y)) or + * ((d[0].w != 0) and (d[0].w != d[0].z))): + * if d[0].f != cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102: # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * if case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":260 + * # If Z is non-zero, it must match Y + * # If W is non-zero, it must match Z + * if (((d[0].y != 0) and (d[0].y != d[0].x)) or # <<<<<<<<<<<<<< + * ((d[0].z != 0) and (d[0].z != d[0].y)) or + * ((d[0].w != 0) and (d[0].w != d[0].z))): +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":265 + * if d[0].f != cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102: + * return cudaErrorInvalidChannelDescriptor + * if case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 0, 0, 0, cudaChannelFormatKindSigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 265, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":266 + * return cudaErrorInvalidChannelDescriptor + * if case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 1 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): +*/ + (__pyx_v_numberOfChannels[0]) = 1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":267 + * if case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 2 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_SIGNED_INT8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":265 + * if d[0].f != cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102: + * return cudaErrorInvalidChannelDescriptor + * if case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":268 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 0, 0, cudaChannelFormatKindSigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 268, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":269 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 2 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): +*/ + (__pyx_v_numberOfChannels[0]) = 2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":270 + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 3 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_SIGNED_INT8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":268 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":271 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 8, 0, cudaChannelFormatKindSigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 271, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":272 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 3 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindSigned): +*/ + (__pyx_v_numberOfChannels[0]) = 3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":273 + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_SIGNED_INT8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":271 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":274 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 8, 8, cudaChannelFormatKindSigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 274, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":275 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":276 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 1 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_SIGNED_INT8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":274 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":277 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 0, 0, 0, cudaChannelFormatKindUnsigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 277, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":278 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 1 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): +*/ + (__pyx_v_numberOfChannels[0]) = 1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":279 + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 2 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_UNSIGNED_INT8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":277 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":280 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 0, 0, cudaChannelFormatKindUnsigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 280, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":281 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 2 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): +*/ + (__pyx_v_numberOfChannels[0]) = 2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":282 + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 3 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_UNSIGNED_INT8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":280 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":283 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 8, 0, cudaChannelFormatKindUnsigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 283, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":284 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 3 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): +*/ + (__pyx_v_numberOfChannels[0]) = 3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":285 + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_UNSIGNED_INT8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":283 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":286 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 8, 8, cudaChannelFormatKindUnsigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 286, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":287 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":288 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 1 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_UNSIGNED_INT8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":286 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":289 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 0, 0, 0, cudaChannelFormatKindSigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 289, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":290 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 1 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): +*/ + (__pyx_v_numberOfChannels[0]) = 1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":291 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 2 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_SIGNED_INT16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":289 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":292 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 16, 0, 0, cudaChannelFormatKindSigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 292, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":293 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 2 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): +*/ + (__pyx_v_numberOfChannels[0]) = 2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":294 + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 3 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_SIGNED_INT16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":292 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":295 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 16, 16, 0, cudaChannelFormatKindSigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 295, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":296 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 3 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindSigned): +*/ + (__pyx_v_numberOfChannels[0]) = 3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":297 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_SIGNED_INT16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":295 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":298 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 16, 16, 16, cudaChannelFormatKindSigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 298, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":299 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":300 + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 1 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_SIGNED_INT16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":298 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":301 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 0, 0, 0, cudaChannelFormatKindUnsigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 301, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":302 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 1 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): +*/ + (__pyx_v_numberOfChannels[0]) = 1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":303 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 2 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_UNSIGNED_INT16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":301 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":304 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 16, 0, 0, cudaChannelFormatKindUnsigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 304, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":305 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 2 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): +*/ + (__pyx_v_numberOfChannels[0]) = 2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":306 + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 3 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_UNSIGNED_INT16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":304 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":307 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 16, 16, 0, cudaChannelFormatKindUnsigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 307, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":308 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 3 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): +*/ + (__pyx_v_numberOfChannels[0]) = 3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":309 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_UNSIGNED_INT16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":307 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":310 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 16, 16, 16, cudaChannelFormatKindUnsigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 310, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":311 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":312 + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 # <<<<<<<<<<<<<< + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 1 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_UNSIGNED_INT16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":310 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":313 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 32, 0, 0, 0, cudaChannelFormatKindSigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 313, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":314 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 1 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): +*/ + (__pyx_v_numberOfChannels[0]) = 1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":315 + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 # <<<<<<<<<<<<<< + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 2 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_SIGNED_INT32; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":313 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":316 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 32, 32, 0, 0, cudaChannelFormatKindSigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 316, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":317 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 2 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): +*/ + (__pyx_v_numberOfChannels[0]) = 2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":318 + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 # <<<<<<<<<<<<<< + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 3 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_SIGNED_INT32; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":316 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":319 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 32, 32, 32, 0, cudaChannelFormatKindSigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 319, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":320 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 3 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindSigned): +*/ + (__pyx_v_numberOfChannels[0]) = 3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":321 + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 # <<<<<<<<<<<<<< + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_SIGNED_INT32; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":319 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":322 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 32, 32, 32, 32, cudaChannelFormatKindSigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 322, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":323 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":324 + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindSigned): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 # <<<<<<<<<<<<<< + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 1 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_SIGNED_INT32; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":322 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindSigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":325 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 32, 0, 0, 0, cudaChannelFormatKindUnsigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 325, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":326 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 1 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): +*/ + (__pyx_v_numberOfChannels[0]) = 1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":327 + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 # <<<<<<<<<<<<<< + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 2 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_UNSIGNED_INT32; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":325 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":328 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 32, 32, 0, 0, cudaChannelFormatKindUnsigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 328, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":329 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 2 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): +*/ + (__pyx_v_numberOfChannels[0]) = 2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":330 + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 # <<<<<<<<<<<<<< + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 3 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_UNSIGNED_INT32; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":328 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":331 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 32, 32, 32, 0, cudaChannelFormatKindUnsigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 331, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":332 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 3 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): +*/ + (__pyx_v_numberOfChannels[0]) = 3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":333 + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 # <<<<<<<<<<<<<< + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_UNSIGNED_INT32; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":331 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":334 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 32, 32, 32, 32, cudaChannelFormatKindUnsigned); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 334, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":335 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":336 + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 1 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_UNSIGNED_INT32; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":334 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":337 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 0, 0, 0, cudaChannelFormatKindFloat); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 337, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":338 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 1 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): +*/ + (__pyx_v_numberOfChannels[0]) = 1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":339 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 2 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_HALF; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":337 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + * elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":340 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 16, 0, 0, cudaChannelFormatKindFloat); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 340, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":341 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 2 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): +*/ + (__pyx_v_numberOfChannels[0]) = 2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":342 + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 3 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_HALF; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":340 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":343 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 16, 16, 0, cudaChannelFormatKindFloat); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 343, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":344 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 3 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindFloat): +*/ + (__pyx_v_numberOfChannels[0]) = 3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":345 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_HALF; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":343 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":346 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 16, 16, 16, cudaChannelFormatKindFloat); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 346, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":347 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":348 + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF # <<<<<<<<<<<<<< + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 1 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_HALF; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":346 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":349 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 32, 0, 0, 0, cudaChannelFormatKindFloat); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 349, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":350 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 1 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): +*/ + (__pyx_v_numberOfChannels[0]) = 1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":351 + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT # <<<<<<<<<<<<<< + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 2 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_FLOAT; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":349 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + * elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":352 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 32, 32, 0, 0, cudaChannelFormatKindFloat); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 352, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":353 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 2 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): +*/ + (__pyx_v_numberOfChannels[0]) = 2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":354 + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT # <<<<<<<<<<<<<< + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 3 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_FLOAT; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":352 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":355 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 32, 32, 32, 0, cudaChannelFormatKindFloat); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 355, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":356 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 3 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindFloat): +*/ + (__pyx_v_numberOfChannels[0]) = 3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":357 + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT # <<<<<<<<<<<<<< + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_FLOAT; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":355 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":358 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 32, 32, 32, 32, cudaChannelFormatKindFloat); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 358, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":359 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindNV12): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":360 + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindFloat): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindNV12): + * numberOfChannels[0] = 3 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_FLOAT; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":358 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindFloat): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":361 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindNV12): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_NV12 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 8, 0, cudaChannelFormatKindNV12); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 361, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":362 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindNV12): + * numberOfChannels[0] = 3 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_NV12 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1): +*/ + (__pyx_v_numberOfChannels[0]) = 3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":363 + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindNV12): + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_NV12 # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_NV12; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":361 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + * elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindNV12): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_NV12 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":364 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_NV12 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 8, 8, cudaChannelFormatKindUnsignedBlockCompressed1); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 364, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":365 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_NV12 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":366 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC1_UNORM; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":364 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_NV12 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":367 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM_SRGB +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 8, 8, cudaChannelFormatKindUnsignedBlockCompressed1SRGB); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 367, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":368 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM_SRGB + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":369 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM_SRGB # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC1_UNORM_SRGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":367 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM_SRGB +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":370 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM_SRGB + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 8, 8, cudaChannelFormatKindUnsignedBlockCompressed2); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 370, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":371 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM_SRGB + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":372 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC2_UNORM; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":370 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM_SRGB + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":373 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM_SRGB +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 8, 8, cudaChannelFormatKindUnsignedBlockCompressed2SRGB); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 373, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":374 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM_SRGB + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":375 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM_SRGB # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC2_UNORM_SRGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":373 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM_SRGB +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":376 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM_SRGB + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 8, 8, cudaChannelFormatKindUnsignedBlockCompressed3); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 376, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":377 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM_SRGB + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":378 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC3_UNORM; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":376 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM_SRGB + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":379 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM_SRGB +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 8, 8, cudaChannelFormatKindUnsignedBlockCompressed3SRGB); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 379, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":380 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM_SRGB + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":381 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM_SRGB # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4): + * numberOfChannels[0] = 1 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC3_UNORM_SRGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":379 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM_SRGB +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":382 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM_SRGB + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_UNORM +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 0, 0, 0, cudaChannelFormatKindUnsignedBlockCompressed4); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 382, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":383 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM_SRGB + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4): + * numberOfChannels[0] = 1 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_UNORM + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4): +*/ + (__pyx_v_numberOfChannels[0]) = 1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":384 + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4): + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_UNORM # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4): + * numberOfChannels[0] = 1 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC4_UNORM; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":382 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM_SRGB + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_UNORM +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":385 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_UNORM + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_SNORM +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 0, 0, 0, cudaChannelFormatKindSignedBlockCompressed4); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 385, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":386 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_UNORM + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4): + * numberOfChannels[0] = 1 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_SNORM + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5): +*/ + (__pyx_v_numberOfChannels[0]) = 1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":387 + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4): + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_SNORM # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5): + * numberOfChannels[0] = 2 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC4_SNORM; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":385 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_UNORM + * elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_SNORM +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":388 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_SNORM + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_UNORM +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 0, 0, cudaChannelFormatKindUnsignedBlockCompressed5); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 388, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":389 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_SNORM + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5): + * numberOfChannels[0] = 2 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_UNORM + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5): +*/ + (__pyx_v_numberOfChannels[0]) = 2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":390 + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5): + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_UNORM # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5): + * numberOfChannels[0] = 2 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC5_UNORM; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":388 + * numberOfChannels[0] = 1 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_SNORM + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_UNORM +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":391 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_UNORM + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_SNORM +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 0, 0, cudaChannelFormatKindSignedBlockCompressed5); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 391, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":392 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_UNORM + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5): + * numberOfChannels[0] = 2 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_SNORM + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H): +*/ + (__pyx_v_numberOfChannels[0]) = 2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":393 + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5): + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_SNORM # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H): + * numberOfChannels[0] = 3 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC5_SNORM; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":391 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_UNORM + * elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_SNORM +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":394 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_SNORM + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_UF16 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 16, 16, 0, cudaChannelFormatKindUnsignedBlockCompressed6H); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 394, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":395 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_SNORM + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H): + * numberOfChannels[0] = 3 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_UF16 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H): +*/ + (__pyx_v_numberOfChannels[0]) = 3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":396 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H): + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_UF16 # <<<<<<<<<<<<<< + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H): + * numberOfChannels[0] = 3 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC6H_UF16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":394 + * numberOfChannels[0] = 2 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_SNORM + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_UF16 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":397 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_UF16 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_SF16 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 16, 16, 16, 0, cudaChannelFormatKindSignedBlockCompressed6H); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 397, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":398 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_UF16 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H): + * numberOfChannels[0] = 3 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_SF16 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7): +*/ + (__pyx_v_numberOfChannels[0]) = 3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":399 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H): + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_SF16 # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC6H_SF16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":397 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_UF16 + * elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_SF16 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":400 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_SF16 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 8, 8, cudaChannelFormatKindUnsignedBlockCompressed7); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 400, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":401 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_SF16 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":402 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM # <<<<<<<<<<<<<< + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC7_UNORM; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":400 + * numberOfChannels[0] = 3 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_SF16 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":403 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM_SRGB +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 8, 8, 8, 8, cudaChannelFormatKindUnsignedBlockCompressed7SRGB); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 403, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":404 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM_SRGB + * elif case_desc(d, 10, 10, 10, 2, cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102): +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":405 + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM_SRGB # <<<<<<<<<<<<<< + * elif case_desc(d, 10, 10, 10, 2, cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102): + * numberOfChannels[0] = 4 +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_BC7_UNORM_SRGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":403 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM + * elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM_SRGB +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":406 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM_SRGB + * elif case_desc(d, 10, 10, 10, 2, cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNORM_INT_101010_2 +*/ + __pyx_t_4 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_case_desc(__pyx_v_d, 10, 10, 10, 2, cudaChannelFormatKindUnsignedNormalized1010102); if (unlikely(__pyx_t_4 == ((int)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 406, __pyx_L1_error) + __pyx_t_3 = (__pyx_t_4 != 0); + if (__pyx_t_3) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":407 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM_SRGB + * elif case_desc(d, 10, 10, 10, 2, cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102): + * numberOfChannels[0] = 4 # <<<<<<<<<<<<<< + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNORM_INT_101010_2 + * else: +*/ + (__pyx_v_numberOfChannels[0]) = 4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":408 + * elif case_desc(d, 10, 10, 10, 2, cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102): + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNORM_INT_101010_2 # <<<<<<<<<<<<<< + * else: + * return cudaErrorInvalidChannelDescriptor +*/ + (__pyx_v_format[0]) = CU_AD_FORMAT_UNORM_INT_101010_2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":406 + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM_SRGB + * elif case_desc(d, 10, 10, 10, 2, cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102): # <<<<<<<<<<<<<< + * numberOfChannels[0] = 4 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNORM_INT_101010_2 +*/ + goto __pyx_L65; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":410 + * format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNORM_INT_101010_2 + * else: + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * + * if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12, +*/ + /*else*/ { + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + } + __pyx_L65:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":412 + * return cudaErrorInvalidChannelDescriptor + * + * if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): +*/ + __pyx_t_1 = (__pyx_v_d[0]).f; + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindNV12); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L67_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":413 + * + * if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12, + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): + * if numberOfChannels[0] != 3: +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindUnsignedBlockCompressed6H); + if (!__pyx_t_2) { + } else { + __pyx_t_3 = __pyx_t_2; + goto __pyx_L67_bool_binop_done; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":412 + * return cudaErrorInvalidChannelDescriptor + * + * if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): +*/ + __pyx_t_2 = (__pyx_t_1 == cudaChannelFormatKindSignedBlockCompressed6H); + __pyx_t_3 = __pyx_t_2; + __pyx_L67_bool_binop_done:; + __pyx_t_2 = __pyx_t_3; + if (__pyx_t_2) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":415 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): + * if numberOfChannels[0] != 3: # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * else: +*/ + __pyx_t_2 = ((__pyx_v_numberOfChannels[0]) != 3); + if (__pyx_t_2) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":416 + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): + * if numberOfChannels[0] != 3: + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * else: + * if (numberOfChannels[0] != 1) and (numberOfChannels[0] != 2) and (numberOfChannels[0] != 4): +*/ + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":415 + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): + * if numberOfChannels[0] != 3: # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * else: +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":412 + * return cudaErrorInvalidChannelDescriptor + * + * if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12, # <<<<<<<<<<<<<< + * cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, + * cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): +*/ + goto __pyx_L66; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":418 + * return cudaErrorInvalidChannelDescriptor + * else: + * if (numberOfChannels[0] != 1) and (numberOfChannels[0] != 2) and (numberOfChannels[0] != 4): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * return cudaSuccess +*/ + /*else*/ { + __pyx_t_3 = ((__pyx_v_numberOfChannels[0]) != 1); + if (__pyx_t_3) { + } else { + __pyx_t_2 = __pyx_t_3; + goto __pyx_L72_bool_binop_done; + } + __pyx_t_3 = ((__pyx_v_numberOfChannels[0]) != 2); + if (__pyx_t_3) { + } else { + __pyx_t_2 = __pyx_t_3; + goto __pyx_L72_bool_binop_done; + } + __pyx_t_3 = ((__pyx_v_numberOfChannels[0]) != 4); + __pyx_t_2 = __pyx_t_3; + __pyx_L72_bool_binop_done:; + if (__pyx_t_2) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":419 + * else: + * if (numberOfChannels[0] != 1) and (numberOfChannels[0] != 2) and (numberOfChannels[0] != 4): + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * return cudaSuccess + * +*/ + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":418 + * return cudaErrorInvalidChannelDescriptor + * else: + * if (numberOfChannels[0] != 1) and (numberOfChannels[0] != 2) and (numberOfChannels[0] != 4): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * return cudaSuccess +*/ + } + } + __pyx_L66:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":420 + * if (numberOfChannels[0] != 1) and (numberOfChannels[0] != 2) and (numberOfChannels[0] != 4): + * return cudaErrorInvalidChannelDescriptor + * return cudaSuccess # <<<<<<<<<<<<<< + * + * cdef cudaError_t getChannelFormatDescFromDriverDesc(cudaChannelFormatDesc* pRuntimeDesc, size_t* pDepth, size_t* pHeight, size_t* pWidth, const cydriver.CUDA_ARRAY3D_DESCRIPTOR_v2* pDriverDesc) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaSuccess; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":205 + * + * + * cdef cudaError_t getDescInfo(const cudaChannelFormatDesc* d, int *numberOfChannels, cydriver.CUarray_format *format) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * # Check validity + * if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSigned, +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime.getDescInfo", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":422 + * return cudaSuccess + * + * cdef cudaError_t getChannelFormatDescFromDriverDesc(cudaChannelFormatDesc* pRuntimeDesc, size_t* pDepth, size_t* pHeight, size_t* pWidth, const cydriver.CUDA_ARRAY3D_DESCRIPTOR_v2* pDriverDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef int channel_size = 0 + * if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT8: +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getChannelFormatDescFromDriverDesc(struct cudaChannelFormatDesc *__pyx_v_pRuntimeDesc, size_t *__pyx_v_pDepth, size_t *__pyx_v_pHeight, size_t *__pyx_v_pWidth, CUDA_ARRAY3D_DESCRIPTOR_v2 const *__pyx_v_pDriverDesc) { + int __pyx_v_channel_size; + cudaError_t __pyx_r; + int __pyx_t_1; + int __pyx_t_2; + unsigned int __pyx_t_3; + size_t __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":423 + * + * cdef cudaError_t getChannelFormatDescFromDriverDesc(cudaChannelFormatDesc* pRuntimeDesc, size_t* pDepth, size_t* pHeight, size_t* pWidth, const cydriver.CUDA_ARRAY3D_DESCRIPTOR_v2* pDriverDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef int channel_size = 0 # <<<<<<<<<<<<<< + * if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT8: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned +*/ + __pyx_v_channel_size = 0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":424 + * cdef cudaError_t getChannelFormatDescFromDriverDesc(cudaChannelFormatDesc* pRuntimeDesc, size_t* pDepth, size_t* pHeight, size_t* pWidth, const cydriver.CUDA_ARRAY3D_DESCRIPTOR_v2* pDriverDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef int channel_size = 0 + * if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT8: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_UNSIGNED_INT8); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":425 + * cdef int channel_size = 0 + * if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT8: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT16: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsigned; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":426 + * if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT8: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT16: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":424 + * cdef cudaError_t getChannelFormatDescFromDriverDesc(cudaChannelFormatDesc* pRuntimeDesc, size_t* pDepth, size_t* pHeight, size_t* pWidth, const cydriver.CUDA_ARRAY3D_DESCRIPTOR_v2* pDriverDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef int channel_size = 0 + * if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT8: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":427 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT16: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 16 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_UNSIGNED_INT16); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":428 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT16: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned # <<<<<<<<<<<<<< + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT32: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsigned; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":429 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT16: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 16 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT32: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned +*/ + __pyx_v_channel_size = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":427 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT16: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 16 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":430 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT32: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 32 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_UNSIGNED_INT32); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":431 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT32: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned # <<<<<<<<<<<<<< + * channel_size = 32 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT8: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsigned; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":432 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT32: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 32 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT8: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned +*/ + __pyx_v_channel_size = 32; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":430 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT32: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 32 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":433 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 32 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT8: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_SIGNED_INT8); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":434 + * channel_size = 32 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT8: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT16: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindSigned; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":435 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT8: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT16: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":433 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + * channel_size = 32 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT8: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":436 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT16: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 16 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_SIGNED_INT16); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":437 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT16: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned # <<<<<<<<<<<<<< + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT32: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindSigned; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":438 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT16: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 16 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT32: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned +*/ + __pyx_v_channel_size = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":436 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT16: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 16 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":439 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT32: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 32 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_SIGNED_INT32); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":440 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT32: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned # <<<<<<<<<<<<<< + * channel_size = 32 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_HALF: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindSigned; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":441 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT32: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 32 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_HALF: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat +*/ + __pyx_v_channel_size = 32; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":439 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT32: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 32 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":442 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 32 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_HALF: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat + * channel_size = 16 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_HALF); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":443 + * channel_size = 32 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_HALF: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat # <<<<<<<<<<<<<< + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_FLOAT: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindFloat; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":444 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_HALF: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat + * channel_size = 16 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_FLOAT: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat +*/ + __pyx_v_channel_size = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":442 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + * channel_size = 32 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_HALF: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat + * channel_size = 16 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":445 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_FLOAT: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat + * channel_size = 32 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_FLOAT); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":446 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_FLOAT: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat # <<<<<<<<<<<<<< + * channel_size = 32 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_NV12: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindFloat; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":447 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_FLOAT: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat + * channel_size = 32 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_NV12: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindNV12 +*/ + __pyx_v_channel_size = 32; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":445 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_FLOAT: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat + * channel_size = 32 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":448 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat + * channel_size = 32 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_NV12: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindNV12 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_NV12); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":449 + * channel_size = 32 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_NV12: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindNV12 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X1: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindNV12; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":450 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_NV12: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindNV12 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X1: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X1 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":448 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat + * channel_size = 32 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_NV12: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindNV12 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":451 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindNV12 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X1: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X1 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_UNORM_INT8X1); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":452 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X1: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X1 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X2: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedNormalized8X1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":453 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X1: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X1 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X2 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":451 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindNV12 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X1: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X1 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":454 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X1 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X2: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X2 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_UNORM_INT8X2); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":455 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X2 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X4: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedNormalized8X2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":456 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X2 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X4: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":454 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X1 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X2: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X2 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":457 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X2 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X4: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_UNORM_INT8X4); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":458 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X4: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X1: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedNormalized8X4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":459 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X4: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X1: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":457 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X2 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X4: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":460 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X1: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_SNORM_INT8X1); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":461 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X1: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X2: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindSignedNormalized8X1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":462 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X1: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":460 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X1: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":463 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X2: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_SNORM_INT8X2); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":464 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X4: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindSignedNormalized8X2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":465 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X4: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":463 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X2: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":466 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X4: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_SNORM_INT8X4); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":467 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X4: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X1: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindSignedNormalized8X4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":468 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X4: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X1: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X1 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":466 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X4: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":469 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X1: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X1 + * channel_size = 16 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_UNORM_INT16X1); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":470 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X1: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X1 # <<<<<<<<<<<<<< + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X2: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedNormalized16X1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":471 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X1: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X1 + * channel_size = 16 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X2 +*/ + __pyx_v_channel_size = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":469 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X1: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X1 + * channel_size = 16 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":472 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X1 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X2: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X2 + * channel_size = 16 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_UNORM_INT16X2); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":473 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X2 # <<<<<<<<<<<<<< + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X4: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedNormalized16X2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":474 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X2 + * channel_size = 16 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X4: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4 +*/ + __pyx_v_channel_size = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":472 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X1 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X2: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X2 + * channel_size = 16 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":475 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X2 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X4: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4 + * channel_size = 16 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_UNORM_INT16X4); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":476 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X4: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4 # <<<<<<<<<<<<<< + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X1: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedNormalized16X4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":477 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X4: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4 + * channel_size = 16 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X1: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1 +*/ + __pyx_v_channel_size = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":475 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X2 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X4: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4 + * channel_size = 16 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":478 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X1: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1 + * channel_size = 16 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_SNORM_INT16X1); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":479 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X1: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1 # <<<<<<<<<<<<<< + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X2: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindSignedNormalized16X1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":480 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X1: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1 + * channel_size = 16 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2 +*/ + __pyx_v_channel_size = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":478 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X1: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1 + * channel_size = 16 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":481 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X2: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2 + * channel_size = 16 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_SNORM_INT16X2); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":482 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2 # <<<<<<<<<<<<<< + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X4: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindSignedNormalized16X2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":483 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2 + * channel_size = 16 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X4: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4 +*/ + __pyx_v_channel_size = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":481 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X2: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2 + * channel_size = 16 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":484 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X4: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4 + * channel_size = 16 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_SNORM_INT16X4); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":485 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X4: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4 # <<<<<<<<<<<<<< + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindSignedNormalized16X4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":486 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X4: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4 + * channel_size = 16 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1 +*/ + __pyx_v_channel_size = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":484 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X4: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4 + * channel_size = 16 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":487 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC1_UNORM); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":488 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM_SRGB: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedBlockCompressed1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":489 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM_SRGB: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":487 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":490 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM_SRGB: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC1_UNORM_SRGB); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":491 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM_SRGB: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedBlockCompressed1SRGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":492 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM_SRGB: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":490 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM_SRGB: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":493 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC2_UNORM); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":494 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM_SRGB: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedBlockCompressed2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":495 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM_SRGB: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":493 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":496 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM_SRGB: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC2_UNORM_SRGB); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":497 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM_SRGB: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedBlockCompressed2SRGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":498 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM_SRGB: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":496 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM_SRGB: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":499 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC3_UNORM); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":500 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM_SRGB: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedBlockCompressed3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":501 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM_SRGB: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":499 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":502 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM_SRGB: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC3_UNORM_SRGB); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":503 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM_SRGB: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_UNORM: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedBlockCompressed3SRGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":504 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM_SRGB: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":502 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM_SRGB: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":505 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_UNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC4_UNORM); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":506 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_SNORM: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedBlockCompressed4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":507 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_SNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":505 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_UNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":508 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_SNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC4_SNORM); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":509 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_SNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_UNORM: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindSignedBlockCompressed4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":510 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_SNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":508 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_SNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":511 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_UNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC5_UNORM); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":512 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_SNORM: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedBlockCompressed5; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":513 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_SNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":511 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_UNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":514 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_SNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC5_SNORM); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":515 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_SNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_UF16: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindSignedBlockCompressed5; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":516 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_SNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_UF16: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":514 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_SNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":517 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_UF16: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H + * channel_size = 16 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC6H_UF16); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":518 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_UF16: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H # <<<<<<<<<<<<<< + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_SF16: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedBlockCompressed6H; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":519 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_UF16: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H + * channel_size = 16 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_SF16: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H +*/ + __pyx_v_channel_size = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":517 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_UF16: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H + * channel_size = 16 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":520 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_SF16: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H + * channel_size = 16 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC6H_SF16); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":521 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_SF16: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H # <<<<<<<<<<<<<< + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindSignedBlockCompressed6H; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":522 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_SF16: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H + * channel_size = 16 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7 +*/ + __pyx_v_channel_size = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":520 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_SF16: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H + * channel_size = 16 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":523 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7 + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC7_UNORM); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":524 + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7 # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM_SRGB: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedBlockCompressed7; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":525 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7 + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM_SRGB: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":523 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H + * channel_size = 16 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7 + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":526 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM_SRGB: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB + * channel_size = 8 +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_BC7_UNORM_SRGB); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":527 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM_SRGB: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB # <<<<<<<<<<<<<< + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT_101010_2: +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedBlockCompressed7SRGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":528 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM_SRGB: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB + * channel_size = 8 # <<<<<<<<<<<<<< + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT_101010_2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102 +*/ + __pyx_v_channel_size = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":526 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM_SRGB: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB + * channel_size = 8 +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":529 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT_101010_2: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102 + * else: +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_UNORM_INT_101010_2); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":530 + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT_101010_2: + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102 # <<<<<<<<<<<<<< + * else: + * return cudaErrorInvalidChannelDescriptor +*/ + (__pyx_v_pRuntimeDesc[0]).f = cudaChannelFormatKindUnsignedNormalized1010102; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":529 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB + * channel_size = 8 + * elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT_101010_2: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102 + * else: +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":532 + * pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102 + * else: + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * + * # populate bits per channel +*/ + /*else*/ { + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + } + __pyx_L3:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":535 + * + * # populate bits per channel + * pRuntimeDesc[0].x = 0 # <<<<<<<<<<<<<< + * pRuntimeDesc[0].y = 0 + * pRuntimeDesc[0].z = 0 +*/ + (__pyx_v_pRuntimeDesc[0]).x = 0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":536 + * # populate bits per channel + * pRuntimeDesc[0].x = 0 + * pRuntimeDesc[0].y = 0 # <<<<<<<<<<<<<< + * pRuntimeDesc[0].z = 0 + * pRuntimeDesc[0].w = 0 +*/ + (__pyx_v_pRuntimeDesc[0]).y = 0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":537 + * pRuntimeDesc[0].x = 0 + * pRuntimeDesc[0].y = 0 + * pRuntimeDesc[0].z = 0 # <<<<<<<<<<<<<< + * pRuntimeDesc[0].w = 0 + * +*/ + (__pyx_v_pRuntimeDesc[0]).z = 0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":538 + * pRuntimeDesc[0].y = 0 + * pRuntimeDesc[0].z = 0 + * pRuntimeDesc[0].w = 0 # <<<<<<<<<<<<<< + * + * if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT_101010_2 and pDriverDesc[0].NumChannels == 4: +*/ + (__pyx_v_pRuntimeDesc[0]).w = 0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":540 + * pRuntimeDesc[0].w = 0 + * + * if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT_101010_2 and pDriverDesc[0].NumChannels == 4: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].w = 2 + * pRuntimeDesc[0].z = 10 +*/ + __pyx_t_2 = ((__pyx_v_pDriverDesc[0]).Format == CU_AD_FORMAT_UNORM_INT_101010_2); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L5_bool_binop_done; + } + __pyx_t_2 = ((__pyx_v_pDriverDesc[0]).NumChannels == 4); + __pyx_t_1 = __pyx_t_2; + __pyx_L5_bool_binop_done:; + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":541 + * + * if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT_101010_2 and pDriverDesc[0].NumChannels == 4: + * pRuntimeDesc[0].w = 2 # <<<<<<<<<<<<<< + * pRuntimeDesc[0].z = 10 + * pRuntimeDesc[0].y = 10 +*/ + (__pyx_v_pRuntimeDesc[0]).w = 2; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":542 + * if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT_101010_2 and pDriverDesc[0].NumChannels == 4: + * pRuntimeDesc[0].w = 2 + * pRuntimeDesc[0].z = 10 # <<<<<<<<<<<<<< + * pRuntimeDesc[0].y = 10 + * pRuntimeDesc[0].x = 10 +*/ + (__pyx_v_pRuntimeDesc[0]).z = 10; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":543 + * pRuntimeDesc[0].w = 2 + * pRuntimeDesc[0].z = 10 + * pRuntimeDesc[0].y = 10 # <<<<<<<<<<<<<< + * pRuntimeDesc[0].x = 10 + * else: +*/ + (__pyx_v_pRuntimeDesc[0]).y = 10; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":544 + * pRuntimeDesc[0].z = 10 + * pRuntimeDesc[0].y = 10 + * pRuntimeDesc[0].x = 10 # <<<<<<<<<<<<<< + * else: + * if pDriverDesc[0].NumChannels >= 4: +*/ + (__pyx_v_pRuntimeDesc[0]).x = 10; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":540 + * pRuntimeDesc[0].w = 0 + * + * if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT_101010_2 and pDriverDesc[0].NumChannels == 4: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].w = 2 + * pRuntimeDesc[0].z = 10 +*/ + goto __pyx_L4; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":546 + * pRuntimeDesc[0].x = 10 + * else: + * if pDriverDesc[0].NumChannels >= 4: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].w = channel_size + * if pDriverDesc[0].NumChannels >= 3: +*/ + /*else*/ { + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).NumChannels >= 4); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":547 + * else: + * if pDriverDesc[0].NumChannels >= 4: + * pRuntimeDesc[0].w = channel_size # <<<<<<<<<<<<<< + * if pDriverDesc[0].NumChannels >= 3: + * pRuntimeDesc[0].z = channel_size +*/ + (__pyx_v_pRuntimeDesc[0]).w = __pyx_v_channel_size; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":546 + * pRuntimeDesc[0].x = 10 + * else: + * if pDriverDesc[0].NumChannels >= 4: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].w = channel_size + * if pDriverDesc[0].NumChannels >= 3: +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":548 + * if pDriverDesc[0].NumChannels >= 4: + * pRuntimeDesc[0].w = channel_size + * if pDriverDesc[0].NumChannels >= 3: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].z = channel_size + * if pDriverDesc[0].NumChannels >= 2: +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).NumChannels >= 3); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":549 + * pRuntimeDesc[0].w = channel_size + * if pDriverDesc[0].NumChannels >= 3: + * pRuntimeDesc[0].z = channel_size # <<<<<<<<<<<<<< + * if pDriverDesc[0].NumChannels >= 2: + * pRuntimeDesc[0].y = channel_size +*/ + (__pyx_v_pRuntimeDesc[0]).z = __pyx_v_channel_size; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":548 + * if pDriverDesc[0].NumChannels >= 4: + * pRuntimeDesc[0].w = channel_size + * if pDriverDesc[0].NumChannels >= 3: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].z = channel_size + * if pDriverDesc[0].NumChannels >= 2: +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":550 + * if pDriverDesc[0].NumChannels >= 3: + * pRuntimeDesc[0].z = channel_size + * if pDriverDesc[0].NumChannels >= 2: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].y = channel_size + * if pDriverDesc[0].NumChannels >= 1: +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).NumChannels >= 2); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":551 + * pRuntimeDesc[0].z = channel_size + * if pDriverDesc[0].NumChannels >= 2: + * pRuntimeDesc[0].y = channel_size # <<<<<<<<<<<<<< + * if pDriverDesc[0].NumChannels >= 1: + * pRuntimeDesc[0].x = channel_size +*/ + (__pyx_v_pRuntimeDesc[0]).y = __pyx_v_channel_size; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":550 + * if pDriverDesc[0].NumChannels >= 3: + * pRuntimeDesc[0].z = channel_size + * if pDriverDesc[0].NumChannels >= 2: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].y = channel_size + * if pDriverDesc[0].NumChannels >= 1: +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":552 + * if pDriverDesc[0].NumChannels >= 2: + * pRuntimeDesc[0].y = channel_size + * if pDriverDesc[0].NumChannels >= 1: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].x = channel_size + * +*/ + __pyx_t_1 = ((__pyx_v_pDriverDesc[0]).NumChannels >= 1); + if (__pyx_t_1) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":553 + * pRuntimeDesc[0].y = channel_size + * if pDriverDesc[0].NumChannels >= 1: + * pRuntimeDesc[0].x = channel_size # <<<<<<<<<<<<<< + * + * if pDriverDesc[0].NumChannels not in (4, 3, 2, 1): +*/ + (__pyx_v_pRuntimeDesc[0]).x = __pyx_v_channel_size; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":552 + * if pDriverDesc[0].NumChannels >= 2: + * pRuntimeDesc[0].y = channel_size + * if pDriverDesc[0].NumChannels >= 1: # <<<<<<<<<<<<<< + * pRuntimeDesc[0].x = channel_size + * +*/ + } + } + __pyx_L4:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":555 + * pRuntimeDesc[0].x = channel_size + * + * if pDriverDesc[0].NumChannels not in (4, 3, 2, 1): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * +*/ + __pyx_t_3 = (__pyx_v_pDriverDesc[0]).NumChannels; + __pyx_t_2 = (__pyx_t_3 != 4); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L12_bool_binop_done; + } + __pyx_t_2 = (__pyx_t_3 != 3); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L12_bool_binop_done; + } + __pyx_t_2 = (__pyx_t_3 != 2); + if (__pyx_t_2) { + } else { + __pyx_t_1 = __pyx_t_2; + goto __pyx_L12_bool_binop_done; + } + __pyx_t_2 = (__pyx_t_3 != 1); + __pyx_t_1 = __pyx_t_2; + __pyx_L12_bool_binop_done:; + __pyx_t_2 = __pyx_t_1; + if (__pyx_t_2) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":556 + * + * if pDriverDesc[0].NumChannels not in (4, 3, 2, 1): + * return cudaErrorInvalidChannelDescriptor # <<<<<<<<<<<<<< + * + * # populate dimensions +*/ + __pyx_r = cudaErrorInvalidChannelDescriptor; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":555 + * pRuntimeDesc[0].x = channel_size + * + * if pDriverDesc[0].NumChannels not in (4, 3, 2, 1): # <<<<<<<<<<<<<< + * return cudaErrorInvalidChannelDescriptor + * +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":559 + * + * # populate dimensions + * if pDepth != NULL: # <<<<<<<<<<<<<< + * pDepth[0] = pDriverDesc[0].Depth + * if pHeight != NULL: +*/ + __pyx_t_2 = (__pyx_v_pDepth != NULL); + if (__pyx_t_2) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":560 + * # populate dimensions + * if pDepth != NULL: + * pDepth[0] = pDriverDesc[0].Depth # <<<<<<<<<<<<<< + * if pHeight != NULL: + * pHeight[0] = pDriverDesc[0].Height +*/ + __pyx_t_4 = (__pyx_v_pDriverDesc[0]).Depth; + (__pyx_v_pDepth[0]) = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":559 + * + * # populate dimensions + * if pDepth != NULL: # <<<<<<<<<<<<<< + * pDepth[0] = pDriverDesc[0].Depth + * if pHeight != NULL: +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":561 + * if pDepth != NULL: + * pDepth[0] = pDriverDesc[0].Depth + * if pHeight != NULL: # <<<<<<<<<<<<<< + * pHeight[0] = pDriverDesc[0].Height + * if pWidth != NULL: +*/ + __pyx_t_2 = (__pyx_v_pHeight != NULL); + if (__pyx_t_2) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":562 + * pDepth[0] = pDriverDesc[0].Depth + * if pHeight != NULL: + * pHeight[0] = pDriverDesc[0].Height # <<<<<<<<<<<<<< + * if pWidth != NULL: + * pWidth[0] = pDriverDesc[0].Width +*/ + __pyx_t_4 = (__pyx_v_pDriverDesc[0]).Height; + (__pyx_v_pHeight[0]) = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":561 + * if pDepth != NULL: + * pDepth[0] = pDriverDesc[0].Depth + * if pHeight != NULL: # <<<<<<<<<<<<<< + * pHeight[0] = pDriverDesc[0].Height + * if pWidth != NULL: +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":563 + * if pHeight != NULL: + * pHeight[0] = pDriverDesc[0].Height + * if pWidth != NULL: # <<<<<<<<<<<<<< + * pWidth[0] = pDriverDesc[0].Width + * return cudaSuccess +*/ + __pyx_t_2 = (__pyx_v_pWidth != NULL); + if (__pyx_t_2) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":564 + * pHeight[0] = pDriverDesc[0].Height + * if pWidth != NULL: + * pWidth[0] = pDriverDesc[0].Width # <<<<<<<<<<<<<< + * return cudaSuccess + * +*/ + __pyx_t_4 = (__pyx_v_pDriverDesc[0]).Width; + (__pyx_v_pWidth[0]) = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":563 + * if pHeight != NULL: + * pHeight[0] = pDriverDesc[0].Height + * if pWidth != NULL: # <<<<<<<<<<<<<< + * pWidth[0] = pDriverDesc[0].Width + * return cudaSuccess +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":565 + * if pWidth != NULL: + * pWidth[0] = pDriverDesc[0].Width + * return cudaSuccess # <<<<<<<<<<<<<< + * + * cdef cudaError_t getDriverEglFrame(cydriver.CUeglFrame *cuEglFrame, cyruntime.cudaEglFrame eglFrame) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaSuccess; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":422 + * return cudaSuccess + * + * cdef cudaError_t getChannelFormatDescFromDriverDesc(cudaChannelFormatDesc* pRuntimeDesc, size_t* pDepth, size_t* pHeight, size_t* pWidth, const cydriver.CUDA_ARRAY3D_DESCRIPTOR_v2* pDriverDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef int channel_size = 0 + * if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT8: +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":567 + * return cudaSuccess + * + * cdef cudaError_t getDriverEglFrame(cydriver.CUeglFrame *cuEglFrame, cyruntime.cudaEglFrame eglFrame) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * cdef unsigned int i = 0 +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getDriverEglFrame(__pyx_t_4cuda_8bindings_8cydriver_CUeglFrame *__pyx_v_cuEglFrame, __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame __pyx_v_eglFrame) { + cudaError_t __pyx_v_err; + unsigned int __pyx_v_i; + cudaError_t __pyx_r; + cudaError_t __pyx_t_1; + int __pyx_t_2; + unsigned int __pyx_t_3; + unsigned int __pyx_t_4; + unsigned int __pyx_t_5; + void *__pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":568 + * + * cdef cudaError_t getDriverEglFrame(cydriver.CUeglFrame *cuEglFrame, cyruntime.cudaEglFrame eglFrame) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * cdef unsigned int i = 0 + * +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":569 + * cdef cudaError_t getDriverEglFrame(cydriver.CUeglFrame *cuEglFrame, cyruntime.cudaEglFrame eglFrame) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess + * cdef unsigned int i = 0 # <<<<<<<<<<<<<< + * + * err = getDescInfo(&eglFrame.planeDesc[0].channelDesc, &cuEglFrame[0].numChannels, &cuEglFrame[0].cuFormat) +*/ + __pyx_v_i = 0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":571 + * cdef unsigned int i = 0 + * + * err = getDescInfo(&eglFrame.planeDesc[0].channelDesc, &cuEglFrame[0].numChannels, &cuEglFrame[0].cuFormat) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getDescInfo((&(__pyx_v_eglFrame.planeDesc[0]).channelDesc), ((int *)(&(__pyx_v_cuEglFrame[0]).numChannels)), (&(__pyx_v_cuEglFrame[0]).cuFormat)); if (unlikely(__pyx_t_1 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 571, __pyx_L1_error) + __pyx_v_err = __pyx_t_1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":572 + * + * err = getDescInfo(&eglFrame.planeDesc[0].channelDesc, &cuEglFrame[0].numChannels, &cuEglFrame[0].cuFormat) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * for i in range(eglFrame.planeCount): +*/ + __pyx_t_2 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_2) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":573 + * err = getDescInfo(&eglFrame.planeDesc[0].channelDesc, &cuEglFrame[0].numChannels, &cuEglFrame[0].cuFormat) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * for i in range(eglFrame.planeCount): + * if eglFrame.frameType == cyruntime.cudaEglFrameTypeArray: +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":572 + * + * err = getDescInfo(&eglFrame.planeDesc[0].channelDesc, &cuEglFrame[0].numChannels, &cuEglFrame[0].cuFormat) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * for i in range(eglFrame.planeCount): +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":574 + * if err != cudaSuccess: + * return err + * for i in range(eglFrame.planeCount): # <<<<<<<<<<<<<< + * if eglFrame.frameType == cyruntime.cudaEglFrameTypeArray: + * cuEglFrame[0].frame.pArray[i] = eglFrame.frame.pArray[i] +*/ + __pyx_t_3 = __pyx_v_eglFrame.planeCount; + __pyx_t_4 = __pyx_t_3; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":575 + * return err + * for i in range(eglFrame.planeCount): + * if eglFrame.frameType == cyruntime.cudaEglFrameTypeArray: # <<<<<<<<<<<<<< + * cuEglFrame[0].frame.pArray[i] = eglFrame.frame.pArray[i] + * else: +*/ + __pyx_t_2 = (__pyx_v_eglFrame.frameType == __pyx_e_4cuda_8bindings_9cyruntime_cudaEglFrameTypeArray); + if (__pyx_t_2) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":576 + * for i in range(eglFrame.planeCount): + * if eglFrame.frameType == cyruntime.cudaEglFrameTypeArray: + * cuEglFrame[0].frame.pArray[i] = eglFrame.frame.pArray[i] # <<<<<<<<<<<<<< + * else: + * cuEglFrame[0].frame.pPitch[i] = eglFrame.frame.pPitch[i].ptr +*/ + ((__pyx_v_cuEglFrame[0]).frame.pArray[__pyx_v_i]) = ((CUarray)(__pyx_v_eglFrame.frame.pArray[__pyx_v_i])); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":575 + * return err + * for i in range(eglFrame.planeCount): + * if eglFrame.frameType == cyruntime.cudaEglFrameTypeArray: # <<<<<<<<<<<<<< + * cuEglFrame[0].frame.pArray[i] = eglFrame.frame.pArray[i] + * else: +*/ + goto __pyx_L6; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":578 + * cuEglFrame[0].frame.pArray[i] = eglFrame.frame.pArray[i] + * else: + * cuEglFrame[0].frame.pPitch[i] = eglFrame.frame.pPitch[i].ptr # <<<<<<<<<<<<<< + * cuEglFrame[0].width = eglFrame.planeDesc[0].width + * cuEglFrame[0].height = eglFrame.planeDesc[0].height +*/ + /*else*/ { + __pyx_t_6 = (__pyx_v_eglFrame.frame.pPitch[__pyx_v_i]).ptr; + ((__pyx_v_cuEglFrame[0]).frame.pPitch[__pyx_v_i]) = __pyx_t_6; + } + __pyx_L6:; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":579 + * else: + * cuEglFrame[0].frame.pPitch[i] = eglFrame.frame.pPitch[i].ptr + * cuEglFrame[0].width = eglFrame.planeDesc[0].width # <<<<<<<<<<<<<< + * cuEglFrame[0].height = eglFrame.planeDesc[0].height + * cuEglFrame[0].depth = eglFrame.planeDesc[0].depth +*/ + __pyx_t_3 = (__pyx_v_eglFrame.planeDesc[0]).width; + (__pyx_v_cuEglFrame[0]).width = __pyx_t_3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":580 + * cuEglFrame[0].frame.pPitch[i] = eglFrame.frame.pPitch[i].ptr + * cuEglFrame[0].width = eglFrame.planeDesc[0].width + * cuEglFrame[0].height = eglFrame.planeDesc[0].height # <<<<<<<<<<<<<< + * cuEglFrame[0].depth = eglFrame.planeDesc[0].depth + * cuEglFrame[0].pitch = eglFrame.planeDesc[0].pitch +*/ + __pyx_t_3 = (__pyx_v_eglFrame.planeDesc[0]).height; + (__pyx_v_cuEglFrame[0]).height = __pyx_t_3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":581 + * cuEglFrame[0].width = eglFrame.planeDesc[0].width + * cuEglFrame[0].height = eglFrame.planeDesc[0].height + * cuEglFrame[0].depth = eglFrame.planeDesc[0].depth # <<<<<<<<<<<<<< + * cuEglFrame[0].pitch = eglFrame.planeDesc[0].pitch + * cuEglFrame[0].planeCount = eglFrame.planeCount +*/ + __pyx_t_3 = (__pyx_v_eglFrame.planeDesc[0]).depth; + (__pyx_v_cuEglFrame[0]).depth = __pyx_t_3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":582 + * cuEglFrame[0].height = eglFrame.planeDesc[0].height + * cuEglFrame[0].depth = eglFrame.planeDesc[0].depth + * cuEglFrame[0].pitch = eglFrame.planeDesc[0].pitch # <<<<<<<<<<<<<< + * cuEglFrame[0].planeCount = eglFrame.planeCount + * if eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar: +*/ + __pyx_t_3 = (__pyx_v_eglFrame.planeDesc[0]).pitch; + (__pyx_v_cuEglFrame[0]).pitch = __pyx_t_3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":583 + * cuEglFrame[0].depth = eglFrame.planeDesc[0].depth + * cuEglFrame[0].pitch = eglFrame.planeDesc[0].pitch + * cuEglFrame[0].planeCount = eglFrame.planeCount # <<<<<<<<<<<<<< + * if eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR +*/ + __pyx_t_3 = __pyx_v_eglFrame.planeCount; + (__pyx_v_cuEglFrame[0]).planeCount = __pyx_t_3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":584 + * cuEglFrame[0].pitch = eglFrame.planeDesc[0].pitch + * cuEglFrame[0].planeCount = eglFrame.planeCount + * if eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar: +*/ + switch (__pyx_v_eglFrame.eglColorFormat) { + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420Planar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":585 + * cuEglFrame[0].planeCount = eglFrame.planeCount + * if eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":584 + * cuEglFrame[0].pitch = eglFrame.planeDesc[0].pitch + * cuEglFrame[0].planeCount = eglFrame.planeCount + * if eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420SemiPlanar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":587 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":586 + * if eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422Planar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV422Planar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":589 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_PLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":588 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422Planar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422SemiPlanar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV422SemiPlanar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":591 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":590 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422SemiPlanar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444Planar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV444Planar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":593 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_PLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":592 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444Planar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444SemiPlanar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV444SemiPlanar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":595 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUYV422: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_422 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":594 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444SemiPlanar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUYV422: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUYV422: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":597 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUYV422: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_422 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY422: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_422 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUYV_422; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":596 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUYV422: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_422 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY422: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY422: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":599 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_422 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY422: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_422 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_422; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":598 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUYV422: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_422 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY422: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_422 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY709: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":601 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_422 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":600 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY422: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_422 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY709: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":603 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_2020 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":602 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY709_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY2020: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":605 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_2020 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatARGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ARGB +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":604 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY2020: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatARGB: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatARGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":607 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatARGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ARGB # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatRGBA: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RGBA +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_ARGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":606 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatARGB: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ARGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatRGBA: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatRGBA: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":609 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ARGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatRGBA: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RGBA # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatABGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ABGR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_RGBA; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":608 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatARGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ARGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatRGBA: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RGBA + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatABGR: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatABGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":611 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RGBA + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatABGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ABGR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBGRA: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BGRA +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_ABGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":610 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatRGBA: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RGBA + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatABGR: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ABGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBGRA: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBGRA: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":613 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ABGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBGRA: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BGRA # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatL: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_L +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BGRA; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":612 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatABGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ABGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBGRA: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BGRA + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatL: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatL: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":615 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BGRA + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatL: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_L # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_R +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_L; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":614 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBGRA: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BGRA + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatL: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_L + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatR: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":617 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_L + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_R # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatA: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_A +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_R; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":616 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatL: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_L + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatR: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_R + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatA: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatA: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":619 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_R + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatA: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_A # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RG +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_A; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":618 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_R + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatA: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_A + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatRG: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatRG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":621 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_A + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RG # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatAYUV: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_RG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":620 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatA: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_A + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatRG: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatAYUV: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatAYUV: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":623 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatAYUV: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_AYUV; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":622 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatAYUV: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444SemiPlanar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU444SemiPlanar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":625 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":624 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatAYUV: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444SemiPlanar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422SemiPlanar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU422SemiPlanar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":627 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":626 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422SemiPlanar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420SemiPlanar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":629 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":628 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_444SemiPlanar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":631 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":630 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":633 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":632 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_444SemiPlanar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":635 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":634 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_420SemiPlanar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":637 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatVYUY_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":636 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatVYUY_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatVYUY_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":639 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatVYUY_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_VYUY_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":638 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatVYUY_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":641 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUYV_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":640 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatVYUY_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUYV_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUYV_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":643 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUYV_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVYU_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUYV_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":642 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUYV_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVYU_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVYU_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":645 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVYU_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUVA_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVYU_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":644 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUYV_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVYU_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUVA_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUVA_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":647 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUVA_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatAYUV_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUVA_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":646 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVYU_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUVA_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatAYUV_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatAYUV_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":649 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatAYUV_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_AYUV_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":648 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUVA_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatAYUV_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444Planar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV444Planar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":651 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":650 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatAYUV_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444Planar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422Planar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV422Planar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":653 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":652 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422Planar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420Planar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":655 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":654 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444SemiPlanar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV444SemiPlanar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":657 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":656 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444SemiPlanar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422SemiPlanar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV422SemiPlanar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":659 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":658 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422SemiPlanar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420SemiPlanar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":661 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":660 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444Planar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU444Planar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":663 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":662 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444Planar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422Planar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU422Planar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":665 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":664 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422Planar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420Planar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":667 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":666 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444SemiPlanar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU444SemiPlanar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":669 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":668 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444SemiPlanar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422SemiPlanar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU422SemiPlanar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":671 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":670 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422SemiPlanar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420SemiPlanar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":673 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerRGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RGGB +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":672 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerRGGB: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerRGGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":675 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerRGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RGGB # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerBGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BGGR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_RGGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":674 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerRGGB: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerBGGR: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerBGGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":677 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerBGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BGGR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerGRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GRBG +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_BGGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":676 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerRGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerBGGR: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerGRBG: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerGRBG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":679 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerGRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GRBG # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerGBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GBRG +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_GRBG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":678 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerBGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerGRBG: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerGBRG: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerGBRG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":681 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerGBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GBRG # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10RGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_RGGB +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_GBRG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":680 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerGRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerGBRG: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10RGGB: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10RGGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":683 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10RGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_RGGB # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10BGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_BGGR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_RGGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":682 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerGBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10RGGB: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10BGGR: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10BGGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":685 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10BGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_BGGR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10GRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GRBG +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_BGGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":684 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10RGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10BGGR: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10GRBG: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10GRBG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":687 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10GRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GRBG # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10GBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GBRG +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_GRBG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":686 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10BGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10GRBG: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10GBRG: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10GBRG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":689 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10GBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GBRG # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12RGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RGGB +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_GBRG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":688 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10GRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10GBRG: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12RGGB: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12RGGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":691 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12RGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RGGB # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12BGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BGGR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_RGGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":690 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10GBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12RGGB: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12BGGR: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12BGGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":693 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12BGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BGGR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12GRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GRBG +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_BGGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":692 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12RGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12BGGR: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12GRBG: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12GRBG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":695 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12GRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GRBG # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12GBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GBRG +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_GRBG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":694 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12BGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12GRBG: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12GBRG: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12GBRG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":697 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12GBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GBRG # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14RGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_RGGB +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_GBRG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":696 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12GRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12GBRG: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14RGGB: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer14RGGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":699 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14RGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_RGGB # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14BGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_BGGR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_RGGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":698 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12GBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14RGGB: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14BGGR: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer14BGGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":701 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14BGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_BGGR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14GRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GRBG +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_BGGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":700 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14RGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14BGGR: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14GRBG: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer14GRBG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":703 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14GRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GRBG # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14GBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GBRG +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_GRBG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":702 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14BGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14GRBG: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14GBRG: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer14GBRG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":705 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14GBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GBRG # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20RGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_RGGB +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_GBRG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":704 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14GRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14GBRG: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20RGGB: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer20RGGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":707 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20RGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_RGGB # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20BGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_BGGR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_RGGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":706 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14GBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20RGGB: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20BGGR: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer20BGGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":709 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20BGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_BGGR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20GRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GRBG +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_BGGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":708 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20RGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20BGGR: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20GRBG: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer20GRBG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":711 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20GRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GRBG # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20GBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GBRG +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_GRBG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":710 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20BGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20GRBG: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20GBRG: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer20GBRG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":713 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20GBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GBRG # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspRGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_GBRG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":712 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20GRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20GBRG: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspRGGB: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerIspRGGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":715 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspRGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspBGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":714 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20GBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspRGGB: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspBGGR: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerIspBGGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":717 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspBGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspGRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":716 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspRGGB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspBGGR: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspGRBG: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerIspGRBG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":719 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspGRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspGBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":718 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspBGGR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspGRBG: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspGBRG: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerIspGBRG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":721 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspGBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":720 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspGRBG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspGBRG: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444Planar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU444Planar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":723 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_PLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":722 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspGBRG: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444Planar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422Planar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU422Planar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":725 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_PLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":724 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422Planar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420Planar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":727 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerBCCR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BCCR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":726 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerBCCR: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerBCCR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":729 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerBCCR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BCCR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerRCCB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RCCB +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_BCCR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":728 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerBCCR: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BCCR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerRCCB: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerRCCB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":731 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BCCR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerRCCB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RCCB # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerCRBC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CRBC +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_RCCB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":730 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerBCCR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BCCR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerRCCB: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RCCB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerCRBC: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerCRBC: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":733 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RCCB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerCRBC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CRBC # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerCBRC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CBRC +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_CRBC; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":732 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerRCCB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RCCB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerCRBC: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CRBC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerCBRC: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerCBRC: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":735 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CRBC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerCBRC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CBRC # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10CCCC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_CCCC +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_CBRC; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":734 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerCRBC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CRBC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerCBRC: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CBRC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10CCCC: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10CCCC: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":737 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CBRC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10CCCC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_CCCC # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12BCCR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BCCR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_CCCC; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":736 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerCBRC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CBRC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10CCCC: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_CCCC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12BCCR: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12BCCR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":739 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_CCCC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12BCCR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BCCR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12RCCB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RCCB +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_BCCR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":738 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10CCCC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_CCCC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12BCCR: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BCCR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12RCCB: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12RCCB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":741 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BCCR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12RCCB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RCCB # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CRBC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CRBC +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_RCCB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":740 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12BCCR: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BCCR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12RCCB: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RCCB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CRBC: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12CRBC: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":743 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RCCB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CRBC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CRBC # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CBRC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CBRC +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CRBC; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":742 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12RCCB: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RCCB + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CRBC: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CRBC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CBRC: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12CBRC: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":745 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CRBC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CBRC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CBRC # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CCCC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CCCC +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CBRC; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":744 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CRBC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CRBC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CBRC: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CBRC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CCCC: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12CCCC: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":747 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CBRC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CCCC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CCCC # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CCCC; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":746 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CBRC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CBRC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CCCC: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CCCC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":749 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CCCC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":748 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CCCC: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CCCC + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_2020: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420SemiPlanar_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":751 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":750 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_2020: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_2020: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420SemiPlanar_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":753 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":752 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_2020: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_2020: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420Planar_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":755 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":754 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_2020: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_2020: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420Planar_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":757 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":756 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_2020: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_709: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420SemiPlanar_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":759 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":758 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_709: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_709: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420SemiPlanar_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":761 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":760 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_709: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_709: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420Planar_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":763 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":762 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_709: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_709: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420Planar_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":765 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":764 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_709: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":767 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":766 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_2020: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":769 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":768 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_2020: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_2020: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_422SemiPlanar_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":771 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":770 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_2020: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_422SemiPlanar: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":773 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709 +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":772 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_2020: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_709: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_422SemiPlanar_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":775 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709 # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":774 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_709: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":777 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_709_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":776 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_709: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":779 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_709_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":778 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY_709_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":781 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_709_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":780 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":783 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_709_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":782 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10_709_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":785 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_709_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":784 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":787 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_709_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUVA: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":786 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12_709_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUVA: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUVA: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":789 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUVA: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVYU: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUVA; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":788 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUVA: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVYU: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVYU: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":791 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVYU: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatVYUY: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVYU; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":790 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUVA: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVYU: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatVYUY: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatVYUY: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":793 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatVYUY: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_VYUY; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":792 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVYU: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatVYUY: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":795 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":794 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatVYUY: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":797 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":796 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_444SemiPlanar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":799 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":798 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":801 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":800 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_420SemiPlanar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":803 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":802 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":805 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":804 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_444SemiPlanar_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":807 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER # <<<<<<<<<<<<<< + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":806 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_444SemiPlanar_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":809 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_709_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER # <<<<<<<<<<<<<< + * else: + * return cudaErrorInvalidValue +*/ + (__pyx_v_cuEglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":808 + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_ER: + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER + * elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_709_ER: # <<<<<<<<<<<<<< + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER + * else: +*/ + break; + default: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":811 + * cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER + * else: + * return cudaErrorInvalidValue # <<<<<<<<<<<<<< + * if eglFrame.frameType == cyruntime.cudaEglFrameTypeArray: + * cuEglFrame[0].frameType = cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY +*/ + __pyx_r = cudaErrorInvalidValue; + goto __pyx_L0; + break; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":812 + * else: + * return cudaErrorInvalidValue + * if eglFrame.frameType == cyruntime.cudaEglFrameTypeArray: # <<<<<<<<<<<<<< + * cuEglFrame[0].frameType = cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY + * elif eglFrame.frameType == cyruntime.cudaEglFrameTypePitch: +*/ + switch (__pyx_v_eglFrame.frameType) { + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglFrameTypeArray: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":813 + * return cudaErrorInvalidValue + * if eglFrame.frameType == cyruntime.cudaEglFrameTypeArray: + * cuEglFrame[0].frameType = cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY # <<<<<<<<<<<<<< + * elif eglFrame.frameType == cyruntime.cudaEglFrameTypePitch: + * cuEglFrame[0].frameType = cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_PITCH +*/ + (__pyx_v_cuEglFrame[0]).frameType = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_FRAME_TYPE_ARRAY; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":812 + * else: + * return cudaErrorInvalidValue + * if eglFrame.frameType == cyruntime.cudaEglFrameTypeArray: # <<<<<<<<<<<<<< + * cuEglFrame[0].frameType = cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY + * elif eglFrame.frameType == cyruntime.cudaEglFrameTypePitch: +*/ + break; + case __pyx_e_4cuda_8bindings_9cyruntime_cudaEglFrameTypePitch: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":815 + * cuEglFrame[0].frameType = cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY + * elif eglFrame.frameType == cyruntime.cudaEglFrameTypePitch: + * cuEglFrame[0].frameType = cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_PITCH # <<<<<<<<<<<<<< + * else: + * return cudaErrorInvalidValue +*/ + (__pyx_v_cuEglFrame[0]).frameType = __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_FRAME_TYPE_PITCH; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":814 + * if eglFrame.frameType == cyruntime.cudaEglFrameTypeArray: + * cuEglFrame[0].frameType = cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY + * elif eglFrame.frameType == cyruntime.cudaEglFrameTypePitch: # <<<<<<<<<<<<<< + * cuEglFrame[0].frameType = cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_PITCH + * else: +*/ + break; + default: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":817 + * cuEglFrame[0].frameType = cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_PITCH + * else: + * return cudaErrorInvalidValue # <<<<<<<<<<<<<< + * + * @cython.show_performance_hints(False) +*/ + __pyx_r = cudaErrorInvalidValue; + goto __pyx_L0; + break; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":567 + * return cudaSuccess + * + * cdef cudaError_t getDriverEglFrame(cydriver.CUeglFrame *cuEglFrame, cyruntime.cudaEglFrame eglFrame) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * cdef cudaError_t err = cudaSuccess + * cdef unsigned int i = 0 +*/ + + /* function exit code */ + __pyx_r = (enum cudaError) 0; + goto __pyx_L0; + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime.getDriverEglFrame", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":819 + * return cudaErrorInvalidValue + * + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaError_t getRuntimeEglFrame(cyruntime.cudaEglFrame *eglFrame, cydriver.CUeglFrame cueglFrame) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getRuntimeEglFrame(__pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame *__pyx_v_eglFrame, __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame __pyx_v_cueglFrame) { + cudaError_t __pyx_v_err; + unsigned int __pyx_v_i; + CUDA_ARRAY3D_DESCRIPTOR_v2 __pyx_v_ad; + struct cudaPitchedPtr __pyx_v_pPtr; + cudaError_t __pyx_r; + unsigned int __pyx_t_1; + unsigned int __pyx_t_2; + unsigned int __pyx_t_3; + unsigned int __pyx_t_4; + CUarray_format __pyx_t_5; + cudaError_t __pyx_t_6; + int __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":821 + * @cython.show_performance_hints(False) + * cdef cudaError_t getRuntimeEglFrame(cyruntime.cudaEglFrame *eglFrame, cydriver.CUeglFrame cueglFrame) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess # <<<<<<<<<<<<<< + * cdef unsigned int i + * cdef cydriver.CUDA_ARRAY3D_DESCRIPTOR_v2 ad +*/ + __pyx_v_err = cudaSuccess; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":825 + * cdef cydriver.CUDA_ARRAY3D_DESCRIPTOR_v2 ad + * cdef cudaPitchedPtr pPtr + * memset(eglFrame, 0, sizeof(eglFrame[0])) # <<<<<<<<<<<<<< + * memset(&ad, 0, sizeof(ad)) + * for i in range(cueglFrame.planeCount): +*/ + (void)(memset(__pyx_v_eglFrame, 0, (sizeof((__pyx_v_eglFrame[0]))))); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":826 + * cdef cudaPitchedPtr pPtr + * memset(eglFrame, 0, sizeof(eglFrame[0])) + * memset(&ad, 0, sizeof(ad)) # <<<<<<<<<<<<<< + * for i in range(cueglFrame.planeCount): + * ad.Depth = cueglFrame.depth +*/ + (void)(memset((&__pyx_v_ad), 0, (sizeof(__pyx_v_ad)))); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":827 + * memset(eglFrame, 0, sizeof(eglFrame[0])) + * memset(&ad, 0, sizeof(ad)) + * for i in range(cueglFrame.planeCount): # <<<<<<<<<<<<<< + * ad.Depth = cueglFrame.depth + * ad.Flags = 0 +*/ + __pyx_t_1 = __pyx_v_cueglFrame.planeCount; + __pyx_t_2 = __pyx_t_1; + for (__pyx_t_3 = 0; __pyx_t_3 < __pyx_t_2; __pyx_t_3+=1) { + __pyx_v_i = __pyx_t_3; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":828 + * memset(&ad, 0, sizeof(ad)) + * for i in range(cueglFrame.planeCount): + * ad.Depth = cueglFrame.depth # <<<<<<<<<<<<<< + * ad.Flags = 0 + * ad.Format = cueglFrame.cuFormat +*/ + __pyx_t_4 = __pyx_v_cueglFrame.depth; + __pyx_v_ad.Depth = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":829 + * for i in range(cueglFrame.planeCount): + * ad.Depth = cueglFrame.depth + * ad.Flags = 0 # <<<<<<<<<<<<<< + * ad.Format = cueglFrame.cuFormat + * ad.Height = cueglFrame.height +*/ + __pyx_v_ad.Flags = 0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":830 + * ad.Depth = cueglFrame.depth + * ad.Flags = 0 + * ad.Format = cueglFrame.cuFormat # <<<<<<<<<<<<<< + * ad.Height = cueglFrame.height + * ad.NumChannels = cueglFrame.numChannels +*/ + __pyx_t_5 = __pyx_v_cueglFrame.cuFormat; + __pyx_v_ad.Format = __pyx_t_5; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":831 + * ad.Flags = 0 + * ad.Format = cueglFrame.cuFormat + * ad.Height = cueglFrame.height # <<<<<<<<<<<<<< + * ad.NumChannels = cueglFrame.numChannels + * ad.Width = cueglFrame.width +*/ + __pyx_t_4 = __pyx_v_cueglFrame.height; + __pyx_v_ad.Height = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":832 + * ad.Format = cueglFrame.cuFormat + * ad.Height = cueglFrame.height + * ad.NumChannels = cueglFrame.numChannels # <<<<<<<<<<<<<< + * ad.Width = cueglFrame.width + * +*/ + __pyx_t_4 = __pyx_v_cueglFrame.numChannels; + __pyx_v_ad.NumChannels = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":833 + * ad.Height = cueglFrame.height + * ad.NumChannels = cueglFrame.numChannels + * ad.Width = cueglFrame.width # <<<<<<<<<<<<<< + * + * err = getChannelFormatDescFromDriverDesc(&eglFrame[0].planeDesc[i].channelDesc, NULL, NULL, NULL, &ad) +*/ + __pyx_t_4 = __pyx_v_cueglFrame.width; + __pyx_v_ad.Width = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":835 + * ad.Width = cueglFrame.width + * + * err = getChannelFormatDescFromDriverDesc(&eglFrame[0].planeDesc[i].channelDesc, NULL, NULL, NULL, &ad) # <<<<<<<<<<<<<< + * if err != cudaSuccess: + * return err +*/ + __pyx_t_6 = __pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getChannelFormatDescFromDriverDesc((&((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).channelDesc), NULL, NULL, NULL, (&__pyx_v_ad)); if (unlikely(__pyx_t_6 == ((cudaError_t)cudaErrorCallRequiresNewerDriver) && __Pyx_ErrOccurredWithGIL())) __PYX_ERR(1, 835, __pyx_L1_error) + __pyx_v_err = __pyx_t_6; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":836 + * + * err = getChannelFormatDescFromDriverDesc(&eglFrame[0].planeDesc[i].channelDesc, NULL, NULL, NULL, &ad) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * +*/ + __pyx_t_7 = (__pyx_v_err != cudaSuccess); + if (__pyx_t_7) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":837 + * err = getChannelFormatDescFromDriverDesc(&eglFrame[0].planeDesc[i].channelDesc, NULL, NULL, NULL, &ad) + * if err != cudaSuccess: + * return err # <<<<<<<<<<<<<< + * + * eglFrame[0].planeDesc[i].depth = cueglFrame.depth +*/ + __pyx_r = __pyx_v_err; + goto __pyx_L0; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":836 + * + * err = getChannelFormatDescFromDriverDesc(&eglFrame[0].planeDesc[i].channelDesc, NULL, NULL, NULL, &ad) + * if err != cudaSuccess: # <<<<<<<<<<<<<< + * return err + * +*/ + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":839 + * return err + * + * eglFrame[0].planeDesc[i].depth = cueglFrame.depth # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].numChannels = cueglFrame.numChannels + * if i == 0: +*/ + __pyx_t_4 = __pyx_v_cueglFrame.depth; + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).depth = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":840 + * + * eglFrame[0].planeDesc[i].depth = cueglFrame.depth + * eglFrame[0].planeDesc[i].numChannels = cueglFrame.numChannels # <<<<<<<<<<<<<< + * if i == 0: + * eglFrame[0].planeDesc[i].width = cueglFrame.width +*/ + __pyx_t_4 = __pyx_v_cueglFrame.numChannels; + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).numChannels = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":841 + * eglFrame[0].planeDesc[i].depth = cueglFrame.depth + * eglFrame[0].planeDesc[i].numChannels = cueglFrame.numChannels + * if i == 0: # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].width = cueglFrame.width + * eglFrame[0].planeDesc[i].height = cueglFrame.height +*/ + __pyx_t_7 = (__pyx_v_i == 0); + if (__pyx_t_7) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":842 + * eglFrame[0].planeDesc[i].numChannels = cueglFrame.numChannels + * if i == 0: + * eglFrame[0].planeDesc[i].width = cueglFrame.width # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch +*/ + __pyx_t_4 = __pyx_v_cueglFrame.width; + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).width = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":843 + * if i == 0: + * eglFrame[0].planeDesc[i].width = cueglFrame.width + * eglFrame[0].planeDesc[i].height = cueglFrame.height # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR or +*/ + __pyx_t_4 = __pyx_v_cueglFrame.height; + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).height = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":844 + * eglFrame[0].planeDesc[i].width = cueglFrame.width + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch # <<<<<<<<<<<<<< + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER or +*/ + __pyx_t_4 = __pyx_v_cueglFrame.pitch; + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).pitch = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":841 + * eglFrame[0].planeDesc[i].depth = cueglFrame.depth + * eglFrame[0].planeDesc[i].numChannels = cueglFrame.numChannels + * if i == 0: # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].width = cueglFrame.width + * eglFrame[0].planeDesc[i].height = cueglFrame.height +*/ + goto __pyx_L6; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":845 + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR or +*/ + switch (__pyx_v_cueglFrame.eglColorFormat) { + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR: + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":846 + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":847 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":848 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":849 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":850 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709): +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":851 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709): + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":845 + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR or +*/ + __pyx_t_7 = 1; + break; + default: + __pyx_t_7 = 0; + break; + } + if (__pyx_t_7) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":853 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709): + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].height = (cueglFrame.height / 2) + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) +*/ + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).width = ((unsigned int)(((double)__pyx_v_cueglFrame.width) / 2.0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":854 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709): + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + * eglFrame[0].planeDesc[i].height = (cueglFrame.height / 2) # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR or +*/ + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).height = ((unsigned int)(((double)__pyx_v_cueglFrame.height) / 2.0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":855 + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + * eglFrame[0].planeDesc[i].height = (cueglFrame.height / 2) + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) # <<<<<<<<<<<<<< + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER or +*/ + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).pitch = ((unsigned int)(((double)__pyx_v_cueglFrame.pitch) / 2.0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":845 + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR or +*/ + goto __pyx_L6; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":856 + * eglFrame[0].planeDesc[i].height = (cueglFrame.height / 2) + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR or +*/ + switch (__pyx_v_cueglFrame.eglColorFormat) { + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR: + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":857 + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":858 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":859 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":860 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":861 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":862 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":863 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":864 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":865 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":866 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":867 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":868 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":869 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER): +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":870 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":856 + * eglFrame[0].planeDesc[i].height = (cueglFrame.height / 2) + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR or +*/ + __pyx_t_7 = 1; + break; + default: + __pyx_t_7 = 0; + break; + } + if (__pyx_t_7) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":872 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].height = (cueglFrame.height / 2) + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) +*/ + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).width = ((unsigned int)(((double)__pyx_v_cueglFrame.width) / 2.0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":873 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + * eglFrame[0].planeDesc[i].height = (cueglFrame.height / 2) # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * eglFrame[0].planeDesc[1].channelDesc.y = 8 +*/ + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).height = ((unsigned int)(((double)__pyx_v_cueglFrame.height) / 2.0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":874 + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + * eglFrame[0].planeDesc[i].height = (cueglFrame.height / 2) + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[1].channelDesc.y = 8 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR or +*/ + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).pitch = ((unsigned int)(((double)__pyx_v_cueglFrame.pitch) / 2.0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":875 + * eglFrame[0].planeDesc[i].height = (cueglFrame.height / 2) + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * eglFrame[0].planeDesc[1].channelDesc.y = 8 # <<<<<<<<<<<<<< + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR or +*/ + ((__pyx_v_eglFrame[0]).planeDesc[1]).channelDesc.y = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":876 + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * eglFrame[0].planeDesc[1].channelDesc.y = 8 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 or +*/ + switch (__pyx_v_cueglFrame.eglColorFormat) { + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR: + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":877 + * eglFrame[0].planeDesc[1].channelDesc.y = 8 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":878 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":879 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":880 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":881 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER): +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":882 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":884 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 # <<<<<<<<<<<<<< + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER or +*/ + ((__pyx_v_eglFrame[0]).planeDesc[1]).channelDesc.y = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":876 + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * eglFrame[0].planeDesc[1].channelDesc.y = 8 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 or +*/ + break; + default: break; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":856 + * eglFrame[0].planeDesc[i].height = (cueglFrame.height / 2) + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR or +*/ + goto __pyx_L6; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":885 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR or +*/ + switch (__pyx_v_cueglFrame.eglColorFormat) { + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_PLANAR: + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":886 + * eglFrame[0].planeDesc[1].channelDesc.y = 16 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER): +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_PLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":887 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER): + * eglFrame[0].planeDesc[i].height = cueglFrame.height +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":885 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR or +*/ + __pyx_t_7 = 1; + break; + default: + __pyx_t_7 = 0; + break; + } + if (__pyx_t_7) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":889 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER): + * eglFrame[0].planeDesc[i].height = cueglFrame.height # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) +*/ + __pyx_t_4 = __pyx_v_cueglFrame.height; + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).height = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":890 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER): + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR or +*/ + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).width = ((unsigned int)(((double)__pyx_v_cueglFrame.width) / 2.0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":891 + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) # <<<<<<<<<<<<<< + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER or +*/ + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).pitch = ((unsigned int)(((double)__pyx_v_cueglFrame.pitch) / 2.0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":885 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR or +*/ + goto __pyx_L6; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":892 + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR or +*/ + switch (__pyx_v_cueglFrame.eglColorFormat) { + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR: + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":893 + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":894 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":895 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":896 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709): +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":897 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709): + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":892 + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR or +*/ + __pyx_t_7 = 1; + break; + default: + __pyx_t_7 = 0; + break; + } + if (__pyx_t_7) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":899 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709): + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) +*/ + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).width = ((unsigned int)(((double)__pyx_v_cueglFrame.width) / 2.0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":900 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709): + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + * eglFrame[0].planeDesc[i].height = cueglFrame.height # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * eglFrame[0].planeDesc[1].channelDesc.y = 8 +*/ + __pyx_t_4 = __pyx_v_cueglFrame.height; + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).height = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":901 + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[1].channelDesc.y = 8 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 or +*/ + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).pitch = ((unsigned int)(((double)__pyx_v_cueglFrame.pitch) / 2.0)); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":902 + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * eglFrame[0].planeDesc[1].channelDesc.y = 8 # <<<<<<<<<<<<<< + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR or +*/ + ((__pyx_v_eglFrame[0]).planeDesc[1]).channelDesc.y = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":903 + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * eglFrame[0].planeDesc[1].channelDesc.y = 8 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709): +*/ + switch (__pyx_v_cueglFrame.eglColorFormat) { + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020: + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":904 + * eglFrame[0].planeDesc[1].channelDesc.y = 8 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":906 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 # <<<<<<<<<<<<<< + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER or +*/ + ((__pyx_v_eglFrame[0]).planeDesc[1]).channelDesc.y = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":903 + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * eglFrame[0].planeDesc[1].channelDesc.y = 8 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709): +*/ + break; + default: break; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":892 + * eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + * eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR or +*/ + goto __pyx_L6; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":907 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR or +*/ + switch (__pyx_v_cueglFrame.eglColorFormat) { + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_PLANAR: + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":908 + * eglFrame[0].planeDesc[1].channelDesc.y = 16 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER): +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_PLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":909 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER): + * eglFrame[0].planeDesc[i].height = cueglFrame.height +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":907 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR or +*/ + __pyx_t_7 = 1; + break; + default: + __pyx_t_7 = 0; + break; + } + if (__pyx_t_7) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":911 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER): + * eglFrame[0].planeDesc[i].height = cueglFrame.height # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].width = cueglFrame.width + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch +*/ + __pyx_t_4 = __pyx_v_cueglFrame.height; + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).height = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":912 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER): + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].width = cueglFrame.width # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR or +*/ + __pyx_t_4 = __pyx_v_cueglFrame.width; + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).width = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":913 + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].width = cueglFrame.width + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch # <<<<<<<<<<<<<< + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER or +*/ + __pyx_t_4 = __pyx_v_cueglFrame.pitch; + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).pitch = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":907 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR or +*/ + goto __pyx_L6; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":914 + * eglFrame[0].planeDesc[i].width = cueglFrame.width + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR or +*/ + switch (__pyx_v_cueglFrame.eglColorFormat) { + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR: + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":915 + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":916 + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":917 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":918 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":919 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":920 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":921 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER): +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":922 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[i].height = cueglFrame.height +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":914 + * eglFrame[0].planeDesc[i].width = cueglFrame.width + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR or +*/ + __pyx_t_7 = 1; + break; + default: + __pyx_t_7 = 0; + break; + } + if (__pyx_t_7) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":924 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[i].height = cueglFrame.height # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].width = cueglFrame.width + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch +*/ + __pyx_t_4 = __pyx_v_cueglFrame.height; + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).height = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":925 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].width = cueglFrame.width # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * eglFrame[0].planeDesc[1].channelDesc.y = 8 +*/ + __pyx_t_4 = __pyx_v_cueglFrame.width; + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).width = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":926 + * eglFrame[0].planeDesc[i].height = cueglFrame.height + * eglFrame[0].planeDesc[i].width = cueglFrame.width + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[1].channelDesc.y = 8 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR or +*/ + __pyx_t_4 = __pyx_v_cueglFrame.pitch; + ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).pitch = __pyx_t_4; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":927 + * eglFrame[0].planeDesc[i].width = cueglFrame.width + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * eglFrame[0].planeDesc[1].channelDesc.y = 8 # <<<<<<<<<<<<<< + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR or +*/ + ((__pyx_v_eglFrame[0]).planeDesc[1]).channelDesc.y = 8; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":928 + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * eglFrame[0].planeDesc[1].channelDesc.y = 8 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER or +*/ + switch (__pyx_v_cueglFrame.eglColorFormat) { + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR: + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":929 + * eglFrame[0].planeDesc[1].channelDesc.y = 8 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":930 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER or +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":931 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER): +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":932 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 +*/ + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":934 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 # <<<<<<<<<<<<<< + * if cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY: + * eglFrame[0].frame.pArray[i] = cueglFrame.frame.pArray[i] +*/ + ((__pyx_v_eglFrame[0]).planeDesc[1]).channelDesc.y = 16; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":928 + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * eglFrame[0].planeDesc[1].channelDesc.y = 8 + * if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER or +*/ + break; + default: break; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":914 + * eglFrame[0].planeDesc[i].width = cueglFrame.width + * eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + * elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR or # <<<<<<<<<<<<<< + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER or + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR or +*/ + } + __pyx_L6:; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":935 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 + * if cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY: # <<<<<<<<<<<<<< + * eglFrame[0].frame.pArray[i] = cueglFrame.frame.pArray[i] + * else: +*/ + __pyx_t_7 = (__pyx_v_cueglFrame.frameType == __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_FRAME_TYPE_ARRAY); + if (__pyx_t_7) { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":936 + * eglFrame[0].planeDesc[1].channelDesc.y = 16 + * if cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY: + * eglFrame[0].frame.pArray[i] = cueglFrame.frame.pArray[i] # <<<<<<<<<<<<<< + * else: + * pPtr = make_cudaPitchedPtr(cueglFrame.frame.pPitch[i], eglFrame[0].planeDesc[i].pitch, +*/ + ((__pyx_v_eglFrame[0]).frame.pArray[__pyx_v_i]) = ((cudaArray_t)(__pyx_v_cueglFrame.frame.pArray[__pyx_v_i])); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":935 + * cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER): + * eglFrame[0].planeDesc[1].channelDesc.y = 16 + * if cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY: # <<<<<<<<<<<<<< + * eglFrame[0].frame.pArray[i] = cueglFrame.frame.pArray[i] + * else: +*/ + goto __pyx_L7; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":938 + * eglFrame[0].frame.pArray[i] = cueglFrame.frame.pArray[i] + * else: + * pPtr = make_cudaPitchedPtr(cueglFrame.frame.pPitch[i], eglFrame[0].planeDesc[i].pitch, # <<<<<<<<<<<<<< + * eglFrame[0].planeDesc[i].width, eglFrame[0].planeDesc[i].height) + * eglFrame[0].frame.pPitch[i] = pPtr +*/ + /*else*/ { + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":939 + * else: + * pPtr = make_cudaPitchedPtr(cueglFrame.frame.pPitch[i], eglFrame[0].planeDesc[i].pitch, + * eglFrame[0].planeDesc[i].width, eglFrame[0].planeDesc[i].height) # <<<<<<<<<<<<<< + * eglFrame[0].frame.pPitch[i] = pPtr + * +*/ + __pyx_v_pPtr = make_cudaPitchedPtr((__pyx_v_cueglFrame.frame.pPitch[__pyx_v_i]), ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).pitch, ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).width, ((__pyx_v_eglFrame[0]).planeDesc[__pyx_v_i]).height); + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":940 + * pPtr = make_cudaPitchedPtr(cueglFrame.frame.pPitch[i], eglFrame[0].planeDesc[i].pitch, + * eglFrame[0].planeDesc[i].width, eglFrame[0].planeDesc[i].height) + * eglFrame[0].frame.pPitch[i] = pPtr # <<<<<<<<<<<<<< + * + * eglFrame[0].planeCount = cueglFrame.planeCount +*/ + ((__pyx_v_eglFrame[0]).frame.pPitch[__pyx_v_i]) = __pyx_v_pPtr; + } + __pyx_L7:; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":942 + * eglFrame[0].frame.pPitch[i] = pPtr + * + * eglFrame[0].planeCount = cueglFrame.planeCount # <<<<<<<<<<<<<< + * if cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar +*/ + __pyx_t_1 = __pyx_v_cueglFrame.planeCount; + (__pyx_v_eglFrame[0]).planeCount = __pyx_t_1; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":943 + * + * eglFrame[0].planeCount = cueglFrame.planeCount + * if cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR: +*/ + switch (__pyx_v_cueglFrame.eglColorFormat) { + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":944 + * eglFrame[0].planeCount = cueglFrame.planeCount + * if cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420Planar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":943 + * + * eglFrame[0].planeCount = cueglFrame.planeCount + * if cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":946 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422Planar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420SemiPlanar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":945 + * if cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_PLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":948 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422Planar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422SemiPlanar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV422Planar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":947 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":950 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422SemiPlanar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444Planar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV422SemiPlanar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":949 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_PLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":952 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444Planar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444SemiPlanar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV444Planar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":951 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":954 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444SemiPlanar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_422: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUYV422 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV444SemiPlanar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":953 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_422: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUYV_422: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":956 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_422: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUYV422 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_422: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY422 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUYV422; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":955 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_422: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUYV422 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_422: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_422: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":958 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUYV422 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_422: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY422 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY709 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY422; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":957 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_422: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUYV422 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_422: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY422 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":960 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY422 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY709 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY709_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":959 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_422: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY422 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":962 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY709_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY2020 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":961 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_2020: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":964 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY2020 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ARGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatARGB +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":963 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_2020: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ARGB: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_ARGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":966 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ARGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatARGB # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RGBA: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatRGBA +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatARGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":965 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ARGB: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatARGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RGBA: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_RGBA: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":968 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatARGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RGBA: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatRGBA # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ABGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatABGR +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatRGBA; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":967 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ARGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatARGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RGBA: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatRGBA + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ABGR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_ABGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":970 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatRGBA + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ABGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatABGR # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BGRA: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBGRA +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatABGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":969 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RGBA: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatRGBA + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ABGR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatABGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BGRA: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BGRA: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":972 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatABGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BGRA: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBGRA # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_L: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatL +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBGRA; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":971 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ABGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatABGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BGRA: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBGRA + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_L: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_L: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":974 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBGRA + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_L: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatL # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_R: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatR +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatL; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":973 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BGRA: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBGRA + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_L: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatL + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_R: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_R: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":976 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatL + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_R: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatR # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_A: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatA +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":975 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_L: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatL + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_R: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_A: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_A: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":978 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_A: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatA # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatRG +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatA; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":977 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_R: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_A: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatA + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RG: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_RG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":980 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatA + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatRG # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatAYUV +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatRG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":979 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_A: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatA + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RG: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_AYUV: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":982 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatAYUV # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444SemiPlanar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatAYUV; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":981 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatAYUV + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":984 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatAYUV + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444SemiPlanar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422SemiPlanar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU444SemiPlanar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":983 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatAYUV + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":986 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422SemiPlanar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU422SemiPlanar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":985 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":988 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420SemiPlanar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":987 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":990 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_444SemiPlanar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":989 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":992 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":991 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":994 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_444SemiPlanar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":993 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":996 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatVYUY_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_420SemiPlanar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":995 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_VYUY_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":998 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatVYUY_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatVYUY_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":997 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatVYUY_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1000 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatVYUY_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUYV_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatUYVY_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":999 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatVYUY_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUYV_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1002 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUYV_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVYU_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUYV_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1001 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUYV_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVYU_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1004 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUYV_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVYU_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUVA_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVYU_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1003 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUYV_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVYU_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUVA_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1006 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVYU_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUVA_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatAYUV_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUVA_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1005 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVYU_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUVA_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_AYUV_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1008 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUVA_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatAYUV_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444Planar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatAYUV_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1007 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUVA_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatAYUV_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1010 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatAYUV_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444Planar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422Planar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV444Planar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1009 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatAYUV_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1012 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422Planar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV422Planar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1011 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1014 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444SemiPlanar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420Planar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1013 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1016 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444SemiPlanar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422SemiPlanar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV444SemiPlanar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1015 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1018 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422SemiPlanar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV422SemiPlanar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1017 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1020 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444Planar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420SemiPlanar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1019 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1022 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444Planar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422Planar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU444Planar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1021 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1024 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422Planar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU422Planar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1023 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1026 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444SemiPlanar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420Planar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1025 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1028 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444SemiPlanar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422SemiPlanar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU444SemiPlanar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1027 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1030 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422SemiPlanar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU422SemiPlanar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1029 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1032 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerRGGB +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420SemiPlanar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1031 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RGGB: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_RGGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1034 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerRGGB # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerBGGR +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerRGGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1033 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RGGB: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerRGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BGGR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_BGGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1036 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerRGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerBGGR # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerGRBG +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerBGGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1035 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerRGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BGGR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerBGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GRBG: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_GRBG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1038 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerBGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerGRBG # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerGBRG +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerGRBG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1037 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerBGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GRBG: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerGRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GBRG: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_GBRG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1040 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerGRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerGBRG # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10RGGB +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerGBRG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1039 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerGRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GBRG: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerGBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_RGGB: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_RGGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1042 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerGBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10RGGB # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10BGGR +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10RGGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1041 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerGBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_RGGB: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10RGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_BGGR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_BGGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1044 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10RGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10BGGR # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10GRBG +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10BGGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1043 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10RGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_BGGR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10BGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GRBG: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_GRBG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1046 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10BGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10GRBG # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10GBRG +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10GRBG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1045 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10BGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GRBG: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10GRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GBRG: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_GBRG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1048 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10GRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10GBRG # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12RGGB +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10GBRG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1047 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10GRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GBRG: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10GBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RGGB: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_RGGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1050 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10GBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12RGGB # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12BGGR +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12RGGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1049 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10GBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RGGB: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12RGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BGGR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_BGGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1052 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12RGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12BGGR # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12GRBG +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12BGGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1051 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12RGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BGGR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12BGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GRBG: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_GRBG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1054 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12BGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12GRBG # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12GBRG +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12GRBG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1053 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12BGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GRBG: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12GRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GBRG: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_GBRG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1056 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12GRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12GBRG # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14RGGB +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12GBRG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1055 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12GRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GBRG: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12GBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_RGGB: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_RGGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1058 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12GBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14RGGB # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14BGGR +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer14RGGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1057 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12GBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_RGGB: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14RGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_BGGR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_BGGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1060 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14RGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14BGGR # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14GRBG +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer14BGGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1059 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14RGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_BGGR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14BGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GRBG: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_GRBG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1062 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14BGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14GRBG # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14GBRG +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer14GRBG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1061 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14BGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GRBG: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14GRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GBRG: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_GBRG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1064 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14GRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14GBRG # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20RGGB +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer14GBRG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1063 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14GRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GBRG: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14GBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_RGGB: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_RGGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1066 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14GBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20RGGB # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20BGGR +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer20RGGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1065 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14GBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_RGGB: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20RGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_BGGR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_BGGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1068 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20RGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20BGGR # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20GRBG +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer20BGGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1067 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20RGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_BGGR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20BGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GRBG: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_GRBG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1070 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20BGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20GRBG # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20GBRG +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer20GRBG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1069 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20BGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GRBG: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20GRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GBRG: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_GBRG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1072 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20GRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20GBRG # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspRGGB +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer20GBRG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1071 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20GRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GBRG: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20GBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1074 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20GBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspRGGB # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspBGGR +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerIspRGGB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1073 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20GBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspRGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1076 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspRGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspBGGR # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspGRBG +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerIspBGGR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1075 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspRGGB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspBGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1078 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspBGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspGRBG # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspGBRG +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerIspGRBG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1077 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspBGGR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspGRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1080 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspGRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspGBRG # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444Planar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerIspGBRG; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1079 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspGRBG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspGBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_PLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1082 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspGBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444Planar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422Planar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU444Planar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1081 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspGBRG + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_PLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1084 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422Planar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU422Planar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1083 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1086 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BCCR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerBCCR +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420Planar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1085 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BCCR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_BCCR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1088 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BCCR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerBCCR # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RCCB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerRCCB +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerBCCR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1087 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BCCR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerBCCR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RCCB: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_RCCB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1090 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerBCCR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RCCB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerRCCB # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CRBC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerCRBC +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerRCCB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1089 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BCCR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerBCCR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RCCB: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerRCCB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CRBC: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_CRBC: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1092 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerRCCB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CRBC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerCRBC # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CBRC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerCBRC +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerCRBC; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1091 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RCCB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerRCCB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CRBC: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerCRBC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CBRC: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_CBRC: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1094 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerCRBC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CBRC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerCBRC # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_CCCC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10CCCC +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayerCBRC; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1093 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CRBC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerCRBC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CBRC: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerCBRC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_CCCC: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_CCCC: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1096 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerCBRC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_CCCC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10CCCC # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BCCR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12BCCR +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer10CCCC; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1095 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CBRC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerCBRC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_CCCC: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10CCCC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BCCR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_BCCR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1098 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10CCCC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BCCR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12BCCR # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RCCB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12RCCB +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12BCCR; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1097 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_CCCC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10CCCC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BCCR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12BCCR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RCCB: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_RCCB: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1100 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12BCCR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RCCB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12RCCB # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CRBC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CRBC +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12RCCB; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1099 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BCCR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12BCCR + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RCCB: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12RCCB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CRBC: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CRBC: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1102 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12RCCB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CRBC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CRBC # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CBRC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CBRC +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12CRBC; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1101 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RCCB: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12RCCB + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CRBC: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CRBC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CBRC: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CBRC: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1104 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CRBC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CBRC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CBRC # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CCCC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CCCC +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12CBRC; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1103 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CRBC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CRBC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CBRC: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CBRC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CCCC: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CCCC: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1106 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CBRC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CCCC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CCCC # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatBayer12CCCC; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1105 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CBRC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CBRC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CCCC: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CCCC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1108 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CCCC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_2020 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1107 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CCCC: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CCCC + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1110 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_2020 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_2020 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420SemiPlanar_2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1109 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1112 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_2020 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_2020 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420SemiPlanar_2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1111 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1114 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_2020 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_2020 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420Planar_2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1113 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1116 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_2020 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_709 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420Planar_2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1115 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1118 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_709 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_709 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420SemiPlanar_709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1117 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1120 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_709 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_709 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420SemiPlanar_709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1119 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1122 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_709 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_709 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUV420Planar_709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1121 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1124 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_709 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVU420Planar_709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1123 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1126 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_2020 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar_709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1125 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1128 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_2020 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_2020 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar_2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1127 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1130 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_2020 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_422SemiPlanar_2020; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1129 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1132 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_709 +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_422SemiPlanar; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1131 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_2020 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1134 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_709 # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_422SemiPlanar_709; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1133 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1136 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY_709_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1135 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_709 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1138 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY_709_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1137 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_709_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1140 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10_709_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1139 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1142 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10_709_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1141 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_709_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1144 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12_709_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1143 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1146 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12_709_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUVA +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1145 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_709_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUVA: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1148 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUVA # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVYU +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYUVA; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1147 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUVA + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVYU: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1150 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUVA + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVYU # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatVYUY +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatYVYU; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1149 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUVA + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVYU + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_VYUY: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1152 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVYU + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatVYUY # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatVYUY; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1151 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVYU + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatVYUY + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1154 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatVYUY + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1153 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatVYUY + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1156 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1155 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1158 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_444SemiPlanar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1157 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1160 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1159 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1162 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_420SemiPlanar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1161 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1164 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1163 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1166 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_ER # <<<<<<<<<<<<<< + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_709_ER +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_444SemiPlanar_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1165 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1168 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_709_ER # <<<<<<<<<<<<<< + * else: + * return cudaErrorInvalidValue +*/ + (__pyx_v_eglFrame[0]).eglColorFormat = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglColorFormatY12V12U12_444SemiPlanar_709_ER; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1167 + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER: + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_ER + * elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER: # <<<<<<<<<<<<<< + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_709_ER + * else: +*/ + break; + default: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1170 + * eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_709_ER + * else: + * return cudaErrorInvalidValue # <<<<<<<<<<<<<< + * if cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY: + * eglFrame[0].frameType = cyruntime.cudaEglFrameTypeArray +*/ + __pyx_r = cudaErrorInvalidValue; + goto __pyx_L0; + break; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1171 + * else: + * return cudaErrorInvalidValue + * if cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY: # <<<<<<<<<<<<<< + * eglFrame[0].frameType = cyruntime.cudaEglFrameTypeArray + * elif cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_PITCH: +*/ + switch (__pyx_v_cueglFrame.frameType) { + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_FRAME_TYPE_ARRAY: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1172 + * return cudaErrorInvalidValue + * if cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY: + * eglFrame[0].frameType = cyruntime.cudaEglFrameTypeArray # <<<<<<<<<<<<<< + * elif cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_PITCH: + * eglFrame[0].frameType = cyruntime.cudaEglFrameTypePitch +*/ + (__pyx_v_eglFrame[0]).frameType = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglFrameTypeArray; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1171 + * else: + * return cudaErrorInvalidValue + * if cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY: # <<<<<<<<<<<<<< + * eglFrame[0].frameType = cyruntime.cudaEglFrameTypeArray + * elif cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_PITCH: +*/ + break; + case __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_FRAME_TYPE_PITCH: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1174 + * eglFrame[0].frameType = cyruntime.cudaEglFrameTypeArray + * elif cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_PITCH: + * eglFrame[0].frameType = cyruntime.cudaEglFrameTypePitch # <<<<<<<<<<<<<< + * else: + * return cudaErrorInvalidValue +*/ + (__pyx_v_eglFrame[0]).frameType = __pyx_e_4cuda_8bindings_9cyruntime_cudaEglFrameTypePitch; + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1173 + * if cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY: + * eglFrame[0].frameType = cyruntime.cudaEglFrameTypeArray + * elif cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_PITCH: # <<<<<<<<<<<<<< + * eglFrame[0].frameType = cyruntime.cudaEglFrameTypePitch + * else: +*/ + break; + default: + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":1176 + * eglFrame[0].frameType = cyruntime.cudaEglFrameTypePitch + * else: + * return cudaErrorInvalidValue # <<<<<<<<<<<<<< +*/ + __pyx_r = cudaErrorInvalidValue; + goto __pyx_L0; + break; + } + + /* "cuda/bindings/_lib/cyruntime/cyruntime.pxi":819 + * return cudaErrorInvalidValue + * + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaError_t getRuntimeEglFrame(cyruntime.cudaEglFrame *eglFrame, cydriver.CUeglFrame cueglFrame) except ?cudaErrorCallRequiresNewerDriver nogil: + * cdef cudaError_t err = cudaSuccess +*/ + + /* function exit code */ + __pyx_r = (enum cudaError) 0; + goto __pyx_L0; + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._bindings.cyruntime.getRuntimeEglFrame", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = cudaErrorCallRequiresNewerDriver; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} +/* #### Code section: module_exttypes ### */ + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + if (__Pyx_ExportFunction("_cudaEGLStreamProducerPresentFrame", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamProducerPresentFrame, "cudaError_t (__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *, __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame, cudaStream_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEGLStreamProducerReturnFrame", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamProducerReturnFrame, "cudaError_t (__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *, __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame *, cudaStream_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsResourceGetMappedEglFrame", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsResourceGetMappedEglFrame, "cudaError_t (__pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame *, cudaGraphicsResource_t, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaVDPAUSetVDPAUDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaVDPAUSetVDPAUDevice, "cudaError_t (int, __pyx_t_4cuda_8bindings_9cyruntime_VdpDevice, __pyx_t_4cuda_8bindings_9cyruntime_VdpGetProcAddress *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaVDPAUGetDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaVDPAUGetDevice, "cudaError_t (int *, __pyx_t_4cuda_8bindings_9cyruntime_VdpDevice, __pyx_t_4cuda_8bindings_9cyruntime_VdpGetProcAddress *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsVDPAURegisterVideoSurface", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsVDPAURegisterVideoSurface, "cudaError_t (struct cudaGraphicsResource **, __pyx_t_4cuda_8bindings_9cyruntime_VdpVideoSurface, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsVDPAURegisterOutputSurface", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsVDPAURegisterOutputSurface, "cudaError_t (struct cudaGraphicsResource **, __pyx_t_4cuda_8bindings_9cyruntime_VdpOutputSurface, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGLGetDevices", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGLGetDevices, "cudaError_t (unsigned int *, int *, unsigned int, enum __pyx_t_4cuda_8bindings_9cyruntime_cudaGLDeviceList)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsGLRegisterImage", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsGLRegisterImage, "cudaError_t (struct cudaGraphicsResource **, __pyx_t_4cuda_8bindings_9cyruntime_GLuint, __pyx_t_4cuda_8bindings_9cyruntime_GLenum, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsGLRegisterBuffer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsGLRegisterBuffer, "cudaError_t (struct cudaGraphicsResource **, __pyx_t_4cuda_8bindings_9cyruntime_GLuint, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsEGLRegisterImage", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsEGLRegisterImage, "cudaError_t (cudaGraphicsResource_t *, __pyx_t_4cuda_8bindings_9cyruntime_EGLImageKHR, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEGLStreamConsumerConnect", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamConsumerConnect, "cudaError_t (__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *, __pyx_t_4cuda_8bindings_9cyruntime_EGLStreamKHR)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEGLStreamConsumerConnectWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamConsumerConnectWithFlags, "cudaError_t (__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *, __pyx_t_4cuda_8bindings_9cyruntime_EGLStreamKHR, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEGLStreamConsumerDisconnect", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamConsumerDisconnect, "cudaError_t (__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEGLStreamConsumerAcquireFrame", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamConsumerAcquireFrame, "cudaError_t (__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *, cudaGraphicsResource_t *, cudaStream_t *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEGLStreamConsumerReleaseFrame", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamConsumerReleaseFrame, "cudaError_t (__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *, cudaGraphicsResource_t, cudaStream_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEGLStreamProducerConnect", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamProducerConnect, "cudaError_t (__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *, __pyx_t_4cuda_8bindings_9cyruntime_EGLStreamKHR, __pyx_t_4cuda_8bindings_9cyruntime_EGLint, __pyx_t_4cuda_8bindings_9cyruntime_EGLint)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEGLStreamProducerDisconnect", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEGLStreamProducerDisconnect, "cudaError_t (__pyx_t_4cuda_8bindings_9cyruntime_cudaEglStreamConnection *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventCreateFromEGLSync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventCreateFromEGLSync, "cudaError_t (cudaEvent_t *, __pyx_t_4cuda_8bindings_9cyruntime_EGLSyncKHR, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("getDriverEglFrame", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getDriverEglFrame, "cudaError_t (__pyx_t_4cuda_8bindings_8cydriver_CUeglFrame *, __pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("getRuntimeEglFrame", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime_getRuntimeEglFrame, "cudaError_t (__pyx_t_4cuda_8bindings_9cyruntime_cudaEglFrame *, __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceReset", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceReset, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceSynchronize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceSynchronize, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceSetLimit", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceSetLimit, "cudaError_t (enum cudaLimit, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetLimit", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetLimit, "cudaError_t (size_t *, enum cudaLimit)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetTexture1DLinearMaxWidth", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetTexture1DLinearMaxWidth, "cudaError_t (size_t *, struct cudaChannelFormatDesc const *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetCacheConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetCacheConfig, "cudaError_t (enum cudaFuncCache *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetStreamPriorityRange", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetStreamPriorityRange, "cudaError_t (int *, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceSetCacheConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceSetCacheConfig, "cudaError_t (enum cudaFuncCache)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetByPCIBusId", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetByPCIBusId, "cudaError_t (int *, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetPCIBusId", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetPCIBusId, "cudaError_t (char *, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaIpcGetEventHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaIpcGetEventHandle, "cudaError_t (cudaIpcEventHandle_t *, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaIpcOpenEventHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaIpcOpenEventHandle, "cudaError_t (cudaEvent_t *, cudaIpcEventHandle_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaIpcGetMemHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaIpcGetMemHandle, "cudaError_t (cudaIpcMemHandle_t *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaIpcOpenMemHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaIpcOpenMemHandle, "cudaError_t (void **, cudaIpcMemHandle_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaIpcCloseMemHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaIpcCloseMemHandle, "cudaError_t (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceFlushGPUDirectRDMAWrites", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceFlushGPUDirectRDMAWrites, "cudaError_t (enum cudaFlushGPUDirectRDMAWritesTarget, enum cudaFlushGPUDirectRDMAWritesScope)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceRegisterAsyncNotification", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceRegisterAsyncNotification, "cudaError_t (int, cudaAsyncCallback, void *, cudaAsyncCallbackHandle_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceUnregisterAsyncNotification", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceUnregisterAsyncNotification, "cudaError_t (int, cudaAsyncCallbackHandle_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetSharedMemConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetSharedMemConfig, "cudaError_t (enum cudaSharedMemConfig *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceSetSharedMemConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceSetSharedMemConfig, "cudaError_t (enum cudaSharedMemConfig)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetLastError", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetLastError, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaPeekAtLastError", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaPeekAtLastError, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetErrorName", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetErrorName, "char const *(cudaError_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetErrorString", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetErrorString, "char const *(cudaError_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetDeviceCount", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetDeviceCount, "cudaError_t (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetDeviceProperties_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetDeviceProperties_v2, "cudaError_t (struct cudaDeviceProp *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetAttribute, "cudaError_t (int *, enum cudaDeviceAttr, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetDefaultMemPool", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetDefaultMemPool, "cudaError_t (cudaMemPool_t *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceSetMemPool", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceSetMemPool, "cudaError_t (int, cudaMemPool_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetMemPool", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetMemPool, "cudaError_t (cudaMemPool_t *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetNvSciSyncAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetNvSciSyncAttributes, "cudaError_t (void *, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetP2PAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetP2PAttribute, "cudaError_t (int *, enum cudaDeviceP2PAttr, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaChooseDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaChooseDevice, "cudaError_t (int *, struct cudaDeviceProp const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaInitDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaInitDevice, "cudaError_t (int, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaSetDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaSetDevice, "cudaError_t (int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetDevice, "cudaError_t (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaSetDeviceFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaSetDeviceFlags, "cudaError_t (unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetDeviceFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetDeviceFlags, "cudaError_t (unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamCreate, "cudaError_t (cudaStream_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamCreateWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamCreateWithFlags, "cudaError_t (cudaStream_t *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamCreateWithPriority", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamCreateWithPriority, "cudaError_t (cudaStream_t *, unsigned int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetPriority", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetPriority, "cudaError_t (cudaStream_t, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetFlags, "cudaError_t (cudaStream_t, unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetId", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetId, "cudaError_t (cudaStream_t, unsigned PY_LONG_LONG *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetDevice, "cudaError_t (cudaStream_t, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaCtxResetPersistingL2Cache", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaCtxResetPersistingL2Cache, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamCopyAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamCopyAttributes, "cudaError_t (cudaStream_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetAttribute, "cudaError_t (cudaStream_t, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaStreamAttrID, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaStreamAttrValue *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamSetAttribute, "cudaError_t (cudaStream_t, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaStreamAttrID, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaStreamAttrValue const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamDestroy, "cudaError_t (cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamWaitEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamWaitEvent, "cudaError_t (cudaStream_t, cudaEvent_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamAddCallback", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamAddCallback, "cudaError_t (cudaStream_t, cudaStreamCallback_t, void *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamSynchronize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamSynchronize, "cudaError_t (cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamQuery", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamQuery, "cudaError_t (cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamAttachMemAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamAttachMemAsync, "cudaError_t (cudaStream_t, void *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamBeginCapture", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamBeginCapture, "cudaError_t (cudaStream_t, enum cudaStreamCaptureMode)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamBeginCaptureToGraph", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamBeginCaptureToGraph, "cudaError_t (cudaStream_t, cudaGraph_t, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t, enum cudaStreamCaptureMode)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaThreadExchangeStreamCaptureMode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaThreadExchangeStreamCaptureMode, "cudaError_t (enum cudaStreamCaptureMode *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamEndCapture", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamEndCapture, "cudaError_t (cudaStream_t, cudaGraph_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamIsCapturing", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamIsCapturing, "cudaError_t (cudaStream_t, enum cudaStreamCaptureStatus *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetCaptureInfo_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetCaptureInfo_v2, "cudaError_t (cudaStream_t, enum cudaStreamCaptureStatus *, unsigned PY_LONG_LONG *, cudaGraph_t *, cudaGraphNode_t const **, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetCaptureInfo_v3", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamGetCaptureInfo_v3, "cudaError_t (cudaStream_t, enum cudaStreamCaptureStatus *, unsigned PY_LONG_LONG *, cudaGraph_t *, cudaGraphNode_t const **, cudaGraphEdgeData const **, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamUpdateCaptureDependencies", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamUpdateCaptureDependencies, "cudaError_t (cudaStream_t, cudaGraphNode_t *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamUpdateCaptureDependencies_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaStreamUpdateCaptureDependencies_v2, "cudaError_t (cudaStream_t, cudaGraphNode_t *, cudaGraphEdgeData const *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventCreate, "cudaError_t (cudaEvent_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventCreateWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventCreateWithFlags, "cudaError_t (cudaEvent_t *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventRecord", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventRecord, "cudaError_t (cudaEvent_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventRecordWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventRecordWithFlags, "cudaError_t (cudaEvent_t, cudaStream_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventQuery", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventQuery, "cudaError_t (cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventSynchronize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventSynchronize, "cudaError_t (cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventDestroy, "cudaError_t (cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventElapsedTime", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventElapsedTime, "cudaError_t (float *, cudaEvent_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventElapsedTime_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaEventElapsedTime_v2, "cudaError_t (float *, cudaEvent_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaImportExternalMemory", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaImportExternalMemory, "cudaError_t (cudaExternalMemory_t *, struct cudaExternalMemoryHandleDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaExternalMemoryGetMappedBuffer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaExternalMemoryGetMappedBuffer, "cudaError_t (void **, cudaExternalMemory_t, struct cudaExternalMemoryBufferDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaExternalMemoryGetMappedMipmappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaExternalMemoryGetMappedMipmappedArray, "cudaError_t (cudaMipmappedArray_t *, cudaExternalMemory_t, struct cudaExternalMemoryMipmappedArrayDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDestroyExternalMemory", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDestroyExternalMemory, "cudaError_t (cudaExternalMemory_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaImportExternalSemaphore", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaImportExternalSemaphore, "cudaError_t (cudaExternalSemaphore_t *, struct cudaExternalSemaphoreHandleDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaSignalExternalSemaphoresAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaSignalExternalSemaphoresAsync_v2, "cudaError_t (cudaExternalSemaphore_t const *, struct cudaExternalSemaphoreSignalParams const *, unsigned int, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaWaitExternalSemaphoresAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaWaitExternalSemaphoresAsync_v2, "cudaError_t (cudaExternalSemaphore_t const *, struct cudaExternalSemaphoreWaitParams const *, unsigned int, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDestroyExternalSemaphore", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDestroyExternalSemaphore, "cudaError_t (cudaExternalSemaphore_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFuncSetCacheConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFuncSetCacheConfig, "cudaError_t (void const *, enum cudaFuncCache)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFuncGetAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFuncGetAttributes, "cudaError_t (struct cudaFuncAttributes *, void const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFuncSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFuncSetAttribute, "cudaError_t (void const *, enum cudaFuncAttribute, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLaunchHostFunc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLaunchHostFunc, "cudaError_t (cudaStream_t, cudaHostFn_t, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFuncSetSharedMemConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFuncSetSharedMemConfig, "cudaError_t (void const *, enum cudaSharedMemConfig)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaOccupancyMaxActiveBlocksPerMultiprocessor", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaOccupancyMaxActiveBlocksPerMultiprocessor, "cudaError_t (int *, void const *, int, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaOccupancyAvailableDynamicSMemPerBlock", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaOccupancyAvailableDynamicSMemPerBlock, "cudaError_t (size_t *, void const *, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, "cudaError_t (int *, void const *, int, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocManaged", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocManaged, "cudaError_t (void **, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMalloc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMalloc, "cudaError_t (void **, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocHost", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocHost, "cudaError_t (void **, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocPitch", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocPitch, "cudaError_t (void **, size_t *, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocArray, "cudaError_t (cudaArray_t *, struct cudaChannelFormatDesc const *, size_t, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFree", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFree, "cudaError_t (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFreeHost", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFreeHost, "cudaError_t (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFreeArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFreeArray, "cudaError_t (cudaArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFreeMipmappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFreeMipmappedArray, "cudaError_t (cudaMipmappedArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaHostAlloc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaHostAlloc, "cudaError_t (void **, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaHostRegister", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaHostRegister, "cudaError_t (void *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaHostUnregister", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaHostUnregister, "cudaError_t (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaHostGetDevicePointer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaHostGetDevicePointer, "cudaError_t (void **, void *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaHostGetFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaHostGetFlags, "cudaError_t (unsigned int *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMalloc3D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMalloc3D, "cudaError_t (struct cudaPitchedPtr *, struct cudaExtent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMalloc3DArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMalloc3DArray, "cudaError_t (cudaArray_t *, struct cudaChannelFormatDesc const *, struct cudaExtent, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocMipmappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocMipmappedArray, "cudaError_t (cudaMipmappedArray_t *, struct cudaChannelFormatDesc const *, struct cudaExtent, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetMipmappedArrayLevel", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetMipmappedArrayLevel, "cudaError_t (cudaArray_t *, cudaMipmappedArray_const_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy3D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy3D, "cudaError_t (struct cudaMemcpy3DParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy3DPeer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy3DPeer, "cudaError_t (struct cudaMemcpy3DPeerParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy3DAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy3DAsync, "cudaError_t (struct cudaMemcpy3DParms const *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy3DPeerAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy3DPeerAsync, "cudaError_t (struct cudaMemcpy3DPeerParms const *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemGetInfo", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemGetInfo, "cudaError_t (size_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaArrayGetInfo", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaArrayGetInfo, "cudaError_t (struct cudaChannelFormatDesc *, struct cudaExtent *, unsigned int *, cudaArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaArrayGetPlane", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaArrayGetPlane, "cudaError_t (cudaArray_t *, cudaArray_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaArrayGetMemoryRequirements", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaArrayGetMemoryRequirements, "cudaError_t (struct cudaArrayMemoryRequirements *, cudaArray_t, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMipmappedArrayGetMemoryRequirements", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMipmappedArrayGetMemoryRequirements, "cudaError_t (struct cudaArrayMemoryRequirements *, cudaMipmappedArray_t, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaArrayGetSparseProperties", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaArrayGetSparseProperties, "cudaError_t (struct cudaArraySparseProperties *, cudaArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMipmappedArrayGetSparseProperties", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMipmappedArrayGetSparseProperties, "cudaError_t (struct cudaArraySparseProperties *, cudaMipmappedArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy, "cudaError_t (void *, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyPeer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyPeer, "cudaError_t (void *, int, void const *, int, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2D, "cudaError_t (void *, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2DToArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2DToArray, "cudaError_t (cudaArray_t, size_t, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2DFromArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2DFromArray, "cudaError_t (void *, size_t, cudaArray_const_t, size_t, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2DArrayToArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2DArrayToArray, "cudaError_t (cudaArray_t, size_t, size_t, cudaArray_const_t, size_t, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyAsync, "cudaError_t (void *, void const *, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyPeerAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyPeerAsync, "cudaError_t (void *, int, void const *, int, size_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyBatchAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyBatchAsync, "cudaError_t (void **, void **, size_t *, size_t, struct cudaMemcpyAttributes *, size_t *, size_t, size_t *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy3DBatchAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy3DBatchAsync, "cudaError_t (size_t, struct cudaMemcpy3DBatchOp *, size_t *, unsigned PY_LONG_LONG, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2DAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2DAsync, "cudaError_t (void *, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2DToArrayAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2DToArrayAsync, "cudaError_t (cudaArray_t, size_t, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2DFromArrayAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpy2DFromArrayAsync, "cudaError_t (void *, size_t, cudaArray_const_t, size_t, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemset", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemset, "cudaError_t (void *, int, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemset2D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemset2D, "cudaError_t (void *, size_t, int, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemset3D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemset3D, "cudaError_t (struct cudaPitchedPtr, int, struct cudaExtent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemsetAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemsetAsync, "cudaError_t (void *, int, size_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemset2DAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemset2DAsync, "cudaError_t (void *, size_t, int, size_t, size_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemset3DAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemset3DAsync, "cudaError_t (struct cudaPitchedPtr, int, struct cudaExtent, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPrefetchAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPrefetchAsync, "cudaError_t (void const *, size_t, int, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPrefetchAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPrefetchAsync_v2, "cudaError_t (void const *, size_t, struct cudaMemLocation, unsigned int, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemAdvise", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemAdvise, "cudaError_t (void const *, size_t, enum cudaMemoryAdvise, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemAdvise_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemAdvise_v2, "cudaError_t (void const *, size_t, enum cudaMemoryAdvise, struct cudaMemLocation)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemRangeGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemRangeGetAttribute, "cudaError_t (void *, size_t, enum cudaMemRangeAttribute, void const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemRangeGetAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemRangeGetAttributes, "cudaError_t (void **, size_t *, enum cudaMemRangeAttribute *, size_t, void const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyToArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyToArray, "cudaError_t (cudaArray_t, size_t, size_t, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyFromArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyFromArray, "cudaError_t (void *, cudaArray_const_t, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyArrayToArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyArrayToArray, "cudaError_t (cudaArray_t, size_t, size_t, cudaArray_const_t, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyToArrayAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyToArrayAsync, "cudaError_t (cudaArray_t, size_t, size_t, void const *, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyFromArrayAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemcpyFromArrayAsync, "cudaError_t (void *, cudaArray_const_t, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocAsync, "cudaError_t (void **, size_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFreeAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaFreeAsync, "cudaError_t (void *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolTrimTo", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolTrimTo, "cudaError_t (cudaMemPool_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolSetAttribute, "cudaError_t (cudaMemPool_t, enum cudaMemPoolAttr, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolGetAttribute, "cudaError_t (cudaMemPool_t, enum cudaMemPoolAttr, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolSetAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolSetAccess, "cudaError_t (cudaMemPool_t, struct cudaMemAccessDesc const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolGetAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolGetAccess, "cudaError_t (enum cudaMemAccessFlags *, cudaMemPool_t, struct cudaMemLocation *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolCreate, "cudaError_t (cudaMemPool_t *, struct cudaMemPoolProps const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolDestroy, "cudaError_t (cudaMemPool_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocFromPoolAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMallocFromPoolAsync, "cudaError_t (void **, size_t, cudaMemPool_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolExportToShareableHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolExportToShareableHandle, "cudaError_t (void *, cudaMemPool_t, enum cudaMemAllocationHandleType, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolImportFromShareableHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolImportFromShareableHandle, "cudaError_t (cudaMemPool_t *, void *, enum cudaMemAllocationHandleType, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolExportPointer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolExportPointer, "cudaError_t (struct cudaMemPoolPtrExportData *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolImportPointer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaMemPoolImportPointer, "cudaError_t (void **, cudaMemPool_t, struct cudaMemPoolPtrExportData *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaPointerGetAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaPointerGetAttributes, "cudaError_t (struct cudaPointerAttributes *, void const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceCanAccessPeer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceCanAccessPeer, "cudaError_t (int *, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceEnablePeerAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceEnablePeerAccess, "cudaError_t (int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceDisablePeerAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceDisablePeerAccess, "cudaError_t (int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsUnregisterResource", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsUnregisterResource, "cudaError_t (cudaGraphicsResource_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsResourceSetMapFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsResourceSetMapFlags, "cudaError_t (cudaGraphicsResource_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsMapResources", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsMapResources, "cudaError_t (int, cudaGraphicsResource_t *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsUnmapResources", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsUnmapResources, "cudaError_t (int, cudaGraphicsResource_t *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsResourceGetMappedPointer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsResourceGetMappedPointer, "cudaError_t (void **, size_t *, cudaGraphicsResource_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsSubResourceGetMappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsSubResourceGetMappedArray, "cudaError_t (cudaArray_t *, cudaGraphicsResource_t, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsResourceGetMappedMipmappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphicsResourceGetMappedMipmappedArray, "cudaError_t (cudaMipmappedArray_t *, cudaGraphicsResource_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetChannelDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetChannelDesc, "cudaError_t (struct cudaChannelFormatDesc *, cudaArray_const_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaCreateChannelDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaCreateChannelDesc, "struct cudaChannelFormatDesc (int, int, int, int, enum cudaChannelFormatKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaCreateTextureObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaCreateTextureObject, "cudaError_t (cudaTextureObject_t *, struct cudaResourceDesc const *, struct cudaTextureDesc const *, struct cudaResourceViewDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDestroyTextureObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDestroyTextureObject, "cudaError_t (cudaTextureObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetTextureObjectResourceDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetTextureObjectResourceDesc, "cudaError_t (struct cudaResourceDesc *, cudaTextureObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetTextureObjectTextureDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetTextureObjectTextureDesc, "cudaError_t (struct cudaTextureDesc *, cudaTextureObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetTextureObjectResourceViewDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetTextureObjectResourceViewDesc, "cudaError_t (struct cudaResourceViewDesc *, cudaTextureObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaCreateSurfaceObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaCreateSurfaceObject, "cudaError_t (cudaSurfaceObject_t *, struct cudaResourceDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDestroySurfaceObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDestroySurfaceObject, "cudaError_t (cudaSurfaceObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetSurfaceObjectResourceDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetSurfaceObjectResourceDesc, "cudaError_t (struct cudaResourceDesc *, cudaSurfaceObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDriverGetVersion", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDriverGetVersion, "cudaError_t (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaRuntimeGetVersion", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaRuntimeGetVersion, "cudaError_t (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphCreate, "cudaError_t (cudaGraph_t *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddKernelNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddKernelNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaKernelNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphKernelNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphKernelNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaKernelNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphKernelNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphKernelNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaKernelNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphKernelNodeCopyAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphKernelNodeCopyAttributes, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphKernelNodeGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphKernelNodeGetAttribute, "cudaError_t (cudaGraphNode_t, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaKernelNodeAttrID, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaKernelNodeAttrValue *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphKernelNodeSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphKernelNodeSetAttribute, "cudaError_t (cudaGraphNode_t, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaKernelNodeAttrID, __pyx_t_4cuda_8bindings_9_bindings_9cyruntime_cudaKernelNodeAttrValue const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddMemcpyNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddMemcpyNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaMemcpy3DParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddMemcpyNode1D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddMemcpyNode1D, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, void *, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemcpyNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemcpyNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemcpy3DParms *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemcpyNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemcpyNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemcpy3DParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemcpyNodeSetParams1D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemcpyNodeSetParams1D, "cudaError_t (cudaGraphNode_t, void *, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddMemsetNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddMemsetNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaMemsetParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemsetNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemsetNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemsetParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemsetNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemsetNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemsetParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddHostNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddHostNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaHostNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphHostNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphHostNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaHostNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphHostNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphHostNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaHostNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddChildGraphNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddChildGraphNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphChildGraphNodeGetGraph", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphChildGraphNodeGetGraph, "cudaError_t (cudaGraphNode_t, cudaGraph_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddEmptyNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddEmptyNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddEventRecordNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddEventRecordNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphEventRecordNodeGetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphEventRecordNodeGetEvent, "cudaError_t (cudaGraphNode_t, cudaEvent_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphEventRecordNodeSetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphEventRecordNodeSetEvent, "cudaError_t (cudaGraphNode_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddEventWaitNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddEventWaitNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphEventWaitNodeGetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphEventWaitNodeGetEvent, "cudaError_t (cudaGraphNode_t, cudaEvent_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphEventWaitNodeSetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphEventWaitNodeSetEvent, "cudaError_t (cudaGraphNode_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddExternalSemaphoresSignalNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddExternalSemaphoresSignalNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaExternalSemaphoreSignalNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExternalSemaphoresSignalNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExternalSemaphoresSignalNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaExternalSemaphoreSignalNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExternalSemaphoresSignalNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExternalSemaphoresSignalNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaExternalSemaphoreSignalNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddExternalSemaphoresWaitNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddExternalSemaphoresWaitNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaExternalSemaphoreWaitNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExternalSemaphoresWaitNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExternalSemaphoresWaitNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaExternalSemaphoreWaitNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExternalSemaphoresWaitNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExternalSemaphoresWaitNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaExternalSemaphoreWaitNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddMemAllocNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddMemAllocNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaMemAllocNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemAllocNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemAllocNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemAllocNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddMemFreeNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddMemFreeNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemFreeNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphMemFreeNodeGetParams, "cudaError_t (cudaGraphNode_t, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGraphMemTrim", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGraphMemTrim, "cudaError_t (int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetGraphMemAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceGetGraphMemAttribute, "cudaError_t (int, enum cudaGraphMemAttributeType, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceSetGraphMemAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaDeviceSetGraphMemAttribute, "cudaError_t (int, enum cudaGraphMemAttributeType, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphClone", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphClone, "cudaError_t (cudaGraph_t *, cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeFindInClone", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeFindInClone, "cudaError_t (cudaGraphNode_t *, cudaGraphNode_t, cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeGetType", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeGetType, "cudaError_t (cudaGraphNode_t, enum cudaGraphNodeType *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphGetNodes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphGetNodes, "cudaError_t (cudaGraph_t, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphGetRootNodes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphGetRootNodes, "cudaError_t (cudaGraph_t, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphGetEdges", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphGetEdges, "cudaError_t (cudaGraph_t, cudaGraphNode_t *, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphGetEdges_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphGetEdges_v2, "cudaError_t (cudaGraph_t, cudaGraphNode_t *, cudaGraphNode_t *, cudaGraphEdgeData *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeGetDependencies", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeGetDependencies, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeGetDependencies_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeGetDependencies_v2, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t *, cudaGraphEdgeData *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeGetDependentNodes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeGetDependentNodes, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeGetDependentNodes_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeGetDependentNodes_v2, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t *, cudaGraphEdgeData *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddDependencies", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddDependencies, "cudaError_t (cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddDependencies_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddDependencies_v2, "cudaError_t (cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphRemoveDependencies", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphRemoveDependencies, "cudaError_t (cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphRemoveDependencies_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphRemoveDependencies_v2, "cudaError_t (cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphDestroyNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphDestroyNode, "cudaError_t (cudaGraphNode_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphInstantiate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphInstantiate, "cudaError_t (cudaGraphExec_t *, cudaGraph_t, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphInstantiateWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphInstantiateWithFlags, "cudaError_t (cudaGraphExec_t *, cudaGraph_t, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphInstantiateWithParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphInstantiateWithParams, "cudaError_t (cudaGraphExec_t *, cudaGraph_t, cudaGraphInstantiateParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecGetFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecGetFlags, "cudaError_t (cudaGraphExec_t, unsigned PY_LONG_LONG *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecKernelNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecKernelNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaKernelNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecMemcpyNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecMemcpyNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaMemcpy3DParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecMemcpyNodeSetParams1D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecMemcpyNodeSetParams1D, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, void *, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecMemsetNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecMemsetNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaMemsetParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecHostNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecHostNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaHostNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecChildGraphNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecChildGraphNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecEventRecordNodeSetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecEventRecordNodeSetEvent, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecEventWaitNodeSetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecEventWaitNodeSetEvent, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecExternalSemaphoresSignalNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecExternalSemaphoresSignalNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaExternalSemaphoreSignalNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecExternalSemaphoresWaitNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecExternalSemaphoresWaitNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaExternalSemaphoreWaitNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeSetEnabled", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeSetEnabled, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeGetEnabled", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeGetEnabled, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecUpdate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecUpdate, "cudaError_t (cudaGraphExec_t, cudaGraph_t, cudaGraphExecUpdateResultInfo *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphUpload", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphUpload, "cudaError_t (cudaGraphExec_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphLaunch", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphLaunch, "cudaError_t (cudaGraphExec_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecDestroy, "cudaError_t (cudaGraphExec_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphDestroy, "cudaError_t (cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphDebugDotPrint", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphDebugDotPrint, "cudaError_t (cudaGraph_t, char const *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaUserObjectCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaUserObjectCreate, "cudaError_t (cudaUserObject_t *, void *, cudaHostFn_t, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaUserObjectRetain", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaUserObjectRetain, "cudaError_t (cudaUserObject_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaUserObjectRelease", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaUserObjectRelease, "cudaError_t (cudaUserObject_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphRetainUserObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphRetainUserObject, "cudaError_t (cudaGraph_t, cudaUserObject_t, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphReleaseUserObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphReleaseUserObject, "cudaError_t (cudaGraph_t, cudaUserObject_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaGraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddNode_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphAddNode_v2, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t, struct cudaGraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaGraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphExecNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaGraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphConditionalHandleCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGraphConditionalHandleCreate, "cudaError_t (cudaGraphConditionalHandle *, cudaGraph_t, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetDriverEntryPoint", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetDriverEntryPoint, "cudaError_t (char const *, void **, unsigned PY_LONG_LONG, enum cudaDriverEntryPointQueryResult *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetDriverEntryPointByVersion", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetDriverEntryPointByVersion, "cudaError_t (char const *, void **, unsigned int, unsigned PY_LONG_LONG, enum cudaDriverEntryPointQueryResult *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryLoadData", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryLoadData, "cudaError_t (cudaLibrary_t *, void const *, enum cudaJitOption *, void **, unsigned int, enum cudaLibraryOption *, void **, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryLoadFromFile", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryLoadFromFile, "cudaError_t (cudaLibrary_t *, char const *, enum cudaJitOption *, void **, unsigned int, enum cudaLibraryOption *, void **, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryUnload", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryUnload, "cudaError_t (cudaLibrary_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryGetKernel", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryGetKernel, "cudaError_t (cudaKernel_t *, cudaLibrary_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryGetGlobal", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryGetGlobal, "cudaError_t (void **, size_t *, cudaLibrary_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryGetManaged", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryGetManaged, "cudaError_t (void **, size_t *, cudaLibrary_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryGetUnifiedFunction", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryGetUnifiedFunction, "cudaError_t (void **, cudaLibrary_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryGetKernelCount", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryGetKernelCount, "cudaError_t (unsigned int *, cudaLibrary_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryEnumerateKernels", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaLibraryEnumerateKernels, "cudaError_t (cudaKernel_t *, unsigned int, cudaLibrary_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaKernelSetAttributeForDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaKernelSetAttributeForDevice, "cudaError_t (cudaKernel_t, enum cudaFuncAttribute, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetExportTable", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetExportTable, "cudaError_t (void const **, cudaUUID_t const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetKernel", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaGetKernel, "cudaError_t (cudaKernel_t *, void const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_make_cudaPitchedPtr", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__make_cudaPitchedPtr, "struct cudaPitchedPtr (void *, size_t, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_make_cudaPos", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__make_cudaPos, "struct cudaPos (size_t, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_make_cudaExtent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__make_cudaExtent, "struct cudaExtent (size_t, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaProfilerStart", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaProfilerStart, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaProfilerStop", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_9cyruntime__cudaProfilerStop, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __pyx_t_1 = PyImport_ImportModule("cuda.bindings._bindings.cydriver"); if (!__pyx_t_1) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuGraphicsEGLRegisterImage", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsEGLRegisterImage, "CUresult (CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_EGLImageKHR, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuEGLStreamConsumerConnect", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerConnect, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuEGLStreamConsumerConnectWithFlags", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerConnectWithFlags, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuEGLStreamConsumerDisconnect", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerDisconnect, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuEGLStreamConsumerAcquireFrame", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerAcquireFrame, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, CUgraphicsResource *, CUstream *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuEGLStreamConsumerReleaseFrame", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamConsumerReleaseFrame, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, CUgraphicsResource, CUstream *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuEGLStreamProducerConnect", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerConnect, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR, __pyx_t_4cuda_8bindings_8cydriver_EGLint, __pyx_t_4cuda_8bindings_8cydriver_EGLint)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuEGLStreamProducerDisconnect", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerDisconnect, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuEGLStreamProducerPresentFrame", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerPresentFrame, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame, CUstream *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuEGLStreamProducerReturnFrame", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEGLStreamProducerReturnFrame, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection *, __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame *, CUstream *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuGraphicsResourceGetMappedEglFrame", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsResourceGetMappedEglFrame, "CUresult (__pyx_t_4cuda_8bindings_8cydriver_CUeglFrame *, CUgraphicsResource, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuEventCreateFromEGLSync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuEventCreateFromEGLSync, "CUresult (CUevent *, __pyx_t_4cuda_8bindings_8cydriver_EGLSyncKHR, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuGraphicsGLRegisterBuffer", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsGLRegisterBuffer, "CUresult (CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_GLuint, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuGraphicsGLRegisterImage", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsGLRegisterImage, "CUresult (CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_GLuint, __pyx_t_4cuda_8bindings_8cydriver_GLenum, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuGLGetDevices_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGLGetDevices_v2, "CUresult (unsigned int *, CUdevice *, unsigned int, __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuVDPAUGetDevice", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuVDPAUGetDevice, "CUresult (CUdevice *, __pyx_t_4cuda_8bindings_8cydriver_VdpDevice, __pyx_t_4cuda_8bindings_8cydriver_VdpGetProcAddress *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuGraphicsVDPAURegisterVideoSurface", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsVDPAURegisterVideoSurface, "CUresult (CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_VdpVideoSurface, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cuGraphicsVDPAURegisterOutputSurface", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_8cydriver__cuGraphicsVDPAURegisterOutputSurface, "CUresult (CUgraphicsResource *, __pyx_t_4cuda_8bindings_8cydriver_VdpOutputSurface, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyImport_ImportModule("cuda.bindings._bindings.cyruntime_ptds"); if (!__pyx_t_1) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceReset", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceReset, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceSynchronize", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSynchronize, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceSetLimit", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetLimit, "cudaError_t (enum cudaLimit, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGetLimit", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetLimit, "cudaError_t (size_t *, enum cudaLimit)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGetTexture1DLinearMaxWidth", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetTexture1DLinearMaxWidth, "cudaError_t (size_t *, struct cudaChannelFormatDesc const *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGetCacheConfig", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetCacheConfig, "cudaError_t (enum cudaFuncCache *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGetStreamPriorityRange", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetStreamPriorityRange, "cudaError_t (int *, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceSetCacheConfig", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetCacheConfig, "cudaError_t (enum cudaFuncCache)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGetByPCIBusId", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetByPCIBusId, "cudaError_t (int *, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGetPCIBusId", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetPCIBusId, "cudaError_t (char *, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaIpcGetEventHandle", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcGetEventHandle, "cudaError_t (cudaIpcEventHandle_t *, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaIpcOpenEventHandle", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcOpenEventHandle, "cudaError_t (cudaEvent_t *, cudaIpcEventHandle_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaIpcGetMemHandle", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcGetMemHandle, "cudaError_t (cudaIpcMemHandle_t *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaIpcOpenMemHandle", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcOpenMemHandle, "cudaError_t (void **, cudaIpcMemHandle_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaIpcCloseMemHandle", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcCloseMemHandle, "cudaError_t (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceFlushGPUDirectRDMAWrites", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceFlushGPUDirectRDMAWrites, "cudaError_t (enum cudaFlushGPUDirectRDMAWritesTarget, enum cudaFlushGPUDirectRDMAWritesScope)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceRegisterAsyncNotification", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceRegisterAsyncNotification, "cudaError_t (int, cudaAsyncCallback, void *, cudaAsyncCallbackHandle_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceUnregisterAsyncNotification", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceUnregisterAsyncNotification, "cudaError_t (int, cudaAsyncCallbackHandle_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGetSharedMemConfig", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetSharedMemConfig, "cudaError_t (enum cudaSharedMemConfig *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceSetSharedMemConfig", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetSharedMemConfig, "cudaError_t (enum cudaSharedMemConfig)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetLastError", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetLastError, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaPeekAtLastError", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaPeekAtLastError, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetErrorName", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetErrorName, "char const *(cudaError_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetErrorString", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetErrorString, "char const *(cudaError_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetDeviceCount", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceCount, "cudaError_t (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetDeviceProperties_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceProperties_v2, "cudaError_t (struct cudaDeviceProp *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGetAttribute", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetAttribute, "cudaError_t (int *, enum cudaDeviceAttr, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGetDefaultMemPool", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetDefaultMemPool, "cudaError_t (cudaMemPool_t *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceSetMemPool", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetMemPool, "cudaError_t (int, cudaMemPool_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGetMemPool", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetMemPool, "cudaError_t (cudaMemPool_t *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGetNvSciSyncAttributes", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetNvSciSyncAttributes, "cudaError_t (void *, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGetP2PAttribute", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetP2PAttribute, "cudaError_t (int *, enum cudaDeviceP2PAttr, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaChooseDevice", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaChooseDevice, "cudaError_t (int *, struct cudaDeviceProp const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaInitDevice", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaInitDevice, "cudaError_t (int, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaSetDevice", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSetDevice, "cudaError_t (int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetDevice", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDevice, "cudaError_t (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaSetDeviceFlags", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSetDeviceFlags, "cudaError_t (unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetDeviceFlags", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceFlags, "cudaError_t (unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamCreate", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreate, "cudaError_t (cudaStream_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamCreateWithFlags", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreateWithFlags, "cudaError_t (cudaStream_t *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamCreateWithPriority", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreateWithPriority, "cudaError_t (cudaStream_t *, unsigned int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamGetPriority", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetPriority, "cudaError_t (cudaStream_t, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamGetFlags", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetFlags, "cudaError_t (cudaStream_t, unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamGetId", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetId, "cudaError_t (cudaStream_t, unsigned PY_LONG_LONG *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamGetDevice", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetDevice, "cudaError_t (cudaStream_t, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaCtxResetPersistingL2Cache", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCtxResetPersistingL2Cache, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamCopyAttributes", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCopyAttributes, "cudaError_t (cudaStream_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamGetAttribute", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetAttribute, "cudaError_t (cudaStream_t, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrID, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrValue *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamSetAttribute", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamSetAttribute, "cudaError_t (cudaStream_t, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrID, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrValue const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamDestroy", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamDestroy, "cudaError_t (cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamWaitEvent", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamWaitEvent, "cudaError_t (cudaStream_t, cudaEvent_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamAddCallback", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamAddCallback, "cudaError_t (cudaStream_t, cudaStreamCallback_t, void *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamSynchronize", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamSynchronize, "cudaError_t (cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamQuery", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamQuery, "cudaError_t (cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamAttachMemAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamAttachMemAsync, "cudaError_t (cudaStream_t, void *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamBeginCapture", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamBeginCapture, "cudaError_t (cudaStream_t, enum cudaStreamCaptureMode)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamBeginCaptureToGraph", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamBeginCaptureToGraph, "cudaError_t (cudaStream_t, cudaGraph_t, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t, enum cudaStreamCaptureMode)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaThreadExchangeStreamCaptureMode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaThreadExchangeStreamCaptureMode, "cudaError_t (enum cudaStreamCaptureMode *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamEndCapture", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamEndCapture, "cudaError_t (cudaStream_t, cudaGraph_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamIsCapturing", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamIsCapturing, "cudaError_t (cudaStream_t, enum cudaStreamCaptureStatus *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamGetCaptureInfo_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetCaptureInfo_v2, "cudaError_t (cudaStream_t, enum cudaStreamCaptureStatus *, unsigned PY_LONG_LONG *, cudaGraph_t *, cudaGraphNode_t const **, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamGetCaptureInfo_v3", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetCaptureInfo_v3, "cudaError_t (cudaStream_t, enum cudaStreamCaptureStatus *, unsigned PY_LONG_LONG *, cudaGraph_t *, cudaGraphNode_t const **, cudaGraphEdgeData const **, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamUpdateCaptureDependencies", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamUpdateCaptureDependencies, "cudaError_t (cudaStream_t, cudaGraphNode_t *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaStreamUpdateCaptureDependencies_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamUpdateCaptureDependencies_v2, "cudaError_t (cudaStream_t, cudaGraphNode_t *, cudaGraphEdgeData const *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaEventCreate", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventCreate, "cudaError_t (cudaEvent_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaEventCreateWithFlags", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventCreateWithFlags, "cudaError_t (cudaEvent_t *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaEventRecord", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventRecord, "cudaError_t (cudaEvent_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaEventRecordWithFlags", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventRecordWithFlags, "cudaError_t (cudaEvent_t, cudaStream_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaEventQuery", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventQuery, "cudaError_t (cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaEventSynchronize", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventSynchronize, "cudaError_t (cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaEventDestroy", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventDestroy, "cudaError_t (cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaEventElapsedTime", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventElapsedTime, "cudaError_t (float *, cudaEvent_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaEventElapsedTime_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventElapsedTime_v2, "cudaError_t (float *, cudaEvent_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaImportExternalMemory", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaImportExternalMemory, "cudaError_t (cudaExternalMemory_t *, struct cudaExternalMemoryHandleDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaExternalMemoryGetMappedBuffer", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaExternalMemoryGetMappedBuffer, "cudaError_t (void **, cudaExternalMemory_t, struct cudaExternalMemoryBufferDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaExternalMemoryGetMappedMipmappedArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaExternalMemoryGetMappedMipmappedArray, "cudaError_t (cudaMipmappedArray_t *, cudaExternalMemory_t, struct cudaExternalMemoryMipmappedArrayDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDestroyExternalMemory", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyExternalMemory, "cudaError_t (cudaExternalMemory_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaImportExternalSemaphore", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaImportExternalSemaphore, "cudaError_t (cudaExternalSemaphore_t *, struct cudaExternalSemaphoreHandleDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaSignalExternalSemaphoresAsync_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSignalExternalSemaphoresAsync_v2, "cudaError_t (cudaExternalSemaphore_t const *, struct cudaExternalSemaphoreSignalParams const *, unsigned int, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaWaitExternalSemaphoresAsync_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaWaitExternalSemaphoresAsync_v2, "cudaError_t (cudaExternalSemaphore_t const *, struct cudaExternalSemaphoreWaitParams const *, unsigned int, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDestroyExternalSemaphore", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyExternalSemaphore, "cudaError_t (cudaExternalSemaphore_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaFuncSetCacheConfig", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetCacheConfig, "cudaError_t (void const *, enum cudaFuncCache)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaFuncGetAttributes", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncGetAttributes, "cudaError_t (struct cudaFuncAttributes *, void const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaFuncSetAttribute", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetAttribute, "cudaError_t (void const *, enum cudaFuncAttribute, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaLaunchHostFunc", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLaunchHostFunc, "cudaError_t (cudaStream_t, cudaHostFn_t, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaFuncSetSharedMemConfig", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetSharedMemConfig, "cudaError_t (void const *, enum cudaSharedMemConfig)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaOccupancyMaxActiveBlocksPerMultiprocessor", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyMaxActiveBlocksPerMultiprocessor, "cudaError_t (int *, void const *, int, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaOccupancyAvailableDynamicSMemPerBlock", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyAvailableDynamicSMemPerBlock, "cudaError_t (size_t *, void const *, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, "cudaError_t (int *, void const *, int, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMallocManaged", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocManaged, "cudaError_t (void **, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMalloc", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc, "cudaError_t (void **, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMallocHost", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocHost, "cudaError_t (void **, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMallocPitch", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocPitch, "cudaError_t (void **, size_t *, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMallocArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocArray, "cudaError_t (cudaArray_t *, struct cudaChannelFormatDesc const *, size_t, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaFree", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFree, "cudaError_t (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaFreeHost", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeHost, "cudaError_t (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaFreeArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeArray, "cudaError_t (cudaArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaFreeMipmappedArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeMipmappedArray, "cudaError_t (cudaMipmappedArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaHostAlloc", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostAlloc, "cudaError_t (void **, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaHostRegister", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostRegister, "cudaError_t (void *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaHostUnregister", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostUnregister, "cudaError_t (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaHostGetDevicePointer", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostGetDevicePointer, "cudaError_t (void **, void *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaHostGetFlags", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostGetFlags, "cudaError_t (unsigned int *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMalloc3D", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc3D, "cudaError_t (struct cudaPitchedPtr *, struct cudaExtent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMalloc3DArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc3DArray, "cudaError_t (cudaArray_t *, struct cudaChannelFormatDesc const *, struct cudaExtent, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMallocMipmappedArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocMipmappedArray, "cudaError_t (cudaMipmappedArray_t *, struct cudaChannelFormatDesc const *, struct cudaExtent, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetMipmappedArrayLevel", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetMipmappedArrayLevel, "cudaError_t (cudaArray_t *, cudaMipmappedArray_const_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpy3D", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3D, "cudaError_t (struct cudaMemcpy3DParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpy3DPeer", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DPeer, "cudaError_t (struct cudaMemcpy3DPeerParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpy3DAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DAsync, "cudaError_t (struct cudaMemcpy3DParms const *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpy3DPeerAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DPeerAsync, "cudaError_t (struct cudaMemcpy3DPeerParms const *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemGetInfo", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemGetInfo, "cudaError_t (size_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaArrayGetInfo", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetInfo, "cudaError_t (struct cudaChannelFormatDesc *, struct cudaExtent *, unsigned int *, cudaArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaArrayGetPlane", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetPlane, "cudaError_t (cudaArray_t *, cudaArray_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaArrayGetMemoryRequirements", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetMemoryRequirements, "cudaError_t (struct cudaArrayMemoryRequirements *, cudaArray_t, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMipmappedArrayGetMemoryRequirements", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMipmappedArrayGetMemoryRequirements, "cudaError_t (struct cudaArrayMemoryRequirements *, cudaMipmappedArray_t, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaArrayGetSparseProperties", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetSparseProperties, "cudaError_t (struct cudaArraySparseProperties *, cudaArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMipmappedArrayGetSparseProperties", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMipmappedArrayGetSparseProperties, "cudaError_t (struct cudaArraySparseProperties *, cudaMipmappedArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpy", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy, "cudaError_t (void *, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpyPeer", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyPeer, "cudaError_t (void *, int, void const *, int, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpy2D", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2D, "cudaError_t (void *, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpy2DToArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DToArray, "cudaError_t (cudaArray_t, size_t, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpy2DFromArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DFromArray, "cudaError_t (void *, size_t, cudaArray_const_t, size_t, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpy2DArrayToArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DArrayToArray, "cudaError_t (cudaArray_t, size_t, size_t, cudaArray_const_t, size_t, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpyAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyAsync, "cudaError_t (void *, void const *, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpyPeerAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyPeerAsync, "cudaError_t (void *, int, void const *, int, size_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpyBatchAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyBatchAsync, "cudaError_t (void **, void **, size_t *, size_t, struct cudaMemcpyAttributes *, size_t *, size_t, size_t *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpy3DBatchAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DBatchAsync, "cudaError_t (size_t, struct cudaMemcpy3DBatchOp *, size_t *, unsigned PY_LONG_LONG, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpy2DAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DAsync, "cudaError_t (void *, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpy2DToArrayAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DToArrayAsync, "cudaError_t (cudaArray_t, size_t, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpy2DFromArrayAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DFromArrayAsync, "cudaError_t (void *, size_t, cudaArray_const_t, size_t, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemset", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset, "cudaError_t (void *, int, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemset2D", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset2D, "cudaError_t (void *, size_t, int, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemset3D", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset3D, "cudaError_t (struct cudaPitchedPtr, int, struct cudaExtent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemsetAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemsetAsync, "cudaError_t (void *, int, size_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemset2DAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset2DAsync, "cudaError_t (void *, size_t, int, size_t, size_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemset3DAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset3DAsync, "cudaError_t (struct cudaPitchedPtr, int, struct cudaExtent, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemPrefetchAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPrefetchAsync, "cudaError_t (void const *, size_t, int, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemPrefetchAsync_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPrefetchAsync_v2, "cudaError_t (void const *, size_t, struct cudaMemLocation, unsigned int, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemAdvise", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemAdvise, "cudaError_t (void const *, size_t, enum cudaMemoryAdvise, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemAdvise_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemAdvise_v2, "cudaError_t (void const *, size_t, enum cudaMemoryAdvise, struct cudaMemLocation)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemRangeGetAttribute", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemRangeGetAttribute, "cudaError_t (void *, size_t, enum cudaMemRangeAttribute, void const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemRangeGetAttributes", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemRangeGetAttributes, "cudaError_t (void **, size_t *, enum cudaMemRangeAttribute *, size_t, void const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpyToArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyToArray, "cudaError_t (cudaArray_t, size_t, size_t, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpyFromArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyFromArray, "cudaError_t (void *, cudaArray_const_t, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpyArrayToArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyArrayToArray, "cudaError_t (cudaArray_t, size_t, size_t, cudaArray_const_t, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpyToArrayAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyToArrayAsync, "cudaError_t (cudaArray_t, size_t, size_t, void const *, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemcpyFromArrayAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyFromArrayAsync, "cudaError_t (void *, cudaArray_const_t, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMallocAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocAsync, "cudaError_t (void **, size_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaFreeAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeAsync, "cudaError_t (void *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemPoolTrimTo", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolTrimTo, "cudaError_t (cudaMemPool_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemPoolSetAttribute", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolSetAttribute, "cudaError_t (cudaMemPool_t, enum cudaMemPoolAttr, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemPoolGetAttribute", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolGetAttribute, "cudaError_t (cudaMemPool_t, enum cudaMemPoolAttr, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemPoolSetAccess", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolSetAccess, "cudaError_t (cudaMemPool_t, struct cudaMemAccessDesc const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemPoolGetAccess", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolGetAccess, "cudaError_t (enum cudaMemAccessFlags *, cudaMemPool_t, struct cudaMemLocation *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemPoolCreate", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolCreate, "cudaError_t (cudaMemPool_t *, struct cudaMemPoolProps const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemPoolDestroy", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolDestroy, "cudaError_t (cudaMemPool_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMallocFromPoolAsync", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocFromPoolAsync, "cudaError_t (void **, size_t, cudaMemPool_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemPoolExportToShareableHandle", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolExportToShareableHandle, "cudaError_t (void *, cudaMemPool_t, enum cudaMemAllocationHandleType, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemPoolImportFromShareableHandle", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolImportFromShareableHandle, "cudaError_t (cudaMemPool_t *, void *, enum cudaMemAllocationHandleType, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemPoolExportPointer", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolExportPointer, "cudaError_t (struct cudaMemPoolPtrExportData *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaMemPoolImportPointer", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolImportPointer, "cudaError_t (void **, cudaMemPool_t, struct cudaMemPoolPtrExportData *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaPointerGetAttributes", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaPointerGetAttributes, "cudaError_t (struct cudaPointerAttributes *, void const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceCanAccessPeer", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceCanAccessPeer, "cudaError_t (int *, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceEnablePeerAccess", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceEnablePeerAccess, "cudaError_t (int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceDisablePeerAccess", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceDisablePeerAccess, "cudaError_t (int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphicsUnregisterResource", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsUnregisterResource, "cudaError_t (cudaGraphicsResource_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphicsResourceSetMapFlags", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceSetMapFlags, "cudaError_t (cudaGraphicsResource_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphicsMapResources", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsMapResources, "cudaError_t (int, cudaGraphicsResource_t *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphicsUnmapResources", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsUnmapResources, "cudaError_t (int, cudaGraphicsResource_t *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphicsResourceGetMappedPointer", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceGetMappedPointer, "cudaError_t (void **, size_t *, cudaGraphicsResource_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphicsSubResourceGetMappedArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsSubResourceGetMappedArray, "cudaError_t (cudaArray_t *, cudaGraphicsResource_t, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphicsResourceGetMappedMipmappedArray", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceGetMappedMipmappedArray, "cudaError_t (cudaMipmappedArray_t *, cudaGraphicsResource_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetChannelDesc", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetChannelDesc, "cudaError_t (struct cudaChannelFormatDesc *, cudaArray_const_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaCreateChannelDesc", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateChannelDesc, "struct cudaChannelFormatDesc (int, int, int, int, enum cudaChannelFormatKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaCreateTextureObject", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateTextureObject, "cudaError_t (cudaTextureObject_t *, struct cudaResourceDesc const *, struct cudaTextureDesc const *, struct cudaResourceViewDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDestroyTextureObject", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyTextureObject, "cudaError_t (cudaTextureObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetTextureObjectResourceDesc", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectResourceDesc, "cudaError_t (struct cudaResourceDesc *, cudaTextureObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetTextureObjectTextureDesc", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectTextureDesc, "cudaError_t (struct cudaTextureDesc *, cudaTextureObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetTextureObjectResourceViewDesc", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectResourceViewDesc, "cudaError_t (struct cudaResourceViewDesc *, cudaTextureObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaCreateSurfaceObject", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateSurfaceObject, "cudaError_t (cudaSurfaceObject_t *, struct cudaResourceDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDestroySurfaceObject", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroySurfaceObject, "cudaError_t (cudaSurfaceObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetSurfaceObjectResourceDesc", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetSurfaceObjectResourceDesc, "cudaError_t (struct cudaResourceDesc *, cudaSurfaceObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDriverGetVersion", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDriverGetVersion, "cudaError_t (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaRuntimeGetVersion", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaRuntimeGetVersion, "cudaError_t (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphCreate", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphCreate, "cudaError_t (cudaGraph_t *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddKernelNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddKernelNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaKernelNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphKernelNodeGetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaKernelNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphKernelNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaKernelNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphKernelNodeCopyAttributes", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeCopyAttributes, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphKernelNodeGetAttribute", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeGetAttribute, "cudaError_t (cudaGraphNode_t, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrID, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrValue *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphKernelNodeSetAttribute", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeSetAttribute, "cudaError_t (cudaGraphNode_t, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrID, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrValue const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddMemcpyNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemcpyNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaMemcpy3DParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddMemcpyNode1D", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemcpyNode1D, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, void *, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphMemcpyNodeGetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemcpy3DParms *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphMemcpyNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemcpy3DParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphMemcpyNodeSetParams1D", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeSetParams1D, "cudaError_t (cudaGraphNode_t, void *, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddMemsetNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemsetNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaMemsetParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphMemsetNodeGetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemsetNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemsetParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphMemsetNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemsetNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemsetParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddHostNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddHostNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaHostNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphHostNodeGetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphHostNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaHostNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphHostNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphHostNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaHostNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddChildGraphNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddChildGraphNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphChildGraphNodeGetGraph", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphChildGraphNodeGetGraph, "cudaError_t (cudaGraphNode_t, cudaGraph_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddEmptyNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEmptyNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddEventRecordNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEventRecordNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphEventRecordNodeGetEvent", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventRecordNodeGetEvent, "cudaError_t (cudaGraphNode_t, cudaEvent_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphEventRecordNodeSetEvent", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventRecordNodeSetEvent, "cudaError_t (cudaGraphNode_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddEventWaitNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEventWaitNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphEventWaitNodeGetEvent", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventWaitNodeGetEvent, "cudaError_t (cudaGraphNode_t, cudaEvent_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphEventWaitNodeSetEvent", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventWaitNodeSetEvent, "cudaError_t (cudaGraphNode_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddExternalSemaphoresSignalNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddExternalSemaphoresSignalNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaExternalSemaphoreSignalNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExternalSemaphoresSignalNodeGetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresSignalNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaExternalSemaphoreSignalNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExternalSemaphoresSignalNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresSignalNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaExternalSemaphoreSignalNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddExternalSemaphoresWaitNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddExternalSemaphoresWaitNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaExternalSemaphoreWaitNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExternalSemaphoresWaitNodeGetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresWaitNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaExternalSemaphoreWaitNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExternalSemaphoresWaitNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresWaitNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaExternalSemaphoreWaitNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddMemAllocNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemAllocNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaMemAllocNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphMemAllocNodeGetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemAllocNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemAllocNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddMemFreeNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemFreeNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphMemFreeNodeGetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemFreeNodeGetParams, "cudaError_t (cudaGraphNode_t, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGraphMemTrim", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGraphMemTrim, "cudaError_t (int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceGetGraphMemAttribute", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetGraphMemAttribute, "cudaError_t (int, enum cudaGraphMemAttributeType, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaDeviceSetGraphMemAttribute", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetGraphMemAttribute, "cudaError_t (int, enum cudaGraphMemAttributeType, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphClone", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphClone, "cudaError_t (cudaGraph_t *, cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphNodeFindInClone", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeFindInClone, "cudaError_t (cudaGraphNode_t *, cudaGraphNode_t, cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphNodeGetType", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetType, "cudaError_t (cudaGraphNode_t, enum cudaGraphNodeType *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphGetNodes", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetNodes, "cudaError_t (cudaGraph_t, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphGetRootNodes", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetRootNodes, "cudaError_t (cudaGraph_t, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphGetEdges", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetEdges, "cudaError_t (cudaGraph_t, cudaGraphNode_t *, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphGetEdges_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetEdges_v2, "cudaError_t (cudaGraph_t, cudaGraphNode_t *, cudaGraphNode_t *, cudaGraphEdgeData *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphNodeGetDependencies", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependencies, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphNodeGetDependencies_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependencies_v2, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t *, cudaGraphEdgeData *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphNodeGetDependentNodes", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependentNodes, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphNodeGetDependentNodes_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependentNodes_v2, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t *, cudaGraphEdgeData *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddDependencies", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddDependencies, "cudaError_t (cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddDependencies_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddDependencies_v2, "cudaError_t (cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphRemoveDependencies", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRemoveDependencies, "cudaError_t (cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphRemoveDependencies_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRemoveDependencies_v2, "cudaError_t (cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphDestroyNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDestroyNode, "cudaError_t (cudaGraphNode_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphInstantiate", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiate, "cudaError_t (cudaGraphExec_t *, cudaGraph_t, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphInstantiateWithFlags", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiateWithFlags, "cudaError_t (cudaGraphExec_t *, cudaGraph_t, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphInstantiateWithParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiateWithParams, "cudaError_t (cudaGraphExec_t *, cudaGraph_t, cudaGraphInstantiateParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecGetFlags", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecGetFlags, "cudaError_t (cudaGraphExec_t, unsigned PY_LONG_LONG *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecKernelNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecKernelNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaKernelNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecMemcpyNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemcpyNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaMemcpy3DParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecMemcpyNodeSetParams1D", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemcpyNodeSetParams1D, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, void *, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecMemsetNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemsetNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaMemsetParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecHostNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecHostNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaHostNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecChildGraphNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecChildGraphNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecEventRecordNodeSetEvent", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecEventRecordNodeSetEvent, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecEventWaitNodeSetEvent", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecEventWaitNodeSetEvent, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecExternalSemaphoresSignalNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecExternalSemaphoresSignalNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaExternalSemaphoreSignalNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecExternalSemaphoresWaitNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecExternalSemaphoresWaitNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaExternalSemaphoreWaitNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphNodeSetEnabled", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeSetEnabled, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphNodeGetEnabled", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetEnabled, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecUpdate", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecUpdate, "cudaError_t (cudaGraphExec_t, cudaGraph_t, cudaGraphExecUpdateResultInfo *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphUpload", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphUpload, "cudaError_t (cudaGraphExec_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphLaunch", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphLaunch, "cudaError_t (cudaGraphExec_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecDestroy", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecDestroy, "cudaError_t (cudaGraphExec_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphDestroy", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDestroy, "cudaError_t (cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphDebugDotPrint", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDebugDotPrint, "cudaError_t (cudaGraph_t, char const *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaUserObjectCreate", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectCreate, "cudaError_t (cudaUserObject_t *, void *, cudaHostFn_t, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaUserObjectRetain", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectRetain, "cudaError_t (cudaUserObject_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaUserObjectRelease", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectRelease, "cudaError_t (cudaUserObject_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphRetainUserObject", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRetainUserObject, "cudaError_t (cudaGraph_t, cudaUserObject_t, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphReleaseUserObject", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphReleaseUserObject, "cudaError_t (cudaGraph_t, cudaUserObject_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddNode", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaGraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphAddNode_v2", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddNode_v2, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t, struct cudaGraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaGraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphExecNodeSetParams", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaGraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGraphConditionalHandleCreate", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphConditionalHandleCreate, "cudaError_t (cudaGraphConditionalHandle *, cudaGraph_t, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetDriverEntryPoint", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDriverEntryPoint, "cudaError_t (char const *, void **, unsigned PY_LONG_LONG, enum cudaDriverEntryPointQueryResult *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetDriverEntryPointByVersion", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDriverEntryPointByVersion, "cudaError_t (char const *, void **, unsigned int, unsigned PY_LONG_LONG, enum cudaDriverEntryPointQueryResult *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaLibraryLoadData", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryLoadData, "cudaError_t (cudaLibrary_t *, void const *, enum cudaJitOption *, void **, unsigned int, enum cudaLibraryOption *, void **, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaLibraryLoadFromFile", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryLoadFromFile, "cudaError_t (cudaLibrary_t *, char const *, enum cudaJitOption *, void **, unsigned int, enum cudaLibraryOption *, void **, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaLibraryUnload", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryUnload, "cudaError_t (cudaLibrary_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaLibraryGetKernel", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetKernel, "cudaError_t (cudaKernel_t *, cudaLibrary_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaLibraryGetGlobal", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetGlobal, "cudaError_t (void **, size_t *, cudaLibrary_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaLibraryGetManaged", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetManaged, "cudaError_t (void **, size_t *, cudaLibrary_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaLibraryGetUnifiedFunction", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetUnifiedFunction, "cudaError_t (void **, cudaLibrary_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaLibraryGetKernelCount", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetKernelCount, "cudaError_t (unsigned int *, cudaLibrary_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaLibraryEnumerateKernels", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryEnumerateKernels, "cudaError_t (cudaKernel_t *, unsigned int, cudaLibrary_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaKernelSetAttributeForDevice", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaKernelSetAttributeForDevice, "cudaError_t (cudaKernel_t, enum cudaFuncAttribute, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetExportTable", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetExportTable, "cudaError_t (void const **, cudaUUID_t const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaGetKernel", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetKernel, "cudaError_t (cudaKernel_t *, void const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_make_cudaPitchedPtr", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaPitchedPtr, "struct cudaPitchedPtr (void *, size_t, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_make_cudaPos", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaPos, "struct cudaPos (size_t, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_make_cudaExtent", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaExtent, "struct cudaExtent (size_t, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaProfilerStart", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaProfilerStart, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ImportFunction_3_1_5(__pyx_t_1, "_cudaProfilerStop", (void (**)(void))&__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaProfilerStop, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_cyruntime(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_cyruntime}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "cyruntime", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_cyruntime(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_cyruntime(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x03090000 + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_cyruntime(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'cyruntime' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "cyruntime" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_NOT_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY +__Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); +if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); +} +#endif + +__Pyx_RefNannySetupContext("PyInit_cyruntime", 0); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #ifdef __Pxy_PyFrame_Initialize_Offsets + __Pxy_PyFrame_Initialize_Offsets(); + #endif + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #if 0 || defined(__Pyx_CyFunction_USED) || defined(__Pyx_FusedFunction_USED) || defined(__Pyx_Coroutine_USED) || defined(__Pyx_Generator_USED) || defined(__Pyx_AsyncGen_USED) + if (__pyx_CommonTypesMetaclass_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_AsyncGen_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + if (__pyx_module_is_main_cuda__bindings___bindings__cyruntime) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "cuda.bindings._bindings.cyruntime")) { + if (unlikely((PyDict_SetItemString(modules, "cuda.bindings._bindings.cyruntime", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_function_export_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_type_init_code(__pyx_mstate); + (void)__Pyx_modinit_type_import_code(__pyx_mstate); + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_function_import_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Execution code ---*/ + + /* "cuda/bindings/_bindings/cyruntime.pyx":7 + * include "../cyruntime_functions.pxi" + * + * import os # <<<<<<<<<<<<<< + * cimport cuda.bindings._bindings.cyruntime_ptds as ptds + * cimport cython +*/ + __pyx_t_2 = __Pyx_ImportDottedModule(__pyx_mstate_global->__pyx_n_u_os, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 7, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_os, __pyx_t_2) < (0)) __PYX_ERR(0, 7, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":11 + * cimport cython + * + * cdef bint __cudaPythonInit = False # <<<<<<<<<<<<<< + * cdef bint __usePTDS = False + * cdef int _cudaPythonInit() except -1 nogil: +*/ + __pyx_v_4cuda_8bindings_9_bindings_9cyruntime___cudaPythonInit = 0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":12 + * + * cdef bint __cudaPythonInit = False + * cdef bint __usePTDS = False # <<<<<<<<<<<<<< + * cdef int _cudaPythonInit() except -1 nogil: + * global __cudaPythonInit +*/ + __pyx_v_4cuda_8bindings_9_bindings_9cyruntime___usePTDS = 0; + + /* "cuda/bindings/_bindings/cyruntime.pyx":1 + * # SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # <<<<<<<<<<<<<< + * # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + * +*/ + __pyx_t_2 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_2) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init cuda.bindings._bindings.cyruntime", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init cuda.bindings._bindings.cyruntime"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ + +typedef struct { + const char *s; +#if 42 <= 65535 + const unsigned short n; +#elif 42 / 2 < INT_MAX + const unsigned int n; +#elif 42 / 2 < LONG_MAX + const unsigned long n; +#else + const Py_ssize_t n; +#endif +#if 1 <= 31 + const unsigned int encoding : 5; +#elif 1 <= 255 + const unsigned char encoding; +#elif 1 <= 65535 + const unsigned short encoding; +#else + const Py_ssize_t encoding; +#endif + const unsigned int is_unicode : 1; + const unsigned int intern : 1; +} __Pyx_StringTabEntry; +static const char * const __pyx_string_tab_encodings[] = { 0 }; +static const __Pyx_StringTabEntry __pyx_string_tab[] = { + {__pyx_k_, sizeof(__pyx_k_), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_ */ + {__pyx_k_CUDA_PYTHON_CUDA_PER_THREAD_DEFA, sizeof(__pyx_k_CUDA_PYTHON_CUDA_PER_THREAD_DEFA), 0, 1, 1}, /* PyObject cname: __pyx_n_u_CUDA_PYTHON_CUDA_PER_THREAD_DEFA */ + {__pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cline_in_traceback */ + {__pyx_k_default, sizeof(__pyx_k_default), 0, 1, 1}, /* PyObject cname: __pyx_n_u_default */ + {__pyx_k_getenv, sizeof(__pyx_k_getenv), 0, 1, 1}, /* PyObject cname: __pyx_n_u_getenv */ + {__pyx_k_initializing, sizeof(__pyx_k_initializing), 0, 1, 1}, /* PyObject cname: __pyx_n_u_initializing */ + {__pyx_k_main, sizeof(__pyx_k_main), 0, 1, 1}, /* PyObject cname: __pyx_n_u_main */ + {__pyx_k_module, sizeof(__pyx_k_module), 0, 1, 1}, /* PyObject cname: __pyx_n_u_module */ + {__pyx_k_name, sizeof(__pyx_k_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name */ + {__pyx_k_os, sizeof(__pyx_k_os), 0, 1, 1}, /* PyObject cname: __pyx_n_u_os */ + {__pyx_k_qualname, sizeof(__pyx_k_qualname), 0, 1, 1}, /* PyObject cname: __pyx_n_u_qualname */ + {__pyx_k_range, sizeof(__pyx_k_range), 0, 1, 1}, /* PyObject cname: __pyx_n_u_range */ + {__pyx_k_spec, sizeof(__pyx_k_spec), 0, 1, 1}, /* PyObject cname: __pyx_n_u_spec */ + {__pyx_k_test, sizeof(__pyx_k_test), 0, 1, 1}, /* PyObject cname: __pyx_n_u_test */ + {0, 0, 0, 0, 0} +}; +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names); + +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_range); if (!__pyx_builtin_range) __PYX_ERR(1, 574, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + __Pyx_RefNannyFinishContext(); + return 0; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + if (__Pyx_InitStrings(__pyx_string_tab, __pyx_mstate->__pyx_string_tab, __pyx_string_tab_encodings) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + return 0; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* PyErrExceptionMatches */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStr */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* PyObjectGetAttrStrNoError */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name); + if (unlikely(!result) && !PyErr_Occurred()) { + PyErr_Format(PyExc_NameError, + "name '%U' is not defined", name); + } + return result; +} + +/* PyDictVersioning */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!__pyx_m)) { + if (!PyErr_Occurred()) + PyErr_SetNone(PyExc_NameError); + return NULL; + } + result = PyObject_GetAttr(__pyx_m, name); + if (likely(result)) { + return result; + } + PyErr_Clear(); +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear(); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return result; + } +#else + result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject *const *args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = __Pyx_PyFrame_GetLocalsplus(f); + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; + PyObject *kwdefs; + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) { + return NULL; + } + if ( + co->co_kwonlyargcount == 0 && + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, (int)nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif + +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = Py_TYPE(func)->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + #if PY_VERSION_HEX < 0x030800B1 + #if CYTHON_FAST_PYCCALL + if (PyCFunction_Check(func)) { + if (kwargs) { + return _PyCFunction_FastCallDict(func, args, nargs, kwargs); + } else { + return _PyCFunction_FastCallKeywords(func, args, nargs, NULL); + } + } + if (!kwargs && __Pyx_IS_TYPE(func, &PyMethodDescr_Type)) { + return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL); + } + #endif + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs); + } + #endif + #endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #elif defined(__Pyx_CyFunction_USED) && CYTHON_BACKPORT_VECTORCALL + if (__Pyx_CyFunction_CheckExact(func)) { + __pyx_vectorcallfunc f = __Pyx_CyFunction_func_vectorcall(func); + if (f) return f(func, args, _nargs, NULL); + } + #elif CYTHON_COMPILING_IN_LIMITED_API && CYTHON_VECTORCALL + return PyObject_Vectorcall(func, args, _nargs, NULL); + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* PyObjectVectorCallKwBuilder */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* ErrOccurredWithGIL */ +static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void) { + int err; + PyGILState_STATE _save = PyGILState_Ensure(); + err = !!PyErr_Occurred(); + PyGILState_Release(_save); + return err; +} + +/* Import */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { + PyObject *module = 0; + PyObject *empty_dict = 0; + PyObject *empty_list = 0; + empty_dict = PyDict_New(); + if (unlikely(!empty_dict)) + goto bad; + if (level == -1) { + const char* package_sep = strchr(__Pyx_MODULE_NAME, '.'); + if (package_sep != (0)) { + module = PyImport_ImportModuleLevelObject( + name, __pyx_mstate_global->__pyx_d, empty_dict, from_list, 1); + if (unlikely(!module)) { + if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError))) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } + if (!module) { + module = PyImport_ImportModuleLevelObject( + name, __pyx_mstate_global->__pyx_d, empty_dict, from_list, level); + } +bad: + Py_XDECREF(empty_dict); + Py_XDECREF(empty_list); + return module; +} + +/* ImportDottedModule */ +static PyObject *__Pyx__ImportDottedModule_Error(PyObject *name, PyObject *parts_tuple, Py_ssize_t count) { + PyObject *partial_name = NULL, *slice = NULL, *sep = NULL; + Py_ssize_t size; + if (unlikely(PyErr_Occurred())) { + PyErr_Clear(); + } +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(parts_tuple); +#else + size = PyTuple_Size(parts_tuple); + if (size < 0) goto bad; +#endif + if (likely(size == count)) { + partial_name = name; + } else { + slice = PySequence_GetSlice(parts_tuple, 0, count); + if (unlikely(!slice)) + goto bad; + sep = PyUnicode_FromStringAndSize(".", 1); + if (unlikely(!sep)) + goto bad; + partial_name = PyUnicode_Join(sep, slice); + } + PyErr_Format( + PyExc_ModuleNotFoundError, + "No module named '%U'", partial_name); +bad: + Py_XDECREF(sep); + Py_XDECREF(slice); + Py_XDECREF(partial_name); + return NULL; +} +static PyObject *__Pyx__ImportDottedModule_Lookup(PyObject *name) { + PyObject *imported_module; +#if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + return NULL; + imported_module = __Pyx_PyDict_GetItemStr(modules, name); + Py_XINCREF(imported_module); +#else + imported_module = PyImport_GetModule(name); +#endif + return imported_module; +} +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple) { + Py_ssize_t i, nparts; +#if CYTHON_ASSUME_SAFE_SIZE + nparts = PyTuple_GET_SIZE(parts_tuple); +#else + nparts = PyTuple_Size(parts_tuple); + if (nparts < 0) return NULL; +#endif + for (i=1; i < nparts && module; i++) { + PyObject *part, *submodule; +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + part = PyTuple_GET_ITEM(parts_tuple, i); +#else + part = __Pyx_PySequence_ITEM(parts_tuple, i); + if (!part) return NULL; +#endif + submodule = __Pyx_PyObject_GetAttrStrNoError(module, part); +#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(part); +#endif + Py_DECREF(module); + module = submodule; + } + if (unlikely(!module)) { + return __Pyx__ImportDottedModule_Error(name, parts_tuple, i); + } + return module; +} +static PyObject *__Pyx__ImportDottedModule(PyObject *name, PyObject *parts_tuple) { + PyObject *imported_module; + PyObject *module = __Pyx_Import(name, NULL, 0); + if (!parts_tuple || unlikely(!module)) + return module; + imported_module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(imported_module)) { + Py_DECREF(module); + return imported_module; + } + PyErr_Clear(); + return __Pyx_ImportDottedModule_WalkParts(module, name, parts_tuple); +} +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple) { +#if CYTHON_COMPILING_IN_CPYTHON + PyObject *module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(module)) { + PyObject *spec = __Pyx_PyObject_GetAttrStrNoError(module, __pyx_mstate_global->__pyx_n_u_spec); + if (likely(spec)) { + PyObject *unsafe = __Pyx_PyObject_GetAttrStrNoError(spec, __pyx_mstate_global->__pyx_n_u_initializing); + if (likely(!unsafe || !__Pyx_PyObject_IsTrue(unsafe))) { + Py_DECREF(spec); + spec = NULL; + } + Py_XDECREF(unsafe); + } + if (likely(!spec)) { + PyErr_Clear(); + return module; + } + Py_DECREF(spec); + Py_DECREF(module); + } else if (PyErr_Occurred()) { + PyErr_Clear(); + } +#endif + return __Pyx__ImportDottedModule(name, parts_tuple); +} + +/* CLineInTraceback */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline; + PyObject *ptype, *pvalue, *ptraceback; +#if CYTHON_COMPILING_IN_CPYTHON + PyObject **cython_runtime_dict; +#endif + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); +#if CYTHON_COMPILING_IN_CPYTHON + cython_runtime_dict = _PyObject_GetDictPtr(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __Pyx_BEGIN_CRITICAL_SECTION(*cython_runtime_dict); + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, *cython_runtime_dict, + __Pyx_PyDict_GetItemStr(*cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback)) + Py_XINCREF(use_cline); + __Pyx_END_CRITICAL_SECTION(); + } else +#endif + { + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback); + if (use_cline_obj) { + use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; + Py_INCREF(use_cline); + Py_DECREF(use_cline_obj); + } else { + PyErr_Clear(); + use_cline = NULL; + } + } + if (!use_cline) { + c_line = 0; + (void) PyObject_SetAttr(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False); + } + else if (use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + (void) __pyx_cfilenm; + (void) __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + py_funcname = PyUnicode_FromString(funcname); + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* CIntFromPyVerify */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_unsigned_int(unsigned int value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const unsigned int neg_one = (unsigned int) -1, const_zero = (unsigned int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(unsigned int) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(unsigned int) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if defined(HAVE_LONG_LONG) && !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(unsigned int) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(unsigned int) <= sizeof(long)) { + return PyLong_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(unsigned int) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(unsigned int), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(unsigned int)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPy */ +static CYTHON_INLINE unsigned int __Pyx_PyLong_As_unsigned_int(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const unsigned int neg_one = (unsigned int) -1, const_zero = (unsigned int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + unsigned int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (unsigned int) -1; + val = __Pyx_PyLong_As_unsigned_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(unsigned int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(unsigned int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(unsigned int) >= 2 * PyLong_SHIFT)) { + return (unsigned int) (((((unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(unsigned int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(unsigned int) >= 3 * PyLong_SHIFT)) { + return (unsigned int) (((((((unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(unsigned int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(unsigned int) >= 4 * PyLong_SHIFT)) { + return (unsigned int) (((((((((unsigned int)digits[3]) << PyLong_SHIFT) | (unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (unsigned int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(unsigned int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(unsigned int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(unsigned int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(unsigned int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(unsigned int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(unsigned int) - 1 > 2 * PyLong_SHIFT)) { + return (unsigned int) (((unsigned int)-1)*(((((unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(unsigned int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(unsigned int) - 1 > 2 * PyLong_SHIFT)) { + return (unsigned int) ((((((unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(unsigned int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(unsigned int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(unsigned int) - 1 > 3 * PyLong_SHIFT)) { + return (unsigned int) (((unsigned int)-1)*(((((((unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(unsigned int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(unsigned int) - 1 > 3 * PyLong_SHIFT)) { + return (unsigned int) ((((((((unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(unsigned int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(unsigned int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(unsigned int) - 1 > 4 * PyLong_SHIFT)) { + return (unsigned int) (((unsigned int)-1)*(((((((((unsigned int)digits[3]) << PyLong_SHIFT) | (unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(unsigned int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(unsigned int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(unsigned int) - 1 > 4 * PyLong_SHIFT)) { + return (unsigned int) ((((((((((unsigned int)digits[3]) << PyLong_SHIFT) | (unsigned int)digits[2]) << PyLong_SHIFT) | (unsigned int)digits[1]) << PyLong_SHIFT) | (unsigned int)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(unsigned int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(unsigned int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(unsigned int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + unsigned int val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (unsigned int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (unsigned int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (unsigned int) -1; + } else { + stepval = v; + } + v = NULL; + val = (unsigned int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(unsigned int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((unsigned int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(unsigned int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((unsigned int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((unsigned int) 1) << (sizeof(unsigned int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (unsigned int) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to unsigned int"); + return (unsigned int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to unsigned int"); + return (unsigned int) -1; +} + +/* FormatTypeName */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static __Pyx_TypeName +__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) +{ + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u_); + } + goto done; +} +#endif + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if defined(HAVE_LONG_LONG) && !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPy */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + long val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (long) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + int val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030b0000 + return Py_Version & ~0xFFUL; +#else + static unsigned long __Pyx_cached_runtime_version = 0; + if (__Pyx_cached_runtime_version == 0) { + const char* rt_version = Py_GetVersion(); + unsigned long version = 0; + unsigned long factor = 0x01000000UL; + unsigned int digit = 0; + int i = 0; + while (factor) { + while ('0' <= rt_version[i] && rt_version[i] <= '9') { + digit = digit * 10 + (unsigned int) (rt_version[i] - '0'); + ++i; + } + version += factor * digit; + if (rt_version[i] != '.') + break; + digit = 0; + factor >>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } + return __Pyx_cached_runtime_version; +#endif +} + +/* CheckBinaryVersion */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } +} + +/* FunctionExport */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig) { + PyObject *d = 0; + PyObject *cobj = 0; + union { + void (*fp)(void); + void *p; + } tmp; + d = PyObject_GetAttrString(__pyx_m, "__pyx_capi__"); + if (!d) { + PyErr_Clear(); + d = PyDict_New(); + if (!d) + goto bad; + Py_INCREF(d); + if (PyModule_AddObject(__pyx_m, "__pyx_capi__", d) < 0) + goto bad; + } + tmp.fp = f; + cobj = PyCapsule_New(tmp.p, sig, 0); + if (!cobj) + goto bad; + if (PyDict_SetItemString(d, name, cobj) < 0) + goto bad; + Py_DECREF(cobj); + Py_DECREF(d); + return 0; +bad: + Py_XDECREF(cobj); + Py_XDECREF(d); + return -1; +} + +/* FunctionImport */ +#ifndef __PYX_HAVE_RT_ImportFunction_3_1_5 +#define __PYX_HAVE_RT_ImportFunction_3_1_5 +static int __Pyx_ImportFunction_3_1_5(PyObject *module, const char *funcname, void (**f)(void), const char *sig) { + PyObject *d = 0; + PyObject *cobj = 0; + union { + void (*fp)(void); + void *p; + } tmp; + d = PyObject_GetAttrString(module, "__pyx_capi__"); + if (!d) + goto bad; +#if (defined(Py_LIMITED_API) && Py_LIMITED_API >= 0x030d0000) || (!defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030d0000) + PyDict_GetItemStringRef(d, funcname, &cobj); +#else + cobj = PyDict_GetItemString(d, funcname); + Py_XINCREF(cobj); +#endif + if (!cobj) { + PyErr_Format(PyExc_ImportError, + "%.200s does not export expected C function %.200s", + PyModule_GetName(module), funcname); + goto bad; + } + if (!PyCapsule_IsValid(cobj, sig)) { + PyErr_Format(PyExc_TypeError, + "C function %.200s.%.200s has wrong signature (expected %.500s, got %.500s)", + PyModule_GetName(module), funcname, sig, PyCapsule_GetName(cobj)); + goto bad; + } + tmp.p = PyCapsule_GetPointer(cobj, sig); + *f = tmp.fp; + if (!(*f)) + goto bad; + Py_DECREF(d); + Py_DECREF(cobj); + return 0; +bad: + Py_XDECREF(d); + Py_XDECREF(cobj); + return -1; +} +#endif + +/* InitStrings */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names) { + while (t->s) { + PyObject *str; + if (t->is_unicode) { + if (t->intern) { + str = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + str = PyUnicode_Decode(t->s, t->n - 1, encoding_names[t->encoding], NULL); + } else { + str = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + str = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + if (!str) + return -1; + *target = str; + if (PyObject_Hash(str) == -1) + return -1; + ++t; + ++target; + } + return 0; +} + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} +#if CYTHON_USE_PYLONG_INTERNALS +static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value) { + if (unlikely(!__Pyx_PyLong_IsCompact(x))) + return 0; + Py_ssize_t value = __Pyx_PyLong_CompactValue(x); + if ((sizeof(long) < sizeof(Py_ssize_t)) && unlikely(value != (long) value)) + return 0; + *return_value = (long) value; + return 1; +} +#endif + + +/* MultiPhaseInitModuleState */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 +#else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 +#endif +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS +#error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" +#endif +#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#define __Pyx_ModuleStateLookup_Lock() +#define __Pyx_ModuleStateLookup_Unlock() +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 +static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; +#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#include +static std::mutex __Pyx_ModuleStateLookup_mutex; +#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() +#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) +#include +static mtx_t __Pyx_ModuleStateLookup_mutex; +static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; +static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); +} +#define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(HAVE_PTHREAD_H) +#include +static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; +#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(_WIN32) +#include // synchapi.h on its own doesn't work +static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; +#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#else +#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." +#endif +typedef struct { + int64_t id; + PyObject *module; +} __Pyx_InterpreterIdAndModule; +typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; +} __Pyx_ModuleStateLookupData; +#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; +#else +static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; +#endif +static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; +} +static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; +#else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); +#endif + return result; + } +} +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); +} +#else +#define __Pyx_ModuleStateLookup_wait_until_no_readers() +#endif +static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; +} +static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; +} +static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); +#else + __Pyx_ModuleStateLookup_data = new_data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return result; +} +static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; +#endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); +#else + __Pyx_ModuleStateLookup_data = data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; +} +#endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime.pxd new file mode 100644 index 0000000000000000000000000000000000000000..274b480485f81e5fb8925a6881f856de2f668316 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime.pxd @@ -0,0 +1,591 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +# This code was automatically generated with version 12.9.0. Do not modify it directly. +include "../cyruntime_types.pxi" + +include "../_lib/cyruntime/cyruntime.pxd" + +cdef cudaError_t _cudaDeviceReset() except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil + +cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil + +cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil + +cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil + +cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil + +cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil + +cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil + +cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime.pyx new file mode 100644 index 0000000000000000000000000000000000000000..e706c2640345f6b18222a0dbe2e860105e8ac1cb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime.pyx @@ -0,0 +1,1782 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +# This code was automatically generated with version 12.9.0. Do not modify it directly. +include "../cyruntime_functions.pxi" + +import os +cimport cuda.bindings._bindings.cyruntime_ptds as ptds +cimport cython + +cdef bint __cudaPythonInit = False +cdef bint __usePTDS = False +cdef int _cudaPythonInit() except -1 nogil: + global __cudaPythonInit + global __usePTDS + + with gil: + __usePTDS = os.getenv('CUDA_PYTHON_CUDA_PER_THREAD_DEFAULT_STREAM', default=False) + __cudaPythonInit = True + return __usePTDS + +# Create a very small function to check whether we are init'ed, so the C +# compiler can inline it. +cdef inline int cudaPythonInit() except -1 nogil: + if __cudaPythonInit: + return __usePTDS + return _cudaPythonInit() + +cdef cudaError_t _cudaDeviceReset() except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceReset() + return cudaDeviceReset() + +cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceSynchronize() + return cudaDeviceSynchronize() + +cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceSetLimit(limit, value) + return cudaDeviceSetLimit(limit, value) + +cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGetLimit(pValue, limit) + return cudaDeviceGetLimit(pValue, limit) + +cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) + return cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) + +cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGetCacheConfig(pCacheConfig) + return cudaDeviceGetCacheConfig(pCacheConfig) + +cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) + return cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) + +cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceSetCacheConfig(cacheConfig) + return cudaDeviceSetCacheConfig(cacheConfig) + +cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGetByPCIBusId(device, pciBusId) + return cudaDeviceGetByPCIBusId(device, pciBusId) + +cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGetPCIBusId(pciBusId, length, device) + return cudaDeviceGetPCIBusId(pciBusId, length, device) + +cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaIpcGetEventHandle(handle, event) + return cudaIpcGetEventHandle(handle, event) + +cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaIpcOpenEventHandle(event, handle) + return cudaIpcOpenEventHandle(event, handle) + +cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaIpcGetMemHandle(handle, devPtr) + return cudaIpcGetMemHandle(handle, devPtr) + +cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaIpcOpenMemHandle(devPtr, handle, flags) + return cudaIpcOpenMemHandle(devPtr, handle, flags) + +cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaIpcCloseMemHandle(devPtr) + return cudaIpcCloseMemHandle(devPtr) + +cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceFlushGPUDirectRDMAWrites(target, scope) + return cudaDeviceFlushGPUDirectRDMAWrites(target, scope) + +cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) + return cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) + +cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceUnregisterAsyncNotification(device, callback) + return cudaDeviceUnregisterAsyncNotification(device, callback) + +cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGetSharedMemConfig(pConfig) + return cudaDeviceGetSharedMemConfig(pConfig) + +cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceSetSharedMemConfig(config) + return cudaDeviceSetSharedMemConfig(config) + +cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetLastError() + return cudaGetLastError() + +cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaPeekAtLastError() + return cudaPeekAtLastError() + +cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetErrorName(error) + return cudaGetErrorName(error) + +cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetErrorString(error) + return cudaGetErrorString(error) + +cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetDeviceCount(count) + return cudaGetDeviceCount(count) + +cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetDeviceProperties_v2(prop, device) + return cudaGetDeviceProperties_v2(prop, device) + +cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGetAttribute(value, attr, device) + return cudaDeviceGetAttribute(value, attr, device) + +cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGetDefaultMemPool(memPool, device) + return cudaDeviceGetDefaultMemPool(memPool, device) + +cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceSetMemPool(device, memPool) + return cudaDeviceSetMemPool(device, memPool) + +cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGetMemPool(memPool, device) + return cudaDeviceGetMemPool(memPool, device) + +cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) + return cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) + +cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) + return cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) + +cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaChooseDevice(device, prop) + return cudaChooseDevice(device, prop) + +cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaInitDevice(device, deviceFlags, flags) + return cudaInitDevice(device, deviceFlags, flags) + +cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaSetDevice(device) + return cudaSetDevice(device) + +cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetDevice(device) + return cudaGetDevice(device) + +cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaSetDeviceFlags(flags) + return cudaSetDeviceFlags(flags) + +cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetDeviceFlags(flags) + return cudaGetDeviceFlags(flags) + +cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamCreate(pStream) + return cudaStreamCreate(pStream) + +cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamCreateWithFlags(pStream, flags) + return cudaStreamCreateWithFlags(pStream, flags) + +cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamCreateWithPriority(pStream, flags, priority) + return cudaStreamCreateWithPriority(pStream, flags, priority) + +cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamGetPriority(hStream, priority) + return cudaStreamGetPriority(hStream, priority) + +cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamGetFlags(hStream, flags) + return cudaStreamGetFlags(hStream, flags) + +cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamGetId(hStream, streamId) + return cudaStreamGetId(hStream, streamId) + +cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamGetDevice(hStream, device) + return cudaStreamGetDevice(hStream, device) + +cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaCtxResetPersistingL2Cache() + return cudaCtxResetPersistingL2Cache() + +cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamCopyAttributes(dst, src) + return cudaStreamCopyAttributes(dst, src) + +cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamGetAttribute(hStream, attr, value_out) + return cudaStreamGetAttribute(hStream, attr, value_out) + +cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamSetAttribute(hStream, attr, value) + return cudaStreamSetAttribute(hStream, attr, value) + +cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamDestroy(stream) + return cudaStreamDestroy(stream) + +cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamWaitEvent(stream, event, flags) + return cudaStreamWaitEvent(stream, event, flags) + +cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamAddCallback(stream, callback, userData, flags) + return cudaStreamAddCallback(stream, callback, userData, flags) + +cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamSynchronize(stream) + return cudaStreamSynchronize(stream) + +cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamQuery(stream) + return cudaStreamQuery(stream) + +cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamAttachMemAsync(stream, devPtr, length, flags) + return cudaStreamAttachMemAsync(stream, devPtr, length, flags) + +cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamBeginCapture(stream, mode) + return cudaStreamBeginCapture(stream, mode) + +cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) + return cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) + +cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaThreadExchangeStreamCaptureMode(mode) + return cudaThreadExchangeStreamCaptureMode(mode) + +cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamEndCapture(stream, pGraph) + return cudaStreamEndCapture(stream, pGraph) + +cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamIsCapturing(stream, pCaptureStatus) + return cudaStreamIsCapturing(stream, pCaptureStatus) + +cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamGetCaptureInfo_v2(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + return cudaStreamGetCaptureInfo_v2(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + +cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + return cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + +cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) + return cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) + +cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) + return cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) + +cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaEventCreate(event) + return cudaEventCreate(event) + +cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaEventCreateWithFlags(event, flags) + return cudaEventCreateWithFlags(event, flags) + +cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaEventRecord(event, stream) + return cudaEventRecord(event, stream) + +cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaEventRecordWithFlags(event, stream, flags) + return cudaEventRecordWithFlags(event, stream, flags) + +cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaEventQuery(event) + return cudaEventQuery(event) + +cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaEventSynchronize(event) + return cudaEventSynchronize(event) + +cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaEventDestroy(event) + return cudaEventDestroy(event) + +cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaEventElapsedTime(ms, start, end) + return cudaEventElapsedTime(ms, start, end) + +cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaEventElapsedTime_v2(ms, start, end) + return cudaEventElapsedTime_v2(ms, start, end) + +cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaImportExternalMemory(extMem_out, memHandleDesc) + return cudaImportExternalMemory(extMem_out, memHandleDesc) + +cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) + return cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) + +cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) + return cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) + +cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDestroyExternalMemory(extMem) + return cudaDestroyExternalMemory(extMem) + +cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaImportExternalSemaphore(extSem_out, semHandleDesc) + return cudaImportExternalSemaphore(extSem_out, semHandleDesc) + +cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaSignalExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + return cudaSignalExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + +cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaWaitExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + return cudaWaitExternalSemaphoresAsync_v2(extSemArray, paramsArray, numExtSems, stream) + +cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDestroyExternalSemaphore(extSem) + return cudaDestroyExternalSemaphore(extSem) + +cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaFuncSetCacheConfig(func, cacheConfig) + return cudaFuncSetCacheConfig(func, cacheConfig) + +cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaFuncGetAttributes(attr, func) + return cudaFuncGetAttributes(attr, func) + +cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaFuncSetAttribute(func, attr, value) + return cudaFuncSetAttribute(func, attr, value) + +cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaLaunchHostFunc(stream, fn, userData) + return cudaLaunchHostFunc(stream, fn, userData) + +cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaFuncSetSharedMemConfig(func, config) + return cudaFuncSetSharedMemConfig(func, config) + +cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) + return cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) + +cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) + return cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) + +cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) + return cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) + +cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMallocManaged(devPtr, size, flags) + return cudaMallocManaged(devPtr, size, flags) + +cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMalloc(devPtr, size) + return cudaMalloc(devPtr, size) + +cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMallocHost(ptr, size) + return cudaMallocHost(ptr, size) + +cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMallocPitch(devPtr, pitch, width, height) + return cudaMallocPitch(devPtr, pitch, width, height) + +cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMallocArray(array, desc, width, height, flags) + return cudaMallocArray(array, desc, width, height, flags) + +cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaFree(devPtr) + return cudaFree(devPtr) + +cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaFreeHost(ptr) + return cudaFreeHost(ptr) + +cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaFreeArray(array) + return cudaFreeArray(array) + +cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaFreeMipmappedArray(mipmappedArray) + return cudaFreeMipmappedArray(mipmappedArray) + +cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaHostAlloc(pHost, size, flags) + return cudaHostAlloc(pHost, size, flags) + +cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaHostRegister(ptr, size, flags) + return cudaHostRegister(ptr, size, flags) + +cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaHostUnregister(ptr) + return cudaHostUnregister(ptr) + +cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaHostGetDevicePointer(pDevice, pHost, flags) + return cudaHostGetDevicePointer(pDevice, pHost, flags) + +cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaHostGetFlags(pFlags, pHost) + return cudaHostGetFlags(pFlags, pHost) + +cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMalloc3D(pitchedDevPtr, extent) + return cudaMalloc3D(pitchedDevPtr, extent) + +cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMalloc3DArray(array, desc, extent, flags) + return cudaMalloc3DArray(array, desc, extent, flags) + +cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) + return cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) + +cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) + return cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) + +cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpy3D(p) + return cudaMemcpy3D(p) + +cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpy3DPeer(p) + return cudaMemcpy3DPeer(p) + +cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpy3DAsync(p, stream) + return cudaMemcpy3DAsync(p, stream) + +cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpy3DPeerAsync(p, stream) + return cudaMemcpy3DPeerAsync(p, stream) + +cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemGetInfo(free, total) + return cudaMemGetInfo(free, total) + +cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaArrayGetInfo(desc, extent, flags, array) + return cudaArrayGetInfo(desc, extent, flags, array) + +cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) + return cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) + +cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaArrayGetMemoryRequirements(memoryRequirements, array, device) + return cudaArrayGetMemoryRequirements(memoryRequirements, array, device) + +cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) + return cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) + +cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaArrayGetSparseProperties(sparseProperties, array) + return cudaArrayGetSparseProperties(sparseProperties, array) + +cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) + return cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) + +cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpy(dst, src, count, kind) + return cudaMemcpy(dst, src, count, kind) + +cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) + return cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) + +cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) + return cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) + +cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) + return cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) + +cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) + return cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) + +cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) + return cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) + +cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpyAsync(dst, src, count, kind, stream) + return cudaMemcpyAsync(dst, src, count, kind, stream) + +cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) + return cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) + +cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) + return cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) + +cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) + return cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) + +cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) + return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) + +cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) + return cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) + +cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) + return cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) + +cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemset(devPtr, value, count) + return cudaMemset(devPtr, value, count) + +cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemset2D(devPtr, pitch, value, width, height) + return cudaMemset2D(devPtr, pitch, value, width, height) + +cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemset3D(pitchedDevPtr, value, extent) + return cudaMemset3D(pitchedDevPtr, value, extent) + +cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemsetAsync(devPtr, value, count, stream) + return cudaMemsetAsync(devPtr, value, count, stream) + +cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) + return cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) + +cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) + return cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) + +cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) + return cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) + +cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) + return cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) + +cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemAdvise(devPtr, count, advice, device) + return cudaMemAdvise(devPtr, count, advice, device) + +cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemAdvise_v2(devPtr, count, advice, location) + return cudaMemAdvise_v2(devPtr, count, advice, location) + +cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) + return cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) + +cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) + return cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) + +cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) + return cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) + +cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) + return cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) + +cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) + return cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) + +cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) + return cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) + +cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) + return cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) + +cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMallocAsync(devPtr, size, hStream) + return cudaMallocAsync(devPtr, size, hStream) + +cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaFreeAsync(devPtr, hStream) + return cudaFreeAsync(devPtr, hStream) + +cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemPoolTrimTo(memPool, minBytesToKeep) + return cudaMemPoolTrimTo(memPool, minBytesToKeep) + +cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemPoolSetAttribute(memPool, attr, value) + return cudaMemPoolSetAttribute(memPool, attr, value) + +cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemPoolGetAttribute(memPool, attr, value) + return cudaMemPoolGetAttribute(memPool, attr, value) + +cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemPoolSetAccess(memPool, descList, count) + return cudaMemPoolSetAccess(memPool, descList, count) + +cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemPoolGetAccess(flags, memPool, location) + return cudaMemPoolGetAccess(flags, memPool, location) + +cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemPoolCreate(memPool, poolProps) + return cudaMemPoolCreate(memPool, poolProps) + +cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemPoolDestroy(memPool) + return cudaMemPoolDestroy(memPool) + +cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMallocFromPoolAsync(ptr, size, memPool, stream) + return cudaMallocFromPoolAsync(ptr, size, memPool, stream) + +cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) + return cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) + +cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) + return cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) + +cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemPoolExportPointer(exportData, ptr) + return cudaMemPoolExportPointer(exportData, ptr) + +cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaMemPoolImportPointer(ptr, memPool, exportData) + return cudaMemPoolImportPointer(ptr, memPool, exportData) + +cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaPointerGetAttributes(attributes, ptr) + return cudaPointerGetAttributes(attributes, ptr) + +cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) + return cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) + +cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceEnablePeerAccess(peerDevice, flags) + return cudaDeviceEnablePeerAccess(peerDevice, flags) + +cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceDisablePeerAccess(peerDevice) + return cudaDeviceDisablePeerAccess(peerDevice) + +cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphicsUnregisterResource(resource) + return cudaGraphicsUnregisterResource(resource) + +cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphicsResourceSetMapFlags(resource, flags) + return cudaGraphicsResourceSetMapFlags(resource, flags) + +cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphicsMapResources(count, resources, stream) + return cudaGraphicsMapResources(count, resources, stream) + +cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphicsUnmapResources(count, resources, stream) + return cudaGraphicsUnmapResources(count, resources, stream) + +cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) + return cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) + +cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) + return cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) + +cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) + return cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) + +cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetChannelDesc(desc, array) + return cudaGetChannelDesc(desc, array) +@cython.show_performance_hints(False) +cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaCreateChannelDesc(x, y, z, w, f) + return cudaCreateChannelDesc(x, y, z, w, f) + +cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) + return cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) + +cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDestroyTextureObject(texObject) + return cudaDestroyTextureObject(texObject) + +cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetTextureObjectResourceDesc(pResDesc, texObject) + return cudaGetTextureObjectResourceDesc(pResDesc, texObject) + +cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetTextureObjectTextureDesc(pTexDesc, texObject) + return cudaGetTextureObjectTextureDesc(pTexDesc, texObject) + +cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) + return cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) + +cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaCreateSurfaceObject(pSurfObject, pResDesc) + return cudaCreateSurfaceObject(pSurfObject, pResDesc) + +cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDestroySurfaceObject(surfObject) + return cudaDestroySurfaceObject(surfObject) + +cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) + return cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) + +cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDriverGetVersion(driverVersion) + return cudaDriverGetVersion(driverVersion) + +cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaRuntimeGetVersion(runtimeVersion) + return cudaRuntimeGetVersion(runtimeVersion) + +cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphCreate(pGraph, flags) + return cudaGraphCreate(pGraph, flags) + +cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + return cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + +cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphKernelNodeGetParams(node, pNodeParams) + return cudaGraphKernelNodeGetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphKernelNodeSetParams(node, pNodeParams) + return cudaGraphKernelNodeSetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphKernelNodeCopyAttributes(hSrc, hDst) + return cudaGraphKernelNodeCopyAttributes(hSrc, hDst) + +cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) + return cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) + +cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphKernelNodeSetAttribute(hNode, attr, value) + return cudaGraphKernelNodeSetAttribute(hNode, attr, value) + +cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) + return cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) + +cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) + return cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) + +cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphMemcpyNodeGetParams(node, pNodeParams) + return cudaGraphMemcpyNodeGetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphMemcpyNodeSetParams(node, pNodeParams) + return cudaGraphMemcpyNodeSetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) + return cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) + +cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) + return cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) + +cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphMemsetNodeGetParams(node, pNodeParams) + return cudaGraphMemsetNodeGetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphMemsetNodeSetParams(node, pNodeParams) + return cudaGraphMemsetNodeSetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + return cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + +cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphHostNodeGetParams(node, pNodeParams) + return cudaGraphHostNodeGetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphHostNodeSetParams(node, pNodeParams) + return cudaGraphHostNodeSetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) + return cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) + +cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphChildGraphNodeGetGraph(node, pGraph) + return cudaGraphChildGraphNodeGetGraph(node, pGraph) + +cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) + return cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) + +cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) + return cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) + +cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphEventRecordNodeGetEvent(node, event_out) + return cudaGraphEventRecordNodeGetEvent(node, event_out) + +cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphEventRecordNodeSetEvent(node, event) + return cudaGraphEventRecordNodeSetEvent(node, event) + +cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) + return cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) + +cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphEventWaitNodeGetEvent(node, event_out) + return cudaGraphEventWaitNodeGetEvent(node, event_out) + +cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphEventWaitNodeSetEvent(node, event) + return cudaGraphEventWaitNodeSetEvent(node, event) + +cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + return cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + +cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) + return cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) + +cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) + return cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) + +cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + return cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + +cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) + return cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) + +cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) + return cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) + +cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + return cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + +cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphMemAllocNodeGetParams(node, params_out) + return cudaGraphMemAllocNodeGetParams(node, params_out) + +cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) + return cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) + +cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphMemFreeNodeGetParams(node, dptr_out) + return cudaGraphMemFreeNodeGetParams(node, dptr_out) + +cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGraphMemTrim(device) + return cudaDeviceGraphMemTrim(device) + +cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceGetGraphMemAttribute(device, attr, value) + return cudaDeviceGetGraphMemAttribute(device, attr, value) + +cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaDeviceSetGraphMemAttribute(device, attr, value) + return cudaDeviceSetGraphMemAttribute(device, attr, value) + +cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphClone(pGraphClone, originalGraph) + return cudaGraphClone(pGraphClone, originalGraph) + +cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) + return cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) + +cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphNodeGetType(node, pType) + return cudaGraphNodeGetType(node, pType) + +cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphGetNodes(graph, nodes, numNodes) + return cudaGraphGetNodes(graph, nodes, numNodes) + +cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) + return cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) + +cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphGetEdges(graph, from_, to, numEdges) + return cudaGraphGetEdges(graph, from_, to, numEdges) + +cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) + return cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) + +cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) + return cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) + +cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) + return cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) + +cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) + return cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) + +cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) + return cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) + +cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddDependencies(graph, from_, to, numDependencies) + return cudaGraphAddDependencies(graph, from_, to, numDependencies) + +cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) + return cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) + +cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphRemoveDependencies(graph, from_, to, numDependencies) + return cudaGraphRemoveDependencies(graph, from_, to, numDependencies) + +cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) + return cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) + +cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphDestroyNode(node) + return cudaGraphDestroyNode(node) + +cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphInstantiate(pGraphExec, graph, flags) + return cudaGraphInstantiate(pGraphExec, graph, flags) + +cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) + return cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) + +cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) + return cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) + +cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecGetFlags(graphExec, flags) + return cudaGraphExecGetFlags(graphExec, flags) + +cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) + return cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) + +cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) + return cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) + +cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) + return cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) + +cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) + return cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) + +cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) + return cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) + +cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) + return cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) + +cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) + return cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) + +cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) + return cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) + +cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) + return cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) + +cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) + return cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) + +cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) + return cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) + +cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) + return cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) + +cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) + return cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) + +cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphUpload(graphExec, stream) + return cudaGraphUpload(graphExec, stream) + +cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphLaunch(graphExec, stream) + return cudaGraphLaunch(graphExec, stream) + +cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecDestroy(graphExec) + return cudaGraphExecDestroy(graphExec) + +cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphDestroy(graph) + return cudaGraphDestroy(graph) + +cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphDebugDotPrint(graph, path, flags) + return cudaGraphDebugDotPrint(graph, path, flags) + +cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) + return cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) + +cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaUserObjectRetain(object, count) + return cudaUserObjectRetain(object, count) + +cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaUserObjectRelease(object, count) + return cudaUserObjectRelease(object, count) + +cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphRetainUserObject(graph, object, count, flags) + return cudaGraphRetainUserObject(graph, object, count, flags) + +cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphReleaseUserObject(graph, object, count) + return cudaGraphReleaseUserObject(graph, object, count) + +cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + return cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + +cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) + return cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) + +cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphNodeSetParams(node, nodeParams) + return cudaGraphNodeSetParams(node, nodeParams) + +cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphExecNodeSetParams(graphExec, node, nodeParams) + return cudaGraphExecNodeSetParams(graphExec, node, nodeParams) + +cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) + return cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) + +cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) + return cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) + +cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) + return cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) + +cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + return cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + +cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + return cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + +cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaLibraryUnload(library) + return cudaLibraryUnload(library) + +cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaLibraryGetKernel(pKernel, library, name) + return cudaLibraryGetKernel(pKernel, library, name) + +cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaLibraryGetGlobal(dptr, numbytes, library, name) + return cudaLibraryGetGlobal(dptr, numbytes, library, name) + +cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaLibraryGetManaged(dptr, numbytes, library, name) + return cudaLibraryGetManaged(dptr, numbytes, library, name) + +cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaLibraryGetUnifiedFunction(fptr, library, symbol) + return cudaLibraryGetUnifiedFunction(fptr, library, symbol) + +cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaLibraryGetKernelCount(count, lib) + return cudaLibraryGetKernelCount(count, lib) + +cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaLibraryEnumerateKernels(kernels, numKernels, lib) + return cudaLibraryEnumerateKernels(kernels, numKernels, lib) + +cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaKernelSetAttributeForDevice(kernel, attr, value, device) + return cudaKernelSetAttributeForDevice(kernel, attr, value, device) + +cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetExportTable(ppExportTable, pExportTableId) + return cudaGetExportTable(ppExportTable, pExportTableId) + +cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaGetKernel(kernelPtr, entryFuncAddr) + return cudaGetKernel(kernelPtr, entryFuncAddr) +@cython.show_performance_hints(False) +cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._make_cudaPitchedPtr(d, p, xsz, ysz) + return make_cudaPitchedPtr(d, p, xsz, ysz) +@cython.show_performance_hints(False) +cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._make_cudaPos(x, y, z) + return make_cudaPos(x, y, z) +@cython.show_performance_hints(False) +cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._make_cudaExtent(w, h, d) + return make_cudaExtent(w, h, d) + +cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaProfilerStart() + return cudaProfilerStart() + +cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil: + cdef bint usePTDS = cudaPythonInit() + if usePTDS: + return ptds._cudaProfilerStop() + return cudaProfilerStop() + + +include "../_lib/cyruntime/cyruntime.pxi" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime_ptds.cpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime_ptds.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6c936dd24a0e0746b1b23a92a729a86f3eddac93 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime_ptds.cpp @@ -0,0 +1,15004 @@ +/* Generated by Cython 3.1.5 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [ + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/cuda_profiler_api.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/cuda_runtime.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/cuda_runtime_api.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/device_types.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/driver_types.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/library_types.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/surface_types.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/texture_types.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/vector_types.h" + ], + "extra_compile_args": [ + "-std=c++14", + "-fpermissive", + "-Wno-deprecated-declarations", + "-fno-var-tracking-assignments", + "-O3" + ], + "include_dirs": [ + "/opt/python/cp312-cp312/include", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include" + ], + "language": "c++", + "libraries": [ + "cudart_static", + "rt" + ], + "library_dirs": [ + "/tmp/build-env-3t3_8d9r/lib/python3.12/site-packages", + "/tmp/build-env-3t3_8d9r/lib", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib64", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib" + ], + "name": "cuda.bindings._bindings.cyruntime_ptds", + "sources": [ + "cuda/bindings/_bindings/cyruntime_ptds.pyx" + ] + }, + "module_name": "cuda.bindings._bindings.cyruntime_ptds" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_1_5" +#define CYTHON_HEX_VERSION 0x030105F0 +#define CYTHON_FUTURE_DIVISION 1 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef HAVE_LONG_LONG + #define HAVE_LONG_LONG +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_CLINE_IN_TRACEBACK + #define CYTHON_CLINE_IN_TRACEBACK 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1) +#endif +#endif +#define CYTHON_BACKPORT_VECTORCALL (CYTHON_METH_FASTCALL && PY_VERSION_HEX < 0x030800B1) +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME + #define CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME 100 +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + #endif + #endif + #if _MSC_VER < 1300 + #ifdef _WIN64 + typedef unsigned long long __pyx_uintptr_t; + #else + typedef unsigned int __pyx_uintptr_t; + #endif + #else + #ifdef _WIN64 + typedef unsigned __int64 __pyx_uintptr_t; + #else + typedef unsigned __int32 __pyx_uintptr_t; + #endif + #endif +#else + #include + typedef uintptr_t __pyx_uintptr_t; +#endif +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#elif CYTHON_BACKPORT_VECTORCALL + typedef PyObject *(*__pyx_vectorcallfunc)(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames); + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1)) + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(((size_t)(n)) & ~__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if __PYX_LIMITED_VERSION_HEX < 0x03090000 + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS && PY_VERSION_HEX >= 0x03080000 +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static CYTHON_INLINE PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__cuda__bindings___bindings__cyruntime_ptds +#define __PYX_HAVE_API__cuda__bindings___bindings__cyruntime_ptds +/* Early includes */ + + #define CUDA_API_PER_THREAD_DEFAULT_STREAM + +#include "vector_types.h" +#include "driver_types.h" +#include "surface_types.h" +#include "texture_types.h" +#include "library_types.h" +#include "cuda_runtime_api.h" +#include "device_types.h" + + #define CUDA_API_PER_THREAD_DEFAULT_STREAM + +#include "cuda_runtime.h" +#include "cuda_profiler_api.h" +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value); + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "cuda/bindings/_bindings/cyruntime_ptds.pyx", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* CriticalSections.proto */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_Begin1(cs, arg) (void)cs +#define __Pyx_PyCriticalSection_Begin2(cs, arg1, arg2) (void)cs +#define __Pyx_PyCriticalSection_End1(cs) +#define __Pyx_PyCriticalSection_End2(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_Begin1 PyCriticalSection_Begin +#define __Pyx_PyCriticalSection_Begin2 PyCriticalSection2_Begin +#define __Pyx_PyCriticalSection_End1 PyCriticalSection_End +#define __Pyx_PyCriticalSection_End2 PyCriticalSection2_End +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* Atomics.proto */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif +#if CYTHON_ATOMICS + #define __pyx_add_acquisition_count(memview)\ + __pyx_atomic_incr_relaxed(__pyx_get_slice_count_pointer(memview)) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_atomic_decr_acq_rel(__pyx_get_slice_count_pointer(memview)) +#else + #define __pyx_add_acquisition_count(memview)\ + __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_sub_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) +#endif + +/* #### Code section: numeric_typedefs ### */ +/* #### Code section: complex_type_declarations ### */ +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ + +/* "cuda/bindings/cyruntime_types.pxi":1553 + * cudaRoundMinInf = 3 + * + * ctypedef cudaLaunchAttributeID cudaStreamAttrID # <<<<<<<<<<<<<< + * + * ctypedef cudaLaunchAttributeID cudaKernelNodeAttrID +*/ +typedef enum cudaLaunchAttributeID __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrID; + +/* "cuda/bindings/cyruntime_types.pxi":1555 + * ctypedef cudaLaunchAttributeID cudaStreamAttrID + * + * ctypedef cudaLaunchAttributeID cudaKernelNodeAttrID # <<<<<<<<<<<<<< + * + * ctypedef cudaLaunchAttributeValue cudaStreamAttrValue +*/ +typedef enum cudaLaunchAttributeID __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrID; + +/* "cuda/bindings/cyruntime_types.pxi":1557 + * ctypedef cudaLaunchAttributeID cudaKernelNodeAttrID + * + * ctypedef cudaLaunchAttributeValue cudaStreamAttrValue # <<<<<<<<<<<<<< + * + * ctypedef cudaLaunchAttributeValue cudaKernelNodeAttrValue +*/ +typedef union cudaLaunchAttributeValue __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrValue; + +/* "cuda/bindings/cyruntime_types.pxi":1559 + * ctypedef cudaLaunchAttributeValue cudaStreamAttrValue + * + * ctypedef cudaLaunchAttributeValue cudaKernelNodeAttrValue # <<<<<<<<<<<<<< +*/ +typedef union cudaLaunchAttributeValue __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrValue; +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyDictVersioning.proto */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __pyx_dict_cached_value;\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* PyErrExceptionMatches.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* PyObjectGetAttrStrNoError.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* CLineInTraceback.proto */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* GCCDiagnostics.proto */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#if !CYTHON_VECTORCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs); +#endif +#define __Pyx_BUILD_ASSERT_EXPR(cond)\ + (sizeof(char [1 - 2*!(cond)]) - 1) +#ifndef Py_MEMBER_SIZE +#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#endif +#if !CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03080000 + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets() + #define __Pyx_PyFrame_GetLocalsplus(frame) ((frame)->f_localsplus) +#else + static size_t __pyx_pyframe_localsplus_offset = 0; + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets()\ + ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ + (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) + #define __Pyx_PyFrame_GetLocalsplus(frame)\ + (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) +#endif +#endif +#endif + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* PyObjectVectorCallKwBuilder.proto */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +/* GetRuntimeVersion.proto */ +static unsigned long __Pyx_get_runtime_version(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* FunctionExport.proto */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + + +/* Module declarations from "cython" */ + +/* Module declarations from "cuda.bindings._bindings.cyruntime_ptds" */ +/* #### Code section: typeinfo ### */ +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "cuda.bindings._bindings.cyruntime_ptds" +extern int __pyx_module_is_main_cuda__bindings___bindings__cyruntime_ptds; +int __pyx_module_is_main_cuda__bindings___bindings__cyruntime_ptds = 0; + +/* Implementation of "cuda.bindings._bindings.cyruntime_ptds" */ +/* #### Code section: global_var ### */ +/* #### Code section: string_decls ### */ +static const char __pyx_k_[] = "?"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_name[] = "__name__"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_module[] = "__module__"; +static const char __pyx_k_qualname[] = "__qualname__"; +static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; +/* #### Code section: decls ### */ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + #ifdef __Pyx_CyFunction_USED + PyTypeObject *__pyx_CyFunctionType; + #endif + #ifdef __Pyx_FusedFunction_USED + PyTypeObject *__pyx_FusedFunctionType; + #endif + #ifdef __Pyx_Generator_USED + PyTypeObject *__pyx_GeneratorType; + #endif + #ifdef __Pyx_IterableCoroutine_USED + PyTypeObject *__pyx_IterableCoroutineType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineAwaitType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineType; + #endif + PyObject *__pyx_string_tab[7]; +/* #### Code section: module_state_contents ### */ +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[1] +#define __pyx_n_u_main __pyx_string_tab[2] +#define __pyx_n_u_module __pyx_string_tab[3] +#define __pyx_n_u_name __pyx_string_tab[4] +#define __pyx_n_u_qualname __pyx_string_tab[5] +#define __pyx_n_u_test __pyx_string_tab[6] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_CLEAR(clear_module_state->__pyx_FusedFunctionType); + #endif + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + for (int i=0; i<7; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_VISIT(traverse_module_state->__pyx_FusedFunctionType); + #endif + for (int i=0; i<7; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":14 + * cimport cython + * + * cdef cudaError_t _cudaDeviceReset() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceReset() + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceReset(void) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":15 + * + * cdef cudaError_t _cudaDeviceReset() except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceReset() # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceReset(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":14 + * cimport cython + * + * cdef cudaError_t _cudaDeviceReset() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceReset() + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":17 + * return cudaDeviceReset() + * + * cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceSynchronize() + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSynchronize(void) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":18 + * + * cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceSynchronize() # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceSynchronize(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":17 + * return cudaDeviceReset() + * + * cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceSynchronize() + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":20 + * return cudaDeviceSynchronize() + * + * cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceSetLimit(limit, value) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetLimit(enum cudaLimit __pyx_v_limit, size_t __pyx_v_value) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":21 + * + * cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceSetLimit(limit, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceSetLimit(__pyx_v_limit, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":20 + * return cudaDeviceSynchronize() + * + * cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceSetLimit(limit, value) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":23 + * return cudaDeviceSetLimit(limit, value) + * + * cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetLimit(pValue, limit) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetLimit(size_t *__pyx_v_pValue, enum cudaLimit __pyx_v_limit) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":24 + * + * cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGetLimit(pValue, limit) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetLimit(__pyx_v_pValue, __pyx_v_limit); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":23 + * return cudaDeviceSetLimit(limit, value) + * + * cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetLimit(pValue, limit) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":26 + * return cudaDeviceGetLimit(pValue, limit) + * + * cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetTexture1DLinearMaxWidth(size_t *__pyx_v_maxWidthInElements, struct cudaChannelFormatDesc const *__pyx_v_fmtDesc, int __pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":27 + * + * cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetTexture1DLinearMaxWidth(__pyx_v_maxWidthInElements, __pyx_v_fmtDesc, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":26 + * return cudaDeviceGetLimit(pValue, limit) + * + * cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":29 + * return cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) + * + * cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetCacheConfig(pCacheConfig) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetCacheConfig(enum cudaFuncCache *__pyx_v_pCacheConfig) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":30 + * + * cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGetCacheConfig(pCacheConfig) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetCacheConfig(__pyx_v_pCacheConfig); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":29 + * return cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) + * + * cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetCacheConfig(pCacheConfig) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":32 + * return cudaDeviceGetCacheConfig(pCacheConfig) + * + * cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetStreamPriorityRange(int *__pyx_v_leastPriority, int *__pyx_v_greatestPriority) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":33 + * + * cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetStreamPriorityRange(__pyx_v_leastPriority, __pyx_v_greatestPriority); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":32 + * return cudaDeviceGetCacheConfig(pCacheConfig) + * + * cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":35 + * return cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) + * + * cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceSetCacheConfig(cacheConfig) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetCacheConfig(enum cudaFuncCache __pyx_v_cacheConfig) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":36 + * + * cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceSetCacheConfig(cacheConfig) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceSetCacheConfig(__pyx_v_cacheConfig); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":35 + * return cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) + * + * cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceSetCacheConfig(cacheConfig) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":38 + * return cudaDeviceSetCacheConfig(cacheConfig) + * + * cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetByPCIBusId(device, pciBusId) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetByPCIBusId(int *__pyx_v_device, char const *__pyx_v_pciBusId) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":39 + * + * cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGetByPCIBusId(device, pciBusId) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetByPCIBusId(__pyx_v_device, __pyx_v_pciBusId); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":38 + * return cudaDeviceSetCacheConfig(cacheConfig) + * + * cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetByPCIBusId(device, pciBusId) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":41 + * return cudaDeviceGetByPCIBusId(device, pciBusId) + * + * cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetPCIBusId(pciBusId, length, device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetPCIBusId(char *__pyx_v_pciBusId, int __pyx_v_length, int __pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":42 + * + * cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGetPCIBusId(pciBusId, length, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetPCIBusId(__pyx_v_pciBusId, __pyx_v_length, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":41 + * return cudaDeviceGetByPCIBusId(device, pciBusId) + * + * cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetPCIBusId(pciBusId, length, device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":44 + * return cudaDeviceGetPCIBusId(pciBusId, length, device) + * + * cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaIpcGetEventHandle(handle, event) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcGetEventHandle(cudaIpcEventHandle_t *__pyx_v_handle, cudaEvent_t __pyx_v_event) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":45 + * + * cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaIpcGetEventHandle(handle, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaIpcGetEventHandle(__pyx_v_handle, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":44 + * return cudaDeviceGetPCIBusId(pciBusId, length, device) + * + * cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaIpcGetEventHandle(handle, event) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":47 + * return cudaIpcGetEventHandle(handle, event) + * + * cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaIpcOpenEventHandle(event, handle) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcOpenEventHandle(cudaEvent_t *__pyx_v_event, cudaIpcEventHandle_t __pyx_v_handle) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":48 + * + * cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaIpcOpenEventHandle(event, handle) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaIpcOpenEventHandle(__pyx_v_event, __pyx_v_handle); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":47 + * return cudaIpcGetEventHandle(handle, event) + * + * cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaIpcOpenEventHandle(event, handle) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":50 + * return cudaIpcOpenEventHandle(event, handle) + * + * cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaIpcGetMemHandle(handle, devPtr) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcGetMemHandle(cudaIpcMemHandle_t *__pyx_v_handle, void *__pyx_v_devPtr) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":51 + * + * cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaIpcGetMemHandle(handle, devPtr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaIpcGetMemHandle(__pyx_v_handle, __pyx_v_devPtr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":50 + * return cudaIpcOpenEventHandle(event, handle) + * + * cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaIpcGetMemHandle(handle, devPtr) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":53 + * return cudaIpcGetMemHandle(handle, devPtr) + * + * cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaIpcOpenMemHandle(devPtr, handle, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcOpenMemHandle(void **__pyx_v_devPtr, cudaIpcMemHandle_t __pyx_v_handle, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":54 + * + * cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaIpcOpenMemHandle(devPtr, handle, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaIpcOpenMemHandle(__pyx_v_devPtr, __pyx_v_handle, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":53 + * return cudaIpcGetMemHandle(handle, devPtr) + * + * cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaIpcOpenMemHandle(devPtr, handle, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":56 + * return cudaIpcOpenMemHandle(devPtr, handle, flags) + * + * cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaIpcCloseMemHandle(devPtr) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcCloseMemHandle(void *__pyx_v_devPtr) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":57 + * + * cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaIpcCloseMemHandle(devPtr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaIpcCloseMemHandle(__pyx_v_devPtr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":56 + * return cudaIpcOpenMemHandle(devPtr, handle, flags) + * + * cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaIpcCloseMemHandle(devPtr) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":59 + * return cudaIpcCloseMemHandle(devPtr) + * + * cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceFlushGPUDirectRDMAWrites(target, scope) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceFlushGPUDirectRDMAWrites(enum cudaFlushGPUDirectRDMAWritesTarget __pyx_v_target, enum cudaFlushGPUDirectRDMAWritesScope __pyx_v_scope) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":60 + * + * cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceFlushGPUDirectRDMAWrites(target, scope) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceFlushGPUDirectRDMAWrites(__pyx_v_target, __pyx_v_scope); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":59 + * return cudaIpcCloseMemHandle(devPtr) + * + * cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceFlushGPUDirectRDMAWrites(target, scope) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":62 + * return cudaDeviceFlushGPUDirectRDMAWrites(target, scope) + * + * cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceRegisterAsyncNotification(int __pyx_v_device, cudaAsyncCallback __pyx_v_callbackFunc, void *__pyx_v_userData, cudaAsyncCallbackHandle_t *__pyx_v_callback) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":63 + * + * cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceRegisterAsyncNotification(__pyx_v_device, __pyx_v_callbackFunc, __pyx_v_userData, __pyx_v_callback); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":62 + * return cudaDeviceFlushGPUDirectRDMAWrites(target, scope) + * + * cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":65 + * return cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) + * + * cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceUnregisterAsyncNotification(device, callback) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceUnregisterAsyncNotification(int __pyx_v_device, cudaAsyncCallbackHandle_t __pyx_v_callback) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":66 + * + * cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceUnregisterAsyncNotification(device, callback) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceUnregisterAsyncNotification(__pyx_v_device, __pyx_v_callback); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":65 + * return cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) + * + * cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceUnregisterAsyncNotification(device, callback) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":68 + * return cudaDeviceUnregisterAsyncNotification(device, callback) + * + * cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetSharedMemConfig(pConfig) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetSharedMemConfig(enum cudaSharedMemConfig *__pyx_v_pConfig) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":69 + * + * cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGetSharedMemConfig(pConfig) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetSharedMemConfig(__pyx_v_pConfig); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":68 + * return cudaDeviceUnregisterAsyncNotification(device, callback) + * + * cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetSharedMemConfig(pConfig) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":71 + * return cudaDeviceGetSharedMemConfig(pConfig) + * + * cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceSetSharedMemConfig(config) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetSharedMemConfig(enum cudaSharedMemConfig __pyx_v_config) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":72 + * + * cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceSetSharedMemConfig(config) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceSetSharedMemConfig(__pyx_v_config); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":71 + * return cudaDeviceGetSharedMemConfig(pConfig) + * + * cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceSetSharedMemConfig(config) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":74 + * return cudaDeviceSetSharedMemConfig(config) + * + * cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetLastError() + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetLastError(void) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":75 + * + * cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetLastError() # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetLastError(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":74 + * return cudaDeviceSetSharedMemConfig(config) + * + * cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetLastError() + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":77 + * return cudaGetLastError() + * + * cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaPeekAtLastError() + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaPeekAtLastError(void) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":78 + * + * cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaPeekAtLastError() # <<<<<<<<<<<<<< + * + * cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil: +*/ + __pyx_r = cudaPeekAtLastError(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":77 + * return cudaGetLastError() + * + * cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaPeekAtLastError() + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":80 + * return cudaPeekAtLastError() + * + * cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil: # <<<<<<<<<<<<<< + * return cudaGetErrorName(error) + * +*/ + +static char const *__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetErrorName(cudaError_t __pyx_v_error) { + char const *__pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":81 + * + * cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil: + * return cudaGetErrorName(error) # <<<<<<<<<<<<<< + * + * cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil: +*/ + __pyx_r = cudaGetErrorName(__pyx_v_error); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":80 + * return cudaPeekAtLastError() + * + * cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil: # <<<<<<<<<<<<<< + * return cudaGetErrorName(error) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":83 + * return cudaGetErrorName(error) + * + * cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil: # <<<<<<<<<<<<<< + * return cudaGetErrorString(error) + * +*/ + +static char const *__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetErrorString(cudaError_t __pyx_v_error) { + char const *__pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":84 + * + * cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil: + * return cudaGetErrorString(error) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetErrorString(__pyx_v_error); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":83 + * return cudaGetErrorName(error) + * + * cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil: # <<<<<<<<<<<<<< + * return cudaGetErrorString(error) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":86 + * return cudaGetErrorString(error) + * + * cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetDeviceCount(count) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceCount(int *__pyx_v_count) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":87 + * + * cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetDeviceCount(count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetDeviceCount(__pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":86 + * return cudaGetErrorString(error) + * + * cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetDeviceCount(count) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":89 + * return cudaGetDeviceCount(count) + * + * cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetDeviceProperties_v2(prop, device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceProperties_v2(struct cudaDeviceProp *__pyx_v_prop, int __pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":90 + * + * cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetDeviceProperties_v2(prop, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetDeviceProperties_v2(__pyx_v_prop, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":89 + * return cudaGetDeviceCount(count) + * + * cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetDeviceProperties_v2(prop, device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":92 + * return cudaGetDeviceProperties_v2(prop, device) + * + * cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetAttribute(value, attr, device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetAttribute(int *__pyx_v_value, enum cudaDeviceAttr __pyx_v_attr, int __pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":93 + * + * cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGetAttribute(value, attr, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetAttribute(__pyx_v_value, __pyx_v_attr, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":92 + * return cudaGetDeviceProperties_v2(prop, device) + * + * cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetAttribute(value, attr, device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":95 + * return cudaDeviceGetAttribute(value, attr, device) + * + * cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetDefaultMemPool(memPool, device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetDefaultMemPool(cudaMemPool_t *__pyx_v_memPool, int __pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":96 + * + * cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGetDefaultMemPool(memPool, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetDefaultMemPool(__pyx_v_memPool, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":95 + * return cudaDeviceGetAttribute(value, attr, device) + * + * cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetDefaultMemPool(memPool, device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":98 + * return cudaDeviceGetDefaultMemPool(memPool, device) + * + * cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceSetMemPool(device, memPool) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetMemPool(int __pyx_v_device, cudaMemPool_t __pyx_v_memPool) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":99 + * + * cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceSetMemPool(device, memPool) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceSetMemPool(__pyx_v_device, __pyx_v_memPool); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":98 + * return cudaDeviceGetDefaultMemPool(memPool, device) + * + * cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceSetMemPool(device, memPool) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":101 + * return cudaDeviceSetMemPool(device, memPool) + * + * cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetMemPool(memPool, device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetMemPool(cudaMemPool_t *__pyx_v_memPool, int __pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":102 + * + * cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGetMemPool(memPool, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetMemPool(__pyx_v_memPool, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":101 + * return cudaDeviceSetMemPool(device, memPool) + * + * cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetMemPool(memPool, device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":104 + * return cudaDeviceGetMemPool(memPool, device) + * + * cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetNvSciSyncAttributes(void *__pyx_v_nvSciSyncAttrList, int __pyx_v_device, int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":105 + * + * cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetNvSciSyncAttributes(__pyx_v_nvSciSyncAttrList, __pyx_v_device, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":104 + * return cudaDeviceGetMemPool(memPool, device) + * + * cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":107 + * return cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) + * + * cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetP2PAttribute(int *__pyx_v_value, enum cudaDeviceP2PAttr __pyx_v_attr, int __pyx_v_srcDevice, int __pyx_v_dstDevice) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":108 + * + * cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetP2PAttribute(__pyx_v_value, __pyx_v_attr, __pyx_v_srcDevice, __pyx_v_dstDevice); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":107 + * return cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) + * + * cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":110 + * return cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) + * + * cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaChooseDevice(device, prop) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaChooseDevice(int *__pyx_v_device, struct cudaDeviceProp const *__pyx_v_prop) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":111 + * + * cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaChooseDevice(device, prop) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaChooseDevice(__pyx_v_device, __pyx_v_prop); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":110 + * return cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) + * + * cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaChooseDevice(device, prop) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":113 + * return cudaChooseDevice(device, prop) + * + * cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaInitDevice(device, deviceFlags, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaInitDevice(int __pyx_v_device, unsigned int __pyx_v_deviceFlags, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":114 + * + * cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaInitDevice(device, deviceFlags, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaInitDevice(__pyx_v_device, __pyx_v_deviceFlags, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":113 + * return cudaChooseDevice(device, prop) + * + * cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaInitDevice(device, deviceFlags, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":116 + * return cudaInitDevice(device, deviceFlags, flags) + * + * cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaSetDevice(device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSetDevice(int __pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":117 + * + * cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaSetDevice(device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaSetDevice(__pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":116 + * return cudaInitDevice(device, deviceFlags, flags) + * + * cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaSetDevice(device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":119 + * return cudaSetDevice(device) + * + * cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetDevice(device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDevice(int *__pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":120 + * + * cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetDevice(device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetDevice(__pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":119 + * return cudaSetDevice(device) + * + * cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetDevice(device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":122 + * return cudaGetDevice(device) + * + * cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaSetDeviceFlags(flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSetDeviceFlags(unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":123 + * + * cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaSetDeviceFlags(flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaSetDeviceFlags(__pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":122 + * return cudaGetDevice(device) + * + * cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaSetDeviceFlags(flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":125 + * return cudaSetDeviceFlags(flags) + * + * cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetDeviceFlags(flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceFlags(unsigned int *__pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":126 + * + * cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetDeviceFlags(flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetDeviceFlags(__pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":125 + * return cudaSetDeviceFlags(flags) + * + * cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetDeviceFlags(flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":128 + * return cudaGetDeviceFlags(flags) + * + * cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamCreate(pStream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreate(cudaStream_t *__pyx_v_pStream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":129 + * + * cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamCreate(pStream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamCreate(__pyx_v_pStream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":128 + * return cudaGetDeviceFlags(flags) + * + * cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamCreate(pStream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":131 + * return cudaStreamCreate(pStream) + * + * cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamCreateWithFlags(pStream, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreateWithFlags(cudaStream_t *__pyx_v_pStream, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":132 + * + * cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamCreateWithFlags(pStream, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamCreateWithFlags(__pyx_v_pStream, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":131 + * return cudaStreamCreate(pStream) + * + * cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamCreateWithFlags(pStream, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":134 + * return cudaStreamCreateWithFlags(pStream, flags) + * + * cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamCreateWithPriority(pStream, flags, priority) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreateWithPriority(cudaStream_t *__pyx_v_pStream, unsigned int __pyx_v_flags, int __pyx_v_priority) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":135 + * + * cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamCreateWithPriority(pStream, flags, priority) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamCreateWithPriority(__pyx_v_pStream, __pyx_v_flags, __pyx_v_priority); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":134 + * return cudaStreamCreateWithFlags(pStream, flags) + * + * cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamCreateWithPriority(pStream, flags, priority) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":137 + * return cudaStreamCreateWithPriority(pStream, flags, priority) + * + * cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetPriority(hStream, priority) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetPriority(cudaStream_t __pyx_v_hStream, int *__pyx_v_priority) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":138 + * + * cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamGetPriority(hStream, priority) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetPriority(__pyx_v_hStream, __pyx_v_priority); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":137 + * return cudaStreamCreateWithPriority(pStream, flags, priority) + * + * cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetPriority(hStream, priority) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":140 + * return cudaStreamGetPriority(hStream, priority) + * + * cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetFlags(hStream, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetFlags(cudaStream_t __pyx_v_hStream, unsigned int *__pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":141 + * + * cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamGetFlags(hStream, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetFlags(__pyx_v_hStream, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":140 + * return cudaStreamGetPriority(hStream, priority) + * + * cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetFlags(hStream, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":143 + * return cudaStreamGetFlags(hStream, flags) + * + * cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetId(hStream, streamId) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetId(cudaStream_t __pyx_v_hStream, unsigned PY_LONG_LONG *__pyx_v_streamId) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":144 + * + * cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamGetId(hStream, streamId) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetId(__pyx_v_hStream, __pyx_v_streamId); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":143 + * return cudaStreamGetFlags(hStream, flags) + * + * cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetId(hStream, streamId) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":146 + * return cudaStreamGetId(hStream, streamId) + * + * cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetDevice(hStream, device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetDevice(cudaStream_t __pyx_v_hStream, int *__pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":147 + * + * cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamGetDevice(hStream, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetDevice(__pyx_v_hStream, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":146 + * return cudaStreamGetId(hStream, streamId) + * + * cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetDevice(hStream, device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":149 + * return cudaStreamGetDevice(hStream, device) + * + * cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaCtxResetPersistingL2Cache() + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCtxResetPersistingL2Cache(void) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":150 + * + * cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaCtxResetPersistingL2Cache() # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaCtxResetPersistingL2Cache(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":149 + * return cudaStreamGetDevice(hStream, device) + * + * cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaCtxResetPersistingL2Cache() + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":152 + * return cudaCtxResetPersistingL2Cache() + * + * cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamCopyAttributes(dst, src) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCopyAttributes(cudaStream_t __pyx_v_dst, cudaStream_t __pyx_v_src) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":153 + * + * cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamCopyAttributes(dst, src) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamCopyAttributes(__pyx_v_dst, __pyx_v_src); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":152 + * return cudaCtxResetPersistingL2Cache() + * + * cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamCopyAttributes(dst, src) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":155 + * return cudaStreamCopyAttributes(dst, src) + * + * cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetAttribute(hStream, attr, value_out) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetAttribute(cudaStream_t __pyx_v_hStream, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrID __pyx_v_attr, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrValue *__pyx_v_value_out) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":156 + * + * cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamGetAttribute(hStream, attr, value_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetAttribute(__pyx_v_hStream, __pyx_v_attr, __pyx_v_value_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":155 + * return cudaStreamCopyAttributes(dst, src) + * + * cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetAttribute(hStream, attr, value_out) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":158 + * return cudaStreamGetAttribute(hStream, attr, value_out) + * + * cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamSetAttribute(hStream, attr, value) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamSetAttribute(cudaStream_t __pyx_v_hStream, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrID __pyx_v_attr, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrValue const *__pyx_v_value) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":159 + * + * cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamSetAttribute(hStream, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamSetAttribute(__pyx_v_hStream, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":158 + * return cudaStreamGetAttribute(hStream, attr, value_out) + * + * cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamSetAttribute(hStream, attr, value) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":161 + * return cudaStreamSetAttribute(hStream, attr, value) + * + * cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamDestroy(stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamDestroy(cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":162 + * + * cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamDestroy(stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamDestroy(__pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":161 + * return cudaStreamSetAttribute(hStream, attr, value) + * + * cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamDestroy(stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":164 + * return cudaStreamDestroy(stream) + * + * cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamWaitEvent(stream, event, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamWaitEvent(cudaStream_t __pyx_v_stream, cudaEvent_t __pyx_v_event, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":165 + * + * cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamWaitEvent(stream, event, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamWaitEvent(__pyx_v_stream, __pyx_v_event, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":164 + * return cudaStreamDestroy(stream) + * + * cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamWaitEvent(stream, event, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":167 + * return cudaStreamWaitEvent(stream, event, flags) + * + * cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamAddCallback(stream, callback, userData, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamAddCallback(cudaStream_t __pyx_v_stream, cudaStreamCallback_t __pyx_v_callback, void *__pyx_v_userData, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":168 + * + * cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamAddCallback(stream, callback, userData, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamAddCallback(__pyx_v_stream, __pyx_v_callback, __pyx_v_userData, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":167 + * return cudaStreamWaitEvent(stream, event, flags) + * + * cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamAddCallback(stream, callback, userData, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":170 + * return cudaStreamAddCallback(stream, callback, userData, flags) + * + * cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamSynchronize(stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamSynchronize(cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":171 + * + * cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamSynchronize(stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamSynchronize(__pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":170 + * return cudaStreamAddCallback(stream, callback, userData, flags) + * + * cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamSynchronize(stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":173 + * return cudaStreamSynchronize(stream) + * + * cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamQuery(stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamQuery(cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":174 + * + * cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamQuery(stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamQuery(__pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":173 + * return cudaStreamSynchronize(stream) + * + * cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamQuery(stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":176 + * return cudaStreamQuery(stream) + * + * cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamAttachMemAsync(stream, devPtr, length, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamAttachMemAsync(cudaStream_t __pyx_v_stream, void *__pyx_v_devPtr, size_t __pyx_v_length, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":177 + * + * cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamAttachMemAsync(stream, devPtr, length, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamAttachMemAsync(__pyx_v_stream, __pyx_v_devPtr, __pyx_v_length, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":176 + * return cudaStreamQuery(stream) + * + * cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamAttachMemAsync(stream, devPtr, length, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":179 + * return cudaStreamAttachMemAsync(stream, devPtr, length, flags) + * + * cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamBeginCapture(stream, mode) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamBeginCapture(cudaStream_t __pyx_v_stream, enum cudaStreamCaptureMode __pyx_v_mode) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":180 + * + * cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamBeginCapture(stream, mode) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamBeginCapture(__pyx_v_stream, __pyx_v_mode); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":179 + * return cudaStreamAttachMemAsync(stream, devPtr, length, flags) + * + * cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamBeginCapture(stream, mode) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":182 + * return cudaStreamBeginCapture(stream, mode) + * + * cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamBeginCaptureToGraph(cudaStream_t __pyx_v_stream, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_dependencies, cudaGraphEdgeData const *__pyx_v_dependencyData, size_t __pyx_v_numDependencies, enum cudaStreamCaptureMode __pyx_v_mode) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":183 + * + * cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamBeginCaptureToGraph(__pyx_v_stream, __pyx_v_graph, __pyx_v_dependencies, __pyx_v_dependencyData, __pyx_v_numDependencies, __pyx_v_mode); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":182 + * return cudaStreamBeginCapture(stream, mode) + * + * cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":185 + * return cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) + * + * cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaThreadExchangeStreamCaptureMode(mode) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaThreadExchangeStreamCaptureMode(enum cudaStreamCaptureMode *__pyx_v_mode) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":186 + * + * cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaThreadExchangeStreamCaptureMode(mode) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaThreadExchangeStreamCaptureMode(__pyx_v_mode); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":185 + * return cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) + * + * cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaThreadExchangeStreamCaptureMode(mode) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":188 + * return cudaThreadExchangeStreamCaptureMode(mode) + * + * cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamEndCapture(stream, pGraph) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamEndCapture(cudaStream_t __pyx_v_stream, cudaGraph_t *__pyx_v_pGraph) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":189 + * + * cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamEndCapture(stream, pGraph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamEndCapture(__pyx_v_stream, __pyx_v_pGraph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":188 + * return cudaThreadExchangeStreamCaptureMode(mode) + * + * cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamEndCapture(stream, pGraph) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":191 + * return cudaStreamEndCapture(stream, pGraph) + * + * cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamIsCapturing(stream, pCaptureStatus) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamIsCapturing(cudaStream_t __pyx_v_stream, enum cudaStreamCaptureStatus *__pyx_v_pCaptureStatus) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":192 + * + * cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamIsCapturing(stream, pCaptureStatus) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamIsCapturing(__pyx_v_stream, __pyx_v_pCaptureStatus); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":191 + * return cudaStreamEndCapture(stream, pGraph) + * + * cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamIsCapturing(stream, pCaptureStatus) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":194 + * return cudaStreamIsCapturing(stream, pCaptureStatus) + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetCaptureInfo_v2_ptsz(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetCaptureInfo_v2(cudaStream_t __pyx_v_stream, enum cudaStreamCaptureStatus *__pyx_v_captureStatus_out, unsigned PY_LONG_LONG *__pyx_v_id_out, cudaGraph_t *__pyx_v_graph_out, cudaGraphNode_t const **__pyx_v_dependencies_out, size_t *__pyx_v_numDependencies_out) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":195 + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamGetCaptureInfo_v2_ptsz(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetCaptureInfo_v2_ptsz(__pyx_v_stream, __pyx_v_captureStatus_out, __pyx_v_id_out, __pyx_v_graph_out, __pyx_v_dependencies_out, __pyx_v_numDependencies_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":194 + * return cudaStreamIsCapturing(stream, pCaptureStatus) + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetCaptureInfo_v2_ptsz(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":197 + * return cudaStreamGetCaptureInfo_v2_ptsz(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetCaptureInfo_v3(cudaStream_t __pyx_v_stream, enum cudaStreamCaptureStatus *__pyx_v_captureStatus_out, unsigned PY_LONG_LONG *__pyx_v_id_out, cudaGraph_t *__pyx_v_graph_out, cudaGraphNode_t const **__pyx_v_dependencies_out, cudaGraphEdgeData const **__pyx_v_edgeData_out, size_t *__pyx_v_numDependencies_out) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":198 + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamGetCaptureInfo_v3(__pyx_v_stream, __pyx_v_captureStatus_out, __pyx_v_id_out, __pyx_v_graph_out, __pyx_v_dependencies_out, __pyx_v_edgeData_out, __pyx_v_numDependencies_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":197 + * return cudaStreamGetCaptureInfo_v2_ptsz(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + * + * cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":200 + * return cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamUpdateCaptureDependencies(cudaStream_t __pyx_v_stream, cudaGraphNode_t *__pyx_v_dependencies, size_t __pyx_v_numDependencies, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":201 + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamUpdateCaptureDependencies(__pyx_v_stream, __pyx_v_dependencies, __pyx_v_numDependencies, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":200 + * return cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":203 + * return cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamUpdateCaptureDependencies_v2(cudaStream_t __pyx_v_stream, cudaGraphNode_t *__pyx_v_dependencies, cudaGraphEdgeData const *__pyx_v_dependencyData, size_t __pyx_v_numDependencies, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":204 + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaStreamUpdateCaptureDependencies_v2(__pyx_v_stream, __pyx_v_dependencies, __pyx_v_dependencyData, __pyx_v_numDependencies, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":203 + * return cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) + * + * cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":206 + * return cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) + * + * cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventCreate(event) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventCreate(cudaEvent_t *__pyx_v_event) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":207 + * + * cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaEventCreate(event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventCreate(__pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":206 + * return cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) + * + * cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventCreate(event) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":209 + * return cudaEventCreate(event) + * + * cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventCreateWithFlags(event, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventCreateWithFlags(cudaEvent_t *__pyx_v_event, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":210 + * + * cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaEventCreateWithFlags(event, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventCreateWithFlags(__pyx_v_event, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":209 + * return cudaEventCreate(event) + * + * cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventCreateWithFlags(event, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":212 + * return cudaEventCreateWithFlags(event, flags) + * + * cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventRecord(event, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventRecord(cudaEvent_t __pyx_v_event, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":213 + * + * cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaEventRecord(event, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventRecord(__pyx_v_event, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":212 + * return cudaEventCreateWithFlags(event, flags) + * + * cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventRecord(event, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":215 + * return cudaEventRecord(event, stream) + * + * cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventRecordWithFlags(event, stream, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventRecordWithFlags(cudaEvent_t __pyx_v_event, cudaStream_t __pyx_v_stream, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":216 + * + * cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaEventRecordWithFlags(event, stream, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventRecordWithFlags(__pyx_v_event, __pyx_v_stream, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":215 + * return cudaEventRecord(event, stream) + * + * cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventRecordWithFlags(event, stream, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":218 + * return cudaEventRecordWithFlags(event, stream, flags) + * + * cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventQuery(event) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventQuery(cudaEvent_t __pyx_v_event) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":219 + * + * cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaEventQuery(event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventQuery(__pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":218 + * return cudaEventRecordWithFlags(event, stream, flags) + * + * cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventQuery(event) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":221 + * return cudaEventQuery(event) + * + * cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventSynchronize(event) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventSynchronize(cudaEvent_t __pyx_v_event) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":222 + * + * cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaEventSynchronize(event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventSynchronize(__pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":221 + * return cudaEventQuery(event) + * + * cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventSynchronize(event) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":224 + * return cudaEventSynchronize(event) + * + * cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventDestroy(event) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventDestroy(cudaEvent_t __pyx_v_event) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":225 + * + * cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaEventDestroy(event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventDestroy(__pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":224 + * return cudaEventSynchronize(event) + * + * cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventDestroy(event) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":227 + * return cudaEventDestroy(event) + * + * cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventElapsedTime(ms, start, end) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventElapsedTime(float *__pyx_v_ms, cudaEvent_t __pyx_v_start, cudaEvent_t __pyx_v_end) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":228 + * + * cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaEventElapsedTime(ms, start, end) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventElapsedTime(__pyx_v_ms, __pyx_v_start, __pyx_v_end); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":227 + * return cudaEventDestroy(event) + * + * cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventElapsedTime(ms, start, end) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":230 + * return cudaEventElapsedTime(ms, start, end) + * + * cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventElapsedTime_v2(ms, start, end) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventElapsedTime_v2(float *__pyx_v_ms, cudaEvent_t __pyx_v_start, cudaEvent_t __pyx_v_end) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":231 + * + * cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaEventElapsedTime_v2(ms, start, end) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaEventElapsedTime_v2(__pyx_v_ms, __pyx_v_start, __pyx_v_end); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":230 + * return cudaEventElapsedTime(ms, start, end) + * + * cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaEventElapsedTime_v2(ms, start, end) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":233 + * return cudaEventElapsedTime_v2(ms, start, end) + * + * cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaImportExternalMemory(extMem_out, memHandleDesc) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaImportExternalMemory(cudaExternalMemory_t *__pyx_v_extMem_out, struct cudaExternalMemoryHandleDesc const *__pyx_v_memHandleDesc) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":234 + * + * cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaImportExternalMemory(extMem_out, memHandleDesc) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaImportExternalMemory(__pyx_v_extMem_out, __pyx_v_memHandleDesc); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":233 + * return cudaEventElapsedTime_v2(ms, start, end) + * + * cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaImportExternalMemory(extMem_out, memHandleDesc) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":236 + * return cudaImportExternalMemory(extMem_out, memHandleDesc) + * + * cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaExternalMemoryGetMappedBuffer(void **__pyx_v_devPtr, cudaExternalMemory_t __pyx_v_extMem, struct cudaExternalMemoryBufferDesc const *__pyx_v_bufferDesc) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":237 + * + * cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaExternalMemoryGetMappedBuffer(__pyx_v_devPtr, __pyx_v_extMem, __pyx_v_bufferDesc); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":236 + * return cudaImportExternalMemory(extMem_out, memHandleDesc) + * + * cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":239 + * return cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) + * + * cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t *__pyx_v_mipmap, cudaExternalMemory_t __pyx_v_extMem, struct cudaExternalMemoryMipmappedArrayDesc const *__pyx_v_mipmapDesc) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":240 + * + * cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaExternalMemoryGetMappedMipmappedArray(__pyx_v_mipmap, __pyx_v_extMem, __pyx_v_mipmapDesc); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":239 + * return cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) + * + * cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":242 + * return cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) + * + * cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDestroyExternalMemory(extMem) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyExternalMemory(cudaExternalMemory_t __pyx_v_extMem) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":243 + * + * cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDestroyExternalMemory(extMem) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDestroyExternalMemory(__pyx_v_extMem); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":242 + * return cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) + * + * cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDestroyExternalMemory(extMem) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":245 + * return cudaDestroyExternalMemory(extMem) + * + * cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaImportExternalSemaphore(extSem_out, semHandleDesc) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaImportExternalSemaphore(cudaExternalSemaphore_t *__pyx_v_extSem_out, struct cudaExternalSemaphoreHandleDesc const *__pyx_v_semHandleDesc) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":246 + * + * cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaImportExternalSemaphore(extSem_out, semHandleDesc) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaImportExternalSemaphore(__pyx_v_extSem_out, __pyx_v_semHandleDesc); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":245 + * return cudaDestroyExternalMemory(extMem) + * + * cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaImportExternalSemaphore(extSem_out, semHandleDesc) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":248 + * return cudaImportExternalSemaphore(extSem_out, semHandleDesc) + * + * cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaSignalExternalSemaphoresAsync_v2_ptsz(extSemArray, paramsArray, numExtSems, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSignalExternalSemaphoresAsync_v2(cudaExternalSemaphore_t const *__pyx_v_extSemArray, struct cudaExternalSemaphoreSignalParams const *__pyx_v_paramsArray, unsigned int __pyx_v_numExtSems, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":249 + * + * cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaSignalExternalSemaphoresAsync_v2_ptsz(extSemArray, paramsArray, numExtSems, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaSignalExternalSemaphoresAsync_v2_ptsz(__pyx_v_extSemArray, __pyx_v_paramsArray, __pyx_v_numExtSems, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":248 + * return cudaImportExternalSemaphore(extSem_out, semHandleDesc) + * + * cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaSignalExternalSemaphoresAsync_v2_ptsz(extSemArray, paramsArray, numExtSems, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":251 + * return cudaSignalExternalSemaphoresAsync_v2_ptsz(extSemArray, paramsArray, numExtSems, stream) + * + * cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaWaitExternalSemaphoresAsync_v2_ptsz(extSemArray, paramsArray, numExtSems, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaWaitExternalSemaphoresAsync_v2(cudaExternalSemaphore_t const *__pyx_v_extSemArray, struct cudaExternalSemaphoreWaitParams const *__pyx_v_paramsArray, unsigned int __pyx_v_numExtSems, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":252 + * + * cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaWaitExternalSemaphoresAsync_v2_ptsz(extSemArray, paramsArray, numExtSems, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaWaitExternalSemaphoresAsync_v2_ptsz(__pyx_v_extSemArray, __pyx_v_paramsArray, __pyx_v_numExtSems, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":251 + * return cudaSignalExternalSemaphoresAsync_v2_ptsz(extSemArray, paramsArray, numExtSems, stream) + * + * cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaWaitExternalSemaphoresAsync_v2_ptsz(extSemArray, paramsArray, numExtSems, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":254 + * return cudaWaitExternalSemaphoresAsync_v2_ptsz(extSemArray, paramsArray, numExtSems, stream) + * + * cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDestroyExternalSemaphore(extSem) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyExternalSemaphore(cudaExternalSemaphore_t __pyx_v_extSem) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":255 + * + * cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDestroyExternalSemaphore(extSem) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDestroyExternalSemaphore(__pyx_v_extSem); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":254 + * return cudaWaitExternalSemaphoresAsync_v2_ptsz(extSemArray, paramsArray, numExtSems, stream) + * + * cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDestroyExternalSemaphore(extSem) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":257 + * return cudaDestroyExternalSemaphore(extSem) + * + * cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFuncSetCacheConfig(func, cacheConfig) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetCacheConfig(void const *__pyx_v_func, enum cudaFuncCache __pyx_v_cacheConfig) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":258 + * + * cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaFuncSetCacheConfig(func, cacheConfig) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFuncSetCacheConfig(__pyx_v_func, __pyx_v_cacheConfig); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":257 + * return cudaDestroyExternalSemaphore(extSem) + * + * cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFuncSetCacheConfig(func, cacheConfig) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":260 + * return cudaFuncSetCacheConfig(func, cacheConfig) + * + * cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFuncGetAttributes(attr, func) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncGetAttributes(struct cudaFuncAttributes *__pyx_v_attr, void const *__pyx_v_func) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":261 + * + * cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaFuncGetAttributes(attr, func) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFuncGetAttributes(__pyx_v_attr, __pyx_v_func); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":260 + * return cudaFuncSetCacheConfig(func, cacheConfig) + * + * cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFuncGetAttributes(attr, func) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":263 + * return cudaFuncGetAttributes(attr, func) + * + * cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFuncSetAttribute(func, attr, value) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetAttribute(void const *__pyx_v_func, enum cudaFuncAttribute __pyx_v_attr, int __pyx_v_value) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":264 + * + * cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaFuncSetAttribute(func, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFuncSetAttribute(__pyx_v_func, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":263 + * return cudaFuncGetAttributes(attr, func) + * + * cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFuncSetAttribute(func, attr, value) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":266 + * return cudaFuncSetAttribute(func, attr, value) + * + * cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLaunchHostFunc(stream, fn, userData) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLaunchHostFunc(cudaStream_t __pyx_v_stream, cudaHostFn_t __pyx_v_fn, void *__pyx_v_userData) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":267 + * + * cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaLaunchHostFunc(stream, fn, userData) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLaunchHostFunc(__pyx_v_stream, __pyx_v_fn, __pyx_v_userData); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":266 + * return cudaFuncSetAttribute(func, attr, value) + * + * cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLaunchHostFunc(stream, fn, userData) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":269 + * return cudaLaunchHostFunc(stream, fn, userData) + * + * cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFuncSetSharedMemConfig(func, config) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetSharedMemConfig(void const *__pyx_v_func, enum cudaSharedMemConfig __pyx_v_config) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":270 + * + * cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaFuncSetSharedMemConfig(func, config) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFuncSetSharedMemConfig(__pyx_v_func, __pyx_v_config); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":269 + * return cudaLaunchHostFunc(stream, fn, userData) + * + * cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFuncSetSharedMemConfig(func, config) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":272 + * return cudaFuncSetSharedMemConfig(func, config) + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyMaxActiveBlocksPerMultiprocessor(int *__pyx_v_numBlocks, void const *__pyx_v_func, int __pyx_v_blockSize, size_t __pyx_v_dynamicSMemSize) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":273 + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaOccupancyMaxActiveBlocksPerMultiprocessor(__pyx_v_numBlocks, __pyx_v_func, __pyx_v_blockSize, __pyx_v_dynamicSMemSize); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":272 + * return cudaFuncSetSharedMemConfig(func, config) + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":275 + * return cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) + * + * cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyAvailableDynamicSMemPerBlock(size_t *__pyx_v_dynamicSmemSize, void const *__pyx_v_func, int __pyx_v_numBlocks, int __pyx_v_blockSize) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":276 + * + * cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaOccupancyAvailableDynamicSMemPerBlock(__pyx_v_dynamicSmemSize, __pyx_v_func, __pyx_v_numBlocks, __pyx_v_blockSize); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":275 + * return cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) + * + * cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":278 + * return cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int *__pyx_v_numBlocks, void const *__pyx_v_func, int __pyx_v_blockSize, size_t __pyx_v_dynamicSMemSize, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":279 + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(__pyx_v_numBlocks, __pyx_v_func, __pyx_v_blockSize, __pyx_v_dynamicSMemSize, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":278 + * return cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) + * + * cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":281 + * return cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) + * + * cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocManaged(devPtr, size, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocManaged(void **__pyx_v_devPtr, size_t __pyx_v_size, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":282 + * + * cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMallocManaged(devPtr, size, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocManaged(__pyx_v_devPtr, __pyx_v_size, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":281 + * return cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) + * + * cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocManaged(devPtr, size, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":284 + * return cudaMallocManaged(devPtr, size, flags) + * + * cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMalloc(devPtr, size) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc(void **__pyx_v_devPtr, size_t __pyx_v_size) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":285 + * + * cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMalloc(devPtr, size) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMalloc(__pyx_v_devPtr, __pyx_v_size); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":284 + * return cudaMallocManaged(devPtr, size, flags) + * + * cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMalloc(devPtr, size) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":287 + * return cudaMalloc(devPtr, size) + * + * cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocHost(ptr, size) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocHost(void **__pyx_v_ptr, size_t __pyx_v_size) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":288 + * + * cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMallocHost(ptr, size) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocHost(__pyx_v_ptr, __pyx_v_size); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":287 + * return cudaMalloc(devPtr, size) + * + * cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocHost(ptr, size) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":290 + * return cudaMallocHost(ptr, size) + * + * cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocPitch(devPtr, pitch, width, height) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocPitch(void **__pyx_v_devPtr, size_t *__pyx_v_pitch, size_t __pyx_v_width, size_t __pyx_v_height) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":291 + * + * cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMallocPitch(devPtr, pitch, width, height) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocPitch(__pyx_v_devPtr, __pyx_v_pitch, __pyx_v_width, __pyx_v_height); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":290 + * return cudaMallocHost(ptr, size) + * + * cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocPitch(devPtr, pitch, width, height) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":293 + * return cudaMallocPitch(devPtr, pitch, width, height) + * + * cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocArray(array, desc, width, height, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocArray(cudaArray_t *__pyx_v_array, struct cudaChannelFormatDesc const *__pyx_v_desc, size_t __pyx_v_width, size_t __pyx_v_height, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":294 + * + * cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMallocArray(array, desc, width, height, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocArray(__pyx_v_array, __pyx_v_desc, __pyx_v_width, __pyx_v_height, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":293 + * return cudaMallocPitch(devPtr, pitch, width, height) + * + * cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocArray(array, desc, width, height, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":296 + * return cudaMallocArray(array, desc, width, height, flags) + * + * cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFree(devPtr) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFree(void *__pyx_v_devPtr) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":297 + * + * cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaFree(devPtr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFree(__pyx_v_devPtr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":296 + * return cudaMallocArray(array, desc, width, height, flags) + * + * cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFree(devPtr) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":299 + * return cudaFree(devPtr) + * + * cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFreeHost(ptr) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeHost(void *__pyx_v_ptr) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":300 + * + * cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaFreeHost(ptr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFreeHost(__pyx_v_ptr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":299 + * return cudaFree(devPtr) + * + * cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFreeHost(ptr) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":302 + * return cudaFreeHost(ptr) + * + * cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFreeArray(array) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeArray(cudaArray_t __pyx_v_array) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":303 + * + * cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaFreeArray(array) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFreeArray(__pyx_v_array); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":302 + * return cudaFreeHost(ptr) + * + * cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFreeArray(array) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":305 + * return cudaFreeArray(array) + * + * cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFreeMipmappedArray(mipmappedArray) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeMipmappedArray(cudaMipmappedArray_t __pyx_v_mipmappedArray) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":306 + * + * cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaFreeMipmappedArray(mipmappedArray) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFreeMipmappedArray(__pyx_v_mipmappedArray); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":305 + * return cudaFreeArray(array) + * + * cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFreeMipmappedArray(mipmappedArray) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":308 + * return cudaFreeMipmappedArray(mipmappedArray) + * + * cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaHostAlloc(pHost, size, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostAlloc(void **__pyx_v_pHost, size_t __pyx_v_size, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":309 + * + * cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaHostAlloc(pHost, size, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaHostAlloc(__pyx_v_pHost, __pyx_v_size, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":308 + * return cudaFreeMipmappedArray(mipmappedArray) + * + * cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaHostAlloc(pHost, size, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":311 + * return cudaHostAlloc(pHost, size, flags) + * + * cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaHostRegister(ptr, size, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostRegister(void *__pyx_v_ptr, size_t __pyx_v_size, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":312 + * + * cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaHostRegister(ptr, size, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaHostRegister(__pyx_v_ptr, __pyx_v_size, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":311 + * return cudaHostAlloc(pHost, size, flags) + * + * cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaHostRegister(ptr, size, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":314 + * return cudaHostRegister(ptr, size, flags) + * + * cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaHostUnregister(ptr) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostUnregister(void *__pyx_v_ptr) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":315 + * + * cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaHostUnregister(ptr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaHostUnregister(__pyx_v_ptr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":314 + * return cudaHostRegister(ptr, size, flags) + * + * cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaHostUnregister(ptr) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":317 + * return cudaHostUnregister(ptr) + * + * cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaHostGetDevicePointer(pDevice, pHost, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostGetDevicePointer(void **__pyx_v_pDevice, void *__pyx_v_pHost, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":318 + * + * cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaHostGetDevicePointer(pDevice, pHost, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaHostGetDevicePointer(__pyx_v_pDevice, __pyx_v_pHost, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":317 + * return cudaHostUnregister(ptr) + * + * cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaHostGetDevicePointer(pDevice, pHost, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":320 + * return cudaHostGetDevicePointer(pDevice, pHost, flags) + * + * cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaHostGetFlags(pFlags, pHost) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostGetFlags(unsigned int *__pyx_v_pFlags, void *__pyx_v_pHost) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":321 + * + * cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaHostGetFlags(pFlags, pHost) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaHostGetFlags(__pyx_v_pFlags, __pyx_v_pHost); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":320 + * return cudaHostGetDevicePointer(pDevice, pHost, flags) + * + * cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaHostGetFlags(pFlags, pHost) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":323 + * return cudaHostGetFlags(pFlags, pHost) + * + * cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMalloc3D(pitchedDevPtr, extent) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc3D(struct cudaPitchedPtr *__pyx_v_pitchedDevPtr, struct cudaExtent __pyx_v_extent) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":324 + * + * cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMalloc3D(pitchedDevPtr, extent) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMalloc3D(__pyx_v_pitchedDevPtr, __pyx_v_extent); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":323 + * return cudaHostGetFlags(pFlags, pHost) + * + * cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMalloc3D(pitchedDevPtr, extent) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":326 + * return cudaMalloc3D(pitchedDevPtr, extent) + * + * cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMalloc3DArray(array, desc, extent, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc3DArray(cudaArray_t *__pyx_v_array, struct cudaChannelFormatDesc const *__pyx_v_desc, struct cudaExtent __pyx_v_extent, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":327 + * + * cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMalloc3DArray(array, desc, extent, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMalloc3DArray(__pyx_v_array, __pyx_v_desc, __pyx_v_extent, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":326 + * return cudaMalloc3D(pitchedDevPtr, extent) + * + * cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMalloc3DArray(array, desc, extent, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":329 + * return cudaMalloc3DArray(array, desc, extent, flags) + * + * cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocMipmappedArray(cudaMipmappedArray_t *__pyx_v_mipmappedArray, struct cudaChannelFormatDesc const *__pyx_v_desc, struct cudaExtent __pyx_v_extent, unsigned int __pyx_v_numLevels, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":330 + * + * cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocMipmappedArray(__pyx_v_mipmappedArray, __pyx_v_desc, __pyx_v_extent, __pyx_v_numLevels, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":329 + * return cudaMalloc3DArray(array, desc, extent, flags) + * + * cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":332 + * return cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) + * + * cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetMipmappedArrayLevel(cudaArray_t *__pyx_v_levelArray, cudaMipmappedArray_const_t __pyx_v_mipmappedArray, unsigned int __pyx_v_level) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":333 + * + * cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetMipmappedArrayLevel(__pyx_v_levelArray, __pyx_v_mipmappedArray, __pyx_v_level); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":332 + * return cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) + * + * cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":335 + * return cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) + * + * cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy3D(p) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3D(struct cudaMemcpy3DParms const *__pyx_v_p) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":336 + * + * cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpy3D(p) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy3D(__pyx_v_p); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":335 + * return cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) + * + * cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy3D(p) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":338 + * return cudaMemcpy3D(p) + * + * cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy3DPeer(p) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DPeer(struct cudaMemcpy3DPeerParms const *__pyx_v_p) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":339 + * + * cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpy3DPeer(p) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy3DPeer(__pyx_v_p); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":338 + * return cudaMemcpy3D(p) + * + * cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy3DPeer(p) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":341 + * return cudaMemcpy3DPeer(p) + * + * cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy3DAsync(p, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DAsync(struct cudaMemcpy3DParms const *__pyx_v_p, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":342 + * + * cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpy3DAsync(p, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy3DAsync(__pyx_v_p, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":341 + * return cudaMemcpy3DPeer(p) + * + * cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy3DAsync(p, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":344 + * return cudaMemcpy3DAsync(p, stream) + * + * cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy3DPeerAsync(p, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DPeerAsync(struct cudaMemcpy3DPeerParms const *__pyx_v_p, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":345 + * + * cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpy3DPeerAsync(p, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy3DPeerAsync(__pyx_v_p, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":344 + * return cudaMemcpy3DAsync(p, stream) + * + * cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy3DPeerAsync(p, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":347 + * return cudaMemcpy3DPeerAsync(p, stream) + * + * cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemGetInfo(free, total) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemGetInfo(size_t *__pyx_v_free, size_t *__pyx_v_total) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":348 + * + * cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemGetInfo(free, total) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemGetInfo(__pyx_v_free, __pyx_v_total); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":347 + * return cudaMemcpy3DPeerAsync(p, stream) + * + * cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemGetInfo(free, total) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":350 + * return cudaMemGetInfo(free, total) + * + * cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaArrayGetInfo(desc, extent, flags, array) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetInfo(struct cudaChannelFormatDesc *__pyx_v_desc, struct cudaExtent *__pyx_v_extent, unsigned int *__pyx_v_flags, cudaArray_t __pyx_v_array) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":351 + * + * cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaArrayGetInfo(desc, extent, flags, array) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaArrayGetInfo(__pyx_v_desc, __pyx_v_extent, __pyx_v_flags, __pyx_v_array); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":350 + * return cudaMemGetInfo(free, total) + * + * cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaArrayGetInfo(desc, extent, flags, array) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":353 + * return cudaArrayGetInfo(desc, extent, flags, array) + * + * cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetPlane(cudaArray_t *__pyx_v_pPlaneArray, cudaArray_t __pyx_v_hArray, unsigned int __pyx_v_planeIdx) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":354 + * + * cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaArrayGetPlane(__pyx_v_pPlaneArray, __pyx_v_hArray, __pyx_v_planeIdx); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":353 + * return cudaArrayGetInfo(desc, extent, flags, array) + * + * cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":356 + * return cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) + * + * cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaArrayGetMemoryRequirements(memoryRequirements, array, device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetMemoryRequirements(struct cudaArrayMemoryRequirements *__pyx_v_memoryRequirements, cudaArray_t __pyx_v_array, int __pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":357 + * + * cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaArrayGetMemoryRequirements(memoryRequirements, array, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaArrayGetMemoryRequirements(__pyx_v_memoryRequirements, __pyx_v_array, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":356 + * return cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) + * + * cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaArrayGetMemoryRequirements(memoryRequirements, array, device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":359 + * return cudaArrayGetMemoryRequirements(memoryRequirements, array, device) + * + * cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMipmappedArrayGetMemoryRequirements(struct cudaArrayMemoryRequirements *__pyx_v_memoryRequirements, cudaMipmappedArray_t __pyx_v_mipmap, int __pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":360 + * + * cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMipmappedArrayGetMemoryRequirements(__pyx_v_memoryRequirements, __pyx_v_mipmap, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":359 + * return cudaArrayGetMemoryRequirements(memoryRequirements, array, device) + * + * cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":362 + * return cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) + * + * cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaArrayGetSparseProperties(sparseProperties, array) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetSparseProperties(struct cudaArraySparseProperties *__pyx_v_sparseProperties, cudaArray_t __pyx_v_array) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":363 + * + * cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaArrayGetSparseProperties(sparseProperties, array) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaArrayGetSparseProperties(__pyx_v_sparseProperties, __pyx_v_array); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":362 + * return cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) + * + * cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaArrayGetSparseProperties(sparseProperties, array) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":365 + * return cudaArrayGetSparseProperties(sparseProperties, array) + * + * cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMipmappedArrayGetSparseProperties(struct cudaArraySparseProperties *__pyx_v_sparseProperties, cudaMipmappedArray_t __pyx_v_mipmap) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":366 + * + * cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMipmappedArrayGetSparseProperties(__pyx_v_sparseProperties, __pyx_v_mipmap); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":365 + * return cudaArrayGetSparseProperties(sparseProperties, array) + * + * cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":368 + * return cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) + * + * cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy(dst, src, count, kind) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy(void *__pyx_v_dst, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":369 + * + * cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpy(dst, src, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy(__pyx_v_dst, __pyx_v_src, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":368 + * return cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) + * + * cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy(dst, src, count, kind) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":371 + * return cudaMemcpy(dst, src, count, kind) + * + * cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyPeer(void *__pyx_v_dst, int __pyx_v_dstDevice, void const *__pyx_v_src, int __pyx_v_srcDevice, size_t __pyx_v_count) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":372 + * + * cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyPeer(__pyx_v_dst, __pyx_v_dstDevice, __pyx_v_src, __pyx_v_srcDevice, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":371 + * return cudaMemcpy(dst, src, count, kind) + * + * cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":374 + * return cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) + * + * cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2D(void *__pyx_v_dst, size_t __pyx_v_dpitch, void const *__pyx_v_src, size_t __pyx_v_spitch, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":375 + * + * cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2D(__pyx_v_dst, __pyx_v_dpitch, __pyx_v_src, __pyx_v_spitch, __pyx_v_width, __pyx_v_height, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":374 + * return cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) + * + * cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":377 + * return cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) + * + * cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DToArray(cudaArray_t __pyx_v_dst, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, void const *__pyx_v_src, size_t __pyx_v_spitch, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":378 + * + * cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2DToArray(__pyx_v_dst, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_src, __pyx_v_spitch, __pyx_v_width, __pyx_v_height, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":377 + * return cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) + * + * cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":380 + * return cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) + * + * cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DFromArray(void *__pyx_v_dst, size_t __pyx_v_dpitch, cudaArray_const_t __pyx_v_src, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":381 + * + * cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2DFromArray(__pyx_v_dst, __pyx_v_dpitch, __pyx_v_src, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_width, __pyx_v_height, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":380 + * return cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) + * + * cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":383 + * return cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) + * + * cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DArrayToArray(cudaArray_t __pyx_v_dst, size_t __pyx_v_wOffsetDst, size_t __pyx_v_hOffsetDst, cudaArray_const_t __pyx_v_src, size_t __pyx_v_wOffsetSrc, size_t __pyx_v_hOffsetSrc, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":384 + * + * cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2DArrayToArray(__pyx_v_dst, __pyx_v_wOffsetDst, __pyx_v_hOffsetDst, __pyx_v_src, __pyx_v_wOffsetSrc, __pyx_v_hOffsetSrc, __pyx_v_width, __pyx_v_height, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":383 + * return cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) + * + * cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":386 + * return cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) + * + * cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyAsync(dst, src, count, kind, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyAsync(void *__pyx_v_dst, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":387 + * + * cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpyAsync(dst, src, count, kind, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyAsync(__pyx_v_dst, __pyx_v_src, __pyx_v_count, __pyx_v_kind, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":386 + * return cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) + * + * cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyAsync(dst, src, count, kind, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":389 + * return cudaMemcpyAsync(dst, src, count, kind, stream) + * + * cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyPeerAsync(void *__pyx_v_dst, int __pyx_v_dstDevice, void const *__pyx_v_src, int __pyx_v_srcDevice, size_t __pyx_v_count, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":390 + * + * cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyPeerAsync(__pyx_v_dst, __pyx_v_dstDevice, __pyx_v_src, __pyx_v_srcDevice, __pyx_v_count, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":389 + * return cudaMemcpyAsync(dst, src, count, kind, stream) + * + * cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":392 + * return cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) + * + * cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyBatchAsync(void **__pyx_v_dsts, void **__pyx_v_srcs, size_t *__pyx_v_sizes, size_t __pyx_v_count, struct cudaMemcpyAttributes *__pyx_v_attrs, size_t *__pyx_v_attrsIdxs, size_t __pyx_v_numAttrs, size_t *__pyx_v_failIdx, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":393 + * + * cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyBatchAsync(__pyx_v_dsts, __pyx_v_srcs, __pyx_v_sizes, __pyx_v_count, __pyx_v_attrs, __pyx_v_attrsIdxs, __pyx_v_numAttrs, __pyx_v_failIdx, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":392 + * return cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) + * + * cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":395 + * return cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) + * + * cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DBatchAsync(size_t __pyx_v_numOps, struct cudaMemcpy3DBatchOp *__pyx_v_opList, size_t *__pyx_v_failIdx, unsigned PY_LONG_LONG __pyx_v_flags, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":396 + * + * cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy3DBatchAsync(__pyx_v_numOps, __pyx_v_opList, __pyx_v_failIdx, __pyx_v_flags, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":395 + * return cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) + * + * cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":398 + * return cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) + * + * cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DAsync(void *__pyx_v_dst, size_t __pyx_v_dpitch, void const *__pyx_v_src, size_t __pyx_v_spitch, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":399 + * + * cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2DAsync(__pyx_v_dst, __pyx_v_dpitch, __pyx_v_src, __pyx_v_spitch, __pyx_v_width, __pyx_v_height, __pyx_v_kind, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":398 + * return cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) + * + * cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":401 + * return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) + * + * cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DToArrayAsync(cudaArray_t __pyx_v_dst, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, void const *__pyx_v_src, size_t __pyx_v_spitch, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":402 + * + * cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2DToArrayAsync(__pyx_v_dst, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_src, __pyx_v_spitch, __pyx_v_width, __pyx_v_height, __pyx_v_kind, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":401 + * return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) + * + * cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":404 + * return cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) + * + * cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DFromArrayAsync(void *__pyx_v_dst, size_t __pyx_v_dpitch, cudaArray_const_t __pyx_v_src, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, size_t __pyx_v_width, size_t __pyx_v_height, enum cudaMemcpyKind __pyx_v_kind, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":405 + * + * cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpy2DFromArrayAsync(__pyx_v_dst, __pyx_v_dpitch, __pyx_v_src, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_width, __pyx_v_height, __pyx_v_kind, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":404 + * return cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) + * + * cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":407 + * return cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) + * + * cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemset(devPtr, value, count) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset(void *__pyx_v_devPtr, int __pyx_v_value, size_t __pyx_v_count) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":408 + * + * cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemset(devPtr, value, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemset(__pyx_v_devPtr, __pyx_v_value, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":407 + * return cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) + * + * cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemset(devPtr, value, count) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":410 + * return cudaMemset(devPtr, value, count) + * + * cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemset2D(devPtr, pitch, value, width, height) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset2D(void *__pyx_v_devPtr, size_t __pyx_v_pitch, int __pyx_v_value, size_t __pyx_v_width, size_t __pyx_v_height) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":411 + * + * cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemset2D(devPtr, pitch, value, width, height) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemset2D(__pyx_v_devPtr, __pyx_v_pitch, __pyx_v_value, __pyx_v_width, __pyx_v_height); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":410 + * return cudaMemset(devPtr, value, count) + * + * cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemset2D(devPtr, pitch, value, width, height) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":413 + * return cudaMemset2D(devPtr, pitch, value, width, height) + * + * cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemset3D(pitchedDevPtr, value, extent) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset3D(struct cudaPitchedPtr __pyx_v_pitchedDevPtr, int __pyx_v_value, struct cudaExtent __pyx_v_extent) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":414 + * + * cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemset3D(pitchedDevPtr, value, extent) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemset3D(__pyx_v_pitchedDevPtr, __pyx_v_value, __pyx_v_extent); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":413 + * return cudaMemset2D(devPtr, pitch, value, width, height) + * + * cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemset3D(pitchedDevPtr, value, extent) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":416 + * return cudaMemset3D(pitchedDevPtr, value, extent) + * + * cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemsetAsync(devPtr, value, count, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemsetAsync(void *__pyx_v_devPtr, int __pyx_v_value, size_t __pyx_v_count, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":417 + * + * cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemsetAsync(devPtr, value, count, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemsetAsync(__pyx_v_devPtr, __pyx_v_value, __pyx_v_count, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":416 + * return cudaMemset3D(pitchedDevPtr, value, extent) + * + * cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemsetAsync(devPtr, value, count, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":419 + * return cudaMemsetAsync(devPtr, value, count, stream) + * + * cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset2DAsync(void *__pyx_v_devPtr, size_t __pyx_v_pitch, int __pyx_v_value, size_t __pyx_v_width, size_t __pyx_v_height, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":420 + * + * cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemset2DAsync(__pyx_v_devPtr, __pyx_v_pitch, __pyx_v_value, __pyx_v_width, __pyx_v_height, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":419 + * return cudaMemsetAsync(devPtr, value, count, stream) + * + * cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":422 + * return cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) + * + * cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset3DAsync(struct cudaPitchedPtr __pyx_v_pitchedDevPtr, int __pyx_v_value, struct cudaExtent __pyx_v_extent, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":423 + * + * cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemset3DAsync(__pyx_v_pitchedDevPtr, __pyx_v_value, __pyx_v_extent, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":422 + * return cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) + * + * cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":425 + * return cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) + * + * cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPrefetchAsync(void const *__pyx_v_devPtr, size_t __pyx_v_count, int __pyx_v_dstDevice, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":426 + * + * cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPrefetchAsync(__pyx_v_devPtr, __pyx_v_count, __pyx_v_dstDevice, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":425 + * return cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) + * + * cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":428 + * return cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) + * + * cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPrefetchAsync_v2(void const *__pyx_v_devPtr, size_t __pyx_v_count, struct cudaMemLocation __pyx_v_location, unsigned int __pyx_v_flags, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":429 + * + * cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPrefetchAsync_v2(__pyx_v_devPtr, __pyx_v_count, __pyx_v_location, __pyx_v_flags, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":428 + * return cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) + * + * cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":431 + * return cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) + * + * cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemAdvise(devPtr, count, advice, device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemAdvise(void const *__pyx_v_devPtr, size_t __pyx_v_count, enum cudaMemoryAdvise __pyx_v_advice, int __pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":432 + * + * cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemAdvise(devPtr, count, advice, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemAdvise(__pyx_v_devPtr, __pyx_v_count, __pyx_v_advice, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":431 + * return cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) + * + * cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemAdvise(devPtr, count, advice, device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":434 + * return cudaMemAdvise(devPtr, count, advice, device) + * + * cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemAdvise_v2(devPtr, count, advice, location) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemAdvise_v2(void const *__pyx_v_devPtr, size_t __pyx_v_count, enum cudaMemoryAdvise __pyx_v_advice, struct cudaMemLocation __pyx_v_location) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":435 + * + * cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemAdvise_v2(devPtr, count, advice, location) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemAdvise_v2(__pyx_v_devPtr, __pyx_v_count, __pyx_v_advice, __pyx_v_location); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":434 + * return cudaMemAdvise(devPtr, count, advice, device) + * + * cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemAdvise_v2(devPtr, count, advice, location) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":437 + * return cudaMemAdvise_v2(devPtr, count, advice, location) + * + * cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemRangeGetAttribute(void *__pyx_v_data, size_t __pyx_v_dataSize, enum cudaMemRangeAttribute __pyx_v_attribute, void const *__pyx_v_devPtr, size_t __pyx_v_count) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":438 + * + * cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemRangeGetAttribute(__pyx_v_data, __pyx_v_dataSize, __pyx_v_attribute, __pyx_v_devPtr, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":437 + * return cudaMemAdvise_v2(devPtr, count, advice, location) + * + * cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":440 + * return cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) + * + * cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemRangeGetAttributes(void **__pyx_v_data, size_t *__pyx_v_dataSizes, enum cudaMemRangeAttribute *__pyx_v_attributes, size_t __pyx_v_numAttributes, void const *__pyx_v_devPtr, size_t __pyx_v_count) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":441 + * + * cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemRangeGetAttributes(__pyx_v_data, __pyx_v_dataSizes, __pyx_v_attributes, __pyx_v_numAttributes, __pyx_v_devPtr, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":440 + * return cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) + * + * cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":443 + * return cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) + * + * cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyToArray(cudaArray_t __pyx_v_dst, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":444 + * + * cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyToArray(__pyx_v_dst, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_src, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":443 + * return cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) + * + * cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":446 + * return cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) + * + * cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyFromArray(void *__pyx_v_dst, cudaArray_const_t __pyx_v_src, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":447 + * + * cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyFromArray(__pyx_v_dst, __pyx_v_src, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":446 + * return cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) + * + * cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":449 + * return cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) + * + * cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyArrayToArray(cudaArray_t __pyx_v_dst, size_t __pyx_v_wOffsetDst, size_t __pyx_v_hOffsetDst, cudaArray_const_t __pyx_v_src, size_t __pyx_v_wOffsetSrc, size_t __pyx_v_hOffsetSrc, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":450 + * + * cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyArrayToArray(__pyx_v_dst, __pyx_v_wOffsetDst, __pyx_v_hOffsetDst, __pyx_v_src, __pyx_v_wOffsetSrc, __pyx_v_hOffsetSrc, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":449 + * return cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) + * + * cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":452 + * return cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) + * + * cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyToArrayAsync(cudaArray_t __pyx_v_dst, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":453 + * + * cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyToArrayAsync(__pyx_v_dst, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_src, __pyx_v_count, __pyx_v_kind, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":452 + * return cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) + * + * cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":455 + * return cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) + * + * cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyFromArrayAsync(void *__pyx_v_dst, cudaArray_const_t __pyx_v_src, size_t __pyx_v_wOffset, size_t __pyx_v_hOffset, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":456 + * + * cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemcpyFromArrayAsync(__pyx_v_dst, __pyx_v_src, __pyx_v_wOffset, __pyx_v_hOffset, __pyx_v_count, __pyx_v_kind, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":455 + * return cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) + * + * cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":458 + * return cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) + * + * cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocAsync(devPtr, size, hStream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocAsync(void **__pyx_v_devPtr, size_t __pyx_v_size, cudaStream_t __pyx_v_hStream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":459 + * + * cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMallocAsync(devPtr, size, hStream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocAsync(__pyx_v_devPtr, __pyx_v_size, __pyx_v_hStream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":458 + * return cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) + * + * cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocAsync(devPtr, size, hStream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":461 + * return cudaMallocAsync(devPtr, size, hStream) + * + * cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFreeAsync(devPtr, hStream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeAsync(void *__pyx_v_devPtr, cudaStream_t __pyx_v_hStream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":462 + * + * cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaFreeAsync(devPtr, hStream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaFreeAsync(__pyx_v_devPtr, __pyx_v_hStream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":461 + * return cudaMallocAsync(devPtr, size, hStream) + * + * cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaFreeAsync(devPtr, hStream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":464 + * return cudaFreeAsync(devPtr, hStream) + * + * cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolTrimTo(memPool, minBytesToKeep) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolTrimTo(cudaMemPool_t __pyx_v_memPool, size_t __pyx_v_minBytesToKeep) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":465 + * + * cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemPoolTrimTo(memPool, minBytesToKeep) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolTrimTo(__pyx_v_memPool, __pyx_v_minBytesToKeep); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":464 + * return cudaFreeAsync(devPtr, hStream) + * + * cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolTrimTo(memPool, minBytesToKeep) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":467 + * return cudaMemPoolTrimTo(memPool, minBytesToKeep) + * + * cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolSetAttribute(memPool, attr, value) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolSetAttribute(cudaMemPool_t __pyx_v_memPool, enum cudaMemPoolAttr __pyx_v_attr, void *__pyx_v_value) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":468 + * + * cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemPoolSetAttribute(memPool, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolSetAttribute(__pyx_v_memPool, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":467 + * return cudaMemPoolTrimTo(memPool, minBytesToKeep) + * + * cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolSetAttribute(memPool, attr, value) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":470 + * return cudaMemPoolSetAttribute(memPool, attr, value) + * + * cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolGetAttribute(memPool, attr, value) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolGetAttribute(cudaMemPool_t __pyx_v_memPool, enum cudaMemPoolAttr __pyx_v_attr, void *__pyx_v_value) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":471 + * + * cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemPoolGetAttribute(memPool, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolGetAttribute(__pyx_v_memPool, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":470 + * return cudaMemPoolSetAttribute(memPool, attr, value) + * + * cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolGetAttribute(memPool, attr, value) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":473 + * return cudaMemPoolGetAttribute(memPool, attr, value) + * + * cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolSetAccess(memPool, descList, count) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolSetAccess(cudaMemPool_t __pyx_v_memPool, struct cudaMemAccessDesc const *__pyx_v_descList, size_t __pyx_v_count) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":474 + * + * cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemPoolSetAccess(memPool, descList, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolSetAccess(__pyx_v_memPool, __pyx_v_descList, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":473 + * return cudaMemPoolGetAttribute(memPool, attr, value) + * + * cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolSetAccess(memPool, descList, count) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":476 + * return cudaMemPoolSetAccess(memPool, descList, count) + * + * cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolGetAccess(flags, memPool, location) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolGetAccess(enum cudaMemAccessFlags *__pyx_v_flags, cudaMemPool_t __pyx_v_memPool, struct cudaMemLocation *__pyx_v_location) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":477 + * + * cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemPoolGetAccess(flags, memPool, location) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolGetAccess(__pyx_v_flags, __pyx_v_memPool, __pyx_v_location); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":476 + * return cudaMemPoolSetAccess(memPool, descList, count) + * + * cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolGetAccess(flags, memPool, location) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":479 + * return cudaMemPoolGetAccess(flags, memPool, location) + * + * cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolCreate(memPool, poolProps) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolCreate(cudaMemPool_t *__pyx_v_memPool, struct cudaMemPoolProps const *__pyx_v_poolProps) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":480 + * + * cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemPoolCreate(memPool, poolProps) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolCreate(__pyx_v_memPool, __pyx_v_poolProps); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":479 + * return cudaMemPoolGetAccess(flags, memPool, location) + * + * cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolCreate(memPool, poolProps) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":482 + * return cudaMemPoolCreate(memPool, poolProps) + * + * cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolDestroy(memPool) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolDestroy(cudaMemPool_t __pyx_v_memPool) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":483 + * + * cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemPoolDestroy(memPool) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolDestroy(__pyx_v_memPool); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":482 + * return cudaMemPoolCreate(memPool, poolProps) + * + * cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolDestroy(memPool) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":485 + * return cudaMemPoolDestroy(memPool) + * + * cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocFromPoolAsync(ptr, size, memPool, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocFromPoolAsync(void **__pyx_v_ptr, size_t __pyx_v_size, cudaMemPool_t __pyx_v_memPool, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":486 + * + * cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMallocFromPoolAsync(ptr, size, memPool, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMallocFromPoolAsync(__pyx_v_ptr, __pyx_v_size, __pyx_v_memPool, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":485 + * return cudaMemPoolDestroy(memPool) + * + * cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMallocFromPoolAsync(ptr, size, memPool, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":488 + * return cudaMallocFromPoolAsync(ptr, size, memPool, stream) + * + * cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolExportToShareableHandle(void *__pyx_v_shareableHandle, cudaMemPool_t __pyx_v_memPool, enum cudaMemAllocationHandleType __pyx_v_handleType, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":489 + * + * cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolExportToShareableHandle(__pyx_v_shareableHandle, __pyx_v_memPool, __pyx_v_handleType, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":488 + * return cudaMallocFromPoolAsync(ptr, size, memPool, stream) + * + * cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":491 + * return cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) + * + * cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolImportFromShareableHandle(cudaMemPool_t *__pyx_v_memPool, void *__pyx_v_shareableHandle, enum cudaMemAllocationHandleType __pyx_v_handleType, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":492 + * + * cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolImportFromShareableHandle(__pyx_v_memPool, __pyx_v_shareableHandle, __pyx_v_handleType, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":491 + * return cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) + * + * cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":494 + * return cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) + * + * cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolExportPointer(exportData, ptr) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolExportPointer(struct cudaMemPoolPtrExportData *__pyx_v_exportData, void *__pyx_v_ptr) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":495 + * + * cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemPoolExportPointer(exportData, ptr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolExportPointer(__pyx_v_exportData, __pyx_v_ptr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":494 + * return cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) + * + * cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolExportPointer(exportData, ptr) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":497 + * return cudaMemPoolExportPointer(exportData, ptr) + * + * cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolImportPointer(ptr, memPool, exportData) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolImportPointer(void **__pyx_v_ptr, cudaMemPool_t __pyx_v_memPool, struct cudaMemPoolPtrExportData *__pyx_v_exportData) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":498 + * + * cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaMemPoolImportPointer(ptr, memPool, exportData) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaMemPoolImportPointer(__pyx_v_ptr, __pyx_v_memPool, __pyx_v_exportData); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":497 + * return cudaMemPoolExportPointer(exportData, ptr) + * + * cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaMemPoolImportPointer(ptr, memPool, exportData) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":500 + * return cudaMemPoolImportPointer(ptr, memPool, exportData) + * + * cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaPointerGetAttributes(attributes, ptr) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaPointerGetAttributes(struct cudaPointerAttributes *__pyx_v_attributes, void const *__pyx_v_ptr) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":501 + * + * cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaPointerGetAttributes(attributes, ptr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaPointerGetAttributes(__pyx_v_attributes, __pyx_v_ptr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":500 + * return cudaMemPoolImportPointer(ptr, memPool, exportData) + * + * cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaPointerGetAttributes(attributes, ptr) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":503 + * return cudaPointerGetAttributes(attributes, ptr) + * + * cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceCanAccessPeer(int *__pyx_v_canAccessPeer, int __pyx_v_device, int __pyx_v_peerDevice) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":504 + * + * cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceCanAccessPeer(__pyx_v_canAccessPeer, __pyx_v_device, __pyx_v_peerDevice); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":503 + * return cudaPointerGetAttributes(attributes, ptr) + * + * cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":506 + * return cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) + * + * cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceEnablePeerAccess(peerDevice, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceEnablePeerAccess(int __pyx_v_peerDevice, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":507 + * + * cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceEnablePeerAccess(peerDevice, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceEnablePeerAccess(__pyx_v_peerDevice, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":506 + * return cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) + * + * cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceEnablePeerAccess(peerDevice, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":509 + * return cudaDeviceEnablePeerAccess(peerDevice, flags) + * + * cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceDisablePeerAccess(peerDevice) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceDisablePeerAccess(int __pyx_v_peerDevice) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":510 + * + * cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceDisablePeerAccess(peerDevice) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceDisablePeerAccess(__pyx_v_peerDevice); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":509 + * return cudaDeviceEnablePeerAccess(peerDevice, flags) + * + * cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceDisablePeerAccess(peerDevice) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":512 + * return cudaDeviceDisablePeerAccess(peerDevice) + * + * cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsUnregisterResource(resource) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsUnregisterResource(cudaGraphicsResource_t __pyx_v_resource) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":513 + * + * cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphicsUnregisterResource(resource) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsUnregisterResource(__pyx_v_resource); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":512 + * return cudaDeviceDisablePeerAccess(peerDevice) + * + * cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsUnregisterResource(resource) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":515 + * return cudaGraphicsUnregisterResource(resource) + * + * cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsResourceSetMapFlags(resource, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t __pyx_v_resource, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":516 + * + * cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphicsResourceSetMapFlags(resource, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsResourceSetMapFlags(__pyx_v_resource, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":515 + * return cudaGraphicsUnregisterResource(resource) + * + * cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsResourceSetMapFlags(resource, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":518 + * return cudaGraphicsResourceSetMapFlags(resource, flags) + * + * cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsMapResources(count, resources, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsMapResources(int __pyx_v_count, cudaGraphicsResource_t *__pyx_v_resources, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":519 + * + * cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphicsMapResources(count, resources, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsMapResources(__pyx_v_count, __pyx_v_resources, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":518 + * return cudaGraphicsResourceSetMapFlags(resource, flags) + * + * cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsMapResources(count, resources, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":521 + * return cudaGraphicsMapResources(count, resources, stream) + * + * cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsUnmapResources(count, resources, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsUnmapResources(int __pyx_v_count, cudaGraphicsResource_t *__pyx_v_resources, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":522 + * + * cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphicsUnmapResources(count, resources, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsUnmapResources(__pyx_v_count, __pyx_v_resources, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":521 + * return cudaGraphicsMapResources(count, resources, stream) + * + * cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsUnmapResources(count, resources, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":524 + * return cudaGraphicsUnmapResources(count, resources, stream) + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceGetMappedPointer(void **__pyx_v_devPtr, size_t *__pyx_v_size, cudaGraphicsResource_t __pyx_v_resource) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":525 + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsResourceGetMappedPointer(__pyx_v_devPtr, __pyx_v_size, __pyx_v_resource); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":524 + * return cudaGraphicsUnmapResources(count, resources, stream) + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":527 + * return cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) + * + * cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsSubResourceGetMappedArray(cudaArray_t *__pyx_v_array, cudaGraphicsResource_t __pyx_v_resource, unsigned int __pyx_v_arrayIndex, unsigned int __pyx_v_mipLevel) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":528 + * + * cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsSubResourceGetMappedArray(__pyx_v_array, __pyx_v_resource, __pyx_v_arrayIndex, __pyx_v_mipLevel); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":527 + * return cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) + * + * cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":530 + * return cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t *__pyx_v_mipmappedArray, cudaGraphicsResource_t __pyx_v_resource) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":531 + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphicsResourceGetMappedMipmappedArray(__pyx_v_mipmappedArray, __pyx_v_resource); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":530 + * return cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) + * + * cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":533 + * return cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) + * + * cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetChannelDesc(desc, array) + * @cython.show_performance_hints(False) +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetChannelDesc(struct cudaChannelFormatDesc *__pyx_v_desc, cudaArray_const_t __pyx_v_array) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":534 + * + * cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetChannelDesc(desc, array) # <<<<<<<<<<<<<< + * @cython.show_performance_hints(False) + * cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil: +*/ + __pyx_r = cudaGetChannelDesc(__pyx_v_desc, __pyx_v_array); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":533 + * return cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) + * + * cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetChannelDesc(desc, array) + * @cython.show_performance_hints(False) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":535 + * cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetChannelDesc(desc, array) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil: + * return cudaCreateChannelDesc(x, y, z, w, f) +*/ + +static struct cudaChannelFormatDesc __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateChannelDesc(int __pyx_v_x, int __pyx_v_y, int __pyx_v_z, int __pyx_v_w, enum cudaChannelFormatKind __pyx_v_f) { + struct cudaChannelFormatDesc __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":537 + * @cython.show_performance_hints(False) + * cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil: + * return cudaCreateChannelDesc(x, y, z, w, f) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaCreateChannelDesc(__pyx_v_x, __pyx_v_y, __pyx_v_z, __pyx_v_w, __pyx_v_f); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":535 + * cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetChannelDesc(desc, array) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil: + * return cudaCreateChannelDesc(x, y, z, w, f) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":539 + * return cudaCreateChannelDesc(x, y, z, w, f) + * + * cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateTextureObject(cudaTextureObject_t *__pyx_v_pTexObject, struct cudaResourceDesc const *__pyx_v_pResDesc, struct cudaTextureDesc const *__pyx_v_pTexDesc, struct cudaResourceViewDesc const *__pyx_v_pResViewDesc) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":540 + * + * cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaCreateTextureObject(__pyx_v_pTexObject, __pyx_v_pResDesc, __pyx_v_pTexDesc, __pyx_v_pResViewDesc); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":539 + * return cudaCreateChannelDesc(x, y, z, w, f) + * + * cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":542 + * return cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) + * + * cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDestroyTextureObject(texObject) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyTextureObject(cudaTextureObject_t __pyx_v_texObject) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":543 + * + * cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDestroyTextureObject(texObject) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDestroyTextureObject(__pyx_v_texObject); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":542 + * return cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) + * + * cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDestroyTextureObject(texObject) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":545 + * return cudaDestroyTextureObject(texObject) + * + * cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetTextureObjectResourceDesc(pResDesc, texObject) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectResourceDesc(struct cudaResourceDesc *__pyx_v_pResDesc, cudaTextureObject_t __pyx_v_texObject) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":546 + * + * cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetTextureObjectResourceDesc(pResDesc, texObject) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetTextureObjectResourceDesc(__pyx_v_pResDesc, __pyx_v_texObject); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":545 + * return cudaDestroyTextureObject(texObject) + * + * cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetTextureObjectResourceDesc(pResDesc, texObject) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":548 + * return cudaGetTextureObjectResourceDesc(pResDesc, texObject) + * + * cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetTextureObjectTextureDesc(pTexDesc, texObject) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectTextureDesc(struct cudaTextureDesc *__pyx_v_pTexDesc, cudaTextureObject_t __pyx_v_texObject) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":549 + * + * cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetTextureObjectTextureDesc(pTexDesc, texObject) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetTextureObjectTextureDesc(__pyx_v_pTexDesc, __pyx_v_texObject); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":548 + * return cudaGetTextureObjectResourceDesc(pResDesc, texObject) + * + * cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetTextureObjectTextureDesc(pTexDesc, texObject) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":551 + * return cudaGetTextureObjectTextureDesc(pTexDesc, texObject) + * + * cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectResourceViewDesc(struct cudaResourceViewDesc *__pyx_v_pResViewDesc, cudaTextureObject_t __pyx_v_texObject) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":552 + * + * cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetTextureObjectResourceViewDesc(__pyx_v_pResViewDesc, __pyx_v_texObject); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":551 + * return cudaGetTextureObjectTextureDesc(pTexDesc, texObject) + * + * cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":554 + * return cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) + * + * cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaCreateSurfaceObject(pSurfObject, pResDesc) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateSurfaceObject(cudaSurfaceObject_t *__pyx_v_pSurfObject, struct cudaResourceDesc const *__pyx_v_pResDesc) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":555 + * + * cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaCreateSurfaceObject(pSurfObject, pResDesc) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaCreateSurfaceObject(__pyx_v_pSurfObject, __pyx_v_pResDesc); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":554 + * return cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) + * + * cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaCreateSurfaceObject(pSurfObject, pResDesc) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":557 + * return cudaCreateSurfaceObject(pSurfObject, pResDesc) + * + * cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDestroySurfaceObject(surfObject) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroySurfaceObject(cudaSurfaceObject_t __pyx_v_surfObject) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":558 + * + * cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDestroySurfaceObject(surfObject) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDestroySurfaceObject(__pyx_v_surfObject); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":557 + * return cudaCreateSurfaceObject(pSurfObject, pResDesc) + * + * cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDestroySurfaceObject(surfObject) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":560 + * return cudaDestroySurfaceObject(surfObject) + * + * cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetSurfaceObjectResourceDesc(struct cudaResourceDesc *__pyx_v_pResDesc, cudaSurfaceObject_t __pyx_v_surfObject) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":561 + * + * cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetSurfaceObjectResourceDesc(__pyx_v_pResDesc, __pyx_v_surfObject); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":560 + * return cudaDestroySurfaceObject(surfObject) + * + * cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":563 + * return cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) + * + * cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDriverGetVersion(driverVersion) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDriverGetVersion(int *__pyx_v_driverVersion) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":564 + * + * cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDriverGetVersion(driverVersion) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDriverGetVersion(__pyx_v_driverVersion); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":563 + * return cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) + * + * cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDriverGetVersion(driverVersion) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":566 + * return cudaDriverGetVersion(driverVersion) + * + * cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaRuntimeGetVersion(runtimeVersion) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaRuntimeGetVersion(int *__pyx_v_runtimeVersion) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":567 + * + * cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaRuntimeGetVersion(runtimeVersion) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaRuntimeGetVersion(__pyx_v_runtimeVersion); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":566 + * return cudaDriverGetVersion(driverVersion) + * + * cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaRuntimeGetVersion(runtimeVersion) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":569 + * return cudaRuntimeGetVersion(runtimeVersion) + * + * cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphCreate(pGraph, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphCreate(cudaGraph_t *__pyx_v_pGraph, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":570 + * + * cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphCreate(pGraph, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphCreate(__pyx_v_pGraph, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":569 + * return cudaRuntimeGetVersion(runtimeVersion) + * + * cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphCreate(pGraph, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":572 + * return cudaGraphCreate(pGraph, flags) + * + * cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddKernelNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaKernelNodeParams const *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":573 + * + * cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddKernelNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":572 + * return cudaGraphCreate(pGraph, flags) + * + * cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":575 + * return cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * + * cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeGetParams(node, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeGetParams(cudaGraphNode_t __pyx_v_node, struct cudaKernelNodeParams *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":576 + * + * cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphKernelNodeGetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphKernelNodeGetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":575 + * return cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * + * cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeGetParams(node, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":578 + * return cudaGraphKernelNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeSetParams(node, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeSetParams(cudaGraphNode_t __pyx_v_node, struct cudaKernelNodeParams const *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":579 + * + * cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphKernelNodeSetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphKernelNodeSetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":578 + * return cudaGraphKernelNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeSetParams(node, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":581 + * return cudaGraphKernelNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeCopyAttributes(hSrc, hDst) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t __pyx_v_hSrc, cudaGraphNode_t __pyx_v_hDst) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":582 + * + * cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphKernelNodeCopyAttributes(hSrc, hDst) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphKernelNodeCopyAttributes(__pyx_v_hSrc, __pyx_v_hDst); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":581 + * return cudaGraphKernelNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeCopyAttributes(hSrc, hDst) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":584 + * return cudaGraphKernelNodeCopyAttributes(hSrc, hDst) + * + * cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeGetAttribute(cudaGraphNode_t __pyx_v_hNode, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrID __pyx_v_attr, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrValue *__pyx_v_value_out) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":585 + * + * cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphKernelNodeGetAttribute(__pyx_v_hNode, __pyx_v_attr, __pyx_v_value_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":584 + * return cudaGraphKernelNodeCopyAttributes(hSrc, hDst) + * + * cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":587 + * return cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) + * + * cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeSetAttribute(hNode, attr, value) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeSetAttribute(cudaGraphNode_t __pyx_v_hNode, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrID __pyx_v_attr, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrValue const *__pyx_v_value) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":588 + * + * cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphKernelNodeSetAttribute(hNode, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphKernelNodeSetAttribute(__pyx_v_hNode, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":587 + * return cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) + * + * cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphKernelNodeSetAttribute(hNode, attr, value) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":590 + * return cudaGraphKernelNodeSetAttribute(hNode, attr, value) + * + * cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemcpyNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaMemcpy3DParms const *__pyx_v_pCopyParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":591 + * + * cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddMemcpyNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_pCopyParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":590 + * return cudaGraphKernelNodeSetAttribute(hNode, attr, value) + * + * cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":593 + * return cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) + * + * cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemcpyNode1D(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, void *__pyx_v_dst, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":594 + * + * cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddMemcpyNode1D(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_dst, __pyx_v_src, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":593 + * return cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) + * + * cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":596 + * return cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) + * + * cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemcpyNodeGetParams(node, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeGetParams(cudaGraphNode_t __pyx_v_node, struct cudaMemcpy3DParms *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":597 + * + * cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphMemcpyNodeGetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemcpyNodeGetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":596 + * return cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) + * + * cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemcpyNodeGetParams(node, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":599 + * return cudaGraphMemcpyNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemcpyNodeSetParams(node, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeSetParams(cudaGraphNode_t __pyx_v_node, struct cudaMemcpy3DParms const *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":600 + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphMemcpyNodeSetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemcpyNodeSetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":599 + * return cudaGraphMemcpyNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemcpyNodeSetParams(node, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":602 + * return cudaGraphMemcpyNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t __pyx_v_node, void *__pyx_v_dst, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":603 + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemcpyNodeSetParams1D(__pyx_v_node, __pyx_v_dst, __pyx_v_src, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":602 + * return cudaGraphMemcpyNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":605 + * return cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) + * + * cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemsetNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaMemsetParams const *__pyx_v_pMemsetParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":606 + * + * cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddMemsetNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_pMemsetParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":605 + * return cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) + * + * cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":608 + * return cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) + * + * cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemsetNodeGetParams(node, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemsetNodeGetParams(cudaGraphNode_t __pyx_v_node, struct cudaMemsetParams *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":609 + * + * cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphMemsetNodeGetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemsetNodeGetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":608 + * return cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) + * + * cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemsetNodeGetParams(node, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":611 + * return cudaGraphMemsetNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemsetNodeSetParams(node, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemsetNodeSetParams(cudaGraphNode_t __pyx_v_node, struct cudaMemsetParams const *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":612 + * + * cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphMemsetNodeSetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemsetNodeSetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":611 + * return cudaGraphMemsetNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemsetNodeSetParams(node, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":614 + * return cudaGraphMemsetNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddHostNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaHostNodeParams const *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":615 + * + * cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddHostNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":614 + * return cudaGraphMemsetNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":617 + * return cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * + * cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphHostNodeGetParams(node, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphHostNodeGetParams(cudaGraphNode_t __pyx_v_node, struct cudaHostNodeParams *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":618 + * + * cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphHostNodeGetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphHostNodeGetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":617 + * return cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + * + * cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphHostNodeGetParams(node, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":620 + * return cudaGraphHostNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphHostNodeSetParams(node, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphHostNodeSetParams(cudaGraphNode_t __pyx_v_node, struct cudaHostNodeParams const *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":621 + * + * cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphHostNodeSetParams(node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphHostNodeSetParams(__pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":620 + * return cudaGraphHostNodeGetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphHostNodeSetParams(node, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":623 + * return cudaGraphHostNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddChildGraphNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, cudaGraph_t __pyx_v_childGraph) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":624 + * + * cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddChildGraphNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_childGraph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":623 + * return cudaGraphHostNodeSetParams(node, pNodeParams) + * + * cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":626 + * return cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) + * + * cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphChildGraphNodeGetGraph(node, pGraph) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t __pyx_v_node, cudaGraph_t *__pyx_v_pGraph) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":627 + * + * cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphChildGraphNodeGetGraph(node, pGraph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphChildGraphNodeGetGraph(__pyx_v_node, __pyx_v_pGraph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":626 + * return cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) + * + * cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphChildGraphNodeGetGraph(node, pGraph) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":629 + * return cudaGraphChildGraphNodeGetGraph(node, pGraph) + * + * cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEmptyNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":630 + * + * cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddEmptyNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":629 + * return cudaGraphChildGraphNodeGetGraph(node, pGraph) + * + * cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":632 + * return cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) + * + * cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEventRecordNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, cudaEvent_t __pyx_v_event) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":633 + * + * cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddEventRecordNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":632 + * return cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) + * + * cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":635 + * return cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) + * + * cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphEventRecordNodeGetEvent(node, event_out) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t __pyx_v_node, cudaEvent_t *__pyx_v_event_out) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":636 + * + * cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphEventRecordNodeGetEvent(node, event_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphEventRecordNodeGetEvent(__pyx_v_node, __pyx_v_event_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":635 + * return cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) + * + * cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphEventRecordNodeGetEvent(node, event_out) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":638 + * return cudaGraphEventRecordNodeGetEvent(node, event_out) + * + * cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphEventRecordNodeSetEvent(node, event) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t __pyx_v_node, cudaEvent_t __pyx_v_event) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":639 + * + * cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphEventRecordNodeSetEvent(node, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphEventRecordNodeSetEvent(__pyx_v_node, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":638 + * return cudaGraphEventRecordNodeGetEvent(node, event_out) + * + * cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphEventRecordNodeSetEvent(node, event) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":641 + * return cudaGraphEventRecordNodeSetEvent(node, event) + * + * cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEventWaitNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, cudaEvent_t __pyx_v_event) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":642 + * + * cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddEventWaitNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":641 + * return cudaGraphEventRecordNodeSetEvent(node, event) + * + * cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":644 + * return cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) + * + * cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphEventWaitNodeGetEvent(node, event_out) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t __pyx_v_node, cudaEvent_t *__pyx_v_event_out) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":645 + * + * cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphEventWaitNodeGetEvent(node, event_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphEventWaitNodeGetEvent(__pyx_v_node, __pyx_v_event_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":644 + * return cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) + * + * cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphEventWaitNodeGetEvent(node, event_out) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":647 + * return cudaGraphEventWaitNodeGetEvent(node, event_out) + * + * cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphEventWaitNodeSetEvent(node, event) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t __pyx_v_node, cudaEvent_t __pyx_v_event) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":648 + * + * cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphEventWaitNodeSetEvent(node, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphEventWaitNodeSetEvent(__pyx_v_node, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":647 + * return cudaGraphEventWaitNodeGetEvent(node, event_out) + * + * cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphEventWaitNodeSetEvent(node, event) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":650 + * return cudaGraphEventWaitNodeSetEvent(node, event) + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaExternalSemaphoreSignalNodeParams const *__pyx_v_nodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":651 + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddExternalSemaphoresSignalNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":650 + * return cudaGraphEventWaitNodeSetEvent(node, event) + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":653 + * return cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t __pyx_v_hNode, struct cudaExternalSemaphoreSignalNodeParams *__pyx_v_params_out) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":654 + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExternalSemaphoresSignalNodeGetParams(__pyx_v_hNode, __pyx_v_params_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":653 + * return cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":656 + * return cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t __pyx_v_hNode, struct cudaExternalSemaphoreSignalNodeParams const *__pyx_v_nodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":657 + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExternalSemaphoresSignalNodeSetParams(__pyx_v_hNode, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":656 + * return cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":659 + * return cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaExternalSemaphoreWaitNodeParams const *__pyx_v_nodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":660 + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddExternalSemaphoresWaitNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":659 + * return cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":662 + * return cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t __pyx_v_hNode, struct cudaExternalSemaphoreWaitNodeParams *__pyx_v_params_out) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":663 + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExternalSemaphoresWaitNodeGetParams(__pyx_v_hNode, __pyx_v_params_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":662 + * return cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":665 + * return cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t __pyx_v_hNode, struct cudaExternalSemaphoreWaitNodeParams const *__pyx_v_nodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":666 + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExternalSemaphoresWaitNodeSetParams(__pyx_v_hNode, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":665 + * return cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) + * + * cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":668 + * return cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemAllocNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaMemAllocNodeParams *__pyx_v_nodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":669 + * + * cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddMemAllocNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":668 + * return cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":671 + * return cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemAllocNodeGetParams(node, params_out) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemAllocNodeGetParams(cudaGraphNode_t __pyx_v_node, struct cudaMemAllocNodeParams *__pyx_v_params_out) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":672 + * + * cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphMemAllocNodeGetParams(node, params_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemAllocNodeGetParams(__pyx_v_node, __pyx_v_params_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":671 + * return cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemAllocNodeGetParams(node, params_out) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":674 + * return cudaGraphMemAllocNodeGetParams(node, params_out) + * + * cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemFreeNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, void *__pyx_v_dptr) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":675 + * + * cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddMemFreeNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_dptr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":674 + * return cudaGraphMemAllocNodeGetParams(node, params_out) + * + * cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":677 + * return cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) + * + * cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemFreeNodeGetParams(node, dptr_out) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemFreeNodeGetParams(cudaGraphNode_t __pyx_v_node, void *__pyx_v_dptr_out) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":678 + * + * cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphMemFreeNodeGetParams(node, dptr_out) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphMemFreeNodeGetParams(__pyx_v_node, __pyx_v_dptr_out); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":677 + * return cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) + * + * cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphMemFreeNodeGetParams(node, dptr_out) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":680 + * return cudaGraphMemFreeNodeGetParams(node, dptr_out) + * + * cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGraphMemTrim(device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGraphMemTrim(int __pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":681 + * + * cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGraphMemTrim(device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGraphMemTrim(__pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":680 + * return cudaGraphMemFreeNodeGetParams(node, dptr_out) + * + * cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGraphMemTrim(device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":683 + * return cudaDeviceGraphMemTrim(device) + * + * cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetGraphMemAttribute(device, attr, value) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetGraphMemAttribute(int __pyx_v_device, enum cudaGraphMemAttributeType __pyx_v_attr, void *__pyx_v_value) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":684 + * + * cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceGetGraphMemAttribute(device, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceGetGraphMemAttribute(__pyx_v_device, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":683 + * return cudaDeviceGraphMemTrim(device) + * + * cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceGetGraphMemAttribute(device, attr, value) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":686 + * return cudaDeviceGetGraphMemAttribute(device, attr, value) + * + * cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceSetGraphMemAttribute(device, attr, value) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetGraphMemAttribute(int __pyx_v_device, enum cudaGraphMemAttributeType __pyx_v_attr, void *__pyx_v_value) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":687 + * + * cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaDeviceSetGraphMemAttribute(device, attr, value) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaDeviceSetGraphMemAttribute(__pyx_v_device, __pyx_v_attr, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":686 + * return cudaDeviceGetGraphMemAttribute(device, attr, value) + * + * cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaDeviceSetGraphMemAttribute(device, attr, value) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":689 + * return cudaDeviceSetGraphMemAttribute(device, attr, value) + * + * cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphClone(pGraphClone, originalGraph) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphClone(cudaGraph_t *__pyx_v_pGraphClone, cudaGraph_t __pyx_v_originalGraph) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":690 + * + * cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphClone(pGraphClone, originalGraph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphClone(__pyx_v_pGraphClone, __pyx_v_originalGraph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":689 + * return cudaDeviceSetGraphMemAttribute(device, attr, value) + * + * cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphClone(pGraphClone, originalGraph) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":692 + * return cudaGraphClone(pGraphClone, originalGraph) + * + * cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeFindInClone(cudaGraphNode_t *__pyx_v_pNode, cudaGraphNode_t __pyx_v_originalNode, cudaGraph_t __pyx_v_clonedGraph) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":693 + * + * cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeFindInClone(__pyx_v_pNode, __pyx_v_originalNode, __pyx_v_clonedGraph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":692 + * return cudaGraphClone(pGraphClone, originalGraph) + * + * cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":695 + * return cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) + * + * cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeGetType(node, pType) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetType(cudaGraphNode_t __pyx_v_node, enum cudaGraphNodeType *__pyx_v_pType) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":696 + * + * cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphNodeGetType(node, pType) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeGetType(__pyx_v_node, __pyx_v_pType); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":695 + * return cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) + * + * cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeGetType(node, pType) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":698 + * return cudaGraphNodeGetType(node, pType) + * + * cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphGetNodes(graph, nodes, numNodes) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetNodes(cudaGraph_t __pyx_v_graph, cudaGraphNode_t *__pyx_v_nodes, size_t *__pyx_v_numNodes) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":699 + * + * cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphGetNodes(graph, nodes, numNodes) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphGetNodes(__pyx_v_graph, __pyx_v_nodes, __pyx_v_numNodes); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":698 + * return cudaGraphNodeGetType(node, pType) + * + * cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphGetNodes(graph, nodes, numNodes) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":701 + * return cudaGraphGetNodes(graph, nodes, numNodes) + * + * cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetRootNodes(cudaGraph_t __pyx_v_graph, cudaGraphNode_t *__pyx_v_pRootNodes, size_t *__pyx_v_pNumRootNodes) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":702 + * + * cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphGetRootNodes(__pyx_v_graph, __pyx_v_pRootNodes, __pyx_v_pNumRootNodes); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":701 + * return cudaGraphGetNodes(graph, nodes, numNodes) + * + * cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":704 + * return cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) + * + * cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphGetEdges(graph, from_, to, numEdges) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetEdges(cudaGraph_t __pyx_v_graph, cudaGraphNode_t *__pyx_v_from_, cudaGraphNode_t *__pyx_v_to, size_t *__pyx_v_numEdges) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":705 + * + * cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphGetEdges(graph, from_, to, numEdges) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphGetEdges(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_numEdges); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":704 + * return cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) + * + * cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphGetEdges(graph, from_, to, numEdges) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":707 + * return cudaGraphGetEdges(graph, from_, to, numEdges) + * + * cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetEdges_v2(cudaGraph_t __pyx_v_graph, cudaGraphNode_t *__pyx_v_from_, cudaGraphNode_t *__pyx_v_to, cudaGraphEdgeData *__pyx_v_edgeData, size_t *__pyx_v_numEdges) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":708 + * + * cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphGetEdges_v2(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_edgeData, __pyx_v_numEdges); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":707 + * return cudaGraphGetEdges(graph, from_, to, numEdges) + * + * cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":710 + * return cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) + * + * cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependencies(cudaGraphNode_t __pyx_v_node, cudaGraphNode_t *__pyx_v_pDependencies, size_t *__pyx_v_pNumDependencies) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":711 + * + * cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeGetDependencies(__pyx_v_node, __pyx_v_pDependencies, __pyx_v_pNumDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":710 + * return cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) + * + * cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":713 + * return cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) + * + * cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependencies_v2(cudaGraphNode_t __pyx_v_node, cudaGraphNode_t *__pyx_v_pDependencies, cudaGraphEdgeData *__pyx_v_edgeData, size_t *__pyx_v_pNumDependencies) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":714 + * + * cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeGetDependencies_v2(__pyx_v_node, __pyx_v_pDependencies, __pyx_v_edgeData, __pyx_v_pNumDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":713 + * return cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) + * + * cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":716 + * return cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependentNodes(cudaGraphNode_t __pyx_v_node, cudaGraphNode_t *__pyx_v_pDependentNodes, size_t *__pyx_v_pNumDependentNodes) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":717 + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeGetDependentNodes(__pyx_v_node, __pyx_v_pDependentNodes, __pyx_v_pNumDependentNodes); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":716 + * return cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":719 + * return cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t __pyx_v_node, cudaGraphNode_t *__pyx_v_pDependentNodes, cudaGraphEdgeData *__pyx_v_edgeData, size_t *__pyx_v_pNumDependentNodes) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":720 + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeGetDependentNodes_v2(__pyx_v_node, __pyx_v_pDependentNodes, __pyx_v_edgeData, __pyx_v_pNumDependentNodes); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":719 + * return cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) + * + * cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":722 + * return cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) + * + * cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddDependencies(graph, from_, to, numDependencies) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddDependencies(cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_from_, cudaGraphNode_t const *__pyx_v_to, size_t __pyx_v_numDependencies) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":723 + * + * cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddDependencies(graph, from_, to, numDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddDependencies(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_numDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":722 + * return cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) + * + * cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddDependencies(graph, from_, to, numDependencies) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":725 + * return cudaGraphAddDependencies(graph, from_, to, numDependencies) + * + * cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddDependencies_v2(cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_from_, cudaGraphNode_t const *__pyx_v_to, cudaGraphEdgeData const *__pyx_v_edgeData, size_t __pyx_v_numDependencies) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":726 + * + * cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddDependencies_v2(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_edgeData, __pyx_v_numDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":725 + * return cudaGraphAddDependencies(graph, from_, to, numDependencies) + * + * cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":728 + * return cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) + * + * cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphRemoveDependencies(graph, from_, to, numDependencies) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRemoveDependencies(cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_from_, cudaGraphNode_t const *__pyx_v_to, size_t __pyx_v_numDependencies) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":729 + * + * cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphRemoveDependencies(graph, from_, to, numDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphRemoveDependencies(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_numDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":728 + * return cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) + * + * cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphRemoveDependencies(graph, from_, to, numDependencies) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":731 + * return cudaGraphRemoveDependencies(graph, from_, to, numDependencies) + * + * cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRemoveDependencies_v2(cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_from_, cudaGraphNode_t const *__pyx_v_to, cudaGraphEdgeData const *__pyx_v_edgeData, size_t __pyx_v_numDependencies) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":732 + * + * cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphRemoveDependencies_v2(__pyx_v_graph, __pyx_v_from_, __pyx_v_to, __pyx_v_edgeData, __pyx_v_numDependencies); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":731 + * return cudaGraphRemoveDependencies(graph, from_, to, numDependencies) + * + * cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":734 + * return cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) + * + * cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphDestroyNode(node) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDestroyNode(cudaGraphNode_t __pyx_v_node) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":735 + * + * cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphDestroyNode(node) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphDestroyNode(__pyx_v_node); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":734 + * return cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) + * + * cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphDestroyNode(node) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":737 + * return cudaGraphDestroyNode(node) + * + * cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphInstantiate(pGraphExec, graph, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiate(cudaGraphExec_t *__pyx_v_pGraphExec, cudaGraph_t __pyx_v_graph, unsigned PY_LONG_LONG __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":738 + * + * cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphInstantiate(pGraphExec, graph, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphInstantiate(__pyx_v_pGraphExec, __pyx_v_graph, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":737 + * return cudaGraphDestroyNode(node) + * + * cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphInstantiate(pGraphExec, graph, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":740 + * return cudaGraphInstantiate(pGraphExec, graph, flags) + * + * cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiateWithFlags(cudaGraphExec_t *__pyx_v_pGraphExec, cudaGraph_t __pyx_v_graph, unsigned PY_LONG_LONG __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":741 + * + * cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphInstantiateWithFlags(__pyx_v_pGraphExec, __pyx_v_graph, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":740 + * return cudaGraphInstantiate(pGraphExec, graph, flags) + * + * cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":743 + * return cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) + * + * cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiateWithParams(cudaGraphExec_t *__pyx_v_pGraphExec, cudaGraph_t __pyx_v_graph, cudaGraphInstantiateParams *__pyx_v_instantiateParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":744 + * + * cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphInstantiateWithParams(__pyx_v_pGraphExec, __pyx_v_graph, __pyx_v_instantiateParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":743 + * return cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) + * + * cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":746 + * return cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) + * + * cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecGetFlags(graphExec, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecGetFlags(cudaGraphExec_t __pyx_v_graphExec, unsigned PY_LONG_LONG *__pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":747 + * + * cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecGetFlags(graphExec, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecGetFlags(__pyx_v_graphExec, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":746 + * return cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) + * + * cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecGetFlags(graphExec, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":749 + * return cudaGraphExecGetFlags(graphExec, flags) + * + * cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecKernelNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_node, struct cudaKernelNodeParams const *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":750 + * + * cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecKernelNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":749 + * return cudaGraphExecGetFlags(graphExec, flags) + * + * cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":752 + * return cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_node, struct cudaMemcpy3DParms const *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":753 + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecMemcpyNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":752 + * return cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":755 + * return cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_node, void *__pyx_v_dst, void const *__pyx_v_src, size_t __pyx_v_count, enum cudaMemcpyKind __pyx_v_kind) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":756 + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecMemcpyNodeSetParams1D(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_dst, __pyx_v_src, __pyx_v_count, __pyx_v_kind); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":755 + * return cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":758 + * return cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) + * + * cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_node, struct cudaMemsetParams const *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":759 + * + * cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecMemsetNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":758 + * return cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) + * + * cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":761 + * return cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecHostNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_node, struct cudaHostNodeParams const *__pyx_v_pNodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":762 + * + * cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecHostNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_pNodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":761 + * return cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":764 + * return cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_node, cudaGraph_t __pyx_v_childGraph) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":765 + * + * cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecChildGraphNodeSetParams(__pyx_v_hGraphExec, __pyx_v_node, __pyx_v_childGraph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":764 + * return cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) + * + * cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":767 + * return cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) + * + * cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_hNode, cudaEvent_t __pyx_v_event) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":768 + * + * cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecEventRecordNodeSetEvent(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":767 + * return cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) + * + * cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":770 + * return cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) + * + * cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_hNode, cudaEvent_t __pyx_v_event) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":771 + * + * cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecEventWaitNodeSetEvent(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_event); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":770 + * return cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) + * + * cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":773 + * return cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_hNode, struct cudaExternalSemaphoreSignalNodeParams const *__pyx_v_nodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":774 + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecExternalSemaphoresSignalNodeSetParams(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":773 + * return cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":776 + * return cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_hNode, struct cudaExternalSemaphoreWaitNodeParams const *__pyx_v_nodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":777 + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecExternalSemaphoresWaitNodeSetParams(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":776 + * return cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":779 + * return cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeSetEnabled(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_hNode, unsigned int __pyx_v_isEnabled) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":780 + * + * cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeSetEnabled(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_isEnabled); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":779 + * return cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) + * + * cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":782 + * return cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) + * + * cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetEnabled(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraphNode_t __pyx_v_hNode, unsigned int *__pyx_v_isEnabled) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":783 + * + * cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeGetEnabled(__pyx_v_hGraphExec, __pyx_v_hNode, __pyx_v_isEnabled); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":782 + * return cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) + * + * cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":785 + * return cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) + * + * cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecUpdate(cudaGraphExec_t __pyx_v_hGraphExec, cudaGraph_t __pyx_v_hGraph, cudaGraphExecUpdateResultInfo *__pyx_v_resultInfo) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":786 + * + * cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecUpdate(__pyx_v_hGraphExec, __pyx_v_hGraph, __pyx_v_resultInfo); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":785 + * return cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) + * + * cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":788 + * return cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) + * + * cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphUpload(graphExec, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphUpload(cudaGraphExec_t __pyx_v_graphExec, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":789 + * + * cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphUpload(graphExec, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphUpload(__pyx_v_graphExec, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":788 + * return cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) + * + * cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphUpload(graphExec, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":791 + * return cudaGraphUpload(graphExec, stream) + * + * cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphLaunch(graphExec, stream) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphLaunch(cudaGraphExec_t __pyx_v_graphExec, cudaStream_t __pyx_v_stream) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":792 + * + * cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphLaunch(graphExec, stream) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphLaunch(__pyx_v_graphExec, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":791 + * return cudaGraphUpload(graphExec, stream) + * + * cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphLaunch(graphExec, stream) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":794 + * return cudaGraphLaunch(graphExec, stream) + * + * cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecDestroy(graphExec) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecDestroy(cudaGraphExec_t __pyx_v_graphExec) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":795 + * + * cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecDestroy(graphExec) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecDestroy(__pyx_v_graphExec); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":794 + * return cudaGraphLaunch(graphExec, stream) + * + * cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecDestroy(graphExec) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":797 + * return cudaGraphExecDestroy(graphExec) + * + * cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphDestroy(graph) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDestroy(cudaGraph_t __pyx_v_graph) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":798 + * + * cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphDestroy(graph) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphDestroy(__pyx_v_graph); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":797 + * return cudaGraphExecDestroy(graphExec) + * + * cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphDestroy(graph) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":800 + * return cudaGraphDestroy(graph) + * + * cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphDebugDotPrint(graph, path, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDebugDotPrint(cudaGraph_t __pyx_v_graph, char const *__pyx_v_path, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":801 + * + * cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphDebugDotPrint(graph, path, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphDebugDotPrint(__pyx_v_graph, __pyx_v_path, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":800 + * return cudaGraphDestroy(graph) + * + * cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphDebugDotPrint(graph, path, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":803 + * return cudaGraphDebugDotPrint(graph, path, flags) + * + * cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectCreate(cudaUserObject_t *__pyx_v_object_out, void *__pyx_v_ptr, cudaHostFn_t __pyx_v_destroy, unsigned int __pyx_v_initialRefcount, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":804 + * + * cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaUserObjectCreate(__pyx_v_object_out, __pyx_v_ptr, __pyx_v_destroy, __pyx_v_initialRefcount, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":803 + * return cudaGraphDebugDotPrint(graph, path, flags) + * + * cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":806 + * return cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) + * + * cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaUserObjectRetain(object, count) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectRetain(cudaUserObject_t __pyx_v_object, unsigned int __pyx_v_count) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":807 + * + * cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaUserObjectRetain(object, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaUserObjectRetain(__pyx_v_object, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":806 + * return cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) + * + * cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaUserObjectRetain(object, count) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":809 + * return cudaUserObjectRetain(object, count) + * + * cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaUserObjectRelease(object, count) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectRelease(cudaUserObject_t __pyx_v_object, unsigned int __pyx_v_count) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":810 + * + * cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaUserObjectRelease(object, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaUserObjectRelease(__pyx_v_object, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":809 + * return cudaUserObjectRetain(object, count) + * + * cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaUserObjectRelease(object, count) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":812 + * return cudaUserObjectRelease(object, count) + * + * cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphRetainUserObject(graph, object, count, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRetainUserObject(cudaGraph_t __pyx_v_graph, cudaUserObject_t __pyx_v_object, unsigned int __pyx_v_count, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":813 + * + * cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphRetainUserObject(graph, object, count, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphRetainUserObject(__pyx_v_graph, __pyx_v_object, __pyx_v_count, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":812 + * return cudaUserObjectRelease(object, count) + * + * cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphRetainUserObject(graph, object, count, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":815 + * return cudaGraphRetainUserObject(graph, object, count, flags) + * + * cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphReleaseUserObject(graph, object, count) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphReleaseUserObject(cudaGraph_t __pyx_v_graph, cudaUserObject_t __pyx_v_object, unsigned int __pyx_v_count) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":816 + * + * cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphReleaseUserObject(graph, object, count) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphReleaseUserObject(__pyx_v_graph, __pyx_v_object, __pyx_v_count); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":815 + * return cudaGraphRetainUserObject(graph, object, count, flags) + * + * cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphReleaseUserObject(graph, object, count) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":818 + * return cudaGraphReleaseUserObject(graph, object, count) + * + * cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddNode(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, size_t __pyx_v_numDependencies, struct cudaGraphNodeParams *__pyx_v_nodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":819 + * + * cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddNode(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_numDependencies, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":818 + * return cudaGraphReleaseUserObject(graph, object, count) + * + * cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":821 + * return cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddNode_v2(cudaGraphNode_t *__pyx_v_pGraphNode, cudaGraph_t __pyx_v_graph, cudaGraphNode_t const *__pyx_v_pDependencies, cudaGraphEdgeData const *__pyx_v_dependencyData, size_t __pyx_v_numDependencies, struct cudaGraphNodeParams *__pyx_v_nodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":822 + * + * cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphAddNode_v2(__pyx_v_pGraphNode, __pyx_v_graph, __pyx_v_pDependencies, __pyx_v_dependencyData, __pyx_v_numDependencies, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":821 + * return cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":824 + * return cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeSetParams(node, nodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeSetParams(cudaGraphNode_t __pyx_v_node, struct cudaGraphNodeParams *__pyx_v_nodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":825 + * + * cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphNodeSetParams(node, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphNodeSetParams(__pyx_v_node, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":824 + * return cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) + * + * cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphNodeSetParams(node, nodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":827 + * return cudaGraphNodeSetParams(node, nodeParams) + * + * cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecNodeSetParams(graphExec, node, nodeParams) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecNodeSetParams(cudaGraphExec_t __pyx_v_graphExec, cudaGraphNode_t __pyx_v_node, struct cudaGraphNodeParams *__pyx_v_nodeParams) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":828 + * + * cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphExecNodeSetParams(graphExec, node, nodeParams) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphExecNodeSetParams(__pyx_v_graphExec, __pyx_v_node, __pyx_v_nodeParams); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":827 + * return cudaGraphNodeSetParams(node, nodeParams) + * + * cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphExecNodeSetParams(graphExec, node, nodeParams) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":830 + * return cudaGraphExecNodeSetParams(graphExec, node, nodeParams) + * + * cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle *__pyx_v_pHandle_out, cudaGraph_t __pyx_v_graph, unsigned int __pyx_v_defaultLaunchValue, unsigned int __pyx_v_flags) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":831 + * + * cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGraphConditionalHandleCreate(__pyx_v_pHandle_out, __pyx_v_graph, __pyx_v_defaultLaunchValue, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":830 + * return cudaGraphExecNodeSetParams(graphExec, node, nodeParams) + * + * cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":833 + * return cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) + * + * cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDriverEntryPoint(char const *__pyx_v_symbol, void **__pyx_v_funcPtr, unsigned PY_LONG_LONG __pyx_v_flags, enum cudaDriverEntryPointQueryResult *__pyx_v_driverStatus) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":834 + * + * cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetDriverEntryPoint(__pyx_v_symbol, __pyx_v_funcPtr, __pyx_v_flags, __pyx_v_driverStatus); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":833 + * return cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) + * + * cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":836 + * return cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) + * + * cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDriverEntryPointByVersion(char const *__pyx_v_symbol, void **__pyx_v_funcPtr, unsigned int __pyx_v_cudaVersion, unsigned PY_LONG_LONG __pyx_v_flags, enum cudaDriverEntryPointQueryResult *__pyx_v_driverStatus) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":837 + * + * cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetDriverEntryPointByVersion(__pyx_v_symbol, __pyx_v_funcPtr, __pyx_v_cudaVersion, __pyx_v_flags, __pyx_v_driverStatus); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":836 + * return cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) + * + * cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":839 + * return cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) + * + * cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryLoadData(cudaLibrary_t *__pyx_v_library, void const *__pyx_v_code, enum cudaJitOption *__pyx_v_jitOptions, void **__pyx_v_jitOptionsValues, unsigned int __pyx_v_numJitOptions, enum cudaLibraryOption *__pyx_v_libraryOptions, void **__pyx_v_libraryOptionValues, unsigned int __pyx_v_numLibraryOptions) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":840 + * + * cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryLoadData(__pyx_v_library, __pyx_v_code, __pyx_v_jitOptions, __pyx_v_jitOptionsValues, __pyx_v_numJitOptions, __pyx_v_libraryOptions, __pyx_v_libraryOptionValues, __pyx_v_numLibraryOptions); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":839 + * return cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) + * + * cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":842 + * return cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * + * cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryLoadFromFile(cudaLibrary_t *__pyx_v_library, char const *__pyx_v_fileName, enum cudaJitOption *__pyx_v_jitOptions, void **__pyx_v_jitOptionsValues, unsigned int __pyx_v_numJitOptions, enum cudaLibraryOption *__pyx_v_libraryOptions, void **__pyx_v_libraryOptionValues, unsigned int __pyx_v_numLibraryOptions) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":843 + * + * cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryLoadFromFile(__pyx_v_library, __pyx_v_fileName, __pyx_v_jitOptions, __pyx_v_jitOptionsValues, __pyx_v_numJitOptions, __pyx_v_libraryOptions, __pyx_v_libraryOptionValues, __pyx_v_numLibraryOptions); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":842 + * return cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * + * cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":845 + * return cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * + * cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryUnload(library) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryUnload(cudaLibrary_t __pyx_v_library) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":846 + * + * cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaLibraryUnload(library) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryUnload(__pyx_v_library); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":845 + * return cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + * + * cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryUnload(library) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":848 + * return cudaLibraryUnload(library) + * + * cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryGetKernel(pKernel, library, name) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetKernel(cudaKernel_t *__pyx_v_pKernel, cudaLibrary_t __pyx_v_library, char const *__pyx_v_name) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":849 + * + * cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaLibraryGetKernel(pKernel, library, name) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryGetKernel(__pyx_v_pKernel, __pyx_v_library, __pyx_v_name); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":848 + * return cudaLibraryUnload(library) + * + * cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryGetKernel(pKernel, library, name) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":851 + * return cudaLibraryGetKernel(pKernel, library, name) + * + * cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryGetGlobal(dptr, numbytes, library, name) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetGlobal(void **__pyx_v_dptr, size_t *__pyx_v_numbytes, cudaLibrary_t __pyx_v_library, char const *__pyx_v_name) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":852 + * + * cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaLibraryGetGlobal(dptr, numbytes, library, name) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryGetGlobal(__pyx_v_dptr, __pyx_v_numbytes, __pyx_v_library, __pyx_v_name); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":851 + * return cudaLibraryGetKernel(pKernel, library, name) + * + * cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryGetGlobal(dptr, numbytes, library, name) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":854 + * return cudaLibraryGetGlobal(dptr, numbytes, library, name) + * + * cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryGetManaged(dptr, numbytes, library, name) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetManaged(void **__pyx_v_dptr, size_t *__pyx_v_numbytes, cudaLibrary_t __pyx_v_library, char const *__pyx_v_name) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":855 + * + * cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaLibraryGetManaged(dptr, numbytes, library, name) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryGetManaged(__pyx_v_dptr, __pyx_v_numbytes, __pyx_v_library, __pyx_v_name); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":854 + * return cudaLibraryGetGlobal(dptr, numbytes, library, name) + * + * cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryGetManaged(dptr, numbytes, library, name) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":857 + * return cudaLibraryGetManaged(dptr, numbytes, library, name) + * + * cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryGetUnifiedFunction(fptr, library, symbol) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetUnifiedFunction(void **__pyx_v_fptr, cudaLibrary_t __pyx_v_library, char const *__pyx_v_symbol) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":858 + * + * cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaLibraryGetUnifiedFunction(fptr, library, symbol) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryGetUnifiedFunction(__pyx_v_fptr, __pyx_v_library, __pyx_v_symbol); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":857 + * return cudaLibraryGetManaged(dptr, numbytes, library, name) + * + * cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryGetUnifiedFunction(fptr, library, symbol) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":860 + * return cudaLibraryGetUnifiedFunction(fptr, library, symbol) + * + * cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryGetKernelCount(count, lib) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetKernelCount(unsigned int *__pyx_v_count, cudaLibrary_t __pyx_v_lib) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":861 + * + * cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaLibraryGetKernelCount(count, lib) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryGetKernelCount(__pyx_v_count, __pyx_v_lib); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":860 + * return cudaLibraryGetUnifiedFunction(fptr, library, symbol) + * + * cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryGetKernelCount(count, lib) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":863 + * return cudaLibraryGetKernelCount(count, lib) + * + * cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryEnumerateKernels(kernels, numKernels, lib) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryEnumerateKernels(cudaKernel_t *__pyx_v_kernels, unsigned int __pyx_v_numKernels, cudaLibrary_t __pyx_v_lib) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":864 + * + * cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaLibraryEnumerateKernels(kernels, numKernels, lib) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaLibraryEnumerateKernels(__pyx_v_kernels, __pyx_v_numKernels, __pyx_v_lib); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":863 + * return cudaLibraryGetKernelCount(count, lib) + * + * cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaLibraryEnumerateKernels(kernels, numKernels, lib) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":866 + * return cudaLibraryEnumerateKernels(kernels, numKernels, lib) + * + * cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaKernelSetAttributeForDevice(kernel, attr, value, device) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaKernelSetAttributeForDevice(cudaKernel_t __pyx_v_kernel, enum cudaFuncAttribute __pyx_v_attr, int __pyx_v_value, int __pyx_v_device) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":867 + * + * cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaKernelSetAttributeForDevice(kernel, attr, value, device) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaKernelSetAttributeForDevice(__pyx_v_kernel, __pyx_v_attr, __pyx_v_value, __pyx_v_device); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":866 + * return cudaLibraryEnumerateKernels(kernels, numKernels, lib) + * + * cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaKernelSetAttributeForDevice(kernel, attr, value, device) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":869 + * return cudaKernelSetAttributeForDevice(kernel, attr, value, device) + * + * cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetExportTable(ppExportTable, pExportTableId) + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetExportTable(void const **__pyx_v_ppExportTable, cudaUUID_t const *__pyx_v_pExportTableId) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":870 + * + * cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetExportTable(ppExportTable, pExportTableId) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaGetExportTable(__pyx_v_ppExportTable, __pyx_v_pExportTableId); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":869 + * return cudaKernelSetAttributeForDevice(kernel, attr, value, device) + * + * cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetExportTable(ppExportTable, pExportTableId) + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":872 + * return cudaGetExportTable(ppExportTable, pExportTableId) + * + * cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetKernel(kernelPtr, entryFuncAddr) + * @cython.show_performance_hints(False) +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetKernel(cudaKernel_t *__pyx_v_kernelPtr, void const *__pyx_v_entryFuncAddr) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":873 + * + * cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetKernel(kernelPtr, entryFuncAddr) # <<<<<<<<<<<<<< + * @cython.show_performance_hints(False) + * cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: +*/ + __pyx_r = cudaGetKernel(__pyx_v_kernelPtr, __pyx_v_entryFuncAddr); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":872 + * return cudaGetExportTable(ppExportTable, pExportTableId) + * + * cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaGetKernel(kernelPtr, entryFuncAddr) + * @cython.show_performance_hints(False) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":874 + * cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetKernel(kernelPtr, entryFuncAddr) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: + * return make_cudaPitchedPtr(d, p, xsz, ysz) +*/ + +static struct cudaPitchedPtr __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaPitchedPtr(void *__pyx_v_d, size_t __pyx_v_p, size_t __pyx_v_xsz, size_t __pyx_v_ysz) { + struct cudaPitchedPtr __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":876 + * @cython.show_performance_hints(False) + * cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: + * return make_cudaPitchedPtr(d, p, xsz, ysz) # <<<<<<<<<<<<<< + * @cython.show_performance_hints(False) + * cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: +*/ + __pyx_r = make_cudaPitchedPtr(__pyx_v_d, __pyx_v_p, __pyx_v_xsz, __pyx_v_ysz); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":874 + * cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaGetKernel(kernelPtr, entryFuncAddr) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: + * return make_cudaPitchedPtr(d, p, xsz, ysz) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":877 + * cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: + * return make_cudaPitchedPtr(d, p, xsz, ysz) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: + * return make_cudaPos(x, y, z) +*/ + +static struct cudaPos __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaPos(size_t __pyx_v_x, size_t __pyx_v_y, size_t __pyx_v_z) { + struct cudaPos __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":879 + * @cython.show_performance_hints(False) + * cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: + * return make_cudaPos(x, y, z) # <<<<<<<<<<<<<< + * @cython.show_performance_hints(False) + * cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil: +*/ + __pyx_r = make_cudaPos(__pyx_v_x, __pyx_v_y, __pyx_v_z); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":877 + * cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: + * return make_cudaPitchedPtr(d, p, xsz, ysz) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: + * return make_cudaPos(x, y, z) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":880 + * cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: + * return make_cudaPos(x, y, z) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil: + * return make_cudaExtent(w, h, d) +*/ + +static struct cudaExtent __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaExtent(size_t __pyx_v_w, size_t __pyx_v_h, size_t __pyx_v_d) { + struct cudaExtent __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":882 + * @cython.show_performance_hints(False) + * cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil: + * return make_cudaExtent(w, h, d) # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = make_cudaExtent(__pyx_v_w, __pyx_v_h, __pyx_v_d); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":880 + * cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: + * return make_cudaPos(x, y, z) + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil: + * return make_cudaExtent(w, h, d) +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":884 + * return make_cudaExtent(w, h, d) + * + * cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaProfilerStart() + * +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaProfilerStart(void) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":885 + * + * cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaProfilerStart() # <<<<<<<<<<<<<< + * + * cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil: +*/ + __pyx_r = cudaProfilerStart(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":884 + * return make_cudaExtent(w, h, d) + * + * cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaProfilerStart() + * +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_bindings/cyruntime_ptds.pyx":887 + * return cudaProfilerStart() + * + * cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaProfilerStop() +*/ + +static cudaError_t __pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaProfilerStop(void) { + cudaError_t __pyx_r; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":888 + * + * cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil: + * return cudaProfilerStop() # <<<<<<<<<<<<<< +*/ + __pyx_r = cudaProfilerStop(); + goto __pyx_L0; + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":887 + * return cudaProfilerStart() + * + * cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil: # <<<<<<<<<<<<<< + * return cudaProfilerStop() +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} +/* #### Code section: module_exttypes ### */ + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + if (__Pyx_ExportFunction("_cudaDeviceReset", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceReset, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceSynchronize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSynchronize, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceSetLimit", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetLimit, "cudaError_t (enum cudaLimit, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetLimit", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetLimit, "cudaError_t (size_t *, enum cudaLimit)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetTexture1DLinearMaxWidth", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetTexture1DLinearMaxWidth, "cudaError_t (size_t *, struct cudaChannelFormatDesc const *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetCacheConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetCacheConfig, "cudaError_t (enum cudaFuncCache *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetStreamPriorityRange", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetStreamPriorityRange, "cudaError_t (int *, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceSetCacheConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetCacheConfig, "cudaError_t (enum cudaFuncCache)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetByPCIBusId", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetByPCIBusId, "cudaError_t (int *, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetPCIBusId", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetPCIBusId, "cudaError_t (char *, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaIpcGetEventHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcGetEventHandle, "cudaError_t (cudaIpcEventHandle_t *, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaIpcOpenEventHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcOpenEventHandle, "cudaError_t (cudaEvent_t *, cudaIpcEventHandle_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaIpcGetMemHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcGetMemHandle, "cudaError_t (cudaIpcMemHandle_t *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaIpcOpenMemHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcOpenMemHandle, "cudaError_t (void **, cudaIpcMemHandle_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaIpcCloseMemHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaIpcCloseMemHandle, "cudaError_t (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceFlushGPUDirectRDMAWrites", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceFlushGPUDirectRDMAWrites, "cudaError_t (enum cudaFlushGPUDirectRDMAWritesTarget, enum cudaFlushGPUDirectRDMAWritesScope)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceRegisterAsyncNotification", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceRegisterAsyncNotification, "cudaError_t (int, cudaAsyncCallback, void *, cudaAsyncCallbackHandle_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceUnregisterAsyncNotification", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceUnregisterAsyncNotification, "cudaError_t (int, cudaAsyncCallbackHandle_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetSharedMemConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetSharedMemConfig, "cudaError_t (enum cudaSharedMemConfig *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceSetSharedMemConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetSharedMemConfig, "cudaError_t (enum cudaSharedMemConfig)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetLastError", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetLastError, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaPeekAtLastError", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaPeekAtLastError, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetErrorName", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetErrorName, "char const *(cudaError_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetErrorString", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetErrorString, "char const *(cudaError_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetDeviceCount", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceCount, "cudaError_t (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetDeviceProperties_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceProperties_v2, "cudaError_t (struct cudaDeviceProp *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetAttribute, "cudaError_t (int *, enum cudaDeviceAttr, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetDefaultMemPool", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetDefaultMemPool, "cudaError_t (cudaMemPool_t *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceSetMemPool", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetMemPool, "cudaError_t (int, cudaMemPool_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetMemPool", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetMemPool, "cudaError_t (cudaMemPool_t *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetNvSciSyncAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetNvSciSyncAttributes, "cudaError_t (void *, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetP2PAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetP2PAttribute, "cudaError_t (int *, enum cudaDeviceP2PAttr, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaChooseDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaChooseDevice, "cudaError_t (int *, struct cudaDeviceProp const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaInitDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaInitDevice, "cudaError_t (int, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaSetDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSetDevice, "cudaError_t (int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDevice, "cudaError_t (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaSetDeviceFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSetDeviceFlags, "cudaError_t (unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetDeviceFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDeviceFlags, "cudaError_t (unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreate, "cudaError_t (cudaStream_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamCreateWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreateWithFlags, "cudaError_t (cudaStream_t *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamCreateWithPriority", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCreateWithPriority, "cudaError_t (cudaStream_t *, unsigned int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetPriority", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetPriority, "cudaError_t (cudaStream_t, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetFlags, "cudaError_t (cudaStream_t, unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetId", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetId, "cudaError_t (cudaStream_t, unsigned PY_LONG_LONG *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetDevice, "cudaError_t (cudaStream_t, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaCtxResetPersistingL2Cache", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCtxResetPersistingL2Cache, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamCopyAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamCopyAttributes, "cudaError_t (cudaStream_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetAttribute, "cudaError_t (cudaStream_t, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrID, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrValue *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamSetAttribute, "cudaError_t (cudaStream_t, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrID, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaStreamAttrValue const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamDestroy, "cudaError_t (cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamWaitEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamWaitEvent, "cudaError_t (cudaStream_t, cudaEvent_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamAddCallback", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamAddCallback, "cudaError_t (cudaStream_t, cudaStreamCallback_t, void *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamSynchronize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamSynchronize, "cudaError_t (cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamQuery", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamQuery, "cudaError_t (cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamAttachMemAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamAttachMemAsync, "cudaError_t (cudaStream_t, void *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamBeginCapture", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamBeginCapture, "cudaError_t (cudaStream_t, enum cudaStreamCaptureMode)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamBeginCaptureToGraph", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamBeginCaptureToGraph, "cudaError_t (cudaStream_t, cudaGraph_t, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t, enum cudaStreamCaptureMode)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaThreadExchangeStreamCaptureMode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaThreadExchangeStreamCaptureMode, "cudaError_t (enum cudaStreamCaptureMode *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamEndCapture", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamEndCapture, "cudaError_t (cudaStream_t, cudaGraph_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamIsCapturing", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamIsCapturing, "cudaError_t (cudaStream_t, enum cudaStreamCaptureStatus *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetCaptureInfo_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetCaptureInfo_v2, "cudaError_t (cudaStream_t, enum cudaStreamCaptureStatus *, unsigned PY_LONG_LONG *, cudaGraph_t *, cudaGraphNode_t const **, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamGetCaptureInfo_v3", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamGetCaptureInfo_v3, "cudaError_t (cudaStream_t, enum cudaStreamCaptureStatus *, unsigned PY_LONG_LONG *, cudaGraph_t *, cudaGraphNode_t const **, cudaGraphEdgeData const **, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamUpdateCaptureDependencies", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamUpdateCaptureDependencies, "cudaError_t (cudaStream_t, cudaGraphNode_t *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaStreamUpdateCaptureDependencies_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaStreamUpdateCaptureDependencies_v2, "cudaError_t (cudaStream_t, cudaGraphNode_t *, cudaGraphEdgeData const *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventCreate, "cudaError_t (cudaEvent_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventCreateWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventCreateWithFlags, "cudaError_t (cudaEvent_t *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventRecord", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventRecord, "cudaError_t (cudaEvent_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventRecordWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventRecordWithFlags, "cudaError_t (cudaEvent_t, cudaStream_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventQuery", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventQuery, "cudaError_t (cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventSynchronize", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventSynchronize, "cudaError_t (cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventDestroy, "cudaError_t (cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventElapsedTime", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventElapsedTime, "cudaError_t (float *, cudaEvent_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaEventElapsedTime_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaEventElapsedTime_v2, "cudaError_t (float *, cudaEvent_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaImportExternalMemory", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaImportExternalMemory, "cudaError_t (cudaExternalMemory_t *, struct cudaExternalMemoryHandleDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaExternalMemoryGetMappedBuffer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaExternalMemoryGetMappedBuffer, "cudaError_t (void **, cudaExternalMemory_t, struct cudaExternalMemoryBufferDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaExternalMemoryGetMappedMipmappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaExternalMemoryGetMappedMipmappedArray, "cudaError_t (cudaMipmappedArray_t *, cudaExternalMemory_t, struct cudaExternalMemoryMipmappedArrayDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDestroyExternalMemory", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyExternalMemory, "cudaError_t (cudaExternalMemory_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaImportExternalSemaphore", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaImportExternalSemaphore, "cudaError_t (cudaExternalSemaphore_t *, struct cudaExternalSemaphoreHandleDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaSignalExternalSemaphoresAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaSignalExternalSemaphoresAsync_v2, "cudaError_t (cudaExternalSemaphore_t const *, struct cudaExternalSemaphoreSignalParams const *, unsigned int, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaWaitExternalSemaphoresAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaWaitExternalSemaphoresAsync_v2, "cudaError_t (cudaExternalSemaphore_t const *, struct cudaExternalSemaphoreWaitParams const *, unsigned int, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDestroyExternalSemaphore", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyExternalSemaphore, "cudaError_t (cudaExternalSemaphore_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFuncSetCacheConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetCacheConfig, "cudaError_t (void const *, enum cudaFuncCache)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFuncGetAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncGetAttributes, "cudaError_t (struct cudaFuncAttributes *, void const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFuncSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetAttribute, "cudaError_t (void const *, enum cudaFuncAttribute, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLaunchHostFunc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLaunchHostFunc, "cudaError_t (cudaStream_t, cudaHostFn_t, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFuncSetSharedMemConfig", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFuncSetSharedMemConfig, "cudaError_t (void const *, enum cudaSharedMemConfig)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaOccupancyMaxActiveBlocksPerMultiprocessor", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyMaxActiveBlocksPerMultiprocessor, "cudaError_t (int *, void const *, int, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaOccupancyAvailableDynamicSMemPerBlock", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyAvailableDynamicSMemPerBlock, "cudaError_t (size_t *, void const *, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags, "cudaError_t (int *, void const *, int, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocManaged", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocManaged, "cudaError_t (void **, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMalloc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc, "cudaError_t (void **, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocHost", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocHost, "cudaError_t (void **, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocPitch", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocPitch, "cudaError_t (void **, size_t *, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocArray, "cudaError_t (cudaArray_t *, struct cudaChannelFormatDesc const *, size_t, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFree", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFree, "cudaError_t (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFreeHost", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeHost, "cudaError_t (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFreeArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeArray, "cudaError_t (cudaArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFreeMipmappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeMipmappedArray, "cudaError_t (cudaMipmappedArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaHostAlloc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostAlloc, "cudaError_t (void **, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaHostRegister", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostRegister, "cudaError_t (void *, size_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaHostUnregister", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostUnregister, "cudaError_t (void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaHostGetDevicePointer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostGetDevicePointer, "cudaError_t (void **, void *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaHostGetFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaHostGetFlags, "cudaError_t (unsigned int *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMalloc3D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc3D, "cudaError_t (struct cudaPitchedPtr *, struct cudaExtent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMalloc3DArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMalloc3DArray, "cudaError_t (cudaArray_t *, struct cudaChannelFormatDesc const *, struct cudaExtent, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocMipmappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocMipmappedArray, "cudaError_t (cudaMipmappedArray_t *, struct cudaChannelFormatDesc const *, struct cudaExtent, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetMipmappedArrayLevel", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetMipmappedArrayLevel, "cudaError_t (cudaArray_t *, cudaMipmappedArray_const_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy3D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3D, "cudaError_t (struct cudaMemcpy3DParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy3DPeer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DPeer, "cudaError_t (struct cudaMemcpy3DPeerParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy3DAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DAsync, "cudaError_t (struct cudaMemcpy3DParms const *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy3DPeerAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DPeerAsync, "cudaError_t (struct cudaMemcpy3DPeerParms const *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemGetInfo", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemGetInfo, "cudaError_t (size_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaArrayGetInfo", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetInfo, "cudaError_t (struct cudaChannelFormatDesc *, struct cudaExtent *, unsigned int *, cudaArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaArrayGetPlane", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetPlane, "cudaError_t (cudaArray_t *, cudaArray_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaArrayGetMemoryRequirements", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetMemoryRequirements, "cudaError_t (struct cudaArrayMemoryRequirements *, cudaArray_t, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMipmappedArrayGetMemoryRequirements", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMipmappedArrayGetMemoryRequirements, "cudaError_t (struct cudaArrayMemoryRequirements *, cudaMipmappedArray_t, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaArrayGetSparseProperties", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaArrayGetSparseProperties, "cudaError_t (struct cudaArraySparseProperties *, cudaArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMipmappedArrayGetSparseProperties", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMipmappedArrayGetSparseProperties, "cudaError_t (struct cudaArraySparseProperties *, cudaMipmappedArray_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy, "cudaError_t (void *, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyPeer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyPeer, "cudaError_t (void *, int, void const *, int, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2D, "cudaError_t (void *, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2DToArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DToArray, "cudaError_t (cudaArray_t, size_t, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2DFromArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DFromArray, "cudaError_t (void *, size_t, cudaArray_const_t, size_t, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2DArrayToArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DArrayToArray, "cudaError_t (cudaArray_t, size_t, size_t, cudaArray_const_t, size_t, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyAsync, "cudaError_t (void *, void const *, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyPeerAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyPeerAsync, "cudaError_t (void *, int, void const *, int, size_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyBatchAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyBatchAsync, "cudaError_t (void **, void **, size_t *, size_t, struct cudaMemcpyAttributes *, size_t *, size_t, size_t *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy3DBatchAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy3DBatchAsync, "cudaError_t (size_t, struct cudaMemcpy3DBatchOp *, size_t *, unsigned PY_LONG_LONG, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2DAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DAsync, "cudaError_t (void *, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2DToArrayAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DToArrayAsync, "cudaError_t (cudaArray_t, size_t, size_t, void const *, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpy2DFromArrayAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpy2DFromArrayAsync, "cudaError_t (void *, size_t, cudaArray_const_t, size_t, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemset", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset, "cudaError_t (void *, int, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemset2D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset2D, "cudaError_t (void *, size_t, int, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemset3D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset3D, "cudaError_t (struct cudaPitchedPtr, int, struct cudaExtent)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemsetAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemsetAsync, "cudaError_t (void *, int, size_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemset2DAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset2DAsync, "cudaError_t (void *, size_t, int, size_t, size_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemset3DAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemset3DAsync, "cudaError_t (struct cudaPitchedPtr, int, struct cudaExtent, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPrefetchAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPrefetchAsync, "cudaError_t (void const *, size_t, int, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPrefetchAsync_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPrefetchAsync_v2, "cudaError_t (void const *, size_t, struct cudaMemLocation, unsigned int, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemAdvise", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemAdvise, "cudaError_t (void const *, size_t, enum cudaMemoryAdvise, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemAdvise_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemAdvise_v2, "cudaError_t (void const *, size_t, enum cudaMemoryAdvise, struct cudaMemLocation)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemRangeGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemRangeGetAttribute, "cudaError_t (void *, size_t, enum cudaMemRangeAttribute, void const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemRangeGetAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemRangeGetAttributes, "cudaError_t (void **, size_t *, enum cudaMemRangeAttribute *, size_t, void const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyToArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyToArray, "cudaError_t (cudaArray_t, size_t, size_t, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyFromArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyFromArray, "cudaError_t (void *, cudaArray_const_t, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyArrayToArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyArrayToArray, "cudaError_t (cudaArray_t, size_t, size_t, cudaArray_const_t, size_t, size_t, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyToArrayAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyToArrayAsync, "cudaError_t (cudaArray_t, size_t, size_t, void const *, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemcpyFromArrayAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemcpyFromArrayAsync, "cudaError_t (void *, cudaArray_const_t, size_t, size_t, size_t, enum cudaMemcpyKind, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocAsync, "cudaError_t (void **, size_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaFreeAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaFreeAsync, "cudaError_t (void *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolTrimTo", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolTrimTo, "cudaError_t (cudaMemPool_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolSetAttribute, "cudaError_t (cudaMemPool_t, enum cudaMemPoolAttr, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolGetAttribute, "cudaError_t (cudaMemPool_t, enum cudaMemPoolAttr, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolSetAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolSetAccess, "cudaError_t (cudaMemPool_t, struct cudaMemAccessDesc const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolGetAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolGetAccess, "cudaError_t (enum cudaMemAccessFlags *, cudaMemPool_t, struct cudaMemLocation *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolCreate, "cudaError_t (cudaMemPool_t *, struct cudaMemPoolProps const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolDestroy, "cudaError_t (cudaMemPool_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMallocFromPoolAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMallocFromPoolAsync, "cudaError_t (void **, size_t, cudaMemPool_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolExportToShareableHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolExportToShareableHandle, "cudaError_t (void *, cudaMemPool_t, enum cudaMemAllocationHandleType, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolImportFromShareableHandle", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolImportFromShareableHandle, "cudaError_t (cudaMemPool_t *, void *, enum cudaMemAllocationHandleType, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolExportPointer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolExportPointer, "cudaError_t (struct cudaMemPoolPtrExportData *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaMemPoolImportPointer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaMemPoolImportPointer, "cudaError_t (void **, cudaMemPool_t, struct cudaMemPoolPtrExportData *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaPointerGetAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaPointerGetAttributes, "cudaError_t (struct cudaPointerAttributes *, void const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceCanAccessPeer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceCanAccessPeer, "cudaError_t (int *, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceEnablePeerAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceEnablePeerAccess, "cudaError_t (int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceDisablePeerAccess", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceDisablePeerAccess, "cudaError_t (int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsUnregisterResource", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsUnregisterResource, "cudaError_t (cudaGraphicsResource_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsResourceSetMapFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceSetMapFlags, "cudaError_t (cudaGraphicsResource_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsMapResources", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsMapResources, "cudaError_t (int, cudaGraphicsResource_t *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsUnmapResources", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsUnmapResources, "cudaError_t (int, cudaGraphicsResource_t *, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsResourceGetMappedPointer", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceGetMappedPointer, "cudaError_t (void **, size_t *, cudaGraphicsResource_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsSubResourceGetMappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsSubResourceGetMappedArray, "cudaError_t (cudaArray_t *, cudaGraphicsResource_t, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphicsResourceGetMappedMipmappedArray", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphicsResourceGetMappedMipmappedArray, "cudaError_t (cudaMipmappedArray_t *, cudaGraphicsResource_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetChannelDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetChannelDesc, "cudaError_t (struct cudaChannelFormatDesc *, cudaArray_const_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaCreateChannelDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateChannelDesc, "struct cudaChannelFormatDesc (int, int, int, int, enum cudaChannelFormatKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaCreateTextureObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateTextureObject, "cudaError_t (cudaTextureObject_t *, struct cudaResourceDesc const *, struct cudaTextureDesc const *, struct cudaResourceViewDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDestroyTextureObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroyTextureObject, "cudaError_t (cudaTextureObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetTextureObjectResourceDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectResourceDesc, "cudaError_t (struct cudaResourceDesc *, cudaTextureObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetTextureObjectTextureDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectTextureDesc, "cudaError_t (struct cudaTextureDesc *, cudaTextureObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetTextureObjectResourceViewDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetTextureObjectResourceViewDesc, "cudaError_t (struct cudaResourceViewDesc *, cudaTextureObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaCreateSurfaceObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaCreateSurfaceObject, "cudaError_t (cudaSurfaceObject_t *, struct cudaResourceDesc const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDestroySurfaceObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDestroySurfaceObject, "cudaError_t (cudaSurfaceObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetSurfaceObjectResourceDesc", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetSurfaceObjectResourceDesc, "cudaError_t (struct cudaResourceDesc *, cudaSurfaceObject_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDriverGetVersion", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDriverGetVersion, "cudaError_t (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaRuntimeGetVersion", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaRuntimeGetVersion, "cudaError_t (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphCreate, "cudaError_t (cudaGraph_t *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddKernelNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddKernelNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaKernelNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphKernelNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaKernelNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphKernelNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaKernelNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphKernelNodeCopyAttributes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeCopyAttributes, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphKernelNodeGetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeGetAttribute, "cudaError_t (cudaGraphNode_t, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrID, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrValue *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphKernelNodeSetAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphKernelNodeSetAttribute, "cudaError_t (cudaGraphNode_t, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrID, __pyx_t_4cuda_8bindings_9_bindings_14cyruntime_ptds_cudaKernelNodeAttrValue const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddMemcpyNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemcpyNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaMemcpy3DParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddMemcpyNode1D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemcpyNode1D, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, void *, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemcpyNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemcpy3DParms *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemcpyNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemcpy3DParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemcpyNodeSetParams1D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemcpyNodeSetParams1D, "cudaError_t (cudaGraphNode_t, void *, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddMemsetNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemsetNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaMemsetParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemsetNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemsetNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemsetParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemsetNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemsetNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemsetParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddHostNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddHostNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaHostNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphHostNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphHostNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaHostNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphHostNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphHostNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaHostNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddChildGraphNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddChildGraphNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphChildGraphNodeGetGraph", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphChildGraphNodeGetGraph, "cudaError_t (cudaGraphNode_t, cudaGraph_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddEmptyNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEmptyNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddEventRecordNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEventRecordNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphEventRecordNodeGetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventRecordNodeGetEvent, "cudaError_t (cudaGraphNode_t, cudaEvent_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphEventRecordNodeSetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventRecordNodeSetEvent, "cudaError_t (cudaGraphNode_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddEventWaitNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddEventWaitNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphEventWaitNodeGetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventWaitNodeGetEvent, "cudaError_t (cudaGraphNode_t, cudaEvent_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphEventWaitNodeSetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphEventWaitNodeSetEvent, "cudaError_t (cudaGraphNode_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddExternalSemaphoresSignalNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddExternalSemaphoresSignalNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaExternalSemaphoreSignalNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExternalSemaphoresSignalNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresSignalNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaExternalSemaphoreSignalNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExternalSemaphoresSignalNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresSignalNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaExternalSemaphoreSignalNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddExternalSemaphoresWaitNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddExternalSemaphoresWaitNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaExternalSemaphoreWaitNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExternalSemaphoresWaitNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresWaitNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaExternalSemaphoreWaitNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExternalSemaphoresWaitNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExternalSemaphoresWaitNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaExternalSemaphoreWaitNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddMemAllocNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemAllocNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaMemAllocNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemAllocNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemAllocNodeGetParams, "cudaError_t (cudaGraphNode_t, struct cudaMemAllocNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddMemFreeNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddMemFreeNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphMemFreeNodeGetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphMemFreeNodeGetParams, "cudaError_t (cudaGraphNode_t, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGraphMemTrim", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGraphMemTrim, "cudaError_t (int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceGetGraphMemAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceGetGraphMemAttribute, "cudaError_t (int, enum cudaGraphMemAttributeType, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaDeviceSetGraphMemAttribute", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaDeviceSetGraphMemAttribute, "cudaError_t (int, enum cudaGraphMemAttributeType, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphClone", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphClone, "cudaError_t (cudaGraph_t *, cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeFindInClone", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeFindInClone, "cudaError_t (cudaGraphNode_t *, cudaGraphNode_t, cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeGetType", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetType, "cudaError_t (cudaGraphNode_t, enum cudaGraphNodeType *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphGetNodes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetNodes, "cudaError_t (cudaGraph_t, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphGetRootNodes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetRootNodes, "cudaError_t (cudaGraph_t, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphGetEdges", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetEdges, "cudaError_t (cudaGraph_t, cudaGraphNode_t *, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphGetEdges_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphGetEdges_v2, "cudaError_t (cudaGraph_t, cudaGraphNode_t *, cudaGraphNode_t *, cudaGraphEdgeData *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeGetDependencies", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependencies, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeGetDependencies_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependencies_v2, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t *, cudaGraphEdgeData *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeGetDependentNodes", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependentNodes, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeGetDependentNodes_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetDependentNodes_v2, "cudaError_t (cudaGraphNode_t, cudaGraphNode_t *, cudaGraphEdgeData *, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddDependencies", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddDependencies, "cudaError_t (cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddDependencies_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddDependencies_v2, "cudaError_t (cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphRemoveDependencies", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRemoveDependencies, "cudaError_t (cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphRemoveDependencies_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRemoveDependencies_v2, "cudaError_t (cudaGraph_t, cudaGraphNode_t const *, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphDestroyNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDestroyNode, "cudaError_t (cudaGraphNode_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphInstantiate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiate, "cudaError_t (cudaGraphExec_t *, cudaGraph_t, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphInstantiateWithFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiateWithFlags, "cudaError_t (cudaGraphExec_t *, cudaGraph_t, unsigned PY_LONG_LONG)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphInstantiateWithParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphInstantiateWithParams, "cudaError_t (cudaGraphExec_t *, cudaGraph_t, cudaGraphInstantiateParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecGetFlags", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecGetFlags, "cudaError_t (cudaGraphExec_t, unsigned PY_LONG_LONG *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecKernelNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecKernelNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaKernelNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecMemcpyNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemcpyNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaMemcpy3DParms const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecMemcpyNodeSetParams1D", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemcpyNodeSetParams1D, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, void *, void const *, size_t, enum cudaMemcpyKind)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecMemsetNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecMemsetNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaMemsetParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecHostNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecHostNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaHostNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecChildGraphNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecChildGraphNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecEventRecordNodeSetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecEventRecordNodeSetEvent, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecEventWaitNodeSetEvent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecEventWaitNodeSetEvent, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, cudaEvent_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecExternalSemaphoresSignalNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecExternalSemaphoresSignalNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaExternalSemaphoreSignalNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecExternalSemaphoresWaitNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecExternalSemaphoresWaitNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaExternalSemaphoreWaitNodeParams const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeSetEnabled", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeSetEnabled, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeGetEnabled", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeGetEnabled, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecUpdate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecUpdate, "cudaError_t (cudaGraphExec_t, cudaGraph_t, cudaGraphExecUpdateResultInfo *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphUpload", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphUpload, "cudaError_t (cudaGraphExec_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphLaunch", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphLaunch, "cudaError_t (cudaGraphExec_t, cudaStream_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecDestroy, "cudaError_t (cudaGraphExec_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDestroy, "cudaError_t (cudaGraph_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphDebugDotPrint", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphDebugDotPrint, "cudaError_t (cudaGraph_t, char const *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaUserObjectCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectCreate, "cudaError_t (cudaUserObject_t *, void *, cudaHostFn_t, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaUserObjectRetain", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectRetain, "cudaError_t (cudaUserObject_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaUserObjectRelease", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaUserObjectRelease, "cudaError_t (cudaUserObject_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphRetainUserObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphRetainUserObject, "cudaError_t (cudaGraph_t, cudaUserObject_t, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphReleaseUserObject", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphReleaseUserObject, "cudaError_t (cudaGraph_t, cudaUserObject_t, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddNode", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddNode, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, size_t, struct cudaGraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphAddNode_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphAddNode_v2, "cudaError_t (cudaGraphNode_t *, cudaGraph_t, cudaGraphNode_t const *, cudaGraphEdgeData const *, size_t, struct cudaGraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphNodeSetParams, "cudaError_t (cudaGraphNode_t, struct cudaGraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphExecNodeSetParams", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphExecNodeSetParams, "cudaError_t (cudaGraphExec_t, cudaGraphNode_t, struct cudaGraphNodeParams *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGraphConditionalHandleCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGraphConditionalHandleCreate, "cudaError_t (cudaGraphConditionalHandle *, cudaGraph_t, unsigned int, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetDriverEntryPoint", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDriverEntryPoint, "cudaError_t (char const *, void **, unsigned PY_LONG_LONG, enum cudaDriverEntryPointQueryResult *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetDriverEntryPointByVersion", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetDriverEntryPointByVersion, "cudaError_t (char const *, void **, unsigned int, unsigned PY_LONG_LONG, enum cudaDriverEntryPointQueryResult *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryLoadData", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryLoadData, "cudaError_t (cudaLibrary_t *, void const *, enum cudaJitOption *, void **, unsigned int, enum cudaLibraryOption *, void **, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryLoadFromFile", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryLoadFromFile, "cudaError_t (cudaLibrary_t *, char const *, enum cudaJitOption *, void **, unsigned int, enum cudaLibraryOption *, void **, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryUnload", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryUnload, "cudaError_t (cudaLibrary_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryGetKernel", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetKernel, "cudaError_t (cudaKernel_t *, cudaLibrary_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryGetGlobal", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetGlobal, "cudaError_t (void **, size_t *, cudaLibrary_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryGetManaged", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetManaged, "cudaError_t (void **, size_t *, cudaLibrary_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryGetUnifiedFunction", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetUnifiedFunction, "cudaError_t (void **, cudaLibrary_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryGetKernelCount", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryGetKernelCount, "cudaError_t (unsigned int *, cudaLibrary_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaLibraryEnumerateKernels", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaLibraryEnumerateKernels, "cudaError_t (cudaKernel_t *, unsigned int, cudaLibrary_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaKernelSetAttributeForDevice", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaKernelSetAttributeForDevice, "cudaError_t (cudaKernel_t, enum cudaFuncAttribute, int, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetExportTable", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetExportTable, "cudaError_t (void const **, cudaUUID_t const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaGetKernel", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaGetKernel, "cudaError_t (cudaKernel_t *, void const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_make_cudaPitchedPtr", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaPitchedPtr, "struct cudaPitchedPtr (void *, size_t, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_make_cudaPos", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaPos, "struct cudaPos (size_t, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_make_cudaExtent", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__make_cudaExtent, "struct cudaExtent (size_t, size_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaProfilerStart", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaProfilerStart, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cudaProfilerStop", (void (*)(void))__pyx_f_4cuda_8bindings_9_bindings_14cyruntime_ptds__cudaProfilerStop, "cudaError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_cyruntime_ptds(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_cyruntime_ptds}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "cyruntime_ptds", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_cyruntime_ptds(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_cyruntime_ptds(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x03090000 + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_cyruntime_ptds(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'cyruntime_ptds' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "cyruntime_ptds" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_NOT_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY +__Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); +if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); +} +#endif + +__Pyx_RefNannySetupContext("PyInit_cyruntime_ptds", 0); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #ifdef __Pxy_PyFrame_Initialize_Offsets + __Pxy_PyFrame_Initialize_Offsets(); + #endif + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #if 0 || defined(__Pyx_CyFunction_USED) || defined(__Pyx_FusedFunction_USED) || defined(__Pyx_Coroutine_USED) || defined(__Pyx_Generator_USED) || defined(__Pyx_AsyncGen_USED) + if (__pyx_CommonTypesMetaclass_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_AsyncGen_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + if (__pyx_module_is_main_cuda__bindings___bindings__cyruntime_ptds) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "cuda.bindings._bindings.cyruntime_ptds")) { + if (unlikely((PyDict_SetItemString(modules, "cuda.bindings._bindings.cyruntime_ptds", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_function_export_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_type_init_code(__pyx_mstate); + (void)__Pyx_modinit_type_import_code(__pyx_mstate); + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + (void)__Pyx_modinit_function_import_code(__pyx_mstate); + /*--- Execution code ---*/ + + /* "cuda/bindings/_bindings/cyruntime_ptds.pyx":1 + * # SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # <<<<<<<<<<<<<< + * # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + * +*/ + __pyx_t_2 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_2) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init cuda.bindings._bindings.cyruntime_ptds", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init cuda.bindings._bindings.cyruntime_ptds"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ + +typedef struct { + const char *s; +#if 18 <= 65535 + const unsigned short n; +#elif 18 / 2 < INT_MAX + const unsigned int n; +#elif 18 / 2 < LONG_MAX + const unsigned long n; +#else + const Py_ssize_t n; +#endif +#if 1 <= 31 + const unsigned int encoding : 5; +#elif 1 <= 255 + const unsigned char encoding; +#elif 1 <= 65535 + const unsigned short encoding; +#else + const Py_ssize_t encoding; +#endif + const unsigned int is_unicode : 1; + const unsigned int intern : 1; +} __Pyx_StringTabEntry; +static const char * const __pyx_string_tab_encodings[] = { 0 }; +static const __Pyx_StringTabEntry __pyx_string_tab[] = { + {__pyx_k_, sizeof(__pyx_k_), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_ */ + {__pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cline_in_traceback */ + {__pyx_k_main, sizeof(__pyx_k_main), 0, 1, 1}, /* PyObject cname: __pyx_n_u_main */ + {__pyx_k_module, sizeof(__pyx_k_module), 0, 1, 1}, /* PyObject cname: __pyx_n_u_module */ + {__pyx_k_name, sizeof(__pyx_k_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name */ + {__pyx_k_qualname, sizeof(__pyx_k_qualname), 0, 1, 1}, /* PyObject cname: __pyx_n_u_qualname */ + {__pyx_k_test, sizeof(__pyx_k_test), 0, 1, 1}, /* PyObject cname: __pyx_n_u_test */ + {0, 0, 0, 0, 0} +}; +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names); + +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + return 0; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + __Pyx_RefNannyFinishContext(); + return 0; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + if (__Pyx_InitStrings(__pyx_string_tab, __pyx_mstate->__pyx_string_tab, __pyx_string_tab_encodings) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + return 0; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* PyDictVersioning */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* PyErrExceptionMatches */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStr */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* PyObjectGetAttrStrNoError */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* CLineInTraceback */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline; + PyObject *ptype, *pvalue, *ptraceback; +#if CYTHON_COMPILING_IN_CPYTHON + PyObject **cython_runtime_dict; +#endif + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); +#if CYTHON_COMPILING_IN_CPYTHON + cython_runtime_dict = _PyObject_GetDictPtr(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __Pyx_BEGIN_CRITICAL_SECTION(*cython_runtime_dict); + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, *cython_runtime_dict, + __Pyx_PyDict_GetItemStr(*cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback)) + Py_XINCREF(use_cline); + __Pyx_END_CRITICAL_SECTION(); + } else +#endif + { + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback); + if (use_cline_obj) { + use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; + Py_INCREF(use_cline); + Py_DECREF(use_cline_obj); + } else { + PyErr_Clear(); + use_cline = NULL; + } + } + if (!use_cline) { + c_line = 0; + (void) PyObject_SetAttr(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False); + } + else if (use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + (void) __pyx_cfilenm; + (void) __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + py_funcname = PyUnicode_FromString(funcname); + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* FormatTypeName */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static __Pyx_TypeName +__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) +{ + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u_); + } + goto done; +} +#endif + +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject *const *args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = __Pyx_PyFrame_GetLocalsplus(f); + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; + PyObject *kwdefs; + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) { + return NULL; + } + if ( + co->co_kwonlyargcount == 0 && + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, (int)nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif + +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = Py_TYPE(func)->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + #if PY_VERSION_HEX < 0x030800B1 + #if CYTHON_FAST_PYCCALL + if (PyCFunction_Check(func)) { + if (kwargs) { + return _PyCFunction_FastCallDict(func, args, nargs, kwargs); + } else { + return _PyCFunction_FastCallKeywords(func, args, nargs, NULL); + } + } + if (!kwargs && __Pyx_IS_TYPE(func, &PyMethodDescr_Type)) { + return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL); + } + #endif + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs); + } + #endif + #endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #elif defined(__Pyx_CyFunction_USED) && CYTHON_BACKPORT_VECTORCALL + if (__Pyx_CyFunction_CheckExact(func)) { + __pyx_vectorcallfunc f = __Pyx_CyFunction_func_vectorcall(func); + if (f) return f(func, args, _nargs, NULL); + } + #elif CYTHON_COMPILING_IN_LIMITED_API && CYTHON_VECTORCALL + return PyObject_Vectorcall(func, args, _nargs, NULL); + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* PyObjectVectorCallKwBuilder */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if defined(HAVE_LONG_LONG) && !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPyVerify */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntFromPy */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + long val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (long) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + int val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030b0000 + return Py_Version & ~0xFFUL; +#else + static unsigned long __Pyx_cached_runtime_version = 0; + if (__Pyx_cached_runtime_version == 0) { + const char* rt_version = Py_GetVersion(); + unsigned long version = 0; + unsigned long factor = 0x01000000UL; + unsigned int digit = 0; + int i = 0; + while (factor) { + while ('0' <= rt_version[i] && rt_version[i] <= '9') { + digit = digit * 10 + (unsigned int) (rt_version[i] - '0'); + ++i; + } + version += factor * digit; + if (rt_version[i] != '.') + break; + digit = 0; + factor >>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } + return __Pyx_cached_runtime_version; +#endif +} + +/* CheckBinaryVersion */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } +} + +/* FunctionExport */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig) { + PyObject *d = 0; + PyObject *cobj = 0; + union { + void (*fp)(void); + void *p; + } tmp; + d = PyObject_GetAttrString(__pyx_m, "__pyx_capi__"); + if (!d) { + PyErr_Clear(); + d = PyDict_New(); + if (!d) + goto bad; + Py_INCREF(d); + if (PyModule_AddObject(__pyx_m, "__pyx_capi__", d) < 0) + goto bad; + } + tmp.fp = f; + cobj = PyCapsule_New(tmp.p, sig, 0); + if (!cobj) + goto bad; + if (PyDict_SetItemString(d, name, cobj) < 0) + goto bad; + Py_DECREF(cobj); + Py_DECREF(d); + return 0; +bad: + Py_XDECREF(cobj); + Py_XDECREF(d); + return -1; +} + +/* InitStrings */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names) { + while (t->s) { + PyObject *str; + if (t->is_unicode) { + if (t->intern) { + str = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + str = PyUnicode_Decode(t->s, t->n - 1, encoding_names[t->encoding], NULL); + } else { + str = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + str = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + if (!str) + return -1; + *target = str; + if (PyObject_Hash(str) == -1) + return -1; + ++t; + ++target; + } + return 0; +} + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} +#if CYTHON_USE_PYLONG_INTERNALS +static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value) { + if (unlikely(!__Pyx_PyLong_IsCompact(x))) + return 0; + Py_ssize_t value = __Pyx_PyLong_CompactValue(x); + if ((sizeof(long) < sizeof(Py_ssize_t)) && unlikely(value != (long) value)) + return 0; + *return_value = (long) value; + return 1; +} +#endif + + +/* MultiPhaseInitModuleState */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 +#else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 +#endif +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS +#error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" +#endif +#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#define __Pyx_ModuleStateLookup_Lock() +#define __Pyx_ModuleStateLookup_Unlock() +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 +static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; +#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#include +static std::mutex __Pyx_ModuleStateLookup_mutex; +#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() +#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) +#include +static mtx_t __Pyx_ModuleStateLookup_mutex; +static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; +static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); +} +#define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(HAVE_PTHREAD_H) +#include +static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; +#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(_WIN32) +#include // synchapi.h on its own doesn't work +static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; +#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#else +#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." +#endif +typedef struct { + int64_t id; + PyObject *module; +} __Pyx_InterpreterIdAndModule; +typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; +} __Pyx_ModuleStateLookupData; +#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; +#else +static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; +#endif +static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; +} +static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; +#else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); +#endif + return result; + } +} +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); +} +#else +#define __Pyx_ModuleStateLookup_wait_until_no_readers() +#endif +static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; +} +static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; +} +static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); +#else + __Pyx_ModuleStateLookup_data = new_data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return result; +} +static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; +#endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); +#else + __Pyx_ModuleStateLookup_data = data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; +} +#endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime_ptds.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime_ptds.pxd new file mode 100644 index 0000000000000000000000000000000000000000..b16e92856bde16a82beb46a98d575f78bc0d4ad6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime_ptds.pxd @@ -0,0 +1,594 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +# This code was automatically generated with version 12.9.0. Do not modify it directly. +cdef extern from "": + """ + #define CUDA_API_PER_THREAD_DEFAULT_STREAM + """ + +include "../cyruntime_types.pxi" + +cdef cudaError_t _cudaDeviceReset() except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil + +cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil + +cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil + +cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil + +cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil + +cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil + +cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil + +cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil + +cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime_ptds.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime_ptds.pyx new file mode 100644 index 0000000000000000000000000000000000000000..4c806962113708f065a0adc9763659f1e4edc9e2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/cyruntime_ptds.pyx @@ -0,0 +1,888 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +# This code was automatically generated with version 12.9.0. Do not modify it directly. +cdef extern from "": + """ + #define CUDA_API_PER_THREAD_DEFAULT_STREAM + """ + +include "../cyruntime_functions.pxi" + +cimport cython + +cdef cudaError_t _cudaDeviceReset() except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceReset() + +cdef cudaError_t _cudaDeviceSynchronize() except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceSynchronize() + +cdef cudaError_t _cudaDeviceSetLimit(cudaLimit limit, size_t value) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceSetLimit(limit, value) + +cdef cudaError_t _cudaDeviceGetLimit(size_t* pValue, cudaLimit limit) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGetLimit(pValue, limit) + +cdef cudaError_t _cudaDeviceGetTexture1DLinearMaxWidth(size_t* maxWidthInElements, const cudaChannelFormatDesc* fmtDesc, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGetTexture1DLinearMaxWidth(maxWidthInElements, fmtDesc, device) + +cdef cudaError_t _cudaDeviceGetCacheConfig(cudaFuncCache* pCacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGetCacheConfig(pCacheConfig) + +cdef cudaError_t _cudaDeviceGetStreamPriorityRange(int* leastPriority, int* greatestPriority) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGetStreamPriorityRange(leastPriority, greatestPriority) + +cdef cudaError_t _cudaDeviceSetCacheConfig(cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceSetCacheConfig(cacheConfig) + +cdef cudaError_t _cudaDeviceGetByPCIBusId(int* device, const char* pciBusId) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGetByPCIBusId(device, pciBusId) + +cdef cudaError_t _cudaDeviceGetPCIBusId(char* pciBusId, int length, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGetPCIBusId(pciBusId, length, device) + +cdef cudaError_t _cudaIpcGetEventHandle(cudaIpcEventHandle_t* handle, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaIpcGetEventHandle(handle, event) + +cdef cudaError_t _cudaIpcOpenEventHandle(cudaEvent_t* event, cudaIpcEventHandle_t handle) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaIpcOpenEventHandle(event, handle) + +cdef cudaError_t _cudaIpcGetMemHandle(cudaIpcMemHandle_t* handle, void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaIpcGetMemHandle(handle, devPtr) + +cdef cudaError_t _cudaIpcOpenMemHandle(void** devPtr, cudaIpcMemHandle_t handle, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaIpcOpenMemHandle(devPtr, handle, flags) + +cdef cudaError_t _cudaIpcCloseMemHandle(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaIpcCloseMemHandle(devPtr) + +cdef cudaError_t _cudaDeviceFlushGPUDirectRDMAWrites(cudaFlushGPUDirectRDMAWritesTarget target, cudaFlushGPUDirectRDMAWritesScope scope) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceFlushGPUDirectRDMAWrites(target, scope) + +cdef cudaError_t _cudaDeviceRegisterAsyncNotification(int device, cudaAsyncCallback callbackFunc, void* userData, cudaAsyncCallbackHandle_t* callback) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceRegisterAsyncNotification(device, callbackFunc, userData, callback) + +cdef cudaError_t _cudaDeviceUnregisterAsyncNotification(int device, cudaAsyncCallbackHandle_t callback) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceUnregisterAsyncNotification(device, callback) + +cdef cudaError_t _cudaDeviceGetSharedMemConfig(cudaSharedMemConfig* pConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGetSharedMemConfig(pConfig) + +cdef cudaError_t _cudaDeviceSetSharedMemConfig(cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceSetSharedMemConfig(config) + +cdef cudaError_t _cudaGetLastError() except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetLastError() + +cdef cudaError_t _cudaPeekAtLastError() except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaPeekAtLastError() + +cdef const char* _cudaGetErrorName(cudaError_t error) except ?NULL nogil: + return cudaGetErrorName(error) + +cdef const char* _cudaGetErrorString(cudaError_t error) except ?NULL nogil: + return cudaGetErrorString(error) + +cdef cudaError_t _cudaGetDeviceCount(int* count) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetDeviceCount(count) + +cdef cudaError_t _cudaGetDeviceProperties_v2(cudaDeviceProp* prop, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetDeviceProperties_v2(prop, device) + +cdef cudaError_t _cudaDeviceGetAttribute(int* value, cudaDeviceAttr attr, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGetAttribute(value, attr, device) + +cdef cudaError_t _cudaDeviceGetDefaultMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGetDefaultMemPool(memPool, device) + +cdef cudaError_t _cudaDeviceSetMemPool(int device, cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceSetMemPool(device, memPool) + +cdef cudaError_t _cudaDeviceGetMemPool(cudaMemPool_t* memPool, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGetMemPool(memPool, device) + +cdef cudaError_t _cudaDeviceGetNvSciSyncAttributes(void* nvSciSyncAttrList, int device, int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGetNvSciSyncAttributes(nvSciSyncAttrList, device, flags) + +cdef cudaError_t _cudaDeviceGetP2PAttribute(int* value, cudaDeviceP2PAttr attr, int srcDevice, int dstDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGetP2PAttribute(value, attr, srcDevice, dstDevice) + +cdef cudaError_t _cudaChooseDevice(int* device, const cudaDeviceProp* prop) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaChooseDevice(device, prop) + +cdef cudaError_t _cudaInitDevice(int device, unsigned int deviceFlags, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaInitDevice(device, deviceFlags, flags) + +cdef cudaError_t _cudaSetDevice(int device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaSetDevice(device) + +cdef cudaError_t _cudaGetDevice(int* device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetDevice(device) + +cdef cudaError_t _cudaSetDeviceFlags(unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaSetDeviceFlags(flags) + +cdef cudaError_t _cudaGetDeviceFlags(unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetDeviceFlags(flags) + +cdef cudaError_t _cudaStreamCreate(cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamCreate(pStream) + +cdef cudaError_t _cudaStreamCreateWithFlags(cudaStream_t* pStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamCreateWithFlags(pStream, flags) + +cdef cudaError_t _cudaStreamCreateWithPriority(cudaStream_t* pStream, unsigned int flags, int priority) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamCreateWithPriority(pStream, flags, priority) + +cdef cudaError_t _cudaStreamGetPriority(cudaStream_t hStream, int* priority) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamGetPriority(hStream, priority) + +cdef cudaError_t _cudaStreamGetFlags(cudaStream_t hStream, unsigned int* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamGetFlags(hStream, flags) + +cdef cudaError_t _cudaStreamGetId(cudaStream_t hStream, unsigned long long* streamId) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamGetId(hStream, streamId) + +cdef cudaError_t _cudaStreamGetDevice(cudaStream_t hStream, int* device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamGetDevice(hStream, device) + +cdef cudaError_t _cudaCtxResetPersistingL2Cache() except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaCtxResetPersistingL2Cache() + +cdef cudaError_t _cudaStreamCopyAttributes(cudaStream_t dst, cudaStream_t src) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamCopyAttributes(dst, src) + +cdef cudaError_t _cudaStreamGetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, cudaStreamAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamGetAttribute(hStream, attr, value_out) + +cdef cudaError_t _cudaStreamSetAttribute(cudaStream_t hStream, cudaStreamAttrID attr, const cudaStreamAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamSetAttribute(hStream, attr, value) + +cdef cudaError_t _cudaStreamDestroy(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamDestroy(stream) + +cdef cudaError_t _cudaStreamWaitEvent(cudaStream_t stream, cudaEvent_t event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamWaitEvent(stream, event, flags) + +cdef cudaError_t _cudaStreamAddCallback(cudaStream_t stream, cudaStreamCallback_t callback, void* userData, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamAddCallback(stream, callback, userData, flags) + +cdef cudaError_t _cudaStreamSynchronize(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamSynchronize(stream) + +cdef cudaError_t _cudaStreamQuery(cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamQuery(stream) + +cdef cudaError_t _cudaStreamAttachMemAsync(cudaStream_t stream, void* devPtr, size_t length, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamAttachMemAsync(stream, devPtr, length, flags) + +cdef cudaError_t _cudaStreamBeginCapture(cudaStream_t stream, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamBeginCapture(stream, mode) + +cdef cudaError_t _cudaStreamBeginCaptureToGraph(cudaStream_t stream, cudaGraph_t graph, const cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaStreamCaptureMode mode) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamBeginCaptureToGraph(stream, graph, dependencies, dependencyData, numDependencies, mode) + +cdef cudaError_t _cudaThreadExchangeStreamCaptureMode(cudaStreamCaptureMode* mode) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaThreadExchangeStreamCaptureMode(mode) + +cdef cudaError_t _cudaStreamEndCapture(cudaStream_t stream, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamEndCapture(stream, pGraph) + +cdef cudaError_t _cudaStreamIsCapturing(cudaStream_t stream, cudaStreamCaptureStatus* pCaptureStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamIsCapturing(stream, pCaptureStatus) + +cdef cudaError_t _cudaStreamGetCaptureInfo_v2(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamGetCaptureInfo_v2_ptsz(stream, captureStatus_out, id_out, graph_out, dependencies_out, numDependencies_out) + +cdef cudaError_t _cudaStreamGetCaptureInfo_v3(cudaStream_t stream, cudaStreamCaptureStatus* captureStatus_out, unsigned long long* id_out, cudaGraph_t* graph_out, const cudaGraphNode_t** dependencies_out, const cudaGraphEdgeData** edgeData_out, size_t* numDependencies_out) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamGetCaptureInfo_v3(stream, captureStatus_out, id_out, graph_out, dependencies_out, edgeData_out, numDependencies_out) + +cdef cudaError_t _cudaStreamUpdateCaptureDependencies(cudaStream_t stream, cudaGraphNode_t* dependencies, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamUpdateCaptureDependencies(stream, dependencies, numDependencies, flags) + +cdef cudaError_t _cudaStreamUpdateCaptureDependencies_v2(cudaStream_t stream, cudaGraphNode_t* dependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaStreamUpdateCaptureDependencies_v2(stream, dependencies, dependencyData, numDependencies, flags) + +cdef cudaError_t _cudaEventCreate(cudaEvent_t* event) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaEventCreate(event) + +cdef cudaError_t _cudaEventCreateWithFlags(cudaEvent_t* event, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaEventCreateWithFlags(event, flags) + +cdef cudaError_t _cudaEventRecord(cudaEvent_t event, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaEventRecord(event, stream) + +cdef cudaError_t _cudaEventRecordWithFlags(cudaEvent_t event, cudaStream_t stream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaEventRecordWithFlags(event, stream, flags) + +cdef cudaError_t _cudaEventQuery(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaEventQuery(event) + +cdef cudaError_t _cudaEventSynchronize(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaEventSynchronize(event) + +cdef cudaError_t _cudaEventDestroy(cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaEventDestroy(event) + +cdef cudaError_t _cudaEventElapsedTime(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaEventElapsedTime(ms, start, end) + +cdef cudaError_t _cudaEventElapsedTime_v2(float* ms, cudaEvent_t start, cudaEvent_t end) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaEventElapsedTime_v2(ms, start, end) + +cdef cudaError_t _cudaImportExternalMemory(cudaExternalMemory_t* extMem_out, const cudaExternalMemoryHandleDesc* memHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaImportExternalMemory(extMem_out, memHandleDesc) + +cdef cudaError_t _cudaExternalMemoryGetMappedBuffer(void** devPtr, cudaExternalMemory_t extMem, const cudaExternalMemoryBufferDesc* bufferDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaExternalMemoryGetMappedBuffer(devPtr, extMem, bufferDesc) + +cdef cudaError_t _cudaExternalMemoryGetMappedMipmappedArray(cudaMipmappedArray_t* mipmap, cudaExternalMemory_t extMem, const cudaExternalMemoryMipmappedArrayDesc* mipmapDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaExternalMemoryGetMappedMipmappedArray(mipmap, extMem, mipmapDesc) + +cdef cudaError_t _cudaDestroyExternalMemory(cudaExternalMemory_t extMem) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDestroyExternalMemory(extMem) + +cdef cudaError_t _cudaImportExternalSemaphore(cudaExternalSemaphore_t* extSem_out, const cudaExternalSemaphoreHandleDesc* semHandleDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaImportExternalSemaphore(extSem_out, semHandleDesc) + +cdef cudaError_t _cudaSignalExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreSignalParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaSignalExternalSemaphoresAsync_v2_ptsz(extSemArray, paramsArray, numExtSems, stream) + +cdef cudaError_t _cudaWaitExternalSemaphoresAsync_v2(const cudaExternalSemaphore_t* extSemArray, const cudaExternalSemaphoreWaitParams* paramsArray, unsigned int numExtSems, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaWaitExternalSemaphoresAsync_v2_ptsz(extSemArray, paramsArray, numExtSems, stream) + +cdef cudaError_t _cudaDestroyExternalSemaphore(cudaExternalSemaphore_t extSem) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDestroyExternalSemaphore(extSem) + +cdef cudaError_t _cudaFuncSetCacheConfig(const void* func, cudaFuncCache cacheConfig) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaFuncSetCacheConfig(func, cacheConfig) + +cdef cudaError_t _cudaFuncGetAttributes(cudaFuncAttributes* attr, const void* func) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaFuncGetAttributes(attr, func) + +cdef cudaError_t _cudaFuncSetAttribute(const void* func, cudaFuncAttribute attr, int value) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaFuncSetAttribute(func, attr, value) + +cdef cudaError_t _cudaLaunchHostFunc(cudaStream_t stream, cudaHostFn_t fn, void* userData) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaLaunchHostFunc(stream, fn, userData) + +cdef cudaError_t _cudaFuncSetSharedMemConfig(const void* func, cudaSharedMemConfig config) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaFuncSetSharedMemConfig(func, config) + +cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessor(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaOccupancyMaxActiveBlocksPerMultiprocessor(numBlocks, func, blockSize, dynamicSMemSize) + +cdef cudaError_t _cudaOccupancyAvailableDynamicSMemPerBlock(size_t* dynamicSmemSize, const void* func, int numBlocks, int blockSize) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaOccupancyAvailableDynamicSMemPerBlock(dynamicSmemSize, func, numBlocks, blockSize) + +cdef cudaError_t _cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(int* numBlocks, const void* func, int blockSize, size_t dynamicSMemSize, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags(numBlocks, func, blockSize, dynamicSMemSize, flags) + +cdef cudaError_t _cudaMallocManaged(void** devPtr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMallocManaged(devPtr, size, flags) + +cdef cudaError_t _cudaMalloc(void** devPtr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMalloc(devPtr, size) + +cdef cudaError_t _cudaMallocHost(void** ptr, size_t size) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMallocHost(ptr, size) + +cdef cudaError_t _cudaMallocPitch(void** devPtr, size_t* pitch, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMallocPitch(devPtr, pitch, width, height) + +cdef cudaError_t _cudaMallocArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, size_t width, size_t height, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMallocArray(array, desc, width, height, flags) + +cdef cudaError_t _cudaFree(void* devPtr) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaFree(devPtr) + +cdef cudaError_t _cudaFreeHost(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaFreeHost(ptr) + +cdef cudaError_t _cudaFreeArray(cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaFreeArray(array) + +cdef cudaError_t _cudaFreeMipmappedArray(cudaMipmappedArray_t mipmappedArray) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaFreeMipmappedArray(mipmappedArray) + +cdef cudaError_t _cudaHostAlloc(void** pHost, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaHostAlloc(pHost, size, flags) + +cdef cudaError_t _cudaHostRegister(void* ptr, size_t size, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaHostRegister(ptr, size, flags) + +cdef cudaError_t _cudaHostUnregister(void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaHostUnregister(ptr) + +cdef cudaError_t _cudaHostGetDevicePointer(void** pDevice, void* pHost, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaHostGetDevicePointer(pDevice, pHost, flags) + +cdef cudaError_t _cudaHostGetFlags(unsigned int* pFlags, void* pHost) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaHostGetFlags(pFlags, pHost) + +cdef cudaError_t _cudaMalloc3D(cudaPitchedPtr* pitchedDevPtr, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMalloc3D(pitchedDevPtr, extent) + +cdef cudaError_t _cudaMalloc3DArray(cudaArray_t* array, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMalloc3DArray(array, desc, extent, flags) + +cdef cudaError_t _cudaMallocMipmappedArray(cudaMipmappedArray_t* mipmappedArray, const cudaChannelFormatDesc* desc, cudaExtent extent, unsigned int numLevels, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMallocMipmappedArray(mipmappedArray, desc, extent, numLevels, flags) + +cdef cudaError_t _cudaGetMipmappedArrayLevel(cudaArray_t* levelArray, cudaMipmappedArray_const_t mipmappedArray, unsigned int level) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetMipmappedArrayLevel(levelArray, mipmappedArray, level) + +cdef cudaError_t _cudaMemcpy3D(const cudaMemcpy3DParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpy3D(p) + +cdef cudaError_t _cudaMemcpy3DPeer(const cudaMemcpy3DPeerParms* p) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpy3DPeer(p) + +cdef cudaError_t _cudaMemcpy3DAsync(const cudaMemcpy3DParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpy3DAsync(p, stream) + +cdef cudaError_t _cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* p, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpy3DPeerAsync(p, stream) + +cdef cudaError_t _cudaMemGetInfo(size_t* free, size_t* total) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemGetInfo(free, total) + +cdef cudaError_t _cudaArrayGetInfo(cudaChannelFormatDesc* desc, cudaExtent* extent, unsigned int* flags, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaArrayGetInfo(desc, extent, flags, array) + +cdef cudaError_t _cudaArrayGetPlane(cudaArray_t* pPlaneArray, cudaArray_t hArray, unsigned int planeIdx) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaArrayGetPlane(pPlaneArray, hArray, planeIdx) + +cdef cudaError_t _cudaArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaArray_t array, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaArrayGetMemoryRequirements(memoryRequirements, array, device) + +cdef cudaError_t _cudaMipmappedArrayGetMemoryRequirements(cudaArrayMemoryRequirements* memoryRequirements, cudaMipmappedArray_t mipmap, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMipmappedArrayGetMemoryRequirements(memoryRequirements, mipmap, device) + +cdef cudaError_t _cudaArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaArray_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaArrayGetSparseProperties(sparseProperties, array) + +cdef cudaError_t _cudaMipmappedArrayGetSparseProperties(cudaArraySparseProperties* sparseProperties, cudaMipmappedArray_t mipmap) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMipmappedArrayGetSparseProperties(sparseProperties, mipmap) + +cdef cudaError_t _cudaMemcpy(void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpy(dst, src, count, kind) + +cdef cudaError_t _cudaMemcpyPeer(void* dst, int dstDevice, const void* src, int srcDevice, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpyPeer(dst, dstDevice, src, srcDevice, count) + +cdef cudaError_t _cudaMemcpy2D(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpy2D(dst, dpitch, src, spitch, width, height, kind) + +cdef cudaError_t _cudaMemcpy2DToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpy2DToArray(dst, wOffset, hOffset, src, spitch, width, height, kind) + +cdef cudaError_t _cudaMemcpy2DFromArray(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpy2DFromArray(dst, dpitch, src, wOffset, hOffset, width, height, kind) + +cdef cudaError_t _cudaMemcpy2DArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t width, size_t height, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpy2DArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, width, height, kind) + +cdef cudaError_t _cudaMemcpyAsync(void* dst, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpyAsync(dst, src, count, kind, stream) + +cdef cudaError_t _cudaMemcpyPeerAsync(void* dst, int dstDevice, const void* src, int srcDevice, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpyPeerAsync(dst, dstDevice, src, srcDevice, count, stream) + +cdef cudaError_t _cudaMemcpyBatchAsync(void** dsts, void** srcs, size_t* sizes, size_t count, cudaMemcpyAttributes* attrs, size_t* attrsIdxs, size_t numAttrs, size_t* failIdx, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpyBatchAsync(dsts, srcs, sizes, count, attrs, attrsIdxs, numAttrs, failIdx, stream) + +cdef cudaError_t _cudaMemcpy3DBatchAsync(size_t numOps, cudaMemcpy3DBatchOp* opList, size_t* failIdx, unsigned long long flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpy3DBatchAsync(numOps, opList, failIdx, flags, stream) + +cdef cudaError_t _cudaMemcpy2DAsync(void* dst, size_t dpitch, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpy2DAsync(dst, dpitch, src, spitch, width, height, kind, stream) + +cdef cudaError_t _cudaMemcpy2DToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t spitch, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpy2DToArrayAsync(dst, wOffset, hOffset, src, spitch, width, height, kind, stream) + +cdef cudaError_t _cudaMemcpy2DFromArrayAsync(void* dst, size_t dpitch, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpy2DFromArrayAsync(dst, dpitch, src, wOffset, hOffset, width, height, kind, stream) + +cdef cudaError_t _cudaMemset(void* devPtr, int value, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemset(devPtr, value, count) + +cdef cudaError_t _cudaMemset2D(void* devPtr, size_t pitch, int value, size_t width, size_t height) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemset2D(devPtr, pitch, value, width, height) + +cdef cudaError_t _cudaMemset3D(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemset3D(pitchedDevPtr, value, extent) + +cdef cudaError_t _cudaMemsetAsync(void* devPtr, int value, size_t count, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemsetAsync(devPtr, value, count, stream) + +cdef cudaError_t _cudaMemset2DAsync(void* devPtr, size_t pitch, int value, size_t width, size_t height, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemset2DAsync(devPtr, pitch, value, width, height, stream) + +cdef cudaError_t _cudaMemset3DAsync(cudaPitchedPtr pitchedDevPtr, int value, cudaExtent extent, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemset3DAsync(pitchedDevPtr, value, extent, stream) + +cdef cudaError_t _cudaMemPrefetchAsync(const void* devPtr, size_t count, int dstDevice, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemPrefetchAsync(devPtr, count, dstDevice, stream) + +cdef cudaError_t _cudaMemPrefetchAsync_v2(const void* devPtr, size_t count, cudaMemLocation location, unsigned int flags, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemPrefetchAsync_v2(devPtr, count, location, flags, stream) + +cdef cudaError_t _cudaMemAdvise(const void* devPtr, size_t count, cudaMemoryAdvise advice, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemAdvise(devPtr, count, advice, device) + +cdef cudaError_t _cudaMemAdvise_v2(const void* devPtr, size_t count, cudaMemoryAdvise advice, cudaMemLocation location) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemAdvise_v2(devPtr, count, advice, location) + +cdef cudaError_t _cudaMemRangeGetAttribute(void* data, size_t dataSize, cudaMemRangeAttribute attribute, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemRangeGetAttribute(data, dataSize, attribute, devPtr, count) + +cdef cudaError_t _cudaMemRangeGetAttributes(void** data, size_t* dataSizes, cudaMemRangeAttribute* attributes, size_t numAttributes, const void* devPtr, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemRangeGetAttributes(data, dataSizes, attributes, numAttributes, devPtr, count) + +cdef cudaError_t _cudaMemcpyToArray(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpyToArray(dst, wOffset, hOffset, src, count, kind) + +cdef cudaError_t _cudaMemcpyFromArray(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpyFromArray(dst, src, wOffset, hOffset, count, kind) + +cdef cudaError_t _cudaMemcpyArrayToArray(cudaArray_t dst, size_t wOffsetDst, size_t hOffsetDst, cudaArray_const_t src, size_t wOffsetSrc, size_t hOffsetSrc, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpyArrayToArray(dst, wOffsetDst, hOffsetDst, src, wOffsetSrc, hOffsetSrc, count, kind) + +cdef cudaError_t _cudaMemcpyToArrayAsync(cudaArray_t dst, size_t wOffset, size_t hOffset, const void* src, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpyToArrayAsync(dst, wOffset, hOffset, src, count, kind, stream) + +cdef cudaError_t _cudaMemcpyFromArrayAsync(void* dst, cudaArray_const_t src, size_t wOffset, size_t hOffset, size_t count, cudaMemcpyKind kind, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemcpyFromArrayAsync(dst, src, wOffset, hOffset, count, kind, stream) + +cdef cudaError_t _cudaMallocAsync(void** devPtr, size_t size, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMallocAsync(devPtr, size, hStream) + +cdef cudaError_t _cudaFreeAsync(void* devPtr, cudaStream_t hStream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaFreeAsync(devPtr, hStream) + +cdef cudaError_t _cudaMemPoolTrimTo(cudaMemPool_t memPool, size_t minBytesToKeep) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemPoolTrimTo(memPool, minBytesToKeep) + +cdef cudaError_t _cudaMemPoolSetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemPoolSetAttribute(memPool, attr, value) + +cdef cudaError_t _cudaMemPoolGetAttribute(cudaMemPool_t memPool, cudaMemPoolAttr attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemPoolGetAttribute(memPool, attr, value) + +cdef cudaError_t _cudaMemPoolSetAccess(cudaMemPool_t memPool, const cudaMemAccessDesc* descList, size_t count) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemPoolSetAccess(memPool, descList, count) + +cdef cudaError_t _cudaMemPoolGetAccess(cudaMemAccessFlags* flags, cudaMemPool_t memPool, cudaMemLocation* location) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemPoolGetAccess(flags, memPool, location) + +cdef cudaError_t _cudaMemPoolCreate(cudaMemPool_t* memPool, const cudaMemPoolProps* poolProps) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemPoolCreate(memPool, poolProps) + +cdef cudaError_t _cudaMemPoolDestroy(cudaMemPool_t memPool) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemPoolDestroy(memPool) + +cdef cudaError_t _cudaMallocFromPoolAsync(void** ptr, size_t size, cudaMemPool_t memPool, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMallocFromPoolAsync(ptr, size, memPool, stream) + +cdef cudaError_t _cudaMemPoolExportToShareableHandle(void* shareableHandle, cudaMemPool_t memPool, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemPoolExportToShareableHandle(shareableHandle, memPool, handleType, flags) + +cdef cudaError_t _cudaMemPoolImportFromShareableHandle(cudaMemPool_t* memPool, void* shareableHandle, cudaMemAllocationHandleType handleType, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemPoolImportFromShareableHandle(memPool, shareableHandle, handleType, flags) + +cdef cudaError_t _cudaMemPoolExportPointer(cudaMemPoolPtrExportData* exportData, void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemPoolExportPointer(exportData, ptr) + +cdef cudaError_t _cudaMemPoolImportPointer(void** ptr, cudaMemPool_t memPool, cudaMemPoolPtrExportData* exportData) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaMemPoolImportPointer(ptr, memPool, exportData) + +cdef cudaError_t _cudaPointerGetAttributes(cudaPointerAttributes* attributes, const void* ptr) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaPointerGetAttributes(attributes, ptr) + +cdef cudaError_t _cudaDeviceCanAccessPeer(int* canAccessPeer, int device, int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceCanAccessPeer(canAccessPeer, device, peerDevice) + +cdef cudaError_t _cudaDeviceEnablePeerAccess(int peerDevice, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceEnablePeerAccess(peerDevice, flags) + +cdef cudaError_t _cudaDeviceDisablePeerAccess(int peerDevice) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceDisablePeerAccess(peerDevice) + +cdef cudaError_t _cudaGraphicsUnregisterResource(cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphicsUnregisterResource(resource) + +cdef cudaError_t _cudaGraphicsResourceSetMapFlags(cudaGraphicsResource_t resource, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphicsResourceSetMapFlags(resource, flags) + +cdef cudaError_t _cudaGraphicsMapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphicsMapResources(count, resources, stream) + +cdef cudaError_t _cudaGraphicsUnmapResources(int count, cudaGraphicsResource_t* resources, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphicsUnmapResources(count, resources, stream) + +cdef cudaError_t _cudaGraphicsResourceGetMappedPointer(void** devPtr, size_t* size, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphicsResourceGetMappedPointer(devPtr, size, resource) + +cdef cudaError_t _cudaGraphicsSubResourceGetMappedArray(cudaArray_t* array, cudaGraphicsResource_t resource, unsigned int arrayIndex, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphicsSubResourceGetMappedArray(array, resource, arrayIndex, mipLevel) + +cdef cudaError_t _cudaGraphicsResourceGetMappedMipmappedArray(cudaMipmappedArray_t* mipmappedArray, cudaGraphicsResource_t resource) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphicsResourceGetMappedMipmappedArray(mipmappedArray, resource) + +cdef cudaError_t _cudaGetChannelDesc(cudaChannelFormatDesc* desc, cudaArray_const_t array) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetChannelDesc(desc, array) +@cython.show_performance_hints(False) +cdef cudaChannelFormatDesc _cudaCreateChannelDesc(int x, int y, int z, int w, cudaChannelFormatKind f) except* nogil: + return cudaCreateChannelDesc(x, y, z, w, f) + +cdef cudaError_t _cudaCreateTextureObject(cudaTextureObject_t* pTexObject, const cudaResourceDesc* pResDesc, const cudaTextureDesc* pTexDesc, const cudaResourceViewDesc* pResViewDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaCreateTextureObject(pTexObject, pResDesc, pTexDesc, pResViewDesc) + +cdef cudaError_t _cudaDestroyTextureObject(cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDestroyTextureObject(texObject) + +cdef cudaError_t _cudaGetTextureObjectResourceDesc(cudaResourceDesc* pResDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetTextureObjectResourceDesc(pResDesc, texObject) + +cdef cudaError_t _cudaGetTextureObjectTextureDesc(cudaTextureDesc* pTexDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetTextureObjectTextureDesc(pTexDesc, texObject) + +cdef cudaError_t _cudaGetTextureObjectResourceViewDesc(cudaResourceViewDesc* pResViewDesc, cudaTextureObject_t texObject) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetTextureObjectResourceViewDesc(pResViewDesc, texObject) + +cdef cudaError_t _cudaCreateSurfaceObject(cudaSurfaceObject_t* pSurfObject, const cudaResourceDesc* pResDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaCreateSurfaceObject(pSurfObject, pResDesc) + +cdef cudaError_t _cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDestroySurfaceObject(surfObject) + +cdef cudaError_t _cudaGetSurfaceObjectResourceDesc(cudaResourceDesc* pResDesc, cudaSurfaceObject_t surfObject) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetSurfaceObjectResourceDesc(pResDesc, surfObject) + +cdef cudaError_t _cudaDriverGetVersion(int* driverVersion) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDriverGetVersion(driverVersion) + +cdef cudaError_t _cudaRuntimeGetVersion(int* runtimeVersion) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaRuntimeGetVersion(runtimeVersion) + +cdef cudaError_t _cudaGraphCreate(cudaGraph_t* pGraph, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphCreate(pGraph, flags) + +cdef cudaError_t _cudaGraphAddKernelNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddKernelNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + +cdef cudaError_t _cudaGraphKernelNodeGetParams(cudaGraphNode_t node, cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphKernelNodeGetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphKernelNodeSetParams(cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphKernelNodeSetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphKernelNodeCopyAttributes(cudaGraphNode_t hSrc, cudaGraphNode_t hDst) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphKernelNodeCopyAttributes(hSrc, hDst) + +cdef cudaError_t _cudaGraphKernelNodeGetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, cudaKernelNodeAttrValue* value_out) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphKernelNodeGetAttribute(hNode, attr, value_out) + +cdef cudaError_t _cudaGraphKernelNodeSetAttribute(cudaGraphNode_t hNode, cudaKernelNodeAttrID attr, const cudaKernelNodeAttrValue* value) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphKernelNodeSetAttribute(hNode, attr, value) + +cdef cudaError_t _cudaGraphAddMemcpyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemcpy3DParms* pCopyParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddMemcpyNode(pGraphNode, graph, pDependencies, numDependencies, pCopyParams) + +cdef cudaError_t _cudaGraphAddMemcpyNode1D(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddMemcpyNode1D(pGraphNode, graph, pDependencies, numDependencies, dst, src, count, kind) + +cdef cudaError_t _cudaGraphMemcpyNodeGetParams(cudaGraphNode_t node, cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphMemcpyNodeGetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphMemcpyNodeSetParams(cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphMemcpyNodeSetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphMemcpyNodeSetParams1D(cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphMemcpyNodeSetParams1D(node, dst, src, count, kind) + +cdef cudaError_t _cudaGraphAddMemsetNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaMemsetParams* pMemsetParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddMemsetNode(pGraphNode, graph, pDependencies, numDependencies, pMemsetParams) + +cdef cudaError_t _cudaGraphMemsetNodeGetParams(cudaGraphNode_t node, cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphMemsetNodeGetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphMemsetNodeSetParams(cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphMemsetNodeSetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphAddHostNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddHostNode(pGraphNode, graph, pDependencies, numDependencies, pNodeParams) + +cdef cudaError_t _cudaGraphHostNodeGetParams(cudaGraphNode_t node, cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphHostNodeGetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphHostNodeSetParams(cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphHostNodeSetParams(node, pNodeParams) + +cdef cudaError_t _cudaGraphAddChildGraphNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddChildGraphNode(pGraphNode, graph, pDependencies, numDependencies, childGraph) + +cdef cudaError_t _cudaGraphChildGraphNodeGetGraph(cudaGraphNode_t node, cudaGraph_t* pGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphChildGraphNodeGetGraph(node, pGraph) + +cdef cudaError_t _cudaGraphAddEmptyNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddEmptyNode(pGraphNode, graph, pDependencies, numDependencies) + +cdef cudaError_t _cudaGraphAddEventRecordNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddEventRecordNode(pGraphNode, graph, pDependencies, numDependencies, event) + +cdef cudaError_t _cudaGraphEventRecordNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphEventRecordNodeGetEvent(node, event_out) + +cdef cudaError_t _cudaGraphEventRecordNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphEventRecordNodeSetEvent(node, event) + +cdef cudaError_t _cudaGraphAddEventWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddEventWaitNode(pGraphNode, graph, pDependencies, numDependencies, event) + +cdef cudaError_t _cudaGraphEventWaitNodeGetEvent(cudaGraphNode_t node, cudaEvent_t* event_out) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphEventWaitNodeGetEvent(node, event_out) + +cdef cudaError_t _cudaGraphEventWaitNodeSetEvent(cudaGraphNode_t node, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphEventWaitNodeSetEvent(node, event) + +cdef cudaError_t _cudaGraphAddExternalSemaphoresSignalNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddExternalSemaphoresSignalNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + +cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreSignalNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExternalSemaphoresSignalNodeGetParams(hNode, params_out) + +cdef cudaError_t _cudaGraphExternalSemaphoresSignalNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExternalSemaphoresSignalNodeSetParams(hNode, nodeParams) + +cdef cudaError_t _cudaGraphAddExternalSemaphoresWaitNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddExternalSemaphoresWaitNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + +cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeGetParams(cudaGraphNode_t hNode, cudaExternalSemaphoreWaitNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExternalSemaphoresWaitNodeGetParams(hNode, params_out) + +cdef cudaError_t _cudaGraphExternalSemaphoresWaitNodeSetParams(cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExternalSemaphoresWaitNodeSetParams(hNode, nodeParams) + +cdef cudaError_t _cudaGraphAddMemAllocNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaMemAllocNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddMemAllocNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + +cdef cudaError_t _cudaGraphMemAllocNodeGetParams(cudaGraphNode_t node, cudaMemAllocNodeParams* params_out) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphMemAllocNodeGetParams(node, params_out) + +cdef cudaError_t _cudaGraphAddMemFreeNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, void* dptr) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddMemFreeNode(pGraphNode, graph, pDependencies, numDependencies, dptr) + +cdef cudaError_t _cudaGraphMemFreeNodeGetParams(cudaGraphNode_t node, void* dptr_out) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphMemFreeNodeGetParams(node, dptr_out) + +cdef cudaError_t _cudaDeviceGraphMemTrim(int device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGraphMemTrim(device) + +cdef cudaError_t _cudaDeviceGetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceGetGraphMemAttribute(device, attr, value) + +cdef cudaError_t _cudaDeviceSetGraphMemAttribute(int device, cudaGraphMemAttributeType attr, void* value) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaDeviceSetGraphMemAttribute(device, attr, value) + +cdef cudaError_t _cudaGraphClone(cudaGraph_t* pGraphClone, cudaGraph_t originalGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphClone(pGraphClone, originalGraph) + +cdef cudaError_t _cudaGraphNodeFindInClone(cudaGraphNode_t* pNode, cudaGraphNode_t originalNode, cudaGraph_t clonedGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphNodeFindInClone(pNode, originalNode, clonedGraph) + +cdef cudaError_t _cudaGraphNodeGetType(cudaGraphNode_t node, cudaGraphNodeType* pType) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphNodeGetType(node, pType) + +cdef cudaError_t _cudaGraphGetNodes(cudaGraph_t graph, cudaGraphNode_t* nodes, size_t* numNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphGetNodes(graph, nodes, numNodes) + +cdef cudaError_t _cudaGraphGetRootNodes(cudaGraph_t graph, cudaGraphNode_t* pRootNodes, size_t* pNumRootNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphGetRootNodes(graph, pRootNodes, pNumRootNodes) + +cdef cudaError_t _cudaGraphGetEdges(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphGetEdges(graph, from_, to, numEdges) + +cdef cudaError_t _cudaGraphGetEdges_v2(cudaGraph_t graph, cudaGraphNode_t* from_, cudaGraphNode_t* to, cudaGraphEdgeData* edgeData, size_t* numEdges) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphGetEdges_v2(graph, from_, to, edgeData, numEdges) + +cdef cudaError_t _cudaGraphNodeGetDependencies(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphNodeGetDependencies(node, pDependencies, pNumDependencies) + +cdef cudaError_t _cudaGraphNodeGetDependencies_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependencies, cudaGraphEdgeData* edgeData, size_t* pNumDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphNodeGetDependencies_v2(node, pDependencies, edgeData, pNumDependencies) + +cdef cudaError_t _cudaGraphNodeGetDependentNodes(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphNodeGetDependentNodes(node, pDependentNodes, pNumDependentNodes) + +cdef cudaError_t _cudaGraphNodeGetDependentNodes_v2(cudaGraphNode_t node, cudaGraphNode_t* pDependentNodes, cudaGraphEdgeData* edgeData, size_t* pNumDependentNodes) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphNodeGetDependentNodes_v2(node, pDependentNodes, edgeData, pNumDependentNodes) + +cdef cudaError_t _cudaGraphAddDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddDependencies(graph, from_, to, numDependencies) + +cdef cudaError_t _cudaGraphAddDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddDependencies_v2(graph, from_, to, edgeData, numDependencies) + +cdef cudaError_t _cudaGraphRemoveDependencies(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphRemoveDependencies(graph, from_, to, numDependencies) + +cdef cudaError_t _cudaGraphRemoveDependencies_v2(cudaGraph_t graph, const cudaGraphNode_t* from_, const cudaGraphNode_t* to, const cudaGraphEdgeData* edgeData, size_t numDependencies) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphRemoveDependencies_v2(graph, from_, to, edgeData, numDependencies) + +cdef cudaError_t _cudaGraphDestroyNode(cudaGraphNode_t node) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphDestroyNode(node) + +cdef cudaError_t _cudaGraphInstantiate(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphInstantiate(pGraphExec, graph, flags) + +cdef cudaError_t _cudaGraphInstantiateWithFlags(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, unsigned long long flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphInstantiateWithFlags(pGraphExec, graph, flags) + +cdef cudaError_t _cudaGraphInstantiateWithParams(cudaGraphExec_t* pGraphExec, cudaGraph_t graph, cudaGraphInstantiateParams* instantiateParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphInstantiateWithParams(pGraphExec, graph, instantiateParams) + +cdef cudaError_t _cudaGraphExecGetFlags(cudaGraphExec_t graphExec, unsigned long long* flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecGetFlags(graphExec, flags) + +cdef cudaError_t _cudaGraphExecKernelNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaKernelNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecKernelNodeSetParams(hGraphExec, node, pNodeParams) + +cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemcpy3DParms* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecMemcpyNodeSetParams(hGraphExec, node, pNodeParams) + +cdef cudaError_t _cudaGraphExecMemcpyNodeSetParams1D(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, void* dst, const void* src, size_t count, cudaMemcpyKind kind) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecMemcpyNodeSetParams1D(hGraphExec, node, dst, src, count, kind) + +cdef cudaError_t _cudaGraphExecMemsetNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaMemsetParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecMemsetNodeSetParams(hGraphExec, node, pNodeParams) + +cdef cudaError_t _cudaGraphExecHostNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, const cudaHostNodeParams* pNodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecHostNodeSetParams(hGraphExec, node, pNodeParams) + +cdef cudaError_t _cudaGraphExecChildGraphNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t node, cudaGraph_t childGraph) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecChildGraphNodeSetParams(hGraphExec, node, childGraph) + +cdef cudaError_t _cudaGraphExecEventRecordNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecEventRecordNodeSetEvent(hGraphExec, hNode, event) + +cdef cudaError_t _cudaGraphExecEventWaitNodeSetEvent(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, cudaEvent_t event) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecEventWaitNodeSetEvent(hGraphExec, hNode, event) + +cdef cudaError_t _cudaGraphExecExternalSemaphoresSignalNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreSignalNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecExternalSemaphoresSignalNodeSetParams(hGraphExec, hNode, nodeParams) + +cdef cudaError_t _cudaGraphExecExternalSemaphoresWaitNodeSetParams(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, const cudaExternalSemaphoreWaitNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecExternalSemaphoresWaitNodeSetParams(hGraphExec, hNode, nodeParams) + +cdef cudaError_t _cudaGraphNodeSetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphNodeSetEnabled(hGraphExec, hNode, isEnabled) + +cdef cudaError_t _cudaGraphNodeGetEnabled(cudaGraphExec_t hGraphExec, cudaGraphNode_t hNode, unsigned int* isEnabled) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphNodeGetEnabled(hGraphExec, hNode, isEnabled) + +cdef cudaError_t _cudaGraphExecUpdate(cudaGraphExec_t hGraphExec, cudaGraph_t hGraph, cudaGraphExecUpdateResultInfo* resultInfo) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecUpdate(hGraphExec, hGraph, resultInfo) + +cdef cudaError_t _cudaGraphUpload(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphUpload(graphExec, stream) + +cdef cudaError_t _cudaGraphLaunch(cudaGraphExec_t graphExec, cudaStream_t stream) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphLaunch(graphExec, stream) + +cdef cudaError_t _cudaGraphExecDestroy(cudaGraphExec_t graphExec) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecDestroy(graphExec) + +cdef cudaError_t _cudaGraphDestroy(cudaGraph_t graph) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphDestroy(graph) + +cdef cudaError_t _cudaGraphDebugDotPrint(cudaGraph_t graph, const char* path, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphDebugDotPrint(graph, path, flags) + +cdef cudaError_t _cudaUserObjectCreate(cudaUserObject_t* object_out, void* ptr, cudaHostFn_t destroy, unsigned int initialRefcount, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaUserObjectCreate(object_out, ptr, destroy, initialRefcount, flags) + +cdef cudaError_t _cudaUserObjectRetain(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaUserObjectRetain(object, count) + +cdef cudaError_t _cudaUserObjectRelease(cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaUserObjectRelease(object, count) + +cdef cudaError_t _cudaGraphRetainUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphRetainUserObject(graph, object, count, flags) + +cdef cudaError_t _cudaGraphReleaseUserObject(cudaGraph_t graph, cudaUserObject_t object, unsigned int count) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphReleaseUserObject(graph, object, count) + +cdef cudaError_t _cudaGraphAddNode(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddNode(pGraphNode, graph, pDependencies, numDependencies, nodeParams) + +cdef cudaError_t _cudaGraphAddNode_v2(cudaGraphNode_t* pGraphNode, cudaGraph_t graph, const cudaGraphNode_t* pDependencies, const cudaGraphEdgeData* dependencyData, size_t numDependencies, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphAddNode_v2(pGraphNode, graph, pDependencies, dependencyData, numDependencies, nodeParams) + +cdef cudaError_t _cudaGraphNodeSetParams(cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphNodeSetParams(node, nodeParams) + +cdef cudaError_t _cudaGraphExecNodeSetParams(cudaGraphExec_t graphExec, cudaGraphNode_t node, cudaGraphNodeParams* nodeParams) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphExecNodeSetParams(graphExec, node, nodeParams) + +cdef cudaError_t _cudaGraphConditionalHandleCreate(cudaGraphConditionalHandle* pHandle_out, cudaGraph_t graph, unsigned int defaultLaunchValue, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGraphConditionalHandleCreate(pHandle_out, graph, defaultLaunchValue, flags) + +cdef cudaError_t _cudaGetDriverEntryPoint(const char* symbol, void** funcPtr, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetDriverEntryPoint(symbol, funcPtr, flags, driverStatus) + +cdef cudaError_t _cudaGetDriverEntryPointByVersion(const char* symbol, void** funcPtr, unsigned int cudaVersion, unsigned long long flags, cudaDriverEntryPointQueryResult* driverStatus) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetDriverEntryPointByVersion(symbol, funcPtr, cudaVersion, flags, driverStatus) + +cdef cudaError_t _cudaLibraryLoadData(cudaLibrary_t* library, const void* code, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaLibraryLoadData(library, code, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + +cdef cudaError_t _cudaLibraryLoadFromFile(cudaLibrary_t* library, const char* fileName, cudaJitOption* jitOptions, void** jitOptionsValues, unsigned int numJitOptions, cudaLibraryOption* libraryOptions, void** libraryOptionValues, unsigned int numLibraryOptions) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaLibraryLoadFromFile(library, fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions) + +cdef cudaError_t _cudaLibraryUnload(cudaLibrary_t library) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaLibraryUnload(library) + +cdef cudaError_t _cudaLibraryGetKernel(cudaKernel_t* pKernel, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaLibraryGetKernel(pKernel, library, name) + +cdef cudaError_t _cudaLibraryGetGlobal(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaLibraryGetGlobal(dptr, numbytes, library, name) + +cdef cudaError_t _cudaLibraryGetManaged(void** dptr, size_t* numbytes, cudaLibrary_t library, const char* name) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaLibraryGetManaged(dptr, numbytes, library, name) + +cdef cudaError_t _cudaLibraryGetUnifiedFunction(void** fptr, cudaLibrary_t library, const char* symbol) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaLibraryGetUnifiedFunction(fptr, library, symbol) + +cdef cudaError_t _cudaLibraryGetKernelCount(unsigned int* count, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaLibraryGetKernelCount(count, lib) + +cdef cudaError_t _cudaLibraryEnumerateKernels(cudaKernel_t* kernels, unsigned int numKernels, cudaLibrary_t lib) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaLibraryEnumerateKernels(kernels, numKernels, lib) + +cdef cudaError_t _cudaKernelSetAttributeForDevice(cudaKernel_t kernel, cudaFuncAttribute attr, int value, int device) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaKernelSetAttributeForDevice(kernel, attr, value, device) + +cdef cudaError_t _cudaGetExportTable(const void** ppExportTable, const cudaUUID_t* pExportTableId) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetExportTable(ppExportTable, pExportTableId) + +cdef cudaError_t _cudaGetKernel(cudaKernel_t* kernelPtr, const void* entryFuncAddr) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaGetKernel(kernelPtr, entryFuncAddr) +@cython.show_performance_hints(False) +cdef cudaPitchedPtr _make_cudaPitchedPtr(void* d, size_t p, size_t xsz, size_t ysz) except* nogil: + return make_cudaPitchedPtr(d, p, xsz, ysz) +@cython.show_performance_hints(False) +cdef cudaPos _make_cudaPos(size_t x, size_t y, size_t z) except* nogil: + return make_cudaPos(x, y, z) +@cython.show_performance_hints(False) +cdef cudaExtent _make_cudaExtent(size_t w, size_t h, size_t d) except* nogil: + return make_cudaExtent(w, h, d) + +cdef cudaError_t _cudaProfilerStart() except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaProfilerStart() + +cdef cudaError_t _cudaProfilerStop() except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaProfilerStop() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/loader.cpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/loader.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a692eddc931316c72b42e975eb9f33f3a1b7a0ea --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/loader.cpp @@ -0,0 +1,350 @@ +// SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +#include +#include +#include +#include "loader.h" + +#define DXCORE_MAX_PATH 260 + +#if defined(_WIN32) +#include "windows.h" +#define _getAddr GetProcAddress +#define _Handle HMODULE +static const size_t sysrootName64_length = (sizeof("System32") - 1); +static const char* sysrootName64 = "System32"; +static const size_t libcudaName64_length = (sizeof("\\nvcuda64.dll") - 1); +static const char* libcudaName64 = "\\nvcuda64.dll"; +static const size_t sysrootNameX86_length = (sizeof("SysWOW64") - 1); +static const char* sysrootNameX86 = "SysWOW64"; +static const size_t libcudaNameX86_length = (sizeof("\\nvcuda32.dll") - 1); +static const char* libcudaNameX86 = "\\nvcuda32.dll"; +static size_t sysrootName_length = NULL; +static const char* sysrootName = NULL; + +#else +#include +#include +#define _getAddr dlsym +#define _Handle void* +static const size_t libcudaNameLinux_length = (sizeof("/libcuda.so.1.1") - 1); +static const char* libcudaNameLinux = "/libcuda.so.1.1"; +#endif +static size_t libcudaName_length = 0; +static const char* libcudaName = NULL; + +struct dxcore_enumAdapters2; +struct dxcore_queryAdapterInfo; + +typedef int (*pfnDxcoreEnumAdapters2)(const dxcore_enumAdapters2 *pParams); +typedef int (*pfnDxcoreQueryAdapterInfo)(const dxcore_queryAdapterInfo *pParams); + +struct dxcore_lib { + _Handle hDxcoreLib; + pfnDxcoreEnumAdapters2 pDxcoreEnumAdapters2; + pfnDxcoreQueryAdapterInfo pDxcoreQueryAdapterInfo; +}; + +struct dxcore_luid +{ + unsigned int lowPart; + int highPart; +}; + +struct dxcore_adapterInfo +{ + unsigned int hAdapter; + struct dxcore_luid AdapterLuid; + unsigned int NumOfSources; + unsigned int bPresentMoveRegionsPreferred; +}; + +struct dxcore_enumAdapters2 +{ + unsigned int NumAdapters; + struct dxcore_adapterInfo *pAdapters; +}; + +enum dxcore_kmtqueryAdapterInfoType +{ + DXCORE_QUERYDRIVERVERSION = 13, + DXCORE_QUERYREGISTRY = 48, +}; + +enum dxcore_queryregistry_type { + DXCORE_QUERYREGISTRY_DRIVERSTOREPATH = 2, +}; + +enum dxcore_queryregistry_status { + DXCORE_QUERYREGISTRY_STATUS_SUCCESS = 0, + DXCORE_QUERYREGISTRY_STATUS_BUFFER_OVERFLOW = 1, + DXCORE_QUERYREGISTRY_STATUS_FAIL = 2, +}; + +struct dxcore_queryregistry_info { + enum dxcore_queryregistry_type QueryType; + unsigned int QueryFlags; + wchar_t ValueName[DXCORE_MAX_PATH]; + unsigned int ValueType; + unsigned int PhysicalAdapterIndex; + unsigned int OutputValueSize; + enum dxcore_queryregistry_status Status; + union { + unsigned long long OutputQword; + wchar_t Output; + }; +}; + +struct dxcore_queryAdapterInfo +{ + unsigned int hAdapter; + enum dxcore_kmtqueryAdapterInfoType Type; + void *pPrivateDriverData; + unsigned int PrivateDriverDataSize; +}; + +static int dxcore_query_adapter_info_helper(struct dxcore_lib* pLib, + unsigned int hAdapter, + enum dxcore_kmtqueryAdapterInfoType type, + void* pPrivateDriverDate, + unsigned int privateDriverDataSize) +{ + struct dxcore_queryAdapterInfo queryAdapterInfo = {}; + + queryAdapterInfo.hAdapter = hAdapter; + queryAdapterInfo.Type = type; + queryAdapterInfo.pPrivateDriverData = pPrivateDriverDate; + queryAdapterInfo.PrivateDriverDataSize = privateDriverDataSize; + + return pLib->pDxcoreQueryAdapterInfo(&queryAdapterInfo); +} + +static int dxcore_query_adapter_wddm_version(struct dxcore_lib* pLib, unsigned int hAdapter, unsigned int* version) +{ + return dxcore_query_adapter_info_helper(pLib, + hAdapter, + DXCORE_QUERYDRIVERVERSION, + (void*)version, + (unsigned int)sizeof(*version)); +} + +static int dxcore_query_adapter_driverstore_path(struct dxcore_lib* pLib, unsigned int hAdapter, char** ppDriverStorePath) +{ + struct dxcore_queryregistry_info params = {}; + struct dxcore_queryregistry_info* pValue = NULL; + wchar_t* pOutput; + size_t outputSizeInBytes; + size_t outputSize; + + // 1. Fetch output size + params.QueryType = DXCORE_QUERYREGISTRY_DRIVERSTOREPATH; + + if (dxcore_query_adapter_info_helper(pLib, + hAdapter, + DXCORE_QUERYREGISTRY, + (void*)¶ms, + (unsigned int)sizeof(struct dxcore_queryregistry_info))) + { + return (-1); + } + + if (params.OutputValueSize > DXCORE_MAX_PATH * sizeof(wchar_t)) { + return (-1); + } + + outputSizeInBytes = (size_t)params.OutputValueSize; + outputSize = outputSizeInBytes / sizeof(wchar_t); + + // 2. Retrieve output + pValue = (struct dxcore_queryregistry_info*)calloc(sizeof(struct dxcore_queryregistry_info) + outputSizeInBytes + sizeof(wchar_t), 1); + if (!pValue) { + return (-1); + } + + pValue->QueryType = DXCORE_QUERYREGISTRY_DRIVERSTOREPATH; + pValue->OutputValueSize = (unsigned int)outputSizeInBytes; + + if (dxcore_query_adapter_info_helper(pLib, + hAdapter, + DXCORE_QUERYREGISTRY, + (void*)pValue, + (unsigned int)(sizeof(struct dxcore_queryregistry_info) + outputSizeInBytes))) + { + free(pValue); + return (-1); + } + pOutput = (wchar_t*)(&pValue->Output); + + // Make sure no matter what happened the wchar_t string is null terminated + pOutput[outputSize] = L'\0'; + + // Convert the output into a regular c string + *ppDriverStorePath = (char*)calloc(outputSize + 1, sizeof(char)); + if (!*ppDriverStorePath) { + free(pValue); + return (-1); + } + wcstombs(*ppDriverStorePath, pOutput, outputSize); + + free(pValue); + + return 0; +} + +static char* replaceSystemPath(char* path) +{ + char *replacedPath = (char*)calloc(DXCORE_MAX_PATH + 1, sizeof(char)); + +#if defined(_WIN32) + wchar_t *systemPath = (wchar_t*)calloc(DXCORE_MAX_PATH + 1, sizeof(wchar_t)); + // Get system root path + if (GetSystemDirectoryW(systemPath, DXCORE_MAX_PATH) == 0) { + free(replacedPath); + free(systemPath); + return NULL; + } + wcstombs(replacedPath, systemPath, DXCORE_MAX_PATH); + free(systemPath); + + // Replace the /SystemRoot/ part of the registry-obtained path with + // the actual system root path from above + char* sysrootPath = strstr(path, sysrootName); + strncat(replacedPath, sysrootPath + sysrootName_length, DXCORE_MAX_PATH - strlen(replacedPath)); +#else + strncat(replacedPath, path, DXCORE_MAX_PATH); +#endif + + // Append nvcuda dll + if (libcudaName_length < DXCORE_MAX_PATH - strlen(replacedPath)) { + strncat(replacedPath, libcudaName, libcudaName_length); + } + else { + strncat(replacedPath, libcudaName, DXCORE_MAX_PATH - strlen(replacedPath)); + } + + return replacedPath; +} + +static int dxcore_check_adapter(struct dxcore_lib *pLib, char *libPath, struct dxcore_adapterInfo *pAdapterInfo) +{ + unsigned int wddmVersion = 0; + char* driverStorePath = NULL; + + if (dxcore_query_adapter_wddm_version(pLib, pAdapterInfo->hAdapter, &wddmVersion)) { + return 1; + } + + if (wddmVersion < 2500) { + return 1; + } + + if (dxcore_query_adapter_driverstore_path(pLib, pAdapterInfo->hAdapter, &driverStorePath)) { + return 1; + } + + // Replace with valid path + char* replacedPath = replaceSystemPath(driverStorePath); + if (!replacedPath) { + free(driverStorePath); + free(replacedPath); + return 1; + } + + // Does file exist? +#if defined(_WIN32) + if (GetFileAttributes(replacedPath) == INVALID_FILE_ATTRIBUTES) { + free(driverStorePath); + free(replacedPath); + return 1; + } +#else + if (access(replacedPath, F_OK) < 0) { + free(driverStorePath); + free(replacedPath); + return 1; + } +#endif + + memcpy(libPath, replacedPath, DXCORE_MAX_PATH); + free(driverStorePath); + free(replacedPath); + + return 0; +} + +static int dxcore_enum_adapters(struct dxcore_lib *pLib, char *libPath) +{ + struct dxcore_enumAdapters2 params = {0}; + unsigned int adapterIndex = 0; + + if (pLib->pDxcoreEnumAdapters2(¶ms)) { + return 1; + } + params.pAdapters = (dxcore_adapterInfo*)calloc(params.NumAdapters, sizeof(struct dxcore_adapterInfo)); + if (pLib->pDxcoreEnumAdapters2(¶ms)) { + free(params.pAdapters); + return 1; + } + + for (adapterIndex = 0; adapterIndex < params.NumAdapters; adapterIndex++) { + if (!dxcore_check_adapter(pLib, libPath, ¶ms.pAdapters[adapterIndex])) { + free(params.pAdapters); + return 0; + } + } + + free(params.pAdapters); + return 1; +} + +int getCUDALibraryPath(char *libPath, bool isBit64) +{ + struct dxcore_lib lib = {0}; + + if (!libPath) { + return 1; + } + + // Configure paths based on app's bit configuration +#if defined(_WIN32) + if (isBit64) { + sysrootName_length = sysrootName64_length; + sysrootName = sysrootName64; + libcudaName_length = libcudaName64_length; + libcudaName = libcudaName64; + } + else { + sysrootName_length = sysrootNameX86_length; + sysrootName = sysrootNameX86; + libcudaName_length = libcudaNameX86_length; + libcudaName = libcudaNameX86; + } +#else + libcudaName_length = libcudaNameLinux_length; + libcudaName = libcudaNameLinux; +#endif + +#if defined(_WIN32) + lib.hDxcoreLib = LoadLibraryExW(L"gdi32.dll", NULL, LOAD_LIBRARY_SEARCH_SYSTEM32); +#else + lib.hDxcoreLib = dlopen("libdxcore.so", RTLD_LAZY); +#endif + if (!lib.hDxcoreLib) { + return 1; + } + + lib.pDxcoreEnumAdapters2 = (pfnDxcoreEnumAdapters2)_getAddr(lib.hDxcoreLib, "D3DKMTEnumAdapters2"); + if (!lib.pDxcoreEnumAdapters2) { + return 1; + } + lib.pDxcoreQueryAdapterInfo = (pfnDxcoreQueryAdapterInfo)_getAddr(lib.hDxcoreLib, "D3DKMTQueryAdapterInfo"); + if (!lib.pDxcoreQueryAdapterInfo) { + return 1; + } + + if (dxcore_enum_adapters(&lib, libPath)) { + return 1; + } + return 0; +} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/loader.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/loader.h new file mode 100644 index 0000000000000000000000000000000000000000..2411037b03256b70ca6e260fc140d94b64b16e00 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/loader.h @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +// +// Please refer to the NVIDIA end user license agreement (EULA) associated +// with this source code for terms and conditions that govern your use of +// this software. Any use, reproduction, disclosure, or distribution of +// this software and related documentation outside the terms of the EULA +// is strictly prohibited. +int getCUDALibraryPath(char *libPath, bool isBit64); diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/loader.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/loader.pxd new file mode 100644 index 0000000000000000000000000000000000000000..805b849cc660295d97cba986c55444db5f21fc47 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_bindings/loader.pxd @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +cdef extern from "loader.h": + int getCUDALibraryPath(char *libPath, bint isBit64) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..912d0c142d98267139755992bb9023bd3ab64a8f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/cufile.cpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/cufile.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2cfa3eb7865e0a5137c97caedfb01efd652cad0d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/cufile.cpp @@ -0,0 +1,18106 @@ +/* Generated by Cython 3.1.5 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [ + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/cuda.h", + "/host/home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include/cudaProfiler.h" + ], + "extra_compile_args": [ + "-std=c++14", + "-fpermissive", + "-Wno-deprecated-declarations", + "-fno-var-tracking-assignments", + "-O3" + ], + "include_dirs": [ + "/opt/python/cp312-cp312/include", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include" + ], + "language": "c++", + "library_dirs": [ + "/tmp/build-env-3t3_8d9r/lib/python3.12/site-packages", + "/tmp/build-env-3t3_8d9r/lib", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib64", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib" + ], + "name": "cuda.bindings._internal.cufile", + "sources": [ + "cuda/bindings/_internal/cufile.pyx" + ] + }, + "module_name": "cuda.bindings._internal.cufile" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_1_5" +#define CYTHON_HEX_VERSION 0x030105F0 +#define CYTHON_FUTURE_DIVISION 1 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef HAVE_LONG_LONG + #define HAVE_LONG_LONG +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_CLINE_IN_TRACEBACK + #define CYTHON_CLINE_IN_TRACEBACK 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1) +#endif +#endif +#define CYTHON_BACKPORT_VECTORCALL (CYTHON_METH_FASTCALL && PY_VERSION_HEX < 0x030800B1) +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME + #define CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME 100 +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + #endif + #endif + #if _MSC_VER < 1300 + #ifdef _WIN64 + typedef unsigned long long __pyx_uintptr_t; + #else + typedef unsigned int __pyx_uintptr_t; + #endif + #else + #ifdef _WIN64 + typedef unsigned __int64 __pyx_uintptr_t; + #else + typedef unsigned __int32 __pyx_uintptr_t; + #endif + #endif +#else + #include + typedef uintptr_t __pyx_uintptr_t; +#endif +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#elif CYTHON_BACKPORT_VECTORCALL + typedef PyObject *(*__pyx_vectorcallfunc)(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames); + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1)) + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(((size_t)(n)) & ~__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if __PYX_LIMITED_VERSION_HEX < 0x03090000 + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS && PY_VERSION_HEX >= 0x03080000 +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static CYTHON_INLINE PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__cuda__bindings___internal__cufile +#define __PYX_HAVE_API__cuda__bindings___internal__cufile +/* Early includes */ +#include +#include +#include +#include +#include "cuda.h" +#include "cudaProfiler.h" +#include +#include + + // This is the missing piece we need to supply to help Cython & C++ compilers. + inline bool operator==(const CUfileError_t& lhs, const CUfileError_t& rhs) { + return (lhs.err == rhs.err) && (lhs.cu_err == rhs.cu_err); + } + static CUfileError_t CUFILE_LOADING_ERROR{(CUfileOpError)-1, (CUresult)-1}; + +#include +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value); + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "cuda/bindings/_internal/cufile.pyx", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* ForceInitThreads.proto */ +#ifndef __PYX_FORCE_INIT_THREADS + #define __PYX_FORCE_INIT_THREADS 0 +#endif + +/* NoFastGil.proto */ +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() + +/* Atomics.proto */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif +#if CYTHON_ATOMICS + #define __pyx_add_acquisition_count(memview)\ + __pyx_atomic_incr_relaxed(__pyx_get_slice_count_pointer(memview)) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_atomic_decr_acq_rel(__pyx_get_slice_count_pointer(memview)) +#else + #define __pyx_add_acquisition_count(memview)\ + __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_sub_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) +#endif + +/* IncludeStructmemberH.proto */ +#include + +/* CriticalSections.proto */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_Begin1(cs, arg) (void)cs +#define __Pyx_PyCriticalSection_Begin2(cs, arg1, arg2) (void)cs +#define __Pyx_PyCriticalSection_End1(cs) +#define __Pyx_PyCriticalSection_End2(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_Begin1 PyCriticalSection_Begin +#define __Pyx_PyCriticalSection_Begin2 PyCriticalSection2_Begin +#define __Pyx_PyCriticalSection_End1 PyCriticalSection_End +#define __Pyx_PyCriticalSection_End2 PyCriticalSection2_End +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* #### Code section: numeric_typedefs ### */ + +/* "cuda/bindings/cydriver.pxd":2602 + * ctypedef CUGLmap_flags_enum CUGLmap_flags + * + * ctypedef unsigned int GLenum # <<<<<<<<<<<<<< + * + * ctypedef unsigned int GLuint +*/ +typedef unsigned int __pyx_t_4cuda_8bindings_8cydriver_GLenum; + +/* "cuda/bindings/cydriver.pxd":2604 + * ctypedef unsigned int GLenum + * + * ctypedef unsigned int GLuint # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef unsigned int __pyx_t_4cuda_8bindings_8cydriver_GLuint; + +/* "cuda/bindings/cydriver.pxd":2616 + * ctypedef void* EGLStreamKHR + * + * ctypedef unsigned int EGLint # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef unsigned int __pyx_t_4cuda_8bindings_8cydriver_EGLint; + +/* "cuda/bindings/cydriver.pxd":2623 + * ctypedef void* EGLSyncKHR + * + * ctypedef uint32_t VdpDevice # <<<<<<<<<<<<<< + * + * ctypedef unsigned long long VdpGetProcAddress +*/ +typedef uint32_t __pyx_t_4cuda_8bindings_8cydriver_VdpDevice; + +/* "cuda/bindings/cydriver.pxd":2625 + * ctypedef uint32_t VdpDevice + * + * ctypedef unsigned long long VdpGetProcAddress # <<<<<<<<<<<<<< + * + * ctypedef uint32_t VdpVideoSurface +*/ +typedef unsigned PY_LONG_LONG __pyx_t_4cuda_8bindings_8cydriver_VdpGetProcAddress; + +/* "cuda/bindings/cydriver.pxd":2627 + * ctypedef unsigned long long VdpGetProcAddress + * + * ctypedef uint32_t VdpVideoSurface # <<<<<<<<<<<<<< + * + * ctypedef uint32_t VdpOutputSurface +*/ +typedef uint32_t __pyx_t_4cuda_8bindings_8cydriver_VdpVideoSurface; + +/* "cuda/bindings/cydriver.pxd":2629 + * ctypedef uint32_t VdpVideoSurface + * + * ctypedef uint32_t VdpOutputSurface # <<<<<<<<<<<<<< + * + * cdef CUresult cuGetErrorString(CUresult error, const char** pStr) except ?CUDA_ERROR_NOT_FOUND nogil +*/ +typedef uint32_t __pyx_t_4cuda_8bindings_8cydriver_VdpOutputSurface; +/* #### Code section: complex_type_declarations ### */ +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ +union __pyx_t_4cuda_8bindings_8cydriver_anon_union15; +struct __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_st; + +/* "cuda/bindings/cydriver.pxd":2429 + * ctypedef CUoutput_mode_enum CUoutput_mode + * + * cdef enum CUeglFrameType_enum: # <<<<<<<<<<<<<< + * CU_EGL_FRAME_TYPE_ARRAY = 0 + * CU_EGL_FRAME_TYPE_PITCH = 1 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUeglFrameType_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_FRAME_TYPE_ARRAY = 0, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_FRAME_TYPE_PITCH = 1 +}; + +/* "cuda/bindings/cydriver.pxd":2435 + * ctypedef CUeglFrameType_enum CUeglFrameType + * + * cdef enum CUeglResourceLocationFlags_enum: # <<<<<<<<<<<<<< + * CU_EGL_RESOURCE_LOCATION_SYSMEM = 0 + * CU_EGL_RESOURCE_LOCATION_VIDMEM = 1 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUeglResourceLocationFlags_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_RESOURCE_LOCATION_SYSMEM = 0, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_RESOURCE_LOCATION_VIDMEM = 1 +}; + +/* "cuda/bindings/cydriver.pxd":2441 + * ctypedef CUeglResourceLocationFlags_enum CUeglResourceLocationFlags + * + * cdef enum CUeglColorFormat_enum: # <<<<<<<<<<<<<< + * CU_EGL_COLOR_FORMAT_YUV420_PLANAR = 0 + * CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR = 1 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUeglColorFormat_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR = 0, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR = 1, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_PLANAR = 2, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR = 3, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_RGB = 4, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BGR = 5, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_ARGB = 6, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_RGBA = 7, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_L = 8, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_R = 9, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_PLANAR = 10, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR = 11, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUYV_422 = 12, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_422 = 13, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_ABGR = 14, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BGRA = 15, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_A = 16, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_RG = 17, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_AYUV = 18, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR = 19, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR = 20, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR = 21, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR = 22, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR = 23, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR = 24, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR = 25, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_VYUY_ER = 26, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_ER = 27, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUYV_ER = 28, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVYU_ER = 29, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV_ER = 30, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUVA_ER = 31, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_AYUV_ER = 32, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER = 33, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER = 34, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER = 35, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER = 36, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER = 37, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER = 38, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER = 39, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER = 40, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER = 41, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER = 42, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER = 43, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER = 44, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_RGGB = 45, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_BGGR = 46, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_GRBG = 47, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_GBRG = 48, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_RGGB = 49, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_BGGR = 50, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_GRBG = 51, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_GBRG = 52, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_RGGB = 53, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_BGGR = 54, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_GRBG = 55, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_GBRG = 56, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_RGGB = 57, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_BGGR = 58, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_GRBG = 59, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER14_GBRG = 60, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_RGGB = 61, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_BGGR = 62, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_GRBG = 63, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER20_GBRG = 64, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU444_PLANAR = 65, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU422_PLANAR = 66, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR = 67, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB = 68, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR = 69, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG = 70, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG = 71, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_BCCR = 72, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_RCCB = 73, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_CRBC = 74, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER_CBRC = 75, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER10_CCCC = 76, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_BCCR = 77, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_RCCB = 78, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CRBC = 79, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CBRC = 80, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_BAYER12_CCCC = 81, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y = 82, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 = 83, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 = 84, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 = 85, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 = 86, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 = 87, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 = 88, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 = 89, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709 = 90, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 = 91, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 = 92, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 = 93, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR = 94, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709 = 95, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y_ER = 96, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y_709_ER = 97, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10_ER = 98, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10_709_ER = 99, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12_ER = 0x64, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12_709_ER = 0x65, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUVA = 0x66, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YUV = 0x67, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_YVYU = 0x68, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_VYUY = 0x69, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER = 0x6A, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER = 0x6B, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER = 0x6C, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER = 0x6D, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER = 0x6E, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER = 0x6F, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER = 0x70, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER = 0x71, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_709 = 0x72, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_709_ER = 0x73, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_UYVY_2020 = 0x74, + __pyx_e_4cuda_8bindings_8cydriver_CU_EGL_COLOR_FORMAT_MAX = 0x75 +}; + +/* "cuda/bindings/cydriver.pxd":2588 + * ctypedef CUeglStreamConnection_st* CUeglStreamConnection + * + * cdef enum CUGLDeviceList_enum: # <<<<<<<<<<<<<< + * CU_GL_DEVICE_LIST_ALL = 1 + * CU_GL_DEVICE_LIST_CURRENT_FRAME = 2 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_DEVICE_LIST_ALL = 1, + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_DEVICE_LIST_CURRENT_FRAME = 2, + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_DEVICE_LIST_NEXT_FRAME = 3 +}; + +/* "cuda/bindings/cydriver.pxd":2595 + * ctypedef CUGLDeviceList_enum CUGLDeviceList + * + * cdef enum CUGLmap_flags_enum: # <<<<<<<<<<<<<< + * CU_GL_MAP_RESOURCE_FLAGS_NONE = 0 + * CU_GL_MAP_RESOURCE_FLAGS_READ_ONLY = 1 +*/ +enum __pyx_t_4cuda_8bindings_8cydriver_CUGLmap_flags_enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_MAP_RESOURCE_FLAGS_NONE = 0, + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_MAP_RESOURCE_FLAGS_READ_ONLY = 1, + __pyx_e_4cuda_8bindings_8cydriver_CU_GL_MAP_RESOURCE_FLAGS_WRITE_DISCARD = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3577 + * cdef CUresult cuGraphicsVDPAURegisterOutputSurface(CUgraphicsResource* pCudaResource, VdpOutputSurface vdpSurface, unsigned int flags) except ?CUDA_ERROR_NOT_FOUND nogil + * + * cdef enum: CUDA_VERSION = 12090 # <<<<<<<<<<<<<< + * + * cdef enum: CU_IPC_HANDLE_SIZE = 64 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_VERSION = 0x2F3A +}; + +/* "cuda/bindings/cydriver.pxd":3579 + * cdef enum: CUDA_VERSION = 12090 + * + * cdef enum: CU_IPC_HANDLE_SIZE = 64 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_LEGACY = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_IPC_HANDLE_SIZE = 64 +}; + +/* "cuda/bindings/cydriver.pxd":3581 + * cdef enum: CU_IPC_HANDLE_SIZE = 64 + * + * cdef enum: CU_STREAM_LEGACY = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_PER_THREAD = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_LEGACY = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3583 + * cdef enum: CU_STREAM_LEGACY = 1 + * + * cdef enum: CU_STREAM_PER_THREAD = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_COMPUTE_ACCELERATED_TARGET_BASE = 65536 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_PER_THREAD = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3585 + * cdef enum: CU_STREAM_PER_THREAD = 2 + * + * cdef enum: CU_COMPUTE_ACCELERATED_TARGET_BASE = 65536 # <<<<<<<<<<<<<< + * + * cdef enum: CU_COMPUTE_FAMILY_TARGET_BASE = 131072 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_COMPUTE_ACCELERATED_TARGET_BASE = 0x10000 +}; + +/* "cuda/bindings/cydriver.pxd":3587 + * cdef enum: CU_COMPUTE_ACCELERATED_TARGET_BASE = 65536 + * + * cdef enum: CU_COMPUTE_FAMILY_TARGET_BASE = 131072 # <<<<<<<<<<<<<< + * + * cdef enum: CU_GRAPH_COND_ASSIGN_DEFAULT = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_COMPUTE_FAMILY_TARGET_BASE = 0x20000 +}; + +/* "cuda/bindings/cydriver.pxd":3589 + * cdef enum: CU_COMPUTE_FAMILY_TARGET_BASE = 131072 + * + * cdef enum: CU_GRAPH_COND_ASSIGN_DEFAULT = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_DEFAULT = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GRAPH_COND_ASSIGN_DEFAULT = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3591 + * cdef enum: CU_GRAPH_COND_ASSIGN_DEFAULT = 1 + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_DEFAULT = 0 # <<<<<<<<<<<<<< + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GRAPH_KERNEL_NODE_PORT_DEFAULT = 0 +}; + +/* "cuda/bindings/cydriver.pxd":3593 + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_DEFAULT = 0 + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3595 + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_PROGRAMMATIC = 1 + * + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3597 + * cdef enum: CU_GRAPH_KERNEL_NODE_PORT_LAUNCH_ORDER = 2 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_COOPERATIVE = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3599 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_COOPERATIVE = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_DIMENSION = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_COOPERATIVE = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3601 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_COOPERATIVE = 2 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_DIMENSION = 4 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_DIMENSION = 4 +}; + +/* "cuda/bindings/cydriver.pxd":3603 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_DIMENSION = 4 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PRIORITY = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 +}; + +/* "cuda/bindings/cydriver.pxd":3605 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE = 5 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PRIORITY = 8 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_PRIORITY = 8 +}; + +/* "cuda/bindings/cydriver.pxd":3607 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PRIORITY = 8 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 +}; + +/* "cuda/bindings/cydriver.pxd":3609 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 +}; + +/* "cuda/bindings/cydriver.pxd":3611 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11 +}; + +/* "cuda/bindings/cydriver.pxd":3613 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_CLUSTER_DIMENSION = 11 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13 # <<<<<<<<<<<<<< + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 14 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13 +}; + +/* "cuda/bindings/cydriver.pxd":3615 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE = 13 + * + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 14 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 14 +}; + +/* "cuda/bindings/cydriver.pxd":3617 + * cdef enum: CU_KERNEL_NODE_ATTRIBUTE_PREFERRED_SHARED_MEMORY_CARVEOUT = 14 + * + * cdef enum: CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3619 + * cdef enum: CU_STREAM_ATTRIBUTE_ACCESS_POLICY_WINDOW = 1 + * + * cdef enum: CU_STREAM_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_PRIORITY = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 +}; + +/* "cuda/bindings/cydriver.pxd":3621 + * cdef enum: CU_STREAM_ATTRIBUTE_SYNCHRONIZATION_POLICY = 3 + * + * cdef enum: CU_STREAM_ATTRIBUTE_PRIORITY = 8 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_PRIORITY = 8 +}; + +/* "cuda/bindings/cydriver.pxd":3623 + * cdef enum: CU_STREAM_ATTRIBUTE_PRIORITY = 8 + * + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 # <<<<<<<<<<<<<< + * + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 +}; + +/* "cuda/bindings/cydriver.pxd":3625 + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN_MAP = 9 + * + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTALLOC_PORTABLE = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 +}; + +/* "cuda/bindings/cydriver.pxd":3627 + * cdef enum: CU_STREAM_ATTRIBUTE_MEM_SYNC_DOMAIN = 10 + * + * cdef enum: CU_MEMHOSTALLOC_PORTABLE = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTALLOC_DEVICEMAP = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTALLOC_PORTABLE = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3629 + * cdef enum: CU_MEMHOSTALLOC_PORTABLE = 1 + * + * cdef enum: CU_MEMHOSTALLOC_DEVICEMAP = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTALLOC_WRITECOMBINED = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTALLOC_DEVICEMAP = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3631 + * cdef enum: CU_MEMHOSTALLOC_DEVICEMAP = 2 + * + * cdef enum: CU_MEMHOSTALLOC_WRITECOMBINED = 4 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTREGISTER_PORTABLE = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTALLOC_WRITECOMBINED = 4 +}; + +/* "cuda/bindings/cydriver.pxd":3633 + * cdef enum: CU_MEMHOSTALLOC_WRITECOMBINED = 4 + * + * cdef enum: CU_MEMHOSTREGISTER_PORTABLE = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTREGISTER_DEVICEMAP = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTREGISTER_PORTABLE = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3635 + * cdef enum: CU_MEMHOSTREGISTER_PORTABLE = 1 + * + * cdef enum: CU_MEMHOSTREGISTER_DEVICEMAP = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTREGISTER_IOMEMORY = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTREGISTER_DEVICEMAP = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3637 + * cdef enum: CU_MEMHOSTREGISTER_DEVICEMAP = 2 + * + * cdef enum: CU_MEMHOSTREGISTER_IOMEMORY = 4 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEMHOSTREGISTER_READ_ONLY = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTREGISTER_IOMEMORY = 4 +}; + +/* "cuda/bindings/cydriver.pxd":3639 + * cdef enum: CU_MEMHOSTREGISTER_IOMEMORY = 4 + * + * cdef enum: CU_MEMHOSTREGISTER_READ_ONLY = 8 # <<<<<<<<<<<<<< + * + * cdef enum: CU_ARRAY_SPARSE_PROPERTIES_SINGLE_MIPTAIL = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEMHOSTREGISTER_READ_ONLY = 8 +}; + +/* "cuda/bindings/cydriver.pxd":3641 + * cdef enum: CU_MEMHOSTREGISTER_READ_ONLY = 8 + * + * cdef enum: CU_ARRAY_SPARSE_PROPERTIES_SINGLE_MIPTAIL = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TENSOR_MAP_NUM_QWORDS = 16 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_ARRAY_SPARSE_PROPERTIES_SINGLE_MIPTAIL = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3643 + * cdef enum: CU_ARRAY_SPARSE_PROPERTIES_SINGLE_MIPTAIL = 1 + * + * cdef enum: CU_TENSOR_MAP_NUM_QWORDS = 16 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_EXTERNAL_MEMORY_DEDICATED = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TENSOR_MAP_NUM_QWORDS = 16 +}; + +/* "cuda/bindings/cydriver.pxd":3645 + * cdef enum: CU_TENSOR_MAP_NUM_QWORDS = 16 + * + * cdef enum: CUDA_EXTERNAL_MEMORY_DEDICATED = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_SIGNAL_SKIP_NVSCIBUF_MEMSYNC = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_EXTERNAL_MEMORY_DEDICATED = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3647 + * cdef enum: CUDA_EXTERNAL_MEMORY_DEDICATED = 1 + * + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_SIGNAL_SKIP_NVSCIBUF_MEMSYNC = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_WAIT_SKIP_NVSCIBUF_MEMSYNC = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_EXTERNAL_SEMAPHORE_SIGNAL_SKIP_NVSCIBUF_MEMSYNC = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3649 + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_SIGNAL_SKIP_NVSCIBUF_MEMSYNC = 1 + * + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_WAIT_SKIP_NVSCIBUF_MEMSYNC = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_NVSCISYNC_ATTR_SIGNAL = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_EXTERNAL_SEMAPHORE_WAIT_SKIP_NVSCIBUF_MEMSYNC = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3651 + * cdef enum: CUDA_EXTERNAL_SEMAPHORE_WAIT_SKIP_NVSCIBUF_MEMSYNC = 2 + * + * cdef enum: CUDA_NVSCISYNC_ATTR_SIGNAL = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_NVSCISYNC_ATTR_WAIT = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_NVSCISYNC_ATTR_SIGNAL = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3653 + * cdef enum: CUDA_NVSCISYNC_ATTR_SIGNAL = 1 + * + * cdef enum: CUDA_NVSCISYNC_ATTR_WAIT = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEM_CREATE_USAGE_TILE_POOL = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_NVSCISYNC_ATTR_WAIT = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3655 + * cdef enum: CUDA_NVSCISYNC_ATTR_WAIT = 2 + * + * cdef enum: CU_MEM_CREATE_USAGE_TILE_POOL = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEM_CREATE_USAGE_HW_DECOMPRESS = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEM_CREATE_USAGE_TILE_POOL = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3657 + * cdef enum: CU_MEM_CREATE_USAGE_TILE_POOL = 1 + * + * cdef enum: CU_MEM_CREATE_USAGE_HW_DECOMPRESS = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEM_CREATE_USAGE_HW_DECOMPRESS = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3659 + * cdef enum: CU_MEM_CREATE_USAGE_HW_DECOMPRESS = 2 + * + * cdef enum: CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3661 + * cdef enum: CU_MEM_POOL_CREATE_USAGE_HW_DECOMPRESS = 2 + * + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3663 + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_PRE_LAUNCH_SYNC = 1 + * + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_LAYERED = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3665 + * cdef enum: CUDA_COOPERATIVE_LAUNCH_MULTI_DEVICE_NO_POST_LAUNCH_SYNC = 2 + * + * cdef enum: CUDA_ARRAY3D_LAYERED = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_2DARRAY = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_LAYERED = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3667 + * cdef enum: CUDA_ARRAY3D_LAYERED = 1 + * + * cdef enum: CUDA_ARRAY3D_2DARRAY = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_SURFACE_LDST = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_2DARRAY = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3669 + * cdef enum: CUDA_ARRAY3D_2DARRAY = 1 + * + * cdef enum: CUDA_ARRAY3D_SURFACE_LDST = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_CUBEMAP = 4 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_SURFACE_LDST = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3671 + * cdef enum: CUDA_ARRAY3D_SURFACE_LDST = 2 + * + * cdef enum: CUDA_ARRAY3D_CUBEMAP = 4 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_TEXTURE_GATHER = 8 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_CUBEMAP = 4 +}; + +/* "cuda/bindings/cydriver.pxd":3673 + * cdef enum: CUDA_ARRAY3D_CUBEMAP = 4 + * + * cdef enum: CUDA_ARRAY3D_TEXTURE_GATHER = 8 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_DEPTH_TEXTURE = 16 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_TEXTURE_GATHER = 8 +}; + +/* "cuda/bindings/cydriver.pxd":3675 + * cdef enum: CUDA_ARRAY3D_TEXTURE_GATHER = 8 + * + * cdef enum: CUDA_ARRAY3D_DEPTH_TEXTURE = 16 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_COLOR_ATTACHMENT = 32 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_DEPTH_TEXTURE = 16 +}; + +/* "cuda/bindings/cydriver.pxd":3677 + * cdef enum: CUDA_ARRAY3D_DEPTH_TEXTURE = 16 + * + * cdef enum: CUDA_ARRAY3D_COLOR_ATTACHMENT = 32 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_SPARSE = 64 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_COLOR_ATTACHMENT = 32 +}; + +/* "cuda/bindings/cydriver.pxd":3679 + * cdef enum: CUDA_ARRAY3D_COLOR_ATTACHMENT = 32 + * + * cdef enum: CUDA_ARRAY3D_SPARSE = 64 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_DEFERRED_MAPPING = 128 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_SPARSE = 64 +}; + +/* "cuda/bindings/cydriver.pxd":3681 + * cdef enum: CUDA_ARRAY3D_SPARSE = 64 + * + * cdef enum: CUDA_ARRAY3D_DEFERRED_MAPPING = 128 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_ARRAY3D_VIDEO_ENCODE_DECODE = 256 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_DEFERRED_MAPPING = 0x80 +}; + +/* "cuda/bindings/cydriver.pxd":3683 + * cdef enum: CUDA_ARRAY3D_DEFERRED_MAPPING = 128 + * + * cdef enum: CUDA_ARRAY3D_VIDEO_ENCODE_DECODE = 256 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSA_OVERRIDE_FORMAT = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_ARRAY3D_VIDEO_ENCODE_DECODE = 0x100 +}; + +/* "cuda/bindings/cydriver.pxd":3685 + * cdef enum: CUDA_ARRAY3D_VIDEO_ENCODE_DECODE = 256 + * + * cdef enum: CU_TRSA_OVERRIDE_FORMAT = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_READ_AS_INTEGER = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSA_OVERRIDE_FORMAT = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3687 + * cdef enum: CU_TRSA_OVERRIDE_FORMAT = 1 + * + * cdef enum: CU_TRSF_READ_AS_INTEGER = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_NORMALIZED_COORDINATES = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_READ_AS_INTEGER = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3689 + * cdef enum: CU_TRSF_READ_AS_INTEGER = 1 + * + * cdef enum: CU_TRSF_NORMALIZED_COORDINATES = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_SRGB = 16 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_NORMALIZED_COORDINATES = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3691 + * cdef enum: CU_TRSF_NORMALIZED_COORDINATES = 2 + * + * cdef enum: CU_TRSF_SRGB = 16 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_DISABLE_TRILINEAR_OPTIMIZATION = 32 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_SRGB = 16 +}; + +/* "cuda/bindings/cydriver.pxd":3693 + * cdef enum: CU_TRSF_SRGB = 16 + * + * cdef enum: CU_TRSF_DISABLE_TRILINEAR_OPTIMIZATION = 32 # <<<<<<<<<<<<<< + * + * cdef enum: CU_TRSF_SEAMLESS_CUBEMAP = 64 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_DISABLE_TRILINEAR_OPTIMIZATION = 32 +}; + +/* "cuda/bindings/cydriver.pxd":3695 + * cdef enum: CU_TRSF_DISABLE_TRILINEAR_OPTIMIZATION = 32 + * + * cdef enum: CU_TRSF_SEAMLESS_CUBEMAP = 64 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_KERNEL_REQUIRED_BLOCK_DIM = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_TRSF_SEAMLESS_CUBEMAP = 64 +}; + +/* "cuda/bindings/cydriver.pxd":3697 + * cdef enum: CU_TRSF_SEAMLESS_CUBEMAP = 64 + * + * cdef enum: CU_LAUNCH_KERNEL_REQUIRED_BLOCK_DIM = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_END_AS_INT = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_KERNEL_REQUIRED_BLOCK_DIM = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3699 + * cdef enum: CU_LAUNCH_KERNEL_REQUIRED_BLOCK_DIM = 1 + * + * cdef enum: CU_LAUNCH_PARAM_END_AS_INT = 0 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_END = 0 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_END_AS_INT = 0 +}; + +/* "cuda/bindings/cydriver.pxd":3701 + * cdef enum: CU_LAUNCH_PARAM_END_AS_INT = 0 + * + * cdef enum: CU_LAUNCH_PARAM_END = 0 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER_AS_INT = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_END = 0 +}; + +/* "cuda/bindings/cydriver.pxd":3703 + * cdef enum: CU_LAUNCH_PARAM_END = 0 + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER_AS_INT = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_BUFFER_POINTER_AS_INT = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3705 + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER_AS_INT = 1 + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER = 1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE_AS_INT = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_BUFFER_POINTER = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3707 + * cdef enum: CU_LAUNCH_PARAM_BUFFER_POINTER = 1 + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE_AS_INT = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE = 2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_BUFFER_SIZE_AS_INT = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3709 + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE_AS_INT = 2 + * + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE = 2 # <<<<<<<<<<<<<< + * + * cdef enum: CU_PARAM_TR_DEFAULT = -1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_LAUNCH_PARAM_BUFFER_SIZE = 2 +}; + +/* "cuda/bindings/cydriver.pxd":3711 + * cdef enum: CU_LAUNCH_PARAM_BUFFER_SIZE = 2 + * + * cdef enum: CU_PARAM_TR_DEFAULT = -1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_DEVICE_CPU = -1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_PARAM_TR_DEFAULT = -1L +}; + +/* "cuda/bindings/cydriver.pxd":3713 + * cdef enum: CU_PARAM_TR_DEFAULT = -1 + * + * cdef enum: CU_DEVICE_CPU = -1 # <<<<<<<<<<<<<< + * + * cdef enum: CU_DEVICE_INVALID = -2 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_DEVICE_CPU = -1L +}; + +/* "cuda/bindings/cydriver.pxd":3715 + * cdef enum: CU_DEVICE_CPU = -1 + * + * cdef enum: CU_DEVICE_INVALID = -2 # <<<<<<<<<<<<<< + * + * cdef enum: RESOURCE_ABI_VERSION = 1 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CU_DEVICE_INVALID = -2L +}; + +/* "cuda/bindings/cydriver.pxd":3717 + * cdef enum: CU_DEVICE_INVALID = -2 + * + * cdef enum: RESOURCE_ABI_VERSION = 1 # <<<<<<<<<<<<<< + * + * cdef enum: RESOURCE_ABI_EXTERNAL_BYTES = 48 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_RESOURCE_ABI_VERSION = 1 +}; + +/* "cuda/bindings/cydriver.pxd":3719 + * cdef enum: RESOURCE_ABI_VERSION = 1 + * + * cdef enum: RESOURCE_ABI_EXTERNAL_BYTES = 48 # <<<<<<<<<<<<<< + * + * cdef enum: MAX_PLANES = 3 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_RESOURCE_ABI_EXTERNAL_BYTES = 48 +}; + +/* "cuda/bindings/cydriver.pxd":3721 + * cdef enum: RESOURCE_ABI_EXTERNAL_BYTES = 48 + * + * cdef enum: MAX_PLANES = 3 # <<<<<<<<<<<<<< + * + * cdef enum: CUDA_EGL_INFINITE_TIMEOUT = 4294967295 +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_MAX_PLANES = 3 +}; + +/* "cuda/bindings/cydriver.pxd":3723 + * cdef enum: MAX_PLANES = 3 + * + * cdef enum: CUDA_EGL_INFINITE_TIMEOUT = 4294967295 # <<<<<<<<<<<<<< +*/ +enum { + __pyx_e_4cuda_8bindings_8cydriver_CUDA_EGL_INFINITE_TIMEOUT = 0xFFFFFFFF +}; + +/* "cuda/bindings/cydriver.pxd":2433 + * CU_EGL_FRAME_TYPE_PITCH = 1 + * + * ctypedef CUeglFrameType_enum CUeglFrameType # <<<<<<<<<<<<<< + * + * cdef enum CUeglResourceLocationFlags_enum: +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUeglFrameType_enum __pyx_t_4cuda_8bindings_8cydriver_CUeglFrameType; + +/* "cuda/bindings/cydriver.pxd":2439 + * CU_EGL_RESOURCE_LOCATION_VIDMEM = 1 + * + * ctypedef CUeglResourceLocationFlags_enum CUeglResourceLocationFlags # <<<<<<<<<<<<<< + * + * cdef enum CUeglColorFormat_enum: +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUeglResourceLocationFlags_enum __pyx_t_4cuda_8bindings_8cydriver_CUeglResourceLocationFlags; + +/* "cuda/bindings/cydriver.pxd":2561 + * CU_EGL_COLOR_FORMAT_MAX = 117 + * + * ctypedef CUeglColorFormat_enum CUeglColorFormat # <<<<<<<<<<<<<< + * + * cdef union anon_union15: +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUeglColorFormat_enum __pyx_t_4cuda_8bindings_8cydriver_CUeglColorFormat; + +/* "cuda/bindings/cydriver.pxd":2563 + * ctypedef CUeglColorFormat_enum CUeglColorFormat + * + * cdef union anon_union15: # <<<<<<<<<<<<<< + * CUarray pArray[3] + * void* pPitch[3] +*/ +union __pyx_t_4cuda_8bindings_8cydriver_anon_union15 { + CUarray pArray[3]; + void *pPitch[3]; +}; + +/* "cuda/bindings/cydriver.pxd":2567 + * void* pPitch[3] + * + * cdef struct CUeglFrame_st: # <<<<<<<<<<<<<< + * anon_union15 frame + * unsigned int width +*/ +struct __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_st { + union __pyx_t_4cuda_8bindings_8cydriver_anon_union15 frame; + unsigned int width; + unsigned int height; + unsigned int depth; + unsigned int pitch; + unsigned int planeCount; + unsigned int numChannels; + __pyx_t_4cuda_8bindings_8cydriver_CUeglFrameType frameType; + __pyx_t_4cuda_8bindings_8cydriver_CUeglColorFormat eglColorFormat; + CUarray_format cuFormat; +}; + +/* "cuda/bindings/cydriver.pxd":2579 + * CUarray_format cuFormat + * + * ctypedef CUeglFrame_st CUeglFrame_v1 # <<<<<<<<<<<<<< + * + * ctypedef CUeglFrame_v1 CUeglFrame +*/ +typedef struct __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_st __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_v1; + +/* "cuda/bindings/cydriver.pxd":2581 + * ctypedef CUeglFrame_st CUeglFrame_v1 + * + * ctypedef CUeglFrame_v1 CUeglFrame # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame_v1 __pyx_t_4cuda_8bindings_8cydriver_CUeglFrame; + +/* "cuda/bindings/cydriver.pxd":2586 + * cdef struct CUeglStreamConnection_st: + * pass + * ctypedef CUeglStreamConnection_st* CUeglStreamConnection # <<<<<<<<<<<<<< + * + * cdef enum CUGLDeviceList_enum: +*/ +typedef struct CUeglStreamConnection_st *__pyx_t_4cuda_8bindings_8cydriver_CUeglStreamConnection; + +/* "cuda/bindings/cydriver.pxd":2593 + * CU_GL_DEVICE_LIST_NEXT_FRAME = 3 + * + * ctypedef CUGLDeviceList_enum CUGLDeviceList # <<<<<<<<<<<<<< + * + * cdef enum CUGLmap_flags_enum: +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList_enum __pyx_t_4cuda_8bindings_8cydriver_CUGLDeviceList; + +/* "cuda/bindings/cydriver.pxd":2600 + * CU_GL_MAP_RESOURCE_FLAGS_WRITE_DISCARD = 2 + * + * ctypedef CUGLmap_flags_enum CUGLmap_flags # <<<<<<<<<<<<<< + * + * ctypedef unsigned int GLenum +*/ +typedef enum __pyx_t_4cuda_8bindings_8cydriver_CUGLmap_flags_enum __pyx_t_4cuda_8bindings_8cydriver_CUGLmap_flags; + +/* "cuda/bindings/cydriver.pxd":2609 + * cdef struct void: + * pass + * ctypedef void* EGLImageKHR # <<<<<<<<<<<<<< + * + * cdef extern from "": +*/ +typedef void *__pyx_t_4cuda_8bindings_8cydriver_EGLImageKHR; + +/* "cuda/bindings/cydriver.pxd":2614 + * cdef struct void: + * pass + * ctypedef void* EGLStreamKHR # <<<<<<<<<<<<<< + * + * ctypedef unsigned int EGLint +*/ +typedef void *__pyx_t_4cuda_8bindings_8cydriver_EGLStreamKHR; + +/* "cuda/bindings/cydriver.pxd":2621 + * cdef struct void: + * pass + * ctypedef void* EGLSyncKHR # <<<<<<<<<<<<<< + * + * ctypedef uint32_t VdpDevice +*/ +typedef void *__pyx_t_4cuda_8bindings_8cydriver_EGLSyncKHR; +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyErrExceptionMatches.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* PyObjectGetAttrStrNoError.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* PyDictVersioning.proto */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __pyx_dict_cached_value;\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) do {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +#define __Pyx_GetModuleGlobalNameUncached(var, name) do {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#if !CYTHON_VECTORCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs); +#endif +#define __Pyx_BUILD_ASSERT_EXPR(cond)\ + (sizeof(char [1 - 2*!(cond)]) - 1) +#ifndef Py_MEMBER_SIZE +#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#endif +#if !CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03080000 + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets() + #define __Pyx_PyFrame_GetLocalsplus(frame) ((frame)->f_localsplus) +#else + static size_t __pyx_pyframe_localsplus_offset = 0; + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets()\ + ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ + (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) + #define __Pyx_PyFrame_GetLocalsplus(frame)\ + (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) +#endif +#endif +#endif + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* PyObjectLookupSpecial.proto */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 0) +#define __Pyx_PyObject_LookupSpecial(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 1) +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error); +#else +#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n) +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/* GetTopmostException.proto */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* TupleAndListFromArray.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +#endif +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/* IncludeStringH.proto */ +#include + +/* BytesEquals.proto */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); + +/* UnicodeEquals.proto */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); + +/* fastcall.proto */ +#if CYTHON_AVOID_BORROWED_REFS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_PySequence_ITEM(args, i) +#elif CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_NewRef(__Pyx_PyTuple_GET_ITEM(args, i)) +#else + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_XNewRef(PyTuple_GetItem(args, i)) +#endif +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_ArgRef_FASTCALL(args, i) __Pyx_NewRef(args[i]) + #define __Pyx_NumKwargs_FASTCALL(kwds) __Pyx_PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs)) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues); + #else + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) + #endif +#else + #define __Pyx_ArgRef_FASTCALL __Pyx_ArgRef_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#if CYTHON_METH_FASTCALL || (CYTHON_COMPILING_IN_CPYTHON && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(args + start, stop - start) +#else +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + +/* DictGetItem.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key); +#define __Pyx_PyObject_Dict_GetItem(obj, name)\ + (likely(PyDict_CheckExact(obj)) ?\ + __Pyx_PyDict_GetItem(obj, name) : PyObject_GetItem(obj, name)) +#else +#define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key) +#define __Pyx_PyObject_Dict_GetItem(obj, name) PyObject_GetItem(obj, name) +#endif + +/* RaiseDoubleKeywords.proto */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywords.proto */ +static CYTHON_INLINE int __Pyx_ParseKeywords( + PyObject *kwds, PyObject *const *kwvalues, PyObject ** const argnames[], + PyObject *kwds2, PyObject *values[], + Py_ssize_t num_pos_args, Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* CallCFunction.proto */ +#define __Pyx_CallCFunction(cfunc, self, args)\ + ((PyCFunction)(void(*)(void))(cfunc)->func)(self, args) +#define __Pyx_CallCFunctionWithKeywords(cfunc, self, args, kwargs)\ + ((PyCFunctionWithKeywords)(void(*)(void))(cfunc)->func)(self, args, kwargs) +#define __Pyx_CallCFunctionFast(cfunc, self, args, nargs)\ + ((__Pyx_PyCFunctionFast)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs) +#define __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, nargs, kwnames)\ + ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs, kwnames) + +/* UnpackUnboundCMethod.proto */ +typedef struct { + PyObject *type; + PyObject **method_name; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS + __pyx_atomic_int_type initialized; +#endif + PyCFunction func; + PyObject *method; + int flag; +} __Pyx_CachedCFunction; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +static CYTHON_INLINE int __Pyx_CachedCFunction_GetAndSetInitializing(__Pyx_CachedCFunction *cfunc) { +#if !CYTHON_ATOMICS + return 1; +#else + __pyx_nonatomic_int_type expected = 0; + if (__pyx_atomic_int_cmp_exchange(&cfunc->initialized, &expected, 1)) { + return 0; + } + return expected; +#endif +} +static CYTHON_INLINE void __Pyx_CachedCFunction_SetFinishedInitializing(__Pyx_CachedCFunction *cfunc) { +#if CYTHON_ATOMICS + __pyx_atomic_store(&cfunc->initialized, 2); +#endif +} +#else +#define __Pyx_CachedCFunction_GetAndSetInitializing(cfunc) 2 +#define __Pyx_CachedCFunction_SetFinishedInitializing(cfunc) +#endif + +/* CallUnboundCMethod2.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2); +#else +#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2) +#endif + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* ArgTypeTest.proto */ +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* RaiseException.proto */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* Import.proto */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); + +/* ImportDottedModule.proto */ +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple); +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple); + +/* ListPack.proto */ +static PyObject *__Pyx_PyList_Pack(Py_ssize_t n, ...); + +/* ImportFrom.proto */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +/* LimitedApiGetTypeDict.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp); +#endif + +/* SetItemOnTypeDict.proto */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v); +#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v) + +/* FixUpExtensionType.proto */ +static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); + +/* FetchSharedCythonModule.proto */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void); + +/* dict_setdefault.proto */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, int is_safe_type); + +/* FetchCommonType.proto */ +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases); + +/* CommonTypesMetaclass.proto */ +static int __pyx_CommonTypesMetaclass_init(PyObject *module); +#define __Pyx_CommonTypesMetaclass_USED + +/* CallTypeTraverse.proto */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0 +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg); +#endif + +/* PyMethodNew.proto */ +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ); + +/* PyVectorcallFastCallDict.proto */ +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); +#endif + +/* CythonFunctionShared.proto */ +#define __Pyx_CyFunction_USED +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f)\ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj)\ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject_HEAD + PyObject *func; +#elif PY_VERSION_HEX < 0x030900B1 + PyCFunctionObject func; +#else + PyCMethodObject func; +#endif +#if CYTHON_BACKPORT_VECTORCALL ||\ + (CYTHON_COMPILING_IN_LIMITED_API && CYTHON_METH_FASTCALL) + __pyx_vectorcallfunc func_vectorcall; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_weakreflist; +#endif + PyObject *func_dict; + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_classobj; +#endif + PyObject *defaults; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; + PyObject *func_is_coroutine; +} __pyx_CyFunctionObject; +#undef __Pyx_CyOrPyCFunction_Check +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_mstate_global->__pyx_CyFunctionType) +#define __Pyx_CyOrPyCFunction_Check(obj) __Pyx_TypeCheck2(obj, __pyx_mstate_global->__pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_CyFunctionType) +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)); +#undef __Pyx_IsSameCFunction +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCyOrCFunction(func, cfunc) +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, + PyTypeObject *defaults_type); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(PyObject *module); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_BACKPORT_VECTORCALL || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif + +/* CythonFunction.proto */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); + +/* CLineInTraceback.proto */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* GCCDiagnostics.proto */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* CIntFromPy.proto */ +static CYTHON_INLINE size_t __Pyx_PyLong_As_size_t(PyObject *); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* PyObjectVectorCallKwBuilder.proto */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +/* GetRuntimeVersion.proto */ +static unsigned long __Pyx_get_runtime_version(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* FunctionExport.proto */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + + +/* Module declarations from "libc.stdint" */ + +/* Module declarations from "libc.stddef" */ + +/* Module declarations from "libc.time" */ + +/* Module declarations from "libcpp" */ + +/* Module declarations from "posix.types" */ + +/* Module declarations from "cuda.bindings.cydriver" */ + +/* Module declarations from "cuda.bindings" */ + +/* Module declarations from "cuda" */ + +/* Module declarations from "cuda.bindings.cycufile" */ + +/* Module declarations from "cython" */ + +/* Module declarations from "cuda.bindings._internal.cufile" */ +static PyObject *__pyx_v_4cuda_8bindings_9_internal_6cufile___symbol_lock = 0; +static int __pyx_v_4cuda_8bindings_9_internal_6cufile___py_cufile_init; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuDriverGetVersion; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleRegister; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleDeregister; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufRegister; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufDeregister; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileRead; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWrite; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverOpen; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose_v2; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileUseCount; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverGetProperties; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetPollMode; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxDirectIOSize; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxCacheSize; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxPinnedMemSize; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSetUp; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSubmit; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOGetStatus; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOCancel; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIODestroy; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileReadAsync; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWriteAsync; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamRegister; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamDeregister; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetVersion; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterSizeT; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterBool; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterString; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterSizeT; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterBool; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterString; +static void *__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose; +static PyObject *__pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs = 0; +static void *__pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(void); /*proto*/ +static int __pyx_f_4cuda_8bindings_9_internal_6cufile___check_or_init_cufile(void); /*proto*/ +static CYTHON_INLINE int __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(void); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_internal_6cufile__inspect_function_pointers(int __pyx_skip_dispatch); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_internal_6cufile__inspect_function_pointer(PyObject *, int __pyx_skip_dispatch); /*proto*/ +/* #### Code section: typeinfo ### */ +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "cuda.bindings._internal.cufile" +extern int __pyx_module_is_main_cuda__bindings___internal__cufile; +int __pyx_module_is_main_cuda__bindings___internal__cufile = 0; + +/* Implementation of "cuda.bindings._internal.cufile" */ +/* #### Code section: global_var ### */ +/* #### Code section: string_decls ### */ +static const char __pyx_k_[] = "."; +static const char __pyx_k__2[] = "?"; +static const char __pyx_k_pop[] = "pop"; +static const char __pyx_k_Lock[] = "Lock"; +static const char __pyx_k_exit[] = "__exit__"; +static const char __pyx_k_func[] = "__func__"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_name[] = "name"; +static const char __pyx_k_spec[] = "__spec__"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_enter[] = "__enter__"; +static const char __pyx_k_utils[] = "utils"; +static const char __pyx_k_cufile[] = "cufile"; +static const char __pyx_k_module[] = "__module__"; +static const char __pyx_k_name_2[] = "__name__"; +static const char __pyx_k_add_note[] = "add_note"; +static const char __pyx_k_qualname[] = "__qualname__"; +static const char __pyx_k_set_name[] = "__set_name__"; +static const char __pyx_k_threading[] = "threading"; +static const char __pyx_k_z_A_a_9AQ[] = "\200\001\330\004\005\330\004\007\200z\220\023\220A\330\010\024\320\024.\250a\330\004\013\2109\220A\220Q"; +static const char __pyx_k_cuFileRead[] = "__cuFileRead"; +static const char __pyx_k_cuFileWrite[] = "__cuFileWrite"; +static const char __pyx_k_handle_uint[] = "_handle_uint"; +static const char __pyx_k_initializing[] = "_initializing"; +static const char __pyx_k_is_coroutine[] = "_is_coroutine"; +static const char __pyx_k_cuFileUseCount[] = "__cuFileUseCount"; +static const char __pyx_k_cuFileReadAsync[] = "__cuFileReadAsync"; +static const char __pyx_k_cuda_pathfinder[] = "cuda.pathfinder"; +static const char __pyx_k_cuFileDriverOpen[] = "__cuFileDriverOpen"; +static const char __pyx_k_cuFileGetVersion[] = "__cuFileGetVersion"; +static const char __pyx_k_cuFileWriteAsync[] = "__cuFileWriteAsync"; +static const char __pyx_k_NotSupportedError[] = "NotSupportedError"; +static const char __pyx_k_cuFileBufRegister[] = "__cuFileBufRegister"; +static const char __pyx_k_cuFileDriverClose[] = "__cuFileDriverClose"; +static const char __pyx_k_asyncio_coroutines[] = "asyncio.coroutines"; +static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; +static const char __pyx_k_cuFileBatchIOSetUp[] = "__cuFileBatchIOSetUp"; +static const char __pyx_k_cuFileBatchIOCancel[] = "__cuFileBatchIOCancel"; +static const char __pyx_k_cuFileBatchIOSubmit[] = "__cuFileBatchIOSubmit"; +static const char __pyx_k_cuFileBufDeregister[] = "__cuFileBufDeregister"; +static const char __pyx_k_cuFileBatchIODestroy[] = "__cuFileBatchIODestroy"; +static const char __pyx_k_cuFileDriverClose_v2[] = "__cuFileDriverClose_v2"; +static const char __pyx_k_cuFileHandleRegister[] = "__cuFileHandleRegister"; +static const char __pyx_k_cuFileStreamRegister[] = "__cuFileStreamRegister"; +static const char __pyx_k_FunctionNotFoundError[] = "FunctionNotFoundError"; +static const char __pyx_k_cuFileBatchIOGetStatus[] = "__cuFileBatchIOGetStatus"; +static const char __pyx_k_cuFileGetParameterBool[] = "__cuFileGetParameterBool"; +static const char __pyx_k_cuFileHandleDeregister[] = "__cuFileHandleDeregister"; +static const char __pyx_k_cuFileSetParameterBool[] = "__cuFileSetParameterBool"; +static const char __pyx_k_cuFileStreamDeregister[] = "__cuFileStreamDeregister"; +static const char __pyx_k_cuFileDriverSetPollMode[] = "__cuFileDriverSetPollMode"; +static const char __pyx_k_cuFileGetParameterSizeT[] = "__cuFileGetParameterSizeT"; +static const char __pyx_k_cuFileSetParameterSizeT[] = "__cuFileSetParameterSizeT"; +static const char __pyx_k_load_nvidia_dynamic_lib[] = "load_nvidia_dynamic_lib"; +static const char __pyx_k_cuFileGetParameterString[] = "__cuFileGetParameterString"; +static const char __pyx_k_cuFileSetParameterString[] = "__cuFileSetParameterString"; +static const char __pyx_k_inspect_function_pointer[] = "_inspect_function_pointer"; +static const char __pyx_k_cuFileDriverGetProperties[] = "__cuFileDriverGetProperties"; +static const char __pyx_k_inspect_function_pointers[] = "_inspect_function_pointers"; +static const char __pyx_k_cuFileDriverSetMaxCacheSize[] = "__cuFileDriverSetMaxCacheSize"; +static const char __pyx_k_cuFileDriverSetMaxDirectIOSize[] = "__cuFileDriverSetMaxDirectIOSize"; +static const char __pyx_k_cuFileDriverSetMaxPinnedMemSiz[] = "__cuFileDriverSetMaxPinnedMemSize"; +static const char __pyx_k_cuda_bindings__internal_cufile[] = "cuda.bindings._internal.cufile"; +static const char __pyx_k_function_cuFileBatchIOCancel_is[] = "function cuFileBatchIOCancel is not found"; +static const char __pyx_k_function_cuFileBatchIOGetStatus[] = "function cuFileBatchIOGetStatus is not found"; +static const char __pyx_k_function_cuFileBatchIOSubmit_is[] = "function cuFileBatchIOSubmit is not found"; +static const char __pyx_k_function_cuFileBufDeregister_is[] = "function cuFileBufDeregister is not found"; +static const char __pyx_k_function_cuFileGetParameterBool[] = "function cuFileGetParameterBool is not found"; +static const char __pyx_k_function_cuFileHandleDeregister[] = "function cuFileHandleDeregister is not found"; +static const char __pyx_k_function_cuFileReadAsync_is_not[] = "function cuFileReadAsync is not found"; +static const char __pyx_k_function_cuFileSetParameterBool[] = "function cuFileSetParameterBool is not found"; +static const char __pyx_k_function_cuFileStreamDeregister[] = "function cuFileStreamDeregister is not found"; +static const char __pyx_k_z_q_Q_Zq_z_A_Ja_Q_Ja_1_Zq_z_A_z[] = "\200\001\330\004\005\330\004\007\200z\220\027\230\001\330\010\017\210q\340\004\031\230\021\330\004\025\220Q\340\004\005\330\004\010\210\001\320\t%\240Z\250q\340\004\005\330\004\010\210\001\320\t'\240z\260\021\340\004\005\330\004\010\210\001\320\t\"\240*\250A\340\004\005\330\004\010\210\001\320\t$\240J\250a\340\004\005\330\004\010\210\001\320\t\033\230:\240Q\340\004\005\330\004\010\210\001\320\t\034\230J\240a\340\004\005\330\004\010\210\001\320\t!\240\032\2501\340\004\005\330\004\010\210\001\320\t%\240Z\250q\340\004\005\330\004\010\210\001\320\t\037\230z\250\021\340\004\005\330\004\010\210\001\320\t*\250*\260A\340\004\005\330\004\010\210\001\320\t(\250\n\260!\340\004\005\330\004\010\210\001\320\t/\250z\270\021\340\004\005\330\004\010\210\001\320\t,\250J\260a\340\004\005\330\004\010\210\001\320\t0\260\n\270!\340\004\005\330\004\010\210\001\320\t#\240:\250Q\340\004\005\330\004\010\210\001\320\t$\240J\250a\340\004\005\330\004\010\210\001\320\t'\240z\260\021\340\004\005\330\004\010\210\001\320\t$\240J\250a\340\004\005\330\004\010\210\001\320\t%\240Z\250q\340\004\005\330\004\010\210\001\320\t \240\n\250!\340\004\005\330\004\010\210\001\320\t!\240\032\2501\340\004\005\330\004\010\210\001\320\t%\240Z\250q\340\004\005\330\004\010\210\001\320\t'\240z\260\021\340\004\005\330\004\010\210\001\320\t!\240\032\2501\340\004\005\330\004\010\210\001\320\t(\250\n\260!\340\004\005\330\004\010\210\001\320\t'\240z\260\021\340\004\005\330\004\010\210\001\320\t)\250\032\2601\340\004\005\330\004\010\210\001\320\t(\250\n\260!\340\004\005\330\004\010\210\001\320\t'\240z\260\021\340\004\005\330\004\010\210\001\320\t)\250\032\2601\340\004\005\330\004\010\210\001\320\t\"\240*\250A\340\004\020\220\001\330\004\013\2101"; +static const char __pyx_k_Note_that_Cython_is_deliberately[] = "Note that Cython is deliberately stricter than PEP-484 and rejects subclasses of builtin types. If you need to pass subclasses then set the 'annotation_typing' directive to False."; +static const char __pyx_k_cuda_bindings__internal_cufile_p[] = "cuda/bindings/_internal/cufile.pyx"; +static const char __pyx_k_function_cuFileBatchIODestroy_is[] = "function cuFileBatchIODestroy is not found"; +static const char __pyx_k_function_cuFileBatchIOSetUp_is_n[] = "function cuFileBatchIOSetUp is not found"; +static const char __pyx_k_function_cuFileBufRegister_is_no[] = "function cuFileBufRegister is not found"; +static const char __pyx_k_function_cuFileDriverClose_is_no[] = "function cuFileDriverClose is not found"; +static const char __pyx_k_function_cuFileDriverClose_v2_is[] = "function cuFileDriverClose_v2 is not found"; +static const char __pyx_k_function_cuFileDriverGetProperti[] = "function cuFileDriverGetProperties is not found"; +static const char __pyx_k_function_cuFileDriverOpen_is_not[] = "function cuFileDriverOpen is not found"; +static const char __pyx_k_function_cuFileDriverSetMaxCache[] = "function cuFileDriverSetMaxCacheSize is not found"; +static const char __pyx_k_function_cuFileDriverSetMaxDirec[] = "function cuFileDriverSetMaxDirectIOSize is not found"; +static const char __pyx_k_function_cuFileDriverSetMaxPinne[] = "function cuFileDriverSetMaxPinnedMemSize is not found"; +static const char __pyx_k_function_cuFileDriverSetPollMode[] = "function cuFileDriverSetPollMode is not found"; +static const char __pyx_k_function_cuFileGetParameterSizeT[] = "function cuFileGetParameterSizeT is not found"; +static const char __pyx_k_function_cuFileGetParameterStrin[] = "function cuFileGetParameterString is not found"; +static const char __pyx_k_function_cuFileGetVersion_is_not[] = "function cuFileGetVersion is not found"; +static const char __pyx_k_function_cuFileHandleRegister_is[] = "function cuFileHandleRegister is not found"; +static const char __pyx_k_function_cuFileRead_is_not_found[] = "function cuFileRead is not found"; +static const char __pyx_k_function_cuFileSetParameterSizeT[] = "function cuFileSetParameterSizeT is not found"; +static const char __pyx_k_function_cuFileSetParameterStrin[] = "function cuFileSetParameterString is not found"; +static const char __pyx_k_function_cuFileStreamRegister_is[] = "function cuFileStreamRegister is not found"; +static const char __pyx_k_function_cuFileUseCount_is_not_f[] = "function cuFileUseCount is not found"; +static const char __pyx_k_function_cuFileWriteAsync_is_not[] = "function cuFileWriteAsync is not found"; +static const char __pyx_k_function_cuFileWrite_is_not_foun[] = "function cuFileWrite is not found"; +/* #### Code section: decls ### */ +static PyObject *__pyx_pf_4cuda_8bindings_9_internal_6cufile__inspect_function_pointers(CYTHON_UNUSED PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_pf_4cuda_8bindings_9_internal_6cufile_2_inspect_function_pointer(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_name); /* proto */ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + #ifdef __Pyx_CyFunction_USED + PyTypeObject *__pyx_CyFunctionType; + #endif + #ifdef __Pyx_FusedFunction_USED + PyTypeObject *__pyx_FusedFunctionType; + #endif + #ifdef __Pyx_Generator_USED + PyTypeObject *__pyx_GeneratorType; + #endif + #ifdef __Pyx_IterableCoroutine_USED + PyTypeObject *__pyx_IterableCoroutineType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineAwaitType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineType; + #endif + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop; + PyObject *__pyx_tuple[1]; + PyObject *__pyx_codeobj_tab[2]; + PyObject *__pyx_string_tab[95]; +/* #### Code section: module_state_contents ### */ +/* CommonTypesMetaclass.module_state_decls */ +PyTypeObject *__pyx_CommonTypesMetaclassType; + +/* CachedMethodType.module_state_decls */ +#if CYTHON_COMPILING_IN_LIMITED_API +PyObject *__Pyx_CachedMethodType; +#endif + +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_n_u_FunctionNotFoundError __pyx_string_tab[1] +#define __pyx_n_u_Lock __pyx_string_tab[2] +#define __pyx_n_u_NotSupportedError __pyx_string_tab[3] +#define __pyx_kp_u_Note_that_Cython_is_deliberately __pyx_string_tab[4] +#define __pyx_kp_u__2 __pyx_string_tab[5] +#define __pyx_kp_u_add_note __pyx_string_tab[6] +#define __pyx_n_u_asyncio_coroutines __pyx_string_tab[7] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[8] +#define __pyx_n_u_cuFileBatchIOCancel __pyx_string_tab[9] +#define __pyx_n_u_cuFileBatchIODestroy __pyx_string_tab[10] +#define __pyx_n_u_cuFileBatchIOGetStatus __pyx_string_tab[11] +#define __pyx_n_u_cuFileBatchIOSetUp __pyx_string_tab[12] +#define __pyx_n_u_cuFileBatchIOSubmit __pyx_string_tab[13] +#define __pyx_n_u_cuFileBufDeregister __pyx_string_tab[14] +#define __pyx_n_u_cuFileBufRegister __pyx_string_tab[15] +#define __pyx_n_u_cuFileDriverClose __pyx_string_tab[16] +#define __pyx_n_u_cuFileDriverClose_v2 __pyx_string_tab[17] +#define __pyx_n_u_cuFileDriverGetProperties __pyx_string_tab[18] +#define __pyx_n_u_cuFileDriverOpen __pyx_string_tab[19] +#define __pyx_n_u_cuFileDriverSetMaxCacheSize __pyx_string_tab[20] +#define __pyx_n_u_cuFileDriverSetMaxDirectIOSize __pyx_string_tab[21] +#define __pyx_n_u_cuFileDriverSetMaxPinnedMemSiz __pyx_string_tab[22] +#define __pyx_n_u_cuFileDriverSetPollMode __pyx_string_tab[23] +#define __pyx_n_u_cuFileGetParameterBool __pyx_string_tab[24] +#define __pyx_n_u_cuFileGetParameterSizeT __pyx_string_tab[25] +#define __pyx_n_u_cuFileGetParameterString __pyx_string_tab[26] +#define __pyx_n_u_cuFileGetVersion __pyx_string_tab[27] +#define __pyx_n_u_cuFileHandleDeregister __pyx_string_tab[28] +#define __pyx_n_u_cuFileHandleRegister __pyx_string_tab[29] +#define __pyx_n_u_cuFileRead __pyx_string_tab[30] +#define __pyx_n_u_cuFileReadAsync __pyx_string_tab[31] +#define __pyx_n_u_cuFileSetParameterBool __pyx_string_tab[32] +#define __pyx_n_u_cuFileSetParameterSizeT __pyx_string_tab[33] +#define __pyx_n_u_cuFileSetParameterString __pyx_string_tab[34] +#define __pyx_n_u_cuFileStreamDeregister __pyx_string_tab[35] +#define __pyx_n_u_cuFileStreamRegister __pyx_string_tab[36] +#define __pyx_n_u_cuFileUseCount __pyx_string_tab[37] +#define __pyx_n_u_cuFileWrite __pyx_string_tab[38] +#define __pyx_n_u_cuFileWriteAsync __pyx_string_tab[39] +#define __pyx_n_u_cuda_bindings__internal_cufile __pyx_string_tab[40] +#define __pyx_kp_u_cuda_bindings__internal_cufile_p __pyx_string_tab[41] +#define __pyx_n_u_cuda_pathfinder __pyx_string_tab[42] +#define __pyx_n_u_cufile __pyx_string_tab[43] +#define __pyx_n_u_enter __pyx_string_tab[44] +#define __pyx_n_u_exit __pyx_string_tab[45] +#define __pyx_n_u_func __pyx_string_tab[46] +#define __pyx_kp_u_function_cuFileBatchIOCancel_is __pyx_string_tab[47] +#define __pyx_kp_u_function_cuFileBatchIODestroy_is __pyx_string_tab[48] +#define __pyx_kp_u_function_cuFileBatchIOGetStatus __pyx_string_tab[49] +#define __pyx_kp_u_function_cuFileBatchIOSetUp_is_n __pyx_string_tab[50] +#define __pyx_kp_u_function_cuFileBatchIOSubmit_is __pyx_string_tab[51] +#define __pyx_kp_u_function_cuFileBufDeregister_is __pyx_string_tab[52] +#define __pyx_kp_u_function_cuFileBufRegister_is_no __pyx_string_tab[53] +#define __pyx_kp_u_function_cuFileDriverClose_is_no __pyx_string_tab[54] +#define __pyx_kp_u_function_cuFileDriverClose_v2_is __pyx_string_tab[55] +#define __pyx_kp_u_function_cuFileDriverGetProperti __pyx_string_tab[56] +#define __pyx_kp_u_function_cuFileDriverOpen_is_not __pyx_string_tab[57] +#define __pyx_kp_u_function_cuFileDriverSetMaxCache __pyx_string_tab[58] +#define __pyx_kp_u_function_cuFileDriverSetMaxDirec __pyx_string_tab[59] +#define __pyx_kp_u_function_cuFileDriverSetMaxPinne __pyx_string_tab[60] +#define __pyx_kp_u_function_cuFileDriverSetPollMode __pyx_string_tab[61] +#define __pyx_kp_u_function_cuFileGetParameterBool __pyx_string_tab[62] +#define __pyx_kp_u_function_cuFileGetParameterSizeT __pyx_string_tab[63] +#define __pyx_kp_u_function_cuFileGetParameterStrin __pyx_string_tab[64] +#define __pyx_kp_u_function_cuFileGetVersion_is_not __pyx_string_tab[65] +#define __pyx_kp_u_function_cuFileHandleDeregister __pyx_string_tab[66] +#define __pyx_kp_u_function_cuFileHandleRegister_is __pyx_string_tab[67] +#define __pyx_kp_u_function_cuFileReadAsync_is_not __pyx_string_tab[68] +#define __pyx_kp_u_function_cuFileRead_is_not_found __pyx_string_tab[69] +#define __pyx_kp_u_function_cuFileSetParameterBool __pyx_string_tab[70] +#define __pyx_kp_u_function_cuFileSetParameterSizeT __pyx_string_tab[71] +#define __pyx_kp_u_function_cuFileSetParameterStrin __pyx_string_tab[72] +#define __pyx_kp_u_function_cuFileStreamDeregister __pyx_string_tab[73] +#define __pyx_kp_u_function_cuFileStreamRegister_is __pyx_string_tab[74] +#define __pyx_kp_u_function_cuFileUseCount_is_not_f __pyx_string_tab[75] +#define __pyx_kp_u_function_cuFileWriteAsync_is_not __pyx_string_tab[76] +#define __pyx_kp_u_function_cuFileWrite_is_not_foun __pyx_string_tab[77] +#define __pyx_n_u_handle_uint __pyx_string_tab[78] +#define __pyx_n_u_initializing __pyx_string_tab[79] +#define __pyx_n_u_inspect_function_pointer __pyx_string_tab[80] +#define __pyx_n_u_inspect_function_pointers __pyx_string_tab[81] +#define __pyx_n_u_is_coroutine __pyx_string_tab[82] +#define __pyx_n_u_load_nvidia_dynamic_lib __pyx_string_tab[83] +#define __pyx_n_u_main __pyx_string_tab[84] +#define __pyx_n_u_module __pyx_string_tab[85] +#define __pyx_n_u_name __pyx_string_tab[86] +#define __pyx_n_u_name_2 __pyx_string_tab[87] +#define __pyx_n_u_pop __pyx_string_tab[88] +#define __pyx_n_u_qualname __pyx_string_tab[89] +#define __pyx_n_u_set_name __pyx_string_tab[90] +#define __pyx_n_u_spec __pyx_string_tab[91] +#define __pyx_n_u_test __pyx_string_tab[92] +#define __pyx_n_u_threading __pyx_string_tab[93] +#define __pyx_n_u_utils __pyx_string_tab[94] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_CLEAR(clear_module_state->__pyx_FusedFunctionType); + #endif + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + for (int i=0; i<1; ++i) { Py_CLEAR(clear_module_state->__pyx_tuple[i]); } + for (int i=0; i<2; ++i) { Py_CLEAR(clear_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<95; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_VISIT(traverse_module_state->__pyx_FusedFunctionType); + #endif + for (int i=0; i<1; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_tuple[i]); } + for (int i=0; i<2; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<95; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "cuda/bindings/_internal/cufile.pyx":77 + * + * + * cdef void* load_library() except* with gil: # <<<<<<<<<<<<<< + * cdef uintptr_t handle = load_nvidia_dynamic_lib("cufile")._handle_uint + * return handle +*/ + +static void *__pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(void) { + uintptr_t __pyx_v_handle; + void *__pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + uintptr_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("load_library", 0); + + /* "cuda/bindings/_internal/cufile.pyx":78 + * + * cdef void* load_library() except* with gil: + * cdef uintptr_t handle = load_nvidia_dynamic_lib("cufile")._handle_uint # <<<<<<<<<<<<<< + * return handle + * +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 78, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_mstate_global->__pyx_n_u_cufile}; + __pyx_t_1 = __Pyx_PyObject_FastCall(__pyx_t_3, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 78, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_handle_uint); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 78, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_5 = __Pyx_PyLong_As_size_t(__pyx_t_3); if (unlikely((__pyx_t_5 == ((uintptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 78, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_handle = __pyx_t_5; + + /* "cuda/bindings/_internal/cufile.pyx":79 + * cdef void* load_library() except* with gil: + * cdef uintptr_t handle = load_nvidia_dynamic_lib("cufile")._handle_uint + * return handle # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = ((void *)__pyx_v_handle); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":77 + * + * + * cdef void* load_library() except* with gil: # <<<<<<<<<<<<<< + * cdef uintptr_t handle = load_nvidia_dynamic_lib("cufile")._handle_uint + * return handle +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("cuda.bindings._internal.cufile.load_library", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":82 + * + * + * cdef int __check_or_init_cufile() except -1 nogil: # <<<<<<<<<<<<<< + * global __py_cufile_init + * +*/ + +static int __pyx_f_4cuda_8bindings_9_internal_6cufile___check_or_init_cufile(void) { + void *__pyx_v_handle; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + void *__pyx_t_10; + PyObject *__pyx_t_11 = NULL; + PyObject *__pyx_t_12 = NULL; + int __pyx_t_13; + int __pyx_t_14; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("__check_or_init_cufile", 1); + + /* "cuda/bindings/_internal/cufile.pyx":85 + * global __py_cufile_init + * + * cdef void* handle = NULL # <<<<<<<<<<<<<< + * + * with gil, __symbol_lock: +*/ + __pyx_v_handle = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":87 + * cdef void* handle = NULL + * + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * # Load function + * global __cuFileHandleRegister +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + /*with:*/ { + __pyx_t_1 = __Pyx_PyObject_LookupSpecial(__pyx_v_4cuda_8bindings_9_internal_6cufile___symbol_lock, __pyx_mstate_global->__pyx_n_u_exit); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 87, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = NULL; + __pyx_t_4 = __Pyx_PyObject_LookupSpecial(__pyx_v_4cuda_8bindings_9_internal_6cufile___symbol_lock, __pyx_mstate_global->__pyx_n_u_enter); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 87, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (likely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_4, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_4, __pyx_callargs+__pyx_t_5, (1-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 87, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + /*try:*/ { + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_6, &__pyx_t_7, &__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":90 + * # Load function + * global __cuFileHandleRegister + * __cuFileHandleRegister = dlsym(RTLD_DEFAULT, 'cuFileHandleRegister') # <<<<<<<<<<<<<< + * if __cuFileHandleRegister == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleRegister = dlsym(RTLD_DEFAULT, ((char const *)"cuFileHandleRegister")); + + /* "cuda/bindings/_internal/cufile.pyx":91 + * global __cuFileHandleRegister + * __cuFileHandleRegister = dlsym(RTLD_DEFAULT, 'cuFileHandleRegister') + * if __cuFileHandleRegister == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleRegister == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":92 + * __cuFileHandleRegister = dlsym(RTLD_DEFAULT, 'cuFileHandleRegister') + * if __cuFileHandleRegister == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileHandleRegister = dlsym(handle, 'cuFileHandleRegister') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":93 + * if __cuFileHandleRegister == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileHandleRegister = dlsym(handle, 'cuFileHandleRegister') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 93, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":92 + * __cuFileHandleRegister = dlsym(RTLD_DEFAULT, 'cuFileHandleRegister') + * if __cuFileHandleRegister == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileHandleRegister = dlsym(handle, 'cuFileHandleRegister') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":94 + * if handle == NULL: + * handle = load_library() + * __cuFileHandleRegister = dlsym(handle, 'cuFileHandleRegister') # <<<<<<<<<<<<<< + * + * global __cuFileHandleDeregister +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleRegister = dlsym(__pyx_v_handle, ((char const *)"cuFileHandleRegister")); + + /* "cuda/bindings/_internal/cufile.pyx":91 + * global __cuFileHandleRegister + * __cuFileHandleRegister = dlsym(RTLD_DEFAULT, 'cuFileHandleRegister') + * if __cuFileHandleRegister == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":97 + * + * global __cuFileHandleDeregister + * __cuFileHandleDeregister = dlsym(RTLD_DEFAULT, 'cuFileHandleDeregister') # <<<<<<<<<<<<<< + * if __cuFileHandleDeregister == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleDeregister = dlsym(RTLD_DEFAULT, ((char const *)"cuFileHandleDeregister")); + + /* "cuda/bindings/_internal/cufile.pyx":98 + * global __cuFileHandleDeregister + * __cuFileHandleDeregister = dlsym(RTLD_DEFAULT, 'cuFileHandleDeregister') + * if __cuFileHandleDeregister == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleDeregister == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":99 + * __cuFileHandleDeregister = dlsym(RTLD_DEFAULT, 'cuFileHandleDeregister') + * if __cuFileHandleDeregister == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileHandleDeregister = dlsym(handle, 'cuFileHandleDeregister') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":100 + * if __cuFileHandleDeregister == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileHandleDeregister = dlsym(handle, 'cuFileHandleDeregister') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 100, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":99 + * __cuFileHandleDeregister = dlsym(RTLD_DEFAULT, 'cuFileHandleDeregister') + * if __cuFileHandleDeregister == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileHandleDeregister = dlsym(handle, 'cuFileHandleDeregister') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":101 + * if handle == NULL: + * handle = load_library() + * __cuFileHandleDeregister = dlsym(handle, 'cuFileHandleDeregister') # <<<<<<<<<<<<<< + * + * global __cuFileBufRegister +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleDeregister = dlsym(__pyx_v_handle, ((char const *)"cuFileHandleDeregister")); + + /* "cuda/bindings/_internal/cufile.pyx":98 + * global __cuFileHandleDeregister + * __cuFileHandleDeregister = dlsym(RTLD_DEFAULT, 'cuFileHandleDeregister') + * if __cuFileHandleDeregister == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":104 + * + * global __cuFileBufRegister + * __cuFileBufRegister = dlsym(RTLD_DEFAULT, 'cuFileBufRegister') # <<<<<<<<<<<<<< + * if __cuFileBufRegister == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufRegister = dlsym(RTLD_DEFAULT, ((char const *)"cuFileBufRegister")); + + /* "cuda/bindings/_internal/cufile.pyx":105 + * global __cuFileBufRegister + * __cuFileBufRegister = dlsym(RTLD_DEFAULT, 'cuFileBufRegister') + * if __cuFileBufRegister == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufRegister == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":106 + * __cuFileBufRegister = dlsym(RTLD_DEFAULT, 'cuFileBufRegister') + * if __cuFileBufRegister == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBufRegister = dlsym(handle, 'cuFileBufRegister') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":107 + * if __cuFileBufRegister == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileBufRegister = dlsym(handle, 'cuFileBufRegister') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 107, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":106 + * __cuFileBufRegister = dlsym(RTLD_DEFAULT, 'cuFileBufRegister') + * if __cuFileBufRegister == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBufRegister = dlsym(handle, 'cuFileBufRegister') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":108 + * if handle == NULL: + * handle = load_library() + * __cuFileBufRegister = dlsym(handle, 'cuFileBufRegister') # <<<<<<<<<<<<<< + * + * global __cuFileBufDeregister +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufRegister = dlsym(__pyx_v_handle, ((char const *)"cuFileBufRegister")); + + /* "cuda/bindings/_internal/cufile.pyx":105 + * global __cuFileBufRegister + * __cuFileBufRegister = dlsym(RTLD_DEFAULT, 'cuFileBufRegister') + * if __cuFileBufRegister == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":111 + * + * global __cuFileBufDeregister + * __cuFileBufDeregister = dlsym(RTLD_DEFAULT, 'cuFileBufDeregister') # <<<<<<<<<<<<<< + * if __cuFileBufDeregister == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufDeregister = dlsym(RTLD_DEFAULT, ((char const *)"cuFileBufDeregister")); + + /* "cuda/bindings/_internal/cufile.pyx":112 + * global __cuFileBufDeregister + * __cuFileBufDeregister = dlsym(RTLD_DEFAULT, 'cuFileBufDeregister') + * if __cuFileBufDeregister == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufDeregister == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":113 + * __cuFileBufDeregister = dlsym(RTLD_DEFAULT, 'cuFileBufDeregister') + * if __cuFileBufDeregister == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBufDeregister = dlsym(handle, 'cuFileBufDeregister') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":114 + * if __cuFileBufDeregister == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileBufDeregister = dlsym(handle, 'cuFileBufDeregister') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 114, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":113 + * __cuFileBufDeregister = dlsym(RTLD_DEFAULT, 'cuFileBufDeregister') + * if __cuFileBufDeregister == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBufDeregister = dlsym(handle, 'cuFileBufDeregister') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":115 + * if handle == NULL: + * handle = load_library() + * __cuFileBufDeregister = dlsym(handle, 'cuFileBufDeregister') # <<<<<<<<<<<<<< + * + * global __cuFileRead +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufDeregister = dlsym(__pyx_v_handle, ((char const *)"cuFileBufDeregister")); + + /* "cuda/bindings/_internal/cufile.pyx":112 + * global __cuFileBufDeregister + * __cuFileBufDeregister = dlsym(RTLD_DEFAULT, 'cuFileBufDeregister') + * if __cuFileBufDeregister == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":118 + * + * global __cuFileRead + * __cuFileRead = dlsym(RTLD_DEFAULT, 'cuFileRead') # <<<<<<<<<<<<<< + * if __cuFileRead == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileRead = dlsym(RTLD_DEFAULT, ((char const *)"cuFileRead")); + + /* "cuda/bindings/_internal/cufile.pyx":119 + * global __cuFileRead + * __cuFileRead = dlsym(RTLD_DEFAULT, 'cuFileRead') + * if __cuFileRead == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileRead == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":120 + * __cuFileRead = dlsym(RTLD_DEFAULT, 'cuFileRead') + * if __cuFileRead == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileRead = dlsym(handle, 'cuFileRead') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":121 + * if __cuFileRead == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileRead = dlsym(handle, 'cuFileRead') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 121, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":120 + * __cuFileRead = dlsym(RTLD_DEFAULT, 'cuFileRead') + * if __cuFileRead == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileRead = dlsym(handle, 'cuFileRead') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":122 + * if handle == NULL: + * handle = load_library() + * __cuFileRead = dlsym(handle, 'cuFileRead') # <<<<<<<<<<<<<< + * + * global __cuFileWrite +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileRead = dlsym(__pyx_v_handle, ((char const *)"cuFileRead")); + + /* "cuda/bindings/_internal/cufile.pyx":119 + * global __cuFileRead + * __cuFileRead = dlsym(RTLD_DEFAULT, 'cuFileRead') + * if __cuFileRead == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":125 + * + * global __cuFileWrite + * __cuFileWrite = dlsym(RTLD_DEFAULT, 'cuFileWrite') # <<<<<<<<<<<<<< + * if __cuFileWrite == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWrite = dlsym(RTLD_DEFAULT, ((char const *)"cuFileWrite")); + + /* "cuda/bindings/_internal/cufile.pyx":126 + * global __cuFileWrite + * __cuFileWrite = dlsym(RTLD_DEFAULT, 'cuFileWrite') + * if __cuFileWrite == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWrite == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":127 + * __cuFileWrite = dlsym(RTLD_DEFAULT, 'cuFileWrite') + * if __cuFileWrite == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileWrite = dlsym(handle, 'cuFileWrite') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":128 + * if __cuFileWrite == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileWrite = dlsym(handle, 'cuFileWrite') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 128, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":127 + * __cuFileWrite = dlsym(RTLD_DEFAULT, 'cuFileWrite') + * if __cuFileWrite == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileWrite = dlsym(handle, 'cuFileWrite') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":129 + * if handle == NULL: + * handle = load_library() + * __cuFileWrite = dlsym(handle, 'cuFileWrite') # <<<<<<<<<<<<<< + * + * global __cuFileDriverOpen +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWrite = dlsym(__pyx_v_handle, ((char const *)"cuFileWrite")); + + /* "cuda/bindings/_internal/cufile.pyx":126 + * global __cuFileWrite + * __cuFileWrite = dlsym(RTLD_DEFAULT, 'cuFileWrite') + * if __cuFileWrite == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":132 + * + * global __cuFileDriverOpen + * __cuFileDriverOpen = dlsym(RTLD_DEFAULT, 'cuFileDriverOpen') # <<<<<<<<<<<<<< + * if __cuFileDriverOpen == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverOpen = dlsym(RTLD_DEFAULT, ((char const *)"cuFileDriverOpen")); + + /* "cuda/bindings/_internal/cufile.pyx":133 + * global __cuFileDriverOpen + * __cuFileDriverOpen = dlsym(RTLD_DEFAULT, 'cuFileDriverOpen') + * if __cuFileDriverOpen == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverOpen == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":134 + * __cuFileDriverOpen = dlsym(RTLD_DEFAULT, 'cuFileDriverOpen') + * if __cuFileDriverOpen == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverOpen = dlsym(handle, 'cuFileDriverOpen') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":135 + * if __cuFileDriverOpen == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileDriverOpen = dlsym(handle, 'cuFileDriverOpen') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 135, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":134 + * __cuFileDriverOpen = dlsym(RTLD_DEFAULT, 'cuFileDriverOpen') + * if __cuFileDriverOpen == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverOpen = dlsym(handle, 'cuFileDriverOpen') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":136 + * if handle == NULL: + * handle = load_library() + * __cuFileDriverOpen = dlsym(handle, 'cuFileDriverOpen') # <<<<<<<<<<<<<< + * + * global __cuFileDriverClose_v2 +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverOpen = dlsym(__pyx_v_handle, ((char const *)"cuFileDriverOpen")); + + /* "cuda/bindings/_internal/cufile.pyx":133 + * global __cuFileDriverOpen + * __cuFileDriverOpen = dlsym(RTLD_DEFAULT, 'cuFileDriverOpen') + * if __cuFileDriverOpen == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":139 + * + * global __cuFileDriverClose_v2 + * __cuFileDriverClose_v2 = dlsym(RTLD_DEFAULT, 'cuFileDriverClose_v2') # <<<<<<<<<<<<<< + * if __cuFileDriverClose_v2 == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose_v2 = dlsym(RTLD_DEFAULT, ((char const *)"cuFileDriverClose_v2")); + + /* "cuda/bindings/_internal/cufile.pyx":140 + * global __cuFileDriverClose_v2 + * __cuFileDriverClose_v2 = dlsym(RTLD_DEFAULT, 'cuFileDriverClose_v2') + * if __cuFileDriverClose_v2 == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose_v2 == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":141 + * __cuFileDriverClose_v2 = dlsym(RTLD_DEFAULT, 'cuFileDriverClose_v2') + * if __cuFileDriverClose_v2 == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverClose_v2 = dlsym(handle, 'cuFileDriverClose_v2') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":142 + * if __cuFileDriverClose_v2 == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileDriverClose_v2 = dlsym(handle, 'cuFileDriverClose_v2') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 142, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":141 + * __cuFileDriverClose_v2 = dlsym(RTLD_DEFAULT, 'cuFileDriverClose_v2') + * if __cuFileDriverClose_v2 == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverClose_v2 = dlsym(handle, 'cuFileDriverClose_v2') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":143 + * if handle == NULL: + * handle = load_library() + * __cuFileDriverClose_v2 = dlsym(handle, 'cuFileDriverClose_v2') # <<<<<<<<<<<<<< + * + * global __cuFileUseCount +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose_v2 = dlsym(__pyx_v_handle, ((char const *)"cuFileDriverClose_v2")); + + /* "cuda/bindings/_internal/cufile.pyx":140 + * global __cuFileDriverClose_v2 + * __cuFileDriverClose_v2 = dlsym(RTLD_DEFAULT, 'cuFileDriverClose_v2') + * if __cuFileDriverClose_v2 == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":146 + * + * global __cuFileUseCount + * __cuFileUseCount = dlsym(RTLD_DEFAULT, 'cuFileUseCount') # <<<<<<<<<<<<<< + * if __cuFileUseCount == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileUseCount = dlsym(RTLD_DEFAULT, ((char const *)"cuFileUseCount")); + + /* "cuda/bindings/_internal/cufile.pyx":147 + * global __cuFileUseCount + * __cuFileUseCount = dlsym(RTLD_DEFAULT, 'cuFileUseCount') + * if __cuFileUseCount == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileUseCount == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":148 + * __cuFileUseCount = dlsym(RTLD_DEFAULT, 'cuFileUseCount') + * if __cuFileUseCount == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileUseCount = dlsym(handle, 'cuFileUseCount') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":149 + * if __cuFileUseCount == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileUseCount = dlsym(handle, 'cuFileUseCount') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 149, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":148 + * __cuFileUseCount = dlsym(RTLD_DEFAULT, 'cuFileUseCount') + * if __cuFileUseCount == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileUseCount = dlsym(handle, 'cuFileUseCount') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":150 + * if handle == NULL: + * handle = load_library() + * __cuFileUseCount = dlsym(handle, 'cuFileUseCount') # <<<<<<<<<<<<<< + * + * global __cuFileDriverGetProperties +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileUseCount = dlsym(__pyx_v_handle, ((char const *)"cuFileUseCount")); + + /* "cuda/bindings/_internal/cufile.pyx":147 + * global __cuFileUseCount + * __cuFileUseCount = dlsym(RTLD_DEFAULT, 'cuFileUseCount') + * if __cuFileUseCount == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":153 + * + * global __cuFileDriverGetProperties + * __cuFileDriverGetProperties = dlsym(RTLD_DEFAULT, 'cuFileDriverGetProperties') # <<<<<<<<<<<<<< + * if __cuFileDriverGetProperties == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverGetProperties = dlsym(RTLD_DEFAULT, ((char const *)"cuFileDriverGetProperties")); + + /* "cuda/bindings/_internal/cufile.pyx":154 + * global __cuFileDriverGetProperties + * __cuFileDriverGetProperties = dlsym(RTLD_DEFAULT, 'cuFileDriverGetProperties') + * if __cuFileDriverGetProperties == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverGetProperties == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":155 + * __cuFileDriverGetProperties = dlsym(RTLD_DEFAULT, 'cuFileDriverGetProperties') + * if __cuFileDriverGetProperties == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverGetProperties = dlsym(handle, 'cuFileDriverGetProperties') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":156 + * if __cuFileDriverGetProperties == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileDriverGetProperties = dlsym(handle, 'cuFileDriverGetProperties') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 156, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":155 + * __cuFileDriverGetProperties = dlsym(RTLD_DEFAULT, 'cuFileDriverGetProperties') + * if __cuFileDriverGetProperties == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverGetProperties = dlsym(handle, 'cuFileDriverGetProperties') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":157 + * if handle == NULL: + * handle = load_library() + * __cuFileDriverGetProperties = dlsym(handle, 'cuFileDriverGetProperties') # <<<<<<<<<<<<<< + * + * global __cuFileDriverSetPollMode +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverGetProperties = dlsym(__pyx_v_handle, ((char const *)"cuFileDriverGetProperties")); + + /* "cuda/bindings/_internal/cufile.pyx":154 + * global __cuFileDriverGetProperties + * __cuFileDriverGetProperties = dlsym(RTLD_DEFAULT, 'cuFileDriverGetProperties') + * if __cuFileDriverGetProperties == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":160 + * + * global __cuFileDriverSetPollMode + * __cuFileDriverSetPollMode = dlsym(RTLD_DEFAULT, 'cuFileDriverSetPollMode') # <<<<<<<<<<<<<< + * if __cuFileDriverSetPollMode == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetPollMode = dlsym(RTLD_DEFAULT, ((char const *)"cuFileDriverSetPollMode")); + + /* "cuda/bindings/_internal/cufile.pyx":161 + * global __cuFileDriverSetPollMode + * __cuFileDriverSetPollMode = dlsym(RTLD_DEFAULT, 'cuFileDriverSetPollMode') + * if __cuFileDriverSetPollMode == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetPollMode == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":162 + * __cuFileDriverSetPollMode = dlsym(RTLD_DEFAULT, 'cuFileDriverSetPollMode') + * if __cuFileDriverSetPollMode == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverSetPollMode = dlsym(handle, 'cuFileDriverSetPollMode') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":163 + * if __cuFileDriverSetPollMode == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileDriverSetPollMode = dlsym(handle, 'cuFileDriverSetPollMode') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 163, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":162 + * __cuFileDriverSetPollMode = dlsym(RTLD_DEFAULT, 'cuFileDriverSetPollMode') + * if __cuFileDriverSetPollMode == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverSetPollMode = dlsym(handle, 'cuFileDriverSetPollMode') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":164 + * if handle == NULL: + * handle = load_library() + * __cuFileDriverSetPollMode = dlsym(handle, 'cuFileDriverSetPollMode') # <<<<<<<<<<<<<< + * + * global __cuFileDriverSetMaxDirectIOSize +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetPollMode = dlsym(__pyx_v_handle, ((char const *)"cuFileDriverSetPollMode")); + + /* "cuda/bindings/_internal/cufile.pyx":161 + * global __cuFileDriverSetPollMode + * __cuFileDriverSetPollMode = dlsym(RTLD_DEFAULT, 'cuFileDriverSetPollMode') + * if __cuFileDriverSetPollMode == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":167 + * + * global __cuFileDriverSetMaxDirectIOSize + * __cuFileDriverSetMaxDirectIOSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxDirectIOSize') # <<<<<<<<<<<<<< + * if __cuFileDriverSetMaxDirectIOSize == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxDirectIOSize = dlsym(RTLD_DEFAULT, ((char const *)"cuFileDriverSetMaxDirectIOSize")); + + /* "cuda/bindings/_internal/cufile.pyx":168 + * global __cuFileDriverSetMaxDirectIOSize + * __cuFileDriverSetMaxDirectIOSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxDirectIOSize') + * if __cuFileDriverSetMaxDirectIOSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxDirectIOSize == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":169 + * __cuFileDriverSetMaxDirectIOSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxDirectIOSize') + * if __cuFileDriverSetMaxDirectIOSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverSetMaxDirectIOSize = dlsym(handle, 'cuFileDriverSetMaxDirectIOSize') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":170 + * if __cuFileDriverSetMaxDirectIOSize == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileDriverSetMaxDirectIOSize = dlsym(handle, 'cuFileDriverSetMaxDirectIOSize') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 170, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":169 + * __cuFileDriverSetMaxDirectIOSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxDirectIOSize') + * if __cuFileDriverSetMaxDirectIOSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverSetMaxDirectIOSize = dlsym(handle, 'cuFileDriverSetMaxDirectIOSize') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":171 + * if handle == NULL: + * handle = load_library() + * __cuFileDriverSetMaxDirectIOSize = dlsym(handle, 'cuFileDriverSetMaxDirectIOSize') # <<<<<<<<<<<<<< + * + * global __cuFileDriverSetMaxCacheSize +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxDirectIOSize = dlsym(__pyx_v_handle, ((char const *)"cuFileDriverSetMaxDirectIOSize")); + + /* "cuda/bindings/_internal/cufile.pyx":168 + * global __cuFileDriverSetMaxDirectIOSize + * __cuFileDriverSetMaxDirectIOSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxDirectIOSize') + * if __cuFileDriverSetMaxDirectIOSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":174 + * + * global __cuFileDriverSetMaxCacheSize + * __cuFileDriverSetMaxCacheSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxCacheSize') # <<<<<<<<<<<<<< + * if __cuFileDriverSetMaxCacheSize == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxCacheSize = dlsym(RTLD_DEFAULT, ((char const *)"cuFileDriverSetMaxCacheSize")); + + /* "cuda/bindings/_internal/cufile.pyx":175 + * global __cuFileDriverSetMaxCacheSize + * __cuFileDriverSetMaxCacheSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxCacheSize') + * if __cuFileDriverSetMaxCacheSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxCacheSize == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":176 + * __cuFileDriverSetMaxCacheSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxCacheSize') + * if __cuFileDriverSetMaxCacheSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverSetMaxCacheSize = dlsym(handle, 'cuFileDriverSetMaxCacheSize') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":177 + * if __cuFileDriverSetMaxCacheSize == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileDriverSetMaxCacheSize = dlsym(handle, 'cuFileDriverSetMaxCacheSize') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 177, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":176 + * __cuFileDriverSetMaxCacheSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxCacheSize') + * if __cuFileDriverSetMaxCacheSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverSetMaxCacheSize = dlsym(handle, 'cuFileDriverSetMaxCacheSize') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":178 + * if handle == NULL: + * handle = load_library() + * __cuFileDriverSetMaxCacheSize = dlsym(handle, 'cuFileDriverSetMaxCacheSize') # <<<<<<<<<<<<<< + * + * global __cuFileDriverSetMaxPinnedMemSize +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxCacheSize = dlsym(__pyx_v_handle, ((char const *)"cuFileDriverSetMaxCacheSize")); + + /* "cuda/bindings/_internal/cufile.pyx":175 + * global __cuFileDriverSetMaxCacheSize + * __cuFileDriverSetMaxCacheSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxCacheSize') + * if __cuFileDriverSetMaxCacheSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":181 + * + * global __cuFileDriverSetMaxPinnedMemSize + * __cuFileDriverSetMaxPinnedMemSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxPinnedMemSize') # <<<<<<<<<<<<<< + * if __cuFileDriverSetMaxPinnedMemSize == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxPinnedMemSize = dlsym(RTLD_DEFAULT, ((char const *)"cuFileDriverSetMaxPinnedMemSize")); + + /* "cuda/bindings/_internal/cufile.pyx":182 + * global __cuFileDriverSetMaxPinnedMemSize + * __cuFileDriverSetMaxPinnedMemSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxPinnedMemSize') + * if __cuFileDriverSetMaxPinnedMemSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxPinnedMemSize == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":183 + * __cuFileDriverSetMaxPinnedMemSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxPinnedMemSize') + * if __cuFileDriverSetMaxPinnedMemSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverSetMaxPinnedMemSize = dlsym(handle, 'cuFileDriverSetMaxPinnedMemSize') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":184 + * if __cuFileDriverSetMaxPinnedMemSize == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileDriverSetMaxPinnedMemSize = dlsym(handle, 'cuFileDriverSetMaxPinnedMemSize') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 184, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":183 + * __cuFileDriverSetMaxPinnedMemSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxPinnedMemSize') + * if __cuFileDriverSetMaxPinnedMemSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverSetMaxPinnedMemSize = dlsym(handle, 'cuFileDriverSetMaxPinnedMemSize') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":185 + * if handle == NULL: + * handle = load_library() + * __cuFileDriverSetMaxPinnedMemSize = dlsym(handle, 'cuFileDriverSetMaxPinnedMemSize') # <<<<<<<<<<<<<< + * + * global __cuFileBatchIOSetUp +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxPinnedMemSize = dlsym(__pyx_v_handle, ((char const *)"cuFileDriverSetMaxPinnedMemSize")); + + /* "cuda/bindings/_internal/cufile.pyx":182 + * global __cuFileDriverSetMaxPinnedMemSize + * __cuFileDriverSetMaxPinnedMemSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxPinnedMemSize') + * if __cuFileDriverSetMaxPinnedMemSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":188 + * + * global __cuFileBatchIOSetUp + * __cuFileBatchIOSetUp = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSetUp') # <<<<<<<<<<<<<< + * if __cuFileBatchIOSetUp == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSetUp = dlsym(RTLD_DEFAULT, ((char const *)"cuFileBatchIOSetUp")); + + /* "cuda/bindings/_internal/cufile.pyx":189 + * global __cuFileBatchIOSetUp + * __cuFileBatchIOSetUp = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSetUp') + * if __cuFileBatchIOSetUp == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSetUp == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":190 + * __cuFileBatchIOSetUp = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSetUp') + * if __cuFileBatchIOSetUp == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBatchIOSetUp = dlsym(handle, 'cuFileBatchIOSetUp') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":191 + * if __cuFileBatchIOSetUp == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileBatchIOSetUp = dlsym(handle, 'cuFileBatchIOSetUp') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 191, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":190 + * __cuFileBatchIOSetUp = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSetUp') + * if __cuFileBatchIOSetUp == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBatchIOSetUp = dlsym(handle, 'cuFileBatchIOSetUp') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":192 + * if handle == NULL: + * handle = load_library() + * __cuFileBatchIOSetUp = dlsym(handle, 'cuFileBatchIOSetUp') # <<<<<<<<<<<<<< + * + * global __cuFileBatchIOSubmit +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSetUp = dlsym(__pyx_v_handle, ((char const *)"cuFileBatchIOSetUp")); + + /* "cuda/bindings/_internal/cufile.pyx":189 + * global __cuFileBatchIOSetUp + * __cuFileBatchIOSetUp = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSetUp') + * if __cuFileBatchIOSetUp == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":195 + * + * global __cuFileBatchIOSubmit + * __cuFileBatchIOSubmit = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSubmit') # <<<<<<<<<<<<<< + * if __cuFileBatchIOSubmit == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSubmit = dlsym(RTLD_DEFAULT, ((char const *)"cuFileBatchIOSubmit")); + + /* "cuda/bindings/_internal/cufile.pyx":196 + * global __cuFileBatchIOSubmit + * __cuFileBatchIOSubmit = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSubmit') + * if __cuFileBatchIOSubmit == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSubmit == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":197 + * __cuFileBatchIOSubmit = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSubmit') + * if __cuFileBatchIOSubmit == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBatchIOSubmit = dlsym(handle, 'cuFileBatchIOSubmit') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":198 + * if __cuFileBatchIOSubmit == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileBatchIOSubmit = dlsym(handle, 'cuFileBatchIOSubmit') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 198, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":197 + * __cuFileBatchIOSubmit = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSubmit') + * if __cuFileBatchIOSubmit == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBatchIOSubmit = dlsym(handle, 'cuFileBatchIOSubmit') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":199 + * if handle == NULL: + * handle = load_library() + * __cuFileBatchIOSubmit = dlsym(handle, 'cuFileBatchIOSubmit') # <<<<<<<<<<<<<< + * + * global __cuFileBatchIOGetStatus +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSubmit = dlsym(__pyx_v_handle, ((char const *)"cuFileBatchIOSubmit")); + + /* "cuda/bindings/_internal/cufile.pyx":196 + * global __cuFileBatchIOSubmit + * __cuFileBatchIOSubmit = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSubmit') + * if __cuFileBatchIOSubmit == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":202 + * + * global __cuFileBatchIOGetStatus + * __cuFileBatchIOGetStatus = dlsym(RTLD_DEFAULT, 'cuFileBatchIOGetStatus') # <<<<<<<<<<<<<< + * if __cuFileBatchIOGetStatus == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOGetStatus = dlsym(RTLD_DEFAULT, ((char const *)"cuFileBatchIOGetStatus")); + + /* "cuda/bindings/_internal/cufile.pyx":203 + * global __cuFileBatchIOGetStatus + * __cuFileBatchIOGetStatus = dlsym(RTLD_DEFAULT, 'cuFileBatchIOGetStatus') + * if __cuFileBatchIOGetStatus == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOGetStatus == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":204 + * __cuFileBatchIOGetStatus = dlsym(RTLD_DEFAULT, 'cuFileBatchIOGetStatus') + * if __cuFileBatchIOGetStatus == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBatchIOGetStatus = dlsym(handle, 'cuFileBatchIOGetStatus') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":205 + * if __cuFileBatchIOGetStatus == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileBatchIOGetStatus = dlsym(handle, 'cuFileBatchIOGetStatus') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 205, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":204 + * __cuFileBatchIOGetStatus = dlsym(RTLD_DEFAULT, 'cuFileBatchIOGetStatus') + * if __cuFileBatchIOGetStatus == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBatchIOGetStatus = dlsym(handle, 'cuFileBatchIOGetStatus') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":206 + * if handle == NULL: + * handle = load_library() + * __cuFileBatchIOGetStatus = dlsym(handle, 'cuFileBatchIOGetStatus') # <<<<<<<<<<<<<< + * + * global __cuFileBatchIOCancel +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOGetStatus = dlsym(__pyx_v_handle, ((char const *)"cuFileBatchIOGetStatus")); + + /* "cuda/bindings/_internal/cufile.pyx":203 + * global __cuFileBatchIOGetStatus + * __cuFileBatchIOGetStatus = dlsym(RTLD_DEFAULT, 'cuFileBatchIOGetStatus') + * if __cuFileBatchIOGetStatus == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":209 + * + * global __cuFileBatchIOCancel + * __cuFileBatchIOCancel = dlsym(RTLD_DEFAULT, 'cuFileBatchIOCancel') # <<<<<<<<<<<<<< + * if __cuFileBatchIOCancel == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOCancel = dlsym(RTLD_DEFAULT, ((char const *)"cuFileBatchIOCancel")); + + /* "cuda/bindings/_internal/cufile.pyx":210 + * global __cuFileBatchIOCancel + * __cuFileBatchIOCancel = dlsym(RTLD_DEFAULT, 'cuFileBatchIOCancel') + * if __cuFileBatchIOCancel == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOCancel == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":211 + * __cuFileBatchIOCancel = dlsym(RTLD_DEFAULT, 'cuFileBatchIOCancel') + * if __cuFileBatchIOCancel == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBatchIOCancel = dlsym(handle, 'cuFileBatchIOCancel') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":212 + * if __cuFileBatchIOCancel == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileBatchIOCancel = dlsym(handle, 'cuFileBatchIOCancel') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 212, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":211 + * __cuFileBatchIOCancel = dlsym(RTLD_DEFAULT, 'cuFileBatchIOCancel') + * if __cuFileBatchIOCancel == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBatchIOCancel = dlsym(handle, 'cuFileBatchIOCancel') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":213 + * if handle == NULL: + * handle = load_library() + * __cuFileBatchIOCancel = dlsym(handle, 'cuFileBatchIOCancel') # <<<<<<<<<<<<<< + * + * global __cuFileBatchIODestroy +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOCancel = dlsym(__pyx_v_handle, ((char const *)"cuFileBatchIOCancel")); + + /* "cuda/bindings/_internal/cufile.pyx":210 + * global __cuFileBatchIOCancel + * __cuFileBatchIOCancel = dlsym(RTLD_DEFAULT, 'cuFileBatchIOCancel') + * if __cuFileBatchIOCancel == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":216 + * + * global __cuFileBatchIODestroy + * __cuFileBatchIODestroy = dlsym(RTLD_DEFAULT, 'cuFileBatchIODestroy') # <<<<<<<<<<<<<< + * if __cuFileBatchIODestroy == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIODestroy = dlsym(RTLD_DEFAULT, ((char const *)"cuFileBatchIODestroy")); + + /* "cuda/bindings/_internal/cufile.pyx":217 + * global __cuFileBatchIODestroy + * __cuFileBatchIODestroy = dlsym(RTLD_DEFAULT, 'cuFileBatchIODestroy') + * if __cuFileBatchIODestroy == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIODestroy == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":218 + * __cuFileBatchIODestroy = dlsym(RTLD_DEFAULT, 'cuFileBatchIODestroy') + * if __cuFileBatchIODestroy == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBatchIODestroy = dlsym(handle, 'cuFileBatchIODestroy') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":219 + * if __cuFileBatchIODestroy == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileBatchIODestroy = dlsym(handle, 'cuFileBatchIODestroy') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 219, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":218 + * __cuFileBatchIODestroy = dlsym(RTLD_DEFAULT, 'cuFileBatchIODestroy') + * if __cuFileBatchIODestroy == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileBatchIODestroy = dlsym(handle, 'cuFileBatchIODestroy') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":220 + * if handle == NULL: + * handle = load_library() + * __cuFileBatchIODestroy = dlsym(handle, 'cuFileBatchIODestroy') # <<<<<<<<<<<<<< + * + * global __cuFileReadAsync +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIODestroy = dlsym(__pyx_v_handle, ((char const *)"cuFileBatchIODestroy")); + + /* "cuda/bindings/_internal/cufile.pyx":217 + * global __cuFileBatchIODestroy + * __cuFileBatchIODestroy = dlsym(RTLD_DEFAULT, 'cuFileBatchIODestroy') + * if __cuFileBatchIODestroy == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":223 + * + * global __cuFileReadAsync + * __cuFileReadAsync = dlsym(RTLD_DEFAULT, 'cuFileReadAsync') # <<<<<<<<<<<<<< + * if __cuFileReadAsync == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileReadAsync = dlsym(RTLD_DEFAULT, ((char const *)"cuFileReadAsync")); + + /* "cuda/bindings/_internal/cufile.pyx":224 + * global __cuFileReadAsync + * __cuFileReadAsync = dlsym(RTLD_DEFAULT, 'cuFileReadAsync') + * if __cuFileReadAsync == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileReadAsync == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":225 + * __cuFileReadAsync = dlsym(RTLD_DEFAULT, 'cuFileReadAsync') + * if __cuFileReadAsync == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileReadAsync = dlsym(handle, 'cuFileReadAsync') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":226 + * if __cuFileReadAsync == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileReadAsync = dlsym(handle, 'cuFileReadAsync') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 226, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":225 + * __cuFileReadAsync = dlsym(RTLD_DEFAULT, 'cuFileReadAsync') + * if __cuFileReadAsync == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileReadAsync = dlsym(handle, 'cuFileReadAsync') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":227 + * if handle == NULL: + * handle = load_library() + * __cuFileReadAsync = dlsym(handle, 'cuFileReadAsync') # <<<<<<<<<<<<<< + * + * global __cuFileWriteAsync +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileReadAsync = dlsym(__pyx_v_handle, ((char const *)"cuFileReadAsync")); + + /* "cuda/bindings/_internal/cufile.pyx":224 + * global __cuFileReadAsync + * __cuFileReadAsync = dlsym(RTLD_DEFAULT, 'cuFileReadAsync') + * if __cuFileReadAsync == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":230 + * + * global __cuFileWriteAsync + * __cuFileWriteAsync = dlsym(RTLD_DEFAULT, 'cuFileWriteAsync') # <<<<<<<<<<<<<< + * if __cuFileWriteAsync == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWriteAsync = dlsym(RTLD_DEFAULT, ((char const *)"cuFileWriteAsync")); + + /* "cuda/bindings/_internal/cufile.pyx":231 + * global __cuFileWriteAsync + * __cuFileWriteAsync = dlsym(RTLD_DEFAULT, 'cuFileWriteAsync') + * if __cuFileWriteAsync == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWriteAsync == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":232 + * __cuFileWriteAsync = dlsym(RTLD_DEFAULT, 'cuFileWriteAsync') + * if __cuFileWriteAsync == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileWriteAsync = dlsym(handle, 'cuFileWriteAsync') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":233 + * if __cuFileWriteAsync == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileWriteAsync = dlsym(handle, 'cuFileWriteAsync') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 233, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":232 + * __cuFileWriteAsync = dlsym(RTLD_DEFAULT, 'cuFileWriteAsync') + * if __cuFileWriteAsync == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileWriteAsync = dlsym(handle, 'cuFileWriteAsync') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":234 + * if handle == NULL: + * handle = load_library() + * __cuFileWriteAsync = dlsym(handle, 'cuFileWriteAsync') # <<<<<<<<<<<<<< + * + * global __cuFileStreamRegister +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWriteAsync = dlsym(__pyx_v_handle, ((char const *)"cuFileWriteAsync")); + + /* "cuda/bindings/_internal/cufile.pyx":231 + * global __cuFileWriteAsync + * __cuFileWriteAsync = dlsym(RTLD_DEFAULT, 'cuFileWriteAsync') + * if __cuFileWriteAsync == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":237 + * + * global __cuFileStreamRegister + * __cuFileStreamRegister = dlsym(RTLD_DEFAULT, 'cuFileStreamRegister') # <<<<<<<<<<<<<< + * if __cuFileStreamRegister == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamRegister = dlsym(RTLD_DEFAULT, ((char const *)"cuFileStreamRegister")); + + /* "cuda/bindings/_internal/cufile.pyx":238 + * global __cuFileStreamRegister + * __cuFileStreamRegister = dlsym(RTLD_DEFAULT, 'cuFileStreamRegister') + * if __cuFileStreamRegister == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamRegister == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":239 + * __cuFileStreamRegister = dlsym(RTLD_DEFAULT, 'cuFileStreamRegister') + * if __cuFileStreamRegister == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileStreamRegister = dlsym(handle, 'cuFileStreamRegister') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":240 + * if __cuFileStreamRegister == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileStreamRegister = dlsym(handle, 'cuFileStreamRegister') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 240, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":239 + * __cuFileStreamRegister = dlsym(RTLD_DEFAULT, 'cuFileStreamRegister') + * if __cuFileStreamRegister == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileStreamRegister = dlsym(handle, 'cuFileStreamRegister') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":241 + * if handle == NULL: + * handle = load_library() + * __cuFileStreamRegister = dlsym(handle, 'cuFileStreamRegister') # <<<<<<<<<<<<<< + * + * global __cuFileStreamDeregister +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamRegister = dlsym(__pyx_v_handle, ((char const *)"cuFileStreamRegister")); + + /* "cuda/bindings/_internal/cufile.pyx":238 + * global __cuFileStreamRegister + * __cuFileStreamRegister = dlsym(RTLD_DEFAULT, 'cuFileStreamRegister') + * if __cuFileStreamRegister == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":244 + * + * global __cuFileStreamDeregister + * __cuFileStreamDeregister = dlsym(RTLD_DEFAULT, 'cuFileStreamDeregister') # <<<<<<<<<<<<<< + * if __cuFileStreamDeregister == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamDeregister = dlsym(RTLD_DEFAULT, ((char const *)"cuFileStreamDeregister")); + + /* "cuda/bindings/_internal/cufile.pyx":245 + * global __cuFileStreamDeregister + * __cuFileStreamDeregister = dlsym(RTLD_DEFAULT, 'cuFileStreamDeregister') + * if __cuFileStreamDeregister == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamDeregister == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":246 + * __cuFileStreamDeregister = dlsym(RTLD_DEFAULT, 'cuFileStreamDeregister') + * if __cuFileStreamDeregister == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileStreamDeregister = dlsym(handle, 'cuFileStreamDeregister') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":247 + * if __cuFileStreamDeregister == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileStreamDeregister = dlsym(handle, 'cuFileStreamDeregister') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 247, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":246 + * __cuFileStreamDeregister = dlsym(RTLD_DEFAULT, 'cuFileStreamDeregister') + * if __cuFileStreamDeregister == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileStreamDeregister = dlsym(handle, 'cuFileStreamDeregister') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":248 + * if handle == NULL: + * handle = load_library() + * __cuFileStreamDeregister = dlsym(handle, 'cuFileStreamDeregister') # <<<<<<<<<<<<<< + * + * global __cuFileGetVersion +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamDeregister = dlsym(__pyx_v_handle, ((char const *)"cuFileStreamDeregister")); + + /* "cuda/bindings/_internal/cufile.pyx":245 + * global __cuFileStreamDeregister + * __cuFileStreamDeregister = dlsym(RTLD_DEFAULT, 'cuFileStreamDeregister') + * if __cuFileStreamDeregister == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":251 + * + * global __cuFileGetVersion + * __cuFileGetVersion = dlsym(RTLD_DEFAULT, 'cuFileGetVersion') # <<<<<<<<<<<<<< + * if __cuFileGetVersion == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetVersion = dlsym(RTLD_DEFAULT, ((char const *)"cuFileGetVersion")); + + /* "cuda/bindings/_internal/cufile.pyx":252 + * global __cuFileGetVersion + * __cuFileGetVersion = dlsym(RTLD_DEFAULT, 'cuFileGetVersion') + * if __cuFileGetVersion == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetVersion == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":253 + * __cuFileGetVersion = dlsym(RTLD_DEFAULT, 'cuFileGetVersion') + * if __cuFileGetVersion == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileGetVersion = dlsym(handle, 'cuFileGetVersion') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":254 + * if __cuFileGetVersion == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileGetVersion = dlsym(handle, 'cuFileGetVersion') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 254, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":253 + * __cuFileGetVersion = dlsym(RTLD_DEFAULT, 'cuFileGetVersion') + * if __cuFileGetVersion == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileGetVersion = dlsym(handle, 'cuFileGetVersion') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":255 + * if handle == NULL: + * handle = load_library() + * __cuFileGetVersion = dlsym(handle, 'cuFileGetVersion') # <<<<<<<<<<<<<< + * + * global __cuFileGetParameterSizeT +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetVersion = dlsym(__pyx_v_handle, ((char const *)"cuFileGetVersion")); + + /* "cuda/bindings/_internal/cufile.pyx":252 + * global __cuFileGetVersion + * __cuFileGetVersion = dlsym(RTLD_DEFAULT, 'cuFileGetVersion') + * if __cuFileGetVersion == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":258 + * + * global __cuFileGetParameterSizeT + * __cuFileGetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileGetParameterSizeT') # <<<<<<<<<<<<<< + * if __cuFileGetParameterSizeT == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterSizeT = dlsym(RTLD_DEFAULT, ((char const *)"cuFileGetParameterSizeT")); + + /* "cuda/bindings/_internal/cufile.pyx":259 + * global __cuFileGetParameterSizeT + * __cuFileGetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileGetParameterSizeT') + * if __cuFileGetParameterSizeT == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterSizeT == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":260 + * __cuFileGetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileGetParameterSizeT') + * if __cuFileGetParameterSizeT == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileGetParameterSizeT = dlsym(handle, 'cuFileGetParameterSizeT') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":261 + * if __cuFileGetParameterSizeT == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileGetParameterSizeT = dlsym(handle, 'cuFileGetParameterSizeT') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 261, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":260 + * __cuFileGetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileGetParameterSizeT') + * if __cuFileGetParameterSizeT == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileGetParameterSizeT = dlsym(handle, 'cuFileGetParameterSizeT') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":262 + * if handle == NULL: + * handle = load_library() + * __cuFileGetParameterSizeT = dlsym(handle, 'cuFileGetParameterSizeT') # <<<<<<<<<<<<<< + * + * global __cuFileGetParameterBool +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterSizeT = dlsym(__pyx_v_handle, ((char const *)"cuFileGetParameterSizeT")); + + /* "cuda/bindings/_internal/cufile.pyx":259 + * global __cuFileGetParameterSizeT + * __cuFileGetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileGetParameterSizeT') + * if __cuFileGetParameterSizeT == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":265 + * + * global __cuFileGetParameterBool + * __cuFileGetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileGetParameterBool') # <<<<<<<<<<<<<< + * if __cuFileGetParameterBool == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterBool = dlsym(RTLD_DEFAULT, ((char const *)"cuFileGetParameterBool")); + + /* "cuda/bindings/_internal/cufile.pyx":266 + * global __cuFileGetParameterBool + * __cuFileGetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileGetParameterBool') + * if __cuFileGetParameterBool == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterBool == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":267 + * __cuFileGetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileGetParameterBool') + * if __cuFileGetParameterBool == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileGetParameterBool = dlsym(handle, 'cuFileGetParameterBool') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":268 + * if __cuFileGetParameterBool == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileGetParameterBool = dlsym(handle, 'cuFileGetParameterBool') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 268, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":267 + * __cuFileGetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileGetParameterBool') + * if __cuFileGetParameterBool == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileGetParameterBool = dlsym(handle, 'cuFileGetParameterBool') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":269 + * if handle == NULL: + * handle = load_library() + * __cuFileGetParameterBool = dlsym(handle, 'cuFileGetParameterBool') # <<<<<<<<<<<<<< + * + * global __cuFileGetParameterString +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterBool = dlsym(__pyx_v_handle, ((char const *)"cuFileGetParameterBool")); + + /* "cuda/bindings/_internal/cufile.pyx":266 + * global __cuFileGetParameterBool + * __cuFileGetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileGetParameterBool') + * if __cuFileGetParameterBool == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":272 + * + * global __cuFileGetParameterString + * __cuFileGetParameterString = dlsym(RTLD_DEFAULT, 'cuFileGetParameterString') # <<<<<<<<<<<<<< + * if __cuFileGetParameterString == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterString = dlsym(RTLD_DEFAULT, ((char const *)"cuFileGetParameterString")); + + /* "cuda/bindings/_internal/cufile.pyx":273 + * global __cuFileGetParameterString + * __cuFileGetParameterString = dlsym(RTLD_DEFAULT, 'cuFileGetParameterString') + * if __cuFileGetParameterString == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterString == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":274 + * __cuFileGetParameterString = dlsym(RTLD_DEFAULT, 'cuFileGetParameterString') + * if __cuFileGetParameterString == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileGetParameterString = dlsym(handle, 'cuFileGetParameterString') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":275 + * if __cuFileGetParameterString == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileGetParameterString = dlsym(handle, 'cuFileGetParameterString') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 275, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":274 + * __cuFileGetParameterString = dlsym(RTLD_DEFAULT, 'cuFileGetParameterString') + * if __cuFileGetParameterString == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileGetParameterString = dlsym(handle, 'cuFileGetParameterString') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":276 + * if handle == NULL: + * handle = load_library() + * __cuFileGetParameterString = dlsym(handle, 'cuFileGetParameterString') # <<<<<<<<<<<<<< + * + * global __cuFileSetParameterSizeT +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterString = dlsym(__pyx_v_handle, ((char const *)"cuFileGetParameterString")); + + /* "cuda/bindings/_internal/cufile.pyx":273 + * global __cuFileGetParameterString + * __cuFileGetParameterString = dlsym(RTLD_DEFAULT, 'cuFileGetParameterString') + * if __cuFileGetParameterString == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":279 + * + * global __cuFileSetParameterSizeT + * __cuFileSetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileSetParameterSizeT') # <<<<<<<<<<<<<< + * if __cuFileSetParameterSizeT == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterSizeT = dlsym(RTLD_DEFAULT, ((char const *)"cuFileSetParameterSizeT")); + + /* "cuda/bindings/_internal/cufile.pyx":280 + * global __cuFileSetParameterSizeT + * __cuFileSetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileSetParameterSizeT') + * if __cuFileSetParameterSizeT == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterSizeT == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":281 + * __cuFileSetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileSetParameterSizeT') + * if __cuFileSetParameterSizeT == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileSetParameterSizeT = dlsym(handle, 'cuFileSetParameterSizeT') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":282 + * if __cuFileSetParameterSizeT == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileSetParameterSizeT = dlsym(handle, 'cuFileSetParameterSizeT') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 282, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":281 + * __cuFileSetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileSetParameterSizeT') + * if __cuFileSetParameterSizeT == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileSetParameterSizeT = dlsym(handle, 'cuFileSetParameterSizeT') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":283 + * if handle == NULL: + * handle = load_library() + * __cuFileSetParameterSizeT = dlsym(handle, 'cuFileSetParameterSizeT') # <<<<<<<<<<<<<< + * + * global __cuFileSetParameterBool +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterSizeT = dlsym(__pyx_v_handle, ((char const *)"cuFileSetParameterSizeT")); + + /* "cuda/bindings/_internal/cufile.pyx":280 + * global __cuFileSetParameterSizeT + * __cuFileSetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileSetParameterSizeT') + * if __cuFileSetParameterSizeT == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":286 + * + * global __cuFileSetParameterBool + * __cuFileSetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileSetParameterBool') # <<<<<<<<<<<<<< + * if __cuFileSetParameterBool == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterBool = dlsym(RTLD_DEFAULT, ((char const *)"cuFileSetParameterBool")); + + /* "cuda/bindings/_internal/cufile.pyx":287 + * global __cuFileSetParameterBool + * __cuFileSetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileSetParameterBool') + * if __cuFileSetParameterBool == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterBool == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":288 + * __cuFileSetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileSetParameterBool') + * if __cuFileSetParameterBool == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileSetParameterBool = dlsym(handle, 'cuFileSetParameterBool') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":289 + * if __cuFileSetParameterBool == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileSetParameterBool = dlsym(handle, 'cuFileSetParameterBool') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 289, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":288 + * __cuFileSetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileSetParameterBool') + * if __cuFileSetParameterBool == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileSetParameterBool = dlsym(handle, 'cuFileSetParameterBool') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":290 + * if handle == NULL: + * handle = load_library() + * __cuFileSetParameterBool = dlsym(handle, 'cuFileSetParameterBool') # <<<<<<<<<<<<<< + * + * global __cuFileSetParameterString +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterBool = dlsym(__pyx_v_handle, ((char const *)"cuFileSetParameterBool")); + + /* "cuda/bindings/_internal/cufile.pyx":287 + * global __cuFileSetParameterBool + * __cuFileSetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileSetParameterBool') + * if __cuFileSetParameterBool == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":293 + * + * global __cuFileSetParameterString + * __cuFileSetParameterString = dlsym(RTLD_DEFAULT, 'cuFileSetParameterString') # <<<<<<<<<<<<<< + * if __cuFileSetParameterString == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterString = dlsym(RTLD_DEFAULT, ((char const *)"cuFileSetParameterString")); + + /* "cuda/bindings/_internal/cufile.pyx":294 + * global __cuFileSetParameterString + * __cuFileSetParameterString = dlsym(RTLD_DEFAULT, 'cuFileSetParameterString') + * if __cuFileSetParameterString == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterString == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":295 + * __cuFileSetParameterString = dlsym(RTLD_DEFAULT, 'cuFileSetParameterString') + * if __cuFileSetParameterString == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileSetParameterString = dlsym(handle, 'cuFileSetParameterString') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":296 + * if __cuFileSetParameterString == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileSetParameterString = dlsym(handle, 'cuFileSetParameterString') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 296, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":295 + * __cuFileSetParameterString = dlsym(RTLD_DEFAULT, 'cuFileSetParameterString') + * if __cuFileSetParameterString == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileSetParameterString = dlsym(handle, 'cuFileSetParameterString') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":297 + * if handle == NULL: + * handle = load_library() + * __cuFileSetParameterString = dlsym(handle, 'cuFileSetParameterString') # <<<<<<<<<<<<<< + * + * global __cuFileDriverClose +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterString = dlsym(__pyx_v_handle, ((char const *)"cuFileSetParameterString")); + + /* "cuda/bindings/_internal/cufile.pyx":294 + * global __cuFileSetParameterString + * __cuFileSetParameterString = dlsym(RTLD_DEFAULT, 'cuFileSetParameterString') + * if __cuFileSetParameterString == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":300 + * + * global __cuFileDriverClose + * __cuFileDriverClose = dlsym(RTLD_DEFAULT, 'cuFileDriverClose') # <<<<<<<<<<<<<< + * if __cuFileDriverClose == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose = dlsym(RTLD_DEFAULT, ((char const *)"cuFileDriverClose")); + + /* "cuda/bindings/_internal/cufile.pyx":301 + * global __cuFileDriverClose + * __cuFileDriverClose = dlsym(RTLD_DEFAULT, 'cuFileDriverClose') + * if __cuFileDriverClose == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":302 + * __cuFileDriverClose = dlsym(RTLD_DEFAULT, 'cuFileDriverClose') + * if __cuFileDriverClose == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverClose = dlsym(handle, 'cuFileDriverClose') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/cufile.pyx":303 + * if __cuFileDriverClose == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __cuFileDriverClose = dlsym(handle, 'cuFileDriverClose') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_6cufile_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 303, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/cufile.pyx":302 + * __cuFileDriverClose = dlsym(RTLD_DEFAULT, 'cuFileDriverClose') + * if __cuFileDriverClose == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __cuFileDriverClose = dlsym(handle, 'cuFileDriverClose') +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":304 + * if handle == NULL: + * handle = load_library() + * __cuFileDriverClose = dlsym(handle, 'cuFileDriverClose') # <<<<<<<<<<<<<< + * + * __py_cufile_init = True +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose = dlsym(__pyx_v_handle, ((char const *)"cuFileDriverClose")); + + /* "cuda/bindings/_internal/cufile.pyx":301 + * global __cuFileDriverClose + * __cuFileDriverClose = dlsym(RTLD_DEFAULT, 'cuFileDriverClose') + * if __cuFileDriverClose == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":306 + * __cuFileDriverClose = dlsym(handle, 'cuFileDriverClose') + * + * __py_cufile_init = True # <<<<<<<<<<<<<< + * return 0 + * +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___py_cufile_init = 1; + + /* "cuda/bindings/_internal/cufile.pyx":307 + * + * __py_cufile_init = True + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L14_try_return; + + /* "cuda/bindings/_internal/cufile.pyx":87 + * cdef void* handle = NULL + * + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * # Load function + * global __cuFileHandleRegister +*/ + } + __pyx_L10_error:; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + /*except:*/ { + __Pyx_AddTraceback("cuda.bindings._internal.cufile.__check_or_init_cufile", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_4, &__pyx_t_3) < 0) __PYX_ERR(0, 87, __pyx_L12_except_error) + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_3); + __pyx_t_11 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 87, __pyx_L12_except_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_11, NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 87, __pyx_L12_except_error) + __Pyx_GOTREF(__pyx_t_12); + __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_12); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + if (__pyx_t_9 < (0)) __PYX_ERR(0, 87, __pyx_L12_except_error) + __pyx_t_13 = (!__pyx_t_9); + if (unlikely(__pyx_t_13)) { + __Pyx_GIVEREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ErrRestoreWithState(__pyx_t_2, __pyx_t_4, __pyx_t_3); + __pyx_t_2 = 0; __pyx_t_4 = 0; __pyx_t_3 = 0; + __PYX_ERR(0, 87, __pyx_L12_except_error) + } + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L11_exception_handled; + } + __pyx_L12_except_error:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L4_error; + __pyx_L14_try_return:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L7_return; + __pyx_L11_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + } + } + /*finally:*/ { + /*normal exit:*/{ + if (__pyx_t_1) { + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_mstate_global->__pyx_tuple[0], NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 87, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + goto __pyx_L9; + } + __pyx_L7_return: { + __pyx_t_14 = __pyx_r; + if (__pyx_t_1) { + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_mstate_global->__pyx_tuple[0], NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 87, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + __pyx_r = __pyx_t_14; + goto __pyx_L3_return; + } + __pyx_L9:; + } + goto __pyx_L81; + __pyx_L6_error:; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L4_error; + __pyx_L81:; + } + } + /*finally:*/ { + /*normal exit:*/{ + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L5; + } + __pyx_L3_return: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L0; + } + __pyx_L4_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + __pyx_L5:; + } + } + + /* "cuda/bindings/_internal/cufile.pyx":82 + * + * + * cdef int __check_or_init_cufile() except -1 nogil: # <<<<<<<<<<<<<< + * global __py_cufile_init + * +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_AddTraceback("cuda.bindings._internal.cufile.__check_or_init_cufile", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":310 + * + * + * cdef inline int _check_or_init_cufile() except -1 nogil: # <<<<<<<<<<<<<< + * if __py_cufile_init: + * return 0 +*/ + +static CYTHON_INLINE int __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(void) { + int __pyx_r; + int __pyx_t_1; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_internal/cufile.pyx":311 + * + * cdef inline int _check_or_init_cufile() except -1 nogil: + * if __py_cufile_init: # <<<<<<<<<<<<<< + * return 0 + * +*/ + if (__pyx_v_4cuda_8bindings_9_internal_6cufile___py_cufile_init) { + + /* "cuda/bindings/_internal/cufile.pyx":312 + * cdef inline int _check_or_init_cufile() except -1 nogil: + * if __py_cufile_init: + * return 0 # <<<<<<<<<<<<<< + * + * return __check_or_init_cufile() +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":311 + * + * cdef inline int _check_or_init_cufile() except -1 nogil: + * if __py_cufile_init: # <<<<<<<<<<<<<< + * return 0 + * +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":314 + * return 0 + * + * return __check_or_init_cufile() # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile___check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 314, __pyx_L1_error) + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":310 + * + * + * cdef inline int _check_or_init_cufile() except -1 nogil: # <<<<<<<<<<<<<< + * if __py_cufile_init: + * return 0 +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._check_or_init_cufile", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":320 + * + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_6cufile_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_internal_6cufile__inspect_function_pointers(CYTHON_UNUSED int __pyx_skip_dispatch) { + PyObject *__pyx_v_data = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointers", 0); + + /* "cuda/bindings/_internal/cufile.pyx":322 + * cpdef dict _inspect_function_pointers(): + * global func_ptrs + * if func_ptrs is not None: # <<<<<<<<<<<<<< + * return func_ptrs + * +*/ + __pyx_t_1 = (__pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs != ((PyObject*)Py_None)); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/cufile.pyx":323 + * global func_ptrs + * if func_ptrs is not None: + * return func_ptrs # <<<<<<<<<<<<<< + * + * _check_or_init_cufile() +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs); + __pyx_r = __pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs; + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":322 + * cpdef dict _inspect_function_pointers(): + * global func_ptrs + * if func_ptrs is not None: # <<<<<<<<<<<<<< + * return func_ptrs + * +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":325 + * return func_ptrs + * + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * cdef dict data = {} + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(0, 325, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":326 + * + * _check_or_init_cufile() + * cdef dict data = {} # <<<<<<<<<<<<<< + * + * global __cuFileHandleRegister +*/ + __pyx_t_3 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 326, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_v_data = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":329 + * + * global __cuFileHandleRegister + * data["__cuFileHandleRegister"] = __cuFileHandleRegister # <<<<<<<<<<<<<< + * + * global __cuFileHandleDeregister +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleRegister)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 329, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileHandleRegister, __pyx_t_3) < 0))) __PYX_ERR(0, 329, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":332 + * + * global __cuFileHandleDeregister + * data["__cuFileHandleDeregister"] = __cuFileHandleDeregister # <<<<<<<<<<<<<< + * + * global __cuFileBufRegister +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleDeregister)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 332, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileHandleDeregister, __pyx_t_3) < 0))) __PYX_ERR(0, 332, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":335 + * + * global __cuFileBufRegister + * data["__cuFileBufRegister"] = __cuFileBufRegister # <<<<<<<<<<<<<< + * + * global __cuFileBufDeregister +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufRegister)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileBufRegister, __pyx_t_3) < 0))) __PYX_ERR(0, 335, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":338 + * + * global __cuFileBufDeregister + * data["__cuFileBufDeregister"] = __cuFileBufDeregister # <<<<<<<<<<<<<< + * + * global __cuFileRead +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufDeregister)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 338, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileBufDeregister, __pyx_t_3) < 0))) __PYX_ERR(0, 338, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":341 + * + * global __cuFileRead + * data["__cuFileRead"] = __cuFileRead # <<<<<<<<<<<<<< + * + * global __cuFileWrite +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileRead)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 341, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileRead, __pyx_t_3) < 0))) __PYX_ERR(0, 341, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":344 + * + * global __cuFileWrite + * data["__cuFileWrite"] = __cuFileWrite # <<<<<<<<<<<<<< + * + * global __cuFileDriverOpen +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWrite)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 344, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileWrite, __pyx_t_3) < 0))) __PYX_ERR(0, 344, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":347 + * + * global __cuFileDriverOpen + * data["__cuFileDriverOpen"] = __cuFileDriverOpen # <<<<<<<<<<<<<< + * + * global __cuFileDriverClose_v2 +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverOpen)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileDriverOpen, __pyx_t_3) < 0))) __PYX_ERR(0, 347, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":350 + * + * global __cuFileDriverClose_v2 + * data["__cuFileDriverClose_v2"] = __cuFileDriverClose_v2 # <<<<<<<<<<<<<< + * + * global __cuFileUseCount +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose_v2)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 350, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileDriverClose_v2, __pyx_t_3) < 0))) __PYX_ERR(0, 350, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":353 + * + * global __cuFileUseCount + * data["__cuFileUseCount"] = __cuFileUseCount # <<<<<<<<<<<<<< + * + * global __cuFileDriverGetProperties +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileUseCount)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileUseCount, __pyx_t_3) < 0))) __PYX_ERR(0, 353, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":356 + * + * global __cuFileDriverGetProperties + * data["__cuFileDriverGetProperties"] = __cuFileDriverGetProperties # <<<<<<<<<<<<<< + * + * global __cuFileDriverSetPollMode +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverGetProperties)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileDriverGetProperties, __pyx_t_3) < 0))) __PYX_ERR(0, 356, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":359 + * + * global __cuFileDriverSetPollMode + * data["__cuFileDriverSetPollMode"] = __cuFileDriverSetPollMode # <<<<<<<<<<<<<< + * + * global __cuFileDriverSetMaxDirectIOSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetPollMode)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 359, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileDriverSetPollMode, __pyx_t_3) < 0))) __PYX_ERR(0, 359, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":362 + * + * global __cuFileDriverSetMaxDirectIOSize + * data["__cuFileDriverSetMaxDirectIOSize"] = __cuFileDriverSetMaxDirectIOSize # <<<<<<<<<<<<<< + * + * global __cuFileDriverSetMaxCacheSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxDirectIOSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 362, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileDriverSetMaxDirectIOSize, __pyx_t_3) < 0))) __PYX_ERR(0, 362, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":365 + * + * global __cuFileDriverSetMaxCacheSize + * data["__cuFileDriverSetMaxCacheSize"] = __cuFileDriverSetMaxCacheSize # <<<<<<<<<<<<<< + * + * global __cuFileDriverSetMaxPinnedMemSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxCacheSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 365, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileDriverSetMaxCacheSize, __pyx_t_3) < 0))) __PYX_ERR(0, 365, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":368 + * + * global __cuFileDriverSetMaxPinnedMemSize + * data["__cuFileDriverSetMaxPinnedMemSize"] = __cuFileDriverSetMaxPinnedMemSize # <<<<<<<<<<<<<< + * + * global __cuFileBatchIOSetUp +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxPinnedMemSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 368, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileDriverSetMaxPinnedMemSiz, __pyx_t_3) < 0))) __PYX_ERR(0, 368, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":371 + * + * global __cuFileBatchIOSetUp + * data["__cuFileBatchIOSetUp"] = __cuFileBatchIOSetUp # <<<<<<<<<<<<<< + * + * global __cuFileBatchIOSubmit +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSetUp)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 371, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileBatchIOSetUp, __pyx_t_3) < 0))) __PYX_ERR(0, 371, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":374 + * + * global __cuFileBatchIOSubmit + * data["__cuFileBatchIOSubmit"] = __cuFileBatchIOSubmit # <<<<<<<<<<<<<< + * + * global __cuFileBatchIOGetStatus +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSubmit)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileBatchIOSubmit, __pyx_t_3) < 0))) __PYX_ERR(0, 374, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":377 + * + * global __cuFileBatchIOGetStatus + * data["__cuFileBatchIOGetStatus"] = __cuFileBatchIOGetStatus # <<<<<<<<<<<<<< + * + * global __cuFileBatchIOCancel +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOGetStatus)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 377, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileBatchIOGetStatus, __pyx_t_3) < 0))) __PYX_ERR(0, 377, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":380 + * + * global __cuFileBatchIOCancel + * data["__cuFileBatchIOCancel"] = __cuFileBatchIOCancel # <<<<<<<<<<<<<< + * + * global __cuFileBatchIODestroy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOCancel)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 380, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileBatchIOCancel, __pyx_t_3) < 0))) __PYX_ERR(0, 380, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":383 + * + * global __cuFileBatchIODestroy + * data["__cuFileBatchIODestroy"] = __cuFileBatchIODestroy # <<<<<<<<<<<<<< + * + * global __cuFileReadAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIODestroy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 383, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileBatchIODestroy, __pyx_t_3) < 0))) __PYX_ERR(0, 383, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":386 + * + * global __cuFileReadAsync + * data["__cuFileReadAsync"] = __cuFileReadAsync # <<<<<<<<<<<<<< + * + * global __cuFileWriteAsync +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileReadAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 386, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileReadAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 386, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":389 + * + * global __cuFileWriteAsync + * data["__cuFileWriteAsync"] = __cuFileWriteAsync # <<<<<<<<<<<<<< + * + * global __cuFileStreamRegister +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWriteAsync)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 389, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileWriteAsync, __pyx_t_3) < 0))) __PYX_ERR(0, 389, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":392 + * + * global __cuFileStreamRegister + * data["__cuFileStreamRegister"] = __cuFileStreamRegister # <<<<<<<<<<<<<< + * + * global __cuFileStreamDeregister +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamRegister)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 392, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileStreamRegister, __pyx_t_3) < 0))) __PYX_ERR(0, 392, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":395 + * + * global __cuFileStreamDeregister + * data["__cuFileStreamDeregister"] = __cuFileStreamDeregister # <<<<<<<<<<<<<< + * + * global __cuFileGetVersion +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamDeregister)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 395, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileStreamDeregister, __pyx_t_3) < 0))) __PYX_ERR(0, 395, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":398 + * + * global __cuFileGetVersion + * data["__cuFileGetVersion"] = __cuFileGetVersion # <<<<<<<<<<<<<< + * + * global __cuFileGetParameterSizeT +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetVersion)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 398, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileGetVersion, __pyx_t_3) < 0))) __PYX_ERR(0, 398, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":401 + * + * global __cuFileGetParameterSizeT + * data["__cuFileGetParameterSizeT"] = __cuFileGetParameterSizeT # <<<<<<<<<<<<<< + * + * global __cuFileGetParameterBool +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterSizeT)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 401, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileGetParameterSizeT, __pyx_t_3) < 0))) __PYX_ERR(0, 401, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":404 + * + * global __cuFileGetParameterBool + * data["__cuFileGetParameterBool"] = __cuFileGetParameterBool # <<<<<<<<<<<<<< + * + * global __cuFileGetParameterString +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterBool)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 404, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileGetParameterBool, __pyx_t_3) < 0))) __PYX_ERR(0, 404, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":407 + * + * global __cuFileGetParameterString + * data["__cuFileGetParameterString"] = __cuFileGetParameterString # <<<<<<<<<<<<<< + * + * global __cuFileSetParameterSizeT +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterString)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 407, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileGetParameterString, __pyx_t_3) < 0))) __PYX_ERR(0, 407, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":410 + * + * global __cuFileSetParameterSizeT + * data["__cuFileSetParameterSizeT"] = __cuFileSetParameterSizeT # <<<<<<<<<<<<<< + * + * global __cuFileSetParameterBool +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterSizeT)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 410, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileSetParameterSizeT, __pyx_t_3) < 0))) __PYX_ERR(0, 410, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":413 + * + * global __cuFileSetParameterBool + * data["__cuFileSetParameterBool"] = __cuFileSetParameterBool # <<<<<<<<<<<<<< + * + * global __cuFileSetParameterString +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterBool)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 413, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileSetParameterBool, __pyx_t_3) < 0))) __PYX_ERR(0, 413, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":416 + * + * global __cuFileSetParameterString + * data["__cuFileSetParameterString"] = __cuFileSetParameterString # <<<<<<<<<<<<<< + * + * global __cuFileDriverClose +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterString)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 416, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileSetParameterString, __pyx_t_3) < 0))) __PYX_ERR(0, 416, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":419 + * + * global __cuFileDriverClose + * data["__cuFileDriverClose"] = __cuFileDriverClose # <<<<<<<<<<<<<< + * + * func_ptrs = data +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 419, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_cuFileDriverClose, __pyx_t_3) < 0))) __PYX_ERR(0, 419, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":421 + * data["__cuFileDriverClose"] = __cuFileDriverClose + * + * func_ptrs = data # <<<<<<<<<<<<<< + * return data + * +*/ + __Pyx_INCREF(__pyx_v_data); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs, __pyx_v_data); + __Pyx_GIVEREF(__pyx_v_data); + + /* "cuda/bindings/_internal/cufile.pyx":422 + * + * func_ptrs = data + * return data # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_data); + __pyx_r = __pyx_v_data; + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":320 + * + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._inspect_function_pointers", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_data); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_6cufile_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +PyDoc_STRVAR(__pyx_doc_4cuda_8bindings_9_internal_6cufile__inspect_function_pointers, "_inspect_function_pointers() -> dict"); +static PyMethodDef __pyx_mdef_4cuda_8bindings_9_internal_6cufile_1_inspect_function_pointers = {"_inspect_function_pointers", (PyCFunction)__pyx_pw_4cuda_8bindings_9_internal_6cufile_1_inspect_function_pointers, METH_NOARGS, __pyx_doc_4cuda_8bindings_9_internal_6cufile__inspect_function_pointers}; +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_6cufile_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_inspect_function_pointers (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_4cuda_8bindings_9_internal_6cufile__inspect_function_pointers(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_4cuda_8bindings_9_internal_6cufile__inspect_function_pointers(CYTHON_UNUSED PyObject *__pyx_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointers", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__inspect_function_pointers(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 320, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._inspect_function_pointers", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":425 + * + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_6cufile_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_internal_6cufile__inspect_function_pointer(PyObject *__pyx_v_name, CYTHON_UNUSED int __pyx_skip_dispatch) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointer", 0); + + /* "cuda/bindings/_internal/cufile.pyx":427 + * cpdef _inspect_function_pointer(str name): + * global func_ptrs + * if func_ptrs is None: # <<<<<<<<<<<<<< + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] +*/ + __pyx_t_1 = (__pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs == ((PyObject*)Py_None)); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/cufile.pyx":428 + * global func_ptrs + * if func_ptrs is None: + * func_ptrs = _inspect_function_pointers() # <<<<<<<<<<<<<< + * return func_ptrs[name] + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_internal_6cufile__inspect_function_pointers(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 428, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs, ((PyObject*)__pyx_t_2)); + __Pyx_GIVEREF(__pyx_t_2); + __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":427 + * cpdef _inspect_function_pointer(str name): + * global func_ptrs + * if func_ptrs is None: # <<<<<<<<<<<<<< + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":429 + * if func_ptrs is None: + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + if (unlikely(__pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 429, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_PyDict_GetItem(__pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs, __pyx_v_name); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 429, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":425 + * + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_6cufile_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_4cuda_8bindings_9_internal_6cufile_2_inspect_function_pointer, "_inspect_function_pointer(str name)"); +static PyMethodDef __pyx_mdef_4cuda_8bindings_9_internal_6cufile_3_inspect_function_pointer = {"_inspect_function_pointer", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cuda_8bindings_9_internal_6cufile_3_inspect_function_pointer, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cuda_8bindings_9_internal_6cufile_2_inspect_function_pointer}; +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_6cufile_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_name = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_inspect_function_pointer (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_name,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 425, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 425, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_inspect_function_pointer", 0) < (0)) __PYX_ERR(0, 425, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_inspect_function_pointer", 1, 1, 1, i); __PYX_ERR(0, 425, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 425, __pyx_L3_error) + } + __pyx_v_name = ((PyObject*)values[0]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_inspect_function_pointer", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 425, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("cuda.bindings._internal.cufile._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_name), (&PyUnicode_Type), 1, "name", 1))) __PYX_ERR(0, 425, __pyx_L1_error) + __pyx_r = __pyx_pf_4cuda_8bindings_9_internal_6cufile_2_inspect_function_pointer(__pyx_self, __pyx_v_name); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_4cuda_8bindings_9_internal_6cufile_2_inspect_function_pointer(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_name) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointer", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__inspect_function_pointer(__pyx_v_name, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":436 + * ############################################################################### + * + * cdef CUfileError_t _cuFileHandleRegister(CUfileHandle_t* fh, CUfileDescr_t* descr) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileHandleRegister + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileHandleRegister(CUfileHandle_t *__pyx_v_fh, CUfileDescr_t *__pyx_v_descr) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileHandleRegister", 1); + + /* "cuda/bindings/_internal/cufile.pyx":438 + * cdef CUfileError_t _cuFileHandleRegister(CUfileHandle_t* fh, CUfileDescr_t* descr) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileHandleRegister + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileHandleRegister == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 438, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":439 + * global __cuFileHandleRegister + * _check_or_init_cufile() + * if __cuFileHandleRegister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileHandleRegister is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleRegister == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":440 + * _check_or_init_cufile() + * if __cuFileHandleRegister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileHandleRegister is not found") + * return (__cuFileHandleRegister)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":441 + * if __cuFileHandleRegister == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileHandleRegister is not found") # <<<<<<<<<<<<<< + * return (__cuFileHandleRegister)( + * fh, descr) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 441, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileHandleRegister_is}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 441, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 441, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":440 + * _check_or_init_cufile() + * if __cuFileHandleRegister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileHandleRegister is not found") + * return (__cuFileHandleRegister)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":439 + * global __cuFileHandleRegister + * _check_or_init_cufile() + * if __cuFileHandleRegister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileHandleRegister is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":442 + * with gil: + * raise FunctionNotFoundError("function cuFileHandleRegister is not found") + * return (__cuFileHandleRegister)( # <<<<<<<<<<<<<< + * fh, descr) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUfileHandle_t *, CUfileDescr_t *))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleRegister)(__pyx_v_fh, __pyx_v_descr); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":436 + * ############################################################################### + * + * cdef CUfileError_t _cuFileHandleRegister(CUfileHandle_t* fh, CUfileDescr_t* descr) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileHandleRegister + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileHandleRegister", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":446 + * + * + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef void _cuFileHandleDeregister(CUfileHandle_t fh) except* nogil: + * global __cuFileHandleDeregister +*/ + +static void __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileHandleDeregister(CUfileHandle_t __pyx_v_fh) { + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileHandleDeregister", 1); + + /* "cuda/bindings/_internal/cufile.pyx":449 + * cdef void _cuFileHandleDeregister(CUfileHandle_t fh) except* nogil: + * global __cuFileHandleDeregister + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileHandleDeregister == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 449, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":450 + * global __cuFileHandleDeregister + * _check_or_init_cufile() + * if __cuFileHandleDeregister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileHandleDeregister is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleDeregister == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":451 + * _check_or_init_cufile() + * if __cuFileHandleDeregister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileHandleDeregister is not found") + * (__cuFileHandleDeregister)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":452 + * if __cuFileHandleDeregister == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileHandleDeregister is not found") # <<<<<<<<<<<<<< + * (__cuFileHandleDeregister)( + * fh) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 452, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileHandleDeregister}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 452, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 452, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":451 + * _check_or_init_cufile() + * if __cuFileHandleDeregister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileHandleDeregister is not found") + * (__cuFileHandleDeregister)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":450 + * global __cuFileHandleDeregister + * _check_or_init_cufile() + * if __cuFileHandleDeregister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileHandleDeregister is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":453 + * with gil: + * raise FunctionNotFoundError("function cuFileHandleDeregister is not found") + * (__cuFileHandleDeregister)( # <<<<<<<<<<<<<< + * fh) + * +*/ + ((void (*)(CUfileHandle_t))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleDeregister)(__pyx_v_fh); + + /* "cuda/bindings/_internal/cufile.pyx":446 + * + * + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef void _cuFileHandleDeregister(CUfileHandle_t fh) except* nogil: + * global __cuFileHandleDeregister +*/ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileHandleDeregister", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() +} + +/* "cuda/bindings/_internal/cufile.pyx":457 + * + * + * cdef CUfileError_t _cuFileBufRegister(const void* bufPtr_base, size_t length, int flags) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileBufRegister + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBufRegister(void const *__pyx_v_bufPtr_base, size_t __pyx_v_length, int __pyx_v_flags) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileBufRegister", 1); + + /* "cuda/bindings/_internal/cufile.pyx":459 + * cdef CUfileError_t _cuFileBufRegister(const void* bufPtr_base, size_t length, int flags) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileBufRegister + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileBufRegister == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 459, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":460 + * global __cuFileBufRegister + * _check_or_init_cufile() + * if __cuFileBufRegister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBufRegister is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufRegister == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":461 + * _check_or_init_cufile() + * if __cuFileBufRegister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBufRegister is not found") + * return (__cuFileBufRegister)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":462 + * if __cuFileBufRegister == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileBufRegister is not found") # <<<<<<<<<<<<<< + * return (__cuFileBufRegister)( + * bufPtr_base, length, flags) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 462, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileBufRegister_is_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 462, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 462, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":461 + * _check_or_init_cufile() + * if __cuFileBufRegister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBufRegister is not found") + * return (__cuFileBufRegister)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":460 + * global __cuFileBufRegister + * _check_or_init_cufile() + * if __cuFileBufRegister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBufRegister is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":463 + * with gil: + * raise FunctionNotFoundError("function cuFileBufRegister is not found") + * return (__cuFileBufRegister)( # <<<<<<<<<<<<<< + * bufPtr_base, length, flags) + * +*/ + __pyx_r = ((CUfileError_t (*)(void const *, size_t, int))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufRegister)(__pyx_v_bufPtr_base, __pyx_v_length, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":457 + * + * + * cdef CUfileError_t _cuFileBufRegister(const void* bufPtr_base, size_t length, int flags) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileBufRegister + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileBufRegister", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":467 + * + * + * cdef CUfileError_t _cuFileBufDeregister(const void* bufPtr_base) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileBufDeregister + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBufDeregister(void const *__pyx_v_bufPtr_base) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileBufDeregister", 1); + + /* "cuda/bindings/_internal/cufile.pyx":469 + * cdef CUfileError_t _cuFileBufDeregister(const void* bufPtr_base) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileBufDeregister + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileBufDeregister == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 469, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":470 + * global __cuFileBufDeregister + * _check_or_init_cufile() + * if __cuFileBufDeregister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBufDeregister is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufDeregister == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":471 + * _check_or_init_cufile() + * if __cuFileBufDeregister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBufDeregister is not found") + * return (__cuFileBufDeregister)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":472 + * if __cuFileBufDeregister == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileBufDeregister is not found") # <<<<<<<<<<<<<< + * return (__cuFileBufDeregister)( + * bufPtr_base) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 472, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileBufDeregister_is}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 472, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 472, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":471 + * _check_or_init_cufile() + * if __cuFileBufDeregister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBufDeregister is not found") + * return (__cuFileBufDeregister)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":470 + * global __cuFileBufDeregister + * _check_or_init_cufile() + * if __cuFileBufDeregister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBufDeregister is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":473 + * with gil: + * raise FunctionNotFoundError("function cuFileBufDeregister is not found") + * return (__cuFileBufDeregister)( # <<<<<<<<<<<<<< + * bufPtr_base) + * +*/ + __pyx_r = ((CUfileError_t (*)(void const *))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufDeregister)(__pyx_v_bufPtr_base); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":467 + * + * + * cdef CUfileError_t _cuFileBufDeregister(const void* bufPtr_base) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileBufDeregister + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileBufDeregister", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":477 + * + * + * cdef ssize_t _cuFileRead(CUfileHandle_t fh, void* bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset) except* nogil: # <<<<<<<<<<<<<< + * global __cuFileRead + * _check_or_init_cufile() +*/ + +static Py_ssize_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileRead(CUfileHandle_t __pyx_v_fh, void *__pyx_v_bufPtr_base, size_t __pyx_v_size, off_t __pyx_v_file_offset, off_t __pyx_v_bufPtr_offset) { + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileRead", 1); + + /* "cuda/bindings/_internal/cufile.pyx":479 + * cdef ssize_t _cuFileRead(CUfileHandle_t fh, void* bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset) except* nogil: + * global __cuFileRead + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileRead == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 479, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":480 + * global __cuFileRead + * _check_or_init_cufile() + * if __cuFileRead == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileRead is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileRead == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":481 + * _check_or_init_cufile() + * if __cuFileRead == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileRead is not found") + * return (__cuFileRead)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":482 + * if __cuFileRead == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileRead is not found") # <<<<<<<<<<<<<< + * return (__cuFileRead)( + * fh, bufPtr_base, size, file_offset, bufPtr_offset) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 482, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileRead_is_not_found}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 482, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 482, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":481 + * _check_or_init_cufile() + * if __cuFileRead == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileRead is not found") + * return (__cuFileRead)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":480 + * global __cuFileRead + * _check_or_init_cufile() + * if __cuFileRead == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileRead is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":483 + * with gil: + * raise FunctionNotFoundError("function cuFileRead is not found") + * return (__cuFileRead)( # <<<<<<<<<<<<<< + * fh, bufPtr_base, size, file_offset, bufPtr_offset) + * +*/ + __pyx_r = ((Py_ssize_t (*)(CUfileHandle_t, void *, size_t, off_t, off_t))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileRead)(__pyx_v_fh, __pyx_v_bufPtr_base, __pyx_v_size, __pyx_v_file_offset, __pyx_v_bufPtr_offset); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":477 + * + * + * cdef ssize_t _cuFileRead(CUfileHandle_t fh, void* bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset) except* nogil: # <<<<<<<<<<<<<< + * global __cuFileRead + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileRead", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":487 + * + * + * cdef ssize_t _cuFileWrite(CUfileHandle_t fh, const void* bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset) except* nogil: # <<<<<<<<<<<<<< + * global __cuFileWrite + * _check_or_init_cufile() +*/ + +static Py_ssize_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileWrite(CUfileHandle_t __pyx_v_fh, void const *__pyx_v_bufPtr_base, size_t __pyx_v_size, off_t __pyx_v_file_offset, off_t __pyx_v_bufPtr_offset) { + Py_ssize_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileWrite", 1); + + /* "cuda/bindings/_internal/cufile.pyx":489 + * cdef ssize_t _cuFileWrite(CUfileHandle_t fh, const void* bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset) except* nogil: + * global __cuFileWrite + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileWrite == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 489, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":490 + * global __cuFileWrite + * _check_or_init_cufile() + * if __cuFileWrite == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileWrite is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWrite == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":491 + * _check_or_init_cufile() + * if __cuFileWrite == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileWrite is not found") + * return (__cuFileWrite)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":492 + * if __cuFileWrite == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileWrite is not found") # <<<<<<<<<<<<<< + * return (__cuFileWrite)( + * fh, bufPtr_base, size, file_offset, bufPtr_offset) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 492, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileWrite_is_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 492, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 492, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":491 + * _check_or_init_cufile() + * if __cuFileWrite == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileWrite is not found") + * return (__cuFileWrite)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":490 + * global __cuFileWrite + * _check_or_init_cufile() + * if __cuFileWrite == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileWrite is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":493 + * with gil: + * raise FunctionNotFoundError("function cuFileWrite is not found") + * return (__cuFileWrite)( # <<<<<<<<<<<<<< + * fh, bufPtr_base, size, file_offset, bufPtr_offset) + * +*/ + __pyx_r = ((Py_ssize_t (*)(CUfileHandle_t, void const *, size_t, off_t, off_t))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWrite)(__pyx_v_fh, __pyx_v_bufPtr_base, __pyx_v_size, __pyx_v_file_offset, __pyx_v_bufPtr_offset); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":487 + * + * + * cdef ssize_t _cuFileWrite(CUfileHandle_t fh, const void* bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset) except* nogil: # <<<<<<<<<<<<<< + * global __cuFileWrite + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileWrite", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":497 + * + * + * cdef CUfileError_t _cuFileDriverOpen() except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverOpen + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverOpen(void) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileDriverOpen", 1); + + /* "cuda/bindings/_internal/cufile.pyx":499 + * cdef CUfileError_t _cuFileDriverOpen() except?CUFILE_LOADING_ERROR nogil: + * global __cuFileDriverOpen + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileDriverOpen == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 499, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":500 + * global __cuFileDriverOpen + * _check_or_init_cufile() + * if __cuFileDriverOpen == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverOpen is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverOpen == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":501 + * _check_or_init_cufile() + * if __cuFileDriverOpen == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverOpen is not found") + * return (__cuFileDriverOpen)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":502 + * if __cuFileDriverOpen == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileDriverOpen is not found") # <<<<<<<<<<<<<< + * return (__cuFileDriverOpen)( + * ) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 502, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileDriverOpen_is_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 502, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 502, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":501 + * _check_or_init_cufile() + * if __cuFileDriverOpen == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverOpen is not found") + * return (__cuFileDriverOpen)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":500 + * global __cuFileDriverOpen + * _check_or_init_cufile() + * if __cuFileDriverOpen == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverOpen is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":503 + * with gil: + * raise FunctionNotFoundError("function cuFileDriverOpen is not found") + * return (__cuFileDriverOpen)( # <<<<<<<<<<<<<< + * ) + * +*/ + __pyx_r = ((CUfileError_t (*)(void))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverOpen)(); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":497 + * + * + * cdef CUfileError_t _cuFileDriverOpen() except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverOpen + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileDriverOpen", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":507 + * + * + * cdef CUfileError_t _cuFileDriverClose_v2() except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverClose_v2 + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverClose_v2(void) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileDriverClose_v2", 1); + + /* "cuda/bindings/_internal/cufile.pyx":509 + * cdef CUfileError_t _cuFileDriverClose_v2() except?CUFILE_LOADING_ERROR nogil: + * global __cuFileDriverClose_v2 + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileDriverClose_v2 == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 509, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":510 + * global __cuFileDriverClose_v2 + * _check_or_init_cufile() + * if __cuFileDriverClose_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverClose_v2 is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose_v2 == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":511 + * _check_or_init_cufile() + * if __cuFileDriverClose_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverClose_v2 is not found") + * return (__cuFileDriverClose_v2)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":512 + * if __cuFileDriverClose_v2 == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileDriverClose_v2 is not found") # <<<<<<<<<<<<<< + * return (__cuFileDriverClose_v2)( + * ) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 512, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileDriverClose_v2_is}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 512, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 512, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":511 + * _check_or_init_cufile() + * if __cuFileDriverClose_v2 == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverClose_v2 is not found") + * return (__cuFileDriverClose_v2)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":510 + * global __cuFileDriverClose_v2 + * _check_or_init_cufile() + * if __cuFileDriverClose_v2 == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverClose_v2 is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":513 + * with gil: + * raise FunctionNotFoundError("function cuFileDriverClose_v2 is not found") + * return (__cuFileDriverClose_v2)( # <<<<<<<<<<<<<< + * ) + * +*/ + __pyx_r = ((CUfileError_t (*)(void))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose_v2)(); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":507 + * + * + * cdef CUfileError_t _cuFileDriverClose_v2() except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverClose_v2 + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileDriverClose_v2", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":517 + * + * + * cdef long _cuFileUseCount() except* nogil: # <<<<<<<<<<<<<< + * global __cuFileUseCount + * _check_or_init_cufile() +*/ + +static long __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileUseCount(void) { + long __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileUseCount", 1); + + /* "cuda/bindings/_internal/cufile.pyx":519 + * cdef long _cuFileUseCount() except* nogil: + * global __cuFileUseCount + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileUseCount == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 519, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":520 + * global __cuFileUseCount + * _check_or_init_cufile() + * if __cuFileUseCount == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileUseCount is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileUseCount == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":521 + * _check_or_init_cufile() + * if __cuFileUseCount == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileUseCount is not found") + * return (__cuFileUseCount)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":522 + * if __cuFileUseCount == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileUseCount is not found") # <<<<<<<<<<<<<< + * return (__cuFileUseCount)( + * ) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 522, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileUseCount_is_not_f}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 522, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 522, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":521 + * _check_or_init_cufile() + * if __cuFileUseCount == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileUseCount is not found") + * return (__cuFileUseCount)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":520 + * global __cuFileUseCount + * _check_or_init_cufile() + * if __cuFileUseCount == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileUseCount is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":523 + * with gil: + * raise FunctionNotFoundError("function cuFileUseCount is not found") + * return (__cuFileUseCount)( # <<<<<<<<<<<<<< + * ) + * +*/ + __pyx_r = ((long (*)(void))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileUseCount)(); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":517 + * + * + * cdef long _cuFileUseCount() except* nogil: # <<<<<<<<<<<<<< + * global __cuFileUseCount + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileUseCount", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":527 + * + * + * cdef CUfileError_t _cuFileDriverGetProperties(CUfileDrvProps_t* props) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverGetProperties + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverGetProperties(CUfileDrvProps_t *__pyx_v_props) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileDriverGetProperties", 1); + + /* "cuda/bindings/_internal/cufile.pyx":529 + * cdef CUfileError_t _cuFileDriverGetProperties(CUfileDrvProps_t* props) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileDriverGetProperties + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileDriverGetProperties == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 529, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":530 + * global __cuFileDriverGetProperties + * _check_or_init_cufile() + * if __cuFileDriverGetProperties == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverGetProperties is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverGetProperties == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":531 + * _check_or_init_cufile() + * if __cuFileDriverGetProperties == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverGetProperties is not found") + * return (__cuFileDriverGetProperties)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":532 + * if __cuFileDriverGetProperties == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileDriverGetProperties is not found") # <<<<<<<<<<<<<< + * return (__cuFileDriverGetProperties)( + * props) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 532, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileDriverGetProperti}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 532, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 532, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":531 + * _check_or_init_cufile() + * if __cuFileDriverGetProperties == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverGetProperties is not found") + * return (__cuFileDriverGetProperties)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":530 + * global __cuFileDriverGetProperties + * _check_or_init_cufile() + * if __cuFileDriverGetProperties == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverGetProperties is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":533 + * with gil: + * raise FunctionNotFoundError("function cuFileDriverGetProperties is not found") + * return (__cuFileDriverGetProperties)( # <<<<<<<<<<<<<< + * props) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUfileDrvProps_t *))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverGetProperties)(__pyx_v_props); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":527 + * + * + * cdef CUfileError_t _cuFileDriverGetProperties(CUfileDrvProps_t* props) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverGetProperties + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileDriverGetProperties", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":537 + * + * + * cdef CUfileError_t _cuFileDriverSetPollMode(cpp_bool poll, size_t poll_threshold_size) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverSetPollMode + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverSetPollMode(bool __pyx_v_poll, size_t __pyx_v_poll_threshold_size) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileDriverSetPollMode", 1); + + /* "cuda/bindings/_internal/cufile.pyx":539 + * cdef CUfileError_t _cuFileDriverSetPollMode(cpp_bool poll, size_t poll_threshold_size) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileDriverSetPollMode + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileDriverSetPollMode == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 539, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":540 + * global __cuFileDriverSetPollMode + * _check_or_init_cufile() + * if __cuFileDriverSetPollMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetPollMode is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetPollMode == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":541 + * _check_or_init_cufile() + * if __cuFileDriverSetPollMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverSetPollMode is not found") + * return (__cuFileDriverSetPollMode)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":542 + * if __cuFileDriverSetPollMode == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetPollMode is not found") # <<<<<<<<<<<<<< + * return (__cuFileDriverSetPollMode)( + * poll, poll_threshold_size) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 542, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileDriverSetPollMode}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 542, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 542, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":541 + * _check_or_init_cufile() + * if __cuFileDriverSetPollMode == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverSetPollMode is not found") + * return (__cuFileDriverSetPollMode)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":540 + * global __cuFileDriverSetPollMode + * _check_or_init_cufile() + * if __cuFileDriverSetPollMode == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetPollMode is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":543 + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetPollMode is not found") + * return (__cuFileDriverSetPollMode)( # <<<<<<<<<<<<<< + * poll, poll_threshold_size) + * +*/ + __pyx_r = ((CUfileError_t (*)(bool, size_t))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetPollMode)(__pyx_v_poll, __pyx_v_poll_threshold_size); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":537 + * + * + * cdef CUfileError_t _cuFileDriverSetPollMode(cpp_bool poll, size_t poll_threshold_size) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverSetPollMode + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileDriverSetPollMode", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":547 + * + * + * cdef CUfileError_t _cuFileDriverSetMaxDirectIOSize(size_t max_direct_io_size) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverSetMaxDirectIOSize + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverSetMaxDirectIOSize(size_t __pyx_v_max_direct_io_size) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileDriverSetMaxDirectIOSize", 1); + + /* "cuda/bindings/_internal/cufile.pyx":549 + * cdef CUfileError_t _cuFileDriverSetMaxDirectIOSize(size_t max_direct_io_size) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileDriverSetMaxDirectIOSize + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileDriverSetMaxDirectIOSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 549, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":550 + * global __cuFileDriverSetMaxDirectIOSize + * _check_or_init_cufile() + * if __cuFileDriverSetMaxDirectIOSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetMaxDirectIOSize is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxDirectIOSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":551 + * _check_or_init_cufile() + * if __cuFileDriverSetMaxDirectIOSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverSetMaxDirectIOSize is not found") + * return (__cuFileDriverSetMaxDirectIOSize)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":552 + * if __cuFileDriverSetMaxDirectIOSize == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetMaxDirectIOSize is not found") # <<<<<<<<<<<<<< + * return (__cuFileDriverSetMaxDirectIOSize)( + * max_direct_io_size) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 552, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileDriverSetMaxDirec}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 552, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 552, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":551 + * _check_or_init_cufile() + * if __cuFileDriverSetMaxDirectIOSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverSetMaxDirectIOSize is not found") + * return (__cuFileDriverSetMaxDirectIOSize)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":550 + * global __cuFileDriverSetMaxDirectIOSize + * _check_or_init_cufile() + * if __cuFileDriverSetMaxDirectIOSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetMaxDirectIOSize is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":553 + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetMaxDirectIOSize is not found") + * return (__cuFileDriverSetMaxDirectIOSize)( # <<<<<<<<<<<<<< + * max_direct_io_size) + * +*/ + __pyx_r = ((CUfileError_t (*)(size_t))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxDirectIOSize)(__pyx_v_max_direct_io_size); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":547 + * + * + * cdef CUfileError_t _cuFileDriverSetMaxDirectIOSize(size_t max_direct_io_size) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverSetMaxDirectIOSize + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileDriverSetMaxDirectIOSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":557 + * + * + * cdef CUfileError_t _cuFileDriverSetMaxCacheSize(size_t max_cache_size) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverSetMaxCacheSize + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverSetMaxCacheSize(size_t __pyx_v_max_cache_size) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileDriverSetMaxCacheSize", 1); + + /* "cuda/bindings/_internal/cufile.pyx":559 + * cdef CUfileError_t _cuFileDriverSetMaxCacheSize(size_t max_cache_size) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileDriverSetMaxCacheSize + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileDriverSetMaxCacheSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 559, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":560 + * global __cuFileDriverSetMaxCacheSize + * _check_or_init_cufile() + * if __cuFileDriverSetMaxCacheSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetMaxCacheSize is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxCacheSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":561 + * _check_or_init_cufile() + * if __cuFileDriverSetMaxCacheSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverSetMaxCacheSize is not found") + * return (__cuFileDriverSetMaxCacheSize)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":562 + * if __cuFileDriverSetMaxCacheSize == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetMaxCacheSize is not found") # <<<<<<<<<<<<<< + * return (__cuFileDriverSetMaxCacheSize)( + * max_cache_size) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 562, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileDriverSetMaxCache}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 562, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 562, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":561 + * _check_or_init_cufile() + * if __cuFileDriverSetMaxCacheSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverSetMaxCacheSize is not found") + * return (__cuFileDriverSetMaxCacheSize)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":560 + * global __cuFileDriverSetMaxCacheSize + * _check_or_init_cufile() + * if __cuFileDriverSetMaxCacheSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetMaxCacheSize is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":563 + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetMaxCacheSize is not found") + * return (__cuFileDriverSetMaxCacheSize)( # <<<<<<<<<<<<<< + * max_cache_size) + * +*/ + __pyx_r = ((CUfileError_t (*)(size_t))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxCacheSize)(__pyx_v_max_cache_size); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":557 + * + * + * cdef CUfileError_t _cuFileDriverSetMaxCacheSize(size_t max_cache_size) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverSetMaxCacheSize + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileDriverSetMaxCacheSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":567 + * + * + * cdef CUfileError_t _cuFileDriverSetMaxPinnedMemSize(size_t max_pinned_size) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverSetMaxPinnedMemSize + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverSetMaxPinnedMemSize(size_t __pyx_v_max_pinned_size) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileDriverSetMaxPinnedMemSize", 1); + + /* "cuda/bindings/_internal/cufile.pyx":569 + * cdef CUfileError_t _cuFileDriverSetMaxPinnedMemSize(size_t max_pinned_size) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileDriverSetMaxPinnedMemSize + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileDriverSetMaxPinnedMemSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 569, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":570 + * global __cuFileDriverSetMaxPinnedMemSize + * _check_or_init_cufile() + * if __cuFileDriverSetMaxPinnedMemSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetMaxPinnedMemSize is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxPinnedMemSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":571 + * _check_or_init_cufile() + * if __cuFileDriverSetMaxPinnedMemSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverSetMaxPinnedMemSize is not found") + * return (__cuFileDriverSetMaxPinnedMemSize)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":572 + * if __cuFileDriverSetMaxPinnedMemSize == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetMaxPinnedMemSize is not found") # <<<<<<<<<<<<<< + * return (__cuFileDriverSetMaxPinnedMemSize)( + * max_pinned_size) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 572, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileDriverSetMaxPinne}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 572, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 572, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":571 + * _check_or_init_cufile() + * if __cuFileDriverSetMaxPinnedMemSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverSetMaxPinnedMemSize is not found") + * return (__cuFileDriverSetMaxPinnedMemSize)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":570 + * global __cuFileDriverSetMaxPinnedMemSize + * _check_or_init_cufile() + * if __cuFileDriverSetMaxPinnedMemSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetMaxPinnedMemSize is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":573 + * with gil: + * raise FunctionNotFoundError("function cuFileDriverSetMaxPinnedMemSize is not found") + * return (__cuFileDriverSetMaxPinnedMemSize)( # <<<<<<<<<<<<<< + * max_pinned_size) + * +*/ + __pyx_r = ((CUfileError_t (*)(size_t))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxPinnedMemSize)(__pyx_v_max_pinned_size); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":567 + * + * + * cdef CUfileError_t _cuFileDriverSetMaxPinnedMemSize(size_t max_pinned_size) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverSetMaxPinnedMemSize + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileDriverSetMaxPinnedMemSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":577 + * + * + * cdef CUfileError_t _cuFileBatchIOSetUp(CUfileBatchHandle_t* batch_idp, unsigned nr) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileBatchIOSetUp + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBatchIOSetUp(CUfileBatchHandle_t *__pyx_v_batch_idp, unsigned int __pyx_v_nr) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileBatchIOSetUp", 1); + + /* "cuda/bindings/_internal/cufile.pyx":579 + * cdef CUfileError_t _cuFileBatchIOSetUp(CUfileBatchHandle_t* batch_idp, unsigned nr) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileBatchIOSetUp + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileBatchIOSetUp == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 579, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":580 + * global __cuFileBatchIOSetUp + * _check_or_init_cufile() + * if __cuFileBatchIOSetUp == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOSetUp is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSetUp == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":581 + * _check_or_init_cufile() + * if __cuFileBatchIOSetUp == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBatchIOSetUp is not found") + * return (__cuFileBatchIOSetUp)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":582 + * if __cuFileBatchIOSetUp == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOSetUp is not found") # <<<<<<<<<<<<<< + * return (__cuFileBatchIOSetUp)( + * batch_idp, nr) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 582, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileBatchIOSetUp_is_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 582, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 582, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":581 + * _check_or_init_cufile() + * if __cuFileBatchIOSetUp == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBatchIOSetUp is not found") + * return (__cuFileBatchIOSetUp)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":580 + * global __cuFileBatchIOSetUp + * _check_or_init_cufile() + * if __cuFileBatchIOSetUp == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOSetUp is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":583 + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOSetUp is not found") + * return (__cuFileBatchIOSetUp)( # <<<<<<<<<<<<<< + * batch_idp, nr) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUfileBatchHandle_t *, unsigned int))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSetUp)(__pyx_v_batch_idp, __pyx_v_nr); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":577 + * + * + * cdef CUfileError_t _cuFileBatchIOSetUp(CUfileBatchHandle_t* batch_idp, unsigned nr) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileBatchIOSetUp + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileBatchIOSetUp", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":587 + * + * + * cdef CUfileError_t _cuFileBatchIOSubmit(CUfileBatchHandle_t batch_idp, unsigned nr, CUfileIOParams_t* iocbp, unsigned int flags) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileBatchIOSubmit + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBatchIOSubmit(CUfileBatchHandle_t __pyx_v_batch_idp, unsigned int __pyx_v_nr, CUfileIOParams_t *__pyx_v_iocbp, unsigned int __pyx_v_flags) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileBatchIOSubmit", 1); + + /* "cuda/bindings/_internal/cufile.pyx":589 + * cdef CUfileError_t _cuFileBatchIOSubmit(CUfileBatchHandle_t batch_idp, unsigned nr, CUfileIOParams_t* iocbp, unsigned int flags) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileBatchIOSubmit + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileBatchIOSubmit == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 589, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":590 + * global __cuFileBatchIOSubmit + * _check_or_init_cufile() + * if __cuFileBatchIOSubmit == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOSubmit is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSubmit == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":591 + * _check_or_init_cufile() + * if __cuFileBatchIOSubmit == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBatchIOSubmit is not found") + * return (__cuFileBatchIOSubmit)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":592 + * if __cuFileBatchIOSubmit == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOSubmit is not found") # <<<<<<<<<<<<<< + * return (__cuFileBatchIOSubmit)( + * batch_idp, nr, iocbp, flags) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 592, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileBatchIOSubmit_is}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 592, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 592, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":591 + * _check_or_init_cufile() + * if __cuFileBatchIOSubmit == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBatchIOSubmit is not found") + * return (__cuFileBatchIOSubmit)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":590 + * global __cuFileBatchIOSubmit + * _check_or_init_cufile() + * if __cuFileBatchIOSubmit == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOSubmit is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":593 + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOSubmit is not found") + * return (__cuFileBatchIOSubmit)( # <<<<<<<<<<<<<< + * batch_idp, nr, iocbp, flags) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUfileBatchHandle_t, unsigned int, CUfileIOParams_t *, unsigned int))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSubmit)(__pyx_v_batch_idp, __pyx_v_nr, __pyx_v_iocbp, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":587 + * + * + * cdef CUfileError_t _cuFileBatchIOSubmit(CUfileBatchHandle_t batch_idp, unsigned nr, CUfileIOParams_t* iocbp, unsigned int flags) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileBatchIOSubmit + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileBatchIOSubmit", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":597 + * + * + * cdef CUfileError_t _cuFileBatchIOGetStatus(CUfileBatchHandle_t batch_idp, unsigned min_nr, unsigned* nr, CUfileIOEvents_t* iocbp, timespec* timeout) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileBatchIOGetStatus + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBatchIOGetStatus(CUfileBatchHandle_t __pyx_v_batch_idp, unsigned int __pyx_v_min_nr, unsigned int *__pyx_v_nr, CUfileIOEvents_t *__pyx_v_iocbp, struct timespec *__pyx_v_timeout) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileBatchIOGetStatus", 1); + + /* "cuda/bindings/_internal/cufile.pyx":599 + * cdef CUfileError_t _cuFileBatchIOGetStatus(CUfileBatchHandle_t batch_idp, unsigned min_nr, unsigned* nr, CUfileIOEvents_t* iocbp, timespec* timeout) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileBatchIOGetStatus + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileBatchIOGetStatus == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 599, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":600 + * global __cuFileBatchIOGetStatus + * _check_or_init_cufile() + * if __cuFileBatchIOGetStatus == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOGetStatus is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOGetStatus == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":601 + * _check_or_init_cufile() + * if __cuFileBatchIOGetStatus == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBatchIOGetStatus is not found") + * return (__cuFileBatchIOGetStatus)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":602 + * if __cuFileBatchIOGetStatus == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOGetStatus is not found") # <<<<<<<<<<<<<< + * return (__cuFileBatchIOGetStatus)( + * batch_idp, min_nr, nr, iocbp, timeout) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 602, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileBatchIOGetStatus}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 602, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 602, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":601 + * _check_or_init_cufile() + * if __cuFileBatchIOGetStatus == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBatchIOGetStatus is not found") + * return (__cuFileBatchIOGetStatus)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":600 + * global __cuFileBatchIOGetStatus + * _check_or_init_cufile() + * if __cuFileBatchIOGetStatus == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOGetStatus is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":603 + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOGetStatus is not found") + * return (__cuFileBatchIOGetStatus)( # <<<<<<<<<<<<<< + * batch_idp, min_nr, nr, iocbp, timeout) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUfileBatchHandle_t, unsigned int, unsigned int *, CUfileIOEvents_t *, struct timespec *))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOGetStatus)(__pyx_v_batch_idp, __pyx_v_min_nr, __pyx_v_nr, __pyx_v_iocbp, __pyx_v_timeout); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":597 + * + * + * cdef CUfileError_t _cuFileBatchIOGetStatus(CUfileBatchHandle_t batch_idp, unsigned min_nr, unsigned* nr, CUfileIOEvents_t* iocbp, timespec* timeout) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileBatchIOGetStatus + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileBatchIOGetStatus", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":607 + * + * + * cdef CUfileError_t _cuFileBatchIOCancel(CUfileBatchHandle_t batch_idp) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileBatchIOCancel + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBatchIOCancel(CUfileBatchHandle_t __pyx_v_batch_idp) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileBatchIOCancel", 1); + + /* "cuda/bindings/_internal/cufile.pyx":609 + * cdef CUfileError_t _cuFileBatchIOCancel(CUfileBatchHandle_t batch_idp) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileBatchIOCancel + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileBatchIOCancel == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 609, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":610 + * global __cuFileBatchIOCancel + * _check_or_init_cufile() + * if __cuFileBatchIOCancel == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOCancel is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOCancel == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":611 + * _check_or_init_cufile() + * if __cuFileBatchIOCancel == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBatchIOCancel is not found") + * return (__cuFileBatchIOCancel)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":612 + * if __cuFileBatchIOCancel == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOCancel is not found") # <<<<<<<<<<<<<< + * return (__cuFileBatchIOCancel)( + * batch_idp) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 612, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileBatchIOCancel_is}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 612, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 612, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":611 + * _check_or_init_cufile() + * if __cuFileBatchIOCancel == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBatchIOCancel is not found") + * return (__cuFileBatchIOCancel)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":610 + * global __cuFileBatchIOCancel + * _check_or_init_cufile() + * if __cuFileBatchIOCancel == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOCancel is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":613 + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIOCancel is not found") + * return (__cuFileBatchIOCancel)( # <<<<<<<<<<<<<< + * batch_idp) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUfileBatchHandle_t))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOCancel)(__pyx_v_batch_idp); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":607 + * + * + * cdef CUfileError_t _cuFileBatchIOCancel(CUfileBatchHandle_t batch_idp) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileBatchIOCancel + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileBatchIOCancel", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":617 + * + * + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef void _cuFileBatchIODestroy(CUfileBatchHandle_t batch_idp) except* nogil: + * global __cuFileBatchIODestroy +*/ + +static void __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBatchIODestroy(CUfileBatchHandle_t __pyx_v_batch_idp) { + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileBatchIODestroy", 1); + + /* "cuda/bindings/_internal/cufile.pyx":620 + * cdef void _cuFileBatchIODestroy(CUfileBatchHandle_t batch_idp) except* nogil: + * global __cuFileBatchIODestroy + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileBatchIODestroy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 620, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":621 + * global __cuFileBatchIODestroy + * _check_or_init_cufile() + * if __cuFileBatchIODestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIODestroy is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIODestroy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":622 + * _check_or_init_cufile() + * if __cuFileBatchIODestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBatchIODestroy is not found") + * (__cuFileBatchIODestroy)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":623 + * if __cuFileBatchIODestroy == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIODestroy is not found") # <<<<<<<<<<<<<< + * (__cuFileBatchIODestroy)( + * batch_idp) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 623, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileBatchIODestroy_is}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 623, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 623, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":622 + * _check_or_init_cufile() + * if __cuFileBatchIODestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileBatchIODestroy is not found") + * (__cuFileBatchIODestroy)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":621 + * global __cuFileBatchIODestroy + * _check_or_init_cufile() + * if __cuFileBatchIODestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIODestroy is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":624 + * with gil: + * raise FunctionNotFoundError("function cuFileBatchIODestroy is not found") + * (__cuFileBatchIODestroy)( # <<<<<<<<<<<<<< + * batch_idp) + * +*/ + ((void (*)(CUfileBatchHandle_t))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIODestroy)(__pyx_v_batch_idp); + + /* "cuda/bindings/_internal/cufile.pyx":617 + * + * + * @cython.show_performance_hints(False) # <<<<<<<<<<<<<< + * cdef void _cuFileBatchIODestroy(CUfileBatchHandle_t batch_idp) except* nogil: + * global __cuFileBatchIODestroy +*/ + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileBatchIODestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() +} + +/* "cuda/bindings/_internal/cufile.pyx":628 + * + * + * cdef CUfileError_t _cuFileReadAsync(CUfileHandle_t fh, void* bufPtr_base, size_t* size_p, off_t* file_offset_p, off_t* bufPtr_offset_p, ssize_t* bytes_read_p, CUstream stream) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileReadAsync + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileReadAsync(CUfileHandle_t __pyx_v_fh, void *__pyx_v_bufPtr_base, size_t *__pyx_v_size_p, off_t *__pyx_v_file_offset_p, off_t *__pyx_v_bufPtr_offset_p, Py_ssize_t *__pyx_v_bytes_read_p, CUstream __pyx_v_stream) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileReadAsync", 1); + + /* "cuda/bindings/_internal/cufile.pyx":630 + * cdef CUfileError_t _cuFileReadAsync(CUfileHandle_t fh, void* bufPtr_base, size_t* size_p, off_t* file_offset_p, off_t* bufPtr_offset_p, ssize_t* bytes_read_p, CUstream stream) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileReadAsync + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileReadAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 630, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":631 + * global __cuFileReadAsync + * _check_or_init_cufile() + * if __cuFileReadAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileReadAsync is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileReadAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":632 + * _check_or_init_cufile() + * if __cuFileReadAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileReadAsync is not found") + * return (__cuFileReadAsync)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":633 + * if __cuFileReadAsync == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileReadAsync is not found") # <<<<<<<<<<<<<< + * return (__cuFileReadAsync)( + * fh, bufPtr_base, size_p, file_offset_p, bufPtr_offset_p, bytes_read_p, stream) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 633, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileReadAsync_is_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 633, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 633, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":632 + * _check_or_init_cufile() + * if __cuFileReadAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileReadAsync is not found") + * return (__cuFileReadAsync)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":631 + * global __cuFileReadAsync + * _check_or_init_cufile() + * if __cuFileReadAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileReadAsync is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":634 + * with gil: + * raise FunctionNotFoundError("function cuFileReadAsync is not found") + * return (__cuFileReadAsync)( # <<<<<<<<<<<<<< + * fh, bufPtr_base, size_p, file_offset_p, bufPtr_offset_p, bytes_read_p, stream) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUfileHandle_t, void *, size_t *, off_t *, off_t *, Py_ssize_t *, CUstream))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileReadAsync)(__pyx_v_fh, __pyx_v_bufPtr_base, __pyx_v_size_p, __pyx_v_file_offset_p, __pyx_v_bufPtr_offset_p, __pyx_v_bytes_read_p, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":628 + * + * + * cdef CUfileError_t _cuFileReadAsync(CUfileHandle_t fh, void* bufPtr_base, size_t* size_p, off_t* file_offset_p, off_t* bufPtr_offset_p, ssize_t* bytes_read_p, CUstream stream) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileReadAsync + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileReadAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":638 + * + * + * cdef CUfileError_t _cuFileWriteAsync(CUfileHandle_t fh, void* bufPtr_base, size_t* size_p, off_t* file_offset_p, off_t* bufPtr_offset_p, ssize_t* bytes_written_p, CUstream stream) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileWriteAsync + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileWriteAsync(CUfileHandle_t __pyx_v_fh, void *__pyx_v_bufPtr_base, size_t *__pyx_v_size_p, off_t *__pyx_v_file_offset_p, off_t *__pyx_v_bufPtr_offset_p, Py_ssize_t *__pyx_v_bytes_written_p, CUstream __pyx_v_stream) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileWriteAsync", 1); + + /* "cuda/bindings/_internal/cufile.pyx":640 + * cdef CUfileError_t _cuFileWriteAsync(CUfileHandle_t fh, void* bufPtr_base, size_t* size_p, off_t* file_offset_p, off_t* bufPtr_offset_p, ssize_t* bytes_written_p, CUstream stream) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileWriteAsync + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileWriteAsync == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 640, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":641 + * global __cuFileWriteAsync + * _check_or_init_cufile() + * if __cuFileWriteAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileWriteAsync is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWriteAsync == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":642 + * _check_or_init_cufile() + * if __cuFileWriteAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileWriteAsync is not found") + * return (__cuFileWriteAsync)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":643 + * if __cuFileWriteAsync == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileWriteAsync is not found") # <<<<<<<<<<<<<< + * return (__cuFileWriteAsync)( + * fh, bufPtr_base, size_p, file_offset_p, bufPtr_offset_p, bytes_written_p, stream) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 643, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileWriteAsync_is_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 643, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 643, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":642 + * _check_or_init_cufile() + * if __cuFileWriteAsync == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileWriteAsync is not found") + * return (__cuFileWriteAsync)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":641 + * global __cuFileWriteAsync + * _check_or_init_cufile() + * if __cuFileWriteAsync == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileWriteAsync is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":644 + * with gil: + * raise FunctionNotFoundError("function cuFileWriteAsync is not found") + * return (__cuFileWriteAsync)( # <<<<<<<<<<<<<< + * fh, bufPtr_base, size_p, file_offset_p, bufPtr_offset_p, bytes_written_p, stream) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUfileHandle_t, void *, size_t *, off_t *, off_t *, Py_ssize_t *, CUstream))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWriteAsync)(__pyx_v_fh, __pyx_v_bufPtr_base, __pyx_v_size_p, __pyx_v_file_offset_p, __pyx_v_bufPtr_offset_p, __pyx_v_bytes_written_p, __pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":638 + * + * + * cdef CUfileError_t _cuFileWriteAsync(CUfileHandle_t fh, void* bufPtr_base, size_t* size_p, off_t* file_offset_p, off_t* bufPtr_offset_p, ssize_t* bytes_written_p, CUstream stream) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileWriteAsync + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileWriteAsync", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":648 + * + * + * cdef CUfileError_t _cuFileStreamRegister(CUstream stream, unsigned flags) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileStreamRegister + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileStreamRegister(CUstream __pyx_v_stream, unsigned int __pyx_v_flags) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileStreamRegister", 1); + + /* "cuda/bindings/_internal/cufile.pyx":650 + * cdef CUfileError_t _cuFileStreamRegister(CUstream stream, unsigned flags) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileStreamRegister + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileStreamRegister == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 650, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":651 + * global __cuFileStreamRegister + * _check_or_init_cufile() + * if __cuFileStreamRegister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileStreamRegister is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamRegister == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":652 + * _check_or_init_cufile() + * if __cuFileStreamRegister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileStreamRegister is not found") + * return (__cuFileStreamRegister)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":653 + * if __cuFileStreamRegister == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileStreamRegister is not found") # <<<<<<<<<<<<<< + * return (__cuFileStreamRegister)( + * stream, flags) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 653, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileStreamRegister_is}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 653, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 653, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":652 + * _check_or_init_cufile() + * if __cuFileStreamRegister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileStreamRegister is not found") + * return (__cuFileStreamRegister)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":651 + * global __cuFileStreamRegister + * _check_or_init_cufile() + * if __cuFileStreamRegister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileStreamRegister is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":654 + * with gil: + * raise FunctionNotFoundError("function cuFileStreamRegister is not found") + * return (__cuFileStreamRegister)( # <<<<<<<<<<<<<< + * stream, flags) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUstream, unsigned int))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamRegister)(__pyx_v_stream, __pyx_v_flags); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":648 + * + * + * cdef CUfileError_t _cuFileStreamRegister(CUstream stream, unsigned flags) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileStreamRegister + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileStreamRegister", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":658 + * + * + * cdef CUfileError_t _cuFileStreamDeregister(CUstream stream) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileStreamDeregister + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileStreamDeregister(CUstream __pyx_v_stream) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileStreamDeregister", 1); + + /* "cuda/bindings/_internal/cufile.pyx":660 + * cdef CUfileError_t _cuFileStreamDeregister(CUstream stream) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileStreamDeregister + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileStreamDeregister == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 660, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":661 + * global __cuFileStreamDeregister + * _check_or_init_cufile() + * if __cuFileStreamDeregister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileStreamDeregister is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamDeregister == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":662 + * _check_or_init_cufile() + * if __cuFileStreamDeregister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileStreamDeregister is not found") + * return (__cuFileStreamDeregister)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":663 + * if __cuFileStreamDeregister == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileStreamDeregister is not found") # <<<<<<<<<<<<<< + * return (__cuFileStreamDeregister)( + * stream) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 663, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileStreamDeregister}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 663, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 663, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":662 + * _check_or_init_cufile() + * if __cuFileStreamDeregister == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileStreamDeregister is not found") + * return (__cuFileStreamDeregister)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":661 + * global __cuFileStreamDeregister + * _check_or_init_cufile() + * if __cuFileStreamDeregister == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileStreamDeregister is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":664 + * with gil: + * raise FunctionNotFoundError("function cuFileStreamDeregister is not found") + * return (__cuFileStreamDeregister)( # <<<<<<<<<<<<<< + * stream) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUstream))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamDeregister)(__pyx_v_stream); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":658 + * + * + * cdef CUfileError_t _cuFileStreamDeregister(CUstream stream) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileStreamDeregister + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileStreamDeregister", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":668 + * + * + * cdef CUfileError_t _cuFileGetVersion(int* version) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileGetVersion + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileGetVersion(int *__pyx_v_version) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileGetVersion", 1); + + /* "cuda/bindings/_internal/cufile.pyx":670 + * cdef CUfileError_t _cuFileGetVersion(int* version) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileGetVersion + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileGetVersion == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 670, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":671 + * global __cuFileGetVersion + * _check_or_init_cufile() + * if __cuFileGetVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileGetVersion is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetVersion == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":672 + * _check_or_init_cufile() + * if __cuFileGetVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileGetVersion is not found") + * return (__cuFileGetVersion)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":673 + * if __cuFileGetVersion == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileGetVersion is not found") # <<<<<<<<<<<<<< + * return (__cuFileGetVersion)( + * version) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 673, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileGetVersion_is_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 673, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 673, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":672 + * _check_or_init_cufile() + * if __cuFileGetVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileGetVersion is not found") + * return (__cuFileGetVersion)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":671 + * global __cuFileGetVersion + * _check_or_init_cufile() + * if __cuFileGetVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileGetVersion is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":674 + * with gil: + * raise FunctionNotFoundError("function cuFileGetVersion is not found") + * return (__cuFileGetVersion)( # <<<<<<<<<<<<<< + * version) + * +*/ + __pyx_r = ((CUfileError_t (*)(int *))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetVersion)(__pyx_v_version); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":668 + * + * + * cdef CUfileError_t _cuFileGetVersion(int* version) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileGetVersion + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileGetVersion", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":678 + * + * + * cdef CUfileError_t _cuFileGetParameterSizeT(CUFileSizeTConfigParameter_t param, size_t* value) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileGetParameterSizeT + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileGetParameterSizeT(CUFileSizeTConfigParameter_t __pyx_v_param, size_t *__pyx_v_value) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileGetParameterSizeT", 1); + + /* "cuda/bindings/_internal/cufile.pyx":680 + * cdef CUfileError_t _cuFileGetParameterSizeT(CUFileSizeTConfigParameter_t param, size_t* value) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileGetParameterSizeT + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileGetParameterSizeT == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 680, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":681 + * global __cuFileGetParameterSizeT + * _check_or_init_cufile() + * if __cuFileGetParameterSizeT == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileGetParameterSizeT is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterSizeT == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":682 + * _check_or_init_cufile() + * if __cuFileGetParameterSizeT == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileGetParameterSizeT is not found") + * return (__cuFileGetParameterSizeT)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":683 + * if __cuFileGetParameterSizeT == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileGetParameterSizeT is not found") # <<<<<<<<<<<<<< + * return (__cuFileGetParameterSizeT)( + * param, value) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 683, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileGetParameterSizeT}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 683, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 683, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":682 + * _check_or_init_cufile() + * if __cuFileGetParameterSizeT == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileGetParameterSizeT is not found") + * return (__cuFileGetParameterSizeT)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":681 + * global __cuFileGetParameterSizeT + * _check_or_init_cufile() + * if __cuFileGetParameterSizeT == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileGetParameterSizeT is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":684 + * with gil: + * raise FunctionNotFoundError("function cuFileGetParameterSizeT is not found") + * return (__cuFileGetParameterSizeT)( # <<<<<<<<<<<<<< + * param, value) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUFileSizeTConfigParameter_t, size_t *))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterSizeT)(__pyx_v_param, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":678 + * + * + * cdef CUfileError_t _cuFileGetParameterSizeT(CUFileSizeTConfigParameter_t param, size_t* value) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileGetParameterSizeT + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileGetParameterSizeT", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":688 + * + * + * cdef CUfileError_t _cuFileGetParameterBool(CUFileBoolConfigParameter_t param, cpp_bool* value) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileGetParameterBool + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileGetParameterBool(CUFileBoolConfigParameter_t __pyx_v_param, bool *__pyx_v_value) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileGetParameterBool", 1); + + /* "cuda/bindings/_internal/cufile.pyx":690 + * cdef CUfileError_t _cuFileGetParameterBool(CUFileBoolConfigParameter_t param, cpp_bool* value) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileGetParameterBool + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileGetParameterBool == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 690, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":691 + * global __cuFileGetParameterBool + * _check_or_init_cufile() + * if __cuFileGetParameterBool == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileGetParameterBool is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterBool == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":692 + * _check_or_init_cufile() + * if __cuFileGetParameterBool == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileGetParameterBool is not found") + * return (__cuFileGetParameterBool)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":693 + * if __cuFileGetParameterBool == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileGetParameterBool is not found") # <<<<<<<<<<<<<< + * return (__cuFileGetParameterBool)( + * param, value) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 693, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileGetParameterBool}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 693, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 693, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":692 + * _check_or_init_cufile() + * if __cuFileGetParameterBool == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileGetParameterBool is not found") + * return (__cuFileGetParameterBool)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":691 + * global __cuFileGetParameterBool + * _check_or_init_cufile() + * if __cuFileGetParameterBool == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileGetParameterBool is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":694 + * with gil: + * raise FunctionNotFoundError("function cuFileGetParameterBool is not found") + * return (__cuFileGetParameterBool)( # <<<<<<<<<<<<<< + * param, value) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUFileBoolConfigParameter_t, bool *))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterBool)(__pyx_v_param, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":688 + * + * + * cdef CUfileError_t _cuFileGetParameterBool(CUFileBoolConfigParameter_t param, cpp_bool* value) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileGetParameterBool + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileGetParameterBool", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":698 + * + * + * cdef CUfileError_t _cuFileGetParameterString(CUFileStringConfigParameter_t param, char* desc_str, int len) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileGetParameterString + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileGetParameterString(CUFileStringConfigParameter_t __pyx_v_param, char *__pyx_v_desc_str, int __pyx_v_len) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileGetParameterString", 1); + + /* "cuda/bindings/_internal/cufile.pyx":700 + * cdef CUfileError_t _cuFileGetParameterString(CUFileStringConfigParameter_t param, char* desc_str, int len) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileGetParameterString + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileGetParameterString == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 700, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":701 + * global __cuFileGetParameterString + * _check_or_init_cufile() + * if __cuFileGetParameterString == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileGetParameterString is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterString == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":702 + * _check_or_init_cufile() + * if __cuFileGetParameterString == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileGetParameterString is not found") + * return (__cuFileGetParameterString)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":703 + * if __cuFileGetParameterString == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileGetParameterString is not found") # <<<<<<<<<<<<<< + * return (__cuFileGetParameterString)( + * param, desc_str, len) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 703, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileGetParameterStrin}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 703, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 703, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":702 + * _check_or_init_cufile() + * if __cuFileGetParameterString == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileGetParameterString is not found") + * return (__cuFileGetParameterString)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":701 + * global __cuFileGetParameterString + * _check_or_init_cufile() + * if __cuFileGetParameterString == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileGetParameterString is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":704 + * with gil: + * raise FunctionNotFoundError("function cuFileGetParameterString is not found") + * return (__cuFileGetParameterString)( # <<<<<<<<<<<<<< + * param, desc_str, len) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUFileStringConfigParameter_t, char *, int))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterString)(__pyx_v_param, __pyx_v_desc_str, __pyx_v_len); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":698 + * + * + * cdef CUfileError_t _cuFileGetParameterString(CUFileStringConfigParameter_t param, char* desc_str, int len) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileGetParameterString + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileGetParameterString", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":708 + * + * + * cdef CUfileError_t _cuFileSetParameterSizeT(CUFileSizeTConfigParameter_t param, size_t value) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileSetParameterSizeT + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileSetParameterSizeT(CUFileSizeTConfigParameter_t __pyx_v_param, size_t __pyx_v_value) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileSetParameterSizeT", 1); + + /* "cuda/bindings/_internal/cufile.pyx":710 + * cdef CUfileError_t _cuFileSetParameterSizeT(CUFileSizeTConfigParameter_t param, size_t value) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileSetParameterSizeT + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileSetParameterSizeT == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 710, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":711 + * global __cuFileSetParameterSizeT + * _check_or_init_cufile() + * if __cuFileSetParameterSizeT == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileSetParameterSizeT is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterSizeT == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":712 + * _check_or_init_cufile() + * if __cuFileSetParameterSizeT == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileSetParameterSizeT is not found") + * return (__cuFileSetParameterSizeT)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":713 + * if __cuFileSetParameterSizeT == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileSetParameterSizeT is not found") # <<<<<<<<<<<<<< + * return (__cuFileSetParameterSizeT)( + * param, value) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 713, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileSetParameterSizeT}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 713, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 713, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":712 + * _check_or_init_cufile() + * if __cuFileSetParameterSizeT == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileSetParameterSizeT is not found") + * return (__cuFileSetParameterSizeT)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":711 + * global __cuFileSetParameterSizeT + * _check_or_init_cufile() + * if __cuFileSetParameterSizeT == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileSetParameterSizeT is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":714 + * with gil: + * raise FunctionNotFoundError("function cuFileSetParameterSizeT is not found") + * return (__cuFileSetParameterSizeT)( # <<<<<<<<<<<<<< + * param, value) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUFileSizeTConfigParameter_t, size_t))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterSizeT)(__pyx_v_param, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":708 + * + * + * cdef CUfileError_t _cuFileSetParameterSizeT(CUFileSizeTConfigParameter_t param, size_t value) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileSetParameterSizeT + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileSetParameterSizeT", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":718 + * + * + * cdef CUfileError_t _cuFileSetParameterBool(CUFileBoolConfigParameter_t param, cpp_bool value) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileSetParameterBool + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileSetParameterBool(CUFileBoolConfigParameter_t __pyx_v_param, bool __pyx_v_value) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileSetParameterBool", 1); + + /* "cuda/bindings/_internal/cufile.pyx":720 + * cdef CUfileError_t _cuFileSetParameterBool(CUFileBoolConfigParameter_t param, cpp_bool value) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileSetParameterBool + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileSetParameterBool == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 720, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":721 + * global __cuFileSetParameterBool + * _check_or_init_cufile() + * if __cuFileSetParameterBool == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileSetParameterBool is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterBool == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":722 + * _check_or_init_cufile() + * if __cuFileSetParameterBool == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileSetParameterBool is not found") + * return (__cuFileSetParameterBool)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":723 + * if __cuFileSetParameterBool == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileSetParameterBool is not found") # <<<<<<<<<<<<<< + * return (__cuFileSetParameterBool)( + * param, value) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 723, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileSetParameterBool}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 723, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 723, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":722 + * _check_or_init_cufile() + * if __cuFileSetParameterBool == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileSetParameterBool is not found") + * return (__cuFileSetParameterBool)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":721 + * global __cuFileSetParameterBool + * _check_or_init_cufile() + * if __cuFileSetParameterBool == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileSetParameterBool is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":724 + * with gil: + * raise FunctionNotFoundError("function cuFileSetParameterBool is not found") + * return (__cuFileSetParameterBool)( # <<<<<<<<<<<<<< + * param, value) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUFileBoolConfigParameter_t, bool))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterBool)(__pyx_v_param, __pyx_v_value); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":718 + * + * + * cdef CUfileError_t _cuFileSetParameterBool(CUFileBoolConfigParameter_t param, cpp_bool value) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileSetParameterBool + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileSetParameterBool", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":728 + * + * + * cdef CUfileError_t _cuFileSetParameterString(CUFileStringConfigParameter_t param, const char* desc_str) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileSetParameterString + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileSetParameterString(CUFileStringConfigParameter_t __pyx_v_param, char const *__pyx_v_desc_str) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileSetParameterString", 1); + + /* "cuda/bindings/_internal/cufile.pyx":730 + * cdef CUfileError_t _cuFileSetParameterString(CUFileStringConfigParameter_t param, const char* desc_str) except?CUFILE_LOADING_ERROR nogil: + * global __cuFileSetParameterString + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileSetParameterString == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 730, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":731 + * global __cuFileSetParameterString + * _check_or_init_cufile() + * if __cuFileSetParameterString == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileSetParameterString is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterString == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":732 + * _check_or_init_cufile() + * if __cuFileSetParameterString == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileSetParameterString is not found") + * return (__cuFileSetParameterString)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":733 + * if __cuFileSetParameterString == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileSetParameterString is not found") # <<<<<<<<<<<<<< + * return (__cuFileSetParameterString)( + * param, desc_str) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 733, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileSetParameterStrin}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 733, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 733, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":732 + * _check_or_init_cufile() + * if __cuFileSetParameterString == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileSetParameterString is not found") + * return (__cuFileSetParameterString)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":731 + * global __cuFileSetParameterString + * _check_or_init_cufile() + * if __cuFileSetParameterString == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileSetParameterString is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":734 + * with gil: + * raise FunctionNotFoundError("function cuFileSetParameterString is not found") + * return (__cuFileSetParameterString)( # <<<<<<<<<<<<<< + * param, desc_str) + * +*/ + __pyx_r = ((CUfileError_t (*)(CUFileStringConfigParameter_t, char const *))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterString)(__pyx_v_param, __pyx_v_desc_str); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":728 + * + * + * cdef CUfileError_t _cuFileSetParameterString(CUFileStringConfigParameter_t param, const char* desc_str) except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileSetParameterString + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileSetParameterString", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/cufile.pyx":738 + * + * + * cdef CUfileError_t _cuFileDriverClose() except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverClose + * _check_or_init_cufile() +*/ + +static CUfileError_t __pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverClose(void) { + CUfileError_t __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_cuFileDriverClose", 1); + + /* "cuda/bindings/_internal/cufile.pyx":740 + * cdef CUfileError_t _cuFileDriverClose() except?CUFILE_LOADING_ERROR nogil: + * global __cuFileDriverClose + * _check_or_init_cufile() # <<<<<<<<<<<<<< + * if __cuFileDriverClose == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_6cufile__check_or_init_cufile(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 740, __pyx_L1_error) + + /* "cuda/bindings/_internal/cufile.pyx":741 + * global __cuFileDriverClose + * _check_or_init_cufile() + * if __cuFileDriverClose == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverClose is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/cufile.pyx":742 + * _check_or_init_cufile() + * if __cuFileDriverClose == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverClose is not found") + * return (__cuFileDriverClose)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/cufile.pyx":743 + * if __cuFileDriverClose == NULL: + * with gil: + * raise FunctionNotFoundError("function cuFileDriverClose is not found") # <<<<<<<<<<<<<< + * return (__cuFileDriverClose)( + * ) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 743, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_cuFileDriverClose_is_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 743, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 743, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/cufile.pyx":742 + * _check_or_init_cufile() + * if __cuFileDriverClose == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function cuFileDriverClose is not found") + * return (__cuFileDriverClose)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/cufile.pyx":741 + * global __cuFileDriverClose + * _check_or_init_cufile() + * if __cuFileDriverClose == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function cuFileDriverClose is not found") +*/ + } + + /* "cuda/bindings/_internal/cufile.pyx":744 + * with gil: + * raise FunctionNotFoundError("function cuFileDriverClose is not found") + * return (__cuFileDriverClose)( # <<<<<<<<<<<<<< + * ) +*/ + __pyx_r = ((CUfileError_t (*)(void))__pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose)(); + goto __pyx_L0; + + /* "cuda/bindings/_internal/cufile.pyx":738 + * + * + * cdef CUfileError_t _cuFileDriverClose() except?CUFILE_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __cuFileDriverClose + * _check_or_init_cufile() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.cufile._cuFileDriverClose", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = CUFILE_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} +/* #### Code section: module_exttypes ### */ + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___symbol_lock = Py_None; Py_INCREF(Py_None); + __pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs = ((PyObject*)Py_None); Py_INCREF(Py_None); + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + if (__Pyx_ExportFunction("_cuFileHandleRegister", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileHandleRegister, "CUfileError_t (CUfileHandle_t *, CUfileDescr_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileHandleDeregister", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileHandleDeregister, "void (CUfileHandle_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileBufRegister", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBufRegister, "CUfileError_t (void const *, size_t, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileBufDeregister", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBufDeregister, "CUfileError_t (void const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileRead", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileRead, "Py_ssize_t (CUfileHandle_t, void *, size_t, off_t, off_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileWrite", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileWrite, "Py_ssize_t (CUfileHandle_t, void const *, size_t, off_t, off_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileDriverOpen", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverOpen, "CUfileError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileDriverClose_v2", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverClose_v2, "CUfileError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileUseCount", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileUseCount, "long (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileDriverGetProperties", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverGetProperties, "CUfileError_t (CUfileDrvProps_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileDriverSetPollMode", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverSetPollMode, "CUfileError_t (bool, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileDriverSetMaxDirectIOSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverSetMaxDirectIOSize, "CUfileError_t (size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileDriverSetMaxCacheSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverSetMaxCacheSize, "CUfileError_t (size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileDriverSetMaxPinnedMemSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverSetMaxPinnedMemSize, "CUfileError_t (size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileBatchIOSetUp", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBatchIOSetUp, "CUfileError_t (CUfileBatchHandle_t *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileBatchIOSubmit", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBatchIOSubmit, "CUfileError_t (CUfileBatchHandle_t, unsigned int, CUfileIOParams_t *, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileBatchIOGetStatus", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBatchIOGetStatus, "CUfileError_t (CUfileBatchHandle_t, unsigned int, unsigned int *, CUfileIOEvents_t *, struct timespec *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileBatchIOCancel", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBatchIOCancel, "CUfileError_t (CUfileBatchHandle_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileBatchIODestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileBatchIODestroy, "void (CUfileBatchHandle_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileReadAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileReadAsync, "CUfileError_t (CUfileHandle_t, void *, size_t *, off_t *, off_t *, Py_ssize_t *, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileWriteAsync", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileWriteAsync, "CUfileError_t (CUfileHandle_t, void *, size_t *, off_t *, off_t *, Py_ssize_t *, CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileStreamRegister", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileStreamRegister, "CUfileError_t (CUstream, unsigned int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileStreamDeregister", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileStreamDeregister, "CUfileError_t (CUstream)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileGetVersion", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileGetVersion, "CUfileError_t (int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileGetParameterSizeT", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileGetParameterSizeT, "CUfileError_t (CUFileSizeTConfigParameter_t, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileGetParameterBool", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileGetParameterBool, "CUfileError_t (CUFileBoolConfigParameter_t, bool *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileGetParameterString", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileGetParameterString, "CUfileError_t (CUFileStringConfigParameter_t, char *, int)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileSetParameterSizeT", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileSetParameterSizeT, "CUfileError_t (CUFileSizeTConfigParameter_t, size_t)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileSetParameterBool", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileSetParameterBool, "CUfileError_t (CUFileBoolConfigParameter_t, bool)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileSetParameterString", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileSetParameterString, "CUfileError_t (CUFileStringConfigParameter_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_cuFileDriverClose", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_6cufile__cuFileDriverClose, "CUfileError_t (void)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_cufile(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_cufile}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "cufile", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_cufile(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_cufile(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x03090000 + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_cufile(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'cufile' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "cufile" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_NOT_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY +__Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); +if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); +} +#endif + +__Pyx_RefNannySetupContext("PyInit_cufile", 0); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #ifdef __Pxy_PyFrame_Initialize_Offsets + __Pxy_PyFrame_Initialize_Offsets(); + #endif + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #if 0 || defined(__Pyx_CyFunction_USED) || defined(__Pyx_FusedFunction_USED) || defined(__Pyx_Coroutine_USED) || defined(__Pyx_Generator_USED) || defined(__Pyx_AsyncGen_USED) + if (__pyx_CommonTypesMetaclass_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_AsyncGen_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + if (__pyx_module_is_main_cuda__bindings___internal__cufile) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name_2, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "cuda.bindings._internal.cufile")) { + if (unlikely((PyDict_SetItemString(modules, "cuda.bindings._internal.cufile", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_function_export_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_type_init_code(__pyx_mstate); + (void)__Pyx_modinit_type_import_code(__pyx_mstate); + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + (void)__Pyx_modinit_function_import_code(__pyx_mstate); + /*--- Execution code ---*/ + + /* "cuda/bindings/_internal/cufile.pyx":8 + * + * from libc.stdint cimport intptr_t, uintptr_t + * import threading # <<<<<<<<<<<<<< + * + * from .utils import FunctionNotFoundError, NotSupportedError +*/ + __pyx_t_2 = __Pyx_ImportDottedModule(__pyx_mstate_global->__pyx_n_u_threading, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_threading, __pyx_t_2) < (0)) __PYX_ERR(0, 8, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":10 + * import threading + * + * from .utils import FunctionNotFoundError, NotSupportedError # <<<<<<<<<<<<<< + * + * from cuda.pathfinder import load_nvidia_dynamic_lib +*/ + __pyx_t_2 = __Pyx_PyList_Pack(2, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError, __pyx_mstate_global->__pyx_n_u_NotSupportedError); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_utils, __pyx_t_2, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError, __pyx_t_2) < (0)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_NotSupportedError); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_NotSupportedError, __pyx_t_2) < (0)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":12 + * from .utils import FunctionNotFoundError, NotSupportedError + * + * from cuda.pathfinder import load_nvidia_dynamic_lib # <<<<<<<<<<<<<< + * + * import cython +*/ + __pyx_t_3 = __Pyx_PyList_Pack(1, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cuda_pathfinder, __pyx_t_3, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_ImportFrom(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib, __pyx_t_3) < (0)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":40 + * ############################################################################### + * + * cdef object __symbol_lock = threading.Lock() # <<<<<<<<<<<<<< + * cdef bint __py_cufile_init = False + * cdef void* __cuDriverGetVersion = NULL +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_threading); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 40, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_Lock); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 40, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 40, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_internal_6cufile___symbol_lock); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_internal_6cufile___symbol_lock, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_2); + __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":41 + * + * cdef object __symbol_lock = threading.Lock() + * cdef bint __py_cufile_init = False # <<<<<<<<<<<<<< + * cdef void* __cuDriverGetVersion = NULL + * +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___py_cufile_init = 0; + + /* "cuda/bindings/_internal/cufile.pyx":42 + * cdef object __symbol_lock = threading.Lock() + * cdef bint __py_cufile_init = False + * cdef void* __cuDriverGetVersion = NULL # <<<<<<<<<<<<<< + * + * cdef void* __cuFileHandleRegister = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuDriverGetVersion = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":44 + * cdef void* __cuDriverGetVersion = NULL + * + * cdef void* __cuFileHandleRegister = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileHandleDeregister = NULL + * cdef void* __cuFileBufRegister = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleRegister = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":45 + * + * cdef void* __cuFileHandleRegister = NULL + * cdef void* __cuFileHandleDeregister = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileBufRegister = NULL + * cdef void* __cuFileBufDeregister = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileHandleDeregister = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":46 + * cdef void* __cuFileHandleRegister = NULL + * cdef void* __cuFileHandleDeregister = NULL + * cdef void* __cuFileBufRegister = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileBufDeregister = NULL + * cdef void* __cuFileRead = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufRegister = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":47 + * cdef void* __cuFileHandleDeregister = NULL + * cdef void* __cuFileBufRegister = NULL + * cdef void* __cuFileBufDeregister = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileRead = NULL + * cdef void* __cuFileWrite = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBufDeregister = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":48 + * cdef void* __cuFileBufRegister = NULL + * cdef void* __cuFileBufDeregister = NULL + * cdef void* __cuFileRead = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileWrite = NULL + * cdef void* __cuFileDriverOpen = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileRead = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":49 + * cdef void* __cuFileBufDeregister = NULL + * cdef void* __cuFileRead = NULL + * cdef void* __cuFileWrite = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileDriverOpen = NULL + * cdef void* __cuFileDriverClose_v2 = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWrite = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":50 + * cdef void* __cuFileRead = NULL + * cdef void* __cuFileWrite = NULL + * cdef void* __cuFileDriverOpen = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileDriverClose_v2 = NULL + * cdef void* __cuFileUseCount = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverOpen = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":51 + * cdef void* __cuFileWrite = NULL + * cdef void* __cuFileDriverOpen = NULL + * cdef void* __cuFileDriverClose_v2 = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileUseCount = NULL + * cdef void* __cuFileDriverGetProperties = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose_v2 = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":52 + * cdef void* __cuFileDriverOpen = NULL + * cdef void* __cuFileDriverClose_v2 = NULL + * cdef void* __cuFileUseCount = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileDriverGetProperties = NULL + * cdef void* __cuFileDriverSetPollMode = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileUseCount = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":53 + * cdef void* __cuFileDriverClose_v2 = NULL + * cdef void* __cuFileUseCount = NULL + * cdef void* __cuFileDriverGetProperties = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileDriverSetPollMode = NULL + * cdef void* __cuFileDriverSetMaxDirectIOSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverGetProperties = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":54 + * cdef void* __cuFileUseCount = NULL + * cdef void* __cuFileDriverGetProperties = NULL + * cdef void* __cuFileDriverSetPollMode = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileDriverSetMaxDirectIOSize = NULL + * cdef void* __cuFileDriverSetMaxCacheSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetPollMode = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":55 + * cdef void* __cuFileDriverGetProperties = NULL + * cdef void* __cuFileDriverSetPollMode = NULL + * cdef void* __cuFileDriverSetMaxDirectIOSize = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileDriverSetMaxCacheSize = NULL + * cdef void* __cuFileDriverSetMaxPinnedMemSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxDirectIOSize = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":56 + * cdef void* __cuFileDriverSetPollMode = NULL + * cdef void* __cuFileDriverSetMaxDirectIOSize = NULL + * cdef void* __cuFileDriverSetMaxCacheSize = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileDriverSetMaxPinnedMemSize = NULL + * cdef void* __cuFileBatchIOSetUp = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxCacheSize = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":57 + * cdef void* __cuFileDriverSetMaxDirectIOSize = NULL + * cdef void* __cuFileDriverSetMaxCacheSize = NULL + * cdef void* __cuFileDriverSetMaxPinnedMemSize = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileBatchIOSetUp = NULL + * cdef void* __cuFileBatchIOSubmit = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverSetMaxPinnedMemSize = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":58 + * cdef void* __cuFileDriverSetMaxCacheSize = NULL + * cdef void* __cuFileDriverSetMaxPinnedMemSize = NULL + * cdef void* __cuFileBatchIOSetUp = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileBatchIOSubmit = NULL + * cdef void* __cuFileBatchIOGetStatus = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSetUp = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":59 + * cdef void* __cuFileDriverSetMaxPinnedMemSize = NULL + * cdef void* __cuFileBatchIOSetUp = NULL + * cdef void* __cuFileBatchIOSubmit = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileBatchIOGetStatus = NULL + * cdef void* __cuFileBatchIOCancel = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOSubmit = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":60 + * cdef void* __cuFileBatchIOSetUp = NULL + * cdef void* __cuFileBatchIOSubmit = NULL + * cdef void* __cuFileBatchIOGetStatus = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileBatchIOCancel = NULL + * cdef void* __cuFileBatchIODestroy = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOGetStatus = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":61 + * cdef void* __cuFileBatchIOSubmit = NULL + * cdef void* __cuFileBatchIOGetStatus = NULL + * cdef void* __cuFileBatchIOCancel = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileBatchIODestroy = NULL + * cdef void* __cuFileReadAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIOCancel = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":62 + * cdef void* __cuFileBatchIOGetStatus = NULL + * cdef void* __cuFileBatchIOCancel = NULL + * cdef void* __cuFileBatchIODestroy = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileReadAsync = NULL + * cdef void* __cuFileWriteAsync = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileBatchIODestroy = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":63 + * cdef void* __cuFileBatchIOCancel = NULL + * cdef void* __cuFileBatchIODestroy = NULL + * cdef void* __cuFileReadAsync = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileWriteAsync = NULL + * cdef void* __cuFileStreamRegister = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileReadAsync = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":64 + * cdef void* __cuFileBatchIODestroy = NULL + * cdef void* __cuFileReadAsync = NULL + * cdef void* __cuFileWriteAsync = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileStreamRegister = NULL + * cdef void* __cuFileStreamDeregister = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileWriteAsync = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":65 + * cdef void* __cuFileReadAsync = NULL + * cdef void* __cuFileWriteAsync = NULL + * cdef void* __cuFileStreamRegister = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileStreamDeregister = NULL + * cdef void* __cuFileGetVersion = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamRegister = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":66 + * cdef void* __cuFileWriteAsync = NULL + * cdef void* __cuFileStreamRegister = NULL + * cdef void* __cuFileStreamDeregister = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileGetVersion = NULL + * cdef void* __cuFileGetParameterSizeT = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileStreamDeregister = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":67 + * cdef void* __cuFileStreamRegister = NULL + * cdef void* __cuFileStreamDeregister = NULL + * cdef void* __cuFileGetVersion = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileGetParameterSizeT = NULL + * cdef void* __cuFileGetParameterBool = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetVersion = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":68 + * cdef void* __cuFileStreamDeregister = NULL + * cdef void* __cuFileGetVersion = NULL + * cdef void* __cuFileGetParameterSizeT = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileGetParameterBool = NULL + * cdef void* __cuFileGetParameterString = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterSizeT = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":69 + * cdef void* __cuFileGetVersion = NULL + * cdef void* __cuFileGetParameterSizeT = NULL + * cdef void* __cuFileGetParameterBool = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileGetParameterString = NULL + * cdef void* __cuFileSetParameterSizeT = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterBool = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":70 + * cdef void* __cuFileGetParameterSizeT = NULL + * cdef void* __cuFileGetParameterBool = NULL + * cdef void* __cuFileGetParameterString = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileSetParameterSizeT = NULL + * cdef void* __cuFileSetParameterBool = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileGetParameterString = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":71 + * cdef void* __cuFileGetParameterBool = NULL + * cdef void* __cuFileGetParameterString = NULL + * cdef void* __cuFileSetParameterSizeT = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileSetParameterBool = NULL + * cdef void* __cuFileSetParameterString = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterSizeT = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":72 + * cdef void* __cuFileGetParameterString = NULL + * cdef void* __cuFileSetParameterSizeT = NULL + * cdef void* __cuFileSetParameterBool = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileSetParameterString = NULL + * cdef void* __cuFileDriverClose = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterBool = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":73 + * cdef void* __cuFileSetParameterSizeT = NULL + * cdef void* __cuFileSetParameterBool = NULL + * cdef void* __cuFileSetParameterString = NULL # <<<<<<<<<<<<<< + * cdef void* __cuFileDriverClose = NULL + * +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileSetParameterString = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":74 + * cdef void* __cuFileSetParameterBool = NULL + * cdef void* __cuFileSetParameterString = NULL + * cdef void* __cuFileDriverClose = NULL # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_4cuda_8bindings_9_internal_6cufile___cuFileDriverClose = NULL; + + /* "cuda/bindings/_internal/cufile.pyx":317 + * + * + * cdef dict func_ptrs = None # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_INCREF(Py_None); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_internal_6cufile_func_ptrs, ((PyObject*)Py_None)); + __Pyx_GIVEREF(Py_None); + + /* "cuda/bindings/_internal/cufile.pyx":320 + * + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_4cuda_8bindings_9_internal_6cufile_1_inspect_function_pointers, 0, __pyx_mstate_global->__pyx_n_u_inspect_function_pointers, NULL, __pyx_mstate_global->__pyx_n_u_cuda_bindings__internal_cufile, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 320, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_inspect_function_pointers, __pyx_t_2) < (0)) __PYX_ERR(0, 320, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":425 + * + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_4cuda_8bindings_9_internal_6cufile_3_inspect_function_pointer, 0, __pyx_mstate_global->__pyx_n_u_inspect_function_pointer, NULL, __pyx_mstate_global->__pyx_n_u_cuda_bindings__internal_cufile, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[1])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 425, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_inspect_function_pointer, __pyx_t_2) < (0)) __PYX_ERR(0, 425, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/cufile.pyx":1 + * # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # <<<<<<<<<<<<<< + * # + * # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +*/ + __pyx_t_2 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_2) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init cuda.bindings._internal.cufile", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init cuda.bindings._internal.cufile"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ + +typedef struct { + const char *s; +#if 179 <= 65535 + const unsigned short n; +#elif 179 / 2 < INT_MAX + const unsigned int n; +#elif 179 / 2 < LONG_MAX + const unsigned long n; +#else + const Py_ssize_t n; +#endif +#if 1 <= 31 + const unsigned int encoding : 5; +#elif 1 <= 255 + const unsigned char encoding; +#elif 1 <= 65535 + const unsigned short encoding; +#else + const Py_ssize_t encoding; +#endif + const unsigned int is_unicode : 1; + const unsigned int intern : 1; +} __Pyx_StringTabEntry; +static const char * const __pyx_string_tab_encodings[] = { 0 }; +static const __Pyx_StringTabEntry __pyx_string_tab[] = { + {__pyx_k_, sizeof(__pyx_k_), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_ */ + {__pyx_k_FunctionNotFoundError, sizeof(__pyx_k_FunctionNotFoundError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_FunctionNotFoundError */ + {__pyx_k_Lock, sizeof(__pyx_k_Lock), 0, 1, 1}, /* PyObject cname: __pyx_n_u_Lock */ + {__pyx_k_NotSupportedError, sizeof(__pyx_k_NotSupportedError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_NotSupportedError */ + {__pyx_k_Note_that_Cython_is_deliberately, sizeof(__pyx_k_Note_that_Cython_is_deliberately), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Note_that_Cython_is_deliberately */ + {__pyx_k__2, sizeof(__pyx_k__2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u__2 */ + {__pyx_k_add_note, sizeof(__pyx_k_add_note), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_add_note */ + {__pyx_k_asyncio_coroutines, sizeof(__pyx_k_asyncio_coroutines), 0, 1, 1}, /* PyObject cname: __pyx_n_u_asyncio_coroutines */ + {__pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cline_in_traceback */ + {__pyx_k_cuFileBatchIOCancel, sizeof(__pyx_k_cuFileBatchIOCancel), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileBatchIOCancel */ + {__pyx_k_cuFileBatchIODestroy, sizeof(__pyx_k_cuFileBatchIODestroy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileBatchIODestroy */ + {__pyx_k_cuFileBatchIOGetStatus, sizeof(__pyx_k_cuFileBatchIOGetStatus), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileBatchIOGetStatus */ + {__pyx_k_cuFileBatchIOSetUp, sizeof(__pyx_k_cuFileBatchIOSetUp), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileBatchIOSetUp */ + {__pyx_k_cuFileBatchIOSubmit, sizeof(__pyx_k_cuFileBatchIOSubmit), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileBatchIOSubmit */ + {__pyx_k_cuFileBufDeregister, sizeof(__pyx_k_cuFileBufDeregister), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileBufDeregister */ + {__pyx_k_cuFileBufRegister, sizeof(__pyx_k_cuFileBufRegister), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileBufRegister */ + {__pyx_k_cuFileDriverClose, sizeof(__pyx_k_cuFileDriverClose), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileDriverClose */ + {__pyx_k_cuFileDriverClose_v2, sizeof(__pyx_k_cuFileDriverClose_v2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileDriverClose_v2 */ + {__pyx_k_cuFileDriverGetProperties, sizeof(__pyx_k_cuFileDriverGetProperties), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileDriverGetProperties */ + {__pyx_k_cuFileDriverOpen, sizeof(__pyx_k_cuFileDriverOpen), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileDriverOpen */ + {__pyx_k_cuFileDriverSetMaxCacheSize, sizeof(__pyx_k_cuFileDriverSetMaxCacheSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileDriverSetMaxCacheSize */ + {__pyx_k_cuFileDriverSetMaxDirectIOSize, sizeof(__pyx_k_cuFileDriverSetMaxDirectIOSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileDriverSetMaxDirectIOSize */ + {__pyx_k_cuFileDriverSetMaxPinnedMemSiz, sizeof(__pyx_k_cuFileDriverSetMaxPinnedMemSiz), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileDriverSetMaxPinnedMemSiz */ + {__pyx_k_cuFileDriverSetPollMode, sizeof(__pyx_k_cuFileDriverSetPollMode), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileDriverSetPollMode */ + {__pyx_k_cuFileGetParameterBool, sizeof(__pyx_k_cuFileGetParameterBool), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileGetParameterBool */ + {__pyx_k_cuFileGetParameterSizeT, sizeof(__pyx_k_cuFileGetParameterSizeT), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileGetParameterSizeT */ + {__pyx_k_cuFileGetParameterString, sizeof(__pyx_k_cuFileGetParameterString), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileGetParameterString */ + {__pyx_k_cuFileGetVersion, sizeof(__pyx_k_cuFileGetVersion), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileGetVersion */ + {__pyx_k_cuFileHandleDeregister, sizeof(__pyx_k_cuFileHandleDeregister), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileHandleDeregister */ + {__pyx_k_cuFileHandleRegister, sizeof(__pyx_k_cuFileHandleRegister), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileHandleRegister */ + {__pyx_k_cuFileRead, sizeof(__pyx_k_cuFileRead), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileRead */ + {__pyx_k_cuFileReadAsync, sizeof(__pyx_k_cuFileReadAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileReadAsync */ + {__pyx_k_cuFileSetParameterBool, sizeof(__pyx_k_cuFileSetParameterBool), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileSetParameterBool */ + {__pyx_k_cuFileSetParameterSizeT, sizeof(__pyx_k_cuFileSetParameterSizeT), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileSetParameterSizeT */ + {__pyx_k_cuFileSetParameterString, sizeof(__pyx_k_cuFileSetParameterString), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileSetParameterString */ + {__pyx_k_cuFileStreamDeregister, sizeof(__pyx_k_cuFileStreamDeregister), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileStreamDeregister */ + {__pyx_k_cuFileStreamRegister, sizeof(__pyx_k_cuFileStreamRegister), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileStreamRegister */ + {__pyx_k_cuFileUseCount, sizeof(__pyx_k_cuFileUseCount), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileUseCount */ + {__pyx_k_cuFileWrite, sizeof(__pyx_k_cuFileWrite), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileWrite */ + {__pyx_k_cuFileWriteAsync, sizeof(__pyx_k_cuFileWriteAsync), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuFileWriteAsync */ + {__pyx_k_cuda_bindings__internal_cufile, sizeof(__pyx_k_cuda_bindings__internal_cufile), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuda_bindings__internal_cufile */ + {__pyx_k_cuda_bindings__internal_cufile_p, sizeof(__pyx_k_cuda_bindings__internal_cufile_p), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_cuda_bindings__internal_cufile_p */ + {__pyx_k_cuda_pathfinder, sizeof(__pyx_k_cuda_pathfinder), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuda_pathfinder */ + {__pyx_k_cufile, sizeof(__pyx_k_cufile), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cufile */ + {__pyx_k_enter, sizeof(__pyx_k_enter), 0, 1, 1}, /* PyObject cname: __pyx_n_u_enter */ + {__pyx_k_exit, sizeof(__pyx_k_exit), 0, 1, 1}, /* PyObject cname: __pyx_n_u_exit */ + {__pyx_k_func, sizeof(__pyx_k_func), 0, 1, 1}, /* PyObject cname: __pyx_n_u_func */ + {__pyx_k_function_cuFileBatchIOCancel_is, sizeof(__pyx_k_function_cuFileBatchIOCancel_is), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileBatchIOCancel_is */ + {__pyx_k_function_cuFileBatchIODestroy_is, sizeof(__pyx_k_function_cuFileBatchIODestroy_is), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileBatchIODestroy_is */ + {__pyx_k_function_cuFileBatchIOGetStatus, sizeof(__pyx_k_function_cuFileBatchIOGetStatus), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileBatchIOGetStatus */ + {__pyx_k_function_cuFileBatchIOSetUp_is_n, sizeof(__pyx_k_function_cuFileBatchIOSetUp_is_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileBatchIOSetUp_is_n */ + {__pyx_k_function_cuFileBatchIOSubmit_is, sizeof(__pyx_k_function_cuFileBatchIOSubmit_is), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileBatchIOSubmit_is */ + {__pyx_k_function_cuFileBufDeregister_is, sizeof(__pyx_k_function_cuFileBufDeregister_is), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileBufDeregister_is */ + {__pyx_k_function_cuFileBufRegister_is_no, sizeof(__pyx_k_function_cuFileBufRegister_is_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileBufRegister_is_no */ + {__pyx_k_function_cuFileDriverClose_is_no, sizeof(__pyx_k_function_cuFileDriverClose_is_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileDriverClose_is_no */ + {__pyx_k_function_cuFileDriverClose_v2_is, sizeof(__pyx_k_function_cuFileDriverClose_v2_is), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileDriverClose_v2_is */ + {__pyx_k_function_cuFileDriverGetProperti, sizeof(__pyx_k_function_cuFileDriverGetProperti), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileDriverGetProperti */ + {__pyx_k_function_cuFileDriverOpen_is_not, sizeof(__pyx_k_function_cuFileDriverOpen_is_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileDriverOpen_is_not */ + {__pyx_k_function_cuFileDriverSetMaxCache, sizeof(__pyx_k_function_cuFileDriverSetMaxCache), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileDriverSetMaxCache */ + {__pyx_k_function_cuFileDriverSetMaxDirec, sizeof(__pyx_k_function_cuFileDriverSetMaxDirec), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileDriverSetMaxDirec */ + {__pyx_k_function_cuFileDriverSetMaxPinne, sizeof(__pyx_k_function_cuFileDriverSetMaxPinne), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileDriverSetMaxPinne */ + {__pyx_k_function_cuFileDriverSetPollMode, sizeof(__pyx_k_function_cuFileDriverSetPollMode), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileDriverSetPollMode */ + {__pyx_k_function_cuFileGetParameterBool, sizeof(__pyx_k_function_cuFileGetParameterBool), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileGetParameterBool */ + {__pyx_k_function_cuFileGetParameterSizeT, sizeof(__pyx_k_function_cuFileGetParameterSizeT), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileGetParameterSizeT */ + {__pyx_k_function_cuFileGetParameterStrin, sizeof(__pyx_k_function_cuFileGetParameterStrin), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileGetParameterStrin */ + {__pyx_k_function_cuFileGetVersion_is_not, sizeof(__pyx_k_function_cuFileGetVersion_is_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileGetVersion_is_not */ + {__pyx_k_function_cuFileHandleDeregister, sizeof(__pyx_k_function_cuFileHandleDeregister), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileHandleDeregister */ + {__pyx_k_function_cuFileHandleRegister_is, sizeof(__pyx_k_function_cuFileHandleRegister_is), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileHandleRegister_is */ + {__pyx_k_function_cuFileReadAsync_is_not, sizeof(__pyx_k_function_cuFileReadAsync_is_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileReadAsync_is_not */ + {__pyx_k_function_cuFileRead_is_not_found, sizeof(__pyx_k_function_cuFileRead_is_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileRead_is_not_found */ + {__pyx_k_function_cuFileSetParameterBool, sizeof(__pyx_k_function_cuFileSetParameterBool), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileSetParameterBool */ + {__pyx_k_function_cuFileSetParameterSizeT, sizeof(__pyx_k_function_cuFileSetParameterSizeT), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileSetParameterSizeT */ + {__pyx_k_function_cuFileSetParameterStrin, sizeof(__pyx_k_function_cuFileSetParameterStrin), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileSetParameterStrin */ + {__pyx_k_function_cuFileStreamDeregister, sizeof(__pyx_k_function_cuFileStreamDeregister), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileStreamDeregister */ + {__pyx_k_function_cuFileStreamRegister_is, sizeof(__pyx_k_function_cuFileStreamRegister_is), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileStreamRegister_is */ + {__pyx_k_function_cuFileUseCount_is_not_f, sizeof(__pyx_k_function_cuFileUseCount_is_not_f), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileUseCount_is_not_f */ + {__pyx_k_function_cuFileWriteAsync_is_not, sizeof(__pyx_k_function_cuFileWriteAsync_is_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileWriteAsync_is_not */ + {__pyx_k_function_cuFileWrite_is_not_foun, sizeof(__pyx_k_function_cuFileWrite_is_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_cuFileWrite_is_not_foun */ + {__pyx_k_handle_uint, sizeof(__pyx_k_handle_uint), 0, 1, 1}, /* PyObject cname: __pyx_n_u_handle_uint */ + {__pyx_k_initializing, sizeof(__pyx_k_initializing), 0, 1, 1}, /* PyObject cname: __pyx_n_u_initializing */ + {__pyx_k_inspect_function_pointer, sizeof(__pyx_k_inspect_function_pointer), 0, 1, 1}, /* PyObject cname: __pyx_n_u_inspect_function_pointer */ + {__pyx_k_inspect_function_pointers, sizeof(__pyx_k_inspect_function_pointers), 0, 1, 1}, /* PyObject cname: __pyx_n_u_inspect_function_pointers */ + {__pyx_k_is_coroutine, sizeof(__pyx_k_is_coroutine), 0, 1, 1}, /* PyObject cname: __pyx_n_u_is_coroutine */ + {__pyx_k_load_nvidia_dynamic_lib, sizeof(__pyx_k_load_nvidia_dynamic_lib), 0, 1, 1}, /* PyObject cname: __pyx_n_u_load_nvidia_dynamic_lib */ + {__pyx_k_main, sizeof(__pyx_k_main), 0, 1, 1}, /* PyObject cname: __pyx_n_u_main */ + {__pyx_k_module, sizeof(__pyx_k_module), 0, 1, 1}, /* PyObject cname: __pyx_n_u_module */ + {__pyx_k_name, sizeof(__pyx_k_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name */ + {__pyx_k_name_2, sizeof(__pyx_k_name_2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name_2 */ + {__pyx_k_pop, sizeof(__pyx_k_pop), 0, 1, 1}, /* PyObject cname: __pyx_n_u_pop */ + {__pyx_k_qualname, sizeof(__pyx_k_qualname), 0, 1, 1}, /* PyObject cname: __pyx_n_u_qualname */ + {__pyx_k_set_name, sizeof(__pyx_k_set_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_set_name */ + {__pyx_k_spec, sizeof(__pyx_k_spec), 0, 1, 1}, /* PyObject cname: __pyx_n_u_spec */ + {__pyx_k_test, sizeof(__pyx_k_test), 0, 1, 1}, /* PyObject cname: __pyx_n_u_test */ + {__pyx_k_threading, sizeof(__pyx_k_threading), 0, 1, 1}, /* PyObject cname: __pyx_n_u_threading */ + {__pyx_k_utils, sizeof(__pyx_k_utils), 0, 1, 1}, /* PyObject cname: __pyx_n_u_utils */ + {0, 0, 0, 0, 0} +}; +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names); + +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + return 0; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "cuda/bindings/_internal/cufile.pyx":87 + * cdef void* handle = NULL + * + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * # Load function + * global __cuFileHandleRegister +*/ + __pyx_mstate_global->__pyx_tuple[0] = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_tuple[0])) __PYX_ERR(0, 87, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.method_name = &__pyx_mstate->__pyx_n_u_pop; + if (__Pyx_InitStrings(__pyx_string_tab, __pyx_mstate->__pyx_string_tab, __pyx_string_tab_encodings) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ +\ + typedef struct { + unsigned int argcount : 1; + unsigned int num_posonly_args : 1; + unsigned int num_kwonly_args : 1; + unsigned int nlocals : 1; + unsigned int flags : 10; + unsigned int first_line : 9; + unsigned int line_table_length : 14; + } __Pyx_PyCode_New_function_description; +/* NewCodeObj.proto */ +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + const char *line_table, + PyObject *tuple_dedup_map +); + + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + PyObject* tuple_dedup_map = PyDict_New(); + if (unlikely(!tuple_dedup_map)) return -1; + { + const __Pyx_PyCode_New_function_description descr = {0, 0, 0, 0, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 320, 504}; + PyObject* const varnames[] = {0}; + __pyx_mstate_global->__pyx_codeobj_tab[0] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_cuda_bindings__internal_cufile_p, __pyx_mstate->__pyx_n_u_inspect_function_pointers, __pyx_k_z_q_Q_Zq_z_A_Ja_Q_Ja_1_Zq_z_A_z, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[0])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 1, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 425, 31}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_name}; + __pyx_mstate_global->__pyx_codeobj_tab[1] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_cuda_bindings__internal_cufile_p, __pyx_mstate->__pyx_n_u_inspect_function_pointer, __pyx_k_z_A_a_9AQ, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[1])) goto bad; + } + Py_DECREF(tuple_dedup_map); + return 0; + bad: + Py_DECREF(tuple_dedup_map); + return -1; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CachedMethodType.init */ + #if CYTHON_COMPILING_IN_LIMITED_API +{ + PyObject *typesModule=NULL; + typesModule = PyImport_ImportModule("types"); + if (typesModule) { + __pyx_mstate_global->__Pyx_CachedMethodType = PyObject_GetAttrString(typesModule, "MethodType"); + Py_DECREF(typesModule); + } +} // error handling follows +#endif + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* PyErrExceptionMatches */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStr */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* PyObjectGetAttrStrNoError */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name); + if (unlikely(!result) && !PyErr_Occurred()) { + PyErr_Format(PyExc_NameError, + "name '%U' is not defined", name); + } + return result; +} + +/* PyDictVersioning */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!__pyx_m)) { + if (!PyErr_Occurred()) + PyErr_SetNone(PyExc_NameError); + return NULL; + } + result = PyObject_GetAttr(__pyx_m, name); + if (likely(result)) { + return result; + } + PyErr_Clear(); +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear(); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return result; + } +#else + result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject *const *args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = __Pyx_PyFrame_GetLocalsplus(f); + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; + PyObject *kwdefs; + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) { + return NULL; + } + if ( + co->co_kwonlyargcount == 0 && + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, (int)nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif + +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = Py_TYPE(func)->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + #if PY_VERSION_HEX < 0x030800B1 + #if CYTHON_FAST_PYCCALL + if (PyCFunction_Check(func)) { + if (kwargs) { + return _PyCFunction_FastCallDict(func, args, nargs, kwargs); + } else { + return _PyCFunction_FastCallKeywords(func, args, nargs, NULL); + } + } + if (!kwargs && __Pyx_IS_TYPE(func, &PyMethodDescr_Type)) { + return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL); + } + #endif + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs); + } + #endif + #endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #elif defined(__Pyx_CyFunction_USED) && CYTHON_BACKPORT_VECTORCALL + if (__Pyx_CyFunction_CheckExact(func)) { + __pyx_vectorcallfunc f = __Pyx_CyFunction_func_vectorcall(func); + if (f) return f(func, args, _nargs, NULL); + } + #elif CYTHON_COMPILING_IN_LIMITED_API && CYTHON_VECTORCALL + return PyObject_Vectorcall(func, args, _nargs, NULL); + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* PyObjectLookupSpecial */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) { + PyObject *res; + PyTypeObject *tp = Py_TYPE(obj); + res = _PyType_Lookup(tp, attr_name); + if (likely(res)) { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (!f) { + Py_INCREF(res); + } else { + res = f(res, obj, (PyObject *)tp); + } + } else if (with_error) { + PyErr_SetObject(PyExc_AttributeError, attr_name); + } + return res; +} +#endif + +/* GetTopmostException */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + +/* SaveResetException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #endif +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); + #else + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); + #endif +} +#endif + +/* GetException */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) +#endif +{ + PyObject *local_type = NULL, *local_value, *local_tb = NULL; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030C0000 + local_value = tstate->current_exception; + tstate->current_exception = 0; + #else + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + #endif +#elif __PYX_LIMITED_VERSION_HEX > 0x030C0000 + local_value = PyErr_GetRaisedException(); +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif +#if __PYX_LIMITED_VERSION_HEX > 0x030C0000 + if (likely(local_value)) { + local_type = (PyObject*) Py_TYPE(local_value); + Py_INCREF(local_type); + local_tb = PyException_GetTraceback(local_value); + } +#else + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } +#endif // __PYX_LIMITED_VERSION_HEX > 0x030C0000 + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if CYTHON_USE_EXC_INFO_STACK + { + _PyErr_StackItem *exc_info = tstate->exc_info; + #if PY_VERSION_HEX >= 0x030B00a4 + tmp_value = exc_info->exc_value; + exc_info->exc_value = local_value; + tmp_type = NULL; + tmp_tb = NULL; + Py_XDECREF(local_type); + Py_XDECREF(local_tb); + #else + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; + #endif + } + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#elif __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + PyErr_SetHandledException(local_value); + Py_XDECREF(local_value); + Py_XDECREF(local_type); + Py_XDECREF(local_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +#if __PYX_LIMITED_VERSION_HEX <= 0x030C0000 +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +#endif +} + +/* TupleAndListFromArray */ +#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + Py_ssize_t i; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + for (i = 0; i < n; i++) { + if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < (0))) { + Py_DECREF(res); + return NULL; + } + Py_INCREF(src[i]); + } + return res; +} +#elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + +/* BytesEquals */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL ||\ + !(CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) + return PyObject_RichCompareBool(s1, s2, equals); +#else + if (s1 == s2) { + return (equals == Py_EQ); + } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { + const char *ps1, *ps2; + Py_ssize_t length = PyBytes_GET_SIZE(s1); + if (length != PyBytes_GET_SIZE(s2)) + return (equals == Py_NE); + ps1 = PyBytes_AS_STRING(s1); + ps2 = PyBytes_AS_STRING(s2); + if (ps1[0] != ps2[0]) { + return (equals == Py_NE); + } else if (length == 1) { + return (equals == Py_EQ); + } else { + int result; +#if CYTHON_USE_UNICODE_INTERNALS && (PY_VERSION_HEX < 0x030B0000) + Py_hash_t hash1, hash2; + hash1 = ((PyBytesObject*)s1)->ob_shash; + hash2 = ((PyBytesObject*)s2)->ob_shash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + return (equals == Py_NE); + } +#endif + result = memcmp(ps1, ps2, (size_t)length); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { + return (equals == Py_NE); + } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) { + return (equals == Py_NE); + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +#endif +} + +/* UnicodeEquals */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL + return PyObject_RichCompareBool(s1, s2, equals); +#else + int s1_is_unicode, s2_is_unicode; + if (s1 == s2) { + goto return_eq; + } + s1_is_unicode = PyUnicode_CheckExact(s1); + s2_is_unicode = PyUnicode_CheckExact(s2); + if (s1_is_unicode & s2_is_unicode) { + Py_ssize_t length, length2; + int kind; + void *data1, *data2; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0)) + return -1; + #endif + length = __Pyx_PyUnicode_GET_LENGTH(s1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length < 0)) return -1; + #endif + length2 = __Pyx_PyUnicode_GET_LENGTH(s2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length2 < 0)) return -1; + #endif + if (length != length2) { + goto return_ne; + } +#if CYTHON_USE_UNICODE_INTERNALS + { + Py_hash_t hash1, hash2; + hash1 = ((PyASCIIObject*)s1)->hash; + hash2 = ((PyASCIIObject*)s2)->hash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + goto return_ne; + } + } +#endif + kind = __Pyx_PyUnicode_KIND(s1); + if (kind != __Pyx_PyUnicode_KIND(s2)) { + goto return_ne; + } + data1 = __Pyx_PyUnicode_DATA(s1); + data2 = __Pyx_PyUnicode_DATA(s2); + if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) { + goto return_ne; + } else if (length == 1) { + goto return_eq; + } else { + int result = memcmp(data1, data2, (size_t)(length * kind)); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & s2_is_unicode) { + goto return_ne; + } else if ((s2 == Py_None) & s1_is_unicode) { + goto return_ne; + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +return_eq: + return (equals == Py_EQ); +return_ne: + return (equals == Py_NE); +#endif +} + +/* fastcall */ +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + Py_ssize_t i, n = __Pyx_PyTuple_GET_SIZE(kwnames); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(n == -1)) return NULL; + #endif + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + if (s == namei) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + int eq = __Pyx_PyUnicode_Equals(s, namei, Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; + return kwvalues[i]; + } + } + return NULL; +} +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues) { + Py_ssize_t i, nkwargs; + PyObject *dict; +#if !CYTHON_ASSUME_SAFE_SIZE + nkwargs = PyTuple_Size(kwnames); + if (unlikely(nkwargs < 0)) return NULL; +#else + nkwargs = PyTuple_GET_SIZE(kwnames); +#endif + dict = PyDict_New(); + if (unlikely(!dict)) + return NULL; + for (i=0; itype, *target->method_name); + if (unlikely(!method)) + return -1; + result = method; +#if CYTHON_COMPILING_IN_CPYTHON + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + { + PyMethodDescrObject *descr = (PyMethodDescrObject*) method; + target->func = descr->d_method->ml_meth; + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); + } else +#endif +#if CYTHON_COMPILING_IN_PYPY +#else + if (PyCFunction_Check(method)) +#endif + { + PyObject *self; + int self_found; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + self = PyObject_GetAttrString(method, "__self__"); + if (!self) { + PyErr_Clear(); + } +#else + self = PyCFunction_GET_SELF(method); +#endif + self_found = (self && self != Py_None); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + Py_XDECREF(self); +#endif + if (self_found) { + PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method); + if (unlikely(!unbound_method)) return -1; + Py_DECREF(method); + result = unbound_method; + } + } +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + if (unlikely(target->method)) { + Py_DECREF(result); + } else +#endif + target->method = result; + return 0; +} + +/* CallUnboundCMethod2 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + PyObject *args[2] = {arg1, arg2}; + if (cfunc->flag == METH_FASTCALL) { + return __Pyx_CallCFunctionFast(cfunc, self, args, 2); + } + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, 2, NULL); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod2(&tmp_cfunc, self, arg1, arg2); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){ + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + PyObject *result = NULL; + PyObject *args = PyTuple_New(2); + if (unlikely(!args)) return NULL; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + if (cfunc->flag & METH_KEYWORDS) + result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL); + else + result = __Pyx_CallCFunction(cfunc, self, args); + Py_DECREF(args); + return result; + } +#endif + { + PyObject *args[4] = {NULL, self, arg1, arg2}; + return __Pyx_PyObject_FastCall(cfunc->method, args+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } +} + +/* ParseKeywords */ +static int __Pyx_ValidateDuplicatePosArgs( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char* function_name) +{ + PyObject ** const *name = argnames; + while (name != first_kw_arg) { + PyObject *key = **name; + int found = PyDict_Contains(kwds, key); + if (unlikely(found)) { + if (found == 1) __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; + } + name++; + } + return 0; +bad: + return -1; +} +#if CYTHON_USE_UNICODE_INTERNALS +static CYTHON_INLINE int __Pyx_UnicodeKeywordsEqual(PyObject *s1, PyObject *s2) { + int kind; + Py_ssize_t len = PyUnicode_GET_LENGTH(s1); + if (len != PyUnicode_GET_LENGTH(s2)) return 0; + kind = PyUnicode_KIND(s1); + if (kind != PyUnicode_KIND(s2)) return 0; + const void *data1 = PyUnicode_DATA(s1); + const void *data2 = PyUnicode_DATA(s2); + return (memcmp(data1, data2, (size_t) len * (size_t) kind) == 0); +} +#endif +static int __Pyx_MatchKeywordArg_str( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + #if CYTHON_USE_UNICODE_INTERNALS + Py_hash_t key_hash = ((PyASCIIObject*)key)->hash; + if (unlikely(key_hash == -1)) { + key_hash = PyObject_Hash(key); + if (unlikely(key_hash == -1)) + goto bad; + } + #endif + name = first_kw_arg; + while (*name) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (key_hash == ((PyASCIIObject*)name_str)->hash && __Pyx_UnicodeKeywordsEqual(name_str, key)) { + *index_found = (size_t) (name - argnames); + return 1; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + *index_found = (size_t) (name - argnames); + return 1; + } + } + #endif + name++; + } + name = argnames; + while (name != first_kw_arg) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (unlikely(key_hash == ((PyASCIIObject*)name_str)->hash)) { + if (__Pyx_UnicodeKeywordsEqual(name_str, key)) + goto arg_passed_twice; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + if (unlikely(name_str == key)) goto arg_passed_twice; + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + } + #endif + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +bad: + return -1; +} +static int __Pyx_MatchKeywordArg_nostr( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; + name = first_kw_arg; + while (*name) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (cmp == 1) { + *index_found = (size_t) (name - argnames); + return 1; + } + if (unlikely(cmp == -1)) goto bad; + name++; + } + name = argnames; + while (name != first_kw_arg) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (unlikely(cmp != 0)) { + if (cmp == 1) goto arg_passed_twice; + else goto bad; + } + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +bad: + return -1; +} +static CYTHON_INLINE int __Pyx_MatchKeywordArg( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + return likely(PyUnicode_CheckExact(key)) ? + __Pyx_MatchKeywordArg_str(key, argnames, first_kw_arg, index_found, function_name) : + __Pyx_MatchKeywordArg_nostr(key, argnames, first_kw_arg, index_found, function_name); +} +static void __Pyx_RejectUnknownKeyword( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char *function_name) +{ + Py_ssize_t pos = 0; + PyObject *key = NULL; + __Pyx_BEGIN_CRITICAL_SECTION(kwds); + while (PyDict_Next(kwds, &pos, &key, NULL)) { + PyObject** const *name = first_kw_arg; + while (*name && (**name != key)) name++; + if (!*name) { + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(key); + #endif + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp != 1) { + if (cmp == 0) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + break; + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + } + } + __Pyx_END_CRITICAL_SECTION(); + assert(PyErr_Occurred()); +} +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t extracted = 0; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + name = first_kw_arg; + while (*name && num_kwargs > extracted) { + PyObject * key = **name; + PyObject *value; + int found = 0; + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + found = PyDict_GetItemRef(kwds, key, &value); + #else + value = PyDict_GetItemWithError(kwds, key); + if (value) { + Py_INCREF(value); + found = 1; + } else { + if (unlikely(PyErr_Occurred())) goto bad; + } + #endif + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + extracted++; + } + name++; + } + if (num_kwargs > extracted) { + if (ignore_unknown_kwargs) { + if (unlikely(__Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name) == -1)) + goto bad; + } else { + __Pyx_RejectUnknownKeyword(kwds, argnames, first_kw_arg, function_name); + goto bad; + } + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t len; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + if (PyDict_Update(kwds2, kwds) < 0) goto bad; + name = first_kw_arg; + while (*name) { + PyObject *key = **name; + PyObject *value; +#if !CYTHON_COMPILING_IN_LIMITED_API && (PY_VERSION_HEX >= 0x030d00A2 || defined(PyDict_Pop)) + int found = PyDict_Pop(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int found = PyDict_GetItemRef(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + if (unlikely(PyDict_DelItem(kwds2, key) < 0)) goto bad; + } +#else + #if CYTHON_COMPILING_IN_CPYTHON + value = _PyDict_Pop(kwds2, key, kwds2); + #else + value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_pop, kwds2, key, kwds2); + #endif + if (value == kwds2) { + Py_DECREF(value); + } else { + if (unlikely(!value)) goto bad; + values[name-argnames] = value; + } +#endif + name++; + } + len = PyDict_Size(kwds2); + if (len > 0) { + return __Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name); + } else if (unlikely(len == -1)) { + goto bad; + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject *key = NULL; + PyObject** const * name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + for (Py_ssize_t pos = 0; pos < num_kwargs; pos++) { +#if CYTHON_AVOID_BORROWED_REFS + key = __Pyx_PySequence_ITEM(kwds, pos); +#else + key = __Pyx_PyTuple_GET_ITEM(kwds, pos); +#endif +#if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!key)) goto bad; +#endif + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + PyObject *value = kwvalues[pos]; + values[name-argnames] = __Pyx_NewRef(value); + } else { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp == 1) { + PyObject *value = kwvalues[pos]; + values[index_found] = __Pyx_NewRef(value); + } else { + if (unlikely(cmp == -1)) goto bad; + if (kwds2) { + PyObject *value = kwvalues[pos]; + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else if (!ignore_unknown_kwargs) { + goto invalid_keyword; + } + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + key = NULL; + #endif + } + return 0; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + goto bad; +bad: + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(key); + #endif + return -1; +} +static int __Pyx_ParseKeywords( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) + return __Pyx_ParseKeywordsTuple(kwds, kwvalues, argnames, kwds2, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); + else if (kwds2) + return __Pyx_ParseKeywordDictToDict(kwds, argnames, kwds2, values, num_pos_args, function_name); + else + return __Pyx_ParseKeywordDict(kwds, argnames, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* ArgTypeTest */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +{ + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; + PyObject *extra_info = __pyx_mstate_global->__pyx_empty_unicode; + int from_annotation_subclass = 0; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (!exact) { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } else if (exact == 2) { + if (__Pyx_TypeCheck(obj, type)) { + from_annotation_subclass = 1; + extra_info = __pyx_mstate_global->__pyx_kp_u_Note_that_Cython_is_deliberately; + } + } + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")" +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + "%s%U" +#endif + , name, type_name, obj_type_name +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + , (from_annotation_subclass ? ". " : ""), extra_info +#endif + ); +#if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + if (exact == 2 && from_annotation_subclass) { + PyObject *res; + PyObject *vargs[2]; + vargs[0] = PyErr_GetRaisedException(); + vargs[1] = extra_info; + res = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_kp_u_add_note, vargs, 2, NULL); + Py_XDECREF(res); + PyErr_SetRaisedException(vargs[0]); + } +#endif + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* RaiseException */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyException_SetTraceback(value, tb); +#elif CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} + +/* Import */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { + PyObject *module = 0; + PyObject *empty_dict = 0; + PyObject *empty_list = 0; + empty_dict = PyDict_New(); + if (unlikely(!empty_dict)) + goto bad; + if (level == -1) { + const char* package_sep = strchr(__Pyx_MODULE_NAME, '.'); + if (package_sep != (0)) { + module = PyImport_ImportModuleLevelObject( + name, __pyx_mstate_global->__pyx_d, empty_dict, from_list, 1); + if (unlikely(!module)) { + if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError))) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } + if (!module) { + module = PyImport_ImportModuleLevelObject( + name, __pyx_mstate_global->__pyx_d, empty_dict, from_list, level); + } +bad: + Py_XDECREF(empty_dict); + Py_XDECREF(empty_list); + return module; +} + +/* ImportDottedModule */ +static PyObject *__Pyx__ImportDottedModule_Error(PyObject *name, PyObject *parts_tuple, Py_ssize_t count) { + PyObject *partial_name = NULL, *slice = NULL, *sep = NULL; + Py_ssize_t size; + if (unlikely(PyErr_Occurred())) { + PyErr_Clear(); + } +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(parts_tuple); +#else + size = PyTuple_Size(parts_tuple); + if (size < 0) goto bad; +#endif + if (likely(size == count)) { + partial_name = name; + } else { + slice = PySequence_GetSlice(parts_tuple, 0, count); + if (unlikely(!slice)) + goto bad; + sep = PyUnicode_FromStringAndSize(".", 1); + if (unlikely(!sep)) + goto bad; + partial_name = PyUnicode_Join(sep, slice); + } + PyErr_Format( + PyExc_ModuleNotFoundError, + "No module named '%U'", partial_name); +bad: + Py_XDECREF(sep); + Py_XDECREF(slice); + Py_XDECREF(partial_name); + return NULL; +} +static PyObject *__Pyx__ImportDottedModule_Lookup(PyObject *name) { + PyObject *imported_module; +#if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + return NULL; + imported_module = __Pyx_PyDict_GetItemStr(modules, name); + Py_XINCREF(imported_module); +#else + imported_module = PyImport_GetModule(name); +#endif + return imported_module; +} +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple) { + Py_ssize_t i, nparts; +#if CYTHON_ASSUME_SAFE_SIZE + nparts = PyTuple_GET_SIZE(parts_tuple); +#else + nparts = PyTuple_Size(parts_tuple); + if (nparts < 0) return NULL; +#endif + for (i=1; i < nparts && module; i++) { + PyObject *part, *submodule; +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + part = PyTuple_GET_ITEM(parts_tuple, i); +#else + part = __Pyx_PySequence_ITEM(parts_tuple, i); + if (!part) return NULL; +#endif + submodule = __Pyx_PyObject_GetAttrStrNoError(module, part); +#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(part); +#endif + Py_DECREF(module); + module = submodule; + } + if (unlikely(!module)) { + return __Pyx__ImportDottedModule_Error(name, parts_tuple, i); + } + return module; +} +static PyObject *__Pyx__ImportDottedModule(PyObject *name, PyObject *parts_tuple) { + PyObject *imported_module; + PyObject *module = __Pyx_Import(name, NULL, 0); + if (!parts_tuple || unlikely(!module)) + return module; + imported_module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(imported_module)) { + Py_DECREF(module); + return imported_module; + } + PyErr_Clear(); + return __Pyx_ImportDottedModule_WalkParts(module, name, parts_tuple); +} +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple) { +#if CYTHON_COMPILING_IN_CPYTHON + PyObject *module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(module)) { + PyObject *spec = __Pyx_PyObject_GetAttrStrNoError(module, __pyx_mstate_global->__pyx_n_u_spec); + if (likely(spec)) { + PyObject *unsafe = __Pyx_PyObject_GetAttrStrNoError(spec, __pyx_mstate_global->__pyx_n_u_initializing); + if (likely(!unsafe || !__Pyx_PyObject_IsTrue(unsafe))) { + Py_DECREF(spec); + spec = NULL; + } + Py_XDECREF(unsafe); + } + if (likely(!spec)) { + PyErr_Clear(); + return module; + } + Py_DECREF(spec); + Py_DECREF(module); + } else if (PyErr_Occurred()) { + PyErr_Clear(); + } +#endif + return __Pyx__ImportDottedModule(name, parts_tuple); +} + +/* ListPack */ +static PyObject *__Pyx_PyList_Pack(Py_ssize_t n, ...) { + va_list va; + PyObject *l = PyList_New(n); + va_start(va, n); + if (unlikely(!l)) goto end; + for (Py_ssize_t i=0; i__pyx_kp_u_); + if (unlikely(!module_dot)) { goto modbad; } + full_name = PyUnicode_Concat(module_dot, name); + if (unlikely(!full_name)) { goto modbad; } + #if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + { + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + goto modbad; + value = PyObject_GetItem(modules, full_name); + } + #else + value = PyImport_GetModule(full_name); + #endif + modbad: + Py_XDECREF(full_name); + Py_XDECREF(module_dot); + Py_XDECREF(module_name); + } + if (unlikely(!value)) { + PyErr_Format(PyExc_ImportError, "cannot import name %S", name); + } + return value; +} + +/* LimitedApiGetTypeDict */ +#if CYTHON_COMPILING_IN_LIMITED_API +static Py_ssize_t __Pyx_GetTypeDictOffset(void) { + PyObject *tp_dictoffset_o; + Py_ssize_t tp_dictoffset; + tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__"); + if (unlikely(!tp_dictoffset_o)) return -1; + tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o); + Py_DECREF(tp_dictoffset_o); + if (unlikely(tp_dictoffset == 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' doesn't have a dictoffset"); + return -1; + } else if (unlikely(tp_dictoffset < 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' has an unexpected negative dictoffset. " + "Please report this as Cython bug"); + return -1; + } + return tp_dictoffset; +} +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) { + static Py_ssize_t tp_dictoffset = 0; + if (unlikely(tp_dictoffset == 0)) { + tp_dictoffset = __Pyx_GetTypeDictOffset(); + if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) { + tp_dictoffset = 0; // try again next time? + return NULL; + } + } + return *(PyObject**)((char*)tp + tp_dictoffset); +} +#endif + +/* SetItemOnTypeDict */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_SetItem(tp_dict, k, v); + if (likely(!result)) { + PyType_Modified(tp); + if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) { + PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name, (PyObject *) tp, k, NULL); + if (!setNameResult) return -1; + Py_DECREF(setNameResult); + } + } + return result; +} + +/* FixUpExtensionType */ +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if __PYX_LIMITED_VERSION_HEX > 0x030900B1 + CYTHON_UNUSED_VAR(spec); + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#else + const PyType_Slot *slot = spec->slots; + int changed = 0; +#if !CYTHON_COMPILING_IN_LIMITED_API + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { +#if !CYTHON_COMPILING_IN_CPYTHON + const +#endif // !CYTHON_COMPILING_IN_CPYTHON) + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); +#if PY_VERSION_HEX >= 0x030800b4 + type->tp_vectorcall_offset = memb->offset; +#else + type->tp_print = (printfunc) memb->offset; +#endif + changed = 1; + } +#endif // CYTHON_METH_FASTCALL +#if !CYTHON_COMPILING_IN_PYPY + else if (strcmp(memb->name, "__module__") == 0) { + PyObject *descr; + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } +#endif // !CYTHON_COMPILING_IN_PYPY + } + memb++; + } + } +#endif // !CYTHON_COMPILING_IN_LIMITED_API +#if !CYTHON_COMPILING_IN_PYPY + slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_getset) + slot++; + if (slot && slot->slot == Py_tp_getset) { + PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc; + while (getset && getset->name) { + if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) { + PyObject *descr = PyDescr_NewGetSet(type, getset); + if (unlikely(!descr)) + return -1; + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *pyname = PyUnicode_FromString(getset->name); + if (unlikely(!pyname)) { + Py_DECREF(descr); + return -1; + } + int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr); + Py_DECREF(pyname); + #else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + #endif + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } + ++getset; + } + } +#endif // !CYTHON_COMPILING_IN_PYPY + if (changed) + PyType_Modified(type); +#endif // PY_VERSION_HEX > 0x030900B1 + return 0; +} + +/* FetchSharedCythonModule */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + return __Pyx_PyImport_AddModuleRef(__PYX_ABI_MODULE_NAME); +} + +/* dict_setdefault */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, + int is_safe_type) { + PyObject* value; + CYTHON_MAYBE_UNUSED_VAR(is_safe_type); +#if CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_CallMethod(d, "setdefault", "OO", key, default_value); +#elif PY_VERSION_HEX >= 0x030d0000 + PyDict_SetDefaultRef(d, key, default_value, &value); +#else + value = PyDict_SetDefault(d, key, default_value); + if (unlikely(!value)) return NULL; + Py_INCREF(value); +#endif + return value; +} + +/* FetchCommonType */ +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject* __Pyx_PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *result = __Pyx_PyType_FromModuleAndSpec(module, spec, bases); + if (result && metaclass) { + PyObject *old_tp = (PyObject*)Py_TYPE(result); + Py_INCREF((PyObject*)metaclass); +#if __PYX_LIMITED_VERSION_HEX >= 0x03090000 + Py_SET_TYPE(result, metaclass); +#else + result->ob_type = metaclass; +#endif + Py_DECREF(old_tp); + } + return result; +} +#else +#define __Pyx_PyType_FromMetaclass(me, mo, s, b) PyType_FromMetaclass(me, mo, s, b) +#endif +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t expected_basicsize) { + Py_ssize_t basicsize; + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (expected_basicsize == 0) { + return 0; // size is inherited, nothing useful to check + } +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) return -1; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = NULL; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; +#else + basicsize = ((PyTypeObject*) cached_type)->tp_basicsize; +#endif + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module = NULL, *cached_type = NULL, *abi_module_dict, *new_cached_type, *py_object_name; + int get_item_ref_result; + const char* object_name = strrchr(spec->name, '.'); + object_name = object_name ? object_name+1 : spec->name; + py_object_name = PyUnicode_FromString(object_name); + if (!py_object_name) return NULL; + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) goto done; + abi_module_dict = PyModule_GetDict(abi_module); + if (!abi_module_dict) goto done; + get_item_ref_result = __Pyx_PyDict_GetItemRef(abi_module_dict, py_object_name, &cached_type); + if (get_item_ref_result == 1) { + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else if (unlikely(get_item_ref_result == -1)) { + goto bad; + } + CYTHON_UNUSED_VAR(module); + cached_type = __Pyx_PyType_FromMetaclass(metaclass, abi_module, spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + new_cached_type = __Pyx_PyDict_SetDefault(abi_module_dict, py_object_name, cached_type, 1); + if (unlikely(new_cached_type != cached_type)) { + if (unlikely(!new_cached_type)) goto bad; + Py_DECREF(cached_type); + cached_type = new_cached_type; + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else { + Py_DECREF(new_cached_type); + } +done: + Py_XDECREF(abi_module); + Py_DECREF(py_object_name); + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* CommonTypesMetaclass */ +static PyObject* __pyx_CommonTypesMetaclass_get_module(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED void* context) { + return PyUnicode_FromString(__PYX_ABI_MODULE_NAME); +} +static PyGetSetDef __pyx_CommonTypesMetaclass_getset[] = { + {"__module__", __pyx_CommonTypesMetaclass_get_module, NULL, NULL, NULL}, + {0, 0, 0, 0, 0} +}; +static PyType_Slot __pyx_CommonTypesMetaclass_slots[] = { + {Py_tp_getset, (void *)__pyx_CommonTypesMetaclass_getset}, + {0, 0} +}; +static PyType_Spec __pyx_CommonTypesMetaclass_spec = { + __PYX_TYPE_MODULE_PREFIX "_common_types_metatype", + 0, + 0, +#if PY_VERSION_HEX >= 0x030A0000 + Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | +#endif + Py_TPFLAGS_DEFAULT, + __pyx_CommonTypesMetaclass_slots +}; +static int __pyx_CommonTypesMetaclass_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + PyObject *bases = PyTuple_Pack(1, &PyType_Type); + if (unlikely(!bases)) { + return -1; + } + mstate->__pyx_CommonTypesMetaclassType = __Pyx_FetchCommonTypeFromSpec(NULL, module, &__pyx_CommonTypesMetaclass_spec, bases); + Py_DECREF(bases); + if (unlikely(mstate->__pyx_CommonTypesMetaclassType == NULL)) { + return -1; + } + return 0; +} + +/* CallTypeTraverse */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) { + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 + if (__Pyx_get_runtime_version() < 0x03090000) return 0; + #endif + if (!always_call) { + PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*); + unsigned long flags = PyType_GetFlags(base); + if (flags & Py_TPFLAGS_HEAPTYPE) { + return 0; + } + } + Py_VISIT((PyObject*)Py_TYPE(o)); + return 0; +} +#endif + +/* PyMethodNew */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + PyObject *result; + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + #if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + { + PyObject *args[] = {func, self}; + result = PyObject_Vectorcall(__pyx_mstate_global->__Pyx_CachedMethodType, args, 2, NULL); + } + #else + result = PyObject_CallFunctionObjArgs(__pyx_mstate_global->__Pyx_CachedMethodType, func, self, NULL); + #endif + return result; +} +#else +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#endif + +/* PyVectorcallFastCallDict */ +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i, pos; + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + #if !CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t nkw = PyDict_Size(kw); + if (unlikely(nkw == -1)) return NULL; + #else + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + #endif + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; + } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (PyDict_Next(kw, &pos, &key, &value)) { + keys_are_strings &= + #if CYTHON_COMPILING_IN_LIMITED_API + PyType_GetFlags(Py_TYPE(key)); + #else + Py_TYPE(key)->tp_flags; + #endif + Py_INCREF(key); + Py_INCREF(value); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(PyTuple_SetItem(kwnames, i, key) < 0)) goto cleanup; + #else + PyTuple_SET_ITEM(kwnames, i, key); + #endif + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + res = vc(func, newargs, nargs, kwnames); +cleanup: + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + Py_ssize_t kw_size = + likely(kw == NULL) ? + 0 : +#if !CYTHON_ASSUME_SAFE_SIZE + PyDict_Size(kw); +#else + PyDict_GET_SIZE(kw); +#endif + if (kw_size == 0) { + return vc(func, args, nargs, NULL); + } +#if !CYTHON_ASSUME_SAFE_SIZE + else if (unlikely(kw_size == -1)) { + return NULL; + } +#endif + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); +} +#endif + +/* CythonFunctionShared */ +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunctionNoMethod(PyObject *func, void (*cfunc)(void)) { + if (__Pyx_CyFunction_Check(func)) { + return PyCFunction_GetFunction(((__pyx_CyFunctionObject*)func)->func) == (PyCFunction) cfunc; + } else if (PyCFunction_Check(func)) { + return PyCFunction_GetFunction(func) == (PyCFunction) cfunc; + } + return 0; +} +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if ((PyObject*)Py_TYPE(func) == __pyx_mstate_global->__Pyx_CachedMethodType) { + int result; + PyObject *newFunc = PyObject_GetAttr(func, __pyx_mstate_global->__pyx_n_u_func); + if (unlikely(!newFunc)) { + PyErr_Clear(); // It's only an optimization, so don't throw an error + return 0; + } + result = __Pyx__IsSameCyOrCFunctionNoMethod(newFunc, cfunc); + Py_DECREF(newFunc); + return result; + } + return __Pyx__IsSameCyOrCFunctionNoMethod(func, cfunc); +} +#else +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if (PyMethod_Check(func)) { + func = PyMethod_GET_FUNCTION(func); + } + return __Pyx_CyOrPyCFunction_Check(func) && __Pyx_CyOrPyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +} +#endif +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#else + __Pyx_Py_XDECREF_SET( + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#endif +} +static PyObject * +__Pyx_CyFunction_get_doc_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_doc == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_doc = PyObject_GetAttrString(op->func, "__doc__"); + if (unlikely(!op->func_doc)) return NULL; +#else + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } +#endif + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + PyObject *result; + CYTHON_UNUSED_VAR(closure); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_doc_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_doc, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_name == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_name = PyObject_GetAttrString(op->func, "__name__"); +#else + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_name_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_name, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + Py_INCREF(op->func_qualname); + result = op->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_dict_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_dict_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL)) { + PyErr_SetString(PyExc_TypeError, + "function's dictionary may not be deleted"); + return -1; + } + if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "setting function's dictionary to a non-dict"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_dict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = __Pyx_PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = __Pyx_PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_defaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_kwdefaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value || value == Py_None) { + value = NULL; + } else if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) { + int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; + if (is_coroutine) { + PyObject *is_coroutine_value, *module, *fromlist, *marker = __pyx_mstate_global->__pyx_n_u_is_coroutine; + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); +#if CYTHON_ASSUME_SAFE_MACROS + PyList_SET_ITEM(fromlist, 0, marker); +#else + if (unlikely(PyList_SetItem(fromlist, 0, marker) < 0)) { + Py_DECREF(marker); + Py_DECREF(fromlist); + return NULL; + } +#endif + module = PyImport_ImportModuleLevelObject(__pyx_mstate_global->__pyx_n_u_asyncio_coroutines, NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + is_coroutine_value = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(is_coroutine_value)) { + return is_coroutine_value; + } +ignore: + PyErr_Clear(); + } + return __Pyx_PyBool_FromLong(is_coroutine); +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + result = __Pyx_CyFunction_get_is_coroutine_value(op); + if (unlikely(!result)) + return NULL; + __Pyx_BEGIN_CRITICAL_SECTION(op); + if (op->func_is_coroutine) { + Py_DECREF(result); + result = __Pyx_NewRef(op->func_is_coroutine); + } else { + op->func_is_coroutine = __Pyx_NewRef(result); + } + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static void __Pyx_CyFunction_raise_argument_count_error(__pyx_CyFunctionObject *func, const char* message, Py_ssize_t size) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + py_name, message, size); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + name, message, size); +#endif +} +static void __Pyx_CyFunction_raise_type_error(__pyx_CyFunctionObject *func, const char* message) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s", + py_name, message); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s", + name, message); +#endif +} +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject * +__Pyx_CyFunction_get_module(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_GetAttrString(op->func, "__module__"); +} +static int +__Pyx_CyFunction_set_module(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_SetAttrString(op->func, "__module__", value); +} +#endif +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {"func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, + {"func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {"__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {"func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API + {"__module__", (getter)__Pyx_CyFunction_get_module, (setter)__Pyx_CyFunction_set_module, 0, 0}, +#endif + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { +#if !CYTHON_COMPILING_IN_LIMITED_API + {"__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, +#endif + {"__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#if CYTHON_METH_FASTCALL +#if CYTHON_BACKPORT_VECTORCALL || CYTHON_COMPILING_IN_LIMITED_API + {"__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, +#else + {"__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {"__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(args); + __Pyx_BEGIN_CRITICAL_SECTION(m); + Py_INCREF(m->func_qualname); + result = m->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { +#if !CYTHON_COMPILING_IN_LIMITED_API + PyCFunctionObject *cf = (PyCFunctionObject*) op; +#endif + if (unlikely(op == NULL)) + return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + op->func = PyCFunction_NewEx(ml, (PyObject*)op, module); + if (unlikely(!op->func)) return NULL; +#endif + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; +#if !CYTHON_COMPILING_IN_LIMITED_API + cf->m_ml = ml; + cf->m_self = (PyObject *) op; +#endif + Py_XINCREF(closure); + op->func_closure = closure; +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_XINCREF(module); + cf->m_module = module; +#endif + op->func_dict = NULL; + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + op->func_classobj = NULL; +#else + ((PyCMethodObject*)op)->mm_class = NULL; +#endif + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func); +#else + Py_CLEAR(((PyCFunctionObject*)m)->m_module); +#endif + Py_CLEAR(m->func_dict); + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API +#if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); +#else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } +#endif +#endif + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); + Py_CLEAR(m->defaults); + return 0; +} +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + __Pyx_PyHeapTypeObject_GC_Del(m); +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + { + int e = __Pyx_call_type_traverse((PyObject*)m, 1, visit, arg); + if (e) return e; + } + Py_VISIT(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func); +#else + Py_VISIT(((PyCFunctionObject*)m)->m_module); +#endif + Py_VISIT(m->func_dict); + __Pyx_VISIT_CONST(m->func_name); + __Pyx_VISIT_CONST(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + __Pyx_VISIT_CONST(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); +#endif + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); + Py_VISIT(m->defaults); + return 0; +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ + PyObject *repr; + __Pyx_BEGIN_CRITICAL_SECTION(op); + repr = PyUnicode_FromFormat("", + op->func_qualname, (void *)op); + __Pyx_END_CRITICAL_SECTION(); + return repr; +} +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *f = ((__pyx_CyFunctionObject*)func)->func; + PyCFunction meth; + int flags; + meth = PyCFunction_GetFunction(f); + if (unlikely(!meth)) return NULL; + flags = PyCFunction_GetFlags(f); + if (unlikely(flags < 0)) return NULL; +#else + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + int flags = f->m_ml->ml_flags; +#endif + Py_ssize_t size; + switch (flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 0)) + return (*meth)(self, NULL); + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes no arguments", size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 1)) { + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = __Pyx_PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif + result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(arg0); + #endif + return result; + } + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes exactly one argument", size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + return NULL; + } + __Pyx_CyFunction_raise_type_error( + (__pyx_CyFunctionObject*)func, "takes no keyword arguments"); + return NULL; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *self, *result; +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)func)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)func)->m_self; +#endif + result = __Pyx_CyFunction_CallMethod(func, self, arg, kw); + return result; +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; +#if CYTHON_ASSUME_SAFE_SIZE + argc = PyTuple_GET_SIZE(args); +#else + argc = PyTuple_Size(args); + if (unlikely(argc < 0)) return NULL; +#endif + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + PyErr_Format(PyExc_TypeError, + "unbound method %.200S() needs an argument", + cyfunc->func_qualname); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; +} +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "needs an argument"); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(__Pyx_PyTuple_GET_SIZE(kwnames))) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "takes no keyword arguments"); + return -1; + } + return ret; +} +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 0)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes no arguments", nargs); + return NULL; + } + return meth(self, NULL); +} +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 1)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes exactly one argument", nargs); + return NULL; + } + return meth(self, args[0]); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))meth)(self, args, nargs, kwnames); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCMethod)(void(*)(void))meth)(self, cls, args, (size_t)nargs, kwnames); +} +#endif +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#if CYTHON_METH_FASTCALL +#if defined(Py_TPFLAGS_HAVE_VECTORCALL) + Py_TPFLAGS_HAVE_VECTORCALL | +#elif defined(_Py_TPFLAGS_HAVE_VECTORCALL) + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif +#endif // CYTHON_METH_FASTCALL +#if PY_VERSION_HEX >= 0x030A0000 + Py_TPFLAGS_IMMUTABLETYPE | +#endif + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, + __pyx_CyFunctionType_slots +}; +static int __pyx_CyFunction_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + mstate->__pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_CyFunctionType_spec, NULL); + if (unlikely(mstate->__pyx_CyFunctionType == NULL)) { + return -1; + } + return 0; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, PyTypeObject *defaults_type) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_CallObject((PyObject*)defaults_type, NULL); // _PyObject_New(defaults_type); + if (unlikely(!m->defaults)) + return NULL; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +/* CythonFunction */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyObject *op = __Pyx_CyFunction_Init( + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_mstate_global->__pyx_CyFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + PyObject_GC_Track(op); + } + return op; +} + +/* CLineInTraceback */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline; + PyObject *ptype, *pvalue, *ptraceback; +#if CYTHON_COMPILING_IN_CPYTHON + PyObject **cython_runtime_dict; +#endif + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); +#if CYTHON_COMPILING_IN_CPYTHON + cython_runtime_dict = _PyObject_GetDictPtr(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __Pyx_BEGIN_CRITICAL_SECTION(*cython_runtime_dict); + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, *cython_runtime_dict, + __Pyx_PyDict_GetItemStr(*cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback)) + Py_XINCREF(use_cline); + __Pyx_END_CRITICAL_SECTION(); + } else +#endif + { + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback); + if (use_cline_obj) { + use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; + Py_INCREF(use_cline); + Py_DECREF(use_cline_obj); + } else { + PyErr_Clear(); + use_cline = NULL; + } + } + if (!use_cline) { + c_line = 0; + (void) PyObject_SetAttr(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False); + } + else if (use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + (void) __pyx_cfilenm; + (void) __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + py_funcname = PyUnicode_FromString(funcname); + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* CIntFromPyVerify */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntFromPy */ +static CYTHON_INLINE size_t __Pyx_PyLong_As_size_t(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + size_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (size_t) -1; + val = __Pyx_PyLong_As_size_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(size_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(size_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 2 * PyLong_SHIFT)) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(size_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 3 * PyLong_SHIFT)) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(size_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 4 * PyLong_SHIFT)) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(size_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(size_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(size_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(size_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(size_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT)) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(size_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(size_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + size_t val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (size_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (size_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (size_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (size_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(size_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((size_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(size_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((size_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((size_t) 1) << (sizeof(size_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (size_t) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to size_t"); + return (size_t) -1; +} + +/* FormatTypeName */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static __Pyx_TypeName +__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) +{ + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u__2); + } + goto done; +} +#endif + +/* PyObjectVectorCallKwBuilder */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if defined(HAVE_LONG_LONG) && !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPy */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + long val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (long) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + int val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030b0000 + return Py_Version & ~0xFFUL; +#else + static unsigned long __Pyx_cached_runtime_version = 0; + if (__Pyx_cached_runtime_version == 0) { + const char* rt_version = Py_GetVersion(); + unsigned long version = 0; + unsigned long factor = 0x01000000UL; + unsigned int digit = 0; + int i = 0; + while (factor) { + while ('0' <= rt_version[i] && rt_version[i] <= '9') { + digit = digit * 10 + (unsigned int) (rt_version[i] - '0'); + ++i; + } + version += factor * digit; + if (rt_version[i] != '.') + break; + digit = 0; + factor >>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } + return __Pyx_cached_runtime_version; +#endif +} + +/* CheckBinaryVersion */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } +} + +/* FunctionExport */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig) { + PyObject *d = 0; + PyObject *cobj = 0; + union { + void (*fp)(void); + void *p; + } tmp; + d = PyObject_GetAttrString(__pyx_m, "__pyx_capi__"); + if (!d) { + PyErr_Clear(); + d = PyDict_New(); + if (!d) + goto bad; + Py_INCREF(d); + if (PyModule_AddObject(__pyx_m, "__pyx_capi__", d) < 0) + goto bad; + } + tmp.fp = f; + cobj = PyCapsule_New(tmp.p, sig, 0); + if (!cobj) + goto bad; + if (PyDict_SetItemString(d, name, cobj) < 0) + goto bad; + Py_DECREF(cobj); + Py_DECREF(d); + return 0; +bad: + Py_XDECREF(cobj); + Py_XDECREF(d); + return -1; +} + +/* NewCodeObj */ +#if CYTHON_COMPILING_IN_LIMITED_API + static PyObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyObject *exception_table = NULL; + PyObject *types_module=NULL, *code_type=NULL, *result=NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + PyObject *version_info; + PyObject *py_minor_version = NULL; + #endif + long minor_version = 0; + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + minor_version = 11; + #else + if (!(version_info = PySys_GetObject("version_info"))) goto end; + if (!(py_minor_version = PySequence_GetItem(version_info, 1))) goto end; + minor_version = PyLong_AsLong(py_minor_version); + Py_DECREF(py_minor_version); + if (minor_version == -1 && PyErr_Occurred()) goto end; + #endif + if (!(types_module = PyImport_ImportModule("types"))) goto end; + if (!(code_type = PyObject_GetAttrString(types_module, "CodeType"))) goto end; + if (minor_version <= 7) { + (void)p; + result = PyObject_CallFunction(code_type, "iiiiiOOOOOOiOOO", a, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else if (minor_version <= 10) { + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOiOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else { + if (!(exception_table = PyBytes_FromStringAndSize(NULL, 0))) goto end; + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOOiOOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, name, fline, lnos, exception_table, fv, cell); + } + end: + Py_XDECREF(code_type); + Py_XDECREF(exception_table); + Py_XDECREF(types_module); + if (type) { + PyErr_Restore(type, value, traceback); + } + return result; + } +#elif PY_VERSION_HEX >= 0x030B0000 + static PyCodeObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyCodeObject *result; + result = + #if PY_VERSION_HEX >= 0x030C0000 + PyUnstable_Code_NewWithPosOnlyArgs + #else + PyCode_NewWithPosOnlyArgs + #endif + (a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, name, fline, lnos, __pyx_mstate_global->__pyx_empty_bytes); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030c00A1 + if (likely(result)) + result->_co_firsttraceable = 0; + #endif + return result; + } +#elif PY_VERSION_HEX >= 0x030800B2 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#else + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#endif +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + const char *line_table, + PyObject *tuple_dedup_map +) { + PyObject *code_obj = NULL, *varnames_tuple_dedup = NULL, *code_bytes = NULL, *line_table_bytes = NULL; + Py_ssize_t var_count = (Py_ssize_t) descr.nlocals; + PyObject *varnames_tuple = PyTuple_New(var_count); + if (unlikely(!varnames_tuple)) return NULL; + for (Py_ssize_t i=0; i < var_count; i++) { + Py_INCREF(varnames[i]); + if (__Pyx_PyTuple_SET_ITEM(varnames_tuple, i, varnames[i]) != (0)) goto done; + } + #if CYTHON_COMPILING_IN_LIMITED_API + varnames_tuple_dedup = PyDict_GetItem(tuple_dedup_map, varnames_tuple); + if (!varnames_tuple_dedup) { + if (unlikely(PyDict_SetItem(tuple_dedup_map, varnames_tuple, varnames_tuple) < 0)) goto done; + varnames_tuple_dedup = varnames_tuple; + } + #else + varnames_tuple_dedup = PyDict_SetDefault(tuple_dedup_map, varnames_tuple, varnames_tuple); + if (unlikely(!varnames_tuple_dedup)) goto done; + #endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(varnames_tuple_dedup); + #endif + if (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table != NULL + && !CYTHON_COMPILING_IN_GRAAL) { + line_table_bytes = PyBytes_FromStringAndSize(line_table, descr.line_table_length); + if (unlikely(!line_table_bytes)) goto done; + Py_ssize_t code_len = (descr.line_table_length * 2 + 4) & ~3; + code_bytes = PyBytes_FromStringAndSize(NULL, code_len); + if (unlikely(!code_bytes)) goto done; + char* c_code_bytes = PyBytes_AsString(code_bytes); + if (unlikely(!c_code_bytes)) goto done; + memset(c_code_bytes, 0, (size_t) code_len); + } + code_obj = (PyObject*) __Pyx__PyCode_New( + (int) descr.argcount, + (int) descr.num_posonly_args, + (int) descr.num_kwonly_args, + (int) descr.nlocals, + 0, + (int) descr.flags, + code_bytes ? code_bytes : __pyx_mstate_global->__pyx_empty_bytes, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + varnames_tuple_dedup, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + filename, + funcname, + (int) descr.first_line, + (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table_bytes) ? line_table_bytes : __pyx_mstate_global->__pyx_empty_bytes + ); +done: + Py_XDECREF(code_bytes); + Py_XDECREF(line_table_bytes); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(varnames_tuple_dedup); + #endif + Py_DECREF(varnames_tuple); + return code_obj; +} + +/* InitStrings */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names) { + while (t->s) { + PyObject *str; + if (t->is_unicode) { + if (t->intern) { + str = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + str = PyUnicode_Decode(t->s, t->n - 1, encoding_names[t->encoding], NULL); + } else { + str = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + str = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + if (!str) + return -1; + *target = str; + if (PyObject_Hash(str) == -1) + return -1; + ++t; + ++target; + } + return 0; +} + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} +#if CYTHON_USE_PYLONG_INTERNALS +static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value) { + if (unlikely(!__Pyx_PyLong_IsCompact(x))) + return 0; + Py_ssize_t value = __Pyx_PyLong_CompactValue(x); + if ((sizeof(long) < sizeof(Py_ssize_t)) && unlikely(value != (long) value)) + return 0; + *return_value = (long) value; + return 1; +} +#endif + + +/* MultiPhaseInitModuleState */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 +#else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 +#endif +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS +#error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" +#endif +#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#define __Pyx_ModuleStateLookup_Lock() +#define __Pyx_ModuleStateLookup_Unlock() +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 +static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; +#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#include +static std::mutex __Pyx_ModuleStateLookup_mutex; +#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() +#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) +#include +static mtx_t __Pyx_ModuleStateLookup_mutex; +static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; +static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); +} +#define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(HAVE_PTHREAD_H) +#include +static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; +#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(_WIN32) +#include // synchapi.h on its own doesn't work +static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; +#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#else +#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." +#endif +typedef struct { + int64_t id; + PyObject *module; +} __Pyx_InterpreterIdAndModule; +typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; +} __Pyx_ModuleStateLookupData; +#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; +#else +static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; +#endif +static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; +} +static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; +#else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); +#endif + return result; + } +} +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); +} +#else +#define __Pyx_ModuleStateLookup_wait_until_no_readers() +#endif +static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; +} +static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; +} +static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); +#else + __Pyx_ModuleStateLookup_data = new_data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return result; +} +static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; +#endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); +#else + __Pyx_ModuleStateLookup_data = data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; +} +#endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/cufile.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/cufile.pxd new file mode 100644 index 0000000000000000000000000000000000000000..880343da3f933b0dd8e4b4cb14ab40f5d79d2cc0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/cufile.pxd @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.9.0 to 12.9.1. Do not modify it directly. + +from ..cycufile cimport * + + +############################################################################### +# Wrapper functions +############################################################################### + +cdef CUfileError_t _cuFileHandleRegister(CUfileHandle_t* fh, CUfileDescr_t* descr) except?CUFILE_LOADING_ERROR nogil +cdef void _cuFileHandleDeregister(CUfileHandle_t fh) except* nogil +cdef CUfileError_t _cuFileBufRegister(const void* bufPtr_base, size_t length, int flags) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileBufDeregister(const void* bufPtr_base) except?CUFILE_LOADING_ERROR nogil +cdef ssize_t _cuFileRead(CUfileHandle_t fh, void* bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset) except* nogil +cdef ssize_t _cuFileWrite(CUfileHandle_t fh, const void* bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset) except* nogil +cdef CUfileError_t _cuFileDriverOpen() except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileDriverClose_v2() except?CUFILE_LOADING_ERROR nogil +cdef long _cuFileUseCount() except* nogil +cdef CUfileError_t _cuFileDriverGetProperties(CUfileDrvProps_t* props) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileDriverSetPollMode(cpp_bool poll, size_t poll_threshold_size) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileDriverSetMaxDirectIOSize(size_t max_direct_io_size) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileDriverSetMaxCacheSize(size_t max_cache_size) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileDriverSetMaxPinnedMemSize(size_t max_pinned_size) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileBatchIOSetUp(CUfileBatchHandle_t* batch_idp, unsigned nr) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileBatchIOSubmit(CUfileBatchHandle_t batch_idp, unsigned nr, CUfileIOParams_t* iocbp, unsigned int flags) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileBatchIOGetStatus(CUfileBatchHandle_t batch_idp, unsigned min_nr, unsigned* nr, CUfileIOEvents_t* iocbp, timespec* timeout) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileBatchIOCancel(CUfileBatchHandle_t batch_idp) except?CUFILE_LOADING_ERROR nogil +cdef void _cuFileBatchIODestroy(CUfileBatchHandle_t batch_idp) except* nogil +cdef CUfileError_t _cuFileReadAsync(CUfileHandle_t fh, void* bufPtr_base, size_t* size_p, off_t* file_offset_p, off_t* bufPtr_offset_p, ssize_t* bytes_read_p, CUstream stream) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileWriteAsync(CUfileHandle_t fh, void* bufPtr_base, size_t* size_p, off_t* file_offset_p, off_t* bufPtr_offset_p, ssize_t* bytes_written_p, CUstream stream) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileStreamRegister(CUstream stream, unsigned flags) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileStreamDeregister(CUstream stream) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileGetVersion(int* version) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileGetParameterSizeT(CUFileSizeTConfigParameter_t param, size_t* value) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileGetParameterBool(CUFileBoolConfigParameter_t param, cpp_bool* value) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileGetParameterString(CUFileStringConfigParameter_t param, char* desc_str, int len) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileSetParameterSizeT(CUFileSizeTConfigParameter_t param, size_t value) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileSetParameterBool(CUFileBoolConfigParameter_t param, cpp_bool value) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileSetParameterString(CUFileStringConfigParameter_t param, const char* desc_str) except?CUFILE_LOADING_ERROR nogil +cdef CUfileError_t _cuFileDriverClose() except?CUFILE_LOADING_ERROR nogil diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/cufile.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/cufile.pyx new file mode 100644 index 0000000000000000000000000000000000000000..dafeec2816a941bafd86f8f1a135b7e4f06117d1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/cufile.pyx @@ -0,0 +1,745 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.9.0 to 12.9.1. Do not modify it directly. + +from libc.stdint cimport intptr_t, uintptr_t +import threading + +from .utils import FunctionNotFoundError, NotSupportedError + +from cuda.pathfinder import load_nvidia_dynamic_lib + +import cython + + +############################################################################### +# Extern +############################################################################### + +cdef extern from "" nogil: + void* dlopen(const char*, int) + char* dlerror() + void* dlsym(void*, const char*) + int dlclose(void*) + + enum: + RTLD_LAZY + RTLD_NOW + RTLD_GLOBAL + RTLD_LOCAL + + const void* RTLD_DEFAULT 'RTLD_DEFAULT' + + +############################################################################### +# Wrapper init +############################################################################### + +cdef object __symbol_lock = threading.Lock() +cdef bint __py_cufile_init = False +cdef void* __cuDriverGetVersion = NULL + +cdef void* __cuFileHandleRegister = NULL +cdef void* __cuFileHandleDeregister = NULL +cdef void* __cuFileBufRegister = NULL +cdef void* __cuFileBufDeregister = NULL +cdef void* __cuFileRead = NULL +cdef void* __cuFileWrite = NULL +cdef void* __cuFileDriverOpen = NULL +cdef void* __cuFileDriverClose_v2 = NULL +cdef void* __cuFileUseCount = NULL +cdef void* __cuFileDriverGetProperties = NULL +cdef void* __cuFileDriverSetPollMode = NULL +cdef void* __cuFileDriverSetMaxDirectIOSize = NULL +cdef void* __cuFileDriverSetMaxCacheSize = NULL +cdef void* __cuFileDriverSetMaxPinnedMemSize = NULL +cdef void* __cuFileBatchIOSetUp = NULL +cdef void* __cuFileBatchIOSubmit = NULL +cdef void* __cuFileBatchIOGetStatus = NULL +cdef void* __cuFileBatchIOCancel = NULL +cdef void* __cuFileBatchIODestroy = NULL +cdef void* __cuFileReadAsync = NULL +cdef void* __cuFileWriteAsync = NULL +cdef void* __cuFileStreamRegister = NULL +cdef void* __cuFileStreamDeregister = NULL +cdef void* __cuFileGetVersion = NULL +cdef void* __cuFileGetParameterSizeT = NULL +cdef void* __cuFileGetParameterBool = NULL +cdef void* __cuFileGetParameterString = NULL +cdef void* __cuFileSetParameterSizeT = NULL +cdef void* __cuFileSetParameterBool = NULL +cdef void* __cuFileSetParameterString = NULL +cdef void* __cuFileDriverClose = NULL + + +cdef void* load_library() except* with gil: + cdef uintptr_t handle = load_nvidia_dynamic_lib("cufile")._handle_uint + return handle + + +cdef int __check_or_init_cufile() except -1 nogil: + global __py_cufile_init + + cdef void* handle = NULL + + with gil, __symbol_lock: + # Load function + global __cuFileHandleRegister + __cuFileHandleRegister = dlsym(RTLD_DEFAULT, 'cuFileHandleRegister') + if __cuFileHandleRegister == NULL: + if handle == NULL: + handle = load_library() + __cuFileHandleRegister = dlsym(handle, 'cuFileHandleRegister') + + global __cuFileHandleDeregister + __cuFileHandleDeregister = dlsym(RTLD_DEFAULT, 'cuFileHandleDeregister') + if __cuFileHandleDeregister == NULL: + if handle == NULL: + handle = load_library() + __cuFileHandleDeregister = dlsym(handle, 'cuFileHandleDeregister') + + global __cuFileBufRegister + __cuFileBufRegister = dlsym(RTLD_DEFAULT, 'cuFileBufRegister') + if __cuFileBufRegister == NULL: + if handle == NULL: + handle = load_library() + __cuFileBufRegister = dlsym(handle, 'cuFileBufRegister') + + global __cuFileBufDeregister + __cuFileBufDeregister = dlsym(RTLD_DEFAULT, 'cuFileBufDeregister') + if __cuFileBufDeregister == NULL: + if handle == NULL: + handle = load_library() + __cuFileBufDeregister = dlsym(handle, 'cuFileBufDeregister') + + global __cuFileRead + __cuFileRead = dlsym(RTLD_DEFAULT, 'cuFileRead') + if __cuFileRead == NULL: + if handle == NULL: + handle = load_library() + __cuFileRead = dlsym(handle, 'cuFileRead') + + global __cuFileWrite + __cuFileWrite = dlsym(RTLD_DEFAULT, 'cuFileWrite') + if __cuFileWrite == NULL: + if handle == NULL: + handle = load_library() + __cuFileWrite = dlsym(handle, 'cuFileWrite') + + global __cuFileDriverOpen + __cuFileDriverOpen = dlsym(RTLD_DEFAULT, 'cuFileDriverOpen') + if __cuFileDriverOpen == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverOpen = dlsym(handle, 'cuFileDriverOpen') + + global __cuFileDriverClose_v2 + __cuFileDriverClose_v2 = dlsym(RTLD_DEFAULT, 'cuFileDriverClose_v2') + if __cuFileDriverClose_v2 == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverClose_v2 = dlsym(handle, 'cuFileDriverClose_v2') + + global __cuFileUseCount + __cuFileUseCount = dlsym(RTLD_DEFAULT, 'cuFileUseCount') + if __cuFileUseCount == NULL: + if handle == NULL: + handle = load_library() + __cuFileUseCount = dlsym(handle, 'cuFileUseCount') + + global __cuFileDriverGetProperties + __cuFileDriverGetProperties = dlsym(RTLD_DEFAULT, 'cuFileDriverGetProperties') + if __cuFileDriverGetProperties == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverGetProperties = dlsym(handle, 'cuFileDriverGetProperties') + + global __cuFileDriverSetPollMode + __cuFileDriverSetPollMode = dlsym(RTLD_DEFAULT, 'cuFileDriverSetPollMode') + if __cuFileDriverSetPollMode == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverSetPollMode = dlsym(handle, 'cuFileDriverSetPollMode') + + global __cuFileDriverSetMaxDirectIOSize + __cuFileDriverSetMaxDirectIOSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxDirectIOSize') + if __cuFileDriverSetMaxDirectIOSize == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverSetMaxDirectIOSize = dlsym(handle, 'cuFileDriverSetMaxDirectIOSize') + + global __cuFileDriverSetMaxCacheSize + __cuFileDriverSetMaxCacheSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxCacheSize') + if __cuFileDriverSetMaxCacheSize == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverSetMaxCacheSize = dlsym(handle, 'cuFileDriverSetMaxCacheSize') + + global __cuFileDriverSetMaxPinnedMemSize + __cuFileDriverSetMaxPinnedMemSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxPinnedMemSize') + if __cuFileDriverSetMaxPinnedMemSize == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverSetMaxPinnedMemSize = dlsym(handle, 'cuFileDriverSetMaxPinnedMemSize') + + global __cuFileBatchIOSetUp + __cuFileBatchIOSetUp = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSetUp') + if __cuFileBatchIOSetUp == NULL: + if handle == NULL: + handle = load_library() + __cuFileBatchIOSetUp = dlsym(handle, 'cuFileBatchIOSetUp') + + global __cuFileBatchIOSubmit + __cuFileBatchIOSubmit = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSubmit') + if __cuFileBatchIOSubmit == NULL: + if handle == NULL: + handle = load_library() + __cuFileBatchIOSubmit = dlsym(handle, 'cuFileBatchIOSubmit') + + global __cuFileBatchIOGetStatus + __cuFileBatchIOGetStatus = dlsym(RTLD_DEFAULT, 'cuFileBatchIOGetStatus') + if __cuFileBatchIOGetStatus == NULL: + if handle == NULL: + handle = load_library() + __cuFileBatchIOGetStatus = dlsym(handle, 'cuFileBatchIOGetStatus') + + global __cuFileBatchIOCancel + __cuFileBatchIOCancel = dlsym(RTLD_DEFAULT, 'cuFileBatchIOCancel') + if __cuFileBatchIOCancel == NULL: + if handle == NULL: + handle = load_library() + __cuFileBatchIOCancel = dlsym(handle, 'cuFileBatchIOCancel') + + global __cuFileBatchIODestroy + __cuFileBatchIODestroy = dlsym(RTLD_DEFAULT, 'cuFileBatchIODestroy') + if __cuFileBatchIODestroy == NULL: + if handle == NULL: + handle = load_library() + __cuFileBatchIODestroy = dlsym(handle, 'cuFileBatchIODestroy') + + global __cuFileReadAsync + __cuFileReadAsync = dlsym(RTLD_DEFAULT, 'cuFileReadAsync') + if __cuFileReadAsync == NULL: + if handle == NULL: + handle = load_library() + __cuFileReadAsync = dlsym(handle, 'cuFileReadAsync') + + global __cuFileWriteAsync + __cuFileWriteAsync = dlsym(RTLD_DEFAULT, 'cuFileWriteAsync') + if __cuFileWriteAsync == NULL: + if handle == NULL: + handle = load_library() + __cuFileWriteAsync = dlsym(handle, 'cuFileWriteAsync') + + global __cuFileStreamRegister + __cuFileStreamRegister = dlsym(RTLD_DEFAULT, 'cuFileStreamRegister') + if __cuFileStreamRegister == NULL: + if handle == NULL: + handle = load_library() + __cuFileStreamRegister = dlsym(handle, 'cuFileStreamRegister') + + global __cuFileStreamDeregister + __cuFileStreamDeregister = dlsym(RTLD_DEFAULT, 'cuFileStreamDeregister') + if __cuFileStreamDeregister == NULL: + if handle == NULL: + handle = load_library() + __cuFileStreamDeregister = dlsym(handle, 'cuFileStreamDeregister') + + global __cuFileGetVersion + __cuFileGetVersion = dlsym(RTLD_DEFAULT, 'cuFileGetVersion') + if __cuFileGetVersion == NULL: + if handle == NULL: + handle = load_library() + __cuFileGetVersion = dlsym(handle, 'cuFileGetVersion') + + global __cuFileGetParameterSizeT + __cuFileGetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileGetParameterSizeT') + if __cuFileGetParameterSizeT == NULL: + if handle == NULL: + handle = load_library() + __cuFileGetParameterSizeT = dlsym(handle, 'cuFileGetParameterSizeT') + + global __cuFileGetParameterBool + __cuFileGetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileGetParameterBool') + if __cuFileGetParameterBool == NULL: + if handle == NULL: + handle = load_library() + __cuFileGetParameterBool = dlsym(handle, 'cuFileGetParameterBool') + + global __cuFileGetParameterString + __cuFileGetParameterString = dlsym(RTLD_DEFAULT, 'cuFileGetParameterString') + if __cuFileGetParameterString == NULL: + if handle == NULL: + handle = load_library() + __cuFileGetParameterString = dlsym(handle, 'cuFileGetParameterString') + + global __cuFileSetParameterSizeT + __cuFileSetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileSetParameterSizeT') + if __cuFileSetParameterSizeT == NULL: + if handle == NULL: + handle = load_library() + __cuFileSetParameterSizeT = dlsym(handle, 'cuFileSetParameterSizeT') + + global __cuFileSetParameterBool + __cuFileSetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileSetParameterBool') + if __cuFileSetParameterBool == NULL: + if handle == NULL: + handle = load_library() + __cuFileSetParameterBool = dlsym(handle, 'cuFileSetParameterBool') + + global __cuFileSetParameterString + __cuFileSetParameterString = dlsym(RTLD_DEFAULT, 'cuFileSetParameterString') + if __cuFileSetParameterString == NULL: + if handle == NULL: + handle = load_library() + __cuFileSetParameterString = dlsym(handle, 'cuFileSetParameterString') + + global __cuFileDriverClose + __cuFileDriverClose = dlsym(RTLD_DEFAULT, 'cuFileDriverClose') + if __cuFileDriverClose == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverClose = dlsym(handle, 'cuFileDriverClose') + + __py_cufile_init = True + return 0 + + +cdef inline int _check_or_init_cufile() except -1 nogil: + if __py_cufile_init: + return 0 + + return __check_or_init_cufile() + + +cdef dict func_ptrs = None + + +cpdef dict _inspect_function_pointers(): + global func_ptrs + if func_ptrs is not None: + return func_ptrs + + _check_or_init_cufile() + cdef dict data = {} + + global __cuFileHandleRegister + data["__cuFileHandleRegister"] = __cuFileHandleRegister + + global __cuFileHandleDeregister + data["__cuFileHandleDeregister"] = __cuFileHandleDeregister + + global __cuFileBufRegister + data["__cuFileBufRegister"] = __cuFileBufRegister + + global __cuFileBufDeregister + data["__cuFileBufDeregister"] = __cuFileBufDeregister + + global __cuFileRead + data["__cuFileRead"] = __cuFileRead + + global __cuFileWrite + data["__cuFileWrite"] = __cuFileWrite + + global __cuFileDriverOpen + data["__cuFileDriverOpen"] = __cuFileDriverOpen + + global __cuFileDriverClose_v2 + data["__cuFileDriverClose_v2"] = __cuFileDriverClose_v2 + + global __cuFileUseCount + data["__cuFileUseCount"] = __cuFileUseCount + + global __cuFileDriverGetProperties + data["__cuFileDriverGetProperties"] = __cuFileDriverGetProperties + + global __cuFileDriverSetPollMode + data["__cuFileDriverSetPollMode"] = __cuFileDriverSetPollMode + + global __cuFileDriverSetMaxDirectIOSize + data["__cuFileDriverSetMaxDirectIOSize"] = __cuFileDriverSetMaxDirectIOSize + + global __cuFileDriverSetMaxCacheSize + data["__cuFileDriverSetMaxCacheSize"] = __cuFileDriverSetMaxCacheSize + + global __cuFileDriverSetMaxPinnedMemSize + data["__cuFileDriverSetMaxPinnedMemSize"] = __cuFileDriverSetMaxPinnedMemSize + + global __cuFileBatchIOSetUp + data["__cuFileBatchIOSetUp"] = __cuFileBatchIOSetUp + + global __cuFileBatchIOSubmit + data["__cuFileBatchIOSubmit"] = __cuFileBatchIOSubmit + + global __cuFileBatchIOGetStatus + data["__cuFileBatchIOGetStatus"] = __cuFileBatchIOGetStatus + + global __cuFileBatchIOCancel + data["__cuFileBatchIOCancel"] = __cuFileBatchIOCancel + + global __cuFileBatchIODestroy + data["__cuFileBatchIODestroy"] = __cuFileBatchIODestroy + + global __cuFileReadAsync + data["__cuFileReadAsync"] = __cuFileReadAsync + + global __cuFileWriteAsync + data["__cuFileWriteAsync"] = __cuFileWriteAsync + + global __cuFileStreamRegister + data["__cuFileStreamRegister"] = __cuFileStreamRegister + + global __cuFileStreamDeregister + data["__cuFileStreamDeregister"] = __cuFileStreamDeregister + + global __cuFileGetVersion + data["__cuFileGetVersion"] = __cuFileGetVersion + + global __cuFileGetParameterSizeT + data["__cuFileGetParameterSizeT"] = __cuFileGetParameterSizeT + + global __cuFileGetParameterBool + data["__cuFileGetParameterBool"] = __cuFileGetParameterBool + + global __cuFileGetParameterString + data["__cuFileGetParameterString"] = __cuFileGetParameterString + + global __cuFileSetParameterSizeT + data["__cuFileSetParameterSizeT"] = __cuFileSetParameterSizeT + + global __cuFileSetParameterBool + data["__cuFileSetParameterBool"] = __cuFileSetParameterBool + + global __cuFileSetParameterString + data["__cuFileSetParameterString"] = __cuFileSetParameterString + + global __cuFileDriverClose + data["__cuFileDriverClose"] = __cuFileDriverClose + + func_ptrs = data + return data + + +cpdef _inspect_function_pointer(str name): + global func_ptrs + if func_ptrs is None: + func_ptrs = _inspect_function_pointers() + return func_ptrs[name] + + +############################################################################### +# Wrapper functions +############################################################################### + +cdef CUfileError_t _cuFileHandleRegister(CUfileHandle_t* fh, CUfileDescr_t* descr) except?CUFILE_LOADING_ERROR nogil: + global __cuFileHandleRegister + _check_or_init_cufile() + if __cuFileHandleRegister == NULL: + with gil: + raise FunctionNotFoundError("function cuFileHandleRegister is not found") + return (__cuFileHandleRegister)( + fh, descr) + + +@cython.show_performance_hints(False) +cdef void _cuFileHandleDeregister(CUfileHandle_t fh) except* nogil: + global __cuFileHandleDeregister + _check_or_init_cufile() + if __cuFileHandleDeregister == NULL: + with gil: + raise FunctionNotFoundError("function cuFileHandleDeregister is not found") + (__cuFileHandleDeregister)( + fh) + + +cdef CUfileError_t _cuFileBufRegister(const void* bufPtr_base, size_t length, int flags) except?CUFILE_LOADING_ERROR nogil: + global __cuFileBufRegister + _check_or_init_cufile() + if __cuFileBufRegister == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBufRegister is not found") + return (__cuFileBufRegister)( + bufPtr_base, length, flags) + + +cdef CUfileError_t _cuFileBufDeregister(const void* bufPtr_base) except?CUFILE_LOADING_ERROR nogil: + global __cuFileBufDeregister + _check_or_init_cufile() + if __cuFileBufDeregister == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBufDeregister is not found") + return (__cuFileBufDeregister)( + bufPtr_base) + + +cdef ssize_t _cuFileRead(CUfileHandle_t fh, void* bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset) except* nogil: + global __cuFileRead + _check_or_init_cufile() + if __cuFileRead == NULL: + with gil: + raise FunctionNotFoundError("function cuFileRead is not found") + return (__cuFileRead)( + fh, bufPtr_base, size, file_offset, bufPtr_offset) + + +cdef ssize_t _cuFileWrite(CUfileHandle_t fh, const void* bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset) except* nogil: + global __cuFileWrite + _check_or_init_cufile() + if __cuFileWrite == NULL: + with gil: + raise FunctionNotFoundError("function cuFileWrite is not found") + return (__cuFileWrite)( + fh, bufPtr_base, size, file_offset, bufPtr_offset) + + +cdef CUfileError_t _cuFileDriverOpen() except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverOpen + _check_or_init_cufile() + if __cuFileDriverOpen == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverOpen is not found") + return (__cuFileDriverOpen)( + ) + + +cdef CUfileError_t _cuFileDriverClose_v2() except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverClose_v2 + _check_or_init_cufile() + if __cuFileDriverClose_v2 == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverClose_v2 is not found") + return (__cuFileDriverClose_v2)( + ) + + +cdef long _cuFileUseCount() except* nogil: + global __cuFileUseCount + _check_or_init_cufile() + if __cuFileUseCount == NULL: + with gil: + raise FunctionNotFoundError("function cuFileUseCount is not found") + return (__cuFileUseCount)( + ) + + +cdef CUfileError_t _cuFileDriverGetProperties(CUfileDrvProps_t* props) except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverGetProperties + _check_or_init_cufile() + if __cuFileDriverGetProperties == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverGetProperties is not found") + return (__cuFileDriverGetProperties)( + props) + + +cdef CUfileError_t _cuFileDriverSetPollMode(cpp_bool poll, size_t poll_threshold_size) except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverSetPollMode + _check_or_init_cufile() + if __cuFileDriverSetPollMode == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverSetPollMode is not found") + return (__cuFileDriverSetPollMode)( + poll, poll_threshold_size) + + +cdef CUfileError_t _cuFileDriverSetMaxDirectIOSize(size_t max_direct_io_size) except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverSetMaxDirectIOSize + _check_or_init_cufile() + if __cuFileDriverSetMaxDirectIOSize == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverSetMaxDirectIOSize is not found") + return (__cuFileDriverSetMaxDirectIOSize)( + max_direct_io_size) + + +cdef CUfileError_t _cuFileDriverSetMaxCacheSize(size_t max_cache_size) except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverSetMaxCacheSize + _check_or_init_cufile() + if __cuFileDriverSetMaxCacheSize == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverSetMaxCacheSize is not found") + return (__cuFileDriverSetMaxCacheSize)( + max_cache_size) + + +cdef CUfileError_t _cuFileDriverSetMaxPinnedMemSize(size_t max_pinned_size) except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverSetMaxPinnedMemSize + _check_or_init_cufile() + if __cuFileDriverSetMaxPinnedMemSize == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverSetMaxPinnedMemSize is not found") + return (__cuFileDriverSetMaxPinnedMemSize)( + max_pinned_size) + + +cdef CUfileError_t _cuFileBatchIOSetUp(CUfileBatchHandle_t* batch_idp, unsigned nr) except?CUFILE_LOADING_ERROR nogil: + global __cuFileBatchIOSetUp + _check_or_init_cufile() + if __cuFileBatchIOSetUp == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBatchIOSetUp is not found") + return (__cuFileBatchIOSetUp)( + batch_idp, nr) + + +cdef CUfileError_t _cuFileBatchIOSubmit(CUfileBatchHandle_t batch_idp, unsigned nr, CUfileIOParams_t* iocbp, unsigned int flags) except?CUFILE_LOADING_ERROR nogil: + global __cuFileBatchIOSubmit + _check_or_init_cufile() + if __cuFileBatchIOSubmit == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBatchIOSubmit is not found") + return (__cuFileBatchIOSubmit)( + batch_idp, nr, iocbp, flags) + + +cdef CUfileError_t _cuFileBatchIOGetStatus(CUfileBatchHandle_t batch_idp, unsigned min_nr, unsigned* nr, CUfileIOEvents_t* iocbp, timespec* timeout) except?CUFILE_LOADING_ERROR nogil: + global __cuFileBatchIOGetStatus + _check_or_init_cufile() + if __cuFileBatchIOGetStatus == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBatchIOGetStatus is not found") + return (__cuFileBatchIOGetStatus)( + batch_idp, min_nr, nr, iocbp, timeout) + + +cdef CUfileError_t _cuFileBatchIOCancel(CUfileBatchHandle_t batch_idp) except?CUFILE_LOADING_ERROR nogil: + global __cuFileBatchIOCancel + _check_or_init_cufile() + if __cuFileBatchIOCancel == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBatchIOCancel is not found") + return (__cuFileBatchIOCancel)( + batch_idp) + + +@cython.show_performance_hints(False) +cdef void _cuFileBatchIODestroy(CUfileBatchHandle_t batch_idp) except* nogil: + global __cuFileBatchIODestroy + _check_or_init_cufile() + if __cuFileBatchIODestroy == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBatchIODestroy is not found") + (__cuFileBatchIODestroy)( + batch_idp) + + +cdef CUfileError_t _cuFileReadAsync(CUfileHandle_t fh, void* bufPtr_base, size_t* size_p, off_t* file_offset_p, off_t* bufPtr_offset_p, ssize_t* bytes_read_p, CUstream stream) except?CUFILE_LOADING_ERROR nogil: + global __cuFileReadAsync + _check_or_init_cufile() + if __cuFileReadAsync == NULL: + with gil: + raise FunctionNotFoundError("function cuFileReadAsync is not found") + return (__cuFileReadAsync)( + fh, bufPtr_base, size_p, file_offset_p, bufPtr_offset_p, bytes_read_p, stream) + + +cdef CUfileError_t _cuFileWriteAsync(CUfileHandle_t fh, void* bufPtr_base, size_t* size_p, off_t* file_offset_p, off_t* bufPtr_offset_p, ssize_t* bytes_written_p, CUstream stream) except?CUFILE_LOADING_ERROR nogil: + global __cuFileWriteAsync + _check_or_init_cufile() + if __cuFileWriteAsync == NULL: + with gil: + raise FunctionNotFoundError("function cuFileWriteAsync is not found") + return (__cuFileWriteAsync)( + fh, bufPtr_base, size_p, file_offset_p, bufPtr_offset_p, bytes_written_p, stream) + + +cdef CUfileError_t _cuFileStreamRegister(CUstream stream, unsigned flags) except?CUFILE_LOADING_ERROR nogil: + global __cuFileStreamRegister + _check_or_init_cufile() + if __cuFileStreamRegister == NULL: + with gil: + raise FunctionNotFoundError("function cuFileStreamRegister is not found") + return (__cuFileStreamRegister)( + stream, flags) + + +cdef CUfileError_t _cuFileStreamDeregister(CUstream stream) except?CUFILE_LOADING_ERROR nogil: + global __cuFileStreamDeregister + _check_or_init_cufile() + if __cuFileStreamDeregister == NULL: + with gil: + raise FunctionNotFoundError("function cuFileStreamDeregister is not found") + return (__cuFileStreamDeregister)( + stream) + + +cdef CUfileError_t _cuFileGetVersion(int* version) except?CUFILE_LOADING_ERROR nogil: + global __cuFileGetVersion + _check_or_init_cufile() + if __cuFileGetVersion == NULL: + with gil: + raise FunctionNotFoundError("function cuFileGetVersion is not found") + return (__cuFileGetVersion)( + version) + + +cdef CUfileError_t _cuFileGetParameterSizeT(CUFileSizeTConfigParameter_t param, size_t* value) except?CUFILE_LOADING_ERROR nogil: + global __cuFileGetParameterSizeT + _check_or_init_cufile() + if __cuFileGetParameterSizeT == NULL: + with gil: + raise FunctionNotFoundError("function cuFileGetParameterSizeT is not found") + return (__cuFileGetParameterSizeT)( + param, value) + + +cdef CUfileError_t _cuFileGetParameterBool(CUFileBoolConfigParameter_t param, cpp_bool* value) except?CUFILE_LOADING_ERROR nogil: + global __cuFileGetParameterBool + _check_or_init_cufile() + if __cuFileGetParameterBool == NULL: + with gil: + raise FunctionNotFoundError("function cuFileGetParameterBool is not found") + return (__cuFileGetParameterBool)( + param, value) + + +cdef CUfileError_t _cuFileGetParameterString(CUFileStringConfigParameter_t param, char* desc_str, int len) except?CUFILE_LOADING_ERROR nogil: + global __cuFileGetParameterString + _check_or_init_cufile() + if __cuFileGetParameterString == NULL: + with gil: + raise FunctionNotFoundError("function cuFileGetParameterString is not found") + return (__cuFileGetParameterString)( + param, desc_str, len) + + +cdef CUfileError_t _cuFileSetParameterSizeT(CUFileSizeTConfigParameter_t param, size_t value) except?CUFILE_LOADING_ERROR nogil: + global __cuFileSetParameterSizeT + _check_or_init_cufile() + if __cuFileSetParameterSizeT == NULL: + with gil: + raise FunctionNotFoundError("function cuFileSetParameterSizeT is not found") + return (__cuFileSetParameterSizeT)( + param, value) + + +cdef CUfileError_t _cuFileSetParameterBool(CUFileBoolConfigParameter_t param, cpp_bool value) except?CUFILE_LOADING_ERROR nogil: + global __cuFileSetParameterBool + _check_or_init_cufile() + if __cuFileSetParameterBool == NULL: + with gil: + raise FunctionNotFoundError("function cuFileSetParameterBool is not found") + return (__cuFileSetParameterBool)( + param, value) + + +cdef CUfileError_t _cuFileSetParameterString(CUFileStringConfigParameter_t param, const char* desc_str) except?CUFILE_LOADING_ERROR nogil: + global __cuFileSetParameterString + _check_or_init_cufile() + if __cuFileSetParameterString == NULL: + with gil: + raise FunctionNotFoundError("function cuFileSetParameterString is not found") + return (__cuFileSetParameterString)( + param, desc_str) + + +cdef CUfileError_t _cuFileDriverClose() except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverClose + _check_or_init_cufile() + if __cuFileDriverClose == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverClose is not found") + return (__cuFileDriverClose)( + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/cufile_linux.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/cufile_linux.pyx new file mode 100644 index 0000000000000000000000000000000000000000..dafeec2816a941bafd86f8f1a135b7e4f06117d1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/cufile_linux.pyx @@ -0,0 +1,745 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.9.0 to 12.9.1. Do not modify it directly. + +from libc.stdint cimport intptr_t, uintptr_t +import threading + +from .utils import FunctionNotFoundError, NotSupportedError + +from cuda.pathfinder import load_nvidia_dynamic_lib + +import cython + + +############################################################################### +# Extern +############################################################################### + +cdef extern from "" nogil: + void* dlopen(const char*, int) + char* dlerror() + void* dlsym(void*, const char*) + int dlclose(void*) + + enum: + RTLD_LAZY + RTLD_NOW + RTLD_GLOBAL + RTLD_LOCAL + + const void* RTLD_DEFAULT 'RTLD_DEFAULT' + + +############################################################################### +# Wrapper init +############################################################################### + +cdef object __symbol_lock = threading.Lock() +cdef bint __py_cufile_init = False +cdef void* __cuDriverGetVersion = NULL + +cdef void* __cuFileHandleRegister = NULL +cdef void* __cuFileHandleDeregister = NULL +cdef void* __cuFileBufRegister = NULL +cdef void* __cuFileBufDeregister = NULL +cdef void* __cuFileRead = NULL +cdef void* __cuFileWrite = NULL +cdef void* __cuFileDriverOpen = NULL +cdef void* __cuFileDriverClose_v2 = NULL +cdef void* __cuFileUseCount = NULL +cdef void* __cuFileDriverGetProperties = NULL +cdef void* __cuFileDriverSetPollMode = NULL +cdef void* __cuFileDriverSetMaxDirectIOSize = NULL +cdef void* __cuFileDriverSetMaxCacheSize = NULL +cdef void* __cuFileDriverSetMaxPinnedMemSize = NULL +cdef void* __cuFileBatchIOSetUp = NULL +cdef void* __cuFileBatchIOSubmit = NULL +cdef void* __cuFileBatchIOGetStatus = NULL +cdef void* __cuFileBatchIOCancel = NULL +cdef void* __cuFileBatchIODestroy = NULL +cdef void* __cuFileReadAsync = NULL +cdef void* __cuFileWriteAsync = NULL +cdef void* __cuFileStreamRegister = NULL +cdef void* __cuFileStreamDeregister = NULL +cdef void* __cuFileGetVersion = NULL +cdef void* __cuFileGetParameterSizeT = NULL +cdef void* __cuFileGetParameterBool = NULL +cdef void* __cuFileGetParameterString = NULL +cdef void* __cuFileSetParameterSizeT = NULL +cdef void* __cuFileSetParameterBool = NULL +cdef void* __cuFileSetParameterString = NULL +cdef void* __cuFileDriverClose = NULL + + +cdef void* load_library() except* with gil: + cdef uintptr_t handle = load_nvidia_dynamic_lib("cufile")._handle_uint + return handle + + +cdef int __check_or_init_cufile() except -1 nogil: + global __py_cufile_init + + cdef void* handle = NULL + + with gil, __symbol_lock: + # Load function + global __cuFileHandleRegister + __cuFileHandleRegister = dlsym(RTLD_DEFAULT, 'cuFileHandleRegister') + if __cuFileHandleRegister == NULL: + if handle == NULL: + handle = load_library() + __cuFileHandleRegister = dlsym(handle, 'cuFileHandleRegister') + + global __cuFileHandleDeregister + __cuFileHandleDeregister = dlsym(RTLD_DEFAULT, 'cuFileHandleDeregister') + if __cuFileHandleDeregister == NULL: + if handle == NULL: + handle = load_library() + __cuFileHandleDeregister = dlsym(handle, 'cuFileHandleDeregister') + + global __cuFileBufRegister + __cuFileBufRegister = dlsym(RTLD_DEFAULT, 'cuFileBufRegister') + if __cuFileBufRegister == NULL: + if handle == NULL: + handle = load_library() + __cuFileBufRegister = dlsym(handle, 'cuFileBufRegister') + + global __cuFileBufDeregister + __cuFileBufDeregister = dlsym(RTLD_DEFAULT, 'cuFileBufDeregister') + if __cuFileBufDeregister == NULL: + if handle == NULL: + handle = load_library() + __cuFileBufDeregister = dlsym(handle, 'cuFileBufDeregister') + + global __cuFileRead + __cuFileRead = dlsym(RTLD_DEFAULT, 'cuFileRead') + if __cuFileRead == NULL: + if handle == NULL: + handle = load_library() + __cuFileRead = dlsym(handle, 'cuFileRead') + + global __cuFileWrite + __cuFileWrite = dlsym(RTLD_DEFAULT, 'cuFileWrite') + if __cuFileWrite == NULL: + if handle == NULL: + handle = load_library() + __cuFileWrite = dlsym(handle, 'cuFileWrite') + + global __cuFileDriverOpen + __cuFileDriverOpen = dlsym(RTLD_DEFAULT, 'cuFileDriverOpen') + if __cuFileDriverOpen == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverOpen = dlsym(handle, 'cuFileDriverOpen') + + global __cuFileDriverClose_v2 + __cuFileDriverClose_v2 = dlsym(RTLD_DEFAULT, 'cuFileDriverClose_v2') + if __cuFileDriverClose_v2 == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverClose_v2 = dlsym(handle, 'cuFileDriverClose_v2') + + global __cuFileUseCount + __cuFileUseCount = dlsym(RTLD_DEFAULT, 'cuFileUseCount') + if __cuFileUseCount == NULL: + if handle == NULL: + handle = load_library() + __cuFileUseCount = dlsym(handle, 'cuFileUseCount') + + global __cuFileDriverGetProperties + __cuFileDriverGetProperties = dlsym(RTLD_DEFAULT, 'cuFileDriverGetProperties') + if __cuFileDriverGetProperties == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverGetProperties = dlsym(handle, 'cuFileDriverGetProperties') + + global __cuFileDriverSetPollMode + __cuFileDriverSetPollMode = dlsym(RTLD_DEFAULT, 'cuFileDriverSetPollMode') + if __cuFileDriverSetPollMode == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverSetPollMode = dlsym(handle, 'cuFileDriverSetPollMode') + + global __cuFileDriverSetMaxDirectIOSize + __cuFileDriverSetMaxDirectIOSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxDirectIOSize') + if __cuFileDriverSetMaxDirectIOSize == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverSetMaxDirectIOSize = dlsym(handle, 'cuFileDriverSetMaxDirectIOSize') + + global __cuFileDriverSetMaxCacheSize + __cuFileDriverSetMaxCacheSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxCacheSize') + if __cuFileDriverSetMaxCacheSize == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverSetMaxCacheSize = dlsym(handle, 'cuFileDriverSetMaxCacheSize') + + global __cuFileDriverSetMaxPinnedMemSize + __cuFileDriverSetMaxPinnedMemSize = dlsym(RTLD_DEFAULT, 'cuFileDriverSetMaxPinnedMemSize') + if __cuFileDriverSetMaxPinnedMemSize == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverSetMaxPinnedMemSize = dlsym(handle, 'cuFileDriverSetMaxPinnedMemSize') + + global __cuFileBatchIOSetUp + __cuFileBatchIOSetUp = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSetUp') + if __cuFileBatchIOSetUp == NULL: + if handle == NULL: + handle = load_library() + __cuFileBatchIOSetUp = dlsym(handle, 'cuFileBatchIOSetUp') + + global __cuFileBatchIOSubmit + __cuFileBatchIOSubmit = dlsym(RTLD_DEFAULT, 'cuFileBatchIOSubmit') + if __cuFileBatchIOSubmit == NULL: + if handle == NULL: + handle = load_library() + __cuFileBatchIOSubmit = dlsym(handle, 'cuFileBatchIOSubmit') + + global __cuFileBatchIOGetStatus + __cuFileBatchIOGetStatus = dlsym(RTLD_DEFAULT, 'cuFileBatchIOGetStatus') + if __cuFileBatchIOGetStatus == NULL: + if handle == NULL: + handle = load_library() + __cuFileBatchIOGetStatus = dlsym(handle, 'cuFileBatchIOGetStatus') + + global __cuFileBatchIOCancel + __cuFileBatchIOCancel = dlsym(RTLD_DEFAULT, 'cuFileBatchIOCancel') + if __cuFileBatchIOCancel == NULL: + if handle == NULL: + handle = load_library() + __cuFileBatchIOCancel = dlsym(handle, 'cuFileBatchIOCancel') + + global __cuFileBatchIODestroy + __cuFileBatchIODestroy = dlsym(RTLD_DEFAULT, 'cuFileBatchIODestroy') + if __cuFileBatchIODestroy == NULL: + if handle == NULL: + handle = load_library() + __cuFileBatchIODestroy = dlsym(handle, 'cuFileBatchIODestroy') + + global __cuFileReadAsync + __cuFileReadAsync = dlsym(RTLD_DEFAULT, 'cuFileReadAsync') + if __cuFileReadAsync == NULL: + if handle == NULL: + handle = load_library() + __cuFileReadAsync = dlsym(handle, 'cuFileReadAsync') + + global __cuFileWriteAsync + __cuFileWriteAsync = dlsym(RTLD_DEFAULT, 'cuFileWriteAsync') + if __cuFileWriteAsync == NULL: + if handle == NULL: + handle = load_library() + __cuFileWriteAsync = dlsym(handle, 'cuFileWriteAsync') + + global __cuFileStreamRegister + __cuFileStreamRegister = dlsym(RTLD_DEFAULT, 'cuFileStreamRegister') + if __cuFileStreamRegister == NULL: + if handle == NULL: + handle = load_library() + __cuFileStreamRegister = dlsym(handle, 'cuFileStreamRegister') + + global __cuFileStreamDeregister + __cuFileStreamDeregister = dlsym(RTLD_DEFAULT, 'cuFileStreamDeregister') + if __cuFileStreamDeregister == NULL: + if handle == NULL: + handle = load_library() + __cuFileStreamDeregister = dlsym(handle, 'cuFileStreamDeregister') + + global __cuFileGetVersion + __cuFileGetVersion = dlsym(RTLD_DEFAULT, 'cuFileGetVersion') + if __cuFileGetVersion == NULL: + if handle == NULL: + handle = load_library() + __cuFileGetVersion = dlsym(handle, 'cuFileGetVersion') + + global __cuFileGetParameterSizeT + __cuFileGetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileGetParameterSizeT') + if __cuFileGetParameterSizeT == NULL: + if handle == NULL: + handle = load_library() + __cuFileGetParameterSizeT = dlsym(handle, 'cuFileGetParameterSizeT') + + global __cuFileGetParameterBool + __cuFileGetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileGetParameterBool') + if __cuFileGetParameterBool == NULL: + if handle == NULL: + handle = load_library() + __cuFileGetParameterBool = dlsym(handle, 'cuFileGetParameterBool') + + global __cuFileGetParameterString + __cuFileGetParameterString = dlsym(RTLD_DEFAULT, 'cuFileGetParameterString') + if __cuFileGetParameterString == NULL: + if handle == NULL: + handle = load_library() + __cuFileGetParameterString = dlsym(handle, 'cuFileGetParameterString') + + global __cuFileSetParameterSizeT + __cuFileSetParameterSizeT = dlsym(RTLD_DEFAULT, 'cuFileSetParameterSizeT') + if __cuFileSetParameterSizeT == NULL: + if handle == NULL: + handle = load_library() + __cuFileSetParameterSizeT = dlsym(handle, 'cuFileSetParameterSizeT') + + global __cuFileSetParameterBool + __cuFileSetParameterBool = dlsym(RTLD_DEFAULT, 'cuFileSetParameterBool') + if __cuFileSetParameterBool == NULL: + if handle == NULL: + handle = load_library() + __cuFileSetParameterBool = dlsym(handle, 'cuFileSetParameterBool') + + global __cuFileSetParameterString + __cuFileSetParameterString = dlsym(RTLD_DEFAULT, 'cuFileSetParameterString') + if __cuFileSetParameterString == NULL: + if handle == NULL: + handle = load_library() + __cuFileSetParameterString = dlsym(handle, 'cuFileSetParameterString') + + global __cuFileDriverClose + __cuFileDriverClose = dlsym(RTLD_DEFAULT, 'cuFileDriverClose') + if __cuFileDriverClose == NULL: + if handle == NULL: + handle = load_library() + __cuFileDriverClose = dlsym(handle, 'cuFileDriverClose') + + __py_cufile_init = True + return 0 + + +cdef inline int _check_or_init_cufile() except -1 nogil: + if __py_cufile_init: + return 0 + + return __check_or_init_cufile() + + +cdef dict func_ptrs = None + + +cpdef dict _inspect_function_pointers(): + global func_ptrs + if func_ptrs is not None: + return func_ptrs + + _check_or_init_cufile() + cdef dict data = {} + + global __cuFileHandleRegister + data["__cuFileHandleRegister"] = __cuFileHandleRegister + + global __cuFileHandleDeregister + data["__cuFileHandleDeregister"] = __cuFileHandleDeregister + + global __cuFileBufRegister + data["__cuFileBufRegister"] = __cuFileBufRegister + + global __cuFileBufDeregister + data["__cuFileBufDeregister"] = __cuFileBufDeregister + + global __cuFileRead + data["__cuFileRead"] = __cuFileRead + + global __cuFileWrite + data["__cuFileWrite"] = __cuFileWrite + + global __cuFileDriverOpen + data["__cuFileDriverOpen"] = __cuFileDriverOpen + + global __cuFileDriverClose_v2 + data["__cuFileDriverClose_v2"] = __cuFileDriverClose_v2 + + global __cuFileUseCount + data["__cuFileUseCount"] = __cuFileUseCount + + global __cuFileDriverGetProperties + data["__cuFileDriverGetProperties"] = __cuFileDriverGetProperties + + global __cuFileDriverSetPollMode + data["__cuFileDriverSetPollMode"] = __cuFileDriverSetPollMode + + global __cuFileDriverSetMaxDirectIOSize + data["__cuFileDriverSetMaxDirectIOSize"] = __cuFileDriverSetMaxDirectIOSize + + global __cuFileDriverSetMaxCacheSize + data["__cuFileDriverSetMaxCacheSize"] = __cuFileDriverSetMaxCacheSize + + global __cuFileDriverSetMaxPinnedMemSize + data["__cuFileDriverSetMaxPinnedMemSize"] = __cuFileDriverSetMaxPinnedMemSize + + global __cuFileBatchIOSetUp + data["__cuFileBatchIOSetUp"] = __cuFileBatchIOSetUp + + global __cuFileBatchIOSubmit + data["__cuFileBatchIOSubmit"] = __cuFileBatchIOSubmit + + global __cuFileBatchIOGetStatus + data["__cuFileBatchIOGetStatus"] = __cuFileBatchIOGetStatus + + global __cuFileBatchIOCancel + data["__cuFileBatchIOCancel"] = __cuFileBatchIOCancel + + global __cuFileBatchIODestroy + data["__cuFileBatchIODestroy"] = __cuFileBatchIODestroy + + global __cuFileReadAsync + data["__cuFileReadAsync"] = __cuFileReadAsync + + global __cuFileWriteAsync + data["__cuFileWriteAsync"] = __cuFileWriteAsync + + global __cuFileStreamRegister + data["__cuFileStreamRegister"] = __cuFileStreamRegister + + global __cuFileStreamDeregister + data["__cuFileStreamDeregister"] = __cuFileStreamDeregister + + global __cuFileGetVersion + data["__cuFileGetVersion"] = __cuFileGetVersion + + global __cuFileGetParameterSizeT + data["__cuFileGetParameterSizeT"] = __cuFileGetParameterSizeT + + global __cuFileGetParameterBool + data["__cuFileGetParameterBool"] = __cuFileGetParameterBool + + global __cuFileGetParameterString + data["__cuFileGetParameterString"] = __cuFileGetParameterString + + global __cuFileSetParameterSizeT + data["__cuFileSetParameterSizeT"] = __cuFileSetParameterSizeT + + global __cuFileSetParameterBool + data["__cuFileSetParameterBool"] = __cuFileSetParameterBool + + global __cuFileSetParameterString + data["__cuFileSetParameterString"] = __cuFileSetParameterString + + global __cuFileDriverClose + data["__cuFileDriverClose"] = __cuFileDriverClose + + func_ptrs = data + return data + + +cpdef _inspect_function_pointer(str name): + global func_ptrs + if func_ptrs is None: + func_ptrs = _inspect_function_pointers() + return func_ptrs[name] + + +############################################################################### +# Wrapper functions +############################################################################### + +cdef CUfileError_t _cuFileHandleRegister(CUfileHandle_t* fh, CUfileDescr_t* descr) except?CUFILE_LOADING_ERROR nogil: + global __cuFileHandleRegister + _check_or_init_cufile() + if __cuFileHandleRegister == NULL: + with gil: + raise FunctionNotFoundError("function cuFileHandleRegister is not found") + return (__cuFileHandleRegister)( + fh, descr) + + +@cython.show_performance_hints(False) +cdef void _cuFileHandleDeregister(CUfileHandle_t fh) except* nogil: + global __cuFileHandleDeregister + _check_or_init_cufile() + if __cuFileHandleDeregister == NULL: + with gil: + raise FunctionNotFoundError("function cuFileHandleDeregister is not found") + (__cuFileHandleDeregister)( + fh) + + +cdef CUfileError_t _cuFileBufRegister(const void* bufPtr_base, size_t length, int flags) except?CUFILE_LOADING_ERROR nogil: + global __cuFileBufRegister + _check_or_init_cufile() + if __cuFileBufRegister == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBufRegister is not found") + return (__cuFileBufRegister)( + bufPtr_base, length, flags) + + +cdef CUfileError_t _cuFileBufDeregister(const void* bufPtr_base) except?CUFILE_LOADING_ERROR nogil: + global __cuFileBufDeregister + _check_or_init_cufile() + if __cuFileBufDeregister == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBufDeregister is not found") + return (__cuFileBufDeregister)( + bufPtr_base) + + +cdef ssize_t _cuFileRead(CUfileHandle_t fh, void* bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset) except* nogil: + global __cuFileRead + _check_or_init_cufile() + if __cuFileRead == NULL: + with gil: + raise FunctionNotFoundError("function cuFileRead is not found") + return (__cuFileRead)( + fh, bufPtr_base, size, file_offset, bufPtr_offset) + + +cdef ssize_t _cuFileWrite(CUfileHandle_t fh, const void* bufPtr_base, size_t size, off_t file_offset, off_t bufPtr_offset) except* nogil: + global __cuFileWrite + _check_or_init_cufile() + if __cuFileWrite == NULL: + with gil: + raise FunctionNotFoundError("function cuFileWrite is not found") + return (__cuFileWrite)( + fh, bufPtr_base, size, file_offset, bufPtr_offset) + + +cdef CUfileError_t _cuFileDriverOpen() except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverOpen + _check_or_init_cufile() + if __cuFileDriverOpen == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverOpen is not found") + return (__cuFileDriverOpen)( + ) + + +cdef CUfileError_t _cuFileDriverClose_v2() except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverClose_v2 + _check_or_init_cufile() + if __cuFileDriverClose_v2 == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverClose_v2 is not found") + return (__cuFileDriverClose_v2)( + ) + + +cdef long _cuFileUseCount() except* nogil: + global __cuFileUseCount + _check_or_init_cufile() + if __cuFileUseCount == NULL: + with gil: + raise FunctionNotFoundError("function cuFileUseCount is not found") + return (__cuFileUseCount)( + ) + + +cdef CUfileError_t _cuFileDriverGetProperties(CUfileDrvProps_t* props) except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverGetProperties + _check_or_init_cufile() + if __cuFileDriverGetProperties == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverGetProperties is not found") + return (__cuFileDriverGetProperties)( + props) + + +cdef CUfileError_t _cuFileDriverSetPollMode(cpp_bool poll, size_t poll_threshold_size) except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverSetPollMode + _check_or_init_cufile() + if __cuFileDriverSetPollMode == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverSetPollMode is not found") + return (__cuFileDriverSetPollMode)( + poll, poll_threshold_size) + + +cdef CUfileError_t _cuFileDriverSetMaxDirectIOSize(size_t max_direct_io_size) except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverSetMaxDirectIOSize + _check_or_init_cufile() + if __cuFileDriverSetMaxDirectIOSize == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverSetMaxDirectIOSize is not found") + return (__cuFileDriverSetMaxDirectIOSize)( + max_direct_io_size) + + +cdef CUfileError_t _cuFileDriverSetMaxCacheSize(size_t max_cache_size) except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverSetMaxCacheSize + _check_or_init_cufile() + if __cuFileDriverSetMaxCacheSize == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverSetMaxCacheSize is not found") + return (__cuFileDriverSetMaxCacheSize)( + max_cache_size) + + +cdef CUfileError_t _cuFileDriverSetMaxPinnedMemSize(size_t max_pinned_size) except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverSetMaxPinnedMemSize + _check_or_init_cufile() + if __cuFileDriverSetMaxPinnedMemSize == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverSetMaxPinnedMemSize is not found") + return (__cuFileDriverSetMaxPinnedMemSize)( + max_pinned_size) + + +cdef CUfileError_t _cuFileBatchIOSetUp(CUfileBatchHandle_t* batch_idp, unsigned nr) except?CUFILE_LOADING_ERROR nogil: + global __cuFileBatchIOSetUp + _check_or_init_cufile() + if __cuFileBatchIOSetUp == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBatchIOSetUp is not found") + return (__cuFileBatchIOSetUp)( + batch_idp, nr) + + +cdef CUfileError_t _cuFileBatchIOSubmit(CUfileBatchHandle_t batch_idp, unsigned nr, CUfileIOParams_t* iocbp, unsigned int flags) except?CUFILE_LOADING_ERROR nogil: + global __cuFileBatchIOSubmit + _check_or_init_cufile() + if __cuFileBatchIOSubmit == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBatchIOSubmit is not found") + return (__cuFileBatchIOSubmit)( + batch_idp, nr, iocbp, flags) + + +cdef CUfileError_t _cuFileBatchIOGetStatus(CUfileBatchHandle_t batch_idp, unsigned min_nr, unsigned* nr, CUfileIOEvents_t* iocbp, timespec* timeout) except?CUFILE_LOADING_ERROR nogil: + global __cuFileBatchIOGetStatus + _check_or_init_cufile() + if __cuFileBatchIOGetStatus == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBatchIOGetStatus is not found") + return (__cuFileBatchIOGetStatus)( + batch_idp, min_nr, nr, iocbp, timeout) + + +cdef CUfileError_t _cuFileBatchIOCancel(CUfileBatchHandle_t batch_idp) except?CUFILE_LOADING_ERROR nogil: + global __cuFileBatchIOCancel + _check_or_init_cufile() + if __cuFileBatchIOCancel == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBatchIOCancel is not found") + return (__cuFileBatchIOCancel)( + batch_idp) + + +@cython.show_performance_hints(False) +cdef void _cuFileBatchIODestroy(CUfileBatchHandle_t batch_idp) except* nogil: + global __cuFileBatchIODestroy + _check_or_init_cufile() + if __cuFileBatchIODestroy == NULL: + with gil: + raise FunctionNotFoundError("function cuFileBatchIODestroy is not found") + (__cuFileBatchIODestroy)( + batch_idp) + + +cdef CUfileError_t _cuFileReadAsync(CUfileHandle_t fh, void* bufPtr_base, size_t* size_p, off_t* file_offset_p, off_t* bufPtr_offset_p, ssize_t* bytes_read_p, CUstream stream) except?CUFILE_LOADING_ERROR nogil: + global __cuFileReadAsync + _check_or_init_cufile() + if __cuFileReadAsync == NULL: + with gil: + raise FunctionNotFoundError("function cuFileReadAsync is not found") + return (__cuFileReadAsync)( + fh, bufPtr_base, size_p, file_offset_p, bufPtr_offset_p, bytes_read_p, stream) + + +cdef CUfileError_t _cuFileWriteAsync(CUfileHandle_t fh, void* bufPtr_base, size_t* size_p, off_t* file_offset_p, off_t* bufPtr_offset_p, ssize_t* bytes_written_p, CUstream stream) except?CUFILE_LOADING_ERROR nogil: + global __cuFileWriteAsync + _check_or_init_cufile() + if __cuFileWriteAsync == NULL: + with gil: + raise FunctionNotFoundError("function cuFileWriteAsync is not found") + return (__cuFileWriteAsync)( + fh, bufPtr_base, size_p, file_offset_p, bufPtr_offset_p, bytes_written_p, stream) + + +cdef CUfileError_t _cuFileStreamRegister(CUstream stream, unsigned flags) except?CUFILE_LOADING_ERROR nogil: + global __cuFileStreamRegister + _check_or_init_cufile() + if __cuFileStreamRegister == NULL: + with gil: + raise FunctionNotFoundError("function cuFileStreamRegister is not found") + return (__cuFileStreamRegister)( + stream, flags) + + +cdef CUfileError_t _cuFileStreamDeregister(CUstream stream) except?CUFILE_LOADING_ERROR nogil: + global __cuFileStreamDeregister + _check_or_init_cufile() + if __cuFileStreamDeregister == NULL: + with gil: + raise FunctionNotFoundError("function cuFileStreamDeregister is not found") + return (__cuFileStreamDeregister)( + stream) + + +cdef CUfileError_t _cuFileGetVersion(int* version) except?CUFILE_LOADING_ERROR nogil: + global __cuFileGetVersion + _check_or_init_cufile() + if __cuFileGetVersion == NULL: + with gil: + raise FunctionNotFoundError("function cuFileGetVersion is not found") + return (__cuFileGetVersion)( + version) + + +cdef CUfileError_t _cuFileGetParameterSizeT(CUFileSizeTConfigParameter_t param, size_t* value) except?CUFILE_LOADING_ERROR nogil: + global __cuFileGetParameterSizeT + _check_or_init_cufile() + if __cuFileGetParameterSizeT == NULL: + with gil: + raise FunctionNotFoundError("function cuFileGetParameterSizeT is not found") + return (__cuFileGetParameterSizeT)( + param, value) + + +cdef CUfileError_t _cuFileGetParameterBool(CUFileBoolConfigParameter_t param, cpp_bool* value) except?CUFILE_LOADING_ERROR nogil: + global __cuFileGetParameterBool + _check_or_init_cufile() + if __cuFileGetParameterBool == NULL: + with gil: + raise FunctionNotFoundError("function cuFileGetParameterBool is not found") + return (__cuFileGetParameterBool)( + param, value) + + +cdef CUfileError_t _cuFileGetParameterString(CUFileStringConfigParameter_t param, char* desc_str, int len) except?CUFILE_LOADING_ERROR nogil: + global __cuFileGetParameterString + _check_or_init_cufile() + if __cuFileGetParameterString == NULL: + with gil: + raise FunctionNotFoundError("function cuFileGetParameterString is not found") + return (__cuFileGetParameterString)( + param, desc_str, len) + + +cdef CUfileError_t _cuFileSetParameterSizeT(CUFileSizeTConfigParameter_t param, size_t value) except?CUFILE_LOADING_ERROR nogil: + global __cuFileSetParameterSizeT + _check_or_init_cufile() + if __cuFileSetParameterSizeT == NULL: + with gil: + raise FunctionNotFoundError("function cuFileSetParameterSizeT is not found") + return (__cuFileSetParameterSizeT)( + param, value) + + +cdef CUfileError_t _cuFileSetParameterBool(CUFileBoolConfigParameter_t param, cpp_bool value) except?CUFILE_LOADING_ERROR nogil: + global __cuFileSetParameterBool + _check_or_init_cufile() + if __cuFileSetParameterBool == NULL: + with gil: + raise FunctionNotFoundError("function cuFileSetParameterBool is not found") + return (__cuFileSetParameterBool)( + param, value) + + +cdef CUfileError_t _cuFileSetParameterString(CUFileStringConfigParameter_t param, const char* desc_str) except?CUFILE_LOADING_ERROR nogil: + global __cuFileSetParameterString + _check_or_init_cufile() + if __cuFileSetParameterString == NULL: + with gil: + raise FunctionNotFoundError("function cuFileSetParameterString is not found") + return (__cuFileSetParameterString)( + param, desc_str) + + +cdef CUfileError_t _cuFileDriverClose() except?CUFILE_LOADING_ERROR nogil: + global __cuFileDriverClose + _check_or_init_cufile() + if __cuFileDriverClose == NULL: + with gil: + raise FunctionNotFoundError("function cuFileDriverClose is not found") + return (__cuFileDriverClose)( + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink.cpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink.cpp new file mode 100644 index 0000000000000000000000000000000000000000..602ae1716aa80831c87ae699ac2dbb8792f13e21 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink.cpp @@ -0,0 +1,12890 @@ +/* Generated by Cython 3.1.5 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [], + "extra_compile_args": [ + "-std=c++14", + "-fpermissive", + "-Wno-deprecated-declarations", + "-fno-var-tracking-assignments", + "-O3" + ], + "include_dirs": [ + "/opt/python/cp312-cp312/include", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include" + ], + "language": "c++", + "library_dirs": [ + "/tmp/build-env-3t3_8d9r/lib/python3.12/site-packages", + "/tmp/build-env-3t3_8d9r/lib", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib64", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib" + ], + "name": "cuda.bindings._internal.nvjitlink", + "sources": [ + "cuda/bindings/_internal/nvjitlink.pyx" + ] + }, + "module_name": "cuda.bindings._internal.nvjitlink" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_1_5" +#define CYTHON_HEX_VERSION 0x030105F0 +#define CYTHON_FUTURE_DIVISION 1 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef HAVE_LONG_LONG + #define HAVE_LONG_LONG +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_CLINE_IN_TRACEBACK + #define CYTHON_CLINE_IN_TRACEBACK 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1) +#endif +#endif +#define CYTHON_BACKPORT_VECTORCALL (CYTHON_METH_FASTCALL && PY_VERSION_HEX < 0x030800B1) +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME + #define CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME 100 +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + #endif + #endif + #if _MSC_VER < 1300 + #ifdef _WIN64 + typedef unsigned long long __pyx_uintptr_t; + #else + typedef unsigned int __pyx_uintptr_t; + #endif + #else + #ifdef _WIN64 + typedef unsigned __int64 __pyx_uintptr_t; + #else + typedef unsigned __int32 __pyx_uintptr_t; + #endif + #endif +#else + #include + typedef uintptr_t __pyx_uintptr_t; +#endif +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#elif CYTHON_BACKPORT_VECTORCALL + typedef PyObject *(*__pyx_vectorcallfunc)(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames); + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1)) + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(((size_t)(n)) & ~__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if __PYX_LIMITED_VERSION_HEX < 0x03090000 + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS && PY_VERSION_HEX >= 0x03080000 +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static CYTHON_INLINE PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__cuda__bindings___internal__nvjitlink +#define __PYX_HAVE_API__cuda__bindings___internal__nvjitlink +/* Early includes */ +#include +#include +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value); + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "cuda/bindings/_internal/nvjitlink.pyx", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* ForceInitThreads.proto */ +#ifndef __PYX_FORCE_INIT_THREADS + #define __PYX_FORCE_INIT_THREADS 0 +#endif + +/* NoFastGil.proto */ +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() + +/* Atomics.proto */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif +#if CYTHON_ATOMICS + #define __pyx_add_acquisition_count(memview)\ + __pyx_atomic_incr_relaxed(__pyx_get_slice_count_pointer(memview)) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_atomic_decr_acq_rel(__pyx_get_slice_count_pointer(memview)) +#else + #define __pyx_add_acquisition_count(memview)\ + __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_sub_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) +#endif + +/* IncludeStructmemberH.proto */ +#include + +/* CriticalSections.proto */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_Begin1(cs, arg) (void)cs +#define __Pyx_PyCriticalSection_Begin2(cs, arg1, arg2) (void)cs +#define __Pyx_PyCriticalSection_End1(cs) +#define __Pyx_PyCriticalSection_End2(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_Begin1 PyCriticalSection_Begin +#define __Pyx_PyCriticalSection_Begin2 PyCriticalSection2_Begin +#define __Pyx_PyCriticalSection_End1 PyCriticalSection_End +#define __Pyx_PyCriticalSection_End2 PyCriticalSection2_End +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* #### Code section: numeric_typedefs ### */ +/* #### Code section: complex_type_declarations ### */ +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ + +/* "cynvjitlink.pxd":15 + * + * # enums + * ctypedef enum nvJitLinkResult "nvJitLinkResult": # <<<<<<<<<<<<<< + * NVJITLINK_SUCCESS "NVJITLINK_SUCCESS" = 0 + * NVJITLINK_ERROR_UNRECOGNIZED_OPTION "NVJITLINK_ERROR_UNRECOGNIZED_OPTION" +*/ +enum nvJitLinkResult { + NVJITLINK_SUCCESS = 0, + NVJITLINK_ERROR_UNRECOGNIZED_OPTION, + NVJITLINK_ERROR_MISSING_ARCH, + NVJITLINK_ERROR_INVALID_INPUT, + NVJITLINK_ERROR_PTX_COMPILE, + NVJITLINK_ERROR_NVVM_COMPILE, + NVJITLINK_ERROR_INTERNAL, + NVJITLINK_ERROR_THREADPOOL, + NVJITLINK_ERROR_UNRECOGNIZED_INPUT, + NVJITLINK_ERROR_FINALIZE, + NVJITLINK_ERROR_NULL_INPUT, + NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS, + NVJITLINK_ERROR_INCORRECT_INPUT_TYPE, + NVJITLINK_ERROR_ARCH_MISMATCH, + NVJITLINK_ERROR_OUTDATED_LIBRARY, + NVJITLINK_ERROR_MISSING_FATBIN, + NVJITLINK_ERROR_UNRECOGNIZED_ARCH, + NVJITLINK_ERROR_UNSUPPORTED_ARCH, + NVJITLINK_ERROR_LTO_NOT_ENABLED, + _NVJITLINKRESULT_INTERNAL_LOADING_ERROR = -42L +}; +typedef enum nvJitLinkResult nvJitLinkResult; + +/* "cynvjitlink.pxd":37 + * _NVJITLINKRESULT_INTERNAL_LOADING_ERROR "_NVJITLINKRESULT_INTERNAL_LOADING_ERROR" = -42 + * + * ctypedef enum nvJitLinkInputType "nvJitLinkInputType": # <<<<<<<<<<<<<< + * NVJITLINK_INPUT_NONE "NVJITLINK_INPUT_NONE" = 0 + * NVJITLINK_INPUT_CUBIN "NVJITLINK_INPUT_CUBIN" = 1 +*/ +enum nvJitLinkInputType { + NVJITLINK_INPUT_NONE = 0, + NVJITLINK_INPUT_CUBIN = 1, + NVJITLINK_INPUT_PTX, + NVJITLINK_INPUT_LTOIR, + NVJITLINK_INPUT_FATBIN, + NVJITLINK_INPUT_OBJECT, + NVJITLINK_INPUT_LIBRARY, + NVJITLINK_INPUT_INDEX, + NVJITLINK_INPUT_ANY = 10 +}; +typedef enum nvJitLinkInputType nvJitLinkInputType; + +/* "cynvjitlink.pxd":50 + * + * # types + * ctypedef void* nvJitLinkHandle 'nvJitLinkHandle' # <<<<<<<<<<<<<< + * + * +*/ +typedef void *nvJitLinkHandle; +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyErrExceptionMatches.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* PyObjectGetAttrStrNoError.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* PyDictVersioning.proto */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __pyx_dict_cached_value;\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) do {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +#define __Pyx_GetModuleGlobalNameUncached(var, name) do {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#if !CYTHON_VECTORCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs); +#endif +#define __Pyx_BUILD_ASSERT_EXPR(cond)\ + (sizeof(char [1 - 2*!(cond)]) - 1) +#ifndef Py_MEMBER_SIZE +#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#endif +#if !CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03080000 + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets() + #define __Pyx_PyFrame_GetLocalsplus(frame) ((frame)->f_localsplus) +#else + static size_t __pyx_pyframe_localsplus_offset = 0; + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets()\ + ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ + (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) + #define __Pyx_PyFrame_GetLocalsplus(frame)\ + (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) +#endif +#endif +#endif + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* PyObjectLookupSpecial.proto */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 0) +#define __Pyx_PyObject_LookupSpecial(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 1) +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error); +#else +#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n) +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/* GetTopmostException.proto */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* TupleAndListFromArray.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +#endif +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/* IncludeStringH.proto */ +#include + +/* BytesEquals.proto */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); + +/* UnicodeEquals.proto */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); + +/* fastcall.proto */ +#if CYTHON_AVOID_BORROWED_REFS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_PySequence_ITEM(args, i) +#elif CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_NewRef(__Pyx_PyTuple_GET_ITEM(args, i)) +#else + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_XNewRef(PyTuple_GetItem(args, i)) +#endif +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_ArgRef_FASTCALL(args, i) __Pyx_NewRef(args[i]) + #define __Pyx_NumKwargs_FASTCALL(kwds) __Pyx_PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs)) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues); + #else + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) + #endif +#else + #define __Pyx_ArgRef_FASTCALL __Pyx_ArgRef_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#if CYTHON_METH_FASTCALL || (CYTHON_COMPILING_IN_CPYTHON && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(args + start, stop - start) +#else +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + +/* DictGetItem.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key); +#define __Pyx_PyObject_Dict_GetItem(obj, name)\ + (likely(PyDict_CheckExact(obj)) ?\ + __Pyx_PyDict_GetItem(obj, name) : PyObject_GetItem(obj, name)) +#else +#define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key) +#define __Pyx_PyObject_Dict_GetItem(obj, name) PyObject_GetItem(obj, name) +#endif + +/* RaiseDoubleKeywords.proto */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywords.proto */ +static CYTHON_INLINE int __Pyx_ParseKeywords( + PyObject *kwds, PyObject *const *kwvalues, PyObject ** const argnames[], + PyObject *kwds2, PyObject *values[], + Py_ssize_t num_pos_args, Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* CallCFunction.proto */ +#define __Pyx_CallCFunction(cfunc, self, args)\ + ((PyCFunction)(void(*)(void))(cfunc)->func)(self, args) +#define __Pyx_CallCFunctionWithKeywords(cfunc, self, args, kwargs)\ + ((PyCFunctionWithKeywords)(void(*)(void))(cfunc)->func)(self, args, kwargs) +#define __Pyx_CallCFunctionFast(cfunc, self, args, nargs)\ + ((__Pyx_PyCFunctionFast)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs) +#define __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, nargs, kwnames)\ + ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs, kwnames) + +/* UnpackUnboundCMethod.proto */ +typedef struct { + PyObject *type; + PyObject **method_name; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS + __pyx_atomic_int_type initialized; +#endif + PyCFunction func; + PyObject *method; + int flag; +} __Pyx_CachedCFunction; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +static CYTHON_INLINE int __Pyx_CachedCFunction_GetAndSetInitializing(__Pyx_CachedCFunction *cfunc) { +#if !CYTHON_ATOMICS + return 1; +#else + __pyx_nonatomic_int_type expected = 0; + if (__pyx_atomic_int_cmp_exchange(&cfunc->initialized, &expected, 1)) { + return 0; + } + return expected; +#endif +} +static CYTHON_INLINE void __Pyx_CachedCFunction_SetFinishedInitializing(__Pyx_CachedCFunction *cfunc) { +#if CYTHON_ATOMICS + __pyx_atomic_store(&cfunc->initialized, 2); +#endif +} +#else +#define __Pyx_CachedCFunction_GetAndSetInitializing(cfunc) 2 +#define __Pyx_CachedCFunction_SetFinishedInitializing(cfunc) +#endif + +/* CallUnboundCMethod2.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2); +#else +#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2) +#endif + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* ArgTypeTest.proto */ +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* RaiseException.proto */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* Import.proto */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); + +/* ImportDottedModule.proto */ +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple); +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple); + +/* ListPack.proto */ +static PyObject *__Pyx_PyList_Pack(Py_ssize_t n, ...); + +/* ImportFrom.proto */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +/* LimitedApiGetTypeDict.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp); +#endif + +/* SetItemOnTypeDict.proto */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v); +#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v) + +/* FixUpExtensionType.proto */ +static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); + +/* FetchSharedCythonModule.proto */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void); + +/* dict_setdefault.proto */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, int is_safe_type); + +/* FetchCommonType.proto */ +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases); + +/* CommonTypesMetaclass.proto */ +static int __pyx_CommonTypesMetaclass_init(PyObject *module); +#define __Pyx_CommonTypesMetaclass_USED + +/* CallTypeTraverse.proto */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0 +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg); +#endif + +/* PyMethodNew.proto */ +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ); + +/* PyVectorcallFastCallDict.proto */ +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); +#endif + +/* CythonFunctionShared.proto */ +#define __Pyx_CyFunction_USED +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f)\ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj)\ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject_HEAD + PyObject *func; +#elif PY_VERSION_HEX < 0x030900B1 + PyCFunctionObject func; +#else + PyCMethodObject func; +#endif +#if CYTHON_BACKPORT_VECTORCALL ||\ + (CYTHON_COMPILING_IN_LIMITED_API && CYTHON_METH_FASTCALL) + __pyx_vectorcallfunc func_vectorcall; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_weakreflist; +#endif + PyObject *func_dict; + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_classobj; +#endif + PyObject *defaults; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; + PyObject *func_is_coroutine; +} __pyx_CyFunctionObject; +#undef __Pyx_CyOrPyCFunction_Check +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_mstate_global->__pyx_CyFunctionType) +#define __Pyx_CyOrPyCFunction_Check(obj) __Pyx_TypeCheck2(obj, __pyx_mstate_global->__pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_CyFunctionType) +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)); +#undef __Pyx_IsSameCFunction +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCyOrCFunction(func, cfunc) +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, + PyTypeObject *defaults_type); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(PyObject *module); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_BACKPORT_VECTORCALL || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif + +/* CythonFunction.proto */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); + +/* CLineInTraceback.proto */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* GCCDiagnostics.proto */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* CIntFromPy.proto */ +static CYTHON_INLINE size_t __Pyx_PyLong_As_size_t(PyObject *); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* PyObjectVectorCallKwBuilder.proto */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +/* GetRuntimeVersion.proto */ +static unsigned long __Pyx_get_runtime_version(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* FunctionExport.proto */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + + +/* Module declarations from "libc.stdint" */ + +/* Module declarations from "cuda.bindings.cynvjitlink" */ + +/* Module declarations from "cuda.bindings._internal.nvjitlink" */ +static PyObject *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___symbol_lock = 0; +static int __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___py_nvjitlink_init; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___cuDriverGetVersion; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkCreate; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkDestroy; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddData; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddFile; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkComplete; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubinSize; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubin; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtxSize; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtx; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLogSize; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLog; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLogSize; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLog; +static void *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkVersion; +static PyObject *__pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs = 0; +static void *__pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(void); /*proto*/ +static int __pyx_f_4cuda_8bindings_9_internal_9nvjitlink___check_or_init_nvjitlink(void); /*proto*/ +static CYTHON_INLINE int __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(void); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__inspect_function_pointers(int __pyx_skip_dispatch); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__inspect_function_pointer(PyObject *, int __pyx_skip_dispatch); /*proto*/ +/* #### Code section: typeinfo ### */ +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "cuda.bindings._internal.nvjitlink" +extern int __pyx_module_is_main_cuda__bindings___internal__nvjitlink; +int __pyx_module_is_main_cuda__bindings___internal__nvjitlink = 0; + +/* Implementation of "cuda.bindings._internal.nvjitlink" */ +/* #### Code section: global_var ### */ +/* #### Code section: string_decls ### */ +static const char __pyx_k_[] = "."; +static const char __pyx_k__2[] = "?"; +static const char __pyx_k_pop[] = "pop"; +static const char __pyx_k_Lock[] = "Lock"; +static const char __pyx_k_exit[] = "__exit__"; +static const char __pyx_k_func[] = "__func__"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_name[] = "name"; +static const char __pyx_k_spec[] = "__spec__"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_enter[] = "__enter__"; +static const char __pyx_k_utils[] = "utils"; +static const char __pyx_k_module[] = "__module__"; +static const char __pyx_k_name_2[] = "__name__"; +static const char __pyx_k_add_note[] = "add_note"; +static const char __pyx_k_qualname[] = "__qualname__"; +static const char __pyx_k_set_name[] = "__set_name__"; +static const char __pyx_k_nvJitLink[] = "nvJitLink"; +static const char __pyx_k_threading[] = "threading"; +static const char __pyx_k_z_A_a_9AQ[] = "\200\001\330\004\005\330\004\007\200z\220\023\220A\330\010\024\320\024.\250a\330\004\013\2109\220A\220Q"; +static const char __pyx_k_handle_uint[] = "_handle_uint"; +static const char __pyx_k_initializing[] = "_initializing"; +static const char __pyx_k_is_coroutine[] = "_is_coroutine"; +static const char __pyx_k_cuda_pathfinder[] = "cuda.pathfinder"; +static const char __pyx_k_nvJitLinkCreate[] = "__nvJitLinkCreate"; +static const char __pyx_k_nvJitLinkAddData[] = "__nvJitLinkAddData"; +static const char __pyx_k_nvJitLinkAddFile[] = "__nvJitLinkAddFile"; +static const char __pyx_k_nvJitLinkDestroy[] = "__nvJitLinkDestroy"; +static const char __pyx_k_nvJitLinkVersion[] = "__nvJitLinkVersion"; +static const char __pyx_k_NotSupportedError[] = "NotSupportedError"; +static const char __pyx_k_nvJitLinkComplete[] = "__nvJitLinkComplete"; +static const char __pyx_k_asyncio_coroutines[] = "asyncio.coroutines"; +static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; +static const char __pyx_k_nvJitLinkGetInfoLog[] = "__nvJitLinkGetInfoLog"; +static const char __pyx_k_nvJitLinkGetErrorLog[] = "__nvJitLinkGetErrorLog"; +static const char __pyx_k_FunctionNotFoundError[] = "FunctionNotFoundError"; +static const char __pyx_k_nvJitLinkGetLinkedPtx[] = "__nvJitLinkGetLinkedPtx"; +static const char __pyx_k_load_nvidia_dynamic_lib[] = "load_nvidia_dynamic_lib"; +static const char __pyx_k_nvJitLinkGetInfoLogSize[] = "__nvJitLinkGetInfoLogSize"; +static const char __pyx_k_nvJitLinkGetLinkedCubin[] = "__nvJitLinkGetLinkedCubin"; +static const char __pyx_k_inspect_function_pointer[] = "_inspect_function_pointer"; +static const char __pyx_k_nvJitLinkGetErrorLogSize[] = "__nvJitLinkGetErrorLogSize"; +static const char __pyx_k_inspect_function_pointers[] = "_inspect_function_pointers"; +static const char __pyx_k_nvJitLinkGetLinkedPtxSize[] = "__nvJitLinkGetLinkedPtxSize"; +static const char __pyx_k_nvJitLinkGetLinkedCubinSize[] = "__nvJitLinkGetLinkedCubinSize"; +static const char __pyx_k_z_q_A_Q_1_1_1_A_Ja_A_j_1_Zq_Ja[] = "\200\001\330\004\005\330\004\007\200z\220\027\230\001\330\010\017\210q\340\004\034\230A\330\004\025\220Q\340\004\005\330\004\010\210\001\320\t \240\n\250!\340\004\005\330\004\010\210\001\320\t!\240\032\2501\340\004\005\330\004\010\210\001\320\t!\240\032\2501\340\004\005\330\004\010\210\001\320\t!\240\032\2501\340\004\005\330\004\010\210\001\320\t\"\240*\250A\340\004\005\330\004\010\210\001\320\t,\250J\260a\340\004\005\330\004\010\210\001\320\t(\250\n\260!\340\004\005\330\004\010\210\001\320\t*\250*\260A\340\004\005\330\004\010\210\001\320\t&\240j\260\001\340\004\005\330\004\010\210\001\320\t)\250\032\2601\340\004\005\330\004\010\210\001\320\t%\240Z\250q\340\004\005\330\004\010\210\001\320\t(\250\n\260!\340\004\005\330\004\010\210\001\320\t$\240J\250a\340\004\005\330\004\010\210\001\320\t!\240\032\2501\340\004\020\220\001\330\004\013\2101"; +static const char __pyx_k_function_nvJitLinkCreate_is_not[] = "function nvJitLinkCreate is not found"; +static const char __pyx_k_function_nvJitLinkGetInfoLog_is[] = "function nvJitLinkGetInfoLog is not found"; +static const char __pyx_k_Note_that_Cython_is_deliberately[] = "Note that Cython is deliberately stricter than PEP-484 and rejects subclasses of builtin types. If you need to pass subclasses then set the 'annotation_typing' directive to False."; +static const char __pyx_k_cuda_bindings__internal_nvjitlin[] = "cuda.bindings._internal.nvjitlink"; +static const char __pyx_k_function_nvJitLinkAddData_is_not[] = "function nvJitLinkAddData is not found"; +static const char __pyx_k_function_nvJitLinkAddFile_is_not[] = "function nvJitLinkAddFile is not found"; +static const char __pyx_k_function_nvJitLinkComplete_is_no[] = "function nvJitLinkComplete is not found"; +static const char __pyx_k_function_nvJitLinkDestroy_is_not[] = "function nvJitLinkDestroy is not found"; +static const char __pyx_k_function_nvJitLinkGetErrorLogSiz[] = "function nvJitLinkGetErrorLogSize is not found"; +static const char __pyx_k_function_nvJitLinkGetErrorLog_is[] = "function nvJitLinkGetErrorLog is not found"; +static const char __pyx_k_function_nvJitLinkGetInfoLogSize[] = "function nvJitLinkGetInfoLogSize is not found"; +static const char __pyx_k_function_nvJitLinkGetLinkedCubin[] = "function nvJitLinkGetLinkedCubinSize is not found"; +static const char __pyx_k_function_nvJitLinkGetLinkedPtxSi[] = "function nvJitLinkGetLinkedPtxSize is not found"; +static const char __pyx_k_function_nvJitLinkGetLinkedPtx_i[] = "function nvJitLinkGetLinkedPtx is not found"; +static const char __pyx_k_function_nvJitLinkVersion_is_not[] = "function nvJitLinkVersion is not found"; +static const char __pyx_k_cuda_bindings__internal_nvjitlin_2[] = "cuda/bindings/_internal/nvjitlink.pyx"; +static const char __pyx_k_function_nvJitLinkGetLinkedCubin_2[] = "function nvJitLinkGetLinkedCubin is not found"; +/* #### Code section: decls ### */ +static PyObject *__pyx_pf_4cuda_8bindings_9_internal_9nvjitlink__inspect_function_pointers(CYTHON_UNUSED PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_pf_4cuda_8bindings_9_internal_9nvjitlink_2_inspect_function_pointer(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_name); /* proto */ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + #ifdef __Pyx_CyFunction_USED + PyTypeObject *__pyx_CyFunctionType; + #endif + #ifdef __Pyx_FusedFunction_USED + PyTypeObject *__pyx_FusedFunctionType; + #endif + #ifdef __Pyx_Generator_USED + PyTypeObject *__pyx_GeneratorType; + #endif + #ifdef __Pyx_IterableCoroutine_USED + PyTypeObject *__pyx_IterableCoroutineType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineAwaitType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineType; + #endif + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop; + PyObject *__pyx_tuple[1]; + PyObject *__pyx_codeobj_tab[2]; + PyObject *__pyx_string_tab[61]; +/* #### Code section: module_state_contents ### */ +/* CommonTypesMetaclass.module_state_decls */ +PyTypeObject *__pyx_CommonTypesMetaclassType; + +/* CachedMethodType.module_state_decls */ +#if CYTHON_COMPILING_IN_LIMITED_API +PyObject *__Pyx_CachedMethodType; +#endif + +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_n_u_FunctionNotFoundError __pyx_string_tab[1] +#define __pyx_n_u_Lock __pyx_string_tab[2] +#define __pyx_n_u_NotSupportedError __pyx_string_tab[3] +#define __pyx_kp_u_Note_that_Cython_is_deliberately __pyx_string_tab[4] +#define __pyx_kp_u__2 __pyx_string_tab[5] +#define __pyx_kp_u_add_note __pyx_string_tab[6] +#define __pyx_n_u_asyncio_coroutines __pyx_string_tab[7] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[8] +#define __pyx_n_u_cuda_bindings__internal_nvjitlin __pyx_string_tab[9] +#define __pyx_kp_u_cuda_bindings__internal_nvjitlin_2 __pyx_string_tab[10] +#define __pyx_n_u_cuda_pathfinder __pyx_string_tab[11] +#define __pyx_n_u_enter __pyx_string_tab[12] +#define __pyx_n_u_exit __pyx_string_tab[13] +#define __pyx_n_u_func __pyx_string_tab[14] +#define __pyx_kp_u_function_nvJitLinkAddData_is_not __pyx_string_tab[15] +#define __pyx_kp_u_function_nvJitLinkAddFile_is_not __pyx_string_tab[16] +#define __pyx_kp_u_function_nvJitLinkComplete_is_no __pyx_string_tab[17] +#define __pyx_kp_u_function_nvJitLinkCreate_is_not __pyx_string_tab[18] +#define __pyx_kp_u_function_nvJitLinkDestroy_is_not __pyx_string_tab[19] +#define __pyx_kp_u_function_nvJitLinkGetErrorLogSiz __pyx_string_tab[20] +#define __pyx_kp_u_function_nvJitLinkGetErrorLog_is __pyx_string_tab[21] +#define __pyx_kp_u_function_nvJitLinkGetInfoLogSize __pyx_string_tab[22] +#define __pyx_kp_u_function_nvJitLinkGetInfoLog_is __pyx_string_tab[23] +#define __pyx_kp_u_function_nvJitLinkGetLinkedCubin __pyx_string_tab[24] +#define __pyx_kp_u_function_nvJitLinkGetLinkedCubin_2 __pyx_string_tab[25] +#define __pyx_kp_u_function_nvJitLinkGetLinkedPtxSi __pyx_string_tab[26] +#define __pyx_kp_u_function_nvJitLinkGetLinkedPtx_i __pyx_string_tab[27] +#define __pyx_kp_u_function_nvJitLinkVersion_is_not __pyx_string_tab[28] +#define __pyx_n_u_handle_uint __pyx_string_tab[29] +#define __pyx_n_u_initializing __pyx_string_tab[30] +#define __pyx_n_u_inspect_function_pointer __pyx_string_tab[31] +#define __pyx_n_u_inspect_function_pointers __pyx_string_tab[32] +#define __pyx_n_u_is_coroutine __pyx_string_tab[33] +#define __pyx_n_u_load_nvidia_dynamic_lib __pyx_string_tab[34] +#define __pyx_n_u_main __pyx_string_tab[35] +#define __pyx_n_u_module __pyx_string_tab[36] +#define __pyx_n_u_name __pyx_string_tab[37] +#define __pyx_n_u_name_2 __pyx_string_tab[38] +#define __pyx_n_u_nvJitLink __pyx_string_tab[39] +#define __pyx_n_u_nvJitLinkAddData __pyx_string_tab[40] +#define __pyx_n_u_nvJitLinkAddFile __pyx_string_tab[41] +#define __pyx_n_u_nvJitLinkComplete __pyx_string_tab[42] +#define __pyx_n_u_nvJitLinkCreate __pyx_string_tab[43] +#define __pyx_n_u_nvJitLinkDestroy __pyx_string_tab[44] +#define __pyx_n_u_nvJitLinkGetErrorLog __pyx_string_tab[45] +#define __pyx_n_u_nvJitLinkGetErrorLogSize __pyx_string_tab[46] +#define __pyx_n_u_nvJitLinkGetInfoLog __pyx_string_tab[47] +#define __pyx_n_u_nvJitLinkGetInfoLogSize __pyx_string_tab[48] +#define __pyx_n_u_nvJitLinkGetLinkedCubin __pyx_string_tab[49] +#define __pyx_n_u_nvJitLinkGetLinkedCubinSize __pyx_string_tab[50] +#define __pyx_n_u_nvJitLinkGetLinkedPtx __pyx_string_tab[51] +#define __pyx_n_u_nvJitLinkGetLinkedPtxSize __pyx_string_tab[52] +#define __pyx_n_u_nvJitLinkVersion __pyx_string_tab[53] +#define __pyx_n_u_pop __pyx_string_tab[54] +#define __pyx_n_u_qualname __pyx_string_tab[55] +#define __pyx_n_u_set_name __pyx_string_tab[56] +#define __pyx_n_u_spec __pyx_string_tab[57] +#define __pyx_n_u_test __pyx_string_tab[58] +#define __pyx_n_u_threading __pyx_string_tab[59] +#define __pyx_n_u_utils __pyx_string_tab[60] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_CLEAR(clear_module_state->__pyx_FusedFunctionType); + #endif + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + for (int i=0; i<1; ++i) { Py_CLEAR(clear_module_state->__pyx_tuple[i]); } + for (int i=0; i<2; ++i) { Py_CLEAR(clear_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<61; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_VISIT(traverse_module_state->__pyx_FusedFunctionType); + #endif + for (int i=0; i<1; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_tuple[i]); } + for (int i=0; i<2; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<61; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "cuda/bindings/_internal/nvjitlink.pyx":58 + * + * + * cdef void* load_library() except* with gil: # <<<<<<<<<<<<<< + * cdef uintptr_t handle = load_nvidia_dynamic_lib("nvJitLink")._handle_uint + * return handle +*/ + +static void *__pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(void) { + uintptr_t __pyx_v_handle; + void *__pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + uintptr_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("load_library", 0); + + /* "cuda/bindings/_internal/nvjitlink.pyx":59 + * + * cdef void* load_library() except* with gil: + * cdef uintptr_t handle = load_nvidia_dynamic_lib("nvJitLink")._handle_uint # <<<<<<<<<<<<<< + * return handle + * +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 59, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_mstate_global->__pyx_n_u_nvJitLink}; + __pyx_t_1 = __Pyx_PyObject_FastCall(__pyx_t_3, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 59, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_handle_uint); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 59, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_5 = __Pyx_PyLong_As_size_t(__pyx_t_3); if (unlikely((__pyx_t_5 == ((uintptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 59, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_handle = __pyx_t_5; + + /* "cuda/bindings/_internal/nvjitlink.pyx":60 + * cdef void* load_library() except* with gil: + * cdef uintptr_t handle = load_nvidia_dynamic_lib("nvJitLink")._handle_uint + * return handle # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = ((void *)__pyx_v_handle); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":58 + * + * + * cdef void* load_library() except* with gil: # <<<<<<<<<<<<<< + * cdef uintptr_t handle = load_nvidia_dynamic_lib("nvJitLink")._handle_uint + * return handle +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink.load_library", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":63 + * + * + * cdef int __check_or_init_nvjitlink() except -1 nogil: # <<<<<<<<<<<<<< + * global __py_nvjitlink_init + * +*/ + +static int __pyx_f_4cuda_8bindings_9_internal_9nvjitlink___check_or_init_nvjitlink(void) { + void *__pyx_v_handle; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + void *__pyx_t_10; + PyObject *__pyx_t_11 = NULL; + PyObject *__pyx_t_12 = NULL; + int __pyx_t_13; + int __pyx_t_14; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("__check_or_init_nvjitlink", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":66 + * global __py_nvjitlink_init + * + * cdef void* handle = NULL # <<<<<<<<<<<<<< + * + * with gil, __symbol_lock: +*/ + __pyx_v_handle = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":68 + * cdef void* handle = NULL + * + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * # Load function + * global __nvJitLinkCreate +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + /*with:*/ { + __pyx_t_1 = __Pyx_PyObject_LookupSpecial(__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___symbol_lock, __pyx_mstate_global->__pyx_n_u_exit); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 68, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = NULL; + __pyx_t_4 = __Pyx_PyObject_LookupSpecial(__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___symbol_lock, __pyx_mstate_global->__pyx_n_u_enter); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 68, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (likely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_4, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_4, __pyx_callargs+__pyx_t_5, (1-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 68, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + /*try:*/ { + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_6, &__pyx_t_7, &__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":71 + * # Load function + * global __nvJitLinkCreate + * __nvJitLinkCreate = dlsym(RTLD_DEFAULT, 'nvJitLinkCreate') # <<<<<<<<<<<<<< + * if __nvJitLinkCreate == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkCreate = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkCreate")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":72 + * global __nvJitLinkCreate + * __nvJitLinkCreate = dlsym(RTLD_DEFAULT, 'nvJitLinkCreate') + * if __nvJitLinkCreate == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkCreate == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":73 + * __nvJitLinkCreate = dlsym(RTLD_DEFAULT, 'nvJitLinkCreate') + * if __nvJitLinkCreate == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkCreate = dlsym(handle, 'nvJitLinkCreate') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":74 + * if __nvJitLinkCreate == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkCreate = dlsym(handle, 'nvJitLinkCreate') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 74, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":73 + * __nvJitLinkCreate = dlsym(RTLD_DEFAULT, 'nvJitLinkCreate') + * if __nvJitLinkCreate == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkCreate = dlsym(handle, 'nvJitLinkCreate') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":75 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkCreate = dlsym(handle, 'nvJitLinkCreate') # <<<<<<<<<<<<<< + * + * global __nvJitLinkDestroy +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkCreate = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkCreate")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":72 + * global __nvJitLinkCreate + * __nvJitLinkCreate = dlsym(RTLD_DEFAULT, 'nvJitLinkCreate') + * if __nvJitLinkCreate == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":78 + * + * global __nvJitLinkDestroy + * __nvJitLinkDestroy = dlsym(RTLD_DEFAULT, 'nvJitLinkDestroy') # <<<<<<<<<<<<<< + * if __nvJitLinkDestroy == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkDestroy = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkDestroy")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":79 + * global __nvJitLinkDestroy + * __nvJitLinkDestroy = dlsym(RTLD_DEFAULT, 'nvJitLinkDestroy') + * if __nvJitLinkDestroy == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkDestroy == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":80 + * __nvJitLinkDestroy = dlsym(RTLD_DEFAULT, 'nvJitLinkDestroy') + * if __nvJitLinkDestroy == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkDestroy = dlsym(handle, 'nvJitLinkDestroy') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":81 + * if __nvJitLinkDestroy == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkDestroy = dlsym(handle, 'nvJitLinkDestroy') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 81, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":80 + * __nvJitLinkDestroy = dlsym(RTLD_DEFAULT, 'nvJitLinkDestroy') + * if __nvJitLinkDestroy == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkDestroy = dlsym(handle, 'nvJitLinkDestroy') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":82 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkDestroy = dlsym(handle, 'nvJitLinkDestroy') # <<<<<<<<<<<<<< + * + * global __nvJitLinkAddData +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkDestroy = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkDestroy")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":79 + * global __nvJitLinkDestroy + * __nvJitLinkDestroy = dlsym(RTLD_DEFAULT, 'nvJitLinkDestroy') + * if __nvJitLinkDestroy == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":85 + * + * global __nvJitLinkAddData + * __nvJitLinkAddData = dlsym(RTLD_DEFAULT, 'nvJitLinkAddData') # <<<<<<<<<<<<<< + * if __nvJitLinkAddData == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddData = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkAddData")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":86 + * global __nvJitLinkAddData + * __nvJitLinkAddData = dlsym(RTLD_DEFAULT, 'nvJitLinkAddData') + * if __nvJitLinkAddData == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddData == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":87 + * __nvJitLinkAddData = dlsym(RTLD_DEFAULT, 'nvJitLinkAddData') + * if __nvJitLinkAddData == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkAddData = dlsym(handle, 'nvJitLinkAddData') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":88 + * if __nvJitLinkAddData == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkAddData = dlsym(handle, 'nvJitLinkAddData') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 88, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":87 + * __nvJitLinkAddData = dlsym(RTLD_DEFAULT, 'nvJitLinkAddData') + * if __nvJitLinkAddData == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkAddData = dlsym(handle, 'nvJitLinkAddData') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":89 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkAddData = dlsym(handle, 'nvJitLinkAddData') # <<<<<<<<<<<<<< + * + * global __nvJitLinkAddFile +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddData = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkAddData")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":86 + * global __nvJitLinkAddData + * __nvJitLinkAddData = dlsym(RTLD_DEFAULT, 'nvJitLinkAddData') + * if __nvJitLinkAddData == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":92 + * + * global __nvJitLinkAddFile + * __nvJitLinkAddFile = dlsym(RTLD_DEFAULT, 'nvJitLinkAddFile') # <<<<<<<<<<<<<< + * if __nvJitLinkAddFile == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddFile = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkAddFile")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":93 + * global __nvJitLinkAddFile + * __nvJitLinkAddFile = dlsym(RTLD_DEFAULT, 'nvJitLinkAddFile') + * if __nvJitLinkAddFile == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddFile == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":94 + * __nvJitLinkAddFile = dlsym(RTLD_DEFAULT, 'nvJitLinkAddFile') + * if __nvJitLinkAddFile == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkAddFile = dlsym(handle, 'nvJitLinkAddFile') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":95 + * if __nvJitLinkAddFile == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkAddFile = dlsym(handle, 'nvJitLinkAddFile') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 95, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":94 + * __nvJitLinkAddFile = dlsym(RTLD_DEFAULT, 'nvJitLinkAddFile') + * if __nvJitLinkAddFile == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkAddFile = dlsym(handle, 'nvJitLinkAddFile') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":96 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkAddFile = dlsym(handle, 'nvJitLinkAddFile') # <<<<<<<<<<<<<< + * + * global __nvJitLinkComplete +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddFile = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkAddFile")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":93 + * global __nvJitLinkAddFile + * __nvJitLinkAddFile = dlsym(RTLD_DEFAULT, 'nvJitLinkAddFile') + * if __nvJitLinkAddFile == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":99 + * + * global __nvJitLinkComplete + * __nvJitLinkComplete = dlsym(RTLD_DEFAULT, 'nvJitLinkComplete') # <<<<<<<<<<<<<< + * if __nvJitLinkComplete == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkComplete = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkComplete")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":100 + * global __nvJitLinkComplete + * __nvJitLinkComplete = dlsym(RTLD_DEFAULT, 'nvJitLinkComplete') + * if __nvJitLinkComplete == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkComplete == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":101 + * __nvJitLinkComplete = dlsym(RTLD_DEFAULT, 'nvJitLinkComplete') + * if __nvJitLinkComplete == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkComplete = dlsym(handle, 'nvJitLinkComplete') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":102 + * if __nvJitLinkComplete == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkComplete = dlsym(handle, 'nvJitLinkComplete') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 102, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":101 + * __nvJitLinkComplete = dlsym(RTLD_DEFAULT, 'nvJitLinkComplete') + * if __nvJitLinkComplete == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkComplete = dlsym(handle, 'nvJitLinkComplete') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":103 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkComplete = dlsym(handle, 'nvJitLinkComplete') # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetLinkedCubinSize +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkComplete = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkComplete")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":100 + * global __nvJitLinkComplete + * __nvJitLinkComplete = dlsym(RTLD_DEFAULT, 'nvJitLinkComplete') + * if __nvJitLinkComplete == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":106 + * + * global __nvJitLinkGetLinkedCubinSize + * __nvJitLinkGetLinkedCubinSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubinSize') # <<<<<<<<<<<<<< + * if __nvJitLinkGetLinkedCubinSize == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubinSize = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkGetLinkedCubinSize")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":107 + * global __nvJitLinkGetLinkedCubinSize + * __nvJitLinkGetLinkedCubinSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubinSize') + * if __nvJitLinkGetLinkedCubinSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubinSize == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":108 + * __nvJitLinkGetLinkedCubinSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubinSize') + * if __nvJitLinkGetLinkedCubinSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetLinkedCubinSize = dlsym(handle, 'nvJitLinkGetLinkedCubinSize') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":109 + * if __nvJitLinkGetLinkedCubinSize == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkGetLinkedCubinSize = dlsym(handle, 'nvJitLinkGetLinkedCubinSize') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 109, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":108 + * __nvJitLinkGetLinkedCubinSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubinSize') + * if __nvJitLinkGetLinkedCubinSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetLinkedCubinSize = dlsym(handle, 'nvJitLinkGetLinkedCubinSize') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":110 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkGetLinkedCubinSize = dlsym(handle, 'nvJitLinkGetLinkedCubinSize') # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetLinkedCubin +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubinSize = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkGetLinkedCubinSize")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":107 + * global __nvJitLinkGetLinkedCubinSize + * __nvJitLinkGetLinkedCubinSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubinSize') + * if __nvJitLinkGetLinkedCubinSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":113 + * + * global __nvJitLinkGetLinkedCubin + * __nvJitLinkGetLinkedCubin = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubin') # <<<<<<<<<<<<<< + * if __nvJitLinkGetLinkedCubin == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubin = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkGetLinkedCubin")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":114 + * global __nvJitLinkGetLinkedCubin + * __nvJitLinkGetLinkedCubin = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubin') + * if __nvJitLinkGetLinkedCubin == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubin == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":115 + * __nvJitLinkGetLinkedCubin = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubin') + * if __nvJitLinkGetLinkedCubin == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetLinkedCubin = dlsym(handle, 'nvJitLinkGetLinkedCubin') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":116 + * if __nvJitLinkGetLinkedCubin == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkGetLinkedCubin = dlsym(handle, 'nvJitLinkGetLinkedCubin') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 116, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":115 + * __nvJitLinkGetLinkedCubin = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubin') + * if __nvJitLinkGetLinkedCubin == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetLinkedCubin = dlsym(handle, 'nvJitLinkGetLinkedCubin') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":117 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkGetLinkedCubin = dlsym(handle, 'nvJitLinkGetLinkedCubin') # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetLinkedPtxSize +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubin = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkGetLinkedCubin")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":114 + * global __nvJitLinkGetLinkedCubin + * __nvJitLinkGetLinkedCubin = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubin') + * if __nvJitLinkGetLinkedCubin == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":120 + * + * global __nvJitLinkGetLinkedPtxSize + * __nvJitLinkGetLinkedPtxSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtxSize') # <<<<<<<<<<<<<< + * if __nvJitLinkGetLinkedPtxSize == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtxSize = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkGetLinkedPtxSize")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":121 + * global __nvJitLinkGetLinkedPtxSize + * __nvJitLinkGetLinkedPtxSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtxSize') + * if __nvJitLinkGetLinkedPtxSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtxSize == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":122 + * __nvJitLinkGetLinkedPtxSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtxSize') + * if __nvJitLinkGetLinkedPtxSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetLinkedPtxSize = dlsym(handle, 'nvJitLinkGetLinkedPtxSize') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":123 + * if __nvJitLinkGetLinkedPtxSize == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkGetLinkedPtxSize = dlsym(handle, 'nvJitLinkGetLinkedPtxSize') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 123, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":122 + * __nvJitLinkGetLinkedPtxSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtxSize') + * if __nvJitLinkGetLinkedPtxSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetLinkedPtxSize = dlsym(handle, 'nvJitLinkGetLinkedPtxSize') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":124 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkGetLinkedPtxSize = dlsym(handle, 'nvJitLinkGetLinkedPtxSize') # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetLinkedPtx +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtxSize = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkGetLinkedPtxSize")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":121 + * global __nvJitLinkGetLinkedPtxSize + * __nvJitLinkGetLinkedPtxSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtxSize') + * if __nvJitLinkGetLinkedPtxSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":127 + * + * global __nvJitLinkGetLinkedPtx + * __nvJitLinkGetLinkedPtx = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtx') # <<<<<<<<<<<<<< + * if __nvJitLinkGetLinkedPtx == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtx = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkGetLinkedPtx")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":128 + * global __nvJitLinkGetLinkedPtx + * __nvJitLinkGetLinkedPtx = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtx') + * if __nvJitLinkGetLinkedPtx == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtx == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":129 + * __nvJitLinkGetLinkedPtx = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtx') + * if __nvJitLinkGetLinkedPtx == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetLinkedPtx = dlsym(handle, 'nvJitLinkGetLinkedPtx') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":130 + * if __nvJitLinkGetLinkedPtx == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkGetLinkedPtx = dlsym(handle, 'nvJitLinkGetLinkedPtx') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 130, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":129 + * __nvJitLinkGetLinkedPtx = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtx') + * if __nvJitLinkGetLinkedPtx == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetLinkedPtx = dlsym(handle, 'nvJitLinkGetLinkedPtx') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":131 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkGetLinkedPtx = dlsym(handle, 'nvJitLinkGetLinkedPtx') # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetErrorLogSize +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtx = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkGetLinkedPtx")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":128 + * global __nvJitLinkGetLinkedPtx + * __nvJitLinkGetLinkedPtx = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtx') + * if __nvJitLinkGetLinkedPtx == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":134 + * + * global __nvJitLinkGetErrorLogSize + * __nvJitLinkGetErrorLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLogSize') # <<<<<<<<<<<<<< + * if __nvJitLinkGetErrorLogSize == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLogSize = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkGetErrorLogSize")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":135 + * global __nvJitLinkGetErrorLogSize + * __nvJitLinkGetErrorLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLogSize') + * if __nvJitLinkGetErrorLogSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLogSize == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":136 + * __nvJitLinkGetErrorLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLogSize') + * if __nvJitLinkGetErrorLogSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetErrorLogSize = dlsym(handle, 'nvJitLinkGetErrorLogSize') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":137 + * if __nvJitLinkGetErrorLogSize == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkGetErrorLogSize = dlsym(handle, 'nvJitLinkGetErrorLogSize') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 137, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":136 + * __nvJitLinkGetErrorLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLogSize') + * if __nvJitLinkGetErrorLogSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetErrorLogSize = dlsym(handle, 'nvJitLinkGetErrorLogSize') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":138 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkGetErrorLogSize = dlsym(handle, 'nvJitLinkGetErrorLogSize') # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetErrorLog +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLogSize = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkGetErrorLogSize")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":135 + * global __nvJitLinkGetErrorLogSize + * __nvJitLinkGetErrorLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLogSize') + * if __nvJitLinkGetErrorLogSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":141 + * + * global __nvJitLinkGetErrorLog + * __nvJitLinkGetErrorLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLog') # <<<<<<<<<<<<<< + * if __nvJitLinkGetErrorLog == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLog = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkGetErrorLog")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":142 + * global __nvJitLinkGetErrorLog + * __nvJitLinkGetErrorLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLog') + * if __nvJitLinkGetErrorLog == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLog == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":143 + * __nvJitLinkGetErrorLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLog') + * if __nvJitLinkGetErrorLog == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetErrorLog = dlsym(handle, 'nvJitLinkGetErrorLog') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":144 + * if __nvJitLinkGetErrorLog == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkGetErrorLog = dlsym(handle, 'nvJitLinkGetErrorLog') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 144, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":143 + * __nvJitLinkGetErrorLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLog') + * if __nvJitLinkGetErrorLog == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetErrorLog = dlsym(handle, 'nvJitLinkGetErrorLog') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":145 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkGetErrorLog = dlsym(handle, 'nvJitLinkGetErrorLog') # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetInfoLogSize +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLog = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkGetErrorLog")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":142 + * global __nvJitLinkGetErrorLog + * __nvJitLinkGetErrorLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLog') + * if __nvJitLinkGetErrorLog == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":148 + * + * global __nvJitLinkGetInfoLogSize + * __nvJitLinkGetInfoLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLogSize') # <<<<<<<<<<<<<< + * if __nvJitLinkGetInfoLogSize == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLogSize = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkGetInfoLogSize")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":149 + * global __nvJitLinkGetInfoLogSize + * __nvJitLinkGetInfoLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLogSize') + * if __nvJitLinkGetInfoLogSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLogSize == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":150 + * __nvJitLinkGetInfoLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLogSize') + * if __nvJitLinkGetInfoLogSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetInfoLogSize = dlsym(handle, 'nvJitLinkGetInfoLogSize') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":151 + * if __nvJitLinkGetInfoLogSize == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkGetInfoLogSize = dlsym(handle, 'nvJitLinkGetInfoLogSize') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 151, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":150 + * __nvJitLinkGetInfoLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLogSize') + * if __nvJitLinkGetInfoLogSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetInfoLogSize = dlsym(handle, 'nvJitLinkGetInfoLogSize') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":152 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkGetInfoLogSize = dlsym(handle, 'nvJitLinkGetInfoLogSize') # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetInfoLog +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLogSize = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkGetInfoLogSize")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":149 + * global __nvJitLinkGetInfoLogSize + * __nvJitLinkGetInfoLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLogSize') + * if __nvJitLinkGetInfoLogSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":155 + * + * global __nvJitLinkGetInfoLog + * __nvJitLinkGetInfoLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLog') # <<<<<<<<<<<<<< + * if __nvJitLinkGetInfoLog == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLog = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkGetInfoLog")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":156 + * global __nvJitLinkGetInfoLog + * __nvJitLinkGetInfoLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLog') + * if __nvJitLinkGetInfoLog == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLog == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":157 + * __nvJitLinkGetInfoLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLog') + * if __nvJitLinkGetInfoLog == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetInfoLog = dlsym(handle, 'nvJitLinkGetInfoLog') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":158 + * if __nvJitLinkGetInfoLog == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkGetInfoLog = dlsym(handle, 'nvJitLinkGetInfoLog') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 158, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":157 + * __nvJitLinkGetInfoLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLog') + * if __nvJitLinkGetInfoLog == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkGetInfoLog = dlsym(handle, 'nvJitLinkGetInfoLog') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":159 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkGetInfoLog = dlsym(handle, 'nvJitLinkGetInfoLog') # <<<<<<<<<<<<<< + * + * global __nvJitLinkVersion +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLog = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkGetInfoLog")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":156 + * global __nvJitLinkGetInfoLog + * __nvJitLinkGetInfoLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLog') + * if __nvJitLinkGetInfoLog == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":162 + * + * global __nvJitLinkVersion + * __nvJitLinkVersion = dlsym(RTLD_DEFAULT, 'nvJitLinkVersion') # <<<<<<<<<<<<<< + * if __nvJitLinkVersion == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkVersion = dlsym(RTLD_DEFAULT, ((char const *)"nvJitLinkVersion")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":163 + * global __nvJitLinkVersion + * __nvJitLinkVersion = dlsym(RTLD_DEFAULT, 'nvJitLinkVersion') + * if __nvJitLinkVersion == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkVersion == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":164 + * __nvJitLinkVersion = dlsym(RTLD_DEFAULT, 'nvJitLinkVersion') + * if __nvJitLinkVersion == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkVersion = dlsym(handle, 'nvJitLinkVersion') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":165 + * if __nvJitLinkVersion == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvJitLinkVersion = dlsym(handle, 'nvJitLinkVersion') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 165, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvjitlink.pyx":164 + * __nvJitLinkVersion = dlsym(RTLD_DEFAULT, 'nvJitLinkVersion') + * if __nvJitLinkVersion == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvJitLinkVersion = dlsym(handle, 'nvJitLinkVersion') +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":166 + * if handle == NULL: + * handle = load_library() + * __nvJitLinkVersion = dlsym(handle, 'nvJitLinkVersion') # <<<<<<<<<<<<<< + * + * __py_nvjitlink_init = True +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkVersion = dlsym(__pyx_v_handle, ((char const *)"nvJitLinkVersion")); + + /* "cuda/bindings/_internal/nvjitlink.pyx":163 + * global __nvJitLinkVersion + * __nvJitLinkVersion = dlsym(RTLD_DEFAULT, 'nvJitLinkVersion') + * if __nvJitLinkVersion == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":168 + * __nvJitLinkVersion = dlsym(handle, 'nvJitLinkVersion') + * + * __py_nvjitlink_init = True # <<<<<<<<<<<<<< + * return 0 + * +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___py_nvjitlink_init = 1; + + /* "cuda/bindings/_internal/nvjitlink.pyx":169 + * + * __py_nvjitlink_init = True + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L14_try_return; + + /* "cuda/bindings/_internal/nvjitlink.pyx":68 + * cdef void* handle = NULL + * + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * # Load function + * global __nvJitLinkCreate +*/ + } + __pyx_L10_error:; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + /*except:*/ { + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink.__check_or_init_nvjitlink", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_4, &__pyx_t_3) < 0) __PYX_ERR(0, 68, __pyx_L12_except_error) + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_3); + __pyx_t_11 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 68, __pyx_L12_except_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_11, NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 68, __pyx_L12_except_error) + __Pyx_GOTREF(__pyx_t_12); + __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_12); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + if (__pyx_t_9 < (0)) __PYX_ERR(0, 68, __pyx_L12_except_error) + __pyx_t_13 = (!__pyx_t_9); + if (unlikely(__pyx_t_13)) { + __Pyx_GIVEREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ErrRestoreWithState(__pyx_t_2, __pyx_t_4, __pyx_t_3); + __pyx_t_2 = 0; __pyx_t_4 = 0; __pyx_t_3 = 0; + __PYX_ERR(0, 68, __pyx_L12_except_error) + } + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L11_exception_handled; + } + __pyx_L12_except_error:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L4_error; + __pyx_L14_try_return:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L7_return; + __pyx_L11_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + } + } + /*finally:*/ { + /*normal exit:*/{ + if (__pyx_t_1) { + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_mstate_global->__pyx_tuple[0], NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 68, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + goto __pyx_L9; + } + __pyx_L7_return: { + __pyx_t_14 = __pyx_r; + if (__pyx_t_1) { + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_mstate_global->__pyx_tuple[0], NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 68, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + __pyx_r = __pyx_t_14; + goto __pyx_L3_return; + } + __pyx_L9:; + } + goto __pyx_L47; + __pyx_L6_error:; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L4_error; + __pyx_L47:; + } + } + /*finally:*/ { + /*normal exit:*/{ + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L5; + } + __pyx_L3_return: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L0; + } + __pyx_L4_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + __pyx_L5:; + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":63 + * + * + * cdef int __check_or_init_nvjitlink() except -1 nogil: # <<<<<<<<<<<<<< + * global __py_nvjitlink_init + * +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink.__check_or_init_nvjitlink", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":172 + * + * + * cdef inline int _check_or_init_nvjitlink() except -1 nogil: # <<<<<<<<<<<<<< + * if __py_nvjitlink_init: + * return 0 +*/ + +static CYTHON_INLINE int __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(void) { + int __pyx_r; + int __pyx_t_1; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_internal/nvjitlink.pyx":173 + * + * cdef inline int _check_or_init_nvjitlink() except -1 nogil: + * if __py_nvjitlink_init: # <<<<<<<<<<<<<< + * return 0 + * +*/ + if (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___py_nvjitlink_init) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":174 + * cdef inline int _check_or_init_nvjitlink() except -1 nogil: + * if __py_nvjitlink_init: + * return 0 # <<<<<<<<<<<<<< + * + * return __check_or_init_nvjitlink() +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":173 + * + * cdef inline int _check_or_init_nvjitlink() except -1 nogil: + * if __py_nvjitlink_init: # <<<<<<<<<<<<<< + * return 0 + * +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":176 + * return 0 + * + * return __check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * + * cdef dict func_ptrs = None +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink___check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 176, __pyx_L1_error) + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":172 + * + * + * cdef inline int _check_or_init_nvjitlink() except -1 nogil: # <<<<<<<<<<<<<< + * if __py_nvjitlink_init: + * return 0 +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._check_or_init_nvjitlink", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":181 + * + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_9nvjitlink_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__inspect_function_pointers(CYTHON_UNUSED int __pyx_skip_dispatch) { + PyObject *__pyx_v_data = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointers", 0); + + /* "cuda/bindings/_internal/nvjitlink.pyx":183 + * cpdef dict _inspect_function_pointers(): + * global func_ptrs + * if func_ptrs is not None: # <<<<<<<<<<<<<< + * return func_ptrs + * +*/ + __pyx_t_1 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs != ((PyObject*)Py_None)); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":184 + * global func_ptrs + * if func_ptrs is not None: + * return func_ptrs # <<<<<<<<<<<<<< + * + * _check_or_init_nvjitlink() +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs); + __pyx_r = __pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs; + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":183 + * cpdef dict _inspect_function_pointers(): + * global func_ptrs + * if func_ptrs is not None: # <<<<<<<<<<<<<< + * return func_ptrs + * +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":186 + * return func_ptrs + * + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * cdef dict data = {} + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(0, 186, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":187 + * + * _check_or_init_nvjitlink() + * cdef dict data = {} # <<<<<<<<<<<<<< + * + * global __nvJitLinkCreate +*/ + __pyx_t_3 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 187, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_v_data = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":190 + * + * global __nvJitLinkCreate + * data["__nvJitLinkCreate"] = __nvJitLinkCreate # <<<<<<<<<<<<<< + * + * global __nvJitLinkDestroy +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkCreate)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 190, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkCreate, __pyx_t_3) < 0))) __PYX_ERR(0, 190, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":193 + * + * global __nvJitLinkDestroy + * data["__nvJitLinkDestroy"] = __nvJitLinkDestroy # <<<<<<<<<<<<<< + * + * global __nvJitLinkAddData +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkDestroy)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkDestroy, __pyx_t_3) < 0))) __PYX_ERR(0, 193, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":196 + * + * global __nvJitLinkAddData + * data["__nvJitLinkAddData"] = __nvJitLinkAddData # <<<<<<<<<<<<<< + * + * global __nvJitLinkAddFile +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddData)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 196, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkAddData, __pyx_t_3) < 0))) __PYX_ERR(0, 196, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":199 + * + * global __nvJitLinkAddFile + * data["__nvJitLinkAddFile"] = __nvJitLinkAddFile # <<<<<<<<<<<<<< + * + * global __nvJitLinkComplete +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddFile)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 199, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkAddFile, __pyx_t_3) < 0))) __PYX_ERR(0, 199, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":202 + * + * global __nvJitLinkComplete + * data["__nvJitLinkComplete"] = __nvJitLinkComplete # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetLinkedCubinSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkComplete)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 202, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkComplete, __pyx_t_3) < 0))) __PYX_ERR(0, 202, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":205 + * + * global __nvJitLinkGetLinkedCubinSize + * data["__nvJitLinkGetLinkedCubinSize"] = __nvJitLinkGetLinkedCubinSize # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetLinkedCubin +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubinSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 205, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkGetLinkedCubinSize, __pyx_t_3) < 0))) __PYX_ERR(0, 205, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":208 + * + * global __nvJitLinkGetLinkedCubin + * data["__nvJitLinkGetLinkedCubin"] = __nvJitLinkGetLinkedCubin # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetLinkedPtxSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubin)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 208, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkGetLinkedCubin, __pyx_t_3) < 0))) __PYX_ERR(0, 208, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":211 + * + * global __nvJitLinkGetLinkedPtxSize + * data["__nvJitLinkGetLinkedPtxSize"] = __nvJitLinkGetLinkedPtxSize # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetLinkedPtx +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtxSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 211, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkGetLinkedPtxSize, __pyx_t_3) < 0))) __PYX_ERR(0, 211, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":214 + * + * global __nvJitLinkGetLinkedPtx + * data["__nvJitLinkGetLinkedPtx"] = __nvJitLinkGetLinkedPtx # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetErrorLogSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtx)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 214, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkGetLinkedPtx, __pyx_t_3) < 0))) __PYX_ERR(0, 214, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":217 + * + * global __nvJitLinkGetErrorLogSize + * data["__nvJitLinkGetErrorLogSize"] = __nvJitLinkGetErrorLogSize # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetErrorLog +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLogSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 217, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkGetErrorLogSize, __pyx_t_3) < 0))) __PYX_ERR(0, 217, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":220 + * + * global __nvJitLinkGetErrorLog + * data["__nvJitLinkGetErrorLog"] = __nvJitLinkGetErrorLog # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetInfoLogSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLog)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 220, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkGetErrorLog, __pyx_t_3) < 0))) __PYX_ERR(0, 220, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":223 + * + * global __nvJitLinkGetInfoLogSize + * data["__nvJitLinkGetInfoLogSize"] = __nvJitLinkGetInfoLogSize # <<<<<<<<<<<<<< + * + * global __nvJitLinkGetInfoLog +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLogSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 223, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkGetInfoLogSize, __pyx_t_3) < 0))) __PYX_ERR(0, 223, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":226 + * + * global __nvJitLinkGetInfoLog + * data["__nvJitLinkGetInfoLog"] = __nvJitLinkGetInfoLog # <<<<<<<<<<<<<< + * + * global __nvJitLinkVersion +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLog)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 226, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkGetInfoLog, __pyx_t_3) < 0))) __PYX_ERR(0, 226, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":229 + * + * global __nvJitLinkVersion + * data["__nvJitLinkVersion"] = __nvJitLinkVersion # <<<<<<<<<<<<<< + * + * func_ptrs = data +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkVersion)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 229, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvJitLinkVersion, __pyx_t_3) < 0))) __PYX_ERR(0, 229, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":231 + * data["__nvJitLinkVersion"] = __nvJitLinkVersion + * + * func_ptrs = data # <<<<<<<<<<<<<< + * return data + * +*/ + __Pyx_INCREF(__pyx_v_data); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs, __pyx_v_data); + __Pyx_GIVEREF(__pyx_v_data); + + /* "cuda/bindings/_internal/nvjitlink.pyx":232 + * + * func_ptrs = data + * return data # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_data); + __pyx_r = __pyx_v_data; + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":181 + * + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._inspect_function_pointers", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_data); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_9nvjitlink_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +PyDoc_STRVAR(__pyx_doc_4cuda_8bindings_9_internal_9nvjitlink__inspect_function_pointers, "_inspect_function_pointers() -> dict"); +static PyMethodDef __pyx_mdef_4cuda_8bindings_9_internal_9nvjitlink_1_inspect_function_pointers = {"_inspect_function_pointers", (PyCFunction)__pyx_pw_4cuda_8bindings_9_internal_9nvjitlink_1_inspect_function_pointers, METH_NOARGS, __pyx_doc_4cuda_8bindings_9_internal_9nvjitlink__inspect_function_pointers}; +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_9nvjitlink_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_inspect_function_pointers (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_4cuda_8bindings_9_internal_9nvjitlink__inspect_function_pointers(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_4cuda_8bindings_9_internal_9nvjitlink__inspect_function_pointers(CYTHON_UNUSED PyObject *__pyx_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointers", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__inspect_function_pointers(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 181, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._inspect_function_pointers", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":235 + * + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_9nvjitlink_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__inspect_function_pointer(PyObject *__pyx_v_name, CYTHON_UNUSED int __pyx_skip_dispatch) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointer", 0); + + /* "cuda/bindings/_internal/nvjitlink.pyx":237 + * cpdef _inspect_function_pointer(str name): + * global func_ptrs + * if func_ptrs is None: # <<<<<<<<<<<<<< + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] +*/ + __pyx_t_1 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs == ((PyObject*)Py_None)); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":238 + * global func_ptrs + * if func_ptrs is None: + * func_ptrs = _inspect_function_pointers() # <<<<<<<<<<<<<< + * return func_ptrs[name] + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__inspect_function_pointers(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 238, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs, ((PyObject*)__pyx_t_2)); + __Pyx_GIVEREF(__pyx_t_2); + __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":237 + * cpdef _inspect_function_pointer(str name): + * global func_ptrs + * if func_ptrs is None: # <<<<<<<<<<<<<< + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":239 + * if func_ptrs is None: + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + if (unlikely(__pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 239, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_PyDict_GetItem(__pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs, __pyx_v_name); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 239, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":235 + * + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_9nvjitlink_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_4cuda_8bindings_9_internal_9nvjitlink_2_inspect_function_pointer, "_inspect_function_pointer(str name)"); +static PyMethodDef __pyx_mdef_4cuda_8bindings_9_internal_9nvjitlink_3_inspect_function_pointer = {"_inspect_function_pointer", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cuda_8bindings_9_internal_9nvjitlink_3_inspect_function_pointer, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cuda_8bindings_9_internal_9nvjitlink_2_inspect_function_pointer}; +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_9nvjitlink_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_name = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_inspect_function_pointer (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_name,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 235, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 235, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_inspect_function_pointer", 0) < (0)) __PYX_ERR(0, 235, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_inspect_function_pointer", 1, 1, 1, i); __PYX_ERR(0, 235, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 235, __pyx_L3_error) + } + __pyx_v_name = ((PyObject*)values[0]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_inspect_function_pointer", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 235, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_name), (&PyUnicode_Type), 1, "name", 1))) __PYX_ERR(0, 235, __pyx_L1_error) + __pyx_r = __pyx_pf_4cuda_8bindings_9_internal_9nvjitlink_2_inspect_function_pointer(__pyx_self, __pyx_v_name); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_4cuda_8bindings_9_internal_9nvjitlink_2_inspect_function_pointer(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_name) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointer", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__inspect_function_pointer(__pyx_v_name, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 235, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":246 + * ############################################################################### + * + * cdef nvJitLinkResult _nvJitLinkCreate(nvJitLinkHandle* handle, uint32_t numOptions, const char** options) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkCreate + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkCreate(nvJitLinkHandle *__pyx_v_handle, uint32_t __pyx_v_numOptions, char const **__pyx_v_options) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkCreate", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":248 + * cdef nvJitLinkResult _nvJitLinkCreate(nvJitLinkHandle* handle, uint32_t numOptions, const char** options) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkCreate + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkCreate == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 248, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":249 + * global __nvJitLinkCreate + * _check_or_init_nvjitlink() + * if __nvJitLinkCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkCreate is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkCreate == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":250 + * _check_or_init_nvjitlink() + * if __nvJitLinkCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkCreate is not found") + * return (__nvJitLinkCreate)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":251 + * if __nvJitLinkCreate == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkCreate is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkCreate)( + * handle, numOptions, options) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 251, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkCreate_is_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 251, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 251, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":250 + * _check_or_init_nvjitlink() + * if __nvJitLinkCreate == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkCreate is not found") + * return (__nvJitLinkCreate)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":249 + * global __nvJitLinkCreate + * _check_or_init_nvjitlink() + * if __nvJitLinkCreate == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkCreate is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":252 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkCreate is not found") + * return (__nvJitLinkCreate)( # <<<<<<<<<<<<<< + * handle, numOptions, options) + * +*/ + __pyx_r = ((nvJitLinkResult (*)(nvJitLinkHandle *, uint32_t, char const **))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkCreate)(__pyx_v_handle, __pyx_v_numOptions, __pyx_v_options); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":246 + * ############################################################################### + * + * cdef nvJitLinkResult _nvJitLinkCreate(nvJitLinkHandle* handle, uint32_t numOptions, const char** options) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkCreate + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkCreate", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":256 + * + * + * cdef nvJitLinkResult _nvJitLinkDestroy(nvJitLinkHandle* handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkDestroy + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkDestroy(nvJitLinkHandle *__pyx_v_handle) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkDestroy", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":258 + * cdef nvJitLinkResult _nvJitLinkDestroy(nvJitLinkHandle* handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkDestroy + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkDestroy == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 258, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":259 + * global __nvJitLinkDestroy + * _check_or_init_nvjitlink() + * if __nvJitLinkDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkDestroy is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkDestroy == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":260 + * _check_or_init_nvjitlink() + * if __nvJitLinkDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkDestroy is not found") + * return (__nvJitLinkDestroy)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":261 + * if __nvJitLinkDestroy == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkDestroy is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkDestroy)( + * handle) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 261, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkDestroy_is_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 261, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 261, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":260 + * _check_or_init_nvjitlink() + * if __nvJitLinkDestroy == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkDestroy is not found") + * return (__nvJitLinkDestroy)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":259 + * global __nvJitLinkDestroy + * _check_or_init_nvjitlink() + * if __nvJitLinkDestroy == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkDestroy is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":262 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkDestroy is not found") + * return (__nvJitLinkDestroy)( # <<<<<<<<<<<<<< + * handle) + * +*/ + __pyx_r = ((nvJitLinkResult (*)(nvJitLinkHandle *))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkDestroy)(__pyx_v_handle); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":256 + * + * + * cdef nvJitLinkResult _nvJitLinkDestroy(nvJitLinkHandle* handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkDestroy + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkDestroy", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":266 + * + * + * cdef nvJitLinkResult _nvJitLinkAddData(nvJitLinkHandle handle, nvJitLinkInputType inputType, const void* data, size_t size, const char* name) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkAddData + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkAddData(nvJitLinkHandle __pyx_v_handle, nvJitLinkInputType __pyx_v_inputType, void const *__pyx_v_data, size_t __pyx_v_size, char const *__pyx_v_name) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkAddData", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":268 + * cdef nvJitLinkResult _nvJitLinkAddData(nvJitLinkHandle handle, nvJitLinkInputType inputType, const void* data, size_t size, const char* name) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkAddData + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkAddData == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 268, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":269 + * global __nvJitLinkAddData + * _check_or_init_nvjitlink() + * if __nvJitLinkAddData == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkAddData is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddData == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":270 + * _check_or_init_nvjitlink() + * if __nvJitLinkAddData == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkAddData is not found") + * return (__nvJitLinkAddData)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":271 + * if __nvJitLinkAddData == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkAddData is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkAddData)( + * handle, inputType, data, size, name) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 271, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkAddData_is_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 271, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 271, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":270 + * _check_or_init_nvjitlink() + * if __nvJitLinkAddData == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkAddData is not found") + * return (__nvJitLinkAddData)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":269 + * global __nvJitLinkAddData + * _check_or_init_nvjitlink() + * if __nvJitLinkAddData == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkAddData is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":272 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkAddData is not found") + * return (__nvJitLinkAddData)( # <<<<<<<<<<<<<< + * handle, inputType, data, size, name) + * +*/ + __pyx_r = ((nvJitLinkResult (*)(nvJitLinkHandle, nvJitLinkInputType, void const *, size_t, char const *))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddData)(__pyx_v_handle, __pyx_v_inputType, __pyx_v_data, __pyx_v_size, __pyx_v_name); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":266 + * + * + * cdef nvJitLinkResult _nvJitLinkAddData(nvJitLinkHandle handle, nvJitLinkInputType inputType, const void* data, size_t size, const char* name) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkAddData + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkAddData", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":276 + * + * + * cdef nvJitLinkResult _nvJitLinkAddFile(nvJitLinkHandle handle, nvJitLinkInputType inputType, const char* fileName) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkAddFile + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkAddFile(nvJitLinkHandle __pyx_v_handle, nvJitLinkInputType __pyx_v_inputType, char const *__pyx_v_fileName) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkAddFile", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":278 + * cdef nvJitLinkResult _nvJitLinkAddFile(nvJitLinkHandle handle, nvJitLinkInputType inputType, const char* fileName) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkAddFile + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkAddFile == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 278, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":279 + * global __nvJitLinkAddFile + * _check_or_init_nvjitlink() + * if __nvJitLinkAddFile == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkAddFile is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddFile == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":280 + * _check_or_init_nvjitlink() + * if __nvJitLinkAddFile == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkAddFile is not found") + * return (__nvJitLinkAddFile)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":281 + * if __nvJitLinkAddFile == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkAddFile is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkAddFile)( + * handle, inputType, fileName) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 281, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkAddFile_is_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 281, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 281, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":280 + * _check_or_init_nvjitlink() + * if __nvJitLinkAddFile == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkAddFile is not found") + * return (__nvJitLinkAddFile)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":279 + * global __nvJitLinkAddFile + * _check_or_init_nvjitlink() + * if __nvJitLinkAddFile == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkAddFile is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":282 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkAddFile is not found") + * return (__nvJitLinkAddFile)( # <<<<<<<<<<<<<< + * handle, inputType, fileName) + * +*/ + __pyx_r = ((nvJitLinkResult (*)(nvJitLinkHandle, nvJitLinkInputType, char const *))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddFile)(__pyx_v_handle, __pyx_v_inputType, __pyx_v_fileName); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":276 + * + * + * cdef nvJitLinkResult _nvJitLinkAddFile(nvJitLinkHandle handle, nvJitLinkInputType inputType, const char* fileName) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkAddFile + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkAddFile", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":286 + * + * + * cdef nvJitLinkResult _nvJitLinkComplete(nvJitLinkHandle handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkComplete + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkComplete(nvJitLinkHandle __pyx_v_handle) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkComplete", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":288 + * cdef nvJitLinkResult _nvJitLinkComplete(nvJitLinkHandle handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkComplete + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkComplete == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 288, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":289 + * global __nvJitLinkComplete + * _check_or_init_nvjitlink() + * if __nvJitLinkComplete == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkComplete is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkComplete == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":290 + * _check_or_init_nvjitlink() + * if __nvJitLinkComplete == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkComplete is not found") + * return (__nvJitLinkComplete)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":291 + * if __nvJitLinkComplete == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkComplete is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkComplete)( + * handle) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 291, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkComplete_is_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 291, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 291, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":290 + * _check_or_init_nvjitlink() + * if __nvJitLinkComplete == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkComplete is not found") + * return (__nvJitLinkComplete)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":289 + * global __nvJitLinkComplete + * _check_or_init_nvjitlink() + * if __nvJitLinkComplete == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkComplete is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":292 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkComplete is not found") + * return (__nvJitLinkComplete)( # <<<<<<<<<<<<<< + * handle) + * +*/ + __pyx_r = ((nvJitLinkResult (*)(nvJitLinkHandle))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkComplete)(__pyx_v_handle); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":286 + * + * + * cdef nvJitLinkResult _nvJitLinkComplete(nvJitLinkHandle handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkComplete + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkComplete", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":296 + * + * + * cdef nvJitLinkResult _nvJitLinkGetLinkedCubinSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetLinkedCubinSize + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetLinkedCubinSize(nvJitLinkHandle __pyx_v_handle, size_t *__pyx_v_size) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkGetLinkedCubinSize", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":298 + * cdef nvJitLinkResult _nvJitLinkGetLinkedCubinSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkGetLinkedCubinSize + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkGetLinkedCubinSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 298, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":299 + * global __nvJitLinkGetLinkedCubinSize + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedCubinSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedCubinSize is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubinSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":300 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedCubinSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetLinkedCubinSize is not found") + * return (__nvJitLinkGetLinkedCubinSize)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":301 + * if __nvJitLinkGetLinkedCubinSize == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedCubinSize is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkGetLinkedCubinSize)( + * handle, size) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 301, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkGetLinkedCubin}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 301, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 301, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":300 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedCubinSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetLinkedCubinSize is not found") + * return (__nvJitLinkGetLinkedCubinSize)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":299 + * global __nvJitLinkGetLinkedCubinSize + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedCubinSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedCubinSize is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":302 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedCubinSize is not found") + * return (__nvJitLinkGetLinkedCubinSize)( # <<<<<<<<<<<<<< + * handle, size) + * +*/ + __pyx_r = ((nvJitLinkResult (*)(nvJitLinkHandle, size_t *))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubinSize)(__pyx_v_handle, __pyx_v_size); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":296 + * + * + * cdef nvJitLinkResult _nvJitLinkGetLinkedCubinSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetLinkedCubinSize + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkGetLinkedCubinSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":306 + * + * + * cdef nvJitLinkResult _nvJitLinkGetLinkedCubin(nvJitLinkHandle handle, void* cubin) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetLinkedCubin + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetLinkedCubin(nvJitLinkHandle __pyx_v_handle, void *__pyx_v_cubin) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkGetLinkedCubin", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":308 + * cdef nvJitLinkResult _nvJitLinkGetLinkedCubin(nvJitLinkHandle handle, void* cubin) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkGetLinkedCubin + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkGetLinkedCubin == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 308, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":309 + * global __nvJitLinkGetLinkedCubin + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedCubin == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedCubin is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubin == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":310 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedCubin == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetLinkedCubin is not found") + * return (__nvJitLinkGetLinkedCubin)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":311 + * if __nvJitLinkGetLinkedCubin == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedCubin is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkGetLinkedCubin)( + * handle, cubin) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 311, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkGetLinkedCubin_2}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 311, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 311, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":310 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedCubin == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetLinkedCubin is not found") + * return (__nvJitLinkGetLinkedCubin)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":309 + * global __nvJitLinkGetLinkedCubin + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedCubin == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedCubin is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":312 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedCubin is not found") + * return (__nvJitLinkGetLinkedCubin)( # <<<<<<<<<<<<<< + * handle, cubin) + * +*/ + __pyx_r = ((nvJitLinkResult (*)(nvJitLinkHandle, void *))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubin)(__pyx_v_handle, __pyx_v_cubin); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":306 + * + * + * cdef nvJitLinkResult _nvJitLinkGetLinkedCubin(nvJitLinkHandle handle, void* cubin) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetLinkedCubin + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkGetLinkedCubin", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":316 + * + * + * cdef nvJitLinkResult _nvJitLinkGetLinkedPtxSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetLinkedPtxSize + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetLinkedPtxSize(nvJitLinkHandle __pyx_v_handle, size_t *__pyx_v_size) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkGetLinkedPtxSize", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":318 + * cdef nvJitLinkResult _nvJitLinkGetLinkedPtxSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkGetLinkedPtxSize + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkGetLinkedPtxSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 318, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":319 + * global __nvJitLinkGetLinkedPtxSize + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedPtxSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedPtxSize is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtxSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":320 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedPtxSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetLinkedPtxSize is not found") + * return (__nvJitLinkGetLinkedPtxSize)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":321 + * if __nvJitLinkGetLinkedPtxSize == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedPtxSize is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkGetLinkedPtxSize)( + * handle, size) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 321, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkGetLinkedPtxSi}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 321, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 321, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":320 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedPtxSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetLinkedPtxSize is not found") + * return (__nvJitLinkGetLinkedPtxSize)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":319 + * global __nvJitLinkGetLinkedPtxSize + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedPtxSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedPtxSize is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":322 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedPtxSize is not found") + * return (__nvJitLinkGetLinkedPtxSize)( # <<<<<<<<<<<<<< + * handle, size) + * +*/ + __pyx_r = ((nvJitLinkResult (*)(nvJitLinkHandle, size_t *))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtxSize)(__pyx_v_handle, __pyx_v_size); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":316 + * + * + * cdef nvJitLinkResult _nvJitLinkGetLinkedPtxSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetLinkedPtxSize + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkGetLinkedPtxSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":326 + * + * + * cdef nvJitLinkResult _nvJitLinkGetLinkedPtx(nvJitLinkHandle handle, char* ptx) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetLinkedPtx + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetLinkedPtx(nvJitLinkHandle __pyx_v_handle, char *__pyx_v_ptx) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkGetLinkedPtx", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":328 + * cdef nvJitLinkResult _nvJitLinkGetLinkedPtx(nvJitLinkHandle handle, char* ptx) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkGetLinkedPtx + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkGetLinkedPtx == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 328, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":329 + * global __nvJitLinkGetLinkedPtx + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedPtx == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedPtx is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtx == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":330 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedPtx == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetLinkedPtx is not found") + * return (__nvJitLinkGetLinkedPtx)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":331 + * if __nvJitLinkGetLinkedPtx == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedPtx is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkGetLinkedPtx)( + * handle, ptx) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 331, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkGetLinkedPtx_i}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 331, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 331, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":330 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedPtx == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetLinkedPtx is not found") + * return (__nvJitLinkGetLinkedPtx)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":329 + * global __nvJitLinkGetLinkedPtx + * _check_or_init_nvjitlink() + * if __nvJitLinkGetLinkedPtx == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedPtx is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":332 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetLinkedPtx is not found") + * return (__nvJitLinkGetLinkedPtx)( # <<<<<<<<<<<<<< + * handle, ptx) + * +*/ + __pyx_r = ((nvJitLinkResult (*)(nvJitLinkHandle, char *))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtx)(__pyx_v_handle, __pyx_v_ptx); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":326 + * + * + * cdef nvJitLinkResult _nvJitLinkGetLinkedPtx(nvJitLinkHandle handle, char* ptx) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetLinkedPtx + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkGetLinkedPtx", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":336 + * + * + * cdef nvJitLinkResult _nvJitLinkGetErrorLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetErrorLogSize + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetErrorLogSize(nvJitLinkHandle __pyx_v_handle, size_t *__pyx_v_size) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkGetErrorLogSize", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":338 + * cdef nvJitLinkResult _nvJitLinkGetErrorLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkGetErrorLogSize + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkGetErrorLogSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 338, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":339 + * global __nvJitLinkGetErrorLogSize + * _check_or_init_nvjitlink() + * if __nvJitLinkGetErrorLogSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetErrorLogSize is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLogSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":340 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetErrorLogSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetErrorLogSize is not found") + * return (__nvJitLinkGetErrorLogSize)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":341 + * if __nvJitLinkGetErrorLogSize == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetErrorLogSize is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkGetErrorLogSize)( + * handle, size) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 341, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkGetErrorLogSiz}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 341, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 341, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":340 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetErrorLogSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetErrorLogSize is not found") + * return (__nvJitLinkGetErrorLogSize)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":339 + * global __nvJitLinkGetErrorLogSize + * _check_or_init_nvjitlink() + * if __nvJitLinkGetErrorLogSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetErrorLogSize is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":342 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetErrorLogSize is not found") + * return (__nvJitLinkGetErrorLogSize)( # <<<<<<<<<<<<<< + * handle, size) + * +*/ + __pyx_r = ((nvJitLinkResult (*)(nvJitLinkHandle, size_t *))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLogSize)(__pyx_v_handle, __pyx_v_size); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":336 + * + * + * cdef nvJitLinkResult _nvJitLinkGetErrorLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetErrorLogSize + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkGetErrorLogSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":346 + * + * + * cdef nvJitLinkResult _nvJitLinkGetErrorLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetErrorLog + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetErrorLog(nvJitLinkHandle __pyx_v_handle, char *__pyx_v_log) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkGetErrorLog", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":348 + * cdef nvJitLinkResult _nvJitLinkGetErrorLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkGetErrorLog + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkGetErrorLog == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 348, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":349 + * global __nvJitLinkGetErrorLog + * _check_or_init_nvjitlink() + * if __nvJitLinkGetErrorLog == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetErrorLog is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLog == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":350 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetErrorLog == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetErrorLog is not found") + * return (__nvJitLinkGetErrorLog)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":351 + * if __nvJitLinkGetErrorLog == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetErrorLog is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkGetErrorLog)( + * handle, log) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 351, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkGetErrorLog_is}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 351, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 351, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":350 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetErrorLog == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetErrorLog is not found") + * return (__nvJitLinkGetErrorLog)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":349 + * global __nvJitLinkGetErrorLog + * _check_or_init_nvjitlink() + * if __nvJitLinkGetErrorLog == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetErrorLog is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":352 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetErrorLog is not found") + * return (__nvJitLinkGetErrorLog)( # <<<<<<<<<<<<<< + * handle, log) + * +*/ + __pyx_r = ((nvJitLinkResult (*)(nvJitLinkHandle, char *))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLog)(__pyx_v_handle, __pyx_v_log); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":346 + * + * + * cdef nvJitLinkResult _nvJitLinkGetErrorLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetErrorLog + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkGetErrorLog", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":356 + * + * + * cdef nvJitLinkResult _nvJitLinkGetInfoLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetInfoLogSize + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetInfoLogSize(nvJitLinkHandle __pyx_v_handle, size_t *__pyx_v_size) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkGetInfoLogSize", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":358 + * cdef nvJitLinkResult _nvJitLinkGetInfoLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkGetInfoLogSize + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkGetInfoLogSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 358, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":359 + * global __nvJitLinkGetInfoLogSize + * _check_or_init_nvjitlink() + * if __nvJitLinkGetInfoLogSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetInfoLogSize is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLogSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":360 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetInfoLogSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetInfoLogSize is not found") + * return (__nvJitLinkGetInfoLogSize)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":361 + * if __nvJitLinkGetInfoLogSize == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetInfoLogSize is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkGetInfoLogSize)( + * handle, size) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 361, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkGetInfoLogSize}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 361, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 361, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":360 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetInfoLogSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetInfoLogSize is not found") + * return (__nvJitLinkGetInfoLogSize)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":359 + * global __nvJitLinkGetInfoLogSize + * _check_or_init_nvjitlink() + * if __nvJitLinkGetInfoLogSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetInfoLogSize is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":362 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetInfoLogSize is not found") + * return (__nvJitLinkGetInfoLogSize)( # <<<<<<<<<<<<<< + * handle, size) + * +*/ + __pyx_r = ((nvJitLinkResult (*)(nvJitLinkHandle, size_t *))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLogSize)(__pyx_v_handle, __pyx_v_size); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":356 + * + * + * cdef nvJitLinkResult _nvJitLinkGetInfoLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetInfoLogSize + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkGetInfoLogSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":366 + * + * + * cdef nvJitLinkResult _nvJitLinkGetInfoLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetInfoLog + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetInfoLog(nvJitLinkHandle __pyx_v_handle, char *__pyx_v_log) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkGetInfoLog", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":368 + * cdef nvJitLinkResult _nvJitLinkGetInfoLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkGetInfoLog + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkGetInfoLog == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 368, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":369 + * global __nvJitLinkGetInfoLog + * _check_or_init_nvjitlink() + * if __nvJitLinkGetInfoLog == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetInfoLog is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLog == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":370 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetInfoLog == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetInfoLog is not found") + * return (__nvJitLinkGetInfoLog)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":371 + * if __nvJitLinkGetInfoLog == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetInfoLog is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkGetInfoLog)( + * handle, log) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 371, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkGetInfoLog_is}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 371, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 371, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":370 + * _check_or_init_nvjitlink() + * if __nvJitLinkGetInfoLog == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkGetInfoLog is not found") + * return (__nvJitLinkGetInfoLog)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":369 + * global __nvJitLinkGetInfoLog + * _check_or_init_nvjitlink() + * if __nvJitLinkGetInfoLog == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetInfoLog is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":372 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkGetInfoLog is not found") + * return (__nvJitLinkGetInfoLog)( # <<<<<<<<<<<<<< + * handle, log) + * +*/ + __pyx_r = ((nvJitLinkResult (*)(nvJitLinkHandle, char *))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLog)(__pyx_v_handle, __pyx_v_log); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":366 + * + * + * cdef nvJitLinkResult _nvJitLinkGetInfoLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkGetInfoLog + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkGetInfoLog", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvjitlink.pyx":376 + * + * + * cdef nvJitLinkResult _nvJitLinkVersion(unsigned int* major, unsigned int* minor) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkVersion + * _check_or_init_nvjitlink() +*/ + +static nvJitLinkResult __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkVersion(unsigned int *__pyx_v_major, unsigned int *__pyx_v_minor) { + nvJitLinkResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvJitLinkVersion", 1); + + /* "cuda/bindings/_internal/nvjitlink.pyx":378 + * cdef nvJitLinkResult _nvJitLinkVersion(unsigned int* major, unsigned int* minor) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvJitLinkVersion + * _check_or_init_nvjitlink() # <<<<<<<<<<<<<< + * if __nvJitLinkVersion == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_9nvjitlink__check_or_init_nvjitlink(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 378, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvjitlink.pyx":379 + * global __nvJitLinkVersion + * _check_or_init_nvjitlink() + * if __nvJitLinkVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkVersion is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkVersion == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvjitlink.pyx":380 + * _check_or_init_nvjitlink() + * if __nvJitLinkVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkVersion is not found") + * return (__nvJitLinkVersion)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvjitlink.pyx":381 + * if __nvJitLinkVersion == NULL: + * with gil: + * raise FunctionNotFoundError("function nvJitLinkVersion is not found") # <<<<<<<<<<<<<< + * return (__nvJitLinkVersion)( + * major, minor) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 381, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvJitLinkVersion_is_not}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 381, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 381, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":380 + * _check_or_init_nvjitlink() + * if __nvJitLinkVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvJitLinkVersion is not found") + * return (__nvJitLinkVersion)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":379 + * global __nvJitLinkVersion + * _check_or_init_nvjitlink() + * if __nvJitLinkVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvJitLinkVersion is not found") +*/ + } + + /* "cuda/bindings/_internal/nvjitlink.pyx":382 + * with gil: + * raise FunctionNotFoundError("function nvJitLinkVersion is not found") + * return (__nvJitLinkVersion)( # <<<<<<<<<<<<<< + * major, minor) +*/ + __pyx_r = ((nvJitLinkResult (*)(unsigned int *, unsigned int *))__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkVersion)(__pyx_v_major, __pyx_v_minor); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":376 + * + * + * cdef nvJitLinkResult _nvJitLinkVersion(unsigned int* major, unsigned int* minor) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvJitLinkVersion + * _check_or_init_nvjitlink() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvjitlink._nvJitLinkVersion", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVJITLINKRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} +/* #### Code section: module_exttypes ### */ + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___symbol_lock = Py_None; Py_INCREF(Py_None); + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs = ((PyObject*)Py_None); Py_INCREF(Py_None); + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + if (__Pyx_ExportFunction("_nvJitLinkCreate", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkCreate, "nvJitLinkResult (nvJitLinkHandle *, uint32_t, char const **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvJitLinkDestroy", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkDestroy, "nvJitLinkResult (nvJitLinkHandle *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvJitLinkAddData", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkAddData, "nvJitLinkResult (nvJitLinkHandle, nvJitLinkInputType, void const *, size_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvJitLinkAddFile", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkAddFile, "nvJitLinkResult (nvJitLinkHandle, nvJitLinkInputType, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvJitLinkComplete", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkComplete, "nvJitLinkResult (nvJitLinkHandle)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvJitLinkGetLinkedCubinSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetLinkedCubinSize, "nvJitLinkResult (nvJitLinkHandle, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvJitLinkGetLinkedCubin", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetLinkedCubin, "nvJitLinkResult (nvJitLinkHandle, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvJitLinkGetLinkedPtxSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetLinkedPtxSize, "nvJitLinkResult (nvJitLinkHandle, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvJitLinkGetLinkedPtx", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetLinkedPtx, "nvJitLinkResult (nvJitLinkHandle, char *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvJitLinkGetErrorLogSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetErrorLogSize, "nvJitLinkResult (nvJitLinkHandle, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvJitLinkGetErrorLog", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetErrorLog, "nvJitLinkResult (nvJitLinkHandle, char *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvJitLinkGetInfoLogSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetInfoLogSize, "nvJitLinkResult (nvJitLinkHandle, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvJitLinkGetInfoLog", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkGetInfoLog, "nvJitLinkResult (nvJitLinkHandle, char *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvJitLinkVersion", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_9nvjitlink__nvJitLinkVersion, "nvJitLinkResult (unsigned int *, unsigned int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_nvjitlink(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_nvjitlink}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "nvjitlink", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_nvjitlink(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_nvjitlink(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x03090000 + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_nvjitlink(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'nvjitlink' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "nvjitlink" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_NOT_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY +__Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); +if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); +} +#endif + +__Pyx_RefNannySetupContext("PyInit_nvjitlink", 0); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #ifdef __Pxy_PyFrame_Initialize_Offsets + __Pxy_PyFrame_Initialize_Offsets(); + #endif + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #if 0 || defined(__Pyx_CyFunction_USED) || defined(__Pyx_FusedFunction_USED) || defined(__Pyx_Coroutine_USED) || defined(__Pyx_Generator_USED) || defined(__Pyx_AsyncGen_USED) + if (__pyx_CommonTypesMetaclass_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_AsyncGen_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + if (__pyx_module_is_main_cuda__bindings___internal__nvjitlink) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name_2, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "cuda.bindings._internal.nvjitlink")) { + if (unlikely((PyDict_SetItemString(modules, "cuda.bindings._internal.nvjitlink", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_function_export_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_type_init_code(__pyx_mstate); + (void)__Pyx_modinit_type_import_code(__pyx_mstate); + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + (void)__Pyx_modinit_function_import_code(__pyx_mstate); + /*--- Execution code ---*/ + + /* "cuda/bindings/_internal/nvjitlink.pyx":9 + * from libc.stdint cimport intptr_t, uintptr_t + * + * import threading # <<<<<<<<<<<<<< + * from .utils import FunctionNotFoundError, NotSupportedError + * +*/ + __pyx_t_2 = __Pyx_ImportDottedModule(__pyx_mstate_global->__pyx_n_u_threading, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_threading, __pyx_t_2) < (0)) __PYX_ERR(0, 9, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":10 + * + * import threading + * from .utils import FunctionNotFoundError, NotSupportedError # <<<<<<<<<<<<<< + * + * from cuda.pathfinder import load_nvidia_dynamic_lib +*/ + __pyx_t_2 = __Pyx_PyList_Pack(2, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError, __pyx_mstate_global->__pyx_n_u_NotSupportedError); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_utils, __pyx_t_2, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError, __pyx_t_2) < (0)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_NotSupportedError); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_NotSupportedError, __pyx_t_2) < (0)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":12 + * from .utils import FunctionNotFoundError, NotSupportedError + * + * from cuda.pathfinder import load_nvidia_dynamic_lib # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_3 = __Pyx_PyList_Pack(1, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cuda_pathfinder, __pyx_t_3, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_ImportFrom(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib, __pyx_t_3) < (0)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":38 + * ############################################################################### + * + * cdef object __symbol_lock = threading.Lock() # <<<<<<<<<<<<<< + * cdef bint __py_nvjitlink_init = False + * cdef void* __cuDriverGetVersion = NULL +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_threading); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_Lock); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 38, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___symbol_lock); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_internal_9nvjitlink___symbol_lock, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_2); + __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":39 + * + * cdef object __symbol_lock = threading.Lock() + * cdef bint __py_nvjitlink_init = False # <<<<<<<<<<<<<< + * cdef void* __cuDriverGetVersion = NULL + * +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___py_nvjitlink_init = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":40 + * cdef object __symbol_lock = threading.Lock() + * cdef bint __py_nvjitlink_init = False + * cdef void* __cuDriverGetVersion = NULL # <<<<<<<<<<<<<< + * + * cdef void* __nvJitLinkCreate = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___cuDriverGetVersion = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":42 + * cdef void* __cuDriverGetVersion = NULL + * + * cdef void* __nvJitLinkCreate = NULL # <<<<<<<<<<<<<< + * cdef void* __nvJitLinkDestroy = NULL + * cdef void* __nvJitLinkAddData = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkCreate = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":43 + * + * cdef void* __nvJitLinkCreate = NULL + * cdef void* __nvJitLinkDestroy = NULL # <<<<<<<<<<<<<< + * cdef void* __nvJitLinkAddData = NULL + * cdef void* __nvJitLinkAddFile = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkDestroy = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":44 + * cdef void* __nvJitLinkCreate = NULL + * cdef void* __nvJitLinkDestroy = NULL + * cdef void* __nvJitLinkAddData = NULL # <<<<<<<<<<<<<< + * cdef void* __nvJitLinkAddFile = NULL + * cdef void* __nvJitLinkComplete = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddData = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":45 + * cdef void* __nvJitLinkDestroy = NULL + * cdef void* __nvJitLinkAddData = NULL + * cdef void* __nvJitLinkAddFile = NULL # <<<<<<<<<<<<<< + * cdef void* __nvJitLinkComplete = NULL + * cdef void* __nvJitLinkGetLinkedCubinSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkAddFile = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":46 + * cdef void* __nvJitLinkAddData = NULL + * cdef void* __nvJitLinkAddFile = NULL + * cdef void* __nvJitLinkComplete = NULL # <<<<<<<<<<<<<< + * cdef void* __nvJitLinkGetLinkedCubinSize = NULL + * cdef void* __nvJitLinkGetLinkedCubin = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkComplete = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":47 + * cdef void* __nvJitLinkAddFile = NULL + * cdef void* __nvJitLinkComplete = NULL + * cdef void* __nvJitLinkGetLinkedCubinSize = NULL # <<<<<<<<<<<<<< + * cdef void* __nvJitLinkGetLinkedCubin = NULL + * cdef void* __nvJitLinkGetLinkedPtxSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubinSize = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":48 + * cdef void* __nvJitLinkComplete = NULL + * cdef void* __nvJitLinkGetLinkedCubinSize = NULL + * cdef void* __nvJitLinkGetLinkedCubin = NULL # <<<<<<<<<<<<<< + * cdef void* __nvJitLinkGetLinkedPtxSize = NULL + * cdef void* __nvJitLinkGetLinkedPtx = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedCubin = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":49 + * cdef void* __nvJitLinkGetLinkedCubinSize = NULL + * cdef void* __nvJitLinkGetLinkedCubin = NULL + * cdef void* __nvJitLinkGetLinkedPtxSize = NULL # <<<<<<<<<<<<<< + * cdef void* __nvJitLinkGetLinkedPtx = NULL + * cdef void* __nvJitLinkGetErrorLogSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtxSize = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":50 + * cdef void* __nvJitLinkGetLinkedCubin = NULL + * cdef void* __nvJitLinkGetLinkedPtxSize = NULL + * cdef void* __nvJitLinkGetLinkedPtx = NULL # <<<<<<<<<<<<<< + * cdef void* __nvJitLinkGetErrorLogSize = NULL + * cdef void* __nvJitLinkGetErrorLog = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetLinkedPtx = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":51 + * cdef void* __nvJitLinkGetLinkedPtxSize = NULL + * cdef void* __nvJitLinkGetLinkedPtx = NULL + * cdef void* __nvJitLinkGetErrorLogSize = NULL # <<<<<<<<<<<<<< + * cdef void* __nvJitLinkGetErrorLog = NULL + * cdef void* __nvJitLinkGetInfoLogSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLogSize = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":52 + * cdef void* __nvJitLinkGetLinkedPtx = NULL + * cdef void* __nvJitLinkGetErrorLogSize = NULL + * cdef void* __nvJitLinkGetErrorLog = NULL # <<<<<<<<<<<<<< + * cdef void* __nvJitLinkGetInfoLogSize = NULL + * cdef void* __nvJitLinkGetInfoLog = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetErrorLog = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":53 + * cdef void* __nvJitLinkGetErrorLogSize = NULL + * cdef void* __nvJitLinkGetErrorLog = NULL + * cdef void* __nvJitLinkGetInfoLogSize = NULL # <<<<<<<<<<<<<< + * cdef void* __nvJitLinkGetInfoLog = NULL + * cdef void* __nvJitLinkVersion = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLogSize = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":54 + * cdef void* __nvJitLinkGetErrorLog = NULL + * cdef void* __nvJitLinkGetInfoLogSize = NULL + * cdef void* __nvJitLinkGetInfoLog = NULL # <<<<<<<<<<<<<< + * cdef void* __nvJitLinkVersion = NULL + * +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkGetInfoLog = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":55 + * cdef void* __nvJitLinkGetInfoLogSize = NULL + * cdef void* __nvJitLinkGetInfoLog = NULL + * cdef void* __nvJitLinkVersion = NULL # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_4cuda_8bindings_9_internal_9nvjitlink___nvJitLinkVersion = NULL; + + /* "cuda/bindings/_internal/nvjitlink.pyx":178 + * return __check_or_init_nvjitlink() + * + * cdef dict func_ptrs = None # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_INCREF(Py_None); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_internal_9nvjitlink_func_ptrs, ((PyObject*)Py_None)); + __Pyx_GIVEREF(Py_None); + + /* "cuda/bindings/_internal/nvjitlink.pyx":181 + * + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_4cuda_8bindings_9_internal_9nvjitlink_1_inspect_function_pointers, 0, __pyx_mstate_global->__pyx_n_u_inspect_function_pointers, NULL, __pyx_mstate_global->__pyx_n_u_cuda_bindings__internal_nvjitlin, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 181, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_inspect_function_pointers, __pyx_t_2) < (0)) __PYX_ERR(0, 181, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":235 + * + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_4cuda_8bindings_9_internal_9nvjitlink_3_inspect_function_pointer, 0, __pyx_mstate_global->__pyx_n_u_inspect_function_pointer, NULL, __pyx_mstate_global->__pyx_n_u_cuda_bindings__internal_nvjitlin, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[1])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 235, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_inspect_function_pointer, __pyx_t_2) < (0)) __PYX_ERR(0, 235, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/nvjitlink.pyx":1 + * # SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # <<<<<<<<<<<<<< + * # + * # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +*/ + __pyx_t_2 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_2) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init cuda.bindings._internal.nvjitlink", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init cuda.bindings._internal.nvjitlink"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ + +typedef struct { + const char *s; +#if 179 <= 65535 + const unsigned short n; +#elif 179 / 2 < INT_MAX + const unsigned int n; +#elif 179 / 2 < LONG_MAX + const unsigned long n; +#else + const Py_ssize_t n; +#endif +#if 1 <= 31 + const unsigned int encoding : 5; +#elif 1 <= 255 + const unsigned char encoding; +#elif 1 <= 65535 + const unsigned short encoding; +#else + const Py_ssize_t encoding; +#endif + const unsigned int is_unicode : 1; + const unsigned int intern : 1; +} __Pyx_StringTabEntry; +static const char * const __pyx_string_tab_encodings[] = { 0 }; +static const __Pyx_StringTabEntry __pyx_string_tab[] = { + {__pyx_k_, sizeof(__pyx_k_), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_ */ + {__pyx_k_FunctionNotFoundError, sizeof(__pyx_k_FunctionNotFoundError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_FunctionNotFoundError */ + {__pyx_k_Lock, sizeof(__pyx_k_Lock), 0, 1, 1}, /* PyObject cname: __pyx_n_u_Lock */ + {__pyx_k_NotSupportedError, sizeof(__pyx_k_NotSupportedError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_NotSupportedError */ + {__pyx_k_Note_that_Cython_is_deliberately, sizeof(__pyx_k_Note_that_Cython_is_deliberately), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Note_that_Cython_is_deliberately */ + {__pyx_k__2, sizeof(__pyx_k__2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u__2 */ + {__pyx_k_add_note, sizeof(__pyx_k_add_note), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_add_note */ + {__pyx_k_asyncio_coroutines, sizeof(__pyx_k_asyncio_coroutines), 0, 1, 1}, /* PyObject cname: __pyx_n_u_asyncio_coroutines */ + {__pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cline_in_traceback */ + {__pyx_k_cuda_bindings__internal_nvjitlin, sizeof(__pyx_k_cuda_bindings__internal_nvjitlin), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuda_bindings__internal_nvjitlin */ + {__pyx_k_cuda_bindings__internal_nvjitlin_2, sizeof(__pyx_k_cuda_bindings__internal_nvjitlin_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_cuda_bindings__internal_nvjitlin_2 */ + {__pyx_k_cuda_pathfinder, sizeof(__pyx_k_cuda_pathfinder), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuda_pathfinder */ + {__pyx_k_enter, sizeof(__pyx_k_enter), 0, 1, 1}, /* PyObject cname: __pyx_n_u_enter */ + {__pyx_k_exit, sizeof(__pyx_k_exit), 0, 1, 1}, /* PyObject cname: __pyx_n_u_exit */ + {__pyx_k_func, sizeof(__pyx_k_func), 0, 1, 1}, /* PyObject cname: __pyx_n_u_func */ + {__pyx_k_function_nvJitLinkAddData_is_not, sizeof(__pyx_k_function_nvJitLinkAddData_is_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkAddData_is_not */ + {__pyx_k_function_nvJitLinkAddFile_is_not, sizeof(__pyx_k_function_nvJitLinkAddFile_is_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkAddFile_is_not */ + {__pyx_k_function_nvJitLinkComplete_is_no, sizeof(__pyx_k_function_nvJitLinkComplete_is_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkComplete_is_no */ + {__pyx_k_function_nvJitLinkCreate_is_not, sizeof(__pyx_k_function_nvJitLinkCreate_is_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkCreate_is_not */ + {__pyx_k_function_nvJitLinkDestroy_is_not, sizeof(__pyx_k_function_nvJitLinkDestroy_is_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkDestroy_is_not */ + {__pyx_k_function_nvJitLinkGetErrorLogSiz, sizeof(__pyx_k_function_nvJitLinkGetErrorLogSiz), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkGetErrorLogSiz */ + {__pyx_k_function_nvJitLinkGetErrorLog_is, sizeof(__pyx_k_function_nvJitLinkGetErrorLog_is), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkGetErrorLog_is */ + {__pyx_k_function_nvJitLinkGetInfoLogSize, sizeof(__pyx_k_function_nvJitLinkGetInfoLogSize), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkGetInfoLogSize */ + {__pyx_k_function_nvJitLinkGetInfoLog_is, sizeof(__pyx_k_function_nvJitLinkGetInfoLog_is), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkGetInfoLog_is */ + {__pyx_k_function_nvJitLinkGetLinkedCubin, sizeof(__pyx_k_function_nvJitLinkGetLinkedCubin), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkGetLinkedCubin */ + {__pyx_k_function_nvJitLinkGetLinkedCubin_2, sizeof(__pyx_k_function_nvJitLinkGetLinkedCubin_2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkGetLinkedCubin_2 */ + {__pyx_k_function_nvJitLinkGetLinkedPtxSi, sizeof(__pyx_k_function_nvJitLinkGetLinkedPtxSi), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkGetLinkedPtxSi */ + {__pyx_k_function_nvJitLinkGetLinkedPtx_i, sizeof(__pyx_k_function_nvJitLinkGetLinkedPtx_i), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkGetLinkedPtx_i */ + {__pyx_k_function_nvJitLinkVersion_is_not, sizeof(__pyx_k_function_nvJitLinkVersion_is_not), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvJitLinkVersion_is_not */ + {__pyx_k_handle_uint, sizeof(__pyx_k_handle_uint), 0, 1, 1}, /* PyObject cname: __pyx_n_u_handle_uint */ + {__pyx_k_initializing, sizeof(__pyx_k_initializing), 0, 1, 1}, /* PyObject cname: __pyx_n_u_initializing */ + {__pyx_k_inspect_function_pointer, sizeof(__pyx_k_inspect_function_pointer), 0, 1, 1}, /* PyObject cname: __pyx_n_u_inspect_function_pointer */ + {__pyx_k_inspect_function_pointers, sizeof(__pyx_k_inspect_function_pointers), 0, 1, 1}, /* PyObject cname: __pyx_n_u_inspect_function_pointers */ + {__pyx_k_is_coroutine, sizeof(__pyx_k_is_coroutine), 0, 1, 1}, /* PyObject cname: __pyx_n_u_is_coroutine */ + {__pyx_k_load_nvidia_dynamic_lib, sizeof(__pyx_k_load_nvidia_dynamic_lib), 0, 1, 1}, /* PyObject cname: __pyx_n_u_load_nvidia_dynamic_lib */ + {__pyx_k_main, sizeof(__pyx_k_main), 0, 1, 1}, /* PyObject cname: __pyx_n_u_main */ + {__pyx_k_module, sizeof(__pyx_k_module), 0, 1, 1}, /* PyObject cname: __pyx_n_u_module */ + {__pyx_k_name, sizeof(__pyx_k_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name */ + {__pyx_k_name_2, sizeof(__pyx_k_name_2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name_2 */ + {__pyx_k_nvJitLink, sizeof(__pyx_k_nvJitLink), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLink */ + {__pyx_k_nvJitLinkAddData, sizeof(__pyx_k_nvJitLinkAddData), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkAddData */ + {__pyx_k_nvJitLinkAddFile, sizeof(__pyx_k_nvJitLinkAddFile), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkAddFile */ + {__pyx_k_nvJitLinkComplete, sizeof(__pyx_k_nvJitLinkComplete), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkComplete */ + {__pyx_k_nvJitLinkCreate, sizeof(__pyx_k_nvJitLinkCreate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkCreate */ + {__pyx_k_nvJitLinkDestroy, sizeof(__pyx_k_nvJitLinkDestroy), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkDestroy */ + {__pyx_k_nvJitLinkGetErrorLog, sizeof(__pyx_k_nvJitLinkGetErrorLog), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkGetErrorLog */ + {__pyx_k_nvJitLinkGetErrorLogSize, sizeof(__pyx_k_nvJitLinkGetErrorLogSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkGetErrorLogSize */ + {__pyx_k_nvJitLinkGetInfoLog, sizeof(__pyx_k_nvJitLinkGetInfoLog), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkGetInfoLog */ + {__pyx_k_nvJitLinkGetInfoLogSize, sizeof(__pyx_k_nvJitLinkGetInfoLogSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkGetInfoLogSize */ + {__pyx_k_nvJitLinkGetLinkedCubin, sizeof(__pyx_k_nvJitLinkGetLinkedCubin), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkGetLinkedCubin */ + {__pyx_k_nvJitLinkGetLinkedCubinSize, sizeof(__pyx_k_nvJitLinkGetLinkedCubinSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkGetLinkedCubinSize */ + {__pyx_k_nvJitLinkGetLinkedPtx, sizeof(__pyx_k_nvJitLinkGetLinkedPtx), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkGetLinkedPtx */ + {__pyx_k_nvJitLinkGetLinkedPtxSize, sizeof(__pyx_k_nvJitLinkGetLinkedPtxSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkGetLinkedPtxSize */ + {__pyx_k_nvJitLinkVersion, sizeof(__pyx_k_nvJitLinkVersion), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvJitLinkVersion */ + {__pyx_k_pop, sizeof(__pyx_k_pop), 0, 1, 1}, /* PyObject cname: __pyx_n_u_pop */ + {__pyx_k_qualname, sizeof(__pyx_k_qualname), 0, 1, 1}, /* PyObject cname: __pyx_n_u_qualname */ + {__pyx_k_set_name, sizeof(__pyx_k_set_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_set_name */ + {__pyx_k_spec, sizeof(__pyx_k_spec), 0, 1, 1}, /* PyObject cname: __pyx_n_u_spec */ + {__pyx_k_test, sizeof(__pyx_k_test), 0, 1, 1}, /* PyObject cname: __pyx_n_u_test */ + {__pyx_k_threading, sizeof(__pyx_k_threading), 0, 1, 1}, /* PyObject cname: __pyx_n_u_threading */ + {__pyx_k_utils, sizeof(__pyx_k_utils), 0, 1, 1}, /* PyObject cname: __pyx_n_u_utils */ + {0, 0, 0, 0, 0} +}; +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names); + +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + return 0; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "cuda/bindings/_internal/nvjitlink.pyx":68 + * cdef void* handle = NULL + * + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * # Load function + * global __nvJitLinkCreate +*/ + __pyx_mstate_global->__pyx_tuple[0] = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_tuple[0])) __PYX_ERR(0, 68, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.method_name = &__pyx_mstate->__pyx_n_u_pop; + if (__Pyx_InitStrings(__pyx_string_tab, __pyx_mstate->__pyx_string_tab, __pyx_string_tab_encodings) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ +\ + typedef struct { + unsigned int argcount : 1; + unsigned int num_posonly_args : 1; + unsigned int num_kwonly_args : 1; + unsigned int nlocals : 1; + unsigned int flags : 10; + unsigned int first_line : 8; + unsigned int line_table_length : 13; + } __Pyx_PyCode_New_function_description; +/* NewCodeObj.proto */ +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + const char *line_table, + PyObject *tuple_dedup_map +); + + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + PyObject* tuple_dedup_map = PyDict_New(); + if (unlikely(!tuple_dedup_map)) return -1; + { + const __Pyx_PyCode_New_function_description descr = {0, 0, 0, 0, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 181, 249}; + PyObject* const varnames[] = {0}; + __pyx_mstate_global->__pyx_codeobj_tab[0] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_cuda_bindings__internal_nvjitlin_2, __pyx_mstate->__pyx_n_u_inspect_function_pointers, __pyx_k_z_q_A_Q_1_1_1_A_Ja_A_j_1_Zq_Ja, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[0])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 1, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 235, 31}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_name}; + __pyx_mstate_global->__pyx_codeobj_tab[1] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_cuda_bindings__internal_nvjitlin_2, __pyx_mstate->__pyx_n_u_inspect_function_pointer, __pyx_k_z_A_a_9AQ, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[1])) goto bad; + } + Py_DECREF(tuple_dedup_map); + return 0; + bad: + Py_DECREF(tuple_dedup_map); + return -1; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CachedMethodType.init */ + #if CYTHON_COMPILING_IN_LIMITED_API +{ + PyObject *typesModule=NULL; + typesModule = PyImport_ImportModule("types"); + if (typesModule) { + __pyx_mstate_global->__Pyx_CachedMethodType = PyObject_GetAttrString(typesModule, "MethodType"); + Py_DECREF(typesModule); + } +} // error handling follows +#endif + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* PyErrExceptionMatches */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStr */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* PyObjectGetAttrStrNoError */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name); + if (unlikely(!result) && !PyErr_Occurred()) { + PyErr_Format(PyExc_NameError, + "name '%U' is not defined", name); + } + return result; +} + +/* PyDictVersioning */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!__pyx_m)) { + if (!PyErr_Occurred()) + PyErr_SetNone(PyExc_NameError); + return NULL; + } + result = PyObject_GetAttr(__pyx_m, name); + if (likely(result)) { + return result; + } + PyErr_Clear(); +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear(); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return result; + } +#else + result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject *const *args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = __Pyx_PyFrame_GetLocalsplus(f); + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; + PyObject *kwdefs; + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) { + return NULL; + } + if ( + co->co_kwonlyargcount == 0 && + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, (int)nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif + +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = Py_TYPE(func)->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + #if PY_VERSION_HEX < 0x030800B1 + #if CYTHON_FAST_PYCCALL + if (PyCFunction_Check(func)) { + if (kwargs) { + return _PyCFunction_FastCallDict(func, args, nargs, kwargs); + } else { + return _PyCFunction_FastCallKeywords(func, args, nargs, NULL); + } + } + if (!kwargs && __Pyx_IS_TYPE(func, &PyMethodDescr_Type)) { + return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL); + } + #endif + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs); + } + #endif + #endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #elif defined(__Pyx_CyFunction_USED) && CYTHON_BACKPORT_VECTORCALL + if (__Pyx_CyFunction_CheckExact(func)) { + __pyx_vectorcallfunc f = __Pyx_CyFunction_func_vectorcall(func); + if (f) return f(func, args, _nargs, NULL); + } + #elif CYTHON_COMPILING_IN_LIMITED_API && CYTHON_VECTORCALL + return PyObject_Vectorcall(func, args, _nargs, NULL); + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* PyObjectLookupSpecial */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) { + PyObject *res; + PyTypeObject *tp = Py_TYPE(obj); + res = _PyType_Lookup(tp, attr_name); + if (likely(res)) { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (!f) { + Py_INCREF(res); + } else { + res = f(res, obj, (PyObject *)tp); + } + } else if (with_error) { + PyErr_SetObject(PyExc_AttributeError, attr_name); + } + return res; +} +#endif + +/* GetTopmostException */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + +/* SaveResetException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #endif +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); + #else + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); + #endif +} +#endif + +/* GetException */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) +#endif +{ + PyObject *local_type = NULL, *local_value, *local_tb = NULL; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030C0000 + local_value = tstate->current_exception; + tstate->current_exception = 0; + #else + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + #endif +#elif __PYX_LIMITED_VERSION_HEX > 0x030C0000 + local_value = PyErr_GetRaisedException(); +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif +#if __PYX_LIMITED_VERSION_HEX > 0x030C0000 + if (likely(local_value)) { + local_type = (PyObject*) Py_TYPE(local_value); + Py_INCREF(local_type); + local_tb = PyException_GetTraceback(local_value); + } +#else + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } +#endif // __PYX_LIMITED_VERSION_HEX > 0x030C0000 + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if CYTHON_USE_EXC_INFO_STACK + { + _PyErr_StackItem *exc_info = tstate->exc_info; + #if PY_VERSION_HEX >= 0x030B00a4 + tmp_value = exc_info->exc_value; + exc_info->exc_value = local_value; + tmp_type = NULL; + tmp_tb = NULL; + Py_XDECREF(local_type); + Py_XDECREF(local_tb); + #else + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; + #endif + } + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#elif __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + PyErr_SetHandledException(local_value); + Py_XDECREF(local_value); + Py_XDECREF(local_type); + Py_XDECREF(local_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +#if __PYX_LIMITED_VERSION_HEX <= 0x030C0000 +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +#endif +} + +/* TupleAndListFromArray */ +#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + Py_ssize_t i; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + for (i = 0; i < n; i++) { + if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < (0))) { + Py_DECREF(res); + return NULL; + } + Py_INCREF(src[i]); + } + return res; +} +#elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + +/* BytesEquals */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL ||\ + !(CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) + return PyObject_RichCompareBool(s1, s2, equals); +#else + if (s1 == s2) { + return (equals == Py_EQ); + } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { + const char *ps1, *ps2; + Py_ssize_t length = PyBytes_GET_SIZE(s1); + if (length != PyBytes_GET_SIZE(s2)) + return (equals == Py_NE); + ps1 = PyBytes_AS_STRING(s1); + ps2 = PyBytes_AS_STRING(s2); + if (ps1[0] != ps2[0]) { + return (equals == Py_NE); + } else if (length == 1) { + return (equals == Py_EQ); + } else { + int result; +#if CYTHON_USE_UNICODE_INTERNALS && (PY_VERSION_HEX < 0x030B0000) + Py_hash_t hash1, hash2; + hash1 = ((PyBytesObject*)s1)->ob_shash; + hash2 = ((PyBytesObject*)s2)->ob_shash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + return (equals == Py_NE); + } +#endif + result = memcmp(ps1, ps2, (size_t)length); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { + return (equals == Py_NE); + } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) { + return (equals == Py_NE); + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +#endif +} + +/* UnicodeEquals */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL + return PyObject_RichCompareBool(s1, s2, equals); +#else + int s1_is_unicode, s2_is_unicode; + if (s1 == s2) { + goto return_eq; + } + s1_is_unicode = PyUnicode_CheckExact(s1); + s2_is_unicode = PyUnicode_CheckExact(s2); + if (s1_is_unicode & s2_is_unicode) { + Py_ssize_t length, length2; + int kind; + void *data1, *data2; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0)) + return -1; + #endif + length = __Pyx_PyUnicode_GET_LENGTH(s1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length < 0)) return -1; + #endif + length2 = __Pyx_PyUnicode_GET_LENGTH(s2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length2 < 0)) return -1; + #endif + if (length != length2) { + goto return_ne; + } +#if CYTHON_USE_UNICODE_INTERNALS + { + Py_hash_t hash1, hash2; + hash1 = ((PyASCIIObject*)s1)->hash; + hash2 = ((PyASCIIObject*)s2)->hash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + goto return_ne; + } + } +#endif + kind = __Pyx_PyUnicode_KIND(s1); + if (kind != __Pyx_PyUnicode_KIND(s2)) { + goto return_ne; + } + data1 = __Pyx_PyUnicode_DATA(s1); + data2 = __Pyx_PyUnicode_DATA(s2); + if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) { + goto return_ne; + } else if (length == 1) { + goto return_eq; + } else { + int result = memcmp(data1, data2, (size_t)(length * kind)); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & s2_is_unicode) { + goto return_ne; + } else if ((s2 == Py_None) & s1_is_unicode) { + goto return_ne; + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +return_eq: + return (equals == Py_EQ); +return_ne: + return (equals == Py_NE); +#endif +} + +/* fastcall */ +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + Py_ssize_t i, n = __Pyx_PyTuple_GET_SIZE(kwnames); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(n == -1)) return NULL; + #endif + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + if (s == namei) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + int eq = __Pyx_PyUnicode_Equals(s, namei, Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; + return kwvalues[i]; + } + } + return NULL; +} +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues) { + Py_ssize_t i, nkwargs; + PyObject *dict; +#if !CYTHON_ASSUME_SAFE_SIZE + nkwargs = PyTuple_Size(kwnames); + if (unlikely(nkwargs < 0)) return NULL; +#else + nkwargs = PyTuple_GET_SIZE(kwnames); +#endif + dict = PyDict_New(); + if (unlikely(!dict)) + return NULL; + for (i=0; itype, *target->method_name); + if (unlikely(!method)) + return -1; + result = method; +#if CYTHON_COMPILING_IN_CPYTHON + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + { + PyMethodDescrObject *descr = (PyMethodDescrObject*) method; + target->func = descr->d_method->ml_meth; + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); + } else +#endif +#if CYTHON_COMPILING_IN_PYPY +#else + if (PyCFunction_Check(method)) +#endif + { + PyObject *self; + int self_found; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + self = PyObject_GetAttrString(method, "__self__"); + if (!self) { + PyErr_Clear(); + } +#else + self = PyCFunction_GET_SELF(method); +#endif + self_found = (self && self != Py_None); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + Py_XDECREF(self); +#endif + if (self_found) { + PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method); + if (unlikely(!unbound_method)) return -1; + Py_DECREF(method); + result = unbound_method; + } + } +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + if (unlikely(target->method)) { + Py_DECREF(result); + } else +#endif + target->method = result; + return 0; +} + +/* CallUnboundCMethod2 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + PyObject *args[2] = {arg1, arg2}; + if (cfunc->flag == METH_FASTCALL) { + return __Pyx_CallCFunctionFast(cfunc, self, args, 2); + } + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, 2, NULL); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod2(&tmp_cfunc, self, arg1, arg2); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){ + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + PyObject *result = NULL; + PyObject *args = PyTuple_New(2); + if (unlikely(!args)) return NULL; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + if (cfunc->flag & METH_KEYWORDS) + result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL); + else + result = __Pyx_CallCFunction(cfunc, self, args); + Py_DECREF(args); + return result; + } +#endif + { + PyObject *args[4] = {NULL, self, arg1, arg2}; + return __Pyx_PyObject_FastCall(cfunc->method, args+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } +} + +/* ParseKeywords */ +static int __Pyx_ValidateDuplicatePosArgs( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char* function_name) +{ + PyObject ** const *name = argnames; + while (name != first_kw_arg) { + PyObject *key = **name; + int found = PyDict_Contains(kwds, key); + if (unlikely(found)) { + if (found == 1) __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; + } + name++; + } + return 0; +bad: + return -1; +} +#if CYTHON_USE_UNICODE_INTERNALS +static CYTHON_INLINE int __Pyx_UnicodeKeywordsEqual(PyObject *s1, PyObject *s2) { + int kind; + Py_ssize_t len = PyUnicode_GET_LENGTH(s1); + if (len != PyUnicode_GET_LENGTH(s2)) return 0; + kind = PyUnicode_KIND(s1); + if (kind != PyUnicode_KIND(s2)) return 0; + const void *data1 = PyUnicode_DATA(s1); + const void *data2 = PyUnicode_DATA(s2); + return (memcmp(data1, data2, (size_t) len * (size_t) kind) == 0); +} +#endif +static int __Pyx_MatchKeywordArg_str( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + #if CYTHON_USE_UNICODE_INTERNALS + Py_hash_t key_hash = ((PyASCIIObject*)key)->hash; + if (unlikely(key_hash == -1)) { + key_hash = PyObject_Hash(key); + if (unlikely(key_hash == -1)) + goto bad; + } + #endif + name = first_kw_arg; + while (*name) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (key_hash == ((PyASCIIObject*)name_str)->hash && __Pyx_UnicodeKeywordsEqual(name_str, key)) { + *index_found = (size_t) (name - argnames); + return 1; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + *index_found = (size_t) (name - argnames); + return 1; + } + } + #endif + name++; + } + name = argnames; + while (name != first_kw_arg) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (unlikely(key_hash == ((PyASCIIObject*)name_str)->hash)) { + if (__Pyx_UnicodeKeywordsEqual(name_str, key)) + goto arg_passed_twice; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + if (unlikely(name_str == key)) goto arg_passed_twice; + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + } + #endif + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +bad: + return -1; +} +static int __Pyx_MatchKeywordArg_nostr( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; + name = first_kw_arg; + while (*name) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (cmp == 1) { + *index_found = (size_t) (name - argnames); + return 1; + } + if (unlikely(cmp == -1)) goto bad; + name++; + } + name = argnames; + while (name != first_kw_arg) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (unlikely(cmp != 0)) { + if (cmp == 1) goto arg_passed_twice; + else goto bad; + } + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +bad: + return -1; +} +static CYTHON_INLINE int __Pyx_MatchKeywordArg( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + return likely(PyUnicode_CheckExact(key)) ? + __Pyx_MatchKeywordArg_str(key, argnames, first_kw_arg, index_found, function_name) : + __Pyx_MatchKeywordArg_nostr(key, argnames, first_kw_arg, index_found, function_name); +} +static void __Pyx_RejectUnknownKeyword( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char *function_name) +{ + Py_ssize_t pos = 0; + PyObject *key = NULL; + __Pyx_BEGIN_CRITICAL_SECTION(kwds); + while (PyDict_Next(kwds, &pos, &key, NULL)) { + PyObject** const *name = first_kw_arg; + while (*name && (**name != key)) name++; + if (!*name) { + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(key); + #endif + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp != 1) { + if (cmp == 0) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + break; + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + } + } + __Pyx_END_CRITICAL_SECTION(); + assert(PyErr_Occurred()); +} +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t extracted = 0; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + name = first_kw_arg; + while (*name && num_kwargs > extracted) { + PyObject * key = **name; + PyObject *value; + int found = 0; + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + found = PyDict_GetItemRef(kwds, key, &value); + #else + value = PyDict_GetItemWithError(kwds, key); + if (value) { + Py_INCREF(value); + found = 1; + } else { + if (unlikely(PyErr_Occurred())) goto bad; + } + #endif + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + extracted++; + } + name++; + } + if (num_kwargs > extracted) { + if (ignore_unknown_kwargs) { + if (unlikely(__Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name) == -1)) + goto bad; + } else { + __Pyx_RejectUnknownKeyword(kwds, argnames, first_kw_arg, function_name); + goto bad; + } + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t len; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + if (PyDict_Update(kwds2, kwds) < 0) goto bad; + name = first_kw_arg; + while (*name) { + PyObject *key = **name; + PyObject *value; +#if !CYTHON_COMPILING_IN_LIMITED_API && (PY_VERSION_HEX >= 0x030d00A2 || defined(PyDict_Pop)) + int found = PyDict_Pop(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int found = PyDict_GetItemRef(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + if (unlikely(PyDict_DelItem(kwds2, key) < 0)) goto bad; + } +#else + #if CYTHON_COMPILING_IN_CPYTHON + value = _PyDict_Pop(kwds2, key, kwds2); + #else + value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_pop, kwds2, key, kwds2); + #endif + if (value == kwds2) { + Py_DECREF(value); + } else { + if (unlikely(!value)) goto bad; + values[name-argnames] = value; + } +#endif + name++; + } + len = PyDict_Size(kwds2); + if (len > 0) { + return __Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name); + } else if (unlikely(len == -1)) { + goto bad; + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject *key = NULL; + PyObject** const * name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + for (Py_ssize_t pos = 0; pos < num_kwargs; pos++) { +#if CYTHON_AVOID_BORROWED_REFS + key = __Pyx_PySequence_ITEM(kwds, pos); +#else + key = __Pyx_PyTuple_GET_ITEM(kwds, pos); +#endif +#if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!key)) goto bad; +#endif + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + PyObject *value = kwvalues[pos]; + values[name-argnames] = __Pyx_NewRef(value); + } else { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp == 1) { + PyObject *value = kwvalues[pos]; + values[index_found] = __Pyx_NewRef(value); + } else { + if (unlikely(cmp == -1)) goto bad; + if (kwds2) { + PyObject *value = kwvalues[pos]; + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else if (!ignore_unknown_kwargs) { + goto invalid_keyword; + } + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + key = NULL; + #endif + } + return 0; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + goto bad; +bad: + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(key); + #endif + return -1; +} +static int __Pyx_ParseKeywords( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) + return __Pyx_ParseKeywordsTuple(kwds, kwvalues, argnames, kwds2, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); + else if (kwds2) + return __Pyx_ParseKeywordDictToDict(kwds, argnames, kwds2, values, num_pos_args, function_name); + else + return __Pyx_ParseKeywordDict(kwds, argnames, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* ArgTypeTest */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +{ + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; + PyObject *extra_info = __pyx_mstate_global->__pyx_empty_unicode; + int from_annotation_subclass = 0; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (!exact) { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } else if (exact == 2) { + if (__Pyx_TypeCheck(obj, type)) { + from_annotation_subclass = 1; + extra_info = __pyx_mstate_global->__pyx_kp_u_Note_that_Cython_is_deliberately; + } + } + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")" +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + "%s%U" +#endif + , name, type_name, obj_type_name +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + , (from_annotation_subclass ? ". " : ""), extra_info +#endif + ); +#if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + if (exact == 2 && from_annotation_subclass) { + PyObject *res; + PyObject *vargs[2]; + vargs[0] = PyErr_GetRaisedException(); + vargs[1] = extra_info; + res = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_kp_u_add_note, vargs, 2, NULL); + Py_XDECREF(res); + PyErr_SetRaisedException(vargs[0]); + } +#endif + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* RaiseException */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyException_SetTraceback(value, tb); +#elif CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} + +/* Import */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { + PyObject *module = 0; + PyObject *empty_dict = 0; + PyObject *empty_list = 0; + empty_dict = PyDict_New(); + if (unlikely(!empty_dict)) + goto bad; + if (level == -1) { + const char* package_sep = strchr(__Pyx_MODULE_NAME, '.'); + if (package_sep != (0)) { + module = PyImport_ImportModuleLevelObject( + name, __pyx_mstate_global->__pyx_d, empty_dict, from_list, 1); + if (unlikely(!module)) { + if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError))) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } + if (!module) { + module = PyImport_ImportModuleLevelObject( + name, __pyx_mstate_global->__pyx_d, empty_dict, from_list, level); + } +bad: + Py_XDECREF(empty_dict); + Py_XDECREF(empty_list); + return module; +} + +/* ImportDottedModule */ +static PyObject *__Pyx__ImportDottedModule_Error(PyObject *name, PyObject *parts_tuple, Py_ssize_t count) { + PyObject *partial_name = NULL, *slice = NULL, *sep = NULL; + Py_ssize_t size; + if (unlikely(PyErr_Occurred())) { + PyErr_Clear(); + } +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(parts_tuple); +#else + size = PyTuple_Size(parts_tuple); + if (size < 0) goto bad; +#endif + if (likely(size == count)) { + partial_name = name; + } else { + slice = PySequence_GetSlice(parts_tuple, 0, count); + if (unlikely(!slice)) + goto bad; + sep = PyUnicode_FromStringAndSize(".", 1); + if (unlikely(!sep)) + goto bad; + partial_name = PyUnicode_Join(sep, slice); + } + PyErr_Format( + PyExc_ModuleNotFoundError, + "No module named '%U'", partial_name); +bad: + Py_XDECREF(sep); + Py_XDECREF(slice); + Py_XDECREF(partial_name); + return NULL; +} +static PyObject *__Pyx__ImportDottedModule_Lookup(PyObject *name) { + PyObject *imported_module; +#if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + return NULL; + imported_module = __Pyx_PyDict_GetItemStr(modules, name); + Py_XINCREF(imported_module); +#else + imported_module = PyImport_GetModule(name); +#endif + return imported_module; +} +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple) { + Py_ssize_t i, nparts; +#if CYTHON_ASSUME_SAFE_SIZE + nparts = PyTuple_GET_SIZE(parts_tuple); +#else + nparts = PyTuple_Size(parts_tuple); + if (nparts < 0) return NULL; +#endif + for (i=1; i < nparts && module; i++) { + PyObject *part, *submodule; +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + part = PyTuple_GET_ITEM(parts_tuple, i); +#else + part = __Pyx_PySequence_ITEM(parts_tuple, i); + if (!part) return NULL; +#endif + submodule = __Pyx_PyObject_GetAttrStrNoError(module, part); +#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(part); +#endif + Py_DECREF(module); + module = submodule; + } + if (unlikely(!module)) { + return __Pyx__ImportDottedModule_Error(name, parts_tuple, i); + } + return module; +} +static PyObject *__Pyx__ImportDottedModule(PyObject *name, PyObject *parts_tuple) { + PyObject *imported_module; + PyObject *module = __Pyx_Import(name, NULL, 0); + if (!parts_tuple || unlikely(!module)) + return module; + imported_module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(imported_module)) { + Py_DECREF(module); + return imported_module; + } + PyErr_Clear(); + return __Pyx_ImportDottedModule_WalkParts(module, name, parts_tuple); +} +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple) { +#if CYTHON_COMPILING_IN_CPYTHON + PyObject *module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(module)) { + PyObject *spec = __Pyx_PyObject_GetAttrStrNoError(module, __pyx_mstate_global->__pyx_n_u_spec); + if (likely(spec)) { + PyObject *unsafe = __Pyx_PyObject_GetAttrStrNoError(spec, __pyx_mstate_global->__pyx_n_u_initializing); + if (likely(!unsafe || !__Pyx_PyObject_IsTrue(unsafe))) { + Py_DECREF(spec); + spec = NULL; + } + Py_XDECREF(unsafe); + } + if (likely(!spec)) { + PyErr_Clear(); + return module; + } + Py_DECREF(spec); + Py_DECREF(module); + } else if (PyErr_Occurred()) { + PyErr_Clear(); + } +#endif + return __Pyx__ImportDottedModule(name, parts_tuple); +} + +/* ListPack */ +static PyObject *__Pyx_PyList_Pack(Py_ssize_t n, ...) { + va_list va; + PyObject *l = PyList_New(n); + va_start(va, n); + if (unlikely(!l)) goto end; + for (Py_ssize_t i=0; i__pyx_kp_u_); + if (unlikely(!module_dot)) { goto modbad; } + full_name = PyUnicode_Concat(module_dot, name); + if (unlikely(!full_name)) { goto modbad; } + #if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + { + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + goto modbad; + value = PyObject_GetItem(modules, full_name); + } + #else + value = PyImport_GetModule(full_name); + #endif + modbad: + Py_XDECREF(full_name); + Py_XDECREF(module_dot); + Py_XDECREF(module_name); + } + if (unlikely(!value)) { + PyErr_Format(PyExc_ImportError, "cannot import name %S", name); + } + return value; +} + +/* LimitedApiGetTypeDict */ +#if CYTHON_COMPILING_IN_LIMITED_API +static Py_ssize_t __Pyx_GetTypeDictOffset(void) { + PyObject *tp_dictoffset_o; + Py_ssize_t tp_dictoffset; + tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__"); + if (unlikely(!tp_dictoffset_o)) return -1; + tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o); + Py_DECREF(tp_dictoffset_o); + if (unlikely(tp_dictoffset == 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' doesn't have a dictoffset"); + return -1; + } else if (unlikely(tp_dictoffset < 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' has an unexpected negative dictoffset. " + "Please report this as Cython bug"); + return -1; + } + return tp_dictoffset; +} +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) { + static Py_ssize_t tp_dictoffset = 0; + if (unlikely(tp_dictoffset == 0)) { + tp_dictoffset = __Pyx_GetTypeDictOffset(); + if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) { + tp_dictoffset = 0; // try again next time? + return NULL; + } + } + return *(PyObject**)((char*)tp + tp_dictoffset); +} +#endif + +/* SetItemOnTypeDict */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_SetItem(tp_dict, k, v); + if (likely(!result)) { + PyType_Modified(tp); + if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) { + PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name, (PyObject *) tp, k, NULL); + if (!setNameResult) return -1; + Py_DECREF(setNameResult); + } + } + return result; +} + +/* FixUpExtensionType */ +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if __PYX_LIMITED_VERSION_HEX > 0x030900B1 + CYTHON_UNUSED_VAR(spec); + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#else + const PyType_Slot *slot = spec->slots; + int changed = 0; +#if !CYTHON_COMPILING_IN_LIMITED_API + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { +#if !CYTHON_COMPILING_IN_CPYTHON + const +#endif // !CYTHON_COMPILING_IN_CPYTHON) + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); +#if PY_VERSION_HEX >= 0x030800b4 + type->tp_vectorcall_offset = memb->offset; +#else + type->tp_print = (printfunc) memb->offset; +#endif + changed = 1; + } +#endif // CYTHON_METH_FASTCALL +#if !CYTHON_COMPILING_IN_PYPY + else if (strcmp(memb->name, "__module__") == 0) { + PyObject *descr; + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } +#endif // !CYTHON_COMPILING_IN_PYPY + } + memb++; + } + } +#endif // !CYTHON_COMPILING_IN_LIMITED_API +#if !CYTHON_COMPILING_IN_PYPY + slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_getset) + slot++; + if (slot && slot->slot == Py_tp_getset) { + PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc; + while (getset && getset->name) { + if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) { + PyObject *descr = PyDescr_NewGetSet(type, getset); + if (unlikely(!descr)) + return -1; + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *pyname = PyUnicode_FromString(getset->name); + if (unlikely(!pyname)) { + Py_DECREF(descr); + return -1; + } + int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr); + Py_DECREF(pyname); + #else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + #endif + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } + ++getset; + } + } +#endif // !CYTHON_COMPILING_IN_PYPY + if (changed) + PyType_Modified(type); +#endif // PY_VERSION_HEX > 0x030900B1 + return 0; +} + +/* FetchSharedCythonModule */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + return __Pyx_PyImport_AddModuleRef(__PYX_ABI_MODULE_NAME); +} + +/* dict_setdefault */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, + int is_safe_type) { + PyObject* value; + CYTHON_MAYBE_UNUSED_VAR(is_safe_type); +#if CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_CallMethod(d, "setdefault", "OO", key, default_value); +#elif PY_VERSION_HEX >= 0x030d0000 + PyDict_SetDefaultRef(d, key, default_value, &value); +#else + value = PyDict_SetDefault(d, key, default_value); + if (unlikely(!value)) return NULL; + Py_INCREF(value); +#endif + return value; +} + +/* FetchCommonType */ +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject* __Pyx_PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *result = __Pyx_PyType_FromModuleAndSpec(module, spec, bases); + if (result && metaclass) { + PyObject *old_tp = (PyObject*)Py_TYPE(result); + Py_INCREF((PyObject*)metaclass); +#if __PYX_LIMITED_VERSION_HEX >= 0x03090000 + Py_SET_TYPE(result, metaclass); +#else + result->ob_type = metaclass; +#endif + Py_DECREF(old_tp); + } + return result; +} +#else +#define __Pyx_PyType_FromMetaclass(me, mo, s, b) PyType_FromMetaclass(me, mo, s, b) +#endif +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t expected_basicsize) { + Py_ssize_t basicsize; + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (expected_basicsize == 0) { + return 0; // size is inherited, nothing useful to check + } +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) return -1; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = NULL; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; +#else + basicsize = ((PyTypeObject*) cached_type)->tp_basicsize; +#endif + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module = NULL, *cached_type = NULL, *abi_module_dict, *new_cached_type, *py_object_name; + int get_item_ref_result; + const char* object_name = strrchr(spec->name, '.'); + object_name = object_name ? object_name+1 : spec->name; + py_object_name = PyUnicode_FromString(object_name); + if (!py_object_name) return NULL; + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) goto done; + abi_module_dict = PyModule_GetDict(abi_module); + if (!abi_module_dict) goto done; + get_item_ref_result = __Pyx_PyDict_GetItemRef(abi_module_dict, py_object_name, &cached_type); + if (get_item_ref_result == 1) { + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else if (unlikely(get_item_ref_result == -1)) { + goto bad; + } + CYTHON_UNUSED_VAR(module); + cached_type = __Pyx_PyType_FromMetaclass(metaclass, abi_module, spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + new_cached_type = __Pyx_PyDict_SetDefault(abi_module_dict, py_object_name, cached_type, 1); + if (unlikely(new_cached_type != cached_type)) { + if (unlikely(!new_cached_type)) goto bad; + Py_DECREF(cached_type); + cached_type = new_cached_type; + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else { + Py_DECREF(new_cached_type); + } +done: + Py_XDECREF(abi_module); + Py_DECREF(py_object_name); + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* CommonTypesMetaclass */ +static PyObject* __pyx_CommonTypesMetaclass_get_module(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED void* context) { + return PyUnicode_FromString(__PYX_ABI_MODULE_NAME); +} +static PyGetSetDef __pyx_CommonTypesMetaclass_getset[] = { + {"__module__", __pyx_CommonTypesMetaclass_get_module, NULL, NULL, NULL}, + {0, 0, 0, 0, 0} +}; +static PyType_Slot __pyx_CommonTypesMetaclass_slots[] = { + {Py_tp_getset, (void *)__pyx_CommonTypesMetaclass_getset}, + {0, 0} +}; +static PyType_Spec __pyx_CommonTypesMetaclass_spec = { + __PYX_TYPE_MODULE_PREFIX "_common_types_metatype", + 0, + 0, +#if PY_VERSION_HEX >= 0x030A0000 + Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | +#endif + Py_TPFLAGS_DEFAULT, + __pyx_CommonTypesMetaclass_slots +}; +static int __pyx_CommonTypesMetaclass_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + PyObject *bases = PyTuple_Pack(1, &PyType_Type); + if (unlikely(!bases)) { + return -1; + } + mstate->__pyx_CommonTypesMetaclassType = __Pyx_FetchCommonTypeFromSpec(NULL, module, &__pyx_CommonTypesMetaclass_spec, bases); + Py_DECREF(bases); + if (unlikely(mstate->__pyx_CommonTypesMetaclassType == NULL)) { + return -1; + } + return 0; +} + +/* CallTypeTraverse */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) { + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 + if (__Pyx_get_runtime_version() < 0x03090000) return 0; + #endif + if (!always_call) { + PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*); + unsigned long flags = PyType_GetFlags(base); + if (flags & Py_TPFLAGS_HEAPTYPE) { + return 0; + } + } + Py_VISIT((PyObject*)Py_TYPE(o)); + return 0; +} +#endif + +/* PyMethodNew */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + PyObject *result; + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + #if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + { + PyObject *args[] = {func, self}; + result = PyObject_Vectorcall(__pyx_mstate_global->__Pyx_CachedMethodType, args, 2, NULL); + } + #else + result = PyObject_CallFunctionObjArgs(__pyx_mstate_global->__Pyx_CachedMethodType, func, self, NULL); + #endif + return result; +} +#else +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#endif + +/* PyVectorcallFastCallDict */ +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i, pos; + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + #if !CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t nkw = PyDict_Size(kw); + if (unlikely(nkw == -1)) return NULL; + #else + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + #endif + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; + } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (PyDict_Next(kw, &pos, &key, &value)) { + keys_are_strings &= + #if CYTHON_COMPILING_IN_LIMITED_API + PyType_GetFlags(Py_TYPE(key)); + #else + Py_TYPE(key)->tp_flags; + #endif + Py_INCREF(key); + Py_INCREF(value); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(PyTuple_SetItem(kwnames, i, key) < 0)) goto cleanup; + #else + PyTuple_SET_ITEM(kwnames, i, key); + #endif + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + res = vc(func, newargs, nargs, kwnames); +cleanup: + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + Py_ssize_t kw_size = + likely(kw == NULL) ? + 0 : +#if !CYTHON_ASSUME_SAFE_SIZE + PyDict_Size(kw); +#else + PyDict_GET_SIZE(kw); +#endif + if (kw_size == 0) { + return vc(func, args, nargs, NULL); + } +#if !CYTHON_ASSUME_SAFE_SIZE + else if (unlikely(kw_size == -1)) { + return NULL; + } +#endif + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); +} +#endif + +/* CythonFunctionShared */ +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunctionNoMethod(PyObject *func, void (*cfunc)(void)) { + if (__Pyx_CyFunction_Check(func)) { + return PyCFunction_GetFunction(((__pyx_CyFunctionObject*)func)->func) == (PyCFunction) cfunc; + } else if (PyCFunction_Check(func)) { + return PyCFunction_GetFunction(func) == (PyCFunction) cfunc; + } + return 0; +} +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if ((PyObject*)Py_TYPE(func) == __pyx_mstate_global->__Pyx_CachedMethodType) { + int result; + PyObject *newFunc = PyObject_GetAttr(func, __pyx_mstate_global->__pyx_n_u_func); + if (unlikely(!newFunc)) { + PyErr_Clear(); // It's only an optimization, so don't throw an error + return 0; + } + result = __Pyx__IsSameCyOrCFunctionNoMethod(newFunc, cfunc); + Py_DECREF(newFunc); + return result; + } + return __Pyx__IsSameCyOrCFunctionNoMethod(func, cfunc); +} +#else +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if (PyMethod_Check(func)) { + func = PyMethod_GET_FUNCTION(func); + } + return __Pyx_CyOrPyCFunction_Check(func) && __Pyx_CyOrPyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +} +#endif +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#else + __Pyx_Py_XDECREF_SET( + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#endif +} +static PyObject * +__Pyx_CyFunction_get_doc_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_doc == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_doc = PyObject_GetAttrString(op->func, "__doc__"); + if (unlikely(!op->func_doc)) return NULL; +#else + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } +#endif + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + PyObject *result; + CYTHON_UNUSED_VAR(closure); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_doc_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_doc, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_name == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_name = PyObject_GetAttrString(op->func, "__name__"); +#else + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_name_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_name, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + Py_INCREF(op->func_qualname); + result = op->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_dict_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_dict_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL)) { + PyErr_SetString(PyExc_TypeError, + "function's dictionary may not be deleted"); + return -1; + } + if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "setting function's dictionary to a non-dict"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_dict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = __Pyx_PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = __Pyx_PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_defaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_kwdefaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value || value == Py_None) { + value = NULL; + } else if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) { + int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; + if (is_coroutine) { + PyObject *is_coroutine_value, *module, *fromlist, *marker = __pyx_mstate_global->__pyx_n_u_is_coroutine; + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); +#if CYTHON_ASSUME_SAFE_MACROS + PyList_SET_ITEM(fromlist, 0, marker); +#else + if (unlikely(PyList_SetItem(fromlist, 0, marker) < 0)) { + Py_DECREF(marker); + Py_DECREF(fromlist); + return NULL; + } +#endif + module = PyImport_ImportModuleLevelObject(__pyx_mstate_global->__pyx_n_u_asyncio_coroutines, NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + is_coroutine_value = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(is_coroutine_value)) { + return is_coroutine_value; + } +ignore: + PyErr_Clear(); + } + return __Pyx_PyBool_FromLong(is_coroutine); +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + result = __Pyx_CyFunction_get_is_coroutine_value(op); + if (unlikely(!result)) + return NULL; + __Pyx_BEGIN_CRITICAL_SECTION(op); + if (op->func_is_coroutine) { + Py_DECREF(result); + result = __Pyx_NewRef(op->func_is_coroutine); + } else { + op->func_is_coroutine = __Pyx_NewRef(result); + } + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static void __Pyx_CyFunction_raise_argument_count_error(__pyx_CyFunctionObject *func, const char* message, Py_ssize_t size) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + py_name, message, size); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + name, message, size); +#endif +} +static void __Pyx_CyFunction_raise_type_error(__pyx_CyFunctionObject *func, const char* message) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s", + py_name, message); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s", + name, message); +#endif +} +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject * +__Pyx_CyFunction_get_module(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_GetAttrString(op->func, "__module__"); +} +static int +__Pyx_CyFunction_set_module(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_SetAttrString(op->func, "__module__", value); +} +#endif +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {"func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, + {"func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {"__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {"func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API + {"__module__", (getter)__Pyx_CyFunction_get_module, (setter)__Pyx_CyFunction_set_module, 0, 0}, +#endif + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { +#if !CYTHON_COMPILING_IN_LIMITED_API + {"__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, +#endif + {"__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#if CYTHON_METH_FASTCALL +#if CYTHON_BACKPORT_VECTORCALL || CYTHON_COMPILING_IN_LIMITED_API + {"__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, +#else + {"__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {"__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(args); + __Pyx_BEGIN_CRITICAL_SECTION(m); + Py_INCREF(m->func_qualname); + result = m->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { +#if !CYTHON_COMPILING_IN_LIMITED_API + PyCFunctionObject *cf = (PyCFunctionObject*) op; +#endif + if (unlikely(op == NULL)) + return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + op->func = PyCFunction_NewEx(ml, (PyObject*)op, module); + if (unlikely(!op->func)) return NULL; +#endif + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; +#if !CYTHON_COMPILING_IN_LIMITED_API + cf->m_ml = ml; + cf->m_self = (PyObject *) op; +#endif + Py_XINCREF(closure); + op->func_closure = closure; +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_XINCREF(module); + cf->m_module = module; +#endif + op->func_dict = NULL; + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + op->func_classobj = NULL; +#else + ((PyCMethodObject*)op)->mm_class = NULL; +#endif + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func); +#else + Py_CLEAR(((PyCFunctionObject*)m)->m_module); +#endif + Py_CLEAR(m->func_dict); + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API +#if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); +#else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } +#endif +#endif + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); + Py_CLEAR(m->defaults); + return 0; +} +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + __Pyx_PyHeapTypeObject_GC_Del(m); +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + { + int e = __Pyx_call_type_traverse((PyObject*)m, 1, visit, arg); + if (e) return e; + } + Py_VISIT(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func); +#else + Py_VISIT(((PyCFunctionObject*)m)->m_module); +#endif + Py_VISIT(m->func_dict); + __Pyx_VISIT_CONST(m->func_name); + __Pyx_VISIT_CONST(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + __Pyx_VISIT_CONST(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); +#endif + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); + Py_VISIT(m->defaults); + return 0; +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ + PyObject *repr; + __Pyx_BEGIN_CRITICAL_SECTION(op); + repr = PyUnicode_FromFormat("", + op->func_qualname, (void *)op); + __Pyx_END_CRITICAL_SECTION(); + return repr; +} +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *f = ((__pyx_CyFunctionObject*)func)->func; + PyCFunction meth; + int flags; + meth = PyCFunction_GetFunction(f); + if (unlikely(!meth)) return NULL; + flags = PyCFunction_GetFlags(f); + if (unlikely(flags < 0)) return NULL; +#else + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + int flags = f->m_ml->ml_flags; +#endif + Py_ssize_t size; + switch (flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 0)) + return (*meth)(self, NULL); + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes no arguments", size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 1)) { + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = __Pyx_PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif + result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(arg0); + #endif + return result; + } + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes exactly one argument", size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + return NULL; + } + __Pyx_CyFunction_raise_type_error( + (__pyx_CyFunctionObject*)func, "takes no keyword arguments"); + return NULL; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *self, *result; +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)func)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)func)->m_self; +#endif + result = __Pyx_CyFunction_CallMethod(func, self, arg, kw); + return result; +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; +#if CYTHON_ASSUME_SAFE_SIZE + argc = PyTuple_GET_SIZE(args); +#else + argc = PyTuple_Size(args); + if (unlikely(argc < 0)) return NULL; +#endif + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + PyErr_Format(PyExc_TypeError, + "unbound method %.200S() needs an argument", + cyfunc->func_qualname); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; +} +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "needs an argument"); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(__Pyx_PyTuple_GET_SIZE(kwnames))) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "takes no keyword arguments"); + return -1; + } + return ret; +} +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 0)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes no arguments", nargs); + return NULL; + } + return meth(self, NULL); +} +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 1)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes exactly one argument", nargs); + return NULL; + } + return meth(self, args[0]); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))meth)(self, args, nargs, kwnames); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCMethod)(void(*)(void))meth)(self, cls, args, (size_t)nargs, kwnames); +} +#endif +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#if CYTHON_METH_FASTCALL +#if defined(Py_TPFLAGS_HAVE_VECTORCALL) + Py_TPFLAGS_HAVE_VECTORCALL | +#elif defined(_Py_TPFLAGS_HAVE_VECTORCALL) + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif +#endif // CYTHON_METH_FASTCALL +#if PY_VERSION_HEX >= 0x030A0000 + Py_TPFLAGS_IMMUTABLETYPE | +#endif + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, + __pyx_CyFunctionType_slots +}; +static int __pyx_CyFunction_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + mstate->__pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_CyFunctionType_spec, NULL); + if (unlikely(mstate->__pyx_CyFunctionType == NULL)) { + return -1; + } + return 0; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, PyTypeObject *defaults_type) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_CallObject((PyObject*)defaults_type, NULL); // _PyObject_New(defaults_type); + if (unlikely(!m->defaults)) + return NULL; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +/* CythonFunction */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyObject *op = __Pyx_CyFunction_Init( + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_mstate_global->__pyx_CyFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + PyObject_GC_Track(op); + } + return op; +} + +/* CLineInTraceback */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline; + PyObject *ptype, *pvalue, *ptraceback; +#if CYTHON_COMPILING_IN_CPYTHON + PyObject **cython_runtime_dict; +#endif + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); +#if CYTHON_COMPILING_IN_CPYTHON + cython_runtime_dict = _PyObject_GetDictPtr(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __Pyx_BEGIN_CRITICAL_SECTION(*cython_runtime_dict); + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, *cython_runtime_dict, + __Pyx_PyDict_GetItemStr(*cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback)) + Py_XINCREF(use_cline); + __Pyx_END_CRITICAL_SECTION(); + } else +#endif + { + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback); + if (use_cline_obj) { + use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; + Py_INCREF(use_cline); + Py_DECREF(use_cline_obj); + } else { + PyErr_Clear(); + use_cline = NULL; + } + } + if (!use_cline) { + c_line = 0; + (void) PyObject_SetAttr(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False); + } + else if (use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + (void) __pyx_cfilenm; + (void) __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + py_funcname = PyUnicode_FromString(funcname); + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* CIntFromPyVerify */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntFromPy */ +static CYTHON_INLINE size_t __Pyx_PyLong_As_size_t(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + size_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (size_t) -1; + val = __Pyx_PyLong_As_size_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(size_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(size_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 2 * PyLong_SHIFT)) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(size_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 3 * PyLong_SHIFT)) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(size_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 4 * PyLong_SHIFT)) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(size_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(size_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(size_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(size_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(size_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT)) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(size_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(size_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + size_t val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (size_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (size_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (size_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (size_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(size_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((size_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(size_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((size_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((size_t) 1) << (sizeof(size_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (size_t) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to size_t"); + return (size_t) -1; +} + +/* FormatTypeName */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static __Pyx_TypeName +__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) +{ + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u__2); + } + goto done; +} +#endif + +/* PyObjectVectorCallKwBuilder */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if defined(HAVE_LONG_LONG) && !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPy */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + long val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (long) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + int val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030b0000 + return Py_Version & ~0xFFUL; +#else + static unsigned long __Pyx_cached_runtime_version = 0; + if (__Pyx_cached_runtime_version == 0) { + const char* rt_version = Py_GetVersion(); + unsigned long version = 0; + unsigned long factor = 0x01000000UL; + unsigned int digit = 0; + int i = 0; + while (factor) { + while ('0' <= rt_version[i] && rt_version[i] <= '9') { + digit = digit * 10 + (unsigned int) (rt_version[i] - '0'); + ++i; + } + version += factor * digit; + if (rt_version[i] != '.') + break; + digit = 0; + factor >>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } + return __Pyx_cached_runtime_version; +#endif +} + +/* CheckBinaryVersion */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } +} + +/* FunctionExport */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig) { + PyObject *d = 0; + PyObject *cobj = 0; + union { + void (*fp)(void); + void *p; + } tmp; + d = PyObject_GetAttrString(__pyx_m, "__pyx_capi__"); + if (!d) { + PyErr_Clear(); + d = PyDict_New(); + if (!d) + goto bad; + Py_INCREF(d); + if (PyModule_AddObject(__pyx_m, "__pyx_capi__", d) < 0) + goto bad; + } + tmp.fp = f; + cobj = PyCapsule_New(tmp.p, sig, 0); + if (!cobj) + goto bad; + if (PyDict_SetItemString(d, name, cobj) < 0) + goto bad; + Py_DECREF(cobj); + Py_DECREF(d); + return 0; +bad: + Py_XDECREF(cobj); + Py_XDECREF(d); + return -1; +} + +/* NewCodeObj */ +#if CYTHON_COMPILING_IN_LIMITED_API + static PyObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyObject *exception_table = NULL; + PyObject *types_module=NULL, *code_type=NULL, *result=NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + PyObject *version_info; + PyObject *py_minor_version = NULL; + #endif + long minor_version = 0; + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + minor_version = 11; + #else + if (!(version_info = PySys_GetObject("version_info"))) goto end; + if (!(py_minor_version = PySequence_GetItem(version_info, 1))) goto end; + minor_version = PyLong_AsLong(py_minor_version); + Py_DECREF(py_minor_version); + if (minor_version == -1 && PyErr_Occurred()) goto end; + #endif + if (!(types_module = PyImport_ImportModule("types"))) goto end; + if (!(code_type = PyObject_GetAttrString(types_module, "CodeType"))) goto end; + if (minor_version <= 7) { + (void)p; + result = PyObject_CallFunction(code_type, "iiiiiOOOOOOiOOO", a, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else if (minor_version <= 10) { + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOiOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else { + if (!(exception_table = PyBytes_FromStringAndSize(NULL, 0))) goto end; + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOOiOOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, name, fline, lnos, exception_table, fv, cell); + } + end: + Py_XDECREF(code_type); + Py_XDECREF(exception_table); + Py_XDECREF(types_module); + if (type) { + PyErr_Restore(type, value, traceback); + } + return result; + } +#elif PY_VERSION_HEX >= 0x030B0000 + static PyCodeObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyCodeObject *result; + result = + #if PY_VERSION_HEX >= 0x030C0000 + PyUnstable_Code_NewWithPosOnlyArgs + #else + PyCode_NewWithPosOnlyArgs + #endif + (a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, name, fline, lnos, __pyx_mstate_global->__pyx_empty_bytes); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030c00A1 + if (likely(result)) + result->_co_firsttraceable = 0; + #endif + return result; + } +#elif PY_VERSION_HEX >= 0x030800B2 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#else + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#endif +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + const char *line_table, + PyObject *tuple_dedup_map +) { + PyObject *code_obj = NULL, *varnames_tuple_dedup = NULL, *code_bytes = NULL, *line_table_bytes = NULL; + Py_ssize_t var_count = (Py_ssize_t) descr.nlocals; + PyObject *varnames_tuple = PyTuple_New(var_count); + if (unlikely(!varnames_tuple)) return NULL; + for (Py_ssize_t i=0; i < var_count; i++) { + Py_INCREF(varnames[i]); + if (__Pyx_PyTuple_SET_ITEM(varnames_tuple, i, varnames[i]) != (0)) goto done; + } + #if CYTHON_COMPILING_IN_LIMITED_API + varnames_tuple_dedup = PyDict_GetItem(tuple_dedup_map, varnames_tuple); + if (!varnames_tuple_dedup) { + if (unlikely(PyDict_SetItem(tuple_dedup_map, varnames_tuple, varnames_tuple) < 0)) goto done; + varnames_tuple_dedup = varnames_tuple; + } + #else + varnames_tuple_dedup = PyDict_SetDefault(tuple_dedup_map, varnames_tuple, varnames_tuple); + if (unlikely(!varnames_tuple_dedup)) goto done; + #endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(varnames_tuple_dedup); + #endif + if (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table != NULL + && !CYTHON_COMPILING_IN_GRAAL) { + line_table_bytes = PyBytes_FromStringAndSize(line_table, descr.line_table_length); + if (unlikely(!line_table_bytes)) goto done; + Py_ssize_t code_len = (descr.line_table_length * 2 + 4) & ~3; + code_bytes = PyBytes_FromStringAndSize(NULL, code_len); + if (unlikely(!code_bytes)) goto done; + char* c_code_bytes = PyBytes_AsString(code_bytes); + if (unlikely(!c_code_bytes)) goto done; + memset(c_code_bytes, 0, (size_t) code_len); + } + code_obj = (PyObject*) __Pyx__PyCode_New( + (int) descr.argcount, + (int) descr.num_posonly_args, + (int) descr.num_kwonly_args, + (int) descr.nlocals, + 0, + (int) descr.flags, + code_bytes ? code_bytes : __pyx_mstate_global->__pyx_empty_bytes, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + varnames_tuple_dedup, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + filename, + funcname, + (int) descr.first_line, + (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table_bytes) ? line_table_bytes : __pyx_mstate_global->__pyx_empty_bytes + ); +done: + Py_XDECREF(code_bytes); + Py_XDECREF(line_table_bytes); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(varnames_tuple_dedup); + #endif + Py_DECREF(varnames_tuple); + return code_obj; +} + +/* InitStrings */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names) { + while (t->s) { + PyObject *str; + if (t->is_unicode) { + if (t->intern) { + str = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + str = PyUnicode_Decode(t->s, t->n - 1, encoding_names[t->encoding], NULL); + } else { + str = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + str = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + if (!str) + return -1; + *target = str; + if (PyObject_Hash(str) == -1) + return -1; + ++t; + ++target; + } + return 0; +} + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} +#if CYTHON_USE_PYLONG_INTERNALS +static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value) { + if (unlikely(!__Pyx_PyLong_IsCompact(x))) + return 0; + Py_ssize_t value = __Pyx_PyLong_CompactValue(x); + if ((sizeof(long) < sizeof(Py_ssize_t)) && unlikely(value != (long) value)) + return 0; + *return_value = (long) value; + return 1; +} +#endif + + +/* MultiPhaseInitModuleState */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 +#else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 +#endif +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS +#error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" +#endif +#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#define __Pyx_ModuleStateLookup_Lock() +#define __Pyx_ModuleStateLookup_Unlock() +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 +static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; +#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#include +static std::mutex __Pyx_ModuleStateLookup_mutex; +#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() +#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) +#include +static mtx_t __Pyx_ModuleStateLookup_mutex; +static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; +static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); +} +#define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(HAVE_PTHREAD_H) +#include +static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; +#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(_WIN32) +#include // synchapi.h on its own doesn't work +static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; +#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#else +#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." +#endif +typedef struct { + int64_t id; + PyObject *module; +} __Pyx_InterpreterIdAndModule; +typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; +} __Pyx_ModuleStateLookupData; +#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; +#else +static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; +#endif +static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; +} +static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; +#else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); +#endif + return result; + } +} +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); +} +#else +#define __Pyx_ModuleStateLookup_wait_until_no_readers() +#endif +static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; +} +static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; +} +static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); +#else + __Pyx_ModuleStateLookup_data = new_data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return result; +} +static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; +#endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); +#else + __Pyx_ModuleStateLookup_data = data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; +} +#endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink.cpython-312-x86_64-linux-gnu.so b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink.cpython-312-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..6a4d6fc26b267af7551f7b93610db3cb7452af1c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink.cpython-312-x86_64-linux-gnu.so differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink.pxd new file mode 100644 index 0000000000000000000000000000000000000000..85d7c843a683be495c39c3222b0cfe804f6aeb4a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink.pxd @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.0.1 to 12.9.1. Do not modify it directly. + +from ..cynvjitlink cimport * + + +############################################################################### +# Wrapper functions +############################################################################### + +cdef nvJitLinkResult _nvJitLinkCreate(nvJitLinkHandle* handle, uint32_t numOptions, const char** options) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult _nvJitLinkDestroy(nvJitLinkHandle* handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult _nvJitLinkAddData(nvJitLinkHandle handle, nvJitLinkInputType inputType, const void* data, size_t size, const char* name) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult _nvJitLinkAddFile(nvJitLinkHandle handle, nvJitLinkInputType inputType, const char* fileName) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult _nvJitLinkComplete(nvJitLinkHandle handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult _nvJitLinkGetLinkedCubinSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult _nvJitLinkGetLinkedCubin(nvJitLinkHandle handle, void* cubin) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult _nvJitLinkGetLinkedPtxSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult _nvJitLinkGetLinkedPtx(nvJitLinkHandle handle, char* ptx) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult _nvJitLinkGetErrorLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult _nvJitLinkGetErrorLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult _nvJitLinkGetInfoLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult _nvJitLinkGetInfoLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult _nvJitLinkVersion(unsigned int* major, unsigned int* minor) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink.pyx new file mode 100644 index 0000000000000000000000000000000000000000..076df1c54456fdc22bb0d9c2a76fca2de2adfdd6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink.pyx @@ -0,0 +1,383 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.0.1 to 12.9.1. Do not modify it directly. + +from libc.stdint cimport intptr_t, uintptr_t + +import threading +from .utils import FunctionNotFoundError, NotSupportedError + +from cuda.pathfinder import load_nvidia_dynamic_lib + + +############################################################################### +# Extern +############################################################################### + +cdef extern from "" nogil: + void* dlopen(const char*, int) + char* dlerror() + void* dlsym(void*, const char*) + int dlclose(void*) + + enum: + RTLD_LAZY + RTLD_NOW + RTLD_GLOBAL + RTLD_LOCAL + + const void* RTLD_DEFAULT 'RTLD_DEFAULT' + + +############################################################################### +# Wrapper init +############################################################################### + +cdef object __symbol_lock = threading.Lock() +cdef bint __py_nvjitlink_init = False +cdef void* __cuDriverGetVersion = NULL + +cdef void* __nvJitLinkCreate = NULL +cdef void* __nvJitLinkDestroy = NULL +cdef void* __nvJitLinkAddData = NULL +cdef void* __nvJitLinkAddFile = NULL +cdef void* __nvJitLinkComplete = NULL +cdef void* __nvJitLinkGetLinkedCubinSize = NULL +cdef void* __nvJitLinkGetLinkedCubin = NULL +cdef void* __nvJitLinkGetLinkedPtxSize = NULL +cdef void* __nvJitLinkGetLinkedPtx = NULL +cdef void* __nvJitLinkGetErrorLogSize = NULL +cdef void* __nvJitLinkGetErrorLog = NULL +cdef void* __nvJitLinkGetInfoLogSize = NULL +cdef void* __nvJitLinkGetInfoLog = NULL +cdef void* __nvJitLinkVersion = NULL + + +cdef void* load_library() except* with gil: + cdef uintptr_t handle = load_nvidia_dynamic_lib("nvJitLink")._handle_uint + return handle + + +cdef int __check_or_init_nvjitlink() except -1 nogil: + global __py_nvjitlink_init + + cdef void* handle = NULL + + with gil, __symbol_lock: + # Load function + global __nvJitLinkCreate + __nvJitLinkCreate = dlsym(RTLD_DEFAULT, 'nvJitLinkCreate') + if __nvJitLinkCreate == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkCreate = dlsym(handle, 'nvJitLinkCreate') + + global __nvJitLinkDestroy + __nvJitLinkDestroy = dlsym(RTLD_DEFAULT, 'nvJitLinkDestroy') + if __nvJitLinkDestroy == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkDestroy = dlsym(handle, 'nvJitLinkDestroy') + + global __nvJitLinkAddData + __nvJitLinkAddData = dlsym(RTLD_DEFAULT, 'nvJitLinkAddData') + if __nvJitLinkAddData == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkAddData = dlsym(handle, 'nvJitLinkAddData') + + global __nvJitLinkAddFile + __nvJitLinkAddFile = dlsym(RTLD_DEFAULT, 'nvJitLinkAddFile') + if __nvJitLinkAddFile == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkAddFile = dlsym(handle, 'nvJitLinkAddFile') + + global __nvJitLinkComplete + __nvJitLinkComplete = dlsym(RTLD_DEFAULT, 'nvJitLinkComplete') + if __nvJitLinkComplete == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkComplete = dlsym(handle, 'nvJitLinkComplete') + + global __nvJitLinkGetLinkedCubinSize + __nvJitLinkGetLinkedCubinSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubinSize') + if __nvJitLinkGetLinkedCubinSize == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetLinkedCubinSize = dlsym(handle, 'nvJitLinkGetLinkedCubinSize') + + global __nvJitLinkGetLinkedCubin + __nvJitLinkGetLinkedCubin = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubin') + if __nvJitLinkGetLinkedCubin == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetLinkedCubin = dlsym(handle, 'nvJitLinkGetLinkedCubin') + + global __nvJitLinkGetLinkedPtxSize + __nvJitLinkGetLinkedPtxSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtxSize') + if __nvJitLinkGetLinkedPtxSize == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetLinkedPtxSize = dlsym(handle, 'nvJitLinkGetLinkedPtxSize') + + global __nvJitLinkGetLinkedPtx + __nvJitLinkGetLinkedPtx = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtx') + if __nvJitLinkGetLinkedPtx == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetLinkedPtx = dlsym(handle, 'nvJitLinkGetLinkedPtx') + + global __nvJitLinkGetErrorLogSize + __nvJitLinkGetErrorLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLogSize') + if __nvJitLinkGetErrorLogSize == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetErrorLogSize = dlsym(handle, 'nvJitLinkGetErrorLogSize') + + global __nvJitLinkGetErrorLog + __nvJitLinkGetErrorLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLog') + if __nvJitLinkGetErrorLog == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetErrorLog = dlsym(handle, 'nvJitLinkGetErrorLog') + + global __nvJitLinkGetInfoLogSize + __nvJitLinkGetInfoLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLogSize') + if __nvJitLinkGetInfoLogSize == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetInfoLogSize = dlsym(handle, 'nvJitLinkGetInfoLogSize') + + global __nvJitLinkGetInfoLog + __nvJitLinkGetInfoLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLog') + if __nvJitLinkGetInfoLog == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetInfoLog = dlsym(handle, 'nvJitLinkGetInfoLog') + + global __nvJitLinkVersion + __nvJitLinkVersion = dlsym(RTLD_DEFAULT, 'nvJitLinkVersion') + if __nvJitLinkVersion == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkVersion = dlsym(handle, 'nvJitLinkVersion') + + __py_nvjitlink_init = True + return 0 + + +cdef inline int _check_or_init_nvjitlink() except -1 nogil: + if __py_nvjitlink_init: + return 0 + + return __check_or_init_nvjitlink() + +cdef dict func_ptrs = None + + +cpdef dict _inspect_function_pointers(): + global func_ptrs + if func_ptrs is not None: + return func_ptrs + + _check_or_init_nvjitlink() + cdef dict data = {} + + global __nvJitLinkCreate + data["__nvJitLinkCreate"] = __nvJitLinkCreate + + global __nvJitLinkDestroy + data["__nvJitLinkDestroy"] = __nvJitLinkDestroy + + global __nvJitLinkAddData + data["__nvJitLinkAddData"] = __nvJitLinkAddData + + global __nvJitLinkAddFile + data["__nvJitLinkAddFile"] = __nvJitLinkAddFile + + global __nvJitLinkComplete + data["__nvJitLinkComplete"] = __nvJitLinkComplete + + global __nvJitLinkGetLinkedCubinSize + data["__nvJitLinkGetLinkedCubinSize"] = __nvJitLinkGetLinkedCubinSize + + global __nvJitLinkGetLinkedCubin + data["__nvJitLinkGetLinkedCubin"] = __nvJitLinkGetLinkedCubin + + global __nvJitLinkGetLinkedPtxSize + data["__nvJitLinkGetLinkedPtxSize"] = __nvJitLinkGetLinkedPtxSize + + global __nvJitLinkGetLinkedPtx + data["__nvJitLinkGetLinkedPtx"] = __nvJitLinkGetLinkedPtx + + global __nvJitLinkGetErrorLogSize + data["__nvJitLinkGetErrorLogSize"] = __nvJitLinkGetErrorLogSize + + global __nvJitLinkGetErrorLog + data["__nvJitLinkGetErrorLog"] = __nvJitLinkGetErrorLog + + global __nvJitLinkGetInfoLogSize + data["__nvJitLinkGetInfoLogSize"] = __nvJitLinkGetInfoLogSize + + global __nvJitLinkGetInfoLog + data["__nvJitLinkGetInfoLog"] = __nvJitLinkGetInfoLog + + global __nvJitLinkVersion + data["__nvJitLinkVersion"] = __nvJitLinkVersion + + func_ptrs = data + return data + + +cpdef _inspect_function_pointer(str name): + global func_ptrs + if func_ptrs is None: + func_ptrs = _inspect_function_pointers() + return func_ptrs[name] + + +############################################################################### +# Wrapper functions +############################################################################### + +cdef nvJitLinkResult _nvJitLinkCreate(nvJitLinkHandle* handle, uint32_t numOptions, const char** options) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkCreate + _check_or_init_nvjitlink() + if __nvJitLinkCreate == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkCreate is not found") + return (__nvJitLinkCreate)( + handle, numOptions, options) + + +cdef nvJitLinkResult _nvJitLinkDestroy(nvJitLinkHandle* handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkDestroy + _check_or_init_nvjitlink() + if __nvJitLinkDestroy == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkDestroy is not found") + return (__nvJitLinkDestroy)( + handle) + + +cdef nvJitLinkResult _nvJitLinkAddData(nvJitLinkHandle handle, nvJitLinkInputType inputType, const void* data, size_t size, const char* name) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkAddData + _check_or_init_nvjitlink() + if __nvJitLinkAddData == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkAddData is not found") + return (__nvJitLinkAddData)( + handle, inputType, data, size, name) + + +cdef nvJitLinkResult _nvJitLinkAddFile(nvJitLinkHandle handle, nvJitLinkInputType inputType, const char* fileName) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkAddFile + _check_or_init_nvjitlink() + if __nvJitLinkAddFile == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkAddFile is not found") + return (__nvJitLinkAddFile)( + handle, inputType, fileName) + + +cdef nvJitLinkResult _nvJitLinkComplete(nvJitLinkHandle handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkComplete + _check_or_init_nvjitlink() + if __nvJitLinkComplete == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkComplete is not found") + return (__nvJitLinkComplete)( + handle) + + +cdef nvJitLinkResult _nvJitLinkGetLinkedCubinSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetLinkedCubinSize + _check_or_init_nvjitlink() + if __nvJitLinkGetLinkedCubinSize == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetLinkedCubinSize is not found") + return (__nvJitLinkGetLinkedCubinSize)( + handle, size) + + +cdef nvJitLinkResult _nvJitLinkGetLinkedCubin(nvJitLinkHandle handle, void* cubin) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetLinkedCubin + _check_or_init_nvjitlink() + if __nvJitLinkGetLinkedCubin == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetLinkedCubin is not found") + return (__nvJitLinkGetLinkedCubin)( + handle, cubin) + + +cdef nvJitLinkResult _nvJitLinkGetLinkedPtxSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetLinkedPtxSize + _check_or_init_nvjitlink() + if __nvJitLinkGetLinkedPtxSize == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetLinkedPtxSize is not found") + return (__nvJitLinkGetLinkedPtxSize)( + handle, size) + + +cdef nvJitLinkResult _nvJitLinkGetLinkedPtx(nvJitLinkHandle handle, char* ptx) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetLinkedPtx + _check_or_init_nvjitlink() + if __nvJitLinkGetLinkedPtx == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetLinkedPtx is not found") + return (__nvJitLinkGetLinkedPtx)( + handle, ptx) + + +cdef nvJitLinkResult _nvJitLinkGetErrorLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetErrorLogSize + _check_or_init_nvjitlink() + if __nvJitLinkGetErrorLogSize == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetErrorLogSize is not found") + return (__nvJitLinkGetErrorLogSize)( + handle, size) + + +cdef nvJitLinkResult _nvJitLinkGetErrorLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetErrorLog + _check_or_init_nvjitlink() + if __nvJitLinkGetErrorLog == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetErrorLog is not found") + return (__nvJitLinkGetErrorLog)( + handle, log) + + +cdef nvJitLinkResult _nvJitLinkGetInfoLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetInfoLogSize + _check_or_init_nvjitlink() + if __nvJitLinkGetInfoLogSize == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetInfoLogSize is not found") + return (__nvJitLinkGetInfoLogSize)( + handle, size) + + +cdef nvJitLinkResult _nvJitLinkGetInfoLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetInfoLog + _check_or_init_nvjitlink() + if __nvJitLinkGetInfoLog == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetInfoLog is not found") + return (__nvJitLinkGetInfoLog)( + handle, log) + + +cdef nvJitLinkResult _nvJitLinkVersion(unsigned int* major, unsigned int* minor) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkVersion + _check_or_init_nvjitlink() + if __nvJitLinkVersion == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkVersion is not found") + return (__nvJitLinkVersion)( + major, minor) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink_linux.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink_linux.pyx new file mode 100644 index 0000000000000000000000000000000000000000..076df1c54456fdc22bb0d9c2a76fca2de2adfdd6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink_linux.pyx @@ -0,0 +1,383 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.0.1 to 12.9.1. Do not modify it directly. + +from libc.stdint cimport intptr_t, uintptr_t + +import threading +from .utils import FunctionNotFoundError, NotSupportedError + +from cuda.pathfinder import load_nvidia_dynamic_lib + + +############################################################################### +# Extern +############################################################################### + +cdef extern from "" nogil: + void* dlopen(const char*, int) + char* dlerror() + void* dlsym(void*, const char*) + int dlclose(void*) + + enum: + RTLD_LAZY + RTLD_NOW + RTLD_GLOBAL + RTLD_LOCAL + + const void* RTLD_DEFAULT 'RTLD_DEFAULT' + + +############################################################################### +# Wrapper init +############################################################################### + +cdef object __symbol_lock = threading.Lock() +cdef bint __py_nvjitlink_init = False +cdef void* __cuDriverGetVersion = NULL + +cdef void* __nvJitLinkCreate = NULL +cdef void* __nvJitLinkDestroy = NULL +cdef void* __nvJitLinkAddData = NULL +cdef void* __nvJitLinkAddFile = NULL +cdef void* __nvJitLinkComplete = NULL +cdef void* __nvJitLinkGetLinkedCubinSize = NULL +cdef void* __nvJitLinkGetLinkedCubin = NULL +cdef void* __nvJitLinkGetLinkedPtxSize = NULL +cdef void* __nvJitLinkGetLinkedPtx = NULL +cdef void* __nvJitLinkGetErrorLogSize = NULL +cdef void* __nvJitLinkGetErrorLog = NULL +cdef void* __nvJitLinkGetInfoLogSize = NULL +cdef void* __nvJitLinkGetInfoLog = NULL +cdef void* __nvJitLinkVersion = NULL + + +cdef void* load_library() except* with gil: + cdef uintptr_t handle = load_nvidia_dynamic_lib("nvJitLink")._handle_uint + return handle + + +cdef int __check_or_init_nvjitlink() except -1 nogil: + global __py_nvjitlink_init + + cdef void* handle = NULL + + with gil, __symbol_lock: + # Load function + global __nvJitLinkCreate + __nvJitLinkCreate = dlsym(RTLD_DEFAULT, 'nvJitLinkCreate') + if __nvJitLinkCreate == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkCreate = dlsym(handle, 'nvJitLinkCreate') + + global __nvJitLinkDestroy + __nvJitLinkDestroy = dlsym(RTLD_DEFAULT, 'nvJitLinkDestroy') + if __nvJitLinkDestroy == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkDestroy = dlsym(handle, 'nvJitLinkDestroy') + + global __nvJitLinkAddData + __nvJitLinkAddData = dlsym(RTLD_DEFAULT, 'nvJitLinkAddData') + if __nvJitLinkAddData == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkAddData = dlsym(handle, 'nvJitLinkAddData') + + global __nvJitLinkAddFile + __nvJitLinkAddFile = dlsym(RTLD_DEFAULT, 'nvJitLinkAddFile') + if __nvJitLinkAddFile == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkAddFile = dlsym(handle, 'nvJitLinkAddFile') + + global __nvJitLinkComplete + __nvJitLinkComplete = dlsym(RTLD_DEFAULT, 'nvJitLinkComplete') + if __nvJitLinkComplete == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkComplete = dlsym(handle, 'nvJitLinkComplete') + + global __nvJitLinkGetLinkedCubinSize + __nvJitLinkGetLinkedCubinSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubinSize') + if __nvJitLinkGetLinkedCubinSize == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetLinkedCubinSize = dlsym(handle, 'nvJitLinkGetLinkedCubinSize') + + global __nvJitLinkGetLinkedCubin + __nvJitLinkGetLinkedCubin = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedCubin') + if __nvJitLinkGetLinkedCubin == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetLinkedCubin = dlsym(handle, 'nvJitLinkGetLinkedCubin') + + global __nvJitLinkGetLinkedPtxSize + __nvJitLinkGetLinkedPtxSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtxSize') + if __nvJitLinkGetLinkedPtxSize == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetLinkedPtxSize = dlsym(handle, 'nvJitLinkGetLinkedPtxSize') + + global __nvJitLinkGetLinkedPtx + __nvJitLinkGetLinkedPtx = dlsym(RTLD_DEFAULT, 'nvJitLinkGetLinkedPtx') + if __nvJitLinkGetLinkedPtx == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetLinkedPtx = dlsym(handle, 'nvJitLinkGetLinkedPtx') + + global __nvJitLinkGetErrorLogSize + __nvJitLinkGetErrorLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLogSize') + if __nvJitLinkGetErrorLogSize == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetErrorLogSize = dlsym(handle, 'nvJitLinkGetErrorLogSize') + + global __nvJitLinkGetErrorLog + __nvJitLinkGetErrorLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetErrorLog') + if __nvJitLinkGetErrorLog == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetErrorLog = dlsym(handle, 'nvJitLinkGetErrorLog') + + global __nvJitLinkGetInfoLogSize + __nvJitLinkGetInfoLogSize = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLogSize') + if __nvJitLinkGetInfoLogSize == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetInfoLogSize = dlsym(handle, 'nvJitLinkGetInfoLogSize') + + global __nvJitLinkGetInfoLog + __nvJitLinkGetInfoLog = dlsym(RTLD_DEFAULT, 'nvJitLinkGetInfoLog') + if __nvJitLinkGetInfoLog == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkGetInfoLog = dlsym(handle, 'nvJitLinkGetInfoLog') + + global __nvJitLinkVersion + __nvJitLinkVersion = dlsym(RTLD_DEFAULT, 'nvJitLinkVersion') + if __nvJitLinkVersion == NULL: + if handle == NULL: + handle = load_library() + __nvJitLinkVersion = dlsym(handle, 'nvJitLinkVersion') + + __py_nvjitlink_init = True + return 0 + + +cdef inline int _check_or_init_nvjitlink() except -1 nogil: + if __py_nvjitlink_init: + return 0 + + return __check_or_init_nvjitlink() + +cdef dict func_ptrs = None + + +cpdef dict _inspect_function_pointers(): + global func_ptrs + if func_ptrs is not None: + return func_ptrs + + _check_or_init_nvjitlink() + cdef dict data = {} + + global __nvJitLinkCreate + data["__nvJitLinkCreate"] = __nvJitLinkCreate + + global __nvJitLinkDestroy + data["__nvJitLinkDestroy"] = __nvJitLinkDestroy + + global __nvJitLinkAddData + data["__nvJitLinkAddData"] = __nvJitLinkAddData + + global __nvJitLinkAddFile + data["__nvJitLinkAddFile"] = __nvJitLinkAddFile + + global __nvJitLinkComplete + data["__nvJitLinkComplete"] = __nvJitLinkComplete + + global __nvJitLinkGetLinkedCubinSize + data["__nvJitLinkGetLinkedCubinSize"] = __nvJitLinkGetLinkedCubinSize + + global __nvJitLinkGetLinkedCubin + data["__nvJitLinkGetLinkedCubin"] = __nvJitLinkGetLinkedCubin + + global __nvJitLinkGetLinkedPtxSize + data["__nvJitLinkGetLinkedPtxSize"] = __nvJitLinkGetLinkedPtxSize + + global __nvJitLinkGetLinkedPtx + data["__nvJitLinkGetLinkedPtx"] = __nvJitLinkGetLinkedPtx + + global __nvJitLinkGetErrorLogSize + data["__nvJitLinkGetErrorLogSize"] = __nvJitLinkGetErrorLogSize + + global __nvJitLinkGetErrorLog + data["__nvJitLinkGetErrorLog"] = __nvJitLinkGetErrorLog + + global __nvJitLinkGetInfoLogSize + data["__nvJitLinkGetInfoLogSize"] = __nvJitLinkGetInfoLogSize + + global __nvJitLinkGetInfoLog + data["__nvJitLinkGetInfoLog"] = __nvJitLinkGetInfoLog + + global __nvJitLinkVersion + data["__nvJitLinkVersion"] = __nvJitLinkVersion + + func_ptrs = data + return data + + +cpdef _inspect_function_pointer(str name): + global func_ptrs + if func_ptrs is None: + func_ptrs = _inspect_function_pointers() + return func_ptrs[name] + + +############################################################################### +# Wrapper functions +############################################################################### + +cdef nvJitLinkResult _nvJitLinkCreate(nvJitLinkHandle* handle, uint32_t numOptions, const char** options) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkCreate + _check_or_init_nvjitlink() + if __nvJitLinkCreate == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkCreate is not found") + return (__nvJitLinkCreate)( + handle, numOptions, options) + + +cdef nvJitLinkResult _nvJitLinkDestroy(nvJitLinkHandle* handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkDestroy + _check_or_init_nvjitlink() + if __nvJitLinkDestroy == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkDestroy is not found") + return (__nvJitLinkDestroy)( + handle) + + +cdef nvJitLinkResult _nvJitLinkAddData(nvJitLinkHandle handle, nvJitLinkInputType inputType, const void* data, size_t size, const char* name) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkAddData + _check_or_init_nvjitlink() + if __nvJitLinkAddData == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkAddData is not found") + return (__nvJitLinkAddData)( + handle, inputType, data, size, name) + + +cdef nvJitLinkResult _nvJitLinkAddFile(nvJitLinkHandle handle, nvJitLinkInputType inputType, const char* fileName) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkAddFile + _check_or_init_nvjitlink() + if __nvJitLinkAddFile == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkAddFile is not found") + return (__nvJitLinkAddFile)( + handle, inputType, fileName) + + +cdef nvJitLinkResult _nvJitLinkComplete(nvJitLinkHandle handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkComplete + _check_or_init_nvjitlink() + if __nvJitLinkComplete == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkComplete is not found") + return (__nvJitLinkComplete)( + handle) + + +cdef nvJitLinkResult _nvJitLinkGetLinkedCubinSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetLinkedCubinSize + _check_or_init_nvjitlink() + if __nvJitLinkGetLinkedCubinSize == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetLinkedCubinSize is not found") + return (__nvJitLinkGetLinkedCubinSize)( + handle, size) + + +cdef nvJitLinkResult _nvJitLinkGetLinkedCubin(nvJitLinkHandle handle, void* cubin) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetLinkedCubin + _check_or_init_nvjitlink() + if __nvJitLinkGetLinkedCubin == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetLinkedCubin is not found") + return (__nvJitLinkGetLinkedCubin)( + handle, cubin) + + +cdef nvJitLinkResult _nvJitLinkGetLinkedPtxSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetLinkedPtxSize + _check_or_init_nvjitlink() + if __nvJitLinkGetLinkedPtxSize == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetLinkedPtxSize is not found") + return (__nvJitLinkGetLinkedPtxSize)( + handle, size) + + +cdef nvJitLinkResult _nvJitLinkGetLinkedPtx(nvJitLinkHandle handle, char* ptx) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetLinkedPtx + _check_or_init_nvjitlink() + if __nvJitLinkGetLinkedPtx == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetLinkedPtx is not found") + return (__nvJitLinkGetLinkedPtx)( + handle, ptx) + + +cdef nvJitLinkResult _nvJitLinkGetErrorLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetErrorLogSize + _check_or_init_nvjitlink() + if __nvJitLinkGetErrorLogSize == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetErrorLogSize is not found") + return (__nvJitLinkGetErrorLogSize)( + handle, size) + + +cdef nvJitLinkResult _nvJitLinkGetErrorLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetErrorLog + _check_or_init_nvjitlink() + if __nvJitLinkGetErrorLog == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetErrorLog is not found") + return (__nvJitLinkGetErrorLog)( + handle, log) + + +cdef nvJitLinkResult _nvJitLinkGetInfoLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetInfoLogSize + _check_or_init_nvjitlink() + if __nvJitLinkGetInfoLogSize == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetInfoLogSize is not found") + return (__nvJitLinkGetInfoLogSize)( + handle, size) + + +cdef nvJitLinkResult _nvJitLinkGetInfoLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetInfoLog + _check_or_init_nvjitlink() + if __nvJitLinkGetInfoLog == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetInfoLog is not found") + return (__nvJitLinkGetInfoLog)( + handle, log) + + +cdef nvJitLinkResult _nvJitLinkVersion(unsigned int* major, unsigned int* minor) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkVersion + _check_or_init_nvjitlink() + if __nvJitLinkVersion == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkVersion is not found") + return (__nvJitLinkVersion)( + major, minor) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink_windows.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink_windows.pyx new file mode 100644 index 0000000000000000000000000000000000000000..1ad0def5ab442ec7e005dd42a9b7862dc5f48280 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvjitlink_windows.pyx @@ -0,0 +1,362 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.0.1 to 12.9.1. Do not modify it directly. + +from libc.stdint cimport intptr_t + +import threading +from .utils import FunctionNotFoundError, NotSupportedError + +from cuda.pathfinder import load_nvidia_dynamic_lib + +from libc.stddef cimport wchar_t +from libc.stdint cimport uintptr_t +from cpython cimport PyUnicode_AsWideCharString, PyMem_Free + +from .utils import NotSupportedError + +cdef extern from "windows.h" nogil: + ctypedef void* HMODULE + ctypedef void* HANDLE + ctypedef void* FARPROC + ctypedef unsigned long DWORD + ctypedef const wchar_t *LPCWSTR + ctypedef const char *LPCSTR + + cdef DWORD LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800 + cdef DWORD LOAD_LIBRARY_SEARCH_DEFAULT_DIRS = 0x00001000 + cdef DWORD LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR = 0x00000100 + + HMODULE _LoadLibraryExW "LoadLibraryExW"( + LPCWSTR lpLibFileName, + HANDLE hFile, + DWORD dwFlags + ) + + FARPROC _GetProcAddress "GetProcAddress"(HMODULE hModule, LPCSTR lpProcName) + +cdef inline uintptr_t LoadLibraryExW(str path, HANDLE hFile, DWORD dwFlags): + cdef uintptr_t result + cdef wchar_t* wpath = PyUnicode_AsWideCharString(path, NULL) + with nogil: + result = _LoadLibraryExW( + wpath, + hFile, + dwFlags + ) + PyMem_Free(wpath) + return result + +cdef inline void *GetProcAddress(uintptr_t hModule, const char* lpProcName) nogil: + return _GetProcAddress(hModule, lpProcName) + +cdef int get_cuda_version(): + cdef int err, driver_ver = 0 + + # Load driver to check version + handle = LoadLibraryExW("nvcuda.dll", NULL, LOAD_LIBRARY_SEARCH_SYSTEM32) + if handle == 0: + raise NotSupportedError('CUDA driver is not found') + cuDriverGetVersion = GetProcAddress(handle, 'cuDriverGetVersion') + if cuDriverGetVersion == NULL: + raise RuntimeError('something went wrong') + err = (cuDriverGetVersion)(&driver_ver) + if err != 0: + raise RuntimeError('something went wrong') + + return driver_ver + + + +############################################################################### +# Wrapper init +############################################################################### + +cdef object __symbol_lock = threading.Lock() +cdef bint __py_nvjitlink_init = False + +cdef void* __nvJitLinkCreate = NULL +cdef void* __nvJitLinkDestroy = NULL +cdef void* __nvJitLinkAddData = NULL +cdef void* __nvJitLinkAddFile = NULL +cdef void* __nvJitLinkComplete = NULL +cdef void* __nvJitLinkGetLinkedCubinSize = NULL +cdef void* __nvJitLinkGetLinkedCubin = NULL +cdef void* __nvJitLinkGetLinkedPtxSize = NULL +cdef void* __nvJitLinkGetLinkedPtx = NULL +cdef void* __nvJitLinkGetErrorLogSize = NULL +cdef void* __nvJitLinkGetErrorLog = NULL +cdef void* __nvJitLinkGetInfoLogSize = NULL +cdef void* __nvJitLinkGetInfoLog = NULL +cdef void* __nvJitLinkVersion = NULL + + +cdef int __check_or_init_nvjitlink() except -1 nogil: + global __py_nvjitlink_init + + with gil, __symbol_lock: + # Load library + handle = load_nvidia_dynamic_lib("nvJitLink")._handle_uint + + # Load function + global __nvJitLinkCreate + __nvJitLinkCreate = GetProcAddress(handle, 'nvJitLinkCreate') + + global __nvJitLinkDestroy + __nvJitLinkDestroy = GetProcAddress(handle, 'nvJitLinkDestroy') + + global __nvJitLinkAddData + __nvJitLinkAddData = GetProcAddress(handle, 'nvJitLinkAddData') + + global __nvJitLinkAddFile + __nvJitLinkAddFile = GetProcAddress(handle, 'nvJitLinkAddFile') + + global __nvJitLinkComplete + __nvJitLinkComplete = GetProcAddress(handle, 'nvJitLinkComplete') + + global __nvJitLinkGetLinkedCubinSize + __nvJitLinkGetLinkedCubinSize = GetProcAddress(handle, 'nvJitLinkGetLinkedCubinSize') + + global __nvJitLinkGetLinkedCubin + __nvJitLinkGetLinkedCubin = GetProcAddress(handle, 'nvJitLinkGetLinkedCubin') + + global __nvJitLinkGetLinkedPtxSize + __nvJitLinkGetLinkedPtxSize = GetProcAddress(handle, 'nvJitLinkGetLinkedPtxSize') + + global __nvJitLinkGetLinkedPtx + __nvJitLinkGetLinkedPtx = GetProcAddress(handle, 'nvJitLinkGetLinkedPtx') + + global __nvJitLinkGetErrorLogSize + __nvJitLinkGetErrorLogSize = GetProcAddress(handle, 'nvJitLinkGetErrorLogSize') + + global __nvJitLinkGetErrorLog + __nvJitLinkGetErrorLog = GetProcAddress(handle, 'nvJitLinkGetErrorLog') + + global __nvJitLinkGetInfoLogSize + __nvJitLinkGetInfoLogSize = GetProcAddress(handle, 'nvJitLinkGetInfoLogSize') + + global __nvJitLinkGetInfoLog + __nvJitLinkGetInfoLog = GetProcAddress(handle, 'nvJitLinkGetInfoLog') + + global __nvJitLinkVersion + __nvJitLinkVersion = GetProcAddress(handle, 'nvJitLinkVersion') + + __py_nvjitlink_init = True + return 0 + + +cdef inline int _check_or_init_nvjitlink() except -1 nogil: + if __py_nvjitlink_init: + return 0 + + return __check_or_init_nvjitlink() + + +cdef dict func_ptrs = None + + +cpdef dict _inspect_function_pointers(): + global func_ptrs + if func_ptrs is not None: + return func_ptrs + + _check_or_init_nvjitlink() + cdef dict data = {} + + global __nvJitLinkCreate + data["__nvJitLinkCreate"] = __nvJitLinkCreate + + global __nvJitLinkDestroy + data["__nvJitLinkDestroy"] = __nvJitLinkDestroy + + global __nvJitLinkAddData + data["__nvJitLinkAddData"] = __nvJitLinkAddData + + global __nvJitLinkAddFile + data["__nvJitLinkAddFile"] = __nvJitLinkAddFile + + global __nvJitLinkComplete + data["__nvJitLinkComplete"] = __nvJitLinkComplete + + global __nvJitLinkGetLinkedCubinSize + data["__nvJitLinkGetLinkedCubinSize"] = __nvJitLinkGetLinkedCubinSize + + global __nvJitLinkGetLinkedCubin + data["__nvJitLinkGetLinkedCubin"] = __nvJitLinkGetLinkedCubin + + global __nvJitLinkGetLinkedPtxSize + data["__nvJitLinkGetLinkedPtxSize"] = __nvJitLinkGetLinkedPtxSize + + global __nvJitLinkGetLinkedPtx + data["__nvJitLinkGetLinkedPtx"] = __nvJitLinkGetLinkedPtx + + global __nvJitLinkGetErrorLogSize + data["__nvJitLinkGetErrorLogSize"] = __nvJitLinkGetErrorLogSize + + global __nvJitLinkGetErrorLog + data["__nvJitLinkGetErrorLog"] = __nvJitLinkGetErrorLog + + global __nvJitLinkGetInfoLogSize + data["__nvJitLinkGetInfoLogSize"] = __nvJitLinkGetInfoLogSize + + global __nvJitLinkGetInfoLog + data["__nvJitLinkGetInfoLog"] = __nvJitLinkGetInfoLog + + global __nvJitLinkVersion + data["__nvJitLinkVersion"] = __nvJitLinkVersion + + func_ptrs = data + return data + + +cpdef _inspect_function_pointer(str name): + global func_ptrs + if func_ptrs is None: + func_ptrs = _inspect_function_pointers() + return func_ptrs[name] + + +############################################################################### +# Wrapper functions +############################################################################### + +cdef nvJitLinkResult _nvJitLinkCreate(nvJitLinkHandle* handle, uint32_t numOptions, const char** options) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkCreate + _check_or_init_nvjitlink() + if __nvJitLinkCreate == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkCreate is not found") + return (__nvJitLinkCreate)( + handle, numOptions, options) + + +cdef nvJitLinkResult _nvJitLinkDestroy(nvJitLinkHandle* handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkDestroy + _check_or_init_nvjitlink() + if __nvJitLinkDestroy == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkDestroy is not found") + return (__nvJitLinkDestroy)( + handle) + + +cdef nvJitLinkResult _nvJitLinkAddData(nvJitLinkHandle handle, nvJitLinkInputType inputType, const void* data, size_t size, const char* name) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkAddData + _check_or_init_nvjitlink() + if __nvJitLinkAddData == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkAddData is not found") + return (__nvJitLinkAddData)( + handle, inputType, data, size, name) + + +cdef nvJitLinkResult _nvJitLinkAddFile(nvJitLinkHandle handle, nvJitLinkInputType inputType, const char* fileName) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkAddFile + _check_or_init_nvjitlink() + if __nvJitLinkAddFile == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkAddFile is not found") + return (__nvJitLinkAddFile)( + handle, inputType, fileName) + + +cdef nvJitLinkResult _nvJitLinkComplete(nvJitLinkHandle handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkComplete + _check_or_init_nvjitlink() + if __nvJitLinkComplete == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkComplete is not found") + return (__nvJitLinkComplete)( + handle) + + +cdef nvJitLinkResult _nvJitLinkGetLinkedCubinSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetLinkedCubinSize + _check_or_init_nvjitlink() + if __nvJitLinkGetLinkedCubinSize == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetLinkedCubinSize is not found") + return (__nvJitLinkGetLinkedCubinSize)( + handle, size) + + +cdef nvJitLinkResult _nvJitLinkGetLinkedCubin(nvJitLinkHandle handle, void* cubin) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetLinkedCubin + _check_or_init_nvjitlink() + if __nvJitLinkGetLinkedCubin == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetLinkedCubin is not found") + return (__nvJitLinkGetLinkedCubin)( + handle, cubin) + + +cdef nvJitLinkResult _nvJitLinkGetLinkedPtxSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetLinkedPtxSize + _check_or_init_nvjitlink() + if __nvJitLinkGetLinkedPtxSize == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetLinkedPtxSize is not found") + return (__nvJitLinkGetLinkedPtxSize)( + handle, size) + + +cdef nvJitLinkResult _nvJitLinkGetLinkedPtx(nvJitLinkHandle handle, char* ptx) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetLinkedPtx + _check_or_init_nvjitlink() + if __nvJitLinkGetLinkedPtx == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetLinkedPtx is not found") + return (__nvJitLinkGetLinkedPtx)( + handle, ptx) + + +cdef nvJitLinkResult _nvJitLinkGetErrorLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetErrorLogSize + _check_or_init_nvjitlink() + if __nvJitLinkGetErrorLogSize == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetErrorLogSize is not found") + return (__nvJitLinkGetErrorLogSize)( + handle, size) + + +cdef nvJitLinkResult _nvJitLinkGetErrorLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetErrorLog + _check_or_init_nvjitlink() + if __nvJitLinkGetErrorLog == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetErrorLog is not found") + return (__nvJitLinkGetErrorLog)( + handle, log) + + +cdef nvJitLinkResult _nvJitLinkGetInfoLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetInfoLogSize + _check_or_init_nvjitlink() + if __nvJitLinkGetInfoLogSize == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetInfoLogSize is not found") + return (__nvJitLinkGetInfoLogSize)( + handle, size) + + +cdef nvJitLinkResult _nvJitLinkGetInfoLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkGetInfoLog + _check_or_init_nvjitlink() + if __nvJitLinkGetInfoLog == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkGetInfoLog is not found") + return (__nvJitLinkGetInfoLog)( + handle, log) + + +cdef nvJitLinkResult _nvJitLinkVersion(unsigned int* major, unsigned int* minor) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvJitLinkVersion + _check_or_init_nvjitlink() + if __nvJitLinkVersion == NULL: + with gil: + raise FunctionNotFoundError("function nvJitLinkVersion is not found") + return (__nvJitLinkVersion)( + major, minor) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm.cpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b64529a31cc80ca5b76602264d7bb08202b30b0f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm.cpp @@ -0,0 +1,13295 @@ +/* Generated by Cython 3.1.5 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [], + "extra_compile_args": [ + "-std=c++14", + "-fpermissive", + "-Wno-deprecated-declarations", + "-fno-var-tracking-assignments", + "-O3" + ], + "include_dirs": [ + "/opt/python/cp312-cp312/include", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include" + ], + "language": "c++", + "library_dirs": [ + "/tmp/build-env-3t3_8d9r/lib/python3.12/site-packages", + "/tmp/build-env-3t3_8d9r/lib", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib64", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib" + ], + "name": "cuda.bindings._internal.nvvm", + "sources": [ + "cuda/bindings/_internal/nvvm.pyx" + ] + }, + "module_name": "cuda.bindings._internal.nvvm" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_1_5" +#define CYTHON_HEX_VERSION 0x030105F0 +#define CYTHON_FUTURE_DIVISION 1 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef HAVE_LONG_LONG + #define HAVE_LONG_LONG +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_CLINE_IN_TRACEBACK + #define CYTHON_CLINE_IN_TRACEBACK 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1) +#endif +#endif +#define CYTHON_BACKPORT_VECTORCALL (CYTHON_METH_FASTCALL && PY_VERSION_HEX < 0x030800B1) +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME + #define CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME 100 +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + #endif + #endif + #if _MSC_VER < 1300 + #ifdef _WIN64 + typedef unsigned long long __pyx_uintptr_t; + #else + typedef unsigned int __pyx_uintptr_t; + #endif + #else + #ifdef _WIN64 + typedef unsigned __int64 __pyx_uintptr_t; + #else + typedef unsigned __int32 __pyx_uintptr_t; + #endif + #endif +#else + #include + typedef uintptr_t __pyx_uintptr_t; +#endif +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#elif CYTHON_BACKPORT_VECTORCALL + typedef PyObject *(*__pyx_vectorcallfunc)(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames); + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1)) + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(((size_t)(n)) & ~__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if __PYX_LIMITED_VERSION_HEX < 0x03090000 + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS && PY_VERSION_HEX >= 0x03080000 +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static CYTHON_INLINE PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__cuda__bindings___internal__nvvm +#define __PYX_HAVE_API__cuda__bindings___internal__nvvm +/* Early includes */ +#include +#include +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value); + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "cuda/bindings/_internal/nvvm.pyx", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* ForceInitThreads.proto */ +#ifndef __PYX_FORCE_INIT_THREADS + #define __PYX_FORCE_INIT_THREADS 0 +#endif + +/* NoFastGil.proto */ +#define __Pyx_PyGILState_Ensure PyGILState_Ensure +#define __Pyx_PyGILState_Release PyGILState_Release +#define __Pyx_FastGIL_Remember() +#define __Pyx_FastGIL_Forget() +#define __Pyx_FastGilFuncInit() + +/* Atomics.proto */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif +#if CYTHON_ATOMICS + #define __pyx_add_acquisition_count(memview)\ + __pyx_atomic_incr_relaxed(__pyx_get_slice_count_pointer(memview)) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_atomic_decr_acq_rel(__pyx_get_slice_count_pointer(memview)) +#else + #define __pyx_add_acquisition_count(memview)\ + __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_sub_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) +#endif + +/* IncludeStructmemberH.proto */ +#include + +/* CriticalSections.proto */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_Begin1(cs, arg) (void)cs +#define __Pyx_PyCriticalSection_Begin2(cs, arg1, arg2) (void)cs +#define __Pyx_PyCriticalSection_End1(cs) +#define __Pyx_PyCriticalSection_End2(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_Begin1 PyCriticalSection_Begin +#define __Pyx_PyCriticalSection_Begin2 PyCriticalSection2_Begin +#define __Pyx_PyCriticalSection_End1 PyCriticalSection_End +#define __Pyx_PyCriticalSection_End2 PyCriticalSection2_End +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* #### Code section: numeric_typedefs ### */ +/* #### Code section: complex_type_declarations ### */ +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ + +/* "cynvvm.pxd":13 + * + * # enums + * ctypedef enum nvvmResult "nvvmResult": # <<<<<<<<<<<<<< + * NVVM_SUCCESS "NVVM_SUCCESS" = 0 + * NVVM_ERROR_OUT_OF_MEMORY "NVVM_ERROR_OUT_OF_MEMORY" = 1 +*/ +enum nvvmResult { + NVVM_SUCCESS = 0, + NVVM_ERROR_OUT_OF_MEMORY = 1, + NVVM_ERROR_PROGRAM_CREATION_FAILURE = 2, + NVVM_ERROR_IR_VERSION_MISMATCH = 3, + NVVM_ERROR_INVALID_INPUT = 4, + NVVM_ERROR_INVALID_PROGRAM = 5, + NVVM_ERROR_INVALID_IR = 6, + NVVM_ERROR_INVALID_OPTION = 7, + NVVM_ERROR_NO_MODULE_IN_PROGRAM = 8, + NVVM_ERROR_COMPILATION = 9, + NVVM_ERROR_CANCELLED = 10, + _NVVMRESULT_INTERNAL_LOADING_ERROR = -42L +}; +typedef enum nvvmResult nvvmResult; + +/* "cynvvm.pxd":29 + * + * # types + * ctypedef void* nvvmProgram 'nvvmProgram' # <<<<<<<<<<<<<< + * + * +*/ +typedef void *nvvmProgram; +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyErrExceptionMatches.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* PyObjectGetAttrStrNoError.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* PyDictVersioning.proto */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __pyx_dict_cached_value;\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* GetModuleGlobalName.proto */ +#if CYTHON_USE_DICT_VERSIONS +#define __Pyx_GetModuleGlobalName(var, name) do {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + (var) = (likely(__pyx_dict_version == __PYX_GET_DICT_VERSION(__pyx_mstate_global->__pyx_d))) ?\ + (likely(__pyx_dict_cached_value) ? __Pyx_NewRef(__pyx_dict_cached_value) : __Pyx_GetBuiltinName(name)) :\ + __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +#define __Pyx_GetModuleGlobalNameUncached(var, name) do {\ + PY_UINT64_T __pyx_dict_version;\ + PyObject *__pyx_dict_cached_value;\ + (var) = __Pyx__GetModuleGlobalName(name, &__pyx_dict_version, &__pyx_dict_cached_value);\ +} while(0) +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value); +#else +#define __Pyx_GetModuleGlobalName(var, name) (var) = __Pyx__GetModuleGlobalName(name) +#define __Pyx_GetModuleGlobalNameUncached(var, name) (var) = __Pyx__GetModuleGlobalName(name) +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name); +#endif + +/* IncludeStringH.proto */ +#include + +/* decode_c_string_utf16.proto */ +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, const char *errors) { + int byteorder = 0; + return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); +} +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16LE(const char *s, Py_ssize_t size, const char *errors) { + int byteorder = -1; + return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); +} +static CYTHON_INLINE PyObject *__Pyx_PyUnicode_DecodeUTF16BE(const char *s, Py_ssize_t size, const char *errors) { + int byteorder = 1; + return PyUnicode_DecodeUTF16(s, size, errors, &byteorder); +} + +/* decode_c_string.proto */ +static CYTHON_INLINE PyObject* __Pyx_decode_c_string( + const char* cstring, Py_ssize_t start, Py_ssize_t stop, + const char* encoding, const char* errors, + PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)); + +/* JoinPyUnicode.proto */ +static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength, + Py_UCS4 max_char); + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#if !CYTHON_VECTORCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs); +#endif +#define __Pyx_BUILD_ASSERT_EXPR(cond)\ + (sizeof(char [1 - 2*!(cond)]) - 1) +#ifndef Py_MEMBER_SIZE +#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#endif +#if !CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03080000 + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets() + #define __Pyx_PyFrame_GetLocalsplus(frame) ((frame)->f_localsplus) +#else + static size_t __pyx_pyframe_localsplus_offset = 0; + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets()\ + ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ + (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) + #define __Pyx_PyFrame_GetLocalsplus(frame)\ + (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) +#endif +#endif +#endif + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* RaiseException.proto */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* BuildPyUnicode.proto */ +static PyObject* __Pyx_PyUnicode_BuildFromAscii(Py_ssize_t ulength, const char* chars, int clength, + int prepend_sign, char padding_char); + +/* COrdinalToPyUnicode.proto */ +static CYTHON_INLINE int __Pyx_CheckUnicodeValue(int value); +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromOrdinal_Padded(int value, Py_ssize_t width, char padding_char); + +/* GCCDiagnostics.proto */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* IncludeStdlibH.proto */ +#include + +/* CIntToPyUnicode.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char, char format_char); + +/* PyObjectLookupSpecial.proto */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 0) +#define __Pyx_PyObject_LookupSpecial(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 1) +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error); +#else +#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n) +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/* GetTopmostException.proto */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* TupleAndListFromArray.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n); +#endif +#if CYTHON_COMPILING_IN_CPYTHON || CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject* __Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n); +#endif + +/* BytesEquals.proto */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); + +/* UnicodeEquals.proto */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); + +/* fastcall.proto */ +#if CYTHON_AVOID_BORROWED_REFS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_PySequence_ITEM(args, i) +#elif CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_NewRef(__Pyx_PyTuple_GET_ITEM(args, i)) +#else + #define __Pyx_ArgRef_VARARGS(args, i) __Pyx_XNewRef(PyTuple_GetItem(args, i)) +#endif +#define __Pyx_NumKwargs_VARARGS(kwds) PyDict_Size(kwds) +#define __Pyx_KwValues_VARARGS(args, nargs) NULL +#define __Pyx_GetKwValue_VARARGS(kw, kwvalues, s) __Pyx_PyDict_GetItemStrWithError(kw, s) +#define __Pyx_KwargsAsDict_VARARGS(kw, kwvalues) PyDict_Copy(kw) +#if CYTHON_METH_FASTCALL + #define __Pyx_ArgRef_FASTCALL(args, i) __Pyx_NewRef(args[i]) + #define __Pyx_NumKwargs_FASTCALL(kwds) __Pyx_PyTuple_GET_SIZE(kwds) + #define __Pyx_KwValues_FASTCALL(args, nargs) ((args) + (nargs)) + static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API + CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues); + #else + #define __Pyx_KwargsAsDict_FASTCALL(kw, kwvalues) _PyStack_AsDict(kwvalues, kw) + #endif +#else + #define __Pyx_ArgRef_FASTCALL __Pyx_ArgRef_VARARGS + #define __Pyx_NumKwargs_FASTCALL __Pyx_NumKwargs_VARARGS + #define __Pyx_KwValues_FASTCALL __Pyx_KwValues_VARARGS + #define __Pyx_GetKwValue_FASTCALL __Pyx_GetKwValue_VARARGS + #define __Pyx_KwargsAsDict_FASTCALL __Pyx_KwargsAsDict_VARARGS +#endif +#define __Pyx_ArgsSlice_VARARGS(args, start, stop) PyTuple_GetSlice(args, start, stop) +#if CYTHON_METH_FASTCALL || (CYTHON_COMPILING_IN_CPYTHON && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) __Pyx_PyTuple_FromArray(args + start, stop - start) +#else +#define __Pyx_ArgsSlice_FASTCALL(args, start, stop) PyTuple_GetSlice(args, start, stop) +#endif + +/* DictGetItem.proto */ +#if !CYTHON_COMPILING_IN_PYPY +static PyObject *__Pyx_PyDict_GetItem(PyObject *d, PyObject* key); +#define __Pyx_PyObject_Dict_GetItem(obj, name)\ + (likely(PyDict_CheckExact(obj)) ?\ + __Pyx_PyDict_GetItem(obj, name) : PyObject_GetItem(obj, name)) +#else +#define __Pyx_PyDict_GetItem(d, key) PyObject_GetItem(d, key) +#define __Pyx_PyObject_Dict_GetItem(obj, name) PyObject_GetItem(obj, name) +#endif + +/* RaiseDoubleKeywords.proto */ +static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); + +/* ParseKeywords.proto */ +static CYTHON_INLINE int __Pyx_ParseKeywords( + PyObject *kwds, PyObject *const *kwvalues, PyObject ** const argnames[], + PyObject *kwds2, PyObject *values[], + Py_ssize_t num_pos_args, Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs +); + +/* CallCFunction.proto */ +#define __Pyx_CallCFunction(cfunc, self, args)\ + ((PyCFunction)(void(*)(void))(cfunc)->func)(self, args) +#define __Pyx_CallCFunctionWithKeywords(cfunc, self, args, kwargs)\ + ((PyCFunctionWithKeywords)(void(*)(void))(cfunc)->func)(self, args, kwargs) +#define __Pyx_CallCFunctionFast(cfunc, self, args, nargs)\ + ((__Pyx_PyCFunctionFast)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs) +#define __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, nargs, kwnames)\ + ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))(PyCFunction)(cfunc)->func)(self, args, nargs, kwnames) + +/* UnpackUnboundCMethod.proto */ +typedef struct { + PyObject *type; + PyObject **method_name; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && CYTHON_ATOMICS + __pyx_atomic_int_type initialized; +#endif + PyCFunction func; + PyObject *method; + int flag; +} __Pyx_CachedCFunction; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +static CYTHON_INLINE int __Pyx_CachedCFunction_GetAndSetInitializing(__Pyx_CachedCFunction *cfunc) { +#if !CYTHON_ATOMICS + return 1; +#else + __pyx_nonatomic_int_type expected = 0; + if (__pyx_atomic_int_cmp_exchange(&cfunc->initialized, &expected, 1)) { + return 0; + } + return expected; +#endif +} +static CYTHON_INLINE void __Pyx_CachedCFunction_SetFinishedInitializing(__Pyx_CachedCFunction *cfunc) { +#if CYTHON_ATOMICS + __pyx_atomic_store(&cfunc->initialized, 2); +#endif +} +#else +#define __Pyx_CachedCFunction_GetAndSetInitializing(cfunc) 2 +#define __Pyx_CachedCFunction_SetFinishedInitializing(cfunc) +#endif + +/* CallUnboundCMethod2.proto */ +CYTHON_UNUSED +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2); +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2); +#else +#define __Pyx_CallUnboundCMethod2(cfunc, self, arg1, arg2) __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2) +#endif + +/* RaiseArgTupleInvalid.proto */ +static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, + Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); + +/* ArgTypeTest.proto */ +#define __Pyx_ArgTypeTest(obj, type, none_allowed, name, exact)\ + ((likely(__Pyx_IS_TYPE(obj, type) | (none_allowed && (obj == Py_None)))) ? 1 :\ + __Pyx__ArgTypeTest(obj, type, name, exact)) +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact); + +/* Import.proto */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); + +/* ImportDottedModule.proto */ +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple); +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple); + +/* ListPack.proto */ +static PyObject *__Pyx_PyList_Pack(Py_ssize_t n, ...); + +/* ImportFrom.proto */ +static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); + +/* LimitedApiGetTypeDict.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp); +#endif + +/* SetItemOnTypeDict.proto */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v); +#define __Pyx_SetItemOnTypeDict(tp, k, v) __Pyx__SetItemOnTypeDict((PyTypeObject*)tp, k, v) + +/* FixUpExtensionType.proto */ +static CYTHON_INLINE int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type); + +/* FetchSharedCythonModule.proto */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void); + +/* dict_setdefault.proto */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, int is_safe_type); + +/* FetchCommonType.proto */ +static PyTypeObject* __Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases); + +/* CommonTypesMetaclass.proto */ +static int __pyx_CommonTypesMetaclass_init(PyObject *module); +#define __Pyx_CommonTypesMetaclass_USED + +/* CallTypeTraverse.proto */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#define __Pyx_call_type_traverse(o, always_call, visit, arg) 0 +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg); +#endif + +/* PyMethodNew.proto */ +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ); + +/* PyVectorcallFastCallDict.proto */ +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw); +#endif + +/* CythonFunctionShared.proto */ +#define __Pyx_CyFunction_USED +#define __Pyx_CYFUNCTION_STATICMETHOD 0x01 +#define __Pyx_CYFUNCTION_CLASSMETHOD 0x02 +#define __Pyx_CYFUNCTION_CCLASS 0x04 +#define __Pyx_CYFUNCTION_COROUTINE 0x08 +#define __Pyx_CyFunction_GetClosure(f)\ + (((__pyx_CyFunctionObject *) (f))->func_closure) +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_CyFunction_GetClassObj(f)\ + (((__pyx_CyFunctionObject *) (f))->func_classobj) +#else + #define __Pyx_CyFunction_GetClassObj(f)\ + ((PyObject*) ((PyCMethodObject *) (f))->mm_class) +#endif +#define __Pyx_CyFunction_SetClassObj(f, classobj)\ + __Pyx__CyFunction_SetClassObj((__pyx_CyFunctionObject *) (f), (classobj)) +#define __Pyx_CyFunction_Defaults(type, f)\ + ((type *)(((__pyx_CyFunctionObject *) (f))->defaults)) +#define __Pyx_CyFunction_SetDefaultsGetter(f, g)\ + ((__pyx_CyFunctionObject *) (f))->defaults_getter = (g) +typedef struct { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject_HEAD + PyObject *func; +#elif PY_VERSION_HEX < 0x030900B1 + PyCFunctionObject func; +#else + PyCMethodObject func; +#endif +#if CYTHON_BACKPORT_VECTORCALL ||\ + (CYTHON_COMPILING_IN_LIMITED_API && CYTHON_METH_FASTCALL) + __pyx_vectorcallfunc func_vectorcall; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_weakreflist; +#endif + PyObject *func_dict; + PyObject *func_name; + PyObject *func_qualname; + PyObject *func_doc; + PyObject *func_globals; + PyObject *func_code; + PyObject *func_closure; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + PyObject *func_classobj; +#endif + PyObject *defaults; + int flags; + PyObject *defaults_tuple; + PyObject *defaults_kwdict; + PyObject *(*defaults_getter)(PyObject *); + PyObject *func_annotations; + PyObject *func_is_coroutine; +} __pyx_CyFunctionObject; +#undef __Pyx_CyOrPyCFunction_Check +#define __Pyx_CyFunction_Check(obj) __Pyx_TypeCheck(obj, __pyx_mstate_global->__pyx_CyFunctionType) +#define __Pyx_CyOrPyCFunction_Check(obj) __Pyx_TypeCheck2(obj, __pyx_mstate_global->__pyx_CyFunctionType, &PyCFunction_Type) +#define __Pyx_CyFunction_CheckExact(obj) __Pyx_IS_TYPE(obj, __pyx_mstate_global->__pyx_CyFunctionType) +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)); +#undef __Pyx_IsSameCFunction +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCyOrCFunction(func, cfunc) +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject* op, PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj); +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, + PyTypeObject *defaults_type); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *m, + PyObject *tuple); +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *m, + PyObject *dict); +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *m, + PyObject *dict); +static int __pyx_CyFunction_init(PyObject *module); +#if CYTHON_METH_FASTCALL +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames); +#if CYTHON_BACKPORT_VECTORCALL || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_func_vectorcall(f) (((__pyx_CyFunctionObject*)f)->func_vectorcall) +#else +#define __Pyx_CyFunction_func_vectorcall(f) (((PyCFunctionObject*)f)->vectorcall) +#endif +#endif + +/* CythonFunction.proto */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, + int flags, PyObject* qualname, + PyObject *closure, + PyObject *module, PyObject *globals, + PyObject* code); + +/* CLineInTraceback.proto */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* CIntFromPy.proto */ +static CYTHON_INLINE size_t __Pyx_PyLong_As_size_t(PyObject *); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* PyObjectVectorCallKwBuilder.proto */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +/* GetRuntimeVersion.proto */ +static unsigned long __Pyx_get_runtime_version(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* FunctionExport.proto */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + + +/* Module declarations from "cuda.bindings.cynvvm" */ + +/* Module declarations from "libc.stdint" */ + +/* Module declarations from "cuda.bindings._internal.nvvm" */ +static PyObject *__pyx_v_4cuda_8bindings_9_internal_4nvvm___symbol_lock = 0; +static int __pyx_v_4cuda_8bindings_9_internal_4nvvm___py_nvvm_init; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___cuDriverGetVersion; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetErrorString; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVersion; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmIRVersion; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCreateProgram; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmDestroyProgram; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmAddModuleToProgram; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmLazyAddModuleToProgram; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCompileProgram; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVerifyProgram; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResultSize; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResult; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLogSize; +static void *__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLog; +static PyObject *__pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs = 0; +static void *__pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(void); /*proto*/ +static int __pyx_f_4cuda_8bindings_9_internal_4nvvm___check_or_init_nvvm(void); /*proto*/ +static int __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(void); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_internal_4nvvm__inspect_function_pointers(int __pyx_skip_dispatch); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_internal_4nvvm__inspect_function_pointer(PyObject *, int __pyx_skip_dispatch); /*proto*/ +/* #### Code section: typeinfo ### */ +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "cuda.bindings._internal.nvvm" +extern int __pyx_module_is_main_cuda__bindings___internal__nvvm; +int __pyx_module_is_main_cuda__bindings___internal__nvvm = 0; + +/* Implementation of "cuda.bindings._internal.nvvm" */ +/* #### Code section: global_var ### */ +static PyObject *__pyx_builtin_RuntimeError; +/* #### Code section: string_decls ### */ +static const char __pyx_k_[] = ")"; +static const char __pyx_k__2[] = "."; +static const char __pyx_k__3[] = "?"; +static const char __pyx_k_pop[] = "pop"; +static const char __pyx_k_Lock[] = "Lock"; +static const char __pyx_k_exit[] = "__exit__"; +static const char __pyx_k_func[] = "__func__"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_name[] = "name"; +static const char __pyx_k_nvvm[] = "nvvm"; +static const char __pyx_k_spec[] = "__spec__"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_enter[] = "__enter__"; +static const char __pyx_k_utils[] = "utils"; +static const char __pyx_k_module[] = "__module__"; +static const char __pyx_k_name_2[] = "__name__"; +static const char __pyx_k_add_note[] = "add_note"; +static const char __pyx_k_qualname[] = "__qualname__"; +static const char __pyx_k_set_name[] = "__set_name__"; +static const char __pyx_k_threading[] = "threading"; +static const char __pyx_k_z_A_a_9AQ[] = "\200\001\330\004\005\330\004\007\200z\220\023\220A\330\010\024\320\024.\250a\330\004\013\2109\220A\220Q"; +static const char __pyx_k_handle_uint[] = "_handle_uint"; +static const char __pyx_k_nvvmVersion[] = "__nvvmVersion"; +static const char __pyx_k_RuntimeError[] = "RuntimeError"; +static const char __pyx_k_initializing[] = "_initializing"; +static const char __pyx_k_is_coroutine[] = "_is_coroutine"; +static const char __pyx_k_nvvmIRVersion[] = "__nvvmIRVersion"; +static const char __pyx_k_cuda_pathfinder[] = "cuda.pathfinder"; +static const char __pyx_k_NotSupportedError[] = "NotSupportedError"; +static const char __pyx_k_nvvmCreateProgram[] = "__nvvmCreateProgram"; +static const char __pyx_k_nvvmGetProgramLog[] = "__nvvmGetProgramLog"; +static const char __pyx_k_nvvmVerifyProgram[] = "__nvvmVerifyProgram"; +static const char __pyx_k_asyncio_coroutines[] = "asyncio.coroutines"; +static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; +static const char __pyx_k_nvvmCompileProgram[] = "__nvvmCompileProgram"; +static const char __pyx_k_nvvmDestroyProgram[] = "__nvvmDestroyProgram"; +static const char __pyx_k_nvvmGetErrorString[] = "__nvvmGetErrorString"; +static const char __pyx_k_FunctionNotFoundError[] = "FunctionNotFoundError"; +static const char __pyx_k_nvvmGetCompiledResult[] = "__nvvmGetCompiledResult"; +static const char __pyx_k_nvvmGetProgramLogSize[] = "__nvvmGetProgramLogSize"; +static const char __pyx_k_nvvmAddModuleToProgram[] = "__nvvmAddModuleToProgram"; +static const char __pyx_k_load_nvidia_dynamic_lib[] = "load_nvidia_dynamic_lib"; +static const char __pyx_k_CUDA_driver_is_not_found[] = "CUDA driver is not found ("; +static const char __pyx_k_inspect_function_pointer[] = "_inspect_function_pointer"; +static const char __pyx_k_inspect_function_pointers[] = "_inspect_function_pointers"; +static const char __pyx_k_nvvmGetCompiledResultSize[] = "__nvvmGetCompiledResultSize"; +static const char __pyx_k_nvvmLazyAddModuleToProgram[] = "__nvvmLazyAddModuleToProgram"; +static const char __pyx_k_cuda_bindings__internal_nvvm[] = "cuda.bindings._internal.nvvm"; +static const char __pyx_k_z_q_q_Q_Q_Ja_j_A_Q_z_Q_Q_A_A_j[] = "\200\001\330\004\005\330\004\007\200z\220\027\230\001\330\010\017\210q\340\004\027\220q\330\004\025\220Q\340\004\005\330\004\010\210\001\320\t#\240:\250Q\340\004\005\330\004\010\210\001\320\t\034\230J\240a\340\004\005\330\004\010\210\001\320\t\036\230j\250\001\340\004\005\330\004\010\210\001\320\t\"\240*\250A\340\004\005\330\004\010\210\001\320\t#\240:\250Q\340\004\005\330\004\010\210\001\320\t'\240z\260\021\340\004\005\330\004\010\210\001\320\t+\250:\260Q\340\004\005\330\004\010\210\001\320\t#\240:\250Q\340\004\005\330\004\010\210\001\320\t\"\240*\250A\340\004\005\330\004\010\210\001\320\t*\250*\260A\340\004\005\330\004\010\210\001\320\t&\240j\260\001\340\004\005\330\004\010\210\001\320\t&\240j\260\001\340\004\005\330\004\010\210\001\320\t\"\240*\250A\340\004\020\220\001\330\004\013\2101"; +static const char __pyx_k_Did_not_find_cuDriverGetVersion[] = "Did not find cuDriverGetVersion symbol in libcuda.so.1"; +static const char __pyx_k_function_nvvmAddModuleToProgram[] = "function nvvmAddModuleToProgram is not found"; +static const char __pyx_k_Note_that_Cython_is_deliberately[] = "Note that Cython is deliberately stricter than PEP-484 and rejects subclasses of builtin types. If you need to pass subclasses then set the 'annotation_typing' directive to False."; +static const char __pyx_k_cuDriverGetVersion_returned_erro[] = "cuDriverGetVersion returned error code "; +static const char __pyx_k_cuda_bindings__internal_nvvm_pyx[] = "cuda/bindings/_internal/nvvm.pyx"; +static const char __pyx_k_function_nvvmCompileProgram_is_n[] = "function nvvmCompileProgram is not found"; +static const char __pyx_k_function_nvvmCreateProgram_is_no[] = "function nvvmCreateProgram is not found"; +static const char __pyx_k_function_nvvmDestroyProgram_is_n[] = "function nvvmDestroyProgram is not found"; +static const char __pyx_k_function_nvvmGetCompiledResultSi[] = "function nvvmGetCompiledResultSize is not found"; +static const char __pyx_k_function_nvvmGetCompiledResult_i[] = "function nvvmGetCompiledResult is not found"; +static const char __pyx_k_function_nvvmGetErrorString_is_n[] = "function nvvmGetErrorString is not found"; +static const char __pyx_k_function_nvvmGetProgramLogSize_i[] = "function nvvmGetProgramLogSize is not found"; +static const char __pyx_k_function_nvvmGetProgramLog_is_no[] = "function nvvmGetProgramLog is not found"; +static const char __pyx_k_function_nvvmIRVersion_is_not_fo[] = "function nvvmIRVersion is not found"; +static const char __pyx_k_function_nvvmLazyAddModuleToProg[] = "function nvvmLazyAddModuleToProgram is not found"; +static const char __pyx_k_function_nvvmVerifyProgram_is_no[] = "function nvvmVerifyProgram is not found"; +static const char __pyx_k_function_nvvmVersion_is_not_foun[] = "function nvvmVersion is not found"; +/* #### Code section: decls ### */ +static PyObject *__pyx_pf_4cuda_8bindings_9_internal_4nvvm__inspect_function_pointers(CYTHON_UNUSED PyObject *__pyx_self); /* proto */ +static PyObject *__pyx_pf_4cuda_8bindings_9_internal_4nvvm_2_inspect_function_pointer(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_name); /* proto */ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + #ifdef __Pyx_CyFunction_USED + PyTypeObject *__pyx_CyFunctionType; + #endif + #ifdef __Pyx_FusedFunction_USED + PyTypeObject *__pyx_FusedFunctionType; + #endif + #ifdef __Pyx_Generator_USED + PyTypeObject *__pyx_GeneratorType; + #endif + #ifdef __Pyx_IterableCoroutine_USED + PyTypeObject *__pyx_IterableCoroutineType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineAwaitType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineType; + #endif + __Pyx_CachedCFunction __pyx_umethod_PyDict_Type_pop; + PyObject *__pyx_tuple[1]; + PyObject *__pyx_codeobj_tab[2]; + PyObject *__pyx_string_tab[64]; +/* #### Code section: module_state_contents ### */ +/* CommonTypesMetaclass.module_state_decls */ +PyTypeObject *__pyx_CommonTypesMetaclassType; + +/* CachedMethodType.module_state_decls */ +#if CYTHON_COMPILING_IN_LIMITED_API +PyObject *__Pyx_CachedMethodType; +#endif + +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_kp_u_CUDA_driver_is_not_found __pyx_string_tab[1] +#define __pyx_kp_u_Did_not_find_cuDriverGetVersion __pyx_string_tab[2] +#define __pyx_n_u_FunctionNotFoundError __pyx_string_tab[3] +#define __pyx_n_u_Lock __pyx_string_tab[4] +#define __pyx_n_u_NotSupportedError __pyx_string_tab[5] +#define __pyx_kp_u_Note_that_Cython_is_deliberately __pyx_string_tab[6] +#define __pyx_n_u_RuntimeError __pyx_string_tab[7] +#define __pyx_kp_u__2 __pyx_string_tab[8] +#define __pyx_kp_u__3 __pyx_string_tab[9] +#define __pyx_kp_u_add_note __pyx_string_tab[10] +#define __pyx_n_u_asyncio_coroutines __pyx_string_tab[11] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[12] +#define __pyx_kp_u_cuDriverGetVersion_returned_erro __pyx_string_tab[13] +#define __pyx_n_u_cuda_bindings__internal_nvvm __pyx_string_tab[14] +#define __pyx_kp_u_cuda_bindings__internal_nvvm_pyx __pyx_string_tab[15] +#define __pyx_n_u_cuda_pathfinder __pyx_string_tab[16] +#define __pyx_n_u_enter __pyx_string_tab[17] +#define __pyx_n_u_exit __pyx_string_tab[18] +#define __pyx_n_u_func __pyx_string_tab[19] +#define __pyx_kp_u_function_nvvmAddModuleToProgram __pyx_string_tab[20] +#define __pyx_kp_u_function_nvvmCompileProgram_is_n __pyx_string_tab[21] +#define __pyx_kp_u_function_nvvmCreateProgram_is_no __pyx_string_tab[22] +#define __pyx_kp_u_function_nvvmDestroyProgram_is_n __pyx_string_tab[23] +#define __pyx_kp_u_function_nvvmGetCompiledResultSi __pyx_string_tab[24] +#define __pyx_kp_u_function_nvvmGetCompiledResult_i __pyx_string_tab[25] +#define __pyx_kp_u_function_nvvmGetErrorString_is_n __pyx_string_tab[26] +#define __pyx_kp_u_function_nvvmGetProgramLogSize_i __pyx_string_tab[27] +#define __pyx_kp_u_function_nvvmGetProgramLog_is_no __pyx_string_tab[28] +#define __pyx_kp_u_function_nvvmIRVersion_is_not_fo __pyx_string_tab[29] +#define __pyx_kp_u_function_nvvmLazyAddModuleToProg __pyx_string_tab[30] +#define __pyx_kp_u_function_nvvmVerifyProgram_is_no __pyx_string_tab[31] +#define __pyx_kp_u_function_nvvmVersion_is_not_foun __pyx_string_tab[32] +#define __pyx_n_u_handle_uint __pyx_string_tab[33] +#define __pyx_n_u_initializing __pyx_string_tab[34] +#define __pyx_n_u_inspect_function_pointer __pyx_string_tab[35] +#define __pyx_n_u_inspect_function_pointers __pyx_string_tab[36] +#define __pyx_n_u_is_coroutine __pyx_string_tab[37] +#define __pyx_n_u_load_nvidia_dynamic_lib __pyx_string_tab[38] +#define __pyx_n_u_main __pyx_string_tab[39] +#define __pyx_n_u_module __pyx_string_tab[40] +#define __pyx_n_u_name __pyx_string_tab[41] +#define __pyx_n_u_name_2 __pyx_string_tab[42] +#define __pyx_n_u_nvvm __pyx_string_tab[43] +#define __pyx_n_u_nvvmAddModuleToProgram __pyx_string_tab[44] +#define __pyx_n_u_nvvmCompileProgram __pyx_string_tab[45] +#define __pyx_n_u_nvvmCreateProgram __pyx_string_tab[46] +#define __pyx_n_u_nvvmDestroyProgram __pyx_string_tab[47] +#define __pyx_n_u_nvvmGetCompiledResult __pyx_string_tab[48] +#define __pyx_n_u_nvvmGetCompiledResultSize __pyx_string_tab[49] +#define __pyx_n_u_nvvmGetErrorString __pyx_string_tab[50] +#define __pyx_n_u_nvvmGetProgramLog __pyx_string_tab[51] +#define __pyx_n_u_nvvmGetProgramLogSize __pyx_string_tab[52] +#define __pyx_n_u_nvvmIRVersion __pyx_string_tab[53] +#define __pyx_n_u_nvvmLazyAddModuleToProgram __pyx_string_tab[54] +#define __pyx_n_u_nvvmVerifyProgram __pyx_string_tab[55] +#define __pyx_n_u_nvvmVersion __pyx_string_tab[56] +#define __pyx_n_u_pop __pyx_string_tab[57] +#define __pyx_n_u_qualname __pyx_string_tab[58] +#define __pyx_n_u_set_name __pyx_string_tab[59] +#define __pyx_n_u_spec __pyx_string_tab[60] +#define __pyx_n_u_test __pyx_string_tab[61] +#define __pyx_n_u_threading __pyx_string_tab[62] +#define __pyx_n_u_utils __pyx_string_tab[63] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_CLEAR(clear_module_state->__pyx_FusedFunctionType); + #endif + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + for (int i=0; i<1; ++i) { Py_CLEAR(clear_module_state->__pyx_tuple[i]); } + for (int i=0; i<2; ++i) { Py_CLEAR(clear_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<64; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_VISIT(traverse_module_state->__pyx_FusedFunctionType); + #endif + for (int i=0; i<1; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_tuple[i]); } + for (int i=0; i<2; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_codeobj_tab[i]); } + for (int i=0; i<64; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "cuda/bindings/_internal/nvvm.pyx":33 + * const void* RTLD_DEFAULT 'RTLD_DEFAULT' + * + * cdef int get_cuda_version(): # <<<<<<<<<<<<<< + * cdef void* handle = NULL + * cdef int err, driver_ver = 0 +*/ + +static int __pyx_f_4cuda_8bindings_9_internal_4nvvm_get_cuda_version(void) { + void *__pyx_v_handle; + int __pyx_v_err; + int __pyx_v_driver_ver; + char *__pyx_v_err_msg; + void *__pyx_v_cuDriverGetVersion; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7[3]; + PyObject *__pyx_t_8 = NULL; + size_t __pyx_t_9; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_cuda_version", 0); + + /* "cuda/bindings/_internal/nvvm.pyx":34 + * + * cdef int get_cuda_version(): + * cdef void* handle = NULL # <<<<<<<<<<<<<< + * cdef int err, driver_ver = 0 + * +*/ + __pyx_v_handle = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":35 + * cdef int get_cuda_version(): + * cdef void* handle = NULL + * cdef int err, driver_ver = 0 # <<<<<<<<<<<<<< + * + * # Load driver to check version +*/ + __pyx_v_driver_ver = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":38 + * + * # Load driver to check version + * handle = dlopen('libcuda.so.1', RTLD_NOW | RTLD_GLOBAL) # <<<<<<<<<<<<<< + * if handle == NULL: + * err_msg = dlerror() +*/ + __pyx_v_handle = dlopen(((char const *)"libcuda.so.1"), (RTLD_NOW | RTLD_GLOBAL)); + + /* "cuda/bindings/_internal/nvvm.pyx":39 + * # Load driver to check version + * handle = dlopen('libcuda.so.1', RTLD_NOW | RTLD_GLOBAL) + * if handle == NULL: # <<<<<<<<<<<<<< + * err_msg = dlerror() + * raise NotSupportedError(f'CUDA driver is not found ({err_msg.decode()})') +*/ + __pyx_t_1 = (__pyx_v_handle == NULL); + if (unlikely(__pyx_t_1)) { + + /* "cuda/bindings/_internal/nvvm.pyx":40 + * handle = dlopen('libcuda.so.1', RTLD_NOW | RTLD_GLOBAL) + * if handle == NULL: + * err_msg = dlerror() # <<<<<<<<<<<<<< + * raise NotSupportedError(f'CUDA driver is not found ({err_msg.decode()})') + * cuDriverGetVersion = dlsym(handle, "cuDriverGetVersion") +*/ + __pyx_v_err_msg = dlerror(); + + /* "cuda/bindings/_internal/nvvm.pyx":41 + * if handle == NULL: + * err_msg = dlerror() + * raise NotSupportedError(f'CUDA driver is not found ({err_msg.decode()})') # <<<<<<<<<<<<<< + * cuDriverGetVersion = dlsym(handle, "cuDriverGetVersion") + * if cuDriverGetVersion == NULL: +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_NotSupportedError); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 41, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_ssize_strlen(__pyx_v_err_msg); if (unlikely(__pyx_t_5 == ((Py_ssize_t)-1))) __PYX_ERR(0, 41, __pyx_L1_error) + __pyx_t_6 = __Pyx_decode_c_string(__pyx_v_err_msg, 0, __pyx_t_5, NULL, NULL, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 41, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7[0] = __pyx_mstate_global->__pyx_kp_u_CUDA_driver_is_not_found; + __pyx_t_7[1] = __pyx_t_6; + __pyx_t_7[2] = __pyx_mstate_global->__pyx_kp_u_; + __pyx_t_8 = __Pyx_PyUnicode_Join(__pyx_t_7, 3, 26 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_6) + 1, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_6)); + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 41, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __pyx_t_9 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_4, __pyx__function); + __pyx_t_9 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, __pyx_t_8}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_4, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 41, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 41, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":39 + * # Load driver to check version + * handle = dlopen('libcuda.so.1', RTLD_NOW | RTLD_GLOBAL) + * if handle == NULL: # <<<<<<<<<<<<<< + * err_msg = dlerror() + * raise NotSupportedError(f'CUDA driver is not found ({err_msg.decode()})') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":42 + * err_msg = dlerror() + * raise NotSupportedError(f'CUDA driver is not found ({err_msg.decode()})') + * cuDriverGetVersion = dlsym(handle, "cuDriverGetVersion") # <<<<<<<<<<<<<< + * if cuDriverGetVersion == NULL: + * raise RuntimeError('Did not find cuDriverGetVersion symbol in libcuda.so.1') +*/ + __pyx_v_cuDriverGetVersion = dlsym(__pyx_v_handle, ((char const *)"cuDriverGetVersion")); + + /* "cuda/bindings/_internal/nvvm.pyx":43 + * raise NotSupportedError(f'CUDA driver is not found ({err_msg.decode()})') + * cuDriverGetVersion = dlsym(handle, "cuDriverGetVersion") + * if cuDriverGetVersion == NULL: # <<<<<<<<<<<<<< + * raise RuntimeError('Did not find cuDriverGetVersion symbol in libcuda.so.1') + * err = (cuDriverGetVersion)(&driver_ver) +*/ + __pyx_t_1 = (__pyx_v_cuDriverGetVersion == NULL); + if (unlikely(__pyx_t_1)) { + + /* "cuda/bindings/_internal/nvvm.pyx":44 + * cuDriverGetVersion = dlsym(handle, "cuDriverGetVersion") + * if cuDriverGetVersion == NULL: + * raise RuntimeError('Did not find cuDriverGetVersion symbol in libcuda.so.1') # <<<<<<<<<<<<<< + * err = (cuDriverGetVersion)(&driver_ver) + * if err != 0: +*/ + __pyx_t_4 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_8 = __pyx_builtin_RuntimeError; + __pyx_t_9 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_Did_not_find_cuDriverGetVersion}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_8, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 44, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 44, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":43 + * raise NotSupportedError(f'CUDA driver is not found ({err_msg.decode()})') + * cuDriverGetVersion = dlsym(handle, "cuDriverGetVersion") + * if cuDriverGetVersion == NULL: # <<<<<<<<<<<<<< + * raise RuntimeError('Did not find cuDriverGetVersion symbol in libcuda.so.1') + * err = (cuDriverGetVersion)(&driver_ver) +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":45 + * if cuDriverGetVersion == NULL: + * raise RuntimeError('Did not find cuDriverGetVersion symbol in libcuda.so.1') + * err = (cuDriverGetVersion)(&driver_ver) # <<<<<<<<<<<<<< + * if err != 0: + * raise RuntimeError(f'cuDriverGetVersion returned error code {err}') +*/ + __pyx_v_err = ((int (*)(int *))__pyx_v_cuDriverGetVersion)((&__pyx_v_driver_ver)); + + /* "cuda/bindings/_internal/nvvm.pyx":46 + * raise RuntimeError('Did not find cuDriverGetVersion symbol in libcuda.so.1') + * err = (cuDriverGetVersion)(&driver_ver) + * if err != 0: # <<<<<<<<<<<<<< + * raise RuntimeError(f'cuDriverGetVersion returned error code {err}') + * +*/ + __pyx_t_1 = (__pyx_v_err != 0); + if (unlikely(__pyx_t_1)) { + + /* "cuda/bindings/_internal/nvvm.pyx":47 + * err = (cuDriverGetVersion)(&driver_ver) + * if err != 0: + * raise RuntimeError(f'cuDriverGetVersion returned error code {err}') # <<<<<<<<<<<<<< + * + * return driver_ver +*/ + __pyx_t_8 = NULL; + __Pyx_INCREF(__pyx_builtin_RuntimeError); + __pyx_t_4 = __pyx_builtin_RuntimeError; + __pyx_t_3 = __Pyx_PyUnicode_From_int(__pyx_v_err, 0, ' ', 'd'); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 47, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_6 = __Pyx_PyUnicode_Concat(__pyx_mstate_global->__pyx_kp_u_cuDriverGetVersion_returned_erro, __pyx_t_3); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 47, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_9 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_8, __pyx_t_6}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_4, __pyx_callargs+__pyx_t_9, (2-__pyx_t_9) | (__pyx_t_9*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_8); __pyx_t_8 = 0; + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 47, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_Raise(__pyx_t_2, 0, 0, 0); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __PYX_ERR(0, 47, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":46 + * raise RuntimeError('Did not find cuDriverGetVersion symbol in libcuda.so.1') + * err = (cuDriverGetVersion)(&driver_ver) + * if err != 0: # <<<<<<<<<<<<<< + * raise RuntimeError(f'cuDriverGetVersion returned error code {err}') + * +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":49 + * raise RuntimeError(f'cuDriverGetVersion returned error code {err}') + * + * return driver_ver # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_driver_ver; + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":33 + * const void* RTLD_DEFAULT 'RTLD_DEFAULT' + * + * cdef int get_cuda_version(): # <<<<<<<<<<<<<< + * cdef void* handle = NULL + * cdef int err, driver_ver = 0 +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm.get_cuda_version", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":75 + * + * + * cdef void* load_library() except* with gil: # <<<<<<<<<<<<<< + * cdef uintptr_t handle = load_nvidia_dynamic_lib("nvvm")._handle_uint + * return handle +*/ + +static void *__pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(void) { + uintptr_t __pyx_v_handle; + void *__pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + size_t __pyx_t_4; + uintptr_t __pyx_t_5; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_RefNannySetupContext("load_library", 0); + + /* "cuda/bindings/_internal/nvvm.pyx":76 + * + * cdef void* load_library() except* with gil: + * cdef uintptr_t handle = load_nvidia_dynamic_lib("nvvm")._handle_uint # <<<<<<<<<<<<<< + * return handle + * +*/ + __pyx_t_2 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_3))) { + __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); + assert(__pyx_t_2); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_3); + __Pyx_INCREF(__pyx_t_2); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_3, __pyx__function); + __pyx_t_4 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_2, __pyx_mstate_global->__pyx_n_u_nvvm}; + __pyx_t_1 = __Pyx_PyObject_FastCall(__pyx_t_3, __pyx_callargs+__pyx_t_4, (2-__pyx_t_4) | (__pyx_t_4*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + } + __pyx_t_3 = __Pyx_PyObject_GetAttrStr(__pyx_t_1, __pyx_mstate_global->__pyx_n_u_handle_uint); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_5 = __Pyx_PyLong_As_size_t(__pyx_t_3); if (unlikely((__pyx_t_5 == ((uintptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 76, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_v_handle = __pyx_t_5; + + /* "cuda/bindings/_internal/nvvm.pyx":77 + * cdef void* load_library() except* with gil: + * cdef uintptr_t handle = load_nvidia_dynamic_lib("nvvm")._handle_uint + * return handle # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = ((void *)__pyx_v_handle); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":75 + * + * + * cdef void* load_library() except* with gil: # <<<<<<<<<<<<<< + * cdef uintptr_t handle = load_nvidia_dynamic_lib("nvvm")._handle_uint + * return handle +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm.load_library", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + __Pyx_PyGILState_Release(__pyx_gilstate_save); + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":80 + * + * + * cdef int __check_or_init_nvvm() except -1 nogil: # <<<<<<<<<<<<<< + * global __py_nvvm_init + * +*/ + +static int __pyx_f_4cuda_8bindings_9_internal_4nvvm___check_or_init_nvvm(void) { + void *__pyx_v_handle; + int __pyx_r; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *__pyx_t_7 = NULL; + PyObject *__pyx_t_8 = NULL; + int __pyx_t_9; + void *__pyx_t_10; + PyObject *__pyx_t_11 = NULL; + PyObject *__pyx_t_12 = NULL; + int __pyx_t_13; + int __pyx_t_14; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("__check_or_init_nvvm", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":83 + * global __py_nvvm_init + * + * cdef void* handle = NULL # <<<<<<<<<<<<<< + * + * with gil, __symbol_lock: +*/ + __pyx_v_handle = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":85 + * cdef void* handle = NULL + * + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * # Load function + * global __nvvmGetErrorString +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + /*with:*/ { + __pyx_t_1 = __Pyx_PyObject_LookupSpecial(__pyx_v_4cuda_8bindings_9_internal_4nvvm___symbol_lock, __pyx_mstate_global->__pyx_n_u_exit); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 85, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_t_3 = NULL; + __pyx_t_4 = __Pyx_PyObject_LookupSpecial(__pyx_v_4cuda_8bindings_9_internal_4nvvm___symbol_lock, __pyx_mstate_global->__pyx_n_u_enter); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 85, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = 1; + #if CYTHON_UNPACK_METHODS + if (likely(PyMethod_Check(__pyx_t_4))) { + __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); + assert(__pyx_t_3); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_4); + __Pyx_INCREF(__pyx_t_3); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_4, __pyx__function); + __pyx_t_5 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_4, __pyx_callargs+__pyx_t_5, (1-__pyx_t_5) | (__pyx_t_5*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 85, __pyx_L6_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + /*try:*/ { + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_6, &__pyx_t_7, &__pyx_t_8); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_7); + __Pyx_XGOTREF(__pyx_t_8); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":88 + * # Load function + * global __nvvmGetErrorString + * __nvvmGetErrorString = dlsym(RTLD_DEFAULT, 'nvvmGetErrorString') # <<<<<<<<<<<<<< + * if __nvvmGetErrorString == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetErrorString = dlsym(RTLD_DEFAULT, ((char const *)"nvvmGetErrorString")); + + /* "cuda/bindings/_internal/nvvm.pyx":89 + * global __nvvmGetErrorString + * __nvvmGetErrorString = dlsym(RTLD_DEFAULT, 'nvvmGetErrorString') + * if __nvvmGetErrorString == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetErrorString == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":90 + * __nvvmGetErrorString = dlsym(RTLD_DEFAULT, 'nvvmGetErrorString') + * if __nvvmGetErrorString == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmGetErrorString = dlsym(handle, 'nvvmGetErrorString') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":91 + * if __nvvmGetErrorString == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvvmGetErrorString = dlsym(handle, 'nvvmGetErrorString') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 91, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvvm.pyx":90 + * __nvvmGetErrorString = dlsym(RTLD_DEFAULT, 'nvvmGetErrorString') + * if __nvvmGetErrorString == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmGetErrorString = dlsym(handle, 'nvvmGetErrorString') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":92 + * if handle == NULL: + * handle = load_library() + * __nvvmGetErrorString = dlsym(handle, 'nvvmGetErrorString') # <<<<<<<<<<<<<< + * + * global __nvvmVersion +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetErrorString = dlsym(__pyx_v_handle, ((char const *)"nvvmGetErrorString")); + + /* "cuda/bindings/_internal/nvvm.pyx":89 + * global __nvvmGetErrorString + * __nvvmGetErrorString = dlsym(RTLD_DEFAULT, 'nvvmGetErrorString') + * if __nvvmGetErrorString == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":95 + * + * global __nvvmVersion + * __nvvmVersion = dlsym(RTLD_DEFAULT, 'nvvmVersion') # <<<<<<<<<<<<<< + * if __nvvmVersion == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVersion = dlsym(RTLD_DEFAULT, ((char const *)"nvvmVersion")); + + /* "cuda/bindings/_internal/nvvm.pyx":96 + * global __nvvmVersion + * __nvvmVersion = dlsym(RTLD_DEFAULT, 'nvvmVersion') + * if __nvvmVersion == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVersion == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":97 + * __nvvmVersion = dlsym(RTLD_DEFAULT, 'nvvmVersion') + * if __nvvmVersion == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmVersion = dlsym(handle, 'nvvmVersion') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":98 + * if __nvvmVersion == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvvmVersion = dlsym(handle, 'nvvmVersion') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 98, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvvm.pyx":97 + * __nvvmVersion = dlsym(RTLD_DEFAULT, 'nvvmVersion') + * if __nvvmVersion == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmVersion = dlsym(handle, 'nvvmVersion') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":99 + * if handle == NULL: + * handle = load_library() + * __nvvmVersion = dlsym(handle, 'nvvmVersion') # <<<<<<<<<<<<<< + * + * global __nvvmIRVersion +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVersion = dlsym(__pyx_v_handle, ((char const *)"nvvmVersion")); + + /* "cuda/bindings/_internal/nvvm.pyx":96 + * global __nvvmVersion + * __nvvmVersion = dlsym(RTLD_DEFAULT, 'nvvmVersion') + * if __nvvmVersion == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":102 + * + * global __nvvmIRVersion + * __nvvmIRVersion = dlsym(RTLD_DEFAULT, 'nvvmIRVersion') # <<<<<<<<<<<<<< + * if __nvvmIRVersion == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmIRVersion = dlsym(RTLD_DEFAULT, ((char const *)"nvvmIRVersion")); + + /* "cuda/bindings/_internal/nvvm.pyx":103 + * global __nvvmIRVersion + * __nvvmIRVersion = dlsym(RTLD_DEFAULT, 'nvvmIRVersion') + * if __nvvmIRVersion == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmIRVersion == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":104 + * __nvvmIRVersion = dlsym(RTLD_DEFAULT, 'nvvmIRVersion') + * if __nvvmIRVersion == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmIRVersion = dlsym(handle, 'nvvmIRVersion') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":105 + * if __nvvmIRVersion == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvvmIRVersion = dlsym(handle, 'nvvmIRVersion') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 105, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvvm.pyx":104 + * __nvvmIRVersion = dlsym(RTLD_DEFAULT, 'nvvmIRVersion') + * if __nvvmIRVersion == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmIRVersion = dlsym(handle, 'nvvmIRVersion') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":106 + * if handle == NULL: + * handle = load_library() + * __nvvmIRVersion = dlsym(handle, 'nvvmIRVersion') # <<<<<<<<<<<<<< + * + * global __nvvmCreateProgram +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmIRVersion = dlsym(__pyx_v_handle, ((char const *)"nvvmIRVersion")); + + /* "cuda/bindings/_internal/nvvm.pyx":103 + * global __nvvmIRVersion + * __nvvmIRVersion = dlsym(RTLD_DEFAULT, 'nvvmIRVersion') + * if __nvvmIRVersion == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":109 + * + * global __nvvmCreateProgram + * __nvvmCreateProgram = dlsym(RTLD_DEFAULT, 'nvvmCreateProgram') # <<<<<<<<<<<<<< + * if __nvvmCreateProgram == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCreateProgram = dlsym(RTLD_DEFAULT, ((char const *)"nvvmCreateProgram")); + + /* "cuda/bindings/_internal/nvvm.pyx":110 + * global __nvvmCreateProgram + * __nvvmCreateProgram = dlsym(RTLD_DEFAULT, 'nvvmCreateProgram') + * if __nvvmCreateProgram == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCreateProgram == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":111 + * __nvvmCreateProgram = dlsym(RTLD_DEFAULT, 'nvvmCreateProgram') + * if __nvvmCreateProgram == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmCreateProgram = dlsym(handle, 'nvvmCreateProgram') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":112 + * if __nvvmCreateProgram == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvvmCreateProgram = dlsym(handle, 'nvvmCreateProgram') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 112, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvvm.pyx":111 + * __nvvmCreateProgram = dlsym(RTLD_DEFAULT, 'nvvmCreateProgram') + * if __nvvmCreateProgram == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmCreateProgram = dlsym(handle, 'nvvmCreateProgram') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":113 + * if handle == NULL: + * handle = load_library() + * __nvvmCreateProgram = dlsym(handle, 'nvvmCreateProgram') # <<<<<<<<<<<<<< + * + * global __nvvmDestroyProgram +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCreateProgram = dlsym(__pyx_v_handle, ((char const *)"nvvmCreateProgram")); + + /* "cuda/bindings/_internal/nvvm.pyx":110 + * global __nvvmCreateProgram + * __nvvmCreateProgram = dlsym(RTLD_DEFAULT, 'nvvmCreateProgram') + * if __nvvmCreateProgram == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":116 + * + * global __nvvmDestroyProgram + * __nvvmDestroyProgram = dlsym(RTLD_DEFAULT, 'nvvmDestroyProgram') # <<<<<<<<<<<<<< + * if __nvvmDestroyProgram == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmDestroyProgram = dlsym(RTLD_DEFAULT, ((char const *)"nvvmDestroyProgram")); + + /* "cuda/bindings/_internal/nvvm.pyx":117 + * global __nvvmDestroyProgram + * __nvvmDestroyProgram = dlsym(RTLD_DEFAULT, 'nvvmDestroyProgram') + * if __nvvmDestroyProgram == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmDestroyProgram == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":118 + * __nvvmDestroyProgram = dlsym(RTLD_DEFAULT, 'nvvmDestroyProgram') + * if __nvvmDestroyProgram == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmDestroyProgram = dlsym(handle, 'nvvmDestroyProgram') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":119 + * if __nvvmDestroyProgram == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvvmDestroyProgram = dlsym(handle, 'nvvmDestroyProgram') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 119, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvvm.pyx":118 + * __nvvmDestroyProgram = dlsym(RTLD_DEFAULT, 'nvvmDestroyProgram') + * if __nvvmDestroyProgram == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmDestroyProgram = dlsym(handle, 'nvvmDestroyProgram') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":120 + * if handle == NULL: + * handle = load_library() + * __nvvmDestroyProgram = dlsym(handle, 'nvvmDestroyProgram') # <<<<<<<<<<<<<< + * + * global __nvvmAddModuleToProgram +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmDestroyProgram = dlsym(__pyx_v_handle, ((char const *)"nvvmDestroyProgram")); + + /* "cuda/bindings/_internal/nvvm.pyx":117 + * global __nvvmDestroyProgram + * __nvvmDestroyProgram = dlsym(RTLD_DEFAULT, 'nvvmDestroyProgram') + * if __nvvmDestroyProgram == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":123 + * + * global __nvvmAddModuleToProgram + * __nvvmAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmAddModuleToProgram') # <<<<<<<<<<<<<< + * if __nvvmAddModuleToProgram == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmAddModuleToProgram = dlsym(RTLD_DEFAULT, ((char const *)"nvvmAddModuleToProgram")); + + /* "cuda/bindings/_internal/nvvm.pyx":124 + * global __nvvmAddModuleToProgram + * __nvvmAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmAddModuleToProgram') + * if __nvvmAddModuleToProgram == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmAddModuleToProgram == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":125 + * __nvvmAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmAddModuleToProgram') + * if __nvvmAddModuleToProgram == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmAddModuleToProgram = dlsym(handle, 'nvvmAddModuleToProgram') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":126 + * if __nvvmAddModuleToProgram == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvvmAddModuleToProgram = dlsym(handle, 'nvvmAddModuleToProgram') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 126, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvvm.pyx":125 + * __nvvmAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmAddModuleToProgram') + * if __nvvmAddModuleToProgram == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmAddModuleToProgram = dlsym(handle, 'nvvmAddModuleToProgram') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":127 + * if handle == NULL: + * handle = load_library() + * __nvvmAddModuleToProgram = dlsym(handle, 'nvvmAddModuleToProgram') # <<<<<<<<<<<<<< + * + * global __nvvmLazyAddModuleToProgram +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmAddModuleToProgram = dlsym(__pyx_v_handle, ((char const *)"nvvmAddModuleToProgram")); + + /* "cuda/bindings/_internal/nvvm.pyx":124 + * global __nvvmAddModuleToProgram + * __nvvmAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmAddModuleToProgram') + * if __nvvmAddModuleToProgram == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":130 + * + * global __nvvmLazyAddModuleToProgram + * __nvvmLazyAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmLazyAddModuleToProgram') # <<<<<<<<<<<<<< + * if __nvvmLazyAddModuleToProgram == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmLazyAddModuleToProgram = dlsym(RTLD_DEFAULT, ((char const *)"nvvmLazyAddModuleToProgram")); + + /* "cuda/bindings/_internal/nvvm.pyx":131 + * global __nvvmLazyAddModuleToProgram + * __nvvmLazyAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmLazyAddModuleToProgram') + * if __nvvmLazyAddModuleToProgram == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmLazyAddModuleToProgram == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":132 + * __nvvmLazyAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmLazyAddModuleToProgram') + * if __nvvmLazyAddModuleToProgram == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmLazyAddModuleToProgram = dlsym(handle, 'nvvmLazyAddModuleToProgram') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":133 + * if __nvvmLazyAddModuleToProgram == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvvmLazyAddModuleToProgram = dlsym(handle, 'nvvmLazyAddModuleToProgram') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 133, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvvm.pyx":132 + * __nvvmLazyAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmLazyAddModuleToProgram') + * if __nvvmLazyAddModuleToProgram == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmLazyAddModuleToProgram = dlsym(handle, 'nvvmLazyAddModuleToProgram') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":134 + * if handle == NULL: + * handle = load_library() + * __nvvmLazyAddModuleToProgram = dlsym(handle, 'nvvmLazyAddModuleToProgram') # <<<<<<<<<<<<<< + * + * global __nvvmCompileProgram +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmLazyAddModuleToProgram = dlsym(__pyx_v_handle, ((char const *)"nvvmLazyAddModuleToProgram")); + + /* "cuda/bindings/_internal/nvvm.pyx":131 + * global __nvvmLazyAddModuleToProgram + * __nvvmLazyAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmLazyAddModuleToProgram') + * if __nvvmLazyAddModuleToProgram == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":137 + * + * global __nvvmCompileProgram + * __nvvmCompileProgram = dlsym(RTLD_DEFAULT, 'nvvmCompileProgram') # <<<<<<<<<<<<<< + * if __nvvmCompileProgram == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCompileProgram = dlsym(RTLD_DEFAULT, ((char const *)"nvvmCompileProgram")); + + /* "cuda/bindings/_internal/nvvm.pyx":138 + * global __nvvmCompileProgram + * __nvvmCompileProgram = dlsym(RTLD_DEFAULT, 'nvvmCompileProgram') + * if __nvvmCompileProgram == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCompileProgram == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":139 + * __nvvmCompileProgram = dlsym(RTLD_DEFAULT, 'nvvmCompileProgram') + * if __nvvmCompileProgram == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmCompileProgram = dlsym(handle, 'nvvmCompileProgram') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":140 + * if __nvvmCompileProgram == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvvmCompileProgram = dlsym(handle, 'nvvmCompileProgram') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 140, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvvm.pyx":139 + * __nvvmCompileProgram = dlsym(RTLD_DEFAULT, 'nvvmCompileProgram') + * if __nvvmCompileProgram == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmCompileProgram = dlsym(handle, 'nvvmCompileProgram') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":141 + * if handle == NULL: + * handle = load_library() + * __nvvmCompileProgram = dlsym(handle, 'nvvmCompileProgram') # <<<<<<<<<<<<<< + * + * global __nvvmVerifyProgram +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCompileProgram = dlsym(__pyx_v_handle, ((char const *)"nvvmCompileProgram")); + + /* "cuda/bindings/_internal/nvvm.pyx":138 + * global __nvvmCompileProgram + * __nvvmCompileProgram = dlsym(RTLD_DEFAULT, 'nvvmCompileProgram') + * if __nvvmCompileProgram == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":144 + * + * global __nvvmVerifyProgram + * __nvvmVerifyProgram = dlsym(RTLD_DEFAULT, 'nvvmVerifyProgram') # <<<<<<<<<<<<<< + * if __nvvmVerifyProgram == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVerifyProgram = dlsym(RTLD_DEFAULT, ((char const *)"nvvmVerifyProgram")); + + /* "cuda/bindings/_internal/nvvm.pyx":145 + * global __nvvmVerifyProgram + * __nvvmVerifyProgram = dlsym(RTLD_DEFAULT, 'nvvmVerifyProgram') + * if __nvvmVerifyProgram == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVerifyProgram == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":146 + * __nvvmVerifyProgram = dlsym(RTLD_DEFAULT, 'nvvmVerifyProgram') + * if __nvvmVerifyProgram == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmVerifyProgram = dlsym(handle, 'nvvmVerifyProgram') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":147 + * if __nvvmVerifyProgram == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvvmVerifyProgram = dlsym(handle, 'nvvmVerifyProgram') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 147, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvvm.pyx":146 + * __nvvmVerifyProgram = dlsym(RTLD_DEFAULT, 'nvvmVerifyProgram') + * if __nvvmVerifyProgram == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmVerifyProgram = dlsym(handle, 'nvvmVerifyProgram') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":148 + * if handle == NULL: + * handle = load_library() + * __nvvmVerifyProgram = dlsym(handle, 'nvvmVerifyProgram') # <<<<<<<<<<<<<< + * + * global __nvvmGetCompiledResultSize +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVerifyProgram = dlsym(__pyx_v_handle, ((char const *)"nvvmVerifyProgram")); + + /* "cuda/bindings/_internal/nvvm.pyx":145 + * global __nvvmVerifyProgram + * __nvvmVerifyProgram = dlsym(RTLD_DEFAULT, 'nvvmVerifyProgram') + * if __nvvmVerifyProgram == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":151 + * + * global __nvvmGetCompiledResultSize + * __nvvmGetCompiledResultSize = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResultSize') # <<<<<<<<<<<<<< + * if __nvvmGetCompiledResultSize == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResultSize = dlsym(RTLD_DEFAULT, ((char const *)"nvvmGetCompiledResultSize")); + + /* "cuda/bindings/_internal/nvvm.pyx":152 + * global __nvvmGetCompiledResultSize + * __nvvmGetCompiledResultSize = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResultSize') + * if __nvvmGetCompiledResultSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResultSize == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":153 + * __nvvmGetCompiledResultSize = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResultSize') + * if __nvvmGetCompiledResultSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmGetCompiledResultSize = dlsym(handle, 'nvvmGetCompiledResultSize') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":154 + * if __nvvmGetCompiledResultSize == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvvmGetCompiledResultSize = dlsym(handle, 'nvvmGetCompiledResultSize') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 154, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvvm.pyx":153 + * __nvvmGetCompiledResultSize = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResultSize') + * if __nvvmGetCompiledResultSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmGetCompiledResultSize = dlsym(handle, 'nvvmGetCompiledResultSize') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":155 + * if handle == NULL: + * handle = load_library() + * __nvvmGetCompiledResultSize = dlsym(handle, 'nvvmGetCompiledResultSize') # <<<<<<<<<<<<<< + * + * global __nvvmGetCompiledResult +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResultSize = dlsym(__pyx_v_handle, ((char const *)"nvvmGetCompiledResultSize")); + + /* "cuda/bindings/_internal/nvvm.pyx":152 + * global __nvvmGetCompiledResultSize + * __nvvmGetCompiledResultSize = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResultSize') + * if __nvvmGetCompiledResultSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":158 + * + * global __nvvmGetCompiledResult + * __nvvmGetCompiledResult = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResult') # <<<<<<<<<<<<<< + * if __nvvmGetCompiledResult == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResult = dlsym(RTLD_DEFAULT, ((char const *)"nvvmGetCompiledResult")); + + /* "cuda/bindings/_internal/nvvm.pyx":159 + * global __nvvmGetCompiledResult + * __nvvmGetCompiledResult = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResult') + * if __nvvmGetCompiledResult == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResult == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":160 + * __nvvmGetCompiledResult = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResult') + * if __nvvmGetCompiledResult == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmGetCompiledResult = dlsym(handle, 'nvvmGetCompiledResult') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":161 + * if __nvvmGetCompiledResult == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvvmGetCompiledResult = dlsym(handle, 'nvvmGetCompiledResult') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 161, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvvm.pyx":160 + * __nvvmGetCompiledResult = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResult') + * if __nvvmGetCompiledResult == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmGetCompiledResult = dlsym(handle, 'nvvmGetCompiledResult') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":162 + * if handle == NULL: + * handle = load_library() + * __nvvmGetCompiledResult = dlsym(handle, 'nvvmGetCompiledResult') # <<<<<<<<<<<<<< + * + * global __nvvmGetProgramLogSize +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResult = dlsym(__pyx_v_handle, ((char const *)"nvvmGetCompiledResult")); + + /* "cuda/bindings/_internal/nvvm.pyx":159 + * global __nvvmGetCompiledResult + * __nvvmGetCompiledResult = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResult') + * if __nvvmGetCompiledResult == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":165 + * + * global __nvvmGetProgramLogSize + * __nvvmGetProgramLogSize = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLogSize') # <<<<<<<<<<<<<< + * if __nvvmGetProgramLogSize == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLogSize = dlsym(RTLD_DEFAULT, ((char const *)"nvvmGetProgramLogSize")); + + /* "cuda/bindings/_internal/nvvm.pyx":166 + * global __nvvmGetProgramLogSize + * __nvvmGetProgramLogSize = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLogSize') + * if __nvvmGetProgramLogSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLogSize == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":167 + * __nvvmGetProgramLogSize = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLogSize') + * if __nvvmGetProgramLogSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmGetProgramLogSize = dlsym(handle, 'nvvmGetProgramLogSize') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":168 + * if __nvvmGetProgramLogSize == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvvmGetProgramLogSize = dlsym(handle, 'nvvmGetProgramLogSize') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 168, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvvm.pyx":167 + * __nvvmGetProgramLogSize = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLogSize') + * if __nvvmGetProgramLogSize == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmGetProgramLogSize = dlsym(handle, 'nvvmGetProgramLogSize') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":169 + * if handle == NULL: + * handle = load_library() + * __nvvmGetProgramLogSize = dlsym(handle, 'nvvmGetProgramLogSize') # <<<<<<<<<<<<<< + * + * global __nvvmGetProgramLog +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLogSize = dlsym(__pyx_v_handle, ((char const *)"nvvmGetProgramLogSize")); + + /* "cuda/bindings/_internal/nvvm.pyx":166 + * global __nvvmGetProgramLogSize + * __nvvmGetProgramLogSize = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLogSize') + * if __nvvmGetProgramLogSize == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":172 + * + * global __nvvmGetProgramLog + * __nvvmGetProgramLog = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLog') # <<<<<<<<<<<<<< + * if __nvvmGetProgramLog == NULL: + * if handle == NULL: +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLog = dlsym(RTLD_DEFAULT, ((char const *)"nvvmGetProgramLog")); + + /* "cuda/bindings/_internal/nvvm.pyx":173 + * global __nvvmGetProgramLog + * __nvvmGetProgramLog = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLog') + * if __nvvmGetProgramLog == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + __pyx_t_9 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLog == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":174 + * __nvvmGetProgramLog = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLog') + * if __nvvmGetProgramLog == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmGetProgramLog = dlsym(handle, 'nvvmGetProgramLog') +*/ + __pyx_t_9 = (__pyx_v_handle == NULL); + if (__pyx_t_9) { + + /* "cuda/bindings/_internal/nvvm.pyx":175 + * if __nvvmGetProgramLog == NULL: + * if handle == NULL: + * handle = load_library() # <<<<<<<<<<<<<< + * __nvvmGetProgramLog = dlsym(handle, 'nvvmGetProgramLog') + * +*/ + __pyx_t_10 = __pyx_f_4cuda_8bindings_9_internal_4nvvm_load_library(); if (unlikely(__pyx_t_10 == ((void *)0) && PyErr_Occurred())) __PYX_ERR(0, 175, __pyx_L10_error) + __pyx_v_handle = __pyx_t_10; + + /* "cuda/bindings/_internal/nvvm.pyx":174 + * __nvvmGetProgramLog = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLog') + * if __nvvmGetProgramLog == NULL: + * if handle == NULL: # <<<<<<<<<<<<<< + * handle = load_library() + * __nvvmGetProgramLog = dlsym(handle, 'nvvmGetProgramLog') +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":176 + * if handle == NULL: + * handle = load_library() + * __nvvmGetProgramLog = dlsym(handle, 'nvvmGetProgramLog') # <<<<<<<<<<<<<< + * + * __py_nvvm_init = True +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLog = dlsym(__pyx_v_handle, ((char const *)"nvvmGetProgramLog")); + + /* "cuda/bindings/_internal/nvvm.pyx":173 + * global __nvvmGetProgramLog + * __nvvmGetProgramLog = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLog') + * if __nvvmGetProgramLog == NULL: # <<<<<<<<<<<<<< + * if handle == NULL: + * handle = load_library() +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":178 + * __nvvmGetProgramLog = dlsym(handle, 'nvvmGetProgramLog') + * + * __py_nvvm_init = True # <<<<<<<<<<<<<< + * return 0 + * +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___py_nvvm_init = 1; + + /* "cuda/bindings/_internal/nvvm.pyx":179 + * + * __py_nvvm_init = True + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L14_try_return; + + /* "cuda/bindings/_internal/nvvm.pyx":85 + * cdef void* handle = NULL + * + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * # Load function + * global __nvvmGetErrorString +*/ + } + __pyx_L10_error:; + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + /*except:*/ { + __Pyx_AddTraceback("cuda.bindings._internal.nvvm.__check_or_init_nvvm", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_2, &__pyx_t_4, &__pyx_t_3) < 0) __PYX_ERR(0, 85, __pyx_L12_except_error) + __Pyx_XGOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_3); + __pyx_t_11 = PyTuple_Pack(3, __pyx_t_2, __pyx_t_4, __pyx_t_3); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 85, __pyx_L12_except_error) + __Pyx_GOTREF(__pyx_t_11); + __pyx_t_12 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_t_11, NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 85, __pyx_L12_except_error) + __Pyx_GOTREF(__pyx_t_12); + __pyx_t_9 = __Pyx_PyObject_IsTrue(__pyx_t_12); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + if (__pyx_t_9 < (0)) __PYX_ERR(0, 85, __pyx_L12_except_error) + __pyx_t_13 = (!__pyx_t_9); + if (unlikely(__pyx_t_13)) { + __Pyx_GIVEREF(__pyx_t_2); + __Pyx_GIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_3); + __Pyx_ErrRestoreWithState(__pyx_t_2, __pyx_t_4, __pyx_t_3); + __pyx_t_2 = 0; __pyx_t_4 = 0; __pyx_t_3 = 0; + __PYX_ERR(0, 85, __pyx_L12_except_error) + } + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + goto __pyx_L11_exception_handled; + } + __pyx_L12_except_error:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L4_error; + __pyx_L14_try_return:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + goto __pyx_L7_return; + __pyx_L11_exception_handled:; + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_7); + __Pyx_XGIVEREF(__pyx_t_8); + __Pyx_ExceptionReset(__pyx_t_6, __pyx_t_7, __pyx_t_8); + } + } + /*finally:*/ { + /*normal exit:*/{ + if (__pyx_t_1) { + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_mstate_global->__pyx_tuple[0], NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 85, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + goto __pyx_L9; + } + __pyx_L7_return: { + __pyx_t_14 = __pyx_r; + if (__pyx_t_1) { + __pyx_t_8 = __Pyx_PyObject_Call(__pyx_t_1, __pyx_mstate_global->__pyx_tuple[0], NULL); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 85, __pyx_L4_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + } + __pyx_r = __pyx_t_14; + goto __pyx_L3_return; + } + __pyx_L9:; + } + goto __pyx_L45; + __pyx_L6_error:; + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + goto __pyx_L4_error; + __pyx_L45:; + } + } + /*finally:*/ { + /*normal exit:*/{ + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L5; + } + __pyx_L3_return: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L0; + } + __pyx_L4_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + __pyx_L5:; + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":80 + * + * + * cdef int __check_or_init_nvvm() except -1 nogil: # <<<<<<<<<<<<<< + * global __py_nvvm_init + * +*/ + + /* function exit code */ + __pyx_r = 0; + goto __pyx_L0; + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm.__check_or_init_nvvm", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":182 + * + * + * cdef int _check_or_init_nvvm() except -1 nogil: # <<<<<<<<<<<<<< + * if __py_nvvm_init: + * return 0 +*/ + +static int __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(void) { + int __pyx_r; + int __pyx_t_1; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + + /* "cuda/bindings/_internal/nvvm.pyx":183 + * + * cdef int _check_or_init_nvvm() except -1 nogil: + * if __py_nvvm_init: # <<<<<<<<<<<<<< + * return 0 + * +*/ + if (__pyx_v_4cuda_8bindings_9_internal_4nvvm___py_nvvm_init) { + + /* "cuda/bindings/_internal/nvvm.pyx":184 + * cdef int _check_or_init_nvvm() except -1 nogil: + * if __py_nvvm_init: + * return 0 # <<<<<<<<<<<<<< + * + * return __check_or_init_nvvm() +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":183 + * + * cdef int _check_or_init_nvvm() except -1 nogil: + * if __py_nvvm_init: # <<<<<<<<<<<<<< + * return 0 + * +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":186 + * return 0 + * + * return __check_or_init_nvvm() # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm___check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 186, __pyx_L1_error) + __pyx_r = __pyx_t_1; + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":182 + * + * + * cdef int _check_or_init_nvvm() except -1 nogil: # <<<<<<<<<<<<<< + * if __py_nvvm_init: + * return 0 +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._check_or_init_nvvm", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":192 + * + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_4nvvm_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_internal_4nvvm__inspect_function_pointers(CYTHON_UNUSED int __pyx_skip_dispatch) { + PyObject *__pyx_v_data = 0; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointers", 0); + + /* "cuda/bindings/_internal/nvvm.pyx":194 + * cpdef dict _inspect_function_pointers(): + * global func_ptrs + * if func_ptrs is not None: # <<<<<<<<<<<<<< + * return func_ptrs + * +*/ + __pyx_t_1 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs != ((PyObject*)Py_None)); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/nvvm.pyx":195 + * global func_ptrs + * if func_ptrs is not None: + * return func_ptrs # <<<<<<<<<<<<<< + * + * _check_or_init_nvvm() +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs); + __pyx_r = __pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs; + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":194 + * cpdef dict _inspect_function_pointers(): + * global func_ptrs + * if func_ptrs is not None: # <<<<<<<<<<<<<< + * return func_ptrs + * +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":197 + * return func_ptrs + * + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * cdef dict data = {} + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_2 == ((int)-1))) __PYX_ERR(0, 197, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":198 + * + * _check_or_init_nvvm() + * cdef dict data = {} # <<<<<<<<<<<<<< + * + * global __nvvmGetErrorString +*/ + __pyx_t_3 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 198, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_v_data = ((PyObject*)__pyx_t_3); + __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":201 + * + * global __nvvmGetErrorString + * data["__nvvmGetErrorString"] = __nvvmGetErrorString # <<<<<<<<<<<<<< + * + * global __nvvmVersion +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetErrorString)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 201, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvvmGetErrorString, __pyx_t_3) < 0))) __PYX_ERR(0, 201, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":204 + * + * global __nvvmVersion + * data["__nvvmVersion"] = __nvvmVersion # <<<<<<<<<<<<<< + * + * global __nvvmIRVersion +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVersion)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 204, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvvmVersion, __pyx_t_3) < 0))) __PYX_ERR(0, 204, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":207 + * + * global __nvvmIRVersion + * data["__nvvmIRVersion"] = __nvvmIRVersion # <<<<<<<<<<<<<< + * + * global __nvvmCreateProgram +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmIRVersion)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 207, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvvmIRVersion, __pyx_t_3) < 0))) __PYX_ERR(0, 207, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":210 + * + * global __nvvmCreateProgram + * data["__nvvmCreateProgram"] = __nvvmCreateProgram # <<<<<<<<<<<<<< + * + * global __nvvmDestroyProgram +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCreateProgram)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 210, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvvmCreateProgram, __pyx_t_3) < 0))) __PYX_ERR(0, 210, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":213 + * + * global __nvvmDestroyProgram + * data["__nvvmDestroyProgram"] = __nvvmDestroyProgram # <<<<<<<<<<<<<< + * + * global __nvvmAddModuleToProgram +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmDestroyProgram)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 213, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvvmDestroyProgram, __pyx_t_3) < 0))) __PYX_ERR(0, 213, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":216 + * + * global __nvvmAddModuleToProgram + * data["__nvvmAddModuleToProgram"] = __nvvmAddModuleToProgram # <<<<<<<<<<<<<< + * + * global __nvvmLazyAddModuleToProgram +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmAddModuleToProgram)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 216, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvvmAddModuleToProgram, __pyx_t_3) < 0))) __PYX_ERR(0, 216, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":219 + * + * global __nvvmLazyAddModuleToProgram + * data["__nvvmLazyAddModuleToProgram"] = __nvvmLazyAddModuleToProgram # <<<<<<<<<<<<<< + * + * global __nvvmCompileProgram +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmLazyAddModuleToProgram)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 219, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvvmLazyAddModuleToProgram, __pyx_t_3) < 0))) __PYX_ERR(0, 219, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":222 + * + * global __nvvmCompileProgram + * data["__nvvmCompileProgram"] = __nvvmCompileProgram # <<<<<<<<<<<<<< + * + * global __nvvmVerifyProgram +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCompileProgram)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 222, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvvmCompileProgram, __pyx_t_3) < 0))) __PYX_ERR(0, 222, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":225 + * + * global __nvvmVerifyProgram + * data["__nvvmVerifyProgram"] = __nvvmVerifyProgram # <<<<<<<<<<<<<< + * + * global __nvvmGetCompiledResultSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVerifyProgram)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 225, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvvmVerifyProgram, __pyx_t_3) < 0))) __PYX_ERR(0, 225, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":228 + * + * global __nvvmGetCompiledResultSize + * data["__nvvmGetCompiledResultSize"] = __nvvmGetCompiledResultSize # <<<<<<<<<<<<<< + * + * global __nvvmGetCompiledResult +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResultSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvvmGetCompiledResultSize, __pyx_t_3) < 0))) __PYX_ERR(0, 228, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":231 + * + * global __nvvmGetCompiledResult + * data["__nvvmGetCompiledResult"] = __nvvmGetCompiledResult # <<<<<<<<<<<<<< + * + * global __nvvmGetProgramLogSize +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResult)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 231, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvvmGetCompiledResult, __pyx_t_3) < 0))) __PYX_ERR(0, 231, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":234 + * + * global __nvvmGetProgramLogSize + * data["__nvvmGetProgramLogSize"] = __nvvmGetProgramLogSize # <<<<<<<<<<<<<< + * + * global __nvvmGetProgramLog +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLogSize)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 234, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvvmGetProgramLogSize, __pyx_t_3) < 0))) __PYX_ERR(0, 234, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":237 + * + * global __nvvmGetProgramLog + * data["__nvvmGetProgramLog"] = __nvvmGetProgramLog # <<<<<<<<<<<<<< + * + * func_ptrs = data +*/ + __pyx_t_3 = PyLong_FromSsize_t(((intptr_t)__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLog)); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 237, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (unlikely((PyDict_SetItem(__pyx_v_data, __pyx_mstate_global->__pyx_n_u_nvvmGetProgramLog, __pyx_t_3) < 0))) __PYX_ERR(0, 237, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":239 + * data["__nvvmGetProgramLog"] = __nvvmGetProgramLog + * + * func_ptrs = data # <<<<<<<<<<<<<< + * return data + * +*/ + __Pyx_INCREF(__pyx_v_data); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs, __pyx_v_data); + __Pyx_GIVEREF(__pyx_v_data); + + /* "cuda/bindings/_internal/nvvm.pyx":240 + * + * func_ptrs = data + * return data # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_data); + __pyx_r = __pyx_v_data; + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":192 + * + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._inspect_function_pointers", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_data); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_4nvvm_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused); /*proto*/ +PyDoc_STRVAR(__pyx_doc_4cuda_8bindings_9_internal_4nvvm__inspect_function_pointers, "_inspect_function_pointers() -> dict"); +static PyMethodDef __pyx_mdef_4cuda_8bindings_9_internal_4nvvm_1_inspect_function_pointers = {"_inspect_function_pointers", (PyCFunction)__pyx_pw_4cuda_8bindings_9_internal_4nvvm_1_inspect_function_pointers, METH_NOARGS, __pyx_doc_4cuda_8bindings_9_internal_4nvvm__inspect_function_pointers}; +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_4nvvm_1_inspect_function_pointers(PyObject *__pyx_self, CYTHON_UNUSED PyObject *unused) { + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_inspect_function_pointers (wrapper)", 0); + __pyx_kwvalues = __Pyx_KwValues_VARARGS(__pyx_args, __pyx_nargs); + __pyx_r = __pyx_pf_4cuda_8bindings_9_internal_4nvvm__inspect_function_pointers(__pyx_self); + + /* function exit code */ + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_4cuda_8bindings_9_internal_4nvvm__inspect_function_pointers(CYTHON_UNUSED PyObject *__pyx_self) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointers", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__inspect_function_pointers(1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 192, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._inspect_function_pointers", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":243 + * + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_4nvvm_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +static PyObject *__pyx_f_4cuda_8bindings_9_internal_4nvvm__inspect_function_pointer(PyObject *__pyx_v_name, CYTHON_UNUSED int __pyx_skip_dispatch) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointer", 0); + + /* "cuda/bindings/_internal/nvvm.pyx":245 + * cpdef _inspect_function_pointer(str name): + * global func_ptrs + * if func_ptrs is None: # <<<<<<<<<<<<<< + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] +*/ + __pyx_t_1 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs == ((PyObject*)Py_None)); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/nvvm.pyx":246 + * global func_ptrs + * if func_ptrs is None: + * func_ptrs = _inspect_function_pointers() # <<<<<<<<<<<<<< + * return func_ptrs[name] + * +*/ + __pyx_t_2 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__inspect_function_pointers(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 246, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs, ((PyObject*)__pyx_t_2)); + __Pyx_GIVEREF(__pyx_t_2); + __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":245 + * cpdef _inspect_function_pointer(str name): + * global func_ptrs + * if func_ptrs is None: # <<<<<<<<<<<<<< + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":247 + * if func_ptrs is None: + * func_ptrs = _inspect_function_pointers() + * return func_ptrs[name] # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + if (unlikely(__pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs == Py_None)) { + PyErr_SetString(PyExc_TypeError, "'NoneType' object is not subscriptable"); + __PYX_ERR(0, 247, __pyx_L1_error) + } + __pyx_t_2 = __Pyx_PyDict_GetItem(__pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs, __pyx_v_name); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 247, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_r = __pyx_t_2; + __pyx_t_2 = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":243 + * + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* Python wrapper */ +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_4nvvm_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +); /*proto*/ +PyDoc_STRVAR(__pyx_doc_4cuda_8bindings_9_internal_4nvvm_2_inspect_function_pointer, "_inspect_function_pointer(str name)"); +static PyMethodDef __pyx_mdef_4cuda_8bindings_9_internal_4nvvm_3_inspect_function_pointer = {"_inspect_function_pointer", (PyCFunction)(void(*)(void))(__Pyx_PyCFunction_FastCallWithKeywords)__pyx_pw_4cuda_8bindings_9_internal_4nvvm_3_inspect_function_pointer, __Pyx_METH_FASTCALL|METH_KEYWORDS, __pyx_doc_4cuda_8bindings_9_internal_4nvvm_2_inspect_function_pointer}; +static PyObject *__pyx_pw_4cuda_8bindings_9_internal_4nvvm_3_inspect_function_pointer(PyObject *__pyx_self, +#if CYTHON_METH_FASTCALL +PyObject *const *__pyx_args, Py_ssize_t __pyx_nargs, PyObject *__pyx_kwds +#else +PyObject *__pyx_args, PyObject *__pyx_kwds +#endif +) { + PyObject *__pyx_v_name = 0; + #if !CYTHON_METH_FASTCALL + CYTHON_UNUSED Py_ssize_t __pyx_nargs; + #endif + CYTHON_UNUSED PyObject *const *__pyx_kwvalues; + PyObject* values[1] = {0}; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyObject *__pyx_r = 0; + __Pyx_RefNannyDeclarations + __Pyx_RefNannySetupContext("_inspect_function_pointer (wrapper)", 0); + #if !CYTHON_METH_FASTCALL + #if CYTHON_ASSUME_SAFE_SIZE + __pyx_nargs = PyTuple_GET_SIZE(__pyx_args); + #else + __pyx_nargs = PyTuple_Size(__pyx_args); if (unlikely(__pyx_nargs < 0)) return NULL; + #endif + #endif + __pyx_kwvalues = __Pyx_KwValues_FASTCALL(__pyx_args, __pyx_nargs); + { + PyObject ** const __pyx_pyargnames[] = {&__pyx_mstate_global->__pyx_n_u_name,0}; + const Py_ssize_t __pyx_kwds_len = (__pyx_kwds) ? __Pyx_NumKwargs_FASTCALL(__pyx_kwds) : 0; + if (unlikely(__pyx_kwds_len) < 0) __PYX_ERR(0, 243, __pyx_L3_error) + if (__pyx_kwds_len > 0) { + switch (__pyx_nargs) { + case 1: + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 243, __pyx_L3_error) + CYTHON_FALLTHROUGH; + case 0: break; + default: goto __pyx_L5_argtuple_error; + } + const Py_ssize_t kwd_pos_args = __pyx_nargs; + if (__Pyx_ParseKeywords(__pyx_kwds, __pyx_kwvalues, __pyx_pyargnames, 0, values, kwd_pos_args, __pyx_kwds_len, "_inspect_function_pointer", 0) < (0)) __PYX_ERR(0, 243, __pyx_L3_error) + for (Py_ssize_t i = __pyx_nargs; i < 1; i++) { + if (unlikely(!values[i])) { __Pyx_RaiseArgtupleInvalid("_inspect_function_pointer", 1, 1, 1, i); __PYX_ERR(0, 243, __pyx_L3_error) } + } + } else if (unlikely(__pyx_nargs != 1)) { + goto __pyx_L5_argtuple_error; + } else { + values[0] = __Pyx_ArgRef_FASTCALL(__pyx_args, 0); + if (!CYTHON_ASSUME_SAFE_MACROS && unlikely(!values[0])) __PYX_ERR(0, 243, __pyx_L3_error) + } + __pyx_v_name = ((PyObject*)values[0]); + } + goto __pyx_L6_skip; + __pyx_L5_argtuple_error:; + __Pyx_RaiseArgtupleInvalid("_inspect_function_pointer", 1, 1, 1, __pyx_nargs); __PYX_ERR(0, 243, __pyx_L3_error) + __pyx_L6_skip:; + goto __pyx_L4_argument_unpacking_done; + __pyx_L3_error:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_RefNannyFinishContext(); + return NULL; + __pyx_L4_argument_unpacking_done:; + if (unlikely(!__Pyx_ArgTypeTest(((PyObject *)__pyx_v_name), (&PyUnicode_Type), 1, "name", 1))) __PYX_ERR(0, 243, __pyx_L1_error) + __pyx_r = __pyx_pf_4cuda_8bindings_9_internal_4nvvm_2_inspect_function_pointer(__pyx_self, __pyx_v_name); + + /* function exit code */ + goto __pyx_L0; + __pyx_L1_error:; + __pyx_r = NULL; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + goto __pyx_L7_cleaned_up; + __pyx_L0:; + for (Py_ssize_t __pyx_temp=0; __pyx_temp < (Py_ssize_t)(sizeof(values)/sizeof(values[0])); ++__pyx_temp) { + Py_XDECREF(values[__pyx_temp]); + } + __pyx_L7_cleaned_up:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static PyObject *__pyx_pf_4cuda_8bindings_9_internal_4nvvm_2_inspect_function_pointer(CYTHON_UNUSED PyObject *__pyx_self, PyObject *__pyx_v_name) { + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("_inspect_function_pointer", 0); + __Pyx_XDECREF(__pyx_r); + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__inspect_function_pointer(__pyx_v_name, 1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 243, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_r = __pyx_t_1; + __pyx_t_1 = 0; + goto __pyx_L0; + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._inspect_function_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = NULL; + __pyx_L0:; + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":254 + * ############################################################################### + * + * cdef const char* _nvvmGetErrorString(nvvmResult result) except?NULL nogil: # <<<<<<<<<<<<<< + * global __nvvmGetErrorString + * _check_or_init_nvvm() +*/ + +static char const *__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmGetErrorString(nvvmResult __pyx_v_result) { + char const *__pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvvmGetErrorString", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":256 + * cdef const char* _nvvmGetErrorString(nvvmResult result) except?NULL nogil: + * global __nvvmGetErrorString + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * if __nvvmGetErrorString == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 256, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":257 + * global __nvvmGetErrorString + * _check_or_init_nvvm() + * if __nvvmGetErrorString == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmGetErrorString is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetErrorString == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvvm.pyx":258 + * _check_or_init_nvvm() + * if __nvvmGetErrorString == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmGetErrorString is not found") + * return (__nvvmGetErrorString)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":259 + * if __nvvmGetErrorString == NULL: + * with gil: + * raise FunctionNotFoundError("function nvvmGetErrorString is not found") # <<<<<<<<<<<<<< + * return (__nvvmGetErrorString)( + * result) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 259, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvvmGetErrorString_is_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 259, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 259, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvvm.pyx":258 + * _check_or_init_nvvm() + * if __nvvmGetErrorString == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmGetErrorString is not found") + * return (__nvvmGetErrorString)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":257 + * global __nvvmGetErrorString + * _check_or_init_nvvm() + * if __nvvmGetErrorString == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmGetErrorString is not found") +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":260 + * with gil: + * raise FunctionNotFoundError("function nvvmGetErrorString is not found") + * return (__nvvmGetErrorString)( # <<<<<<<<<<<<<< + * result) + * +*/ + __pyx_r = ((char const *(*)(nvvmResult))__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetErrorString)(__pyx_v_result); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":254 + * ############################################################################### + * + * cdef const char* _nvvmGetErrorString(nvvmResult result) except?NULL nogil: # <<<<<<<<<<<<<< + * global __nvvmGetErrorString + * _check_or_init_nvvm() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._nvvmGetErrorString", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":264 + * + * + * cdef nvvmResult _nvvmVersion(int* major, int* minor) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmVersion + * _check_or_init_nvvm() +*/ + +static nvvmResult __pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmVersion(int *__pyx_v_major, int *__pyx_v_minor) { + nvvmResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvvmVersion", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":266 + * cdef nvvmResult _nvvmVersion(int* major, int* minor) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvvmVersion + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * if __nvvmVersion == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 266, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":267 + * global __nvvmVersion + * _check_or_init_nvvm() + * if __nvvmVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmVersion is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVersion == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvvm.pyx":268 + * _check_or_init_nvvm() + * if __nvvmVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmVersion is not found") + * return (__nvvmVersion)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":269 + * if __nvvmVersion == NULL: + * with gil: + * raise FunctionNotFoundError("function nvvmVersion is not found") # <<<<<<<<<<<<<< + * return (__nvvmVersion)( + * major, minor) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 269, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvvmVersion_is_not_foun}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 269, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 269, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvvm.pyx":268 + * _check_or_init_nvvm() + * if __nvvmVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmVersion is not found") + * return (__nvvmVersion)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":267 + * global __nvvmVersion + * _check_or_init_nvvm() + * if __nvvmVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmVersion is not found") +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":270 + * with gil: + * raise FunctionNotFoundError("function nvvmVersion is not found") + * return (__nvvmVersion)( # <<<<<<<<<<<<<< + * major, minor) + * +*/ + __pyx_r = ((nvvmResult (*)(int *, int *))__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVersion)(__pyx_v_major, __pyx_v_minor); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":264 + * + * + * cdef nvvmResult _nvvmVersion(int* major, int* minor) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmVersion + * _check_or_init_nvvm() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._nvvmVersion", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVVMRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":274 + * + * + * cdef nvvmResult _nvvmIRVersion(int* majorIR, int* minorIR, int* majorDbg, int* minorDbg) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmIRVersion + * _check_or_init_nvvm() +*/ + +static nvvmResult __pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmIRVersion(int *__pyx_v_majorIR, int *__pyx_v_minorIR, int *__pyx_v_majorDbg, int *__pyx_v_minorDbg) { + nvvmResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvvmIRVersion", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":276 + * cdef nvvmResult _nvvmIRVersion(int* majorIR, int* minorIR, int* majorDbg, int* minorDbg) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvvmIRVersion + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * if __nvvmIRVersion == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 276, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":277 + * global __nvvmIRVersion + * _check_or_init_nvvm() + * if __nvvmIRVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmIRVersion is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmIRVersion == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvvm.pyx":278 + * _check_or_init_nvvm() + * if __nvvmIRVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmIRVersion is not found") + * return (__nvvmIRVersion)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":279 + * if __nvvmIRVersion == NULL: + * with gil: + * raise FunctionNotFoundError("function nvvmIRVersion is not found") # <<<<<<<<<<<<<< + * return (__nvvmIRVersion)( + * majorIR, minorIR, majorDbg, minorDbg) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 279, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvvmIRVersion_is_not_fo}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 279, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 279, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvvm.pyx":278 + * _check_or_init_nvvm() + * if __nvvmIRVersion == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmIRVersion is not found") + * return (__nvvmIRVersion)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":277 + * global __nvvmIRVersion + * _check_or_init_nvvm() + * if __nvvmIRVersion == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmIRVersion is not found") +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":280 + * with gil: + * raise FunctionNotFoundError("function nvvmIRVersion is not found") + * return (__nvvmIRVersion)( # <<<<<<<<<<<<<< + * majorIR, minorIR, majorDbg, minorDbg) + * +*/ + __pyx_r = ((nvvmResult (*)(int *, int *, int *, int *))__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmIRVersion)(__pyx_v_majorIR, __pyx_v_minorIR, __pyx_v_majorDbg, __pyx_v_minorDbg); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":274 + * + * + * cdef nvvmResult _nvvmIRVersion(int* majorIR, int* minorIR, int* majorDbg, int* minorDbg) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmIRVersion + * _check_or_init_nvvm() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._nvvmIRVersion", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVVMRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":284 + * + * + * cdef nvvmResult _nvvmCreateProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmCreateProgram + * _check_or_init_nvvm() +*/ + +static nvvmResult __pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmCreateProgram(nvvmProgram *__pyx_v_prog) { + nvvmResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvvmCreateProgram", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":286 + * cdef nvvmResult _nvvmCreateProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvvmCreateProgram + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * if __nvvmCreateProgram == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 286, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":287 + * global __nvvmCreateProgram + * _check_or_init_nvvm() + * if __nvvmCreateProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmCreateProgram is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCreateProgram == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvvm.pyx":288 + * _check_or_init_nvvm() + * if __nvvmCreateProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmCreateProgram is not found") + * return (__nvvmCreateProgram)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":289 + * if __nvvmCreateProgram == NULL: + * with gil: + * raise FunctionNotFoundError("function nvvmCreateProgram is not found") # <<<<<<<<<<<<<< + * return (__nvvmCreateProgram)( + * prog) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 289, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvvmCreateProgram_is_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 289, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 289, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvvm.pyx":288 + * _check_or_init_nvvm() + * if __nvvmCreateProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmCreateProgram is not found") + * return (__nvvmCreateProgram)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":287 + * global __nvvmCreateProgram + * _check_or_init_nvvm() + * if __nvvmCreateProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmCreateProgram is not found") +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":290 + * with gil: + * raise FunctionNotFoundError("function nvvmCreateProgram is not found") + * return (__nvvmCreateProgram)( # <<<<<<<<<<<<<< + * prog) + * +*/ + __pyx_r = ((nvvmResult (*)(nvvmProgram *))__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCreateProgram)(__pyx_v_prog); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":284 + * + * + * cdef nvvmResult _nvvmCreateProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmCreateProgram + * _check_or_init_nvvm() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._nvvmCreateProgram", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVVMRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":294 + * + * + * cdef nvvmResult _nvvmDestroyProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmDestroyProgram + * _check_or_init_nvvm() +*/ + +static nvvmResult __pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmDestroyProgram(nvvmProgram *__pyx_v_prog) { + nvvmResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvvmDestroyProgram", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":296 + * cdef nvvmResult _nvvmDestroyProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvvmDestroyProgram + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * if __nvvmDestroyProgram == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 296, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":297 + * global __nvvmDestroyProgram + * _check_or_init_nvvm() + * if __nvvmDestroyProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmDestroyProgram is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmDestroyProgram == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvvm.pyx":298 + * _check_or_init_nvvm() + * if __nvvmDestroyProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmDestroyProgram is not found") + * return (__nvvmDestroyProgram)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":299 + * if __nvvmDestroyProgram == NULL: + * with gil: + * raise FunctionNotFoundError("function nvvmDestroyProgram is not found") # <<<<<<<<<<<<<< + * return (__nvvmDestroyProgram)( + * prog) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 299, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvvmDestroyProgram_is_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 299, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 299, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvvm.pyx":298 + * _check_or_init_nvvm() + * if __nvvmDestroyProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmDestroyProgram is not found") + * return (__nvvmDestroyProgram)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":297 + * global __nvvmDestroyProgram + * _check_or_init_nvvm() + * if __nvvmDestroyProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmDestroyProgram is not found") +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":300 + * with gil: + * raise FunctionNotFoundError("function nvvmDestroyProgram is not found") + * return (__nvvmDestroyProgram)( # <<<<<<<<<<<<<< + * prog) + * +*/ + __pyx_r = ((nvvmResult (*)(nvvmProgram *))__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmDestroyProgram)(__pyx_v_prog); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":294 + * + * + * cdef nvvmResult _nvvmDestroyProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmDestroyProgram + * _check_or_init_nvvm() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._nvvmDestroyProgram", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVVMRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":304 + * + * + * cdef nvvmResult _nvvmAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmAddModuleToProgram + * _check_or_init_nvvm() +*/ + +static nvvmResult __pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmAddModuleToProgram(nvvmProgram __pyx_v_prog, char const *__pyx_v_buffer, size_t __pyx_v_size, char const *__pyx_v_name) { + nvvmResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvvmAddModuleToProgram", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":306 + * cdef nvvmResult _nvvmAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvvmAddModuleToProgram + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * if __nvvmAddModuleToProgram == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 306, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":307 + * global __nvvmAddModuleToProgram + * _check_or_init_nvvm() + * if __nvvmAddModuleToProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmAddModuleToProgram is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmAddModuleToProgram == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvvm.pyx":308 + * _check_or_init_nvvm() + * if __nvvmAddModuleToProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmAddModuleToProgram is not found") + * return (__nvvmAddModuleToProgram)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":309 + * if __nvvmAddModuleToProgram == NULL: + * with gil: + * raise FunctionNotFoundError("function nvvmAddModuleToProgram is not found") # <<<<<<<<<<<<<< + * return (__nvvmAddModuleToProgram)( + * prog, buffer, size, name) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 309, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvvmAddModuleToProgram}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 309, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 309, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvvm.pyx":308 + * _check_or_init_nvvm() + * if __nvvmAddModuleToProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmAddModuleToProgram is not found") + * return (__nvvmAddModuleToProgram)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":307 + * global __nvvmAddModuleToProgram + * _check_or_init_nvvm() + * if __nvvmAddModuleToProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmAddModuleToProgram is not found") +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":310 + * with gil: + * raise FunctionNotFoundError("function nvvmAddModuleToProgram is not found") + * return (__nvvmAddModuleToProgram)( # <<<<<<<<<<<<<< + * prog, buffer, size, name) + * +*/ + __pyx_r = ((nvvmResult (*)(nvvmProgram, char const *, size_t, char const *))__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmAddModuleToProgram)(__pyx_v_prog, __pyx_v_buffer, __pyx_v_size, __pyx_v_name); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":304 + * + * + * cdef nvvmResult _nvvmAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmAddModuleToProgram + * _check_or_init_nvvm() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._nvvmAddModuleToProgram", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVVMRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":314 + * + * + * cdef nvvmResult _nvvmLazyAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmLazyAddModuleToProgram + * _check_or_init_nvvm() +*/ + +static nvvmResult __pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmLazyAddModuleToProgram(nvvmProgram __pyx_v_prog, char const *__pyx_v_buffer, size_t __pyx_v_size, char const *__pyx_v_name) { + nvvmResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvvmLazyAddModuleToProgram", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":316 + * cdef nvvmResult _nvvmLazyAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvvmLazyAddModuleToProgram + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * if __nvvmLazyAddModuleToProgram == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 316, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":317 + * global __nvvmLazyAddModuleToProgram + * _check_or_init_nvvm() + * if __nvvmLazyAddModuleToProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmLazyAddModuleToProgram is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmLazyAddModuleToProgram == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvvm.pyx":318 + * _check_or_init_nvvm() + * if __nvvmLazyAddModuleToProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmLazyAddModuleToProgram is not found") + * return (__nvvmLazyAddModuleToProgram)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":319 + * if __nvvmLazyAddModuleToProgram == NULL: + * with gil: + * raise FunctionNotFoundError("function nvvmLazyAddModuleToProgram is not found") # <<<<<<<<<<<<<< + * return (__nvvmLazyAddModuleToProgram)( + * prog, buffer, size, name) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 319, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvvmLazyAddModuleToProg}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 319, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 319, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvvm.pyx":318 + * _check_or_init_nvvm() + * if __nvvmLazyAddModuleToProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmLazyAddModuleToProgram is not found") + * return (__nvvmLazyAddModuleToProgram)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":317 + * global __nvvmLazyAddModuleToProgram + * _check_or_init_nvvm() + * if __nvvmLazyAddModuleToProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmLazyAddModuleToProgram is not found") +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":320 + * with gil: + * raise FunctionNotFoundError("function nvvmLazyAddModuleToProgram is not found") + * return (__nvvmLazyAddModuleToProgram)( # <<<<<<<<<<<<<< + * prog, buffer, size, name) + * +*/ + __pyx_r = ((nvvmResult (*)(nvvmProgram, char const *, size_t, char const *))__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmLazyAddModuleToProgram)(__pyx_v_prog, __pyx_v_buffer, __pyx_v_size, __pyx_v_name); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":314 + * + * + * cdef nvvmResult _nvvmLazyAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmLazyAddModuleToProgram + * _check_or_init_nvvm() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._nvvmLazyAddModuleToProgram", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVVMRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":324 + * + * + * cdef nvvmResult _nvvmCompileProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmCompileProgram + * _check_or_init_nvvm() +*/ + +static nvvmResult __pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmCompileProgram(nvvmProgram __pyx_v_prog, int __pyx_v_numOptions, char const **__pyx_v_options) { + nvvmResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvvmCompileProgram", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":326 + * cdef nvvmResult _nvvmCompileProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvvmCompileProgram + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * if __nvvmCompileProgram == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 326, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":327 + * global __nvvmCompileProgram + * _check_or_init_nvvm() + * if __nvvmCompileProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmCompileProgram is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCompileProgram == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvvm.pyx":328 + * _check_or_init_nvvm() + * if __nvvmCompileProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmCompileProgram is not found") + * return (__nvvmCompileProgram)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":329 + * if __nvvmCompileProgram == NULL: + * with gil: + * raise FunctionNotFoundError("function nvvmCompileProgram is not found") # <<<<<<<<<<<<<< + * return (__nvvmCompileProgram)( + * prog, numOptions, options) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 329, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvvmCompileProgram_is_n}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 329, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 329, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvvm.pyx":328 + * _check_or_init_nvvm() + * if __nvvmCompileProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmCompileProgram is not found") + * return (__nvvmCompileProgram)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":327 + * global __nvvmCompileProgram + * _check_or_init_nvvm() + * if __nvvmCompileProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmCompileProgram is not found") +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":330 + * with gil: + * raise FunctionNotFoundError("function nvvmCompileProgram is not found") + * return (__nvvmCompileProgram)( # <<<<<<<<<<<<<< + * prog, numOptions, options) + * +*/ + __pyx_r = ((nvvmResult (*)(nvvmProgram, int, char const **))__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCompileProgram)(__pyx_v_prog, __pyx_v_numOptions, __pyx_v_options); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":324 + * + * + * cdef nvvmResult _nvvmCompileProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmCompileProgram + * _check_or_init_nvvm() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._nvvmCompileProgram", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVVMRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":334 + * + * + * cdef nvvmResult _nvvmVerifyProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmVerifyProgram + * _check_or_init_nvvm() +*/ + +static nvvmResult __pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmVerifyProgram(nvvmProgram __pyx_v_prog, int __pyx_v_numOptions, char const **__pyx_v_options) { + nvvmResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvvmVerifyProgram", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":336 + * cdef nvvmResult _nvvmVerifyProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvvmVerifyProgram + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * if __nvvmVerifyProgram == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 336, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":337 + * global __nvvmVerifyProgram + * _check_or_init_nvvm() + * if __nvvmVerifyProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmVerifyProgram is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVerifyProgram == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvvm.pyx":338 + * _check_or_init_nvvm() + * if __nvvmVerifyProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmVerifyProgram is not found") + * return (__nvvmVerifyProgram)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":339 + * if __nvvmVerifyProgram == NULL: + * with gil: + * raise FunctionNotFoundError("function nvvmVerifyProgram is not found") # <<<<<<<<<<<<<< + * return (__nvvmVerifyProgram)( + * prog, numOptions, options) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 339, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvvmVerifyProgram_is_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 339, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 339, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvvm.pyx":338 + * _check_or_init_nvvm() + * if __nvvmVerifyProgram == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmVerifyProgram is not found") + * return (__nvvmVerifyProgram)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":337 + * global __nvvmVerifyProgram + * _check_or_init_nvvm() + * if __nvvmVerifyProgram == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmVerifyProgram is not found") +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":340 + * with gil: + * raise FunctionNotFoundError("function nvvmVerifyProgram is not found") + * return (__nvvmVerifyProgram)( # <<<<<<<<<<<<<< + * prog, numOptions, options) + * +*/ + __pyx_r = ((nvvmResult (*)(nvvmProgram, int, char const **))__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVerifyProgram)(__pyx_v_prog, __pyx_v_numOptions, __pyx_v_options); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":334 + * + * + * cdef nvvmResult _nvvmVerifyProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmVerifyProgram + * _check_or_init_nvvm() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._nvvmVerifyProgram", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVVMRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":344 + * + * + * cdef nvvmResult _nvvmGetCompiledResultSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmGetCompiledResultSize + * _check_or_init_nvvm() +*/ + +static nvvmResult __pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmGetCompiledResultSize(nvvmProgram __pyx_v_prog, size_t *__pyx_v_bufferSizeRet) { + nvvmResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvvmGetCompiledResultSize", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":346 + * cdef nvvmResult _nvvmGetCompiledResultSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvvmGetCompiledResultSize + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * if __nvvmGetCompiledResultSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 346, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":347 + * global __nvvmGetCompiledResultSize + * _check_or_init_nvvm() + * if __nvvmGetCompiledResultSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmGetCompiledResultSize is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResultSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvvm.pyx":348 + * _check_or_init_nvvm() + * if __nvvmGetCompiledResultSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmGetCompiledResultSize is not found") + * return (__nvvmGetCompiledResultSize)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":349 + * if __nvvmGetCompiledResultSize == NULL: + * with gil: + * raise FunctionNotFoundError("function nvvmGetCompiledResultSize is not found") # <<<<<<<<<<<<<< + * return (__nvvmGetCompiledResultSize)( + * prog, bufferSizeRet) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 349, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvvmGetCompiledResultSi}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 349, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 349, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvvm.pyx":348 + * _check_or_init_nvvm() + * if __nvvmGetCompiledResultSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmGetCompiledResultSize is not found") + * return (__nvvmGetCompiledResultSize)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":347 + * global __nvvmGetCompiledResultSize + * _check_or_init_nvvm() + * if __nvvmGetCompiledResultSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmGetCompiledResultSize is not found") +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":350 + * with gil: + * raise FunctionNotFoundError("function nvvmGetCompiledResultSize is not found") + * return (__nvvmGetCompiledResultSize)( # <<<<<<<<<<<<<< + * prog, bufferSizeRet) + * +*/ + __pyx_r = ((nvvmResult (*)(nvvmProgram, size_t *))__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResultSize)(__pyx_v_prog, __pyx_v_bufferSizeRet); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":344 + * + * + * cdef nvvmResult _nvvmGetCompiledResultSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmGetCompiledResultSize + * _check_or_init_nvvm() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._nvvmGetCompiledResultSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVVMRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":354 + * + * + * cdef nvvmResult _nvvmGetCompiledResult(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmGetCompiledResult + * _check_or_init_nvvm() +*/ + +static nvvmResult __pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmGetCompiledResult(nvvmProgram __pyx_v_prog, char *__pyx_v_buffer) { + nvvmResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvvmGetCompiledResult", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":356 + * cdef nvvmResult _nvvmGetCompiledResult(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvvmGetCompiledResult + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * if __nvvmGetCompiledResult == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 356, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":357 + * global __nvvmGetCompiledResult + * _check_or_init_nvvm() + * if __nvvmGetCompiledResult == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmGetCompiledResult is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResult == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvvm.pyx":358 + * _check_or_init_nvvm() + * if __nvvmGetCompiledResult == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmGetCompiledResult is not found") + * return (__nvvmGetCompiledResult)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":359 + * if __nvvmGetCompiledResult == NULL: + * with gil: + * raise FunctionNotFoundError("function nvvmGetCompiledResult is not found") # <<<<<<<<<<<<<< + * return (__nvvmGetCompiledResult)( + * prog, buffer) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 359, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvvmGetCompiledResult_i}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 359, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 359, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvvm.pyx":358 + * _check_or_init_nvvm() + * if __nvvmGetCompiledResult == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmGetCompiledResult is not found") + * return (__nvvmGetCompiledResult)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":357 + * global __nvvmGetCompiledResult + * _check_or_init_nvvm() + * if __nvvmGetCompiledResult == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmGetCompiledResult is not found") +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":360 + * with gil: + * raise FunctionNotFoundError("function nvvmGetCompiledResult is not found") + * return (__nvvmGetCompiledResult)( # <<<<<<<<<<<<<< + * prog, buffer) + * +*/ + __pyx_r = ((nvvmResult (*)(nvvmProgram, char *))__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResult)(__pyx_v_prog, __pyx_v_buffer); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":354 + * + * + * cdef nvvmResult _nvvmGetCompiledResult(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmGetCompiledResult + * _check_or_init_nvvm() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._nvvmGetCompiledResult", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVVMRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":364 + * + * + * cdef nvvmResult _nvvmGetProgramLogSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmGetProgramLogSize + * _check_or_init_nvvm() +*/ + +static nvvmResult __pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmGetProgramLogSize(nvvmProgram __pyx_v_prog, size_t *__pyx_v_bufferSizeRet) { + nvvmResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvvmGetProgramLogSize", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":366 + * cdef nvvmResult _nvvmGetProgramLogSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvvmGetProgramLogSize + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * if __nvvmGetProgramLogSize == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 366, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":367 + * global __nvvmGetProgramLogSize + * _check_or_init_nvvm() + * if __nvvmGetProgramLogSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmGetProgramLogSize is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLogSize == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvvm.pyx":368 + * _check_or_init_nvvm() + * if __nvvmGetProgramLogSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmGetProgramLogSize is not found") + * return (__nvvmGetProgramLogSize)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":369 + * if __nvvmGetProgramLogSize == NULL: + * with gil: + * raise FunctionNotFoundError("function nvvmGetProgramLogSize is not found") # <<<<<<<<<<<<<< + * return (__nvvmGetProgramLogSize)( + * prog, bufferSizeRet) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 369, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvvmGetProgramLogSize_i}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 369, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 369, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvvm.pyx":368 + * _check_or_init_nvvm() + * if __nvvmGetProgramLogSize == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmGetProgramLogSize is not found") + * return (__nvvmGetProgramLogSize)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":367 + * global __nvvmGetProgramLogSize + * _check_or_init_nvvm() + * if __nvvmGetProgramLogSize == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmGetProgramLogSize is not found") +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":370 + * with gil: + * raise FunctionNotFoundError("function nvvmGetProgramLogSize is not found") + * return (__nvvmGetProgramLogSize)( # <<<<<<<<<<<<<< + * prog, bufferSizeRet) + * +*/ + __pyx_r = ((nvvmResult (*)(nvvmProgram, size_t *))__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLogSize)(__pyx_v_prog, __pyx_v_bufferSizeRet); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":364 + * + * + * cdef nvvmResult _nvvmGetProgramLogSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmGetProgramLogSize + * _check_or_init_nvvm() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._nvvmGetProgramLogSize", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVVMRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} + +/* "cuda/bindings/_internal/nvvm.pyx":374 + * + * + * cdef nvvmResult _nvvmGetProgramLog(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmGetProgramLog + * _check_or_init_nvvm() +*/ + +static nvvmResult __pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmGetProgramLog(nvvmProgram __pyx_v_prog, char *__pyx_v_buffer) { + nvvmResult __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + PyGILState_STATE __pyx_gilstate_save; + __Pyx_RefNannySetupContext("_nvvmGetProgramLog", 1); + + /* "cuda/bindings/_internal/nvvm.pyx":376 + * cdef nvvmResult _nvvmGetProgramLog(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + * global __nvvmGetProgramLog + * _check_or_init_nvvm() # <<<<<<<<<<<<<< + * if __nvvmGetProgramLog == NULL: + * with gil: +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_4nvvm__check_or_init_nvvm(); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(0, 376, __pyx_L1_error) + + /* "cuda/bindings/_internal/nvvm.pyx":377 + * global __nvvmGetProgramLog + * _check_or_init_nvvm() + * if __nvvmGetProgramLog == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmGetProgramLog is not found") +*/ + __pyx_t_2 = (__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLog == NULL); + if (unlikely(__pyx_t_2)) { + + /* "cuda/bindings/_internal/nvvm.pyx":378 + * _check_or_init_nvvm() + * if __nvvmGetProgramLog == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmGetProgramLog is not found") + * return (__nvvmGetProgramLog)( +*/ + { + PyGILState_STATE __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + /*try:*/ { + + /* "cuda/bindings/_internal/nvvm.pyx":379 + * if __nvvmGetProgramLog == NULL: + * with gil: + * raise FunctionNotFoundError("function nvvmGetProgramLog is not found") # <<<<<<<<<<<<<< + * return (__nvvmGetProgramLog)( + * prog, buffer) +*/ + __pyx_t_4 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_5, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 379, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_5); + __pyx_t_6 = 1; + #if CYTHON_UNPACK_METHODS + if (unlikely(PyMethod_Check(__pyx_t_5))) { + __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_5); + assert(__pyx_t_4); + PyObject* __pyx__function = PyMethod_GET_FUNCTION(__pyx_t_5); + __Pyx_INCREF(__pyx_t_4); + __Pyx_INCREF(__pyx__function); + __Pyx_DECREF_SET(__pyx_t_5, __pyx__function); + __pyx_t_6 = 0; + } + #endif + { + PyObject *__pyx_callargs[2] = {__pyx_t_4, __pyx_mstate_global->__pyx_kp_u_function_nvvmGetProgramLog_is_no}; + __pyx_t_3 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (2-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 379, __pyx_L5_error) + __Pyx_GOTREF(__pyx_t_3); + } + __Pyx_Raise(__pyx_t_3, 0, 0, 0); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __PYX_ERR(0, 379, __pyx_L5_error) + } + + /* "cuda/bindings/_internal/nvvm.pyx":378 + * _check_or_init_nvvm() + * if __nvvmGetProgramLog == NULL: + * with gil: # <<<<<<<<<<<<<< + * raise FunctionNotFoundError("function nvvmGetProgramLog is not found") + * return (__nvvmGetProgramLog)( +*/ + /*finally:*/ { + __pyx_L5_error: { + __Pyx_PyGILState_Release(__pyx_gilstate_save); + goto __pyx_L1_error; + } + } + } + + /* "cuda/bindings/_internal/nvvm.pyx":377 + * global __nvvmGetProgramLog + * _check_or_init_nvvm() + * if __nvvmGetProgramLog == NULL: # <<<<<<<<<<<<<< + * with gil: + * raise FunctionNotFoundError("function nvvmGetProgramLog is not found") +*/ + } + + /* "cuda/bindings/_internal/nvvm.pyx":380 + * with gil: + * raise FunctionNotFoundError("function nvvmGetProgramLog is not found") + * return (__nvvmGetProgramLog)( # <<<<<<<<<<<<<< + * prog, buffer) +*/ + __pyx_r = ((nvvmResult (*)(nvvmProgram, char *))__pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLog)(__pyx_v_prog, __pyx_v_buffer); + goto __pyx_L0; + + /* "cuda/bindings/_internal/nvvm.pyx":374 + * + * + * cdef nvvmResult _nvvmGetProgramLog(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: # <<<<<<<<<<<<<< + * global __nvvmGetProgramLog + * _check_or_init_nvvm() +*/ + + /* function exit code */ + __pyx_L1_error:; + __pyx_gilstate_save = __Pyx_PyGILState_Ensure(); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.nvvm._nvvmGetProgramLog", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = _NVVMRESULT_INTERNAL_LOADING_ERROR; + __Pyx_PyGILState_Release(__pyx_gilstate_save); + __pyx_L0:; + __Pyx_RefNannyFinishContextNogil() + return __pyx_r; +} +/* #### Code section: module_exttypes ### */ + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___symbol_lock = Py_None; Py_INCREF(Py_None); + __pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs = ((PyObject*)Py_None); Py_INCREF(Py_None); + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + if (__Pyx_ExportFunction("_nvvmGetErrorString", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmGetErrorString, "char const *(nvvmResult)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvvmVersion", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmVersion, "nvvmResult (int *, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvvmIRVersion", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmIRVersion, "nvvmResult (int *, int *, int *, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvvmCreateProgram", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmCreateProgram, "nvvmResult (nvvmProgram *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvvmDestroyProgram", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmDestroyProgram, "nvvmResult (nvvmProgram *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvvmAddModuleToProgram", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmAddModuleToProgram, "nvvmResult (nvvmProgram, char const *, size_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvvmLazyAddModuleToProgram", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmLazyAddModuleToProgram, "nvvmResult (nvvmProgram, char const *, size_t, char const *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvvmCompileProgram", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmCompileProgram, "nvvmResult (nvvmProgram, int, char const **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvvmVerifyProgram", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmVerifyProgram, "nvvmResult (nvvmProgram, int, char const **)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvvmGetCompiledResultSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmGetCompiledResultSize, "nvvmResult (nvvmProgram, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvvmGetCompiledResult", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmGetCompiledResult, "nvvmResult (nvvmProgram, char *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvvmGetProgramLogSize", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmGetProgramLogSize, "nvvmResult (nvvmProgram, size_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("_nvvmGetProgramLog", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_4nvvm__nvvmGetProgramLog, "nvvmResult (nvvmProgram, char *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_nvvm(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_nvvm}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "nvvm", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_nvvm(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_nvvm(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x03090000 + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_nvvm(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + size_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'nvvm' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "nvvm" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_NOT_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY +__Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); +if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); +} +#endif + +__Pyx_RefNannySetupContext("PyInit_nvvm", 0); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #ifdef __Pxy_PyFrame_Initialize_Offsets + __Pxy_PyFrame_Initialize_Offsets(); + #endif + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #if 0 || defined(__Pyx_CyFunction_USED) || defined(__Pyx_FusedFunction_USED) || defined(__Pyx_Coroutine_USED) || defined(__Pyx_Generator_USED) || defined(__Pyx_AsyncGen_USED) + if (__pyx_CommonTypesMetaclass_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_AsyncGen_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + if (__pyx_module_is_main_cuda__bindings___internal__nvvm) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name_2, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "cuda.bindings._internal.nvvm")) { + if (unlikely((PyDict_SetItemString(modules, "cuda.bindings._internal.nvvm", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_function_export_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_type_init_code(__pyx_mstate); + (void)__Pyx_modinit_type_import_code(__pyx_mstate); + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + (void)__Pyx_modinit_function_import_code(__pyx_mstate); + /*--- Execution code ---*/ + + /* "cuda/bindings/_internal/nvvm.pyx":9 + * from libc.stdint cimport intptr_t, uintptr_t + * + * import threading # <<<<<<<<<<<<<< + * from .utils import FunctionNotFoundError, NotSupportedError + * +*/ + __pyx_t_2 = __Pyx_ImportDottedModule(__pyx_mstate_global->__pyx_n_u_threading, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_threading, __pyx_t_2) < (0)) __PYX_ERR(0, 9, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":10 + * + * import threading + * from .utils import FunctionNotFoundError, NotSupportedError # <<<<<<<<<<<<<< + * + * from cuda.pathfinder import load_nvidia_dynamic_lib +*/ + __pyx_t_2 = __Pyx_PyList_Pack(2, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError, __pyx_mstate_global->__pyx_n_u_NotSupportedError); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_utils, __pyx_t_2, 1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError, __pyx_t_2) < (0)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __pyx_t_2 = __Pyx_ImportFrom(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_NotSupportedError); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_NotSupportedError, __pyx_t_2) < (0)) __PYX_ERR(0, 10, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":12 + * from .utils import FunctionNotFoundError, NotSupportedError + * + * from cuda.pathfinder import load_nvidia_dynamic_lib # <<<<<<<<<<<<<< + * + * +*/ + __pyx_t_3 = __Pyx_PyList_Pack(1, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_2 = __Pyx_Import(__pyx_mstate_global->__pyx_n_u_cuda_pathfinder, __pyx_t_3, 0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __pyx_t_3 = __Pyx_ImportFrom(__pyx_t_2, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_load_nvidia_dynamic_lib, __pyx_t_3) < (0)) __PYX_ERR(0, 12, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":56 + * ############################################################################### + * + * cdef object __symbol_lock = threading.Lock() # <<<<<<<<<<<<<< + * cdef bint __py_nvvm_init = False + * cdef void* __cuDriverGetVersion = NULL +*/ + __pyx_t_3 = NULL; + __Pyx_GetModuleGlobalName(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_threading); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 56, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + __pyx_t_5 = __Pyx_PyObject_GetAttrStr(__pyx_t_4, __pyx_mstate_global->__pyx_n_u_Lock); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 56, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __pyx_t_6 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_3, NULL}; + __pyx_t_2 = __Pyx_PyObject_FastCall(__pyx_t_5, __pyx_callargs+__pyx_t_6, (1-__pyx_t_6) | (__pyx_t_6*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 56, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + } + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_internal_4nvvm___symbol_lock); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_internal_4nvvm___symbol_lock, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_2); + __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":57 + * + * cdef object __symbol_lock = threading.Lock() + * cdef bint __py_nvvm_init = False # <<<<<<<<<<<<<< + * cdef void* __cuDriverGetVersion = NULL + * +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___py_nvvm_init = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":58 + * cdef object __symbol_lock = threading.Lock() + * cdef bint __py_nvvm_init = False + * cdef void* __cuDriverGetVersion = NULL # <<<<<<<<<<<<<< + * + * cdef void* __nvvmGetErrorString = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___cuDriverGetVersion = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":60 + * cdef void* __cuDriverGetVersion = NULL + * + * cdef void* __nvvmGetErrorString = NULL # <<<<<<<<<<<<<< + * cdef void* __nvvmVersion = NULL + * cdef void* __nvvmIRVersion = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetErrorString = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":61 + * + * cdef void* __nvvmGetErrorString = NULL + * cdef void* __nvvmVersion = NULL # <<<<<<<<<<<<<< + * cdef void* __nvvmIRVersion = NULL + * cdef void* __nvvmCreateProgram = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVersion = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":62 + * cdef void* __nvvmGetErrorString = NULL + * cdef void* __nvvmVersion = NULL + * cdef void* __nvvmIRVersion = NULL # <<<<<<<<<<<<<< + * cdef void* __nvvmCreateProgram = NULL + * cdef void* __nvvmDestroyProgram = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmIRVersion = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":63 + * cdef void* __nvvmVersion = NULL + * cdef void* __nvvmIRVersion = NULL + * cdef void* __nvvmCreateProgram = NULL # <<<<<<<<<<<<<< + * cdef void* __nvvmDestroyProgram = NULL + * cdef void* __nvvmAddModuleToProgram = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCreateProgram = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":64 + * cdef void* __nvvmIRVersion = NULL + * cdef void* __nvvmCreateProgram = NULL + * cdef void* __nvvmDestroyProgram = NULL # <<<<<<<<<<<<<< + * cdef void* __nvvmAddModuleToProgram = NULL + * cdef void* __nvvmLazyAddModuleToProgram = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmDestroyProgram = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":65 + * cdef void* __nvvmCreateProgram = NULL + * cdef void* __nvvmDestroyProgram = NULL + * cdef void* __nvvmAddModuleToProgram = NULL # <<<<<<<<<<<<<< + * cdef void* __nvvmLazyAddModuleToProgram = NULL + * cdef void* __nvvmCompileProgram = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmAddModuleToProgram = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":66 + * cdef void* __nvvmDestroyProgram = NULL + * cdef void* __nvvmAddModuleToProgram = NULL + * cdef void* __nvvmLazyAddModuleToProgram = NULL # <<<<<<<<<<<<<< + * cdef void* __nvvmCompileProgram = NULL + * cdef void* __nvvmVerifyProgram = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmLazyAddModuleToProgram = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":67 + * cdef void* __nvvmAddModuleToProgram = NULL + * cdef void* __nvvmLazyAddModuleToProgram = NULL + * cdef void* __nvvmCompileProgram = NULL # <<<<<<<<<<<<<< + * cdef void* __nvvmVerifyProgram = NULL + * cdef void* __nvvmGetCompiledResultSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmCompileProgram = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":68 + * cdef void* __nvvmLazyAddModuleToProgram = NULL + * cdef void* __nvvmCompileProgram = NULL + * cdef void* __nvvmVerifyProgram = NULL # <<<<<<<<<<<<<< + * cdef void* __nvvmGetCompiledResultSize = NULL + * cdef void* __nvvmGetCompiledResult = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmVerifyProgram = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":69 + * cdef void* __nvvmCompileProgram = NULL + * cdef void* __nvvmVerifyProgram = NULL + * cdef void* __nvvmGetCompiledResultSize = NULL # <<<<<<<<<<<<<< + * cdef void* __nvvmGetCompiledResult = NULL + * cdef void* __nvvmGetProgramLogSize = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResultSize = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":70 + * cdef void* __nvvmVerifyProgram = NULL + * cdef void* __nvvmGetCompiledResultSize = NULL + * cdef void* __nvvmGetCompiledResult = NULL # <<<<<<<<<<<<<< + * cdef void* __nvvmGetProgramLogSize = NULL + * cdef void* __nvvmGetProgramLog = NULL +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetCompiledResult = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":71 + * cdef void* __nvvmGetCompiledResultSize = NULL + * cdef void* __nvvmGetCompiledResult = NULL + * cdef void* __nvvmGetProgramLogSize = NULL # <<<<<<<<<<<<<< + * cdef void* __nvvmGetProgramLog = NULL + * +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLogSize = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":72 + * cdef void* __nvvmGetCompiledResult = NULL + * cdef void* __nvvmGetProgramLogSize = NULL + * cdef void* __nvvmGetProgramLog = NULL # <<<<<<<<<<<<<< + * + * +*/ + __pyx_v_4cuda_8bindings_9_internal_4nvvm___nvvmGetProgramLog = NULL; + + /* "cuda/bindings/_internal/nvvm.pyx":189 + * + * + * cdef dict func_ptrs = None # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_INCREF(Py_None); + __Pyx_XGOTREF(__pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs); + __Pyx_DECREF_SET(__pyx_v_4cuda_8bindings_9_internal_4nvvm_func_ptrs, ((PyObject*)Py_None)); + __Pyx_GIVEREF(Py_None); + + /* "cuda/bindings/_internal/nvvm.pyx":192 + * + * + * cpdef dict _inspect_function_pointers(): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is not None: +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_4cuda_8bindings_9_internal_4nvvm_1_inspect_function_pointers, 0, __pyx_mstate_global->__pyx_n_u_inspect_function_pointers, NULL, __pyx_mstate_global->__pyx_n_u_cuda_bindings__internal_nvvm, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[0])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 192, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_inspect_function_pointers, __pyx_t_2) < (0)) __PYX_ERR(0, 192, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":243 + * + * + * cpdef _inspect_function_pointer(str name): # <<<<<<<<<<<<<< + * global func_ptrs + * if func_ptrs is None: +*/ + __pyx_t_2 = __Pyx_CyFunction_New(&__pyx_mdef_4cuda_8bindings_9_internal_4nvvm_3_inspect_function_pointer, 0, __pyx_mstate_global->__pyx_n_u_inspect_function_pointer, NULL, __pyx_mstate_global->__pyx_n_u_cuda_bindings__internal_nvvm, __pyx_mstate_global->__pyx_d, ((PyObject *)__pyx_mstate_global->__pyx_codeobj_tab[1])); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 243, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_inspect_function_pointer, __pyx_t_2) < (0)) __PYX_ERR(0, 243, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/nvvm.pyx":1 + * # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # <<<<<<<<<<<<<< + * # + * # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +*/ + __pyx_t_2 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_2) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init cuda.bindings._internal.nvvm", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init cuda.bindings._internal.nvvm"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ + +typedef struct { + const char *s; +#if 179 <= 65535 + const unsigned short n; +#elif 179 / 2 < INT_MAX + const unsigned int n; +#elif 179 / 2 < LONG_MAX + const unsigned long n; +#else + const Py_ssize_t n; +#endif +#if 1 <= 31 + const unsigned int encoding : 5; +#elif 1 <= 255 + const unsigned char encoding; +#elif 1 <= 65535 + const unsigned short encoding; +#else + const Py_ssize_t encoding; +#endif + const unsigned int is_unicode : 1; + const unsigned int intern : 1; +} __Pyx_StringTabEntry; +static const char * const __pyx_string_tab_encodings[] = { 0 }; +static const __Pyx_StringTabEntry __pyx_string_tab[] = { + {__pyx_k_, sizeof(__pyx_k_), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_ */ + {__pyx_k_CUDA_driver_is_not_found, sizeof(__pyx_k_CUDA_driver_is_not_found), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_CUDA_driver_is_not_found */ + {__pyx_k_Did_not_find_cuDriverGetVersion, sizeof(__pyx_k_Did_not_find_cuDriverGetVersion), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Did_not_find_cuDriverGetVersion */ + {__pyx_k_FunctionNotFoundError, sizeof(__pyx_k_FunctionNotFoundError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_FunctionNotFoundError */ + {__pyx_k_Lock, sizeof(__pyx_k_Lock), 0, 1, 1}, /* PyObject cname: __pyx_n_u_Lock */ + {__pyx_k_NotSupportedError, sizeof(__pyx_k_NotSupportedError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_NotSupportedError */ + {__pyx_k_Note_that_Cython_is_deliberately, sizeof(__pyx_k_Note_that_Cython_is_deliberately), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_Note_that_Cython_is_deliberately */ + {__pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_RuntimeError */ + {__pyx_k__2, sizeof(__pyx_k__2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u__2 */ + {__pyx_k__3, sizeof(__pyx_k__3), 0, 1, 0}, /* PyObject cname: __pyx_kp_u__3 */ + {__pyx_k_add_note, sizeof(__pyx_k_add_note), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_add_note */ + {__pyx_k_asyncio_coroutines, sizeof(__pyx_k_asyncio_coroutines), 0, 1, 1}, /* PyObject cname: __pyx_n_u_asyncio_coroutines */ + {__pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cline_in_traceback */ + {__pyx_k_cuDriverGetVersion_returned_erro, sizeof(__pyx_k_cuDriverGetVersion_returned_erro), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_cuDriverGetVersion_returned_erro */ + {__pyx_k_cuda_bindings__internal_nvvm, sizeof(__pyx_k_cuda_bindings__internal_nvvm), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuda_bindings__internal_nvvm */ + {__pyx_k_cuda_bindings__internal_nvvm_pyx, sizeof(__pyx_k_cuda_bindings__internal_nvvm_pyx), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_cuda_bindings__internal_nvvm_pyx */ + {__pyx_k_cuda_pathfinder, sizeof(__pyx_k_cuda_pathfinder), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuda_pathfinder */ + {__pyx_k_enter, sizeof(__pyx_k_enter), 0, 1, 1}, /* PyObject cname: __pyx_n_u_enter */ + {__pyx_k_exit, sizeof(__pyx_k_exit), 0, 1, 1}, /* PyObject cname: __pyx_n_u_exit */ + {__pyx_k_func, sizeof(__pyx_k_func), 0, 1, 1}, /* PyObject cname: __pyx_n_u_func */ + {__pyx_k_function_nvvmAddModuleToProgram, sizeof(__pyx_k_function_nvvmAddModuleToProgram), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvvmAddModuleToProgram */ + {__pyx_k_function_nvvmCompileProgram_is_n, sizeof(__pyx_k_function_nvvmCompileProgram_is_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvvmCompileProgram_is_n */ + {__pyx_k_function_nvvmCreateProgram_is_no, sizeof(__pyx_k_function_nvvmCreateProgram_is_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvvmCreateProgram_is_no */ + {__pyx_k_function_nvvmDestroyProgram_is_n, sizeof(__pyx_k_function_nvvmDestroyProgram_is_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvvmDestroyProgram_is_n */ + {__pyx_k_function_nvvmGetCompiledResultSi, sizeof(__pyx_k_function_nvvmGetCompiledResultSi), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvvmGetCompiledResultSi */ + {__pyx_k_function_nvvmGetCompiledResult_i, sizeof(__pyx_k_function_nvvmGetCompiledResult_i), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvvmGetCompiledResult_i */ + {__pyx_k_function_nvvmGetErrorString_is_n, sizeof(__pyx_k_function_nvvmGetErrorString_is_n), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvvmGetErrorString_is_n */ + {__pyx_k_function_nvvmGetProgramLogSize_i, sizeof(__pyx_k_function_nvvmGetProgramLogSize_i), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvvmGetProgramLogSize_i */ + {__pyx_k_function_nvvmGetProgramLog_is_no, sizeof(__pyx_k_function_nvvmGetProgramLog_is_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvvmGetProgramLog_is_no */ + {__pyx_k_function_nvvmIRVersion_is_not_fo, sizeof(__pyx_k_function_nvvmIRVersion_is_not_fo), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvvmIRVersion_is_not_fo */ + {__pyx_k_function_nvvmLazyAddModuleToProg, sizeof(__pyx_k_function_nvvmLazyAddModuleToProg), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvvmLazyAddModuleToProg */ + {__pyx_k_function_nvvmVerifyProgram_is_no, sizeof(__pyx_k_function_nvvmVerifyProgram_is_no), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvvmVerifyProgram_is_no */ + {__pyx_k_function_nvvmVersion_is_not_foun, sizeof(__pyx_k_function_nvvmVersion_is_not_foun), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_function_nvvmVersion_is_not_foun */ + {__pyx_k_handle_uint, sizeof(__pyx_k_handle_uint), 0, 1, 1}, /* PyObject cname: __pyx_n_u_handle_uint */ + {__pyx_k_initializing, sizeof(__pyx_k_initializing), 0, 1, 1}, /* PyObject cname: __pyx_n_u_initializing */ + {__pyx_k_inspect_function_pointer, sizeof(__pyx_k_inspect_function_pointer), 0, 1, 1}, /* PyObject cname: __pyx_n_u_inspect_function_pointer */ + {__pyx_k_inspect_function_pointers, sizeof(__pyx_k_inspect_function_pointers), 0, 1, 1}, /* PyObject cname: __pyx_n_u_inspect_function_pointers */ + {__pyx_k_is_coroutine, sizeof(__pyx_k_is_coroutine), 0, 1, 1}, /* PyObject cname: __pyx_n_u_is_coroutine */ + {__pyx_k_load_nvidia_dynamic_lib, sizeof(__pyx_k_load_nvidia_dynamic_lib), 0, 1, 1}, /* PyObject cname: __pyx_n_u_load_nvidia_dynamic_lib */ + {__pyx_k_main, sizeof(__pyx_k_main), 0, 1, 1}, /* PyObject cname: __pyx_n_u_main */ + {__pyx_k_module, sizeof(__pyx_k_module), 0, 1, 1}, /* PyObject cname: __pyx_n_u_module */ + {__pyx_k_name, sizeof(__pyx_k_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name */ + {__pyx_k_name_2, sizeof(__pyx_k_name_2), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name_2 */ + {__pyx_k_nvvm, sizeof(__pyx_k_nvvm), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvm */ + {__pyx_k_nvvmAddModuleToProgram, sizeof(__pyx_k_nvvmAddModuleToProgram), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvmAddModuleToProgram */ + {__pyx_k_nvvmCompileProgram, sizeof(__pyx_k_nvvmCompileProgram), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvmCompileProgram */ + {__pyx_k_nvvmCreateProgram, sizeof(__pyx_k_nvvmCreateProgram), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvmCreateProgram */ + {__pyx_k_nvvmDestroyProgram, sizeof(__pyx_k_nvvmDestroyProgram), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvmDestroyProgram */ + {__pyx_k_nvvmGetCompiledResult, sizeof(__pyx_k_nvvmGetCompiledResult), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvmGetCompiledResult */ + {__pyx_k_nvvmGetCompiledResultSize, sizeof(__pyx_k_nvvmGetCompiledResultSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvmGetCompiledResultSize */ + {__pyx_k_nvvmGetErrorString, sizeof(__pyx_k_nvvmGetErrorString), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvmGetErrorString */ + {__pyx_k_nvvmGetProgramLog, sizeof(__pyx_k_nvvmGetProgramLog), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvmGetProgramLog */ + {__pyx_k_nvvmGetProgramLogSize, sizeof(__pyx_k_nvvmGetProgramLogSize), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvmGetProgramLogSize */ + {__pyx_k_nvvmIRVersion, sizeof(__pyx_k_nvvmIRVersion), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvmIRVersion */ + {__pyx_k_nvvmLazyAddModuleToProgram, sizeof(__pyx_k_nvvmLazyAddModuleToProgram), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvmLazyAddModuleToProgram */ + {__pyx_k_nvvmVerifyProgram, sizeof(__pyx_k_nvvmVerifyProgram), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvmVerifyProgram */ + {__pyx_k_nvvmVersion, sizeof(__pyx_k_nvvmVersion), 0, 1, 1}, /* PyObject cname: __pyx_n_u_nvvmVersion */ + {__pyx_k_pop, sizeof(__pyx_k_pop), 0, 1, 1}, /* PyObject cname: __pyx_n_u_pop */ + {__pyx_k_qualname, sizeof(__pyx_k_qualname), 0, 1, 1}, /* PyObject cname: __pyx_n_u_qualname */ + {__pyx_k_set_name, sizeof(__pyx_k_set_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_set_name */ + {__pyx_k_spec, sizeof(__pyx_k_spec), 0, 1, 1}, /* PyObject cname: __pyx_n_u_spec */ + {__pyx_k_test, sizeof(__pyx_k_test), 0, 1, 1}, /* PyObject cname: __pyx_n_u_test */ + {__pyx_k_threading, sizeof(__pyx_k_threading), 0, 1, 1}, /* PyObject cname: __pyx_n_u_threading */ + {__pyx_k_utils, sizeof(__pyx_k_utils), 0, 1, 1}, /* PyObject cname: __pyx_n_u_utils */ + {0, 0, 0, 0, 0} +}; +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names); + +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(0, 44, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "cuda/bindings/_internal/nvvm.pyx":85 + * cdef void* handle = NULL + * + * with gil, __symbol_lock: # <<<<<<<<<<<<<< + * # Load function + * global __nvvmGetErrorString +*/ + __pyx_mstate_global->__pyx_tuple[0] = PyTuple_Pack(3, Py_None, Py_None, Py_None); if (unlikely(!__pyx_mstate_global->__pyx_tuple[0])) __PYX_ERR(0, 85, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.type = (PyObject*)&PyDict_Type; + __pyx_mstate->__pyx_umethod_PyDict_Type_pop.method_name = &__pyx_mstate->__pyx_n_u_pop; + if (__Pyx_InitStrings(__pyx_string_tab, __pyx_mstate->__pyx_string_tab, __pyx_string_tab_encodings) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ +\ + typedef struct { + unsigned int argcount : 1; + unsigned int num_posonly_args : 1; + unsigned int num_kwonly_args : 1; + unsigned int nlocals : 1; + unsigned int flags : 10; + unsigned int first_line : 8; + unsigned int line_table_length : 13; + } __Pyx_PyCode_New_function_description; +/* NewCodeObj.proto */ +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + const char *line_table, + PyObject *tuple_dedup_map +); + + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + PyObject* tuple_dedup_map = PyDict_New(); + if (unlikely(!tuple_dedup_map)) return -1; + { + const __Pyx_PyCode_New_function_description descr = {0, 0, 0, 0, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 192, 234}; + PyObject* const varnames[] = {0}; + __pyx_mstate_global->__pyx_codeobj_tab[0] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_cuda_bindings__internal_nvvm_pyx, __pyx_mstate->__pyx_n_u_inspect_function_pointers, __pyx_k_z_q_q_Q_Q_Ja_j_A_Q_z_Q_Q_A_A_j, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[0])) goto bad; + } + { + const __Pyx_PyCode_New_function_description descr = {1, 0, 0, 1, (unsigned int)(CO_OPTIMIZED|CO_NEWLOCALS), 243, 31}; + PyObject* const varnames[] = {__pyx_mstate->__pyx_n_u_name}; + __pyx_mstate_global->__pyx_codeobj_tab[1] = __Pyx_PyCode_New(descr, varnames, __pyx_mstate->__pyx_kp_u_cuda_bindings__internal_nvvm_pyx, __pyx_mstate->__pyx_n_u_inspect_function_pointer, __pyx_k_z_A_a_9AQ, tuple_dedup_map); if (unlikely(!__pyx_mstate_global->__pyx_codeobj_tab[1])) goto bad; + } + Py_DECREF(tuple_dedup_map); + return 0; + bad: + Py_DECREF(tuple_dedup_map); + return -1; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* CachedMethodType.init */ + #if CYTHON_COMPILING_IN_LIMITED_API +{ + PyObject *typesModule=NULL; + typesModule = PyImport_ImportModule("types"); + if (typesModule) { + __pyx_mstate_global->__Pyx_CachedMethodType = PyObject_GetAttrString(typesModule, "MethodType"); + Py_DECREF(typesModule); + } +} // error handling follows +#endif + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* PyErrExceptionMatches */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStr */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* PyObjectGetAttrStrNoError */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name); + if (unlikely(!result) && !PyErr_Occurred()) { + PyErr_Format(PyExc_NameError, + "name '%U' is not defined", name); + } + return result; +} + +/* PyDictVersioning */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* GetModuleGlobalName */ +#if CYTHON_USE_DICT_VERSIONS +static PyObject *__Pyx__GetModuleGlobalName(PyObject *name, PY_UINT64_T *dict_version, PyObject **dict_cached_value) +#else +static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) +#endif +{ + PyObject *result; +#if CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!__pyx_m)) { + if (!PyErr_Occurred()) + PyErr_SetNone(PyExc_NameError); + return NULL; + } + result = PyObject_GetAttr(__pyx_m, name); + if (likely(result)) { + return result; + } + PyErr_Clear(); +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + if (unlikely(__Pyx_PyDict_GetItemRef(__pyx_mstate_global->__pyx_d, name, &result) == -1)) PyErr_Clear(); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return result; + } +#else + result = _PyDict_GetItem_KnownHash(__pyx_mstate_global->__pyx_d, name, ((PyASCIIObject *) name)->hash); + __PYX_UPDATE_DICT_CACHE(__pyx_mstate_global->__pyx_d, result, *dict_cached_value, *dict_version) + if (likely(result)) { + return __Pyx_NewRef(result); + } + PyErr_Clear(); +#endif + return __Pyx_GetBuiltinName(name); +} + +/* decode_c_string */ +static CYTHON_INLINE PyObject* __Pyx_decode_c_string( + const char* cstring, Py_ssize_t start, Py_ssize_t stop, + const char* encoding, const char* errors, + PyObject* (*decode_func)(const char *s, Py_ssize_t size, const char *errors)) { + Py_ssize_t length; + if (unlikely((start < 0) | (stop < 0))) { + size_t slen = strlen(cstring); + if (unlikely(slen > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, + "c-string too long to convert to Python"); + return NULL; + } + length = (Py_ssize_t) slen; + if (start < 0) { + start += length; + if (start < 0) + start = 0; + } + if (stop < 0) + stop += length; + } + if (unlikely(stop <= start)) + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_unicode); + length = stop - start; + cstring += start; + if (decode_func) { + return decode_func(cstring, length, errors); + } else { + return PyUnicode_Decode(cstring, length, encoding, errors); + } +} + +/* JoinPyUnicode */ +static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength, + Py_UCS4 max_char) { +#if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + PyObject *result_uval; + int result_ukind, kind_shift; + Py_ssize_t i, char_pos; + void *result_udata; + if (max_char > 1114111) max_char = 1114111; + result_uval = PyUnicode_New(result_ulength, max_char); + if (unlikely(!result_uval)) return NULL; + result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; + kind_shift = (result_ukind == PyUnicode_4BYTE_KIND) ? 2 : result_ukind - 1; + result_udata = PyUnicode_DATA(result_uval); + assert(kind_shift == 2 || kind_shift == 1 || kind_shift == 0); + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - result_ulength < 0)) + goto overflow; + char_pos = 0; + for (i=0; i < value_count; i++) { + int ukind; + Py_ssize_t ulength; + void *udata; + PyObject *uval = values[i]; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (__Pyx_PyUnicode_READY(uval) == (-1)) + goto bad; + #endif + ulength = __Pyx_PyUnicode_GET_LENGTH(uval); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(ulength < 0)) goto bad; + #endif + if (unlikely(!ulength)) + continue; + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - ulength < char_pos)) + goto overflow; + ukind = __Pyx_PyUnicode_KIND(uval); + udata = __Pyx_PyUnicode_DATA(uval); + if (ukind == result_ukind) { + memcpy((char *)result_udata + (char_pos << kind_shift), udata, (size_t) (ulength << kind_shift)); + } else { + #if PY_VERSION_HEX >= 0x030d0000 + if (unlikely(PyUnicode_CopyCharacters(result_uval, char_pos, uval, 0, ulength) < 0)) goto bad; + #elif CYTHON_COMPILING_IN_CPYTHON || defined(_PyUnicode_FastCopyCharacters) + _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); + #else + Py_ssize_t j; + for (j=0; j < ulength; j++) { + Py_UCS4 uchar = __Pyx_PyUnicode_READ(ukind, udata, j); + __Pyx_PyUnicode_WRITE(result_ukind, result_udata, char_pos+j, uchar); + } + #endif + } + char_pos += ulength; + } + return result_uval; +overflow: + PyErr_SetString(PyExc_OverflowError, "join() result is too long for a Python string"); +bad: + Py_DECREF(result_uval); + return NULL; +#else + Py_ssize_t i; + PyObject *result = NULL; + PyObject *value_tuple = PyTuple_New(value_count); + if (unlikely(!value_tuple)) return NULL; + CYTHON_UNUSED_VAR(max_char); + CYTHON_UNUSED_VAR(result_ulength); + for (i=0; i__pyx_empty_unicode, value_tuple); +bad: + Py_DECREF(value_tuple); + return result; +#endif +} + +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject *const *args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = __Pyx_PyFrame_GetLocalsplus(f); + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; + PyObject *kwdefs; + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) { + return NULL; + } + if ( + co->co_kwonlyargcount == 0 && + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, (int)nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif + +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = Py_TYPE(func)->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + #if PY_VERSION_HEX < 0x030800B1 + #if CYTHON_FAST_PYCCALL + if (PyCFunction_Check(func)) { + if (kwargs) { + return _PyCFunction_FastCallDict(func, args, nargs, kwargs); + } else { + return _PyCFunction_FastCallKeywords(func, args, nargs, NULL); + } + } + if (!kwargs && __Pyx_IS_TYPE(func, &PyMethodDescr_Type)) { + return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL); + } + #endif + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs); + } + #endif + #endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #elif defined(__Pyx_CyFunction_USED) && CYTHON_BACKPORT_VECTORCALL + if (__Pyx_CyFunction_CheckExact(func)) { + __pyx_vectorcallfunc f = __Pyx_CyFunction_func_vectorcall(func); + if (f) return f(func, args, _nargs, NULL); + } + #elif CYTHON_COMPILING_IN_LIMITED_API && CYTHON_VECTORCALL + return PyObject_Vectorcall(func, args, _nargs, NULL); + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* RaiseException */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyException_SetTraceback(value, tb); +#elif CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} + +/* CIntToDigits */ +static const char DIGIT_PAIRS_10[2*10*10+1] = { + "00010203040506070809" + "10111213141516171819" + "20212223242526272829" + "30313233343536373839" + "40414243444546474849" + "50515253545556575859" + "60616263646566676869" + "70717273747576777879" + "80818283848586878889" + "90919293949596979899" +}; +static const char DIGIT_PAIRS_8[2*8*8+1] = { + "0001020304050607" + "1011121314151617" + "2021222324252627" + "3031323334353637" + "4041424344454647" + "5051525354555657" + "6061626364656667" + "7071727374757677" +}; +static const char DIGITS_HEX[2*16+1] = { + "0123456789abcdef" + "0123456789ABCDEF" +}; + +/* BuildPyUnicode */ +static PyObject* __Pyx_PyUnicode_BuildFromAscii(Py_ssize_t ulength, const char* chars, int clength, + int prepend_sign, char padding_char) { + PyObject *uval; + Py_ssize_t uoffset = ulength - clength; +#if CYTHON_USE_UNICODE_INTERNALS + Py_ssize_t i; + void *udata; + uval = PyUnicode_New(ulength, 127); + if (unlikely(!uval)) return NULL; + udata = PyUnicode_DATA(uval); + if (uoffset > 0) { + i = 0; + if (prepend_sign) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, 0, '-'); + i++; + } + for (; i < uoffset; i++) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, i, padding_char); + } + } + for (i=0; i < clength; i++) { + __Pyx_PyUnicode_WRITE(PyUnicode_1BYTE_KIND, udata, uoffset+i, chars[i]); + } +#else + { + PyObject *sign = NULL, *padding = NULL; + uval = NULL; + if (uoffset > 0) { + prepend_sign = !!prepend_sign; + if (uoffset > prepend_sign) { + padding = PyUnicode_FromOrdinal(padding_char); + if (likely(padding) && uoffset > prepend_sign + 1) { + PyObject *tmp = PySequence_Repeat(padding, uoffset - prepend_sign); + Py_DECREF(padding); + padding = tmp; + } + if (unlikely(!padding)) goto done_or_error; + } + if (prepend_sign) { + sign = PyUnicode_FromOrdinal('-'); + if (unlikely(!sign)) goto done_or_error; + } + } + uval = PyUnicode_DecodeASCII(chars, clength, NULL); + if (likely(uval) && padding) { + PyObject *tmp = PyUnicode_Concat(padding, uval); + Py_DECREF(uval); + uval = tmp; + } + if (likely(uval) && sign) { + PyObject *tmp = PyUnicode_Concat(sign, uval); + Py_DECREF(uval); + uval = tmp; + } +done_or_error: + Py_XDECREF(padding); + Py_XDECREF(sign); + } +#endif + return uval; +} + +/* COrdinalToPyUnicode */ +static CYTHON_INLINE int __Pyx_CheckUnicodeValue(int value) { + return value <= 1114111; +} +static PyObject* __Pyx_PyUnicode_FromOrdinal_Padded(int value, Py_ssize_t ulength, char padding_char) { + if (likely(ulength <= 250)) { + char chars[256]; + if (value <= 255) { + memset(chars, padding_char, (size_t) (ulength - 1)); + chars[ulength-1] = (char) value; + return PyUnicode_DecodeLatin1(chars, ulength, NULL); + } + char *cpos = chars + sizeof(chars); + if (value < 0x800) { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xc0 | (value & 0x1f)); + } else if (value < 0x10000) { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xe0 | (value & 0x0f)); + } else { + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0x80 | (value & 0x3f)); + value >>= 6; + *--cpos = (char) (0xf0 | (value & 0x07)); + } + cpos -= ulength; + memset(cpos, padding_char, (size_t) (ulength - 1)); + return PyUnicode_DecodeUTF8(cpos, chars + sizeof(chars) - cpos, NULL); + } + if (value <= 127 && CYTHON_USE_UNICODE_INTERNALS) { + const char chars[1] = {(char) value}; + return __Pyx_PyUnicode_BuildFromAscii(ulength, chars, 1, 0, padding_char); + } + { + PyObject *uchar, *padding_uchar, *padding, *result; + padding_uchar = PyUnicode_FromOrdinal(padding_char); + if (unlikely(!padding_uchar)) return NULL; + padding = PySequence_Repeat(padding_uchar, ulength - 1); + Py_DECREF(padding_uchar); + if (unlikely(!padding)) return NULL; + uchar = PyUnicode_FromOrdinal(value); + if (unlikely(!uchar)) { + Py_DECREF(padding); + return NULL; + } + result = PyUnicode_Concat(padding, uchar); + Py_DECREF(padding); + Py_DECREF(uchar); + return result; + } +} + +/* CIntToPyUnicode */ +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_From_int(int value, Py_ssize_t width, char padding_char, char format_char) { + char digits[sizeof(int)*3+2]; + char *dpos, *end = digits + sizeof(int)*3+2; + const char *hex_digits = DIGITS_HEX; + Py_ssize_t length, ulength; + int prepend_sign, last_one_off; + int remaining; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (format_char == 'c') { + if (unlikely(!(is_unsigned || value == 0 || value > 0) || + !(sizeof(value) <= 2 || value & ~ (int) 0x01fffff || __Pyx_CheckUnicodeValue((int) value)))) { + PyErr_SetString(PyExc_OverflowError, "%c arg not in range(0x110000)"); + return NULL; + } + if (width <= 1) { + return PyUnicode_FromOrdinal((int) value); + } + return __Pyx_PyUnicode_FromOrdinal_Padded((int) value, width, padding_char); + } + if (format_char == 'X') { + hex_digits += 16; + format_char = 'x'; + } + remaining = value; + last_one_off = 0; + dpos = end; + do { + int digit_pos; + switch (format_char) { + case 'o': + digit_pos = abs((int)(remaining % (8*8))); + remaining = (int) (remaining / (8*8)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_8 + digit_pos * 2, 2); + last_one_off = (digit_pos < 8); + break; + case 'd': + digit_pos = abs((int)(remaining % (10*10))); + remaining = (int) (remaining / (10*10)); + dpos -= 2; + memcpy(dpos, DIGIT_PAIRS_10 + digit_pos * 2, 2); + last_one_off = (digit_pos < 10); + break; + case 'x': + *(--dpos) = hex_digits[abs((int)(remaining % 16))]; + remaining = (int) (remaining / 16); + break; + default: + assert(0); + break; + } + } while (unlikely(remaining != 0)); + assert(!last_one_off || *dpos == '0'); + dpos += last_one_off; + length = end - dpos; + ulength = length; + prepend_sign = 0; + if (!is_unsigned && value <= neg_one) { + if (padding_char == ' ' || width <= length + 1) { + *(--dpos) = '-'; + ++length; + } else { + prepend_sign = 1; + } + ++ulength; + } + if (width > ulength) { + ulength = width; + } + if (ulength == 1) { + return PyUnicode_FromOrdinal(*dpos); + } + return __Pyx_PyUnicode_BuildFromAscii(ulength, dpos, (int) length, prepend_sign, padding_char); +} + +/* PyObjectLookupSpecial */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) { + PyObject *res; + PyTypeObject *tp = Py_TYPE(obj); + res = _PyType_Lookup(tp, attr_name); + if (likely(res)) { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (!f) { + Py_INCREF(res); + } else { + res = f(res, obj, (PyObject *)tp); + } + } else if (with_error) { + PyErr_SetObject(PyExc_AttributeError, attr_name); + } + return res; +} +#endif + +/* GetTopmostException */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + +/* SaveResetException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #endif +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); + #else + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); + #endif +} +#endif + +/* GetException */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) +#endif +{ + PyObject *local_type = NULL, *local_value, *local_tb = NULL; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030C0000 + local_value = tstate->current_exception; + tstate->current_exception = 0; + #else + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + #endif +#elif __PYX_LIMITED_VERSION_HEX > 0x030C0000 + local_value = PyErr_GetRaisedException(); +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif +#if __PYX_LIMITED_VERSION_HEX > 0x030C0000 + if (likely(local_value)) { + local_type = (PyObject*) Py_TYPE(local_value); + Py_INCREF(local_type); + local_tb = PyException_GetTraceback(local_value); + } +#else + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } +#endif // __PYX_LIMITED_VERSION_HEX > 0x030C0000 + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if CYTHON_USE_EXC_INFO_STACK + { + _PyErr_StackItem *exc_info = tstate->exc_info; + #if PY_VERSION_HEX >= 0x030B00a4 + tmp_value = exc_info->exc_value; + exc_info->exc_value = local_value; + tmp_type = NULL; + tmp_tb = NULL; + Py_XDECREF(local_type); + Py_XDECREF(local_tb); + #else + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; + #endif + } + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#elif __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + PyErr_SetHandledException(local_value); + Py_XDECREF(local_value); + Py_XDECREF(local_type); + Py_XDECREF(local_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +#if __PYX_LIMITED_VERSION_HEX <= 0x030C0000 +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +#endif +} + +/* TupleAndListFromArray */ +#if !CYTHON_COMPILING_IN_CPYTHON && CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + Py_ssize_t i; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + for (i = 0; i < n; i++) { + if (unlikely(__Pyx_PyTuple_SET_ITEM(res, i, src[i]) < (0))) { + Py_DECREF(res); + return NULL; + } + Py_INCREF(src[i]); + } + return res; +} +#elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE void __Pyx_copy_object_array(PyObject *const *CYTHON_RESTRICT src, PyObject** CYTHON_RESTRICT dest, Py_ssize_t length) { + PyObject *v; + Py_ssize_t i; + for (i = 0; i < length; i++) { + v = dest[i] = src[i]; + Py_INCREF(v); + } +} +static CYTHON_INLINE PyObject * +__Pyx_PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return __Pyx_NewRef(__pyx_mstate_global->__pyx_empty_tuple); + } + res = PyTuple_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyTupleObject*)res)->ob_item, n); + return res; +} +static CYTHON_INLINE PyObject * +__Pyx_PyList_FromArray(PyObject *const *src, Py_ssize_t n) +{ + PyObject *res; + if (n <= 0) { + return PyList_New(0); + } + res = PyList_New(n); + if (unlikely(res == NULL)) return NULL; + __Pyx_copy_object_array(src, ((PyListObject*)res)->ob_item, n); + return res; +} +#endif + +/* BytesEquals */ +static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL ||\ + !(CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) + return PyObject_RichCompareBool(s1, s2, equals); +#else + if (s1 == s2) { + return (equals == Py_EQ); + } else if (PyBytes_CheckExact(s1) & PyBytes_CheckExact(s2)) { + const char *ps1, *ps2; + Py_ssize_t length = PyBytes_GET_SIZE(s1); + if (length != PyBytes_GET_SIZE(s2)) + return (equals == Py_NE); + ps1 = PyBytes_AS_STRING(s1); + ps2 = PyBytes_AS_STRING(s2); + if (ps1[0] != ps2[0]) { + return (equals == Py_NE); + } else if (length == 1) { + return (equals == Py_EQ); + } else { + int result; +#if CYTHON_USE_UNICODE_INTERNALS && (PY_VERSION_HEX < 0x030B0000) + Py_hash_t hash1, hash2; + hash1 = ((PyBytesObject*)s1)->ob_shash; + hash2 = ((PyBytesObject*)s2)->ob_shash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + return (equals == Py_NE); + } +#endif + result = memcmp(ps1, ps2, (size_t)length); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & PyBytes_CheckExact(s2)) { + return (equals == Py_NE); + } else if ((s2 == Py_None) & PyBytes_CheckExact(s1)) { + return (equals == Py_NE); + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +#endif +} + +/* UnicodeEquals */ +static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_GRAAL + return PyObject_RichCompareBool(s1, s2, equals); +#else + int s1_is_unicode, s2_is_unicode; + if (s1 == s2) { + goto return_eq; + } + s1_is_unicode = PyUnicode_CheckExact(s1); + s2_is_unicode = PyUnicode_CheckExact(s2); + if (s1_is_unicode & s2_is_unicode) { + Py_ssize_t length, length2; + int kind; + void *data1, *data2; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(__Pyx_PyUnicode_READY(s1) < 0) || unlikely(__Pyx_PyUnicode_READY(s2) < 0)) + return -1; + #endif + length = __Pyx_PyUnicode_GET_LENGTH(s1); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length < 0)) return -1; + #endif + length2 = __Pyx_PyUnicode_GET_LENGTH(s2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(length2 < 0)) return -1; + #endif + if (length != length2) { + goto return_ne; + } +#if CYTHON_USE_UNICODE_INTERNALS + { + Py_hash_t hash1, hash2; + hash1 = ((PyASCIIObject*)s1)->hash; + hash2 = ((PyASCIIObject*)s2)->hash; + if (hash1 != hash2 && hash1 != -1 && hash2 != -1) { + goto return_ne; + } + } +#endif + kind = __Pyx_PyUnicode_KIND(s1); + if (kind != __Pyx_PyUnicode_KIND(s2)) { + goto return_ne; + } + data1 = __Pyx_PyUnicode_DATA(s1); + data2 = __Pyx_PyUnicode_DATA(s2); + if (__Pyx_PyUnicode_READ(kind, data1, 0) != __Pyx_PyUnicode_READ(kind, data2, 0)) { + goto return_ne; + } else if (length == 1) { + goto return_eq; + } else { + int result = memcmp(data1, data2, (size_t)(length * kind)); + return (equals == Py_EQ) ? (result == 0) : (result != 0); + } + } else if ((s1 == Py_None) & s2_is_unicode) { + goto return_ne; + } else if ((s2 == Py_None) & s1_is_unicode) { + goto return_ne; + } else { + int result; + PyObject* py_result = PyObject_RichCompare(s1, s2, equals); + if (!py_result) + return -1; + result = __Pyx_PyObject_IsTrue(py_result); + Py_DECREF(py_result); + return result; + } +return_eq: + return (equals == Py_EQ); +return_ne: + return (equals == Py_NE); +#endif +} + +/* fastcall */ +#if CYTHON_METH_FASTCALL +static CYTHON_INLINE PyObject * __Pyx_GetKwValue_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues, PyObject *s) +{ + Py_ssize_t i, n = __Pyx_PyTuple_GET_SIZE(kwnames); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(n == -1)) return NULL; + #endif + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + if (s == namei) return kwvalues[i]; + } + for (i = 0; i < n; i++) + { + PyObject *namei = __Pyx_PyTuple_GET_ITEM(kwnames, i); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!namei)) return NULL; + #endif + int eq = __Pyx_PyUnicode_Equals(s, namei, Py_EQ); + if (unlikely(eq != 0)) { + if (unlikely(eq < 0)) return NULL; + return kwvalues[i]; + } + } + return NULL; +} +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +CYTHON_UNUSED static PyObject *__Pyx_KwargsAsDict_FASTCALL(PyObject *kwnames, PyObject *const *kwvalues) { + Py_ssize_t i, nkwargs; + PyObject *dict; +#if !CYTHON_ASSUME_SAFE_SIZE + nkwargs = PyTuple_Size(kwnames); + if (unlikely(nkwargs < 0)) return NULL; +#else + nkwargs = PyTuple_GET_SIZE(kwnames); +#endif + dict = PyDict_New(); + if (unlikely(!dict)) + return NULL; + for (i=0; itype, *target->method_name); + if (unlikely(!method)) + return -1; + result = method; +#if CYTHON_COMPILING_IN_CPYTHON + if (likely(__Pyx_TypeCheck(method, &PyMethodDescr_Type))) + { + PyMethodDescrObject *descr = (PyMethodDescrObject*) method; + target->func = descr->d_method->ml_meth; + target->flag = descr->d_method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST | METH_STACKLESS); + } else +#endif +#if CYTHON_COMPILING_IN_PYPY +#else + if (PyCFunction_Check(method)) +#endif + { + PyObject *self; + int self_found; +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + self = PyObject_GetAttrString(method, "__self__"); + if (!self) { + PyErr_Clear(); + } +#else + self = PyCFunction_GET_SELF(method); +#endif + self_found = (self && self != Py_None); +#if CYTHON_COMPILING_IN_LIMITED_API || CYTHON_COMPILING_IN_PYPY + Py_XDECREF(self); +#endif + if (self_found) { + PyObject *unbound_method = PyCFunction_New(&__Pyx_UnboundCMethod_Def, method); + if (unlikely(!unbound_method)) return -1; + Py_DECREF(method); + result = unbound_method; + } + } +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + if (unlikely(target->method)) { + Py_DECREF(result); + } else +#endif + target->method = result; + return 0; +} + +/* CallUnboundCMethod2 */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject *__Pyx_CallUnboundCMethod2(__Pyx_CachedCFunction *cfunc, PyObject *self, PyObject *arg1, PyObject *arg2) { + int was_initialized = __Pyx_CachedCFunction_GetAndSetInitializing(cfunc); + if (likely(was_initialized == 2 && cfunc->func)) { + PyObject *args[2] = {arg1, arg2}; + if (cfunc->flag == METH_FASTCALL) { + return __Pyx_CallCFunctionFast(cfunc, self, args, 2); + } + if (cfunc->flag == (METH_FASTCALL | METH_KEYWORDS)) + return __Pyx_CallCFunctionFastWithKeywords(cfunc, self, args, 2, NULL); + } +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + else if (unlikely(was_initialized == 1)) { + __Pyx_CachedCFunction tmp_cfunc = { +#ifndef __cplusplus + 0 +#endif + }; + tmp_cfunc.type = cfunc->type; + tmp_cfunc.method_name = cfunc->method_name; + return __Pyx__CallUnboundCMethod2(&tmp_cfunc, self, arg1, arg2); + } +#endif + PyObject *result = __Pyx__CallUnboundCMethod2(cfunc, self, arg1, arg2); + __Pyx_CachedCFunction_SetFinishedInitializing(cfunc); + return result; +} +#endif +static PyObject* __Pyx__CallUnboundCMethod2(__Pyx_CachedCFunction* cfunc, PyObject* self, PyObject* arg1, PyObject* arg2){ + if (unlikely(!cfunc->func && !cfunc->method) && unlikely(__Pyx_TryUnpackUnboundCMethod(cfunc) < 0)) return NULL; +#if CYTHON_COMPILING_IN_CPYTHON + if (cfunc->func && (cfunc->flag & METH_VARARGS)) { + PyObject *result = NULL; + PyObject *args = PyTuple_New(2); + if (unlikely(!args)) return NULL; + Py_INCREF(arg1); + PyTuple_SET_ITEM(args, 0, arg1); + Py_INCREF(arg2); + PyTuple_SET_ITEM(args, 1, arg2); + if (cfunc->flag & METH_KEYWORDS) + result = __Pyx_CallCFunctionWithKeywords(cfunc, self, args, NULL); + else + result = __Pyx_CallCFunction(cfunc, self, args); + Py_DECREF(args); + return result; + } +#endif + { + PyObject *args[4] = {NULL, self, arg1, arg2}; + return __Pyx_PyObject_FastCall(cfunc->method, args+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); + } +} + +/* ParseKeywords */ +static int __Pyx_ValidateDuplicatePosArgs( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char* function_name) +{ + PyObject ** const *name = argnames; + while (name != first_kw_arg) { + PyObject *key = **name; + int found = PyDict_Contains(kwds, key); + if (unlikely(found)) { + if (found == 1) __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; + } + name++; + } + return 0; +bad: + return -1; +} +#if CYTHON_USE_UNICODE_INTERNALS +static CYTHON_INLINE int __Pyx_UnicodeKeywordsEqual(PyObject *s1, PyObject *s2) { + int kind; + Py_ssize_t len = PyUnicode_GET_LENGTH(s1); + if (len != PyUnicode_GET_LENGTH(s2)) return 0; + kind = PyUnicode_KIND(s1); + if (kind != PyUnicode_KIND(s2)) return 0; + const void *data1 = PyUnicode_DATA(s1); + const void *data2 = PyUnicode_DATA(s2); + return (memcmp(data1, data2, (size_t) len * (size_t) kind) == 0); +} +#endif +static int __Pyx_MatchKeywordArg_str( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + #if CYTHON_USE_UNICODE_INTERNALS + Py_hash_t key_hash = ((PyASCIIObject*)key)->hash; + if (unlikely(key_hash == -1)) { + key_hash = PyObject_Hash(key); + if (unlikely(key_hash == -1)) + goto bad; + } + #endif + name = first_kw_arg; + while (*name) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (key_hash == ((PyASCIIObject*)name_str)->hash && __Pyx_UnicodeKeywordsEqual(name_str, key)) { + *index_found = (size_t) (name - argnames); + return 1; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) { + *index_found = (size_t) (name - argnames); + return 1; + } + } + #endif + name++; + } + name = argnames; + while (name != first_kw_arg) { + PyObject *name_str = **name; + #if CYTHON_USE_UNICODE_INTERNALS + if (unlikely(key_hash == ((PyASCIIObject*)name_str)->hash)) { + if (__Pyx_UnicodeKeywordsEqual(name_str, key)) + goto arg_passed_twice; + } + #else + #if CYTHON_ASSUME_SAFE_SIZE + if (PyUnicode_GET_LENGTH(name_str) == PyUnicode_GET_LENGTH(key)) + #endif + { + if (unlikely(name_str == key)) goto arg_passed_twice; + int cmp = PyUnicode_Compare(name_str, key); + if (cmp < 0 && unlikely(PyErr_Occurred())) goto bad; + if (cmp == 0) goto arg_passed_twice; + } + #endif + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +bad: + return -1; +} +static int __Pyx_MatchKeywordArg_nostr( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + PyObject ** const *name; + if (unlikely(!PyUnicode_Check(key))) goto invalid_keyword_type; + name = first_kw_arg; + while (*name) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (cmp == 1) { + *index_found = (size_t) (name - argnames); + return 1; + } + if (unlikely(cmp == -1)) goto bad; + name++; + } + name = argnames; + while (name != first_kw_arg) { + int cmp = PyObject_RichCompareBool(**name, key, Py_EQ); + if (unlikely(cmp != 0)) { + if (cmp == 1) goto arg_passed_twice; + else goto bad; + } + name++; + } + return 0; +arg_passed_twice: + __Pyx_RaiseDoubleKeywordsError(function_name, key); + goto bad; +invalid_keyword_type: + PyErr_Format(PyExc_TypeError, + "%.200s() keywords must be strings", function_name); + goto bad; +bad: + return -1; +} +static CYTHON_INLINE int __Pyx_MatchKeywordArg( + PyObject *key, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + size_t *index_found, + const char *function_name) +{ + return likely(PyUnicode_CheckExact(key)) ? + __Pyx_MatchKeywordArg_str(key, argnames, first_kw_arg, index_found, function_name) : + __Pyx_MatchKeywordArg_nostr(key, argnames, first_kw_arg, index_found, function_name); +} +static void __Pyx_RejectUnknownKeyword( + PyObject *kwds, + PyObject ** const argnames[], + PyObject ** const *first_kw_arg, + const char *function_name) +{ + Py_ssize_t pos = 0; + PyObject *key = NULL; + __Pyx_BEGIN_CRITICAL_SECTION(kwds); + while (PyDict_Next(kwds, &pos, &key, NULL)) { + PyObject** const *name = first_kw_arg; + while (*name && (**name != key)) name++; + if (!*name) { + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(key); + #endif + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp != 1) { + if (cmp == 0) { + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + break; + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + #endif + } + } + __Pyx_END_CRITICAL_SECTION(); + assert(PyErr_Occurred()); +} +static int __Pyx_ParseKeywordDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t extracted = 0; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + name = first_kw_arg; + while (*name && num_kwargs > extracted) { + PyObject * key = **name; + PyObject *value; + int found = 0; + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + found = PyDict_GetItemRef(kwds, key, &value); + #else + value = PyDict_GetItemWithError(kwds, key); + if (value) { + Py_INCREF(value); + found = 1; + } else { + if (unlikely(PyErr_Occurred())) goto bad; + } + #endif + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + extracted++; + } + name++; + } + if (num_kwargs > extracted) { + if (ignore_unknown_kwargs) { + if (unlikely(__Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name) == -1)) + goto bad; + } else { + __Pyx_RejectUnknownKeyword(kwds, argnames, first_kw_arg, function_name); + goto bad; + } + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordDictToDict( + PyObject *kwds, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + const char* function_name) +{ + PyObject** const *name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + Py_ssize_t len; +#if !CYTHON_COMPILING_IN_PYPY || defined(PyArg_ValidateKeywordArguments) + if (unlikely(!PyArg_ValidateKeywordArguments(kwds))) return -1; +#endif + if (PyDict_Update(kwds2, kwds) < 0) goto bad; + name = first_kw_arg; + while (*name) { + PyObject *key = **name; + PyObject *value; +#if !CYTHON_COMPILING_IN_LIMITED_API && (PY_VERSION_HEX >= 0x030d00A2 || defined(PyDict_Pop)) + int found = PyDict_Pop(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + } +#elif __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + int found = PyDict_GetItemRef(kwds2, key, &value); + if (found) { + if (unlikely(found < 0)) goto bad; + values[name-argnames] = value; + if (unlikely(PyDict_DelItem(kwds2, key) < 0)) goto bad; + } +#else + #if CYTHON_COMPILING_IN_CPYTHON + value = _PyDict_Pop(kwds2, key, kwds2); + #else + value = __Pyx_CallUnboundCMethod2(&__pyx_mstate_global->__pyx_umethod_PyDict_Type_pop, kwds2, key, kwds2); + #endif + if (value == kwds2) { + Py_DECREF(value); + } else { + if (unlikely(!value)) goto bad; + values[name-argnames] = value; + } +#endif + name++; + } + len = PyDict_Size(kwds2); + if (len > 0) { + return __Pyx_ValidateDuplicatePosArgs(kwds, argnames, first_kw_arg, function_name); + } else if (unlikely(len == -1)) { + goto bad; + } + return 0; +bad: + return -1; +} +static int __Pyx_ParseKeywordsTuple( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + PyObject *key = NULL; + PyObject** const * name; + PyObject** const *first_kw_arg = argnames + num_pos_args; + for (Py_ssize_t pos = 0; pos < num_kwargs; pos++) { +#if CYTHON_AVOID_BORROWED_REFS + key = __Pyx_PySequence_ITEM(kwds, pos); +#else + key = __Pyx_PyTuple_GET_ITEM(kwds, pos); +#endif +#if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(!key)) goto bad; +#endif + name = first_kw_arg; + while (*name && (**name != key)) name++; + if (*name) { + PyObject *value = kwvalues[pos]; + values[name-argnames] = __Pyx_NewRef(value); + } else { + size_t index_found = 0; + int cmp = __Pyx_MatchKeywordArg(key, argnames, first_kw_arg, &index_found, function_name); + if (cmp == 1) { + PyObject *value = kwvalues[pos]; + values[index_found] = __Pyx_NewRef(value); + } else { + if (unlikely(cmp == -1)) goto bad; + if (kwds2) { + PyObject *value = kwvalues[pos]; + if (unlikely(PyDict_SetItem(kwds2, key, value))) goto bad; + } else if (!ignore_unknown_kwargs) { + goto invalid_keyword; + } + } + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(key); + key = NULL; + #endif + } + return 0; +invalid_keyword: + PyErr_Format(PyExc_TypeError, + "%s() got an unexpected keyword argument '%U'", + function_name, key); + goto bad; +bad: + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(key); + #endif + return -1; +} +static int __Pyx_ParseKeywords( + PyObject *kwds, + PyObject * const *kwvalues, + PyObject ** const argnames[], + PyObject *kwds2, + PyObject *values[], + Py_ssize_t num_pos_args, + Py_ssize_t num_kwargs, + const char* function_name, + int ignore_unknown_kwargs) +{ + if (CYTHON_METH_FASTCALL && likely(PyTuple_Check(kwds))) + return __Pyx_ParseKeywordsTuple(kwds, kwvalues, argnames, kwds2, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); + else if (kwds2) + return __Pyx_ParseKeywordDictToDict(kwds, argnames, kwds2, values, num_pos_args, function_name); + else + return __Pyx_ParseKeywordDict(kwds, argnames, values, num_pos_args, num_kwargs, function_name, ignore_unknown_kwargs); +} + +/* RaiseArgTupleInvalid */ +static void __Pyx_RaiseArgtupleInvalid( + const char* func_name, + int exact, + Py_ssize_t num_min, + Py_ssize_t num_max, + Py_ssize_t num_found) +{ + Py_ssize_t num_expected; + const char *more_or_less; + if (num_found < num_min) { + num_expected = num_min; + more_or_less = "at least"; + } else { + num_expected = num_max; + more_or_less = "at most"; + } + if (exact) { + more_or_less = "exactly"; + } + PyErr_Format(PyExc_TypeError, + "%.200s() takes %.8s %" CYTHON_FORMAT_SSIZE_T "d positional argument%.1s (%" CYTHON_FORMAT_SSIZE_T "d given)", + func_name, more_or_less, num_expected, + (num_expected == 1) ? "" : "s", num_found); +} + +/* ArgTypeTest */ +static int __Pyx__ArgTypeTest(PyObject *obj, PyTypeObject *type, const char *name, int exact) +{ + __Pyx_TypeName type_name; + __Pyx_TypeName obj_type_name; + PyObject *extra_info = __pyx_mstate_global->__pyx_empty_unicode; + int from_annotation_subclass = 0; + if (unlikely(!type)) { + PyErr_SetString(PyExc_SystemError, "Missing type object"); + return 0; + } + else if (!exact) { + if (likely(__Pyx_TypeCheck(obj, type))) return 1; + } else if (exact == 2) { + if (__Pyx_TypeCheck(obj, type)) { + from_annotation_subclass = 1; + extra_info = __pyx_mstate_global->__pyx_kp_u_Note_that_Cython_is_deliberately; + } + } + type_name = __Pyx_PyType_GetFullyQualifiedName(type); + obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, + "Argument '%.200s' has incorrect type (expected " __Pyx_FMT_TYPENAME + ", got " __Pyx_FMT_TYPENAME ")" +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + "%s%U" +#endif + , name, type_name, obj_type_name +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 + , (from_annotation_subclass ? ". " : ""), extra_info +#endif + ); +#if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + if (exact == 2 && from_annotation_subclass) { + PyObject *res; + PyObject *vargs[2]; + vargs[0] = PyErr_GetRaisedException(); + vargs[1] = extra_info; + res = PyObject_VectorcallMethod(__pyx_mstate_global->__pyx_kp_u_add_note, vargs, 2, NULL); + Py_XDECREF(res); + PyErr_SetRaisedException(vargs[0]); + } +#endif + __Pyx_DECREF_TypeName(type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* Import */ +static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { + PyObject *module = 0; + PyObject *empty_dict = 0; + PyObject *empty_list = 0; + empty_dict = PyDict_New(); + if (unlikely(!empty_dict)) + goto bad; + if (level == -1) { + const char* package_sep = strchr(__Pyx_MODULE_NAME, '.'); + if (package_sep != (0)) { + module = PyImport_ImportModuleLevelObject( + name, __pyx_mstate_global->__pyx_d, empty_dict, from_list, 1); + if (unlikely(!module)) { + if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError))) + goto bad; + PyErr_Clear(); + } + } + level = 0; + } + if (!module) { + module = PyImport_ImportModuleLevelObject( + name, __pyx_mstate_global->__pyx_d, empty_dict, from_list, level); + } +bad: + Py_XDECREF(empty_dict); + Py_XDECREF(empty_list); + return module; +} + +/* ImportDottedModule */ +static PyObject *__Pyx__ImportDottedModule_Error(PyObject *name, PyObject *parts_tuple, Py_ssize_t count) { + PyObject *partial_name = NULL, *slice = NULL, *sep = NULL; + Py_ssize_t size; + if (unlikely(PyErr_Occurred())) { + PyErr_Clear(); + } +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(parts_tuple); +#else + size = PyTuple_Size(parts_tuple); + if (size < 0) goto bad; +#endif + if (likely(size == count)) { + partial_name = name; + } else { + slice = PySequence_GetSlice(parts_tuple, 0, count); + if (unlikely(!slice)) + goto bad; + sep = PyUnicode_FromStringAndSize(".", 1); + if (unlikely(!sep)) + goto bad; + partial_name = PyUnicode_Join(sep, slice); + } + PyErr_Format( + PyExc_ModuleNotFoundError, + "No module named '%U'", partial_name); +bad: + Py_XDECREF(sep); + Py_XDECREF(slice); + Py_XDECREF(partial_name); + return NULL; +} +static PyObject *__Pyx__ImportDottedModule_Lookup(PyObject *name) { + PyObject *imported_module; +#if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + return NULL; + imported_module = __Pyx_PyDict_GetItemStr(modules, name); + Py_XINCREF(imported_module); +#else + imported_module = PyImport_GetModule(name); +#endif + return imported_module; +} +static PyObject *__Pyx_ImportDottedModule_WalkParts(PyObject *module, PyObject *name, PyObject *parts_tuple) { + Py_ssize_t i, nparts; +#if CYTHON_ASSUME_SAFE_SIZE + nparts = PyTuple_GET_SIZE(parts_tuple); +#else + nparts = PyTuple_Size(parts_tuple); + if (nparts < 0) return NULL; +#endif + for (i=1; i < nparts && module; i++) { + PyObject *part, *submodule; +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + part = PyTuple_GET_ITEM(parts_tuple, i); +#else + part = __Pyx_PySequence_ITEM(parts_tuple, i); + if (!part) return NULL; +#endif + submodule = __Pyx_PyObject_GetAttrStrNoError(module, part); +#if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(part); +#endif + Py_DECREF(module); + module = submodule; + } + if (unlikely(!module)) { + return __Pyx__ImportDottedModule_Error(name, parts_tuple, i); + } + return module; +} +static PyObject *__Pyx__ImportDottedModule(PyObject *name, PyObject *parts_tuple) { + PyObject *imported_module; + PyObject *module = __Pyx_Import(name, NULL, 0); + if (!parts_tuple || unlikely(!module)) + return module; + imported_module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(imported_module)) { + Py_DECREF(module); + return imported_module; + } + PyErr_Clear(); + return __Pyx_ImportDottedModule_WalkParts(module, name, parts_tuple); +} +static PyObject *__Pyx_ImportDottedModule(PyObject *name, PyObject *parts_tuple) { +#if CYTHON_COMPILING_IN_CPYTHON + PyObject *module = __Pyx__ImportDottedModule_Lookup(name); + if (likely(module)) { + PyObject *spec = __Pyx_PyObject_GetAttrStrNoError(module, __pyx_mstate_global->__pyx_n_u_spec); + if (likely(spec)) { + PyObject *unsafe = __Pyx_PyObject_GetAttrStrNoError(spec, __pyx_mstate_global->__pyx_n_u_initializing); + if (likely(!unsafe || !__Pyx_PyObject_IsTrue(unsafe))) { + Py_DECREF(spec); + spec = NULL; + } + Py_XDECREF(unsafe); + } + if (likely(!spec)) { + PyErr_Clear(); + return module; + } + Py_DECREF(spec); + Py_DECREF(module); + } else if (PyErr_Occurred()) { + PyErr_Clear(); + } +#endif + return __Pyx__ImportDottedModule(name, parts_tuple); +} + +/* ListPack */ +static PyObject *__Pyx_PyList_Pack(Py_ssize_t n, ...) { + va_list va; + PyObject *l = PyList_New(n); + va_start(va, n); + if (unlikely(!l)) goto end; + for (Py_ssize_t i=0; i__pyx_kp_u__2); + if (unlikely(!module_dot)) { goto modbad; } + full_name = PyUnicode_Concat(module_dot, name); + if (unlikely(!full_name)) { goto modbad; } + #if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030400) ||\ + CYTHON_COMPILING_IN_GRAAL + { + PyObject *modules = PyImport_GetModuleDict(); + if (unlikely(!modules)) + goto modbad; + value = PyObject_GetItem(modules, full_name); + } + #else + value = PyImport_GetModule(full_name); + #endif + modbad: + Py_XDECREF(full_name); + Py_XDECREF(module_dot); + Py_XDECREF(module_name); + } + if (unlikely(!value)) { + PyErr_Format(PyExc_ImportError, "cannot import name %S", name); + } + return value; +} + +/* LimitedApiGetTypeDict */ +#if CYTHON_COMPILING_IN_LIMITED_API +static Py_ssize_t __Pyx_GetTypeDictOffset(void) { + PyObject *tp_dictoffset_o; + Py_ssize_t tp_dictoffset; + tp_dictoffset_o = PyObject_GetAttrString((PyObject*)(&PyType_Type), "__dictoffset__"); + if (unlikely(!tp_dictoffset_o)) return -1; + tp_dictoffset = PyLong_AsSsize_t(tp_dictoffset_o); + Py_DECREF(tp_dictoffset_o); + if (unlikely(tp_dictoffset == 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' doesn't have a dictoffset"); + return -1; + } else if (unlikely(tp_dictoffset < 0)) { + PyErr_SetString( + PyExc_TypeError, + "'type' has an unexpected negative dictoffset. " + "Please report this as Cython bug"); + return -1; + } + return tp_dictoffset; +} +static PyObject *__Pyx_GetTypeDict(PyTypeObject *tp) { + static Py_ssize_t tp_dictoffset = 0; + if (unlikely(tp_dictoffset == 0)) { + tp_dictoffset = __Pyx_GetTypeDictOffset(); + if (unlikely(tp_dictoffset == -1 && PyErr_Occurred())) { + tp_dictoffset = 0; // try again next time? + return NULL; + } + } + return *(PyObject**)((char*)tp + tp_dictoffset); +} +#endif + +/* SetItemOnTypeDict */ +static int __Pyx__SetItemOnTypeDict(PyTypeObject *tp, PyObject *k, PyObject *v) { + int result; + PyObject *tp_dict; +#if CYTHON_COMPILING_IN_LIMITED_API + tp_dict = __Pyx_GetTypeDict(tp); + if (unlikely(!tp_dict)) return -1; +#else + tp_dict = tp->tp_dict; +#endif + result = PyDict_SetItem(tp_dict, k, v); + if (likely(!result)) { + PyType_Modified(tp); + if (unlikely(PyObject_HasAttr(v, __pyx_mstate_global->__pyx_n_u_set_name))) { + PyObject *setNameResult = PyObject_CallMethodObjArgs(v, __pyx_mstate_global->__pyx_n_u_set_name, (PyObject *) tp, k, NULL); + if (!setNameResult) return -1; + Py_DECREF(setNameResult); + } + } + return result; +} + +/* FixUpExtensionType */ +static int __Pyx_fix_up_extension_type_from_spec(PyType_Spec *spec, PyTypeObject *type) { +#if __PYX_LIMITED_VERSION_HEX > 0x030900B1 + CYTHON_UNUSED_VAR(spec); + CYTHON_UNUSED_VAR(type); + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); +#else + const PyType_Slot *slot = spec->slots; + int changed = 0; +#if !CYTHON_COMPILING_IN_LIMITED_API + while (slot && slot->slot && slot->slot != Py_tp_members) + slot++; + if (slot && slot->slot == Py_tp_members) { +#if !CYTHON_COMPILING_IN_CPYTHON + const +#endif // !CYTHON_COMPILING_IN_CPYTHON) + PyMemberDef *memb = (PyMemberDef*) slot->pfunc; + while (memb && memb->name) { + if (memb->name[0] == '_' && memb->name[1] == '_') { + if (strcmp(memb->name, "__weaklistoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_weaklistoffset = memb->offset; + changed = 1; + } + else if (strcmp(memb->name, "__dictoffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); + type->tp_dictoffset = memb->offset; + changed = 1; + } +#if CYTHON_METH_FASTCALL + else if (strcmp(memb->name, "__vectorcalloffset__") == 0) { + assert(memb->type == T_PYSSIZET); + assert(memb->flags == READONLY); +#if PY_VERSION_HEX >= 0x030800b4 + type->tp_vectorcall_offset = memb->offset; +#else + type->tp_print = (printfunc) memb->offset; +#endif + changed = 1; + } +#endif // CYTHON_METH_FASTCALL +#if !CYTHON_COMPILING_IN_PYPY + else if (strcmp(memb->name, "__module__") == 0) { + PyObject *descr; + assert(memb->type == T_OBJECT); + assert(memb->flags == 0 || memb->flags == READONLY); + descr = PyDescr_NewMember(type, memb); + if (unlikely(!descr)) + return -1; + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } +#endif // !CYTHON_COMPILING_IN_PYPY + } + memb++; + } + } +#endif // !CYTHON_COMPILING_IN_LIMITED_API +#if !CYTHON_COMPILING_IN_PYPY + slot = spec->slots; + while (slot && slot->slot && slot->slot != Py_tp_getset) + slot++; + if (slot && slot->slot == Py_tp_getset) { + PyGetSetDef *getset = (PyGetSetDef*) slot->pfunc; + while (getset && getset->name) { + if (getset->name[0] == '_' && getset->name[1] == '_' && strcmp(getset->name, "__module__") == 0) { + PyObject *descr = PyDescr_NewGetSet(type, getset); + if (unlikely(!descr)) + return -1; + #if CYTHON_COMPILING_IN_LIMITED_API + PyObject *pyname = PyUnicode_FromString(getset->name); + if (unlikely(!pyname)) { + Py_DECREF(descr); + return -1; + } + int set_item_result = __Pyx_SetItemOnTypeDict(type, pyname, descr); + Py_DECREF(pyname); + #else + CYTHON_UNUSED_VAR(__Pyx__SetItemOnTypeDict); + int set_item_result = PyDict_SetItem(type->tp_dict, PyDescr_NAME(descr), descr); + #endif + Py_DECREF(descr); + if (unlikely(set_item_result < 0)) { + return -1; + } + changed = 1; + } + ++getset; + } + } +#endif // !CYTHON_COMPILING_IN_PYPY + if (changed) + PyType_Modified(type); +#endif // PY_VERSION_HEX > 0x030900B1 + return 0; +} + +/* FetchSharedCythonModule */ +static PyObject *__Pyx_FetchSharedCythonABIModule(void) { + return __Pyx_PyImport_AddModuleRef(__PYX_ABI_MODULE_NAME); +} + +/* dict_setdefault */ +static CYTHON_INLINE PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value, + int is_safe_type) { + PyObject* value; + CYTHON_MAYBE_UNUSED_VAR(is_safe_type); +#if CYTHON_COMPILING_IN_LIMITED_API + value = PyObject_CallMethod(d, "setdefault", "OO", key, default_value); +#elif PY_VERSION_HEX >= 0x030d0000 + PyDict_SetDefaultRef(d, key, default_value, &value); +#else + value = PyDict_SetDefault(d, key, default_value); + if (unlikely(!value)) return NULL; + Py_INCREF(value); +#endif + return value; +} + +/* FetchCommonType */ +#if __PYX_LIMITED_VERSION_HEX < 0x030C0000 +static PyObject* __Pyx_PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *result = __Pyx_PyType_FromModuleAndSpec(module, spec, bases); + if (result && metaclass) { + PyObject *old_tp = (PyObject*)Py_TYPE(result); + Py_INCREF((PyObject*)metaclass); +#if __PYX_LIMITED_VERSION_HEX >= 0x03090000 + Py_SET_TYPE(result, metaclass); +#else + result->ob_type = metaclass; +#endif + Py_DECREF(old_tp); + } + return result; +} +#else +#define __Pyx_PyType_FromMetaclass(me, mo, s, b) PyType_FromMetaclass(me, mo, s, b) +#endif +static int __Pyx_VerifyCachedType(PyObject *cached_type, + const char *name, + Py_ssize_t expected_basicsize) { + Py_ssize_t basicsize; + if (!PyType_Check(cached_type)) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s is not a type object", name); + return -1; + } + if (expected_basicsize == 0) { + return 0; // size is inherited, nothing useful to check + } +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_basicsize; + py_basicsize = PyObject_GetAttrString(cached_type, "__basicsize__"); + if (unlikely(!py_basicsize)) return -1; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = NULL; + if (unlikely(basicsize == (Py_ssize_t)-1) && PyErr_Occurred()) return -1; +#else + basicsize = ((PyTypeObject*) cached_type)->tp_basicsize; +#endif + if (basicsize != expected_basicsize) { + PyErr_Format(PyExc_TypeError, + "Shared Cython type %.200s has the wrong size, try recompiling", + name); + return -1; + } + return 0; +} +static PyTypeObject *__Pyx_FetchCommonTypeFromSpec(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) { + PyObject *abi_module = NULL, *cached_type = NULL, *abi_module_dict, *new_cached_type, *py_object_name; + int get_item_ref_result; + const char* object_name = strrchr(spec->name, '.'); + object_name = object_name ? object_name+1 : spec->name; + py_object_name = PyUnicode_FromString(object_name); + if (!py_object_name) return NULL; + abi_module = __Pyx_FetchSharedCythonABIModule(); + if (!abi_module) goto done; + abi_module_dict = PyModule_GetDict(abi_module); + if (!abi_module_dict) goto done; + get_item_ref_result = __Pyx_PyDict_GetItemRef(abi_module_dict, py_object_name, &cached_type); + if (get_item_ref_result == 1) { + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else if (unlikely(get_item_ref_result == -1)) { + goto bad; + } + CYTHON_UNUSED_VAR(module); + cached_type = __Pyx_PyType_FromMetaclass(metaclass, abi_module, spec, bases); + if (unlikely(!cached_type)) goto bad; + if (unlikely(__Pyx_fix_up_extension_type_from_spec(spec, (PyTypeObject *) cached_type) < 0)) goto bad; + new_cached_type = __Pyx_PyDict_SetDefault(abi_module_dict, py_object_name, cached_type, 1); + if (unlikely(new_cached_type != cached_type)) { + if (unlikely(!new_cached_type)) goto bad; + Py_DECREF(cached_type); + cached_type = new_cached_type; + if (__Pyx_VerifyCachedType( + cached_type, + object_name, + spec->basicsize) < 0) { + goto bad; + } + goto done; + } else { + Py_DECREF(new_cached_type); + } +done: + Py_XDECREF(abi_module); + Py_DECREF(py_object_name); + assert(cached_type == NULL || PyType_Check(cached_type)); + return (PyTypeObject *) cached_type; +bad: + Py_XDECREF(cached_type); + cached_type = NULL; + goto done; +} + +/* CommonTypesMetaclass */ +static PyObject* __pyx_CommonTypesMetaclass_get_module(CYTHON_UNUSED PyObject *self, CYTHON_UNUSED void* context) { + return PyUnicode_FromString(__PYX_ABI_MODULE_NAME); +} +static PyGetSetDef __pyx_CommonTypesMetaclass_getset[] = { + {"__module__", __pyx_CommonTypesMetaclass_get_module, NULL, NULL, NULL}, + {0, 0, 0, 0, 0} +}; +static PyType_Slot __pyx_CommonTypesMetaclass_slots[] = { + {Py_tp_getset, (void *)__pyx_CommonTypesMetaclass_getset}, + {0, 0} +}; +static PyType_Spec __pyx_CommonTypesMetaclass_spec = { + __PYX_TYPE_MODULE_PREFIX "_common_types_metatype", + 0, + 0, +#if PY_VERSION_HEX >= 0x030A0000 + Py_TPFLAGS_IMMUTABLETYPE | + Py_TPFLAGS_DISALLOW_INSTANTIATION | +#endif + Py_TPFLAGS_DEFAULT, + __pyx_CommonTypesMetaclass_slots +}; +static int __pyx_CommonTypesMetaclass_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + PyObject *bases = PyTuple_Pack(1, &PyType_Type); + if (unlikely(!bases)) { + return -1; + } + mstate->__pyx_CommonTypesMetaclassType = __Pyx_FetchCommonTypeFromSpec(NULL, module, &__pyx_CommonTypesMetaclass_spec, bases); + Py_DECREF(bases); + if (unlikely(mstate->__pyx_CommonTypesMetaclassType == NULL)) { + return -1; + } + return 0; +} + +/* CallTypeTraverse */ +#if !CYTHON_USE_TYPE_SPECS || (!CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x03090000) +#else +static int __Pyx_call_type_traverse(PyObject *o, int always_call, visitproc visit, void *arg) { + #if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 + if (__Pyx_get_runtime_version() < 0x03090000) return 0; + #endif + if (!always_call) { + PyTypeObject *base = __Pyx_PyObject_GetSlot(o, tp_base, PyTypeObject*); + unsigned long flags = PyType_GetFlags(base); + if (flags & Py_TPFLAGS_HEAPTYPE) { + return 0; + } + } + Py_VISIT((PyObject*)Py_TYPE(o)); + return 0; +} +#endif + +/* PyMethodNew */ +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + PyObject *result; + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + #if __PYX_LIMITED_VERSION_HEX >= 0x030C0000 + { + PyObject *args[] = {func, self}; + result = PyObject_Vectorcall(__pyx_mstate_global->__Pyx_CachedMethodType, args, 2, NULL); + } + #else + result = PyObject_CallFunctionObjArgs(__pyx_mstate_global->__Pyx_CachedMethodType, func, self, NULL); + #endif + return result; +} +#else +static PyObject *__Pyx_PyMethod_New(PyObject *func, PyObject *self, PyObject *typ) { + CYTHON_UNUSED_VAR(typ); + if (!self) + return __Pyx_NewRef(func); + return PyMethod_New(func, self); +} +#endif + +/* PyVectorcallFastCallDict */ +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static PyObject *__Pyx_PyVectorcall_FastCallDict_kw(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + PyObject *res = NULL; + PyObject *kwnames; + PyObject **newargs; + PyObject **kwvalues; + Py_ssize_t i, pos; + size_t j; + PyObject *key, *value; + unsigned long keys_are_strings; + #if !CYTHON_ASSUME_SAFE_SIZE + Py_ssize_t nkw = PyDict_Size(kw); + if (unlikely(nkw == -1)) return NULL; + #else + Py_ssize_t nkw = PyDict_GET_SIZE(kw); + #endif + newargs = (PyObject **)PyMem_Malloc((nargs + (size_t)nkw) * sizeof(args[0])); + if (unlikely(newargs == NULL)) { + PyErr_NoMemory(); + return NULL; + } + for (j = 0; j < nargs; j++) newargs[j] = args[j]; + kwnames = PyTuple_New(nkw); + if (unlikely(kwnames == NULL)) { + PyMem_Free(newargs); + return NULL; + } + kwvalues = newargs + nargs; + pos = i = 0; + keys_are_strings = Py_TPFLAGS_UNICODE_SUBCLASS; + while (PyDict_Next(kw, &pos, &key, &value)) { + keys_are_strings &= + #if CYTHON_COMPILING_IN_LIMITED_API + PyType_GetFlags(Py_TYPE(key)); + #else + Py_TYPE(key)->tp_flags; + #endif + Py_INCREF(key); + Py_INCREF(value); + #if !CYTHON_ASSUME_SAFE_MACROS + if (unlikely(PyTuple_SetItem(kwnames, i, key) < 0)) goto cleanup; + #else + PyTuple_SET_ITEM(kwnames, i, key); + #endif + kwvalues[i] = value; + i++; + } + if (unlikely(!keys_are_strings)) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + goto cleanup; + } + res = vc(func, newargs, nargs, kwnames); +cleanup: + Py_DECREF(kwnames); + for (i = 0; i < nkw; i++) + Py_DECREF(kwvalues[i]); + PyMem_Free(newargs); + return res; +} +static CYTHON_INLINE PyObject *__Pyx_PyVectorcall_FastCallDict(PyObject *func, __pyx_vectorcallfunc vc, PyObject *const *args, size_t nargs, PyObject *kw) +{ + Py_ssize_t kw_size = + likely(kw == NULL) ? + 0 : +#if !CYTHON_ASSUME_SAFE_SIZE + PyDict_Size(kw); +#else + PyDict_GET_SIZE(kw); +#endif + if (kw_size == 0) { + return vc(func, args, nargs, NULL); + } +#if !CYTHON_ASSUME_SAFE_SIZE + else if (unlikely(kw_size == -1)) { + return NULL; + } +#endif + return __Pyx_PyVectorcall_FastCallDict_kw(func, vc, args, nargs, kw); +} +#endif + +/* CythonFunctionShared */ +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunctionNoMethod(PyObject *func, void (*cfunc)(void)) { + if (__Pyx_CyFunction_Check(func)) { + return PyCFunction_GetFunction(((__pyx_CyFunctionObject*)func)->func) == (PyCFunction) cfunc; + } else if (PyCFunction_Check(func)) { + return PyCFunction_GetFunction(func) == (PyCFunction) cfunc; + } + return 0; +} +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if ((PyObject*)Py_TYPE(func) == __pyx_mstate_global->__Pyx_CachedMethodType) { + int result; + PyObject *newFunc = PyObject_GetAttr(func, __pyx_mstate_global->__pyx_n_u_func); + if (unlikely(!newFunc)) { + PyErr_Clear(); // It's only an optimization, so don't throw an error + return 0; + } + result = __Pyx__IsSameCyOrCFunctionNoMethod(newFunc, cfunc); + Py_DECREF(newFunc); + return result; + } + return __Pyx__IsSameCyOrCFunctionNoMethod(func, cfunc); +} +#else +static CYTHON_INLINE int __Pyx__IsSameCyOrCFunction(PyObject *func, void (*cfunc)(void)) { + if (PyMethod_Check(func)) { + func = PyMethod_GET_FUNCTION(func); + } + return __Pyx_CyOrPyCFunction_Check(func) && __Pyx_CyOrPyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +} +#endif +static CYTHON_INLINE void __Pyx__CyFunction_SetClassObj(__pyx_CyFunctionObject* f, PyObject* classobj) { +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + __Pyx_Py_XDECREF_SET( + __Pyx_CyFunction_GetClassObj(f), + ((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#else + __Pyx_Py_XDECREF_SET( + ((PyCMethodObject *) (f))->mm_class, + (PyTypeObject*)((classobj) ? __Pyx_NewRef(classobj) : NULL)); +#endif +} +static PyObject * +__Pyx_CyFunction_get_doc_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_doc == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_doc = PyObject_GetAttrString(op->func, "__doc__"); + if (unlikely(!op->func_doc)) return NULL; +#else + if (((PyCFunctionObject*)op)->m_ml->ml_doc) { + op->func_doc = PyUnicode_FromString(((PyCFunctionObject*)op)->m_ml->ml_doc); + if (unlikely(op->func_doc == NULL)) + return NULL; + } else { + Py_INCREF(Py_None); + return Py_None; + } +#endif + } + Py_INCREF(op->func_doc); + return op->func_doc; +} +static PyObject * +__Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, void *closure) { + PyObject *result; + CYTHON_UNUSED_VAR(closure); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_doc_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_doc(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (value == NULL) { + value = Py_None; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_doc, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_name_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_name == NULL)) { +#if CYTHON_COMPILING_IN_LIMITED_API + op->func_name = PyObject_GetAttrString(op->func, "__name__"); +#else + op->func_name = PyUnicode_InternFromString(((PyCFunctionObject*)op)->m_ml->ml_name); +#endif + if (unlikely(op->func_name == NULL)) + return NULL; + } + Py_INCREF(op->func_name); + return op->func_name; +} +static PyObject * +__Pyx_CyFunction_get_name(__pyx_CyFunctionObject *op, void *context) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_name_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_name(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__name__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_name, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_qualname(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + Py_INCREF(op->func_qualname); + result = op->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_qualname(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL || !PyUnicode_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__qualname__ must be set to a string object"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_qualname, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_dict_locked(__pyx_CyFunctionObject *op) +{ + if (unlikely(op->func_dict == NULL)) { + op->func_dict = PyDict_New(); + if (unlikely(op->func_dict == NULL)) + return NULL; + } + Py_INCREF(op->func_dict); + return op->func_dict; +} +static PyObject * +__Pyx_CyFunction_get_dict(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + PyObject *result; + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_dict_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_dict(__pyx_CyFunctionObject *op, PyObject *value, void *context) +{ + CYTHON_UNUSED_VAR(context); + if (unlikely(value == NULL)) { + PyErr_SetString(PyExc_TypeError, + "function's dictionary may not be deleted"); + return -1; + } + if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "setting function's dictionary to a non-dict"); + return -1; + } + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_dict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_globals(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(context); + Py_INCREF(op->func_globals); + return op->func_globals; +} +static PyObject * +__Pyx_CyFunction_get_closure(__pyx_CyFunctionObject *op, void *context) +{ + CYTHON_UNUSED_VAR(op); + CYTHON_UNUSED_VAR(context); + Py_INCREF(Py_None); + return Py_None; +} +static PyObject * +__Pyx_CyFunction_get_code(__pyx_CyFunctionObject *op, void *context) +{ + PyObject* result = (op->func_code) ? op->func_code : Py_None; + CYTHON_UNUSED_VAR(context); + Py_INCREF(result); + return result; +} +static int +__Pyx_CyFunction_init_defaults(__pyx_CyFunctionObject *op) { + int result = 0; + PyObject *res = op->defaults_getter((PyObject *) op); + if (unlikely(!res)) + return -1; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + op->defaults_tuple = PyTuple_GET_ITEM(res, 0); + Py_INCREF(op->defaults_tuple); + op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); + Py_INCREF(op->defaults_kwdict); + #else + op->defaults_tuple = __Pyx_PySequence_ITEM(res, 0); + if (unlikely(!op->defaults_tuple)) result = -1; + else { + op->defaults_kwdict = __Pyx_PySequence_ITEM(res, 1); + if (unlikely(!op->defaults_kwdict)) result = -1; + } + #endif + Py_DECREF(res); + return result; +} +static int +__Pyx_CyFunction_set_defaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyTuple_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__defaults__ must be set to a tuple object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__defaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_tuple, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_defaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_tuple; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_tuple; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_defaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result = NULL; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_defaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_kwdefaults(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value) { + value = Py_None; + } else if (unlikely(value != Py_None && !PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__kwdefaults__ must be set to a dict object"); + return -1; + } + PyErr_WarnEx(PyExc_RuntimeWarning, "changes to cyfunction.__kwdefaults__ will not " + "currently affect the values used in function calls", 1); + Py_INCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->defaults_kwdict, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->defaults_kwdict; + if (unlikely(!result)) { + if (op->defaults_getter) { + if (unlikely(__Pyx_CyFunction_init_defaults(op) < 0)) return NULL; + result = op->defaults_kwdict; + } else { + result = Py_None; + } + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_kwdefaults(__pyx_CyFunctionObject *op, void *context) { + PyObject* result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_kwdefaults_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static int +__Pyx_CyFunction_set_annotations(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + if (!value || value == Py_None) { + value = NULL; + } else if (unlikely(!PyDict_Check(value))) { + PyErr_SetString(PyExc_TypeError, + "__annotations__ must be set to a dict object"); + return -1; + } + Py_XINCREF(value); + __Pyx_BEGIN_CRITICAL_SECTION(op); + __Pyx_Py_XDECREF_SET(op->func_annotations, value); + __Pyx_END_CRITICAL_SECTION(); + return 0; +} +static PyObject * +__Pyx_CyFunction_get_annotations_locked(__pyx_CyFunctionObject *op) { + PyObject* result = op->func_annotations; + if (unlikely(!result)) { + result = PyDict_New(); + if (unlikely(!result)) return NULL; + op->func_annotations = result; + } + Py_INCREF(result); + return result; +} +static PyObject * +__Pyx_CyFunction_get_annotations(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + __Pyx_BEGIN_CRITICAL_SECTION(op); + result = __Pyx_CyFunction_get_annotations_locked(op); + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine_value(__pyx_CyFunctionObject *op) { + int is_coroutine = op->flags & __Pyx_CYFUNCTION_COROUTINE; + if (is_coroutine) { + PyObject *is_coroutine_value, *module, *fromlist, *marker = __pyx_mstate_global->__pyx_n_u_is_coroutine; + fromlist = PyList_New(1); + if (unlikely(!fromlist)) return NULL; + Py_INCREF(marker); +#if CYTHON_ASSUME_SAFE_MACROS + PyList_SET_ITEM(fromlist, 0, marker); +#else + if (unlikely(PyList_SetItem(fromlist, 0, marker) < 0)) { + Py_DECREF(marker); + Py_DECREF(fromlist); + return NULL; + } +#endif + module = PyImport_ImportModuleLevelObject(__pyx_mstate_global->__pyx_n_u_asyncio_coroutines, NULL, NULL, fromlist, 0); + Py_DECREF(fromlist); + if (unlikely(!module)) goto ignore; + is_coroutine_value = __Pyx_PyObject_GetAttrStr(module, marker); + Py_DECREF(module); + if (likely(is_coroutine_value)) { + return is_coroutine_value; + } +ignore: + PyErr_Clear(); + } + return __Pyx_PyBool_FromLong(is_coroutine); +} +static PyObject * +__Pyx_CyFunction_get_is_coroutine(__pyx_CyFunctionObject *op, void *context) { + PyObject *result; + CYTHON_UNUSED_VAR(context); + if (op->func_is_coroutine) { + return __Pyx_NewRef(op->func_is_coroutine); + } + result = __Pyx_CyFunction_get_is_coroutine_value(op); + if (unlikely(!result)) + return NULL; + __Pyx_BEGIN_CRITICAL_SECTION(op); + if (op->func_is_coroutine) { + Py_DECREF(result); + result = __Pyx_NewRef(op->func_is_coroutine); + } else { + op->func_is_coroutine = __Pyx_NewRef(result); + } + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static void __Pyx_CyFunction_raise_argument_count_error(__pyx_CyFunctionObject *func, const char* message, Py_ssize_t size) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + py_name, message, size); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s (%" CYTHON_FORMAT_SSIZE_T "d given)", + name, message, size); +#endif +} +static void __Pyx_CyFunction_raise_type_error(__pyx_CyFunctionObject *func, const char* message) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *py_name = __Pyx_CyFunction_get_name(func, NULL); + if (!py_name) return; + PyErr_Format(PyExc_TypeError, + "%.200S() %s", + py_name, message); + Py_DECREF(py_name); +#else + const char* name = ((PyCFunctionObject*)func)->m_ml->ml_name; + PyErr_Format(PyExc_TypeError, + "%.200s() %s", + name, message); +#endif +} +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject * +__Pyx_CyFunction_get_module(__pyx_CyFunctionObject *op, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_GetAttrString(op->func, "__module__"); +} +static int +__Pyx_CyFunction_set_module(__pyx_CyFunctionObject *op, PyObject* value, void *context) { + CYTHON_UNUSED_VAR(context); + return PyObject_SetAttrString(op->func, "__module__", value); +} +#endif +static PyGetSetDef __pyx_CyFunction_getsets[] = { + {"func_doc", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"__doc__", (getter)__Pyx_CyFunction_get_doc, (setter)__Pyx_CyFunction_set_doc, 0, 0}, + {"func_name", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__name__", (getter)__Pyx_CyFunction_get_name, (setter)__Pyx_CyFunction_set_name, 0, 0}, + {"__qualname__", (getter)__Pyx_CyFunction_get_qualname, (setter)__Pyx_CyFunction_set_qualname, 0, 0}, + {"func_dict", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {"__dict__", (getter)__Pyx_CyFunction_get_dict, (setter)__Pyx_CyFunction_set_dict, 0, 0}, + {"func_globals", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"__globals__", (getter)__Pyx_CyFunction_get_globals, 0, 0, 0}, + {"func_closure", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"__closure__", (getter)__Pyx_CyFunction_get_closure, 0, 0, 0}, + {"func_code", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"__code__", (getter)__Pyx_CyFunction_get_code, 0, 0, 0}, + {"func_defaults", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__defaults__", (getter)__Pyx_CyFunction_get_defaults, (setter)__Pyx_CyFunction_set_defaults, 0, 0}, + {"__kwdefaults__", (getter)__Pyx_CyFunction_get_kwdefaults, (setter)__Pyx_CyFunction_set_kwdefaults, 0, 0}, + {"__annotations__", (getter)__Pyx_CyFunction_get_annotations, (setter)__Pyx_CyFunction_set_annotations, 0, 0}, + {"_is_coroutine", (getter)__Pyx_CyFunction_get_is_coroutine, 0, 0, 0}, +#if CYTHON_COMPILING_IN_LIMITED_API + {"__module__", (getter)__Pyx_CyFunction_get_module, (setter)__Pyx_CyFunction_set_module, 0, 0}, +#endif + {0, 0, 0, 0, 0} +}; +static PyMemberDef __pyx_CyFunction_members[] = { +#if !CYTHON_COMPILING_IN_LIMITED_API + {"__module__", T_OBJECT, offsetof(PyCFunctionObject, m_module), 0, 0}, +#endif + {"__dictoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_dict), READONLY, 0}, +#if CYTHON_METH_FASTCALL +#if CYTHON_BACKPORT_VECTORCALL || CYTHON_COMPILING_IN_LIMITED_API + {"__vectorcalloffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_vectorcall), READONLY, 0}, +#else + {"__vectorcalloffset__", T_PYSSIZET, offsetof(PyCFunctionObject, vectorcall), READONLY, 0}, +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + {"__weaklistoffset__", T_PYSSIZET, offsetof(__pyx_CyFunctionObject, func_weakreflist), READONLY, 0}, +#else + {"__weaklistoffset__", T_PYSSIZET, offsetof(PyCFunctionObject, m_weakreflist), READONLY, 0}, +#endif +#endif + {0, 0, 0, 0, 0} +}; +static PyObject * +__Pyx_CyFunction_reduce(__pyx_CyFunctionObject *m, PyObject *args) +{ + PyObject *result = NULL; + CYTHON_UNUSED_VAR(args); + __Pyx_BEGIN_CRITICAL_SECTION(m); + Py_INCREF(m->func_qualname); + result = m->func_qualname; + __Pyx_END_CRITICAL_SECTION(); + return result; +} +static PyMethodDef __pyx_CyFunction_methods[] = { + {"__reduce__", (PyCFunction)__Pyx_CyFunction_reduce, METH_VARARGS, 0}, + {0, 0, 0, 0} +}; +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyFunction_weakreflist(cyfunc) ((cyfunc)->func_weakreflist) +#else +#define __Pyx_CyFunction_weakreflist(cyfunc) (((PyCFunctionObject*)cyfunc)->m_weakreflist) +#endif +static PyObject *__Pyx_CyFunction_Init(__pyx_CyFunctionObject *op, PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { +#if !CYTHON_COMPILING_IN_LIMITED_API + PyCFunctionObject *cf = (PyCFunctionObject*) op; +#endif + if (unlikely(op == NULL)) + return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + op->func = PyCFunction_NewEx(ml, (PyObject*)op, module); + if (unlikely(!op->func)) return NULL; +#endif + op->flags = flags; + __Pyx_CyFunction_weakreflist(op) = NULL; +#if !CYTHON_COMPILING_IN_LIMITED_API + cf->m_ml = ml; + cf->m_self = (PyObject *) op; +#endif + Py_XINCREF(closure); + op->func_closure = closure; +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_XINCREF(module); + cf->m_module = module; +#endif + op->func_dict = NULL; + op->func_name = NULL; + Py_INCREF(qualname); + op->func_qualname = qualname; + op->func_doc = NULL; +#if PY_VERSION_HEX < 0x030900B1 || CYTHON_COMPILING_IN_LIMITED_API + op->func_classobj = NULL; +#else + ((PyCMethodObject*)op)->mm_class = NULL; +#endif + op->func_globals = globals; + Py_INCREF(op->func_globals); + Py_XINCREF(code); + op->func_code = code; + op->defaults = NULL; + op->defaults_tuple = NULL; + op->defaults_kwdict = NULL; + op->defaults_getter = NULL; + op->func_annotations = NULL; + op->func_is_coroutine = NULL; +#if CYTHON_METH_FASTCALL + switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS | METH_METHOD)) { + case METH_NOARGS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_NOARGS; + break; + case METH_O: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_O; + break; + case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD; + break; + case METH_FASTCALL | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS; + break; + case METH_VARARGS | METH_KEYWORDS: + __Pyx_CyFunction_func_vectorcall(op) = NULL; + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + Py_DECREF(op); + return NULL; + } +#endif + return (PyObject *) op; +} +static int +__Pyx_CyFunction_clear(__pyx_CyFunctionObject *m) +{ + Py_CLEAR(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_CLEAR(m->func); +#else + Py_CLEAR(((PyCFunctionObject*)m)->m_module); +#endif + Py_CLEAR(m->func_dict); + Py_CLEAR(m->func_name); + Py_CLEAR(m->func_qualname); + Py_CLEAR(m->func_doc); + Py_CLEAR(m->func_globals); + Py_CLEAR(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API +#if PY_VERSION_HEX < 0x030900B1 + Py_CLEAR(__Pyx_CyFunction_GetClassObj(m)); +#else + { + PyObject *cls = (PyObject*) ((PyCMethodObject *) (m))->mm_class; + ((PyCMethodObject *) (m))->mm_class = NULL; + Py_XDECREF(cls); + } +#endif +#endif + Py_CLEAR(m->defaults_tuple); + Py_CLEAR(m->defaults_kwdict); + Py_CLEAR(m->func_annotations); + Py_CLEAR(m->func_is_coroutine); + Py_CLEAR(m->defaults); + return 0; +} +static void __Pyx__CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + if (__Pyx_CyFunction_weakreflist(m) != NULL) + PyObject_ClearWeakRefs((PyObject *) m); + __Pyx_CyFunction_clear(m); + __Pyx_PyHeapTypeObject_GC_Del(m); +} +static void __Pyx_CyFunction_dealloc(__pyx_CyFunctionObject *m) +{ + PyObject_GC_UnTrack(m); + __Pyx__CyFunction_dealloc(m); +} +static int __Pyx_CyFunction_traverse(__pyx_CyFunctionObject *m, visitproc visit, void *arg) +{ + { + int e = __Pyx_call_type_traverse((PyObject*)m, 1, visit, arg); + if (e) return e; + } + Py_VISIT(m->func_closure); +#if CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(m->func); +#else + Py_VISIT(((PyCFunctionObject*)m)->m_module); +#endif + Py_VISIT(m->func_dict); + __Pyx_VISIT_CONST(m->func_name); + __Pyx_VISIT_CONST(m->func_qualname); + Py_VISIT(m->func_doc); + Py_VISIT(m->func_globals); + __Pyx_VISIT_CONST(m->func_code); +#if !CYTHON_COMPILING_IN_LIMITED_API + Py_VISIT(__Pyx_CyFunction_GetClassObj(m)); +#endif + Py_VISIT(m->defaults_tuple); + Py_VISIT(m->defaults_kwdict); + Py_VISIT(m->func_is_coroutine); + Py_VISIT(m->defaults); + return 0; +} +static PyObject* +__Pyx_CyFunction_repr(__pyx_CyFunctionObject *op) +{ + PyObject *repr; + __Pyx_BEGIN_CRITICAL_SECTION(op); + repr = PyUnicode_FromFormat("", + op->func_qualname, (void *)op); + __Pyx_END_CRITICAL_SECTION(); + return repr; +} +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { +#if CYTHON_COMPILING_IN_LIMITED_API + PyObject *f = ((__pyx_CyFunctionObject*)func)->func; + PyCFunction meth; + int flags; + meth = PyCFunction_GetFunction(f); + if (unlikely(!meth)) return NULL; + flags = PyCFunction_GetFlags(f); + if (unlikely(flags < 0)) return NULL; +#else + PyCFunctionObject* f = (PyCFunctionObject*)func; + PyCFunction meth = f->m_ml->ml_meth; + int flags = f->m_ml->ml_flags; +#endif + Py_ssize_t size; + switch (flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { + case METH_VARARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) + return (*meth)(self, arg); + break; + case METH_VARARGS | METH_KEYWORDS: + return (*(PyCFunctionWithKeywords)(void(*)(void))meth)(self, arg, kw); + case METH_NOARGS: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 0)) + return (*meth)(self, NULL); + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes no arguments", size); + return NULL; + } + break; + case METH_O: + if (likely(kw == NULL || PyDict_Size(kw) == 0)) { +#if CYTHON_ASSUME_SAFE_SIZE + size = PyTuple_GET_SIZE(arg); +#else + size = PyTuple_Size(arg); + if (unlikely(size < 0)) return NULL; +#endif + if (likely(size == 1)) { + PyObject *result, *arg0; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + arg0 = PyTuple_GET_ITEM(arg, 0); + #else + arg0 = __Pyx_PySequence_ITEM(arg, 0); if (unlikely(!arg0)) return NULL; + #endif + result = (*meth)(self, arg0); + #if !(CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS) + Py_DECREF(arg0); + #endif + return result; + } + __Pyx_CyFunction_raise_argument_count_error( + (__pyx_CyFunctionObject*)func, + "takes exactly one argument", size); + return NULL; + } + break; + default: + PyErr_SetString(PyExc_SystemError, "Bad call flags for CyFunction"); + return NULL; + } + __Pyx_CyFunction_raise_type_error( + (__pyx_CyFunctionObject*)func, "takes no keyword arguments"); + return NULL; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *self, *result; +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)func)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)func)->m_self; +#endif + result = __Pyx_CyFunction_CallMethod(func, self, arg, kw); + return result; +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) + __pyx_vectorcallfunc vc = __Pyx_CyFunction_func_vectorcall(cyfunc); + if (vc) { +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE + return __Pyx_PyVectorcall_FastCallDict(func, vc, &PyTuple_GET_ITEM(args, 0), (size_t)PyTuple_GET_SIZE(args), kw); +#else + (void) &__Pyx_PyVectorcall_FastCallDict; + return PyVectorcall_Call(func, args, kw); +#endif + } +#endif + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; +#if CYTHON_ASSUME_SAFE_SIZE + argc = PyTuple_GET_SIZE(args); +#else + argc = PyTuple_Size(args); + if (unlikely(argc < 0)) return NULL; +#endif + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + PyErr_Format(PyExc_TypeError, + "unbound method %.200S() needs an argument", + cyfunc->func_qualname); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; +} +#if CYTHON_METH_FASTCALL && (CYTHON_VECTORCALL || CYTHON_BACKPORT_VECTORCALL) +static CYTHON_INLINE int __Pyx_CyFunction_Vectorcall_CheckArgs(__pyx_CyFunctionObject *cyfunc, Py_ssize_t nargs, PyObject *kwnames) +{ + int ret = 0; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + if (unlikely(nargs < 1)) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "needs an argument"); + return -1; + } + ret = 1; + } + if (unlikely(kwnames) && unlikely(__Pyx_PyTuple_GET_SIZE(kwnames))) { + __Pyx_CyFunction_raise_type_error( + cyfunc, "takes no keyword arguments"); + return -1; + } + return ret; +} +static PyObject * __Pyx_CyFunction_Vectorcall_NOARGS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 0)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes no arguments", nargs); + return NULL; + } + return meth(self, NULL); +} +static PyObject * __Pyx_CyFunction_Vectorcall_O(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, kwnames)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + if (unlikely(nargs != 1)) { + __Pyx_CyFunction_raise_argument_count_error( + cyfunc, "takes exactly one argument", nargs); + return NULL; + } + return meth(self, args[0]); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCFunctionFastWithKeywords)(void(*)(void))meth)(self, args, nargs, kwnames); +} +static PyObject * __Pyx_CyFunction_Vectorcall_FASTCALL_KEYWORDS_METHOD(PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) +{ + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *)func; + PyTypeObject *cls = (PyTypeObject *) __Pyx_CyFunction_GetClassObj(cyfunc); +#if CYTHON_BACKPORT_VECTORCALL + Py_ssize_t nargs = (Py_ssize_t)nargsf; +#else + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); +#endif + PyObject *self; +#if CYTHON_COMPILING_IN_LIMITED_API + PyCFunction meth = PyCFunction_GetFunction(cyfunc->func); + if (unlikely(!meth)) return NULL; +#else + PyCFunction meth = ((PyCFunctionObject*)cyfunc)->m_ml->ml_meth; +#endif + switch (__Pyx_CyFunction_Vectorcall_CheckArgs(cyfunc, nargs, NULL)) { + case 1: + self = args[0]; + args += 1; + nargs -= 1; + break; + case 0: +#if CYTHON_COMPILING_IN_LIMITED_API + self = PyCFunction_GetSelf(((__pyx_CyFunctionObject*)cyfunc)->func); + if (unlikely(!self) && PyErr_Occurred()) return NULL; +#else + self = ((PyCFunctionObject*)cyfunc)->m_self; +#endif + break; + default: + return NULL; + } + return ((__Pyx_PyCMethod)(void(*)(void))meth)(self, cls, args, (size_t)nargs, kwnames); +} +#endif +static PyType_Slot __pyx_CyFunctionType_slots[] = { + {Py_tp_dealloc, (void *)__Pyx_CyFunction_dealloc}, + {Py_tp_repr, (void *)__Pyx_CyFunction_repr}, + {Py_tp_call, (void *)__Pyx_CyFunction_CallAsMethod}, + {Py_tp_traverse, (void *)__Pyx_CyFunction_traverse}, + {Py_tp_clear, (void *)__Pyx_CyFunction_clear}, + {Py_tp_methods, (void *)__pyx_CyFunction_methods}, + {Py_tp_members, (void *)__pyx_CyFunction_members}, + {Py_tp_getset, (void *)__pyx_CyFunction_getsets}, + {Py_tp_descr_get, (void *)__Pyx_PyMethod_New}, + {0, 0}, +}; +static PyType_Spec __pyx_CyFunctionType_spec = { + __PYX_TYPE_MODULE_PREFIX "cython_function_or_method", + sizeof(__pyx_CyFunctionObject), + 0, +#ifdef Py_TPFLAGS_METHOD_DESCRIPTOR + Py_TPFLAGS_METHOD_DESCRIPTOR | +#endif +#if CYTHON_METH_FASTCALL +#if defined(Py_TPFLAGS_HAVE_VECTORCALL) + Py_TPFLAGS_HAVE_VECTORCALL | +#elif defined(_Py_TPFLAGS_HAVE_VECTORCALL) + _Py_TPFLAGS_HAVE_VECTORCALL | +#endif +#endif // CYTHON_METH_FASTCALL +#if PY_VERSION_HEX >= 0x030A0000 + Py_TPFLAGS_IMMUTABLETYPE | +#endif + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, + __pyx_CyFunctionType_slots +}; +static int __pyx_CyFunction_init(PyObject *module) { + __pyx_mstatetype *mstate = __Pyx_PyModule_GetState(module); + mstate->__pyx_CyFunctionType = __Pyx_FetchCommonTypeFromSpec( + mstate->__pyx_CommonTypesMetaclassType, module, &__pyx_CyFunctionType_spec, NULL); + if (unlikely(mstate->__pyx_CyFunctionType == NULL)) { + return -1; + } + return 0; +} +static CYTHON_INLINE PyObject *__Pyx_CyFunction_InitDefaults(PyObject *func, PyTypeObject *defaults_type) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults = PyObject_CallObject((PyObject*)defaults_type, NULL); // _PyObject_New(defaults_type); + if (unlikely(!m->defaults)) + return NULL; + return m->defaults; +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsTuple(PyObject *func, PyObject *tuple) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_tuple = tuple; + Py_INCREF(tuple); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetDefaultsKwDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->defaults_kwdict = dict; + Py_INCREF(dict); +} +static CYTHON_INLINE void __Pyx_CyFunction_SetAnnotationsDict(PyObject *func, PyObject *dict) { + __pyx_CyFunctionObject *m = (__pyx_CyFunctionObject *) func; + m->func_annotations = dict; + Py_INCREF(dict); +} + +/* CythonFunction */ +static PyObject *__Pyx_CyFunction_New(PyMethodDef *ml, int flags, PyObject* qualname, + PyObject *closure, PyObject *module, PyObject* globals, PyObject* code) { + PyObject *op = __Pyx_CyFunction_Init( + PyObject_GC_New(__pyx_CyFunctionObject, __pyx_mstate_global->__pyx_CyFunctionType), + ml, flags, qualname, closure, module, globals, code + ); + if (likely(op)) { + PyObject_GC_Track(op); + } + return op; +} + +/* CLineInTraceback */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline; + PyObject *ptype, *pvalue, *ptraceback; +#if CYTHON_COMPILING_IN_CPYTHON + PyObject **cython_runtime_dict; +#endif + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); +#if CYTHON_COMPILING_IN_CPYTHON + cython_runtime_dict = _PyObject_GetDictPtr(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __Pyx_BEGIN_CRITICAL_SECTION(*cython_runtime_dict); + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, *cython_runtime_dict, + __Pyx_PyDict_GetItemStr(*cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback)) + Py_XINCREF(use_cline); + __Pyx_END_CRITICAL_SECTION(); + } else +#endif + { + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback); + if (use_cline_obj) { + use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; + Py_INCREF(use_cline); + Py_DECREF(use_cline_obj); + } else { + PyErr_Clear(); + use_cline = NULL; + } + } + if (!use_cline) { + c_line = 0; + (void) PyObject_SetAttr(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False); + } + else if (use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + (void) __pyx_cfilenm; + (void) __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + py_funcname = PyUnicode_FromString(funcname); + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* CIntFromPyVerify */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntFromPy */ +static CYTHON_INLINE size_t __Pyx_PyLong_As_size_t(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + size_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (size_t) -1; + val = __Pyx_PyLong_As_size_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(size_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(size_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 2 * PyLong_SHIFT)) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(size_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 3 * PyLong_SHIFT)) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(size_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 4 * PyLong_SHIFT)) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(size_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(size_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(size_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(size_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(size_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT)) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(size_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(size_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + size_t val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (size_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (size_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (size_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (size_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(size_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((size_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(size_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((size_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((size_t) 1) << (sizeof(size_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (size_t) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to size_t"); + return (size_t) -1; +} + +/* FormatTypeName */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static __Pyx_TypeName +__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) +{ + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u__3); + } + goto done; +} +#endif + +/* PyObjectVectorCallKwBuilder */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if defined(HAVE_LONG_LONG) && !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPy */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + long val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (long) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + int val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030b0000 + return Py_Version & ~0xFFUL; +#else + static unsigned long __Pyx_cached_runtime_version = 0; + if (__Pyx_cached_runtime_version == 0) { + const char* rt_version = Py_GetVersion(); + unsigned long version = 0; + unsigned long factor = 0x01000000UL; + unsigned int digit = 0; + int i = 0; + while (factor) { + while ('0' <= rt_version[i] && rt_version[i] <= '9') { + digit = digit * 10 + (unsigned int) (rt_version[i] - '0'); + ++i; + } + version += factor * digit; + if (rt_version[i] != '.') + break; + digit = 0; + factor >>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } + return __Pyx_cached_runtime_version; +#endif +} + +/* CheckBinaryVersion */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } +} + +/* FunctionExport */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig) { + PyObject *d = 0; + PyObject *cobj = 0; + union { + void (*fp)(void); + void *p; + } tmp; + d = PyObject_GetAttrString(__pyx_m, "__pyx_capi__"); + if (!d) { + PyErr_Clear(); + d = PyDict_New(); + if (!d) + goto bad; + Py_INCREF(d); + if (PyModule_AddObject(__pyx_m, "__pyx_capi__", d) < 0) + goto bad; + } + tmp.fp = f; + cobj = PyCapsule_New(tmp.p, sig, 0); + if (!cobj) + goto bad; + if (PyDict_SetItemString(d, name, cobj) < 0) + goto bad; + Py_DECREF(cobj); + Py_DECREF(d); + return 0; +bad: + Py_XDECREF(cobj); + Py_XDECREF(d); + return -1; +} + +/* NewCodeObj */ +#if CYTHON_COMPILING_IN_LIMITED_API + static PyObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyObject *exception_table = NULL; + PyObject *types_module=NULL, *code_type=NULL, *result=NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + PyObject *version_info; + PyObject *py_minor_version = NULL; + #endif + long minor_version = 0; + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + #if __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + minor_version = 11; + #else + if (!(version_info = PySys_GetObject("version_info"))) goto end; + if (!(py_minor_version = PySequence_GetItem(version_info, 1))) goto end; + minor_version = PyLong_AsLong(py_minor_version); + Py_DECREF(py_minor_version); + if (minor_version == -1 && PyErr_Occurred()) goto end; + #endif + if (!(types_module = PyImport_ImportModule("types"))) goto end; + if (!(code_type = PyObject_GetAttrString(types_module, "CodeType"))) goto end; + if (minor_version <= 7) { + (void)p; + result = PyObject_CallFunction(code_type, "iiiiiOOOOOOiOOO", a, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else if (minor_version <= 10) { + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOiOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, fline, lnos, fv, cell); + } else { + if (!(exception_table = PyBytes_FromStringAndSize(NULL, 0))) goto end; + result = PyObject_CallFunction(code_type, "iiiiiiOOOOOOOiOOOO", a,p, k, l, s, f, code, + c, n, v, fn, name, name, fline, lnos, exception_table, fv, cell); + } + end: + Py_XDECREF(code_type); + Py_XDECREF(exception_table); + Py_XDECREF(types_module); + if (type) { + PyErr_Restore(type, value, traceback); + } + return result; + } +#elif PY_VERSION_HEX >= 0x030B0000 + static PyCodeObject* __Pyx__PyCode_New(int a, int p, int k, int l, int s, int f, + PyObject *code, PyObject *c, PyObject* n, PyObject *v, + PyObject *fv, PyObject *cell, PyObject* fn, + PyObject *name, int fline, PyObject *lnos) { + PyCodeObject *result; + result = + #if PY_VERSION_HEX >= 0x030C0000 + PyUnstable_Code_NewWithPosOnlyArgs + #else + PyCode_NewWithPosOnlyArgs + #endif + (a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, name, fline, lnos, __pyx_mstate_global->__pyx_empty_bytes); + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030c00A1 + if (likely(result)) + result->_co_firsttraceable = 0; + #endif + return result; + } +#elif PY_VERSION_HEX >= 0x030800B2 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_NewWithPosOnlyArgs(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#else + #define __Pyx__PyCode_New(a, p, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos)\ + PyCode_New(a, k, l, s, f, code, c, n, v, fv, cell, fn, name, fline, lnos) +#endif +static PyObject* __Pyx_PyCode_New( + const __Pyx_PyCode_New_function_description descr, + PyObject * const *varnames, + PyObject *filename, + PyObject *funcname, + const char *line_table, + PyObject *tuple_dedup_map +) { + PyObject *code_obj = NULL, *varnames_tuple_dedup = NULL, *code_bytes = NULL, *line_table_bytes = NULL; + Py_ssize_t var_count = (Py_ssize_t) descr.nlocals; + PyObject *varnames_tuple = PyTuple_New(var_count); + if (unlikely(!varnames_tuple)) return NULL; + for (Py_ssize_t i=0; i < var_count; i++) { + Py_INCREF(varnames[i]); + if (__Pyx_PyTuple_SET_ITEM(varnames_tuple, i, varnames[i]) != (0)) goto done; + } + #if CYTHON_COMPILING_IN_LIMITED_API + varnames_tuple_dedup = PyDict_GetItem(tuple_dedup_map, varnames_tuple); + if (!varnames_tuple_dedup) { + if (unlikely(PyDict_SetItem(tuple_dedup_map, varnames_tuple, varnames_tuple) < 0)) goto done; + varnames_tuple_dedup = varnames_tuple; + } + #else + varnames_tuple_dedup = PyDict_SetDefault(tuple_dedup_map, varnames_tuple, varnames_tuple); + if (unlikely(!varnames_tuple_dedup)) goto done; + #endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(varnames_tuple_dedup); + #endif + if (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table != NULL + && !CYTHON_COMPILING_IN_GRAAL) { + line_table_bytes = PyBytes_FromStringAndSize(line_table, descr.line_table_length); + if (unlikely(!line_table_bytes)) goto done; + Py_ssize_t code_len = (descr.line_table_length * 2 + 4) & ~3; + code_bytes = PyBytes_FromStringAndSize(NULL, code_len); + if (unlikely(!code_bytes)) goto done; + char* c_code_bytes = PyBytes_AsString(code_bytes); + if (unlikely(!c_code_bytes)) goto done; + memset(c_code_bytes, 0, (size_t) code_len); + } + code_obj = (PyObject*) __Pyx__PyCode_New( + (int) descr.argcount, + (int) descr.num_posonly_args, + (int) descr.num_kwonly_args, + (int) descr.nlocals, + 0, + (int) descr.flags, + code_bytes ? code_bytes : __pyx_mstate_global->__pyx_empty_bytes, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + varnames_tuple_dedup, + __pyx_mstate_global->__pyx_empty_tuple, + __pyx_mstate_global->__pyx_empty_tuple, + filename, + funcname, + (int) descr.first_line, + (__PYX_LIMITED_VERSION_HEX >= (0x030b0000) && line_table_bytes) ? line_table_bytes : __pyx_mstate_global->__pyx_empty_bytes + ); +done: + Py_XDECREF(code_bytes); + Py_XDECREF(line_table_bytes); + #if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(varnames_tuple_dedup); + #endif + Py_DECREF(varnames_tuple); + return code_obj; +} + +/* InitStrings */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names) { + while (t->s) { + PyObject *str; + if (t->is_unicode) { + if (t->intern) { + str = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + str = PyUnicode_Decode(t->s, t->n - 1, encoding_names[t->encoding], NULL); + } else { + str = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + str = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + if (!str) + return -1; + *target = str; + if (PyObject_Hash(str) == -1) + return -1; + ++t; + ++target; + } + return 0; +} + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} +#if CYTHON_USE_PYLONG_INTERNALS +static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value) { + if (unlikely(!__Pyx_PyLong_IsCompact(x))) + return 0; + Py_ssize_t value = __Pyx_PyLong_CompactValue(x); + if ((sizeof(long) < sizeof(Py_ssize_t)) && unlikely(value != (long) value)) + return 0; + *return_value = (long) value; + return 1; +} +#endif + + +/* MultiPhaseInitModuleState */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 +#else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 +#endif +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS +#error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" +#endif +#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#define __Pyx_ModuleStateLookup_Lock() +#define __Pyx_ModuleStateLookup_Unlock() +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 +static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; +#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#include +static std::mutex __Pyx_ModuleStateLookup_mutex; +#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() +#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) +#include +static mtx_t __Pyx_ModuleStateLookup_mutex; +static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; +static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); +} +#define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(HAVE_PTHREAD_H) +#include +static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; +#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(_WIN32) +#include // synchapi.h on its own doesn't work +static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; +#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#else +#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." +#endif +typedef struct { + int64_t id; + PyObject *module; +} __Pyx_InterpreterIdAndModule; +typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; +} __Pyx_ModuleStateLookupData; +#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; +#else +static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; +#endif +static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; +} +static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; +#else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); +#endif + return result; + } +} +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); +} +#else +#define __Pyx_ModuleStateLookup_wait_until_no_readers() +#endif +static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; +} +static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; +} +static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); +#else + __Pyx_ModuleStateLookup_data = new_data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return result; +} +static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; +#endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); +#else + __Pyx_ModuleStateLookup_data = data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; +} +#endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm.cpython-312-x86_64-linux-gnu.so b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm.cpython-312-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..23607424d0b6b7d0e0940d3168604f11c3287deb Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm.cpython-312-x86_64-linux-gnu.so differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm.pxd new file mode 100644 index 0000000000000000000000000000000000000000..8ca17dbc4d4da532160b2ba21eed0074b1502d75 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm.pxd @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.0.1 to 12.9.1. Do not modify it directly. + +from ..cynvvm cimport * + + +############################################################################### +# Wrapper functions +############################################################################### + +cdef const char* _nvvmGetErrorString(nvvmResult result) except?NULL nogil +cdef nvvmResult _nvvmVersion(int* major, int* minor) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvvmResult _nvvmIRVersion(int* majorIR, int* minorIR, int* majorDbg, int* minorDbg) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvvmResult _nvvmCreateProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvvmResult _nvvmDestroyProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvvmResult _nvvmAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvvmResult _nvvmLazyAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvvmResult _nvvmCompileProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvvmResult _nvvmVerifyProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvvmResult _nvvmGetCompiledResultSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvvmResult _nvvmGetCompiledResult(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvvmResult _nvvmGetProgramLogSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvvmResult _nvvmGetProgramLog(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm.pyx new file mode 100644 index 0000000000000000000000000000000000000000..7e82d2a9aa216eb3586e7f94a0339e1d9ed6b50d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm.pyx @@ -0,0 +1,381 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.0.1 to 12.9.1. Do not modify it directly. + +from libc.stdint cimport intptr_t, uintptr_t + +import threading +from .utils import FunctionNotFoundError, NotSupportedError + +from cuda.pathfinder import load_nvidia_dynamic_lib + + +############################################################################### +# Extern +############################################################################### + +cdef extern from "" nogil: + void* dlopen(const char*, int) + char* dlerror() + void* dlsym(void*, const char*) + int dlclose(void*) + + enum: + RTLD_LAZY + RTLD_NOW + RTLD_GLOBAL + RTLD_LOCAL + + const void* RTLD_DEFAULT 'RTLD_DEFAULT' + +cdef int get_cuda_version(): + cdef void* handle = NULL + cdef int err, driver_ver = 0 + + # Load driver to check version + handle = dlopen('libcuda.so.1', RTLD_NOW | RTLD_GLOBAL) + if handle == NULL: + err_msg = dlerror() + raise NotSupportedError(f'CUDA driver is not found ({err_msg.decode()})') + cuDriverGetVersion = dlsym(handle, "cuDriverGetVersion") + if cuDriverGetVersion == NULL: + raise RuntimeError('Did not find cuDriverGetVersion symbol in libcuda.so.1') + err = (cuDriverGetVersion)(&driver_ver) + if err != 0: + raise RuntimeError(f'cuDriverGetVersion returned error code {err}') + + return driver_ver + + +############################################################################### +# Wrapper init +############################################################################### + +cdef object __symbol_lock = threading.Lock() +cdef bint __py_nvvm_init = False +cdef void* __cuDriverGetVersion = NULL + +cdef void* __nvvmGetErrorString = NULL +cdef void* __nvvmVersion = NULL +cdef void* __nvvmIRVersion = NULL +cdef void* __nvvmCreateProgram = NULL +cdef void* __nvvmDestroyProgram = NULL +cdef void* __nvvmAddModuleToProgram = NULL +cdef void* __nvvmLazyAddModuleToProgram = NULL +cdef void* __nvvmCompileProgram = NULL +cdef void* __nvvmVerifyProgram = NULL +cdef void* __nvvmGetCompiledResultSize = NULL +cdef void* __nvvmGetCompiledResult = NULL +cdef void* __nvvmGetProgramLogSize = NULL +cdef void* __nvvmGetProgramLog = NULL + + +cdef void* load_library() except* with gil: + cdef uintptr_t handle = load_nvidia_dynamic_lib("nvvm")._handle_uint + return handle + + +cdef int __check_or_init_nvvm() except -1 nogil: + global __py_nvvm_init + + cdef void* handle = NULL + + with gil, __symbol_lock: + # Load function + global __nvvmGetErrorString + __nvvmGetErrorString = dlsym(RTLD_DEFAULT, 'nvvmGetErrorString') + if __nvvmGetErrorString == NULL: + if handle == NULL: + handle = load_library() + __nvvmGetErrorString = dlsym(handle, 'nvvmGetErrorString') + + global __nvvmVersion + __nvvmVersion = dlsym(RTLD_DEFAULT, 'nvvmVersion') + if __nvvmVersion == NULL: + if handle == NULL: + handle = load_library() + __nvvmVersion = dlsym(handle, 'nvvmVersion') + + global __nvvmIRVersion + __nvvmIRVersion = dlsym(RTLD_DEFAULT, 'nvvmIRVersion') + if __nvvmIRVersion == NULL: + if handle == NULL: + handle = load_library() + __nvvmIRVersion = dlsym(handle, 'nvvmIRVersion') + + global __nvvmCreateProgram + __nvvmCreateProgram = dlsym(RTLD_DEFAULT, 'nvvmCreateProgram') + if __nvvmCreateProgram == NULL: + if handle == NULL: + handle = load_library() + __nvvmCreateProgram = dlsym(handle, 'nvvmCreateProgram') + + global __nvvmDestroyProgram + __nvvmDestroyProgram = dlsym(RTLD_DEFAULT, 'nvvmDestroyProgram') + if __nvvmDestroyProgram == NULL: + if handle == NULL: + handle = load_library() + __nvvmDestroyProgram = dlsym(handle, 'nvvmDestroyProgram') + + global __nvvmAddModuleToProgram + __nvvmAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmAddModuleToProgram') + if __nvvmAddModuleToProgram == NULL: + if handle == NULL: + handle = load_library() + __nvvmAddModuleToProgram = dlsym(handle, 'nvvmAddModuleToProgram') + + global __nvvmLazyAddModuleToProgram + __nvvmLazyAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmLazyAddModuleToProgram') + if __nvvmLazyAddModuleToProgram == NULL: + if handle == NULL: + handle = load_library() + __nvvmLazyAddModuleToProgram = dlsym(handle, 'nvvmLazyAddModuleToProgram') + + global __nvvmCompileProgram + __nvvmCompileProgram = dlsym(RTLD_DEFAULT, 'nvvmCompileProgram') + if __nvvmCompileProgram == NULL: + if handle == NULL: + handle = load_library() + __nvvmCompileProgram = dlsym(handle, 'nvvmCompileProgram') + + global __nvvmVerifyProgram + __nvvmVerifyProgram = dlsym(RTLD_DEFAULT, 'nvvmVerifyProgram') + if __nvvmVerifyProgram == NULL: + if handle == NULL: + handle = load_library() + __nvvmVerifyProgram = dlsym(handle, 'nvvmVerifyProgram') + + global __nvvmGetCompiledResultSize + __nvvmGetCompiledResultSize = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResultSize') + if __nvvmGetCompiledResultSize == NULL: + if handle == NULL: + handle = load_library() + __nvvmGetCompiledResultSize = dlsym(handle, 'nvvmGetCompiledResultSize') + + global __nvvmGetCompiledResult + __nvvmGetCompiledResult = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResult') + if __nvvmGetCompiledResult == NULL: + if handle == NULL: + handle = load_library() + __nvvmGetCompiledResult = dlsym(handle, 'nvvmGetCompiledResult') + + global __nvvmGetProgramLogSize + __nvvmGetProgramLogSize = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLogSize') + if __nvvmGetProgramLogSize == NULL: + if handle == NULL: + handle = load_library() + __nvvmGetProgramLogSize = dlsym(handle, 'nvvmGetProgramLogSize') + + global __nvvmGetProgramLog + __nvvmGetProgramLog = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLog') + if __nvvmGetProgramLog == NULL: + if handle == NULL: + handle = load_library() + __nvvmGetProgramLog = dlsym(handle, 'nvvmGetProgramLog') + + __py_nvvm_init = True + return 0 + + +cdef int _check_or_init_nvvm() except -1 nogil: + if __py_nvvm_init: + return 0 + + return __check_or_init_nvvm() + + +cdef dict func_ptrs = None + + +cpdef dict _inspect_function_pointers(): + global func_ptrs + if func_ptrs is not None: + return func_ptrs + + _check_or_init_nvvm() + cdef dict data = {} + + global __nvvmGetErrorString + data["__nvvmGetErrorString"] = __nvvmGetErrorString + + global __nvvmVersion + data["__nvvmVersion"] = __nvvmVersion + + global __nvvmIRVersion + data["__nvvmIRVersion"] = __nvvmIRVersion + + global __nvvmCreateProgram + data["__nvvmCreateProgram"] = __nvvmCreateProgram + + global __nvvmDestroyProgram + data["__nvvmDestroyProgram"] = __nvvmDestroyProgram + + global __nvvmAddModuleToProgram + data["__nvvmAddModuleToProgram"] = __nvvmAddModuleToProgram + + global __nvvmLazyAddModuleToProgram + data["__nvvmLazyAddModuleToProgram"] = __nvvmLazyAddModuleToProgram + + global __nvvmCompileProgram + data["__nvvmCompileProgram"] = __nvvmCompileProgram + + global __nvvmVerifyProgram + data["__nvvmVerifyProgram"] = __nvvmVerifyProgram + + global __nvvmGetCompiledResultSize + data["__nvvmGetCompiledResultSize"] = __nvvmGetCompiledResultSize + + global __nvvmGetCompiledResult + data["__nvvmGetCompiledResult"] = __nvvmGetCompiledResult + + global __nvvmGetProgramLogSize + data["__nvvmGetProgramLogSize"] = __nvvmGetProgramLogSize + + global __nvvmGetProgramLog + data["__nvvmGetProgramLog"] = __nvvmGetProgramLog + + func_ptrs = data + return data + + +cpdef _inspect_function_pointer(str name): + global func_ptrs + if func_ptrs is None: + func_ptrs = _inspect_function_pointers() + return func_ptrs[name] + + +############################################################################### +# Wrapper functions +############################################################################### + +cdef const char* _nvvmGetErrorString(nvvmResult result) except?NULL nogil: + global __nvvmGetErrorString + _check_or_init_nvvm() + if __nvvmGetErrorString == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetErrorString is not found") + return (__nvvmGetErrorString)( + result) + + +cdef nvvmResult _nvvmVersion(int* major, int* minor) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmVersion + _check_or_init_nvvm() + if __nvvmVersion == NULL: + with gil: + raise FunctionNotFoundError("function nvvmVersion is not found") + return (__nvvmVersion)( + major, minor) + + +cdef nvvmResult _nvvmIRVersion(int* majorIR, int* minorIR, int* majorDbg, int* minorDbg) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmIRVersion + _check_or_init_nvvm() + if __nvvmIRVersion == NULL: + with gil: + raise FunctionNotFoundError("function nvvmIRVersion is not found") + return (__nvvmIRVersion)( + majorIR, minorIR, majorDbg, minorDbg) + + +cdef nvvmResult _nvvmCreateProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmCreateProgram + _check_or_init_nvvm() + if __nvvmCreateProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmCreateProgram is not found") + return (__nvvmCreateProgram)( + prog) + + +cdef nvvmResult _nvvmDestroyProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmDestroyProgram + _check_or_init_nvvm() + if __nvvmDestroyProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmDestroyProgram is not found") + return (__nvvmDestroyProgram)( + prog) + + +cdef nvvmResult _nvvmAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmAddModuleToProgram + _check_or_init_nvvm() + if __nvvmAddModuleToProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmAddModuleToProgram is not found") + return (__nvvmAddModuleToProgram)( + prog, buffer, size, name) + + +cdef nvvmResult _nvvmLazyAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmLazyAddModuleToProgram + _check_or_init_nvvm() + if __nvvmLazyAddModuleToProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmLazyAddModuleToProgram is not found") + return (__nvvmLazyAddModuleToProgram)( + prog, buffer, size, name) + + +cdef nvvmResult _nvvmCompileProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmCompileProgram + _check_or_init_nvvm() + if __nvvmCompileProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmCompileProgram is not found") + return (__nvvmCompileProgram)( + prog, numOptions, options) + + +cdef nvvmResult _nvvmVerifyProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmVerifyProgram + _check_or_init_nvvm() + if __nvvmVerifyProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmVerifyProgram is not found") + return (__nvvmVerifyProgram)( + prog, numOptions, options) + + +cdef nvvmResult _nvvmGetCompiledResultSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmGetCompiledResultSize + _check_or_init_nvvm() + if __nvvmGetCompiledResultSize == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetCompiledResultSize is not found") + return (__nvvmGetCompiledResultSize)( + prog, bufferSizeRet) + + +cdef nvvmResult _nvvmGetCompiledResult(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmGetCompiledResult + _check_or_init_nvvm() + if __nvvmGetCompiledResult == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetCompiledResult is not found") + return (__nvvmGetCompiledResult)( + prog, buffer) + + +cdef nvvmResult _nvvmGetProgramLogSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmGetProgramLogSize + _check_or_init_nvvm() + if __nvvmGetProgramLogSize == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetProgramLogSize is not found") + return (__nvvmGetProgramLogSize)( + prog, bufferSizeRet) + + +cdef nvvmResult _nvvmGetProgramLog(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmGetProgramLog + _check_or_init_nvvm() + if __nvvmGetProgramLog == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetProgramLog is not found") + return (__nvvmGetProgramLog)( + prog, buffer) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm_linux.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm_linux.pyx new file mode 100644 index 0000000000000000000000000000000000000000..7e82d2a9aa216eb3586e7f94a0339e1d9ed6b50d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm_linux.pyx @@ -0,0 +1,381 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.0.1 to 12.9.1. Do not modify it directly. + +from libc.stdint cimport intptr_t, uintptr_t + +import threading +from .utils import FunctionNotFoundError, NotSupportedError + +from cuda.pathfinder import load_nvidia_dynamic_lib + + +############################################################################### +# Extern +############################################################################### + +cdef extern from "" nogil: + void* dlopen(const char*, int) + char* dlerror() + void* dlsym(void*, const char*) + int dlclose(void*) + + enum: + RTLD_LAZY + RTLD_NOW + RTLD_GLOBAL + RTLD_LOCAL + + const void* RTLD_DEFAULT 'RTLD_DEFAULT' + +cdef int get_cuda_version(): + cdef void* handle = NULL + cdef int err, driver_ver = 0 + + # Load driver to check version + handle = dlopen('libcuda.so.1', RTLD_NOW | RTLD_GLOBAL) + if handle == NULL: + err_msg = dlerror() + raise NotSupportedError(f'CUDA driver is not found ({err_msg.decode()})') + cuDriverGetVersion = dlsym(handle, "cuDriverGetVersion") + if cuDriverGetVersion == NULL: + raise RuntimeError('Did not find cuDriverGetVersion symbol in libcuda.so.1') + err = (cuDriverGetVersion)(&driver_ver) + if err != 0: + raise RuntimeError(f'cuDriverGetVersion returned error code {err}') + + return driver_ver + + +############################################################################### +# Wrapper init +############################################################################### + +cdef object __symbol_lock = threading.Lock() +cdef bint __py_nvvm_init = False +cdef void* __cuDriverGetVersion = NULL + +cdef void* __nvvmGetErrorString = NULL +cdef void* __nvvmVersion = NULL +cdef void* __nvvmIRVersion = NULL +cdef void* __nvvmCreateProgram = NULL +cdef void* __nvvmDestroyProgram = NULL +cdef void* __nvvmAddModuleToProgram = NULL +cdef void* __nvvmLazyAddModuleToProgram = NULL +cdef void* __nvvmCompileProgram = NULL +cdef void* __nvvmVerifyProgram = NULL +cdef void* __nvvmGetCompiledResultSize = NULL +cdef void* __nvvmGetCompiledResult = NULL +cdef void* __nvvmGetProgramLogSize = NULL +cdef void* __nvvmGetProgramLog = NULL + + +cdef void* load_library() except* with gil: + cdef uintptr_t handle = load_nvidia_dynamic_lib("nvvm")._handle_uint + return handle + + +cdef int __check_or_init_nvvm() except -1 nogil: + global __py_nvvm_init + + cdef void* handle = NULL + + with gil, __symbol_lock: + # Load function + global __nvvmGetErrorString + __nvvmGetErrorString = dlsym(RTLD_DEFAULT, 'nvvmGetErrorString') + if __nvvmGetErrorString == NULL: + if handle == NULL: + handle = load_library() + __nvvmGetErrorString = dlsym(handle, 'nvvmGetErrorString') + + global __nvvmVersion + __nvvmVersion = dlsym(RTLD_DEFAULT, 'nvvmVersion') + if __nvvmVersion == NULL: + if handle == NULL: + handle = load_library() + __nvvmVersion = dlsym(handle, 'nvvmVersion') + + global __nvvmIRVersion + __nvvmIRVersion = dlsym(RTLD_DEFAULT, 'nvvmIRVersion') + if __nvvmIRVersion == NULL: + if handle == NULL: + handle = load_library() + __nvvmIRVersion = dlsym(handle, 'nvvmIRVersion') + + global __nvvmCreateProgram + __nvvmCreateProgram = dlsym(RTLD_DEFAULT, 'nvvmCreateProgram') + if __nvvmCreateProgram == NULL: + if handle == NULL: + handle = load_library() + __nvvmCreateProgram = dlsym(handle, 'nvvmCreateProgram') + + global __nvvmDestroyProgram + __nvvmDestroyProgram = dlsym(RTLD_DEFAULT, 'nvvmDestroyProgram') + if __nvvmDestroyProgram == NULL: + if handle == NULL: + handle = load_library() + __nvvmDestroyProgram = dlsym(handle, 'nvvmDestroyProgram') + + global __nvvmAddModuleToProgram + __nvvmAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmAddModuleToProgram') + if __nvvmAddModuleToProgram == NULL: + if handle == NULL: + handle = load_library() + __nvvmAddModuleToProgram = dlsym(handle, 'nvvmAddModuleToProgram') + + global __nvvmLazyAddModuleToProgram + __nvvmLazyAddModuleToProgram = dlsym(RTLD_DEFAULT, 'nvvmLazyAddModuleToProgram') + if __nvvmLazyAddModuleToProgram == NULL: + if handle == NULL: + handle = load_library() + __nvvmLazyAddModuleToProgram = dlsym(handle, 'nvvmLazyAddModuleToProgram') + + global __nvvmCompileProgram + __nvvmCompileProgram = dlsym(RTLD_DEFAULT, 'nvvmCompileProgram') + if __nvvmCompileProgram == NULL: + if handle == NULL: + handle = load_library() + __nvvmCompileProgram = dlsym(handle, 'nvvmCompileProgram') + + global __nvvmVerifyProgram + __nvvmVerifyProgram = dlsym(RTLD_DEFAULT, 'nvvmVerifyProgram') + if __nvvmVerifyProgram == NULL: + if handle == NULL: + handle = load_library() + __nvvmVerifyProgram = dlsym(handle, 'nvvmVerifyProgram') + + global __nvvmGetCompiledResultSize + __nvvmGetCompiledResultSize = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResultSize') + if __nvvmGetCompiledResultSize == NULL: + if handle == NULL: + handle = load_library() + __nvvmGetCompiledResultSize = dlsym(handle, 'nvvmGetCompiledResultSize') + + global __nvvmGetCompiledResult + __nvvmGetCompiledResult = dlsym(RTLD_DEFAULT, 'nvvmGetCompiledResult') + if __nvvmGetCompiledResult == NULL: + if handle == NULL: + handle = load_library() + __nvvmGetCompiledResult = dlsym(handle, 'nvvmGetCompiledResult') + + global __nvvmGetProgramLogSize + __nvvmGetProgramLogSize = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLogSize') + if __nvvmGetProgramLogSize == NULL: + if handle == NULL: + handle = load_library() + __nvvmGetProgramLogSize = dlsym(handle, 'nvvmGetProgramLogSize') + + global __nvvmGetProgramLog + __nvvmGetProgramLog = dlsym(RTLD_DEFAULT, 'nvvmGetProgramLog') + if __nvvmGetProgramLog == NULL: + if handle == NULL: + handle = load_library() + __nvvmGetProgramLog = dlsym(handle, 'nvvmGetProgramLog') + + __py_nvvm_init = True + return 0 + + +cdef int _check_or_init_nvvm() except -1 nogil: + if __py_nvvm_init: + return 0 + + return __check_or_init_nvvm() + + +cdef dict func_ptrs = None + + +cpdef dict _inspect_function_pointers(): + global func_ptrs + if func_ptrs is not None: + return func_ptrs + + _check_or_init_nvvm() + cdef dict data = {} + + global __nvvmGetErrorString + data["__nvvmGetErrorString"] = __nvvmGetErrorString + + global __nvvmVersion + data["__nvvmVersion"] = __nvvmVersion + + global __nvvmIRVersion + data["__nvvmIRVersion"] = __nvvmIRVersion + + global __nvvmCreateProgram + data["__nvvmCreateProgram"] = __nvvmCreateProgram + + global __nvvmDestroyProgram + data["__nvvmDestroyProgram"] = __nvvmDestroyProgram + + global __nvvmAddModuleToProgram + data["__nvvmAddModuleToProgram"] = __nvvmAddModuleToProgram + + global __nvvmLazyAddModuleToProgram + data["__nvvmLazyAddModuleToProgram"] = __nvvmLazyAddModuleToProgram + + global __nvvmCompileProgram + data["__nvvmCompileProgram"] = __nvvmCompileProgram + + global __nvvmVerifyProgram + data["__nvvmVerifyProgram"] = __nvvmVerifyProgram + + global __nvvmGetCompiledResultSize + data["__nvvmGetCompiledResultSize"] = __nvvmGetCompiledResultSize + + global __nvvmGetCompiledResult + data["__nvvmGetCompiledResult"] = __nvvmGetCompiledResult + + global __nvvmGetProgramLogSize + data["__nvvmGetProgramLogSize"] = __nvvmGetProgramLogSize + + global __nvvmGetProgramLog + data["__nvvmGetProgramLog"] = __nvvmGetProgramLog + + func_ptrs = data + return data + + +cpdef _inspect_function_pointer(str name): + global func_ptrs + if func_ptrs is None: + func_ptrs = _inspect_function_pointers() + return func_ptrs[name] + + +############################################################################### +# Wrapper functions +############################################################################### + +cdef const char* _nvvmGetErrorString(nvvmResult result) except?NULL nogil: + global __nvvmGetErrorString + _check_or_init_nvvm() + if __nvvmGetErrorString == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetErrorString is not found") + return (__nvvmGetErrorString)( + result) + + +cdef nvvmResult _nvvmVersion(int* major, int* minor) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmVersion + _check_or_init_nvvm() + if __nvvmVersion == NULL: + with gil: + raise FunctionNotFoundError("function nvvmVersion is not found") + return (__nvvmVersion)( + major, minor) + + +cdef nvvmResult _nvvmIRVersion(int* majorIR, int* minorIR, int* majorDbg, int* minorDbg) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmIRVersion + _check_or_init_nvvm() + if __nvvmIRVersion == NULL: + with gil: + raise FunctionNotFoundError("function nvvmIRVersion is not found") + return (__nvvmIRVersion)( + majorIR, minorIR, majorDbg, minorDbg) + + +cdef nvvmResult _nvvmCreateProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmCreateProgram + _check_or_init_nvvm() + if __nvvmCreateProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmCreateProgram is not found") + return (__nvvmCreateProgram)( + prog) + + +cdef nvvmResult _nvvmDestroyProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmDestroyProgram + _check_or_init_nvvm() + if __nvvmDestroyProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmDestroyProgram is not found") + return (__nvvmDestroyProgram)( + prog) + + +cdef nvvmResult _nvvmAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmAddModuleToProgram + _check_or_init_nvvm() + if __nvvmAddModuleToProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmAddModuleToProgram is not found") + return (__nvvmAddModuleToProgram)( + prog, buffer, size, name) + + +cdef nvvmResult _nvvmLazyAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmLazyAddModuleToProgram + _check_or_init_nvvm() + if __nvvmLazyAddModuleToProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmLazyAddModuleToProgram is not found") + return (__nvvmLazyAddModuleToProgram)( + prog, buffer, size, name) + + +cdef nvvmResult _nvvmCompileProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmCompileProgram + _check_or_init_nvvm() + if __nvvmCompileProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmCompileProgram is not found") + return (__nvvmCompileProgram)( + prog, numOptions, options) + + +cdef nvvmResult _nvvmVerifyProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmVerifyProgram + _check_or_init_nvvm() + if __nvvmVerifyProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmVerifyProgram is not found") + return (__nvvmVerifyProgram)( + prog, numOptions, options) + + +cdef nvvmResult _nvvmGetCompiledResultSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmGetCompiledResultSize + _check_or_init_nvvm() + if __nvvmGetCompiledResultSize == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetCompiledResultSize is not found") + return (__nvvmGetCompiledResultSize)( + prog, bufferSizeRet) + + +cdef nvvmResult _nvvmGetCompiledResult(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmGetCompiledResult + _check_or_init_nvvm() + if __nvvmGetCompiledResult == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetCompiledResult is not found") + return (__nvvmGetCompiledResult)( + prog, buffer) + + +cdef nvvmResult _nvvmGetProgramLogSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmGetProgramLogSize + _check_or_init_nvvm() + if __nvvmGetProgramLogSize == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetProgramLogSize is not found") + return (__nvvmGetProgramLogSize)( + prog, bufferSizeRet) + + +cdef nvvmResult _nvvmGetProgramLog(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmGetProgramLog + _check_or_init_nvvm() + if __nvvmGetProgramLog == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetProgramLog is not found") + return (__nvvmGetProgramLog)( + prog, buffer) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm_windows.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm_windows.pyx new file mode 100644 index 0000000000000000000000000000000000000000..a667642958699d4289beff162934e98b7c1cdf8e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/nvvm_windows.pyx @@ -0,0 +1,345 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.0.1 to 12.9.1. Do not modify it directly. + +from libc.stdint cimport intptr_t + +import threading +from .utils import FunctionNotFoundError, NotSupportedError + +from cuda.pathfinder import load_nvidia_dynamic_lib + +from libc.stddef cimport wchar_t +from libc.stdint cimport uintptr_t +from cpython cimport PyUnicode_AsWideCharString, PyMem_Free + +from .utils import NotSupportedError + +cdef extern from "windows.h" nogil: + ctypedef void* HMODULE + ctypedef void* HANDLE + ctypedef void* FARPROC + ctypedef unsigned long DWORD + ctypedef const wchar_t *LPCWSTR + ctypedef const char *LPCSTR + + cdef DWORD LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800 + cdef DWORD LOAD_LIBRARY_SEARCH_DEFAULT_DIRS = 0x00001000 + cdef DWORD LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR = 0x00000100 + + HMODULE _LoadLibraryExW "LoadLibraryExW"( + LPCWSTR lpLibFileName, + HANDLE hFile, + DWORD dwFlags + ) + + FARPROC _GetProcAddress "GetProcAddress"(HMODULE hModule, LPCSTR lpProcName) + +cdef inline uintptr_t LoadLibraryExW(str path, HANDLE hFile, DWORD dwFlags): + cdef uintptr_t result + cdef wchar_t* wpath = PyUnicode_AsWideCharString(path, NULL) + with nogil: + result = _LoadLibraryExW( + wpath, + hFile, + dwFlags + ) + PyMem_Free(wpath) + return result + +cdef inline void *GetProcAddress(uintptr_t hModule, const char* lpProcName) nogil: + return _GetProcAddress(hModule, lpProcName) + +cdef int get_cuda_version(): + cdef int err, driver_ver = 0 + + # Load driver to check version + handle = LoadLibraryExW("nvcuda.dll", NULL, LOAD_LIBRARY_SEARCH_SYSTEM32) + if handle == 0: + raise NotSupportedError('CUDA driver is not found') + cuDriverGetVersion = GetProcAddress(handle, 'cuDriverGetVersion') + if cuDriverGetVersion == NULL: + raise RuntimeError('something went wrong') + err = (cuDriverGetVersion)(&driver_ver) + if err != 0: + raise RuntimeError('something went wrong') + + return driver_ver + + + +############################################################################### +# Wrapper init +############################################################################### + +cdef object __symbol_lock = threading.Lock() +cdef bint __py_nvvm_init = False + +cdef void* __nvvmGetErrorString = NULL +cdef void* __nvvmVersion = NULL +cdef void* __nvvmIRVersion = NULL +cdef void* __nvvmCreateProgram = NULL +cdef void* __nvvmDestroyProgram = NULL +cdef void* __nvvmAddModuleToProgram = NULL +cdef void* __nvvmLazyAddModuleToProgram = NULL +cdef void* __nvvmCompileProgram = NULL +cdef void* __nvvmVerifyProgram = NULL +cdef void* __nvvmGetCompiledResultSize = NULL +cdef void* __nvvmGetCompiledResult = NULL +cdef void* __nvvmGetProgramLogSize = NULL +cdef void* __nvvmGetProgramLog = NULL + + +cdef int __check_or_init_nvvm() except -1 nogil: + global __py_nvvm_init + + with gil, __symbol_lock: + # Load library + handle = load_nvidia_dynamic_lib("nvvm")._handle_uint + + # Load function + global __nvvmGetErrorString + __nvvmGetErrorString = GetProcAddress(handle, 'nvvmGetErrorString') + + global __nvvmVersion + __nvvmVersion = GetProcAddress(handle, 'nvvmVersion') + + global __nvvmIRVersion + __nvvmIRVersion = GetProcAddress(handle, 'nvvmIRVersion') + + global __nvvmCreateProgram + __nvvmCreateProgram = GetProcAddress(handle, 'nvvmCreateProgram') + + global __nvvmDestroyProgram + __nvvmDestroyProgram = GetProcAddress(handle, 'nvvmDestroyProgram') + + global __nvvmAddModuleToProgram + __nvvmAddModuleToProgram = GetProcAddress(handle, 'nvvmAddModuleToProgram') + + global __nvvmLazyAddModuleToProgram + __nvvmLazyAddModuleToProgram = GetProcAddress(handle, 'nvvmLazyAddModuleToProgram') + + global __nvvmCompileProgram + __nvvmCompileProgram = GetProcAddress(handle, 'nvvmCompileProgram') + + global __nvvmVerifyProgram + __nvvmVerifyProgram = GetProcAddress(handle, 'nvvmVerifyProgram') + + global __nvvmGetCompiledResultSize + __nvvmGetCompiledResultSize = GetProcAddress(handle, 'nvvmGetCompiledResultSize') + + global __nvvmGetCompiledResult + __nvvmGetCompiledResult = GetProcAddress(handle, 'nvvmGetCompiledResult') + + global __nvvmGetProgramLogSize + __nvvmGetProgramLogSize = GetProcAddress(handle, 'nvvmGetProgramLogSize') + + global __nvvmGetProgramLog + __nvvmGetProgramLog = GetProcAddress(handle, 'nvvmGetProgramLog') + + __py_nvvm_init = True + return 0 + + +cdef inline int _check_or_init_nvvm() except -1 nogil: + if __py_nvvm_init: + return 0 + + return __check_or_init_nvvm() + + +cdef dict func_ptrs = None + + +cpdef dict _inspect_function_pointers(): + global func_ptrs + if func_ptrs is not None: + return func_ptrs + + _check_or_init_nvvm() + cdef dict data = {} + + global __nvvmGetErrorString + data["__nvvmGetErrorString"] = __nvvmGetErrorString + + global __nvvmVersion + data["__nvvmVersion"] = __nvvmVersion + + global __nvvmIRVersion + data["__nvvmIRVersion"] = __nvvmIRVersion + + global __nvvmCreateProgram + data["__nvvmCreateProgram"] = __nvvmCreateProgram + + global __nvvmDestroyProgram + data["__nvvmDestroyProgram"] = __nvvmDestroyProgram + + global __nvvmAddModuleToProgram + data["__nvvmAddModuleToProgram"] = __nvvmAddModuleToProgram + + global __nvvmLazyAddModuleToProgram + data["__nvvmLazyAddModuleToProgram"] = __nvvmLazyAddModuleToProgram + + global __nvvmCompileProgram + data["__nvvmCompileProgram"] = __nvvmCompileProgram + + global __nvvmVerifyProgram + data["__nvvmVerifyProgram"] = __nvvmVerifyProgram + + global __nvvmGetCompiledResultSize + data["__nvvmGetCompiledResultSize"] = __nvvmGetCompiledResultSize + + global __nvvmGetCompiledResult + data["__nvvmGetCompiledResult"] = __nvvmGetCompiledResult + + global __nvvmGetProgramLogSize + data["__nvvmGetProgramLogSize"] = __nvvmGetProgramLogSize + + global __nvvmGetProgramLog + data["__nvvmGetProgramLog"] = __nvvmGetProgramLog + + func_ptrs = data + return data + + +cpdef _inspect_function_pointer(str name): + global func_ptrs + if func_ptrs is None: + func_ptrs = _inspect_function_pointers() + return func_ptrs[name] + + +############################################################################### +# Wrapper functions +############################################################################### + +cdef const char* _nvvmGetErrorString(nvvmResult result) except?NULL nogil: + global __nvvmGetErrorString + _check_or_init_nvvm() + if __nvvmGetErrorString == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetErrorString is not found") + return (__nvvmGetErrorString)( + result) + + +cdef nvvmResult _nvvmVersion(int* major, int* minor) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmVersion + _check_or_init_nvvm() + if __nvvmVersion == NULL: + with gil: + raise FunctionNotFoundError("function nvvmVersion is not found") + return (__nvvmVersion)( + major, minor) + + +cdef nvvmResult _nvvmIRVersion(int* majorIR, int* minorIR, int* majorDbg, int* minorDbg) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmIRVersion + _check_or_init_nvvm() + if __nvvmIRVersion == NULL: + with gil: + raise FunctionNotFoundError("function nvvmIRVersion is not found") + return (__nvvmIRVersion)( + majorIR, minorIR, majorDbg, minorDbg) + + +cdef nvvmResult _nvvmCreateProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmCreateProgram + _check_or_init_nvvm() + if __nvvmCreateProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmCreateProgram is not found") + return (__nvvmCreateProgram)( + prog) + + +cdef nvvmResult _nvvmDestroyProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmDestroyProgram + _check_or_init_nvvm() + if __nvvmDestroyProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmDestroyProgram is not found") + return (__nvvmDestroyProgram)( + prog) + + +cdef nvvmResult _nvvmAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmAddModuleToProgram + _check_or_init_nvvm() + if __nvvmAddModuleToProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmAddModuleToProgram is not found") + return (__nvvmAddModuleToProgram)( + prog, buffer, size, name) + + +cdef nvvmResult _nvvmLazyAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmLazyAddModuleToProgram + _check_or_init_nvvm() + if __nvvmLazyAddModuleToProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmLazyAddModuleToProgram is not found") + return (__nvvmLazyAddModuleToProgram)( + prog, buffer, size, name) + + +cdef nvvmResult _nvvmCompileProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmCompileProgram + _check_or_init_nvvm() + if __nvvmCompileProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmCompileProgram is not found") + return (__nvvmCompileProgram)( + prog, numOptions, options) + + +cdef nvvmResult _nvvmVerifyProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmVerifyProgram + _check_or_init_nvvm() + if __nvvmVerifyProgram == NULL: + with gil: + raise FunctionNotFoundError("function nvvmVerifyProgram is not found") + return (__nvvmVerifyProgram)( + prog, numOptions, options) + + +cdef nvvmResult _nvvmGetCompiledResultSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmGetCompiledResultSize + _check_or_init_nvvm() + if __nvvmGetCompiledResultSize == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetCompiledResultSize is not found") + return (__nvvmGetCompiledResultSize)( + prog, bufferSizeRet) + + +cdef nvvmResult _nvvmGetCompiledResult(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmGetCompiledResult + _check_or_init_nvvm() + if __nvvmGetCompiledResult == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetCompiledResult is not found") + return (__nvvmGetCompiledResult)( + prog, buffer) + + +cdef nvvmResult _nvvmGetProgramLogSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmGetProgramLogSize + _check_or_init_nvvm() + if __nvvmGetProgramLogSize == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetProgramLogSize is not found") + return (__nvvmGetProgramLogSize)( + prog, bufferSizeRet) + + +cdef nvvmResult _nvvmGetProgramLog(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + global __nvvmGetProgramLog + _check_or_init_nvvm() + if __nvvmGetProgramLog == NULL: + with gil: + raise FunctionNotFoundError("function nvvmGetProgramLog is not found") + return (__nvvmGetProgramLog)( + prog, buffer) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/utils.cpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/utils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b3861bf63b6e4e6949d317447cf9d8e41950e185 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/utils.cpp @@ -0,0 +1,12909 @@ +/* Generated by Cython 3.1.5 */ + +/* BEGIN: Cython Metadata +{ + "distutils": { + "depends": [], + "extra_compile_args": [ + "-std=c++14", + "-fpermissive", + "-Wno-deprecated-declarations", + "-fno-var-tracking-assignments", + "-O3" + ], + "include_dirs": [ + "/opt/python/cp312-cp312/include", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/include" + ], + "language": "c++", + "library_dirs": [ + "/tmp/build-env-3t3_8d9r/lib/python3.12/site-packages", + "/tmp/build-env-3t3_8d9r/lib", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib64", + "/host//home/runner/_work/cuda-python/cuda-python/cuda_toolkit/lib" + ], + "name": "cuda.bindings._internal.utils", + "sources": [ + "cuda/bindings/_internal/utils.pyx" + ] + }, + "module_name": "cuda.bindings._internal.utils" +} +END: Cython Metadata */ + +#ifndef PY_SSIZE_T_CLEAN +#define PY_SSIZE_T_CLEAN +#endif /* PY_SSIZE_T_CLEAN */ +/* InitLimitedAPI */ +#if defined(Py_LIMITED_API) + #if !defined(CYTHON_LIMITED_API) + #define CYTHON_LIMITED_API 1 + #endif +#elif defined(CYTHON_LIMITED_API) + #ifdef _MSC_VER + #pragma message ("Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead.") + #else + #warning Limited API usage is enabled with 'CYTHON_LIMITED_API' but 'Py_LIMITED_API' does not define a Python target version. Consider setting 'Py_LIMITED_API' instead. + #endif +#endif + +#include "Python.h" +#ifndef Py_PYTHON_H + #error Python headers needed to compile C extensions, please install development version of Python. +#elif PY_VERSION_HEX < 0x03080000 + #error Cython requires Python 3.8+. +#else +#define __PYX_ABI_VERSION "3_1_5" +#define CYTHON_HEX_VERSION 0x030105F0 +#define CYTHON_FUTURE_DIVISION 1 +/* CModulePreamble */ +#include +#ifndef offsetof + #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) +#endif +#if !defined(_WIN32) && !defined(WIN32) && !defined(MS_WINDOWS) + #ifndef __stdcall + #define __stdcall + #endif + #ifndef __cdecl + #define __cdecl + #endif + #ifndef __fastcall + #define __fastcall + #endif +#endif +#ifndef DL_IMPORT + #define DL_IMPORT(t) t +#endif +#ifndef DL_EXPORT + #define DL_EXPORT(t) t +#endif +#define __PYX_COMMA , +#ifndef HAVE_LONG_LONG + #define HAVE_LONG_LONG +#endif +#ifndef PY_LONG_LONG + #define PY_LONG_LONG LONG_LONG +#endif +#ifndef Py_HUGE_VAL + #define Py_HUGE_VAL HUGE_VAL +#endif +#define __PYX_LIMITED_VERSION_HEX PY_VERSION_HEX +#if defined(GRAALVM_PYTHON) + /* For very preliminary testing purposes. Most variables are set the same as PyPy. + The existence of this section does not imply that anything works or is even tested */ + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 1 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(PYPY_VERSION) + #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #if PY_VERSION_HEX < 0x03090000 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #elif !defined(CYTHON_PEP489_MULTI_PHASE_INIT) + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #undef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE (PYPY_VERSION_NUM >= 0x07030C00) + #endif + #undef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 0 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC (PYPY_VERSION_NUM >= 0x07031100) + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#elif defined(CYTHON_LIMITED_API) + #ifdef Py_LIMITED_API + #undef __PYX_LIMITED_VERSION_HEX + #define __PYX_LIMITED_VERSION_HEX Py_LIMITED_API + #endif + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + #define CYTHON_COMPILING_IN_GRAAL 0 + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #undef CYTHON_CLINE_IN_TRACEBACK + #define CYTHON_CLINE_IN_TRACEBACK 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 1 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #endif + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #undef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING 0 + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 0 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND (__PYX_LIMITED_VERSION_HEX >= 0x030A0000) + #endif + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 0 + #endif + #undef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS 0 +#else + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 + #define CYTHON_COMPILING_IN_GRAAL 0 + #ifdef Py_GIL_DISABLED + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 1 + #else + #define CYTHON_COMPILING_IN_CPYTHON_FREETHREADING 0 + #endif + #if PY_VERSION_HEX < 0x030A0000 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #elif !defined(CYTHON_USE_TYPE_SLOTS) + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #ifndef CYTHON_USE_TYPE_SPECS + #define CYTHON_USE_TYPE_SPECS 0 + #endif + #ifndef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 1 + #endif + #ifndef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLIST_INTERNALS) + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING || PY_VERSION_HEX >= 0x030B00A2 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 1 + #elif !defined(CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS) + #define CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_ASSUME_SAFE_SIZE + #define CYTHON_ASSUME_SAFE_SIZE 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #elif !defined(CYTHON_FAST_GIL) + #define CYTHON_FAST_GIL (PY_VERSION_HEX < 0x030C00A6) + #endif + #ifndef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif + #ifndef CYTHON_PEP487_INIT_SUBCLASS + #define CYTHON_PEP487_INIT_SUBCLASS 1 + #endif + #ifndef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 1 + #endif + #ifndef CYTHON_USE_MODULE_STATE + #define CYTHON_USE_MODULE_STATE 0 + #endif + #ifndef CYTHON_USE_SYS_MONITORING + #define CYTHON_USE_SYS_MONITORING (PY_VERSION_HEX >= 0x030d00B1) + #endif + #ifndef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #endif + #ifndef CYTHON_USE_AM_SEND + #define CYTHON_USE_AM_SEND 1 + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #elif !defined(CYTHON_USE_DICT_VERSIONS) + #define CYTHON_USE_DICT_VERSIONS (PY_VERSION_HEX < 0x030C00A5 && !CYTHON_USE_MODULE_STATE) + #endif + #ifndef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 1 + #endif + #ifndef CYTHON_UPDATE_DESCRIPTOR_DOC + #define CYTHON_UPDATE_DESCRIPTOR_DOC 1 + #endif + #ifndef CYTHON_USE_FREELISTS + #define CYTHON_USE_FREELISTS (!CYTHON_COMPILING_IN_CPYTHON_FREETHREADING) + #endif +#endif +#ifndef CYTHON_FAST_PYCCALL +#define CYTHON_FAST_PYCCALL CYTHON_FAST_PYCALL +#endif +#ifndef CYTHON_VECTORCALL +#if CYTHON_COMPILING_IN_LIMITED_API +#define CYTHON_VECTORCALL (__PYX_LIMITED_VERSION_HEX >= 0x030C0000) +#else +#define CYTHON_VECTORCALL (CYTHON_FAST_PYCCALL && PY_VERSION_HEX >= 0x030800B1) +#endif +#endif +#define CYTHON_BACKPORT_VECTORCALL (CYTHON_METH_FASTCALL && PY_VERSION_HEX < 0x030800B1) +#if CYTHON_USE_PYLONG_INTERNALS + #undef SHIFT + #undef BASE + #undef MASK + #ifdef SIZEOF_VOID_P + enum { __pyx_check_sizeof_voidp = 1 / (int)(SIZEOF_VOID_P == sizeof(void*)) }; + #endif +#endif +#ifndef CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME + #define CYTHON_LOCK_AND_GIL_DEADLOCK_AVOIDANCE_TIME 100 +#endif +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif +#ifndef __has_cpp_attribute + #define __has_cpp_attribute(x) 0 +#endif +#ifndef CYTHON_RESTRICT + #if defined(__GNUC__) + #define CYTHON_RESTRICT __restrict__ + #elif defined(_MSC_VER) && _MSC_VER >= 1400 + #define CYTHON_RESTRICT __restrict + #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define CYTHON_RESTRICT restrict + #else + #define CYTHON_RESTRICT + #endif +#endif +#ifndef CYTHON_UNUSED + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(maybe_unused) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(maybe_unused) + #define CYTHON_UNUSED [[maybe_unused]] + #endif + #endif + #endif +#endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR + #define CYTHON_MAYBE_UNUSED_VAR(x) CYTHON_UNUSED_VAR(x) +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON && !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_USE_CPP_STD_MOVE + #if defined(__cplusplus) && (\ + __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600)) + #define CYTHON_USE_CPP_STD_MOVE 1 + #else + #define CYTHON_USE_CPP_STD_MOVE 0 + #endif +#endif +#define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) +#ifdef _MSC_VER + #ifndef _MSC_STDINT_H_ + #if _MSC_VER < 1300 + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; + #else + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; + #endif + #endif + #if _MSC_VER < 1300 + #ifdef _WIN64 + typedef unsigned long long __pyx_uintptr_t; + #else + typedef unsigned int __pyx_uintptr_t; + #endif + #else + #ifdef _WIN64 + typedef unsigned __int64 __pyx_uintptr_t; + #else + typedef unsigned __int32 __pyx_uintptr_t; + #endif + #endif +#else + #include + typedef uintptr_t __pyx_uintptr_t; +#endif +#ifndef CYTHON_FALLTHROUGH + #if defined(__cplusplus) + /* for clang __has_cpp_attribute(fallthrough) is true even before C++17 + * but leads to warnings with -pedantic, since it is a C++17 feature */ + #if ((defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) || __cplusplus >= 201703L) + #if __has_cpp_attribute(fallthrough) + #define CYTHON_FALLTHROUGH [[fallthrough]] + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_cpp_attribute(clang::fallthrough) + #define CYTHON_FALLTHROUGH [[clang::fallthrough]] + #elif __has_cpp_attribute(gnu::fallthrough) + #define CYTHON_FALLTHROUGH [[gnu::fallthrough]] + #endif + #endif + #endif + #ifndef CYTHON_FALLTHROUGH + #if __has_attribute(fallthrough) + #define CYTHON_FALLTHROUGH __attribute__((fallthrough)) + #else + #define CYTHON_FALLTHROUGH + #endif + #endif + #if defined(__clang__) && defined(__apple_build_version__) + #if __apple_build_version__ < 7000000 + #undef CYTHON_FALLTHROUGH + #define CYTHON_FALLTHROUGH + #endif + #endif +#endif +#ifndef Py_UNREACHABLE + #define Py_UNREACHABLE() assert(0); abort() +#endif +#ifdef __cplusplus + template + struct __PYX_IS_UNSIGNED_IMPL {static const bool value = T(0) < T(-1);}; + #define __PYX_IS_UNSIGNED(type) (__PYX_IS_UNSIGNED_IMPL::value) +#else + #define __PYX_IS_UNSIGNED(type) (((type)-1) > 0) +#endif +#if CYTHON_COMPILING_IN_PYPY == 1 + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x030A0000) +#else + #define __PYX_NEED_TP_PRINT_SLOT (PY_VERSION_HEX >= 0x030800b4 && PY_VERSION_HEX < 0x03090000) +#endif +#define __PYX_REINTERPRET_FUNCION(func_pointer, other_pointer) ((func_pointer)(void(*)(void))(other_pointer)) + +/* CppInitCode */ +#ifndef __cplusplus + #error "Cython files generated with the C++ option must be compiled with a C++ compiler." +#endif +#ifndef CYTHON_INLINE + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #else + #define CYTHON_INLINE inline + #endif +#endif +template +void __Pyx_call_destructor(T& x) { + x.~T(); +} +template +class __Pyx_FakeReference { + public: + __Pyx_FakeReference() : ptr(NULL) { } + __Pyx_FakeReference(const T& ref) : ptr(const_cast(&ref)) { } + T *operator->() { return ptr; } + T *operator&() { return ptr; } + operator T&() { return *ptr; } + template bool operator ==(const U& other) const { return *ptr == other; } + template bool operator !=(const U& other) const { return *ptr != other; } + template bool operator==(const __Pyx_FakeReference& other) const { return *ptr == *other.ptr; } + template bool operator!=(const __Pyx_FakeReference& other) const { return *ptr != *other.ptr; } + private: + T *ptr; +}; + +/* PythonCompatibility */ +#define __PYX_BUILD_PY_SSIZE_T "n" +#define CYTHON_FORMAT_SSIZE_T "z" +#define __Pyx_BUILTIN_MODULE_NAME "builtins" +#define __Pyx_DefaultClassType PyType_Type +#if CYTHON_COMPILING_IN_LIMITED_API + #ifndef CO_OPTIMIZED + static int CO_OPTIMIZED; + #endif + #ifndef CO_NEWLOCALS + static int CO_NEWLOCALS; + #endif + #ifndef CO_VARARGS + static int CO_VARARGS; + #endif + #ifndef CO_VARKEYWORDS + static int CO_VARKEYWORDS; + #endif + #ifndef CO_ASYNC_GENERATOR + static int CO_ASYNC_GENERATOR; + #endif + #ifndef CO_GENERATOR + static int CO_GENERATOR; + #endif + #ifndef CO_COROUTINE + static int CO_COROUTINE; + #endif +#else + #ifndef CO_COROUTINE + #define CO_COROUTINE 0x80 + #endif + #ifndef CO_ASYNC_GENERATOR + #define CO_ASYNC_GENERATOR 0x200 + #endif +#endif +static int __Pyx_init_co_variables(void); +#if PY_VERSION_HEX >= 0x030900A4 || defined(Py_IS_TYPE) + #define __Pyx_IS_TYPE(ob, type) Py_IS_TYPE(ob, type) +#else + #define __Pyx_IS_TYPE(ob, type) (((const PyObject*)ob)->ob_type == (type)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_Is) + #define __Pyx_Py_Is(x, y) Py_Is(x, y) +#else + #define __Pyx_Py_Is(x, y) ((x) == (y)) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsNone) + #define __Pyx_Py_IsNone(ob) Py_IsNone(ob) +#else + #define __Pyx_Py_IsNone(ob) __Pyx_Py_Is((ob), Py_None) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsTrue) + #define __Pyx_Py_IsTrue(ob) Py_IsTrue(ob) +#else + #define __Pyx_Py_IsTrue(ob) __Pyx_Py_Is((ob), Py_True) +#endif +#if PY_VERSION_HEX >= 0x030A00B1 || defined(Py_IsFalse) + #define __Pyx_Py_IsFalse(ob) Py_IsFalse(ob) +#else + #define __Pyx_Py_IsFalse(ob) __Pyx_Py_Is((ob), Py_False) +#endif +#define __Pyx_NoneAsNull(obj) (__Pyx_Py_IsNone(obj) ? NULL : (obj)) +#if PY_VERSION_HEX >= 0x030900F0 && !CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyObject_GC_IsFinalized(o) PyObject_GC_IsFinalized(o) +#else + #define __Pyx_PyObject_GC_IsFinalized(o) _PyGC_FINALIZED(o) +#endif +#ifndef Py_TPFLAGS_CHECKTYPES + #define Py_TPFLAGS_CHECKTYPES 0 +#endif +#ifndef Py_TPFLAGS_HAVE_INDEX + #define Py_TPFLAGS_HAVE_INDEX 0 +#endif +#ifndef Py_TPFLAGS_HAVE_NEWBUFFER + #define Py_TPFLAGS_HAVE_NEWBUFFER 0 +#endif +#ifndef Py_TPFLAGS_HAVE_FINALIZE + #define Py_TPFLAGS_HAVE_FINALIZE 0 +#endif +#ifndef Py_TPFLAGS_SEQUENCE + #define Py_TPFLAGS_SEQUENCE 0 +#endif +#ifndef Py_TPFLAGS_MAPPING + #define Py_TPFLAGS_MAPPING 0 +#endif +#ifndef METH_STACKLESS + #define METH_STACKLESS 0 +#endif +#ifndef METH_FASTCALL + #ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + #endif + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject *const *args, Py_ssize_t nargs); + typedef PyObject *(*__Pyx_PyCFunctionFastWithKeywords) (PyObject *self, PyObject *const *args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #if PY_VERSION_HEX >= 0x030d00A4 + # define __Pyx_PyCFunctionFast PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords PyCFunctionFastWithKeywords + #else + # define __Pyx_PyCFunctionFast _PyCFunctionFast + # define __Pyx_PyCFunctionFastWithKeywords _PyCFunctionFastWithKeywords + #endif +#endif +#if CYTHON_METH_FASTCALL + #define __Pyx_METH_FASTCALL METH_FASTCALL + #define __Pyx_PyCFunction_FastCall __Pyx_PyCFunctionFast + #define __Pyx_PyCFunction_FastCallWithKeywords __Pyx_PyCFunctionFastWithKeywords +#else + #define __Pyx_METH_FASTCALL METH_VARARGS + #define __Pyx_PyCFunction_FastCall PyCFunction + #define __Pyx_PyCFunction_FastCallWithKeywords PyCFunctionWithKeywords +#endif +#if CYTHON_VECTORCALL + #define __pyx_vectorcallfunc vectorcallfunc + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET PY_VECTORCALL_ARGUMENTS_OFFSET + #define __Pyx_PyVectorcall_NARGS(n) PyVectorcall_NARGS((size_t)(n)) +#elif CYTHON_BACKPORT_VECTORCALL + typedef PyObject *(*__pyx_vectorcallfunc)(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames); + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET ((size_t)1 << (8 * sizeof(size_t) - 1)) + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(((size_t)(n)) & ~__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)) +#else + #define __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET 0 + #define __Pyx_PyVectorcall_NARGS(n) ((Py_ssize_t)(n)) +#endif +#if PY_VERSION_HEX >= 0x030900B1 +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_CheckExact(func) +#else +#define __Pyx_PyCFunction_CheckExact(func) PyCFunction_Check(func) +#endif +#define __Pyx_CyOrPyCFunction_Check(func) PyCFunction_Check(func) +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) (((PyCFunctionObject*)(func))->m_ml->ml_meth) +#elif !CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_CyOrPyCFunction_GET_FUNCTION(func) PyCFunction_GET_FUNCTION(func) +#endif +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_CyOrPyCFunction_GET_FLAGS(func) (((PyCFunctionObject*)(func))->m_ml->ml_flags) +static CYTHON_INLINE PyObject* __Pyx_CyOrPyCFunction_GET_SELF(PyObject *func) { + return (__Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_STATIC) ? NULL : ((PyCFunctionObject*)func)->m_self; +} +#endif +static CYTHON_INLINE int __Pyx__IsSameCFunction(PyObject *func, void (*cfunc)(void)) { +#if CYTHON_COMPILING_IN_LIMITED_API + return PyCFunction_Check(func) && PyCFunction_GetFunction(func) == (PyCFunction) cfunc; +#else + return PyCFunction_Check(func) && PyCFunction_GET_FUNCTION(func) == (PyCFunction) cfunc; +#endif +} +#define __Pyx_IsSameCFunction(func, cfunc) __Pyx__IsSameCFunction(func, cfunc) +#if __PYX_LIMITED_VERSION_HEX < 0x03090000 + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) ((void)m, PyType_FromSpecWithBases(s, b)) + typedef PyObject *(*__Pyx_PyCMethod)(PyObject *, PyTypeObject *, PyObject *const *, size_t, PyObject *); +#else + #define __Pyx_PyType_FromModuleAndSpec(m, s, b) PyType_FromModuleAndSpec(m, s, b) + #define __Pyx_PyCMethod PyCMethod +#endif +#ifndef METH_METHOD + #define METH_METHOD 0x200 +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_GRAAL + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) _PyFrame_SetLineNumber((frame), (lineno)) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE + #define __Pyx_PyThreadState_Current PyThreadState_GET() +#elif PY_VERSION_HEX >= 0x030d00A1 + #define __Pyx_PyThreadState_Current PyThreadState_GetUnchecked() +#else + #define __Pyx_PyThreadState_Current _PyThreadState_UncheckedGet() +#endif +#if CYTHON_USE_MODULE_STATE +static CYTHON_INLINE void *__Pyx__PyModule_GetState(PyObject *op) +{ + void *result; + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#define __Pyx_PyModule_GetState(o) (__pyx_mstatetype *)__Pyx__PyModule_GetState(o) +#else +#define __Pyx_PyModule_GetState(op) ((void)op,__pyx_mstate_global) +#endif +#define __Pyx_PyObject_GetSlot(obj, name, func_ctype) __Pyx_PyType_GetSlot(Py_TYPE((PyObject *) obj), name, func_ctype) +#define __Pyx_PyObject_TryGetSlot(obj, name, func_ctype) __Pyx_PyType_TryGetSlot(Py_TYPE(obj), name, func_ctype) +#define __Pyx_PyObject_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#define __Pyx_PyObject_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSubSlot(Py_TYPE(obj), sub, name, func_ctype) +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((type)->name) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype) __Pyx_PyType_GetSlot(type, name, func_ctype) + #define __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) (((type)->sub) ? ((type)->sub->name) : NULL) + #define __Pyx_PyType_TryGetSubSlot(type, sub, name, func_ctype) __Pyx_PyType_GetSubSlot(type, sub, name, func_ctype) +#else + #define __Pyx_PyType_GetSlot(type, name, func_ctype) ((func_ctype) PyType_GetSlot((type), Py_##name)) + #define __Pyx_PyType_TryGetSlot(type, name, func_ctype)\ + ((__PYX_LIMITED_VERSION_HEX >= 0x030A0000 ||\ + (PyType_GetFlags(type) & Py_TPFLAGS_HEAPTYPE) || __Pyx_get_runtime_version() >= 0x030A0000) ?\ + __Pyx_PyType_GetSlot(type, name, func_ctype) : NULL) + #define __Pyx_PyType_GetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_GetSlot(obj, name, func_ctype) + #define __Pyx_PyType_TryGetSubSlot(obj, sub, name, func_ctype) __Pyx_PyType_TryGetSlot(obj, name, func_ctype) +#endif +#if CYTHON_COMPILING_IN_CPYTHON || defined(_PyDict_NewPresized) +#define __Pyx_PyDict_NewPresized(n) ((n <= 8) ? PyDict_New() : _PyDict_NewPresized(n)) +#else +#define __Pyx_PyDict_NewPresized(n) PyDict_New() +#endif +#define __Pyx_PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y) +#define __Pyx_PyNumber_InPlaceDivide(x,y) PyNumber_InPlaceTrueDivide(x,y) +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_UNICODE_INTERNALS +#define __Pyx_PyDict_GetItemStrWithError(dict, name) _PyDict_GetItem_KnownHash(dict, name, ((PyASCIIObject *) name)->hash) +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStr(PyObject *dict, PyObject *name) { + PyObject *res = __Pyx_PyDict_GetItemStrWithError(dict, name); + if (res == NULL) PyErr_Clear(); + return res; +} +#elif !CYTHON_COMPILING_IN_PYPY || PYPY_VERSION_NUM >= 0x07020000 +#define __Pyx_PyDict_GetItemStrWithError PyDict_GetItemWithError +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#else +static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, PyObject *name) { +#if CYTHON_COMPILING_IN_PYPY + return PyDict_GetItem(dict, name); +#else + PyDictEntry *ep; + PyDictObject *mp = (PyDictObject*) dict; + long hash = ((PyStringObject *) name)->ob_shash; + assert(hash != -1); + ep = (mp->ma_lookup)(mp, name, hash); + if (ep == NULL) { + return NULL; + } + return ep->me_value; +#endif +} +#define __Pyx_PyDict_GetItemStr PyDict_GetItem +#endif +#if CYTHON_USE_TYPE_SLOTS + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) + #define __Pyx_PyType_HasFeature(type, feature) ((__Pyx_PyType_GetFlags(type) & (feature)) != 0) +#else + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) + #define __Pyx_PyType_HasFeature(type, feature) PyType_HasFeature(type, feature) +#endif +#define __Pyx_PyObject_GetIterNextFunc(iterator) __Pyx_PyObject_GetSlot(iterator, tp_iternext, iternextfunc) +#if CYTHON_USE_TYPE_SPECS && PY_VERSION_HEX >= 0x03080000 +#define __Pyx_PyHeapTypeObject_GC_Del(obj) {\ + PyTypeObject *type = Py_TYPE((PyObject*)obj);\ + assert(__Pyx_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE));\ + PyObject_GC_Del(obj);\ + Py_DECREF(type);\ +} +#else +#define __Pyx_PyHeapTypeObject_GC_Del(obj) PyObject_GC_Del(obj) +#endif +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_ReadChar(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((void)u, 1114111U) + #define __Pyx_PyUnicode_KIND(u) ((void)u, (0)) + #define __Pyx_PyUnicode_DATA(u) ((void*)u) + #define __Pyx_PyUnicode_READ(k, d, i) ((void)k, PyUnicode_ReadChar((PyObject*)(d), i)) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetLength(u)) +#else + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_READY(op) (0) + #else + #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ + 0 : _PyUnicode_Ready((PyObject *)(op))) + #endif + #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) + #define __Pyx_PyUnicode_KIND(u) ((int)PyUnicode_KIND(u)) + #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) + #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, (Py_UCS4) ch) + #if PY_VERSION_HEX >= 0x030C0000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_LENGTH(u)) + #else + #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x03090000 + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : ((PyCompactUnicodeObject *)(u))->wstr_length)) + #else + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) + #endif + #endif +#endif +#if CYTHON_COMPILING_IN_PYPY + #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) PyNumber_Add(a, b) +#else + #define __Pyx_PyUnicode_Concat(a, b) PyUnicode_Concat(a, b) + #define __Pyx_PyUnicode_ConcatSafe(a, b) ((unlikely((a) == Py_None) || unlikely((b) == Py_None)) ?\ + PyNumber_Add(a, b) : __Pyx_PyUnicode_Concat(a, b)) +#endif +#if CYTHON_COMPILING_IN_PYPY + #if !defined(PyUnicode_DecodeUnicodeEscape) + #define PyUnicode_DecodeUnicodeEscape(s, size, errors) PyUnicode_Decode(s, size, "unicode_escape", errors) + #endif + #if !defined(PyUnicode_Contains) + #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) + #endif + #if !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) + #endif + #if !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) + #endif +#endif +#define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None || (PyUnicode_Check(b) && !PyUnicode_CheckExact(b)))) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) +#if CYTHON_COMPILING_IN_CPYTHON + #define __Pyx_PySequence_ListKeepNew(obj)\ + (likely(PyList_CheckExact(obj) && Py_REFCNT(obj) == 1) ? __Pyx_NewRef(obj) : PySequence_List(obj)) +#else + #define __Pyx_PySequence_ListKeepNew(obj) PySequence_List(obj) +#endif +#ifndef PySet_CheckExact + #define PySet_CheckExact(obj) __Pyx_IS_TYPE(obj, &PySet_Type) +#endif +#if PY_VERSION_HEX >= 0x030900A4 + #define __Pyx_SET_REFCNT(obj, refcnt) Py_SET_REFCNT(obj, refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SET_SIZE(obj, size) +#else + #define __Pyx_SET_REFCNT(obj, refcnt) Py_REFCNT(obj) = (refcnt) + #define __Pyx_SET_SIZE(obj, size) Py_SIZE(obj) = (size) +#endif +#if CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyList_GetItemRef(o, i) (likely((i) >= 0) ? PySequence_GetItem(o, i) : (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) + #else + #define __Pyx_PyList_GetItemRef(o, i) PySequence_ITEM(o, i) + #endif +#elif CYTHON_COMPILING_IN_LIMITED_API || !CYTHON_ASSUME_SAFE_MACROS + #if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyList_GetItemRef(o, i) PyList_GetItemRef(o, i) + #else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_XNewRef(PyList_GetItem(o, i)) + #endif +#else + #define __Pyx_PyList_GetItemRef(o, i) __Pyx_NewRef(PyList_GET_ITEM(o, i)) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyDict_GetItemRef(dict, key, result) PyDict_GetItemRef(dict, key, result) +#elif CYTHON_AVOID_BORROWED_REFS || CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyObject_GetItem(dict, key); + if (*result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return 0; + } + return -1; + } + return 1; +} +#else +static CYTHON_INLINE int __Pyx_PyDict_GetItemRef(PyObject *dict, PyObject *key, PyObject **result) { + *result = PyDict_GetItemWithError(dict, key); + if (*result == NULL) { + return PyErr_Occurred() ? -1 : 0; + } + Py_INCREF(*result); + return 1; +} +#endif +#if defined(CYTHON_DEBUG_VISIT_CONST) && CYTHON_DEBUG_VISIT_CONST + #define __Pyx_VISIT_CONST(obj) Py_VISIT(obj) +#else + #define __Pyx_VISIT_CONST(obj) +#endif +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PySequence_ITEM(o, i) PySequence_ITEM(o, i) + #define __Pyx_PySequence_SIZE(seq) Py_SIZE(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) (PyTuple_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GET_ITEM(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) (PyList_SET_ITEM(o, i, v), (0)) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GET_ITEM(o, i) +#else + #define __Pyx_PySequence_ITEM(o, i) PySequence_GetItem(o, i) + #define __Pyx_PySequence_SIZE(seq) PySequence_Size(seq) + #define __Pyx_PyTuple_SET_ITEM(o, i, v) PyTuple_SetItem(o, i, v) + #define __Pyx_PyTuple_GET_ITEM(o, i) PyTuple_GetItem(o, i) + #define __Pyx_PyList_SET_ITEM(o, i, v) PyList_SetItem(o, i, v) + #define __Pyx_PyList_GET_ITEM(o, i) PyList_GetItem(o, i) +#endif +#if CYTHON_ASSUME_SAFE_SIZE + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_GET_SIZE(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_GET_SIZE(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_GET_SIZE(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_GET_SIZE(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_GET_SIZE(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GET_LENGTH(o) +#else + #define __Pyx_PyTuple_GET_SIZE(o) PyTuple_Size(o) + #define __Pyx_PyList_GET_SIZE(o) PyList_Size(o) + #define __Pyx_PySet_GET_SIZE(o) PySet_Size(o) + #define __Pyx_PyBytes_GET_SIZE(o) PyBytes_Size(o) + #define __Pyx_PyByteArray_GET_SIZE(o) PyByteArray_Size(o) + #define __Pyx_PyUnicode_GET_LENGTH(o) PyUnicode_GetLength(o) +#endif +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + #define __Pyx_PyImport_AddModuleRef(name) PyImport_AddModuleRef(name) +#else + static CYTHON_INLINE PyObject *__Pyx_PyImport_AddModuleRef(const char *name) { + PyObject *module = PyImport_AddModule(name); + Py_XINCREF(module); + return module; + } +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_InternFromString) + #define PyUnicode_InternFromString(s) PyUnicode_FromString(s) +#endif +#define __Pyx_PyLong_FromHash_t PyLong_FromSsize_t +#define __Pyx_PyLong_AsHash_t __Pyx_PyIndex_AsSsize_t +#if __PYX_LIMITED_VERSION_HEX >= 0x030A0000 + #define __Pyx_PySendResult PySendResult +#else + typedef enum { + PYGEN_RETURN = 0, + PYGEN_ERROR = -1, + PYGEN_NEXT = 1, + } __Pyx_PySendResult; +#endif +#if CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX < 0x030A00A3 + typedef __Pyx_PySendResult (*__Pyx_pyiter_sendfunc)(PyObject *iter, PyObject *value, PyObject **result); +#else + #define __Pyx_pyiter_sendfunc sendfunc +#endif +#if !CYTHON_USE_AM_SEND +#define __PYX_HAS_PY_AM_SEND 0 +#elif __PYX_LIMITED_VERSION_HEX >= 0x030A0000 +#define __PYX_HAS_PY_AM_SEND 1 +#else +#define __PYX_HAS_PY_AM_SEND 2 // our own backported implementation +#endif +#if __PYX_HAS_PY_AM_SEND < 2 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods +#else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + __Pyx_pyiter_sendfunc am_send; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_SlotTpAsAsync(s) ((PyAsyncMethods*)(s)) +#endif +#if CYTHON_USE_AM_SEND && PY_VERSION_HEX < 0x030A00F0 + #define __Pyx_TPFLAGS_HAVE_AM_SEND (1UL << 21) +#else + #define __Pyx_TPFLAGS_HAVE_AM_SEND (0) +#endif +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_PyInterpreterState_Get() PyInterpreterState_Get() +#else +#define __Pyx_PyInterpreterState_Get() PyThreadState_Get()->interp +#endif +#if CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030A0000 +#ifdef __cplusplus +extern "C" +#endif +PyAPI_FUNC(void *) PyMem_Calloc(size_t nelem, size_t elsize); +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_init_co_variable(PyObject *inspect, const char* name, int *write_to) { + int value; + PyObject *py_value = PyObject_GetAttrString(inspect, name); + if (!py_value) return 0; + value = (int) PyLong_AsLong(py_value); + Py_DECREF(py_value); + *write_to = value; + return value != -1 || !PyErr_Occurred(); +} +static int __Pyx_init_co_variables(void) { + PyObject *inspect; + int result; + inspect = PyImport_ImportModule("inspect"); + result = +#if !defined(CO_OPTIMIZED) + __Pyx_init_co_variable(inspect, "CO_OPTIMIZED", &CO_OPTIMIZED) && +#endif +#if !defined(CO_NEWLOCALS) + __Pyx_init_co_variable(inspect, "CO_NEWLOCALS", &CO_NEWLOCALS) && +#endif +#if !defined(CO_VARARGS) + __Pyx_init_co_variable(inspect, "CO_VARARGS", &CO_VARARGS) && +#endif +#if !defined(CO_VARKEYWORDS) + __Pyx_init_co_variable(inspect, "CO_VARKEYWORDS", &CO_VARKEYWORDS) && +#endif +#if !defined(CO_ASYNC_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_ASYNC_GENERATOR", &CO_ASYNC_GENERATOR) && +#endif +#if !defined(CO_GENERATOR) + __Pyx_init_co_variable(inspect, "CO_GENERATOR", &CO_GENERATOR) && +#endif +#if !defined(CO_COROUTINE) + __Pyx_init_co_variable(inspect, "CO_COROUTINE", &CO_COROUTINE) && +#endif + 1; + Py_DECREF(inspect); + return result ? 0 : -1; +} +#else +static int __Pyx_init_co_variables(void) { + return 0; // It's a limited API-only feature +} +#endif + +/* MathInitCode */ +#if defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS) + #ifndef _USE_MATH_DEFINES + #define _USE_MATH_DEFINES + #endif +#endif +#include +#ifdef NAN +#define __PYX_NAN() ((float) NAN) +#else +static CYTHON_INLINE float __PYX_NAN() { + float value; + memset(&value, 0xFF, sizeof(value)); + return value; +} +#endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif + +#ifndef CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#define CYTHON_CLINE_IN_TRACEBACK_RUNTIME 0 +#endif +#ifndef CYTHON_CLINE_IN_TRACEBACK +#define CYTHON_CLINE_IN_TRACEBACK CYTHON_CLINE_IN_TRACEBACK_RUNTIME +#endif +#if CYTHON_CLINE_IN_TRACEBACK +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; __pyx_clineno = __LINE__; (void) __pyx_clineno; } +#else +#define __PYX_MARK_ERR_POS(f_index, lineno) { __pyx_filename = __pyx_f[f_index]; (void) __pyx_filename; __pyx_lineno = lineno; (void) __pyx_lineno; (void) __pyx_clineno; } +#endif +#define __PYX_ERR(f_index, lineno, Ln_error) \ + { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; } + +#ifdef CYTHON_EXTERN_C + #undef __PYX_EXTERN_C + #define __PYX_EXTERN_C CYTHON_EXTERN_C +#elif defined(__PYX_EXTERN_C) + #ifdef _MSC_VER + #pragma message ("Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead.") + #else + #warning Please do not define the '__PYX_EXTERN_C' macro externally. Use 'CYTHON_EXTERN_C' instead. + #endif +#else + #define __PYX_EXTERN_C extern "C++" +#endif + +#define __PYX_HAVE__cuda__bindings___internal__utils +#define __PYX_HAVE_API__cuda__bindings___internal__utils +/* Early includes */ +#include +#include "ios" +#include "new" +#include "stdexcept" +#include "typeinfo" +#include +#include + + template + class nullable_unique_ptr { + public: + nullable_unique_ptr() noexcept = default; + + nullable_unique_ptr(std::nullptr_t) noexcept = delete; + + explicit nullable_unique_ptr(T* data, bool own_data): + own_data_(own_data) + { + if (own_data) + manager_.reset(data); + else + raw_data_ = data; + } + + nullable_unique_ptr(const nullable_unique_ptr&) = delete; + + nullable_unique_ptr& operator=(const nullable_unique_ptr&) = delete; + + nullable_unique_ptr(nullable_unique_ptr&& other) noexcept + { + own_data_ = other.own_data_; + other.own_data_ = false; // ownership is transferred + if (own_data_) + { + manager_ = std::move(other.manager_); + raw_data_ = nullptr; // just in case + } + else + { + manager_.reset(nullptr); // just in case + raw_data_ = other.raw_data_; + } + } + + nullable_unique_ptr& operator=(nullable_unique_ptr&& other) noexcept + { + own_data_ = other.own_data_; + other.own_data_ = false; // ownership is transferred + if (own_data_) + { + manager_ = std::move(other.manager_); + raw_data_ = nullptr; // just in case + } + else + { + manager_.reset(nullptr); // just in case + raw_data_ = other.raw_data_; + } + return *this; + } + + ~nullable_unique_ptr() = default; + + void reset(T* data, bool own_data) + { + own_data_ = own_data; + if (own_data_) + { + manager_.reset(data); + raw_data_ = nullptr; + } + else + { + manager_.reset(nullptr); + raw_data_ = data; + } + } + + void swap(nullable_unique_ptr& other) noexcept + { + std::swap(manager_, other.manager_); + std::swap(raw_data_, other.raw_data_); + std::swap(own_data_, other.own_data_); + } + + /* + * Get the pointer to the underlying object (this is different from data()!). + */ + T* get() const noexcept + { + if (own_data_) + return manager_.get(); + else + return raw_data_; + } + + /* + * Get the pointer to the underlying buffer (this is different from get()!). + */ + void* data() noexcept + { + if (own_data_) + return manager_.get()->data(); + else + return raw_data_; + } + + T& operator*() + { + if (own_data_) + return *manager_; + else + return *raw_data_; + } + + private: + std::unique_ptr manager_{}; + T* raw_data_{nullptr}; + bool own_data_{false}; + }; + +#include +#include + + #if PY_MAJOR_VERSION >= 3 + #define __Pyx_PyFloat_FromString(obj) PyFloat_FromString(obj) + #else + #define __Pyx_PyFloat_FromString(obj) PyFloat_FromString(obj, NULL) + #endif + +#include + + #if PY_MAJOR_VERSION <= 2 + #define PyDict_GetItemWithError _PyDict_GetItemWithError + #endif + + + #if PY_VERSION_HEX < 0x030d0000 + static CYTHON_INLINE int __Pyx_PyWeakref_GetRef(PyObject *ref, PyObject **pobj) + { + PyObject *obj = PyWeakref_GetObject(ref); + if (obj == NULL) { + // SystemError if ref is NULL + *pobj = NULL; + return -1; + } + if (obj == Py_None) { + *pobj = NULL; + return 0; + } + Py_INCREF(obj); + *pobj = obj; + return 1; + } + #else + #define __Pyx_PyWeakref_GetRef PyWeakref_GetRef + #endif + +#include "pythread.h" + + #if (CYTHON_COMPILING_IN_PYPY && PYPY_VERSION_NUM < 0x07030600) && !defined(PyContextVar_Get) + #define PyContextVar_Get(var, d, v) ((d) ? ((void)(var), Py_INCREF(d), (v)[0] = (d), 0) : ((v)[0] = NULL, 0) ) + #endif + +#include + + #if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1600) + // move should be defined for these versions of MSVC, but __cplusplus isn't set usefully + #include + + namespace cython_std { + template typename std::remove_reference::type&& move(T& t) noexcept { return std::move(t); } + template typename std::remove_reference::type&& move(T&& t) noexcept { return std::move(t); } + } + + #endif + +#ifdef _OPENMP +#include +#endif /* _OPENMP */ + +#if defined(PYREX_WITHOUT_ASSERTIONS) && !defined(CYTHON_WITHOUT_ASSERTIONS) +#define CYTHON_WITHOUT_ASSERTIONS +#endif + +#define __PYX_DEFAULT_STRING_ENCODING_IS_ASCII 0 +#define __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 0 +#define __PYX_DEFAULT_STRING_ENCODING "" +#define __Pyx_PyObject_FromString __Pyx_PyBytes_FromString +#define __Pyx_PyObject_FromStringAndSize __Pyx_PyBytes_FromStringAndSize +#define __Pyx_uchar_cast(c) ((unsigned char)c) +#define __Pyx_long_cast(x) ((long)x) +#define __Pyx_fits_Py_ssize_t(v, type, is_signed) (\ + (sizeof(type) < sizeof(Py_ssize_t)) ||\ + (sizeof(type) > sizeof(Py_ssize_t) &&\ + likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX) &&\ + (!is_signed || likely(v > (type)PY_SSIZE_T_MIN ||\ + v == (type)PY_SSIZE_T_MIN))) ||\ + (sizeof(type) == sizeof(Py_ssize_t) &&\ + (is_signed || likely(v < (type)PY_SSIZE_T_MAX ||\ + v == (type)PY_SSIZE_T_MAX))) ) +static CYTHON_INLINE int __Pyx_is_valid_index(Py_ssize_t i, Py_ssize_t limit) { + return (size_t) i < (size_t) limit; +} +#if defined (__cplusplus) && __cplusplus >= 201103L + #include + #define __Pyx_sst_abs(value) std::abs(value) +#elif SIZEOF_INT >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) abs(value) +#elif SIZEOF_LONG >= SIZEOF_SIZE_T + #define __Pyx_sst_abs(value) labs(value) +#elif defined (_MSC_VER) + #define __Pyx_sst_abs(value) ((Py_ssize_t)_abs64(value)) +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L + #define __Pyx_sst_abs(value) llabs(value) +#elif defined (__GNUC__) + #define __Pyx_sst_abs(value) __builtin_llabs(value) +#else + #define __Pyx_sst_abs(value) ((value<0) ? -value : value) +#endif +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s); +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject*); +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject*, Py_ssize_t* length); +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char*); +#define __Pyx_PyByteArray_FromStringAndSize(s, l) PyByteArray_FromStringAndSize((const char*)s, l) +#define __Pyx_PyBytes_FromString PyBytes_FromString +#define __Pyx_PyBytes_FromStringAndSize PyBytes_FromStringAndSize +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); +#if CYTHON_ASSUME_SAFE_MACROS + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AS_STRING(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AS_STRING(s) +#else + #define __Pyx_PyBytes_AsWritableString(s) ((char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableSString(s) ((signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsWritableUString(s) ((unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsString(s) ((const char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsSString(s) ((const signed char*) PyBytes_AsString(s)) + #define __Pyx_PyBytes_AsUString(s) ((const unsigned char*) PyBytes_AsString(s)) + #define __Pyx_PyByteArray_AsString(s) PyByteArray_AsString(s) +#endif +#define __Pyx_PyObject_AsWritableString(s) ((char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableSString(s) ((signed char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsWritableUString(s) ((unsigned char*)(__pyx_uintptr_t) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsSString(s) ((const signed char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_AsUString(s) ((const unsigned char*) __Pyx_PyObject_AsString(s)) +#define __Pyx_PyObject_FromCString(s) __Pyx_PyObject_FromString((const char*)s) +#define __Pyx_PyBytes_FromCString(s) __Pyx_PyBytes_FromString((const char*)s) +#define __Pyx_PyByteArray_FromCString(s) __Pyx_PyByteArray_FromString((const char*)s) +#define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) +#define __Pyx_PyUnicode_FromOrdinal(o) PyUnicode_FromOrdinal((int)o) +#define __Pyx_PyUnicode_AsUnicode PyUnicode_AsUnicode +static CYTHON_INLINE PyObject *__Pyx_NewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_NewRef) + return Py_NewRef(obj); +#else + Py_INCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_XNewRef(PyObject *obj) { +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030a0000 || defined(Py_XNewRef) + return Py_XNewRef(obj); +#else + Py_XINCREF(obj); + return obj; +#endif +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b); +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b); +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject*); +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject*); +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x); +#define __Pyx_PySequence_Tuple(obj)\ + (likely(PyTuple_CheckExact(obj)) ? __Pyx_NewRef(obj) : PySequence_Tuple(obj)) +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t); +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject*); +#if CYTHON_ASSUME_SAFE_MACROS +#define __Pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AS_DOUBLE(x) +#else +#define __Pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) +#define __Pyx_PyFloat_AS_DOUBLE(x) PyFloat_AsDouble(x) +#endif +#define __Pyx_PyFloat_AsFloat(x) ((float) __Pyx_PyFloat_AsDouble(x)) +#define __Pyx_PyNumber_Int(x) (PyLong_CheckExact(x) ? __Pyx_NewRef(x) : PyNumber_Long(x)) +#if CYTHON_USE_PYLONG_INTERNALS + #if PY_VERSION_HEX >= 0x030C00A7 + #ifndef _PyLong_SIGN_MASK + #define _PyLong_SIGN_MASK 3 + #endif + #ifndef _PyLong_NON_SIZE_BITS + #define _PyLong_NON_SIZE_BITS 3 + #endif + #define __Pyx_PyLong_Sign(x) (((PyLongObject*)x)->long_value.lv_tag & _PyLong_SIGN_MASK) + #define __Pyx_PyLong_IsNeg(x) ((__Pyx_PyLong_Sign(x) & 2) != 0) + #define __Pyx_PyLong_IsNonNeg(x) (!__Pyx_PyLong_IsNeg(x)) + #define __Pyx_PyLong_IsZero(x) (__Pyx_PyLong_Sign(x) & 1) + #define __Pyx_PyLong_IsPos(x) (__Pyx_PyLong_Sign(x) == 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) (__Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) ((Py_ssize_t) (((PyLongObject*)x)->long_value.lv_tag >> _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_SignedDigitCount(x)\ + ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * __Pyx_PyLong_DigitCount(x)) + #if defined(PyUnstable_Long_IsCompact) && defined(PyUnstable_Long_CompactValue) + #define __Pyx_PyLong_IsCompact(x) PyUnstable_Long_IsCompact((PyLongObject*) x) + #define __Pyx_PyLong_CompactValue(x) PyUnstable_Long_CompactValue((PyLongObject*) x) + #else + #define __Pyx_PyLong_IsCompact(x) (((PyLongObject*)x)->long_value.lv_tag < (2 << _PyLong_NON_SIZE_BITS)) + #define __Pyx_PyLong_CompactValue(x) ((1 - (Py_ssize_t) __Pyx_PyLong_Sign(x)) * (Py_ssize_t) __Pyx_PyLong_Digits(x)[0]) + #endif + typedef Py_ssize_t __Pyx_compact_pylong; + typedef size_t __Pyx_compact_upylong; + #else + #define __Pyx_PyLong_IsNeg(x) (Py_SIZE(x) < 0) + #define __Pyx_PyLong_IsNonNeg(x) (Py_SIZE(x) >= 0) + #define __Pyx_PyLong_IsZero(x) (Py_SIZE(x) == 0) + #define __Pyx_PyLong_IsPos(x) (Py_SIZE(x) > 0) + #define __Pyx_PyLong_CompactValueUnsigned(x) ((Py_SIZE(x) == 0) ? 0 : __Pyx_PyLong_Digits(x)[0]) + #define __Pyx_PyLong_DigitCount(x) __Pyx_sst_abs(Py_SIZE(x)) + #define __Pyx_PyLong_SignedDigitCount(x) Py_SIZE(x) + #define __Pyx_PyLong_IsCompact(x) (Py_SIZE(x) == 0 || Py_SIZE(x) == 1 || Py_SIZE(x) == -1) + #define __Pyx_PyLong_CompactValue(x)\ + ((Py_SIZE(x) == 0) ? (sdigit) 0 : ((Py_SIZE(x) < 0) ? -(sdigit)__Pyx_PyLong_Digits(x)[0] : (sdigit)__Pyx_PyLong_Digits(x)[0])) + typedef sdigit __Pyx_compact_pylong; + typedef digit __Pyx_compact_upylong; + #endif + static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value); + #if PY_VERSION_HEX >= 0x030C00A5 + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->long_value.ob_digit) + #else + #define __Pyx_PyLong_Digits(x) (((PyLongObject*)x)->ob_digit) + #endif +#endif +#if __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeUTF8(c_str, size, NULL) +#elif __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_DecodeASCII(c_str, size, NULL) +#else + #define __Pyx_PyUnicode_FromStringAndSize(c_str, size) PyUnicode_Decode(c_str, size, __PYX_DEFAULT_STRING_ENCODING, NULL) +#endif + + +/* Test for GCC > 2.95 */ +#if defined(__GNUC__) && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))) + #define likely(x) __builtin_expect(!!(x), 1) + #define unlikely(x) __builtin_expect(!!(x), 0) +#else /* !__GNUC__ or GCC < 2.95 */ + #define likely(x) (x) + #define unlikely(x) (x) +#endif /* __GNUC__ */ +/* PretendToInitialize */ +#ifdef __cplusplus +#if __cplusplus > 201103L +#include +#endif +template +static void __Pyx_pretend_to_initialize(T* ptr) { +#if __cplusplus > 201103L + if ((std::is_trivially_default_constructible::value)) +#endif + *ptr = T(); + (void)ptr; +} +#else +static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; } +#endif + + +#if !CYTHON_USE_MODULE_STATE +static PyObject *__pyx_m = NULL; +#endif +static int __pyx_lineno; +static int __pyx_clineno = 0; +static const char * const __pyx_cfilenm = __FILE__; +static const char *__pyx_filename; + +/* #### Code section: filename_table ### */ + +static const char* const __pyx_f[] = { + "cuda/bindings/_internal/utils.pyx", + "", + "cpython/contextvars.pxd", + "cpython/type.pxd", + "cpython/bool.pxd", + "cpython/complex.pxd", +}; +/* #### Code section: utility_code_proto_before_types ### */ +/* CriticalSections.proto */ +#if !CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __Pyx_PyCriticalSection void* +#define __Pyx_PyCriticalSection2 void* +#define __Pyx_PyCriticalSection_Begin1(cs, arg) (void)cs +#define __Pyx_PyCriticalSection_Begin2(cs, arg1, arg2) (void)cs +#define __Pyx_PyCriticalSection_End1(cs) +#define __Pyx_PyCriticalSection_End2(cs) +#else +#define __Pyx_PyCriticalSection PyCriticalSection +#define __Pyx_PyCriticalSection2 PyCriticalSection2 +#define __Pyx_PyCriticalSection_Begin1 PyCriticalSection_Begin +#define __Pyx_PyCriticalSection_Begin2 PyCriticalSection2_Begin +#define __Pyx_PyCriticalSection_End1 PyCriticalSection_End +#define __Pyx_PyCriticalSection_End2 PyCriticalSection2_End +#endif +#if PY_VERSION_HEX < 0x030d0000 || CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_BEGIN_CRITICAL_SECTION(o) { +#define __Pyx_END_CRITICAL_SECTION() } +#else +#define __Pyx_BEGIN_CRITICAL_SECTION Py_BEGIN_CRITICAL_SECTION +#define __Pyx_END_CRITICAL_SECTION Py_END_CRITICAL_SECTION +#endif + +/* Atomics.proto */ +#include +#ifndef CYTHON_ATOMICS + #define CYTHON_ATOMICS 1 +#endif +#define __PYX_CYTHON_ATOMICS_ENABLED() CYTHON_ATOMICS +#define __PYX_GET_CYTHON_COMPILING_IN_CPYTHON_FREETHREADING() CYTHON_COMPILING_IN_CPYTHON_FREETHREADING +#define __pyx_atomic_int_type int +#define __pyx_nonatomic_int_type int +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__)) + #include +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ + (defined(_MSC_VER) && _MSC_VER >= 1700))) + #include +#endif +#if CYTHON_ATOMICS && (defined(__STDC_VERSION__) &&\ + (__STDC_VERSION__ >= 201112L) &&\ + !defined(__STDC_NO_ATOMICS__) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type atomic_int + #define __pyx_atomic_ptr_type atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) atomic_fetch_add_explicit(value, 1, memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) atomic_fetch_add_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) atomic_fetch_sub_explicit(value, 1, memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) atomic_load(value) + #define __pyx_atomic_store(value, new_value) atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) atomic_load_explicit(value, memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) atomic_load_explicit(value, memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C atomics" + #endif +#elif CYTHON_ATOMICS && (defined(__cplusplus) && (\ + (__cplusplus >= 201103L) ||\ +\ + (defined(_MSC_VER) && _MSC_VER >= 1700)) &&\ + ATOMIC_INT_LOCK_FREE == 2) + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type std::atomic_int + #define __pyx_atomic_ptr_type std::atomic_uintptr_t + #define __pyx_nonatomic_ptr_type uintptr_t + #define __pyx_atomic_incr_relaxed(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_relaxed) + #define __pyx_atomic_incr_acq_rel(value) std::atomic_fetch_add_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_decr_acq_rel(value) std::atomic_fetch_sub_explicit(value, 1, std::memory_order_acq_rel) + #define __pyx_atomic_sub(value, arg) std::atomic_fetch_sub(value, arg) + #define __pyx_atomic_int_cmp_exchange(value, expected, desired) std::atomic_compare_exchange_strong(value, expected, desired) + #define __pyx_atomic_load(value) std::atomic_load(value) + #define __pyx_atomic_store(value, new_value) std::atomic_store(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) std::atomic_load_explicit(value, std::memory_order_relaxed) + #define __pyx_atomic_pointer_load_acquire(value) std::atomic_load_explicit(value, std::memory_order_acquire) + #define __pyx_atomic_pointer_exchange(value, new_value) std::atomic_exchange(value, (__pyx_nonatomic_ptr_type)new_value) + #if defined(__PYX_DEBUG_ATOMICS) && defined(_MSC_VER) + #pragma message ("Using standard C++ atomics") + #elif defined(__PYX_DEBUG_ATOMICS) + #warning "Using standard C++ atomics" + #endif +#elif CYTHON_ATOMICS && (__GNUC__ >= 5 || (__GNUC__ == 4 &&\ + (__GNUC_MINOR__ > 1 ||\ + (__GNUC_MINOR__ == 1 && __GNUC_PATCHLEVEL__ >= 2)))) + #define __pyx_atomic_ptr_type void* + #define __pyx_atomic_incr_relaxed(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_incr_acq_rel(value) __sync_fetch_and_add(value, 1) + #define __pyx_atomic_decr_acq_rel(value) __sync_fetch_and_sub(value, 1) + #define __pyx_atomic_sub(value, arg) __sync_fetch_and_sub(value, arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = __sync_val_compare_and_swap(value, *expected, desired); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_store(value, new_value) __sync_lock_test_and_set(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_load_acquire(value) __sync_fetch_and_add(value, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) __sync_lock_test_and_set(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #warning "Using GNU atomics" + #endif +#elif CYTHON_ATOMICS && defined(_MSC_VER) + #include + #undef __pyx_atomic_int_type + #define __pyx_atomic_int_type long + #define __pyx_atomic_ptr_type void* + #undef __pyx_nonatomic_int_type + #define __pyx_nonatomic_int_type long + #pragma intrinsic (_InterlockedExchangeAdd, _InterlockedExchange, _InterlockedCompareExchange, _InterlockedCompareExchangePointer, _InterlockedExchangePointer) + #define __pyx_atomic_incr_relaxed(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_incr_acq_rel(value) _InterlockedExchangeAdd(value, 1) + #define __pyx_atomic_decr_acq_rel(value) _InterlockedExchangeAdd(value, -1) + #define __pyx_atomic_sub(value, arg) _InterlockedExchangeAdd(value, -arg) + static CYTHON_INLINE int __pyx_atomic_int_cmp_exchange(__pyx_atomic_int_type* value, __pyx_nonatomic_int_type* expected, __pyx_nonatomic_int_type desired) { + __pyx_nonatomic_int_type old = _InterlockedCompareExchange(value, desired, *expected); + int result = old == *expected; + *expected = old; + return result; + } + #define __pyx_atomic_load(value) _InterlockedExchangeAdd(value, 0) + #define __pyx_atomic_store(value, new_value) _InterlockedExchange(value, new_value) + #define __pyx_atomic_pointer_load_relaxed(value) *(void * volatile *)value + #define __pyx_atomic_pointer_load_acquire(value) _InterlockedCompareExchangePointer(value, 0, 0) + #define __pyx_atomic_pointer_exchange(value, new_value) _InterlockedExchangePointer(value, (__pyx_atomic_ptr_type)new_value) + #ifdef __PYX_DEBUG_ATOMICS + #pragma message ("Using MSVC atomics") + #endif +#else + #undef CYTHON_ATOMICS + #define CYTHON_ATOMICS 0 + #ifdef __PYX_DEBUG_ATOMICS + #warning "Not using atomics" + #endif +#endif +#if CYTHON_ATOMICS + #define __pyx_add_acquisition_count(memview)\ + __pyx_atomic_incr_relaxed(__pyx_get_slice_count_pointer(memview)) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_atomic_decr_acq_rel(__pyx_get_slice_count_pointer(memview)) +#else + #define __pyx_add_acquisition_count(memview)\ + __pyx_add_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) + #define __pyx_sub_acquisition_count(memview)\ + __pyx_sub_acquisition_count_locked(__pyx_get_slice_count_pointer(memview), memview->lock) +#endif + +/* #### Code section: numeric_typedefs ### */ +/* #### Code section: complex_type_declarations ### */ +/* #### Code section: type_declarations ### */ + +/*--- Type declarations ---*/ +struct __pyx_opt_args_7cpython_11contextvars_get_value; +struct __pyx_opt_args_7cpython_11contextvars_get_value_no_default; + +/* "cpython/contextvars.pxd":116 + * + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") + * cdef inline object get_value(var, default_value=None): # <<<<<<<<<<<<<< + * """Return a new reference to the value of the context variable, + * or the default value of the context variable, +*/ +struct __pyx_opt_args_7cpython_11contextvars_get_value { + int __pyx_n; + PyObject *default_value; +}; + +/* "cpython/contextvars.pxd":134 + * + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") + * cdef inline object get_value_no_default(var, default_value=None): # <<<<<<<<<<<<<< + * """Return a new reference to the value of the context variable, + * or the provided default value if no such value was found. +*/ +struct __pyx_opt_args_7cpython_11contextvars_get_value_no_default { + int __pyx_n; + PyObject *default_value; +}; +template +struct __pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource; +struct __pyx_opt_args_4cuda_8bindings_9_internal_5utils_get_buffer_pointer; + +/* "cuda/bindings/_internal/utils.pxd":156 + * + * + * cdef cppclass nested_resource[T]: # <<<<<<<<<<<<<< + * nullable_unique_ptr[ vector[intptr_t] ] ptrs + * nullable_unique_ptr[ vector[vector[T]] ] nested_resource_ptr +*/ +template +struct __pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource { + nullable_unique_ptr > ptrs; + nullable_unique_ptr > > nested_resource_ptr; +}; + +/* "cuda/bindings/_internal/utils.pxd":167 + * + * cdef bint is_nested_sequence(data) + * cdef void* get_buffer_pointer(buf, Py_ssize_t size, readonly=*) except* # <<<<<<<<<<<<<< +*/ +struct __pyx_opt_args_4cuda_8bindings_9_internal_5utils_get_buffer_pointer { + int __pyx_n; + PyObject *readonly; +}; +/* #### Code section: utility_code_proto ### */ + +/* --- Runtime support code (head) --- */ +/* Refnanny.proto */ +#ifndef CYTHON_REFNANNY + #define CYTHON_REFNANNY 0 +#endif +#if CYTHON_REFNANNY + typedef struct { + void (*INCREF)(void*, PyObject*, Py_ssize_t); + void (*DECREF)(void*, PyObject*, Py_ssize_t); + void (*GOTREF)(void*, PyObject*, Py_ssize_t); + void (*GIVEREF)(void*, PyObject*, Py_ssize_t); + void* (*SetupContext)(const char*, Py_ssize_t, const char*); + void (*FinishContext)(void**); + } __Pyx_RefNannyAPIStruct; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNanny = NULL; + static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname); + #define __Pyx_RefNannyDeclarations void *__pyx_refnanny = NULL; + #define __Pyx_RefNannySetupContext(name, acquire_gil)\ + if (acquire_gil) {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + PyGILState_Release(__pyx_gilstate_save);\ + } else {\ + __pyx_refnanny = __Pyx_RefNanny->SetupContext((name), (__LINE__), (__FILE__));\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContextNogil() {\ + PyGILState_STATE __pyx_gilstate_save = PyGILState_Ensure();\ + __Pyx_RefNannyFinishContext();\ + PyGILState_Release(__pyx_gilstate_save);\ + } + #define __Pyx_RefNannyFinishContext()\ + __Pyx_RefNanny->FinishContext(&__pyx_refnanny) + #define __Pyx_INCREF(r) __Pyx_RefNanny->INCREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_DECREF(r) __Pyx_RefNanny->DECREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GOTREF(r) __Pyx_RefNanny->GOTREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_GIVEREF(r) __Pyx_RefNanny->GIVEREF(__pyx_refnanny, (PyObject *)(r), (__LINE__)) + #define __Pyx_XINCREF(r) do { if((r) == NULL); else {__Pyx_INCREF(r); }} while(0) + #define __Pyx_XDECREF(r) do { if((r) == NULL); else {__Pyx_DECREF(r); }} while(0) + #define __Pyx_XGOTREF(r) do { if((r) == NULL); else {__Pyx_GOTREF(r); }} while(0) + #define __Pyx_XGIVEREF(r) do { if((r) == NULL); else {__Pyx_GIVEREF(r);}} while(0) +#else + #define __Pyx_RefNannyDeclarations + #define __Pyx_RefNannySetupContext(name, acquire_gil) + #define __Pyx_RefNannyFinishContextNogil() + #define __Pyx_RefNannyFinishContext() + #define __Pyx_INCREF(r) Py_INCREF(r) + #define __Pyx_DECREF(r) Py_DECREF(r) + #define __Pyx_GOTREF(r) + #define __Pyx_GIVEREF(r) + #define __Pyx_XINCREF(r) Py_XINCREF(r) + #define __Pyx_XDECREF(r) Py_XDECREF(r) + #define __Pyx_XGOTREF(r) + #define __Pyx_XGIVEREF(r) +#endif +#define __Pyx_Py_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; Py_XDECREF(tmp);\ + } while (0) +#define __Pyx_XDECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_XDECREF(tmp);\ + } while (0) +#define __Pyx_DECREF_SET(r, v) do {\ + PyObject *tmp = (PyObject *) r;\ + r = v; __Pyx_DECREF(tmp);\ + } while (0) +#define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) +#define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) + +/* PyErrExceptionMatches.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_ExceptionMatches(err) __Pyx_PyErr_ExceptionMatchesInState(__pyx_tstate, err) +static CYTHON_INLINE int __Pyx_PyErr_ExceptionMatchesInState(PyThreadState* tstate, PyObject* err); +#else +#define __Pyx_PyErr_ExceptionMatches(err) PyErr_ExceptionMatches(err) +#endif + +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; +#define __Pyx_PyThreadState_assign __pyx_tstate = __Pyx_PyThreadState_Current; +#if PY_VERSION_HEX >= 0x030C00A6 +#define __Pyx_PyErr_Occurred() (__pyx_tstate->current_exception != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->current_exception ? (PyObject*) Py_TYPE(__pyx_tstate->current_exception) : (PyObject*) NULL) +#else +#define __Pyx_PyErr_Occurred() (__pyx_tstate->curexc_type != NULL) +#define __Pyx_PyErr_CurrentExceptionType() (__pyx_tstate->curexc_type) +#endif +#else +#define __Pyx_PyThreadState_declare +#define __Pyx_PyThreadState_assign +#define __Pyx_PyErr_Occurred() (PyErr_Occurred() != NULL) +#define __Pyx_PyErr_CurrentExceptionType() PyErr_Occurred() +#endif + +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_PyErr_Clear() __Pyx_ErrRestore(NULL, NULL, NULL) +#define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) __Pyx_ErrFetchInState(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A6 +#define __Pyx_PyErr_SetNone(exc) (Py_INCREF(exc), __Pyx_ErrRestore((exc), NULL, NULL)) +#else +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#endif +#else +#define __Pyx_PyErr_Clear() PyErr_Clear() +#define __Pyx_PyErr_SetNone(exc) PyErr_SetNone(exc) +#define __Pyx_ErrRestoreWithState(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchWithState(type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestoreInState(tstate, type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetchInState(tstate, type, value, tb) PyErr_Fetch(type, value, tb) +#define __Pyx_ErrRestore(type, value, tb) PyErr_Restore(type, value, tb) +#define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) +#endif + +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name); +#else +#define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) +#endif + +/* PyObjectGetAttrStrNoError.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name); + +/* GetBuiltinName.proto */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name); + +/* AssertionsEnabled.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API || (CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x030C0000) + static int __pyx_assertions_enabled_flag; + #define __pyx_assertions_enabled() (__pyx_assertions_enabled_flag) + static int __Pyx_init_assertions_enabled(void) { + PyObject *builtins, *debug, *debug_str; + int flag; + builtins = PyEval_GetBuiltins(); + if (!builtins) goto bad; + debug_str = PyUnicode_FromStringAndSize("__debug__", 9); + if (!debug_str) goto bad; + debug = PyObject_GetItem(builtins, debug_str); + Py_DECREF(debug_str); + if (!debug) goto bad; + flag = PyObject_IsTrue(debug); + Py_DECREF(debug); + if (flag == -1) goto bad; + __pyx_assertions_enabled_flag = flag; + return 0; + bad: + __pyx_assertions_enabled_flag = 1; + return -1; + } +#else + #define __Pyx_init_assertions_enabled() (0) + #define __pyx_assertions_enabled() (!Py_OptimizeFlag) +#endif + +/* RaiseException.proto */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); + +/* GetTopmostException.proto */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * __Pyx_PyErr_GetTopmostException(PyThreadState *tstate); +#endif + +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb); +#else +#define __Pyx_ExceptionSave(type, value, tb) PyErr_GetExcInfo(type, value, tb) +#define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) +#endif + +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* PyUnicode_Unicode.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Unicode(PyObject *obj); + +/* IncludeStringH.proto */ +#include + +/* JoinPyUnicode.proto */ +static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength, + Py_UCS4 max_char); + +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#if !CYTHON_VECTORCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs); +#endif +#define __Pyx_BUILD_ASSERT_EXPR(cond)\ + (sizeof(char [1 - 2*!(cond)]) - 1) +#ifndef Py_MEMBER_SIZE +#define Py_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#endif +#if !CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03080000 + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets() + #define __Pyx_PyFrame_GetLocalsplus(frame) ((frame)->f_localsplus) +#else + static size_t __pyx_pyframe_localsplus_offset = 0; + #include "frameobject.h" + #define __Pxy_PyFrame_Initialize_Offsets()\ + ((void)__Pyx_BUILD_ASSERT_EXPR(sizeof(PyFrameObject) == offsetof(PyFrameObject, f_localsplus) + Py_MEMBER_SIZE(PyFrameObject, f_localsplus)),\ + (void)(__pyx_pyframe_localsplus_offset = ((size_t)PyFrame_Type.tp_basicsize) - Py_MEMBER_SIZE(PyFrameObject, f_localsplus))) + #define __Pyx_PyFrame_GetLocalsplus(frame)\ + (assert(__pyx_pyframe_localsplus_offset), (PyObject **)(((char *)(frame)) + __pyx_pyframe_localsplus_offset)) +#endif +#endif +#endif + +/* PyObjectCall.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); +#else +#define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) +#endif + +/* PyObjectCallMethO.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); +#endif + +/* PyObjectFastCall.proto */ +#define __Pyx_PyObject_FastCall(func, args, nargs) __Pyx_PyObject_FastCallDict(func, args, (size_t)(nargs), NULL) +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs); + +/* SwapException.proto */ +#if CYTHON_FAST_THREAD_STATE +#define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); +#endif + +/* GetItemInt.proto */ +#define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck) :\ + (is_list ? (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL) :\ + __Pyx_GetItemInt_Generic(o, to_py_func(i)))) +#define __Pyx_GetItemInt_List(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_List_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\ + (PyErr_SetString(PyExc_IndexError, "list index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck); +#define __Pyx_GetItemInt_Tuple(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck, has_gil)\ + (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ + __Pyx_GetItemInt_Tuple_Fast(o, (Py_ssize_t)i, wraparound, boundscheck) :\ + (PyErr_SetString(PyExc_IndexError, "tuple index out of range"), (PyObject*)NULL)) +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + int wraparound, int boundscheck); +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j); +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, + int is_list, int wraparound, int boundscheck); + +/* RaiseUnexpectedTypeError.proto */ +static int __Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj); + +/* TypeImport.proto */ +#ifndef __PYX_HAVE_RT_ImportType_proto_3_1_5 +#define __PYX_HAVE_RT_ImportType_proto_3_1_5 +#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +#include +#endif +#if (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) || __cplusplus >= 201103L +#define __PYX_GET_STRUCT_ALIGNMENT_3_1_5(s) alignof(s) +#else +#define __PYX_GET_STRUCT_ALIGNMENT_3_1_5(s) sizeof(void*) +#endif +enum __Pyx_ImportType_CheckSize_3_1_5 { + __Pyx_ImportType_CheckSize_Error_3_1_5 = 0, + __Pyx_ImportType_CheckSize_Warn_3_1_5 = 1, + __Pyx_ImportType_CheckSize_Ignore_3_1_5 = 2 +}; +static PyTypeObject *__Pyx_ImportType_3_1_5(PyObject* module, const char *module_name, const char *class_name, size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_1_5 check_size); +#endif + +/* PyObjectCallOneArg.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); + +/* Py3UpdateBases.proto */ +static PyObject* __Pyx_PEP560_update_bases(PyObject *bases); + +/* CalculateMetaclass.proto */ +static PyObject *__Pyx_CalculateMetaclass(PyTypeObject *metaclass, PyObject *bases); + +/* PyObjectCall2Args.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2); + +/* PyObjectLookupSpecial.proto */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +#define __Pyx_PyObject_LookupSpecialNoError(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 0) +#define __Pyx_PyObject_LookupSpecial(obj, attr_name) __Pyx__PyObject_LookupSpecial(obj, attr_name, 1) +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error); +#else +#define __Pyx_PyObject_LookupSpecialNoError(o,n) __Pyx_PyObject_GetAttrStrNoError(o,n) +#define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) +#endif + +/* Py3ClassCreate.proto */ +static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, PyObject *qualname, + PyObject *mkw, PyObject *modname, PyObject *doc); +static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, PyObject *dict, + PyObject *mkw, int calculate_metaclass, int allow_py2_metaclass); + +/* PyDictVersioning.proto */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +#define __PYX_DICT_VERSION_INIT ((PY_UINT64_T) -1) +#define __PYX_GET_DICT_VERSION(dict) (((PyDictObject*)(dict))->ma_version_tag) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var)\ + (version_var) = __PYX_GET_DICT_VERSION(dict);\ + (cache_var) = (value); +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) {\ + static PY_UINT64_T __pyx_dict_version = 0;\ + static PyObject *__pyx_dict_cached_value = NULL;\ + if (likely(__PYX_GET_DICT_VERSION(DICT) == __pyx_dict_version)) {\ + (VAR) = __pyx_dict_cached_value;\ + } else {\ + (VAR) = __pyx_dict_cached_value = (LOOKUP);\ + __pyx_dict_version = __PYX_GET_DICT_VERSION(DICT);\ + }\ +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj); +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj); +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version); +#else +#define __PYX_GET_DICT_VERSION(dict) (0) +#define __PYX_UPDATE_DICT_CACHE(dict, value, cache_var, version_var) +#define __PYX_PY_DICT_LOOKUP_IF_MODIFIED(VAR, DICT, LOOKUP) (VAR) = (LOOKUP); +#endif + +/* CLineInTraceback.proto */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line); +#else +#define __Pyx_CLineForTraceback(tstate, c_line) (((CYTHON_CLINE_IN_TRACEBACK)) ? c_line : 0) +#endif + +/* CodeObjectCache.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject __Pyx_CachedCodeObjectType; +#else +typedef PyCodeObject __Pyx_CachedCodeObjectType; +#endif +typedef struct { + __Pyx_CachedCodeObjectType* code_object; + int code_line; +} __Pyx_CodeObjectCacheEntry; +struct __Pyx_CodeObjectCache { + int count; + int max_count; + __Pyx_CodeObjectCacheEntry* entries; + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_int_type accessor_count; + #endif +}; +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line); +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object); + +/* AddTraceback.proto */ +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename); + +/* CppExceptionConversion.proto */ +#ifndef __Pyx_CppExn2PyErr +#include +#include +#include +#include +static void __Pyx_CppExn2PyErr() { + try { + if (PyErr_Occurred()) + ; // let the latest Python exn pass through and ignore the current one + else + throw; + } catch (const std::bad_alloc& exn) { + PyErr_SetString(PyExc_MemoryError, exn.what()); + } catch (const std::bad_cast& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::bad_typeid& exn) { + PyErr_SetString(PyExc_TypeError, exn.what()); + } catch (const std::domain_error& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::invalid_argument& exn) { + PyErr_SetString(PyExc_ValueError, exn.what()); + } catch (const std::ios_base::failure& exn) { + PyErr_SetString(PyExc_IOError, exn.what()); + } catch (const std::out_of_range& exn) { + PyErr_SetString(PyExc_IndexError, exn.what()); + } catch (const std::overflow_error& exn) { + PyErr_SetString(PyExc_OverflowError, exn.what()); + } catch (const std::range_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::underflow_error& exn) { + PyErr_SetString(PyExc_ArithmeticError, exn.what()); + } catch (const std::exception& exn) { + PyErr_SetString(PyExc_RuntimeError, exn.what()); + } + catch (...) + { + PyErr_SetString(PyExc_RuntimeError, "Unknown exception"); + } +} +#endif + +/* GCCDiagnostics.proto */ +#if !defined(__INTEL_COMPILER) && defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) +#define __Pyx_HAS_GCC_DIAGNOSTIC +#endif + +/* LengthHint.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +#define __Pyx_PyObject_LengthHint(o, defaultval) (defaultval) +#else +#define __Pyx_PyObject_LengthHint(o, defaultval) PyObject_LengthHint(o, defaultval) +#endif + +/* CIntFromPy.proto */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int32_t __Pyx_PyLong_As_int32_t(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE int64_t __Pyx_PyLong_As_int64_t(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE char __Pyx_PyLong_As_char(PyObject *); + +/* CIntFromPy.proto */ +static CYTHON_INLINE size_t __Pyx_PyLong_As_size_t(PyObject *); + +/* FormatTypeName.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API +typedef PyObject *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%U" +#define __Pyx_DECREF_TypeName(obj) Py_XDECREF(obj) +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 +#define __Pyx_PyType_GetFullyQualifiedName PyType_GetFullyQualifiedName +#else +static __Pyx_TypeName __Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp); +#endif +#else // !LIMITED_API +typedef const char *__Pyx_TypeName; +#define __Pyx_FMT_TYPENAME "%.200s" +#define __Pyx_PyType_GetFullyQualifiedName(tp) ((tp)->tp_name) +#define __Pyx_DECREF_TypeName(obj) +#endif + +/* PyObjectVectorCallKwBuilder.proto */ +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#if CYTHON_VECTORCALL +#if PY_VERSION_HEX >= 0x03090000 +#define __Pyx_Object_Vectorcall_CallFromBuilder PyObject_Vectorcall +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder _PyObject_Vectorcall +#endif +#define __Pyx_MakeVectorcallBuilderKwds(n) PyTuple_New(n) +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n); +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n); +#else +#define __Pyx_Object_Vectorcall_CallFromBuilder __Pyx_PyObject_FastCallDict +#define __Pyx_MakeVectorcallBuilderKwds(n) __Pyx_PyDict_NewPresized(n) +#define __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n) PyDict_SetItem(builder, key, value) +#define __Pyx_VectorcallBuilder_AddArgStr(key, value, builder, args, n) PyDict_SetItemString(builder, key, value) +#endif + +/* CIntToPy.proto */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value); + +/* CIntFromPy.proto */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *); + +/* FastTypeChecks.proto */ +#if CYTHON_COMPILING_IN_CPYTHON +#define __Pyx_TypeCheck(obj, type) __Pyx_IsSubtype(Py_TYPE(obj), (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) __Pyx_IsAnySubtype2(Py_TYPE(obj), (PyTypeObject *)type1, (PyTypeObject *)type2) +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches(PyObject *err, PyObject *type); +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2); +#else +#define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_TypeCheck2(obj, type1, type2) (PyObject_TypeCheck(obj, (PyTypeObject *)type1) || PyObject_TypeCheck(obj, (PyTypeObject *)type2)) +#define __Pyx_PyErr_GivenExceptionMatches(err, type) PyErr_GivenExceptionMatches(err, type) +static CYTHON_INLINE int __Pyx_PyErr_GivenExceptionMatches2(PyObject *err, PyObject *type1, PyObject *type2) { + return PyErr_GivenExceptionMatches(err, type1) || PyErr_GivenExceptionMatches(err, type2); +} +#endif +#define __Pyx_PyErr_ExceptionMatches2(err1, err2) __Pyx_PyErr_GivenExceptionMatches2(__Pyx_PyErr_CurrentExceptionType(), err1, err2) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) +#ifdef PyExceptionInstance_Check + #define __Pyx_PyBaseException_Check(obj) PyExceptionInstance_Check(obj) +#else + #define __Pyx_PyBaseException_Check(obj) __Pyx_TypeCheck(obj, PyExc_BaseException) +#endif + +/* GetRuntimeVersion.proto */ +static unsigned long __Pyx_get_runtime_version(void); + +/* CheckBinaryVersion.proto */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer); + +/* FunctionExport.proto */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig); + +/* MultiPhaseInitModuleState.proto */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +static PyObject *__Pyx_State_FindModule(void*); +static int __Pyx_State_AddModule(PyObject* module, void*); +static int __Pyx_State_RemoveModule(void*); +#elif CYTHON_USE_MODULE_STATE +#define __Pyx_State_FindModule PyState_FindModule +#define __Pyx_State_AddModule PyState_AddModule +#define __Pyx_State_RemoveModule PyState_RemoveModule +#endif + +/* #### Code section: module_declarations ### */ +/* CythonABIVersion.proto */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if CYTHON_METH_FASTCALL + #define __PYX_FASTCALL_ABI_SUFFIX "_fastcall" + #else + #define __PYX_FASTCALL_ABI_SUFFIX + #endif + #define __PYX_LIMITED_ABI_SUFFIX "limited" __PYX_FASTCALL_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#else + #define __PYX_LIMITED_ABI_SUFFIX +#endif +#if __PYX_HAS_PY_AM_SEND == 1 + #define __PYX_AM_SEND_ABI_SUFFIX +#elif __PYX_HAS_PY_AM_SEND == 2 + #define __PYX_AM_SEND_ABI_SUFFIX "amsendbackport" +#else + #define __PYX_AM_SEND_ABI_SUFFIX "noamsend" +#endif +#ifndef __PYX_MONITORING_ABI_SUFFIX + #define __PYX_MONITORING_ABI_SUFFIX +#endif +#if CYTHON_USE_TP_FINALIZE + #define __PYX_TP_FINALIZE_ABI_SUFFIX +#else + #define __PYX_TP_FINALIZE_ABI_SUFFIX "nofinalize" +#endif +#if CYTHON_USE_FREELISTS || !defined(__Pyx_AsyncGen_USED) + #define __PYX_FREELISTS_ABI_SUFFIX +#else + #define __PYX_FREELISTS_ABI_SUFFIX "nofreelists" +#endif +#define CYTHON_ABI __PYX_ABI_VERSION __PYX_LIMITED_ABI_SUFFIX __PYX_MONITORING_ABI_SUFFIX __PYX_TP_FINALIZE_ABI_SUFFIX __PYX_FREELISTS_ABI_SUFFIX __PYX_AM_SEND_ABI_SUFFIX +#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI +#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "." + +#if !CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE double __pyx_f_7cpython_7complex_7complex_4real_real(PyComplexObject *__pyx_v_self); /* proto*/ +#endif +#if !CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE double __pyx_f_7cpython_7complex_7complex_4imag_imag(PyComplexObject *__pyx_v_self); /* proto*/ +#endif + +/* Module declarations from "libc.stdint" */ + +/* Module declarations from "libcpp.vector" */ + +/* Module declarations from "libcpp" */ + +/* Module declarations from "libcpp.memory" */ + +/* Module declarations from "cpython.version" */ + +/* Module declarations from "__builtin__" */ + +/* Module declarations from "cpython.type" */ + +/* Module declarations from "libc.string" */ + +/* Module declarations from "libc.stdio" */ + +/* Module declarations from "cpython.object" */ + +/* Module declarations from "cpython.ref" */ + +/* Module declarations from "cpython.exc" */ + +/* Module declarations from "cpython.module" */ + +/* Module declarations from "cpython.mem" */ + +/* Module declarations from "cpython.tuple" */ + +/* Module declarations from "cpython.list" */ + +/* Module declarations from "cpython.sequence" */ + +/* Module declarations from "cpython.mapping" */ + +/* Module declarations from "cpython.iterator" */ + +/* Module declarations from "cpython.number" */ + +/* Module declarations from "__builtin__" */ + +/* Module declarations from "cpython.bool" */ + +/* Module declarations from "cpython.long" */ + +/* Module declarations from "cpython.float" */ + +/* Module declarations from "cython" */ + +/* Module declarations from "__builtin__" */ + +/* Module declarations from "cpython.complex" */ + +/* Module declarations from "libc.stddef" */ + +/* Module declarations from "cpython.unicode" */ + +/* Module declarations from "cpython.pyport" */ + +/* Module declarations from "cpython.dict" */ + +/* Module declarations from "cpython.instance" */ + +/* Module declarations from "cpython.function" */ + +/* Module declarations from "cpython.method" */ + +/* Module declarations from "cpython.weakref" */ + +/* Module declarations from "cpython.getargs" */ + +/* Module declarations from "cpython.pythread" */ + +/* Module declarations from "cpython.pystate" */ + +/* Module declarations from "cpython.set" */ + +/* Module declarations from "cpython.buffer" */ + +/* Module declarations from "cpython.bytes" */ + +/* Module declarations from "cpython.pycapsule" */ + +/* Module declarations from "cpython.contextvars" */ + +/* Module declarations from "cpython" */ + +/* Module declarations from "libcpp.utility" */ + +/* Module declarations from "cuda.bindings._internal.utils" */ +static int __pyx_f_4cuda_8bindings_9_internal_5utils_is_nested_sequence(PyObject *); /*proto*/ +static int __pyx_fuse_0__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr(nullable_unique_ptr > &, PyObject *, int *); /*proto*/ +static int __pyx_fuse_1__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr(nullable_unique_ptr > &, PyObject *, int32_t *); /*proto*/ +static int __pyx_fuse_2__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr(nullable_unique_ptr > &, PyObject *, int64_t *); /*proto*/ +static int __pyx_fuse_3__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr(nullable_unique_ptr > &, PyObject *, char *); /*proto*/ +static int __pyx_fuse_4__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr(nullable_unique_ptr > &, PyObject *, float *); /*proto*/ +static int __pyx_fuse_5__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr(nullable_unique_ptr > &, PyObject *, double *); /*proto*/ +static int __pyx_fuse_0__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptrs(nullable_unique_ptr > &, PyObject *, void *); /*proto*/ +static int __pyx_fuse_0__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr(__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &, PyObject *, int *); /*proto*/ +static int __pyx_fuse_1__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr(__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &, PyObject *, int32_t *); /*proto*/ +static int __pyx_fuse_2__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr(__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &, PyObject *, int64_t *); /*proto*/ +static int __pyx_fuse_3__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr(__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &, PyObject *, char *); /*proto*/ +static int __pyx_fuse_4__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr(__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &, PyObject *, float *); /*proto*/ +static int __pyx_fuse_5__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr(__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &, PyObject *, double *); /*proto*/ +static std::vector __pyx_convert_vector_from_py_int(PyObject *); /*proto*/ +static std::vector __pyx_convert_vector_from_py_int32_t(PyObject *); /*proto*/ +static std::vector __pyx_convert_vector_from_py_int64_t(PyObject *); /*proto*/ +static std::vector __pyx_convert_vector_from_py_float(PyObject *); /*proto*/ +static std::vector __pyx_convert_vector_from_py_double(PyObject *); /*proto*/ +/* #### Code section: typeinfo ### */ +/* #### Code section: before_global_var ### */ +#define __Pyx_MODULE_NAME "cuda.bindings._internal.utils" +extern int __pyx_module_is_main_cuda__bindings___internal__utils; +int __pyx_module_is_main_cuda__bindings___internal__utils = 0; + +/* Implementation of "cuda.bindings._internal.utils" */ +/* #### Code section: global_var ### */ +static PyObject *__pyx_builtin_RuntimeError; +static PyObject *__pyx_builtin_AssertionError; +static PyObject *__pyx_builtin_ValueError; +static PyObject *__pyx_builtin_range; +static PyObject *__pyx_builtin_enumerate; +/* #### Code section: string_decls ### */ +static const char __pyx_k_[] = ""; +static const char __pyx_k__2[] = "?"; +static const char __pyx_k_doc[] = "__doc__"; +static const char __pyx_k_None[] = "None"; +static const char __pyx_k_main[] = "__main__"; +static const char __pyx_k_name[] = "__name__"; +static const char __pyx_k_test[] = "__test__"; +static const char __pyx_k_range[] = "range"; +static const char __pyx_k_module[] = "__module__"; +static const char __pyx_k_prepare[] = "__prepare__"; +static const char __pyx_k_qualname[] = "__qualname__"; +static const char __pyx_k_writable[] = "writable "; +static const char __pyx_k_enumerate[] = "enumerate"; +static const char __pyx_k_metaclass[] = "__metaclass__"; +static const char __pyx_k_ValueError[] = "ValueError"; +static const char __pyx_k_mro_entries[] = "__mro_entries__"; +static const char __pyx_k_RuntimeError[] = "RuntimeError"; +static const char __pyx_k_AssertionError[] = "AssertionError"; +static const char __pyx_k_NotSupportedError[] = "NotSupportedError"; +static const char __pyx_k_cline_in_traceback[] = "cline_in_traceback"; +static const char __pyx_k_buffer_of_size_bytes[] = "buffer, of size bytes"; +static const char __pyx_k_FunctionNotFoundError[] = "FunctionNotFoundError"; +static const char __pyx_k_cuda_bindings__internal_utils[] = "cuda.bindings._internal.utils"; +static const char __pyx_k_buf_must_be_either_a_Python_int[] = "buf must be either a Python int representing the pointer address to a valid buffer, or a 1D contiguous "; +/* #### Code section: decls ### */ +/* #### Code section: late_includes ### */ +/* #### Code section: module_state ### */ +/* SmallCodeConfig */ +#ifndef CYTHON_SMALL_CODE +#if defined(__clang__) + #define CYTHON_SMALL_CODE +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) + #define CYTHON_SMALL_CODE __attribute__((cold)) +#else + #define CYTHON_SMALL_CODE +#endif +#endif + +typedef struct { + PyObject *__pyx_d; + PyObject *__pyx_b; + PyObject *__pyx_cython_runtime; + PyObject *__pyx_empty_tuple; + PyObject *__pyx_empty_bytes; + PyObject *__pyx_empty_unicode; + #ifdef __Pyx_CyFunction_USED + PyTypeObject *__pyx_CyFunctionType; + #endif + #ifdef __Pyx_FusedFunction_USED + PyTypeObject *__pyx_FusedFunctionType; + #endif + #ifdef __Pyx_Generator_USED + PyTypeObject *__pyx_GeneratorType; + #endif + #ifdef __Pyx_IterableCoroutine_USED + PyTypeObject *__pyx_IterableCoroutineType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineAwaitType; + #endif + #ifdef __Pyx_Coroutine_USED + PyTypeObject *__pyx_CoroutineType; + #endif + PyTypeObject *__pyx_ptype_7cpython_4type_type; + PyTypeObject *__pyx_ptype_7cpython_4bool_bool; + PyTypeObject *__pyx_ptype_7cpython_7complex_complex; + PyObject *__pyx_tuple[4]; + PyObject *__pyx_string_tab[24]; +/* #### Code section: module_state_contents ### */ +/* CodeObjectCache.module_state_decls */ +struct __Pyx_CodeObjectCache __pyx_code_cache; + +/* #### Code section: module_state_end ### */ +} __pyx_mstatetype; + +#if CYTHON_USE_MODULE_STATE +#ifdef __cplusplus +namespace { +extern struct PyModuleDef __pyx_moduledef; +} /* anonymous namespace */ +#else +static struct PyModuleDef __pyx_moduledef; +#endif + +#define __pyx_mstate_global (__Pyx_PyModule_GetState(__Pyx_State_FindModule(&__pyx_moduledef))) + +#define __pyx_m (__Pyx_State_FindModule(&__pyx_moduledef)) +#else +static __pyx_mstatetype __pyx_mstate_global_static = +#ifdef __cplusplus + {}; +#else + {0}; +#endif +static __pyx_mstatetype * const __pyx_mstate_global = &__pyx_mstate_global_static; +#endif +/* #### Code section: constant_name_defines ### */ +#define __pyx_kp_u_ __pyx_string_tab[0] +#define __pyx_n_u_AssertionError __pyx_string_tab[1] +#define __pyx_n_u_FunctionNotFoundError __pyx_string_tab[2] +#define __pyx_kp_u_None __pyx_string_tab[3] +#define __pyx_n_u_NotSupportedError __pyx_string_tab[4] +#define __pyx_n_u_RuntimeError __pyx_string_tab[5] +#define __pyx_n_u_ValueError __pyx_string_tab[6] +#define __pyx_kp_u__2 __pyx_string_tab[7] +#define __pyx_kp_u_buf_must_be_either_a_Python_int __pyx_string_tab[8] +#define __pyx_kp_u_buffer_of_size_bytes __pyx_string_tab[9] +#define __pyx_n_u_cline_in_traceback __pyx_string_tab[10] +#define __pyx_n_u_cuda_bindings__internal_utils __pyx_string_tab[11] +#define __pyx_n_u_doc __pyx_string_tab[12] +#define __pyx_n_u_enumerate __pyx_string_tab[13] +#define __pyx_n_u_main __pyx_string_tab[14] +#define __pyx_n_u_metaclass __pyx_string_tab[15] +#define __pyx_n_u_module __pyx_string_tab[16] +#define __pyx_n_u_mro_entries __pyx_string_tab[17] +#define __pyx_n_u_name __pyx_string_tab[18] +#define __pyx_n_u_prepare __pyx_string_tab[19] +#define __pyx_n_u_qualname __pyx_string_tab[20] +#define __pyx_n_u_range __pyx_string_tab[21] +#define __pyx_n_u_test __pyx_string_tab[22] +#define __pyx_kp_u_writable __pyx_string_tab[23] +/* #### Code section: module_state_clear ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_clear(PyObject *m) { + __pyx_mstatetype *clear_module_state = __Pyx_PyModule_GetState(m); + if (!clear_module_state) return 0; + Py_CLEAR(clear_module_state->__pyx_d); + Py_CLEAR(clear_module_state->__pyx_b); + Py_CLEAR(clear_module_state->__pyx_cython_runtime); + Py_CLEAR(clear_module_state->__pyx_empty_tuple); + Py_CLEAR(clear_module_state->__pyx_empty_bytes); + Py_CLEAR(clear_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_CLEAR(clear_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_CLEAR(clear_module_state->__pyx_FusedFunctionType); + #endif + #if CYTHON_PEP489_MULTI_PHASE_INIT + __Pyx_State_RemoveModule(NULL); + #endif + Py_CLEAR(clear_module_state->__pyx_ptype_7cpython_4type_type); + Py_CLEAR(clear_module_state->__pyx_ptype_7cpython_4bool_bool); + Py_CLEAR(clear_module_state->__pyx_ptype_7cpython_7complex_complex); + for (int i=0; i<4; ++i) { Py_CLEAR(clear_module_state->__pyx_tuple[i]); } + for (int i=0; i<24; ++i) { Py_CLEAR(clear_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_state_traverse ### */ +#if CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __pyx_m_traverse(PyObject *m, visitproc visit, void *arg) { + __pyx_mstatetype *traverse_module_state = __Pyx_PyModule_GetState(m); + if (!traverse_module_state) return 0; + Py_VISIT(traverse_module_state->__pyx_d); + Py_VISIT(traverse_module_state->__pyx_b); + Py_VISIT(traverse_module_state->__pyx_cython_runtime); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_tuple); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_bytes); + __Pyx_VISIT_CONST(traverse_module_state->__pyx_empty_unicode); + #ifdef __Pyx_CyFunction_USED + Py_VISIT(traverse_module_state->__pyx_CyFunctionType); + #endif + #ifdef __Pyx_FusedFunction_USED + Py_VISIT(traverse_module_state->__pyx_FusedFunctionType); + #endif + Py_VISIT(traverse_module_state->__pyx_ptype_7cpython_4type_type); + Py_VISIT(traverse_module_state->__pyx_ptype_7cpython_4bool_bool); + Py_VISIT(traverse_module_state->__pyx_ptype_7cpython_7complex_complex); + for (int i=0; i<4; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_tuple[i]); } + for (int i=0; i<24; ++i) { __Pyx_VISIT_CONST(traverse_module_state->__pyx_string_tab[i]); } + return 0; +} +#endif +/* #### Code section: module_code ### */ + +/* "vector.from_py":51 + * cdef Py_ssize_t __Pyx_PyObject_LengthHint(object o, Py_ssize_t defaultval) except -1 + * + * @cname("__pyx_convert_vector_from_py_int") # <<<<<<<<<<<<<< + * cdef vector[X] __pyx_convert_vector_from_py_int(object o) except *: + * +*/ + +static std::vector __pyx_convert_vector_from_py_int(PyObject *__pyx_v_o) { + std::vector __pyx_v_v; + Py_ssize_t __pyx_v_s; + PyObject *__pyx_v_item = NULL; + std::vector __pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *(*__pyx_t_4)(PyObject *); + PyObject *__pyx_t_5 = NULL; + int __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_convert_vector_from_py_int", 0); + + /* "vector.from_py":55 + * + * cdef vector[X] v + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) # <<<<<<<<<<<<<< + * + * if s > 0: +*/ + __pyx_t_1 = __Pyx_PyObject_LengthHint(__pyx_v_o, 0); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(1, 55, __pyx_L1_error) + __pyx_v_s = __pyx_t_1; + + /* "vector.from_py":57 + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) + * + * if s > 0: # <<<<<<<<<<<<<< + * v.reserve( s) + * +*/ + __pyx_t_2 = (__pyx_v_s > 0); + if (__pyx_t_2) { + + /* "vector.from_py":58 + * + * if s > 0: + * v.reserve( s) # <<<<<<<<<<<<<< + * + * for item in o: +*/ + try { + __pyx_v_v.reserve(((size_t)__pyx_v_s)); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(1, 58, __pyx_L1_error) + } + + /* "vector.from_py":57 + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) + * + * if s > 0: # <<<<<<<<<<<<<< + * v.reserve( s) + * +*/ + } + + /* "vector.from_py":60 + * v.reserve( s) + * + * for item in o: # <<<<<<<<<<<<<< + * v.push_back(item) + * +*/ + if (likely(PyList_CheckExact(__pyx_v_o)) || PyTuple_CheckExact(__pyx_v_o)) { + __pyx_t_3 = __pyx_v_o; __Pyx_INCREF(__pyx_t_3); + __pyx_t_1 = 0; + __pyx_t_4 = NULL; + } else { + __pyx_t_1 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 60, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 60, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_4)) { + if (likely(PyList_CheckExact(__pyx_t_3))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_3); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 60, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + __pyx_t_5 = __Pyx_PyList_GetItemRef(__pyx_t_3, __pyx_t_1); + ++__pyx_t_1; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_3); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 60, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_1)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_3, __pyx_t_1); + #endif + ++__pyx_t_1; + } + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 60, __pyx_L1_error) + } else { + __pyx_t_5 = __pyx_t_4(__pyx_t_3); + if (unlikely(!__pyx_t_5)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 60, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_5); + __pyx_t_5 = 0; + + /* "vector.from_py":61 + * + * for item in o: + * v.push_back(item) # <<<<<<<<<<<<<< + * + * return v +*/ + __pyx_t_6 = __Pyx_PyLong_As_int(__pyx_v_item); if (unlikely((__pyx_t_6 == (int)-1) && PyErr_Occurred())) __PYX_ERR(1, 61, __pyx_L1_error) + try { + __pyx_v_v.push_back(((int)__pyx_t_6)); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(1, 61, __pyx_L1_error) + } + + /* "vector.from_py":60 + * v.reserve( s) + * + * for item in o: # <<<<<<<<<<<<<< + * v.push_back(item) + * +*/ + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "vector.from_py":63 + * v.push_back(item) + * + * return v # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_v; + goto __pyx_L0; + + /* "vector.from_py":51 + * cdef Py_ssize_t __Pyx_PyObject_LengthHint(object o, Py_ssize_t defaultval) except -1 + * + * @cname("__pyx_convert_vector_from_py_int") # <<<<<<<<<<<<<< + * cdef vector[X] __pyx_convert_vector_from_py_int(object o) except *: + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("vector.from_py.__pyx_convert_vector_from_py_int", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_pretend_to_initialize(&__pyx_r); + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_item); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static std::vector __pyx_convert_vector_from_py_int32_t(PyObject *__pyx_v_o) { + std::vector __pyx_v_v; + Py_ssize_t __pyx_v_s; + PyObject *__pyx_v_item = NULL; + std::vector __pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *(*__pyx_t_4)(PyObject *); + PyObject *__pyx_t_5 = NULL; + int32_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_convert_vector_from_py_int32_t", 0); + + /* "vector.from_py":55 + * + * cdef vector[X] v + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) # <<<<<<<<<<<<<< + * + * if s > 0: +*/ + __pyx_t_1 = __Pyx_PyObject_LengthHint(__pyx_v_o, 0); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(1, 55, __pyx_L1_error) + __pyx_v_s = __pyx_t_1; + + /* "vector.from_py":57 + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) + * + * if s > 0: # <<<<<<<<<<<<<< + * v.reserve( s) + * +*/ + __pyx_t_2 = (__pyx_v_s > 0); + if (__pyx_t_2) { + + /* "vector.from_py":58 + * + * if s > 0: + * v.reserve( s) # <<<<<<<<<<<<<< + * + * for item in o: +*/ + try { + __pyx_v_v.reserve(((size_t)__pyx_v_s)); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(1, 58, __pyx_L1_error) + } + + /* "vector.from_py":57 + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) + * + * if s > 0: # <<<<<<<<<<<<<< + * v.reserve( s) + * +*/ + } + + /* "vector.from_py":60 + * v.reserve( s) + * + * for item in o: # <<<<<<<<<<<<<< + * v.push_back(item) + * +*/ + if (likely(PyList_CheckExact(__pyx_v_o)) || PyTuple_CheckExact(__pyx_v_o)) { + __pyx_t_3 = __pyx_v_o; __Pyx_INCREF(__pyx_t_3); + __pyx_t_1 = 0; + __pyx_t_4 = NULL; + } else { + __pyx_t_1 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 60, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 60, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_4)) { + if (likely(PyList_CheckExact(__pyx_t_3))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_3); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 60, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + __pyx_t_5 = __Pyx_PyList_GetItemRef(__pyx_t_3, __pyx_t_1); + ++__pyx_t_1; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_3); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 60, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_1)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_3, __pyx_t_1); + #endif + ++__pyx_t_1; + } + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 60, __pyx_L1_error) + } else { + __pyx_t_5 = __pyx_t_4(__pyx_t_3); + if (unlikely(!__pyx_t_5)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 60, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_5); + __pyx_t_5 = 0; + + /* "vector.from_py":61 + * + * for item in o: + * v.push_back(item) # <<<<<<<<<<<<<< + * + * return v +*/ + __pyx_t_6 = __Pyx_PyLong_As_int32_t(__pyx_v_item); if (unlikely((__pyx_t_6 == ((int32_t)-1)) && PyErr_Occurred())) __PYX_ERR(1, 61, __pyx_L1_error) + try { + __pyx_v_v.push_back(((int32_t)__pyx_t_6)); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(1, 61, __pyx_L1_error) + } + + /* "vector.from_py":60 + * v.reserve( s) + * + * for item in o: # <<<<<<<<<<<<<< + * v.push_back(item) + * +*/ + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "vector.from_py":63 + * v.push_back(item) + * + * return v # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_v; + goto __pyx_L0; + + /* "vector.from_py":51 + * cdef Py_ssize_t __Pyx_PyObject_LengthHint(object o, Py_ssize_t defaultval) except -1 + * + * @cname("__pyx_convert_vector_from_py_int32_t") # <<<<<<<<<<<<<< + * cdef vector[X] __pyx_convert_vector_from_py_int32_t(object o) except *: + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("vector.from_py.__pyx_convert_vector_from_py_int32_t", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_pretend_to_initialize(&__pyx_r); + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_item); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static std::vector __pyx_convert_vector_from_py_int64_t(PyObject *__pyx_v_o) { + std::vector __pyx_v_v; + Py_ssize_t __pyx_v_s; + PyObject *__pyx_v_item = NULL; + std::vector __pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *(*__pyx_t_4)(PyObject *); + PyObject *__pyx_t_5 = NULL; + int64_t __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_convert_vector_from_py_int64_t", 0); + + /* "vector.from_py":55 + * + * cdef vector[X] v + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) # <<<<<<<<<<<<<< + * + * if s > 0: +*/ + __pyx_t_1 = __Pyx_PyObject_LengthHint(__pyx_v_o, 0); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(1, 55, __pyx_L1_error) + __pyx_v_s = __pyx_t_1; + + /* "vector.from_py":57 + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) + * + * if s > 0: # <<<<<<<<<<<<<< + * v.reserve( s) + * +*/ + __pyx_t_2 = (__pyx_v_s > 0); + if (__pyx_t_2) { + + /* "vector.from_py":58 + * + * if s > 0: + * v.reserve( s) # <<<<<<<<<<<<<< + * + * for item in o: +*/ + try { + __pyx_v_v.reserve(((size_t)__pyx_v_s)); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(1, 58, __pyx_L1_error) + } + + /* "vector.from_py":57 + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) + * + * if s > 0: # <<<<<<<<<<<<<< + * v.reserve( s) + * +*/ + } + + /* "vector.from_py":60 + * v.reserve( s) + * + * for item in o: # <<<<<<<<<<<<<< + * v.push_back(item) + * +*/ + if (likely(PyList_CheckExact(__pyx_v_o)) || PyTuple_CheckExact(__pyx_v_o)) { + __pyx_t_3 = __pyx_v_o; __Pyx_INCREF(__pyx_t_3); + __pyx_t_1 = 0; + __pyx_t_4 = NULL; + } else { + __pyx_t_1 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 60, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 60, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_4)) { + if (likely(PyList_CheckExact(__pyx_t_3))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_3); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 60, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + __pyx_t_5 = __Pyx_PyList_GetItemRef(__pyx_t_3, __pyx_t_1); + ++__pyx_t_1; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_3); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 60, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_1)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_3, __pyx_t_1); + #endif + ++__pyx_t_1; + } + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 60, __pyx_L1_error) + } else { + __pyx_t_5 = __pyx_t_4(__pyx_t_3); + if (unlikely(!__pyx_t_5)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 60, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_5); + __pyx_t_5 = 0; + + /* "vector.from_py":61 + * + * for item in o: + * v.push_back(item) # <<<<<<<<<<<<<< + * + * return v +*/ + __pyx_t_6 = __Pyx_PyLong_As_int64_t(__pyx_v_item); if (unlikely((__pyx_t_6 == ((int64_t)-1)) && PyErr_Occurred())) __PYX_ERR(1, 61, __pyx_L1_error) + try { + __pyx_v_v.push_back(((int64_t)__pyx_t_6)); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(1, 61, __pyx_L1_error) + } + + /* "vector.from_py":60 + * v.reserve( s) + * + * for item in o: # <<<<<<<<<<<<<< + * v.push_back(item) + * +*/ + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "vector.from_py":63 + * v.push_back(item) + * + * return v # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_v; + goto __pyx_L0; + + /* "vector.from_py":51 + * cdef Py_ssize_t __Pyx_PyObject_LengthHint(object o, Py_ssize_t defaultval) except -1 + * + * @cname("__pyx_convert_vector_from_py_int64_t") # <<<<<<<<<<<<<< + * cdef vector[X] __pyx_convert_vector_from_py_int64_t(object o) except *: + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("vector.from_py.__pyx_convert_vector_from_py_int64_t", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_pretend_to_initialize(&__pyx_r); + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_item); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static std::vector __pyx_convert_vector_from_py_float(PyObject *__pyx_v_o) { + std::vector __pyx_v_v; + Py_ssize_t __pyx_v_s; + PyObject *__pyx_v_item = NULL; + std::vector __pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *(*__pyx_t_4)(PyObject *); + PyObject *__pyx_t_5 = NULL; + float __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_convert_vector_from_py_float", 0); + + /* "vector.from_py":55 + * + * cdef vector[X] v + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) # <<<<<<<<<<<<<< + * + * if s > 0: +*/ + __pyx_t_1 = __Pyx_PyObject_LengthHint(__pyx_v_o, 0); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(1, 55, __pyx_L1_error) + __pyx_v_s = __pyx_t_1; + + /* "vector.from_py":57 + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) + * + * if s > 0: # <<<<<<<<<<<<<< + * v.reserve( s) + * +*/ + __pyx_t_2 = (__pyx_v_s > 0); + if (__pyx_t_2) { + + /* "vector.from_py":58 + * + * if s > 0: + * v.reserve( s) # <<<<<<<<<<<<<< + * + * for item in o: +*/ + try { + __pyx_v_v.reserve(((size_t)__pyx_v_s)); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(1, 58, __pyx_L1_error) + } + + /* "vector.from_py":57 + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) + * + * if s > 0: # <<<<<<<<<<<<<< + * v.reserve( s) + * +*/ + } + + /* "vector.from_py":60 + * v.reserve( s) + * + * for item in o: # <<<<<<<<<<<<<< + * v.push_back(item) + * +*/ + if (likely(PyList_CheckExact(__pyx_v_o)) || PyTuple_CheckExact(__pyx_v_o)) { + __pyx_t_3 = __pyx_v_o; __Pyx_INCREF(__pyx_t_3); + __pyx_t_1 = 0; + __pyx_t_4 = NULL; + } else { + __pyx_t_1 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 60, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 60, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_4)) { + if (likely(PyList_CheckExact(__pyx_t_3))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_3); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 60, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + __pyx_t_5 = __Pyx_PyList_GetItemRef(__pyx_t_3, __pyx_t_1); + ++__pyx_t_1; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_3); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 60, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_1)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_3, __pyx_t_1); + #endif + ++__pyx_t_1; + } + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 60, __pyx_L1_error) + } else { + __pyx_t_5 = __pyx_t_4(__pyx_t_3); + if (unlikely(!__pyx_t_5)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 60, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_5); + __pyx_t_5 = 0; + + /* "vector.from_py":61 + * + * for item in o: + * v.push_back(item) # <<<<<<<<<<<<<< + * + * return v +*/ + __pyx_t_6 = __Pyx_PyFloat_AsFloat(__pyx_v_item); if (unlikely((__pyx_t_6 == (float)-1) && PyErr_Occurred())) __PYX_ERR(1, 61, __pyx_L1_error) + try { + __pyx_v_v.push_back(((float)__pyx_t_6)); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(1, 61, __pyx_L1_error) + } + + /* "vector.from_py":60 + * v.reserve( s) + * + * for item in o: # <<<<<<<<<<<<<< + * v.push_back(item) + * +*/ + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "vector.from_py":63 + * v.push_back(item) + * + * return v # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_v; + goto __pyx_L0; + + /* "vector.from_py":51 + * cdef Py_ssize_t __Pyx_PyObject_LengthHint(object o, Py_ssize_t defaultval) except -1 + * + * @cname("__pyx_convert_vector_from_py_float") # <<<<<<<<<<<<<< + * cdef vector[X] __pyx_convert_vector_from_py_float(object o) except *: + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("vector.from_py.__pyx_convert_vector_from_py_float", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_pretend_to_initialize(&__pyx_r); + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_item); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static std::vector __pyx_convert_vector_from_py_double(PyObject *__pyx_v_o) { + std::vector __pyx_v_v; + Py_ssize_t __pyx_v_s; + PyObject *__pyx_v_item = NULL; + std::vector __pyx_r; + __Pyx_RefNannyDeclarations + Py_ssize_t __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + PyObject *(*__pyx_t_4)(PyObject *); + PyObject *__pyx_t_5 = NULL; + double __pyx_t_6; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_convert_vector_from_py_double", 0); + + /* "vector.from_py":55 + * + * cdef vector[X] v + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) # <<<<<<<<<<<<<< + * + * if s > 0: +*/ + __pyx_t_1 = __Pyx_PyObject_LengthHint(__pyx_v_o, 0); if (unlikely(__pyx_t_1 == ((Py_ssize_t)-1))) __PYX_ERR(1, 55, __pyx_L1_error) + __pyx_v_s = __pyx_t_1; + + /* "vector.from_py":57 + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) + * + * if s > 0: # <<<<<<<<<<<<<< + * v.reserve( s) + * +*/ + __pyx_t_2 = (__pyx_v_s > 0); + if (__pyx_t_2) { + + /* "vector.from_py":58 + * + * if s > 0: + * v.reserve( s) # <<<<<<<<<<<<<< + * + * for item in o: +*/ + try { + __pyx_v_v.reserve(((size_t)__pyx_v_s)); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(1, 58, __pyx_L1_error) + } + + /* "vector.from_py":57 + * cdef Py_ssize_t s = __Pyx_PyObject_LengthHint(o, 0) + * + * if s > 0: # <<<<<<<<<<<<<< + * v.reserve( s) + * +*/ + } + + /* "vector.from_py":60 + * v.reserve( s) + * + * for item in o: # <<<<<<<<<<<<<< + * v.push_back(item) + * +*/ + if (likely(PyList_CheckExact(__pyx_v_o)) || PyTuple_CheckExact(__pyx_v_o)) { + __pyx_t_3 = __pyx_v_o; __Pyx_INCREF(__pyx_t_3); + __pyx_t_1 = 0; + __pyx_t_4 = NULL; + } else { + __pyx_t_1 = -1; __pyx_t_3 = PyObject_GetIter(__pyx_v_o); if (unlikely(!__pyx_t_3)) __PYX_ERR(1, 60, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_3); if (unlikely(!__pyx_t_4)) __PYX_ERR(1, 60, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_4)) { + if (likely(PyList_CheckExact(__pyx_t_3))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_3); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 60, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + __pyx_t_5 = __Pyx_PyList_GetItemRef(__pyx_t_3, __pyx_t_1); + ++__pyx_t_1; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_3); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(1, 60, __pyx_L1_error) + #endif + if (__pyx_t_1 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_3, __pyx_t_1)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_3, __pyx_t_1); + #endif + ++__pyx_t_1; + } + if (unlikely(!__pyx_t_5)) __PYX_ERR(1, 60, __pyx_L1_error) + } else { + __pyx_t_5 = __pyx_t_4(__pyx_t_3); + if (unlikely(!__pyx_t_5)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(1, 60, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_item, __pyx_t_5); + __pyx_t_5 = 0; + + /* "vector.from_py":61 + * + * for item in o: + * v.push_back(item) # <<<<<<<<<<<<<< + * + * return v +*/ + __pyx_t_6 = __Pyx_PyFloat_AsDouble(__pyx_v_item); if (unlikely((__pyx_t_6 == (double)-1) && PyErr_Occurred())) __PYX_ERR(1, 61, __pyx_L1_error) + try { + __pyx_v_v.push_back(((double)__pyx_t_6)); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(1, 61, __pyx_L1_error) + } + + /* "vector.from_py":60 + * v.reserve( s) + * + * for item in o: # <<<<<<<<<<<<<< + * v.push_back(item) + * +*/ + } + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + + /* "vector.from_py":63 + * v.push_back(item) + * + * return v # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_v; + goto __pyx_L0; + + /* "vector.from_py":51 + * cdef Py_ssize_t __Pyx_PyObject_LengthHint(object o, Py_ssize_t defaultval) except -1 + * + * @cname("__pyx_convert_vector_from_py_double") # <<<<<<<<<<<<<< + * cdef vector[X] __pyx_convert_vector_from_py_double(object o) except *: + * +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("vector.from_py.__pyx_convert_vector_from_py_double", __pyx_clineno, __pyx_lineno, __pyx_filename); + __Pyx_pretend_to_initialize(&__pyx_r); + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_item); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cpython/complex.pxd":20 + * + * # unavailable in limited API + * @property # <<<<<<<<<<<<<< + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") + * cdef inline double real(self) noexcept: +*/ + +#if !CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE double __pyx_f_7cpython_7complex_7complex_4real_real(PyComplexObject *__pyx_v_self) { + double __pyx_r; + + /* "cpython/complex.pxd":23 + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") + * cdef inline double real(self) noexcept: + * return self.cval.real # <<<<<<<<<<<<<< + * + * # unavailable in limited API +*/ + __pyx_r = __pyx_v_self->cval.real; + goto __pyx_L0; + + /* "cpython/complex.pxd":20 + * + * # unavailable in limited API + * @property # <<<<<<<<<<<<<< + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") + * cdef inline double real(self) noexcept: +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} +#endif /*!(#if !CYTHON_COMPILING_IN_LIMITED_API)*/ + +/* "cpython/complex.pxd":26 + * + * # unavailable in limited API + * @property # <<<<<<<<<<<<<< + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") + * cdef inline double imag(self) noexcept: +*/ + +#if !CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE double __pyx_f_7cpython_7complex_7complex_4imag_imag(PyComplexObject *__pyx_v_self) { + double __pyx_r; + + /* "cpython/complex.pxd":29 + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") + * cdef inline double imag(self) noexcept: + * return self.cval.imag # <<<<<<<<<<<<<< + * + * # PyTypeObject PyComplex_Type +*/ + __pyx_r = __pyx_v_self->cval.imag; + goto __pyx_L0; + + /* "cpython/complex.pxd":26 + * + * # unavailable in limited API + * @property # <<<<<<<<<<<<<< + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") + * cdef inline double imag(self) noexcept: +*/ + + /* function exit code */ + __pyx_L0:; + return __pyx_r; +} +#endif /*!(#if !CYTHON_COMPILING_IN_LIMITED_API)*/ + +/* "cpython/contextvars.pxd":115 + * + * + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") # <<<<<<<<<<<<<< + * cdef inline object get_value(var, default_value=None): + * """Return a new reference to the value of the context variable, +*/ + +#if !CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE PyObject *__pyx_f_7cpython_11contextvars_get_value(PyObject *__pyx_v_var, struct __pyx_opt_args_7cpython_11contextvars_get_value *__pyx_optional_args) { + + /* "cpython/contextvars.pxd":116 + * + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") + * cdef inline object get_value(var, default_value=None): # <<<<<<<<<<<<<< + * """Return a new reference to the value of the context variable, + * or the default value of the context variable, +*/ + PyObject *__pyx_v_default_value = ((PyObject *)Py_None); + PyObject *__pyx_v_value; + PyObject *__pyx_v_pyvalue = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + PyObject *__pyx_t_3 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_value", 0); + if (__pyx_optional_args) { + if (__pyx_optional_args->__pyx_n > 0) { + __pyx_v_default_value = __pyx_optional_args->default_value; + } + } + + /* "cpython/contextvars.pxd":121 + * or None if no such value or default was found. + * """ + * cdef PyObject *value = NULL # <<<<<<<<<<<<<< + * PyContextVar_Get(var, NULL, &value) + * if value is NULL: +*/ + __pyx_v_value = NULL; + + /* "cpython/contextvars.pxd":122 + * """ + * cdef PyObject *value = NULL + * PyContextVar_Get(var, NULL, &value) # <<<<<<<<<<<<<< + * if value is NULL: + * # context variable does not have a default +*/ + __pyx_t_1 = PyContextVar_Get(__pyx_v_var, NULL, (&__pyx_v_value)); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(2, 122, __pyx_L1_error) + + /* "cpython/contextvars.pxd":123 + * cdef PyObject *value = NULL + * PyContextVar_Get(var, NULL, &value) + * if value is NULL: # <<<<<<<<<<<<<< + * # context variable does not have a default + * pyvalue = default_value +*/ + __pyx_t_2 = (__pyx_v_value == NULL); + if (__pyx_t_2) { + + /* "cpython/contextvars.pxd":125 + * if value is NULL: + * # context variable does not have a default + * pyvalue = default_value # <<<<<<<<<<<<<< + * else: + * # value or default value of context variable +*/ + __Pyx_INCREF(__pyx_v_default_value); + __pyx_v_pyvalue = __pyx_v_default_value; + + /* "cpython/contextvars.pxd":123 + * cdef PyObject *value = NULL + * PyContextVar_Get(var, NULL, &value) + * if value is NULL: # <<<<<<<<<<<<<< + * # context variable does not have a default + * pyvalue = default_value +*/ + goto __pyx_L3; + } + + /* "cpython/contextvars.pxd":128 + * else: + * # value or default value of context variable + * pyvalue = value # <<<<<<<<<<<<<< + * Py_XDECREF(value) # PyContextVar_Get() returned an owned reference as 'PyObject*' + * return pyvalue +*/ + /*else*/ { + __pyx_t_3 = ((PyObject *)__pyx_v_value); + __Pyx_INCREF(__pyx_t_3); + __pyx_v_pyvalue = __pyx_t_3; + __pyx_t_3 = 0; + + /* "cpython/contextvars.pxd":129 + * # value or default value of context variable + * pyvalue = value + * Py_XDECREF(value) # PyContextVar_Get() returned an owned reference as 'PyObject*' # <<<<<<<<<<<<<< + * return pyvalue + * +*/ + Py_XDECREF(__pyx_v_value); + } + __pyx_L3:; + + /* "cpython/contextvars.pxd":130 + * pyvalue = value + * Py_XDECREF(value) # PyContextVar_Get() returned an owned reference as 'PyObject*' + * return pyvalue # <<<<<<<<<<<<<< + * + * +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_pyvalue); + __pyx_r = __pyx_v_pyvalue; + goto __pyx_L0; + + /* "cpython/contextvars.pxd":115 + * + * + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") # <<<<<<<<<<<<<< + * cdef inline object get_value(var, default_value=None): + * """Return a new reference to the value of the context variable, +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_3); + __Pyx_AddTraceback("cpython.contextvars.get_value", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_pyvalue); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +#endif /*!(#if !CYTHON_COMPILING_IN_LIMITED_API)*/ + +/* "cpython/contextvars.pxd":133 + * + * + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") # <<<<<<<<<<<<<< + * cdef inline object get_value_no_default(var, default_value=None): + * """Return a new reference to the value of the context variable, +*/ + +#if !CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE PyObject *__pyx_f_7cpython_11contextvars_get_value_no_default(PyObject *__pyx_v_var, struct __pyx_opt_args_7cpython_11contextvars_get_value_no_default *__pyx_optional_args) { + + /* "cpython/contextvars.pxd":134 + * + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") + * cdef inline object get_value_no_default(var, default_value=None): # <<<<<<<<<<<<<< + * """Return a new reference to the value of the context variable, + * or the provided default value if no such value was found. +*/ + PyObject *__pyx_v_default_value = ((PyObject *)Py_None); + PyObject *__pyx_v_value; + PyObject *__pyx_v_pyvalue = NULL; + PyObject *__pyx_r = NULL; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_value_no_default", 0); + if (__pyx_optional_args) { + if (__pyx_optional_args->__pyx_n > 0) { + __pyx_v_default_value = __pyx_optional_args->default_value; + } + } + + /* "cpython/contextvars.pxd":140 + * Ignores the default value of the context variable, if any. + * """ + * cdef PyObject *value = NULL # <<<<<<<<<<<<<< + * PyContextVar_Get(var, default_value, &value) + * # value of context variable or 'default_value' +*/ + __pyx_v_value = NULL; + + /* "cpython/contextvars.pxd":141 + * """ + * cdef PyObject *value = NULL + * PyContextVar_Get(var, default_value, &value) # <<<<<<<<<<<<<< + * # value of context variable or 'default_value' + * pyvalue = value +*/ + __pyx_t_1 = PyContextVar_Get(__pyx_v_var, ((PyObject *)__pyx_v_default_value), (&__pyx_v_value)); if (unlikely(__pyx_t_1 == ((int)-1))) __PYX_ERR(2, 141, __pyx_L1_error) + + /* "cpython/contextvars.pxd":143 + * PyContextVar_Get(var, default_value, &value) + * # value of context variable or 'default_value' + * pyvalue = value # <<<<<<<<<<<<<< + * Py_XDECREF(value) # PyContextVar_Get() returned an owned reference as 'PyObject*' + * return pyvalue +*/ + __pyx_t_2 = ((PyObject *)__pyx_v_value); + __Pyx_INCREF(__pyx_t_2); + __pyx_v_pyvalue = __pyx_t_2; + __pyx_t_2 = 0; + + /* "cpython/contextvars.pxd":144 + * # value of context variable or 'default_value' + * pyvalue = value + * Py_XDECREF(value) # PyContextVar_Get() returned an owned reference as 'PyObject*' # <<<<<<<<<<<<<< + * return pyvalue +*/ + Py_XDECREF(__pyx_v_value); + + /* "cpython/contextvars.pxd":145 + * pyvalue = value + * Py_XDECREF(value) # PyContextVar_Get() returned an owned reference as 'PyObject*' + * return pyvalue # <<<<<<<<<<<<<< +*/ + __Pyx_XDECREF(__pyx_r); + __Pyx_INCREF(__pyx_v_pyvalue); + __pyx_r = __pyx_v_pyvalue; + goto __pyx_L0; + + /* "cpython/contextvars.pxd":133 + * + * + * @_cython.c_compile_guard("!CYTHON_COMPILING_IN_LIMITED_API") # <<<<<<<<<<<<<< + * cdef inline object get_value_no_default(var, default_value=None): + * """Return a new reference to the value of the context variable, +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_AddTraceback("cpython.contextvars.get_value_no_default", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_pyvalue); + __Pyx_XGIVEREF(__pyx_r); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +#endif /*!(#if !CYTHON_COMPILING_IN_LIMITED_API)*/ + +/* "cuda/bindings/_internal/utils.pyx":11 + * + * + * cdef bint is_nested_sequence(data): # <<<<<<<<<<<<<< + * if not cpython.PySequence_Check(data): + * return False +*/ + +static int __pyx_f_4cuda_8bindings_9_internal_5utils_is_nested_sequence(PyObject *__pyx_v_data) { + PyObject *__pyx_v_i = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + PyObject *__pyx_t_2 = NULL; + Py_ssize_t __pyx_t_3; + PyObject *(*__pyx_t_4)(PyObject *); + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("is_nested_sequence", 0); + + /* "cuda/bindings/_internal/utils.pyx":12 + * + * cdef bint is_nested_sequence(data): + * if not cpython.PySequence_Check(data): # <<<<<<<<<<<<<< + * return False + * else: +*/ + __pyx_t_1 = (!PySequence_Check(__pyx_v_data)); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":13 + * cdef bint is_nested_sequence(data): + * if not cpython.PySequence_Check(data): + * return False # <<<<<<<<<<<<<< + * else: + * for i in data: +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":12 + * + * cdef bint is_nested_sequence(data): + * if not cpython.PySequence_Check(data): # <<<<<<<<<<<<<< + * return False + * else: +*/ + } + + /* "cuda/bindings/_internal/utils.pyx":15 + * return False + * else: + * for i in data: # <<<<<<<<<<<<<< + * if not cpython.PySequence_Check(i): + * return False +*/ + /*else*/ { + if (likely(PyList_CheckExact(__pyx_v_data)) || PyTuple_CheckExact(__pyx_v_data)) { + __pyx_t_2 = __pyx_v_data; __Pyx_INCREF(__pyx_t_2); + __pyx_t_3 = 0; + __pyx_t_4 = NULL; + } else { + __pyx_t_3 = -1; __pyx_t_2 = PyObject_GetIter(__pyx_v_data); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 15, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_4 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_2); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 15, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_4)) { + if (likely(PyList_CheckExact(__pyx_t_2))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 15, __pyx_L1_error) + #endif + if (__pyx_t_3 >= __pyx_temp) break; + } + __pyx_t_5 = __Pyx_PyList_GetItemRef(__pyx_t_2, __pyx_t_3); + ++__pyx_t_3; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_2); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 15, __pyx_L1_error) + #endif + if (__pyx_t_3 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_5 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_2, __pyx_t_3)); + #else + __pyx_t_5 = __Pyx_PySequence_ITEM(__pyx_t_2, __pyx_t_3); + #endif + ++__pyx_t_3; + } + if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 15, __pyx_L1_error) + } else { + __pyx_t_5 = __pyx_t_4(__pyx_t_2); + if (unlikely(!__pyx_t_5)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 15, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_5); + __Pyx_XDECREF_SET(__pyx_v_i, __pyx_t_5); + __pyx_t_5 = 0; + + /* "cuda/bindings/_internal/utils.pyx":16 + * else: + * for i in data: + * if not cpython.PySequence_Check(i): # <<<<<<<<<<<<<< + * return False + * else: +*/ + __pyx_t_1 = (!PySequence_Check(__pyx_v_i)); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":17 + * for i in data: + * if not cpython.PySequence_Check(i): + * return False # <<<<<<<<<<<<<< + * else: + * return True +*/ + __pyx_r = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":16 + * else: + * for i in data: + * if not cpython.PySequence_Check(i): # <<<<<<<<<<<<<< + * return False + * else: +*/ + } + + /* "cuda/bindings/_internal/utils.pyx":15 + * return False + * else: + * for i in data: # <<<<<<<<<<<<<< + * if not cpython.PySequence_Check(i): + * return False +*/ + } + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + /*else*/ { + + /* "cuda/bindings/_internal/utils.pyx":19 + * return False + * else: + * return True # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 1; + goto __pyx_L0; + } + } + + /* "cuda/bindings/_internal/utils.pyx":11 + * + * + * cdef bint is_nested_sequence(data): # <<<<<<<<<<<<<< + * if not cpython.PySequence_Check(data): + * return False +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_5); + __Pyx_AddTraceback("cuda.bindings._internal.utils.is_nested_sequence", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = -1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_i); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cuda/bindings/_internal/utils.pyx":22 + * + * + * cdef void* get_buffer_pointer(buf, Py_ssize_t size, readonly=True) except*: # <<<<<<<<<<<<<< + * """The caller must ensure ``buf`` is alive when the returned pointer is in use.""" + * cdef void* bufPtr +*/ + +static void *__pyx_f_4cuda_8bindings_9_internal_5utils_get_buffer_pointer(PyObject *__pyx_v_buf, Py_ssize_t __pyx_v_size, struct __pyx_opt_args_4cuda_8bindings_9_internal_5utils_get_buffer_pointer *__pyx_optional_args) { + PyObject *__pyx_v_readonly = ((PyObject *)Py_True); + void *__pyx_v_bufPtr; + int __pyx_v_flags; + int __pyx_v_status; + Py_buffer __pyx_v_view; + PyObject *__pyx_v_e = NULL; + PyObject *__pyx_v_adj = NULL; + void *__pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + int __pyx_t_2; + intptr_t __pyx_t_3; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + PyObject *__pyx_t_6 = NULL; + int __pyx_t_7; + void *__pyx_t_8; + PyObject *__pyx_t_9 = NULL; + PyObject *__pyx_t_10 = NULL; + PyObject *__pyx_t_11 = NULL; + PyObject *__pyx_t_12 = NULL; + PyObject *__pyx_t_13 = NULL; + PyObject *__pyx_t_14 = NULL; + PyObject *__pyx_t_15 = NULL; + PyObject *__pyx_t_16[3]; + PyObject *__pyx_t_17 = NULL; + size_t __pyx_t_18; + int __pyx_t_19; + char const *__pyx_t_20; + PyObject *__pyx_t_21 = NULL; + PyObject *__pyx_t_22 = NULL; + PyObject *__pyx_t_23 = NULL; + PyObject *__pyx_t_24 = NULL; + PyObject *__pyx_t_25 = NULL; + PyObject *__pyx_t_26 = NULL; + char const *__pyx_t_27; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("get_buffer_pointer", 0); + if (__pyx_optional_args) { + if (__pyx_optional_args->__pyx_n > 0) { + __pyx_v_readonly = __pyx_optional_args->readonly; + } + } + + /* "cuda/bindings/_internal/utils.pyx":25 + * """The caller must ensure ``buf`` is alive when the returned pointer is in use.""" + * cdef void* bufPtr + * cdef int flags = cpython.PyBUF_ANY_CONTIGUOUS # <<<<<<<<<<<<<< + * if not readonly: + * flags |= cpython.PyBUF_WRITABLE +*/ + __pyx_v_flags = PyBUF_ANY_CONTIGUOUS; + + /* "cuda/bindings/_internal/utils.pyx":26 + * cdef void* bufPtr + * cdef int flags = cpython.PyBUF_ANY_CONTIGUOUS + * if not readonly: # <<<<<<<<<<<<<< + * flags |= cpython.PyBUF_WRITABLE + * cdef int status = -1 +*/ + __pyx_t_1 = __Pyx_PyObject_IsTrue(__pyx_v_readonly); if (unlikely((__pyx_t_1 < 0))) __PYX_ERR(0, 26, __pyx_L1_error) + __pyx_t_2 = (!__pyx_t_1); + if (__pyx_t_2) { + + /* "cuda/bindings/_internal/utils.pyx":27 + * cdef int flags = cpython.PyBUF_ANY_CONTIGUOUS + * if not readonly: + * flags |= cpython.PyBUF_WRITABLE # <<<<<<<<<<<<<< + * cdef int status = -1 + * cdef cpython.Py_buffer view +*/ + __pyx_v_flags = (__pyx_v_flags | PyBUF_WRITABLE); + + /* "cuda/bindings/_internal/utils.pyx":26 + * cdef void* bufPtr + * cdef int flags = cpython.PyBUF_ANY_CONTIGUOUS + * if not readonly: # <<<<<<<<<<<<<< + * flags |= cpython.PyBUF_WRITABLE + * cdef int status = -1 +*/ + } + + /* "cuda/bindings/_internal/utils.pyx":28 + * if not readonly: + * flags |= cpython.PyBUF_WRITABLE + * cdef int status = -1 # <<<<<<<<<<<<<< + * cdef cpython.Py_buffer view + * +*/ + __pyx_v_status = -1; + + /* "cuda/bindings/_internal/utils.pyx":31 + * cdef cpython.Py_buffer view + * + * if isinstance(buf, int): # <<<<<<<<<<<<<< + * bufPtr = buf + * else: # try buffer protocol +*/ + __pyx_t_2 = PyLong_Check(__pyx_v_buf); + if (__pyx_t_2) { + + /* "cuda/bindings/_internal/utils.pyx":32 + * + * if isinstance(buf, int): + * bufPtr = buf # <<<<<<<<<<<<<< + * else: # try buffer protocol + * try: +*/ + __pyx_t_3 = PyLong_AsSsize_t(__pyx_v_buf); if (unlikely((__pyx_t_3 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 32, __pyx_L1_error) + __pyx_v_bufPtr = ((void *)((intptr_t)__pyx_t_3)); + + /* "cuda/bindings/_internal/utils.pyx":31 + * cdef cpython.Py_buffer view + * + * if isinstance(buf, int): # <<<<<<<<<<<<<< + * bufPtr = buf + * else: # try buffer protocol +*/ + goto __pyx_L4; + } + + /* "cuda/bindings/_internal/utils.pyx":34 + * bufPtr = buf + * else: # try buffer protocol + * try: # <<<<<<<<<<<<<< + * status = cpython.PyObject_GetBuffer(buf, &view, flags) + * # when the caller does not provide a size, it is set to -1 at generate-time by cybind +*/ + /*else*/ { + /*try:*/ { + { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __Pyx_ExceptionSave(&__pyx_t_4, &__pyx_t_5, &__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_6); + /*try:*/ { + + /* "cuda/bindings/_internal/utils.pyx":35 + * else: # try buffer protocol + * try: + * status = cpython.PyObject_GetBuffer(buf, &view, flags) # <<<<<<<<<<<<<< + * # when the caller does not provide a size, it is set to -1 at generate-time by cybind + * if size != -1: +*/ + __pyx_t_7 = PyObject_GetBuffer(__pyx_v_buf, (&__pyx_v_view), __pyx_v_flags); if (unlikely(__pyx_t_7 == ((int)-1))) __PYX_ERR(0, 35, __pyx_L8_error) + __pyx_v_status = __pyx_t_7; + + /* "cuda/bindings/_internal/utils.pyx":37 + * status = cpython.PyObject_GetBuffer(buf, &view, flags) + * # when the caller does not provide a size, it is set to -1 at generate-time by cybind + * if size != -1: # <<<<<<<<<<<<<< + * assert view.len == size + * assert view.ndim == 1 +*/ + __pyx_t_2 = (__pyx_v_size != -1L); + if (__pyx_t_2) { + + /* "cuda/bindings/_internal/utils.pyx":38 + * # when the caller does not provide a size, it is set to -1 at generate-time by cybind + * if size != -1: + * assert view.len == size # <<<<<<<<<<<<<< + * assert view.ndim == 1 + * except Exception as e: +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_2 = (__pyx_v_view.len == __pyx_v_size); + if (unlikely(!__pyx_t_2)) { + __Pyx_Raise(__pyx_builtin_AssertionError, 0, 0, 0); + __PYX_ERR(0, 38, __pyx_L8_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 38, __pyx_L8_error) + #endif + + /* "cuda/bindings/_internal/utils.pyx":37 + * status = cpython.PyObject_GetBuffer(buf, &view, flags) + * # when the caller does not provide a size, it is set to -1 at generate-time by cybind + * if size != -1: # <<<<<<<<<<<<<< + * assert view.len == size + * assert view.ndim == 1 +*/ + } + + /* "cuda/bindings/_internal/utils.pyx":39 + * if size != -1: + * assert view.len == size + * assert view.ndim == 1 # <<<<<<<<<<<<<< + * except Exception as e: + * adj = "writable " if not readonly else "" +*/ + #ifndef CYTHON_WITHOUT_ASSERTIONS + if (unlikely(__pyx_assertions_enabled())) { + __pyx_t_2 = (__pyx_v_view.ndim == 1); + if (unlikely(!__pyx_t_2)) { + __Pyx_Raise(__pyx_builtin_AssertionError, 0, 0, 0); + __PYX_ERR(0, 39, __pyx_L8_error) + } + } + #else + if ((1)); else __PYX_ERR(0, 39, __pyx_L8_error) + #endif + + /* "cuda/bindings/_internal/utils.pyx":34 + * bufPtr = buf + * else: # try buffer protocol + * try: # <<<<<<<<<<<<<< + * status = cpython.PyObject_GetBuffer(buf, &view, flags) + * # when the caller does not provide a size, it is set to -1 at generate-time by cybind +*/ + } + + /* "cuda/bindings/_internal/utils.pyx":47 + * "buffer, of size bytes") from e + * else: + * bufPtr = view.buf # <<<<<<<<<<<<<< + * finally: + * if status == 0: +*/ + /*else:*/ { + __pyx_t_8 = __pyx_v_view.buf; + __pyx_v_bufPtr = __pyx_t_8; + } + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + goto __pyx_L13_try_end; + __pyx_L8_error:; + + /* "cuda/bindings/_internal/utils.pyx":40 + * assert view.len == size + * assert view.ndim == 1 + * except Exception as e: # <<<<<<<<<<<<<< + * adj = "writable " if not readonly else "" + * raise ValueError( +*/ + __pyx_t_7 = __Pyx_PyErr_ExceptionMatches(((PyObject *)(((PyTypeObject*)PyExc_Exception)))); + if (__pyx_t_7) { + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_buffer_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + if (__Pyx_GetException(&__pyx_t_9, &__pyx_t_10, &__pyx_t_11) < 0) __PYX_ERR(0, 40, __pyx_L10_except_error) + __Pyx_XGOTREF(__pyx_t_9); + __Pyx_XGOTREF(__pyx_t_10); + __Pyx_XGOTREF(__pyx_t_11); + __Pyx_INCREF(__pyx_t_10); + __pyx_v_e = __pyx_t_10; + /*try:*/ { + + /* "cuda/bindings/_internal/utils.pyx":41 + * assert view.ndim == 1 + * except Exception as e: + * adj = "writable " if not readonly else "" # <<<<<<<<<<<<<< + * raise ValueError( + * "buf must be either a Python int representing the pointer " +*/ + __pyx_t_2 = __Pyx_PyObject_IsTrue(__pyx_v_readonly); if (unlikely((__pyx_t_2 < 0))) __PYX_ERR(0, 41, __pyx_L20_error) + __pyx_t_1 = (!__pyx_t_2); + if (__pyx_t_1) { + __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_writable); + __pyx_t_12 = __pyx_mstate_global->__pyx_kp_u_writable; + } else { + __Pyx_INCREF(__pyx_mstate_global->__pyx_kp_u_); + __pyx_t_12 = __pyx_mstate_global->__pyx_kp_u_; + } + __pyx_v_adj = ((PyObject*)__pyx_t_12); + __pyx_t_12 = 0; + + /* "cuda/bindings/_internal/utils.pyx":42 + * except Exception as e: + * adj = "writable " if not readonly else "" + * raise ValueError( # <<<<<<<<<<<<<< + * "buf must be either a Python int representing the pointer " + * f"address to a valid buffer, or a 1D contiguous {adj}" +*/ + __pyx_t_13 = NULL; + __Pyx_INCREF(__pyx_builtin_ValueError); + __pyx_t_14 = __pyx_builtin_ValueError; + + /* "cuda/bindings/_internal/utils.pyx":44 + * raise ValueError( + * "buf must be either a Python int representing the pointer " + * f"address to a valid buffer, or a 1D contiguous {adj}" # <<<<<<<<<<<<<< + * "buffer, of size bytes") from e + * else: +*/ + __pyx_t_15 = __Pyx_PyUnicode_Unicode(__pyx_v_adj); if (unlikely(!__pyx_t_15)) __PYX_ERR(0, 44, __pyx_L20_error) + __Pyx_GOTREF(__pyx_t_15); + __pyx_t_16[0] = __pyx_mstate_global->__pyx_kp_u_buf_must_be_either_a_Python_int; + __pyx_t_16[1] = __pyx_t_15; + __pyx_t_16[2] = __pyx_mstate_global->__pyx_kp_u_buffer_of_size_bytes; + + /* "cuda/bindings/_internal/utils.pyx":43 + * adj = "writable " if not readonly else "" + * raise ValueError( + * "buf must be either a Python int representing the pointer " # <<<<<<<<<<<<<< + * f"address to a valid buffer, or a 1D contiguous {adj}" + * "buffer, of size bytes") from e +*/ + __pyx_t_17 = __Pyx_PyUnicode_Join(__pyx_t_16, 3, 103 + __Pyx_PyUnicode_GET_LENGTH(__pyx_t_15) + 21, 127 | __Pyx_PyUnicode_MAX_CHAR_VALUE(__pyx_t_15)); + if (unlikely(!__pyx_t_17)) __PYX_ERR(0, 43, __pyx_L20_error) + __Pyx_GOTREF(__pyx_t_17); + __Pyx_DECREF(__pyx_t_15); __pyx_t_15 = 0; + __pyx_t_18 = 1; + { + PyObject *__pyx_callargs[2] = {__pyx_t_13, __pyx_t_17}; + __pyx_t_12 = __Pyx_PyObject_FastCall(__pyx_t_14, __pyx_callargs+__pyx_t_18, (2-__pyx_t_18) | (__pyx_t_18*__Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET)); + __Pyx_XDECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_DECREF(__pyx_t_17); __pyx_t_17 = 0; + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + if (unlikely(!__pyx_t_12)) __PYX_ERR(0, 42, __pyx_L20_error) + __Pyx_GOTREF(__pyx_t_12); + } + + /* "cuda/bindings/_internal/utils.pyx":45 + * "buf must be either a Python int representing the pointer " + * f"address to a valid buffer, or a 1D contiguous {adj}" + * "buffer, of size bytes") from e # <<<<<<<<<<<<<< + * else: + * bufPtr = view.buf +*/ + __Pyx_Raise(__pyx_t_12, 0, 0, __pyx_v_e); + __Pyx_DECREF(__pyx_t_12); __pyx_t_12 = 0; + __PYX_ERR(0, 42, __pyx_L20_error) + } + + /* "cuda/bindings/_internal/utils.pyx":40 + * assert view.len == size + * assert view.ndim == 1 + * except Exception as e: # <<<<<<<<<<<<<< + * adj = "writable " if not readonly else "" + * raise ValueError( +*/ + /*finally:*/ { + __pyx_L20_error:; + /*exception exit:*/{ + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __pyx_t_21 = 0; __pyx_t_22 = 0; __pyx_t_23 = 0; __pyx_t_24 = 0; __pyx_t_25 = 0; __pyx_t_26 = 0; + __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_XDECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_XDECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_XDECREF(__pyx_t_15); __pyx_t_15 = 0; + __Pyx_XDECREF(__pyx_t_17); __pyx_t_17 = 0; + __Pyx_ExceptionSwap(&__pyx_t_24, &__pyx_t_25, &__pyx_t_26); + if ( unlikely(__Pyx_GetException(&__pyx_t_21, &__pyx_t_22, &__pyx_t_23) < 0)) __Pyx_ErrFetch(&__pyx_t_21, &__pyx_t_22, &__pyx_t_23); + __Pyx_XGOTREF(__pyx_t_21); + __Pyx_XGOTREF(__pyx_t_22); + __Pyx_XGOTREF(__pyx_t_23); + __Pyx_XGOTREF(__pyx_t_24); + __Pyx_XGOTREF(__pyx_t_25); + __Pyx_XGOTREF(__pyx_t_26); + __pyx_t_7 = __pyx_lineno; __pyx_t_19 = __pyx_clineno; __pyx_t_20 = __pyx_filename; + { + __Pyx_DECREF(__pyx_v_e); __pyx_v_e = 0; + } + __Pyx_XGIVEREF(__pyx_t_24); + __Pyx_XGIVEREF(__pyx_t_25); + __Pyx_XGIVEREF(__pyx_t_26); + __Pyx_ExceptionReset(__pyx_t_24, __pyx_t_25, __pyx_t_26); + __Pyx_XGIVEREF(__pyx_t_21); + __Pyx_XGIVEREF(__pyx_t_22); + __Pyx_XGIVEREF(__pyx_t_23); + __Pyx_ErrRestore(__pyx_t_21, __pyx_t_22, __pyx_t_23); + __pyx_t_21 = 0; __pyx_t_22 = 0; __pyx_t_23 = 0; __pyx_t_24 = 0; __pyx_t_25 = 0; __pyx_t_26 = 0; + __pyx_lineno = __pyx_t_7; __pyx_clineno = __pyx_t_19; __pyx_filename = __pyx_t_20; + goto __pyx_L10_except_error; + } + } + } + goto __pyx_L10_except_error; + + /* "cuda/bindings/_internal/utils.pyx":34 + * bufPtr = buf + * else: # try buffer protocol + * try: # <<<<<<<<<<<<<< + * status = cpython.PyObject_GetBuffer(buf, &view, flags) + * # when the caller does not provide a size, it is set to -1 at generate-time by cybind +*/ + __pyx_L10_except_error:; + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_XGIVEREF(__pyx_t_5); + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_ExceptionReset(__pyx_t_4, __pyx_t_5, __pyx_t_6); + goto __pyx_L6_error; + __pyx_L13_try_end:; + } + } + + /* "cuda/bindings/_internal/utils.pyx":49 + * bufPtr = view.buf + * finally: + * if status == 0: # <<<<<<<<<<<<<< + * cpython.PyBuffer_Release(&view) + * +*/ + /*finally:*/ { + /*normal exit:*/{ + __pyx_t_1 = (__pyx_v_status == 0); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":50 + * finally: + * if status == 0: + * cpython.PyBuffer_Release(&view) # <<<<<<<<<<<<<< + * + * return bufPtr +*/ + PyBuffer_Release((&__pyx_v_view)); + + /* "cuda/bindings/_internal/utils.pyx":49 + * bufPtr = view.buf + * finally: + * if status == 0: # <<<<<<<<<<<<<< + * cpython.PyBuffer_Release(&view) + * +*/ + } + goto __pyx_L7; + } + __pyx_L6_error:; + /*exception exit:*/{ + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + __pyx_t_6 = 0; __pyx_t_5 = 0; __pyx_t_4 = 0; __pyx_t_26 = 0; __pyx_t_25 = 0; __pyx_t_24 = 0; + __Pyx_XDECREF(__pyx_t_10); __pyx_t_10 = 0; + __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_XDECREF(__pyx_t_12); __pyx_t_12 = 0; + __Pyx_XDECREF(__pyx_t_13); __pyx_t_13 = 0; + __Pyx_XDECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_XDECREF(__pyx_t_15); __pyx_t_15 = 0; + __Pyx_XDECREF(__pyx_t_17); __pyx_t_17 = 0; + __Pyx_XDECREF(__pyx_t_9); __pyx_t_9 = 0; + __Pyx_ExceptionSwap(&__pyx_t_26, &__pyx_t_25, &__pyx_t_24); + if ( unlikely(__Pyx_GetException(&__pyx_t_6, &__pyx_t_5, &__pyx_t_4) < 0)) __Pyx_ErrFetch(&__pyx_t_6, &__pyx_t_5, &__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_6); + __Pyx_XGOTREF(__pyx_t_5); + __Pyx_XGOTREF(__pyx_t_4); + __Pyx_XGOTREF(__pyx_t_26); + __Pyx_XGOTREF(__pyx_t_25); + __Pyx_XGOTREF(__pyx_t_24); + __pyx_t_19 = __pyx_lineno; __pyx_t_7 = __pyx_clineno; __pyx_t_27 = __pyx_filename; + { + __pyx_t_1 = (__pyx_v_status == 0); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":50 + * finally: + * if status == 0: + * cpython.PyBuffer_Release(&view) # <<<<<<<<<<<<<< + * + * return bufPtr +*/ + PyBuffer_Release((&__pyx_v_view)); + + /* "cuda/bindings/_internal/utils.pyx":49 + * bufPtr = view.buf + * finally: + * if status == 0: # <<<<<<<<<<<<<< + * cpython.PyBuffer_Release(&view) + * +*/ + } + } + __Pyx_XGIVEREF(__pyx_t_26); + __Pyx_XGIVEREF(__pyx_t_25); + __Pyx_XGIVEREF(__pyx_t_24); + __Pyx_ExceptionReset(__pyx_t_26, __pyx_t_25, __pyx_t_24); + __Pyx_XGIVEREF(__pyx_t_6); + __Pyx_XGIVEREF(__pyx_t_5); + __Pyx_XGIVEREF(__pyx_t_4); + __Pyx_ErrRestore(__pyx_t_6, __pyx_t_5, __pyx_t_4); + __pyx_t_6 = 0; __pyx_t_5 = 0; __pyx_t_4 = 0; __pyx_t_26 = 0; __pyx_t_25 = 0; __pyx_t_24 = 0; + __pyx_lineno = __pyx_t_19; __pyx_clineno = __pyx_t_7; __pyx_filename = __pyx_t_27; + goto __pyx_L1_error; + } + __pyx_L7:; + } + } + __pyx_L4:; + + /* "cuda/bindings/_internal/utils.pyx":52 + * cpython.PyBuffer_Release(&view) + * + * return bufPtr # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = __pyx_v_bufPtr; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":22 + * + * + * cdef void* get_buffer_pointer(buf, Py_ssize_t size, readonly=True) except*: # <<<<<<<<<<<<<< + * """The caller must ensure ``buf`` is alive when the returned pointer is in use.""" + * cdef void* bufPtr +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_9); + __Pyx_XDECREF(__pyx_t_10); + __Pyx_XDECREF(__pyx_t_11); + __Pyx_XDECREF(__pyx_t_12); + __Pyx_XDECREF(__pyx_t_13); + __Pyx_XDECREF(__pyx_t_14); + __Pyx_XDECREF(__pyx_t_15); + __Pyx_XDECREF(__pyx_t_17); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_buffer_pointer", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 0; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_e); + __Pyx_XDECREF(__pyx_v_adj); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cuda/bindings/_internal/utils.pyx":57 + * # Cython can't infer the ResT overload when it is wrapped in nullable_unique_ptr, + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * if cpython.PySequence_Check(obj): + * vec = new vector[ResT](len(obj)) +*/ + +static int __pyx_fuse_0__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr(nullable_unique_ptr > &__pyx_v_in_out_ptr, PyObject *__pyx_v_obj, CYTHON_UNUSED int *__pyx_v___unused) { + std::vector *__pyx_v_vec; + Py_ssize_t __pyx_v_i; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + std::vector *__pyx_t_3; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + int __pyx_t_7; + intptr_t __pyx_t_8; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_0get_resource_ptr", 0); + + /* "cuda/bindings/_internal/utils.pyx":58 + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[ResT](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + __pyx_t_1 = PySequence_Check(__pyx_v_obj); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":59 + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): + * vec = new vector[ResT](len(obj)) # <<<<<<<<<<<<<< + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 59, __pyx_L1_error) + try { + __pyx_t_3 = new std::vector (__pyx_t_2); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 59, __pyx_L1_error) + } + __pyx_v_vec = __pyx_t_3; + + /* "cuda/bindings/_internal/utils.pyx":62 + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) # <<<<<<<<<<<<<< + * for i in range(len(obj)): + * deref(vec)[i] = obj[i] +*/ + __pyx_v_in_out_ptr.reset(__pyx_v_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":63 + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): # <<<<<<<<<<<<<< + * deref(vec)[i] = obj[i] + * else: +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 63, __pyx_L1_error) + __pyx_t_4 = __pyx_t_2; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "cuda/bindings/_internal/utils.pyx":64 + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): + * deref(vec)[i] = obj[i] # <<<<<<<<<<<<<< + * else: + * in_out_ptr.reset(obj, False) +*/ + __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_obj, __pyx_v_i, Py_ssize_t, 1, PyLong_FromSsize_t, 0, 1, 1, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 64, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __Pyx_PyLong_As_int(__pyx_t_6); if (unlikely((__pyx_t_7 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 64, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + ((*__pyx_v_vec)[__pyx_v_i]) = __pyx_t_7; + } + + /* "cuda/bindings/_internal/utils.pyx":58 + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[ResT](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":66 + * deref(vec)[i] = obj[i] + * else: + * in_out_ptr.reset(obj, False) # <<<<<<<<<<<<<< + * return 0 + * +*/ + /*else*/ { + __pyx_t_8 = PyLong_AsSsize_t(__pyx_v_obj); if (unlikely((__pyx_t_8 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 66, __pyx_L1_error) + __pyx_v_in_out_ptr.reset(((std::vector *)((intptr_t)__pyx_t_8)), 0); + } + __pyx_L3:; + + /* "cuda/bindings/_internal/utils.pyx":67 + * else: + * in_out_ptr.reset(obj, False) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":57 + * # Cython can't infer the ResT overload when it is wrapped in nullable_unique_ptr, + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * if cpython.PySequence_Check(obj): + * vec = new vector[ResT](len(obj)) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_resource_ptr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_fuse_1__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr(nullable_unique_ptr > &__pyx_v_in_out_ptr, PyObject *__pyx_v_obj, CYTHON_UNUSED int32_t *__pyx_v___unused) { + std::vector *__pyx_v_vec; + Py_ssize_t __pyx_v_i; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + std::vector *__pyx_t_3; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + int32_t __pyx_t_7; + intptr_t __pyx_t_8; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_1get_resource_ptr", 0); + + /* "cuda/bindings/_internal/utils.pyx":58 + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[ResT](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + __pyx_t_1 = PySequence_Check(__pyx_v_obj); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":59 + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): + * vec = new vector[ResT](len(obj)) # <<<<<<<<<<<<<< + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 59, __pyx_L1_error) + try { + __pyx_t_3 = new std::vector (__pyx_t_2); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 59, __pyx_L1_error) + } + __pyx_v_vec = __pyx_t_3; + + /* "cuda/bindings/_internal/utils.pyx":62 + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) # <<<<<<<<<<<<<< + * for i in range(len(obj)): + * deref(vec)[i] = obj[i] +*/ + __pyx_v_in_out_ptr.reset(__pyx_v_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":63 + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): # <<<<<<<<<<<<<< + * deref(vec)[i] = obj[i] + * else: +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 63, __pyx_L1_error) + __pyx_t_4 = __pyx_t_2; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "cuda/bindings/_internal/utils.pyx":64 + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): + * deref(vec)[i] = obj[i] # <<<<<<<<<<<<<< + * else: + * in_out_ptr.reset(obj, False) +*/ + __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_obj, __pyx_v_i, Py_ssize_t, 1, PyLong_FromSsize_t, 0, 1, 1, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 64, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __Pyx_PyLong_As_int32_t(__pyx_t_6); if (unlikely((__pyx_t_7 == ((int32_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 64, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + ((*__pyx_v_vec)[__pyx_v_i]) = __pyx_t_7; + } + + /* "cuda/bindings/_internal/utils.pyx":58 + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[ResT](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":66 + * deref(vec)[i] = obj[i] + * else: + * in_out_ptr.reset(obj, False) # <<<<<<<<<<<<<< + * return 0 + * +*/ + /*else*/ { + __pyx_t_8 = PyLong_AsSsize_t(__pyx_v_obj); if (unlikely((__pyx_t_8 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 66, __pyx_L1_error) + __pyx_v_in_out_ptr.reset(((std::vector *)((intptr_t)__pyx_t_8)), 0); + } + __pyx_L3:; + + /* "cuda/bindings/_internal/utils.pyx":67 + * else: + * in_out_ptr.reset(obj, False) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":57 + * # Cython can't infer the ResT overload when it is wrapped in nullable_unique_ptr, + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * if cpython.PySequence_Check(obj): + * vec = new vector[ResT](len(obj)) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_resource_ptr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_fuse_2__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr(nullable_unique_ptr > &__pyx_v_in_out_ptr, PyObject *__pyx_v_obj, CYTHON_UNUSED int64_t *__pyx_v___unused) { + std::vector *__pyx_v_vec; + Py_ssize_t __pyx_v_i; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + std::vector *__pyx_t_3; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + int64_t __pyx_t_7; + intptr_t __pyx_t_8; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_2get_resource_ptr", 0); + + /* "cuda/bindings/_internal/utils.pyx":58 + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[ResT](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + __pyx_t_1 = PySequence_Check(__pyx_v_obj); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":59 + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): + * vec = new vector[ResT](len(obj)) # <<<<<<<<<<<<<< + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 59, __pyx_L1_error) + try { + __pyx_t_3 = new std::vector (__pyx_t_2); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 59, __pyx_L1_error) + } + __pyx_v_vec = __pyx_t_3; + + /* "cuda/bindings/_internal/utils.pyx":62 + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) # <<<<<<<<<<<<<< + * for i in range(len(obj)): + * deref(vec)[i] = obj[i] +*/ + __pyx_v_in_out_ptr.reset(__pyx_v_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":63 + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): # <<<<<<<<<<<<<< + * deref(vec)[i] = obj[i] + * else: +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 63, __pyx_L1_error) + __pyx_t_4 = __pyx_t_2; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "cuda/bindings/_internal/utils.pyx":64 + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): + * deref(vec)[i] = obj[i] # <<<<<<<<<<<<<< + * else: + * in_out_ptr.reset(obj, False) +*/ + __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_obj, __pyx_v_i, Py_ssize_t, 1, PyLong_FromSsize_t, 0, 1, 1, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 64, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __Pyx_PyLong_As_int64_t(__pyx_t_6); if (unlikely((__pyx_t_7 == ((int64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 64, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + ((*__pyx_v_vec)[__pyx_v_i]) = __pyx_t_7; + } + + /* "cuda/bindings/_internal/utils.pyx":58 + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[ResT](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":66 + * deref(vec)[i] = obj[i] + * else: + * in_out_ptr.reset(obj, False) # <<<<<<<<<<<<<< + * return 0 + * +*/ + /*else*/ { + __pyx_t_8 = PyLong_AsSsize_t(__pyx_v_obj); if (unlikely((__pyx_t_8 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 66, __pyx_L1_error) + __pyx_v_in_out_ptr.reset(((std::vector *)((intptr_t)__pyx_t_8)), 0); + } + __pyx_L3:; + + /* "cuda/bindings/_internal/utils.pyx":67 + * else: + * in_out_ptr.reset(obj, False) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":57 + * # Cython can't infer the ResT overload when it is wrapped in nullable_unique_ptr, + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * if cpython.PySequence_Check(obj): + * vec = new vector[ResT](len(obj)) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_resource_ptr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_fuse_3__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr(nullable_unique_ptr > &__pyx_v_in_out_ptr, PyObject *__pyx_v_obj, CYTHON_UNUSED char *__pyx_v___unused) { + std::vector *__pyx_v_vec; + Py_ssize_t __pyx_v_i; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + std::vector *__pyx_t_3; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + char __pyx_t_7; + intptr_t __pyx_t_8; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_3get_resource_ptr", 0); + + /* "cuda/bindings/_internal/utils.pyx":58 + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[ResT](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + __pyx_t_1 = PySequence_Check(__pyx_v_obj); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":59 + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): + * vec = new vector[ResT](len(obj)) # <<<<<<<<<<<<<< + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 59, __pyx_L1_error) + try { + __pyx_t_3 = new std::vector (__pyx_t_2); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 59, __pyx_L1_error) + } + __pyx_v_vec = __pyx_t_3; + + /* "cuda/bindings/_internal/utils.pyx":62 + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) # <<<<<<<<<<<<<< + * for i in range(len(obj)): + * deref(vec)[i] = obj[i] +*/ + __pyx_v_in_out_ptr.reset(__pyx_v_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":63 + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): # <<<<<<<<<<<<<< + * deref(vec)[i] = obj[i] + * else: +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 63, __pyx_L1_error) + __pyx_t_4 = __pyx_t_2; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "cuda/bindings/_internal/utils.pyx":64 + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): + * deref(vec)[i] = obj[i] # <<<<<<<<<<<<<< + * else: + * in_out_ptr.reset(obj, False) +*/ + __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_obj, __pyx_v_i, Py_ssize_t, 1, PyLong_FromSsize_t, 0, 1, 1, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 64, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __Pyx_PyLong_As_char(__pyx_t_6); if (unlikely((__pyx_t_7 == (char)-1) && PyErr_Occurred())) __PYX_ERR(0, 64, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + ((*__pyx_v_vec)[__pyx_v_i]) = __pyx_t_7; + } + + /* "cuda/bindings/_internal/utils.pyx":58 + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[ResT](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":66 + * deref(vec)[i] = obj[i] + * else: + * in_out_ptr.reset(obj, False) # <<<<<<<<<<<<<< + * return 0 + * +*/ + /*else*/ { + __pyx_t_8 = PyLong_AsSsize_t(__pyx_v_obj); if (unlikely((__pyx_t_8 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 66, __pyx_L1_error) + __pyx_v_in_out_ptr.reset(((std::vector *)((intptr_t)__pyx_t_8)), 0); + } + __pyx_L3:; + + /* "cuda/bindings/_internal/utils.pyx":67 + * else: + * in_out_ptr.reset(obj, False) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":57 + * # Cython can't infer the ResT overload when it is wrapped in nullable_unique_ptr, + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * if cpython.PySequence_Check(obj): + * vec = new vector[ResT](len(obj)) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_resource_ptr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_fuse_4__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr(nullable_unique_ptr > &__pyx_v_in_out_ptr, PyObject *__pyx_v_obj, CYTHON_UNUSED float *__pyx_v___unused) { + std::vector *__pyx_v_vec; + Py_ssize_t __pyx_v_i; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + std::vector *__pyx_t_3; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + float __pyx_t_7; + intptr_t __pyx_t_8; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_4get_resource_ptr", 0); + + /* "cuda/bindings/_internal/utils.pyx":58 + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[ResT](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + __pyx_t_1 = PySequence_Check(__pyx_v_obj); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":59 + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): + * vec = new vector[ResT](len(obj)) # <<<<<<<<<<<<<< + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 59, __pyx_L1_error) + try { + __pyx_t_3 = new std::vector (__pyx_t_2); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 59, __pyx_L1_error) + } + __pyx_v_vec = __pyx_t_3; + + /* "cuda/bindings/_internal/utils.pyx":62 + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) # <<<<<<<<<<<<<< + * for i in range(len(obj)): + * deref(vec)[i] = obj[i] +*/ + __pyx_v_in_out_ptr.reset(__pyx_v_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":63 + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): # <<<<<<<<<<<<<< + * deref(vec)[i] = obj[i] + * else: +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 63, __pyx_L1_error) + __pyx_t_4 = __pyx_t_2; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "cuda/bindings/_internal/utils.pyx":64 + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): + * deref(vec)[i] = obj[i] # <<<<<<<<<<<<<< + * else: + * in_out_ptr.reset(obj, False) +*/ + __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_obj, __pyx_v_i, Py_ssize_t, 1, PyLong_FromSsize_t, 0, 1, 1, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 64, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __Pyx_PyFloat_AsFloat(__pyx_t_6); if (unlikely((__pyx_t_7 == (float)-1) && PyErr_Occurred())) __PYX_ERR(0, 64, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + ((*__pyx_v_vec)[__pyx_v_i]) = __pyx_t_7; + } + + /* "cuda/bindings/_internal/utils.pyx":58 + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[ResT](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":66 + * deref(vec)[i] = obj[i] + * else: + * in_out_ptr.reset(obj, False) # <<<<<<<<<<<<<< + * return 0 + * +*/ + /*else*/ { + __pyx_t_8 = PyLong_AsSsize_t(__pyx_v_obj); if (unlikely((__pyx_t_8 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 66, __pyx_L1_error) + __pyx_v_in_out_ptr.reset(((std::vector *)((intptr_t)__pyx_t_8)), 0); + } + __pyx_L3:; + + /* "cuda/bindings/_internal/utils.pyx":67 + * else: + * in_out_ptr.reset(obj, False) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":57 + * # Cython can't infer the ResT overload when it is wrapped in nullable_unique_ptr, + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * if cpython.PySequence_Check(obj): + * vec = new vector[ResT](len(obj)) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_resource_ptr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_fuse_5__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr(nullable_unique_ptr > &__pyx_v_in_out_ptr, PyObject *__pyx_v_obj, CYTHON_UNUSED double *__pyx_v___unused) { + std::vector *__pyx_v_vec; + Py_ssize_t __pyx_v_i; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + std::vector *__pyx_t_3; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + double __pyx_t_7; + intptr_t __pyx_t_8; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_5get_resource_ptr", 0); + + /* "cuda/bindings/_internal/utils.pyx":58 + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[ResT](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + __pyx_t_1 = PySequence_Check(__pyx_v_obj); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":59 + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): + * vec = new vector[ResT](len(obj)) # <<<<<<<<<<<<<< + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 59, __pyx_L1_error) + try { + __pyx_t_3 = new std::vector (__pyx_t_2); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 59, __pyx_L1_error) + } + __pyx_v_vec = __pyx_t_3; + + /* "cuda/bindings/_internal/utils.pyx":62 + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) # <<<<<<<<<<<<<< + * for i in range(len(obj)): + * deref(vec)[i] = obj[i] +*/ + __pyx_v_in_out_ptr.reset(__pyx_v_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":63 + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): # <<<<<<<<<<<<<< + * deref(vec)[i] = obj[i] + * else: +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 63, __pyx_L1_error) + __pyx_t_4 = __pyx_t_2; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "cuda/bindings/_internal/utils.pyx":64 + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): + * deref(vec)[i] = obj[i] # <<<<<<<<<<<<<< + * else: + * in_out_ptr.reset(obj, False) +*/ + __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_obj, __pyx_v_i, Py_ssize_t, 1, PyLong_FromSsize_t, 0, 1, 1, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 64, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = __Pyx_PyFloat_AsDouble(__pyx_t_6); if (unlikely((__pyx_t_7 == (double)-1) && PyErr_Occurred())) __PYX_ERR(0, 64, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + ((*__pyx_v_vec)[__pyx_v_i]) = __pyx_t_7; + } + + /* "cuda/bindings/_internal/utils.pyx":58 + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[ResT](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":66 + * deref(vec)[i] = obj[i] + * else: + * in_out_ptr.reset(obj, False) # <<<<<<<<<<<<<< + * return 0 + * +*/ + /*else*/ { + __pyx_t_8 = PyLong_AsSsize_t(__pyx_v_obj); if (unlikely((__pyx_t_8 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 66, __pyx_L1_error) + __pyx_v_in_out_ptr.reset(((std::vector *)((intptr_t)__pyx_t_8)), 0); + } + __pyx_L3:; + + /* "cuda/bindings/_internal/utils.pyx":67 + * else: + * in_out_ptr.reset(obj, False) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":57 + * # Cython can't infer the ResT overload when it is wrapped in nullable_unique_ptr, + * # so we need a dummy (__unused) input argument to help it + * cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * if cpython.PySequence_Check(obj): + * vec = new vector[ResT](len(obj)) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_resource_ptr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cuda/bindings/_internal/utils.pyx":70 + * + * + * cdef int get_resource_ptrs(nullable_unique_ptr[ vector[PtrT*] ] &in_out_ptr, object obj, PtrT* __unused) except 1: # <<<<<<<<<<<<<< + * if cpython.PySequence_Check(obj): + * vec = new vector[PtrT*](len(obj)) +*/ + +static int __pyx_fuse_0__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptrs(nullable_unique_ptr > &__pyx_v_in_out_ptr, PyObject *__pyx_v_obj, CYTHON_UNUSED void *__pyx_v___unused) { + std::vector *__pyx_v_vec; + Py_ssize_t __pyx_v_i; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + std::vector *__pyx_t_3; + Py_ssize_t __pyx_t_4; + Py_ssize_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + intptr_t __pyx_t_7; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_0get_resource_ptrs", 0); + + /* "cuda/bindings/_internal/utils.pyx":71 + * + * cdef int get_resource_ptrs(nullable_unique_ptr[ vector[PtrT*] ] &in_out_ptr, object obj, PtrT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[PtrT*](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + __pyx_t_1 = PySequence_Check(__pyx_v_obj); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":72 + * cdef int get_resource_ptrs(nullable_unique_ptr[ vector[PtrT*] ] &in_out_ptr, object obj, PtrT* __unused) except 1: + * if cpython.PySequence_Check(obj): + * vec = new vector[PtrT*](len(obj)) # <<<<<<<<<<<<<< + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 72, __pyx_L1_error) + try { + __pyx_t_3 = new std::vector (__pyx_t_2); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 72, __pyx_L1_error) + } + __pyx_v_vec = __pyx_t_3; + + /* "cuda/bindings/_internal/utils.pyx":75 + * # set the ownership immediately to avoid leaking the `vec` memory in + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) # <<<<<<<<<<<<<< + * for i in range(len(obj)): + * deref(vec)[i] = (obj[i]) +*/ + __pyx_v_in_out_ptr.reset(__pyx_v_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":76 + * # case of exception in the following loop + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): # <<<<<<<<<<<<<< + * deref(vec)[i] = (obj[i]) + * else: +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 76, __pyx_L1_error) + __pyx_t_4 = __pyx_t_2; + for (__pyx_t_5 = 0; __pyx_t_5 < __pyx_t_4; __pyx_t_5+=1) { + __pyx_v_i = __pyx_t_5; + + /* "cuda/bindings/_internal/utils.pyx":77 + * in_out_ptr.reset(vec, True) + * for i in range(len(obj)): + * deref(vec)[i] = (obj[i]) # <<<<<<<<<<<<<< + * else: + * in_out_ptr.reset(obj, False) +*/ + __pyx_t_6 = __Pyx_GetItemInt(__pyx_v_obj, __pyx_v_i, Py_ssize_t, 1, PyLong_FromSsize_t, 0, 1, 1, 1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 77, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = PyLong_AsSsize_t(__pyx_t_6); if (unlikely((__pyx_t_7 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 77, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + ((*__pyx_v_vec)[__pyx_v_i]) = ((void *)((intptr_t)__pyx_t_7)); + } + + /* "cuda/bindings/_internal/utils.pyx":71 + * + * cdef int get_resource_ptrs(nullable_unique_ptr[ vector[PtrT*] ] &in_out_ptr, object obj, PtrT* __unused) except 1: + * if cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * vec = new vector[PtrT*](len(obj)) + * # set the ownership immediately to avoid leaking the `vec` memory in +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":79 + * deref(vec)[i] = (obj[i]) + * else: + * in_out_ptr.reset(obj, False) # <<<<<<<<<<<<<< + * return 0 + * +*/ + /*else*/ { + __pyx_t_7 = PyLong_AsSsize_t(__pyx_v_obj); if (unlikely((__pyx_t_7 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 79, __pyx_L1_error) + __pyx_v_in_out_ptr.reset(((std::vector *)((intptr_t)__pyx_t_7)), 0); + } + __pyx_L3:; + + /* "cuda/bindings/_internal/utils.pyx":80 + * else: + * in_out_ptr.reset(obj, False) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":70 + * + * + * cdef int get_resource_ptrs(nullable_unique_ptr[ vector[PtrT*] ] &in_out_ptr, object obj, PtrT* __unused) except 1: # <<<<<<<<<<<<<< + * if cpython.PySequence_Check(obj): + * vec = new vector[PtrT*](len(obj)) +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_resource_ptrs", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 1; + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +/* "cuda/bindings/_internal/utils.pyx":83 + * + * + * cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr +*/ + +static int __pyx_fuse_0__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr(__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &__pyx_v_in_out_ptr, PyObject *__pyx_v_obj, CYTHON_UNUSED int *__pyx_v___unused) { + nullable_unique_ptr > __pyx_v_nested_ptr; + nullable_unique_ptr > > __pyx_v_nested_res_ptr; + std::vector *__pyx_v_nested_vec; + std::vector > *__pyx_v_nested_res_vec; + size_t __pyx_v_i; + size_t __pyx_v_length; + intptr_t __pyx_v_addr; + PyObject *__pyx_v_obj_i = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + std::vector > *__pyx_t_3; + std::vector *__pyx_t_4; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + std::vector __pyx_t_9; + intptr_t __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_0get_nested_resource_ptr", 0); + + /* "cuda/bindings/_internal/utils.pyx":86 + * cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr + * cdef vector[intptr_t]* nested_vec = NULL # <<<<<<<<<<<<<< + * cdef vector[vector[ResT]]* nested_res_vec = NULL + * cdef size_t i = 0, length = 0 +*/ + __pyx_v_nested_vec = NULL; + + /* "cuda/bindings/_internal/utils.pyx":87 + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr + * cdef vector[intptr_t]* nested_vec = NULL + * cdef vector[vector[ResT]]* nested_res_vec = NULL # <<<<<<<<<<<<<< + * cdef size_t i = 0, length = 0 + * cdef intptr_t addr +*/ + __pyx_v_nested_res_vec = NULL; + + /* "cuda/bindings/_internal/utils.pyx":88 + * cdef vector[intptr_t]* nested_vec = NULL + * cdef vector[vector[ResT]]* nested_res_vec = NULL + * cdef size_t i = 0, length = 0 # <<<<<<<<<<<<<< + * cdef intptr_t addr + * +*/ + __pyx_v_i = 0; + __pyx_v_length = 0; + + /* "cuda/bindings/_internal/utils.pyx":91 + * cdef intptr_t addr + * + * if is_nested_sequence(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_5utils_is_nested_sequence(__pyx_v_obj); if (unlikely(__pyx_t_1 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(0, 91, __pyx_L1_error) + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":92 + * + * if is_nested_sequence(obj): + * length = len(obj) # <<<<<<<<<<<<<< + * nested_res_vec = new vector[vector[ResT]](length) + * nested_vec = new vector[intptr_t](length) +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_v_length = __pyx_t_2; + + /* "cuda/bindings/_internal/utils.pyx":93 + * if is_nested_sequence(obj): + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) # <<<<<<<<<<<<<< + * nested_vec = new vector[intptr_t](length) + * # set the ownership immediately to avoid leaking memory in case of +*/ + try { + __pyx_t_3 = new std::vector > (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 93, __pyx_L1_error) + } + __pyx_v_nested_res_vec = __pyx_t_3; + + /* "cuda/bindings/_internal/utils.pyx":94 + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) + * nested_vec = new vector[intptr_t](length) # <<<<<<<<<<<<<< + * # set the ownership immediately to avoid leaking memory in case of + * # exception in the following loop +*/ + try { + __pyx_t_4 = new std::vector (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 94, __pyx_L1_error) + } + __pyx_v_nested_vec = __pyx_t_4; + + /* "cuda/bindings/_internal/utils.pyx":97 + * # set the ownership immediately to avoid leaking memory in case of + * # exception in the following loop + * nested_res_ptr.reset(nested_res_vec, True) # <<<<<<<<<<<<<< + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): +*/ + __pyx_v_nested_res_ptr.reset(__pyx_v_nested_res_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":98 + * # exception in the following loop + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) # <<<<<<<<<<<<<< + * for i, obj_i in enumerate(obj): + * if ResT is char: +*/ + __pyx_v_nested_ptr.reset(__pyx_v_nested_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":99 + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): # <<<<<<<<<<<<<< + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_obj)) || PyTuple_CheckExact(__pyx_v_obj)) { + __pyx_t_6 = __pyx_v_obj; __Pyx_INCREF(__pyx_t_6); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_obj); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 99, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 99, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 99, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_2); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 99, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_2)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 99, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 99, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_obj_i, __pyx_t_8); + __pyx_t_8 = 0; + __pyx_v_i = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "cuda/bindings/_internal/utils.pyx":108 + * deref(nested_res_vec)[i].assign(obj_i_ptr, obj_i_ptr + str_len) + * else: + * deref(nested_res_vec)[i] = obj_i # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): +*/ + __pyx_t_9 = __pyx_convert_vector_from_py_int(__pyx_v_obj_i); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 108, __pyx_L1_error) + ((*__pyx_v_nested_res_vec)[__pyx_v_i]) = __pyx_t_9; + + /* "cuda/bindings/_internal/utils.pyx":109 + * else: + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) # <<<<<<<<<<<<<< + * elif cpython.PySequence_Check(obj): + * length = len(obj) +*/ + ((*__pyx_v_nested_vec)[__pyx_v_i]) = ((intptr_t)((*__pyx_v_nested_res_vec)[__pyx_v_i]).data()); + + /* "cuda/bindings/_internal/utils.pyx":99 + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): # <<<<<<<<<<<<<< + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "cuda/bindings/_internal/utils.pyx":91 + * cdef intptr_t addr + * + * if is_nested_sequence(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":110 + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_vec = new vector[intptr_t](length) +*/ + __pyx_t_1 = PySequence_Check(__pyx_v_obj); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":111 + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): + * length = len(obj) # <<<<<<<<<<<<<< + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 111, __pyx_L1_error) + __pyx_v_length = __pyx_t_2; + + /* "cuda/bindings/_internal/utils.pyx":112 + * elif cpython.PySequence_Check(obj): + * length = len(obj) + * nested_vec = new vector[intptr_t](length) # <<<<<<<<<<<<<< + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): +*/ + try { + __pyx_t_4 = new std::vector (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 112, __pyx_L1_error) + } + __pyx_v_nested_vec = __pyx_t_4; + + /* "cuda/bindings/_internal/utils.pyx":113 + * length = len(obj) + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) # <<<<<<<<<<<<<< + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr +*/ + __pyx_v_nested_ptr.reset(__pyx_v_nested_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":114 + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_obj)) || PyTuple_CheckExact(__pyx_v_obj)) { + __pyx_t_6 = __pyx_v_obj; __Pyx_INCREF(__pyx_t_6); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_obj); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 114, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 114, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_2); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 114, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_2)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 114, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 114, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_10 = PyLong_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_10 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_addr = __pyx_t_10; + __pyx_v_i = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "cuda/bindings/_internal/utils.pyx":115 + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr # <<<<<<<<<<<<<< + * nested_res_ptr.reset(NULL, False) + * else: +*/ + ((*__pyx_v_nested_vec)[__pyx_v_i]) = __pyx_v_addr; + + /* "cuda/bindings/_internal/utils.pyx":114 + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "cuda/bindings/_internal/utils.pyx":116 + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) # <<<<<<<<<<<<<< + * else: + * # obj is an int (ResT**) +*/ + __pyx_v_nested_res_ptr.reset(NULL, 0); + + /* "cuda/bindings/_internal/utils.pyx":110 + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_vec = new vector[intptr_t](length) +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":119 + * else: + * # obj is an int (ResT**) + * nested_res_ptr.reset(NULL, False) # <<<<<<<<<<<<<< + * nested_ptr.reset(obj, False) + * +*/ + /*else*/ { + __pyx_v_nested_res_ptr.reset(NULL, 0); + + /* "cuda/bindings/_internal/utils.pyx":120 + * # obj is an int (ResT**) + * nested_res_ptr.reset(NULL, False) + * nested_ptr.reset(obj, False) # <<<<<<<<<<<<<< + * + * in_out_ptr.ptrs = move(nested_ptr) +*/ + __pyx_t_10 = PyLong_AsSsize_t(__pyx_v_obj); if (unlikely((__pyx_t_10 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 120, __pyx_L1_error) + __pyx_v_nested_ptr.reset(((std::vector *)((intptr_t)__pyx_t_10)), 0); + } + __pyx_L3:; + + /* "cuda/bindings/_internal/utils.pyx":122 + * nested_ptr.reset(obj, False) + * + * in_out_ptr.ptrs = move(nested_ptr) # <<<<<<<<<<<<<< + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) + * return 0 +*/ + __pyx_v_in_out_ptr.ptrs = cython_std::move > >(__pyx_v_nested_ptr); + + /* "cuda/bindings/_internal/utils.pyx":123 + * + * in_out_ptr.ptrs = move(nested_ptr) + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) # <<<<<<<<<<<<<< + * return 0 + * +*/ + __pyx_v_in_out_ptr.nested_resource_ptr = cython_std::move > > >(__pyx_v_nested_res_ptr); + + /* "cuda/bindings/_internal/utils.pyx":124 + * in_out_ptr.ptrs = move(nested_ptr) + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":83 + * + * + * cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_nested_resource_ptr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_obj_i); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_fuse_1__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr(__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &__pyx_v_in_out_ptr, PyObject *__pyx_v_obj, CYTHON_UNUSED int32_t *__pyx_v___unused) { + nullable_unique_ptr > __pyx_v_nested_ptr; + nullable_unique_ptr > > __pyx_v_nested_res_ptr; + std::vector *__pyx_v_nested_vec; + std::vector > *__pyx_v_nested_res_vec; + size_t __pyx_v_i; + size_t __pyx_v_length; + intptr_t __pyx_v_addr; + PyObject *__pyx_v_obj_i = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + std::vector > *__pyx_t_3; + std::vector *__pyx_t_4; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + std::vector __pyx_t_9; + intptr_t __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_1get_nested_resource_ptr", 0); + + /* "cuda/bindings/_internal/utils.pyx":86 + * cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr + * cdef vector[intptr_t]* nested_vec = NULL # <<<<<<<<<<<<<< + * cdef vector[vector[ResT]]* nested_res_vec = NULL + * cdef size_t i = 0, length = 0 +*/ + __pyx_v_nested_vec = NULL; + + /* "cuda/bindings/_internal/utils.pyx":87 + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr + * cdef vector[intptr_t]* nested_vec = NULL + * cdef vector[vector[ResT]]* nested_res_vec = NULL # <<<<<<<<<<<<<< + * cdef size_t i = 0, length = 0 + * cdef intptr_t addr +*/ + __pyx_v_nested_res_vec = NULL; + + /* "cuda/bindings/_internal/utils.pyx":88 + * cdef vector[intptr_t]* nested_vec = NULL + * cdef vector[vector[ResT]]* nested_res_vec = NULL + * cdef size_t i = 0, length = 0 # <<<<<<<<<<<<<< + * cdef intptr_t addr + * +*/ + __pyx_v_i = 0; + __pyx_v_length = 0; + + /* "cuda/bindings/_internal/utils.pyx":91 + * cdef intptr_t addr + * + * if is_nested_sequence(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_5utils_is_nested_sequence(__pyx_v_obj); if (unlikely(__pyx_t_1 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(0, 91, __pyx_L1_error) + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":92 + * + * if is_nested_sequence(obj): + * length = len(obj) # <<<<<<<<<<<<<< + * nested_res_vec = new vector[vector[ResT]](length) + * nested_vec = new vector[intptr_t](length) +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_v_length = __pyx_t_2; + + /* "cuda/bindings/_internal/utils.pyx":93 + * if is_nested_sequence(obj): + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) # <<<<<<<<<<<<<< + * nested_vec = new vector[intptr_t](length) + * # set the ownership immediately to avoid leaking memory in case of +*/ + try { + __pyx_t_3 = new std::vector > (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 93, __pyx_L1_error) + } + __pyx_v_nested_res_vec = __pyx_t_3; + + /* "cuda/bindings/_internal/utils.pyx":94 + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) + * nested_vec = new vector[intptr_t](length) # <<<<<<<<<<<<<< + * # set the ownership immediately to avoid leaking memory in case of + * # exception in the following loop +*/ + try { + __pyx_t_4 = new std::vector (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 94, __pyx_L1_error) + } + __pyx_v_nested_vec = __pyx_t_4; + + /* "cuda/bindings/_internal/utils.pyx":97 + * # set the ownership immediately to avoid leaking memory in case of + * # exception in the following loop + * nested_res_ptr.reset(nested_res_vec, True) # <<<<<<<<<<<<<< + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): +*/ + __pyx_v_nested_res_ptr.reset(__pyx_v_nested_res_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":98 + * # exception in the following loop + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) # <<<<<<<<<<<<<< + * for i, obj_i in enumerate(obj): + * if ResT is char: +*/ + __pyx_v_nested_ptr.reset(__pyx_v_nested_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":99 + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): # <<<<<<<<<<<<<< + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_obj)) || PyTuple_CheckExact(__pyx_v_obj)) { + __pyx_t_6 = __pyx_v_obj; __Pyx_INCREF(__pyx_t_6); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_obj); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 99, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 99, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 99, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_2); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 99, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_2)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 99, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 99, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_obj_i, __pyx_t_8); + __pyx_t_8 = 0; + __pyx_v_i = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "cuda/bindings/_internal/utils.pyx":108 + * deref(nested_res_vec)[i].assign(obj_i_ptr, obj_i_ptr + str_len) + * else: + * deref(nested_res_vec)[i] = obj_i # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): +*/ + __pyx_t_9 = __pyx_convert_vector_from_py_int32_t(__pyx_v_obj_i); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 108, __pyx_L1_error) + ((*__pyx_v_nested_res_vec)[__pyx_v_i]) = __pyx_t_9; + + /* "cuda/bindings/_internal/utils.pyx":109 + * else: + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) # <<<<<<<<<<<<<< + * elif cpython.PySequence_Check(obj): + * length = len(obj) +*/ + ((*__pyx_v_nested_vec)[__pyx_v_i]) = ((intptr_t)((*__pyx_v_nested_res_vec)[__pyx_v_i]).data()); + + /* "cuda/bindings/_internal/utils.pyx":99 + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): # <<<<<<<<<<<<<< + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "cuda/bindings/_internal/utils.pyx":91 + * cdef intptr_t addr + * + * if is_nested_sequence(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":110 + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_vec = new vector[intptr_t](length) +*/ + __pyx_t_1 = PySequence_Check(__pyx_v_obj); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":111 + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): + * length = len(obj) # <<<<<<<<<<<<<< + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 111, __pyx_L1_error) + __pyx_v_length = __pyx_t_2; + + /* "cuda/bindings/_internal/utils.pyx":112 + * elif cpython.PySequence_Check(obj): + * length = len(obj) + * nested_vec = new vector[intptr_t](length) # <<<<<<<<<<<<<< + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): +*/ + try { + __pyx_t_4 = new std::vector (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 112, __pyx_L1_error) + } + __pyx_v_nested_vec = __pyx_t_4; + + /* "cuda/bindings/_internal/utils.pyx":113 + * length = len(obj) + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) # <<<<<<<<<<<<<< + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr +*/ + __pyx_v_nested_ptr.reset(__pyx_v_nested_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":114 + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_obj)) || PyTuple_CheckExact(__pyx_v_obj)) { + __pyx_t_6 = __pyx_v_obj; __Pyx_INCREF(__pyx_t_6); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_obj); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 114, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 114, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_2); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 114, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_2)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 114, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 114, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_10 = PyLong_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_10 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_addr = __pyx_t_10; + __pyx_v_i = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "cuda/bindings/_internal/utils.pyx":115 + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr # <<<<<<<<<<<<<< + * nested_res_ptr.reset(NULL, False) + * else: +*/ + ((*__pyx_v_nested_vec)[__pyx_v_i]) = __pyx_v_addr; + + /* "cuda/bindings/_internal/utils.pyx":114 + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "cuda/bindings/_internal/utils.pyx":116 + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) # <<<<<<<<<<<<<< + * else: + * # obj is an int (ResT**) +*/ + __pyx_v_nested_res_ptr.reset(NULL, 0); + + /* "cuda/bindings/_internal/utils.pyx":110 + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_vec = new vector[intptr_t](length) +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":119 + * else: + * # obj is an int (ResT**) + * nested_res_ptr.reset(NULL, False) # <<<<<<<<<<<<<< + * nested_ptr.reset(obj, False) + * +*/ + /*else*/ { + __pyx_v_nested_res_ptr.reset(NULL, 0); + + /* "cuda/bindings/_internal/utils.pyx":120 + * # obj is an int (ResT**) + * nested_res_ptr.reset(NULL, False) + * nested_ptr.reset(obj, False) # <<<<<<<<<<<<<< + * + * in_out_ptr.ptrs = move(nested_ptr) +*/ + __pyx_t_10 = PyLong_AsSsize_t(__pyx_v_obj); if (unlikely((__pyx_t_10 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 120, __pyx_L1_error) + __pyx_v_nested_ptr.reset(((std::vector *)((intptr_t)__pyx_t_10)), 0); + } + __pyx_L3:; + + /* "cuda/bindings/_internal/utils.pyx":122 + * nested_ptr.reset(obj, False) + * + * in_out_ptr.ptrs = move(nested_ptr) # <<<<<<<<<<<<<< + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) + * return 0 +*/ + __pyx_v_in_out_ptr.ptrs = cython_std::move > >(__pyx_v_nested_ptr); + + /* "cuda/bindings/_internal/utils.pyx":123 + * + * in_out_ptr.ptrs = move(nested_ptr) + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) # <<<<<<<<<<<<<< + * return 0 + * +*/ + __pyx_v_in_out_ptr.nested_resource_ptr = ((nullable_unique_ptr > > )cython_std::move > > >(__pyx_v_nested_res_ptr)); + + /* "cuda/bindings/_internal/utils.pyx":124 + * in_out_ptr.ptrs = move(nested_ptr) + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":83 + * + * + * cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_nested_resource_ptr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_obj_i); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_fuse_2__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr(__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &__pyx_v_in_out_ptr, PyObject *__pyx_v_obj, CYTHON_UNUSED int64_t *__pyx_v___unused) { + nullable_unique_ptr > __pyx_v_nested_ptr; + nullable_unique_ptr > > __pyx_v_nested_res_ptr; + std::vector *__pyx_v_nested_vec; + std::vector > *__pyx_v_nested_res_vec; + size_t __pyx_v_i; + size_t __pyx_v_length; + intptr_t __pyx_v_addr; + PyObject *__pyx_v_obj_i = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + std::vector > *__pyx_t_3; + std::vector *__pyx_t_4; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + std::vector __pyx_t_9; + intptr_t __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_2get_nested_resource_ptr", 0); + + /* "cuda/bindings/_internal/utils.pyx":86 + * cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr + * cdef vector[intptr_t]* nested_vec = NULL # <<<<<<<<<<<<<< + * cdef vector[vector[ResT]]* nested_res_vec = NULL + * cdef size_t i = 0, length = 0 +*/ + __pyx_v_nested_vec = NULL; + + /* "cuda/bindings/_internal/utils.pyx":87 + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr + * cdef vector[intptr_t]* nested_vec = NULL + * cdef vector[vector[ResT]]* nested_res_vec = NULL # <<<<<<<<<<<<<< + * cdef size_t i = 0, length = 0 + * cdef intptr_t addr +*/ + __pyx_v_nested_res_vec = NULL; + + /* "cuda/bindings/_internal/utils.pyx":88 + * cdef vector[intptr_t]* nested_vec = NULL + * cdef vector[vector[ResT]]* nested_res_vec = NULL + * cdef size_t i = 0, length = 0 # <<<<<<<<<<<<<< + * cdef intptr_t addr + * +*/ + __pyx_v_i = 0; + __pyx_v_length = 0; + + /* "cuda/bindings/_internal/utils.pyx":91 + * cdef intptr_t addr + * + * if is_nested_sequence(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_5utils_is_nested_sequence(__pyx_v_obj); if (unlikely(__pyx_t_1 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(0, 91, __pyx_L1_error) + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":92 + * + * if is_nested_sequence(obj): + * length = len(obj) # <<<<<<<<<<<<<< + * nested_res_vec = new vector[vector[ResT]](length) + * nested_vec = new vector[intptr_t](length) +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_v_length = __pyx_t_2; + + /* "cuda/bindings/_internal/utils.pyx":93 + * if is_nested_sequence(obj): + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) # <<<<<<<<<<<<<< + * nested_vec = new vector[intptr_t](length) + * # set the ownership immediately to avoid leaking memory in case of +*/ + try { + __pyx_t_3 = new std::vector > (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 93, __pyx_L1_error) + } + __pyx_v_nested_res_vec = __pyx_t_3; + + /* "cuda/bindings/_internal/utils.pyx":94 + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) + * nested_vec = new vector[intptr_t](length) # <<<<<<<<<<<<<< + * # set the ownership immediately to avoid leaking memory in case of + * # exception in the following loop +*/ + try { + __pyx_t_4 = new std::vector (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 94, __pyx_L1_error) + } + __pyx_v_nested_vec = __pyx_t_4; + + /* "cuda/bindings/_internal/utils.pyx":97 + * # set the ownership immediately to avoid leaking memory in case of + * # exception in the following loop + * nested_res_ptr.reset(nested_res_vec, True) # <<<<<<<<<<<<<< + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): +*/ + __pyx_v_nested_res_ptr.reset(__pyx_v_nested_res_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":98 + * # exception in the following loop + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) # <<<<<<<<<<<<<< + * for i, obj_i in enumerate(obj): + * if ResT is char: +*/ + __pyx_v_nested_ptr.reset(__pyx_v_nested_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":99 + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): # <<<<<<<<<<<<<< + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_obj)) || PyTuple_CheckExact(__pyx_v_obj)) { + __pyx_t_6 = __pyx_v_obj; __Pyx_INCREF(__pyx_t_6); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_obj); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 99, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 99, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 99, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_2); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 99, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_2)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 99, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 99, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_obj_i, __pyx_t_8); + __pyx_t_8 = 0; + __pyx_v_i = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "cuda/bindings/_internal/utils.pyx":108 + * deref(nested_res_vec)[i].assign(obj_i_ptr, obj_i_ptr + str_len) + * else: + * deref(nested_res_vec)[i] = obj_i # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): +*/ + __pyx_t_9 = __pyx_convert_vector_from_py_int64_t(__pyx_v_obj_i); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 108, __pyx_L1_error) + ((*__pyx_v_nested_res_vec)[__pyx_v_i]) = __pyx_t_9; + + /* "cuda/bindings/_internal/utils.pyx":109 + * else: + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) # <<<<<<<<<<<<<< + * elif cpython.PySequence_Check(obj): + * length = len(obj) +*/ + ((*__pyx_v_nested_vec)[__pyx_v_i]) = ((intptr_t)((*__pyx_v_nested_res_vec)[__pyx_v_i]).data()); + + /* "cuda/bindings/_internal/utils.pyx":99 + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): # <<<<<<<<<<<<<< + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "cuda/bindings/_internal/utils.pyx":91 + * cdef intptr_t addr + * + * if is_nested_sequence(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":110 + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_vec = new vector[intptr_t](length) +*/ + __pyx_t_1 = PySequence_Check(__pyx_v_obj); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":111 + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): + * length = len(obj) # <<<<<<<<<<<<<< + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 111, __pyx_L1_error) + __pyx_v_length = __pyx_t_2; + + /* "cuda/bindings/_internal/utils.pyx":112 + * elif cpython.PySequence_Check(obj): + * length = len(obj) + * nested_vec = new vector[intptr_t](length) # <<<<<<<<<<<<<< + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): +*/ + try { + __pyx_t_4 = new std::vector (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 112, __pyx_L1_error) + } + __pyx_v_nested_vec = __pyx_t_4; + + /* "cuda/bindings/_internal/utils.pyx":113 + * length = len(obj) + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) # <<<<<<<<<<<<<< + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr +*/ + __pyx_v_nested_ptr.reset(__pyx_v_nested_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":114 + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_obj)) || PyTuple_CheckExact(__pyx_v_obj)) { + __pyx_t_6 = __pyx_v_obj; __Pyx_INCREF(__pyx_t_6); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_obj); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 114, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 114, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_2); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 114, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_2)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 114, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 114, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_10 = PyLong_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_10 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_addr = __pyx_t_10; + __pyx_v_i = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "cuda/bindings/_internal/utils.pyx":115 + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr # <<<<<<<<<<<<<< + * nested_res_ptr.reset(NULL, False) + * else: +*/ + ((*__pyx_v_nested_vec)[__pyx_v_i]) = __pyx_v_addr; + + /* "cuda/bindings/_internal/utils.pyx":114 + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "cuda/bindings/_internal/utils.pyx":116 + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) # <<<<<<<<<<<<<< + * else: + * # obj is an int (ResT**) +*/ + __pyx_v_nested_res_ptr.reset(NULL, 0); + + /* "cuda/bindings/_internal/utils.pyx":110 + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_vec = new vector[intptr_t](length) +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":119 + * else: + * # obj is an int (ResT**) + * nested_res_ptr.reset(NULL, False) # <<<<<<<<<<<<<< + * nested_ptr.reset(obj, False) + * +*/ + /*else*/ { + __pyx_v_nested_res_ptr.reset(NULL, 0); + + /* "cuda/bindings/_internal/utils.pyx":120 + * # obj is an int (ResT**) + * nested_res_ptr.reset(NULL, False) + * nested_ptr.reset(obj, False) # <<<<<<<<<<<<<< + * + * in_out_ptr.ptrs = move(nested_ptr) +*/ + __pyx_t_10 = PyLong_AsSsize_t(__pyx_v_obj); if (unlikely((__pyx_t_10 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 120, __pyx_L1_error) + __pyx_v_nested_ptr.reset(((std::vector *)((intptr_t)__pyx_t_10)), 0); + } + __pyx_L3:; + + /* "cuda/bindings/_internal/utils.pyx":122 + * nested_ptr.reset(obj, False) + * + * in_out_ptr.ptrs = move(nested_ptr) # <<<<<<<<<<<<<< + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) + * return 0 +*/ + __pyx_v_in_out_ptr.ptrs = cython_std::move > >(__pyx_v_nested_ptr); + + /* "cuda/bindings/_internal/utils.pyx":123 + * + * in_out_ptr.ptrs = move(nested_ptr) + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) # <<<<<<<<<<<<<< + * return 0 + * +*/ + __pyx_v_in_out_ptr.nested_resource_ptr = ((nullable_unique_ptr > > )cython_std::move > > >(__pyx_v_nested_res_ptr)); + + /* "cuda/bindings/_internal/utils.pyx":124 + * in_out_ptr.ptrs = move(nested_ptr) + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":83 + * + * + * cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_nested_resource_ptr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_obj_i); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_fuse_3__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr(__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &__pyx_v_in_out_ptr, PyObject *__pyx_v_obj, CYTHON_UNUSED char *__pyx_v___unused) { + nullable_unique_ptr > __pyx_v_nested_ptr; + nullable_unique_ptr > > __pyx_v_nested_res_ptr; + std::vector *__pyx_v_nested_vec; + std::vector > *__pyx_v_nested_res_vec; + size_t __pyx_v_i; + size_t __pyx_v_length; + intptr_t __pyx_v_addr; + PyObject *__pyx_v_obj_i = NULL; + PyObject *__pyx_v_obj_i_bytes = NULL; + size_t __pyx_v_str_len; + char *__pyx_v_obj_i_ptr; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + std::vector > *__pyx_t_3; + std::vector *__pyx_t_4; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + Py_ssize_t __pyx_t_9; + char *__pyx_t_10; + intptr_t __pyx_t_11; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_3get_nested_resource_ptr", 0); + + /* "cuda/bindings/_internal/utils.pyx":86 + * cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr + * cdef vector[intptr_t]* nested_vec = NULL # <<<<<<<<<<<<<< + * cdef vector[vector[ResT]]* nested_res_vec = NULL + * cdef size_t i = 0, length = 0 +*/ + __pyx_v_nested_vec = NULL; + + /* "cuda/bindings/_internal/utils.pyx":87 + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr + * cdef vector[intptr_t]* nested_vec = NULL + * cdef vector[vector[ResT]]* nested_res_vec = NULL # <<<<<<<<<<<<<< + * cdef size_t i = 0, length = 0 + * cdef intptr_t addr +*/ + __pyx_v_nested_res_vec = NULL; + + /* "cuda/bindings/_internal/utils.pyx":88 + * cdef vector[intptr_t]* nested_vec = NULL + * cdef vector[vector[ResT]]* nested_res_vec = NULL + * cdef size_t i = 0, length = 0 # <<<<<<<<<<<<<< + * cdef intptr_t addr + * +*/ + __pyx_v_i = 0; + __pyx_v_length = 0; + + /* "cuda/bindings/_internal/utils.pyx":91 + * cdef intptr_t addr + * + * if is_nested_sequence(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_5utils_is_nested_sequence(__pyx_v_obj); if (unlikely(__pyx_t_1 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(0, 91, __pyx_L1_error) + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":92 + * + * if is_nested_sequence(obj): + * length = len(obj) # <<<<<<<<<<<<<< + * nested_res_vec = new vector[vector[ResT]](length) + * nested_vec = new vector[intptr_t](length) +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_v_length = __pyx_t_2; + + /* "cuda/bindings/_internal/utils.pyx":93 + * if is_nested_sequence(obj): + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) # <<<<<<<<<<<<<< + * nested_vec = new vector[intptr_t](length) + * # set the ownership immediately to avoid leaking memory in case of +*/ + try { + __pyx_t_3 = new std::vector > (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 93, __pyx_L1_error) + } + __pyx_v_nested_res_vec = __pyx_t_3; + + /* "cuda/bindings/_internal/utils.pyx":94 + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) + * nested_vec = new vector[intptr_t](length) # <<<<<<<<<<<<<< + * # set the ownership immediately to avoid leaking memory in case of + * # exception in the following loop +*/ + try { + __pyx_t_4 = new std::vector (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 94, __pyx_L1_error) + } + __pyx_v_nested_vec = __pyx_t_4; + + /* "cuda/bindings/_internal/utils.pyx":97 + * # set the ownership immediately to avoid leaking memory in case of + * # exception in the following loop + * nested_res_ptr.reset(nested_res_vec, True) # <<<<<<<<<<<<<< + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): +*/ + __pyx_v_nested_res_ptr.reset(__pyx_v_nested_res_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":98 + * # exception in the following loop + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) # <<<<<<<<<<<<<< + * for i, obj_i in enumerate(obj): + * if ResT is char: +*/ + __pyx_v_nested_ptr.reset(__pyx_v_nested_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":99 + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): # <<<<<<<<<<<<<< + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_obj)) || PyTuple_CheckExact(__pyx_v_obj)) { + __pyx_t_6 = __pyx_v_obj; __Pyx_INCREF(__pyx_t_6); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_obj); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 99, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 99, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 99, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_2); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 99, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_2)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 99, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 99, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_obj_i, __pyx_t_8); + __pyx_t_8 = 0; + __pyx_v_i = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "cuda/bindings/_internal/utils.pyx":101 + * for i, obj_i in enumerate(obj): + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() # <<<<<<<<<<<<<< + * str_len = (len(obj_i_bytes)) + 1 # including null termination + * deref(nested_res_vec)[i].resize(str_len) +*/ + if (!(likely(PyUnicode_CheckExact(__pyx_v_obj_i)) || __Pyx_RaiseUnexpectedTypeError("str", __pyx_v_obj_i))) __PYX_ERR(0, 101, __pyx_L1_error) + if (unlikely(__pyx_v_obj_i == Py_None)) { + PyErr_Format(PyExc_AttributeError, "'NoneType' object has no attribute '%.30s'", "encode"); + __PYX_ERR(0, 101, __pyx_L1_error) + } + __pyx_t_8 = PyUnicode_AsEncodedString(((PyObject*)__pyx_v_obj_i), NULL, NULL); if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 101, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_obj_i_bytes, ((PyObject*)__pyx_t_8)); + __pyx_t_8 = 0; + + /* "cuda/bindings/_internal/utils.pyx":102 + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() + * str_len = (len(obj_i_bytes)) + 1 # including null termination # <<<<<<<<<<<<<< + * deref(nested_res_vec)[i].resize(str_len) + * obj_i_ptr = (obj_i_bytes) +*/ + __pyx_t_9 = __Pyx_PyBytes_GET_SIZE(__pyx_v_obj_i_bytes); if (unlikely(__pyx_t_9 == ((Py_ssize_t)-1))) __PYX_ERR(0, 102, __pyx_L1_error) + __pyx_v_str_len = (((size_t)__pyx_t_9) + 1); + + /* "cuda/bindings/_internal/utils.pyx":103 + * obj_i_bytes = ((obj_i)).encode() + * str_len = (len(obj_i_bytes)) + 1 # including null termination + * deref(nested_res_vec)[i].resize(str_len) # <<<<<<<<<<<<<< + * obj_i_ptr = (obj_i_bytes) + * # cast to size_t explicitly to work around a potentially Cython bug +*/ + try { + ((*__pyx_v_nested_res_vec)[__pyx_v_i]).resize(__pyx_v_str_len); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 103, __pyx_L1_error) + } + + /* "cuda/bindings/_internal/utils.pyx":104 + * str_len = (len(obj_i_bytes)) + 1 # including null termination + * deref(nested_res_vec)[i].resize(str_len) + * obj_i_ptr = (obj_i_bytes) # <<<<<<<<<<<<<< + * # cast to size_t explicitly to work around a potentially Cython bug + * deref(nested_res_vec)[i].assign(obj_i_ptr, obj_i_ptr + str_len) +*/ + __pyx_t_10 = __Pyx_PyBytes_AsWritableString(__pyx_v_obj_i_bytes); if (unlikely((!__pyx_t_10) && PyErr_Occurred())) __PYX_ERR(0, 104, __pyx_L1_error) + __pyx_v_obj_i_ptr = ((char *)__pyx_t_10); + + /* "cuda/bindings/_internal/utils.pyx":106 + * obj_i_ptr = (obj_i_bytes) + * # cast to size_t explicitly to work around a potentially Cython bug + * deref(nested_res_vec)[i].assign(obj_i_ptr, obj_i_ptr + str_len) # <<<<<<<<<<<<<< + * else: + * deref(nested_res_vec)[i] = obj_i +*/ + try { + ((*__pyx_v_nested_res_vec)[__pyx_v_i]).assign(__pyx_v_obj_i_ptr, (__pyx_v_obj_i_ptr + ((size_t)__pyx_v_str_len))); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 106, __pyx_L1_error) + } + + /* "cuda/bindings/_internal/utils.pyx":109 + * else: + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) # <<<<<<<<<<<<<< + * elif cpython.PySequence_Check(obj): + * length = len(obj) +*/ + ((*__pyx_v_nested_vec)[__pyx_v_i]) = ((intptr_t)((*__pyx_v_nested_res_vec)[__pyx_v_i]).data()); + + /* "cuda/bindings/_internal/utils.pyx":99 + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): # <<<<<<<<<<<<<< + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "cuda/bindings/_internal/utils.pyx":91 + * cdef intptr_t addr + * + * if is_nested_sequence(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":110 + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_vec = new vector[intptr_t](length) +*/ + __pyx_t_1 = PySequence_Check(__pyx_v_obj); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":111 + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): + * length = len(obj) # <<<<<<<<<<<<<< + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 111, __pyx_L1_error) + __pyx_v_length = __pyx_t_2; + + /* "cuda/bindings/_internal/utils.pyx":112 + * elif cpython.PySequence_Check(obj): + * length = len(obj) + * nested_vec = new vector[intptr_t](length) # <<<<<<<<<<<<<< + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): +*/ + try { + __pyx_t_4 = new std::vector (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 112, __pyx_L1_error) + } + __pyx_v_nested_vec = __pyx_t_4; + + /* "cuda/bindings/_internal/utils.pyx":113 + * length = len(obj) + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) # <<<<<<<<<<<<<< + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr +*/ + __pyx_v_nested_ptr.reset(__pyx_v_nested_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":114 + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_obj)) || PyTuple_CheckExact(__pyx_v_obj)) { + __pyx_t_6 = __pyx_v_obj; __Pyx_INCREF(__pyx_t_6); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_obj); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 114, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 114, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_2); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 114, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_2)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 114, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 114, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_11 = PyLong_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_11 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_addr = __pyx_t_11; + __pyx_v_i = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "cuda/bindings/_internal/utils.pyx":115 + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr # <<<<<<<<<<<<<< + * nested_res_ptr.reset(NULL, False) + * else: +*/ + ((*__pyx_v_nested_vec)[__pyx_v_i]) = __pyx_v_addr; + + /* "cuda/bindings/_internal/utils.pyx":114 + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "cuda/bindings/_internal/utils.pyx":116 + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) # <<<<<<<<<<<<<< + * else: + * # obj is an int (ResT**) +*/ + __pyx_v_nested_res_ptr.reset(NULL, 0); + + /* "cuda/bindings/_internal/utils.pyx":110 + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_vec = new vector[intptr_t](length) +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":119 + * else: + * # obj is an int (ResT**) + * nested_res_ptr.reset(NULL, False) # <<<<<<<<<<<<<< + * nested_ptr.reset(obj, False) + * +*/ + /*else*/ { + __pyx_v_nested_res_ptr.reset(NULL, 0); + + /* "cuda/bindings/_internal/utils.pyx":120 + * # obj is an int (ResT**) + * nested_res_ptr.reset(NULL, False) + * nested_ptr.reset(obj, False) # <<<<<<<<<<<<<< + * + * in_out_ptr.ptrs = move(nested_ptr) +*/ + __pyx_t_11 = PyLong_AsSsize_t(__pyx_v_obj); if (unlikely((__pyx_t_11 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 120, __pyx_L1_error) + __pyx_v_nested_ptr.reset(((std::vector *)((intptr_t)__pyx_t_11)), 0); + } + __pyx_L3:; + + /* "cuda/bindings/_internal/utils.pyx":122 + * nested_ptr.reset(obj, False) + * + * in_out_ptr.ptrs = move(nested_ptr) # <<<<<<<<<<<<<< + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) + * return 0 +*/ + __pyx_v_in_out_ptr.ptrs = cython_std::move > >(__pyx_v_nested_ptr); + + /* "cuda/bindings/_internal/utils.pyx":123 + * + * in_out_ptr.ptrs = move(nested_ptr) + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) # <<<<<<<<<<<<<< + * return 0 + * +*/ + __pyx_v_in_out_ptr.nested_resource_ptr = cython_std::move > > >(__pyx_v_nested_res_ptr); + + /* "cuda/bindings/_internal/utils.pyx":124 + * in_out_ptr.ptrs = move(nested_ptr) + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":83 + * + * + * cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_nested_resource_ptr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_obj_i); + __Pyx_XDECREF(__pyx_v_obj_i_bytes); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_fuse_4__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr(__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &__pyx_v_in_out_ptr, PyObject *__pyx_v_obj, CYTHON_UNUSED float *__pyx_v___unused) { + nullable_unique_ptr > __pyx_v_nested_ptr; + nullable_unique_ptr > > __pyx_v_nested_res_ptr; + std::vector *__pyx_v_nested_vec; + std::vector > *__pyx_v_nested_res_vec; + size_t __pyx_v_i; + size_t __pyx_v_length; + intptr_t __pyx_v_addr; + PyObject *__pyx_v_obj_i = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + std::vector > *__pyx_t_3; + std::vector *__pyx_t_4; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + std::vector __pyx_t_9; + intptr_t __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_4get_nested_resource_ptr", 0); + + /* "cuda/bindings/_internal/utils.pyx":86 + * cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr + * cdef vector[intptr_t]* nested_vec = NULL # <<<<<<<<<<<<<< + * cdef vector[vector[ResT]]* nested_res_vec = NULL + * cdef size_t i = 0, length = 0 +*/ + __pyx_v_nested_vec = NULL; + + /* "cuda/bindings/_internal/utils.pyx":87 + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr + * cdef vector[intptr_t]* nested_vec = NULL + * cdef vector[vector[ResT]]* nested_res_vec = NULL # <<<<<<<<<<<<<< + * cdef size_t i = 0, length = 0 + * cdef intptr_t addr +*/ + __pyx_v_nested_res_vec = NULL; + + /* "cuda/bindings/_internal/utils.pyx":88 + * cdef vector[intptr_t]* nested_vec = NULL + * cdef vector[vector[ResT]]* nested_res_vec = NULL + * cdef size_t i = 0, length = 0 # <<<<<<<<<<<<<< + * cdef intptr_t addr + * +*/ + __pyx_v_i = 0; + __pyx_v_length = 0; + + /* "cuda/bindings/_internal/utils.pyx":91 + * cdef intptr_t addr + * + * if is_nested_sequence(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_5utils_is_nested_sequence(__pyx_v_obj); if (unlikely(__pyx_t_1 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(0, 91, __pyx_L1_error) + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":92 + * + * if is_nested_sequence(obj): + * length = len(obj) # <<<<<<<<<<<<<< + * nested_res_vec = new vector[vector[ResT]](length) + * nested_vec = new vector[intptr_t](length) +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_v_length = __pyx_t_2; + + /* "cuda/bindings/_internal/utils.pyx":93 + * if is_nested_sequence(obj): + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) # <<<<<<<<<<<<<< + * nested_vec = new vector[intptr_t](length) + * # set the ownership immediately to avoid leaking memory in case of +*/ + try { + __pyx_t_3 = new std::vector > (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 93, __pyx_L1_error) + } + __pyx_v_nested_res_vec = __pyx_t_3; + + /* "cuda/bindings/_internal/utils.pyx":94 + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) + * nested_vec = new vector[intptr_t](length) # <<<<<<<<<<<<<< + * # set the ownership immediately to avoid leaking memory in case of + * # exception in the following loop +*/ + try { + __pyx_t_4 = new std::vector (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 94, __pyx_L1_error) + } + __pyx_v_nested_vec = __pyx_t_4; + + /* "cuda/bindings/_internal/utils.pyx":97 + * # set the ownership immediately to avoid leaking memory in case of + * # exception in the following loop + * nested_res_ptr.reset(nested_res_vec, True) # <<<<<<<<<<<<<< + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): +*/ + __pyx_v_nested_res_ptr.reset(__pyx_v_nested_res_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":98 + * # exception in the following loop + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) # <<<<<<<<<<<<<< + * for i, obj_i in enumerate(obj): + * if ResT is char: +*/ + __pyx_v_nested_ptr.reset(__pyx_v_nested_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":99 + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): # <<<<<<<<<<<<<< + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_obj)) || PyTuple_CheckExact(__pyx_v_obj)) { + __pyx_t_6 = __pyx_v_obj; __Pyx_INCREF(__pyx_t_6); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_obj); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 99, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 99, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 99, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_2); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 99, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_2)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 99, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 99, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_obj_i, __pyx_t_8); + __pyx_t_8 = 0; + __pyx_v_i = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "cuda/bindings/_internal/utils.pyx":108 + * deref(nested_res_vec)[i].assign(obj_i_ptr, obj_i_ptr + str_len) + * else: + * deref(nested_res_vec)[i] = obj_i # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): +*/ + __pyx_t_9 = __pyx_convert_vector_from_py_float(__pyx_v_obj_i); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 108, __pyx_L1_error) + ((*__pyx_v_nested_res_vec)[__pyx_v_i]) = __pyx_t_9; + + /* "cuda/bindings/_internal/utils.pyx":109 + * else: + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) # <<<<<<<<<<<<<< + * elif cpython.PySequence_Check(obj): + * length = len(obj) +*/ + ((*__pyx_v_nested_vec)[__pyx_v_i]) = ((intptr_t)((*__pyx_v_nested_res_vec)[__pyx_v_i]).data()); + + /* "cuda/bindings/_internal/utils.pyx":99 + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): # <<<<<<<<<<<<<< + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "cuda/bindings/_internal/utils.pyx":91 + * cdef intptr_t addr + * + * if is_nested_sequence(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":110 + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_vec = new vector[intptr_t](length) +*/ + __pyx_t_1 = PySequence_Check(__pyx_v_obj); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":111 + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): + * length = len(obj) # <<<<<<<<<<<<<< + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 111, __pyx_L1_error) + __pyx_v_length = __pyx_t_2; + + /* "cuda/bindings/_internal/utils.pyx":112 + * elif cpython.PySequence_Check(obj): + * length = len(obj) + * nested_vec = new vector[intptr_t](length) # <<<<<<<<<<<<<< + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): +*/ + try { + __pyx_t_4 = new std::vector (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 112, __pyx_L1_error) + } + __pyx_v_nested_vec = __pyx_t_4; + + /* "cuda/bindings/_internal/utils.pyx":113 + * length = len(obj) + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) # <<<<<<<<<<<<<< + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr +*/ + __pyx_v_nested_ptr.reset(__pyx_v_nested_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":114 + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_obj)) || PyTuple_CheckExact(__pyx_v_obj)) { + __pyx_t_6 = __pyx_v_obj; __Pyx_INCREF(__pyx_t_6); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_obj); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 114, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 114, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_2); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 114, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_2)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 114, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 114, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_10 = PyLong_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_10 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_addr = __pyx_t_10; + __pyx_v_i = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "cuda/bindings/_internal/utils.pyx":115 + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr # <<<<<<<<<<<<<< + * nested_res_ptr.reset(NULL, False) + * else: +*/ + ((*__pyx_v_nested_vec)[__pyx_v_i]) = __pyx_v_addr; + + /* "cuda/bindings/_internal/utils.pyx":114 + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "cuda/bindings/_internal/utils.pyx":116 + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) # <<<<<<<<<<<<<< + * else: + * # obj is an int (ResT**) +*/ + __pyx_v_nested_res_ptr.reset(NULL, 0); + + /* "cuda/bindings/_internal/utils.pyx":110 + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_vec = new vector[intptr_t](length) +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":119 + * else: + * # obj is an int (ResT**) + * nested_res_ptr.reset(NULL, False) # <<<<<<<<<<<<<< + * nested_ptr.reset(obj, False) + * +*/ + /*else*/ { + __pyx_v_nested_res_ptr.reset(NULL, 0); + + /* "cuda/bindings/_internal/utils.pyx":120 + * # obj is an int (ResT**) + * nested_res_ptr.reset(NULL, False) + * nested_ptr.reset(obj, False) # <<<<<<<<<<<<<< + * + * in_out_ptr.ptrs = move(nested_ptr) +*/ + __pyx_t_10 = PyLong_AsSsize_t(__pyx_v_obj); if (unlikely((__pyx_t_10 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 120, __pyx_L1_error) + __pyx_v_nested_ptr.reset(((std::vector *)((intptr_t)__pyx_t_10)), 0); + } + __pyx_L3:; + + /* "cuda/bindings/_internal/utils.pyx":122 + * nested_ptr.reset(obj, False) + * + * in_out_ptr.ptrs = move(nested_ptr) # <<<<<<<<<<<<<< + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) + * return 0 +*/ + __pyx_v_in_out_ptr.ptrs = cython_std::move > >(__pyx_v_nested_ptr); + + /* "cuda/bindings/_internal/utils.pyx":123 + * + * in_out_ptr.ptrs = move(nested_ptr) + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) # <<<<<<<<<<<<<< + * return 0 + * +*/ + __pyx_v_in_out_ptr.nested_resource_ptr = cython_std::move > > >(__pyx_v_nested_res_ptr); + + /* "cuda/bindings/_internal/utils.pyx":124 + * in_out_ptr.ptrs = move(nested_ptr) + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":83 + * + * + * cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_nested_resource_ptr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_obj_i); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} + +static int __pyx_fuse_5__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr(__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &__pyx_v_in_out_ptr, PyObject *__pyx_v_obj, CYTHON_UNUSED double *__pyx_v___unused) { + nullable_unique_ptr > __pyx_v_nested_ptr; + nullable_unique_ptr > > __pyx_v_nested_res_ptr; + std::vector *__pyx_v_nested_vec; + std::vector > *__pyx_v_nested_res_vec; + size_t __pyx_v_i; + size_t __pyx_v_length; + intptr_t __pyx_v_addr; + PyObject *__pyx_v_obj_i = NULL; + int __pyx_r; + __Pyx_RefNannyDeclarations + int __pyx_t_1; + Py_ssize_t __pyx_t_2; + std::vector > *__pyx_t_3; + std::vector *__pyx_t_4; + size_t __pyx_t_5; + PyObject *__pyx_t_6 = NULL; + PyObject *(*__pyx_t_7)(PyObject *); + PyObject *__pyx_t_8 = NULL; + std::vector __pyx_t_9; + intptr_t __pyx_t_10; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__pyx_fuse_5get_nested_resource_ptr", 0); + + /* "cuda/bindings/_internal/utils.pyx":86 + * cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr + * cdef vector[intptr_t]* nested_vec = NULL # <<<<<<<<<<<<<< + * cdef vector[vector[ResT]]* nested_res_vec = NULL + * cdef size_t i = 0, length = 0 +*/ + __pyx_v_nested_vec = NULL; + + /* "cuda/bindings/_internal/utils.pyx":87 + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr + * cdef vector[intptr_t]* nested_vec = NULL + * cdef vector[vector[ResT]]* nested_res_vec = NULL # <<<<<<<<<<<<<< + * cdef size_t i = 0, length = 0 + * cdef intptr_t addr +*/ + __pyx_v_nested_res_vec = NULL; + + /* "cuda/bindings/_internal/utils.pyx":88 + * cdef vector[intptr_t]* nested_vec = NULL + * cdef vector[vector[ResT]]* nested_res_vec = NULL + * cdef size_t i = 0, length = 0 # <<<<<<<<<<<<<< + * cdef intptr_t addr + * +*/ + __pyx_v_i = 0; + __pyx_v_length = 0; + + /* "cuda/bindings/_internal/utils.pyx":91 + * cdef intptr_t addr + * + * if is_nested_sequence(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) +*/ + __pyx_t_1 = __pyx_f_4cuda_8bindings_9_internal_5utils_is_nested_sequence(__pyx_v_obj); if (unlikely(__pyx_t_1 == ((int)-1) && PyErr_Occurred())) __PYX_ERR(0, 91, __pyx_L1_error) + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":92 + * + * if is_nested_sequence(obj): + * length = len(obj) # <<<<<<<<<<<<<< + * nested_res_vec = new vector[vector[ResT]](length) + * nested_vec = new vector[intptr_t](length) +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 92, __pyx_L1_error) + __pyx_v_length = __pyx_t_2; + + /* "cuda/bindings/_internal/utils.pyx":93 + * if is_nested_sequence(obj): + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) # <<<<<<<<<<<<<< + * nested_vec = new vector[intptr_t](length) + * # set the ownership immediately to avoid leaking memory in case of +*/ + try { + __pyx_t_3 = new std::vector > (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 93, __pyx_L1_error) + } + __pyx_v_nested_res_vec = __pyx_t_3; + + /* "cuda/bindings/_internal/utils.pyx":94 + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) + * nested_vec = new vector[intptr_t](length) # <<<<<<<<<<<<<< + * # set the ownership immediately to avoid leaking memory in case of + * # exception in the following loop +*/ + try { + __pyx_t_4 = new std::vector (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 94, __pyx_L1_error) + } + __pyx_v_nested_vec = __pyx_t_4; + + /* "cuda/bindings/_internal/utils.pyx":97 + * # set the ownership immediately to avoid leaking memory in case of + * # exception in the following loop + * nested_res_ptr.reset(nested_res_vec, True) # <<<<<<<<<<<<<< + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): +*/ + __pyx_v_nested_res_ptr.reset(__pyx_v_nested_res_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":98 + * # exception in the following loop + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) # <<<<<<<<<<<<<< + * for i, obj_i in enumerate(obj): + * if ResT is char: +*/ + __pyx_v_nested_ptr.reset(__pyx_v_nested_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":99 + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): # <<<<<<<<<<<<<< + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_obj)) || PyTuple_CheckExact(__pyx_v_obj)) { + __pyx_t_6 = __pyx_v_obj; __Pyx_INCREF(__pyx_t_6); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_obj); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 99, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 99, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 99, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_2); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 99, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_2)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 99, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 99, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __Pyx_XDECREF_SET(__pyx_v_obj_i, __pyx_t_8); + __pyx_t_8 = 0; + __pyx_v_i = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "cuda/bindings/_internal/utils.pyx":108 + * deref(nested_res_vec)[i].assign(obj_i_ptr, obj_i_ptr + str_len) + * else: + * deref(nested_res_vec)[i] = obj_i # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): +*/ + __pyx_t_9 = __pyx_convert_vector_from_py_double(__pyx_v_obj_i); if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 108, __pyx_L1_error) + ((*__pyx_v_nested_res_vec)[__pyx_v_i]) = __pyx_t_9; + + /* "cuda/bindings/_internal/utils.pyx":109 + * else: + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) # <<<<<<<<<<<<<< + * elif cpython.PySequence_Check(obj): + * length = len(obj) +*/ + ((*__pyx_v_nested_vec)[__pyx_v_i]) = ((intptr_t)((*__pyx_v_nested_res_vec)[__pyx_v_i]).data()); + + /* "cuda/bindings/_internal/utils.pyx":99 + * nested_res_ptr.reset(nested_res_vec, True) + * nested_ptr.reset(nested_vec, True) + * for i, obj_i in enumerate(obj): # <<<<<<<<<<<<<< + * if ResT is char: + * obj_i_bytes = ((obj_i)).encode() +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "cuda/bindings/_internal/utils.pyx":91 + * cdef intptr_t addr + * + * if is_nested_sequence(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_res_vec = new vector[vector[ResT]](length) +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":110 + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_vec = new vector[intptr_t](length) +*/ + __pyx_t_1 = PySequence_Check(__pyx_v_obj); + if (__pyx_t_1) { + + /* "cuda/bindings/_internal/utils.pyx":111 + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): + * length = len(obj) # <<<<<<<<<<<<<< + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) +*/ + __pyx_t_2 = PyObject_Length(__pyx_v_obj); if (unlikely(__pyx_t_2 == ((Py_ssize_t)-1))) __PYX_ERR(0, 111, __pyx_L1_error) + __pyx_v_length = __pyx_t_2; + + /* "cuda/bindings/_internal/utils.pyx":112 + * elif cpython.PySequence_Check(obj): + * length = len(obj) + * nested_vec = new vector[intptr_t](length) # <<<<<<<<<<<<<< + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): +*/ + try { + __pyx_t_4 = new std::vector (__pyx_v_length); + } catch(...) { + __Pyx_CppExn2PyErr(); + __PYX_ERR(0, 112, __pyx_L1_error) + } + __pyx_v_nested_vec = __pyx_t_4; + + /* "cuda/bindings/_internal/utils.pyx":113 + * length = len(obj) + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) # <<<<<<<<<<<<<< + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr +*/ + __pyx_v_nested_ptr.reset(__pyx_v_nested_vec, 1); + + /* "cuda/bindings/_internal/utils.pyx":114 + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) +*/ + __pyx_t_5 = 0; + if (likely(PyList_CheckExact(__pyx_v_obj)) || PyTuple_CheckExact(__pyx_v_obj)) { + __pyx_t_6 = __pyx_v_obj; __Pyx_INCREF(__pyx_t_6); + __pyx_t_2 = 0; + __pyx_t_7 = NULL; + } else { + __pyx_t_2 = -1; __pyx_t_6 = PyObject_GetIter(__pyx_v_obj); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __pyx_t_7 = (CYTHON_COMPILING_IN_LIMITED_API) ? PyIter_Next : __Pyx_PyObject_GetIterNextFunc(__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 114, __pyx_L1_error) + } + for (;;) { + if (likely(!__pyx_t_7)) { + if (likely(PyList_CheckExact(__pyx_t_6))) { + { + Py_ssize_t __pyx_temp = __Pyx_PyList_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 114, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + __pyx_t_8 = __Pyx_PyList_GetItemRef(__pyx_t_6, __pyx_t_2); + ++__pyx_t_2; + } else { + { + Py_ssize_t __pyx_temp = __Pyx_PyTuple_GET_SIZE(__pyx_t_6); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely((__pyx_temp < 0))) __PYX_ERR(0, 114, __pyx_L1_error) + #endif + if (__pyx_t_2 >= __pyx_temp) break; + } + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_8 = __Pyx_NewRef(PyTuple_GET_ITEM(__pyx_t_6, __pyx_t_2)); + #else + __pyx_t_8 = __Pyx_PySequence_ITEM(__pyx_t_6, __pyx_t_2); + #endif + ++__pyx_t_2; + } + if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 114, __pyx_L1_error) + } else { + __pyx_t_8 = __pyx_t_7(__pyx_t_6); + if (unlikely(!__pyx_t_8)) { + PyObject* exc_type = PyErr_Occurred(); + if (exc_type) { + if (unlikely(!__Pyx_PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration))) __PYX_ERR(0, 114, __pyx_L1_error) + PyErr_Clear(); + } + break; + } + } + __Pyx_GOTREF(__pyx_t_8); + __pyx_t_10 = PyLong_AsSsize_t(__pyx_t_8); if (unlikely((__pyx_t_10 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 114, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_8); __pyx_t_8 = 0; + __pyx_v_addr = __pyx_t_10; + __pyx_v_i = __pyx_t_5; + __pyx_t_5 = (__pyx_t_5 + 1); + + /* "cuda/bindings/_internal/utils.pyx":115 + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr # <<<<<<<<<<<<<< + * nested_res_ptr.reset(NULL, False) + * else: +*/ + ((*__pyx_v_nested_vec)[__pyx_v_i]) = __pyx_v_addr; + + /* "cuda/bindings/_internal/utils.pyx":114 + * nested_vec = new vector[intptr_t](length) + * nested_ptr.reset(nested_vec, True) + * for i, addr in enumerate(obj): # <<<<<<<<<<<<<< + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) +*/ + } + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + + /* "cuda/bindings/_internal/utils.pyx":116 + * for i, addr in enumerate(obj): + * deref(nested_vec)[i] = addr + * nested_res_ptr.reset(NULL, False) # <<<<<<<<<<<<<< + * else: + * # obj is an int (ResT**) +*/ + __pyx_v_nested_res_ptr.reset(NULL, 0); + + /* "cuda/bindings/_internal/utils.pyx":110 + * deref(nested_res_vec)[i] = obj_i + * deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + * elif cpython.PySequence_Check(obj): # <<<<<<<<<<<<<< + * length = len(obj) + * nested_vec = new vector[intptr_t](length) +*/ + goto __pyx_L3; + } + + /* "cuda/bindings/_internal/utils.pyx":119 + * else: + * # obj is an int (ResT**) + * nested_res_ptr.reset(NULL, False) # <<<<<<<<<<<<<< + * nested_ptr.reset(obj, False) + * +*/ + /*else*/ { + __pyx_v_nested_res_ptr.reset(NULL, 0); + + /* "cuda/bindings/_internal/utils.pyx":120 + * # obj is an int (ResT**) + * nested_res_ptr.reset(NULL, False) + * nested_ptr.reset(obj, False) # <<<<<<<<<<<<<< + * + * in_out_ptr.ptrs = move(nested_ptr) +*/ + __pyx_t_10 = PyLong_AsSsize_t(__pyx_v_obj); if (unlikely((__pyx_t_10 == ((intptr_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 120, __pyx_L1_error) + __pyx_v_nested_ptr.reset(((std::vector *)((intptr_t)__pyx_t_10)), 0); + } + __pyx_L3:; + + /* "cuda/bindings/_internal/utils.pyx":122 + * nested_ptr.reset(obj, False) + * + * in_out_ptr.ptrs = move(nested_ptr) # <<<<<<<<<<<<<< + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) + * return 0 +*/ + __pyx_v_in_out_ptr.ptrs = cython_std::move > >(__pyx_v_nested_ptr); + + /* "cuda/bindings/_internal/utils.pyx":123 + * + * in_out_ptr.ptrs = move(nested_ptr) + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) # <<<<<<<<<<<<<< + * return 0 + * +*/ + __pyx_v_in_out_ptr.nested_resource_ptr = cython_std::move > > >(__pyx_v_nested_res_ptr); + + /* "cuda/bindings/_internal/utils.pyx":124 + * in_out_ptr.ptrs = move(nested_ptr) + * in_out_ptr.nested_resource_ptr = move(nested_res_ptr) + * return 0 # <<<<<<<<<<<<<< + * + * +*/ + __pyx_r = 0; + goto __pyx_L0; + + /* "cuda/bindings/_internal/utils.pyx":83 + * + * + * cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, ResT* __unused) except 1: # <<<<<<<<<<<<<< + * cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + * cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr +*/ + + /* function exit code */ + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_6); + __Pyx_XDECREF(__pyx_t_8); + __Pyx_AddTraceback("cuda.bindings._internal.utils.get_nested_resource_ptr", __pyx_clineno, __pyx_lineno, __pyx_filename); + __pyx_r = 1; + __pyx_L0:; + __Pyx_XDECREF(__pyx_v_obj_i); + __Pyx_RefNannyFinishContext(); + return __pyx_r; +} +/* #### Code section: module_exttypes ### */ + +static PyMethodDef __pyx_methods[] = { + {0, 0, 0, 0} +}; +/* #### Code section: initfunc_declarations ### */ +static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate); /*proto*/ +static CYTHON_SMALL_CODE int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate); /*proto*/ +/* #### Code section: init_module ### */ + +static int __Pyx_modinit_global_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_global_init_code", 0); + /*--- Global init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_variable_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_export_code", 0); + /*--- Variable export code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_export_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_function_export_code", 0); + /*--- Function export code ---*/ + if (__Pyx_ExportFunction("is_nested_sequence", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_5utils_is_nested_sequence, "int (PyObject *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("get_buffer_pointer", (void (*)(void))__pyx_f_4cuda_8bindings_9_internal_5utils_get_buffer_pointer, "void *(PyObject *, Py_ssize_t, struct __pyx_opt_args_4cuda_8bindings_9_internal_5utils_get_buffer_pointer *__pyx_optional_args)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("__pyx_fuse_0get_resource_ptr", (void (*)(void))__pyx_fuse_0__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr, "int (nullable_unique_ptr > &, PyObject *, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("__pyx_fuse_1get_resource_ptr", (void (*)(void))__pyx_fuse_1__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr, "int (nullable_unique_ptr > &, PyObject *, int32_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("__pyx_fuse_2get_resource_ptr", (void (*)(void))__pyx_fuse_2__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr, "int (nullable_unique_ptr > &, PyObject *, int64_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("__pyx_fuse_3get_resource_ptr", (void (*)(void))__pyx_fuse_3__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr, "int (nullable_unique_ptr > &, PyObject *, char *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("__pyx_fuse_4get_resource_ptr", (void (*)(void))__pyx_fuse_4__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr, "int (nullable_unique_ptr > &, PyObject *, float *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("__pyx_fuse_5get_resource_ptr", (void (*)(void))__pyx_fuse_5__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptr, "int (nullable_unique_ptr > &, PyObject *, double *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("__pyx_fuse_0get_resource_ptrs", (void (*)(void))__pyx_fuse_0__pyx_f_4cuda_8bindings_9_internal_5utils_get_resource_ptrs, "int (nullable_unique_ptr > &, PyObject *, void *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("__pyx_fuse_0get_nested_resource_ptr", (void (*)(void))__pyx_fuse_0__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr, "int (__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &, PyObject *, int *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("__pyx_fuse_1get_nested_resource_ptr", (void (*)(void))__pyx_fuse_1__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr, "int (__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &, PyObject *, int32_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("__pyx_fuse_2get_nested_resource_ptr", (void (*)(void))__pyx_fuse_2__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr, "int (__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &, PyObject *, int64_t *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("__pyx_fuse_3get_nested_resource_ptr", (void (*)(void))__pyx_fuse_3__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr, "int (__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &, PyObject *, char *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("__pyx_fuse_4get_nested_resource_ptr", (void (*)(void))__pyx_fuse_4__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr, "int (__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &, PyObject *, float *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_ExportFunction("__pyx_fuse_5get_nested_resource_ptr", (void (*)(void))__pyx_fuse_5__pyx_f_4cuda_8bindings_9_internal_5utils_get_nested_resource_ptr, "int (__pyx_t_4cuda_8bindings_9_internal_5utils_nested_resource &, PyObject *, double *)") < 0) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_type_init_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_type_init_code", 0); + /*--- Type init code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_type_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + PyObject *__pyx_t_1 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannySetupContext("__Pyx_modinit_type_import_code", 0); + /*--- Type import code ---*/ + __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(3, 9, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_7cpython_4type_type = __Pyx_ImportType_3_1_5(__pyx_t_1, __Pyx_BUILTIN_MODULE_NAME, "type", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyTypeObject), __PYX_GET_STRUCT_ALIGNMENT_3_1_5(PyTypeObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + 0, 0, + #else + sizeof(PyHeapTypeObject), __PYX_GET_STRUCT_ALIGNMENT_3_1_5(PyHeapTypeObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_1_5); if (!__pyx_mstate->__pyx_ptype_7cpython_4type_type) __PYX_ERR(3, 9, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(4, 8, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_7cpython_4bool_bool = __Pyx_ImportType_3_1_5(__pyx_t_1, __Pyx_BUILTIN_MODULE_NAME, "bool", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyLongObject), __PYX_GET_STRUCT_ALIGNMENT_3_1_5(PyLongObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + 0, 0, + #else + sizeof(PyLongObject), __PYX_GET_STRUCT_ALIGNMENT_3_1_5(PyLongObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_1_5); if (!__pyx_mstate->__pyx_ptype_7cpython_4bool_bool) __PYX_ERR(4, 8, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __pyx_t_1 = PyImport_ImportModule(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_t_1)) __PYX_ERR(5, 16, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __pyx_mstate->__pyx_ptype_7cpython_7complex_complex = __Pyx_ImportType_3_1_5(__pyx_t_1, __Pyx_BUILTIN_MODULE_NAME, "complex", + #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000 + sizeof(PyComplexObject), __PYX_GET_STRUCT_ALIGNMENT_3_1_5(PyComplexObject), + #elif CYTHON_COMPILING_IN_LIMITED_API + 0, 0, + #else + sizeof(PyComplexObject), __PYX_GET_STRUCT_ALIGNMENT_3_1_5(PyComplexObject), + #endif + __Pyx_ImportType_CheckSize_Warn_3_1_5); if (!__pyx_mstate->__pyx_ptype_7cpython_7complex_complex) __PYX_ERR(5, 16, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_1); + __Pyx_RefNannyFinishContext(); + return -1; +} + +static int __Pyx_modinit_variable_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_variable_import_code", 0); + /*--- Variable import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +static int __Pyx_modinit_function_import_code(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_modinit_function_import_code", 0); + /*--- Function import code ---*/ + __Pyx_RefNannyFinishContext(); + return 0; +} + +#if CYTHON_PEP489_MULTI_PHASE_INIT +static PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def); /*proto*/ +static int __pyx_pymod_exec_utils(PyObject* module); /*proto*/ +static PyModuleDef_Slot __pyx_moduledef_slots[] = { + {Py_mod_create, (void*)__pyx_pymod_create}, + {Py_mod_exec, (void*)__pyx_pymod_exec_utils}, + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + #endif + #if PY_VERSION_HEX >= 0x030C0000 && CYTHON_USE_MODULE_STATE + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + #endif + {0, NULL} +}; +#endif + +#ifdef __cplusplus +namespace { + struct PyModuleDef __pyx_moduledef = + #else + static struct PyModuleDef __pyx_moduledef = + #endif + { + PyModuleDef_HEAD_INIT, + "utils", + 0, /* m_doc */ + #if CYTHON_USE_MODULE_STATE + sizeof(__pyx_mstatetype), /* m_size */ + #else + (CYTHON_PEP489_MULTI_PHASE_INIT) ? 0 : -1, /* m_size */ + #endif + __pyx_methods /* m_methods */, + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_moduledef_slots, /* m_slots */ + #else + NULL, /* m_reload */ + #endif + #if CYTHON_USE_MODULE_STATE + __pyx_m_traverse, /* m_traverse */ + __pyx_m_clear, /* m_clear */ + NULL /* m_free */ + #else + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ + #endif + }; + #ifdef __cplusplus +} /* anonymous namespace */ +#endif + +/* PyModInitFuncType */ +#ifndef CYTHON_NO_PYINIT_EXPORT + #define __Pyx_PyMODINIT_FUNC PyMODINIT_FUNC +#else + #ifdef __cplusplus + #define __Pyx_PyMODINIT_FUNC extern "C" PyObject * + #else + #define __Pyx_PyMODINIT_FUNC PyObject * + #endif +#endif + +__Pyx_PyMODINIT_FUNC PyInit_utils(void) CYTHON_SMALL_CODE; /*proto*/ +__Pyx_PyMODINIT_FUNC PyInit_utils(void) +#if CYTHON_PEP489_MULTI_PHASE_INIT +{ + return PyModuleDef_Init(&__pyx_moduledef); +} +/* ModuleCreationPEP489 */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x03090000 +static PY_INT64_T __Pyx_GetCurrentInterpreterId(void) { + { + PyObject *module = PyImport_ImportModule("_interpreters"); // 3.13+ I think + if (!module) { + PyErr_Clear(); // just try the 3.8-3.12 version + module = PyImport_ImportModule("_xxsubinterpreters"); + if (!module) goto bad; + } + PyObject *current = PyObject_CallMethod(module, "get_current", NULL); + Py_DECREF(module); + if (!current) goto bad; + if (PyTuple_Check(current)) { + PyObject *new_current = PySequence_GetItem(current, 0); + Py_DECREF(current); + current = new_current; + if (!new_current) goto bad; + } + long long as_c_int = PyLong_AsLongLong(current); + Py_DECREF(current); + return as_c_int; + } + bad: + PySys_WriteStderr("__Pyx_GetCurrentInterpreterId failed. Try setting the C define CYTHON_PEP489_MULTI_PHASE_INIT=0\n"); + return -1; +} +#endif +#if !CYTHON_USE_MODULE_STATE +static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { + static PY_INT64_T main_interpreter_id = -1; +#if CYTHON_COMPILING_IN_GRAAL + PY_INT64_T current_id = PyInterpreterState_GetIDFromThreadState(PyThreadState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX >= 0x03090000 + PY_INT64_T current_id = PyInterpreterState_GetID(PyInterpreterState_Get()); +#elif CYTHON_COMPILING_IN_LIMITED_API + PY_INT64_T current_id = __Pyx_GetCurrentInterpreterId(); +#else + PY_INT64_T current_id = PyInterpreterState_GetID(PyThreadState_Get()->interp); +#endif + if (unlikely(current_id == -1)) { + return -1; + } + if (main_interpreter_id == -1) { + main_interpreter_id = current_id; + return 0; + } else if (unlikely(main_interpreter_id != current_id)) { + PyErr_SetString( + PyExc_ImportError, + "Interpreter change detected - this module can only be loaded into one interpreter per process."); + return -1; + } + return 0; +} +#endif +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) +{ + PyObject *value = PyObject_GetAttrString(spec, from_name); + int result = 0; + if (likely(value)) { + if (allow_none || value != Py_None) { + result = PyDict_SetItemString(moddict, to_name, value); + } + Py_DECREF(value); + } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + } else { + result = -1; + } + return result; +} +static CYTHON_SMALL_CODE PyObject* __pyx_pymod_create(PyObject *spec, PyModuleDef *def) { + PyObject *module = NULL, *moddict, *modname; + CYTHON_UNUSED_VAR(def); + #if !CYTHON_USE_MODULE_STATE + if (__Pyx_check_single_interpreter()) + return NULL; + #endif + if (__pyx_m) + return __Pyx_NewRef(__pyx_m); + modname = PyObject_GetAttrString(spec, "name"); + if (unlikely(!modname)) goto bad; + module = PyModule_NewObject(modname); + Py_DECREF(modname); + if (unlikely(!module)) goto bad; + moddict = PyModule_GetDict(module); + if (unlikely(!moddict)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "parent", "__package__", 1) < 0)) goto bad; + if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "submodule_search_locations", "__path__", 0) < 0)) goto bad; + return module; +bad: + Py_XDECREF(module); + return NULL; +} + + +static CYTHON_SMALL_CODE int __pyx_pymod_exec_utils(PyObject *__pyx_pyinit_module) +#endif +{ + int stringtab_initialized = 0; + #if CYTHON_USE_MODULE_STATE + int pystate_addmodule_run = 0; + #endif + __pyx_mstatetype *__pyx_mstate = NULL; + PyObject *__pyx_t_1 = NULL; + PyObject *__pyx_t_2 = NULL; + PyObject *__pyx_t_3 = NULL; + PyObject *__pyx_t_4 = NULL; + PyObject *__pyx_t_5 = NULL; + int __pyx_lineno = 0; + const char *__pyx_filename = NULL; + int __pyx_clineno = 0; + __Pyx_RefNannyDeclarations + #if CYTHON_PEP489_MULTI_PHASE_INIT + if (__pyx_m) { + if (__pyx_m == __pyx_pyinit_module) return 0; + PyErr_SetString(PyExc_RuntimeError, "Module 'utils' has already been imported. Re-initialisation is not supported."); + return -1; + } + #else + if (__pyx_m) return __Pyx_NewRef(__pyx_m); + #endif + /*--- Module creation code ---*/ + #if CYTHON_PEP489_MULTI_PHASE_INIT + __pyx_t_1 = __pyx_pyinit_module; + Py_INCREF(__pyx_t_1); + #else + __pyx_t_1 = PyModule_Create(&__pyx_moduledef); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #if CYTHON_USE_MODULE_STATE + { + int add_module_result = __Pyx_State_AddModule(__pyx_t_1, &__pyx_moduledef); + __pyx_t_1 = 0; /* transfer ownership from __pyx_t_1 to "utils" pseudovariable */ + if (unlikely((add_module_result < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + pystate_addmodule_run = 1; + } + #else + __pyx_m = __pyx_t_1; + #endif + #if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + PyUnstable_Module_SetGIL(__pyx_m, Py_MOD_GIL_NOT_USED); + #endif + __pyx_mstate = __pyx_mstate_global; + CYTHON_UNUSED_VAR(__pyx_t_1); + __pyx_mstate->__pyx_d = PyModule_GetDict(__pyx_m); if (unlikely(!__pyx_mstate->__pyx_d)) __PYX_ERR(0, 1, __pyx_L1_error) + Py_INCREF(__pyx_mstate->__pyx_d); + __pyx_mstate->__pyx_b = __Pyx_PyImport_AddModuleRef(__Pyx_BUILTIN_MODULE_NAME); if (unlikely(!__pyx_mstate->__pyx_b)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_cython_runtime = __Pyx_PyImport_AddModuleRef("cython_runtime"); if (unlikely(!__pyx_mstate->__pyx_cython_runtime)) __PYX_ERR(0, 1, __pyx_L1_error) + if (PyObject_SetAttrString(__pyx_m, "__builtins__", __pyx_mstate->__pyx_b) < 0) __PYX_ERR(0, 1, __pyx_L1_error) + /* ImportRefnannyAPI */ + #if CYTHON_REFNANNY +__Pyx_RefNanny = __Pyx_RefNannyImportAPI("refnanny"); +if (!__Pyx_RefNanny) { + PyErr_Clear(); + __Pyx_RefNanny = __Pyx_RefNannyImportAPI("Cython.Runtime.refnanny"); + if (!__Pyx_RefNanny) + Py_FatalError("failed to import 'refnanny' module"); +} +#endif + +__Pyx_RefNannySetupContext("PyInit_utils", 0); + if (__Pyx_check_binary_version(__PYX_LIMITED_VERSION_HEX, __Pyx_get_runtime_version(), CYTHON_COMPILING_IN_LIMITED_API) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #ifdef __Pxy_PyFrame_Initialize_Offsets + __Pxy_PyFrame_Initialize_Offsets(); + #endif + __pyx_mstate->__pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_mstate->__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_mstate->__pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_mstate->__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Initialize various global constants etc. ---*/ + if (__Pyx_InitConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + stringtab_initialized = 1; + if (__Pyx_InitGlobals() < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #if 0 || defined(__Pyx_CyFunction_USED) || defined(__Pyx_FusedFunction_USED) || defined(__Pyx_Coroutine_USED) || defined(__Pyx_Generator_USED) || defined(__Pyx_AsyncGen_USED) + if (__pyx_CommonTypesMetaclass_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_CyFunction_USED + if (__pyx_CyFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_FusedFunction_USED + if (__pyx_FusedFunction_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Coroutine_USED + if (__pyx_Coroutine_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_Generator_USED + if (__pyx_Generator_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + #ifdef __Pyx_AsyncGen_USED + if (__pyx_AsyncGen_init(__pyx_m) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + #endif + /*--- Library function declarations ---*/ + if (__pyx_module_is_main_cuda__bindings___internal__utils) { + if (PyObject_SetAttr(__pyx_m, __pyx_mstate_global->__pyx_n_u_name, __pyx_mstate_global->__pyx_n_u_main) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + } + { + PyObject *modules = PyImport_GetModuleDict(); if (unlikely(!modules)) __PYX_ERR(0, 1, __pyx_L1_error) + if (!PyDict_GetItemString(modules, "cuda.bindings._internal.utils")) { + if (unlikely((PyDict_SetItemString(modules, "cuda.bindings._internal.utils", __pyx_m) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + } + } + /*--- Builtin init code ---*/ + if (__Pyx_InitCachedBuiltins(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Constants init code ---*/ + if (__Pyx_InitCachedConstants(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + if (__Pyx_CreateCodeObjects(__pyx_mstate) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + /*--- Global type/function init code ---*/ + (void)__Pyx_modinit_global_init_code(__pyx_mstate); + (void)__Pyx_modinit_variable_export_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_function_export_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_type_init_code(__pyx_mstate); + if (unlikely((__Pyx_modinit_type_import_code(__pyx_mstate) < 0))) __PYX_ERR(0, 1, __pyx_L1_error) + (void)__Pyx_modinit_variable_import_code(__pyx_mstate); + (void)__Pyx_modinit_function_import_code(__pyx_mstate); + /*--- Execution code ---*/ + + /* "cuda/bindings/_internal/utils.pyx":127 + * + * + * class FunctionNotFoundError(RuntimeError): pass # <<<<<<<<<<<<<< + * + * class NotSupportedError(RuntimeError): pass +*/ + __pyx_t_2 = __Pyx_PEP560_update_bases(__pyx_mstate_global->__pyx_tuple[1]); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 127, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_CalculateMetaclass(NULL, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 127, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_Py3MetaclassPrepare(__pyx_t_3, __pyx_t_2, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError, (PyObject *) NULL, __pyx_mstate_global->__pyx_n_u_cuda_bindings__internal_utils, (PyObject *) NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 127, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (__pyx_t_2 != __pyx_mstate_global->__pyx_tuple[1]) { + if (unlikely((PyDict_SetItemString(__pyx_t_4, "__orig_bases__", __pyx_mstate_global->__pyx_tuple[1]) < 0))) __PYX_ERR(0, 127, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_Py3ClassCreate(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError, __pyx_t_2, __pyx_t_4, NULL, 0, 0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 127, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_FunctionNotFoundError, __pyx_t_5) < (0)) __PYX_ERR(0, 127, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/utils.pyx":129 + * class FunctionNotFoundError(RuntimeError): pass + * + * class NotSupportedError(RuntimeError): pass # <<<<<<<<<<<<<< +*/ + __pyx_t_2 = __Pyx_PEP560_update_bases(__pyx_mstate_global->__pyx_tuple[3]); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 129, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __pyx_t_3 = __Pyx_CalculateMetaclass(NULL, __pyx_t_2); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 129, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + __pyx_t_4 = __Pyx_Py3MetaclassPrepare(__pyx_t_3, __pyx_t_2, __pyx_mstate_global->__pyx_n_u_NotSupportedError, __pyx_mstate_global->__pyx_n_u_NotSupportedError, (PyObject *) NULL, __pyx_mstate_global->__pyx_n_u_cuda_bindings__internal_utils, (PyObject *) NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 129, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_4); + if (__pyx_t_2 != __pyx_mstate_global->__pyx_tuple[3]) { + if (unlikely((PyDict_SetItemString(__pyx_t_4, "__orig_bases__", __pyx_mstate_global->__pyx_tuple[3]) < 0))) __PYX_ERR(0, 129, __pyx_L1_error) + } + __pyx_t_5 = __Pyx_Py3ClassCreate(__pyx_t_3, __pyx_mstate_global->__pyx_n_u_NotSupportedError, __pyx_t_2, __pyx_t_4, NULL, 0, 0); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 129, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_NotSupportedError, __pyx_t_5) < (0)) __PYX_ERR(0, 129, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /* "cuda/bindings/_internal/utils.pyx":1 + * # SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # <<<<<<<<<<<<<< + * # + * # SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +*/ + __pyx_t_2 = __Pyx_PyDict_NewPresized(0); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + if (PyDict_SetItem(__pyx_mstate_global->__pyx_d, __pyx_mstate_global->__pyx_n_u_test, __pyx_t_2) < (0)) __PYX_ERR(0, 1, __pyx_L1_error) + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + + /*--- Wrapped vars code ---*/ + + goto __pyx_L0; + __pyx_L1_error:; + __Pyx_XDECREF(__pyx_t_2); + __Pyx_XDECREF(__pyx_t_3); + __Pyx_XDECREF(__pyx_t_4); + __Pyx_XDECREF(__pyx_t_5); + if (__pyx_m) { + if (__pyx_mstate->__pyx_d && stringtab_initialized) { + __Pyx_AddTraceback("init cuda.bindings._internal.utils", __pyx_clineno, __pyx_lineno, __pyx_filename); + } + #if !CYTHON_USE_MODULE_STATE + Py_CLEAR(__pyx_m); + #else + Py_DECREF(__pyx_m); + if (pystate_addmodule_run) { + PyObject *tp, *value, *tb; + PyErr_Fetch(&tp, &value, &tb); + PyState_RemoveModule(&__pyx_moduledef); + PyErr_Restore(tp, value, tb); + } + #endif + } else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_ImportError, "init cuda.bindings._internal.utils"); + } + __pyx_L0:; + __Pyx_RefNannyFinishContext(); + #if CYTHON_PEP489_MULTI_PHASE_INIT + return (__pyx_m != NULL) ? 0 : -1; + #else + return __pyx_m; + #endif +} +/* #### Code section: pystring_table ### */ + +typedef struct { + const char *s; +#if 103 <= 65535 + const unsigned short n; +#elif 103 / 2 < INT_MAX + const unsigned int n; +#elif 103 / 2 < LONG_MAX + const unsigned long n; +#else + const Py_ssize_t n; +#endif +#if 1 <= 31 + const unsigned int encoding : 5; +#elif 1 <= 255 + const unsigned char encoding; +#elif 1 <= 65535 + const unsigned short encoding; +#else + const Py_ssize_t encoding; +#endif + const unsigned int is_unicode : 1; + const unsigned int intern : 1; +} __Pyx_StringTabEntry; +static const char * const __pyx_string_tab_encodings[] = { 0 }; +static const __Pyx_StringTabEntry __pyx_string_tab[] = { + {__pyx_k_, sizeof(__pyx_k_), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_ */ + {__pyx_k_AssertionError, sizeof(__pyx_k_AssertionError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_AssertionError */ + {__pyx_k_FunctionNotFoundError, sizeof(__pyx_k_FunctionNotFoundError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_FunctionNotFoundError */ + {__pyx_k_None, sizeof(__pyx_k_None), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_None */ + {__pyx_k_NotSupportedError, sizeof(__pyx_k_NotSupportedError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_NotSupportedError */ + {__pyx_k_RuntimeError, sizeof(__pyx_k_RuntimeError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_RuntimeError */ + {__pyx_k_ValueError, sizeof(__pyx_k_ValueError), 0, 1, 1}, /* PyObject cname: __pyx_n_u_ValueError */ + {__pyx_k__2, sizeof(__pyx_k__2), 0, 1, 0}, /* PyObject cname: __pyx_kp_u__2 */ + {__pyx_k_buf_must_be_either_a_Python_int, sizeof(__pyx_k_buf_must_be_either_a_Python_int), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_buf_must_be_either_a_Python_int */ + {__pyx_k_buffer_of_size_bytes, sizeof(__pyx_k_buffer_of_size_bytes), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_buffer_of_size_bytes */ + {__pyx_k_cline_in_traceback, sizeof(__pyx_k_cline_in_traceback), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cline_in_traceback */ + {__pyx_k_cuda_bindings__internal_utils, sizeof(__pyx_k_cuda_bindings__internal_utils), 0, 1, 1}, /* PyObject cname: __pyx_n_u_cuda_bindings__internal_utils */ + {__pyx_k_doc, sizeof(__pyx_k_doc), 0, 1, 1}, /* PyObject cname: __pyx_n_u_doc */ + {__pyx_k_enumerate, sizeof(__pyx_k_enumerate), 0, 1, 1}, /* PyObject cname: __pyx_n_u_enumerate */ + {__pyx_k_main, sizeof(__pyx_k_main), 0, 1, 1}, /* PyObject cname: __pyx_n_u_main */ + {__pyx_k_metaclass, sizeof(__pyx_k_metaclass), 0, 1, 1}, /* PyObject cname: __pyx_n_u_metaclass */ + {__pyx_k_module, sizeof(__pyx_k_module), 0, 1, 1}, /* PyObject cname: __pyx_n_u_module */ + {__pyx_k_mro_entries, sizeof(__pyx_k_mro_entries), 0, 1, 1}, /* PyObject cname: __pyx_n_u_mro_entries */ + {__pyx_k_name, sizeof(__pyx_k_name), 0, 1, 1}, /* PyObject cname: __pyx_n_u_name */ + {__pyx_k_prepare, sizeof(__pyx_k_prepare), 0, 1, 1}, /* PyObject cname: __pyx_n_u_prepare */ + {__pyx_k_qualname, sizeof(__pyx_k_qualname), 0, 1, 1}, /* PyObject cname: __pyx_n_u_qualname */ + {__pyx_k_range, sizeof(__pyx_k_range), 0, 1, 1}, /* PyObject cname: __pyx_n_u_range */ + {__pyx_k_test, sizeof(__pyx_k_test), 0, 1, 1}, /* PyObject cname: __pyx_n_u_test */ + {__pyx_k_writable, sizeof(__pyx_k_writable), 0, 1, 0}, /* PyObject cname: __pyx_kp_u_writable */ + {0, 0, 0, 0, 0} +}; +/* InitStrings.proto */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names); + +/* #### Code section: cached_builtins ### */ + +static int __Pyx_InitCachedBuiltins(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + __pyx_builtin_RuntimeError = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_RuntimeError); if (!__pyx_builtin_RuntimeError) __PYX_ERR(0, 127, __pyx_L1_error) + __pyx_builtin_AssertionError = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_AssertionError); if (!__pyx_builtin_AssertionError) __PYX_ERR(0, 38, __pyx_L1_error) + __pyx_builtin_ValueError = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_ValueError); if (!__pyx_builtin_ValueError) __PYX_ERR(0, 42, __pyx_L1_error) + __pyx_builtin_range = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_range); if (!__pyx_builtin_range) __PYX_ERR(0, 63, __pyx_L1_error) + __pyx_builtin_enumerate = __Pyx_GetBuiltinName(__pyx_mstate->__pyx_n_u_enumerate); if (!__pyx_builtin_enumerate) __PYX_ERR(0, 99, __pyx_L1_error) + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cached_constants ### */ + +static int __Pyx_InitCachedConstants(__pyx_mstatetype *__pyx_mstate) { + __Pyx_RefNannyDeclarations + CYTHON_UNUSED_VAR(__pyx_mstate); + __Pyx_RefNannySetupContext("__Pyx_InitCachedConstants", 0); + + /* "cuda/bindings/_internal/utils.pyx":127 + * + * + * class FunctionNotFoundError(RuntimeError): pass # <<<<<<<<<<<<<< + * + * class NotSupportedError(RuntimeError): pass +*/ + __pyx_mstate_global->__pyx_tuple[0] = PyTuple_Pack(1, __pyx_builtin_RuntimeError); if (unlikely(!__pyx_mstate_global->__pyx_tuple[0])) __PYX_ERR(0, 127, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[0]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[0]); + __pyx_mstate_global->__pyx_tuple[1] = PyTuple_Pack(1, __pyx_builtin_RuntimeError); if (unlikely(!__pyx_mstate_global->__pyx_tuple[1])) __PYX_ERR(0, 127, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[1]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[1]); + + /* "cuda/bindings/_internal/utils.pyx":129 + * class FunctionNotFoundError(RuntimeError): pass + * + * class NotSupportedError(RuntimeError): pass # <<<<<<<<<<<<<< +*/ + __pyx_mstate_global->__pyx_tuple[2] = PyTuple_Pack(1, __pyx_builtin_RuntimeError); if (unlikely(!__pyx_mstate_global->__pyx_tuple[2])) __PYX_ERR(0, 129, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[2]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[2]); + __pyx_mstate_global->__pyx_tuple[3] = PyTuple_Pack(1, __pyx_builtin_RuntimeError); if (unlikely(!__pyx_mstate_global->__pyx_tuple[3])) __PYX_ERR(0, 129, __pyx_L1_error) + __Pyx_GOTREF(__pyx_mstate_global->__pyx_tuple[3]); + __Pyx_GIVEREF(__pyx_mstate_global->__pyx_tuple[3]); + __Pyx_RefNannyFinishContext(); + return 0; + __pyx_L1_error:; + __Pyx_RefNannyFinishContext(); + return -1; +} +/* #### Code section: init_constants ### */ + +static int __Pyx_InitConstants(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + if (__Pyx_InitStrings(__pyx_string_tab, __pyx_mstate->__pyx_string_tab, __pyx_string_tab_encodings) < 0) __PYX_ERR(0, 1, __pyx_L1_error); + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: init_codeobjects ### */ + +static int __Pyx_CreateCodeObjects(__pyx_mstatetype *__pyx_mstate) { + CYTHON_UNUSED_VAR(__pyx_mstate); + return 0; +} +/* #### Code section: init_globals ### */ + +static int __Pyx_InitGlobals(void) { + /* PythonCompatibility.init */ + if (likely(__Pyx_init_co_variables() == 0)); else + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + /* AssertionsEnabled.init */ + if (likely(__Pyx_init_assertions_enabled() == 0)); else + +if (unlikely(PyErr_Occurred())) __PYX_ERR(0, 1, __pyx_L1_error) + + return 0; + __pyx_L1_error:; + return -1; +} +/* #### Code section: cleanup_globals ### */ +/* #### Code section: cleanup_module ### */ +/* #### Code section: main_method ### */ +/* #### Code section: utility_code_pragmas ### */ +#ifdef _MSC_VER +#pragma warning( push ) +/* Warning 4127: conditional expression is constant + * Cython uses constant conditional expressions to allow in inline functions to be optimized at + * compile-time, so this warning is not useful + */ +#pragma warning( disable : 4127 ) +#endif + + + +/* #### Code section: utility_code_def ### */ + +/* --- Runtime support code --- */ +/* Refnanny */ +#if CYTHON_REFNANNY +static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { + PyObject *m = NULL, *p = NULL; + void *r = NULL; + m = PyImport_ImportModule(modname); + if (!m) goto end; + p = PyObject_GetAttrString(m, "RefNannyAPI"); + if (!p) goto end; + r = PyLong_AsVoidPtr(p); +end: + Py_XDECREF(p); + Py_XDECREF(m); + return (__Pyx_RefNannyAPIStruct *)r; +} +#endif + +/* PyErrExceptionMatches */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx_PyErr_ExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030C00A6 + PyObject *current_exception = tstate->current_exception; + if (unlikely(!current_exception)) return 0; + exc_type = (PyObject*) Py_TYPE(current_exception); + if (exc_type == err) return 1; +#else + exc_type = tstate->curexc_type; + if (exc_type == err) return 1; + if (unlikely(!exc_type)) return 0; +#endif + #if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(exc_type); + #endif + if (unlikely(PyTuple_Check(err))) { + result = __Pyx_PyErr_ExceptionMatchesTuple(exc_type, err); + } else { + result = __Pyx_PyErr_GivenExceptionMatches(exc_type, err); + } + #if CYTHON_AVOID_BORROWED_REFS + Py_DECREF(exc_type); + #endif + return result; +} +#endif + +/* PyErrFetchRestore */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject *tmp_value; + assert(type == NULL || (value != NULL && type == (PyObject*) Py_TYPE(value))); + if (value) { + #if CYTHON_COMPILING_IN_CPYTHON + if (unlikely(((PyBaseExceptionObject*) value)->traceback != tb)) + #endif + PyException_SetTraceback(value, tb); + } + tmp_value = tstate->current_exception; + tstate->current_exception = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + tmp_type = tstate->curexc_type; + tmp_value = tstate->curexc_value; + tmp_tb = tstate->curexc_traceback; + tstate->curexc_type = type; + tstate->curexc_value = value; + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#endif +} +static CYTHON_INLINE void __Pyx_ErrFetchInState(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyObject* exc_value; + exc_value = tstate->current_exception; + tstate->current_exception = 0; + *value = exc_value; + *type = NULL; + *tb = NULL; + if (exc_value) { + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + #if CYTHON_COMPILING_IN_CPYTHON + *tb = ((PyBaseExceptionObject*) exc_value)->traceback; + Py_XINCREF(*tb); + #else + *tb = PyException_GetTraceback(exc_value); + #endif + } +#else + *type = tstate->curexc_type; + *value = tstate->curexc_value; + *tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; +#endif +} +#endif + +/* PyObjectGetAttrStr */ +#if CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro)) + return tp->tp_getattro(obj, attr_name); + return PyObject_GetAttr(obj, attr_name); +} +#endif + +/* PyObjectGetAttrStrNoError */ +#if __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static void __Pyx_PyObject_GetAttrStr_ClearAttributeError(void) { + __Pyx_PyThreadState_declare + __Pyx_PyThreadState_assign + if (likely(__Pyx_PyErr_ExceptionMatches(PyExc_AttributeError))) + __Pyx_PyErr_Clear(); +} +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStrNoError(PyObject* obj, PyObject* attr_name) { + PyObject *result; +#if __PYX_LIMITED_VERSION_HEX >= 0x030d0000 + (void) PyObject_GetOptionalAttr(obj, attr_name, &result); + return result; +#else +#if CYTHON_COMPILING_IN_CPYTHON && CYTHON_USE_TYPE_SLOTS + PyTypeObject* tp = Py_TYPE(obj); + if (likely(tp->tp_getattro == PyObject_GenericGetAttr)) { + return _PyObject_GenericGetAttrWithDict(obj, attr_name, NULL, 1); + } +#endif + result = __Pyx_PyObject_GetAttrStr(obj, attr_name); + if (unlikely(!result)) { + __Pyx_PyObject_GetAttrStr_ClearAttributeError(); + } + return result; +#endif +} + +/* GetBuiltinName */ +static PyObject *__Pyx_GetBuiltinName(PyObject *name) { + PyObject* result = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_b, name); + if (unlikely(!result) && !PyErr_Occurred()) { + PyErr_Format(PyExc_NameError, + "name '%U' is not defined", name); + } + return result; +} + +/* RaiseException */ +static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause) { + PyObject* owned_instance = NULL; + if (tb == Py_None) { + tb = 0; + } else if (tb && !PyTraceBack_Check(tb)) { + PyErr_SetString(PyExc_TypeError, + "raise: arg 3 must be a traceback or None"); + goto bad; + } + if (value == Py_None) + value = 0; + if (PyExceptionInstance_Check(type)) { + if (value) { + PyErr_SetString(PyExc_TypeError, + "instance exception may not have a separate value"); + goto bad; + } + value = type; + type = (PyObject*) Py_TYPE(value); + } else if (PyExceptionClass_Check(type)) { + PyObject *instance_class = NULL; + if (value && PyExceptionInstance_Check(value)) { + instance_class = (PyObject*) Py_TYPE(value); + if (instance_class != type) { + int is_subclass = PyObject_IsSubclass(instance_class, type); + if (!is_subclass) { + instance_class = NULL; + } else if (unlikely(is_subclass == -1)) { + goto bad; + } else { + type = instance_class; + } + } + } + if (!instance_class) { + PyObject *args; + if (!value) + args = PyTuple_New(0); + else if (PyTuple_Check(value)) { + Py_INCREF(value); + args = value; + } else + args = PyTuple_Pack(1, value); + if (!args) + goto bad; + owned_instance = PyObject_Call(type, args, NULL); + Py_DECREF(args); + if (!owned_instance) + goto bad; + value = owned_instance; + if (!PyExceptionInstance_Check(value)) { + PyErr_Format(PyExc_TypeError, + "calling %R should have returned an instance of " + "BaseException, not %R", + type, Py_TYPE(value)); + goto bad; + } + } + } else { + PyErr_SetString(PyExc_TypeError, + "raise: exception class must be a subclass of BaseException"); + goto bad; + } + if (cause) { + PyObject *fixed_cause; + if (cause == Py_None) { + fixed_cause = NULL; + } else if (PyExceptionClass_Check(cause)) { + fixed_cause = PyObject_CallObject(cause, NULL); + if (fixed_cause == NULL) + goto bad; + } else if (PyExceptionInstance_Check(cause)) { + fixed_cause = cause; + Py_INCREF(fixed_cause); + } else { + PyErr_SetString(PyExc_TypeError, + "exception causes must derive from " + "BaseException"); + goto bad; + } + PyException_SetCause(value, fixed_cause); + } + PyErr_SetObject(type, value); + if (tb) { +#if PY_VERSION_HEX >= 0x030C00A6 + PyException_SetTraceback(value, tb); +#elif CYTHON_FAST_THREAD_STATE + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject* tmp_tb = tstate->curexc_traceback; + if (tb != tmp_tb) { + Py_INCREF(tb); + tstate->curexc_traceback = tb; + Py_XDECREF(tmp_tb); + } +#else + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); + Py_INCREF(tb); + PyErr_Restore(tmp_type, tmp_value, tb); + Py_XDECREF(tmp_tb); +#endif + } +bad: + Py_XDECREF(owned_instance); + return; +} + +/* GetTopmostException */ +#if CYTHON_USE_EXC_INFO_STACK && CYTHON_FAST_THREAD_STATE +static _PyErr_StackItem * +__Pyx_PyErr_GetTopmostException(PyThreadState *tstate) +{ + _PyErr_StackItem *exc_info = tstate->exc_info; + while ((exc_info->exc_value == NULL || exc_info->exc_value == Py_None) && + exc_info->previous_item != NULL) + { + exc_info = exc_info->previous_item; + } + return exc_info; +} +#endif + +/* SaveResetException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + PyObject *exc_value = exc_info->exc_value; + if (exc_value == NULL || exc_value == Py_None) { + *value = NULL; + *type = NULL; + *tb = NULL; + } else { + *value = exc_value; + Py_INCREF(*value); + *type = (PyObject*) Py_TYPE(exc_value); + Py_INCREF(*type); + *tb = PyException_GetTraceback(exc_value); + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = __Pyx_PyErr_GetTopmostException(tstate); + *type = exc_info->exc_type; + *value = exc_info->exc_value; + *tb = exc_info->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #else + *type = tstate->exc_type; + *value = tstate->exc_value; + *tb = tstate->exc_traceback; + Py_XINCREF(*type); + Py_XINCREF(*value); + Py_XINCREF(*tb); + #endif +} +static CYTHON_INLINE void __Pyx__ExceptionReset(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + PyObject *tmp_value = exc_info->exc_value; + exc_info->exc_value = value; + Py_XDECREF(tmp_value); + Py_XDECREF(type); + Py_XDECREF(tb); + #else + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = type; + exc_info->exc_value = value; + exc_info->exc_traceback = tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = type; + tstate->exc_value = value; + tstate->exc_traceback = tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); + #endif +} +#endif + +/* GetException */ +#if CYTHON_FAST_THREAD_STATE +static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) +#else +static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) +#endif +{ + PyObject *local_type = NULL, *local_value, *local_tb = NULL; +#if CYTHON_FAST_THREAD_STATE + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if PY_VERSION_HEX >= 0x030C0000 + local_value = tstate->current_exception; + tstate->current_exception = 0; + #else + local_type = tstate->curexc_type; + local_value = tstate->curexc_value; + local_tb = tstate->curexc_traceback; + tstate->curexc_type = 0; + tstate->curexc_value = 0; + tstate->curexc_traceback = 0; + #endif +#elif __PYX_LIMITED_VERSION_HEX > 0x030C0000 + local_value = PyErr_GetRaisedException(); +#else + PyErr_Fetch(&local_type, &local_value, &local_tb); +#endif +#if __PYX_LIMITED_VERSION_HEX > 0x030C0000 + if (likely(local_value)) { + local_type = (PyObject*) Py_TYPE(local_value); + Py_INCREF(local_type); + local_tb = PyException_GetTraceback(local_value); + } +#else + PyErr_NormalizeException(&local_type, &local_value, &local_tb); +#if CYTHON_FAST_THREAD_STATE + if (unlikely(tstate->curexc_type)) +#else + if (unlikely(PyErr_Occurred())) +#endif + goto bad; + if (local_tb) { + if (unlikely(PyException_SetTraceback(local_value, local_tb) < 0)) + goto bad; + } +#endif // __PYX_LIMITED_VERSION_HEX > 0x030C0000 + Py_XINCREF(local_tb); + Py_XINCREF(local_type); + Py_XINCREF(local_value); + *type = local_type; + *value = local_value; + *tb = local_tb; +#if CYTHON_FAST_THREAD_STATE + #if CYTHON_USE_EXC_INFO_STACK + { + _PyErr_StackItem *exc_info = tstate->exc_info; + #if PY_VERSION_HEX >= 0x030B00a4 + tmp_value = exc_info->exc_value; + exc_info->exc_value = local_value; + tmp_type = NULL; + tmp_tb = NULL; + Py_XDECREF(local_type); + Py_XDECREF(local_tb); + #else + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = local_type; + exc_info->exc_value = local_value; + exc_info->exc_traceback = local_tb; + #endif + } + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = local_type; + tstate->exc_value = local_value; + tstate->exc_traceback = local_tb; + #endif + Py_XDECREF(tmp_type); + Py_XDECREF(tmp_value); + Py_XDECREF(tmp_tb); +#elif __PYX_LIMITED_VERSION_HEX >= 0x030b0000 + PyErr_SetHandledException(local_value); + Py_XDECREF(local_value); + Py_XDECREF(local_type); + Py_XDECREF(local_tb); +#else + PyErr_SetExcInfo(local_type, local_value, local_tb); +#endif + return 0; +#if __PYX_LIMITED_VERSION_HEX <= 0x030C0000 +bad: + *type = 0; + *value = 0; + *tb = 0; + Py_XDECREF(local_type); + Py_XDECREF(local_value); + Py_XDECREF(local_tb); + return -1; +#endif +} + +/* PyUnicode_Unicode */ +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_Unicode(PyObject *obj) { + if (unlikely(obj == Py_None)) + obj = __pyx_mstate_global->__pyx_kp_u_None; + return __Pyx_NewRef(obj); +} + +/* JoinPyUnicode */ +static PyObject* __Pyx_PyUnicode_Join(PyObject** values, Py_ssize_t value_count, Py_ssize_t result_ulength, + Py_UCS4 max_char) { +#if CYTHON_USE_UNICODE_INTERNALS && CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + PyObject *result_uval; + int result_ukind, kind_shift; + Py_ssize_t i, char_pos; + void *result_udata; + if (max_char > 1114111) max_char = 1114111; + result_uval = PyUnicode_New(result_ulength, max_char); + if (unlikely(!result_uval)) return NULL; + result_ukind = (max_char <= 255) ? PyUnicode_1BYTE_KIND : (max_char <= 65535) ? PyUnicode_2BYTE_KIND : PyUnicode_4BYTE_KIND; + kind_shift = (result_ukind == PyUnicode_4BYTE_KIND) ? 2 : result_ukind - 1; + result_udata = PyUnicode_DATA(result_uval); + assert(kind_shift == 2 || kind_shift == 1 || kind_shift == 0); + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - result_ulength < 0)) + goto overflow; + char_pos = 0; + for (i=0; i < value_count; i++) { + int ukind; + Py_ssize_t ulength; + void *udata; + PyObject *uval = values[i]; + #if !CYTHON_COMPILING_IN_LIMITED_API + if (__Pyx_PyUnicode_READY(uval) == (-1)) + goto bad; + #endif + ulength = __Pyx_PyUnicode_GET_LENGTH(uval); + #if !CYTHON_ASSUME_SAFE_SIZE + if (unlikely(ulength < 0)) goto bad; + #endif + if (unlikely(!ulength)) + continue; + if (unlikely((PY_SSIZE_T_MAX >> kind_shift) - ulength < char_pos)) + goto overflow; + ukind = __Pyx_PyUnicode_KIND(uval); + udata = __Pyx_PyUnicode_DATA(uval); + if (ukind == result_ukind) { + memcpy((char *)result_udata + (char_pos << kind_shift), udata, (size_t) (ulength << kind_shift)); + } else { + #if PY_VERSION_HEX >= 0x030d0000 + if (unlikely(PyUnicode_CopyCharacters(result_uval, char_pos, uval, 0, ulength) < 0)) goto bad; + #elif CYTHON_COMPILING_IN_CPYTHON || defined(_PyUnicode_FastCopyCharacters) + _PyUnicode_FastCopyCharacters(result_uval, char_pos, uval, 0, ulength); + #else + Py_ssize_t j; + for (j=0; j < ulength; j++) { + Py_UCS4 uchar = __Pyx_PyUnicode_READ(ukind, udata, j); + __Pyx_PyUnicode_WRITE(result_ukind, result_udata, char_pos+j, uchar); + } + #endif + } + char_pos += ulength; + } + return result_uval; +overflow: + PyErr_SetString(PyExc_OverflowError, "join() result is too long for a Python string"); +bad: + Py_DECREF(result_uval); + return NULL; +#else + Py_ssize_t i; + PyObject *result = NULL; + PyObject *value_tuple = PyTuple_New(value_count); + if (unlikely(!value_tuple)) return NULL; + CYTHON_UNUSED_VAR(max_char); + CYTHON_UNUSED_VAR(result_ulength); + for (i=0; i__pyx_empty_unicode, value_tuple); +bad: + Py_DECREF(value_tuple); + return result; +#endif +} + +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL && !CYTHON_VECTORCALL +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject *const *args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = __Pyx_PyFrame_GetLocalsplus(f); + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; + PyObject *kwdefs; + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) { + return NULL; + } + if ( + co->co_kwonlyargcount == 0 && + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, (int)nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif + +/* PyObjectCall */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { + PyObject *result; + ternaryfunc call = Py_TYPE(func)->tp_call; + if (unlikely(!call)) + return PyObject_Call(func, arg, kw); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = (*call)(func, arg, kw); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectCallMethO */ +#if CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { + PyObject *self, *result; + PyCFunction cfunc; + cfunc = __Pyx_CyOrPyCFunction_GET_FUNCTION(func); + self = __Pyx_CyOrPyCFunction_GET_SELF(func); + if (unlikely(Py_EnterRecursiveCall(" while calling a Python object"))) + return NULL; + result = cfunc(self, arg); + Py_LeaveRecursiveCall(); + if (unlikely(!result) && unlikely(!PyErr_Occurred())) { + PyErr_SetString( + PyExc_SystemError, + "NULL result without error in PyObject_Call"); + } + return result; +} +#endif + +/* PyObjectFastCall */ +#if PY_VERSION_HEX < 0x03090000 || CYTHON_COMPILING_IN_LIMITED_API +static PyObject* __Pyx_PyObject_FastCall_fallback(PyObject *func, PyObject * const*args, size_t nargs, PyObject *kwargs) { + PyObject *argstuple; + PyObject *result = 0; + size_t i; + argstuple = PyTuple_New((Py_ssize_t)nargs); + if (unlikely(!argstuple)) return NULL; + for (i = 0; i < nargs; i++) { + Py_INCREF(args[i]); + if (__Pyx_PyTuple_SET_ITEM(argstuple, (Py_ssize_t)i, args[i]) != (0)) goto bad; + } + result = __Pyx_PyObject_Call(func, argstuple, kwargs); + bad: + Py_DECREF(argstuple); + return result; +} +#endif +#if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + #if PY_VERSION_HEX < 0x03090000 + #define __Pyx_PyVectorcall_Function(callable) _PyVectorcall_Function(callable) + #elif CYTHON_COMPILING_IN_CPYTHON +static CYTHON_INLINE vectorcallfunc __Pyx_PyVectorcall_Function(PyObject *callable) { + PyTypeObject *tp = Py_TYPE(callable); + #if defined(__Pyx_CyFunction_USED) + if (__Pyx_CyFunction_CheckExact(callable)) { + return __Pyx_CyFunction_func_vectorcall(callable); + } + #endif + if (!PyType_HasFeature(tp, Py_TPFLAGS_HAVE_VECTORCALL)) { + return NULL; + } + assert(PyCallable_Check(callable)); + Py_ssize_t offset = tp->tp_vectorcall_offset; + assert(offset > 0); + vectorcallfunc ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; +} + #else + #define __Pyx_PyVectorcall_Function(callable) PyVectorcall_Function(callable) + #endif +#endif +static CYTHON_INLINE PyObject* __Pyx_PyObject_FastCallDict(PyObject *func, PyObject *const *args, size_t _nargs, PyObject *kwargs) { + Py_ssize_t nargs = __Pyx_PyVectorcall_NARGS(_nargs); +#if CYTHON_COMPILING_IN_CPYTHON + if (nargs == 0 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_NOARGS)) + return __Pyx_PyObject_CallMethO(func, NULL); + } + else if (nargs == 1 && kwargs == NULL) { + if (__Pyx_CyOrPyCFunction_Check(func) && likely( __Pyx_CyOrPyCFunction_GET_FLAGS(func) & METH_O)) + return __Pyx_PyObject_CallMethO(func, args[0]); + } +#endif + #if PY_VERSION_HEX < 0x030800B1 + #if CYTHON_FAST_PYCCALL + if (PyCFunction_Check(func)) { + if (kwargs) { + return _PyCFunction_FastCallDict(func, args, nargs, kwargs); + } else { + return _PyCFunction_FastCallKeywords(func, args, nargs, NULL); + } + } + if (!kwargs && __Pyx_IS_TYPE(func, &PyMethodDescr_Type)) { + return _PyMethodDescr_FastCallKeywords(func, args, nargs, NULL); + } + #endif + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs); + } + #endif + #endif + if (kwargs == NULL) { + #if CYTHON_VECTORCALL && !CYTHON_COMPILING_IN_LIMITED_API + vectorcallfunc f = __Pyx_PyVectorcall_Function(func); + if (f) { + return f(func, args, _nargs, NULL); + } + #elif defined(__Pyx_CyFunction_USED) && CYTHON_BACKPORT_VECTORCALL + if (__Pyx_CyFunction_CheckExact(func)) { + __pyx_vectorcallfunc f = __Pyx_CyFunction_func_vectorcall(func); + if (f) return f(func, args, _nargs, NULL); + } + #elif CYTHON_COMPILING_IN_LIMITED_API && CYTHON_VECTORCALL + return PyObject_Vectorcall(func, args, _nargs, NULL); + #endif + } + if (nargs == 0) { + return __Pyx_PyObject_Call(func, __pyx_mstate_global->__pyx_empty_tuple, kwargs); + } + #if PY_VERSION_HEX >= 0x03090000 && !CYTHON_COMPILING_IN_LIMITED_API + return PyObject_VectorcallDict(func, args, (size_t)nargs, kwargs); + #else + return __Pyx_PyObject_FastCall_fallback(func, args, (size_t)nargs, kwargs); + #endif +} + +/* SwapException */ +#if CYTHON_FAST_THREAD_STATE +static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + #if CYTHON_USE_EXC_INFO_STACK && PY_VERSION_HEX >= 0x030B00a4 + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_value = exc_info->exc_value; + exc_info->exc_value = *value; + if (tmp_value == NULL || tmp_value == Py_None) { + Py_XDECREF(tmp_value); + tmp_value = NULL; + tmp_type = NULL; + tmp_tb = NULL; + } else { + tmp_type = (PyObject*) Py_TYPE(tmp_value); + Py_INCREF(tmp_type); + #if CYTHON_COMPILING_IN_CPYTHON + tmp_tb = ((PyBaseExceptionObject*) tmp_value)->traceback; + Py_XINCREF(tmp_tb); + #else + tmp_tb = PyException_GetTraceback(tmp_value); + #endif + } + #elif CYTHON_USE_EXC_INFO_STACK + _PyErr_StackItem *exc_info = tstate->exc_info; + tmp_type = exc_info->exc_type; + tmp_value = exc_info->exc_value; + tmp_tb = exc_info->exc_traceback; + exc_info->exc_type = *type; + exc_info->exc_value = *value; + exc_info->exc_traceback = *tb; + #else + tmp_type = tstate->exc_type; + tmp_value = tstate->exc_value; + tmp_tb = tstate->exc_traceback; + tstate->exc_type = *type; + tstate->exc_value = *value; + tstate->exc_traceback = *tb; + #endif + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#else +static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb) { + PyObject *tmp_type, *tmp_value, *tmp_tb; + PyErr_GetExcInfo(&tmp_type, &tmp_value, &tmp_tb); + PyErr_SetExcInfo(*type, *value, *tb); + *type = tmp_type; + *value = tmp_value; + *tb = tmp_tb; +} +#endif + +/* GetItemInt */ +static PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { + PyObject *r; + if (unlikely(!j)) return NULL; + r = PyObject_GetItem(o, j); + Py_DECREF(j); + return r; +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS && !CYTHON_AVOID_THREAD_UNSAFE_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyList_GET_SIZE(o); + } + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyList_GET_SIZE(o)))) { + PyObject *r = PyList_GET_ITEM(o, wrapped_i); + Py_INCREF(r); + return r; + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS + Py_ssize_t wrapped_i = i; + if (wraparound & unlikely(i < 0)) { + wrapped_i += PyTuple_GET_SIZE(o); + } + if ((!boundscheck) || likely(__Pyx_is_valid_index(wrapped_i, PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, wrapped_i); + Py_INCREF(r); + return r; + } + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +#else + return PySequence_GetItem(o, i); +#endif +} +static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, + CYTHON_NCP_UNUSED int wraparound, + CYTHON_NCP_UNUSED int boundscheck) { +#if CYTHON_ASSUME_SAFE_MACROS && CYTHON_ASSUME_SAFE_SIZE && !CYTHON_AVOID_BORROWED_REFS && CYTHON_USE_TYPE_SLOTS + if (is_list || PyList_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); + if ((!boundscheck) || (likely(__Pyx_is_valid_index(n, PyList_GET_SIZE(o))))) { + return __Pyx_PyList_GetItemRef(o, n); + } + } + else if (PyTuple_CheckExact(o)) { + Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyTuple_GET_SIZE(o); + if ((!boundscheck) || likely(__Pyx_is_valid_index(n, PyTuple_GET_SIZE(o)))) { + PyObject *r = PyTuple_GET_ITEM(o, n); + Py_INCREF(r); + return r; + } + } else { + PyMappingMethods *mm = Py_TYPE(o)->tp_as_mapping; + PySequenceMethods *sm = Py_TYPE(o)->tp_as_sequence; + if (mm && mm->mp_subscript) { + PyObject *r, *key = PyLong_FromSsize_t(i); + if (unlikely(!key)) return NULL; + r = mm->mp_subscript(o, key); + Py_DECREF(key); + return r; + } + if (likely(sm && sm->sq_item)) { + if (wraparound && unlikely(i < 0) && likely(sm->sq_length)) { + Py_ssize_t l = sm->sq_length(o); + if (likely(l >= 0)) { + i += l; + } else { + if (!PyErr_ExceptionMatches(PyExc_OverflowError)) + return NULL; + PyErr_Clear(); + } + } + return sm->sq_item(o, i); + } + } +#else + if (is_list || !PyMapping_Check(o)) { + return PySequence_GetItem(o, i); + } +#endif + return __Pyx_GetItemInt_Generic(o, PyLong_FromSsize_t(i)); +} + +/* RaiseUnexpectedTypeError */ +static int +__Pyx_RaiseUnexpectedTypeError(const char *expected, PyObject *obj) +{ + __Pyx_TypeName obj_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(obj)); + PyErr_Format(PyExc_TypeError, "Expected %s, got " __Pyx_FMT_TYPENAME, + expected, obj_type_name); + __Pyx_DECREF_TypeName(obj_type_name); + return 0; +} + +/* TypeImport */ +#ifndef __PYX_HAVE_RT_ImportType_3_1_5 +#define __PYX_HAVE_RT_ImportType_3_1_5 +static PyTypeObject *__Pyx_ImportType_3_1_5(PyObject *module, const char *module_name, const char *class_name, + size_t size, size_t alignment, enum __Pyx_ImportType_CheckSize_3_1_5 check_size) +{ + PyObject *result = 0; + Py_ssize_t basicsize; + Py_ssize_t itemsize; +#if defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API) + PyObject *py_basicsize; + PyObject *py_itemsize; +#endif + result = PyObject_GetAttrString(module, class_name); + if (!result) + goto bad; + if (!PyType_Check(result)) { + PyErr_Format(PyExc_TypeError, + "%.200s.%.200s is not a type object", + module_name, class_name); + goto bad; + } +#if !( defined(Py_LIMITED_API) || (defined(CYTHON_COMPILING_IN_LIMITED_API) && CYTHON_COMPILING_IN_LIMITED_API) ) + basicsize = ((PyTypeObject *)result)->tp_basicsize; + itemsize = ((PyTypeObject *)result)->tp_itemsize; +#else + if (size == 0) { + return (PyTypeObject *)result; + } + py_basicsize = PyObject_GetAttrString(result, "__basicsize__"); + if (!py_basicsize) + goto bad; + basicsize = PyLong_AsSsize_t(py_basicsize); + Py_DECREF(py_basicsize); + py_basicsize = 0; + if (basicsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; + py_itemsize = PyObject_GetAttrString(result, "__itemsize__"); + if (!py_itemsize) + goto bad; + itemsize = PyLong_AsSsize_t(py_itemsize); + Py_DECREF(py_itemsize); + py_itemsize = 0; + if (itemsize == (Py_ssize_t)-1 && PyErr_Occurred()) + goto bad; +#endif + if (itemsize) { + if (size % alignment) { + alignment = size % alignment; + } + if (itemsize < (Py_ssize_t)alignment) + itemsize = (Py_ssize_t)alignment; + } + if ((size_t)(basicsize + itemsize) < size) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize+itemsize); + goto bad; + } + if (check_size == __Pyx_ImportType_CheckSize_Error_3_1_5 && + ((size_t)basicsize > size || (size_t)(basicsize + itemsize) < size)) { + PyErr_Format(PyExc_ValueError, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd-%zd from PyObject", + module_name, class_name, size, basicsize, basicsize+itemsize); + goto bad; + } + else if (check_size == __Pyx_ImportType_CheckSize_Warn_3_1_5 && (size_t)basicsize > size) { + if (PyErr_WarnFormat(NULL, 0, + "%.200s.%.200s size changed, may indicate binary incompatibility. " + "Expected %zd from C header, got %zd from PyObject", + module_name, class_name, size, basicsize) < 0) { + goto bad; + } + } + return (PyTypeObject *)result; +bad: + Py_XDECREF(result); + return NULL; +} +#endif + +/* PyObjectCallOneArg */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { + PyObject *args[2] = {NULL, arg}; + return __Pyx_PyObject_FastCall(func, args+1, 1 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* Py3UpdateBases */ +static PyObject* +__Pyx_PEP560_update_bases(PyObject *bases) +{ + Py_ssize_t i, j, size_bases; + PyObject *base = NULL, *meth, *new_base, *result, *new_bases = NULL; +#if CYTHON_ASSUME_SAFE_SIZE + size_bases = PyTuple_GET_SIZE(bases); +#else + size_bases = PyTuple_Size(bases); + if (size_bases < 0) return NULL; +#endif + for (i = 0; i < size_bases; i++) { +#if CYTHON_AVOID_BORROWED_REFS + Py_CLEAR(base); +#endif +#if CYTHON_ASSUME_SAFE_MACROS + base = PyTuple_GET_ITEM(bases, i); +#else + base = PyTuple_GetItem(bases, i); + if (!base) goto error; +#endif +#if CYTHON_AVOID_BORROWED_REFS + Py_INCREF(base); +#endif + if (PyType_Check(base)) { + if (new_bases) { + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + meth = __Pyx_PyObject_GetAttrStrNoError(base, __pyx_mstate_global->__pyx_n_u_mro_entries); + if (!meth && PyErr_Occurred()) { + goto error; + } + if (!meth) { + if (new_bases) { + if (PyList_Append(new_bases, base) < 0) { + goto error; + } + } + continue; + } + new_base = __Pyx_PyObject_CallOneArg(meth, bases); + Py_DECREF(meth); + if (!new_base) { + goto error; + } + if (!PyTuple_Check(new_base)) { + PyErr_SetString(PyExc_TypeError, + "__mro_entries__ must return a tuple"); + Py_DECREF(new_base); + goto error; + } + if (!new_bases) { + if (!(new_bases = PyList_New(i))) { + goto error; + } + for (j = 0; j < i; j++) { + PyObject *base_from_list; +#if CYTHON_ASSUME_SAFE_MACROS + base_from_list = PyTuple_GET_ITEM(bases, j); + PyList_SET_ITEM(new_bases, j, base_from_list); + Py_INCREF(base_from_list); +#else + base_from_list = PyTuple_GetItem(bases, j); + if (!base_from_list) goto error; + Py_INCREF(base_from_list); + if (PyList_SetItem(new_bases, j, base_from_list) < 0) goto error; +#endif + } + } +#if CYTHON_ASSUME_SAFE_SIZE + j = PyList_GET_SIZE(new_bases); +#else + j = PyList_Size(new_bases); + if (j < 0) goto error; +#endif + if (PyList_SetSlice(new_bases, j, j, new_base) < 0) { + goto error; + } + Py_DECREF(new_base); + } + if (!new_bases) { + Py_INCREF(bases); + return bases; + } + result = PyList_AsTuple(new_bases); + Py_DECREF(new_bases); +#if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(base); +#endif + return result; +error: + Py_XDECREF(new_bases); +#if CYTHON_AVOID_BORROWED_REFS + Py_XDECREF(base); +#endif + return NULL; +} + +/* CalculateMetaclass */ +static PyObject *__Pyx_CalculateMetaclass(PyTypeObject *metaclass, PyObject *bases) { + Py_ssize_t i, nbases; +#if CYTHON_ASSUME_SAFE_SIZE + nbases = PyTuple_GET_SIZE(bases); +#else + nbases = PyTuple_Size(bases); + if (nbases < 0) return NULL; +#endif + for (i=0; i < nbases; i++) { + PyTypeObject *tmptype; +#if CYTHON_ASSUME_SAFE_MACROS + PyObject *tmp = PyTuple_GET_ITEM(bases, i); +#else + PyObject *tmp = PyTuple_GetItem(bases, i); + if (!tmp) return NULL; +#endif + tmptype = Py_TYPE(tmp); + if (!metaclass) { + metaclass = tmptype; + continue; + } + if (PyType_IsSubtype(metaclass, tmptype)) + continue; + if (PyType_IsSubtype(tmptype, metaclass)) { + metaclass = tmptype; + continue; + } + PyErr_SetString(PyExc_TypeError, + "metaclass conflict: " + "the metaclass of a derived class " + "must be a (non-strict) subclass " + "of the metaclasses of all its bases"); + return NULL; + } + if (!metaclass) { + metaclass = &PyType_Type; + } + Py_INCREF((PyObject*) metaclass); + return (PyObject*) metaclass; +} + +/* PyObjectCall2Args */ +static CYTHON_INLINE PyObject* __Pyx_PyObject_Call2Args(PyObject* function, PyObject* arg1, PyObject* arg2) { + PyObject *args[3] = {NULL, arg1, arg2}; + return __Pyx_PyObject_FastCall(function, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET); +} + +/* PyObjectLookupSpecial */ +#if CYTHON_USE_PYTYPE_LOOKUP && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PyObject* __Pyx__PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name, int with_error) { + PyObject *res; + PyTypeObject *tp = Py_TYPE(obj); + res = _PyType_Lookup(tp, attr_name); + if (likely(res)) { + descrgetfunc f = Py_TYPE(res)->tp_descr_get; + if (!f) { + Py_INCREF(res); + } else { + res = f(res, obj, (PyObject *)tp); + } + } else if (with_error) { + PyErr_SetObject(PyExc_AttributeError, attr_name); + } + return res; +} +#endif + +/* Py3ClassCreate */ +static PyObject *__Pyx_Py3MetaclassPrepare(PyObject *metaclass, PyObject *bases, PyObject *name, + PyObject *qualname, PyObject *mkw, PyObject *modname, PyObject *doc) { + PyObject *ns; + if (metaclass) { + PyObject *prep = __Pyx_PyObject_GetAttrStrNoError(metaclass, __pyx_mstate_global->__pyx_n_u_prepare); + if (prep) { + PyObject *pargs[3] = {NULL, name, bases}; + ns = __Pyx_PyObject_FastCallDict(prep, pargs+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, mkw); + Py_DECREF(prep); + } else { + if (unlikely(PyErr_Occurred())) + return NULL; + ns = PyDict_New(); + } + } else { + ns = PyDict_New(); + } + if (unlikely(!ns)) + return NULL; + if (unlikely(PyObject_SetItem(ns, __pyx_mstate_global->__pyx_n_u_module, modname) < 0)) goto bad; + if (unlikely(PyObject_SetItem(ns, __pyx_mstate_global->__pyx_n_u_qualname, qualname) < 0)) goto bad; + if (unlikely(doc && PyObject_SetItem(ns, __pyx_mstate_global->__pyx_n_u_doc, doc) < 0)) goto bad; + return ns; +bad: + Py_DECREF(ns); + return NULL; +} +static PyObject *__Pyx_Py3ClassCreate(PyObject *metaclass, PyObject *name, PyObject *bases, + PyObject *dict, PyObject *mkw, + int calculate_metaclass, int allow_py2_metaclass) { + PyObject *result; + PyObject *owned_metaclass = NULL; + PyObject *margs[4] = {NULL, name, bases, dict}; + if (allow_py2_metaclass) { + owned_metaclass = PyObject_GetItem(dict, __pyx_mstate_global->__pyx_n_u_metaclass); + if (owned_metaclass) { + metaclass = owned_metaclass; + } else if (likely(PyErr_ExceptionMatches(PyExc_KeyError))) { + PyErr_Clear(); + } else { + return NULL; + } + } + if (calculate_metaclass && (!metaclass || PyType_Check(metaclass))) { + metaclass = __Pyx_CalculateMetaclass((PyTypeObject*) metaclass, bases); + Py_XDECREF(owned_metaclass); + if (unlikely(!metaclass)) + return NULL; + owned_metaclass = metaclass; + } + result = __Pyx_PyObject_FastCallDict(metaclass, margs+1, 3 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, mkw); + Py_XDECREF(owned_metaclass); + return result; +} + +/* PyDictVersioning */ +#if CYTHON_USE_DICT_VERSIONS && CYTHON_USE_TYPE_SLOTS +static CYTHON_INLINE PY_UINT64_T __Pyx_get_tp_dict_version(PyObject *obj) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + return likely(dict) ? __PYX_GET_DICT_VERSION(dict) : 0; +} +static CYTHON_INLINE PY_UINT64_T __Pyx_get_object_dict_version(PyObject *obj) { + PyObject **dictptr = NULL; + Py_ssize_t offset = Py_TYPE(obj)->tp_dictoffset; + if (offset) { +#if CYTHON_COMPILING_IN_CPYTHON + dictptr = (likely(offset > 0)) ? (PyObject **) ((char *)obj + offset) : _PyObject_GetDictPtr(obj); +#else + dictptr = _PyObject_GetDictPtr(obj); +#endif + } + return (dictptr && *dictptr) ? __PYX_GET_DICT_VERSION(*dictptr) : 0; +} +static CYTHON_INLINE int __Pyx_object_dict_version_matches(PyObject* obj, PY_UINT64_T tp_dict_version, PY_UINT64_T obj_dict_version) { + PyObject *dict = Py_TYPE(obj)->tp_dict; + if (unlikely(!dict) || unlikely(tp_dict_version != __PYX_GET_DICT_VERSION(dict))) + return 0; + return obj_dict_version == __Pyx_get_object_dict_version(obj); +} +#endif + +/* CLineInTraceback */ +#if CYTHON_CLINE_IN_TRACEBACK && CYTHON_CLINE_IN_TRACEBACK_RUNTIME +static int __Pyx_CLineForTraceback(PyThreadState *tstate, int c_line) { + PyObject *use_cline; + PyObject *ptype, *pvalue, *ptraceback; +#if CYTHON_COMPILING_IN_CPYTHON + PyObject **cython_runtime_dict; +#endif + CYTHON_MAYBE_UNUSED_VAR(tstate); + if (unlikely(!__pyx_mstate_global->__pyx_cython_runtime)) { + return c_line; + } + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); +#if CYTHON_COMPILING_IN_CPYTHON + cython_runtime_dict = _PyObject_GetDictPtr(__pyx_mstate_global->__pyx_cython_runtime); + if (likely(cython_runtime_dict)) { + __Pyx_BEGIN_CRITICAL_SECTION(*cython_runtime_dict); + __PYX_PY_DICT_LOOKUP_IF_MODIFIED( + use_cline, *cython_runtime_dict, + __Pyx_PyDict_GetItemStr(*cython_runtime_dict, __pyx_mstate_global->__pyx_n_u_cline_in_traceback)) + Py_XINCREF(use_cline); + __Pyx_END_CRITICAL_SECTION(); + } else +#endif + { + PyObject *use_cline_obj = __Pyx_PyObject_GetAttrStrNoError(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback); + if (use_cline_obj) { + use_cline = PyObject_Not(use_cline_obj) ? Py_False : Py_True; + Py_INCREF(use_cline); + Py_DECREF(use_cline_obj); + } else { + PyErr_Clear(); + use_cline = NULL; + } + } + if (!use_cline) { + c_line = 0; + (void) PyObject_SetAttr(__pyx_mstate_global->__pyx_cython_runtime, __pyx_mstate_global->__pyx_n_u_cline_in_traceback, Py_False); + } + else if (use_cline == Py_False || (use_cline != Py_True && PyObject_Not(use_cline) != 0)) { + c_line = 0; + } + Py_XDECREF(use_cline); + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + return c_line; +} +#endif + +/* CodeObjectCache */ +static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { + int start = 0, mid = 0, end = count - 1; + if (end >= 0 && code_line > entries[end].code_line) { + return count; + } + while (start < end) { + mid = start + (end - start) / 2; + if (code_line < entries[mid].code_line) { + end = mid; + } else if (code_line > entries[mid].code_line) { + start = mid + 1; + } else { + return mid; + } + } + if (code_line <= entries[mid].code_line) { + return mid; + } else { + return mid + 1; + } +} +static __Pyx_CachedCodeObjectType *__pyx__find_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line) { + __Pyx_CachedCodeObjectType* code_object; + int pos; + if (unlikely(!code_line) || unlikely(!code_cache->entries)) { + return NULL; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if (unlikely(pos >= code_cache->count) || unlikely(code_cache->entries[pos].code_line != code_line)) { + return NULL; + } + code_object = code_cache->entries[pos].code_object; + Py_INCREF(code_object); + return code_object; +} +static __Pyx_CachedCodeObjectType *__pyx_find_code_object(int code_line) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__find_code_object; + return NULL; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just miss. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type old_count = __pyx_atomic_incr_acq_rel(&code_cache->accessor_count); + if (old_count < 0) { + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); + return NULL; + } +#endif + __Pyx_CachedCodeObjectType *result = __pyx__find_code_object(code_cache, code_line); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_decr_acq_rel(&code_cache->accessor_count); +#endif + return result; +#endif +} +static void __pyx__insert_code_object(struct __Pyx_CodeObjectCache *code_cache, int code_line, __Pyx_CachedCodeObjectType* code_object) +{ + int pos, i; + __Pyx_CodeObjectCacheEntry* entries = code_cache->entries; + if (unlikely(!code_line)) { + return; + } + if (unlikely(!entries)) { + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Malloc(64*sizeof(__Pyx_CodeObjectCacheEntry)); + if (likely(entries)) { + code_cache->entries = entries; + code_cache->max_count = 64; + code_cache->count = 1; + entries[0].code_line = code_line; + entries[0].code_object = code_object; + Py_INCREF(code_object); + } + return; + } + pos = __pyx_bisect_code_objects(code_cache->entries, code_cache->count, code_line); + if ((pos < code_cache->count) && unlikely(code_cache->entries[pos].code_line == code_line)) { + __Pyx_CachedCodeObjectType* tmp = entries[pos].code_object; + entries[pos].code_object = code_object; + Py_INCREF(code_object); + Py_DECREF(tmp); + return; + } + if (code_cache->count == code_cache->max_count) { + int new_max = code_cache->max_count + 64; + entries = (__Pyx_CodeObjectCacheEntry*)PyMem_Realloc( + code_cache->entries, ((size_t)new_max) * sizeof(__Pyx_CodeObjectCacheEntry)); + if (unlikely(!entries)) { + return; + } + code_cache->entries = entries; + code_cache->max_count = new_max; + } + for (i=code_cache->count; i>pos; i--) { + entries[i] = entries[i-1]; + } + entries[pos].code_line = code_line; + entries[pos].code_object = code_object; + code_cache->count++; + Py_INCREF(code_object); +} +static void __pyx_insert_code_object(int code_line, __Pyx_CachedCodeObjectType* code_object) { +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING && !CYTHON_ATOMICS + (void)__pyx__insert_code_object; + return; // Most implementation should have atomics. But otherwise, don't make it thread-safe, just fail. +#else + struct __Pyx_CodeObjectCache *code_cache = &__pyx_mstate_global->__pyx_code_cache; +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_nonatomic_int_type expected = 0; + if (!__pyx_atomic_int_cmp_exchange(&code_cache->accessor_count, &expected, INT_MIN)) { + return; + } +#endif + __pyx__insert_code_object(code_cache, code_line, code_object); +#if CYTHON_COMPILING_IN_CPYTHON_FREETHREADING + __pyx_atomic_sub(&code_cache->accessor_count, INT_MIN); +#endif +#endif +} + +/* AddTraceback */ +#include "compile.h" +#include "frameobject.h" +#include "traceback.h" +#if PY_VERSION_HEX >= 0x030b00a6 && !CYTHON_COMPILING_IN_LIMITED_API && !defined(PYPY_VERSION) + #ifndef Py_BUILD_CORE + #define Py_BUILD_CORE 1 + #endif + #include "internal/pycore_frame.h" +#endif +#if CYTHON_COMPILING_IN_LIMITED_API +static PyObject *__Pyx_PyCode_Replace_For_AddTraceback(PyObject *code, PyObject *scratch_dict, + PyObject *firstlineno, PyObject *name) { + PyObject *replace = NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_firstlineno", firstlineno))) return NULL; + if (unlikely(PyDict_SetItemString(scratch_dict, "co_name", name))) return NULL; + replace = PyObject_GetAttrString(code, "replace"); + if (likely(replace)) { + PyObject *result = PyObject_Call(replace, __pyx_mstate_global->__pyx_empty_tuple, scratch_dict); + Py_DECREF(replace); + return result; + } + PyErr_Clear(); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyObject *code_object = NULL, *py_py_line = NULL, *py_funcname = NULL, *dict = NULL; + PyObject *replace = NULL, *getframe = NULL, *frame = NULL; + PyObject *exc_type, *exc_value, *exc_traceback; + int success = 0; + if (c_line) { + (void) __pyx_cfilenm; + (void) __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + PyErr_Fetch(&exc_type, &exc_value, &exc_traceback); + code_object = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!code_object) { + code_object = Py_CompileString("_getframe()", filename, Py_eval_input); + if (unlikely(!code_object)) goto bad; + py_py_line = PyLong_FromLong(py_line); + if (unlikely(!py_py_line)) goto bad; + py_funcname = PyUnicode_FromString(funcname); + if (unlikely(!py_funcname)) goto bad; + dict = PyDict_New(); + if (unlikely(!dict)) goto bad; + { + PyObject *old_code_object = code_object; + code_object = __Pyx_PyCode_Replace_For_AddTraceback(code_object, dict, py_py_line, py_funcname); + Py_DECREF(old_code_object); + } + if (unlikely(!code_object)) goto bad; + __pyx_insert_code_object(c_line ? -c_line : py_line, code_object); + } else { + dict = PyDict_New(); + } + getframe = PySys_GetObject("_getframe"); + if (unlikely(!getframe)) goto bad; + if (unlikely(PyDict_SetItemString(dict, "_getframe", getframe))) goto bad; + frame = PyEval_EvalCode(code_object, dict, dict); + if (unlikely(!frame) || frame == Py_None) goto bad; + success = 1; + bad: + PyErr_Restore(exc_type, exc_value, exc_traceback); + Py_XDECREF(code_object); + Py_XDECREF(py_py_line); + Py_XDECREF(py_funcname); + Py_XDECREF(dict); + Py_XDECREF(replace); + if (success) { + PyTraceBack_Here( + (struct _frame*)frame); + } + Py_XDECREF(frame); +} +#else +static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( + const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = NULL; + PyObject *py_funcname = NULL; + if (c_line) { + py_funcname = PyUnicode_FromFormat( "%s (%s:%d)", funcname, __pyx_cfilenm, c_line); + if (!py_funcname) goto bad; + funcname = PyUnicode_AsUTF8(py_funcname); + if (!funcname) goto bad; + } + py_code = PyCode_NewEmpty(filename, funcname, py_line); + Py_XDECREF(py_funcname); + return py_code; +bad: + Py_XDECREF(py_funcname); + return NULL; +} +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + PyCodeObject *py_code = 0; + PyFrameObject *py_frame = 0; + PyThreadState *tstate = __Pyx_PyThreadState_Current; + PyObject *ptype, *pvalue, *ptraceback; + if (c_line) { + c_line = __Pyx_CLineForTraceback(tstate, c_line); + } + py_code = __pyx_find_code_object(c_line ? -c_line : py_line); + if (!py_code) { + __Pyx_ErrFetchInState(tstate, &ptype, &pvalue, &ptraceback); + py_code = __Pyx_CreateCodeObjectForTraceback( + funcname, c_line, py_line, filename); + if (!py_code) { + /* If the code object creation fails, then we should clear the + fetched exception references and propagate the new exception */ + Py_XDECREF(ptype); + Py_XDECREF(pvalue); + Py_XDECREF(ptraceback); + goto bad; + } + __Pyx_ErrRestoreInState(tstate, ptype, pvalue, ptraceback); + __pyx_insert_code_object(c_line ? -c_line : py_line, py_code); + } + py_frame = PyFrame_New( + tstate, /*PyThreadState *tstate,*/ + py_code, /*PyCodeObject *code,*/ + __pyx_mstate_global->__pyx_d, /*PyObject *globals,*/ + 0 /*PyObject *locals*/ + ); + if (!py_frame) goto bad; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); + PyTraceBack_Here(py_frame); +bad: + Py_XDECREF(py_code); + Py_XDECREF(py_frame); +} +#endif + +/* CIntFromPyVerify */ +#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) +#define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ + __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) +#define __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, exc)\ + {\ + func_type value = func_value;\ + if (sizeof(target_type) < sizeof(func_type)) {\ + if (unlikely(value != (func_type) (target_type) value)) {\ + func_type zero = 0;\ + if (exc && unlikely(value == (func_type)-1 && PyErr_Occurred()))\ + return (target_type) -1;\ + if (is_unsigned && unlikely(value < zero))\ + goto raise_neg_overflow;\ + else\ + goto raise_overflow;\ + }\ + }\ + return (target_type) value;\ + } + +/* CIntFromPy */ +static CYTHON_INLINE int __Pyx_PyLong_As_int(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int neg_one = (int) -1, const_zero = (int) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int) -1; + val = __Pyx_PyLong_As_int(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 2 * PyLong_SHIFT)) { + return (int) (((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 3 * PyLong_SHIFT)) { + return (int) (((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) >= 4 * PyLong_SHIFT)) { + return (int) (((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + return (int) ((((((int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + return (int) ((((((((int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) (((int)-1)*(((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int) - 1 > 4 * PyLong_SHIFT)) { + return (int) ((((((((((int)digits[3]) << PyLong_SHIFT) | (int)digits[2]) << PyLong_SHIFT) | (int)digits[1]) << PyLong_SHIFT) | (int)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + int val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int) -1; + } else { + stepval = v; + } + v = NULL; + val = (int) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int) 1) << (sizeof(int) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int"); + return (int) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int"); + return (int) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE int32_t __Pyx_PyLong_As_int32_t(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int32_t neg_one = (int32_t) -1, const_zero = (int32_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int32_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int32_t) -1; + val = __Pyx_PyLong_As_int32_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int32_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int32_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) >= 2 * PyLong_SHIFT)) { + return (int32_t) (((((int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int32_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) >= 3 * PyLong_SHIFT)) { + return (int32_t) (((((((int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int32_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) >= 4 * PyLong_SHIFT)) { + return (int32_t) (((((((((int32_t)digits[3]) << PyLong_SHIFT) | (int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int32_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int32_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int32_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int32_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int32_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 2 * PyLong_SHIFT)) { + return (int32_t) (((int32_t)-1)*(((((int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int32_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 2 * PyLong_SHIFT)) { + return (int32_t) ((((((int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int32_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 3 * PyLong_SHIFT)) { + return (int32_t) (((int32_t)-1)*(((((((int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int32_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 3 * PyLong_SHIFT)) { + return (int32_t) ((((((((int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int32_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 4 * PyLong_SHIFT)) { + return (int32_t) (((int32_t)-1)*(((((((((int32_t)digits[3]) << PyLong_SHIFT) | (int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int32_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int32_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int32_t) - 1 > 4 * PyLong_SHIFT)) { + return (int32_t) ((((((((((int32_t)digits[3]) << PyLong_SHIFT) | (int32_t)digits[2]) << PyLong_SHIFT) | (int32_t)digits[1]) << PyLong_SHIFT) | (int32_t)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int32_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int32_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int32_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + int32_t val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int32_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int32_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int32_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (int32_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int32_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int32_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int32_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int32_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int32_t) 1) << (sizeof(int32_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int32_t) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int32_t"); + return (int32_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int32_t"); + return (int32_t) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE int64_t __Pyx_PyLong_As_int64_t(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const int64_t neg_one = (int64_t) -1, const_zero = (int64_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + int64_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (int64_t) -1; + val = __Pyx_PyLong_As_int64_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int64_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(int64_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) >= 2 * PyLong_SHIFT)) { + return (int64_t) (((((int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(int64_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) >= 3 * PyLong_SHIFT)) { + return (int64_t) (((((((int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(int64_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) >= 4 * PyLong_SHIFT)) { + return (int64_t) (((((((((int64_t)digits[3]) << PyLong_SHIFT) | (int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (int64_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(int64_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(int64_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int64_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int64_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(int64_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(int64_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) - 1 > 2 * PyLong_SHIFT)) { + return (int64_t) (((int64_t)-1)*(((((int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(int64_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) - 1 > 2 * PyLong_SHIFT)) { + return (int64_t) ((((((int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(int64_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) - 1 > 3 * PyLong_SHIFT)) { + return (int64_t) (((int64_t)-1)*(((((((int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(int64_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) - 1 > 3 * PyLong_SHIFT)) { + return (int64_t) ((((((((int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(int64_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) - 1 > 4 * PyLong_SHIFT)) { + return (int64_t) (((int64_t)-1)*(((((((((int64_t)digits[3]) << PyLong_SHIFT) | (int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(int64_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(int64_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(int64_t) - 1 > 4 * PyLong_SHIFT)) { + return (int64_t) ((((((((((int64_t)digits[3]) << PyLong_SHIFT) | (int64_t)digits[2]) << PyLong_SHIFT) | (int64_t)digits[1]) << PyLong_SHIFT) | (int64_t)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(int64_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(int64_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(int64_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(int64_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + int64_t val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (int64_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (int64_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (int64_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (int64_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(int64_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((int64_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(int64_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((int64_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((int64_t) 1) << (sizeof(int64_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (int64_t) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to int64_t"); + return (int64_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to int64_t"); + return (int64_t) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE char __Pyx_PyLong_As_char(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const char neg_one = (char) -1, const_zero = (char) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + char val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (char) -1; + val = __Pyx_PyLong_As_char(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(char, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(char) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 2 * PyLong_SHIFT)) { + return (char) (((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(char) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 3 * PyLong_SHIFT)) { + return (char) (((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(char) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) >= 4 * PyLong_SHIFT)) { + return (char) (((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (char) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(char) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(char) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(char, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(char, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(char) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(char) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + return (char) ((((((char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(char) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(char) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + return (char) ((((((((char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(char) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 4 * PyLong_SHIFT)) { + return (char) (((char)-1)*(((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(char) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(char, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(char) - 1 > 4 * PyLong_SHIFT)) { + return (char) ((((((((((char)digits[3]) << PyLong_SHIFT) | (char)digits[2]) << PyLong_SHIFT) | (char)digits[1]) << PyLong_SHIFT) | (char)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(char) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(char, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(char) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(char, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + char val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (char) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (char) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (char) -1; + } else { + stepval = v; + } + v = NULL; + val = (char) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(char) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((char) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(char) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((char) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((char) 1) << (sizeof(char) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (char) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to char"); + return (char) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to char"); + return (char) -1; +} + +/* CIntFromPy */ +static CYTHON_INLINE size_t __Pyx_PyLong_As_size_t(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + size_t val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (size_t) -1; + val = __Pyx_PyLong_As_size_t(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(size_t, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(size_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 2 * PyLong_SHIFT)) { + return (size_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(size_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 3 * PyLong_SHIFT)) { + return (size_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(size_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) >= 4 * PyLong_SHIFT)) { + return (size_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (size_t) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(size_t) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(size_t, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(size_t) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + return (size_t) ((((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(size_t) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(size_t) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + return (size_t) ((((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(size_t) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT)) { + return (size_t) (((size_t)-1)*(((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(size_t) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(size_t, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(size_t) - 1 > 4 * PyLong_SHIFT)) { + return (size_t) ((((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(size_t) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(size_t) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + size_t val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (size_t) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (size_t) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (size_t) -1; + } else { + stepval = v; + } + v = NULL; + val = (size_t) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(size_t) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((size_t) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(size_t) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((size_t) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((size_t) 1) << (sizeof(size_t) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (size_t) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to size_t"); + return (size_t) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to size_t"); + return (size_t) -1; +} + +/* FormatTypeName */ +#if CYTHON_COMPILING_IN_LIMITED_API && __PYX_LIMITED_VERSION_HEX < 0x030d0000 +static __Pyx_TypeName +__Pyx_PyType_GetFullyQualifiedName(PyTypeObject* tp) +{ + PyObject *module = NULL, *name = NULL, *result = NULL; + #if __PYX_LIMITED_VERSION_HEX < 0x030b0000 + name = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_qualname); + #else + name = PyType_GetQualName(tp); + #endif + if (unlikely(name == NULL) || unlikely(!PyUnicode_Check(name))) goto bad; + module = __Pyx_PyObject_GetAttrStr((PyObject *)tp, + __pyx_mstate_global->__pyx_n_u_module); + if (unlikely(module == NULL) || unlikely(!PyUnicode_Check(module))) goto bad; + if (PyUnicode_CompareWithASCIIString(module, "builtins") == 0) { + result = name; + name = NULL; + goto done; + } + result = PyUnicode_FromFormat("%U.%U", module, name); + if (unlikely(result == NULL)) goto bad; + done: + Py_XDECREF(name); + Py_XDECREF(module); + return result; + bad: + PyErr_Clear(); + if (name) { + result = name; + name = NULL; + } else { + result = __Pyx_NewRef(__pyx_mstate_global->__pyx_kp_u__2); + } + goto done; +} +#endif + +/* PyObjectVectorCallKwBuilder */ +#if CYTHON_VECTORCALL +static int __Pyx_VectorcallBuilder_AddArg(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_PyObject_FastCallDict; + if (__Pyx_PyTuple_SET_ITEM(builder, n, key) != (0)) return -1; + Py_INCREF(key); + args[n] = value; + return 0; +} +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + (void)__Pyx_VectorcallBuilder_AddArgStr; + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return __Pyx_VectorcallBuilder_AddArg(key, value, builder, args, n); +} +static int __Pyx_VectorcallBuilder_AddArgStr(const char *key, PyObject *value, PyObject *builder, PyObject **args, int n) { + PyObject *pyKey = PyUnicode_FromString(key); + if (!pyKey) return -1; + return __Pyx_VectorcallBuilder_AddArg(pyKey, value, builder, args, n); +} +#else // CYTHON_VECTORCALL +CYTHON_UNUSED static int __Pyx_VectorcallBuilder_AddArg_Check(PyObject *key, PyObject *value, PyObject *builder, CYTHON_UNUSED PyObject **args, CYTHON_UNUSED int n) { + if (unlikely(!PyUnicode_Check(key))) { + PyErr_SetString(PyExc_TypeError, "keywords must be strings"); + return -1; + } + return PyDict_SetItem(builder, key, value); +} +#endif + +/* CIntToPy */ +static CYTHON_INLINE PyObject* __Pyx_PyLong_From_long(long value) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (is_unsigned) { + if (sizeof(long) < sizeof(long)) { + return PyLong_FromLong((long) value); + } else if (sizeof(long) <= sizeof(unsigned long)) { + return PyLong_FromUnsignedLong((unsigned long) value); +#if defined(HAVE_LONG_LONG) && !CYTHON_COMPILING_IN_PYPY + } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif + } + } else { + if (sizeof(long) <= sizeof(long)) { + return PyLong_FromLong((long) value); +#ifdef HAVE_LONG_LONG + } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { + return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif + } + } + { + unsigned char *bytes = (unsigned char *)&value; +#if !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d00A4 + if (is_unsigned) { + return PyLong_FromUnsignedNativeBytes(bytes, sizeof(value), -1); + } else { + return PyLong_FromNativeBytes(bytes, sizeof(value), -1); + } +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX < 0x030d0000 + int one = 1; int little = (int)*(unsigned char *)&one; + return _PyLong_FromByteArray(bytes, sizeof(long), + little, !is_unsigned); +#else + int one = 1; int little = (int)*(unsigned char *)&one; + PyObject *from_bytes, *result = NULL, *kwds = NULL; + PyObject *py_bytes = NULL, *order_str = NULL; + from_bytes = PyObject_GetAttrString((PyObject*)&PyLong_Type, "from_bytes"); + if (!from_bytes) return NULL; + py_bytes = PyBytes_FromStringAndSize((char*)bytes, sizeof(long)); + if (!py_bytes) goto limited_bad; + order_str = PyUnicode_FromString(little ? "little" : "big"); + if (!order_str) goto limited_bad; + { + PyObject *args[3+(CYTHON_VECTORCALL ? 1 : 0)] = { NULL, py_bytes, order_str }; + if (!is_unsigned) { + kwds = __Pyx_MakeVectorcallBuilderKwds(1); + if (!kwds) goto limited_bad; + if (__Pyx_VectorcallBuilder_AddArgStr("signed", __Pyx_NewRef(Py_True), kwds, args+3, 0) < 0) goto limited_bad; + } + result = __Pyx_Object_Vectorcall_CallFromBuilder(from_bytes, args+1, 2 | __Pyx_PY_VECTORCALL_ARGUMENTS_OFFSET, kwds); + } + limited_bad: + Py_XDECREF(kwds); + Py_XDECREF(order_str); + Py_XDECREF(py_bytes); + Py_XDECREF(from_bytes); + return result; +#endif + } +} + +/* CIntFromPy */ +static CYTHON_INLINE long __Pyx_PyLong_As_long(PyObject *x) { +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wconversion" +#endif + const long neg_one = (long) -1, const_zero = (long) 0; +#ifdef __Pyx_HAS_GCC_DIAGNOSTIC +#pragma GCC diagnostic pop +#endif + const int is_unsigned = neg_one > const_zero; + if (unlikely(!PyLong_Check(x))) { + long val; + PyObject *tmp = __Pyx_PyNumber_Long(x); + if (!tmp) return (long) -1; + val = __Pyx_PyLong_As_long(tmp); + Py_DECREF(tmp); + return val; + } + if (is_unsigned) { +#if CYTHON_USE_PYLONG_INTERNALS + if (unlikely(__Pyx_PyLong_IsNeg(x))) { + goto raise_neg_overflow; + } else if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_upylong, __Pyx_PyLong_CompactValueUnsigned(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_DigitCount(x)) { + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 2 * PyLong_SHIFT)) { + return (long) (((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 3 * PyLong_SHIFT)) { + return (long) (((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) >= 4 * PyLong_SHIFT)) { + return (long) (((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0])); + } + } + break; + } + } +#endif +#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX < 0x030C00A7 + if (unlikely(Py_SIZE(x) < 0)) { + goto raise_neg_overflow; + } +#else + { + int result = PyObject_RichCompareBool(x, Py_False, Py_LT); + if (unlikely(result < 0)) + return (long) -1; + if (unlikely(result == 1)) + goto raise_neg_overflow; + } +#endif + if ((sizeof(long) <= sizeof(unsigned long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(unsigned PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif + } + } else { +#if CYTHON_USE_PYLONG_INTERNALS + if (__Pyx_PyLong_IsCompact(x)) { + __PYX_VERIFY_RETURN_INT(long, __Pyx_compact_pylong, __Pyx_PyLong_CompactValue(x)) + } else { + const digit* digits = __Pyx_PyLong_Digits(x); + assert(__Pyx_PyLong_DigitCount(x) > 1); + switch (__Pyx_PyLong_SignedDigitCount(x)) { + case -2: + if ((8 * sizeof(long) - 1 > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 2: + if ((8 * sizeof(long) > 1 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 2 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + return (long) ((((((long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -3: + if ((8 * sizeof(long) - 1 > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 3: + if ((8 * sizeof(long) > 2 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 3 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + return (long) ((((((((long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case -4: + if ((8 * sizeof(long) - 1 > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, long, -(long) (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) (((long)-1)*(((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + case 4: + if ((8 * sizeof(long) > 3 * PyLong_SHIFT)) { + if ((8 * sizeof(unsigned long) > 4 * PyLong_SHIFT)) { + __PYX_VERIFY_RETURN_INT(long, unsigned long, (((((((((unsigned long)digits[3]) << PyLong_SHIFT) | (unsigned long)digits[2]) << PyLong_SHIFT) | (unsigned long)digits[1]) << PyLong_SHIFT) | (unsigned long)digits[0]))) + } else if ((8 * sizeof(long) - 1 > 4 * PyLong_SHIFT)) { + return (long) ((((((((((long)digits[3]) << PyLong_SHIFT) | (long)digits[2]) << PyLong_SHIFT) | (long)digits[1]) << PyLong_SHIFT) | (long)digits[0]))); + } + } + break; + } + } +#endif + if ((sizeof(long) <= sizeof(long))) { + __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG + } else if ((sizeof(long) <= sizeof(PY_LONG_LONG))) { + __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif + } + } + { + long val; + int ret = -1; +#if PY_VERSION_HEX >= 0x030d00A6 && !CYTHON_COMPILING_IN_LIMITED_API + Py_ssize_t bytes_copied = PyLong_AsNativeBytes( + x, &val, sizeof(val), Py_ASNATIVEBYTES_NATIVE_ENDIAN | (is_unsigned ? Py_ASNATIVEBYTES_UNSIGNED_BUFFER | Py_ASNATIVEBYTES_REJECT_NEGATIVE : 0)); + if (unlikely(bytes_copied == -1)) { + } else if (unlikely(bytes_copied > (Py_ssize_t) sizeof(val))) { + goto raise_overflow; + } else { + ret = 0; + } +#elif PY_VERSION_HEX < 0x030d0000 && !(CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) || defined(_PyLong_AsByteArray) + int one = 1; int is_little = (int)*(unsigned char *)&one; + unsigned char *bytes = (unsigned char *)&val; + ret = _PyLong_AsByteArray((PyLongObject *)x, + bytes, sizeof(val), + is_little, !is_unsigned); +#else + PyObject *v; + PyObject *stepval = NULL, *mask = NULL, *shift = NULL; + int bits, remaining_bits, is_negative = 0; + int chunk_size = (sizeof(long) < 8) ? 30 : 62; + if (likely(PyLong_CheckExact(x))) { + v = __Pyx_NewRef(x); + } else { + v = PyNumber_Long(x); + if (unlikely(!v)) return (long) -1; + assert(PyLong_CheckExact(v)); + } + { + int result = PyObject_RichCompareBool(v, Py_False, Py_LT); + if (unlikely(result < 0)) { + Py_DECREF(v); + return (long) -1; + } + is_negative = result == 1; + } + if (is_unsigned && unlikely(is_negative)) { + Py_DECREF(v); + goto raise_neg_overflow; + } else if (is_negative) { + stepval = PyNumber_Invert(v); + Py_DECREF(v); + if (unlikely(!stepval)) + return (long) -1; + } else { + stepval = v; + } + v = NULL; + val = (long) 0; + mask = PyLong_FromLong((1L << chunk_size) - 1); if (unlikely(!mask)) goto done; + shift = PyLong_FromLong(chunk_size); if (unlikely(!shift)) goto done; + for (bits = 0; bits < (int) sizeof(long) * 8 - chunk_size; bits += chunk_size) { + PyObject *tmp, *digit; + long idigit; + digit = PyNumber_And(stepval, mask); + if (unlikely(!digit)) goto done; + idigit = PyLong_AsLong(digit); + Py_DECREF(digit); + if (unlikely(idigit < 0)) goto done; + val |= ((long) idigit) << bits; + tmp = PyNumber_Rshift(stepval, shift); + if (unlikely(!tmp)) goto done; + Py_DECREF(stepval); stepval = tmp; + } + Py_DECREF(shift); shift = NULL; + Py_DECREF(mask); mask = NULL; + { + long idigit = PyLong_AsLong(stepval); + if (unlikely(idigit < 0)) goto done; + remaining_bits = ((int) sizeof(long) * 8) - bits - (is_unsigned ? 0 : 1); + if (unlikely(idigit >= (1L << remaining_bits))) + goto raise_overflow; + val |= ((long) idigit) << bits; + } + if (!is_unsigned) { + if (unlikely(val & (((long) 1) << (sizeof(long) * 8 - 1)))) + goto raise_overflow; + if (is_negative) + val = ~val; + } + ret = 0; + done: + Py_XDECREF(shift); + Py_XDECREF(mask); + Py_XDECREF(stepval); +#endif + if (unlikely(ret)) + return (long) -1; + return val; + } +raise_overflow: + PyErr_SetString(PyExc_OverflowError, + "value too large to convert to long"); + return (long) -1; +raise_neg_overflow: + PyErr_SetString(PyExc_OverflowError, + "can't convert negative value to long"); + return (long) -1; +} + +/* FastTypeChecks */ +#if CYTHON_COMPILING_IN_CPYTHON +static int __Pyx_InBases(PyTypeObject *a, PyTypeObject *b) { + while (a) { + a = __Pyx_PyType_GetSlot(a, tp_base, PyTypeObject*); + if (a == b) + return 1; + } + return b == &PyBaseObject_Type; +} +static CYTHON_INLINE int __Pyx_IsSubtype(PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (a == b) return 1; + mro = a->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + if (PyTuple_GET_ITEM(mro, i) == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(a, b); +} +static CYTHON_INLINE int __Pyx_IsAnySubtype2(PyTypeObject *cls, PyTypeObject *a, PyTypeObject *b) { + PyObject *mro; + if (cls == a || cls == b) return 1; + mro = cls->tp_mro; + if (likely(mro)) { + Py_ssize_t i, n; + n = PyTuple_GET_SIZE(mro); + for (i = 0; i < n; i++) { + PyObject *base = PyTuple_GET_ITEM(mro, i); + if (base == (PyObject *)a || base == (PyObject *)b) + return 1; + } + return 0; + } + return __Pyx_InBases(cls, a) || __Pyx_InBases(cls, b); +} +static CYTHON_INLINE int __Pyx_inner_PyErr_GivenExceptionMatches2(PyObject *err, PyObject* exc_type1, PyObject *exc_type2) { + if (exc_type1) { + return __Pyx_IsAnySubtype2((PyTypeObject*)err, (PyTypeObject*)exc_type1, (PyTypeObject*)exc_type2); + } else { + return __Pyx_IsSubtype((PyTypeObject*)err, (PyTypeObject*)exc_type2); + } +} +static int __Pyx_PyErr_GivenExceptionMatchesTuple(PyObject *exc_type, PyObject *tuple) { + Py_ssize_t i, n; + assert(PyExceptionClass_Check(exc_type)); + n = PyTuple_GET_SIZE(tuple); + for (i=0; i= 0x030b0000 + return Py_Version & ~0xFFUL; +#else + static unsigned long __Pyx_cached_runtime_version = 0; + if (__Pyx_cached_runtime_version == 0) { + const char* rt_version = Py_GetVersion(); + unsigned long version = 0; + unsigned long factor = 0x01000000UL; + unsigned int digit = 0; + int i = 0; + while (factor) { + while ('0' <= rt_version[i] && rt_version[i] <= '9') { + digit = digit * 10 + (unsigned int) (rt_version[i] - '0'); + ++i; + } + version += factor * digit; + if (rt_version[i] != '.') + break; + digit = 0; + factor >>= 8; + ++i; + } + __Pyx_cached_runtime_version = version; + } + return __Pyx_cached_runtime_version; +#endif +} + +/* CheckBinaryVersion */ +static int __Pyx_check_binary_version(unsigned long ct_version, unsigned long rt_version, int allow_newer) { + const unsigned long MAJOR_MINOR = 0xFFFF0000UL; + if ((rt_version & MAJOR_MINOR) == (ct_version & MAJOR_MINOR)) + return 0; + if (likely(allow_newer && (rt_version & MAJOR_MINOR) > (ct_version & MAJOR_MINOR))) + return 1; + { + char message[200]; + PyOS_snprintf(message, sizeof(message), + "compile time Python version %d.%d " + "of module '%.100s' " + "%s " + "runtime version %d.%d", + (int) (ct_version >> 24), (int) ((ct_version >> 16) & 0xFF), + __Pyx_MODULE_NAME, + (allow_newer) ? "was newer than" : "does not match", + (int) (rt_version >> 24), (int) ((rt_version >> 16) & 0xFF) + ); + return PyErr_WarnEx(NULL, message, 1); + } +} + +/* FunctionExport */ +static int __Pyx_ExportFunction(const char *name, void (*f)(void), const char *sig) { + PyObject *d = 0; + PyObject *cobj = 0; + union { + void (*fp)(void); + void *p; + } tmp; + d = PyObject_GetAttrString(__pyx_m, "__pyx_capi__"); + if (!d) { + PyErr_Clear(); + d = PyDict_New(); + if (!d) + goto bad; + Py_INCREF(d); + if (PyModule_AddObject(__pyx_m, "__pyx_capi__", d) < 0) + goto bad; + } + tmp.fp = f; + cobj = PyCapsule_New(tmp.p, sig, 0); + if (!cobj) + goto bad; + if (PyDict_SetItemString(d, name, cobj) < 0) + goto bad; + Py_DECREF(cobj); + Py_DECREF(d); + return 0; +bad: + Py_XDECREF(cobj); + Py_XDECREF(d); + return -1; +} + +/* InitStrings */ +static int __Pyx_InitStrings(__Pyx_StringTabEntry const *t, PyObject **target, const char* const* encoding_names) { + while (t->s) { + PyObject *str; + if (t->is_unicode) { + if (t->intern) { + str = PyUnicode_InternFromString(t->s); + } else if (t->encoding) { + str = PyUnicode_Decode(t->s, t->n - 1, encoding_names[t->encoding], NULL); + } else { + str = PyUnicode_FromStringAndSize(t->s, t->n - 1); + } + } else { + str = PyBytes_FromStringAndSize(t->s, t->n - 1); + } + if (!str) + return -1; + *target = str; + if (PyObject_Hash(str) == -1) + return -1; + ++t; + ++target; + } + return 0; +} + +#include +static CYTHON_INLINE Py_ssize_t __Pyx_ssize_strlen(const char *s) { + size_t len = strlen(s); + if (unlikely(len > (size_t) PY_SSIZE_T_MAX)) { + PyErr_SetString(PyExc_OverflowError, "byte string is too long"); + return -1; + } + return (Py_ssize_t) len; +} +static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return __Pyx_PyUnicode_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE PyObject* __Pyx_PyByteArray_FromString(const char* c_str) { + Py_ssize_t len = __Pyx_ssize_strlen(c_str); + if (unlikely(len < 0)) return NULL; + return PyByteArray_FromStringAndSize(c_str, len); +} +static CYTHON_INLINE const char* __Pyx_PyObject_AsString(PyObject* o) { + Py_ssize_t ignore; + return __Pyx_PyObject_AsStringAndSize(o, &ignore); +} +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 +static CYTHON_INLINE const char* __Pyx_PyUnicode_AsStringAndSize(PyObject* o, Py_ssize_t *length) { + if (unlikely(__Pyx_PyUnicode_READY(o) == -1)) return NULL; +#if CYTHON_COMPILING_IN_LIMITED_API + { + const char* result; + Py_ssize_t unicode_length; + CYTHON_MAYBE_UNUSED_VAR(unicode_length); // only for __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + #if __PYX_LIMITED_VERSION_HEX < 0x030A0000 + if (unlikely(PyArg_Parse(o, "s#", &result, length) < 0)) return NULL; + #else + result = PyUnicode_AsUTF8AndSize(o, length); + #endif + #if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + unicode_length = PyUnicode_GetLength(o); + if (unlikely(unicode_length < 0)) return NULL; + if (unlikely(unicode_length != *length)) { + PyUnicode_AsASCIIString(o); + return NULL; + } + #endif + return result; + } +#else +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII + if (likely(PyUnicode_IS_ASCII(o))) { + *length = PyUnicode_GET_LENGTH(o); + return PyUnicode_AsUTF8(o); + } else { + PyUnicode_AsASCIIString(o); + return NULL; + } +#else + return PyUnicode_AsUTF8AndSize(o, length); +#endif +#endif +} +#endif +static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ssize_t *length) { +#if __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_UTF8 + if (PyUnicode_Check(o)) { + return __Pyx_PyUnicode_AsStringAndSize(o, length); + } else +#endif + if (PyByteArray_Check(o)) { +#if (CYTHON_ASSUME_SAFE_SIZE && CYTHON_ASSUME_SAFE_MACROS) || (CYTHON_COMPILING_IN_PYPY && (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE))) + *length = PyByteArray_GET_SIZE(o); + return PyByteArray_AS_STRING(o); +#else + *length = PyByteArray_Size(o); + if (*length == -1) return NULL; + return PyByteArray_AsString(o); +#endif + } else + { + char* result; + int r = PyBytes_AsStringAndSize(o, &result, length); + if (unlikely(r < 0)) { + return NULL; + } else { + return result; + } + } +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrue(PyObject* x) { + int is_true = x == Py_True; + if (is_true | (x == Py_False) | (x == Py_None)) return is_true; + else return PyObject_IsTrue(x); +} +static CYTHON_INLINE int __Pyx_PyObject_IsTrueAndDecref(PyObject* x) { + int retval; + if (unlikely(!x)) return -1; + retval = __Pyx_PyObject_IsTrue(x); + Py_DECREF(x); + return retval; +} +static PyObject* __Pyx_PyNumber_LongWrongResultType(PyObject* result) { + __Pyx_TypeName result_type_name = __Pyx_PyType_GetFullyQualifiedName(Py_TYPE(result)); + if (PyLong_Check(result)) { + if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME "). " + "The ability to return an instance of a strict subclass of int is deprecated, " + "and may be removed in a future version of Python.", + result_type_name)) { + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; + } + __Pyx_DECREF_TypeName(result_type_name); + return result; + } + PyErr_Format(PyExc_TypeError, + "__int__ returned non-int (type " __Pyx_FMT_TYPENAME ")", + result_type_name); + __Pyx_DECREF_TypeName(result_type_name); + Py_DECREF(result); + return NULL; +} +static CYTHON_INLINE PyObject* __Pyx_PyNumber_Long(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS + PyNumberMethods *m; +#endif + PyObject *res = NULL; + if (likely(PyLong_Check(x))) + return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS + m = Py_TYPE(x)->tp_as_number; + if (likely(m && m->nb_int)) { + res = m->nb_int(x); + } +#else + if (!PyBytes_CheckExact(x) && !PyUnicode_CheckExact(x)) { + res = PyNumber_Long(x); + } +#endif + if (likely(res)) { + if (unlikely(!PyLong_CheckExact(res))) { + return __Pyx_PyNumber_LongWrongResultType(res); + } + } + else if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_TypeError, + "an integer is required"); + } + return res; +} +static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject* b) { + Py_ssize_t ival; + PyObject *x; + if (likely(PyLong_CheckExact(b))) { + #if CYTHON_USE_PYLONG_INTERNALS + if (likely(__Pyx_PyLong_IsCompact(b))) { + return __Pyx_PyLong_CompactValue(b); + } else { + const digit* digits = __Pyx_PyLong_Digits(b); + const Py_ssize_t size = __Pyx_PyLong_SignedDigitCount(b); + switch (size) { + case 2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return (Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -2: + if (8 * sizeof(Py_ssize_t) > 2 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -3: + if (8 * sizeof(Py_ssize_t) > 3 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case 4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return (Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + case -4: + if (8 * sizeof(Py_ssize_t) > 4 * PyLong_SHIFT) { + return -(Py_ssize_t) (((((((((size_t)digits[3]) << PyLong_SHIFT) | (size_t)digits[2]) << PyLong_SHIFT) | (size_t)digits[1]) << PyLong_SHIFT) | (size_t)digits[0])); + } + break; + } + } + #endif + return PyLong_AsSsize_t(b); + } + x = PyNumber_Index(b); + if (!x) return -1; + ival = PyLong_AsSsize_t(x); + Py_DECREF(x); + return ival; +} +static CYTHON_INLINE Py_hash_t __Pyx_PyIndex_AsHash_t(PyObject* o) { + if (sizeof(Py_hash_t) == sizeof(Py_ssize_t)) { + return (Py_hash_t) __Pyx_PyIndex_AsSsize_t(o); + } else { + Py_ssize_t ival; + PyObject *x; + x = PyNumber_Index(o); + if (!x) return -1; + ival = PyLong_AsLong(x); + Py_DECREF(x); + return ival; + } +} +static CYTHON_INLINE PyObject *__Pyx_Owned_Py_None(int b) { + CYTHON_UNUSED_VAR(b); + return __Pyx_NewRef(Py_None); +} +static CYTHON_INLINE PyObject * __Pyx_PyBool_FromLong(long b) { + return b ? __Pyx_NewRef(Py_True) : __Pyx_NewRef(Py_False); +} +static CYTHON_INLINE PyObject * __Pyx_PyLong_FromSize_t(size_t ival) { + return PyLong_FromSize_t(ival); +} +#if CYTHON_USE_PYLONG_INTERNALS +static CYTHON_INLINE int __Pyx_PyLong_CompactAsLong(PyObject *x, long *return_value) { + if (unlikely(!__Pyx_PyLong_IsCompact(x))) + return 0; + Py_ssize_t value = __Pyx_PyLong_CompactValue(x); + if ((sizeof(long) < sizeof(Py_ssize_t)) && unlikely(value != (long) value)) + return 0; + *return_value = (long) value; + return 1; +} +#endif + + +/* MultiPhaseInitModuleState */ +#if CYTHON_PEP489_MULTI_PHASE_INIT && CYTHON_USE_MODULE_STATE +#ifndef CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#if (CYTHON_COMPILING_IN_LIMITED_API || PY_VERSION_HEX >= 0x030C0000) + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 1 +#else + #define CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE 0 +#endif +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE && !CYTHON_ATOMICS +#error "Module state with PEP489 requires atomics. Currently that's one of\ + C11, C++11, gcc atomic intrinsics or MSVC atomic intrinsics" +#endif +#if !CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +#define __Pyx_ModuleStateLookup_Lock() +#define __Pyx_ModuleStateLookup_Unlock() +#elif !CYTHON_COMPILING_IN_LIMITED_API && PY_VERSION_HEX >= 0x030d0000 +static PyMutex __Pyx_ModuleStateLookup_mutex = {0}; +#define __Pyx_ModuleStateLookup_Lock() PyMutex_Lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() PyMutex_Unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(__cplusplus) && __cplusplus >= 201103L +#include +static std::mutex __Pyx_ModuleStateLookup_mutex; +#define __Pyx_ModuleStateLookup_Lock() __Pyx_ModuleStateLookup_mutex.lock() +#define __Pyx_ModuleStateLookup_Unlock() __Pyx_ModuleStateLookup_mutex.unlock() +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201112L) && !defined(__STDC_NO_THREADS__) +#include +static mtx_t __Pyx_ModuleStateLookup_mutex; +static once_flag __Pyx_ModuleStateLookup_mutex_once_flag = ONCE_FLAG_INIT; +static void __Pyx_ModuleStateLookup_initialize_mutex(void) { + mtx_init(&__Pyx_ModuleStateLookup_mutex, mtx_plain); +} +#define __Pyx_ModuleStateLookup_Lock()\ + call_once(&__Pyx_ModuleStateLookup_mutex_once_flag, __Pyx_ModuleStateLookup_initialize_mutex);\ + mtx_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() mtx_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(HAVE_PTHREAD_H) +#include +static pthread_mutex_t __Pyx_ModuleStateLookup_mutex = PTHREAD_MUTEX_INITIALIZER; +#define __Pyx_ModuleStateLookup_Lock() pthread_mutex_lock(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() pthread_mutex_unlock(&__Pyx_ModuleStateLookup_mutex) +#elif defined(_WIN32) +#include // synchapi.h on its own doesn't work +static SRWLOCK __Pyx_ModuleStateLookup_mutex = SRWLOCK_INIT; +#define __Pyx_ModuleStateLookup_Lock() AcquireSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#define __Pyx_ModuleStateLookup_Unlock() ReleaseSRWLockExclusive(&__Pyx_ModuleStateLookup_mutex) +#else +#error "No suitable lock available for CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE.\ + Requires C standard >= C11, or C++ standard >= C++11,\ + or pthreads, or the Windows 32 API, or Python >= 3.13." +#endif +typedef struct { + int64_t id; + PyObject *module; +} __Pyx_InterpreterIdAndModule; +typedef struct { + char interpreter_id_as_index; + Py_ssize_t count; + Py_ssize_t allocated; + __Pyx_InterpreterIdAndModule table[1]; +} __Pyx_ModuleStateLookupData; +#define __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE 32 +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_int_type __Pyx_ModuleStateLookup_read_counter = 0; +#endif +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static __pyx_atomic_ptr_type __Pyx_ModuleStateLookup_data = 0; +#else +static __Pyx_ModuleStateLookupData* __Pyx_ModuleStateLookup_data = NULL; +#endif +static __Pyx_InterpreterIdAndModule* __Pyx_State_FindModuleStateLookupTableLowerBound( + __Pyx_InterpreterIdAndModule* table, + Py_ssize_t count, + int64_t interpreterId) { + __Pyx_InterpreterIdAndModule* begin = table; + __Pyx_InterpreterIdAndModule* end = begin + count; + if (begin->id == interpreterId) { + return begin; + } + while ((end - begin) > __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + __Pyx_InterpreterIdAndModule* halfway = begin + (end - begin)/2; + if (halfway->id == interpreterId) { + return halfway; + } + if (halfway->id < interpreterId) { + begin = halfway; + } else { + end = halfway; + } + } + for (; begin < end; ++begin) { + if (begin->id >= interpreterId) return begin; + } + return begin; +} +static PyObject *__Pyx_State_FindModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return NULL; +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData* data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + { + __pyx_atomic_incr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + if (likely(data)) { + __Pyx_ModuleStateLookupData* new_data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_acquire(&__Pyx_ModuleStateLookup_data); + if (likely(data == new_data)) { + goto read_finished; + } + } + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); + __Pyx_ModuleStateLookup_Lock(); + __pyx_atomic_incr_relaxed(&__Pyx_ModuleStateLookup_read_counter); + data = (__Pyx_ModuleStateLookupData*)__pyx_atomic_pointer_load_relaxed(&__Pyx_ModuleStateLookup_data); + __Pyx_ModuleStateLookup_Unlock(); + } + read_finished:; +#else + __Pyx_ModuleStateLookupData* data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_InterpreterIdAndModule* found = NULL; + if (unlikely(!data)) goto end; + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + found = data->table+interpreter_id; + } + } else { + found = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + } + end: + { + PyObject *result=NULL; + if (found && found->id == interpreter_id) { + result = found->module; + } +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_decr_acq_rel(&__Pyx_ModuleStateLookup_read_counter); +#endif + return result; + } +} +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE +static void __Pyx_ModuleStateLookup_wait_until_no_readers(void) { + while (__pyx_atomic_load(&__Pyx_ModuleStateLookup_read_counter) != 0); +} +#else +#define __Pyx_ModuleStateLookup_wait_until_no_readers() +#endif +static int __Pyx_State_AddModuleInterpIdAsIndex(__Pyx_ModuleStateLookupData **old_data, PyObject* module, int64_t interpreter_id) { + Py_ssize_t to_allocate = (*old_data)->allocated; + while (to_allocate <= interpreter_id) { + if (to_allocate == 0) to_allocate = 1; + else to_allocate *= 2; + } + __Pyx_ModuleStateLookupData *new_data = *old_data; + if (to_allocate != (*old_data)->allocated) { + new_data = (__Pyx_ModuleStateLookupData *)realloc( + *old_data, + sizeof(__Pyx_ModuleStateLookupData)+(to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + PyErr_NoMemory(); + return -1; + } + for (Py_ssize_t i = new_data->allocated; i < to_allocate; ++i) { + new_data->table[i].id = i; + new_data->table[i].module = NULL; + } + new_data->allocated = to_allocate; + } + new_data->table[interpreter_id].module = module; + if (new_data->count < interpreter_id+1) { + new_data->count = interpreter_id+1; + } + *old_data = new_data; + return 0; +} +static void __Pyx_State_ConvertFromInterpIdAsIndex(__Pyx_ModuleStateLookupData *data) { + __Pyx_InterpreterIdAndModule *read = data->table; + __Pyx_InterpreterIdAndModule *write = data->table; + __Pyx_InterpreterIdAndModule *end = read + data->count; + for (; readmodule) { + write->id = read->id; + write->module = read->module; + ++write; + } + } + data->count = write - data->table; + for (; writeid = 0; + write->module = NULL; + } + data->interpreter_id_as_index = 0; +} +static int __Pyx_State_AddModule(PyObject* module, CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + int result = 0; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *old_data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *old_data = __Pyx_ModuleStateLookup_data; +#endif + __Pyx_ModuleStateLookupData *new_data = old_data; + if (!new_data) { + new_data = (__Pyx_ModuleStateLookupData *)calloc(1, sizeof(__Pyx_ModuleStateLookupData)); + if (!new_data) { + result = -1; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = 1; + new_data->interpreter_id_as_index = 1; + } + __Pyx_ModuleStateLookup_wait_until_no_readers(); + if (new_data->interpreter_id_as_index) { + if (interpreter_id < __PYX_MODULE_STATE_LOOKUP_SMALL_SIZE) { + result = __Pyx_State_AddModuleInterpIdAsIndex(&new_data, module, interpreter_id); + goto end; + } + __Pyx_State_ConvertFromInterpIdAsIndex(new_data); + } + { + Py_ssize_t insert_at = 0; + { + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + new_data->table, new_data->count, interpreter_id); + assert(lower_bound); + insert_at = lower_bound - new_data->table; + if (unlikely(insert_at < new_data->count && lower_bound->id == interpreter_id)) { + lower_bound->module = module; + goto end; // already in table, nothing more to do + } + } + if (new_data->count+1 >= new_data->allocated) { + Py_ssize_t to_allocate = (new_data->count+1)*2; + new_data = + (__Pyx_ModuleStateLookupData*)realloc( + new_data, + sizeof(__Pyx_ModuleStateLookupData) + + (to_allocate-1)*sizeof(__Pyx_InterpreterIdAndModule)); + if (!new_data) { + result = -1; + new_data = old_data; + PyErr_NoMemory(); + goto end; + } + new_data->allocated = to_allocate; + } + ++new_data->count; + int64_t last_id = interpreter_id; + PyObject *last_module = module; + for (Py_ssize_t i=insert_at; icount; ++i) { + int64_t current_id = new_data->table[i].id; + new_data->table[i].id = last_id; + last_id = current_id; + PyObject *current_module = new_data->table[i].module; + new_data->table[i].module = last_module; + last_module = current_module; + } + } + end: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, new_data); +#else + __Pyx_ModuleStateLookup_data = new_data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return result; +} +static int __Pyx_State_RemoveModule(CYTHON_UNUSED void* dummy) { + int64_t interpreter_id = PyInterpreterState_GetID(__Pyx_PyInterpreterState_Get()); + if (interpreter_id == -1) return -1; + __Pyx_ModuleStateLookup_Lock(); +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __Pyx_ModuleStateLookupData *data = (__Pyx_ModuleStateLookupData *) + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, 0); +#else + __Pyx_ModuleStateLookupData *data = __Pyx_ModuleStateLookup_data; +#endif + if (data->interpreter_id_as_index) { + if (interpreter_id < data->count) { + data->table[interpreter_id].module = NULL; + } + goto done; + } + { + __Pyx_ModuleStateLookup_wait_until_no_readers(); + __Pyx_InterpreterIdAndModule* lower_bound = __Pyx_State_FindModuleStateLookupTableLowerBound( + data->table, data->count, interpreter_id); + if (!lower_bound) goto done; + if (lower_bound->id != interpreter_id) goto done; + __Pyx_InterpreterIdAndModule *end = data->table+data->count; + for (;lower_boundid = (lower_bound+1)->id; + lower_bound->module = (lower_bound+1)->module; + } + } + --data->count; + if (data->count == 0) { + free(data); + data = NULL; + } + done: +#if CYTHON_MODULE_STATE_LOOKUP_THREAD_SAFE + __pyx_atomic_pointer_exchange(&__Pyx_ModuleStateLookup_data, data); +#else + __Pyx_ModuleStateLookup_data = data; +#endif + __Pyx_ModuleStateLookup_Unlock(); + return 0; +} +#endif + +/* #### Code section: utility_code_pragmas_end ### */ +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + + + +/* #### Code section: end ### */ +#endif /* Py_PYTHON_H */ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/utils.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/utils.pxd new file mode 100644 index 0000000000000000000000000000000000000000..30f7935afb99d8bef09356bbd0ab9ee8f50e11ee --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/utils.pxd @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +from libc.stdint cimport int32_t, int64_t, intptr_t +from libcpp.vector cimport vector +from libcpp cimport bool as cppbool +from libcpp cimport nullptr_t, nullptr +from libcpp.memory cimport unique_ptr + + +cdef extern from * nogil: + """ + template + class nullable_unique_ptr { + public: + nullable_unique_ptr() noexcept = default; + + nullable_unique_ptr(std::nullptr_t) noexcept = delete; + + explicit nullable_unique_ptr(T* data, bool own_data): + own_data_(own_data) + { + if (own_data) + manager_.reset(data); + else + raw_data_ = data; + } + + nullable_unique_ptr(const nullable_unique_ptr&) = delete; + + nullable_unique_ptr& operator=(const nullable_unique_ptr&) = delete; + + nullable_unique_ptr(nullable_unique_ptr&& other) noexcept + { + own_data_ = other.own_data_; + other.own_data_ = false; // ownership is transferred + if (own_data_) + { + manager_ = std::move(other.manager_); + raw_data_ = nullptr; // just in case + } + else + { + manager_.reset(nullptr); // just in case + raw_data_ = other.raw_data_; + } + } + + nullable_unique_ptr& operator=(nullable_unique_ptr&& other) noexcept + { + own_data_ = other.own_data_; + other.own_data_ = false; // ownership is transferred + if (own_data_) + { + manager_ = std::move(other.manager_); + raw_data_ = nullptr; // just in case + } + else + { + manager_.reset(nullptr); // just in case + raw_data_ = other.raw_data_; + } + return *this; + } + + ~nullable_unique_ptr() = default; + + void reset(T* data, bool own_data) + { + own_data_ = own_data; + if (own_data_) + { + manager_.reset(data); + raw_data_ = nullptr; + } + else + { + manager_.reset(nullptr); + raw_data_ = data; + } + } + + void swap(nullable_unique_ptr& other) noexcept + { + std::swap(manager_, other.manager_); + std::swap(raw_data_, other.raw_data_); + std::swap(own_data_, other.own_data_); + } + + /* + * Get the pointer to the underlying object (this is different from data()!). + */ + T* get() const noexcept + { + if (own_data_) + return manager_.get(); + else + return raw_data_; + } + + /* + * Get the pointer to the underlying buffer (this is different from get()!). + */ + void* data() noexcept + { + if (own_data_) + return manager_.get()->data(); + else + return raw_data_; + } + + T& operator*() + { + if (own_data_) + return *manager_; + else + return *raw_data_; + } + + private: + std::unique_ptr manager_{}; + T* raw_data_{nullptr}; + bool own_data_{false}; + }; + """ + # xref: cython/Cython/Includes/libcpp/memory.pxd + cdef cppclass nullable_unique_ptr[T]: + nullable_unique_ptr() + nullable_unique_ptr(T*, cppbool) + nullable_unique_ptr(nullable_unique_ptr[T]&) + + # Modifiers + void reset(T*, cppbool) + void swap(nullable_unique_ptr&) + + # Observers + T* get() + T& operator*() + void* data() + + +ctypedef fused ResT: + int + int32_t + int64_t + char + float + double + + +ctypedef fused PtrT: + void + + +cdef cppclass nested_resource[T]: + nullable_unique_ptr[ vector[intptr_t] ] ptrs + nullable_unique_ptr[ vector[vector[T]] ] nested_resource_ptr + + +# accepts the output pointer as input to use the return value for exception propagation +cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1 +cdef int get_resource_ptrs(nullable_unique_ptr[ vector[PtrT*] ] &in_out_ptr, object obj, PtrT* __unused) except 1 +cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, ResT* __unused) except 1 + +cdef bint is_nested_sequence(data) +cdef void* get_buffer_pointer(buf, Py_ssize_t size, readonly=*) except* diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/utils.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/utils.pyx new file mode 100644 index 0000000000000000000000000000000000000000..aa78e6cff011015fecca0ba773763c4f5b1ccf70 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_internal/utils.pyx @@ -0,0 +1,129 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +cimport cpython +from libc.stdint cimport intptr_t +from libcpp.utility cimport move +from cython.operator cimport dereference as deref + + +cdef bint is_nested_sequence(data): + if not cpython.PySequence_Check(data): + return False + else: + for i in data: + if not cpython.PySequence_Check(i): + return False + else: + return True + + +cdef void* get_buffer_pointer(buf, Py_ssize_t size, readonly=True) except*: + """The caller must ensure ``buf`` is alive when the returned pointer is in use.""" + cdef void* bufPtr + cdef int flags = cpython.PyBUF_ANY_CONTIGUOUS + if not readonly: + flags |= cpython.PyBUF_WRITABLE + cdef int status = -1 + cdef cpython.Py_buffer view + + if isinstance(buf, int): + bufPtr = buf + else: # try buffer protocol + try: + status = cpython.PyObject_GetBuffer(buf, &view, flags) + # when the caller does not provide a size, it is set to -1 at generate-time by cybind + if size != -1: + assert view.len == size + assert view.ndim == 1 + except Exception as e: + adj = "writable " if not readonly else "" + raise ValueError( + "buf must be either a Python int representing the pointer " + f"address to a valid buffer, or a 1D contiguous {adj}" + "buffer, of size bytes") from e + else: + bufPtr = view.buf + finally: + if status == 0: + cpython.PyBuffer_Release(&view) + + return bufPtr + + +# Cython can't infer the ResT overload when it is wrapped in nullable_unique_ptr, +# so we need a dummy (__unused) input argument to help it +cdef int get_resource_ptr(nullable_unique_ptr[vector[ResT]] &in_out_ptr, object obj, ResT* __unused) except 1: + if cpython.PySequence_Check(obj): + vec = new vector[ResT](len(obj)) + # set the ownership immediately to avoid leaking the `vec` memory in + # case of exception in the following loop + in_out_ptr.reset(vec, True) + for i in range(len(obj)): + deref(vec)[i] = obj[i] + else: + in_out_ptr.reset(obj, False) + return 0 + + +cdef int get_resource_ptrs(nullable_unique_ptr[ vector[PtrT*] ] &in_out_ptr, object obj, PtrT* __unused) except 1: + if cpython.PySequence_Check(obj): + vec = new vector[PtrT*](len(obj)) + # set the ownership immediately to avoid leaking the `vec` memory in + # case of exception in the following loop + in_out_ptr.reset(vec, True) + for i in range(len(obj)): + deref(vec)[i] = (obj[i]) + else: + in_out_ptr.reset(obj, False) + return 0 + + +cdef int get_nested_resource_ptr(nested_resource[ResT] &in_out_ptr, object obj, ResT* __unused) except 1: + cdef nullable_unique_ptr[ vector[intptr_t] ] nested_ptr + cdef nullable_unique_ptr[ vector[vector[ResT]] ] nested_res_ptr + cdef vector[intptr_t]* nested_vec = NULL + cdef vector[vector[ResT]]* nested_res_vec = NULL + cdef size_t i = 0, length = 0 + cdef intptr_t addr + + if is_nested_sequence(obj): + length = len(obj) + nested_res_vec = new vector[vector[ResT]](length) + nested_vec = new vector[intptr_t](length) + # set the ownership immediately to avoid leaking memory in case of + # exception in the following loop + nested_res_ptr.reset(nested_res_vec, True) + nested_ptr.reset(nested_vec, True) + for i, obj_i in enumerate(obj): + if ResT is char: + obj_i_bytes = ((obj_i)).encode() + str_len = (len(obj_i_bytes)) + 1 # including null termination + deref(nested_res_vec)[i].resize(str_len) + obj_i_ptr = (obj_i_bytes) + # cast to size_t explicitly to work around a potentially Cython bug + deref(nested_res_vec)[i].assign(obj_i_ptr, obj_i_ptr + str_len) + else: + deref(nested_res_vec)[i] = obj_i + deref(nested_vec)[i] = (deref(nested_res_vec)[i].data()) + elif cpython.PySequence_Check(obj): + length = len(obj) + nested_vec = new vector[intptr_t](length) + nested_ptr.reset(nested_vec, True) + for i, addr in enumerate(obj): + deref(nested_vec)[i] = addr + nested_res_ptr.reset(NULL, False) + else: + # obj is an int (ResT**) + nested_res_ptr.reset(NULL, False) + nested_ptr.reset(obj, False) + + in_out_ptr.ptrs = move(nested_ptr) + in_out_ptr.nested_resource_ptr = move(nested_res_ptr) + return 0 + + +class FunctionNotFoundError(RuntimeError): pass + +class NotSupportedError(RuntimeError): pass diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..37c006d8960a741697f13f4ccd22f4ca41ac2f25 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/cyruntime/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/cyruntime/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/cyruntime/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/cyruntime/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..35b4059053e59a4f66874d0f5de5e90725891d12 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/cyruntime/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/cyruntime/cyruntime.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/cyruntime/cyruntime.pxd new file mode 100644 index 0000000000000000000000000000000000000000..48f87f29caf4e93aef95c2d197c48064d7f4da19 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/cyruntime/cyruntime.pxd @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +cimport cuda.bindings.cyruntime as cyruntime +cimport cuda.bindings._bindings.cydriver as _cydriver + +# These graphics API are the reimplemented version of what's supported by CUDA Runtime. +# Issue https://github.com/NVIDIA/cuda-python/issues/488 will remove them by letting us +# use call into the static library directly. +# +# This is an ABI breaking change which can only happen in a major version bump. + +# This file is included from cuda/bindings/_bindings/cyruntime.pxd.in but kept in a +# separate file to keep it separated from the auto-generated code there. + +# Prior to https://github.com/NVIDIA/cuda-python/pull/914, this was two +# independent modules (c.b._lib.cyruntime.cyruntime and +# c.b._lib.cyruntime.utils), but was merged into one. + +cdef cudaError_t _cudaEGLStreamProducerPresentFrame(cyruntime.cudaEglStreamConnection* conn, cyruntime.cudaEglFrame eglframe, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaEGLStreamProducerReturnFrame(cyruntime.cudaEglStreamConnection* conn, cyruntime.cudaEglFrame* eglframe, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaGraphicsResourceGetMappedEglFrame(cyruntime.cudaEglFrame* eglFrame, cudaGraphicsResource_t resource, unsigned int index, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaVDPAUSetVDPAUDevice(int device, cyruntime.VdpDevice vdpDevice, cyruntime.VdpGetProcAddress* vdpGetProcAddress) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaVDPAUGetDevice(int* device, cyruntime.VdpDevice vdpDevice, cyruntime.VdpGetProcAddress* vdpGetProcAddress) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaGraphicsVDPAURegisterVideoSurface(cudaGraphicsResource** resource, cyruntime.VdpVideoSurface vdpSurface, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaGraphicsVDPAURegisterOutputSurface(cudaGraphicsResource** resource, cyruntime.VdpOutputSurface vdpSurface, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaGLGetDevices(unsigned int* pCudaDeviceCount, int* pCudaDevices, unsigned int cudaDeviceCount, cyruntime.cudaGLDeviceList deviceList) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaGraphicsGLRegisterImage(cudaGraphicsResource** resource, cyruntime.GLuint image, cyruntime.GLenum target, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaGraphicsGLRegisterBuffer(cudaGraphicsResource** resource, cyruntime.GLuint buffer, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaGraphicsEGLRegisterImage(cudaGraphicsResource_t* pCudaResource, cyruntime.EGLImageKHR image, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaEGLStreamConsumerConnect(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaEGLStreamConsumerConnectWithFlags(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaEGLStreamConsumerDisconnect(cyruntime.cudaEglStreamConnection* conn) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaEGLStreamConsumerAcquireFrame(cyruntime.cudaEglStreamConnection* conn, cudaGraphicsResource_t* pCudaResource, cudaStream_t* pStream, unsigned int timeout) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaEGLStreamConsumerReleaseFrame(cyruntime.cudaEglStreamConnection* conn, cudaGraphicsResource_t pCudaResource, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaEGLStreamProducerConnect(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream, cyruntime.EGLint width, cyruntime.EGLint height) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaEGLStreamProducerDisconnect(cyruntime.cudaEglStreamConnection* conn) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t _cudaEventCreateFromEGLSync(cudaEvent_t* phEvent, cyruntime.EGLSyncKHR eglSync, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil + +# utility functions + +cdef cudaError_t getDriverEglFrame(_cydriver.CUeglFrame *cuEglFrame, cyruntime.cudaEglFrame eglFrame) except ?cudaErrorCallRequiresNewerDriver nogil +cdef cudaError_t getRuntimeEglFrame(cyruntime.cudaEglFrame *eglFrame, _cydriver.CUeglFrame cueglFrame) except ?cudaErrorCallRequiresNewerDriver nogil diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/cyruntime/cyruntime.pxi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/cyruntime/cyruntime.pxi new file mode 100644 index 0000000000000000000000000000000000000000..c18bd1ca2ea4427e3dfc859bceae6e794b307277 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/cyruntime/cyruntime.pxi @@ -0,0 +1,1176 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +# These graphics API are the reimplemented version of what's supported by CUDA Runtime. +# Issue https://github.com/NVIDIA/cuda-python/issues/488 will remove them by letting us +# use call into the static library directly. + +# This file is included from cuda/bindings/_bindings/cyruntime.pyx.in but kept in a +# separate file to keep it separated from the auto-generated code there. + +# Prior to https://github.com/NVIDIA/cuda-python/pull/914, this was two +# independent modules (c.b._lib.cyruntime.cyruntime and +# c.b._lib.cyruntime.utils), but was merged into one. + +from libc.string cimport memset +cimport cuda.bindings._bindings.cydriver as cydriver + +cdef cudaError_t _cudaEGLStreamProducerPresentFrame(cyruntime.cudaEglStreamConnection* conn, cyruntime.cudaEglFrame eglframe, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + cdef cydriver.CUeglFrame cueglFrame + err = getDriverEglFrame(&cueglFrame, eglframe) + if err != cudaSuccess: + return err + err = cydriver._cuEGLStreamProducerPresentFrame(conn, cueglFrame, pStream) + return err + +cdef cudaError_t _cudaEGLStreamProducerReturnFrame(cyruntime.cudaEglStreamConnection* conn, cyruntime.cudaEglFrame* eglframe, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + if eglframe == NULL: + err = cudaErrorInvalidResourceHandle + return err + cdef cydriver.CUeglFrame cueglFrame + err = cydriver._cuEGLStreamProducerReturnFrame(conn, &cueglFrame, pStream) + if err != cudaSuccess: + return err + err = getRuntimeEglFrame(eglframe, cueglFrame) + return err + +cdef cudaError_t _cudaGraphicsResourceGetMappedEglFrame(cyruntime.cudaEglFrame* eglFrame, cudaGraphicsResource_t resource, unsigned int index, unsigned int mipLevel) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + cdef cydriver.CUeglFrame cueglFrame + memset(&cueglFrame, 0, sizeof(cueglFrame)) + err = cydriver._cuGraphicsResourceGetMappedEglFrame(&cueglFrame, resource, index, mipLevel) + if err != cudaSuccess: + return err + err = getRuntimeEglFrame(eglFrame, cueglFrame) + return err + +cdef cudaError_t _cudaVDPAUSetVDPAUDevice(int device, cyruntime.VdpDevice vdpDevice, cyruntime.VdpGetProcAddress* vdpGetProcAddress) except ?cudaErrorCallRequiresNewerDriver nogil: + return cudaErrorNotSupported + +cdef cudaError_t _cudaVDPAUGetDevice(int* device, cyruntime.VdpDevice vdpDevice, cyruntime.VdpGetProcAddress* vdpGetProcAddress) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuVDPAUGetDevice(device, vdpDevice, vdpGetProcAddress) + return err + +cdef cudaError_t _cudaGraphicsVDPAURegisterVideoSurface(cudaGraphicsResource** resource, cyruntime.VdpVideoSurface vdpSurface, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuGraphicsVDPAURegisterVideoSurface(resource, vdpSurface, flags) + return err + +cdef cudaError_t _cudaGraphicsVDPAURegisterOutputSurface(cudaGraphicsResource** resource, cyruntime.VdpOutputSurface vdpSurface, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuGraphicsVDPAURegisterOutputSurface(resource, vdpSurface, flags) + return err + +cdef cudaError_t _cudaGLGetDevices(unsigned int* pCudaDeviceCount, int* pCudaDevices, unsigned int cudaDeviceCount, cyruntime.cudaGLDeviceList deviceList) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuGLGetDevices_v2(pCudaDeviceCount, pCudaDevices, cudaDeviceCount, deviceList) + return err + +cdef cudaError_t _cudaGraphicsGLRegisterImage(cudaGraphicsResource** resource, cyruntime.GLuint image, cyruntime.GLenum target, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuGraphicsGLRegisterImage(resource, image, target, flags) + return err + +cdef cudaError_t _cudaGraphicsGLRegisterBuffer(cudaGraphicsResource** resource, cyruntime.GLuint buffer, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuGraphicsGLRegisterBuffer(resource, buffer, flags) + return err + +cdef cudaError_t _cudaGraphicsEGLRegisterImage(cudaGraphicsResource_t* pCudaResource, cyruntime.EGLImageKHR image, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuGraphicsEGLRegisterImage(pCudaResource, image, flags) + return err + +cdef cudaError_t _cudaEGLStreamConsumerConnect(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuEGLStreamConsumerConnect(conn, eglStream) + return err + +cdef cudaError_t _cudaEGLStreamConsumerConnectWithFlags(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuEGLStreamConsumerConnectWithFlags(conn, eglStream, flags) + return err + +cdef cudaError_t _cudaEGLStreamConsumerDisconnect(cyruntime.cudaEglStreamConnection* conn) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuEGLStreamConsumerDisconnect(conn) + return err + +cdef cudaError_t _cudaEGLStreamConsumerAcquireFrame(cyruntime.cudaEglStreamConnection* conn, cudaGraphicsResource_t* pCudaResource, cudaStream_t* pStream, unsigned int timeout) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuEGLStreamConsumerAcquireFrame(conn, pCudaResource, pStream, timeout) + return err + +cdef cudaError_t _cudaEGLStreamConsumerReleaseFrame(cyruntime.cudaEglStreamConnection* conn, cudaGraphicsResource_t pCudaResource, cudaStream_t* pStream) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuEGLStreamConsumerReleaseFrame(conn, pCudaResource, pStream) + return err + +cdef cudaError_t _cudaEGLStreamProducerConnect(cyruntime.cudaEglStreamConnection* conn, cyruntime.EGLStreamKHR eglStream, cyruntime.EGLint width, cyruntime.EGLint height) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuEGLStreamProducerConnect(conn, eglStream, width, height) + return err + +cdef cudaError_t _cudaEGLStreamProducerDisconnect(cyruntime.cudaEglStreamConnection* conn) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuEGLStreamProducerDisconnect(conn) + return err + +cdef cudaError_t _cudaEventCreateFromEGLSync(cudaEvent_t* phEvent, cyruntime.EGLSyncKHR eglSync, unsigned int flags) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + # cudaFree(0) is a NOP operations that initializes the context state + err = cudaFree(0) + if err != cudaSuccess: + return err + err = cydriver._cuEventCreateFromEGLSync(phEvent, eglSync, flags) + return err + +## utility functions + +cdef int case_desc(const cudaChannelFormatDesc* d, int x, int y, int z, int w, int f) except ?cudaErrorCallRequiresNewerDriver nogil: + return d[0].x == x and d[0].y == y and d[0].z == z and d[0].w == w and d[0].f == f + + +cdef cudaError_t getDescInfo(const cudaChannelFormatDesc* d, int *numberOfChannels, cydriver.CUarray_format *format) except ?cudaErrorCallRequiresNewerDriver nogil: + # Check validity + if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSigned, + cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + if (d[0].x != 8) and (d[0].x != 16) and (d[0].x != 32): + return cudaErrorInvalidChannelDescriptor + elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindFloat,): + if (d[0].x != 16) and (d[0].x != 32): + return cudaErrorInvalidChannelDescriptor + elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12,): + if (d[0].x != 8) or (d[0].y != 8) or (d[0].z != 8) or (d[0].w != 0): + return cudaErrorInvalidChannelDescriptor + elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1, + cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2, + cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X1, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X2, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4,): + if (d[0].x != 8): + return cudaErrorInvalidChannelDescriptor + elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1, + cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2, + cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X1, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X2, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4,): + if (d[0].x != 16): + return cudaErrorInvalidChannelDescriptor + elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4, + cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5, + cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB,): + if (d[0].x != 8): + return cudaErrorInvalidChannelDescriptor + elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, + cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): + if (d[0].x != 16) or (d[0].y != 16) or (d[0].z != 16) or (d[0].w != 0): + return cudaErrorInvalidChannelDescriptor + elif d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102,): + if (d[0].x != 10) or (d[0].y != 10) or (d[0].z != 10) or (d[0].w != 2): + return cudaErrorInvalidChannelDescriptor + else: + return cudaErrorInvalidChannelDescriptor + + # If Y is non-zero, it must match X + # If Z is non-zero, it must match Y + # If W is non-zero, it must match Z + if (((d[0].y != 0) and (d[0].y != d[0].x)) or + ((d[0].z != 0) and (d[0].z != d[0].y)) or + ((d[0].w != 0) and (d[0].w != d[0].z))): + if d[0].f != cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102: + return cudaErrorInvalidChannelDescriptor + if case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + numberOfChannels[0] = 1 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + numberOfChannels[0] = 2 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + numberOfChannels[0] = 3 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindSigned): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT8 + elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + numberOfChannels[0] = 1 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + numberOfChannels[0] = 2 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + numberOfChannels[0] = 3 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT8 + elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + numberOfChannels[0] = 1 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + numberOfChannels[0] = 2 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + numberOfChannels[0] = 3 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindSigned): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT16 + elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + numberOfChannels[0] = 1 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + numberOfChannels[0] = 2 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + numberOfChannels[0] = 3 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT16 + elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + numberOfChannels[0] = 1 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + numberOfChannels[0] = 2 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindSigned): + numberOfChannels[0] = 3 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindSigned): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_SIGNED_INT32 + elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + numberOfChannels[0] = 1 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + numberOfChannels[0] = 2 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + numberOfChannels[0] = 3 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindUnsigned): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNSIGNED_INT32 + elif case_desc(d, 16, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + numberOfChannels[0] = 1 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + elif case_desc(d, 16, 16, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + numberOfChannels[0] = 2 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + numberOfChannels[0] = 3 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + elif case_desc(d, 16, 16, 16, 16, cudaChannelFormatKind.cudaChannelFormatKindFloat): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_HALF + elif case_desc(d, 32, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + numberOfChannels[0] = 1 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + elif case_desc(d, 32, 32, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + numberOfChannels[0] = 2 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + elif case_desc(d, 32, 32, 32, 0, cudaChannelFormatKind.cudaChannelFormatKindFloat): + numberOfChannels[0] = 3 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + elif case_desc(d, 32, 32, 32, 32, cudaChannelFormatKind.cudaChannelFormatKindFloat): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_FLOAT + elif case_desc(d, 8, 8, 8, 0, cudaChannelFormatKind.cudaChannelFormatKindNV12): + numberOfChannels[0] = 3 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_NV12 + elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM + elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC1_UNORM_SRGB + elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM + elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC2_UNORM_SRGB + elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM + elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC3_UNORM_SRGB + elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4): + numberOfChannels[0] = 1 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_UNORM + elif case_desc(d, 8, 0, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4): + numberOfChannels[0] = 1 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC4_SNORM + elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5): + numberOfChannels[0] = 2 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_UNORM + elif case_desc(d, 8, 8, 0, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5): + numberOfChannels[0] = 2 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC5_SNORM + elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H): + numberOfChannels[0] = 3 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_UF16 + elif case_desc(d, 16, 16, 16, 0, cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H): + numberOfChannels[0] = 3 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC6H_SF16 + elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM + elif case_desc(d, 8, 8, 8, 8, cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_BC7_UNORM_SRGB + elif case_desc(d, 10, 10, 10, 2, cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102): + numberOfChannels[0] = 4 + format[0] = cydriver.CUarray_format_enum.CU_AD_FORMAT_UNORM_INT_101010_2 + else: + return cudaErrorInvalidChannelDescriptor + + if d[0].f in (cudaChannelFormatKind.cudaChannelFormatKindNV12, + cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H, + cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H,): + if numberOfChannels[0] != 3: + return cudaErrorInvalidChannelDescriptor + else: + if (numberOfChannels[0] != 1) and (numberOfChannels[0] != 2) and (numberOfChannels[0] != 4): + return cudaErrorInvalidChannelDescriptor + return cudaSuccess + +cdef cudaError_t getChannelFormatDescFromDriverDesc(cudaChannelFormatDesc* pRuntimeDesc, size_t* pDepth, size_t* pHeight, size_t* pWidth, const cydriver.CUDA_ARRAY3D_DESCRIPTOR_v2* pDriverDesc) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef int channel_size = 0 + if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT8: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT16: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + channel_size = 16 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNSIGNED_INT32: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsigned + channel_size = 32 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT8: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT16: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + channel_size = 16 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SIGNED_INT32: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSigned + channel_size = 32 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_HALF: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat + channel_size = 16 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_FLOAT: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindFloat + channel_size = 32 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_NV12: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindNV12 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X1: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X1 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X2: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X2 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT8X4: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized8X4 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X1: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X1 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X2: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X2 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT8X4: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized8X4 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X1: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X1 + channel_size = 16 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X2: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X2 + channel_size = 16 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT16X4: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized16X4 + channel_size = 16 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X1: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X1 + channel_size = 16 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X2: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X2 + channel_size = 16 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_SNORM_INT16X4: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedNormalized16X4 + channel_size = 16 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC1_UNORM_SRGB: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed1SRGB + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC2_UNORM_SRGB: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed2SRGB + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC3_UNORM_SRGB: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed3SRGB + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_UNORM: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed4 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC4_SNORM: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed4 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_UNORM: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed5 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC5_SNORM: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed5 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_UF16: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed6H + channel_size = 16 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC6H_SF16: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindSignedBlockCompressed6H + channel_size = 16 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7 + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_BC7_UNORM_SRGB: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedBlockCompressed7SRGB + channel_size = 8 + elif pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT_101010_2: + pRuntimeDesc[0].f = cudaChannelFormatKind.cudaChannelFormatKindUnsignedNormalized1010102 + else: + return cudaErrorInvalidChannelDescriptor + + # populate bits per channel + pRuntimeDesc[0].x = 0 + pRuntimeDesc[0].y = 0 + pRuntimeDesc[0].z = 0 + pRuntimeDesc[0].w = 0 + + if pDriverDesc[0].Format == cydriver.CU_AD_FORMAT_UNORM_INT_101010_2 and pDriverDesc[0].NumChannels == 4: + pRuntimeDesc[0].w = 2 + pRuntimeDesc[0].z = 10 + pRuntimeDesc[0].y = 10 + pRuntimeDesc[0].x = 10 + else: + if pDriverDesc[0].NumChannels >= 4: + pRuntimeDesc[0].w = channel_size + if pDriverDesc[0].NumChannels >= 3: + pRuntimeDesc[0].z = channel_size + if pDriverDesc[0].NumChannels >= 2: + pRuntimeDesc[0].y = channel_size + if pDriverDesc[0].NumChannels >= 1: + pRuntimeDesc[0].x = channel_size + + if pDriverDesc[0].NumChannels not in (4, 3, 2, 1): + return cudaErrorInvalidChannelDescriptor + + # populate dimensions + if pDepth != NULL: + pDepth[0] = pDriverDesc[0].Depth + if pHeight != NULL: + pHeight[0] = pDriverDesc[0].Height + if pWidth != NULL: + pWidth[0] = pDriverDesc[0].Width + return cudaSuccess + +cdef cudaError_t getDriverEglFrame(cydriver.CUeglFrame *cuEglFrame, cyruntime.cudaEglFrame eglFrame) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + cdef unsigned int i = 0 + + err = getDescInfo(&eglFrame.planeDesc[0].channelDesc, &cuEglFrame[0].numChannels, &cuEglFrame[0].cuFormat) + if err != cudaSuccess: + return err + for i in range(eglFrame.planeCount): + if eglFrame.frameType == cyruntime.cudaEglFrameTypeArray: + cuEglFrame[0].frame.pArray[i] = eglFrame.frame.pArray[i] + else: + cuEglFrame[0].frame.pPitch[i] = eglFrame.frame.pPitch[i].ptr + cuEglFrame[0].width = eglFrame.planeDesc[0].width + cuEglFrame[0].height = eglFrame.planeDesc[0].height + cuEglFrame[0].depth = eglFrame.planeDesc[0].depth + cuEglFrame[0].pitch = eglFrame.planeDesc[0].pitch + cuEglFrame[0].planeCount = eglFrame.planeCount + if eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422Planar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422SemiPlanar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444Planar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444SemiPlanar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUYV422: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_422 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY422: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_422 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY709: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY709_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY2020: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_2020 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatARGB: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ARGB + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatRGBA: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RGBA + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatABGR: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ABGR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBGRA: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BGRA + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatL: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_L + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatR: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_R + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatA: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_A + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatRG: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RG + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatAYUV: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444SemiPlanar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422SemiPlanar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatVYUY_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatUYVY_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUYV_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVYU_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUVA_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatAYUV_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444Planar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422Planar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV444SemiPlanar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV422SemiPlanar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444Planar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422Planar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444SemiPlanar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422SemiPlanar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerRGGB: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RGGB + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerBGGR: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BGGR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerGRBG: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GRBG + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerGBRG: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GBRG + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10RGGB: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_RGGB + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10BGGR: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_BGGR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10GRBG: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GRBG + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10GBRG: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GBRG + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12RGGB: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RGGB + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12BGGR: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BGGR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12GRBG: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GRBG + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12GBRG: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GBRG + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14RGGB: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_RGGB + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14BGGR: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_BGGR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14GRBG: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GRBG + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer14GBRG: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GBRG + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20RGGB: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_RGGB + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20BGGR: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_BGGR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20GRBG: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GRBG + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer20GBRG: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GBRG + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspRGGB: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspBGGR: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspGRBG: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerIspGBRG: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU444Planar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU422Planar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerBCCR: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BCCR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerRCCB: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RCCB + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerCRBC: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CRBC + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayerCBRC: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CBRC + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer10CCCC: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_CCCC + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12BCCR: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BCCR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12RCCB: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RCCB + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CRBC: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CRBC + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CBRC: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CBRC + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatBayer12CCCC: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CCCC + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_2020: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_2020: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_2020: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_2020: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420SemiPlanar_709: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420SemiPlanar_709: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUV420Planar_709: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVU420Planar_709: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_2020: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_2020: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_709: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709 + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY_709_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_709_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10_709_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_709_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12_709_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_709_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYUVA: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatYVYU: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatVYUY: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER + elif eglFrame.eglColorFormat == cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_709_ER: + cuEglFrame[0].eglColorFormat = cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER + else: + return cudaErrorInvalidValue + if eglFrame.frameType == cyruntime.cudaEglFrameTypeArray: + cuEglFrame[0].frameType = cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY + elif eglFrame.frameType == cyruntime.cudaEglFrameTypePitch: + cuEglFrame[0].frameType = cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_PITCH + else: + return cudaErrorInvalidValue + +@cython.show_performance_hints(False) +cdef cudaError_t getRuntimeEglFrame(cyruntime.cudaEglFrame *eglFrame, cydriver.CUeglFrame cueglFrame) except ?cudaErrorCallRequiresNewerDriver nogil: + cdef cudaError_t err = cudaSuccess + cdef unsigned int i + cdef cydriver.CUDA_ARRAY3D_DESCRIPTOR_v2 ad + cdef cudaPitchedPtr pPtr + memset(eglFrame, 0, sizeof(eglFrame[0])) + memset(&ad, 0, sizeof(ad)) + for i in range(cueglFrame.planeCount): + ad.Depth = cueglFrame.depth + ad.Flags = 0 + ad.Format = cueglFrame.cuFormat + ad.Height = cueglFrame.height + ad.NumChannels = cueglFrame.numChannels + ad.Width = cueglFrame.width + + err = getChannelFormatDescFromDriverDesc(&eglFrame[0].planeDesc[i].channelDesc, NULL, NULL, NULL, &ad) + if err != cudaSuccess: + return err + + eglFrame[0].planeDesc[i].depth = cueglFrame.depth + eglFrame[0].planeDesc[i].numChannels = cueglFrame.numChannels + if i == 0: + eglFrame[0].planeDesc[i].width = cueglFrame.width + eglFrame[0].planeDesc[i].height = cueglFrame.height + eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020 or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020 or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709 or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709): + eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + eglFrame[0].planeDesc[i].height = (cueglFrame.height / 2) + eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020 or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020 or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709 or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709 or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER): + eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + eglFrame[0].planeDesc[i].height = (cueglFrame.height / 2) + eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + eglFrame[0].planeDesc[1].channelDesc.y = 8 + if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709 or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020 or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER): + eglFrame[0].planeDesc[1].channelDesc.y = 16 + elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER): + eglFrame[0].planeDesc[i].height = cueglFrame.height + eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709): + eglFrame[0].planeDesc[i].width = (cueglFrame.width / 2) + eglFrame[0].planeDesc[i].height = cueglFrame.height + eglFrame[0].planeDesc[i].pitch = (cueglFrame.pitch / 2) + eglFrame[0].planeDesc[1].channelDesc.y = 8 + if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020 or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709): + eglFrame[0].planeDesc[1].channelDesc.y = 16 + elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER): + eglFrame[0].planeDesc[i].height = cueglFrame.height + eglFrame[0].planeDesc[i].width = cueglFrame.width + eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + elif (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER): + eglFrame[0].planeDesc[i].height = cueglFrame.height + eglFrame[0].planeDesc[i].width = cueglFrame.width + eglFrame[0].planeDesc[i].pitch = cueglFrame.pitch + eglFrame[0].planeDesc[1].channelDesc.y = 8 + if (cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER or + cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER): + eglFrame[0].planeDesc[1].channelDesc.y = 16 + if cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY: + eglFrame[0].frame.pArray[i] = cueglFrame.frame.pArray[i] + else: + pPtr = make_cudaPitchedPtr(cueglFrame.frame.pPitch[i], eglFrame[0].planeDesc[i].pitch, + eglFrame[0].planeDesc[i].width, eglFrame[0].planeDesc[i].height) + eglFrame[0].frame.pPitch[i] = pPtr + + eglFrame[0].planeCount = cueglFrame.planeCount + if cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422Planar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422SemiPlanar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444Planar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444SemiPlanar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_422: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUYV422 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_422: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY422 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY709 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_709_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY709_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_2020: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY2020 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ARGB: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatARGB + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RGBA: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatRGBA + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_ABGR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatABGR + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BGRA: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBGRA + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_L: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatL + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_R: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatR + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_A: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatA + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_RG: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatRG + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatAYUV + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444SemiPlanar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422SemiPlanar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatVYUY_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_UYVY_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatUYVY_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUYV_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUYV_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVYU_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUVA_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_AYUV_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatAYUV_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_PLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444Planar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_PLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422Planar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV444_SEMIPLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV444SemiPlanar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV422_SEMIPLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV422SemiPlanar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444Planar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422Planar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_SEMIPLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444SemiPlanar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_SEMIPLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422SemiPlanar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RGGB: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerRGGB + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BGGR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerBGGR + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GRBG: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerGRBG + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_GBRG: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerGBRG + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_RGGB: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10RGGB + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_BGGR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10BGGR + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GRBG: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10GRBG + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_GBRG: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10GBRG + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RGGB: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12RGGB + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BGGR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12BGGR + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GRBG: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12GRBG + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_GBRG: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12GBRG + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_RGGB: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14RGGB + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_BGGR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14BGGR + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GRBG: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14GRBG + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER14_GBRG: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer14GBRG + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_RGGB: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20RGGB + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_BGGR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20BGGR + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GRBG: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20GRBG + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER20_GBRG: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer20GBRG + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_RGGB: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspRGGB + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_BGGR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspBGGR + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GRBG: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspGRBG + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_ISP_GBRG: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerIspGBRG + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU444_PLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU444Planar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU422_PLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU422Planar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_BCCR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerBCCR + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_RCCB: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerRCCB + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CRBC: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerCRBC + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER_CBRC: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayerCBRC + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER10_CCCC: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer10CCCC + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_BCCR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12BCCR + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_RCCB: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12RCCB + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CRBC: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CRBC + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CBRC: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CBRC + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_BAYER12_CCCC: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatBayer12CCCC + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_2020: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_2020 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_2020: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_2020 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_2020: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_2020 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_2020: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_2020 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_SEMIPLANAR_709: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420SemiPlanar_709 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_SEMIPLANAR_709: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420SemiPlanar_709 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUV420_PLANAR_709: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUV420Planar_709 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVU420_PLANAR_709: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVU420Planar_709 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_2020: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_2020 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_2020: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_2020 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_422_SEMIPLANAR_709: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_422SemiPlanar_709 + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y_709_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY_709_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10_709_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10_709_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12_709_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12_709_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YUVA: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYUVA + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_YVYU: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatYVYU + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_VYUY: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatVYUY + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_420_SEMIPLANAR_709_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_420SemiPlanar_709_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y10V10U10_444_SEMIPLANAR_709_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY10V10U10_444SemiPlanar_709_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_420_SEMIPLANAR_709_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_420SemiPlanar_709_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_ER + elif cueglFrame.eglColorFormat == cydriver.CUeglColorFormat_enum.CU_EGL_COLOR_FORMAT_Y12V12U12_444_SEMIPLANAR_709_ER: + eglFrame[0].eglColorFormat = cyruntime.cudaEglColorFormatY12V12U12_444SemiPlanar_709_ER + else: + return cudaErrorInvalidValue + if cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_ARRAY: + eglFrame[0].frameType = cyruntime.cudaEglFrameTypeArray + elif cueglFrame.frameType == cydriver.CUeglFrameType_enum.CU_EGL_FRAME_TYPE_PITCH: + eglFrame[0].frameType = cyruntime.cudaEglFrameTypePitch + else: + return cudaErrorInvalidValue diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/dlfcn.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/dlfcn.pxd new file mode 100644 index 0000000000000000000000000000000000000000..2ae95814396cab6134db80754e52e6f7b68bf979 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/dlfcn.pxd @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +cdef extern from "" nogil: + void *dlopen(const char *, int) + char *dlerror() + void *dlsym(void *, const char *) + int dlclose(void *) + + enum: + RTLD_LAZY + RTLD_NOW + RTLD_GLOBAL + RTLD_LOCAL diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/param_packer.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/param_packer.h new file mode 100644 index 0000000000000000000000000000000000000000..96c56b4fe4e0f2d3ec4d7dc87d7f66b3df545d79 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/param_packer.h @@ -0,0 +1,152 @@ +// SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +// Please refer to the NVIDIA end user license agreement (EULA) associated +// with this source code for terms and conditions that govern your use of +// this software. Any use, reproduction, disclosure, or distribution of +// this software and related documentation outside the terms of the EULA +// is strictly prohibited. + +#include + +#include +#include +#include +#include + +static PyObject* ctypes_module = nullptr; + +static PyTypeObject* ctypes_c_char = nullptr; +static PyTypeObject* ctypes_c_bool = nullptr; +static PyTypeObject* ctypes_c_wchar = nullptr; +static PyTypeObject* ctypes_c_byte = nullptr; +static PyTypeObject* ctypes_c_ubyte = nullptr; +static PyTypeObject* ctypes_c_short = nullptr; +static PyTypeObject* ctypes_c_ushort = nullptr; +static PyTypeObject* ctypes_c_int = nullptr; +static PyTypeObject* ctypes_c_uint = nullptr; +static PyTypeObject* ctypes_c_long = nullptr; +static PyTypeObject* ctypes_c_ulong = nullptr; +static PyTypeObject* ctypes_c_longlong = nullptr; +static PyTypeObject* ctypes_c_ulonglong = nullptr; +static PyTypeObject* ctypes_c_size_t = nullptr; +static PyTypeObject* ctypes_c_float = nullptr; +static PyTypeObject* ctypes_c_double = nullptr; +static PyTypeObject* ctypes_c_void_p = nullptr; + +static void fetch_ctypes() +{ + ctypes_module = PyImport_ImportModule("ctypes"); + if (ctypes_module == nullptr) + throw std::runtime_error("Cannot import ctypes module"); + // get method addressof + PyObject* ctypes_dict = PyModule_GetDict(ctypes_module); + if (ctypes_dict == nullptr) + throw std::runtime_error(std::string("FAILURE @ ") + std::string(__FILE__) + " : " + std::to_string(__LINE__)); + // supportedtypes + ctypes_c_char = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_char"); + ctypes_c_bool = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_bool"); + ctypes_c_wchar = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_wchar"); + ctypes_c_byte = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_byte"); + ctypes_c_ubyte = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_ubyte"); + ctypes_c_short = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_short"); + ctypes_c_ushort = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_ushort"); + ctypes_c_int = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_int"); + ctypes_c_uint = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_uint"); + ctypes_c_long = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_long"); + ctypes_c_ulong = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_ulong"); + ctypes_c_longlong = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_longlong"); + ctypes_c_ulonglong = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_ulonglong"); + ctypes_c_size_t = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_size_t"); + ctypes_c_float = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_float"); + ctypes_c_double = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_double"); + ctypes_c_void_p = (PyTypeObject*) PyDict_GetItemString(ctypes_dict, "c_void_p"); // == c_voidp +} + + +// (target type, source type) +static std::map, std::function> m_feeders; + +static void populate_feeders(PyTypeObject* target_t, PyTypeObject* source_t) +{ + if (target_t == ctypes_c_int) + { + if (source_t == &PyLong_Type) + { + m_feeders[{target_t,source_t}] = [](void* ptr, PyObject* value) -> int + { + *((int*)ptr) = (int)PyLong_AsLong(value); + return sizeof(int); + }; + return; + } + } else if (target_t == ctypes_c_bool) { + if (source_t == &PyBool_Type) + { + m_feeders[{target_t,source_t}] = [](void* ptr, PyObject* value) -> int + { + *((bool*)ptr) = (value == Py_True); + return sizeof(bool); + }; + return; + } + } else if (target_t == ctypes_c_byte) { + if (source_t == &PyLong_Type) + { + m_feeders[{target_t,source_t}] = [](void* ptr, PyObject* value) -> int + { + *((int8_t*)ptr) = (int8_t)PyLong_AsLong(value); + return sizeof(int8_t); + }; + return; + } + } else if (target_t == ctypes_c_double) { + if (source_t == &PyFloat_Type) + { + m_feeders[{target_t,source_t}] = [](void* ptr, PyObject* value) -> int + { + *((double*)ptr) = (double)PyFloat_AsDouble(value); + return sizeof(double); + }; + return; + } + } else if (target_t == ctypes_c_float) { + if (source_t == &PyFloat_Type) + { + m_feeders[{target_t,source_t}] = [](void* ptr, PyObject* value) -> int + { + *((float*)ptr) = (float)PyFloat_AsDouble(value); + return sizeof(float); + }; + return; + } + } else if (target_t == ctypes_c_longlong) { + if (source_t == &PyLong_Type) + { + m_feeders[{target_t,source_t}] = [](void* ptr, PyObject* value) -> int + { + *((long long*)ptr) = (long long)PyLong_AsLongLong(value); + return sizeof(long long); + }; + return; + } + } +} + +static int feed(void* ptr, PyObject* value, PyObject* type) +{ + PyTypeObject* pto = (PyTypeObject*)type; + if (ctypes_c_int == nullptr) + fetch_ctypes(); + auto found = m_feeders.find({pto,value->ob_type}); + if (found == m_feeders.end()) + { + populate_feeders(pto, value->ob_type); + found = m_feeders.find({pto,value->ob_type}); + } + if (found != m_feeders.end()) + { + return found->second(ptr, value); + } + return 0; +} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/param_packer.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/param_packer.pxd new file mode 100644 index 0000000000000000000000000000000000000000..ad7fd95668fb3065b8ffa142ddad22970d23ed55 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/param_packer.pxd @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +# Include "param_packer.h" so its contents get compiled into every +# Cython extension module that depends on param_packer.pxd. +cdef extern from "param_packer.h": + int feed(void* ptr, object o, object ct) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/utils.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/utils.pxd new file mode 100644 index 0000000000000000000000000000000000000000..514b0658466ea975ce2cc8ceff5778633940cd82 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/utils.pxd @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +cimport cuda.bindings.driver as driver +cimport cuda.bindings.cydriver as cydriver +cimport cuda.bindings.cyruntime as cyruntime +from libcpp.vector cimport vector + +cdef class _HelperKernelParams: + cdef Py_buffer _pybuffer + cdef bint _pyobj_acquired + cdef void** _ckernelParams + cdef char* _ckernelParamsData + cdef int _length + cdef bint _malloc_list_created + +cdef class _HelperInputVoidPtr: + cdef Py_buffer _pybuffer + cdef void* _cptr + cdef bint _pyobj_acquired + +cdef class _HelperCUmemPool_attribute: + cdef void* _cptr + cdef cydriver.CUmemPool_attribute_enum _attr + cdef bint _is_getter + + # Return values + cdef int _int_val + cdef driver.cuuint64_t _cuuint64_t_val + +cdef class _HelperCUmem_range_attribute: + cdef void* _cptr + cdef cydriver.CUmem_range_attribute_enum _attr + cdef size_t _data_size + + # Return values + cdef int _int_val # 32 bit integer + cdef int* _int_val_list # 32 bit integer array + +cdef class _HelperCUpointer_attribute: + cdef void* _cptr + cdef cydriver.CUpointer_attribute_enum _attr + cdef bint _is_getter + + # Return values + cdef driver.CUcontext _ctx + cdef unsigned int _uint + cdef driver.CUdeviceptr _devptr + cdef void** _void + cdef driver.CUDA_POINTER_ATTRIBUTE_P2P_TOKENS _token + cdef bint _bool + cdef unsigned long long _ull + cdef size_t _size + cdef driver.CUmemoryPool _mempool + +cdef class _HelperCUgraphMem_attribute: + cdef void* _cptr + cdef cydriver.CUgraphMem_attribute_enum _attr + cdef bint _is_getter + + # Return values + cdef driver.cuuint64_t _cuuint64_t_val + +cdef class _HelperCUjit_option: + cdef void* _cptr + cdef cydriver.CUjit_option_enum _attr + + # Return values + cdef unsigned int _uint + cdef float _float + cdef char* _charstar + cdef cydriver.CUjit_target_enum _target + cdef cydriver.CUjit_fallback_enum _fallback + cdef int _int + cdef cydriver.CUjit_cacheMode_enum _cacheMode + cdef vector[char*] _charstarstar # list of names + cdef _InputVoidPtrPtrHelper _voidstarstar # list of addresses + +cdef class _HelperCudaJitOption: + cdef void* _cptr + cdef cyruntime.cudaJitOption _attr + + # Return values + cdef unsigned int _uint + cdef float _float + cdef char* _charstar + cdef cyruntime.cudaJit_Fallback _fallback + cdef int _int + cdef cyruntime.cudaJit_CacheMode _cacheMode + +cdef class _HelperCUlibraryOption: + cdef void* _cptr + cdef cydriver.CUlibraryOption_enum _attr + + # Return values + cdef unsigned int _uint + +cdef class _HelperCudaLibraryOption: + cdef void* _cptr + cdef cyruntime.cudaLibraryOption _attr + + # Return values + cdef unsigned int _uint + +cdef class _HelperCUmemAllocationHandleType: + cdef void* _cptr + cdef cydriver.CUmemAllocationHandleType_enum _type + + # Return values + cdef int _int + cdef void* _handle + cdef unsigned int _d3dkmt_handle + cdef driver.CUmemFabricHandle _mem_fabric_handle + +cdef class _InputVoidPtrPtrHelper: + cdef void** _cptr + +cdef class _HelperCUcoredumpSettings: + cdef void* _cptr + cdef cydriver.CUcoredumpSettings_enum _attrib + cdef bint _is_getter + cdef size_t _size + + # Return values + cdef bint _bool + cdef char* _charstar diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/utils.pxi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/utils.pxi new file mode 100644 index 0000000000000000000000000000000000000000..1084d3c948e42b9f7a88a1a49864d0a62c0afdcc --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/utils.pxi @@ -0,0 +1,604 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +from cpython.buffer cimport PyObject_CheckBuffer, PyObject_GetBuffer, PyBuffer_Release, PyBUF_SIMPLE, PyBUF_ANY_CONTIGUOUS +from libc.stdlib cimport calloc, free +from libc.stdint cimport int32_t, uint32_t, int64_t, uint64_t +from libc.stddef cimport wchar_t +from libc.string cimport memcpy +from enum import Enum as _Enum +import ctypes as _ctypes +cimport cuda.bindings.cydriver as cydriver +import cuda.bindings.driver as _driver +cimport cuda.bindings._lib.param_packer as param_packer + +cdef void* _callocWrapper(length, size): + cdef void* out = calloc(length, size) + if out is NULL: + raise MemoryError('Failed to allocated length x size memory: {}x{}'.format(length, size)) + return out + +cdef class _HelperKernelParams: + supported_types = { # excluding void_p and None, which are handled specially + _ctypes.c_bool, + _ctypes.c_char, + _ctypes.c_wchar, + _ctypes.c_byte, + _ctypes.c_ubyte, + _ctypes.c_short, + _ctypes.c_ushort, + _ctypes.c_int, + _ctypes.c_uint, + _ctypes.c_long, + _ctypes.c_ulong, + _ctypes.c_longlong, + _ctypes.c_ulonglong, + _ctypes.c_size_t, + _ctypes.c_float, + _ctypes.c_double + } + + max_param_size = max(_ctypes.sizeof(max(_HelperKernelParams.supported_types, key=lambda t:_ctypes.sizeof(t))), sizeof(void_ptr)) + + def __cinit__(self, kernelParams): + self._pyobj_acquired = False + self._malloc_list_created = False + if kernelParams is None: + self._ckernelParams = NULL + elif isinstance(kernelParams, (int)): + # Easy run, user gave us an already configured void** address + self._ckernelParams = kernelParams + elif PyObject_CheckBuffer(kernelParams): + # Easy run, get address from Python Buffer Protocol + err_buffer = PyObject_GetBuffer(kernelParams, &self._pybuffer, PyBUF_SIMPLE | PyBUF_ANY_CONTIGUOUS) + if err_buffer == -1: + raise RuntimeError("Argument 'kernelParams' failed to retrieve buffer through Buffer Protocol") + self._pyobj_acquired = True + self._ckernelParams = self._pybuffer.buf + elif isinstance(kernelParams, (tuple)) and len(kernelParams) == 2 and isinstance(kernelParams[0], (tuple)) and isinstance(kernelParams[1], (tuple)): + # Hard run, construct and fill out contigues memory using provided kernel values and types based + if len(kernelParams[0]) != len(kernelParams[1]): + raise TypeError("Argument 'kernelParams' has tuples with different length") + if len(kernelParams[0]) != 0: + self._length = len(kernelParams[0]) + self._ckernelParams = _callocWrapper(len(kernelParams[0]), sizeof(void*)) + self._ckernelParamsData = _callocWrapper(len(kernelParams[0]), _HelperKernelParams.max_param_size) + self._malloc_list_created = True + + idx = 0 + data_idx = 0 + for value, ctype in zip(kernelParams[0], kernelParams[1]): + if ctype is None: + # special cases for None + if callable(getattr(value, 'getPtr', None)): + self._ckernelParams[idx] = value.getPtr() + elif isinstance(value, (_ctypes.Structure)): + self._ckernelParams[idx] = _ctypes.addressof(value) + elif isinstance(value, (_Enum)): + self._ckernelParams[idx] = &(self._ckernelParamsData[data_idx]) + (self._ckernelParams[idx])[0] = value.value + data_idx += sizeof(int) + else: + raise TypeError("Provided argument is of type {} but expected Type {}, {} or CUDA Binding structure with getPtr() attribute".format(type(value), type(_ctypes.Structure), type(_ctypes.c_void_p))) + elif ctype in _HelperKernelParams.supported_types: + self._ckernelParams[idx] = &(self._ckernelParamsData[data_idx]) + + # handle case where a float is passed as a double + if ctype == _ctypes.c_double and isinstance(value, _ctypes.c_float): + value = ctype(value.value) + if not isinstance(value, ctype): # make it a ctype + size = param_packer.feed(self._ckernelParams[idx], value, ctype) + if size == 0: # feed failed + value = ctype(value) + size = _ctypes.sizeof(ctype) + addr = (_ctypes.addressof(value)) + memcpy(self._ckernelParams[idx], addr, size) + else: + size = _ctypes.sizeof(ctype) + addr = (_ctypes.addressof(value)) + memcpy(self._ckernelParams[idx], addr, size) + data_idx += size + elif ctype == _ctypes.c_void_p: + # special cases for void_p + if isinstance(value, (int, _ctypes.c_void_p)): + self._ckernelParams[idx] = &(self._ckernelParamsData[data_idx]) + (self._ckernelParams[idx])[0] = value.value if isinstance(value, (_ctypes.c_void_p)) else value + data_idx += sizeof(void_ptr) + elif callable(getattr(value, 'getPtr', None)): + self._ckernelParams[idx] = &(self._ckernelParamsData[data_idx]) + (self._ckernelParams[idx])[0] = value.getPtr() + data_idx += sizeof(void_ptr) + else: + raise TypeError("Provided argument is of type {} but expected Type {}, {} or CUDA Binding structure with getPtr() attribute".format(type(value), type(int), type(_ctypes.c_void_p))) + else: + raise TypeError("Unsupported type: " + str(type(ctype))) + idx += 1 + else: + raise TypeError("Argument 'kernelParams' is not a valid type: tuple[tuple[Any, ...], tuple[Any, ...]] or PyObject implimenting Buffer Protocol or Int") + + def __dealloc__(self): + if self._pyobj_acquired is True: + PyBuffer_Release(&self._pybuffer) + if self._malloc_list_created is True: + free(self._ckernelParams) + free(self._ckernelParamsData) + + @property + def ckernelParams(self): + return self._ckernelParams + +cdef class _HelperInputVoidPtr: + def __cinit__(self, ptr): + self._pyobj_acquired = False + if ptr is None: + self._cptr = NULL + elif isinstance(ptr, (int)): + # Easy run, user gave us an already configured void** address + self._cptr = ptr + elif isinstance(ptr, (_driver.CUdeviceptr)): + self._cptr = int(ptr) + elif PyObject_CheckBuffer(ptr): + # Easy run, get address from Python Buffer Protocol + err_buffer = PyObject_GetBuffer(ptr, &self._pybuffer, PyBUF_SIMPLE | PyBUF_ANY_CONTIGUOUS) + if err_buffer == -1: + raise RuntimeError("Failed to retrieve buffer through Buffer Protocol") + self._pyobj_acquired = True + self._cptr = self._pybuffer.buf + else: + raise TypeError("Provided argument is of type {} but expected Type {}, {} or object with Buffer Protocol".format(type(ptr), type(None), type(int))) + + def __dealloc__(self): + if self._pyobj_acquired is True: + PyBuffer_Release(&self._pybuffer) + + @property + def cptr(self): + return self._cptr + +cdef class _HelperCUmemPool_attribute: + def __cinit__(self, attr, init_value, is_getter=False): + self._is_getter = is_getter + self._attr = attr.value + if self._attr in (cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_REUSE_FOLLOW_EVENT_DEPENDENCIES, + cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_REUSE_ALLOW_OPPORTUNISTIC, + cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_REUSE_ALLOW_INTERNAL_DEPENDENCIES,): + self._int_val = init_value + self._cptr = &self._int_val + elif self._attr in (cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_RELEASE_THRESHOLD, + cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_RESERVED_MEM_CURRENT, + cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_RESERVED_MEM_HIGH, + cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_USED_MEM_CURRENT, + cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_USED_MEM_HIGH,): + if self._is_getter: + self._cuuint64_t_val = _driver.cuuint64_t() + self._cptr = self._cuuint64_t_val.getPtr() + else: + self._cptr = init_value.getPtr() + else: + raise TypeError('Unsupported attribute: {}'.format(attr.name)) + + def __dealloc__(self): + pass + + @property + def cptr(self): + return self._cptr + + def pyObj(self): + assert(self._is_getter == True) + if self._attr in (cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_REUSE_FOLLOW_EVENT_DEPENDENCIES, + cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_REUSE_ALLOW_OPPORTUNISTIC, + cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_REUSE_ALLOW_INTERNAL_DEPENDENCIES,): + return self._int_val + elif self._attr in (cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_RELEASE_THRESHOLD, + cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_RESERVED_MEM_CURRENT, + cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_RESERVED_MEM_HIGH, + cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_USED_MEM_CURRENT, + cydriver.CUmemPool_attribute_enum.CU_MEMPOOL_ATTR_USED_MEM_HIGH,): + return self._cuuint64_t_val + else: + raise TypeError('Unsupported attribute value: {}'.format(self._attr)) + +cdef class _HelperCUmem_range_attribute: + def __cinit__(self, attr, data_size): + self._data_size = data_size + self._attr = attr.value + if self._attr in (cydriver.CUmem_range_attribute_enum.CU_MEM_RANGE_ATTRIBUTE_READ_MOSTLY, + cydriver.CUmem_range_attribute_enum.CU_MEM_RANGE_ATTRIBUTE_PREFERRED_LOCATION, + cydriver.CUmem_range_attribute_enum.CU_MEM_RANGE_ATTRIBUTE_LAST_PREFETCH_LOCATION,): + self._cptr = &self._int_val + elif self._attr in (cydriver.CUmem_range_attribute_enum.CU_MEM_RANGE_ATTRIBUTE_ACCESSED_BY,): + self._cptr = _callocWrapper(1, self._data_size) + self._int_val_list = self._cptr + else: + raise TypeError('Unsupported attribute: {}'.format(attr.name)) + + def __dealloc__(self): + if self._attr in (cydriver.CUmem_range_attribute_enum.CU_MEM_RANGE_ATTRIBUTE_ACCESSED_BY,): + free(self._cptr) + + @property + def cptr(self): + return self._cptr + + def pyObj(self): + if self._attr in (cydriver.CUmem_range_attribute_enum.CU_MEM_RANGE_ATTRIBUTE_READ_MOSTLY, + cydriver.CUmem_range_attribute_enum.CU_MEM_RANGE_ATTRIBUTE_PREFERRED_LOCATION, + cydriver.CUmem_range_attribute_enum.CU_MEM_RANGE_ATTRIBUTE_LAST_PREFETCH_LOCATION,): + return self._int_val + elif self._attr in (cydriver.CUmem_range_attribute_enum.CU_MEM_RANGE_ATTRIBUTE_ACCESSED_BY,): + return [self._int_val_list[idx] for idx in range(int(self._data_size/4))] + else: + raise TypeError('Unsupported attribute value: {}'.format(self._attr)) + +cdef class _HelperCUpointer_attribute: + def __cinit__(self, attr, init_value, is_getter=False): + self._is_getter = is_getter + self._attr = attr.value + if self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_CONTEXT,): + if self._is_getter: + self._ctx = _driver.CUcontext() + self._cptr = self._ctx.getPtr() + else: + self._cptr = init_value.getPtr() + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_MEMORY_TYPE, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_DEVICE_ORDINAL, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_ALLOWED_HANDLE_TYPES, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_IS_GPU_DIRECT_RDMA_CAPABLE, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_ACCESS_FLAGS,): + self._uint = init_value + self._cptr = &self._uint + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_DEVICE_POINTER, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_RANGE_START_ADDR,): + if self._is_getter: + self._devptr = _driver.CUdeviceptr() + self._cptr = self._devptr.getPtr() + else: + self._cptr = init_value.getPtr() + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_HOST_POINTER,): + self._void = init_value + self._cptr = &self._void + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_P2P_TOKENS,): + if self._is_getter: + self._token = _driver.CUDA_POINTER_ATTRIBUTE_P2P_TOKENS() + self._cptr = self._token.getPtr() + else: + self._cptr = init_value.getPtr() + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_SYNC_MEMOPS, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_IS_MANAGED, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_IS_LEGACY_CUDA_IPC_CAPABLE, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_MAPPED,): + self._bool = init_value + self._cptr = &self._bool + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_BUFFER_ID,): + self._ull = init_value + self._cptr = &self._ull + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_RANGE_SIZE,): + self._size = init_value + self._cptr = &self._size + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE,): + if self._is_getter: + self._mempool = _driver.CUmemoryPool() + self._cptr = self._mempool.getPtr() + else: + self._cptr = init_value.getPtr() + else: + raise TypeError('Unsupported attribute: {}'.format(attr.name)) + + def __dealloc__(self): + pass + + @property + def cptr(self): + return self._cptr + + def pyObj(self): + assert(self._is_getter == True) + if self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_CONTEXT,): + return self._ctx + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_MEMORY_TYPE, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_DEVICE_ORDINAL, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_ALLOWED_HANDLE_TYPES, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_IS_GPU_DIRECT_RDMA_CAPABLE, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_ACCESS_FLAGS,): + return self._uint + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_DEVICE_POINTER, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_RANGE_START_ADDR,): + return self._devptr + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_HOST_POINTER,): + return self._void + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_P2P_TOKENS,): + return self._token + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_SYNC_MEMOPS, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_IS_MANAGED, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_IS_LEGACY_CUDA_IPC_CAPABLE, + cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_MAPPED,): + return self._bool + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_BUFFER_ID,): + return self._ull + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_RANGE_SIZE,): + return self._size + elif self._attr in (cydriver.CUpointer_attribute_enum.CU_POINTER_ATTRIBUTE_MEMPOOL_HANDLE,): + return self._mempool + else: + raise TypeError('Unsupported attribute value: {}'.format(self._attr)) + +cdef class _HelperCUgraphMem_attribute: + def __cinit__(self, attr, init_value, is_getter=False): + self._is_getter = is_getter + self._attr = attr.value + if self._attr in (cydriver.CUgraphMem_attribute_enum.CU_GRAPH_MEM_ATTR_USED_MEM_CURRENT, + cydriver.CUgraphMem_attribute_enum.CU_GRAPH_MEM_ATTR_USED_MEM_HIGH, + cydriver.CUgraphMem_attribute_enum.CU_GRAPH_MEM_ATTR_RESERVED_MEM_CURRENT, + cydriver.CUgraphMem_attribute_enum.CU_GRAPH_MEM_ATTR_RESERVED_MEM_HIGH,): + if self._is_getter: + self._cuuint64_t_val = _driver.cuuint64_t() + self._cptr = self._cuuint64_t_val.getPtr() + else: + self._cptr = init_value.getPtr() + else: + raise TypeError('Unsupported attribute: {}'.format(attr.name)) + + def __dealloc__(self): + pass + + @property + def cptr(self): + return self._cptr + + def pyObj(self): + assert(self._is_getter == True) + if self._attr in (cydriver.CUgraphMem_attribute_enum.CU_GRAPH_MEM_ATTR_USED_MEM_CURRENT, + cydriver.CUgraphMem_attribute_enum.CU_GRAPH_MEM_ATTR_USED_MEM_HIGH, + cydriver.CUgraphMem_attribute_enum.CU_GRAPH_MEM_ATTR_RESERVED_MEM_CURRENT, + cydriver.CUgraphMem_attribute_enum.CU_GRAPH_MEM_ATTR_RESERVED_MEM_HIGH,): + return self._cuuint64_t_val + else: + raise TypeError('Unsupported attribute value: {}'.format(self._attr)) + +cdef class _HelperCUjit_option: + def __cinit__(self, attr, init_value): + self._attr = attr.value + if self._attr in (cydriver.CUjit_option_enum.CU_JIT_MAX_REGISTERS, + cydriver.CUjit_option_enum.CU_JIT_THREADS_PER_BLOCK, + cydriver.CUjit_option_enum.CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES, + cydriver.CUjit_option_enum.CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES, + cydriver.CUjit_option_enum.CU_JIT_OPTIMIZATION_LEVEL, + cydriver.CUjit_option_enum.CU_JIT_GLOBAL_SYMBOL_COUNT, + cydriver.CUjit_option_enum.CU_JIT_TARGET_FROM_CUCONTEXT, + cydriver.CUjit_option_enum.CU_JIT_REFERENCED_KERNEL_COUNT, + cydriver.CUjit_option_enum.CU_JIT_REFERENCED_VARIABLE_COUNT, + cydriver.CUjit_option_enum.CU_JIT_MIN_CTA_PER_SM,): + self._uint = init_value + self._cptr = self._uint + elif self._attr in (cydriver.CUjit_option_enum.CU_JIT_WALL_TIME,): + self._float = init_value + self._cptr = self._float + elif self._attr in (cydriver.CUjit_option_enum.CU_JIT_INFO_LOG_BUFFER, + cydriver.CUjit_option_enum.CU_JIT_ERROR_LOG_BUFFER): + self._charstar = init_value + self._cptr = self._charstar + elif self._attr in (cydriver.CUjit_option_enum.CU_JIT_TARGET,): + self._target = init_value.value + self._cptr = self._target + elif self._attr in (cydriver.CUjit_option_enum.CU_JIT_FALLBACK_STRATEGY,): + self._fallback = init_value.value + self._cptr = self._fallback + elif self._attr in (cydriver.CUjit_option_enum.CU_JIT_GENERATE_DEBUG_INFO, + cydriver.CUjit_option_enum.CU_JIT_LOG_VERBOSE, + cydriver.CUjit_option_enum.CU_JIT_GENERATE_LINE_INFO, + cydriver.CUjit_option_enum.CU_JIT_LTO, + cydriver.CUjit_option_enum.CU_JIT_FTZ, + cydriver.CUjit_option_enum.CU_JIT_PREC_DIV, + cydriver.CUjit_option_enum.CU_JIT_PREC_SQRT, + cydriver.CUjit_option_enum.CU_JIT_FMA, + cydriver.CUjit_option_enum.CU_JIT_OPTIMIZE_UNUSED_DEVICE_VARIABLES,): + self._int = init_value + self._cptr = self._int + elif self._attr in (cydriver.CUjit_option_enum.CU_JIT_CACHE_MODE,): + self._cacheMode = init_value.value + self._cptr = self._cacheMode + elif self._attr in (cydriver.CUjit_option_enum.CU_JIT_GLOBAL_SYMBOL_NAMES, + cydriver.CUjit_option_enum.CU_JIT_REFERENCED_KERNEL_NAMES, + cydriver.CUjit_option_enum.CU_JIT_REFERENCED_VARIABLE_NAMES,): + self._charstarstar = init_value + self._cptr = &self._charstarstar[0] + elif self._attr in (cydriver.CUjit_option_enum.CU_JIT_GLOBAL_SYMBOL_ADDRESSES,): + pylist = [_HelperInputVoidPtr(val) for val in init_value] + self._voidstarstar = _InputVoidPtrPtrHelper(pylist) + self._cptr = self._voidstarstar.cptr + else: + raise TypeError('Unsupported attribute: {}'.format(attr.name)) + + def __dealloc__(self): + pass + + @property + def cptr(self): + return self._cptr + +cdef class _HelperCudaJitOption: + def __cinit__(self, attr, init_value): + self._attr = attr.value + if self._attr in (cyruntime.cudaJitOption.cudaJitMaxRegisters, + cyruntime.cudaJitOption.cudaJitThreadsPerBlock, + cyruntime.cudaJitOption.cudaJitInfoLogBufferSizeBytes, + cyruntime.cudaJitOption.cudaJitErrorLogBufferSizeBytes, + cyruntime.cudaJitOption.cudaJitOptimizationLevel, + cyruntime.cudaJitOption.cudaJitMinCtaPerSm,): + self._uint = init_value + self._cptr = self._uint + elif self._attr in (cyruntime.cudaJitOption.cudaJitWallTime,): + self._float = init_value + self._cptr = self._float + elif self._attr in (cyruntime.cudaJitOption.cudaJitInfoLogBuffer, + cyruntime.cudaJitOption.cudaJitErrorLogBuffer): + self._charstar = init_value + self._cptr = self._charstar + elif self._attr in (cyruntime.cudaJitOption.cudaJitFallbackStrategy,): + self._fallback = init_value.value + self._cptr = self._fallback + elif self._attr in (cyruntime.cudaJitOption.cudaJitGenerateDebugInfo, + cyruntime.cudaJitOption.cudaJitLogVerbose, + cyruntime.cudaJitOption.cudaJitGenerateLineInfo, + cyruntime.cudaJitOption.cudaJitPositionIndependentCode, + cyruntime.cudaJitOption.cudaJitMaxThreadsPerBlock, + cyruntime.cudaJitOption.cudaJitOverrideDirectiveValues,): + self._int = init_value + self._cptr = self._int + elif self._attr in (cyruntime.cudaJitOption.cudaJitCacheMode,): + self._cacheMode = init_value.value + self._cptr = self._cacheMode + else: + raise TypeError('Unsupported attribute: {}'.format(attr.name)) + + def __dealloc__(self): + pass + + @property + def cptr(self): + return self._cptr + +cdef class _HelperCUlibraryOption: + def __cinit__(self, attr, init_value): + self._attr = attr.value + if False: + pass + elif self._attr in (cydriver.CUlibraryOption_enum.CU_LIBRARY_HOST_UNIVERSAL_FUNCTION_AND_DATA_TABLE,): + self._cptr = init_value.getPtr() + elif self._attr in (cydriver.CUlibraryOption_enum.CU_LIBRARY_BINARY_IS_PRESERVED,): + self._uint = init_value + self._cptr = self._uint + else: + raise TypeError('Unsupported attribute: {}'.format(attr.name)) + + def __dealloc__(self): + pass + + @property + def cptr(self): + return self._cptr + +cdef class _HelperCudaLibraryOption: + def __cinit__(self, attr, init_value): + self._attr = attr.value + if False: + pass + elif self._attr in (cyruntime.cudaLibraryOption.cudaLibraryHostUniversalFunctionAndDataTable,): + self._cptr = init_value.getPtr() + elif self._attr in (cyruntime.cudaLibraryOption.cudaLibraryBinaryIsPreserved,): + self._uint = init_value + self._cptr = self._uint + else: + raise TypeError('Unsupported attribute: {}'.format(attr.name)) + + def __dealloc__(self): + pass + + @property + def cptr(self): + return self._cptr + +cdef class _HelperCUmemAllocationHandleType: + def __cinit__(self, attr): + self._type = attr.value + if False: + pass + elif self._type in (cydriver.CUmemAllocationHandleType_enum.CU_MEM_HANDLE_TYPE_NONE,): + self._cptr = &self._int + elif self._type in (cydriver.CUmemAllocationHandleType_enum.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR,): + self._cptr = &self._int + elif self._type in (cydriver.CUmemAllocationHandleType_enum.CU_MEM_HANDLE_TYPE_WIN32,): + self._cptr = &self._handle + elif self._type in (cydriver.CUmemAllocationHandleType_enum.CU_MEM_HANDLE_TYPE_WIN32_KMT,): + self._cptr = &self._d3dkmt_handle + elif self._type in (cydriver.CUmemAllocationHandleType_enum.CU_MEM_HANDLE_TYPE_FABRIC,): + self._mem_fabric_handle = _driver.CUmemFabricHandle() + self._cptr = self._mem_fabric_handle.getPtr() + else: + raise TypeError('Unsupported attribute: {}'.format(attr.name)) + + def __dealloc__(self): + pass + + @property + def cptr(self): + return self._cptr + + def pyObj(self): + if False: + pass + elif self._type in (cydriver.CUmemAllocationHandleType_enum.CU_MEM_HANDLE_TYPE_NONE,): + return self._int + elif self._type in (cydriver.CUmemAllocationHandleType_enum.CU_MEM_HANDLE_TYPE_POSIX_FILE_DESCRIPTOR,): + return self._int + elif self._type in (cydriver.CUmemAllocationHandleType_enum.CU_MEM_HANDLE_TYPE_WIN32,): + return self._handle + elif self._type in (cydriver.CUmemAllocationHandleType_enum.CU_MEM_HANDLE_TYPE_WIN32_KMT,): + return self._d3dkmt_handle + elif self._type in (cydriver.CUmemAllocationHandleType_enum.CU_MEM_HANDLE_TYPE_FABRIC,): + return self._mem_fabric_handle + else: + raise TypeError('Unsupported attribute: {}'.format(self._type)) + +cdef class _InputVoidPtrPtrHelper: + def __cinit__(self, lst): + self._cptr = _callocWrapper(len(lst), sizeof(void*)) + for idx in range(len(lst)): + self._cptr[idx] = lst[idx].cptr + + def __dealloc__(self): + free(self._cptr) + + @property + def cptr(self): + return self._cptr + +cdef class _HelperCUcoredumpSettings: + def __cinit__(self, attr, init_value, is_getter=False): + self._is_getter = is_getter + self._attrib = attr.value + if self._attrib in (cydriver.CUcoredumpSettings_enum.CU_COREDUMP_FILE, + cydriver.CUcoredumpSettings_enum.CU_COREDUMP_PIPE,): + if self._is_getter: + self._charstar = _callocWrapper(1024, 1) + self._cptr = self._charstar + self._size = 1024 + else: + self._charstar = init_value + self._cptr = self._charstar + self._size = len(init_value) + elif self._attrib in (cydriver.CUcoredumpSettings_enum.CU_COREDUMP_ENABLE_ON_EXCEPTION, + cydriver.CUcoredumpSettings_enum.CU_COREDUMP_TRIGGER_HOST, + cydriver.CUcoredumpSettings_enum.CU_COREDUMP_LIGHTWEIGHT, + cydriver.CUcoredumpSettings_enum.CU_COREDUMP_ENABLE_USER_TRIGGER,): + if self._is_getter == False: + self._bool = init_value + + self._cptr = &self._bool + self._size = 1 + else: + raise TypeError('Unsupported attribute: {}'.format(attr.name)) + + def __dealloc__(self): + pass + + @property + def cptr(self): + return self._cptr + + def size(self): + return self._size + + def pyObj(self): + assert(self._is_getter == True) + if self._attrib in (cydriver.CUcoredumpSettings_enum.CU_COREDUMP_FILE, + cydriver.CUcoredumpSettings_enum.CU_COREDUMP_PIPE,): + return self._charstar + elif self._attrib in (cydriver.CUcoredumpSettings_enum.CU_COREDUMP_ENABLE_ON_EXCEPTION, + cydriver.CUcoredumpSettings_enum.CU_COREDUMP_TRIGGER_HOST, + cydriver.CUcoredumpSettings_enum.CU_COREDUMP_LIGHTWEIGHT, + cydriver.CUcoredumpSettings_enum.CU_COREDUMP_ENABLE_USER_TRIGGER,): + return self._bool + else: + raise TypeError('Unsupported attribute value: {}'.format(self._attrib)) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/windll.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/windll.pxd new file mode 100644 index 0000000000000000000000000000000000000000..7b190f35959d98d17da3a5c7d1433dc7a8c17cc9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/_lib/windll.pxd @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2021-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +from libc.stddef cimport wchar_t +from libc.stdint cimport uintptr_t +from cpython cimport PyUnicode_AsWideCharString, PyMem_Free + +cdef extern from "windows.h" nogil: + ctypedef void* HMODULE + ctypedef void* HANDLE + ctypedef void* FARPROC + ctypedef unsigned long DWORD + ctypedef const wchar_t *LPCWSTR + ctypedef const char *LPCSTR + ctypedef int BOOL + + cdef DWORD LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800 + + HMODULE _LoadLibraryExW "LoadLibraryExW"( + LPCWSTR lpLibFileName, + HANDLE hFile, + DWORD dwFlags + ) + + FARPROC _GetProcAddress "GetProcAddress"(HMODULE hModule, LPCSTR lpProcName) + + BOOL _FreeLibrary "FreeLibrary"(HMODULE hLibModule) + +cdef inline uintptr_t LoadLibraryExW(str path, HANDLE hFile, DWORD dwFlags): + cdef uintptr_t result + cdef wchar_t* wpath = PyUnicode_AsWideCharString(path, NULL) + with nogil: + result = _LoadLibraryExW( + wpath, + hFile, + dwFlags + ) + PyMem_Free(wpath) + return result + +cdef inline FARPROC GetProcAddress(uintptr_t hModule, const char* lpProcName) nogil: + return _GetProcAddress(hModule, lpProcName) + +cdef inline BOOL FreeLibrary(uintptr_t hLibModule) nogil: + return _FreeLibrary(hLibModule) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/cycufile.cpython-312-x86_64-linux-gnu.so b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/cycufile.cpython-312-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..9e783f6b20957b6d56af5b04d908b967bc98c9a5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/cycufile.cpython-312-x86_64-linux-gnu.so differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/cynvjitlink.pxd b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/cynvjitlink.pxd new file mode 100644 index 0000000000000000000000000000000000000000..c1ba67a51700b8b32667a016184ac24370578f1a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/cynvjitlink.pxd @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.0.1 to 12.9.1. Do not modify it directly. + +from libc.stdint cimport intptr_t, uint32_t + + +############################################################################### +# Types (structs, enums, ...) +############################################################################### + +# enums +ctypedef enum nvJitLinkResult "nvJitLinkResult": + NVJITLINK_SUCCESS "NVJITLINK_SUCCESS" = 0 + NVJITLINK_ERROR_UNRECOGNIZED_OPTION "NVJITLINK_ERROR_UNRECOGNIZED_OPTION" + NVJITLINK_ERROR_MISSING_ARCH "NVJITLINK_ERROR_MISSING_ARCH" + NVJITLINK_ERROR_INVALID_INPUT "NVJITLINK_ERROR_INVALID_INPUT" + NVJITLINK_ERROR_PTX_COMPILE "NVJITLINK_ERROR_PTX_COMPILE" + NVJITLINK_ERROR_NVVM_COMPILE "NVJITLINK_ERROR_NVVM_COMPILE" + NVJITLINK_ERROR_INTERNAL "NVJITLINK_ERROR_INTERNAL" + NVJITLINK_ERROR_THREADPOOL "NVJITLINK_ERROR_THREADPOOL" + NVJITLINK_ERROR_UNRECOGNIZED_INPUT "NVJITLINK_ERROR_UNRECOGNIZED_INPUT" + NVJITLINK_ERROR_FINALIZE "NVJITLINK_ERROR_FINALIZE" + NVJITLINK_ERROR_NULL_INPUT "NVJITLINK_ERROR_NULL_INPUT" + NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS "NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS" + NVJITLINK_ERROR_INCORRECT_INPUT_TYPE "NVJITLINK_ERROR_INCORRECT_INPUT_TYPE" + NVJITLINK_ERROR_ARCH_MISMATCH "NVJITLINK_ERROR_ARCH_MISMATCH" + NVJITLINK_ERROR_OUTDATED_LIBRARY "NVJITLINK_ERROR_OUTDATED_LIBRARY" + NVJITLINK_ERROR_MISSING_FATBIN "NVJITLINK_ERROR_MISSING_FATBIN" + NVJITLINK_ERROR_UNRECOGNIZED_ARCH "NVJITLINK_ERROR_UNRECOGNIZED_ARCH" + NVJITLINK_ERROR_UNSUPPORTED_ARCH "NVJITLINK_ERROR_UNSUPPORTED_ARCH" + NVJITLINK_ERROR_LTO_NOT_ENABLED "NVJITLINK_ERROR_LTO_NOT_ENABLED" + _NVJITLINKRESULT_INTERNAL_LOADING_ERROR "_NVJITLINKRESULT_INTERNAL_LOADING_ERROR" = -42 + +ctypedef enum nvJitLinkInputType "nvJitLinkInputType": + NVJITLINK_INPUT_NONE "NVJITLINK_INPUT_NONE" = 0 + NVJITLINK_INPUT_CUBIN "NVJITLINK_INPUT_CUBIN" = 1 + NVJITLINK_INPUT_PTX "NVJITLINK_INPUT_PTX" + NVJITLINK_INPUT_LTOIR "NVJITLINK_INPUT_LTOIR" + NVJITLINK_INPUT_FATBIN "NVJITLINK_INPUT_FATBIN" + NVJITLINK_INPUT_OBJECT "NVJITLINK_INPUT_OBJECT" + NVJITLINK_INPUT_LIBRARY "NVJITLINK_INPUT_LIBRARY" + NVJITLINK_INPUT_INDEX "NVJITLINK_INPUT_INDEX" + NVJITLINK_INPUT_ANY "NVJITLINK_INPUT_ANY" = 10 + + +# types +ctypedef void* nvJitLinkHandle 'nvJitLinkHandle' + + +############################################################################### +# Functions +############################################################################### + +cdef nvJitLinkResult nvJitLinkCreate(nvJitLinkHandle* handle, uint32_t numOptions, const char** options) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult nvJitLinkDestroy(nvJitLinkHandle* handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult nvJitLinkAddData(nvJitLinkHandle handle, nvJitLinkInputType inputType, const void* data, size_t size, const char* name) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult nvJitLinkAddFile(nvJitLinkHandle handle, nvJitLinkInputType inputType, const char* fileName) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult nvJitLinkComplete(nvJitLinkHandle handle) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult nvJitLinkGetLinkedCubinSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult nvJitLinkGetLinkedCubin(nvJitLinkHandle handle, void* cubin) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult nvJitLinkGetLinkedPtxSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult nvJitLinkGetLinkedPtx(nvJitLinkHandle handle, char* ptx) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult nvJitLinkGetErrorLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult nvJitLinkGetErrorLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult nvJitLinkGetInfoLogSize(nvJitLinkHandle handle, size_t* size) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult nvJitLinkGetInfoLog(nvJitLinkHandle handle, char* log) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil +cdef nvJitLinkResult nvJitLinkVersion(unsigned int* major, unsigned int* minor) except?_NVJITLINKRESULT_INTERNAL_LOADING_ERROR nogil diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/cynvvm.cpython-312-x86_64-linux-gnu.so b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/cynvvm.cpython-312-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..3ee9eb5a08b655b5fe414eff05edca32dc5a8494 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/cynvvm.cpython-312-x86_64-linux-gnu.so differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/cynvvm.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/cynvvm.pyx new file mode 100644 index 0000000000000000000000000000000000000000..bb5b5b4c05c423545436eb34581a77f7fa986d04 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/cynvvm.pyx @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.0.1 to 12.9.1. Do not modify it directly. + +from ._internal cimport nvvm as _nvvm + + +############################################################################### +# Wrapper functions +############################################################################### + +cdef const char* nvvmGetErrorString(nvvmResult result) except?NULL nogil: + return _nvvm._nvvmGetErrorString(result) + + +cdef nvvmResult nvvmVersion(int* major, int* minor) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + return _nvvm._nvvmVersion(major, minor) + + +cdef nvvmResult nvvmIRVersion(int* majorIR, int* minorIR, int* majorDbg, int* minorDbg) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + return _nvvm._nvvmIRVersion(majorIR, minorIR, majorDbg, minorDbg) + + +cdef nvvmResult nvvmCreateProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + return _nvvm._nvvmCreateProgram(prog) + + +cdef nvvmResult nvvmDestroyProgram(nvvmProgram* prog) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + return _nvvm._nvvmDestroyProgram(prog) + + +cdef nvvmResult nvvmAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + return _nvvm._nvvmAddModuleToProgram(prog, buffer, size, name) + + +cdef nvvmResult nvvmLazyAddModuleToProgram(nvvmProgram prog, const char* buffer, size_t size, const char* name) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + return _nvvm._nvvmLazyAddModuleToProgram(prog, buffer, size, name) + + +cdef nvvmResult nvvmCompileProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + return _nvvm._nvvmCompileProgram(prog, numOptions, options) + + +cdef nvvmResult nvvmVerifyProgram(nvvmProgram prog, int numOptions, const char** options) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + return _nvvm._nvvmVerifyProgram(prog, numOptions, options) + + +cdef nvvmResult nvvmGetCompiledResultSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + return _nvvm._nvvmGetCompiledResultSize(prog, bufferSizeRet) + + +cdef nvvmResult nvvmGetCompiledResult(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + return _nvvm._nvvmGetCompiledResult(prog, buffer) + + +cdef nvvmResult nvvmGetProgramLogSize(nvvmProgram prog, size_t* bufferSizeRet) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + return _nvvm._nvvmGetProgramLogSize(prog, bufferSizeRet) + + +cdef nvvmResult nvvmGetProgramLog(nvvmProgram prog, char* buffer) except?_NVVMRESULT_INTERNAL_LOADING_ERROR nogil: + return _nvvm._nvvmGetProgramLog(prog, buffer) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/nvjitlink.pyx b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/nvjitlink.pyx new file mode 100644 index 0000000000000000000000000000000000000000..6dd8770d7a3ab429f46866fae93f1775ac724c96 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/nvjitlink.pyx @@ -0,0 +1,326 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE +# +# This code was automatically generated across versions from 12.0.1 to 12.9.1. Do not modify it directly. + +cimport cython # NOQA + +from ._internal.utils cimport (get_resource_ptr, get_nested_resource_ptr, nested_resource, nullable_unique_ptr, + get_buffer_pointer, get_resource_ptrs) + +from enum import IntEnum as _IntEnum +from libcpp.vector cimport vector + + +############################################################################### +# Enum +############################################################################### + +class Result(_IntEnum): + """See `nvJitLinkResult`.""" + SUCCESS = NVJITLINK_SUCCESS + ERROR_UNRECOGNIZED_OPTION = NVJITLINK_ERROR_UNRECOGNIZED_OPTION + ERROR_MISSING_ARCH = NVJITLINK_ERROR_MISSING_ARCH + ERROR_INVALID_INPUT = NVJITLINK_ERROR_INVALID_INPUT + ERROR_PTX_COMPILE = NVJITLINK_ERROR_PTX_COMPILE + ERROR_NVVM_COMPILE = NVJITLINK_ERROR_NVVM_COMPILE + ERROR_INTERNAL = NVJITLINK_ERROR_INTERNAL + ERROR_THREADPOOL = NVJITLINK_ERROR_THREADPOOL + ERROR_UNRECOGNIZED_INPUT = NVJITLINK_ERROR_UNRECOGNIZED_INPUT + ERROR_FINALIZE = NVJITLINK_ERROR_FINALIZE + ERROR_NULL_INPUT = NVJITLINK_ERROR_NULL_INPUT + ERROR_INCOMPATIBLE_OPTIONS = NVJITLINK_ERROR_INCOMPATIBLE_OPTIONS + ERROR_INCORRECT_INPUT_TYPE = NVJITLINK_ERROR_INCORRECT_INPUT_TYPE + ERROR_ARCH_MISMATCH = NVJITLINK_ERROR_ARCH_MISMATCH + ERROR_OUTDATED_LIBRARY = NVJITLINK_ERROR_OUTDATED_LIBRARY + ERROR_MISSING_FATBIN = NVJITLINK_ERROR_MISSING_FATBIN + ERROR_UNRECOGNIZED_ARCH = NVJITLINK_ERROR_UNRECOGNIZED_ARCH + ERROR_UNSUPPORTED_ARCH = NVJITLINK_ERROR_UNSUPPORTED_ARCH + ERROR_LTO_NOT_ENABLED = NVJITLINK_ERROR_LTO_NOT_ENABLED + +class InputType(_IntEnum): + """See `nvJitLinkInputType`.""" + NONE = NVJITLINK_INPUT_NONE + CUBIN = NVJITLINK_INPUT_CUBIN + PTX = NVJITLINK_INPUT_PTX + LTOIR = NVJITLINK_INPUT_LTOIR + FATBIN = NVJITLINK_INPUT_FATBIN + OBJECT = NVJITLINK_INPUT_OBJECT + LIBRARY = NVJITLINK_INPUT_LIBRARY + INDEX = NVJITLINK_INPUT_INDEX + ANY = NVJITLINK_INPUT_ANY + + +############################################################################### +# Error handling +############################################################################### + +class nvJitLinkError(Exception): + + def __init__(self, status): + self.status = status + s = Result(status) + cdef str err = f"{s.name} ({s.value})" + super(nvJitLinkError, self).__init__(err) + + def __reduce__(self): + return (type(self), (self.status,)) + + +@cython.profile(False) +cdef int check_status(int status) except 1 nogil: + if status != 0: + with gil: + raise nvJitLinkError(status) + return status + + +############################################################################### +# Wrapper functions +############################################################################### + +cpdef destroy(intptr_t handle): + """nvJitLinkDestroy frees the memory associated with the given handle. + + Args: + handle (intptr_t): nvJitLink handle. + + .. seealso:: `nvJitLinkDestroy` + """ + cdef Handle h = handle + with nogil: + status = nvJitLinkDestroy(&h) + check_status(status) + + +cpdef intptr_t create(uint32_t num_options, options) except -1: + """nvJitLinkCreate creates an instance of nvJitLinkHandle with the given input options, and sets the output parameter ``handle``. + + Args: + num_options (uint32_t): Number of options passed. + options (object): Array of size ``num_options`` of option strings. It can be: + + - an :class:`int` as the pointer address to the nested sequence, or + - a Python sequence of :class:`int`\s, each of which is a pointer address + to a valid sequence of 'char', or + - a nested Python sequence of ``str``. + + + Returns: + intptr_t: Address of nvJitLink handle. + + .. seealso:: `nvJitLinkCreate` + """ + cdef nested_resource[ char ] _options_ + get_nested_resource_ptr[char](_options_, options, NULL) + cdef Handle handle + with nogil: + status = nvJitLinkCreate(&handle, num_options, (_options_.ptrs.data())) + check_status(status) + return handle + + +cpdef add_data(intptr_t handle, int input_type, data, size_t size, name): + """nvJitLinkAddData adds data image to the link. + + Args: + handle (intptr_t): nvJitLink handle. + input_type (InputType): kind of input. + data (bytes): pointer to data image in memory. + size (size_t): size of the data. + name (str): name of input object. + + .. seealso:: `nvJitLinkAddData` + """ + cdef void* _data_ = get_buffer_pointer(data, size, readonly=True) + if not isinstance(name, str): + raise TypeError("name must be a Python str") + cdef bytes _temp_name_ = (name).encode() + cdef char* _name_ = _temp_name_ + with nogil: + status = nvJitLinkAddData(handle, <_InputType>input_type, _data_, size, _name_) + check_status(status) + + +cpdef add_file(intptr_t handle, int input_type, file_name): + """nvJitLinkAddFile reads data from file and links it in. + + Args: + handle (intptr_t): nvJitLink handle. + input_type (InputType): kind of input. + file_name (str): name of file. + + .. seealso:: `nvJitLinkAddFile` + """ + if not isinstance(file_name, str): + raise TypeError("file_name must be a Python str") + cdef bytes _temp_file_name_ = (file_name).encode() + cdef char* _file_name_ = _temp_file_name_ + with nogil: + status = nvJitLinkAddFile(handle, <_InputType>input_type, _file_name_) + check_status(status) + + +cpdef complete(intptr_t handle): + """nvJitLinkComplete does the actual link. + + Args: + handle (intptr_t): nvJitLink handle. + + .. seealso:: `nvJitLinkComplete` + """ + with nogil: + status = nvJitLinkComplete(handle) + check_status(status) + + +cpdef size_t get_linked_cubin_size(intptr_t handle) except? 0: + """nvJitLinkGetLinkedCubinSize gets the size of the linked cubin. + + Args: + handle (intptr_t): nvJitLink handle. + + Returns: + size_t: Size of the linked cubin. + + .. seealso:: `nvJitLinkGetLinkedCubinSize` + """ + cdef size_t size + with nogil: + status = nvJitLinkGetLinkedCubinSize(handle, &size) + check_status(status) + return size + + +cpdef get_linked_cubin(intptr_t handle, cubin): + """nvJitLinkGetLinkedCubin gets the linked cubin. + + Args: + handle (intptr_t): nvJitLink handle. + cubin (bytes): The linked cubin. + + .. seealso:: `nvJitLinkGetLinkedCubin` + """ + cdef void* _cubin_ = get_buffer_pointer(cubin, -1, readonly=False) + with nogil: + status = nvJitLinkGetLinkedCubin(handle, _cubin_) + check_status(status) + + +cpdef size_t get_linked_ptx_size(intptr_t handle) except? 0: + """nvJitLinkGetLinkedPtxSize gets the size of the linked ptx. + + Args: + handle (intptr_t): nvJitLink handle. + + Returns: + size_t: Size of the linked PTX. + + .. seealso:: `nvJitLinkGetLinkedPtxSize` + """ + cdef size_t size + with nogil: + status = nvJitLinkGetLinkedPtxSize(handle, &size) + check_status(status) + return size + + +cpdef get_linked_ptx(intptr_t handle, ptx): + """nvJitLinkGetLinkedPtx gets the linked ptx. + + Args: + handle (intptr_t): nvJitLink handle. + ptx (bytes): The linked PTX. + + .. seealso:: `nvJitLinkGetLinkedPtx` + """ + cdef void* _ptx_ = get_buffer_pointer(ptx, -1, readonly=False) + with nogil: + status = nvJitLinkGetLinkedPtx(handle, _ptx_) + check_status(status) + + +cpdef size_t get_error_log_size(intptr_t handle) except? 0: + """nvJitLinkGetErrorLogSize gets the size of the error log. + + Args: + handle (intptr_t): nvJitLink handle. + + Returns: + size_t: Size of the error log. + + .. seealso:: `nvJitLinkGetErrorLogSize` + """ + cdef size_t size + with nogil: + status = nvJitLinkGetErrorLogSize(handle, &size) + check_status(status) + return size + + +cpdef get_error_log(intptr_t handle, log): + """nvJitLinkGetErrorLog puts any error messages in the log. + + Args: + handle (intptr_t): nvJitLink handle. + log (bytes): The error log. + + .. seealso:: `nvJitLinkGetErrorLog` + """ + cdef void* _log_ = get_buffer_pointer(log, -1, readonly=False) + with nogil: + status = nvJitLinkGetErrorLog(handle, _log_) + check_status(status) + + +cpdef size_t get_info_log_size(intptr_t handle) except? 0: + """nvJitLinkGetInfoLogSize gets the size of the info log. + + Args: + handle (intptr_t): nvJitLink handle. + + Returns: + size_t: Size of the info log. + + .. seealso:: `nvJitLinkGetInfoLogSize` + """ + cdef size_t size + with nogil: + status = nvJitLinkGetInfoLogSize(handle, &size) + check_status(status) + return size + + +cpdef get_info_log(intptr_t handle, log): + """nvJitLinkGetInfoLog puts any info messages in the log. + + Args: + handle (intptr_t): nvJitLink handle. + log (bytes): The info log. + + .. seealso:: `nvJitLinkGetInfoLog` + """ + cdef void* _log_ = get_buffer_pointer(log, -1, readonly=False) + with nogil: + status = nvJitLinkGetInfoLog(handle, _log_) + check_status(status) + + +cpdef tuple version(): + """nvJitLinkVersion returns the current version of nvJitLink. + + Returns: + A 2-tuple containing: + + - unsigned int: The major version. + - unsigned int: The minor version. + + .. seealso:: `nvJitLinkVersion` + """ + cdef unsigned int major + cdef unsigned int minor + with nogil: + status = nvJitLinkVersion(&major, &minor) + check_status(status) + return (major, minor) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/utils/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5f9288b81e400cc115fdc36b4660203a02f1ea2b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/utils/__init__.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +from typing import Any, Callable + +from ._ptx_utils import get_minimal_required_cuda_ver_from_ptx_ver, get_ptx_ver + +_handle_getters: dict[type, Callable[[Any], int]] = {} + + +def _add_cuda_native_handle_getter(t: type, getter: Callable[[Any], int]) -> None: + _handle_getters[t] = getter + + +def get_cuda_native_handle(obj: Any) -> int: + """Returns the address of the provided CUDA Python object as a Python int. + + Parameters + ---------- + obj : Any + CUDA Python object + + Returns + ------- + int : The object address. + """ + obj_type = type(obj) + try: + return _handle_getters[obj_type](obj) + except KeyError: + raise TypeError("Unknown type: " + str(obj_type)) from None diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/utils/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6532c03d0800b747cbd3d0af517a3741518ece80 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/utils/__pycache__/_ptx_utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/utils/__pycache__/_ptx_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..028e23d13d7a76a080603ed8aa78f5868f5a12df Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/utils/__pycache__/_ptx_utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/utils/_ptx_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/utils/_ptx_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d303d5980bc4575d974afef017dcd13262e7424c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/bindings/utils/_ptx_utils.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE + +import re + +# Mapping based on the official PTX ISA <-> CUDA Release table +# https://docs.nvidia.com/cuda/parallel-thread-execution/#release-notes-ptx-release-history +_ptx_to_cuda = { + "1.0": (1, 0), + "1.1": (1, 1), + "1.2": (2, 0), + "1.3": (2, 1), + "1.4": (2, 2), + "2.0": (3, 0), + "2.1": (3, 1), + "2.2": (3, 2), + "2.3": (4, 0), + "3.0": (4, 1), + "3.1": (5, 0), + "3.2": (5, 5), + "4.0": (6, 0), + "4.1": (6, 5), + "4.2": (7, 0), + "4.3": (7, 5), + "5.0": (8, 0), + "6.0": (9, 0), + "6.1": (9, 1), + "6.2": (9, 2), + "6.3": (10, 0), + "6.4": (10, 1), + "6.5": (10, 2), + "7.0": (11, 0), + "7.1": (11, 1), + "7.2": (11, 2), + "7.3": (11, 3), + "7.4": (11, 4), + "7.5": (11, 5), + "7.6": (11, 6), + "7.7": (11, 7), + "7.8": (11, 8), + "8.0": (12, 0), + "8.1": (12, 1), + "8.2": (12, 2), + "8.3": (12, 3), + "8.4": (12, 4), + "8.5": (12, 5), + "8.6": (12, 7), + "8.7": (12, 8), + "8.8": (12, 9), +} + + +def get_minimal_required_cuda_ver_from_ptx_ver(ptx_version: str) -> int: + """ + Maps the PTX ISA version to the minimal CUDA driver, nvPTXCompiler, or nvJitLink version + that is needed to load a PTX of the given ISA version. + + Parameters + ---------- + ptx_version : str + PTX ISA version as a string, e.g. "8.8" for PTX ISA 8.8. This is the ``.version`` + directive in the PTX header. + + Returns + ------- + int + Minimal CUDA version as 1000 * major + 10 * minor, e.g. 12090 for CUDA 12.9. + + Raises + ------ + ValueError + If the PTX version is unknown. + + Examples + -------- + >>> get_minimal_required_driver_ver_from_ptx_ver("8.8") + 12090 + >>> get_minimal_required_driver_ver_from_ptx_ver("7.0") + 11000 + """ + try: + major, minor = _ptx_to_cuda[ptx_version] + return 1000 * major + 10 * minor + except KeyError: + raise ValueError(f"Unknown or unsupported PTX ISA version: {ptx_version}") from None + + +# Regex pattern to match .version directive and capture the version number +# TODO: if import speed is a concern, consider lazy-initializing it. +_ptx_ver_pattern = re.compile(r"\.version\s+([0-9]+\.[0-9]+)") + + +def get_ptx_ver(ptx: str) -> str: + """ + Extract the PTX ISA version string from PTX source code. + + Parameters + ---------- + ptx : str + The PTX assembly source code as a string. + + Returns + ------- + str + The PTX ISA version string, e.g., "8.8". + + Raises + ------ + ValueError + If the .version directive is not found in the PTX source. + + Examples + -------- + >>> ptx = r''' + ... .version 8.8 + ... .target sm_86 + ... .address_size 64 + ... + ... .visible .entry test_kernel() + ... { + ... ret; + ... } + ... ''' + >>> get_ptx_ver(ptx) + '8.8' + """ + m = _ptx_ver_pattern.search(ptx) + if m: + return m.group(1) + else: + raise ValueError("No .version directive found in PTX source. Is it a valid PTX?") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9f2acc6ba28ddb25cf69cc0a44b3b592b0eb4143 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/__pycache__/_version.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/__pycache__/_version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d9c19fea84834b70379f1b5edb4346266c91fccf Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/__pycache__/_version.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_binaries/__pycache__/find_nvidia_binary_utility.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_binaries/__pycache__/find_nvidia_binary_utility.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e58fb5d5088fbb29c4f619269fccb0dccbad75b5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_binaries/__pycache__/find_nvidia_binary_utility.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_binaries/__pycache__/supported_nvidia_binaries.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_binaries/__pycache__/supported_nvidia_binaries.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5762806c6e4432060128f77a4253edd3caaf05b0 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_binaries/__pycache__/supported_nvidia_binaries.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_binaries/find_nvidia_binary_utility.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_binaries/find_nvidia_binary_utility.py new file mode 100644 index 0000000000000000000000000000000000000000..1d8278e103647b73adaf8dee1eb93fd393369dbc --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_binaries/find_nvidia_binary_utility.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import functools +import os +import shutil + +from cuda.pathfinder._binaries import supported_nvidia_binaries +from cuda.pathfinder._utils.env_vars import get_cuda_home_or_path +from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs_all_sitepackages +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS + + +class UnsupportedBinaryError(Exception): + def __init__(self, utility: str) -> None: + super().__init__(utility) + self.utility = utility + + def __str__(self) -> str: + supported_utilities = ", ".join(supported_nvidia_binaries.SUPPORTED_BINARIES) + return f"Binary '{self.utility}' is not supported. Supported utilities are: {supported_utilities}" + + +def _normalize_utility_name(utility_name: str) -> str: + """Normalize utility name by adding .exe on Windows if needed.""" + if IS_WINDOWS and not utility_name.lower().endswith((".exe", ".bat", ".cmd")): + return f"{utility_name}.exe" + return utility_name + + +@functools.cache +def find_nvidia_binary_utility(utility_name: str) -> str | None: + """Locate a CUDA binary utility executable. + + Args: + utility_name (str): The name of the binary utility to find + (e.g., ``"nvdisasm"``, ``"cuobjdump"``). On Windows, the ``.exe`` + extension will be automatically appended if not present. The function + also recognizes ``.bat`` and ``.cmd`` files on Windows. + + Returns: + str or None: Absolute path to the discovered executable, or ``None`` + if the utility cannot be found. The returned path is normalized + (absolute and with resolved separators). + + Raises: + UnsupportedBinaryError: If ``utility_name`` is not in the supported set + (see ``SUPPORTED_BINARY_UTILITIES``). + + Search order: + 1. **NVIDIA Python wheels** + + - Scan installed distributions (``site-packages``) for binary layouts + shipped in NVIDIA wheels (e.g., ``cuda-nvcc``). + + 2. **Conda environments** + + - Check Conda-style installation prefixes via ``CONDA_PREFIX`` + environment variable, which use platform-specific bin directory + layouts (``Library/bin`` on Windows, ``bin`` on Linux). + + 3. **CUDA Toolkit environment variables** + + - Use ``CUDA_HOME`` or ``CUDA_PATH`` (in that order), searching + ``bin/x64``, ``bin/x86_64``, and ``bin`` subdirectories on Windows, + or just ``bin`` on Linux. + + Note: + Results are cached using ``@functools.cache`` for performance. The cache + persists for the lifetime of the process. + + On Windows, executables are identified by their file extensions + (``.exe``, ``.bat``, ``.cmd``). On Unix-like systems, executables + are identified by the ``X_OK`` (execute) permission bit. + + Example: + >>> from cuda.pathfinder import find_nvidia_binary_utility + >>> nvdisasm = find_nvidia_binary_utility("nvdisasm") + >>> if nvdisasm: + ... print(f"Found nvdisasm at: {nvdisasm}") + """ + if utility_name not in supported_nvidia_binaries.SUPPORTED_BINARIES: + raise UnsupportedBinaryError(utility_name) + + # 1. Search in site-packages (NVIDIA wheels) + candidate_dirs = supported_nvidia_binaries.SITE_PACKAGES_BINDIRS.get(utility_name, ()) + dirs = [] + + for sub_dir in candidate_dirs: + dirs.extend(find_sub_dirs_all_sitepackages(sub_dir.split(os.sep))) + + # 2. Search in Conda environment + if (conda_prefix := os.environ.get("CONDA_PREFIX")) is not None: + if IS_WINDOWS: + dirs.append(os.path.join(conda_prefix, "Library", "bin")) + else: + dirs.append(os.path.join(conda_prefix, "bin")) + + # 3. Search in CUDA Toolkit (CUDA_HOME/CUDA_PATH) + if (cuda_home := get_cuda_home_or_path()) is not None: + if IS_WINDOWS: + dirs.append(os.path.join(cuda_home, "bin", "x64")) + dirs.append(os.path.join(cuda_home, "bin", "x86_64")) + dirs.append(os.path.join(cuda_home, "bin")) + + normalized_name = _normalize_utility_name(utility_name) + return shutil.which(normalized_name, path=os.pathsep.join(dirs)) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_binaries/supported_nvidia_binaries.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_binaries/supported_nvidia_binaries.py new file mode 100644 index 0000000000000000000000000000000000000000..bf0a3689c89d347dd51b06082c5dd1f0792834e3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_binaries/supported_nvidia_binaries.py @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import os + +# Site-packages bin directories where binaries might be found +# Based on NVIDIA wheel layouts (same for Linux and Windows) +_CUDA_NVCC_BIN = os.path.join("nvidia", "cuda_nvcc", "bin") +_NSIGHT_SYSTEMS_BIN = os.path.join("nvidia", "nsight_systems", "bin") +_NSIGHT_COMPUTE_BIN = os.path.join("nvidia", "nsight_compute", "bin") + +# Common CUDA binary utilities available on both Linux and Windows +SITE_PACKAGES_BINDIRS = { + # Core compilation tools + "nvcc": (_CUDA_NVCC_BIN,), + "nvdisasm": (_CUDA_NVCC_BIN,), + "cuobjdump": (_CUDA_NVCC_BIN,), + "nvprune": (_CUDA_NVCC_BIN,), + "fatbinary": (_CUDA_NVCC_BIN,), + "bin2c": (_CUDA_NVCC_BIN,), + "nvlink": (_CUDA_NVCC_BIN,), + # Runtime/debugging tools + "cuda-gdb": (_CUDA_NVCC_BIN,), + "cuda-gdbserver": (_CUDA_NVCC_BIN,), + "compute-sanitizer": (_CUDA_NVCC_BIN,), + # Profiling tools + "nvprof": (_CUDA_NVCC_BIN,), + "nsys": (_NSIGHT_SYSTEMS_BIN,), + "nsight-sys": (_NSIGHT_SYSTEMS_BIN,), + "ncu": (_NSIGHT_COMPUTE_BIN,), + "nsight-compute": (_NSIGHT_COMPUTE_BIN,), +} + +SUPPORTED_BINARIES_ALL = SUPPORTED_BINARIES = tuple(SITE_PACKAGES_BINDIRS.keys()) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/canary_probe_subprocess.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/canary_probe_subprocess.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0879ee5017b046f7e4142e993dce5fd44edb10e4 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/canary_probe_subprocess.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/find_nvidia_dynamic_lib.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/find_nvidia_dynamic_lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2faf40bf16fa396740e04cb27fd9c2014f5bfe5a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/find_nvidia_dynamic_lib.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/load_dl_common.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/load_dl_common.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b4f6db8ecca2bbb5b103132e29ea3b3dbe01bb1a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/load_dl_common.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/load_dl_linux.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/load_dl_linux.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6b603f0ccf6b3d1aea294eb75f395dcf56d9af57 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/load_dl_linux.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/load_dl_windows.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/load_dl_windows.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..febe6fc94a70f7afc34c78900966709f025e9a30 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/load_dl_windows.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/load_nvidia_dynamic_lib.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/load_nvidia_dynamic_lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c3803cca539a57ef77c9293e1d6935f54b342e01 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/load_nvidia_dynamic_lib.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/supported_nvidia_libs.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/supported_nvidia_libs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4199c999fd984a44e424da078ac8e009627030cc Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/__pycache__/supported_nvidia_libs.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/canary_probe_subprocess.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/canary_probe_subprocess.py new file mode 100644 index 0000000000000000000000000000000000000000..3f1525dd48e8bd0de4664d14298d623b0df24214 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/canary_probe_subprocess.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json + +from cuda.pathfinder._dynamic_libs.load_dl_common import DynamicLibNotFoundError, LoadedDL +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS + +if IS_WINDOWS: + from cuda.pathfinder._dynamic_libs.load_dl_windows import load_with_system_search +else: + from cuda.pathfinder._dynamic_libs.load_dl_linux import load_with_system_search + + +def _probe_canary_abs_path(libname: str) -> str | None: + try: + loaded: LoadedDL | None = load_with_system_search(libname) + except DynamicLibNotFoundError: + return None + if loaded is None: + return None + abs_path = loaded.abs_path + if not isinstance(abs_path, str): + return None + return abs_path + + +def probe_canary_abs_path_and_print_json(libname: str) -> None: + print(json.dumps(_probe_canary_abs_path(libname))) # noqa: T201 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/find_nvidia_dynamic_lib.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/find_nvidia_dynamic_lib.py new file mode 100644 index 0000000000000000000000000000000000000000..b93523f36a8dc1d6b7cf14b46bd2f6ba5d7f6933 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/find_nvidia_dynamic_lib.py @@ -0,0 +1,277 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import glob +import os +from collections.abc import Sequence + +from cuda.pathfinder._dynamic_libs.load_dl_common import DynamicLibNotFoundError +from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import ( + SITE_PACKAGES_LIBDIRS_LINUX, + SITE_PACKAGES_LIBDIRS_WINDOWS, + is_suppressed_dll_file, +) +from cuda.pathfinder._utils.env_vars import get_cuda_home_or_path +from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs_all_sitepackages +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS + + +def _no_such_file_in_sub_dirs( + sub_dirs: Sequence[str], file_wild: str, error_messages: list[str], attachments: list[str] +) -> None: + error_messages.append(f"No such file: {file_wild}") + for sub_dir in find_sub_dirs_all_sitepackages(sub_dirs): + attachments.append(f' listdir("{sub_dir}"):') + for node in sorted(os.listdir(sub_dir)): + attachments.append(f" {node}") + + +def _find_so_using_nvidia_lib_dirs( + libname: str, so_basename: str, error_messages: list[str], attachments: list[str] +) -> str | None: + rel_dirs = SITE_PACKAGES_LIBDIRS_LINUX.get(libname) + if rel_dirs is not None: + sub_dirs_searched = [] + file_wild = so_basename + "*" + for rel_dir in rel_dirs: + sub_dir = tuple(rel_dir.split(os.path.sep)) + for abs_dir in find_sub_dirs_all_sitepackages(sub_dir): + # First look for an exact match + so_name = os.path.join(abs_dir, so_basename) + if os.path.isfile(so_name): + return so_name + # Look for a versioned library + # Using sort here mainly to make the result deterministic. + for so_name in sorted(glob.glob(os.path.join(abs_dir, file_wild))): + if os.path.isfile(so_name): + return so_name + sub_dirs_searched.append(sub_dir) + for sub_dir in sub_dirs_searched: + _no_such_file_in_sub_dirs(sub_dir, file_wild, error_messages, attachments) + return None + + +def _find_dll_under_dir(dirpath: str, file_wild: str) -> str | None: + for path in sorted(glob.glob(os.path.join(dirpath, file_wild))): + if not os.path.isfile(path): + continue + if not is_suppressed_dll_file(os.path.basename(path)): + return path + return None + + +def _find_dll_using_nvidia_bin_dirs( + libname: str, lib_searched_for: str, error_messages: list[str], attachments: list[str] +) -> str | None: + rel_dirs = SITE_PACKAGES_LIBDIRS_WINDOWS.get(libname) + if rel_dirs is not None: + sub_dirs_searched = [] + for rel_dir in rel_dirs: + sub_dir = tuple(rel_dir.split(os.path.sep)) + for abs_dir in find_sub_dirs_all_sitepackages(sub_dir): + dll_name = _find_dll_under_dir(abs_dir, lib_searched_for) + if dll_name is not None: + return dll_name + sub_dirs_searched.append(sub_dir) + for sub_dir in sub_dirs_searched: + _no_such_file_in_sub_dirs(sub_dir, lib_searched_for, error_messages, attachments) + return None + + +def _find_lib_dir_using_anchor_point(libname: str, anchor_point: str, linux_lib_dir: str) -> str | None: + # Resolve paths for the four cases: + # Windows/Linux x nvvm yes/no + if IS_WINDOWS: + if libname == "nvvm": # noqa: SIM108 + rel_paths = [ + "nvvm/bin/*", # CTK 13 + "nvvm/bin", # CTK 12 + ] + else: + rel_paths = [ + "bin/x64", # CTK 13 + "bin", # CTK 12 + ] + else: + if libname == "nvvm": # noqa: SIM108 + rel_paths = ["nvvm/lib64"] + else: + rel_paths = [linux_lib_dir] + + for rel_path in rel_paths: + for dirname in sorted(glob.glob(os.path.join(anchor_point, rel_path))): + if os.path.isdir(dirname): + return os.path.normpath(dirname) + + return None + + +def _find_lib_dir_using_cuda_home(libname: str) -> str | None: + cuda_home = get_cuda_home_or_path() + if cuda_home is None: + return None + return _find_lib_dir_using_anchor_point(libname, anchor_point=cuda_home, linux_lib_dir="lib64") + + +def _find_lib_dir_using_conda_prefix(libname: str) -> str | None: + conda_prefix = os.environ.get("CONDA_PREFIX") + if not conda_prefix: + return None + return _find_lib_dir_using_anchor_point( + libname, anchor_point=os.path.join(conda_prefix, "Library") if IS_WINDOWS else conda_prefix, linux_lib_dir="lib" + ) + + +def _find_so_using_lib_dir( + lib_dir: str, so_basename: str, error_messages: list[str], attachments: list[str] +) -> str | None: + so_name = os.path.join(lib_dir, so_basename) + if os.path.isfile(so_name): + return so_name + error_messages.append(f"No such file: {so_name}") + attachments.append(f' listdir("{lib_dir}"):') + if not os.path.isdir(lib_dir): + attachments.append(" DIRECTORY DOES NOT EXIST") + else: + for node in sorted(os.listdir(lib_dir)): + attachments.append(f" {node}") + return None + + +def _find_dll_using_lib_dir( + lib_dir: str, libname: str, error_messages: list[str], attachments: list[str] +) -> str | None: + file_wild = libname + "*.dll" + dll_name = _find_dll_under_dir(lib_dir, file_wild) + if dll_name is not None: + return dll_name + error_messages.append(f"No such file: {file_wild}") + attachments.append(f' listdir("{lib_dir}"):') + for node in sorted(os.listdir(lib_dir)): + attachments.append(f" {node}") + return None + + +def _derive_ctk_root_linux(resolved_lib_path: str) -> str | None: + """Derive the CTK installation root from a resolved library path on Linux. + + Standard system CTK layout: ``$CTK_ROOT/lib64/libfoo.so.XX`` + (some installs use ``lib`` instead of ``lib64``). + Also handles target-specific layouts: + ``$CTK_ROOT/targets//lib64/libfoo.so.XX`` (or ``lib``). + + Returns None if the path doesn't match a recognized layout. + """ + lib_dir = os.path.dirname(resolved_lib_path) + basename = os.path.basename(lib_dir) + if basename in ("lib64", "lib"): + parent = os.path.dirname(lib_dir) + grandparent = os.path.dirname(parent) + if os.path.basename(grandparent) == "targets": + # This corresponds to /.../targets//lib{,64} + return os.path.dirname(grandparent) + return parent + return None + + +def _derive_ctk_root_windows(resolved_lib_path: str) -> str | None: + """Derive the CTK installation root from a resolved library path on Windows. + + Handles two CTK layouts: + - CTK 13: ``$CTK_ROOT/bin/x64/foo.dll`` + - CTK 12: ``$CTK_ROOT/bin/foo.dll`` + + Returns None if the path doesn't match a recognized layout. + + Uses ``ntpath`` explicitly so the function is testable on any platform. + """ + import ntpath + + lib_dir = ntpath.dirname(resolved_lib_path) + basename = ntpath.basename(lib_dir).lower() + if basename == "x64": + parent = ntpath.dirname(lib_dir) + if ntpath.basename(parent).lower() == "bin": + return ntpath.dirname(parent) + elif basename == "bin": + return ntpath.dirname(lib_dir) + return None + + +def derive_ctk_root(resolved_lib_path: str) -> str | None: + """Derive the CTK installation root from a resolved library path. + + Given the absolute path of a loaded CTK shared library, walk up the + directory tree to find the CTK root. Returns None if the path doesn't + match any recognized CTK directory layout. + """ + if IS_WINDOWS: + return _derive_ctk_root_windows(resolved_lib_path) + return _derive_ctk_root_linux(resolved_lib_path) + + +class _FindNvidiaDynamicLib: + def __init__(self, libname: str): + self.libname = libname + if IS_WINDOWS: + self.lib_searched_for = f"{libname}*.dll" + else: + self.lib_searched_for = f"lib{libname}.so" + self.error_messages: list[str] = [] + self.attachments: list[str] = [] + self.abs_path: str | None = None + + def try_site_packages(self) -> str | None: + if IS_WINDOWS: + return _find_dll_using_nvidia_bin_dirs( + self.libname, + self.lib_searched_for, + self.error_messages, + self.attachments, + ) + else: + return _find_so_using_nvidia_lib_dirs( + self.libname, + self.lib_searched_for, + self.error_messages, + self.attachments, + ) + + def try_with_conda_prefix(self) -> str | None: + return self._find_using_lib_dir(_find_lib_dir_using_conda_prefix(self.libname)) + + def try_with_cuda_home(self) -> str | None: + return self._find_using_lib_dir(_find_lib_dir_using_cuda_home(self.libname)) + + def try_via_ctk_root(self, ctk_root: str) -> str | None: + """Find the library under a derived CTK root directory. + + Uses :func:`_find_lib_dir_using_anchor_point` which already knows + about non-standard sub-paths (e.g. ``nvvm/lib64`` for nvvm). + """ + return self._find_using_lib_dir( + _find_lib_dir_using_anchor_point(self.libname, anchor_point=ctk_root, linux_lib_dir="lib64") + ) + + def _find_using_lib_dir(self, lib_dir: str | None) -> str | None: + if lib_dir is None: + return None + if IS_WINDOWS: + return _find_dll_using_lib_dir( + lib_dir, + self.libname, + self.error_messages, + self.attachments, + ) + else: + return _find_so_using_lib_dir( + lib_dir, + self.lib_searched_for, + self.error_messages, + self.attachments, + ) + + def raise_not_found_error(self) -> None: + err = ", ".join(self.error_messages) + att = "\n".join(self.attachments) + raise DynamicLibNotFoundError(f'Failure finding "{self.lib_searched_for}": {err}\n{att}') diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/load_dl_common.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/load_dl_common.py new file mode 100644 index 0000000000000000000000000000000000000000..1204a23c15cc9461d078bf7f814375eb850905cf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/load_dl_common.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Callable +from dataclasses import dataclass + +from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import DIRECT_DEPENDENCIES + + +class DynamicLibNotFoundError(RuntimeError): + pass + + +class DynamicLibNotAvailableError(DynamicLibNotFoundError): + pass + + +class DynamicLibUnknownError(DynamicLibNotFoundError): + pass + + +@dataclass +class LoadedDL: + abs_path: str | None + was_already_loaded_from_elsewhere: bool + _handle_uint: int # Platform-agnostic unsigned pointer value + found_via: str + + +def load_dependencies(libname: str, load_func: Callable[[str], LoadedDL]) -> None: + for dep in DIRECT_DEPENDENCIES.get(libname, ()): + load_func(dep) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/load_dl_linux.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/load_dl_linux.py new file mode 100644 index 0000000000000000000000000000000000000000..4d2bae5b90ece2f970001a207d710aa326cab922 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/load_dl_linux.py @@ -0,0 +1,216 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import contextlib +import ctypes +import ctypes.util +import os +from typing import cast + +from cuda.pathfinder._dynamic_libs.load_dl_common import LoadedDL +from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import ( + LIBNAMES_REQUIRING_RTLD_DEEPBIND, + SUPPORTED_LINUX_SONAMES, +) + +CDLL_MODE = os.RTLD_NOW | os.RTLD_GLOBAL + + +def _load_libdl() -> ctypes.CDLL: + # In normal glibc-based Linux environments, find_library("dl") should return + # something like "libdl.so.2". In minimal or stripped-down environments + # (no ldconfig/gcc, incomplete linker cache), this can return None even + # though libdl is present. In that case, we fall back to the stable SONAME. + name = ctypes.util.find_library("dl") or "libdl.so.2" + try: + return ctypes.CDLL(name) + except OSError as e: + raise RuntimeError(f"Could not load {name!r} (required for dlinfo/dlerror on Linux)") from e + + +LIBDL = _load_libdl() + +# dlinfo +LIBDL.dlinfo.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p] +LIBDL.dlinfo.restype = ctypes.c_int + +# dlerror (thread-local error string; cleared after read) +LIBDL.dlerror.argtypes = [] +LIBDL.dlerror.restype = ctypes.c_char_p + +# First appeared in 2004-era glibc. Universally correct on Linux for all practical purposes. +RTLD_DI_LINKMAP = 2 +RTLD_DI_ORIGIN = 6 + + +class _LinkMapLNameView(ctypes.Structure): + """ + Prefix-only view of glibc's `struct link_map` used **solely** to read `l_name`. + + Background: + - `dlinfo(handle, RTLD_DI_LINKMAP, ...)` returns a `struct link_map*`. + - The first few members of `struct link_map` (including `l_name`) have been + stable on glibc for decades and are documented as debugger-visible. + - We only need the offset/layout of `l_name`, not the full struct. + + Safety constraints: + - This is a **partial** definition (prefix). It must only be used via a pointer + returned by `dlinfo(...)`. + - Do **not** instantiate it or pass it **by value** to any C function. + - Do **not** access any members beyond those declared here. + - Do **not** rely on `ctypes.sizeof(LinkMapPrefix)` for allocation. + + Rationale: + - Defining only the leading fields avoids depending on internal/unstable + tail members while keeping code more readable than raw pointer arithmetic. + """ + + _fields_ = ( + ("l_addr", ctypes.c_void_p), # ElfW(Addr) + ("l_name", ctypes.c_char_p), # char* + ) + + +# Defensive assertions, mainly to document the invariants we depend on +assert _LinkMapLNameView.l_addr.offset == 0 +assert _LinkMapLNameView.l_name.offset == ctypes.sizeof(ctypes.c_void_p) + + +def _dl_last_error() -> str | None: + msg_bytes = cast(bytes | None, LIBDL.dlerror()) + if not msg_bytes: + return None # no pending error + # Never raises; undecodable bytes are mapped to U+DC80..U+DCFF + return msg_bytes.decode("utf-8", "surrogateescape") + + +def l_name_for_dynamic_library(libname: str, handle: ctypes.CDLL) -> str: + lm_view = ctypes.POINTER(_LinkMapLNameView)() + rc = LIBDL.dlinfo(ctypes.c_void_p(handle._handle), RTLD_DI_LINKMAP, ctypes.byref(lm_view)) + if rc != 0: + err = _dl_last_error() + raise OSError(f"dlinfo failed for {libname=!r} (rc={rc})" + (f": {err}" if err else "")) + if not lm_view: # NULL link_map** + raise OSError(f"dlinfo returned NULL link_map pointer for {libname=!r}") + + l_name_bytes = lm_view.contents.l_name + if not l_name_bytes: + raise OSError(f"dlinfo returned empty link_map->l_name for {libname=!r}") + + path = os.fsdecode(l_name_bytes) + if not path: + raise OSError(f"dlinfo returned empty l_name string for {libname=!r}") + + return path + + +def l_origin_for_dynamic_library(libname: str, handle: ctypes.CDLL) -> str: + l_origin_buf = ctypes.create_string_buffer(4096) + rc = LIBDL.dlinfo(ctypes.c_void_p(handle._handle), RTLD_DI_ORIGIN, l_origin_buf) + if rc != 0: + err = _dl_last_error() + raise OSError(f"dlinfo failed for {libname=!r} (rc={rc})" + (f": {err}" if err else "")) + + path = os.fsdecode(l_origin_buf.value) + if not path: + raise OSError(f"dlinfo returned empty l_origin string for {libname=!r}") + + return path + + +def abs_path_for_dynamic_library(libname: str, handle: ctypes.CDLL) -> str: + l_name = l_name_for_dynamic_library(libname, handle) + l_origin = l_origin_for_dynamic_library(libname, handle) + return os.path.join(l_origin, os.path.basename(l_name)) + + +def get_candidate_sonames(libname: str) -> list[str]: + # Reverse tabulated names to achieve new → old search order. + candidate_sonames = list(reversed(SUPPORTED_LINUX_SONAMES.get(libname, ()))) + candidate_sonames.append(f"lib{libname}.so") + return candidate_sonames + + +def check_if_already_loaded_from_elsewhere(libname: str, _have_abs_path: bool) -> LoadedDL | None: + for soname in get_candidate_sonames(libname): + try: + handle = ctypes.CDLL(soname, mode=os.RTLD_NOLOAD) + except OSError: + continue + else: + return LoadedDL( + abs_path_for_dynamic_library(libname, handle), True, handle._handle, "was-already-loaded-from-elsewhere" + ) + return None + + +def _load_lib(libname: str, filename: str) -> ctypes.CDLL: + cdll_mode = CDLL_MODE + if libname in LIBNAMES_REQUIRING_RTLD_DEEPBIND: + cdll_mode |= os.RTLD_DEEPBIND + return ctypes.CDLL(filename, cdll_mode) + + +def load_with_system_search(libname: str) -> LoadedDL | None: + """Try to load a library using system search paths. + + Args: + libname: The name of the library to load + + Returns: + A LoadedDL object if successful, None if the library cannot be loaded + + Raises: + RuntimeError: If the library is loaded but no expected symbol is found + """ + for soname in get_candidate_sonames(libname): + try: + handle = _load_lib(libname, soname) + except OSError: + pass + else: + abs_path = abs_path_for_dynamic_library(libname, handle) + if abs_path is None: + raise RuntimeError(f"No expected symbol for {libname=!r}") + return LoadedDL(abs_path, False, handle._handle, "system-search") + return None + + +def _work_around_known_bugs(libname: str, found_path: str) -> None: + if libname == "nvrtc": + # Work around bug/oversight in + # nvidia_cuda_nvrtc-13.0.48-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl + # Issue: libnvrtc.so.13 RUNPATH is not set. + # This workaround is highly specific + # - for simplicity. + # - to not mask bugs in future nvidia-cuda-nvrtc releases. + # - because a more general workaround is complicated. + dirname, basename = os.path.split(found_path) + if basename == "libnvrtc.so.13": + dep_basename = "libnvrtc-builtins.so.13.0" + dep_path = os.path.join(dirname, dep_basename) + if os.path.isfile(dep_path): + # In case of failure, defer to primary load, which is almost certain to fail, too. + with contextlib.suppress(OSError): + ctypes.CDLL(dep_path, CDLL_MODE) + + +def load_with_abs_path(libname: str, found_path: str, found_via: str | None = None) -> LoadedDL: + """Load a dynamic library from the given path. + + Args: + libname: The name of the library to load + found_path: The absolute path to the library file + + Returns: + A LoadedDL object representing the loaded library + + Raises: + RuntimeError: If the library cannot be loaded + """ + _work_around_known_bugs(libname, found_path) + try: + handle = _load_lib(libname, found_path) + except OSError as e: + raise RuntimeError(f"Failed to dlopen {found_path}: {e}") from e + return LoadedDL(found_path, False, handle._handle, found_via) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/load_dl_windows.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/load_dl_windows.py new file mode 100644 index 0000000000000000000000000000000000000000..b9f15ea50b20e2c0a34bb572837e63abd3c873a4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/load_dl_windows.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import ctypes +import ctypes.wintypes +import os +import struct + +from cuda.pathfinder._dynamic_libs.load_dl_common import LoadedDL +from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import ( + LIBNAMES_REQUIRING_OS_ADD_DLL_DIRECTORY, + SUPPORTED_WINDOWS_DLLS, +) + +# Mirrors WinBase.h (unfortunately not defined already elsewhere) +WINBASE_LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR = 0x00000100 +WINBASE_LOAD_LIBRARY_SEARCH_DEFAULT_DIRS = 0x00001000 + +POINTER_ADDRESS_SPACE = 2 ** (struct.calcsize("P") * 8) + +# Set up kernel32 functions with proper types +kernel32 = ctypes.windll.kernel32 # type: ignore[attr-defined] + +# GetModuleHandleW +kernel32.GetModuleHandleW.argtypes = [ctypes.wintypes.LPCWSTR] +kernel32.GetModuleHandleW.restype = ctypes.wintypes.HMODULE + +# LoadLibraryExW +kernel32.LoadLibraryExW.argtypes = [ + ctypes.wintypes.LPCWSTR, # lpLibFileName + ctypes.wintypes.HANDLE, # hFile (reserved, must be NULL) + ctypes.wintypes.DWORD, # dwFlags +] +kernel32.LoadLibraryExW.restype = ctypes.wintypes.HMODULE + +# GetModuleFileNameW +kernel32.GetModuleFileNameW.argtypes = [ + ctypes.wintypes.HMODULE, # hModule + ctypes.wintypes.LPWSTR, # lpFilename + ctypes.wintypes.DWORD, # nSize +] +kernel32.GetModuleFileNameW.restype = ctypes.wintypes.DWORD + +# AddDllDirectory (Windows 7+) +kernel32.AddDllDirectory.argtypes = [ctypes.wintypes.LPCWSTR] +kernel32.AddDllDirectory.restype = ctypes.c_void_p # DLL_DIRECTORY_COOKIE + + +def ctypes_handle_to_unsigned_int(handle: ctypes.wintypes.HMODULE) -> int: + """Convert ctypes HMODULE to unsigned int.""" + handle_uint = int(handle) + if handle_uint < 0: + # Convert from signed to unsigned representation + handle_uint += POINTER_ADDRESS_SPACE + return handle_uint + + +def add_dll_directory(dll_abs_path: str) -> None: + """Add a DLL directory to the search path and update PATH environment variable. + + Args: + dll_abs_path: Absolute path to the DLL file + + Raises: + AssertionError: If the directory containing the DLL does not exist + """ + dirpath = os.path.dirname(dll_abs_path) + assert os.path.isdir(dirpath), dll_abs_path + + # Add the DLL directory to the search path + result = kernel32.AddDllDirectory(dirpath) + if not result: + # Fallback: just update PATH if AddDllDirectory fails + pass + + # Update PATH as a fallback for dependent DLL resolution + curr_path = os.environ.get("PATH") + os.environ["PATH"] = dirpath if curr_path is None else os.pathsep.join((curr_path, dirpath)) + + +def abs_path_for_dynamic_library(libname: str, handle: ctypes.wintypes.HMODULE) -> str: + """Get the absolute path of a loaded dynamic library on Windows.""" + # Create buffer for the path + buffer = ctypes.create_unicode_buffer(260) # MAX_PATH + length = kernel32.GetModuleFileNameW(handle, buffer, len(buffer)) + + if length == 0: + error_code = ctypes.GetLastError() # type: ignore[attr-defined] + raise RuntimeError(f"GetModuleFileNameW failed for {libname!r} (error code: {error_code})") + + # If buffer was too small, try with larger buffer + if length == len(buffer): + buffer = ctypes.create_unicode_buffer(32768) # Extended path length + length = kernel32.GetModuleFileNameW(handle, buffer, len(buffer)) + if length == 0: + error_code = ctypes.GetLastError() # type: ignore[attr-defined] + raise RuntimeError(f"GetModuleFileNameW failed for {libname!r} (error code: {error_code})") + + return buffer.value + + +def check_if_already_loaded_from_elsewhere(libname: str, have_abs_path: bool) -> LoadedDL | None: + for dll_name in SUPPORTED_WINDOWS_DLLS.get(libname, ()): + handle = kernel32.GetModuleHandleW(dll_name) + if handle: + abs_path = abs_path_for_dynamic_library(libname, handle) + if have_abs_path and libname in LIBNAMES_REQUIRING_OS_ADD_DLL_DIRECTORY: + # This is a side-effect if the pathfinder loads the library via + # load_with_abs_path(). To make the side-effect more deterministic, + # activate it even if the library was already loaded from elsewhere. + add_dll_directory(abs_path) + return LoadedDL(abs_path, True, ctypes_handle_to_unsigned_int(handle), "was-already-loaded-from-elsewhere") + return None + + +def load_with_system_search(libname: str) -> LoadedDL | None: + """Try to load a DLL using system search paths. + + Args: + libname: The name of the library to load + + Returns: + A LoadedDL object if successful, None if the library cannot be loaded + """ + # Reverse tabulated names to achieve new → old search order. + for dll_name in reversed(SUPPORTED_WINDOWS_DLLS.get(libname, ())): + handle = kernel32.LoadLibraryExW(dll_name, None, 0) + if handle: + abs_path = abs_path_for_dynamic_library(libname, handle) + return LoadedDL(abs_path, False, ctypes_handle_to_unsigned_int(handle), "system-search") + + return None + + +def load_with_abs_path(libname: str, found_path: str, found_via: str | None = None) -> LoadedDL: + """Load a dynamic library from the given path. + + Args: + libname: The name of the library to load + found_path: The absolute path to the DLL file + + Returns: + A LoadedDL object representing the loaded library + + Raises: + RuntimeError: If the DLL cannot be loaded + """ + if libname in LIBNAMES_REQUIRING_OS_ADD_DLL_DIRECTORY: + add_dll_directory(found_path) + + flags = WINBASE_LOAD_LIBRARY_SEARCH_DEFAULT_DIRS | WINBASE_LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR + handle = kernel32.LoadLibraryExW(found_path, None, flags) + + if not handle: + error_code = ctypes.GetLastError() # type: ignore[attr-defined] + raise RuntimeError(f"Failed to load DLL at {found_path}: Windows error {error_code}") + + return LoadedDL(found_path, False, ctypes_handle_to_unsigned_int(handle), found_via) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/load_nvidia_dynamic_lib.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/load_nvidia_dynamic_lib.py new file mode 100644 index 0000000000000000000000000000000000000000..9df1c5de230d9203278ce6c3f2588bd420f4c12a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/load_nvidia_dynamic_lib.py @@ -0,0 +1,298 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import functools +import json +import struct +import sys + +from cuda.pathfinder._dynamic_libs.canary_probe_subprocess import probe_canary_abs_path_and_print_json +from cuda.pathfinder._dynamic_libs.find_nvidia_dynamic_lib import ( + _FindNvidiaDynamicLib, + derive_ctk_root, +) +from cuda.pathfinder._dynamic_libs.load_dl_common import ( + DynamicLibNotAvailableError, + DynamicLibNotFoundError, + DynamicLibUnknownError, + LoadedDL, + load_dependencies, +) +from cuda.pathfinder._dynamic_libs.supported_nvidia_libs import ( + _CTK_ROOT_CANARY_ANCHOR_LIBNAMES, + _CTK_ROOT_CANARY_DISCOVERABLE_LIBNAMES, + SUPPORTED_LINUX_SONAMES, + SUPPORTED_WINDOWS_DLLS, +) +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS +from cuda.pathfinder._utils.spawned_process_runner import run_in_spawned_child_process + +if IS_WINDOWS: + from cuda.pathfinder._dynamic_libs.load_dl_windows import ( + check_if_already_loaded_from_elsewhere, + load_with_abs_path, + load_with_system_search, + ) +else: + from cuda.pathfinder._dynamic_libs.load_dl_linux import ( + check_if_already_loaded_from_elsewhere, + load_with_abs_path, + load_with_system_search, + ) + +# All libnames recognized by load_nvidia_dynamic_lib, across all categories +# (CTK, third-party, driver). Built from the platform-appropriate soname/DLL +# registry so that platform-specific libs (e.g. cufile on Linux) are included +# only where they apply. +_ALL_SUPPORTED_LIBNAMES: frozenset[str] = frozenset( + (SUPPORTED_WINDOWS_DLLS if IS_WINDOWS else SUPPORTED_LINUX_SONAMES).keys() +) +_ALL_KNOWN_LIBNAMES: frozenset[str] = frozenset(SUPPORTED_LINUX_SONAMES) | frozenset(SUPPORTED_WINDOWS_DLLS) +_PLATFORM_NAME = "Windows" if IS_WINDOWS else "Linux" + +# Driver libraries: shipped with the NVIDIA display driver, always on the +# system linker path. These skip all CTK search steps (site-packages, +# conda, CUDA_HOME, canary) and go straight to system search. +_DRIVER_ONLY_LIBNAMES = frozenset(("cuda", "nvml")) + + +def _load_driver_lib_no_cache(libname: str) -> LoadedDL: + """Load an NVIDIA driver library (system-search only). + + Driver libs (libcuda, libnvidia-ml) are part of the display driver, not + the CUDA Toolkit. They are always on the system linker path, so the + full CTK search cascade (site-packages, conda, CUDA_HOME, canary) is + unnecessary. + """ + loaded = check_if_already_loaded_from_elsewhere(libname, False) + if loaded is not None: + return loaded + loaded = load_with_system_search(libname) + if loaded is not None: + return loaded + raise DynamicLibNotFoundError( + f'"{libname}" is an NVIDIA driver library and can only be found via' + f" system search. Ensure the NVIDIA display driver is installed." + ) + + +@functools.cache +def _resolve_system_loaded_abs_path_in_subprocess(libname: str) -> str | None: + """Resolve a library's system-search absolute path in a child process. + + This runs in a spawned (not forked) child process. Spawning is important + because it starts from a fresh interpreter state, so the child does not + inherit already-loaded CUDA dynamic libraries from the parent process + (especially the well-known canary probe library). + + That keeps any side-effects of loading the canary library scoped to the + child process instead of polluting the current process, and ensures the + canary probe is an independent system-search attempt. + """ + result = run_in_spawned_child_process( + probe_canary_abs_path_and_print_json, + args=(libname,), + timeout=10.0, + rethrow=True, + ) + + # Read the final non-empty stdout line in case earlier lines are emitted. + lines = [line for line in result.stdout.splitlines() if line.strip()] + if not lines: + raise RuntimeError(f"Canary probe child process produced no stdout payload for {libname!r}") + try: + payload = json.loads(lines[-1]) + except json.JSONDecodeError: + raise RuntimeError( + f"Canary probe child process emitted invalid JSON payload for {libname!r}: {lines[-1]!r}" + ) from None + if isinstance(payload, str): + return payload + if payload is None: + return None + raise RuntimeError(f"Canary probe child process emitted unexpected payload for {libname!r}: {payload!r}") + + +def _try_ctk_root_canary(finder: _FindNvidiaDynamicLib) -> str | None: + """Derive the CTK root from a system-installed canary lib. + + For discoverable libs (currently nvvm) whose shared object doesn't reside + on the standard linker path, we locate a well-known CTK lib that IS on + the linker path via system search, derive the CTK installation root from + its resolved path, and then look for the target lib relative to that root. + + The canary load is performed in a subprocess to avoid introducing loader + state into the current process. + """ + for canary_libname in _CTK_ROOT_CANARY_ANCHOR_LIBNAMES: + canary_abs_path = _resolve_system_loaded_abs_path_in_subprocess(canary_libname) + if canary_abs_path is None: + continue + ctk_root = derive_ctk_root(canary_abs_path) + if ctk_root is None: + continue + abs_path: str | None = finder.try_via_ctk_root(ctk_root) + if abs_path is not None: + return abs_path + return None + + +def _load_lib_no_cache(libname: str) -> LoadedDL: + if libname in _DRIVER_ONLY_LIBNAMES: + return _load_driver_lib_no_cache(libname) + + finder = _FindNvidiaDynamicLib(libname) + abs_path = finder.try_site_packages() + if abs_path is not None: + found_via = "site-packages" + else: + abs_path = finder.try_with_conda_prefix() + if abs_path is not None: + found_via = "conda" + + # If the library was already loaded by someone else, reproduce any OS-specific + # side-effects we would have applied on a direct absolute-path load (e.g., + # AddDllDirectory on Windows for libs that require it). + loaded = check_if_already_loaded_from_elsewhere(libname, abs_path is not None) + + # Load dependencies regardless of who loaded the primary lib first. + # Doing this *after* the side-effect ensures dependencies resolve consistently + # relative to the actually loaded location. + load_dependencies(libname, load_nvidia_dynamic_lib) + + if loaded is not None: + return loaded + + if abs_path is None: + loaded = load_with_system_search(libname) + if loaded is not None: + return loaded + + abs_path = finder.try_with_cuda_home() + if abs_path is not None: + found_via = "CUDA_HOME" + else: + if libname not in _CTK_ROOT_CANARY_DISCOVERABLE_LIBNAMES: + finder.raise_not_found_error() + + # Canary probe (discoverable libs only): if the direct system + # search and CUDA_HOME both failed (e.g. nvvm isn't on the linker + # path and CUDA_HOME is unset), try to discover the CTK root by + # loading a well-known CTK lib in a subprocess, then look for the + # target lib relative to that root. + abs_path = _try_ctk_root_canary(finder) + if abs_path is not None: + found_via = "system-ctk-root" + else: + finder.raise_not_found_error() + + return load_with_abs_path(libname, abs_path, found_via) + + +@functools.cache +def load_nvidia_dynamic_lib(libname: str) -> LoadedDL: + """Load an NVIDIA dynamic library by name. + + Args: + libname (str): The short name of the library to load (e.g., ``"cudart"``, + ``"nvvm"``, etc.). + + Returns: + LoadedDL: Object containing the OS library handle and absolute path. + + **Important:** + + **Never close the returned handle.** Do **not** call ``dlclose`` (Linux) or + ``FreeLibrary`` (Windows) on the ``LoadedDL._handle_uint``. + + **Why:** the return value is cached (``functools.cache``) and shared across the + process. Closing the handle can unload the module while other code still uses + it, leading to crashes or subtle failures. + + This applies to Linux and Windows. For context, see issue #1011: + https://github.com/NVIDIA/cuda-python/issues/1011 + + Raises: + DynamicLibUnknownError: If ``libname`` is not a recognized library name. + DynamicLibNotAvailableError: If ``libname`` is recognized but not + supported on this platform. + DynamicLibNotFoundError: If the library cannot be found or loaded. + RuntimeError: If Python is not 64-bit. + + Search order: + 0. **Already loaded in the current process** + + - If a matching library is already loaded by some other component, + return its absolute path and handle and skip the rest of the search. + + 1. **NVIDIA Python wheels** + + - Scan installed distributions (``site-packages``) to find libraries + shipped in NVIDIA wheels. + + 2. **Conda environment** + + - Conda installations are discovered via ``CONDA_PREFIX``, which is + defined automatically in activated conda environments (see + https://docs.conda.io/projects/conda-build/en/stable/user-guide/environment-variables.html). + + 3. **OS default mechanisms** + + - Fall back to the native loader: + + - Linux: ``dlopen()`` + + - Windows: ``LoadLibraryW()`` + + - CUDA Toolkit (CTK) system installs with system config updates are often + discovered via: + + - Linux: ``/etc/ld.so.conf.d/*cuda*.conf`` + + - Windows: ``C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\vX.Y\\bin`` + on the system ``PATH``. + + 4. **Environment variables** + + - If set, use ``CUDA_HOME`` or ``CUDA_PATH`` (in that order). + + 5. **CTK root canary probe (discoverable libs only)** + + - For selected libraries whose shared object doesn't reside on the + standard linker path (currently ``nvvm``), + attempt to discover the CTK installation root by system-loading a + well-known CTK library (``cudart``) in a subprocess, then derive + the root from its resolved absolute path. + + **Driver libraries** (``"cuda"``, ``"nvml"``): + + These are part of the NVIDIA display driver (not the CUDA Toolkit) and + are always on the system linker path. For these libraries the search + is simplified to: + + 0. Already loaded in the current process + 1. OS default mechanisms (``dlopen`` / ``LoadLibraryW``) + + The CTK-specific steps (site-packages, conda, ``CUDA_HOME``, canary + probe) are skipped entirely. + + Notes: + The search is performed **per library**. There is currently no mechanism to + guarantee that multiple libraries are all resolved from the same location. + + """ + pointer_size_bits = struct.calcsize("P") * 8 + if pointer_size_bits != 64: + raise RuntimeError( + f"cuda.pathfinder.load_nvidia_dynamic_lib() requires 64-bit Python." + f" Currently running: {pointer_size_bits}-bit Python" + f" {sys.version_info.major}.{sys.version_info.minor}" + ) + if libname not in _ALL_KNOWN_LIBNAMES: + raise DynamicLibUnknownError(f"Unknown library name: {libname!r}. Known names: {sorted(_ALL_KNOWN_LIBNAMES)}") + if libname not in _ALL_SUPPORTED_LIBNAMES: + raise DynamicLibNotAvailableError( + f"Library name {libname!r} is known but not available on {_PLATFORM_NAME}. " + f"Supported names on {_PLATFORM_NAME}: {sorted(_ALL_SUPPORTED_LIBNAMES)}" + ) + return _load_lib_no_cache(libname) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py new file mode 100644 index 0000000000000000000000000000000000000000..d4225233c2d203079afe509a834dabe4ebeec66b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_dynamic_libs/supported_nvidia_libs.py @@ -0,0 +1,460 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# THIS FILE NEEDS TO BE REVIEWED/UPDATED FOR EACH CTK RELEASE +# Likely candidates for updates are: +# SUPPORTED_LIBNAMES +# SUPPORTED_WINDOWS_DLLS +# SUPPORTED_LINUX_SONAMES + +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS + +SUPPORTED_LIBNAMES_COMMON = ( + # Core CUDA Runtime and Compiler + "cudart", + "nvfatbin", + "nvJitLink", + "nvrtc", + "nvvm", + # Math Libraries + "cublas", + "cublasLt", + "cufft", + "cufftw", + "curand", + "cusolver", + "cusolverMg", + "cusparse", + "nppc", + "nppial", + "nppicc", + "nppidei", + "nppif", + "nppig", + "nppim", + "nppist", + "nppisu", + "nppitc", + "npps", + "nvblas", + # Other + "nvjpeg", +) + +# Note: The `cufile_rdma` information is intentionally retained (commented out) +# despite not being actively used in the current build. It took a nontrivial +# amount of effort to determine the SONAME, dependencies, and expected symbols +# for this special-case library, especially given its RDMA/MLX5 dependencies +# and limited availability. Keeping this as a reference avoids having to +# reconstruct the information from scratch in the future. + +SUPPORTED_LIBNAMES_LINUX_ONLY = ( + "cufile", + # "cufile_rdma", # Requires libmlx5.so +) +SUPPORTED_LIBNAMES_LINUX = SUPPORTED_LIBNAMES_COMMON + SUPPORTED_LIBNAMES_LINUX_ONLY + +SUPPORTED_LIBNAMES_WINDOWS_ONLY = () +SUPPORTED_LIBNAMES_WINDOWS = SUPPORTED_LIBNAMES_COMMON + SUPPORTED_LIBNAMES_WINDOWS_ONLY + +SUPPORTED_LIBNAMES_ALL = SUPPORTED_LIBNAMES_COMMON + SUPPORTED_LIBNAMES_LINUX_ONLY + SUPPORTED_LIBNAMES_WINDOWS_ONLY +SUPPORTED_LIBNAMES = SUPPORTED_LIBNAMES_WINDOWS if IS_WINDOWS else SUPPORTED_LIBNAMES_LINUX + +# Based on ldd output for Linux x86_64 nvidia-*-cu12 wheels (12.8.1) +DIRECT_DEPENDENCIES_CTK = { + "cublas": ("cublasLt",), + "cufftw": ("cufft",), + # "cufile_rdma": ("cufile",), + "cusolver": ("nvJitLink", "cusparse", "cublasLt", "cublas"), + "cusolverMg": ("nvJitLink", "cublasLt", "cublas"), + "cusparse": ("nvJitLink",), + "nppial": ("nppc",), + "nppicc": ("nppc",), + "nppidei": ("nppc",), + "nppif": ("nppc",), + "nppig": ("nppc",), + "nppim": ("nppc",), + "nppist": ("nppc",), + "nppisu": ("nppc",), + "nppitc": ("nppc",), + "npps": ("nppc",), + "nvblas": ("cublas", "cublasLt"), +} +DIRECT_DEPENDENCIES = DIRECT_DEPENDENCIES_CTK | { + "mathdx": ("nvrtc",), + "cublasmp": ("cublas", "cublasLt", "nvshmem_host"), + "cufftMp": ("nvshmem_host",), + "cudss": ("cublas", "cublasLt"), + "cutensor": ("cublasLt",), + "cutensorMg": ("cutensor", "cublasLt"), +} + +# Based on these files: +# cuda_12.0.1_525.85.12_linux.run +# cuda_12.1.1_530.30.02_linux.run +# cuda_12.2.2_535.104.05_linux.run +# cuda_12.3.2_545.23.08_linux.run +# cuda_12.4.1_550.54.15_linux.run +# cuda_12.5.1_555.42.06_linux.run +# cuda_12.6.3_560.35.05_linux.run +# cuda_12.8.1_570.124.06_linux.run +# cuda_12.9.1_575.57.08_linux.run +# cuda_13.0.2_580.95.05_linux.run +# cuda_13.1.0_590.44.01_linux.run +# Generated with toolshed/build_pathfinder_sonames.py +# Please keep in old → new sort order. +SUPPORTED_LINUX_SONAMES_CTK = { + "cublas": ( + "libcublas.so.12", + "libcublas.so.13", + ), + "cublasLt": ( + "libcublasLt.so.12", + "libcublasLt.so.13", + ), + "cudart": ( + "libcudart.so.12", + "libcudart.so.13", + ), + "cufft": ( + "libcufft.so.11", + "libcufft.so.12", + ), + "cufftw": ( + "libcufftw.so.11", + "libcufftw.so.12", + ), + "cufile": ("libcufile.so.0",), + # "cufile_rdma": ("libcufile_rdma.so.1",), + "curand": ("libcurand.so.10",), + "cusolver": ( + "libcusolver.so.11", + "libcusolver.so.12", + ), + "cusolverMg": ( + "libcusolverMg.so.11", + "libcusolverMg.so.12", + ), + "cusparse": ("libcusparse.so.12",), + "nppc": ( + "libnppc.so.12", + "libnppc.so.13", + ), + "nppial": ( + "libnppial.so.12", + "libnppial.so.13", + ), + "nppicc": ( + "libnppicc.so.12", + "libnppicc.so.13", + ), + "nppidei": ( + "libnppidei.so.12", + "libnppidei.so.13", + ), + "nppif": ( + "libnppif.so.12", + "libnppif.so.13", + ), + "nppig": ( + "libnppig.so.12", + "libnppig.so.13", + ), + "nppim": ( + "libnppim.so.12", + "libnppim.so.13", + ), + "nppist": ( + "libnppist.so.12", + "libnppist.so.13", + ), + "nppisu": ( + "libnppisu.so.12", + "libnppisu.so.13", + ), + "nppitc": ( + "libnppitc.so.12", + "libnppitc.so.13", + ), + "npps": ( + "libnpps.so.12", + "libnpps.so.13", + ), + "nvJitLink": ( + "libnvJitLink.so.12", + "libnvJitLink.so.13", + ), + "nvblas": ( + "libnvblas.so.12", + "libnvblas.so.13", + ), + "nvfatbin": ( + "libnvfatbin.so.12", + "libnvfatbin.so.13", + ), + "nvjpeg": ( + "libnvjpeg.so.12", + "libnvjpeg.so.13", + ), + "nvrtc": ( + "libnvrtc.so.12", + "libnvrtc.so.13", + ), + "nvvm": ("libnvvm.so.4",), +} +SUPPORTED_LINUX_SONAMES_OTHER = { + "cublasmp": ("libcublasmp.so.0",), + "cufftMp": ("libcufftMp.so.12", "libcufftMp.so.11"), + "mathdx": ("libmathdx.so.0",), + "cudss": ("libcudss.so.0",), + "cusparseLt": ("libcusparseLt.so.0",), + "cutensor": ("libcutensor.so.2",), + "cutensorMg": ("libcutensorMg.so.2",), + "nccl": ("libnccl.so.2",), + "nvpl_fftw": ("libnvpl_fftw.so.0",), + "nvshmem_host": ("libnvshmem_host.so.3",), +} +# Driver libraries: shipped with the NVIDIA driver, always on the system +# linker path. Only system search is needed (no site-packages / conda / +# CUDA_HOME). +SUPPORTED_LINUX_SONAMES_DRIVER = { + "cuda": ("libcuda.so.1",), + "nvml": ("libnvidia-ml.so.1",), +} +SUPPORTED_LINUX_SONAMES = SUPPORTED_LINUX_SONAMES_CTK | SUPPORTED_LINUX_SONAMES_OTHER | SUPPORTED_LINUX_SONAMES_DRIVER + +# Based on these files: +# cuda_12.0.1_528.33_windows.exe +# cuda_12.1.1_531.14_windows.exe +# cuda_12.2.2_537.13_windows.exe +# cuda_12.3.2_546.12_windows.exe +# cuda_12.4.1_551.78_windows.exe +# cuda_12.5.1_555.85_windows.exe +# cuda_12.6.3_561.17_windows.exe +# cuda_12.8.1_572.61_windows.exe +# cuda_12.9.1_576.57_windows.exe +# cuda_13.0.2_windows.exe +# cuda_13.1.0_windows.exe +# Generated with toolshed/build_pathfinder_dlls.py +# Please keep in old → new sort order. +SUPPORTED_WINDOWS_DLLS_CTK = { + "cublas": ( + "cublas64_12.dll", + "cublas64_13.dll", + ), + "cublasLt": ( + "cublasLt64_12.dll", + "cublasLt64_13.dll", + ), + "cudart": ( + "cudart64_12.dll", + "cudart64_13.dll", + ), + "cufft": ( + "cufft64_11.dll", + "cufft64_12.dll", + ), + "cufftw": ( + "cufftw64_11.dll", + "cufftw64_12.dll", + ), + "curand": ("curand64_10.dll",), + "cusolver": ( + "cusolver64_11.dll", + "cusolver64_12.dll", + ), + "cusolverMg": ( + "cusolverMg64_11.dll", + "cusolverMg64_12.dll", + ), + "cusparse": ("cusparse64_12.dll",), + "nppc": ( + "nppc64_12.dll", + "nppc64_13.dll", + ), + "nppial": ( + "nppial64_12.dll", + "nppial64_13.dll", + ), + "nppicc": ( + "nppicc64_12.dll", + "nppicc64_13.dll", + ), + "nppidei": ( + "nppidei64_12.dll", + "nppidei64_13.dll", + ), + "nppif": ( + "nppif64_12.dll", + "nppif64_13.dll", + ), + "nppig": ( + "nppig64_12.dll", + "nppig64_13.dll", + ), + "nppim": ( + "nppim64_12.dll", + "nppim64_13.dll", + ), + "nppist": ( + "nppist64_12.dll", + "nppist64_13.dll", + ), + "nppisu": ( + "nppisu64_12.dll", + "nppisu64_13.dll", + ), + "nppitc": ( + "nppitc64_12.dll", + "nppitc64_13.dll", + ), + "npps": ( + "npps64_12.dll", + "npps64_13.dll", + ), + "nvJitLink": ( + "nvJitLink_120_0.dll", + "nvJitLink_130_0.dll", + ), + "nvblas": ( + "nvblas64_12.dll", + "nvblas64_13.dll", + ), + "nvfatbin": ( + "nvfatbin_120_0.dll", + "nvfatbin_130_0.dll", + ), + "nvjpeg": ( + "nvjpeg64_12.dll", + "nvjpeg64_13.dll", + ), + "nvrtc": ( + "nvrtc64_120_0.dll", + "nvrtc64_130_0.dll", + ), + "nvvm": ( + "nvvm64.dll", + "nvvm64_40_0.dll", + "nvvm70.dll", + ), +} +SUPPORTED_WINDOWS_DLLS_OTHER = { + "mathdx": ("mathdx64_0.dll",), + "cudss": ("cudss64_0.dll",), + "cusparseLt": ("cusparseLt.dll",), + "cutensor": ("cutensor.dll",), + "cutensorMg": ("cutensorMg.dll",), +} +SUPPORTED_WINDOWS_DLLS_DRIVER = { + "cuda": ("nvcuda.dll",), + "nvml": ("nvml.dll",), +} +SUPPORTED_WINDOWS_DLLS = SUPPORTED_WINDOWS_DLLS_CTK | SUPPORTED_WINDOWS_DLLS_OTHER | SUPPORTED_WINDOWS_DLLS_DRIVER + +LIBNAMES_REQUIRING_OS_ADD_DLL_DIRECTORY = ( + "cufft", + "nvrtc", +) + +LIBNAMES_REQUIRING_RTLD_DEEPBIND = ("cufftMp",) + +# CTK root canary probe config: +# - anchor libs: expected on the standard system loader path and used to derive +# CTK root in an isolated child process. +# - discoverable libs: libs that are allowed to use the CTK-root canary fallback. +_CTK_ROOT_CANARY_ANCHOR_LIBNAMES = ("cudart",) +_CTK_ROOT_CANARY_DISCOVERABLE_LIBNAMES = ("nvvm",) + +# Based on output of toolshed/make_site_packages_libdirs_linux.py +SITE_PACKAGES_LIBDIRS_LINUX_CTK = { + "cublas": ("nvidia/cu13/lib", "nvidia/cublas/lib"), + "cublasLt": ("nvidia/cu13/lib", "nvidia/cublas/lib"), + "cudart": ("nvidia/cu13/lib", "nvidia/cuda_runtime/lib"), + "cufft": ("nvidia/cu13/lib", "nvidia/cufft/lib"), + "cufftw": ("nvidia/cu13/lib", "nvidia/cufft/lib"), + "cufile": ("nvidia/cu13/lib", "nvidia/cufile/lib"), + # "cufile_rdma": ("nvidia/cu13/lib", "nvidia/cufile/lib"), + "curand": ("nvidia/cu13/lib", "nvidia/curand/lib"), + "cusolver": ("nvidia/cu13/lib", "nvidia/cusolver/lib"), + "cusolverMg": ("nvidia/cu13/lib", "nvidia/cusolver/lib"), + "cusparse": ("nvidia/cu13/lib", "nvidia/cusparse/lib"), + "nppc": ("nvidia/cu13/lib", "nvidia/npp/lib"), + "nppial": ("nvidia/cu13/lib", "nvidia/npp/lib"), + "nppicc": ("nvidia/cu13/lib", "nvidia/npp/lib"), + "nppidei": ("nvidia/cu13/lib", "nvidia/npp/lib"), + "nppif": ("nvidia/cu13/lib", "nvidia/npp/lib"), + "nppig": ("nvidia/cu13/lib", "nvidia/npp/lib"), + "nppim": ("nvidia/cu13/lib", "nvidia/npp/lib"), + "nppist": ("nvidia/cu13/lib", "nvidia/npp/lib"), + "nppisu": ("nvidia/cu13/lib", "nvidia/npp/lib"), + "nppitc": ("nvidia/cu13/lib", "nvidia/npp/lib"), + "npps": ("nvidia/cu13/lib", "nvidia/npp/lib"), + "nvJitLink": ("nvidia/cu13/lib", "nvidia/nvjitlink/lib"), + "nvblas": ("nvidia/cu13/lib", "nvidia/cublas/lib"), + "nvfatbin": ("nvidia/cu13/lib", "nvidia/nvfatbin/lib"), + "nvjpeg": ("nvidia/cu13/lib", "nvidia/nvjpeg/lib"), + "nvrtc": ("nvidia/cu13/lib", "nvidia/cuda_nvrtc/lib"), + "nvvm": ("nvidia/cu13/lib", "nvidia/cuda_nvcc/nvvm/lib64"), +} +SITE_PACKAGES_LIBDIRS_LINUX_OTHER = { + "cublasmp": ("nvidia/cublasmp/cu13/lib", "nvidia/cublasmp/cu12/lib"), + "cudss": ("nvidia/cu13/lib", "nvidia/cu12/lib"), + "cufftMp": ("nvidia/cufftmp/cu13/lib", "nvidia/cufftmp/cu12/lib"), + "cusparseLt": ("nvidia/cusparselt/lib",), + "cutensor": ("cutensor/lib",), + "cutensorMg": ("cutensor/lib",), + "mathdx": ("nvidia/cu13/lib", "nvidia/cu12/lib"), + "nccl": ("nvidia/nccl/lib",), + "nvpl_fftw": ("nvpl/lib",), + "nvshmem_host": ("nvidia/nvshmem/lib",), +} +SITE_PACKAGES_LIBDIRS_LINUX = SITE_PACKAGES_LIBDIRS_LINUX_CTK | SITE_PACKAGES_LIBDIRS_LINUX_OTHER + +# Based on output of toolshed/make_site_packages_libdirs_windows.py +SITE_PACKAGES_LIBDIRS_WINDOWS_CTK = { + "cublas": ("nvidia/cu13/bin/x86_64", "nvidia/cublas/bin"), + "cublasLt": ("nvidia/cu13/bin/x86_64", "nvidia/cublas/bin"), + "cudart": ("nvidia/cu13/bin/x86_64", "nvidia/cuda_runtime/bin"), + "cufft": ("nvidia/cu13/bin/x86_64", "nvidia/cufft/bin"), + "cufftw": ("nvidia/cu13/bin/x86_64", "nvidia/cufft/bin"), + "curand": ("nvidia/cu13/bin/x86_64", "nvidia/curand/bin"), + "cusolver": ("nvidia/cu13/bin/x86_64", "nvidia/cusolver/bin"), + "cusolverMg": ("nvidia/cu13/bin/x86_64", "nvidia/cusolver/bin"), + "cusparse": ("nvidia/cu13/bin/x86_64", "nvidia/cusparse/bin"), + "nppc": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"), + "nppial": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"), + "nppicc": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"), + "nppidei": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"), + "nppif": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"), + "nppig": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"), + "nppim": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"), + "nppist": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"), + "nppisu": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"), + "nppitc": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"), + "npps": ("nvidia/cu13/bin/x86_64", "nvidia/npp/bin"), + "nvJitLink": ("nvidia/cu13/bin/x86_64", "nvidia/nvjitlink/bin"), + "nvblas": ("nvidia/cu13/bin/x86_64", "nvidia/cublas/bin"), + "nvfatbin": ("nvidia/cu13/bin/x86_64", "nvidia/nvfatbin/bin"), + "nvjpeg": ("nvidia/cu13/bin/x86_64", "nvidia/nvjpeg/bin"), + "nvrtc": ("nvidia/cu13/bin/x86_64", "nvidia/cuda_nvrtc/bin"), + "nvvm": ("nvidia/cu13/bin/x86_64", "nvidia/cuda_nvcc/nvvm/bin"), +} +SITE_PACKAGES_LIBDIRS_WINDOWS_OTHER = { + "cudss": ("nvidia/cu13/bin", "nvidia/cu12/bin"), + "mathdx": ("nvidia/cu13/bin/x86_64", "nvidia/cu12/bin"), + "cusparseLt": ("nvidia/cusparselt/bin",), + "cutensor": ("cutensor/bin",), + "cutensorMg": ("cutensor/bin",), +} +SITE_PACKAGES_LIBDIRS_WINDOWS = SITE_PACKAGES_LIBDIRS_WINDOWS_CTK | SITE_PACKAGES_LIBDIRS_WINDOWS_OTHER + + +def is_suppressed_dll_file(path_basename: str) -> bool: + if path_basename.startswith("nvrtc"): + # nvidia_cuda_nvrtc_cu12-12.8.93-py3-none-win_amd64.whl: + # nvidia\cuda_nvrtc\bin\ + # nvrtc-builtins64_128.dll + # nvrtc64_120_0.alt.dll + # nvrtc64_120_0.dll + return path_basename.endswith(".alt.dll") or "-builtins" in path_basename + return path_basename.startswith(("cudart32_", "nvvm32")) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_headers/__pycache__/find_nvidia_headers.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_headers/__pycache__/find_nvidia_headers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..20720530fdd01c2312f9d4e8dc688dcd5924af95 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_headers/__pycache__/find_nvidia_headers.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_headers/__pycache__/supported_nvidia_headers.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_headers/__pycache__/supported_nvidia_headers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a3dd0379db193673d2a7567f4f648ded4f91c794 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_headers/__pycache__/supported_nvidia_headers.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_headers/find_nvidia_headers.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_headers/find_nvidia_headers.py new file mode 100644 index 0000000000000000000000000000000000000000..637ecabfcb6cabb04e9a9b2796fad974bdf62567 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_headers/find_nvidia_headers.py @@ -0,0 +1,201 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import functools +import glob +import os +from dataclasses import dataclass + +from cuda.pathfinder._headers import supported_nvidia_headers +from cuda.pathfinder._utils.env_vars import get_cuda_home_or_path +from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs_all_sitepackages +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS + + +@dataclass +class LocatedHeaderDir: + abs_path: str | None + found_via: str + + def __post_init__(self) -> None: + self.abs_path = _abs_norm(self.abs_path) + + +def _abs_norm(path: str | None) -> str | None: + if path: + return os.path.normpath(os.path.abspath(path)) + return None + + +def _joined_isfile(dirpath: str, basename: str) -> bool: + return os.path.isfile(os.path.join(dirpath, basename)) + + +def _locate_under_site_packages(sub_dir: str, h_basename: str) -> LocatedHeaderDir | None: + # Installed from a wheel + hdr_dir: str # help mypy + for hdr_dir in find_sub_dirs_all_sitepackages(tuple(sub_dir.split("/"))): + if _joined_isfile(hdr_dir, h_basename): + return LocatedHeaderDir(abs_path=hdr_dir, found_via="site-packages") + return None + + +def _locate_based_on_ctk_layout(libname: str, h_basename: str, anchor_point: str) -> str | None: + parts = [anchor_point] + if libname == "nvvm": + parts.append(libname) + parts.append("include") + idir = os.path.join(*parts) + if libname == "cccl": + if IS_WINDOWS: + cdir_ctk12 = os.path.join(idir, "targets", "x64") # conda has this anomaly + cdir_ctk13 = os.path.join(cdir_ctk12, "cccl") + if _joined_isfile(cdir_ctk13, h_basename): + return cdir_ctk13 + if _joined_isfile(cdir_ctk12, h_basename): + return cdir_ctk12 + cdir = os.path.join(idir, "cccl") # CTK 13 + if _joined_isfile(cdir, h_basename): + return cdir + if _joined_isfile(idir, h_basename): + return idir + return None + + +def _find_based_on_conda_layout(libname: str, h_basename: str, ctk_layout: bool) -> LocatedHeaderDir | None: + conda_prefix = os.environ.get("CONDA_PREFIX") + if not conda_prefix: + return None + if IS_WINDOWS: + anchor_point = os.path.join(conda_prefix, "Library") + if not os.path.isdir(anchor_point): + return None + else: + if ctk_layout: + targets_include_path = glob.glob(os.path.join(conda_prefix, "targets", "*", "include")) + if not targets_include_path: + return None + if len(targets_include_path) != 1: + # Conda does not support multiple architectures. + # QUESTION(PR#956): Do we want to issue a warning? + return None + include_path = targets_include_path[0] + else: + include_path = os.path.join(conda_prefix, "include") + anchor_point = os.path.dirname(include_path) + found_header_path = _locate_based_on_ctk_layout(libname, h_basename, anchor_point) + if found_header_path: + return LocatedHeaderDir(abs_path=found_header_path, found_via="conda") + return None + + +def _find_ctk_header_directory(libname: str) -> LocatedHeaderDir | None: + h_basename = supported_nvidia_headers.SUPPORTED_HEADERS_CTK[libname] + candidate_dirs = supported_nvidia_headers.SUPPORTED_SITE_PACKAGE_HEADER_DIRS_CTK[libname] + + for cdir in candidate_dirs: + if hdr_dir := _locate_under_site_packages(cdir, h_basename): + return hdr_dir + + if hdr_dir := _find_based_on_conda_layout(libname, h_basename, True): + return hdr_dir + + cuda_home = get_cuda_home_or_path() + if cuda_home: # noqa: SIM102 + if result := _locate_based_on_ctk_layout(libname, h_basename, cuda_home): + return LocatedHeaderDir(abs_path=result, found_via="CUDA_HOME") + + return None + + +@functools.cache +def locate_nvidia_header_directory(libname: str) -> LocatedHeaderDir | None: + """Locate the header directory for a supported NVIDIA library. + + Args: + libname (str): The short name of the library whose headers are needed + (e.g., ``"nvrtc"``, ``"cusolver"``, ``"nvshmem"``). + + Returns: + LocatedHeaderDir or None: A LocatedHeaderDir object containing the absolute path + to the discovered header directory and information about where it was found, + or ``None`` if the headers cannot be found. + + Raises: + RuntimeError: If ``libname`` is not in the supported set. + + Search order: + 1. **NVIDIA Python wheels** + + - Scan installed distributions (``site-packages``) for header layouts + shipped in NVIDIA wheels (e.g., ``cuda-toolkit[nvrtc]``). + + 2. **Conda environments** + + - Check Conda-style installation prefixes, which use platform-specific + include directory layouts. + + 3. **CUDA Toolkit environment variables** + + - Use ``CUDA_HOME`` or ``CUDA_PATH`` (in that order). + """ + + if libname in supported_nvidia_headers.SUPPORTED_HEADERS_CTK: + return _find_ctk_header_directory(libname) + + h_basename = supported_nvidia_headers.SUPPORTED_HEADERS_NON_CTK.get(libname) + if h_basename is None: + raise RuntimeError(f"UNKNOWN {libname=}") + + candidate_dirs = supported_nvidia_headers.SUPPORTED_SITE_PACKAGE_HEADER_DIRS_NON_CTK.get(libname, []) + + for cdir in candidate_dirs: + if found_hdr := _locate_under_site_packages(cdir, h_basename): + return found_hdr + + if found_hdr := _find_based_on_conda_layout(libname, h_basename, False): + return found_hdr + + # Fall back to system install directories + candidate_dirs = supported_nvidia_headers.SUPPORTED_INSTALL_DIRS_NON_CTK.get(libname, []) + for cdir in candidate_dirs: + for hdr_dir in sorted(glob.glob(cdir), reverse=True): + if _joined_isfile(hdr_dir, h_basename): + # For system installs, we don't have a clear found_via, so use "system" + return LocatedHeaderDir(abs_path=hdr_dir, found_via="supported_install_dir") + return None + + +def find_nvidia_header_directory(libname: str) -> str | None: + """Locate the header directory for a supported NVIDIA library. + + Args: + libname (str): The short name of the library whose headers are needed + (e.g., ``"nvrtc"``, ``"cusolver"``, ``"nvshmem"``). + + Returns: + str or None: Absolute path to the discovered header directory, or ``None`` + if the headers cannot be found. + + Raises: + RuntimeError: If ``libname`` is not in the supported set. + + Search order: + 1. **NVIDIA Python wheels** + + - Scan installed distributions (``site-packages``) for header layouts + shipped in NVIDIA wheels (e.g., ``cuda-toolkit[nvrtc]``). + + 2. **Conda environments** + + - Check Conda-style installation prefixes, which use platform-specific + include directory layouts. + + 3. **CUDA Toolkit environment variables** + + - Use ``CUDA_HOME`` or ``CUDA_PATH`` (in that order). + """ + found = locate_nvidia_header_directory(libname) + return found.abs_path if found else None diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_headers/supported_nvidia_headers.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_headers/supported_nvidia_headers.py new file mode 100644 index 0000000000000000000000000000000000000000..af29551dd834ab9a0051b5b83eefa142855d3a55 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_headers/supported_nvidia_headers.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from typing import Final + +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS + +SUPPORTED_HEADERS_CTK_COMMON = { + "cccl": "cuda/std/version", + "cublas": "cublas.h", + "cudart": "cuda_runtime.h", + "cufft": "cufft.h", + "curand": "curand.h", + "cusolver": "cusolverDn.h", + "cusparse": "cusparse.h", + "npp": "npp.h", + "nvcc": "fatbinary_section.h", + "nvfatbin": "nvFatbin.h", + "nvjitlink": "nvJitLink.h", + "nvjpeg": "nvjpeg.h", + "nvrtc": "nvrtc.h", + "nvvm": "nvvm.h", +} + +SUPPORTED_HEADERS_CTK_LINUX_ONLY = { + "cufile": "cufile.h", +} +SUPPORTED_HEADERS_CTK_LINUX = SUPPORTED_HEADERS_CTK_COMMON | SUPPORTED_HEADERS_CTK_LINUX_ONLY + +SUPPORTED_HEADERS_CTK_WINDOWS_ONLY: dict[str, str] = {} +SUPPORTED_HEADERS_CTK_WINDOWS = SUPPORTED_HEADERS_CTK_COMMON | SUPPORTED_HEADERS_CTK_WINDOWS_ONLY + +SUPPORTED_HEADERS_CTK_ALL = ( + SUPPORTED_HEADERS_CTK_COMMON | SUPPORTED_HEADERS_CTK_LINUX_ONLY | SUPPORTED_HEADERS_CTK_WINDOWS_ONLY +) +SUPPORTED_HEADERS_CTK: Final[dict[str, str]] = ( + SUPPORTED_HEADERS_CTK_WINDOWS if IS_WINDOWS else SUPPORTED_HEADERS_CTK_LINUX +) + +SUPPORTED_SITE_PACKAGE_HEADER_DIRS_CTK = { + "cccl": ( + "nvidia/cu13/include/cccl", # cuda-toolkit[cccl]==13.* + "nvidia/cuda_cccl/include", # cuda-toolkit[cccl]==12.* + ), + "cublas": ("nvidia/cu13/include", "nvidia/cublas/include"), + "cudart": ("nvidia/cu13/include", "nvidia/cuda_runtime/include"), + "cufft": ("nvidia/cu13/include", "nvidia/cufft/include"), + "cufile": ("nvidia/cu13/include", "nvidia/cufile/include"), + "curand": ("nvidia/cu13/include", "nvidia/curand/include"), + "cusolver": ("nvidia/cu13/include", "nvidia/cusolver/include"), + "cusparse": ("nvidia/cu13/include", "nvidia/cusparse/include"), + "npp": ("nvidia/cu13/include", "nvidia/npp/include"), + "nvcc": ("nvidia/cu13/include", "nvidia/cuda_nvcc/include"), + "nvfatbin": ("nvidia/cu13/include", "nvidia/nvfatbin/include"), + "nvjitlink": ("nvidia/cu13/include", "nvidia/nvjitlink/include"), + "nvjpeg": ("nvidia/cu13/include", "nvidia/nvjpeg/include"), + "nvrtc": ("nvidia/cu13/include", "nvidia/cuda_nvrtc/include"), + "nvvm": ("nvidia/cu13/include", "nvidia/cuda_nvcc/nvvm/include"), +} + +SUPPORTED_HEADERS_NON_CTK_COMMON = { + "cusparseLt": "cusparseLt.h", + "cutensor": "cutensor.h", +} +SUPPORTED_HEADERS_NON_CTK_LINUX_ONLY = { + "nvshmem": "nvshmem.h", +} +SUPPORTED_HEADERS_NON_CTK_WINDOWS_ONLY: Final[dict[str, str]] = {} +SUPPORTED_HEADERS_NON_CTK_LINUX = SUPPORTED_HEADERS_NON_CTK_COMMON | SUPPORTED_HEADERS_NON_CTK_LINUX_ONLY +SUPPORTED_HEADERS_NON_CTK_WINDOWS = SUPPORTED_HEADERS_NON_CTK_COMMON | SUPPORTED_HEADERS_NON_CTK_WINDOWS_ONLY +SUPPORTED_HEADERS_NON_CTK_ALL = ( + SUPPORTED_HEADERS_NON_CTK_COMMON | SUPPORTED_HEADERS_NON_CTK_LINUX_ONLY | SUPPORTED_HEADERS_NON_CTK_WINDOWS_ONLY +) +SUPPORTED_HEADERS_NON_CTK: Final[dict[str, str]] = ( + SUPPORTED_HEADERS_NON_CTK_WINDOWS if IS_WINDOWS else SUPPORTED_HEADERS_NON_CTK_LINUX +) + +SUPPORTED_SITE_PACKAGE_HEADER_DIRS_NON_CTK = { + "cusparseLt": ("nvidia/cusparselt/include",), + "cutensor": ("cutensor/include",), + "nvshmem": ("nvidia/nvshmem/include",), +} + +SUPPORTED_INSTALL_DIRS_NON_CTK = { + "nvshmem": ("/usr/include/nvshmem_*",), +} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_static_libs/__pycache__/find_bitcode_lib.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_static_libs/__pycache__/find_bitcode_lib.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b1e31ca1a9d8379e480a38328883dea97a69c8d7 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_static_libs/__pycache__/find_bitcode_lib.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_static_libs/find_bitcode_lib.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_static_libs/find_bitcode_lib.py new file mode 100644 index 0000000000000000000000000000000000000000..109f9eb53f2e367005e43cfbeda9014adc8db8b0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_static_libs/find_bitcode_lib.py @@ -0,0 +1,162 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import functools +import os +from dataclasses import dataclass +from typing import NoReturn, TypedDict + +from cuda.pathfinder._utils.env_vars import get_cuda_home_or_path +from cuda.pathfinder._utils.find_sub_dirs import find_sub_dirs_all_sitepackages +from cuda.pathfinder._utils.platform_aware import IS_WINDOWS + + +class BitcodeLibNotFoundError(RuntimeError): + """Raised when a bitcode library cannot be found.""" + + +@dataclass(frozen=True) +class LocatedBitcodeLib: + """Information about a located bitcode library.""" + + name: str + abs_path: str + filename: str + found_via: str + + +class _BitcodeLibInfo(TypedDict): + filename: str + rel_path: str + site_packages_dirs: tuple[str, ...] + + +_SUPPORTED_BITCODE_LIBS_INFO: dict[str, _BitcodeLibInfo] = { + "device": { + "filename": "libdevice.10.bc", + "rel_path": os.path.join("nvvm", "libdevice"), + "site_packages_dirs": ( + "nvidia/cu13/nvvm/libdevice", + "nvidia/cuda_nvcc/nvvm/libdevice", + ), + }, +} + +# Public API: just the supported library names +SUPPORTED_BITCODE_LIBS: tuple[str, ...] = tuple(sorted(_SUPPORTED_BITCODE_LIBS_INFO.keys())) + + +def _no_such_file_in_dir(dir_path: str, filename: str, error_messages: list[str], attachments: list[str]) -> None: + error_messages.append(f"No such file: {os.path.join(dir_path, filename)}") + if os.path.isdir(dir_path): + attachments.append(f' listdir("{dir_path}"):') + for node in sorted(os.listdir(dir_path)): + attachments.append(f" {node}") + else: + attachments.append(f' Directory does not exist: "{dir_path}"') + + +class _FindBitcodeLib: + def __init__(self, name: str) -> None: + if name not in _SUPPORTED_BITCODE_LIBS_INFO: # Updated reference + raise ValueError(f"Unknown bitcode library: '{name}'. Supported: {', '.join(SUPPORTED_BITCODE_LIBS)}") + self.name: str = name + self.config: _BitcodeLibInfo = _SUPPORTED_BITCODE_LIBS_INFO[name] # Updated reference + self.filename: str = self.config["filename"] + self.rel_path: str = self.config["rel_path"] + self.site_packages_dirs: tuple[str, ...] = self.config["site_packages_dirs"] + self.error_messages: list[str] = [] + self.attachments: list[str] = [] + + def try_site_packages(self) -> str | None: + for rel_dir in self.site_packages_dirs: + sub_dir = tuple(rel_dir.split("/")) + for abs_dir in find_sub_dirs_all_sitepackages(sub_dir): + file_path = os.path.join(abs_dir, self.filename) + if os.path.isfile(file_path): + return file_path + return None + + def try_with_conda_prefix(self) -> str | None: + conda_prefix = os.environ.get("CONDA_PREFIX") + if not conda_prefix: + return None + + anchor = os.path.join(conda_prefix, "Library") if IS_WINDOWS else conda_prefix + file_path = os.path.join(anchor, self.rel_path, self.filename) + if os.path.isfile(file_path): + return file_path + return None + + def try_with_cuda_home(self) -> str | None: + cuda_home = get_cuda_home_or_path() + if cuda_home is None: + self.error_messages.append("CUDA_HOME/CUDA_PATH not set") + return None + + file_path = os.path.join(cuda_home, self.rel_path, self.filename) + if os.path.isfile(file_path): + return file_path + + _no_such_file_in_dir( + os.path.join(cuda_home, self.rel_path), + self.filename, + self.error_messages, + self.attachments, + ) + return None + + def raise_not_found_error(self) -> NoReturn: + err = ", ".join(self.error_messages) if self.error_messages else "No search paths available" + att = "\n".join(self.attachments) if self.attachments else "" + raise BitcodeLibNotFoundError(f'Failure finding "{self.filename}": {err}\n{att}') + + +def locate_bitcode_lib(name: str) -> LocatedBitcodeLib: + """Locate a bitcode library by name. + + Raises: + ValueError: If ``name`` is not a supported bitcode library. + BitcodeLibNotFoundError: If the bitcode library cannot be found. + """ + finder = _FindBitcodeLib(name) + + abs_path = finder.try_site_packages() + if abs_path is not None: + return LocatedBitcodeLib( + name=name, + abs_path=abs_path, + filename=finder.filename, + found_via="site-packages", + ) + + abs_path = finder.try_with_conda_prefix() + if abs_path is not None: + return LocatedBitcodeLib( + name=name, + abs_path=abs_path, + filename=finder.filename, + found_via="conda", + ) + + abs_path = finder.try_with_cuda_home() + if abs_path is not None: + return LocatedBitcodeLib( + name=name, + abs_path=abs_path, + filename=finder.filename, + found_via="CUDA_HOME", + ) + + finder.raise_not_found_error() + + +@functools.cache +def find_bitcode_lib(name: str) -> str: + """Find the absolute path to a bitcode library. + + Raises: + ValueError: If ``name`` is not a supported bitcode library. + BitcodeLibNotFoundError: If the bitcode library cannot be found. + """ + return locate_bitcode_lib(name).abs_path diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/env_vars.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/env_vars.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0eb7af3ce84b9da6b9141f799e3373b7f43ea99a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/env_vars.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/find_site_packages_dll.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/find_site_packages_dll.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b30a850925e0978f79eb2c5a0434a57fd8bbcd4e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/find_site_packages_dll.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/find_site_packages_so.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/find_site_packages_so.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5a4c701feb3ec405948c82156c344d02a5ec8706 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/find_site_packages_so.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/find_sub_dirs.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/find_sub_dirs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4c1b130ffb61c6494c73b3e3b2bfc24b8551e993 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/find_sub_dirs.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/platform_aware.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/platform_aware.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a279c17d8e33c8a4705f4ac8319df733da432564 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/platform_aware.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/spawned_process_runner.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/spawned_process_runner.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8c731d5181c178f4b9e9ccdfe925248ccac83b2a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/__pycache__/spawned_process_runner.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/env_vars.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/env_vars.py new file mode 100644 index 0000000000000000000000000000000000000000..cf78a627cb12f58dc3510eaad03dae6988772fc5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/env_vars.py @@ -0,0 +1,51 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +import warnings + + +def _paths_differ(a: str, b: str) -> bool: + """ + Return True if paths are observably different. + + Strategy: + 1) Compare os.path.normcase(os.path.normpath(...)) for quick, robust textual equality. + - Handles trailing slashes and case-insensitivity on Windows. + 2) If still different AND both exist, use os.path.samefile to resolve symlinks/junctions. + 3) Otherwise (nonexistent paths or samefile unavailable), treat as different. + """ + norm_a = os.path.normcase(os.path.normpath(a)) + norm_b = os.path.normcase(os.path.normpath(b)) + if norm_a == norm_b: + return False + + try: + if os.path.exists(a) and os.path.exists(b): + # samefile raises on non-existent paths; only call when both exist. + return not os.path.samefile(a, b) + except OSError: + # Fall through to "different" if samefile isn't applicable/available. + pass + + # If normalized strings differ and we couldn't prove they're the same entry, treat as different. + return True + + +def get_cuda_home_or_path() -> str | None: + cuda_home = os.environ.get("CUDA_HOME") + cuda_path = os.environ.get("CUDA_PATH") + + if cuda_home and cuda_path and _paths_differ(cuda_home, cuda_path): + warnings.warn( + "Both CUDA_HOME and CUDA_PATH are set but differ:\n" + f" CUDA_HOME={cuda_home}\n" + f" CUDA_PATH={cuda_path}\n" + "Using CUDA_HOME (higher priority).", + UserWarning, + stacklevel=2, + ) + + if cuda_home is not None: + return cuda_home + return cuda_path diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/find_site_packages_dll.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/find_site_packages_dll.py new file mode 100644 index 0000000000000000000000000000000000000000..507355727f98ece01d53398d0154113abe8061f5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/find_site_packages_dll.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import collections +import functools +import importlib.metadata + + +@functools.cache +def find_all_dll_files_via_metadata() -> dict[str, tuple[str, ...]]: + results: collections.defaultdict[str, list[str]] = collections.defaultdict(list) + + # sort dists for deterministic output + + for dist in sorted( + importlib.metadata.distributions(), + # `get` exists before 3.12, even though the hints only exist for Python >=3.12 + key=lambda d: (d.metadata.get("Name", ""), d.version), # type: ignore[attr-defined] + ): + files = dist.files + if not files: + continue + for relpath in sorted(files, key=lambda p: str(p)): # deterministic + relname = relpath.name.lower() + if not relname.endswith(".dll"): + continue + abs_path = str(dist.locate_file(relpath)) + results[relname].append(abs_path) + + # plain dicts; sort inner list for stability + return {k: tuple(sorted(v)) for k, v in results.items()} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/find_site_packages_so.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/find_site_packages_so.py new file mode 100644 index 0000000000000000000000000000000000000000..33ee1f1bcfb07cc22000b732090ecb535f5335a7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/find_site_packages_so.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import collections +import functools +import importlib.metadata +import re + +_SO_RE = re.compile(r"\.so(?:$|\.)") # matches libfoo.so or libfoo.so.1.2.3 + + +def split_so_version_suffix(so_filename: str) -> tuple[str, str]: + idx = so_filename.rfind(".so") + assert idx > 0, so_filename + idx += 3 + return (so_filename[:idx], so_filename[idx:]) + + +@functools.cache +def find_all_so_files_via_metadata() -> dict[str, dict[str, tuple[str, ...]]]: + results: collections.defaultdict[str, collections.defaultdict[str, list[str]]] = collections.defaultdict( + lambda: collections.defaultdict(list) + ) + + # sort dists for deterministic output + for dist in sorted( + importlib.metadata.distributions(), + # `get` exists before 3.12, even though the hints only exist for Python >=3.12 + key=lambda d: (d.metadata.get("Name", ""), d.version), # type: ignore[attr-defined] + ): + files = dist.files + if not files: + continue + for relpath in sorted(files, key=lambda p: str(p)): # deterministic + relname = relpath.name + if not _SO_RE.search(relname): + continue + so_basename, so_version_suffix = split_so_version_suffix(relname) + abs_path = str(dist.locate_file(relpath)) + results[so_basename][so_version_suffix].append(abs_path) + + # plain dicts; sort inner lists for stability + return {k: {kk: tuple(sorted(vv)) for kk, vv in v.items()} for k, v in results.items()} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/find_sub_dirs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/find_sub_dirs.py new file mode 100644 index 0000000000000000000000000000000000000000..a37f2dc8244969877f5af7989970f9754b1e2ecf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/find_sub_dirs.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import functools +import os +import site +import sys +from collections.abc import Sequence + + +def find_sub_dirs_no_cache(parent_dirs: Sequence[str], sub_dirs: Sequence[str]) -> list[str]: + results = [] + for base in parent_dirs: + stack = [(base, 0)] # (current_path, index into sub_dirs) + while stack: + current_path, idx = stack.pop() + if idx == len(sub_dirs): + if os.path.isdir(current_path): + results.append(current_path) + continue + + sub = sub_dirs[idx] + if sub == "*": + try: + entries = sorted(os.listdir(current_path)) + except OSError: + continue + for entry in entries: + entry_path = os.path.join(current_path, entry) + if os.path.isdir(entry_path): + stack.append((entry_path, idx + 1)) + else: + next_path = os.path.join(current_path, sub) + if os.path.isdir(next_path): + stack.append((next_path, idx + 1)) + return results + + +@functools.cache +def find_sub_dirs_cached(parent_dirs: Sequence[str], sub_dirs: Sequence[str]) -> list[str]: + return find_sub_dirs_no_cache(parent_dirs, sub_dirs) + + +def find_sub_dirs(parent_dirs: Sequence[str], sub_dirs: Sequence[str]) -> list[str]: + return find_sub_dirs_cached(tuple(parent_dirs), tuple(sub_dirs)) + + +def find_sub_dirs_sys_path(sub_dirs: Sequence[str]) -> list[str]: + return find_sub_dirs(sys.path, sub_dirs) + + +def find_sub_dirs_all_sitepackages(sub_dirs: Sequence[str]) -> list[str]: + parent_dirs = list(site.getsitepackages()) + if site.ENABLE_USER_SITE: + user_site = site.getusersitepackages() + if user_site: + parent_dirs.insert(0, user_site) + return find_sub_dirs(parent_dirs, sub_dirs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/platform_aware.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/platform_aware.py new file mode 100644 index 0000000000000000000000000000000000000000..72ecbc535932a7aa4252bdc6f4973326a8706cb6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/platform_aware.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import sys + +IS_WINDOWS = sys.platform == "win32" + + +def quote_for_shell(s: str) -> str: + if IS_WINDOWS: + # This is a relatively heavy import; keep pathfinder lean if possible. + from subprocess import list2cmdline + + return list2cmdline([s]) + else: + import shlex + + return shlex.quote(s) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/spawned_process_runner.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/spawned_process_runner.py new file mode 100644 index 0000000000000000000000000000000000000000..a5358d3d8d852ca16b5800a8e9cdeaa19634df13 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/cuda/pathfinder/_utils/spawned_process_runner.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import multiprocessing +import queue # for Empty +import sys +import traceback +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from io import StringIO +from typing import Any + +PROCESS_KILLED = -9 +PROCESS_NO_RESULT = -999 + + +# Similar to https://docs.python.org/3/library/subprocess.html#subprocess.CompletedProcess +# (args, check_returncode() are intentionally not supported here.) +@dataclass +class CompletedProcess: + returncode: int + stdout: str + stderr: str + + +class ChildProcessWrapper: + def __init__( + self, + result_queue: Any, + target: Callable[..., None], + args: Sequence[Any] | None, + kwargs: dict[str, Any] | None, + ) -> None: + self.target = target + self.args = () if args is None else args + self.kwargs = {} if kwargs is None else kwargs + self.result_queue = result_queue + + def __call__(self) -> None: + # Capture stdout/stderr + old_stdout = sys.stdout + old_stderr = sys.stderr + sys.stdout = StringIO() + sys.stderr = StringIO() + + try: + self.target(*self.args, **self.kwargs) + returncode = 0 + except SystemExit as e: # Handle sys.exit() + returncode = e.code if isinstance(e.code, int) else 0 + except BaseException: + traceback.print_exc() + returncode = 1 + finally: + # Collect outputs and restore streams + stdout = sys.stdout.getvalue() + stderr = sys.stderr.getvalue() + sys.stdout = old_stdout + sys.stderr = old_stderr + try: # noqa: SIM105 + self.result_queue.put((returncode, stdout, stderr)) + except Exception: # noqa: S110 + # If the queue is broken (e.g., parent gone), best effort logging + pass + + +def run_in_spawned_child_process( + target: Callable[..., None], + *, + args: Sequence[Any] | None = None, + kwargs: dict[str, Any] | None = None, + timeout: float | None = None, + rethrow: bool = False, +) -> CompletedProcess: + """Run `target` in a spawned child process, capturing stdout/stderr. + + The provided `target` must be defined at the top level of a module, and must + be importable in the spawned child process. Lambdas, closures, or interactively + defined functions (e.g., in Jupyter notebooks) will not work. + + If `rethrow=True` and the child process exits with a nonzero code, + raises ChildProcessError with the captured stderr. + """ + ctx = multiprocessing.get_context("spawn") + result_queue = ctx.Queue() + process = ctx.Process(target=ChildProcessWrapper(result_queue, target, args, kwargs)) + process.start() + + try: + process.join(timeout) + if process.is_alive(): + process.terminate() + process.join() + result = CompletedProcess( + returncode=PROCESS_KILLED, + stdout="", + stderr=f"Process timed out after {timeout} seconds and was terminated.", + ) + else: + try: + returncode, stdout, stderr = result_queue.get(timeout=1.0) + except (queue.Empty, EOFError): + result = CompletedProcess( + returncode=PROCESS_NO_RESULT, + stdout="", + stderr="Process exited or crashed before returning results.", + ) + else: + result = CompletedProcess( + returncode=returncode, + stdout=stdout, + stderr=stderr, + ) + + if rethrow and result.returncode != 0: + raise ChildProcessError( + f"Child process exited with code {result.returncode}.\n" + "--- stderr-from-child-process ---\n" + f"{result.stderr}" + "\n" + ) + + return result + + finally: + try: + result_queue.close() + result_queue.join_thread() + except Exception: # noqa: S110 + pass + if process.is_alive(): + process.kill() + process.join() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..12294c561897e6065f95ae5a53a54d70bf1651e9 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/arrow_reader.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/arrow_reader.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f482111b36edb9c2d9a0a72f8f8924a5a8fb9239 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/arrow_reader.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/arrow_writer.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/arrow_writer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8202de23fa55b054d6a9ccc7f4f9d4932614688c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/arrow_writer.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/combine.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/combine.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a35bd03fd4d15c0df6c121f046130766a7f187c5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/combine.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/config.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a6575f8e48be8af746182ec3ad9ad63177cbf831 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/config.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/data_files.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/data_files.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..facf9ed5a057a71642864c483661c80acd307431 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/data_files.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/distributed.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/distributed.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0dd7344a302bf50c1409411155f70b8be63dc4d4 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/distributed.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/exceptions.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57041ec89ba50b1d5337e9e27a9263edcb1c1d07 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/exceptions.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/fingerprint.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/fingerprint.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b5e086591e5130dfc91ad1d5649282287677093f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/fingerprint.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/hub.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/hub.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c8bb3fe7569d33d305500f6e88d0158999465e98 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/hub.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/info.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/info.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba523b0958184bc5dc0b1e1eabb2abdfe1372b2b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/info.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/inspect.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/inspect.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..826b491d65eea4888466ac9170dbd3014e287c24 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/inspect.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/keyhash.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/keyhash.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe4e0507e1694945d18c8ac66e47c9cde686d502 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/keyhash.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/metric.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/metric.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d46620456d5aa8e648f861b8498c8b00cd3b49f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/metric.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/naming.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/naming.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bdba72fca04242b53dc7950af862fe946df43bad Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/naming.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/search.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/search.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..33142bb33bdc46ec62c6f16d051cf2a8fe3affb8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/search.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/splits.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/splits.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ba9c74f7e12b28dd6762a0c9543e49e72813d731 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/splits.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/streaming.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/streaming.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e0f56e57312ef8f610d6082cc12b31272d447525 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/__pycache__/streaming.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..905e753955a348a8e486302e1b6f5e8f53ec7bf4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__init__.py @@ -0,0 +1,13 @@ +from abc import ABC, abstractmethod +from argparse import ArgumentParser + + +class BaseDatasetsCLICommand(ABC): + @staticmethod + @abstractmethod + def register_subcommand(parser: ArgumentParser): + raise NotImplementedError() + + @abstractmethod + def run(self): + raise NotImplementedError() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c939a0824cef16f1d07d641febb0d7d390894747 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/convert.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/convert.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ee7048160bf4a84effdf59acf206d06eebabcff3 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/convert.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/convert_to_parquet.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/convert_to_parquet.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5b95f9a0a2436afb32f5028eb0407b285573c240 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/convert_to_parquet.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e238d9b27fe73be7048819d66d47d0f5e6ebe497 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/datasets_cli.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4ec3004112dd49d98f094b61fdb0585638519484 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/delete_from_hub.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/dummy_data.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/dummy_data.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b0217b15670b999c3865f22abcd44316493d8fab Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/dummy_data.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/env.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/env.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..29452183c287194138597f0def273502d41e0ee0 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/env.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/run_beam.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/run_beam.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4beb1e44c9b52e9aa9cb7e3e1ffaea2c57e40073 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/run_beam.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/test.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/test.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7234f5c8d7156ed12b2e6df8e78a50478ecab9f7 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/__pycache__/test.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/convert.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/convert.py new file mode 100644 index 0000000000000000000000000000000000000000..f50d6aae5ba2e5c8b3c9766fa639c68ba87b2988 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/convert.py @@ -0,0 +1,195 @@ +import os +import re +import shutil +from argparse import ArgumentParser, Namespace + +from datasets.commands import BaseDatasetsCLICommand +from datasets.utils.logging import get_logger + + +HIGHLIGHT_MESSAGE_PRE = """<<<<<<< This should probably be modified because it mentions: """ + +HIGHLIGHT_MESSAGE_POST = """======= +>>>>>>> +""" + +TO_HIGHLIGHT = [ + "TextEncoderConfig", + "ByteTextEncoder", + "SubwordTextEncoder", + "encoder_config", + "maybe_build_from_corpus", + "manual_dir", +] + +TO_CONVERT = [ + # (pattern, replacement) + # Order is important here for some replacements + (r"tfds\.core", r"datasets"), + (r"tf\.io\.gfile\.GFile", r"open"), + (r"tf\.([\w\d]+)", r"datasets.Value('\1')"), + (r"tfds\.features\.Text\(\)", r"datasets.Value('string')"), + (r"tfds\.features\.Text\(", r"datasets.Value('string'),"), + (r"features\s*=\s*tfds.features.FeaturesDict\(", r"features=datasets.Features("), + (r"tfds\.features\.FeaturesDict\(", r"dict("), + (r"The TensorFlow Datasets Authors", r"The TensorFlow Datasets Authors and the HuggingFace Datasets Authors"), + (r"tfds\.", r"datasets."), + (r"dl_manager\.manual_dir", r"self.config.data_dir"), + (r"self\.builder_config", r"self.config"), +] + + +def convert_command_factory(args: Namespace): + """ + Factory function used to convert a model TF 1.0 checkpoint in a PyTorch checkpoint. + + Returns: ConvertCommand + """ + return ConvertCommand(args.tfds_path, args.datasets_directory) + + +class ConvertCommand(BaseDatasetsCLICommand): + @staticmethod + def register_subcommand(parser: ArgumentParser): + """ + Register this command to argparse so it's available for the datasets-cli + + Args: + parser: Root parser to register command-specific arguments + """ + train_parser = parser.add_parser( + "convert", + help="Convert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.", + ) + train_parser.add_argument( + "--tfds_path", + type=str, + required=True, + help="Path to a TensorFlow Datasets folder to convert or a single tfds file to convert.", + ) + train_parser.add_argument( + "--datasets_directory", type=str, required=True, help="Path to the HuggingFace Datasets folder." + ) + train_parser.set_defaults(func=convert_command_factory) + + def __init__(self, tfds_path: str, datasets_directory: str, *args): + self._logger = get_logger("datasets-cli/converting") + + self._tfds_path = tfds_path + self._datasets_directory = datasets_directory + + def run(self): + if os.path.isdir(self._tfds_path): + abs_tfds_path = os.path.abspath(self._tfds_path) + elif os.path.isfile(self._tfds_path): + abs_tfds_path = os.path.dirname(self._tfds_path) + else: + raise ValueError("--tfds_path is neither a directory nor a file. Please check path.") + + abs_datasets_path = os.path.abspath(self._datasets_directory) + + self._logger.info(f"Converting datasets from {abs_tfds_path} to {abs_datasets_path}") + + utils_files = [] + with_manual_update = [] + imports_to_builder_map = {} + + if os.path.isdir(self._tfds_path): + file_names = os.listdir(abs_tfds_path) + else: + file_names = [os.path.basename(self._tfds_path)] + + for f_name in file_names: + self._logger.info(f"Looking at file {f_name}") + input_file = os.path.join(abs_tfds_path, f_name) + output_file = os.path.join(abs_datasets_path, f_name) + + if not os.path.isfile(input_file) or "__init__" in f_name or "_test" in f_name or ".py" not in f_name: + self._logger.info("Skipping file") + continue + + with open(input_file, encoding="utf-8") as f: + lines = f.readlines() + + out_lines = [] + is_builder = False + needs_manual_update = False + tfds_imports = [] + for line in lines: + out_line = line + + # Convert imports + if "import tensorflow.compat.v2 as tf" in out_line: + continue + elif "@tfds.core" in out_line: + continue + elif "builder=self" in out_line: + continue + elif "import tensorflow_datasets.public_api as tfds" in out_line: + out_line = "import datasets\n" + elif "import tensorflow" in out_line: + # order is important here + out_line = "" + continue + elif "from absl import logging" in out_line: + out_line = "from datasets import logging\n" + elif "getLogger" in out_line: + out_line = out_line.replace("getLogger", "get_logger") + elif any(expression in out_line for expression in TO_HIGHLIGHT): + needs_manual_update = True + to_remove = list(filter(lambda e: e in out_line, TO_HIGHLIGHT)) + out_lines.append(HIGHLIGHT_MESSAGE_PRE + str(to_remove) + "\n") + out_lines.append(out_line) + out_lines.append(HIGHLIGHT_MESSAGE_POST) + continue + else: + for pattern, replacement in TO_CONVERT: + out_line = re.sub(pattern, replacement, out_line) + + # Take care of saving utilities (to later move them together with main script) + if "tensorflow_datasets" in out_line: + match = re.match(r"from\stensorflow_datasets.*import\s([^\.\r\n]+)", out_line) + tfds_imports.extend(imp.strip() for imp in match.group(1).split(",")) + out_line = "from . import " + match.group(1) + + # Check we have not forget anything + if "tf." in out_line or "tfds." in out_line or "tensorflow_datasets" in out_line: + raise ValueError(f"Error converting {out_line.strip()}") + + if "GeneratorBasedBuilder" in out_line or "BeamBasedBuilder" in out_line: + is_builder = True + out_lines.append(out_line) + + if is_builder or "wmt" in f_name: + # We create a new directory for each dataset + dir_name = f_name.replace(".py", "") + output_dir = os.path.join(abs_datasets_path, dir_name) + output_file = os.path.join(output_dir, f_name) + os.makedirs(output_dir, exist_ok=True) + self._logger.info(f"Adding directory {output_dir}") + imports_to_builder_map.update({imp: output_dir for imp in tfds_imports}) + else: + # Utilities will be moved at the end + utils_files.append(output_file) + + if needs_manual_update: + with_manual_update.append(output_file) + + with open(output_file, "w", encoding="utf-8") as f: + f.writelines(out_lines) + self._logger.info(f"Converted in {output_file}") + + for utils_file in utils_files: + try: + f_name = os.path.basename(utils_file) + dest_folder = imports_to_builder_map[f_name.replace(".py", "")] + self._logger.info(f"Moving {dest_folder} to {utils_file}") + shutil.copy(utils_file, dest_folder) + except KeyError: + self._logger.error(f"Cannot find destination folder for {utils_file}. Please copy manually.") + + if with_manual_update: + for file_path in with_manual_update: + self._logger.warning( + f"You need to manually update file {file_path} to remove configurations using 'TextEncoderConfig'." + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/convert_to_parquet.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/convert_to_parquet.py new file mode 100644 index 0000000000000000000000000000000000000000..43e82de802bf3687678ddee537edf662d7639bb7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/convert_to_parquet.py @@ -0,0 +1,46 @@ +from argparse import ArgumentParser +from typing import Optional + +from datasets.commands import BaseDatasetsCLICommand +from datasets.hub import convert_to_parquet + + +def _command_factory(args): + return ConvertToParquetCommand( + args.dataset_id, + args.token, + args.revision, + args.trust_remote_code, + ) + + +class ConvertToParquetCommand(BaseDatasetsCLICommand): + @staticmethod + def register_subcommand(parser): + parser: ArgumentParser = parser.add_parser("convert_to_parquet", help="Convert dataset to Parquet") + parser.add_argument( + "dataset_id", help="source dataset ID, e.g. USERNAME/DATASET_NAME or ORGANIZATION/DATASET_NAME" + ) + parser.add_argument("--token", help="access token to the Hugging Face Hub (defaults to logged-in user's one)") + parser.add_argument("--revision", help="source revision") + parser.add_argument( + "--trust_remote_code", action="store_true", help="whether to trust the code execution of the load script" + ) + parser.set_defaults(func=_command_factory) + + def __init__( + self, + dataset_id: str, + token: Optional[str], + revision: Optional[str], + trust_remote_code: bool, + ): + self._dataset_id = dataset_id + self._token = token + self._revision = revision + self._trust_remote_code = trust_remote_code + + def run(self) -> None: + _ = convert_to_parquet( + self._dataset_id, revision=self._revision, token=self._token, trust_remote_code=self._trust_remote_code + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/datasets_cli.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/datasets_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..eb76c11afdf818bfa344015b415d3ce44c300fb1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/datasets_cli.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +from argparse import ArgumentParser + +from datasets.commands.convert import ConvertCommand +from datasets.commands.convert_to_parquet import ConvertToParquetCommand +from datasets.commands.delete_from_hub import DeleteFromHubCommand +from datasets.commands.dummy_data import DummyDataCommand +from datasets.commands.env import EnvironmentCommand +from datasets.commands.run_beam import RunBeamCommand +from datasets.commands.test import TestCommand +from datasets.utils.logging import set_verbosity_info + + +def parse_unknown_args(unknown_args): + return {key.lstrip("-"): value for key, value in zip(unknown_args[::2], unknown_args[1::2])} + + +def main(): + parser = ArgumentParser( + "HuggingFace Datasets CLI tool", usage="datasets-cli []", allow_abbrev=False + ) + commands_parser = parser.add_subparsers(help="datasets-cli command helpers") + set_verbosity_info() + + # Register commands + ConvertCommand.register_subcommand(commands_parser) + EnvironmentCommand.register_subcommand(commands_parser) + TestCommand.register_subcommand(commands_parser) + RunBeamCommand.register_subcommand(commands_parser) + DummyDataCommand.register_subcommand(commands_parser) + ConvertToParquetCommand.register_subcommand(commands_parser) + DeleteFromHubCommand.register_subcommand(commands_parser) + + # Parse args + args, unknown_args = parser.parse_known_args() + if not hasattr(args, "func"): + parser.print_help() + exit(1) + kwargs = parse_unknown_args(unknown_args) + + # Run + service = args.func(args, **kwargs) + service.run() + + +if __name__ == "__main__": + main() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/delete_from_hub.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/delete_from_hub.py new file mode 100644 index 0000000000000000000000000000000000000000..126a029c246ab584df47c4ba1b908a798b9003b6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/delete_from_hub.py @@ -0,0 +1,42 @@ +from argparse import ArgumentParser +from typing import Optional + +from datasets.commands import BaseDatasetsCLICommand +from datasets.hub import delete_from_hub + + +def _command_factory(args): + return DeleteFromHubCommand( + args.dataset_id, + args.config_name, + args.token, + args.revision, + ) + + +class DeleteFromHubCommand(BaseDatasetsCLICommand): + @staticmethod + def register_subcommand(parser): + parser: ArgumentParser = parser.add_parser("delete_from_hub", help="Delete dataset config from the Hub") + parser.add_argument( + "dataset_id", help="source dataset ID, e.g. USERNAME/DATASET_NAME or ORGANIZATION/DATASET_NAME" + ) + parser.add_argument("config_name", help="config name to delete") + parser.add_argument("--token", help="access token to the Hugging Face Hub") + parser.add_argument("--revision", help="source revision") + parser.set_defaults(func=_command_factory) + + def __init__( + self, + dataset_id: str, + config_name: str, + token: Optional[str], + revision: Optional[str], + ): + self._dataset_id = dataset_id + self._config_name = config_name + self._token = token + self._revision = revision + + def run(self) -> None: + _ = delete_from_hub(self._dataset_id, self._config_name, revision=self._revision, token=self._token) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/dummy_data.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/dummy_data.py new file mode 100644 index 0000000000000000000000000000000000000000..c4321696e67258d80d40422a327dccb35859545d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/dummy_data.py @@ -0,0 +1,468 @@ +import fnmatch +import json +import os +import shutil +import tempfile +import xml.etree.ElementTree as ET +from argparse import ArgumentParser +from pathlib import Path +from typing import Optional + +from datasets import config +from datasets.commands import BaseDatasetsCLICommand +from datasets.download.download_config import DownloadConfig +from datasets.download.download_manager import DownloadManager +from datasets.download.mock_download_manager import MockDownloadManager +from datasets.load import dataset_module_factory, import_main_class +from datasets.utils.deprecation_utils import deprecated +from datasets.utils.logging import get_logger, set_verbosity_warning +from datasets.utils.py_utils import map_nested + + +logger = get_logger(__name__) + +DEFAULT_ENCODING = "utf-8" + + +def dummy_data_command_factory(args): + return DummyDataCommand( + args.path_to_dataset, + args.auto_generate, + args.n_lines, + args.json_field, + args.xml_tag, + args.match_text_files, + args.keep_uncompressed, + args.cache_dir, + args.encoding, + ) + + +class DummyDataGeneratorDownloadManager(DownloadManager): + def __init__(self, mock_download_manager, *args, **kwargs): + super().__init__(*args, **kwargs) + self.mock_download_manager = mock_download_manager + self.downloaded_dummy_paths = [] + self.expected_dummy_paths = [] + + def download(self, url_or_urls): + output = super().download(url_or_urls) + dummy_output = self.mock_download_manager.download(url_or_urls) + map_nested(self.downloaded_dummy_paths.append, output, map_tuple=True) + map_nested(self.expected_dummy_paths.append, dummy_output, map_tuple=True) + return output + + def download_and_extract(self, url_or_urls): + output = super().extract(super().download(url_or_urls)) + dummy_output = self.mock_download_manager.download(url_or_urls) + map_nested(self.downloaded_dummy_paths.append, output, map_tuple=True) + map_nested(self.expected_dummy_paths.append, dummy_output, map_tuple=True) + return output + + def auto_generate_dummy_data_folder( + self, + n_lines: int = 5, + json_field: Optional[str] = None, + xml_tag: Optional[str] = None, + match_text_files: Optional[str] = None, + encoding: Optional[str] = None, + ) -> bool: + os.makedirs( + os.path.join( + self.mock_download_manager.datasets_scripts_dir, + self.mock_download_manager.dataset_name, + self.mock_download_manager.dummy_data_folder, + "dummy_data", + ), + exist_ok=True, + ) + total = 0 + self.mock_download_manager.load_existing_dummy_data = False + for src_path, relative_dst_path in zip(self.downloaded_dummy_paths, self.expected_dummy_paths): + dst_path = os.path.join( + self.mock_download_manager.datasets_scripts_dir, + self.mock_download_manager.dataset_name, + self.mock_download_manager.dummy_data_folder, + relative_dst_path, + ) + total += self._create_dummy_data( + src_path, + dst_path, + n_lines=n_lines, + json_field=json_field, + xml_tag=xml_tag, + match_text_files=match_text_files, + encoding=encoding, + ) + if total == 0: + logger.error( + "Dummy data generation failed: no dummy files were created. " + "Make sure the data files format is supported by the auto-generation." + ) + return total > 0 + + def _create_dummy_data( + self, + src_path: str, + dst_path: str, + n_lines: int, + json_field: Optional[str] = None, + xml_tag: Optional[str] = None, + match_text_files: Optional[str] = None, + encoding: Optional[str] = None, + ) -> int: + encoding = encoding or DEFAULT_ENCODING + if os.path.isfile(src_path): + logger.debug(f"Trying to generate dummy data file {dst_path}") + dst_path_extensions = Path(dst_path).suffixes + line_by_line_extensions = [".txt", ".csv", ".jsonl", ".tsv"] + is_line_by_line_text_file = any(extension in dst_path_extensions for extension in line_by_line_extensions) + if match_text_files is not None: + file_name = os.path.basename(dst_path) + for pattern in match_text_files.split(","): + is_line_by_line_text_file |= fnmatch.fnmatch(file_name, pattern) + # Line by line text file (txt, csv etc.) + if is_line_by_line_text_file: + Path(dst_path).parent.mkdir(exist_ok=True, parents=True) + with open(src_path, encoding=encoding) as src_file: + with open(dst_path, "w", encoding=encoding) as dst_file: + first_lines = [] + for i, line in enumerate(src_file): + if i >= n_lines: + break + first_lines.append(line) + dst_file.write("".join(first_lines).strip()) + return 1 + # json file + elif ".json" in dst_path_extensions: + with open(src_path, encoding=encoding) as src_file: + json_data = json.load(src_file) + if json_field is not None: + json_data = json_data[json_field] + if isinstance(json_data, dict): + if not all(isinstance(v, list) for v in json_data.values()): + raise ValueError( + f"Couldn't parse columns {list(json_data.keys())}. " + "Maybe specify which json field must be used " + "to read the data with --json_field ." + ) + first_json_data = {k: v[:n_lines] for k, v in json_data.items()} + else: + first_json_data = json_data[:n_lines] + if json_field is not None: + first_json_data = {json_field: first_json_data} + Path(dst_path).parent.mkdir(exist_ok=True, parents=True) + with open(dst_path, "w", encoding=encoding) as dst_file: + json.dump(first_json_data, dst_file) + return 1 + # xml file + elif any(extension in dst_path_extensions for extension in [".xml", ".txm"]): + if xml_tag is None: + logger.warning("Found xml file but 'xml_tag' is set to None. Please provide --xml_tag") + else: + self._create_xml_dummy_data(src_path, dst_path, xml_tag, n_lines=n_lines, encoding=encoding) + return 1 + logger.warning( + f"Couldn't generate dummy file '{dst_path}'. " "Ignore that if this file is not useful for dummy data." + ) + return 0 + # directory, iterate through all files + elif os.path.isdir(src_path): + total = 0 + for path, _, files in os.walk(src_path): + for name in files: + if not name.startswith("."): # ignore files like .DS_Store etc. + src_file_path = os.path.join(path, name) + dst_file_path = os.path.join(dst_path, Path(src_file_path).relative_to(src_path)) + total += self._create_dummy_data( + src_file_path, + dst_file_path, + n_lines=n_lines, + json_field=json_field, + xml_tag=xml_tag, + match_text_files=match_text_files, + encoding=encoding, + ) + return total + + @staticmethod + def _create_xml_dummy_data(src_path, dst_path, xml_tag, n_lines=5, encoding=DEFAULT_ENCODING): + Path(dst_path).parent.mkdir(exist_ok=True, parents=True) + with open(src_path, encoding=encoding) as src_file: + n_line = 0 + parents = [] + for event, elem in ET.iterparse(src_file, events=("start", "end")): + if event == "start": + parents.append(elem) + else: + _ = parents.pop() + if elem.tag == xml_tag: + if n_line < n_lines: + n_line += 1 + else: + if parents: + parents[-1].remove(elem) + ET.ElementTree(element=elem).write(dst_path, encoding=encoding) + + def compress_autogenerated_dummy_data(self, path_to_dataset): + root_dir = os.path.join(path_to_dataset, self.mock_download_manager.dummy_data_folder) + base_name = os.path.join(root_dir, "dummy_data") + base_dir = "dummy_data" + logger.info(f"Compressing dummy data folder to '{base_name}.zip'") + shutil.make_archive(base_name, "zip", root_dir, base_dir) + shutil.rmtree(base_name) + + +@deprecated( + "The `datasets` repository does not host the dataset scripts anymore. Therefore, dummy data is no longer needed to test their loading with CI." +) +class DummyDataCommand(BaseDatasetsCLICommand): + @staticmethod + def register_subcommand(parser: ArgumentParser): + test_parser = parser.add_parser("dummy_data", help="Generate dummy data.") + test_parser.add_argument("--auto_generate", action="store_true", help="Automatically generate dummy data") + test_parser.add_argument( + "--n_lines", type=int, default=5, help="Number of lines or samples to keep when auto-generating dummy data" + ) + test_parser.add_argument( + "--json_field", + type=str, + default=None, + help="Optional, json field to read the data from when auto-generating dummy data. In the json data files, this field must point to a list of samples as json objects (ex: the 'data' field for squad-like files)", + ) + test_parser.add_argument( + "--xml_tag", + type=str, + default=None, + help="Optional, xml tag name of the samples inside the xml files when auto-generating dummy data.", + ) + test_parser.add_argument( + "--match_text_files", + type=str, + default=None, + help="Optional, a comma separated list of file patterns that looks for line-by-line text files other than *.txt or *.csv. Example: --match_text_files *.label", + ) + test_parser.add_argument( + "--keep_uncompressed", + action="store_true", + help="Whether to leave the dummy data folders uncompressed when auto-generating dummy data. Useful for debugging for to do manual adjustements before compressing.", + ) + test_parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="Cache directory to download and cache files when auto-generating dummy data", + ) + test_parser.add_argument( + "--encoding", + type=str, + default=None, + help=f"Encoding to use when auto-generating dummy data. Defaults to {DEFAULT_ENCODING}", + ) + test_parser.add_argument("path_to_dataset", type=str, help="Path to the dataset (example: ./datasets/squad)") + test_parser.set_defaults(func=dummy_data_command_factory) + + def __init__( + self, + path_to_dataset: str, + auto_generate: bool, + n_lines: int, + json_field: Optional[str], + xml_tag: Optional[str], + match_text_files: Optional[str], + keep_uncompressed: bool, + cache_dir: Optional[str], + encoding: Optional[str], + ): + self._path_to_dataset = path_to_dataset + if os.path.isdir(path_to_dataset): + self._dataset_name = path_to_dataset.replace(os.sep, "/").split("/")[-1] + else: + self._dataset_name = path_to_dataset.replace(os.sep, "/").split("/")[-2] + cache_dir = os.path.expanduser(cache_dir or config.HF_DATASETS_CACHE) + self._auto_generate = auto_generate + self._n_lines = n_lines + self._json_field = json_field + self._xml_tag = xml_tag + self._match_text_files = match_text_files + self._keep_uncompressed = keep_uncompressed + self._cache_dir = cache_dir + self._encoding = encoding + + def run(self): + set_verbosity_warning() + dataset_module = dataset_module_factory(self._path_to_dataset) + builder_cls = import_main_class(dataset_module.module_path) + + # use `None` as config if no configs + builder_configs = builder_cls.BUILDER_CONFIGS or [None] + auto_generate_results = [] + with tempfile.TemporaryDirectory() as tmp_dir: + for builder_config in builder_configs: + config_name = builder_config.name if builder_config else None + dataset_builder = builder_cls(config_name=config_name, hash=dataset_module.hash, cache_dir=tmp_dir) + version = builder_config.version if builder_config else dataset_builder.config.version + mock_dl_manager = MockDownloadManager( + dataset_name=self._dataset_name, + config=builder_config, + version=version, + use_local_dummy_data=True, + load_existing_dummy_data=False, + ) + + if self._auto_generate: + auto_generate_results.append( + self._autogenerate_dummy_data( + dataset_builder=dataset_builder, + mock_dl_manager=mock_dl_manager, + keep_uncompressed=self._keep_uncompressed, + ) + ) + else: + self._print_dummy_data_instructions( + dataset_builder=dataset_builder, mock_dl_manager=mock_dl_manager + ) + if self._auto_generate and not self._keep_uncompressed: + if all(auto_generate_results): + print(f"Automatic dummy data generation succeeded for all configs of '{self._path_to_dataset}'") + else: + print(f"Automatic dummy data generation failed for some configs of '{self._path_to_dataset}'") + + def _autogenerate_dummy_data(self, dataset_builder, mock_dl_manager, keep_uncompressed) -> Optional[bool]: + dl_cache_dir = ( + os.path.join(self._cache_dir, config.DOWNLOADED_DATASETS_DIR) + if self._cache_dir + else config.DOWNLOADED_DATASETS_PATH + ) + download_config = DownloadConfig(cache_dir=dl_cache_dir) + dl_manager = DummyDataGeneratorDownloadManager( + dataset_name=self._dataset_name, mock_download_manager=mock_dl_manager, download_config=download_config + ) + dataset_builder._split_generators(dl_manager) + mock_dl_manager.load_existing_dummy_data = False # don't use real dummy data + dl_manager.auto_generate_dummy_data_folder( + n_lines=self._n_lines, + json_field=self._json_field, + xml_tag=self._xml_tag, + match_text_files=self._match_text_files, + encoding=self._encoding, + ) + if not keep_uncompressed: + path_do_dataset = os.path.join(mock_dl_manager.datasets_scripts_dir, mock_dl_manager.dataset_name) + dl_manager.compress_autogenerated_dummy_data(path_do_dataset) + # now test that the dummy_data.zip file actually works + mock_dl_manager.load_existing_dummy_data = True # use real dummy data + n_examples_per_split = {} + os.makedirs(dataset_builder._cache_dir, exist_ok=True) + try: + split_generators = dataset_builder._split_generators(mock_dl_manager) + for split_generator in split_generators: + dataset_builder._prepare_split(split_generator, check_duplicate_keys=False) + n_examples_per_split[split_generator.name] = split_generator.split_info.num_examples + except OSError as e: + logger.error( + f"Failed to load dummy data for config '{dataset_builder.config.name}''.\nOriginal error:\n" + + str(e) + ) + return False + else: + if all(n_examples > 0 for n_examples in n_examples_per_split.values()): + logger.warning( + f"Dummy data generation done and dummy data test succeeded for config '{dataset_builder.config.name}''." + ) + return True + else: + empty_splits = [ + split_name for split_name in n_examples_per_split if n_examples_per_split[split_name] == 0 + ] + logger.warning( + f"Dummy data generation done but dummy data test failed since splits {empty_splits} have 0 examples for config '{dataset_builder.config.name}''." + ) + return False + else: + generated_dummy_data_dir = os.path.join(self._path_to_dataset, mock_dl_manager.dummy_data_folder) + logger.info( + f"Dummy data generated in directory '{generated_dummy_data_dir}' but kept uncompressed. " + "Please compress this directory into a zip file to use it for dummy data tests." + ) + + def _print_dummy_data_instructions(self, dataset_builder, mock_dl_manager): + dummy_data_folder = os.path.join(self._path_to_dataset, mock_dl_manager.dummy_data_folder) + logger.info(f"Creating dummy folder structure for {dummy_data_folder}... ") + os.makedirs(dummy_data_folder, exist_ok=True) + + try: + generator_splits = dataset_builder._split_generators(mock_dl_manager) + except FileNotFoundError as e: + print( + f"Dataset {self._dataset_name} with config {mock_dl_manager.config} seems to already open files in the method `_split_generators(...)`. You might consider to instead only open files in the method `_generate_examples(...)` instead. If this is not possible the dummy data has to be created with less guidance. Make sure you create the file {e.filename}." + ) + + files_to_create = set() + split_names = [] + dummy_file_name = mock_dl_manager.dummy_file_name + + for split in generator_splits: + logger.info(f"Collecting dummy data file paths to create for {split.name}") + split_names.append(split.name) + gen_kwargs = split.gen_kwargs + generator = dataset_builder._generate_examples(**gen_kwargs) + + try: + dummy_data_guidance_print = "\n" + 30 * "=" + "DUMMY DATA INSTRUCTIONS" + 30 * "=" + "\n" + config_string = ( + f"config {mock_dl_manager.config.name} of " if mock_dl_manager.config is not None else "" + ) + dummy_data_guidance_print += ( + "- In order to create the dummy data for " + + config_string + + f"{self._dataset_name}, please go into the folder '{dummy_data_folder}' with `cd {dummy_data_folder}` . \n\n" + ) + + # trigger generate function + for key, record in generator: + pass + + dummy_data_guidance_print += f"- It appears that the function `_generate_examples(...)` expects one or more files in the folder {dummy_file_name} using the function `glob.glob(...)`. In this case, please refer to the `_generate_examples(...)` method to see under which filename the dummy data files should be created. \n\n" + + except FileNotFoundError as e: + files_to_create.add(e.filename) + + split_names = ", ".join(split_names) + if len(files_to_create) > 0: + # no glob.glob(...) in `_generate_examples(...)` + if len(files_to_create) == 1 and next(iter(files_to_create)) == dummy_file_name: + dummy_data_guidance_print += f"- Please create a single dummy data file called '{next(iter(files_to_create))}' from the folder '{dummy_data_folder}'. Make sure that the dummy data file provides at least one example for the split(s) '{split_names}' \n\n" + files_string = dummy_file_name + else: + files_string = ", ".join(files_to_create) + dummy_data_guidance_print += f"- Please create the following dummy data files '{files_string}' from the folder '{dummy_data_folder}'\n\n" + + dummy_data_guidance_print += f"- For each of the splits '{split_names}', make sure that one or more of the dummy data files provide at least one example \n\n" + + dummy_data_guidance_print += f"- If the method `_generate_examples(...)` includes multiple `open()` statements, you might have to create other files in addition to '{files_string}'. In this case please refer to the `_generate_examples(...)` method \n\n" + + if len(files_to_create) == 1 and next(iter(files_to_create)) == dummy_file_name: + dummy_data_guidance_print += f"- After the dummy data file is created, it should be zipped to '{dummy_file_name}.zip' with the command `zip {dummy_file_name}.zip {dummy_file_name}` \n\n" + + dummy_data_guidance_print += ( + f"- You can now delete the file '{dummy_file_name}' with the command `rm {dummy_file_name}` \n\n" + ) + + dummy_data_guidance_print += f"- To get the file '{dummy_file_name}' back for further changes to the dummy data, simply unzip {dummy_file_name}.zip with the command `unzip {dummy_file_name}.zip` \n\n" + else: + dummy_data_guidance_print += f"- After all dummy data files are created, they should be zipped recursively to '{dummy_file_name}.zip' with the command `zip -r {dummy_file_name}.zip {dummy_file_name}/` \n\n" + + dummy_data_guidance_print += ( + f"- You can now delete the folder '{dummy_file_name}' with the command `rm -r {dummy_file_name}` \n\n" + ) + + dummy_data_guidance_print += f"- To get the folder '{dummy_file_name}' back for further changes to the dummy data, simply unzip {dummy_file_name}.zip with the command `unzip {dummy_file_name}.zip` \n\n" + + dummy_data_guidance_print += ( + f"- Make sure you have created the file '{dummy_file_name}.zip' in '{dummy_data_folder}' \n" + ) + + dummy_data_guidance_print += 83 * "=" + "\n" + + print(dummy_data_guidance_print) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/env.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/env.py new file mode 100644 index 0000000000000000000000000000000000000000..40b2a3654c8f1c6c080222a80b12d108972a5dc9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/env.py @@ -0,0 +1,41 @@ +import platform +from argparse import ArgumentParser + +import fsspec +import huggingface_hub +import pandas +import pyarrow + +from datasets import __version__ as version +from datasets.commands import BaseDatasetsCLICommand + + +def info_command_factory(_): + return EnvironmentCommand() + + +class EnvironmentCommand(BaseDatasetsCLICommand): + @staticmethod + def register_subcommand(parser: ArgumentParser): + download_parser = parser.add_parser("env", help="Print relevant system environment info.") + download_parser.set_defaults(func=info_command_factory) + + def run(self): + info = { + "`datasets` version": version, + "Platform": platform.platform(), + "Python version": platform.python_version(), + "`huggingface_hub` version": huggingface_hub.__version__, + "PyArrow version": pyarrow.__version__, + "Pandas version": pandas.__version__, + "`fsspec` version": fsspec.__version__, + } + + print("\nCopy-and-paste the text below in your GitHub issue.\n") + print(self.format_dict(info)) + + return info + + @staticmethod + def format_dict(d): + return "\n".join([f"- {prop}: {val}" for prop, val in d.items()]) + "\n" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/run_beam.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/run_beam.py new file mode 100644 index 0000000000000000000000000000000000000000..1d59e464344076315a2f51c608e04502189a3c2a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/run_beam.py @@ -0,0 +1,168 @@ +import os +from argparse import ArgumentParser +from pathlib import Path +from shutil import copyfile +from typing import List + +from datasets import config +from datasets.builder import DatasetBuilder +from datasets.commands import BaseDatasetsCLICommand +from datasets.download.download_config import DownloadConfig +from datasets.download.download_manager import DownloadMode +from datasets.load import dataset_module_factory, import_main_class +from datasets.utils.deprecation_utils import deprecated +from datasets.utils.info_utils import VerificationMode + + +def run_beam_command_factory(args, **kwargs): + return RunBeamCommand( + args.dataset, + args.name, + args.cache_dir, + args.beam_pipeline_options, + args.data_dir, + args.all_configs, + args.save_info or args.save_infos, + args.ignore_verifications, + args.force_redownload, + **kwargs, + ) + + +@deprecated( + "`BeamBasedBuilder` and `datasets-cli run_beam` are deprecated and will be removed in v3.0.0. Please use `GeneratorBasedBuilder` or `ArrowBasedBuilder` instead." +) +class RunBeamCommand(BaseDatasetsCLICommand): + @staticmethod + def register_subcommand(parser: ArgumentParser): + run_beam_parser = parser.add_parser("run_beam", help="Run a Beam dataset processing pipeline") + run_beam_parser.add_argument("dataset", type=str, help="Name of the dataset to download") + run_beam_parser.add_argument("--name", type=str, default=None, help="Dataset config name") + run_beam_parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="Cache directory where the datasets are stored", + ) + run_beam_parser.add_argument( + "--beam_pipeline_options", + type=str, + default="", + help="Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`", + ) + run_beam_parser.add_argument( + "--data_dir", + type=str, + default=None, + help="Can be used to specify a manual directory to get the files from", + ) + run_beam_parser.add_argument("--all_configs", action="store_true", help="Test all dataset configurations") + run_beam_parser.add_argument("--save_info", action="store_true", help="Save the dataset infos file") + run_beam_parser.add_argument( + "--ignore_verifications", action="store_true", help="Run the test without checksums and splits checks" + ) + run_beam_parser.add_argument("--force_redownload", action="store_true", help="Force dataset redownload") + # aliases + run_beam_parser.add_argument("--save_infos", action="store_true", help="alias for save_info") + run_beam_parser.set_defaults(func=run_beam_command_factory) + + def __init__( + self, + dataset: str, + name: str, + cache_dir: str, + beam_pipeline_options: str, + data_dir: str, + all_configs: bool, + save_infos: bool, + ignore_verifications: bool, + force_redownload: bool, + **config_kwargs, + ): + self._dataset = dataset + self._name = name + self._cache_dir = cache_dir + self._beam_pipeline_options = beam_pipeline_options + self._data_dir = data_dir + self._all_configs = all_configs + self._save_infos = save_infos + self._ignore_verifications = ignore_verifications + self._force_redownload = force_redownload + self._config_kwargs = config_kwargs + + def run(self): + import apache_beam as beam + + if self._name is not None and self._all_configs: + print("Both parameters `name` and `all_configs` can't be used at once.") + exit(1) + path, config_name = self._dataset, self._name + dataset_module = dataset_module_factory(path) + builder_cls = import_main_class(dataset_module.module_path) + builders: List[DatasetBuilder] = [] + if self._beam_pipeline_options: + beam_options = beam.options.pipeline_options.PipelineOptions( + flags=[f"--{opt.strip()}" for opt in self._beam_pipeline_options.split(",") if opt] + ) + else: + beam_options = None + if self._all_configs and len(builder_cls.BUILDER_CONFIGS) > 0: + for builder_config in builder_cls.BUILDER_CONFIGS: + builders.append( + builder_cls( + config_name=builder_config.name, + data_dir=self._data_dir, + hash=dataset_module.hash, + beam_options=beam_options, + cache_dir=self._cache_dir, + base_path=dataset_module.builder_kwargs.get("base_path"), + ) + ) + else: + builders.append( + builder_cls( + config_name=config_name, + data_dir=self._data_dir, + beam_options=beam_options, + cache_dir=self._cache_dir, + base_path=dataset_module.builder_kwargs.get("base_path"), + **self._config_kwargs, + ) + ) + + for builder in builders: + builder.download_and_prepare( + download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS + if not self._force_redownload + else DownloadMode.FORCE_REDOWNLOAD, + download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH), + verification_mode=VerificationMode.NO_CHECKS + if self._ignore_verifications + else VerificationMode.ALL_CHECKS, + ) + if self._save_infos: + builder._save_infos() + + print("Apache beam run successful.") + + # If save_infos=True, the dataset infos file is created next to the loaded module file. + # Let's move it to the original directory of the dataset script, to allow the user to + # upload them on S3 at the same time afterwards. + if self._save_infos: + dataset_infos_path = os.path.join(builder_cls.get_imported_module_dir(), config.DATASETDICT_INFOS_FILENAME) + + name = Path(path).name + ".py" + + combined_path = os.path.join(path, name) + if os.path.isfile(path): + dataset_dir = os.path.dirname(path) + elif os.path.isfile(combined_path): + dataset_dir = path + else: # in case of a remote dataset + print(f"Dataset Infos file saved at {dataset_infos_path}") + exit(1) + + # Move datasetinfo back to the user + user_dataset_infos_path = os.path.join(dataset_dir, config.DATASETDICT_INFOS_FILENAME) + copyfile(dataset_infos_path, user_dataset_infos_path) + print(f"Dataset Infos file saved at {user_dataset_infos_path}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/test.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/test.py new file mode 100644 index 0000000000000000000000000000000000000000..986bf490a1d80225a0b94431dd7bdb2912199bb1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/commands/test.py @@ -0,0 +1,207 @@ +import logging +import os +from argparse import ArgumentParser +from pathlib import Path +from shutil import copyfile, rmtree +from typing import Generator, Optional + +import datasets.config +from datasets.builder import DatasetBuilder +from datasets.commands import BaseDatasetsCLICommand +from datasets.download.download_manager import DownloadMode +from datasets.load import dataset_module_factory, import_main_class +from datasets.utils.info_utils import VerificationMode +from datasets.utils.logging import ERROR, get_logger + + +logger = get_logger(__name__) + + +def _test_command_factory(args): + return TestCommand( + args.dataset, + args.name, + args.cache_dir, + args.data_dir, + args.all_configs, + args.save_info or args.save_infos, + args.ignore_verifications, + args.force_redownload, + args.clear_cache, + args.num_proc, + args.trust_remote_code, + ) + + +class TestCommand(BaseDatasetsCLICommand): + __test__ = False # to tell pytest it's not a test class + + @staticmethod + def register_subcommand(parser: ArgumentParser): + test_parser = parser.add_parser("test", help="Test dataset implementation.") + test_parser.add_argument("--name", type=str, default=None, help="Dataset processing name") + test_parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="Cache directory where the datasets are stored.", + ) + test_parser.add_argument( + "--data_dir", + type=str, + default=None, + help="Can be used to specify a manual directory to get the files from.", + ) + test_parser.add_argument("--all_configs", action="store_true", help="Test all dataset configurations") + test_parser.add_argument( + "--save_info", action="store_true", help="Save the dataset infos in the dataset card (README.md)" + ) + test_parser.add_argument( + "--ignore_verifications", + action="store_true", + help="Run the test without checksums and splits checks.", + ) + test_parser.add_argument("--force_redownload", action="store_true", help="Force dataset redownload") + test_parser.add_argument( + "--clear_cache", + action="store_true", + help="Remove downloaded files and cached datasets after each config test", + ) + test_parser.add_argument("--num_proc", type=int, default=None, help="Number of processes") + test_parser.add_argument( + "--trust_remote_code", action="store_true", help="whether to trust the code execution of the load script" + ) + # aliases + test_parser.add_argument("--save_infos", action="store_true", help="alias to save_info") + test_parser.add_argument("dataset", type=str, help="Name of the dataset to download") + test_parser.set_defaults(func=_test_command_factory) + + def __init__( + self, + dataset: str, + name: str, + cache_dir: str, + data_dir: str, + all_configs: bool, + save_infos: bool, + ignore_verifications: bool, + force_redownload: bool, + clear_cache: bool, + num_proc: int, + trust_remote_code: Optional[bool], + ): + self._dataset = dataset + self._name = name + self._cache_dir = cache_dir + self._data_dir = data_dir + self._all_configs = all_configs + self._save_infos = save_infos + self._ignore_verifications = ignore_verifications + self._force_redownload = force_redownload + self._clear_cache = clear_cache + self._num_proc = num_proc + self._trust_remote_code = trust_remote_code + if clear_cache and not cache_dir: + print( + "When --clear_cache is used, specifying a cache directory is mandatory.\n" + "The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.\n" + "Please provide a --cache_dir that will be used to test the dataset script." + ) + exit(1) + if save_infos: + self._ignore_verifications = True + + def run(self): + logging.getLogger("filelock").setLevel(ERROR) + if self._name is not None and self._all_configs: + print("Both parameters `config` and `all_configs` can't be used at once.") + exit(1) + path, config_name = self._dataset, self._name + module = dataset_module_factory(path, trust_remote_code=self._trust_remote_code) + builder_cls = import_main_class(module.module_path) + n_builders = len(builder_cls.BUILDER_CONFIGS) if self._all_configs and builder_cls.BUILDER_CONFIGS else 1 + + def get_builders() -> Generator[DatasetBuilder, None, None]: + if self._all_configs and builder_cls.BUILDER_CONFIGS: + for i, config in enumerate(builder_cls.BUILDER_CONFIGS): + if "config_name" in module.builder_kwargs: + yield builder_cls( + cache_dir=self._cache_dir, + data_dir=self._data_dir, + **module.builder_kwargs, + ) + else: + yield builder_cls( + config_name=config.name, + cache_dir=self._cache_dir, + data_dir=self._data_dir, + **module.builder_kwargs, + ) + else: + if "config_name" in module.builder_kwargs: + yield builder_cls(cache_dir=self._cache_dir, data_dir=self._data_dir, **module.builder_kwargs) + else: + yield builder_cls( + config_name=config_name, + cache_dir=self._cache_dir, + data_dir=self._data_dir, + **module.builder_kwargs, + ) + + for j, builder in enumerate(get_builders()): + print(f"Testing builder '{builder.config.name}' ({j + 1}/{n_builders})") + builder._record_infos = os.path.exists( + os.path.join(builder.get_imported_module_dir(), datasets.config.DATASETDICT_INFOS_FILENAME) + ) # record checksums only if we need to update a (deprecated) dataset_infos.json + builder.download_and_prepare( + download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS + if not self._force_redownload + else DownloadMode.FORCE_REDOWNLOAD, + verification_mode=VerificationMode.NO_CHECKS + if self._ignore_verifications + else VerificationMode.ALL_CHECKS, + try_from_hf_gcs=False, + num_proc=self._num_proc, + ) + builder.as_dataset() + if self._save_infos: + builder._save_infos() + + # If save_infos=True, the dataset card (README.md) is created next to the loaded module file. + # The dataset_infos are saved in the YAML part of the README.md + + # Let's move it to the original directory of the dataset script, to allow the user to + # upload them on S3 at the same time afterwards. + if self._save_infos: + dataset_readme_path = os.path.join( + builder_cls.get_imported_module_dir(), datasets.config.REPOCARD_FILENAME + ) + name = Path(path).name + ".py" + combined_path = os.path.join(path, name) + if os.path.isfile(path): + dataset_dir = os.path.dirname(path) + elif os.path.isfile(combined_path): + dataset_dir = path + elif os.path.isdir(path): # for local directories containing only data files + dataset_dir = path + else: # in case of a remote dataset + dataset_dir = None + print(f"Dataset card saved at {dataset_readme_path}") + + # Move dataset_info back to the user + if dataset_dir is not None: + user_dataset_readme_path = os.path.join(dataset_dir, datasets.config.REPOCARD_FILENAME) + copyfile(dataset_readme_path, user_dataset_readme_path) + print(f"Dataset card saved at {user_dataset_readme_path}") + + # If clear_cache=True, the download folder and the dataset builder cache directory are deleted + if self._clear_cache: + if os.path.isdir(builder._cache_dir): + logger.warning(f"Clearing cache at {builder._cache_dir}") + rmtree(builder._cache_dir) + download_dir = os.path.join(self._cache_dir, datasets.config.DOWNLOADED_DATASETS_DIR) + if os.path.isdir(download_dir): + logger.warning(f"Clearing cache at {download_dir}") + rmtree(download_dir) + + print("Test successful.") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8ae0d436504dc2e609b0ca8851509c72a161dbde --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__init__.py @@ -0,0 +1,10 @@ +__all__ = [ + "DownloadConfig", + "DownloadManager", + "DownloadMode", + "StreamingDownloadManager", +] + +from .download_config import DownloadConfig +from .download_manager import DownloadManager, DownloadMode +from .streaming_download_manager import StreamingDownloadManager diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d13ae78b5ca18818049df0750634cfc838362f38 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/download_config.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/download_config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..150e67d41c41561a5d0bc59f4a2326144c201cdc Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/download_config.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/download_manager.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/download_manager.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1c30096bc11ebf985858e86c0baea175c34a2eea Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/download_manager.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/mock_download_manager.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/mock_download_manager.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9cbe6d233019c027020e5ddb8abfdb3471b05fc6 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/mock_download_manager.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..32e016f9adfe3a69b9b0a57b61733693abe6d6d6 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/__pycache__/streaming_download_manager.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/download_config.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/download_config.py new file mode 100644 index 0000000000000000000000000000000000000000..c1fdf9be6a2379a9f3a2d0933aebe3c45689f337 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/download_config.py @@ -0,0 +1,108 @@ +import copy +import warnings +from dataclasses import InitVar, dataclass, field +from pathlib import Path +from typing import Any, Dict, Optional, Union + +from .. import config + + +@dataclass +class DownloadConfig: + """Configuration for our cached path manager. + + Attributes: + cache_dir (`str` or `Path`, *optional*): + Specify a cache directory to save the file to (overwrite the + default cache dir). + force_download (`bool`, defaults to `False`): + If `True`, re-dowload the file even if it's already cached in + the cache dir. + resume_download (`bool`, defaults to `False`): + If `True`, resume the download if an incompletely received file is + found. + proxies (`dict`, *optional*): + user_agent (`str`, *optional*): + Optional string or dict that will be appended to the user-agent on remote + requests. + extract_compressed_file (`bool`, defaults to `False`): + If `True` and the path point to a zip or tar file, + extract the compressed file in a folder along the archive. + force_extract (`bool`, defaults to `False`): + If `True` when `extract_compressed_file` is `True` and the archive + was already extracted, re-extract the archive and override the folder where it was extracted. + delete_extracted (`bool`, defaults to `False`): + Whether to delete (or keep) the extracted files. + extract_on_the_fly (`bool`, defaults to `False`): + If `True`, extract compressed files while they are being read. + use_etag (`bool`, defaults to `True`): + Whether to use the ETag HTTP response header to validate the cached files. + num_proc (`int`, *optional*): + The number of processes to launch to download the files in parallel. + max_retries (`int`, default to `1`): + The number of times to retry an HTTP request if it fails. + token (`str` or `bool`, *optional*): + Optional string or boolean to use as Bearer token + for remote files on the Datasets Hub. If `True`, or not specified, will get token from `~/.huggingface`. + use_auth_token (`str` or `bool`, *optional*): + Optional string or boolean to use as Bearer token + for remote files on the Datasets Hub. If `True`, or not specified, will get token from `~/.huggingface`. + + + + `use_auth_token` was deprecated in favor of `token` in version 2.14.0 and will be removed in 3.0.0. + + + + ignore_url_params (`bool`, defaults to `False`): + Whether to strip all query parameters and fragments from + the download URL before using it for caching the file. + storage_options (`dict`, *optional*): + Key/value pairs to be passed on to the dataset file-system backend, if any. + download_desc (`str`, *optional*): + A description to be displayed alongside with the progress bar while downloading the files. + disable_tqdm (`bool`, defaults to `False`): + Whether to disable the individual files download progress bar + """ + + cache_dir: Optional[Union[str, Path]] = None + force_download: bool = False + resume_download: bool = False + local_files_only: bool = False + proxies: Optional[Dict] = None + user_agent: Optional[str] = None + extract_compressed_file: bool = False + force_extract: bool = False + delete_extracted: bool = False + extract_on_the_fly: bool = False + use_etag: bool = True + num_proc: Optional[int] = None + max_retries: int = 1 + token: Optional[Union[str, bool]] = None + use_auth_token: InitVar[Optional[Union[str, bool]]] = "deprecated" + ignore_url_params: bool = False + storage_options: Dict[str, Any] = field(default_factory=dict) + download_desc: Optional[str] = None + disable_tqdm: bool = False + + def __post_init__(self, use_auth_token): + if use_auth_token != "deprecated": + warnings.warn( + "'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0.\n" + f"You can remove this warning by passing 'token={use_auth_token}' instead.", + FutureWarning, + ) + self.token = use_auth_token + if "hf" not in self.storage_options: + self.storage_options["hf"] = {"token": self.token, "endpoint": config.HF_ENDPOINT} + + def copy(self) -> "DownloadConfig": + return self.__class__(**{k: copy.deepcopy(v) for k, v in self.__dict__.items()}) + + def __setattr__(self, name, value): + if name == "token" and getattr(self, "storage_options", None) is not None: + if "hf" not in self.storage_options: + self.storage_options["hf"] = {"token": value, "endpoint": config.HF_ENDPOINT} + elif getattr(self.storage_options["hf"], "token", None) is None: + self.storage_options["hf"]["token"] = value + super().__setattr__(name, value) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/download_manager.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/download_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..b6c45d94c15209aefdd5bd7ef300b30c1e0f1f56 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/download_manager.py @@ -0,0 +1,448 @@ +# Copyright 2020 The TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Download manager interface.""" + +import enum +import io +import multiprocessing +import os +import posixpath +import warnings +from datetime import datetime +from functools import partial +from typing import Dict, List, Optional, Union + +import fsspec +from fsspec.core import url_to_fs +from tqdm.contrib.concurrent import thread_map + +from .. import config +from ..utils import tqdm as hf_tqdm +from ..utils.deprecation_utils import DeprecatedEnum, deprecated +from ..utils.file_utils import ( + ArchiveIterable, + FilesIterable, + cached_path, + get_from_cache, + hash_url_to_filename, + is_relative_path, + stack_multiprocessing_download_progress_bars, + url_or_path_join, +) +from ..utils.info_utils import get_size_checksum_dict +from ..utils.logging import get_logger, tqdm +from ..utils.py_utils import NestedDataStructure, map_nested, size_str +from ..utils.track import tracked_str +from .download_config import DownloadConfig + + +logger = get_logger(__name__) + + +class DownloadMode(enum.Enum): + """`Enum` for how to treat pre-existing downloads and data. + + The default mode is `REUSE_DATASET_IF_EXISTS`, which will reuse both + raw downloads and the prepared dataset if they exist. + + The generations modes: + + | | Downloads | Dataset | + |-------------------------------------|-----------|---------| + | `REUSE_DATASET_IF_EXISTS` (default) | Reuse | Reuse | + | `REUSE_CACHE_IF_EXISTS` | Reuse | Fresh | + | `FORCE_REDOWNLOAD` | Fresh | Fresh | + + """ + + REUSE_DATASET_IF_EXISTS = "reuse_dataset_if_exists" + REUSE_CACHE_IF_EXISTS = "reuse_cache_if_exists" + FORCE_REDOWNLOAD = "force_redownload" + + +class GenerateMode(DeprecatedEnum): + REUSE_DATASET_IF_EXISTS = "reuse_dataset_if_exists" + REUSE_CACHE_IF_EXISTS = "reuse_cache_if_exists" + FORCE_REDOWNLOAD = "force_redownload" + + @property + def help_message(self): + return "Use 'DownloadMode' instead." + + +class DownloadManager: + is_streaming = False + + def __init__( + self, + dataset_name: Optional[str] = None, + data_dir: Optional[str] = None, + download_config: Optional[DownloadConfig] = None, + base_path: Optional[str] = None, + record_checksums=True, + ): + """Download manager constructor. + + Args: + data_dir: + can be used to specify a manual directory to get the files from. + dataset_name (`str`): + name of dataset this instance will be used for. If + provided, downloads will contain which datasets they were used for. + download_config (`DownloadConfig`): + to specify the cache directory and other + download options + base_path (`str`): + base path that is used when relative paths are used to + download files. This can be a remote url. + record_checksums (`bool`, defaults to `True`): + Whether to record the checksums of the downloaded files. If None, the value is inferred from the builder. + """ + self._dataset_name = dataset_name + self._data_dir = data_dir + self._base_path = base_path or os.path.abspath(".") + # To record what is being used: {url: {num_bytes: int, checksum: str}} + self._recorded_sizes_checksums: Dict[str, Dict[str, Optional[Union[int, str]]]] = {} + self.record_checksums = record_checksums + self.download_config = download_config or DownloadConfig() + self.downloaded_paths = {} + self.extracted_paths = {} + + @property + def manual_dir(self): + return self._data_dir + + @property + def downloaded_size(self): + """Returns the total size of downloaded files.""" + return sum(checksums_dict["num_bytes"] for checksums_dict in self._recorded_sizes_checksums.values()) + + @staticmethod + def ship_files_with_pipeline(downloaded_path_or_paths, pipeline): + """Ship the files using Beam FileSystems to the pipeline temp dir. + + Args: + downloaded_path_or_paths (`str` or `list[str]` or `dict[str, str]`): + Nested structure containing the + downloaded path(s). + pipeline ([`utils.beam_utils.BeamPipeline`]): + Apache Beam Pipeline. + + Returns: + `str` or `list[str]` or `dict[str, str]` + """ + from ..utils.beam_utils import upload_local_to_remote + + remote_dir = pipeline._options.get_all_options().get("temp_location") + if remote_dir is None: + raise ValueError("You need to specify 'temp_location' in PipelineOptions to upload files") + + def upload(local_file_path): + remote_file_path = posixpath.join( + remote_dir, config.DOWNLOADED_DATASETS_DIR, os.path.basename(local_file_path) + ) + logger.info( + f"Uploading {local_file_path} ({size_str(os.path.getsize(local_file_path))}) to {remote_file_path}." + ) + upload_local_to_remote(local_file_path, remote_file_path) + return remote_file_path + + uploaded_path_or_paths = map_nested( + lambda local_file_path: upload(local_file_path), + downloaded_path_or_paths, + ) + return uploaded_path_or_paths + + def _record_sizes_checksums(self, url_or_urls: NestedDataStructure, downloaded_path_or_paths: NestedDataStructure): + """Record size/checksum of downloaded files.""" + delay = 5 + for url, path in hf_tqdm( + list(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten())), + delay=delay, + desc="Computing checksums", + ): + # call str to support PathLike objects + self._recorded_sizes_checksums[str(url)] = get_size_checksum_dict( + path, record_checksum=self.record_checksums + ) + + @deprecated("Use `.download`/`.download_and_extract` with `fsspec` URLs instead.") + def download_custom(self, url_or_urls, custom_download): + """ + Download given urls(s) by calling `custom_download`. + + Args: + url_or_urls (`str` or `list` or `dict`): + URL or `list` or `dict` of URLs to download and extract. Each URL is a `str`. + custom_download (`Callable[src_url, dst_path]`): + The source URL and destination path. For example + `tf.io.gfile.copy`, that lets you download from Google storage. + + Returns: + downloaded_path(s): `str`, The downloaded paths matching the given input + `url_or_urls`. + + Example: + + ```py + >>> downloaded_files = dl_manager.download_custom('s3://my-bucket/data.zip', custom_download_for_my_private_bucket) + ``` + """ + cache_dir = self.download_config.cache_dir or config.DOWNLOADED_DATASETS_PATH + max_retries = self.download_config.max_retries + + def url_to_downloaded_path(url): + return os.path.join(cache_dir, hash_url_to_filename(url)) + + downloaded_path_or_paths = map_nested(url_to_downloaded_path, url_or_urls) + url_or_urls = NestedDataStructure(url_or_urls) + downloaded_path_or_paths = NestedDataStructure(downloaded_path_or_paths) + for url, path in zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten()): + try: + get_from_cache( + url, cache_dir=cache_dir, local_files_only=True, use_etag=False, max_retries=max_retries + ) + cached = True + except FileNotFoundError: + cached = False + if not cached or self.download_config.force_download: + custom_download(url, path) + get_from_cache( + url, cache_dir=cache_dir, local_files_only=True, use_etag=False, max_retries=max_retries + ) + self._record_sizes_checksums(url_or_urls, downloaded_path_or_paths) + return downloaded_path_or_paths.data + + def download(self, url_or_urls): + """Download given URL(s). + + By default, only one process is used for download. Pass customized `download_config.num_proc` to change this behavior. + + Args: + url_or_urls (`str` or `list` or `dict`): + URL or `list` or `dict` of URLs to download. Each URL is a `str`. + + Returns: + `str` or `list` or `dict`: + The downloaded paths matching the given input `url_or_urls`. + + Example: + + ```py + >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + ``` + """ + download_config = self.download_config.copy() + download_config.extract_compressed_file = False + if download_config.download_desc is None: + download_config.download_desc = "Downloading data" + + download_func = partial(self._download_batched, download_config=download_config) + + start_time = datetime.now() + with stack_multiprocessing_download_progress_bars(): + downloaded_path_or_paths = map_nested( + download_func, + url_or_urls, + map_tuple=True, + num_proc=download_config.num_proc, + desc="Downloading data files", + batched=True, + batch_size=-1, + ) + duration = datetime.now() - start_time + logger.info(f"Downloading took {duration.total_seconds() // 60} min") + url_or_urls = NestedDataStructure(url_or_urls) + downloaded_path_or_paths = NestedDataStructure(downloaded_path_or_paths) + self.downloaded_paths.update(dict(zip(url_or_urls.flatten(), downloaded_path_or_paths.flatten()))) + + start_time = datetime.now() + self._record_sizes_checksums(url_or_urls, downloaded_path_or_paths) + duration = datetime.now() - start_time + logger.info(f"Checksum Computation took {duration.total_seconds() // 60} min") + + return downloaded_path_or_paths.data + + def _download_batched( + self, + url_or_filenames: List[str], + download_config: DownloadConfig, + ) -> List[str]: + if len(url_or_filenames) >= 16: + download_config = download_config.copy() + download_config.disable_tqdm = True + download_func = partial(self._download_single, download_config=download_config) + + fs: fsspec.AbstractFileSystem + fs, path = url_to_fs(url_or_filenames[0], **download_config.storage_options) + size = 0 + try: + size = fs.info(path).get("size", 0) + except Exception: + pass + max_workers = ( + config.HF_DATASETS_MULTITHREADING_MAX_WORKERS if size < (20 << 20) else 1 + ) # enable multithreading if files are small + + return thread_map( + download_func, + url_or_filenames, + desc=download_config.download_desc or "Downloading", + unit="files", + position=multiprocessing.current_process()._identity[-1] # contains the ranks of subprocesses + if os.environ.get("HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS") == "1" + and multiprocessing.current_process()._identity + else None, + max_workers=max_workers, + tqdm_class=tqdm, + ) + else: + return [ + self._download_single(url_or_filename, download_config=download_config) + for url_or_filename in url_or_filenames + ] + + def _download_single(self, url_or_filename: str, download_config: DownloadConfig) -> str: + url_or_filename = str(url_or_filename) + if is_relative_path(url_or_filename): + # append the relative path to the base_path + url_or_filename = url_or_path_join(self._base_path, url_or_filename) + out = cached_path(url_or_filename, download_config=download_config) + out = tracked_str(out) + out.set_origin(url_or_filename) + return out + + def iter_archive(self, path_or_buf: Union[str, io.BufferedReader]): + """Iterate over files within an archive. + + Args: + path_or_buf (`str` or `io.BufferedReader`): + Archive path or archive binary file object. + + Yields: + `tuple[str, io.BufferedReader]`: + 2-tuple (path_within_archive, file_object). + File object is opened in binary mode. + + Example: + + ```py + >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + >>> files = dl_manager.iter_archive(archive) + ``` + """ + + if hasattr(path_or_buf, "read"): + return ArchiveIterable.from_buf(path_or_buf) + else: + return ArchiveIterable.from_urlpath(path_or_buf) + + def iter_files(self, paths: Union[str, List[str]]): + """Iterate over file paths. + + Args: + paths (`str` or `list` of `str`): + Root paths. + + Yields: + `str`: File path. + + Example: + + ```py + >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip') + >>> files = dl_manager.iter_files(files) + ``` + """ + return FilesIterable.from_urlpaths(paths) + + def extract(self, path_or_paths, num_proc="deprecated"): + """Extract given path(s). + + Args: + path_or_paths (path or `list` or `dict`): + Path of file to extract. Each path is a `str`. + num_proc (`int`): + Use multi-processing if `num_proc` > 1 and the length of + `path_or_paths` is larger than `num_proc`. + + + + Pass `DownloadConfig(num_proc=)` to the initializer instead. + + + + Returns: + extracted_path(s): `str`, The extracted paths matching the given input + path_or_paths. + + Example: + + ```py + >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + >>> extracted_files = dl_manager.extract(downloaded_files) + ``` + """ + if num_proc != "deprecated": + warnings.warn( + "'num_proc' was deprecated in version 2.6.2 and will be removed in 3.0.0. Pass `DownloadConfig(num_proc=)` to the initializer instead.", + FutureWarning, + ) + download_config = self.download_config.copy() + download_config.extract_compressed_file = True + extract_func = partial(self._download_single, download_config=download_config) + extracted_paths = map_nested( + extract_func, + path_or_paths, + num_proc=download_config.num_proc, + desc="Extracting data files", + ) + path_or_paths = NestedDataStructure(path_or_paths) + extracted_paths = NestedDataStructure(extracted_paths) + self.extracted_paths.update(dict(zip(path_or_paths.flatten(), extracted_paths.flatten()))) + return extracted_paths.data + + def download_and_extract(self, url_or_urls): + """Download and extract given `url_or_urls`. + + Is roughly equivalent to: + + ``` + extracted_paths = dl_manager.extract(dl_manager.download(url_or_urls)) + ``` + + Args: + url_or_urls (`str` or `list` or `dict`): + URL or `list` or `dict` of URLs to download and extract. Each URL is a `str`. + + Returns: + extracted_path(s): `str`, extracted paths of given URL(s). + """ + return self.extract(self.download(url_or_urls)) + + def get_recorded_sizes_checksums(self): + return self._recorded_sizes_checksums.copy() + + def delete_extracted_files(self): + paths_to_delete = set(self.extracted_paths.values()) - set(self.downloaded_paths.values()) + for key, path in list(self.extracted_paths.items()): + if path in paths_to_delete and os.path.isfile(path): + os.remove(path) + del self.extracted_paths[key] + + def manage_extracted_files(self): + if self.download_config.delete_extracted: + self.delete_extracted_files() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/mock_download_manager.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/mock_download_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..7c71103a536b2a725ebb1d1dfe239b80baedc740 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/mock_download_manager.py @@ -0,0 +1,244 @@ +# Copyright 2020 The TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Mock download manager interface.""" + +import os +import re +import urllib.parse +from pathlib import Path +from typing import Callable, List, Optional, Union +from zipfile import ZipFile + +from ..utils.file_utils import cached_path, hf_github_url +from ..utils.logging import get_logger +from ..utils.version import Version + + +logger = get_logger(__name__) + + +class MockDownloadManager: + dummy_file_name = "dummy_data" + datasets_scripts_dir = "datasets" + is_streaming = False + + def __init__( + self, + dataset_name: str, + config: str, + version: Union[Version, str], + cache_dir: Optional[str] = None, + use_local_dummy_data: bool = False, + load_existing_dummy_data: bool = True, + download_callbacks: Optional[List[Callable]] = None, + ): + self.downloaded_size = 0 + self.dataset_name = dataset_name + self.cache_dir = cache_dir + self.use_local_dummy_data = use_local_dummy_data + self.config = config + # download_callbacks take a single url as input + self.download_callbacks: List[Callable] = download_callbacks or [] + # if False, it doesn't load existing files and it returns the paths of the dummy files relative + # to the dummy_data zip file root + self.load_existing_dummy_data = load_existing_dummy_data + + # TODO(PVP, QL) might need to make this more general + self.version_name = str(version) + # to be downloaded + self._dummy_file = None + self._bucket_url = None + + @property + def dummy_file(self): + if self._dummy_file is None: + self._dummy_file = self.download_dummy_data() + return self._dummy_file + + @property + def dummy_data_folder(self): + if self.config is not None: + # structure is dummy / config_name / version_name + return os.path.join("dummy", self.config.name, self.version_name) + # structure is dummy / version_name + return os.path.join("dummy", self.version_name) + + @property + def dummy_zip_file(self): + return os.path.join(self.dummy_data_folder, "dummy_data.zip") + + def download_dummy_data(self): + path_to_dummy_data_dir = ( + self.local_path_to_dummy_data if self.use_local_dummy_data is True else self.github_path_to_dummy_data + ) + + local_path = cached_path( + path_to_dummy_data_dir, cache_dir=self.cache_dir, extract_compressed_file=True, force_extract=True + ) + + return os.path.join(local_path, self.dummy_file_name) + + @property + def local_path_to_dummy_data(self): + return os.path.join(self.datasets_scripts_dir, self.dataset_name, self.dummy_zip_file) + + @property + def github_path_to_dummy_data(self): + if self._bucket_url is None: + self._bucket_url = hf_github_url(self.dataset_name, self.dummy_zip_file.replace(os.sep, "/")) + return self._bucket_url + + @property + def manual_dir(self): + # return full path if its a dir + if os.path.isdir(self.dummy_file): + return self.dummy_file + # else cut off path to file -> example `xsum`. + return "/".join(self.dummy_file.replace(os.sep, "/").split("/")[:-1]) + + # this function has to be in the manager under this name so that testing works + def download_and_extract(self, data_url, *args): + if self.load_existing_dummy_data: + # dummy data is downloaded and tested + dummy_file = self.dummy_file + else: + # dummy data cannot be downloaded and only the path to dummy file is returned + dummy_file = self.dummy_file_name + + # special case when data_url is a dict + if isinstance(data_url, dict): + return self.create_dummy_data_dict(dummy_file, data_url) + elif isinstance(data_url, (list, tuple)): + return self.create_dummy_data_list(dummy_file, data_url) + else: + return self.create_dummy_data_single(dummy_file, data_url) + + # this function has to be in the manager under this name so that testing works + def download(self, data_url, *args): + return self.download_and_extract(data_url) + + # this function has to be in the manager under this name so that testing works + def download_custom(self, data_url, custom_download): + return self.download_and_extract(data_url) + + # this function has to be in the manager under this name so that testing works + def extract(self, path, *args, **kwargs): + return path + + # this function has to be in the manager under this name so that testing works + def get_recorded_sizes_checksums(self): + return {} + + def create_dummy_data_dict(self, path_to_dummy_data, data_url): + dummy_data_dict = {} + for key, single_urls in data_url.items(): + for download_callback in self.download_callbacks: + if isinstance(single_urls, list): + for single_url in single_urls: + download_callback(single_url) + else: + single_url = single_urls + download_callback(single_url) + # we force the name of each key to be the last file / folder name of the url path + # if the url has arguments, we need to encode them with urllib.parse.quote_plus + if isinstance(single_urls, list): + value = [os.path.join(path_to_dummy_data, urllib.parse.quote_plus(Path(x).name)) for x in single_urls] + else: + single_url = single_urls + value = os.path.join(path_to_dummy_data, urllib.parse.quote_plus(Path(single_url).name)) + dummy_data_dict[key] = value + + # make sure that values are unique + if all(isinstance(i, str) for i in dummy_data_dict.values()) and len(set(dummy_data_dict.values())) < len( + dummy_data_dict.values() + ): + # append key to value to make its name unique + dummy_data_dict = {key: value + key for key, value in dummy_data_dict.items()} + + return dummy_data_dict + + def create_dummy_data_list(self, path_to_dummy_data, data_url): + dummy_data_list = [] + # trick: if there are many shards named like `data.txt-000001-of-00300`, only use the first one + is_tf_records = all(bool(re.findall("[0-9]{3,}-of-[0-9]{3,}", url)) for url in data_url) + is_pubmed_records = all( + url.startswith("https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmed") for url in data_url + ) + if data_url and (is_tf_records or is_pubmed_records): + data_url = [data_url[0]] * len(data_url) + for single_url in data_url: + for download_callback in self.download_callbacks: + download_callback(single_url) + # we force the name of each key to be the last file / folder name of the url path + # if the url has arguments, we need to encode them with urllib.parse.quote_plus + value = os.path.join(path_to_dummy_data, urllib.parse.quote_plus(single_url.split("/")[-1])) + dummy_data_list.append(value) + return dummy_data_list + + def create_dummy_data_single(self, path_to_dummy_data, data_url): + for download_callback in self.download_callbacks: + download_callback(data_url) + # we force the name of each key to be the last file / folder name of the url path + # if the url has arguments, we need to encode them with urllib.parse.quote_plus + value = os.path.join(path_to_dummy_data, urllib.parse.quote_plus(data_url.split("/")[-1])) + if os.path.exists(value) or not self.load_existing_dummy_data: + return value + else: + # Backward compatibility, maybe deprecate at one point. + # For many datasets with single url calls to dl_manager.download_and_extract, + # the dummy_data.zip file is actually the zipped downloaded file + # while now we expected the dummy_data.zip file to be a directory containing + # the downloaded file. + return path_to_dummy_data + + def delete_extracted_files(self): + pass + + def manage_extracted_files(self): + pass + + def iter_archive(self, path): + def _iter_archive_members(path): + # this preserves the order of the members inside the ZIP archive + dummy_parent_path = Path(self.dummy_file).parent + relative_path = path.relative_to(dummy_parent_path) + with ZipFile(self.local_path_to_dummy_data) as zip_file: + members = zip_file.namelist() + for member in members: + if member.startswith(relative_path.as_posix()): + yield dummy_parent_path.joinpath(member) + + path = Path(path) + file_paths = _iter_archive_members(path) if self.use_local_dummy_data else path.rglob("*") + for file_path in file_paths: + if file_path.is_file() and not file_path.name.startswith((".", "__")): + yield file_path.relative_to(path).as_posix(), file_path.open("rb") + + def iter_files(self, paths): + if not isinstance(paths, list): + paths = [paths] + for path in paths: + if os.path.isfile(path): + yield path + else: + for dirpath, dirnames, filenames in os.walk(path): + if os.path.basename(dirpath).startswith((".", "__")): + continue + dirnames.sort() + for filename in sorted(filenames): + if filename.startswith((".", "__")): + continue + yield os.path.join(dirpath, filename) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/streaming_download_manager.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/streaming_download_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..9a1d1e3a53cfc1db3e873e1f9ab3da9a157db3c0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/download/streaming_download_manager.py @@ -0,0 +1,210 @@ +import io +import os +from typing import Iterable, List, Optional, Tuple, Union + +from ..utils.file_utils import ( # noqa: F401 # backward compatibility + SINGLE_FILE_COMPRESSION_PROTOCOLS, + ArchiveIterable, + FilesIterable, + _get_extraction_protocol, + _get_path_extension, + _prepare_path_and_storage_options, + is_relative_path, + url_or_path_join, + xbasename, + xdirname, + xet_parse, + xexists, + xgetsize, + xglob, + xgzip_open, + xisdir, + xisfile, + xjoin, + xlistdir, + xnumpy_load, + xopen, + xpandas_read_csv, + xpandas_read_excel, + xPath, + xpyarrow_parquet_read_table, + xrelpath, + xsio_loadmat, + xsplit, + xsplitext, + xwalk, + xxml_dom_minidom_parse, +) +from ..utils.logging import get_logger +from ..utils.py_utils import map_nested +from .download_config import DownloadConfig + + +logger = get_logger(__name__) + + +class StreamingDownloadManager: + """ + Download manager that uses the "::" separator to navigate through (possibly remote) compressed archives. + Contrary to the regular `DownloadManager`, the `download` and `extract` methods don't actually download nor extract + data, but they rather return the path or url that could be opened using the `xopen` function which extends the + built-in `open` function to stream data from remote files. + """ + + is_streaming = True + + def __init__( + self, + dataset_name: Optional[str] = None, + data_dir: Optional[str] = None, + download_config: Optional[DownloadConfig] = None, + base_path: Optional[str] = None, + ): + self._dataset_name = dataset_name + self._data_dir = data_dir + self._base_path = base_path or os.path.abspath(".") + self.download_config = download_config or DownloadConfig() + + @property + def manual_dir(self): + return self._data_dir + + def download(self, url_or_urls): + """Normalize URL(s) of files to stream data from. + This is the lazy version of `DownloadManager.download` for streaming. + + Args: + url_or_urls (`str` or `list` or `dict`): + URL(s) of files to stream data from. Each url is a `str`. + + Returns: + url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input url_or_urls. + + Example: + + ```py + >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + ``` + """ + url_or_urls = map_nested(self._download_single, url_or_urls, map_tuple=True) + return url_or_urls + + def _download_single(self, urlpath: str) -> str: + urlpath = str(urlpath) + if is_relative_path(urlpath): + # append the relative path to the base_path + urlpath = url_or_path_join(self._base_path, urlpath) + return urlpath + + def extract(self, url_or_urls): + """Add extraction protocol for given url(s) for streaming. + + This is the lazy version of `DownloadManager.extract` for streaming. + + Args: + url_or_urls (`str` or `list` or `dict`): + URL(s) of files to stream data from. Each url is a `str`. + + Returns: + url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`. + + Example: + + ```py + >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + >>> extracted_files = dl_manager.extract(downloaded_files) + ``` + """ + urlpaths = map_nested(self._extract, url_or_urls, map_tuple=True) + return urlpaths + + def _extract(self, urlpath: str) -> str: + urlpath = str(urlpath) + protocol = _get_extraction_protocol(urlpath, download_config=self.download_config) + # get inner file: zip://train-00000.json.gz::https://foo.bar/data.zip -> zip://train-00000.json.gz + path = urlpath.split("::")[0] + extension = _get_path_extension(path) + if extension in ["tgz", "tar"] or path.endswith((".tar.gz", ".tar.bz2", ".tar.xz")): + raise NotImplementedError( + f"Extraction protocol for TAR archives like '{urlpath}' is not implemented in streaming mode. " + f"Please use `dl_manager.iter_archive` instead.\n\n" + f"Example usage:\n\n" + f"\turl = dl_manager.download(url)\n" + f"\ttar_archive_iterator = dl_manager.iter_archive(url)\n\n" + f"\tfor filename, file in tar_archive_iterator:\n" + f"\t\t..." + ) + if protocol is None: + # no extraction + return urlpath + elif protocol in SINGLE_FILE_COMPRESSION_PROTOCOLS: + # there is one single file which is the uncompressed file + inner_file = os.path.basename(urlpath.split("::")[0]) + inner_file = inner_file[: inner_file.rindex(".")] if "." in inner_file else inner_file + return f"{protocol}://{inner_file}::{urlpath}" + else: + return f"{protocol}://::{urlpath}" + + def download_and_extract(self, url_or_urls): + """Prepare given `url_or_urls` for streaming (add extraction protocol). + + This is the lazy version of `DownloadManager.download_and_extract` for streaming. + + Is equivalent to: + + ``` + urls = dl_manager.extract(dl_manager.download(url_or_urls)) + ``` + + Args: + url_or_urls (`str` or `list` or `dict`): + URL(s) to stream from data from. Each url is a `str`. + + Returns: + url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`. + """ + return self.extract(self.download(url_or_urls)) + + def iter_archive(self, urlpath_or_buf: Union[str, io.BufferedReader]) -> Iterable[Tuple]: + """Iterate over files within an archive. + + Args: + urlpath_or_buf (`str` or `io.BufferedReader`): + Archive path or archive binary file object. + + Yields: + `tuple[str, io.BufferedReader]`: + 2-tuple (path_within_archive, file_object). + File object is opened in binary mode. + + Example: + + ```py + >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') + >>> files = dl_manager.iter_archive(archive) + ``` + """ + + if hasattr(urlpath_or_buf, "read"): + return ArchiveIterable.from_buf(urlpath_or_buf) + else: + return ArchiveIterable.from_urlpath(urlpath_or_buf, download_config=self.download_config) + + def iter_files(self, urlpaths: Union[str, List[str]]) -> Iterable[str]: + """Iterate over files. + + Args: + urlpaths (`str` or `list` of `str`): + Root paths. + + Yields: + str: File URL path. + + Example: + + ```py + >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip') + >>> files = dl_manager.iter_files(files) + ``` + """ + return FilesIterable.from_urlpaths(urlpaths, download_config=self.download_config) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b3c03fbfed7fcee5a0bb3c3a2ff0d63fdf3dd62b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__init__.py @@ -0,0 +1,18 @@ +__all__ = [ + "Audio", + "Array2D", + "Array3D", + "Array4D", + "Array5D", + "ClassLabel", + "Features", + "Sequence", + "Value", + "Image", + "Translation", + "TranslationVariableLanguages", +] +from .audio import Audio +from .features import Array2D, Array3D, Array4D, Array5D, ClassLabel, Features, Sequence, Value +from .image import Image +from .translation import Translation, TranslationVariableLanguages diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d050b22559d07de29461384e557a906d89029318 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__pycache__/audio.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__pycache__/audio.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eb752e7ac0753e7f0eac76ad4cdc2c6e1414a0de Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__pycache__/audio.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__pycache__/image.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__pycache__/image.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..339953d5b9ebc6c9a30b83ab2718c27b7835a885 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__pycache__/image.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__pycache__/translation.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__pycache__/translation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b4a8a8ccad1d2142957c7f6456a516f58e4d65d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/__pycache__/translation.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/audio.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/audio.py new file mode 100644 index 0000000000000000000000000000000000000000..f7df47b7a061801196db39eca222c9e4d6f9e599 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/audio.py @@ -0,0 +1,277 @@ +import os +from dataclasses import dataclass, field +from io import BytesIO +from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union + +import numpy as np +import pyarrow as pa + +from .. import config +from ..download.download_config import DownloadConfig +from ..table import array_cast +from ..utils.file_utils import xopen, xsplitext +from ..utils.py_utils import no_op_if_value_is_null, string_to_dict + + +if TYPE_CHECKING: + from .features import FeatureType + + +@dataclass +class Audio: + """Audio [`Feature`] to extract audio data from an audio file. + + Input: The Audio feature accepts as input: + - A `str`: Absolute path to the audio file (i.e. random access is allowed). + - A `dict` with the keys: + + - `path`: String with relative path of the audio file to the archive file. + - `bytes`: Bytes content of the audio file. + + This is useful for archived files with sequential access. + + - A `dict` with the keys: + + - `path`: String with relative path of the audio file to the archive file. + - `array`: Array containing the audio sample + - `sampling_rate`: Integer corresponding to the sampling rate of the audio sample. + + This is useful for archived files with sequential access. + + Args: + sampling_rate (`int`, *optional*): + Target sampling rate. If `None`, the native sampling rate is used. + mono (`bool`, defaults to `True`): + Whether to convert the audio signal to mono by averaging samples across + channels. + decode (`bool`, defaults to `True`): + Whether to decode the audio data. If `False`, + returns the underlying dictionary in the format `{"path": audio_path, "bytes": audio_bytes}`. + + Example: + + ```py + >>> from datasets import load_dataset, Audio + >>> ds = load_dataset("PolyAI/minds14", name="en-US", split="train") + >>> ds = ds.cast_column("audio", Audio(sampling_rate=16000)) + >>> ds[0]["audio"] + {'array': array([ 2.3443763e-05, 2.1729663e-04, 2.2145823e-04, ..., + 3.8356509e-05, -7.3497440e-06, -2.1754686e-05], dtype=float32), + 'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav', + 'sampling_rate': 16000} + ``` + """ + + sampling_rate: Optional[int] = None + mono: bool = True + decode: bool = True + id: Optional[str] = None + # Automatically constructed + dtype: ClassVar[str] = "dict" + pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()}) + _type: str = field(default="Audio", init=False, repr=False) + + def __call__(self): + return self.pa_type + + def encode_example(self, value: Union[str, bytes, dict]) -> dict: + """Encode example into a format for Arrow. + + Args: + value (`str` or `dict`): + Data passed as input to Audio feature. + + Returns: + `dict` + """ + try: + import soundfile as sf # soundfile is a dependency of librosa, needed to decode audio files. + except ImportError as err: + raise ImportError("To support encoding audio data, please install 'soundfile'.") from err + if isinstance(value, str): + return {"bytes": None, "path": value} + elif isinstance(value, bytes): + return {"bytes": value, "path": None} + elif "array" in value: + # convert the audio array to wav bytes + buffer = BytesIO() + sf.write(buffer, value["array"], value["sampling_rate"], format="wav") + return {"bytes": buffer.getvalue(), "path": None} + elif value.get("path") is not None and os.path.isfile(value["path"]): + # we set "bytes": None to not duplicate the data if they're already available locally + if value["path"].endswith("pcm"): + # "PCM" only has raw audio bytes + if value.get("sampling_rate") is None: + # At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate + raise KeyError("To use PCM files, please specify a 'sampling_rate' in Audio object") + if value.get("bytes"): + # If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!) + bytes_value = np.frombuffer(value["bytes"], dtype=np.int16).astype(np.float32) / 32767 + else: + bytes_value = np.memmap(value["path"], dtype="h", mode="r").astype(np.float32) / 32767 + + buffer = BytesIO(bytes()) + sf.write(buffer, bytes_value, value["sampling_rate"], format="wav") + return {"bytes": buffer.getvalue(), "path": None} + else: + return {"bytes": None, "path": value.get("path")} + elif value.get("bytes") is not None or value.get("path") is not None: + # store the audio bytes, and path is used to infer the audio format using the file extension + return {"bytes": value.get("bytes"), "path": value.get("path")} + else: + raise ValueError( + f"An audio sample should have one of 'path' or 'bytes' but they are missing or None in {value}." + ) + + def decode_example( + self, value: dict, token_per_repo_id: Optional[Dict[str, Union[str, bool, None]]] = None + ) -> dict: + """Decode example audio file into audio data. + + Args: + value (`dict`): + A dictionary with keys: + + - `path`: String with relative audio file path. + - `bytes`: Bytes of the audio file. + token_per_repo_id (`dict`, *optional*): + To access and decode + audio files from private repositories on the Hub, you can pass + a dictionary repo_id (`str`) -> token (`bool` or `str`) + + Returns: + `dict` + """ + if not self.decode: + raise RuntimeError("Decoding is disabled for this feature. Please use Audio(decode=True) instead.") + + path, file = (value["path"], BytesIO(value["bytes"])) if value["bytes"] is not None else (value["path"], None) + if path is None and file is None: + raise ValueError(f"An audio sample should have one of 'path' or 'bytes' but both are None in {value}.") + + try: + import librosa + import soundfile as sf + except ImportError as err: + raise ImportError("To support decoding audio files, please install 'librosa' and 'soundfile'.") from err + + audio_format = xsplitext(path)[1][1:].lower() if path is not None else None + if not config.IS_OPUS_SUPPORTED and audio_format == "opus": + raise RuntimeError( + "Decoding 'opus' files requires system library 'libsndfile'>=1.0.31, " + 'You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. ' + ) + elif not config.IS_MP3_SUPPORTED and audio_format == "mp3": + raise RuntimeError( + "Decoding 'mp3' files requires system library 'libsndfile'>=1.1.0, " + 'You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. ' + ) + + if file is None: + token_per_repo_id = token_per_repo_id or {} + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL if source_url.startswith(config.HF_ENDPOINT) else config.HUB_DATASETS_HFFS_URL + ) + try: + repo_id = string_to_dict(source_url, pattern)["repo_id"] + token = token_per_repo_id[repo_id] + except (ValueError, KeyError): + token = None + + download_config = DownloadConfig(token=token) + with xopen(path, "rb", download_config=download_config) as f: + array, sampling_rate = sf.read(f) + + else: + array, sampling_rate = sf.read(file) + + array = array.T + if self.mono: + array = librosa.to_mono(array) + if self.sampling_rate and self.sampling_rate != sampling_rate: + array = librosa.resample(array, orig_sr=sampling_rate, target_sr=self.sampling_rate) + sampling_rate = self.sampling_rate + + return {"path": path, "array": array, "sampling_rate": sampling_rate} + + def flatten(self) -> Union["FeatureType", Dict[str, "FeatureType"]]: + """If in the decodable state, raise an error, otherwise flatten the feature into a dictionary.""" + from .features import Value + + if self.decode: + raise ValueError("Cannot flatten a decoded Audio feature.") + return { + "bytes": Value("binary"), + "path": Value("string"), + } + + def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray]) -> pa.StructArray: + """Cast an Arrow array to the Audio arrow storage type. + The Arrow types that can be converted to the Audio pyarrow storage type are: + + - `pa.string()` - it must contain the "path" data + - `pa.binary()` - it must contain the audio bytes + - `pa.struct({"bytes": pa.binary()})` + - `pa.struct({"path": pa.string()})` + - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter + + Args: + storage (`Union[pa.StringArray, pa.StructArray]`): + PyArrow array to cast. + + Returns: + `pa.StructArray`: Array in the Audio arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})` + """ + if pa.types.is_string(storage.type): + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_binary(storage.type): + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_struct(storage.type) and storage.type.get_all_field_indices("array"): + storage = pa.array([Audio().encode_example(x) if x is not None else None for x in storage.to_pylist()]) + elif pa.types.is_struct(storage.type): + if storage.type.get_field_index("bytes") >= 0: + bytes_array = storage.field("bytes") + else: + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + if storage.type.get_field_index("path") >= 0: + path_array = storage.field("path") + else: + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + return array_cast(storage, self.pa_type) + + def embed_storage(self, storage: pa.StructArray) -> pa.StructArray: + """Embed audio files into the Arrow array. + + Args: + storage (`pa.StructArray`): + PyArrow array to embed. + + Returns: + `pa.StructArray`: Array in the Audio arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + + @no_op_if_value_is_null + def path_to_bytes(path): + with xopen(path, "rb") as f: + bytes_ = f.read() + return bytes_ + + bytes_array = pa.array( + [ + (path_to_bytes(x["path"]) if x["bytes"] is None else x["bytes"]) if x is not None else None + for x in storage.to_pylist() + ], + type=pa.binary(), + ) + path_array = pa.array( + [os.path.basename(path) if path is not None else None for path in storage.field("path").to_pylist()], + type=pa.string(), + ) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null()) + return array_cast(storage, self.pa_type) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/features.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/features.py new file mode 100644 index 0000000000000000000000000000000000000000..5400ccdcc3f7c78e920112374d8e9a7e41c1c0e8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/features.py @@ -0,0 +1,2213 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""This class handle features definition in datasets and some utilities to display table type.""" + +import copy +import json +import re +import sys +from collections.abc import Iterable, Mapping +from collections.abc import Sequence as SequenceABC +from dataclasses import InitVar, dataclass, field, fields +from functools import reduce, wraps +from operator import mul +from typing import Any, Callable, ClassVar, Dict, List, Optional, Tuple, Union +from typing import Sequence as Sequence_ + +import numpy as np +import pandas as pd +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.types +import pyarrow_hotfix # noqa: F401 # to fix vulnerability on pyarrow<14.0.1 +from pandas.api.extensions import ExtensionArray as PandasExtensionArray +from pandas.api.extensions import ExtensionDtype as PandasExtensionDtype + +from .. import config +from ..naming import camelcase_to_snakecase, snakecase_to_camelcase +from ..table import array_cast +from ..utils import experimental, logging +from ..utils.py_utils import asdict, first_non_null_value, zip_dict +from .audio import Audio +from .image import Image, encode_pil_image +from .translation import Translation, TranslationVariableLanguages + + +logger = logging.get_logger(__name__) + + +def _arrow_to_datasets_dtype(arrow_type: pa.DataType) -> str: + """ + _arrow_to_datasets_dtype takes a pyarrow.DataType and converts it to a datasets string dtype. + In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))` + """ + if pyarrow.types.is_null(arrow_type): + return "null" + elif pyarrow.types.is_boolean(arrow_type): + return "bool" + elif pyarrow.types.is_int8(arrow_type): + return "int8" + elif pyarrow.types.is_int16(arrow_type): + return "int16" + elif pyarrow.types.is_int32(arrow_type): + return "int32" + elif pyarrow.types.is_int64(arrow_type): + return "int64" + elif pyarrow.types.is_uint8(arrow_type): + return "uint8" + elif pyarrow.types.is_uint16(arrow_type): + return "uint16" + elif pyarrow.types.is_uint32(arrow_type): + return "uint32" + elif pyarrow.types.is_uint64(arrow_type): + return "uint64" + elif pyarrow.types.is_float16(arrow_type): + return "float16" # pyarrow dtype is "halffloat" + elif pyarrow.types.is_float32(arrow_type): + return "float32" # pyarrow dtype is "float" + elif pyarrow.types.is_float64(arrow_type): + return "float64" # pyarrow dtype is "double" + elif pyarrow.types.is_time32(arrow_type): + return f"time32[{pa.type_for_alias(str(arrow_type)).unit}]" + elif pyarrow.types.is_time64(arrow_type): + return f"time64[{pa.type_for_alias(str(arrow_type)).unit}]" + elif pyarrow.types.is_timestamp(arrow_type): + if arrow_type.tz is None: + return f"timestamp[{arrow_type.unit}]" + elif arrow_type.tz: + return f"timestamp[{arrow_type.unit}, tz={arrow_type.tz}]" + else: + raise ValueError(f"Unexpected timestamp object {arrow_type}.") + elif pyarrow.types.is_date32(arrow_type): + return "date32" # pyarrow dtype is "date32[day]" + elif pyarrow.types.is_date64(arrow_type): + return "date64" # pyarrow dtype is "date64[ms]" + elif pyarrow.types.is_duration(arrow_type): + return f"duration[{arrow_type.unit}]" + elif pyarrow.types.is_decimal128(arrow_type): + return f"decimal128({arrow_type.precision}, {arrow_type.scale})" + elif pyarrow.types.is_decimal256(arrow_type): + return f"decimal256({arrow_type.precision}, {arrow_type.scale})" + elif pyarrow.types.is_binary(arrow_type): + return "binary" + elif pyarrow.types.is_large_binary(arrow_type): + return "large_binary" + elif pyarrow.types.is_string(arrow_type): + return "string" + elif pyarrow.types.is_large_string(arrow_type): + return "large_string" + elif pyarrow.types.is_dictionary(arrow_type): + return _arrow_to_datasets_dtype(arrow_type.value_type) + else: + raise ValueError(f"Arrow type {arrow_type} does not have a datasets dtype equivalent.") + + +def string_to_arrow(datasets_dtype: str) -> pa.DataType: + """ + string_to_arrow takes a datasets string dtype and converts it to a pyarrow.DataType. + + In effect, `dt == string_to_arrow(_arrow_to_datasets_dtype(dt))` + + This is necessary because the datasets.Value() primitive type is constructed using a string dtype + + Value(dtype=str) + + But Features.type (via `get_nested_type()` expects to resolve Features into a pyarrow Schema, + which means that each Value() must be able to resolve into a corresponding pyarrow.DataType, which is the + purpose of this function. + """ + + def _dtype_error_msg(dtype, pa_dtype, examples=None, urls=None): + msg = f"{dtype} is not a validly formatted string representation of the pyarrow {pa_dtype} type." + if examples: + examples = ", ".join(examples[:-1]) + " or " + examples[-1] if len(examples) > 1 else examples[0] + msg += f"\nValid examples include: {examples}." + if urls: + urls = ", ".join(urls[:-1]) + " and " + urls[-1] if len(urls) > 1 else urls[0] + msg += f"\nFor more insformation, see: {urls}." + return msg + + if datasets_dtype in pa.__dict__: + return pa.__dict__[datasets_dtype]() + + if (datasets_dtype + "_") in pa.__dict__: + return pa.__dict__[datasets_dtype + "_"]() + + timestamp_matches = re.search(r"^timestamp\[(.*)\]$", datasets_dtype) + if timestamp_matches: + timestamp_internals = timestamp_matches.group(1) + internals_matches = re.search(r"^(s|ms|us|ns),\s*tz=([a-zA-Z0-9/_+\-:]*)$", timestamp_internals) + if timestamp_internals in ["s", "ms", "us", "ns"]: + return pa.timestamp(timestamp_internals) + elif internals_matches: + return pa.timestamp(internals_matches.group(1), internals_matches.group(2)) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "timestamp", + examples=["timestamp[us]", "timestamp[us, tz=America/New_York"], + urls=["https://arrow.apache.org/docs/python/generated/pyarrow.timestamp.html"], + ) + ) + + duration_matches = re.search(r"^duration\[(.*)\]$", datasets_dtype) + if duration_matches: + duration_internals = duration_matches.group(1) + if duration_internals in ["s", "ms", "us", "ns"]: + return pa.duration(duration_internals) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "duration", + examples=["duration[s]", "duration[us]"], + urls=["https://arrow.apache.org/docs/python/generated/pyarrow.duration.html"], + ) + ) + + time_matches = re.search(r"^time(.*)\[(.*)\]$", datasets_dtype) + if time_matches: + time_internals_bits = time_matches.group(1) + if time_internals_bits == "32": + time_internals_unit = time_matches.group(2) + if time_internals_unit in ["s", "ms"]: + return pa.time32(time_internals_unit) + else: + raise ValueError( + f"{time_internals_unit} is not a valid unit for the pyarrow time32 type. Supported units: s (second) and ms (millisecond)." + ) + elif time_internals_bits == "64": + time_internals_unit = time_matches.group(2) + if time_internals_unit in ["us", "ns"]: + return pa.time64(time_internals_unit) + else: + raise ValueError( + f"{time_internals_unit} is not a valid unit for the pyarrow time64 type. Supported units: us (microsecond) and ns (nanosecond)." + ) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "time", + examples=["time32[s]", "time64[us]"], + urls=[ + "https://arrow.apache.org/docs/python/generated/pyarrow.time32.html", + "https://arrow.apache.org/docs/python/generated/pyarrow.time64.html", + ], + ) + ) + + decimal_matches = re.search(r"^decimal(.*)\((.*)\)$", datasets_dtype) + if decimal_matches: + decimal_internals_bits = decimal_matches.group(1) + if decimal_internals_bits == "128": + decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2)) + if decimal_internals_precision_and_scale: + precision = decimal_internals_precision_and_scale.group(1) + scale = decimal_internals_precision_and_scale.group(2) + return pa.decimal128(int(precision), int(scale)) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "decimal128", + examples=["decimal128(10, 2)", "decimal128(4, -2)"], + urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html"], + ) + ) + elif decimal_internals_bits == "256": + decimal_internals_precision_and_scale = re.search(r"^(\d+),\s*(-?\d+)$", decimal_matches.group(2)) + if decimal_internals_precision_and_scale: + precision = decimal_internals_precision_and_scale.group(1) + scale = decimal_internals_precision_and_scale.group(2) + return pa.decimal256(int(precision), int(scale)) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "decimal256", + examples=["decimal256(30, 2)", "decimal256(38, -4)"], + urls=["https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html"], + ) + ) + else: + raise ValueError( + _dtype_error_msg( + datasets_dtype, + "decimal", + examples=["decimal128(12, 3)", "decimal256(40, 6)"], + urls=[ + "https://arrow.apache.org/docs/python/generated/pyarrow.decimal128.html", + "https://arrow.apache.org/docs/python/generated/pyarrow.decimal256.html", + ], + ) + ) + + raise ValueError( + f"Neither {datasets_dtype} nor {datasets_dtype + '_'} seems to be a pyarrow data type. " + f"Please make sure to use a correct data type, see: " + f"https://arrow.apache.org/docs/python/api/datatypes.html#factory-functions" + ) + + +def _cast_to_python_objects(obj: Any, only_1d_for_numpy: bool, optimize_list_casting: bool) -> Tuple[Any, bool]: + """ + Cast pytorch/tensorflow/pandas objects to python numpy array/lists. + It works recursively. + + If `optimize_list_casting` is True, to avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted. + If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same. + This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example. + + Args: + obj: the object (nested struct) to cast. + only_1d_for_numpy (bool): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to + nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays. + Indeed Arrow only support converting 1-dimensional array values. + optimize_list_casting (bool): whether to optimize list casting by checking the first non-null element to see if it needs to be casted + and if it doesn't, not checking the rest of the list elements. + + Returns: + casted_obj: the casted object + has_changed (bool): True if the object has been changed, False if it is identical + """ + + if config.TF_AVAILABLE and "tensorflow" in sys.modules: + import tensorflow as tf + + if config.TORCH_AVAILABLE and "torch" in sys.modules: + import torch + + if config.JAX_AVAILABLE and "jax" in sys.modules: + import jax.numpy as jnp + + if config.PIL_AVAILABLE and "PIL" in sys.modules: + import PIL.Image + + if isinstance(obj, np.ndarray): + if obj.ndim == 0: + return obj[()], True + elif not only_1d_for_numpy or obj.ndim == 1: + return obj, False + else: + return ( + [ + _cast_to_python_objects( + x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for x in obj + ], + True, + ) + elif config.TORCH_AVAILABLE and "torch" in sys.modules and isinstance(obj, torch.Tensor): + if obj.dtype == torch.bfloat16: + return _cast_to_python_objects( + obj.detach().to(torch.float).cpu().numpy(), + only_1d_for_numpy=only_1d_for_numpy, + optimize_list_casting=optimize_list_casting, + )[0], True + if obj.ndim == 0: + return obj.detach().cpu().numpy()[()], True + elif not only_1d_for_numpy or obj.ndim == 1: + return obj.detach().cpu().numpy(), True + else: + return ( + [ + _cast_to_python_objects( + x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for x in obj.detach().cpu().numpy() + ], + True, + ) + elif config.TF_AVAILABLE and "tensorflow" in sys.modules and isinstance(obj, tf.Tensor): + if obj.ndim == 0: + return obj.numpy()[()], True + elif not only_1d_for_numpy or obj.ndim == 1: + return obj.numpy(), True + else: + return ( + [ + _cast_to_python_objects( + x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for x in obj.numpy() + ], + True, + ) + elif config.JAX_AVAILABLE and "jax" in sys.modules and isinstance(obj, jnp.ndarray): + if obj.ndim == 0: + return np.asarray(obj)[()], True + elif not only_1d_for_numpy or obj.ndim == 1: + return np.asarray(obj), True + else: + return ( + [ + _cast_to_python_objects( + x, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for x in np.asarray(obj) + ], + True, + ) + elif config.PIL_AVAILABLE and "PIL" in sys.modules and isinstance(obj, PIL.Image.Image): + return encode_pil_image(obj), True + elif isinstance(obj, pd.Series): + return ( + _cast_to_python_objects( + obj.tolist(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0], + True, + ) + elif isinstance(obj, pd.DataFrame): + return ( + { + key: _cast_to_python_objects( + value, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for key, value in obj.to_dict("series").items() + }, + True, + ) + elif isinstance(obj, pd.Timestamp): + return obj.to_pydatetime(), True + elif isinstance(obj, pd.Timedelta): + return obj.to_pytimedelta(), True + elif isinstance(obj, Mapping): + has_changed = not isinstance(obj, dict) + output = {} + for k, v in obj.items(): + casted_v, has_changed_v = _cast_to_python_objects( + v, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + ) + has_changed |= has_changed_v + output[k] = casted_v + return output if has_changed else obj, has_changed + elif hasattr(obj, "__array__"): + return ( + _cast_to_python_objects( + obj.__array__(), only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0], + True, + ) + elif isinstance(obj, (list, tuple)): + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt): + break + casted_first_elmt, has_changed_first_elmt = _cast_to_python_objects( + first_elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + ) + if has_changed_first_elmt or not optimize_list_casting: + return ( + [ + _cast_to_python_objects( + elmt, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + for elmt in obj + ], + True, + ) + else: + if isinstance(obj, (list, tuple)): + return obj, False + else: + return list(obj), True + else: + return obj, False + else: + return obj, False + + +def cast_to_python_objects(obj: Any, only_1d_for_numpy=False, optimize_list_casting=True) -> Any: + """ + Cast numpy/pytorch/tensorflow/pandas objects to python lists. + It works recursively. + + If `optimize_list_casting` is True, To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be casted. + If the first element needs to be casted, then all the elements of the list will be casted, otherwise they'll stay the same. + This trick allows to cast objects that contain tokenizers outputs without iterating over every single token for example. + + Args: + obj: the object (nested struct) to cast + only_1d_for_numpy (bool, default ``False``): whether to keep the full multi-dim tensors as multi-dim numpy arrays, or convert them to + nested lists of 1-dimensional numpy arrays. This can be useful to keep only 1-d arrays to instantiate Arrow arrays. + Indeed Arrow only support converting 1-dimensional array values. + optimize_list_casting (bool, default ``True``): whether to optimize list casting by checking the first non-null element to see if it needs to be casted + and if it doesn't, not checking the rest of the list elements. + + Returns: + casted_obj: the casted object + """ + return _cast_to_python_objects( + obj, only_1d_for_numpy=only_1d_for_numpy, optimize_list_casting=optimize_list_casting + )[0] + + +@dataclass +class Value: + """ + The `Value` dtypes are as follows: + + - `null` + - `bool` + - `int8` + - `int16` + - `int32` + - `int64` + - `uint8` + - `uint16` + - `uint32` + - `uint64` + - `float16` + - `float32` (alias float) + - `float64` (alias double) + - `time32[(s|ms)]` + - `time64[(us|ns)]` + - `timestamp[(s|ms|us|ns)]` + - `timestamp[(s|ms|us|ns), tz=(tzstring)]` + - `date32` + - `date64` + - `duration[(s|ms|us|ns)]` + - `decimal128(precision, scale)` + - `decimal256(precision, scale)` + - `binary` + - `large_binary` + - `string` + - `large_string` + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'stars': Value(dtype='int32')}) + >>> features + {'stars': Value(dtype='int32', id=None)} + ``` + """ + + dtype: str + id: Optional[str] = None + # Automatically constructed + pa_type: ClassVar[Any] = None + _type: str = field(default="Value", init=False, repr=False) + + def __post_init__(self): + if self.dtype == "double": # fix inferred type + self.dtype = "float64" + if self.dtype == "float": # fix inferred type + self.dtype = "float32" + self.pa_type = string_to_arrow(self.dtype) + + def __call__(self): + return self.pa_type + + def encode_example(self, value): + if pa.types.is_boolean(self.pa_type): + return bool(value) + elif pa.types.is_integer(self.pa_type): + return int(value) + elif pa.types.is_floating(self.pa_type): + return float(value) + elif pa.types.is_string(self.pa_type): + return str(value) + else: + return value + + +class _ArrayXD: + def __post_init__(self): + self.shape = tuple(self.shape) + + def __call__(self): + pa_type = globals()[self.__class__.__name__ + "ExtensionType"](self.shape, self.dtype) + return pa_type + + def encode_example(self, value): + return value + + +@dataclass +class Array2D(_ArrayXD): + """Create a two-dimensional array. + + Args: + shape (`tuple`): + The size of each dimension. + dtype (`str`): + The value of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'x': Array2D(shape=(1, 3), dtype='int32')}) + ``` + """ + + shape: tuple + dtype: str + id: Optional[str] = None + # Automatically constructed + _type: str = field(default="Array2D", init=False, repr=False) + + +@dataclass +class Array3D(_ArrayXD): + """Create a three-dimensional array. + + Args: + shape (`tuple`): + The size of each dimension. + dtype (`str`): + The value of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'x': Array3D(shape=(1, 2, 3), dtype='int32')}) + ``` + """ + + shape: tuple + dtype: str + id: Optional[str] = None + # Automatically constructed + _type: str = field(default="Array3D", init=False, repr=False) + + +@dataclass +class Array4D(_ArrayXD): + """Create a four-dimensional array. + + Args: + shape (`tuple`): + The size of each dimension. + dtype (`str`): + The value of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'x': Array4D(shape=(1, 2, 2, 3), dtype='int32')}) + ``` + """ + + shape: tuple + dtype: str + id: Optional[str] = None + # Automatically constructed + _type: str = field(default="Array4D", init=False, repr=False) + + +@dataclass +class Array5D(_ArrayXD): + """Create a five-dimensional array. + + Args: + shape (`tuple`): + The size of each dimension. + dtype (`str`): + The value of the data type. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'x': Array5D(shape=(1, 2, 2, 3, 3), dtype='int32')}) + ``` + """ + + shape: tuple + dtype: str + id: Optional[str] = None + # Automatically constructed + _type: str = field(default="Array5D", init=False, repr=False) + + +class _ArrayXDExtensionType(pa.ExtensionType): + ndims: Optional[int] = None + + def __init__(self, shape: tuple, dtype: str): + if self.ndims is None or self.ndims <= 1: + raise ValueError("You must instantiate an array type with a value for dim that is > 1") + if len(shape) != self.ndims: + raise ValueError(f"shape={shape} and ndims={self.ndims} don't match") + for dim in range(1, self.ndims): + if shape[dim] is None: + raise ValueError(f"Support only dynamic size on first dimension. Got: {shape}") + self.shape = tuple(shape) + self.value_type = dtype + self.storage_dtype = self._generate_dtype(self.value_type) + pa.ExtensionType.__init__(self, self.storage_dtype, f"{self.__class__.__module__}.{self.__class__.__name__}") + + def __arrow_ext_serialize__(self): + return json.dumps((self.shape, self.value_type)).encode() + + @classmethod + def __arrow_ext_deserialize__(cls, storage_type, serialized): + args = json.loads(serialized) + return cls(*args) + + # This was added to pa.ExtensionType in pyarrow >= 13.0.0 + def __reduce__(self): + return self.__arrow_ext_deserialize__, (self.storage_type, self.__arrow_ext_serialize__()) + + def __hash__(self): + return hash((self.__class__, self.shape, self.value_type)) + + def __arrow_ext_class__(self): + return ArrayExtensionArray + + def _generate_dtype(self, dtype): + dtype = string_to_arrow(dtype) + for d in reversed(self.shape): + dtype = pa.list_(dtype) + # Don't specify the size of the list, since fixed length list arrays have issues + # being validated after slicing in pyarrow 0.17.1 + return dtype + + def to_pandas_dtype(self): + return PandasArrayExtensionDtype(self.value_type) + + +class Array2DExtensionType(_ArrayXDExtensionType): + ndims = 2 + + +class Array3DExtensionType(_ArrayXDExtensionType): + ndims = 3 + + +class Array4DExtensionType(_ArrayXDExtensionType): + ndims = 4 + + +class Array5DExtensionType(_ArrayXDExtensionType): + ndims = 5 + + +# Register the extension types for deserialization +pa.register_extension_type(Array2DExtensionType((1, 2), "int64")) +pa.register_extension_type(Array3DExtensionType((1, 2, 3), "int64")) +pa.register_extension_type(Array4DExtensionType((1, 2, 3, 4), "int64")) +pa.register_extension_type(Array5DExtensionType((1, 2, 3, 4, 5), "int64")) + + +def _is_zero_copy_only(pa_type: pa.DataType, unnest: bool = False) -> bool: + """ + When converting a pyarrow array to a numpy array, we must know whether this could be done in zero-copy or not. + This function returns the value of the ``zero_copy_only`` parameter to pass to ``.to_numpy()``, given the type of the pyarrow array. + + # zero copy is available for all primitive types except booleans and temporal types (date, time, timestamp or duration) + # primitive types are types for which the physical representation in arrow and in numpy + # https://github.com/wesm/arrow/blob/c07b9b48cf3e0bbbab493992a492ae47e5b04cad/python/pyarrow/types.pxi#L821 + # see https://arrow.apache.org/docs/python/generated/pyarrow.Array.html#pyarrow.Array.to_numpy + # and https://issues.apache.org/jira/browse/ARROW-2871?jql=text%20~%20%22boolean%20to_numpy%22 + """ + + def _unnest_pa_type(pa_type: pa.DataType) -> pa.DataType: + if pa.types.is_list(pa_type): + return _unnest_pa_type(pa_type.value_type) + return pa_type + + if unnest: + pa_type = _unnest_pa_type(pa_type) + return pa.types.is_primitive(pa_type) and not (pa.types.is_boolean(pa_type) or pa.types.is_temporal(pa_type)) + + +class ArrayExtensionArray(pa.ExtensionArray): + def __array__(self): + zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True) + return self.to_numpy(zero_copy_only=zero_copy_only) + + def __getitem__(self, i): + return self.storage[i] + + def to_numpy(self, zero_copy_only=True): + storage: pa.ListArray = self.storage + null_mask = storage.is_null().to_numpy(zero_copy_only=False) + + if self.type.shape[0] is not None: + size = 1 + null_indices = np.arange(len(storage))[null_mask] - np.arange(np.sum(null_mask)) + + for i in range(self.type.ndims): + size *= self.type.shape[i] + storage = storage.flatten() + numpy_arr = storage.to_numpy(zero_copy_only=zero_copy_only) + numpy_arr = numpy_arr.reshape(len(self) - len(null_indices), *self.type.shape) + + if len(null_indices): + numpy_arr = np.insert(numpy_arr.astype(np.float64), null_indices, np.nan, axis=0) + + else: + shape = self.type.shape + ndims = self.type.ndims + arrays = [] + first_dim_offsets = np.array([off.as_py() for off in storage.offsets]) + for i, is_null in enumerate(null_mask): + if is_null: + arrays.append(np.nan) + else: + storage_el = storage[i : i + 1] + first_dim = first_dim_offsets[i + 1] - first_dim_offsets[i] + # flatten storage + for _ in range(ndims): + storage_el = storage_el.flatten() + + numpy_arr = storage_el.to_numpy(zero_copy_only=zero_copy_only) + arrays.append(numpy_arr.reshape(first_dim, *shape[1:])) + + if len(np.unique(np.diff(first_dim_offsets))) > 1: + # ragged + numpy_arr = np.empty(len(arrays), dtype=object) + numpy_arr[:] = arrays + else: + numpy_arr = np.array(arrays) + + return numpy_arr + + def to_pylist(self): + zero_copy_only = _is_zero_copy_only(self.storage.type, unnest=True) + numpy_arr = self.to_numpy(zero_copy_only=zero_copy_only) + if self.type.shape[0] is None and numpy_arr.dtype == object: + return [arr.tolist() for arr in numpy_arr.tolist()] + else: + return numpy_arr.tolist() + + +class PandasArrayExtensionDtype(PandasExtensionDtype): + _metadata = "value_type" + + def __init__(self, value_type: Union["PandasArrayExtensionDtype", np.dtype]): + self._value_type = value_type + + def __from_arrow__(self, array: Union[pa.Array, pa.ChunkedArray]): + if isinstance(array, pa.ChunkedArray): + array = array.type.wrap_array(pa.concat_arrays([chunk.storage for chunk in array.chunks])) + zero_copy_only = _is_zero_copy_only(array.storage.type, unnest=True) + numpy_arr = array.to_numpy(zero_copy_only=zero_copy_only) + return PandasArrayExtensionArray(numpy_arr) + + @classmethod + def construct_array_type(cls): + return PandasArrayExtensionArray + + @property + def type(self) -> type: + return np.ndarray + + @property + def kind(self) -> str: + return "O" + + @property + def name(self) -> str: + return f"array[{self.value_type}]" + + @property + def value_type(self) -> np.dtype: + return self._value_type + + +class PandasArrayExtensionArray(PandasExtensionArray): + def __init__(self, data: np.ndarray, copy: bool = False): + self._data = data if not copy else np.array(data) + self._dtype = PandasArrayExtensionDtype(data.dtype) + + def __array__(self, dtype=None): + """ + Convert to NumPy Array. + Note that Pandas expects a 1D array when dtype is set to object. + But for other dtypes, the returned shape is the same as the one of ``data``. + + More info about pandas 1D requirement for PandasExtensionArray here: + https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.api.extensions.ExtensionArray.html#pandas.api.extensions.ExtensionArray + + """ + if dtype == object: + out = np.empty(len(self._data), dtype=object) + for i in range(len(self._data)): + out[i] = self._data[i] + return out + if dtype is None: + return self._data + else: + return self._data.astype(dtype) + + def copy(self, deep: bool = False) -> "PandasArrayExtensionArray": + return PandasArrayExtensionArray(self._data, copy=True) + + @classmethod + def _from_sequence( + cls, scalars, dtype: Optional[PandasArrayExtensionDtype] = None, copy: bool = False + ) -> "PandasArrayExtensionArray": + if len(scalars) > 1 and all( + isinstance(x, np.ndarray) and x.shape == scalars[0].shape and x.dtype == scalars[0].dtype for x in scalars + ): + data = np.array(scalars, dtype=dtype if dtype is None else dtype.value_type, copy=copy) + else: + data = np.empty(len(scalars), dtype=object) + data[:] = scalars + return cls(data, copy=copy) + + @classmethod + def _concat_same_type(cls, to_concat: Sequence_["PandasArrayExtensionArray"]) -> "PandasArrayExtensionArray": + if len(to_concat) > 1 and all( + va._data.shape == to_concat[0]._data.shape and va._data.dtype == to_concat[0]._data.dtype + for va in to_concat + ): + data = np.vstack([va._data for va in to_concat]) + else: + data = np.empty(len(to_concat), dtype=object) + data[:] = [va._data for va in to_concat] + return cls(data, copy=False) + + @property + def dtype(self) -> PandasArrayExtensionDtype: + return self._dtype + + @property + def nbytes(self) -> int: + return self._data.nbytes + + def isna(self) -> np.ndarray: + return np.array([pd.isna(arr).any() for arr in self._data]) + + def __setitem__(self, key: Union[int, slice, np.ndarray], value: Any) -> None: + raise NotImplementedError() + + def __getitem__(self, item: Union[int, slice, np.ndarray]) -> Union[np.ndarray, "PandasArrayExtensionArray"]: + if isinstance(item, int): + return self._data[item] + return PandasArrayExtensionArray(self._data[item], copy=False) + + def take( + self, indices: Sequence_[int], allow_fill: bool = False, fill_value: bool = None + ) -> "PandasArrayExtensionArray": + indices: np.ndarray = np.asarray(indices, dtype=int) + if allow_fill: + fill_value = ( + self.dtype.na_value if fill_value is None else np.asarray(fill_value, dtype=self.dtype.value_type) + ) + mask = indices == -1 + if (indices < -1).any(): + raise ValueError("Invalid value in `indices`, must be all >= -1 for `allow_fill` is True") + elif len(self) > 0: + pass + elif not np.all(mask): + raise IndexError("Invalid take for empty PandasArrayExtensionArray, must be all -1.") + else: + data = np.array([fill_value] * len(indices), dtype=self.dtype.value_type) + return PandasArrayExtensionArray(data, copy=False) + took = self._data.take(indices, axis=0) + if allow_fill and mask.any(): + took[mask] = [fill_value] * np.sum(mask) + return PandasArrayExtensionArray(took, copy=False) + + def __len__(self) -> int: + return len(self._data) + + def __eq__(self, other) -> np.ndarray: + if not isinstance(other, PandasArrayExtensionArray): + raise NotImplementedError(f"Invalid type to compare to: {type(other)}") + return (self._data == other._data).all() + + +def pandas_types_mapper(dtype): + if isinstance(dtype, _ArrayXDExtensionType): + return PandasArrayExtensionDtype(dtype.value_type) + + +@dataclass +class ClassLabel: + """Feature type for integer class labels. + + There are 3 ways to define a `ClassLabel`, which correspond to the 3 arguments: + + * `num_classes`: Create 0 to (num_classes-1) labels. + * `names`: List of label strings. + * `names_file`: File containing the list of labels. + + Under the hood the labels are stored as integers. + You can use negative integers to represent unknown/missing labels. + + Args: + num_classes (`int`, *optional*): + Number of classes. All labels must be < `num_classes`. + names (`list` of `str`, *optional*): + String names for the integer classes. + The order in which the names are provided is kept. + names_file (`str`, *optional*): + Path to a file with names for the integer classes, one per line. + + Example: + + ```py + >>> from datasets import Features + >>> features = Features({'label': ClassLabel(num_classes=3, names=['bad', 'ok', 'good'])}) + >>> features + {'label': ClassLabel(num_classes=3, names=['bad', 'ok', 'good'], id=None)} + ``` + """ + + num_classes: InitVar[Optional[int]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict + names: List[str] = None + names_file: InitVar[Optional[str]] = None # Pseudo-field: ignored by asdict/fields when converting to/from dict + id: Optional[str] = None + # Automatically constructed + dtype: ClassVar[str] = "int64" + pa_type: ClassVar[Any] = pa.int64() + _str2int: ClassVar[Dict[str, int]] = None + _int2str: ClassVar[Dict[int, int]] = None + _type: str = field(default="ClassLabel", init=False, repr=False) + + def __post_init__(self, num_classes, names_file): + self.num_classes = num_classes + self.names_file = names_file + if self.names_file is not None and self.names is not None: + raise ValueError("Please provide either names or names_file but not both.") + # Set self.names + if self.names is None: + if self.names_file is not None: + self.names = self._load_names_from_file(self.names_file) + elif self.num_classes is not None: + self.names = [str(i) for i in range(self.num_classes)] + else: + raise ValueError("Please provide either num_classes, names or names_file.") + elif not isinstance(self.names, SequenceABC): + raise TypeError(f"Please provide names as a list, is {type(self.names)}") + # Set self.num_classes + if self.num_classes is None: + self.num_classes = len(self.names) + elif self.num_classes != len(self.names): + raise ValueError( + "ClassLabel number of names do not match the defined num_classes. " + f"Got {len(self.names)} names VS {self.num_classes} num_classes" + ) + # Prepare mappings + self._int2str = [str(name) for name in self.names] + self._str2int = {name: i for i, name in enumerate(self._int2str)} + if len(self._int2str) != len(self._str2int): + raise ValueError("Some label names are duplicated. Each label name should be unique.") + + def __call__(self): + return self.pa_type + + def str2int(self, values: Union[str, Iterable]) -> Union[int, Iterable]: + """Conversion class name `string` => `integer`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("rotten_tomatoes", split="train") + >>> ds.features["label"].str2int('neg') + 0 + ``` + """ + if not isinstance(values, str) and not isinstance(values, Iterable): + raise ValueError( + f"Values {values} should be a string or an Iterable (list, numpy array, pytorch, tensorflow tensors)" + ) + return_list = True + if isinstance(values, str): + values = [values] + return_list = False + + output = [self._strval2int(value) for value in values] + return output if return_list else output[0] + + def _strval2int(self, value: str) -> int: + failed_parse = False + value = str(value) + # first attempt - raw string value + int_value = self._str2int.get(value) + if int_value is None: + # second attempt - strip whitespace + int_value = self._str2int.get(value.strip()) + if int_value is None: + # third attempt - convert str to int + try: + int_value = int(value) + except ValueError: + failed_parse = True + else: + if int_value < -1 or int_value >= self.num_classes: + failed_parse = True + if failed_parse: + raise ValueError(f"Invalid string class label {value}") + return int_value + + def int2str(self, values: Union[int, Iterable]) -> Union[str, Iterable]: + """Conversion `integer` => class name `string`. + + Regarding unknown/missing labels: passing negative integers raises `ValueError`. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("rotten_tomatoes", split="train") + >>> ds.features["label"].int2str(0) + 'neg' + ``` + """ + if not isinstance(values, int) and not isinstance(values, Iterable): + raise ValueError( + f"Values {values} should be an integer or an Iterable (list, numpy array, pytorch, tensorflow tensors)" + ) + return_list = True + if isinstance(values, int): + values = [values] + return_list = False + + for v in values: + if not 0 <= v < self.num_classes: + raise ValueError(f"Invalid integer class label {v:d}") + + output = [self._int2str[int(v)] for v in values] + return output if return_list else output[0] + + def encode_example(self, example_data): + if self.num_classes is None: + raise ValueError( + "Trying to use ClassLabel feature with undefined number of class. " + "Please set ClassLabel.names or num_classes." + ) + + # If a string is given, convert to associated integer + if isinstance(example_data, str): + example_data = self.str2int(example_data) + + # Allowing -1 to mean no label. + if not -1 <= example_data < self.num_classes: + raise ValueError(f"Class label {example_data:d} greater than configured num_classes {self.num_classes}") + return example_data + + def cast_storage(self, storage: Union[pa.StringArray, pa.IntegerArray]) -> pa.Int64Array: + """Cast an Arrow array to the `ClassLabel` arrow storage type. + The Arrow types that can be converted to the `ClassLabel` pyarrow storage type are: + + - `pa.string()` + - `pa.int()` + + Args: + storage (`Union[pa.StringArray, pa.IntegerArray]`): + PyArrow array to cast. + + Returns: + `pa.Int64Array`: Array in the `ClassLabel` arrow storage type. + """ + if isinstance(storage, pa.IntegerArray) and len(storage) > 0: + min_max = pc.min_max(storage).as_py() + if min_max["max"] is not None and min_max["max"] >= self.num_classes: + raise ValueError( + f"Class label {min_max['max']} greater than configured num_classes {self.num_classes}" + ) + elif isinstance(storage, pa.StringArray): + storage = pa.array( + [self._strval2int(label) if label is not None else None for label in storage.to_pylist()] + ) + return array_cast(storage, self.pa_type) + + @staticmethod + def _load_names_from_file(names_filepath): + with open(names_filepath, encoding="utf-8") as f: + return [name.strip() for name in f.read().split("\n") if name.strip()] # Filter empty names + + +@dataclass +class Sequence: + """Construct a list of feature from a single type or a dict of types. + Mostly here for compatiblity with tfds. + + Args: + feature: + A list of features of a single type or a dictionary of types. + length (`int`): + Length of the sequence. + + Example: + + ```py + >>> from datasets import Features, Sequence, Value, ClassLabel + >>> features = Features({'post': Sequence(feature={'text': Value(dtype='string'), 'upvotes': Value(dtype='int32'), 'label': ClassLabel(num_classes=2, names=['hot', 'cold'])})}) + >>> features + {'post': Sequence(feature={'text': Value(dtype='string', id=None), 'upvotes': Value(dtype='int32', id=None), 'label': ClassLabel(num_classes=2, names=['hot', 'cold'], id=None)}, length=-1, id=None)} + ``` + """ + + feature: Any + length: int = -1 + id: Optional[str] = None + # Automatically constructed + dtype: ClassVar[str] = "list" + pa_type: ClassVar[Any] = None + _type: str = field(default="Sequence", init=False, repr=False) + + +FeatureType = Union[ + dict, + list, + tuple, + Value, + ClassLabel, + Translation, + TranslationVariableLanguages, + Sequence, + Array2D, + Array3D, + Array4D, + Array5D, + Audio, + Image, +] + + +def _check_non_null_non_empty_recursive(obj, schema: Optional[FeatureType] = None) -> bool: + """ + Check if the object is not None. + If the object is a list or a tuple, recursively check the first element of the sequence and stop if at any point the first element is not a sequence or is an empty sequence. + """ + if obj is None: + return False + elif isinstance(obj, (list, tuple)) and (schema is None or isinstance(schema, (list, tuple, Sequence))): + if len(obj) > 0: + if schema is None: + pass + elif isinstance(schema, (list, tuple)): + schema = schema[0] + else: + schema = schema.feature + return _check_non_null_non_empty_recursive(obj[0], schema) + else: + return False + else: + return True + + +def get_nested_type(schema: FeatureType) -> pa.DataType: + """ + get_nested_type() converts a datasets.FeatureType into a pyarrow.DataType, and acts as the inverse of + generate_from_arrow_type(). + + It performs double-duty as the implementation of Features.type and handles the conversion of + datasets.Feature->pa.struct + """ + # Nested structures: we allow dict, list/tuples, sequences + if isinstance(schema, Features): + return pa.struct( + {key: get_nested_type(schema[key]) for key in schema} + ) # Features is subclass of dict, and dict order is deterministic since Python 3.6 + elif isinstance(schema, dict): + return pa.struct( + {key: get_nested_type(schema[key]) for key in schema} + ) # however don't sort on struct types since the order matters + elif isinstance(schema, (list, tuple)): + if len(schema) != 1: + raise ValueError("When defining list feature, you should just provide one example of the inner type") + value_type = get_nested_type(schema[0]) + return pa.list_(value_type) + elif isinstance(schema, Sequence): + value_type = get_nested_type(schema.feature) + # We allow to reverse list of dict => dict of list for compatibility with tfds + if isinstance(schema.feature, dict): + return pa.struct({f.name: pa.list_(f.type, schema.length) for f in value_type}) + return pa.list_(value_type, schema.length) + + # Other objects are callable which returns their data type (ClassLabel, Array2D, Translation, Arrow datatype creation methods) + return schema() + + +def encode_nested_example(schema, obj, level=0): + """Encode a nested example. + This is used since some features (in particular ClassLabel) have some logic during encoding. + + To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be encoded. + If the first element needs to be encoded, then all the elements of the list will be encoded, otherwise they'll stay the same. + """ + # Nested structures: we allow dict, list/tuples, sequences + if isinstance(schema, dict): + if level == 0 and obj is None: + raise ValueError("Got None but expected a dictionary instead") + return ( + {k: encode_nested_example(schema[k], obj.get(k), level=level + 1) for k in schema} + if obj is not None + else None + ) + + elif isinstance(schema, (list, tuple)): + sub_schema = schema[0] + if obj is None: + return None + elif isinstance(obj, np.ndarray): + return encode_nested_example(schema, obj.tolist()) + else: + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, sub_schema): + break + if encode_nested_example(sub_schema, first_elmt, level=level + 1) != first_elmt: + return [encode_nested_example(sub_schema, o, level=level + 1) for o in obj] + return list(obj) + elif isinstance(schema, Sequence): + if obj is None: + return None + # We allow to reverse list of dict => dict of list for compatiblity with tfds + if isinstance(schema.feature, dict): + # dict of list to fill + list_dict = {} + if isinstance(obj, (list, tuple)): + # obj is a list of dict + for k in schema.feature: + list_dict[k] = [encode_nested_example(schema.feature[k], o.get(k), level=level + 1) for o in obj] + return list_dict + else: + # obj is a single dict + for k in schema.feature: + list_dict[k] = ( + [encode_nested_example(schema.feature[k], o, level=level + 1) for o in obj[k]] + if k in obj + else None + ) + return list_dict + # schema.feature is not a dict + if isinstance(obj, str): # don't interpret a string as a list + raise ValueError(f"Got a string but expected a list instead: '{obj}'") + else: + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, schema.feature): + break + # be careful when comparing tensors here + if ( + not isinstance(first_elmt, list) + or encode_nested_example(schema.feature, first_elmt, level=level + 1) != first_elmt + ): + return [encode_nested_example(schema.feature, o, level=level + 1) for o in obj] + return list(obj) + # Object with special encoding: + # ClassLabel will convert from string to int, TranslationVariableLanguages does some checks + elif isinstance(schema, (Audio, Image, ClassLabel, TranslationVariableLanguages, Value, _ArrayXD)): + return schema.encode_example(obj) if obj is not None else None + # Other object should be directly convertible to a native Arrow type (like Translation and Translation) + return obj + + +def decode_nested_example(schema, obj, token_per_repo_id: Optional[Dict[str, Union[str, bool, None]]] = None): + """Decode a nested example. + This is used since some features (in particular Audio and Image) have some logic during decoding. + + To avoid iterating over possibly long lists, it first checks (recursively) if the first element that is not None or empty (if it is a sequence) has to be decoded. + If the first element needs to be decoded, then all the elements of the list will be decoded, otherwise they'll stay the same. + """ + # Nested structures: we allow dict, list/tuples, sequences + if isinstance(schema, dict): + return ( + {k: decode_nested_example(sub_schema, sub_obj) for k, (sub_schema, sub_obj) in zip_dict(schema, obj)} + if obj is not None + else None + ) + elif isinstance(schema, (list, tuple)): + sub_schema = schema[0] + if obj is None: + return None + else: + if len(obj) > 0: + for first_elmt in obj: + if _check_non_null_non_empty_recursive(first_elmt, sub_schema): + break + if decode_nested_example(sub_schema, first_elmt) != first_elmt: + return [decode_nested_example(sub_schema, o) for o in obj] + return list(obj) + elif isinstance(schema, Sequence): + # We allow to reverse list of dict => dict of list for compatiblity with tfds + if isinstance(schema.feature, dict): + return {k: decode_nested_example([schema.feature[k]], obj[k]) for k in schema.feature} + else: + return decode_nested_example([schema.feature], obj) + # Object with special decoding: + elif isinstance(schema, (Audio, Image)): + # we pass the token to read and decode files from private repositories in streaming mode + if obj is not None and schema.decode: + return schema.decode_example(obj, token_per_repo_id=token_per_repo_id) + return obj + + +_FEATURE_TYPES: Dict[str, FeatureType] = { + Value.__name__: Value, + ClassLabel.__name__: ClassLabel, + Translation.__name__: Translation, + TranslationVariableLanguages.__name__: TranslationVariableLanguages, + Sequence.__name__: Sequence, + Array2D.__name__: Array2D, + Array3D.__name__: Array3D, + Array4D.__name__: Array4D, + Array5D.__name__: Array5D, + Audio.__name__: Audio, + Image.__name__: Image, +} + + +@experimental +def register_feature( + feature_cls: type, + feature_type: str, +): + """ + Register a Feature object using a name and class. + This function must be used on a Feature class. + """ + if feature_type in _FEATURE_TYPES: + logger.warning( + f"Overwriting feature type '{feature_type}' ({_FEATURE_TYPES[feature_type].__name__} -> {feature_cls.__name__})" + ) + _FEATURE_TYPES[feature_type] = feature_cls + + +def generate_from_dict(obj: Any): + """Regenerate the nested feature object from a deserialized dict. + We use the '_type' fields to get the dataclass name to load. + + generate_from_dict is the recursive helper for Features.from_dict, and allows for a convenient constructor syntax + to define features from deserialized JSON dictionaries. This function is used in particular when deserializing + a :class:`DatasetInfo` that was dumped to a JSON object. This acts as an analogue to + :meth:`Features.from_arrow_schema` and handles the recursive field-by-field instantiation, but doesn't require any + mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive dtypes + that :class:`Value` automatically performs. + """ + # Nested structures: we allow dict, list/tuples, sequences + if isinstance(obj, list): + return [generate_from_dict(value) for value in obj] + # Otherwise we have a dict or a dataclass + if "_type" not in obj or isinstance(obj["_type"], dict): + return {key: generate_from_dict(value) for key, value in obj.items()} + obj = dict(obj) + _type = obj.pop("_type") + class_type = _FEATURE_TYPES.get(_type, None) or globals().get(_type, None) + + if class_type is None: + raise ValueError(f"Feature type '{_type}' not found. Available feature types: {list(_FEATURE_TYPES.keys())}") + + if class_type == Sequence: + return Sequence(feature=generate_from_dict(obj["feature"]), length=obj.get("length", -1)) + + field_names = {f.name for f in fields(class_type)} + return class_type(**{k: v for k, v in obj.items() if k in field_names}) + + +def generate_from_arrow_type(pa_type: pa.DataType) -> FeatureType: + """ + generate_from_arrow_type accepts an arrow DataType and returns a datasets FeatureType to be used as the type for + a single field. + + This is the high-level arrow->datasets type conversion and is inverted by get_nested_type(). + + This operates at the individual *field* level, whereas Features.from_arrow_schema() operates at the + full schema level and holds the methods that represent the bijection from Features<->pyarrow.Schema + """ + if isinstance(pa_type, pa.StructType): + return {field.name: generate_from_arrow_type(field.type) for field in pa_type} + elif isinstance(pa_type, pa.FixedSizeListType): + return Sequence(feature=generate_from_arrow_type(pa_type.value_type), length=pa_type.list_size) + elif isinstance(pa_type, pa.ListType): + feature = generate_from_arrow_type(pa_type.value_type) + if isinstance(feature, (dict, tuple, list)): + return [feature] + return Sequence(feature=feature) + elif isinstance(pa_type, _ArrayXDExtensionType): + array_feature = [None, None, Array2D, Array3D, Array4D, Array5D][pa_type.ndims] + return array_feature(shape=pa_type.shape, dtype=pa_type.value_type) + elif isinstance(pa_type, pa.DataType): + return Value(dtype=_arrow_to_datasets_dtype(pa_type)) + else: + raise ValueError(f"Cannot convert {pa_type} to a Feature type.") + + +def numpy_to_pyarrow_listarray(arr: np.ndarray, type: pa.DataType = None) -> pa.ListArray: + """Build a PyArrow ListArray from a multidimensional NumPy array""" + arr = np.array(arr) + values = pa.array(arr.flatten(), type=type) + for i in range(arr.ndim - 1): + n_offsets = reduce(mul, arr.shape[: arr.ndim - i - 1], 1) + step_offsets = arr.shape[arr.ndim - i - 1] + offsets = pa.array(np.arange(n_offsets + 1) * step_offsets, type=pa.int32()) + values = pa.ListArray.from_arrays(offsets, values) + return values + + +def list_of_pa_arrays_to_pyarrow_listarray(l_arr: List[Optional[pa.Array]]) -> pa.ListArray: + null_mask = np.array([arr is None for arr in l_arr]) + null_indices = np.arange(len(null_mask))[null_mask] - np.arange(np.sum(null_mask)) + l_arr = [arr for arr in l_arr if arr is not None] + offsets = np.cumsum( + [0] + [len(arr) for arr in l_arr], dtype=object + ) # convert to dtype object to allow None insertion + offsets = np.insert(offsets, null_indices, None) + offsets = pa.array(offsets, type=pa.int32()) + values = pa.concat_arrays(l_arr) + return pa.ListArray.from_arrays(offsets, values) + + +def list_of_np_array_to_pyarrow_listarray(l_arr: List[np.ndarray], type: pa.DataType = None) -> pa.ListArray: + """Build a PyArrow ListArray from a possibly nested list of NumPy arrays""" + if len(l_arr) > 0: + return list_of_pa_arrays_to_pyarrow_listarray( + [numpy_to_pyarrow_listarray(arr, type=type) if arr is not None else None for arr in l_arr] + ) + else: + return pa.array([], type=type) + + +def contains_any_np_array(data: Any): + """Return `True` if data is a NumPy ndarray or (recursively) if first non-null value in list is a NumPy ndarray. + + Args: + data (Any): Data. + + Returns: + bool + """ + if isinstance(data, np.ndarray): + return True + elif isinstance(data, list): + return contains_any_np_array(first_non_null_value(data)[1]) + else: + return False + + +def any_np_array_to_pyarrow_listarray(data: Union[np.ndarray, List], type: pa.DataType = None) -> pa.ListArray: + """Convert to PyArrow ListArray either a NumPy ndarray or (recursively) a list that may contain any NumPy ndarray. + + Args: + data (Union[np.ndarray, List]): Data. + type (pa.DataType): Explicit PyArrow DataType passed to coerce the ListArray data type. + + Returns: + pa.ListArray + """ + if isinstance(data, np.ndarray): + return numpy_to_pyarrow_listarray(data, type=type) + elif isinstance(data, list): + return list_of_pa_arrays_to_pyarrow_listarray([any_np_array_to_pyarrow_listarray(i, type=type) for i in data]) + + +def to_pyarrow_listarray(data: Any, pa_type: _ArrayXDExtensionType) -> pa.Array: + """Convert to PyArrow ListArray. + + Args: + data (Any): Sequence, iterable, np.ndarray or pd.Series. + pa_type (_ArrayXDExtensionType): Any of the ArrayNDExtensionType. + + Returns: + pyarrow.Array + """ + if contains_any_np_array(data): + return any_np_array_to_pyarrow_listarray(data, type=pa_type.value_type) + else: + return pa.array(data, pa_type.storage_dtype) + + +def _visit(feature: FeatureType, func: Callable[[FeatureType], Optional[FeatureType]]) -> FeatureType: + """Visit a (possibly nested) feature. + + Args: + feature (FeatureType): the feature type to be checked + Returns: + visited feature (FeatureType) + """ + if isinstance(feature, dict): + out = func({k: _visit(f, func) for k, f in feature.items()}) + elif isinstance(feature, (list, tuple)): + out = func([_visit(feature[0], func)]) + elif isinstance(feature, Sequence): + out = func(Sequence(_visit(feature.feature, func), length=feature.length)) + else: + out = func(feature) + return feature if out is None else out + + +def require_decoding(feature: FeatureType, ignore_decode_attribute: bool = False) -> bool: + """Check if a (possibly nested) feature requires decoding. + + Args: + feature (FeatureType): the feature type to be checked + ignore_decode_attribute (:obj:`bool`, default ``False``): Whether to ignore the current value + of the `decode` attribute of the decodable feature types. + Returns: + :obj:`bool` + """ + if isinstance(feature, dict): + return any(require_decoding(f) for f in feature.values()) + elif isinstance(feature, (list, tuple)): + return require_decoding(feature[0]) + elif isinstance(feature, Sequence): + return require_decoding(feature.feature) + else: + return hasattr(feature, "decode_example") and (feature.decode if not ignore_decode_attribute else True) + + +def require_storage_cast(feature: FeatureType) -> bool: + """Check if a (possibly nested) feature requires storage casting. + + Args: + feature (FeatureType): the feature type to be checked + Returns: + :obj:`bool` + """ + if isinstance(feature, dict): + return any(require_storage_cast(f) for f in feature.values()) + elif isinstance(feature, (list, tuple)): + return require_storage_cast(feature[0]) + elif isinstance(feature, Sequence): + return require_storage_cast(feature.feature) + else: + return hasattr(feature, "cast_storage") + + +def require_storage_embed(feature: FeatureType) -> bool: + """Check if a (possibly nested) feature requires embedding data into storage. + + Args: + feature (FeatureType): the feature type to be checked + Returns: + :obj:`bool` + """ + if isinstance(feature, dict): + return any(require_storage_cast(f) for f in feature.values()) + elif isinstance(feature, (list, tuple)): + return require_storage_cast(feature[0]) + elif isinstance(feature, Sequence): + return require_storage_cast(feature.feature) + else: + return hasattr(feature, "embed_storage") + + +def keep_features_dicts_synced(func): + """ + Wrapper to keep the secondary dictionary, which tracks whether keys are decodable, of the :class:`datasets.Features` object + in sync with the main dictionary. + """ + + @wraps(func) + def wrapper(*args, **kwargs): + if args: + self: "Features" = args[0] + args = args[1:] + else: + self: "Features" = kwargs.pop("self") + out = func(self, *args, **kwargs) + assert hasattr(self, "_column_requires_decoding") + self._column_requires_decoding = {col: require_decoding(feature) for col, feature in self.items()} + return out + + wrapper._decorator_name_ = "_keep_dicts_synced" + return wrapper + + +class Features(dict): + """A special dictionary that defines the internal structure of a dataset. + + Instantiated with a dictionary of type `dict[str, FieldType]`, where keys are the desired column names, + and values are the type of that column. + + `FieldType` can be one of the following: + - a [`~datasets.Value`] feature specifies a single typed value, e.g. `int64` or `string`. + - a [`~datasets.ClassLabel`] feature specifies a field with a predefined set of classes which can have labels + associated to them and will be stored as integers in the dataset. + - a python `dict` which specifies that the field is a nested field containing a mapping of sub-fields to sub-fields + features. It's possible to have nested fields of nested fields in an arbitrary manner. + - a python `list` or a [`~datasets.Sequence`] specifies that the field contains a list of objects. The python + `list` or [`~datasets.Sequence`] should be provided with a single sub-feature as an example of the feature + type hosted in this list. + + + + A [`~datasets.Sequence`] with a internal dictionary feature will be automatically converted into a dictionary of + lists. This behavior is implemented to have a compatilbity layer with the TensorFlow Datasets library but may be + un-wanted in some cases. If you don't want this behavior, you can use a python `list` instead of the + [`~datasets.Sequence`]. + + + + - a [`Array2D`], [`Array3D`], [`Array4D`] or [`Array5D`] feature for multidimensional arrays. + - an [`Audio`] feature to store the absolute path to an audio file or a dictionary with the relative path + to an audio file ("path" key) and its bytes content ("bytes" key). This feature extracts the audio data. + - an [`Image`] feature to store the absolute path to an image file, an `np.ndarray` object, a `PIL.Image.Image` object + or a dictionary with the relative path to an image file ("path" key) and its bytes content ("bytes" key). This feature extracts the image data. + - [`~datasets.Translation`] and [`~datasets.TranslationVariableLanguages`], the two features specific to Machine Translation. + """ + + def __init__(*args, **kwargs): + # self not in the signature to allow passing self as a kwarg + if not args: + raise TypeError("descriptor '__init__' of 'Features' object needs an argument") + self, *args = args + super(Features, self).__init__(*args, **kwargs) + self._column_requires_decoding: Dict[str, bool] = { + col: require_decoding(feature) for col, feature in self.items() + } + + __setitem__ = keep_features_dicts_synced(dict.__setitem__) + __delitem__ = keep_features_dicts_synced(dict.__delitem__) + update = keep_features_dicts_synced(dict.update) + setdefault = keep_features_dicts_synced(dict.setdefault) + pop = keep_features_dicts_synced(dict.pop) + popitem = keep_features_dicts_synced(dict.popitem) + clear = keep_features_dicts_synced(dict.clear) + + def __reduce__(self): + return Features, (dict(self),) + + @property + def type(self): + """ + Features field types. + + Returns: + :obj:`pyarrow.DataType` + """ + return get_nested_type(self) + + @property + def arrow_schema(self): + """ + Features schema. + + Returns: + :obj:`pyarrow.Schema` + """ + hf_metadata = {"info": {"features": self.to_dict()}} + return pa.schema(self.type).with_metadata({"huggingface": json.dumps(hf_metadata)}) + + @classmethod + def from_arrow_schema(cls, pa_schema: pa.Schema) -> "Features": + """ + Construct [`Features`] from Arrow Schema. + It also checks the schema metadata for Hugging Face Datasets features. + Non-nullable fields are not supported and set to nullable. + + Also, pa.dictionary is not supported and it uses its underlying type instead. + Therefore datasets convert DictionaryArray objects to their actual values. + + Args: + pa_schema (`pyarrow.Schema`): + Arrow Schema. + + Returns: + [`Features`] + """ + # try to load features from the arrow schema metadata + metadata_features = Features() + if pa_schema.metadata is not None and "huggingface".encode("utf-8") in pa_schema.metadata: + metadata = json.loads(pa_schema.metadata["huggingface".encode("utf-8")].decode()) + if "info" in metadata and "features" in metadata["info"] and metadata["info"]["features"] is not None: + metadata_features = Features.from_dict(metadata["info"]["features"]) + metadata_features_schema = metadata_features.arrow_schema + obj = { + field.name: ( + metadata_features[field.name] + if field.name in metadata_features and metadata_features_schema.field(field.name) == field + else generate_from_arrow_type(field.type) + ) + for field in pa_schema + } + return cls(**obj) + + @classmethod + def from_dict(cls, dic) -> "Features": + """ + Construct [`Features`] from dict. + + Regenerate the nested feature object from a deserialized dict. + We use the `_type` key to infer the dataclass name of the feature `FieldType`. + + It allows for a convenient constructor syntax + to define features from deserialized JSON dictionaries. This function is used in particular when deserializing + a [`DatasetInfo`] that was dumped to a JSON object. This acts as an analogue to + [`Features.from_arrow_schema`] and handles the recursive field-by-field instantiation, but doesn't require + any mapping to/from pyarrow, except for the fact that it takes advantage of the mapping of pyarrow primitive + dtypes that [`Value`] automatically performs. + + Args: + dic (`dict[str, Any]`): + Python dictionary. + + Returns: + `Features` + + Example:: + >>> Features.from_dict({'_type': {'dtype': 'string', 'id': None, '_type': 'Value'}}) + {'_type': Value(dtype='string', id=None)} + """ + obj = generate_from_dict(dic) + return cls(**obj) + + def to_dict(self): + return asdict(self) + + def _to_yaml_list(self) -> list: + # we compute the YAML list from the dict representation that is used for JSON dump + yaml_data = self.to_dict() + + def simplify(feature: dict) -> dict: + if not isinstance(feature, dict): + raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}") + + # + # sequence: -> sequence: int32 + # dtype: int32 -> + # + if isinstance(feature.get("sequence"), dict) and list(feature["sequence"]) == ["dtype"]: + feature["sequence"] = feature["sequence"]["dtype"] + + # + # sequence: -> sequence: + # struct: -> - name: foo + # - name: foo -> dtype: int32 + # dtype: int32 -> + # + if isinstance(feature.get("sequence"), dict) and list(feature["sequence"]) == ["struct"]: + feature["sequence"] = feature["sequence"]["struct"] + + # + # list: -> list: int32 + # dtype: int32 -> + # + if isinstance(feature.get("list"), dict) and list(feature["list"]) == ["dtype"]: + feature["list"] = feature["list"]["dtype"] + + # + # list: -> list: + # struct: -> - name: foo + # - name: foo -> dtype: int32 + # dtype: int32 -> + # + if isinstance(feature.get("list"), dict) and list(feature["list"]) == ["struct"]: + feature["list"] = feature["list"]["struct"] + + # + # class_label: -> class_label: + # names: -> names: + # - negative -> '0': negative + # - positive -> '1': positive + # + if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), list): + # server-side requirement: keys must be strings + feature["class_label"]["names"] = { + str(label_id): label_name for label_id, label_name in enumerate(feature["class_label"]["names"]) + } + return feature + + def to_yaml_inner(obj: Union[dict, list]) -> dict: + if isinstance(obj, dict): + _type = obj.pop("_type", None) + if _type == "Sequence": + _feature = obj.pop("feature") + return simplify({"sequence": to_yaml_inner(_feature), **obj}) + elif _type == "Value": + return obj + elif _type and not obj: + return {"dtype": camelcase_to_snakecase(_type)} + elif _type: + return {"dtype": simplify({camelcase_to_snakecase(_type): obj})} + else: + return {"struct": [{"name": name, **to_yaml_inner(_feature)} for name, _feature in obj.items()]} + elif isinstance(obj, list): + return simplify({"list": simplify(to_yaml_inner(obj[0]))}) + elif isinstance(obj, tuple): + return to_yaml_inner(list(obj)) + else: + raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}") + + def to_yaml_types(obj: dict) -> dict: + if isinstance(obj, dict): + return {k: to_yaml_types(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [to_yaml_types(v) for v in obj] + elif isinstance(obj, tuple): + return to_yaml_types(list(obj)) + else: + return obj + + return to_yaml_types(to_yaml_inner(yaml_data)["struct"]) + + @classmethod + def _from_yaml_list(cls, yaml_data: list) -> "Features": + yaml_data = copy.deepcopy(yaml_data) + + # we convert the list obtained from YAML data into the dict representation that is used for JSON dump + + def unsimplify(feature: dict) -> dict: + if not isinstance(feature, dict): + raise TypeError(f"Expected a dict but got a {type(feature)}: {feature}") + # + # sequence: int32 -> sequence: + # -> dtype: int32 + # + if isinstance(feature.get("sequence"), str): + feature["sequence"] = {"dtype": feature["sequence"]} + # + # list: int32 -> list: + # -> dtype: int32 + # + if isinstance(feature.get("list"), str): + feature["list"] = {"dtype": feature["list"]} + + # + # class_label: -> class_label: + # names: -> names: + # '0': negative -> - negative + # '1': positive -> - positive + # + if isinstance(feature.get("class_label"), dict) and isinstance(feature["class_label"].get("names"), dict): + label_ids = sorted(feature["class_label"]["names"], key=int) + if label_ids and [int(label_id) for label_id in label_ids] != list(range(int(label_ids[-1]) + 1)): + raise ValueError( + f"ClassLabel expected a value for all label ids [0:{int(label_ids[-1]) + 1}] but some ids are missing." + ) + feature["class_label"]["names"] = [feature["class_label"]["names"][label_id] for label_id in label_ids] + return feature + + def from_yaml_inner(obj: Union[dict, list]) -> Union[dict, list]: + if isinstance(obj, dict): + if not obj: + return {} + _type = next(iter(obj)) + if _type == "sequence": + _feature = unsimplify(obj).pop(_type) + return {"feature": from_yaml_inner(_feature), **obj, "_type": "Sequence"} + if _type == "list": + return [from_yaml_inner(unsimplify(obj)[_type])] + if _type == "struct": + return from_yaml_inner(obj["struct"]) + elif _type == "dtype": + if isinstance(obj["dtype"], str): + # e.g. int32, float64, string, audio, image + try: + Value(obj["dtype"]) + return {**obj, "_type": "Value"} + except ValueError: + # e.g. Audio, Image, ArrayXD + return {"_type": snakecase_to_camelcase(obj["dtype"])} + else: + return from_yaml_inner(obj["dtype"]) + else: + return {"_type": snakecase_to_camelcase(_type), **unsimplify(obj)[_type]} + elif isinstance(obj, list): + names = [_feature.pop("name") for _feature in obj] + return {name: from_yaml_inner(_feature) for name, _feature in zip(names, obj)} + else: + raise TypeError(f"Expected a dict or a list but got {type(obj)}: {obj}") + + return cls.from_dict(from_yaml_inner(yaml_data)) + + def encode_example(self, example): + """ + Encode example into a format for Arrow. + + Args: + example (`dict[str, Any]`): + Data in a Dataset row. + + Returns: + `dict[str, Any]` + """ + example = cast_to_python_objects(example) + return encode_nested_example(self, example) + + def encode_column(self, column, column_name: str): + """ + Encode column into a format for Arrow. + + Args: + column (`list[Any]`): + Data in a Dataset column. + column_name (`str`): + Dataset column name. + + Returns: + `list[Any]` + """ + column = cast_to_python_objects(column) + return [encode_nested_example(self[column_name], obj, level=1) for obj in column] + + def encode_batch(self, batch): + """ + Encode batch into a format for Arrow. + + Args: + batch (`dict[str, list[Any]]`): + Data in a Dataset batch. + + Returns: + `dict[str, list[Any]]` + """ + encoded_batch = {} + if set(batch) != set(self): + raise ValueError(f"Column mismatch between batch {set(batch)} and features {set(self)}") + for key, column in batch.items(): + column = cast_to_python_objects(column) + encoded_batch[key] = [encode_nested_example(self[key], obj, level=1) for obj in column] + return encoded_batch + + def decode_example(self, example: dict, token_per_repo_id: Optional[Dict[str, Union[str, bool, None]]] = None): + """Decode example with custom feature decoding. + + Args: + example (`dict[str, Any]`): + Dataset row data. + token_per_repo_id (`dict`, *optional*): + To access and decode audio or image files from private repositories on the Hub, you can pass + a dictionary `repo_id (str) -> token (bool or str)`. + + Returns: + `dict[str, Any]` + """ + + return { + column_name: decode_nested_example(feature, value, token_per_repo_id=token_per_repo_id) + if self._column_requires_decoding[column_name] + else value + for column_name, (feature, value) in zip_dict( + {key: value for key, value in self.items() if key in example}, example + ) + } + + def decode_column(self, column: list, column_name: str): + """Decode column with custom feature decoding. + + Args: + column (`list[Any]`): + Dataset column data. + column_name (`str`): + Dataset column name. + + Returns: + `list[Any]` + """ + return ( + [decode_nested_example(self[column_name], value) if value is not None else None for value in column] + if self._column_requires_decoding[column_name] + else column + ) + + def decode_batch(self, batch: dict, token_per_repo_id: Optional[Dict[str, Union[str, bool, None]]] = None): + """Decode batch with custom feature decoding. + + Args: + batch (`dict[str, list[Any]]`): + Dataset batch data. + token_per_repo_id (`dict`, *optional*): + To access and decode audio or image files from private repositories on the Hub, you can pass + a dictionary repo_id (str) -> token (bool or str) + + Returns: + `dict[str, list[Any]]` + """ + decoded_batch = {} + for column_name, column in batch.items(): + decoded_batch[column_name] = ( + [ + decode_nested_example(self[column_name], value, token_per_repo_id=token_per_repo_id) + if value is not None + else None + for value in column + ] + if self._column_requires_decoding[column_name] + else column + ) + return decoded_batch + + def copy(self) -> "Features": + """ + Make a deep copy of [`Features`]. + + Returns: + [`Features`] + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("rotten_tomatoes", split="train") + >>> copy_of_features = ds.features.copy() + >>> copy_of_features + {'label': ClassLabel(num_classes=2, names=['neg', 'pos'], id=None), + 'text': Value(dtype='string', id=None)} + ``` + """ + return copy.deepcopy(self) + + def reorder_fields_as(self, other: "Features") -> "Features": + """ + Reorder Features fields to match the field order of other [`Features`]. + + The order of the fields is important since it matters for the underlying arrow data. + Re-ordering the fields allows to make the underlying arrow data type match. + + Args: + other ([`Features`]): + The other [`Features`] to align with. + + Returns: + [`Features`] + + Example:: + + >>> from datasets import Features, Sequence, Value + >>> # let's say we have to features with a different order of nested fields (for a and b for example) + >>> f1 = Features({"root": Sequence({"a": Value("string"), "b": Value("string")})}) + >>> f2 = Features({"root": {"b": Sequence(Value("string")), "a": Sequence(Value("string"))}}) + >>> assert f1.type != f2.type + >>> # re-ordering keeps the base structure (here Sequence is defined at the root level), but make the fields order match + >>> f1.reorder_fields_as(f2) + {'root': Sequence(feature={'b': Value(dtype='string', id=None), 'a': Value(dtype='string', id=None)}, length=-1, id=None)} + >>> assert f1.reorder_fields_as(f2).type == f2.type + """ + + def recursive_reorder(source, target, stack=""): + stack_position = " at " + stack[1:] if stack else "" + if isinstance(target, Sequence): + target = target.feature + if isinstance(target, dict): + target = {k: [v] for k, v in target.items()} + else: + target = [target] + if isinstance(source, Sequence): + source, id_, length = source.feature, source.id, source.length + if isinstance(source, dict): + source = {k: [v] for k, v in source.items()} + reordered = recursive_reorder(source, target, stack) + return Sequence({k: v[0] for k, v in reordered.items()}, id=id_, length=length) + else: + source = [source] + reordered = recursive_reorder(source, target, stack) + return Sequence(reordered[0], id=id_, length=length) + elif isinstance(source, dict): + if not isinstance(target, dict): + raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position) + if sorted(source) != sorted(target): + message = ( + f"Keys mismatch: between {source} (source) and {target} (target).\n" + f"{source.keys()-target.keys()} are missing from target " + f"and {target.keys()-source.keys()} are missing from source" + stack_position + ) + raise ValueError(message) + return {key: recursive_reorder(source[key], target[key], stack + f".{key}") for key in target} + elif isinstance(source, list): + if not isinstance(target, list): + raise ValueError(f"Type mismatch: between {source} and {target}" + stack_position) + if len(source) != len(target): + raise ValueError(f"Length mismatch: between {source} and {target}" + stack_position) + return [recursive_reorder(source[i], target[i], stack + ".") for i in range(len(target))] + else: + return source + + return Features(recursive_reorder(self, other)) + + def flatten(self, max_depth=16) -> "Features": + """Flatten the features. Every dictionary column is removed and is replaced by + all the subfields it contains. The new fields are named by concatenating the + name of the original column and the subfield name like this: `.`. + + If a column contains nested dictionaries, then all the lower-level subfields names are + also concatenated to form new columns: `..`, etc. + + Returns: + [`Features`]: + The flattened features. + + Example: + + ```py + >>> from datasets import load_dataset + >>> ds = load_dataset("squad", split="train") + >>> ds.features.flatten() + {'answers.answer_start': Sequence(feature=Value(dtype='int32', id=None), length=-1, id=None), + 'answers.text': Sequence(feature=Value(dtype='string', id=None), length=-1, id=None), + 'context': Value(dtype='string', id=None), + 'id': Value(dtype='string', id=None), + 'question': Value(dtype='string', id=None), + 'title': Value(dtype='string', id=None)} + ``` + """ + for depth in range(1, max_depth): + no_change = True + flattened = self.copy() + for column_name, subfeature in self.items(): + if isinstance(subfeature, dict): + no_change = False + flattened.update({f"{column_name}.{k}": v for k, v in subfeature.items()}) + del flattened[column_name] + elif isinstance(subfeature, Sequence) and isinstance(subfeature.feature, dict): + no_change = False + flattened.update( + { + f"{column_name}.{k}": Sequence(v) if not isinstance(v, dict) else [v] + for k, v in subfeature.feature.items() + } + ) + del flattened[column_name] + elif hasattr(subfeature, "flatten") and subfeature.flatten() != subfeature: + no_change = False + flattened.update({f"{column_name}.{k}": v for k, v in subfeature.flatten().items()}) + del flattened[column_name] + self = flattened + if no_change: + break + return self + + +def _align_features(features_list: List[Features]) -> List[Features]: + """Align dictionaries of features so that the keys that are found in multiple dictionaries share the same feature.""" + name2feature = {} + for features in features_list: + for k, v in features.items(): + if k in name2feature and isinstance(v, dict): + # Recursively align features. + name2feature[k] = _align_features([name2feature[k], v])[0] + elif k not in name2feature or (isinstance(name2feature[k], Value) and name2feature[k].dtype == "null"): + name2feature[k] = v + + return [Features({k: name2feature[k] for k in features.keys()}) for features in features_list] + + +def _check_if_features_can_be_aligned(features_list: List[Features]): + """Check if the dictionaries of features can be aligned. + + Two dictonaries of features can be aligned if the keys they share have the same type or some of them is of type `Value("null")`. + """ + name2feature = {} + for features in features_list: + for k, v in features.items(): + if k not in name2feature or (isinstance(name2feature[k], Value) and name2feature[k].dtype == "null"): + name2feature[k] = v + + for features in features_list: + for k, v in features.items(): + if isinstance(v, dict) and isinstance(name2feature[k], dict): + # Deep checks for structure. + _check_if_features_can_be_aligned([name2feature[k], v]) + elif not (isinstance(v, Value) and v.dtype == "null") and name2feature[k] != v: + raise ValueError( + f'The features can\'t be aligned because the key {k} of features {features} has unexpected type - {v} (expected either {name2feature[k]} or Value("null").' + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/image.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/image.py new file mode 100644 index 0000000000000000000000000000000000000000..c63d4d439641a41d592c38a79343195e2beb591e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/image.py @@ -0,0 +1,383 @@ +import os +import sys +import warnings +from dataclasses import dataclass, field +from io import BytesIO +from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union + +import numpy as np +import pyarrow as pa + +from .. import config +from ..download.download_config import DownloadConfig +from ..table import array_cast +from ..utils.file_utils import is_local_path, xopen +from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict + + +if TYPE_CHECKING: + import PIL.Image + + from .features import FeatureType + + +_IMAGE_COMPRESSION_FORMATS: Optional[List[str]] = None +_NATIVE_BYTEORDER = "<" if sys.byteorder == "little" else ">" +# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image +_VALID_IMAGE_ARRAY_DTPYES = [ + np.dtype("|b1"), + np.dtype("|u1"), + np.dtype("u2"), + np.dtype("i2"), + np.dtype("u4"), + np.dtype("i4"), + np.dtype("f4"), + np.dtype("f8"), +] + + +@dataclass +class Image: + """Image [`Feature`] to read image data from an image file. + + Input: The Image feature accepts as input: + - A `str`: Absolute path to the image file (i.e. random access is allowed). + - A `dict` with the keys: + + - `path`: String with relative path of the image file to the archive file. + - `bytes`: Bytes of the image file. + + This is useful for archived files with sequential access. + + - An `np.ndarray`: NumPy array representing an image. + - A `PIL.Image.Image`: PIL image object. + + Args: + mode (`str`, *optional*): + The mode to convert the image to. If `None`, the native mode of the image is used. + decode (`bool`, defaults to `True`): + Whether to decode the image data. If `False`, + returns the underlying dictionary in the format `{"path": image_path, "bytes": image_bytes}`. + + Examples: + + ```py + >>> from datasets import load_dataset, Image + >>> ds = load_dataset("beans", split="train") + >>> ds.features["image"] + Image(decode=True, id=None) + >>> ds[0]["image"] + + >>> ds = ds.cast_column('image', Image(decode=False)) + {'bytes': None, + 'path': '/root/.cache/huggingface/datasets/downloads/extracted/b0a21163f78769a2cf11f58dfc767fb458fc7cea5c05dccc0144a2c0f0bc1292/train/healthy/healthy_train.85.jpg'} + ``` + """ + + mode: Optional[str] = None + decode: bool = True + id: Optional[str] = None + # Automatically constructed + dtype: ClassVar[str] = "PIL.Image.Image" + pa_type: ClassVar[Any] = pa.struct({"bytes": pa.binary(), "path": pa.string()}) + _type: str = field(default="Image", init=False, repr=False) + + def __call__(self): + return self.pa_type + + def encode_example(self, value: Union[str, bytes, dict, np.ndarray, "PIL.Image.Image"]) -> dict: + """Encode example into a format for Arrow. + + Args: + value (`str`, `np.ndarray`, `PIL.Image.Image` or `dict`): + Data passed as input to Image feature. + + Returns: + `dict` with "path" and "bytes" fields + """ + if config.PIL_AVAILABLE: + import PIL.Image + else: + raise ImportError("To support encoding images, please install 'Pillow'.") + + if isinstance(value, list): + value = np.array(value) + + if isinstance(value, str): + return {"path": value, "bytes": None} + elif isinstance(value, bytes): + return {"path": None, "bytes": value} + elif isinstance(value, np.ndarray): + # convert the image array to PNG/TIFF bytes + return encode_np_array(value) + elif isinstance(value, PIL.Image.Image): + # convert the PIL image to bytes (default format is PNG/TIFF) + return encode_pil_image(value) + elif value.get("path") is not None and os.path.isfile(value["path"]): + # we set "bytes": None to not duplicate the data if they're already available locally + return {"bytes": None, "path": value.get("path")} + elif value.get("bytes") is not None or value.get("path") is not None: + # store the image bytes, and path is used to infer the image format using the file extension + return {"bytes": value.get("bytes"), "path": value.get("path")} + else: + raise ValueError( + f"An image sample should have one of 'path' or 'bytes' but they are missing or None in {value}." + ) + + def decode_example(self, value: dict, token_per_repo_id=None) -> "PIL.Image.Image": + """Decode example image file into image data. + + Args: + value (`str` or `dict`): + A string with the absolute image file path, a dictionary with + keys: + + - `path`: String with absolute or relative image file path. + - `bytes`: The bytes of the image file. + token_per_repo_id (`dict`, *optional*): + To access and decode + image files from private repositories on the Hub, you can pass + a dictionary repo_id (`str`) -> token (`bool` or `str`). + + Returns: + `PIL.Image.Image` + """ + if not self.decode: + raise RuntimeError("Decoding is disabled for this feature. Please use Image(decode=True) instead.") + + if config.PIL_AVAILABLE: + import PIL.Image + import PIL.ImageOps + else: + raise ImportError("To support decoding images, please install 'Pillow'.") + + if token_per_repo_id is None: + token_per_repo_id = {} + + path, bytes_ = value["path"], value["bytes"] + if bytes_ is None: + if path is None: + raise ValueError(f"An image should have one of 'path' or 'bytes' but both are None in {value}.") + else: + if is_local_path(path): + image = PIL.Image.open(path) + else: + source_url = path.split("::")[-1] + pattern = ( + config.HUB_DATASETS_URL + if source_url.startswith(config.HF_ENDPOINT) + else config.HUB_DATASETS_HFFS_URL + ) + try: + repo_id = string_to_dict(source_url, pattern)["repo_id"] + token = token_per_repo_id.get(repo_id) + except ValueError: + token = None + download_config = DownloadConfig(token=token) + with xopen(path, "rb", download_config=download_config) as f: + bytes_ = BytesIO(f.read()) + image = PIL.Image.open(bytes_) + else: + image = PIL.Image.open(BytesIO(bytes_)) + image.load() # to avoid "Too many open files" errors + if image.getexif().get(PIL.Image.ExifTags.Base.Orientation) is not None: + image = PIL.ImageOps.exif_transpose(image) + if self.mode and self.mode != image.mode: + image = image.convert(self.mode) + return image + + def flatten(self) -> Union["FeatureType", Dict[str, "FeatureType"]]: + """If in the decodable state, return the feature itself, otherwise flatten the feature into a dictionary.""" + from .features import Value + + return ( + self + if self.decode + else { + "bytes": Value("binary"), + "path": Value("string"), + } + ) + + def cast_storage(self, storage: Union[pa.StringArray, pa.StructArray, pa.ListArray]) -> pa.StructArray: + """Cast an Arrow array to the Image arrow storage type. + The Arrow types that can be converted to the Image pyarrow storage type are: + + - `pa.string()` - it must contain the "path" data + - `pa.binary()` - it must contain the image bytes + - `pa.struct({"bytes": pa.binary()})` + - `pa.struct({"path": pa.string()})` + - `pa.struct({"bytes": pa.binary(), "path": pa.string()})` - order doesn't matter + - `pa.list(*)` - it must contain the image array data + + Args: + storage (`Union[pa.StringArray, pa.StructArray, pa.ListArray]`): + PyArrow array to cast. + + Returns: + `pa.StructArray`: Array in the Image arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + if pa.types.is_string(storage.type): + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + storage = pa.StructArray.from_arrays([bytes_array, storage], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_binary(storage.type): + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([storage, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_struct(storage.type): + if storage.type.get_field_index("bytes") >= 0: + bytes_array = storage.field("bytes") + else: + bytes_array = pa.array([None] * len(storage), type=pa.binary()) + if storage.type.get_field_index("path") >= 0: + path_array = storage.field("path") + else: + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=storage.is_null()) + elif pa.types.is_list(storage.type): + bytes_array = pa.array( + [encode_np_array(np.array(arr))["bytes"] if arr is not None else None for arr in storage.to_pylist()], + type=pa.binary(), + ) + path_array = pa.array([None] * len(storage), type=pa.string()) + storage = pa.StructArray.from_arrays( + [bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null() + ) + return array_cast(storage, self.pa_type) + + def embed_storage(self, storage: pa.StructArray) -> pa.StructArray: + """Embed image files into the Arrow array. + + Args: + storage (`pa.StructArray`): + PyArrow array to embed. + + Returns: + `pa.StructArray`: Array in the Image arrow storage type, that is + `pa.struct({"bytes": pa.binary(), "path": pa.string()})`. + """ + + @no_op_if_value_is_null + def path_to_bytes(path): + with xopen(path, "rb") as f: + bytes_ = f.read() + return bytes_ + + bytes_array = pa.array( + [ + (path_to_bytes(x["path"]) if x["bytes"] is None else x["bytes"]) if x is not None else None + for x in storage.to_pylist() + ], + type=pa.binary(), + ) + path_array = pa.array( + [os.path.basename(path) if path is not None else None for path in storage.field("path").to_pylist()], + type=pa.string(), + ) + storage = pa.StructArray.from_arrays([bytes_array, path_array], ["bytes", "path"], mask=bytes_array.is_null()) + return array_cast(storage, self.pa_type) + + +def list_image_compression_formats() -> List[str]: + if config.PIL_AVAILABLE: + import PIL.Image + else: + raise ImportError("To support encoding images, please install 'Pillow'.") + + global _IMAGE_COMPRESSION_FORMATS + if _IMAGE_COMPRESSION_FORMATS is None: + PIL.Image.init() + _IMAGE_COMPRESSION_FORMATS = list(set(PIL.Image.OPEN.keys()) & set(PIL.Image.SAVE.keys())) + return _IMAGE_COMPRESSION_FORMATS + + +def image_to_bytes(image: "PIL.Image.Image") -> bytes: + """Convert a PIL Image object to bytes using native compression if possible, otherwise use PNG/TIFF compression.""" + buffer = BytesIO() + if image.format in list_image_compression_formats(): + format = image.format + else: + format = "PNG" if image.mode in ["1", "L", "LA", "RGB", "RGBA"] else "TIFF" + image.save(buffer, format=format) + return buffer.getvalue() + + +def encode_pil_image(image: "PIL.Image.Image") -> dict: + if hasattr(image, "filename") and image.filename != "": + return {"path": image.filename, "bytes": None} + else: + return {"path": None, "bytes": image_to_bytes(image)} + + +def encode_np_array(array: np.ndarray) -> dict: + if config.PIL_AVAILABLE: + import PIL.Image + else: + raise ImportError("To support encoding images, please install 'Pillow'.") + + dtype = array.dtype + dtype_byteorder = dtype.byteorder if dtype.byteorder != "=" else _NATIVE_BYTEORDER + dtype_kind = dtype.kind + dtype_itemsize = dtype.itemsize + + dest_dtype = None + + # Multi-channel array case (only np.dtype("|u1") is allowed) + if array.shape[2:]: + if dtype_kind not in ["u", "i"]: + raise TypeError( + f"Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays." + ) + dest_dtype = np.dtype("|u1") + if dtype != dest_dtype: + warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'") + # Exact match + elif dtype in _VALID_IMAGE_ARRAY_DTPYES: + dest_dtype = dtype + else: # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually) + while dtype_itemsize >= 1: + dtype_str = dtype_byteorder + dtype_kind + str(dtype_itemsize) + if np.dtype(dtype_str) in _VALID_IMAGE_ARRAY_DTPYES: + dest_dtype = np.dtype(dtype_str) + warnings.warn(f"Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'") + break + else: + dtype_itemsize //= 2 + if dest_dtype is None: + raise TypeError( + f"Cannot downcast dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}" + ) + + image = PIL.Image.fromarray(array.astype(dest_dtype)) + return {"path": None, "bytes": image_to_bytes(image)} + + +def objects_to_list_of_image_dicts( + objs: Union[List[str], List[dict], List[np.ndarray], List["PIL.Image.Image"]], +) -> List[dict]: + """Encode a list of objects into a format suitable for creating an extension array of type `ImageExtensionType`.""" + if config.PIL_AVAILABLE: + import PIL.Image + else: + raise ImportError("To support encoding images, please install 'Pillow'.") + + if objs: + _, obj = first_non_null_value(objs) + if isinstance(obj, str): + return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs] + if isinstance(obj, np.ndarray): + obj_to_image_dict_func = no_op_if_value_is_null(encode_np_array) + return [obj_to_image_dict_func(obj) for obj in objs] + elif isinstance(obj, PIL.Image.Image): + obj_to_image_dict_func = no_op_if_value_is_null(encode_pil_image) + return [obj_to_image_dict_func(obj) for obj in objs] + else: + return objs + else: + return objs diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/translation.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/translation.py new file mode 100644 index 0000000000000000000000000000000000000000..1d3eb1af4bbb15397afe4f1e0a5afd54060fcda3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/features/translation.py @@ -0,0 +1,129 @@ +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union + +import pyarrow as pa + + +if TYPE_CHECKING: + from .features import FeatureType + + +@dataclass +class Translation: + """`FeatureConnector` for translations with fixed languages per example. + Here for compatiblity with tfds. + + Args: + languages (`dict`): + A dictionary for each example mapping string language codes to string translations. + + Example: + + ```python + >>> # At construction time: + >>> datasets.features.Translation(languages=['en', 'fr', 'de']) + >>> # During data generation: + >>> yield { + ... 'en': 'the cat', + ... 'fr': 'le chat', + ... 'de': 'die katze' + ... } + ``` + """ + + languages: List[str] + id: Optional[str] = None + # Automatically constructed + dtype: ClassVar[str] = "dict" + pa_type: ClassVar[Any] = None + _type: str = field(default="Translation", init=False, repr=False) + + def __call__(self): + return pa.struct({lang: pa.string() for lang in sorted(self.languages)}) + + def flatten(self) -> Union["FeatureType", Dict[str, "FeatureType"]]: + """Flatten the Translation feature into a dictionary.""" + from .features import Value + + return {k: Value("string") for k in sorted(self.languages)} + + +@dataclass +class TranslationVariableLanguages: + """`FeatureConnector` for translations with variable languages per example. + Here for compatiblity with tfds. + + Args: + languages (`dict`): + A dictionary for each example mapping string language codes to one or more string translations. + The languages present may vary from example to example. + + Returns: + - `language` or `translation` (variable-length 1D `tf.Tensor` of `tf.string`): + Language codes sorted in ascending order or plain text translations, sorted to align with language codes. + + Example: + + ```python + >>> # At construction time: + >>> datasets.features.TranslationVariableLanguages(languages=['en', 'fr', 'de']) + >>> # During data generation: + >>> yield { + ... 'en': 'the cat', + ... 'fr': ['le chat', 'la chatte,'] + ... 'de': 'die katze' + ... } + >>> # Tensor returned : + >>> { + ... 'language': ['en', 'de', 'fr', 'fr'], + ... 'translation': ['the cat', 'die katze', 'la chatte', 'le chat'], + ... } + ``` + """ + + languages: Optional[List] = None + num_languages: Optional[int] = None + id: Optional[str] = None + # Automatically constructed + dtype: ClassVar[str] = "dict" + pa_type: ClassVar[Any] = None + _type: str = field(default="TranslationVariableLanguages", init=False, repr=False) + + def __post_init__(self): + self.languages = sorted(set(self.languages)) if self.languages else None + self.num_languages = len(self.languages) if self.languages else None + + def __call__(self): + return pa.struct({"language": pa.list_(pa.string()), "translation": pa.list_(pa.string())}) + + def encode_example(self, translation_dict): + lang_set = set(self.languages) + if set(translation_dict) == {"language", "translation"}: + return translation_dict + elif self.languages and set(translation_dict) - lang_set: + raise ValueError( + f'Some languages in example ({", ".join(sorted(set(translation_dict) - lang_set))}) are not in valid set ({", ".join(lang_set)}).' + ) + + # Convert dictionary into tuples, splitting out cases where there are + # multiple translations for a single language. + translation_tuples = [] + for lang, text in translation_dict.items(): + if isinstance(text, str): + translation_tuples.append((lang, text)) + else: + translation_tuples.extend([(lang, el) for el in text]) + + # Ensure translations are in ascending order by language code. + languages, translations = zip(*sorted(translation_tuples)) + + return {"language": languages, "translation": translations} + + def flatten(self) -> Union["FeatureType", Dict[str, "FeatureType"]]: + """Flatten the TranslationVariableLanguages feature into a dictionary.""" + from .features import Sequence, Value + + return { + "language": Sequence(Value("string")), + "translation": Sequence(Value("string")), + } diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5c48c7069ce62e781527ba1edd641a1dc6f9c3d8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/__init__.py @@ -0,0 +1,69 @@ +import importlib +import shutil +import warnings +from typing import List + +import fsspec +import fsspec.asyn +from fsspec.implementations.local import LocalFileSystem + +from ..utils.deprecation_utils import deprecated +from . import compression + + +_has_s3fs = importlib.util.find_spec("s3fs") is not None + +if _has_s3fs: + from .s3filesystem import S3FileSystem + +COMPRESSION_FILESYSTEMS: List[compression.BaseCompressedFileFileSystem] = [ + compression.Bz2FileSystem, + compression.GzipFileSystem, + compression.Lz4FileSystem, + compression.XzFileSystem, + compression.ZstdFileSystem, +] + +# Register custom filesystems +for fs_class in COMPRESSION_FILESYSTEMS: + if fs_class.protocol in fsspec.registry and fsspec.registry[fs_class.protocol] is not fs_class: + warnings.warn(f"A filesystem protocol was already set for {fs_class.protocol} and will be overwritten.") + fsspec.register_implementation(fs_class.protocol, fs_class, clobber=True) + + +@deprecated( + "This function is deprecated and will be removed in a future version. Please use `fsspec.core.strip_protocol` instead." +) +def extract_path_from_uri(dataset_path: str) -> str: + """ + Preprocesses `dataset_path` and removes remote filesystem (e.g. removing `s3://`). + + Args: + dataset_path (`str`): + Path (e.g. `dataset/train`) or remote uri (e.g. `s3://my-bucket/dataset/train`) of the dataset directory. + """ + if "://" in dataset_path: + dataset_path = dataset_path.split("://")[1] + return dataset_path + + +def is_remote_filesystem(fs: fsspec.AbstractFileSystem) -> bool: + """ + Checks if `fs` is a remote filesystem. + + Args: + fs (`fsspec.spec.AbstractFileSystem`): + An abstract super-class for pythonic file-systems, e.g. `fsspec.filesystem(\'file\')` or [`datasets.filesystems.S3FileSystem`]. + """ + return not isinstance(fs, LocalFileSystem) + + +def rename(fs: fsspec.AbstractFileSystem, src: str, dst: str): + """ + Renames the file `src` in `fs` to `dst`. + """ + if not is_remote_filesystem(fs): + # LocalFileSystem.mv does copy + rm, it is more efficient to simply move a local directory + shutil.move(fs._strip_protocol(src), fs._strip_protocol(dst)) + else: + fs.mv(src, dst, recursive=True) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5fe397f1bf180d2ed625912ddb417ca227adae4c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/__pycache__/compression.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/__pycache__/compression.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..104e4f1239dc88992225fb6ae0c333089821836b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/__pycache__/compression.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/__pycache__/s3filesystem.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/__pycache__/s3filesystem.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..60c3f6d57e9777652cd291bcf69846f6a765d553 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/__pycache__/s3filesystem.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/compression.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/compression.py new file mode 100644 index 0000000000000000000000000000000000000000..416d5b15f822dcfbaf0d37cac42ee61d3c0734d2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/compression.py @@ -0,0 +1,127 @@ +import os +from functools import partial +from typing import Optional + +import fsspec +from fsspec.archive import AbstractArchiveFileSystem + + +class BaseCompressedFileFileSystem(AbstractArchiveFileSystem): + """Read contents of compressed file as a filesystem with one file inside.""" + + root_marker = "" + protocol: str = ( + None # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz + ) + compression: str = None # compression type in fsspec. ex: "gzip" + extension: str = None # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz + + def __init__( + self, fo: str = "", target_protocol: Optional[str] = None, target_options: Optional[dict] = None, **kwargs + ): + """ + The compressed file system can be instantiated from any compressed file. + It reads the contents of compressed file as a filesystem with one file inside, as if it was an archive. + + The single file inside the filesystem is named after the compresssed file, + without the compression extension at the end of the filename. + + Args: + fo (:obj:``str``): Path to compressed file. Will fetch file using ``fsspec.open()`` + mode (:obj:``str``): Currently, only 'rb' accepted + target_protocol(:obj:``str``, optional): To override the FS protocol inferred from a URL. + target_options (:obj:``dict``, optional): Kwargs passed when instantiating the target FS. + """ + super().__init__(self, **kwargs) + self.fo = fo.__fspath__() if hasattr(fo, "__fspath__") else fo + # always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode + self._open_with_fsspec = partial( + fsspec.open, + self.fo, + mode="rb", + protocol=target_protocol, + compression=self.compression, + client_kwargs={ + "requote_redirect_url": False, # see https://github.com/huggingface/datasets/pull/5459 + "trust_env": True, # Enable reading proxy env variables. + **(target_options or {}).pop("client_kwargs", {}), # To avoid issues if it was already passed. + }, + **(target_options or {}), + ) + self.compressed_name = os.path.basename(self.fo.split("::")[0]) + self.uncompressed_name = ( + self.compressed_name[: self.compressed_name.rindex(".")] + if "." in self.compressed_name + else self.compressed_name + ) + self.dir_cache = None + + @classmethod + def _strip_protocol(cls, path): + # compressed file paths are always relative to the archive root + return super()._strip_protocol(path).lstrip("/") + + def _get_dirs(self): + if self.dir_cache is None: + f = {**self._open_with_fsspec().fs.info(self.fo), "name": self.uncompressed_name} + self.dir_cache = {f["name"]: f} + + def cat(self, path: str): + with self._open_with_fsspec().open() as f: + return f.read() + + def _open( + self, + path: str, + mode: str = "rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + path = self._strip_protocol(path) + if mode != "rb": + raise ValueError(f"Tried to read with mode {mode} on file {self.fo} opened with mode 'rb'") + return self._open_with_fsspec().open() + + +class Bz2FileSystem(BaseCompressedFileFileSystem): + """Read contents of BZ2 file as a filesystem with one file inside.""" + + protocol = "bz2" + compression = "bz2" + extension = ".bz2" + + +class GzipFileSystem(BaseCompressedFileFileSystem): + """Read contents of GZIP file as a filesystem with one file inside.""" + + protocol = "gzip" + compression = "gzip" + extension = ".gz" + + +class Lz4FileSystem(BaseCompressedFileFileSystem): + """Read contents of LZ4 file as a filesystem with one file inside.""" + + protocol = "lz4" + compression = "lz4" + extension = ".lz4" + + +class XzFileSystem(BaseCompressedFileFileSystem): + """Read contents of .xz (LZMA) file as a filesystem with one file inside.""" + + protocol = "xz" + compression = "xz" + extension = ".xz" + + +class ZstdFileSystem(BaseCompressedFileFileSystem): + """ + Read contents of .zstd file as a filesystem with one file inside. + """ + + protocol = "zstd" + compression = "zstd" + extension = ".zst" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/s3filesystem.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/s3filesystem.py new file mode 100644 index 0000000000000000000000000000000000000000..4d204f1f8738e51411cacac0201fd67e5c185422 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/filesystems/s3filesystem.py @@ -0,0 +1,116 @@ +import s3fs + +from ..utils.deprecation_utils import deprecated + + +@deprecated("Use s3fs.S3FileSystem instead.") +class S3FileSystem(s3fs.S3FileSystem): + """ + `datasets.filesystems.S3FileSystem` is a subclass of [`s3fs.S3FileSystem`](https://s3fs.readthedocs.io/en/latest/api.html). + + Users can use this class to access S3 as if it were a file system. It exposes a filesystem-like API (ls, cp, open, etc.) on top of S3 storage. Provide credentials either explicitly (`key=`, `secret=`) or with boto's credential methods. See botocore documentation for more information. If no credentials are available, use `anon=True`. + + Args: + anon (`bool`, default to `False`): + Whether to use anonymous connection (public buckets only). If `False`, uses the key/secret given, + or boto's credential resolver (client_kwargs, environment, variables, config files, EC2 IAM server, in that order). + key (`str`): + If not anonymous, use this access key ID, if specified. + secret (`str`): + If not anonymous, use this secret access key, if specified. + token (`str`): + If not anonymous, use this security token, if specified. + use_ssl (`bool`, defaults to `True`): + Whether to use SSL in connections to S3; may be faster without, but insecure. If `use_ssl` is + also set in `client_kwargs`, the value set in `client_kwargs` will take priority. + s3_additional_kwargs (`dict`): + Parameters that are used when calling S3 API methods. Typically used for things + like ServerSideEncryption. + client_kwargs (`dict`): + Parameters for the botocore client. + requester_pays (`bool`, defaults to `False`): + Whether `RequesterPays` buckets are supported. + default_block_size (`int`): + If given, the default block size value used for `open()`, if no specific value is given at all time. + The built-in default is 5MB. + default_fill_cache (`bool`, defaults to `True`): + Whether to use cache filling with open by default. Refer to `S3File.open`. + default_cache_type (`str`, defaults to `bytes`): + If given, the default `cache_type` value used for `open()`. Set to `none` if no + caching is desired. See fsspec's documentation for other available `cache_type` values. + version_aware (`bool`, defaults to `False`): + Whether to support bucket versioning. If enable this will require the user to have + the necessary IAM permissions for dealing with versioned objects. + cache_regions (`bool`, defaults to `False`): + Whether to cache bucket regions. Whenever a new bucket is used, it will + first find out which region it belongs to and then use the client for that region. + asynchronous (`bool`, defaults to `False`): + Whether this instance is to be used from inside coroutines. + config_kwargs (`dict`): + Parameters passed to `botocore.client.Config`. + **kwargs: + Other parameters for core session. + session (`aiobotocore.session.AioSession`): + Session to be used for all connections. This session will be used inplace of creating + a new session inside S3FileSystem. For example: `aiobotocore.session.AioSession(profile='test_user')`. + skip_instance_cache (`bool`): + Control reuse of instances. Passed on to `fsspec`. + use_listings_cache (`bool`): + Control reuse of directory listings. Passed on to `fsspec`. + listings_expiry_time (`int` or `float`): + Control reuse of directory listings. Passed on to `fsspec`. + max_paths (`int`): Control reuse of directory listings. Passed on to `fsspec`. + + Examples: + + Listing files from public S3 bucket. + + ```py + >>> import datasets + >>> s3 = datasets.filesystems.S3FileSystem(anon=True) # doctest: +SKIP + >>> s3.ls('public-datasets/imdb/train') # doctest: +SKIP + ['dataset_info.json.json','dataset.arrow','state.json'] + ``` + + Listing files from private S3 bucket using `aws_access_key_id` and `aws_secret_access_key`. + + ```py + >>> import datasets + >>> s3 = datasets.filesystems.S3FileSystem(key=aws_access_key_id, secret=aws_secret_access_key) # doctest: +SKIP + >>> s3.ls('my-private-datasets/imdb/train') # doctest: +SKIP + ['dataset_info.json.json','dataset.arrow','state.json'] + ``` + + Using `S3Filesystem` with `botocore.session.Session` and custom `aws_profile`. + + ```py + >>> import botocore + >>> from datasets.filesystems import S3Filesystem + + >>> s3_session = botocore.session.Session(profile_name='my_profile_name') + >>> s3 = S3FileSystem(session=s3_session) # doctest: +SKIP + ``` + + Loading dataset from S3 using `S3Filesystem` and [`load_from_disk`]. + + ```py + >>> from datasets import load_from_disk + >>> from datasets.filesystems import S3Filesystem + + >>> s3 = S3FileSystem(key=aws_access_key_id, secret=aws_secret_access_key) # doctest: +SKIP + >>> dataset = load_from_disk('s3://my-private-datasets/imdb/train', storage_options=s3.storage_options) # doctest: +SKIP + >>> print(len(dataset)) + 25000 + ``` + + Saving dataset to S3 using `S3Filesystem` and [`Dataset.save_to_disk`]. + + ```py + >>> from datasets import load_dataset + >>> from datasets.filesystems import S3Filesystem + + >>> dataset = load_dataset("imdb") + >>> s3 = S3FileSystem(key=aws_access_key_id, secret=aws_secret_access_key) # doctest: +SKIP + >>> dataset.save_to_disk('s3://my-private-datasets/imdb/train', storage_options=s3.storage_options) # doctest: +SKIP + ``` + """ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8aa21d37bd2e5188f2ceb4235dc14c7dc3af2453 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__init__.py @@ -0,0 +1,135 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List, Optional, Type + +from .. import config +from ..utils import logging +from .formatting import ( + ArrowFormatter, + CustomFormatter, + Formatter, + PandasFormatter, + PythonFormatter, + TensorFormatter, + format_table, + query_table, +) +from .np_formatter import NumpyFormatter + + +logger = logging.get_logger(__name__) + +_FORMAT_TYPES: Dict[Optional[str], Type[Formatter]] = {} +_FORMAT_TYPES_ALIASES: Dict[Optional[str], str] = {} +_FORMAT_TYPES_ALIASES_UNAVAILABLE: Dict[Optional[str], Exception] = {} + + +def _register_formatter( + formatter_cls: type, + format_type: Optional[str], + aliases: Optional[List[str]] = None, +): + """ + Register a Formatter object using a name and optional aliases. + This function must be used on a Formatter class. + """ + aliases = aliases if aliases is not None else [] + if format_type in _FORMAT_TYPES: + logger.warning( + f"Overwriting format type '{format_type}' ({_FORMAT_TYPES[format_type].__name__} -> {formatter_cls.__name__})" + ) + _FORMAT_TYPES[format_type] = formatter_cls + for alias in set(aliases + [format_type]): + if alias in _FORMAT_TYPES_ALIASES: + logger.warning( + f"Overwriting format type alias '{alias}' ({_FORMAT_TYPES_ALIASES[alias]} -> {format_type})" + ) + _FORMAT_TYPES_ALIASES[alias] = format_type + + +def _register_unavailable_formatter( + unavailable_error: Exception, format_type: Optional[str], aliases: Optional[List[str]] = None +): + """ + Register an unavailable Formatter object using a name and optional aliases. + This function must be used on an Exception object that is raised when trying to get the unavailable formatter. + """ + aliases = aliases if aliases is not None else [] + for alias in set(aliases + [format_type]): + _FORMAT_TYPES_ALIASES_UNAVAILABLE[alias] = unavailable_error + + +# Here we define all the available formatting functions that can be used by `Dataset.set_format` +_register_formatter(PythonFormatter, None, aliases=["python"]) +_register_formatter(ArrowFormatter, "arrow", aliases=["pa", "pyarrow"]) +_register_formatter(NumpyFormatter, "numpy", aliases=["np"]) +_register_formatter(PandasFormatter, "pandas", aliases=["pd"]) +_register_formatter(CustomFormatter, "custom") + +if config.POLARS_AVAILABLE: + from .polars_formatter import PolarsFormatter + + _register_formatter(PolarsFormatter, "polars", aliases=["pl"]) +else: + _polars_error = ValueError("Polars needs to be installed to be able to return Polars dataframes.") + _register_unavailable_formatter(_polars_error, "polars", aliases=["pl"]) + +if config.TORCH_AVAILABLE: + from .torch_formatter import TorchFormatter + + _register_formatter(TorchFormatter, "torch", aliases=["pt", "pytorch"]) +else: + _torch_error = ValueError("PyTorch needs to be installed to be able to return PyTorch tensors.") + _register_unavailable_formatter(_torch_error, "torch", aliases=["pt", "pytorch"]) + +if config.TF_AVAILABLE: + from .tf_formatter import TFFormatter + + _register_formatter(TFFormatter, "tensorflow", aliases=["tf"]) +else: + _tf_error = ValueError("Tensorflow needs to be installed to be able to return Tensorflow tensors.") + _register_unavailable_formatter(_tf_error, "tensorflow", aliases=["tf"]) + +if config.JAX_AVAILABLE: + from .jax_formatter import JaxFormatter + + _register_formatter(JaxFormatter, "jax", aliases=[]) +else: + _jax_error = ValueError("JAX needs to be installed to be able to return JAX arrays.") + _register_unavailable_formatter(_jax_error, "jax", aliases=[]) + + +def get_format_type_from_alias(format_type: Optional[str]) -> Optional[str]: + """If the given format type is a known alias, then return its main type name. Otherwise return the type with no change.""" + if format_type in _FORMAT_TYPES_ALIASES: + return _FORMAT_TYPES_ALIASES[format_type] + else: + return format_type + + +def get_formatter(format_type: Optional[str], **format_kwargs) -> Formatter: + """ + Factory function to get a Formatter given its type name and keyword arguments. + A formatter is an object that extracts and formats data from pyarrow table. + It defines the formatting for rows, colums and batches. + If the formatter for a given type name doesn't exist or is not available, an error is raised. + """ + format_type = get_format_type_from_alias(format_type) + if format_type in _FORMAT_TYPES: + return _FORMAT_TYPES[format_type](**format_kwargs) + if format_type in _FORMAT_TYPES_ALIASES_UNAVAILABLE: + raise _FORMAT_TYPES_ALIASES_UNAVAILABLE[format_type] + else: + raise ValueError(f"Format type should be one of {list(_FORMAT_TYPES.keys())}, but got '{format_type}'") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d36d3f940861a5046bf3e32326d2475e61e794e8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/formatting.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/formatting.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e98e1dbfc4575f3f8ebd5c49dfc18f3ea9d77068 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/formatting.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/jax_formatter.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/jax_formatter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..324d7e24cff75257b363a6a0353243d881cf78f2 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/jax_formatter.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/np_formatter.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/np_formatter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d042938fbb50decaf7cf5ccf1a80f7ae8d2febb2 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/np_formatter.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/polars_formatter.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/polars_formatter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7d249f80ccbaa775827e32dfa5aef05a2924c43c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/polars_formatter.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/tf_formatter.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/tf_formatter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d01af3b9e5e61de249bf42c4229bd451bb002a00 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/tf_formatter.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/torch_formatter.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/torch_formatter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6bd6d8ed541a46526b2fee89788973e65631db32 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/__pycache__/torch_formatter.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/formatting.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/formatting.py new file mode 100644 index 0000000000000000000000000000000000000000..9570fb1808ca2a46b35da691c033802cd6eb601b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/formatting.py @@ -0,0 +1,653 @@ +# Copyright 2020 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import operator +from collections.abc import Mapping, MutableMapping +from functools import partial + +# Lint as: python3 +from typing import Any, Callable, Dict, Generic, Iterable, List, Optional, TypeVar, Union + +import numpy as np +import pandas as pd +import pyarrow as pa +from packaging import version + +from .. import config +from ..features import Features +from ..features.features import _ArrayXDExtensionType, _is_zero_copy_only, decode_nested_example, pandas_types_mapper +from ..table import Table +from ..utils.py_utils import no_op_if_value_is_null + + +T = TypeVar("T") + +RowFormat = TypeVar("RowFormat") +ColumnFormat = TypeVar("ColumnFormat") +BatchFormat = TypeVar("BatchFormat") + + +def _is_range_contiguous(key: range) -> bool: + return key.step == 1 and key.stop >= key.start + + +def _raise_bad_key_type(key: Any): + raise TypeError( + f"Wrong key type: '{key}' of type '{type(key)}'. Expected one of int, slice, range, str or Iterable." + ) + + +def _query_table_with_indices_mapping( + table: Table, key: Union[int, slice, range, str, Iterable], indices: Table +) -> pa.Table: + """ + Query a pyarrow Table to extract the subtable that correspond to the given key. + The :obj:`indices` parameter corresponds to the indices mapping in case we cant to take into + account a shuffling or an indices selection for example. + The indices table must contain one column named "indices" of type uint64. + """ + if isinstance(key, int): + key = indices.fast_slice(key % indices.num_rows, 1).column(0)[0].as_py() + return _query_table(table, key) + if isinstance(key, slice): + key = range(*key.indices(indices.num_rows)) + if isinstance(key, range): + if _is_range_contiguous(key) and key.start >= 0: + return _query_table( + table, [i.as_py() for i in indices.fast_slice(key.start, key.stop - key.start).column(0)] + ) + else: + pass # treat as an iterable + if isinstance(key, str): + table = table.select([key]) + return _query_table(table, indices.column(0).to_pylist()) + if isinstance(key, Iterable): + return _query_table(table, [indices.fast_slice(i, 1).column(0)[0].as_py() for i in key]) + + _raise_bad_key_type(key) + + +def _query_table(table: Table, key: Union[int, slice, range, str, Iterable]) -> pa.Table: + """ + Query a pyarrow Table to extract the subtable that correspond to the given key. + """ + if isinstance(key, int): + return table.fast_slice(key % table.num_rows, 1) + if isinstance(key, slice): + key = range(*key.indices(table.num_rows)) + if isinstance(key, range): + if _is_range_contiguous(key) and key.start >= 0: + return table.fast_slice(key.start, key.stop - key.start) + else: + pass # treat as an iterable + if isinstance(key, str): + return table.table.drop([column for column in table.column_names if column != key]) + if isinstance(key, Iterable): + key = np.fromiter(key, np.int64) + if len(key) == 0: + return table.table.slice(0, 0) + # don't use pyarrow.Table.take even for pyarrow >=1.0 (see https://issues.apache.org/jira/browse/ARROW-9773) + return table.fast_gather(key % table.num_rows) + + _raise_bad_key_type(key) + + +def _is_array_with_nulls(pa_array: pa.Array) -> bool: + return pa_array.null_count > 0 + + +class BaseArrowExtractor(Generic[RowFormat, ColumnFormat, BatchFormat]): + """ + Arrow extractor are used to extract data from pyarrow tables. + It makes it possible to extract rows, columns and batches. + These three extractions types have to be implemented. + """ + + def extract_row(self, pa_table: pa.Table) -> RowFormat: + raise NotImplementedError + + def extract_column(self, pa_table: pa.Table) -> ColumnFormat: + raise NotImplementedError + + def extract_batch(self, pa_table: pa.Table) -> BatchFormat: + raise NotImplementedError + + +def _unnest(py_dict: Dict[str, List[T]]) -> Dict[str, T]: + """Return the first element of a batch (dict) as a row (dict)""" + return {key: array[0] for key, array in py_dict.items()} + + +class SimpleArrowExtractor(BaseArrowExtractor[pa.Table, pa.Array, pa.Table]): + def extract_row(self, pa_table: pa.Table) -> pa.Table: + return pa_table + + def extract_column(self, pa_table: pa.Table) -> pa.Array: + return pa_table.column(0) + + def extract_batch(self, pa_table: pa.Table) -> pa.Table: + return pa_table + + +class PythonArrowExtractor(BaseArrowExtractor[dict, list, dict]): + def extract_row(self, pa_table: pa.Table) -> dict: + return _unnest(pa_table.to_pydict()) + + def extract_column(self, pa_table: pa.Table) -> list: + return pa_table.column(0).to_pylist() + + def extract_batch(self, pa_table: pa.Table) -> dict: + return pa_table.to_pydict() + + +class NumpyArrowExtractor(BaseArrowExtractor[dict, np.ndarray, dict]): + def __init__(self, **np_array_kwargs): + self.np_array_kwargs = np_array_kwargs + + def extract_row(self, pa_table: pa.Table) -> dict: + return _unnest(self.extract_batch(pa_table)) + + def extract_column(self, pa_table: pa.Table) -> np.ndarray: + return self._arrow_array_to_numpy(pa_table[pa_table.column_names[0]]) + + def extract_batch(self, pa_table: pa.Table) -> dict: + return {col: self._arrow_array_to_numpy(pa_table[col]) for col in pa_table.column_names} + + def _arrow_array_to_numpy(self, pa_array: pa.Array) -> np.ndarray: + if isinstance(pa_array, pa.ChunkedArray): + if isinstance(pa_array.type, _ArrayXDExtensionType): + # don't call to_pylist() to preserve dtype of the fixed-size array + zero_copy_only = _is_zero_copy_only(pa_array.type.storage_dtype, unnest=True) + array: List = [ + row for chunk in pa_array.chunks for row in chunk.to_numpy(zero_copy_only=zero_copy_only) + ] + else: + zero_copy_only = _is_zero_copy_only(pa_array.type) and all( + not _is_array_with_nulls(chunk) for chunk in pa_array.chunks + ) + array: List = [ + row for chunk in pa_array.chunks for row in chunk.to_numpy(zero_copy_only=zero_copy_only) + ] + else: + if isinstance(pa_array.type, _ArrayXDExtensionType): + # don't call to_pylist() to preserve dtype of the fixed-size array + zero_copy_only = _is_zero_copy_only(pa_array.type.storage_dtype, unnest=True) + array: List = pa_array.to_numpy(zero_copy_only=zero_copy_only) + else: + zero_copy_only = _is_zero_copy_only(pa_array.type) and not _is_array_with_nulls(pa_array) + array: List = pa_array.to_numpy(zero_copy_only=zero_copy_only).tolist() + if len(array) > 0: + if any( + (isinstance(x, np.ndarray) and (x.dtype == object or x.shape != array[0].shape)) + or (isinstance(x, float) and np.isnan(x)) + for x in array + ): + return np.array(array, copy=False, dtype=object) + return np.array(array, copy=False) + + +class PandasArrowExtractor(BaseArrowExtractor[pd.DataFrame, pd.Series, pd.DataFrame]): + def extract_row(self, pa_table: pa.Table) -> pd.DataFrame: + return pa_table.slice(length=1).to_pandas(types_mapper=pandas_types_mapper) + + def extract_column(self, pa_table: pa.Table) -> pd.Series: + return pa_table.select([0]).to_pandas(types_mapper=pandas_types_mapper)[pa_table.column_names[0]] + + def extract_batch(self, pa_table: pa.Table) -> pd.DataFrame: + return pa_table.to_pandas(types_mapper=pandas_types_mapper) + + +class PythonFeaturesDecoder: + def __init__(self, features: Optional[Features]): + self.features = features + + def decode_row(self, row: dict) -> dict: + return self.features.decode_example(row) if self.features else row + + def decode_column(self, column: list, column_name: str) -> list: + return self.features.decode_column(column, column_name) if self.features else column + + def decode_batch(self, batch: dict) -> dict: + return self.features.decode_batch(batch) if self.features else batch + + +class PandasFeaturesDecoder: + def __init__(self, features: Optional[Features]): + self.features = features + + def decode_row(self, row: pd.DataFrame) -> pd.DataFrame: + decode = ( + { + column_name: no_op_if_value_is_null(partial(decode_nested_example, feature)) + for column_name, feature in self.features.items() + if self.features._column_requires_decoding[column_name] + } + if self.features + else {} + ) + if decode: + row[list(decode.keys())] = row.transform(decode) + return row + + def decode_column(self, column: pd.Series, column_name: str) -> pd.Series: + decode = ( + no_op_if_value_is_null(partial(decode_nested_example, self.features[column_name])) + if self.features and column_name in self.features and self.features._column_requires_decoding[column_name] + else None + ) + if decode: + column = column.transform(decode) + return column + + def decode_batch(self, batch: pd.DataFrame) -> pd.DataFrame: + return self.decode_row(batch) + + +class LazyDict(MutableMapping): + """A dictionary backed by Arrow data. The values are formatted on-the-fly when accessing the dictionary.""" + + def __init__(self, pa_table: pa.Table, formatter: "Formatter"): + self.pa_table = pa_table + self.formatter = formatter + + self.data = {key: None for key in pa_table.column_names} + self.keys_to_format = set(self.data.keys()) + + def __len__(self): + return len(self.data) + + def __getitem__(self, key): + value = self.data[key] + if key in self.keys_to_format: + value = self.format(key) + self.data[key] = value + self.keys_to_format.remove(key) + return value + + def __setitem__(self, key, value): + if key in self.keys_to_format: + self.keys_to_format.remove(key) + self.data[key] = value + + def __delitem__(self, key) -> None: + if key in self.keys_to_format: + self.keys_to_format.remove(key) + del self.data[key] + + def __iter__(self): + return iter(self.data) + + def __contains__(self, key): + return key in self.data + + def __repr__(self): + self._format_all() + return repr(self.data) + + if config.PY_VERSION >= version.parse("3.9"): + # merging with the union ("|") operator is supported in Python 3.9+ + + def __or__(self, other): + if isinstance(other, LazyDict): + inst = self.copy() + other = other.copy() + other._format_all() + inst.keys_to_format -= other.data.keys() + inst.data = inst.data | other.data + return inst + if isinstance(other, dict): + inst = self.copy() + inst.keys_to_format -= other.keys() + inst.data = inst.data | other + return inst + return NotImplemented + + def __ror__(self, other): + if isinstance(other, LazyDict): + inst = self.copy() + other = other.copy() + other._format_all() + inst.keys_to_format -= other.data.keys() + inst.data = other.data | inst.data + return inst + if isinstance(other, dict): + inst = self.copy() + inst.keys_to_format -= other.keys() + inst.data = other | inst.data + return inst + return NotImplemented + + def __ior__(self, other): + if isinstance(other, LazyDict): + other = other.copy() + other._format_all() + self.keys_to_format -= other.data.keys() + self.data |= other.data + else: + self.keys_to_format -= other.keys() + self.data |= other + return self + + def __copy__(self): + # Identical to `UserDict.__copy__` + inst = self.__class__.__new__(self.__class__) + inst.__dict__.update(self.__dict__) + # Create a copy and avoid triggering descriptors + inst.__dict__["data"] = self.__dict__["data"].copy() + inst.__dict__["keys_to_format"] = self.__dict__["keys_to_format"].copy() + return inst + + def copy(self): + import copy + + return copy.copy(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + raise NotImplementedError + + def format(self, key): + raise NotImplementedError + + def _format_all(self): + for key in self.keys_to_format: + self.data[key] = self.format(key) + self.keys_to_format.clear() + + +class LazyRow(LazyDict): + def format(self, key): + return self.formatter.format_column(self.pa_table.select([key]))[0] + + +class LazyBatch(LazyDict): + def format(self, key): + return self.formatter.format_column(self.pa_table.select([key])) + + +class Formatter(Generic[RowFormat, ColumnFormat, BatchFormat]): + """ + A formatter is an object that extracts and formats data from pyarrow tables. + It defines the formatting for rows, columns and batches. + """ + + simple_arrow_extractor = SimpleArrowExtractor + python_arrow_extractor = PythonArrowExtractor + numpy_arrow_extractor = NumpyArrowExtractor + pandas_arrow_extractor = PandasArrowExtractor + + def __init__(self, features: Optional[Features] = None): + self.features = features + self.python_features_decoder = PythonFeaturesDecoder(self.features) + self.pandas_features_decoder = PandasFeaturesDecoder(self.features) + + def __call__(self, pa_table: pa.Table, query_type: str) -> Union[RowFormat, ColumnFormat, BatchFormat]: + if query_type == "row": + return self.format_row(pa_table) + elif query_type == "column": + return self.format_column(pa_table) + elif query_type == "batch": + return self.format_batch(pa_table) + + def format_row(self, pa_table: pa.Table) -> RowFormat: + raise NotImplementedError + + def format_column(self, pa_table: pa.Table) -> ColumnFormat: + raise NotImplementedError + + def format_batch(self, pa_table: pa.Table) -> BatchFormat: + raise NotImplementedError + + +class TensorFormatter(Formatter[RowFormat, ColumnFormat, BatchFormat]): + def recursive_tensorize(self, data_struct: dict): + raise NotImplementedError + + +class ArrowFormatter(Formatter[pa.Table, pa.Array, pa.Table]): + def format_row(self, pa_table: pa.Table) -> pa.Table: + return self.simple_arrow_extractor().extract_row(pa_table) + + def format_column(self, pa_table: pa.Table) -> pa.Array: + return self.simple_arrow_extractor().extract_column(pa_table) + + def format_batch(self, pa_table: pa.Table) -> pa.Table: + return self.simple_arrow_extractor().extract_batch(pa_table) + + +class PythonFormatter(Formatter[Mapping, list, Mapping]): + def __init__(self, features=None, lazy=False): + super().__init__(features) + self.lazy = lazy + + def format_row(self, pa_table: pa.Table) -> Mapping: + if self.lazy: + return LazyRow(pa_table, self) + row = self.python_arrow_extractor().extract_row(pa_table) + row = self.python_features_decoder.decode_row(row) + return row + + def format_column(self, pa_table: pa.Table) -> list: + column = self.python_arrow_extractor().extract_column(pa_table) + column = self.python_features_decoder.decode_column(column, pa_table.column_names[0]) + return column + + def format_batch(self, pa_table: pa.Table) -> Mapping: + if self.lazy: + return LazyBatch(pa_table, self) + batch = self.python_arrow_extractor().extract_batch(pa_table) + batch = self.python_features_decoder.decode_batch(batch) + return batch + + +class PandasFormatter(Formatter[pd.DataFrame, pd.Series, pd.DataFrame]): + def format_row(self, pa_table: pa.Table) -> pd.DataFrame: + row = self.pandas_arrow_extractor().extract_row(pa_table) + row = self.pandas_features_decoder.decode_row(row) + return row + + def format_column(self, pa_table: pa.Table) -> pd.Series: + column = self.pandas_arrow_extractor().extract_column(pa_table) + column = self.pandas_features_decoder.decode_column(column, pa_table.column_names[0]) + return column + + def format_batch(self, pa_table: pa.Table) -> pd.DataFrame: + row = self.pandas_arrow_extractor().extract_batch(pa_table) + row = self.pandas_features_decoder.decode_batch(row) + return row + + +class CustomFormatter(Formatter[dict, ColumnFormat, dict]): + """ + A user-defined custom formatter function defined by a ``transform``. + The transform must take as input a batch of data extracted for an arrow table using the python extractor, + and return a batch. + If the output batch is not a dict, then output_all_columns won't work. + If the ouput batch has several fields, then querying a single column won't work since we don't know which field + to return. + """ + + def __init__(self, transform: Callable[[dict], dict], features=None, **kwargs): + super().__init__(features=features) + self.transform = transform + + def format_row(self, pa_table: pa.Table) -> dict: + formatted_batch = self.format_batch(pa_table) + try: + return _unnest(formatted_batch) + except Exception as exc: + raise TypeError( + f"Custom formatting function must return a dict of sequences to be able to pick a row, but got {formatted_batch}" + ) from exc + + def format_column(self, pa_table: pa.Table) -> ColumnFormat: + formatted_batch = self.format_batch(pa_table) + if hasattr(formatted_batch, "keys"): + if len(formatted_batch.keys()) > 1: + raise TypeError( + "Tried to query a column but the custom formatting function returns too many columns. " + f"Only one column was expected but got columns {list(formatted_batch.keys())}." + ) + else: + raise TypeError( + f"Custom formatting function must return a dict to be able to pick a row, but got {formatted_batch}" + ) + try: + return formatted_batch[pa_table.column_names[0]] + except Exception as exc: + raise TypeError( + f"Custom formatting function must return a dict to be able to pick a row, but got {formatted_batch}" + ) from exc + + def format_batch(self, pa_table: pa.Table) -> dict: + batch = self.python_arrow_extractor().extract_batch(pa_table) + batch = self.python_features_decoder.decode_batch(batch) + return self.transform(batch) + + +def _check_valid_column_key(key: str, columns: List[str]) -> None: + if key not in columns: + raise KeyError(f"Column {key} not in the dataset. Current columns in the dataset: {columns}") + + +def _check_valid_index_key(key: Union[int, slice, range, Iterable], size: int) -> None: + if isinstance(key, int): + if (key < 0 and key + size < 0) or (key >= size): + raise IndexError(f"Invalid key: {key} is out of bounds for size {size}") + return + elif isinstance(key, slice): + pass + elif isinstance(key, range): + if len(key) > 0: + _check_valid_index_key(max(key), size=size) + _check_valid_index_key(min(key), size=size) + elif isinstance(key, Iterable): + if len(key) > 0: + _check_valid_index_key(int(max(key)), size=size) + _check_valid_index_key(int(min(key)), size=size) + else: + _raise_bad_key_type(key) + + +def key_to_query_type(key: Union[int, slice, range, str, Iterable]) -> str: + if isinstance(key, int): + return "row" + elif isinstance(key, str): + return "column" + elif isinstance(key, (slice, range, Iterable)): + return "batch" + _raise_bad_key_type(key) + + +def query_table( + table: Table, + key: Union[int, slice, range, str, Iterable], + indices: Optional[Table] = None, +) -> pa.Table: + """ + Query a Table to extract the subtable that correspond to the given key. + + Args: + table (``datasets.table.Table``): The input Table to query from + key (``Union[int, slice, range, str, Iterable]``): The key can be of different types: + - an integer i: the subtable containing only the i-th row + - a slice [i:j:k]: the subtable containing the rows that correspond to this slice + - a range(i, j, k): the subtable containing the rows that correspond to this range + - a string c: the subtable containing all the rows but only the column c + - an iterable l: the subtable that is the concatenation of all the i-th rows for all i in the iterable + indices (Optional ``datasets.table.Table``): If not None, it is used to re-map the given key to the table rows. + The indices table must contain one column named "indices" of type uint64. + This is used in case of shuffling or rows selection. + + + Returns: + ``pyarrow.Table``: the result of the query on the input table + """ + # Check if key is valid + if not isinstance(key, (int, slice, range, str, Iterable)): + try: + key = operator.index(key) + except TypeError: + _raise_bad_key_type(key) + if isinstance(key, str): + _check_valid_column_key(key, table.column_names) + else: + size = indices.num_rows if indices is not None else table.num_rows + _check_valid_index_key(key, size) + # Query the main table + if indices is None: + pa_subtable = _query_table(table, key) + else: + pa_subtable = _query_table_with_indices_mapping(table, key, indices=indices) + return pa_subtable + + +def format_table( + table: Table, + key: Union[int, slice, range, str, Iterable], + formatter: Formatter, + format_columns: Optional[list] = None, + output_all_columns=False, +): + """ + Format a Table depending on the key that was used and a Formatter object. + + Args: + table (``datasets.table.Table``): The input Table to format + key (``Union[int, slice, range, str, Iterable]``): Depending on the key that was used, the formatter formats + the table as either a row, a column or a batch. + formatter (``datasets.formatting.formatting.Formatter``): Any subclass of a Formatter such as + PythonFormatter, NumpyFormatter, etc. + format_columns (:obj:`List[str]`, optional): if not None, it defines the columns that will be formatted using the + given formatter. Other columns are discarded (unless ``output_all_columns`` is True) + output_all_columns (:obj:`bool`, defaults to False). If True, the formatted output is completed using the columns + that are not in the ``format_columns`` list. For these columns, the PythonFormatter is used. + + + Returns: + A row, column or batch formatted object defined by the Formatter: + - the PythonFormatter returns a dictionary for a row or a batch, and a list for a column. + - the NumpyFormatter returns a dictionary for a row or a batch, and a np.array for a column. + - the PandasFormatter returns a pd.DataFrame for a row or a batch, and a pd.Series for a column. + - the TorchFormatter returns a dictionary for a row or a batch, and a torch.Tensor for a column. + - the TFFormatter returns a dictionary for a row or a batch, and a tf.Tensor for a column. + """ + if isinstance(table, Table): + pa_table = table.table + else: + pa_table = table + query_type = key_to_query_type(key) + python_formatter = PythonFormatter(features=formatter.features) + if format_columns is None: + return formatter(pa_table, query_type=query_type) + elif query_type == "column": + if key in format_columns: + return formatter(pa_table, query_type) + else: + return python_formatter(pa_table, query_type=query_type) + else: + pa_table_to_format = pa_table.drop(col for col in pa_table.column_names if col not in format_columns) + formatted_output = formatter(pa_table_to_format, query_type=query_type) + if output_all_columns: + if isinstance(formatted_output, MutableMapping): + pa_table_with_remaining_columns = pa_table.drop( + col for col in pa_table.column_names if col in format_columns + ) + remaining_columns_dict = python_formatter(pa_table_with_remaining_columns, query_type=query_type) + formatted_output.update(remaining_columns_dict) + else: + raise TypeError( + f"Custom formatting function must return a dict to work with output_all_columns=True, but got {formatted_output}" + ) + return formatted_output diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/jax_formatter.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/jax_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..8035341c5cd2794345163b388945b3a092708916 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/jax_formatter.py @@ -0,0 +1,160 @@ +# Copyright 2021 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +import sys +from collections.abc import Mapping +from typing import TYPE_CHECKING, Dict, Optional + +import numpy as np +import pyarrow as pa + +from .. import config +from ..utils.logging import get_logger +from ..utils.py_utils import map_nested +from .formatting import TensorFormatter + + +if TYPE_CHECKING: + import jax + import jaxlib + +logger = get_logger() + +DEVICE_MAPPING: Optional[dict] = None + + +class JaxFormatter(TensorFormatter[Mapping, "jax.Array", Mapping]): + def __init__(self, features=None, device=None, **jnp_array_kwargs): + super().__init__(features=features) + import jax + from jaxlib.xla_client import Device + + if isinstance(device, Device): + raise ValueError( + f"Expected {device} to be a `str` not {type(device)}, as `jaxlib.xla_extension.Device` " + "is not serializable neither with `pickle` nor with `dill`. Instead you can surround " + "the device with `str()` to get its string identifier that will be internally mapped " + "to the actual `jaxlib.xla_extension.Device`." + ) + self.device = device if isinstance(device, str) else str(jax.devices()[0]) + # using global variable since `jaxlib.xla_extension.Device` is not serializable neither + # with `pickle` nor with `dill`, so we need to use a global variable instead + global DEVICE_MAPPING + if DEVICE_MAPPING is None: + DEVICE_MAPPING = self._map_devices_to_str() + if self.device not in list(DEVICE_MAPPING.keys()): + logger.warning( + f"Device with string identifier {self.device} not listed among the available " + f"devices: {list(DEVICE_MAPPING.keys())}, so falling back to the default " + f"device: {str(jax.devices()[0])}." + ) + self.device = str(jax.devices()[0]) + self.jnp_array_kwargs = jnp_array_kwargs + + @staticmethod + def _map_devices_to_str() -> Dict[str, "jaxlib.xla_extension.Device"]: + import jax + + return {str(device): device for device in jax.devices()} + + def _consolidate(self, column): + import jax + import jax.numpy as jnp + + if isinstance(column, list) and column: + if all( + isinstance(x, jax.Array) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column + ): + return jnp.stack(column, axis=0) + return column + + def _tensorize(self, value): + import jax + import jax.numpy as jnp + + if isinstance(value, (str, bytes, type(None))): + return value + elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character): + return value.tolist() + + default_dtype = {} + + if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer): + # the default int precision depends on the jax config + # see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision + if jax.config.jax_enable_x64: + default_dtype = {"dtype": jnp.int64} + else: + default_dtype = {"dtype": jnp.int32} + elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating): + default_dtype = {"dtype": jnp.float32} + elif config.PIL_AVAILABLE and "PIL" in sys.modules: + import PIL.Image + + if isinstance(value, PIL.Image.Image): + value = np.asarray(value) + + # using global variable since `jaxlib.xla_extension.Device` is not serializable neither + # with `pickle` nor with `dill`, so we need to use a global variable instead + global DEVICE_MAPPING + if DEVICE_MAPPING is None: + DEVICE_MAPPING = self._map_devices_to_str() + + with jax.default_device(DEVICE_MAPPING[self.device]): + # calling jnp.array on a np.ndarray does copy the data + # see https://github.com/google/jax/issues/4486 + return jnp.array(value, **{**default_dtype, **self.jnp_array_kwargs}) + + def _recursive_tensorize(self, data_struct): + import jax + + # support for torch, tf, jax etc. + if config.TORCH_AVAILABLE and "torch" in sys.modules: + import torch + + if isinstance(data_struct, torch.Tensor): + return self._tensorize(data_struct.detach().cpu().numpy()[()]) + if hasattr(data_struct, "__array__") and not isinstance(data_struct, jax.Array): + data_struct = data_struct.__array__() + # support for nested types like struct of list of struct + if isinstance(data_struct, np.ndarray): + if data_struct.dtype == object: # jax arrays cannot be instantied from an array of objects + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + elif isinstance(data_struct, (list, tuple)): + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + return self._tensorize(data_struct) + + def recursive_tensorize(self, data_struct: dict): + return map_nested(self._recursive_tensorize, data_struct, map_list=False) + + def format_row(self, pa_table: pa.Table) -> Mapping: + row = self.numpy_arrow_extractor().extract_row(pa_table) + row = self.python_features_decoder.decode_row(row) + return self.recursive_tensorize(row) + + def format_column(self, pa_table: pa.Table) -> "jax.Array": + column = self.numpy_arrow_extractor().extract_column(pa_table) + column = self.python_features_decoder.decode_column(column, pa_table.column_names[0]) + column = self.recursive_tensorize(column) + column = self._consolidate(column) + return column + + def format_batch(self, pa_table: pa.Table) -> Mapping: + batch = self.numpy_arrow_extractor().extract_batch(pa_table) + batch = self.python_features_decoder.decode_batch(batch) + batch = self.recursive_tensorize(batch) + for column_name in batch: + batch[column_name] = self._consolidate(batch[column_name]) + return batch diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/np_formatter.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/np_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..95bcff2b51728fdd9647dad382639724df163ce2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/np_formatter.py @@ -0,0 +1,106 @@ +# Copyright 2020 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from collections.abc import Mapping + +import numpy as np +import pyarrow as pa + +from .. import config +from ..utils.py_utils import map_nested +from .formatting import TensorFormatter + + +class NumpyFormatter(TensorFormatter[Mapping, np.ndarray, Mapping]): + def __init__(self, features=None, **np_array_kwargs): + super().__init__(features=features) + self.np_array_kwargs = np_array_kwargs + + def _consolidate(self, column): + if isinstance(column, list): + if column and all( + isinstance(x, np.ndarray) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column + ): + return np.stack(column) + else: + # don't use np.array(column, dtype=object) + # since it fails in certain cases + # see https://stackoverflow.com/q/51005699 + out = np.empty(len(column), dtype=object) + out[:] = column + return out + return column + + def _tensorize(self, value): + if isinstance(value, (str, bytes, type(None))): + return value + elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character): + return value + elif isinstance(value, np.number): + return value + + default_dtype = {} + + if isinstance(value, np.ndarray) and np.issubdtype(value.dtype, np.integer): + default_dtype = {"dtype": np.int64} + elif isinstance(value, np.ndarray) and np.issubdtype(value.dtype, np.floating): + default_dtype = {"dtype": np.float32} + elif config.PIL_AVAILABLE and "PIL" in sys.modules: + import PIL.Image + + if isinstance(value, PIL.Image.Image): + return np.asarray(value, **self.np_array_kwargs) + + return np.asarray(value, **{**default_dtype, **self.np_array_kwargs}) + + def _recursive_tensorize(self, data_struct): + # support for torch, tf, jax etc. + if config.TORCH_AVAILABLE and "torch" in sys.modules: + import torch + + if isinstance(data_struct, torch.Tensor): + return self._tensorize(data_struct.detach().cpu().numpy()[()]) + if hasattr(data_struct, "__array__") and not isinstance(data_struct, (np.ndarray, np.character, np.number)): + data_struct = data_struct.__array__() + # support for nested types like struct of list of struct + if isinstance(data_struct, np.ndarray): + if data_struct.dtype == object: + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + if isinstance(data_struct, (list, tuple)): + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + return self._tensorize(data_struct) + + def recursive_tensorize(self, data_struct: dict): + return map_nested(self._recursive_tensorize, data_struct, map_list=False) + + def format_row(self, pa_table: pa.Table) -> Mapping: + row = self.numpy_arrow_extractor().extract_row(pa_table) + row = self.python_features_decoder.decode_row(row) + return self.recursive_tensorize(row) + + def format_column(self, pa_table: pa.Table) -> np.ndarray: + column = self.numpy_arrow_extractor().extract_column(pa_table) + column = self.python_features_decoder.decode_column(column, pa_table.column_names[0]) + column = self.recursive_tensorize(column) + column = self._consolidate(column) + return column + + def format_batch(self, pa_table: pa.Table) -> Mapping: + batch = self.numpy_arrow_extractor().extract_batch(pa_table) + batch = self.python_features_decoder.decode_batch(batch) + batch = self.recursive_tensorize(batch) + for column_name in batch: + batch[column_name] = self._consolidate(batch[column_name]) + return batch diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/polars_formatter.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/polars_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..543bde52dd0fb29f2732bce5ee2edcdf9f253109 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/polars_formatter.py @@ -0,0 +1,122 @@ +# Copyright 2020 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from collections.abc import Mapping +from functools import partial +from typing import TYPE_CHECKING, Optional + +import pyarrow as pa + +from .. import config +from ..features import Features +from ..features.features import decode_nested_example +from ..utils.py_utils import no_op_if_value_is_null +from .formatting import BaseArrowExtractor, TensorFormatter + + +if TYPE_CHECKING: + import polars as pl + + +class PolarsArrowExtractor(BaseArrowExtractor["pl.DataFrame", "pl.Series", "pl.DataFrame"]): + def extract_row(self, pa_table: pa.Table) -> "pl.DataFrame": + if config.POLARS_AVAILABLE: + if "polars" not in sys.modules: + import polars + else: + polars = sys.modules["polars"] + + return polars.from_arrow(pa_table.slice(length=1)) + else: + raise ValueError("Polars needs to be installed to be able to return Polars dataframes.") + + def extract_column(self, pa_table: pa.Table) -> "pl.Series": + if config.POLARS_AVAILABLE: + if "polars" not in sys.modules: + import polars + else: + polars = sys.modules["polars"] + + return polars.from_arrow(pa_table.select([0]))[pa_table.column_names[0]] + else: + raise ValueError("Polars needs to be installed to be able to return Polars dataframes.") + + def extract_batch(self, pa_table: pa.Table) -> "pl.DataFrame": + if config.POLARS_AVAILABLE: + if "polars" not in sys.modules: + import polars + else: + polars = sys.modules["polars"] + + return polars.from_arrow(pa_table) + else: + raise ValueError("Polars needs to be installed to be able to return Polars dataframes.") + + +class PolarsFeaturesDecoder: + def __init__(self, features: Optional[Features]): + self.features = features + import polars as pl # noqa: F401 - import pl at initialization + + def decode_row(self, row: "pl.DataFrame") -> "pl.DataFrame": + decode = ( + { + column_name: no_op_if_value_is_null(partial(decode_nested_example, feature)) + for column_name, feature in self.features.items() + if self.features._column_requires_decoding[column_name] + } + if self.features + else {} + ) + if decode: + row[list(decode.keys())] = row.map_rows(decode) + return row + + def decode_column(self, column: "pl.Series", column_name: str) -> "pl.Series": + decode = ( + no_op_if_value_is_null(partial(decode_nested_example, self.features[column_name])) + if self.features and column_name in self.features and self.features._column_requires_decoding[column_name] + else None + ) + if decode: + column = column.map_elements(decode) + return column + + def decode_batch(self, batch: "pl.DataFrame") -> "pl.DataFrame": + return self.decode_row(batch) + + +class PolarsFormatter(TensorFormatter[Mapping, "pl.DataFrame", Mapping]): + def __init__(self, features=None, **np_array_kwargs): + super().__init__(features=features) + self.np_array_kwargs = np_array_kwargs + self.polars_arrow_extractor = PolarsArrowExtractor + self.polars_features_decoder = PolarsFeaturesDecoder(features) + import polars as pl # noqa: F401 - import pl at initialization + + def format_row(self, pa_table: pa.Table) -> "pl.DataFrame": + row = self.polars_arrow_extractor().extract_row(pa_table) + row = self.polars_features_decoder.decode_row(row) + return row + + def format_column(self, pa_table: pa.Table) -> "pl.Series": + column = self.polars_arrow_extractor().extract_column(pa_table) + column = self.polars_features_decoder.decode_column(column, pa_table.column_names[0]) + return column + + def format_batch(self, pa_table: pa.Table) -> "pl.DataFrame": + row = self.polars_arrow_extractor().extract_batch(pa_table) + row = self.polars_features_decoder.decode_batch(row) + return row diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/tf_formatter.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/tf_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..adb15cda3815d77fa0272562e83fda029d1babee --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/tf_formatter.py @@ -0,0 +1,115 @@ +# Copyright 2020 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +import sys +from collections.abc import Mapping +from typing import TYPE_CHECKING + +import numpy as np +import pyarrow as pa + +from .. import config +from ..utils.py_utils import map_nested +from .formatting import TensorFormatter + + +if TYPE_CHECKING: + import tensorflow as tf + + +class TFFormatter(TensorFormatter[Mapping, "tf.Tensor", Mapping]): + def __init__(self, features=None, **tf_tensor_kwargs): + super().__init__(features=features) + self.tf_tensor_kwargs = tf_tensor_kwargs + import tensorflow as tf # noqa: F401 - import tf at initialization + + def _consolidate(self, column): + import tensorflow as tf + + if isinstance(column, list) and column: + if all( + isinstance(x, tf.Tensor) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column + ): + return tf.stack(column) + elif all( + isinstance(x, (tf.Tensor, tf.RaggedTensor)) and x.ndim == 1 and x.dtype == column[0].dtype + for x in column + ): + # only rag 1-D tensors, otherwise some dimensions become ragged even though they were consolidated + return tf.ragged.stack(column) + + return column + + def _tensorize(self, value): + import tensorflow as tf + + if value is None: + return value + + default_dtype = {} + + if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer): + default_dtype = {"dtype": tf.int64} + elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating): + default_dtype = {"dtype": tf.float32} + elif config.PIL_AVAILABLE and "PIL" in sys.modules: + import PIL.Image + + if isinstance(value, PIL.Image.Image): + value = np.asarray(value) + + return tf.convert_to_tensor(value, **{**default_dtype, **self.tf_tensor_kwargs}) + + def _recursive_tensorize(self, data_struct): + import tensorflow as tf + + # support for torch, tf, jax etc. + if config.TORCH_AVAILABLE and "torch" in sys.modules: + import torch + + if isinstance(data_struct, torch.Tensor): + return self._tensorize(data_struct.detach().cpu().numpy()[()]) + if hasattr(data_struct, "__array__") and not isinstance(data_struct, tf.Tensor): + data_struct = data_struct.__array__() + # support for nested types like struct of list of struct + if isinstance(data_struct, np.ndarray): + if data_struct.dtype == object: # tf tensors cannot be instantied from an array of objects + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + elif isinstance(data_struct, (list, tuple)): + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + return self._tensorize(data_struct) + + def recursive_tensorize(self, data_struct: dict): + return map_nested(self._recursive_tensorize, data_struct, map_list=False) + + def format_row(self, pa_table: pa.Table) -> Mapping: + row = self.numpy_arrow_extractor().extract_row(pa_table) + row = self.python_features_decoder.decode_row(row) + return self.recursive_tensorize(row) + + def format_column(self, pa_table: pa.Table) -> "tf.Tensor": + column = self.numpy_arrow_extractor().extract_column(pa_table) + column = self.python_features_decoder.decode_column(column, pa_table.column_names[0]) + column = self.recursive_tensorize(column) + column = self._consolidate(column) + return column + + def format_batch(self, pa_table: pa.Table) -> Mapping: + batch = self.numpy_arrow_extractor().extract_batch(pa_table) + batch = self.python_features_decoder.decode_batch(batch) + batch = self.recursive_tensorize(batch) + for column_name in batch: + batch[column_name] = self._consolidate(batch[column_name]) + return batch diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/torch_formatter.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/torch_formatter.py new file mode 100644 index 0000000000000000000000000000000000000000..8efe759a1443a74b94d59fe38944f6527ac18cf7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/formatting/torch_formatter.py @@ -0,0 +1,115 @@ +# Copyright 2020 The HuggingFace Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +import sys +from collections.abc import Mapping +from typing import TYPE_CHECKING + +import numpy as np +import pyarrow as pa + +from .. import config +from ..utils.py_utils import map_nested +from .formatting import TensorFormatter + + +if TYPE_CHECKING: + import torch + + +class TorchFormatter(TensorFormatter[Mapping, "torch.Tensor", Mapping]): + def __init__(self, features=None, **torch_tensor_kwargs): + super().__init__(features=features) + self.torch_tensor_kwargs = torch_tensor_kwargs + import torch # noqa import torch at initialization + + def _consolidate(self, column): + import torch + + if isinstance(column, list) and column: + if all( + isinstance(x, torch.Tensor) and x.shape == column[0].shape and x.dtype == column[0].dtype + for x in column + ): + return torch.stack(column) + return column + + def _tensorize(self, value): + import torch + + if isinstance(value, (str, bytes, type(None))): + return value + elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character): + return value.tolist() + + default_dtype = {} + + if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer): + default_dtype = {"dtype": torch.int64} + + # Convert dtype to np.int64 if it's either np.uint16 or np.uint32 to ensure compatibility. + # np.uint64 is excluded from this conversion as there is no compatible PyTorch dtype that can handle it without loss. + if value.dtype in [np.uint16, np.uint32]: + value = value.astype(np.int64) + + elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating): + default_dtype = {"dtype": torch.float32} + elif config.PIL_AVAILABLE and "PIL" in sys.modules: + import PIL.Image + + if isinstance(value, PIL.Image.Image): + value = np.asarray(value) + if value.ndim == 2: + value = value[:, :, np.newaxis] + + value = value.transpose((2, 0, 1)) + return torch.tensor(value, **{**default_dtype, **self.torch_tensor_kwargs}) + + def _recursive_tensorize(self, data_struct): + import torch + + # support for torch, tf, jax etc. + if hasattr(data_struct, "__array__") and not isinstance(data_struct, torch.Tensor): + data_struct = data_struct.__array__() + # support for nested types like struct of list of struct + if isinstance(data_struct, np.ndarray): + if data_struct.dtype == object: # torch tensors cannot be instantied from an array of objects + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + elif isinstance(data_struct, (list, tuple)): + return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct]) + return self._tensorize(data_struct) + + def recursive_tensorize(self, data_struct: dict): + return map_nested(self._recursive_tensorize, data_struct, map_list=False) + + def format_row(self, pa_table: pa.Table) -> Mapping: + row = self.numpy_arrow_extractor().extract_row(pa_table) + row = self.python_features_decoder.decode_row(row) + return self.recursive_tensorize(row) + + def format_column(self, pa_table: pa.Table) -> "torch.Tensor": + column = self.numpy_arrow_extractor().extract_column(pa_table) + column = self.python_features_decoder.decode_column(column, pa_table.column_names[0]) + column = self.recursive_tensorize(column) + column = self._consolidate(column) + return column + + def format_batch(self, pa_table: pa.Table) -> Mapping: + batch = self.numpy_arrow_extractor().extract_batch(pa_table) + batch = self.python_features_decoder.decode_batch(batch) + batch = self.recursive_tensorize(batch) + for column_name in batch: + batch[column_name] = self._consolidate(batch[column_name]) + return batch diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..29f8e8d2e3ec47fb80666db335c27d466a377773 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/abc.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/abc.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6f906c1dfe52221f97a88e475827666bcca0a55 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/abc.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/csv.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/csv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7db7746612b6a2faf4946d1c5245d3fc31e53b6 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/csv.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/generator.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/generator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..091feed3e42ea3f41337f2b8430fce5ed6320662 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/generator.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/json.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/json.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..113b0f0029b054fa3dd73fa384f4137656c189b6 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/json.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/parquet.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/parquet.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..af43583ec23270cc5f34fba39ac587cd03a90e97 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/parquet.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/spark.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/spark.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b107ede0b594136f04846a4dcf9543bee155659a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/spark.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/sql.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/sql.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a0c7b56b66a7b2beb4d7af158966f0e151c54f7b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/sql.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/text.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/text.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24a215ee11fa6659c2ee65f31b3f1908dc33ac77 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/__pycache__/text.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/abc.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/abc.py new file mode 100644 index 0000000000000000000000000000000000000000..a1913cc20e3fd748ef912e2fb3d7c1e18f16ac8c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/abc.py @@ -0,0 +1,53 @@ +from abc import ABC, abstractmethod +from typing import Optional, Union + +from .. import Dataset, DatasetDict, Features, IterableDataset, IterableDatasetDict, NamedSplit +from ..utils.typing import NestedDataStructureLike, PathLike + + +class AbstractDatasetReader(ABC): + def __init__( + self, + path_or_paths: Optional[NestedDataStructureLike[PathLike]] = None, + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ): + self.path_or_paths = path_or_paths + self.split = split if split or isinstance(path_or_paths, dict) else "train" + self.features = features + self.cache_dir = cache_dir + self.keep_in_memory = keep_in_memory + self.streaming = streaming + self.num_proc = num_proc + self.kwargs = kwargs + + @abstractmethod + def read(self) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]: + pass + + +class AbstractDatasetInputStream(ABC): + def __init__( + self, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ): + self.features = features + self.cache_dir = cache_dir + self.keep_in_memory = keep_in_memory + self.streaming = streaming + self.num_proc = num_proc + self.kwargs = kwargs + + @abstractmethod + def read(self) -> Union[Dataset, IterableDataset]: + pass diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/csv.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/csv.py new file mode 100644 index 0000000000000000000000000000000000000000..4ac2ea1135b579ae0f38f7dafc73ae5c2db1a6b6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/csv.py @@ -0,0 +1,145 @@ +import multiprocessing +import os +from typing import BinaryIO, Optional, Union + +import fsspec + +from .. import Dataset, Features, NamedSplit, config +from ..formatting import query_table +from ..packaged_modules.csv.csv import Csv +from ..utils import tqdm as hf_tqdm +from ..utils.typing import NestedDataStructureLike, PathLike +from .abc import AbstractDatasetReader + + +class CsvDatasetReader(AbstractDatasetReader): + def __init__( + self, + path_or_paths: NestedDataStructureLike[PathLike], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ): + super().__init__( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + num_proc=num_proc, + **kwargs, + ) + path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths} + self.builder = Csv( + cache_dir=cache_dir, + data_files=path_or_paths, + features=features, + **kwargs, + ) + + def read(self): + # Build iterable dataset + if self.streaming: + dataset = self.builder.as_streaming_dataset(split=self.split) + # Build regular (map-style) dataset + else: + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + num_proc=self.num_proc, + ) + dataset = self.builder.as_dataset( + split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory + ) + return dataset + + +class CsvDatasetWriter: + def __init__( + self, + dataset: Dataset, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **to_csv_kwargs, + ): + if num_proc is not None and num_proc <= 0: + raise ValueError(f"num_proc {num_proc} must be an integer > 0.") + + self.dataset = dataset + self.path_or_buf = path_or_buf + self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE + self.num_proc = num_proc + self.encoding = "utf-8" + self.storage_options = storage_options or {} + self.to_csv_kwargs = to_csv_kwargs + + def write(self) -> int: + _ = self.to_csv_kwargs.pop("path_or_buf", None) + header = self.to_csv_kwargs.pop("header", True) + index = self.to_csv_kwargs.pop("index", False) + + if isinstance(self.path_or_buf, (str, bytes, os.PathLike)): + with fsspec.open(self.path_or_buf, "wb", **(self.storage_options or {})) as buffer: + written = self._write(file_obj=buffer, header=header, index=index, **self.to_csv_kwargs) + else: + written = self._write(file_obj=self.path_or_buf, header=header, index=index, **self.to_csv_kwargs) + return written + + def _batch_csv(self, args): + offset, header, index, to_csv_kwargs = args + + batch = query_table( + table=self.dataset.data, + key=slice(offset, offset + self.batch_size), + indices=self.dataset._indices, + ) + csv_str = batch.to_pandas().to_csv( + path_or_buf=None, header=header if (offset == 0) else False, index=index, **to_csv_kwargs + ) + return csv_str.encode(self.encoding) + + def _write(self, file_obj: BinaryIO, header, index, **to_csv_kwargs) -> int: + """Writes the pyarrow table as CSV to a binary file handle. + + Caller is responsible for opening and closing the handle. + """ + written = 0 + + if self.num_proc is None or self.num_proc == 1: + for offset in hf_tqdm( + range(0, len(self.dataset), self.batch_size), + unit="ba", + desc="Creating CSV from Arrow format", + ): + csv_str = self._batch_csv((offset, header, index, to_csv_kwargs)) + written += file_obj.write(csv_str) + + else: + num_rows, batch_size = len(self.dataset), self.batch_size + with multiprocessing.Pool(self.num_proc) as pool: + for csv_str in hf_tqdm( + pool.imap( + self._batch_csv, + [(offset, header, index, to_csv_kwargs) for offset in range(0, num_rows, batch_size)], + ), + total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size, + unit="ba", + desc="Creating CSV from Arrow format", + ): + written += file_obj.write(csv_str) + + return written diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/generator.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/generator.py new file mode 100644 index 0000000000000000000000000000000000000000..2566d5fcdcca6525f67afe9106dcf9ba2842d92c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/generator.py @@ -0,0 +1,57 @@ +from typing import Callable, Optional + +from .. import Features +from ..packaged_modules.generator.generator import Generator +from .abc import AbstractDatasetInputStream + + +class GeneratorDatasetInputStream(AbstractDatasetInputStream): + def __init__( + self, + generator: Callable, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + gen_kwargs: Optional[dict] = None, + num_proc: Optional[int] = None, + **kwargs, + ): + super().__init__( + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + num_proc=num_proc, + **kwargs, + ) + self.builder = Generator( + cache_dir=cache_dir, + features=features, + generator=generator, + gen_kwargs=gen_kwargs, + **kwargs, + ) + + def read(self): + # Build iterable dataset + if self.streaming: + dataset = self.builder.as_streaming_dataset(split="train") + # Build regular (map-style) dataset + else: + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + num_proc=self.num_proc, + ) + dataset = self.builder.as_dataset( + split="train", verification_mode=verification_mode, in_memory=self.keep_in_memory + ) + return dataset diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/json.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/json.py new file mode 100644 index 0000000000000000000000000000000000000000..c1d89b0fd04c83394e34104442776ba15628cb58 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/json.py @@ -0,0 +1,170 @@ +import multiprocessing +import os +from typing import BinaryIO, Optional, Union + +import fsspec + +from .. import Dataset, Features, NamedSplit, config +from ..formatting import query_table +from ..packaged_modules.json.json import Json +from ..utils import tqdm as hf_tqdm +from ..utils.typing import NestedDataStructureLike, PathLike +from .abc import AbstractDatasetReader + + +class JsonDatasetReader(AbstractDatasetReader): + def __init__( + self, + path_or_paths: NestedDataStructureLike[PathLike], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + field: Optional[str] = None, + num_proc: Optional[int] = None, + **kwargs, + ): + super().__init__( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + num_proc=num_proc, + **kwargs, + ) + self.field = field + path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths} + self.builder = Json( + cache_dir=cache_dir, + data_files=path_or_paths, + features=features, + field=field, + **kwargs, + ) + + def read(self): + # Build iterable dataset + if self.streaming: + dataset = self.builder.as_streaming_dataset(split=self.split) + # Build regular (map-style) dataset + else: + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + # try_from_hf_gcs=try_from_hf_gcs, + base_path=base_path, + num_proc=self.num_proc, + ) + dataset = self.builder.as_dataset( + split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory + ) + return dataset + + +class JsonDatasetWriter: + def __init__( + self, + dataset: Dataset, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + num_proc: Optional[int] = None, + storage_options: Optional[dict] = None, + **to_json_kwargs, + ): + if num_proc is not None and num_proc <= 0: + raise ValueError(f"num_proc {num_proc} must be an integer > 0.") + + self.dataset = dataset + self.path_or_buf = path_or_buf + self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE + self.num_proc = num_proc + self.encoding = "utf-8" + self.storage_options = storage_options or {} + self.to_json_kwargs = to_json_kwargs + + def write(self) -> int: + _ = self.to_json_kwargs.pop("path_or_buf", None) + orient = self.to_json_kwargs.pop("orient", "records") + lines = self.to_json_kwargs.pop("lines", True if orient == "records" else False) + if "index" not in self.to_json_kwargs and orient in ["split", "table"]: + self.to_json_kwargs["index"] = False + + # Determine the default compression value based on self.path_or_buf type + default_compression = "infer" if isinstance(self.path_or_buf, (str, bytes, os.PathLike)) else None + compression = self.to_json_kwargs.pop("compression", default_compression) + + if compression not in [None, "infer", "gzip", "bz2", "xz"]: + raise NotImplementedError(f"`datasets` currently does not support {compression} compression") + + if isinstance(self.path_or_buf, (str, bytes, os.PathLike)): + with fsspec.open( + self.path_or_buf, "wb", compression=compression, **(self.storage_options or {}) + ) as buffer: + written = self._write(file_obj=buffer, orient=orient, lines=lines, **self.to_json_kwargs) + else: + if compression: + raise NotImplementedError( + f"The compression parameter is not supported when writing to a buffer, but compression={compression}" + " was passed. Please provide a local path instead." + ) + written = self._write(file_obj=self.path_or_buf, orient=orient, lines=lines, **self.to_json_kwargs) + return written + + def _batch_json(self, args): + offset, orient, lines, to_json_kwargs = args + + batch = query_table( + table=self.dataset.data, + key=slice(offset, offset + self.batch_size), + indices=self.dataset._indices, + ) + json_str = batch.to_pandas().to_json(path_or_buf=None, orient=orient, lines=lines, **to_json_kwargs) + if not json_str.endswith("\n"): + json_str += "\n" + return json_str.encode(self.encoding) + + def _write( + self, + file_obj: BinaryIO, + orient, + lines, + **to_json_kwargs, + ) -> int: + """Writes the pyarrow table as JSON lines to a binary file handle. + + Caller is responsible for opening and closing the handle. + """ + written = 0 + + if self.num_proc is None or self.num_proc == 1: + for offset in hf_tqdm( + range(0, len(self.dataset), self.batch_size), + unit="ba", + desc="Creating json from Arrow format", + ): + json_str = self._batch_json((offset, orient, lines, to_json_kwargs)) + written += file_obj.write(json_str) + else: + num_rows, batch_size = len(self.dataset), self.batch_size + with multiprocessing.Pool(self.num_proc) as pool: + for json_str in hf_tqdm( + pool.imap( + self._batch_json, + [(offset, orient, lines, to_json_kwargs) for offset in range(0, num_rows, batch_size)], + ), + total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size, + unit="ba", + desc="Creating json from Arrow format", + ): + written += file_obj.write(json_str) + + return written diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/parquet.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/parquet.py new file mode 100644 index 0000000000000000000000000000000000000000..51434106fb1f623500ceabbb6b0b3077fa20eee9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/parquet.py @@ -0,0 +1,158 @@ +import os +from typing import BinaryIO, Optional, Union + +import fsspec +import numpy as np +import pyarrow.parquet as pq + +from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config +from ..features.features import FeatureType, _visit +from ..formatting import query_table +from ..packaged_modules import _PACKAGED_DATASETS_MODULES +from ..packaged_modules.parquet.parquet import Parquet +from ..utils import tqdm as hf_tqdm +from ..utils.typing import NestedDataStructureLike, PathLike +from .abc import AbstractDatasetReader + + +def get_writer_batch_size(features: Features) -> Optional[int]: + """ + Get the writer_batch_size that defines the maximum row group size in the parquet files. + The default in `datasets` is 1,000 but we lower it to 100 for image datasets. + This allows to optimize random access to parquet file, since accessing 1 row requires + to read its entire row group. + + This can be improved to get optimized size for querying/iterating + but at least it matches the dataset viewer expectations on HF. + + Args: + ds_config_info (`datasets.info.DatasetInfo`): + Dataset info from `datasets`. + Returns: + writer_batch_size (`Optional[int]`): + Writer batch size to pass to a dataset builder. + If `None`, then it will use the `datasets` default. + """ + + batch_size = np.inf + + def set_batch_size(feature: FeatureType) -> None: + nonlocal batch_size + if isinstance(feature, Image): + batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS) + elif isinstance(feature, Audio): + batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS) + elif isinstance(feature, Value) and feature.dtype == "binary": + batch_size = min(batch_size, config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS) + + _visit(features, set_batch_size) + + return None if batch_size is np.inf else batch_size + + +class ParquetDatasetReader(AbstractDatasetReader): + def __init__( + self, + path_or_paths: NestedDataStructureLike[PathLike], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ): + super().__init__( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + num_proc=num_proc, + **kwargs, + ) + path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths} + hash = _PACKAGED_DATASETS_MODULES["parquet"][1] + self.builder = Parquet( + cache_dir=cache_dir, + data_files=path_or_paths, + features=features, + hash=hash, + **kwargs, + ) + + def read(self): + # Build iterable dataset + if self.streaming: + dataset = self.builder.as_streaming_dataset(split=self.split) + # Build regular (map-style) dataset + else: + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + num_proc=self.num_proc, + ) + dataset = self.builder.as_dataset( + split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory + ) + return dataset + + +class ParquetDatasetWriter: + def __init__( + self, + dataset: Dataset, + path_or_buf: Union[PathLike, BinaryIO], + batch_size: Optional[int] = None, + storage_options: Optional[dict] = None, + **parquet_writer_kwargs, + ): + self.dataset = dataset + self.path_or_buf = path_or_buf + self.batch_size = batch_size or get_writer_batch_size(dataset.features) + self.storage_options = storage_options or {} + self.parquet_writer_kwargs = parquet_writer_kwargs + + def write(self) -> int: + batch_size = self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE + + if isinstance(self.path_or_buf, (str, bytes, os.PathLike)): + with fsspec.open(self.path_or_buf, "wb", **(self.storage_options or {})) as buffer: + written = self._write(file_obj=buffer, batch_size=batch_size, **self.parquet_writer_kwargs) + else: + written = self._write(file_obj=self.path_or_buf, batch_size=batch_size, **self.parquet_writer_kwargs) + return written + + def _write(self, file_obj: BinaryIO, batch_size: int, **parquet_writer_kwargs) -> int: + """Writes the pyarrow table as Parquet to a binary file handle. + + Caller is responsible for opening and closing the handle. + """ + written = 0 + _ = parquet_writer_kwargs.pop("path_or_buf", None) + schema = self.dataset.features.arrow_schema + + writer = pq.ParquetWriter(file_obj, schema=schema, **parquet_writer_kwargs) + + for offset in hf_tqdm( + range(0, len(self.dataset), batch_size), + unit="ba", + desc="Creating parquet from Arrow format", + ): + batch = query_table( + table=self.dataset._data, + key=slice(offset, offset + batch_size), + indices=self.dataset._indices, + ) + writer.write_table(batch) + written += batch.nbytes + writer.close() + return written diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/spark.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/spark.py new file mode 100644 index 0000000000000000000000000000000000000000..7562ba1fb5f77ed8f82374e3021fcb3a93b1da8d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/spark.py @@ -0,0 +1,57 @@ +from typing import Optional + +import pyspark + +from .. import Features, NamedSplit +from ..download import DownloadMode +from ..packaged_modules.spark.spark import Spark +from .abc import AbstractDatasetReader + + +class SparkDatasetReader(AbstractDatasetReader): + """A dataset reader that reads from a Spark DataFrame. + + When caching, cache materialization is parallelized over Spark; an NFS that is accessible to the driver must be + provided. Streaming is not currently supported. + """ + + def __init__( + self, + df: pyspark.sql.DataFrame, + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + streaming: bool = True, + cache_dir: str = None, + keep_in_memory: bool = False, + working_dir: str = None, + load_from_cache_file: bool = True, + file_format: str = "arrow", + **kwargs, + ): + super().__init__( + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + **kwargs, + ) + self._load_from_cache_file = load_from_cache_file + self._file_format = file_format + self.builder = Spark( + df=df, + features=features, + cache_dir=cache_dir, + working_dir=working_dir, + **kwargs, + ) + + def read(self): + if self.streaming: + return self.builder.as_streaming_dataset(split=self.split) + download_mode = None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD + self.builder.download_and_prepare( + download_mode=download_mode, + file_format=self._file_format, + ) + return self.builder.as_dataset(split=self.split) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/sql.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/sql.py new file mode 100644 index 0000000000000000000000000000000000000000..2331e3e6407fa81cd26569306ff2c7952dad4920 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/sql.py @@ -0,0 +1,124 @@ +import multiprocessing +from typing import TYPE_CHECKING, Optional, Union + +from .. import Dataset, Features, config +from ..formatting import query_table +from ..packaged_modules.sql.sql import Sql +from ..utils import tqdm as hf_tqdm +from .abc import AbstractDatasetInputStream + + +if TYPE_CHECKING: + import sqlite3 + + import sqlalchemy + + +class SqlDatasetReader(AbstractDatasetInputStream): + def __init__( + self, + sql: Union[str, "sqlalchemy.sql.Selectable"], + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"], + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + **kwargs, + ): + super().__init__(features=features, cache_dir=cache_dir, keep_in_memory=keep_in_memory, **kwargs) + self.builder = Sql( + cache_dir=cache_dir, + features=features, + sql=sql, + con=con, + **kwargs, + ) + + def read(self): + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + ) + + # Build dataset for splits + dataset = self.builder.as_dataset( + split="train", verification_mode=verification_mode, in_memory=self.keep_in_memory + ) + return dataset + + +class SqlDatasetWriter: + def __init__( + self, + dataset: Dataset, + name: str, + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"], + batch_size: Optional[int] = None, + num_proc: Optional[int] = None, + **to_sql_kwargs, + ): + if num_proc is not None and num_proc <= 0: + raise ValueError(f"num_proc {num_proc} must be an integer > 0.") + + self.dataset = dataset + self.name = name + self.con = con + self.batch_size = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE + self.num_proc = num_proc + self.to_sql_kwargs = to_sql_kwargs + + def write(self) -> int: + _ = self.to_sql_kwargs.pop("sql", None) + _ = self.to_sql_kwargs.pop("con", None) + index = self.to_sql_kwargs.pop("index", False) + + written = self._write(index=index, **self.to_sql_kwargs) + return written + + def _batch_sql(self, args): + offset, index, to_sql_kwargs = args + to_sql_kwargs = {**to_sql_kwargs, "if_exists": "append"} if offset > 0 else to_sql_kwargs + batch = query_table( + table=self.dataset.data, + key=slice(offset, offset + self.batch_size), + indices=self.dataset._indices, + ) + df = batch.to_pandas() + num_rows = df.to_sql(self.name, self.con, index=index, **to_sql_kwargs) + return num_rows or len(df) + + def _write(self, index, **to_sql_kwargs) -> int: + """Writes the pyarrow table as SQL to a database. + + Caller is responsible for opening and closing the SQL connection. + """ + written = 0 + + if self.num_proc is None or self.num_proc == 1: + for offset in hf_tqdm( + range(0, len(self.dataset), self.batch_size), + unit="ba", + desc="Creating SQL from Arrow format", + ): + written += self._batch_sql((offset, index, to_sql_kwargs)) + else: + num_rows, batch_size = len(self.dataset), self.batch_size + with multiprocessing.Pool(self.num_proc) as pool: + for num_rows in hf_tqdm( + pool.imap( + self._batch_sql, + [(offset, index, to_sql_kwargs) for offset in range(0, num_rows, batch_size)], + ), + total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size, + unit="ba", + desc="Creating SQL from Arrow format", + ): + written += num_rows + + return written diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/text.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/text.py new file mode 100644 index 0000000000000000000000000000000000000000..58963f3c7ab1779d79226943e341200b782e1add --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/io/text.py @@ -0,0 +1,60 @@ +from typing import Optional + +from .. import Features, NamedSplit +from ..packaged_modules.text.text import Text +from ..utils.typing import NestedDataStructureLike, PathLike +from .abc import AbstractDatasetReader + + +class TextDatasetReader(AbstractDatasetReader): + def __init__( + self, + path_or_paths: NestedDataStructureLike[PathLike], + split: Optional[NamedSplit] = None, + features: Optional[Features] = None, + cache_dir: str = None, + keep_in_memory: bool = False, + streaming: bool = False, + num_proc: Optional[int] = None, + **kwargs, + ): + super().__init__( + path_or_paths, + split=split, + features=features, + cache_dir=cache_dir, + keep_in_memory=keep_in_memory, + streaming=streaming, + num_proc=num_proc, + **kwargs, + ) + path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths} + self.builder = Text( + cache_dir=cache_dir, + data_files=path_or_paths, + features=features, + **kwargs, + ) + + def read(self): + # Build iterable dataset + if self.streaming: + dataset = self.builder.as_streaming_dataset(split=self.split) + # Build regular (map-style) dataset + else: + download_config = None + download_mode = None + verification_mode = None + base_path = None + + self.builder.download_and_prepare( + download_config=download_config, + download_mode=download_mode, + verification_mode=verification_mode, + base_path=base_path, + num_proc=self.num_proc, + ) + dataset = self.builder.as_dataset( + split=self.split, verification_mode=verification_mode, in_memory=self.keep_in_memory + ) + return dataset diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3513f9ae59e7c48e1f41756f48c14a1bb217ffdf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/__init__.py @@ -0,0 +1,83 @@ +import inspect +import re +from typing import Dict, List, Tuple + +from huggingface_hub.utils import insecure_hashlib + +from .arrow import arrow +from .audiofolder import audiofolder +from .cache import cache +from .csv import csv +from .imagefolder import imagefolder +from .json import json +from .pandas import pandas +from .parquet import parquet +from .sql import sql +from .text import text +from .webdataset import webdataset + + +def _hash_python_lines(lines: List[str]) -> str: + filtered_lines = [] + for line in lines: + line = re.sub(r"#.*", "", line) # remove comments + if line: + filtered_lines.append(line) + full_str = "\n".join(filtered_lines) + + # Make a hash from all this code + full_bytes = full_str.encode("utf-8") + return insecure_hashlib.sha256(full_bytes).hexdigest() + + +# get importable module names and hash for caching +_PACKAGED_DATASETS_MODULES = { + "csv": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())), + "json": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())), + "pandas": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())), + "parquet": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())), + "arrow": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())), + "text": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())), + "imagefolder": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())), + "audiofolder": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())), + "webdataset": (webdataset.__name__, _hash_python_lines(inspect.getsource(webdataset).splitlines())), +} + +# get importable module names and hash for caching +_PACKAGED_DATASETS_MODULES_2_15_HASHES = { + "csv": "eea64c71ca8b46dd3f537ed218fc9bf495d5707789152eb2764f5c78fa66d59d", + "json": "8bb11242116d547c741b2e8a1f18598ffdd40a1d4f2a2872c7a28b697434bc96", + "pandas": "3ac4ffc4563c796122ef66899b9485a3f1a977553e2d2a8a318c72b8cc6f2202", + "parquet": "ca31c69184d9832faed373922c2acccec0b13a0bb5bbbe19371385c3ff26f1d1", + "arrow": "74f69db2c14c2860059d39860b1f400a03d11bf7fb5a8258ca38c501c878c137", + "text": "c4a140d10f020282918b5dd1b8a49f0104729c6177f60a6b49ec2a365ec69f34", + "imagefolder": "7b7ce5247a942be131d49ad4f3de5866083399a0f250901bd8dc202f8c5f7ce5", + "audiofolder": "d3c1655c66c8f72e4efb5c79e952975fa6e2ce538473a6890241ddbddee9071c", +} + +# Used to infer the module to use based on the data files extensions +_EXTENSION_TO_MODULE: Dict[str, Tuple[str, dict]] = { + ".csv": ("csv", {}), + ".tsv": ("csv", {"sep": "\t"}), + ".json": ("json", {}), + ".jsonl": ("json", {}), + ".parquet": ("parquet", {}), + ".geoparquet": ("parquet", {}), + ".gpq": ("parquet", {}), + ".arrow": ("arrow", {}), + ".txt": ("text", {}), + ".tar": ("webdataset", {}), +} +_EXTENSION_TO_MODULE.update({ext: ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext.upper(): ("imagefolder", {}) for ext in imagefolder.ImageFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext: ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS}) +_EXTENSION_TO_MODULE.update({ext.upper(): ("audiofolder", {}) for ext in audiofolder.AudioFolder.EXTENSIONS}) +_MODULE_SUPPORTS_METADATA = {"imagefolder", "audiofolder"} + +# Used to filter data files based on extensions given a module name +_MODULE_TO_EXTENSIONS: Dict[str, List[str]] = {} +for _ext, (_module, _) in _EXTENSION_TO_MODULE.items(): + _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext) + +for _module in _MODULE_TO_EXTENSIONS: + _MODULE_TO_EXTENSIONS[_module].append(".zip") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d24862c574a02fd8003dcd86f0bb18c6e9b8a93 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2eee578f557db02b19e876c42218b9db5210b349 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__pycache__/arrow.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__pycache__/arrow.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9577976c314a99d4c5d9c88b02fbc8431521015a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/arrow/__pycache__/arrow.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/arrow/arrow.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/arrow/arrow.py new file mode 100644 index 0000000000000000000000000000000000000000..223a74602c35b09208ce06b226ff3bdd6e6dfdd2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/arrow/arrow.py @@ -0,0 +1,70 @@ +import itertools +from dataclasses import dataclass +from typing import Optional + +import pyarrow as pa + +import datasets +from datasets.table import table_cast + + +logger = datasets.utils.logging.get_logger(__name__) + + +@dataclass +class ArrowConfig(datasets.BuilderConfig): + """BuilderConfig for Arrow.""" + + features: Optional[datasets.Features] = None + + def __post_init__(self): + super().__post_init__() + + +class Arrow(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = ArrowConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + data_files = dl_manager.download_and_extract(self.config.data_files) + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + # Use `dl_manager.iter_files` to skip hidden files in an extracted archive + files = [dl_manager.iter_files(file) for file in files] + # Infer features if they are stored in the arrow schema + if self.info.features is None: + for file in itertools.chain.from_iterable(files): + with open(file, "rb") as f: + self.info.features = datasets.Features.from_arrow_schema(pa.ipc.open_stream(f).schema) + break + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.info.features is not None: + # more expensive cast to support nested features with keys in a different order + # allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, self.info.features.arrow_schema) + return pa_table + + def _generate_tables(self, files): + for file_idx, file in enumerate(itertools.chain.from_iterable(files)): + with open(file, "rb") as f: + try: + for batch_idx, record_batch in enumerate(pa.ipc.open_stream(f)): + pa_table = pa.Table.from_batches([record_batch]) + # Uncomment for debugging (will print the Arrow table size and elements) + # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}") + # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows))) + yield f"{file_idx}_{batch_idx}", self._cast_table(pa_table) + except ValueError as e: + logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}") + raise diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fc19521dd8b442f9797cc0c8df0c85397c4d550d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__pycache__/audiofolder.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__pycache__/audiofolder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..989773aaa439b872f04bddac8414fd570d8559fd Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/__pycache__/audiofolder.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/audiofolder.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/audiofolder.py new file mode 100644 index 0000000000000000000000000000000000000000..338fa09d755d802d4191fe0ec8827d9118c57297 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/audiofolder/audiofolder.py @@ -0,0 +1,71 @@ +from typing import List + +import datasets +from datasets.tasks import AudioClassification + +from ..folder_based_builder import folder_based_builder + + +logger = datasets.utils.logging.get_logger(__name__) + + +class AudioFolderConfig(folder_based_builder.FolderBasedBuilderConfig): + """Builder Config for AudioFolder.""" + + drop_labels: bool = None + drop_metadata: bool = None + + def __post_init__(self): + super().__post_init__() + + +class AudioFolder(folder_based_builder.FolderBasedBuilder): + BASE_FEATURE = datasets.Audio + BASE_COLUMN_NAME = "audio" + BUILDER_CONFIG_CLASS = AudioFolderConfig + EXTENSIONS: List[str] # definition at the bottom of the script + CLASSIFICATION_TASK = AudioClassification(audio_column="audio", label_column="label") + + +# Obtained with: +# ``` +# import soundfile as sf +# +# AUDIO_EXTENSIONS = [f".{format.lower()}" for format in sf.available_formats().keys()] +# +# # .opus decoding is supported if libsndfile >= 1.0.31: +# AUDIO_EXTENSIONS.extend([".opus"]) +# ``` +# We intentionally do not run this code on launch because: +# (1) Soundfile is an optional dependency, so importing it in global namespace is not allowed +# (2) To ensure the list of supported extensions is deterministic +AUDIO_EXTENSIONS = [ + ".aiff", + ".au", + ".avr", + ".caf", + ".flac", + ".htk", + ".svx", + ".mat4", + ".mat5", + ".mpc2k", + ".ogg", + ".paf", + ".pvf", + ".raw", + ".rf64", + ".sd2", + ".sds", + ".ircam", + ".voc", + ".w64", + ".wav", + ".nist", + ".wavex", + ".wve", + ".xi", + ".mp3", + ".opus", +] +AudioFolder.EXTENSIONS = AUDIO_EXTENSIONS diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/cache/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/cache/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/cache/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/cache/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..725114795ae4f56439bddde0e2ea9505991bfe17 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/cache/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/cache/__pycache__/cache.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/cache/__pycache__/cache.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ffa2b7fc03bf9783b4d2682f12fe7ac3392dfc43 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/cache/__pycache__/cache.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/cache/cache.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/cache/cache.py new file mode 100644 index 0000000000000000000000000000000000000000..9085b22078b6010ce1e4137573b5b884f56f487b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/cache/cache.py @@ -0,0 +1,207 @@ +import glob +import json +import os +import shutil +import time +import warnings +from pathlib import Path +from typing import List, Optional, Tuple, Union + +import pyarrow as pa + +import datasets +import datasets.config +import datasets.data_files +from datasets.naming import camelcase_to_snakecase, filenames_for_dataset_split + + +logger = datasets.utils.logging.get_logger(__name__) + + +def _get_modification_time(cached_directory_path): + return (Path(cached_directory_path)).stat().st_mtime + + +def _find_hash_in_cache( + dataset_name: str, + config_name: Optional[str], + cache_dir: Optional[str], + config_kwargs: dict, + custom_features: Optional[datasets.Features], +) -> Tuple[str, str, str]: + if config_name or config_kwargs or custom_features: + config_id = datasets.BuilderConfig(config_name or "default").create_config_id( + config_kwargs=config_kwargs, custom_features=custom_features + ) + else: + config_id = None + cache_dir = os.path.expanduser(str(cache_dir or datasets.config.HF_DATASETS_CACHE)) + namespace_and_dataset_name = dataset_name.split("/") + namespace_and_dataset_name[-1] = camelcase_to_snakecase(namespace_and_dataset_name[-1]) + cached_relative_path = "___".join(namespace_and_dataset_name) + cached_datasets_directory_path_root = os.path.join(cache_dir, cached_relative_path) + cached_directory_paths = [ + cached_directory_path + for cached_directory_path in glob.glob( + os.path.join(cached_datasets_directory_path_root, config_id or "*", "*", "*") + ) + if os.path.isdir(cached_directory_path) + and ( + config_kwargs + or custom_features + or json.loads(Path(cached_directory_path, "dataset_info.json").read_text(encoding="utf-8"))["config_name"] + == Path(cached_directory_path).parts[-3] # no extra params => config_id == config_name + ) + ] + if not cached_directory_paths: + cached_directory_paths = [ + cached_directory_path + for cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", "*", "*")) + if os.path.isdir(cached_directory_path) + ] + available_configs = sorted( + {Path(cached_directory_path).parts[-3] for cached_directory_path in cached_directory_paths} + ) + raise ValueError( + f"Couldn't find cache for {dataset_name}" + + (f" for config '{config_id}'" if config_id else "") + + (f"\nAvailable configs in the cache: {available_configs}" if available_configs else "") + ) + # get most recent + cached_directory_path = Path(sorted(cached_directory_paths, key=_get_modification_time)[-1]) + version, hash = cached_directory_path.parts[-2:] + other_configs = [ + Path(_cached_directory_path).parts[-3] + for _cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", version, hash)) + if os.path.isdir(_cached_directory_path) + and ( + config_kwargs + or custom_features + or json.loads(Path(_cached_directory_path, "dataset_info.json").read_text(encoding="utf-8"))["config_name"] + == Path(_cached_directory_path).parts[-3] # no extra params => config_id == config_name + ) + ] + if not config_id and len(other_configs) > 1: + raise ValueError( + f"There are multiple '{dataset_name}' configurations in the cache: {', '.join(other_configs)}" + f"\nPlease specify which configuration to reload from the cache, e.g." + f"\n\tload_dataset('{dataset_name}', '{other_configs[0]}')" + ) + config_name = cached_directory_path.parts[-3] + warning_msg = ( + f"Found the latest cached dataset configuration '{config_name}' at {cached_directory_path} " + f"(last modified on {time.ctime(_get_modification_time(cached_directory_path))})." + ) + logger.warning(warning_msg) + return config_name, version, hash + + +class Cache(datasets.ArrowBasedBuilder): + def __init__( + self, + cache_dir: Optional[str] = None, + dataset_name: Optional[str] = None, + config_name: Optional[str] = None, + version: Optional[str] = "0.0.0", + hash: Optional[str] = None, + base_path: Optional[str] = None, + info: Optional[datasets.DatasetInfo] = None, + features: Optional[datasets.Features] = None, + token: Optional[Union[bool, str]] = None, + use_auth_token="deprecated", + repo_id: Optional[str] = None, + data_files: Optional[Union[str, list, dict, datasets.data_files.DataFilesDict]] = None, + data_dir: Optional[str] = None, + storage_options: Optional[dict] = None, + writer_batch_size: Optional[int] = None, + name="deprecated", + **config_kwargs, + ): + if use_auth_token != "deprecated": + warnings.warn( + "'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0.\n" + f"You can remove this warning by passing 'token={use_auth_token}' instead.", + FutureWarning, + ) + token = use_auth_token + if name != "deprecated": + warnings.warn( + "Parameter 'name' was renamed to 'config_name' in version 2.3.0 and will be removed in 3.0.0.", + category=FutureWarning, + ) + config_name = name + if repo_id is None and dataset_name is None: + raise ValueError("repo_id or dataset_name is required for the Cache dataset builder") + if data_files is not None: + config_kwargs["data_files"] = data_files + if data_dir is not None: + config_kwargs["data_dir"] = data_dir + if hash == "auto" and version == "auto": + config_name, version, hash = _find_hash_in_cache( + dataset_name=repo_id or dataset_name, + config_name=config_name, + cache_dir=cache_dir, + config_kwargs=config_kwargs, + custom_features=features, + ) + elif hash == "auto" or version == "auto": + raise NotImplementedError("Pass both hash='auto' and version='auto' instead") + super().__init__( + cache_dir=cache_dir, + dataset_name=dataset_name, + config_name=config_name, + version=version, + hash=hash, + base_path=base_path, + info=info, + token=token, + repo_id=repo_id, + storage_options=storage_options, + writer_batch_size=writer_batch_size, + ) + + def _info(self) -> datasets.DatasetInfo: + return datasets.DatasetInfo() + + def download_and_prepare(self, output_dir: Optional[str] = None, *args, **kwargs): + if not os.path.exists(self.cache_dir): + raise ValueError(f"Cache directory for {self.dataset_name} doesn't exist at {self.cache_dir}") + if output_dir is not None and output_dir != self.cache_dir: + shutil.copytree(self.cache_dir, output_dir) + + def _split_generators(self, dl_manager): + # used to stream from cache + if isinstance(self.info.splits, datasets.SplitDict): + split_infos: List[datasets.SplitInfo] = list(self.info.splits.values()) + else: + raise ValueError(f"Missing splits info for {self.dataset_name} in cache directory {self.cache_dir}") + return [ + datasets.SplitGenerator( + name=split_info.name, + gen_kwargs={ + "files": filenames_for_dataset_split( + self.cache_dir, + dataset_name=self.dataset_name, + split=split_info.name, + filetype_suffix="arrow", + shard_lengths=split_info.shard_lengths, + ) + }, + ) + for split_info in split_infos + ] + + def _generate_tables(self, files): + # used to stream from cache + for file_idx, file in enumerate(files): + with open(file, "rb") as f: + try: + for batch_idx, record_batch in enumerate(pa.ipc.open_stream(f)): + pa_table = pa.Table.from_batches([record_batch]) + # Uncomment for debugging (will print the Arrow table size and elements) + # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}") + # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows))) + yield f"{file_idx}_{batch_idx}", pa_table + except ValueError as e: + logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}") + raise diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/csv/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/csv/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7ee16b63e8f47a14d7f0897a53b7b055efedc1c7 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8dba01e79e7ce1f6497dd320efbb20360b730d6a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/csv/__pycache__/csv.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/csv/csv.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/csv/csv.py new file mode 100644 index 0000000000000000000000000000000000000000..16f004aa3b48b6346a3e7307b688e7aecec25ab7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/csv/csv.py @@ -0,0 +1,198 @@ +import itertools +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Union + +import pandas as pd +import pyarrow as pa + +import datasets +import datasets.config +from datasets.features.features import require_storage_cast +from datasets.table import table_cast +from datasets.utils.py_utils import Literal + + +logger = datasets.utils.logging.get_logger(__name__) + +_PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS = ["names", "prefix"] +_PANDAS_READ_CSV_DEPRECATED_PARAMETERS = ["warn_bad_lines", "error_bad_lines", "mangle_dupe_cols"] +_PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS = ["encoding_errors", "on_bad_lines"] +_PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS = ["date_format"] +_PANDAS_READ_CSV_DEPRECATED_2_2_0_PARAMETERS = ["verbose"] + + +@dataclass +class CsvConfig(datasets.BuilderConfig): + """BuilderConfig for CSV.""" + + sep: str = "," + delimiter: Optional[str] = None + header: Optional[Union[int, List[int], str]] = "infer" + names: Optional[List[str]] = None + column_names: Optional[List[str]] = None + index_col: Optional[Union[int, str, List[int], List[str]]] = None + usecols: Optional[Union[List[int], List[str]]] = None + prefix: Optional[str] = None + mangle_dupe_cols: bool = True + engine: Optional[Literal["c", "python", "pyarrow"]] = None + converters: Dict[Union[int, str], Callable[[Any], Any]] = None + true_values: Optional[list] = None + false_values: Optional[list] = None + skipinitialspace: bool = False + skiprows: Optional[Union[int, List[int]]] = None + nrows: Optional[int] = None + na_values: Optional[Union[str, List[str]]] = None + keep_default_na: bool = True + na_filter: bool = True + verbose: bool = False + skip_blank_lines: bool = True + thousands: Optional[str] = None + decimal: str = "." + lineterminator: Optional[str] = None + quotechar: str = '"' + quoting: int = 0 + escapechar: Optional[str] = None + comment: Optional[str] = None + encoding: Optional[str] = None + dialect: Optional[str] = None + error_bad_lines: bool = True + warn_bad_lines: bool = True + skipfooter: int = 0 + doublequote: bool = True + memory_map: bool = False + float_precision: Optional[str] = None + chunksize: int = 10_000 + features: Optional[datasets.Features] = None + encoding_errors: Optional[str] = "strict" + on_bad_lines: Literal["error", "warn", "skip"] = "error" + date_format: Optional[str] = None + + def __post_init__(self): + super().__post_init__() + if self.delimiter is not None: + self.sep = self.delimiter + if self.column_names is not None: + self.names = self.column_names + + @property + def pd_read_csv_kwargs(self): + pd_read_csv_kwargs = { + "sep": self.sep, + "header": self.header, + "names": self.names, + "index_col": self.index_col, + "usecols": self.usecols, + "prefix": self.prefix, + "mangle_dupe_cols": self.mangle_dupe_cols, + "engine": self.engine, + "converters": self.converters, + "true_values": self.true_values, + "false_values": self.false_values, + "skipinitialspace": self.skipinitialspace, + "skiprows": self.skiprows, + "nrows": self.nrows, + "na_values": self.na_values, + "keep_default_na": self.keep_default_na, + "na_filter": self.na_filter, + "verbose": self.verbose, + "skip_blank_lines": self.skip_blank_lines, + "thousands": self.thousands, + "decimal": self.decimal, + "lineterminator": self.lineterminator, + "quotechar": self.quotechar, + "quoting": self.quoting, + "escapechar": self.escapechar, + "comment": self.comment, + "encoding": self.encoding, + "dialect": self.dialect, + "error_bad_lines": self.error_bad_lines, + "warn_bad_lines": self.warn_bad_lines, + "skipfooter": self.skipfooter, + "doublequote": self.doublequote, + "memory_map": self.memory_map, + "float_precision": self.float_precision, + "chunksize": self.chunksize, + "encoding_errors": self.encoding_errors, + "on_bad_lines": self.on_bad_lines, + "date_format": self.date_format, + } + + # some kwargs must not be passed if they don't have a default value + # some others are deprecated and we can also not pass them if they are the default value + for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS: + if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig(), pd_read_csv_parameter): + del pd_read_csv_kwargs[pd_read_csv_parameter] + + # Remove 1.3 new arguments + if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3): + for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS: + del pd_read_csv_kwargs[pd_read_csv_parameter] + + # Remove 2.0 new arguments + if not (datasets.config.PANDAS_VERSION.major >= 2): + for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS: + del pd_read_csv_kwargs[pd_read_csv_parameter] + + # Remove 2.2 deprecated arguments + if datasets.config.PANDAS_VERSION.release >= (2, 2): + for pd_read_csv_parameter in _PANDAS_READ_CSV_DEPRECATED_2_2_0_PARAMETERS: + if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig(), pd_read_csv_parameter): + del pd_read_csv_kwargs[pd_read_csv_parameter] + + return pd_read_csv_kwargs + + +class Csv(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = CsvConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + data_files = dl_manager.download_and_extract(self.config.data_files) + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + files = [dl_manager.iter_files(file) for file in files] + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + schema = self.config.features.arrow_schema + if all(not require_storage_cast(feature) for feature in self.config.features.values()): + # cheaper cast + pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema) + else: + # more expensive cast; allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, schema) + return pa_table + + def _generate_tables(self, files): + schema = self.config.features.arrow_schema if self.config.features else None + # dtype allows reading an int column as str + dtype = ( + { + name: dtype.to_pandas_dtype() if not require_storage_cast(feature) else object + for name, dtype, feature in zip(schema.names, schema.types, self.config.features.values()) + } + if schema is not None + else None + ) + for file_idx, file in enumerate(itertools.chain.from_iterable(files)): + csv_file_reader = pd.read_csv(file, iterator=True, dtype=dtype, **self.config.pd_read_csv_kwargs) + try: + for batch_idx, df in enumerate(csv_file_reader): + pa_table = pa.Table.from_pandas(df) + # Uncomment for debugging (will print the Arrow table size and elements) + # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}") + # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows))) + yield (file_idx, batch_idx), self._cast_table(pa_table) + except ValueError as e: + logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}") + raise diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1f0c878d30b8e8c04b0a4fb727ac4d5ca89465b7 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c4fb569967ef67dea3c8513c11dfd5f98db6609 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/__pycache__/folder_based_builder.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..7b71dc407ae4d0f212cd5623763de63d11e60670 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/folder_based_builder/folder_based_builder.py @@ -0,0 +1,409 @@ +import collections +import itertools +import os +from dataclasses import dataclass +from typing import List, Optional, Tuple, Type + +import pandas as pd +import pyarrow as pa +import pyarrow.json as paj + +import datasets +from datasets.features.features import FeatureType +from datasets.tasks.base import TaskTemplate + + +logger = datasets.utils.logging.get_logger(__name__) + + +def count_path_segments(path): + return path.replace("\\", "/").count("/") + + +@dataclass +class FolderBasedBuilderConfig(datasets.BuilderConfig): + """BuilderConfig for AutoFolder.""" + + features: Optional[datasets.Features] = None + drop_labels: bool = None + drop_metadata: bool = None + + def __post_init__(self): + super().__post_init__() + + +class FolderBasedBuilder(datasets.GeneratorBasedBuilder): + """ + Base class for generic data loaders for vision and image data. + + + Abstract class attributes to be overridden by a child class: + BASE_FEATURE: feature object to decode data (i.e. datasets.Image, datasets.Audio, ...) + BASE_COLUMN_NAME: string key name of a base feature (i.e. "image", "audio", ...) + BUILDER_CONFIG_CLASS: builder config inherited from `folder_based_builder.FolderBasedBuilderConfig` + EXTENSIONS: list of allowed extensions (only files with these extensions and METADATA_FILENAME files + will be included in a dataset) + CLASSIFICATION_TASK: classification task to use if labels are obtained from the folder structure + """ + + BASE_FEATURE: Type[FeatureType] + BASE_COLUMN_NAME: str + BUILDER_CONFIG_CLASS: FolderBasedBuilderConfig + EXTENSIONS: List[str] + CLASSIFICATION_TASK: TaskTemplate + + METADATA_FILENAMES: List[str] = ["metadata.csv", "metadata.jsonl"] + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + # Do an early pass if: + # * `drop_labels` is None (default) or False, to infer the class labels + # * `drop_metadata` is None (default) or False, to find the metadata files + do_analyze = not self.config.drop_labels or not self.config.drop_metadata + labels, path_depths = set(), set() + metadata_files = collections.defaultdict(set) + + def analyze(files_or_archives, downloaded_files_or_dirs, split): + if len(downloaded_files_or_dirs) == 0: + return + # The files are separated from the archives at this point, so check the first sample + # to see if it's a file or a directory and iterate accordingly + if os.path.isfile(downloaded_files_or_dirs[0]): + original_files, downloaded_files = files_or_archives, downloaded_files_or_dirs + for original_file, downloaded_file in zip(original_files, downloaded_files): + original_file, downloaded_file = str(original_file), str(downloaded_file) + _, original_file_ext = os.path.splitext(original_file) + if original_file_ext.lower() in self.EXTENSIONS: + if not self.config.drop_labels: + labels.add(os.path.basename(os.path.dirname(original_file))) + path_depths.add(count_path_segments(original_file)) + elif os.path.basename(original_file) in self.METADATA_FILENAMES: + metadata_files[split].add((original_file, downloaded_file)) + else: + original_file_name = os.path.basename(original_file) + logger.debug( + f"The file '{original_file_name}' was ignored: it is not an image, and is not {self.METADATA_FILENAMES} either." + ) + else: + archives, downloaded_dirs = files_or_archives, downloaded_files_or_dirs + for archive, downloaded_dir in zip(archives, downloaded_dirs): + archive, downloaded_dir = str(archive), str(downloaded_dir) + for downloaded_dir_file in dl_manager.iter_files(downloaded_dir): + _, downloaded_dir_file_ext = os.path.splitext(downloaded_dir_file) + if downloaded_dir_file_ext in self.EXTENSIONS: + if not self.config.drop_labels: + labels.add(os.path.basename(os.path.dirname(downloaded_dir_file))) + path_depths.add(count_path_segments(downloaded_dir_file)) + elif os.path.basename(downloaded_dir_file) in self.METADATA_FILENAMES: + metadata_files[split].add((None, downloaded_dir_file)) + else: + archive_file_name = os.path.basename(archive) + original_file_name = os.path.basename(downloaded_dir_file) + logger.debug( + f"The file '{original_file_name}' from the archive '{archive_file_name}' was ignored: it is not an {self.BASE_COLUMN_NAME}, and is not {self.METADATA_FILENAMES} either." + ) + + data_files = self.config.data_files + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + files, archives = self._split_files_and_archives(files) + downloaded_files = dl_manager.download(files) + downloaded_dirs = dl_manager.download_and_extract(archives) + if do_analyze: # drop_metadata is None or False, drop_labels is None or False + logger.info(f"Searching for labels and/or metadata files in {split_name} data files...") + analyze(files, downloaded_files, split_name) + analyze(archives, downloaded_dirs, split_name) + + if metadata_files: + # add metadata if `metadata_files` are found and `drop_metadata` is None (default) or False + add_metadata = not self.config.drop_metadata + # if `metadata_files` are found, add labels only if + # `drop_labels` is set up to False explicitly (not-default behavior) + add_labels = self.config.drop_labels is False + else: + # if `metadata_files` are not found, don't add metadata + add_metadata = False + # if `metadata_files` are not found and `drop_labels` is None (default) - + # add labels if files are on the same level in directory hierarchy and there is more than one label + add_labels = ( + (len(labels) > 1 and len(path_depths) == 1) + if self.config.drop_labels is None + else not self.config.drop_labels + ) + + if add_labels: + logger.info("Adding the labels inferred from data directories to the dataset's features...") + if add_metadata: + logger.info("Adding metadata to the dataset...") + else: + add_labels, add_metadata, metadata_files = False, False, {} + + splits.append( + datasets.SplitGenerator( + name=split_name, + gen_kwargs={ + "files": list(zip(files, downloaded_files)) + + [(None, dl_manager.iter_files(downloaded_dir)) for downloaded_dir in downloaded_dirs], + "metadata_files": metadata_files, + "split_name": split_name, + "add_labels": add_labels, + "add_metadata": add_metadata, + }, + ) + ) + + if add_metadata: + # Verify that: + # * all metadata files have the same set of features + # * the `file_name` key is one of the metadata keys and is of type string + features_per_metadata_file: List[Tuple[str, datasets.Features]] = [] + + # Check that all metadata files share the same format + metadata_ext = { + os.path.splitext(original_metadata_file)[-1] + for original_metadata_file, _ in itertools.chain.from_iterable(metadata_files.values()) + } + if len(metadata_ext) > 1: + raise ValueError(f"Found metadata files with different extensions: {list(metadata_ext)}") + metadata_ext = metadata_ext.pop() + + for _, downloaded_metadata_file in itertools.chain.from_iterable(metadata_files.values()): + pa_metadata_table = self._read_metadata(downloaded_metadata_file, metadata_ext=metadata_ext) + features_per_metadata_file.append( + (downloaded_metadata_file, datasets.Features.from_arrow_schema(pa_metadata_table.schema)) + ) + for downloaded_metadata_file, metadata_features in features_per_metadata_file: + if metadata_features != features_per_metadata_file[0][1]: + raise ValueError( + f"Metadata files {downloaded_metadata_file} and {features_per_metadata_file[0][0]} have different features: {features_per_metadata_file[0]} != {metadata_features}" + ) + metadata_features = features_per_metadata_file[0][1] + if "file_name" not in metadata_features: + raise ValueError("`file_name` must be present as dictionary key in metadata files") + if metadata_features["file_name"] != datasets.Value("string"): + raise ValueError("`file_name` key must be a string") + del metadata_features["file_name"] + else: + metadata_features = None + + # Normally, we would do this in _info, but we need to know the labels and/or metadata + # before building the features + if self.config.features is None: + if add_labels: + self.info.features = datasets.Features( + { + self.BASE_COLUMN_NAME: self.BASE_FEATURE(), + "label": datasets.ClassLabel(names=sorted(labels)), + } + ) + self.info.task_templates = [self.CLASSIFICATION_TASK.align_with_features(self.info.features)] + else: + self.info.features = datasets.Features({self.BASE_COLUMN_NAME: self.BASE_FEATURE()}) + + if add_metadata: + # Warn if there are duplicated keys in metadata compared to the existing features + # (`BASE_COLUMN_NAME`, optionally "label") + duplicated_keys = set(self.info.features) & set(metadata_features) + if duplicated_keys: + logger.warning( + f"Ignoring metadata columns {list(duplicated_keys)} as they are already present in " + f"the features dictionary." + ) + # skip metadata duplicated keys + self.info.features.update( + { + feature: metadata_features[feature] + for feature in metadata_features + if feature not in duplicated_keys + } + ) + + return splits + + def _split_files_and_archives(self, data_files): + files, archives = [], [] + for data_file in data_files: + _, data_file_ext = os.path.splitext(data_file) + if data_file_ext.lower() in self.EXTENSIONS: + files.append(data_file) + elif os.path.basename(data_file) in self.METADATA_FILENAMES: + files.append(data_file) + else: + archives.append(data_file) + return files, archives + + def _read_metadata(self, metadata_file, metadata_ext: str = ""): + if metadata_ext == ".csv": + # Use `pd.read_csv` (although slower) instead of `pyarrow.csv.read_csv` for reading CSV files for consistency with the CSV packaged module + return pa.Table.from_pandas(pd.read_csv(metadata_file)) + else: + with open(metadata_file, "rb") as f: + return paj.read_json(f) + + def _generate_examples(self, files, metadata_files, split_name, add_metadata, add_labels): + split_metadata_files = metadata_files.get(split_name, []) + sample_empty_metadata = ( + {k: None for k in self.info.features if k != self.BASE_COLUMN_NAME} if self.info.features else {} + ) + last_checked_dir = None + metadata_dir = None + metadata_dict = None + downloaded_metadata_file = None + + metadata_ext = "" + if split_metadata_files: + metadata_ext = { + os.path.splitext(original_metadata_file)[-1] for original_metadata_file, _ in split_metadata_files + } + metadata_ext = metadata_ext.pop() + + file_idx = 0 + for original_file, downloaded_file_or_dir in files: + if original_file is not None: + _, original_file_ext = os.path.splitext(original_file) + if original_file_ext.lower() in self.EXTENSIONS: + if add_metadata: + # If the file is a file of a needed type, and we've just entered a new directory, + # find the nereast metadata file (by counting path segments) for the directory + current_dir = os.path.dirname(original_file) + if last_checked_dir is None or last_checked_dir != current_dir: + last_checked_dir = current_dir + metadata_file_candidates = [ + ( + os.path.relpath(original_file, os.path.dirname(metadata_file_candidate)), + metadata_file_candidate, + downloaded_metadata_file, + ) + for metadata_file_candidate, downloaded_metadata_file in split_metadata_files + if metadata_file_candidate + is not None # ignore metadata_files that are inside archives + and not os.path.relpath( + original_file, os.path.dirname(metadata_file_candidate) + ).startswith("..") + ] + if metadata_file_candidates: + _, metadata_file, downloaded_metadata_file = min( + metadata_file_candidates, key=lambda x: count_path_segments(x[0]) + ) + pa_metadata_table = self._read_metadata( + downloaded_metadata_file, metadata_ext=metadata_ext + ) + pa_file_name_array = pa_metadata_table["file_name"] + pa_metadata_table = pa_metadata_table.drop(["file_name"]) + metadata_dir = os.path.dirname(metadata_file) + metadata_dict = { + os.path.normpath(file_name).replace("\\", "/"): sample_metadata + for file_name, sample_metadata in zip( + pa_file_name_array.to_pylist(), pa_metadata_table.to_pylist() + ) + } + else: + raise ValueError( + f"One or several metadata{metadata_ext} were found, but not in the same directory or in a parent directory of {downloaded_file_or_dir}." + ) + if metadata_dir is not None and downloaded_metadata_file is not None: + file_relpath = os.path.relpath(original_file, metadata_dir) + file_relpath = file_relpath.replace("\\", "/") + if file_relpath not in metadata_dict: + raise ValueError( + f"{self.BASE_COLUMN_NAME} at {file_relpath} doesn't have metadata in {downloaded_metadata_file}." + ) + sample_metadata = metadata_dict[file_relpath] + else: + raise ValueError( + f"One or several metadata{metadata_ext} were found, but not in the same directory or in a parent directory of {downloaded_file_or_dir}." + ) + else: + sample_metadata = {} + if add_labels: + sample_label = {"label": os.path.basename(os.path.dirname(original_file))} + else: + sample_label = {} + yield ( + file_idx, + { + **sample_empty_metadata, + self.BASE_COLUMN_NAME: downloaded_file_or_dir, + **sample_metadata, + **sample_label, + }, + ) + file_idx += 1 + else: + for downloaded_dir_file in downloaded_file_or_dir: + _, downloaded_dir_file_ext = os.path.splitext(downloaded_dir_file) + if downloaded_dir_file_ext.lower() in self.EXTENSIONS: + if add_metadata: + current_dir = os.path.dirname(downloaded_dir_file) + if last_checked_dir is None or last_checked_dir != current_dir: + last_checked_dir = current_dir + metadata_file_candidates = [ + ( + os.path.relpath( + downloaded_dir_file, os.path.dirname(downloaded_metadata_file) + ), + metadata_file_candidate, + downloaded_metadata_file, + ) + for metadata_file_candidate, downloaded_metadata_file in split_metadata_files + if metadata_file_candidate + is None # ignore metadata_files that are not inside archives + and not os.path.relpath( + downloaded_dir_file, os.path.dirname(downloaded_metadata_file) + ).startswith("..") + ] + if metadata_file_candidates: + _, metadata_file, downloaded_metadata_file = min( + metadata_file_candidates, key=lambda x: count_path_segments(x[0]) + ) + pa_metadata_table = self._read_metadata( + downloaded_metadata_file, metadata_ext=metadata_ext + ) + pa_file_name_array = pa_metadata_table["file_name"] + pa_metadata_table = pa_metadata_table.drop(["file_name"]) + metadata_dir = os.path.dirname(downloaded_metadata_file) + metadata_dict = { + os.path.normpath(file_name).replace("\\", "/"): sample_metadata + for file_name, sample_metadata in zip( + pa_file_name_array.to_pylist(), pa_metadata_table.to_pylist() + ) + } + else: + raise ValueError( + f"One or several metadata{metadata_ext} were found, but not in the same directory or in a parent directory of {downloaded_dir_file}." + ) + if metadata_dir is not None and downloaded_metadata_file is not None: + downloaded_dir_file_relpath = os.path.relpath(downloaded_dir_file, metadata_dir) + downloaded_dir_file_relpath = downloaded_dir_file_relpath.replace("\\", "/") + if downloaded_dir_file_relpath not in metadata_dict: + raise ValueError( + f"{self.BASE_COLUMN_NAME} at {downloaded_dir_file_relpath} doesn't have metadata in {downloaded_metadata_file}." + ) + sample_metadata = metadata_dict[downloaded_dir_file_relpath] + else: + raise ValueError( + f"One or several metadata{metadata_ext} were found, but not in the same directory or in a parent directory of {downloaded_dir_file}." + ) + else: + sample_metadata = {} + if add_labels: + sample_label = {"label": os.path.basename(os.path.dirname(downloaded_dir_file))} + else: + sample_label = {} + yield ( + file_idx, + { + **sample_empty_metadata, + self.BASE_COLUMN_NAME: downloaded_dir_file, + **sample_metadata, + **sample_label, + }, + ) + file_idx += 1 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/generator/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/generator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..318671e61ac3b4715678df8a407e5659aafe03f0 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..158946b84b79f2a49cb2e9214ae11189041c02e6 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/generator/__pycache__/generator.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/generator/generator.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/generator/generator.py new file mode 100644 index 0000000000000000000000000000000000000000..336942f2edc522edaeb33e387f024d968bfc595a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/generator/generator.py @@ -0,0 +1,33 @@ +from dataclasses import dataclass +from typing import Callable, Optional + +import datasets + + +@dataclass +class GeneratorConfig(datasets.BuilderConfig): + generator: Optional[Callable] = None + gen_kwargs: Optional[dict] = None + features: Optional[datasets.Features] = None + + def __post_init__(self): + super().__post_init__() + if self.generator is None: + raise ValueError("generator must be specified") + + if self.gen_kwargs is None: + self.gen_kwargs = {} + + +class Generator(datasets.GeneratorBasedBuilder): + BUILDER_CONFIG_CLASS = GeneratorConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs=self.config.gen_kwargs)] + + def _generate_examples(self, **gen_kwargs): + for idx, ex in enumerate(self.config.generator(**gen_kwargs)): + yield idx, ex diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d1b2635acd24ee7b954eff05161eabf5b9e3d559 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..75d8ebacf5720abce36f52bfe832edce1cdaee93 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/__pycache__/imagefolder.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py new file mode 100644 index 0000000000000000000000000000000000000000..16fbcd005d459b74e45f17586fa75634848f2442 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/imagefolder/imagefolder.py @@ -0,0 +1,107 @@ +from typing import List + +import datasets +from datasets.tasks import ImageClassification + +from ..folder_based_builder import folder_based_builder + + +logger = datasets.utils.logging.get_logger(__name__) + + +class ImageFolderConfig(folder_based_builder.FolderBasedBuilderConfig): + """BuilderConfig for ImageFolder.""" + + drop_labels: bool = None + drop_metadata: bool = None + + def __post_init__(self): + super().__post_init__() + + +class ImageFolder(folder_based_builder.FolderBasedBuilder): + BASE_FEATURE = datasets.Image + BASE_COLUMN_NAME = "image" + BUILDER_CONFIG_CLASS = ImageFolderConfig + EXTENSIONS: List[str] # definition at the bottom of the script + CLASSIFICATION_TASK = ImageClassification(image_column="image", label_column="label") + + +# Obtained with: +# ``` +# import PIL.Image +# IMAGE_EXTENSIONS = [] +# PIL.Image.init() +# for ext, format in PIL.Image.EXTENSION.items(): +# if format in PIL.Image.OPEN: +# IMAGE_EXTENSIONS.append(ext[1:]) +# ``` +# We intentionally do not run this code on launch because: +# (1) Pillow is an optional dependency, so importing Pillow in global namespace is not allowed +# (2) To ensure the list of supported extensions is deterministic +IMAGE_EXTENSIONS = [ + ".blp", + ".bmp", + ".dib", + ".bufr", + ".cur", + ".pcx", + ".dcx", + ".dds", + ".ps", + ".eps", + ".fit", + ".fits", + ".fli", + ".flc", + ".ftc", + ".ftu", + ".gbr", + ".gif", + ".grib", + ".h5", + ".hdf", + ".png", + ".apng", + ".jp2", + ".j2k", + ".jpc", + ".jpf", + ".jpx", + ".j2c", + ".icns", + ".ico", + ".im", + ".iim", + ".tif", + ".tiff", + ".jfif", + ".jpe", + ".jpg", + ".jpeg", + ".mpg", + ".mpeg", + ".msp", + ".pcd", + ".pxr", + ".pbm", + ".pgm", + ".ppm", + ".pnm", + ".psd", + ".bw", + ".rgb", + ".rgba", + ".sgi", + ".ras", + ".tga", + ".icb", + ".vda", + ".vst", + ".webp", + ".wmf", + ".emf", + ".xbm", + ".xpm", +] +ImageFolder.EXTENSIONS = IMAGE_EXTENSIONS diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/json/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/json/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/json/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/json/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b3c7a3844a720556bbe3e91d7d87cad878f8d7fa Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/json/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/json/__pycache__/json.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/json/__pycache__/json.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a343167247f837c9fcf698dfdd05033f19f4d5fc Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/json/__pycache__/json.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/json/json.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/json/json.py new file mode 100644 index 0000000000000000000000000000000000000000..40539b625c90326f35830aaa3a867f5c3c7c1207 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/json/json.py @@ -0,0 +1,171 @@ +import io +import itertools +from dataclasses import dataclass +from typing import Optional + +import pandas as pd +import pyarrow as pa +import pyarrow.json as paj + +import datasets +from datasets.table import table_cast +from datasets.utils.file_utils import readline + + +logger = datasets.utils.logging.get_logger(__name__) + + +def ujson_dumps(*args, **kwargs): + try: + return pd.io.json.ujson_dumps(*args, **kwargs) + except AttributeError: + # Before pandas-2.2.0, ujson_dumps was renamed to dumps: import ujson_dumps as dumps + return pd.io.json.dumps(*args, **kwargs) + + +def ujson_loads(*args, **kwargs): + try: + return pd.io.json.ujson_loads(*args, **kwargs) + except AttributeError: + # Before pandas-2.2.0, ujson_loads was renamed to loads: import ujson_loads as loads + return pd.io.json.loads(*args, **kwargs) + + +@dataclass +class JsonConfig(datasets.BuilderConfig): + """BuilderConfig for JSON.""" + + features: Optional[datasets.Features] = None + encoding: str = "utf-8" + encoding_errors: Optional[str] = None + field: Optional[str] = None + use_threads: bool = True # deprecated + block_size: Optional[int] = None # deprecated + chunksize: int = 10 << 20 # 10MB + newlines_in_values: Optional[bool] = None + + def __post_init__(self): + super().__post_init__() + + +class Json(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = JsonConfig + + def _info(self): + if self.config.block_size is not None: + logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead") + self.config.chunksize = self.config.block_size + if self.config.use_threads is not True: + logger.warning( + "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore." + ) + if self.config.newlines_in_values is not None: + raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported") + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + data_files = dl_manager.download_and_extract(self.config.data_files) + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + files = [dl_manager.iter_files(file) for file in files] + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + # adding missing columns + for column_name in set(self.config.features) - set(pa_table.column_names): + type = self.config.features.arrow_schema.field(column_name).type + pa_table = pa_table.append_column(column_name, pa.array([None] * len(pa_table), type=type)) + # more expensive cast to support nested structures with keys in a different order + # allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, self.config.features.arrow_schema) + return pa_table + + def _generate_tables(self, files): + for file_idx, file in enumerate(itertools.chain.from_iterable(files)): + # If the file is one json object and if we need to look at the items in one specific field + if self.config.field is not None: + with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f: + dataset = ujson_loads(f.read()) + # We keep only the field we are interested in + dataset = dataset[self.config.field] + df = pd.read_json(io.StringIO(ujson_dumps(dataset)), dtype_backend="pyarrow") + if df.columns.tolist() == [0]: + df.columns = list(self.config.features) if self.config.features else ["text"] + pa_table = pa.Table.from_pandas(df, preserve_index=False) + yield file_idx, self._cast_table(pa_table) + + # If the file has one json object per line + else: + with open(file, "rb") as f: + batch_idx = 0 + # Use block_size equal to the chunk size divided by 32 to leverage multithreading + # Set a default minimum value of 16kB if the chunk size is really small + block_size = max(self.config.chunksize // 32, 16 << 10) + encoding_errors = ( + self.config.encoding_errors if self.config.encoding_errors is not None else "strict" + ) + while True: + batch = f.read(self.config.chunksize) + if not batch: + break + # Finish current line + try: + batch += f.readline() + except (AttributeError, io.UnsupportedOperation): + batch += readline(f) + # PyArrow only accepts utf-8 encoded bytes + if self.config.encoding != "utf-8": + batch = batch.decode(self.config.encoding, errors=encoding_errors).encode("utf-8") + try: + while True: + try: + pa_table = paj.read_json( + io.BytesIO(batch), read_options=paj.ReadOptions(block_size=block_size) + ) + break + except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e: + if ( + isinstance(e, pa.ArrowInvalid) + and "straddling" not in str(e) + or block_size > len(batch) + ): + raise + else: + # Increase the block size in case it was too small. + # The block size will be reset for the next file. + logger.debug( + f"Batch of {len(batch)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}." + ) + block_size *= 2 + except pa.ArrowInvalid as e: + try: + with open( + file, encoding=self.config.encoding, errors=self.config.encoding_errors + ) as f: + df = pd.read_json(f, dtype_backend="pyarrow") + except ValueError: + logger.error(f"Failed to load JSON from file '{file}' with error {type(e)}: {e}") + raise e + if df.columns.tolist() == [0]: + df.columns = list(self.config.features) if self.config.features else ["text"] + try: + pa_table = pa.Table.from_pandas(df, preserve_index=False) + except pa.ArrowInvalid as e: + logger.error( + f"Failed to convert pandas DataFrame to Arrow Table from file '{file}' with error {type(e)}: {e}" + ) + raise ValueError( + f"Failed to convert pandas DataFrame to Arrow Table from file {file}." + ) from None + yield file_idx, self._cast_table(pa_table) + break + yield (file_idx, batch_idx), self._cast_table(pa_table) + batch_idx += 1 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cf2904fbdf1a6ce14f2b505acfa862038574971e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__pycache__/pandas.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__pycache__/pandas.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e5ca698c02f4105cccdbdac5b9250ada68f2ae9a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/pandas/__pycache__/pandas.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/pandas/pandas.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/pandas/pandas.py new file mode 100644 index 0000000000000000000000000000000000000000..d1eb50d33c899d83421167bf6dc70680e437e1e4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/pandas/pandas.py @@ -0,0 +1,65 @@ +import itertools +import warnings +from dataclasses import dataclass +from typing import Optional + +import pandas as pd +import pyarrow as pa + +import datasets +from datasets.table import table_cast + + +@dataclass +class PandasConfig(datasets.BuilderConfig): + """BuilderConfig for Pandas.""" + + features: Optional[datasets.Features] = None + + def __post_init__(self): + super().__post_init__() + + +class Pandas(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = PandasConfig + + def _info(self): + warnings.warn( + "The Pandas builder is deprecated and will be removed in the next major version of datasets.", + FutureWarning, + ) + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + data_files = dl_manager.download_and_extract(self.config.data_files) + if isinstance(data_files, (str, list, tuple)): + files = data_files + if isinstance(files, str): + files = [files] + # Use `dl_manager.iter_files` to skip hidden files in an extracted archive + files = [dl_manager.iter_files(file) for file in files] + return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"files": files})] + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + # Use `dl_manager.iter_files` to skip hidden files in an extracted archive + files = [dl_manager.iter_files(file) for file in files] + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + # more expensive cast to support nested features with keys in a different order + # allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, self.config.features.arrow_schema) + return pa_table + + def _generate_tables(self, files): + for i, file in enumerate(itertools.chain.from_iterable(files)): + with open(file, "rb") as f: + pa_table = pa.Table.from_pandas(pd.read_pickle(f)) + yield i, self._cast_table(pa_table) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b203561895b901255f2d730931cf6562240f4dc1 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__pycache__/parquet.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__pycache__/parquet.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca03565d1802c2ea2b8eb361ebe92e7e65c2866f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/parquet/__pycache__/parquet.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/parquet/parquet.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/parquet/parquet.py new file mode 100644 index 0000000000000000000000000000000000000000..f6ec2e06cc095eb5559ce99d59392f81f60b3ace --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/parquet/parquet.py @@ -0,0 +1,96 @@ +import itertools +from dataclasses import dataclass +from typing import List, Optional + +import pyarrow as pa +import pyarrow.parquet as pq + +import datasets +from datasets.table import table_cast + + +logger = datasets.utils.logging.get_logger(__name__) + + +@dataclass +class ParquetConfig(datasets.BuilderConfig): + """BuilderConfig for Parquet.""" + + batch_size: Optional[int] = None + columns: Optional[List[str]] = None + features: Optional[datasets.Features] = None + + def __post_init__(self): + super().__post_init__() + + +class Parquet(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = ParquetConfig + + def _info(self): + if ( + self.config.columns is not None + and self.config.features is not None + and set(self.config.columns) != set(self.config.features) + ): + raise ValueError( + "The columns and features argument must contain the same columns, but got ", + f"{self.config.columns} and {self.config.features}", + ) + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + data_files = dl_manager.download_and_extract(self.config.data_files) + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + # Use `dl_manager.iter_files` to skip hidden files in an extracted archive + files = [dl_manager.iter_files(file) for file in files] + # Infer features if they are stored in the arrow schema + if self.info.features is None: + for file in itertools.chain.from_iterable(files): + with open(file, "rb") as f: + self.info.features = datasets.Features.from_arrow_schema(pq.read_schema(f)) + break + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + if self.config.columns is not None and set(self.config.columns) != set(self.info.features): + self.info.features = datasets.Features( + {col: feat for col, feat in self.info.features.items() if col in self.config.columns} + ) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.info.features is not None: + # more expensive cast to support nested features with keys in a different order + # allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, self.info.features.arrow_schema) + return pa_table + + def _generate_tables(self, files): + if self.config.features is not None and self.config.columns is not None: + if sorted(field.name for field in self.info.features.arrow_schema) != sorted(self.config.columns): + raise ValueError( + f"Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'" + ) + for file_idx, file in enumerate(itertools.chain.from_iterable(files)): + with open(file, "rb") as f: + parquet_file = pq.ParquetFile(f) + if parquet_file.metadata.num_row_groups > 0: + batch_size = self.config.batch_size or parquet_file.metadata.row_group(0).num_rows + try: + for batch_idx, record_batch in enumerate( + parquet_file.iter_batches(batch_size=batch_size, columns=self.config.columns) + ): + pa_table = pa.Table.from_batches([record_batch]) + # Uncomment for debugging (will print the Arrow table size and elements) + # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}") + # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows))) + yield f"{file_idx}_{batch_idx}", self._cast_table(pa_table) + except ValueError as e: + logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}") + raise diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/spark/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/spark/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/spark/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/spark/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7660e4905db427254f807e23dfbd134469eb72d5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/spark/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/spark/__pycache__/spark.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/spark/__pycache__/spark.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f1a8a1af70dc4aa35705b3b3169960b585f5fb4 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/spark/__pycache__/spark.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/spark/spark.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/spark/spark.py new file mode 100644 index 0000000000000000000000000000000000000000..c21cb3dd981ca599f65738dcb7a54eb62ebc3f0d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/spark/spark.py @@ -0,0 +1,366 @@ +import os +import posixpath +import uuid +from dataclasses import dataclass +from itertools import islice +from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, Union + +import numpy as np +import pyarrow as pa + +import datasets +from datasets.arrow_writer import ArrowWriter, ParquetWriter +from datasets.config import MAX_SHARD_SIZE +from datasets.filesystems import ( + is_remote_filesystem, + rename, +) +from datasets.iterable_dataset import _BaseExamplesIterable +from datasets.utils import experimental +from datasets.utils.py_utils import convert_file_size_to_int + + +logger = datasets.utils.logging.get_logger(__name__) + +if TYPE_CHECKING: + import pyspark + import pyspark.sql + + +@dataclass +class SparkConfig(datasets.BuilderConfig): + """BuilderConfig for Spark.""" + + features: Optional[datasets.Features] = None + + def __post_init__(self): + super().__post_init__() + + +def _reorder_dataframe_by_partition(df: "pyspark.sql.DataFrame", new_partition_order: List[int]): + df_combined = df.select("*").where(f"part_id = {new_partition_order[0]}") + for partition_id in new_partition_order[1:]: + partition_df = df.select("*").where(f"part_id = {partition_id}") + df_combined = df_combined.union(partition_df) + return df_combined + + +def _generate_iterable_examples( + df: "pyspark.sql.DataFrame", + partition_order: List[int], + state_dict: Optional[dict] = None, +): + import pyspark + + df_with_partition_id = df.select("*", pyspark.sql.functions.spark_partition_id().alias("part_id")) + partition_idx_start = state_dict["partition_idx"] if state_dict else 0 + partition_df = _reorder_dataframe_by_partition(df_with_partition_id, partition_order[partition_idx_start:]) + # pipeline next partition in parallel to hide latency + rows = partition_df.toLocalIterator(prefetchPartitions=True) + curr_partition = None + row_id = state_dict["partition_example_idx"] if state_dict else 0 + for row in islice(rows, row_id, None): + row_as_dict = row.asDict() + part_id = row_as_dict["part_id"] + row_as_dict.pop("part_id") + if curr_partition != part_id: + if state_dict and curr_partition is not None: + state_dict["partition_idx"] += 1 + curr_partition = part_id + row_id = 0 + if state_dict: + state_dict["partition_example_idx"] = row_id + 1 + yield f"{part_id}_{row_id}", row_as_dict + row_id += 1 + + +class SparkExamplesIterable(_BaseExamplesIterable): + def __init__( + self, + df: "pyspark.sql.DataFrame", + partition_order=None, + ): + super().__init__() + self.df = df + self.partition_order = partition_order or range(self.df.rdd.getNumPartitions()) + + def _init_state_dict(self) -> dict: + self._state_dict = {"partition_idx": 0, "partition_example_idx": 0} + return self._state_dict + + @experimental + def load_state_dict(self, state_dict: dict) -> dict: + return super().load_state_dict(state_dict) + + def __iter__(self): + yield from _generate_iterable_examples(self.df, self.partition_order, self._state_dict) + + def shuffle_data_sources(self, generator: np.random.Generator) -> "SparkExamplesIterable": + partition_order = list(range(self.df.rdd.getNumPartitions())) + generator.shuffle(partition_order) + return SparkExamplesIterable(self.df, partition_order=partition_order) + + def shard_data_sources(self, worker_id: int, num_workers: int) -> "SparkExamplesIterable": + partition_order = self.split_shard_indices_by_worker(worker_id, num_workers) + return SparkExamplesIterable(self.df, partition_order=partition_order) + + @property + def n_shards(self) -> int: + return len(self.partition_order) + + +class Spark(datasets.DatasetBuilder): + BUILDER_CONFIG_CLASS = SparkConfig + + def __init__( + self, + df: "pyspark.sql.DataFrame", + cache_dir: str = None, + working_dir: str = None, + **config_kwargs, + ): + import pyspark + + self._spark = pyspark.sql.SparkSession.builder.getOrCreate() + self.df = df + self._working_dir = working_dir + + super().__init__( + cache_dir=cache_dir, + config_name=str(self.df.semanticHash()), + **config_kwargs, + ) + + def _validate_cache_dir(self): + # Define this so that we don't reference self in create_cache_and_write_probe, which will result in a pickling + # error due to pickling the SparkContext. + cache_dir = self._cache_dir + + # Returns the path of the created file. + def create_cache_and_write_probe(context): + # makedirs with exist_ok will recursively create the directory. It will not throw an error if directories + # already exist. + os.makedirs(cache_dir, exist_ok=True) + probe_file = os.path.join(cache_dir, "fs_test" + uuid.uuid4().hex) + # Opening the file in append mode will create a new file unless it already exists, in which case it will not + # change the file contents. + open(probe_file, "a") + return [probe_file] + + if self._spark.conf.get("spark.master", "").startswith("local"): + return + + # If the cluster is multi-node, make sure that the user provided a cache_dir and that it is on an NFS + # accessible to the driver. + # TODO: Stream batches to the driver using ArrowCollectSerializer instead of throwing an error. + if self._cache_dir: + probe = ( + self._spark.sparkContext.parallelize(range(1), 1).mapPartitions(create_cache_and_write_probe).collect() + ) + if os.path.isfile(probe[0]): + return + + raise ValueError( + "When using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir" + ) + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager: datasets.download.download_manager.DownloadManager): + return [datasets.SplitGenerator(name=datasets.Split.TRAIN)] + + def _repartition_df_if_needed(self, max_shard_size): + import pyspark + + def get_arrow_batch_size(it): + for batch in it: + yield pa.RecordBatch.from_pydict({"batch_bytes": [batch.nbytes]}) + + df_num_rows = self.df.count() + sample_num_rows = df_num_rows if df_num_rows <= 100 else 100 + # Approximate the size of each row (in Arrow format) by averaging over a max-100-row sample. + approx_bytes_per_row = ( + self.df.limit(sample_num_rows) + .repartition(1) + .mapInArrow(get_arrow_batch_size, "batch_bytes: long") + .agg(pyspark.sql.functions.sum("batch_bytes").alias("sample_bytes")) + .collect()[0] + .sample_bytes + / sample_num_rows + ) + approx_total_size = approx_bytes_per_row * df_num_rows + if approx_total_size > max_shard_size: + # Make sure there is at least one row per partition. + new_num_partitions = min(df_num_rows, int(approx_total_size / max_shard_size)) + self.df = self.df.repartition(new_num_partitions) + + def _prepare_split_single( + self, + fpath: str, + file_format: str, + max_shard_size: int, + ) -> Iterable[Tuple[int, bool, Union[int, tuple]]]: + import pyspark + + writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter + working_fpath = os.path.join(self._working_dir, os.path.basename(fpath)) if self._working_dir else fpath + embed_local_files = file_format == "parquet" + + # Define these so that we don't reference self in write_arrow, which will result in a pickling error due to + # pickling the SparkContext. + features = self.config.features + writer_batch_size = self._writer_batch_size + storage_options = self._fs.storage_options + + def write_arrow(it): + # Within the same SparkContext, no two task attempts will share the same attempt ID. + task_id = pyspark.TaskContext().taskAttemptId() + first_batch = next(it, None) + if first_batch is None: + # Some partitions might not receive any data. + return pa.RecordBatch.from_arrays( + [[task_id], [0], [0]], + names=["task_id", "num_examples", "num_bytes"], + ) + shard_id = 0 + writer = writer_class( + features=features, + path=working_fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"), + writer_batch_size=writer_batch_size, + storage_options=storage_options, + embed_local_files=embed_local_files, + ) + table = pa.Table.from_batches([first_batch]) + writer.write_table(table) + for batch in it: + if max_shard_size is not None and writer._num_bytes >= max_shard_size: + num_examples, num_bytes = writer.finalize() + writer.close() + yield pa.RecordBatch.from_arrays( + [[task_id], [num_examples], [num_bytes]], + names=["task_id", "num_examples", "num_bytes"], + ) + shard_id += 1 + writer = writer_class( + features=writer._features, + path=working_fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"), + writer_batch_size=writer_batch_size, + storage_options=storage_options, + embed_local_files=embed_local_files, + ) + table = pa.Table.from_batches([batch]) + writer.write_table(table) + + if writer._num_bytes > 0: + num_examples, num_bytes = writer.finalize() + writer.close() + yield pa.RecordBatch.from_arrays( + [[task_id], [num_examples], [num_bytes]], + names=["task_id", "num_examples", "num_bytes"], + ) + + if working_fpath != fpath: + for file in os.listdir(os.path.dirname(working_fpath)): + dest = os.path.join(os.path.dirname(fpath), os.path.basename(file)) + shutil.move(file, dest) + + stats = ( + self.df.mapInArrow(write_arrow, "task_id: long, num_examples: long, num_bytes: long") + .groupBy("task_id") + .agg( + pyspark.sql.functions.sum("num_examples").alias("total_num_examples"), + pyspark.sql.functions.sum("num_bytes").alias("total_num_bytes"), + pyspark.sql.functions.count("num_bytes").alias("num_shards"), + pyspark.sql.functions.collect_list("num_examples").alias("shard_lengths"), + ) + .collect() + ) + for row in stats: + yield row.task_id, (row.total_num_examples, row.total_num_bytes, row.num_shards, row.shard_lengths) + + def _prepare_split( + self, + split_generator: "datasets.SplitGenerator", + file_format: str = "arrow", + max_shard_size: Optional[Union[str, int]] = None, + num_proc: Optional[int] = None, + **kwargs, + ): + self._validate_cache_dir() + + max_shard_size = convert_file_size_to_int(max_shard_size or MAX_SHARD_SIZE) + self._repartition_df_if_needed(max_shard_size) + is_local = not is_remote_filesystem(self._fs) + path_join = os.path.join if is_local else posixpath.join + + SUFFIX = "-TTTTT-SSSSS-of-NNNNN" + fname = f"{self.name}-{split_generator.name}{SUFFIX}.{file_format}" + fpath = path_join(self._output_dir, fname) + + total_num_examples = 0 + total_num_bytes = 0 + total_shards = 0 + task_id_and_num_shards = [] + all_shard_lengths = [] + + for task_id, content in self._prepare_split_single(fpath, file_format, max_shard_size): + ( + num_examples, + num_bytes, + num_shards, + shard_lengths, + ) = content + if num_bytes > 0: + total_num_examples += num_examples + total_num_bytes += num_bytes + total_shards += num_shards + task_id_and_num_shards.append((task_id, num_shards)) + all_shard_lengths.extend(shard_lengths) + + split_generator.split_info.num_examples = total_num_examples + split_generator.split_info.num_bytes = total_num_bytes + + # should rename everything at the end + logger.debug(f"Renaming {total_shards} shards.") + if total_shards > 1: + split_generator.split_info.shard_lengths = all_shard_lengths + + # Define fs outside of _rename_shard so that we don't reference self in the function, which will result in a + # pickling error due to pickling the SparkContext. + fs = self._fs + + # use the -SSSSS-of-NNNNN pattern + def _rename_shard( + task_id: int, + shard_id: int, + global_shard_id: int, + ): + rename( + fs, + fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"), + fpath.replace("TTTTT-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"), + ) + + args = [] + global_shard_id = 0 + for i in range(len(task_id_and_num_shards)): + task_id, num_shards = task_id_and_num_shards[i] + for shard_id in range(num_shards): + args.append([task_id, shard_id, global_shard_id]) + global_shard_id += 1 + self._spark.sparkContext.parallelize(args, len(args)).map(lambda args: _rename_shard(*args)).collect() + else: + # don't use any pattern + shard_id = 0 + task_id = task_id_and_num_shards[0][0] + self._rename( + fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"), + fpath.replace(SUFFIX, ""), + ) + + def _get_examples_iterable_for_split( + self, + split_generator: "datasets.SplitGenerator", + ) -> SparkExamplesIterable: + return SparkExamplesIterable(self.df) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/sql/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/sql/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/sql/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/sql/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..63cf7b5df40809ab4a6d178f38ac6c3b5b54f56c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/sql/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/sql/__pycache__/sql.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/sql/__pycache__/sql.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..435dc498850e3d012f9e7fd8599c196366233645 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/sql/__pycache__/sql.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/sql/sql.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/sql/sql.py new file mode 100644 index 0000000000000000000000000000000000000000..152a8dc2089c97df2db68266ea93201c59d73324 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/sql/sql.py @@ -0,0 +1,119 @@ +import sys +from dataclasses import dataclass +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union + +import pandas as pd +import pyarrow as pa + +import datasets +import datasets.config +from datasets.features.features import require_storage_cast +from datasets.table import table_cast + + +if TYPE_CHECKING: + import sqlite3 + + import sqlalchemy + + +logger = datasets.utils.logging.get_logger(__name__) + + +@dataclass +class SqlConfig(datasets.BuilderConfig): + """BuilderConfig for SQL.""" + + sql: Union[str, "sqlalchemy.sql.Selectable"] = None + con: Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"] = None + index_col: Optional[Union[str, List[str]]] = None + coerce_float: bool = True + params: Optional[Union[List, Tuple, Dict]] = None + parse_dates: Optional[Union[List, Dict]] = None + columns: Optional[List[str]] = None + chunksize: Optional[int] = 10_000 + features: Optional[datasets.Features] = None + + def __post_init__(self): + super().__post_init__() + if self.sql is None: + raise ValueError("sql must be specified") + if self.con is None: + raise ValueError("con must be specified") + + def create_config_id( + self, + config_kwargs: dict, + custom_features: Optional[datasets.Features] = None, + ) -> str: + config_kwargs = config_kwargs.copy() + # We need to stringify the Selectable object to make its hash deterministic + + # The process of stringifying is explained here: http://docs.sqlalchemy.org/en/latest/faq/sqlexpressions.html + sql = config_kwargs["sql"] + if not isinstance(sql, str): + if datasets.config.SQLALCHEMY_AVAILABLE and "sqlalchemy" in sys.modules: + import sqlalchemy + + if isinstance(sql, sqlalchemy.sql.Selectable): + engine = sqlalchemy.create_engine(config_kwargs["con"].split("://")[0] + "://") + sql_str = str(sql.compile(dialect=engine.dialect)) + config_kwargs["sql"] = sql_str + else: + raise TypeError( + f"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(sql)}: {sql}" + ) + else: + raise TypeError( + f"Supported types for 'sql' are string and sqlalchemy.sql.Selectable but got {type(sql)}: {sql}" + ) + con = config_kwargs["con"] + if not isinstance(con, str): + config_kwargs["con"] = id(con) + logger.info( + f"SQL connection 'con' of type {type(con)} couldn't be hashed properly. To enable hashing, specify 'con' as URI string instead." + ) + + return super().create_config_id(config_kwargs, custom_features=custom_features) + + @property + def pd_read_sql_kwargs(self): + pd_read_sql_kwargs = { + "index_col": self.index_col, + "columns": self.columns, + "params": self.params, + "coerce_float": self.coerce_float, + "parse_dates": self.parse_dates, + } + return pd_read_sql_kwargs + + +class Sql(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = SqlConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={})] + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + schema = self.config.features.arrow_schema + if all(not require_storage_cast(feature) for feature in self.config.features.values()): + # cheaper cast + pa_table = pa.Table.from_arrays([pa_table[field.name] for field in schema], schema=schema) + else: + # more expensive cast; allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, schema) + return pa_table + + def _generate_tables(self): + chunksize = self.config.chunksize + sql_reader = pd.read_sql( + self.config.sql, self.config.con, chunksize=chunksize, **self.config.pd_read_sql_kwargs + ) + sql_reader = [sql_reader] if chunksize is None else sql_reader + for chunk_idx, df in enumerate(sql_reader): + pa_table = pa.Table.from_pandas(df) + yield chunk_idx, self._cast_table(pa_table) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/text/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/text/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/text/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/text/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3bb405531afd067872859d1385461247da885d07 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/text/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/text/__pycache__/text.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/text/__pycache__/text.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff7db87e92e146dde44a93f7c6f9454dbfc887d5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/text/__pycache__/text.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/text/text.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/text/text.py new file mode 100644 index 0000000000000000000000000000000000000000..e0e0d7fedb9d8731721ee340ab74b9cf392ee946 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/text/text.py @@ -0,0 +1,124 @@ +import itertools +import warnings +from dataclasses import InitVar, dataclass +from io import StringIO +from typing import Optional + +import pyarrow as pa + +import datasets +from datasets.features.features import require_storage_cast +from datasets.table import table_cast + + +logger = datasets.utils.logging.get_logger(__name__) + + +@dataclass +class TextConfig(datasets.BuilderConfig): + """BuilderConfig for text files.""" + + features: Optional[datasets.Features] = None + encoding: str = "utf-8" + errors: InitVar[Optional[str]] = "deprecated" + encoding_errors: Optional[str] = None + chunksize: int = 10 << 20 # 10MB + keep_linebreaks: bool = False + sample_by: str = "line" + + def __post_init__(self, errors): + super().__post_init__() + if errors != "deprecated": + warnings.warn( + "'errors' was deprecated in favor of 'encoding_errors' in version 2.14.0 and will be removed in 3.0.0.\n" + f"You can remove this warning by passing 'encoding_errors={errors}' instead.", + FutureWarning, + ) + self.encoding_errors = errors + + +class Text(datasets.ArrowBasedBuilder): + BUILDER_CONFIG_CLASS = TextConfig + + def _info(self): + return datasets.DatasetInfo(features=self.config.features) + + def _split_generators(self, dl_manager): + """The `data_files` kwarg in load_dataset() can be a str, List[str], Dict[str,str], or Dict[str,List[str]]. + + If str or List[str], then the dataset returns only the 'train' split. + If dict, then keys should be from the `datasets.Split` enum. + """ + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + dl_manager.download_config.extract_on_the_fly = True + data_files = dl_manager.download_and_extract(self.config.data_files) + splits = [] + for split_name, files in data_files.items(): + if isinstance(files, str): + files = [files] + files = [dl_manager.iter_files(file) for file in files] + splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files})) + return splits + + def _cast_table(self, pa_table: pa.Table) -> pa.Table: + if self.config.features is not None: + schema = self.config.features.arrow_schema + if all(not require_storage_cast(feature) for feature in self.config.features.values()): + # cheaper cast + pa_table = pa_table.cast(schema) + else: + # more expensive cast; allows str <-> int/float or str to Audio for example + pa_table = table_cast(pa_table, schema) + return pa_table + else: + return pa_table.cast(pa.schema({"text": pa.string()})) + + def _generate_tables(self, files): + pa_table_names = list(self.config.features) if self.config.features is not None else ["text"] + for file_idx, file in enumerate(itertools.chain.from_iterable(files)): + # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n" + with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f: + if self.config.sample_by == "line": + batch_idx = 0 + while True: + batch = f.read(self.config.chunksize) + if not batch: + break + batch += f.readline() # finish current line + # StringIO.readlines, by default splits only on "\n" (and keeps line breaks) + batch = StringIO(batch).readlines() + if not self.config.keep_linebreaks: + batch = [line.rstrip("\n") for line in batch] + pa_table = pa.Table.from_arrays([pa.array(batch)], names=pa_table_names) + # Uncomment for debugging (will print the Arrow table size and elements) + # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}") + # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows))) + yield (file_idx, batch_idx), self._cast_table(pa_table) + batch_idx += 1 + elif self.config.sample_by == "paragraph": + batch_idx = 0 + batch = "" + while True: + new_batch = f.read(self.config.chunksize) + if not new_batch: + break + batch += new_batch + batch += f.readline() # finish current line + batch = batch.split("\n\n") + pa_table = pa.Table.from_arrays( + [pa.array([example for example in batch[:-1] if example])], names=pa_table_names + ) + # Uncomment for debugging (will print the Arrow table size and elements) + # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}") + # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows))) + yield (file_idx, batch_idx), self._cast_table(pa_table) + batch_idx += 1 + batch = batch[-1] + if batch: + pa_table = pa.Table.from_arrays([pa.array([batch])], names=pa_table_names) + yield (file_idx, batch_idx), self._cast_table(pa_table) + elif self.config.sample_by == "document": + text = f.read() + pa_table = pa.Table.from_arrays([pa.array([text])], names=pa_table_names) + yield file_idx, self._cast_table(pa_table) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b82dc2e266363021be71409a221cb0179e911c14 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/_tenbin.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/_tenbin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..905d38f5f8d60ae8607f4efae9f25fd3e523b17a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/_tenbin.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/webdataset.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/webdataset.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b489571c06f5cb9fa43956eba24ba742441de06c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/__pycache__/webdataset.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/_tenbin.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/_tenbin.py new file mode 100644 index 0000000000000000000000000000000000000000..cd8c054842e090dc09bdf3d2fee59241a1a928c5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/_tenbin.py @@ -0,0 +1,285 @@ +# +# Copyright (c) 2017-2021 NVIDIA CORPORATION. All rights reserved. +# This file coems from the WebDataset library. +# See the LICENSE file for licensing terms (BSD-style). +# + +""" +Binary tensor encodings for PyTorch and NumPy. + +This defines efficient binary encodings for tensors. The format is 8 byte +aligned and can be used directly for computations when transmitted, say, +via RDMA. The format is supported by WebDataset with the `.ten` filename +extension. It is also used by Tensorcom, Tensorcom RDMA, and can be used +for fast tensor storage with LMDB and in disk files (which can be memory +mapped) + +Data is encoded as a series of chunks: + +- magic number (int64) +- length in bytes (int64) +- bytes (multiple of 64 bytes long) + +Arrays are a header chunk followed by a data chunk. +Header chunks have the following structure: + +- dtype (int64) +- 8 byte array name +- ndim (int64) +- dim[0] +- dim[1] +- ... +""" + +import struct +import sys + +import numpy as np + + +def bytelen(a): + """Determine the length of a in bytes.""" + if hasattr(a, "nbytes"): + return a.nbytes + elif isinstance(a, (bytearray, bytes)): + return len(a) + else: + raise ValueError(a, "cannot determine nbytes") + + +def bytedata(a): + """Return a the raw data corresponding to a.""" + if isinstance(a, (bytearray, bytes, memoryview)): + return a + elif hasattr(a, "data"): + return a.data + else: + raise ValueError(a, "cannot return bytedata") + + +# tables for converting between long/short NumPy dtypes + +long_to_short = """ +float16 f2 +float32 f4 +float64 f8 +int8 i1 +int16 i2 +int32 i4 +int64 i8 +uint8 u1 +uint16 u2 +unit32 u4 +uint64 u8 +""".strip() +long_to_short = [x.split() for x in long_to_short.split("\n")] +long_to_short = {x[0]: x[1] for x in long_to_short} +short_to_long = {v: k for k, v in long_to_short.items()} + + +def check_acceptable_input_type(data, allow64): + """Check that the data has an acceptable type for tensor encoding. + + :param data: array + :param allow64: allow 64 bit types + """ + for a in data: + if a.dtype.name not in long_to_short: + raise ValueError("unsupported dataypte") + if not allow64 and a.dtype.name not in ["float64", "int64", "uint64"]: + raise ValueError("64 bit datatypes not allowed unless explicitly enabled") + + +def str64(s): + """Convert a string to an int64.""" + s = s + "\0" * (8 - len(s)) + s = s.encode("ascii") + return struct.unpack("@q", s)[0] + + +def unstr64(i): + """Convert an int64 to a string.""" + b = struct.pack("@q", i) + return b.decode("ascii").strip("\0") + + +def check_infos(data, infos, required_infos=None): + """Verify the info strings.""" + if required_infos is False or required_infos is None: + return data + if required_infos is True: + return data, infos + if not isinstance(required_infos, (tuple, list)): + raise ValueError("required_infos must be tuple or list") + for required, actual in zip(required_infos, infos): + raise ValueError(f"actual info {actual} doesn't match required info {required}") + return data + + +def encode_header(a, info=""): + """Encode an array header as a byte array.""" + if a.ndim >= 10: + raise ValueError("too many dimensions") + if a.nbytes != np.prod(a.shape) * a.itemsize: + raise ValueError("mismatch between size and shape") + if a.dtype.name not in long_to_short: + raise ValueError("unsupported array type") + header = [str64(long_to_short[a.dtype.name]), str64(info), len(a.shape)] + list(a.shape) + return bytedata(np.array(header, dtype="i8")) + + +def decode_header(h): + """Decode a byte array into an array header.""" + h = np.frombuffer(h, dtype="i8") + if unstr64(h[0]) not in short_to_long: + raise ValueError("unsupported array type") + dtype = np.dtype(short_to_long[unstr64(h[0])]) + info = unstr64(h[1]) + rank = int(h[2]) + shape = tuple(h[3 : 3 + rank]) + return shape, dtype, info + + +def encode_list(l, infos=None): # noqa: E741 + """Given a list of arrays, encode them into a list of byte arrays.""" + if infos is None: + infos = [""] + else: + if len(l) != len(infos): + raise ValueError(f"length of list {l} must muatch length of infos {infos}") + result = [] + for i, a in enumerate(l): + header = encode_header(a, infos[i % len(infos)]) + result += [header, bytedata(a)] + return result + + +def decode_list(l, infos=False): # noqa: E741 + """Given a list of byte arrays, decode them into arrays.""" + result = [] + infos0 = [] + for header, data in zip(l[::2], l[1::2]): + shape, dtype, info = decode_header(header) + a = np.frombuffer(data, dtype=dtype, count=np.prod(shape)).reshape(*shape) + result += [a] + infos0 += [info] + return check_infos(result, infos0, infos) + + +magic_str = "~TenBin~" +magic = str64(magic_str) +magic_bytes = unstr64(magic).encode("ascii") + + +def roundup(n, k=64): + """Round up to the next multiple of 64.""" + return k * ((n + k - 1) // k) + + +def encode_chunks(l): # noqa: E741 + """Encode a list of chunks into a single byte array, with lengths and magics..""" + size = sum(16 + roundup(b.nbytes) for b in l) + result = bytearray(size) + offset = 0 + for b in l: + result[offset : offset + 8] = magic_bytes + offset += 8 + result[offset : offset + 8] = struct.pack("@q", b.nbytes) + offset += 8 + result[offset : offset + bytelen(b)] = b + offset += roundup(bytelen(b)) + return result + + +def decode_chunks(buf): + """Decode a byte array into a list of chunks.""" + result = [] + offset = 0 + total = bytelen(buf) + while offset < total: + if magic_bytes != buf[offset : offset + 8]: + raise ValueError("magic bytes mismatch") + offset += 8 + nbytes = struct.unpack("@q", buf[offset : offset + 8])[0] + offset += 8 + b = buf[offset : offset + nbytes] + offset += roundup(nbytes) + result.append(b) + return result + + +def encode_buffer(l, infos=None): # noqa: E741 + """Encode a list of arrays into a single byte array.""" + if not isinstance(l, list): + raise ValueError("requires list") + return encode_chunks(encode_list(l, infos=infos)) + + +def decode_buffer(buf, infos=False): + """Decode a byte array into a list of arrays.""" + return decode_list(decode_chunks(buf), infos=infos) + + +def write_chunk(stream, buf): + """Write a byte chunk to the stream with magics, length, and padding.""" + nbytes = bytelen(buf) + stream.write(magic_bytes) + stream.write(struct.pack("@q", nbytes)) + stream.write(bytedata(buf)) + padding = roundup(nbytes) - nbytes + if padding > 0: + stream.write(b"\0" * padding) + + +def read_chunk(stream): + """Read a byte chunk from a stream with magics, length, and padding.""" + magic = stream.read(8) + if magic == b"": + return None + if magic != magic_bytes: + raise ValueError("magic number does not match") + nbytes = stream.read(8) + nbytes = struct.unpack("@q", nbytes)[0] + if nbytes < 0: + raise ValueError("negative nbytes") + data = stream.read(nbytes) + padding = roundup(nbytes) - nbytes + if padding > 0: + stream.read(padding) + return data + + +def write(stream, l, infos=None): # noqa: E741 + """Write a list of arrays to a stream, with magics, length, and padding.""" + for chunk in encode_list(l, infos=infos): + write_chunk(stream, chunk) + + +def read(stream, n=sys.maxsize, infos=False): + """Read a list of arrays from a stream, with magics, length, and padding.""" + chunks = [] + for _ in range(n): + header = read_chunk(stream) + if header is None: + break + data = read_chunk(stream) + if data is None: + raise ValueError("premature EOF") + chunks += [header, data] + return decode_list(chunks, infos=infos) + + +def save(fname, *args, infos=None, nocheck=False): + """Save a list of arrays to a file, with magics, length, and padding.""" + if not nocheck and not fname.endswith(".ten"): + raise ValueError("file name should end in .ten") + with open(fname, "wb") as stream: + write(stream, args, infos=infos) + + +def load(fname, infos=False, nocheck=False): + """Read a list of arrays from a file, with magics, length, and padding.""" + if not nocheck and not fname.endswith(".ten"): + raise ValueError("file name should end in .ten") + with open(fname, "rb") as stream: + return read(stream, infos=infos) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/webdataset.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/webdataset.py new file mode 100644 index 0000000000000000000000000000000000000000..fc1567d4ac09acf0d721d17b0aec6c3f59fb6c77 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/packaged_modules/webdataset/webdataset.py @@ -0,0 +1,311 @@ +import io +import json +from itertools import islice +from typing import Any, Callable, Dict, List + +import fsspec +import numpy as np +import pyarrow as pa + +import datasets +from datasets.features.features import cast_to_python_objects +from datasets.utils.file_utils import SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL, xbasename + + +logger = datasets.utils.logging.get_logger(__name__) + + +class WebDataset(datasets.GeneratorBasedBuilder): + DEFAULT_WRITER_BATCH_SIZE = 100 + IMAGE_EXTENSIONS: List[str] # definition at the bottom of the script + AUDIO_EXTENSIONS: List[str] # definition at the bottom of the script + DECODERS: Dict[str, Callable[[Any], Any]] # definition at the bottom of the script + NUM_EXAMPLES_FOR_FEATURES_INFERENCE = 5 + + @classmethod + def _get_pipeline_from_tar(cls, tar_path, tar_iterator): + current_example = {} + fs: fsspec.AbstractFileSystem = fsspec.filesystem("memory") + streaming_download_manager = datasets.StreamingDownloadManager() + for filename, f in tar_iterator: + if "." in filename: + example_key, field_name = filename.split(".", 1) + if current_example and current_example["__key__"] != example_key: + yield current_example + current_example = {} + current_example["__key__"] = example_key + current_example["__url__"] = tar_path + current_example[field_name.lower()] = f.read() + if field_name.split(".")[-1] in SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL: + fs.write_bytes(filename, current_example[field_name.lower()]) + extracted_file_path = streaming_download_manager.extract(f"memory://{filename}") + with fsspec.open(extracted_file_path) as f: + current_example[field_name.lower()] = f.read() + fs.delete(filename) + data_extension = xbasename(extracted_file_path).split(".")[-1] + else: + data_extension = field_name.split(".")[-1] + if data_extension in cls.DECODERS: + current_example[field_name] = cls.DECODERS[data_extension](current_example[field_name]) + if current_example: + yield current_example + + def _info(self) -> datasets.DatasetInfo: + return datasets.DatasetInfo() + + def _split_generators(self, dl_manager): + """We handle string, list and dicts in datafiles""" + # Download the data files + if not self.config.data_files: + raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}") + data_files = dl_manager.download(self.config.data_files) + splits = [] + for split_name, tar_paths in data_files.items(): + if isinstance(tar_paths, str): + tar_paths = [tar_paths] + tar_iterators = [dl_manager.iter_archive(tar_path) for tar_path in tar_paths] + splits.append( + datasets.SplitGenerator( + name=split_name, gen_kwargs={"tar_paths": tar_paths, "tar_iterators": tar_iterators} + ) + ) + if not self.info.features: + # Get one example to get the feature types + pipeline = self._get_pipeline_from_tar(tar_paths[0], tar_iterators[0]) + first_examples = list(islice(pipeline, self.NUM_EXAMPLES_FOR_FEATURES_INFERENCE)) + if any(example.keys() != first_examples[0].keys() for example in first_examples): + raise ValueError( + "The TAR archives of the dataset should be in WebDataset format, " + "but the files in the archive don't share the same prefix or the same types." + ) + pa_tables = [ + pa.Table.from_pylist(cast_to_python_objects([example], only_1d_for_numpy=True)) + for example in first_examples + ] + if datasets.config.PYARROW_VERSION.major < 14: + inferred_arrow_schema = pa.concat_tables(pa_tables, promote=True).schema + else: + inferred_arrow_schema = pa.concat_tables(pa_tables, promote_options="default").schema + features = datasets.Features.from_arrow_schema(inferred_arrow_schema) + + # Set Image types + for field_name in first_examples[0]: + extension = field_name.rsplit(".", 1)[-1] + if extension in self.IMAGE_EXTENSIONS: + features[field_name] = datasets.Image() + # Set Audio types + for field_name in first_examples[0]: + extension = field_name.rsplit(".", 1)[-1] + if extension in self.AUDIO_EXTENSIONS: + features[field_name] = datasets.Audio() + self.info.features = features + + return splits + + def _generate_examples(self, tar_paths, tar_iterators): + image_field_names = [ + field_name for field_name, feature in self.info.features.items() if isinstance(feature, datasets.Image) + ] + audio_field_names = [ + field_name for field_name, feature in self.info.features.items() if isinstance(feature, datasets.Audio) + ] + for tar_idx, (tar_path, tar_iterator) in enumerate(zip(tar_paths, tar_iterators)): + for example_idx, example in enumerate(self._get_pipeline_from_tar(tar_path, tar_iterator)): + for field_name in image_field_names + audio_field_names: + example[field_name] = {"path": example["__key__"] + "." + field_name, "bytes": example[field_name]} + yield f"{tar_idx}_{example_idx}", example + + +# Obtained with: +# ``` +# import PIL.Image +# IMAGE_EXTENSIONS = [] +# PIL.Image.init() +# for ext, format in PIL.Image.EXTENSION.items(): +# if format in PIL.Image.OPEN: +# IMAGE_EXTENSIONS.append(ext[1:]) +# ``` +# We intentionally do not run this code on launch because: +# (1) Pillow is an optional dependency, so importing Pillow in global namespace is not allowed +# (2) To ensure the list of supported extensions is deterministic +IMAGE_EXTENSIONS = [ + "blp", + "bmp", + "dib", + "bufr", + "cur", + "pcx", + "dcx", + "dds", + "ps", + "eps", + "fit", + "fits", + "fli", + "flc", + "ftc", + "ftu", + "gbr", + "gif", + "grib", + "h5", + "hdf", + "png", + "apng", + "jp2", + "j2k", + "jpc", + "jpf", + "jpx", + "j2c", + "icns", + "ico", + "im", + "iim", + "tif", + "tiff", + "jfif", + "jpe", + "jpg", + "jpeg", + "mpg", + "mpeg", + "msp", + "pcd", + "pxr", + "pbm", + "pgm", + "ppm", + "pnm", + "psd", + "bw", + "rgb", + "rgba", + "sgi", + "ras", + "tga", + "icb", + "vda", + "vst", + "webp", + "wmf", + "emf", + "xbm", + "xpm", +] +WebDataset.IMAGE_EXTENSIONS = IMAGE_EXTENSIONS + + +# Obtained with: +# ``` +# import soundfile as sf +# +# AUDIO_EXTENSIONS = [f".{format.lower()}" for format in sf.available_formats().keys()] +# +# # .opus decoding is supported if libsndfile >= 1.0.31: +# AUDIO_EXTENSIONS.extend([".mp3", ".opus"]) +# ``` +# We intentionally do not run this code on launch because: +# (1) Soundfile is an optional dependency, so importing it in global namespace is not allowed +# (2) To ensure the list of supported extensions is deterministic +AUDIO_EXTENSIONS = [ + "aiff", + "au", + "avr", + "caf", + "flac", + "htk", + "svx", + "mat4", + "mat5", + "mpc2k", + "ogg", + "paf", + "pvf", + "raw", + "rf64", + "sd2", + "sds", + "ircam", + "voc", + "w64", + "wav", + "nist", + "wavex", + "wve", + "xi", + "mp3", + "opus", +] +WebDataset.AUDIO_EXTENSIONS = AUDIO_EXTENSIONS + + +def text_loads(data: bytes): + return data.decode("utf-8") + + +def tenbin_loads(data: bytes): + from . import _tenbin + + return _tenbin.decode_buffer(data) + + +def msgpack_loads(data: bytes): + import msgpack + + return msgpack.unpackb(data) + + +def npy_loads(data: bytes): + import numpy.lib.format + + stream = io.BytesIO(data) + return numpy.lib.format.read_array(stream, allow_pickle=False) + + +def npz_loads(data: bytes): + return np.load(io.BytesIO(data), allow_pickle=False) + + +def cbor_loads(data: bytes): + import cbor + + return cbor.loads(data) + + +def torch_loads(data: bytes): + import torch + + return torch.load(io.BytesIO(data), weights_only=True) + + +# Obtained by checking `decoders` in `webdataset.autodecode` +# and removing unsafe extension decoders. +# Removed Pickle decoders: +# - "pyd": lambda data: pickle.loads(data) +# - "pickle": lambda data: pickle.loads(data) +# Modified NumPy decoders to fix CVE-2019-6446 (add allow_pickle=False and weights_only=True): +# - "npy": npy_loads, +# - "npz": lambda data: np.load(io.BytesIO(data)), +# - "pth": lambda data: torch_loads(data) +DECODERS = { + "txt": text_loads, + "text": text_loads, + "transcript": text_loads, + "cls": int, + "cls2": int, + "index": int, + "inx": int, + "id": int, + "json": json.loads, + "jsn": json.loads, + "ten": tenbin_loads, + "tb": tenbin_loads, + "mp": msgpack_loads, + "msg": msgpack_loads, + "npy": npy_loads, + "npz": npz_loads, + "cbor": cbor_loads, + "pth": torch_loads, +} +WebDataset.DECODERS = DECODERS diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/parallel/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/parallel/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a8d527f4d09946b3b7b7a65ec22afa953ddfd315 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/parallel/__init__.py @@ -0,0 +1 @@ +from .parallel import ParallelBackendConfig, parallel_backend, parallel_map diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/parallel/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/parallel/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6093969bb2e9a8457b4767cec1817077d2fd41a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/parallel/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/parallel/__pycache__/parallel.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/parallel/__pycache__/parallel.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6088e2aa1a0a35bd50d1eda06d43ca3c44e819d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/parallel/__pycache__/parallel.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/parallel/parallel.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/parallel/parallel.py new file mode 100644 index 0000000000000000000000000000000000000000..5cad2c48ba2b0a6e71c62babd4eb052a3dd1dfd4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/parallel/parallel.py @@ -0,0 +1,120 @@ +import contextlib +from multiprocessing import Pool, RLock + +from tqdm.auto import tqdm + +from ..utils import experimental, logging + + +logger = logging.get_logger(__name__) + + +class ParallelBackendConfig: + backend_name = None + + +@experimental +def parallel_map(function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func): + """ + **Experimental.** Apply a function to iterable elements in parallel, where the implementation uses either + multiprocessing.Pool or joblib for parallelization. + + Args: + function (`Callable[[Any], Any]`): Function to be applied to `iterable`. + iterable (`list`, `tuple` or `np.ndarray`): Iterable elements to apply function to. + num_proc (`int`): Number of processes (if no backend specified) or jobs (using joblib). + types (`tuple`): Additional types (besides `dict` values) to apply `function` recursively to their elements. + disable_tqdm (`bool`): Whether to disable the tqdm progressbar. + desc (`str`): Prefix for the tqdm progressbar. + single_map_nested_func (`Callable`): Map function that applies `function` to an element from `iterable`. + Takes a tuple of function, data_struct, types, rank, disable_tqdm, desc as input, where data_struct is an + element of `iterable`, and `rank` is used for progress bar. + """ + if ParallelBackendConfig.backend_name is None: + return _map_with_multiprocessing_pool( + function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func + ) + + return _map_with_joblib( + function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func + ) + + +def _map_with_multiprocessing_pool( + function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func +): + num_proc = num_proc if num_proc <= len(iterable) else len(iterable) + split_kwds = [] # We organize the splits ourselve (contiguous splits) + for index in range(num_proc): + div = len(iterable) // num_proc + mod = len(iterable) % num_proc + start = div * index + min(index, mod) + end = start + div + (1 if index < mod else 0) + split_kwds.append((function, iterable[start:end], batched, batch_size, types, index, disable_tqdm, desc)) + + if len(iterable) != sum(len(i[1]) for i in split_kwds): + raise ValueError( + f"Error dividing inputs iterable among processes. " + f"Total number of objects {len(iterable)}, " + f"length: {sum(len(i[1]) for i in split_kwds)}" + ) + + logger.info( + f"Spawning {num_proc} processes for {len(iterable)} objects in slices of {[len(i[1]) for i in split_kwds]}" + ) + initargs, initializer = None, None + if not disable_tqdm: + initargs, initializer = (RLock(),), tqdm.set_lock + with Pool(num_proc, initargs=initargs, initializer=initializer) as pool: + mapped = pool.map(single_map_nested_func, split_kwds) + logger.info(f"Finished {num_proc} processes") + mapped = [obj for proc_res in mapped for obj in proc_res] + logger.info(f"Unpacked {len(mapped)} objects") + + return mapped + + +def _map_with_joblib( + function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func +): + # progress bar is not yet supported for _map_with_joblib, because tqdm couldn't accurately be applied to joblib, + # and it requires monkey-patching joblib internal classes which is subject to change + import joblib + + with joblib.parallel_backend(ParallelBackendConfig.backend_name, n_jobs=num_proc): + return joblib.Parallel()( + joblib.delayed(single_map_nested_func)((function, obj, batched, batch_size, types, None, True, None)) + for obj in iterable + ) + + +@experimental +@contextlib.contextmanager +def parallel_backend(backend_name: str): + """ + **Experimental.** Configures the parallel backend for parallelized dataset loading, which uses the parallelization + implemented by joblib. + + Args: + backend_name (str): Name of backend for parallelization implementation, has to be supported by joblib. + + Example usage: + ```py + with parallel_backend('spark'): + dataset = load_dataset(..., num_proc=2) + ``` + """ + ParallelBackendConfig.backend_name = backend_name + + if backend_name == "spark": + from joblibspark import register_spark + + register_spark() + + # TODO: call create_cache_and_write_probe if "download" in steps + # TODO: raise NotImplementedError when Dataset.map etc is called + + try: + yield + finally: + ParallelBackendConfig.backend_name = None diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9a18a1e79349cfb32a743aeca4c3e9a809645a75 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__init__.py @@ -0,0 +1,46 @@ +from typing import Optional + +from ..utils.logging import get_logger +from .audio_classification import AudioClassification +from .automatic_speech_recognition import AutomaticSpeechRecognition +from .base import TaskTemplate +from .image_classification import ImageClassification +from .language_modeling import LanguageModeling +from .question_answering import QuestionAnsweringExtractive +from .summarization import Summarization +from .text_classification import TextClassification + + +__all__ = [ + "AutomaticSpeechRecognition", + "AudioClassification", + "ImageClassification", + "LanguageModeling", + "QuestionAnsweringExtractive", + "Summarization", + "TaskTemplate", + "TextClassification", +] + +logger = get_logger(__name__) + + +NAME2TEMPLATE = { + AutomaticSpeechRecognition.task: AutomaticSpeechRecognition, + AudioClassification.task: AudioClassification, + ImageClassification.task: ImageClassification, + LanguageModeling.task: LanguageModeling, + QuestionAnsweringExtractive.task: QuestionAnsweringExtractive, + Summarization.task: Summarization, + TextClassification.task: TextClassification, +} + + +def task_template_from_dict(task_template_dict: dict) -> Optional[TaskTemplate]: + """Create one of the supported task templates in :py:mod:`datasets.tasks` from a dictionary.""" + task_name = task_template_dict.get("task") + if task_name is None: + logger.warning(f"Couldn't find template for task '{task_name}'. Available templates: {list(NAME2TEMPLATE)}") + return None + template = NAME2TEMPLATE.get(task_name) + return template.from_dict(task_template_dict) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a0709e37471f7f3a2376b4071eeaa0ec5c3e4ebf Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/audio_classification.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/audio_classification.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cbc0c4fcff657220df560695d90ee66d4741759f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/audio_classification.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/automatic_speech_recognition.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/automatic_speech_recognition.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a00894ba6df114078befc6c664e3bac66453e08b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/automatic_speech_recognition.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/base.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/base.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24d898b420ecda4b17b1e6b039ea8ca230c4a3e2 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/base.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/image_classification.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/image_classification.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2bf0121e58dbe639b3736eba280d67f02d98304d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/image_classification.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/language_modeling.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/language_modeling.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..70dce8faf2a46b44b7d772c177d44195ccd21246 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/language_modeling.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/question_answering.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/question_answering.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61df98000a45ae1c91e4a76656bc7eb25aad3586 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/question_answering.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/summarization.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/summarization.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d4dd6a3055cb30a1d52a23753519d07c54cb0ac Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/summarization.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/text_classification.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/text_classification.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a92307ef3a307c307674d2c8c9be849f6c1fba71 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/__pycache__/text_classification.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/audio_classification.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/audio_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..6f9fe402f3814b4db0eb1832405adcfaef77503e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/audio_classification.py @@ -0,0 +1,33 @@ +import copy +from dataclasses import dataclass, field +from typing import ClassVar, Dict + +from ..features import Audio, ClassLabel, Features +from .base import TaskTemplate + + +@dataclass(frozen=True) +class AudioClassification(TaskTemplate): + task: str = field(default="audio-classification", metadata={"include_in_asdict_even_if_is_default": True}) + input_schema: ClassVar[Features] = Features({"audio": Audio()}) + label_schema: ClassVar[Features] = Features({"labels": ClassLabel}) + audio_column: str = "audio" + label_column: str = "labels" + + def align_with_features(self, features): + if self.label_column not in features: + raise ValueError(f"Column {self.label_column} is not present in features.") + if not isinstance(features[self.label_column], ClassLabel): + raise ValueError(f"Column {self.label_column} is not a ClassLabel.") + task_template = copy.deepcopy(self) + label_schema = self.label_schema.copy() + label_schema["labels"] = features[self.label_column] + task_template.__dict__["label_schema"] = label_schema + return task_template + + @property + def column_mapping(self) -> Dict[str, str]: + return { + self.audio_column: "audio", + self.label_column: "labels", + } diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/automatic_speech_recognition.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/automatic_speech_recognition.py new file mode 100644 index 0000000000000000000000000000000000000000..103a98a1bc9774de6b652bbc69b41501a419f0f8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/automatic_speech_recognition.py @@ -0,0 +1,30 @@ +import copy +from dataclasses import dataclass, field +from typing import ClassVar, Dict + +from ..features import Audio, Features, Value +from .base import TaskTemplate + + +@dataclass(frozen=True) +class AutomaticSpeechRecognition(TaskTemplate): + task: str = field(default="automatic-speech-recognition", metadata={"include_in_asdict_even_if_is_default": True}) + input_schema: ClassVar[Features] = Features({"audio": Audio()}) + label_schema: ClassVar[Features] = Features({"transcription": Value("string")}) + audio_column: str = "audio" + transcription_column: str = "transcription" + + def align_with_features(self, features): + if self.audio_column not in features: + raise ValueError(f"Column {self.audio_column} is not present in features.") + if not isinstance(features[self.audio_column], Audio): + raise ValueError(f"Column {self.audio_column} is not an Audio type.") + task_template = copy.deepcopy(self) + input_schema = self.input_schema.copy() + input_schema["audio"] = features[self.audio_column] + task_template.__dict__["input_schema"] = input_schema + return task_template + + @property + def column_mapping(self) -> Dict[str, str]: + return {self.audio_column: "audio", self.transcription_column: "transcription"} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/base.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/base.py new file mode 100644 index 0000000000000000000000000000000000000000..21a5337ffc0784a1ed12f4617a9a0ef6ba7253e5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/base.py @@ -0,0 +1,39 @@ +import abc +import copy +import dataclasses +from dataclasses import dataclass +from typing import ClassVar, Dict, Type, TypeVar + +from ..features import Features + + +T = TypeVar("T", bound="TaskTemplate") + + +@dataclass(frozen=True) +class TaskTemplate(abc.ABC): + # `task` is not a ClassVar since we want it to be part of the `asdict` output for JSON serialization + task: str + input_schema: ClassVar[Features] + label_schema: ClassVar[Features] + + def align_with_features(self: T, features: Features) -> T: + """ + Align features with the task template. + """ + # No-op + return copy.deepcopy(self) + + @property + def features(self) -> Features: + return Features(**self.input_schema, **self.label_schema) + + @property + @abc.abstractmethod + def column_mapping(self) -> Dict[str, str]: + raise NotImplementedError + + @classmethod + def from_dict(cls: Type[T], template_dict: dict) -> T: + field_names = {f.name for f in dataclasses.fields(cls)} + return cls(**{k: v for k, v in template_dict.items() if k in field_names}) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/image_classification.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/image_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..20a19e0408a7ec8061ac4fac700d83e6dcbadcdf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/image_classification.py @@ -0,0 +1,33 @@ +import copy +from dataclasses import dataclass, field +from typing import ClassVar, Dict + +from ..features import ClassLabel, Features, Image +from .base import TaskTemplate + + +@dataclass(frozen=True) +class ImageClassification(TaskTemplate): + task: str = field(default="image-classification", metadata={"include_in_asdict_even_if_is_default": True}) + input_schema: ClassVar[Features] = Features({"image": Image()}) + label_schema: ClassVar[Features] = Features({"labels": ClassLabel}) + image_column: str = "image" + label_column: str = "labels" + + def align_with_features(self, features): + if self.label_column not in features: + raise ValueError(f"Column {self.label_column} is not present in features.") + if not isinstance(features[self.label_column], ClassLabel): + raise ValueError(f"Column {self.label_column} is not a ClassLabel.") + task_template = copy.deepcopy(self) + label_schema = self.label_schema.copy() + label_schema["labels"] = features[self.label_column] + task_template.__dict__["label_schema"] = label_schema + return task_template + + @property + def column_mapping(self) -> Dict[str, str]: + return { + self.image_column: "image", + self.label_column: "labels", + } diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/language_modeling.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/language_modeling.py new file mode 100644 index 0000000000000000000000000000000000000000..b2837744fa1718e57ffbeeca1a6e9a60c9468d8f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/language_modeling.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass, field +from typing import ClassVar, Dict + +from ..features import Features, Value +from .base import TaskTemplate + + +@dataclass(frozen=True) +class LanguageModeling(TaskTemplate): + task: str = field(default="language-modeling", metadata={"include_in_asdict_even_if_is_default": True}) + + input_schema: ClassVar[Features] = Features({"text": Value("string")}) + label_schema: ClassVar[Features] = Features({}) + text_column: str = "text" + + @property + def column_mapping(self) -> Dict[str, str]: + return {self.text_column: "text"} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/question_answering.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/question_answering.py new file mode 100644 index 0000000000000000000000000000000000000000..349fd54141762631eec025681015cedd97c23b63 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/question_answering.py @@ -0,0 +1,29 @@ +from dataclasses import dataclass, field +from typing import ClassVar, Dict + +from ..features import Features, Sequence, Value +from .base import TaskTemplate + + +@dataclass(frozen=True) +class QuestionAnsweringExtractive(TaskTemplate): + # `task` is not a ClassVar since we want it to be part of the `asdict` output for JSON serialization + task: str = field(default="question-answering-extractive", metadata={"include_in_asdict_even_if_is_default": True}) + input_schema: ClassVar[Features] = Features({"question": Value("string"), "context": Value("string")}) + label_schema: ClassVar[Features] = Features( + { + "answers": Sequence( + { + "text": Value("string"), + "answer_start": Value("int32"), + } + ) + } + ) + question_column: str = "question" + context_column: str = "context" + answers_column: str = "answers" + + @property + def column_mapping(self) -> Dict[str, str]: + return {self.question_column: "question", self.context_column: "context", self.answers_column: "answers"} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/summarization.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/summarization.py new file mode 100644 index 0000000000000000000000000000000000000000..a0057b07b4f62947c1bfde1962bf06be1427c363 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/summarization.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass, field +from typing import ClassVar, Dict + +from ..features import Features, Value +from .base import TaskTemplate + + +@dataclass(frozen=True) +class Summarization(TaskTemplate): + # `task` is not a ClassVar since we want it to be part of the `asdict` output for JSON serialization + task: str = field(default="summarization", metadata={"include_in_asdict_even_if_is_default": True}) + input_schema: ClassVar[Features] = Features({"text": Value("string")}) + label_schema: ClassVar[Features] = Features({"summary": Value("string")}) + text_column: str = "text" + summary_column: str = "summary" + + @property + def column_mapping(self) -> Dict[str, str]: + return {self.text_column: "text", self.summary_column: "summary"} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/text_classification.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/text_classification.py new file mode 100644 index 0000000000000000000000000000000000000000..13584b73e8ae668bd6c145b60598cd6859be5146 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/tasks/text_classification.py @@ -0,0 +1,34 @@ +import copy +from dataclasses import dataclass, field +from typing import ClassVar, Dict + +from ..features import ClassLabel, Features, Value +from .base import TaskTemplate + + +@dataclass(frozen=True) +class TextClassification(TaskTemplate): + # `task` is not a ClassVar since we want it to be part of the `asdict` output for JSON serialization + task: str = field(default="text-classification", metadata={"include_in_asdict_even_if_is_default": True}) + input_schema: ClassVar[Features] = Features({"text": Value("string")}) + label_schema: ClassVar[Features] = Features({"labels": ClassLabel}) + text_column: str = "text" + label_column: str = "labels" + + def align_with_features(self, features): + if self.label_column not in features: + raise ValueError(f"Column {self.label_column} is not present in features.") + if not isinstance(features[self.label_column], ClassLabel): + raise ValueError(f"Column {self.label_column} is not a ClassLabel.") + task_template = copy.deepcopy(self) + label_schema = self.label_schema.copy() + label_schema["labels"] = features[self.label_column] + task_template.__dict__["label_schema"] = label_schema + return task_template + + @property + def column_mapping(self) -> Dict[str, str]: + return { + self.text_column: "text", + self.label_column: "labels", + } diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a1c3d2935994956e230a441f3f2d65695ebaaf2d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import tqdm as _tqdm # _tqdm is the module +from .experimental import experimental +from .info_utils import VerificationMode +from .logging import disable_progress_bar, enable_progress_bar, is_progress_bar_enabled +from .tqdm import ( + are_progress_bars_disabled, + disable_progress_bars, + enable_progress_bars, + tqdm, +) +from .version import Version diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ea5f3b155764fcfc903c8f6c4d40a93e347b8bd2 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/_dataset_viewer.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/_dataset_viewer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b6916252e7c7a187eab0103febe1ac3cd1889e3b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/_dataset_viewer.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/_dill.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/_dill.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a4e109cae329154bd17e0fb035d45532d86d9e49 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/_dill.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/_filelock.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/_filelock.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..22cb8ea481e0c6162c4110030fa4abaf4c5c9ada Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/_filelock.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/beam_utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/beam_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6c4f5daed2f0e4469788b5c4dd58c92634955417 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/beam_utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/cache.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/cache.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..32851f376c1070f5fc8177e630d7d638c90c0494 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/cache.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/deprecation_utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/deprecation_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..38f454a5fd2cf11d2951dfd0703f3356670883d6 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/deprecation_utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/doc_utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/doc_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a36e42b3813b825d3a8c86835e15ab238315440e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/doc_utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/download_manager.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/download_manager.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..79496ec045b40d92af9da98841ec04f2f5506769 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/download_manager.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/experimental.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/experimental.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3beedf4513d0c9c8c78e67a014929c4e780989b1 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/experimental.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/extract.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/extract.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cc2555ae0a6b8f542afc1ad7273fda39ac80cbef Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/extract.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/file_utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/file_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e3d7e5c9fc5209492fa9456b5dadbf98b9fcfe71 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/file_utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/filelock.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/filelock.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..30bcf233cce206db575894ddde38b5af3931cc00 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/filelock.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/hub.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/hub.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..029c62240598fb688a50ead6bb8ac18063d60ba7 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/hub.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/info_utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/info_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5a393d5aa6e19e2d2c46d5735e6898db21e07f8e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/info_utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/logging.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/logging.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..030c869696d86a743c0cc364430eace45ced4725 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/logging.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/metadata.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/metadata.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d9ebfcc445722cb1a33cad18d471ba7fdc57076 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/metadata.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/patching.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/patching.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..421b20ffd03412da14a045bf24faeb0b07510996 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/patching.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/py_utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/py_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d98869bb2dbeceb42abc18732fb9cf3ff9f0309 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/py_utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/readme.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/readme.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e116130c3040e1dbe88fa9c13cc495fd1bb9e527 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/readme.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/sharding.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/sharding.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b9e0882c4dcfe0d523b65c5def0d0841a053ad02 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/sharding.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/stratify.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/stratify.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..30a7160293cd0507a7c88e47c05db9a0dfd0cb8c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/stratify.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/tf_utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/tf_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4a3a4b84e2a14fdbc14cc6eedd651c2555e25d05 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/tf_utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/tqdm.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/tqdm.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..370604f5abee2af1891f905269a8b26c10b69943 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/tqdm.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/track.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/track.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6aca56481c418762c035c3dfba3219c6569bd8a1 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/track.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/typing.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/typing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a89ff24ce3576c6d4c3f42af45d3962f4c843193 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/typing.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/version.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1995dbc9958e07227a36935ad94648d0a0ea84a0 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/__pycache__/version.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/_dataset_viewer.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/_dataset_viewer.py new file mode 100644 index 0000000000000000000000000000000000000000..58edecb1cc14cec752afc34c1551c49c7490ca4c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/_dataset_viewer.py @@ -0,0 +1,96 @@ +from typing import Any, Dict, List, Optional, Union + +from .. import config +from ..exceptions import DatasetsError +from .file_utils import ( + get_authentication_headers_for_url, + http_get, +) +from .logging import get_logger + + +logger = get_logger(__name__) + + +class DatasetViewerError(DatasetsError): + """Dataset viewer error. + + Raised when trying to use the dataset viewer HTTP API and when trying to access: + - a missing dataset, or + - a private/gated dataset and the user is not authenticated. + - unavailable /parquet or /info responses + """ + + +def get_exported_parquet_files(dataset: str, revision: str, token: Optional[Union[str, bool]]) -> List[Dict[str, Any]]: + """ + Get the dataset exported parquet files + Docs: https://huggingface.co/docs/datasets-server/parquet + """ + dataset_viewer_parquet_url = config.HF_ENDPOINT.replace("://", "://datasets-server.") + "/parquet?dataset=" + try: + parquet_data_files_response = http_get( + url=dataset_viewer_parquet_url + dataset, + temp_file=None, + headers=get_authentication_headers_for_url(config.HF_ENDPOINT + f"datasets/{dataset}", token=token), + timeout=100.0, + max_retries=3, + ) + parquet_data_files_response.raise_for_status() + if "X-Revision" in parquet_data_files_response.headers: + if parquet_data_files_response.headers["X-Revision"] == revision or revision is None: + parquet_data_files_response_json = parquet_data_files_response.json() + if ( + parquet_data_files_response_json.get("partial") is False + and not parquet_data_files_response_json.get("pending", True) + and not parquet_data_files_response_json.get("failed", True) + and "parquet_files" in parquet_data_files_response_json + ): + return parquet_data_files_response_json["parquet_files"] + else: + logger.debug(f"Parquet export for {dataset} is not completely ready yet.") + else: + logger.debug( + f"Parquet export for {dataset} is available but outdated (revision='{parquet_data_files_response.headers['X-Revision']}')" + ) + except Exception as e: # noqa catch any exception of the dataset viewer API and consider the parquet export doesn't exist + logger.debug(f"No parquet export for {dataset} available ({type(e).__name__}: {e})") + raise DatasetViewerError("No exported Parquet files available.") + + +def get_exported_dataset_infos( + dataset: str, revision: str, token: Optional[Union[str, bool]] +) -> Dict[str, Dict[str, Any]]: + """ + Get the dataset information, can be useful to get e.g. the dataset features. + Docs: https://huggingface.co/docs/datasets-server/info + """ + dataset_viewer_info_url = config.HF_ENDPOINT.replace("://", "://datasets-server.") + "/info?dataset=" + try: + info_response = http_get( + url=dataset_viewer_info_url + dataset, + temp_file=None, + headers=get_authentication_headers_for_url(config.HF_ENDPOINT + f"datasets/{dataset}", token=token), + timeout=100.0, + max_retries=3, + ) + info_response.raise_for_status() + if "X-Revision" in info_response.headers: + if info_response.headers["X-Revision"] == revision or revision is None: + info_response = info_response.json() + if ( + info_response.get("partial") is False + and not info_response.get("pending", True) + and not info_response.get("failed", True) + and "dataset_info" in info_response + ): + return info_response["dataset_info"] + else: + logger.debug(f"Dataset info for {dataset} is not completely ready yet.") + else: + logger.debug( + f"Dataset info for {dataset} is available but outdated (revision='{info_response.headers['X-Revision']}')" + ) + except Exception as e: # noqa catch any exception of the dataset viewer API and consider the dataset info doesn't exist + logger.debug(f"No dataset info for {dataset} available ({type(e).__name__}: {e})") + raise DatasetViewerError("No exported dataset infos available.") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/_dill.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/_dill.py new file mode 100644 index 0000000000000000000000000000000000000000..15578198a39622340f937a3dfdd9091af26d5453 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/_dill.py @@ -0,0 +1,459 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Extends `dill` to support pickling more types and produce more consistent dumps.""" + +import os +import sys +from io import BytesIO +from types import CodeType, FunctionType + +import dill +from packaging import version + +from .. import config + + +class Pickler(dill.Pickler): + dispatch = dill._dill.MetaCatchingDict(dill.Pickler.dispatch.copy()) + _legacy_no_dict_keys_sorting = False + + def save(self, obj, save_persistent_id=True): + obj_type = type(obj) + if obj_type not in self.dispatch: + if "regex" in sys.modules: + import regex # type: ignore + + if obj_type is regex.Pattern: + pklregister(obj_type)(_save_regexPattern) + if "spacy" in sys.modules: + import spacy # type: ignore + + if issubclass(obj_type, spacy.Language): + pklregister(obj_type)(_save_spacyLanguage) + if "tiktoken" in sys.modules: + import tiktoken # type: ignore + + if obj_type is tiktoken.Encoding: + pklregister(obj_type)(_save_tiktokenEncoding) + if "torch" in sys.modules: + import torch # type: ignore + + if issubclass(obj_type, torch.Tensor): + pklregister(obj_type)(_save_torchTensor) + + if obj_type is torch.Generator: + pklregister(obj_type)(_save_torchGenerator) + + # Unwrap `torch.compile`-ed modules + if issubclass(obj_type, torch.nn.Module): + obj = getattr(obj, "_orig_mod", obj) + if "transformers" in sys.modules: + import transformers # type: ignore + + if issubclass(obj_type, transformers.PreTrainedTokenizerBase): + pklregister(obj_type)(_save_transformersPreTrainedTokenizerBase) + + # Unwrap `torch.compile`-ed functions + if obj_type is FunctionType: + obj = getattr(obj, "_torchdynamo_orig_callable", obj) + dill.Pickler.save(self, obj, save_persistent_id=save_persistent_id) + + def _batch_setitems(self, items): + if self._legacy_no_dict_keys_sorting: + return super()._batch_setitems(items) + # Ignore the order of keys in a dict + try: + # Faster, but fails for unorderable elements + items = sorted(items) + except Exception: # TypeError, decimal.InvalidOperation, etc. + from datasets.fingerprint import Hasher + + items = sorted(items, key=lambda x: Hasher.hash(x[0])) + dill.Pickler._batch_setitems(self, items) + + def memoize(self, obj): + # Don't memoize strings since two identical strings can have different Python ids + if type(obj) is not str: # noqa: E721 + dill.Pickler.memoize(self, obj) + + +def pklregister(t): + """Register a custom reducer for the type.""" + + def proxy(func): + Pickler.dispatch[t] = func + return func + + return proxy + + +def dump(obj, file): + """Pickle an object to a file.""" + Pickler(file, recurse=True).dump(obj) + + +def dumps(obj): + """Pickle an object to a string.""" + file = BytesIO() + dump(obj, file) + return file.getvalue() + + +if config.DILL_VERSION < version.parse("0.3.6"): + + def log(pickler, msg): + dill._dill.log.info(msg) + +elif config.DILL_VERSION.release[:3] in [ + version.parse("0.3.6").release, + version.parse("0.3.7").release, + version.parse("0.3.8").release, +]: + + def log(pickler, msg): + dill._dill.logger.trace(pickler, msg) + + +@pklregister(set) +def _save_set(pickler, obj): + log(pickler, f"Se: {obj}") + try: + # Faster, but fails for unorderable elements + args = (sorted(obj),) + except Exception: # TypeError, decimal.InvalidOperation, etc. + from datasets.fingerprint import Hasher + + args = (sorted(obj, key=Hasher.hash),) + + pickler.save_reduce(set, args, obj=obj) + log(pickler, "# Se") + + +def _save_regexPattern(pickler, obj): + import regex # type: ignore + + log(pickler, f"Re: {obj}") + args = (obj.pattern, obj.flags) + pickler.save_reduce(regex.compile, args, obj=obj) + log(pickler, "# Re") + + +def _save_tiktokenEncoding(pickler, obj): + import tiktoken # type: ignore + + log(pickler, f"Enc: {obj}") + args = (obj.name, obj._pat_str, obj._mergeable_ranks, obj._special_tokens) + pickler.save_reduce(tiktoken.Encoding, args, obj=obj) + log(pickler, "# Enc") + + +def _save_torchTensor(pickler, obj): + import torch # type: ignore + + # `torch.from_numpy` is not picklable in `torch>=1.11.0` + def create_torchTensor(np_array): + return torch.from_numpy(np_array) + + log(pickler, f"To: {obj}") + args = (obj.detach().cpu().numpy(),) + pickler.save_reduce(create_torchTensor, args, obj=obj) + log(pickler, "# To") + + +def _save_torchGenerator(pickler, obj): + import torch # type: ignore + + def create_torchGenerator(state): + generator = torch.Generator() + generator.set_state(state) + return generator + + log(pickler, f"Ge: {obj}") + args = (obj.get_state(),) + pickler.save_reduce(create_torchGenerator, args, obj=obj) + log(pickler, "# Ge") + + +def _save_spacyLanguage(pickler, obj): + import spacy # type: ignore + + def create_spacyLanguage(config, bytes): + lang_cls = spacy.util.get_lang_class(config["nlp"]["lang"]) + lang_inst = lang_cls.from_config(config) + return lang_inst.from_bytes(bytes) + + log(pickler, f"Sp: {obj}") + args = (obj.config, obj.to_bytes()) + pickler.save_reduce(create_spacyLanguage, args, obj=obj) + log(pickler, "# Sp") + + +def _save_transformersPreTrainedTokenizerBase(pickler, obj): + log(pickler, f"Tok: {obj}") + # Ignore the `cache` attribute + state = obj.__dict__ + if "cache" in state and isinstance(state["cache"], dict): + state["cache"] = {} + pickler.save_reduce(type(obj), (), state=state, obj=obj) + log(pickler, "# Tok") + + +if config.DILL_VERSION < version.parse("0.3.6"): + + @pklregister(CodeType) + def _save_code(pickler, obj): + """ + From dill._dill.save_code + This is a modified version that removes the origin (filename + line no.) + of functions created in notebooks or shells for example. + """ + dill._dill.log.info(f"Co: {obj}") + # The filename of a function is the .py file where it is defined. + # Filenames of functions created in notebooks or shells start with '<' + # ex: for ipython, and for shell + # Filenames of functions created in ipykernel the filename + # look like f"{tempdir}/ipykernel_{id1}/{id2}.py" + # Moreover lambda functions have a special name: '' + # ex: (lambda x: x).__code__.co_name == "" # True + # + # For the hashing mechanism we ignore where the function has been defined + # More specifically: + # - we ignore the filename of special functions (filename starts with '<') + # - we always ignore the line number + # - we only use the base name of the file instead of the whole path, + # to be robust in case a script is moved for example. + # + # Only those two lines are different from the original implementation: + co_filename = ( + "" + if obj.co_filename.startswith("<") + or ( + len(obj.co_filename.split(os.path.sep)) > 1 + and obj.co_filename.split(os.path.sep)[-2].startswith("ipykernel_") + ) + or obj.co_name == "" + else os.path.basename(obj.co_filename) + ) + co_firstlineno = 1 + # The rest is the same as in the original dill implementation + if dill._dill.PY3: + if hasattr(obj, "co_posonlyargcount"): + args = ( + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, + obj.co_name, + co_firstlineno, + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + else: + args = ( + obj.co_argcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, + obj.co_name, + co_firstlineno, + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + else: + args = ( + obj.co_argcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, + obj.co_name, + co_firstlineno, + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + pickler.save_reduce(CodeType, args, obj=obj) + dill._dill.log.info("# Co") + return + +elif config.DILL_VERSION.release[:3] in [ + version.parse("0.3.6").release, + version.parse("0.3.7").release, + version.parse("0.3.8").release, +]: + # From: https://github.com/uqfoundation/dill/blob/dill-0.3.6/dill/_dill.py#L1104 + @pklregister(CodeType) + def save_code(pickler, obj): + dill._dill.logger.trace(pickler, "Co: %s", obj) + + ############################################################################################################ + # Modification here for huggingface/datasets + # The filename of a function is the .py file where it is defined. + # Filenames of functions created in notebooks or shells start with '<' + # ex: for ipython, and for shell + # Filenames of functions created in ipykernel the filename + # look like f"{tempdir}/ipykernel_{id1}/{id2}.py" + # Moreover lambda functions have a special name: '' + # ex: (lambda x: x).__code__.co_name == "" # True + # + # For the hashing mechanism we ignore where the function has been defined + # More specifically: + # - we ignore the filename of special functions (filename starts with '<') + # - we always ignore the line number + # - we only use the base name of the file instead of the whole path, + # to be robust in case a script is moved for example. + # + # Only those two lines are different from the original implementation: + co_filename = ( + "" + if obj.co_filename.startswith("<") + or ( + len(obj.co_filename.split(os.path.sep)) > 1 + and obj.co_filename.split(os.path.sep)[-2].startswith("ipykernel_") + ) + or obj.co_name == "" + else os.path.basename(obj.co_filename) + ) + co_firstlineno = 1 + # The rest is the same as in the original dill implementation, except for the replacements: + # - obj.co_filename => co_filename + # - obj.co_firstlineno => co_firstlineno + ############################################################################################################ + + if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + obj.co_qualname, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_linetable, + obj.co_endlinetable, + obj.co_columntable, + obj.co_exceptiontable, + obj.co_freevars, + obj.co_cellvars, + ) + elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + obj.co_qualname, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_linetable, + obj.co_exceptiontable, + obj.co_freevars, + obj.co_cellvars, + ) + elif hasattr(obj, "co_linetable"): # python 3.10 (16 args) + args = ( + obj.co_lnotab, # for < python 3.10 [not counted in args] + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_linetable, + obj.co_freevars, + obj.co_cellvars, + ) + elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) + args = ( + obj.co_argcount, + obj.co_posonlyargcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + else: # python 3.7 (15 args) + args = ( + obj.co_argcount, + obj.co_kwonlyargcount, + obj.co_nlocals, + obj.co_stacksize, + obj.co_flags, + obj.co_code, + obj.co_consts, + obj.co_names, + obj.co_varnames, + co_filename, # Modification for huggingface/datasets ############################################ + obj.co_name, + co_firstlineno, # Modification for huggingface/datasets ######################################### + obj.co_lnotab, + obj.co_freevars, + obj.co_cellvars, + ) + + pickler.save_reduce(dill._dill._create_code, args, obj=obj) + dill._dill.logger.trace(pickler, "# Co") + return diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/_filelock.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/_filelock.py new file mode 100644 index 0000000000000000000000000000000000000000..19620e6e777505eaf314366f7f3c657fafc515e0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/_filelock.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2023 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +"""Utilities to handle file locking in `datasets`.""" + +import os + +from filelock import FileLock as FileLock_ +from filelock import UnixFileLock +from filelock import __version__ as _filelock_version +from packaging import version + + +class FileLock(FileLock_): + """ + A `filelock.FileLock` initializer that handles long paths. + It also uses the current umask for lock files. + """ + + MAX_FILENAME_LENGTH = 255 + + def __init__(self, lock_file, *args, **kwargs): + # The "mode" argument is required if we want to use the current umask in filelock >= 3.10 + # In previous previous it was already using the current umask. + if "mode" not in kwargs and version.parse(_filelock_version) >= version.parse("3.10.0"): + umask = os.umask(0o666) + os.umask(umask) + kwargs["mode"] = 0o666 & ~umask + lock_file = self.hash_filename_if_too_long(lock_file) + super().__init__(lock_file, *args, **kwargs) + + @classmethod + def hash_filename_if_too_long(cls, path: str) -> str: + path = os.path.abspath(os.path.expanduser(path)) + filename = os.path.basename(path) + max_filename_length = cls.MAX_FILENAME_LENGTH + if issubclass(cls, UnixFileLock): + max_filename_length = min(max_filename_length, os.statvfs(os.path.dirname(path)).f_namemax) + if len(filename) > max_filename_length: + dirname = os.path.dirname(path) + hashed_filename = str(hash(filename)) + new_filename = ( + filename[: max_filename_length - len(hashed_filename) - 8] + "..." + hashed_filename + ".lock" + ) + return os.path.join(dirname, new_filename) + else: + return path diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/beam_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/beam_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..831354397cf2bb1c0ee464093484d53c037aa95c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/beam_utils.py @@ -0,0 +1,52 @@ +import os + +from apache_beam.io.filesystems import FileSystems +from apache_beam.pipeline import Pipeline + +from .logging import get_logger + + +CHUNK_SIZE = 2 << 20 # 2mb +logger = get_logger(__name__) + + +class BeamPipeline(Pipeline): + """Wrapper over `apache_beam.pipeline.Pipeline` for convenience""" + + def is_local(self): + runner = self._options.get_all_options().get("runner") + return runner in [None, "DirectRunner", "PortableRunner"] + + +def upload_local_to_remote(local_file_path, remote_file_path, force_upload=False): + """Use the Beam Filesystems to upload to a remote directory on gcs/s3/hdfs...""" + fs = FileSystems + if fs.exists(remote_file_path): + if force_upload: + logger.info(f"Remote path already exist: {remote_file_path}. Overwriting it as force_upload=True.") + else: + logger.info(f"Remote path already exist: {remote_file_path}. Skipping it as force_upload=False.") + return + with fs.create(remote_file_path) as remote_file: + with open(local_file_path, "rb") as local_file: + chunk = local_file.read(CHUNK_SIZE) + while chunk: + remote_file.write(chunk) + chunk = local_file.read(CHUNK_SIZE) + + +def download_remote_to_local(remote_file_path, local_file_path, force_download=False): + """Use the Beam Filesystems to download from a remote directory on gcs/s3/hdfs...""" + fs = FileSystems + if os.path.exists(local_file_path): + if force_download: + logger.info(f"Local path already exist: {remote_file_path}. Overwriting it as force_upload=True.") + else: + logger.info(f"Local path already exist: {remote_file_path}. Skipping it as force_upload=False.") + return + with fs.open(remote_file_path) as remote_file: + with open(local_file_path, "wb") as local_file: + chunk = remote_file.read(CHUNK_SIZE) + while chunk: + local_file.write(chunk) + chunk = remote_file.read(CHUNK_SIZE) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/cache.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/cache.py new file mode 100644 index 0000000000000000000000000000000000000000..5485441439d17051eb9a59bc7bc4155321923958 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/cache.py @@ -0,0 +1,283 @@ +import json +import os +import re +import shutil +import tempfile +from contextlib import contextmanager +from functools import partial +from pathlib import Path +from urllib.parse import urljoin, urlparse + +import requests + +from datasets import DownloadConfig, config +from datasets.utils.extract import ExtractManager +from datasets.utils.file_utils import ( + _raise_if_offline_mode_is_enabled, + ftp_get, + ftp_head, + get_authentication_headers_for_url, + hash_url_to_filename, + http_get, + http_head, + is_local_path, + is_remote_url, + logger, +) +from datasets.utils.filelock import FileLock + + +def cached_path( + url_or_filename, + download_config=None, + **download_kwargs, +) -> str: + """ + Given something that might be a URL (or might be a local path), + determine which. If it's a URL, download the file and cache it, and + return the path to the cached file. If it's already a local path, + make sure the file exists and then return the path. + + Return: + Local path (string) + + Raises: + FileNotFoundError: in case of non-recoverable file + (non-existent or no cache on disk) + ConnectionError: in case of unreachable url + and no cache on disk + ValueError: if it couldn't parse the url or filename correctly + requests.exceptions.ConnectionError: in case of internet connection issue + """ + if download_config is None: + download_config = DownloadConfig(**download_kwargs) + + cache_dir = download_config.cache_dir or config.DOWNLOADED_DATASETS_PATH + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + if isinstance(url_or_filename, Path): + url_or_filename = str(url_or_filename) + + if is_remote_url(url_or_filename): + # URL, so get it from the cache (downloading if necessary) + output_path = get_from_cache( + url_or_filename, + cache_dir=cache_dir, + force_download=download_config.force_download, + proxies=download_config.proxies, + resume_download=download_config.resume_download, + user_agent=download_config.user_agent, + local_files_only=download_config.local_files_only, + use_etag=download_config.use_etag, + max_retries=download_config.max_retries, + use_auth_token=download_config.use_auth_token, + ignore_url_params=download_config.ignore_url_params, + download_desc=download_config.download_desc, + ) + elif os.path.exists(url_or_filename): + # File, and it exists. + output_path = url_or_filename + elif is_local_path(url_or_filename): + # File, but it doesn't exist. + raise FileNotFoundError(f"Local file {url_or_filename} doesn't exist") + else: + # Something unknown + raise ValueError(f"unable to parse {url_or_filename} as a URL or as a local path") + + if output_path is None: + return output_path + + if download_config.extract_compressed_file: + output_path = ExtractManager(cache_dir=download_config.cache_dir).extract( + output_path, force_extract=download_config.force_extract + ) + + return output_path + + +def get_from_cache( + url, + cache_dir=None, + force_download=False, + proxies=None, + etag_timeout=100, + resume_download=False, + user_agent=None, + local_files_only=False, + use_etag=True, + max_retries=0, + use_auth_token=None, + ignore_url_params=False, + download_desc=None, +) -> str: + """ + Given a URL, look for the corresponding file in the local cache. + If it's not there, download it. Then return the path to the cached file. + + Return: + Local path (string) + + Raises: + FileNotFoundError: in case of non-recoverable file + (non-existent or no cache on disk) + ConnectionError: in case of unreachable url + and no cache on disk + """ + if cache_dir is None: + cache_dir = config.HF_DATASETS_CACHE + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + os.makedirs(cache_dir, exist_ok=True) + + if ignore_url_params: + # strip all query parameters and #fragments from the URL + cached_url = urljoin(url, urlparse(url).path) + else: + cached_url = url # additional parameters may be added to the given URL + + connected = False + response = None + cookies = None + etag = None + head_error = None + + # Try a first time to file the file on the local file system without eTag (None) + # if we don't ask for 'force_download' then we spare a request + filename = hash_url_to_filename(cached_url, etag=None) + cache_path = os.path.join(cache_dir, filename) + + if os.path.exists(cache_path) and not force_download and not use_etag: + return cache_path + + # Prepare headers for authentication + headers = get_authentication_headers_for_url(url, use_auth_token=use_auth_token) + if user_agent is not None: + headers["user-agent"] = user_agent + + # We don't have the file locally or we need an eTag + if not local_files_only: + if url.startswith("ftp://"): + connected = ftp_head(url) + try: + response = http_head( + url, + allow_redirects=True, + proxies=proxies, + timeout=etag_timeout, + max_retries=max_retries, + headers=headers, + ) + if response.status_code == 200: # ok + etag = response.headers.get("ETag") if use_etag else None + for k, v in response.cookies.items(): + # In some edge cases, we need to get a confirmation token + if k.startswith("download_warning") and "drive.google.com" in url: + url += "&confirm=" + v + cookies = response.cookies + connected = True + # Fix Google Drive URL to avoid Virus scan warning + if "drive.google.com" in url and "confirm=" not in url: + url += "&confirm=t" + # In some edge cases, head request returns 400 but the connection is actually ok + elif ( + (response.status_code == 400 and "firebasestorage.googleapis.com" in url) + or (response.status_code == 405 and "drive.google.com" in url) + or ( + response.status_code == 403 + and ( + re.match(r"^https?://github.com/.*?/.*?/releases/download/.*?/.*?$", url) + or re.match(r"^https://.*?s3.*?amazonaws.com/.*?$", response.url) + ) + ) + or (response.status_code == 403 and "ndownloader.figstatic.com" in url) + ): + connected = True + logger.info(f"Couldn't get ETag version for url {url}") + elif response.status_code == 401 and config.HF_ENDPOINT in url and use_auth_token is None: + raise ConnectionError( + f"Unauthorized for URL {url}. Please use the parameter `use_auth_token=True` after logging in with `huggingface-cli login`" + ) + except (OSError, requests.exceptions.Timeout) as e: + # not connected + head_error = e + pass + + # connected == False = we don't have a connection, or url doesn't exist, or is otherwise inaccessible. + # try to get the last downloaded one + if not connected: + if os.path.exists(cache_path) and not force_download: + return cache_path + if local_files_only: + raise FileNotFoundError( + f"Cannot find the requested files in the cached path at {cache_path} and outgoing traffic has been" + " disabled. To enable file online look-ups, set 'local_files_only' to False." + ) + elif response is not None and response.status_code == 404: + raise FileNotFoundError(f"Couldn't find file at {url}") + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + if head_error is not None: + raise ConnectionError(f"Couldn't reach {url} ({repr(head_error)})") + elif response is not None: + raise ConnectionError(f"Couldn't reach {url} (error {response.status_code})") + else: + raise ConnectionError(f"Couldn't reach {url}") + + # Try a second time + filename = hash_url_to_filename(cached_url, etag) + cache_path = os.path.join(cache_dir, filename) + + if os.path.exists(cache_path) and not force_download: + return cache_path + + # From now on, connected is True. + # Prevent parallel downloads of the same file with a lock. + lock_path = cache_path + ".lock" + with FileLock(lock_path): + if resume_download: + incomplete_path = cache_path + ".incomplete" + + @contextmanager + def _resumable_file_manager(): + with open(incomplete_path, "a+b") as f: + yield f + + temp_file_manager = _resumable_file_manager + if os.path.exists(incomplete_path): + resume_size = os.stat(incomplete_path).st_size + else: + resume_size = 0 + else: + temp_file_manager = partial(tempfile.NamedTemporaryFile, dir=cache_dir, delete=False) + resume_size = 0 + + # Download to temporary file, then copy to cache dir once finished. + # Otherwise you get corrupt cache entries if the download gets interrupted. + with temp_file_manager() as temp_file: + logger.info(f"{url} not found in cache or force_download set to True, downloading to {temp_file.name}") + + # GET file object + if url.startswith("ftp://"): + ftp_get(url, temp_file) + else: + http_get( + url, + temp_file, + proxies=proxies, + resume_size=resume_size, + headers=headers, + cookies=cookies, + max_retries=max_retries, + desc=download_desc, + ) + + logger.info(f"storing {url} in cache at {cache_path}") + shutil.move(temp_file.name, cache_path) + + logger.info(f"creating metadata file for {cache_path}") + meta = {"url": url, "etag": etag} + meta_path = cache_path + ".json" + with open(meta_path, "w", encoding="utf-8") as meta_file: + json.dump(meta, meta_file) + + return cache_path diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/deprecation_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/deprecation_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f05ecbeaa3eae5476e99c461dbede9ebfa111eb0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/deprecation_utils.py @@ -0,0 +1,105 @@ +import enum +import inspect +import warnings +from functools import wraps +from typing import Callable, Optional + +from .logging import get_logger + + +_emitted_deprecation_warnings = set() +logger = get_logger(__name__) + + +def deprecated(help_message: Optional[str] = None): + """Decorator to mark a class or a function as deprecated. + + Args: + help_message (:obj:`str`, optional): An optional message to guide the user on how to + switch to non-deprecated usage of the library. + """ + + def decorator(deprecated_class_or_function: Callable): + global _emitted_deprecation_warnings + + if inspect.isclass(deprecated_class_or_function): + deprecated_function = deprecated_class_or_function.__init__ + name = deprecated_class_or_function.__name__ + else: + deprecated_function = deprecated_class_or_function + name = deprecated_function.__name__ + # Support deprecating __init__ class method: class name instead + name = name if name != "__init__" else deprecated_function.__qualname__.split(".")[-2] + + warning_msg = ( + f"{name} is deprecated and will be removed in the next major version of datasets." + f" {help_message}" + if help_message + else "" + ) + + @wraps(deprecated_function) + def wrapper(*args, **kwargs): + func_hash = hash(deprecated_function) + if func_hash not in _emitted_deprecation_warnings: + warnings.warn(warning_msg, category=FutureWarning, stacklevel=2) + _emitted_deprecation_warnings.add(func_hash) + return deprecated_function(*args, **kwargs) + + wrapper._decorator_name_ = "deprecated" + + if inspect.isclass(deprecated_class_or_function): + deprecated_class_or_function.__init__ = wrapper + return deprecated_class_or_function + else: + return wrapper + + return decorator + + +class OnAccess(enum.EnumMeta): + """ + Enum metaclass that calls a user-specified function whenever a member is accessed. + """ + + def __getattribute__(cls, name): + obj = super().__getattribute__(name) + if isinstance(obj, enum.Enum) and obj._on_access: + obj._on_access() + return obj + + def __getitem__(cls, name): + member = super().__getitem__(name) + if member._on_access: + member._on_access() + return member + + def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1): + obj = super().__call__(value, names, module=module, qualname=qualname, type=type, start=start) + if isinstance(obj, enum.Enum) and obj._on_access: + obj._on_access() + return obj + + +class DeprecatedEnum(enum.Enum, metaclass=OnAccess): + """ + Enum class that calls `deprecate` method whenever a member is accessed. + """ + + def __new__(cls, value): + member = object.__new__(cls) + member._value_ = value + member._on_access = member.deprecate + return member + + @property + def help_message(self): + return "" + + def deprecate(self): + help_message = f" {self.help_message}" if self.help_message else "" + warnings.warn( + f"'{self.__objclass__.__name__}' is deprecated and will be removed in the next major version of datasets." + + help_message, + FutureWarning, + stacklevel=3, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/doc_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/doc_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6ef8bcb4e70725ad086cb817e0ec4551d1c0966e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/doc_utils.py @@ -0,0 +1,15 @@ +from typing import Callable + + +def is_documented_by(function_with_docstring: Callable): + """Decorator to share docstrings across common functions. + + Args: + function_with_docstring (`Callable`): Name of the function with the docstring. + """ + + def wrapper(target_function): + target_function.__doc__ = function_with_docstring.__doc__ + return target_function + + return wrapper diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/download_manager.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/download_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..b524c2f9686f65d083c424a4e17d001395b743b6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/download_manager.py @@ -0,0 +1 @@ +# deprecated, please use datasets.download.download_manager diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/experimental.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/experimental.py new file mode 100644 index 0000000000000000000000000000000000000000..cc406154e9347f4df83b1f7b08c32a961d469f6a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/experimental.py @@ -0,0 +1,43 @@ +"""Contains utilities to flag a feature as "experimental" in datasets.""" + +import warnings +from functools import wraps +from typing import Callable + + +def experimental(fn: Callable) -> Callable: + """Decorator to flag a feature as experimental. + + An experimental feature trigger a warning when used as it might be subject to breaking changes in the future. + + Args: + fn (`Callable`): + The function to flag as experimental. + + Returns: + `Callable`: The decorated function. + + Example: + + ```python + >>> from datasets.utils import experimental + + >>> @experimental + ... def my_function(): + ... print("Hello world!") + + >>> my_function() + UserWarning: 'my_function' is experimental and might be subject to breaking changes in the future. + Hello world! + ``` + """ + + @wraps(fn) + def _inner_fn(*args, **kwargs): + warnings.warn( + (f"'{fn.__name__}' is experimental and might be subject to breaking changes in the future."), + UserWarning, + ) + return fn(*args, **kwargs) + + return _inner_fn diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/extract.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/extract.py new file mode 100644 index 0000000000000000000000000000000000000000..39e3babf97b7563d39d8dbf59f2203b460c8ac21 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/extract.py @@ -0,0 +1,351 @@ +import bz2 +import gzip +import lzma +import os +import shutil +import struct +import tarfile +import warnings +import zipfile +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Dict, List, Optional, Type, Union + +from .. import config +from ._filelock import FileLock +from .logging import get_logger + + +logger = get_logger(__name__) + + +class ExtractManager: + def __init__(self, cache_dir: Optional[str] = None): + self.extract_dir = ( + os.path.join(cache_dir, config.EXTRACTED_DATASETS_DIR) if cache_dir else config.EXTRACTED_DATASETS_PATH + ) + self.extractor = Extractor + + def _get_output_path(self, path: str) -> str: + from .file_utils import hash_url_to_filename + + # Path where we extract compressed archives + # We extract in the cache dir, and get the extracted path name by hashing the original path" + abs_path = os.path.abspath(path) + return os.path.join(self.extract_dir, hash_url_to_filename(abs_path)) + + def _do_extract(self, output_path: str, force_extract: bool) -> bool: + return force_extract or ( + not os.path.isfile(output_path) and not (os.path.isdir(output_path) and os.listdir(output_path)) + ) + + def extract(self, input_path: str, force_extract: bool = False) -> str: + extractor_format = self.extractor.infer_extractor_format(input_path) + if not extractor_format: + return input_path + output_path = self._get_output_path(input_path) + if self._do_extract(output_path, force_extract): + self.extractor.extract(input_path, output_path, extractor_format) + return output_path + + +class BaseExtractor(ABC): + @classmethod + @abstractmethod + def is_extractable(cls, path: Union[Path, str], **kwargs) -> bool: ... + + @staticmethod + @abstractmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: ... + + +class MagicNumberBaseExtractor(BaseExtractor, ABC): + magic_numbers: List[bytes] = [] + + @staticmethod + def read_magic_number(path: Union[Path, str], magic_number_length: int): + with open(path, "rb") as f: + return f.read(magic_number_length) + + @classmethod + def is_extractable(cls, path: Union[Path, str], magic_number: bytes = b"") -> bool: + if not magic_number: + magic_number_length = max(len(cls_magic_number) for cls_magic_number in cls.magic_numbers) + try: + magic_number = cls.read_magic_number(path, magic_number_length) + except OSError: + return False + return any(magic_number.startswith(cls_magic_number) for cls_magic_number in cls.magic_numbers) + + +class TarExtractor(BaseExtractor): + @classmethod + def is_extractable(cls, path: Union[Path, str], **kwargs) -> bool: + return tarfile.is_tarfile(path) + + @staticmethod + def safemembers(members, output_path): + """ + Fix for CVE-2007-4559 + Desc: + Directory traversal vulnerability in the (1) extract and (2) extractall functions in the tarfile + module in Python allows user-assisted remote attackers to overwrite arbitrary files via a .. (dot dot) + sequence in filenames in a TAR archive, a related issue to CVE-2001-1267. + See: https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2007-4559 + From: https://stackoverflow.com/a/10077309 + """ + + def resolved(path: str) -> str: + return os.path.realpath(os.path.abspath(path)) + + def badpath(path: str, base: str) -> bool: + # joinpath will ignore base if path is absolute + return not resolved(os.path.join(base, path)).startswith(base) + + def badlink(info, base: str) -> bool: + # Links are interpreted relative to the directory containing the link + tip = resolved(os.path.join(base, os.path.dirname(info.name))) + return badpath(info.linkname, base=tip) + + base = resolved(output_path) + + for finfo in members: + if badpath(finfo.name, base): + logger.error(f"Extraction of {finfo.name} is blocked (illegal path)") + elif finfo.issym() and badlink(finfo, base): + logger.error(f"Extraction of {finfo.name} is blocked: Symlink to {finfo.linkname}") + elif finfo.islnk() and badlink(finfo, base): + logger.error(f"Extraction of {finfo.name} is blocked: Hard link to {finfo.linkname}") + else: + yield finfo + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + os.makedirs(output_path, exist_ok=True) + tar_file = tarfile.open(input_path) + tar_file.extractall(output_path, members=TarExtractor.safemembers(tar_file, output_path)) + tar_file.close() + + +class GzipExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x1f\x8b"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + with gzip.open(input_path, "rb") as gzip_file: + with open(output_path, "wb") as extracted_file: + shutil.copyfileobj(gzip_file, extracted_file) + + +class ZipExtractor(MagicNumberBaseExtractor): + magic_numbers = [ + b"PK\x03\x04", + b"PK\x05\x06", # empty archive + b"PK\x07\x08", # spanned archive + ] + + @classmethod + def is_extractable(cls, path: Union[Path, str], magic_number: bytes = b"") -> bool: + if super().is_extractable(path, magic_number=magic_number): + return True + try: + # Alternative version of zipfile.is_zipfile that has less false positives, but misses executable zip archives. + # From: https://github.com/python/cpython/pull/5053 + from zipfile import ( + _CD_SIGNATURE, + _ECD_DISK_NUMBER, + _ECD_DISK_START, + _ECD_ENTRIES_TOTAL, + _ECD_OFFSET, + _ECD_SIZE, + _EndRecData, + sizeCentralDir, + stringCentralDir, + structCentralDir, + ) + + with open(path, "rb") as fp: + endrec = _EndRecData(fp) + if endrec: + if endrec[_ECD_ENTRIES_TOTAL] == 0 and endrec[_ECD_SIZE] == 0 and endrec[_ECD_OFFSET] == 0: + return True # Empty zipfiles are still zipfiles + elif endrec[_ECD_DISK_NUMBER] == endrec[_ECD_DISK_START]: + fp.seek(endrec[_ECD_OFFSET]) # Central directory is on the same disk + if fp.tell() == endrec[_ECD_OFFSET] and endrec[_ECD_SIZE] >= sizeCentralDir: + data = fp.read(sizeCentralDir) # CD is where we expect it to be + if len(data) == sizeCentralDir: + centdir = struct.unpack(structCentralDir, data) # CD is the right size + if centdir[_CD_SIGNATURE] == stringCentralDir: + return True # First central directory entry has correct magic number + return False + except Exception: # catch all errors in case future python versions change the zipfile internals + return False + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + os.makedirs(output_path, exist_ok=True) + with zipfile.ZipFile(input_path, "r") as zip_file: + zip_file.extractall(output_path) + zip_file.close() + + +class XzExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"\xfd\x37\x7a\x58\x5a\x00"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + with lzma.open(input_path) as compressed_file: + with open(output_path, "wb") as extracted_file: + shutil.copyfileobj(compressed_file, extracted_file) + + +class RarExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"Rar!\x1a\x07\x00", b"Rar!\x1a\x07\x01\x00"] # RAR_ID # RAR5_ID + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + if not config.RARFILE_AVAILABLE: + raise ImportError("Please pip install rarfile") + import rarfile + + os.makedirs(output_path, exist_ok=True) + rf = rarfile.RarFile(input_path) + rf.extractall(output_path) + rf.close() + + +class ZstdExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x28\xb5\x2f\xfd"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + if not config.ZSTANDARD_AVAILABLE: + raise ImportError("Please pip install zstandard") + import zstandard as zstd + + dctx = zstd.ZstdDecompressor() + with open(input_path, "rb") as ifh, open(output_path, "wb") as ofh: + dctx.copy_stream(ifh, ofh) + + +class Bzip2Extractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x42\x5a\x68"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + with bz2.open(input_path, "rb") as compressed_file: + with open(output_path, "wb") as extracted_file: + shutil.copyfileobj(compressed_file, extracted_file) + + +class SevenZipExtractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x37\x7a\xbc\xaf\x27\x1c"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + if not config.PY7ZR_AVAILABLE: + raise ImportError("Please pip install py7zr") + import py7zr + + os.makedirs(output_path, exist_ok=True) + with py7zr.SevenZipFile(input_path, "r") as archive: + archive.extractall(output_path) + + +class Lz4Extractor(MagicNumberBaseExtractor): + magic_numbers = [b"\x04\x22\x4d\x18"] + + @staticmethod + def extract(input_path: Union[Path, str], output_path: Union[Path, str]) -> None: + if not config.LZ4_AVAILABLE: + raise ImportError("Please pip install lz4") + import lz4.frame + + with lz4.frame.open(input_path, "rb") as compressed_file: + with open(output_path, "wb") as extracted_file: + shutil.copyfileobj(compressed_file, extracted_file) + + +class Extractor: + # Put zip file to the last, b/c it is possible wrongly detected as zip (I guess it means: as tar or gzip) + extractors: Dict[str, Type[BaseExtractor]] = { + "tar": TarExtractor, + "gzip": GzipExtractor, + "zip": ZipExtractor, + "xz": XzExtractor, + "rar": RarExtractor, + "zstd": ZstdExtractor, + "bz2": Bzip2Extractor, + "7z": SevenZipExtractor, # + "lz4": Lz4Extractor, # + } + + @classmethod + def _get_magic_number_max_length(cls): + return max( + len(extractor_magic_number) + for extractor in cls.extractors.values() + if issubclass(extractor, MagicNumberBaseExtractor) + for extractor_magic_number in extractor.magic_numbers + ) + + @staticmethod + def _read_magic_number(path: Union[Path, str], magic_number_length: int): + try: + return MagicNumberBaseExtractor.read_magic_number(path, magic_number_length=magic_number_length) + except OSError: + return b"" + + @classmethod + def is_extractable(cls, path: Union[Path, str], return_extractor: bool = False) -> bool: + warnings.warn( + "Method 'is_extractable' was deprecated in version 2.4.0 and will be removed in 3.0.0. " + "Use 'infer_extractor_format' instead.", + category=FutureWarning, + ) + extractor_format = cls.infer_extractor_format(path) + if extractor_format: + return True if not return_extractor else (True, cls.extractors[extractor_format]) + return False if not return_extractor else (False, None) + + @classmethod + def infer_extractor_format(cls, path: Union[Path, str]) -> Optional[str]: # + magic_number_max_length = cls._get_magic_number_max_length() + magic_number = cls._read_magic_number(path, magic_number_max_length) + for extractor_format, extractor in cls.extractors.items(): + if extractor.is_extractable(path, magic_number=magic_number): + return extractor_format + + @classmethod + def extract( + cls, + input_path: Union[Path, str], + output_path: Union[Path, str], + extractor_format: Optional[str] = None, # + extractor: Optional[BaseExtractor] = "deprecated", + ) -> None: + os.makedirs(os.path.dirname(output_path), exist_ok=True) + # Prevent parallel extractions + lock_path = str(Path(output_path).with_suffix(".lock")) + with FileLock(lock_path): + shutil.rmtree(output_path, ignore_errors=True) + if extractor_format or extractor != "deprecated": + if extractor != "deprecated" or not isinstance(extractor_format, str): # passed as positional arg + warnings.warn( + "Parameter 'extractor' was deprecated in version 2.4.0 and will be removed in 3.0.0. " + "Use 'extractor_format' instead.", + category=FutureWarning, + ) + extractor = extractor if extractor != "deprecated" else extractor_format + else: + extractor = cls.extractors[extractor_format] + return extractor.extract(input_path, output_path) + else: + warnings.warn( + "Parameter 'extractor_format' was made required in version 2.4.0 and not passing it will raise an " + "exception in 3.0.0.", + category=FutureWarning, + ) + for extractor in cls.extractors.values(): + if extractor.is_extractable(input_path): + return extractor.extract(input_path, output_path) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/file_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/file_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..261ea25899241ee03b41bc803eaf9b9d6d93129e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/file_utils.py @@ -0,0 +1,1677 @@ +""" +Utilities for working with the local dataset cache. +This file is adapted from the AllenNLP library at https://github.com/allenai/allennlp +Copyright by the AllenNLP authors. +""" + +import copy +import glob +import io +import json +import multiprocessing +import os +import posixpath +import re +import shutil +import sys +import tarfile +import time +import urllib +import warnings +import xml.dom.minidom +import zipfile +from asyncio import TimeoutError +from contextlib import closing, contextmanager +from functools import partial +from io import BytesIO +from itertools import chain +from pathlib import Path, PurePosixPath +from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, TypeVar, Union +from unittest.mock import patch +from urllib.parse import urljoin, urlparse +from xml.etree import ElementTree as ET + +import fsspec +import huggingface_hub +import requests +from aiohttp.client_exceptions import ClientError +from fsspec.core import strip_protocol, url_to_fs +from fsspec.utils import can_be_local +from huggingface_hub.utils import EntryNotFoundError, insecure_hashlib +from packaging import version + +from .. import __version__, config +from ..download.download_config import DownloadConfig +from ..filesystems import COMPRESSION_FILESYSTEMS +from . import _tqdm, logging +from . import tqdm as hf_tqdm +from ._filelock import FileLock +from .extract import ExtractManager +from .track import TrackedIterable + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +INCOMPLETE_SUFFIX = ".incomplete" + +T = TypeVar("T", str, Path) + + +def init_hf_modules(hf_modules_cache: Optional[Union[Path, str]] = None) -> str: + """ + Add hf_modules_cache to the python path. + By default hf_modules_cache='~/.cache/huggingface/modules'. + It can also be set with the environment variable HF_MODULES_CACHE. + This is used to add modules such as `datasets_modules` + """ + hf_modules_cache = hf_modules_cache if hf_modules_cache is not None else config.HF_MODULES_CACHE + hf_modules_cache = str(hf_modules_cache) + if hf_modules_cache not in sys.path: + sys.path.append(hf_modules_cache) + + os.makedirs(hf_modules_cache, exist_ok=True) + if not os.path.exists(os.path.join(hf_modules_cache, "__init__.py")): + with open(os.path.join(hf_modules_cache, "__init__.py"), "w"): + pass + return hf_modules_cache + + +def is_remote_url(url_or_filename: str) -> bool: + return urlparse(url_or_filename).scheme != "" and not os.path.ismount(urlparse(url_or_filename).scheme + ":/") + + +def is_local_path(url_or_filename: str) -> bool: + # On unix the scheme of a local path is empty (for both absolute and relative), + # while on windows the scheme is the drive name (ex: "c") for absolute paths. + # for details on the windows behavior, see https://bugs.python.org/issue42215 + return urlparse(url_or_filename).scheme == "" or os.path.ismount(urlparse(url_or_filename).scheme + ":/") + + +def is_relative_path(url_or_filename: str) -> bool: + return urlparse(url_or_filename).scheme == "" and not os.path.isabs(url_or_filename) + + +def relative_to_absolute_path(path: T) -> T: + """Convert relative path to absolute path.""" + abs_path_str = os.path.abspath(os.path.expanduser(os.path.expandvars(str(path)))) + return Path(abs_path_str) if isinstance(path, Path) else abs_path_str + + +def hf_bucket_url(identifier: str, filename: str, use_cdn=False, dataset=True) -> str: + if dataset: + endpoint = config.CLOUDFRONT_DATASETS_DISTRIB_PREFIX if use_cdn else config.S3_DATASETS_BUCKET_PREFIX + else: + endpoint = config.CLOUDFRONT_METRICS_DISTRIB_PREFIX if use_cdn else config.S3_METRICS_BUCKET_PREFIX + return "/".join((endpoint, identifier, filename)) + + +def head_hf_s3( + identifier: str, filename: str, use_cdn=False, dataset=True, max_retries=0 +) -> Union[requests.Response, Exception]: + return http_head( + hf_bucket_url(identifier=identifier, filename=filename, use_cdn=use_cdn, dataset=dataset), + max_retries=max_retries, + ) + + +def hf_github_url(path: str, name: str, dataset=True, revision: Optional[str] = None) -> str: + default_revision = "main" if version.parse(__version__).is_devrelease else __version__ + revision = revision or default_revision + if dataset: + return config.REPO_DATASETS_URL.format(revision=revision, path=path, name=name) + else: + return config.REPO_METRICS_URL.format(revision=revision, path=path, name=name) + + +def url_or_path_join(base_name: str, *pathnames: str) -> str: + if is_remote_url(base_name): + return posixpath.join(base_name, *(str(pathname).replace(os.sep, "/").lstrip("/") for pathname in pathnames)) + else: + return Path(base_name, *pathnames).as_posix() + + +def url_or_path_parent(url_or_path: str) -> str: + if is_remote_url(url_or_path): + return url_or_path[: url_or_path.rindex("/")] + else: + return os.path.dirname(url_or_path) + + +def hash_url_to_filename(url, etag=None): + """ + Convert `url` into a hashed filename in a repeatable way. + If `etag` is specified, append its hash to the url's, delimited + by a period. + If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name + so that TF 2.0 can identify it as a HDF5 file + (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) + """ + url_bytes = url.encode("utf-8") + url_hash = insecure_hashlib.sha256(url_bytes) + filename = url_hash.hexdigest() + + if etag: + etag_bytes = etag.encode("utf-8") + etag_hash = insecure_hashlib.sha256(etag_bytes) + filename += "." + etag_hash.hexdigest() + + if url.endswith(".py"): + filename += ".py" + + return filename + + +def cached_path( + url_or_filename, + download_config=None, + **download_kwargs, +) -> str: + """ + Given something that might be a URL (or might be a local path), + determine which. If it's a URL, download the file and cache it, and + return the path to the cached file. If it's already a local path, + make sure the file exists and then return the path. + + Return: + Local path (string) + + Raises: + FileNotFoundError: in case of non-recoverable file + (non-existent or no cache on disk) + ConnectionError: in case of unreachable url + and no cache on disk + ValueError: if it couldn't parse the url or filename correctly + requests.exceptions.ConnectionError: in case of internet connection issue + """ + if download_config is None: + download_config = DownloadConfig(**download_kwargs) + + cache_dir = download_config.cache_dir or config.DOWNLOADED_DATASETS_PATH + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + if isinstance(url_or_filename, Path): + url_or_filename = str(url_or_filename) + + # Convert fsspec URL in the format "file://local/path" to "local/path" + if can_be_local(url_or_filename): + url_or_filename = strip_protocol(url_or_filename) + + if is_remote_url(url_or_filename): + # URL, so get it from the cache (downloading if necessary) + output_path = get_from_cache( + url_or_filename, + cache_dir=cache_dir, + force_download=download_config.force_download, + proxies=download_config.proxies, + resume_download=download_config.resume_download, + user_agent=download_config.user_agent, + local_files_only=download_config.local_files_only, + use_etag=download_config.use_etag, + max_retries=download_config.max_retries, + token=download_config.token, + ignore_url_params=download_config.ignore_url_params, + storage_options=download_config.storage_options, + download_desc=download_config.download_desc, + disable_tqdm=download_config.disable_tqdm, + ) + elif os.path.exists(url_or_filename): + # File, and it exists. + output_path = url_or_filename + elif is_local_path(url_or_filename): + # File, but it doesn't exist. + raise FileNotFoundError(f"Local file {url_or_filename} doesn't exist") + else: + # Something unknown + raise ValueError(f"unable to parse {url_or_filename} as a URL or as a local path") + + if output_path is None: + return output_path + + if download_config.extract_compressed_file: + if download_config.extract_on_the_fly: + # Add a compression prefix to the compressed file so that it can be extracted + # as it's being read using xopen. + protocol = _get_extraction_protocol(output_path, download_config=download_config) + extension = _get_path_extension(url_or_filename.split("::")[0]) + if ( + protocol + and extension not in ["tgz", "tar"] + and not url_or_filename.split("::")[0].endswith((".tar.gz", ".tar.bz2", ".tar.xz")) + ): + output_path = relative_to_absolute_path(output_path) + if protocol in SINGLE_FILE_COMPRESSION_PROTOCOLS: + # there is one single file which is the uncompressed file + inner_file = os.path.basename(output_path) + inner_file = inner_file[: inner_file.rindex(".")] if "." in inner_file else inner_file + output_path = f"{protocol}://{inner_file}::{output_path}" + else: + output_path = f"{protocol}://::{output_path}" + return output_path + + # Eager extraction + output_path = ExtractManager(cache_dir=download_config.cache_dir).extract( + output_path, force_extract=download_config.force_extract + ) + return relative_to_absolute_path(output_path) + + +def get_datasets_user_agent(user_agent: Optional[Union[str, dict]] = None) -> str: + ua = f"datasets/{__version__}" + ua += f"; python/{config.PY_VERSION}" + ua += f"; huggingface_hub/{huggingface_hub.__version__}" + ua += f"; pyarrow/{config.PYARROW_VERSION}" + if config.TORCH_AVAILABLE: + ua += f"; torch/{config.TORCH_VERSION}" + if config.TF_AVAILABLE: + ua += f"; tensorflow/{config.TF_VERSION}" + if config.JAX_AVAILABLE: + ua += f"; jax/{config.JAX_VERSION}" + if config.BEAM_AVAILABLE: + ua += f"; apache_beam/{config.BEAM_VERSION}" + if isinstance(user_agent, dict): + ua += f"; {'; '.join(f'{k}/{v}' for k, v in user_agent.items())}" + elif isinstance(user_agent, str): + ua += "; " + user_agent + return ua + + +def get_authentication_headers_for_url( + url: str, token: Optional[Union[str, bool]] = None, use_auth_token: Optional[Union[str, bool]] = "deprecated" +) -> dict: + """Handle the HF authentication""" + if use_auth_token != "deprecated": + warnings.warn( + "'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0.\n" + f"You can remove this warning by passing 'token={use_auth_token}' instead.", + FutureWarning, + ) + token = use_auth_token + if url.startswith(config.HF_ENDPOINT): + return huggingface_hub.utils.build_hf_headers( + token=token, library_name="datasets", library_version=__version__ + ) + else: + return {} + + +class OfflineModeIsEnabled(ConnectionError): + pass + + +def _raise_if_offline_mode_is_enabled(msg: Optional[str] = None): + """Raise an OfflineModeIsEnabled error (subclass of ConnectionError) if HF_DATASETS_OFFLINE is True.""" + if config.HF_DATASETS_OFFLINE: + raise OfflineModeIsEnabled( + "Offline mode is enabled." if msg is None else "Offline mode is enabled. " + str(msg) + ) + + +def _request_with_retry( + method: str, + url: str, + max_retries: int = 0, + base_wait_time: float = 0.5, + max_wait_time: float = 2, + timeout: float = 10.0, + **params, +) -> requests.Response: + """Wrapper around requests to retry in case it fails with a ConnectTimeout, with exponential backoff. + + Note that if the environment variable HF_DATASETS_OFFLINE is set to 1, then a OfflineModeIsEnabled error is raised. + + Args: + method (str): HTTP method, such as 'GET' or 'HEAD'. + url (str): The URL of the resource to fetch. + max_retries (int): Maximum number of retries, defaults to 0 (no retries). + base_wait_time (float): Duration (in seconds) to wait before retrying the first time. Wait time between + retries then grows exponentially, capped by max_wait_time. + max_wait_time (float): Maximum amount of time between two retries, in seconds. + **params (additional keyword arguments): Params to pass to :obj:`requests.request`. + """ + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + tries, success = 0, False + while not success: + tries += 1 + try: + response = requests.request(method=method.upper(), url=url, timeout=timeout, **params) + success = True + except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError) as err: + if tries > max_retries: + raise err + else: + logger.info(f"{method} request to {url} timed out, retrying... [{tries/max_retries}]") + sleep_time = min(max_wait_time, base_wait_time * 2 ** (tries - 1)) # Exponential backoff + time.sleep(sleep_time) + return response + + +def fsspec_head(url, storage_options=None): + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + fs, path = url_to_fs(url, **(storage_options or {})) + return fs.info(path) + + +def stack_multiprocessing_download_progress_bars(): + # Stack downloads progress bars automatically using HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS=1 + # We use environment variables since the download may happen in a subprocess + return patch.dict(os.environ, {"HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS": "1"}) + + +class TqdmCallback(fsspec.callbacks.TqdmCallback): + def __init__(self, tqdm_kwargs=None, *args, **kwargs): + if config.FSSPEC_VERSION < version.parse("2024.2.0"): + super().__init__(tqdm_kwargs, *args, **kwargs) + self._tqdm = _tqdm # replace tqdm module by datasets.utils.tqdm module + else: + kwargs["tqdm_cls"] = _tqdm.tqdm + super().__init__(tqdm_kwargs, *args, **kwargs) + + +def fsspec_get(url, temp_file, storage_options=None, desc=None, disable_tqdm=False): + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + fs, path = url_to_fs(url, **(storage_options or {})) + callback = TqdmCallback( + tqdm_kwargs={ + "desc": desc or "Downloading", + "unit": "B", + "unit_scale": True, + "position": multiprocessing.current_process()._identity[-1] # contains the ranks of subprocesses + if os.environ.get("HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS") == "1" + and multiprocessing.current_process()._identity + else None, + "disable": disable_tqdm, + } + ) + fs.get_file(path, temp_file.name, callback=callback) + + +def ftp_head(url, timeout=10.0): + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + try: + with closing(urllib.request.urlopen(url, timeout=timeout)) as r: + r.read(1) + except Exception: + return False + return True + + +def ftp_get(url, temp_file, timeout=10.0): + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + try: + logger.info(f"Getting through FTP {url} into {temp_file.name}") + with closing(urllib.request.urlopen(url, timeout=timeout)) as r: + shutil.copyfileobj(r, temp_file) + except urllib.error.URLError as e: + raise ConnectionError(e) from None + + +def http_get( + url, + temp_file, + proxies=None, + resume_size=0, + headers=None, + cookies=None, + timeout=100.0, + max_retries=0, + desc=None, + disable_tqdm=False, +) -> Optional[requests.Response]: + headers = dict(headers) if headers is not None else {} + headers["user-agent"] = get_datasets_user_agent(user_agent=headers.get("user-agent")) + if resume_size > 0: + headers["Range"] = f"bytes={resume_size:d}-" + response = _request_with_retry( + method="GET", + url=url, + stream=True, + proxies=proxies, + headers=headers, + cookies=cookies, + max_retries=max_retries, + timeout=timeout, + ) + if temp_file is None: + return response + if response.status_code == 416: # Range not satisfiable + return + content_length = response.headers.get("Content-Length") + total = resume_size + int(content_length) if content_length is not None else None + with hf_tqdm( + unit="B", + unit_scale=True, + total=total, + initial=resume_size, + desc=desc or "Downloading", + position=multiprocessing.current_process()._identity[-1] # contains the ranks of subprocesses + if os.environ.get("HF_DATASETS_STACK_MULTIPROCESSING_DOWNLOAD_PROGRESS_BARS") == "1" + and multiprocessing.current_process()._identity + else None, + disable=disable_tqdm, + ) as progress: + for chunk in response.iter_content(chunk_size=1024): + progress.update(len(chunk)) + temp_file.write(chunk) + + +def http_head( + url, proxies=None, headers=None, cookies=None, allow_redirects=True, timeout=10.0, max_retries=0 +) -> requests.Response: + headers = copy.deepcopy(headers) or {} + headers["user-agent"] = get_datasets_user_agent(user_agent=headers.get("user-agent")) + response = _request_with_retry( + method="HEAD", + url=url, + proxies=proxies, + headers=headers, + cookies=cookies, + allow_redirects=allow_redirects, + timeout=timeout, + max_retries=max_retries, + ) + return response + + +def request_etag( + url: str, token: Optional[Union[str, bool]] = None, use_auth_token: Optional[Union[str, bool]] = "deprecated" +) -> Optional[str]: + if use_auth_token != "deprecated": + warnings.warn( + "'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0.\n" + f"You can remove this warning by passing 'token={use_auth_token}' instead.", + FutureWarning, + ) + token = use_auth_token + if urlparse(url).scheme not in ("http", "https"): + return None + headers = get_authentication_headers_for_url(url, token=token) + response = http_head(url, headers=headers, max_retries=3) + response.raise_for_status() + etag = response.headers.get("ETag") if response.ok else None + return etag + + +def get_from_cache( + url, + cache_dir=None, + force_download=False, + proxies=None, + etag_timeout=100, + resume_download=False, + user_agent=None, + local_files_only=False, + use_etag=True, + max_retries=0, + token=None, + use_auth_token="deprecated", + ignore_url_params=False, + storage_options=None, + download_desc=None, + disable_tqdm=False, +) -> str: + """ + Given a URL, look for the corresponding file in the local cache. + If it's not there, download it. Then return the path to the cached file. + + Return: + Local path (string) + + Raises: + FileNotFoundError: in case of non-recoverable file + (non-existent or no cache on disk) + ConnectionError: in case of unreachable url + and no cache on disk + """ + if use_auth_token != "deprecated": + warnings.warn( + "'use_auth_token' was deprecated in favor of 'token' in version 2.14.0 and will be removed in 3.0.0.\n" + f"You can remove this warning by passing 'token={use_auth_token}' instead.", + FutureWarning, + ) + token = use_auth_token + if cache_dir is None: + cache_dir = config.HF_DATASETS_CACHE + if isinstance(cache_dir, Path): + cache_dir = str(cache_dir) + + os.makedirs(cache_dir, exist_ok=True) + + if ignore_url_params: + # strip all query parameters and #fragments from the URL + cached_url = urljoin(url, urlparse(url).path) + else: + cached_url = url # additional parameters may be added to the given URL + + connected = False + response = None + cookies = None + etag = None + head_error = None + scheme = None + + # Try a first time to file the file on the local file system without eTag (None) + # if we don't ask for 'force_download' then we spare a request + filename = hash_url_to_filename(cached_url, etag=None) + cache_path = os.path.join(cache_dir, filename) + + if os.path.exists(cache_path) and not force_download and not use_etag: + return cache_path + + # Prepare headers for authentication + headers = get_authentication_headers_for_url(url, token=token) + if user_agent is not None: + headers["user-agent"] = user_agent + + # We don't have the file locally or we need an eTag + if not local_files_only: + scheme = urlparse(url).scheme + if scheme == "ftp": + connected = ftp_head(url) + elif scheme not in ("http", "https"): + response = fsspec_head(url, storage_options=storage_options) + # s3fs uses "ETag", gcsfs uses "etag" + etag = (response.get("ETag", None) or response.get("etag", None)) if use_etag else None + connected = True + try: + response = http_head( + url, + allow_redirects=True, + proxies=proxies, + timeout=etag_timeout, + max_retries=max_retries, + headers=headers, + ) + if response.status_code == 200: # ok + etag = response.headers.get("ETag") if use_etag else None + for k, v in response.cookies.items(): + # In some edge cases, we need to get a confirmation token + if k.startswith("download_warning") and "drive.google.com" in url: + url += "&confirm=" + v + cookies = response.cookies + connected = True + # Fix Google Drive URL to avoid Virus scan warning + if "drive.google.com" in url and "confirm=" not in url: + url += "&confirm=t" + # In some edge cases, head request returns 400 but the connection is actually ok + elif ( + (response.status_code == 400 and "firebasestorage.googleapis.com" in url) + or (response.status_code == 405 and "drive.google.com" in url) + or ( + response.status_code == 403 + and ( + re.match(r"^https?://github.com/.*?/.*?/releases/download/.*?/.*?$", url) + or re.match(r"^https://.*?s3.*?amazonaws.com/.*?$", response.url) + ) + ) + or (response.status_code == 403 and "ndownloader.figstatic.com" in url) + ): + connected = True + logger.info(f"Couldn't get ETag version for url {url}") + elif response.status_code == 401 and config.HF_ENDPOINT in url and token is None: + raise ConnectionError( + f"Unauthorized for URL {url}. Please use the parameter `token=True` after logging in with `huggingface-cli login`" + ) + except (OSError, requests.exceptions.Timeout) as e: + # not connected + head_error = e + pass + + # connected == False = we don't have a connection, or url doesn't exist, or is otherwise inaccessible. + # try to get the last downloaded one + if not connected: + if os.path.exists(cache_path) and not force_download: + return cache_path + if local_files_only: + raise FileNotFoundError( + f"Cannot find the requested files in the cached path at {cache_path} and outgoing traffic has been" + " disabled. To enable file online look-ups, set 'local_files_only' to False." + ) + elif response is not None and response.status_code == 404: + raise FileNotFoundError(f"Couldn't find file at {url}") + _raise_if_offline_mode_is_enabled(f"Tried to reach {url}") + if head_error is not None: + raise ConnectionError(f"Couldn't reach {url} ({repr(head_error)})") + elif response is not None: + raise ConnectionError(f"Couldn't reach {url} (error {response.status_code})") + else: + raise ConnectionError(f"Couldn't reach {url}") + + # Try a second time + filename = hash_url_to_filename(cached_url, etag) + cache_path = os.path.join(cache_dir, filename) + + if os.path.exists(cache_path) and not force_download: + return cache_path + + # From now on, connected is True. + # Prevent parallel downloads of the same file with a lock. + lock_path = cache_path + ".lock" + with FileLock(lock_path): + # Retry in case previously locked processes just enter after the precedent process releases the lock + if os.path.exists(cache_path) and not force_download: + return cache_path + + incomplete_path = cache_path + ".incomplete" + + @contextmanager + def temp_file_manager(mode="w+b"): + with open(incomplete_path, mode) as f: + yield f + + resume_size = 0 + if resume_download: + temp_file_manager = partial(temp_file_manager, mode="a+b") + if os.path.exists(incomplete_path): + resume_size = os.stat(incomplete_path).st_size + + # Download to temporary file, then copy to cache path once finished. + # Otherwise, you get corrupt cache entries if the download gets interrupted. + with temp_file_manager() as temp_file: + logger.info(f"{url} not found in cache or force_download set to True, downloading to {temp_file.name}") + + # GET file object + if scheme == "ftp": + ftp_get(url, temp_file) + elif scheme not in ("http", "https"): + fsspec_get( + url, temp_file, storage_options=storage_options, desc=download_desc, disable_tqdm=disable_tqdm + ) + else: + http_get( + url, + temp_file=temp_file, + proxies=proxies, + resume_size=resume_size, + headers=headers, + cookies=cookies, + max_retries=max_retries, + desc=download_desc, + disable_tqdm=disable_tqdm, + ) + + logger.info(f"storing {url} in cache at {cache_path}") + shutil.move(temp_file.name, cache_path) + umask = os.umask(0o666) + os.umask(umask) + os.chmod(cache_path, 0o666 & ~umask) + + logger.info(f"creating metadata file for {cache_path}") + meta = {"url": url, "etag": etag} + meta_path = cache_path + ".json" + with open(meta_path, "w", encoding="utf-8") as meta_file: + json.dump(meta, meta_file) + + return cache_path + + +def add_start_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = "".join(docstr) + "\n\n" + (fn.__doc__ if fn.__doc__ is not None else "") + return fn + + return docstring_decorator + + +def add_end_docstrings(*docstr): + def docstring_decorator(fn): + fn.__doc__ = (fn.__doc__ if fn.__doc__ is not None else "") + "\n\n" + "".join(docstr) + return fn + + return docstring_decorator + + +def estimate_dataset_size(paths): + return sum(path.stat().st_size for path in paths) + + +def readline(f: io.RawIOBase): + # From: https://github.com/python/cpython/blob/d27e2f4d118e7a9909b6a3e5da06c5ff95806a85/Lib/_pyio.py#L525 + res = bytearray() + while True: + b = f.read(1) + if not b: + break + res += b + if res.endswith(b"\n"): + break + return bytes(res) + + +####################### +# Streaming utilities # +####################### + +BASE_KNOWN_EXTENSIONS = [ + "txt", + "csv", + "json", + "jsonl", + "tsv", + "conll", + "conllu", + "orig", + "parquet", + "pkl", + "pickle", + "rel", + "xml", +] +COMPRESSION_EXTENSION_TO_PROTOCOL = { + # single file compression + **{fs_class.extension.lstrip("."): fs_class.protocol for fs_class in COMPRESSION_FILESYSTEMS}, + # archive compression + "zip": "zip", +} +SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL = { + fs_class.extension.lstrip("."): fs_class.protocol for fs_class in COMPRESSION_FILESYSTEMS +} +SINGLE_FILE_COMPRESSION_PROTOCOLS = {fs_class.protocol for fs_class in COMPRESSION_FILESYSTEMS} +SINGLE_SLASH_AFTER_PROTOCOL_PATTERN = re.compile(r"(? str: + # Get extension: https://foo.bar/train.json.gz -> gz + extension = path.split(".")[-1] + # Remove query params ("dl=1", "raw=true"): gz?dl=1 -> gz + # Remove shards infos (".txt_1", ".txt-00000-of-00100"): txt_1 -> txt + for symb in "?-_": + extension = extension.split(symb)[0] + return extension + + +def _get_extraction_protocol_with_magic_number(f) -> Optional[str]: + """read the magic number from a file-like object and return the compression protocol""" + # Check if the file object is seekable even before reading the magic number (to avoid https://bugs.python.org/issue26440) + try: + f.seek(0) + except (AttributeError, io.UnsupportedOperation): + return None + magic_number = f.read(MAGIC_NUMBER_MAX_LENGTH) + f.seek(0) + for i in range(MAGIC_NUMBER_MAX_LENGTH): + compression = MAGIC_NUMBER_TO_COMPRESSION_PROTOCOL.get(magic_number[: MAGIC_NUMBER_MAX_LENGTH - i]) + if compression is not None: + return compression + compression = MAGIC_NUMBER_TO_UNSUPPORTED_COMPRESSION_PROTOCOL.get(magic_number[: MAGIC_NUMBER_MAX_LENGTH - i]) + if compression is not None: + raise NotImplementedError(f"Compression protocol '{compression}' not implemented.") + + +def _get_extraction_protocol(urlpath: str, download_config: Optional[DownloadConfig] = None) -> Optional[str]: + # get inner file: zip://train-00000.json.gz::https://foo.bar/data.zip -> zip://train-00000.json.gz + urlpath = str(urlpath) + path = urlpath.split("::")[0] + extension = _get_path_extension(path) + if ( + extension in BASE_KNOWN_EXTENSIONS + or extension in ["tgz", "tar"] + or path.endswith((".tar.gz", ".tar.bz2", ".tar.xz")) + ): + return None + elif extension in COMPRESSION_EXTENSION_TO_PROTOCOL: + return COMPRESSION_EXTENSION_TO_PROTOCOL[extension] + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + try: + with fsspec.open(urlpath, **(storage_options or {})) as f: + return _get_extraction_protocol_with_magic_number(f) + except FileNotFoundError: + if urlpath.startswith(config.HF_ENDPOINT): + raise FileNotFoundError( + urlpath + "\nIf the repo is private or gated, make sure to log in with `huggingface-cli login`." + ) from None + else: + raise + + +def xjoin(a, *p): + """ + This function extends os.path.join to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xjoin function allows you to apply the join on the first path of the chain. + + Example:: + + >>> xjoin("zip://folder1::https://host.com/archive.zip", "file.txt") + zip://folder1/file.txt::https://host.com/archive.zip + """ + a, *b = str(a).split("::") + if is_local_path(a): + return os.path.join(a, *p) + else: + a = posixpath.join(a, *p) + return "::".join([a] + b) + + +def xdirname(a): + """ + This function extends os.path.dirname to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xdirname function allows you to apply the dirname on the first path of the chain. + + Example:: + + >>> xdirname("zip://folder1/file.txt::https://host.com/archive.zip") + zip://folder1::https://host.com/archive.zip + """ + a, *b = str(a).split("::") + if is_local_path(a): + a = os.path.dirname(Path(a).as_posix()) + else: + a = posixpath.dirname(a) + # if we end up at the root of the protocol, we get for example a = 'http:' + # so we have to fix it by adding the '//' that was removed: + if a.endswith(":"): + a += "//" + return "::".join([a] + b) + + +def xexists(urlpath: str, download_config: Optional[DownloadConfig] = None): + """Extend `os.path.exists` function to support both local and remote files. + + Args: + urlpath (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `bool` + """ + + main_hop, *rest_hops = _as_str(urlpath).split("::") + if is_local_path(main_hop): + return os.path.exists(main_hop) + else: + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + main_hop, *rest_hops = urlpath.split("::") + fs, *_ = url_to_fs(urlpath, **storage_options) + return fs.exists(main_hop) + + +def xbasename(a): + """ + This function extends os.path.basename to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xbasename function allows you to apply the basename on the first path of the chain. + + Example:: + + >>> xbasename("zip://folder1/file.txt::https://host.com/archive.zip") + file.txt + """ + a, *b = str(a).split("::") + if is_local_path(a): + return os.path.basename(Path(a).as_posix()) + else: + return posixpath.basename(a) + + +def xsplit(a): + """ + This function extends os.path.split to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xsplit function allows you to apply the xsplit on the first path of the chain. + + Example:: + + >>> xsplit("zip://folder1/file.txt::https://host.com/archive.zip") + ('zip://folder1::https://host.com/archive.zip', 'file.txt') + """ + a, *b = str(a).split("::") + if is_local_path(a): + return os.path.split(Path(a).as_posix()) + else: + a, tail = posixpath.split(a) + return "::".join([a + "//" if a.endswith(":") else a] + b), tail + + +def xsplitext(a): + """ + This function extends os.path.splitext to support the "::" hop separator. It supports both paths and urls. + + A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". + This is used to access files inside a zip file over http for example. + + Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. + Then you can just chain the url this way: + + zip://folder1/file.txt::https://host.com/archive.zip + + The xsplitext function allows you to apply the splitext on the first path of the chain. + + Example:: + + >>> xsplitext("zip://folder1/file.txt::https://host.com/archive.zip") + ('zip://folder1/file::https://host.com/archive.zip', '.txt') + """ + a, *b = str(a).split("::") + if is_local_path(a): + return os.path.splitext(Path(a).as_posix()) + else: + a, ext = posixpath.splitext(a) + return "::".join([a] + b), ext + + +def xisfile(path, download_config: Optional[DownloadConfig] = None) -> bool: + """Extend `os.path.isfile` function to support remote files. + + Args: + path (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `bool` + """ + main_hop, *rest_hops = str(path).split("::") + if is_local_path(main_hop): + return os.path.isfile(path) + else: + path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config) + main_hop, *rest_hops = path.split("::") + fs, *_ = url_to_fs(path, **storage_options) + return fs.isfile(main_hop) + + +def xgetsize(path, download_config: Optional[DownloadConfig] = None) -> int: + """Extend `os.path.getsize` function to support remote files. + + Args: + path (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `int`: optional + """ + main_hop, *rest_hops = str(path).split("::") + if is_local_path(main_hop): + return os.path.getsize(path) + else: + path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config) + main_hop, *rest_hops = path.split("::") + fs, *_ = fs, *_ = url_to_fs(path, **storage_options) + try: + size = fs.size(main_hop) + except EntryNotFoundError: + raise FileNotFoundError(f"No such file: {path}") + if size is None: + # use xopen instead of fs.open to make data fetching more robust + with xopen(path, download_config=download_config) as f: + size = len(f.read()) + return size + + +def xisdir(path, download_config: Optional[DownloadConfig] = None) -> bool: + """Extend `os.path.isdir` function to support remote files. + + Args: + path (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `bool` + """ + main_hop, *rest_hops = str(path).split("::") + if is_local_path(main_hop): + return os.path.isdir(path) + else: + path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config) + main_hop, *rest_hops = path.split("::") + fs, *_ = fs, *_ = url_to_fs(path, **storage_options) + inner_path = main_hop.split("://")[-1] + if not inner_path.strip("/"): + return True + return fs.isdir(inner_path) + + +def xrelpath(path, start=None): + """Extend `os.path.relpath` function to support remote files. + + Args: + path (`str`): URL path. + start (`str`): Start URL directory path. + + Returns: + `str` + """ + main_hop, *rest_hops = str(path).split("::") + if is_local_path(main_hop): + return os.path.relpath(main_hop, start=start) if start else os.path.relpath(main_hop) + else: + return posixpath.relpath(main_hop, start=str(start).split("::")[0]) if start else os.path.relpath(main_hop) + + +def _add_retries_to_file_obj_read_method(file_obj): + read = file_obj.read + max_retries = config.STREAMING_READ_MAX_RETRIES + + def read_with_retries(*args, **kwargs): + disconnect_err = None + for retry in range(1, max_retries + 1): + try: + out = read(*args, **kwargs) + break + except (ClientError, TimeoutError) as err: + disconnect_err = err + logger.warning( + f"Got disconnected from remote data host. Retrying in {config.STREAMING_READ_RETRY_INTERVAL}sec [{retry}/{max_retries}]" + ) + time.sleep(config.STREAMING_READ_RETRY_INTERVAL) + else: + raise ConnectionError("Server Disconnected") from disconnect_err + return out + + try: + file_obj.read = read_with_retries + except AttributeError: # read-only attribute + orig_file_obj = file_obj + file_obj = io.RawIOBase() + file_obj.read = read_with_retries + file_obj.__getattr__ = lambda _, attr: getattr(orig_file_obj, attr) + return file_obj + + +def _prepare_path_and_storage_options( + urlpath: str, download_config: Optional[DownloadConfig] = None +) -> Tuple[str, Dict[str, Dict[str, Any]]]: + prepared_urlpath = [] + prepared_storage_options = {} + for hop in urlpath.split("::"): + hop, storage_options = _prepare_single_hop_path_and_storage_options(hop, download_config=download_config) + prepared_urlpath.append(hop) + prepared_storage_options.update(storage_options) + return "::".join(prepared_urlpath), storage_options + + +def _prepare_single_hop_path_and_storage_options( + urlpath: str, download_config: Optional[DownloadConfig] = None +) -> Tuple[str, Dict[str, Dict[str, Any]]]: + """ + Prepare the URL and the kwargs that must be passed to the HttpFileSystem or to requests.get/head + + In particular it resolves google drive URLs + It also adds the authentication headers for the Hugging Face Hub, for both https:// and hf:// paths. + + Storage options are formatted in the form {protocol: storage_options_for_protocol} + """ + token = None if download_config is None else download_config.token + if urlpath.startswith(config.HF_ENDPOINT) and "/resolve/" in urlpath: + urlpath = "hf://" + urlpath[len(config.HF_ENDPOINT) + 1 :].replace("/resolve/", "@", 1) + protocol = urlpath.split("://")[0] if "://" in urlpath else "file" + if download_config is not None and protocol in download_config.storage_options: + storage_options = download_config.storage_options[protocol] + elif download_config is not None and protocol not in download_config.storage_options: + storage_options = { + option_name: option_value + for option_name, option_value in download_config.storage_options.items() + if option_name not in fsspec.available_protocols() + } + else: + storage_options = {} + if storage_options: + storage_options = {protocol: storage_options} + if protocol in ["http", "https"]: + storage_options[protocol] = { + "headers": { + **get_authentication_headers_for_url(urlpath, token=token), + "user-agent": get_datasets_user_agent(), + }, + "client_kwargs": {"trust_env": True}, # Enable reading proxy env variables. + **(storage_options.get(protocol, {})), + } + if "drive.google.com" in urlpath: + response = http_head(urlpath) + cookies = None + for k, v in response.cookies.items(): + if k.startswith("download_warning"): + urlpath += "&confirm=" + v + cookies = response.cookies + storage_options[protocol] = {"cookies": cookies, **storage_options.get(protocol, {})} + # Fix Google Drive URL to avoid Virus scan warning + if "drive.google.com" in urlpath and "confirm=" not in urlpath: + urlpath += "&confirm=t" + if urlpath.startswith("https://raw.githubusercontent.com/"): + # Workaround for served data with gzip content-encoding: https://github.com/fsspec/filesystem_spec/issues/389 + storage_options[protocol]["headers"]["Accept-Encoding"] = "identity" + elif protocol == "hf": + storage_options[protocol] = { + "token": token, + "endpoint": config.HF_ENDPOINT, + **storage_options.get(protocol, {}), + } + # streaming with block_size=0 is only implemented in 0.21 (see https://github.com/huggingface/huggingface_hub/pull/1967) + if config.HF_HUB_VERSION < version.parse("0.21.0"): + storage_options[protocol]["block_size"] = "default" + return urlpath, storage_options + + +def xopen(file: str, mode="r", *args, download_config: Optional[DownloadConfig] = None, **kwargs): + """Extend `open` function to support remote files using `fsspec`. + + It also has a retry mechanism in case connection fails. + The `args` and `kwargs` are passed to `fsspec.open`, except `token` which is used for queries to private repos on huggingface.co + + Args: + file (`str`): Path name of the file to be opened. + mode (`str`, *optional*, default "r"): Mode in which the file is opened. + *args: Arguments to be passed to `fsspec.open`. + download_config : mainly use token or storage_options to support different platforms and auth types. + **kwargs: Keyword arguments to be passed to `fsspec.open`. + + Returns: + file object + """ + # This works as well for `xopen(str(Path(...)))` + file_str = _as_str(file) + main_hop, *rest_hops = file_str.split("::") + if is_local_path(main_hop): + # ignore fsspec-specific kwargs + kwargs.pop("block_size", None) + return open(main_hop, mode, *args, **kwargs) + # add headers and cookies for authentication on the HF Hub and for Google Drive + file, storage_options = _prepare_path_and_storage_options(file_str, download_config=download_config) + kwargs = {**kwargs, **(storage_options or {})} + try: + file_obj = fsspec.open(file, mode=mode, *args, **kwargs).open() + except ValueError as e: + if str(e) == "Cannot seek streaming HTTP file": + raise NonStreamableDatasetError( + "Streaming is not possible for this dataset because data host server doesn't support HTTP range " + "requests. You can still load this dataset in non-streaming mode by passing `streaming=False` (default)" + ) from e + else: + raise + except FileNotFoundError: + if file.startswith(config.HF_ENDPOINT): + raise FileNotFoundError( + file + "\nIf the repo is private or gated, make sure to log in with `huggingface-cli login`." + ) from None + else: + raise + file_obj = _add_retries_to_file_obj_read_method(file_obj) + return file_obj + + +def xlistdir(path: str, download_config: Optional[DownloadConfig] = None) -> List[str]: + """Extend `os.listdir` function to support remote files. + + Args: + path (`str`): URL path. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `list` of `str` + """ + main_hop, *rest_hops = _as_str(path).split("::") + if is_local_path(main_hop): + return os.listdir(path) + else: + # globbing inside a zip in a private repo requires authentication + path, storage_options = _prepare_path_and_storage_options(path, download_config=download_config) + main_hop, *rest_hops = path.split("::") + fs, *_ = url_to_fs(path, **storage_options) + inner_path = main_hop.split("://")[-1] + if inner_path.strip("/") and not fs.isdir(inner_path): + raise FileNotFoundError(f"Directory doesn't exist: {path}") + paths = fs.listdir(inner_path, detail=False) + return [os.path.basename(path.rstrip("/")) for path in paths] + + +def xglob(urlpath, *, recursive=False, download_config: Optional[DownloadConfig] = None): + """Extend `glob.glob` function to support remote files. + + Args: + urlpath (`str`): URL path with shell-style wildcard patterns. + recursive (`bool`, default `False`): Whether to match the "**" pattern recursively to zero or more + directories or subdirectories. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `list` of `str` + """ + main_hop, *rest_hops = _as_str(urlpath).split("::") + if is_local_path(main_hop): + return glob.glob(main_hop, recursive=recursive) + else: + # globbing inside a zip in a private repo requires authentication + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + main_hop, *rest_hops = urlpath.split("::") + fs, *_ = url_to_fs(urlpath, **storage_options) + inner_path = main_hop.split("://")[1] + globbed_paths = fs.glob(inner_path) + protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[-1] + return ["::".join([f"{protocol}://{globbed_path}"] + rest_hops) for globbed_path in globbed_paths] + + +def xwalk(urlpath, download_config: Optional[DownloadConfig] = None, **kwargs): + """Extend `os.walk` function to support remote files. + + Args: + urlpath (`str`): URL root path. + download_config : mainly use token or storage_options to support different platforms and auth types. + **kwargs: Additional keyword arguments forwarded to the underlying filesystem. + + + Yields: + `tuple`: 3-tuple (dirpath, dirnames, filenames). + """ + main_hop, *rest_hops = _as_str(urlpath).split("::") + if is_local_path(main_hop): + yield from os.walk(main_hop, **kwargs) + else: + # walking inside a zip in a private repo requires authentication + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + main_hop, *rest_hops = urlpath.split("::") + fs, *_ = url_to_fs(urlpath, **storage_options) + inner_path = main_hop.split("://")[-1] + if inner_path.strip("/") and not fs.isdir(inner_path): + return [] + protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[-1] + for dirpath, dirnames, filenames in fs.walk(inner_path, **kwargs): + yield "::".join([f"{protocol}://{dirpath}"] + rest_hops), dirnames, filenames + + +class xPath(type(Path())): + """Extension of `pathlib.Path` to support both local paths and remote URLs.""" + + def __str__(self): + path_str = super().__str__() + main_hop, *rest_hops = path_str.split("::") + if is_local_path(main_hop): + return main_hop + path_as_posix = path_str.replace("\\", "/") + path_as_posix = SINGLE_SLASH_AFTER_PROTOCOL_PATTERN.sub("://", path_as_posix) + path_as_posix += "//" if path_as_posix.endswith(":") else "" # Add slashes to root of the protocol + return path_as_posix + + def exists(self, download_config: Optional[DownloadConfig] = None): + """Extend `pathlib.Path.exists` method to support both local and remote files. + + Args: + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `bool` + """ + return xexists(str(self), download_config=download_config) + + def glob(self, pattern, download_config: Optional[DownloadConfig] = None): + """Glob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Args: + pattern (`str`): Pattern that resulting paths must match. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Yields: + [`xPath`] + """ + posix_path = self.as_posix() + main_hop, *rest_hops = posix_path.split("::") + if is_local_path(main_hop): + yield from Path(main_hop).glob(pattern) + else: + # globbing inside a zip in a private repo requires authentication + if rest_hops: + urlpath = rest_hops[0] + urlpath, storage_options = _prepare_path_and_storage_options(urlpath, download_config=download_config) + storage_options = {urlpath.split("://")[0]: storage_options} + posix_path = "::".join([main_hop, urlpath, *rest_hops[1:]]) + else: + storage_options = None + fs, *_ = url_to_fs(xjoin(posix_path, pattern), **(storage_options or {})) + globbed_paths = fs.glob(xjoin(main_hop, pattern)) + for globbed_path in globbed_paths: + yield type(self)("::".join([f"{fs.protocol}://{globbed_path}"] + rest_hops)) + + def rglob(self, pattern, **kwargs): + """Rglob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Args: + pattern (`str`): Pattern that resulting paths must match. + + Yields: + [`xPath`] + """ + return self.glob("**/" + pattern, **kwargs) + + @property + def parent(self) -> "xPath": + """Name function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Returns: + [`xPath`] + """ + return type(self)(xdirname(self.as_posix())) + + @property + def name(self) -> str: + """Name function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Returns: + `str` + """ + return PurePosixPath(self.as_posix().split("::")[0]).name + + @property + def stem(self) -> str: + """Stem function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Returns: + `str` + """ + return PurePosixPath(self.as_posix().split("::")[0]).stem + + @property + def suffix(self) -> str: + """Suffix function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. + + Returns: + `str` + """ + return PurePosixPath(self.as_posix().split("::")[0]).suffix + + def open(self, *args, **kwargs): + """Extend :func:`xopen` to support argument of type :obj:`~pathlib.Path`. + + Args: + **args: Arguments passed to :func:`fsspec.open`. + **kwargs: Keyword arguments passed to :func:`fsspec.open`. + + Returns: + `io.FileIO`: File-like object. + """ + return xopen(str(self), *args, **kwargs) + + def joinpath(self, *p: Tuple[str, ...]) -> "xPath": + """Extend :func:`xjoin` to support argument of type :obj:`~pathlib.Path`. + + Args: + *p (`tuple` of `str`): Other path components. + + Returns: + [`xPath`] + """ + return type(self)(xjoin(self.as_posix(), *p)) + + def __truediv__(self, p: str) -> "xPath": + return self.joinpath(p) + + def with_suffix(self, suffix): + main_hop, *rest_hops = str(self).split("::") + if is_local_path(main_hop): + return type(self)(str(super().with_suffix(suffix))) + return type(self)("::".join([type(self)(PurePosixPath(main_hop).with_suffix(suffix)).as_posix()] + rest_hops)) + + +def _as_str(path: Union[str, Path, xPath]): + return str(path) if isinstance(path, xPath) else str(xPath(str(path))) + + +def xgzip_open(filepath_or_buffer, *args, download_config: Optional[DownloadConfig] = None, **kwargs): + import gzip + + if hasattr(filepath_or_buffer, "read"): + return gzip.open(filepath_or_buffer, *args, **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + return gzip.open(xopen(filepath_or_buffer, "rb", download_config=download_config), *args, **kwargs) + + +def xnumpy_load(filepath_or_buffer, *args, download_config: Optional[DownloadConfig] = None, **kwargs): + import numpy as np + + if hasattr(filepath_or_buffer, "read"): + return np.load(filepath_or_buffer, *args, **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + return np.load(xopen(filepath_or_buffer, "rb", download_config=download_config), *args, **kwargs) + + +def xpandas_read_csv(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs): + import pandas as pd + + if hasattr(filepath_or_buffer, "read"): + return pd.read_csv(filepath_or_buffer, **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + if kwargs.get("compression", "infer") == "infer": + kwargs["compression"] = _get_extraction_protocol(filepath_or_buffer, download_config=download_config) + return pd.read_csv(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs) + + +def xpandas_read_excel(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs): + import pandas as pd + + if hasattr(filepath_or_buffer, "read"): + try: + return pd.read_excel(filepath_or_buffer, **kwargs) + except ValueError: # Cannot seek streaming HTTP file + return pd.read_excel(BytesIO(filepath_or_buffer.read()), **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + try: + return pd.read_excel(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs) + except ValueError: # Cannot seek streaming HTTP file + return pd.read_excel( + BytesIO(xopen(filepath_or_buffer, "rb", download_config=download_config).read()), **kwargs + ) + + +def xpyarrow_parquet_read_table(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs): + import pyarrow.parquet as pq + + if hasattr(filepath_or_buffer, "read"): + return pq.read_table(filepath_or_buffer, **kwargs) + else: + filepath_or_buffer = str(filepath_or_buffer) + return pq.read_table(xopen(filepath_or_buffer, mode="rb", download_config=download_config), **kwargs) + + +def xsio_loadmat(filepath_or_buffer, download_config: Optional[DownloadConfig] = None, **kwargs): + import scipy.io as sio + + if hasattr(filepath_or_buffer, "read"): + return sio.loadmat(filepath_or_buffer, **kwargs) + else: + return sio.loadmat(xopen(filepath_or_buffer, "rb", download_config=download_config), **kwargs) + + +def xet_parse(source, parser=None, download_config: Optional[DownloadConfig] = None): + """Extend `xml.etree.ElementTree.parse` function to support remote files. + + Args: + source: File path or file object. + parser (`XMLParser`, *optional*, default `XMLParser`): Parser instance. + download_config : mainly use token or storage_options to support different platforms and auth types. + + Returns: + `xml.etree.ElementTree.Element`: Root element of the given source document. + """ + if hasattr(source, "read"): + return ET.parse(source, parser=parser) + else: + with xopen(source, "rb", download_config=download_config) as f: + return ET.parse(f, parser=parser) + + +def xxml_dom_minidom_parse(filename_or_file, download_config: Optional[DownloadConfig] = None, **kwargs): + """Extend `xml.dom.minidom.parse` function to support remote files. + + Args: + filename_or_file (`str` or file): File path or file object. + download_config : mainly use token or storage_options to support different platforms and auth types. + **kwargs (optional): Additional keyword arguments passed to `xml.dom.minidom.parse`. + + Returns: + :obj:`xml.dom.minidom.Document`: Parsed document. + """ + if hasattr(filename_or_file, "read"): + return xml.dom.minidom.parse(filename_or_file, **kwargs) + else: + with xopen(filename_or_file, "rb", download_config=download_config) as f: + return xml.dom.minidom.parse(f, **kwargs) + + +class _IterableFromGenerator(TrackedIterable): + """Utility class to create an iterable from a generator function, in order to reset the generator when needed.""" + + def __init__(self, generator: Callable, *args, **kwargs): + super().__init__() + self.generator = generator + self.args = args + self.kwargs = kwargs + + def __iter__(self): + for x in self.generator(*self.args, **self.kwargs): + self.last_item = x + yield x + self.last_item = None + + +class ArchiveIterable(_IterableFromGenerator): + """An iterable of (path, fileobj) from a TAR archive, used by `iter_archive`""" + + @staticmethod + def _iter_tar(f): + stream = tarfile.open(fileobj=f, mode="r|*") + for tarinfo in stream: + file_path = tarinfo.name + if not tarinfo.isreg(): + continue + if file_path is None: + continue + if os.path.basename(file_path).startswith((".", "__")): + # skipping hidden files + continue + file_obj = stream.extractfile(tarinfo) + yield file_path, file_obj + stream.members = [] + del stream + + @staticmethod + def _iter_zip(f): + zipf = zipfile.ZipFile(f) + for member in zipf.infolist(): + file_path = member.filename + if member.is_dir(): + continue + if file_path is None: + continue + if os.path.basename(file_path).startswith((".", "__")): + # skipping hidden files + continue + file_obj = zipf.open(member) + yield file_path, file_obj + + @classmethod + def _iter_from_fileobj(cls, f) -> Generator[Tuple, None, None]: + compression = _get_extraction_protocol_with_magic_number(f) + if compression == "zip": + yield from cls._iter_zip(f) + else: + yield from cls._iter_tar(f) + + @classmethod + def _iter_from_urlpath( + cls, urlpath: str, download_config: Optional[DownloadConfig] = None + ) -> Generator[Tuple, None, None]: + compression = _get_extraction_protocol(urlpath, download_config=download_config) + # Set block_size=0 to get faster streaming + # (e.g. for hf:// and https:// it uses streaming Requests file-like instances) + with xopen(urlpath, "rb", download_config=download_config, block_size=0) as f: + if compression == "zip": + yield from cls._iter_zip(f) + else: + yield from cls._iter_tar(f) + + @classmethod + def from_buf(cls, fileobj) -> "ArchiveIterable": + return cls(cls._iter_from_fileobj, fileobj) + + @classmethod + def from_urlpath(cls, urlpath_or_buf, download_config: Optional[DownloadConfig] = None) -> "ArchiveIterable": + return cls(cls._iter_from_urlpath, urlpath_or_buf, download_config) + + +class FilesIterable(_IterableFromGenerator): + """An iterable of paths from a list of directories or files""" + + @classmethod + def _iter_from_urlpaths( + cls, urlpaths: Union[str, List[str]], download_config: Optional[DownloadConfig] = None + ) -> Generator[str, None, None]: + if not isinstance(urlpaths, list): + urlpaths = [urlpaths] + for urlpath in urlpaths: + if xisfile(urlpath, download_config=download_config): + yield urlpath + elif xisdir(urlpath, download_config=download_config): + for dirpath, dirnames, filenames in xwalk(urlpath, download_config=download_config): + # in-place modification to prune the search + dirnames[:] = sorted([dirname for dirname in dirnames if not dirname.startswith((".", "__"))]) + if xbasename(dirpath).startswith((".", "__")): + # skipping hidden directories + continue + for filename in sorted(filenames): + if filename.startswith((".", "__")): + # skipping hidden files + continue + yield xjoin(dirpath, filename) + else: + raise FileNotFoundError(urlpath) + + @classmethod + def from_urlpaths(cls, urlpaths, download_config: Optional[DownloadConfig] = None) -> "FilesIterable": + return cls(cls._iter_from_urlpaths, urlpaths, download_config) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/filelock.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/filelock.py new file mode 100644 index 0000000000000000000000000000000000000000..df0728efe644d8eb32f0e578a85e39ba366e9743 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/filelock.py @@ -0,0 +1,11 @@ +# deprecated, please use the `filelock` package instead + +from filelock import ( # noqa: F401 # imported for backward compatibility TODO: remove in 3.0.0 + BaseFileLock, + SoftFileLock, + Timeout, + UnixFileLock, + WindowsFileLock, +) + +from ._filelock import FileLock # noqa: F401 # imported for backward compatibility. TODO: remove in 3.0.0 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/hub.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/hub.py new file mode 100644 index 0000000000000000000000000000000000000000..6d784333b23328c113ec7be444d0a77410f1b857 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/hub.py @@ -0,0 +1,6 @@ +from functools import partial + +from huggingface_hub import hf_hub_url + + +hf_dataset_url = partial(hf_hub_url, repo_type="dataset") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/info_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/info_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d93f5b4509f47c5c12ad98c11fb34acfdab9b714 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/info_utils.py @@ -0,0 +1,106 @@ +import enum +import os +from typing import Optional + +from huggingface_hub.utils import insecure_hashlib + +from .. import config +from ..exceptions import ( + ExpectedMoreDownloadedFilesError, + ExpectedMoreSplitsError, + NonMatchingChecksumError, + NonMatchingSplitsSizesError, + UnexpectedDownloadedFileError, + UnexpectedSplitsError, +) +from .logging import get_logger + + +logger = get_logger(__name__) + + +class VerificationMode(enum.Enum): + """`Enum` that specifies which verification checks to run. + + The default mode is `BASIC_CHECKS`, which will perform only rudimentary checks to avoid slowdowns + when generating/downloading a dataset for the first time. + + The verification modes: + + | | Verification checks | + |---------------------------|------------------------------------------------------------------------------ | + | `ALL_CHECKS` | Split checks, uniqueness of the keys yielded in case of the GeneratorBuilder | + | | and the validity (number of files, checksums, etc.) of downloaded files | + | `BASIC_CHECKS` (default) | Same as `ALL_CHECKS` but without checking downloaded files | + | `NO_CHECKS` | None | + + """ + + ALL_CHECKS = "all_checks" + BASIC_CHECKS = "basic_checks" + NO_CHECKS = "no_checks" + + +def verify_checksums(expected_checksums: Optional[dict], recorded_checksums: dict, verification_name=None): + if expected_checksums is None: + logger.info("Unable to verify checksums.") + return + if len(set(expected_checksums) - set(recorded_checksums)) > 0: + raise ExpectedMoreDownloadedFilesError(str(set(expected_checksums) - set(recorded_checksums))) + if len(set(recorded_checksums) - set(expected_checksums)) > 0: + raise UnexpectedDownloadedFileError(str(set(recorded_checksums) - set(expected_checksums))) + bad_urls = [url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]] + for_verification_name = " for " + verification_name if verification_name is not None else "" + if len(bad_urls) > 0: + raise NonMatchingChecksumError( + f"Checksums didn't match{for_verification_name}:\n" + f"{bad_urls}\n" + "Set `verification_mode='no_checks'` to skip checksums verification and ignore this error" + ) + logger.info("All the checksums matched successfully" + for_verification_name) + + +def verify_splits(expected_splits: Optional[dict], recorded_splits: dict): + if expected_splits is None: + logger.info("Unable to verify splits sizes.") + return + if len(set(expected_splits) - set(recorded_splits)) > 0: + raise ExpectedMoreSplitsError(str(set(expected_splits) - set(recorded_splits))) + if len(set(recorded_splits) - set(expected_splits)) > 0: + raise UnexpectedSplitsError(str(set(recorded_splits) - set(expected_splits))) + bad_splits = [ + {"expected": expected_splits[name], "recorded": recorded_splits[name]} + for name in expected_splits + if expected_splits[name].num_examples != recorded_splits[name].num_examples + ] + if len(bad_splits) > 0: + raise NonMatchingSplitsSizesError(str(bad_splits)) + logger.info("All the splits matched successfully.") + + +def get_size_checksum_dict(path: str, record_checksum: bool = True) -> dict: + """Compute the file size and the sha256 checksum of a file""" + if record_checksum: + m = insecure_hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + m.update(chunk) + checksum = m.hexdigest() + else: + checksum = None + return {"num_bytes": os.path.getsize(path), "checksum": checksum} + + +def is_small_dataset(dataset_size): + """Check if `dataset_size` is smaller than `config.IN_MEMORY_MAX_SIZE`. + + Args: + dataset_size (int): Dataset size in bytes. + + Returns: + bool: Whether `dataset_size` is smaller than `config.IN_MEMORY_MAX_SIZE`. + """ + if dataset_size and config.IN_MEMORY_MAX_SIZE: + return dataset_size < config.IN_MEMORY_MAX_SIZE + else: + return False diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/logging.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..dffd5ce46e0d2da5cbbfb023003c3f4caae86093 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/logging.py @@ -0,0 +1,179 @@ +# Copyright 2020 Optuna, Hugging Face +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Logging utilities.""" + +import logging +import os +from logging import ( + CRITICAL, # NOQA + DEBUG, # NOQA + ERROR, # NOQA + FATAL, # NOQA + INFO, # NOQA + NOTSET, # NOQA + WARN, # NOQA + WARNING, # NOQA +) +from typing import Optional + +from .tqdm import ( # noqa: F401 # imported for backward compatibility + disable_progress_bar, + enable_progress_bar, + is_progress_bar_enabled, + tqdm, +) + + +log_levels = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + +_default_log_level = logging.WARNING + + +def _get_default_logging_level(): + """ + If DATASETS_VERBOSITY env var is set to one of the valid choices return that as the new default level. + If it is not - fall back to ``_default_log_level`` + """ + env_level_str = os.getenv("DATASETS_VERBOSITY", None) + if env_level_str: + if env_level_str in log_levels: + return log_levels[env_level_str] + else: + logging.getLogger().warning( + f"Unknown option DATASETS_VERBOSITY={env_level_str}, " + f"has to be one of: { ', '.join(log_levels.keys()) }" + ) + return _default_log_level + + +def _get_library_name() -> str: + return __name__.split(".")[0] + + +def _get_library_root_logger() -> logging.Logger: + return logging.getLogger(_get_library_name()) + + +def _configure_library_root_logger() -> None: + # Apply our default configuration to the library root logger. + library_root_logger = _get_library_root_logger() + library_root_logger.addHandler(logging.StreamHandler()) + library_root_logger.setLevel(_get_default_logging_level()) + + +def _reset_library_root_logger() -> None: + library_root_logger = _get_library_root_logger() + library_root_logger.setLevel(logging.NOTSET) + + +def get_logger(name: Optional[str] = None) -> logging.Logger: + """Return a logger with the specified name. + This function can be used in dataset scripts. + """ + if name is None: + name = _get_library_name() + return logging.getLogger(name) + + +def get_verbosity() -> int: + """Return the current level for the HuggingFace datasets library's root logger. + Returns: + Logging level, e.g., `datasets.logging.DEBUG` and `datasets.logging.INFO`. + + + + HuggingFace datasets library has following logging levels: + - `datasets.logging.CRITICAL`, `datasets.logging.FATAL` + - `datasets.logging.ERROR` + - `datasets.logging.WARNING`, `datasets.logging.WARN` + - `datasets.logging.INFO` + - `datasets.logging.DEBUG` + + + """ + return _get_library_root_logger().getEffectiveLevel() + + +def set_verbosity(verbosity: int) -> None: + """Set the level for the Hugging Face Datasets library's root logger. + Args: + verbosity: + Logging level, e.g., `datasets.logging.DEBUG` and `datasets.logging.INFO`. + """ + _get_library_root_logger().setLevel(verbosity) + + +def set_verbosity_info(): + """Set the level for the Hugging Face datasets library's root logger to `INFO`. + + This will display most of the logging information and tqdm bars. + + Shortcut to `datasets.logging.set_verbosity(datasets.logging.INFO)`. + """ + return set_verbosity(INFO) + + +def set_verbosity_warning(): + """Set the level for the Hugging Face datasets library's root logger to `WARNING`. + + This will display only the warning and errors logging information and tqdm bars. + + Shortcut to `datasets.logging.set_verbosity(datasets.logging.WARNING)`. + """ + return set_verbosity(WARNING) + + +def set_verbosity_debug(): + """Set the level for the Hugging Face datasets library's root logger to `DEBUG`. + + This will display all the logging information and tqdm bars. + + Shortcut to `datasets.logging.set_verbosity(datasets.logging.DEBUG)`. + """ + return set_verbosity(DEBUG) + + +def set_verbosity_error(): + """Set the level for the Hugging Face datasets library's root logger to `ERROR`. + + This will display only the errors logging information and tqdm bars. + + Shortcut to `datasets.logging.set_verbosity(datasets.logging.ERROR)`. + """ + return set_verbosity(ERROR) + + +def disable_propagation() -> None: + """Disable propagation of the library log outputs. + Note that log propagation is disabled by default. + """ + _get_library_root_logger().propagate = False + + +def enable_propagation() -> None: + """Enable propagation of the library log outputs. + Please disable the Hugging Face datasets library's default handler to prevent double logging if the root logger has + been configured. + """ + _get_library_root_logger().propagate = True + + +# Configure the library root logger at the module level (singleton-like) +_configure_library_root_logger() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/metadata.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..d2e763e9cb6a150db3c7a6a46b0f60bd7f799b1e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/metadata.py @@ -0,0 +1,320 @@ +import re +import textwrap +from collections import Counter +from itertools import groupby +from operator import itemgetter +from pathlib import Path +from typing import Any, ClassVar, Dict, List, Optional, Tuple, Union + +import yaml +from huggingface_hub import DatasetCardData + +from ..config import METADATA_CONFIGS_FIELD +from ..info import DatasetInfo, DatasetInfosDict +from ..naming import _split_re +from ..utils.logging import get_logger +from .deprecation_utils import deprecated + + +logger = get_logger(__name__) + + +class _NoDuplicateSafeLoader(yaml.SafeLoader): + def _check_no_duplicates_on_constructed_node(self, node): + keys = [self.constructed_objects[key_node] for key_node, _ in node.value] + keys = [tuple(key) if isinstance(key, list) else key for key in keys] + counter = Counter(keys) + duplicate_keys = [key for key in counter if counter[key] > 1] + if duplicate_keys: + raise TypeError(f"Got duplicate yaml keys: {duplicate_keys}") + + def construct_mapping(self, node, deep=False): + mapping = super().construct_mapping(node, deep=deep) + self._check_no_duplicates_on_constructed_node(node) + return mapping + + +def _split_yaml_from_readme(readme_content: str) -> Tuple[Optional[str], str]: + full_content = list(readme_content.splitlines()) + if full_content and full_content[0] == "---" and "---" in full_content[1:]: + sep_idx = full_content[1:].index("---") + 1 + yamlblock = "\n".join(full_content[1:sep_idx]) + return yamlblock, "\n".join(full_content[sep_idx + 1 :]) + + return None, "\n".join(full_content) + + +@deprecated("Use `huggingface_hub.DatasetCardData` instead.") +class DatasetMetadata(dict): + # class attributes + _FIELDS_WITH_DASHES = {"train_eval_index"} # train-eval-index in the YAML metadata + + @classmethod + def from_readme(cls, path: Union[Path, str]) -> "DatasetMetadata": + """Loads and validates the dataset metadata from its dataset card (README.md) + + Args: + path (:obj:`Path`): Path to the dataset card (its README.md file) + + Returns: + :class:`DatasetMetadata`: The dataset's metadata + + Raises: + :obj:`TypeError`: If the dataset's metadata is invalid + """ + with open(path, encoding="utf-8") as readme_file: + yaml_string, _ = _split_yaml_from_readme(readme_file.read()) + if yaml_string is not None: + return cls.from_yaml_string(yaml_string) + else: + return cls() + + def to_readme(self, path: Path): + if path.exists(): + with open(path, encoding="utf-8") as readme_file: + readme_content = readme_file.read() + else: + readme_content = None + updated_readme_content = self._to_readme(readme_content) + with open(path, "w", encoding="utf-8") as readme_file: + readme_file.write(updated_readme_content) + + def _to_readme(self, readme_content: Optional[str] = None) -> str: + if readme_content is not None: + _, content = _split_yaml_from_readme(readme_content) + full_content = "---\n" + self.to_yaml_string() + "---\n" + content + else: + full_content = "---\n" + self.to_yaml_string() + "---\n" + return full_content + + @classmethod + def from_yaml_string(cls, string: str) -> "DatasetMetadata": + """Loads and validates the dataset metadata from a YAML string + + Args: + string (:obj:`str`): The YAML string + + Returns: + :class:`DatasetMetadata`: The dataset's metadata + + Raises: + :obj:`TypeError`: If the dataset's metadata is invalid + """ + metadata_dict = yaml.load(string, Loader=_NoDuplicateSafeLoader) or {} + + # Convert the YAML keys to DatasetMetadata fields + metadata_dict = { + (key.replace("-", "_") if key.replace("-", "_") in cls._FIELDS_WITH_DASHES else key): value + for key, value in metadata_dict.items() + } + return cls(**metadata_dict) + + def to_yaml_string(self) -> str: + return yaml.safe_dump( + { + (key.replace("_", "-") if key in self._FIELDS_WITH_DASHES else key): value + for key, value in self.items() + }, + sort_keys=False, + allow_unicode=True, + encoding="utf-8", + ).decode("utf-8") + + +class MetadataConfigs(Dict[str, Dict[str, Any]]): + """Should be in format {config_name: {**config_params}}.""" + + FIELD_NAME: ClassVar[str] = METADATA_CONFIGS_FIELD + + @staticmethod + def _raise_if_data_files_field_not_valid(metadata_config: dict): + yaml_data_files = metadata_config.get("data_files") + if yaml_data_files is not None: + yaml_error_message = textwrap.dedent( + f""" + Expected data_files in YAML to be either a string or a list of strings + or a list of dicts with two keys: 'split' and 'path', but got {yaml_data_files} + Examples of data_files in YAML: + + data_files: data.csv + + data_files: data/*.png + + data_files: + - part0/* + - part1/* + + data_files: + - split: train + path: train/* + - split: test + path: test/* + + data_files: + - split: train + path: + - train/part1/* + - train/part2/* + - split: test + path: test/* + + PS: some symbols like dashes '-' are not allowed in split names + """ + ) + if not isinstance(yaml_data_files, (list, str)): + raise ValueError(yaml_error_message) + if isinstance(yaml_data_files, list): + for yaml_data_files_item in yaml_data_files: + if ( + not isinstance(yaml_data_files_item, (str, dict)) + or isinstance(yaml_data_files_item, dict) + and not ( + len(yaml_data_files_item) == 2 + and "split" in yaml_data_files_item + and re.match(_split_re, yaml_data_files_item["split"]) + and isinstance(yaml_data_files_item.get("path"), (str, list)) + ) + ): + raise ValueError(yaml_error_message) + + @classmethod + def _from_exported_parquet_files_and_dataset_infos( + cls, + revision: str, + exported_parquet_files: List[Dict[str, Any]], + dataset_infos: DatasetInfosDict, + ) -> "MetadataConfigs": + metadata_configs = { + config_name: { + "data_files": [ + { + "split": split_name, + "path": [ + parquet_file["url"].replace("refs%2Fconvert%2Fparquet", revision) + for parquet_file in parquet_files_for_split + ], + } + for split_name, parquet_files_for_split in groupby(parquet_files_for_config, itemgetter("split")) + ], + "version": str(dataset_infos.get(config_name, DatasetInfo()).version or "0.0.0"), + } + for config_name, parquet_files_for_config in groupby(exported_parquet_files, itemgetter("config")) + } + if dataset_infos: + # Preserve order of configs and splits + metadata_configs = { + config_name: { + "data_files": [ + data_file + for split_name in dataset_info.splits + for data_file in metadata_configs[config_name]["data_files"] + if data_file["split"] == split_name + ], + "version": metadata_configs[config_name]["version"], + } + for config_name, dataset_info in dataset_infos.items() + } + return cls(metadata_configs) + + @classmethod + def from_dataset_card_data(cls, dataset_card_data: DatasetCardData) -> "MetadataConfigs": + if dataset_card_data.get(cls.FIELD_NAME): + metadata_configs = dataset_card_data[cls.FIELD_NAME] + if not isinstance(metadata_configs, list): + raise ValueError(f"Expected {cls.FIELD_NAME} to be a list, but got '{metadata_configs}'") + for metadata_config in metadata_configs: + if "config_name" not in metadata_config: + raise ValueError( + f"Each config must include `config_name` field with a string name of a config, " + f"but got {metadata_config}. " + ) + cls._raise_if_data_files_field_not_valid(metadata_config) + return cls( + { + config["config_name"]: {param: value for param, value in config.items() if param != "config_name"} + for config in metadata_configs + } + ) + return cls() + + def to_dataset_card_data(self, dataset_card_data: DatasetCardData) -> None: + if self: + for metadata_config in self.values(): + self._raise_if_data_files_field_not_valid(metadata_config) + current_metadata_configs = self.from_dataset_card_data(dataset_card_data) + total_metadata_configs = dict(sorted({**current_metadata_configs, **self}.items())) + for config_name, config_metadata in total_metadata_configs.items(): + config_metadata.pop("config_name", None) + dataset_card_data[self.FIELD_NAME] = [ + {"config_name": config_name, **config_metadata} + for config_name, config_metadata in total_metadata_configs.items() + ] + + def get_default_config_name(self) -> Optional[str]: + default_config_name = None + for config_name, metadata_config in self.items(): + if len(self) == 1 or config_name == "default" or metadata_config.get("default"): + if default_config_name is None: + default_config_name = config_name + else: + raise ValueError( + f"Dataset has several default configs: '{default_config_name}' and '{config_name}'." + ) + return default_config_name + + +# DEPRECATED - just here to support old versions of evaluate like 0.2.2 +# To support new tasks on the Hugging Face Hub, please open a PR for this file: +# https://github.com/huggingface/huggingface.js/blob/main/packages/tasks/src/pipelines.ts +known_task_ids = { + "image-classification": [], + "translation": [], + "image-segmentation": [], + "fill-mask": [], + "automatic-speech-recognition": [], + "token-classification": [], + "sentence-similarity": [], + "audio-classification": [], + "question-answering": [], + "summarization": [], + "zero-shot-classification": [], + "table-to-text": [], + "feature-extraction": [], + "other": [], + "multiple-choice": [], + "text-classification": [], + "text-to-image": [], + "text2text-generation": [], + "zero-shot-image-classification": [], + "tabular-classification": [], + "tabular-regression": [], + "image-to-image": [], + "tabular-to-text": [], + "unconditional-image-generation": [], + "text-retrieval": [], + "text-to-speech": [], + "object-detection": [], + "audio-to-audio": [], + "text-generation": [], + "conversational": [], + "table-question-answering": [], + "visual-question-answering": [], + "image-to-text": [], + "reinforcement-learning": [], + "voice-activity-detection": [], + "time-series-forecasting": [], + "document-question-answering": [], +} + + +if __name__ == "__main__": + from argparse import ArgumentParser + + ap = ArgumentParser(usage="Validate the yaml metadata block of a README.md file.") + ap.add_argument("readme_filepath") + args = ap.parse_args() + + readme_filepath = Path(args.readme_filepath) + dataset_metadata = DatasetMetadata.from_readme(readme_filepath) + print(dataset_metadata) + dataset_metadata.to_readme(readme_filepath) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/patching.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/patching.py new file mode 100644 index 0000000000000000000000000000000000000000..f245cabd97065d9e82a1320d02999f9ec03bda36 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/patching.py @@ -0,0 +1,119 @@ +from importlib import import_module + +from .logging import get_logger + + +logger = get_logger(__name__) + + +class _PatchedModuleObj: + """Set all the modules components as attributes of the _PatchedModuleObj object.""" + + def __init__(self, module, attrs=None): + attrs = attrs or [] + if module is not None: + for key in module.__dict__: + if key in attrs or not key.startswith("__"): + setattr(self, key, getattr(module, key)) + self._original_module = module._original_module if isinstance(module, _PatchedModuleObj) else module + + +class patch_submodule: + """ + Patch a submodule attribute of an object, by keeping all other submodules intact at all levels. + + Example:: + + >>> import importlib + >>> from datasets.load import dataset_module_factory + >>> from datasets.streaming import patch_submodule, xjoin + >>> + >>> dataset_module = dataset_module_factory("snli") + >>> snli_module = importlib.import_module(dataset_module.module_path) + >>> patcher = patch_submodule(snli_module, "os.path.join", xjoin) + >>> patcher.start() + >>> assert snli_module.os.path.join is xjoin + """ + + _active_patches = [] + + def __init__(self, obj, target: str, new, attrs=None): + self.obj = obj + self.target = target + self.new = new + self.key = target.split(".")[0] + self.original = {} + self.attrs = attrs or [] + + def __enter__(self): + *submodules, target_attr = self.target.split(".") + + # Patch modules: + # it's used to patch attributes of submodules like "os.path.join"; + # in this case we need to patch "os" and "os.path" + + for i in range(len(submodules)): + try: + submodule = import_module(".".join(submodules[: i + 1])) + except ModuleNotFoundError: + continue + # We iterate over all the globals in self.obj in case we find "os" or "os.path" + for attr in self.obj.__dir__(): + obj_attr = getattr(self.obj, attr) + # We don't check for the name of the global, but rather if its value *is* "os" or "os.path". + # This allows to patch renamed modules like "from os import path as ospath". + if obj_attr is submodule or ( + isinstance(obj_attr, _PatchedModuleObj) and obj_attr._original_module is submodule + ): + self.original[attr] = obj_attr + # patch at top level + setattr(self.obj, attr, _PatchedModuleObj(obj_attr, attrs=self.attrs)) + patched = getattr(self.obj, attr) + # construct lower levels patches + for key in submodules[i + 1 :]: + setattr(patched, key, _PatchedModuleObj(getattr(patched, key, None), attrs=self.attrs)) + patched = getattr(patched, key) + # finally set the target attribute + setattr(patched, target_attr, self.new) + + # Patch attribute itself: + # it's used for builtins like "open", + # and also to patch "os.path.join" we may also need to patch "join" + # itself if it was imported as "from os.path import join". + + if submodules: # if it's an attribute of a submodule like "os.path.join" + try: + attr_value = getattr(import_module(".".join(submodules)), target_attr) + except (AttributeError, ModuleNotFoundError): + return + # We iterate over all the globals in self.obj in case we find "os.path.join" + for attr in self.obj.__dir__(): + # We don't check for the name of the global, but rather if its value *is* "os.path.join". + # This allows to patch renamed attributes like "from os.path import join as pjoin". + if getattr(self.obj, attr) is attr_value: + self.original[attr] = getattr(self.obj, attr) + setattr(self.obj, attr, self.new) + elif target_attr in globals()["__builtins__"]: # if it'a s builtin like "open" + self.original[target_attr] = globals()["__builtins__"][target_attr] + setattr(self.obj, target_attr, self.new) + else: + raise RuntimeError(f"Tried to patch attribute {target_attr} instead of a submodule.") + + def __exit__(self, *exc_info): + for attr in list(self.original): + setattr(self.obj, attr, self.original.pop(attr)) + + def start(self): + """Activate a patch.""" + self.__enter__() + self._active_patches.append(self) + + def stop(self): + """Stop an active patch.""" + try: + self._active_patches.remove(self) + except ValueError: + # If the patch hasn't been started this will fail + return None + + return self.__exit__() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/py_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/py_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..97dd6fecf523e96b1ff28b62a7c4316b12502239 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/py_utils.py @@ -0,0 +1,734 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Some python utils function and classes.""" + +import copy +import functools +import itertools +import multiprocessing.pool +import os +import queue +import re +import types +import warnings +from contextlib import contextmanager +from dataclasses import fields, is_dataclass +from multiprocessing import Manager +from pathlib import Path +from queue import Empty +from shutil import disk_usage +from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, TypeVar, Union +from urllib.parse import urlparse + +import multiprocess +import multiprocess.pool +import numpy as np +from tqdm.auto import tqdm + +from .. import config +from ..parallel import parallel_map +from . import logging +from . import tqdm as hf_tqdm +from ._dill import ( # noqa: F401 # imported for backward compatibility. TODO: remove in 3.0.0 + Pickler, + dump, + dumps, + pklregister, +) +from ._filelock import FileLock + + +try: # pragma: no branch + import typing_extensions as _typing_extensions + from typing_extensions import Final, Literal +except ImportError: + _typing_extensions = Literal = Final = None + + +logger = logging.get_logger(__name__) + + +# NOTE: When used on an instance method, the cache is shared across all +# instances and IS NOT per-instance. +# See +# https://stackoverflow.com/questions/14946264/python-lru-cache-decorator-per-instance +# For @property methods, use @memoized_property below. +memoize = functools.lru_cache + + +def size_str(size_in_bytes): + """Returns a human readable size string. + + If size_in_bytes is None, then returns "Unknown size". + + For example `size_str(1.5 * datasets.units.GiB) == "1.50 GiB"`. + + Args: + size_in_bytes: `int` or `None`, the size, in bytes, that we want to + format as a human-readable size string. + """ + if not size_in_bytes: + return "Unknown size" + + _NAME_LIST = [("PiB", 2**50), ("TiB", 2**40), ("GiB", 2**30), ("MiB", 2**20), ("KiB", 2**10)] + + size_in_bytes = float(size_in_bytes) + for name, size_bytes in _NAME_LIST: + value = size_in_bytes / size_bytes + if value >= 1.0: + return f"{value:.2f} {name}" + return f"{int(size_in_bytes)} bytes" + + +def convert_file_size_to_int(size: Union[int, str]) -> int: + """ + Converts a size expressed as a string with digits an unit (like `"50MB"`) to an integer (in bytes). + + Args: + size (`int` or `str`): The size to convert. Will be directly returned if an `int`. + + Example: + + ```py + >>> convert_file_size_to_int("1MiB") + 1048576 + ``` + """ + if isinstance(size, int): + return size + if size.upper().endswith("PIB"): + return int(size[:-3]) * (2**50) + if size.upper().endswith("TIB"): + return int(size[:-3]) * (2**40) + if size.upper().endswith("GIB"): + return int(size[:-3]) * (2**30) + if size.upper().endswith("MIB"): + return int(size[:-3]) * (2**20) + if size.upper().endswith("KIB"): + return int(size[:-3]) * (2**10) + if size.upper().endswith("PB"): + int_size = int(size[:-2]) * (10**15) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("TB"): + int_size = int(size[:-2]) * (10**12) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("GB"): + int_size = int(size[:-2]) * (10**9) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("MB"): + int_size = int(size[:-2]) * (10**6) + return int_size // 8 if size.endswith("b") else int_size + if size.upper().endswith("KB"): + int_size = int(size[:-2]) * (10**3) + return int_size // 8 if size.endswith("b") else int_size + raise ValueError(f"`size={size}` is not in a valid format. Use an integer followed by the unit, e.g., '5GB'.") + + +def glob_pattern_to_regex(pattern): + # partially taken from fsspec: + # https://github.com/fsspec/filesystem_spec/blob/697d0f8133d8a5fbc3926e4761d7ecd51337ce50/fsspec/asyn.py#L735 + return ( + pattern.replace("\\", r"\\") + .replace(".", r"\.") + .replace("*", ".*") + .replace("+", r"\+") + .replace("//", "/") + .replace("(", r"\(") + .replace(")", r"\)") + .replace("|", r"\|") + .replace("^", r"\^") + .replace("$", r"\$") + .rstrip("/") + .replace("?", ".") + ) + + +def string_to_dict(string: str, pattern: str) -> Dict[str, str]: + """Un-format a string using a python f-string pattern. + From https://stackoverflow.com/a/36838374 + + Example:: + + >>> p = 'hello, my name is {name} and I am a {age} year old {what}' + >>> s = p.format(name='cody', age=18, what='quarterback') + >>> s + 'hello, my name is cody and I am a 18 year old quarterback' + >>> string_to_dict(s, p) + {'age': '18', 'name': 'cody', 'what': 'quarterback'} + + Args: + string (str): input string + pattern (str): pattern formatted like a python f-string + + Returns: + Dict[str, str]: dictionary of variable -> value, retrieved from the input using the pattern + + Raises: + ValueError: if the string doesn't match the pattern + """ + regex = re.sub(r"{(.+?)}", r"(?P<_\1>.+)", pattern) + result = re.search(regex, string) + if result is None: + raise ValueError(f"String {string} doesn't match the pattern {pattern}") + values = list(result.groups()) + keys = re.findall(r"{(.+?)}", pattern) + _dict = dict(zip(keys, values)) + return _dict + + +def asdict(obj): + """Convert an object to its dictionary representation recursively. + + + """ + + # Implementation based on https://docs.python.org/3/library/dataclasses.html#dataclasses.asdict + + def _is_dataclass_instance(obj): + # https://docs.python.org/3/library/dataclasses.html#dataclasses.is_dataclass + return is_dataclass(obj) and not isinstance(obj, type) + + def _asdict_inner(obj): + if _is_dataclass_instance(obj): + result = {} + for f in fields(obj): + value = _asdict_inner(getattr(obj, f.name)) + if not f.init or value != f.default or f.metadata.get("include_in_asdict_even_if_is_default", False): + result[f.name] = value + return result + elif isinstance(obj, tuple) and hasattr(obj, "_fields"): + # obj is a namedtuple + return type(obj)(*[_asdict_inner(v) for v in obj]) + elif isinstance(obj, (list, tuple)): + # Assume we can create an object of this type by passing in a + # generator (which is not true for namedtuples, handled + # above). + return type(obj)(_asdict_inner(v) for v in obj) + elif isinstance(obj, dict): + return {_asdict_inner(k): _asdict_inner(v) for k, v in obj.items()} + else: + return copy.deepcopy(obj) + + if not isinstance(obj, dict) and not _is_dataclass_instance(obj): + raise TypeError(f"{obj} is not a dict or a dataclass") + + return _asdict_inner(obj) + + +@contextmanager +def temporary_assignment(obj, attr, value): + """Temporarily assign obj.attr to value.""" + original = getattr(obj, attr, None) + setattr(obj, attr, value) + try: + yield + finally: + setattr(obj, attr, original) + + +@contextmanager +def temp_seed(seed: int, set_pytorch=False, set_tensorflow=False): + """Temporarily set the random seed. This works for python numpy, pytorch and tensorflow.""" + np_state = np.random.get_state() + np.random.seed(seed) + + if set_pytorch and config.TORCH_AVAILABLE: + import torch + + torch_state = torch.random.get_rng_state() + torch.random.manual_seed(seed) + + if torch.cuda.is_available(): + torch_cuda_states = torch.cuda.get_rng_state_all() + torch.cuda.manual_seed_all(seed) + + if set_tensorflow and config.TF_AVAILABLE: + import tensorflow as tf + from tensorflow.python.eager import context as tfpycontext + + tf_state = tf.random.get_global_generator() + temp_gen = tf.random.Generator.from_seed(seed) + tf.random.set_global_generator(temp_gen) + + if not tf.executing_eagerly(): + raise ValueError("Setting random seed for TensorFlow is only available in eager mode") + + tf_context = tfpycontext.context() # eager mode context + tf_seed = tf_context._seed + tf_rng_initialized = hasattr(tf_context, "_rng") + if tf_rng_initialized: + tf_rng = tf_context._rng + tf_context._set_global_seed(seed) + + try: + yield + finally: + np.random.set_state(np_state) + + if set_pytorch and config.TORCH_AVAILABLE: + torch.random.set_rng_state(torch_state) + if torch.cuda.is_available(): + torch.cuda.set_rng_state_all(torch_cuda_states) + + if set_tensorflow and config.TF_AVAILABLE: + tf.random.set_global_generator(tf_state) + + tf_context._seed = tf_seed + if tf_rng_initialized: + tf_context._rng = tf_rng + else: + delattr(tf_context, "_rng") + + +def unique_values(values): + """Iterate over iterable and return only unique values in order.""" + seen = set() + for value in values: + if value not in seen: + seen.add(value) + yield value + + +def no_op_if_value_is_null(func): + """If the value is None, return None, else call `func`.""" + + def wrapper(value): + return func(value) if value is not None else None + + return wrapper + + +def first_non_null_value(iterable): + """Return the index and the value of the first non-null value in the iterable. If all values are None, return -1 as index.""" + for i, value in enumerate(iterable): + if value is not None: + return i, value + return -1, None + + +def zip_dict(*dicts): + """Iterate over items of dictionaries grouped by their keys.""" + for key in unique_values(itertools.chain(*dicts)): # set merge all keys + # Will raise KeyError if the dict don't have the same keys + yield key, tuple(d[key] for d in dicts) + + +class NonMutableDict(dict): + """Dict where keys can only be added but not modified. + + Will raise an error if the user try to overwrite one key. The error message + can be customized during construction. It will be formatted using {key} for + the overwritten key. + """ + + def __init__(self, *args, **kwargs): + self._error_msg = kwargs.pop( + "error_msg", + "Try to overwrite existing key: {key}", + ) + if kwargs: + raise ValueError("NonMutableDict cannot be initialized with kwargs.") + super().__init__(*args, **kwargs) + + def __setitem__(self, key, value): + if key in self: + raise ValueError(self._error_msg.format(key=key)) + return super().__setitem__(key, value) + + def update(self, other): + if any(k in self for k in other): + raise ValueError(self._error_msg.format(key=set(self) & set(other))) + return super().update(other) + + +class classproperty(property): # pylint: disable=invalid-name + """Descriptor to be used as decorator for @classmethods.""" + + def __get__(self, obj, objtype=None): + return self.fget.__get__(None, objtype)() + + +def _single_map_nested(args): + """Apply a function recursively to each element of a nested data struct.""" + function, data_struct, batched, batch_size, types, rank, disable_tqdm, desc = args + + # Singleton first to spare some computation + if not isinstance(data_struct, dict) and not isinstance(data_struct, types): + if batched: + return function([data_struct])[0] + else: + return function(data_struct) + if ( + batched + and not isinstance(data_struct, dict) + and isinstance(data_struct, types) + and all(not isinstance(v, (dict, types)) for v in data_struct) + ): + return [mapped_item for batch in iter_batched(data_struct, batch_size) for mapped_item in function(batch)] + + # Reduce logging to keep things readable in multiprocessing with tqdm + if rank is not None and logging.get_verbosity() < logging.WARNING: + logging.set_verbosity_warning() + # Print at least one thing to fix tqdm in notebooks in multiprocessing + # see https://github.com/tqdm/tqdm/issues/485#issuecomment-473338308 + if rank is not None and not disable_tqdm and any("notebook" in tqdm_cls.__name__ for tqdm_cls in tqdm.__mro__): + print(" ", end="", flush=True) + + # Loop over single examples or batches and write to buffer/file if examples are to be updated + pbar_iterable = data_struct.items() if isinstance(data_struct, dict) else data_struct + pbar_desc = (desc + " " if desc is not None else "") + "#" + str(rank) if rank is not None else desc + with hf_tqdm(pbar_iterable, disable=disable_tqdm, position=rank, unit="obj", desc=pbar_desc) as pbar: + if isinstance(data_struct, dict): + return { + k: _single_map_nested((function, v, batched, batch_size, types, None, True, None)) for k, v in pbar + } + else: + mapped = [_single_map_nested((function, v, batched, batch_size, types, None, True, None)) for v in pbar] + if isinstance(data_struct, list): + return mapped + elif isinstance(data_struct, tuple): + return tuple(mapped) + else: + return np.array(mapped) + + +def map_nested( + function: Callable[[Any], Any], + data_struct: Any, + dict_only: bool = False, + map_list: bool = True, + map_tuple: bool = False, + map_numpy: bool = False, + num_proc: Optional[int] = None, + parallel_min_length: int = 2, + batched: bool = False, + batch_size: Optional[int] = 1000, + types: Optional[tuple] = None, + disable_tqdm: bool = True, + desc: Optional[str] = None, +) -> Any: + """Apply a function recursively to each element of a nested data struct. + + Use multiprocessing if num_proc > 1 and the length of data_struct is greater than or equal to + `parallel_min_length`. + + + + Before version 2.5.0, multiprocessing was not used if `num_proc` was greater than or equal to ``len(iterable)``. + + Now, if `num_proc` is greater than or equal to ``len(iterable)``, `num_proc` is set to ``len(iterable)`` and + multiprocessing is used. + + + + Args: + function (`Callable`): Function to be applied to `data_struct`. + data_struct (`Any`): Data structure to apply `function` to. + dict_only (`bool`, default `False`): Whether only apply `function` recursively to `dict` values in + `data_struct`. + map_list (`bool`, default `True`): Whether also apply `function` recursively to `list` elements (besides `dict` + values). + map_tuple (`bool`, default `False`): Whether also apply `function` recursively to `tuple` elements (besides + `dict` values). + map_numpy (`bool, default `False`): Whether also apply `function` recursively to `numpy.array` elements (besides + `dict` values). + num_proc (`int`, *optional*): Number of processes. + The level in the data struct used for multiprocessing is the first level that has smaller sub-structs, + starting from the root. + parallel_min_length (`int`, default `2`): Minimum length of `data_struct` required for parallel + processing. + + batched (`bool`, defaults to `False`): + Provide batch of items to `function`. + + batch_size (`int`, *optional*, defaults to `1000`): + Number of items per batch provided to `function` if `batched=True`. + If `batch_size <= 0` or `batch_size == None`, provide the full iterable as a single batch to `function`. + + types (`tuple`, *optional*): Additional types (besides `dict` values) to apply `function` recursively to their + elements. + disable_tqdm (`bool`, default `True`): Whether to disable the tqdm progressbar. + desc (`str`, *optional*): Prefix for the tqdm progressbar. + + Returns: + `Any` + """ + if types is None: + types = [] + if not dict_only: + if map_list: + types.append(list) + if map_tuple: + types.append(tuple) + if map_numpy: + types.append(np.ndarray) + types = tuple(types) + + # Singleton + if not isinstance(data_struct, dict) and not isinstance(data_struct, types): + if batched: + data_struct = [data_struct] + mapped = function(data_struct) + if batched: + mapped = mapped[0] + return mapped + + iterable = list(data_struct.values()) if isinstance(data_struct, dict) else data_struct + + if num_proc is None: + num_proc = 1 + if any(isinstance(v, types) and len(v) > len(iterable) for v in iterable): + mapped = [ + map_nested( + function=function, + data_struct=obj, + num_proc=num_proc, + parallel_min_length=parallel_min_length, + batched=batched, + batch_size=batch_size, + types=types, + ) + for obj in iterable + ] + elif num_proc != -1 and num_proc <= 1 or len(iterable) < parallel_min_length: + if batched: + if batch_size is None or batch_size <= 0: + batch_size = max(len(iterable) // num_proc + int(len(iterable) % num_proc > 0), 1) + iterable = list(iter_batched(iterable, batch_size)) + mapped = [ + _single_map_nested((function, obj, batched, batch_size, types, None, True, None)) + for obj in hf_tqdm(iterable, disable=disable_tqdm, desc=desc) + ] + if batched: + mapped = [mapped_item for mapped_batch in mapped for mapped_item in mapped_batch] + else: + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + message=".* is experimental and might be subject to breaking changes in the future\\.$", + category=UserWarning, + ) + if batched: + if batch_size is None or batch_size <= 0: + batch_size = len(iterable) // num_proc + int(len(iterable) % num_proc > 0) + iterable = list(iter_batched(iterable, batch_size)) + mapped = parallel_map( + function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, _single_map_nested + ) + if batched: + mapped = [mapped_item for mapped_batch in mapped for mapped_item in mapped_batch] + + if isinstance(data_struct, dict): + return dict(zip(data_struct.keys(), mapped)) + else: + if isinstance(data_struct, list): + return mapped + elif isinstance(data_struct, tuple): + return tuple(mapped) + else: + return np.array(mapped) + + +class NestedDataStructure: + def __init__(self, data=None): + self.data = data if data is not None else [] + + def flatten(self, data=None): + data = data if data is not None else self.data + if isinstance(data, dict): + return self.flatten(list(data.values())) + elif isinstance(data, (list, tuple)): + return [flattened for item in data for flattened in self.flatten(item)] + else: + return [data] + + +def has_sufficient_disk_space(needed_bytes, directory="."): + try: + free_bytes = disk_usage(os.path.abspath(directory)).free + except OSError: + return True + return needed_bytes < free_bytes + + +def _convert_github_url(url_path: str) -> Tuple[str, Optional[str]]: + """Convert a link to a file on a github repo in a link to the raw github object.""" + parsed = urlparse(url_path) + sub_directory = None + if parsed.scheme in ("http", "https", "s3") and parsed.netloc == "github.com": + if "blob" in url_path: + if not url_path.endswith(".py"): + raise ValueError(f"External import from github at {url_path} should point to a file ending with '.py'") + url_path = url_path.replace("blob", "raw") # Point to the raw file + else: + # Parse github url to point to zip + github_path = parsed.path[1:] + repo_info, branch = github_path.split("/tree/") if "/tree/" in github_path else (github_path, "master") + repo_owner, repo_name = repo_info.split("/") + url_path = f"https://github.com/{repo_owner}/{repo_name}/archive/{branch}.zip" + sub_directory = f"{repo_name}-{branch}" + return url_path, sub_directory + + +def lock_importable_file(importable_local_file: str) -> FileLock: + # Check the directory with a unique name in our dataset folder + # path is: ./datasets/dataset_name/hash_from_code/script.py + # we use a hash as subdirectory_name to be able to have multiple versions of a dataset/metric processing file together + importable_directory_path = str(Path(importable_local_file).resolve().parent.parent) + lock_path = importable_directory_path + ".lock" + return FileLock(lock_path) + + +def get_imports(file_path: str) -> Tuple[str, str, str, str]: + """Find whether we should import or clone additional files for a given processing script. + And list the import. + + We allow: + - library dependencies, + - local dependencies and + - external dependencies whose url is specified with a comment starting from "# From:' followed by the raw url to a file, an archive or a github repository. + external dependencies will be downloaded (and extracted if needed in the dataset folder). + We also add an `__init__.py` to each sub-folder of a downloaded folder so the user can import from them in the script. + + Note that only direct import in the dataset processing script will be handled + We don't recursively explore the additional import to download further files. + + Example:: + + import tensorflow + import .c4_utils + import .clicr.dataset-code.build_json_dataset # From: https://raw.githubusercontent.com/clips/clicr/master/dataset-code/build_json_dataset + """ + lines = [] + with open(file_path, encoding="utf-8") as f: + lines.extend(f.readlines()) + + logger.debug(f"Checking {file_path} for additional imports.") + imports: List[Tuple[str, str, str, Optional[str]]] = [] + is_in_docstring = False + for line in lines: + docstr_start_match = re.findall(r'[\s\S]*?"""[\s\S]*?', line) + + if len(docstr_start_match) == 1: + # flip True <=> False only if doctstring + # starts at line without finishing + is_in_docstring = not is_in_docstring + + if is_in_docstring: + # import statements in doctstrings should + # not be added as required dependencies + continue + + match = re.match(r"^import\s+(\.?)([^\s\.]+)[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)", line, flags=re.MULTILINE) + if match is None: + match = re.match( + r"^from\s+(\.?)([^\s\.]+)(?:[^\s]*)\s+import\s+[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)", + line, + flags=re.MULTILINE, + ) + if match is None: + continue + if match.group(1): + # The import starts with a '.', we will download the relevant file + if any(imp[1] == match.group(2) for imp in imports): + # We already have this import + continue + if match.group(3): + # The import has a comment with 'From:', we'll retrieve it from the given url + url_path = match.group(3) + url_path, sub_directory = _convert_github_url(url_path) + imports.append(("external", match.group(2), url_path, sub_directory)) + elif match.group(2): + # The import should be at the same place as the file + imports.append(("internal", match.group(2), match.group(2), None)) + else: + if match.group(3): + # The import has a comment with `From: git+https:...`, asks user to pip install from git. + url_path = match.group(3) + imports.append(("library", match.group(2), url_path, None)) + else: + imports.append(("library", match.group(2), match.group(2), None)) + + return imports + + +def copyfunc(func): + result = types.FunctionType(func.__code__, func.__globals__, func.__name__, func.__defaults__, func.__closure__) + result.__kwdefaults__ = func.__kwdefaults__ + return result + + +Y = TypeVar("Y") + + +def _write_generator_to_queue(queue: queue.Queue, func: Callable[..., Iterable[Y]], kwargs: dict) -> int: + for i, result in enumerate(func(**kwargs)): + queue.put(result) + return i + + +def _get_pool_pid(pool: Union[multiprocessing.pool.Pool, multiprocess.pool.Pool]) -> Set[int]: + return {f.pid for f in pool._pool} + + +def iflatmap_unordered( + pool: Union[multiprocessing.pool.Pool, multiprocess.pool.Pool], + func: Callable[..., Iterable[Y]], + *, + kwargs_iterable: Iterable[dict], +) -> Iterable[Y]: + initial_pool_pid = _get_pool_pid(pool) + pool_changed = False + manager_cls = Manager if isinstance(pool, multiprocessing.pool.Pool) else multiprocess.Manager + with manager_cls() as manager: + queue = manager.Queue() + async_results = [ + pool.apply_async(_write_generator_to_queue, (queue, func, kwargs)) for kwargs in kwargs_iterable + ] + try: + while True: + try: + yield queue.get(timeout=0.05) + except Empty: + if all(async_result.ready() for async_result in async_results) and queue.empty(): + break + if _get_pool_pid(pool) != initial_pool_pid: + pool_changed = True + # One of the subprocesses has died. We should not wait forever. + raise RuntimeError( + "One of the subprocesses has abruptly died during map operation." + "To debug the error, disable multiprocessing." + ) + finally: + if not pool_changed: + # we get the result in case there's an error to raise + [async_result.get(timeout=0.05) for async_result in async_results] + + +T = TypeVar("T") + + +def iter_batched(iterable: Iterable[T], n: int) -> Iterable[List[T]]: + if n < 1: + raise ValueError(f"Invalid batch size {n}") + batch = [] + for item in iterable: + batch.append(item) + if len(batch) == n: + yield batch + batch = [] + if batch: + yield batch diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/readme.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/readme.py new file mode 100644 index 0000000000000000000000000000000000000000..66ed087f7d67181c6840179fa634e8b8e4238f85 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/readme.py @@ -0,0 +1,277 @@ +# loading package files: https://stackoverflow.com/a/20885799 +import importlib.resources as pkg_resources +import logging +from pathlib import Path +from typing import Any, List, Tuple + +import yaml + +from . import resources +from .deprecation_utils import deprecated + + +BASE_REF_URL = "https://github.com/huggingface/datasets/tree/main/src/datasets/utils" +this_url = f"{BASE_REF_URL}/{__file__}" +logger = logging.getLogger(__name__) + + +def load_yaml_resource(resource: str) -> Tuple[Any, str]: + content = pkg_resources.read_text(resources, resource) + return yaml.safe_load(content), f"{BASE_REF_URL}/resources/{resource}" + + +readme_structure, known_readme_structure_url = load_yaml_resource("readme_structure.yaml") + +FILLER_TEXT = [ + "[Needs More Information]", + "[More Information Needed]", + "(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)", +] + +# Dictionary representation of section/readme, error_list, warning_list +ReadmeValidatorOutput = Tuple[dict, List[str], List[str]] + + +class Section: + def __init__(self, name: str, level: str, lines: List[str] = None, suppress_parsing_errors: bool = False): + self.name = name + self.level = level + self.lines = lines + self.text = "" + self.is_empty_text = True + self.content = {} + self.parsing_error_list = [] + self.parsing_warning_list = [] + if self.lines is not None: + self.parse(suppress_parsing_errors=suppress_parsing_errors) + + def parse(self, suppress_parsing_errors: bool = False): + current_sub_level = "" + current_lines = [] + code_start = False + for line in self.lines: + if line.strip(" \n") == "": + continue + elif line.strip(" \n")[:3] == "```": + code_start = not code_start + elif line.split()[0] == self.level + "#" and not code_start: + if current_sub_level != "": + self.content[current_sub_level] = Section(current_sub_level, self.level + "#", current_lines) + current_lines = [] + else: + if current_lines != []: + self.text += "".join(current_lines).strip() + if self.text != "" and self.text not in FILLER_TEXT: + self.is_empty_text = False + current_lines = [] + + current_sub_level = " ".join(line.split()[1:]).strip(" \n") + else: + current_lines.append(line) + else: + if current_sub_level != "": + if current_sub_level in self.content: + self.parsing_error_list.append( + f"Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections." + ) + self.content[current_sub_level] = Section(current_sub_level, self.level + "#", current_lines) + else: + if current_lines != []: + self.text += "".join(current_lines).strip() + if self.text != "" and self.text not in FILLER_TEXT: + self.is_empty_text = False + + if self.level == "" and not suppress_parsing_errors: + if self.parsing_error_list != [] or self.parsing_warning_list != []: + errors = errors = "\n".join("-\t" + x for x in self.parsing_error_list + self.parsing_warning_list) + error_string = f"The following issues were found while parsing the README at `{self.name}`:\n" + errors + raise ValueError(error_string) + + def validate(self, structure: dict) -> ReadmeValidatorOutput: + """Validates a Section class object recursively using the structure provided as a dictionary. + + Args: + structute (:obj: `dict`): The dictionary representing expected structure. + + Returns: + :obj: `ReadmeValidatorOutput`: The dictionary representation of the section, and the errors. + """ + # Header text validation + error_list = [] + warning_list = [] + if structure["allow_empty"] is False: + # If content is expected + if self.is_empty_text and self.content == {}: + # If no content is found, mention it in the error_list + error_list.append(f"Expected some content in section `{self.name}` but it is empty.") + + if structure["allow_empty_text"] is False: + # If some text is expected + if self.is_empty_text: + # If no text is found, mention it in the error_list + error_list.append( + f"Expected some text in section `{self.name}` but it is empty (text in subsections are ignored)." + ) + # Subsections Validation + if structure["subsections"] is not None: + # If subsections are expected + if self.content == {}: + # If no subsections are present + values = [subsection["name"] for subsection in structure["subsections"]] + # Mention the expected values in the error_list + error_list.append( + f"Section `{self.name}` expected the following subsections: {', '.join(['`'+x+'`' for x in values])}. Found 'None'." + ) + else: + # If some subsections are present + structure_names = [subsection["name"] for subsection in structure["subsections"]] + has_missing_subsections = False + for idx, name in enumerate(structure_names): + if name not in self.content: + # If the expected subsection is not present + error_list.append(f"Section `{self.name}` is missing subsection: `{name}`.") + has_missing_subsections = True + else: + # If the subsection is present, validate subsection, return the result + # and concat the errors from subsection to section error_list + + # Skip sublevel validation if current level is `###` + if self.level == "###": + continue + else: + _, subsec_error_list, subsec_warning_list = self.content[name].validate( + structure["subsections"][idx] + ) + error_list += subsec_error_list + warning_list += subsec_warning_list + + if has_missing_subsections: # we only allow to have extra subsections if all the other ones are here + for name in self.content: + if name not in structure_names: + # If an extra subsection is present + warning_list.append( + f"`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown." + ) + if error_list: + # If there are errors, do not return the dictionary as it is invalid + return {}, error_list, warning_list + else: + return self.to_dict(), error_list, warning_list + + def to_dict(self) -> dict: + """Returns the dictionary representation of a section.""" + return { + "name": self.name, + "text": self.text, + "is_empty_text": self.is_empty_text, + "subsections": [value.to_dict() for value in self.content.values()], + } + + +@deprecated("Use `huggingface_hub.DatasetCard` instead.") +class ReadMe(Section): # Level 0 + def __init__(self, name: str, lines: List[str], structure: dict = None, suppress_parsing_errors: bool = False): + super().__init__(name=name, level="") # Not using lines here as we need to use a child class parse + self.structure = structure + self.yaml_tags_line_count = -2 + self.tag_count = 0 + self.lines = lines + if self.lines is not None: + self.parse(suppress_parsing_errors=suppress_parsing_errors) + + def validate(self): + if self.structure is None: + content, error_list, warning_list = self._validate(readme_structure) + else: + content, error_list, warning_list = self._validate(self.structure) + if error_list != [] or warning_list != []: + errors = "\n".join(["-\t" + x for x in error_list + warning_list]) + error_string = f"The following issues were found for the README at `{self.name}`:\n" + errors + raise ValueError(error_string) + + @classmethod + def from_readme(cls, path: Path, structure: dict = None, suppress_parsing_errors: bool = False): + with open(path, encoding="utf-8") as f: + lines = f.readlines() + return cls(path, lines, structure, suppress_parsing_errors=suppress_parsing_errors) + + @classmethod + def from_string( + cls, string: str, structure: dict = None, root_name: str = "root", suppress_parsing_errors: bool = False + ): + lines = string.split("\n") + return cls(root_name, lines, structure, suppress_parsing_errors=suppress_parsing_errors) + + def parse(self, suppress_parsing_errors: bool = False): + # Skip Tags + line_count = 0 + + for line in self.lines: + self.yaml_tags_line_count += 1 + if line.strip(" \n") == "---": + self.tag_count += 1 + if self.tag_count == 2: + break + line_count += 1 + if self.tag_count == 2: + self.lines = self.lines[line_count + 1 :] # Get the last + 1 th item. + else: + self.lines = self.lines[self.tag_count :] + super().parse(suppress_parsing_errors=suppress_parsing_errors) + + def __str__(self): + """Returns the string of dictionary representation of the ReadMe.""" + return str(self.to_dict()) + + def _validate(self, readme_structure): + error_list = [] + warning_list = [] + if self.yaml_tags_line_count == 0: + warning_list.append("Empty YAML markers are present in the README.") + elif self.tag_count == 0: + warning_list.append("No YAML markers are present in the README.") + elif self.tag_count == 1: + warning_list.append("Only the start of YAML tags present in the README.") + # Check how many first level sections are present. + num_first_level_keys = len(self.content.keys()) + if num_first_level_keys > 1: + # If more than one, add to the error list, continue + error_list.append( + f"The README has several first-level headings: {', '.join(['`'+x+'`' for x in list(self.content.keys())])}. Only one heading is expected. Skipping further validation for this README." + ) + elif num_first_level_keys < 1: + # If less than one, append error. + error_list.append( + "The README has no first-level headings. One heading is expected. Skipping further validation for this README." + ) + + else: + # If one exactly + start_key = list(self.content.keys())[0] # Get the key + if start_key.startswith("Dataset Card for"): # Check correct start + # If the starting is correct, validate all the sections + _, sec_error_list, sec_warning_list = self.content[start_key].validate( + readme_structure["subsections"][0] + ) + error_list += sec_error_list + warning_list += sec_warning_list + else: + # If not found, append error + error_list.append( + "No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README." + ) + if error_list: + # If there are errors, do not return the dictionary as it is invalid + return {}, error_list, warning_list + else: + return self.to_dict(), error_list, warning_list + + +if __name__ == "__main__": + from argparse import ArgumentParser + + ap = ArgumentParser(usage="Validate the content (excluding YAML tags) of a README.md file.") + ap.add_argument("readme_filepath") + args = ap.parse_args() + readme_filepath = Path(args.readme_filepath) + readme = ReadMe.from_readme(readme_filepath) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3488f7bb73345e37b62511fd3a684de042b409cc Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/creators.json b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/creators.json new file mode 100644 index 0000000000000000000000000000000000000000..d9e15f0039cc27ed8abd9fdf394423a3fada2c95 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/creators.json @@ -0,0 +1,17 @@ +{ + "language": [ + "found", + "crowdsourced", + "expert-generated", + "machine-generated", + "other" + ], + "annotations": [ + "found", + "crowdsourced", + "expert-generated", + "machine-generated", + "no-annotation", + "other" + ] +} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/languages.json b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/languages.json new file mode 100644 index 0000000000000000000000000000000000000000..ea7686f956b898af3faf97b86be89b71d88855d4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/languages.json @@ -0,0 +1,8026 @@ +{ + "code": "Programming language (C++, Java, Javascript, Python, etc.)", + "aa": "Afar", + "aaa": "Ghotuo", + "aab": "Alumu-Tesu", + "aac": "Ari", + "aad": "Amal", + "aae": "Arbëreshë Albanian", + "aaf": "Aranadan", + "aag": "Ambrak", + "aah": "Abu' Arapesh", + "aai": "Arifama-Miniafia", + "aak": "Ankave", + "aal": "Afade", + "aan": "Anambé", + "aao": "Algerian Saharan Arabic", + "aap": "Pará Arára", + "aaq": "Eastern Abnaki", + "aas": "Aasáx", + "aat": "Arvanitika Albanian", + "aau": "Abau", + "aav": "Austro-Asiatic languages", + "aaw": "Solong", + "aax": "Mandobo Atas", + "aaz": "Amarasi", + "ab": "Abkhazian", + "aba": "Abé", + "abb": "Bankon", + "abc": "Ambala Ayta", + "abd": "Manide", + "abe": "Western Abnaki", + "abf": "Abai Sungai", + "abg": "Abaga", + "abh": "Tajiki Arabic", + "abi": "Abidji", + "abj": "Aka-Bea", + "abl": "Lampung Nyo", + "abm": "Abanyom", + "abn": "Abua", + "abo": "Abon", + "abp": "Abellen Ayta", + "abq": "Abaza", + "abr": "Abron", + "abs": "Ambonese Malay", + "abt": "Ambulas", + "abu": "Abure", + "abv": "Baharna Arabic", + "abw": "Pal", + "abx": "Inabaknon", + "aby": "Aneme Wake", + "abz": "Abui", + "aca": "Achagua", + "acb": "Áncá", + "acd": "Gikyode", + "ace": "Achinese", + "acf": "Saint Lucian Creole French", + "ach": "Acoli", + "aci": "Aka-Cari", + "ack": "Aka-Kora", + "acl": "Akar-Bale", + "acm": "Mesopotamian Arabic", + "acn": "Achang", + "acp": "Eastern Acipa", + "acq": "Ta'izzi-Adeni Arabic", + "acr": "Achi", + "acs": "Acroá", + "act": "Achterhoeks", + "acu": "Achuar-Shiwiar", + "acv": "Achumawi", + "acw": "Hijazi Arabic", + "acx": "Omani Arabic", + "acy": "Cypriot Arabic", + "acz": "Acheron", + "ada": "Adangme", + "adb": "Atauran", + "add": "Lidzonka; Dzodinka", + "ade": "Adele", + "adf": "Dhofari Arabic", + "adg": "Andegerebinha", + "adh": "Adhola", + "adi": "Adi", + "adj": "Adioukrou", + "adl": "Galo", + "adn": "Adang", + "ado": "Abu", + "adq": "Adangbe", + "adr": "Adonara", + "ads": "Adamorobe Sign Language", + "adt": "Adnyamathanha", + "adu": "Aduge", + "adw": "Amundava", + "adx": "Amdo Tibetan", + "ady": "Adyghe; Adygei", + "adz": "Adzera", + "ae": "Avestan", + "aea": "Areba", + "aeb": "Tunisian Arabic", + "aec": "Saidi Arabic", + "aed": "Argentine Sign Language", + "aee": "Northeast Pashai; Northeast Pashayi", + "aek": "Haeke", + "ael": "Ambele", + "aem": "Arem", + "aen": "Armenian Sign Language", + "aeq": "Aer", + "aer": "Eastern Arrernte", + "aes": "Alsea", + "aeu": "Akeu", + "aew": "Ambakich", + "aey": "Amele", + "aez": "Aeka", + "af": "Afrikaans", + "afa": "Afro-Asiatic languages", + "afb": "Gulf Arabic", + "afd": "Andai", + "afe": "Putukwam", + "afg": "Afghan Sign Language", + "afh": "Afrihili", + "afi": "Akrukay; Chini", + "afk": "Nanubae", + "afn": "Defaka", + "afo": "Eloyi", + "afp": "Tapei", + "afs": "Afro-Seminole Creole", + "aft": "Afitti", + "afu": "Awutu", + "afz": "Obokuitai", + "aga": "Aguano", + "agb": "Legbo", + "agc": "Agatu", + "agd": "Agarabi", + "age": "Angal", + "agf": "Arguni", + "agg": "Angor", + "agh": "Ngelima", + "agi": "Agariya", + "agj": "Argobba", + "agk": "Isarog Agta", + "agl": "Fembe", + "agm": "Angaataha", + "agn": "Agutaynen", + "ago": "Tainae", + "agq": "Aghem", + "agr": "Aguaruna", + "ags": "Esimbi", + "agt": "Central Cagayan Agta", + "agu": "Aguacateco", + "agv": "Remontado Dumagat", + "agw": "Kahua", + "agx": "Aghul", + "agy": "Southern Alta", + "agz": "Mt. Iriga Agta", + "aha": "Ahanta", + "ahb": "Axamb", + "ahg": "Qimant", + "ahh": "Aghu", + "ahi": "Tiagbamrin Aizi", + "ahk": "Akha", + "ahl": "Igo", + "ahm": "Mobumrin Aizi", + "ahn": "Àhàn", + "aho": "Ahom", + "ahp": "Aproumu Aizi", + "ahr": "Ahirani", + "ahs": "Ashe", + "aht": "Ahtena", + "aia": "Arosi", + "aib": "Ainu (China)", + "aic": "Ainbai", + "aid": "Alngith", + "aie": "Amara", + "aif": "Agi", + "aig": "Antigua and Barbuda Creole English", + "aih": "Ai-Cham", + "aii": "Assyrian Neo-Aramaic", + "aij": "Lishanid Noshan", + "aik": "Ake", + "ail": "Aimele", + "aim": "Aimol", + "ain": "Ainu (Japan)", + "aio": "Aiton", + "aip": "Burumakok", + "aiq": "Aimaq", + "air": "Airoran", + "ait": "Arikem", + "aiw": "Aari", + "aix": "Aighon", + "aiy": "Ali", + "aja": "Aja (South Sudan)", + "ajg": "Aja (Benin)", + "aji": "Ajië", + "ajn": "Andajin", + "ajp": "South Levantine Arabic", + "ajs": "Algerian Jewish Sign Language", + "aju": "Judeo-Moroccan Arabic", + "ajw": "Ajawa", + "ajz": "Amri Karbi", + "ak": "Akan", + "akb": "Batak Angkola", + "akc": "Mpur", + "akd": "Ukpet-Ehom", + "ake": "Akawaio", + "akf": "Akpa", + "akg": "Anakalangu", + "akh": "Angal Heneng", + "aki": "Aiome", + "akj": "Aka-Jeru", + "akk": "Akkadian", + "akl": "Aklanon", + "akm": "Aka-Bo", + "ako": "Akurio", + "akp": "Siwu", + "akq": "Ak", + "akr": "Araki", + "aks": "Akaselem", + "akt": "Akolet", + "aku": "Akum", + "akv": "Akhvakh", + "akw": "Akwa", + "akx": "Aka-Kede", + "aky": "Aka-Kol", + "akz": "Alabama", + "ala": "Alago", + "alc": "Qawasqar", + "ald": "Alladian", + "ale": "Aleut", + "alf": "Alege", + "alg": "Algonquian languages", + "alh": "Alawa", + "ali": "Amaimon", + "alj": "Alangan", + "alk": "Alak", + "all": "Allar", + "alm": "Amblong", + "aln": "Gheg Albanian", + "alo": "Larike-Wakasihu", + "alp": "Alune", + "alq": "Algonquin", + "alr": "Alutor", + "als": "Tosk Albanian", + "alt": "Southern Altai", + "alu": "'Are'are", + "alv": "Atlantic-Congo languages", + "alw": "Alaba-K’abeena; Wanbasana", + "alx": "Amol", + "aly": "Alyawarr", + "alz": "Alur", + "am": "Amharic", + "ama": "Amanayé", + "amb": "Ambo", + "amc": "Amahuaca", + "ame": "Yanesha'", + "amf": "Hamer-Banna", + "amg": "Amurdak", + "ami": "Amis", + "amj": "Amdang", + "amk": "Ambai", + "aml": "War-Jaintia", + "amm": "Ama (Papua New Guinea)", + "amn": "Amanab", + "amo": "Amo", + "amp": "Alamblak", + "amq": "Amahai", + "amr": "Amarakaeri", + "ams": "Southern Amami-Oshima", + "amt": "Amto", + "amu": "Guerrero Amuzgo", + "amv": "Ambelau", + "amw": "Western Neo-Aramaic", + "amx": "Anmatyerre", + "amy": "Ami", + "amz": "Atampaya", + "an": "Aragonese", + "ana": "Andaqui", + "anb": "Andoa", + "anc": "Ngas", + "and": "Ansus", + "ane": "Xârâcùù", + "anf": "Animere", + "ang": "Old English (ca. 450-1100)", + "anh": "Nend", + "ani": "Andi", + "anj": "Anor", + "ank": "Goemai", + "anl": "Anu-Hkongso Chin", + "anm": "Anal", + "ann": "Obolo", + "ano": "Andoque", + "anp": "Angika", + "anq": "Jarawa (India)", + "anr": "Andh", + "ans": "Anserma", + "ant": "Antakarinya; Antikarinya", + "anu": "Anuak", + "anv": "Denya", + "anw": "Anaang", + "anx": "Andra-Hus", + "any": "Anyin", + "anz": "Anem", + "aoa": "Angolar", + "aob": "Abom", + "aoc": "Pemon", + "aod": "Andarum", + "aoe": "Angal Enen", + "aof": "Bragat", + "aog": "Angoram", + "aoi": "Anindilyakwa", + "aoj": "Mufian", + "aok": "Arhö", + "aol": "Alor", + "aom": "Ömie", + "aon": "Bumbita Arapesh", + "aor": "Aore", + "aos": "Taikat", + "aot": "Atong (India); A'tong", + "aou": "A'ou", + "aox": "Atorada", + "aoz": "Uab Meto", + "apa": "Apache languages", + "apb": "Sa'a", + "apc": "North Levantine Arabic", + "apd": "Sudanese Arabic", + "ape": "Bukiyip", + "apf": "Pahanan Agta", + "apg": "Ampanang", + "aph": "Athpariya", + "api": "Apiaká", + "apj": "Jicarilla Apache", + "apk": "Kiowa Apache", + "apl": "Lipan Apache", + "apm": "Mescalero-Chiricahua Apache", + "apn": "Apinayé", + "apo": "Ambul", + "app": "Apma", + "apq": "A-Pucikwar", + "apr": "Arop-Lokep", + "aps": "Arop-Sissano", + "apt": "Apatani", + "apu": "Apurinã", + "apv": "Alapmunte", + "apw": "Western Apache", + "apx": "Aputai", + "apy": "Apalaí", + "apz": "Safeyoka", + "aqa": "Alacalufan languages", + "aqc": "Archi", + "aqd": "Ampari Dogon", + "aqg": "Arigidi", + "aqk": "Aninka", + "aql": "Algic languages", + "aqm": "Atohwaim", + "aqn": "Northern Alta", + "aqp": "Atakapa", + "aqr": "Arhâ", + "aqt": "Angaité", + "aqz": "Akuntsu", + "ar": "Arabic", + "arb": "Standard Arabic", + "arc": "Official Aramaic (700-300 BCE); Imperial Aramaic (700-300 BCE)", + "ard": "Arabana", + "are": "Western Arrarnta", + "arh": "Arhuaco", + "ari": "Arikara", + "arj": "Arapaso", + "ark": "Arikapú", + "arl": "Arabela", + "arn": "Mapudungun; Mapuche", + "aro": "Araona", + "arp": "Arapaho", + "arq": "Algerian Arabic", + "arr": "Karo (Brazil)", + "ars": "Najdi Arabic", + "art": "Artificial languages", + "aru": "Aruá (Amazonas State); Arawá", + "arv": "Arbore", + "arw": "Arawak", + "arx": "Aruá (Rodonia State)", + "ary": "Moroccan Arabic", + "arz": "Egyptian Arabic", + "as": "Assamese", + "asa": "Asu (Tanzania)", + "asb": "Assiniboine", + "asc": "Casuarina Coast Asmat", + "ase": "American Sign Language", + "asf": "Auslan; Australian Sign Language", + "asg": "Cishingini", + "ash": "Abishira", + "asi": "Buruwai", + "asj": "Sari", + "ask": "Ashkun", + "asl": "Asilulu", + "asn": "Xingú Asuriní", + "aso": "Dano", + "asp": "Algerian Sign Language", + "asq": "Austrian Sign Language", + "asr": "Asuri", + "ass": "Ipulo", + "ast": "Asturian; Asturleonese; Bable; Leonese", + "asu": "Tocantins Asurini", + "asv": "Asoa", + "asw": "Australian Aborigines Sign Language", + "asx": "Muratayak", + "asy": "Yaosakor Asmat", + "asz": "As", + "ata": "Pele-Ata", + "atb": "Zaiwa", + "atc": "Atsahuaca", + "atd": "Ata Manobo", + "ate": "Atemble", + "atg": "Ivbie North-Okpela-Arhe", + "ath": "Athapascan languages", + "ati": "Attié", + "atj": "Atikamekw", + "atk": "Ati", + "atl": "Mt. Iraya Agta", + "atm": "Ata", + "atn": "Ashtiani", + "ato": "Atong (Cameroon)", + "atp": "Pudtol Atta", + "atq": "Aralle-Tabulahan", + "atr": "Waimiri-Atroari", + "ats": "Gros Ventre", + "att": "Pamplona Atta", + "atu": "Reel", + "atv": "Northern Altai", + "atw": "Atsugewi", + "atx": "Arutani", + "aty": "Aneityum", + "atz": "Arta", + "aua": "Asumboa", + "aub": "Alugu", + "auc": "Waorani", + "aud": "Anuta", + "auf": "Arauan languages", + "aug": "Aguna", + "auh": "Aushi", + "aui": "Anuki", + "auj": "Awjilah", + "auk": "Heyo", + "aul": "Aulua", + "aum": "Asu (Nigeria)", + "aun": "Molmo One", + "auo": "Auyokawa", + "aup": "Makayam", + "auq": "Anus; Korur", + "aur": "Aruek", + "aus": "Australian languages", + "aut": "Austral", + "auu": "Auye", + "auw": "Awyi", + "aux": "Aurá", + "auy": "Awiyaana", + "auz": "Uzbeki Arabic", + "av": "Avaric", + "avb": "Avau", + "avd": "Alviri-Vidari", + "avi": "Avikam", + "avk": "Kotava", + "avl": "Eastern Egyptian Bedawi Arabic", + "avm": "Angkamuthi", + "avn": "Avatime", + "avo": "Agavotaguerra", + "avs": "Aushiri", + "avt": "Au", + "avu": "Avokaya", + "avv": "Avá-Canoeiro", + "awa": "Awadhi", + "awb": "Awa (Papua New Guinea)", + "awc": "Cicipu", + "awd": "Arawakan languages", + "awe": "Awetí", + "awg": "Anguthimri", + "awh": "Awbono", + "awi": "Aekyom", + "awk": "Awabakal", + "awm": "Arawum", + "awn": "Awngi", + "awo": "Awak", + "awr": "Awera", + "aws": "South Awyu", + "awt": "Araweté", + "awu": "Central Awyu", + "awv": "Jair Awyu", + "aww": "Awun", + "awx": "Awara", + "awy": "Edera Awyu", + "axb": "Abipon", + "axe": "Ayerrerenge", + "axg": "Mato Grosso Arára", + "axk": "Yaka (Central African Republic)", + "axl": "Lower Southern Aranda", + "axm": "Middle Armenian", + "axx": "Xârâgurè", + "ay": "Aymara", + "aya": "Awar", + "ayb": "Ayizo Gbe", + "ayc": "Southern Aymara", + "ayd": "Ayabadhu", + "aye": "Ayere", + "ayg": "Ginyanga", + "ayh": "Hadrami Arabic", + "ayi": "Leyigha", + "ayk": "Akuku", + "ayl": "Libyan Arabic", + "ayn": "Sanaani Arabic", + "ayo": "Ayoreo", + "ayp": "North Mesopotamian Arabic", + "ayq": "Ayi (Papua New Guinea)", + "ayr": "Central Aymara", + "ays": "Sorsogon Ayta", + "ayt": "Magbukun Ayta", + "ayu": "Ayu", + "ayz": "Mai Brat", + "az": "Azerbaijani", + "aza": "Azha", + "azb": "South Azerbaijani", + "azc": "Uto-Aztecan languages", + "azd": "Eastern Durango Nahuatl", + "azg": "San Pedro Amuzgos Amuzgo", + "azj": "North Azerbaijani", + "azm": "Ipalapa Amuzgo", + "azn": "Western Durango Nahuatl", + "azo": "Awing", + "azt": "Faire Atta", + "azz": "Highland Puebla Nahuatl", + "ba": "Bashkir", + "baa": "Babatana", + "bab": "Bainouk-Gunyuño", + "bac": "Badui", + "bad": "Banda languages", + "bae": "Baré", + "baf": "Nubaca", + "bag": "Tuki", + "bah": "Bahamas Creole English", + "bai": "Bamileke languages", + "baj": "Barakai", + "bal": "Baluchi", + "ban": "Balinese", + "bao": "Waimaha", + "bap": "Bantawa", + "bar": "Bavarian", + "bas": "Basa (Cameroon)", + "bat": "Baltic languages", + "bau": "Bada (Nigeria)", + "bav": "Vengo", + "baw": "Bambili-Bambui", + "bax": "Bamun", + "bay": "Batuley", + "bba": "Baatonum", + "bbb": "Barai", + "bbc": "Batak Toba", + "bbd": "Bau", + "bbe": "Bangba", + "bbf": "Baibai", + "bbg": "Barama", + "bbh": "Bugan", + "bbi": "Barombi", + "bbj": "Ghomálá'", + "bbk": "Babanki", + "bbl": "Bats", + "bbm": "Babango", + "bbn": "Uneapa", + "bbo": "Northern Bobo Madaré; Konabéré", + "bbp": "West Central Banda", + "bbq": "Bamali", + "bbr": "Girawa", + "bbs": "Bakpinka", + "bbt": "Mburku", + "bbu": "Kulung (Nigeria)", + "bbv": "Karnai", + "bbw": "Baba", + "bbx": "Bubia", + "bby": "Befang", + "bca": "Central Bai", + "bcb": "Bainouk-Samik", + "bcc": "Southern Balochi", + "bcd": "North Babar", + "bce": "Bamenyam", + "bcf": "Bamu", + "bcg": "Baga Pokur", + "bch": "Bariai", + "bci": "Baoulé", + "bcj": "Bardi", + "bck": "Bunuba", + "bcl": "Central Bikol", + "bcm": "Bannoni", + "bcn": "Bali (Nigeria)", + "bco": "Kaluli", + "bcp": "Bali (Democratic Republic of Congo)", + "bcq": "Bench", + "bcr": "Babine", + "bcs": "Kohumono", + "bct": "Bendi", + "bcu": "Awad Bing", + "bcv": "Shoo-Minda-Nye", + "bcw": "Bana", + "bcy": "Bacama", + "bcz": "Bainouk-Gunyaamolo", + "bda": "Bayot", + "bdb": "Basap", + "bdc": "Emberá-Baudó", + "bdd": "Bunama", + "bde": "Bade", + "bdf": "Biage", + "bdg": "Bonggi", + "bdh": "Baka (South Sudan)", + "bdi": "Burun", + "bdj": "Bai (South Sudan); Bai", + "bdk": "Budukh", + "bdl": "Indonesian Bajau", + "bdm": "Buduma", + "bdn": "Baldemu", + "bdo": "Morom", + "bdp": "Bende", + "bdq": "Bahnar", + "bdr": "West Coast Bajau", + "bds": "Burunge", + "bdt": "Bokoto", + "bdu": "Oroko", + "bdv": "Bodo Parja", + "bdw": "Baham", + "bdx": "Budong-Budong", + "bdy": "Bandjalang", + "bdz": "Badeshi", + "be": "Belarusian", + "bea": "Beaver", + "beb": "Bebele", + "bec": "Iceve-Maci", + "bed": "Bedoanas", + "bee": "Byangsi", + "bef": "Benabena", + "beg": "Belait", + "beh": "Biali", + "bei": "Bekati'", + "bej": "Beja; Bedawiyet", + "bek": "Bebeli", + "bem": "Bemba (Zambia)", + "beo": "Beami", + "bep": "Besoa", + "beq": "Beembe", + "ber": "Berber languages", + "bes": "Besme", + "bet": "Guiberoua Béte", + "beu": "Blagar", + "bev": "Daloa Bété", + "bew": "Betawi", + "bex": "Jur Modo", + "bey": "Beli (Papua New Guinea)", + "bez": "Bena (Tanzania)", + "bfa": "Bari", + "bfb": "Pauri Bareli", + "bfc": "Panyi Bai; Northern Bai", + "bfd": "Bafut", + "bfe": "Betaf; Tena", + "bff": "Bofi", + "bfg": "Busang Kayan", + "bfh": "Blafe", + "bfi": "British Sign Language", + "bfj": "Bafanji", + "bfk": "Ban Khor Sign Language", + "bfl": "Banda-Ndélé", + "bfm": "Mmen", + "bfn": "Bunak", + "bfo": "Malba Birifor", + "bfp": "Beba", + "bfq": "Badaga", + "bfr": "Bazigar", + "bfs": "Southern Bai", + "bft": "Balti", + "bfu": "Gahri", + "bfw": "Bondo", + "bfx": "Bantayanon", + "bfy": "Bagheli", + "bfz": "Mahasu Pahari", + "bg": "Bulgarian", + "bga": "Gwamhi-Wuri", + "bgb": "Bobongko", + "bgc": "Haryanvi", + "bgd": "Rathwi Bareli", + "bge": "Bauria", + "bgf": "Bangandu", + "bgg": "Bugun", + "bgi": "Giangan", + "bgj": "Bangolan", + "bgk": "Bit; Buxinhua", + "bgl": "Bo (Laos)", + "bgn": "Western Balochi", + "bgo": "Baga Koga", + "bgp": "Eastern Balochi", + "bgq": "Bagri", + "bgr": "Bawm Chin", + "bgs": "Tagabawa", + "bgt": "Bughotu", + "bgu": "Mbongno", + "bgv": "Warkay-Bipim", + "bgw": "Bhatri", + "bgx": "Balkan Gagauz Turkish", + "bgy": "Benggoi", + "bgz": "Banggai", + "bh": "Bihari languages", + "bha": "Bharia", + "bhb": "Bhili", + "bhc": "Biga", + "bhd": "Bhadrawahi", + "bhe": "Bhaya", + "bhf": "Odiai", + "bhg": "Binandere", + "bhh": "Bukharic", + "bhi": "Bhilali", + "bhj": "Bahing", + "bhl": "Bimin", + "bhm": "Bathari", + "bhn": "Bohtan Neo-Aramaic", + "bho": "Bhojpuri", + "bhp": "Bima", + "bhq": "Tukang Besi South", + "bhr": "Bara Malagasy", + "bhs": "Buwal", + "bht": "Bhattiyali", + "bhu": "Bhunjia", + "bhv": "Bahau", + "bhw": "Biak", + "bhx": "Bhalay", + "bhy": "Bhele", + "bhz": "Bada (Indonesia)", + "bi": "Bislama", + "bia": "Badimaya", + "bib": "Bissa; Bisa", + "bid": "Bidiyo", + "bie": "Bepour", + "bif": "Biafada", + "big": "Biangai", + "bik": "Bikol", + "bil": "Bile", + "bim": "Bimoba", + "bin": "Bini; Edo", + "bio": "Nai", + "bip": "Bila", + "biq": "Bipi", + "bir": "Bisorio", + "bit": "Berinomo", + "biu": "Biete", + "biv": "Southern Birifor", + "biw": "Kol (Cameroon)", + "bix": "Bijori", + "biy": "Birhor", + "biz": "Baloi", + "bja": "Budza", + "bjb": "Banggarla", + "bjc": "Bariji", + "bje": "Biao-Jiao Mien", + "bjf": "Barzani Jewish Neo-Aramaic", + "bjg": "Bidyogo", + "bjh": "Bahinemo", + "bji": "Burji", + "bjj": "Kanauji", + "bjk": "Barok", + "bjl": "Bulu (Papua New Guinea)", + "bjm": "Bajelani", + "bjn": "Banjar", + "bjo": "Mid-Southern Banda", + "bjp": "Fanamaket", + "bjr": "Binumarien", + "bjs": "Bajan", + "bjt": "Balanta-Ganja", + "bju": "Busuu", + "bjv": "Bedjond", + "bjw": "Bakwé", + "bjx": "Banao Itneg", + "bjy": "Bayali", + "bjz": "Baruga", + "bka": "Kyak", + "bkc": "Baka (Cameroon)", + "bkd": "Binukid; Talaandig", + "bkf": "Beeke", + "bkg": "Buraka", + "bkh": "Bakoko", + "bki": "Baki", + "bkj": "Pande", + "bkk": "Brokskat", + "bkl": "Berik", + "bkm": "Kom (Cameroon)", + "bkn": "Bukitan", + "bko": "Kwa'", + "bkp": "Boko (Democratic Republic of Congo)", + "bkq": "Bakairí", + "bkr": "Bakumpai", + "bks": "Northern Sorsoganon", + "bkt": "Boloki", + "bku": "Buhid", + "bkv": "Bekwarra", + "bkw": "Bekwel", + "bkx": "Baikeno", + "bky": "Bokyi", + "bkz": "Bungku", + "bla": "Siksika", + "blb": "Bilua", + "blc": "Bella Coola", + "bld": "Bolango", + "ble": "Balanta-Kentohe", + "blf": "Buol", + "blh": "Kuwaa", + "bli": "Bolia", + "blj": "Bolongan", + "blk": "Pa'o Karen; Pa'O", + "bll": "Biloxi", + "blm": "Beli (South Sudan)", + "bln": "Southern Catanduanes Bikol", + "blo": "Anii", + "blp": "Blablanga", + "blq": "Baluan-Pam", + "blr": "Blang", + "bls": "Balaesang", + "blt": "Tai Dam", + "blv": "Kibala; Bolo", + "blw": "Balangao", + "blx": "Mag-Indi Ayta", + "bly": "Notre", + "blz": "Balantak", + "bm": "Bambara", + "bma": "Lame", + "bmb": "Bembe", + "bmc": "Biem", + "bmd": "Baga Manduri", + "bme": "Limassa", + "bmf": "Bom-Kim", + "bmg": "Bamwe", + "bmh": "Kein", + "bmi": "Bagirmi", + "bmj": "Bote-Majhi", + "bmk": "Ghayavi", + "bml": "Bomboli", + "bmm": "Northern Betsimisaraka Malagasy", + "bmn": "Bina (Papua New Guinea)", + "bmo": "Bambalang", + "bmp": "Bulgebi", + "bmq": "Bomu", + "bmr": "Muinane", + "bms": "Bilma Kanuri", + "bmt": "Biao Mon", + "bmu": "Somba-Siawari", + "bmv": "Bum", + "bmw": "Bomwali", + "bmx": "Baimak", + "bmz": "Baramu", + "bn": "Bengali; Bangla", + "bna": "Bonerate", + "bnb": "Bookan", + "bnc": "Bontok", + "bnd": "Banda (Indonesia)", + "bne": "Bintauna", + "bnf": "Masiwang", + "bng": "Benga", + "bni": "Bangi", + "bnj": "Eastern Tawbuid", + "bnk": "Bierebo", + "bnl": "Boon", + "bnm": "Batanga", + "bnn": "Bunun", + "bno": "Bantoanon", + "bnp": "Bola", + "bnq": "Bantik", + "bnr": "Butmas-Tur", + "bns": "Bundeli", + "bnt": "Bantu languages", + "bnu": "Bentong", + "bnv": "Bonerif; Beneraf; Edwas", + "bnw": "Bisis", + "bnx": "Bangubangu", + "bny": "Bintulu", + "bnz": "Beezen", + "bo": "Tibetan", + "boa": "Bora", + "bob": "Aweer", + "boe": "Mundabli", + "bof": "Bolon", + "bog": "Bamako Sign Language", + "boh": "Boma", + "boi": "Barbareño", + "boj": "Anjam", + "bok": "Bonjo", + "bol": "Bole", + "bom": "Berom", + "bon": "Bine", + "boo": "Tiemacèwè Bozo", + "bop": "Bonkiman", + "boq": "Bogaya", + "bor": "Borôro", + "bot": "Bongo", + "bou": "Bondei", + "bov": "Tuwuli", + "bow": "Rema", + "box": "Buamu", + "boy": "Bodo (Central African Republic)", + "boz": "Tiéyaxo Bozo", + "bpa": "Daakaka", + "bpc": "Mbuk", + "bpd": "Banda-Banda", + "bpe": "Bauni", + "bpg": "Bonggo", + "bph": "Botlikh", + "bpi": "Bagupi", + "bpj": "Binji", + "bpk": "Orowe; 'Ôrôê", + "bpl": "Broome Pearling Lugger Pidgin", + "bpm": "Biyom", + "bpn": "Dzao Min", + "bpo": "Anasi", + "bpp": "Kaure", + "bpq": "Banda Malay", + "bpr": "Koronadal Blaan", + "bps": "Sarangani Blaan", + "bpt": "Barrow Point", + "bpu": "Bongu", + "bpv": "Bian Marind", + "bpw": "Bo (Papua New Guinea)", + "bpx": "Palya Bareli", + "bpy": "Bishnupriya", + "bpz": "Bilba", + "bqa": "Tchumbuli", + "bqb": "Bagusa", + "bqc": "Boko (Benin); Boo", + "bqd": "Bung", + "bqf": "Baga Kaloum", + "bqg": "Bago-Kusuntu", + "bqh": "Baima", + "bqi": "Bakhtiari", + "bqj": "Bandial", + "bqk": "Banda-Mbrès", + "bql": "Bilakura", + "bqm": "Wumboko", + "bqn": "Bulgarian Sign Language", + "bqo": "Balo", + "bqp": "Busa", + "bqq": "Biritai", + "bqr": "Burusu", + "bqs": "Bosngun", + "bqt": "Bamukumbit", + "bqu": "Boguru", + "bqv": "Koro Wachi; Begbere-Ejar", + "bqw": "Buru (Nigeria)", + "bqx": "Baangi", + "bqy": "Bengkala Sign Language", + "bqz": "Bakaka", + "br": "Breton", + "bra": "Braj", + "brb": "Brao; Lave", + "brc": "Berbice Creole Dutch", + "brd": "Baraamu", + "brf": "Bira", + "brg": "Baure", + "brh": "Brahui", + "bri": "Mokpwe", + "brj": "Bieria", + "brk": "Birked", + "brl": "Birwa", + "brm": "Barambu", + "brn": "Boruca", + "bro": "Brokkat", + "brp": "Barapasi", + "brq": "Breri", + "brr": "Birao", + "brs": "Baras", + "brt": "Bitare", + "bru": "Eastern Bru", + "brv": "Western Bru", + "brw": "Bellari", + "brx": "Bodo (India)", + "bry": "Burui", + "brz": "Bilbil", + "bs": "Bosnian", + "bsa": "Abinomn", + "bsb": "Brunei Bisaya", + "bsc": "Bassari; Oniyan", + "bse": "Wushi", + "bsf": "Bauchi", + "bsg": "Bashkardi", + "bsh": "Kati", + "bsi": "Bassossi", + "bsj": "Bangwinji", + "bsk": "Burushaski", + "bsl": "Basa-Gumna", + "bsm": "Busami", + "bsn": "Barasana-Eduria", + "bso": "Buso", + "bsp": "Baga Sitemu", + "bsq": "Bassa", + "bsr": "Bassa-Kontagora", + "bss": "Akoose", + "bst": "Basketo", + "bsu": "Bahonsuai", + "bsv": "Baga Sobané", + "bsw": "Baiso", + "bsx": "Yangkam", + "bsy": "Sabah Bisaya", + "bta": "Bata", + "btc": "Bati (Cameroon)", + "btd": "Batak Dairi", + "bte": "Gamo-Ningi", + "btf": "Birgit", + "btg": "Gagnoa Bété", + "bth": "Biatah Bidayuh", + "bti": "Burate", + "btj": "Bacanese Malay", + "btk": "Batak languages", + "btm": "Batak Mandailing", + "btn": "Ratagnon", + "bto": "Rinconada Bikol", + "btp": "Budibud", + "btq": "Batek", + "btr": "Baetora", + "bts": "Batak Simalungun", + "btt": "Bete-Bendi", + "btu": "Batu", + "btv": "Bateri", + "btw": "Butuanon", + "btx": "Batak Karo", + "bty": "Bobot", + "btz": "Batak Alas-Kluet", + "bua": "Buriat", + "bub": "Bua", + "buc": "Bushi", + "bud": "Ntcham", + "bue": "Beothuk", + "buf": "Bushoong", + "bug": "Buginese", + "buh": "Younuo Bunu", + "bui": "Bongili", + "buj": "Basa-Gurmana", + "buk": "Bugawac", + "bum": "Bulu (Cameroon)", + "bun": "Sherbro", + "buo": "Terei", + "bup": "Busoa", + "buq": "Brem", + "bus": "Bokobaru", + "but": "Bungain", + "buu": "Budu", + "buv": "Bun", + "buw": "Bubi", + "bux": "Boghom", + "buy": "Bullom So", + "buz": "Bukwen", + "bva": "Barein", + "bvb": "Bube", + "bvc": "Baelelea", + "bvd": "Baeggu", + "bve": "Berau Malay", + "bvf": "Boor", + "bvg": "Bonkeng", + "bvh": "Bure", + "bvi": "Belanda Viri", + "bvj": "Baan", + "bvk": "Bukat", + "bvl": "Bolivian Sign Language", + "bvm": "Bamunka", + "bvn": "Buna", + "bvo": "Bolgo", + "bvp": "Bumang", + "bvq": "Birri", + "bvr": "Burarra", + "bvt": "Bati (Indonesia)", + "bvu": "Bukit Malay", + "bvv": "Baniva", + "bvw": "Boga", + "bvx": "Dibole", + "bvy": "Baybayanon", + "bvz": "Bauzi", + "bwa": "Bwatoo", + "bwb": "Namosi-Naitasiri-Serua", + "bwc": "Bwile", + "bwd": "Bwaidoka", + "bwe": "Bwe Karen", + "bwf": "Boselewa", + "bwg": "Barwe", + "bwh": "Bishuo", + "bwi": "Baniwa", + "bwj": "Láá Láá Bwamu", + "bwk": "Bauwaki", + "bwl": "Bwela", + "bwm": "Biwat", + "bwn": "Wunai Bunu", + "bwo": "Boro (Ethiopia); Borna (Ethiopia)", + "bwp": "Mandobo Bawah", + "bwq": "Southern Bobo Madaré", + "bwr": "Bura-Pabir", + "bws": "Bomboma", + "bwt": "Bafaw-Balong", + "bwu": "Buli (Ghana)", + "bww": "Bwa", + "bwx": "Bu-Nao Bunu", + "bwy": "Cwi Bwamu", + "bwz": "Bwisi", + "bxa": "Tairaha", + "bxb": "Belanda Bor", + "bxc": "Molengue", + "bxd": "Pela", + "bxe": "Birale", + "bxf": "Bilur; Minigir", + "bxg": "Bangala", + "bxh": "Buhutu", + "bxi": "Pirlatapa", + "bxj": "Bayungu", + "bxk": "Bukusu; Lubukusu", + "bxl": "Jalkunan", + "bxm": "Mongolia Buriat", + "bxn": "Burduna", + "bxo": "Barikanchi", + "bxp": "Bebil", + "bxq": "Beele", + "bxr": "Russia Buriat", + "bxs": "Busam", + "bxu": "China Buriat", + "bxv": "Berakou", + "bxw": "Bankagooma", + "bxz": "Binahari", + "bya": "Batak", + "byb": "Bikya", + "byc": "Ubaghara", + "byd": "Benyadu'", + "bye": "Pouye", + "byf": "Bete", + "byg": "Baygo", + "byh": "Bhujel", + "byi": "Buyu", + "byj": "Bina (Nigeria)", + "byk": "Biao", + "byl": "Bayono", + "bym": "Bidjara", + "byn": "Bilin; Blin", + "byo": "Biyo", + "byp": "Bumaji", + "byq": "Basay", + "byr": "Baruya; Yipma", + "bys": "Burak", + "byt": "Berti", + "byv": "Medumba", + "byw": "Belhariya", + "byx": "Qaqet", + "byz": "Banaro", + "bza": "Bandi", + "bzb": "Andio", + "bzc": "Southern Betsimisaraka Malagasy", + "bzd": "Bribri", + "bze": "Jenaama Bozo", + "bzf": "Boikin", + "bzg": "Babuza", + "bzh": "Mapos Buang", + "bzi": "Bisu", + "bzj": "Belize Kriol English", + "bzk": "Nicaragua Creole English", + "bzl": "Boano (Sulawesi)", + "bzm": "Bolondo", + "bzn": "Boano (Maluku)", + "bzo": "Bozaba", + "bzp": "Kemberano", + "bzq": "Buli (Indonesia)", + "bzr": "Biri", + "bzs": "Brazilian Sign Language", + "bzt": "Brithenig", + "bzu": "Burmeso", + "bzv": "Naami", + "bzw": "Basa (Nigeria)", + "bzx": "Kɛlɛngaxo Bozo", + "bzy": "Obanliku", + "bzz": "Evant", + "ca": "Catalan; Valencian", + "caa": "Chortí", + "cab": "Garifuna", + "cac": "Chuj", + "cad": "Caddo", + "cae": "Lehar; Laalaa", + "caf": "Southern Carrier", + "cag": "Nivaclé", + "cah": "Cahuarano", + "cai": "Central American Indian languages", + "caj": "Chané", + "cak": "Kaqchikel; Cakchiquel", + "cal": "Carolinian", + "cam": "Cemuhî", + "can": "Chambri", + "cao": "Chácobo", + "cap": "Chipaya", + "caq": "Car Nicobarese", + "car": "Galibi Carib", + "cas": "Tsimané", + "cau": "Caucasian languages", + "cav": "Cavineña", + "caw": "Callawalla", + "cax": "Chiquitano", + "cay": "Cayuga", + "caz": "Canichana", + "cba": "Chibchan languages", + "cbb": "Cabiyarí", + "cbc": "Carapana", + "cbd": "Carijona", + "cbg": "Chimila", + "cbi": "Chachi", + "cbj": "Ede Cabe", + "cbk": "Chavacano", + "cbl": "Bualkhaw Chin", + "cbn": "Nyahkur", + "cbo": "Izora", + "cbq": "Tsucuba; Cuba", + "cbr": "Cashibo-Cacataibo", + "cbs": "Cashinahua", + "cbt": "Chayahuita", + "cbu": "Candoshi-Shapra", + "cbv": "Cacua", + "cbw": "Kinabalian", + "cby": "Carabayo", + "ccc": "Chamicuro", + "ccd": "Cafundo Creole", + "cce": "Chopi", + "ccg": "Samba Daka", + "cch": "Atsam", + "ccj": "Kasanga", + "ccl": "Cutchi-Swahili", + "ccm": "Malaccan Creole Malay", + "ccn": "North Caucasian languages", + "cco": "Comaltepec Chinantec", + "ccp": "Chakma", + "ccr": "Cacaopera", + "ccs": "South Caucasian languages", + "cda": "Choni", + "cdc": "Chadic languages", + "cdd": "Caddoan languages", + "cde": "Chenchu", + "cdf": "Chiru", + "cdh": "Chambeali", + "cdi": "Chodri", + "cdj": "Churahi", + "cdm": "Chepang", + "cdn": "Chaudangsi", + "cdo": "Min Dong Chinese", + "cdr": "Cinda-Regi-Tiyal", + "cds": "Chadian Sign Language", + "cdy": "Chadong", + "cdz": "Koda", + "ce": "Chechen", + "cea": "Lower Chehalis", + "ceb": "Cebuano", + "ceg": "Chamacoco", + "cek": "Eastern Khumi Chin", + "cel": "Celtic languages", + "cen": "Cen", + "cet": "Centúúm", + "cey": "Ekai Chin", + "cfa": "Dijim-Bwilim", + "cfd": "Cara", + "cfg": "Como Karim", + "cfm": "Falam Chin", + "cga": "Changriwa", + "cgc": "Kagayanen", + "cgg": "Chiga", + "cgk": "Chocangacakha", + "ch": "Chamorro", + "chb": "Chibcha", + "chc": "Catawba", + "chd": "Highland Oaxaca Chontal", + "chf": "Tabasco Chontal", + "chg": "Chagatai", + "chh": "Chinook", + "chj": "Ojitlán Chinantec", + "chk": "Chuukese", + "chl": "Cahuilla", + "chm": "Mari (Russia)", + "chn": "Chinook jargon", + "cho": "Choctaw", + "chp": "Chipewyan; Dene Suline", + "chq": "Quiotepec Chinantec", + "chr": "Cherokee", + "cht": "Cholón", + "chw": "Chuwabu", + "chx": "Chantyal", + "chy": "Cheyenne", + "chz": "Ozumacín Chinantec", + "cia": "Cia-Cia", + "cib": "Ci Gbe", + "cic": "Chickasaw", + "cid": "Chimariko", + "cie": "Cineni", + "cih": "Chinali", + "cik": "Chitkuli Kinnauri", + "cim": "Cimbrian", + "cin": "Cinta Larga", + "cip": "Chiapanec", + "cir": "Tiri; Haméa; Méa", + "ciw": "Chippewa", + "ciy": "Chaima", + "cja": "Western Cham", + "cje": "Chru", + "cjh": "Upper Chehalis", + "cji": "Chamalal", + "cjk": "Chokwe", + "cjm": "Eastern Cham", + "cjn": "Chenapian", + "cjo": "Ashéninka Pajonal", + "cjp": "Cabécar", + "cjs": "Shor", + "cjv": "Chuave", + "cjy": "Jinyu Chinese", + "ckb": "Central Kurdish", + "ckh": "Chak", + "ckl": "Cibak", + "ckm": "Chakavian", + "ckn": "Kaang Chin", + "cko": "Anufo", + "ckq": "Kajakse", + "ckr": "Kairak", + "cks": "Tayo", + "ckt": "Chukot", + "cku": "Koasati", + "ckv": "Kavalan", + "ckx": "Caka", + "cky": "Cakfem-Mushere", + "ckz": "Cakchiquel-Quiché Mixed Language", + "cla": "Ron", + "clc": "Chilcotin", + "cld": "Chaldean Neo-Aramaic", + "cle": "Lealao Chinantec", + "clh": "Chilisso", + "cli": "Chakali", + "clj": "Laitu Chin", + "clk": "Idu-Mishmi", + "cll": "Chala", + "clm": "Clallam", + "clo": "Lowland Oaxaca Chontal", + "clt": "Lautu Chin", + "clu": "Caluyanun", + "clw": "Chulym", + "cly": "Eastern Highland Chatino", + "cma": "Maa", + "cmc": "Chamic languages", + "cme": "Cerma", + "cmg": "Classical Mongolian", + "cmi": "Emberá-Chamí", + "cml": "Campalagian", + "cmm": "Michigamea", + "cmn": "Mandarin Chinese", + "cmo": "Central Mnong", + "cmr": "Mro-Khimi Chin", + "cms": "Messapic", + "cmt": "Camtho", + "cna": "Changthang", + "cnb": "Chinbon Chin", + "cnc": "Côông", + "cng": "Northern Qiang", + "cnh": "Hakha Chin; Haka Chin", + "cni": "Asháninka", + "cnk": "Khumi Chin", + "cnl": "Lalana Chinantec", + "cno": "Con", + "cnp": "Northern Ping Chinese; Northern Pinghua", + "cnq": "Chung", + "cnr": "Montenegrin", + "cns": "Central Asmat", + "cnt": "Tepetotutla Chinantec", + "cnu": "Chenoua", + "cnw": "Ngawn Chin", + "cnx": "Middle Cornish", + "co": "Corsican", + "coa": "Cocos Islands Malay", + "cob": "Chicomuceltec", + "coc": "Cocopa", + "cod": "Cocama-Cocamilla", + "coe": "Koreguaje", + "cof": "Colorado", + "cog": "Chong", + "coh": "Chonyi-Dzihana-Kauma; Chichonyi-Chidzihana-Chikauma", + "coj": "Cochimi", + "cok": "Santa Teresa Cora", + "col": "Columbia-Wenatchi", + "com": "Comanche", + "con": "Cofán", + "coo": "Comox", + "cop": "Coptic", + "coq": "Coquille", + "cot": "Caquinte", + "cou": "Wamey", + "cov": "Cao Miao", + "cow": "Cowlitz", + "cox": "Nanti", + "coz": "Chochotec", + "cpa": "Palantla Chinantec", + "cpb": "Ucayali-Yurúa Ashéninka", + "cpc": "Ajyíninka Apurucayali", + "cpe": "English-based creoles and pidgins", + "cpf": "French-based creoles and pidgins", + "cpg": "Cappadocian Greek", + "cpi": "Chinese Pidgin English", + "cpn": "Cherepon", + "cpo": "Kpeego", + "cpp": "Portuguese-based creoles and pidgins", + "cps": "Capiznon", + "cpu": "Pichis Ashéninka", + "cpx": "Pu-Xian Chinese", + "cpy": "South Ucayali Ashéninka", + "cqd": "Chuanqiandian Cluster Miao", + "cr": "Cree", + "cra": "Chara", + "crb": "Island Carib", + "crc": "Lonwolwol", + "crd": "Coeur d'Alene", + "crf": "Caramanta", + "crg": "Michif", + "crh": "Crimean Tatar; Crimean Turkish", + "cri": "Sãotomense", + "crj": "Southern East Cree", + "crk": "Plains Cree", + "crl": "Northern East Cree", + "crm": "Moose Cree", + "crn": "El Nayar Cora", + "cro": "Crow", + "crp": "Creoles and pidgins", + "crq": "Iyo'wujwa Chorote", + "crr": "Carolina Algonquian", + "crs": "Seselwa Creole French", + "crt": "Iyojwa'ja Chorote", + "crv": "Chaura", + "crw": "Chrau", + "crx": "Carrier", + "cry": "Cori", + "crz": "Cruzeño", + "cs": "Czech", + "csa": "Chiltepec Chinantec", + "csb": "Kashubian", + "csc": "Catalan Sign Language; Lengua de señas catalana; Llengua de Signes Catalana", + "csd": "Chiangmai Sign Language", + "cse": "Czech Sign Language", + "csf": "Cuba Sign Language", + "csg": "Chilean Sign Language", + "csh": "Asho Chin", + "csi": "Coast Miwok", + "csj": "Songlai Chin", + "csk": "Jola-Kasa", + "csl": "Chinese Sign Language", + "csm": "Central Sierra Miwok", + "csn": "Colombian Sign Language", + "cso": "Sochiapam Chinantec; Sochiapan Chinantec", + "csp": "Southern Ping Chinese; Southern Pinghua", + "csq": "Croatia Sign Language", + "csr": "Costa Rican Sign Language", + "css": "Southern Ohlone", + "cst": "Northern Ohlone", + "csu": "Central Sudanic languages", + "csv": "Sumtu Chin", + "csw": "Swampy Cree", + "csx": "Cambodian Sign Language", + "csy": "Siyin Chin", + "csz": "Coos", + "cta": "Tataltepec Chatino", + "ctc": "Chetco", + "ctd": "Tedim Chin", + "cte": "Tepinapa Chinantec", + "ctg": "Chittagonian", + "cth": "Thaiphum Chin", + "ctl": "Tlacoatzintepec Chinantec", + "ctm": "Chitimacha", + "ctn": "Chhintange", + "cto": "Emberá-Catío", + "ctp": "Western Highland Chatino", + "cts": "Northern Catanduanes Bikol", + "ctt": "Wayanad Chetti", + "ctu": "Chol", + "cty": "Moundadan Chetty", + "ctz": "Zacatepec Chatino", + "cu": "Church Slavic; Church Slavonic; Old Bulgarian; Old Church Slavonic; Old Slavonic", + "cua": "Cua", + "cub": "Cubeo", + "cuc": "Usila Chinantec", + "cuh": "Chuka; Gichuka", + "cui": "Cuiba", + "cuj": "Mashco Piro", + "cuk": "San Blas Kuna", + "cul": "Culina; Kulina", + "cuo": "Cumanagoto", + "cup": "Cupeño", + "cuq": "Cun", + "cur": "Chhulung", + "cus": "Cushitic languages", + "cut": "Teutila Cuicatec", + "cuu": "Tai Ya", + "cuv": "Cuvok", + "cuw": "Chukwa", + "cux": "Tepeuxila Cuicatec", + "cuy": "Cuitlatec", + "cv": "Chuvash", + "cvg": "Chug", + "cvn": "Valle Nacional Chinantec", + "cwa": "Kabwa", + "cwb": "Maindo", + "cwd": "Woods Cree", + "cwe": "Kwere", + "cwg": "Chewong; Cheq Wong", + "cwt": "Kuwaataay", + "cy": "Welsh", + "cya": "Nopala Chatino", + "cyb": "Cayubaba", + "cyo": "Cuyonon", + "czh": "Huizhou Chinese", + "czk": "Knaanic", + "czn": "Zenzontepec Chatino", + "czo": "Min Zhong Chinese", + "czt": "Zotung Chin", + "da": "Danish", + "daa": "Dangaléat", + "dac": "Dambi", + "dad": "Marik", + "dae": "Duupa", + "dag": "Dagbani", + "dah": "Gwahatike", + "dai": "Day", + "daj": "Dar Fur Daju", + "dak": "Dakota", + "dal": "Dahalo", + "dam": "Damakawa", + "dao": "Daai Chin", + "daq": "Dandami Maria", + "dar": "Dargwa", + "das": "Daho-Doo", + "dau": "Dar Sila Daju", + "dav": "Taita; Dawida", + "daw": "Davawenyo", + "dax": "Dayi", + "day": "Land Dayak languages", + "daz": "Dao", + "dba": "Bangime", + "dbb": "Deno", + "dbd": "Dadiya", + "dbe": "Dabe", + "dbf": "Edopi", + "dbg": "Dogul Dom Dogon", + "dbi": "Doka", + "dbj": "Ida'an", + "dbl": "Dyirbal", + "dbm": "Duguri", + "dbn": "Duriankere", + "dbo": "Dulbu", + "dbp": "Duwai", + "dbq": "Daba", + "dbr": "Dabarre", + "dbt": "Ben Tey Dogon", + "dbu": "Bondum Dom Dogon", + "dbv": "Dungu", + "dbw": "Bankan Tey Dogon", + "dby": "Dibiyaso", + "dcc": "Deccan", + "dcr": "Negerhollands", + "dda": "Dadi Dadi", + "ddd": "Dongotono", + "dde": "Doondo", + "ddg": "Fataluku", + "ddi": "West Goodenough", + "ddj": "Jaru", + "ddn": "Dendi (Benin)", + "ddo": "Dido", + "ddr": "Dhudhuroa", + "dds": "Donno So Dogon", + "ddw": "Dawera-Daweloor", + "de": "German", + "dec": "Dagik", + "ded": "Dedua", + "dee": "Dewoin", + "def": "Dezfuli", + "deg": "Degema", + "deh": "Dehwari", + "dei": "Demisa", + "dek": "Dek", + "del": "Delaware", + "dem": "Dem", + "den": "Slave (Athapascan)", + "dep": "Pidgin Delaware", + "deq": "Dendi (Central African Republic)", + "der": "Deori", + "des": "Desano", + "dev": "Domung", + "dez": "Dengese", + "dga": "Southern Dagaare", + "dgb": "Bunoge Dogon", + "dgc": "Casiguran Dumagat Agta", + "dgd": "Dagaari Dioula", + "dge": "Degenan", + "dgg": "Doga", + "dgh": "Dghwede", + "dgi": "Northern Dagara", + "dgk": "Dagba", + "dgl": "Andaandi; Dongolawi", + "dgn": "Dagoman", + "dgo": "Dogri (individual language)", + "dgr": "Dogrib; Tłı̨chǫ", + "dgs": "Dogoso", + "dgt": "Ndra'ngith", + "dgw": "Daungwurrung", + "dgx": "Doghoro", + "dgz": "Daga", + "dhd": "Dhundari", + "dhg": "Dhangu-Djangu; Dhangu; Djangu", + "dhi": "Dhimal", + "dhl": "Dhalandji", + "dhm": "Zemba", + "dhn": "Dhanki", + "dho": "Dhodia", + "dhr": "Dhargari", + "dhs": "Dhaiso", + "dhu": "Dhurga", + "dhv": "Dehu; Drehu", + "dhw": "Dhanwar (Nepal)", + "dhx": "Dhungaloo", + "dia": "Dia", + "dib": "South Central Dinka", + "dic": "Lakota Dida", + "did": "Didinga", + "dif": "Dieri; Diyari", + "dig": "Digo; Chidigo", + "dih": "Kumiai", + "dii": "Dimbong", + "dij": "Dai", + "dik": "Southwestern Dinka", + "dil": "Dilling", + "dim": "Dime", + "din": "Dinka", + "dio": "Dibo", + "dip": "Northeastern Dinka", + "diq": "Dimli (individual language)", + "dir": "Dirim", + "dis": "Dimasa", + "diu": "Diriku", + "diw": "Northwestern Dinka", + "dix": "Dixon Reef", + "diy": "Diuwe", + "diz": "Ding", + "dja": "Djadjawurrung", + "djb": "Djinba", + "djc": "Dar Daju Daju", + "djd": "Djamindjung; Ngaliwurru", + "dje": "Zarma", + "djf": "Djangun", + "dji": "Djinang", + "djj": "Djeebbana", + "djk": "Eastern Maroon Creole; Businenge Tongo; Nenge", + "djm": "Jamsay Dogon", + "djn": "Jawoyn; Djauan", + "djo": "Jangkang", + "djr": "Djambarrpuyngu", + "dju": "Kapriman", + "djw": "Djawi", + "dka": "Dakpakha", + "dkg": "Kadung", + "dkk": "Dakka", + "dkr": "Kuijau", + "dks": "Southeastern Dinka", + "dkx": "Mazagway", + "dlg": "Dolgan", + "dlk": "Dahalik", + "dlm": "Dalmatian", + "dln": "Darlong", + "dma": "Duma", + "dmb": "Mombo Dogon", + "dmc": "Gavak", + "dmd": "Madhi Madhi", + "dme": "Dugwor", + "dmf": "Medefaidrin", + "dmg": "Upper Kinabatangan", + "dmk": "Domaaki", + "dml": "Dameli", + "dmm": "Dama", + "dmn": "Mande languages", + "dmo": "Kemedzung", + "dmr": "East Damar", + "dms": "Dampelas", + "dmu": "Dubu; Tebi", + "dmv": "Dumpas", + "dmw": "Mudburra", + "dmx": "Dema", + "dmy": "Demta; Sowari", + "dna": "Upper Grand Valley Dani", + "dnd": "Daonda", + "dne": "Ndendeule", + "dng": "Dungan", + "dni": "Lower Grand Valley Dani", + "dnj": "Dan", + "dnk": "Dengka", + "dnn": "Dzùùngoo", + "dno": "Ndrulo; Northern Lendu", + "dnr": "Danaru", + "dnt": "Mid Grand Valley Dani", + "dnu": "Danau", + "dnv": "Danu", + "dnw": "Western Dani", + "dny": "Dení", + "doa": "Dom", + "dob": "Dobu", + "doc": "Northern Dong", + "doe": "Doe", + "dof": "Domu", + "doh": "Dong", + "doi": "Dogri (macrolanguage)", + "dok": "Dondo", + "dol": "Doso", + "don": "Toura (Papua New Guinea)", + "doo": "Dongo", + "dop": "Lukpa", + "doq": "Dominican Sign Language", + "dor": "Dori'o", + "dos": "Dogosé", + "dot": "Dass", + "dov": "Dombe", + "dow": "Doyayo", + "dox": "Bussa", + "doy": "Dompo", + "doz": "Dorze", + "dpp": "Papar", + "dra": "Dravidian languages", + "drb": "Dair", + "drc": "Minderico", + "drd": "Darmiya", + "dre": "Dolpo", + "drg": "Rungus", + "dri": "C'Lela", + "drl": "Paakantyi", + "drn": "West Damar", + "dro": "Daro-Matu Melanau", + "drq": "Dura", + "drs": "Gedeo", + "drt": "Drents", + "dru": "Rukai", + "dry": "Darai", + "dsb": "Lower Sorbian", + "dse": "Dutch Sign Language", + "dsh": "Daasanach", + "dsi": "Disa", + "dsl": "Danish Sign Language", + "dsn": "Dusner", + "dso": "Desiya", + "dsq": "Tadaksahak", + "dsz": "Mardin Sign Language", + "dta": "Daur", + "dtb": "Labuk-Kinabatangan Kadazan", + "dtd": "Ditidaht", + "dth": "Adithinngithigh", + "dti": "Ana Tinga Dogon", + "dtk": "Tene Kan Dogon", + "dtm": "Tomo Kan Dogon", + "dtn": "Daatsʼíin", + "dto": "Tommo So Dogon", + "dtp": "Kadazan Dusun; Central Dusun", + "dtr": "Lotud", + "dts": "Toro So Dogon", + "dtt": "Toro Tegu Dogon", + "dtu": "Tebul Ure Dogon", + "dty": "Dotyali", + "dua": "Duala", + "dub": "Dubli", + "duc": "Duna", + "due": "Umiray Dumaget Agta", + "duf": "Dumbea; Drubea", + "dug": "Duruma; Chiduruma", + "duh": "Dungra Bhil", + "dui": "Dumun", + "duk": "Uyajitaya", + "dul": "Alabat Island Agta", + "dum": "Middle Dutch (ca. 1050-1350)", + "dun": "Dusun Deyah", + "duo": "Dupaninan Agta", + "dup": "Duano", + "duq": "Dusun Malang", + "dur": "Dii", + "dus": "Dumi", + "duu": "Drung", + "duv": "Duvle", + "duw": "Dusun Witu", + "dux": "Duungooma", + "duy": "Dicamay Agta", + "duz": "Duli-Gey", + "dv": "Dhivehi; Divehi; Maldivian", + "dva": "Duau", + "dwa": "Diri", + "dwk": "Dawik Kui", + "dwr": "Dawro", + "dws": "Dutton World Speedwords", + "dwu": "Dhuwal", + "dww": "Dawawa", + "dwy": "Dhuwaya", + "dwz": "Dewas Rai", + "dya": "Dyan", + "dyb": "Dyaberdyaber", + "dyd": "Dyugun", + "dyg": "Villa Viciosa Agta", + "dyi": "Djimini Senoufo", + "dym": "Yanda Dom Dogon", + "dyn": "Dyangadi; Dhanggatti", + "dyo": "Jola-Fonyi", + "dyu": "Dyula", + "dyy": "Djabugay; Dyaabugay", + "dz": "Dzongkha", + "dza": "Tunzu", + "dze": "Djiwarli", + "dzg": "Dazaga", + "dzl": "Dzalakha", + "dzn": "Dzando", + "eaa": "Karenggapa", + "ebc": "Beginci", + "ebg": "Ebughu", + "ebk": "Eastern Bontok", + "ebo": "Teke-Ebo", + "ebr": "Ebrié", + "ebu": "Embu; Kiembu", + "ecr": "Eteocretan", + "ecs": "Ecuadorian Sign Language", + "ecy": "Eteocypriot", + "ee": "Ewe", + "eee": "E", + "efa": "Efai", + "efe": "Efe", + "efi": "Efik", + "ega": "Ega", + "egl": "Emilian", + "egm": "Benamanga", + "ego": "Eggon", + "egx": "Egyptian languages", + "egy": "Egyptian (Ancient)", + "ehs": "Miyakubo Sign Language", + "ehu": "Ehueun", + "eip": "Eipomek", + "eit": "Eitiep", + "eiv": "Askopan", + "eja": "Ejamat", + "eka": "Ekajuk", + "eke": "Ekit", + "ekg": "Ekari", + "eki": "Eki", + "ekk": "Standard Estonian", + "ekl": "Kol (Bangladesh); Kol", + "ekm": "Elip", + "eko": "Koti", + "ekp": "Ekpeye", + "ekr": "Yace", + "eky": "Eastern Kayah", + "el": "Modern Greek (1453-)", + "ele": "Elepi", + "elh": "El Hugeirat", + "eli": "Nding", + "elk": "Elkei", + "elm": "Eleme", + "elo": "El Molo", + "elu": "Elu", + "elx": "Elamite", + "ema": "Emai-Iuleha-Ora", + "emb": "Embaloh", + "eme": "Emerillon", + "emg": "Eastern Meohang", + "emi": "Mussau-Emira", + "emk": "Eastern Maninkakan", + "emm": "Mamulique", + "emn": "Eman", + "emp": "Northern Emberá", + "emq": "Eastern Minyag", + "ems": "Pacific Gulf Yupik", + "emu": "Eastern Muria", + "emw": "Emplawas", + "emx": "Erromintxela", + "emy": "Epigraphic Mayan", + "emz": "Mbessa", + "en": "English", + "ena": "Apali", + "enb": "Markweeta", + "enc": "En", + "end": "Ende", + "enf": "Forest Enets", + "enh": "Tundra Enets", + "enl": "Enlhet", + "enm": "Middle English (1100-1500)", + "enn": "Engenni", + "eno": "Enggano", + "enq": "Enga", + "enr": "Emumu; Emem", + "enu": "Enu", + "env": "Enwan (Edo State)", + "enw": "Enwan (Akwa Ibom State)", + "enx": "Enxet", + "eo": "Esperanto", + "eot": "Beti (Côte d'Ivoire)", + "epi": "Epie", + "era": "Eravallan", + "erg": "Sie", + "erh": "Eruwa", + "eri": "Ogea", + "erk": "South Efate", + "ero": "Horpa", + "err": "Erre", + "ers": "Ersu", + "ert": "Eritai", + "erw": "Erokwanas", + "es": "Spanish; Castilian", + "ese": "Ese Ejja", + "esg": "Aheri Gondi", + "esh": "Eshtehardi", + "esi": "North Alaskan Inupiatun", + "esk": "Northwest Alaska Inupiatun", + "esl": "Egypt Sign Language", + "esm": "Esuma", + "esn": "Salvadoran Sign Language", + "eso": "Estonian Sign Language", + "esq": "Esselen", + "ess": "Central Siberian Yupik", + "esu": "Central Yupik", + "esx": "Eskimo-Aleut languages", + "esy": "Eskayan", + "et": "Estonian", + "etb": "Etebi", + "etc": "Etchemin", + "eth": "Ethiopian Sign Language", + "etn": "Eton (Vanuatu)", + "eto": "Eton (Cameroon)", + "etr": "Edolo", + "ets": "Yekhee", + "ett": "Etruscan", + "etu": "Ejagham", + "etx": "Eten", + "etz": "Semimi", + "eu": "Basque", + "euq": "Basque (family)", + "eve": "Even", + "evh": "Uvbie", + "evn": "Evenki", + "ewo": "Ewondo", + "ext": "Extremaduran", + "eya": "Eyak", + "eyo": "Keiyo", + "eza": "Ezaa", + "eze": "Uzekwe", + "fa": "Persian", + "faa": "Fasu", + "fab": "Fa d'Ambu", + "fad": "Wagi", + "faf": "Fagani", + "fag": "Finongan", + "fah": "Baissa Fali", + "fai": "Faiwol", + "faj": "Faita", + "fak": "Fang (Cameroon)", + "fal": "South Fali", + "fam": "Fam", + "fan": "Fang (Equatorial Guinea)", + "fap": "Paloor", + "far": "Fataleka", + "fat": "Fanti", + "fau": "Fayu", + "fax": "Fala", + "fay": "Southwestern Fars", + "faz": "Northwestern Fars", + "fbl": "West Albay Bikol", + "fcs": "Quebec Sign Language", + "fer": "Feroge", + "ff": "Fulah", + "ffi": "Foia Foia", + "ffm": "Maasina Fulfulde", + "fgr": "Fongoro", + "fi": "Finnish", + "fia": "Nobiin", + "fie": "Fyer", + "fif": "Faifi", + "fil": "Filipino; Pilipino", + "fip": "Fipa", + "fir": "Firan", + "fit": "Tornedalen Finnish; Meänkieli", + "fiu": "Finno-Ugrian languages", + "fiw": "Fiwaga", + "fj": "Fijian", + "fkk": "Kirya-Konzəl", + "fkv": "Kven Finnish", + "fla": "Kalispel-Pend d'Oreille", + "flh": "Foau", + "fli": "Fali", + "fll": "North Fali", + "fln": "Flinders Island", + "flr": "Fuliiru", + "fly": "Flaaitaal; Tsotsitaal", + "fmp": "Fe'fe'", + "fmu": "Far Western Muria", + "fnb": "Fanbak", + "fng": "Fanagalo", + "fni": "Fania", + "fo": "Faroese", + "fod": "Foodo", + "foi": "Foi", + "fom": "Foma", + "fon": "Fon", + "for": "Fore", + "fos": "Siraya", + "fox": "Formosan languages", + "fpe": "Fernando Po Creole English", + "fqs": "Fas", + "fr": "French", + "frc": "Cajun French", + "frd": "Fordata", + "frk": "Frankish", + "frm": "Middle French (ca. 1400-1600)", + "fro": "Old French (842-ca. 1400)", + "frp": "Arpitan; Francoprovençal", + "frq": "Forak", + "frr": "Northern Frisian", + "frs": "Eastern Frisian", + "frt": "Fortsenal", + "fse": "Finnish Sign Language", + "fsl": "French Sign Language", + "fss": "Finland-Swedish Sign Language; finlandssvenskt teckenspråk; suomenruotsalainen viittomakieli", + "fub": "Adamawa Fulfulde", + "fuc": "Pulaar", + "fud": "East Futuna", + "fue": "Borgu Fulfulde", + "fuf": "Pular", + "fuh": "Western Niger Fulfulde", + "fui": "Bagirmi Fulfulde", + "fuj": "Ko", + "fum": "Fum", + "fun": "Fulniô", + "fuq": "Central-Eastern Niger Fulfulde", + "fur": "Friulian", + "fut": "Futuna-Aniwa", + "fuu": "Furu", + "fuv": "Nigerian Fulfulde", + "fuy": "Fuyug", + "fvr": "Fur", + "fwa": "Fwâi", + "fwe": "Fwe", + "fy": "Western Frisian", + "ga": "Irish", + "gaa": "Ga", + "gab": "Gabri", + "gac": "Mixed Great Andamanese", + "gad": "Gaddang", + "gae": "Guarequena", + "gaf": "Gende", + "gag": "Gagauz", + "gah": "Alekano", + "gai": "Borei", + "gaj": "Gadsup", + "gak": "Gamkonora", + "gal": "Galolen", + "gam": "Kandawo", + "gan": "Gan Chinese", + "gao": "Gants", + "gap": "Gal", + "gaq": "Gata'", + "gar": "Galeya", + "gas": "Adiwasi Garasia", + "gat": "Kenati", + "gau": "Mudhili Gadaba", + "gaw": "Nobonob", + "gax": "Borana-Arsi-Guji Oromo", + "gay": "Gayo", + "gaz": "West Central Oromo", + "gba": "Gbaya (Central African Republic)", + "gbb": "Kaytetye", + "gbd": "Karajarri", + "gbe": "Niksek", + "gbf": "Gaikundi", + "gbg": "Gbanziri", + "gbh": "Defi Gbe", + "gbi": "Galela", + "gbj": "Bodo Gadaba", + "gbk": "Gaddi", + "gbl": "Gamit", + "gbm": "Garhwali", + "gbn": "Mo'da", + "gbo": "Northern Grebo", + "gbp": "Gbaya-Bossangoa", + "gbq": "Gbaya-Bozoum", + "gbr": "Gbagyi", + "gbs": "Gbesi Gbe", + "gbu": "Gagadu", + "gbv": "Gbanu", + "gbw": "Gabi-Gabi", + "gbx": "Eastern Xwla Gbe", + "gby": "Gbari", + "gbz": "Zoroastrian Dari", + "gcc": "Mali", + "gcd": "Ganggalida", + "gce": "Galice", + "gcf": "Guadeloupean Creole French", + "gcl": "Grenadian Creole English", + "gcn": "Gaina", + "gcr": "Guianese Creole French", + "gct": "Colonia Tovar German", + "gd": "Scottish Gaelic; Gaelic", + "gda": "Gade Lohar", + "gdb": "Pottangi Ollar Gadaba", + "gdc": "Gugu Badhun", + "gdd": "Gedaged", + "gde": "Gude", + "gdf": "Guduf-Gava", + "gdg": "Ga'dang", + "gdh": "Gadjerawang; Gajirrabeng", + "gdi": "Gundi", + "gdj": "Gurdjar", + "gdk": "Gadang", + "gdl": "Dirasha", + "gdm": "Laal", + "gdn": "Umanakaina", + "gdo": "Ghodoberi", + "gdq": "Mehri", + "gdr": "Wipi", + "gds": "Ghandruk Sign Language", + "gdt": "Kungardutyi", + "gdu": "Gudu", + "gdx": "Godwari", + "gea": "Geruma", + "geb": "Kire", + "gec": "Gboloo Grebo", + "ged": "Gade", + "gef": "Gerai", + "geg": "Gengle", + "geh": "Hutterite German; Hutterisch", + "gei": "Gebe", + "gej": "Gen", + "gek": "Ywom", + "gel": "ut-Ma'in", + "gem": "Germanic languages", + "geq": "Geme", + "ges": "Geser-Gorom", + "gev": "Eviya", + "gew": "Gera", + "gex": "Garre", + "gey": "Enya", + "gez": "Geez", + "gfk": "Patpatar", + "gft": "Gafat", + "gga": "Gao", + "ggb": "Gbii", + "ggd": "Gugadj", + "gge": "Gurr-goni", + "ggg": "Gurgula", + "ggk": "Kungarakany", + "ggl": "Ganglau", + "ggt": "Gitua", + "ggu": "Gagu; Gban", + "ggw": "Gogodala", + "gha": "Ghadamès", + "ghc": "Hiberno-Scottish Gaelic", + "ghe": "Southern Ghale", + "ghh": "Northern Ghale", + "ghk": "Geko Karen", + "ghl": "Ghulfan", + "ghn": "Ghanongga", + "gho": "Ghomara", + "ghr": "Ghera", + "ghs": "Guhu-Samane", + "ght": "Kuke; Kutang Ghale", + "gia": "Kija", + "gib": "Gibanawa", + "gic": "Gail", + "gid": "Gidar", + "gie": "Gaɓogbo; Guébie", + "gig": "Goaria", + "gih": "Githabul", + "gii": "Girirra", + "gil": "Gilbertese", + "gim": "Gimi (Eastern Highlands)", + "gin": "Hinukh", + "gip": "Gimi (West New Britain)", + "giq": "Green Gelao", + "gir": "Red Gelao", + "gis": "North Giziga", + "git": "Gitxsan", + "giu": "Mulao", + "giw": "White Gelao", + "gix": "Gilima", + "giy": "Giyug", + "giz": "South Giziga", + "gjk": "Kachi Koli", + "gjm": "Gunditjmara", + "gjn": "Gonja", + "gjr": "Gurindji Kriol", + "gju": "Gujari", + "gka": "Guya", + "gkd": "Magɨ (Madang Province)", + "gke": "Ndai", + "gkn": "Gokana", + "gko": "Kok-Nar", + "gkp": "Guinea Kpelle", + "gku": "ǂUngkue", + "gl": "Galician", + "glb": "Belning", + "glc": "Bon Gula", + "gld": "Nanai", + "glh": "Northwest Pashai; Northwest Pashayi", + "glj": "Gula Iro", + "glk": "Gilaki", + "gll": "Garlali", + "glo": "Galambu", + "glr": "Glaro-Twabo", + "glu": "Gula (Chad)", + "glw": "Glavda", + "gly": "Gule", + "gma": "Gambera", + "gmb": "Gula'alaa", + "gmd": "Mághdì", + "gme": "East Germanic languages", + "gmg": "Magɨyi", + "gmh": "Middle High German (ca. 1050-1500)", + "gml": "Middle Low German", + "gmm": "Gbaya-Mbodomo", + "gmn": "Gimnime", + "gmq": "North Germanic languages", + "gmr": "Mirning; Mirniny", + "gmu": "Gumalu", + "gmv": "Gamo", + "gmw": "West Germanic languages", + "gmx": "Magoma", + "gmy": "Mycenaean Greek", + "gmz": "Mgbolizhia", + "gn": "Guarani", + "gna": "Kaansa", + "gnb": "Gangte", + "gnc": "Guanche", + "gnd": "Zulgo-Gemzek", + "gne": "Ganang", + "gng": "Ngangam", + "gnh": "Lere", + "gni": "Gooniyandi", + "gnj": "Ngen", + "gnk": "ǁGana", + "gnl": "Gangulu", + "gnm": "Ginuman", + "gnn": "Gumatj", + "gno": "Northern Gondi", + "gnq": "Gana", + "gnr": "Gureng Gureng", + "gnt": "Guntai", + "gnu": "Gnau", + "gnw": "Western Bolivian Guaraní", + "gnz": "Ganzi", + "goa": "Guro", + "gob": "Playero", + "goc": "Gorakor", + "god": "Godié", + "goe": "Gongduk", + "gof": "Gofa", + "gog": "Gogo", + "goh": "Old High German (ca. 750-1050)", + "goi": "Gobasi", + "goj": "Gowlan", + "gok": "Gowli", + "gol": "Gola", + "gom": "Goan Konkani", + "gon": "Gondi", + "goo": "Gone Dau", + "gop": "Yeretuar", + "goq": "Gorap", + "gor": "Gorontalo", + "gos": "Gronings", + "got": "Gothic", + "gou": "Gavar", + "gov": "Goo", + "gow": "Gorowa", + "gox": "Gobu", + "goy": "Goundo", + "goz": "Gozarkhani", + "gpa": "Gupa-Abawa", + "gpe": "Ghanaian Pidgin English", + "gpn": "Taiap", + "gqa": "Ga'anda", + "gqi": "Guiqiong", + "gqn": "Guana (Brazil)", + "gqr": "Gor", + "gqu": "Qau", + "gra": "Rajput Garasia", + "grb": "Grebo", + "grc": "Ancient Greek (to 1453)", + "grd": "Guruntum-Mbaaru", + "grg": "Madi", + "grh": "Gbiri-Niragu", + "gri": "Ghari", + "grj": "Southern Grebo", + "grk": "Greek languages", + "grm": "Kota Marudu Talantang", + "gro": "Groma", + "grq": "Gorovu", + "grr": "Taznatit", + "grs": "Gresi", + "grt": "Garo", + "gru": "Kistane", + "grv": "Central Grebo", + "grw": "Gweda", + "grx": "Guriaso", + "gry": "Barclayville Grebo", + "grz": "Guramalum", + "gse": "Ghanaian Sign Language", + "gsg": "German Sign Language", + "gsl": "Gusilay", + "gsm": "Guatemalan Sign Language", + "gsn": "Nema; Gusan", + "gso": "Southwest Gbaya", + "gsp": "Wasembo", + "gss": "Greek Sign Language", + "gsw": "Swiss German; Alemannic; Alsatian", + "gta": "Guató", + "gtu": "Aghu-Tharnggala", + "gu": "Gujarati", + "gua": "Shiki", + "gub": "Guajajára", + "guc": "Wayuu", + "gud": "Yocoboué Dida", + "gue": "Gurindji", + "guf": "Gupapuyngu", + "gug": "Paraguayan Guaraní", + "guh": "Guahibo", + "gui": "Eastern Bolivian Guaraní", + "guk": "Gumuz", + "gul": "Sea Island Creole English", + "gum": "Guambiano", + "gun": "Mbyá Guaraní", + "guo": "Guayabero", + "gup": "Gunwinggu", + "guq": "Aché", + "gur": "Farefare", + "gus": "Guinean Sign Language", + "gut": "Maléku Jaíka", + "guu": "Yanomamö", + "guw": "Gun", + "gux": "Gourmanchéma", + "guz": "Gusii; Ekegusii", + "gv": "Manx", + "gva": "Guana (Paraguay)", + "gvc": "Guanano", + "gve": "Duwet", + "gvf": "Golin", + "gvj": "Guajá", + "gvl": "Gulay", + "gvm": "Gurmana", + "gvn": "Kuku-Yalanji", + "gvo": "Gavião Do Jiparaná", + "gvp": "Pará Gavião", + "gvr": "Gurung", + "gvs": "Gumawana", + "gvy": "Guyani", + "gwa": "Mbato", + "gwb": "Gwa", + "gwc": "Gawri; Kalami", + "gwd": "Gawwada", + "gwe": "Gweno", + "gwf": "Gowro", + "gwg": "Moo", + "gwi": "Gwichʼin", + "gwj": "ǀGwi", + "gwm": "Awngthim", + "gwn": "Gwandara", + "gwr": "Gwere", + "gwt": "Gawar-Bati", + "gwu": "Guwamu", + "gww": "Kwini", + "gwx": "Gua", + "gxx": "Wè Southern", + "gya": "Northwest Gbaya", + "gyb": "Garus", + "gyd": "Kayardild", + "gye": "Gyem", + "gyf": "Gungabula", + "gyg": "Gbayi", + "gyi": "Gyele", + "gyl": "Gayil", + "gym": "Ngäbere", + "gyn": "Guyanese Creole English", + "gyo": "Gyalsumdo", + "gyr": "Guarayu", + "gyy": "Gunya", + "gyz": "Geji; Gyaazi", + "gza": "Ganza", + "gzi": "Gazi", + "gzn": "Gane", + "ha": "Hausa", + "haa": "Han", + "hab": "Hanoi Sign Language", + "hac": "Gurani", + "had": "Hatam", + "hae": "Eastern Oromo", + "haf": "Haiphong Sign Language", + "hag": "Hanga", + "hah": "Hahon", + "hai": "Haida", + "haj": "Hajong", + "hak": "Hakka Chinese", + "hal": "Halang", + "ham": "Hewa", + "han": "Hangaza", + "hao": "Hakö", + "hap": "Hupla", + "haq": "Ha", + "har": "Harari", + "has": "Haisla", + "hav": "Havu", + "haw": "Hawaiian", + "hax": "Southern Haida", + "hay": "Haya", + "haz": "Hazaragi", + "hba": "Hamba", + "hbb": "Huba", + "hbn": "Heiban", + "hbo": "Ancient Hebrew", + "hbu": "Habu", + "hca": "Andaman Creole Hindi", + "hch": "Huichol", + "hdn": "Northern Haida", + "hds": "Honduras Sign Language", + "hdy": "Hadiyya", + "he": "Hebrew", + "hea": "Northern Qiandong Miao", + "hed": "Herdé", + "heg": "Helong", + "heh": "Hehe", + "hei": "Heiltsuk", + "hem": "Hemba", + "hgm": "Haiǁom", + "hgw": "Haigwai", + "hhi": "Hoia Hoia", + "hhr": "Kerak", + "hhy": "Hoyahoya", + "hi": "Hindi", + "hia": "Lamang", + "hib": "Hibito", + "hid": "Hidatsa", + "hif": "Fiji Hindi", + "hig": "Kamwe", + "hih": "Pamosu", + "hii": "Hinduri", + "hij": "Hijuk", + "hik": "Seit-Kaitetu", + "hil": "Hiligaynon", + "him": "Himachali languages; Western Pahari languages", + "hio": "Tsoa", + "hir": "Himarimã", + "hit": "Hittite", + "hiw": "Hiw", + "hix": "Hixkaryána", + "hji": "Haji", + "hka": "Kahe", + "hke": "Hunde", + "hkh": "Khah; Poguli", + "hkk": "Hunjara-Kaina Ke", + "hkn": "Mel-Khaonh", + "hks": "Hong Kong Sign Language; Heung Kong Sau Yue", + "hla": "Halia", + "hlb": "Halbi", + "hld": "Halang Doan", + "hle": "Hlersu", + "hlt": "Matu Chin", + "hlu": "Hieroglyphic Luwian", + "hma": "Southern Mashan Hmong; Southern Mashan Miao", + "hmb": "Humburi Senni Songhay", + "hmc": "Central Huishui Hmong; Central Huishui Miao", + "hmd": "Large Flowery Miao; A-hmaos; Da-Hua Miao", + "hme": "Eastern Huishui Hmong; Eastern Huishui Miao", + "hmf": "Hmong Don", + "hmg": "Southwestern Guiyang Hmong", + "hmh": "Southwestern Huishui Hmong; Southwestern Huishui Miao", + "hmi": "Northern Huishui Hmong; Northern Huishui Miao", + "hmj": "Ge; Gejia", + "hmk": "Maek", + "hml": "Luopohe Hmong; Luopohe Miao", + "hmm": "Central Mashan Hmong; Central Mashan Miao", + "hmn": "Hmong; Mong", + "hmp": "Northern Mashan Hmong; Northern Mashan Miao", + "hmq": "Eastern Qiandong Miao", + "hmr": "Hmar", + "hms": "Southern Qiandong Miao", + "hmt": "Hamtai", + "hmu": "Hamap", + "hmv": "Hmong Dô", + "hmw": "Western Mashan Hmong; Western Mashan Miao", + "hmx": "Hmong-Mien languages", + "hmy": "Southern Guiyang Hmong; Southern Guiyang Miao", + "hmz": "Hmong Shua; Sinicized Miao", + "hna": "Mina (Cameroon)", + "hnd": "Southern Hindko", + "hne": "Chhattisgarhi", + "hng": "Hungu", + "hnh": "ǁAni", + "hni": "Hani", + "hnj": "Hmong Njua; Mong Leng; Mong Njua", + "hnn": "Hanunoo", + "hno": "Northern Hindko", + "hns": "Caribbean Hindustani", + "hnu": "Hung", + "ho": "Hiri Motu", + "hoa": "Hoava", + "hob": "Mari (Madang Province)", + "hoc": "Ho", + "hod": "Holma", + "hoe": "Horom", + "hoh": "Hobyót", + "hoi": "Holikachuk", + "hoj": "Hadothi; Haroti", + "hok": "Hokan languages", + "hol": "Holu", + "hom": "Homa", + "hoo": "Holoholo", + "hop": "Hopi", + "hor": "Horo", + "hos": "Ho Chi Minh City Sign Language", + "hot": "Hote; Malê", + "hov": "Hovongan", + "how": "Honi", + "hoy": "Holiya", + "hoz": "Hozo", + "hpo": "Hpon", + "hps": "Hawai'i Sign Language (HSL); Hawai'i Pidgin Sign Language", + "hr": "Croatian", + "hra": "Hrangkhol", + "hrc": "Niwer Mil", + "hre": "Hre", + "hrk": "Haruku", + "hrm": "Horned Miao", + "hro": "Haroi", + "hrp": "Nhirrpi", + "hrt": "Hértevin", + "hru": "Hruso", + "hrw": "Warwar Feni", + "hrx": "Hunsrik", + "hrz": "Harzani", + "hsb": "Upper Sorbian", + "hsh": "Hungarian Sign Language", + "hsl": "Hausa Sign Language", + "hsn": "Xiang Chinese", + "hss": "Harsusi", + "ht": "Haitian; Haitian Creole", + "hti": "Hoti", + "hto": "Minica Huitoto", + "hts": "Hadza", + "htu": "Hitu", + "htx": "Middle Hittite", + "hu": "Hungarian", + "hub": "Huambisa", + "huc": "ǂHua; ǂʼAmkhoe", + "hud": "Huaulu", + "hue": "San Francisco Del Mar Huave", + "huf": "Humene", + "hug": "Huachipaeri", + "huh": "Huilliche", + "hui": "Huli", + "huj": "Northern Guiyang Hmong; Northern Guiyang Miao", + "huk": "Hulung", + "hul": "Hula", + "hum": "Hungana", + "huo": "Hu", + "hup": "Hupa", + "huq": "Tsat", + "hur": "Halkomelem", + "hus": "Huastec", + "hut": "Humla", + "huu": "Murui Huitoto", + "huv": "San Mateo Del Mar Huave", + "huw": "Hukumina", + "hux": "Nüpode Huitoto", + "huy": "Hulaulá", + "huz": "Hunzib", + "hvc": "Haitian Vodoun Culture Language", + "hve": "San Dionisio Del Mar Huave", + "hvk": "Haveke", + "hvn": "Sabu", + "hvv": "Santa María Del Mar Huave", + "hwa": "Wané", + "hwc": "Hawai'i Creole English; Hawai'i Pidgin", + "hwo": "Hwana", + "hy": "Armenian", + "hya": "Hya", + "hyw": "Western Armenian", + "hyx": "Armenian (family)", + "hz": "Herero", + "ia": "Interlingua (International Auxiliary Language Association)", + "iai": "Iaai", + "ian": "Iatmul", + "iar": "Purari", + "iba": "Iban", + "ibb": "Ibibio", + "ibd": "Iwaidja", + "ibe": "Akpes", + "ibg": "Ibanag", + "ibh": "Bih", + "ibl": "Ibaloi", + "ibm": "Agoi", + "ibn": "Ibino", + "ibr": "Ibuoro", + "ibu": "Ibu", + "iby": "Ibani", + "ica": "Ede Ica", + "ich": "Etkywan", + "icl": "Icelandic Sign Language", + "icr": "Islander Creole English", + "id": "Indonesian", + "ida": "Idakho-Isukha-Tiriki; Luidakho-Luisukha-Lutirichi", + "idb": "Indo-Portuguese", + "idc": "Idon; Ajiya", + "idd": "Ede Idaca", + "ide": "Idere", + "idi": "Idi", + "idr": "Indri", + "ids": "Idesa", + "idt": "Idaté", + "idu": "Idoma", + "ie": "Interlingue; Occidental", + "ifa": "Amganad Ifugao", + "ifb": "Batad Ifugao; Ayangan Ifugao", + "ife": "Ifè", + "iff": "Ifo", + "ifk": "Tuwali Ifugao", + "ifm": "Teke-Fuumu", + "ifu": "Mayoyao Ifugao", + "ify": "Keley-I Kallahan", + "ig": "Igbo", + "igb": "Ebira", + "ige": "Igede", + "igg": "Igana", + "igl": "Igala", + "igm": "Kanggape", + "ign": "Ignaciano", + "igo": "Isebe", + "igs": "Interglossa", + "igw": "Igwe", + "ihb": "Iha Based Pidgin", + "ihi": "Ihievbe", + "ihp": "Iha", + "ihw": "Bidhawal", + "ii": "Sichuan Yi; Nuosu", + "iin": "Thiin", + "iir": "Indo-Iranian languages", + "ijc": "Izon", + "ije": "Biseni", + "ijj": "Ede Ije", + "ijn": "Kalabari", + "ijo": "Ijo languages", + "ijs": "Southeast Ijo", + "ik": "Inupiaq", + "ike": "Eastern Canadian Inuktitut", + "iki": "Iko", + "ikk": "Ika", + "ikl": "Ikulu", + "iko": "Olulumo-Ikom", + "ikp": "Ikpeshi", + "ikr": "Ikaranggal", + "iks": "Inuit Sign Language", + "ikt": "Inuinnaqtun; Western Canadian Inuktitut", + "ikv": "Iku-Gora-Ankwa", + "ikw": "Ikwere", + "ikx": "Ik", + "ikz": "Ikizu", + "ila": "Ile Ape", + "ilb": "Ila", + "ilg": "Garig-Ilgar", + "ili": "Ili Turki", + "ilk": "Ilongot", + "ilm": "Iranun (Malaysia)", + "ilo": "Iloko", + "ilp": "Iranun (Philippines)", + "ils": "International Sign", + "ilu": "Ili'uun", + "ilv": "Ilue", + "ima": "Mala Malasar", + "imi": "Anamgura", + "iml": "Miluk", + "imn": "Imonda", + "imo": "Imbongu", + "imr": "Imroing", + "ims": "Marsian", + "imt": "Imotong", + "imy": "Milyan", + "inb": "Inga", + "inc": "Indic languages", + "ine": "Indo-European languages", + "ing": "Degexit'an", + "inh": "Ingush", + "inj": "Jungle Inga", + "inl": "Indonesian Sign Language", + "inm": "Minaean", + "inn": "Isinai", + "ino": "Inoke-Yate", + "inp": "Iñapari", + "ins": "Indian Sign Language", + "int": "Intha", + "inz": "Ineseño", + "io": "Ido", + "ior": "Inor", + "iou": "Tuma-Irumu", + "iow": "Iowa-Oto", + "ipi": "Ipili", + "ipo": "Ipiko", + "iqu": "Iquito", + "iqw": "Ikwo", + "ira": "Iranian languages", + "ire": "Iresim", + "irh": "Irarutu", + "iri": "Rigwe; Irigwe", + "irk": "Iraqw", + "irn": "Irántxe", + "iro": "Iroquoian languages", + "irr": "Ir", + "iru": "Irula", + "irx": "Kamberau", + "iry": "Iraya", + "is": "Icelandic", + "isa": "Isabi", + "isc": "Isconahua", + "isd": "Isnag", + "ise": "Italian Sign Language", + "isg": "Irish Sign Language", + "ish": "Esan", + "isi": "Nkem-Nkum", + "isk": "Ishkashimi", + "ism": "Masimasi", + "isn": "Isanzu", + "iso": "Isoko", + "isr": "Israeli Sign Language", + "ist": "Istriot", + "isu": "Isu (Menchum Division)", + "it": "Italian", + "itb": "Binongan Itneg", + "itc": "Italic languages", + "itd": "Southern Tidung", + "ite": "Itene", + "iti": "Inlaod Itneg", + "itk": "Judeo-Italian", + "itl": "Itelmen", + "itm": "Itu Mbon Uzo", + "ito": "Itonama", + "itr": "Iteri", + "its": "Isekiri", + "itt": "Maeng Itneg", + "itv": "Itawit", + "itw": "Ito", + "itx": "Itik", + "ity": "Moyadan Itneg", + "itz": "Itzá", + "iu": "Inuktitut", + "ium": "Iu Mien", + "ivb": "Ibatan", + "ivv": "Ivatan", + "iwk": "I-Wak", + "iwm": "Iwam", + "iwo": "Iwur", + "iws": "Sepik Iwam", + "ixc": "Ixcatec", + "ixl": "Ixil", + "iya": "Iyayu", + "iyo": "Mesaka", + "iyx": "Yaka (Congo)", + "izh": "Ingrian", + "izr": "Izere", + "izz": "Izii", + "ja": "Japanese", + "jaa": "Jamamadí", + "jab": "Hyam", + "jac": "Popti'; Jakalteko", + "jad": "Jahanka", + "jae": "Yabem", + "jaf": "Jara", + "jah": "Jah Hut", + "jaj": "Zazao", + "jak": "Jakun", + "jal": "Yalahatan", + "jam": "Jamaican Creole English", + "jan": "Jandai", + "jao": "Yanyuwa", + "jaq": "Yaqay", + "jas": "New Caledonian Javanese", + "jat": "Jakati", + "jau": "Yaur", + "jax": "Jambi Malay", + "jay": "Yan-nhangu; Nhangu", + "jaz": "Jawe", + "jbe": "Judeo-Berber", + "jbi": "Badjiri", + "jbj": "Arandai", + "jbk": "Barikewa", + "jbm": "Bijim", + "jbn": "Nafusi", + "jbo": "Lojban", + "jbr": "Jofotek-Bromnya", + "jbt": "Jabutí", + "jbu": "Jukun Takum", + "jbw": "Yawijibaya", + "jcs": "Jamaican Country Sign Language", + "jct": "Krymchak", + "jda": "Jad", + "jdg": "Jadgali", + "jdt": "Judeo-Tat", + "jeb": "Jebero", + "jee": "Jerung", + "jeh": "Jeh", + "jei": "Yei", + "jek": "Jeri Kuo", + "jel": "Yelmek", + "jen": "Dza", + "jer": "Jere", + "jet": "Manem", + "jeu": "Jonkor Bourmataguil", + "jgb": "Ngbee", + "jge": "Judeo-Georgian", + "jgk": "Gwak", + "jgo": "Ngomba", + "jhi": "Jehai", + "jhs": "Jhankot Sign Language", + "jia": "Jina", + "jib": "Jibu", + "jic": "Tol", + "jid": "Bu (Kaduna State)", + "jie": "Jilbe", + "jig": "Jingulu; Djingili", + "jih": "sTodsde; Shangzhai", + "jii": "Jiiddu", + "jil": "Jilim", + "jim": "Jimi (Cameroon)", + "jio": "Jiamao", + "jiq": "Guanyinqiao; Lavrung", + "jit": "Jita", + "jiu": "Youle Jinuo", + "jiv": "Shuar", + "jiy": "Buyuan Jinuo", + "jje": "Jejueo", + "jjr": "Bankal", + "jka": "Kaera", + "jkm": "Mobwa Karen", + "jko": "Kubo", + "jkp": "Paku Karen", + "jkr": "Koro (India)", + "jks": "Amami Koniya Sign Language", + "jku": "Labir", + "jle": "Ngile", + "jls": "Jamaican Sign Language", + "jma": "Dima", + "jmb": "Zumbun", + "jmc": "Machame", + "jmd": "Yamdena", + "jmi": "Jimi (Nigeria)", + "jml": "Jumli", + "jmn": "Makuri Naga", + "jmr": "Kamara", + "jms": "Mashi (Nigeria)", + "jmw": "Mouwase", + "jmx": "Western Juxtlahuaca Mixtec", + "jna": "Jangshung", + "jnd": "Jandavra", + "jng": "Yangman", + "jni": "Janji", + "jnj": "Yemsa", + "jnl": "Rawat", + "jns": "Jaunsari", + "job": "Joba", + "jod": "Wojenaka", + "jog": "Jogi", + "jor": "Jorá", + "jos": "Jordanian Sign Language", + "jow": "Jowulu", + "jpa": "Jewish Palestinian Aramaic", + "jpr": "Judeo-Persian", + "jpx": "Japanese (family)", + "jqr": "Jaqaru", + "jra": "Jarai", + "jrb": "Judeo-Arabic", + "jrr": "Jiru", + "jrt": "Jakattoe", + "jru": "Japrería", + "jsl": "Japanese Sign Language", + "jua": "Júma", + "jub": "Wannu", + "juc": "Jurchen", + "jud": "Worodougou", + "juh": "Hõne", + "jui": "Ngadjuri", + "juk": "Wapan", + "jul": "Jirel", + "jum": "Jumjum", + "jun": "Juang", + "juo": "Jiba", + "jup": "Hupdë", + "jur": "Jurúna", + "jus": "Jumla Sign Language", + "jut": "Jutish", + "juu": "Ju", + "juw": "Wãpha", + "juy": "Juray", + "jv": "Javanese", + "jvd": "Javindo", + "jvn": "Caribbean Javanese", + "jwi": "Jwira-Pepesa", + "jya": "Jiarong", + "jye": "Judeo-Yemeni Arabic", + "jyy": "Jaya", + "ka": "Georgian", + "kaa": "Kara-Kalpak; Karakalpak", + "kab": "Kabyle", + "kac": "Kachin; Jingpho", + "kad": "Adara", + "kae": "Ketangalan", + "kaf": "Katso", + "kag": "Kajaman", + "kah": "Kara (Central African Republic)", + "kai": "Karekare", + "kaj": "Jju", + "kak": "Kalanguya; Kayapa Kallahan", + "kam": "Kamba (Kenya)", + "kao": "Xaasongaxango", + "kap": "Bezhta", + "kaq": "Capanahua", + "kar": "Karen languages", + "kav": "Katukína", + "kaw": "Kawi", + "kax": "Kao", + "kay": "Kamayurá", + "kba": "Kalarko", + "kbb": "Kaxuiâna", + "kbc": "Kadiwéu", + "kbd": "Kabardian", + "kbe": "Kanju", + "kbg": "Khamba", + "kbh": "Camsá", + "kbi": "Kaptiau", + "kbj": "Kari", + "kbk": "Grass Koiari", + "kbl": "Kanembu", + "kbm": "Iwal", + "kbn": "Kare (Central African Republic)", + "kbo": "Keliko", + "kbp": "Kabiyè", + "kbq": "Kamano", + "kbr": "Kafa", + "kbs": "Kande", + "kbt": "Abadi", + "kbu": "Kabutra", + "kbv": "Dera (Indonesia)", + "kbw": "Kaiep", + "kbx": "Ap Ma", + "kby": "Manga Kanuri", + "kbz": "Duhwa", + "kca": "Khanty", + "kcb": "Kawacha", + "kcc": "Lubila", + "kcd": "Ngkâlmpw Kanum", + "kce": "Kaivi", + "kcf": "Ukaan", + "kcg": "Tyap", + "kch": "Vono", + "kci": "Kamantan", + "kcj": "Kobiana", + "kck": "Kalanga", + "kcl": "Kela (Papua New Guinea); Kala", + "kcm": "Gula (Central African Republic)", + "kcn": "Nubi", + "kco": "Kinalakna", + "kcp": "Kanga", + "kcq": "Kamo", + "kcr": "Katla", + "kcs": "Koenoem", + "kct": "Kaian", + "kcu": "Kami (Tanzania)", + "kcv": "Kete", + "kcw": "Kabwari", + "kcx": "Kachama-Ganjule", + "kcy": "Korandje", + "kcz": "Konongo", + "kda": "Worimi", + "kdc": "Kutu", + "kdd": "Yankunytjatjara", + "kde": "Makonde", + "kdf": "Mamusi", + "kdg": "Seba", + "kdh": "Tem", + "kdi": "Kumam", + "kdj": "Karamojong", + "kdk": "Numèè; Kwényi", + "kdl": "Tsikimba", + "kdm": "Kagoma", + "kdn": "Kunda", + "kdo": "Kordofanian languages", + "kdp": "Kaningdon-Nindem", + "kdq": "Koch", + "kdr": "Karaim", + "kdt": "Kuy", + "kdu": "Kadaru", + "kdw": "Koneraw", + "kdx": "Kam", + "kdy": "Keder; Keijar", + "kdz": "Kwaja", + "kea": "Kabuverdianu", + "keb": "Kélé", + "kec": "Keiga", + "ked": "Kerewe", + "kee": "Eastern Keres", + "kef": "Kpessi", + "keg": "Tese", + "keh": "Keak", + "kei": "Kei", + "kej": "Kadar", + "kek": "Kekchí", + "kel": "Kela (Democratic Republic of Congo)", + "kem": "Kemak", + "ken": "Kenyang", + "keo": "Kakwa", + "kep": "Kaikadi", + "keq": "Kamar", + "ker": "Kera", + "kes": "Kugbo", + "ket": "Ket", + "keu": "Akebu", + "kev": "Kanikkaran", + "kew": "West Kewa", + "kex": "Kukna", + "key": "Kupia", + "kez": "Kukele", + "kfa": "Kodava", + "kfb": "Northwestern Kolami", + "kfc": "Konda-Dora", + "kfd": "Korra Koraga", + "kfe": "Kota (India)", + "kff": "Koya", + "kfg": "Kudiya", + "kfh": "Kurichiya", + "kfi": "Kannada Kurumba", + "kfj": "Kemiehua", + "kfk": "Kinnauri", + "kfl": "Kung", + "kfm": "Khunsari", + "kfn": "Kuk", + "kfo": "Koro (Côte d'Ivoire)", + "kfp": "Korwa", + "kfq": "Korku", + "kfr": "Kachhi; Kutchi", + "kfs": "Bilaspuri", + "kft": "Kanjari", + "kfu": "Katkari", + "kfv": "Kurmukar", + "kfw": "Kharam Naga", + "kfx": "Kullu Pahari", + "kfy": "Kumaoni", + "kfz": "Koromfé", + "kg": "Kongo", + "kga": "Koyaga", + "kgb": "Kawe", + "kge": "Komering", + "kgf": "Kube", + "kgg": "Kusunda", + "kgi": "Selangor Sign Language", + "kgj": "Gamale Kham", + "kgk": "Kaiwá", + "kgl": "Kunggari", + "kgm": "Karipúna", + "kgn": "Karingani", + "kgo": "Krongo", + "kgp": "Kaingang", + "kgq": "Kamoro", + "kgr": "Abun", + "kgs": "Kumbainggar", + "kgt": "Somyev", + "kgu": "Kobol", + "kgv": "Karas", + "kgw": "Karon Dori", + "kgx": "Kamaru", + "kgy": "Kyerung", + "kha": "Khasi", + "khb": "Lü", + "khc": "Tukang Besi North", + "khd": "Bädi Kanum", + "khe": "Korowai", + "khf": "Khuen", + "khg": "Khams Tibetan", + "khh": "Kehu", + "khi": "Khoisan languages", + "khj": "Kuturmi", + "khk": "Halh Mongolian", + "khl": "Lusi", + "khn": "Khandesi", + "kho": "Khotanese; Sakan", + "khp": "Kapori; Kapauri", + "khq": "Koyra Chiini Songhay", + "khr": "Kharia", + "khs": "Kasua", + "kht": "Khamti", + "khu": "Nkhumbi", + "khv": "Khvarshi", + "khw": "Khowar", + "khx": "Kanu", + "khy": "Kele (Democratic Republic of Congo)", + "khz": "Keapara", + "ki": "Kikuyu; Gikuyu", + "kia": "Kim", + "kib": "Koalib", + "kic": "Kickapoo", + "kid": "Koshin", + "kie": "Kibet", + "kif": "Eastern Parbate Kham", + "kig": "Kimaama; Kimaghima", + "kih": "Kilmeri", + "kii": "Kitsai", + "kij": "Kilivila", + "kil": "Kariya", + "kim": "Karagas", + "kio": "Kiowa", + "kip": "Sheshi Kham", + "kiq": "Kosadle; Kosare", + "kis": "Kis", + "kit": "Agob", + "kiu": "Kirmanjki (individual language)", + "kiv": "Kimbu", + "kiw": "Northeast Kiwai", + "kix": "Khiamniungan Naga", + "kiy": "Kirikiri", + "kiz": "Kisi", + "kj": "Kuanyama; Kwanyama", + "kja": "Mlap", + "kjb": "Q'anjob'al; Kanjobal", + "kjc": "Coastal Konjo", + "kjd": "Southern Kiwai", + "kje": "Kisar", + "kjg": "Khmu", + "kjh": "Khakas", + "kji": "Zabana", + "kjj": "Khinalugh", + "kjk": "Highland Konjo", + "kjl": "Western Parbate Kham", + "kjm": "Kháng", + "kjn": "Kunjen", + "kjo": "Harijan Kinnauri", + "kjp": "Pwo Eastern Karen", + "kjq": "Western Keres", + "kjr": "Kurudu", + "kjs": "East Kewa", + "kjt": "Phrae Pwo Karen", + "kju": "Kashaya", + "kjv": "Kaikavian Literary Language", + "kjx": "Ramopa", + "kjy": "Erave", + "kjz": "Bumthangkha", + "kk": "Kazakh", + "kka": "Kakanda", + "kkb": "Kwerisa", + "kkc": "Odoodee", + "kkd": "Kinuku", + "kke": "Kakabe", + "kkf": "Kalaktang Monpa", + "kkg": "Mabaka Valley Kalinga", + "kkh": "Khün", + "kki": "Kagulu", + "kkj": "Kako", + "kkk": "Kokota", + "kkl": "Kosarek Yale", + "kkm": "Kiong", + "kkn": "Kon Keu", + "kko": "Karko", + "kkp": "Gugubera; Koko-Bera", + "kkq": "Kaeku", + "kkr": "Kir-Balar", + "kks": "Giiwo", + "kkt": "Koi", + "kku": "Tumi", + "kkv": "Kangean", + "kkw": "Teke-Kukuya", + "kkx": "Kohin", + "kky": "Guugu Yimidhirr; Guguyimidjir", + "kkz": "Kaska", + "kl": "Kalaallisut; Greenlandic", + "kla": "Klamath-Modoc", + "klb": "Kiliwa", + "klc": "Kolbila", + "kld": "Gamilaraay", + "kle": "Kulung (Nepal)", + "klf": "Kendeje", + "klg": "Tagakaulo", + "klh": "Weliki", + "kli": "Kalumpang", + "klj": "Khalaj", + "klk": "Kono (Nigeria)", + "kll": "Kagan Kalagan", + "klm": "Migum", + "kln": "Kalenjin", + "klo": "Kapya", + "klp": "Kamasa", + "klq": "Rumu", + "klr": "Khaling", + "kls": "Kalasha", + "klt": "Nukna", + "klu": "Klao", + "klv": "Maskelynes", + "klw": "Tado; Lindu", + "klx": "Koluwawa", + "kly": "Kalao", + "klz": "Kabola", + "km": "Khmer; Central Khmer", + "kma": "Konni", + "kmb": "Kimbundu", + "kmc": "Southern Dong", + "kmd": "Majukayang Kalinga", + "kme": "Bakole", + "kmf": "Kare (Papua New Guinea)", + "kmg": "Kâte", + "kmh": "Kalam", + "kmi": "Kami (Nigeria)", + "kmj": "Kumarbhag Paharia", + "kmk": "Limos Kalinga", + "kml": "Tanudan Kalinga", + "kmm": "Kom (India)", + "kmn": "Awtuw", + "kmo": "Kwoma", + "kmp": "Gimme", + "kmq": "Kwama", + "kmr": "Northern Kurdish", + "kms": "Kamasau", + "kmt": "Kemtuik", + "kmu": "Kanite", + "kmv": "Karipúna Creole French", + "kmw": "Komo (Democratic Republic of Congo)", + "kmx": "Waboda", + "kmy": "Koma", + "kmz": "Khorasani Turkish", + "kn": "Kannada", + "kna": "Dera (Nigeria)", + "knb": "Lubuagan Kalinga", + "knc": "Central Kanuri", + "knd": "Konda", + "kne": "Kankanaey", + "knf": "Mankanya", + "kng": "Koongo", + "kni": "Kanufi", + "knj": "Western Kanjobal", + "knk": "Kuranko", + "knl": "Keninjal", + "knm": "Kanamarí", + "knn": "Konkani (individual language)", + "kno": "Kono (Sierra Leone)", + "knp": "Kwanja", + "knq": "Kintaq", + "knr": "Kaningra", + "kns": "Kensiu", + "knt": "Panoan Katukína", + "knu": "Kono (Guinea)", + "knv": "Tabo", + "knw": "Kung-Ekoka", + "knx": "Kendayan; Salako", + "kny": "Kanyok", + "knz": "Kalamsé", + "ko": "Korean", + "koa": "Konomala", + "koc": "Kpati", + "kod": "Kodi", + "koe": "Kacipo-Bale Suri", + "kof": "Kubi", + "kog": "Cogui; Kogi", + "koh": "Koyo", + "koi": "Komi-Permyak", + "kok": "Konkani (macrolanguage)", + "kol": "Kol (Papua New Guinea)", + "koo": "Konzo", + "kop": "Waube", + "koq": "Kota (Gabon)", + "kos": "Kosraean", + "kot": "Lagwan", + "kou": "Koke", + "kov": "Kudu-Camo", + "kow": "Kugama", + "koy": "Koyukon", + "koz": "Korak", + "kpa": "Kutto", + "kpb": "Mullu Kurumba", + "kpc": "Curripaco", + "kpd": "Koba", + "kpe": "Kpelle", + "kpf": "Komba", + "kpg": "Kapingamarangi", + "kph": "Kplang", + "kpi": "Kofei", + "kpj": "Karajá", + "kpk": "Kpan", + "kpl": "Kpala", + "kpm": "Koho", + "kpn": "Kepkiriwát", + "kpo": "Ikposo", + "kpq": "Korupun-Sela", + "kpr": "Korafe-Yegha", + "kps": "Tehit", + "kpt": "Karata", + "kpu": "Kafoa", + "kpv": "Komi-Zyrian", + "kpw": "Kobon", + "kpx": "Mountain Koiali", + "kpy": "Koryak", + "kpz": "Kupsabiny", + "kqa": "Mum", + "kqb": "Kovai", + "kqc": "Doromu-Koki", + "kqd": "Koy Sanjaq Surat", + "kqe": "Kalagan", + "kqf": "Kakabai", + "kqg": "Khe", + "kqh": "Kisankasa", + "kqi": "Koitabu", + "kqj": "Koromira", + "kqk": "Kotafon Gbe", + "kql": "Kyenele", + "kqm": "Khisa", + "kqn": "Kaonde", + "kqo": "Eastern Krahn", + "kqp": "Kimré", + "kqq": "Krenak", + "kqr": "Kimaragang", + "kqs": "Northern Kissi", + "kqt": "Klias River Kadazan", + "kqu": "Seroa", + "kqv": "Okolod", + "kqw": "Kandas", + "kqx": "Mser", + "kqy": "Koorete", + "kqz": "Korana", + "kr": "Kanuri", + "kra": "Kumhali", + "krb": "Karkin", + "krc": "Karachay-Balkar", + "krd": "Kairui-Midiki", + "kre": "Panará", + "krf": "Koro (Vanuatu)", + "krh": "Kurama", + "kri": "Krio", + "krj": "Kinaray-A", + "krk": "Kerek", + "krl": "Karelian", + "krn": "Sapo", + "kro": "Kru languages", + "krp": "Korop", + "krr": "Krung", + "krs": "Gbaya (Sudan)", + "krt": "Tumari Kanuri", + "kru": "Kurukh", + "krv": "Kavet", + "krw": "Western Krahn", + "krx": "Karon", + "kry": "Kryts", + "krz": "Sota Kanum", + "ks": "Kashmiri", + "ksa": "Shuwa-Zamani", + "ksb": "Shambala", + "ksc": "Southern Kalinga", + "ksd": "Kuanua", + "kse": "Kuni", + "ksf": "Bafia", + "ksg": "Kusaghe", + "ksh": "Kölsch", + "ksi": "Krisa; I'saka", + "ksj": "Uare", + "ksk": "Kansa", + "ksl": "Kumalu", + "ksm": "Kumba", + "ksn": "Kasiguranin", + "kso": "Kofa", + "ksp": "Kaba", + "ksq": "Kwaami", + "ksr": "Borong", + "kss": "Southern Kisi", + "kst": "Winyé", + "ksu": "Khamyang", + "ksv": "Kusu", + "ksw": "S'gaw Karen", + "ksx": "Kedang", + "ksy": "Kharia Thar", + "ksz": "Kodaku", + "kta": "Katua", + "ktb": "Kambaata", + "ktc": "Kholok", + "ktd": "Kokata; Kukatha", + "kte": "Nubri", + "ktf": "Kwami", + "ktg": "Kalkutung", + "kth": "Karanga", + "kti": "North Muyu", + "ktj": "Plapo Krumen", + "ktk": "Kaniet", + "ktl": "Koroshi", + "ktm": "Kurti", + "ktn": "Karitiâna", + "kto": "Kuot", + "ktp": "Kaduo", + "ktq": "Katabaga", + "kts": "South Muyu", + "ktt": "Ketum", + "ktu": "Kituba (Democratic Republic of Congo)", + "ktv": "Eastern Katu", + "ktw": "Kato", + "ktx": "Kaxararí", + "kty": "Kango (Bas-Uélé District)", + "ktz": "Juǀʼhoan; Juǀʼhoansi", + "ku": "Kurdish", + "kub": "Kutep", + "kuc": "Kwinsu", + "kud": "'Auhelawa", + "kue": "Kuman (Papua New Guinea)", + "kuf": "Western Katu", + "kug": "Kupa", + "kuh": "Kushi", + "kui": "Kuikúro-Kalapálo; Kalapalo", + "kuj": "Kuria", + "kuk": "Kepo'", + "kul": "Kulere", + "kum": "Kumyk", + "kun": "Kunama", + "kuo": "Kumukio", + "kup": "Kunimaipa", + "kuq": "Karipuna", + "kus": "Kusaal", + "kut": "Kutenai", + "kuu": "Upper Kuskokwim", + "kuv": "Kur", + "kuw": "Kpagua", + "kux": "Kukatja", + "kuy": "Kuuku-Ya'u", + "kuz": "Kunza", + "kv": "Komi", + "kva": "Bagvalal", + "kvb": "Kubu", + "kvc": "Kove", + "kvd": "Kui (Indonesia)", + "kve": "Kalabakan", + "kvf": "Kabalai", + "kvg": "Kuni-Boazi", + "kvh": "Komodo", + "kvi": "Kwang", + "kvj": "Psikye", + "kvk": "Korean Sign Language", + "kvl": "Kayaw", + "kvm": "Kendem", + "kvn": "Border Kuna", + "kvo": "Dobel", + "kvp": "Kompane", + "kvq": "Geba Karen", + "kvr": "Kerinci", + "kvt": "Lahta Karen; Lahta", + "kvu": "Yinbaw Karen", + "kvv": "Kola", + "kvw": "Wersing", + "kvx": "Parkari Koli", + "kvy": "Yintale Karen; Yintale", + "kvz": "Tsakwambo; Tsaukambo", + "kw": "Cornish", + "kwa": "Dâw", + "kwb": "Kwa", + "kwc": "Likwala", + "kwd": "Kwaio", + "kwe": "Kwerba", + "kwf": "Kwara'ae", + "kwg": "Sara Kaba Deme", + "kwh": "Kowiai", + "kwi": "Awa-Cuaiquer", + "kwj": "Kwanga", + "kwk": "Kwakiutl", + "kwl": "Kofyar", + "kwm": "Kwambi", + "kwn": "Kwangali", + "kwo": "Kwomtari", + "kwp": "Kodia", + "kwr": "Kwer", + "kws": "Kwese", + "kwt": "Kwesten", + "kwu": "Kwakum", + "kwv": "Sara Kaba Náà", + "kww": "Kwinti", + "kwx": "Khirwar", + "kwy": "San Salvador Kongo", + "kwz": "Kwadi", + "kxa": "Kairiru", + "kxb": "Krobu", + "kxc": "Konso; Khonso", + "kxd": "Brunei", + "kxf": "Manumanaw Karen; Manumanaw", + "kxh": "Karo (Ethiopia)", + "kxi": "Keningau Murut", + "kxj": "Kulfa", + "kxk": "Zayein Karen", + "kxm": "Northern Khmer", + "kxn": "Kanowit-Tanjong Melanau", + "kxo": "Kanoé", + "kxp": "Wadiyara Koli", + "kxq": "Smärky Kanum", + "kxr": "Koro (Papua New Guinea)", + "kxs": "Kangjia", + "kxt": "Koiwat", + "kxv": "Kuvi", + "kxw": "Konai", + "kxx": "Likuba", + "kxy": "Kayong", + "kxz": "Kerewo", + "ky": "Kirghiz; Kyrgyz", + "kya": "Kwaya", + "kyb": "Butbut Kalinga", + "kyc": "Kyaka", + "kyd": "Karey", + "kye": "Krache", + "kyf": "Kouya", + "kyg": "Keyagana", + "kyh": "Karok", + "kyi": "Kiput", + "kyj": "Karao", + "kyk": "Kamayo", + "kyl": "Kalapuya", + "kym": "Kpatili", + "kyn": "Northern Binukidnon", + "kyo": "Kelon", + "kyp": "Kang", + "kyq": "Kenga", + "kyr": "Kuruáya", + "kys": "Baram Kayan", + "kyt": "Kayagar", + "kyu": "Western Kayah", + "kyv": "Kayort", + "kyw": "Kudmali", + "kyx": "Rapoisi", + "kyy": "Kambaira", + "kyz": "Kayabí", + "kza": "Western Karaboro", + "kzb": "Kaibobo", + "kzc": "Bondoukou Kulango", + "kzd": "Kadai", + "kze": "Kosena", + "kzf": "Da'a Kaili", + "kzg": "Kikai", + "kzi": "Kelabit", + "kzk": "Kazukuru", + "kzl": "Kayeli", + "kzm": "Kais", + "kzn": "Kokola", + "kzo": "Kaningi", + "kzp": "Kaidipang", + "kzq": "Kaike", + "kzr": "Karang", + "kzs": "Sugut Dusun", + "kzu": "Kayupulau", + "kzv": "Komyandaret", + "kzw": "Karirí-Xocó", + "kzx": "Kamarian", + "kzy": "Kango (Tshopo District)", + "kzz": "Kalabra", + "la": "Latin", + "laa": "Southern Subanen", + "lab": "Linear A", + "lac": "Lacandon", + "lad": "Ladino", + "lae": "Pattani", + "laf": "Lafofa", + "lag": "Langi", + "lah": "Lahnda", + "lai": "Lambya", + "laj": "Lango (Uganda)", + "lal": "Lalia", + "lam": "Lamba", + "lan": "Laru", + "lap": "Laka (Chad)", + "laq": "Qabiao", + "lar": "Larteh", + "las": "Lama (Togo)", + "lau": "Laba", + "law": "Lauje", + "lax": "Tiwa", + "lay": "Lama Bai", + "laz": "Aribwatsa", + "lb": "Luxembourgish; Letzeburgesch", + "lbb": "Label", + "lbc": "Lakkia", + "lbe": "Lak", + "lbf": "Tinani", + "lbg": "Laopang", + "lbi": "La'bi", + "lbj": "Ladakhi", + "lbk": "Central Bontok", + "lbl": "Libon Bikol", + "lbm": "Lodhi", + "lbn": "Rmeet", + "lbo": "Laven", + "lbq": "Wampar", + "lbr": "Lohorung", + "lbs": "Libyan Sign Language", + "lbt": "Lachi", + "lbu": "Labu", + "lbv": "Lavatbura-Lamusong", + "lbw": "Tolaki", + "lbx": "Lawangan", + "lby": "Lamalama; Lamu-Lamu", + "lbz": "Lardil", + "lcc": "Legenyem", + "lcd": "Lola", + "lce": "Loncong; Sekak", + "lcf": "Lubu", + "lch": "Luchazi", + "lcl": "Lisela", + "lcm": "Tungag", + "lcp": "Western Lawa", + "lcq": "Luhu", + "lcs": "Lisabata-Nuniali", + "lda": "Kla-Dan", + "ldb": "Dũya", + "ldd": "Luri", + "ldg": "Lenyima", + "ldh": "Lamja-Dengsa-Tola", + "ldi": "Laari", + "ldj": "Lemoro", + "ldk": "Leelau", + "ldl": "Kaan", + "ldm": "Landoma", + "ldn": "Láadan", + "ldo": "Loo", + "ldp": "Tso", + "ldq": "Lufu", + "lea": "Lega-Shabunda", + "leb": "Lala-Bisa", + "lec": "Leco", + "led": "Lendu", + "lee": "Lyélé", + "lef": "Lelemi", + "leh": "Lenje", + "lei": "Lemio", + "lej": "Lengola", + "lek": "Leipon", + "lel": "Lele (Democratic Republic of Congo)", + "lem": "Nomaande", + "len": "Lenca", + "leo": "Leti (Cameroon)", + "lep": "Lepcha", + "leq": "Lembena", + "ler": "Lenkau", + "les": "Lese", + "let": "Lesing-Gelimi; Amio-Gelimi", + "leu": "Kara (Papua New Guinea)", + "lev": "Lamma", + "lew": "Ledo Kaili", + "lex": "Luang", + "ley": "Lemolang", + "lez": "Lezghian", + "lfa": "Lefa", + "lfn": "Lingua Franca Nova", + "lg": "Ganda; Luganda", + "lga": "Lungga", + "lgb": "Laghu", + "lgg": "Lugbara", + "lgh": "Laghuu", + "lgi": "Lengilu", + "lgk": "Lingarak; Neverver", + "lgl": "Wala", + "lgm": "Lega-Mwenga", + "lgn": "T'apo; Opuuo", + "lgo": "Lango (South Sudan)", + "lgq": "Logba", + "lgr": "Lengo", + "lgt": "Pahi", + "lgu": "Longgu", + "lgz": "Ligenza", + "lha": "Laha (Viet Nam)", + "lhh": "Laha (Indonesia)", + "lhi": "Lahu Shi", + "lhl": "Lahul Lohar", + "lhm": "Lhomi", + "lhn": "Lahanan", + "lhp": "Lhokpu", + "lhs": "Mlahsö", + "lht": "Lo-Toga", + "lhu": "Lahu", + "li": "Limburgan; Limburger; Limburgish", + "lia": "West-Central Limba", + "lib": "Likum", + "lic": "Hlai", + "lid": "Nyindrou", + "lie": "Likila", + "lif": "Limbu", + "lig": "Ligbi", + "lih": "Lihir", + "lij": "Ligurian", + "lik": "Lika", + "lil": "Lillooet", + "lio": "Liki", + "lip": "Sekpele", + "liq": "Libido", + "lir": "Liberian English", + "lis": "Lisu", + "liu": "Logorik", + "liv": "Liv", + "liw": "Col", + "lix": "Liabuku", + "liy": "Banda-Bambari", + "liz": "Libinza", + "lja": "Golpa", + "lje": "Rampi", + "lji": "Laiyolo", + "ljl": "Li'o", + "ljp": "Lampung Api", + "ljw": "Yirandali", + "ljx": "Yuru", + "lka": "Lakalei", + "lkb": "Kabras; Lukabaras", + "lkc": "Kucong", + "lkd": "Lakondê", + "lke": "Kenyi", + "lkh": "Lakha", + "lki": "Laki", + "lkj": "Remun", + "lkl": "Laeko-Libuat", + "lkm": "Kalaamaya", + "lkn": "Lakon; Vure", + "lko": "Khayo; Olukhayo", + "lkr": "Päri", + "lks": "Kisa; Olushisa", + "lkt": "Lakota", + "lku": "Kungkari", + "lky": "Lokoya", + "lla": "Lala-Roba", + "llb": "Lolo", + "llc": "Lele (Guinea)", + "lld": "Ladin", + "lle": "Lele (Papua New Guinea)", + "llf": "Hermit", + "llg": "Lole", + "llh": "Lamu", + "lli": "Teke-Laali", + "llj": "Ladji Ladji", + "llk": "Lelak", + "lll": "Lilau", + "llm": "Lasalimu", + "lln": "Lele (Chad)", + "llp": "North Efate", + "llq": "Lolak", + "lls": "Lithuanian Sign Language", + "llu": "Lau", + "llx": "Lauan", + "lma": "East Limba", + "lmb": "Merei", + "lmc": "Limilngan", + "lmd": "Lumun", + "lme": "Pévé", + "lmf": "South Lembata", + "lmg": "Lamogai", + "lmh": "Lambichhong", + "lmi": "Lombi", + "lmj": "West Lembata", + "lmk": "Lamkang", + "lml": "Hano", + "lmn": "Lambadi", + "lmo": "Lombard", + "lmp": "Limbum", + "lmq": "Lamatuka", + "lmr": "Lamalera", + "lmu": "Lamenu", + "lmv": "Lomaiviti", + "lmw": "Lake Miwok", + "lmx": "Laimbue", + "lmy": "Lamboya", + "ln": "Lingala", + "lna": "Langbashe", + "lnb": "Mbalanhu", + "lnd": "Lundayeh; Lun Bawang", + "lng": "Langobardic", + "lnh": "Lanoh", + "lni": "Daantanai'", + "lnj": "Leningitij", + "lnl": "South Central Banda", + "lnm": "Langam", + "lnn": "Lorediakarkar", + "lns": "Lamnso'", + "lnu": "Longuda", + "lnw": "Lanima", + "lnz": "Lonzo", + "lo": "Lao", + "loa": "Loloda", + "lob": "Lobi", + "loc": "Inonhan", + "loe": "Saluan", + "lof": "Logol", + "log": "Logo", + "loh": "Narim", + "loi": "Loma (Côte d'Ivoire)", + "loj": "Lou", + "lok": "Loko", + "lol": "Mongo", + "lom": "Loma (Liberia)", + "lon": "Malawi Lomwe", + "loo": "Lombo", + "lop": "Lopa", + "loq": "Lobala", + "lor": "Téén", + "los": "Loniu", + "lot": "Otuho", + "lou": "Louisiana Creole", + "lov": "Lopi", + "low": "Tampias Lobu", + "lox": "Loun", + "loy": "Loke", + "loz": "Lozi", + "lpa": "Lelepa", + "lpe": "Lepki", + "lpn": "Long Phuri Naga", + "lpo": "Lipo", + "lpx": "Lopit", + "lqr": "Logir", + "lra": "Rara Bakati'", + "lrc": "Northern Luri", + "lre": "Laurentian", + "lrg": "Laragia", + "lri": "Marachi; Olumarachi", + "lrk": "Loarki", + "lrl": "Lari", + "lrm": "Marama; Olumarama", + "lrn": "Lorang", + "lro": "Laro", + "lrr": "Southern Yamphu", + "lrt": "Larantuka Malay", + "lrv": "Larevat", + "lrz": "Lemerig", + "lsa": "Lasgerdi", + "lsb": "Burundian Sign Language; Langue des Signes Burundaise", + "lsc": "Albarradas Sign Language; Lengua de señas Albarradas", + "lsd": "Lishana Deni", + "lse": "Lusengo", + "lsh": "Lish", + "lsi": "Lashi", + "lsl": "Latvian Sign Language", + "lsm": "Saamia; Olusamia", + "lsn": "Tibetan Sign Language", + "lso": "Laos Sign Language", + "lsp": "Panamanian Sign Language; Lengua de Señas Panameñas", + "lsr": "Aruop", + "lss": "Lasi", + "lst": "Trinidad and Tobago Sign Language", + "lsv": "Sivia Sign Language", + "lsw": "Seychelles Sign Language; Lalang Siny Seselwa; Langue des Signes Seychelloise", + "lsy": "Mauritian Sign Language", + "lt": "Lithuanian", + "ltc": "Late Middle Chinese", + "ltg": "Latgalian", + "lth": "Thur", + "lti": "Leti (Indonesia)", + "ltn": "Latundê", + "lto": "Tsotso; Olutsotso", + "lts": "Tachoni; Lutachoni", + "ltu": "Latu", + "lu": "Luba-Katanga", + "lua": "Luba-Lulua", + "luc": "Aringa", + "lud": "Ludian", + "lue": "Luvale", + "luf": "Laua", + "lui": "Luiseno", + "luj": "Luna", + "luk": "Lunanakha", + "lul": "Olu'bo", + "lum": "Luimbi", + "lun": "Lunda", + "luo": "Luo (Kenya and Tanzania); Dholuo", + "lup": "Lumbu", + "luq": "Lucumi", + "lur": "Laura", + "lus": "Lushai", + "lut": "Lushootseed", + "luu": "Lumba-Yakkha", + "luv": "Luwati", + "luw": "Luo (Cameroon)", + "luy": "Luyia; Oluluyia", + "luz": "Southern Luri", + "lv": "Latvian", + "lva": "Maku'a", + "lvi": "Lavi", + "lvk": "Lavukaleve", + "lvs": "Standard Latvian", + "lvu": "Levuka", + "lwa": "Lwalu", + "lwe": "Lewo Eleng", + "lwg": "Wanga; Oluwanga", + "lwh": "White Lachi", + "lwl": "Eastern Lawa", + "lwm": "Laomian", + "lwo": "Luwo", + "lws": "Malawian Sign Language", + "lwt": "Lewotobi", + "lwu": "Lawu", + "lww": "Lewo", + "lxm": "Lakurumau", + "lya": "Layakha", + "lyg": "Lyngngam", + "lyn": "Luyana", + "lzh": "Literary Chinese", + "lzl": "Litzlitz", + "lzn": "Leinong Naga", + "lzz": "Laz", + "maa": "San Jerónimo Tecóatl Mazatec", + "mab": "Yutanduchi Mixtec", + "mad": "Madurese", + "mae": "Bo-Rukul", + "maf": "Mafa", + "mag": "Magahi", + "mai": "Maithili", + "maj": "Jalapa De Díaz Mazatec", + "mak": "Makasar", + "mam": "Mam", + "man": "Mandingo; Manding", + "map": "Austronesian languages", + "maq": "Chiquihuitlán Mazatec", + "mas": "Masai", + "mat": "San Francisco Matlatzinca", + "mau": "Huautla Mazatec", + "mav": "Sateré-Mawé", + "maw": "Mampruli", + "max": "North Moluccan Malay", + "maz": "Central Mazahua", + "mba": "Higaonon", + "mbb": "Western Bukidnon Manobo", + "mbc": "Macushi", + "mbd": "Dibabawon Manobo", + "mbe": "Molale", + "mbf": "Baba Malay", + "mbh": "Mangseng", + "mbi": "Ilianen Manobo", + "mbj": "Nadëb", + "mbk": "Malol", + "mbl": "Maxakalí", + "mbm": "Ombamba", + "mbn": "Macaguán", + "mbo": "Mbo (Cameroon)", + "mbp": "Malayo", + "mbq": "Maisin", + "mbr": "Nukak Makú", + "mbs": "Sarangani Manobo", + "mbt": "Matigsalug Manobo", + "mbu": "Mbula-Bwazza", + "mbv": "Mbulungish", + "mbw": "Maring", + "mbx": "Mari (East Sepik Province)", + "mby": "Memoni", + "mbz": "Amoltepec Mixtec", + "mca": "Maca", + "mcb": "Machiguenga", + "mcc": "Bitur", + "mcd": "Sharanahua", + "mce": "Itundujia Mixtec", + "mcf": "Matsés", + "mcg": "Mapoyo", + "mch": "Maquiritari", + "mci": "Mese", + "mcj": "Mvanip", + "mck": "Mbunda", + "mcl": "Macaguaje", + "mcm": "Malaccan Creole Portuguese", + "mcn": "Masana", + "mco": "Coatlán Mixe", + "mcp": "Makaa", + "mcq": "Ese", + "mcr": "Menya", + "mcs": "Mambai", + "mct": "Mengisa", + "mcu": "Cameroon Mambila", + "mcv": "Minanibai", + "mcw": "Mawa (Chad)", + "mcx": "Mpiemo", + "mcy": "South Watut", + "mcz": "Mawan", + "mda": "Mada (Nigeria)", + "mdb": "Morigi", + "mdc": "Male (Papua New Guinea)", + "mdd": "Mbum", + "mde": "Maba (Chad)", + "mdf": "Moksha", + "mdg": "Massalat", + "mdh": "Maguindanaon", + "mdi": "Mamvu", + "mdj": "Mangbetu", + "mdk": "Mangbutu", + "mdl": "Maltese Sign Language", + "mdm": "Mayogo", + "mdn": "Mbati", + "mdp": "Mbala", + "mdq": "Mbole", + "mdr": "Mandar", + "mds": "Maria (Papua New Guinea)", + "mdt": "Mbere", + "mdu": "Mboko", + "mdv": "Santa Lucía Monteverde Mixtec", + "mdw": "Mbosi", + "mdx": "Dizin", + "mdy": "Male (Ethiopia)", + "mdz": "Suruí Do Pará", + "mea": "Menka", + "meb": "Ikobi", + "mec": "Marra", + "med": "Melpa", + "mee": "Mengen", + "mef": "Megam", + "meh": "Southwestern Tlaxiaco Mixtec", + "mei": "Midob", + "mej": "Meyah", + "mek": "Mekeo", + "mel": "Central Melanau", + "mem": "Mangala", + "men": "Mende (Sierra Leone)", + "meo": "Kedah Malay", + "mep": "Miriwoong", + "meq": "Merey", + "mer": "Meru", + "mes": "Masmaje", + "met": "Mato", + "meu": "Motu", + "mev": "Mano", + "mew": "Maaka", + "mey": "Hassaniyya", + "mez": "Menominee", + "mfa": "Pattani Malay", + "mfb": "Bangka", + "mfc": "Mba", + "mfd": "Mendankwe-Nkwen", + "mfe": "Morisyen", + "mff": "Naki", + "mfg": "Mogofin", + "mfh": "Matal", + "mfi": "Wandala", + "mfj": "Mefele", + "mfk": "North Mofu", + "mfl": "Putai", + "mfm": "Marghi South", + "mfn": "Cross River Mbembe", + "mfo": "Mbe", + "mfp": "Makassar Malay", + "mfq": "Moba", + "mfr": "Marrithiyel", + "mfs": "Mexican Sign Language", + "mft": "Mokerang", + "mfu": "Mbwela", + "mfv": "Mandjak", + "mfw": "Mulaha", + "mfx": "Melo", + "mfy": "Mayo", + "mfz": "Mabaan", + "mg": "Malagasy", + "mga": "Middle Irish (900-1200)", + "mgb": "Mararit", + "mgc": "Morokodo", + "mgd": "Moru", + "mge": "Mango", + "mgf": "Maklew", + "mgg": "Mpumpong", + "mgh": "Makhuwa-Meetto", + "mgi": "Lijili", + "mgj": "Abureni", + "mgk": "Mawes", + "mgl": "Maleu-Kilenge", + "mgm": "Mambae", + "mgn": "Mbangi", + "mgo": "Meta'", + "mgp": "Eastern Magar", + "mgq": "Malila", + "mgr": "Mambwe-Lungu", + "mgs": "Manda (Tanzania)", + "mgt": "Mongol", + "mgu": "Mailu", + "mgv": "Matengo", + "mgw": "Matumbi", + "mgy": "Mbunga", + "mgz": "Mbugwe", + "mh": "Marshallese", + "mha": "Manda (India)", + "mhb": "Mahongwe", + "mhc": "Mocho", + "mhd": "Mbugu", + "mhe": "Besisi; Mah Meri", + "mhf": "Mamaa", + "mhg": "Margu", + "mhi": "Ma'di", + "mhj": "Mogholi", + "mhk": "Mungaka", + "mhl": "Mauwake", + "mhm": "Makhuwa-Moniga", + "mhn": "Mócheno", + "mho": "Mashi (Zambia)", + "mhp": "Balinese Malay", + "mhq": "Mandan", + "mhr": "Eastern Mari", + "mhs": "Buru (Indonesia)", + "mht": "Mandahuaca", + "mhu": "Digaro-Mishmi; Darang Deng", + "mhw": "Mbukushu", + "mhx": "Maru; Lhaovo", + "mhy": "Ma'anyan", + "mhz": "Mor (Mor Islands)", + "mi": "Maori", + "mia": "Miami", + "mib": "Atatláhuca Mixtec", + "mic": "Mi'kmaq; Micmac", + "mid": "Mandaic", + "mie": "Ocotepec Mixtec", + "mif": "Mofu-Gudur", + "mig": "San Miguel El Grande Mixtec", + "mih": "Chayuco Mixtec", + "mii": "Chigmecatitlán Mixtec", + "mij": "Abar; Mungbam", + "mik": "Mikasuki", + "mil": "Peñoles Mixtec", + "mim": "Alacatlatzala Mixtec", + "min": "Minangkabau", + "mio": "Pinotepa Nacional Mixtec", + "mip": "Apasco-Apoala Mixtec", + "miq": "Mískito", + "mir": "Isthmus Mixe", + "mit": "Southern Puebla Mixtec", + "miu": "Cacaloxtepec Mixtec", + "miw": "Akoye", + "mix": "Mixtepec Mixtec", + "miy": "Ayutla Mixtec", + "miz": "Coatzospan Mixtec", + "mjb": "Makalero", + "mjc": "San Juan Colorado Mixtec", + "mjd": "Northwest Maidu", + "mje": "Muskum", + "mjg": "Tu", + "mjh": "Mwera (Nyasa)", + "mji": "Kim Mun", + "mjj": "Mawak", + "mjk": "Matukar", + "mjl": "Mandeali", + "mjm": "Medebur", + "mjn": "Ma (Papua New Guinea)", + "mjo": "Malankuravan", + "mjp": "Malapandaram", + "mjq": "Malaryan", + "mjr": "Malavedan", + "mjs": "Miship", + "mjt": "Sauria Paharia", + "mju": "Manna-Dora", + "mjv": "Mannan", + "mjw": "Karbi", + "mjx": "Mahali", + "mjy": "Mahican", + "mjz": "Majhi", + "mk": "Macedonian", + "mka": "Mbre", + "mkb": "Mal Paharia", + "mkc": "Siliput", + "mke": "Mawchi", + "mkf": "Miya", + "mkg": "Mak (China)", + "mkh": "Mon-Khmer languages", + "mki": "Dhatki", + "mkj": "Mokilese", + "mkk": "Byep", + "mkl": "Mokole", + "mkm": "Moklen", + "mkn": "Kupang Malay", + "mko": "Mingang Doso", + "mkp": "Moikodi", + "mkq": "Bay Miwok", + "mkr": "Malas", + "mks": "Silacayoapan Mixtec", + "mkt": "Vamale", + "mku": "Konyanka Maninka", + "mkv": "Mafea", + "mkw": "Kituba (Congo)", + "mkx": "Kinamiging Manobo", + "mky": "East Makian", + "mkz": "Makasae", + "ml": "Malayalam", + "mla": "Malo", + "mlb": "Mbule", + "mlc": "Cao Lan", + "mle": "Manambu", + "mlf": "Mal", + "mlh": "Mape", + "mli": "Malimpung", + "mlj": "Miltu", + "mlk": "Ilwana; Kiwilwana", + "mll": "Malua Bay", + "mlm": "Mulam", + "mln": "Malango", + "mlo": "Mlomp", + "mlp": "Bargam", + "mlq": "Western Maninkakan", + "mlr": "Vame", + "mls": "Masalit", + "mlu": "To'abaita", + "mlv": "Motlav; Mwotlap", + "mlw": "Moloko", + "mlx": "Malfaxal; Naha'ai", + "mlz": "Malaynon", + "mma": "Mama", + "mmb": "Momina", + "mmc": "Michoacán Mazahua", + "mmd": "Maonan", + "mme": "Mae", + "mmf": "Mundat", + "mmg": "North Ambrym", + "mmh": "Mehináku", + "mmi": "Musar", + "mmj": "Majhwar", + "mmk": "Mukha-Dora", + "mml": "Man Met", + "mmm": "Maii", + "mmn": "Mamanwa", + "mmo": "Mangga Buang", + "mmp": "Siawi", + "mmq": "Musak", + "mmr": "Western Xiangxi Miao", + "mmt": "Malalamai", + "mmu": "Mmaala", + "mmv": "Miriti", + "mmw": "Emae", + "mmx": "Madak", + "mmy": "Migaama", + "mmz": "Mabaale", + "mn": "Mongolian", + "mna": "Mbula", + "mnb": "Muna", + "mnc": "Manchu", + "mnd": "Mondé", + "mne": "Naba", + "mnf": "Mundani", + "mng": "Eastern Mnong", + "mnh": "Mono (Democratic Republic of Congo)", + "mni": "Manipuri", + "mnj": "Munji", + "mnk": "Mandinka", + "mnl": "Tiale", + "mnm": "Mapena", + "mnn": "Southern Mnong", + "mno": "Manobo languages", + "mnp": "Min Bei Chinese", + "mnq": "Minriq", + "mnr": "Mono (USA)", + "mns": "Mansi", + "mnu": "Mer", + "mnv": "Rennell-Bellona", + "mnw": "Mon", + "mnx": "Manikion", + "mny": "Manyawa", + "mnz": "Moni", + "moa": "Mwan", + "moc": "Mocoví", + "mod": "Mobilian", + "moe": "Innu; Montagnais", + "mog": "Mongondow", + "moh": "Mohawk", + "moi": "Mboi", + "moj": "Monzombo", + "mok": "Morori", + "mom": "Mangue", + "moo": "Monom", + "mop": "Mopán Maya", + "moq": "Mor (Bomberai Peninsula)", + "mor": "Moro", + "mos": "Mossi", + "mot": "Barí", + "mou": "Mogum", + "mov": "Mohave", + "mow": "Moi (Congo)", + "mox": "Molima", + "moy": "Shekkacho", + "moz": "Mukulu; Gergiko", + "mpa": "Mpoto", + "mpb": "Malak Malak; Mullukmulluk", + "mpc": "Mangarrayi", + "mpd": "Machinere", + "mpe": "Majang", + "mpg": "Marba", + "mph": "Maung", + "mpi": "Mpade", + "mpj": "Martu Wangka; Wangkajunga", + "mpk": "Mbara (Chad)", + "mpl": "Middle Watut", + "mpm": "Yosondúa Mixtec", + "mpn": "Mindiri", + "mpo": "Miu", + "mpp": "Migabac", + "mpq": "Matís", + "mpr": "Vangunu", + "mps": "Dadibi", + "mpt": "Mian", + "mpu": "Makuráp", + "mpv": "Mungkip", + "mpw": "Mapidian", + "mpx": "Misima-Panaeati", + "mpy": "Mapia", + "mpz": "Mpi", + "mqa": "Maba (Indonesia)", + "mqb": "Mbuko", + "mqc": "Mangole", + "mqe": "Matepi", + "mqf": "Momuna", + "mqg": "Kota Bangun Kutai Malay", + "mqh": "Tlazoyaltepec Mixtec", + "mqi": "Mariri", + "mqj": "Mamasa", + "mqk": "Rajah Kabunsuwan Manobo", + "mql": "Mbelime", + "mqm": "South Marquesan", + "mqn": "Moronene", + "mqo": "Modole", + "mqp": "Manipa", + "mqq": "Minokok", + "mqr": "Mander", + "mqs": "West Makian", + "mqt": "Mok", + "mqu": "Mandari", + "mqv": "Mosimo", + "mqw": "Murupi", + "mqx": "Mamuju", + "mqy": "Manggarai", + "mqz": "Pano", + "mr": "Marathi", + "mra": "Mlabri", + "mrb": "Marino", + "mrc": "Maricopa", + "mrd": "Western Magar", + "mre": "Martha's Vineyard Sign Language", + "mrf": "Elseng", + "mrg": "Mising", + "mrh": "Mara Chin", + "mrj": "Western Mari", + "mrk": "Hmwaveke", + "mrl": "Mortlockese", + "mrm": "Merlav; Mwerlap", + "mrn": "Cheke Holo", + "mro": "Mru", + "mrp": "Morouas", + "mrq": "North Marquesan", + "mrr": "Maria (India)", + "mrs": "Maragus", + "mrt": "Marghi Central", + "mru": "Mono (Cameroon)", + "mrv": "Mangareva", + "mrw": "Maranao", + "mrx": "Maremgi; Dineor", + "mry": "Mandaya", + "mrz": "Marind", + "ms": "Malay (macrolanguage)", + "msb": "Masbatenyo", + "msc": "Sankaran Maninka", + "msd": "Yucatec Maya Sign Language", + "mse": "Musey", + "msf": "Mekwei", + "msg": "Moraid", + "msh": "Masikoro Malagasy", + "msi": "Sabah Malay", + "msj": "Ma (Democratic Republic of Congo)", + "msk": "Mansaka", + "msl": "Molof; Poule", + "msm": "Agusan Manobo", + "msn": "Vurës", + "mso": "Mombum", + "msp": "Maritsauá", + "msq": "Caac", + "msr": "Mongolian Sign Language", + "mss": "West Masela", + "msu": "Musom", + "msv": "Maslam", + "msw": "Mansoanka", + "msx": "Moresada", + "msy": "Aruamu", + "msz": "Momare", + "mt": "Maltese", + "mta": "Cotabato Manobo", + "mtb": "Anyin Morofo", + "mtc": "Munit", + "mtd": "Mualang", + "mte": "Mono (Solomon Islands)", + "mtf": "Murik (Papua New Guinea)", + "mtg": "Una", + "mth": "Munggui", + "mti": "Maiwa (Papua New Guinea)", + "mtj": "Moskona", + "mtk": "Mbe'", + "mtl": "Montol", + "mtm": "Mator", + "mtn": "Matagalpa", + "mto": "Totontepec Mixe", + "mtp": "Wichí Lhamtés Nocten", + "mtq": "Muong", + "mtr": "Mewari", + "mts": "Yora", + "mtt": "Mota", + "mtu": "Tututepec Mixtec", + "mtv": "Asaro'o", + "mtw": "Southern Binukidnon", + "mtx": "Tidaá Mixtec", + "mty": "Nabi", + "mua": "Mundang", + "mub": "Mubi", + "muc": "Ajumbu", + "mud": "Mednyj Aleut", + "mue": "Media Lengua", + "mug": "Musgu", + "muh": "Mündü", + "mui": "Musi", + "muj": "Mabire", + "muk": "Mugom", + "mum": "Maiwala", + "mun": "Munda languages", + "muo": "Nyong", + "mup": "Malvi", + "muq": "Eastern Xiangxi Miao", + "mur": "Murle", + "mus": "Creek", + "mut": "Western Muria", + "muu": "Yaaku", + "muv": "Muthuvan", + "mux": "Bo-Ung", + "muy": "Muyang", + "muz": "Mursi", + "mva": "Manam", + "mvb": "Mattole", + "mvd": "Mamboru", + "mve": "Marwari (Pakistan)", + "mvf": "Peripheral Mongolian", + "mvg": "Yucuañe Mixtec", + "mvh": "Mulgi", + "mvi": "Miyako", + "mvk": "Mekmek", + "mvl": "Mbara (Australia)", + "mvn": "Minaveha", + "mvo": "Marovo", + "mvp": "Duri", + "mvq": "Moere", + "mvr": "Marau", + "mvs": "Massep", + "mvt": "Mpotovoro", + "mvu": "Marfa", + "mvv": "Tagal Murut", + "mvw": "Machinga", + "mvx": "Meoswar", + "mvy": "Indus Kohistani", + "mvz": "Mesqan", + "mwa": "Mwatebu", + "mwb": "Juwal", + "mwc": "Are", + "mwe": "Mwera (Chimwera)", + "mwf": "Murrinh-Patha", + "mwg": "Aiklep", + "mwh": "Mouk-Aria", + "mwi": "Labo; Ninde", + "mwk": "Kita Maninkakan", + "mwl": "Mirandese", + "mwm": "Sar", + "mwn": "Nyamwanga", + "mwo": "Central Maewo", + "mwp": "Kala Lagaw Ya", + "mwq": "Mün Chin", + "mwr": "Marwari", + "mws": "Mwimbi-Muthambi", + "mwt": "Moken", + "mwu": "Mittu", + "mwv": "Mentawai", + "mww": "Hmong Daw", + "mwz": "Moingi", + "mxa": "Northwest Oaxaca Mixtec", + "mxb": "Tezoatlán Mixtec", + "mxc": "Manyika", + "mxd": "Modang", + "mxe": "Mele-Fila", + "mxf": "Malgbe", + "mxg": "Mbangala", + "mxh": "Mvuba", + "mxi": "Mozarabic", + "mxj": "Miju-Mishmi; Geman Deng", + "mxk": "Monumbo", + "mxl": "Maxi Gbe", + "mxm": "Meramera", + "mxn": "Moi (Indonesia)", + "mxo": "Mbowe", + "mxp": "Tlahuitoltepec Mixe", + "mxq": "Juquila Mixe", + "mxr": "Murik (Malaysia)", + "mxs": "Huitepec Mixtec", + "mxt": "Jamiltepec Mixtec", + "mxu": "Mada (Cameroon)", + "mxv": "Metlatónoc Mixtec", + "mxw": "Namo", + "mxx": "Mahou; Mawukakan", + "mxy": "Southeastern Nochixtlán Mixtec", + "mxz": "Central Masela", + "my": "Burmese", + "myb": "Mbay", + "myc": "Mayeka", + "mye": "Myene", + "myf": "Bambassi", + "myg": "Manta", + "myh": "Makah", + "myj": "Mangayat", + "myk": "Mamara Senoufo", + "myl": "Moma", + "mym": "Me'en", + "myn": "Mayan languages", + "myo": "Anfillo", + "myp": "Pirahã", + "myr": "Muniche", + "mys": "Mesmes", + "myu": "Mundurukú", + "myv": "Erzya", + "myw": "Muyuw", + "myx": "Masaaba", + "myy": "Macuna", + "myz": "Classical Mandaic", + "mza": "Santa María Zacatepec Mixtec", + "mzb": "Tumzabt", + "mzc": "Madagascar Sign Language", + "mzd": "Malimba", + "mze": "Morawa", + "mzg": "Monastic Sign Language", + "mzh": "Wichí Lhamtés Güisnay", + "mzi": "Ixcatlán Mazatec", + "mzj": "Manya", + "mzk": "Nigeria Mambila", + "mzl": "Mazatlán Mixe", + "mzm": "Mumuye", + "mzn": "Mazanderani", + "mzo": "Matipuhy", + "mzp": "Movima", + "mzq": "Mori Atas", + "mzr": "Marúbo", + "mzs": "Macanese", + "mzt": "Mintil", + "mzu": "Inapang", + "mzv": "Manza", + "mzw": "Deg", + "mzx": "Mawayana", + "mzy": "Mozambican Sign Language", + "mzz": "Maiadomu", + "na": "Nauru", + "naa": "Namla", + "nab": "Southern Nambikuára", + "nac": "Narak", + "nae": "Naka'ela", + "naf": "Nabak", + "nag": "Naga Pidgin", + "nah": "Nahuatl languages", + "nai": "North American Indian languages", + "naj": "Nalu", + "nak": "Nakanai", + "nal": "Nalik", + "nam": "Ngan'gityemerri", + "nan": "Min Nan Chinese", + "nao": "Naaba", + "nap": "Neapolitan", + "naq": "Khoekhoe; Nama (Namibia)", + "nar": "Iguta", + "nas": "Naasioi", + "nat": "Ca̱hungwa̱rya̱; Hungworo", + "naw": "Nawuri", + "nax": "Nakwi", + "nay": "Ngarrindjeri", + "naz": "Coatepec Nahuatl", + "nb": "Norwegian Bokmål", + "nba": "Nyemba", + "nbb": "Ndoe", + "nbc": "Chang Naga", + "nbd": "Ngbinda", + "nbe": "Konyak Naga", + "nbg": "Nagarchal", + "nbh": "Ngamo", + "nbi": "Mao Naga", + "nbj": "Ngarinyman", + "nbk": "Nake", + "nbm": "Ngbaka Ma'bo", + "nbn": "Kuri", + "nbo": "Nkukoli", + "nbp": "Nnam", + "nbq": "Nggem", + "nbr": "Numana", + "nbs": "Namibian Sign Language", + "nbt": "Na", + "nbu": "Rongmei Naga", + "nbv": "Ngamambo", + "nbw": "Southern Ngbandi", + "nby": "Ningera", + "nca": "Iyo", + "ncb": "Central Nicobarese", + "ncc": "Ponam", + "ncd": "Nachering", + "nce": "Yale", + "ncf": "Notsi", + "ncg": "Nisga'a", + "nch": "Central Huasteca Nahuatl", + "nci": "Classical Nahuatl", + "ncj": "Northern Puebla Nahuatl", + "nck": "Na-kara", + "ncl": "Michoacán Nahuatl", + "ncm": "Nambo", + "ncn": "Nauna", + "nco": "Sibe", + "ncq": "Northern Katang", + "ncr": "Ncane", + "ncs": "Nicaraguan Sign Language", + "nct": "Chothe Naga", + "ncu": "Chumburung", + "ncx": "Central Puebla Nahuatl", + "ncz": "Natchez", + "nd": "North Ndebele", + "nda": "Ndasa", + "ndb": "Kenswei Nsei", + "ndc": "Ndau", + "ndd": "Nde-Nsele-Nta", + "ndf": "Nadruvian", + "ndg": "Ndengereko", + "ndh": "Ndali", + "ndi": "Samba Leko", + "ndj": "Ndamba", + "ndk": "Ndaka", + "ndl": "Ndolo", + "ndm": "Ndam", + "ndn": "Ngundi", + "ndp": "Ndo", + "ndq": "Ndombe", + "ndr": "Ndoola", + "nds": "Low German; Low Saxon", + "ndt": "Ndunga", + "ndu": "Dugun", + "ndv": "Ndut", + "ndw": "Ndobo", + "ndx": "Nduga", + "ndy": "Lutos", + "ndz": "Ndogo", + "ne": "Nepali (macrolanguage)", + "nea": "Eastern Ngad'a", + "neb": "Toura (Côte d'Ivoire)", + "nec": "Nedebang", + "ned": "Nde-Gbite", + "nee": "Nêlêmwa-Nixumwak", + "nef": "Nefamese", + "neg": "Negidal", + "neh": "Nyenkha", + "nei": "Neo-Hittite", + "nej": "Neko", + "nek": "Neku", + "nem": "Nemi", + "nen": "Nengone", + "neo": "Ná-Meo", + "neq": "North Central Mixe", + "ner": "Yahadian", + "nes": "Bhoti Kinnauri", + "net": "Nete", + "neu": "Neo", + "nev": "Nyaheun", + "new": "Newari; Nepal Bhasa", + "nex": "Neme", + "ney": "Neyo", + "nez": "Nez Perce", + "nfa": "Dhao", + "nfd": "Ahwai", + "nfl": "Ayiwo; Äiwoo", + "nfr": "Nafaanra", + "nfu": "Mfumte", + "ng": "Ndonga", + "nga": "Ngbaka", + "ngb": "Northern Ngbandi", + "ngc": "Ngombe (Democratic Republic of Congo)", + "ngd": "Ngando (Central African Republic)", + "nge": "Ngemba", + "ngf": "Trans-New Guinea languages", + "ngg": "Ngbaka Manza", + "ngh": "Nǁng", + "ngi": "Ngizim", + "ngj": "Ngie", + "ngk": "Dalabon", + "ngl": "Lomwe", + "ngm": "Ngatik Men's Creole", + "ngn": "Ngwo", + "ngp": "Ngulu", + "ngq": "Ngurimi; Ngoreme", + "ngr": "Engdewu", + "ngs": "Gvoko", + "ngt": "Kriang; Ngeq", + "ngu": "Guerrero Nahuatl", + "ngv": "Nagumi", + "ngw": "Ngwaba", + "ngx": "Nggwahyi", + "ngy": "Tibea", + "ngz": "Ngungwel", + "nha": "Nhanda", + "nhb": "Beng", + "nhc": "Tabasco Nahuatl", + "nhd": "Chiripá; Ava Guaraní", + "nhe": "Eastern Huasteca Nahuatl", + "nhf": "Nhuwala", + "nhg": "Tetelcingo Nahuatl", + "nhh": "Nahari", + "nhi": "Zacatlán-Ahuacatlán-Tepetzintla Nahuatl", + "nhk": "Isthmus-Cosoleacaque Nahuatl", + "nhm": "Morelos Nahuatl", + "nhn": "Central Nahuatl", + "nho": "Takuu", + "nhp": "Isthmus-Pajapan Nahuatl", + "nhq": "Huaxcaleca Nahuatl", + "nhr": "Naro", + "nht": "Ometepec Nahuatl", + "nhu": "Noone", + "nhv": "Temascaltepec Nahuatl", + "nhw": "Western Huasteca Nahuatl", + "nhx": "Isthmus-Mecayapan Nahuatl", + "nhy": "Northern Oaxaca Nahuatl", + "nhz": "Santa María La Alta Nahuatl", + "nia": "Nias", + "nib": "Nakame", + "nic": "Niger-Kordofanian languages", + "nid": "Ngandi", + "nie": "Niellim", + "nif": "Nek", + "nig": "Ngalakgan", + "nih": "Nyiha (Tanzania)", + "nii": "Nii", + "nij": "Ngaju", + "nik": "Southern Nicobarese", + "nil": "Nila", + "nim": "Nilamba", + "nin": "Ninzo", + "nio": "Nganasan", + "niq": "Nandi", + "nir": "Nimboran", + "nis": "Nimi", + "nit": "Southeastern Kolami", + "niu": "Niuean", + "niv": "Gilyak", + "niw": "Nimo", + "nix": "Hema", + "niy": "Ngiti", + "niz": "Ningil", + "nja": "Nzanyi", + "njb": "Nocte Naga", + "njd": "Ndonde Hamba", + "njh": "Lotha Naga", + "nji": "Gudanji", + "njj": "Njen", + "njl": "Njalgulgule", + "njm": "Angami Naga", + "njn": "Liangmai Naga", + "njo": "Ao Naga", + "njr": "Njerep", + "njs": "Nisa", + "njt": "Ndyuka-Trio Pidgin", + "nju": "Ngadjunmaya", + "njx": "Kunyi", + "njy": "Njyem", + "njz": "Nyishi", + "nka": "Nkoya", + "nkb": "Khoibu Naga", + "nkc": "Nkongho", + "nkd": "Koireng", + "nke": "Duke", + "nkf": "Inpui Naga", + "nkg": "Nekgini", + "nkh": "Khezha Naga", + "nki": "Thangal Naga", + "nkj": "Nakai", + "nkk": "Nokuku", + "nkm": "Namat", + "nkn": "Nkangala", + "nko": "Nkonya", + "nkp": "Niuatoputapu", + "nkq": "Nkami", + "nkr": "Nukuoro", + "nks": "North Asmat", + "nkt": "Nyika (Tanzania)", + "nku": "Bouna Kulango", + "nkv": "Nyika (Malawi and Zambia)", + "nkw": "Nkutu", + "nkx": "Nkoroo", + "nkz": "Nkari", + "nl": "Dutch; Flemish", + "nla": "Ngombale", + "nlc": "Nalca", + "nle": "East Nyala", + "nlg": "Gela", + "nli": "Grangali", + "nlj": "Nyali", + "nlk": "Ninia Yali", + "nll": "Nihali", + "nlm": "Mankiyali", + "nlo": "Ngul", + "nlq": "Lao Naga", + "nlu": "Nchumbulu", + "nlv": "Orizaba Nahuatl", + "nlw": "Walangama", + "nlx": "Nahali", + "nly": "Nyamal", + "nlz": "Nalögo", + "nma": "Maram Naga", + "nmb": "Big Nambas; V'ënen Taut", + "nmc": "Ngam", + "nmd": "Ndumu", + "nme": "Mzieme Naga", + "nmf": "Tangkhul Naga (India)", + "nmg": "Kwasio", + "nmh": "Monsang Naga", + "nmi": "Nyam", + "nmj": "Ngombe (Central African Republic)", + "nmk": "Namakura", + "nml": "Ndemli", + "nmm": "Manangba", + "nmn": "ǃXóõ", + "nmo": "Moyon Naga", + "nmp": "Nimanbur", + "nmq": "Nambya", + "nmr": "Nimbari", + "nms": "Letemboi", + "nmt": "Namonuito", + "nmu": "Northeast Maidu", + "nmv": "Ngamini", + "nmw": "Nimoa; Rifao", + "nmx": "Nama (Papua New Guinea)", + "nmy": "Namuyi", + "nmz": "Nawdm", + "nn": "Norwegian Nynorsk", + "nna": "Nyangumarta", + "nnb": "Nande", + "nnc": "Nancere", + "nnd": "West Ambae", + "nne": "Ngandyera", + "nnf": "Ngaing", + "nng": "Maring Naga", + "nnh": "Ngiemboon", + "nni": "North Nuaulu", + "nnj": "Nyangatom", + "nnk": "Nankina", + "nnl": "Northern Rengma Naga", + "nnm": "Namia", + "nnn": "Ngete", + "nnp": "Wancho Naga", + "nnq": "Ngindo", + "nnr": "Narungga", + "nnt": "Nanticoke", + "nnu": "Dwang", + "nnv": "Nugunu (Australia)", + "nnw": "Southern Nuni", + "nny": "Nyangga", + "nnz": "Nda'nda'", + "no": "Norwegian", + "noa": "Woun Meu", + "noc": "Nuk", + "nod": "Northern Thai", + "noe": "Nimadi", + "nof": "Nomane", + "nog": "Nogai", + "noh": "Nomu", + "noi": "Noiri", + "noj": "Nonuya", + "nok": "Nooksack", + "nol": "Nomlaki", + "nom": "Nocamán", + "non": "Old Norse", + "nop": "Numanggang", + "noq": "Ngongo", + "nos": "Eastern Nisu", + "not": "Nomatsiguenga", + "nou": "Ewage-Notu", + "nov": "Novial", + "now": "Nyambo", + "noy": "Noy", + "noz": "Nayi", + "npa": "Nar Phu", + "npb": "Nupbikha", + "npg": "Ponyo-Gongwang Naga", + "nph": "Phom Naga", + "npi": "Nepali (individual language)", + "npl": "Southeastern Puebla Nahuatl", + "npn": "Mondropolon", + "npo": "Pochuri Naga", + "nps": "Nipsan", + "npu": "Puimei Naga", + "npx": "Noipx", + "npy": "Napu", + "nqg": "Southern Nago", + "nqk": "Kura Ede Nago", + "nql": "Ngendelengo", + "nqm": "Ndom", + "nqn": "Nen", + "nqo": "N'Ko; N’Ko", + "nqq": "Kyan-Karyaw Naga", + "nqt": "Nteng", + "nqy": "Akyaung Ari Naga", + "nr": "South Ndebele", + "nra": "Ngom", + "nrb": "Nara", + "nrc": "Noric", + "nre": "Southern Rengma Naga", + "nrf": "Jèrriais; Guernésiais", + "nrg": "Narango", + "nri": "Chokri Naga", + "nrk": "Ngarla", + "nrl": "Ngarluma", + "nrm": "Narom", + "nrn": "Norn", + "nrp": "North Picene", + "nrr": "Norra; Nora", + "nrt": "Northern Kalapuya", + "nru": "Narua", + "nrx": "Ngurmbur", + "nrz": "Lala", + "nsa": "Sangtam Naga", + "nsb": "Lower Nossob", + "nsc": "Nshi", + "nsd": "Southern Nisu", + "nse": "Nsenga", + "nsf": "Northwestern Nisu", + "nsg": "Ngasa", + "nsh": "Ngoshie", + "nsi": "Nigerian Sign Language", + "nsk": "Naskapi", + "nsl": "Norwegian Sign Language", + "nsm": "Sumi Naga", + "nsn": "Nehan", + "nso": "Pedi; Northern Sotho; Sepedi", + "nsp": "Nepalese Sign Language", + "nsq": "Northern Sierra Miwok", + "nsr": "Maritime Sign Language", + "nss": "Nali", + "nst": "Tase Naga", + "nsu": "Sierra Negra Nahuatl", + "nsv": "Southwestern Nisu", + "nsw": "Navut", + "nsx": "Nsongo", + "nsy": "Nasal", + "nsz": "Nisenan", + "ntd": "Northern Tidung", + "nte": "Nathembo", + "ntg": "Ngantangarra", + "nti": "Natioro", + "ntj": "Ngaanyatjarra", + "ntk": "Ikoma-Nata-Isenye", + "ntm": "Nateni", + "nto": "Ntomba", + "ntp": "Northern Tepehuan", + "ntr": "Delo", + "ntu": "Natügu", + "ntw": "Nottoway", + "ntx": "Tangkhul Naga (Myanmar)", + "nty": "Mantsi", + "ntz": "Natanzi", + "nua": "Yuanga", + "nub": "Nubian languages", + "nuc": "Nukuini", + "nud": "Ngala", + "nue": "Ngundu", + "nuf": "Nusu", + "nug": "Nungali", + "nuh": "Ndunda", + "nui": "Ngumbi", + "nuj": "Nyole", + "nuk": "Nuu-chah-nulth; Nuuchahnulth", + "nul": "Nusa Laut", + "num": "Niuafo'ou", + "nun": "Anong", + "nuo": "Nguôn", + "nup": "Nupe-Nupe-Tako", + "nuq": "Nukumanu", + "nur": "Nukuria", + "nus": "Nuer", + "nut": "Nung (Viet Nam)", + "nuu": "Ngbundu", + "nuv": "Northern Nuni", + "nuw": "Nguluwan", + "nux": "Mehek", + "nuy": "Nunggubuyu", + "nuz": "Tlamacazapa Nahuatl", + "nv": "Navajo; Navaho", + "nvh": "Nasarian", + "nvm": "Namiae", + "nvo": "Nyokon", + "nwa": "Nawathinehena", + "nwb": "Nyabwa", + "nwc": "Classical Newari; Classical Nepal Bhasa; Old Newari", + "nwe": "Ngwe", + "nwg": "Ngayawung", + "nwi": "Southwest Tanna", + "nwm": "Nyamusa-Molo", + "nwo": "Nauo", + "nwr": "Nawaru", + "nww": "Ndwewe", + "nwx": "Middle Newar", + "nwy": "Nottoway-Meherrin", + "nxa": "Nauete", + "nxd": "Ngando (Democratic Republic of Congo)", + "nxe": "Nage", + "nxg": "Ngad'a", + "nxi": "Nindi", + "nxk": "Koki Naga", + "nxl": "South Nuaulu", + "nxm": "Numidian", + "nxn": "Ngawun", + "nxo": "Ndambomo", + "nxq": "Naxi", + "nxr": "Ninggerum", + "nxx": "Nafri", + "ny": "Nyanja; Chewa; Chichewa", + "nyb": "Nyangbo", + "nyc": "Nyanga-li", + "nyd": "Nyore; Olunyole", + "nye": "Nyengo", + "nyf": "Giryama; Kigiryama", + "nyg": "Nyindu", + "nyh": "Nyikina", + "nyi": "Ama (Sudan)", + "nyj": "Nyanga", + "nyk": "Nyaneka", + "nyl": "Nyeu", + "nym": "Nyamwezi", + "nyn": "Nyankole", + "nyo": "Nyoro", + "nyp": "Nyang'i", + "nyq": "Nayini", + "nyr": "Nyiha (Malawi)", + "nys": "Nyungar", + "nyt": "Nyawaygi", + "nyu": "Nyungwe", + "nyv": "Nyulnyul", + "nyw": "Nyaw", + "nyx": "Nganyaywana", + "nyy": "Nyakyusa-Ngonde", + "nza": "Tigon Mbembe", + "nzb": "Njebi", + "nzd": "Nzadi", + "nzi": "Nzima", + "nzk": "Nzakara", + "nzm": "Zeme Naga", + "nzs": "New Zealand Sign Language", + "nzu": "Teke-Nzikou", + "nzy": "Nzakambay", + "nzz": "Nanga Dama Dogon", + "oaa": "Orok", + "oac": "Oroch", + "oar": "Old Aramaic (up to 700 BCE); Ancient Aramaic (up to 700 BCE)", + "oav": "Old Avar", + "obi": "Obispeño", + "obk": "Southern Bontok", + "obl": "Oblo", + "obm": "Moabite", + "obo": "Obo Manobo", + "obr": "Old Burmese", + "obt": "Old Breton", + "obu": "Obulom", + "oc": "Occitan (post 1500)", + "oca": "Ocaina", + "och": "Old Chinese", + "ocm": "Old Cham", + "oco": "Old Cornish", + "ocu": "Atzingo Matlatzinca", + "oda": "Odut", + "odk": "Od", + "odt": "Old Dutch", + "odu": "Odual", + "ofo": "Ofo", + "ofs": "Old Frisian", + "ofu": "Efutop", + "ogb": "Ogbia", + "ogc": "Ogbah", + "oge": "Old Georgian", + "ogg": "Ogbogolo", + "ogo": "Khana", + "ogu": "Ogbronuagum", + "oht": "Old Hittite", + "ohu": "Old Hungarian", + "oia": "Oirata", + "oie": "Okolie", + "oin": "Inebu One", + "oj": "Ojibwa", + "ojb": "Northwestern Ojibwa", + "ojc": "Central Ojibwa", + "ojg": "Eastern Ojibwa", + "ojp": "Old Japanese", + "ojs": "Severn Ojibwa", + "ojv": "Ontong Java", + "ojw": "Western Ojibwa", + "oka": "Okanagan", + "okb": "Okobo", + "okc": "Kobo", + "okd": "Okodia", + "oke": "Okpe (Southwestern Edo)", + "okg": "Koko Babangk", + "okh": "Koresh-e Rostam", + "oki": "Okiek", + "okj": "Oko-Juwoi", + "okk": "Kwamtim One", + "okl": "Old Kentish Sign Language", + "okm": "Middle Korean (10th-16th cent.)", + "okn": "Oki-No-Erabu", + "oko": "Old Korean (3rd-9th cent.)", + "okr": "Kirike", + "oks": "Oko-Eni-Osayen", + "oku": "Oku", + "okv": "Orokaiva", + "okx": "Okpe (Northwestern Edo)", + "okz": "Old Khmer", + "ola": "Walungge", + "old": "Mochi", + "ole": "Olekha", + "olk": "Olkol", + "olm": "Oloma", + "olo": "Livvi", + "olr": "Olrat", + "olt": "Old Lithuanian", + "olu": "Kuvale", + "om": "Oromo", + "oma": "Omaha-Ponca", + "omb": "East Ambae", + "omc": "Mochica", + "omg": "Omagua", + "omi": "Omi", + "omk": "Omok", + "oml": "Ombo", + "omn": "Minoan", + "omo": "Utarmbung", + "omp": "Old Manipuri", + "omq": "Oto-Manguean languages", + "omr": "Old Marathi", + "omt": "Omotik", + "omu": "Omurano", + "omv": "Omotic languages", + "omw": "South Tairora", + "omx": "Old Mon", + "omy": "Old Malay", + "ona": "Ona", + "onb": "Lingao", + "one": "Oneida", + "ong": "Olo", + "oni": "Onin", + "onj": "Onjob", + "onk": "Kabore One", + "onn": "Onobasulu", + "ono": "Onondaga", + "onp": "Sartang", + "onr": "Northern One", + "ons": "Ono", + "ont": "Ontenu", + "onu": "Unua", + "onw": "Old Nubian", + "onx": "Onin Based Pidgin", + "ood": "Tohono O'odham", + "oog": "Ong", + "oon": "Önge", + "oor": "Oorlams", + "oos": "Old Ossetic", + "opa": "Okpamheri", + "opk": "Kopkaka", + "opm": "Oksapmin", + "opo": "Opao", + "opt": "Opata", + "opy": "Ofayé", + "or": "Oriya (macrolanguage); Odia (macrolanguage)", + "ora": "Oroha", + "orc": "Orma", + "ore": "Orejón", + "org": "Oring", + "orh": "Oroqen", + "orn": "Orang Kanaq", + "oro": "Orokolo", + "orr": "Oruma", + "ors": "Orang Seletar", + "ort": "Adivasi Oriya", + "oru": "Ormuri", + "orv": "Old Russian", + "orw": "Oro Win", + "orx": "Oro", + "ory": "Odia (individual language); Oriya (individual language)", + "orz": "Ormu", + "os": "Ossetian; Ossetic", + "osa": "Osage", + "osc": "Oscan", + "osi": "Osing", + "osn": "Old Sundanese", + "oso": "Ososo", + "osp": "Old Spanish", + "ost": "Osatu", + "osu": "Southern One", + "osx": "Old Saxon", + "ota": "Ottoman Turkish (1500-1928)", + "otb": "Old Tibetan", + "otd": "Ot Danum", + "ote": "Mezquital Otomi", + "oti": "Oti", + "otk": "Old Turkish", + "otl": "Tilapa Otomi", + "otm": "Eastern Highland Otomi", + "otn": "Tenango Otomi", + "oto": "Otomian languages", + "otq": "Querétaro Otomi", + "otr": "Otoro", + "ots": "Estado de México Otomi", + "ott": "Temoaya Otomi", + "otu": "Otuke", + "otw": "Ottawa", + "otx": "Texcatepec Otomi", + "oty": "Old Tamil", + "otz": "Ixtenco Otomi", + "oua": "Tagargrent", + "oub": "Glio-Oubi", + "oue": "Oune", + "oui": "Old Uighur", + "oum": "Ouma", + "ovd": "Elfdalian; Övdalian", + "owi": "Owiniga", + "owl": "Old Welsh", + "oyb": "Oy", + "oyd": "Oyda", + "oym": "Wayampi", + "oyy": "Oya'oya", + "ozm": "Koonzime", + "pa": "Panjabi; Punjabi", + "paa": "Papuan languages", + "pab": "Parecís", + "pac": "Pacoh", + "pad": "Paumarí", + "pae": "Pagibete", + "paf": "Paranawát", + "pag": "Pangasinan", + "pah": "Tenharim", + "pai": "Pe", + "pak": "Parakanã", + "pal": "Pahlavi", + "pam": "Pampanga; Kapampangan", + "pao": "Northern Paiute", + "pap": "Papiamento", + "paq": "Parya", + "par": "Panamint; Timbisha", + "pas": "Papasena", + "pau": "Palauan", + "pav": "Pakaásnovos", + "paw": "Pawnee", + "pax": "Pankararé", + "pay": "Pech", + "paz": "Pankararú", + "pbb": "Páez", + "pbc": "Patamona", + "pbe": "Mezontla Popoloca", + "pbf": "Coyotepec Popoloca", + "pbg": "Paraujano", + "pbh": "E'ñapa Woromaipu", + "pbi": "Parkwa", + "pbl": "Mak (Nigeria)", + "pbm": "Puebla Mazatec", + "pbn": "Kpasam", + "pbo": "Papel", + "pbp": "Badyara", + "pbr": "Pangwa", + "pbs": "Central Pame", + "pbt": "Southern Pashto", + "pbu": "Northern Pashto", + "pbv": "Pnar", + "pby": "Pyu (Papua New Guinea)", + "pca": "Santa Inés Ahuatempan Popoloca", + "pcb": "Pear", + "pcc": "Bouyei", + "pcd": "Picard", + "pce": "Ruching Palaung", + "pcf": "Paliyan", + "pcg": "Paniya", + "pch": "Pardhan", + "pci": "Duruwa", + "pcj": "Parenga", + "pck": "Paite Chin", + "pcl": "Pardhi", + "pcm": "Nigerian Pidgin", + "pcn": "Piti", + "pcp": "Pacahuara", + "pcw": "Pyapun", + "pda": "Anam", + "pdc": "Pennsylvania German", + "pdi": "Pa Di", + "pdn": "Podena; Fedan", + "pdo": "Padoe", + "pdt": "Plautdietsch", + "pdu": "Kayan", + "pea": "Peranakan Indonesian", + "peb": "Eastern Pomo", + "ped": "Mala (Papua New Guinea)", + "pee": "Taje", + "pef": "Northeastern Pomo", + "peg": "Pengo", + "peh": "Bonan", + "pei": "Chichimeca-Jonaz", + "pej": "Northern Pomo", + "pek": "Penchal", + "pel": "Pekal", + "pem": "Phende", + "peo": "Old Persian (ca. 600-400 B.C.)", + "pep": "Kunja", + "peq": "Southern Pomo", + "pes": "Iranian Persian", + "pev": "Pémono", + "pex": "Petats", + "pey": "Petjo", + "pez": "Eastern Penan", + "pfa": "Pááfang", + "pfe": "Pere", + "pfl": "Pfaelzisch", + "pga": "Sudanese Creole Arabic", + "pgd": "Gāndhārī", + "pgg": "Pangwali", + "pgi": "Pagi", + "pgk": "Rerep", + "pgl": "Primitive Irish", + "pgn": "Paelignian", + "pgs": "Pangseng", + "pgu": "Pagu", + "pgz": "Papua New Guinean Sign Language", + "pha": "Pa-Hng", + "phd": "Phudagi", + "phg": "Phuong", + "phh": "Phukha", + "phi": "Philippine languages", + "phj": "Pahari", + "phk": "Phake", + "phl": "Phalura; Palula", + "phm": "Phimbi", + "phn": "Phoenician", + "pho": "Phunoi", + "phq": "Phana'", + "phr": "Pahari-Potwari", + "pht": "Phu Thai", + "phu": "Phuan", + "phv": "Pahlavani", + "phw": "Phangduwali", + "pi": "Pali", + "pia": "Pima Bajo", + "pib": "Yine", + "pic": "Pinji", + "pid": "Piaroa", + "pie": "Piro", + "pif": "Pingelapese", + "pig": "Pisabo", + "pih": "Pitcairn-Norfolk", + "pij": "Pijao", + "pil": "Yom", + "pim": "Powhatan", + "pin": "Piame", + "pio": "Piapoco", + "pip": "Pero", + "pir": "Piratapuyo", + "pis": "Pijin", + "pit": "Pitta Pitta", + "piu": "Pintupi-Luritja", + "piv": "Pileni; Vaeakau-Taumako", + "piw": "Pimbwe", + "pix": "Piu", + "piy": "Piya-Kwonci", + "piz": "Pije", + "pjt": "Pitjantjatjara", + "pka": "Ardhamāgadhī Prākrit", + "pkb": "Pokomo; Kipfokomo", + "pkc": "Paekche", + "pkg": "Pak-Tong", + "pkh": "Pankhu", + "pkn": "Pakanha", + "pko": "Pökoot", + "pkp": "Pukapuka", + "pkr": "Attapady Kurumba", + "pks": "Pakistan Sign Language", + "pkt": "Maleng", + "pku": "Paku", + "pl": "Polish", + "pla": "Miani", + "plb": "Polonombauk", + "plc": "Central Palawano", + "pld": "Polari", + "ple": "Palu'e", + "plf": "Central Malayo-Polynesian languages", + "plg": "Pilagá", + "plh": "Paulohi", + "plj": "Polci", + "plk": "Kohistani Shina", + "pll": "Shwe Palaung", + "pln": "Palenquero", + "plo": "Oluta Popoluca", + "plq": "Palaic", + "plr": "Palaka Senoufo", + "pls": "San Marcos Tlacoyalco Popoloca; San Marcos Tlalcoyalco Popoloca", + "plt": "Plateau Malagasy", + "plu": "Palikúr", + "plv": "Southwest Palawano", + "plw": "Brooke's Point Palawano", + "ply": "Bolyu", + "plz": "Paluan", + "pma": "Paama", + "pmb": "Pambia", + "pmd": "Pallanganmiddang", + "pme": "Pwaamei", + "pmf": "Pamona", + "pmh": "Māhārāṣṭri Prākrit", + "pmi": "Northern Pumi", + "pmj": "Southern Pumi", + "pmk": "Pamlico", + "pml": "Lingua Franca", + "pmm": "Pomo", + "pmn": "Pam", + "pmo": "Pom", + "pmq": "Northern Pame", + "pmr": "Paynamar", + "pms": "Piemontese", + "pmt": "Tuamotuan", + "pmw": "Plains Miwok", + "pmx": "Poumei Naga", + "pmy": "Papuan Malay", + "pmz": "Southern Pame", + "pna": "Punan Bah-Biau", + "pnb": "Western Panjabi", + "pnc": "Pannei", + "pnd": "Mpinda", + "pne": "Western Penan", + "png": "Pangu; Pongu", + "pnh": "Penrhyn", + "pni": "Aoheng", + "pnj": "Pinjarup", + "pnk": "Paunaka", + "pnl": "Paleni", + "pnm": "Punan Batu 1", + "pnn": "Pinai-Hagahai", + "pno": "Panobo", + "pnp": "Pancana", + "pnq": "Pana (Burkina Faso)", + "pnr": "Panim", + "pns": "Ponosakan", + "pnt": "Pontic", + "pnu": "Jiongnai Bunu", + "pnv": "Pinigura", + "pnw": "Banyjima; Panytyima", + "pnx": "Phong-Kniang", + "pny": "Pinyin", + "pnz": "Pana (Central African Republic)", + "poc": "Poqomam", + "poe": "San Juan Atzingo Popoloca", + "pof": "Poke", + "pog": "Potiguára", + "poh": "Poqomchi'", + "poi": "Highland Popoluca", + "pok": "Pokangá", + "pom": "Southeastern Pomo", + "pon": "Pohnpeian", + "poo": "Central Pomo", + "pop": "Pwapwâ", + "poq": "Texistepec Popoluca", + "pos": "Sayula Popoluca", + "pot": "Potawatomi", + "pov": "Upper Guinea Crioulo", + "pow": "San Felipe Otlaltepec Popoloca", + "pox": "Polabian", + "poy": "Pogolo", + "poz": "Malayo-Polynesian languages", + "ppe": "Papi", + "ppi": "Paipai", + "ppk": "Uma", + "ppl": "Pipil; Nicarao", + "ppm": "Papuma", + "ppn": "Papapana", + "ppo": "Folopa", + "ppp": "Pelende", + "ppq": "Pei", + "pps": "San Luís Temalacayuca Popoloca", + "ppt": "Pare", + "ppu": "Papora", + "pqa": "Pa'a", + "pqe": "Eastern Malayo-Polynesian languages", + "pqm": "Malecite-Passamaquoddy", + "pqw": "Western Malayo-Polynesian languages", + "pra": "Prakrit languages", + "prc": "Parachi", + "prd": "Parsi-Dari", + "pre": "Principense", + "prf": "Paranan", + "prg": "Prussian", + "prh": "Porohanon", + "pri": "Paicî", + "prk": "Parauk", + "prl": "Peruvian Sign Language", + "prm": "Kibiri", + "prn": "Prasuni", + "pro": "Old Provençal (to 1500); Old Occitan (to 1500)", + "prp": "Parsi", + "prq": "Ashéninka Perené", + "prr": "Puri", + "prs": "Dari; Afghan Persian", + "prt": "Phai", + "pru": "Puragi", + "prw": "Parawen", + "prx": "Purik", + "prz": "Providencia Sign Language", + "ps": "Pushto; Pashto", + "psa": "Asue Awyu", + "psc": "Iranian Sign Language; Persian Sign Language", + "psd": "Plains Indian Sign Language", + "pse": "Central Malay", + "psg": "Penang Sign Language", + "psh": "Southwest Pashai; Southwest Pashayi", + "psi": "Southeast Pashai; Southeast Pashayi", + "psl": "Puerto Rican Sign Language", + "psm": "Pauserna", + "psn": "Panasuan", + "pso": "Polish Sign Language", + "psp": "Philippine Sign Language", + "psq": "Pasi", + "psr": "Portuguese Sign Language", + "pss": "Kaulong", + "pst": "Central Pashto", + "psu": "Sauraseni Prākrit", + "psw": "Port Sandwich", + "psy": "Piscataway", + "pt": "Portuguese", + "pta": "Pai Tavytera", + "pth": "Pataxó Hã-Ha-Hãe", + "pti": "Pindiini; Wangkatha", + "ptn": "Patani", + "pto": "Zo'é", + "ptp": "Patep", + "ptq": "Pattapu", + "ptr": "Piamatsina", + "ptt": "Enrekang", + "ptu": "Bambam", + "ptv": "Port Vato", + "ptw": "Pentlatch", + "pty": "Pathiya", + "pua": "Western Highland Purepecha", + "pub": "Purum", + "puc": "Punan Merap", + "pud": "Punan Aput", + "pue": "Puelche", + "puf": "Punan Merah", + "pug": "Phuie", + "pui": "Puinave", + "puj": "Punan Tubu", + "pum": "Puma", + "puo": "Puoc", + "pup": "Pulabu", + "puq": "Puquina", + "pur": "Puruborá", + "put": "Putoh", + "puu": "Punu", + "puw": "Puluwatese", + "pux": "Puare", + "puy": "Purisimeño", + "pwa": "Pawaia", + "pwb": "Panawa", + "pwg": "Gapapaiwa", + "pwi": "Patwin", + "pwm": "Molbog", + "pwn": "Paiwan", + "pwo": "Pwo Western Karen", + "pwr": "Powari", + "pww": "Pwo Northern Karen", + "pxm": "Quetzaltepec Mixe", + "pye": "Pye Krumen", + "pym": "Fyam", + "pyn": "Poyanáwa", + "pys": "Paraguayan Sign Language; Lengua de Señas del Paraguay", + "pyu": "Puyuma", + "pyx": "Pyu (Myanmar)", + "pyy": "Pyen", + "pzh": "Pazeh", + "pzn": "Jejara Naga; Para Naga", + "qu": "Quechua", + "qua": "Quapaw", + "qub": "Huallaga Huánuco Quechua", + "quc": "K'iche'; Quiché", + "qud": "Calderón Highland Quichua", + "quf": "Lambayeque Quechua", + "qug": "Chimborazo Highland Quichua", + "quh": "South Bolivian Quechua", + "qui": "Quileute", + "quk": "Chachapoyas Quechua", + "qul": "North Bolivian Quechua", + "qum": "Sipacapense", + "qun": "Quinault", + "qup": "Southern Pastaza Quechua", + "quq": "Quinqui", + "qur": "Yanahuanca Pasco Quechua", + "qus": "Santiago del Estero Quichua", + "quv": "Sacapulteco", + "quw": "Tena Lowland Quichua", + "qux": "Yauyos Quechua", + "quy": "Ayacucho Quechua", + "quz": "Cusco Quechua", + "qva": "Ambo-Pasco Quechua", + "qvc": "Cajamarca Quechua", + "qve": "Eastern Apurímac Quechua", + "qvh": "Huamalíes-Dos de Mayo Huánuco Quechua", + "qvi": "Imbabura Highland Quichua", + "qvj": "Loja Highland Quichua", + "qvl": "Cajatambo North Lima Quechua", + "qvm": "Margos-Yarowilca-Lauricocha Quechua", + "qvn": "North Junín Quechua", + "qvo": "Napo Lowland Quechua", + "qvp": "Pacaraos Quechua", + "qvs": "San Martín Quechua", + "qvw": "Huaylla Wanca Quechua", + "qvy": "Queyu", + "qvz": "Northern Pastaza Quichua", + "qwa": "Corongo Ancash Quechua", + "qwc": "Classical Quechua", + "qwe": "Quechuan (family)", + "qwh": "Huaylas Ancash Quechua", + "qwm": "Kuman (Russia)", + "qws": "Sihuas Ancash Quechua", + "qwt": "Kwalhioqua-Tlatskanai", + "qxa": "Chiquián Ancash Quechua", + "qxc": "Chincha Quechua", + "qxh": "Panao Huánuco Quechua", + "qxl": "Salasaca Highland Quichua", + "qxn": "Northern Conchucos Ancash Quechua", + "qxo": "Southern Conchucos Ancash Quechua", + "qxp": "Puno Quechua", + "qxq": "Qashqa'i", + "qxr": "Cañar Highland Quichua", + "qxs": "Southern Qiang", + "qxt": "Santa Ana de Tusi Pasco Quechua", + "qxu": "Arequipa-La Unión Quechua", + "qxw": "Jauja Wanca Quechua", + "qya": "Quenya", + "qyp": "Quiripi", + "raa": "Dungmali", + "rab": "Camling", + "rac": "Rasawa", + "rad": "Rade", + "raf": "Western Meohang", + "rag": "Logooli; Lulogooli", + "rah": "Rabha", + "rai": "Ramoaaina", + "raj": "Rajasthani", + "rak": "Tulu-Bohuai", + "ral": "Ralte", + "ram": "Canela", + "ran": "Riantana", + "rao": "Rao", + "rap": "Rapanui", + "raq": "Saam", + "rar": "Rarotongan; Cook Islands Maori", + "ras": "Tegali", + "rat": "Razajerdi", + "rau": "Raute", + "rav": "Sampang", + "raw": "Rawang", + "rax": "Rang", + "ray": "Rapa", + "raz": "Rahambuu", + "rbb": "Rumai Palaung", + "rbk": "Northern Bontok", + "rbl": "Miraya Bikol", + "rbp": "Barababaraba", + "rcf": "Réunion Creole French", + "rdb": "Rudbari", + "rea": "Rerau", + "reb": "Rembong", + "ree": "Rejang Kayan", + "reg": "Kara (Tanzania)", + "rei": "Reli", + "rej": "Rejang", + "rel": "Rendille", + "rem": "Remo", + "ren": "Rengao", + "rer": "Rer Bare", + "res": "Reshe", + "ret": "Retta", + "rey": "Reyesano", + "rga": "Roria", + "rge": "Romano-Greek", + "rgk": "Rangkas", + "rgn": "Romagnol", + "rgr": "Resígaro", + "rgs": "Southern Roglai", + "rgu": "Ringgou", + "rhg": "Rohingya", + "rhp": "Yahang", + "ria": "Riang (India)", + "rib": "Bribri Sign Language", + "rif": "Tarifit", + "ril": "Riang Lang; Riang (Myanmar)", + "rim": "Nyaturu", + "rin": "Nungu", + "rir": "Ribun", + "rit": "Ritharrngu", + "riu": "Riung", + "rjg": "Rajong", + "rji": "Raji", + "rjs": "Rajbanshi", + "rka": "Kraol", + "rkb": "Rikbaktsa", + "rkh": "Rakahanga-Manihiki", + "rki": "Rakhine", + "rkm": "Marka", + "rkt": "Rangpuri; Kamta", + "rkw": "Arakwal", + "rm": "Romansh", + "rma": "Rama", + "rmb": "Rembarrnga", + "rmc": "Carpathian Romani", + "rmd": "Traveller Danish", + "rme": "Angloromani", + "rmf": "Kalo Finnish Romani", + "rmg": "Traveller Norwegian", + "rmh": "Murkim", + "rmi": "Lomavren", + "rmk": "Romkun", + "rml": "Baltic Romani", + "rmm": "Roma", + "rmn": "Balkan Romani", + "rmo": "Sinte Romani", + "rmp": "Rempi", + "rmq": "Caló", + "rms": "Romanian Sign Language", + "rmt": "Domari", + "rmu": "Tavringer Romani", + "rmv": "Romanova", + "rmw": "Welsh Romani", + "rmx": "Romam", + "rmy": "Vlax Romani", + "rmz": "Marma", + "rn": "Rundi", + "rnb": "Brunca Sign Language", + "rnd": "Ruund", + "rng": "Ronga", + "rnl": "Ranglong", + "rnn": "Roon", + "rnp": "Rongpo", + "rnr": "Nari Nari", + "rnw": "Rungwa", + "ro": "Romanian; Moldavian; Moldovan", + "roa": "Romance languages", + "rob": "Tae'", + "roc": "Cacgia Roglai", + "rod": "Rogo", + "roe": "Ronji", + "rof": "Rombo", + "rog": "Northern Roglai", + "rol": "Romblomanon", + "rom": "Romany", + "roo": "Rotokas", + "rop": "Kriol", + "ror": "Rongga", + "rou": "Runga", + "row": "Dela-Oenale", + "rpn": "Repanbitip", + "rpt": "Rapting", + "rri": "Ririo", + "rro": "Waima", + "rrt": "Arritinngithigh", + "rsb": "Romano-Serbian", + "rsk": "Ruthenian; Rusyn", + "rsl": "Russian Sign Language", + "rsm": "Miriwoong Sign Language", + "rsn": "Rwandan Sign Language", + "rtc": "Rungtu Chin", + "rth": "Ratahan", + "rtm": "Rotuman", + "rts": "Yurats", + "rtw": "Rathawi", + "ru": "Russian", + "rub": "Gungu", + "ruc": "Ruuli", + "rue": "Rusyn", + "ruf": "Luguru", + "rug": "Roviana", + "ruh": "Ruga", + "rui": "Rufiji", + "ruk": "Che", + "ruo": "Istro Romanian", + "rup": "Macedo-Romanian; Aromanian; Arumanian", + "ruq": "Megleno Romanian", + "rut": "Rutul", + "ruu": "Lanas Lobu", + "ruy": "Mala (Nigeria)", + "ruz": "Ruma", + "rw": "Kinyarwanda", + "rwa": "Rawo", + "rwk": "Rwa", + "rwl": "Ruwila", + "rwm": "Amba (Uganda)", + "rwo": "Rawa", + "rwr": "Marwari (India)", + "rxd": "Ngardi", + "rxw": "Karuwali; Garuwali", + "ryn": "Northern Amami-Oshima", + "rys": "Yaeyama", + "ryu": "Central Okinawan", + "rzh": "Rāziḥī", + "sa": "Sanskrit", + "saa": "Saba", + "sab": "Buglere", + "sac": "Meskwaki", + "sad": "Sandawe", + "sae": "Sabanê", + "saf": "Safaliba", + "sah": "Yakut", + "sai": "South American Indian languages", + "saj": "Sahu", + "sak": "Sake", + "sal": "Salishan languages", + "sam": "Samaritan Aramaic", + "sao": "Sause", + "saq": "Samburu", + "sar": "Saraveca", + "sas": "Sasak", + "sat": "Santali", + "sau": "Saleman", + "sav": "Saafi-Saafi", + "saw": "Sawi", + "sax": "Sa", + "say": "Saya", + "saz": "Saurashtra", + "sba": "Ngambay", + "sbb": "Simbo", + "sbc": "Kele (Papua New Guinea)", + "sbd": "Southern Samo", + "sbe": "Saliba", + "sbf": "Chabu; Shabo", + "sbg": "Seget", + "sbh": "Sori-Harengan", + "sbi": "Seti", + "sbj": "Surbakhal", + "sbk": "Safwa", + "sbl": "Botolan Sambal", + "sbm": "Sagala", + "sbn": "Sindhi Bhil", + "sbo": "Sabüm", + "sbp": "Sangu (Tanzania)", + "sbq": "Sileibi", + "sbr": "Sembakung Murut", + "sbs": "Subiya", + "sbt": "Kimki", + "sbu": "Stod Bhoti", + "sbv": "Sabine", + "sbw": "Simba", + "sbx": "Seberuang", + "sby": "Soli", + "sbz": "Sara Kaba", + "sc": "Sardinian", + "scb": "Chut", + "sce": "Dongxiang", + "scf": "San Miguel Creole French", + "scg": "Sanggau", + "sch": "Sakachep", + "sci": "Sri Lankan Creole Malay", + "sck": "Sadri", + "scl": "Shina", + "scn": "Sicilian", + "sco": "Scots", + "scp": "Hyolmo; Helambu Sherpa", + "scq": "Sa'och", + "scs": "North Slavey", + "sct": "Southern Katang", + "scu": "Shumcho", + "scv": "Sheni", + "scw": "Sha", + "scx": "Sicel", + "sd": "Sindhi", + "sda": "Toraja-Sa'dan", + "sdb": "Shabak", + "sdc": "Sassarese Sardinian", + "sde": "Surubu", + "sdf": "Sarli", + "sdg": "Savi", + "sdh": "Southern Kurdish", + "sdj": "Suundi", + "sdk": "Sos Kundi", + "sdl": "Saudi Arabian Sign Language", + "sdn": "Gallurese Sardinian", + "sdo": "Bukar-Sadung Bidayuh", + "sdp": "Sherdukpen", + "sdq": "Semandang", + "sdr": "Oraon Sadri", + "sds": "Sened", + "sdt": "Shuadit", + "sdu": "Sarudu", + "sdv": "Eastern Sudanic languages", + "sdx": "Sibu Melanau", + "sdz": "Sallands", + "se": "Northern Sami", + "sea": "Semai", + "seb": "Shempire Senoufo", + "sec": "Sechelt", + "sed": "Sedang", + "see": "Seneca", + "sef": "Cebaara Senoufo", + "seg": "Segeju", + "seh": "Sena", + "sei": "Seri", + "sej": "Sene", + "sek": "Sekani", + "sel": "Selkup", + "sem": "Semitic languages", + "sen": "Nanerigé Sénoufo", + "seo": "Suarmin", + "sep": "Sìcìté Sénoufo", + "seq": "Senara Sénoufo", + "ser": "Serrano", + "ses": "Koyraboro Senni Songhai", + "set": "Sentani", + "seu": "Serui-Laut", + "sev": "Nyarafolo Senoufo", + "sew": "Sewa Bay", + "sey": "Secoya", + "sez": "Senthang Chin", + "sfb": "Langue des signes de Belgique Francophone; French Belgian Sign Language", + "sfe": "Eastern Subanen", + "sfm": "Small Flowery Miao", + "sfs": "South African Sign Language", + "sfw": "Sehwi", + "sg": "Sango", + "sga": "Old Irish (to 900)", + "sgb": "Mag-antsi Ayta", + "sgc": "Kipsigis", + "sgd": "Surigaonon", + "sge": "Segai", + "sgg": "Swiss-German Sign Language", + "sgh": "Shughni", + "sgi": "Suga", + "sgj": "Surgujia", + "sgk": "Sangkong", + "sgm": "Singa", + "sgn": "Sign languages", + "sgp": "Singpho", + "sgr": "Sangisari", + "sgs": "Samogitian", + "sgt": "Brokpake", + "sgu": "Salas", + "sgw": "Sebat Bet Gurage", + "sgx": "Sierra Leone Sign Language", + "sgy": "Sanglechi", + "sgz": "Sursurunga", + "sh": "Serbo-Croatian", + "sha": "Shall-Zwall", + "shb": "Ninam", + "shc": "Sonde", + "shd": "Kundal Shahi", + "she": "Sheko", + "shg": "Shua", + "shh": "Shoshoni", + "shi": "Tachelhit", + "shj": "Shatt", + "shk": "Shilluk", + "shl": "Shendu", + "shm": "Shahrudi", + "shn": "Shan", + "sho": "Shanga", + "shp": "Shipibo-Conibo", + "shq": "Sala", + "shr": "Shi", + "shs": "Shuswap", + "sht": "Shasta", + "shu": "Chadian Arabic", + "shv": "Shehri", + "shw": "Shwai", + "shx": "She", + "shy": "Tachawit", + "shz": "Syenara Senoufo", + "si": "Sinhala; Sinhalese", + "sia": "Akkala Sami", + "sib": "Sebop", + "sid": "Sidamo", + "sie": "Simaa", + "sif": "Siamou", + "sig": "Paasaal", + "sih": "Zire; Sîshëë", + "sii": "Shom Peng", + "sij": "Numbami", + "sik": "Sikiana", + "sil": "Tumulung Sisaala", + "sim": "Mende (Papua New Guinea)", + "sio": "Siouan languages", + "sip": "Sikkimese", + "siq": "Sonia", + "sir": "Siri", + "sis": "Siuslaw", + "sit": "Sino-Tibetan languages", + "siu": "Sinagen", + "siv": "Sumariup", + "siw": "Siwai", + "six": "Sumau", + "siy": "Sivandi", + "siz": "Siwi", + "sja": "Epena", + "sjb": "Sajau Basap", + "sjd": "Kildin Sami", + "sje": "Pite Sami", + "sjg": "Assangori", + "sjk": "Kemi Sami", + "sjl": "Sajalong; Miji", + "sjm": "Mapun", + "sjn": "Sindarin", + "sjo": "Xibe", + "sjp": "Surjapuri", + "sjr": "Siar-Lak", + "sjs": "Senhaja De Srair", + "sjt": "Ter Sami", + "sju": "Ume Sami", + "sjw": "Shawnee", + "sk": "Slovak", + "ska": "Skagit", + "skb": "Saek", + "skc": "Ma Manda", + "skd": "Southern Sierra Miwok", + "ske": "Seke (Vanuatu)", + "skf": "Sakirabiá", + "skg": "Sakalava Malagasy", + "skh": "Sikule", + "ski": "Sika", + "skj": "Seke (Nepal)", + "skm": "Kutong", + "skn": "Kolibugan Subanon", + "sko": "Seko Tengah", + "skp": "Sekapan", + "skq": "Sininkere", + "skr": "Saraiki; Seraiki", + "sks": "Maia", + "skt": "Sakata", + "sku": "Sakao", + "skv": "Skou", + "skw": "Skepi Creole Dutch", + "skx": "Seko Padang", + "sky": "Sikaiana", + "skz": "Sekar", + "sl": "Slovenian", + "sla": "Slavic languages", + "slc": "Sáliba", + "sld": "Sissala", + "sle": "Sholaga", + "slf": "Swiss-Italian Sign Language", + "slg": "Selungai Murut", + "slh": "Southern Puget Sound Salish", + "sli": "Lower Silesian", + "slj": "Salumá", + "sll": "Salt-Yui", + "slm": "Pangutaran Sama", + "sln": "Salinan", + "slp": "Lamaholot", + "slq": "Salchuq", + "slr": "Salar", + "sls": "Singapore Sign Language", + "slt": "Sila", + "slu": "Selaru", + "slw": "Sialum", + "slx": "Salampasu", + "sly": "Selayar", + "slz": "Ma'ya", + "sm": "Samoan", + "sma": "Southern Sami", + "smb": "Simbari", + "smc": "Som", + "smf": "Auwe", + "smg": "Simbali", + "smh": "Samei", + "smi": "Sami languages", + "smj": "Lule Sami", + "smk": "Bolinao", + "sml": "Central Sama", + "smm": "Musasa", + "smn": "Inari Sami", + "smp": "Samaritan", + "smq": "Samo", + "smr": "Simeulue", + "sms": "Skolt Sami", + "smt": "Simte", + "smu": "Somray", + "smv": "Samvedi", + "smw": "Sumbawa", + "smx": "Samba", + "smy": "Semnani", + "smz": "Simeku", + "sn": "Shona", + "snc": "Sinaugoro", + "sne": "Bau Bidayuh", + "snf": "Noon", + "sng": "Sanga (Democratic Republic of Congo)", + "sni": "Sensi", + "snj": "Riverain Sango", + "snk": "Soninke", + "snl": "Sangil", + "snm": "Southern Ma'di", + "snn": "Siona", + "sno": "Snohomish", + "snp": "Siane", + "snq": "Sangu (Gabon)", + "snr": "Sihan", + "sns": "South West Bay; Nahavaq", + "snu": "Senggi; Viid", + "snv": "Sa'ban", + "snw": "Selee", + "snx": "Sam", + "sny": "Saniyo-Hiyewe", + "snz": "Kou", + "so": "Somali", + "soa": "Thai Song", + "sob": "Sobei", + "soc": "So (Democratic Republic of Congo)", + "sod": "Songoora", + "soe": "Songomeno", + "sog": "Sogdian", + "soh": "Aka", + "soi": "Sonha", + "soj": "Soi", + "sok": "Sokoro", + "sol": "Solos", + "son": "Songhai languages", + "soo": "Songo", + "sop": "Songe", + "soq": "Kanasi", + "sor": "Somrai", + "sos": "Seeku", + "sou": "Southern Thai", + "sov": "Sonsorol", + "sow": "Sowanda", + "sox": "Swo", + "soy": "Miyobe", + "soz": "Temi", + "spb": "Sepa (Indonesia)", + "spc": "Sapé", + "spd": "Saep", + "spe": "Sepa (Papua New Guinea)", + "spg": "Sian", + "spi": "Saponi", + "spk": "Sengo", + "spl": "Selepet", + "spm": "Akukem", + "spn": "Sanapaná", + "spo": "Spokane", + "spp": "Supyire Senoufo", + "spq": "Loreto-Ucayali Spanish", + "spr": "Saparua", + "sps": "Saposa", + "spt": "Spiti Bhoti", + "spu": "Sapuan", + "spv": "Sambalpuri; Kosli", + "spx": "South Picene", + "spy": "Sabaot", + "sq": "Albanian", + "sqa": "Shama-Sambuga", + "sqh": "Shau", + "sqj": "Albanian languages", + "sqk": "Albanian Sign Language", + "sqm": "Suma", + "sqn": "Susquehannock", + "sqo": "Sorkhei", + "sqq": "Sou", + "sqr": "Siculo Arabic", + "sqs": "Sri Lankan Sign Language", + "sqt": "Soqotri", + "squ": "Squamish", + "sqx": "Kufr Qassem Sign Language (KQSL)", + "sr": "Serbian", + "sra": "Saruga", + "srb": "Sora", + "src": "Logudorese Sardinian", + "sre": "Sara", + "srf": "Nafi", + "srg": "Sulod", + "srh": "Sarikoli", + "sri": "Siriano", + "srk": "Serudung Murut", + "srl": "Isirawa", + "srm": "Saramaccan", + "srn": "Sranan Tongo", + "sro": "Campidanese Sardinian", + "srq": "Sirionó", + "srr": "Serer", + "srs": "Sarsi", + "srt": "Sauri", + "sru": "Suruí", + "srv": "Southern Sorsoganon", + "srw": "Serua", + "srx": "Sirmauri", + "sry": "Sera", + "srz": "Shahmirzadi", + "ss": "Swati", + "ssa": "Nilo-Saharan languages", + "ssb": "Southern Sama", + "ssc": "Suba-Simbiti", + "ssd": "Siroi", + "sse": "Balangingi; Bangingih Sama", + "ssf": "Thao", + "ssg": "Seimat", + "ssh": "Shihhi Arabic", + "ssi": "Sansi", + "ssj": "Sausi", + "ssk": "Sunam", + "ssl": "Western Sisaala", + "ssm": "Semnam", + "ssn": "Waata", + "sso": "Sissano", + "ssp": "Spanish Sign Language", + "ssq": "So'a", + "ssr": "Swiss-French Sign Language", + "sss": "Sô", + "sst": "Sinasina", + "ssu": "Susuami", + "ssv": "Shark Bay", + "ssx": "Samberigi", + "ssy": "Saho", + "ssz": "Sengseng", + "st": "Southern Sotho", + "sta": "Settla", + "stb": "Northern Subanen", + "std": "Sentinel", + "ste": "Liana-Seti", + "stf": "Seta", + "stg": "Trieng", + "sth": "Shelta", + "sti": "Bulo Stieng", + "stj": "Matya Samo", + "stk": "Arammba", + "stl": "Stellingwerfs", + "stm": "Setaman", + "stn": "Owa", + "sto": "Stoney", + "stp": "Southeastern Tepehuan", + "stq": "Saterfriesisch", + "str": "Straits Salish", + "sts": "Shumashti", + "stt": "Budeh Stieng", + "stu": "Samtao", + "stv": "Silt'e", + "stw": "Satawalese", + "sty": "Siberian Tatar", + "su": "Sundanese", + "sua": "Sulka", + "sub": "Suku", + "suc": "Western Subanon", + "sue": "Suena", + "sug": "Suganga", + "sui": "Suki", + "suj": "Shubi", + "suk": "Sukuma", + "suo": "Bouni", + "suq": "Tirmaga-Chai Suri; Suri", + "sur": "Mwaghavul", + "sus": "Susu", + "sut": "Subtiaba", + "suv": "Puroik", + "suw": "Sumbwa", + "sux": "Sumerian", + "suy": "Suyá", + "suz": "Sunwar", + "sv": "Swedish", + "sva": "Svan", + "svb": "Ulau-Suain", + "svc": "Vincentian Creole English", + "sve": "Serili", + "svk": "Slovakian Sign Language", + "svm": "Slavomolisano", + "svs": "Savosavo", + "svx": "Skalvian", + "sw": "Swahili (macrolanguage)", + "swb": "Maore Comorian", + "swc": "Congo Swahili", + "swf": "Sere", + "swg": "Swabian", + "swh": "Swahili (individual language); Kiswahili", + "swi": "Sui", + "swj": "Sira", + "swk": "Malawi Sena", + "swl": "Swedish Sign Language", + "swm": "Samosa", + "swn": "Sawknah", + "swo": "Shanenawa", + "swp": "Suau", + "swq": "Sharwa", + "swr": "Saweru", + "sws": "Seluwasan", + "swt": "Sawila", + "swu": "Suwawa", + "swv": "Shekhawati", + "sww": "Sowa", + "swx": "Suruahá", + "swy": "Sarua", + "sxb": "Suba", + "sxc": "Sicanian", + "sxe": "Sighu", + "sxg": "Shuhi; Shixing", + "sxk": "Southern Kalapuya", + "sxl": "Selian", + "sxm": "Samre", + "sxn": "Sangir", + "sxo": "Sorothaptic", + "sxr": "Saaroa", + "sxs": "Sasaru", + "sxu": "Upper Saxon", + "sxw": "Saxwe Gbe", + "sya": "Siang", + "syb": "Central Subanen", + "syc": "Classical Syriac", + "syd": "Samoyedic languages", + "syi": "Seki", + "syk": "Sukur", + "syl": "Sylheti", + "sym": "Maya Samo", + "syn": "Senaya", + "syo": "Suoy", + "syr": "Syriac", + "sys": "Sinyar", + "syw": "Kagate", + "syx": "Samay", + "syy": "Al-Sayyid Bedouin Sign Language", + "sza": "Semelai", + "szb": "Ngalum", + "szc": "Semaq Beri", + "szd": "Seru", + "sze": "Seze", + "szg": "Sengele", + "szl": "Silesian", + "szn": "Sula", + "szp": "Suabo", + "szs": "Solomon Islands Sign Language", + "szv": "Isu (Fako Division)", + "szw": "Sawai", + "szy": "Sakizaya", + "ta": "Tamil", + "taa": "Lower Tanana", + "tab": "Tabassaran", + "tac": "Lowland Tarahumara", + "tad": "Tause", + "tae": "Tariana", + "taf": "Tapirapé", + "tag": "Tagoi", + "tai": "Tai languages", + "taj": "Eastern Tamang", + "tak": "Tala", + "tal": "Tal", + "tan": "Tangale", + "tao": "Yami", + "tap": "Taabwa", + "taq": "Tamasheq", + "tar": "Central Tarahumara", + "tas": "Tay Boi", + "tau": "Upper Tanana", + "tav": "Tatuyo", + "taw": "Tai", + "tax": "Tamki", + "tay": "Atayal", + "taz": "Tocho", + "tba": "Aikanã", + "tbc": "Takia", + "tbd": "Kaki Ae", + "tbe": "Tanimbili", + "tbf": "Mandara", + "tbg": "North Tairora", + "tbh": "Dharawal; Thurawal", + "tbi": "Gaam", + "tbj": "Tiang", + "tbk": "Calamian Tagbanwa", + "tbl": "Tboli", + "tbm": "Tagbu", + "tbn": "Barro Negro Tunebo", + "tbo": "Tawala", + "tbp": "Taworta; Diebroud", + "tbq": "Tibeto-Burman languages", + "tbr": "Tumtum", + "tbs": "Tanguat", + "tbt": "Tembo (Kitembo)", + "tbu": "Tubar", + "tbv": "Tobo", + "tbw": "Tagbanwa", + "tbx": "Kapin", + "tby": "Tabaru", + "tbz": "Ditammari", + "tca": "Ticuna", + "tcb": "Tanacross", + "tcc": "Datooga", + "tcd": "Tafi", + "tce": "Southern Tutchone", + "tcf": "Malinaltepec Me'phaa; Malinaltepec Tlapanec", + "tcg": "Tamagario", + "tch": "Turks And Caicos Creole English", + "tci": "Wára", + "tck": "Tchitchege", + "tcl": "Taman (Myanmar)", + "tcm": "Tanahmerah", + "tcn": "Tichurong", + "tco": "Taungyo", + "tcp": "Tawr Chin", + "tcq": "Kaiy", + "tcs": "Torres Strait Creole; Yumplatok", + "tct": "T'en", + "tcu": "Southeastern Tarahumara", + "tcw": "Tecpatlán Totonac", + "tcx": "Toda", + "tcy": "Tulu", + "tcz": "Thado Chin", + "tda": "Tagdal", + "tdb": "Panchpargania", + "tdc": "Emberá-Tadó", + "tdd": "Tai Nüa", + "tde": "Tiranige Diga Dogon", + "tdf": "Talieng", + "tdg": "Western Tamang", + "tdh": "Thulung", + "tdi": "Tomadino", + "tdj": "Tajio", + "tdk": "Tambas", + "tdl": "Sur", + "tdm": "Taruma", + "tdn": "Tondano", + "tdo": "Teme", + "tdq": "Tita", + "tdr": "Todrah", + "tds": "Doutai", + "tdt": "Tetun Dili", + "tdv": "Toro", + "tdx": "Tandroy-Mahafaly Malagasy", + "tdy": "Tadyawan", + "te": "Telugu", + "tea": "Temiar", + "teb": "Tetete", + "tec": "Terik", + "ted": "Tepo Krumen", + "tee": "Huehuetla Tepehua", + "tef": "Teressa", + "teg": "Teke-Tege", + "teh": "Tehuelche", + "tei": "Torricelli", + "tek": "Ibali Teke", + "tem": "Timne", + "ten": "Tama (Colombia)", + "teo": "Teso", + "tep": "Tepecano", + "teq": "Temein", + "ter": "Tereno", + "tes": "Tengger", + "tet": "Tetum", + "teu": "Soo", + "tev": "Teor", + "tew": "Tewa (USA)", + "tex": "Tennet", + "tey": "Tulishi", + "tez": "Tetserret", + "tfi": "Tofin Gbe", + "tfn": "Tanaina", + "tfo": "Tefaro", + "tfr": "Teribe", + "tft": "Ternate", + "tg": "Tajik", + "tga": "Sagalla", + "tgb": "Tobilung", + "tgc": "Tigak", + "tgd": "Ciwogai", + "tge": "Eastern Gorkha Tamang", + "tgf": "Chalikha", + "tgh": "Tobagonian Creole English", + "tgi": "Lawunuia", + "tgj": "Tagin", + "tgn": "Tandaganon", + "tgo": "Sudest", + "tgp": "Tangoa", + "tgq": "Tring", + "tgr": "Tareng", + "tgs": "Nume", + "tgt": "Central Tagbanwa", + "tgu": "Tanggu", + "tgv": "Tingui-Boto", + "tgw": "Tagwana Senoufo", + "tgx": "Tagish", + "tgy": "Togoyo", + "tgz": "Tagalaka", + "th": "Thai", + "thd": "Kuuk Thaayorre; Thayore", + "the": "Chitwania Tharu", + "thf": "Thangmi", + "thh": "Northern Tarahumara", + "thi": "Tai Long", + "thk": "Tharaka; Kitharaka", + "thl": "Dangaura Tharu", + "thm": "Aheu", + "thn": "Thachanadan", + "thp": "Thompson", + "thq": "Kochila Tharu", + "thr": "Rana Tharu", + "ths": "Thakali", + "tht": "Tahltan", + "thu": "Thuri", + "thv": "Tahaggart Tamahaq", + "thy": "Tha", + "thz": "Tayart Tamajeq", + "ti": "Tigrinya", + "tia": "Tidikelt Tamazight", + "tic": "Tira", + "tif": "Tifal", + "tig": "Tigre", + "tih": "Timugon Murut", + "tii": "Tiene", + "tij": "Tilung", + "tik": "Tikar", + "til": "Tillamook", + "tim": "Timbe", + "tin": "Tindi", + "tio": "Teop", + "tip": "Trimuris", + "tiq": "Tiéfo", + "tis": "Masadiit Itneg", + "tit": "Tinigua", + "tiu": "Adasen", + "tiv": "Tiv", + "tiw": "Tiwi", + "tix": "Southern Tiwa", + "tiy": "Tiruray", + "tiz": "Tai Hongjin", + "tja": "Tajuasohn", + "tjg": "Tunjung", + "tji": "Northern Tujia", + "tjj": "Tjungundji", + "tjl": "Tai Laing", + "tjm": "Timucua", + "tjn": "Tonjon", + "tjo": "Temacine Tamazight", + "tjp": "Tjupany", + "tjs": "Southern Tujia", + "tju": "Tjurruru", + "tjw": "Djabwurrung", + "tk": "Turkmen", + "tka": "Truká", + "tkb": "Buksa", + "tkd": "Tukudede", + "tke": "Takwane", + "tkf": "Tukumanféd", + "tkg": "Tesaka Malagasy", + "tkl": "Tokelau", + "tkm": "Takelma", + "tkn": "Toku-No-Shima", + "tkp": "Tikopia", + "tkq": "Tee", + "tkr": "Tsakhur", + "tks": "Takestani", + "tkt": "Kathoriya Tharu", + "tku": "Upper Necaxa Totonac", + "tkv": "Mur Pano", + "tkw": "Teanu", + "tkx": "Tangko", + "tkz": "Takua", + "tl": "Tagalog", + "tla": "Southwestern Tepehuan", + "tlb": "Tobelo", + "tlc": "Yecuatla Totonac", + "tld": "Talaud", + "tlf": "Telefol", + "tlg": "Tofanma", + "tlh": "Klingon; tlhIngan Hol", + "tli": "Tlingit", + "tlj": "Talinga-Bwisi", + "tlk": "Taloki", + "tll": "Tetela", + "tlm": "Tolomako", + "tln": "Talondo'", + "tlo": "Talodi", + "tlp": "Filomena Mata-Coahuitlán Totonac", + "tlq": "Tai Loi", + "tlr": "Talise", + "tls": "Tambotalo", + "tlt": "Sou Nama; Teluti", + "tlu": "Tulehu", + "tlv": "Taliabu", + "tlx": "Khehek", + "tly": "Talysh", + "tma": "Tama (Chad)", + "tmb": "Katbol; Avava", + "tmc": "Tumak", + "tmd": "Haruai", + "tme": "Tremembé", + "tmf": "Toba-Maskoy", + "tmg": "Ternateño", + "tmh": "Tamashek", + "tmi": "Tutuba", + "tmj": "Samarokena", + "tmk": "Northwestern Tamang", + "tml": "Tamnim Citak", + "tmm": "Tai Thanh", + "tmn": "Taman (Indonesia)", + "tmo": "Temoq", + "tmq": "Tumleo", + "tmr": "Jewish Babylonian Aramaic (ca. 200-1200 CE)", + "tms": "Tima", + "tmt": "Tasmate", + "tmu": "Iau", + "tmv": "Tembo (Motembo)", + "tmw": "Temuan", + "tmy": "Tami", + "tmz": "Tamanaku", + "tn": "Tswana", + "tna": "Tacana", + "tnb": "Western Tunebo", + "tnc": "Tanimuca-Retuarã", + "tnd": "Angosturas Tunebo", + "tng": "Tobanga", + "tnh": "Maiani", + "tni": "Tandia", + "tnk": "Kwamera", + "tnl": "Lenakel", + "tnm": "Tabla", + "tnn": "North Tanna", + "tno": "Toromono", + "tnp": "Whitesands", + "tnq": "Taino", + "tnr": "Ménik", + "tns": "Tenis", + "tnt": "Tontemboan", + "tnu": "Tay Khang", + "tnv": "Tangchangya", + "tnw": "Tonsawang", + "tnx": "Tanema", + "tny": "Tongwe", + "tnz": "Ten'edn", + "to": "Tonga (Tonga Islands)", + "tob": "Toba", + "toc": "Coyutla Totonac", + "tod": "Toma", + "tof": "Gizrra", + "tog": "Tonga (Nyasa)", + "toh": "Gitonga", + "toi": "Tonga (Zambia)", + "toj": "Tojolabal", + "tok": "Toki Pona", + "tol": "Tolowa", + "tom": "Tombulu", + "too": "Xicotepec De Juárez Totonac", + "top": "Papantla Totonac", + "toq": "Toposa", + "tor": "Togbo-Vara Banda", + "tos": "Highland Totonac", + "tou": "Tho", + "tov": "Upper Taromi", + "tow": "Jemez", + "tox": "Tobian", + "toy": "Topoiyo", + "toz": "To", + "tpa": "Taupota", + "tpc": "Azoyú Me'phaa; Azoyú Tlapanec", + "tpe": "Tippera", + "tpf": "Tarpia", + "tpg": "Kula", + "tpi": "Tok Pisin", + "tpj": "Tapieté", + "tpk": "Tupinikin", + "tpl": "Tlacoapa Me'phaa; Tlacoapa Tlapanec", + "tpm": "Tampulma", + "tpn": "Tupinambá", + "tpo": "Tai Pao", + "tpp": "Pisaflores Tepehua", + "tpq": "Tukpa", + "tpr": "Tuparí", + "tpt": "Tlachichilco Tepehua", + "tpu": "Tampuan", + "tpv": "Tanapag", + "tpw": "Tupí", + "tpx": "Acatepec Me'phaa; Acatepec Tlapanec", + "tpy": "Trumai", + "tpz": "Tinputz", + "tqb": "Tembé", + "tql": "Lehali", + "tqm": "Turumsa", + "tqn": "Tenino", + "tqo": "Toaripi", + "tqp": "Tomoip", + "tqq": "Tunni", + "tqr": "Torona", + "tqt": "Western Totonac", + "tqu": "Touo", + "tqw": "Tonkawa", + "tr": "Turkish", + "tra": "Tirahi", + "trb": "Terebu", + "trc": "Copala Triqui", + "trd": "Turi", + "tre": "East Tarangan", + "trf": "Trinidadian Creole English", + "trg": "Lishán Didán", + "trh": "Turaka", + "tri": "Trió", + "trj": "Toram", + "trk": "Turkic languages", + "trl": "Traveller Scottish", + "trm": "Tregami", + "trn": "Trinitario", + "tro": "Tarao Naga", + "trp": "Kok Borok", + "trq": "San Martín Itunyoso Triqui", + "trr": "Taushiro", + "trs": "Chicahuaxtla Triqui", + "trt": "Tunggare", + "tru": "Turoyo; Surayt", + "trv": "Sediq; Seediq; Taroko", + "trw": "Torwali", + "trx": "Tringgus-Sembaan Bidayuh", + "try": "Turung", + "trz": "Torá", + "ts": "Tsonga", + "tsa": "Tsaangi", + "tsb": "Tsamai", + "tsc": "Tswa", + "tsd": "Tsakonian", + "tse": "Tunisian Sign Language", + "tsg": "Tausug", + "tsh": "Tsuvan", + "tsi": "Tsimshian", + "tsj": "Tshangla", + "tsk": "Tseku", + "tsl": "Ts'ün-Lao", + "tsm": "Turkish Sign Language; Türk İşaret Dili", + "tsp": "Northern Toussian", + "tsq": "Thai Sign Language", + "tsr": "Akei", + "tss": "Taiwan Sign Language", + "tst": "Tondi Songway Kiini", + "tsu": "Tsou", + "tsv": "Tsogo", + "tsw": "Tsishingini", + "tsx": "Mubami", + "tsy": "Tebul Sign Language", + "tsz": "Purepecha", + "tt": "Tatar", + "tta": "Tutelo", + "ttb": "Gaa", + "ttc": "Tektiteko", + "ttd": "Tauade", + "tte": "Bwanabwana", + "ttf": "Tuotomb", + "ttg": "Tutong", + "tth": "Upper Ta'oih", + "tti": "Tobati", + "ttj": "Tooro", + "ttk": "Totoro", + "ttl": "Totela", + "ttm": "Northern Tutchone", + "ttn": "Towei", + "tto": "Lower Ta'oih", + "ttp": "Tombelala", + "ttq": "Tawallammat Tamajaq", + "ttr": "Tera", + "tts": "Northeastern Thai", + "ttt": "Muslim Tat", + "ttu": "Torau", + "ttv": "Titan", + "ttw": "Long Wat", + "tty": "Sikaritai", + "ttz": "Tsum", + "tua": "Wiarumus", + "tub": "Tübatulabal", + "tuc": "Mutu", + "tud": "Tuxá", + "tue": "Tuyuca", + "tuf": "Central Tunebo", + "tug": "Tunia", + "tuh": "Taulil", + "tui": "Tupuri", + "tuj": "Tugutil", + "tul": "Tula", + "tum": "Tumbuka", + "tun": "Tunica", + "tuo": "Tucano", + "tup": "Tupi languages", + "tuq": "Tedaga", + "tus": "Tuscarora", + "tut": "Altaic languages", + "tuu": "Tututni", + "tuv": "Turkana", + "tuw": "Tungus languages", + "tux": "Tuxináwa", + "tuy": "Tugen", + "tuz": "Turka", + "tva": "Vaghua", + "tvd": "Tsuvadi", + "tve": "Te'un", + "tvk": "Southeast Ambrym", + "tvl": "Tuvalu", + "tvm": "Tela-Masbuar", + "tvn": "Tavoyan", + "tvo": "Tidore", + "tvs": "Taveta", + "tvt": "Tutsa Naga", + "tvu": "Tunen", + "tvw": "Sedoa", + "tvx": "Taivoan", + "tvy": "Timor Pidgin", + "tw": "Twi", + "twa": "Twana", + "twb": "Western Tawbuid", + "twc": "Teshenawa", + "twd": "Twents", + "twe": "Tewa (Indonesia)", + "twf": "Northern Tiwa", + "twg": "Tereweng", + "twh": "Tai Dón", + "twl": "Tawara", + "twm": "Tawang Monpa", + "twn": "Twendi", + "two": "Tswapong", + "twp": "Ere", + "twq": "Tasawaq", + "twr": "Southwestern Tarahumara", + "twt": "Turiwára", + "twu": "Termanu", + "tww": "Tuwari", + "twx": "Tewe", + "twy": "Tawoyan", + "txa": "Tombonuo", + "txb": "Tokharian B", + "txc": "Tsetsaut", + "txe": "Totoli", + "txg": "Tangut", + "txh": "Thracian", + "txi": "Ikpeng", + "txj": "Tarjumo", + "txm": "Tomini", + "txn": "West Tarangan", + "txo": "Toto", + "txq": "Tii", + "txr": "Tartessian", + "txs": "Tonsea", + "txt": "Citak", + "txu": "Kayapó", + "txx": "Tatana", + "txy": "Tanosy Malagasy", + "ty": "Tahitian", + "tya": "Tauya", + "tye": "Kyanga", + "tyh": "O'du", + "tyi": "Teke-Tsaayi", + "tyj": "Tai Do; Tai Yo", + "tyl": "Thu Lao", + "tyn": "Kombai", + "typ": "Thaypan", + "tyr": "Tai Daeng", + "tys": "Tày Sa Pa", + "tyt": "Tày Tac", + "tyu": "Kua", + "tyv": "Tuvinian", + "tyx": "Teke-Tyee", + "tyy": "Tiyaa", + "tyz": "Tày", + "tza": "Tanzanian Sign Language", + "tzh": "Tzeltal", + "tzj": "Tz'utujil", + "tzl": "Talossan", + "tzm": "Central Atlas Tamazight", + "tzn": "Tugun", + "tzo": "Tzotzil", + "tzx": "Tabriak", + "uam": "Uamué", + "uan": "Kuan", + "uar": "Tairuma", + "uba": "Ubang", + "ubi": "Ubi", + "ubl": "Buhi'non Bikol", + "ubr": "Ubir", + "ubu": "Umbu-Ungu", + "uby": "Ubykh", + "uda": "Uda", + "ude": "Udihe", + "udg": "Muduga", + "udi": "Udi", + "udj": "Ujir", + "udl": "Wuzlam", + "udm": "Udmurt", + "udu": "Uduk", + "ues": "Kioko", + "ufi": "Ufim", + "ug": "Uighur; Uyghur", + "uga": "Ugaritic", + "ugb": "Kuku-Ugbanh", + "uge": "Ughele", + "ugh": "Kubachi", + "ugn": "Ugandan Sign Language", + "ugo": "Ugong", + "ugy": "Uruguayan Sign Language", + "uha": "Uhami", + "uhn": "Damal", + "uis": "Uisai", + "uiv": "Iyive", + "uji": "Tanjijili", + "uk": "Ukrainian", + "uka": "Kaburi", + "ukg": "Ukuriguma", + "ukh": "Ukhwejo", + "uki": "Kui (India)", + "ukk": "Muak Sa-aak", + "ukl": "Ukrainian Sign Language", + "ukp": "Ukpe-Bayobiri", + "ukq": "Ukwa", + "uks": "Urubú-Kaapor Sign Language; Kaapor Sign Language", + "uku": "Ukue", + "ukv": "Kuku", + "ukw": "Ukwuani-Aboh-Ndoni", + "uky": "Kuuk-Yak", + "ula": "Fungwa", + "ulb": "Ulukwumi", + "ulc": "Ulch", + "ule": "Lule", + "ulf": "Usku; Afra", + "uli": "Ulithian", + "ulk": "Meriam Mir", + "ull": "Ullatan", + "ulm": "Ulumanda'", + "uln": "Unserdeutsch", + "ulu": "Uma' Lung", + "ulw": "Ulwa", + "uma": "Umatilla", + "umb": "Umbundu", + "umc": "Marrucinian", + "umd": "Umbindhamu", + "umg": "Morrobalama; Umbuygamu", + "umi": "Ukit", + "umm": "Umon", + "umn": "Makyan Naga", + "umo": "Umotína", + "ump": "Umpila", + "umr": "Umbugarla", + "ums": "Pendau", + "umu": "Munsee", + "una": "North Watut", + "und": "Undetermined", + "une": "Uneme", + "ung": "Ngarinyin", + "uni": "Uni", + "unk": "Enawené-Nawé", + "unm": "Unami", + "unn": "Kurnai", + "unr": "Mundari", + "unu": "Unubahe", + "unx": "Munda", + "unz": "Unde Kaili", + "uon": "Kulon", + "upi": "Umeda", + "upv": "Uripiv-Wala-Rano-Atchin", + "ur": "Urdu", + "ura": "Urarina", + "urb": "Urubú-Kaapor; Kaapor", + "urc": "Urningangg", + "ure": "Uru", + "urf": "Uradhi", + "urg": "Urigina", + "urh": "Urhobo", + "uri": "Urim", + "urj": "Uralic languages", + "urk": "Urak Lawoi'", + "url": "Urali", + "urm": "Urapmin", + "urn": "Uruangnirin", + "uro": "Ura (Papua New Guinea)", + "urp": "Uru-Pa-In", + "urr": "Lehalurup; Löyöp", + "urt": "Urat", + "uru": "Urumi", + "urv": "Uruava", + "urw": "Sop", + "urx": "Urimo", + "ury": "Orya", + "urz": "Uru-Eu-Wau-Wau", + "usa": "Usarufa", + "ush": "Ushojo", + "usi": "Usui", + "usk": "Usaghade", + "usp": "Uspanteco", + "uss": "us-Saare", + "usu": "Uya", + "uta": "Otank", + "ute": "Ute-Southern Paiute", + "uth": "ut-Hun", + "utp": "Amba (Solomon Islands)", + "utr": "Etulo", + "utu": "Utu", + "uum": "Urum", + "uur": "Ura (Vanuatu)", + "uuu": "U", + "uve": "West Uvean; Fagauvea", + "uvh": "Uri", + "uvl": "Lote", + "uwa": "Kuku-Uwanh", + "uya": "Doko-Uyanga", + "uz": "Uzbek", + "uzn": "Northern Uzbek", + "uzs": "Southern Uzbek", + "vaa": "Vaagri Booli", + "vae": "Vale", + "vaf": "Vafsi", + "vag": "Vagla", + "vah": "Varhadi-Nagpuri", + "vai": "Vai", + "vaj": "Sekele; Northwestern ǃKung; Vasekele", + "val": "Vehes", + "vam": "Vanimo", + "van": "Valman", + "vao": "Vao", + "vap": "Vaiphei", + "var": "Huarijio", + "vas": "Vasavi", + "vau": "Vanuma", + "vav": "Varli", + "vay": "Wayu", + "vbb": "Southeast Babar", + "vbk": "Southwestern Bontok", + "ve": "Venda", + "vec": "Venetian", + "ved": "Veddah", + "vel": "Veluws", + "vem": "Vemgo-Mabas", + "veo": "Ventureño", + "vep": "Veps", + "ver": "Mom Jango", + "vgr": "Vaghri", + "vgt": "Vlaamse Gebarentaal; Flemish Sign Language", + "vi": "Vietnamese", + "vic": "Virgin Islands Creole English", + "vid": "Vidunda", + "vif": "Vili", + "vig": "Viemo", + "vil": "Vilela", + "vin": "Vinza", + "vis": "Vishavan", + "vit": "Viti", + "viv": "Iduna", + "vka": "Kariyarra", + "vkj": "Kujarge", + "vkk": "Kaur", + "vkl": "Kulisusu", + "vkm": "Kamakan", + "vkn": "Koro Nulu", + "vko": "Kodeoha", + "vkp": "Korlai Creole Portuguese", + "vkt": "Tenggarong Kutai Malay", + "vku": "Kurrama", + "vkz": "Koro Zuba", + "vlp": "Valpei", + "vls": "Vlaams", + "vma": "Martuyhunira", + "vmb": "Barbaram", + "vmc": "Juxtlahuaca Mixtec", + "vmd": "Mudu Koraga", + "vme": "East Masela", + "vmf": "Mainfränkisch", + "vmg": "Lungalunga", + "vmh": "Maraghei", + "vmi": "Miwa", + "vmj": "Ixtayutla Mixtec", + "vmk": "Makhuwa-Shirima", + "vml": "Malgana", + "vmm": "Mitlatongo Mixtec", + "vmp": "Soyaltepec Mazatec", + "vmq": "Soyaltepec Mixtec", + "vmr": "Marenje", + "vms": "Moksela", + "vmu": "Muluridyi", + "vmv": "Valley Maidu", + "vmw": "Makhuwa", + "vmx": "Tamazola Mixtec", + "vmy": "Ayautla Mazatec", + "vmz": "Mazatlán Mazatec", + "vnk": "Vano; Lovono", + "vnm": "Vinmavis; Neve'ei", + "vnp": "Vunapu", + "vo": "Volapük", + "vor": "Voro", + "vot": "Votic", + "vra": "Vera'a", + "vro": "Võro", + "vrs": "Varisi", + "vrt": "Burmbar; Banam Bay", + "vsi": "Moldova Sign Language", + "vsl": "Venezuelan Sign Language", + "vsv": "Valencian Sign Language; Llengua de signes valenciana", + "vto": "Vitou", + "vum": "Vumbu", + "vun": "Vunjo", + "vut": "Vute", + "vwa": "Awa (China)", + "wa": "Walloon", + "waa": "Walla Walla", + "wab": "Wab", + "wac": "Wasco-Wishram", + "wad": "Wamesa; Wondama", + "wae": "Walser", + "waf": "Wakoná", + "wag": "Wa'ema", + "wah": "Watubela", + "wai": "Wares", + "waj": "Waffa", + "wak": "Wakashan languages", + "wal": "Wolaytta; Wolaitta", + "wam": "Wampanoag", + "wan": "Wan", + "wao": "Wappo", + "wap": "Wapishana", + "waq": "Wagiman", + "war": "Waray (Philippines)", + "was": "Washo", + "wat": "Kaninuwa", + "wau": "Waurá", + "wav": "Waka", + "waw": "Waiwai", + "wax": "Watam; Marangis", + "way": "Wayana", + "waz": "Wampur", + "wba": "Warao", + "wbb": "Wabo", + "wbe": "Waritai", + "wbf": "Wara", + "wbh": "Wanda", + "wbi": "Vwanji", + "wbj": "Alagwa", + "wbk": "Waigali", + "wbl": "Wakhi", + "wbm": "Wa", + "wbp": "Warlpiri", + "wbq": "Waddar", + "wbr": "Wagdi", + "wbs": "West Bengal Sign Language", + "wbt": "Warnman", + "wbv": "Wajarri", + "wbw": "Woi", + "wca": "Yanomámi", + "wci": "Waci Gbe", + "wdd": "Wandji", + "wdg": "Wadaginam", + "wdj": "Wadjiginy", + "wdk": "Wadikali", + "wdt": "Wendat", + "wdu": "Wadjigu", + "wdy": "Wadjabangayi", + "wea": "Wewaw", + "wec": "Wè Western", + "wed": "Wedau", + "weg": "Wergaia", + "weh": "Weh", + "wei": "Kiunum", + "wem": "Weme Gbe", + "wen": "Sorbian languages", + "weo": "Wemale", + "wep": "Westphalien", + "wer": "Weri", + "wes": "Cameroon Pidgin", + "wet": "Perai", + "weu": "Rawngtu Chin", + "wew": "Wejewa", + "wfg": "Yafi; Zorop", + "wga": "Wagaya", + "wgb": "Wagawaga", + "wgg": "Wangkangurru; Wangganguru", + "wgi": "Wahgi", + "wgo": "Waigeo", + "wgu": "Wirangu", + "wgy": "Warrgamay", + "wha": "Sou Upaa; Manusela", + "whg": "North Wahgi", + "whk": "Wahau Kenyah", + "whu": "Wahau Kayan", + "wib": "Southern Toussian", + "wic": "Wichita", + "wie": "Wik-Epa", + "wif": "Wik-Keyangan", + "wig": "Wik Ngathan", + "wih": "Wik-Me'anha", + "wii": "Minidien", + "wij": "Wik-Iiyanh", + "wik": "Wikalkan", + "wil": "Wilawila", + "wim": "Wik-Mungkan", + "win": "Ho-Chunk", + "wir": "Wiraféd", + "wiu": "Wiru", + "wiv": "Vitu", + "wiy": "Wiyot", + "wja": "Waja", + "wji": "Warji", + "wka": "Kw'adza", + "wkb": "Kumbaran", + "wkd": "Wakde; Mo", + "wkl": "Kalanadi", + "wkr": "Keerray-Woorroong", + "wku": "Kunduvadi", + "wkw": "Wakawaka", + "wky": "Wangkayutyuru", + "wla": "Walio", + "wlc": "Mwali Comorian", + "wle": "Wolane", + "wlg": "Kunbarlang", + "wlh": "Welaun", + "wli": "Waioli", + "wlk": "Wailaki", + "wll": "Wali (Sudan)", + "wlm": "Middle Welsh", + "wlo": "Wolio", + "wlr": "Wailapa", + "wls": "Wallisian", + "wlu": "Wuliwuli", + "wlv": "Wichí Lhamtés Vejoz", + "wlw": "Walak", + "wlx": "Wali (Ghana)", + "wly": "Waling", + "wma": "Mawa (Nigeria)", + "wmb": "Wambaya", + "wmc": "Wamas", + "wmd": "Mamaindé", + "wme": "Wambule", + "wmg": "Western Minyag", + "wmh": "Waima'a", + "wmi": "Wamin", + "wmm": "Maiwa (Indonesia)", + "wmn": "Waamwang", + "wmo": "Wom (Papua New Guinea)", + "wms": "Wambon", + "wmt": "Walmajarri", + "wmw": "Mwani", + "wmx": "Womo", + "wnb": "Wanambre", + "wnc": "Wantoat", + "wnd": "Wandarang", + "wne": "Waneci", + "wng": "Wanggom", + "wni": "Ndzwani Comorian", + "wnk": "Wanukaka", + "wnm": "Wanggamala", + "wnn": "Wunumara", + "wno": "Wano", + "wnp": "Wanap", + "wnu": "Usan", + "wnw": "Wintu", + "wny": "Wanyi; Waanyi", + "wo": "Wolof", + "woa": "Kuwema; Tyaraity", + "wob": "Wè Northern", + "woc": "Wogeo", + "wod": "Wolani", + "woe": "Woleaian", + "wof": "Gambian Wolof", + "wog": "Wogamusin", + "woi": "Kamang", + "wok": "Longto", + "wom": "Wom (Nigeria)", + "won": "Wongo", + "woo": "Manombai", + "wor": "Woria", + "wos": "Hanga Hundi", + "wow": "Wawonii", + "woy": "Weyto", + "wpc": "Maco", + "wrb": "Waluwarra; Warluwara", + "wrg": "Warungu; Gudjal", + "wrh": "Wiradjuri", + "wri": "Wariyangga", + "wrk": "Garrwa", + "wrl": "Warlmanpa", + "wrm": "Warumungu", + "wrn": "Warnang", + "wro": "Worrorra", + "wrp": "Waropen", + "wrr": "Wardaman", + "wrs": "Waris", + "wru": "Waru", + "wrv": "Waruna", + "wrw": "Gugu Warra", + "wrx": "Wae Rana", + "wry": "Merwari", + "wrz": "Waray (Australia)", + "wsa": "Warembori", + "wsg": "Adilabad Gondi", + "wsi": "Wusi", + "wsk": "Waskia", + "wsr": "Owenia", + "wss": "Wasa", + "wsu": "Wasu", + "wsv": "Wotapuri-Katarqalai", + "wtf": "Watiwa", + "wth": "Wathawurrung", + "wti": "Berta", + "wtk": "Watakataui", + "wtm": "Mewati", + "wtw": "Wotu", + "wua": "Wikngenchera", + "wub": "Wunambal", + "wud": "Wudu", + "wuh": "Wutunhua", + "wul": "Silimo", + "wum": "Wumbvu", + "wun": "Bungu", + "wur": "Wurrugu", + "wut": "Wutung", + "wuu": "Wu Chinese", + "wuv": "Wuvulu-Aua", + "wux": "Wulna", + "wuy": "Wauyai", + "wwa": "Waama", + "wwb": "Wakabunga", + "wwo": "Wetamut; Dorig", + "wwr": "Warrwa", + "www": "Wawa", + "wxa": "Waxianghua", + "wxw": "Wardandi", + "wyb": "Wangaaybuwan-Ngiyambaa", + "wyi": "Woiwurrung", + "wym": "Wymysorys", + "wyn": "Wyandot", + "wyr": "Wayoró", + "wyy": "Western Fijian", + "xaa": "Andalusian Arabic", + "xab": "Sambe", + "xac": "Kachari", + "xad": "Adai", + "xae": "Aequian", + "xag": "Aghwan", + "xai": "Kaimbé", + "xaj": "Ararandewára", + "xak": "Máku", + "xal": "Kalmyk; Oirat", + "xam": "ǀXam", + "xan": "Xamtanga", + "xao": "Khao", + "xap": "Apalachee", + "xaq": "Aquitanian", + "xar": "Karami", + "xas": "Kamas", + "xat": "Katawixi", + "xau": "Kauwera", + "xav": "Xavánte", + "xaw": "Kawaiisu", + "xay": "Kayan Mahakam", + "xbb": "Lower Burdekin", + "xbc": "Bactrian", + "xbd": "Bindal", + "xbe": "Bigambal", + "xbg": "Bunganditj", + "xbi": "Kombio", + "xbj": "Birrpayi", + "xbm": "Middle Breton", + "xbn": "Kenaboi", + "xbo": "Bolgarian", + "xbp": "Bibbulman", + "xbr": "Kambera", + "xbw": "Kambiwá", + "xby": "Batjala; Batyala", + "xcb": "Cumbric", + "xcc": "Camunic", + "xce": "Celtiberian", + "xcg": "Cisalpine Gaulish", + "xch": "Chemakum; Chimakum", + "xcl": "Classical Armenian", + "xcm": "Comecrudo", + "xcn": "Cotoname", + "xco": "Chorasmian", + "xcr": "Carian", + "xct": "Classical Tibetan", + "xcu": "Curonian", + "xcv": "Chuvantsy", + "xcw": "Coahuilteco", + "xcy": "Cayuse", + "xda": "Darkinyung", + "xdc": "Dacian", + "xdk": "Dharuk", + "xdm": "Edomite", + "xdo": "Kwandu", + "xdq": "Kaitag", + "xdy": "Malayic Dayak", + "xeb": "Eblan", + "xed": "Hdi", + "xeg": "ǁXegwi", + "xel": "Kelo", + "xem": "Kembayan", + "xep": "Epi-Olmec", + "xer": "Xerénte", + "xes": "Kesawai", + "xet": "Xetá", + "xeu": "Keoru-Ahia", + "xfa": "Faliscan", + "xga": "Galatian", + "xgb": "Gbin", + "xgd": "Gudang", + "xgf": "Gabrielino-Fernandeño", + "xgg": "Goreng", + "xgi": "Garingbal", + "xgl": "Galindan", + "xgm": "Dharumbal; Guwinmal", + "xgn": "Mongolian languages", + "xgr": "Garza", + "xgu": "Unggumi", + "xgw": "Guwa", + "xh": "Xhosa", + "xha": "Harami", + "xhc": "Hunnic", + "xhd": "Hadrami", + "xhe": "Khetrani", + "xhm": "Middle Khmer (1400 to 1850 CE)", + "xhr": "Hernican", + "xht": "Hattic", + "xhu": "Hurrian", + "xhv": "Khua", + "xib": "Iberian", + "xii": "Xiri", + "xil": "Illyrian", + "xin": "Xinca", + "xir": "Xiriâna", + "xis": "Kisan", + "xiv": "Indus Valley Language", + "xiy": "Xipaya", + "xjb": "Minjungbal", + "xjt": "Jaitmatang", + "xka": "Kalkoti", + "xkb": "Northern Nago", + "xkc": "Kho'ini", + "xkd": "Mendalam Kayan", + "xke": "Kereho", + "xkf": "Khengkha", + "xkg": "Kagoro", + "xki": "Kenyan Sign Language", + "xkj": "Kajali", + "xkk": "Kachok; Kaco'", + "xkl": "Mainstream Kenyah", + "xkn": "Kayan River Kayan", + "xko": "Kiorr", + "xkp": "Kabatei", + "xkq": "Koroni", + "xkr": "Xakriabá", + "xks": "Kumbewaha", + "xkt": "Kantosi", + "xku": "Kaamba", + "xkv": "Kgalagadi", + "xkw": "Kembra", + "xkx": "Karore", + "xky": "Uma' Lasan", + "xkz": "Kurtokha", + "xla": "Kamula", + "xlb": "Loup B", + "xlc": "Lycian", + "xld": "Lydian", + "xle": "Lemnian", + "xlg": "Ligurian (Ancient)", + "xli": "Liburnian", + "xln": "Alanic", + "xlo": "Loup A", + "xlp": "Lepontic", + "xls": "Lusitanian", + "xlu": "Cuneiform Luwian", + "xly": "Elymian", + "xma": "Mushungulu", + "xmb": "Mbonga", + "xmc": "Makhuwa-Marrevone", + "xmd": "Mbudum", + "xme": "Median", + "xmf": "Mingrelian", + "xmg": "Mengaka", + "xmh": "Kugu-Muminh", + "xmj": "Majera", + "xmk": "Ancient Macedonian", + "xml": "Malaysian Sign Language", + "xmm": "Manado Malay", + "xmn": "Manichaean Middle Persian", + "xmo": "Morerebi", + "xmp": "Kuku-Mu'inh", + "xmq": "Kuku-Mangk", + "xmr": "Meroitic", + "xms": "Moroccan Sign Language", + "xmt": "Matbat", + "xmu": "Kamu", + "xmv": "Antankarana Malagasy; Tankarana Malagasy", + "xmw": "Tsimihety Malagasy", + "xmx": "Salawati; Maden", + "xmy": "Mayaguduna", + "xmz": "Mori Bawah", + "xna": "Ancient North Arabian", + "xnb": "Kanakanabu", + "xnd": "Na-Dene languages", + "xng": "Middle Mongolian", + "xnh": "Kuanhua", + "xni": "Ngarigu", + "xnj": "Ngoni (Tanzania)", + "xnk": "Nganakarti", + "xnm": "Ngumbarl", + "xnn": "Northern Kankanay", + "xno": "Anglo-Norman", + "xnq": "Ngoni (Mozambique)", + "xnr": "Kangri", + "xns": "Kanashi", + "xnt": "Narragansett", + "xnu": "Nukunul", + "xny": "Nyiyaparli", + "xnz": "Kenzi; Mattoki", + "xoc": "O'chi'chi'", + "xod": "Kokoda", + "xog": "Soga", + "xoi": "Kominimung", + "xok": "Xokleng", + "xom": "Komo (Sudan)", + "xon": "Konkomba", + "xoo": "Xukurú", + "xop": "Kopar", + "xor": "Korubo", + "xow": "Kowaki", + "xpa": "Pirriya", + "xpb": "Northeastern Tasmanian; Pyemmairrener", + "xpc": "Pecheneg", + "xpd": "Oyster Bay Tasmanian", + "xpe": "Liberia Kpelle", + "xpf": "Southeast Tasmanian; Nuenonne", + "xpg": "Phrygian", + "xph": "North Midlands Tasmanian; Tyerrenoterpanner", + "xpi": "Pictish", + "xpj": "Mpalitjanh", + "xpk": "Kulina Pano", + "xpl": "Port Sorell Tasmanian", + "xpm": "Pumpokol", + "xpn": "Kapinawá", + "xpo": "Pochutec", + "xpp": "Puyo-Paekche", + "xpq": "Mohegan-Pequot", + "xpr": "Parthian", + "xps": "Pisidian", + "xpt": "Punthamara", + "xpu": "Punic", + "xpv": "Northern Tasmanian; Tommeginne", + "xpw": "Northwestern Tasmanian; Peerapper", + "xpx": "Southwestern Tasmanian; Toogee", + "xpy": "Puyo", + "xpz": "Bruny Island Tasmanian", + "xqa": "Karakhanid", + "xqt": "Qatabanian", + "xra": "Krahô", + "xrb": "Eastern Karaboro", + "xrd": "Gundungurra", + "xre": "Kreye", + "xrg": "Minang", + "xri": "Krikati-Timbira", + "xrm": "Armazic", + "xrn": "Arin", + "xrr": "Raetic", + "xrt": "Aranama-Tamique", + "xru": "Marriammu", + "xrw": "Karawa", + "xsa": "Sabaean", + "xsb": "Sambal", + "xsc": "Scythian", + "xsd": "Sidetic", + "xse": "Sempan", + "xsh": "Shamang", + "xsi": "Sio", + "xsj": "Subi", + "xsl": "South Slavey", + "xsm": "Kasem", + "xsn": "Sanga (Nigeria)", + "xso": "Solano", + "xsp": "Silopi", + "xsq": "Makhuwa-Saka", + "xsr": "Sherpa", + "xss": "Assan", + "xsu": "Sanumá", + "xsv": "Sudovian", + "xsy": "Saisiyat", + "xta": "Alcozauca Mixtec", + "xtb": "Chazumba Mixtec", + "xtc": "Katcha-Kadugli-Miri", + "xtd": "Diuxi-Tilantongo Mixtec", + "xte": "Ketengban", + "xtg": "Transalpine Gaulish", + "xth": "Yitha Yitha", + "xti": "Sinicahua Mixtec", + "xtj": "San Juan Teita Mixtec", + "xtl": "Tijaltepec Mixtec", + "xtm": "Magdalena Peñasco Mixtec", + "xtn": "Northern Tlaxiaco Mixtec", + "xto": "Tokharian A", + "xtp": "San Miguel Piedras Mixtec", + "xtq": "Tumshuqese", + "xtr": "Early Tripuri", + "xts": "Sindihui Mixtec", + "xtt": "Tacahua Mixtec", + "xtu": "Cuyamecalco Mixtec", + "xtv": "Thawa", + "xtw": "Tawandê", + "xty": "Yoloxochitl Mixtec", + "xua": "Alu Kurumba", + "xub": "Betta Kurumba", + "xud": "Umiida", + "xug": "Kunigami", + "xuj": "Jennu Kurumba", + "xul": "Ngunawal; Nunukul", + "xum": "Umbrian", + "xun": "Unggaranggu", + "xuo": "Kuo", + "xup": "Upper Umpqua", + "xur": "Urartian", + "xut": "Kuthant", + "xuu": "Kxoe; Khwedam", + "xve": "Venetic", + "xvi": "Kamviri", + "xvn": "Vandalic", + "xvo": "Volscian", + "xvs": "Vestinian", + "xwa": "Kwaza", + "xwc": "Woccon", + "xwd": "Wadi Wadi", + "xwe": "Xwela Gbe", + "xwg": "Kwegu", + "xwj": "Wajuk", + "xwk": "Wangkumara", + "xwl": "Western Xwla Gbe", + "xwo": "Written Oirat", + "xwr": "Kwerba Mamberamo", + "xwt": "Wotjobaluk", + "xww": "Wemba Wemba", + "xxb": "Boro (Ghana)", + "xxk": "Ke'o", + "xxm": "Minkin", + "xxr": "Koropó", + "xxt": "Tambora", + "xya": "Yaygir", + "xyb": "Yandjibara", + "xyj": "Mayi-Yapi", + "xyk": "Mayi-Kulan", + "xyl": "Yalakalore", + "xyt": "Mayi-Thakurti", + "xyy": "Yorta Yorta", + "xzh": "Zhang-Zhung", + "xzm": "Zemgalian", + "xzp": "Ancient Zapotec", + "yaa": "Yaminahua", + "yab": "Yuhup", + "yac": "Pass Valley Yali", + "yad": "Yagua", + "yae": "Pumé", + "yaf": "Yaka (Democratic Republic of Congo)", + "yag": "Yámana", + "yah": "Yazgulyam", + "yai": "Yagnobi", + "yaj": "Banda-Yangere", + "yak": "Yakama", + "yal": "Yalunka", + "yam": "Yamba", + "yan": "Mayangna", + "yao": "Yao", + "yap": "Yapese", + "yaq": "Yaqui", + "yar": "Yabarana", + "yas": "Nugunu (Cameroon)", + "yat": "Yambeta", + "yau": "Yuwana", + "yav": "Yangben", + "yaw": "Yawalapití", + "yax": "Yauma", + "yay": "Agwagwune", + "yaz": "Lokaa", + "yba": "Yala", + "ybb": "Yemba", + "ybe": "West Yugur", + "ybh": "Yakha", + "ybi": "Yamphu", + "ybj": "Hasha", + "ybk": "Bokha", + "ybl": "Yukuben", + "ybm": "Yaben", + "ybn": "Yabaâna", + "ybo": "Yabong", + "ybx": "Yawiyo", + "yby": "Yaweyuha", + "ych": "Chesu", + "ycl": "Lolopo", + "ycn": "Yucuna", + "ycp": "Chepya", + "yda": "Yanda", + "ydd": "Eastern Yiddish", + "yde": "Yangum Dey", + "ydg": "Yidgha", + "ydk": "Yoidik", + "yea": "Ravula", + "yec": "Yeniche", + "yee": "Yimas", + "yei": "Yeni", + "yej": "Yevanic", + "yel": "Yela", + "yer": "Tarok", + "yes": "Nyankpa", + "yet": "Yetfa", + "yeu": "Yerukula", + "yev": "Yapunda", + "yey": "Yeyi", + "yga": "Malyangapa", + "ygi": "Yiningayi", + "ygl": "Yangum Gel", + "ygm": "Yagomi", + "ygp": "Gepo", + "ygr": "Yagaria", + "ygs": "Yolŋu Sign Language", + "ygu": "Yugul", + "ygw": "Yagwoia", + "yha": "Baha Buyang", + "yhd": "Judeo-Iraqi Arabic", + "yhl": "Hlepho Phowa", + "yhs": "Yan-nhaŋu Sign Language", + "yi": "Yiddish", + "yia": "Yinggarda", + "yif": "Ache", + "yig": "Wusa Nasu", + "yih": "Western Yiddish", + "yii": "Yidiny", + "yij": "Yindjibarndi", + "yik": "Dongshanba Lalo", + "yil": "Yindjilandji", + "yim": "Yimchungru Naga", + "yin": "Riang Lai; Yinchia", + "yip": "Pholo", + "yiq": "Miqie", + "yir": "North Awyu", + "yis": "Yis", + "yit": "Eastern Lalu", + "yiu": "Awu", + "yiv": "Northern Nisu", + "yix": "Axi Yi", + "yiz": "Azhe", + "yka": "Yakan", + "ykg": "Northern Yukaghir", + "yki": "Yoke", + "ykk": "Yakaikeke", + "ykl": "Khlula", + "ykm": "Kap", + "ykn": "Kua-nsi", + "yko": "Yasa", + "ykr": "Yekora", + "ykt": "Kathu", + "yku": "Kuamasi", + "yky": "Yakoma", + "yla": "Yaul", + "ylb": "Yaleba", + "yle": "Yele", + "ylg": "Yelogu", + "yli": "Angguruk Yali", + "yll": "Yil", + "ylm": "Limi", + "yln": "Langnian Buyang", + "ylo": "Naluo Yi", + "ylr": "Yalarnnga", + "ylu": "Aribwaung", + "yly": "Nyâlayu; Nyelâyu", + "ymb": "Yambes", + "ymc": "Southern Muji", + "ymd": "Muda", + "yme": "Yameo", + "ymg": "Yamongeri", + "ymh": "Mili", + "ymi": "Moji", + "ymk": "Makwe", + "yml": "Iamalele", + "ymm": "Maay", + "ymn": "Yamna; Sunum", + "ymo": "Yangum Mon", + "ymp": "Yamap", + "ymq": "Qila Muji", + "ymr": "Malasar", + "yms": "Mysian", + "ymx": "Northern Muji", + "ymz": "Muzi", + "yna": "Aluo", + "ynd": "Yandruwandha", + "yne": "Lang'e", + "yng": "Yango", + "ynk": "Naukan Yupik", + "ynl": "Yangulam", + "ynn": "Yana", + "yno": "Yong", + "ynq": "Yendang", + "yns": "Yansi", + "ynu": "Yahuna", + "yo": "Yoruba", + "yob": "Yoba", + "yog": "Yogad", + "yoi": "Yonaguni", + "yok": "Yokuts", + "yol": "Yola", + "yom": "Yombe", + "yon": "Yongkom", + "yot": "Yotti", + "yox": "Yoron", + "yoy": "Yoy", + "ypa": "Phala", + "ypb": "Labo Phowa", + "ypg": "Phola", + "yph": "Phupha", + "ypk": "Yupik languages", + "ypm": "Phuma", + "ypn": "Ani Phowa", + "ypo": "Alo Phola", + "ypp": "Phupa", + "ypz": "Phuza", + "yra": "Yerakai", + "yrb": "Yareba", + "yre": "Yaouré", + "yrk": "Nenets", + "yrl": "Nhengatu", + "yrm": "Yirrk-Mel", + "yrn": "Yerong", + "yro": "Yaroamë", + "yrs": "Yarsun", + "yrw": "Yarawata", + "yry": "Yarluyandi", + "ysc": "Yassic", + "ysd": "Samatao", + "ysg": "Sonaga", + "ysl": "Yugoslavian Sign Language", + "ysm": "Myanmar Sign Language", + "ysn": "Sani", + "yso": "Nisi (China)", + "ysp": "Southern Lolopo", + "ysr": "Sirenik Yupik", + "yss": "Yessan-Mayo", + "ysy": "Sanie", + "yta": "Talu", + "ytl": "Tanglang", + "ytp": "Thopho", + "ytw": "Yout Wam", + "yty": "Yatay", + "yua": "Yucateco; Yucatec Maya", + "yub": "Yugambal", + "yuc": "Yuchi", + "yud": "Judeo-Tripolitanian Arabic", + "yue": "Yue Chinese; Cantonese", + "yuf": "Havasupai-Walapai-Yavapai", + "yug": "Yug", + "yui": "Yurutí", + "yuj": "Karkar-Yuri", + "yuk": "Yuki", + "yul": "Yulu", + "yum": "Quechan", + "yun": "Bena (Nigeria)", + "yup": "Yukpa", + "yuq": "Yuqui", + "yur": "Yurok", + "yut": "Yopno", + "yuw": "Yau (Morobe Province)", + "yux": "Southern Yukaghir", + "yuy": "East Yugur", + "yuz": "Yuracare", + "yva": "Yawa", + "yvt": "Yavitero", + "ywa": "Kalou", + "ywg": "Yinhawangka", + "ywl": "Western Lalu", + "ywn": "Yawanawa", + "ywq": "Wuding-Luquan Yi", + "ywr": "Yawuru", + "ywt": "Xishanba Lalo; Central Lalo", + "ywu": "Wumeng Nasu", + "yww": "Yawarawarga", + "yxa": "Mayawali", + "yxg": "Yagara", + "yxl": "Yardliyawarra", + "yxm": "Yinwum", + "yxu": "Yuyu", + "yxy": "Yabula Yabula", + "yyr": "Yir Yoront", + "yyu": "Yau (Sandaun Province)", + "yyz": "Ayizi", + "yzg": "E'ma Buyang", + "yzk": "Zokhuo", + "za": "Zhuang; Chuang", + "zaa": "Sierra de Juárez Zapotec", + "zab": "Western Tlacolula Valley Zapotec; San Juan Guelavía Zapotec", + "zac": "Ocotlán Zapotec", + "zad": "Cajonos Zapotec", + "zae": "Yareni Zapotec", + "zaf": "Ayoquesco Zapotec", + "zag": "Zaghawa", + "zah": "Zangwal", + "zai": "Isthmus Zapotec", + "zaj": "Zaramo", + "zak": "Zanaki", + "zal": "Zauzou", + "zam": "Miahuatlán Zapotec", + "zao": "Ozolotepec Zapotec", + "zap": "Zapotec", + "zaq": "Aloápam Zapotec", + "zar": "Rincón Zapotec", + "zas": "Santo Domingo Albarradas Zapotec", + "zat": "Tabaa Zapotec", + "zau": "Zangskari", + "zav": "Yatzachi Zapotec", + "zaw": "Mitla Zapotec", + "zax": "Xadani Zapotec", + "zay": "Zayse-Zergulla; Zaysete", + "zaz": "Zari", + "zba": "Balaibalan", + "zbc": "Central Berawan", + "zbe": "East Berawan", + "zbl": "Blissymbols; Bliss; Blissymbolics", + "zbt": "Batui", + "zbu": "Bu (Bauchi State)", + "zbw": "West Berawan", + "zca": "Coatecas Altas Zapotec", + "zcd": "Las Delicias Zapotec", + "zch": "Central Hongshuihe Zhuang", + "zdj": "Ngazidja Comorian", + "zea": "Zeeuws", + "zeg": "Zenag", + "zeh": "Eastern Hongshuihe Zhuang", + "zen": "Zenaga", + "zga": "Kinga", + "zgb": "Guibei Zhuang", + "zgh": "Standard Moroccan Tamazight", + "zgm": "Minz Zhuang", + "zgn": "Guibian Zhuang", + "zgr": "Magori", + "zh": "Chinese", + "zhb": "Zhaba", + "zhd": "Dai Zhuang", + "zhi": "Zhire", + "zhn": "Nong Zhuang", + "zhw": "Zhoa", + "zhx": "Chinese (family)", + "zia": "Zia", + "zib": "Zimbabwe Sign Language", + "zik": "Zimakani", + "zil": "Zialo", + "zim": "Mesme", + "zin": "Zinza", + "ziw": "Zigula", + "ziz": "Zizilivakan", + "zka": "Kaimbulawa", + "zkb": "Koibal", + "zkd": "Kadu", + "zkg": "Koguryo", + "zkh": "Khorezmian", + "zkk": "Karankawa", + "zkn": "Kanan", + "zko": "Kott", + "zkp": "São Paulo Kaingáng", + "zkr": "Zakhring", + "zkt": "Kitan", + "zku": "Kaurna", + "zkv": "Krevinian", + "zkz": "Khazar", + "zla": "Zula", + "zle": "East Slavic languages", + "zlj": "Liujiang Zhuang", + "zlm": "Malay (individual language)", + "zln": "Lianshan Zhuang", + "zlq": "Liuqian Zhuang", + "zls": "South Slavic languages", + "zlw": "West Slavic languages", + "zma": "Manda (Australia)", + "zmb": "Zimba", + "zmc": "Margany", + "zmd": "Maridan", + "zme": "Mangerr", + "zmf": "Mfinu", + "zmg": "Marti Ke", + "zmh": "Makolkol", + "zmi": "Negeri Sembilan Malay", + "zmj": "Maridjabin", + "zmk": "Mandandanyi", + "zml": "Matngala", + "zmm": "Marimanindji; Marramaninyshi", + "zmn": "Mbangwe", + "zmo": "Molo", + "zmp": "Mpuono", + "zmq": "Mituku", + "zmr": "Maranunggu", + "zms": "Mbesa", + "zmt": "Maringarr", + "zmu": "Muruwari", + "zmv": "Mbariman-Gudhinma", + "zmw": "Mbo (Democratic Republic of Congo)", + "zmx": "Bomitaba", + "zmy": "Mariyedi", + "zmz": "Mbandja", + "zna": "Zan Gula", + "znd": "Zande languages", + "zne": "Zande (individual language)", + "zng": "Mang", + "znk": "Manangkari", + "zns": "Mangas", + "zoc": "Copainalá Zoque", + "zoh": "Chimalapa Zoque", + "zom": "Zou", + "zoo": "Asunción Mixtepec Zapotec", + "zoq": "Tabasco Zoque", + "zor": "Rayón Zoque", + "zos": "Francisco León Zoque", + "zpa": "Lachiguiri Zapotec", + "zpb": "Yautepec Zapotec", + "zpc": "Choapan Zapotec", + "zpd": "Southeastern Ixtlán Zapotec", + "zpe": "Petapa Zapotec", + "zpf": "San Pedro Quiatoni Zapotec", + "zpg": "Guevea De Humboldt Zapotec", + "zph": "Totomachapan Zapotec", + "zpi": "Santa María Quiegolani Zapotec", + "zpj": "Quiavicuzas Zapotec", + "zpk": "Tlacolulita Zapotec", + "zpl": "Lachixío Zapotec", + "zpm": "Mixtepec Zapotec", + "zpn": "Santa Inés Yatzechi Zapotec", + "zpo": "Amatlán Zapotec", + "zpp": "El Alto Zapotec", + "zpq": "Zoogocho Zapotec", + "zpr": "Santiago Xanica Zapotec", + "zps": "Coatlán Zapotec", + "zpt": "San Vicente Coatlán Zapotec", + "zpu": "Yalálag Zapotec", + "zpv": "Chichicapan Zapotec", + "zpw": "Zaniza Zapotec", + "zpx": "San Baltazar Loxicha Zapotec", + "zpy": "Mazaltepec Zapotec", + "zpz": "Texmelucan Zapotec", + "zqe": "Qiubei Zhuang", + "zra": "Kara (Korea)", + "zrg": "Mirgan", + "zrn": "Zerenkel", + "zro": "Záparo", + "zrp": "Zarphatic", + "zrs": "Mairasi", + "zsa": "Sarasira", + "zsk": "Kaskean", + "zsl": "Zambian Sign Language", + "zsm": "Standard Malay", + "zsr": "Southern Rincon Zapotec", + "zsu": "Sukurum", + "zte": "Elotepec Zapotec", + "ztg": "Xanaguía Zapotec", + "ztl": "Lapaguía-Guivini Zapotec", + "ztm": "San Agustín Mixtepec Zapotec", + "ztn": "Santa Catarina Albarradas Zapotec", + "ztp": "Loxicha Zapotec", + "ztq": "Quioquitani-Quierí Zapotec", + "zts": "Tilquiapan Zapotec", + "ztt": "Tejalapan Zapotec", + "ztu": "Güilá Zapotec", + "ztx": "Zaachila Zapotec", + "zty": "Yatee Zapotec", + "zu": "Zulu", + "zua": "Zeem", + "zuh": "Tokano", + "zum": "Kumzari", + "zun": "Zuni", + "zuy": "Zumaya", + "zwa": "Zay", + "zyb": "Yongbei Zhuang", + "zyg": "Yang Zhuang", + "zyj": "Youjiang Zhuang", + "zyn": "Yongnan Zhuang", + "zyp": "Zyphe Chin", + "zza": "Zaza; Dimili; Dimli (macrolanguage); Kirdki; Kirmanjki (macrolanguage); Zazaki", + "zzj": "Zuojiang Zhuang" +} \ No newline at end of file diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/multilingualities.json b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/multilingualities.json new file mode 100644 index 0000000000000000000000000000000000000000..a35c79f03dfcf7c8a116b7fc8ee1b383ab5022fa --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/multilingualities.json @@ -0,0 +1,6 @@ +{ + "monolingual": "contains a single language", + "multilingual": "contains multiple languages", + "translation": "contains translated or aligned text", + "other": "other type of language distribution" +} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/readme_structure.yaml b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/readme_structure.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b781e11e8258a446874ebf96104f642d0c190cf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/readme_structure.yaml @@ -0,0 +1,116 @@ +name: "" # Filename comes here +allow_empty: false +allow_empty_text: true +subsections: + - name: "Dataset Card for X" # First-level markdown heading + allow_empty: false + allow_empty_text: true + subsections: + - name: "Table of Contents" + allow_empty: false + allow_empty_text: false + subsections: null # meaning it should not be checked. + - name: "Dataset Description" + allow_empty: false + allow_empty_text: false + subsections: + - name: "Dataset Summary" + allow_empty: false + allow_empty_text: false + subsections: null + - name: "Supported Tasks and Leaderboards" + allow_empty: true + allow_empty_text: true + subsections: null + - name: Languages + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Dataset Structure" + allow_empty: false + allow_empty_text: true + subsections: + - name: "Data Instances" + allow_empty: false + allow_empty_text: true + subsections: null + - name: "Data Fields" + allow_empty: false + allow_empty_text: true + subsections: null + - name: "Data Splits" + allow_empty: false + allow_empty_text: true + subsections: null + - name: "Dataset Creation" + allow_empty: false + allow_empty_text: true + subsections: + - name: "Curation Rationale" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Source Data" + allow_empty: false + allow_empty_text: true + subsections: + - name: "Initial Data Collection and Normalization" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Who are the source language producers?" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Annotations" + allow_empty: false + allow_empty_text: true + subsections: + - name: "Annotation process" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Who are the annotators?" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Personal and Sensitive Information" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Considerations for Using the Data" + allow_empty: true + allow_empty_text: true + subsections: + - name: "Social Impact of Dataset" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Discussion of Biases" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Other Known Limitations" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Additional Information" + allow_empty: true + allow_empty_text: true + subsections: + - name: "Dataset Curators" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Licensing Information" + allow_empty: true + allow_empty_text: true + subsections: null + - name: "Citation Information" + allow_empty: false + allow_empty_text: true + subsections: null + - name: "Contributions" + allow_empty: false + allow_empty_text: false + subsections: null diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/size_categories.json b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/size_categories.json new file mode 100644 index 0000000000000000000000000000000000000000..983ce0c10dbb2e2245f90ae47e9de4c1025d5bb1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/resources/size_categories.json @@ -0,0 +1,14 @@ +[ + "unknown", + "n<1K", + "1K1T" +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/sharding.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/sharding.py new file mode 100644 index 0000000000000000000000000000000000000000..7ee3133b80ea927a076eebc7eedc2e7b25013ffa --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/sharding.py @@ -0,0 +1,96 @@ +from typing import List + +import numpy as np + + +def _number_of_shards_in_gen_kwargs(gen_kwargs: dict) -> int: + """Return the number of possible shards according to the input gen_kwargs""" + # Having lists of different sizes makes sharding ambigious, raise an error in this case + # until we decide how to define sharding without ambiguity for users + lists_lengths = {key: len(value) for key, value in gen_kwargs.items() if isinstance(value, list)} + if len(set(lists_lengths.values())) > 1: + raise RuntimeError( + ( + "Sharding is ambiguous for this dataset: " + + "we found several data sources lists of different lengths, and we don't know over which list we should parallelize:\n" + + "\n".join(f"\t- key {key} has length {length}" for key, length in lists_lengths.items()) + + "\nTo fix this, check the 'gen_kwargs' and make sure to use lists only for data sources, " + + "and use tuples otherwise. In the end there should only be one single list, or several lists with the same length." + ) + ) + max_length = max(lists_lengths.values(), default=0) + return max(1, max_length) + + +def _distribute_shards(num_shards: int, max_num_jobs: int) -> List[range]: + """ + Get the range of shard indices per job. + If num_shards>> _distribute_shards(2, max_num_jobs=4) + [range(0, 1), range(1, 2)] + >>> _distribute_shards(10, max_num_jobs=3) + [range(0, 4), range(4, 7), range(7, 10)] + ``` + """ + shards_indices_per_group = [] + for group_idx in range(max_num_jobs): + num_shards_to_add = num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs)) + if num_shards_to_add == 0: + break + start = shards_indices_per_group[-1].stop if shards_indices_per_group else 0 + shard_indices = range(start, start + num_shards_to_add) + shards_indices_per_group.append(shard_indices) + return shards_indices_per_group + + +def _split_gen_kwargs(gen_kwargs: dict, max_num_jobs: int) -> List[dict]: + """Split the gen_kwargs into `max_num_job` gen_kwargs""" + # Having lists of different sizes makes sharding ambigious, raise an error in this case + num_shards = _number_of_shards_in_gen_kwargs(gen_kwargs) + if num_shards == 1: + return [dict(gen_kwargs)] + else: + shard_indices_per_group = _distribute_shards(num_shards=num_shards, max_num_jobs=max_num_jobs) + return [ + { + key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]] + if isinstance(value, list) + else value + for key, value in gen_kwargs.items() + } + for group_idx in range(len(shard_indices_per_group)) + ] + + +def _merge_gen_kwargs(gen_kwargs_list: List[dict]) -> dict: + return { + key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]] + if isinstance(gen_kwargs_list[0][key], list) + else gen_kwargs_list[0][key] + for key in gen_kwargs_list[0] + } + + +def _shuffle_gen_kwargs(rng: np.random.Generator, gen_kwargs: dict) -> dict: + """Return a shuffled copy of the input gen_kwargs""" + # We must shuffle all the lists, and lists of the same size must have the same shuffling. + # This way entangled lists of (shard, shard_metadata) are still in the right order. + + # First, let's generate the shuffled indices per list size + list_sizes = {len(value) for value in gen_kwargs.values() if isinstance(value, list)} + indices_per_size = {} + for size in list_sizes: + indices_per_size[size] = list(range(size)) + rng.shuffle(indices_per_size[size]) + # Now let's copy the gen_kwargs and shuffle the lists based on their sizes + shuffled_kwargs = dict(gen_kwargs) + for key, value in shuffled_kwargs.items(): + if isinstance(value, list): + shuffled_kwargs[key] = [value[i] for i in indices_per_size[len(value)]] + return shuffled_kwargs diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/stratify.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/stratify.py new file mode 100644 index 0000000000000000000000000000000000000000..d0967aa1abb790f741af5ff920c67e615d1b01da --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/stratify.py @@ -0,0 +1,107 @@ +import numpy as np + + +def approximate_mode(class_counts, n_draws, rng): + """Computes approximate mode of multivariate hypergeometric. + This is an approximation to the mode of the multivariate + hypergeometric given by class_counts and n_draws. + It shouldn't be off by more than one. + It is the mostly likely outcome of drawing n_draws many + samples from the population given by class_counts. + Args + ---------- + class_counts : ndarray of int + Population per class. + n_draws : int + Number of draws (samples to draw) from the overall population. + rng : random state + Used to break ties. + Returns + ------- + sampled_classes : ndarray of int + Number of samples drawn from each class. + np.sum(sampled_classes) == n_draws + + """ + # this computes a bad approximation to the mode of the + # multivariate hypergeometric given by class_counts and n_draws + continuous = n_draws * class_counts / class_counts.sum() + # floored means we don't overshoot n_samples, but probably undershoot + floored = np.floor(continuous) + # we add samples according to how much "left over" probability + # they had, until we arrive at n_samples + need_to_add = int(n_draws - floored.sum()) + if need_to_add > 0: + remainder = continuous - floored + values = np.sort(np.unique(remainder))[::-1] + # add according to remainder, but break ties + # randomly to avoid biases + for value in values: + (inds,) = np.where(remainder == value) + # if we need_to_add less than what's in inds + # we draw randomly from them. + # if we need to add more, we add them all and + # go to the next value + add_now = min(len(inds), need_to_add) + inds = rng.choice(inds, size=add_now, replace=False) + floored[inds] += 1 + need_to_add -= add_now + if need_to_add == 0: + break + return floored.astype(np.int64) + + +def stratified_shuffle_split_generate_indices(y, n_train, n_test, rng, n_splits=10): + """ + + Provides train/test indices to split data in train/test sets. + It's reference is taken from StratifiedShuffleSplit implementation + of scikit-learn library. + + Args + ---------- + + n_train : int, + represents the absolute number of train samples. + + n_test : int, + represents the absolute number of test samples. + + random_state : int or RandomState instance, default=None + Controls the randomness of the training and testing indices produced. + Pass an int for reproducible output across multiple function calls. + + n_splits : int, default=10 + Number of re-shuffling & splitting iterations. + """ + classes, y_indices = np.unique(y, return_inverse=True) + n_classes = classes.shape[0] + class_counts = np.bincount(y_indices) + if np.min(class_counts) < 2: + raise ValueError("Minimum class count error") + if n_train < n_classes: + raise ValueError( + "The train_size = %d should be greater or " "equal to the number of classes = %d" % (n_train, n_classes) + ) + if n_test < n_classes: + raise ValueError( + "The test_size = %d should be greater or " "equal to the number of classes = %d" % (n_test, n_classes) + ) + class_indices = np.split(np.argsort(y_indices, kind="mergesort"), np.cumsum(class_counts)[:-1]) + for _ in range(n_splits): + n_i = approximate_mode(class_counts, n_train, rng) + class_counts_remaining = class_counts - n_i + t_i = approximate_mode(class_counts_remaining, n_test, rng) + + train = [] + test = [] + + for i in range(n_classes): + permutation = rng.permutation(class_counts[i]) + perm_indices_class_i = class_indices[i].take(permutation, mode="clip") + train.extend(perm_indices_class_i[: n_i[i]]) + test.extend(perm_indices_class_i[n_i[i] : n_i[i] + t_i[i]]) + train = rng.permutation(train) + test = rng.permutation(test) + + yield train, test diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/tf_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/tf_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b69f5c85b2c38bb47506a4b2fedb5a69e1d37c00 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/tf_utils.py @@ -0,0 +1,582 @@ +# Copyright 2022 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TF-specific utils import.""" + +import os +import warnings +from functools import partial +from math import ceil +from uuid import uuid4 + +import numpy as np +import pyarrow as pa +from multiprocess import get_context + + +try: + from multiprocess.shared_memory import SharedMemory +except ImportError: + SharedMemory = None # Version checks should prevent this being called on older Python versions + +from .. import config + + +def minimal_tf_collate_fn(features): + if isinstance(features, dict): # case batch_size=None: nothing to collate + return features + elif config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + first = features[0] + batch = {} + for k, v in first.items(): + if isinstance(v, np.ndarray): + batch[k] = np.stack([f[k] for f in features]) + elif isinstance(v, tf.Tensor): + batch[k] = tf.stack([f[k] for f in features]) + else: + batch[k] = np.array([f[k] for f in features]) + return batch + + +def minimal_tf_collate_fn_with_renaming(features): + batch = minimal_tf_collate_fn(features) + if "label" in batch: + batch["labels"] = batch["label"] + del batch["label"] + return batch + + +def is_numeric_pa_type(pa_type): + if pa.types.is_list(pa_type): + return is_numeric_pa_type(pa_type.value_type) + return pa.types.is_integer(pa_type) or pa.types.is_floating(pa_type) or pa.types.is_decimal(pa_type) + + +def is_numeric_feature(feature): + from .. import ClassLabel, Sequence, Value + from ..features.features import _ArrayXD + + if isinstance(feature, Sequence): + return is_numeric_feature(feature.feature) + elif isinstance(feature, list): + return is_numeric_feature(feature[0]) + elif isinstance(feature, _ArrayXD): + return is_numeric_pa_type(feature().storage_dtype) + elif isinstance(feature, Value): + return is_numeric_pa_type(feature()) + elif isinstance(feature, ClassLabel): + return True + else: + return False + + +def np_get_batch( + indices, dataset, cols_to_retain, collate_fn, collate_fn_args, columns_to_np_types, return_dict=False +): + if not isinstance(indices, np.ndarray): + indices = indices.numpy() + + is_batched = True + # Optimization - if we're loading a sequential batch, do it with slicing instead of a list of indices + if isinstance(indices, np.integer): + batch = dataset[indices.item()] + is_batched = False + elif np.all(np.diff(indices) == 1): + batch = dataset[indices[0] : indices[-1] + 1] + elif isinstance(indices, np.ndarray): + batch = dataset[indices] + else: + raise RuntimeError("Unexpected type for indices: {}".format(type(indices))) + + if cols_to_retain is not None: + batch = { + key: value + for key, value in batch.items() + if key in cols_to_retain or key in ("label", "label_ids", "labels") + } + + if is_batched: + actual_size = len(list(batch.values())[0]) # Get the length of one of the arrays, assume all same + # Our collators expect a list of dicts, not a dict of lists/arrays, so we invert + batch = [{key: value[i] for key, value in batch.items()} for i in range(actual_size)] + batch = collate_fn(batch, **collate_fn_args) + + if return_dict: + out_batch = {} + for col, cast_dtype in columns_to_np_types.items(): + # In case the collate_fn returns something strange + array = np.array(batch[col]) + array = array.astype(cast_dtype) + out_batch[col] = array + else: + out_batch = [] + for col, cast_dtype in columns_to_np_types.items(): + # In case the collate_fn returns something strange + array = np.array(batch[col]) + array = array.astype(cast_dtype) + out_batch.append(array) + return out_batch + + +def dataset_to_tf( + dataset, + cols_to_retain, + collate_fn, + collate_fn_args, + columns_to_np_types, + output_signature, + shuffle, + batch_size, + drop_remainder, +): + """Create a tf.data.Dataset from the underlying Dataset. This is a single-process method - the multiprocess + equivalent is multiprocess_dataset_to_tf. + + Args: + dataset (`Dataset`): Dataset to wrap with tf.data.Dataset. + cols_to_retain (`List[str]`): Dataset column(s) to load in the + tf.data.Dataset. It is acceptable to include column names that are created by the `collate_fn` and + that do not exist in the original dataset. + collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate + lists of samples into a batch. + collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the + `collate_fn`. Can be empty. + columns_to_np_types (`Dict[str, np.dtype]`): A `dict` mapping column names to numpy dtypes. + output_signature (`Dict[str, tf.TensorSpec]`): A `dict` mapping column names to + `tf.TensorSpec` objects. + shuffle(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for + validation/evaluation. + batch_size (`int`, default `None`): Size of batches to load from the dataset. Defaults to `None`, which implies that + the dataset won't be batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`. + drop_remainder(`bool`, default `None`): Drop the last incomplete batch when loading. If not provided, + defaults to the same setting as shuffle. + + Returns: + `tf.data.Dataset` + """ + if config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + # TODO Matt: When our minimum Python version is 3.8 or higher, we can delete all of this and move everything + # to the NumPy multiprocessing path. + if hasattr(tf, "random_index_shuffle"): + random_index_shuffle = tf.random_index_shuffle + elif hasattr(tf.random.experimental, "index_shuffle"): + random_index_shuffle = tf.random.experimental.index_shuffle + else: + if len(dataset) > 10_000_000: + warnings.warn( + "to_tf_dataset() can be memory-inefficient on versions of TensorFlow older than 2.9. " + "If you are iterating over a dataset with a very large number of samples, consider " + "upgrading to TF >= 2.9." + ) + random_index_shuffle = None + + getter_fn = partial( + np_get_batch, + dataset=dataset, + cols_to_retain=cols_to_retain, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + columns_to_np_types=columns_to_np_types, + return_dict=False, + ) + + # This works because dictionaries always output in the same order + tout = [tf.dtypes.as_dtype(dtype) for dtype in columns_to_np_types.values()] + + @tf.function(input_signature=[tf.TensorSpec(None, tf.int64)]) + def fetch_function(indices): + output = tf.py_function( + getter_fn, + inp=[indices], + Tout=tout, + ) + return {key: output[i] for i, key in enumerate(columns_to_np_types.keys())} + + tf_dataset = tf.data.Dataset.range(len(dataset)) + + if shuffle and random_index_shuffle is not None: + base_seed = tf.fill((3,), value=tf.cast(-1, dtype=tf.int64)) + + def scan_random_index(state, index): + if tf.reduce_all(state == -1): + # This generates a new random seed once per epoch only, + # to ensure that we iterate over each sample exactly once per epoch + state = tf.random.uniform(shape=(3,), maxval=2**62, dtype=tf.int64) + shuffled_index = random_index_shuffle(index=index, seed=state, max_index=len(dataset) - 1) + return state, shuffled_index + + tf_dataset = tf_dataset.scan(base_seed, scan_random_index) + elif shuffle: + tf_dataset = tf_dataset.shuffle(tf_dataset.cardinality()) + + if batch_size is not None: + tf_dataset = tf_dataset.batch(batch_size, drop_remainder=drop_remainder) + + tf_dataset = tf_dataset.map(fetch_function) + + if batch_size is not None: + + def ensure_shapes(input_dict): + return {key: tf.ensure_shape(val, output_signature[key].shape) for key, val in input_dict.items()} + + else: + # Ensure shape but remove batch dimension of output_signature[key].shape + def ensure_shapes(input_dict): + return {key: tf.ensure_shape(val, output_signature[key].shape[1:]) for key, val in input_dict.items()} + + return tf_dataset.map(ensure_shapes) + + +class SharedMemoryContext: + # This is a context manager for creating shared memory that ensures cleanup happens even if a process is interrupted + # The process that creates shared memory is always the one responsible for unlinking it in the end + def __init__(self): + self.created_shms = [] + self.opened_shms = [] + + def get_shm(self, name, size, create): + shm = SharedMemory(size=int(size), name=name, create=create) + if create: + # We only unlink the ones we created in this context + self.created_shms.append(shm) + else: + # If we didn't create it, we only close it when done, we don't unlink it + self.opened_shms.append(shm) + return shm + + def get_array(self, name, shape, dtype, create): + shm = self.get_shm(name=name, size=np.prod(shape) * np.dtype(dtype).itemsize, create=create) + return np.ndarray(shape, dtype=dtype, buffer=shm.buf) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + for shm in self.created_shms: + shm.close() + shm.unlink() + for shm in self.opened_shms: + shm.close() + + +class NumpyMultiprocessingGenerator: + def __init__( + self, + dataset, + cols_to_retain, + collate_fn, + collate_fn_args, + columns_to_np_types, + output_signature, + shuffle, + batch_size, + drop_remainder, + num_workers, + ): + self.dataset = dataset + self.cols_to_retain = cols_to_retain + self.collate_fn = collate_fn + self.collate_fn_args = collate_fn_args + self.string_columns = [col for col, dtype in columns_to_np_types.items() if dtype in (np.unicode_, np.str_)] + # Strings will be converted to arrays of single unicode chars, so that we can have a constant itemsize + self.columns_to_np_types = { + col: dtype if col not in self.string_columns else np.dtype("U1") + for col, dtype in columns_to_np_types.items() + } + self.output_signature = output_signature + self.shuffle = shuffle + self.batch_size = batch_size + self.drop_remainder = drop_remainder + self.num_workers = num_workers + # Because strings are converted to characters, we need to add one extra dimension to the shape + self.columns_to_ranks = { + col: int(spec.shape.rank) if col not in self.string_columns else int(spec.shape.rank) + 1 + for col, spec in output_signature.items() + } + + def __iter__(self): + # Make sure we only spawn workers if they have work to do + num_workers = min(self.num_workers, int(ceil(len(self.dataset) / self.batch_size))) + # Do the shuffling in iter so that it's done at the start of each epoch + per_worker_batches, final_batch, final_batch_worker = self.distribute_batches( + self.dataset, self.batch_size, self.drop_remainder, num_workers, self.shuffle + ) + ctx = get_context("spawn") + names = [] + shape_arrays = [] + workers = [] + array_ready_events = [ctx.Event() for _ in range(num_workers)] + array_loaded_events = [ctx.Event() for _ in range(num_workers)] + + base_args = { + "dataset": self.dataset, + "cols_to_retain": self.cols_to_retain, + "collate_fn": self.collate_fn, + "collate_fn_args": self.collate_fn_args, + "columns_to_np_types": self.columns_to_np_types, + "columns_to_ranks": self.columns_to_ranks, + "string_columns": self.string_columns, + } + with SharedMemoryContext() as shm_ctx: + for i in range(num_workers): + worker_random_id = str(uuid4()) + worker_name = f"dw_{i}_{worker_random_id}"[:10] + names.append(worker_name) + + worker_shape_arrays = { + col: shm_ctx.get_array(f"{worker_name}_{col}_shape", shape=(rank,), dtype=np.int64, create=True) + for col, rank in self.columns_to_ranks.items() + } + shape_arrays.append(worker_shape_arrays) + + worker_indices = per_worker_batches[i] + if i == final_batch_worker and final_batch is not None: + final_batch_arg = final_batch + else: + final_batch_arg = None + worker_kwargs = { + "worker_name": worker_name, + "indices": worker_indices, + "extra_batch": final_batch_arg, + "array_ready_event": array_ready_events[i], + "array_loaded_event": array_loaded_events[i], + **base_args, + } + worker = ctx.Process(target=self.worker_loop, kwargs=worker_kwargs, daemon=True) + worker.start() + workers.append(worker) + + end_signal_received = False + while not end_signal_received: + for i in range(num_workers): + if not array_ready_events[i].wait(timeout=60): + raise TimeoutError("Data loading worker timed out!") + array_ready_events[i].clear() + array_shapes = shape_arrays[i] + if any(np.any(shape < 0) for shape in array_shapes.values()): + # Child processes send negative array shapes to indicate + # that no more data is going to be sent + end_signal_received = True + break + # Matt: Because array shapes are variable we recreate the shared memory each iteration. + # I suspect repeatedly opening lots of shared memory is the bottleneck for the parent process. + # A future optimization, at the cost of some code complexity, could be to reuse shared memory + # between iterations, but this would require knowing in advance the maximum size, or having + # a system to only create a new memory block when a new maximum size is seen. + # Another potential optimization would be to figure out which memory copies are necessary, + # or whether we can yield objects straight out of shared memory. + with SharedMemoryContext() as batch_shm_ctx: + # This memory context only lasts long enough to copy everything out of the batch + arrays = { + col: batch_shm_ctx.get_array( + f"{names[i]}_{col}", + shape=shape, + dtype=self.columns_to_np_types[col], + create=False, + ) + for col, shape in array_shapes.items() + } + # Copy everything out of shm because the memory + # will be unlinked by the child process at some point + arrays = {col: np.copy(arr) for col, arr in arrays.items()} + # Now we convert any unicode char arrays to strings + for string_col in self.string_columns: + arrays[string_col] = ( + arrays[string_col].view(f"U{arrays[string_col].shape[-1]}").squeeze(-1) + ) + yield arrays + array_loaded_events[i].set() + # Now we just do some cleanup + # Shared memory is cleaned up by the context manager, so we just make sure workers finish + for worker in workers: + worker.join() + + def __call__(self): + return self + + @staticmethod + def worker_loop( + dataset, + cols_to_retain, + collate_fn, + collate_fn_args, + columns_to_np_types, + columns_to_ranks, + string_columns, + indices, + extra_batch, + worker_name, + array_ready_event, + array_loaded_event, + ): + os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" + + if config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + tf.config.set_visible_devices([], "GPU") # Make sure workers don't try to allocate GPU memory + + def send_batch_to_parent(indices): + batch = np_get_batch( + indices=indices, + dataset=dataset, + cols_to_retain=cols_to_retain, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + columns_to_np_types=columns_to_np_types, + return_dict=True, + ) + + # Now begins the fun part where we start shovelling shared memory at the parent process + out_arrays = {} + with SharedMemoryContext() as batch_shm_ctx: + # The batch shared memory context exists only as long as it takes for the parent process + # to read everything, after which it cleans everything up again + for col, cast_dtype in columns_to_np_types.items(): + # Everything has to be np.array for this to work, even if the collate_fn is giving us tf.Tensor + array = batch[col] + if col in string_columns: + # We can't send unicode arrays over shared memory, so we convert to single chars ("U1") + # which have a fixed width of 4 bytes. The parent process will convert these back to strings. + array = array.view("U1").reshape(array.shape + (-1,)) + shape_arrays[col][:] = array.shape + out_arrays[col] = batch_shm_ctx.get_array( + f"{worker_name}_{col}", shape=array.shape, dtype=cast_dtype, create=True + ) + out_arrays[col][:] = array + + array_ready_event.set() + array_loaded_event.wait() + array_loaded_event.clear() + + with SharedMemoryContext() as shm_ctx: + shape_arrays = { + col: shm_ctx.get_array(f"{worker_name}_{col}_shape", shape=(rank,), dtype=np.int64, create=False) + for col, rank in columns_to_ranks.items() + } + + for batch in indices: + send_batch_to_parent(batch) + if extra_batch is not None: + send_batch_to_parent(extra_batch) + # Now we send a batsignal to the parent process that we're done + for col, array in shape_arrays.items(): + array[:] = -1 + array_ready_event.set() + + @staticmethod + def distribute_batches(dataset, batch_size, drop_remainder, num_workers, shuffle): + indices = np.arange(len(dataset)) + if shuffle: + np.random.shuffle(indices) + num_samples = len(indices) + # We distribute the batches so that reading from the workers in round-robin order yields the exact + # order specified in indices. This is only important when shuffle is False, but we do it regardless. + incomplete_batch_cutoff = num_samples - (num_samples % batch_size) + indices, last_incomplete_batch = np.split(indices, [incomplete_batch_cutoff]) + if drop_remainder or len(last_incomplete_batch) == 0: + last_incomplete_batch = None + + indices = indices.reshape(-1, batch_size) + num_batches = len(indices) + final_batches_cutoff = num_batches - (num_batches % num_workers) + indices, final_batches = np.split(indices, [final_batches_cutoff]) + indices = indices.reshape(-1, num_workers, batch_size) + + per_worker_indices = np.split(indices, indices.shape[1], axis=1) + per_worker_indices = [np.squeeze(worker_indices, 1) for worker_indices in per_worker_indices] + # Distribute the final batches to the first workers + for i in range(len(final_batches)): + # len(final_batches) can be zero, and is always less than num_workers + per_worker_indices[i] = np.concatenate([per_worker_indices[i], final_batches[i].reshape(1, -1)], axis=0) + # Add the last incomplete batch to the next worker, which might be the first worker + if last_incomplete_batch is not None: + incomplete_batch_worker_idx = len(final_batches) + else: + incomplete_batch_worker_idx = None + return per_worker_indices, last_incomplete_batch, incomplete_batch_worker_idx + + +def multiprocess_dataset_to_tf( + dataset, + cols_to_retain, + collate_fn, + collate_fn_args, + columns_to_np_types, + output_signature, + shuffle, + batch_size, + drop_remainder, + num_workers, +): + """Create a tf.data.Dataset from the underlying Dataset. This is a multi-process method - the single-process + equivalent is dataset_to_tf. + + Args: + dataset (`Dataset`): Dataset to wrap with tf.data.Dataset. + cols_to_retain (`List[str]`): Dataset column(s) to load in the + tf.data.Dataset. It is acceptable to include column names that are created by the `collate_fn` and + that do not exist in the original dataset. + collate_fn(`Callable`): A function or callable object (such as a `DataCollator`) that will collate + lists of samples into a batch. + collate_fn_args (`Dict`): A `dict` of keyword arguments to be passed to the + `collate_fn`. Can be empty. + columns_to_np_types (`Dict[str, np.dtype]`): A `dict` mapping column names to numpy dtypes. + output_signature (`Dict[str, tf.TensorSpec]`): A `dict` mapping column names to + `tf.TensorSpec` objects. + shuffle(`bool`): Shuffle the dataset order when loading. Recommended True for training, False for + validation/evaluation. + batch_size (`int`, default `None`): Size of batches to load from the dataset. Defaults to `None`, which implies that + the dataset won't be batched, but the returned dataset can be batched later with `tf_dataset.batch(batch_size)`. + drop_remainder(`bool`, default `None`): Drop the last incomplete batch when loading. If not provided, + defaults to the same setting as shuffle. + num_workers (`int`): Number of workers to use for loading the dataset. Should be >= 1. + + Returns: + `tf.data.Dataset` + """ + if config.TF_AVAILABLE: + import tensorflow as tf + else: + raise ImportError("Called a Tensorflow-specific function but Tensorflow is not installed.") + + data_generator = NumpyMultiprocessingGenerator( + dataset=dataset, + cols_to_retain=cols_to_retain, + collate_fn=collate_fn, + collate_fn_args=collate_fn_args, + columns_to_np_types=columns_to_np_types, + output_signature=output_signature, + shuffle=shuffle, + batch_size=batch_size, + drop_remainder=drop_remainder, + num_workers=num_workers, + ) + + tf_dataset = tf.data.Dataset.from_generator(data_generator, output_signature=output_signature) + if drop_remainder: + dataset_length = int(len(dataset) // batch_size) + else: + dataset_length = int(ceil(len(dataset) / batch_size)) + return tf_dataset.apply(tf.data.experimental.assert_cardinality(dataset_length)) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/tqdm.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/tqdm.py new file mode 100644 index 0000000000000000000000000000000000000000..e28a8ff7ccfda7d4fd9a6195636d181f285ceb65 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/tqdm.py @@ -0,0 +1,131 @@ +"""Utility helpers to handle progress bars in `datasets`. + +Example: + 1. Use `datasets.utils.tqdm` as you would use `tqdm.tqdm` or `tqdm.auto.tqdm`. + 2. To disable progress bars, either use `disable_progress_bars()` helper or set the + environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS` to 1. + 3. To re-enable progress bars, use `enable_progress_bars()`. + 4. To check whether progress bars are disabled, use `are_progress_bars_disabled()`. + +NOTE: Environment variable `HF_DATASETS_DISABLE_PROGRESS_BARS` has the priority. + +Example: + ```py + from datasets.utils import ( + are_progress_bars_disabled, + disable_progress_bars, + enable_progress_bars, + tqdm, + ) + + # Disable progress bars globally + disable_progress_bars() + + # Use as normal `tqdm` + for _ in tqdm(range(5)): + do_something() + + # Still not showing progress bars, as `disable=False` is overwritten to `True`. + for _ in tqdm(range(5), disable=False): + do_something() + + are_progress_bars_disabled() # True + + # Re-enable progress bars globally + enable_progress_bars() + + # Progress bar will be shown ! + for _ in tqdm(range(5)): + do_something() + ``` +""" + +import warnings + +from tqdm.auto import tqdm as old_tqdm + +from ..config import HF_DATASETS_DISABLE_PROGRESS_BARS + + +# `HF_DATASETS_DISABLE_PROGRESS_BARS` is `Optional[bool]` while `_hf_datasets_progress_bars_disabled` +# is a `bool`. If `HF_DATASETS_DISABLE_PROGRESS_BARS` is set to True or False, it has priority. +# If `HF_DATASETS_DISABLE_PROGRESS_BARS` is None, it means the user have not set the +# environment variable and is free to enable/disable progress bars programmatically. +# TL;DR: env variable has priority over code. +# +# By default, progress bars are enabled. +_hf_datasets_progress_bars_disabled: bool = HF_DATASETS_DISABLE_PROGRESS_BARS or False + + +def disable_progress_bars() -> None: + """ + Disable globally progress bars used in `datasets` except if `HF_DATASETS_DISABLE_PROGRESS_BAR` environment + variable has been set. + + Use [`~utils.enable_progress_bars`] to re-enable them. + """ + if HF_DATASETS_DISABLE_PROGRESS_BARS is False: + warnings.warn( + "Cannot disable progress bars: environment variable `HF_DATASETS_DISABLE_PROGRESS_BAR=0` is set and has" + " priority." + ) + return + global _hf_datasets_progress_bars_disabled + _hf_datasets_progress_bars_disabled = True + + +def enable_progress_bars() -> None: + """ + Enable globally progress bars used in `datasets` except if `HF_DATASETS_DISABLE_PROGRESS_BAR` environment + variable has been set. + + Use [`~utils.disable_progress_bars`] to disable them. + """ + if HF_DATASETS_DISABLE_PROGRESS_BARS is True: + warnings.warn( + "Cannot enable progress bars: environment variable `HF_DATASETS_DISABLE_PROGRESS_BAR=1` is set and has" + " priority." + ) + return + global _hf_datasets_progress_bars_disabled + _hf_datasets_progress_bars_disabled = False + + +def are_progress_bars_disabled() -> bool: + """Return whether progress bars are globally disabled or not. + + Progress bars used in `datasets` can be enable or disabled globally using [`~utils.enable_progress_bars`] + and [`~utils.disable_progress_bars`] or by setting `HF_DATASETS_DISABLE_PROGRESS_BAR` as environment variable. + """ + global _hf_datasets_progress_bars_disabled + return _hf_datasets_progress_bars_disabled + + +class tqdm(old_tqdm): + """ + Class to override `disable` argument in case progress bars are globally disabled. + + Taken from https://github.com/tqdm/tqdm/issues/619#issuecomment-619639324. + """ + + def __init__(self, *args, **kwargs): + if are_progress_bars_disabled(): + kwargs["disable"] = True + super().__init__(*args, **kwargs) + + def __delattr__(self, attr: str) -> None: + """Fix for https://github.com/huggingface/datasets/issues/6066""" + try: + super().__delattr__(attr) + except AttributeError: + if attr != "_lock": + raise + + +# backward compatibility +enable_progress_bar = enable_progress_bars +disable_progress_bar = disable_progress_bars + + +def is_progress_bar_enabled(): + return not are_progress_bars_disabled() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/track.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/track.py new file mode 100644 index 0000000000000000000000000000000000000000..11a3787c7d8595cc7160994973f28db1f709b3b2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/track.py @@ -0,0 +1,49 @@ +from collections.abc import Iterator +from typing import Iterable + + +class tracked_str(str): + origins = {} + + def set_origin(self, origin: str): + if super().__repr__() not in self.origins: + self.origins[super().__repr__()] = origin + + def get_origin(self): + return self.origins.get(super().__repr__(), str(self)) + + def __repr__(self) -> str: + if super().__repr__() not in self.origins or self.origins[super().__repr__()] == self: + return super().__repr__() + else: + return f"{str(self)} (origin={self.origins[super().__repr__()]})" + + +class tracked_list(list): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.last_item = None + + def __iter__(self) -> Iterator: + for x in super().__iter__(): + self.last_item = x + yield x + self.last_item = None + + def __repr__(self) -> str: + if self.last_item is None: + return super().__repr__() + else: + return f"{self.__class__.__name__}(current={self.last_item})" + + +class TrackedIterable(Iterable): + def __init__(self) -> None: + super().__init__() + self.last_item = None + + def __repr__(self) -> str: + if self.last_item is None: + super().__repr__() + else: + return f"{self.__class__.__name__}(current={self.last_item})" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/typing.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/typing.py new file mode 100644 index 0000000000000000000000000000000000000000..7ad087fc98d2a6de2d3e493120135fc9ea49e605 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/typing.py @@ -0,0 +1,9 @@ +import os +from typing import Dict, List, Tuple, TypeVar, Union + + +T = TypeVar("T") + +ListLike = Union[List[T], Tuple[T, ...]] +NestedDataStructureLike = Union[T, List[T], Dict[str, T]] +PathLike = Union[str, bytes, os.PathLike] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/version.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/version.py new file mode 100644 index 0000000000000000000000000000000000000000..75cf4c39d5f9b916ade8f8d88a0f0ca9e5769217 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/datasets/utils/version.py @@ -0,0 +1,106 @@ +# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Lint as: python3 +"""Version utils.""" + +import dataclasses +import re +from dataclasses import dataclass +from functools import total_ordering +from typing import Optional, Union + + +_VERSION_REG = re.compile(r"^(?P\d+)" r"\.(?P\d+)" r"\.(?P\d+)$") + + +@total_ordering +@dataclass +class Version: + """Dataset version `MAJOR.MINOR.PATCH`. + + Args: + version_str (`str`): + The dataset version. + description (`str`): + A description of what is new in this version. + major (`str`): + minor (`str`): + patch (`str`): + + Example: + + ```py + >>> VERSION = datasets.Version("1.0.0") + ``` + """ + + version_str: str + description: Optional[str] = None + major: Optional[Union[str, int]] = None + minor: Optional[Union[str, int]] = None + patch: Optional[Union[str, int]] = None + + def __post_init__(self): + self.major, self.minor, self.patch = _str_to_version_tuple(self.version_str) + + def __repr__(self): + return f"{self.tuple[0]}.{self.tuple[1]}.{self.tuple[2]}" + + @property + def tuple(self): + return self.major, self.minor, self.patch + + def _validate_operand(self, other): + if isinstance(other, str): + return Version(other) + elif isinstance(other, Version): + return other + raise TypeError(f"{other} (type {type(other)}) cannot be compared to version.") + + def __eq__(self, other): + try: + other = self._validate_operand(other) + except (TypeError, ValueError): + return False + else: + return self.tuple == other.tuple + + def __lt__(self, other): + other = self._validate_operand(other) + return self.tuple < other.tuple + + def __hash__(self): + return hash(_version_tuple_to_str(self.tuple)) + + @classmethod + def from_dict(cls, dic): + field_names = {f.name for f in dataclasses.fields(cls)} + return cls(**{k: v for k, v in dic.items() if k in field_names}) + + def _to_yaml_string(self) -> str: + return self.version_str + + +def _str_to_version_tuple(version_str): + """Return the tuple (major, minor, patch) version extracted from the str.""" + res = _VERSION_REG.match(version_str) + if not res: + raise ValueError(f"Invalid version '{version_str}'. Format should be x.y.z with {{x,y,z}} being digits.") + return tuple(int(v) for v in [res.group("major"), res.group("minor"), res.group("patch")]) + + +def _version_tuple_to_str(version_tuple): + """Return the str version from the version tuple (major, minor, patch).""" + return ".".join(str(v) for v in version_tuple) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3158cce702be1cc8f349a6d70bde3efb7105a5d8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/__version__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/__version__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b28126547391dfb755f84bf37e5664a3c10e62c4 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/__version__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_api.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_api.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5a78d43757b9d508b1f39c582acfe1b6325edf53 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_api.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_auth.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_auth.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2fa8e115a6854361fb38c582e0cfa5b8e75741af Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_auth.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_client.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_client.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..274010428ef03fc0d25338c5c7cb9946a23de9e2 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_client.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_config.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe747be12444fe1640b07b7e7a283f6f209b8b0b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_config.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_content.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_content.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..192074ff7b98c3c370d728404c0ae61a1f482dfc Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_content.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_decoders.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_decoders.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d59ef6a1f8e221f361c4c417d5e31f5e1682456d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_decoders.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_exceptions.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2644b48ab335a92fe34bff33d73632ebc8b97abd Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_exceptions.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_main.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_main.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0d1a6359704d26dcd8123a3a97fedcf4d7b39a8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_main.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_models.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_models.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..818e03bcefe35f74cb1b5efb25be37c51dd5e395 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_models.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_multipart.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_multipart.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a1827f0601f3ec6d923624f4c928663133513859 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_multipart.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_status_codes.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_status_codes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..af6f7af7aa1ded9303fbf679811327f7cba6bfb9 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_status_codes.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_types.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_types.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0aeecffcb6a4145d535504f90fd985a9c4d23a0 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_types.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_urlparse.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_urlparse.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a31ee42bd0079c4dcf694369b700e9813d360d38 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_urlparse.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_urls.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_urls.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e8524f5bc99be86b2ef2d465730bee4d02d50c1f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_urls.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3bc2168f10a28bedcea040451da7ab6cc936e650 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/__pycache__/_utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7a321053b29bcd48698cf2bd74a1d19c8556aefb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__init__.py @@ -0,0 +1,15 @@ +from .asgi import * +from .base import * +from .default import * +from .mock import * +from .wsgi import * + +__all__ = [ + "ASGITransport", + "AsyncBaseTransport", + "BaseTransport", + "AsyncHTTPTransport", + "HTTPTransport", + "MockTransport", + "WSGITransport", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3bd10db79c4c13586fdfd9c2f3b2260963237372 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/asgi.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/asgi.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d95d1556b2ca34d77a7f1f7734b7317e53cbd54b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/asgi.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/base.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/base.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dfc1125d0ef8fb2b3c736a29ab13d48099164495 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/base.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/default.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/default.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6734bcbabf6e6ac63504db88a6a8dc53e52fad99 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/default.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/mock.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/mock.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d232234a0db184a7d507b167fea879240227c730 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/mock.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/wsgi.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/wsgi.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3d9a032dd62510ad9b805cecbb7f5080da7fad15 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/__pycache__/wsgi.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/asgi.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/asgi.py new file mode 100644 index 0000000000000000000000000000000000000000..2bc4efae0e1b14620f75f712eb15ecf500d14eef --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/asgi.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import typing + +from .._models import Request, Response +from .._types import AsyncByteStream +from .base import AsyncBaseTransport + +if typing.TYPE_CHECKING: # pragma: no cover + import asyncio + + import trio + + Event = typing.Union[asyncio.Event, trio.Event] + + +_Message = typing.MutableMapping[str, typing.Any] +_Receive = typing.Callable[[], typing.Awaitable[_Message]] +_Send = typing.Callable[ + [typing.MutableMapping[str, typing.Any]], typing.Awaitable[None] +] +_ASGIApp = typing.Callable[ + [typing.MutableMapping[str, typing.Any], _Receive, _Send], typing.Awaitable[None] +] + +__all__ = ["ASGITransport"] + + +def is_running_trio() -> bool: + try: + # sniffio is a dependency of trio. + + # See https://github.com/python-trio/trio/issues/2802 + import sniffio + + if sniffio.current_async_library() == "trio": + return True + except ImportError: # pragma: nocover + pass + + return False + + +def create_event() -> Event: + if is_running_trio(): + import trio + + return trio.Event() + + import asyncio + + return asyncio.Event() + + +class ASGIResponseStream(AsyncByteStream): + def __init__(self, body: list[bytes]) -> None: + self._body = body + + async def __aiter__(self) -> typing.AsyncIterator[bytes]: + yield b"".join(self._body) + + +class ASGITransport(AsyncBaseTransport): + """ + A custom AsyncTransport that handles sending requests directly to an ASGI app. + + ```python + transport = httpx.ASGITransport( + app=app, + root_path="/submount", + client=("1.2.3.4", 123) + ) + client = httpx.AsyncClient(transport=transport) + ``` + + Arguments: + + * `app` - The ASGI application. + * `raise_app_exceptions` - Boolean indicating if exceptions in the application + should be raised. Default to `True`. Can be set to `False` for use cases + such as testing the content of a client 500 response. + * `root_path` - The root path on which the ASGI application should be mounted. + * `client` - A two-tuple indicating the client IP and port of incoming requests. + ``` + """ + + def __init__( + self, + app: _ASGIApp, + raise_app_exceptions: bool = True, + root_path: str = "", + client: tuple[str, int] = ("127.0.0.1", 123), + ) -> None: + self.app = app + self.raise_app_exceptions = raise_app_exceptions + self.root_path = root_path + self.client = client + + async def handle_async_request( + self, + request: Request, + ) -> Response: + assert isinstance(request.stream, AsyncByteStream) + + # ASGI scope. + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": request.method, + "headers": [(k.lower(), v) for (k, v) in request.headers.raw], + "scheme": request.url.scheme, + "path": request.url.path, + "raw_path": request.url.raw_path.split(b"?")[0], + "query_string": request.url.query, + "server": (request.url.host, request.url.port), + "client": self.client, + "root_path": self.root_path, + } + + # Request. + request_body_chunks = request.stream.__aiter__() + request_complete = False + + # Response. + status_code = None + response_headers = None + body_parts = [] + response_started = False + response_complete = create_event() + + # ASGI callables. + + async def receive() -> dict[str, typing.Any]: + nonlocal request_complete + + if request_complete: + await response_complete.wait() + return {"type": "http.disconnect"} + + try: + body = await request_body_chunks.__anext__() + except StopAsyncIteration: + request_complete = True + return {"type": "http.request", "body": b"", "more_body": False} + return {"type": "http.request", "body": body, "more_body": True} + + async def send(message: typing.MutableMapping[str, typing.Any]) -> None: + nonlocal status_code, response_headers, response_started + + if message["type"] == "http.response.start": + assert not response_started + + status_code = message["status"] + response_headers = message.get("headers", []) + response_started = True + + elif message["type"] == "http.response.body": + assert not response_complete.is_set() + body = message.get("body", b"") + more_body = message.get("more_body", False) + + if body and request.method != "HEAD": + body_parts.append(body) + + if not more_body: + response_complete.set() + + try: + await self.app(scope, receive, send) + except Exception: # noqa: PIE-786 + if self.raise_app_exceptions: + raise + + response_complete.set() + if status_code is None: + status_code = 500 + if response_headers is None: + response_headers = {} + + assert response_complete.is_set() + assert status_code is not None + assert response_headers is not None + + stream = ASGIResponseStream(body_parts) + + return Response(status_code, headers=response_headers, stream=stream) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/base.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/base.py new file mode 100644 index 0000000000000000000000000000000000000000..66fd99d702480b555c06694fe14715ea6df3dfc3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/base.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import typing +from types import TracebackType + +from .._models import Request, Response + +T = typing.TypeVar("T", bound="BaseTransport") +A = typing.TypeVar("A", bound="AsyncBaseTransport") + +__all__ = ["AsyncBaseTransport", "BaseTransport"] + + +class BaseTransport: + def __enter__(self: T) -> T: + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + self.close() + + def handle_request(self, request: Request) -> Response: + """ + Send a single HTTP request and return a response. + + Developers shouldn't typically ever need to call into this API directly, + since the Client class provides all the higher level user-facing API + niceties. + + In order to properly release any network resources, the response + stream should *either* be consumed immediately, with a call to + `response.stream.read()`, or else the `handle_request` call should + be followed with a try/finally block to ensuring the stream is + always closed. + + Example usage: + + with httpx.HTTPTransport() as transport: + req = httpx.Request( + method=b"GET", + url=(b"https", b"www.example.com", 443, b"/"), + headers=[(b"Host", b"www.example.com")], + ) + resp = transport.handle_request(req) + body = resp.stream.read() + print(resp.status_code, resp.headers, body) + + + Takes a `Request` instance as the only argument. + + Returns a `Response` instance. + """ + raise NotImplementedError( + "The 'handle_request' method must be implemented." + ) # pragma: no cover + + def close(self) -> None: + pass + + +class AsyncBaseTransport: + async def __aenter__(self: A) -> A: + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + await self.aclose() + + async def handle_async_request( + self, + request: Request, + ) -> Response: + raise NotImplementedError( + "The 'handle_async_request' method must be implemented." + ) # pragma: no cover + + async def aclose(self) -> None: + pass diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/default.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/default.py new file mode 100644 index 0000000000000000000000000000000000000000..d5aa05ff234fd3fbf4fee88c4a7d3e3c151a538f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/default.py @@ -0,0 +1,406 @@ +""" +Custom transports, with nicely configured defaults. + +The following additional keyword arguments are currently supported by httpcore... + +* uds: str +* local_address: str +* retries: int + +Example usages... + +# Disable HTTP/2 on a single specific domain. +mounts = { + "all://": httpx.HTTPTransport(http2=True), + "all://*example.org": httpx.HTTPTransport() +} + +# Using advanced httpcore configuration, with connection retries. +transport = httpx.HTTPTransport(retries=1) +client = httpx.Client(transport=transport) + +# Using advanced httpcore configuration, with unix domain sockets. +transport = httpx.HTTPTransport(uds="socket.uds") +client = httpx.Client(transport=transport) +""" + +from __future__ import annotations + +import contextlib +import typing +from types import TracebackType + +if typing.TYPE_CHECKING: + import ssl # pragma: no cover + + import httpx # pragma: no cover + +from .._config import DEFAULT_LIMITS, Limits, Proxy, create_ssl_context +from .._exceptions import ( + ConnectError, + ConnectTimeout, + LocalProtocolError, + NetworkError, + PoolTimeout, + ProtocolError, + ProxyError, + ReadError, + ReadTimeout, + RemoteProtocolError, + TimeoutException, + UnsupportedProtocol, + WriteError, + WriteTimeout, +) +from .._models import Request, Response +from .._types import AsyncByteStream, CertTypes, ProxyTypes, SyncByteStream +from .._urls import URL +from .base import AsyncBaseTransport, BaseTransport + +T = typing.TypeVar("T", bound="HTTPTransport") +A = typing.TypeVar("A", bound="AsyncHTTPTransport") + +SOCKET_OPTION = typing.Union[ + typing.Tuple[int, int, int], + typing.Tuple[int, int, typing.Union[bytes, bytearray]], + typing.Tuple[int, int, None, int], +] + +__all__ = ["AsyncHTTPTransport", "HTTPTransport"] + +HTTPCORE_EXC_MAP: dict[type[Exception], type[httpx.HTTPError]] = {} + + +def _load_httpcore_exceptions() -> dict[type[Exception], type[httpx.HTTPError]]: + import httpcore + + return { + httpcore.TimeoutException: TimeoutException, + httpcore.ConnectTimeout: ConnectTimeout, + httpcore.ReadTimeout: ReadTimeout, + httpcore.WriteTimeout: WriteTimeout, + httpcore.PoolTimeout: PoolTimeout, + httpcore.NetworkError: NetworkError, + httpcore.ConnectError: ConnectError, + httpcore.ReadError: ReadError, + httpcore.WriteError: WriteError, + httpcore.ProxyError: ProxyError, + httpcore.UnsupportedProtocol: UnsupportedProtocol, + httpcore.ProtocolError: ProtocolError, + httpcore.LocalProtocolError: LocalProtocolError, + httpcore.RemoteProtocolError: RemoteProtocolError, + } + + +@contextlib.contextmanager +def map_httpcore_exceptions() -> typing.Iterator[None]: + global HTTPCORE_EXC_MAP + if len(HTTPCORE_EXC_MAP) == 0: + HTTPCORE_EXC_MAP = _load_httpcore_exceptions() + try: + yield + except Exception as exc: + mapped_exc = None + + for from_exc, to_exc in HTTPCORE_EXC_MAP.items(): + if not isinstance(exc, from_exc): + continue + # We want to map to the most specific exception we can find. + # Eg if `exc` is an `httpcore.ReadTimeout`, we want to map to + # `httpx.ReadTimeout`, not just `httpx.TimeoutException`. + if mapped_exc is None or issubclass(to_exc, mapped_exc): + mapped_exc = to_exc + + if mapped_exc is None: # pragma: no cover + raise + + message = str(exc) + raise mapped_exc(message) from exc + + +class ResponseStream(SyncByteStream): + def __init__(self, httpcore_stream: typing.Iterable[bytes]) -> None: + self._httpcore_stream = httpcore_stream + + def __iter__(self) -> typing.Iterator[bytes]: + with map_httpcore_exceptions(): + for part in self._httpcore_stream: + yield part + + def close(self) -> None: + if hasattr(self._httpcore_stream, "close"): + self._httpcore_stream.close() + + +class HTTPTransport(BaseTransport): + def __init__( + self, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + proxy: ProxyTypes | None = None, + uds: str | None = None, + local_address: str | None = None, + retries: int = 0, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + import httpcore + + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + if proxy is None: + self._pool = httpcore.ConnectionPool( + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + uds=uds, + local_address=local_address, + retries=retries, + socket_options=socket_options, + ) + elif proxy.url.scheme in ("http", "https"): + self._pool = httpcore.HTTPProxy( + proxy_url=httpcore.URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + proxy_headers=proxy.headers.raw, + ssl_context=ssl_context, + proxy_ssl_context=proxy.ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + socket_options=socket_options, + ) + elif proxy.url.scheme in ("socks5", "socks5h"): + try: + import socksio # noqa + except ImportError: # pragma: no cover + raise ImportError( + "Using SOCKS proxy, but the 'socksio' package is not installed. " + "Make sure to install httpx using `pip install httpx[socks]`." + ) from None + + self._pool = httpcore.SOCKSProxy( + proxy_url=httpcore.URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + ) + else: # pragma: no cover + raise ValueError( + "Proxy protocol must be either 'http', 'https', 'socks5', or 'socks5h'," + f" but got {proxy.url.scheme!r}." + ) + + def __enter__(self: T) -> T: # Use generics for subclass support. + self._pool.__enter__() + return self + + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + with map_httpcore_exceptions(): + self._pool.__exit__(exc_type, exc_value, traceback) + + def handle_request( + self, + request: Request, + ) -> Response: + assert isinstance(request.stream, SyncByteStream) + import httpcore + + req = httpcore.Request( + method=request.method, + url=httpcore.URL( + scheme=request.url.raw_scheme, + host=request.url.raw_host, + port=request.url.port, + target=request.url.raw_path, + ), + headers=request.headers.raw, + content=request.stream, + extensions=request.extensions, + ) + with map_httpcore_exceptions(): + resp = self._pool.handle_request(req) + + assert isinstance(resp.stream, typing.Iterable) + + return Response( + status_code=resp.status, + headers=resp.headers, + stream=ResponseStream(resp.stream), + extensions=resp.extensions, + ) + + def close(self) -> None: + self._pool.close() + + +class AsyncResponseStream(AsyncByteStream): + def __init__(self, httpcore_stream: typing.AsyncIterable[bytes]) -> None: + self._httpcore_stream = httpcore_stream + + async def __aiter__(self) -> typing.AsyncIterator[bytes]: + with map_httpcore_exceptions(): + async for part in self._httpcore_stream: + yield part + + async def aclose(self) -> None: + if hasattr(self._httpcore_stream, "aclose"): + await self._httpcore_stream.aclose() + + +class AsyncHTTPTransport(AsyncBaseTransport): + def __init__( + self, + verify: ssl.SSLContext | str | bool = True, + cert: CertTypes | None = None, + trust_env: bool = True, + http1: bool = True, + http2: bool = False, + limits: Limits = DEFAULT_LIMITS, + proxy: ProxyTypes | None = None, + uds: str | None = None, + local_address: str | None = None, + retries: int = 0, + socket_options: typing.Iterable[SOCKET_OPTION] | None = None, + ) -> None: + import httpcore + + proxy = Proxy(url=proxy) if isinstance(proxy, (str, URL)) else proxy + ssl_context = create_ssl_context(verify=verify, cert=cert, trust_env=trust_env) + + if proxy is None: + self._pool = httpcore.AsyncConnectionPool( + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + uds=uds, + local_address=local_address, + retries=retries, + socket_options=socket_options, + ) + elif proxy.url.scheme in ("http", "https"): + self._pool = httpcore.AsyncHTTPProxy( + proxy_url=httpcore.URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + proxy_headers=proxy.headers.raw, + proxy_ssl_context=proxy.ssl_context, + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + socket_options=socket_options, + ) + elif proxy.url.scheme in ("socks5", "socks5h"): + try: + import socksio # noqa + except ImportError: # pragma: no cover + raise ImportError( + "Using SOCKS proxy, but the 'socksio' package is not installed. " + "Make sure to install httpx using `pip install httpx[socks]`." + ) from None + + self._pool = httpcore.AsyncSOCKSProxy( + proxy_url=httpcore.URL( + scheme=proxy.url.raw_scheme, + host=proxy.url.raw_host, + port=proxy.url.port, + target=proxy.url.raw_path, + ), + proxy_auth=proxy.raw_auth, + ssl_context=ssl_context, + max_connections=limits.max_connections, + max_keepalive_connections=limits.max_keepalive_connections, + keepalive_expiry=limits.keepalive_expiry, + http1=http1, + http2=http2, + ) + else: # pragma: no cover + raise ValueError( + "Proxy protocol must be either 'http', 'https', 'socks5', or 'socks5h'," + " but got {proxy.url.scheme!r}." + ) + + async def __aenter__(self: A) -> A: # Use generics for subclass support. + await self._pool.__aenter__() + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: + with map_httpcore_exceptions(): + await self._pool.__aexit__(exc_type, exc_value, traceback) + + async def handle_async_request( + self, + request: Request, + ) -> Response: + assert isinstance(request.stream, AsyncByteStream) + import httpcore + + req = httpcore.Request( + method=request.method, + url=httpcore.URL( + scheme=request.url.raw_scheme, + host=request.url.raw_host, + port=request.url.port, + target=request.url.raw_path, + ), + headers=request.headers.raw, + content=request.stream, + extensions=request.extensions, + ) + with map_httpcore_exceptions(): + resp = await self._pool.handle_async_request(req) + + assert isinstance(resp.stream, typing.AsyncIterable) + + return Response( + status_code=resp.status, + headers=resp.headers, + stream=AsyncResponseStream(resp.stream), + extensions=resp.extensions, + ) + + async def aclose(self) -> None: + await self._pool.aclose() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/mock.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/mock.py new file mode 100644 index 0000000000000000000000000000000000000000..8c418f59e06cae43abdbb626ec21cafc7e8c6277 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/mock.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import typing + +from .._models import Request, Response +from .base import AsyncBaseTransport, BaseTransport + +SyncHandler = typing.Callable[[Request], Response] +AsyncHandler = typing.Callable[[Request], typing.Coroutine[None, None, Response]] + + +__all__ = ["MockTransport"] + + +class MockTransport(AsyncBaseTransport, BaseTransport): + def __init__(self, handler: SyncHandler | AsyncHandler) -> None: + self.handler = handler + + def handle_request( + self, + request: Request, + ) -> Response: + request.read() + response = self.handler(request) + if not isinstance(response, Response): # pragma: no cover + raise TypeError("Cannot use an async handler in a sync Client") + return response + + async def handle_async_request( + self, + request: Request, + ) -> Response: + await request.aread() + response = self.handler(request) + + # Allow handler to *optionally* be an `async` function. + # If it is, then the `response` variable need to be awaited to actually + # return the result. + + if not isinstance(response, Response): + response = await response + + return response diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/wsgi.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/wsgi.py new file mode 100644 index 0000000000000000000000000000000000000000..8592ffe017a87367cc7578184540096a9682908d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/httpx/_transports/wsgi.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import io +import itertools +import sys +import typing + +from .._models import Request, Response +from .._types import SyncByteStream +from .base import BaseTransport + +if typing.TYPE_CHECKING: + from _typeshed import OptExcInfo # pragma: no cover + from _typeshed.wsgi import WSGIApplication # pragma: no cover + +_T = typing.TypeVar("_T") + + +__all__ = ["WSGITransport"] + + +def _skip_leading_empty_chunks(body: typing.Iterable[_T]) -> typing.Iterable[_T]: + body = iter(body) + for chunk in body: + if chunk: + return itertools.chain([chunk], body) + return [] + + +class WSGIByteStream(SyncByteStream): + def __init__(self, result: typing.Iterable[bytes]) -> None: + self._close = getattr(result, "close", None) + self._result = _skip_leading_empty_chunks(result) + + def __iter__(self) -> typing.Iterator[bytes]: + for part in self._result: + yield part + + def close(self) -> None: + if self._close is not None: + self._close() + + +class WSGITransport(BaseTransport): + """ + A custom transport that handles sending requests directly to an WSGI app. + The simplest way to use this functionality is to use the `app` argument. + + ``` + client = httpx.Client(app=app) + ``` + + Alternatively, you can setup the transport instance explicitly. + This allows you to include any additional configuration arguments specific + to the WSGITransport class: + + ``` + transport = httpx.WSGITransport( + app=app, + script_name="/submount", + remote_addr="1.2.3.4" + ) + client = httpx.Client(transport=transport) + ``` + + Arguments: + + * `app` - The WSGI application. + * `raise_app_exceptions` - Boolean indicating if exceptions in the application + should be raised. Default to `True`. Can be set to `False` for use cases + such as testing the content of a client 500 response. + * `script_name` - The root path on which the WSGI application should be mounted. + * `remote_addr` - A string indicating the client IP of incoming requests. + ``` + """ + + def __init__( + self, + app: WSGIApplication, + raise_app_exceptions: bool = True, + script_name: str = "", + remote_addr: str = "127.0.0.1", + wsgi_errors: typing.TextIO | None = None, + ) -> None: + self.app = app + self.raise_app_exceptions = raise_app_exceptions + self.script_name = script_name + self.remote_addr = remote_addr + self.wsgi_errors = wsgi_errors + + def handle_request(self, request: Request) -> Response: + request.read() + wsgi_input = io.BytesIO(request.content) + + port = request.url.port or {"http": 80, "https": 443}[request.url.scheme] + environ = { + "wsgi.version": (1, 0), + "wsgi.url_scheme": request.url.scheme, + "wsgi.input": wsgi_input, + "wsgi.errors": self.wsgi_errors or sys.stderr, + "wsgi.multithread": True, + "wsgi.multiprocess": False, + "wsgi.run_once": False, + "REQUEST_METHOD": request.method, + "SCRIPT_NAME": self.script_name, + "PATH_INFO": request.url.path, + "QUERY_STRING": request.url.query.decode("ascii"), + "SERVER_NAME": request.url.host, + "SERVER_PORT": str(port), + "SERVER_PROTOCOL": "HTTP/1.1", + "REMOTE_ADDR": self.remote_addr, + } + for header_key, header_value in request.headers.raw: + key = header_key.decode("ascii").upper().replace("-", "_") + if key not in ("CONTENT_TYPE", "CONTENT_LENGTH"): + key = "HTTP_" + key + environ[key] = header_value.decode("ascii") + + seen_status = None + seen_response_headers = None + seen_exc_info = None + + def start_response( + status: str, + response_headers: list[tuple[str, str]], + exc_info: OptExcInfo | None = None, + ) -> typing.Callable[[bytes], typing.Any]: + nonlocal seen_status, seen_response_headers, seen_exc_info + seen_status = status + seen_response_headers = response_headers + seen_exc_info = exc_info + return lambda _: None + + result = self.app(environ, start_response) + + stream = WSGIByteStream(result) + + assert seen_status is not None + assert seen_response_headers is not None + if seen_exc_info and seen_exc_info[0] and self.raise_app_exceptions: + raise seen_exc_info[1] + + status_code = int(seen_status.split()[0]) + headers = [ + (key.encode("ascii"), value.encode("ascii")) + for key, value in seen_response_headers + ] + + return Response(status_code, headers=headers, stream=stream) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2-3.1.6.dist-info/licenses/LICENSE.txt b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2-3.1.6.dist-info/licenses/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..c37cae49ec77ad6ebb25568c1605f1fee5313cfb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/jinja2-3.1.6.dist-info/licenses/LICENSE.txt @@ -0,0 +1,28 @@ +Copyright 2007 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/portalocker-3.2.0.dist-info/licenses/LICENSE b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/portalocker-3.2.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..b638bda0d3b793bf3c52b37c9ea1f5b6dc0ce98b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/portalocker-3.2.0.dist-info/licenses/LICENSE @@ -0,0 +1,11 @@ +Copyright 2022 Rick van Hattem + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/pyarrow_hotfix-0.7.dist-info/licenses/LICENSE.txt b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/pyarrow_hotfix-0.7.dist-info/licenses/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..d645695673349e3947e8e5ae42332d0ac3164cd7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/pyarrow_hotfix-0.7.dist-info/licenses/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cc6d5111de76d544a10e29ac652b9e6b7c0ad1c5 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/matchers.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/matchers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dca8ecd111e6c72e7287da83c27243fcd4e2709d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/matchers.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/registries.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/registries.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..030d51801222ab9bb8d8f5ea0431cedd8d204b94 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/registries.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/test_matchers.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/test_matchers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ce9066e38f9a39d0bf52be50bb744ead9b48b15c Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/test_matchers.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/test_registries.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/test_registries.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e2ed198c26ad5cce2cc90e9884d0f34788aab30e Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/test_registries.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/test_responses.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/test_responses.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..85c991994734c65661d6c8b4aa1da00a38e6afaa Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/responses/__pycache__/test_responses.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..948fee10ca2311d9d66739b928c05dfd722101f8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/__pycache__/_impl.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/__pycache__/_impl.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da9f6376aeabca45a9a0d987693c4bed0fbfcff8 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/__pycache__/_impl.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/__pycache__/_version.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/__pycache__/_version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e5928faeae55491c925b618362aa1fa23490f74a Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/__pycache__/_version.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/_tests/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/_tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/_tests/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/_tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1677a8800bbfd6034d2473bf01bf44f258464fac Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/_tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/_tests/__pycache__/test_sniffio.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/_tests/__pycache__/test_sniffio.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7772f525cb47d642dc10a950e2d81407cf9d9834 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/_tests/__pycache__/test_sniffio.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/_tests/test_sniffio.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/_tests/test_sniffio.py new file mode 100644 index 0000000000000000000000000000000000000000..02945a947527a9bfb396d0cd35ffee22eda665bb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/sniffio/_tests/test_sniffio.py @@ -0,0 +1,84 @@ +import os +import sys + +import pytest + +from .. import ( + current_async_library, AsyncLibraryNotFoundError, + current_async_library_cvar, thread_local +) + + +def test_basics_cvar(): + with pytest.raises(AsyncLibraryNotFoundError): + current_async_library() + + token = current_async_library_cvar.set("generic-lib") + try: + assert current_async_library() == "generic-lib" + finally: + current_async_library_cvar.reset(token) + + with pytest.raises(AsyncLibraryNotFoundError): + current_async_library() + + +def test_basics_tlocal(): + with pytest.raises(AsyncLibraryNotFoundError): + current_async_library() + + old_name, thread_local.name = thread_local.name, "generic-lib" + try: + assert current_async_library() == "generic-lib" + finally: + thread_local.name = old_name + + with pytest.raises(AsyncLibraryNotFoundError): + current_async_library() + + +def test_asyncio(): + import asyncio + + with pytest.raises(AsyncLibraryNotFoundError): + current_async_library() + + ran = [] + + async def this_is_asyncio(): + assert current_async_library() == "asyncio" + # Call it a second time to exercise the caching logic + assert current_async_library() == "asyncio" + ran.append(True) + + asyncio.run(this_is_asyncio()) + assert ran == [True] + + with pytest.raises(AsyncLibraryNotFoundError): + current_async_library() + + +@pytest.mark.skipif( + sys.version_info >= (3, 12), + reason= + "curio broken on 3.12 (https://github.com/python-trio/sniffio/pull/42)", +) +def test_curio(): + import curio + + with pytest.raises(AsyncLibraryNotFoundError): + current_async_library() + + ran = [] + + async def this_is_curio(): + assert current_async_library() == "curio" + # Call it a second time to exercise the caching logic + assert current_async_library() == "curio" + ran.append(True) + + curio.run(this_is_curio) + assert ran == [True] + + with pytest.raises(AsyncLibraryNotFoundError): + current_async_library() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_VariableFunctions.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_VariableFunctions.pyi new file mode 100644 index 0000000000000000000000000000000000000000..9fd18ba29fa23b662defcaaed6d53c4918da67fe --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_VariableFunctions.pyi @@ -0,0 +1,33783 @@ +# @generated by tools/pyi/gen_pyi.py from torch/_C/_VariableFunctions.pyi.in +# mypy: disable-error-code="type-arg" +# mypy: allow-untyped-defs +# ruff: noqa: F401,PYI054 + +from collections.abc import Callable, Sequence +from types import EllipsisType +from typing import Any, Literal, overload, TypeVar + +import torch +from torch import ( + contiguous_format, + Generator, + inf, + memory_format, + strided, + SymInt, + Tensor, +) +from torch._prims_common import DeviceLikeType +from torch.types import ( + _bool, + _complex, + _device, + _dtype, + _float, + _int, + _layout, + _qscheme, + _size, + Device, + Number, +) + +__all__ = [ + "__and__", + "__lshift__", + "__or__", + "__rshift__", + "__xor__", + "_adaptive_avg_pool2d", + "_adaptive_avg_pool3d", + "_add_batch_dim", + "_add_relu", + "_add_relu_", + "_addmm_activation", + "_aminmax", + "_amp_foreach_non_finite_check_and_unscale_", + "_amp_update_scale_", + "_assert_async", + "_assert_scalar", + "_assert_tensor_metadata", + "_batch_norm_impl_index", + "_cast_Byte", + "_cast_Char", + "_cast_Double", + "_cast_Float", + "_cast_Half", + "_cast_Int", + "_cast_Long", + "_cast_Short", + "_choose_qparams_per_tensor", + "_chunk_cat", + "_coalesce", + "_compute_linear_combination", + "_conj", + "_conj_copy", + "_conj_physical", + "_convert_indices_from_coo_to_csr", + "_convert_indices_from_csr_to_coo", + "_convert_weight_to_int4pack", + "_convert_weight_to_int4pack_for_cpu", + "_convolution", + "_convolution_mode", + "_copy_from", + "_copy_from_and_resize", + "_cslt_compress", + "_cslt_sparse_mm", + "_cslt_sparse_mm_search", + "_ctc_loss", + "_cudnn_ctc_loss", + "_cudnn_init_dropout_state", + "_cudnn_rnn", + "_cudnn_rnn_flatten_weight", + "_cufft_clear_plan_cache", + "_cufft_get_plan_cache_max_size", + "_cufft_get_plan_cache_size", + "_cufft_set_plan_cache_max_size", + "_cummax_helper", + "_cummin_helper", + "_debug_has_internal_overlap", + "_dim_arange", + "_dirichlet_grad", + "_disable_functionalization", + "_dyn_quant_matmul_4bit", + "_dyn_quant_pack_4bit_weight", + "_efficientzerotensor", + "_embedding_bag", + "_embedding_bag_forward_only", + "_empty_affine_quantized", + "_empty_per_channel_affine_quantized", + "_enable_functionalization", + "_euclidean_dist", + "_fake_quantize_learnable_per_channel_affine", + "_fake_quantize_learnable_per_tensor_affine", + "_fake_quantize_per_tensor_affine_cachemask_tensor_qparams", + "_fft_c2c", + "_fft_c2r", + "_fft_r2c", + "_fill_mem_eff_dropout_mask_", + "_foobar", + "_foreach_abs", + "_foreach_abs_", + "_foreach_acos", + "_foreach_acos_", + "_foreach_add", + "_foreach_add_", + "_foreach_addcdiv", + "_foreach_addcdiv_", + "_foreach_addcmul", + "_foreach_addcmul_", + "_foreach_asin", + "_foreach_asin_", + "_foreach_atan", + "_foreach_atan_", + "_foreach_ceil", + "_foreach_ceil_", + "_foreach_clamp_max", + "_foreach_clamp_max_", + "_foreach_clamp_min", + "_foreach_clamp_min_", + "_foreach_copy_", + "_foreach_cos", + "_foreach_cos_", + "_foreach_cosh", + "_foreach_cosh_", + "_foreach_div", + "_foreach_div_", + "_foreach_erf", + "_foreach_erf_", + "_foreach_erfc", + "_foreach_erfc_", + "_foreach_exp", + "_foreach_exp_", + "_foreach_expm1", + "_foreach_expm1_", + "_foreach_floor", + "_foreach_floor_", + "_foreach_frac", + "_foreach_frac_", + "_foreach_lerp", + "_foreach_lerp_", + "_foreach_lgamma", + "_foreach_lgamma_", + "_foreach_log", + "_foreach_log10", + "_foreach_log10_", + "_foreach_log1p", + "_foreach_log1p_", + "_foreach_log2", + "_foreach_log2_", + "_foreach_log_", + "_foreach_max", + "_foreach_maximum", + "_foreach_maximum_", + "_foreach_minimum", + "_foreach_minimum_", + "_foreach_mul", + "_foreach_mul_", + "_foreach_neg", + "_foreach_neg_", + "_foreach_norm", + "_foreach_pow", + "_foreach_pow_", + "_foreach_reciprocal", + "_foreach_reciprocal_", + "_foreach_round", + "_foreach_round_", + "_foreach_rsqrt", + "_foreach_rsqrt_", + "_foreach_sigmoid", + "_foreach_sigmoid_", + "_foreach_sign", + "_foreach_sign_", + "_foreach_sin", + "_foreach_sin_", + "_foreach_sinh", + "_foreach_sinh_", + "_foreach_sqrt", + "_foreach_sqrt_", + "_foreach_sub", + "_foreach_sub_", + "_foreach_tan", + "_foreach_tan_", + "_foreach_tanh", + "_foreach_tanh_", + "_foreach_trunc", + "_foreach_trunc_", + "_foreach_zero_", + "_from_functional_tensor", + "_functional_assert_async", + "_functional_assert_scalar", + "_functional_sym_constrain_range", + "_functional_sym_constrain_range_for_size", + "_functionalize_apply_view_metas", + "_functionalize_are_all_mutations_hidden_from_autograd", + "_functionalize_are_all_mutations_under_no_grad_or_inference_mode", + "_functionalize_commit_update", + "_functionalize_has_metadata_mutation", + "_functionalize_inductor_storage_resized_counter", + "_functionalize_is_symbolic", + "_functionalize_mark_mutation_hidden_from_autograd", + "_functionalize_mark_storage_changed", + "_functionalize_mutation_counter", + "_functionalize_replace", + "_functionalize_storage_changed_counter", + "_functionalize_sync", + "_functionalize_unsafe_set", + "_functionalize_was_inductor_storage_resized", + "_functionalize_was_storage_changed", + "_fused_adagrad_", + "_fused_adam_", + "_fused_adamw_", + "_fused_dropout", + "_fused_moving_avg_obs_fq_helper", + "_fused_rms_norm", + "_fused_sdp_choice", + "_fused_sgd_", + "_fw_primal_copy", + "_grid_sampler_2d_cpu_fallback", + "_grouped_mm", + "_has_compatible_shallow_copy_type", + "_histogramdd_bin_edges", + "_histogramdd_from_bin_cts", + "_histogramdd_from_bin_tensors", + "_index_put_impl_", + "_indices_copy", + "_int_mm", + "_is_all_true", + "_is_any_true", + "_is_functional_tensor", + "_is_functional_tensor_base", + "_is_zerotensor", + "_lazy_clone", + "_linalg_check_errors", + "_linalg_det", + "_linalg_eigh", + "_linalg_slogdet", + "_linalg_solve_ex", + "_linalg_svd", + "_log_softmax", + "_log_softmax_backward_data", + "_logcumsumexp", + "_lstm_mps", + "_lu_with_info", + "_make_dep_token", + "_make_dual", + "_make_dual_copy", + "_make_per_channel_quantized_tensor", + "_make_per_tensor_quantized_tensor", + "_masked_scale", + "_masked_softmax", + "_mixed_dtypes_linear", + "_mkldnn_reshape", + "_mkldnn_transpose", + "_mkldnn_transpose_", + "_mps_convolution", + "_mps_convolution_transpose", + "_native_batch_norm_legit", + "_native_batch_norm_legit_no_training", + "_native_multi_head_attention", + "_neg_view", + "_neg_view_copy", + "_nested_compute_contiguous_strides_offsets", + "_nested_from_padded", + "_nested_from_padded_and_nested_example", + "_nested_from_padded_tensor", + "_nested_get_jagged_dummy", + "_nested_get_lengths", + "_nested_get_max_seqlen", + "_nested_get_min_seqlen", + "_nested_get_offsets", + "_nested_get_ragged_idx", + "_nested_get_values", + "_nested_get_values_copy", + "_nested_tensor_from_mask", + "_nested_tensor_from_mask_left_aligned", + "_nested_tensor_from_tensor_list", + "_nested_tensor_softmax_with_shape", + "_nested_view_from_buffer", + "_nested_view_from_buffer_copy", + "_nested_view_from_jagged", + "_nested_view_from_jagged_copy", + "_nnpack_available", + "_nnpack_spatial_convolution", + "_pack_padded_sequence", + "_pad_packed_sequence", + "_pin_memory", + "_prelu_kernel", + "_print", + "_propagate_xla_data", + "_remove_batch_dim", + "_reshape_alias_copy", + "_reshape_from_tensor", + "_resize_output_", + "_rowwise_prune", + "_safe_softmax", + "_sample_dirichlet", + "_saturate_weight_to_fp16", + "_scaled_dot_product_attention_math", + "_scaled_dot_product_attention_math_for_mps", + "_scaled_dot_product_cudnn_attention", + "_scaled_dot_product_efficient_attention", + "_scaled_dot_product_flash_attention", + "_scaled_dot_product_flash_attention_for_cpu", + "_scaled_grouped_mm", + "_scaled_grouped_mm_v2", + "_scaled_mm", + "_scaled_mm_v2", + "_shape_as_tensor", + "_sobol_engine_draw", + "_sobol_engine_ff_", + "_sobol_engine_initialize_state_", + "_sobol_engine_scramble_", + "_softmax", + "_softmax_backward_data", + "_sparse_broadcast_to", + "_sparse_broadcast_to_copy", + "_sparse_csr_prod", + "_sparse_csr_sum", + "_sparse_log_softmax_backward_data", + "_sparse_semi_structured_addmm", + "_sparse_semi_structured_apply", + "_sparse_semi_structured_apply_dense", + "_sparse_semi_structured_linear", + "_sparse_semi_structured_mm", + "_sparse_semi_structured_tile", + "_sparse_softmax_backward_data", + "_sparse_sparse_matmul", + "_sparse_sum", + "_stack", + "_standard_gamma", + "_standard_gamma_grad", + "_sync", + "_test_autograd_multiple_dispatch", + "_test_autograd_multiple_dispatch_view", + "_test_autograd_multiple_dispatch_view_copy", + "_test_check_tensor", + "_test_functorch_fallback", + "_test_parallel_materialize", + "_test_serialization_subcmul", + "_to_cpu", + "_to_functional_tensor", + "_to_sparse_semi_structured", + "_transform_bias_rescale_qkv", + "_transformer_encoder_layer_fwd", + "_trilinear", + "_triton_multi_head_attention", + "_triton_scaled_dot_attention", + "_unique", + "_unique2", + "_unpack_dual", + "_unsafe_index", + "_unsafe_index_put", + "_unsafe_masked_index", + "_unsafe_masked_index_put_accumulate", + "_use_cudnn_ctc_loss", + "_use_cudnn_rnn_flatten_weight", + "_validate_compressed_sparse_indices", + "_validate_sparse_bsc_tensor_args", + "_validate_sparse_bsr_tensor_args", + "_validate_sparse_compressed_tensor_args", + "_validate_sparse_coo_tensor_args", + "_validate_sparse_csc_tensor_args", + "_validate_sparse_csr_tensor_args", + "_values_copy", + "_weight_int4pack_mm", + "_weight_int4pack_mm_for_cpu", + "_weight_int4pack_mm_with_scales_and_zeros", + "_weight_int8pack_mm", + "_weight_norm", + "_weight_norm_interface", + "_wrapped_linear_prepack", + "_wrapped_quantized_linear_prepacked", + "abs", + "abs_", + "absolute", + "acos", + "acos_", + "acosh", + "acosh_", + "adaptive_avg_pool1d", + "adaptive_max_pool1d", + "add", + "addbmm", + "addcdiv", + "addcmul", + "addmm", + "addmv", + "addmv_", + "addr", + "adjoint", + "affine_grid_generator", + "alias_copy", + "all", + "allclose", + "alpha_dropout", + "alpha_dropout_", + "amax", + "amin", + "aminmax", + "angle", + "any", + "arange", + "arccos", + "arccos_", + "arccosh", + "arccosh_", + "arcsin", + "arcsin_", + "arcsinh", + "arcsinh_", + "arctan", + "arctan2", + "arctan_", + "arctanh", + "arctanh_", + "argmax", + "argmin", + "argsort", + "argwhere", + "as_strided", + "as_strided_", + "as_strided_copy", + "as_strided_scatter", + "as_tensor", + "asarray", + "asin", + "asin_", + "asinh", + "asinh_", + "atan", + "atan2", + "atan_", + "atanh", + "atanh_", + "avg_pool1d", + "baddbmm", + "bartlett_window", + "batch_norm", + "batch_norm_backward_elemt", + "batch_norm_backward_reduce", + "batch_norm_elemt", + "batch_norm_gather_stats", + "batch_norm_gather_stats_with_counts", + "batch_norm_stats", + "batch_norm_update_stats", + "bernoulli", + "bilinear", + "binary_cross_entropy_with_logits", + "bincount", + "binomial", + "bitwise_and", + "bitwise_left_shift", + "bitwise_not", + "bitwise_or", + "bitwise_right_shift", + "bitwise_xor", + "blackman_window", + "bmm", + "broadcast_to", + "bucketize", + "can_cast", + "cat", + "ccol_indices_copy", + "ceil", + "ceil_", + "celu", + "celu_", + "channel_shuffle", + "cholesky", + "cholesky_inverse", + "cholesky_solve", + "choose_qparams_optimized", + "chunk", + "clamp", + "clamp_", + "clamp_max", + "clamp_max_", + "clamp_min", + "clamp_min_", + "clip", + "clip_", + "clone", + "col_indices_copy", + "column_stack", + "combinations", + "complex", + "concat", + "concatenate", + "conj", + "conj_physical", + "conj_physical_", + "constant_pad_nd", + "conv1d", + "conv2d", + "conv3d", + "conv_tbc", + "conv_transpose1d", + "conv_transpose2d", + "conv_transpose3d", + "convolution", + "copysign", + "corrcoef", + "cos", + "cos_", + "cosh", + "cosh_", + "cosine_embedding_loss", + "cosine_similarity", + "count_nonzero", + "cov", + "cross", + "crow_indices_copy", + "ctc_loss", + "cudnn_affine_grid_generator", + "cudnn_batch_norm", + "cudnn_convolution", + "cudnn_convolution_add_relu", + "cudnn_convolution_relu", + "cudnn_convolution_transpose", + "cudnn_grid_sampler", + "cudnn_is_acceptable", + "cummax", + "cummin", + "cumprod", + "cumsum", + "cumulative_trapezoid", + "deg2rad", + "deg2rad_", + "dequantize", + "det", + "detach", + "detach_", + "detach_copy", + "diag", + "diag_embed", + "diagflat", + "diagonal", + "diagonal_copy", + "diagonal_scatter", + "diff", + "digamma", + "dist", + "div", + "divide", + "dot", + "dropout", + "dropout_", + "dsmm", + "dsplit", + "dstack", + "embedding", + "embedding_bag", + "embedding_renorm_", + "empty", + "empty_like", + "empty_permuted", + "empty_quantized", + "empty_strided", + "eq", + "equal", + "erf", + "erf_", + "erfc", + "erfc_", + "erfinv", + "exp", + "exp2", + "exp2_", + "exp_", + "expand_copy", + "expm1", + "expm1_", + "eye", + "fake_quantize_per_channel_affine", + "fake_quantize_per_tensor_affine", + "fbgemm_linear_fp16_weight", + "fbgemm_linear_fp16_weight_fp32_activation", + "fbgemm_linear_int8_weight", + "fbgemm_linear_int8_weight_fp32_activation", + "fbgemm_linear_quantize_weight", + "fbgemm_pack_gemm_matrix_fp16", + "fbgemm_pack_quantized_matrix", + "feature_alpha_dropout", + "feature_alpha_dropout_", + "feature_dropout", + "feature_dropout_", + "fill", + "fill_", + "fix", + "fix_", + "flatten", + "flip", + "fliplr", + "flipud", + "float_power", + "floor", + "floor_", + "floor_divide", + "fmax", + "fmin", + "fmod", + "frac", + "frac_", + "frexp", + "frobenius_norm", + "from_file", + "from_numpy", + "frombuffer", + "full", + "full_like", + "fused_moving_avg_obs_fake_quant", + "gather", + "gcd", + "gcd_", + "ge", + "geqrf", + "ger", + "get_default_dtype", + "get_num_interop_threads", + "get_num_threads", + "gradient", + "greater", + "greater_equal", + "grid_sampler", + "grid_sampler_2d", + "grid_sampler_3d", + "group_norm", + "gru", + "gru_cell", + "gt", + "hamming_window", + "hann_window", + "hardshrink", + "hash_tensor", + "heaviside", + "hinge_embedding_loss", + "histc", + "histogram", + "histogramdd", + "hsmm", + "hsplit", + "hspmm", + "hstack", + "hypot", + "i0", + "i0_", + "igamma", + "igammac", + "imag", + "index_add", + "index_copy", + "index_fill", + "index_put", + "index_put_", + "index_reduce", + "index_select", + "indices_copy", + "init_num_threads", + "inner", + "instance_norm", + "int_repr", + "inverse", + "is_complex", + "is_conj", + "is_distributed", + "is_floating_point", + "is_grad_enabled", + "is_inference", + "is_inference_mode_enabled", + "is_neg", + "is_nonzero", + "is_same_size", + "is_signed", + "is_vulkan_available", + "isclose", + "isfinite", + "isin", + "isinf", + "isnan", + "isneginf", + "isposinf", + "isreal", + "istft", + "kaiser_window", + "kl_div", + "kron", + "kthvalue", + "layer_norm", + "lcm", + "lcm_", + "ldexp", + "ldexp_", + "le", + "lerp", + "less", + "less_equal", + "lgamma", + "linspace", + "log", + "log10", + "log10_", + "log1p", + "log1p_", + "log2", + "log2_", + "log_", + "log_softmax", + "logaddexp", + "logaddexp2", + "logcumsumexp", + "logdet", + "logical_and", + "logical_not", + "logical_or", + "logical_xor", + "logit", + "logit_", + "logspace", + "logsumexp", + "lstm", + "lstm_cell", + "lt", + "lu_solve", + "lu_unpack", + "margin_ranking_loss", + "masked_fill", + "masked_scatter", + "masked_select", + "matmul", + "matrix_exp", + "matrix_power", + "max", + "max_pool1d", + "max_pool1d_with_indices", + "max_pool2d", + "max_pool3d", + "maximum", + "mean", + "median", + "min", + "minimum", + "miopen_batch_norm", + "miopen_convolution", + "miopen_convolution_add_relu", + "miopen_convolution_relu", + "miopen_convolution_transpose", + "miopen_depthwise_convolution", + "miopen_rnn", + "mkldnn_adaptive_avg_pool2d", + "mkldnn_convolution", + "mkldnn_linear_backward_weights", + "mkldnn_max_pool2d", + "mkldnn_max_pool3d", + "mkldnn_rnn_layer", + "mm", + "mode", + "moveaxis", + "movedim", + "msort", + "mul", + "multinomial", + "multiply", + "mv", + "mvlgamma", + "nan_to_num", + "nan_to_num_", + "nanmean", + "nanmedian", + "nanquantile", + "nansum", + "narrow", + "narrow_copy", + "native_batch_norm", + "native_channel_shuffle", + "native_dropout", + "native_group_norm", + "native_layer_norm", + "native_norm", + "ne", + "neg", + "neg_", + "negative", + "negative_", + "nextafter", + "nonzero", + "nonzero_static", + "norm_except_dim", + "normal", + "not_equal", + "nuclear_norm", + "numel", + "ones", + "ones_like", + "orgqr", + "ormqr", + "outer", + "pairwise_distance", + "pdist", + "permute", + "permute_copy", + "pinverse", + "pixel_shuffle", + "pixel_unshuffle", + "poisson", + "poisson_nll_loss", + "polar", + "polygamma", + "positive", + "pow", + "prelu", + "prod", + "promote_types", + "put", + "q_per_channel_axis", + "q_per_channel_scales", + "q_per_channel_zero_points", + "q_scale", + "q_zero_point", + "qr", + "quantile", + "quantize_per_channel", + "quantize_per_tensor", + "quantize_per_tensor_dynamic", + "quantized_batch_norm", + "quantized_gru_cell", + "quantized_lstm_cell", + "quantized_max_pool1d", + "quantized_max_pool2d", + "quantized_max_pool3d", + "quantized_rnn_relu_cell", + "quantized_rnn_tanh_cell", + "rad2deg", + "rad2deg_", + "rand", + "rand_like", + "randint", + "randint_like", + "randn", + "randn_like", + "randperm", + "range", + "ravel", + "real", + "reciprocal", + "reciprocal_", + "relu", + "relu_", + "remainder", + "renorm", + "repeat_interleave", + "reshape", + "resize_as_", + "resize_as_sparse_", + "resolve_conj", + "resolve_neg", + "result_type", + "rms_norm", + "rnn_relu", + "rnn_relu_cell", + "rnn_tanh", + "rnn_tanh_cell", + "roll", + "rot90", + "round", + "round_", + "row_indices_copy", + "row_stack", + "rrelu", + "rrelu_", + "rsqrt", + "rsqrt_", + "rsub", + "saddmm", + "scalar_tensor", + "scatter", + "scatter_add", + "scatter_reduce", + "searchsorted", + "segment_reduce", + "select", + "select_copy", + "select_scatter", + "selu", + "selu_", + "set_flush_denormal", + "set_num_interop_threads", + "set_num_threads", + "sgn", + "sigmoid", + "sigmoid_", + "sign", + "signbit", + "sin", + "sin_", + "sinc", + "sinc_", + "sinh", + "sinh_", + "slice_copy", + "slice_inverse", + "slice_scatter", + "slogdet", + "smm", + "softmax", + "sort", + "sparse_bsc_tensor", + "sparse_bsr_tensor", + "sparse_compressed_tensor", + "sparse_coo_tensor", + "sparse_csc_tensor", + "sparse_csr_tensor", + "split_copy", + "split_with_sizes", + "split_with_sizes_copy", + "spmm", + "sqrt", + "sqrt_", + "square", + "square_", + "squeeze", + "squeeze_copy", + "sspaddmm", + "stack", + "std", + "std_mean", + "sub", + "subtract", + "sum", + "svd", + "swapaxes", + "swapdims", + "sym_constrain_range", + "sym_constrain_range_for_size", + "t", + "t_copy", + "take", + "take_along_dim", + "tan", + "tan_", + "tanh", + "tanh_", + "tensor", + "tensor_split", + "threshold", + "threshold_", + "tile", + "topk", + "trace", + "transpose", + "transpose_copy", + "trapezoid", + "trapz", + "triangular_solve", + "tril", + "tril_indices", + "triplet_margin_loss", + "triu", + "triu_indices", + "true_divide", + "trunc", + "trunc_", + "unbind", + "unbind_copy", + "unflatten", + "unfold_copy", + "unique_dim", + "unsafe_chunk", + "unsafe_split", + "unsafe_split_with_sizes", + "unsqueeze", + "unsqueeze_copy", + "values_copy", + "vander", + "var", + "var_mean", + "vdot", + "view_as_complex", + "view_as_complex_copy", + "view_as_real", + "view_as_real_copy", + "view_copy", + "vsplit", + "vstack", + "where", + "xlogy", + "xlogy_", + "zero_", + "zeros", + "zeros_like", +] + +@overload +def __and__(input: Tensor, other: Tensor) -> Tensor: ... +@overload +def __and__(input: Tensor, other: Number | _complex) -> Tensor: ... +@overload +def __lshift__(input: Tensor, other: Tensor) -> Tensor: ... +@overload +def __lshift__(input: Tensor, other: Number | _complex) -> Tensor: ... +@overload +def __or__(input: Tensor, other: Tensor) -> Tensor: ... +@overload +def __or__(input: Tensor, other: Number | _complex) -> Tensor: ... +@overload +def __rshift__(input: Tensor, other: Tensor) -> Tensor: ... +@overload +def __rshift__(input: Tensor, other: Number | _complex) -> Tensor: ... +@overload +def __xor__(input: Tensor, other: Tensor) -> Tensor: ... +@overload +def __xor__(input: Tensor, other: Number | _complex) -> Tensor: ... +def _adaptive_avg_pool2d( + input: Tensor, + output_size: _int | SymInt | Sequence[_int | SymInt], +) -> Tensor: ... +def _adaptive_avg_pool3d( + input: Tensor, + output_size: _int | SymInt | Sequence[_int | SymInt], +) -> Tensor: ... +def _add_batch_dim(input: Tensor, batch_dim: _int, level: _int) -> Tensor: ... +@overload +def _add_relu( + input: Tensor, + other: Tensor, + *, + alpha: Number | _complex = 1, + out: Tensor | None = None, +) -> Tensor: ... +@overload +def _add_relu( + input: Tensor, + other: Number | _complex, + alpha: Number | _complex = 1, +) -> Tensor: ... +@overload +def _add_relu_( + input: Tensor, + other: Tensor, + *, + alpha: Number | _complex = 1, +) -> Tensor: ... +@overload +def _add_relu_( + input: Tensor, + other: Number | _complex, + alpha: Number | _complex = 1, +) -> Tensor: ... +def _addmm_activation( + input: Tensor, + mat1: Tensor, + mat2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + use_gelu: _bool = False, + out: Tensor | None = None, +) -> Tensor: ... +@overload +def _aminmax(input: Tensor) -> tuple[Tensor, Tensor]: ... +@overload +def _aminmax( + input: Tensor, + dim: _int, + keepdim: _bool = False, +) -> tuple[Tensor, Tensor]: ... +def _amp_foreach_non_finite_check_and_unscale_( + self: tuple[Tensor, ...] | list[Tensor] | None, + found_inf: Tensor, + inv_scale: Tensor, +) -> None: ... +def _amp_update_scale_( + input: Tensor, + growth_tracker: Tensor, + found_inf: Tensor, + scale_growth_factor: _float, + scale_backoff_factor: _float, + growth_interval: _int, +) -> Tensor: ... +@overload +def _assert_async(input: Tensor) -> None: + r""" + _assert_async(tensor) -> void + + Asynchronously assert that the contents of tensor are nonzero. For CPU tensors, + this is equivalent to ``assert tensor`` or ``assert tensor.is_nonzero()``; for + CUDA tensors, we DO NOT synchronize and you may only find out the assertion + failed at a later CUDA kernel launch. Asynchronous assertion can be helpful for + testing invariants in CUDA tensors without giving up performance. This function + is NOT intended to be used for regular error checking, as it will trash your CUDA + context if the assert fails (forcing you to restart your PyTorch process.) + + Args: + tensor (Tensor): a one element tensor to test to see if it is nonzero. Zero + elements (including False for boolean tensors) cause an assertion failure + to be raised. + """ + +@overload +def _assert_async(input: Tensor, assert_msg: str) -> None: + r""" + _assert_async(tensor) -> void + + Asynchronously assert that the contents of tensor are nonzero. For CPU tensors, + this is equivalent to ``assert tensor`` or ``assert tensor.is_nonzero()``; for + CUDA tensors, we DO NOT synchronize and you may only find out the assertion + failed at a later CUDA kernel launch. Asynchronous assertion can be helpful for + testing invariants in CUDA tensors without giving up performance. This function + is NOT intended to be used for regular error checking, as it will trash your CUDA + context if the assert fails (forcing you to restart your PyTorch process.) + + Args: + tensor (Tensor): a one element tensor to test to see if it is nonzero. Zero + elements (including False for boolean tensors) cause an assertion failure + to be raised. + """ + +def _assert_scalar(self: Number | _complex, assert_msg: str) -> None: ... +def _assert_tensor_metadata( + a: Tensor, + size: Sequence[_int | SymInt] | None = None, + stride: Sequence[_int | SymInt] | None = None, + dtype: _dtype | None = None, + *, + device: DeviceLikeType | None = None, + layout: _layout | None = None, +) -> None: ... +def _batch_norm_impl_index( + input: Tensor, + weight: Tensor | None, + bias: Tensor | None, + running_mean: Tensor | None, + running_var: Tensor | None, + training: _bool, + momentum: _float, + eps: _float, + cudnn_enabled: _bool, +) -> tuple[Tensor, Tensor, Tensor, Tensor, _int]: ... +def _cast_Byte(input: Tensor, non_blocking: _bool = False) -> Tensor: ... +def _cast_Char(input: Tensor, non_blocking: _bool = False) -> Tensor: ... +def _cast_Double(input: Tensor, non_blocking: _bool = False) -> Tensor: ... +def _cast_Float(input: Tensor, non_blocking: _bool = False) -> Tensor: ... +def _cast_Half(input: Tensor, non_blocking: _bool = False) -> Tensor: ... +def _cast_Int(input: Tensor, non_blocking: _bool = False) -> Tensor: ... +def _cast_Long(input: Tensor, non_blocking: _bool = False) -> Tensor: ... +def _cast_Short(input: Tensor, non_blocking: _bool = False) -> Tensor: ... +def _choose_qparams_per_tensor( + input: Tensor, + reduce_range: _bool = False, +) -> tuple[_float, _int]: ... +def _chunk_cat( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + dim: _int, + num_chunks: _int, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _coalesce(input: Tensor) -> Tensor: ... +def _compute_linear_combination( + input: Tensor, + coefficients: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _conj(input: Tensor) -> Tensor: ... +def _conj_copy(input: Tensor, *, out: Tensor | None = None) -> Tensor: ... +def _conj_physical(input: Tensor) -> Tensor: ... +def _convert_indices_from_coo_to_csr( + input: Tensor, + size: _int, + *, + out_int32: _bool = False, + out: Tensor | None = None, +) -> Tensor: ... +def _convert_indices_from_csr_to_coo( + crow_indices: Tensor, + col_indices: Tensor, + *, + out_int32: _bool = False, + transpose: _bool = False, + out: Tensor | None = None, +) -> Tensor: ... +def _convert_weight_to_int4pack(input: Tensor, innerKTiles: _int) -> Tensor: ... +def _convert_weight_to_int4pack_for_cpu( + input: Tensor, + innerKTiles: _int, +) -> Tensor: ... +@overload +def _convolution( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + stride: Sequence[_int | SymInt], + padding: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + transposed: _bool, + output_padding: _size, + groups: _int | SymInt, + benchmark: _bool, + deterministic: _bool, + cudnn_enabled: _bool, +) -> Tensor: ... +@overload +def _convolution( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + stride: Sequence[_int | SymInt], + padding: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + transposed: _bool, + output_padding: Sequence[_int | SymInt], + groups: _int | SymInt, + benchmark: _bool, + deterministic: _bool, + cudnn_enabled: _bool, + allow_tf32: _bool, +) -> Tensor: ... +def _convolution_mode( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + stride: Sequence[_int | SymInt], + padding: str, + dilation: Sequence[_int | SymInt], + groups: _int | SymInt, +) -> Tensor: ... +def _copy_from( + input: Tensor, + dst: Tensor, + non_blocking: _bool = False, +) -> Tensor: ... +def _copy_from_and_resize(input: Tensor, dst: Tensor) -> Tensor: ... +def _cslt_compress(input: Tensor) -> Tensor: ... +def _cslt_sparse_mm( + compressed_A: Tensor, + dense_B: Tensor, + bias: Tensor | None = None, + alpha: Tensor | None = None, + out_dtype: _dtype | None = None, + transpose_result: _bool = False, + alg_id: _int = 0, + split_k: _int = 1, + split_k_mode: _int = -1, +) -> Tensor: ... +def _cslt_sparse_mm_search( + compressed_A: Tensor, + dense_B: Tensor, + bias: Tensor | None = None, + alpha: Tensor | None = None, + out_dtype: _dtype | None = None, + transpose_result: _bool = False, +) -> _int: ... +@overload +def _ctc_loss( + log_probs: Tensor, + targets: Tensor, + input_lengths: _size, + target_lengths: _size, + blank: _int = 0, + zero_infinity: _bool = False, +) -> tuple[Tensor, Tensor]: ... +@overload +def _ctc_loss( + log_probs: Tensor, + targets: Tensor, + input_lengths: Tensor, + target_lengths: Tensor, + blank: _int = 0, + zero_infinity: _bool = False, +) -> tuple[Tensor, Tensor]: ... +@overload +def _cudnn_ctc_loss( + log_probs: Tensor, + targets: Tensor, + input_lengths: _size, + target_lengths: _size, + blank: _int, + deterministic: _bool, + zero_infinity: _bool, +) -> tuple[Tensor, Tensor]: ... +@overload +def _cudnn_ctc_loss( + log_probs: Tensor, + targets: Tensor, + input_lengths: Tensor, + target_lengths: Tensor, + blank: _int, + deterministic: _bool, + zero_infinity: _bool, +) -> tuple[Tensor, Tensor]: ... +def _cudnn_init_dropout_state( + dropout: _float, + train: _bool, + dropout_seed: _int, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: ... +def _cudnn_rnn( + input: Tensor, + weight: tuple[Tensor, ...] | list[Tensor] | None, + weight_stride0: _int, + weight_buf: Tensor | None, + hx: Tensor, + cx: Tensor | None, + mode: _int, + hidden_size: _int | SymInt, + proj_size: _int | SymInt, + num_layers: _int, + batch_first: _bool, + dropout: _float, + train: _bool, + bidirectional: _bool, + batch_sizes: Sequence[_int | SymInt], + dropout_state: Tensor | None, +) -> tuple[Tensor, Tensor, Tensor, Tensor, Tensor]: ... +def _cudnn_rnn_flatten_weight( + weight_arr: tuple[Tensor, ...] | list[Tensor] | None, + weight_stride0: _int, + input_size: _int | SymInt, + mode: _int, + hidden_size: _int | SymInt, + proj_size: _int | SymInt, + num_layers: _int, + batch_first: _bool, + bidirectional: _bool, +) -> Tensor: ... +def _cufft_clear_plan_cache(device_index: _int) -> None: ... +def _cufft_get_plan_cache_max_size(device_index: _int) -> _int: ... +def _cufft_get_plan_cache_size(device_index: _int) -> _int: ... +def _cufft_set_plan_cache_max_size( + device_index: _int, + max_size: _int, +) -> None: ... +def _cummax_helper( + input: Tensor, + values: Tensor, + indices: Tensor, + dim: _int, +) -> None: ... +def _cummin_helper( + input: Tensor, + values: Tensor, + indices: Tensor, + dim: _int, +) -> None: ... +def _debug_has_internal_overlap(input: Tensor) -> _int: ... +def _dim_arange(like: Tensor, dim: _int) -> Tensor: ... +def _dirichlet_grad(x: Tensor, alpha: Tensor, total: Tensor) -> Tensor: ... +def _disable_functionalization(): ... +def _dyn_quant_matmul_4bit( + inp: Tensor, + packed_weights: Tensor, + block_size: _int, + in_features: _int, + out_features: _int, +) -> Tensor: ... +def _dyn_quant_pack_4bit_weight( + weights: Tensor, + scales_zeros: Tensor, + bias: Tensor | None, + block_size: _int, + in_features: _int, + out_features: _int, +) -> Tensor: ... +@overload +def _efficientzerotensor( + size: Sequence[_int | SymInt], + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: ... +@overload +def _efficientzerotensor( + *size: _int | SymInt, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: ... +def _embedding_bag( + weight: Tensor, + indices: Tensor, + offsets: Tensor, + scale_grad_by_freq: _bool = False, + mode: _int = 0, + sparse: _bool = False, + per_sample_weights: Tensor | None = None, + include_last_offset: _bool = False, + padding_idx: _int = -1, +) -> tuple[Tensor, Tensor, Tensor, Tensor]: ... +def _embedding_bag_forward_only( + weight: Tensor, + indices: Tensor, + offsets: Tensor, + scale_grad_by_freq: _bool = False, + mode: _int = 0, + sparse: _bool = False, + per_sample_weights: Tensor | None = None, + include_last_offset: _bool = False, + padding_idx: _int = -1, +) -> tuple[Tensor, Tensor, Tensor, Tensor]: ... +@overload +def _empty_affine_quantized( + size: Sequence[_int | SymInt], + *, + scale: _float = 1, + zero_point: _int = 0, + memory_format: memory_format | None = contiguous_format, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: ... +@overload +def _empty_affine_quantized( + *size: _int | SymInt, + scale: _float = 1, + zero_point: _int = 0, + memory_format: memory_format | None = contiguous_format, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: ... +@overload +def _empty_per_channel_affine_quantized( + size: Sequence[_int | SymInt], + *, + scales: Tensor, + zero_points: Tensor, + axis: _int, + memory_format: memory_format | None = contiguous_format, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: ... +@overload +def _empty_per_channel_affine_quantized( + *size: _int | SymInt, + scales: Tensor, + zero_points: Tensor, + axis: _int, + memory_format: memory_format | None = contiguous_format, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: ... +def _enable_functionalization(*, reapply_views: _bool = False) -> None: ... +def _euclidean_dist(x1: Tensor, x2: Tensor) -> Tensor: ... +def _fake_quantize_learnable_per_channel_affine( + input: Tensor, + scale: Tensor, + zero_point: Tensor, + axis: _int, + quant_min: _int, + quant_max: _int, + grad_factor: _float = 1.0, +) -> Tensor: ... +def _fake_quantize_learnable_per_tensor_affine( + input: Tensor, + scale: Tensor, + zero_point: Tensor, + quant_min: _int, + quant_max: _int, + grad_factor: _float = 1.0, +) -> Tensor: ... +def _fake_quantize_per_tensor_affine_cachemask_tensor_qparams( + input: Tensor, + scale: Tensor, + zero_point: Tensor, + fake_quant_enabled: Tensor, + quant_min: _int, + quant_max: _int, +) -> torch.return_types._fake_quantize_per_tensor_affine_cachemask_tensor_qparams: # fmt: skip + ... +def _fft_c2c( + input: Tensor, + dim: Sequence[_int | SymInt], + normalization: _int, + forward: _bool, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _fft_c2r( + input: Tensor, + dim: _size, + normalization: _int, + last_dim_size: _int | SymInt, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _fft_r2c( + input: Tensor, + dim: _size, + normalization: _int, + onesided: _bool, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _fill_mem_eff_dropout_mask_( + input: Tensor, + dropout_p: _float, + seed: _int, + offset: _int, +) -> Tensor: ... +def _foobar( + input: Tensor, + arg1: _bool = True, + arg2: _bool = True, + *, + arg3: _bool = True, +) -> Tensor: ... +def _foreach_abs( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_abs(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.abs` to each Tensor of the input list. + """ + +def _foreach_abs_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_abs_(self: List[Tensor]) -> None + + Apply :func:`torch.abs` to each Tensor of the input list. + """ + +def _foreach_acos( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_acos(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.acos` to each Tensor of the input list. + """ + +def _foreach_acos_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_acos_(self: List[Tensor]) -> None + + Apply :func:`torch.acos` to each Tensor of the input list. + """ + +@overload +def _foreach_add( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_add( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, + *, + alpha: Number | _complex = 1, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_add( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: Tensor, + *, + alpha: Number | _complex = 1, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_add( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_add_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> None: ... +@overload +def _foreach_add_( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, + *, + alpha: Number | _complex = 1, +) -> None: ... +@overload +def _foreach_add_( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: Tensor, + *, + alpha: Number | _complex = 1, +) -> None: ... +@overload +def _foreach_add_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> None: ... +@overload +def _foreach_addcdiv( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensor1: tuple[Tensor, ...] | list[Tensor] | None, + tensor2: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_addcdiv( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensor1: tuple[Tensor, ...] | list[Tensor] | None, + tensor2: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Tensor, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_addcdiv( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensor1: tuple[Tensor, ...] | list[Tensor] | None, + tensor2: tuple[Tensor, ...] | list[Tensor] | None, + value: Number | _complex = 1, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_addcdiv_( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensor1: tuple[Tensor, ...] | list[Tensor] | None, + tensor2: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> None: ... +@overload +def _foreach_addcdiv_( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensor1: tuple[Tensor, ...] | list[Tensor] | None, + tensor2: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Tensor, +) -> None: ... +@overload +def _foreach_addcdiv_( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensor1: tuple[Tensor, ...] | list[Tensor] | None, + tensor2: tuple[Tensor, ...] | list[Tensor] | None, + value: Number | _complex = 1, +) -> None: ... +@overload +def _foreach_addcmul( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensor1: tuple[Tensor, ...] | list[Tensor] | None, + tensor2: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_addcmul( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensor1: tuple[Tensor, ...] | list[Tensor] | None, + tensor2: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Tensor, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_addcmul( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensor1: tuple[Tensor, ...] | list[Tensor] | None, + tensor2: tuple[Tensor, ...] | list[Tensor] | None, + value: Number | _complex = 1, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_addcmul_( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensor1: tuple[Tensor, ...] | list[Tensor] | None, + tensor2: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> None: ... +@overload +def _foreach_addcmul_( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensor1: tuple[Tensor, ...] | list[Tensor] | None, + tensor2: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Tensor, +) -> None: ... +@overload +def _foreach_addcmul_( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensor1: tuple[Tensor, ...] | list[Tensor] | None, + tensor2: tuple[Tensor, ...] | list[Tensor] | None, + value: Number | _complex = 1, +) -> None: ... +def _foreach_asin( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_asin(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.asin` to each Tensor of the input list. + """ + +def _foreach_asin_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_asin_(self: List[Tensor]) -> None + + Apply :func:`torch.asin` to each Tensor of the input list. + """ + +def _foreach_atan( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_atan(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.atan` to each Tensor of the input list. + """ + +def _foreach_atan_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_atan_(self: List[Tensor]) -> None + + Apply :func:`torch.atan` to each Tensor of the input list. + """ + +def _foreach_ceil( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_ceil(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.ceil` to each Tensor of the input list. + """ + +def _foreach_ceil_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_ceil_(self: List[Tensor]) -> None + + Apply :func:`torch.ceil` to each Tensor of the input list. + """ + +@overload +def _foreach_clamp_max( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_clamp_max( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_clamp_max( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_clamp_max_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> None: ... +@overload +def _foreach_clamp_max_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> None: ... +@overload +def _foreach_clamp_max_( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, +) -> None: ... +@overload +def _foreach_clamp_min( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_clamp_min( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_clamp_min( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_clamp_min_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> None: ... +@overload +def _foreach_clamp_min_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> None: ... +@overload +def _foreach_clamp_min_( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, +) -> None: ... +def _foreach_copy_( + self: tuple[Tensor, ...] | list[Tensor] | None, + src: tuple[Tensor, ...] | list[Tensor] | None, + non_blocking: _bool = False, +) -> None: ... +def _foreach_cos( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_cos(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.cos` to each Tensor of the input list. + """ + +def _foreach_cos_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_cos_(self: List[Tensor]) -> None + + Apply :func:`torch.cos` to each Tensor of the input list. + """ + +def _foreach_cosh( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_cosh(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.cosh` to each Tensor of the input list. + """ + +def _foreach_cosh_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_cosh_(self: List[Tensor]) -> None + + Apply :func:`torch.cosh` to each Tensor of the input list. + """ + +@overload +def _foreach_div( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_div( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: Tensor, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_div( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_div( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_div_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> None: ... +@overload +def _foreach_div_( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: Tensor, +) -> None: ... +@overload +def _foreach_div_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> None: ... +@overload +def _foreach_div_( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, +) -> None: ... +def _foreach_erf( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_erf(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.erf` to each Tensor of the input list. + """ + +def _foreach_erf_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_erf_(self: List[Tensor]) -> None + + Apply :func:`torch.erf` to each Tensor of the input list. + """ + +def _foreach_erfc( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_erfc(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.erfc` to each Tensor of the input list. + """ + +def _foreach_erfc_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_erfc_(self: List[Tensor]) -> None + + Apply :func:`torch.erfc` to each Tensor of the input list. + """ + +def _foreach_exp( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_exp(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.exp` to each Tensor of the input list. + """ + +def _foreach_exp_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_exp_(self: List[Tensor]) -> None + + Apply :func:`torch.exp` to each Tensor of the input list. + """ + +def _foreach_expm1( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_expm1(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.expm1` to each Tensor of the input list. + """ + +def _foreach_expm1_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_expm1_(self: List[Tensor]) -> None + + Apply :func:`torch.expm1` to each Tensor of the input list. + """ + +def _foreach_floor( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_floor(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.floor` to each Tensor of the input list. + """ + +def _foreach_floor_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_floor_(self: List[Tensor]) -> None + + Apply :func:`torch.floor` to each Tensor of the input list. + """ + +def _foreach_frac( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_frac(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.frac` to each Tensor of the input list. + """ + +def _foreach_frac_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_frac_(self: List[Tensor]) -> None + + Apply :func:`torch.frac` to each Tensor of the input list. + """ + +@overload +def _foreach_lerp( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensors1: tuple[Tensor, ...] | list[Tensor] | None, + weight: Number | _complex, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_lerp( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensors1: tuple[Tensor, ...] | list[Tensor] | None, + weight: Sequence[Number | _complex], +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_lerp( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensors1: tuple[Tensor, ...] | list[Tensor] | None, + weights: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_lerp_( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensors1: tuple[Tensor, ...] | list[Tensor] | None, + weight: Number | _complex, +) -> None: ... +@overload +def _foreach_lerp_( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensors1: tuple[Tensor, ...] | list[Tensor] | None, + weight: Sequence[Number | _complex], +) -> None: ... +@overload +def _foreach_lerp_( + self: tuple[Tensor, ...] | list[Tensor] | None, + tensors1: tuple[Tensor, ...] | list[Tensor] | None, + weights: tuple[Tensor, ...] | list[Tensor] | None, +) -> None: ... +def _foreach_lgamma( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_lgamma(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.lgamma` to each Tensor of the input list. + """ + +def _foreach_lgamma_( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> None: + r""" + _foreach_lgamma_(self: List[Tensor]) -> None + + Apply :func:`torch.lgamma` to each Tensor of the input list. + """ + +def _foreach_log( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_log(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.log` to each Tensor of the input list. + """ + +def _foreach_log10( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_log10(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.log10` to each Tensor of the input list. + """ + +def _foreach_log10_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_log10_(self: List[Tensor]) -> None + + Apply :func:`torch.log10` to each Tensor of the input list. + """ + +def _foreach_log1p( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_log1p(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.log1p` to each Tensor of the input list. + """ + +def _foreach_log1p_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_log1p_(self: List[Tensor]) -> None + + Apply :func:`torch.log1p` to each Tensor of the input list. + """ + +def _foreach_log2( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_log2(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.log2` to each Tensor of the input list. + """ + +def _foreach_log2_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_log2_(self: List[Tensor]) -> None + + Apply :func:`torch.log2` to each Tensor of the input list. + """ + +def _foreach_log_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_log_(self: List[Tensor]) -> None + + Apply :func:`torch.log` to each Tensor of the input list. + """ + +def _foreach_max( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_maximum( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_maximum( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_maximum( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_maximum_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> None: ... +@overload +def _foreach_maximum_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> None: ... +@overload +def _foreach_maximum_( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, +) -> None: ... +@overload +def _foreach_minimum( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_minimum( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_minimum( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_minimum_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> None: ... +@overload +def _foreach_minimum_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> None: ... +@overload +def _foreach_minimum_( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, +) -> None: ... +@overload +def _foreach_mul( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_mul( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: Tensor, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_mul( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_mul( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_mul_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> None: ... +@overload +def _foreach_mul_( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: Tensor, +) -> None: ... +@overload +def _foreach_mul_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> None: ... +@overload +def _foreach_mul_( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, +) -> None: ... +def _foreach_neg( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_neg(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.neg` to each Tensor of the input list. + """ + +def _foreach_neg_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_neg_(self: List[Tensor]) -> None + + Apply :func:`torch.neg` to each Tensor of the input list. + """ + +def _foreach_norm( + self: tuple[Tensor, ...] | list[Tensor] | None, + ord: Number | _complex = 2, + dtype: _dtype | None = None, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_pow( + self: tuple[Tensor, ...] | list[Tensor] | None, + exponent: Sequence[Number | _complex], +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_pow( + self: tuple[Tensor, ...] | list[Tensor] | None, + exponent: Number | _complex, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_pow( + self: tuple[Tensor, ...] | list[Tensor] | None, + exponent: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_pow( + self: Number | _complex, + exponent: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_pow_( + self: tuple[Tensor, ...] | list[Tensor] | None, + exponent: Sequence[Number | _complex], +) -> None: ... +@overload +def _foreach_pow_( + self: tuple[Tensor, ...] | list[Tensor] | None, + exponent: Number | _complex, +) -> None: ... +@overload +def _foreach_pow_( + self: tuple[Tensor, ...] | list[Tensor] | None, + exponent: tuple[Tensor, ...] | list[Tensor] | None, +) -> None: ... +def _foreach_reciprocal( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_reciprocal(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.reciprocal` to each Tensor of the input list. + """ + +def _foreach_reciprocal_( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> None: + r""" + _foreach_reciprocal_(self: List[Tensor]) -> None + + Apply :func:`torch.reciprocal` to each Tensor of the input list. + """ + +def _foreach_round( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_round(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.round` to each Tensor of the input list. + """ + +def _foreach_round_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_round_(self: List[Tensor]) -> None + + Apply :func:`torch.round` to each Tensor of the input list. + """ + +def _foreach_rsqrt( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: ... +def _foreach_rsqrt_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: ... +def _foreach_sigmoid( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_sigmoid(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.sigmoid` to each Tensor of the input list. + """ + +def _foreach_sigmoid_( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> None: + r""" + _foreach_sigmoid_(self: List[Tensor]) -> None + + Apply :func:`torch.sigmoid` to each Tensor of the input list. + """ + +def _foreach_sign( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: ... +def _foreach_sign_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: ... +def _foreach_sin( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_sin(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.sin` to each Tensor of the input list. + """ + +def _foreach_sin_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_sin_(self: List[Tensor]) -> None + + Apply :func:`torch.sin` to each Tensor of the input list. + """ + +def _foreach_sinh( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_sinh(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.sinh` to each Tensor of the input list. + """ + +def _foreach_sinh_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_sinh_(self: List[Tensor]) -> None + + Apply :func:`torch.sinh` to each Tensor of the input list. + """ + +def _foreach_sqrt( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_sqrt(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.sqrt` to each Tensor of the input list. + """ + +def _foreach_sqrt_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_sqrt_(self: List[Tensor]) -> None + + Apply :func:`torch.sqrt` to each Tensor of the input list. + """ + +@overload +def _foreach_sub( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_sub( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, + *, + alpha: Number | _complex = 1, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_sub( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> tuple[Tensor, ...]: ... +@overload +def _foreach_sub_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalars: Sequence[Number | _complex], +) -> None: ... +@overload +def _foreach_sub_( + self: tuple[Tensor, ...] | list[Tensor] | None, + other: tuple[Tensor, ...] | list[Tensor] | None, + *, + alpha: Number | _complex = 1, +) -> None: ... +@overload +def _foreach_sub_( + self: tuple[Tensor, ...] | list[Tensor] | None, + scalar: Number | _complex, +) -> None: ... +def _foreach_tan( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_tan(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.tan` to each Tensor of the input list. + """ + +def _foreach_tan_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_tan_(self: List[Tensor]) -> None + + Apply :func:`torch.tan` to each Tensor of the input list. + """ + +def _foreach_tanh( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_tanh(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.tanh` to each Tensor of the input list. + """ + +def _foreach_tanh_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_tanh_(self: List[Tensor]) -> None + + Apply :func:`torch.tanh` to each Tensor of the input list. + """ + +def _foreach_trunc( + self: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + _foreach_trunc(self: List[Tensor]) -> List[Tensor] + + Apply :func:`torch.trunc` to each Tensor of the input list. + """ + +def _foreach_trunc_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_trunc_(self: List[Tensor]) -> None + + Apply :func:`torch.trunc` to each Tensor of the input list. + """ + +def _foreach_zero_(self: tuple[Tensor, ...] | list[Tensor] | None) -> None: + r""" + _foreach_zero_(self: List[Tensor]) -> None + + Apply :func:`torch.zero` to each Tensor of the input list. + """ + +def _from_functional_tensor(t: Tensor) -> Tensor: ... +def _functional_assert_async( + input: Tensor, + assert_msg: str, + dep_token: Tensor, +) -> Tensor: ... +def _functional_assert_scalar( + self: Number | _complex, + assert_msg: str, + dep_token: Tensor, +) -> Tensor: ... +def _functional_sym_constrain_range( + size: Number | _complex, + min: _int | None, + max: _int | None, + dep_token: Tensor, +) -> Tensor: ... +def _functional_sym_constrain_range_for_size( + size: Number | _complex, + min: _int | None, + max: _int | None, + dep_token: Tensor, +) -> Tensor: ... +def _functionalize_apply_view_metas(tensor: Tensor, base: Tensor) -> Tensor: ... +def _functionalize_are_all_mutations_hidden_from_autograd( + t: Tensor, +) -> _bool: ... +def _functionalize_are_all_mutations_under_no_grad_or_inference_mode( + t: Tensor, +) -> _bool: ... +def _functionalize_commit_update(t: Tensor) -> None: ... +def _functionalize_has_metadata_mutation(tensor: Tensor) -> _bool: ... +def _functionalize_inductor_storage_resized_counter(t: Tensor) -> _int: ... +def _functionalize_is_symbolic(tensor: Tensor) -> _bool: ... +def _functionalize_mark_mutation_hidden_from_autograd(t: Tensor) -> None: ... +def _functionalize_mark_storage_changed(tensor: Tensor) -> _bool: ... +def _functionalize_mutation_counter(t: Tensor) -> _int: ... +def _functionalize_replace(self_: Tensor, other: Tensor) -> None: ... +def _functionalize_storage_changed_counter(t: Tensor) -> _int: ... +def _functionalize_sync(t: Tensor) -> None: ... +def _functionalize_unsafe_set(dst: Tensor, src: Tensor) -> None: ... +def _functionalize_was_inductor_storage_resized(t: Tensor) -> _bool: ... +def _functionalize_was_storage_changed(tensor: Tensor) -> _bool: ... +@overload +def _fused_adagrad_( + self: tuple[Tensor, ...] | list[Tensor] | None, + grads: tuple[Tensor, ...] | list[Tensor] | None, + state_sums: tuple[Tensor, ...] | list[Tensor] | None, + state_steps: tuple[Tensor, ...] | list[Tensor] | None, + *, + lr: Tensor, + lr_decay: _float, + weight_decay: _float, + eps: _float, + maximize: _bool, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, +) -> None: ... +@overload +def _fused_adagrad_( + self: tuple[Tensor, ...] | list[Tensor] | None, + grads: tuple[Tensor, ...] | list[Tensor] | None, + state_sums: tuple[Tensor, ...] | list[Tensor] | None, + state_steps: tuple[Tensor, ...] | list[Tensor] | None, + *, + lr: _float, + lr_decay: _float, + weight_decay: _float, + eps: _float, + maximize: _bool, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, +) -> None: ... +@overload +def _fused_adam_( + self: tuple[Tensor, ...] | list[Tensor] | None, + grads: tuple[Tensor, ...] | list[Tensor] | None, + exp_avgs: tuple[Tensor, ...] | list[Tensor] | None, + exp_avg_sqs: tuple[Tensor, ...] | list[Tensor] | None, + max_exp_avg_sqs: tuple[Tensor, ...] | list[Tensor] | None, + state_steps: tuple[Tensor, ...] | list[Tensor] | None, + *, + lr: Tensor, + beta1: _float, + beta2: _float, + weight_decay: _float, + eps: _float, + amsgrad: _bool, + maximize: _bool, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, +) -> None: ... +@overload +def _fused_adam_( + self: tuple[Tensor, ...] | list[Tensor] | None, + grads: tuple[Tensor, ...] | list[Tensor] | None, + exp_avgs: tuple[Tensor, ...] | list[Tensor] | None, + exp_avg_sqs: tuple[Tensor, ...] | list[Tensor] | None, + max_exp_avg_sqs: tuple[Tensor, ...] | list[Tensor] | None, + state_steps: tuple[Tensor, ...] | list[Tensor] | None, + *, + lr: _float, + beta1: _float, + beta2: _float, + weight_decay: _float, + eps: _float, + amsgrad: _bool, + maximize: _bool, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, +) -> None: ... +@overload +def _fused_adamw_( + self: tuple[Tensor, ...] | list[Tensor] | None, + grads: tuple[Tensor, ...] | list[Tensor] | None, + exp_avgs: tuple[Tensor, ...] | list[Tensor] | None, + exp_avg_sqs: tuple[Tensor, ...] | list[Tensor] | None, + max_exp_avg_sqs: tuple[Tensor, ...] | list[Tensor] | None, + state_steps: tuple[Tensor, ...] | list[Tensor] | None, + *, + lr: Tensor, + beta1: _float, + beta2: _float, + weight_decay: _float, + eps: _float, + amsgrad: _bool, + maximize: _bool, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, +) -> None: ... +@overload +def _fused_adamw_( + self: tuple[Tensor, ...] | list[Tensor] | None, + grads: tuple[Tensor, ...] | list[Tensor] | None, + exp_avgs: tuple[Tensor, ...] | list[Tensor] | None, + exp_avg_sqs: tuple[Tensor, ...] | list[Tensor] | None, + max_exp_avg_sqs: tuple[Tensor, ...] | list[Tensor] | None, + state_steps: tuple[Tensor, ...] | list[Tensor] | None, + *, + lr: _float, + beta1: _float, + beta2: _float, + weight_decay: _float, + eps: _float, + amsgrad: _bool, + maximize: _bool, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, +) -> None: ... +def _fused_dropout( + input: Tensor, + p: _float, + generator: Generator | None = None, +) -> tuple[Tensor, Tensor]: ... +def _fused_moving_avg_obs_fq_helper( + input: Tensor, + observer_on: Tensor, + fake_quant_on: Tensor, + running_min: Tensor, + running_max: Tensor, + scale: Tensor, + zero_point: Tensor, + averaging_const: _float, + quant_min: _int, + quant_max: _int, + ch_axis: _int, + per_row_fake_quant: _bool = False, + symmetric_quant: _bool = False, +) -> torch.return_types._fused_moving_avg_obs_fq_helper: ... +def _fused_rms_norm( + input: Tensor, + normalized_shape: _size, + weight: Tensor | None, + eps: _float | None, +) -> tuple[Tensor, Tensor]: ... +def _fused_sdp_choice( + query: Tensor, + key: Tensor, + value: Tensor, + attn_mask: Tensor | None = None, + dropout_p: _float = 0.0, + is_causal: _bool = False, + *, + scale: _float | None = None, + enable_gqa: _bool = False, +) -> _int: ... +@overload +def _fused_sgd_( + self: tuple[Tensor, ...] | list[Tensor] | None, + grads: tuple[Tensor, ...] | list[Tensor] | None, + momentum_buffer_list: tuple[Tensor, ...] | list[Tensor] | None, + *, + weight_decay: _float, + momentum: _float, + lr: Tensor, + dampening: _float, + nesterov: _bool, + maximize: _bool, + is_first_step: _bool, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, +) -> None: ... +@overload +def _fused_sgd_( + self: tuple[Tensor, ...] | list[Tensor] | None, + grads: tuple[Tensor, ...] | list[Tensor] | None, + momentum_buffer_list: tuple[Tensor, ...] | list[Tensor] | None, + *, + weight_decay: _float, + momentum: _float, + lr: _float, + dampening: _float, + nesterov: _bool, + maximize: _bool, + is_first_step: _bool, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, +) -> None: ... +def _fw_primal_copy( + input: Tensor, + level: _int, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _grid_sampler_2d_cpu_fallback( + input: Tensor, + grid: Tensor, + interpolation_mode: _int, + padding_mode: _int, + align_corners: _bool, +) -> Tensor: ... +def _grouped_mm( + input: Tensor, + mat2: Tensor, + offs: Tensor | None = None, + bias: Tensor | None = None, + out_dtype: _dtype | None = None, +) -> Tensor: ... +def _has_compatible_shallow_copy_type( + input: Tensor, + from_: Tensor, +) -> _bool: ... +def _histogramdd_bin_edges( + input: Tensor, + bins: _size, + *, + range: Sequence[_float] | None = None, + weight: Tensor | None = None, + density: _bool = False, +) -> tuple[Tensor, ...]: ... +def _histogramdd_from_bin_cts( + input: Tensor, + bins: _size, + *, + range: Sequence[_float] | None = None, + weight: Tensor | None = None, + density: _bool = False, +) -> Tensor: ... +def _histogramdd_from_bin_tensors( + input: Tensor, + bins: tuple[Tensor, ...] | list[Tensor] | None, + *, + weight: Tensor | None = None, + density: _bool = False, +) -> Tensor: ... +def _index_put_impl_( + input: Tensor, + indices: tuple[Tensor, ...] | list[Tensor] | None, + values: Tensor, + accumulate: _bool = False, + unsafe: _bool = False, +) -> Tensor: ... +def _indices_copy(input: Tensor, *, out: Tensor | None = None) -> Tensor: ... +def _int_mm( + input: Tensor, + mat2: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _is_all_true(input: Tensor) -> Tensor: ... +def _is_any_true(input: Tensor) -> Tensor: ... +def _is_functional_tensor(t: Tensor) -> _bool: ... +def _is_functional_tensor_base(t: Tensor) -> _bool: ... +def _is_zerotensor(input: Tensor) -> _bool: ... +def _lazy_clone(input: Tensor) -> Tensor: ... +def _linalg_check_errors( + info: Tensor, + api_name: str, + *, + is_matrix: _bool, +) -> None: ... +def _linalg_det( + A: Tensor, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types._linalg_det: ... +def _linalg_eigh( + A: Tensor, + UPLO: str = "L", + compute_v: _bool = True, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types._linalg_eigh: ... +def _linalg_slogdet( + A: Tensor, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types._linalg_slogdet: ... +def _linalg_solve_ex( + A: Tensor, + B: Tensor, + *, + left: _bool = True, + check_errors: _bool = False, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types._linalg_solve_ex: ... +def _linalg_svd( + A: Tensor, + full_matrices: _bool = False, + compute_uv: _bool = True, + *, + driver: str | None = None, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types._linalg_svd: ... +def _log_softmax( + input: Tensor, + dim: _int, + half_to_float: _bool, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _log_softmax_backward_data( + grad_output: Tensor, + output: Tensor, + dim: _int, + input_dtype: _dtype, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _logcumsumexp( + input: Tensor, + dim: _int, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _lstm_mps( + input: Tensor, + hx: tuple[Tensor, ...] | list[Tensor] | None, + params: tuple[Tensor, ...] | list[Tensor] | None, + has_biases: _bool, + num_layers: _int, + dropout: _float, + train: _bool, + bidirectional: _bool, + batch_first: _bool, +) -> tuple[Tensor, Tensor, Tensor, Tensor, Tensor, Tensor]: ... +def _lu_with_info( + input: Tensor, + pivot: _bool = True, + check_errors: _bool = True, +) -> torch.return_types._lu_with_info: ... +def _make_dep_token( + *, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: ... +def _make_dual(primal: Tensor, tangent: Tensor, level: _int) -> Tensor: ... +def _make_dual_copy( + primal: Tensor, + tangent: Tensor, + level: _int, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _make_per_channel_quantized_tensor( + input: Tensor, + scale: Tensor, + zero_point: Tensor, + axis: _int, +) -> Tensor: ... +def _make_per_tensor_quantized_tensor( + input: Tensor, + scale: _float, + zero_point: _int, +) -> Tensor: ... +def _masked_scale(input: Tensor, mask: Tensor, scale: _float) -> Tensor: ... +def _masked_softmax( + input: Tensor, + mask: Tensor, + dim: _int | None = None, + mask_type: _int | None = None, +) -> Tensor: ... +def _mixed_dtypes_linear( + input: Tensor, + weight: Tensor, + scale: Tensor, + *, + bias: Tensor | None = None, + activation: str | None = None, +) -> Tensor: ... +def _mkldnn_reshape(input: Tensor, shape: _size) -> Tensor: ... +def _mkldnn_transpose(input: Tensor, dim0: _int, dim1: _int) -> Tensor: ... +def _mkldnn_transpose_(input: Tensor, dim0: _int, dim1: _int) -> Tensor: ... +def _mps_convolution( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + padding: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + groups: _int | SymInt, +) -> Tensor: ... +def _mps_convolution_transpose( + input: Tensor, + weight: Tensor, + padding: Sequence[_int | SymInt], + output_padding: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + groups: _int | SymInt, +) -> Tensor: ... +@overload +def _native_batch_norm_legit( + input: Tensor, + weight: Tensor | None, + bias: Tensor | None, + running_mean: Tensor, + running_var: Tensor, + training: _bool, + momentum: _float, + eps: _float, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> tuple[Tensor, Tensor, Tensor]: ... +@overload +def _native_batch_norm_legit( + input: Tensor, + weight: Tensor | None, + bias: Tensor | None, + training: _bool, + momentum: _float, + eps: _float, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> tuple[Tensor, Tensor, Tensor]: ... +def _native_batch_norm_legit_no_training( + input: Tensor, + weight: Tensor | None, + bias: Tensor | None, + running_mean: Tensor, + running_var: Tensor, + momentum: _float, + eps: _float, +) -> tuple[Tensor, Tensor, Tensor]: ... +def _native_multi_head_attention( + query: Tensor, + key: Tensor, + value: Tensor, + embed_dim: _int, + num_head: _int, + qkv_weight: Tensor, + qkv_bias: Tensor, + proj_weight: Tensor, + proj_bias: Tensor, + mask: Tensor | None = None, + need_weights: _bool = True, + average_attn_weights: _bool = True, + mask_type: _int | None = None, +) -> tuple[Tensor, Tensor]: ... +def _neg_view(input: Tensor) -> Tensor: ... +def _neg_view_copy(input: Tensor, *, out: Tensor | None = None) -> Tensor: ... +def _nested_compute_contiguous_strides_offsets( + nested_size: Tensor, +) -> tuple[Tensor, Tensor]: ... +def _nested_from_padded( + padded: Tensor, + cpu_nested_shape_example: Tensor, + fuse_transform_0213: _bool = False, +) -> Tensor: ... +def _nested_from_padded_and_nested_example( + padded: Tensor, + nt_example: Tensor, +) -> Tensor: ... +def _nested_from_padded_tensor( + padded: Tensor, + offsets: Tensor, + dummy: Tensor, + ragged_idx: _int = 1, + min_seqlen: Tensor | None = None, + max_seqlen: Tensor | None = None, + sum_S: _int | SymInt | None = None, +) -> Tensor: ... +def _nested_get_jagged_dummy(any: Tensor) -> Tensor: ... +def _nested_get_lengths(input: Tensor) -> Tensor: ... +def _nested_get_max_seqlen(input: Tensor) -> Tensor: ... +def _nested_get_min_seqlen(input: Tensor) -> Tensor: ... +def _nested_get_offsets(input: Tensor) -> Tensor: ... +def _nested_get_ragged_idx(input: Tensor) -> _int: ... +def _nested_get_values(input: Tensor) -> Tensor: ... +def _nested_get_values_copy( + input: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _nested_tensor_from_mask( + t: Tensor, + mask: Tensor, + mask_check: _bool = True, +) -> Tensor: ... +def _nested_tensor_from_mask_left_aligned(t: Tensor, mask: Tensor) -> _bool: ... +def _nested_tensor_from_tensor_list( + list: tuple[Tensor, ...] | list[Tensor] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = None, +) -> Tensor: ... +def _nested_tensor_softmax_with_shape( + input: Tensor, + query: Tensor, +) -> Tensor: ... +def _nested_view_from_buffer( + input: Tensor, + nested_size: Tensor, + nested_strides: Tensor, + offsets: Tensor, +) -> Tensor: ... +def _nested_view_from_buffer_copy( + input: Tensor, + nested_size: Tensor, + nested_strides: Tensor, + offsets: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _nested_view_from_jagged( + input: Tensor, + offsets: Tensor, + dummy: Tensor, + lengths: Tensor | None = None, + ragged_idx: _int = 1, + min_seqlen: Tensor | None = None, + max_seqlen: Tensor | None = None, +) -> Tensor: ... +def _nested_view_from_jagged_copy( + input: Tensor, + offsets: Tensor, + dummy: Tensor, + lengths: Tensor | None = None, + ragged_idx: _int = 1, + min_seqlen: Tensor | None = None, + max_seqlen: Tensor | None = None, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _nnpack_available() -> _bool: ... +def _nnpack_spatial_convolution( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + padding: _int | SymInt | Sequence[_int | SymInt], + stride: _int | SymInt | Sequence[_int | SymInt] = 1, +) -> Tensor: ... +def _pack_padded_sequence( + input: Tensor, + lengths: Tensor, + batch_first: _bool, +) -> tuple[Tensor, Tensor]: ... +def _pad_packed_sequence( + data: Tensor, + batch_sizes: Tensor, + batch_first: _bool, + padding_value: Number | _complex, + total_length: _int, +) -> tuple[Tensor, Tensor]: ... +def _pin_memory( + input: Tensor, + device: DeviceLikeType | None = None, +) -> Tensor: ... +def _prelu_kernel(input: Tensor, weight: Tensor) -> Tensor: ... +def _print(s: str) -> None: ... +def _propagate_xla_data(input: Tensor, output: Tensor) -> None: ... +def _remove_batch_dim( + input: Tensor, + level: _int, + batch_size: _int | SymInt, + out_dim: _int, +) -> Tensor: ... +def _reshape_alias_copy( + input: Tensor, + size: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + *, + out: Tensor | None = None, +) -> Tensor: ... +def _reshape_from_tensor(input: Tensor, shape: Tensor) -> Tensor: ... +def _resize_output_( + input: Tensor, + size: Sequence[_int | SymInt], + device: DeviceLikeType | None, +) -> Tensor: ... +def _rowwise_prune( + weight: Tensor, + mask: Tensor, + compressed_indices_dtype: _dtype, +) -> tuple[Tensor, Tensor]: ... +def _safe_softmax( + input: Tensor, + dim: _int, + dtype: _dtype | None = None, +) -> Tensor: ... +def _sample_dirichlet( + input: Tensor, + generator: Generator | None = None, +) -> Tensor: ... +def _saturate_weight_to_fp16(weight: Tensor) -> Tensor: ... +def _scaled_dot_product_attention_math( + query: Tensor, + key: Tensor, + value: Tensor, + attn_mask: Tensor | None = None, + dropout_p: _float = 0.0, + is_causal: _bool = False, + dropout_mask: Tensor | None = None, + *, + scale: _float | None = None, + enable_gqa: _bool = False, +) -> tuple[Tensor, Tensor]: ... +def _scaled_dot_product_attention_math_for_mps( + query: Tensor, + key: Tensor, + value: Tensor, + attn_mask: Tensor | None = None, + dropout_p: _float = 0.0, + is_causal: _bool = False, + dropout_mask: Tensor | None = None, + *, + scale: _float | None = None, +) -> tuple[Tensor, Tensor]: ... +def _scaled_dot_product_cudnn_attention( + query: Tensor, + key: Tensor, + value: Tensor, + attn_bias: Tensor | None, + compute_log_sumexp: _bool, + dropout_p: _float = 0.0, + is_causal: _bool = False, + return_debug_mask: _bool = False, + *, + scale: _float | None = None, +) -> torch.return_types._scaled_dot_product_cudnn_attention: ... +def _scaled_dot_product_efficient_attention( + query: Tensor, + key: Tensor, + value: Tensor, + attn_bias: Tensor | None, + compute_log_sumexp: _bool, + dropout_p: _float = 0.0, + is_causal: _bool = False, + *, + scale: _float | None = None, +) -> torch.return_types._scaled_dot_product_efficient_attention: ... +def _scaled_dot_product_flash_attention( + query: Tensor, + key: Tensor, + value: Tensor, + dropout_p: _float = 0.0, + is_causal: _bool = False, + return_debug_mask: _bool = False, + *, + scale: _float | None = None, +) -> torch.return_types._scaled_dot_product_flash_attention: ... +def _scaled_dot_product_flash_attention_for_cpu( + query: Tensor, + key: Tensor, + value: Tensor, + dropout_p: _float = 0.0, + is_causal: _bool = False, + *, + attn_mask: Tensor | None = None, + scale: _float | None = None, +) -> torch.return_types._scaled_dot_product_flash_attention_for_cpu: ... +def _scaled_grouped_mm( + input: Tensor, + mat2: Tensor, + scale_a: Tensor, + scale_b: Tensor, + offs: Tensor | None = None, + bias: Tensor | None = None, + scale_result: Tensor | None = None, + out_dtype: _dtype | None = None, + use_fast_accum: _bool = False, +) -> Tensor: ... +def _scaled_grouped_mm_v2( + input: Tensor, + mat2: Tensor, + scale_a: tuple[Tensor, ...] | list[Tensor] | None, + recipe_a: _size, + swizzle_a: _size, + scale_b: tuple[Tensor, ...] | list[Tensor] | None, + recipe_b: _size, + swizzle_b: _size, + offs: Tensor | None = None, + bias: Tensor | None = None, + out_dtype: _dtype | None = None, + contraction_dim: _size = (), + use_fast_accum: _bool = False, +) -> Tensor: ... +def _scaled_mm( + input: Tensor, + mat2: Tensor, + scale_a: Tensor, + scale_b: Tensor, + bias: Tensor | None = None, + scale_result: Tensor | None = None, + out_dtype: _dtype | None = None, + use_fast_accum: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _scaled_mm_v2( + input: Tensor, + mat2: Tensor, + scale_a: tuple[Tensor, ...] | list[Tensor] | None, + recipe_a: _size, + swizzle_a: _size, + scale_b: tuple[Tensor, ...] | list[Tensor] | None, + recipe_b: _size, + swizzle_b: _size, + bias: Tensor | None, + out_dtype: _dtype | None, + contraction_dim: _size = (), + use_fast_accum: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _shape_as_tensor(input: Tensor) -> Tensor: ... +def _sobol_engine_draw( + quasi: Tensor, + n: _int, + sobolstate: Tensor, + dimension: _int, + num_generated: _int, + dtype: _dtype | None, +) -> tuple[Tensor, Tensor]: ... +def _sobol_engine_ff_( + input: Tensor, + n: _int, + sobolstate: Tensor, + dimension: _int, + num_generated: _int, +) -> Tensor: ... +def _sobol_engine_initialize_state_( + input: Tensor, + dimension: _int, +) -> Tensor: ... +def _sobol_engine_scramble_( + input: Tensor, + ltm: Tensor, + dimension: _int, +) -> Tensor: ... +def _softmax( + input: Tensor, + dim: _int, + half_to_float: _bool, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _softmax_backward_data( + grad_output: Tensor, + output: Tensor, + dim: _int, + input_dtype: _dtype, + *, + grad_input: Tensor | None = None, +) -> Tensor: ... +def _sparse_broadcast_to(input: Tensor, size: _size) -> Tensor: ... +def _sparse_broadcast_to_copy( + input: Tensor, + size: _size, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _sparse_csr_prod( + input: Tensor, + dim: _int | _size, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, +) -> Tensor: ... +def _sparse_csr_sum( + input: Tensor, + dim: _int | _size, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, +) -> Tensor: ... +def _sparse_log_softmax_backward_data( + grad_output: Tensor, + output: Tensor, + dim: _int, + input: Tensor, +) -> Tensor: ... +def _sparse_semi_structured_addmm( + input: Tensor, + mat1: Tensor, + mat1_meta: Tensor, + mat2: Tensor, + *, + alpha: Number | _complex = 1, + beta: Number | _complex = 1, + out_dtype: _dtype | None = None, +) -> Tensor: ... +def _sparse_semi_structured_apply( + input: Tensor, + thread_masks: Tensor, +) -> tuple[Tensor, Tensor]: ... +def _sparse_semi_structured_apply_dense( + input: Tensor, + thread_masks: Tensor, +) -> Tensor: ... +def _sparse_semi_structured_linear( + input: Tensor, + weight: Tensor, + meta: Tensor, + *, + bias: Tensor | None = None, + activation: str | None = None, + out_dtype: _dtype | None = None, +) -> Tensor: ... +def _sparse_semi_structured_mm( + mat1: Tensor, + mat1_meta: Tensor, + mat2: Tensor, + *, + out_dtype: _dtype | None = None, +) -> Tensor: ... +def _sparse_semi_structured_tile( + input: Tensor, + algorithm: str = "", + use_cutlass: _bool = True, +) -> tuple[Tensor, Tensor, Tensor, Tensor, Tensor]: ... +def _sparse_softmax_backward_data( + grad_output: Tensor, + output: Tensor, + dim: _int, + input: Tensor, +) -> Tensor: ... +def _sparse_sparse_matmul(input: Tensor, other: Tensor) -> Tensor: ... +@overload +def _sparse_sum(input: Tensor) -> Tensor: ... +@overload +def _sparse_sum(input: Tensor, *, dtype: _dtype) -> Tensor: ... +@overload +def _sparse_sum(input: Tensor, dim: _int | _size) -> Tensor: ... +@overload +def _sparse_sum( + input: Tensor, + dim: _int | _size, + *, + dtype: _dtype, +) -> Tensor: ... +def _stack( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + dim: _int = 0, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _standard_gamma( + input: Tensor, + generator: Generator | None = None, +) -> Tensor: ... +def _standard_gamma_grad(input: Tensor, output: Tensor) -> Tensor: ... +def _sync(t: Tensor) -> None: ... +@overload +def _test_autograd_multiple_dispatch(input: Tensor) -> Tensor: ... +@overload +def _test_autograd_multiple_dispatch(input: Tensor, b: _bool) -> Tensor: ... +def _test_autograd_multiple_dispatch_view(input: Tensor) -> Tensor: ... +def _test_autograd_multiple_dispatch_view_copy( + input: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: ... +def _test_check_tensor(input: Tensor) -> Tensor: ... +def _test_functorch_fallback(input: Tensor, other: Tensor) -> Tensor: ... +def _test_parallel_materialize( + input: Tensor, + num_parallel: _int, + skip_first: _bool = False, +) -> Tensor: ... +def _test_serialization_subcmul( + input: Tensor, + other: Tensor, + alpha: Number | _complex = 1, +) -> Tensor: ... +def _to_cpu( + tensors: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: ... +def _to_functional_tensor(t: Tensor) -> Tensor: ... +def _to_sparse_semi_structured(dense: Tensor) -> tuple[Tensor, Tensor]: ... +def _transform_bias_rescale_qkv( + qkv: Tensor, + qkv_bias: Tensor, + num_heads: _int, +) -> tuple[Tensor, Tensor, Tensor]: ... +def _transformer_encoder_layer_fwd( + src: Tensor, + embed_dim: _int, + num_heads: _int, + qkv_weight: Tensor, + qkv_bias: Tensor, + proj_weight: Tensor, + proj_bias: Tensor, + use_gelu: _bool, + norm_first: _bool, + eps: _float, + norm_weight_1: Tensor, + norm_bias_1: Tensor, + norm_weight_2: Tensor, + norm_bias_2: Tensor, + ffn_weight_1: Tensor, + ffn_bias_1: Tensor, + ffn_weight_2: Tensor, + ffn_bias_2: Tensor, + mask: Tensor | None = None, + mask_type: _int | None = None, +) -> Tensor: ... +def _trilinear( + i1: Tensor, + i2: Tensor, + i3: Tensor, + expand1: _size, + expand2: _size, + expand3: _size, + sumdim: _size, + unroll_dim: _int = 1, +) -> Tensor: ... +def _triton_multi_head_attention( + query: Tensor, + key: Tensor, + value: Tensor, + embed_dim: _int, + num_head: _int, + qkv_weight: Tensor, + qkv_bias: Tensor, + proj_weight: Tensor, + proj_bias: Tensor, + mask: Tensor | None = None, +) -> Tensor: ... +def _triton_scaled_dot_attention( + q: Tensor, + k: Tensor, + v: Tensor, + dropout_p: _float = 0.0, +) -> Tensor: ... +def _unique( + input: Tensor, + sorted: _bool = True, + return_inverse: _bool = False, +) -> tuple[Tensor, Tensor]: ... +def _unique2( + input: Tensor, + sorted: _bool = True, + return_inverse: _bool = False, + return_counts: _bool = False, +) -> tuple[Tensor, Tensor, Tensor]: ... +def _unpack_dual( + dual: Tensor, + level: _int, +) -> torch.return_types._unpack_dual: ... +def _unsafe_index( + input: Tensor, + indices: tuple[Tensor, ...] | list[Tensor] | None, +) -> Tensor: ... +def _unsafe_index_put( + input: Tensor, + indices: tuple[Tensor, ...] | list[Tensor] | None, + values: Tensor, + accumulate: _bool = False, +) -> Tensor: ... +def _unsafe_masked_index( + input: Tensor, + mask: Tensor, + indices: tuple[Tensor, ...] | list[Tensor] | None, + fill: Number | _complex, +) -> Tensor: ... +def _unsafe_masked_index_put_accumulate( + input: Tensor, + mask: Tensor, + indices: tuple[Tensor, ...] | list[Tensor] | None, + values: Tensor, +) -> Tensor: ... +@overload +def _use_cudnn_ctc_loss( + log_probs: Tensor, + targets: Tensor, + input_lengths: Tensor, + target_lengths: Tensor, + blank: _int, +) -> _bool: ... +@overload +def _use_cudnn_ctc_loss( + log_probs: Tensor, + targets: Tensor, + input_lengths: _size, + target_lengths: _size, + blank: _int, +) -> _bool: ... +def _use_cudnn_rnn_flatten_weight() -> _bool: ... +def _validate_compressed_sparse_indices( + is_crow: _bool, + compressed_idx: Tensor, + plain_idx: Tensor, + cdim: _int, + dim: _int, + nnz: _int, +) -> None: ... +def _validate_sparse_bsc_tensor_args( + ccol_indices: Tensor, + row_indices: Tensor, + values: Tensor, + size: _size, + check_pinning: _bool | None = None, +) -> None: ... +def _validate_sparse_bsr_tensor_args( + crow_indices: Tensor, + col_indices: Tensor, + values: Tensor, + size: _size, + check_pinning: _bool | None = None, +) -> None: ... +def _validate_sparse_compressed_tensor_args( + compressed_indices: Tensor, + plain_indices: Tensor, + values: Tensor, + size: _size, + layout: _layout, + check_pinning: _bool | None = None, +) -> None: ... +def _validate_sparse_coo_tensor_args( + indices: Tensor, + values: Tensor, + size: _size, + is_coalesced: _bool | None = None, + check_pinning: _bool | None = None, +) -> None: ... +def _validate_sparse_csc_tensor_args( + ccol_indices: Tensor, + row_indices: Tensor, + values: Tensor, + size: _size, + check_pinning: _bool | None = None, +) -> None: ... +def _validate_sparse_csr_tensor_args( + crow_indices: Tensor, + col_indices: Tensor, + values: Tensor, + size: _size, + check_pinning: _bool | None = None, +) -> None: ... +def _values_copy(input: Tensor, *, out: Tensor | None = None) -> Tensor: ... +def _weight_int4pack_mm( + input: Tensor, + mat2: Tensor, + qGroupSize: _int, + qScaleAndZeros: Tensor, +) -> Tensor: ... +def _weight_int4pack_mm_for_cpu( + input: Tensor, + mat2: Tensor, + qGroupSize: _int, + qScaleAndZeros: Tensor, +) -> Tensor: ... +def _weight_int4pack_mm_with_scales_and_zeros( + input: Tensor, + mat2: Tensor, + qGroupSize: _int, + qScale: Tensor, + qZeros: Tensor, +) -> Tensor: ... +def _weight_int8pack_mm( + input: Tensor, + mat2: Tensor, + scales: Tensor, +) -> Tensor: ... +def _weight_norm(v: Tensor, g: Tensor, dim: _int = 0) -> Tensor: ... +def _weight_norm_interface( + v: Tensor, + g: Tensor, + dim: _int = 0, +) -> tuple[Tensor, Tensor]: ... +def _wrapped_linear_prepack( + weight: Tensor, + weight_scale: Tensor, + weight_zero_point: Tensor, + bias: Tensor, +) -> Tensor: ... +def _wrapped_quantized_linear_prepacked( + input: Tensor, + input_scale: Tensor, + input_zero_point: Tensor, + packed_weight: Tensor, + output_scale: Tensor, + output_zero_point: Tensor, + out_channel: _int, +) -> Tensor: ... +def abs(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + abs(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Computes the absolute value of each element in :attr:`input`. + + .. math:: + \text{out}_{i} = |\text{input}_{i}| + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.abs(torch.tensor([-1, -2, 3])) + tensor([ 1, 2, 3]) + """ + +def abs_(input: Tensor) -> Tensor: ... +def absolute(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + absolute(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Alias for :func:`torch.abs` + """ + +def acos(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + acos(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Returns a new tensor with the arccosine (in radians) of each element in :attr:`input`. + + .. math:: + \text{out}_{i} = \cos^{-1}(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([ 0.3348, -0.5889, 0.2005, -0.1584]) + >>> torch.acos(a) + tensor([ 1.2294, 2.2004, 1.3690, 1.7298]) + """ + +def acos_(input: Tensor) -> Tensor: ... +def acosh(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + acosh(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Returns a new tensor with the inverse hyperbolic cosine of the elements of :attr:`input`. + + .. math:: + \text{out}_{i} = \cosh^{-1}(\text{input}_{i}) + + Note: + The domain of the inverse hyperbolic cosine is `[1, inf)` and values outside this range + will be mapped to ``NaN``, except for `+ INF` for which the output is mapped to `+ INF`. + + Args: + input (Tensor): the input tensor. + + Keyword arguments: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4).uniform_(1, 2) + >>> a + tensor([ 1.3192, 1.9915, 1.9674, 1.7151 ]) + >>> torch.acosh(a) + tensor([ 0.7791, 1.3120, 1.2979, 1.1341 ]) + """ + +def acosh_(input: Tensor) -> Tensor: ... +def adaptive_avg_pool1d(input: Tensor, output_size: _int | _size) -> Tensor: ... +def adaptive_max_pool1d( + input: Tensor, + output_size: _int | _size, +) -> tuple[Tensor, Tensor]: ... +@overload +def add( + input: Tensor | Number | _complex, + other: Tensor | Number | _complex, + *, + alpha: Number | _complex | None = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + add(input, other, *, alpha=1, out=None) -> Tensor + + Adds :attr:`other`, scaled by :attr:`alpha`, to :attr:`input`. + + .. math:: + \text{{out}}_i = \text{{input}}_i + \text{{alpha}} \times \text{{other}}_i + + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer, float, and complex inputs. + + Args: + input (Tensor): the input tensor. + other (Tensor or Number): the tensor or number to add to :attr:`input`. + + Keyword arguments: + alpha (Number): the multiplier for :attr:`other`. + out (Tensor, optional): the output tensor. + + Examples:: + + >>> a = torch.randn(4) + >>> a + tensor([ 0.0202, 1.0985, 1.3506, -0.6056]) + >>> torch.add(a, 20) + tensor([ 20.0202, 21.0985, 21.3506, 19.3944]) + + >>> b = torch.randn(4) + >>> b + tensor([-0.9732, -0.3497, 0.6245, 0.4022]) + >>> c = torch.randn(4, 1) + >>> c + tensor([[ 0.3743], + [-1.7724], + [-0.5811], + [-0.8017]]) + >>> torch.add(b, c, alpha=10) + tensor([[ 2.7695, 3.3930, 4.3672, 4.1450], + [-18.6971, -18.0736, -17.0994, -17.3216], + [ -6.7845, -6.1610, -5.1868, -5.4090], + [ -8.9902, -8.3667, -7.3925, -7.6147]]) + """ + +@overload +def add(self: Tensor, alpha: Number | _complex, other: Tensor) -> Tensor: + r""" + add(input, other, *, alpha=1, out=None) -> Tensor + + Adds :attr:`other`, scaled by :attr:`alpha`, to :attr:`input`. + + .. math:: + \text{{out}}_i = \text{{input}}_i + \text{{alpha}} \times \text{{other}}_i + + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer, float, and complex inputs. + + Args: + input (Tensor): the input tensor. + other (Tensor or Number): the tensor or number to add to :attr:`input`. + + Keyword arguments: + alpha (Number): the multiplier for :attr:`other`. + out (Tensor, optional): the output tensor. + + Examples:: + + >>> a = torch.randn(4) + >>> a + tensor([ 0.0202, 1.0985, 1.3506, -0.6056]) + >>> torch.add(a, 20) + tensor([ 20.0202, 21.0985, 21.3506, 19.3944]) + + >>> b = torch.randn(4) + >>> b + tensor([-0.9732, -0.3497, 0.6245, 0.4022]) + >>> c = torch.randn(4, 1) + >>> c + tensor([[ 0.3743], + [-1.7724], + [-0.5811], + [-0.8017]]) + >>> torch.add(b, c, alpha=10) + tensor([[ 2.7695, 3.3930, 4.3672, 4.1450], + [-18.6971, -18.0736, -17.0994, -17.3216], + [ -6.7845, -6.1610, -5.1868, -5.4090], + [ -8.9902, -8.3667, -7.3925, -7.6147]]) + """ + +@overload +def add( + self: Tensor, + alpha: Number | _complex, + other: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + add(input, other, *, alpha=1, out=None) -> Tensor + + Adds :attr:`other`, scaled by :attr:`alpha`, to :attr:`input`. + + .. math:: + \text{{out}}_i = \text{{input}}_i + \text{{alpha}} \times \text{{other}}_i + + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer, float, and complex inputs. + + Args: + input (Tensor): the input tensor. + other (Tensor or Number): the tensor or number to add to :attr:`input`. + + Keyword arguments: + alpha (Number): the multiplier for :attr:`other`. + out (Tensor, optional): the output tensor. + + Examples:: + + >>> a = torch.randn(4) + >>> a + tensor([ 0.0202, 1.0985, 1.3506, -0.6056]) + >>> torch.add(a, 20) + tensor([ 20.0202, 21.0985, 21.3506, 19.3944]) + + >>> b = torch.randn(4) + >>> b + tensor([-0.9732, -0.3497, 0.6245, 0.4022]) + >>> c = torch.randn(4, 1) + >>> c + tensor([[ 0.3743], + [-1.7724], + [-0.5811], + [-0.8017]]) + >>> torch.add(b, c, alpha=10) + tensor([[ 2.7695, 3.3930, 4.3672, 4.1450], + [-18.6971, -18.0736, -17.0994, -17.3216], + [ -6.7845, -6.1610, -5.1868, -5.4090], + [ -8.9902, -8.3667, -7.3925, -7.6147]]) + """ + +@overload +def addbmm( + beta: Number | _complex, + self: Tensor, + alpha: Number | _complex, + batch1: Tensor, + batch2: Tensor, +) -> Tensor: + r""" + addbmm(input, batch1, batch2, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a batch matrix-matrix product of matrices stored + in :attr:`batch1` and :attr:`batch2`, + with a reduced add step (all matrix multiplications get accumulated + along the first dimension). + :attr:`input` is added to the final result. + + :attr:`batch1` and :attr:`batch2` must be 3-D tensors each containing the + same number of matrices. + + If :attr:`batch1` is a :math:`(b \times n \times m)` tensor, :attr:`batch2` is a + :math:`(b \times m \times p)` tensor, :attr:`input` must be + :ref:`broadcastable ` with a :math:`(n \times p)` tensor + and :attr:`out` will be a :math:`(n \times p)` tensor. + + .. math:: + out = \beta\ \text{input} + \alpha\ (\sum_{i=0}^{b-1} \text{batch1}_i \mathbin{@} \text{batch2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and :attr:`alpha` + must be real numbers, otherwise they should be integers. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): matrix to be added + batch1 (Tensor): the first batch of matrices to be multiplied + batch2 (Tensor): the second batch of matrices to be multiplied + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for `batch1 @ batch2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(3, 5) + >>> batch1 = torch.randn(10, 3, 4) + >>> batch2 = torch.randn(10, 4, 5) + >>> torch.addbmm(M, batch1, batch2) + tensor([[ 6.6311, 0.0503, 6.9768, -12.0362, -2.1653], + [ -4.8185, -1.4255, -6.6760, 8.9453, 2.5743], + [ -3.8202, 4.3691, 1.0943, -1.1109, 5.4730]]) + """ + +@overload +def addbmm( + beta: Number | _complex, + self: Tensor, + alpha: Number | _complex, + batch1: Tensor, + batch2: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + addbmm(input, batch1, batch2, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a batch matrix-matrix product of matrices stored + in :attr:`batch1` and :attr:`batch2`, + with a reduced add step (all matrix multiplications get accumulated + along the first dimension). + :attr:`input` is added to the final result. + + :attr:`batch1` and :attr:`batch2` must be 3-D tensors each containing the + same number of matrices. + + If :attr:`batch1` is a :math:`(b \times n \times m)` tensor, :attr:`batch2` is a + :math:`(b \times m \times p)` tensor, :attr:`input` must be + :ref:`broadcastable ` with a :math:`(n \times p)` tensor + and :attr:`out` will be a :math:`(n \times p)` tensor. + + .. math:: + out = \beta\ \text{input} + \alpha\ (\sum_{i=0}^{b-1} \text{batch1}_i \mathbin{@} \text{batch2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and :attr:`alpha` + must be real numbers, otherwise they should be integers. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): matrix to be added + batch1 (Tensor): the first batch of matrices to be multiplied + batch2 (Tensor): the second batch of matrices to be multiplied + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for `batch1 @ batch2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(3, 5) + >>> batch1 = torch.randn(10, 3, 4) + >>> batch2 = torch.randn(10, 4, 5) + >>> torch.addbmm(M, batch1, batch2) + tensor([[ 6.6311, 0.0503, 6.9768, -12.0362, -2.1653], + [ -4.8185, -1.4255, -6.6760, 8.9453, 2.5743], + [ -3.8202, 4.3691, 1.0943, -1.1109, 5.4730]]) + """ + +@overload +def addbmm( + input: Tensor, + batch1: Tensor, + batch2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + addbmm(input, batch1, batch2, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a batch matrix-matrix product of matrices stored + in :attr:`batch1` and :attr:`batch2`, + with a reduced add step (all matrix multiplications get accumulated + along the first dimension). + :attr:`input` is added to the final result. + + :attr:`batch1` and :attr:`batch2` must be 3-D tensors each containing the + same number of matrices. + + If :attr:`batch1` is a :math:`(b \times n \times m)` tensor, :attr:`batch2` is a + :math:`(b \times m \times p)` tensor, :attr:`input` must be + :ref:`broadcastable ` with a :math:`(n \times p)` tensor + and :attr:`out` will be a :math:`(n \times p)` tensor. + + .. math:: + out = \beta\ \text{input} + \alpha\ (\sum_{i=0}^{b-1} \text{batch1}_i \mathbin{@} \text{batch2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and :attr:`alpha` + must be real numbers, otherwise they should be integers. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): matrix to be added + batch1 (Tensor): the first batch of matrices to be multiplied + batch2 (Tensor): the second batch of matrices to be multiplied + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for `batch1 @ batch2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(3, 5) + >>> batch1 = torch.randn(10, 3, 4) + >>> batch2 = torch.randn(10, 4, 5) + >>> torch.addbmm(M, batch1, batch2) + tensor([[ 6.6311, 0.0503, 6.9768, -12.0362, -2.1653], + [ -4.8185, -1.4255, -6.6760, 8.9453, 2.5743], + [ -3.8202, 4.3691, 1.0943, -1.1109, 5.4730]]) + """ + +@overload +def addbmm( + beta: Number | _complex, + self: Tensor, + batch1: Tensor, + batch2: Tensor, +) -> Tensor: + r""" + addbmm(input, batch1, batch2, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a batch matrix-matrix product of matrices stored + in :attr:`batch1` and :attr:`batch2`, + with a reduced add step (all matrix multiplications get accumulated + along the first dimension). + :attr:`input` is added to the final result. + + :attr:`batch1` and :attr:`batch2` must be 3-D tensors each containing the + same number of matrices. + + If :attr:`batch1` is a :math:`(b \times n \times m)` tensor, :attr:`batch2` is a + :math:`(b \times m \times p)` tensor, :attr:`input` must be + :ref:`broadcastable ` with a :math:`(n \times p)` tensor + and :attr:`out` will be a :math:`(n \times p)` tensor. + + .. math:: + out = \beta\ \text{input} + \alpha\ (\sum_{i=0}^{b-1} \text{batch1}_i \mathbin{@} \text{batch2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and :attr:`alpha` + must be real numbers, otherwise they should be integers. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): matrix to be added + batch1 (Tensor): the first batch of matrices to be multiplied + batch2 (Tensor): the second batch of matrices to be multiplied + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for `batch1 @ batch2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(3, 5) + >>> batch1 = torch.randn(10, 3, 4) + >>> batch2 = torch.randn(10, 4, 5) + >>> torch.addbmm(M, batch1, batch2) + tensor([[ 6.6311, 0.0503, 6.9768, -12.0362, -2.1653], + [ -4.8185, -1.4255, -6.6760, 8.9453, 2.5743], + [ -3.8202, 4.3691, 1.0943, -1.1109, 5.4730]]) + """ + +@overload +def addbmm( + beta: Number | _complex, + self: Tensor, + batch1: Tensor, + batch2: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + addbmm(input, batch1, batch2, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a batch matrix-matrix product of matrices stored + in :attr:`batch1` and :attr:`batch2`, + with a reduced add step (all matrix multiplications get accumulated + along the first dimension). + :attr:`input` is added to the final result. + + :attr:`batch1` and :attr:`batch2` must be 3-D tensors each containing the + same number of matrices. + + If :attr:`batch1` is a :math:`(b \times n \times m)` tensor, :attr:`batch2` is a + :math:`(b \times m \times p)` tensor, :attr:`input` must be + :ref:`broadcastable ` with a :math:`(n \times p)` tensor + and :attr:`out` will be a :math:`(n \times p)` tensor. + + .. math:: + out = \beta\ \text{input} + \alpha\ (\sum_{i=0}^{b-1} \text{batch1}_i \mathbin{@} \text{batch2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and :attr:`alpha` + must be real numbers, otherwise they should be integers. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): matrix to be added + batch1 (Tensor): the first batch of matrices to be multiplied + batch2 (Tensor): the second batch of matrices to be multiplied + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for `batch1 @ batch2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(3, 5) + >>> batch1 = torch.randn(10, 3, 4) + >>> batch2 = torch.randn(10, 4, 5) + >>> torch.addbmm(M, batch1, batch2) + tensor([[ 6.6311, 0.0503, 6.9768, -12.0362, -2.1653], + [ -4.8185, -1.4255, -6.6760, 8.9453, 2.5743], + [ -3.8202, 4.3691, 1.0943, -1.1109, 5.4730]]) + """ + +@overload +def addcdiv( + self: Tensor, + value: Number | _complex, + tensor1: Tensor, + tensor2: Tensor, +) -> Tensor: + r""" + addcdiv(input, tensor1, tensor2, *, value=1, out=None) -> Tensor + + Performs the element-wise division of :attr:`tensor1` by :attr:`tensor2`, + multiplies the result by the scalar :attr:`value` and adds it to :attr:`input`. + + .. warning:: + Integer division with addcdiv is no longer supported, and in a future + release addcdiv will perform a true division of tensor1 and tensor2. + The historic addcdiv behavior can be implemented as + (input + value * torch.trunc(tensor1 / tensor2)).to(input.dtype) + for integer inputs and as (input + value * tensor1 / tensor2) for float inputs. + The future addcdiv behavior is just the latter implementation: + (input + value * tensor1 / tensor2), for all dtypes. + + .. math:: + \text{out}_i = \text{input}_i + \text{value} \times \frac{\text{tensor1}_i}{\text{tensor2}_i} + + + The shapes of :attr:`input`, :attr:`tensor1`, and :attr:`tensor2` must be + :ref:`broadcastable `. + + For inputs of type `FloatTensor` or `DoubleTensor`, :attr:`value` must be + a real number, otherwise an integer. + + Args: + input (Tensor): the tensor to be added + tensor1 (Tensor): the numerator tensor + tensor2 (Tensor): the denominator tensor + + Keyword args: + value (Number, optional): multiplier for :math:`\text{tensor1} / \text{tensor2}` + out (Tensor, optional): the output tensor. + + Example:: + + >>> t = torch.randn(1, 3) + >>> t1 = torch.randn(3, 1) + >>> t2 = torch.randn(1, 3) + >>> torch.addcdiv(t, t1, t2, value=0.1) + tensor([[-0.2312, -3.6496, 0.1312], + [-1.0428, 3.4292, -0.1030], + [-0.5369, -0.9829, 0.0430]]) + """ + +@overload +def addcdiv( + self: Tensor, + value: Number | _complex, + tensor1: Tensor, + tensor2: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + addcdiv(input, tensor1, tensor2, *, value=1, out=None) -> Tensor + + Performs the element-wise division of :attr:`tensor1` by :attr:`tensor2`, + multiplies the result by the scalar :attr:`value` and adds it to :attr:`input`. + + .. warning:: + Integer division with addcdiv is no longer supported, and in a future + release addcdiv will perform a true division of tensor1 and tensor2. + The historic addcdiv behavior can be implemented as + (input + value * torch.trunc(tensor1 / tensor2)).to(input.dtype) + for integer inputs and as (input + value * tensor1 / tensor2) for float inputs. + The future addcdiv behavior is just the latter implementation: + (input + value * tensor1 / tensor2), for all dtypes. + + .. math:: + \text{out}_i = \text{input}_i + \text{value} \times \frac{\text{tensor1}_i}{\text{tensor2}_i} + + + The shapes of :attr:`input`, :attr:`tensor1`, and :attr:`tensor2` must be + :ref:`broadcastable `. + + For inputs of type `FloatTensor` or `DoubleTensor`, :attr:`value` must be + a real number, otherwise an integer. + + Args: + input (Tensor): the tensor to be added + tensor1 (Tensor): the numerator tensor + tensor2 (Tensor): the denominator tensor + + Keyword args: + value (Number, optional): multiplier for :math:`\text{tensor1} / \text{tensor2}` + out (Tensor, optional): the output tensor. + + Example:: + + >>> t = torch.randn(1, 3) + >>> t1 = torch.randn(3, 1) + >>> t2 = torch.randn(1, 3) + >>> torch.addcdiv(t, t1, t2, value=0.1) + tensor([[-0.2312, -3.6496, 0.1312], + [-1.0428, 3.4292, -0.1030], + [-0.5369, -0.9829, 0.0430]]) + """ + +@overload +def addcdiv( + input: Tensor, + tensor1: Tensor, + tensor2: Tensor, + *, + value: Number | _complex = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + addcdiv(input, tensor1, tensor2, *, value=1, out=None) -> Tensor + + Performs the element-wise division of :attr:`tensor1` by :attr:`tensor2`, + multiplies the result by the scalar :attr:`value` and adds it to :attr:`input`. + + .. warning:: + Integer division with addcdiv is no longer supported, and in a future + release addcdiv will perform a true division of tensor1 and tensor2. + The historic addcdiv behavior can be implemented as + (input + value * torch.trunc(tensor1 / tensor2)).to(input.dtype) + for integer inputs and as (input + value * tensor1 / tensor2) for float inputs. + The future addcdiv behavior is just the latter implementation: + (input + value * tensor1 / tensor2), for all dtypes. + + .. math:: + \text{out}_i = \text{input}_i + \text{value} \times \frac{\text{tensor1}_i}{\text{tensor2}_i} + + + The shapes of :attr:`input`, :attr:`tensor1`, and :attr:`tensor2` must be + :ref:`broadcastable `. + + For inputs of type `FloatTensor` or `DoubleTensor`, :attr:`value` must be + a real number, otherwise an integer. + + Args: + input (Tensor): the tensor to be added + tensor1 (Tensor): the numerator tensor + tensor2 (Tensor): the denominator tensor + + Keyword args: + value (Number, optional): multiplier for :math:`\text{tensor1} / \text{tensor2}` + out (Tensor, optional): the output tensor. + + Example:: + + >>> t = torch.randn(1, 3) + >>> t1 = torch.randn(3, 1) + >>> t2 = torch.randn(1, 3) + >>> torch.addcdiv(t, t1, t2, value=0.1) + tensor([[-0.2312, -3.6496, 0.1312], + [-1.0428, 3.4292, -0.1030], + [-0.5369, -0.9829, 0.0430]]) + """ + +@overload +def addcmul( + self: Tensor, + value: Number | _complex, + tensor1: Tensor, + tensor2: Tensor, +) -> Tensor: + r""" + addcmul(input, tensor1, tensor2, *, value=1, out=None) -> Tensor + + Performs the element-wise multiplication of :attr:`tensor1` + by :attr:`tensor2`, multiplies the result by the scalar :attr:`value` + and adds it to :attr:`input`. + + .. math:: + \text{out}_i = \text{input}_i + \text{value} \times \text{tensor1}_i \times \text{tensor2}_i + + The shapes of :attr:`tensor`, :attr:`tensor1`, and :attr:`tensor2` must be + :ref:`broadcastable `. + + For inputs of type `FloatTensor` or `DoubleTensor`, :attr:`value` must be + a real number, otherwise an integer. + + Args: + input (Tensor): the tensor to be added + tensor1 (Tensor): the tensor to be multiplied + tensor2 (Tensor): the tensor to be multiplied + + Keyword args: + value (Number, optional): multiplier for :math:`tensor1 .* tensor2` + out (Tensor, optional): the output tensor. + + Example:: + + >>> t = torch.randn(1, 3) + >>> t1 = torch.randn(3, 1) + >>> t2 = torch.randn(1, 3) + >>> torch.addcmul(t, t1, t2, value=0.1) + tensor([[-0.8635, -0.6391, 1.6174], + [-0.7617, -0.5879, 1.7388], + [-0.8353, -0.6249, 1.6511]]) + """ + +@overload +def addcmul( + self: Tensor, + value: Number | _complex, + tensor1: Tensor, + tensor2: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + addcmul(input, tensor1, tensor2, *, value=1, out=None) -> Tensor + + Performs the element-wise multiplication of :attr:`tensor1` + by :attr:`tensor2`, multiplies the result by the scalar :attr:`value` + and adds it to :attr:`input`. + + .. math:: + \text{out}_i = \text{input}_i + \text{value} \times \text{tensor1}_i \times \text{tensor2}_i + + The shapes of :attr:`tensor`, :attr:`tensor1`, and :attr:`tensor2` must be + :ref:`broadcastable `. + + For inputs of type `FloatTensor` or `DoubleTensor`, :attr:`value` must be + a real number, otherwise an integer. + + Args: + input (Tensor): the tensor to be added + tensor1 (Tensor): the tensor to be multiplied + tensor2 (Tensor): the tensor to be multiplied + + Keyword args: + value (Number, optional): multiplier for :math:`tensor1 .* tensor2` + out (Tensor, optional): the output tensor. + + Example:: + + >>> t = torch.randn(1, 3) + >>> t1 = torch.randn(3, 1) + >>> t2 = torch.randn(1, 3) + >>> torch.addcmul(t, t1, t2, value=0.1) + tensor([[-0.8635, -0.6391, 1.6174], + [-0.7617, -0.5879, 1.7388], + [-0.8353, -0.6249, 1.6511]]) + """ + +@overload +def addcmul( + input: Tensor, + tensor1: Tensor, + tensor2: Tensor, + *, + value: Number | _complex = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + addcmul(input, tensor1, tensor2, *, value=1, out=None) -> Tensor + + Performs the element-wise multiplication of :attr:`tensor1` + by :attr:`tensor2`, multiplies the result by the scalar :attr:`value` + and adds it to :attr:`input`. + + .. math:: + \text{out}_i = \text{input}_i + \text{value} \times \text{tensor1}_i \times \text{tensor2}_i + + The shapes of :attr:`tensor`, :attr:`tensor1`, and :attr:`tensor2` must be + :ref:`broadcastable `. + + For inputs of type `FloatTensor` or `DoubleTensor`, :attr:`value` must be + a real number, otherwise an integer. + + Args: + input (Tensor): the tensor to be added + tensor1 (Tensor): the tensor to be multiplied + tensor2 (Tensor): the tensor to be multiplied + + Keyword args: + value (Number, optional): multiplier for :math:`tensor1 .* tensor2` + out (Tensor, optional): the output tensor. + + Example:: + + >>> t = torch.randn(1, 3) + >>> t1 = torch.randn(3, 1) + >>> t2 = torch.randn(1, 3) + >>> torch.addcmul(t, t1, t2, value=0.1) + tensor([[-0.8635, -0.6391, 1.6174], + [-0.7617, -0.5879, 1.7388], + [-0.8353, -0.6249, 1.6511]]) + """ + +@overload +def addmm( + beta: Number | _complex, + self: Tensor, + alpha: Number | _complex, + mat1: Tensor, + mat2: Tensor, +) -> Tensor: + r""" + addmm(input, mat1, mat2, out_dtype=None, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a matrix multiplication of the matrices :attr:`mat1` and :attr:`mat2`. + The matrix :attr:`input` is added to the final result. + + If :attr:`mat1` is a :math:`(n \times m)` tensor, :attr:`mat2` is a + :math:`(m \times p)` tensor, then :attr:`input` must be + :ref:`broadcastable ` with a :math:`(n \times p)` tensor + and :attr:`out` will be a :math:`(n \times p)` tensor. + + :attr:`alpha` and :attr:`beta` are scaling factors on matrix-vector product between + :attr:`mat1` and :attr:`mat2` and the added matrix :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{mat1}_i \mathbin{@} \text{mat2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + This operation has support for arguments with :ref:`sparse layouts`. If + :attr:`input` is sparse the result will have the same layout and if :attr:`out` + is provided it must have the same layout as :attr:`input`. + + + .. warning:: + Sparse support is a beta feature and some layout(s)/dtype/device combinations may not be supported, + or may not have autograd support. If you notice missing functionality please + open a feature request. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): matrix to be added + mat1 (Tensor): the first matrix to be matrix multiplied + mat2 (Tensor): the second matrix to be matrix multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat1 @ mat2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(2, 3) + >>> mat1 = torch.randn(2, 3) + >>> mat2 = torch.randn(3, 3) + >>> torch.addmm(M, mat1, mat2) + tensor([[-4.8716, 1.4671, -1.3746], + [ 0.7573, -3.9555, -2.8681]]) + """ + +@overload +def addmm( + beta: Number | _complex, + self: Tensor, + alpha: Number | _complex, + mat1: Tensor, + mat2: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + addmm(input, mat1, mat2, out_dtype=None, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a matrix multiplication of the matrices :attr:`mat1` and :attr:`mat2`. + The matrix :attr:`input` is added to the final result. + + If :attr:`mat1` is a :math:`(n \times m)` tensor, :attr:`mat2` is a + :math:`(m \times p)` tensor, then :attr:`input` must be + :ref:`broadcastable ` with a :math:`(n \times p)` tensor + and :attr:`out` will be a :math:`(n \times p)` tensor. + + :attr:`alpha` and :attr:`beta` are scaling factors on matrix-vector product between + :attr:`mat1` and :attr:`mat2` and the added matrix :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{mat1}_i \mathbin{@} \text{mat2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + This operation has support for arguments with :ref:`sparse layouts`. If + :attr:`input` is sparse the result will have the same layout and if :attr:`out` + is provided it must have the same layout as :attr:`input`. + + + .. warning:: + Sparse support is a beta feature and some layout(s)/dtype/device combinations may not be supported, + or may not have autograd support. If you notice missing functionality please + open a feature request. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): matrix to be added + mat1 (Tensor): the first matrix to be matrix multiplied + mat2 (Tensor): the second matrix to be matrix multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat1 @ mat2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(2, 3) + >>> mat1 = torch.randn(2, 3) + >>> mat2 = torch.randn(3, 3) + >>> torch.addmm(M, mat1, mat2) + tensor([[-4.8716, 1.4671, -1.3746], + [ 0.7573, -3.9555, -2.8681]]) + """ + +@overload +def addmm( + input: Tensor, + mat1: Tensor, + mat2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + addmm(input, mat1, mat2, out_dtype=None, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a matrix multiplication of the matrices :attr:`mat1` and :attr:`mat2`. + The matrix :attr:`input` is added to the final result. + + If :attr:`mat1` is a :math:`(n \times m)` tensor, :attr:`mat2` is a + :math:`(m \times p)` tensor, then :attr:`input` must be + :ref:`broadcastable ` with a :math:`(n \times p)` tensor + and :attr:`out` will be a :math:`(n \times p)` tensor. + + :attr:`alpha` and :attr:`beta` are scaling factors on matrix-vector product between + :attr:`mat1` and :attr:`mat2` and the added matrix :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{mat1}_i \mathbin{@} \text{mat2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + This operation has support for arguments with :ref:`sparse layouts`. If + :attr:`input` is sparse the result will have the same layout and if :attr:`out` + is provided it must have the same layout as :attr:`input`. + + + .. warning:: + Sparse support is a beta feature and some layout(s)/dtype/device combinations may not be supported, + or may not have autograd support. If you notice missing functionality please + open a feature request. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): matrix to be added + mat1 (Tensor): the first matrix to be matrix multiplied + mat2 (Tensor): the second matrix to be matrix multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat1 @ mat2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(2, 3) + >>> mat1 = torch.randn(2, 3) + >>> mat2 = torch.randn(3, 3) + >>> torch.addmm(M, mat1, mat2) + tensor([[-4.8716, 1.4671, -1.3746], + [ 0.7573, -3.9555, -2.8681]]) + """ + +@overload +def addmm( + input: Tensor, + mat1: Tensor, + mat2: Tensor, + out_dtype: _dtype, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + addmm(input, mat1, mat2, out_dtype=None, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a matrix multiplication of the matrices :attr:`mat1` and :attr:`mat2`. + The matrix :attr:`input` is added to the final result. + + If :attr:`mat1` is a :math:`(n \times m)` tensor, :attr:`mat2` is a + :math:`(m \times p)` tensor, then :attr:`input` must be + :ref:`broadcastable ` with a :math:`(n \times p)` tensor + and :attr:`out` will be a :math:`(n \times p)` tensor. + + :attr:`alpha` and :attr:`beta` are scaling factors on matrix-vector product between + :attr:`mat1` and :attr:`mat2` and the added matrix :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{mat1}_i \mathbin{@} \text{mat2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + This operation has support for arguments with :ref:`sparse layouts`. If + :attr:`input` is sparse the result will have the same layout and if :attr:`out` + is provided it must have the same layout as :attr:`input`. + + + .. warning:: + Sparse support is a beta feature and some layout(s)/dtype/device combinations may not be supported, + or may not have autograd support. If you notice missing functionality please + open a feature request. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): matrix to be added + mat1 (Tensor): the first matrix to be matrix multiplied + mat2 (Tensor): the second matrix to be matrix multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat1 @ mat2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(2, 3) + >>> mat1 = torch.randn(2, 3) + >>> mat2 = torch.randn(3, 3) + >>> torch.addmm(M, mat1, mat2) + tensor([[-4.8716, 1.4671, -1.3746], + [ 0.7573, -3.9555, -2.8681]]) + """ + +@overload +def addmm( + beta: Number | _complex, + self: Tensor, + mat1: Tensor, + mat2: Tensor, +) -> Tensor: + r""" + addmm(input, mat1, mat2, out_dtype=None, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a matrix multiplication of the matrices :attr:`mat1` and :attr:`mat2`. + The matrix :attr:`input` is added to the final result. + + If :attr:`mat1` is a :math:`(n \times m)` tensor, :attr:`mat2` is a + :math:`(m \times p)` tensor, then :attr:`input` must be + :ref:`broadcastable ` with a :math:`(n \times p)` tensor + and :attr:`out` will be a :math:`(n \times p)` tensor. + + :attr:`alpha` and :attr:`beta` are scaling factors on matrix-vector product between + :attr:`mat1` and :attr:`mat2` and the added matrix :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{mat1}_i \mathbin{@} \text{mat2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + This operation has support for arguments with :ref:`sparse layouts`. If + :attr:`input` is sparse the result will have the same layout and if :attr:`out` + is provided it must have the same layout as :attr:`input`. + + + .. warning:: + Sparse support is a beta feature and some layout(s)/dtype/device combinations may not be supported, + or may not have autograd support. If you notice missing functionality please + open a feature request. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): matrix to be added + mat1 (Tensor): the first matrix to be matrix multiplied + mat2 (Tensor): the second matrix to be matrix multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat1 @ mat2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(2, 3) + >>> mat1 = torch.randn(2, 3) + >>> mat2 = torch.randn(3, 3) + >>> torch.addmm(M, mat1, mat2) + tensor([[-4.8716, 1.4671, -1.3746], + [ 0.7573, -3.9555, -2.8681]]) + """ + +@overload +def addmm( + beta: Number | _complex, + self: Tensor, + mat1: Tensor, + mat2: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + addmm(input, mat1, mat2, out_dtype=None, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a matrix multiplication of the matrices :attr:`mat1` and :attr:`mat2`. + The matrix :attr:`input` is added to the final result. + + If :attr:`mat1` is a :math:`(n \times m)` tensor, :attr:`mat2` is a + :math:`(m \times p)` tensor, then :attr:`input` must be + :ref:`broadcastable ` with a :math:`(n \times p)` tensor + and :attr:`out` will be a :math:`(n \times p)` tensor. + + :attr:`alpha` and :attr:`beta` are scaling factors on matrix-vector product between + :attr:`mat1` and :attr:`mat2` and the added matrix :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{mat1}_i \mathbin{@} \text{mat2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + This operation has support for arguments with :ref:`sparse layouts`. If + :attr:`input` is sparse the result will have the same layout and if :attr:`out` + is provided it must have the same layout as :attr:`input`. + + + .. warning:: + Sparse support is a beta feature and some layout(s)/dtype/device combinations may not be supported, + or may not have autograd support. If you notice missing functionality please + open a feature request. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): matrix to be added + mat1 (Tensor): the first matrix to be matrix multiplied + mat2 (Tensor): the second matrix to be matrix multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat1 @ mat2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(2, 3) + >>> mat1 = torch.randn(2, 3) + >>> mat2 = torch.randn(3, 3) + >>> torch.addmm(M, mat1, mat2) + tensor([[-4.8716, 1.4671, -1.3746], + [ 0.7573, -3.9555, -2.8681]]) + """ + +@overload +def addmv( + beta: Number | _complex, + self: Tensor, + alpha: Number | _complex, + mat: Tensor, + vec: Tensor, +) -> Tensor: + r""" + addmv(input, mat, vec, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a matrix-vector product of the matrix :attr:`mat` and + the vector :attr:`vec`. + The vector :attr:`input` is added to the final result. + + If :attr:`mat` is a :math:`(n \times m)` tensor, :attr:`vec` is a 1-D tensor of + size `m`, then :attr:`input` must be + :ref:`broadcastable ` with a 1-D tensor of size `n` and + :attr:`out` will be 1-D tensor of size `n`. + + :attr:`alpha` and :attr:`beta` are scaling factors on matrix-vector product between + :attr:`mat` and :attr:`vec` and the added tensor :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{mat} \mathbin{@} \text{vec}) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + Args: + input (Tensor): vector to be added + mat (Tensor): matrix to be matrix multiplied + vec (Tensor): vector to be matrix multiplied + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat @ vec` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(2) + >>> mat = torch.randn(2, 3) + >>> vec = torch.randn(3) + >>> torch.addmv(M, mat, vec) + tensor([-0.3768, -5.5565]) + """ + +@overload +def addmv( + beta: Number | _complex, + self: Tensor, + alpha: Number | _complex, + mat: Tensor, + vec: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + addmv(input, mat, vec, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a matrix-vector product of the matrix :attr:`mat` and + the vector :attr:`vec`. + The vector :attr:`input` is added to the final result. + + If :attr:`mat` is a :math:`(n \times m)` tensor, :attr:`vec` is a 1-D tensor of + size `m`, then :attr:`input` must be + :ref:`broadcastable ` with a 1-D tensor of size `n` and + :attr:`out` will be 1-D tensor of size `n`. + + :attr:`alpha` and :attr:`beta` are scaling factors on matrix-vector product between + :attr:`mat` and :attr:`vec` and the added tensor :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{mat} \mathbin{@} \text{vec}) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + Args: + input (Tensor): vector to be added + mat (Tensor): matrix to be matrix multiplied + vec (Tensor): vector to be matrix multiplied + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat @ vec` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(2) + >>> mat = torch.randn(2, 3) + >>> vec = torch.randn(3) + >>> torch.addmv(M, mat, vec) + tensor([-0.3768, -5.5565]) + """ + +@overload +def addmv( + input: Tensor, + mat: Tensor, + vec: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + addmv(input, mat, vec, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a matrix-vector product of the matrix :attr:`mat` and + the vector :attr:`vec`. + The vector :attr:`input` is added to the final result. + + If :attr:`mat` is a :math:`(n \times m)` tensor, :attr:`vec` is a 1-D tensor of + size `m`, then :attr:`input` must be + :ref:`broadcastable ` with a 1-D tensor of size `n` and + :attr:`out` will be 1-D tensor of size `n`. + + :attr:`alpha` and :attr:`beta` are scaling factors on matrix-vector product between + :attr:`mat` and :attr:`vec` and the added tensor :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{mat} \mathbin{@} \text{vec}) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + Args: + input (Tensor): vector to be added + mat (Tensor): matrix to be matrix multiplied + vec (Tensor): vector to be matrix multiplied + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat @ vec` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(2) + >>> mat = torch.randn(2, 3) + >>> vec = torch.randn(3) + >>> torch.addmv(M, mat, vec) + tensor([-0.3768, -5.5565]) + """ + +@overload +def addmv( + beta: Number | _complex, + self: Tensor, + mat: Tensor, + vec: Tensor, +) -> Tensor: + r""" + addmv(input, mat, vec, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a matrix-vector product of the matrix :attr:`mat` and + the vector :attr:`vec`. + The vector :attr:`input` is added to the final result. + + If :attr:`mat` is a :math:`(n \times m)` tensor, :attr:`vec` is a 1-D tensor of + size `m`, then :attr:`input` must be + :ref:`broadcastable ` with a 1-D tensor of size `n` and + :attr:`out` will be 1-D tensor of size `n`. + + :attr:`alpha` and :attr:`beta` are scaling factors on matrix-vector product between + :attr:`mat` and :attr:`vec` and the added tensor :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{mat} \mathbin{@} \text{vec}) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + Args: + input (Tensor): vector to be added + mat (Tensor): matrix to be matrix multiplied + vec (Tensor): vector to be matrix multiplied + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat @ vec` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(2) + >>> mat = torch.randn(2, 3) + >>> vec = torch.randn(3) + >>> torch.addmv(M, mat, vec) + tensor([-0.3768, -5.5565]) + """ + +@overload +def addmv( + beta: Number | _complex, + self: Tensor, + mat: Tensor, + vec: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + addmv(input, mat, vec, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a matrix-vector product of the matrix :attr:`mat` and + the vector :attr:`vec`. + The vector :attr:`input` is added to the final result. + + If :attr:`mat` is a :math:`(n \times m)` tensor, :attr:`vec` is a 1-D tensor of + size `m`, then :attr:`input` must be + :ref:`broadcastable ` with a 1-D tensor of size `n` and + :attr:`out` will be 1-D tensor of size `n`. + + :attr:`alpha` and :attr:`beta` are scaling factors on matrix-vector product between + :attr:`mat` and :attr:`vec` and the added tensor :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{mat} \mathbin{@} \text{vec}) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + Args: + input (Tensor): vector to be added + mat (Tensor): matrix to be matrix multiplied + vec (Tensor): vector to be matrix multiplied + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat @ vec` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(2) + >>> mat = torch.randn(2, 3) + >>> vec = torch.randn(3) + >>> torch.addmv(M, mat, vec) + tensor([-0.3768, -5.5565]) + """ + +@overload +def addmv_( + beta: Number | _complex, + self: Tensor, + alpha: Number | _complex, + mat: Tensor, + vec: Tensor, +) -> Tensor: ... +@overload +def addmv_( + input: Tensor, + mat: Tensor, + vec: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, +) -> Tensor: ... +@overload +def addmv_( + beta: Number | _complex, + self: Tensor, + mat: Tensor, + vec: Tensor, +) -> Tensor: ... +@overload +def addr( + beta: Number | _complex, + self: Tensor, + alpha: Number | _complex, + vec1: Tensor, + vec2: Tensor, +) -> Tensor: + r""" + addr(input, vec1, vec2, *, beta=1, alpha=1, out=None) -> Tensor + + Performs the outer-product of vectors :attr:`vec1` and :attr:`vec2` + and adds it to the matrix :attr:`input`. + + Optional values :attr:`beta` and :attr:`alpha` are scaling factors on the + outer product between :attr:`vec1` and :attr:`vec2` and the added matrix + :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{vec1} \otimes \text{vec2}) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + If :attr:`vec1` is a vector of size `n` and :attr:`vec2` is a vector + of size `m`, then :attr:`input` must be + :ref:`broadcastable ` with a matrix of size + :math:`(n \times m)` and :attr:`out` will be a matrix of size + :math:`(n \times m)`. + + Args: + input (Tensor): matrix to be added + vec1 (Tensor): the first vector of the outer product + vec2 (Tensor): the second vector of the outer product + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`\text{vec1} \otimes \text{vec2}` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> vec1 = torch.arange(1., 4.) + >>> vec2 = torch.arange(1., 3.) + >>> M = torch.zeros(3, 2) + >>> torch.addr(M, vec1, vec2) + tensor([[ 1., 2.], + [ 2., 4.], + [ 3., 6.]]) + """ + +@overload +def addr( + beta: Number | _complex, + self: Tensor, + alpha: Number | _complex, + vec1: Tensor, + vec2: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + addr(input, vec1, vec2, *, beta=1, alpha=1, out=None) -> Tensor + + Performs the outer-product of vectors :attr:`vec1` and :attr:`vec2` + and adds it to the matrix :attr:`input`. + + Optional values :attr:`beta` and :attr:`alpha` are scaling factors on the + outer product between :attr:`vec1` and :attr:`vec2` and the added matrix + :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{vec1} \otimes \text{vec2}) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + If :attr:`vec1` is a vector of size `n` and :attr:`vec2` is a vector + of size `m`, then :attr:`input` must be + :ref:`broadcastable ` with a matrix of size + :math:`(n \times m)` and :attr:`out` will be a matrix of size + :math:`(n \times m)`. + + Args: + input (Tensor): matrix to be added + vec1 (Tensor): the first vector of the outer product + vec2 (Tensor): the second vector of the outer product + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`\text{vec1} \otimes \text{vec2}` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> vec1 = torch.arange(1., 4.) + >>> vec2 = torch.arange(1., 3.) + >>> M = torch.zeros(3, 2) + >>> torch.addr(M, vec1, vec2) + tensor([[ 1., 2.], + [ 2., 4.], + [ 3., 6.]]) + """ + +@overload +def addr( + input: Tensor, + vec1: Tensor, + vec2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + addr(input, vec1, vec2, *, beta=1, alpha=1, out=None) -> Tensor + + Performs the outer-product of vectors :attr:`vec1` and :attr:`vec2` + and adds it to the matrix :attr:`input`. + + Optional values :attr:`beta` and :attr:`alpha` are scaling factors on the + outer product between :attr:`vec1` and :attr:`vec2` and the added matrix + :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{vec1} \otimes \text{vec2}) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + If :attr:`vec1` is a vector of size `n` and :attr:`vec2` is a vector + of size `m`, then :attr:`input` must be + :ref:`broadcastable ` with a matrix of size + :math:`(n \times m)` and :attr:`out` will be a matrix of size + :math:`(n \times m)`. + + Args: + input (Tensor): matrix to be added + vec1 (Tensor): the first vector of the outer product + vec2 (Tensor): the second vector of the outer product + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`\text{vec1} \otimes \text{vec2}` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> vec1 = torch.arange(1., 4.) + >>> vec2 = torch.arange(1., 3.) + >>> M = torch.zeros(3, 2) + >>> torch.addr(M, vec1, vec2) + tensor([[ 1., 2.], + [ 2., 4.], + [ 3., 6.]]) + """ + +@overload +def addr( + beta: Number | _complex, + self: Tensor, + vec1: Tensor, + vec2: Tensor, +) -> Tensor: + r""" + addr(input, vec1, vec2, *, beta=1, alpha=1, out=None) -> Tensor + + Performs the outer-product of vectors :attr:`vec1` and :attr:`vec2` + and adds it to the matrix :attr:`input`. + + Optional values :attr:`beta` and :attr:`alpha` are scaling factors on the + outer product between :attr:`vec1` and :attr:`vec2` and the added matrix + :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{vec1} \otimes \text{vec2}) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + If :attr:`vec1` is a vector of size `n` and :attr:`vec2` is a vector + of size `m`, then :attr:`input` must be + :ref:`broadcastable ` with a matrix of size + :math:`(n \times m)` and :attr:`out` will be a matrix of size + :math:`(n \times m)`. + + Args: + input (Tensor): matrix to be added + vec1 (Tensor): the first vector of the outer product + vec2 (Tensor): the second vector of the outer product + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`\text{vec1} \otimes \text{vec2}` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> vec1 = torch.arange(1., 4.) + >>> vec2 = torch.arange(1., 3.) + >>> M = torch.zeros(3, 2) + >>> torch.addr(M, vec1, vec2) + tensor([[ 1., 2.], + [ 2., 4.], + [ 3., 6.]]) + """ + +@overload +def addr( + beta: Number | _complex, + self: Tensor, + vec1: Tensor, + vec2: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + addr(input, vec1, vec2, *, beta=1, alpha=1, out=None) -> Tensor + + Performs the outer-product of vectors :attr:`vec1` and :attr:`vec2` + and adds it to the matrix :attr:`input`. + + Optional values :attr:`beta` and :attr:`alpha` are scaling factors on the + outer product between :attr:`vec1` and :attr:`vec2` and the added matrix + :attr:`input` respectively. + + .. math:: + \text{out} = \beta\ \text{input} + \alpha\ (\text{vec1} \otimes \text{vec2}) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + If :attr:`vec1` is a vector of size `n` and :attr:`vec2` is a vector + of size `m`, then :attr:`input` must be + :ref:`broadcastable ` with a matrix of size + :math:`(n \times m)` and :attr:`out` will be a matrix of size + :math:`(n \times m)`. + + Args: + input (Tensor): matrix to be added + vec1 (Tensor): the first vector of the outer product + vec2 (Tensor): the second vector of the outer product + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`\text{vec1} \otimes \text{vec2}` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> vec1 = torch.arange(1., 4.) + >>> vec2 = torch.arange(1., 3.) + >>> M = torch.zeros(3, 2) + >>> torch.addr(M, vec1, vec2) + tensor([[ 1., 2.], + [ 2., 4.], + [ 3., 6.]]) + """ + +def adjoint(input: Tensor) -> Tensor: + r""" + adjoint(input: Tensor) -> Tensor + Returns a view of the tensor conjugated and with the last two dimensions transposed. + + ``x.adjoint()`` is equivalent to ``x.transpose(-2, -1).conj()`` for complex tensors and + to ``x.transpose(-2, -1)`` for real tensors. + + Args: + {input} + + Example:: + + >>> x = torch.arange(4, dtype=torch.float) + >>> A = torch.complex(x, x).reshape(2, 2) + >>> A + tensor([[0.+0.j, 1.+1.j], + [2.+2.j, 3.+3.j]]) + >>> A.adjoint() + tensor([[0.-0.j, 2.-2.j], + [1.-1.j, 3.-3.j]]) + >>> (A.adjoint() == A.mH).all() + tensor(True) + """ + +def affine_grid_generator( + theta: Tensor, + size: Sequence[_int | SymInt], + align_corners: _bool, +) -> Tensor: ... +def alias_copy(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + Performs the same operation as :func:`torch.alias`, but all output tensors + are freshly created instead of aliasing the input. + """ + +@overload +def all(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + all(input: Tensor, *, out=None) -> Tensor + + Tests if all elements in :attr:`input` evaluate to `True`. + + .. note:: This function matches the behaviour of NumPy in returning + output of dtype `bool` for all supported dtypes except `uint8`. + For `uint8` the dtype of output is `uint8` itself. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(1, 2).bool() + >>> a + tensor([[False, True]], dtype=torch.bool) + >>> torch.all(a) + tensor(False, dtype=torch.bool) + >>> a = torch.arange(0, 3) + >>> a + tensor([0, 1, 2]) + >>> torch.all(a) + tensor(False) + + .. function:: all(input, dim, keepdim=False, *, out=None) -> Tensor + :noindex: + + For each row of :attr:`input` in the given dimension :attr:`dim`, + returns `True` if all elements in the row evaluate to `True` and `False` otherwise. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(4, 2).bool() + >>> a + tensor([[True, True], + [True, False], + [True, True], + [True, True]], dtype=torch.bool) + >>> torch.all(a, dim=1) + tensor([ True, False, True, True], dtype=torch.bool) + >>> torch.all(a, dim=0) + tensor([ True, False], dtype=torch.bool) + """ + +@overload +def all( + input: Tensor, + dim: _size | None = None, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + all(input: Tensor, *, out=None) -> Tensor + + Tests if all elements in :attr:`input` evaluate to `True`. + + .. note:: This function matches the behaviour of NumPy in returning + output of dtype `bool` for all supported dtypes except `uint8`. + For `uint8` the dtype of output is `uint8` itself. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(1, 2).bool() + >>> a + tensor([[False, True]], dtype=torch.bool) + >>> torch.all(a) + tensor(False, dtype=torch.bool) + >>> a = torch.arange(0, 3) + >>> a + tensor([0, 1, 2]) + >>> torch.all(a) + tensor(False) + + .. function:: all(input, dim, keepdim=False, *, out=None) -> Tensor + :noindex: + + For each row of :attr:`input` in the given dimension :attr:`dim`, + returns `True` if all elements in the row evaluate to `True` and `False` otherwise. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(4, 2).bool() + >>> a + tensor([[True, True], + [True, False], + [True, True], + [True, True]], dtype=torch.bool) + >>> torch.all(a, dim=1) + tensor([ True, False, True, True], dtype=torch.bool) + >>> torch.all(a, dim=0) + tensor([ True, False], dtype=torch.bool) + """ + +@overload +def all( + input: Tensor, + dim: _int, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + all(input: Tensor, *, out=None) -> Tensor + + Tests if all elements in :attr:`input` evaluate to `True`. + + .. note:: This function matches the behaviour of NumPy in returning + output of dtype `bool` for all supported dtypes except `uint8`. + For `uint8` the dtype of output is `uint8` itself. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(1, 2).bool() + >>> a + tensor([[False, True]], dtype=torch.bool) + >>> torch.all(a) + tensor(False, dtype=torch.bool) + >>> a = torch.arange(0, 3) + >>> a + tensor([0, 1, 2]) + >>> torch.all(a) + tensor(False) + + .. function:: all(input, dim, keepdim=False, *, out=None) -> Tensor + :noindex: + + For each row of :attr:`input` in the given dimension :attr:`dim`, + returns `True` if all elements in the row evaluate to `True` and `False` otherwise. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(4, 2).bool() + >>> a + tensor([[True, True], + [True, False], + [True, True], + [True, True]], dtype=torch.bool) + >>> torch.all(a, dim=1) + tensor([ True, False, True, True], dtype=torch.bool) + >>> torch.all(a, dim=0) + tensor([ True, False], dtype=torch.bool) + """ + +@overload +def all( + input: Tensor, + dim: str | EllipsisType | None, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + all(input: Tensor, *, out=None) -> Tensor + + Tests if all elements in :attr:`input` evaluate to `True`. + + .. note:: This function matches the behaviour of NumPy in returning + output of dtype `bool` for all supported dtypes except `uint8`. + For `uint8` the dtype of output is `uint8` itself. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(1, 2).bool() + >>> a + tensor([[False, True]], dtype=torch.bool) + >>> torch.all(a) + tensor(False, dtype=torch.bool) + >>> a = torch.arange(0, 3) + >>> a + tensor([0, 1, 2]) + >>> torch.all(a) + tensor(False) + + .. function:: all(input, dim, keepdim=False, *, out=None) -> Tensor + :noindex: + + For each row of :attr:`input` in the given dimension :attr:`dim`, + returns `True` if all elements in the row evaluate to `True` and `False` otherwise. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(4, 2).bool() + >>> a + tensor([[True, True], + [True, False], + [True, True], + [True, True]], dtype=torch.bool) + >>> torch.all(a, dim=1) + tensor([ True, False, True, True], dtype=torch.bool) + >>> torch.all(a, dim=0) + tensor([ True, False], dtype=torch.bool) + """ + +def allclose( + input: Tensor, + other: Tensor, + rtol: _float = 1e-05, + atol: _float = 1e-08, + equal_nan: _bool = False, +) -> _bool: + r""" + allclose(input: Tensor, other: Tensor, rtol: float = 1e-05, atol: float = 1e-08, equal_nan: bool = False) -> bool + + This function checks if :attr:`input` and :attr:`other` satisfy the condition: + + .. math:: + \lvert \text{input}_i - \text{other}_i \rvert \leq \texttt{atol} + \texttt{rtol} \times \lvert \text{other}_i \rvert + + elementwise, for all elements of :attr:`input` and :attr:`other`. The behaviour of this function is analogous to + `numpy.allclose `_ + + Args: + input (Tensor): first tensor to compare + other (Tensor): second tensor to compare + atol (float, optional): absolute tolerance. Default: 1e-08 + rtol (float, optional): relative tolerance. Default: 1e-05 + equal_nan (bool, optional): if ``True``, then two ``NaN`` s will be considered equal. Default: ``False`` + + Example:: + + >>> torch.allclose(torch.tensor([10000., 1e-07]), torch.tensor([10000.1, 1e-08])) + False + >>> torch.allclose(torch.tensor([10000., 1e-08]), torch.tensor([10000.1, 1e-09])) + True + >>> torch.allclose(torch.tensor([1.0, float('nan')]), torch.tensor([1.0, float('nan')])) + False + >>> torch.allclose(torch.tensor([1.0, float('nan')]), torch.tensor([1.0, float('nan')]), equal_nan=True) + True + """ + +def alpha_dropout(input: Tensor, p: _float, train: _bool) -> Tensor: ... +def alpha_dropout_(input: Tensor, p: _float, train: _bool) -> Tensor: ... +def amax( + input: Tensor, + dim: _int | _size = (), + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + amax(input, dim, keepdim=False, *, out=None) -> Tensor + + Returns the maximum value of each slice of the :attr:`input` tensor in the given + dimension(s) :attr:`dim`. + + .. note:: + The difference between ``max``/``min`` and ``amax``/``amin`` is: + - ``amax``/``amin`` supports reducing on multiple dimensions, + - ``amax``/``amin`` does not return indices. + + Both ``amax``/``amin`` evenly distribute gradients between equal values + when there are multiple input elements with the same minimum or maximum value. + + For ``max``/``min``: + - If reduce over all dimensions(no dim specified), gradients evenly distribute between equally ``max``/``min`` values. + - If reduce over one specified axis, only propagate to the indexed element. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[ 0.8177, 1.4878, -0.2491, 0.9130], + [-0.7158, 1.1775, 2.0992, 0.4817], + [-0.0053, 0.0164, -1.3738, -0.0507], + [ 1.9700, 1.1106, -1.0318, -1.0816]]) + >>> torch.amax(a, 1) + tensor([1.4878, 2.0992, 0.0164, 1.9700]) + """ + +def amin( + input: Tensor, + dim: _int | _size = (), + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + amin(input, dim, keepdim=False, *, out=None) -> Tensor + + Returns the minimum value of each slice of the :attr:`input` tensor in the given + dimension(s) :attr:`dim`. + + .. note:: + The difference between ``max``/``min`` and ``amax``/``amin`` is: + - ``amax``/``amin`` supports reducing on multiple dimensions, + - ``amax``/``amin`` does not return indices. + + Both ``amax``/``amin`` evenly distribute gradients between equal values + when there are multiple input elements with the same minimum or maximum value. + + For ``max``/``min``: + - If reduce over all dimensions(no dim specified), gradients evenly distribute between equally ``max``/``min`` values. + - If reduce over one specified axis, only propagate to the indexed element. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[ 0.6451, -0.4866, 0.2987, -1.3312], + [-0.5744, 1.2980, 1.8397, -0.2713], + [ 0.9128, 0.9214, -1.7268, -0.2995], + [ 0.9023, 0.4853, 0.9075, -1.6165]]) + >>> torch.amin(a, 1) + tensor([-1.3312, -0.5744, -1.7268, -1.6165]) + """ + +def aminmax( + input: Tensor, + *, + dim: _int | None = None, + keepdim: _bool = False, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.aminmax: + r""" + aminmax(input, *, dim=None, keepdim=False, out=None) -> (Tensor min, Tensor max) + + Computes the minimum and maximum values of the :attr:`input` tensor. + + Args: + input (Tensor): + The input tensor + + Keyword Args: + dim (Optional[int]): + The dimension along which to compute the values. If `None`, + computes the values over the entire :attr:`input` tensor. + Default is `None`. + keepdim (bool): + If `True`, the reduced dimensions will be kept in the output + tensor as dimensions with size 1 for broadcasting, otherwise + they will be removed, as if calling (:func:`torch.squeeze`). + Default is `False`. + out (Optional[Tuple[Tensor, Tensor]]): + Optional tensors on which to write the result. Must have the same + shape and dtype as the expected output. + Default is `None`. + + Returns: + A named tuple `(min, max)` containing the minimum and maximum values. + + Raises: + RuntimeError + If any of the dimensions to compute the values over has size 0. + + .. note:: + NaN values are propagated to the output if at least one value is NaN. + + .. seealso:: + :func:`torch.amin` computes just the minimum value + :func:`torch.amax` computes just the maximum value + + Example:: + + >>> torch.aminmax(torch.tensor([1, -3, 5])) + torch.return_types.aminmax( + min=tensor(-3), + max=tensor(5)) + + >>> # aminmax propagates NaNs + >>> torch.aminmax(torch.tensor([1, -3, 5, torch.nan])) + torch.return_types.aminmax( + min=tensor(nan), + max=tensor(nan)) + + >>> t = torch.arange(10).view(2, 5) + >>> t + tensor([[0, 1, 2, 3, 4], + [5, 6, 7, 8, 9]]) + >>> t.aminmax(dim=0, keepdim=True) + torch.return_types.aminmax( + min=tensor([[0, 1, 2, 3, 4]]), + max=tensor([[5, 6, 7, 8, 9]])) + """ + +def angle(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + angle(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Computes the element-wise angle (in radians) of the given :attr:`input` tensor. + + .. math:: + \text{out}_{i} = angle(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + .. note:: Starting in PyTorch 1.8, angle returns pi for negative real numbers, + zero for non-negative real numbers, and propagates NaNs. Previously + the function would return zero for all real numbers and not propagate + floating-point NaNs. + + Example:: + + >>> torch.angle(torch.tensor([-1 + 1j, -2 + 2j, 3 - 3j]))*180/3.14159 + tensor([ 135., 135, -45]) + """ + +@overload +def any(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + any(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Tests if any element in :attr:`input` evaluates to `True`. + + .. note:: This function matches the behaviour of NumPy in returning + output of dtype `bool` for all supported dtypes except `uint8`. + For `uint8` the dtype of output is `uint8` itself. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(1, 2).bool() + >>> a + tensor([[False, True]], dtype=torch.bool) + >>> torch.any(a) + tensor(True, dtype=torch.bool) + >>> a = torch.arange(0, 3) + >>> a + tensor([0, 1, 2]) + >>> torch.any(a) + tensor(True) + + .. function:: any(input, dim, keepdim=False, *, out=None) -> Tensor + :noindex: + + For each row of :attr:`input` in the given dimension :attr:`dim`, + returns `True` if any element in the row evaluate to `True` and `False` otherwise. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4, 2) < 0 + >>> a + tensor([[ True, True], + [False, True], + [ True, True], + [False, False]]) + >>> torch.any(a, 1) + tensor([ True, True, True, False]) + >>> torch.any(a, 0) + tensor([True, True]) + """ + +@overload +def any( + input: Tensor, + dim: _size | None = None, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + any(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Tests if any element in :attr:`input` evaluates to `True`. + + .. note:: This function matches the behaviour of NumPy in returning + output of dtype `bool` for all supported dtypes except `uint8`. + For `uint8` the dtype of output is `uint8` itself. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(1, 2).bool() + >>> a + tensor([[False, True]], dtype=torch.bool) + >>> torch.any(a) + tensor(True, dtype=torch.bool) + >>> a = torch.arange(0, 3) + >>> a + tensor([0, 1, 2]) + >>> torch.any(a) + tensor(True) + + .. function:: any(input, dim, keepdim=False, *, out=None) -> Tensor + :noindex: + + For each row of :attr:`input` in the given dimension :attr:`dim`, + returns `True` if any element in the row evaluate to `True` and `False` otherwise. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4, 2) < 0 + >>> a + tensor([[ True, True], + [False, True], + [ True, True], + [False, False]]) + >>> torch.any(a, 1) + tensor([ True, True, True, False]) + >>> torch.any(a, 0) + tensor([True, True]) + """ + +@overload +def any( + input: Tensor, + dim: _int, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + any(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Tests if any element in :attr:`input` evaluates to `True`. + + .. note:: This function matches the behaviour of NumPy in returning + output of dtype `bool` for all supported dtypes except `uint8`. + For `uint8` the dtype of output is `uint8` itself. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(1, 2).bool() + >>> a + tensor([[False, True]], dtype=torch.bool) + >>> torch.any(a) + tensor(True, dtype=torch.bool) + >>> a = torch.arange(0, 3) + >>> a + tensor([0, 1, 2]) + >>> torch.any(a) + tensor(True) + + .. function:: any(input, dim, keepdim=False, *, out=None) -> Tensor + :noindex: + + For each row of :attr:`input` in the given dimension :attr:`dim`, + returns `True` if any element in the row evaluate to `True` and `False` otherwise. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4, 2) < 0 + >>> a + tensor([[ True, True], + [False, True], + [ True, True], + [False, False]]) + >>> torch.any(a, 1) + tensor([ True, True, True, False]) + >>> torch.any(a, 0) + tensor([True, True]) + """ + +@overload +def any( + input: Tensor, + dim: str | EllipsisType | None, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + any(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Tests if any element in :attr:`input` evaluates to `True`. + + .. note:: This function matches the behaviour of NumPy in returning + output of dtype `bool` for all supported dtypes except `uint8`. + For `uint8` the dtype of output is `uint8` itself. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(1, 2).bool() + >>> a + tensor([[False, True]], dtype=torch.bool) + >>> torch.any(a) + tensor(True, dtype=torch.bool) + >>> a = torch.arange(0, 3) + >>> a + tensor([0, 1, 2]) + >>> torch.any(a) + tensor(True) + + .. function:: any(input, dim, keepdim=False, *, out=None) -> Tensor + :noindex: + + For each row of :attr:`input` in the given dimension :attr:`dim`, + returns `True` if any element in the row evaluate to `True` and `False` otherwise. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4, 2) < 0 + >>> a + tensor([[ True, True], + [False, True], + [ True, True], + [False, False]]) + >>> torch.any(a, 1) + tensor([ True, True, True, False]) + >>> torch.any(a, 0) + tensor([True, True]) + """ + +@overload +def arange( + start: Number, + end: Number, + step: Number, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + pin_memory: _bool = False, +) -> Tensor: + r""" + arange(start=0, end, step=1, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a 1-D tensor of size :math:`\left\lceil \frac{\text{end} - \text{start}}{\text{step}} \right\rceil` + with values from the interval ``[start, end)`` taken with common difference + :attr:`step` beginning from `start`. + + Note: When using floating-point dtypes (especially reduced precision types like ``bfloat16``), + the results may be affected by floating-point rounding behavior. Some values in the sequence + might not be exactly representable in certain floating-point formats, which can lead to + repeated values or unexpected rounding. For precise sequences, it is recommended to use + integer dtypes instead of floating-point dtypes. + + Note that non-integer :attr:`step` is subject to floating point rounding errors when + comparing against :attr:`end`; to avoid inconsistency, we advise subtracting a small epsilon from :attr:`end` + in such cases. + + .. math:: + \text{out}_{{i+1}} = \text{out}_{i} + \text{step} + + Args: + start (Number, optional): the starting value for the set of points. Default: ``0``. + end (Number): the ending value for the set of points + step (Number, optional): the gap between each pair of adjacent points. Default: ``1``. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). If `dtype` is not given, infer the data type from the other input + arguments. If any of `start`, `end`, or `stop` are floating-point, the + `dtype` is inferred to be the default dtype, see + :meth:`~torch.get_default_dtype`. Otherwise, the `dtype` is inferred to + be `torch.int64`. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.arange(5) + tensor([ 0, 1, 2, 3, 4]) + >>> torch.arange(1, 4) + tensor([ 1, 2, 3]) + >>> torch.arange(1, 2.5, 0.5) + tensor([ 1.0000, 1.5000, 2.0000]) + """ + +@overload +def arange( + start: Number, + end: Number, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + pin_memory: _bool = False, +) -> Tensor: + r""" + arange(start=0, end, step=1, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a 1-D tensor of size :math:`\left\lceil \frac{\text{end} - \text{start}}{\text{step}} \right\rceil` + with values from the interval ``[start, end)`` taken with common difference + :attr:`step` beginning from `start`. + + Note: When using floating-point dtypes (especially reduced precision types like ``bfloat16``), + the results may be affected by floating-point rounding behavior. Some values in the sequence + might not be exactly representable in certain floating-point formats, which can lead to + repeated values or unexpected rounding. For precise sequences, it is recommended to use + integer dtypes instead of floating-point dtypes. + + Note that non-integer :attr:`step` is subject to floating point rounding errors when + comparing against :attr:`end`; to avoid inconsistency, we advise subtracting a small epsilon from :attr:`end` + in such cases. + + .. math:: + \text{out}_{{i+1}} = \text{out}_{i} + \text{step} + + Args: + start (Number, optional): the starting value for the set of points. Default: ``0``. + end (Number): the ending value for the set of points + step (Number, optional): the gap between each pair of adjacent points. Default: ``1``. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). If `dtype` is not given, infer the data type from the other input + arguments. If any of `start`, `end`, or `stop` are floating-point, the + `dtype` is inferred to be the default dtype, see + :meth:`~torch.get_default_dtype`. Otherwise, the `dtype` is inferred to + be `torch.int64`. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.arange(5) + tensor([ 0, 1, 2, 3, 4]) + >>> torch.arange(1, 4) + tensor([ 1, 2, 3]) + >>> torch.arange(1, 2.5, 0.5) + tensor([ 1.0000, 1.5000, 2.0000]) + """ + +@overload +def arange( + end: Number, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + pin_memory: _bool = False, +) -> Tensor: + r""" + arange(start=0, end, step=1, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a 1-D tensor of size :math:`\left\lceil \frac{\text{end} - \text{start}}{\text{step}} \right\rceil` + with values from the interval ``[start, end)`` taken with common difference + :attr:`step` beginning from `start`. + + Note: When using floating-point dtypes (especially reduced precision types like ``bfloat16``), + the results may be affected by floating-point rounding behavior. Some values in the sequence + might not be exactly representable in certain floating-point formats, which can lead to + repeated values or unexpected rounding. For precise sequences, it is recommended to use + integer dtypes instead of floating-point dtypes. + + Note that non-integer :attr:`step` is subject to floating point rounding errors when + comparing against :attr:`end`; to avoid inconsistency, we advise subtracting a small epsilon from :attr:`end` + in such cases. + + .. math:: + \text{out}_{{i+1}} = \text{out}_{i} + \text{step} + + Args: + start (Number, optional): the starting value for the set of points. Default: ``0``. + end (Number): the ending value for the set of points + step (Number, optional): the gap between each pair of adjacent points. Default: ``1``. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). If `dtype` is not given, infer the data type from the other input + arguments. If any of `start`, `end`, or `stop` are floating-point, the + `dtype` is inferred to be the default dtype, see + :meth:`~torch.get_default_dtype`. Otherwise, the `dtype` is inferred to + be `torch.int64`. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.arange(5) + tensor([ 0, 1, 2, 3, 4]) + >>> torch.arange(1, 4) + tensor([ 1, 2, 3]) + >>> torch.arange(1, 2.5, 0.5) + tensor([ 1.0000, 1.5000, 2.0000]) + """ + +@overload +def arange( + end: Number | _complex, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + arange(start=0, end, step=1, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a 1-D tensor of size :math:`\left\lceil \frac{\text{end} - \text{start}}{\text{step}} \right\rceil` + with values from the interval ``[start, end)`` taken with common difference + :attr:`step` beginning from `start`. + + Note: When using floating-point dtypes (especially reduced precision types like ``bfloat16``), + the results may be affected by floating-point rounding behavior. Some values in the sequence + might not be exactly representable in certain floating-point formats, which can lead to + repeated values or unexpected rounding. For precise sequences, it is recommended to use + integer dtypes instead of floating-point dtypes. + + Note that non-integer :attr:`step` is subject to floating point rounding errors when + comparing against :attr:`end`; to avoid inconsistency, we advise subtracting a small epsilon from :attr:`end` + in such cases. + + .. math:: + \text{out}_{{i+1}} = \text{out}_{i} + \text{step} + + Args: + start (Number, optional): the starting value for the set of points. Default: ``0``. + end (Number): the ending value for the set of points + step (Number, optional): the gap between each pair of adjacent points. Default: ``1``. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). If `dtype` is not given, infer the data type from the other input + arguments. If any of `start`, `end`, or `stop` are floating-point, the + `dtype` is inferred to be the default dtype, see + :meth:`~torch.get_default_dtype`. Otherwise, the `dtype` is inferred to + be `torch.int64`. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.arange(5) + tensor([ 0, 1, 2, 3, 4]) + >>> torch.arange(1, 4) + tensor([ 1, 2, 3]) + >>> torch.arange(1, 2.5, 0.5) + tensor([ 1.0000, 1.5000, 2.0000]) + """ + +@overload +def arange( + start: Number | _complex, + end: Number | _complex, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + arange(start=0, end, step=1, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a 1-D tensor of size :math:`\left\lceil \frac{\text{end} - \text{start}}{\text{step}} \right\rceil` + with values from the interval ``[start, end)`` taken with common difference + :attr:`step` beginning from `start`. + + Note: When using floating-point dtypes (especially reduced precision types like ``bfloat16``), + the results may be affected by floating-point rounding behavior. Some values in the sequence + might not be exactly representable in certain floating-point formats, which can lead to + repeated values or unexpected rounding. For precise sequences, it is recommended to use + integer dtypes instead of floating-point dtypes. + + Note that non-integer :attr:`step` is subject to floating point rounding errors when + comparing against :attr:`end`; to avoid inconsistency, we advise subtracting a small epsilon from :attr:`end` + in such cases. + + .. math:: + \text{out}_{{i+1}} = \text{out}_{i} + \text{step} + + Args: + start (Number, optional): the starting value for the set of points. Default: ``0``. + end (Number): the ending value for the set of points + step (Number, optional): the gap between each pair of adjacent points. Default: ``1``. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). If `dtype` is not given, infer the data type from the other input + arguments. If any of `start`, `end`, or `stop` are floating-point, the + `dtype` is inferred to be the default dtype, see + :meth:`~torch.get_default_dtype`. Otherwise, the `dtype` is inferred to + be `torch.int64`. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.arange(5) + tensor([ 0, 1, 2, 3, 4]) + >>> torch.arange(1, 4) + tensor([ 1, 2, 3]) + >>> torch.arange(1, 2.5, 0.5) + tensor([ 1.0000, 1.5000, 2.0000]) + """ + +@overload +def arange( + start: Number | _complex, + end: Number | _complex, + step: Number | _complex = 1, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + arange(start=0, end, step=1, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a 1-D tensor of size :math:`\left\lceil \frac{\text{end} - \text{start}}{\text{step}} \right\rceil` + with values from the interval ``[start, end)`` taken with common difference + :attr:`step` beginning from `start`. + + Note: When using floating-point dtypes (especially reduced precision types like ``bfloat16``), + the results may be affected by floating-point rounding behavior. Some values in the sequence + might not be exactly representable in certain floating-point formats, which can lead to + repeated values or unexpected rounding. For precise sequences, it is recommended to use + integer dtypes instead of floating-point dtypes. + + Note that non-integer :attr:`step` is subject to floating point rounding errors when + comparing against :attr:`end`; to avoid inconsistency, we advise subtracting a small epsilon from :attr:`end` + in such cases. + + .. math:: + \text{out}_{{i+1}} = \text{out}_{i} + \text{step} + + Args: + start (Number, optional): the starting value for the set of points. Default: ``0``. + end (Number): the ending value for the set of points + step (Number, optional): the gap between each pair of adjacent points. Default: ``1``. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). If `dtype` is not given, infer the data type from the other input + arguments. If any of `start`, `end`, or `stop` are floating-point, the + `dtype` is inferred to be the default dtype, see + :meth:`~torch.get_default_dtype`. Otherwise, the `dtype` is inferred to + be `torch.int64`. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.arange(5) + tensor([ 0, 1, 2, 3, 4]) + >>> torch.arange(1, 4) + tensor([ 1, 2, 3]) + >>> torch.arange(1, 2.5, 0.5) + tensor([ 1.0000, 1.5000, 2.0000]) + """ + +def arccos(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + arccos(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Alias for :func:`torch.acos`. + """ + +def arccos_(input: Tensor) -> Tensor: ... +def arccosh(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + arccosh(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Alias for :func:`torch.acosh`. + """ + +def arccosh_(input: Tensor) -> Tensor: ... +def arcsin(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + arcsin(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Alias for :func:`torch.asin`. + """ + +def arcsin_(input: Tensor) -> Tensor: ... +def arcsinh(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + arcsinh(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Alias for :func:`torch.asinh`. + """ + +def arcsinh_(input: Tensor) -> Tensor: ... +def arctan(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + arctan(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Alias for :func:`torch.atan`. + """ + +def arctan2( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + arctan2(input: Tensor, other: Tensor, *, out: Optional[Tensor]) -> Tensor + Alias for :func:`torch.atan2`. + """ + +def arctan_(input: Tensor) -> Tensor: ... +def arctanh(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + arctanh(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Alias for :func:`torch.atanh`. + """ + +def arctanh_(input: Tensor) -> Tensor: ... +def argmax( + input: Tensor, + dim: _int | None = None, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + argmax(input) -> LongTensor + + Returns the indices of the maximum value of all elements in the :attr:`input` tensor. + + This is the second value returned by :meth:`torch.max`. See its + documentation for the exact semantics of this method. + + .. note:: If there are multiple maximal values then the indices of the first maximal value are returned. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[ 1.3398, 0.2663, -0.2686, 0.2450], + [-0.7401, -0.8805, -0.3402, -1.1936], + [ 0.4907, -1.3948, -1.0691, -0.3132], + [-1.6092, 0.5419, -0.2993, 0.3195]]) + >>> torch.argmax(a) + tensor(0) + + .. function:: argmax(input, dim, keepdim=False) -> LongTensor + :noindex: + + Returns the indices of the maximum values of a tensor across a dimension. + + This is the second value returned by :meth:`torch.max`. See its + documentation for the exact semantics of this method. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. + If ``None``, the argmax of the flattened input is returned. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[ 1.3398, 0.2663, -0.2686, 0.2450], + [-0.7401, -0.8805, -0.3402, -1.1936], + [ 0.4907, -1.3948, -1.0691, -0.3132], + [-1.6092, 0.5419, -0.2993, 0.3195]]) + >>> torch.argmax(a, dim=1) + tensor([ 0, 2, 0, 1]) + """ + +def argmin( + input: Tensor, + dim: _int | None = None, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + argmin(input, dim=None, keepdim=False) -> LongTensor + + Returns the indices of the minimum value(s) of the flattened tensor or along a dimension + + This is the second value returned by :meth:`torch.min`. See its + documentation for the exact semantics of this method. + + .. note:: If there are multiple minimal values then the indices of the first minimal value are returned. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. + If ``None``, the argmin of the flattened input is returned. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[ 0.1139, 0.2254, -0.1381, 0.3687], + [ 1.0100, -1.1975, -0.0102, -0.4732], + [-0.9240, 0.1207, -0.7506, -1.0213], + [ 1.7809, -1.2960, 0.9384, 0.1438]]) + >>> torch.argmin(a) + tensor(13) + >>> torch.argmin(a, dim=1) + tensor([ 2, 1, 3, 1]) + >>> torch.argmin(a, dim=1, keepdim=True) + tensor([[2], + [1], + [3], + [1]]) + """ + +@overload +def argsort( + input: Tensor, + *, + stable: _bool, + dim: _int = -1, + descending: _bool = False, + out: Tensor | None = None, +) -> Tensor: + r""" + argsort(input, dim=-1, descending=False, *, stable=False) -> Tensor + + Returns the indices that sort a tensor along a given dimension in ascending + order by value. + + This is the second value returned by :meth:`torch.sort`. See its documentation + for the exact semantics of this method. + + If :attr:`stable` is ``True`` then the sorting routine becomes stable, preserving + the order of equivalent elements. If ``False``, the relative order of values + which compare equal is not guaranteed. ``True`` is slower. + + Args: + input (Tensor): the input tensor. + dim (int, optional): the dimension to sort along + descending (bool, optional): controls the sorting order (ascending or descending) + + Keyword args: + stable (bool, optional): controls the relative order of equivalent elements + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[ 0.0785, 1.5267, -0.8521, 0.4065], + [ 0.1598, 0.0788, -0.0745, -1.2700], + [ 1.2208, 1.0722, -0.7064, 1.2564], + [ 0.0669, -0.2318, -0.8229, -0.9280]]) + + + >>> torch.argsort(a, dim=1) + tensor([[2, 0, 3, 1], + [3, 2, 1, 0], + [2, 1, 0, 3], + [3, 2, 1, 0]]) + """ + +@overload +def argsort( + input: Tensor, + dim: _int = -1, + descending: _bool = False, +) -> Tensor: + r""" + argsort(input, dim=-1, descending=False, *, stable=False) -> Tensor + + Returns the indices that sort a tensor along a given dimension in ascending + order by value. + + This is the second value returned by :meth:`torch.sort`. See its documentation + for the exact semantics of this method. + + If :attr:`stable` is ``True`` then the sorting routine becomes stable, preserving + the order of equivalent elements. If ``False``, the relative order of values + which compare equal is not guaranteed. ``True`` is slower. + + Args: + input (Tensor): the input tensor. + dim (int, optional): the dimension to sort along + descending (bool, optional): controls the sorting order (ascending or descending) + + Keyword args: + stable (bool, optional): controls the relative order of equivalent elements + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[ 0.0785, 1.5267, -0.8521, 0.4065], + [ 0.1598, 0.0788, -0.0745, -1.2700], + [ 1.2208, 1.0722, -0.7064, 1.2564], + [ 0.0669, -0.2318, -0.8229, -0.9280]]) + + + >>> torch.argsort(a, dim=1) + tensor([[2, 0, 3, 1], + [3, 2, 1, 0], + [2, 1, 0, 3], + [3, 2, 1, 0]]) + """ + +@overload +def argsort( + input: Tensor, + dim: str | EllipsisType | None, + descending: _bool = False, +) -> Tensor: + r""" + argsort(input, dim=-1, descending=False, *, stable=False) -> Tensor + + Returns the indices that sort a tensor along a given dimension in ascending + order by value. + + This is the second value returned by :meth:`torch.sort`. See its documentation + for the exact semantics of this method. + + If :attr:`stable` is ``True`` then the sorting routine becomes stable, preserving + the order of equivalent elements. If ``False``, the relative order of values + which compare equal is not guaranteed. ``True`` is slower. + + Args: + input (Tensor): the input tensor. + dim (int, optional): the dimension to sort along + descending (bool, optional): controls the sorting order (ascending or descending) + + Keyword args: + stable (bool, optional): controls the relative order of equivalent elements + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[ 0.0785, 1.5267, -0.8521, 0.4065], + [ 0.1598, 0.0788, -0.0745, -1.2700], + [ 1.2208, 1.0722, -0.7064, 1.2564], + [ 0.0669, -0.2318, -0.8229, -0.9280]]) + + + >>> torch.argsort(a, dim=1) + tensor([[2, 0, 3, 1], + [3, 2, 1, 0], + [2, 1, 0, 3], + [3, 2, 1, 0]]) + """ + +def argwhere(input: Tensor) -> Tensor: + r""" + argwhere(input) -> Tensor + + Returns a tensor containing the indices of all non-zero elements of + :attr:`input`. Each row in the result contains the indices of a non-zero + element in :attr:`input`. The result is sorted lexicographically, with + the last index changing the fastest (C-style). + + If :attr:`input` has :math:`n` dimensions, then the resulting indices tensor + :attr:`out` is of size :math:`(z \times n)`, where :math:`z` is the total number of + non-zero elements in the :attr:`input` tensor. + + .. note:: + This function is similar to NumPy's `argwhere`. + + When :attr:`input` is on CUDA, this function causes host-device synchronization. + + Args: + {input} + + Example:: + + >>> t = torch.tensor([1, 0, 1]) + >>> torch.argwhere(t) + tensor([[0], + [2]]) + >>> t = torch.tensor([[1, 0, 1], [0, 1, 1]]) + >>> torch.argwhere(t) + tensor([[0, 0], + [0, 2], + [1, 1], + [1, 2]]) + """ + +def as_strided( + input: Tensor, + size: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + storage_offset: _int | SymInt | None = None, +) -> Tensor: + r""" + as_strided(input, size, stride, storage_offset=None) -> Tensor + + Create a view of an existing `torch.Tensor` :attr:`input` with specified + :attr:`size`, :attr:`stride` and :attr:`storage_offset`. + + .. warning:: + Prefer using other view functions, like :meth:`torch.Tensor.view` or + :meth:`torch.Tensor.expand`, to setting a view's strides manually with + `as_strided`, as this function will throw an error on non-standard Pytorch + backends (that do not have a concept of stride) and the result will depend + on the current layout in memory. The constructed view must only refer to + elements within the Tensor's storage or a runtime error will be thrown. + If the generated view is "overlapped" (with multiple indices referring to + the same element in memory), the behavior of inplace operations on this view + is undefined (and might not throw runtime errors). + + Args: + input (Tensor): the input tensor. + size (tuple or ints): the shape of the output tensor + stride (tuple or ints): the stride of the output tensor + storage_offset (int, optional): the offset in the underlying storage of the output tensor. + If ``None``, the storage_offset of the output tensor will match the input tensor. + + Example:: + + >>> x = torch.randn(3, 3) + >>> x + tensor([[ 0.9039, 0.6291, 1.0795], + [ 0.1586, 2.1939, -0.4900], + [-0.1909, -0.7503, 1.9355]]) + >>> t = torch.as_strided(x, (2, 2), (1, 2)) + >>> t + tensor([[0.9039, 1.0795], + [0.6291, 0.1586]]) + >>> t = torch.as_strided(x, (2, 2), (1, 2), 1) + tensor([[0.6291, 0.1586], + [1.0795, 2.1939]]) + """ + +def as_strided_( + input: Tensor, + size: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + storage_offset: _int | SymInt | None = None, +) -> Tensor: ... +def as_strided_copy( + input: Tensor, + size: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + storage_offset: _int | SymInt | None = None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.as_strided`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def as_strided_scatter( + input: Tensor, + src: Tensor, + size: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + storage_offset: _int | SymInt | None = None, +) -> Tensor: + r""" + as_strided_scatter(input, src, size, stride, storage_offset=None) -> Tensor + + Embeds the values of the :attr:`src` tensor into :attr:`input` along + the elements corresponding to the result of calling + input.as_strided(size, stride, storage_offset). + + This function returns a tensor with fresh storage; it does not + return a view. + + Args: + input (Tensor): the input tensor. + size (tuple or ints): the shape of the output tensor + stride (tuple or ints): the stride of the output tensor + storage_offset (int, optional): the offset in the underlying storage of the output tensor + + .. note:: + + :attr:`src` must be of the proper size in order to be embedded + into :attr:`input`. Specifically, it should have the same shape as + `torch.as_strided(input, size, stride, storage_offset)` + + Example:: + + >>> a = torch.arange(4).reshape(2, 2) + 1 + >>> a + tensor([[1, 2], + [3, 4]]) + >>> b = torch.zeros(3, 3) + >>> b + tensor([[0., 0., 0.], + [0., 0., 0.], + [0., 0., 0.]]) + >>> torch.as_strided_scatter(b, a, (2, 2), (1, 2)) + tensor([[1., 3., 2.], + [4., 0., 0.], + [0., 0., 0.]]) + """ + +def as_tensor( + data: Any, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, +) -> Tensor: + r""" + as_tensor(data: Any, dtype: Optional[dtype] = None, device: Optional[DeviceLikeType]) -> Tensor + + Converts :attr:`data` into a tensor, sharing data and preserving autograd + history if possible. + + If :attr:`data` is already a tensor with the requested dtype and device + then :attr:`data` itself is returned, but if :attr:`data` is a + tensor with a different dtype or device then it's copied as if using + `data.to(dtype=dtype, device=device)`. + + If :attr:`data` is a NumPy array (an ndarray) with the same dtype and device then a + tensor is constructed using :func:`torch.from_numpy`. + + If :attr:`data` is a CuPy array, the returned tensor will be located on the same device as the CuPy array unless + specifically overwritten by :attr:`device` or a default device. The device of the CuPy array is inferred from the + pointer of the array using `cudaPointerGetAttributes` unless :attr:`device` is provided with an explicit device index. + + .. seealso:: + + :func:`torch.tensor` never shares its data and creates a new "leaf tensor" (see :doc:`/notes/autograd`). + + + Args: + data (array_like): Initial data for the tensor. Can be a list, tuple, + NumPy ``ndarray``, scalar, and other types. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, infers data type from :attr:`data`. + device (:class:`torch.device`, optional): the device of the constructed tensor. If None and data is a tensor + then the device of data is used. If None and data is not a tensor then + the result tensor is constructed on the current device. + + + Example:: + + >>> a = numpy.array([1, 2, 3]) + >>> t = torch.as_tensor(a) + >>> t + tensor([ 1, 2, 3]) + >>> t[0] = -1 + >>> a + array([-1, 2, 3]) + + >>> a = numpy.array([1, 2, 3]) + >>> t = torch.as_tensor(a, device=torch.device('cuda')) + >>> t + tensor([ 1, 2, 3]) + >>> t[0] = -1 + >>> a + array([1, 2, 3]) + """ + +def asarray( + obj: Any, + *, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + copy: _bool | None = None, + requires_grad: _bool = False, +) -> Tensor: + r""" + asarray(obj: Any, *, dtype: Optional[dtype], device: Optional[DeviceLikeType], copy: Optional[bool] = None, requires_grad: bool = False) -> Tensor # noqa: B950 + + Converts :attr:`obj` to a tensor. + + :attr:`obj` can be one of: + + 1. a tensor + 2. a NumPy array or a NumPy scalar + 3. a DLPack capsule + 4. an object that implements Python's buffer protocol + 5. a scalar + 6. a sequence of scalars + + When :attr:`obj` is a tensor, NumPy array, or DLPack capsule the returned tensor will, + by default, not require a gradient, have the same datatype as :attr:`obj`, be on the + same device, and share memory with it. These properties can be controlled with the + :attr:`dtype`, :attr:`device`, :attr:`copy`, and :attr:`requires_grad` keyword arguments. + If the returned tensor is of a different datatype, on a different device, or a copy is + requested then it will not share its memory with :attr:`obj`. If :attr:`requires_grad` + is ``True`` then the returned tensor will require a gradient, and if :attr:`obj` is + also a tensor with an autograd history then the returned tensor will have the same history. + + When :attr:`obj` is not a tensor, NumPy array, or DLPack capsule but implements Python's + buffer protocol then the buffer is interpreted as an array of bytes grouped according to + the size of the datatype passed to the :attr:`dtype` keyword argument. (If no datatype is + passed then the default floating point datatype is used, instead.) The returned tensor + will have the specified datatype (or default floating point datatype if none is specified) + and, by default, be on the CPU device and share memory with the buffer. + + When :attr:`obj` is a NumPy scalar, the returned tensor will be a 0-dimensional tensor on + the CPU and that doesn't share its memory (i.e. ``copy=True``). By default datatype will + be the PyTorch datatype corresponding to the NumPy's scalar's datatype. + + When :attr:`obj` is none of the above but a scalar, or a sequence of scalars then the + returned tensor will, by default, infer its datatype from the scalar values, be on the + current default device, and not share its memory. + + .. seealso:: + + :func:`torch.tensor` creates a tensor that always copies the data from the input object. + :func:`torch.from_numpy` creates a tensor that always shares memory from NumPy arrays. + :func:`torch.frombuffer` creates a tensor that always shares memory from objects that + implement the buffer protocol. + :func:`torch.from_dlpack` creates a tensor that always shares memory from + DLPack capsules. + + Args: + obj (object): a tensor, NumPy array, DLPack Capsule, object that implements Python's + buffer protocol, scalar, or sequence of scalars. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the datatype of the returned tensor. + Default: ``None``, which causes the datatype of the returned tensor to be + inferred from :attr:`obj`. + copy (bool, optional): controls whether the returned tensor shares memory with :attr:`obj`. + Default: ``None``, which causes the returned tensor to share memory with :attr:`obj` + whenever possible. If ``True`` then the returned tensor does not share its memory. + If ``False`` then the returned tensor shares its memory with :attr:`obj` and an + error is thrown if it cannot. + device (:class:`torch.device`, optional): the device of the returned tensor. + Default: ``None``, which causes the device of :attr:`obj` to be used. Or, if + :attr:`obj` is a Python sequence, the current default device will be used. + requires_grad (bool, optional): whether the returned tensor requires grad. + Default: ``False``, which causes the returned tensor not to require a gradient. + If ``True``, then the returned tensor will require a gradient, and if :attr:`obj` + is also a tensor with an autograd history then the returned tensor will have + the same history. + + Example:: + + >>> a = torch.tensor([1, 2, 3]) + >>> # Shares memory with tensor 'a' + >>> b = torch.asarray(a) + >>> a.data_ptr() == b.data_ptr() + True + >>> # Forces memory copy + >>> c = torch.asarray(a, copy=True) + >>> a.data_ptr() == c.data_ptr() + False + + >>> a = torch.tensor([1., 2., 3.], requires_grad=True) + >>> b = a + 2 + >>> b + tensor([3., 4., 5.], grad_fn=) + >>> # Shares memory with tensor 'b', with no grad + >>> c = torch.asarray(b) + >>> c + tensor([3., 4., 5.]) + >>> # Shares memory with tensor 'b', retaining autograd history + >>> d = torch.asarray(b, requires_grad=True) + >>> d + tensor([3., 4., 5.], grad_fn=) + + >>> array = numpy.array([1, 2, 3]) + >>> # Shares memory with array 'array' + >>> t1 = torch.asarray(array) + >>> array.__array_interface__['data'][0] == t1.data_ptr() + True + >>> # Copies memory due to dtype mismatch + >>> t2 = torch.asarray(array, dtype=torch.float32) + >>> array.__array_interface__['data'][0] == t2.data_ptr() + False + + >>> scalar = numpy.float64(0.5) + >>> torch.asarray(scalar) + tensor(0.5000, dtype=torch.float64) + """ + +def asin(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + asin(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Returns a new tensor with the arcsine of the elements (in radians) in the :attr:`input` tensor. + + .. math:: + \text{out}_{i} = \sin^{-1}(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([-0.5962, 1.4985, -0.4396, 1.4525]) + >>> torch.asin(a) + tensor([-0.6387, nan, -0.4552, nan]) + """ + +def asin_(input: Tensor) -> Tensor: ... +def asinh(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + asinh(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Returns a new tensor with the inverse hyperbolic sine of the elements of :attr:`input`. + + .. math:: + \text{out}_{i} = \sinh^{-1}(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword arguments: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([ 0.1606, -1.4267, -1.0899, -1.0250 ]) + >>> torch.asinh(a) + tensor([ 0.1599, -1.1534, -0.9435, -0.8990 ]) + """ + +def asinh_(input: Tensor) -> Tensor: ... +def atan(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + atan(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Returns a new tensor with the arctangent of the elements (in radians) in the :attr:`input` tensor. + + .. math:: + \text{out}_{i} = \tan^{-1}(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([ 0.2341, 0.2539, -0.6256, -0.6448]) + >>> torch.atan(a) + tensor([ 0.2299, 0.2487, -0.5591, -0.5727]) + """ + +def atan2( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + atan2(input: Tensor, other: Tensor, *, out: Optional[Tensor]) -> Tensor + + Element-wise arctangent of :math:`\text{input}_{i} / \text{other}_{i}` + with consideration of the quadrant. Returns a new tensor with the signed angles + in radians between vector :math:`(\text{other}_{i}, \text{input}_{i})` + and vector :math:`(1, 0)`. (Note that :math:`\text{other}_{i}`, the second + parameter, is the x-coordinate, while :math:`\text{input}_{i}`, the first + parameter, is the y-coordinate.) + + The shapes of ``input`` and ``other`` must be + :ref:`broadcastable `. + + Args: + input (Tensor): the first input tensor + other (Tensor): the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([ 0.9041, 0.0196, -0.3108, -2.4423]) + >>> torch.atan2(a, torch.randn(4)) + tensor([ 0.9833, 0.0811, -1.9743, -1.4151]) + """ + +def atan_(input: Tensor) -> Tensor: ... +def atanh(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + atanh(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Returns a new tensor with the inverse hyperbolic tangent of the elements of :attr:`input`. + + Note: + The domain of the inverse hyperbolic tangent is `(-1, 1)` and values outside this range + will be mapped to ``NaN``, except for the values `1` and `-1` for which the output is + mapped to `+/-INF` respectively. + + .. math:: + \text{out}_{i} = \tanh^{-1}(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword arguments: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4).uniform_(-1, 1) + >>> a + tensor([ -0.9385, 0.2968, -0.8591, -0.1871 ]) + >>> torch.atanh(a) + tensor([ -1.7253, 0.3060, -1.2899, -0.1893 ]) + """ + +def atanh_(input: Tensor) -> Tensor: ... +def avg_pool1d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size = (), + padding: _int | _size = 0, + ceil_mode: _bool = False, + count_include_pad: _bool = True, +) -> Tensor: ... +@overload +def baddbmm( + beta: Number | _complex, + self: Tensor, + alpha: Number | _complex, + batch1: Tensor, + batch2: Tensor, +) -> Tensor: + r""" + baddbmm(input, batch1, batch2, out_dtype=None, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a batch matrix-matrix product of matrices in :attr:`batch1` + and :attr:`batch2`. + :attr:`input` is added to the final result. + + :attr:`batch1` and :attr:`batch2` must be 3-D tensors each containing the same + number of matrices. + + If :attr:`batch1` is a :math:`(b \times n \times m)` tensor, :attr:`batch2` is a + :math:`(b \times m \times p)` tensor, then :attr:`input` must be + :ref:`broadcastable ` with a + :math:`(b \times n \times p)` tensor and :attr:`out` will be a + :math:`(b \times n \times p)` tensor. Both :attr:`alpha` and :attr:`beta` mean the + same as the scaling factors used in :meth:`torch.addbmm`. + + .. math:: + \text{out}_i = \beta\ \text{input}_i + \alpha\ (\text{batch1}_i \mathbin{@} \text{batch2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): the tensor to be added + batch1 (Tensor): the first batch of matrices to be multiplied + batch2 (Tensor): the second batch of matrices to be multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`\text{batch1} \mathbin{@} \text{batch2}` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(10, 3, 5) + >>> batch1 = torch.randn(10, 3, 4) + >>> batch2 = torch.randn(10, 4, 5) + >>> torch.baddbmm(M, batch1, batch2).size() + torch.Size([10, 3, 5]) + """ + +@overload +def baddbmm( + beta: Number | _complex, + self: Tensor, + alpha: Number | _complex, + batch1: Tensor, + batch2: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + baddbmm(input, batch1, batch2, out_dtype=None, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a batch matrix-matrix product of matrices in :attr:`batch1` + and :attr:`batch2`. + :attr:`input` is added to the final result. + + :attr:`batch1` and :attr:`batch2` must be 3-D tensors each containing the same + number of matrices. + + If :attr:`batch1` is a :math:`(b \times n \times m)` tensor, :attr:`batch2` is a + :math:`(b \times m \times p)` tensor, then :attr:`input` must be + :ref:`broadcastable ` with a + :math:`(b \times n \times p)` tensor and :attr:`out` will be a + :math:`(b \times n \times p)` tensor. Both :attr:`alpha` and :attr:`beta` mean the + same as the scaling factors used in :meth:`torch.addbmm`. + + .. math:: + \text{out}_i = \beta\ \text{input}_i + \alpha\ (\text{batch1}_i \mathbin{@} \text{batch2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): the tensor to be added + batch1 (Tensor): the first batch of matrices to be multiplied + batch2 (Tensor): the second batch of matrices to be multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`\text{batch1} \mathbin{@} \text{batch2}` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(10, 3, 5) + >>> batch1 = torch.randn(10, 3, 4) + >>> batch2 = torch.randn(10, 4, 5) + >>> torch.baddbmm(M, batch1, batch2).size() + torch.Size([10, 3, 5]) + """ + +@overload +def baddbmm( + input: Tensor, + batch1: Tensor, + batch2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + baddbmm(input, batch1, batch2, out_dtype=None, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a batch matrix-matrix product of matrices in :attr:`batch1` + and :attr:`batch2`. + :attr:`input` is added to the final result. + + :attr:`batch1` and :attr:`batch2` must be 3-D tensors each containing the same + number of matrices. + + If :attr:`batch1` is a :math:`(b \times n \times m)` tensor, :attr:`batch2` is a + :math:`(b \times m \times p)` tensor, then :attr:`input` must be + :ref:`broadcastable ` with a + :math:`(b \times n \times p)` tensor and :attr:`out` will be a + :math:`(b \times n \times p)` tensor. Both :attr:`alpha` and :attr:`beta` mean the + same as the scaling factors used in :meth:`torch.addbmm`. + + .. math:: + \text{out}_i = \beta\ \text{input}_i + \alpha\ (\text{batch1}_i \mathbin{@} \text{batch2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): the tensor to be added + batch1 (Tensor): the first batch of matrices to be multiplied + batch2 (Tensor): the second batch of matrices to be multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`\text{batch1} \mathbin{@} \text{batch2}` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(10, 3, 5) + >>> batch1 = torch.randn(10, 3, 4) + >>> batch2 = torch.randn(10, 4, 5) + >>> torch.baddbmm(M, batch1, batch2).size() + torch.Size([10, 3, 5]) + """ + +@overload +def baddbmm( + input: Tensor, + batch1: Tensor, + batch2: Tensor, + out_dtype: _dtype, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + baddbmm(input, batch1, batch2, out_dtype=None, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a batch matrix-matrix product of matrices in :attr:`batch1` + and :attr:`batch2`. + :attr:`input` is added to the final result. + + :attr:`batch1` and :attr:`batch2` must be 3-D tensors each containing the same + number of matrices. + + If :attr:`batch1` is a :math:`(b \times n \times m)` tensor, :attr:`batch2` is a + :math:`(b \times m \times p)` tensor, then :attr:`input` must be + :ref:`broadcastable ` with a + :math:`(b \times n \times p)` tensor and :attr:`out` will be a + :math:`(b \times n \times p)` tensor. Both :attr:`alpha` and :attr:`beta` mean the + same as the scaling factors used in :meth:`torch.addbmm`. + + .. math:: + \text{out}_i = \beta\ \text{input}_i + \alpha\ (\text{batch1}_i \mathbin{@} \text{batch2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): the tensor to be added + batch1 (Tensor): the first batch of matrices to be multiplied + batch2 (Tensor): the second batch of matrices to be multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`\text{batch1} \mathbin{@} \text{batch2}` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(10, 3, 5) + >>> batch1 = torch.randn(10, 3, 4) + >>> batch2 = torch.randn(10, 4, 5) + >>> torch.baddbmm(M, batch1, batch2).size() + torch.Size([10, 3, 5]) + """ + +@overload +def baddbmm( + beta: Number | _complex, + self: Tensor, + batch1: Tensor, + batch2: Tensor, +) -> Tensor: + r""" + baddbmm(input, batch1, batch2, out_dtype=None, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a batch matrix-matrix product of matrices in :attr:`batch1` + and :attr:`batch2`. + :attr:`input` is added to the final result. + + :attr:`batch1` and :attr:`batch2` must be 3-D tensors each containing the same + number of matrices. + + If :attr:`batch1` is a :math:`(b \times n \times m)` tensor, :attr:`batch2` is a + :math:`(b \times m \times p)` tensor, then :attr:`input` must be + :ref:`broadcastable ` with a + :math:`(b \times n \times p)` tensor and :attr:`out` will be a + :math:`(b \times n \times p)` tensor. Both :attr:`alpha` and :attr:`beta` mean the + same as the scaling factors used in :meth:`torch.addbmm`. + + .. math:: + \text{out}_i = \beta\ \text{input}_i + \alpha\ (\text{batch1}_i \mathbin{@} \text{batch2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): the tensor to be added + batch1 (Tensor): the first batch of matrices to be multiplied + batch2 (Tensor): the second batch of matrices to be multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`\text{batch1} \mathbin{@} \text{batch2}` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(10, 3, 5) + >>> batch1 = torch.randn(10, 3, 4) + >>> batch2 = torch.randn(10, 4, 5) + >>> torch.baddbmm(M, batch1, batch2).size() + torch.Size([10, 3, 5]) + """ + +@overload +def baddbmm( + beta: Number | _complex, + self: Tensor, + batch1: Tensor, + batch2: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + baddbmm(input, batch1, batch2, out_dtype=None, *, beta=1, alpha=1, out=None) -> Tensor + + Performs a batch matrix-matrix product of matrices in :attr:`batch1` + and :attr:`batch2`. + :attr:`input` is added to the final result. + + :attr:`batch1` and :attr:`batch2` must be 3-D tensors each containing the same + number of matrices. + + If :attr:`batch1` is a :math:`(b \times n \times m)` tensor, :attr:`batch2` is a + :math:`(b \times m \times p)` tensor, then :attr:`input` must be + :ref:`broadcastable ` with a + :math:`(b \times n \times p)` tensor and :attr:`out` will be a + :math:`(b \times n \times p)` tensor. Both :attr:`alpha` and :attr:`beta` mean the + same as the scaling factors used in :meth:`torch.addbmm`. + + .. math:: + \text{out}_i = \beta\ \text{input}_i + \alpha\ (\text{batch1}_i \mathbin{@} \text{batch2}_i) + + If :attr:`beta` is 0, then the content of :attr:`input` will be ignored, and `nan` and `inf` in + it will not be propagated. + + For inputs of type `FloatTensor` or `DoubleTensor`, arguments :attr:`beta` and + :attr:`alpha` must be real numbers, otherwise they should be integers. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): the tensor to be added + batch1 (Tensor): the first batch of matrices to be multiplied + batch2 (Tensor): the second batch of matrices to be multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`\text{batch1} \mathbin{@} \text{batch2}` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + + Example:: + + >>> M = torch.randn(10, 3, 5) + >>> batch1 = torch.randn(10, 3, 4) + >>> batch2 = torch.randn(10, 4, 5) + >>> torch.baddbmm(M, batch1, batch2).size() + torch.Size([10, 3, 5]) + """ + +@overload +def bartlett_window( + window_length: _int, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + bartlett_window(window_length, periodic=True, *, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Bartlett window function. + + .. math:: + w[n] = 1 - \left| \frac{2n}{N-1} - 1 \right| = \begin{cases} + \frac{2n}{N - 1} & \text{if } 0 \leq n \leq \frac{N - 1}{2} \\ + 2 - \frac{2n}{N - 1} & \text{if } \frac{N - 1}{2} < n < N \\ + \end{cases}, + + where :math:`N` is the full window size. + + The input :attr:`window_length` is a positive integer controlling the + returned window size. :attr:`periodic` flag determines whether the returned + window trims off the last duplicate value from the symmetric window and is + ready to be used as a periodic window with functions like + :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in + above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have + ``torch.bartlett_window(L, periodic=True)`` equal to + ``torch.bartlett_window(L + 1, periodic=False)[:-1])``. + + .. note:: + If :attr:`window_length` :math:`=1`, the returned window contains a single value 1. + + Arguments: + window_length (int): the size of returned window + periodic (bool, optional): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window + """ + +@overload +def bartlett_window( + window_length: _int, + periodic: _bool, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + bartlett_window(window_length, periodic=True, *, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Bartlett window function. + + .. math:: + w[n] = 1 - \left| \frac{2n}{N-1} - 1 \right| = \begin{cases} + \frac{2n}{N - 1} & \text{if } 0 \leq n \leq \frac{N - 1}{2} \\ + 2 - \frac{2n}{N - 1} & \text{if } \frac{N - 1}{2} < n < N \\ + \end{cases}, + + where :math:`N` is the full window size. + + The input :attr:`window_length` is a positive integer controlling the + returned window size. :attr:`periodic` flag determines whether the returned + window trims off the last duplicate value from the symmetric window and is + ready to be used as a periodic window with functions like + :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in + above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have + ``torch.bartlett_window(L, periodic=True)`` equal to + ``torch.bartlett_window(L + 1, periodic=False)[:-1])``. + + .. note:: + If :attr:`window_length` :math:`=1`, the returned window contains a single value 1. + + Arguments: + window_length (int): the size of returned window + periodic (bool, optional): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window + """ + +def batch_norm( + input: Tensor, + weight: Tensor | None, + bias: Tensor | None, + running_mean: Tensor | None, + running_var: Tensor | None, + training: _bool, + momentum: _float, + eps: _float, + cudnn_enabled: _bool, +) -> Tensor: ... +def batch_norm_backward_elemt( + grad_out: Tensor, + input: Tensor, + mean: Tensor, + invstd: Tensor, + weight: Tensor | None, + sum_dy: Tensor, + sum_dy_xmu: Tensor, + count: Tensor, +) -> Tensor: ... +def batch_norm_backward_reduce( + grad_out: Tensor, + input: Tensor, + mean: Tensor, + invstd: Tensor, + weight: Tensor | None, + input_g: _bool, + weight_g: _bool, + bias_g: _bool, +) -> tuple[Tensor, Tensor, Tensor, Tensor]: ... +def batch_norm_elemt( + input: Tensor, + weight: Tensor | None, + bias: Tensor | None, + mean: Tensor, + invstd: Tensor, + eps: _float, + *, + out: Tensor | None = None, +) -> Tensor: ... +def batch_norm_gather_stats( + input: Tensor, + mean: Tensor, + invstd: Tensor, + running_mean: Tensor | None, + running_var: Tensor | None, + momentum: _float, + eps: _float, + count: _int, +) -> tuple[Tensor, Tensor]: ... +def batch_norm_gather_stats_with_counts( + input: Tensor, + mean: Tensor, + invstd: Tensor, + running_mean: Tensor | None, + running_var: Tensor | None, + momentum: _float, + eps: _float, + counts: Tensor, +) -> tuple[Tensor, Tensor]: ... +def batch_norm_stats(input: Tensor, eps: _float) -> tuple[Tensor, Tensor]: ... +def batch_norm_update_stats( + input: Tensor, + running_mean: Tensor | None, + running_var: Tensor | None, + momentum: _float, +) -> tuple[Tensor, Tensor]: ... +@overload +def bernoulli( + input: Tensor, + *, + generator: Generator | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + bernoulli(input: Tensor, *, generator: Optional[Generator], out: Optional[Tensor]) -> Tensor + + Draws binary random numbers (0 or 1) from a Bernoulli distribution. + + The :attr:`input` tensor should be a tensor containing probabilities + to be used for drawing the binary random number. + Hence, all values in :attr:`input` have to be in the range: + :math:`0 \leq \text{input}_i \leq 1`. + + The :math:`\text{i}^{th}` element of the output tensor will draw a + value :math:`1` according to the :math:`\text{i}^{th}` probability value given + in :attr:`input`. + + .. math:: + \text{out}_{i} \sim \mathrm{Bernoulli}(p = \text{input}_{i}) + + The returned :attr:`out` tensor only has values 0 or 1 and is of the same + shape as :attr:`input`. + + :attr:`out` can have integral ``dtype``, but :attr:`input` must have floating + point ``dtype``. + + Args: + input (Tensor): the input tensor of probability values for the Bernoulli distribution + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.empty(3, 3).uniform_(0, 1) # generate a uniform random matrix with range [0, 1] + >>> a + tensor([[ 0.1737, 0.0950, 0.3609], + [ 0.7148, 0.0289, 0.2676], + [ 0.9456, 0.8937, 0.7202]]) + >>> torch.bernoulli(a) + tensor([[ 1., 0., 0.], + [ 0., 0., 0.], + [ 1., 1., 1.]]) + + >>> a = torch.ones(3, 3) # probability of drawing "1" is 1 + >>> torch.bernoulli(a) + tensor([[ 1., 1., 1.], + [ 1., 1., 1.], + [ 1., 1., 1.]]) + >>> a = torch.zeros(3, 3) # probability of drawing "1" is 0 + >>> torch.bernoulli(a) + tensor([[ 0., 0., 0.], + [ 0., 0., 0.], + [ 0., 0., 0.]]) + """ + +@overload +def bernoulli( + input: Tensor, + p: _float, + *, + generator: Generator | None = None, +) -> Tensor: + r""" + bernoulli(input: Tensor, *, generator: Optional[Generator], out: Optional[Tensor]) -> Tensor + + Draws binary random numbers (0 or 1) from a Bernoulli distribution. + + The :attr:`input` tensor should be a tensor containing probabilities + to be used for drawing the binary random number. + Hence, all values in :attr:`input` have to be in the range: + :math:`0 \leq \text{input}_i \leq 1`. + + The :math:`\text{i}^{th}` element of the output tensor will draw a + value :math:`1` according to the :math:`\text{i}^{th}` probability value given + in :attr:`input`. + + .. math:: + \text{out}_{i} \sim \mathrm{Bernoulli}(p = \text{input}_{i}) + + The returned :attr:`out` tensor only has values 0 or 1 and is of the same + shape as :attr:`input`. + + :attr:`out` can have integral ``dtype``, but :attr:`input` must have floating + point ``dtype``. + + Args: + input (Tensor): the input tensor of probability values for the Bernoulli distribution + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.empty(3, 3).uniform_(0, 1) # generate a uniform random matrix with range [0, 1] + >>> a + tensor([[ 0.1737, 0.0950, 0.3609], + [ 0.7148, 0.0289, 0.2676], + [ 0.9456, 0.8937, 0.7202]]) + >>> torch.bernoulli(a) + tensor([[ 1., 0., 0.], + [ 0., 0., 0.], + [ 1., 1., 1.]]) + + >>> a = torch.ones(3, 3) # probability of drawing "1" is 1 + >>> torch.bernoulli(a) + tensor([[ 1., 1., 1.], + [ 1., 1., 1.], + [ 1., 1., 1.]]) + >>> a = torch.zeros(3, 3) # probability of drawing "1" is 0 + >>> torch.bernoulli(a) + tensor([[ 0., 0., 0.], + [ 0., 0., 0.], + [ 0., 0., 0.]]) + """ + +def bilinear( + input1: Tensor, + input2: Tensor, + weight: Tensor, + bias: Tensor | None = None, +) -> Tensor: ... +def binary_cross_entropy_with_logits( + input: Tensor, + target: Tensor, + weight: Tensor | None = None, + pos_weight: Tensor | None = None, + reduction: _int = 1, +) -> Tensor: ... +def bincount( + input: Tensor, + weights: Tensor | None = None, + minlength: _int | SymInt = 0, +) -> Tensor: + r""" + bincount(input, weights=None, minlength=0) -> Tensor + + Count the frequency of each value in an array of non-negative ints. + + The number of bins (size 1) is one larger than the largest value in + :attr:`input` unless :attr:`input` is empty, in which case the result is a + tensor of size 0. If :attr:`minlength` is specified, the number of bins is at least + :attr:`minlength` and if :attr:`input` is empty, then the result is tensor of size + :attr:`minlength` filled with zeros. If ``n`` is the value at position ``i``, + ``out[n] += weights[i]`` if :attr:`weights` is specified else + ``out[n] += 1``. + + Note: + This operation may produce nondeterministic gradients when given tensors on a CUDA device. See :doc:`/notes/randomness` for more information. + + Arguments: + input (Tensor): 1-d int tensor + weights (Tensor): optional, weight for each value in the input tensor. + Should be of same size as input tensor. + minlength (int): optional, minimum number of bins. Should be non-negative. + + Returns: + output (Tensor): a tensor of shape ``Size([max(input) + 1])`` if + :attr:`input` is non-empty, else ``Size(0)`` + + Example:: + + >>> input = torch.randint(0, 8, (5,), dtype=torch.int64) + >>> weights = torch.linspace(0, 1, steps=5) + >>> input, weights + (tensor([4, 3, 6, 3, 4]), + tensor([ 0.0000, 0.2500, 0.5000, 0.7500, 1.0000]) + + >>> torch.bincount(input) + tensor([0, 0, 0, 2, 2, 0, 1]) + + >>> input.bincount(weights) + tensor([0.0000, 0.0000, 0.0000, 1.0000, 1.0000, 0.0000, 0.5000]) + """ + +def binomial( + count: Tensor, + prob: Tensor, + generator: Generator | None = None, +) -> Tensor: ... +@overload +def bitwise_and( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + bitwise_and(input, other, *, out=None) -> Tensor + + Computes the bitwise AND of :attr:`input` and :attr:`other`. The input tensor must be of + integral or Boolean types. For bool tensors, it computes the logical AND. + + Args: + input: the first input tensor + other: the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_and(torch.tensor([-1, -2, 3], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([1, 0, 3], dtype=torch.int8) + >>> torch.bitwise_and(torch.tensor([True, True, False]), torch.tensor([False, True, False])) + tensor([ False, True, False]) + """ + +@overload +def bitwise_and(self: Number | _complex, other: Tensor) -> Tensor: + r""" + bitwise_and(input, other, *, out=None) -> Tensor + + Computes the bitwise AND of :attr:`input` and :attr:`other`. The input tensor must be of + integral or Boolean types. For bool tensors, it computes the logical AND. + + Args: + input: the first input tensor + other: the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_and(torch.tensor([-1, -2, 3], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([1, 0, 3], dtype=torch.int8) + >>> torch.bitwise_and(torch.tensor([True, True, False]), torch.tensor([False, True, False])) + tensor([ False, True, False]) + """ + +@overload +def bitwise_and( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + bitwise_and(input, other, *, out=None) -> Tensor + + Computes the bitwise AND of :attr:`input` and :attr:`other`. The input tensor must be of + integral or Boolean types. For bool tensors, it computes the logical AND. + + Args: + input: the first input tensor + other: the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_and(torch.tensor([-1, -2, 3], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([1, 0, 3], dtype=torch.int8) + >>> torch.bitwise_and(torch.tensor([True, True, False]), torch.tensor([False, True, False])) + tensor([ False, True, False]) + """ + +@overload +def bitwise_left_shift( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + bitwise_left_shift(input, other, *, out=None) -> Tensor + + Computes the left arithmetic shift of :attr:`input` by :attr:`other` bits. + The input tensor must be of integral type. This operator supports + :ref:`broadcasting to a common shape ` and + :ref:`type promotion `. + + The operation applied is: + + .. math:: + \text{out}_i = \text{input}_i << \text{other}_i + + Args: + input (Tensor or Scalar): the first input tensor + other (Tensor or Scalar): the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_left_shift(torch.tensor([-1, -2, 3], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([-2, -2, 24], dtype=torch.int8) + """ + +@overload +def bitwise_left_shift(self: Number | _complex, other: Tensor) -> Tensor: + r""" + bitwise_left_shift(input, other, *, out=None) -> Tensor + + Computes the left arithmetic shift of :attr:`input` by :attr:`other` bits. + The input tensor must be of integral type. This operator supports + :ref:`broadcasting to a common shape ` and + :ref:`type promotion `. + + The operation applied is: + + .. math:: + \text{out}_i = \text{input}_i << \text{other}_i + + Args: + input (Tensor or Scalar): the first input tensor + other (Tensor or Scalar): the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_left_shift(torch.tensor([-1, -2, 3], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([-2, -2, 24], dtype=torch.int8) + """ + +@overload +def bitwise_left_shift( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + bitwise_left_shift(input, other, *, out=None) -> Tensor + + Computes the left arithmetic shift of :attr:`input` by :attr:`other` bits. + The input tensor must be of integral type. This operator supports + :ref:`broadcasting to a common shape ` and + :ref:`type promotion `. + + The operation applied is: + + .. math:: + \text{out}_i = \text{input}_i << \text{other}_i + + Args: + input (Tensor or Scalar): the first input tensor + other (Tensor or Scalar): the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_left_shift(torch.tensor([-1, -2, 3], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([-2, -2, 24], dtype=torch.int8) + """ + +def bitwise_not(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + bitwise_not(input, *, out=None) -> Tensor + + Computes the bitwise NOT of the given input tensor. The input tensor must be of + integral or Boolean types. For bool tensors, it computes the logical NOT. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_not(torch.tensor([-1, -2, 3], dtype=torch.int8)) + tensor([ 0, 1, -4], dtype=torch.int8) + """ + +@overload +def bitwise_or( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + bitwise_or(input: Tensor, other: Tensor, *, out: Optional[Tensor]) -> Tensor + + Computes the bitwise OR of :attr:`input` and :attr:`other`. The input tensor must be of + integral or Boolean types. For bool tensors, it computes the logical OR. + + Args: + input: the first input tensor + other: the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_or(torch.tensor([-1, -2, 3], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([-1, -2, 3], dtype=torch.int8) + >>> torch.bitwise_or(torch.tensor([True, True, False]), torch.tensor([False, True, False])) + tensor([ True, True, False]) + """ + +@overload +def bitwise_or(self: Number | _complex, other: Tensor) -> Tensor: + r""" + bitwise_or(input: Tensor, other: Tensor, *, out: Optional[Tensor]) -> Tensor + + Computes the bitwise OR of :attr:`input` and :attr:`other`. The input tensor must be of + integral or Boolean types. For bool tensors, it computes the logical OR. + + Args: + input: the first input tensor + other: the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_or(torch.tensor([-1, -2, 3], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([-1, -2, 3], dtype=torch.int8) + >>> torch.bitwise_or(torch.tensor([True, True, False]), torch.tensor([False, True, False])) + tensor([ True, True, False]) + """ + +@overload +def bitwise_or( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + bitwise_or(input: Tensor, other: Tensor, *, out: Optional[Tensor]) -> Tensor + + Computes the bitwise OR of :attr:`input` and :attr:`other`. The input tensor must be of + integral or Boolean types. For bool tensors, it computes the logical OR. + + Args: + input: the first input tensor + other: the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_or(torch.tensor([-1, -2, 3], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([-1, -2, 3], dtype=torch.int8) + >>> torch.bitwise_or(torch.tensor([True, True, False]), torch.tensor([False, True, False])) + tensor([ True, True, False]) + """ + +@overload +def bitwise_right_shift( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + bitwise_right_shift(input, other, *, out=None) -> Tensor + + Computes the right arithmetic shift of :attr:`input` by :attr:`other` bits. + The input tensor must be of integral type. This operator supports + :ref:`broadcasting to a common shape ` and + :ref:`type promotion `. + In any case, if the value of the right operand is negative or is greater + or equal to the number of bits in the promoted left operand, the behavior is undefined. + + The operation applied is: + + .. math:: + \text{out}_i = \text{input}_i >> \text{other}_i + + Args: + input (Tensor or Scalar): the first input tensor + other (Tensor or Scalar): the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_right_shift(torch.tensor([-2, -7, 31], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([-1, -7, 3], dtype=torch.int8) + """ + +@overload +def bitwise_right_shift(self: Number | _complex, other: Tensor) -> Tensor: + r""" + bitwise_right_shift(input, other, *, out=None) -> Tensor + + Computes the right arithmetic shift of :attr:`input` by :attr:`other` bits. + The input tensor must be of integral type. This operator supports + :ref:`broadcasting to a common shape ` and + :ref:`type promotion `. + In any case, if the value of the right operand is negative or is greater + or equal to the number of bits in the promoted left operand, the behavior is undefined. + + The operation applied is: + + .. math:: + \text{out}_i = \text{input}_i >> \text{other}_i + + Args: + input (Tensor or Scalar): the first input tensor + other (Tensor or Scalar): the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_right_shift(torch.tensor([-2, -7, 31], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([-1, -7, 3], dtype=torch.int8) + """ + +@overload +def bitwise_right_shift( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + bitwise_right_shift(input, other, *, out=None) -> Tensor + + Computes the right arithmetic shift of :attr:`input` by :attr:`other` bits. + The input tensor must be of integral type. This operator supports + :ref:`broadcasting to a common shape ` and + :ref:`type promotion `. + In any case, if the value of the right operand is negative or is greater + or equal to the number of bits in the promoted left operand, the behavior is undefined. + + The operation applied is: + + .. math:: + \text{out}_i = \text{input}_i >> \text{other}_i + + Args: + input (Tensor or Scalar): the first input tensor + other (Tensor or Scalar): the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_right_shift(torch.tensor([-2, -7, 31], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([-1, -7, 3], dtype=torch.int8) + """ + +@overload +def bitwise_xor( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + bitwise_xor(input, other, *, out=None) -> Tensor + + Computes the bitwise XOR of :attr:`input` and :attr:`other`. The input tensor must be of + integral or Boolean types. For bool tensors, it computes the logical XOR. + + Args: + input: the first input tensor + other: the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_xor(torch.tensor([-1, -2, 3], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([-2, -2, 0], dtype=torch.int8) + >>> torch.bitwise_xor(torch.tensor([True, True, False]), torch.tensor([False, True, False])) + tensor([ True, False, False]) + """ + +@overload +def bitwise_xor(self: Number | _complex, other: Tensor) -> Tensor: + r""" + bitwise_xor(input, other, *, out=None) -> Tensor + + Computes the bitwise XOR of :attr:`input` and :attr:`other`. The input tensor must be of + integral or Boolean types. For bool tensors, it computes the logical XOR. + + Args: + input: the first input tensor + other: the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_xor(torch.tensor([-1, -2, 3], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([-2, -2, 0], dtype=torch.int8) + >>> torch.bitwise_xor(torch.tensor([True, True, False]), torch.tensor([False, True, False])) + tensor([ True, False, False]) + """ + +@overload +def bitwise_xor( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + bitwise_xor(input, other, *, out=None) -> Tensor + + Computes the bitwise XOR of :attr:`input` and :attr:`other`. The input tensor must be of + integral or Boolean types. For bool tensors, it computes the logical XOR. + + Args: + input: the first input tensor + other: the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.bitwise_xor(torch.tensor([-1, -2, 3], dtype=torch.int8), torch.tensor([1, 0, 3], dtype=torch.int8)) + tensor([-2, -2, 0], dtype=torch.int8) + >>> torch.bitwise_xor(torch.tensor([True, True, False]), torch.tensor([False, True, False])) + tensor([ True, False, False]) + """ + +@overload +def blackman_window( + window_length: _int, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + blackman_window(window_length, periodic=True, *, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Blackman window function. + + .. math:: + w[n] = 0.42 - 0.5 \cos \left( \frac{2 \pi n}{N - 1} \right) + 0.08 \cos \left( \frac{4 \pi n}{N - 1} \right) + + where :math:`N` is the full window size. + + The input :attr:`window_length` is a positive integer controlling the + returned window size. :attr:`periodic` flag determines whether the returned + window trims off the last duplicate value from the symmetric window and is + ready to be used as a periodic window with functions like + :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in + above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have + ``torch.blackman_window(L, periodic=True)`` equal to + ``torch.blackman_window(L + 1, periodic=False)[:-1]``. + + .. note:: + If :attr:`window_length` :math:`=1`, the returned window contains a single value 1. + + Arguments: + window_length (int): the size of returned window + periodic (bool, optional): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window + """ + +@overload +def blackman_window( + window_length: _int, + periodic: _bool, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + blackman_window(window_length, periodic=True, *, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Blackman window function. + + .. math:: + w[n] = 0.42 - 0.5 \cos \left( \frac{2 \pi n}{N - 1} \right) + 0.08 \cos \left( \frac{4 \pi n}{N - 1} \right) + + where :math:`N` is the full window size. + + The input :attr:`window_length` is a positive integer controlling the + returned window size. :attr:`periodic` flag determines whether the returned + window trims off the last duplicate value from the symmetric window and is + ready to be used as a periodic window with functions like + :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in + above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have + ``torch.blackman_window(L, periodic=True)`` equal to + ``torch.blackman_window(L + 1, periodic=False)[:-1]``. + + .. note:: + If :attr:`window_length` :math:`=1`, the returned window contains a single value 1. + + Arguments: + window_length (int): the size of returned window + periodic (bool, optional): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window + """ + +@overload +def bmm( + input: Tensor, + mat2: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + bmm(input, mat2, out_dtype=None, *, out=None) -> Tensor + + Performs a batch matrix-matrix product of matrices stored in :attr:`input` + and :attr:`mat2`. + + :attr:`input` and :attr:`mat2` must be 3-D tensors each containing + the same number of matrices. + + If :attr:`input` is a :math:`(b \times n \times m)` tensor, :attr:`mat2` is a + :math:`(b \times m \times p)` tensor, :attr:`out` will be a + :math:`(b \times n \times p)` tensor. + + .. math:: + \text{out}_i = \text{input}_i \mathbin{@} \text{mat2}_i + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + .. note:: This function does not :ref:`broadcast `. + For broadcasting matrix products, see :func:`torch.matmul`. + + Args: + input (Tensor): the first batch of matrices to be multiplied + mat2 (Tensor): the second batch of matrices to be multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword Args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> input = torch.randn(10, 3, 4) + >>> mat2 = torch.randn(10, 4, 5) + >>> res = torch.bmm(input, mat2) + >>> res.size() + torch.Size([10, 3, 5]) + """ + +@overload +def bmm( + input: Tensor, + mat2: Tensor, + out_dtype: _dtype, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + bmm(input, mat2, out_dtype=None, *, out=None) -> Tensor + + Performs a batch matrix-matrix product of matrices stored in :attr:`input` + and :attr:`mat2`. + + :attr:`input` and :attr:`mat2` must be 3-D tensors each containing + the same number of matrices. + + If :attr:`input` is a :math:`(b \times n \times m)` tensor, :attr:`mat2` is a + :math:`(b \times m \times p)` tensor, :attr:`out` will be a + :math:`(b \times n \times p)` tensor. + + .. math:: + \text{out}_i = \text{input}_i \mathbin{@} \text{mat2}_i + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + .. note:: This function does not :ref:`broadcast `. + For broadcasting matrix products, see :func:`torch.matmul`. + + Args: + input (Tensor): the first batch of matrices to be multiplied + mat2 (Tensor): the second batch of matrices to be multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword Args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> input = torch.randn(10, 3, 4) + >>> mat2 = torch.randn(10, 4, 5) + >>> res = torch.bmm(input, mat2) + >>> res.size() + torch.Size([10, 3, 5]) + """ + +def broadcast_to(input: Tensor, size: Sequence[_int | SymInt]) -> Tensor: + r""" + broadcast_to(input, shape) -> Tensor + + Broadcasts :attr:`input` to the shape :attr:`\shape`. + Equivalent to calling ``input.expand(shape)``. See :meth:`~Tensor.expand` for details. + + Args: + input (Tensor): the input tensor. + shape (list, tuple, or :class:`torch.Size`): the new shape. + + Example:: + + >>> x = torch.tensor([1, 2, 3]) + >>> torch.broadcast_to(x, (3, 3)) + tensor([[1, 2, 3], + [1, 2, 3], + [1, 2, 3]]) + """ + +@overload +def bucketize( + input: Tensor, + boundaries: Tensor, + *, + out_int32: _bool = False, + right: _bool = False, + out: Tensor | None = None, +) -> Tensor: + r""" + bucketize(input, boundaries, *, out_int32=False, right=False, out=None) -> Tensor + + Returns the indices of the buckets to which each value in the :attr:`input` belongs, where the + boundaries of the buckets are set by :attr:`boundaries`. Return a new tensor with the same size + as :attr:`input`. If :attr:`right` is False (default), then the left boundary is open. Note that + this behavior is opposite the behavior of + `numpy.digitize `_. + More formally, the returned index satisfies the following rules: + + .. list-table:: + :widths: 15 85 + :header-rows: 1 + + * - :attr:`right` + - *returned index satisfies* + * - False + - ``boundaries[i-1] < input[m][n]...[l][x] <= boundaries[i]`` + * - True + - ``boundaries[i-1] <= input[m][n]...[l][x] < boundaries[i]`` + + Args: + input (Tensor or Scalar): N-D tensor or a Scalar containing the search value(s). + boundaries (Tensor): 1-D tensor, must contain a strictly increasing sequence, or the return value is undefined. + + Keyword args: + out_int32 (bool, optional): indicate the output data type. torch.int32 if True, torch.int64 otherwise. + Default value is False, i.e. default output data type is torch.int64. + right (bool, optional): determines the behavior for values in :attr:`boundaries`. See the table above. + out (Tensor, optional): the output tensor, must be the same size as :attr:`input` if provided. + + + Example:: + + >>> boundaries = torch.tensor([1, 3, 5, 7, 9]) + >>> boundaries + tensor([1, 3, 5, 7, 9]) + >>> v = torch.tensor([[3, 6, 9], [3, 6, 9]]) + >>> v + tensor([[3, 6, 9], + [3, 6, 9]]) + >>> torch.bucketize(v, boundaries) + tensor([[1, 3, 4], + [1, 3, 4]]) + >>> torch.bucketize(v, boundaries, right=True) + tensor([[2, 3, 5], + [2, 3, 5]]) + """ + +@overload +def bucketize( + self: Number | _complex, + boundaries: Tensor, + *, + out_int32: _bool = False, + right: _bool = False, +) -> Tensor: + r""" + bucketize(input, boundaries, *, out_int32=False, right=False, out=None) -> Tensor + + Returns the indices of the buckets to which each value in the :attr:`input` belongs, where the + boundaries of the buckets are set by :attr:`boundaries`. Return a new tensor with the same size + as :attr:`input`. If :attr:`right` is False (default), then the left boundary is open. Note that + this behavior is opposite the behavior of + `numpy.digitize `_. + More formally, the returned index satisfies the following rules: + + .. list-table:: + :widths: 15 85 + :header-rows: 1 + + * - :attr:`right` + - *returned index satisfies* + * - False + - ``boundaries[i-1] < input[m][n]...[l][x] <= boundaries[i]`` + * - True + - ``boundaries[i-1] <= input[m][n]...[l][x] < boundaries[i]`` + + Args: + input (Tensor or Scalar): N-D tensor or a Scalar containing the search value(s). + boundaries (Tensor): 1-D tensor, must contain a strictly increasing sequence, or the return value is undefined. + + Keyword args: + out_int32 (bool, optional): indicate the output data type. torch.int32 if True, torch.int64 otherwise. + Default value is False, i.e. default output data type is torch.int64. + right (bool, optional): determines the behavior for values in :attr:`boundaries`. See the table above. + out (Tensor, optional): the output tensor, must be the same size as :attr:`input` if provided. + + + Example:: + + >>> boundaries = torch.tensor([1, 3, 5, 7, 9]) + >>> boundaries + tensor([1, 3, 5, 7, 9]) + >>> v = torch.tensor([[3, 6, 9], [3, 6, 9]]) + >>> v + tensor([[3, 6, 9], + [3, 6, 9]]) + >>> torch.bucketize(v, boundaries) + tensor([[1, 3, 4], + [1, 3, 4]]) + >>> torch.bucketize(v, boundaries, right=True) + tensor([[2, 3, 5], + [2, 3, 5]]) + """ + +def can_cast(from_: _dtype, to: _dtype) -> _bool: + r""" + can_cast(from_, to) -> bool + + Determines if a type conversion is allowed under PyTorch casting rules + described in the type promotion :ref:`documentation `. + + Args: + from\_ (dtype): The original :class:`torch.dtype`. + to (dtype): The target :class:`torch.dtype`. + + Example:: + + >>> torch.can_cast(torch.double, torch.float) + True + >>> torch.can_cast(torch.float, torch.int) + False + """ + +@overload +def cat( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + dim: _int = 0, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + cat(tensors, dim=0, *, out=None) -> Tensor + + Concatenates the given sequence of tensors in :attr:`tensors` in the given dimension. + All tensors must either have the same shape (except in the concatenating + dimension) or be a 1-D empty tensor with size ``(0,)``. + + :func:`torch.cat` can be seen as an inverse operation for :func:`torch.split` + and :func:`torch.chunk`. + + :func:`torch.cat` can be best understood via examples. + + .. seealso:: + + :func:`torch.stack` concatenates the given sequence along a new dimension. + + Args: + tensors (sequence of Tensors): Non-empty tensors provided must have the same shape, + except in the cat dimension. + + dim (int, optional): the dimension over which the tensors are concatenated + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> x = torch.randn(2, 3) + >>> x + tensor([[ 0.6580, -1.0969, -0.4614], + [-0.1034, -0.5790, 0.1497]]) + >>> torch.cat((x, x, x), 0) + tensor([[ 0.6580, -1.0969, -0.4614], + [-0.1034, -0.5790, 0.1497], + [ 0.6580, -1.0969, -0.4614], + [-0.1034, -0.5790, 0.1497], + [ 0.6580, -1.0969, -0.4614], + [-0.1034, -0.5790, 0.1497]]) + >>> torch.cat((x, x, x), 1) + tensor([[ 0.6580, -1.0969, -0.4614, 0.6580, -1.0969, -0.4614, 0.6580, + -1.0969, -0.4614], + [-0.1034, -0.5790, 0.1497, -0.1034, -0.5790, 0.1497, -0.1034, + -0.5790, 0.1497]]) + """ + +@overload +def cat( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + dim: str | EllipsisType | None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + cat(tensors, dim=0, *, out=None) -> Tensor + + Concatenates the given sequence of tensors in :attr:`tensors` in the given dimension. + All tensors must either have the same shape (except in the concatenating + dimension) or be a 1-D empty tensor with size ``(0,)``. + + :func:`torch.cat` can be seen as an inverse operation for :func:`torch.split` + and :func:`torch.chunk`. + + :func:`torch.cat` can be best understood via examples. + + .. seealso:: + + :func:`torch.stack` concatenates the given sequence along a new dimension. + + Args: + tensors (sequence of Tensors): Non-empty tensors provided must have the same shape, + except in the cat dimension. + + dim (int, optional): the dimension over which the tensors are concatenated + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> x = torch.randn(2, 3) + >>> x + tensor([[ 0.6580, -1.0969, -0.4614], + [-0.1034, -0.5790, 0.1497]]) + >>> torch.cat((x, x, x), 0) + tensor([[ 0.6580, -1.0969, -0.4614], + [-0.1034, -0.5790, 0.1497], + [ 0.6580, -1.0969, -0.4614], + [-0.1034, -0.5790, 0.1497], + [ 0.6580, -1.0969, -0.4614], + [-0.1034, -0.5790, 0.1497]]) + >>> torch.cat((x, x, x), 1) + tensor([[ 0.6580, -1.0969, -0.4614, 0.6580, -1.0969, -0.4614, 0.6580, + -1.0969, -0.4614], + [-0.1034, -0.5790, 0.1497, -0.1034, -0.5790, 0.1497, -0.1034, + -0.5790, 0.1497]]) + """ + +def ccol_indices_copy( + input: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: ... +def ceil(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + ceil(input, *, out=None) -> Tensor + + Returns a new tensor with the ceil of the elements of :attr:`input`, + the smallest integer greater than or equal to each element. + + For integer inputs, follows the array-api convention of returning a + copy of the input tensor. + + .. math:: + \text{out}_{i} = \left\lceil \text{input}_{i} \right\rceil + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([-0.6341, -1.4208, -1.0900, 0.5826]) + >>> torch.ceil(a) + tensor([-0., -1., -1., 1.]) + """ + +def ceil_(input: Tensor) -> Tensor: ... +def celu(input: Tensor, alpha: Number | _complex = 1.0) -> Tensor: ... +def celu_(input: Tensor, alpha: Number | _complex = 1.0) -> Tensor: ... +def channel_shuffle(input: Tensor, groups: _int | SymInt) -> Tensor: ... +def cholesky( + input: Tensor, + upper: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + cholesky(input, upper=False, *, out=None) -> Tensor + + Computes the Cholesky decomposition of a symmetric positive-definite + matrix :math:`A` or for batches of symmetric positive-definite matrices. + + If :attr:`upper` is ``True``, the returned matrix ``U`` is upper-triangular, and + the decomposition has the form: + + .. math:: + + A = U^TU + + If :attr:`upper` is ``False``, the returned matrix ``L`` is lower-triangular, and + the decomposition has the form: + + .. math:: + + A = LL^T + + If :attr:`upper` is ``True``, and :math:`A` is a batch of symmetric positive-definite + matrices, then the returned tensor will be composed of upper-triangular Cholesky factors + of each of the individual matrices. Similarly, when :attr:`upper` is ``False``, the returned + tensor will be composed of lower-triangular Cholesky factors of each of the individual + matrices. + + .. warning:: + + :func:`torch.cholesky` is deprecated in favor of :func:`torch.linalg.cholesky` + and will be removed in a future PyTorch release. + + ``L = torch.cholesky(A)`` should be replaced with + + .. code:: python + + L = torch.linalg.cholesky(A) + + ``U = torch.cholesky(A, upper=True)`` should be replaced with + + .. code:: python + + U = torch.linalg.cholesky(A).mH + + This transform will produce equivalent results for all valid (symmetric positive definite) inputs. + + Args: + input (Tensor): the input tensor :math:`A` of size :math:`(*, n, n)` where `*` is zero or more + batch dimensions consisting of symmetric positive-definite matrices. + upper (bool, optional): flag that indicates whether to return a + upper or lower triangular matrix. Default: ``False`` + + Keyword args: + out (Tensor, optional): the output matrix + + Example:: + + >>> a = torch.randn(3, 3) + >>> a = a @ a.mT + 1e-3 # make symmetric positive-definite + >>> l = torch.cholesky(a) + >>> a + tensor([[ 2.4112, -0.7486, 1.4551], + [-0.7486, 1.3544, 0.1294], + [ 1.4551, 0.1294, 1.6724]]) + >>> l + tensor([[ 1.5528, 0.0000, 0.0000], + [-0.4821, 1.0592, 0.0000], + [ 0.9371, 0.5487, 0.7023]]) + >>> l @ l.mT + tensor([[ 2.4112, -0.7486, 1.4551], + [-0.7486, 1.3544, 0.1294], + [ 1.4551, 0.1294, 1.6724]]) + >>> a = torch.randn(3, 2, 2) # Example for batched input + >>> a = a @ a.mT + 1e-03 # make symmetric positive-definite + >>> l = torch.cholesky(a) + >>> z = l @ l.mT + >>> torch.dist(z, a) + tensor(2.3842e-07) + """ + +def cholesky_inverse( + input: Tensor, + upper: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + cholesky_inverse(L, upper=False, *, out=None) -> Tensor + + Computes the inverse of a complex Hermitian or real symmetric + positive-definite matrix given its Cholesky decomposition. + + Let :math:`A` be a complex Hermitian or real symmetric positive-definite matrix, + and :math:`L` its Cholesky decomposition such that: + + .. math:: + + A = LL^{\text{H}} + + where :math:`L^{\text{H}}` is the conjugate transpose when :math:`L` is complex, + and the transpose when :math:`L` is real-valued. + + Computes the inverse matrix :math:`A^{-1}`. + + Supports input of float, double, cfloat and cdouble dtypes. + Also supports batches of matrices, and if :math:`A` is a batch of matrices + then the output has the same batch dimensions. + + Args: + L (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions + consisting of lower or upper triangular Cholesky decompositions of + symmetric or Hermitian positive-definite matrices. + upper (bool, optional): flag that indicates whether :math:`L` is lower triangular + or upper triangular. Default: ``False`` + + Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + + Example:: + + >>> A = torch.randn(3, 3) + >>> A = A @ A.T + torch.eye(3) * 1e-3 # Creates a symmetric positive-definite matrix + >>> L = torch.linalg.cholesky(A) # Extract Cholesky decomposition + >>> torch.cholesky_inverse(L) + tensor([[ 1.9314, 1.2251, -0.0889], + [ 1.2251, 2.4439, 0.2122], + [-0.0889, 0.2122, 0.1412]]) + >>> A.inverse() + tensor([[ 1.9314, 1.2251, -0.0889], + [ 1.2251, 2.4439, 0.2122], + [-0.0889, 0.2122, 0.1412]]) + + >>> A = torch.randn(3, 2, 2, dtype=torch.complex64) + >>> A = A @ A.mH + torch.eye(2) * 1e-3 # Batch of Hermitian positive-definite matrices + >>> L = torch.linalg.cholesky(A) + >>> torch.dist(torch.inverse(A), torch.cholesky_inverse(L)) + tensor(5.6358e-7) + """ + +def cholesky_solve( + input: Tensor, + input2: Tensor, + upper: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + cholesky_solve(B, L, upper=False, *, out=None) -> Tensor + + Computes the solution of a system of linear equations with complex Hermitian + or real symmetric positive-definite lhs given its Cholesky decomposition. + + Let :math:`A` be a complex Hermitian or real symmetric positive-definite matrix, + and :math:`L` its Cholesky decomposition such that: + + .. math:: + + A = LL^{\text{H}} + + where :math:`L^{\text{H}}` is the conjugate transpose when :math:`L` is complex, + and the transpose when :math:`L` is real-valued. + + Returns the solution :math:`X` of the following linear system: + + .. math:: + + AX = B + + Supports inputs of float, double, cfloat and cdouble dtypes. + Also supports batches of matrices, and if :math:`A` or :math:`B` is a batch of matrices + then the output has the same batch dimensions. + + Args: + B (Tensor): right-hand side tensor of shape `(*, n, k)` + where :math:`*` is zero or more batch dimensions + L (Tensor): tensor of shape `(*, n, n)` where `*` is zero or more batch dimensions + consisting of lower or upper triangular Cholesky decompositions of + symmetric or Hermitian positive-definite matrices. + upper (bool, optional): flag that indicates whether :math:`L` is lower triangular + or upper triangular. Default: ``False``. + + Keyword args: + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + + Example:: + + >>> A = torch.randn(3, 3) + >>> A = A @ A.T + torch.eye(3) * 1e-3 # Creates a symmetric positive-definite matrix + >>> L = torch.linalg.cholesky(A) # Extract Cholesky decomposition + >>> B = torch.randn(3, 2) + >>> torch.cholesky_solve(B, L) + tensor([[ -8.1625, 19.6097], + [ -5.8398, 14.2387], + [ -4.3771, 10.4173]]) + >>> A.inverse() @ B + tensor([[ -8.1626, 19.6097], + [ -5.8398, 14.2387], + [ -4.3771, 10.4173]]) + + >>> A = torch.randn(3, 2, 2, dtype=torch.complex64) + >>> A = A @ A.mH + torch.eye(2) * 1e-3 # Batch of Hermitian positive-definite matrices + >>> L = torch.linalg.cholesky(A) + >>> B = torch.randn(2, 1, dtype=torch.complex64) + >>> X = torch.cholesky_solve(B, L) + >>> torch.dist(X, A.inverse() @ B) + tensor(1.6881e-5) + """ + +def choose_qparams_optimized( + input: Tensor, + numel: _int, + n_bins: _int, + ratio: _float, + bit_width: _int, +) -> tuple[Tensor, Tensor]: ... +def chunk(input: Tensor, chunks: _int, dim: _int = 0) -> tuple[Tensor, ...]: + r""" + chunk(input: Tensor, chunks: int, dim: int = 0) -> Tuple[Tensor, ...] + + Attempts to split a tensor into the specified number of chunks. Each chunk is a view of + the input tensor. + + + .. note:: + + This function may return fewer than the specified number of chunks! + + .. seealso:: + + :func:`torch.tensor_split` a function that always returns exactly the specified number of chunks + + If the tensor size along the given dimension :attr:`dim` is divisible by :attr:`chunks`, + all returned chunks will be the same size. + If the tensor size along the given dimension :attr:`dim` is not divisible by :attr:`chunks`, + all returned chunks will be the same size, except the last one. + If such division is not possible, this function may return fewer + than the specified number of chunks. + + Arguments: + input (Tensor): the tensor to split + chunks (int): number of chunks to return + dim (int): dimension along which to split the tensor + + Example: + >>> torch.arange(11).chunk(6) + (tensor([0, 1]), + tensor([2, 3]), + tensor([4, 5]), + tensor([6, 7]), + tensor([8, 9]), + tensor([10])) + >>> torch.arange(12).chunk(6) + (tensor([0, 1]), + tensor([2, 3]), + tensor([4, 5]), + tensor([6, 7]), + tensor([8, 9]), + tensor([10, 11])) + >>> torch.arange(13).chunk(6) + (tensor([0, 1, 2]), + tensor([3, 4, 5]), + tensor([6, 7, 8]), + tensor([ 9, 10, 11]), + tensor([12])) + """ + +@overload +def clamp( + input: Tensor, + min: Tensor | None = None, + max: Tensor | None = None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + clamp(input, min=None, max=None, *, out=None) -> Tensor + + Clamps all elements in :attr:`input` into the range `[` :attr:`min`, :attr:`max` `]`. + Letting min_value and max_value be :attr:`min` and :attr:`max`, respectively, this returns: + + .. math:: + y_i = \min(\max(x_i, \text{min\_value}_i), \text{max\_value}_i) + + If :attr:`min` is ``None``, there is no lower bound. + Or, if :attr:`max` is ``None`` there is no upper bound. + + + .. note:: + If :attr:`min` is greater than :attr:`max` :func:`torch.clamp(..., min, max) ` + sets all elements in :attr:`input` to the value of :attr:`max`. + + Args: + input (Tensor): the input tensor. + min (Number or Tensor, optional): lower-bound of the range to be clamped to + max (Number or Tensor, optional): upper-bound of the range to be clamped to + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([-1.7120, 0.1734, -0.0478, -0.0922]) + >>> torch.clamp(a, min=-0.5, max=0.5) + tensor([-0.5000, 0.1734, -0.0478, -0.0922]) + + >>> min = torch.linspace(-1, 1, steps=4) + >>> torch.clamp(a, min=min) + tensor([-1.0000, 0.1734, 0.3333, 1.0000]) + """ + +@overload +def clamp( + input: Tensor, + min: Number | _complex | None = None, + max: Number | _complex | None = None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + clamp(input, min=None, max=None, *, out=None) -> Tensor + + Clamps all elements in :attr:`input` into the range `[` :attr:`min`, :attr:`max` `]`. + Letting min_value and max_value be :attr:`min` and :attr:`max`, respectively, this returns: + + .. math:: + y_i = \min(\max(x_i, \text{min\_value}_i), \text{max\_value}_i) + + If :attr:`min` is ``None``, there is no lower bound. + Or, if :attr:`max` is ``None`` there is no upper bound. + + + .. note:: + If :attr:`min` is greater than :attr:`max` :func:`torch.clamp(..., min, max) ` + sets all elements in :attr:`input` to the value of :attr:`max`. + + Args: + input (Tensor): the input tensor. + min (Number or Tensor, optional): lower-bound of the range to be clamped to + max (Number or Tensor, optional): upper-bound of the range to be clamped to + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([-1.7120, 0.1734, -0.0478, -0.0922]) + >>> torch.clamp(a, min=-0.5, max=0.5) + tensor([-0.5000, 0.1734, -0.0478, -0.0922]) + + >>> min = torch.linspace(-1, 1, steps=4) + >>> torch.clamp(a, min=min) + tensor([-1.0000, 0.1734, 0.3333, 1.0000]) + """ + +@overload +def clamp_( + input: Tensor, + min: Tensor | None = None, + max: Tensor | None = None, +) -> Tensor: ... +@overload +def clamp_( + input: Tensor, + min: Number | _complex | None = None, + max: Number | _complex | None = None, +) -> Tensor: ... +@overload +def clamp_max( + input: Tensor, + max: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: ... +@overload +def clamp_max( + input: Tensor, + max: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: ... +@overload +def clamp_max_(input: Tensor, max: Tensor) -> Tensor: ... +@overload +def clamp_max_(input: Tensor, max: Number | _complex) -> Tensor: ... +@overload +def clamp_min( + input: Tensor, + min: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: ... +@overload +def clamp_min( + input: Tensor, + min: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: ... +@overload +def clamp_min_(input: Tensor, min: Tensor) -> Tensor: ... +@overload +def clamp_min_(input: Tensor, min: Number | _complex) -> Tensor: ... +@overload +def clip( + input: Tensor, + min: Tensor | None = None, + max: Tensor | None = None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + clip(input, min=None, max=None, *, out=None) -> Tensor + + Alias for :func:`torch.clamp`. + """ + +@overload +def clip( + input: Tensor, + min: Number | _complex | None = None, + max: Number | _complex | None = None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + clip(input, min=None, max=None, *, out=None) -> Tensor + + Alias for :func:`torch.clamp`. + """ + +@overload +def clip_( + input: Tensor, + min: Tensor | None = None, + max: Tensor | None = None, +) -> Tensor: ... +@overload +def clip_( + input: Tensor, + min: Number | _complex | None = None, + max: Number | _complex | None = None, +) -> Tensor: ... +def clone( + input: Tensor, + *, + memory_format: memory_format | None = None, +) -> Tensor: + r""" + clone(input, *, memory_format=torch.preserve_format) -> Tensor + + Returns a copy of :attr:`input`. + + .. note:: + + This function is differentiable, so gradients will flow back from the + result of this operation to :attr:`input`. To create a tensor without an + autograd relationship to :attr:`input` see :meth:`~Tensor.detach`. + + In addition, when ``torch.preserve_format`` is used: + If the input tensor is dense (i.e., non-overlapping strided), + its memory format (including strides) is retained. + Otherwise (e.g., a non-dense view like a stepped slice), + the output is converted to the dense (contiguous) format. + + Args: + input (Tensor): the input tensor. + + Keyword args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned tensor. Default: ``torch.preserve_format``. + """ + +def col_indices_copy(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + Performs the same operation as :func:`torch.col_indices`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def column_stack( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + column_stack(tensors, *, out=None) -> Tensor + + Creates a new tensor by horizontally stacking the tensors in :attr:`tensors`. + + Equivalent to ``torch.hstack(tensors)``, except each zero or one dimensional tensor ``t`` + in :attr:`tensors` is first reshaped into a ``(t.numel(), 1)`` column before being stacked horizontally. + + Args: + tensors (sequence of Tensors): sequence of tensors to concatenate + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([1, 2, 3]) + >>> b = torch.tensor([4, 5, 6]) + >>> torch.column_stack((a, b)) + tensor([[1, 4], + [2, 5], + [3, 6]]) + >>> a = torch.arange(5) + >>> b = torch.arange(10).reshape(5, 2) + >>> torch.column_stack((a, b, b)) + tensor([[0, 0, 1, 0, 1], + [1, 2, 3, 2, 3], + [2, 4, 5, 4, 5], + [3, 6, 7, 6, 7], + [4, 8, 9, 8, 9]]) + """ + +def combinations( + input: Tensor, + r: _int = 2, + with_replacement: _bool = False, +) -> Tensor: + r""" + combinations(input: Tensor, r: int = 2, with_replacement: bool = False) -> seq + + Compute combinations of length :math:`r` of the given tensor. The behavior is similar to + python's `itertools.combinations` when `with_replacement` is set to `False`, and + `itertools.combinations_with_replacement` when `with_replacement` is set to `True`. + + Arguments: + input (Tensor): 1D vector. + r (int, optional): number of elements to combine + with_replacement (bool, optional): whether to allow duplication in combination + + Returns: + Tensor: A tensor equivalent to converting all the input tensors into lists, do + `itertools.combinations` or `itertools.combinations_with_replacement` on these + lists, and finally convert the resulting list into tensor. + + Example:: + + >>> a = [1, 2, 3] + >>> list(itertools.combinations(a, r=2)) + [(1, 2), (1, 3), (2, 3)] + >>> list(itertools.combinations(a, r=3)) + [(1, 2, 3)] + >>> list(itertools.combinations_with_replacement(a, r=2)) + [(1, 1), (1, 2), (1, 3), (2, 2), (2, 3), (3, 3)] + >>> tensor_a = torch.tensor(a) + >>> torch.combinations(tensor_a) + tensor([[1, 2], + [1, 3], + [2, 3]]) + >>> torch.combinations(tensor_a, r=3) + tensor([[1, 2, 3]]) + >>> torch.combinations(tensor_a, with_replacement=True) + tensor([[1, 1], + [1, 2], + [1, 3], + [2, 2], + [2, 3], + [3, 3]]) + """ + +def complex( + real: Tensor, + imag: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + complex(real, imag, *, out=None) -> Tensor + + Constructs a complex tensor with its real part equal to :attr:`real` and its + imaginary part equal to :attr:`imag`. + + Args: + real (Tensor): The real part of the complex tensor. Must be half, float or double. + imag (Tensor): The imaginary part of the complex tensor. Must be same dtype + as :attr:`real`. + + Keyword args: + out (Tensor): If the inputs are ``torch.float32``, must be + ``torch.complex64``. If the inputs are ``torch.float64``, must be + ``torch.complex128``. + + Example:: + + >>> real = torch.tensor([1, 2], dtype=torch.float32) + >>> imag = torch.tensor([3, 4], dtype=torch.float32) + >>> z = torch.complex(real, imag) + >>> z + tensor([(1.+3.j), (2.+4.j)]) + >>> z.dtype + torch.complex64 + """ + +@overload +def concat( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + dim: _int = 0, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + concat(tensors, dim=0, *, out=None) -> Tensor + + Alias of :func:`torch.cat`. + """ + +@overload +def concat( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + dim: str | EllipsisType | None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + concat(tensors, dim=0, *, out=None) -> Tensor + + Alias of :func:`torch.cat`. + """ + +@overload +def concatenate( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + dim: _int = 0, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + concatenate(tensors, axis=0, out=None) -> Tensor + + Alias of :func:`torch.cat`. + """ + +@overload +def concatenate( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + dim: str | EllipsisType | None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + concatenate(tensors, axis=0, out=None) -> Tensor + + Alias of :func:`torch.cat`. + """ + +def conj(input: Tensor) -> Tensor: + r""" + conj(input) -> Tensor + + Returns a view of :attr:`input` with a flipped conjugate bit. If :attr:`input` has a non-complex dtype, + this function just returns :attr:`input`. + + .. note:: + :func:`torch.conj` performs a lazy conjugation, but the actual conjugated tensor can be materialized + at any time using :func:`torch.resolve_conj`. + + .. warning:: In the future, :func:`torch.conj` may return a non-writeable view for an :attr:`input` of + non-complex dtype. It's recommended that programs not modify the tensor returned by :func:`torch.conj_physical` + when :attr:`input` is of non-complex dtype to be compatible with this change. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> x = torch.tensor([-1 + 1j, -2 + 2j, 3 - 3j]) + >>> x.is_conj() + False + >>> y = torch.conj(x) + >>> y.is_conj() + True + """ + +def conj_physical(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + conj_physical(input, *, out=None) -> Tensor + + Computes the element-wise conjugate of the given :attr:`input` tensor. + If :attr:`input` has a non-complex dtype, this function just returns :attr:`input`. + + .. note:: + This performs the conjugate operation regardless of the fact conjugate bit is set or not. + + .. warning:: In the future, :func:`torch.conj_physical` may return a non-writeable view for an :attr:`input` of + non-complex dtype. It's recommended that programs not modify the tensor returned by :func:`torch.conj_physical` + when :attr:`input` is of non-complex dtype to be compatible with this change. + + .. math:: + \text{out}_{i} = conj(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.conj_physical(torch.tensor([-1 + 1j, -2 + 2j, 3 - 3j])) + tensor([-1 - 1j, -2 - 2j, 3 + 3j]) + """ + +def conj_physical_(input: Tensor) -> Tensor: ... +def constant_pad_nd( + input: Tensor, + pad: Sequence[_int | SymInt], + value: Number | _complex = 0, +) -> Tensor: ... +@overload +def conv1d( + input: Tensor, + weight: Tensor, + bias: Tensor | None = None, + stride: _int | SymInt | Sequence[_int | SymInt] = 1, + padding: _int | SymInt | Sequence[_int | SymInt] = 0, + dilation: _int | SymInt | Sequence[_int | SymInt] = 1, + groups: _int | SymInt = 1, +) -> Tensor: ... +@overload +def conv1d( + input: Tensor, + weight: Tensor, + bias: Tensor | None = None, + stride: _int | SymInt | Sequence[_int | SymInt] = 1, + padding: str = "valid", + dilation: _int | SymInt | Sequence[_int | SymInt] = 1, + groups: _int | SymInt = 1, +) -> Tensor: ... +@overload +def conv2d( + input: Tensor, + weight: Tensor, + bias: Tensor | None = None, + stride: _int | SymInt | Sequence[_int | SymInt] = 1, + padding: _int | SymInt | Sequence[_int | SymInt] = 0, + dilation: _int | SymInt | Sequence[_int | SymInt] = 1, + groups: _int | SymInt = 1, +) -> Tensor: ... +@overload +def conv2d( + input: Tensor, + weight: Tensor, + bias: Tensor | None = None, + stride: _int | SymInt | Sequence[_int | SymInt] = 1, + padding: str = "valid", + dilation: _int | SymInt | Sequence[_int | SymInt] = 1, + groups: _int | SymInt = 1, +) -> Tensor: ... +@overload +def conv3d( + input: Tensor, + weight: Tensor, + bias: Tensor | None = None, + stride: _int | SymInt | Sequence[_int | SymInt] = 1, + padding: _int | SymInt | Sequence[_int | SymInt] = 0, + dilation: _int | SymInt | Sequence[_int | SymInt] = 1, + groups: _int | SymInt = 1, +) -> Tensor: ... +@overload +def conv3d( + input: Tensor, + weight: Tensor, + bias: Tensor | None = None, + stride: _int | SymInt | Sequence[_int | SymInt] = 1, + padding: str = "valid", + dilation: _int | SymInt | Sequence[_int | SymInt] = 1, + groups: _int | SymInt = 1, +) -> Tensor: ... +def conv_tbc( + input: Tensor, + weight: Tensor, + bias: Tensor, + pad: _int = 0, +) -> Tensor: ... +def conv_transpose1d( + input: Tensor, + weight: Tensor, + bias: Tensor | None = None, + stride: _int | SymInt | Sequence[_int | SymInt] = 1, + padding: _int | SymInt | Sequence[_int | SymInt] = 0, + output_padding: _int | SymInt | Sequence[_int | SymInt] = 0, + groups: _int | SymInt = 1, + dilation: _int | SymInt | Sequence[_int | SymInt] = 1, +) -> Tensor: ... +def conv_transpose2d( + input: Tensor, + weight: Tensor, + bias: Tensor | None = None, + stride: _int | SymInt | Sequence[_int | SymInt] = 1, + padding: _int | SymInt | Sequence[_int | SymInt] = 0, + output_padding: _int | SymInt | Sequence[_int | SymInt] = 0, + groups: _int | SymInt = 1, + dilation: _int | SymInt | Sequence[_int | SymInt] = 1, +) -> Tensor: ... +def conv_transpose3d( + input: Tensor, + weight: Tensor, + bias: Tensor | None = None, + stride: _int | SymInt | Sequence[_int | SymInt] = 1, + padding: _int | SymInt | Sequence[_int | SymInt] = 0, + output_padding: _int | SymInt | Sequence[_int | SymInt] = 0, + groups: _int | SymInt = 1, + dilation: _int | SymInt | Sequence[_int | SymInt] = 1, +) -> Tensor: ... +def convolution( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + stride: Sequence[_int | SymInt], + padding: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + transposed: _bool, + output_padding: Sequence[_int | SymInt], + groups: _int | SymInt, +) -> Tensor: ... +@overload +def copysign( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + copysign(input, other, *, out=None) -> Tensor + + Create a new floating-point tensor with the magnitude of :attr:`input` and the sign of :attr:`other`, elementwise. + + .. math:: + \text{out}_{i} = \begin{cases} + -|\text{input}_{i}| & \text{if } \text{other}_{i} \leq -0.0 \\ + |\text{input}_{i}| & \text{if } \text{other}_{i} \geq 0.0 \\ + \end{cases} + + + Supports :ref:`broadcasting to a common shape `, + and integer and float inputs. + + Args: + input (Tensor): magnitudes. + other (Tensor or Number): contains value(s) whose signbit(s) are + applied to the magnitudes in :attr:`input`. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(5) + >>> a + tensor([-1.2557, -0.0026, -0.5387, 0.4740, -0.9244]) + >>> torch.copysign(a, 1) + tensor([1.2557, 0.0026, 0.5387, 0.4740, 0.9244]) + >>> a = torch.randn(4, 4) + >>> a + tensor([[ 0.7079, 0.2778, -1.0249, 0.5719], + [-0.0059, -0.2600, -0.4475, -1.3948], + [ 0.3667, -0.9567, -2.5757, -0.1751], + [ 0.2046, -0.0742, 0.2998, -0.1054]]) + >>> b = torch.randn(4) + tensor([ 0.2373, 0.3120, 0.3190, -1.1128]) + >>> torch.copysign(a, b) + tensor([[ 0.7079, 0.2778, 1.0249, -0.5719], + [ 0.0059, 0.2600, 0.4475, -1.3948], + [ 0.3667, 0.9567, 2.5757, -0.1751], + [ 0.2046, 0.0742, 0.2998, -0.1054]]) + >>> a = torch.tensor([1.]) + >>> b = torch.tensor([-0.]) + >>> torch.copysign(a, b) + tensor([-1.]) + + .. note:: + copysign handles signed zeros. If the other argument has a negative zero (-0), + the corresponding output value will be negative. + """ + +@overload +def copysign( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + copysign(input, other, *, out=None) -> Tensor + + Create a new floating-point tensor with the magnitude of :attr:`input` and the sign of :attr:`other`, elementwise. + + .. math:: + \text{out}_{i} = \begin{cases} + -|\text{input}_{i}| & \text{if } \text{other}_{i} \leq -0.0 \\ + |\text{input}_{i}| & \text{if } \text{other}_{i} \geq 0.0 \\ + \end{cases} + + + Supports :ref:`broadcasting to a common shape `, + and integer and float inputs. + + Args: + input (Tensor): magnitudes. + other (Tensor or Number): contains value(s) whose signbit(s) are + applied to the magnitudes in :attr:`input`. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(5) + >>> a + tensor([-1.2557, -0.0026, -0.5387, 0.4740, -0.9244]) + >>> torch.copysign(a, 1) + tensor([1.2557, 0.0026, 0.5387, 0.4740, 0.9244]) + >>> a = torch.randn(4, 4) + >>> a + tensor([[ 0.7079, 0.2778, -1.0249, 0.5719], + [-0.0059, -0.2600, -0.4475, -1.3948], + [ 0.3667, -0.9567, -2.5757, -0.1751], + [ 0.2046, -0.0742, 0.2998, -0.1054]]) + >>> b = torch.randn(4) + tensor([ 0.2373, 0.3120, 0.3190, -1.1128]) + >>> torch.copysign(a, b) + tensor([[ 0.7079, 0.2778, 1.0249, -0.5719], + [ 0.0059, 0.2600, 0.4475, -1.3948], + [ 0.3667, 0.9567, 2.5757, -0.1751], + [ 0.2046, 0.0742, 0.2998, -0.1054]]) + >>> a = torch.tensor([1.]) + >>> b = torch.tensor([-0.]) + >>> torch.copysign(a, b) + tensor([-1.]) + + .. note:: + copysign handles signed zeros. If the other argument has a negative zero (-0), + the corresponding output value will be negative. + """ + +def corrcoef(input: Tensor) -> Tensor: + r""" + corrcoef(input) -> Tensor + + Estimates the Pearson product-moment correlation coefficient matrix of the variables given by the :attr:`input` matrix, + where rows are the variables and columns are the observations. + + .. note:: + + The correlation coefficient matrix R is computed using the covariance matrix C as given by + :math:`R_{ij} = \frac{ C_{ij} } { \sqrt{ C_{ii} * C_{jj} } }` + + .. note:: + + Due to floating point rounding, the resulting array may not be Hermitian and its diagonal elements may not be 1. + The real and imaginary values are clipped to the interval [-1, 1] in an attempt to improve this situation. + + Args: + input (Tensor): A 2D matrix containing multiple variables and observations, or a + Scalar or 1D vector representing a single variable. + + Returns: + (Tensor) The correlation coefficient matrix of the variables. + + .. seealso:: + + :func:`torch.cov` covariance matrix. + + Example:: + + >>> x = torch.tensor([[0, 1, 2], [2, 1, 0]]) + >>> torch.corrcoef(x) + tensor([[ 1., -1.], + [-1., 1.]]) + >>> x = torch.randn(2, 4) + >>> x + tensor([[-0.2678, -0.0908, -0.3766, 0.2780], + [-0.5812, 0.1535, 0.2387, 0.2350]]) + >>> torch.corrcoef(x) + tensor([[1.0000, 0.3582], + [0.3582, 1.0000]]) + >>> torch.corrcoef(x[0]) + tensor(1.) + """ + +def cos(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + cos(input, *, out=None) -> Tensor + + Returns a new tensor with the cosine of the elements of :attr:`input` given in radians. + + .. math:: + \text{out}_{i} = \cos(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([ 1.4309, 1.2706, -0.8562, 0.9796]) + >>> torch.cos(a) + tensor([ 0.1395, 0.2957, 0.6553, 0.5574]) + """ + +def cos_(input: Tensor) -> Tensor: ... +def cosh(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + cosh(input, *, out=None) -> Tensor + + Returns a new tensor with the hyperbolic cosine of the elements of + :attr:`input`. + + .. math:: + \text{out}_{i} = \cosh(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([ 0.1632, 1.1835, -0.6979, -0.7325]) + >>> torch.cosh(a) + tensor([ 1.0133, 1.7860, 1.2536, 1.2805]) + + .. note:: + When :attr:`input` is on the CPU, the implementation of torch.cosh may use + the Sleef library, which rounds very large results to infinity or negative + infinity. See `here `_ for details. + """ + +def cosh_(input: Tensor) -> Tensor: ... +def cosine_embedding_loss( + input1: Tensor, + input2: Tensor, + target: Tensor, + margin: _float = 0.0, + reduction: _int = 1, +) -> Tensor: ... +def cosine_similarity( + x1: Tensor, + x2: Tensor, + dim: _int = 1, + eps: _float = 1e-08, +) -> Tensor: ... +@overload +def count_nonzero(input: Tensor, dim: _int | None = None) -> Tensor: + r""" + count_nonzero(input, dim=None) -> Tensor + + Counts the number of non-zero values in the tensor :attr:`input` along the given :attr:`dim`. + If no dim is specified then all non-zeros in the tensor are counted. + + Args: + input (Tensor): the input tensor. + dim (int or tuple of ints, optional): Dim or tuple of dims along which to count non-zeros. + + Example:: + + >>> x = torch.zeros(3,3) + >>> x[torch.randn(3,3) > 0.5] = 1 + >>> x + tensor([[0., 1., 1.], + [0., 0., 0.], + [0., 0., 1.]]) + >>> torch.count_nonzero(x) + tensor(3) + >>> torch.count_nonzero(x, dim=0) + tensor([0, 1, 2]) + """ + +@overload +def count_nonzero(input: Tensor, dim: _size) -> Tensor: + r""" + count_nonzero(input, dim=None) -> Tensor + + Counts the number of non-zero values in the tensor :attr:`input` along the given :attr:`dim`. + If no dim is specified then all non-zeros in the tensor are counted. + + Args: + input (Tensor): the input tensor. + dim (int or tuple of ints, optional): Dim or tuple of dims along which to count non-zeros. + + Example:: + + >>> x = torch.zeros(3,3) + >>> x[torch.randn(3,3) > 0.5] = 1 + >>> x + tensor([[0., 1., 1.], + [0., 0., 0.], + [0., 0., 1.]]) + >>> torch.count_nonzero(x) + tensor(3) + >>> torch.count_nonzero(x, dim=0) + tensor([0, 1, 2]) + """ + +def cov( + input: Tensor, + *, + correction: _int = 1, + fweights: Tensor | None = None, + aweights: Tensor | None = None, +) -> Tensor: + r""" + cov(input, *, correction=1, fweights=None, aweights=None) -> Tensor + + Estimates the covariance matrix of the variables given by the :attr:`input` matrix, where rows are + the variables and columns are the observations. + + A covariance matrix is a square matrix giving the covariance of each pair of variables. The diagonal contains + the variance of each variable (covariance of a variable with itself). By definition, if :attr:`input` represents + a single variable (Scalar or 1D) then its variance is returned. + + The sample covariance of the variables :math:`x` and :math:`y` is given by: + + .. math:: + \text{cov}(x,y) = \frac{\sum^{N}_{i = 1}(x_{i} - \bar{x})(y_{i} - \bar{y})}{\max(0,~N~-~\delta N)} + + where :math:`\bar{x}` and :math:`\bar{y}` are the simple means of the :math:`x` and :math:`y` respectively, and + :math:`\delta N` is the :attr:`correction`. + + If :attr:`fweights` and/or :attr:`aweights` are provided, the weighted covariance + is calculated, which is given by: + + .. math:: + \text{cov}_w(x,y) = \frac{\sum^{N}_{i = 1}w_i(x_{i} - \mu_x^*)(y_{i} - \mu_y^*)} + {\max(0,~\sum^{N}_{i = 1}w_i~-~\frac{\sum^{N}_{i = 1}w_ia_i}{\sum^{N}_{i = 1}w_i}~\delta N)} + + where :math:`w` denotes :attr:`fweights` or :attr:`aweights` (``f`` and ``a`` for brevity) based on whichever is + provided, or :math:`w = f \times a` if both are provided, and + :math:`\mu_x^* = \frac{\sum^{N}_{i = 1}w_ix_{i} }{\sum^{N}_{i = 1}w_i}` is the weighted mean of the variable. If not + provided, ``f`` and/or ``a`` can be seen as a :math:`\mathbb{1}` vector of appropriate size. + + Args: + input (Tensor): A 2D matrix containing multiple variables and observations, or a + Scalar or 1D vector representing a single variable. + + Keyword Args: + correction (int, optional): difference between the sample size and sample degrees of freedom. + Defaults to Bessel's correction, ``correction = 1`` which returns the unbiased estimate, + even if both :attr:`fweights` and :attr:`aweights` are specified. ``correction = 0`` + will return the simple average. Defaults to ``1``. + fweights (tensor, optional): A Scalar or 1D tensor of observation vector frequencies representing the number of + times each observation should be repeated. Its numel must equal the number of columns of :attr:`input`. + Must have integral dtype. Ignored if ``None``. Defaults to ``None``. + aweights (tensor, optional): A Scalar or 1D array of observation vector weights. + These relative weights are typically large for observations considered "important" and smaller for + observations considered less "important". Its numel must equal the number of columns of :attr:`input`. + Must have floating point dtype. Ignored if ``None``. Defaults to ``None``. + + Returns: + (Tensor) The covariance matrix of the variables. + + .. seealso:: + + :func:`torch.corrcoef` normalized covariance matrix. + + Example:: + + >>> x = torch.tensor([[0, 2], [1, 1], [2, 0]]).T + >>> x + tensor([[0, 1, 2], + [2, 1, 0]]) + >>> torch.cov(x) + tensor([[ 1., -1.], + [-1., 1.]]) + >>> torch.cov(x, correction=0) + tensor([[ 0.6667, -0.6667], + [-0.6667, 0.6667]]) + >>> fw = torch.randint(1, 10, (3,)) + >>> fw + tensor([1, 6, 9]) + >>> aw = torch.rand(3) + >>> aw + tensor([0.4282, 0.0255, 0.4144]) + >>> torch.cov(x, fweights=fw, aweights=aw) + tensor([[ 0.4169, -0.4169], + [-0.4169, 0.4169]]) + """ + +def cross( + input: Tensor, + other: Tensor, + dim: _int | None = None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + cross(input, other, dim=None, *, out=None) -> Tensor + + + Returns the cross product of vectors in dimension :attr:`dim` of :attr:`input` + and :attr:`other`. + + Supports input of float, double, cfloat and cdouble dtypes. Also supports batches + of vectors, for which it computes the product along the dimension :attr:`dim`. + In this case, the output has the same batch dimensions as the inputs. + + .. warning:: + If :attr:`dim` is not given, it defaults to the first dimension found + with the size 3. Note that this might be unexpected. + + This behavior is deprecated and will be changed to match that of :func:`torch.linalg.cross` + in a future release. + + .. seealso:: + :func:`torch.linalg.cross` which has dim=-1 as default. + + + Args: + input (Tensor): the input tensor. + other (Tensor): the second input tensor + dim (int, optional): the dimension to take the cross-product in. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4, 3) + >>> a + tensor([[-0.3956, 1.1455, 1.6895], + [-0.5849, 1.3672, 0.3599], + [-1.1626, 0.7180, -0.0521], + [-0.1339, 0.9902, -2.0225]]) + >>> b = torch.randn(4, 3) + >>> b + tensor([[-0.0257, -1.4725, -1.2251], + [-1.1479, -0.7005, -1.9757], + [-1.3904, 0.3726, -1.1836], + [-0.9688, -0.7153, 0.2159]]) + >>> torch.cross(a, b, dim=1) + tensor([[ 1.0844, -0.5281, 0.6120], + [-2.4490, -1.5687, 1.9792], + [-0.8304, -1.3037, 0.5650], + [-1.2329, 1.9883, 1.0551]]) + >>> torch.cross(a, b) + tensor([[ 1.0844, -0.5281, 0.6120], + [-2.4490, -1.5687, 1.9792], + [-0.8304, -1.3037, 0.5650], + [-1.2329, 1.9883, 1.0551]]) + """ + +def crow_indices_copy( + input: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.crow_indices`, but all output tensors + are freshly created instead of aliasing the input. + """ + +@overload +def ctc_loss( + log_probs: Tensor, + targets: Tensor, + input_lengths: _size, + target_lengths: _size, + blank: _int = 0, + reduction: _int = 1, + zero_infinity: _bool = False, +) -> Tensor: ... +@overload +def ctc_loss( + log_probs: Tensor, + targets: Tensor, + input_lengths: Tensor, + target_lengths: Tensor, + blank: _int = 0, + reduction: _int = 1, + zero_infinity: _bool = False, +) -> Tensor: ... +def cudnn_affine_grid_generator( + theta: Tensor, + N: _int, + C: _int, + H: _int, + W: _int, +) -> Tensor: ... +def cudnn_batch_norm( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + running_mean: Tensor | None, + running_var: Tensor | None, + training: _bool, + exponential_average_factor: _float, + epsilon: _float, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> tuple[Tensor, Tensor, Tensor, Tensor]: ... +def cudnn_convolution( + input: Tensor, + weight: Tensor, + padding: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + groups: _int | SymInt, + benchmark: _bool, + deterministic: _bool, + allow_tf32: _bool, + *, + out: Tensor | None = None, +) -> Tensor: ... +def cudnn_convolution_add_relu( + input: Tensor, + weight: Tensor, + z: Tensor, + alpha: Number | _complex | None, + bias: Tensor | None, + stride: Sequence[_int | SymInt], + padding: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + groups: _int | SymInt, +) -> Tensor: ... +def cudnn_convolution_relu( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + stride: Sequence[_int | SymInt], + padding: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + groups: _int | SymInt, +) -> Tensor: ... +def cudnn_convolution_transpose( + input: Tensor, + weight: Tensor, + padding: Sequence[_int | SymInt], + output_padding: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + groups: _int | SymInt, + benchmark: _bool, + deterministic: _bool, + allow_tf32: _bool, +) -> Tensor: ... +def cudnn_grid_sampler(input: Tensor, grid: Tensor) -> Tensor: ... +def cudnn_is_acceptable(input: Tensor) -> _bool: ... +@overload +def cummax( + input: Tensor, + dim: _int, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.cummax: + r""" + cummax(input, dim, *, out=None) -> (Tensor, LongTensor) + Returns a namedtuple ``(values, indices)`` where ``values`` is the cumulative maximum of + elements of :attr:`input` in the dimension :attr:`dim`. And ``indices`` is the index + location of each maximum value found in the dimension :attr:`dim`. + + .. math:: + y_i = max(x_1, x_2, x_3, \dots, x_i) + + Args: + input (Tensor): the input tensor. + dim (int): the dimension to do the operation over + + Keyword args: + out (tuple, optional): the result tuple of two output tensors (values, indices) + + Example:: + + >>> a = torch.randn(10) + >>> a + tensor([-0.3449, -1.5447, 0.0685, -1.5104, -1.1706, 0.2259, 1.4696, -1.3284, + 1.9946, -0.8209]) + >>> torch.cummax(a, dim=0) + torch.return_types.cummax( + values=tensor([-0.3449, -0.3449, 0.0685, 0.0685, 0.0685, 0.2259, 1.4696, 1.4696, + 1.9946, 1.9946]), + indices=tensor([0, 0, 2, 2, 2, 5, 6, 6, 8, 8])) + """ + +@overload +def cummax( + input: Tensor, + dim: str | EllipsisType | None, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.cummax: + r""" + cummax(input, dim, *, out=None) -> (Tensor, LongTensor) + Returns a namedtuple ``(values, indices)`` where ``values`` is the cumulative maximum of + elements of :attr:`input` in the dimension :attr:`dim`. And ``indices`` is the index + location of each maximum value found in the dimension :attr:`dim`. + + .. math:: + y_i = max(x_1, x_2, x_3, \dots, x_i) + + Args: + input (Tensor): the input tensor. + dim (int): the dimension to do the operation over + + Keyword args: + out (tuple, optional): the result tuple of two output tensors (values, indices) + + Example:: + + >>> a = torch.randn(10) + >>> a + tensor([-0.3449, -1.5447, 0.0685, -1.5104, -1.1706, 0.2259, 1.4696, -1.3284, + 1.9946, -0.8209]) + >>> torch.cummax(a, dim=0) + torch.return_types.cummax( + values=tensor([-0.3449, -0.3449, 0.0685, 0.0685, 0.0685, 0.2259, 1.4696, 1.4696, + 1.9946, 1.9946]), + indices=tensor([0, 0, 2, 2, 2, 5, 6, 6, 8, 8])) + """ + +@overload +def cummin( + input: Tensor, + dim: _int, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.cummin: + r""" + cummin(input, dim, *, out=None) -> (Tensor, LongTensor) + Returns a namedtuple ``(values, indices)`` where ``values`` is the cumulative minimum of + elements of :attr:`input` in the dimension :attr:`dim`. And ``indices`` is the index + location of each maximum value found in the dimension :attr:`dim`. + + .. math:: + y_i = min(x_1, x_2, x_3, \dots, x_i) + + Args: + input (Tensor): the input tensor. + dim (int): the dimension to do the operation over + + Keyword args: + out (tuple, optional): the result tuple of two output tensors (values, indices) + + Example:: + + >>> a = torch.randn(10) + >>> a + tensor([-0.2284, -0.6628, 0.0975, 0.2680, -1.3298, -0.4220, -0.3885, 1.1762, + 0.9165, 1.6684]) + >>> torch.cummin(a, dim=0) + torch.return_types.cummin( + values=tensor([-0.2284, -0.6628, -0.6628, -0.6628, -1.3298, -1.3298, -1.3298, -1.3298, + -1.3298, -1.3298]), + indices=tensor([0, 1, 1, 1, 4, 4, 4, 4, 4, 4])) + """ + +@overload +def cummin( + input: Tensor, + dim: str | EllipsisType | None, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.cummin: + r""" + cummin(input, dim, *, out=None) -> (Tensor, LongTensor) + Returns a namedtuple ``(values, indices)`` where ``values`` is the cumulative minimum of + elements of :attr:`input` in the dimension :attr:`dim`. And ``indices`` is the index + location of each maximum value found in the dimension :attr:`dim`. + + .. math:: + y_i = min(x_1, x_2, x_3, \dots, x_i) + + Args: + input (Tensor): the input tensor. + dim (int): the dimension to do the operation over + + Keyword args: + out (tuple, optional): the result tuple of two output tensors (values, indices) + + Example:: + + >>> a = torch.randn(10) + >>> a + tensor([-0.2284, -0.6628, 0.0975, 0.2680, -1.3298, -0.4220, -0.3885, 1.1762, + 0.9165, 1.6684]) + >>> torch.cummin(a, dim=0) + torch.return_types.cummin( + values=tensor([-0.2284, -0.6628, -0.6628, -0.6628, -1.3298, -1.3298, -1.3298, -1.3298, + -1.3298, -1.3298]), + indices=tensor([0, 1, 1, 1, 4, 4, 4, 4, 4, 4])) + """ + +@overload +def cumprod( + input: Tensor, + dim: _int, + *, + dtype: _dtype | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + cumprod(input, dim, *, dtype=None, out=None) -> Tensor + + Returns the cumulative product of elements of :attr:`input` in the dimension + :attr:`dim`. + + For example, if :attr:`input` is a vector of size N, the result will also be + a vector of size N, with elements. + + .. math:: + y_i = x_1 \times x_2\times x_3\times \dots \times x_i + + Args: + input (Tensor): the input tensor. + dim (int): the dimension to do the operation over + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(10) + >>> a + tensor([ 0.6001, 0.2069, -0.1919, 0.9792, 0.6727, 1.0062, 0.4126, + -0.2129, -0.4206, 0.1968]) + >>> torch.cumprod(a, dim=0) + tensor([ 0.6001, 0.1241, -0.0238, -0.0233, -0.0157, -0.0158, -0.0065, + 0.0014, -0.0006, -0.0001]) + + >>> a[5] = 0.0 + >>> torch.cumprod(a, dim=0) + tensor([ 0.6001, 0.1241, -0.0238, -0.0233, -0.0157, -0.0000, -0.0000, + 0.0000, -0.0000, -0.0000]) + """ + +@overload +def cumprod( + input: Tensor, + dim: str | EllipsisType | None, + *, + dtype: _dtype | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + cumprod(input, dim, *, dtype=None, out=None) -> Tensor + + Returns the cumulative product of elements of :attr:`input` in the dimension + :attr:`dim`. + + For example, if :attr:`input` is a vector of size N, the result will also be + a vector of size N, with elements. + + .. math:: + y_i = x_1 \times x_2\times x_3\times \dots \times x_i + + Args: + input (Tensor): the input tensor. + dim (int): the dimension to do the operation over + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(10) + >>> a + tensor([ 0.6001, 0.2069, -0.1919, 0.9792, 0.6727, 1.0062, 0.4126, + -0.2129, -0.4206, 0.1968]) + >>> torch.cumprod(a, dim=0) + tensor([ 0.6001, 0.1241, -0.0238, -0.0233, -0.0157, -0.0158, -0.0065, + 0.0014, -0.0006, -0.0001]) + + >>> a[5] = 0.0 + >>> torch.cumprod(a, dim=0) + tensor([ 0.6001, 0.1241, -0.0238, -0.0233, -0.0157, -0.0000, -0.0000, + 0.0000, -0.0000, -0.0000]) + """ + +@overload +def cumsum( + input: Tensor, + dim: _int, + *, + dtype: _dtype | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + cumsum(input, dim, *, dtype=None, out=None) -> Tensor + + Returns the cumulative sum of elements of :attr:`input` in the dimension + :attr:`dim`. + + For example, if :attr:`input` is a vector of size N, the result will also be + a vector of size N, with elements. + + .. math:: + y_i = x_1 + x_2 + x_3 + \dots + x_i + + Args: + input (Tensor): the input tensor. + dim (int): the dimension to do the operation over + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randint(1, 20, (10,)) + >>> a + tensor([13, 7, 3, 10, 13, 3, 15, 10, 9, 10]) + >>> torch.cumsum(a, dim=0) + tensor([13, 20, 23, 33, 46, 49, 64, 74, 83, 93]) + """ + +@overload +def cumsum( + input: Tensor, + dim: str | EllipsisType | None, + *, + dtype: _dtype | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + cumsum(input, dim, *, dtype=None, out=None) -> Tensor + + Returns the cumulative sum of elements of :attr:`input` in the dimension + :attr:`dim`. + + For example, if :attr:`input` is a vector of size N, the result will also be + a vector of size N, with elements. + + .. math:: + y_i = x_1 + x_2 + x_3 + \dots + x_i + + Args: + input (Tensor): the input tensor. + dim (int): the dimension to do the operation over + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randint(1, 20, (10,)) + >>> a + tensor([13, 7, 3, 10, 13, 3, 15, 10, 9, 10]) + >>> torch.cumsum(a, dim=0) + tensor([13, 20, 23, 33, 46, 49, 64, 74, 83, 93]) + """ + +@overload +def cumulative_trapezoid(y: Tensor, x: Tensor, *, dim: _int = -1) -> Tensor: + r""" + cumulative_trapezoid(y, x=None, *, dx=None, dim=-1) -> Tensor + + Cumulatively computes the `trapezoidal rule `_ + along :attr:`dim`. By default the spacing between elements is assumed to be 1, but + :attr:`dx` can be used to specify a different constant spacing, and :attr:`x` can be + used to specify arbitrary spacing along :attr:`dim`. + + For more details, please read :func:`torch.trapezoid`. The difference between :func:`torch.trapezoid` + and this function is that, :func:`torch.trapezoid` returns a value for each integration, + where as this function returns a cumulative value for every spacing within the integration. This + is analogous to how `.sum` returns a value and `.cumsum` returns a cumulative sum. + + Arguments: + y (Tensor): Values to use when computing the trapezoidal rule. + x (Tensor): If specified, defines spacing between values as specified above. + + Keyword arguments: + dx (float): constant spacing between values. If neither :attr:`x` or :attr:`dx` + are specified then this defaults to 1. Effectively multiplies the result by its value. + dim (int): The dimension along which to compute the trapezoidal rule. + The last (inner-most) dimension by default. + + Examples:: + + >>> # Cumulatively computes the trapezoidal rule in 1D, spacing is implicitly 1. + >>> y = torch.tensor([1, 5, 10]) + >>> torch.cumulative_trapezoid(y) + tensor([3., 10.5]) + + >>> # Computes the same trapezoidal rule directly up to each element to verify + >>> (1 + 5) / 2 + 3.0 + >>> (1 + 10 + 10) / 2 + 10.5 + + >>> # Cumulatively computes the trapezoidal rule in 1D with constant spacing of 2 + >>> # NOTE: the result is the same as before, but multiplied by 2 + >>> torch.cumulative_trapezoid(y, dx=2) + tensor([6., 21.]) + + >>> # Cumulatively computes the trapezoidal rule in 1D with arbitrary spacing + >>> x = torch.tensor([1, 3, 6]) + >>> torch.cumulative_trapezoid(y, x) + tensor([6., 28.5]) + + >>> # Computes the same trapezoidal rule directly up to each element to verify + >>> ((3 - 1) * (1 + 5)) / 2 + 6.0 + >>> ((3 - 1) * (1 + 5) + (6 - 3) * (5 + 10)) / 2 + 28.5 + + >>> # Cumulatively computes the trapezoidal rule for each row of a 3x3 matrix + >>> y = torch.arange(9).reshape(3, 3) + tensor([[0, 1, 2], + [3, 4, 5], + [6, 7, 8]]) + >>> torch.cumulative_trapezoid(y) + tensor([[ 0.5, 2.], + [ 3.5, 8.], + [ 6.5, 14.]]) + + >>> # Cumulatively computes the trapezoidal rule for each column of the matrix + >>> torch.cumulative_trapezoid(y, dim=0) + tensor([[ 1.5, 2.5, 3.5], + [ 6.0, 8.0, 10.0]]) + + >>> # Cumulatively computes the trapezoidal rule for each row of a 3x3 ones matrix + >>> # with the same arbitrary spacing + >>> y = torch.ones(3, 3) + >>> x = torch.tensor([1, 3, 6]) + >>> torch.cumulative_trapezoid(y, x) + tensor([[2., 5.], + [2., 5.], + [2., 5.]]) + + >>> # Cumulatively computes the trapezoidal rule for each row of a 3x3 ones matrix + >>> # with different arbitrary spacing per row + >>> y = torch.ones(3, 3) + >>> x = torch.tensor([[1, 2, 3], [1, 3, 5], [1, 4, 7]]) + >>> torch.cumulative_trapezoid(y, x) + tensor([[1., 2.], + [2., 4.], + [3., 6.]]) + """ + +@overload +def cumulative_trapezoid( + y: Tensor, + *, + dx: Number | _complex = 1, + dim: _int = -1, +) -> Tensor: + r""" + cumulative_trapezoid(y, x=None, *, dx=None, dim=-1) -> Tensor + + Cumulatively computes the `trapezoidal rule `_ + along :attr:`dim`. By default the spacing between elements is assumed to be 1, but + :attr:`dx` can be used to specify a different constant spacing, and :attr:`x` can be + used to specify arbitrary spacing along :attr:`dim`. + + For more details, please read :func:`torch.trapezoid`. The difference between :func:`torch.trapezoid` + and this function is that, :func:`torch.trapezoid` returns a value for each integration, + where as this function returns a cumulative value for every spacing within the integration. This + is analogous to how `.sum` returns a value and `.cumsum` returns a cumulative sum. + + Arguments: + y (Tensor): Values to use when computing the trapezoidal rule. + x (Tensor): If specified, defines spacing between values as specified above. + + Keyword arguments: + dx (float): constant spacing between values. If neither :attr:`x` or :attr:`dx` + are specified then this defaults to 1. Effectively multiplies the result by its value. + dim (int): The dimension along which to compute the trapezoidal rule. + The last (inner-most) dimension by default. + + Examples:: + + >>> # Cumulatively computes the trapezoidal rule in 1D, spacing is implicitly 1. + >>> y = torch.tensor([1, 5, 10]) + >>> torch.cumulative_trapezoid(y) + tensor([3., 10.5]) + + >>> # Computes the same trapezoidal rule directly up to each element to verify + >>> (1 + 5) / 2 + 3.0 + >>> (1 + 10 + 10) / 2 + 10.5 + + >>> # Cumulatively computes the trapezoidal rule in 1D with constant spacing of 2 + >>> # NOTE: the result is the same as before, but multiplied by 2 + >>> torch.cumulative_trapezoid(y, dx=2) + tensor([6., 21.]) + + >>> # Cumulatively computes the trapezoidal rule in 1D with arbitrary spacing + >>> x = torch.tensor([1, 3, 6]) + >>> torch.cumulative_trapezoid(y, x) + tensor([6., 28.5]) + + >>> # Computes the same trapezoidal rule directly up to each element to verify + >>> ((3 - 1) * (1 + 5)) / 2 + 6.0 + >>> ((3 - 1) * (1 + 5) + (6 - 3) * (5 + 10)) / 2 + 28.5 + + >>> # Cumulatively computes the trapezoidal rule for each row of a 3x3 matrix + >>> y = torch.arange(9).reshape(3, 3) + tensor([[0, 1, 2], + [3, 4, 5], + [6, 7, 8]]) + >>> torch.cumulative_trapezoid(y) + tensor([[ 0.5, 2.], + [ 3.5, 8.], + [ 6.5, 14.]]) + + >>> # Cumulatively computes the trapezoidal rule for each column of the matrix + >>> torch.cumulative_trapezoid(y, dim=0) + tensor([[ 1.5, 2.5, 3.5], + [ 6.0, 8.0, 10.0]]) + + >>> # Cumulatively computes the trapezoidal rule for each row of a 3x3 ones matrix + >>> # with the same arbitrary spacing + >>> y = torch.ones(3, 3) + >>> x = torch.tensor([1, 3, 6]) + >>> torch.cumulative_trapezoid(y, x) + tensor([[2., 5.], + [2., 5.], + [2., 5.]]) + + >>> # Cumulatively computes the trapezoidal rule for each row of a 3x3 ones matrix + >>> # with different arbitrary spacing per row + >>> y = torch.ones(3, 3) + >>> x = torch.tensor([[1, 2, 3], [1, 3, 5], [1, 4, 7]]) + >>> torch.cumulative_trapezoid(y, x) + tensor([[1., 2.], + [2., 4.], + [3., 6.]]) + """ + +def deg2rad(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + deg2rad(input, *, out=None) -> Tensor + + Returns a new tensor with each of the elements of :attr:`input` + converted from angles in degrees to radians. + + Args: + input (Tensor): the input tensor. + + Keyword arguments: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([[180.0, -180.0], [360.0, -360.0], [90.0, -90.0]]) + >>> torch.deg2rad(a) + tensor([[ 3.1416, -3.1416], + [ 6.2832, -6.2832], + [ 1.5708, -1.5708]]) + """ + +def deg2rad_(input: Tensor) -> Tensor: ... +@overload +def dequantize(input: Tensor) -> Tensor: + r""" + dequantize(tensor) -> Tensor + + Returns an fp32 Tensor by dequantizing a quantized Tensor + + Args: + tensor (Tensor): A quantized Tensor + + .. function:: dequantize(tensors) -> sequence of Tensors + :noindex: + + Given a list of quantized Tensors, dequantize them and return a list of fp32 Tensors + + Args: + tensors (sequence of Tensors): A list of quantized Tensors + """ + +@overload +def dequantize( + tensors: tuple[Tensor, ...] | list[Tensor] | None, +) -> tuple[Tensor, ...]: + r""" + dequantize(tensor) -> Tensor + + Returns an fp32 Tensor by dequantizing a quantized Tensor + + Args: + tensor (Tensor): A quantized Tensor + + .. function:: dequantize(tensors) -> sequence of Tensors + :noindex: + + Given a list of quantized Tensors, dequantize them and return a list of fp32 Tensors + + Args: + tensors (sequence of Tensors): A list of quantized Tensors + """ + +def det(input: Tensor) -> Tensor: + r""" + det(input) -> Tensor + + Alias for :func:`torch.linalg.det` + """ + +def detach(input: Tensor) -> Tensor: ... +def detach_(input: Tensor) -> Tensor: ... +def detach_copy(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + Performs the same operation as :func:`torch.detach`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def diag( + input: Tensor, + diagonal: _int = 0, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + diag(input, diagonal=0, *, out=None) -> Tensor + + - If :attr:`input` is a vector (1-D tensor), then returns a 2-D square tensor + with the elements of :attr:`input` as the diagonal. + - If :attr:`input` is a matrix (2-D tensor), then returns a 1-D tensor with + the diagonal elements of :attr:`input`. + + The argument :attr:`diagonal` controls which diagonal to consider: + + - If :attr:`diagonal` = 0, it is the main diagonal. + - If :attr:`diagonal` > 0, it is above the main diagonal. + - If :attr:`diagonal` < 0, it is below the main diagonal. + + Args: + input (Tensor): the input tensor. + diagonal (int, optional): the diagonal to consider + + Keyword args: + out (Tensor, optional): the output tensor. + + .. seealso:: + + :func:`torch.diagonal` always returns the diagonal of its input. + + :func:`torch.diagflat` always constructs a tensor with diagonal elements + specified by the input. + + Examples: + + Get the square matrix where the input vector is the diagonal:: + + >>> a = torch.randn(3) + >>> a + tensor([ 0.5950,-0.0872, 2.3298]) + >>> torch.diag(a) + tensor([[ 0.5950, 0.0000, 0.0000], + [ 0.0000,-0.0872, 0.0000], + [ 0.0000, 0.0000, 2.3298]]) + >>> torch.diag(a, 1) + tensor([[ 0.0000, 0.5950, 0.0000, 0.0000], + [ 0.0000, 0.0000,-0.0872, 0.0000], + [ 0.0000, 0.0000, 0.0000, 2.3298], + [ 0.0000, 0.0000, 0.0000, 0.0000]]) + + Get the k-th diagonal of a given matrix:: + + >>> a = torch.randn(3, 3) + >>> a + tensor([[-0.4264, 0.0255,-0.1064], + [ 0.8795,-0.2429, 0.1374], + [ 0.1029,-0.6482,-1.6300]]) + >>> torch.diag(a, 0) + tensor([-0.4264,-0.2429,-1.6300]) + >>> torch.diag(a, 1) + tensor([ 0.0255, 0.1374]) + """ + +def diag_embed( + input: Tensor, + offset: _int = 0, + dim1: _int = -2, + dim2: _int = -1, +) -> Tensor: + r""" + diag_embed(input, offset=0, dim1=-2, dim2=-1) -> Tensor + + Creates a tensor whose diagonals of certain 2D planes (specified by + :attr:`dim1` and :attr:`dim2`) are filled by :attr:`input`. + To facilitate creating batched diagonal matrices, the 2D planes formed by + the last two dimensions of the returned tensor are chosen by default. + + The argument :attr:`offset` controls which diagonal to consider: + + - If :attr:`offset` = 0, it is the main diagonal. + - If :attr:`offset` > 0, it is above the main diagonal. + - If :attr:`offset` < 0, it is below the main diagonal. + + The size of the new matrix will be calculated to make the specified diagonal + of the size of the last input dimension. + Note that for :attr:`offset` other than :math:`0`, the order of :attr:`dim1` + and :attr:`dim2` matters. Exchanging them is equivalent to changing the + sign of :attr:`offset`. + + Applying :meth:`torch.diagonal` to the output of this function with + the same arguments yields a matrix identical to input. However, + :meth:`torch.diagonal` has different default dimensions, so those + need to be explicitly specified. + + Args: + input (Tensor): the input tensor. Must be at least 1-dimensional. + offset (int, optional): which diagonal to consider. Default: 0 + (main diagonal). + dim1 (int, optional): first dimension with respect to which to + take diagonal. Default: -2. + dim2 (int, optional): second dimension with respect to which to + take diagonal. Default: -1. + + Example:: + + >>> a = torch.randn(2, 3) + >>> torch.diag_embed(a) + tensor([[[ 1.5410, 0.0000, 0.0000], + [ 0.0000, -0.2934, 0.0000], + [ 0.0000, 0.0000, -2.1788]], + + [[ 0.5684, 0.0000, 0.0000], + [ 0.0000, -1.0845, 0.0000], + [ 0.0000, 0.0000, -1.3986]]]) + + >>> torch.diag_embed(a, offset=1, dim1=0, dim2=2) + tensor([[[ 0.0000, 1.5410, 0.0000, 0.0000], + [ 0.0000, 0.5684, 0.0000, 0.0000]], + + [[ 0.0000, 0.0000, -0.2934, 0.0000], + [ 0.0000, 0.0000, -1.0845, 0.0000]], + + [[ 0.0000, 0.0000, 0.0000, -2.1788], + [ 0.0000, 0.0000, 0.0000, -1.3986]], + + [[ 0.0000, 0.0000, 0.0000, 0.0000], + [ 0.0000, 0.0000, 0.0000, 0.0000]]]) + """ + +def diagflat(input: Tensor, offset: _int = 0) -> Tensor: + r""" + diagflat(input, offset=0) -> Tensor + + - If :attr:`input` is a vector (1-D tensor), then returns a 2-D square tensor + with the elements of :attr:`input` as the diagonal. + - If :attr:`input` is a tensor with more than one dimension, then returns a + 2-D tensor with diagonal elements equal to a flattened :attr:`input`. + + The argument :attr:`offset` controls which diagonal to consider: + + - If :attr:`offset` = 0, it is the main diagonal. + - If :attr:`offset` > 0, it is above the main diagonal. + - If :attr:`offset` < 0, it is below the main diagonal. + + Args: + input (Tensor): the input tensor. + offset (int, optional): the diagonal to consider. Default: 0 (main + diagonal). + + Examples:: + + >>> a = torch.randn(3) + >>> a + tensor([-0.2956, -0.9068, 0.1695]) + >>> torch.diagflat(a) + tensor([[-0.2956, 0.0000, 0.0000], + [ 0.0000, -0.9068, 0.0000], + [ 0.0000, 0.0000, 0.1695]]) + >>> torch.diagflat(a, 1) + tensor([[ 0.0000, -0.2956, 0.0000, 0.0000], + [ 0.0000, 0.0000, -0.9068, 0.0000], + [ 0.0000, 0.0000, 0.0000, 0.1695], + [ 0.0000, 0.0000, 0.0000, 0.0000]]) + + >>> a = torch.randn(2, 2) + >>> a + tensor([[ 0.2094, -0.3018], + [-0.1516, 1.9342]]) + >>> torch.diagflat(a) + tensor([[ 0.2094, 0.0000, 0.0000, 0.0000], + [ 0.0000, -0.3018, 0.0000, 0.0000], + [ 0.0000, 0.0000, -0.1516, 0.0000], + [ 0.0000, 0.0000, 0.0000, 1.9342]]) + """ + +@overload +def diagonal( + input: Tensor, + offset: _int = 0, + dim1: _int = 0, + dim2: _int = 1, +) -> Tensor: + r""" + diagonal(input, offset=0, dim1=0, dim2=1) -> Tensor + + Returns a partial view of :attr:`input` with the its diagonal elements + with respect to :attr:`dim1` and :attr:`dim2` appended as a dimension + at the end of the shape. + + The argument :attr:`offset` controls which diagonal to consider: + + - If :attr:`offset` = 0, it is the main diagonal. + - If :attr:`offset` > 0, it is above the main diagonal. + - If :attr:`offset` < 0, it is below the main diagonal. + + Applying :meth:`torch.diag_embed` to the output of this function with + the same arguments yields a diagonal matrix with the diagonal entries + of the input. However, :meth:`torch.diag_embed` has different default + dimensions, so those need to be explicitly specified. + + Args: + input (Tensor): the input tensor. Must be at least 2-dimensional. + offset (int, optional): which diagonal to consider. Default: 0 + (main diagonal). + dim1 (int, optional): first dimension with respect to which to + take diagonal. Default: 0. + dim2 (int, optional): second dimension with respect to which to + take diagonal. Default: 1. + + .. note:: To take a batch diagonal, pass in dim1=-2, dim2=-1. + + Examples:: + + >>> a = torch.randn(3, 3) + >>> a + tensor([[-1.0854, 1.1431, -0.1752], + [ 0.8536, -0.0905, 0.0360], + [ 0.6927, -0.3735, -0.4945]]) + + + >>> torch.diagonal(a) + tensor([-1.0854, -0.0905, -0.4945]) + + + >>> torch.diagonal(a, 1) + tensor([ 1.1431, 0.0360]) + + >>> b = torch.randn(2, 5) + >>> b + tensor([[-1.7948, -1.2731, -0.3181, 2.0200, -1.6745], + [ 1.8262, -1.5049, 0.4114, 1.0704, -1.2607]]) + + >>> torch.diagonal(b, 1, 1, 0) + tensor([1.8262]) + + >>> x = torch.randn(2, 5, 4, 2) + >>> torch.diagonal(x, offset=-1, dim1=1, dim2=2) + tensor([[[-1.2631, 0.3755, -1.5977, -1.8172], + [-1.1065, 1.0401, -0.2235, -0.7938]], + + [[-1.7325, -0.3081, 0.6166, 0.2335], + [ 1.0500, 0.7336, -0.3836, -1.1015]]]) + """ + +@overload +def diagonal( + input: Tensor, + *, + outdim: str | EllipsisType | None, + dim1: str | EllipsisType | None, + dim2: str | EllipsisType | None, + offset: _int = 0, +) -> Tensor: + r""" + diagonal(input, offset=0, dim1=0, dim2=1) -> Tensor + + Returns a partial view of :attr:`input` with the its diagonal elements + with respect to :attr:`dim1` and :attr:`dim2` appended as a dimension + at the end of the shape. + + The argument :attr:`offset` controls which diagonal to consider: + + - If :attr:`offset` = 0, it is the main diagonal. + - If :attr:`offset` > 0, it is above the main diagonal. + - If :attr:`offset` < 0, it is below the main diagonal. + + Applying :meth:`torch.diag_embed` to the output of this function with + the same arguments yields a diagonal matrix with the diagonal entries + of the input. However, :meth:`torch.diag_embed` has different default + dimensions, so those need to be explicitly specified. + + Args: + input (Tensor): the input tensor. Must be at least 2-dimensional. + offset (int, optional): which diagonal to consider. Default: 0 + (main diagonal). + dim1 (int, optional): first dimension with respect to which to + take diagonal. Default: 0. + dim2 (int, optional): second dimension with respect to which to + take diagonal. Default: 1. + + .. note:: To take a batch diagonal, pass in dim1=-2, dim2=-1. + + Examples:: + + >>> a = torch.randn(3, 3) + >>> a + tensor([[-1.0854, 1.1431, -0.1752], + [ 0.8536, -0.0905, 0.0360], + [ 0.6927, -0.3735, -0.4945]]) + + + >>> torch.diagonal(a) + tensor([-1.0854, -0.0905, -0.4945]) + + + >>> torch.diagonal(a, 1) + tensor([ 1.1431, 0.0360]) + + >>> b = torch.randn(2, 5) + >>> b + tensor([[-1.7948, -1.2731, -0.3181, 2.0200, -1.6745], + [ 1.8262, -1.5049, 0.4114, 1.0704, -1.2607]]) + + >>> torch.diagonal(b, 1, 1, 0) + tensor([1.8262]) + + >>> x = torch.randn(2, 5, 4, 2) + >>> torch.diagonal(x, offset=-1, dim1=1, dim2=2) + tensor([[[-1.2631, 0.3755, -1.5977, -1.8172], + [-1.1065, 1.0401, -0.2235, -0.7938]], + + [[-1.7325, -0.3081, 0.6166, 0.2335], + [ 1.0500, 0.7336, -0.3836, -1.1015]]]) + """ + +def diagonal_copy( + input: Tensor, + offset: _int = 0, + dim1: _int = 0, + dim2: _int = 1, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.diagonal`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def diagonal_scatter( + input: Tensor, + src: Tensor, + offset: _int = 0, + dim1: _int = 0, + dim2: _int = 1, +) -> Tensor: + r""" + diagonal_scatter(input, src, offset=0, dim1=0, dim2=1) -> Tensor + + Embeds the values of the :attr:`src` tensor into :attr:`input` along + the diagonal elements of :attr:`input`, with respect to :attr:`dim1` + and :attr:`dim2`. + + This function returns a tensor with fresh storage; it does not + return a view. + + The argument :attr:`offset` controls which diagonal to consider: + + - If :attr:`offset` = 0, it is the main diagonal. + - If :attr:`offset` > 0, it is above the main diagonal. + - If :attr:`offset` < 0, it is below the main diagonal. + + Args: + input (Tensor): the input tensor. Must be at least 2-dimensional. + src (Tensor): the tensor to embed into :attr:`input`. + offset (int, optional): which diagonal to consider. Default: 0 + (main diagonal). + dim1 (int, optional): first dimension with respect to which to + take diagonal. Default: 0. + dim2 (int, optional): second dimension with respect to which to + take diagonal. Default: 1. + + .. note:: + + :attr:`src` must be of the proper size in order to be embedded + into :attr:`input`. Specifically, it should have the same shape as + ``torch.diagonal(input, offset, dim1, dim2)`` + + Examples:: + + >>> a = torch.zeros(3, 3) + >>> a + tensor([[0., 0., 0.], + [0., 0., 0.], + [0., 0., 0.]]) + + >>> torch.diagonal_scatter(a, torch.ones(3), 0) + tensor([[1., 0., 0.], + [0., 1., 0.], + [0., 0., 1.]]) + + >>> torch.diagonal_scatter(a, torch.ones(2), 1) + tensor([[0., 1., 0.], + [0., 0., 1.], + [0., 0., 0.]]) + """ + +def diff( + input: Tensor, + n: _int = 1, + dim: _int = -1, + prepend: Tensor | None = None, + append: Tensor | None = None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + diff(input, n=1, dim=-1, prepend=None, append=None) -> Tensor + + Computes the n-th forward difference along the given dimension. + + The first-order differences are given by `out[i] = input[i + 1] - input[i]`. Higher-order + differences are calculated by using :func:`torch.diff` recursively. + + Args: + input (Tensor): the tensor to compute the differences on + n (int, optional): the number of times to recursively compute the difference + dim (int, optional): the dimension to compute the difference along. + Default is the last dimension. + prepend, append (Tensor, optional): values to prepend or append to + :attr:`input` along :attr:`dim` before computing the difference. + Their dimensions must be equivalent to that of input, and their shapes + must match input's shape except on :attr:`dim`. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([1, 3, 2]) + >>> torch.diff(a) + tensor([ 2, -1]) + >>> b = torch.tensor([4, 5]) + >>> torch.diff(a, append=b) + tensor([ 2, -1, 2, 1]) + >>> c = torch.tensor([[1, 2, 3], [3, 4, 5]]) + >>> torch.diff(c, dim=0) + tensor([[2, 2, 2]]) + >>> torch.diff(c, dim=1) + tensor([[1, 1], + [1, 1]]) + """ + +def digamma(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + digamma(input, *, out=None) -> Tensor + + Alias for :func:`torch.special.digamma`. + """ + +def dist(input: Tensor, other: Tensor, p: Number | _complex = 2) -> Tensor: + r""" + dist(input, other, p=2) -> Tensor + + Returns the p-norm of (:attr:`input` - :attr:`other`) + + The shapes of :attr:`input` and :attr:`other` must be + :ref:`broadcastable `. + + Args: + input (Tensor): the input tensor. + other (Tensor): the Right-hand-side input tensor + p (float, optional): the norm to be computed + + Example:: + + >>> x = torch.randn(4) + >>> x + tensor([-1.5393, -0.8675, 0.5916, 1.6321]) + >>> y = torch.randn(4) + >>> y + tensor([ 0.0967, -1.0511, 0.6295, 0.8360]) + >>> torch.dist(x, y, 3.5) + tensor(1.6727) + >>> torch.dist(x, y, 3) + tensor(1.6973) + >>> torch.dist(x, y, 0) + tensor(4.) + >>> torch.dist(x, y, 1) + tensor(2.6537) + """ + +def div( + input: Tensor | Number, + other: Tensor | Number, + *, + rounding_mode: str | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + div(input, other, *, rounding_mode=None, out=None) -> Tensor + + Divides each element of the input ``input`` by the corresponding element of + :attr:`other`. + + .. math:: + \text{out}_i = \frac{\text{input}_i}{\text{other}_i} + + .. note:: + By default, this performs a "true" division like Python 3. + See the :attr:`rounding_mode` argument for floor division. + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer, float, and complex inputs. + Always promotes integer types to the default scalar type. + + Args: + input (Tensor): the dividend + other (Tensor or Number): the divisor + + Keyword args: + rounding_mode (str, optional): Type of rounding applied to the result: + + * None - default behavior. Performs no rounding and, if both :attr:`input` and + :attr:`other` are integer types, promotes the inputs to the default scalar type. + Equivalent to true division in Python (the ``/`` operator) and NumPy's ``np.true_divide``. + * ``"trunc"`` - rounds the results of the division towards zero. + Equivalent to C-style integer division. + * ``"floor"`` - rounds the results of the division down. + Equivalent to floor division in Python (the ``//`` operator) and NumPy's ``np.floor_divide``. + + out (Tensor, optional): the output tensor. + + Examples:: + + >>> x = torch.tensor([ 0.3810, 1.2774, -0.2972, -0.3719, 0.4637]) + >>> torch.div(x, 0.5) + tensor([ 0.7620, 2.5548, -0.5944, -0.7438, 0.9274]) + + >>> a = torch.tensor([[-0.3711, -1.9353, -0.4605, -0.2917], + ... [ 0.1815, -1.0111, 0.9805, -1.5923], + ... [ 0.1062, 1.4581, 0.7759, -1.2344], + ... [-0.1830, -0.0313, 1.1908, -1.4757]]) + >>> b = torch.tensor([ 0.8032, 0.2930, -0.8113, -0.2308]) + >>> torch.div(a, b) + tensor([[-0.4620, -6.6051, 0.5676, 1.2639], + [ 0.2260, -3.4509, -1.2086, 6.8990], + [ 0.1322, 4.9764, -0.9564, 5.3484], + [-0.2278, -0.1068, -1.4678, 6.3938]]) + + >>> torch.div(a, b, rounding_mode='trunc') + tensor([[-0., -6., 0., 1.], + [ 0., -3., -1., 6.], + [ 0., 4., -0., 5.], + [-0., -0., -1., 6.]]) + + >>> torch.div(a, b, rounding_mode='floor') + tensor([[-1., -7., 0., 1.], + [ 0., -4., -2., 6.], + [ 0., 4., -1., 5.], + [-1., -1., -2., 6.]]) + """ + +@overload +def divide( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + divide(input, other, *, rounding_mode=None, out=None) -> Tensor + + Alias for :func:`torch.div`. + """ + +@overload +def divide( + input: Tensor, + other: Tensor, + *, + rounding_mode: str | None, + out: Tensor | None = None, +) -> Tensor: + r""" + divide(input, other, *, rounding_mode=None, out=None) -> Tensor + + Alias for :func:`torch.div`. + """ + +@overload +def divide( + input: Tensor, + other: Number | _complex, + *, + rounding_mode: str | None, +) -> Tensor: + r""" + divide(input, other, *, rounding_mode=None, out=None) -> Tensor + + Alias for :func:`torch.div`. + """ + +@overload +def divide(input: Tensor, other: Number | _complex) -> Tensor: + r""" + divide(input, other, *, rounding_mode=None, out=None) -> Tensor + + Alias for :func:`torch.div`. + """ + +def dot( + input: Tensor, + tensor: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + dot(input, tensor, *, out=None) -> Tensor + + Computes the dot product of two 1D tensors. + + .. note:: + + Unlike NumPy's dot, torch.dot intentionally only supports computing the dot product + of two 1D tensors with the same number of elements. + + Args: + input (Tensor): first tensor in the dot product, must be 1D. + tensor (Tensor): second tensor in the dot product, must be 1D. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.dot(torch.tensor([2, 3]), torch.tensor([2, 1])) + tensor(7) + + >>> t1, t2 = torch.tensor([0, 1]), torch.tensor([2, 3]) + >>> torch.dot(t1, t2) + tensor(3) + """ + +def dropout(input: Tensor, p: _float, train: _bool) -> Tensor: ... +def dropout_(input: Tensor, p: _float, train: _bool) -> Tensor: ... +def dsmm(input: Tensor, mat2: Tensor) -> Tensor: ... +@overload +def dsplit(input: Tensor, sections: _int) -> tuple[Tensor, ...]: + r""" + dsplit(input, indices_or_sections) -> List of Tensors + + Splits :attr:`input`, a tensor with three or more dimensions, into multiple tensors + depthwise according to :attr:`indices_or_sections`. Each split is a view of + :attr:`input`. + + This is equivalent to calling torch.tensor_split(input, indices_or_sections, dim=2) + (the split dimension is 2), except that if :attr:`indices_or_sections` is an integer + it must evenly divide the split dimension or a runtime error will be thrown. + + This function is based on NumPy's :func:`numpy.dsplit`. + + Args: + input (Tensor): tensor to split. + indices_or_sections (int or list or tuple of ints): See argument in :func:`torch.tensor_split`. + + Example:: + + >>> t = torch.arange(16.0).reshape(2, 2, 4) + >>> t + tensor([[[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.]], + [[ 8., 9., 10., 11.], + [12., 13., 14., 15.]]]) + >>> torch.dsplit(t, 2) + (tensor([[[ 0., 1.], + [ 4., 5.]], + [[ 8., 9.], + [12., 13.]]]), + tensor([[[ 2., 3.], + [ 6., 7.]], + [[10., 11.], + [14., 15.]]])) + + >>> torch.dsplit(t, [3, 6]) + (tensor([[[ 0., 1., 2.], + [ 4., 5., 6.]], + [[ 8., 9., 10.], + [12., 13., 14.]]]), + tensor([[[ 3.], + [ 7.]], + [[11.], + [15.]]]), + tensor([], size=(2, 2, 0))) + """ + +@overload +def dsplit(input: Tensor, indices: _size) -> tuple[Tensor, ...]: + r""" + dsplit(input, indices_or_sections) -> List of Tensors + + Splits :attr:`input`, a tensor with three or more dimensions, into multiple tensors + depthwise according to :attr:`indices_or_sections`. Each split is a view of + :attr:`input`. + + This is equivalent to calling torch.tensor_split(input, indices_or_sections, dim=2) + (the split dimension is 2), except that if :attr:`indices_or_sections` is an integer + it must evenly divide the split dimension or a runtime error will be thrown. + + This function is based on NumPy's :func:`numpy.dsplit`. + + Args: + input (Tensor): tensor to split. + indices_or_sections (int or list or tuple of ints): See argument in :func:`torch.tensor_split`. + + Example:: + + >>> t = torch.arange(16.0).reshape(2, 2, 4) + >>> t + tensor([[[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.]], + [[ 8., 9., 10., 11.], + [12., 13., 14., 15.]]]) + >>> torch.dsplit(t, 2) + (tensor([[[ 0., 1.], + [ 4., 5.]], + [[ 8., 9.], + [12., 13.]]]), + tensor([[[ 2., 3.], + [ 6., 7.]], + [[10., 11.], + [14., 15.]]])) + + >>> torch.dsplit(t, [3, 6]) + (tensor([[[ 0., 1., 2.], + [ 4., 5., 6.]], + [[ 8., 9., 10.], + [12., 13., 14.]]]), + tensor([[[ 3.], + [ 7.]], + [[11.], + [15.]]]), + tensor([], size=(2, 2, 0))) + """ + +def dstack( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + dstack(tensors, *, out=None) -> Tensor + + Stack tensors in sequence depthwise (along third axis). + + This is equivalent to concatenation along the third axis after 1-D and 2-D tensors have been reshaped by :func:`torch.atleast_3d`. + + Args: + tensors (sequence of Tensors): sequence of tensors to concatenate + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([1, 2, 3]) + >>> b = torch.tensor([4, 5, 6]) + >>> torch.dstack((a,b)) + tensor([[[1, 4], + [2, 5], + [3, 6]]]) + >>> a = torch.tensor([[1],[2],[3]]) + >>> b = torch.tensor([[4],[5],[6]]) + >>> torch.dstack((a,b)) + tensor([[[1, 4]], + [[2, 5]], + [[3, 6]]]) + """ + +def embedding( + weight: Tensor, + indices: Tensor, + padding_idx: _int | SymInt = -1, + scale_grad_by_freq: _bool = False, + sparse: _bool = False, +) -> Tensor: ... +@overload +def embedding_bag( + weight: Tensor, + indices: Tensor, + offsets: Tensor, + scale_grad_by_freq: _bool, + mode: _int, + sparse: _bool, + per_sample_weights: Tensor | None, + include_last_offset: _bool, + padding_idx: _int | None, +) -> tuple[Tensor, Tensor, Tensor, Tensor]: ... +@overload +def embedding_bag( + weight: Tensor, + indices: Tensor, + offsets: Tensor, + scale_grad_by_freq: _bool = False, + mode: _int = 0, + sparse: _bool = False, + per_sample_weights: Tensor | None = None, + include_last_offset: _bool = False, +) -> tuple[Tensor, Tensor, Tensor, Tensor]: ... +def embedding_renorm_( + input: Tensor, + indices: Tensor, + max_norm: _float, + norm_type: _float, +) -> Tensor: ... +@overload +def empty( + size: Sequence[_int | SymInt], + *, + memory_format: memory_format | None = None, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + empty(*size, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False, memory_format=torch.contiguous_format) -> Tensor + + Returns a tensor filled with uninitialized data. The shape of the tensor is + defined by the variable argument :attr:`size`. + + .. note:: + If :func:`torch.use_deterministic_algorithms()` and + :attr:`torch.utils.deterministic.fill_uninitialized_memory` are both set to + ``True``, the output tensor is initialized to prevent any possible + nondeterministic behavior from using the data as an input to an operation. + Floating point and complex tensors are filled with NaN, and integer tensors + are filled with the maximum value. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.contiguous_format``. + + Example:: + + >>> torch.empty((2,3), dtype=torch.int64) + tensor([[ 9.4064e+13, 2.8000e+01, 9.3493e+13], + [ 7.5751e+18, 7.1428e+18, 7.5955e+18]]) + """ + +@overload +def empty( + *size: _int | SymInt, + memory_format: memory_format | None = None, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + empty(*size, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False, memory_format=torch.contiguous_format) -> Tensor + + Returns a tensor filled with uninitialized data. The shape of the tensor is + defined by the variable argument :attr:`size`. + + .. note:: + If :func:`torch.use_deterministic_algorithms()` and + :attr:`torch.utils.deterministic.fill_uninitialized_memory` are both set to + ``True``, the output tensor is initialized to prevent any possible + nondeterministic behavior from using the data as an input to an operation. + Floating point and complex tensors are filled with NaN, and integer tensors + are filled with the maximum value. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.contiguous_format``. + + Example:: + + >>> torch.empty((2,3), dtype=torch.int64) + tensor([[ 9.4064e+13, 2.8000e+01, 9.3493e+13], + [ 7.5751e+18, 7.1428e+18, 7.5955e+18]]) + """ + +@overload +def empty( + size: _size, + *, + names: Sequence[str | EllipsisType | None] | None, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + empty(*size, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False, memory_format=torch.contiguous_format) -> Tensor + + Returns a tensor filled with uninitialized data. The shape of the tensor is + defined by the variable argument :attr:`size`. + + .. note:: + If :func:`torch.use_deterministic_algorithms()` and + :attr:`torch.utils.deterministic.fill_uninitialized_memory` are both set to + ``True``, the output tensor is initialized to prevent any possible + nondeterministic behavior from using the data as an input to an operation. + Floating point and complex tensors are filled with NaN, and integer tensors + are filled with the maximum value. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.contiguous_format``. + + Example:: + + >>> torch.empty((2,3), dtype=torch.int64) + tensor([[ 9.4064e+13, 2.8000e+01, 9.3493e+13], + [ 7.5751e+18, 7.1428e+18, 7.5955e+18]]) + """ + +@overload +def empty( + *size: _int, + names: Sequence[str | EllipsisType | None] | None, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + empty(*size, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False, memory_format=torch.contiguous_format) -> Tensor + + Returns a tensor filled with uninitialized data. The shape of the tensor is + defined by the variable argument :attr:`size`. + + .. note:: + If :func:`torch.use_deterministic_algorithms()` and + :attr:`torch.utils.deterministic.fill_uninitialized_memory` are both set to + ``True``, the output tensor is initialized to prevent any possible + nondeterministic behavior from using the data as an input to an operation. + Floating point and complex tensors are filled with NaN, and integer tensors + are filled with the maximum value. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.contiguous_format``. + + Example:: + + >>> torch.empty((2,3), dtype=torch.int64) + tensor([[ 9.4064e+13, 2.8000e+01, 9.3493e+13], + [ 7.5751e+18, 7.1428e+18, 7.5955e+18]]) + """ + +def empty_like( + input: Tensor, + *, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + empty_like(input, *, dtype=None, layout=None, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns an uninitialized tensor with the same size as :attr:`input`. + ``torch.empty_like(input)`` is equivalent to + ``torch.empty(input.size(), dtype=input.dtype, layout=input.layout, device=input.device)``. + + .. note:: + If :func:`torch.use_deterministic_algorithms()` and + :attr:`torch.utils.deterministic.fill_uninitialized_memory` are both set to + ``True``, the output tensor is initialized to prevent any possible + nondeterministic behavior from using the data as an input to an operation. + Floating point and complex tensors are filled with NaN, and integer tensors + are filled with the maximum value. + + When ``torch.preserve_format`` is used: + If the input tensor is dense (i.e., non-overlapping strided), + its memory format (including strides) is retained. + Otherwise (e.g., a non-dense view like a stepped slice), + the output is converted to the dense format. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + + Example:: + + >>> a=torch.empty((2,3), dtype=torch.int32, device = 'cuda') + >>> torch.empty_like(a) + tensor([[0, 0, 0], + [0, 0, 0]], device='cuda:0', dtype=torch.int32) + """ + +def empty_permuted( + size: Sequence[_int | SymInt], + physical_layout: _size, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + empty_permuted(size, physical_layout, *, dtype=None, layout=None, device=None, requires_grad=False, pin_memory=False) -> Tensor + + Creates an uninitialized, non-overlapping and dense tensor with the + specified :attr:`size`, with :attr:`physical_layout` specifying how the + dimensions are physically laid out in memory (each logical dimension is listed + from outermost to innermost). :attr:`physical_layout` is a generalization + of NCHW/NHWC notation: if each dimension is assigned a number according to + what order they occur in size (N=0, C=1, H=2, W=3), then NCHW is ``(0, 1, 2, 3)`` + while NHWC is ``(0, 2, 3, 1)``. Equivalently, the strides of the output + tensor ``t`` are such that ``t.stride(physical_layout[i]) == contiguous_strides[i]`` + (notably, this function is *not* equivalent to ``torch.empty(size).permute(physical_layout)``). + + Unlike :func:`torch.empty_strided`, this is guaranteed to produce a dense + tensor with no overlaps. If possible, prefer using this function over + :func:`torch.empty_strided` or manual use of :func:`torch.as_strided`. + + .. note:: + If :func:`torch.use_deterministic_algorithms()` and + :attr:`torch.utils.deterministic.fill_uninitialized_memory` are both set to + ``True``, the output tensor is initialized to prevent any possible + nondeterministic behavior from using the data as an input to an operation. + Floating point and complex tensors are filled with NaN, and integer tensors + are filled with the maximum value. + + Args: + size (tuple of int): the shape of the output tensor + physical_layout (tuple of int): the ordering of dimensions physically in memory + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Examples: + + >>> torch.empty((2, 3, 5, 7)).stride() + (105, 35, 7, 1) + >>> torch.empty_permuted((2, 3, 5, 7), (0, 1, 2, 3)).stride() + (105, 35, 7, 1) + >>> torch.empty((2, 3, 5, 7), memory_format=torch.channels_last).stride() + (105, 1, 21, 3) + >>> torch.empty_permuted((2, 3, 5, 7), (0, 2, 3, 1)).stride() + (105, 1, 21, 3) + >>> torch.empty_permuted((2, 3, 5, 7), (0, 2, 3, 1)).dim_order() + (0, 2, 3, 1) + """ + +def empty_quantized( + size: _size, + qtensor: Tensor, + *, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: ... +def empty_strided( + size: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + empty_strided(size, stride, *, dtype=None, layout=None, device=None, requires_grad=False, pin_memory=False) -> Tensor + + Creates a tensor with the specified :attr:`size` and :attr:`stride` and filled with undefined data. + + .. warning:: + If the constructed tensor is "overlapped" (with multiple indices referring to the same element + in memory) its behavior is undefined. + + .. note:: + If :func:`torch.use_deterministic_algorithms()` and + :attr:`torch.utils.deterministic.fill_uninitialized_memory` are both set to + ``True``, the output tensor is initialized to prevent any possible + nondeterministic behavior from using the data as an input to an operation. + Floating point and complex tensors are filled with NaN, and integer tensors + are filled with the maximum value. + + Args: + size (tuple of int): the shape of the output tensor + stride (tuple of int): the strides of the output tensor + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> a = torch.empty_strided((2, 3), (1, 2)) + >>> a + tensor([[8.9683e-44, 4.4842e-44, 5.1239e+07], + [0.0000e+00, 0.0000e+00, 3.0705e-41]]) + >>> a.stride() + (1, 2) + >>> a.size() + torch.Size([2, 3]) + """ + +@overload +def eq( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + eq(input, other, *, out=None) -> Tensor + + Computes element-wise equality + + The second argument can be a number or a tensor whose shape is + :ref:`broadcastable ` with the first argument. + + Args: + input (Tensor): the tensor to compare + other (Tensor or float): the tensor or value to compare + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is equal to :attr:`other` and False elsewhere + + Example:: + + >>> torch.eq(torch.tensor([[1, 2], [3, 4]]), torch.tensor([[1, 1], [4, 4]])) + tensor([[ True, False], + [False, True]]) + """ + +@overload +def eq( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + eq(input, other, *, out=None) -> Tensor + + Computes element-wise equality + + The second argument can be a number or a tensor whose shape is + :ref:`broadcastable ` with the first argument. + + Args: + input (Tensor): the tensor to compare + other (Tensor or float): the tensor or value to compare + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is equal to :attr:`other` and False elsewhere + + Example:: + + >>> torch.eq(torch.tensor([[1, 2], [3, 4]]), torch.tensor([[1, 1], [4, 4]])) + tensor([[ True, False], + [False, True]]) + """ + +def equal(input: Tensor, other: Tensor) -> _bool: + r""" + equal(input, other) -> bool + + ``True`` if two tensors have the same size and elements, ``False`` otherwise. + + .. note:: + + Tensors containing NaNs are never equal to each other. Additionally, this function does not + differentiate between the data types of the tensors during comparison. For more thorough tensor checks, + use :meth:`torch.testing.assert_close`. + + Example:: + + >>> torch.equal(torch.tensor([1, 2]), torch.tensor([1, 2])) + True + >>> torch.equal(torch.tensor([3, torch.nan]), torch.tensor([3, torch.nan])) + False + >>> torch.equal(torch.tensor([1, 2, 3], dtype=torch.int32), torch.tensor([1, 2, 3], dtype=torch.float32)) + True + """ + +def erf(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + erf(input, *, out=None) -> Tensor + + Alias for :func:`torch.special.erf`. + """ + +def erf_(input: Tensor) -> Tensor: ... +def erfc(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + erfc(input, *, out=None) -> Tensor + + Alias for :func:`torch.special.erfc`. + """ + +def erfc_(input: Tensor) -> Tensor: ... +def erfinv(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + erfinv(input, *, out=None) -> Tensor + + Alias for :func:`torch.special.erfinv`. + """ + +def exp(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + exp(input, *, out=None) -> Tensor + + Returns a new tensor with the exponential of the elements + of the input tensor :attr:`input`. + + .. math:: + y_{i} = e^{x_{i}} + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.exp(torch.tensor([0, math.log(2.)])) + tensor([ 1., 2.]) + """ + +def exp2(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + exp2(input, *, out=None) -> Tensor + + Alias for :func:`torch.special.exp2`. + """ + +def exp2_(input: Tensor) -> Tensor: ... +def exp_(input: Tensor) -> Tensor: ... +def expand_copy( + input: Tensor, + size: Sequence[_int | SymInt], + *, + implicit: _bool = False, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.Tensor.expand`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def expm1(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + expm1(input, *, out=None) -> Tensor + + Alias for :func:`torch.special.expm1`. + """ + +def expm1_(input: Tensor) -> Tensor: ... +@overload +def eye( + n: _int | SymInt, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + eye(n, m=None, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a 2-D tensor with ones on the diagonal and zeros elsewhere. + + Args: + n (int): the number of rows + m (int, optional): the number of columns with default being :attr:`n` + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 2-D tensor with ones on the diagonal and zeros elsewhere + + Example:: + + >>> torch.eye(3) + tensor([[ 1., 0., 0.], + [ 0., 1., 0.], + [ 0., 0., 1.]]) + """ + +@overload +def eye( + n: _int | SymInt, + m: _int | SymInt, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + eye(n, m=None, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a 2-D tensor with ones on the diagonal and zeros elsewhere. + + Args: + n (int): the number of rows + m (int, optional): the number of columns with default being :attr:`n` + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 2-D tensor with ones on the diagonal and zeros elsewhere + + Example:: + + >>> torch.eye(3) + tensor([[ 1., 0., 0.], + [ 0., 1., 0.], + [ 0., 0., 1.]]) + """ + +def fake_quantize_per_channel_affine( + input: Tensor, + scale: Tensor, + zero_point: Tensor, + axis: _int, + quant_min: _int, + quant_max: _int, +) -> Tensor: + r""" + fake_quantize_per_channel_affine(input, scale, zero_point, axis, quant_min, quant_max) -> Tensor + + Returns a new tensor with the data in :attr:`input` fake quantized per channel using :attr:`scale`, + :attr:`zero_point`, :attr:`quant_min` and :attr:`quant_max`, across the channel specified by :attr:`axis`. + + .. math:: + \text{output} = ( + min( + \text{quant\_max}, + max( + \text{quant\_min}, + \text{std::nearby\_int}(\text{input} / \text{scale}) + \text{zero\_point} + ) + ) - \text{zero\_point} + ) \times \text{scale} + + Args: + input (Tensor): the input value(s), in ``torch.float32`` + scale (Tensor): quantization scale, per channel in ``torch.float32`` + zero_point (Tensor): quantization zero_point, per channel in ``torch.int32`` or ``torch.half`` or ``torch.float32`` + axis (int32): channel axis + quant_min (int64): lower bound of the quantized domain + quant_max (int64): upper bound of the quantized domain + + Returns: + Tensor: A newly fake_quantized per channel ``torch.float32`` tensor + + Example:: + + >>> x = torch.randn(2, 2, 2) + >>> x + tensor([[[-0.2525, -0.0466], + [ 0.3491, -0.2168]], + + [[-0.5906, 1.6258], + [ 0.6444, -0.0542]]]) + >>> scales = (torch.randn(2) + 1) * 0.05 + >>> scales + tensor([0.0475, 0.0486]) + >>> zero_points = torch.zeros(2).to(torch.int32) + >>> zero_points + tensor([0, 0]) + >>> torch.fake_quantize_per_channel_affine(x, scales, zero_points, 1, 0, 255) + tensor([[[0.0000, 0.0000], + [0.3405, 0.0000]], + + [[0.0000, 1.6134], + [0.6323, 0.0000]]]) + """ + +@overload +def fake_quantize_per_tensor_affine( + input: Tensor, + scale: _float, + zero_point: _int, + quant_min: _int, + quant_max: _int, +) -> Tensor: + r""" + fake_quantize_per_tensor_affine(input, scale, zero_point, quant_min, quant_max) -> Tensor + + Returns a new tensor with the data in :attr:`input` fake quantized using :attr:`scale`, + :attr:`zero_point`, :attr:`quant_min` and :attr:`quant_max`. + + .. math:: + \text{output} = ( + min( + \text{quant\_max}, + max( + \text{quant\_min}, + \text{std::nearby\_int}(\text{input} / \text{scale}) + \text{zero\_point} + ) + ) - \text{zero\_point} + ) \times \text{scale} + + Args: + input (Tensor): the input value(s), ``torch.float32`` tensor + scale (double scalar or ``float32`` Tensor): quantization scale + zero_point (int64 scalar or ``int32`` Tensor): quantization zero_point + quant_min (int64): lower bound of the quantized domain + quant_max (int64): upper bound of the quantized domain + + Returns: + Tensor: A newly fake_quantized ``torch.float32`` tensor + + Example:: + + >>> x = torch.randn(4) + >>> x + tensor([ 0.0552, 0.9730, 0.3973, -1.0780]) + >>> torch.fake_quantize_per_tensor_affine(x, 0.1, 0, 0, 255) + tensor([0.1000, 1.0000, 0.4000, 0.0000]) + >>> torch.fake_quantize_per_tensor_affine(x, torch.tensor(0.1), torch.tensor(0), 0, 255) + tensor([0.1000, 1.0000, 0.4000, 0.0000]) + """ + +@overload +def fake_quantize_per_tensor_affine( + input: Tensor, + scale: Tensor, + zero_point: Tensor, + quant_min: _int, + quant_max: _int, +) -> Tensor: + r""" + fake_quantize_per_tensor_affine(input, scale, zero_point, quant_min, quant_max) -> Tensor + + Returns a new tensor with the data in :attr:`input` fake quantized using :attr:`scale`, + :attr:`zero_point`, :attr:`quant_min` and :attr:`quant_max`. + + .. math:: + \text{output} = ( + min( + \text{quant\_max}, + max( + \text{quant\_min}, + \text{std::nearby\_int}(\text{input} / \text{scale}) + \text{zero\_point} + ) + ) - \text{zero\_point} + ) \times \text{scale} + + Args: + input (Tensor): the input value(s), ``torch.float32`` tensor + scale (double scalar or ``float32`` Tensor): quantization scale + zero_point (int64 scalar or ``int32`` Tensor): quantization zero_point + quant_min (int64): lower bound of the quantized domain + quant_max (int64): upper bound of the quantized domain + + Returns: + Tensor: A newly fake_quantized ``torch.float32`` tensor + + Example:: + + >>> x = torch.randn(4) + >>> x + tensor([ 0.0552, 0.9730, 0.3973, -1.0780]) + >>> torch.fake_quantize_per_tensor_affine(x, 0.1, 0, 0, 255) + tensor([0.1000, 1.0000, 0.4000, 0.0000]) + >>> torch.fake_quantize_per_tensor_affine(x, torch.tensor(0.1), torch.tensor(0), 0, 255) + tensor([0.1000, 1.0000, 0.4000, 0.0000]) + """ + +@overload +def fbgemm_linear_fp16_weight( + input: Tensor, + packed_weight: Tensor, + bias: Tensor, +) -> Tensor: ... +@overload +def fbgemm_linear_fp16_weight( + input: Tensor, + packed_weight: Tensor, + bias: Tensor, + output: Tensor, +) -> Tensor: ... +@overload +def fbgemm_linear_fp16_weight_fp32_activation( + input: Tensor, + packed_weight: Tensor, + bias: Tensor | None, +) -> Tensor: ... +@overload +def fbgemm_linear_fp16_weight_fp32_activation( + input: Tensor, + packed_weight: Tensor, + bias: Tensor | None, + output: Tensor, +) -> Tensor: ... +def fbgemm_linear_int8_weight( + input: Tensor, + weight: Tensor, + packed: Tensor, + col_offsets: Tensor, + weight_scale: Number | _complex, + weight_zero_point: Number | _complex, + bias: Tensor, +) -> Tensor: ... +def fbgemm_linear_int8_weight_fp32_activation( + input: Tensor, + weight: Tensor, + packed: Tensor, + col_offsets: Tensor, + weight_scale: Number | _complex, + weight_zero_point: Number | _complex, + bias: Tensor, +) -> Tensor: ... +def fbgemm_linear_quantize_weight( + input: Tensor, +) -> tuple[Tensor, Tensor, _float, _int]: ... +def fbgemm_pack_gemm_matrix_fp16(input: Tensor) -> Tensor: ... +@overload +def fbgemm_pack_quantized_matrix(input: Tensor) -> Tensor: ... +@overload +def fbgemm_pack_quantized_matrix(input: Tensor, K: _int, N: _int) -> Tensor: ... +def feature_alpha_dropout(input: Tensor, p: _float, train: _bool) -> Tensor: ... +def feature_alpha_dropout_( + input: Tensor, + p: _float, + train: _bool, +) -> Tensor: ... +def feature_dropout(input: Tensor, p: _float, train: _bool) -> Tensor: ... +def feature_dropout_(input: Tensor, p: _float, train: _bool) -> Tensor: ... +@overload +def fill(input: Tensor, value: Tensor) -> Tensor: ... +@overload +def fill(input: Tensor, value: Number | _complex) -> Tensor: ... +@overload +def fill_(input: Tensor, value: Tensor) -> Tensor: ... +@overload +def fill_(input: Tensor, value: Number | _complex) -> Tensor: ... +def fix(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + fix(input, *, out=None) -> Tensor + + Alias for :func:`torch.trunc` + """ + +def fix_(input: Tensor) -> Tensor: ... +@overload +def flatten( + input: Tensor, + start_dim: _int = 0, + end_dim: _int = -1, +) -> Tensor: + r""" + flatten(input, start_dim=0, end_dim=-1) -> Tensor + + Flattens :attr:`input` by reshaping it into a one-dimensional tensor. If :attr:`start_dim` or :attr:`end_dim` + are passed, only dimensions starting with :attr:`start_dim` and ending with :attr:`end_dim` are flattened. + The order of elements in :attr:`input` is unchanged. + + Unlike NumPy's flatten, which always copies input's data, this function may return the original object, a view, + or copy. If no dimensions are flattened, then the original object :attr:`input` is returned. Otherwise, if input can + be viewed as the flattened shape, then that view is returned. Finally, only if the input cannot be viewed as the + flattened shape is input's data copied. See :meth:`torch.Tensor.view` for details on when a view will be returned. + + .. note:: + Flattening a zero-dimensional tensor will return a one-dimensional view. + + Args: + input (Tensor): the input tensor. + start_dim (int): the first dim to flatten + end_dim (int): the last dim to flatten + + Example:: + + >>> t = torch.tensor([[[1, 2], + ... [3, 4]], + ... [[5, 6], + ... [7, 8]]]) + >>> torch.flatten(t) + tensor([1, 2, 3, 4, 5, 6, 7, 8]) + >>> torch.flatten(t, start_dim=1) + tensor([[1, 2, 3, 4], + [5, 6, 7, 8]]) + """ + +@overload +def flatten( + input: Tensor, + start_dim: _int, + end_dim: _int, + out_dim: str | EllipsisType | None, +) -> Tensor: + r""" + flatten(input, start_dim=0, end_dim=-1) -> Tensor + + Flattens :attr:`input` by reshaping it into a one-dimensional tensor. If :attr:`start_dim` or :attr:`end_dim` + are passed, only dimensions starting with :attr:`start_dim` and ending with :attr:`end_dim` are flattened. + The order of elements in :attr:`input` is unchanged. + + Unlike NumPy's flatten, which always copies input's data, this function may return the original object, a view, + or copy. If no dimensions are flattened, then the original object :attr:`input` is returned. Otherwise, if input can + be viewed as the flattened shape, then that view is returned. Finally, only if the input cannot be viewed as the + flattened shape is input's data copied. See :meth:`torch.Tensor.view` for details on when a view will be returned. + + .. note:: + Flattening a zero-dimensional tensor will return a one-dimensional view. + + Args: + input (Tensor): the input tensor. + start_dim (int): the first dim to flatten + end_dim (int): the last dim to flatten + + Example:: + + >>> t = torch.tensor([[[1, 2], + ... [3, 4]], + ... [[5, 6], + ... [7, 8]]]) + >>> torch.flatten(t) + tensor([1, 2, 3, 4, 5, 6, 7, 8]) + >>> torch.flatten(t, start_dim=1) + tensor([[1, 2, 3, 4], + [5, 6, 7, 8]]) + """ + +@overload +def flatten( + input: Tensor, + start_dim: str | EllipsisType | None, + end_dim: str | EllipsisType | None, + out_dim: str | EllipsisType | None, +) -> Tensor: + r""" + flatten(input, start_dim=0, end_dim=-1) -> Tensor + + Flattens :attr:`input` by reshaping it into a one-dimensional tensor. If :attr:`start_dim` or :attr:`end_dim` + are passed, only dimensions starting with :attr:`start_dim` and ending with :attr:`end_dim` are flattened. + The order of elements in :attr:`input` is unchanged. + + Unlike NumPy's flatten, which always copies input's data, this function may return the original object, a view, + or copy. If no dimensions are flattened, then the original object :attr:`input` is returned. Otherwise, if input can + be viewed as the flattened shape, then that view is returned. Finally, only if the input cannot be viewed as the + flattened shape is input's data copied. See :meth:`torch.Tensor.view` for details on when a view will be returned. + + .. note:: + Flattening a zero-dimensional tensor will return a one-dimensional view. + + Args: + input (Tensor): the input tensor. + start_dim (int): the first dim to flatten + end_dim (int): the last dim to flatten + + Example:: + + >>> t = torch.tensor([[[1, 2], + ... [3, 4]], + ... [[5, 6], + ... [7, 8]]]) + >>> torch.flatten(t) + tensor([1, 2, 3, 4, 5, 6, 7, 8]) + >>> torch.flatten(t, start_dim=1) + tensor([[1, 2, 3, 4], + [5, 6, 7, 8]]) + """ + +@overload +def flatten( + input: Tensor, + dims: Sequence[str | EllipsisType | None], + out_dim: str | EllipsisType | None, +) -> Tensor: + r""" + flatten(input, start_dim=0, end_dim=-1) -> Tensor + + Flattens :attr:`input` by reshaping it into a one-dimensional tensor. If :attr:`start_dim` or :attr:`end_dim` + are passed, only dimensions starting with :attr:`start_dim` and ending with :attr:`end_dim` are flattened. + The order of elements in :attr:`input` is unchanged. + + Unlike NumPy's flatten, which always copies input's data, this function may return the original object, a view, + or copy. If no dimensions are flattened, then the original object :attr:`input` is returned. Otherwise, if input can + be viewed as the flattened shape, then that view is returned. Finally, only if the input cannot be viewed as the + flattened shape is input's data copied. See :meth:`torch.Tensor.view` for details on when a view will be returned. + + .. note:: + Flattening a zero-dimensional tensor will return a one-dimensional view. + + Args: + input (Tensor): the input tensor. + start_dim (int): the first dim to flatten + end_dim (int): the last dim to flatten + + Example:: + + >>> t = torch.tensor([[[1, 2], + ... [3, 4]], + ... [[5, 6], + ... [7, 8]]]) + >>> torch.flatten(t) + tensor([1, 2, 3, 4, 5, 6, 7, 8]) + >>> torch.flatten(t, start_dim=1) + tensor([[1, 2, 3, 4], + [5, 6, 7, 8]]) + """ + +def flip(input: Tensor, dims: _size) -> Tensor: + r""" + flip(input, dims) -> Tensor + + Reverse the order of an n-D tensor along given axis in dims. + + .. note:: + `torch.flip` makes a copy of :attr:`input`'s data. This is different from NumPy's `np.flip`, + which returns a view in constant time. Since copying a tensor's data is more work than viewing that data, + `torch.flip` is expected to be slower than `np.flip`. + + Args: + input (Tensor): the input tensor. + dims (a list or tuple): axis to flip on + + Example:: + + >>> x = torch.arange(8).view(2, 2, 2) + >>> x + tensor([[[ 0, 1], + [ 2, 3]], + + [[ 4, 5], + [ 6, 7]]]) + >>> torch.flip(x, [0, 1]) + tensor([[[ 6, 7], + [ 4, 5]], + + [[ 2, 3], + [ 0, 1]]]) + """ + +def fliplr(input: Tensor) -> Tensor: + r""" + fliplr(input) -> Tensor + + Flip tensor in the left/right direction, returning a new tensor. + + Flip the entries in each row in the left/right direction. + Columns are preserved, but appear in a different order than before. + + Note: + Requires the tensor to be at least 2-D. + + .. note:: + `torch.fliplr` makes a copy of :attr:`input`'s data. This is different from NumPy's `np.fliplr`, + which returns a view in constant time. Since copying a tensor's data is more work than viewing that data, + `torch.fliplr` is expected to be slower than `np.fliplr`. + + Args: + input (Tensor): Must be at least 2-dimensional. + + Example:: + + >>> x = torch.arange(4).view(2, 2) + >>> x + tensor([[0, 1], + [2, 3]]) + >>> torch.fliplr(x) + tensor([[1, 0], + [3, 2]]) + """ + +def flipud(input: Tensor) -> Tensor: + r""" + flipud(input) -> Tensor + + Flip tensor in the up/down direction, returning a new tensor. + + Flip the entries in each column in the up/down direction. + Rows are preserved, but appear in a different order than before. + + Note: + Requires the tensor to be at least 1-D. + + .. note:: + `torch.flipud` makes a copy of :attr:`input`'s data. This is different from NumPy's `np.flipud`, + which returns a view in constant time. Since copying a tensor's data is more work than viewing that data, + `torch.flipud` is expected to be slower than `np.flipud`. + + Args: + input (Tensor): Must be at least 1-dimensional. + + Example:: + + >>> x = torch.arange(4).view(2, 2) + >>> x + tensor([[0, 1], + [2, 3]]) + >>> torch.flipud(x) + tensor([[2, 3], + [0, 1]]) + """ + +@overload +def float_power( + input: Tensor, + exponent: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + float_power(input, exponent, *, out=None) -> Tensor + + Raises :attr:`input` to the power of :attr:`exponent`, elementwise, in double precision. + If neither input is complex returns a ``torch.float64`` tensor, + and if one or more inputs is complex returns a ``torch.complex128`` tensor. + + .. note:: + This function always computes in double precision, unlike :func:`torch.pow`, + which implements more typical :ref:`type promotion `. + This is useful when the computation needs to be performed in a wider or more precise dtype, + or the results of the computation may contain fractional values not representable in the input dtypes, + like when an integer base is raised to a negative integer exponent. + + Args: + input (Tensor or Number): the base value(s) + exponent (Tensor or Number): the exponent value(s) + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randint(10, (4,)) + >>> a + tensor([6, 4, 7, 1]) + >>> torch.float_power(a, 2) + tensor([36., 16., 49., 1.], dtype=torch.float64) + + >>> a = torch.arange(1, 5) + >>> a + tensor([ 1, 2, 3, 4]) + >>> exp = torch.tensor([2, -3, 4, -5]) + >>> exp + tensor([ 2, -3, 4, -5]) + >>> torch.float_power(a, exp) + tensor([1.0000e+00, 1.2500e-01, 8.1000e+01, 9.7656e-04], dtype=torch.float64) + """ + +@overload +def float_power( + self: Number | _complex, + exponent: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + float_power(input, exponent, *, out=None) -> Tensor + + Raises :attr:`input` to the power of :attr:`exponent`, elementwise, in double precision. + If neither input is complex returns a ``torch.float64`` tensor, + and if one or more inputs is complex returns a ``torch.complex128`` tensor. + + .. note:: + This function always computes in double precision, unlike :func:`torch.pow`, + which implements more typical :ref:`type promotion `. + This is useful when the computation needs to be performed in a wider or more precise dtype, + or the results of the computation may contain fractional values not representable in the input dtypes, + like when an integer base is raised to a negative integer exponent. + + Args: + input (Tensor or Number): the base value(s) + exponent (Tensor or Number): the exponent value(s) + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randint(10, (4,)) + >>> a + tensor([6, 4, 7, 1]) + >>> torch.float_power(a, 2) + tensor([36., 16., 49., 1.], dtype=torch.float64) + + >>> a = torch.arange(1, 5) + >>> a + tensor([ 1, 2, 3, 4]) + >>> exp = torch.tensor([2, -3, 4, -5]) + >>> exp + tensor([ 2, -3, 4, -5]) + >>> torch.float_power(a, exp) + tensor([1.0000e+00, 1.2500e-01, 8.1000e+01, 9.7656e-04], dtype=torch.float64) + """ + +@overload +def float_power( + input: Tensor, + exponent: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + float_power(input, exponent, *, out=None) -> Tensor + + Raises :attr:`input` to the power of :attr:`exponent`, elementwise, in double precision. + If neither input is complex returns a ``torch.float64`` tensor, + and if one or more inputs is complex returns a ``torch.complex128`` tensor. + + .. note:: + This function always computes in double precision, unlike :func:`torch.pow`, + which implements more typical :ref:`type promotion `. + This is useful when the computation needs to be performed in a wider or more precise dtype, + or the results of the computation may contain fractional values not representable in the input dtypes, + like when an integer base is raised to a negative integer exponent. + + Args: + input (Tensor or Number): the base value(s) + exponent (Tensor or Number): the exponent value(s) + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randint(10, (4,)) + >>> a + tensor([6, 4, 7, 1]) + >>> torch.float_power(a, 2) + tensor([36., 16., 49., 1.], dtype=torch.float64) + + >>> a = torch.arange(1, 5) + >>> a + tensor([ 1, 2, 3, 4]) + >>> exp = torch.tensor([2, -3, 4, -5]) + >>> exp + tensor([ 2, -3, 4, -5]) + >>> torch.float_power(a, exp) + tensor([1.0000e+00, 1.2500e-01, 8.1000e+01, 9.7656e-04], dtype=torch.float64) + """ + +def floor(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + floor(input, *, out=None) -> Tensor + + Returns a new tensor with the floor of the elements of :attr:`input`, + the largest integer less than or equal to each element. + + For integer inputs, follows the array-api convention of returning a + copy of the input tensor. + + .. math:: + \text{out}_{i} = \left\lfloor \text{input}_{i} \right\rfloor + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([-0.8166, 1.5308, -0.2530, -0.2091]) + >>> torch.floor(a) + tensor([-1., 1., -1., -1.]) + """ + +def floor_(input: Tensor) -> Tensor: ... +def floor_divide( + input: Tensor | Number, + other: Tensor | Number, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + floor_divide(input, other, *, out=None) -> Tensor + + .. note:: + + Before PyTorch 1.13 :func:`torch.floor_divide` incorrectly performed + truncation division. To restore the previous behavior use + :func:`torch.div` with ``rounding_mode='trunc'``. + + Computes :attr:`input` divided by :attr:`other`, elementwise, and floors + the result. + + .. math:: + \text{{out}}_i = \text{floor} \left( \frac{{\text{{input}}_i}}{{\text{{other}}_i}} \right) + + + + Supports broadcasting to a common shape, type promotion, and integer and float inputs. + + Args: + input (Tensor or Number): the dividend + other (Tensor or Number): the divisor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([4.0, 3.0]) + >>> b = torch.tensor([2.0, 2.0]) + >>> torch.floor_divide(a, b) + tensor([2.0, 1.0]) + >>> torch.floor_divide(a, 1.4) + tensor([2.0, 2.0]) + """ + +def fmax( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + fmax(input, other, *, out=None) -> Tensor + + Computes the element-wise maximum of :attr:`input` and :attr:`other`. + + This is like :func:`torch.maximum` except it handles NaNs differently: + if exactly one of the two elements being compared is a NaN then the non-NaN element is taken as the maximum. + Only if both elements are NaN is NaN propagated. + + This function is a wrapper around C++'s ``std::fmax`` and is similar to NumPy's ``fmax`` function. + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer and floating-point inputs. + + Args: + input (Tensor): the input tensor. + other (Tensor): the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([9.7, float('nan'), 3.1, float('nan')]) + >>> b = torch.tensor([-2.2, 0.5, float('nan'), float('nan')]) + >>> torch.fmax(a, b) + tensor([9.7000, 0.5000, 3.1000, nan]) + """ + +def fmin( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + fmin(input, other, *, out=None) -> Tensor + + Computes the element-wise minimum of :attr:`input` and :attr:`other`. + + This is like :func:`torch.minimum` except it handles NaNs differently: + if exactly one of the two elements being compared is a NaN then the non-NaN element is taken as the minimum. + Only if both elements are NaN is NaN propagated. + + This function is a wrapper around C++'s ``std::fmin`` and is similar to NumPy's ``fmin`` function. + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer and floating-point inputs. + + Args: + input (Tensor): the input tensor. + other (Tensor): the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([2.2, float('nan'), 2.1, float('nan')]) + >>> b = torch.tensor([-9.3, 0.1, float('nan'), float('nan')]) + >>> torch.fmin(a, b) + tensor([-9.3000, 0.1000, 2.1000, nan]) + """ + +@overload +def fmod( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + fmod(input, other, *, out=None) -> Tensor + + Applies C++'s `std::fmod `_ entrywise. + The result has the same sign as the dividend :attr:`input` and its absolute value + is less than that of :attr:`other`. + + This function may be defined in terms of :func:`torch.div` as + + .. code:: python + + torch.fmod(a, b) == a - a.div(b, rounding_mode="trunc") * b + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer and float inputs. + + .. note:: + + When the divisor is zero, returns ``NaN`` for floating point dtypes + on both CPU and GPU; raises ``RuntimeError`` for integer division by + zero on CPU; Integer division by zero on GPU may return any value. + + .. note:: + + Complex inputs are not supported. In some cases, it is not mathematically + possible to satisfy the definition of a modulo operation with complex numbers. + + .. seealso:: + + :func:`torch.remainder` which implements Python's modulus operator. + This one is defined using division rounding down the result. + + Args: + input (Tensor): the dividend + other (Tensor or Scalar): the divisor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.fmod(torch.tensor([-3., -2, -1, 1, 2, 3]), 2) + tensor([-1., -0., -1., 1., 0., 1.]) + >>> torch.fmod(torch.tensor([1, 2, 3, 4, 5]), -1.5) + tensor([1.0000, 0.5000, 0.0000, 1.0000, 0.5000]) + """ + +@overload +def fmod( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + fmod(input, other, *, out=None) -> Tensor + + Applies C++'s `std::fmod `_ entrywise. + The result has the same sign as the dividend :attr:`input` and its absolute value + is less than that of :attr:`other`. + + This function may be defined in terms of :func:`torch.div` as + + .. code:: python + + torch.fmod(a, b) == a - a.div(b, rounding_mode="trunc") * b + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer and float inputs. + + .. note:: + + When the divisor is zero, returns ``NaN`` for floating point dtypes + on both CPU and GPU; raises ``RuntimeError`` for integer division by + zero on CPU; Integer division by zero on GPU may return any value. + + .. note:: + + Complex inputs are not supported. In some cases, it is not mathematically + possible to satisfy the definition of a modulo operation with complex numbers. + + .. seealso:: + + :func:`torch.remainder` which implements Python's modulus operator. + This one is defined using division rounding down the result. + + Args: + input (Tensor): the dividend + other (Tensor or Scalar): the divisor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.fmod(torch.tensor([-3., -2, -1, 1, 2, 3]), 2) + tensor([-1., -0., -1., 1., 0., 1.]) + >>> torch.fmod(torch.tensor([1, 2, 3, 4, 5]), -1.5) + tensor([1.0000, 0.5000, 0.0000, 1.0000, 0.5000]) + """ + +def frac(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + frac(input, *, out=None) -> Tensor + + Computes the fractional portion of each element in :attr:`input`. + + .. math:: + \text{out}_{i} = \text{input}_{i} - \left\lfloor |\text{input}_{i}| \right\rfloor * \operatorname{sgn}(\text{input}_{i}) + + Example:: + + >>> torch.frac(torch.tensor([1, 2.5, -3.2])) + tensor([ 0.0000, 0.5000, -0.2000]) + """ + +def frac_(input: Tensor) -> Tensor: ... +def frexp( + input: Tensor, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.frexp: + r""" + frexp(input, *, out=None) -> (Tensor mantissa, Tensor exponent) + + Decomposes :attr:`input` into mantissa and exponent tensors + such that :math:`\text{input} = \text{mantissa} \times 2^{\text{exponent}}`. + + The range of mantissa is the open interval (-1, 1). + + Supports float inputs. + + Args: + input (Tensor): the input tensor + + + Keyword args: + out (tuple, optional): the output tensors + + Example:: + + >>> x = torch.arange(9.) + >>> mantissa, exponent = torch.frexp(x) + >>> mantissa + tensor([0.0000, 0.5000, 0.5000, 0.7500, 0.5000, 0.6250, 0.7500, 0.8750, 0.5000]) + >>> exponent + tensor([0, 1, 2, 2, 3, 3, 3, 3, 4], dtype=torch.int32) + >>> torch.ldexp(mantissa, exponent) + tensor([0., 1., 2., 3., 4., 5., 6., 7., 8.]) + """ + +def frobenius_norm( + input: Tensor, + dim: _int | _size, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: ... +def from_file( + filename: str, + shared: _bool | None = None, + size: _int | None = 0, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + from_file(filename, shared=None, size=0, *, dtype=None, layout=None, device=None, pin_memory=False) + + Creates a CPU tensor with a storage backed by a memory-mapped file. + + If ``shared`` is True, then memory is shared between processes. All changes are written to the file. + If ``shared`` is False, then changes to the tensor do not affect the file. + + ``size`` is the number of elements in the Tensor. If ``shared`` is ``False``, then the file must contain + at least ``size * sizeof(dtype)`` bytes. If ``shared`` is ``True`` the file will be created if needed. + + .. note:: + Only CPU tensors can be mapped to files. + + .. note:: + For now, tensors with storages backed by a memory-mapped file cannot be created in pinned memory. + + + Args: + filename (str): file name to map + shared (bool): whether to share memory (whether ``MAP_SHARED`` or ``MAP_PRIVATE`` is passed to the + underlying `mmap(2) call `_) + size (int): number of elements in the tensor + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> t = torch.randn(2, 5, dtype=torch.float64) + >>> t.numpy().tofile('storage.pt') + >>> t_mapped = torch.from_file('storage.pt', shared=False, size=10, dtype=torch.float64) + """ + +def from_numpy(ndarray) -> Tensor: + r""" + from_numpy(ndarray) -> Tensor + + Creates a :class:`Tensor` from a :class:`numpy.ndarray`. + + The returned tensor and :attr:`ndarray` share the same memory. Modifications to + the tensor will be reflected in the :attr:`ndarray` and vice versa. The returned + tensor is not resizable. + + It currently accepts :attr:`ndarray` with dtypes of ``numpy.float64``, + ``numpy.float32``, ``numpy.float16``, ``numpy.complex64``, ``numpy.complex128``, + ``numpy.int64``, ``numpy.int32``, ``numpy.int16``, ``numpy.int8``, ``numpy.uint8``, + and ``bool``. + + .. warning:: + Writing to a tensor created from a read-only NumPy array is not supported and will result in undefined behavior. + + Example:: + + >>> a = numpy.array([1, 2, 3]) + >>> t = torch.from_numpy(a) + >>> t + tensor([ 1, 2, 3]) + >>> t[0] = -1 + >>> a + array([-1, 2, 3]) + """ + +def frombuffer( + buffer: Any, + *, + dtype: _dtype, + count: int = -1, + offset: int = 0, + requires_grad: _bool = False, +) -> Tensor: + r""" + frombuffer(buffer, *, dtype, count=-1, offset=0, requires_grad=False) -> Tensor + + Creates a 1-dimensional :class:`Tensor` from an object that implements + the Python buffer protocol. + + Skips the first :attr:`offset` bytes in the buffer, and interprets the rest of + the raw bytes as a 1-dimensional tensor of type :attr:`dtype` with :attr:`count` + elements. + + Note that either of the following must be true: + + 1. :attr:`count` is a positive non-zero number, and the total number of bytes + in the buffer is more than :attr:`offset` plus :attr:`count` times the size + (in bytes) of :attr:`dtype`. + + 2. :attr:`count` is negative, and the length (number of bytes) of the buffer + subtracted by the :attr:`offset` is a multiple of the size (in bytes) of + :attr:`dtype`. + + The returned tensor and buffer share the same memory. Modifications to + the tensor will be reflected in the buffer and vice versa. The returned + tensor is not resizable. + + .. note:: + This function increments the reference count for the object that + owns the shared memory. Therefore, such memory will not be deallocated + before the returned tensor goes out of scope. + + .. warning:: + This function's behavior is undefined when passed an object implementing + the buffer protocol whose data is not on the CPU. Doing so is likely to + cause a segmentation fault. + + .. warning:: + This function does not try to infer the :attr:`dtype` (hence, it is not + optional). Passing a different :attr:`dtype` than its source may result + in unexpected behavior. + + Args: + buffer (object): a Python object that exposes the buffer interface. + + Keyword args: + dtype (:class:`torch.dtype`): the desired data type of returned tensor. + count (int, optional): the number of desired elements to be read. + If negative, all the elements (until the end of the buffer) will be + read. Default: -1. + offset (int, optional): the number of bytes to skip at the start of + the buffer. Default: 0. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> import array + >>> a = array.array('i', [1, 2, 3]) + >>> t = torch.frombuffer(a, dtype=torch.int32) + >>> t + tensor([ 1, 2, 3]) + >>> t[0] = -1 + >>> a + array([-1, 2, 3]) + + >>> # Interprets the signed char bytes as 32-bit integers. + >>> # Each 4 signed char elements will be interpreted as + >>> # 1 signed 32-bit integer. + >>> import array + >>> a = array.array('b', [-1, 0, 0, 0]) + >>> torch.frombuffer(a, dtype=torch.int32) + tensor([255], dtype=torch.int32) + """ + +@overload +def full( + size: _size, + fill_value: Number | _complex, + *, + out: Tensor | None = None, + layout: _layout = strided, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + pin_memory: _bool = False, +) -> Tensor: + r""" + full(size, fill_value, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Creates a tensor of size :attr:`size` filled with :attr:`fill_value`. The + tensor's dtype is inferred from :attr:`fill_value`. + + Args: + size (int...): a list, tuple, or :class:`torch.Size` of integers defining the + shape of the output tensor. + fill_value (Scalar): the value to fill the output tensor with. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.full((2, 3), 3.141592) + tensor([[ 3.1416, 3.1416, 3.1416], + [ 3.1416, 3.1416, 3.1416]]) + """ + +@overload +def full( + size: _size, + fill_value: Number | _complex, + *, + names: list[str | None], + layout: _layout = strided, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + pin_memory: _bool = False, +) -> Tensor: + r""" + full(size, fill_value, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Creates a tensor of size :attr:`size` filled with :attr:`fill_value`. The + tensor's dtype is inferred from :attr:`fill_value`. + + Args: + size (int...): a list, tuple, or :class:`torch.Size` of integers defining the + shape of the output tensor. + fill_value (Scalar): the value to fill the output tensor with. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.full((2, 3), 3.141592) + tensor([[ 3.1416, 3.1416, 3.1416], + [ 3.1416, 3.1416, 3.1416]]) + """ + +@overload +def full( + size: Sequence[_int | SymInt], + fill_value: Number | _complex, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + full(size, fill_value, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Creates a tensor of size :attr:`size` filled with :attr:`fill_value`. The + tensor's dtype is inferred from :attr:`fill_value`. + + Args: + size (int...): a list, tuple, or :class:`torch.Size` of integers defining the + shape of the output tensor. + fill_value (Scalar): the value to fill the output tensor with. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.full((2, 3), 3.141592) + tensor([[ 3.1416, 3.1416, 3.1416], + [ 3.1416, 3.1416, 3.1416]]) + """ + +@overload +def full( + size: _size, + fill_value: Number | _complex, + *, + names: Sequence[str | EllipsisType | None] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + full(size, fill_value, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Creates a tensor of size :attr:`size` filled with :attr:`fill_value`. The + tensor's dtype is inferred from :attr:`fill_value`. + + Args: + size (int...): a list, tuple, or :class:`torch.Size` of integers defining the + shape of the output tensor. + fill_value (Scalar): the value to fill the output tensor with. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.full((2, 3), 3.141592) + tensor([[ 3.1416, 3.1416, 3.1416], + [ 3.1416, 3.1416, 3.1416]]) + """ + +def full_like( + input: Tensor, + fill_value: Number | _complex, + *, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + full_like(input, fill_value, \*, dtype=None, layout=torch.strided, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns a tensor with the same size as :attr:`input` filled with :attr:`fill_value`. + ``torch.full_like(input, fill_value)`` is equivalent to + ``torch.full(input.size(), fill_value, dtype=input.dtype, layout=input.layout, device=input.device)``. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + fill_value: the number to fill the output tensor with. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + + Example:: + + >>> x = torch.ones(2, 3) + >>> torch.full_like(x, 3.141592) + tensor([[ 3.1416, 3.1416, 3.1416], + [ 3.1416, 3.1416, 3.1416]]) + >>> torch.full_like(x, 7) + tensor([[7., 7., 7.], + [7., 7., 7.]]) + >>> torch.full_like(x, 0.5, dtype=torch.int32) + tensor([[0, 0, 0], + [0, 0, 0]], dtype=torch.int32) + >>> y = torch.randn(3, 4, dtype=torch.float64) + >>> torch.full_like(y, -1.0) + tensor([[-1., -1., -1., -1.], + [-1., -1., -1., -1.], + [-1., -1., -1., -1.]], dtype=torch.float64) + """ + +def fused_moving_avg_obs_fake_quant( + input: Tensor, + observer_on: Tensor, + fake_quant_on: Tensor, + running_min: Tensor, + running_max: Tensor, + scale: Tensor, + zero_point: Tensor, + averaging_const: _float, + quant_min: _int, + quant_max: _int, + ch_axis: _int, + per_row_fake_quant: _bool = False, + symmetric_quant: _bool = False, +) -> Tensor: ... +@overload +def gather( + input: Tensor, + dim: _int, + index: Tensor, + *, + sparse_grad: _bool = False, + out: Tensor | None = None, +) -> Tensor: + r""" + gather(input, dim, index, *, sparse_grad=False, out=None) -> Tensor + + Gathers values along an axis specified by `dim`. + + For a 3-D tensor the output is specified by:: + + out[i][j][k] = input[index[i][j][k]][j][k] # if dim == 0 + out[i][j][k] = input[i][index[i][j][k]][k] # if dim == 1 + out[i][j][k] = input[i][j][index[i][j][k]] # if dim == 2 + + :attr:`input` and :attr:`index` must have the same number of dimensions. + It is also required that ``index.size(d) <= input.size(d)`` for all + dimensions ``d != dim``. :attr:`out` will have the same shape as :attr:`index`. + Note that ``input`` and ``index`` do not broadcast against each other. + When :attr:`index` is empty, we always return an empty output with the same shape + without further error checking. + + Args: + input (Tensor): the source tensor + dim (int): the axis along which to index + index (LongTensor): the indices of elements to gather + + Keyword arguments: + sparse_grad (bool, optional): If ``True``, gradient w.r.t. :attr:`input` will be a sparse tensor. + out (Tensor, optional): the destination tensor + + Example:: + + >>> t = torch.tensor([[1, 2], [3, 4]]) + >>> torch.gather(t, 1, torch.tensor([[0, 0], [1, 0]])) + tensor([[ 1, 1], + [ 4, 3]]) + """ + +@overload +def gather( + input: Tensor, + dim: str | EllipsisType | None, + index: Tensor, + *, + sparse_grad: _bool = False, + out: Tensor | None = None, +) -> Tensor: + r""" + gather(input, dim, index, *, sparse_grad=False, out=None) -> Tensor + + Gathers values along an axis specified by `dim`. + + For a 3-D tensor the output is specified by:: + + out[i][j][k] = input[index[i][j][k]][j][k] # if dim == 0 + out[i][j][k] = input[i][index[i][j][k]][k] # if dim == 1 + out[i][j][k] = input[i][j][index[i][j][k]] # if dim == 2 + + :attr:`input` and :attr:`index` must have the same number of dimensions. + It is also required that ``index.size(d) <= input.size(d)`` for all + dimensions ``d != dim``. :attr:`out` will have the same shape as :attr:`index`. + Note that ``input`` and ``index`` do not broadcast against each other. + When :attr:`index` is empty, we always return an empty output with the same shape + without further error checking. + + Args: + input (Tensor): the source tensor + dim (int): the axis along which to index + index (LongTensor): the indices of elements to gather + + Keyword arguments: + sparse_grad (bool, optional): If ``True``, gradient w.r.t. :attr:`input` will be a sparse tensor. + out (Tensor, optional): the destination tensor + + Example:: + + >>> t = torch.tensor([[1, 2], [3, 4]]) + >>> torch.gather(t, 1, torch.tensor([[0, 0], [1, 0]])) + tensor([[ 1, 1], + [ 4, 3]]) + """ + +def gcd( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + gcd(input, other, *, out=None) -> Tensor + + Computes the element-wise greatest common divisor (GCD) of :attr:`input` and :attr:`other`. + + Both :attr:`input` and :attr:`other` must have integer types. + + .. note:: + This defines :math:`gcd(0, 0) = 0`. + + Args: + input (Tensor): the input tensor. + other (Tensor): the second input tensor + + Keyword arguments: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([5, 10, 15]) + >>> b = torch.tensor([3, 4, 5]) + >>> torch.gcd(a, b) + tensor([1, 2, 5]) + >>> c = torch.tensor([3]) + >>> torch.gcd(a, c) + tensor([1, 1, 3]) + """ + +def gcd_(input: Tensor, other: Tensor) -> Tensor: ... +@overload +def ge( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + ge(input, other, *, out=None) -> Tensor + + Computes :math:`\text{input} \geq \text{other}` element-wise. + + + The second argument can be a number or a tensor whose shape is + :ref:`broadcastable ` with the first argument. + + Args: + input (Tensor): the tensor to compare + other (Tensor or float): the tensor or value to compare + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is greater than or equal to :attr:`other` and False elsewhere + + Example:: + + >>> torch.ge(torch.tensor([[1, 2], [3, 4]]), torch.tensor([[1, 1], [4, 4]])) + tensor([[True, True], [False, True]]) + """ + +@overload +def ge( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + ge(input, other, *, out=None) -> Tensor + + Computes :math:`\text{input} \geq \text{other}` element-wise. + + + The second argument can be a number or a tensor whose shape is + :ref:`broadcastable ` with the first argument. + + Args: + input (Tensor): the tensor to compare + other (Tensor or float): the tensor or value to compare + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is greater than or equal to :attr:`other` and False elsewhere + + Example:: + + >>> torch.ge(torch.tensor([[1, 2], [3, 4]]), torch.tensor([[1, 1], [4, 4]])) + tensor([[True, True], [False, True]]) + """ + +def geqrf( + input: Tensor, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.geqrf: + r""" + geqrf(input, *, out=None) -> (Tensor, Tensor) + + This is a low-level function for calling LAPACK's geqrf directly. This function + returns a namedtuple (a, tau) as defined in `LAPACK documentation for geqrf`_ . + + Computes a QR decomposition of :attr:`input`. + Both `Q` and `R` matrices are stored in the same output tensor `a`. + The elements of `R` are stored on and above the diagonal. + Elementary reflectors (or Householder vectors) implicitly defining matrix `Q` + are stored below the diagonal. + The results of this function can be used together with :func:`torch.linalg.householder_product` + to obtain the `Q` matrix or + with :func:`torch.ormqr`, which uses an implicit representation of the `Q` matrix, + for an efficient matrix-matrix multiplication. + + See `LAPACK documentation for geqrf`_ for further details. + + .. note:: + See also :func:`torch.linalg.qr`, which computes Q and R matrices, and :func:`torch.linalg.lstsq` + with the ``driver="gels"`` option for a function that can solve matrix equations using a QR decomposition. + + Args: + input (Tensor): the input matrix + + Keyword args: + out (tuple, optional): the output tuple of (Tensor, Tensor). Ignored if `None`. Default: `None`. + + .. _LAPACK documentation for geqrf: + http://www.netlib.org/lapack/explore-html/df/dc5/group__variants_g_ecomputational_ga3766ea903391b5cf9008132f7440ec7b.html + """ + +def ger( + input: Tensor, + vec2: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + ger(input, vec2, *, out=None) -> Tensor + + Alias of :func:`torch.outer`. + + .. warning:: + This function is deprecated and will be removed in a future PyTorch release. + Use :func:`torch.outer` instead. + """ + +def get_default_dtype() -> _dtype: + r""" + get_default_dtype() -> torch.dtype + + Get the current default floating point :class:`torch.dtype`. + + Example:: + + >>> torch.get_default_dtype() # initial default for floating point is torch.float32 + torch.float32 + >>> torch.set_default_dtype(torch.float64) + >>> torch.get_default_dtype() # default is now changed to torch.float64 + torch.float64 + """ + +def get_num_interop_threads() -> _int: + r""" + get_num_interop_threads() -> int + + Returns the number of threads used for inter-op parallelism on CPU + (e.g. in JIT interpreter) + """ + +def get_num_threads() -> _int: + r""" + get_num_threads() -> int + + Returns the number of threads used for parallelizing CPU operations + """ + +@overload +def gradient( + input: Tensor, + *, + spacing: Number | _complex | None = None, + dim: _int | None = None, + edge_order: _int = 1, +) -> tuple[Tensor, ...]: + r""" + gradient(input, *, spacing=1, dim=None, edge_order=1) -> List of Tensors + + Estimates the gradient of a function :math:`g : \mathbb{R}^n \rightarrow \mathbb{R}` in + one or more dimensions using the `second-order accurate central differences method + `_ and + either first or second order estimates at the boundaries. + + The gradient of :math:`g` is estimated using samples. By default, when :attr:`spacing` is not + specified, the samples are entirely described by :attr:`input`, and the mapping of input coordinates + to an output is the same as the tensor's mapping of indices to values. For example, for a three-dimensional + :attr:`input` the function described is :math:`g : \mathbb{R}^3 \rightarrow \mathbb{R}`, and + :math:`g(1, 2, 3)\ == input[1, 2, 3]`. + + When :attr:`spacing` is specified, it modifies the relationship between :attr:`input` and input coordinates. + This is detailed in the "Keyword Arguments" section below. + + The gradient is estimated by estimating each partial derivative of :math:`g` independently. This estimation is + accurate if :math:`g` is in :math:`C^3` (it has at least 3 continuous derivatives), and the estimation can be + improved by providing closer samples. Mathematically, the value at each interior point of a partial derivative + is estimated using `Taylor's theorem with remainder `_. + Letting :math:`x` be an interior point with :math:`x-h_l` and :math:`x+h_r` be points neighboring + it to the left and right respectively, :math:`f(x+h_r)` and :math:`f(x-h_l)` can be estimated using: + + .. math:: + \begin{aligned} + f(x+h_r) = f(x) + h_r f'(x) + {h_r}^2 \frac{f''(x)}{2} + {h_r}^3 \frac{f'''(\xi_1)}{6}, \xi_1 \in (x, x+h_r) \\ + f(x-h_l) = f(x) - h_l f'(x) + {h_l}^2 \frac{f''(x)}{2} - {h_l}^3 \frac{f'''(\xi_2)}{6}, \xi_2 \in (x, x-h_l) \\ + \end{aligned} + + Using the fact that :math:`f \in C^3` and solving the linear system, we derive: + + .. math:: + f'(x) \approx \frac{ {h_l}^2 f(x+h_r) - {h_r}^2 f(x-h_l) + + ({h_r}^2-{h_l}^2 ) f(x) }{ {h_r} {h_l}^2 + {h_r}^2 {h_l} } + + .. note:: + We estimate the gradient of functions in complex domain + :math:`g : \mathbb{C}^n \rightarrow \mathbb{C}` in the same way. + + The value of each partial derivative at the boundary points is computed differently. See edge_order below. + + Args: + input (``Tensor``): the tensor that represents the values of the function + + Keyword args: + spacing (``scalar``, ``list of scalar``, ``list of Tensor``, optional): :attr:`spacing` can be used to modify + how the :attr:`input` tensor's indices relate to sample coordinates. If :attr:`spacing` is a scalar then + the indices are multiplied by the scalar to produce the coordinates. For example, if :attr:`spacing=2` the + indices (1, 2, 3) become coordinates (2, 4, 6). If :attr:`spacing` is a list of scalars then the corresponding + indices are multiplied. For example, if :attr:`spacing=(2, -1, 3)` the indices (1, 2, 3) become coordinates (2, -2, 9). + Finally, if :attr:`spacing` is a list of one-dimensional tensors then each tensor specifies the coordinates for + the corresponding dimension. For example, if the indices are (1, 2, 3) and the tensors are (t0, t1, t2), then + the coordinates are (t0[1], t1[2], t2[3]) + + dim (``int``, ``list of int``, optional): the dimension or dimensions to approximate the gradient over. By default + the partial gradient in every dimension is computed. Note that when :attr:`dim` is specified the elements of + the :attr:`spacing` argument must correspond with the specified dims." + + edge_order (``int``, optional): 1 or 2, for `first-order + `_ or + `second-order `_ + estimation of the boundary ("edge") values, respectively. Note that when :attr:`edge_order` is specified, each + dimension size of :attr:`input` should be at least edge_order+1 + + Examples:: + + >>> # Estimates the gradient of f(x)=x^2 at points [-2, -1, 2, 4] + >>> coordinates = (torch.tensor([-2., -1., 1., 4.]),) + >>> values = torch.tensor([4., 1., 1., 16.], ) + >>> torch.gradient(values, spacing = coordinates) + (tensor([-3., -2., 2., 5.]),) + + >>> # Estimates the gradient of the R^2 -> R function whose samples are + >>> # described by the tensor t. Implicit coordinates are [0, 1] for the outermost + >>> # dimension and [0, 1, 2, 3] for the innermost dimension, and function estimates + >>> # partial derivative for both dimensions. + >>> t = torch.tensor([[1, 2, 4, 8], [10, 20, 40, 80]]) + >>> torch.gradient(t) + (tensor([[ 9., 18., 36., 72.], + [ 9., 18., 36., 72.]]), + tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]])) + + >>> # A scalar value for spacing modifies the relationship between tensor indices + >>> # and input coordinates by multiplying the indices to find the + >>> # coordinates. For example, below the indices of the innermost + >>> # 0, 1, 2, 3 translate to coordinates of [0, 2, 4, 6], and the indices of + >>> # the outermost dimension 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = 2.0) # dim = None (implicitly [0, 1]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.5000, 0.7500, 1.5000, 2.0000], + [ 5.0000, 7.5000, 15.0000, 20.0000]])) + >>> # doubling the spacing between samples halves the estimated partial gradients. + + >>> + >>> # Estimates only the partial derivative for dimension 1 + >>> torch.gradient(t, dim = 1) # spacing = None (implicitly 1.) + (tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]]),) + + >>> # When spacing is a list of scalars, the relationship between the tensor + >>> # indices and input coordinates changes based on dimension. + >>> # For example, below, the indices of the innermost dimension 0, 1, 2, 3 translate + >>> # to coordinates of [0, 3, 6, 9], and the indices of the outermost dimension + >>> # 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = [3., 2.]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + + >>> # The following example is a replication of the previous one with explicit + >>> # coordinates. + >>> coords = (torch.tensor([0, 2]), torch.tensor([0, 3, 6, 9])) + >>> torch.gradient(t, spacing = coords) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + """ + +@overload +def gradient( + input: Tensor, + *, + spacing: Sequence[Number | _complex], + dim: _int | None = None, + edge_order: _int = 1, +) -> tuple[Tensor, ...]: + r""" + gradient(input, *, spacing=1, dim=None, edge_order=1) -> List of Tensors + + Estimates the gradient of a function :math:`g : \mathbb{R}^n \rightarrow \mathbb{R}` in + one or more dimensions using the `second-order accurate central differences method + `_ and + either first or second order estimates at the boundaries. + + The gradient of :math:`g` is estimated using samples. By default, when :attr:`spacing` is not + specified, the samples are entirely described by :attr:`input`, and the mapping of input coordinates + to an output is the same as the tensor's mapping of indices to values. For example, for a three-dimensional + :attr:`input` the function described is :math:`g : \mathbb{R}^3 \rightarrow \mathbb{R}`, and + :math:`g(1, 2, 3)\ == input[1, 2, 3]`. + + When :attr:`spacing` is specified, it modifies the relationship between :attr:`input` and input coordinates. + This is detailed in the "Keyword Arguments" section below. + + The gradient is estimated by estimating each partial derivative of :math:`g` independently. This estimation is + accurate if :math:`g` is in :math:`C^3` (it has at least 3 continuous derivatives), and the estimation can be + improved by providing closer samples. Mathematically, the value at each interior point of a partial derivative + is estimated using `Taylor's theorem with remainder `_. + Letting :math:`x` be an interior point with :math:`x-h_l` and :math:`x+h_r` be points neighboring + it to the left and right respectively, :math:`f(x+h_r)` and :math:`f(x-h_l)` can be estimated using: + + .. math:: + \begin{aligned} + f(x+h_r) = f(x) + h_r f'(x) + {h_r}^2 \frac{f''(x)}{2} + {h_r}^3 \frac{f'''(\xi_1)}{6}, \xi_1 \in (x, x+h_r) \\ + f(x-h_l) = f(x) - h_l f'(x) + {h_l}^2 \frac{f''(x)}{2} - {h_l}^3 \frac{f'''(\xi_2)}{6}, \xi_2 \in (x, x-h_l) \\ + \end{aligned} + + Using the fact that :math:`f \in C^3` and solving the linear system, we derive: + + .. math:: + f'(x) \approx \frac{ {h_l}^2 f(x+h_r) - {h_r}^2 f(x-h_l) + + ({h_r}^2-{h_l}^2 ) f(x) }{ {h_r} {h_l}^2 + {h_r}^2 {h_l} } + + .. note:: + We estimate the gradient of functions in complex domain + :math:`g : \mathbb{C}^n \rightarrow \mathbb{C}` in the same way. + + The value of each partial derivative at the boundary points is computed differently. See edge_order below. + + Args: + input (``Tensor``): the tensor that represents the values of the function + + Keyword args: + spacing (``scalar``, ``list of scalar``, ``list of Tensor``, optional): :attr:`spacing` can be used to modify + how the :attr:`input` tensor's indices relate to sample coordinates. If :attr:`spacing` is a scalar then + the indices are multiplied by the scalar to produce the coordinates. For example, if :attr:`spacing=2` the + indices (1, 2, 3) become coordinates (2, 4, 6). If :attr:`spacing` is a list of scalars then the corresponding + indices are multiplied. For example, if :attr:`spacing=(2, -1, 3)` the indices (1, 2, 3) become coordinates (2, -2, 9). + Finally, if :attr:`spacing` is a list of one-dimensional tensors then each tensor specifies the coordinates for + the corresponding dimension. For example, if the indices are (1, 2, 3) and the tensors are (t0, t1, t2), then + the coordinates are (t0[1], t1[2], t2[3]) + + dim (``int``, ``list of int``, optional): the dimension or dimensions to approximate the gradient over. By default + the partial gradient in every dimension is computed. Note that when :attr:`dim` is specified the elements of + the :attr:`spacing` argument must correspond with the specified dims." + + edge_order (``int``, optional): 1 or 2, for `first-order + `_ or + `second-order `_ + estimation of the boundary ("edge") values, respectively. Note that when :attr:`edge_order` is specified, each + dimension size of :attr:`input` should be at least edge_order+1 + + Examples:: + + >>> # Estimates the gradient of f(x)=x^2 at points [-2, -1, 2, 4] + >>> coordinates = (torch.tensor([-2., -1., 1., 4.]),) + >>> values = torch.tensor([4., 1., 1., 16.], ) + >>> torch.gradient(values, spacing = coordinates) + (tensor([-3., -2., 2., 5.]),) + + >>> # Estimates the gradient of the R^2 -> R function whose samples are + >>> # described by the tensor t. Implicit coordinates are [0, 1] for the outermost + >>> # dimension and [0, 1, 2, 3] for the innermost dimension, and function estimates + >>> # partial derivative for both dimensions. + >>> t = torch.tensor([[1, 2, 4, 8], [10, 20, 40, 80]]) + >>> torch.gradient(t) + (tensor([[ 9., 18., 36., 72.], + [ 9., 18., 36., 72.]]), + tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]])) + + >>> # A scalar value for spacing modifies the relationship between tensor indices + >>> # and input coordinates by multiplying the indices to find the + >>> # coordinates. For example, below the indices of the innermost + >>> # 0, 1, 2, 3 translate to coordinates of [0, 2, 4, 6], and the indices of + >>> # the outermost dimension 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = 2.0) # dim = None (implicitly [0, 1]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.5000, 0.7500, 1.5000, 2.0000], + [ 5.0000, 7.5000, 15.0000, 20.0000]])) + >>> # doubling the spacing between samples halves the estimated partial gradients. + + >>> + >>> # Estimates only the partial derivative for dimension 1 + >>> torch.gradient(t, dim = 1) # spacing = None (implicitly 1.) + (tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]]),) + + >>> # When spacing is a list of scalars, the relationship between the tensor + >>> # indices and input coordinates changes based on dimension. + >>> # For example, below, the indices of the innermost dimension 0, 1, 2, 3 translate + >>> # to coordinates of [0, 3, 6, 9], and the indices of the outermost dimension + >>> # 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = [3., 2.]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + + >>> # The following example is a replication of the previous one with explicit + >>> # coordinates. + >>> coords = (torch.tensor([0, 2]), torch.tensor([0, 3, 6, 9])) + >>> torch.gradient(t, spacing = coords) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + """ + +@overload +def gradient( + input: Tensor, + *, + spacing: Sequence[Number | _complex], + dim: _size, + edge_order: _int = 1, +) -> tuple[Tensor, ...]: + r""" + gradient(input, *, spacing=1, dim=None, edge_order=1) -> List of Tensors + + Estimates the gradient of a function :math:`g : \mathbb{R}^n \rightarrow \mathbb{R}` in + one or more dimensions using the `second-order accurate central differences method + `_ and + either first or second order estimates at the boundaries. + + The gradient of :math:`g` is estimated using samples. By default, when :attr:`spacing` is not + specified, the samples are entirely described by :attr:`input`, and the mapping of input coordinates + to an output is the same as the tensor's mapping of indices to values. For example, for a three-dimensional + :attr:`input` the function described is :math:`g : \mathbb{R}^3 \rightarrow \mathbb{R}`, and + :math:`g(1, 2, 3)\ == input[1, 2, 3]`. + + When :attr:`spacing` is specified, it modifies the relationship between :attr:`input` and input coordinates. + This is detailed in the "Keyword Arguments" section below. + + The gradient is estimated by estimating each partial derivative of :math:`g` independently. This estimation is + accurate if :math:`g` is in :math:`C^3` (it has at least 3 continuous derivatives), and the estimation can be + improved by providing closer samples. Mathematically, the value at each interior point of a partial derivative + is estimated using `Taylor's theorem with remainder `_. + Letting :math:`x` be an interior point with :math:`x-h_l` and :math:`x+h_r` be points neighboring + it to the left and right respectively, :math:`f(x+h_r)` and :math:`f(x-h_l)` can be estimated using: + + .. math:: + \begin{aligned} + f(x+h_r) = f(x) + h_r f'(x) + {h_r}^2 \frac{f''(x)}{2} + {h_r}^3 \frac{f'''(\xi_1)}{6}, \xi_1 \in (x, x+h_r) \\ + f(x-h_l) = f(x) - h_l f'(x) + {h_l}^2 \frac{f''(x)}{2} - {h_l}^3 \frac{f'''(\xi_2)}{6}, \xi_2 \in (x, x-h_l) \\ + \end{aligned} + + Using the fact that :math:`f \in C^3` and solving the linear system, we derive: + + .. math:: + f'(x) \approx \frac{ {h_l}^2 f(x+h_r) - {h_r}^2 f(x-h_l) + + ({h_r}^2-{h_l}^2 ) f(x) }{ {h_r} {h_l}^2 + {h_r}^2 {h_l} } + + .. note:: + We estimate the gradient of functions in complex domain + :math:`g : \mathbb{C}^n \rightarrow \mathbb{C}` in the same way. + + The value of each partial derivative at the boundary points is computed differently. See edge_order below. + + Args: + input (``Tensor``): the tensor that represents the values of the function + + Keyword args: + spacing (``scalar``, ``list of scalar``, ``list of Tensor``, optional): :attr:`spacing` can be used to modify + how the :attr:`input` tensor's indices relate to sample coordinates. If :attr:`spacing` is a scalar then + the indices are multiplied by the scalar to produce the coordinates. For example, if :attr:`spacing=2` the + indices (1, 2, 3) become coordinates (2, 4, 6). If :attr:`spacing` is a list of scalars then the corresponding + indices are multiplied. For example, if :attr:`spacing=(2, -1, 3)` the indices (1, 2, 3) become coordinates (2, -2, 9). + Finally, if :attr:`spacing` is a list of one-dimensional tensors then each tensor specifies the coordinates for + the corresponding dimension. For example, if the indices are (1, 2, 3) and the tensors are (t0, t1, t2), then + the coordinates are (t0[1], t1[2], t2[3]) + + dim (``int``, ``list of int``, optional): the dimension or dimensions to approximate the gradient over. By default + the partial gradient in every dimension is computed. Note that when :attr:`dim` is specified the elements of + the :attr:`spacing` argument must correspond with the specified dims." + + edge_order (``int``, optional): 1 or 2, for `first-order + `_ or + `second-order `_ + estimation of the boundary ("edge") values, respectively. Note that when :attr:`edge_order` is specified, each + dimension size of :attr:`input` should be at least edge_order+1 + + Examples:: + + >>> # Estimates the gradient of f(x)=x^2 at points [-2, -1, 2, 4] + >>> coordinates = (torch.tensor([-2., -1., 1., 4.]),) + >>> values = torch.tensor([4., 1., 1., 16.], ) + >>> torch.gradient(values, spacing = coordinates) + (tensor([-3., -2., 2., 5.]),) + + >>> # Estimates the gradient of the R^2 -> R function whose samples are + >>> # described by the tensor t. Implicit coordinates are [0, 1] for the outermost + >>> # dimension and [0, 1, 2, 3] for the innermost dimension, and function estimates + >>> # partial derivative for both dimensions. + >>> t = torch.tensor([[1, 2, 4, 8], [10, 20, 40, 80]]) + >>> torch.gradient(t) + (tensor([[ 9., 18., 36., 72.], + [ 9., 18., 36., 72.]]), + tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]])) + + >>> # A scalar value for spacing modifies the relationship between tensor indices + >>> # and input coordinates by multiplying the indices to find the + >>> # coordinates. For example, below the indices of the innermost + >>> # 0, 1, 2, 3 translate to coordinates of [0, 2, 4, 6], and the indices of + >>> # the outermost dimension 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = 2.0) # dim = None (implicitly [0, 1]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.5000, 0.7500, 1.5000, 2.0000], + [ 5.0000, 7.5000, 15.0000, 20.0000]])) + >>> # doubling the spacing between samples halves the estimated partial gradients. + + >>> + >>> # Estimates only the partial derivative for dimension 1 + >>> torch.gradient(t, dim = 1) # spacing = None (implicitly 1.) + (tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]]),) + + >>> # When spacing is a list of scalars, the relationship between the tensor + >>> # indices and input coordinates changes based on dimension. + >>> # For example, below, the indices of the innermost dimension 0, 1, 2, 3 translate + >>> # to coordinates of [0, 3, 6, 9], and the indices of the outermost dimension + >>> # 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = [3., 2.]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + + >>> # The following example is a replication of the previous one with explicit + >>> # coordinates. + >>> coords = (torch.tensor([0, 2]), torch.tensor([0, 3, 6, 9])) + >>> torch.gradient(t, spacing = coords) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + """ + +@overload +def gradient( + input: Tensor, + *, + spacing: tuple[Tensor, ...] | list[Tensor] | None, + dim: _int | None = None, + edge_order: _int = 1, +) -> tuple[Tensor, ...]: + r""" + gradient(input, *, spacing=1, dim=None, edge_order=1) -> List of Tensors + + Estimates the gradient of a function :math:`g : \mathbb{R}^n \rightarrow \mathbb{R}` in + one or more dimensions using the `second-order accurate central differences method + `_ and + either first or second order estimates at the boundaries. + + The gradient of :math:`g` is estimated using samples. By default, when :attr:`spacing` is not + specified, the samples are entirely described by :attr:`input`, and the mapping of input coordinates + to an output is the same as the tensor's mapping of indices to values. For example, for a three-dimensional + :attr:`input` the function described is :math:`g : \mathbb{R}^3 \rightarrow \mathbb{R}`, and + :math:`g(1, 2, 3)\ == input[1, 2, 3]`. + + When :attr:`spacing` is specified, it modifies the relationship between :attr:`input` and input coordinates. + This is detailed in the "Keyword Arguments" section below. + + The gradient is estimated by estimating each partial derivative of :math:`g` independently. This estimation is + accurate if :math:`g` is in :math:`C^3` (it has at least 3 continuous derivatives), and the estimation can be + improved by providing closer samples. Mathematically, the value at each interior point of a partial derivative + is estimated using `Taylor's theorem with remainder `_. + Letting :math:`x` be an interior point with :math:`x-h_l` and :math:`x+h_r` be points neighboring + it to the left and right respectively, :math:`f(x+h_r)` and :math:`f(x-h_l)` can be estimated using: + + .. math:: + \begin{aligned} + f(x+h_r) = f(x) + h_r f'(x) + {h_r}^2 \frac{f''(x)}{2} + {h_r}^3 \frac{f'''(\xi_1)}{6}, \xi_1 \in (x, x+h_r) \\ + f(x-h_l) = f(x) - h_l f'(x) + {h_l}^2 \frac{f''(x)}{2} - {h_l}^3 \frac{f'''(\xi_2)}{6}, \xi_2 \in (x, x-h_l) \\ + \end{aligned} + + Using the fact that :math:`f \in C^3` and solving the linear system, we derive: + + .. math:: + f'(x) \approx \frac{ {h_l}^2 f(x+h_r) - {h_r}^2 f(x-h_l) + + ({h_r}^2-{h_l}^2 ) f(x) }{ {h_r} {h_l}^2 + {h_r}^2 {h_l} } + + .. note:: + We estimate the gradient of functions in complex domain + :math:`g : \mathbb{C}^n \rightarrow \mathbb{C}` in the same way. + + The value of each partial derivative at the boundary points is computed differently. See edge_order below. + + Args: + input (``Tensor``): the tensor that represents the values of the function + + Keyword args: + spacing (``scalar``, ``list of scalar``, ``list of Tensor``, optional): :attr:`spacing` can be used to modify + how the :attr:`input` tensor's indices relate to sample coordinates. If :attr:`spacing` is a scalar then + the indices are multiplied by the scalar to produce the coordinates. For example, if :attr:`spacing=2` the + indices (1, 2, 3) become coordinates (2, 4, 6). If :attr:`spacing` is a list of scalars then the corresponding + indices are multiplied. For example, if :attr:`spacing=(2, -1, 3)` the indices (1, 2, 3) become coordinates (2, -2, 9). + Finally, if :attr:`spacing` is a list of one-dimensional tensors then each tensor specifies the coordinates for + the corresponding dimension. For example, if the indices are (1, 2, 3) and the tensors are (t0, t1, t2), then + the coordinates are (t0[1], t1[2], t2[3]) + + dim (``int``, ``list of int``, optional): the dimension or dimensions to approximate the gradient over. By default + the partial gradient in every dimension is computed. Note that when :attr:`dim` is specified the elements of + the :attr:`spacing` argument must correspond with the specified dims." + + edge_order (``int``, optional): 1 or 2, for `first-order + `_ or + `second-order `_ + estimation of the boundary ("edge") values, respectively. Note that when :attr:`edge_order` is specified, each + dimension size of :attr:`input` should be at least edge_order+1 + + Examples:: + + >>> # Estimates the gradient of f(x)=x^2 at points [-2, -1, 2, 4] + >>> coordinates = (torch.tensor([-2., -1., 1., 4.]),) + >>> values = torch.tensor([4., 1., 1., 16.], ) + >>> torch.gradient(values, spacing = coordinates) + (tensor([-3., -2., 2., 5.]),) + + >>> # Estimates the gradient of the R^2 -> R function whose samples are + >>> # described by the tensor t. Implicit coordinates are [0, 1] for the outermost + >>> # dimension and [0, 1, 2, 3] for the innermost dimension, and function estimates + >>> # partial derivative for both dimensions. + >>> t = torch.tensor([[1, 2, 4, 8], [10, 20, 40, 80]]) + >>> torch.gradient(t) + (tensor([[ 9., 18., 36., 72.], + [ 9., 18., 36., 72.]]), + tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]])) + + >>> # A scalar value for spacing modifies the relationship between tensor indices + >>> # and input coordinates by multiplying the indices to find the + >>> # coordinates. For example, below the indices of the innermost + >>> # 0, 1, 2, 3 translate to coordinates of [0, 2, 4, 6], and the indices of + >>> # the outermost dimension 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = 2.0) # dim = None (implicitly [0, 1]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.5000, 0.7500, 1.5000, 2.0000], + [ 5.0000, 7.5000, 15.0000, 20.0000]])) + >>> # doubling the spacing between samples halves the estimated partial gradients. + + >>> + >>> # Estimates only the partial derivative for dimension 1 + >>> torch.gradient(t, dim = 1) # spacing = None (implicitly 1.) + (tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]]),) + + >>> # When spacing is a list of scalars, the relationship between the tensor + >>> # indices and input coordinates changes based on dimension. + >>> # For example, below, the indices of the innermost dimension 0, 1, 2, 3 translate + >>> # to coordinates of [0, 3, 6, 9], and the indices of the outermost dimension + >>> # 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = [3., 2.]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + + >>> # The following example is a replication of the previous one with explicit + >>> # coordinates. + >>> coords = (torch.tensor([0, 2]), torch.tensor([0, 3, 6, 9])) + >>> torch.gradient(t, spacing = coords) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + """ + +@overload +def gradient( + input: Tensor, + *, + spacing: Number | _complex, + dim: _size, + edge_order: _int = 1, +) -> tuple[Tensor, ...]: + r""" + gradient(input, *, spacing=1, dim=None, edge_order=1) -> List of Tensors + + Estimates the gradient of a function :math:`g : \mathbb{R}^n \rightarrow \mathbb{R}` in + one or more dimensions using the `second-order accurate central differences method + `_ and + either first or second order estimates at the boundaries. + + The gradient of :math:`g` is estimated using samples. By default, when :attr:`spacing` is not + specified, the samples are entirely described by :attr:`input`, and the mapping of input coordinates + to an output is the same as the tensor's mapping of indices to values. For example, for a three-dimensional + :attr:`input` the function described is :math:`g : \mathbb{R}^3 \rightarrow \mathbb{R}`, and + :math:`g(1, 2, 3)\ == input[1, 2, 3]`. + + When :attr:`spacing` is specified, it modifies the relationship between :attr:`input` and input coordinates. + This is detailed in the "Keyword Arguments" section below. + + The gradient is estimated by estimating each partial derivative of :math:`g` independently. This estimation is + accurate if :math:`g` is in :math:`C^3` (it has at least 3 continuous derivatives), and the estimation can be + improved by providing closer samples. Mathematically, the value at each interior point of a partial derivative + is estimated using `Taylor's theorem with remainder `_. + Letting :math:`x` be an interior point with :math:`x-h_l` and :math:`x+h_r` be points neighboring + it to the left and right respectively, :math:`f(x+h_r)` and :math:`f(x-h_l)` can be estimated using: + + .. math:: + \begin{aligned} + f(x+h_r) = f(x) + h_r f'(x) + {h_r}^2 \frac{f''(x)}{2} + {h_r}^3 \frac{f'''(\xi_1)}{6}, \xi_1 \in (x, x+h_r) \\ + f(x-h_l) = f(x) - h_l f'(x) + {h_l}^2 \frac{f''(x)}{2} - {h_l}^3 \frac{f'''(\xi_2)}{6}, \xi_2 \in (x, x-h_l) \\ + \end{aligned} + + Using the fact that :math:`f \in C^3` and solving the linear system, we derive: + + .. math:: + f'(x) \approx \frac{ {h_l}^2 f(x+h_r) - {h_r}^2 f(x-h_l) + + ({h_r}^2-{h_l}^2 ) f(x) }{ {h_r} {h_l}^2 + {h_r}^2 {h_l} } + + .. note:: + We estimate the gradient of functions in complex domain + :math:`g : \mathbb{C}^n \rightarrow \mathbb{C}` in the same way. + + The value of each partial derivative at the boundary points is computed differently. See edge_order below. + + Args: + input (``Tensor``): the tensor that represents the values of the function + + Keyword args: + spacing (``scalar``, ``list of scalar``, ``list of Tensor``, optional): :attr:`spacing` can be used to modify + how the :attr:`input` tensor's indices relate to sample coordinates. If :attr:`spacing` is a scalar then + the indices are multiplied by the scalar to produce the coordinates. For example, if :attr:`spacing=2` the + indices (1, 2, 3) become coordinates (2, 4, 6). If :attr:`spacing` is a list of scalars then the corresponding + indices are multiplied. For example, if :attr:`spacing=(2, -1, 3)` the indices (1, 2, 3) become coordinates (2, -2, 9). + Finally, if :attr:`spacing` is a list of one-dimensional tensors then each tensor specifies the coordinates for + the corresponding dimension. For example, if the indices are (1, 2, 3) and the tensors are (t0, t1, t2), then + the coordinates are (t0[1], t1[2], t2[3]) + + dim (``int``, ``list of int``, optional): the dimension or dimensions to approximate the gradient over. By default + the partial gradient in every dimension is computed. Note that when :attr:`dim` is specified the elements of + the :attr:`spacing` argument must correspond with the specified dims." + + edge_order (``int``, optional): 1 or 2, for `first-order + `_ or + `second-order `_ + estimation of the boundary ("edge") values, respectively. Note that when :attr:`edge_order` is specified, each + dimension size of :attr:`input` should be at least edge_order+1 + + Examples:: + + >>> # Estimates the gradient of f(x)=x^2 at points [-2, -1, 2, 4] + >>> coordinates = (torch.tensor([-2., -1., 1., 4.]),) + >>> values = torch.tensor([4., 1., 1., 16.], ) + >>> torch.gradient(values, spacing = coordinates) + (tensor([-3., -2., 2., 5.]),) + + >>> # Estimates the gradient of the R^2 -> R function whose samples are + >>> # described by the tensor t. Implicit coordinates are [0, 1] for the outermost + >>> # dimension and [0, 1, 2, 3] for the innermost dimension, and function estimates + >>> # partial derivative for both dimensions. + >>> t = torch.tensor([[1, 2, 4, 8], [10, 20, 40, 80]]) + >>> torch.gradient(t) + (tensor([[ 9., 18., 36., 72.], + [ 9., 18., 36., 72.]]), + tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]])) + + >>> # A scalar value for spacing modifies the relationship between tensor indices + >>> # and input coordinates by multiplying the indices to find the + >>> # coordinates. For example, below the indices of the innermost + >>> # 0, 1, 2, 3 translate to coordinates of [0, 2, 4, 6], and the indices of + >>> # the outermost dimension 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = 2.0) # dim = None (implicitly [0, 1]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.5000, 0.7500, 1.5000, 2.0000], + [ 5.0000, 7.5000, 15.0000, 20.0000]])) + >>> # doubling the spacing between samples halves the estimated partial gradients. + + >>> + >>> # Estimates only the partial derivative for dimension 1 + >>> torch.gradient(t, dim = 1) # spacing = None (implicitly 1.) + (tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]]),) + + >>> # When spacing is a list of scalars, the relationship between the tensor + >>> # indices and input coordinates changes based on dimension. + >>> # For example, below, the indices of the innermost dimension 0, 1, 2, 3 translate + >>> # to coordinates of [0, 3, 6, 9], and the indices of the outermost dimension + >>> # 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = [3., 2.]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + + >>> # The following example is a replication of the previous one with explicit + >>> # coordinates. + >>> coords = (torch.tensor([0, 2]), torch.tensor([0, 3, 6, 9])) + >>> torch.gradient(t, spacing = coords) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + """ + +@overload +def gradient( + input: Tensor, + *, + spacing: tuple[Tensor, ...] | list[Tensor] | None, + dim: _size, + edge_order: _int = 1, +) -> tuple[Tensor, ...]: + r""" + gradient(input, *, spacing=1, dim=None, edge_order=1) -> List of Tensors + + Estimates the gradient of a function :math:`g : \mathbb{R}^n \rightarrow \mathbb{R}` in + one or more dimensions using the `second-order accurate central differences method + `_ and + either first or second order estimates at the boundaries. + + The gradient of :math:`g` is estimated using samples. By default, when :attr:`spacing` is not + specified, the samples are entirely described by :attr:`input`, and the mapping of input coordinates + to an output is the same as the tensor's mapping of indices to values. For example, for a three-dimensional + :attr:`input` the function described is :math:`g : \mathbb{R}^3 \rightarrow \mathbb{R}`, and + :math:`g(1, 2, 3)\ == input[1, 2, 3]`. + + When :attr:`spacing` is specified, it modifies the relationship between :attr:`input` and input coordinates. + This is detailed in the "Keyword Arguments" section below. + + The gradient is estimated by estimating each partial derivative of :math:`g` independently. This estimation is + accurate if :math:`g` is in :math:`C^3` (it has at least 3 continuous derivatives), and the estimation can be + improved by providing closer samples. Mathematically, the value at each interior point of a partial derivative + is estimated using `Taylor's theorem with remainder `_. + Letting :math:`x` be an interior point with :math:`x-h_l` and :math:`x+h_r` be points neighboring + it to the left and right respectively, :math:`f(x+h_r)` and :math:`f(x-h_l)` can be estimated using: + + .. math:: + \begin{aligned} + f(x+h_r) = f(x) + h_r f'(x) + {h_r}^2 \frac{f''(x)}{2} + {h_r}^3 \frac{f'''(\xi_1)}{6}, \xi_1 \in (x, x+h_r) \\ + f(x-h_l) = f(x) - h_l f'(x) + {h_l}^2 \frac{f''(x)}{2} - {h_l}^3 \frac{f'''(\xi_2)}{6}, \xi_2 \in (x, x-h_l) \\ + \end{aligned} + + Using the fact that :math:`f \in C^3` and solving the linear system, we derive: + + .. math:: + f'(x) \approx \frac{ {h_l}^2 f(x+h_r) - {h_r}^2 f(x-h_l) + + ({h_r}^2-{h_l}^2 ) f(x) }{ {h_r} {h_l}^2 + {h_r}^2 {h_l} } + + .. note:: + We estimate the gradient of functions in complex domain + :math:`g : \mathbb{C}^n \rightarrow \mathbb{C}` in the same way. + + The value of each partial derivative at the boundary points is computed differently. See edge_order below. + + Args: + input (``Tensor``): the tensor that represents the values of the function + + Keyword args: + spacing (``scalar``, ``list of scalar``, ``list of Tensor``, optional): :attr:`spacing` can be used to modify + how the :attr:`input` tensor's indices relate to sample coordinates. If :attr:`spacing` is a scalar then + the indices are multiplied by the scalar to produce the coordinates. For example, if :attr:`spacing=2` the + indices (1, 2, 3) become coordinates (2, 4, 6). If :attr:`spacing` is a list of scalars then the corresponding + indices are multiplied. For example, if :attr:`spacing=(2, -1, 3)` the indices (1, 2, 3) become coordinates (2, -2, 9). + Finally, if :attr:`spacing` is a list of one-dimensional tensors then each tensor specifies the coordinates for + the corresponding dimension. For example, if the indices are (1, 2, 3) and the tensors are (t0, t1, t2), then + the coordinates are (t0[1], t1[2], t2[3]) + + dim (``int``, ``list of int``, optional): the dimension or dimensions to approximate the gradient over. By default + the partial gradient in every dimension is computed. Note that when :attr:`dim` is specified the elements of + the :attr:`spacing` argument must correspond with the specified dims." + + edge_order (``int``, optional): 1 or 2, for `first-order + `_ or + `second-order `_ + estimation of the boundary ("edge") values, respectively. Note that when :attr:`edge_order` is specified, each + dimension size of :attr:`input` should be at least edge_order+1 + + Examples:: + + >>> # Estimates the gradient of f(x)=x^2 at points [-2, -1, 2, 4] + >>> coordinates = (torch.tensor([-2., -1., 1., 4.]),) + >>> values = torch.tensor([4., 1., 1., 16.], ) + >>> torch.gradient(values, spacing = coordinates) + (tensor([-3., -2., 2., 5.]),) + + >>> # Estimates the gradient of the R^2 -> R function whose samples are + >>> # described by the tensor t. Implicit coordinates are [0, 1] for the outermost + >>> # dimension and [0, 1, 2, 3] for the innermost dimension, and function estimates + >>> # partial derivative for both dimensions. + >>> t = torch.tensor([[1, 2, 4, 8], [10, 20, 40, 80]]) + >>> torch.gradient(t) + (tensor([[ 9., 18., 36., 72.], + [ 9., 18., 36., 72.]]), + tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]])) + + >>> # A scalar value for spacing modifies the relationship between tensor indices + >>> # and input coordinates by multiplying the indices to find the + >>> # coordinates. For example, below the indices of the innermost + >>> # 0, 1, 2, 3 translate to coordinates of [0, 2, 4, 6], and the indices of + >>> # the outermost dimension 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = 2.0) # dim = None (implicitly [0, 1]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.5000, 0.7500, 1.5000, 2.0000], + [ 5.0000, 7.5000, 15.0000, 20.0000]])) + >>> # doubling the spacing between samples halves the estimated partial gradients. + + >>> + >>> # Estimates only the partial derivative for dimension 1 + >>> torch.gradient(t, dim = 1) # spacing = None (implicitly 1.) + (tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]]),) + + >>> # When spacing is a list of scalars, the relationship between the tensor + >>> # indices and input coordinates changes based on dimension. + >>> # For example, below, the indices of the innermost dimension 0, 1, 2, 3 translate + >>> # to coordinates of [0, 3, 6, 9], and the indices of the outermost dimension + >>> # 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = [3., 2.]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + + >>> # The following example is a replication of the previous one with explicit + >>> # coordinates. + >>> coords = (torch.tensor([0, 2]), torch.tensor([0, 3, 6, 9])) + >>> torch.gradient(t, spacing = coords) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + """ + +@overload +def gradient( + input: Tensor, + *, + dim: _size, + edge_order: _int = 1, +) -> tuple[Tensor, ...]: + r""" + gradient(input, *, spacing=1, dim=None, edge_order=1) -> List of Tensors + + Estimates the gradient of a function :math:`g : \mathbb{R}^n \rightarrow \mathbb{R}` in + one or more dimensions using the `second-order accurate central differences method + `_ and + either first or second order estimates at the boundaries. + + The gradient of :math:`g` is estimated using samples. By default, when :attr:`spacing` is not + specified, the samples are entirely described by :attr:`input`, and the mapping of input coordinates + to an output is the same as the tensor's mapping of indices to values. For example, for a three-dimensional + :attr:`input` the function described is :math:`g : \mathbb{R}^3 \rightarrow \mathbb{R}`, and + :math:`g(1, 2, 3)\ == input[1, 2, 3]`. + + When :attr:`spacing` is specified, it modifies the relationship between :attr:`input` and input coordinates. + This is detailed in the "Keyword Arguments" section below. + + The gradient is estimated by estimating each partial derivative of :math:`g` independently. This estimation is + accurate if :math:`g` is in :math:`C^3` (it has at least 3 continuous derivatives), and the estimation can be + improved by providing closer samples. Mathematically, the value at each interior point of a partial derivative + is estimated using `Taylor's theorem with remainder `_. + Letting :math:`x` be an interior point with :math:`x-h_l` and :math:`x+h_r` be points neighboring + it to the left and right respectively, :math:`f(x+h_r)` and :math:`f(x-h_l)` can be estimated using: + + .. math:: + \begin{aligned} + f(x+h_r) = f(x) + h_r f'(x) + {h_r}^2 \frac{f''(x)}{2} + {h_r}^3 \frac{f'''(\xi_1)}{6}, \xi_1 \in (x, x+h_r) \\ + f(x-h_l) = f(x) - h_l f'(x) + {h_l}^2 \frac{f''(x)}{2} - {h_l}^3 \frac{f'''(\xi_2)}{6}, \xi_2 \in (x, x-h_l) \\ + \end{aligned} + + Using the fact that :math:`f \in C^3` and solving the linear system, we derive: + + .. math:: + f'(x) \approx \frac{ {h_l}^2 f(x+h_r) - {h_r}^2 f(x-h_l) + + ({h_r}^2-{h_l}^2 ) f(x) }{ {h_r} {h_l}^2 + {h_r}^2 {h_l} } + + .. note:: + We estimate the gradient of functions in complex domain + :math:`g : \mathbb{C}^n \rightarrow \mathbb{C}` in the same way. + + The value of each partial derivative at the boundary points is computed differently. See edge_order below. + + Args: + input (``Tensor``): the tensor that represents the values of the function + + Keyword args: + spacing (``scalar``, ``list of scalar``, ``list of Tensor``, optional): :attr:`spacing` can be used to modify + how the :attr:`input` tensor's indices relate to sample coordinates. If :attr:`spacing` is a scalar then + the indices are multiplied by the scalar to produce the coordinates. For example, if :attr:`spacing=2` the + indices (1, 2, 3) become coordinates (2, 4, 6). If :attr:`spacing` is a list of scalars then the corresponding + indices are multiplied. For example, if :attr:`spacing=(2, -1, 3)` the indices (1, 2, 3) become coordinates (2, -2, 9). + Finally, if :attr:`spacing` is a list of one-dimensional tensors then each tensor specifies the coordinates for + the corresponding dimension. For example, if the indices are (1, 2, 3) and the tensors are (t0, t1, t2), then + the coordinates are (t0[1], t1[2], t2[3]) + + dim (``int``, ``list of int``, optional): the dimension or dimensions to approximate the gradient over. By default + the partial gradient in every dimension is computed. Note that when :attr:`dim` is specified the elements of + the :attr:`spacing` argument must correspond with the specified dims." + + edge_order (``int``, optional): 1 or 2, for `first-order + `_ or + `second-order `_ + estimation of the boundary ("edge") values, respectively. Note that when :attr:`edge_order` is specified, each + dimension size of :attr:`input` should be at least edge_order+1 + + Examples:: + + >>> # Estimates the gradient of f(x)=x^2 at points [-2, -1, 2, 4] + >>> coordinates = (torch.tensor([-2., -1., 1., 4.]),) + >>> values = torch.tensor([4., 1., 1., 16.], ) + >>> torch.gradient(values, spacing = coordinates) + (tensor([-3., -2., 2., 5.]),) + + >>> # Estimates the gradient of the R^2 -> R function whose samples are + >>> # described by the tensor t. Implicit coordinates are [0, 1] for the outermost + >>> # dimension and [0, 1, 2, 3] for the innermost dimension, and function estimates + >>> # partial derivative for both dimensions. + >>> t = torch.tensor([[1, 2, 4, 8], [10, 20, 40, 80]]) + >>> torch.gradient(t) + (tensor([[ 9., 18., 36., 72.], + [ 9., 18., 36., 72.]]), + tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]])) + + >>> # A scalar value for spacing modifies the relationship between tensor indices + >>> # and input coordinates by multiplying the indices to find the + >>> # coordinates. For example, below the indices of the innermost + >>> # 0, 1, 2, 3 translate to coordinates of [0, 2, 4, 6], and the indices of + >>> # the outermost dimension 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = 2.0) # dim = None (implicitly [0, 1]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.5000, 0.7500, 1.5000, 2.0000], + [ 5.0000, 7.5000, 15.0000, 20.0000]])) + >>> # doubling the spacing between samples halves the estimated partial gradients. + + >>> + >>> # Estimates only the partial derivative for dimension 1 + >>> torch.gradient(t, dim = 1) # spacing = None (implicitly 1.) + (tensor([[ 1.0000, 1.5000, 3.0000, 4.0000], + [10.0000, 15.0000, 30.0000, 40.0000]]),) + + >>> # When spacing is a list of scalars, the relationship between the tensor + >>> # indices and input coordinates changes based on dimension. + >>> # For example, below, the indices of the innermost dimension 0, 1, 2, 3 translate + >>> # to coordinates of [0, 3, 6, 9], and the indices of the outermost dimension + >>> # 0, 1 translate to coordinates of [0, 2]. + >>> torch.gradient(t, spacing = [3., 2.]) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + + >>> # The following example is a replication of the previous one with explicit + >>> # coordinates. + >>> coords = (torch.tensor([0, 2]), torch.tensor([0, 3, 6, 9])) + >>> torch.gradient(t, spacing = coords) + (tensor([[ 4.5000, 9.0000, 18.0000, 36.0000], + [ 4.5000, 9.0000, 18.0000, 36.0000]]), + tensor([[ 0.3333, 0.5000, 1.0000, 1.3333], + [ 3.3333, 5.0000, 10.0000, 13.3333]])) + """ + +@overload +def greater( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + greater(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.gt`. + """ + +@overload +def greater( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + greater(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.gt`. + """ + +@overload +def greater_equal( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + greater_equal(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.ge`. + """ + +@overload +def greater_equal( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + greater_equal(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.ge`. + """ + +def grid_sampler( + input: Tensor, + grid: Tensor, + interpolation_mode: _int, + padding_mode: _int, + align_corners: _bool, +) -> Tensor: ... +def grid_sampler_2d( + input: Tensor, + grid: Tensor, + interpolation_mode: _int, + padding_mode: _int, + align_corners: _bool, +) -> Tensor: ... +def grid_sampler_3d( + input: Tensor, + grid: Tensor, + interpolation_mode: _int, + padding_mode: _int, + align_corners: _bool, +) -> Tensor: ... +def group_norm( + input: Tensor, + num_groups: _int, + weight: Tensor | None = None, + bias: Tensor | None = None, + eps: _float = 1e-05, + cudnn_enabled: _bool = True, +) -> Tensor: ... +@overload +def gru( + data: Tensor, + batch_sizes: Tensor, + hx: Tensor, + params: tuple[Tensor, ...] | list[Tensor] | None, + has_biases: _bool, + num_layers: _int, + dropout: _float, + train: _bool, + bidirectional: _bool, +) -> tuple[Tensor, Tensor]: ... +@overload +def gru( + input: Tensor, + hx: Tensor, + params: tuple[Tensor, ...] | list[Tensor] | None, + has_biases: _bool, + num_layers: _int, + dropout: _float, + train: _bool, + bidirectional: _bool, + batch_first: _bool, +) -> tuple[Tensor, Tensor]: ... +def gru_cell( + input: Tensor, + hx: Tensor, + w_ih: Tensor, + w_hh: Tensor, + b_ih: Tensor | None = None, + b_hh: Tensor | None = None, +) -> Tensor: ... +@overload +def gt( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + gt(input, other, *, out=None) -> Tensor + + Computes :math:`\text{input} > \text{other}` element-wise. + + + The second argument can be a number or a tensor whose shape is + :ref:`broadcastable ` with the first argument. + + Args: + input (Tensor): the tensor to compare + other (Tensor or float): the tensor or value to compare + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is greater than :attr:`other` and False elsewhere + + Example:: + + >>> torch.gt(torch.tensor([[1, 2], [3, 4]]), torch.tensor([[1, 1], [4, 4]])) + tensor([[False, True], [False, False]]) + """ + +@overload +def gt( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + gt(input, other, *, out=None) -> Tensor + + Computes :math:`\text{input} > \text{other}` element-wise. + + + The second argument can be a number or a tensor whose shape is + :ref:`broadcastable ` with the first argument. + + Args: + input (Tensor): the tensor to compare + other (Tensor or float): the tensor or value to compare + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is greater than :attr:`other` and False elsewhere + + Example:: + + >>> torch.gt(torch.tensor([[1, 2], [3, 4]]), torch.tensor([[1, 1], [4, 4]])) + tensor([[False, True], [False, False]]) + """ + +@overload +def hamming_window( + window_length: _int, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + hamming_window(window_length, *, dtype=None, layout=None, device=None, pin_memory=False, requires_grad=False) -> Tensor + + Hamming window function. + + .. math:: + w[n] = \alpha - \beta\ \cos \left( \frac{2 \pi n}{N - 1} \right), + + where :math:`N` is the full window size. + + The input :attr:`window_length` is a positive integer controlling the + returned window size. :attr:`periodic` flag determines whether the returned + window trims off the last duplicate value from the symmetric window and is + ready to be used as a periodic window with functions like + :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in + above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have + ``torch.hamming_window(L, periodic=True)`` equal to + ``torch.hamming_window(L + 1, periodic=False)[:-1])``. + + .. note:: + If :attr:`window_length` :math:`=1`, the returned window contains a single value 1. + + .. note:: + This is a generalized version of :meth:`torch.hann_window`. + + Arguments: + window_length (int): the size of returned window + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + + .. function:: hamming_window(window_length, periodic, *, dtype=None, layout=None, device=None, \ + pin_memory=False, requires_grad=False) -> Tensor + :noindex: + + Hamming window function with periodic specified. + + Arguments: + window_length (int): the size of returned window + periodic (bool): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + + .. function:: hamming_window(window_length, periodic, float alpha, *, dtype=None, layout=None, device=None, \ + pin_memory=False, requires_grad=False) -> Tensor + :noindex: + + Hamming window function with periodic and alpha specified. + + Arguments: + window_length (int): the size of returned window + periodic (bool): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + alpha (float): The coefficient :math:`\alpha` in the equation above + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + + .. function:: hamming_window(window_length, periodic, float alpha, float beta, *, dtype=None, layout=None, \ + device=None, pin_memory=False, requires_grad=False) -> Tensor + :noindex: + + Hamming window function with periodic, alpha and beta specified. + + Arguments: + window_length (int): the size of returned window + periodic (bool): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + alpha (float): The coefficient :math:`\alpha` in the equation above + beta (float): The coefficient :math:`\beta` in the equation above + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + """ + +@overload +def hamming_window( + window_length: _int, + periodic: _bool, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + hamming_window(window_length, *, dtype=None, layout=None, device=None, pin_memory=False, requires_grad=False) -> Tensor + + Hamming window function. + + .. math:: + w[n] = \alpha - \beta\ \cos \left( \frac{2 \pi n}{N - 1} \right), + + where :math:`N` is the full window size. + + The input :attr:`window_length` is a positive integer controlling the + returned window size. :attr:`periodic` flag determines whether the returned + window trims off the last duplicate value from the symmetric window and is + ready to be used as a periodic window with functions like + :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in + above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have + ``torch.hamming_window(L, periodic=True)`` equal to + ``torch.hamming_window(L + 1, periodic=False)[:-1])``. + + .. note:: + If :attr:`window_length` :math:`=1`, the returned window contains a single value 1. + + .. note:: + This is a generalized version of :meth:`torch.hann_window`. + + Arguments: + window_length (int): the size of returned window + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + + .. function:: hamming_window(window_length, periodic, *, dtype=None, layout=None, device=None, \ + pin_memory=False, requires_grad=False) -> Tensor + :noindex: + + Hamming window function with periodic specified. + + Arguments: + window_length (int): the size of returned window + periodic (bool): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + + .. function:: hamming_window(window_length, periodic, float alpha, *, dtype=None, layout=None, device=None, \ + pin_memory=False, requires_grad=False) -> Tensor + :noindex: + + Hamming window function with periodic and alpha specified. + + Arguments: + window_length (int): the size of returned window + periodic (bool): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + alpha (float): The coefficient :math:`\alpha` in the equation above + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + + .. function:: hamming_window(window_length, periodic, float alpha, float beta, *, dtype=None, layout=None, \ + device=None, pin_memory=False, requires_grad=False) -> Tensor + :noindex: + + Hamming window function with periodic, alpha and beta specified. + + Arguments: + window_length (int): the size of returned window + periodic (bool): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + alpha (float): The coefficient :math:`\alpha` in the equation above + beta (float): The coefficient :math:`\beta` in the equation above + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + """ + +@overload +def hamming_window( + window_length: _int, + periodic: _bool, + alpha: _float, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + hamming_window(window_length, *, dtype=None, layout=None, device=None, pin_memory=False, requires_grad=False) -> Tensor + + Hamming window function. + + .. math:: + w[n] = \alpha - \beta\ \cos \left( \frac{2 \pi n}{N - 1} \right), + + where :math:`N` is the full window size. + + The input :attr:`window_length` is a positive integer controlling the + returned window size. :attr:`periodic` flag determines whether the returned + window trims off the last duplicate value from the symmetric window and is + ready to be used as a periodic window with functions like + :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in + above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have + ``torch.hamming_window(L, periodic=True)`` equal to + ``torch.hamming_window(L + 1, periodic=False)[:-1])``. + + .. note:: + If :attr:`window_length` :math:`=1`, the returned window contains a single value 1. + + .. note:: + This is a generalized version of :meth:`torch.hann_window`. + + Arguments: + window_length (int): the size of returned window + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + + .. function:: hamming_window(window_length, periodic, *, dtype=None, layout=None, device=None, \ + pin_memory=False, requires_grad=False) -> Tensor + :noindex: + + Hamming window function with periodic specified. + + Arguments: + window_length (int): the size of returned window + periodic (bool): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + + .. function:: hamming_window(window_length, periodic, float alpha, *, dtype=None, layout=None, device=None, \ + pin_memory=False, requires_grad=False) -> Tensor + :noindex: + + Hamming window function with periodic and alpha specified. + + Arguments: + window_length (int): the size of returned window + periodic (bool): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + alpha (float): The coefficient :math:`\alpha` in the equation above + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + + .. function:: hamming_window(window_length, periodic, float alpha, float beta, *, dtype=None, layout=None, \ + device=None, pin_memory=False, requires_grad=False) -> Tensor + :noindex: + + Hamming window function with periodic, alpha and beta specified. + + Arguments: + window_length (int): the size of returned window + periodic (bool): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + alpha (float): The coefficient :math:`\alpha` in the equation above + beta (float): The coefficient :math:`\beta` in the equation above + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + """ + +@overload +def hamming_window( + window_length: _int, + periodic: _bool, + alpha: _float, + beta: _float, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + hamming_window(window_length, *, dtype=None, layout=None, device=None, pin_memory=False, requires_grad=False) -> Tensor + + Hamming window function. + + .. math:: + w[n] = \alpha - \beta\ \cos \left( \frac{2 \pi n}{N - 1} \right), + + where :math:`N` is the full window size. + + The input :attr:`window_length` is a positive integer controlling the + returned window size. :attr:`periodic` flag determines whether the returned + window trims off the last duplicate value from the symmetric window and is + ready to be used as a periodic window with functions like + :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in + above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have + ``torch.hamming_window(L, periodic=True)`` equal to + ``torch.hamming_window(L + 1, periodic=False)[:-1])``. + + .. note:: + If :attr:`window_length` :math:`=1`, the returned window contains a single value 1. + + .. note:: + This is a generalized version of :meth:`torch.hann_window`. + + Arguments: + window_length (int): the size of returned window + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + + .. function:: hamming_window(window_length, periodic, *, dtype=None, layout=None, device=None, \ + pin_memory=False, requires_grad=False) -> Tensor + :noindex: + + Hamming window function with periodic specified. + + Arguments: + window_length (int): the size of returned window + periodic (bool): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + + .. function:: hamming_window(window_length, periodic, float alpha, *, dtype=None, layout=None, device=None, \ + pin_memory=False, requires_grad=False) -> Tensor + :noindex: + + Hamming window function with periodic and alpha specified. + + Arguments: + window_length (int): the size of returned window + periodic (bool): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + alpha (float): The coefficient :math:`\alpha` in the equation above + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + + .. function:: hamming_window(window_length, periodic, float alpha, float beta, *, dtype=None, layout=None, \ + device=None, pin_memory=False, requires_grad=False) -> Tensor + :noindex: + + Hamming window function with periodic, alpha and beta specified. + + Arguments: + window_length (int): the size of returned window + periodic (bool): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + alpha (float): The coefficient :math:`\alpha` in the equation above + beta (float): The coefficient :math:`\beta` in the equation above + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window. + """ + +@overload +def hann_window( + window_length: _int, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + hann_window(window_length, periodic=True, *, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Hann window function. + + .. math:: + w[n] = \frac{1}{2}\ \left[1 - \cos \left( \frac{2 \pi n}{N - 1} \right)\right] = + \sin^2 \left( \frac{\pi n}{N - 1} \right), + + where :math:`N` is the full window size. + + The input :attr:`window_length` is a positive integer controlling the + returned window size. :attr:`periodic` flag determines whether the returned + window trims off the last duplicate value from the symmetric window and is + ready to be used as a periodic window with functions like + :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in + above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have + ``torch.hann_window(L, periodic=True)`` equal to + ``torch.hann_window(L + 1, periodic=False)[:-1])``. + + .. note:: + If :attr:`window_length` :math:`=1`, the returned window contains a single value 1. + + Arguments: + window_length (int): the size of returned window + periodic (bool, optional): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window + """ + +@overload +def hann_window( + window_length: _int, + periodic: _bool, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + hann_window(window_length, periodic=True, *, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Hann window function. + + .. math:: + w[n] = \frac{1}{2}\ \left[1 - \cos \left( \frac{2 \pi n}{N - 1} \right)\right] = + \sin^2 \left( \frac{\pi n}{N - 1} \right), + + where :math:`N` is the full window size. + + The input :attr:`window_length` is a positive integer controlling the + returned window size. :attr:`periodic` flag determines whether the returned + window trims off the last duplicate value from the symmetric window and is + ready to be used as a periodic window with functions like + :meth:`torch.stft`. Therefore, if :attr:`periodic` is true, the :math:`N` in + above formula is in fact :math:`\text{window\_length} + 1`. Also, we always have + ``torch.hann_window(L, periodic=True)`` equal to + ``torch.hann_window(L + 1, periodic=False)[:-1])``. + + .. note:: + If :attr:`window_length` :math:`=1`, the returned window contains a single value 1. + + Arguments: + window_length (int): the size of returned window + periodic (bool, optional): If True, returns a window to be used as periodic + function. If False, return a symmetric window. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). Only floating point types are supported. + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Returns: + Tensor: A 1-D tensor of size :math:`(\text{window\_length},)` containing the window + """ + +def hardshrink( + input: Tensor, + lambd: Number | _complex = 0.5, + *, + out: Tensor | None = None, +) -> Tensor: ... +def hash_tensor( + input: Tensor, + dim: _int | _size = (), + *, + keepdim: _bool = False, + mode: _int = 0, + out: Tensor | None = None, +) -> Tensor: + r""" + hash_tensor(input, *, mode=0) -> Tensor + + Returns a hash of all elements in the :attr:`input` tensor. + + Currently only mode=0 (reduction via xor) is supported. The output will always + be of type ``torch.uint64``. The elements of ``input`` are upcasted to their + 64 bit float / integer equivalent and bitcasted to ``torch.uint64`` before + reduction via xor. + + Args: + input (Tensor): the input tensor. + + Keyword Args: + mode (int) : The hash to use. Default: 0 (xor_reduction) + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 1.1918, -1.1813, 0.3373]]) + >>> torch.hash_tensor(a) + tensor(13822780554648485888, dtype=torch.uint64) + + .. function:: hash_tensor(input, dim, *, keepdim=False, mode=0) -> Tensor + :noindex: + + Returns the hash of each row of the :attr:`input` tensor in the given + dimension :attr:`dim` given by mode. If :attr:`dim` is a list of dimensions, + reduce over all of them. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword Args: + mode (int) : The hash to use. Default: 0 (xor_reduction) + + Example:: + + >>> a = torch.randn(2, 4) + >>> a + tensor([[ 0.1317, -0.5554, -1.4724, -1.1391], + [ 0.0778, -0.6070, 0.6375, 0.1798]]) + >>> torch.hash_tensor(a, 1) + tensor([9233691267014066176, 9255993250844508160], dtype=torch.uint64) + """ + +def heaviside( + input: Tensor, + values: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + heaviside(input, values, *, out=None) -> Tensor + + Computes the Heaviside step function for each element in :attr:`input`. + The Heaviside step function is defined as: + + .. math:: + \text{{heaviside}}(input, values) = \begin{cases} + 0, & \text{if input < 0}\\ + values, & \text{if input == 0}\\ + 1, & \text{if input > 0} + \end{cases} + + + Args: + input (Tensor): the input tensor. + values (Tensor): The values to use where :attr:`input` is zero. + + Keyword arguments: + out (Tensor, optional): the output tensor. + + Example:: + + >>> input = torch.tensor([-1.5, 0, 2.0]) + >>> values = torch.tensor([0.5]) + >>> torch.heaviside(input, values) + tensor([0.0000, 0.5000, 1.0000]) + >>> values = torch.tensor([1.2, -2.0, 3.5]) + >>> torch.heaviside(input, values) + tensor([0., -2., 1.]) + """ + +def hinge_embedding_loss( + input: Tensor, + target: Tensor, + margin: _float = 1.0, + reduction: _int = 1, +) -> Tensor: ... +def histc( + input: Tensor, + bins: _int = 100, + min: Number | _complex = 0, + max: Number | _complex = 0, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + histc(input, bins=100, min=0, max=0, *, out=None) -> Tensor + + Computes the histogram of a tensor. + + The elements are sorted into equal width bins between :attr:`min` and + :attr:`max`. If :attr:`min` and :attr:`max` are both zero, the minimum and + maximum values of the data are used. + + Elements lower than min and higher than max and ``NaN`` elements are ignored. + + Args: + input (Tensor): the input tensor. + bins (int): number of histogram bins + min (Scalar): lower end of the range (inclusive) + max (Scalar): upper end of the range (inclusive) + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + Tensor: Histogram represented as a tensor + + Example:: + + >>> torch.histc(torch.tensor([1., 2, 1]), bins=4, min=0, max=3) + tensor([ 0., 2., 1., 0.]) + """ + +@overload +def histogram( + input: Tensor, + bins: Tensor, + *, + weight: Tensor | None = None, + density: _bool = False, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.histogram: + r""" + histogram(input, bins, *, range=None, weight=None, density=False, out=None) -> (Tensor, Tensor) + + Computes a histogram of the values in a tensor. + + :attr:`bins` can be an integer or a 1D tensor. + + If :attr:`bins` is an int, it specifies the number of equal-width bins. + By default, the lower and upper range of the bins is determined by the + minimum and maximum elements of the input tensor. The :attr:`range` + argument can be provided to specify a range for the bins. + + If :attr:`bins` is a 1D tensor, it specifies the sequence of bin edges + including the rightmost edge. It should contain at least 2 elements + and its elements should be increasing. + + Args: + input (Tensor): the input tensor. + bins: int or 1D Tensor. If int, defines the number of equal-width bins. If tensor, + defines the sequence of bin edges including the rightmost edge. + + Keyword args: + range (tuple of float): Defines the range of the bins. + weight (Tensor): If provided, weight should have the same shape as input. Each value in + input contributes its associated weight towards its bin's result. + density (bool): If False, the result will contain the count (or total weight) in each bin. + If True, the result is the value of the probability density function over the bins, + normalized such that the integral over the range of the bins is 1. + out (Tensor, optional): the output tensor. (tuple, optional): The result tuple of two output tensors (hist, bin_edges). + + Returns: + hist (Tensor): 1D Tensor containing the values of the histogram. + bin_edges(Tensor): 1D Tensor containing the edges of the histogram bins. + + Example:: + + >>> torch.histogram(torch.tensor([1., 2, 1]), bins=4, range=(0., 3.), weight=torch.tensor([1., 2., 4.])) + (tensor([ 0., 5., 2., 0.]), tensor([0., 0.75, 1.5, 2.25, 3.])) + >>> torch.histogram(torch.tensor([1., 2, 1]), bins=4, range=(0., 3.), weight=torch.tensor([1., 2., 4.]), density=True) + (tensor([ 0., 0.9524, 0.3810, 0.]), tensor([0., 0.75, 1.5, 2.25, 3.])) + """ + +@overload +def histogram( + input: Tensor, + bins: _int = 100, + *, + range: Sequence[_float] | None = None, + weight: Tensor | None = None, + density: _bool = False, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.histogram: + r""" + histogram(input, bins, *, range=None, weight=None, density=False, out=None) -> (Tensor, Tensor) + + Computes a histogram of the values in a tensor. + + :attr:`bins` can be an integer or a 1D tensor. + + If :attr:`bins` is an int, it specifies the number of equal-width bins. + By default, the lower and upper range of the bins is determined by the + minimum and maximum elements of the input tensor. The :attr:`range` + argument can be provided to specify a range for the bins. + + If :attr:`bins` is a 1D tensor, it specifies the sequence of bin edges + including the rightmost edge. It should contain at least 2 elements + and its elements should be increasing. + + Args: + input (Tensor): the input tensor. + bins: int or 1D Tensor. If int, defines the number of equal-width bins. If tensor, + defines the sequence of bin edges including the rightmost edge. + + Keyword args: + range (tuple of float): Defines the range of the bins. + weight (Tensor): If provided, weight should have the same shape as input. Each value in + input contributes its associated weight towards its bin's result. + density (bool): If False, the result will contain the count (or total weight) in each bin. + If True, the result is the value of the probability density function over the bins, + normalized such that the integral over the range of the bins is 1. + out (Tensor, optional): the output tensor. (tuple, optional): The result tuple of two output tensors (hist, bin_edges). + + Returns: + hist (Tensor): 1D Tensor containing the values of the histogram. + bin_edges(Tensor): 1D Tensor containing the edges of the histogram bins. + + Example:: + + >>> torch.histogram(torch.tensor([1., 2, 1]), bins=4, range=(0., 3.), weight=torch.tensor([1., 2., 4.])) + (tensor([ 0., 5., 2., 0.]), tensor([0., 0.75, 1.5, 2.25, 3.])) + >>> torch.histogram(torch.tensor([1., 2, 1]), bins=4, range=(0., 3.), weight=torch.tensor([1., 2., 4.]), density=True) + (tensor([ 0., 0.9524, 0.3810, 0.]), tensor([0., 0.75, 1.5, 2.25, 3.])) + """ + +@overload +def histogramdd( + input: Tensor, + bins: _int, + range: Sequence[_float] | None = None, + weight: Tensor | None = None, + density: _bool = False, +) -> torch.return_types.histogramdd: + r""" + histogramdd(input, bins, *, range=None, weight=None, density=False, out=None) -> (Tensor, Tensor[]) + + Computes a multi-dimensional histogram of the values in a tensor. + + Interprets the elements of an input tensor whose innermost dimension has size N + as a collection of N-dimensional points. Maps each of the points into a set of + N-dimensional bins and returns the number of points (or total weight) in each bin. + + :attr:`input` must be a tensor with at least 2 dimensions. + If input has shape (M, N), each of its M rows defines a point in N-dimensional space. + If input has three or more dimensions, all but the last dimension are flattened. + + Each dimension is independently associated with its own strictly increasing sequence + of bin edges. Bin edges may be specified explicitly by passing a sequence of 1D + tensors. Alternatively, bin edges may be constructed automatically by passing a + sequence of integers specifying the number of equal-width bins in each dimension. + + For each N-dimensional point in input: + - Each of its coordinates is binned independently among the bin edges + corresponding to its dimension + - Binning results are combined to identify the N-dimensional bin (if any) + into which the point falls + - If the point falls into a bin, the bin's count (or total weight) is incremented + - Points which do not fall into any bin do not contribute to the output + + :attr:`bins` can be a sequence of N 1D tensors, a sequence of N ints, or a single int. + + If :attr:`bins` is a sequence of N 1D tensors, it explicitly specifies the N sequences + of bin edges. Each 1D tensor should contain a strictly increasing sequence with at + least one element. A sequence of K bin edges defines K-1 bins, explicitly specifying + the left and right edges of all bins. Every bin is inclusive of its left edge. Only + the rightmost bin is inclusive of its right edge. + + If :attr:`bins` is a sequence of N ints, it specifies the number of equal-width bins + in each dimension. By default, the leftmost and rightmost bin edges in each dimension + are determined by the minimum and maximum elements of the input tensor in the + corresponding dimension. The :attr:`range` argument can be provided to manually + specify the leftmost and rightmost bin edges in each dimension. + + If :attr:`bins` is an int, it specifies the number of equal-width bins for all dimensions. + + .. note:: + See also :func:`torch.histogram`, which specifically computes 1D histograms. + While :func:`torch.histogramdd` infers the dimensionality of its bins and + binned values from the shape of :attr:`input`, :func:`torch.histogram` + accepts and flattens :attr:`input` of any shape. + + Args: + input (Tensor): the input tensor. + bins: Tensor[], int[], or int. + If Tensor[], defines the sequences of bin edges. + If int[], defines the number of equal-width bins in each dimension. + If int, defines the number of equal-width bins for all dimensions. + Keyword args: + range (sequence of float): Defines the leftmost and rightmost bin edges + in each dimension. + weight (Tensor): By default, each value in the input has weight 1. If a weight + tensor is passed, each N-dimensional coordinate in input + contributes its associated weight towards its bin's result. + The weight tensor should have the same shape as the :attr:`input` + tensor excluding its innermost dimension N. + density (bool): If False (default), the result will contain the count (or total weight) + in each bin. If True, each count (weight) is divided by the total count + (total weight), then divided by the volume of its associated bin. + Returns: + hist (Tensor): N-dimensional Tensor containing the values of the histogram. + bin_edges(Tensor[]): sequence of N 1D Tensors containing the bin edges. + + Example:: + + >>> torch.histogramdd(torch.tensor([[0., 1.], [1., 0.], [2., 0.], [2., 2.]]), bins=[3, 3], + ... weight=torch.tensor([1., 2., 4., 8.])) + torch.return_types.histogramdd( + hist=tensor([[0., 1., 0.], + [2., 0., 0.], + [4., 0., 8.]]), + bin_edges=(tensor([0.0000, 0.6667, 1.3333, 2.0000]), + tensor([0.0000, 0.6667, 1.3333, 2.0000]))) + + >>> torch.histogramdd(torch.tensor([[0., 0.], [1., 1.], [2., 2.]]), bins=[2, 2], + ... range=[0., 1., 0., 1.], density=True) + torch.return_types.histogramdd( + hist=tensor([[2., 0.], + [0., 2.]]), + bin_edges=(tensor([0.0000, 0.5000, 1.0000]), + tensor([0.0000, 0.5000, 1.0000]))) + """ + +@overload +def histogramdd( + input: Tensor, + bins: _size, + range: Sequence[_float] | None = None, + weight: Tensor | None = None, + density: _bool = False, +) -> torch.return_types.histogramdd: + r""" + histogramdd(input, bins, *, range=None, weight=None, density=False, out=None) -> (Tensor, Tensor[]) + + Computes a multi-dimensional histogram of the values in a tensor. + + Interprets the elements of an input tensor whose innermost dimension has size N + as a collection of N-dimensional points. Maps each of the points into a set of + N-dimensional bins and returns the number of points (or total weight) in each bin. + + :attr:`input` must be a tensor with at least 2 dimensions. + If input has shape (M, N), each of its M rows defines a point in N-dimensional space. + If input has three or more dimensions, all but the last dimension are flattened. + + Each dimension is independently associated with its own strictly increasing sequence + of bin edges. Bin edges may be specified explicitly by passing a sequence of 1D + tensors. Alternatively, bin edges may be constructed automatically by passing a + sequence of integers specifying the number of equal-width bins in each dimension. + + For each N-dimensional point in input: + - Each of its coordinates is binned independently among the bin edges + corresponding to its dimension + - Binning results are combined to identify the N-dimensional bin (if any) + into which the point falls + - If the point falls into a bin, the bin's count (or total weight) is incremented + - Points which do not fall into any bin do not contribute to the output + + :attr:`bins` can be a sequence of N 1D tensors, a sequence of N ints, or a single int. + + If :attr:`bins` is a sequence of N 1D tensors, it explicitly specifies the N sequences + of bin edges. Each 1D tensor should contain a strictly increasing sequence with at + least one element. A sequence of K bin edges defines K-1 bins, explicitly specifying + the left and right edges of all bins. Every bin is inclusive of its left edge. Only + the rightmost bin is inclusive of its right edge. + + If :attr:`bins` is a sequence of N ints, it specifies the number of equal-width bins + in each dimension. By default, the leftmost and rightmost bin edges in each dimension + are determined by the minimum and maximum elements of the input tensor in the + corresponding dimension. The :attr:`range` argument can be provided to manually + specify the leftmost and rightmost bin edges in each dimension. + + If :attr:`bins` is an int, it specifies the number of equal-width bins for all dimensions. + + .. note:: + See also :func:`torch.histogram`, which specifically computes 1D histograms. + While :func:`torch.histogramdd` infers the dimensionality of its bins and + binned values from the shape of :attr:`input`, :func:`torch.histogram` + accepts and flattens :attr:`input` of any shape. + + Args: + input (Tensor): the input tensor. + bins: Tensor[], int[], or int. + If Tensor[], defines the sequences of bin edges. + If int[], defines the number of equal-width bins in each dimension. + If int, defines the number of equal-width bins for all dimensions. + Keyword args: + range (sequence of float): Defines the leftmost and rightmost bin edges + in each dimension. + weight (Tensor): By default, each value in the input has weight 1. If a weight + tensor is passed, each N-dimensional coordinate in input + contributes its associated weight towards its bin's result. + The weight tensor should have the same shape as the :attr:`input` + tensor excluding its innermost dimension N. + density (bool): If False (default), the result will contain the count (or total weight) + in each bin. If True, each count (weight) is divided by the total count + (total weight), then divided by the volume of its associated bin. + Returns: + hist (Tensor): N-dimensional Tensor containing the values of the histogram. + bin_edges(Tensor[]): sequence of N 1D Tensors containing the bin edges. + + Example:: + + >>> torch.histogramdd(torch.tensor([[0., 1.], [1., 0.], [2., 0.], [2., 2.]]), bins=[3, 3], + ... weight=torch.tensor([1., 2., 4., 8.])) + torch.return_types.histogramdd( + hist=tensor([[0., 1., 0.], + [2., 0., 0.], + [4., 0., 8.]]), + bin_edges=(tensor([0.0000, 0.6667, 1.3333, 2.0000]), + tensor([0.0000, 0.6667, 1.3333, 2.0000]))) + + >>> torch.histogramdd(torch.tensor([[0., 0.], [1., 1.], [2., 2.]]), bins=[2, 2], + ... range=[0., 1., 0., 1.], density=True) + torch.return_types.histogramdd( + hist=tensor([[2., 0.], + [0., 2.]]), + bin_edges=(tensor([0.0000, 0.5000, 1.0000]), + tensor([0.0000, 0.5000, 1.0000]))) + """ + +@overload +def histogramdd( + input: Tensor, + bins: tuple[Tensor, ...] | list[Tensor] | None, + range: Sequence[_float] | None = None, + weight: Tensor | None = None, + density: _bool = False, +) -> torch.return_types.histogramdd: + r""" + histogramdd(input, bins, *, range=None, weight=None, density=False, out=None) -> (Tensor, Tensor[]) + + Computes a multi-dimensional histogram of the values in a tensor. + + Interprets the elements of an input tensor whose innermost dimension has size N + as a collection of N-dimensional points. Maps each of the points into a set of + N-dimensional bins and returns the number of points (or total weight) in each bin. + + :attr:`input` must be a tensor with at least 2 dimensions. + If input has shape (M, N), each of its M rows defines a point in N-dimensional space. + If input has three or more dimensions, all but the last dimension are flattened. + + Each dimension is independently associated with its own strictly increasing sequence + of bin edges. Bin edges may be specified explicitly by passing a sequence of 1D + tensors. Alternatively, bin edges may be constructed automatically by passing a + sequence of integers specifying the number of equal-width bins in each dimension. + + For each N-dimensional point in input: + - Each of its coordinates is binned independently among the bin edges + corresponding to its dimension + - Binning results are combined to identify the N-dimensional bin (if any) + into which the point falls + - If the point falls into a bin, the bin's count (or total weight) is incremented + - Points which do not fall into any bin do not contribute to the output + + :attr:`bins` can be a sequence of N 1D tensors, a sequence of N ints, or a single int. + + If :attr:`bins` is a sequence of N 1D tensors, it explicitly specifies the N sequences + of bin edges. Each 1D tensor should contain a strictly increasing sequence with at + least one element. A sequence of K bin edges defines K-1 bins, explicitly specifying + the left and right edges of all bins. Every bin is inclusive of its left edge. Only + the rightmost bin is inclusive of its right edge. + + If :attr:`bins` is a sequence of N ints, it specifies the number of equal-width bins + in each dimension. By default, the leftmost and rightmost bin edges in each dimension + are determined by the minimum and maximum elements of the input tensor in the + corresponding dimension. The :attr:`range` argument can be provided to manually + specify the leftmost and rightmost bin edges in each dimension. + + If :attr:`bins` is an int, it specifies the number of equal-width bins for all dimensions. + + .. note:: + See also :func:`torch.histogram`, which specifically computes 1D histograms. + While :func:`torch.histogramdd` infers the dimensionality of its bins and + binned values from the shape of :attr:`input`, :func:`torch.histogram` + accepts and flattens :attr:`input` of any shape. + + Args: + input (Tensor): the input tensor. + bins: Tensor[], int[], or int. + If Tensor[], defines the sequences of bin edges. + If int[], defines the number of equal-width bins in each dimension. + If int, defines the number of equal-width bins for all dimensions. + Keyword args: + range (sequence of float): Defines the leftmost and rightmost bin edges + in each dimension. + weight (Tensor): By default, each value in the input has weight 1. If a weight + tensor is passed, each N-dimensional coordinate in input + contributes its associated weight towards its bin's result. + The weight tensor should have the same shape as the :attr:`input` + tensor excluding its innermost dimension N. + density (bool): If False (default), the result will contain the count (or total weight) + in each bin. If True, each count (weight) is divided by the total count + (total weight), then divided by the volume of its associated bin. + Returns: + hist (Tensor): N-dimensional Tensor containing the values of the histogram. + bin_edges(Tensor[]): sequence of N 1D Tensors containing the bin edges. + + Example:: + + >>> torch.histogramdd(torch.tensor([[0., 1.], [1., 0.], [2., 0.], [2., 2.]]), bins=[3, 3], + ... weight=torch.tensor([1., 2., 4., 8.])) + torch.return_types.histogramdd( + hist=tensor([[0., 1., 0.], + [2., 0., 0.], + [4., 0., 8.]]), + bin_edges=(tensor([0.0000, 0.6667, 1.3333, 2.0000]), + tensor([0.0000, 0.6667, 1.3333, 2.0000]))) + + >>> torch.histogramdd(torch.tensor([[0., 0.], [1., 1.], [2., 2.]]), bins=[2, 2], + ... range=[0., 1., 0., 1.], density=True) + torch.return_types.histogramdd( + hist=tensor([[2., 0.], + [0., 2.]]), + bin_edges=(tensor([0.0000, 0.5000, 1.0000]), + tensor([0.0000, 0.5000, 1.0000]))) + """ + +def hsmm(input: Tensor, mat2: Tensor) -> Tensor: ... +@overload +def hsplit(input: Tensor, sections: _int) -> tuple[Tensor, ...]: + r""" + hsplit(input, indices_or_sections) -> List of Tensors + + Splits :attr:`input`, a tensor with one or more dimensions, into multiple tensors + horizontally according to :attr:`indices_or_sections`. Each split is a view of + :attr:`input`. + + If :attr:`input` is one dimensional this is equivalent to calling + torch.tensor_split(input, indices_or_sections, dim=0) (the split dimension is + zero), and if :attr:`input` has two or more dimensions it's equivalent to calling + torch.tensor_split(input, indices_or_sections, dim=1) (the split dimension is 1), + except that if :attr:`indices_or_sections` is an integer it must evenly divide + the split dimension or a runtime error will be thrown. + + This function is based on NumPy's :func:`numpy.hsplit`. + + Args: + input (Tensor): tensor to split. + indices_or_sections (int or list or tuple of ints): See argument in :func:`torch.tensor_split`. + + Example:: + + >>> t = torch.arange(16.0).reshape(4,4) + >>> t + tensor([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [12., 13., 14., 15.]]) + >>> torch.hsplit(t, 2) + (tensor([[ 0., 1.], + [ 4., 5.], + [ 8., 9.], + [12., 13.]]), + tensor([[ 2., 3.], + [ 6., 7.], + [10., 11.], + [14., 15.]])) + >>> torch.hsplit(t, [3, 6]) + (tensor([[ 0., 1., 2.], + [ 4., 5., 6.], + [ 8., 9., 10.], + [12., 13., 14.]]), + tensor([[ 3.], + [ 7.], + [11.], + [15.]]), + tensor([], size=(4, 0))) + """ + +@overload +def hsplit(input: Tensor, indices: _size) -> tuple[Tensor, ...]: + r""" + hsplit(input, indices_or_sections) -> List of Tensors + + Splits :attr:`input`, a tensor with one or more dimensions, into multiple tensors + horizontally according to :attr:`indices_or_sections`. Each split is a view of + :attr:`input`. + + If :attr:`input` is one dimensional this is equivalent to calling + torch.tensor_split(input, indices_or_sections, dim=0) (the split dimension is + zero), and if :attr:`input` has two or more dimensions it's equivalent to calling + torch.tensor_split(input, indices_or_sections, dim=1) (the split dimension is 1), + except that if :attr:`indices_or_sections` is an integer it must evenly divide + the split dimension or a runtime error will be thrown. + + This function is based on NumPy's :func:`numpy.hsplit`. + + Args: + input (Tensor): tensor to split. + indices_or_sections (int or list or tuple of ints): See argument in :func:`torch.tensor_split`. + + Example:: + + >>> t = torch.arange(16.0).reshape(4,4) + >>> t + tensor([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [12., 13., 14., 15.]]) + >>> torch.hsplit(t, 2) + (tensor([[ 0., 1.], + [ 4., 5.], + [ 8., 9.], + [12., 13.]]), + tensor([[ 2., 3.], + [ 6., 7.], + [10., 11.], + [14., 15.]])) + >>> torch.hsplit(t, [3, 6]) + (tensor([[ 0., 1., 2.], + [ 4., 5., 6.], + [ 8., 9., 10.], + [12., 13., 14.]]), + tensor([[ 3.], + [ 7.], + [11.], + [15.]]), + tensor([], size=(4, 0))) + """ + +def hspmm( + mat1: Tensor, + mat2: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + hspmm(mat1, mat2, *, out=None) -> Tensor + + Performs a matrix multiplication of a :ref:`sparse COO matrix + ` :attr:`mat1` and a strided matrix :attr:`mat2`. The + result is a (1 + 1)-dimensional :ref:`hybrid COO matrix + `. + + Args: + mat1 (Tensor): the first sparse matrix to be matrix multiplied + mat2 (Tensor): the second strided matrix to be matrix multiplied + + Keyword args: + out (Tensor, optional): the output tensor. + """ + +def hstack( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + hstack(tensors, *, out=None) -> Tensor + + Stack tensors in sequence horizontally (column wise). + + This is equivalent to concatenation along the first axis for 1-D tensors, and along the second axis for all other tensors. + + Args: + tensors (sequence of Tensors): sequence of tensors to concatenate + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([1, 2, 3]) + >>> b = torch.tensor([4, 5, 6]) + >>> torch.hstack((a,b)) + tensor([1, 2, 3, 4, 5, 6]) + >>> a = torch.tensor([[1],[2],[3]]) + >>> b = torch.tensor([[4],[5],[6]]) + >>> torch.hstack((a,b)) + tensor([[1, 4], + [2, 5], + [3, 6]]) + """ + +def hypot( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + hypot(input, other, *, out=None) -> Tensor + + Given the legs of a right triangle, return its hypotenuse. + + .. math:: + \text{out}_{i} = \sqrt{\text{input}_{i}^{2} + \text{other}_{i}^{2}} + + The shapes of ``input`` and ``other`` must be + :ref:`broadcastable `. + + Args: + input (Tensor): the first input tensor + other (Tensor): the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.hypot(torch.tensor([4.0]), torch.tensor([3.0, 4.0, 5.0])) + tensor([5.0000, 5.6569, 6.4031]) + """ + +def i0(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + i0(input, *, out=None) -> Tensor + + Alias for :func:`torch.special.i0`. + """ + +def i0_(input: Tensor) -> Tensor: ... +def igamma( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + igamma(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.special.gammainc`. + """ + +def igammac( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + igammac(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.special.gammaincc`. + """ + +def imag(input: Tensor) -> Tensor: + r""" + imag(input) -> Tensor + + Returns a new tensor containing imaginary values of the :attr:`self` tensor. + The returned tensor and :attr:`self` share the same underlying storage. + + .. warning:: + :func:`imag` is only supported for tensors with complex dtypes. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> x=torch.randn(4, dtype=torch.cfloat) + >>> x + tensor([(0.3100+0.3553j), (-0.5445-0.7896j), (-1.6492-0.0633j), (-0.0638-0.8119j)]) + >>> x.imag + tensor([ 0.3553, -0.7896, -0.0633, -0.8119]) + """ + +@overload +def index_add( + input: Tensor, + dim: _int, + index: Tensor, + source: Tensor, + *, + alpha: Number | _complex = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + index_add(input: Tensor, dim: int, index: Tensor, source: Tensor, *, alpha: Union[Number, _complex] = 1, out: Optional[Tensor]) -> Tensor # noqa: B950 + + See :meth:`~Tensor.index_add_` for function description. + """ + +@overload +def index_add( + input: Tensor, + dim: str | EllipsisType | None, + index: Tensor, + source: Tensor, + *, + alpha: Number | _complex = 1, +) -> Tensor: + r""" + index_add(input: Tensor, dim: int, index: Tensor, source: Tensor, *, alpha: Union[Number, _complex] = 1, out: Optional[Tensor]) -> Tensor # noqa: B950 + + See :meth:`~Tensor.index_add_` for function description. + """ + +@overload +def index_copy( + input: Tensor, + dim: _int, + index: Tensor, + source: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + index_copy(input: Tensor, dim: int, index: Tensor, source: Tensor, *, out: Optional[Tensor]) -> Tensor + + See :meth:`~Tensor.index_add_` for function description. + """ + +@overload +def index_copy( + input: Tensor, + dim: str | EllipsisType | None, + index: Tensor, + source: Tensor, +) -> Tensor: + r""" + index_copy(input: Tensor, dim: int, index: Tensor, source: Tensor, *, out: Optional[Tensor]) -> Tensor + + See :meth:`~Tensor.index_add_` for function description. + """ + +@overload +def index_fill( + input: Tensor, + dim: _int, + index: Tensor, + value: Tensor, +) -> Tensor: ... +@overload +def index_fill( + input: Tensor, + dim: str | EllipsisType | None, + index: Tensor, + value: Tensor, +) -> Tensor: ... +@overload +def index_fill( + input: Tensor, + dim: _int, + index: Tensor, + value: Number | _complex, +) -> Tensor: ... +@overload +def index_fill( + input: Tensor, + dim: str | EllipsisType | None, + index: Tensor, + value: Number | _complex, +) -> Tensor: ... +def index_put( + input: Tensor, + indices: tuple[Tensor, ...] | list[Tensor] | None, + values: Tensor, + accumulate: _bool = False, +) -> Tensor: ... +def index_put_( + input: Tensor, + indices: tuple[Tensor, ...] | list[Tensor] | None, + values: Tensor, + accumulate: _bool = False, +) -> Tensor: ... +def index_reduce( + input: Tensor, + dim: _int, + index: Tensor, + source: Tensor, + reduce: str, + *, + include_self: _bool = True, + out: Tensor | None = None, +) -> Tensor: + r""" + index_reduce(input: Tensor, dim: int, index: Tensor, source: Tensor, reduce: str, *, include_self: bool = True, out: Optional[Tensor]) -> Tensor # noqa: B950 + + See :meth:`~Tensor.index_reduce_` for function description. + """ + +@overload +def index_select( + input: Tensor, + dim: _int, + index: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + index_select(input, dim, index, *, out=None) -> Tensor + + Returns a new tensor which indexes the :attr:`input` tensor along dimension + :attr:`dim` using the entries in :attr:`index`. + + The returned tensor has the same number of dimensions as the original tensor + (:attr:`input`). The :attr:`dim`\ th dimension has the same size as the length + of :attr:`index`; other dimensions have the same size as in the original tensor. + + .. note:: The returned tensor does **not** use the same storage as the original + tensor. If :attr:`out` has a different shape than expected, we + silently change it to the correct shape, reallocating the underlying + storage if necessary. + + Args: + input (Tensor): the input tensor. + dim (int): the dimension in which we index + index (IntTensor or LongTensor): the 1-D tensor containing the indices to index + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> x = torch.randn(3, 4) + >>> x + tensor([[ 0.1427, 0.0231, -0.5414, -1.0009], + [-0.4664, 0.2647, -0.1228, -1.1068], + [-1.1734, -0.6571, 0.7230, -0.6004]]) + >>> indices = torch.tensor([0, 2]) + >>> torch.index_select(x, 0, indices) + tensor([[ 0.1427, 0.0231, -0.5414, -1.0009], + [-1.1734, -0.6571, 0.7230, -0.6004]]) + >>> torch.index_select(x, 1, indices) + tensor([[ 0.1427, -0.5414], + [-0.4664, -0.1228], + [-1.1734, 0.7230]]) + """ + +@overload +def index_select( + input: Tensor, + dim: str | EllipsisType | None, + index: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + index_select(input, dim, index, *, out=None) -> Tensor + + Returns a new tensor which indexes the :attr:`input` tensor along dimension + :attr:`dim` using the entries in :attr:`index`. + + The returned tensor has the same number of dimensions as the original tensor + (:attr:`input`). The :attr:`dim`\ th dimension has the same size as the length + of :attr:`index`; other dimensions have the same size as in the original tensor. + + .. note:: The returned tensor does **not** use the same storage as the original + tensor. If :attr:`out` has a different shape than expected, we + silently change it to the correct shape, reallocating the underlying + storage if necessary. + + Args: + input (Tensor): the input tensor. + dim (int): the dimension in which we index + index (IntTensor or LongTensor): the 1-D tensor containing the indices to index + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> x = torch.randn(3, 4) + >>> x + tensor([[ 0.1427, 0.0231, -0.5414, -1.0009], + [-0.4664, 0.2647, -0.1228, -1.1068], + [-1.1734, -0.6571, 0.7230, -0.6004]]) + >>> indices = torch.tensor([0, 2]) + >>> torch.index_select(x, 0, indices) + tensor([[ 0.1427, 0.0231, -0.5414, -1.0009], + [-1.1734, -0.6571, 0.7230, -0.6004]]) + >>> torch.index_select(x, 1, indices) + tensor([[ 0.1427, -0.5414], + [-0.4664, -0.1228], + [-1.1734, 0.7230]]) + """ + +def indices_copy(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + Performs the same operation as :func:`torch.indices`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def init_num_threads() -> None: ... +def inner( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + inner(input, other, *, out=None) -> Tensor + + Computes the dot product for 1D tensors. For higher dimensions, sums the product + of elements from :attr:`input` and :attr:`other` along their last dimension. + + .. note:: + + If either :attr:`input` or :attr:`other` is a scalar, the result is equivalent + to `torch.mul(input, other)`. + + If both :attr:`input` and :attr:`other` are non-scalars, the size of their last + dimension must match and the result is equivalent to `torch.tensordot(input, + other, dims=([-1], [-1]))` + + Args: + input (Tensor): First input tensor + other (Tensor): Second input tensor + + Keyword args: + out (Tensor, optional): Optional output tensor to write result into. The output + shape is `input.shape[:-1] + other.shape[:-1]`. + + Example:: + + # Dot product + >>> torch.inner(torch.tensor([1, 2, 3]), torch.tensor([0, 2, 1])) + tensor(7) + + # Multidimensional input tensors + >>> a = torch.randn(2, 3) + >>> a + tensor([[0.8173, 1.0874, 1.1784], + [0.3279, 0.1234, 2.7894]]) + >>> b = torch.randn(2, 4, 3) + >>> b + tensor([[[-0.4682, -0.7159, 0.1506], + [ 0.4034, -0.3657, 1.0387], + [ 0.9892, -0.6684, 0.1774], + [ 0.9482, 1.3261, 0.3917]], + + [[ 0.4537, 0.7493, 1.1724], + [ 0.2291, 0.5749, -0.2267], + [-0.7920, 0.3607, -0.3701], + [ 1.3666, -0.5850, -1.7242]]]) + >>> torch.inner(a, b) + tensor([[[-0.9837, 1.1560, 0.2907, 2.6785], + [ 2.5671, 0.5452, -0.6912, -1.5509]], + + [[ 0.1782, 2.9843, 0.7366, 1.5672], + [ 3.5115, -0.4864, -1.2476, -4.4337]]]) + + # Scalar input + >>> torch.inner(a, torch.tensor(2)) + tensor([[1.6347, 2.1748, 2.3567], + [0.6558, 0.2469, 5.5787]]) + """ + +def instance_norm( + input: Tensor, + weight: Tensor | None, + bias: Tensor | None, + running_mean: Tensor | None, + running_var: Tensor | None, + use_input_stats: _bool, + momentum: _float, + eps: _float, + cudnn_enabled: _bool, +) -> Tensor: ... +def int_repr(input: Tensor) -> Tensor: ... +def inverse(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + inverse(input, *, out=None) -> Tensor + + Alias for :func:`torch.linalg.inv` + """ + +def is_complex(input: Tensor) -> _bool: + r""" + is_complex(input: Tensor) -> bool + + Returns True if the data type of :attr:`input` is a complex data type i.e., + one of ``torch.complex64``, and ``torch.complex128``. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> torch.is_complex(torch.tensor([1, 2, 3], dtype=torch.complex64)) + True + >>> torch.is_complex(torch.tensor([1, 2, 3], dtype=torch.complex128)) + True + >>> torch.is_complex(torch.tensor([1, 2, 3], dtype=torch.int32)) + False + >>> torch.is_complex(torch.tensor([1.0, 2.0, 3.0], dtype=torch.float16)) + False + """ + +def is_conj(input: Tensor) -> _bool: + r""" + is_conj(input) -> (bool) + + Returns True if the :attr:`input` is a conjugated tensor, i.e. its conjugate bit is set to `True`. + + Args: + input (Tensor): the input tensor. + """ + +def is_distributed(input: Tensor) -> _bool: ... +def is_floating_point(input: Tensor) -> _bool: + r""" + is_floating_point(input: Tensor) -> bool + + Returns True if the data type of :attr:`input` is a floating point data type i.e., + one of ``torch.float64``, ``torch.float32``, ``torch.float16``, and ``torch.bfloat16``. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> torch.is_floating_point(torch.tensor([1.0, 2.0, 3.0])) + True + >>> torch.is_floating_point(torch.tensor([1, 2, 3], dtype=torch.int32)) + False + >>> torch.is_floating_point(torch.tensor([1.0, 2.0, 3.0], dtype=torch.float16)) + True + >>> torch.is_floating_point(torch.tensor([1, 2, 3], dtype=torch.complex64)) + False + """ + +def is_grad_enabled() -> _bool: + r""" + is_grad_enabled() -> (bool) + + Returns True if grad mode is currently enabled. + """ + +def is_inference(input: Tensor) -> _bool: + r""" + is_inference(input) -> (bool) + + Returns True if :attr:`input` is an inference tensor. + + A non-view tensor is an inference tensor if and only if it was + allocated during inference mode. A view tensor is an inference + tensor if and only if the tensor it is a view of is an inference tensor. + + For details on inference mode please see + `Inference Mode `_. + + Args: + input (Tensor): the input tensor. + """ + +def is_inference_mode_enabled() -> _bool: + r""" + is_inference_mode_enabled() -> (bool) + + Returns True if inference mode is currently enabled. + """ + +def is_neg(input: Tensor) -> _bool: ... +def is_nonzero(input: Tensor) -> _bool: + r""" + is_nonzero(input) -> (bool) + + Returns True if the :attr:`input` is a single element tensor which is not equal to zero + after type conversions. + i.e. not equal to ``torch.tensor([0.])`` or ``torch.tensor([0])`` or + ``torch.tensor([False])``. + Throws a ``RuntimeError`` if ``torch.numel() != 1`` (even in case + of sparse tensors). + + Args: + input (Tensor): the input tensor. + + Examples:: + + >>> torch.is_nonzero(torch.tensor([0.])) + False + >>> torch.is_nonzero(torch.tensor([1.5])) + True + >>> torch.is_nonzero(torch.tensor([False])) + False + >>> torch.is_nonzero(torch.tensor([3])) + True + >>> torch.is_nonzero(torch.tensor([1, 3, 5])) + Traceback (most recent call last): + ... + RuntimeError: Boolean value of Tensor with more than one value is ambiguous + >>> torch.is_nonzero(torch.tensor([])) + Traceback (most recent call last): + ... + RuntimeError: Boolean value of Tensor with no values is ambiguous + """ + +def is_same_size(input: Tensor, other: Tensor) -> _bool: ... +def is_signed(input: Tensor) -> _bool: ... +def is_vulkan_available() -> _bool: ... +def isclose( + input: Tensor, + other: Tensor, + rtol: _float = 1e-05, + atol: _float = 1e-08, + equal_nan: _bool = False, +) -> Tensor: + r""" + isclose(input, other, rtol=1e-05, atol=1e-08, equal_nan=False) -> Tensor + + Returns a new tensor with boolean elements representing if each element of + :attr:`input` is "close" to the corresponding element of :attr:`other`. + Closeness is defined as: + + .. math:: + \lvert \text{input}_i - \text{other}_i \rvert \leq \texttt{rtol} \times \lvert \text{other}_i \rvert + \texttt{atol} + + + where :attr:`input` and :attr:`other` are finite. Where :attr:`input` + and/or :attr:`other` are nonfinite they are close if and only if + they are equal, with NaNs being considered equal to each other when + :attr:`equal_nan` is True. + + Args: + input (Tensor): first tensor to compare + other (Tensor): second tensor to compare + rtol (float, optional): relative tolerance. Default: 1e-05 + atol (float, optional): absolute tolerance. Default: 1e-08 + equal_nan (bool, optional): if ``True``, then two ``NaN`` s will be considered equal. Default: ``False`` + + Examples:: + + >>> torch.isclose(torch.tensor((1., 2, 3)), torch.tensor((1 + 1e-10, 3, 4))) + tensor([ True, False, False]) + >>> torch.isclose(torch.tensor((float('inf'), 4)), torch.tensor((float('inf'), 6)), rtol=.5) + tensor([True, True]) + """ + +def isfinite(input: Tensor) -> Tensor: + r""" + isfinite(input) -> Tensor + + Returns a new tensor with boolean elements representing if each element is `finite` or not. + + Real values are finite when they are not NaN, negative infinity, or infinity. + Complex values are finite when both their real and imaginary parts are finite. + + Args: + input (Tensor): the input tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is finite and False elsewhere + + Example:: + + >>> torch.isfinite(torch.tensor([1, float('inf'), 2, float('-inf'), float('nan')])) + tensor([True, False, True, False, False]) + """ + +@overload +def isin( + elements: Tensor, + test_elements: Tensor, + *, + assume_unique: _bool = False, + invert: _bool = False, + out: Tensor | None = None, +) -> Tensor: + r""" + isin(elements, test_elements, *, assume_unique=False, invert=False) -> Tensor + + Tests if each element of :attr:`elements` is in :attr:`test_elements`. Returns + a boolean tensor of the same shape as :attr:`elements` that is True for elements + in :attr:`test_elements` and False otherwise. + + .. note:: + One of :attr:`elements` or :attr:`test_elements` can be a scalar, but not both. + + Args: + elements (Tensor or Scalar): Input elements + test_elements (Tensor or Scalar): Values against which to test for each input element + assume_unique (bool, optional): If True, assumes both :attr:`elements` and + :attr:`test_elements` contain unique elements, which can speed up the + calculation. Default: False + invert (bool, optional): If True, inverts the boolean return tensor, resulting in True + values for elements *not* in :attr:`test_elements`. Default: False + + Returns: + A boolean tensor of the same shape as :attr:`elements` that is True for elements in + :attr:`test_elements` and False otherwise + + Example: + >>> torch.isin(torch.tensor([[1, 2], [3, 4]]), torch.tensor([2, 3])) + tensor([[False, True], + [ True, False]]) + """ + +@overload +def isin( + element: Number | _complex, + test_elements: Tensor, + *, + assume_unique: _bool = False, + invert: _bool = False, + out: Tensor | None = None, +) -> Tensor: + r""" + isin(elements, test_elements, *, assume_unique=False, invert=False) -> Tensor + + Tests if each element of :attr:`elements` is in :attr:`test_elements`. Returns + a boolean tensor of the same shape as :attr:`elements` that is True for elements + in :attr:`test_elements` and False otherwise. + + .. note:: + One of :attr:`elements` or :attr:`test_elements` can be a scalar, but not both. + + Args: + elements (Tensor or Scalar): Input elements + test_elements (Tensor or Scalar): Values against which to test for each input element + assume_unique (bool, optional): If True, assumes both :attr:`elements` and + :attr:`test_elements` contain unique elements, which can speed up the + calculation. Default: False + invert (bool, optional): If True, inverts the boolean return tensor, resulting in True + values for elements *not* in :attr:`test_elements`. Default: False + + Returns: + A boolean tensor of the same shape as :attr:`elements` that is True for elements in + :attr:`test_elements` and False otherwise + + Example: + >>> torch.isin(torch.tensor([[1, 2], [3, 4]]), torch.tensor([2, 3])) + tensor([[False, True], + [ True, False]]) + """ + +@overload +def isin( + elements: Tensor, + test_element: Number | _complex, + *, + assume_unique: _bool = False, + invert: _bool = False, + out: Tensor | None = None, +) -> Tensor: + r""" + isin(elements, test_elements, *, assume_unique=False, invert=False) -> Tensor + + Tests if each element of :attr:`elements` is in :attr:`test_elements`. Returns + a boolean tensor of the same shape as :attr:`elements` that is True for elements + in :attr:`test_elements` and False otherwise. + + .. note:: + One of :attr:`elements` or :attr:`test_elements` can be a scalar, but not both. + + Args: + elements (Tensor or Scalar): Input elements + test_elements (Tensor or Scalar): Values against which to test for each input element + assume_unique (bool, optional): If True, assumes both :attr:`elements` and + :attr:`test_elements` contain unique elements, which can speed up the + calculation. Default: False + invert (bool, optional): If True, inverts the boolean return tensor, resulting in True + values for elements *not* in :attr:`test_elements`. Default: False + + Returns: + A boolean tensor of the same shape as :attr:`elements` that is True for elements in + :attr:`test_elements` and False otherwise + + Example: + >>> torch.isin(torch.tensor([[1, 2], [3, 4]]), torch.tensor([2, 3])) + tensor([[False, True], + [ True, False]]) + """ + +def isinf(input: Tensor) -> Tensor: + r""" + isinf(input) -> Tensor + + Tests if each element of :attr:`input` is infinite + (positive or negative infinity) or not. + + .. note:: + Complex values are infinite when their real or imaginary part is + infinite. + + Args: + input (Tensor): the input tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is infinite and False elsewhere + + Example:: + + >>> torch.isinf(torch.tensor([1, float('inf'), 2, float('-inf'), float('nan')])) + tensor([False, True, False, True, False]) + """ + +def isnan(input: Tensor) -> Tensor: + r""" + isnan(input) -> Tensor + + Returns a new tensor with boolean elements representing if each element of :attr:`input` + is NaN or not. Complex values are considered NaN when either their real + and/or imaginary part is NaN. + + Arguments: + input (Tensor): the input tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is NaN and False elsewhere + + Example:: + + >>> torch.isnan(torch.tensor([1, float('nan'), 2])) + tensor([False, True, False]) + """ + +def isneginf(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + isneginf(input, *, out=None) -> Tensor + Tests if each element of :attr:`input` is negative infinity or not. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([-float('inf'), float('inf'), 1.2]) + >>> torch.isneginf(a) + tensor([ True, False, False]) + """ + +def isposinf(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + isposinf(input, *, out=None) -> Tensor + Tests if each element of :attr:`input` is positive infinity or not. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([-float('inf'), float('inf'), 1.2]) + >>> torch.isposinf(a) + tensor([False, True, False]) + """ + +def isreal(input: Tensor) -> Tensor: + r""" + isreal(input) -> Tensor + + Returns a new tensor with boolean elements representing if each element of :attr:`input` is real-valued or not. + All real-valued types are considered real. Complex values are considered real when their imaginary part is 0. + + Arguments: + input (Tensor): the input tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is real and False elsewhere + + Example:: + + >>> torch.isreal(torch.tensor([1, 1+1j, 2+0j])) + tensor([True, False, True]) + """ + +def istft( + input: Tensor, + n_fft: _int, + hop_length: _int | None = None, + win_length: _int | None = None, + window: Tensor | None = None, + center: _bool = True, + normalized: _bool = False, + onesided: _bool | None = None, + length: _int | None = None, + return_complex: _bool = False, +) -> Tensor: ... +@overload +def kaiser_window( + window_length: _int, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + kaiser_window(window_length, periodic=True, beta=12.0, *, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Computes the Kaiser window with window length :attr:`window_length` and shape parameter :attr:`beta`. + + Let I_0 be the zeroth order modified Bessel function of the first kind (see :func:`torch.i0`) and + ``N = L - 1`` if :attr:`periodic` is False and ``L`` if :attr:`periodic` is True, + where ``L`` is the :attr:`window_length`. This function computes: + + .. math:: + out_i = I_0 \left( \beta \sqrt{1 - \left( {\frac{i - N/2}{N/2}} \right) ^2 } \right) / I_0( \beta ) + + Calling ``torch.kaiser_window(L, B, periodic=True)`` is equivalent to calling + ``torch.kaiser_window(L + 1, B, periodic=False)[:-1])``. + The :attr:`periodic` argument is intended as a helpful shorthand + to produce a periodic window as input to functions like :func:`torch.stft`. + + .. note:: + If :attr:`window_length` is one, then the returned window is a single element tensor containing a one. + + + Args: + window_length (int): length of the window. + periodic (bool, optional): If True, returns a periodic window suitable for use in spectral analysis. + If False, returns a symmetric window suitable for use in filter design. + beta (float, optional): shape parameter for the window. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + """ + +@overload +def kaiser_window( + window_length: _int, + periodic: _bool, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + kaiser_window(window_length, periodic=True, beta=12.0, *, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Computes the Kaiser window with window length :attr:`window_length` and shape parameter :attr:`beta`. + + Let I_0 be the zeroth order modified Bessel function of the first kind (see :func:`torch.i0`) and + ``N = L - 1`` if :attr:`periodic` is False and ``L`` if :attr:`periodic` is True, + where ``L`` is the :attr:`window_length`. This function computes: + + .. math:: + out_i = I_0 \left( \beta \sqrt{1 - \left( {\frac{i - N/2}{N/2}} \right) ^2 } \right) / I_0( \beta ) + + Calling ``torch.kaiser_window(L, B, periodic=True)`` is equivalent to calling + ``torch.kaiser_window(L + 1, B, periodic=False)[:-1])``. + The :attr:`periodic` argument is intended as a helpful shorthand + to produce a periodic window as input to functions like :func:`torch.stft`. + + .. note:: + If :attr:`window_length` is one, then the returned window is a single element tensor containing a one. + + + Args: + window_length (int): length of the window. + periodic (bool, optional): If True, returns a periodic window suitable for use in spectral analysis. + If False, returns a symmetric window suitable for use in filter design. + beta (float, optional): shape parameter for the window. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + """ + +@overload +def kaiser_window( + window_length: _int, + periodic: _bool, + beta: _float, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + kaiser_window(window_length, periodic=True, beta=12.0, *, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Computes the Kaiser window with window length :attr:`window_length` and shape parameter :attr:`beta`. + + Let I_0 be the zeroth order modified Bessel function of the first kind (see :func:`torch.i0`) and + ``N = L - 1`` if :attr:`periodic` is False and ``L`` if :attr:`periodic` is True, + where ``L`` is the :attr:`window_length`. This function computes: + + .. math:: + out_i = I_0 \left( \beta \sqrt{1 - \left( {\frac{i - N/2}{N/2}} \right) ^2 } \right) / I_0( \beta ) + + Calling ``torch.kaiser_window(L, B, periodic=True)`` is equivalent to calling + ``torch.kaiser_window(L + 1, B, periodic=False)[:-1])``. + The :attr:`periodic` argument is intended as a helpful shorthand + to produce a periodic window as input to functions like :func:`torch.stft`. + + .. note:: + If :attr:`window_length` is one, then the returned window is a single element tensor containing a one. + + + Args: + window_length (int): length of the window. + periodic (bool, optional): If True, returns a periodic window suitable for use in spectral analysis. + If False, returns a symmetric window suitable for use in filter design. + beta (float, optional): shape parameter for the window. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned window tensor. Only + ``torch.strided`` (dense layout) is supported. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + """ + +def kl_div( + input: Tensor, + target: Tensor, + reduction: _int = 1, + *, + log_target: _bool = False, +) -> Tensor: ... +def kron( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + kron(input, other, *, out=None) -> Tensor + + Computes the Kronecker product, denoted by :math:`\otimes`, of :attr:`input` and :attr:`other`. + + If :attr:`input` is a :math:`(a_0 \times a_1 \times \dots \times a_n)` tensor and :attr:`other` is a + :math:`(b_0 \times b_1 \times \dots \times b_n)` tensor, the result will be a + :math:`(a_0*b_0 \times a_1*b_1 \times \dots \times a_n*b_n)` tensor with the following entries: + + .. math:: + (\text{input} \otimes \text{other})_{k_0, k_1, \dots, k_n} = + \text{input}_{i_0, i_1, \dots, i_n} * \text{other}_{j_0, j_1, \dots, j_n}, + + where :math:`k_t = i_t * b_t + j_t` for :math:`0 \leq t \leq n`. + If one tensor has fewer dimensions than the other it is unsqueezed until it has the same number of dimensions. + + Supports real-valued and complex-valued inputs. + + .. note:: + This function generalizes the typical definition of the Kronecker product for two matrices to two tensors, + as described above. When :attr:`input` is a :math:`(m \times n)` matrix and :attr:`other` is a + :math:`(p \times q)` matrix, the result will be a :math:`(p*m \times q*n)` block matrix: + + .. math:: + \mathbf{A} \otimes \mathbf{B}=\begin{bmatrix} + a_{11} \mathbf{B} & \cdots & a_{1 n} \mathbf{B} \\ + \vdots & \ddots & \vdots \\ + a_{m 1} \mathbf{B} & \cdots & a_{m n} \mathbf{B} \end{bmatrix} + + where :attr:`input` is :math:`\mathbf{A}` and :attr:`other` is :math:`\mathbf{B}`. + + Arguments: + input (Tensor) + other (Tensor) + + Keyword args: + out (Tensor, optional): The output tensor. Ignored if ``None``. Default: ``None`` + + Examples:: + + >>> mat1 = torch.eye(2) + >>> mat2 = torch.ones(2, 2) + >>> torch.kron(mat1, mat2) + tensor([[1., 1., 0., 0.], + [1., 1., 0., 0.], + [0., 0., 1., 1.], + [0., 0., 1., 1.]]) + + >>> mat1 = torch.eye(2) + >>> mat2 = torch.arange(1, 5).reshape(2, 2) + >>> torch.kron(mat1, mat2) + tensor([[1., 2., 0., 0.], + [3., 4., 0., 0.], + [0., 0., 1., 2.], + [0., 0., 3., 4.]]) + """ + +@overload +def kthvalue( + input: Tensor, + k: _int | SymInt, + dim: _int = -1, + keepdim: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.kthvalue: + r""" + kthvalue(input, k, dim=None, keepdim=False, *, out=None) -> (Tensor, LongTensor) + + Returns a namedtuple ``(values, indices)`` where ``values`` is the :attr:`k` th + smallest element of each row of the :attr:`input` tensor in the given dimension + :attr:`dim`. And ``indices`` is the index location of each element found. + + If :attr:`dim` is not given, the last dimension of the `input` is chosen. + + If :attr:`keepdim` is ``True``, both the :attr:`values` and :attr:`indices` tensors + are the same size as :attr:`input`, except in the dimension :attr:`dim` where + they are of size 1. Otherwise, :attr:`dim` is squeezed + (see :func:`torch.squeeze`), resulting in both the :attr:`values` and + :attr:`indices` tensors having 1 fewer dimension than the :attr:`input` tensor. + + .. note:: + When :attr:`input` is a CUDA tensor and there are multiple valid + :attr:`k` th values, this function may nondeterministically return + :attr:`indices` for any of them. + + Args: + input (Tensor): the input tensor. + k (int): k for the k-th smallest element + dim (int, optional): the dimension to find the kth value along + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (tuple, optional): the output tuple of (Tensor, LongTensor) + can be optionally given to be used as output buffers + + Example:: + + >>> x = torch.arange(1., 6.) + >>> x + tensor([ 1., 2., 3., 4., 5.]) + >>> torch.kthvalue(x, 4) + torch.return_types.kthvalue(values=tensor(4.), indices=tensor(3)) + + >>> x=torch.arange(1.,7.).resize_(2,3) + >>> x + tensor([[ 1., 2., 3.], + [ 4., 5., 6.]]) + >>> torch.kthvalue(x, 2, 0, True) + torch.return_types.kthvalue(values=tensor([[4., 5., 6.]]), indices=tensor([[1, 1, 1]])) + """ + +@overload +def kthvalue( + input: Tensor, + k: _int | SymInt, + dim: str | EllipsisType | None, + keepdim: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.kthvalue: + r""" + kthvalue(input, k, dim=None, keepdim=False, *, out=None) -> (Tensor, LongTensor) + + Returns a namedtuple ``(values, indices)`` where ``values`` is the :attr:`k` th + smallest element of each row of the :attr:`input` tensor in the given dimension + :attr:`dim`. And ``indices`` is the index location of each element found. + + If :attr:`dim` is not given, the last dimension of the `input` is chosen. + + If :attr:`keepdim` is ``True``, both the :attr:`values` and :attr:`indices` tensors + are the same size as :attr:`input`, except in the dimension :attr:`dim` where + they are of size 1. Otherwise, :attr:`dim` is squeezed + (see :func:`torch.squeeze`), resulting in both the :attr:`values` and + :attr:`indices` tensors having 1 fewer dimension than the :attr:`input` tensor. + + .. note:: + When :attr:`input` is a CUDA tensor and there are multiple valid + :attr:`k` th values, this function may nondeterministically return + :attr:`indices` for any of them. + + Args: + input (Tensor): the input tensor. + k (int): k for the k-th smallest element + dim (int, optional): the dimension to find the kth value along + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (tuple, optional): the output tuple of (Tensor, LongTensor) + can be optionally given to be used as output buffers + + Example:: + + >>> x = torch.arange(1., 6.) + >>> x + tensor([ 1., 2., 3., 4., 5.]) + >>> torch.kthvalue(x, 4) + torch.return_types.kthvalue(values=tensor(4.), indices=tensor(3)) + + >>> x=torch.arange(1.,7.).resize_(2,3) + >>> x + tensor([[ 1., 2., 3.], + [ 4., 5., 6.]]) + >>> torch.kthvalue(x, 2, 0, True) + torch.return_types.kthvalue(values=tensor([[4., 5., 6.]]), indices=tensor([[1, 1, 1]])) + """ + +def layer_norm( + input: Tensor, + normalized_shape: Sequence[_int | SymInt], + weight: Tensor | None = None, + bias: Tensor | None = None, + eps: _float = 1e-05, + cudnn_enable: _bool = True, +) -> Tensor: ... +def lcm( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + lcm(input, other, *, out=None) -> Tensor + + Computes the element-wise least common multiple (LCM) of :attr:`input` and :attr:`other`. + + Both :attr:`input` and :attr:`other` must have integer types. + + .. note:: + This defines :math:`lcm(0, 0) = 0` and :math:`lcm(0, a) = 0`. + + Args: + input (Tensor): the input tensor. + other (Tensor): the second input tensor + + Keyword arguments: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([5, 10, 15]) + >>> b = torch.tensor([3, 4, 5]) + >>> torch.lcm(a, b) + tensor([15, 20, 15]) + >>> c = torch.tensor([3]) + >>> torch.lcm(a, c) + tensor([15, 30, 15]) + """ + +def lcm_(input: Tensor, other: Tensor) -> Tensor: ... +def ldexp( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + ldexp(input, other, *, out=None) -> Tensor + + Multiplies :attr:`input` by 2 ** :attr:`other`. + + .. math:: + \text{{out}}_i = \text{{input}}_i * 2^\text{{other}}_i + + + Typically this function is used to construct floating point numbers by multiplying + mantissas in :attr:`input` with integral powers of two created from the exponents + in :attr:`other`. + + Args: + input (Tensor): the input tensor. + other (Tensor): a tensor of exponents, typically integers. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.ldexp(torch.tensor([1.]), torch.tensor([1])) + tensor([2.]) + >>> torch.ldexp(torch.tensor([1.0]), torch.tensor([1, 2, 3, 4])) + tensor([ 2., 4., 8., 16.]) + """ + +def ldexp_(input: Tensor, other: Tensor) -> Tensor: ... +@overload +def le( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + le(input, other, *, out=None) -> Tensor + + Computes :math:`\text{input} \leq \text{other}` element-wise. + + + The second argument can be a number or a tensor whose shape is + :ref:`broadcastable ` with the first argument. + + Args: + input (Tensor): the tensor to compare + other (Tensor or Scalar): the tensor or value to compare + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is less than or equal to + :attr:`other` and False elsewhere + + Example:: + + >>> torch.le(torch.tensor([[1, 2], [3, 4]]), torch.tensor([[1, 1], [4, 4]])) + tensor([[True, False], [True, True]]) + """ + +@overload +def le( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + le(input, other, *, out=None) -> Tensor + + Computes :math:`\text{input} \leq \text{other}` element-wise. + + + The second argument can be a number or a tensor whose shape is + :ref:`broadcastable ` with the first argument. + + Args: + input (Tensor): the tensor to compare + other (Tensor or Scalar): the tensor or value to compare + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is less than or equal to + :attr:`other` and False elsewhere + + Example:: + + >>> torch.le(torch.tensor([[1, 2], [3, 4]]), torch.tensor([[1, 1], [4, 4]])) + tensor([[True, False], [True, True]]) + """ + +@overload +def lerp( + input: Tensor, + end: Tensor, + weight: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + lerp(input, end, weight, *, out=None) + + Does a linear interpolation of two tensors :attr:`start` (given by :attr:`input`) and :attr:`end` based + on a scalar or tensor :attr:`weight` and returns the resulting :attr:`out` tensor. + + .. math:: + \text{out}_i = \text{start}_i + \text{weight}_i \times (\text{end}_i - \text{start}_i) + + The shapes of :attr:`start` and :attr:`end` must be + :ref:`broadcastable `. If :attr:`weight` is a tensor, then + the shapes of :attr:`weight`, :attr:`start`, and :attr:`end` must be :ref:`broadcastable `. + + Args: + input (Tensor): the tensor with the starting points + end (Tensor): the tensor with the ending points + weight (float or tensor): the weight for the interpolation formula + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> start = torch.arange(1., 5.) + >>> end = torch.empty(4).fill_(10) + >>> start + tensor([ 1., 2., 3., 4.]) + >>> end + tensor([ 10., 10., 10., 10.]) + >>> torch.lerp(start, end, 0.5) + tensor([ 5.5000, 6.0000, 6.5000, 7.0000]) + >>> torch.lerp(start, end, torch.full_like(start, 0.5)) + tensor([ 5.5000, 6.0000, 6.5000, 7.0000]) + """ + +@overload +def lerp( + input: Tensor, + end: Tensor, + weight: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + lerp(input, end, weight, *, out=None) + + Does a linear interpolation of two tensors :attr:`start` (given by :attr:`input`) and :attr:`end` based + on a scalar or tensor :attr:`weight` and returns the resulting :attr:`out` tensor. + + .. math:: + \text{out}_i = \text{start}_i + \text{weight}_i \times (\text{end}_i - \text{start}_i) + + The shapes of :attr:`start` and :attr:`end` must be + :ref:`broadcastable `. If :attr:`weight` is a tensor, then + the shapes of :attr:`weight`, :attr:`start`, and :attr:`end` must be :ref:`broadcastable `. + + Args: + input (Tensor): the tensor with the starting points + end (Tensor): the tensor with the ending points + weight (float or tensor): the weight for the interpolation formula + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> start = torch.arange(1., 5.) + >>> end = torch.empty(4).fill_(10) + >>> start + tensor([ 1., 2., 3., 4.]) + >>> end + tensor([ 10., 10., 10., 10.]) + >>> torch.lerp(start, end, 0.5) + tensor([ 5.5000, 6.0000, 6.5000, 7.0000]) + >>> torch.lerp(start, end, torch.full_like(start, 0.5)) + tensor([ 5.5000, 6.0000, 6.5000, 7.0000]) + """ + +@overload +def less( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + less(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.lt`. + """ + +@overload +def less( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + less(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.lt`. + """ + +@overload +def less_equal( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + less_equal(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.le`. + """ + +@overload +def less_equal( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + less_equal(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.le`. + """ + +def lgamma(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + lgamma(input, *, out=None) -> Tensor + + Computes the natural logarithm of the absolute value of the gamma function on :attr:`input`. + + .. math:: + \text{out}_{i} = \ln |\Gamma(\text{input}_{i})| + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.arange(0.5, 2, 0.5) + >>> torch.lgamma(a) + tensor([ 0.5724, 0.0000, -0.1208]) + """ + +@overload +def linspace( + start: Number, + end: Number, + steps: _int | None = None, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + pin_memory: _bool = False, +) -> Tensor: + r""" + linspace(start, end, steps, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Creates a one-dimensional tensor of size :attr:`steps` whose values are evenly + spaced from :attr:`start` to :attr:`end`, inclusive. That is, the value are: + + .. math:: + (\text{start}, + \text{start} + \frac{\text{end} - \text{start}}{\text{steps} - 1}, + \ldots, + \text{start} + (\text{steps} - 2) * \frac{\text{end} - \text{start}}{\text{steps} - 1}, + \text{end}) + + + From PyTorch 1.11 linspace requires the steps argument. Use steps=100 to restore the previous behavior. + + Args: + start (float or Tensor): the starting value for the set of points. If `Tensor`, it must be 0-dimensional + end (float or Tensor): the ending value for the set of points. If `Tensor`, it must be 0-dimensional + steps (int): size of the constructed tensor + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the data type to perform the computation in. + Default: if None, uses the global default dtype (see torch.get_default_dtype()) + when both :attr:`start` and :attr:`end` are real, + and corresponding complex dtype when either is complex. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + + Example:: + + >>> torch.linspace(3, 10, steps=5) + tensor([ 3.0000, 4.7500, 6.5000, 8.2500, 10.0000]) + >>> torch.linspace(-10, 10, steps=5) + tensor([-10., -5., 0., 5., 10.]) + >>> torch.linspace(start=-10, end=10, steps=5) + tensor([-10., -5., 0., 5., 10.]) + >>> torch.linspace(start=-10, end=10, steps=1) + tensor([-10.]) + """ + +@overload +def linspace( + start: Tensor, + end: Tensor, + steps: _int, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + linspace(start, end, steps, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Creates a one-dimensional tensor of size :attr:`steps` whose values are evenly + spaced from :attr:`start` to :attr:`end`, inclusive. That is, the value are: + + .. math:: + (\text{start}, + \text{start} + \frac{\text{end} - \text{start}}{\text{steps} - 1}, + \ldots, + \text{start} + (\text{steps} - 2) * \frac{\text{end} - \text{start}}{\text{steps} - 1}, + \text{end}) + + + From PyTorch 1.11 linspace requires the steps argument. Use steps=100 to restore the previous behavior. + + Args: + start (float or Tensor): the starting value for the set of points. If `Tensor`, it must be 0-dimensional + end (float or Tensor): the ending value for the set of points. If `Tensor`, it must be 0-dimensional + steps (int): size of the constructed tensor + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the data type to perform the computation in. + Default: if None, uses the global default dtype (see torch.get_default_dtype()) + when both :attr:`start` and :attr:`end` are real, + and corresponding complex dtype when either is complex. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + + Example:: + + >>> torch.linspace(3, 10, steps=5) + tensor([ 3.0000, 4.7500, 6.5000, 8.2500, 10.0000]) + >>> torch.linspace(-10, 10, steps=5) + tensor([-10., -5., 0., 5., 10.]) + >>> torch.linspace(start=-10, end=10, steps=5) + tensor([-10., -5., 0., 5., 10.]) + >>> torch.linspace(start=-10, end=10, steps=1) + tensor([-10.]) + """ + +@overload +def linspace( + start: Number | _complex, + end: Tensor, + steps: _int, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + linspace(start, end, steps, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Creates a one-dimensional tensor of size :attr:`steps` whose values are evenly + spaced from :attr:`start` to :attr:`end`, inclusive. That is, the value are: + + .. math:: + (\text{start}, + \text{start} + \frac{\text{end} - \text{start}}{\text{steps} - 1}, + \ldots, + \text{start} + (\text{steps} - 2) * \frac{\text{end} - \text{start}}{\text{steps} - 1}, + \text{end}) + + + From PyTorch 1.11 linspace requires the steps argument. Use steps=100 to restore the previous behavior. + + Args: + start (float or Tensor): the starting value for the set of points. If `Tensor`, it must be 0-dimensional + end (float or Tensor): the ending value for the set of points. If `Tensor`, it must be 0-dimensional + steps (int): size of the constructed tensor + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the data type to perform the computation in. + Default: if None, uses the global default dtype (see torch.get_default_dtype()) + when both :attr:`start` and :attr:`end` are real, + and corresponding complex dtype when either is complex. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + + Example:: + + >>> torch.linspace(3, 10, steps=5) + tensor([ 3.0000, 4.7500, 6.5000, 8.2500, 10.0000]) + >>> torch.linspace(-10, 10, steps=5) + tensor([-10., -5., 0., 5., 10.]) + >>> torch.linspace(start=-10, end=10, steps=5) + tensor([-10., -5., 0., 5., 10.]) + >>> torch.linspace(start=-10, end=10, steps=1) + tensor([-10.]) + """ + +@overload +def linspace( + start: Tensor, + end: Number | _complex, + steps: _int, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + linspace(start, end, steps, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Creates a one-dimensional tensor of size :attr:`steps` whose values are evenly + spaced from :attr:`start` to :attr:`end`, inclusive. That is, the value are: + + .. math:: + (\text{start}, + \text{start} + \frac{\text{end} - \text{start}}{\text{steps} - 1}, + \ldots, + \text{start} + (\text{steps} - 2) * \frac{\text{end} - \text{start}}{\text{steps} - 1}, + \text{end}) + + + From PyTorch 1.11 linspace requires the steps argument. Use steps=100 to restore the previous behavior. + + Args: + start (float or Tensor): the starting value for the set of points. If `Tensor`, it must be 0-dimensional + end (float or Tensor): the ending value for the set of points. If `Tensor`, it must be 0-dimensional + steps (int): size of the constructed tensor + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the data type to perform the computation in. + Default: if None, uses the global default dtype (see torch.get_default_dtype()) + when both :attr:`start` and :attr:`end` are real, + and corresponding complex dtype when either is complex. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + + Example:: + + >>> torch.linspace(3, 10, steps=5) + tensor([ 3.0000, 4.7500, 6.5000, 8.2500, 10.0000]) + >>> torch.linspace(-10, 10, steps=5) + tensor([-10., -5., 0., 5., 10.]) + >>> torch.linspace(start=-10, end=10, steps=5) + tensor([-10., -5., 0., 5., 10.]) + >>> torch.linspace(start=-10, end=10, steps=1) + tensor([-10.]) + """ + +@overload +def linspace( + start: Number | _complex, + end: Number | _complex, + steps: _int, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + linspace(start, end, steps, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Creates a one-dimensional tensor of size :attr:`steps` whose values are evenly + spaced from :attr:`start` to :attr:`end`, inclusive. That is, the value are: + + .. math:: + (\text{start}, + \text{start} + \frac{\text{end} - \text{start}}{\text{steps} - 1}, + \ldots, + \text{start} + (\text{steps} - 2) * \frac{\text{end} - \text{start}}{\text{steps} - 1}, + \text{end}) + + + From PyTorch 1.11 linspace requires the steps argument. Use steps=100 to restore the previous behavior. + + Args: + start (float or Tensor): the starting value for the set of points. If `Tensor`, it must be 0-dimensional + end (float or Tensor): the ending value for the set of points. If `Tensor`, it must be 0-dimensional + steps (int): size of the constructed tensor + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the data type to perform the computation in. + Default: if None, uses the global default dtype (see torch.get_default_dtype()) + when both :attr:`start` and :attr:`end` are real, + and corresponding complex dtype when either is complex. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + + Example:: + + >>> torch.linspace(3, 10, steps=5) + tensor([ 3.0000, 4.7500, 6.5000, 8.2500, 10.0000]) + >>> torch.linspace(-10, 10, steps=5) + tensor([-10., -5., 0., 5., 10.]) + >>> torch.linspace(start=-10, end=10, steps=5) + tensor([-10., -5., 0., 5., 10.]) + >>> torch.linspace(start=-10, end=10, steps=1) + tensor([-10.]) + """ + +def log(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + log(input, *, out=None) -> Tensor + + Returns a new tensor with the natural logarithm of the elements + of :attr:`input`. + + .. math:: + y_{i} = \log_{e} (x_{i}) + + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(5) * 5 + >>> a + tensor([4.7767, 4.3234, 1.2156, 0.2411, 4.5739]) + >>> torch.log(a) + tensor([ 1.5637, 1.4640, 0.1952, -1.4226, 1.5204]) + """ + +def log10(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + log10(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Returns a new tensor with the logarithm to the base 10 of the elements + of :attr:`input`. + + .. math:: + y_{i} = \log_{10} (x_{i}) + + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(5) + >>> a + tensor([ 0.5224, 0.9354, 0.7257, 0.1301, 0.2251]) + + + >>> torch.log10(a) + tensor([-0.2820, -0.0290, -0.1392, -0.8857, -0.6476]) + """ + +def log10_(input: Tensor) -> Tensor: ... +def log1p(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + log1p(input, *, out=None) -> Tensor + + Returns a new tensor with the natural logarithm of (1 + :attr:`input`). + + .. math:: + y_i = \log_{e} (x_i + 1) + + .. note:: This function is more accurate than :func:`torch.log` for small + values of :attr:`input` + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(5) + >>> a + tensor([-1.0090, -0.9923, 1.0249, -0.5372, 0.2492]) + >>> torch.log1p(a) + tensor([ nan, -4.8653, 0.7055, -0.7705, 0.2225]) + """ + +def log1p_(input: Tensor) -> Tensor: ... +def log2(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + log2(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Returns a new tensor with the logarithm to the base 2 of the elements + of :attr:`input`. + + .. math:: + y_{i} = \log_{2} (x_{i}) + + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.rand(5) + >>> a + tensor([ 0.8419, 0.8003, 0.9971, 0.5287, 0.0490]) + + + >>> torch.log2(a) + tensor([-0.2483, -0.3213, -0.0042, -0.9196, -4.3504]) + """ + +def log2_(input: Tensor) -> Tensor: ... +def log_(input: Tensor) -> Tensor: ... +@overload +def log_softmax( + input: Tensor, + dim: _int, + dtype: _dtype | None = None, + *, + out: Tensor | None = None, +) -> Tensor: ... +@overload +def log_softmax( + input: Tensor, + dim: str | EllipsisType | None, + *, + dtype: _dtype | None = None, +) -> Tensor: ... +def logaddexp( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + logaddexp(input, other, *, out=None) -> Tensor + + Logarithm of the sum of exponentiations of the inputs. + + Calculates pointwise :math:`\log\left(e^x + e^y\right)`. This function is useful + in statistics where the calculated probabilities of events may be so small as to + exceed the range of normal floating point numbers. In such cases the logarithm + of the calculated probability is stored. This function allows adding + probabilities stored in such a fashion. + + This op should be disambiguated with :func:`torch.logsumexp` which performs a + reduction on a single tensor. + + Args: + input (Tensor): the input tensor. + other (Tensor): the second input tensor + + Keyword arguments: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.logaddexp(torch.tensor([-1.0]), torch.tensor([-1.0, -2, -3])) + tensor([-0.3069, -0.6867, -0.8731]) + >>> torch.logaddexp(torch.tensor([-100.0, -200, -300]), torch.tensor([-1.0, -2, -3])) + tensor([-1., -2., -3.]) + >>> torch.logaddexp(torch.tensor([1.0, 2000, 30000]), torch.tensor([-1.0, -2, -3])) + tensor([1.1269e+00, 2.0000e+03, 3.0000e+04]) + """ + +def logaddexp2( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + logaddexp2(input, other, *, out=None) -> Tensor + + Logarithm of the sum of exponentiations of the inputs in base-2. + + Calculates pointwise :math:`\log_2\left(2^x + 2^y\right)`. See + :func:`torch.logaddexp` for more details. + + Args: + input (Tensor): the input tensor. + other (Tensor): the second input tensor + + Keyword arguments: + out (Tensor, optional): the output tensor. + """ + +@overload +def logcumsumexp( + input: Tensor, + dim: _int, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + logcumsumexp(input, dim, *, out=None) -> Tensor + Returns the logarithm of the cumulative summation of the exponentiation of + elements of :attr:`input` in the dimension :attr:`dim`. + + For summation index :math:`j` given by `dim` and other indices :math:`i`, the result is + + .. math:: + \text{logcumsumexp}(x)_{ij} = \log \sum\limits_{k=0}^{j} \exp(x_{ik}) + + Args: + input (Tensor): the input tensor. + dim (int): the dimension to do the operation over + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(10) + >>> torch.logcumsumexp(a, dim=0) + tensor([-0.42296738, -0.04462666, 0.86278635, 0.94622083, 1.05277811, + 1.39202815, 1.83525007, 1.84492621, 2.06084887, 2.06844475])) + """ + +@overload +def logcumsumexp( + input: Tensor, + dim: str | EllipsisType | None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + logcumsumexp(input, dim, *, out=None) -> Tensor + Returns the logarithm of the cumulative summation of the exponentiation of + elements of :attr:`input` in the dimension :attr:`dim`. + + For summation index :math:`j` given by `dim` and other indices :math:`i`, the result is + + .. math:: + \text{logcumsumexp}(x)_{ij} = \log \sum\limits_{k=0}^{j} \exp(x_{ik}) + + Args: + input (Tensor): the input tensor. + dim (int): the dimension to do the operation over + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(10) + >>> torch.logcumsumexp(a, dim=0) + tensor([-0.42296738, -0.04462666, 0.86278635, 0.94622083, 1.05277811, + 1.39202815, 1.83525007, 1.84492621, 2.06084887, 2.06844475])) + """ + +def logdet(input: Tensor) -> Tensor: + r""" + logdet(input) -> Tensor + + Calculates log determinant of a square matrix or batches of square matrices. + + It returns ``-inf`` if the input has a determinant of zero, and ``NaN`` if it has + a negative determinant. + + .. note:: + Backward through :meth:`logdet` internally uses SVD results when :attr:`input` + is not invertible. In this case, double backward through :meth:`logdet` will + be unstable in when :attr:`input` doesn't have distinct singular values. See + :func:`torch.linalg.svd` for details. + + .. seealso:: + + :func:`torch.linalg.slogdet` computes the sign (resp. angle) and natural logarithm of the + absolute value of the determinant of real-valued (resp. complex) square matrices. + + Arguments: + input (Tensor): the input tensor of size ``(*, n, n)`` where ``*`` is zero or more + batch dimensions. + + Example:: + + >>> A = torch.randn(3, 3) + >>> torch.det(A) + tensor(0.2611) + >>> torch.logdet(A) + tensor(-1.3430) + >>> A + tensor([[[ 0.9254, -0.6213], + [-0.5787, 1.6843]], + + [[ 0.3242, -0.9665], + [ 0.4539, -0.0887]], + + [[ 1.1336, -0.4025], + [-0.7089, 0.9032]]]) + >>> A.det() + tensor([1.1990, 0.4099, 0.7386]) + >>> A.det().log() + tensor([ 0.1815, -0.8917, -0.3031]) + """ + +def logical_and( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + logical_and(input, other, *, out=None) -> Tensor + + Computes the element-wise logical AND of the given input tensors. Zeros are treated as ``False`` and nonzeros are + treated as ``True``. + + Args: + input (Tensor): the input tensor. + other (Tensor): the tensor to compute AND with + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.logical_and(torch.tensor([True, False, True]), torch.tensor([True, False, False])) + tensor([ True, False, False]) + >>> a = torch.tensor([0, 1, 10, 0], dtype=torch.int8) + >>> b = torch.tensor([4, 0, 1, 0], dtype=torch.int8) + >>> torch.logical_and(a, b) + tensor([False, False, True, False]) + >>> torch.logical_and(a.double(), b.double()) + tensor([False, False, True, False]) + >>> torch.logical_and(a.double(), b) + tensor([False, False, True, False]) + >>> torch.logical_and(a, b, out=torch.empty(4, dtype=torch.bool)) + tensor([False, False, True, False]) + """ + +def logical_not(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + logical_not(input, *, out=None) -> Tensor + + Computes the element-wise logical NOT of the given input tensor. If not specified, the output tensor will have the bool + dtype. If the input tensor is not a bool tensor, zeros are treated as ``False`` and non-zeros are treated as ``True``. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.logical_not(torch.tensor([True, False])) + tensor([False, True]) + >>> torch.logical_not(torch.tensor([0, 1, -10], dtype=torch.int8)) + tensor([ True, False, False]) + >>> torch.logical_not(torch.tensor([0., 1.5, -10.], dtype=torch.double)) + tensor([ True, False, False]) + >>> torch.logical_not(torch.tensor([0., 1., -10.], dtype=torch.double), out=torch.empty(3, dtype=torch.int16)) + tensor([1, 0, 0], dtype=torch.int16) + """ + +def logical_or( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + logical_or(input, other, *, out=None) -> Tensor + + Computes the element-wise logical OR of the given input tensors. Zeros are treated as ``False`` and nonzeros are + treated as ``True``. + + Args: + input (Tensor): the input tensor. + other (Tensor): the tensor to compute OR with + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.logical_or(torch.tensor([True, False, True]), torch.tensor([True, False, False])) + tensor([ True, False, True]) + >>> a = torch.tensor([0, 1, 10, 0], dtype=torch.int8) + >>> b = torch.tensor([4, 0, 1, 0], dtype=torch.int8) + >>> torch.logical_or(a, b) + tensor([ True, True, True, False]) + >>> torch.logical_or(a.double(), b.double()) + tensor([ True, True, True, False]) + >>> torch.logical_or(a.double(), b) + tensor([ True, True, True, False]) + >>> torch.logical_or(a, b, out=torch.empty(4, dtype=torch.bool)) + tensor([ True, True, True, False]) + """ + +def logical_xor( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + logical_xor(input: Tensor, other: Tensor, *, out: Optional[Tensor]) -> Tensor + + Computes the element-wise logical XOR of the given input tensors. Zeros are treated as ``False`` and nonzeros are + treated as ``True``. + + Args: + input (Tensor): the input tensor. + other (Tensor): the tensor to compute XOR with + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.logical_xor(torch.tensor([True, False, True]), torch.tensor([True, False, False])) + tensor([False, False, True]) + >>> a = torch.tensor([0, 1, 10, 0], dtype=torch.int8) + >>> b = torch.tensor([4, 0, 1, 0], dtype=torch.int8) + >>> torch.logical_xor(a, b) + tensor([ True, True, False, False]) + >>> torch.logical_xor(a.double(), b.double()) + tensor([ True, True, False, False]) + >>> torch.logical_xor(a.double(), b) + tensor([ True, True, False, False]) + >>> torch.logical_xor(a, b, out=torch.empty(4, dtype=torch.bool)) + tensor([ True, True, False, False]) + """ + +def logit( + input: Tensor, + eps: _float | None = None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + logit(input, eps=None, *, out=None) -> Tensor + + Alias for :func:`torch.special.logit`. + """ + +def logit_(input: Tensor, eps: _float | None = None) -> Tensor: ... +@overload +def logspace( + start: Number, + end: Number, + steps: _int | None = None, + base: _float = 10.0, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + pin_memory: _bool = False, +) -> Tensor: + r""" + logspace(start, end, steps, base=10.0, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + + Creates a one-dimensional tensor of size :attr:`steps` whose values are evenly + spaced from :math:`{{\text{{base}}}}^{{\text{{start}}}}` to + :math:`{{\text{{base}}}}^{{\text{{end}}}}`, inclusive, on a logarithmic scale + with base :attr:`base`. That is, the values are: + + .. math:: + (\text{base}^{\text{start}}, + \text{base}^{(\text{start} + \frac{\text{end} - \text{start}}{ \text{steps} - 1})}, + \ldots, + \text{base}^{(\text{start} + (\text{steps} - 2) * \frac{\text{end} - \text{start}}{ \text{steps} - 1})}, + \text{base}^{\text{end}}) + + + + From PyTorch 1.11 logspace requires the steps argument. Use steps=100 to restore the previous behavior. + + Args: + start (float or Tensor): the starting value for the set of points. If `Tensor`, it must be 0-dimensional + end (float or Tensor): the ending value for the set of points. If `Tensor`, it must be 0-dimensional + steps (int): size of the constructed tensor + base (float, optional): base of the logarithm function. Default: ``10.0``. + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the data type to perform the computation in. + Default: if None, uses the global default dtype (see torch.get_default_dtype()) + when both :attr:`start` and :attr:`end` are real, + and corresponding complex dtype when either is complex. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.logspace(start=-10, end=10, steps=5) + tensor([ 1.0000e-10, 1.0000e-05, 1.0000e+00, 1.0000e+05, 1.0000e+10]) + >>> torch.logspace(start=0.1, end=1.0, steps=5) + tensor([ 1.2589, 2.1135, 3.5481, 5.9566, 10.0000]) + >>> torch.logspace(start=0.1, end=1.0, steps=1) + tensor([1.2589]) + >>> torch.logspace(start=2, end=2, steps=1, base=2) + tensor([4.0]) + """ + +@overload +def logspace( + start: Tensor, + end: Tensor, + steps: _int, + base: _float = 10.0, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + logspace(start, end, steps, base=10.0, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + + Creates a one-dimensional tensor of size :attr:`steps` whose values are evenly + spaced from :math:`{{\text{{base}}}}^{{\text{{start}}}}` to + :math:`{{\text{{base}}}}^{{\text{{end}}}}`, inclusive, on a logarithmic scale + with base :attr:`base`. That is, the values are: + + .. math:: + (\text{base}^{\text{start}}, + \text{base}^{(\text{start} + \frac{\text{end} - \text{start}}{ \text{steps} - 1})}, + \ldots, + \text{base}^{(\text{start} + (\text{steps} - 2) * \frac{\text{end} - \text{start}}{ \text{steps} - 1})}, + \text{base}^{\text{end}}) + + + + From PyTorch 1.11 logspace requires the steps argument. Use steps=100 to restore the previous behavior. + + Args: + start (float or Tensor): the starting value for the set of points. If `Tensor`, it must be 0-dimensional + end (float or Tensor): the ending value for the set of points. If `Tensor`, it must be 0-dimensional + steps (int): size of the constructed tensor + base (float, optional): base of the logarithm function. Default: ``10.0``. + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the data type to perform the computation in. + Default: if None, uses the global default dtype (see torch.get_default_dtype()) + when both :attr:`start` and :attr:`end` are real, + and corresponding complex dtype when either is complex. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.logspace(start=-10, end=10, steps=5) + tensor([ 1.0000e-10, 1.0000e-05, 1.0000e+00, 1.0000e+05, 1.0000e+10]) + >>> torch.logspace(start=0.1, end=1.0, steps=5) + tensor([ 1.2589, 2.1135, 3.5481, 5.9566, 10.0000]) + >>> torch.logspace(start=0.1, end=1.0, steps=1) + tensor([1.2589]) + >>> torch.logspace(start=2, end=2, steps=1, base=2) + tensor([4.0]) + """ + +@overload +def logspace( + start: Number | _complex, + end: Tensor, + steps: _int, + base: _float = 10.0, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + logspace(start, end, steps, base=10.0, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + + Creates a one-dimensional tensor of size :attr:`steps` whose values are evenly + spaced from :math:`{{\text{{base}}}}^{{\text{{start}}}}` to + :math:`{{\text{{base}}}}^{{\text{{end}}}}`, inclusive, on a logarithmic scale + with base :attr:`base`. That is, the values are: + + .. math:: + (\text{base}^{\text{start}}, + \text{base}^{(\text{start} + \frac{\text{end} - \text{start}}{ \text{steps} - 1})}, + \ldots, + \text{base}^{(\text{start} + (\text{steps} - 2) * \frac{\text{end} - \text{start}}{ \text{steps} - 1})}, + \text{base}^{\text{end}}) + + + + From PyTorch 1.11 logspace requires the steps argument. Use steps=100 to restore the previous behavior. + + Args: + start (float or Tensor): the starting value for the set of points. If `Tensor`, it must be 0-dimensional + end (float or Tensor): the ending value for the set of points. If `Tensor`, it must be 0-dimensional + steps (int): size of the constructed tensor + base (float, optional): base of the logarithm function. Default: ``10.0``. + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the data type to perform the computation in. + Default: if None, uses the global default dtype (see torch.get_default_dtype()) + when both :attr:`start` and :attr:`end` are real, + and corresponding complex dtype when either is complex. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.logspace(start=-10, end=10, steps=5) + tensor([ 1.0000e-10, 1.0000e-05, 1.0000e+00, 1.0000e+05, 1.0000e+10]) + >>> torch.logspace(start=0.1, end=1.0, steps=5) + tensor([ 1.2589, 2.1135, 3.5481, 5.9566, 10.0000]) + >>> torch.logspace(start=0.1, end=1.0, steps=1) + tensor([1.2589]) + >>> torch.logspace(start=2, end=2, steps=1, base=2) + tensor([4.0]) + """ + +@overload +def logspace( + start: Tensor, + end: Number | _complex, + steps: _int, + base: _float = 10.0, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + logspace(start, end, steps, base=10.0, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + + Creates a one-dimensional tensor of size :attr:`steps` whose values are evenly + spaced from :math:`{{\text{{base}}}}^{{\text{{start}}}}` to + :math:`{{\text{{base}}}}^{{\text{{end}}}}`, inclusive, on a logarithmic scale + with base :attr:`base`. That is, the values are: + + .. math:: + (\text{base}^{\text{start}}, + \text{base}^{(\text{start} + \frac{\text{end} - \text{start}}{ \text{steps} - 1})}, + \ldots, + \text{base}^{(\text{start} + (\text{steps} - 2) * \frac{\text{end} - \text{start}}{ \text{steps} - 1})}, + \text{base}^{\text{end}}) + + + + From PyTorch 1.11 logspace requires the steps argument. Use steps=100 to restore the previous behavior. + + Args: + start (float or Tensor): the starting value for the set of points. If `Tensor`, it must be 0-dimensional + end (float or Tensor): the ending value for the set of points. If `Tensor`, it must be 0-dimensional + steps (int): size of the constructed tensor + base (float, optional): base of the logarithm function. Default: ``10.0``. + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the data type to perform the computation in. + Default: if None, uses the global default dtype (see torch.get_default_dtype()) + when both :attr:`start` and :attr:`end` are real, + and corresponding complex dtype when either is complex. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.logspace(start=-10, end=10, steps=5) + tensor([ 1.0000e-10, 1.0000e-05, 1.0000e+00, 1.0000e+05, 1.0000e+10]) + >>> torch.logspace(start=0.1, end=1.0, steps=5) + tensor([ 1.2589, 2.1135, 3.5481, 5.9566, 10.0000]) + >>> torch.logspace(start=0.1, end=1.0, steps=1) + tensor([1.2589]) + >>> torch.logspace(start=2, end=2, steps=1, base=2) + tensor([4.0]) + """ + +@overload +def logspace( + start: Number | _complex, + end: Number | _complex, + steps: _int, + base: _float = 10.0, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + logspace(start, end, steps, base=10.0, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + + Creates a one-dimensional tensor of size :attr:`steps` whose values are evenly + spaced from :math:`{{\text{{base}}}}^{{\text{{start}}}}` to + :math:`{{\text{{base}}}}^{{\text{{end}}}}`, inclusive, on a logarithmic scale + with base :attr:`base`. That is, the values are: + + .. math:: + (\text{base}^{\text{start}}, + \text{base}^{(\text{start} + \frac{\text{end} - \text{start}}{ \text{steps} - 1})}, + \ldots, + \text{base}^{(\text{start} + (\text{steps} - 2) * \frac{\text{end} - \text{start}}{ \text{steps} - 1})}, + \text{base}^{\text{end}}) + + + + From PyTorch 1.11 logspace requires the steps argument. Use steps=100 to restore the previous behavior. + + Args: + start (float or Tensor): the starting value for the set of points. If `Tensor`, it must be 0-dimensional + end (float or Tensor): the ending value for the set of points. If `Tensor`, it must be 0-dimensional + steps (int): size of the constructed tensor + base (float, optional): base of the logarithm function. Default: ``10.0``. + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the data type to perform the computation in. + Default: if None, uses the global default dtype (see torch.get_default_dtype()) + when both :attr:`start` and :attr:`end` are real, + and corresponding complex dtype when either is complex. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.logspace(start=-10, end=10, steps=5) + tensor([ 1.0000e-10, 1.0000e-05, 1.0000e+00, 1.0000e+05, 1.0000e+10]) + >>> torch.logspace(start=0.1, end=1.0, steps=5) + tensor([ 1.2589, 2.1135, 3.5481, 5.9566, 10.0000]) + >>> torch.logspace(start=0.1, end=1.0, steps=1) + tensor([1.2589]) + >>> torch.logspace(start=2, end=2, steps=1, base=2) + tensor([4.0]) + """ + +@overload +def logsumexp( + input: Tensor, + dim: _int | _size, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + logsumexp(input, dim, keepdim=False, *, out=None) + + Returns the log of summed exponentials of each row of the :attr:`input` + tensor in the given dimension :attr:`dim`. The computation is numerically + stabilized. + + For summation index :math:`j` given by `dim` and other indices :math:`i`, the result is + + .. math:: + \text{logsumexp}(x)_{i} = \log \sum_j \exp(x_{ij}) + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + dim (int or tuple of ints): the dimension or dimensions to reduce. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(3, 3) + >>> torch.logsumexp(a, 1) + tensor([1.4907, 1.0593, 1.5696]) + >>> torch.dist(torch.logsumexp(a, 1), torch.log(torch.sum(torch.exp(a), 1))) + tensor(1.6859e-07) + """ + +@overload +def logsumexp( + input: Tensor, + dim: Sequence[str | EllipsisType | None], + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + logsumexp(input, dim, keepdim=False, *, out=None) + + Returns the log of summed exponentials of each row of the :attr:`input` + tensor in the given dimension :attr:`dim`. The computation is numerically + stabilized. + + For summation index :math:`j` given by `dim` and other indices :math:`i`, the result is + + .. math:: + \text{logsumexp}(x)_{i} = \log \sum_j \exp(x_{ij}) + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + dim (int or tuple of ints): the dimension or dimensions to reduce. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(3, 3) + >>> torch.logsumexp(a, 1) + tensor([1.4907, 1.0593, 1.5696]) + >>> torch.dist(torch.logsumexp(a, 1), torch.log(torch.sum(torch.exp(a), 1))) + tensor(1.6859e-07) + """ + +@overload +def lstm( + data: Tensor, + batch_sizes: Tensor, + hx: tuple[Tensor, ...] | list[Tensor] | None, + params: tuple[Tensor, ...] | list[Tensor] | None, + has_biases: _bool, + num_layers: _int, + dropout: _float, + train: _bool, + bidirectional: _bool, +) -> tuple[Tensor, Tensor, Tensor]: ... +@overload +def lstm( + input: Tensor, + hx: tuple[Tensor, ...] | list[Tensor] | None, + params: tuple[Tensor, ...] | list[Tensor] | None, + has_biases: _bool, + num_layers: _int, + dropout: _float, + train: _bool, + bidirectional: _bool, + batch_first: _bool, +) -> tuple[Tensor, Tensor, Tensor]: ... +def lstm_cell( + input: Tensor, + hx: tuple[Tensor, ...] | list[Tensor] | None, + w_ih: Tensor, + w_hh: Tensor, + b_ih: Tensor | None = None, + b_hh: Tensor | None = None, +) -> tuple[Tensor, Tensor]: ... +@overload +def lt( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + lt(input, other, *, out=None) -> Tensor + + Computes :math:`\text{input} < \text{other}` element-wise. + + + The second argument can be a number or a tensor whose shape is + :ref:`broadcastable ` with the first argument. + + Args: + input (Tensor): the tensor to compare + other (Tensor or float): the tensor or value to compare + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is less than :attr:`other` and False elsewhere + + Example:: + + >>> torch.lt(torch.tensor([[1, 2], [3, 4]]), torch.tensor([[1, 1], [4, 4]])) + tensor([[False, False], [True, False]]) + """ + +@overload +def lt( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + lt(input, other, *, out=None) -> Tensor + + Computes :math:`\text{input} < \text{other}` element-wise. + + + The second argument can be a number or a tensor whose shape is + :ref:`broadcastable ` with the first argument. + + Args: + input (Tensor): the tensor to compare + other (Tensor or float): the tensor or value to compare + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is less than :attr:`other` and False elsewhere + + Example:: + + >>> torch.lt(torch.tensor([[1, 2], [3, 4]]), torch.tensor([[1, 1], [4, 4]])) + tensor([[False, False], [True, False]]) + """ + +def lu_solve( + input: Tensor, + LU_data: Tensor, + LU_pivots: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + lu_solve(b, LU_data, LU_pivots, *, out=None) -> Tensor + + Returns the LU solve of the linear system :math:`Ax = b` using the partially pivoted + LU factorization of A from :func:`~linalg.lu_factor`. + + This function supports ``float``, ``double``, ``cfloat`` and ``cdouble`` dtypes for :attr:`input`. + + .. warning:: + + :func:`torch.lu_solve` is deprecated in favor of :func:`torch.linalg.lu_solve`. + :func:`torch.lu_solve` will be removed in a future PyTorch release. + ``X = torch.lu_solve(B, LU, pivots)`` should be replaced with + + .. code:: python + + X = linalg.lu_solve(LU, pivots, B) + + Arguments: + b (Tensor): the RHS tensor of size :math:`(*, m, k)`, where :math:`*` + is zero or more batch dimensions. + LU_data (Tensor): the pivoted LU factorization of A from :meth:`~linalg.lu_factor` of size :math:`(*, m, m)`, + where :math:`*` is zero or more batch dimensions. + LU_pivots (IntTensor): the pivots of the LU factorization from :meth:`~linalg.lu_factor` of size :math:`(*, m)`, + where :math:`*` is zero or more batch dimensions. + The batch dimensions of :attr:`LU_pivots` must be equal to the batch dimensions of + :attr:`LU_data`. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> A = torch.randn(2, 3, 3) + >>> b = torch.randn(2, 3, 1) + >>> LU, pivots = torch.linalg.lu_factor(A) + >>> x = torch.lu_solve(b, LU, pivots) + >>> torch.dist(A @ x, b) + tensor(1.00000e-07 * + 2.8312) + """ + +def lu_unpack( + LU_data: Tensor, + LU_pivots: Tensor, + unpack_data: _bool = True, + unpack_pivots: _bool = True, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.lu_unpack: + r""" + lu_unpack(LU_data, LU_pivots, unpack_data=True, unpack_pivots=True, *, out=None) -> (Tensor, Tensor, Tensor) + + Unpacks the LU decomposition returned by :func:`~linalg.lu_factor` into the `P, L, U` matrices. + + .. seealso:: + + :func:`~linalg.lu` returns the matrices from the LU decomposition. Its gradient formula is more efficient + than that of doing :func:`~linalg.lu_factor` followed by :func:`~linalg.lu_unpack`. + + Args: + LU_data (Tensor): the packed LU factorization data + LU_pivots (Tensor): the packed LU factorization pivots + unpack_data (bool): flag indicating if the data should be unpacked. + If ``False``, then the returned ``L`` and ``U`` are empty tensors. + Default: ``True`` + unpack_pivots (bool): flag indicating if the pivots should be unpacked into a permutation matrix ``P``. + If ``False``, then the returned ``P`` is an empty tensor. + Default: ``True`` + + Keyword args: + out (tuple, optional): output tuple of three tensors. Ignored if `None`. + + Returns: + A namedtuple ``(P, L, U)`` + + Examples:: + + >>> A = torch.randn(2, 3, 3) + >>> LU, pivots = torch.linalg.lu_factor(A) + >>> P, L, U = torch.lu_unpack(LU, pivots) + >>> # We can recover A from the factorization + >>> A_ = P @ L @ U + >>> torch.allclose(A, A_) + True + + >>> # LU factorization of a rectangular matrix: + >>> A = torch.randn(2, 3, 2) + >>> LU, pivots = torch.linalg.lu_factor(A) + >>> P, L, U = torch.lu_unpack(LU, pivots) + >>> # P, L, U are the same as returned by linalg.lu + >>> P_, L_, U_ = torch.linalg.lu(A) + >>> torch.allclose(P, P_) and torch.allclose(L, L_) and torch.allclose(U, U_) + True + """ + +def margin_ranking_loss( + input1: Tensor, + input2: Tensor, + target: Tensor, + margin: _float = 0.0, + reduction: _int = 1, +) -> Tensor: ... +@overload +def masked_fill(input: Tensor, mask: Tensor, value: Tensor) -> Tensor: ... +@overload +def masked_fill( + input: Tensor, + mask: Tensor, + value: Number | _complex, +) -> Tensor: ... +def masked_scatter(input: Tensor, mask: Tensor, source: Tensor) -> Tensor: ... +def masked_select( + input: Tensor, + mask: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + masked_select(input, mask, *, out=None) -> Tensor + + Returns a new 1-D tensor which indexes the :attr:`input` tensor according to + the boolean mask :attr:`mask` which is a `BoolTensor`. + + The shapes of the :attr:`mask` tensor and the :attr:`input` tensor don't need + to match, but they must be :ref:`broadcastable `. + + .. note:: The returned tensor does **not** use the same storage + as the original tensor + + Args: + input (Tensor): the input tensor. + mask (BoolTensor): the tensor containing the binary mask to index with + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> x = torch.randn(3, 4) + >>> x + tensor([[ 0.3552, -2.3825, -0.8297, 0.3477], + [-1.2035, 1.2252, 0.5002, 0.6248], + [ 0.1307, -2.0608, 0.1244, 2.0139]]) + >>> mask = x.ge(0.5) + >>> mask + tensor([[False, False, False, False], + [False, True, True, True], + [False, False, False, True]]) + >>> torch.masked_select(x, mask) + tensor([ 1.2252, 0.5002, 0.6248, 2.0139]) + """ + +def matmul( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + matmul(input, other, *, out=None) -> Tensor + + Matrix product of two tensors. + + The behavior depends on the dimensionality of the tensors as follows: + + - If both tensors are 1-dimensional, the dot product (scalar) is returned. + - If both arguments are 2-dimensional, the matrix-matrix product is returned. + - If the first argument is 1-dimensional and the second argument is 2-dimensional, + a 1 is prepended to its dimension for the purpose of the matrix multiply. + After the matrix multiply, the prepended dimension is removed. + - If the first argument is 2-dimensional and the second argument is 1-dimensional, + the matrix-vector product is returned. + - If both arguments are at least 1-dimensional and at least one argument is + N-dimensional (where N > 2), then a batched matrix multiply is returned. If the first + argument is 1-dimensional, a 1 is prepended to its dimension for the purpose of the + batched matrix multiply and removed after. If the second argument is 1-dimensional, a + 1 is appended to its dimension for the purpose of the batched matrix multiply and removed after. + + The first N-2 dimensions of each argument, the batch dimensions, are + :ref:`broadcast ` (and thus must be broadcastable). + The last 2, the matrix dimensions, are handled as in the matrix-matrix product. + + For example, if :attr:`input` is a + :math:`(j \times 1 \times n \times m)` tensor and :attr:`other` is a :math:`(k \times m \times p)` + tensor, the batch dimensions are :math:`(j \times 1)` and :math:`(k)`, + and the matrix dimensions are :math:`(n \times m)` and :math:`(m \times p)`. + :attr:`out` will be a :math:`(j \times k \times n \times p)` tensor. + + This operation has support for arguments with :ref:`sparse layouts`. In particular the + matrix-matrix (both arguments 2-dimensional) supports sparse arguments with the same restrictions + as :func:`torch.mm` + + + .. warning:: + Sparse support is a beta feature and some layout(s)/dtype/device combinations may not be supported, + or may not have autograd support. If you notice missing functionality please + open a feature request. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + .. note:: + + The 1-dimensional dot product version of this function does not support an :attr:`out` parameter. + + Arguments: + input (Tensor): the first tensor to be multiplied + other (Tensor): the second tensor to be multiplied + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> # vector x vector + >>> tensor1 = torch.randn(3) + >>> tensor2 = torch.randn(3) + >>> torch.matmul(tensor1, tensor2).size() + torch.Size([]) + >>> # matrix x vector + >>> tensor1 = torch.randn(3, 4) + >>> tensor2 = torch.randn(4) + >>> torch.matmul(tensor1, tensor2).size() + torch.Size([3]) + >>> # batched matrix x broadcasted vector + >>> tensor1 = torch.randn(10, 3, 4) + >>> tensor2 = torch.randn(4) + >>> torch.matmul(tensor1, tensor2).size() + torch.Size([10, 3]) + >>> # batched matrix x batched matrix + >>> tensor1 = torch.randn(10, 3, 4) + >>> tensor2 = torch.randn(10, 4, 5) + >>> torch.matmul(tensor1, tensor2).size() + torch.Size([10, 3, 5]) + >>> # batched matrix x broadcasted matrix + >>> tensor1 = torch.randn(10, 3, 4) + >>> tensor2 = torch.randn(4, 5) + >>> torch.matmul(tensor1, tensor2).size() + torch.Size([10, 3, 5]) + """ + +def matrix_exp(input: Tensor) -> Tensor: + r""" + matrix_exp(A) -> Tensor + + Alias for :func:`torch.linalg.matrix_exp`. + """ + +def matrix_power( + input: Tensor, + n: _int, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + matrix_power(input, n, *, out=None) -> Tensor + + Alias for :func:`torch.linalg.matrix_power` + """ + +@overload +def max(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + max(input, *, out=None) -> Tensor + + Returns the maximum value of all elements in the ``input`` tensor. + + .. note:: + The difference between ``max``/``min`` and ``amax``/``amin`` is: + - ``amax``/``amin`` supports reducing on multiple dimensions, + - ``amax``/``amin`` does not return indices. + + Both ``amax``/``amin`` evenly distribute gradients between equal values + when there are multiple input elements with the same minimum or maximum value. + + For ``max``/``min``: + - If reduce over all dimensions(no dim specified), gradients evenly distribute between equally ``max``/``min`` values. + - If reduce over one specified axis, only propagate to the indexed element. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.6763, 0.7445, -2.2369]]) + >>> torch.max(a) + tensor(0.7445) + + .. function:: max(input, dim, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` is the maximum + value of each row of the :attr:`input` tensor in the given dimension + :attr:`dim`. And ``indices`` is the index location of each maximum value found + (argmax). + + If ``keepdim`` is ``True``, the output tensors are of the same size + as ``input`` except in the dimension ``dim`` where they are of size 1. + Otherwise, ``dim`` is squeezed (see :func:`torch.squeeze`), resulting + in the output tensors having 1 fewer dimension than ``input``. + + .. note:: If there are multiple maximal values in a reduced row then + the indices of the first maximal value are returned. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. If omitted, all dimensions are reduced. Explicit ``None`` is not supported. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (tuple, optional): the result tuple of two output tensors (max, max_indices) + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[-1.2360, -0.2942, -0.1222, 0.8475], + [ 1.1949, -1.1127, -2.2379, -0.6702], + [ 1.5717, -0.9207, 0.1297, -1.8768], + [-0.6172, 1.0036, -0.6060, -0.2432]]) + >>> torch.max(a, 1) + torch.return_types.max(values=tensor([0.8475, 1.1949, 1.5717, 1.0036]), indices=tensor([3, 0, 0, 1])) + >>> a = torch.tensor([[1.0, 2.0], [3.0, 4.0]]) + >>> a.max(dim=1, keepdim=True) + torch.return_types.max( + values=tensor([[2.], [4.]]), + indices=tensor([[1], [1]])) + >>> a.max(dim=1, keepdim=False) + torch.return_types.max( + values=tensor([2., 4.]), + indices=tensor([1, 1])) + + .. function:: max(input, other, *, out=None) -> Tensor + :noindex: + + See :func:`torch.maximum`. + """ + +@overload +def max( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + max(input, *, out=None) -> Tensor + + Returns the maximum value of all elements in the ``input`` tensor. + + .. note:: + The difference between ``max``/``min`` and ``amax``/``amin`` is: + - ``amax``/``amin`` supports reducing on multiple dimensions, + - ``amax``/``amin`` does not return indices. + + Both ``amax``/``amin`` evenly distribute gradients between equal values + when there are multiple input elements with the same minimum or maximum value. + + For ``max``/``min``: + - If reduce over all dimensions(no dim specified), gradients evenly distribute between equally ``max``/``min`` values. + - If reduce over one specified axis, only propagate to the indexed element. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.6763, 0.7445, -2.2369]]) + >>> torch.max(a) + tensor(0.7445) + + .. function:: max(input, dim, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` is the maximum + value of each row of the :attr:`input` tensor in the given dimension + :attr:`dim`. And ``indices`` is the index location of each maximum value found + (argmax). + + If ``keepdim`` is ``True``, the output tensors are of the same size + as ``input`` except in the dimension ``dim`` where they are of size 1. + Otherwise, ``dim`` is squeezed (see :func:`torch.squeeze`), resulting + in the output tensors having 1 fewer dimension than ``input``. + + .. note:: If there are multiple maximal values in a reduced row then + the indices of the first maximal value are returned. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. If omitted, all dimensions are reduced. Explicit ``None`` is not supported. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (tuple, optional): the result tuple of two output tensors (max, max_indices) + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[-1.2360, -0.2942, -0.1222, 0.8475], + [ 1.1949, -1.1127, -2.2379, -0.6702], + [ 1.5717, -0.9207, 0.1297, -1.8768], + [-0.6172, 1.0036, -0.6060, -0.2432]]) + >>> torch.max(a, 1) + torch.return_types.max(values=tensor([0.8475, 1.1949, 1.5717, 1.0036]), indices=tensor([3, 0, 0, 1])) + >>> a = torch.tensor([[1.0, 2.0], [3.0, 4.0]]) + >>> a.max(dim=1, keepdim=True) + torch.return_types.max( + values=tensor([[2.], [4.]]), + indices=tensor([[1], [1]])) + >>> a.max(dim=1, keepdim=False) + torch.return_types.max( + values=tensor([2., 4.]), + indices=tensor([1, 1])) + + .. function:: max(input, other, *, out=None) -> Tensor + :noindex: + + See :func:`torch.maximum`. + """ + +@overload +def max( + input: Tensor, + dim: _int, + keepdim: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.max: + r""" + max(input, *, out=None) -> Tensor + + Returns the maximum value of all elements in the ``input`` tensor. + + .. note:: + The difference between ``max``/``min`` and ``amax``/``amin`` is: + - ``amax``/``amin`` supports reducing on multiple dimensions, + - ``amax``/``amin`` does not return indices. + + Both ``amax``/``amin`` evenly distribute gradients between equal values + when there are multiple input elements with the same minimum or maximum value. + + For ``max``/``min``: + - If reduce over all dimensions(no dim specified), gradients evenly distribute between equally ``max``/``min`` values. + - If reduce over one specified axis, only propagate to the indexed element. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.6763, 0.7445, -2.2369]]) + >>> torch.max(a) + tensor(0.7445) + + .. function:: max(input, dim, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` is the maximum + value of each row of the :attr:`input` tensor in the given dimension + :attr:`dim`. And ``indices`` is the index location of each maximum value found + (argmax). + + If ``keepdim`` is ``True``, the output tensors are of the same size + as ``input`` except in the dimension ``dim`` where they are of size 1. + Otherwise, ``dim`` is squeezed (see :func:`torch.squeeze`), resulting + in the output tensors having 1 fewer dimension than ``input``. + + .. note:: If there are multiple maximal values in a reduced row then + the indices of the first maximal value are returned. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. If omitted, all dimensions are reduced. Explicit ``None`` is not supported. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (tuple, optional): the result tuple of two output tensors (max, max_indices) + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[-1.2360, -0.2942, -0.1222, 0.8475], + [ 1.1949, -1.1127, -2.2379, -0.6702], + [ 1.5717, -0.9207, 0.1297, -1.8768], + [-0.6172, 1.0036, -0.6060, -0.2432]]) + >>> torch.max(a, 1) + torch.return_types.max(values=tensor([0.8475, 1.1949, 1.5717, 1.0036]), indices=tensor([3, 0, 0, 1])) + >>> a = torch.tensor([[1.0, 2.0], [3.0, 4.0]]) + >>> a.max(dim=1, keepdim=True) + torch.return_types.max( + values=tensor([[2.], [4.]]), + indices=tensor([[1], [1]])) + >>> a.max(dim=1, keepdim=False) + torch.return_types.max( + values=tensor([2., 4.]), + indices=tensor([1, 1])) + + .. function:: max(input, other, *, out=None) -> Tensor + :noindex: + + See :func:`torch.maximum`. + """ + +@overload +def max( + input: Tensor, + dim: str | EllipsisType | None, + keepdim: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.max: + r""" + max(input, *, out=None) -> Tensor + + Returns the maximum value of all elements in the ``input`` tensor. + + .. note:: + The difference between ``max``/``min`` and ``amax``/``amin`` is: + - ``amax``/``amin`` supports reducing on multiple dimensions, + - ``amax``/``amin`` does not return indices. + + Both ``amax``/``amin`` evenly distribute gradients between equal values + when there are multiple input elements with the same minimum or maximum value. + + For ``max``/``min``: + - If reduce over all dimensions(no dim specified), gradients evenly distribute between equally ``max``/``min`` values. + - If reduce over one specified axis, only propagate to the indexed element. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.6763, 0.7445, -2.2369]]) + >>> torch.max(a) + tensor(0.7445) + + .. function:: max(input, dim, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` is the maximum + value of each row of the :attr:`input` tensor in the given dimension + :attr:`dim`. And ``indices`` is the index location of each maximum value found + (argmax). + + If ``keepdim`` is ``True``, the output tensors are of the same size + as ``input`` except in the dimension ``dim`` where they are of size 1. + Otherwise, ``dim`` is squeezed (see :func:`torch.squeeze`), resulting + in the output tensors having 1 fewer dimension than ``input``. + + .. note:: If there are multiple maximal values in a reduced row then + the indices of the first maximal value are returned. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. If omitted, all dimensions are reduced. Explicit ``None`` is not supported. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (tuple, optional): the result tuple of two output tensors (max, max_indices) + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[-1.2360, -0.2942, -0.1222, 0.8475], + [ 1.1949, -1.1127, -2.2379, -0.6702], + [ 1.5717, -0.9207, 0.1297, -1.8768], + [-0.6172, 1.0036, -0.6060, -0.2432]]) + >>> torch.max(a, 1) + torch.return_types.max(values=tensor([0.8475, 1.1949, 1.5717, 1.0036]), indices=tensor([3, 0, 0, 1])) + >>> a = torch.tensor([[1.0, 2.0], [3.0, 4.0]]) + >>> a.max(dim=1, keepdim=True) + torch.return_types.max( + values=tensor([[2.], [4.]]), + indices=tensor([[1], [1]])) + >>> a.max(dim=1, keepdim=False) + torch.return_types.max( + values=tensor([2., 4.]), + indices=tensor([1, 1])) + + .. function:: max(input, other, *, out=None) -> Tensor + :noindex: + + See :func:`torch.maximum`. + """ + +def max_pool1d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size = (), + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: _bool = False, +) -> Tensor: ... +def max_pool1d_with_indices( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size = (), + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: _bool = False, +) -> tuple[Tensor, Tensor]: ... +def max_pool2d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size = (), + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: _bool = False, +) -> Tensor: ... +def max_pool3d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size = (), + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: _bool = False, +) -> Tensor: ... +def maximum( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + maximum(input, other, *, out=None) -> Tensor + + Computes the element-wise maximum of :attr:`input` and :attr:`other`. + + .. note:: + If one of the elements being compared is a NaN, then that element is returned. + :func:`maximum` is not supported for tensors with complex dtypes. + + Args: + input (Tensor): the input tensor. + other (Tensor): the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor((1, 2, -1)) + >>> b = torch.tensor((3, 0, 4)) + >>> torch.maximum(a, b) + tensor([3, 2, 4]) + """ + +@overload +def mean( + input: Tensor, + *, + dtype: _dtype | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + mean(input, *, dtype=None) -> Tensor + + .. note:: + If the `input` tensor is empty, ``torch.mean()`` returns ``nan``. + This behavior is consistent with NumPy and follows the definition + that the mean over an empty set is undefined. + + + Returns the mean value of all elements in the :attr:`input` tensor. Input must be floating point or complex. + + Args: + input (Tensor): + the input tensor, either of floating point or complex dtype + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.2294, -0.5481, 1.3288]]) + >>> torch.mean(a) + tensor(0.3367) + + .. function:: mean(input, dim, keepdim=False, *, dtype=None, out=None) -> Tensor + :noindex: + + Returns the mean value of each row of the :attr:`input` tensor in the given + dimension :attr:`dim`. If :attr:`dim` is a list of dimensions, + reduce over all of them. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + out (Tensor, optional): the output tensor. + + .. seealso:: + + :func:`torch.nanmean` computes the mean value of `non-NaN` elements. + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[-0.3841, 0.6320, 0.4254, -0.7384], + [-0.9644, 1.0131, -0.6549, -1.4279], + [-0.2951, -1.3350, -0.7694, 0.5600], + [ 1.0842, -0.9580, 0.3623, 0.2343]]) + >>> torch.mean(a, 1) + tensor([-0.0163, -0.5085, -0.4599, 0.1807]) + >>> torch.mean(a, 1, True) + tensor([[-0.0163], + [-0.5085], + [-0.4599], + [ 0.1807]]) + """ + +@overload +def mean( + input: Tensor, + dim: _int | _size | None, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + mean(input, *, dtype=None) -> Tensor + + .. note:: + If the `input` tensor is empty, ``torch.mean()`` returns ``nan``. + This behavior is consistent with NumPy and follows the definition + that the mean over an empty set is undefined. + + + Returns the mean value of all elements in the :attr:`input` tensor. Input must be floating point or complex. + + Args: + input (Tensor): + the input tensor, either of floating point or complex dtype + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.2294, -0.5481, 1.3288]]) + >>> torch.mean(a) + tensor(0.3367) + + .. function:: mean(input, dim, keepdim=False, *, dtype=None, out=None) -> Tensor + :noindex: + + Returns the mean value of each row of the :attr:`input` tensor in the given + dimension :attr:`dim`. If :attr:`dim` is a list of dimensions, + reduce over all of them. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + out (Tensor, optional): the output tensor. + + .. seealso:: + + :func:`torch.nanmean` computes the mean value of `non-NaN` elements. + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[-0.3841, 0.6320, 0.4254, -0.7384], + [-0.9644, 1.0131, -0.6549, -1.4279], + [-0.2951, -1.3350, -0.7694, 0.5600], + [ 1.0842, -0.9580, 0.3623, 0.2343]]) + >>> torch.mean(a, 1) + tensor([-0.0163, -0.5085, -0.4599, 0.1807]) + >>> torch.mean(a, 1, True) + tensor([[-0.0163], + [-0.5085], + [-0.4599], + [ 0.1807]]) + """ + +@overload +def mean( + input: Tensor, + dim: Sequence[str | EllipsisType | None], + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + mean(input, *, dtype=None) -> Tensor + + .. note:: + If the `input` tensor is empty, ``torch.mean()`` returns ``nan``. + This behavior is consistent with NumPy and follows the definition + that the mean over an empty set is undefined. + + + Returns the mean value of all elements in the :attr:`input` tensor. Input must be floating point or complex. + + Args: + input (Tensor): + the input tensor, either of floating point or complex dtype + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.2294, -0.5481, 1.3288]]) + >>> torch.mean(a) + tensor(0.3367) + + .. function:: mean(input, dim, keepdim=False, *, dtype=None, out=None) -> Tensor + :noindex: + + Returns the mean value of each row of the :attr:`input` tensor in the given + dimension :attr:`dim`. If :attr:`dim` is a list of dimensions, + reduce over all of them. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + out (Tensor, optional): the output tensor. + + .. seealso:: + + :func:`torch.nanmean` computes the mean value of `non-NaN` elements. + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[-0.3841, 0.6320, 0.4254, -0.7384], + [-0.9644, 1.0131, -0.6549, -1.4279], + [-0.2951, -1.3350, -0.7694, 0.5600], + [ 1.0842, -0.9580, 0.3623, 0.2343]]) + >>> torch.mean(a, 1) + tensor([-0.0163, -0.5085, -0.4599, 0.1807]) + >>> torch.mean(a, 1, True) + tensor([[-0.0163], + [-0.5085], + [-0.4599], + [ 0.1807]]) + """ + +@overload +def median(input: Tensor) -> Tensor: + r""" + median(input) -> Tensor + + Returns the median of the values in :attr:`input`. + + .. note:: + The median is not unique for :attr:`input` tensors with an even number + of elements. In this case the lower of the two medians is returned. To + compute the mean of both medians, use :func:`torch.quantile` with ``q=0.5`` instead. + + .. warning:: + This function produces deterministic (sub)gradients unlike ``median(dim=0)`` + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 1.5219, -1.5212, 0.2202]]) + >>> torch.median(a) + tensor(0.2202) + + .. function:: median(input, dim=-1, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` contains the median of each row of :attr:`input` + in the dimension :attr:`dim`, and ``indices`` contains the index of the median values found in the dimension :attr:`dim`. + + By default, :attr:`dim` is the last dimension of the :attr:`input` tensor. + + If :attr:`keepdim` is ``True``, the output tensors are of the same size + as :attr:`input` except in the dimension :attr:`dim` where they are of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in + the outputs tensor having 1 fewer dimension than :attr:`input`. + + .. note:: + The median is not unique for :attr:`input` tensors with an even number + of elements in the dimension :attr:`dim`. In this case the lower of the + two medians is returned. To compute the mean of both medians in + :attr:`input`, use :func:`torch.quantile` with ``q=0.5`` instead. + + .. warning:: + ``indices`` does not necessarily contain the first occurrence of each + median value found, unless it is unique. + The exact implementation details are device-specific. + Do not expect the same result when run on CPU and GPU in general. + For the same reason do not expect the gradients to be deterministic. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out ((Tensor, Tensor), optional): The first tensor will be populated with the median values and the second + tensor, which must have dtype long, with their indices in the dimension + :attr:`dim` of :attr:`input`. + + Example:: + + >>> a = torch.randn(4, 5) + >>> a + tensor([[ 0.2505, -0.3982, -0.9948, 0.3518, -1.3131], + [ 0.3180, -0.6993, 1.0436, 0.0438, 0.2270], + [-0.2751, 0.7303, 0.2192, 0.3321, 0.2488], + [ 1.0778, -1.9510, 0.7048, 0.4742, -0.7125]]) + >>> torch.median(a, 1) + torch.return_types.median(values=tensor([-0.3982, 0.2270, 0.2488, 0.4742]), indices=tensor([1, 4, 4, 3])) + """ + +@overload +def median( + input: Tensor, + dim: _int, + keepdim: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.median: + r""" + median(input) -> Tensor + + Returns the median of the values in :attr:`input`. + + .. note:: + The median is not unique for :attr:`input` tensors with an even number + of elements. In this case the lower of the two medians is returned. To + compute the mean of both medians, use :func:`torch.quantile` with ``q=0.5`` instead. + + .. warning:: + This function produces deterministic (sub)gradients unlike ``median(dim=0)`` + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 1.5219, -1.5212, 0.2202]]) + >>> torch.median(a) + tensor(0.2202) + + .. function:: median(input, dim=-1, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` contains the median of each row of :attr:`input` + in the dimension :attr:`dim`, and ``indices`` contains the index of the median values found in the dimension :attr:`dim`. + + By default, :attr:`dim` is the last dimension of the :attr:`input` tensor. + + If :attr:`keepdim` is ``True``, the output tensors are of the same size + as :attr:`input` except in the dimension :attr:`dim` where they are of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in + the outputs tensor having 1 fewer dimension than :attr:`input`. + + .. note:: + The median is not unique for :attr:`input` tensors with an even number + of elements in the dimension :attr:`dim`. In this case the lower of the + two medians is returned. To compute the mean of both medians in + :attr:`input`, use :func:`torch.quantile` with ``q=0.5`` instead. + + .. warning:: + ``indices`` does not necessarily contain the first occurrence of each + median value found, unless it is unique. + The exact implementation details are device-specific. + Do not expect the same result when run on CPU and GPU in general. + For the same reason do not expect the gradients to be deterministic. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out ((Tensor, Tensor), optional): The first tensor will be populated with the median values and the second + tensor, which must have dtype long, with their indices in the dimension + :attr:`dim` of :attr:`input`. + + Example:: + + >>> a = torch.randn(4, 5) + >>> a + tensor([[ 0.2505, -0.3982, -0.9948, 0.3518, -1.3131], + [ 0.3180, -0.6993, 1.0436, 0.0438, 0.2270], + [-0.2751, 0.7303, 0.2192, 0.3321, 0.2488], + [ 1.0778, -1.9510, 0.7048, 0.4742, -0.7125]]) + >>> torch.median(a, 1) + torch.return_types.median(values=tensor([-0.3982, 0.2270, 0.2488, 0.4742]), indices=tensor([1, 4, 4, 3])) + """ + +@overload +def median( + input: Tensor, + dim: str | EllipsisType | None, + keepdim: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.median: + r""" + median(input) -> Tensor + + Returns the median of the values in :attr:`input`. + + .. note:: + The median is not unique for :attr:`input` tensors with an even number + of elements. In this case the lower of the two medians is returned. To + compute the mean of both medians, use :func:`torch.quantile` with ``q=0.5`` instead. + + .. warning:: + This function produces deterministic (sub)gradients unlike ``median(dim=0)`` + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 1.5219, -1.5212, 0.2202]]) + >>> torch.median(a) + tensor(0.2202) + + .. function:: median(input, dim=-1, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` contains the median of each row of :attr:`input` + in the dimension :attr:`dim`, and ``indices`` contains the index of the median values found in the dimension :attr:`dim`. + + By default, :attr:`dim` is the last dimension of the :attr:`input` tensor. + + If :attr:`keepdim` is ``True``, the output tensors are of the same size + as :attr:`input` except in the dimension :attr:`dim` where they are of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in + the outputs tensor having 1 fewer dimension than :attr:`input`. + + .. note:: + The median is not unique for :attr:`input` tensors with an even number + of elements in the dimension :attr:`dim`. In this case the lower of the + two medians is returned. To compute the mean of both medians in + :attr:`input`, use :func:`torch.quantile` with ``q=0.5`` instead. + + .. warning:: + ``indices`` does not necessarily contain the first occurrence of each + median value found, unless it is unique. + The exact implementation details are device-specific. + Do not expect the same result when run on CPU and GPU in general. + For the same reason do not expect the gradients to be deterministic. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out ((Tensor, Tensor), optional): The first tensor will be populated with the median values and the second + tensor, which must have dtype long, with their indices in the dimension + :attr:`dim` of :attr:`input`. + + Example:: + + >>> a = torch.randn(4, 5) + >>> a + tensor([[ 0.2505, -0.3982, -0.9948, 0.3518, -1.3131], + [ 0.3180, -0.6993, 1.0436, 0.0438, 0.2270], + [-0.2751, 0.7303, 0.2192, 0.3321, 0.2488], + [ 1.0778, -1.9510, 0.7048, 0.4742, -0.7125]]) + >>> torch.median(a, 1) + torch.return_types.median(values=tensor([-0.3982, 0.2270, 0.2488, 0.4742]), indices=tensor([1, 4, 4, 3])) + """ + +@overload +def min(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + min(input, *, out=None) -> Tensor + + Returns the minimum value of all elements in the :attr:`input` tensor. + + .. note:: + The difference between ``max``/``min`` and ``amax``/``amin`` is: + - ``amax``/``amin`` supports reducing on multiple dimensions, + - ``amax``/``amin`` does not return indices. + + Both ``amax``/``amin`` evenly distribute gradients between equal values + when there are multiple input elements with the same minimum or maximum value. + + For ``max``/``min``: + - If reduce over all dimensions(no dim specified), gradients evenly distribute between equally ``max``/``min`` values. + - If reduce over one specified axis, only propagate to the indexed element. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.6750, 1.0857, 1.7197]]) + >>> torch.min(a) + tensor(0.6750) + + .. function:: min(input, dim, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` is the minimum + value of each row of the :attr:`input` tensor in the given dimension + :attr:`dim`. And ``indices`` is the index location of each minimum value found + (argmin). + + If :attr:`keepdim` is ``True``, the output tensors are of the same size as + :attr:`input` except in the dimension :attr:`dim` where they are of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in + the output tensors having 1 fewer dimension than :attr:`input`. + + .. note:: If there are multiple minimal values in a reduced row then + the indices of the first minimal value are returned. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. If omitted, all dimensions are reduced. Explicit ``None`` is not supported. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (tuple, optional): the tuple of two output tensors (min, min_indices) + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[-0.6248, 1.1334, -1.1899, -0.2803], + [-1.4644, -0.2635, -0.3651, 0.6134], + [ 0.2457, 0.0384, 1.0128, 0.7015], + [-0.1153, 2.9849, 2.1458, 0.5788]]) + >>> torch.min(a, 1) + torch.return_types.min(values=tensor([-1.1899, -1.4644, 0.0384, -0.1153]), indices=tensor([2, 0, 1, 0])) + + .. function:: min(input, other, *, out=None) -> Tensor + :noindex: + + See :func:`torch.minimum`. + """ + +@overload +def min( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + min(input, *, out=None) -> Tensor + + Returns the minimum value of all elements in the :attr:`input` tensor. + + .. note:: + The difference between ``max``/``min`` and ``amax``/``amin`` is: + - ``amax``/``amin`` supports reducing on multiple dimensions, + - ``amax``/``amin`` does not return indices. + + Both ``amax``/``amin`` evenly distribute gradients between equal values + when there are multiple input elements with the same minimum or maximum value. + + For ``max``/``min``: + - If reduce over all dimensions(no dim specified), gradients evenly distribute between equally ``max``/``min`` values. + - If reduce over one specified axis, only propagate to the indexed element. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.6750, 1.0857, 1.7197]]) + >>> torch.min(a) + tensor(0.6750) + + .. function:: min(input, dim, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` is the minimum + value of each row of the :attr:`input` tensor in the given dimension + :attr:`dim`. And ``indices`` is the index location of each minimum value found + (argmin). + + If :attr:`keepdim` is ``True``, the output tensors are of the same size as + :attr:`input` except in the dimension :attr:`dim` where they are of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in + the output tensors having 1 fewer dimension than :attr:`input`. + + .. note:: If there are multiple minimal values in a reduced row then + the indices of the first minimal value are returned. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. If omitted, all dimensions are reduced. Explicit ``None`` is not supported. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (tuple, optional): the tuple of two output tensors (min, min_indices) + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[-0.6248, 1.1334, -1.1899, -0.2803], + [-1.4644, -0.2635, -0.3651, 0.6134], + [ 0.2457, 0.0384, 1.0128, 0.7015], + [-0.1153, 2.9849, 2.1458, 0.5788]]) + >>> torch.min(a, 1) + torch.return_types.min(values=tensor([-1.1899, -1.4644, 0.0384, -0.1153]), indices=tensor([2, 0, 1, 0])) + + .. function:: min(input, other, *, out=None) -> Tensor + :noindex: + + See :func:`torch.minimum`. + """ + +@overload +def min( + input: Tensor, + dim: _int, + keepdim: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.min: + r""" + min(input, *, out=None) -> Tensor + + Returns the minimum value of all elements in the :attr:`input` tensor. + + .. note:: + The difference between ``max``/``min`` and ``amax``/``amin`` is: + - ``amax``/``amin`` supports reducing on multiple dimensions, + - ``amax``/``amin`` does not return indices. + + Both ``amax``/``amin`` evenly distribute gradients between equal values + when there are multiple input elements with the same minimum or maximum value. + + For ``max``/``min``: + - If reduce over all dimensions(no dim specified), gradients evenly distribute between equally ``max``/``min`` values. + - If reduce over one specified axis, only propagate to the indexed element. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.6750, 1.0857, 1.7197]]) + >>> torch.min(a) + tensor(0.6750) + + .. function:: min(input, dim, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` is the minimum + value of each row of the :attr:`input` tensor in the given dimension + :attr:`dim`. And ``indices`` is the index location of each minimum value found + (argmin). + + If :attr:`keepdim` is ``True``, the output tensors are of the same size as + :attr:`input` except in the dimension :attr:`dim` where they are of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in + the output tensors having 1 fewer dimension than :attr:`input`. + + .. note:: If there are multiple minimal values in a reduced row then + the indices of the first minimal value are returned. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. If omitted, all dimensions are reduced. Explicit ``None`` is not supported. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (tuple, optional): the tuple of two output tensors (min, min_indices) + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[-0.6248, 1.1334, -1.1899, -0.2803], + [-1.4644, -0.2635, -0.3651, 0.6134], + [ 0.2457, 0.0384, 1.0128, 0.7015], + [-0.1153, 2.9849, 2.1458, 0.5788]]) + >>> torch.min(a, 1) + torch.return_types.min(values=tensor([-1.1899, -1.4644, 0.0384, -0.1153]), indices=tensor([2, 0, 1, 0])) + + .. function:: min(input, other, *, out=None) -> Tensor + :noindex: + + See :func:`torch.minimum`. + """ + +@overload +def min( + input: Tensor, + dim: str | EllipsisType | None, + keepdim: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.min: + r""" + min(input, *, out=None) -> Tensor + + Returns the minimum value of all elements in the :attr:`input` tensor. + + .. note:: + The difference between ``max``/``min`` and ``amax``/``amin`` is: + - ``amax``/``amin`` supports reducing on multiple dimensions, + - ``amax``/``amin`` does not return indices. + + Both ``amax``/``amin`` evenly distribute gradients between equal values + when there are multiple input elements with the same minimum or maximum value. + + For ``max``/``min``: + - If reduce over all dimensions(no dim specified), gradients evenly distribute between equally ``max``/``min`` values. + - If reduce over one specified axis, only propagate to the indexed element. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.6750, 1.0857, 1.7197]]) + >>> torch.min(a) + tensor(0.6750) + + .. function:: min(input, dim, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` is the minimum + value of each row of the :attr:`input` tensor in the given dimension + :attr:`dim`. And ``indices`` is the index location of each minimum value found + (argmin). + + If :attr:`keepdim` is ``True``, the output tensors are of the same size as + :attr:`input` except in the dimension :attr:`dim` where they are of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in + the output tensors having 1 fewer dimension than :attr:`input`. + + .. note:: If there are multiple minimal values in a reduced row then + the indices of the first minimal value are returned. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. If omitted, all dimensions are reduced. Explicit ``None`` is not supported. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (tuple, optional): the tuple of two output tensors (min, min_indices) + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[-0.6248, 1.1334, -1.1899, -0.2803], + [-1.4644, -0.2635, -0.3651, 0.6134], + [ 0.2457, 0.0384, 1.0128, 0.7015], + [-0.1153, 2.9849, 2.1458, 0.5788]]) + >>> torch.min(a, 1) + torch.return_types.min(values=tensor([-1.1899, -1.4644, 0.0384, -0.1153]), indices=tensor([2, 0, 1, 0])) + + .. function:: min(input, other, *, out=None) -> Tensor + :noindex: + + See :func:`torch.minimum`. + """ + +def minimum( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + minimum(input, other, *, out=None) -> Tensor + + Computes the element-wise minimum of :attr:`input` and :attr:`other`. + + .. note:: + If one of the elements being compared is a NaN, then that element is returned. + :func:`minimum` is not supported for tensors with complex dtypes. + + Args: + input (Tensor): the input tensor. + other (Tensor): the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor((1, 2, -1)) + >>> b = torch.tensor((3, 0, 4)) + >>> torch.minimum(a, b) + tensor([1, 0, -1]) + """ + +def miopen_batch_norm( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + running_mean: Tensor | None, + running_var: Tensor | None, + training: _bool, + exponential_average_factor: _float, + epsilon: _float, +) -> tuple[Tensor, Tensor, Tensor]: ... +def miopen_convolution( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + padding: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + groups: _int | SymInt, + benchmark: _bool, + deterministic: _bool, +) -> Tensor: ... +def miopen_convolution_add_relu( + input: Tensor, + weight: Tensor, + z: Tensor, + alpha: Number | _complex | None, + bias: Tensor | None, + stride: Sequence[_int | SymInt], + padding: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + groups: _int | SymInt, +) -> Tensor: ... +def miopen_convolution_relu( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + stride: Sequence[_int | SymInt], + padding: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + groups: _int | SymInt, +) -> Tensor: ... +def miopen_convolution_transpose( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + padding: Sequence[_int | SymInt], + output_padding: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + groups: _int | SymInt, + benchmark: _bool, + deterministic: _bool, +) -> Tensor: ... +def miopen_depthwise_convolution( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + padding: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + groups: _int | SymInt, + benchmark: _bool, + deterministic: _bool, +) -> Tensor: ... +def miopen_rnn( + input: Tensor, + weight: tuple[Tensor, ...] | list[Tensor] | None, + weight_stride0: _int, + hx: Tensor, + cx: Tensor | None, + mode: _int, + hidden_size: _int, + num_layers: _int, + batch_first: _bool, + dropout: _float, + train: _bool, + bidirectional: _bool, + batch_sizes: _size, + dropout_state: Tensor | None, +) -> tuple[Tensor, Tensor, Tensor, Tensor, Tensor]: ... +def mkldnn_adaptive_avg_pool2d( + input: Tensor, + output_size: _int | _size, + *, + out: Tensor | None = None, +) -> Tensor: ... +def mkldnn_convolution( + input: Tensor, + weight: Tensor, + bias: Tensor | None, + padding: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + dilation: Sequence[_int | SymInt], + groups: _int | SymInt, +) -> Tensor: ... +def mkldnn_linear_backward_weights( + grad_output: Tensor, + input: Tensor, + weight: Tensor, + bias_defined: _bool, +) -> tuple[Tensor, Tensor]: ... +def mkldnn_max_pool2d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size = (), + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: _bool = False, +) -> Tensor: ... +def mkldnn_max_pool3d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size = (), + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: _bool = False, +) -> Tensor: ... +def mkldnn_rnn_layer( + input: Tensor, + weight0: Tensor, + weight1: Tensor, + weight2: Tensor, + weight3: Tensor, + hx_: Tensor, + cx_: Tensor, + reverse: _bool, + batch_sizes: _size, + mode: _int, + hidden_size: _int, + num_layers: _int, + has_biases: _bool, + bidirectional: _bool, + batch_first: _bool, + train: _bool, +) -> tuple[Tensor, Tensor, Tensor, Tensor]: ... +@overload +def mm(input: Tensor, mat2: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + mm(input, mat2, out_dtype=None, *, out=None) -> Tensor + + Performs a matrix multiplication of the matrices :attr:`input` and :attr:`mat2`. + + If :attr:`input` is a :math:`(n \times m)` tensor, :attr:`mat2` is a + :math:`(m \times p)` tensor, :attr:`out` will be a :math:`(n \times p)` tensor. + + .. note:: This function does not :ref:`broadcast `. + For broadcasting matrix products, see :func:`torch.matmul`. + + Supports strided and sparse 2-D tensors as inputs, autograd with + respect to strided inputs. + + This operation has support for arguments with :ref:`sparse layouts`. + If :attr:`out` is provided its layout will be used. Otherwise, the result + layout will be deduced from that of :attr:`input`. + + + .. warning:: + Sparse support is a beta feature and some layout(s)/dtype/device combinations may not be supported, + or may not have autograd support. If you notice missing functionality please + open a feature request. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): the first matrix to be matrix multiplied + mat2 (Tensor): the second matrix to be matrix multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> mat1 = torch.randn(2, 3) + >>> mat2 = torch.randn(3, 3) + >>> torch.mm(mat1, mat2) + tensor([[ 0.4851, 0.5037, -0.3633], + [-0.0760, -3.6705, 2.4784]]) + """ + +@overload +def mm( + input: Tensor, + mat2: Tensor, + out_dtype: _dtype, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + mm(input, mat2, out_dtype=None, *, out=None) -> Tensor + + Performs a matrix multiplication of the matrices :attr:`input` and :attr:`mat2`. + + If :attr:`input` is a :math:`(n \times m)` tensor, :attr:`mat2` is a + :math:`(m \times p)` tensor, :attr:`out` will be a :math:`(n \times p)` tensor. + + .. note:: This function does not :ref:`broadcast `. + For broadcasting matrix products, see :func:`torch.matmul`. + + Supports strided and sparse 2-D tensors as inputs, autograd with + respect to strided inputs. + + This operation has support for arguments with :ref:`sparse layouts`. + If :attr:`out` is provided its layout will be used. Otherwise, the result + layout will be deduced from that of :attr:`input`. + + + .. warning:: + Sparse support is a beta feature and some layout(s)/dtype/device combinations may not be supported, + or may not have autograd support. If you notice missing functionality please + open a feature request. + + This operator supports :ref:`TensorFloat32`. + + On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. + + Args: + input (Tensor): the first matrix to be matrix multiplied + mat2 (Tensor): the second matrix to be matrix multiplied + out_dtype (dtype, optional): the dtype of the output tensor, + Supported only on CUDA and for torch.float32 given + torch.float16/torch.bfloat16 input dtypes + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> mat1 = torch.randn(2, 3) + >>> mat2 = torch.randn(3, 3) + >>> torch.mm(mat1, mat2) + tensor([[ 0.4851, 0.5037, -0.3633], + [-0.0760, -3.6705, 2.4784]]) + """ + +@overload +def mode( + input: Tensor, + dim: _int = -1, + keepdim: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.mode: + r""" + mode(input, dim=-1, keepdim=False, *, out=None) -> (Tensor, LongTensor) + + Returns a namedtuple ``(values, indices)`` where ``values`` is the mode + value of each row of the :attr:`input` tensor in the given dimension + :attr:`dim`, i.e. a value which appears most often + in that row, and ``indices`` is the index location of each mode value found. + + By default, :attr:`dim` is the last dimension of the :attr:`input` tensor. + + If :attr:`keepdim` is ``True``, the output tensors are of the same size as + :attr:`input` except in the dimension :attr:`dim` where they are of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting + in the output tensors having 1 fewer dimension than :attr:`input`. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (tuple, optional): the result tuple of two output tensors (values, indices) + + Example:: + + >>> b = torch.tensor([[0, 0, 0, 2, 0, 0, 2], + ... [0, 3, 0, 0, 2, 0, 1], + ... [2, 2, 2, 0, 0, 0, 3], + ... [2, 2, 3, 0, 1, 1, 0], + ... [1, 1, 0, 0, 2, 0, 2]]) + >>> torch.mode(b, 0) + torch.return_types.mode( + values=tensor([0, 2, 0, 0, 0, 0, 2]), + indices=tensor([1, 3, 4, 4, 2, 4, 4])) + """ + +@overload +def mode( + input: Tensor, + dim: str | EllipsisType | None, + keepdim: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.mode: + r""" + mode(input, dim=-1, keepdim=False, *, out=None) -> (Tensor, LongTensor) + + Returns a namedtuple ``(values, indices)`` where ``values`` is the mode + value of each row of the :attr:`input` tensor in the given dimension + :attr:`dim`, i.e. a value which appears most often + in that row, and ``indices`` is the index location of each mode value found. + + By default, :attr:`dim` is the last dimension of the :attr:`input` tensor. + + If :attr:`keepdim` is ``True``, the output tensors are of the same size as + :attr:`input` except in the dimension :attr:`dim` where they are of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting + in the output tensors having 1 fewer dimension than :attr:`input`. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out (tuple, optional): the result tuple of two output tensors (values, indices) + + Example:: + + >>> b = torch.tensor([[0, 0, 0, 2, 0, 0, 2], + ... [0, 3, 0, 0, 2, 0, 1], + ... [2, 2, 2, 0, 0, 0, 3], + ... [2, 2, 3, 0, 1, 1, 0], + ... [1, 1, 0, 0, 2, 0, 2]]) + >>> torch.mode(b, 0) + torch.return_types.mode( + values=tensor([0, 2, 0, 0, 0, 0, 2]), + indices=tensor([1, 3, 4, 4, 2, 4, 4])) + """ + +@overload +def moveaxis(input: Tensor, source: _int, destination: _int) -> Tensor: + r""" + moveaxis(input, source, destination) -> Tensor + + Alias for :func:`torch.movedim`. + + This function is equivalent to NumPy's moveaxis function. + + Examples:: + + >>> t = torch.randn(3,2,1) + >>> t + tensor([[[-0.3362], + [-0.8437]], + + [[-0.9627], + [ 0.1727]], + + [[ 0.5173], + [-0.1398]]]) + >>> torch.moveaxis(t, 1, 0).shape + torch.Size([2, 3, 1]) + >>> torch.moveaxis(t, 1, 0) + tensor([[[-0.3362], + [-0.9627], + [ 0.5173]], + + [[-0.8437], + [ 0.1727], + [-0.1398]]]) + >>> torch.moveaxis(t, (1, 2), (0, 1)).shape + torch.Size([2, 1, 3]) + >>> torch.moveaxis(t, (1, 2), (0, 1)) + tensor([[[-0.3362, -0.9627, 0.5173]], + + [[-0.8437, 0.1727, -0.1398]]]) + """ + +@overload +def moveaxis(input: Tensor, source: _size, destination: _size) -> Tensor: + r""" + moveaxis(input, source, destination) -> Tensor + + Alias for :func:`torch.movedim`. + + This function is equivalent to NumPy's moveaxis function. + + Examples:: + + >>> t = torch.randn(3,2,1) + >>> t + tensor([[[-0.3362], + [-0.8437]], + + [[-0.9627], + [ 0.1727]], + + [[ 0.5173], + [-0.1398]]]) + >>> torch.moveaxis(t, 1, 0).shape + torch.Size([2, 3, 1]) + >>> torch.moveaxis(t, 1, 0) + tensor([[[-0.3362], + [-0.9627], + [ 0.5173]], + + [[-0.8437], + [ 0.1727], + [-0.1398]]]) + >>> torch.moveaxis(t, (1, 2), (0, 1)).shape + torch.Size([2, 1, 3]) + >>> torch.moveaxis(t, (1, 2), (0, 1)) + tensor([[[-0.3362, -0.9627, 0.5173]], + + [[-0.8437, 0.1727, -0.1398]]]) + """ + +@overload +def movedim(input: Tensor, source: _int, destination: _int) -> Tensor: + r""" + movedim(input, source, destination) -> Tensor + + Moves the dimension(s) of :attr:`input` at the position(s) in :attr:`source` + to the position(s) in :attr:`destination`. + + Other dimensions of :attr:`input` that are not explicitly moved remain in + their original order and appear at the positions not specified in :attr:`destination`. + + Args: + input (Tensor): the input tensor. + source (int or tuple of ints): Original positions of the dims to move. These must be unique. + destination (int or tuple of ints): Destination positions for each of the original dims. These must also be unique. + + Examples:: + + >>> t = torch.randn(3,2,1) + >>> t + tensor([[[-0.3362], + [-0.8437]], + + [[-0.9627], + [ 0.1727]], + + [[ 0.5173], + [-0.1398]]]) + >>> torch.movedim(t, 1, 0).shape + torch.Size([2, 3, 1]) + >>> torch.movedim(t, 1, 0) + tensor([[[-0.3362], + [-0.9627], + [ 0.5173]], + + [[-0.8437], + [ 0.1727], + [-0.1398]]]) + >>> torch.movedim(t, (1, 2), (0, 1)).shape + torch.Size([2, 1, 3]) + >>> torch.movedim(t, (1, 2), (0, 1)) + tensor([[[-0.3362, -0.9627, 0.5173]], + + [[-0.8437, 0.1727, -0.1398]]]) + """ + +@overload +def movedim(input: Tensor, source: _size, destination: _size) -> Tensor: + r""" + movedim(input, source, destination) -> Tensor + + Moves the dimension(s) of :attr:`input` at the position(s) in :attr:`source` + to the position(s) in :attr:`destination`. + + Other dimensions of :attr:`input` that are not explicitly moved remain in + their original order and appear at the positions not specified in :attr:`destination`. + + Args: + input (Tensor): the input tensor. + source (int or tuple of ints): Original positions of the dims to move. These must be unique. + destination (int or tuple of ints): Destination positions for each of the original dims. These must also be unique. + + Examples:: + + >>> t = torch.randn(3,2,1) + >>> t + tensor([[[-0.3362], + [-0.8437]], + + [[-0.9627], + [ 0.1727]], + + [[ 0.5173], + [-0.1398]]]) + >>> torch.movedim(t, 1, 0).shape + torch.Size([2, 3, 1]) + >>> torch.movedim(t, 1, 0) + tensor([[[-0.3362], + [-0.9627], + [ 0.5173]], + + [[-0.8437], + [ 0.1727], + [-0.1398]]]) + >>> torch.movedim(t, (1, 2), (0, 1)).shape + torch.Size([2, 1, 3]) + >>> torch.movedim(t, (1, 2), (0, 1)) + tensor([[[-0.3362, -0.9627, 0.5173]], + + [[-0.8437, 0.1727, -0.1398]]]) + """ + +def msort(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + msort(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Sorts the elements of the :attr:`input` tensor along its first dimension + in ascending order by value. + + .. note:: `torch.msort(t)` is equivalent to `torch.sort(t, dim=0)[0]`. + See also :func:`torch.sort`. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> t = torch.randn(3, 4) + >>> t + tensor([[-0.1321, 0.4370, -1.2631, -1.1289], + [-2.0527, -1.1250, 0.2275, 0.3077], + [-0.0881, -0.1259, -0.5495, 1.0284]]) + >>> torch.msort(t) + tensor([[-2.0527, -1.1250, -1.2631, -1.1289], + [-0.1321, -0.1259, -0.5495, 0.3077], + [-0.0881, 0.4370, 0.2275, 1.0284]]) + """ + +def mul( + input: Tensor | Number | _complex, + other: Tensor | Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + mul(input, other, *, out=None) -> Tensor + + Multiplies :attr:`input` by :attr:`other`. + + + .. math:: + \text{out}_i = \text{input}_i \times \text{other}_i + + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer, float, and complex inputs. + + Args: + input (Tensor): the input tensor. + other (Tensor or Number): the tensor or number to multiply input by. + + Keyword args: + out (Tensor, optional): the output tensor. + + Examples:: + + >>> a = torch.randn(3) + >>> a + tensor([ 0.2015, -0.4255, 2.6087]) + >>> torch.mul(a, 100) + tensor([ 20.1494, -42.5491, 260.8663]) + + >>> b = torch.randn(4, 1) + >>> b + tensor([[ 1.1207], + [-0.3137], + [ 0.0700], + [ 0.8378]]) + >>> c = torch.randn(1, 4) + >>> c + tensor([[ 0.5146, 0.1216, -0.5244, 2.2382]]) + >>> torch.mul(b, c) + tensor([[ 0.5767, 0.1363, -0.5877, 2.5083], + [-0.1614, -0.0382, 0.1645, -0.7021], + [ 0.0360, 0.0085, -0.0367, 0.1567], + [ 0.4312, 0.1019, -0.4394, 1.8753]]) + """ + +def multinomial( + input: Tensor, + num_samples: _int | SymInt, + replacement: _bool = False, + *, + generator: Generator | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + multinomial(input, num_samples, replacement=False, *, generator=None, out=None) -> LongTensor + + Returns a tensor where each row contains :attr:`num_samples` indices sampled + from the multinomial (a stricter definition would be multivariate, + refer to :class:`torch.distributions.multinomial.Multinomial` for more details) + probability distribution located in the corresponding row + of tensor :attr:`input`. + + .. note:: + The rows of :attr:`input` do not need to sum to one (in which case we use + the values as weights), but must be non-negative, finite and have + a non-zero sum. + + Indices are ordered from left to right according to when each was sampled + (first samples are placed in first column). + + If :attr:`input` is a vector, :attr:`out` is a vector of size :attr:`num_samples`. + + If :attr:`input` is a matrix with `m` rows, :attr:`out` is an matrix of shape + :math:`(m \times \text{num\_samples})`. + + If replacement is ``True``, samples are drawn with replacement. + + If not, they are drawn without replacement, which means that when a + sample index is drawn for a row, it cannot be drawn again for that row. + + .. note:: + When drawn without replacement, :attr:`num_samples` must be lower than + number of non-zero elements in :attr:`input` (or the min number of non-zero + elements in each row of :attr:`input` if it is a matrix). + + Args: + input (Tensor): the input tensor containing probabilities + num_samples (int): number of samples to draw + replacement (bool, optional): whether to draw with replacement or not + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + + Example:: + + >>> weights = torch.tensor([0, 10, 3, 0], dtype=torch.float) # create a tensor of weights + >>> torch.multinomial(weights, 2) + tensor([1, 2]) + >>> torch.multinomial(weights, 5) # ERROR! + RuntimeError: cannot sample n_sample > prob_dist.size(-1) samples without replacement + >>> torch.multinomial(weights, 4, replacement=True) + tensor([ 2, 1, 1, 1]) + """ + +@overload +def multiply( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + multiply(input, other, *, out=None) + + Alias for :func:`torch.mul`. + """ + +@overload +def multiply(input: Tensor, other: Number | _complex) -> Tensor: + r""" + multiply(input, other, *, out=None) + + Alias for :func:`torch.mul`. + """ + +def mv(input: Tensor, vec: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + mv(input, vec, *, out=None) -> Tensor + + Performs a matrix-vector product of the matrix :attr:`input` and the vector + :attr:`vec`. + + If :attr:`input` is a :math:`(n \times m)` tensor, :attr:`vec` is a 1-D tensor of + size :math:`m`, :attr:`out` will be 1-D of size :math:`n`. + + .. note:: This function does not :ref:`broadcast `. + + Args: + input (Tensor): matrix to be multiplied + vec (Tensor): vector to be multiplied + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> mat = torch.randn(2, 3) + >>> vec = torch.randn(3) + >>> torch.mv(mat, vec) + tensor([ 1.0404, -0.6361]) + """ + +def mvlgamma( + input: Tensor, + p: _int, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + mvlgamma(input, p, *, out=None) -> Tensor + + Alias for :func:`torch.special.multigammaln`. + """ + +def nan_to_num( + input: Tensor, + nan: _float | None = None, + posinf: _float | None = None, + neginf: _float | None = None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + nan_to_num(input, nan=0.0, posinf=None, neginf=None, *, out=None) -> Tensor + + Replaces :literal:`NaN`, positive infinity, and negative infinity values in :attr:`input` + with the values specified by :attr:`nan`, :attr:`posinf`, and :attr:`neginf`, respectively. + By default, :literal:`NaN`\ s are replaced with zero, positive infinity is replaced with the + greatest finite value representable by :attr:`input`'s dtype, and negative infinity + is replaced with the least finite value representable by :attr:`input`'s dtype. + + Args: + input (Tensor): the input tensor. + nan (Number, optional): the value to replace :literal:`NaN`\s with. Default is zero. + posinf (Number, optional): if a Number, the value to replace positive infinity values with. + If None, positive infinity values are replaced with the greatest finite value representable by :attr:`input`'s dtype. + Default is None. + neginf (Number, optional): if a Number, the value to replace negative infinity values with. + If None, negative infinity values are replaced with the lowest finite value representable by :attr:`input`'s dtype. + Default is None. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> x = torch.tensor([float('nan'), float('inf'), -float('inf'), 3.14]) + >>> torch.nan_to_num(x) + tensor([ 0.0000e+00, 3.4028e+38, -3.4028e+38, 3.1400e+00]) + >>> torch.nan_to_num(x, nan=2.0) + tensor([ 2.0000e+00, 3.4028e+38, -3.4028e+38, 3.1400e+00]) + >>> torch.nan_to_num(x, nan=2.0, posinf=1.0) + tensor([ 2.0000e+00, 1.0000e+00, -3.4028e+38, 3.1400e+00]) + """ + +def nan_to_num_( + input: Tensor, + nan: _float | None = None, + posinf: _float | None = None, + neginf: _float | None = None, +) -> Tensor: ... +def nanmean( + input: Tensor, + dim: _int | _size | None = None, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + nanmean(input, dim=None, keepdim=False, *, dtype=None, out=None) -> Tensor + + Computes the mean of all `non-NaN` elements along the specified dimensions. + Input must be floating point or complex. + + This function is identical to :func:`torch.mean` when there are no `NaN` values + in the :attr:`input` tensor. In the presence of `NaN`, :func:`torch.mean` will + propagate the `NaN` to the output whereas :func:`torch.nanmean` will ignore the + `NaN` values (`torch.nanmean(a)` is equivalent to `torch.mean(a[~a.isnan()])`). + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor, either of floating point or complex dtype + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + out (Tensor, optional): the output tensor. + + .. seealso:: + + :func:`torch.mean` computes the mean value, propagating `NaN`. + + Example:: + + >>> x = torch.tensor([[torch.nan, 1, 2], [1, 2, 3]]) + >>> x.mean() + tensor(nan) + >>> x.nanmean() + tensor(1.8000) + >>> x.mean(dim=0) + tensor([ nan, 1.5000, 2.5000]) + >>> x.nanmean(dim=0) + tensor([1.0000, 1.5000, 2.5000]) + + # If all elements in the reduced dimensions are NaN then the result is NaN + >>> torch.tensor([torch.nan]).nanmean() + tensor(nan) + """ + +@overload +def nanmedian(input: Tensor) -> Tensor: + r""" + nanmedian(input) -> Tensor + + Returns the median of the values in :attr:`input`, ignoring ``NaN`` values. + + This function is identical to :func:`torch.median` when there are no ``NaN`` values in :attr:`input`. + When :attr:`input` has one or more ``NaN`` values, :func:`torch.median` will always return ``NaN``, + while this function will return the median of the non-``NaN`` elements in :attr:`input`. + If all the elements in :attr:`input` are ``NaN`` it will also return ``NaN``. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> a = torch.tensor([1, float('nan'), 3, 2]) + >>> a.median() + tensor(nan) + >>> a.nanmedian() + tensor(2.) + + .. function:: nanmedian(input, dim=-1, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` contains the median of each row of :attr:`input` + in the dimension :attr:`dim`, ignoring ``NaN`` values, and ``indices`` contains the index of the median values + found in the dimension :attr:`dim`. + + This function is identical to :func:`torch.median` when there are no ``NaN`` values in a reduced row. When a reduced row has + one or more ``NaN`` values, :func:`torch.median` will always reduce it to ``NaN``, while this function will reduce it to the + median of the non-``NaN`` elements. If all the elements in a reduced row are ``NaN`` then it will be reduced to ``NaN``, too. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out ((Tensor, Tensor), optional): The first tensor will be populated with the median values and the second + tensor, which must have dtype long, with their indices in the dimension + :attr:`dim` of :attr:`input`. + + Example:: + + >>> a = torch.tensor([[2, 3, 1], [float('nan'), 1, float('nan')]]) + >>> a + tensor([[2., 3., 1.], + [nan, 1., nan]]) + >>> a.median(0) + torch.return_types.median(values=tensor([nan, 1., nan]), indices=tensor([1, 1, 1])) + >>> a.nanmedian(0) + torch.return_types.nanmedian(values=tensor([2., 1., 1.]), indices=tensor([0, 1, 0])) + """ + +@overload +def nanmedian( + input: Tensor, + dim: _int, + keepdim: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.nanmedian: + r""" + nanmedian(input) -> Tensor + + Returns the median of the values in :attr:`input`, ignoring ``NaN`` values. + + This function is identical to :func:`torch.median` when there are no ``NaN`` values in :attr:`input`. + When :attr:`input` has one or more ``NaN`` values, :func:`torch.median` will always return ``NaN``, + while this function will return the median of the non-``NaN`` elements in :attr:`input`. + If all the elements in :attr:`input` are ``NaN`` it will also return ``NaN``. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> a = torch.tensor([1, float('nan'), 3, 2]) + >>> a.median() + tensor(nan) + >>> a.nanmedian() + tensor(2.) + + .. function:: nanmedian(input, dim=-1, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` contains the median of each row of :attr:`input` + in the dimension :attr:`dim`, ignoring ``NaN`` values, and ``indices`` contains the index of the median values + found in the dimension :attr:`dim`. + + This function is identical to :func:`torch.median` when there are no ``NaN`` values in a reduced row. When a reduced row has + one or more ``NaN`` values, :func:`torch.median` will always reduce it to ``NaN``, while this function will reduce it to the + median of the non-``NaN`` elements. If all the elements in a reduced row are ``NaN`` then it will be reduced to ``NaN``, too. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out ((Tensor, Tensor), optional): The first tensor will be populated with the median values and the second + tensor, which must have dtype long, with their indices in the dimension + :attr:`dim` of :attr:`input`. + + Example:: + + >>> a = torch.tensor([[2, 3, 1], [float('nan'), 1, float('nan')]]) + >>> a + tensor([[2., 3., 1.], + [nan, 1., nan]]) + >>> a.median(0) + torch.return_types.median(values=tensor([nan, 1., nan]), indices=tensor([1, 1, 1])) + >>> a.nanmedian(0) + torch.return_types.nanmedian(values=tensor([2., 1., 1.]), indices=tensor([0, 1, 0])) + """ + +@overload +def nanmedian( + input: Tensor, + dim: str | EllipsisType | None, + keepdim: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.nanmedian: + r""" + nanmedian(input) -> Tensor + + Returns the median of the values in :attr:`input`, ignoring ``NaN`` values. + + This function is identical to :func:`torch.median` when there are no ``NaN`` values in :attr:`input`. + When :attr:`input` has one or more ``NaN`` values, :func:`torch.median` will always return ``NaN``, + while this function will return the median of the non-``NaN`` elements in :attr:`input`. + If all the elements in :attr:`input` are ``NaN`` it will also return ``NaN``. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> a = torch.tensor([1, float('nan'), 3, 2]) + >>> a.median() + tensor(nan) + >>> a.nanmedian() + tensor(2.) + + .. function:: nanmedian(input, dim=-1, keepdim=False, *, out=None) -> (Tensor, LongTensor) + :noindex: + + Returns a namedtuple ``(values, indices)`` where ``values`` contains the median of each row of :attr:`input` + in the dimension :attr:`dim`, ignoring ``NaN`` values, and ``indices`` contains the index of the median values + found in the dimension :attr:`dim`. + + This function is identical to :func:`torch.median` when there are no ``NaN`` values in a reduced row. When a reduced row has + one or more ``NaN`` values, :func:`torch.median` will always reduce it to ``NaN``, while this function will reduce it to the + median of the non-``NaN`` elements. If all the elements in a reduced row are ``NaN`` then it will be reduced to ``NaN``, too. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + out ((Tensor, Tensor), optional): The first tensor will be populated with the median values and the second + tensor, which must have dtype long, with their indices in the dimension + :attr:`dim` of :attr:`input`. + + Example:: + + >>> a = torch.tensor([[2, 3, 1], [float('nan'), 1, float('nan')]]) + >>> a + tensor([[2., 3., 1.], + [nan, 1., nan]]) + >>> a.median(0) + torch.return_types.median(values=tensor([nan, 1., nan]), indices=tensor([1, 1, 1])) + >>> a.nanmedian(0) + torch.return_types.nanmedian(values=tensor([2., 1., 1.]), indices=tensor([0, 1, 0])) + """ + +@overload +def nanquantile( + input: Tensor, + q: Tensor, + dim: _int | None = None, + keepdim: _bool = False, + *, + interpolation: str = "linear", + out: Tensor | None = None, +) -> Tensor: + r""" + nanquantile(input, q, dim=None, keepdim=False, *, interpolation='linear', out=None) -> Tensor + + This is a variant of :func:`torch.quantile` that "ignores" ``NaN`` values, + computing the quantiles :attr:`q` as if ``NaN`` values in :attr:`input` did + not exist. If all values in a reduced row are ``NaN`` then the quantiles for + that reduction will be ``NaN``. See the documentation for :func:`torch.quantile`. + + Args: + input (Tensor): the input tensor. + q (float or Tensor): a scalar or 1D tensor of quantile values in the range [0, 1] + + dim (int, optional): the dimension to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword arguments: + interpolation (str): interpolation method to use when the desired quantile lies between two data points. + Can be ``linear``, ``lower``, ``higher``, ``midpoint`` and ``nearest``. + Default is ``linear``. + out (Tensor, optional): the output tensor. + + Example:: + + >>> t = torch.tensor([float('nan'), 1, 2]) + >>> t.quantile(0.5) + tensor(nan) + >>> t.nanquantile(0.5) + tensor(1.5000) + >>> t = torch.tensor([[float('nan'), float('nan')], [1, 2]]) + >>> t + tensor([[nan, nan], + [1., 2.]]) + >>> t.nanquantile(0.5, dim=0) + tensor([1., 2.]) + >>> t.nanquantile(0.5, dim=1) + tensor([ nan, 1.5000]) + """ + +@overload +def nanquantile( + input: Tensor, + q: _float, + dim: _int | None = None, + keepdim: _bool = False, + *, + interpolation: str = "linear", + out: Tensor | None = None, +) -> Tensor: + r""" + nanquantile(input, q, dim=None, keepdim=False, *, interpolation='linear', out=None) -> Tensor + + This is a variant of :func:`torch.quantile` that "ignores" ``NaN`` values, + computing the quantiles :attr:`q` as if ``NaN`` values in :attr:`input` did + not exist. If all values in a reduced row are ``NaN`` then the quantiles for + that reduction will be ``NaN``. See the documentation for :func:`torch.quantile`. + + Args: + input (Tensor): the input tensor. + q (float or Tensor): a scalar or 1D tensor of quantile values in the range [0, 1] + + dim (int, optional): the dimension to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword arguments: + interpolation (str): interpolation method to use when the desired quantile lies between two data points. + Can be ``linear``, ``lower``, ``higher``, ``midpoint`` and ``nearest``. + Default is ``linear``. + out (Tensor, optional): the output tensor. + + Example:: + + >>> t = torch.tensor([float('nan'), 1, 2]) + >>> t.quantile(0.5) + tensor(nan) + >>> t.nanquantile(0.5) + tensor(1.5000) + >>> t = torch.tensor([[float('nan'), float('nan')], [1, 2]]) + >>> t + tensor([[nan, nan], + [1., 2.]]) + >>> t.nanquantile(0.5, dim=0) + tensor([1., 2.]) + >>> t.nanquantile(0.5, dim=1) + tensor([ nan, 1.5000]) + """ + +def nansum( + input: Tensor, + dim: _int | _size | None = None, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + nansum(input, *, dtype=None) -> Tensor + + Returns the sum of all elements, treating Not a Numbers (NaNs) as zero. + + Args: + input (Tensor): the input tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> a = torch.tensor([1., 2., float('nan'), 4.]) + >>> torch.nansum(a) + tensor(7.) + + .. function:: nansum(input, dim, keepdim=False, *, dtype=None) -> Tensor + :noindex: + + Returns the sum of each row of the :attr:`input` tensor in the given + dimension :attr:`dim`, treating Not a Numbers (NaNs) as zero. + If :attr:`dim` is a list of dimensions, reduce over all of them. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> torch.nansum(torch.tensor([1., float("nan")])) + tensor(1.) + >>> a = torch.tensor([[1, 2], [3., float("nan")]]) + >>> torch.nansum(a) + tensor(6.) + >>> torch.nansum(a, dim=0) + tensor([4., 2.]) + >>> torch.nansum(a, dim=1) + tensor([3., 3.]) + """ + +@overload +def narrow( + input: Tensor, + dim: _int, + start: Tensor, + length: _int | SymInt, +) -> Tensor: + r""" + narrow(input, dim, start, length) -> Tensor + + Returns a new tensor that is a narrowed version of :attr:`input` tensor. The + dimension :attr:`dim` is input from :attr:`start` to ``start + length``. The + returned tensor and :attr:`input` tensor share the same underlying storage. + + Args: + input (Tensor): the tensor to narrow + dim (int): the dimension along which to narrow + start (int or Tensor): index of the element to start the narrowed dimension + from. Can be negative, which means indexing from the end of `dim`. If + `Tensor`, it must be an 0-dim integral `Tensor` (bools not allowed) + length (int): length of the narrowed dimension, must be weakly positive + + Example:: + + >>> x = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) + >>> torch.narrow(x, 0, 0, 2) + tensor([[ 1, 2, 3], + [ 4, 5, 6]]) + >>> torch.narrow(x, 1, 1, 2) + tensor([[ 2, 3], + [ 5, 6], + [ 8, 9]]) + >>> torch.narrow(x, -1, torch.tensor(-1), 1) + tensor([[3], + [6], + [9]]) + """ + +@overload +def narrow( + input: Tensor, + dim: _int, + start: _int | SymInt, + length: _int | SymInt, +) -> Tensor: + r""" + narrow(input, dim, start, length) -> Tensor + + Returns a new tensor that is a narrowed version of :attr:`input` tensor. The + dimension :attr:`dim` is input from :attr:`start` to ``start + length``. The + returned tensor and :attr:`input` tensor share the same underlying storage. + + Args: + input (Tensor): the tensor to narrow + dim (int): the dimension along which to narrow + start (int or Tensor): index of the element to start the narrowed dimension + from. Can be negative, which means indexing from the end of `dim`. If + `Tensor`, it must be an 0-dim integral `Tensor` (bools not allowed) + length (int): length of the narrowed dimension, must be weakly positive + + Example:: + + >>> x = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) + >>> torch.narrow(x, 0, 0, 2) + tensor([[ 1, 2, 3], + [ 4, 5, 6]]) + >>> torch.narrow(x, 1, 1, 2) + tensor([[ 2, 3], + [ 5, 6], + [ 8, 9]]) + >>> torch.narrow(x, -1, torch.tensor(-1), 1) + tensor([[3], + [6], + [9]]) + """ + +def narrow_copy( + input: Tensor, + dim: _int, + start: _int | SymInt, + length: _int | SymInt, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + narrow_copy(input, dim, start, length, *, out=None) -> Tensor + + Same as :meth:`Tensor.narrow` except this returns a copy rather + than shared storage. This is primarily for sparse tensors, which + do not have a shared-storage narrow method. + + Args: + input (Tensor): the tensor to narrow + dim (int): the dimension along which to narrow + start (int): index of the element to start the narrowed dimension from. Can + be negative, which means indexing from the end of `dim` + length (int): length of the narrowed dimension, must be weakly positive + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> x = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) + >>> torch.narrow_copy(x, 0, 0, 2) + tensor([[ 1, 2, 3], + [ 4, 5, 6]]) + >>> torch.narrow_copy(x, 1, 1, 2) + tensor([[ 2, 3], + [ 5, 6], + [ 8, 9]]) + >>> s = torch.arange(16).reshape(2, 2, 2, 2).to_sparse(2) + >>> torch.narrow_copy(s, 0, 0, 1) + tensor(indices=tensor([[0, 0], + [0, 1]]), + values=tensor([[[0, 1], + [2, 3]], + + [[4, 5], + [6, 7]]]), + size=(1, 2, 2, 2), nnz=2, layout=torch.sparse_coo) + + .. seealso:: + + :func:`torch.narrow` for a non copy variant + """ + +def native_batch_norm( + input: Tensor, + weight: Tensor | None, + bias: Tensor | None, + running_mean: Tensor | None, + running_var: Tensor | None, + training: _bool, + momentum: _float, + eps: _float, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> tuple[Tensor, Tensor, Tensor]: ... +def native_channel_shuffle(input: Tensor, groups: _int | SymInt) -> Tensor: ... +def native_dropout( + input: Tensor, + p: _float, + train: _bool | None, +) -> tuple[Tensor, Tensor]: ... +def native_group_norm( + input: Tensor, + weight: Tensor | None, + bias: Tensor | None, + N: _int | SymInt, + C: _int | SymInt, + HxW: _int | SymInt, + group: _int, + eps: _float, +) -> tuple[Tensor, Tensor, Tensor]: ... +def native_layer_norm( + input: Tensor, + normalized_shape: Sequence[_int | SymInt], + weight: Tensor | None, + bias: Tensor | None, + eps: _float, +) -> tuple[Tensor, Tensor, Tensor]: ... +@overload +def native_norm( + input: Tensor, + p: Number | _complex | None, + dim: _int | _size, + keepdim: _bool, + dtype: _dtype | None, +) -> Tensor: ... +@overload +def native_norm(input: Tensor, p: Number | _complex = 2) -> Tensor: ... +@overload +def ne( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + ne(input, other, *, out=None) -> Tensor + + Computes :math:`\text{input} \neq \text{other}` element-wise. + + + The second argument can be a number or a tensor whose shape is + :ref:`broadcastable ` with the first argument. + + Args: + input (Tensor): the tensor to compare + other (Tensor or float): the tensor or value to compare + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is not equal to :attr:`other` and False elsewhere + + Example:: + + >>> torch.ne(torch.tensor([[1, 2], [3, 4]]), torch.tensor([[1, 1], [4, 4]])) + tensor([[False, True], [True, False]]) + """ + +@overload +def ne( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + ne(input, other, *, out=None) -> Tensor + + Computes :math:`\text{input} \neq \text{other}` element-wise. + + + The second argument can be a number or a tensor whose shape is + :ref:`broadcastable ` with the first argument. + + Args: + input (Tensor): the tensor to compare + other (Tensor or float): the tensor or value to compare + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + A boolean tensor that is True where :attr:`input` is not equal to :attr:`other` and False elsewhere + + Example:: + + >>> torch.ne(torch.tensor([[1, 2], [3, 4]]), torch.tensor([[1, 1], [4, 4]])) + tensor([[False, True], [True, False]]) + """ + +def neg(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + neg(input, *, out=None) -> Tensor + + Returns a new tensor with the negative of the elements of :attr:`input`. + + .. math:: + \text{out} = -1 \times \text{input} + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(5) + >>> a + tensor([ 0.0090, -0.2262, -0.0682, -0.2866, 0.3940]) + >>> torch.neg(a) + tensor([-0.0090, 0.2262, 0.0682, 0.2866, -0.3940]) + """ + +def neg_(input: Tensor) -> Tensor: ... +def negative(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + negative(input, *, out=None) -> Tensor + + Alias for :func:`torch.neg` + """ + +def negative_(input: Tensor) -> Tensor: ... +def nextafter( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + nextafter(input, other, *, out=None) -> Tensor + + Return the next floating-point value after :attr:`input` towards :attr:`other`, elementwise. + + The shapes of ``input`` and ``other`` must be + :ref:`broadcastable `. + + Args: + input (Tensor): the first input tensor + other (Tensor): the second input tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> eps = torch.finfo(torch.float32).eps + >>> torch.nextafter(torch.tensor([1.0, 2.0]), torch.tensor([2.0, 1.0])) == torch.tensor([eps + 1, 2 - eps]) + tensor([True, True]) + """ + +@overload +def nonzero( + input: Tensor, + *, + as_tuple: Literal[False] = False, + out: Tensor | None = None, +) -> Tensor: + r""" + nonzero(input, *, out=None, as_tuple=False) -> LongTensor or tuple of LongTensors + + .. note:: + :func:`torch.nonzero(..., as_tuple=False) ` (default) returns a + 2-D tensor where each row is the index for a nonzero value. + + :func:`torch.nonzero(..., as_tuple=True) ` returns a tuple of 1-D + index tensors, allowing for advanced indexing, so ``x[x.nonzero(as_tuple=True)]`` + gives all nonzero values of tensor ``x``. Of the returned tuple, each index tensor + contains nonzero indices for a certain dimension. + + See below for more details on the two behaviors. + + When :attr:`input` is on CUDA, :func:`torch.nonzero() ` causes + host-device synchronization. + + **When** :attr:`as_tuple` **is** ``False`` **(default)**: + + Returns a tensor containing the indices of all non-zero elements of + :attr:`input`. Each row in the result contains the indices of a non-zero + element in :attr:`input`. The result is sorted lexicographically, with + the last index changing the fastest (C-style). + + If :attr:`input` has :math:`n` dimensions, then the resulting indices tensor + :attr:`out` is of size :math:`(z \times n)`, where :math:`z` is the total number of + non-zero elements in the :attr:`input` tensor. + + **When** :attr:`as_tuple` **is** ``True``: + + Returns a tuple of 1-D tensors, one for each dimension in :attr:`input`, + each containing the indices (in that dimension) of all non-zero elements of + :attr:`input` . + + If :attr:`input` has :math:`n` dimensions, then the resulting tuple contains :math:`n` + tensors of size :math:`z`, where :math:`z` is the total number of + non-zero elements in the :attr:`input` tensor. + + As a special case, when :attr:`input` has zero dimensions and a nonzero scalar + value, it is treated as a one-dimensional tensor with one element. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (LongTensor, optional): the output tensor containing indices + + Returns: + LongTensor or tuple of LongTensor: If :attr:`as_tuple` is ``False``, the output + tensor containing indices. If :attr:`as_tuple` is ``True``, one 1-D tensor for + each dimension, containing the indices of each nonzero element along that + dimension. + + Example:: + + >>> torch.nonzero(torch.tensor([1, 1, 1, 0, 1])) + tensor([[ 0], + [ 1], + [ 2], + [ 4]]) + >>> torch.nonzero(torch.tensor([[0.6, 0.0, 0.0, 0.0], + ... [0.0, 0.4, 0.0, 0.0], + ... [0.0, 0.0, 1.2, 0.0], + ... [0.0, 0.0, 0.0,-0.4]])) + tensor([[ 0, 0], + [ 1, 1], + [ 2, 2], + [ 3, 3]]) + >>> torch.nonzero(torch.tensor([1, 1, 1, 0, 1]), as_tuple=True) + (tensor([0, 1, 2, 4]),) + >>> torch.nonzero(torch.tensor([[0.6, 0.0, 0.0, 0.0], + ... [0.0, 0.4, 0.0, 0.0], + ... [0.0, 0.0, 1.2, 0.0], + ... [0.0, 0.0, 0.0,-0.4]]), as_tuple=True) + (tensor([0, 1, 2, 3]), tensor([0, 1, 2, 3])) + >>> torch.nonzero(torch.tensor(5), as_tuple=True) + (tensor([0]),) + """ + +@overload +def nonzero( + input: Tensor, + *, + as_tuple: Literal[True], +) -> tuple[Tensor, ...]: + r""" + nonzero(input, *, out=None, as_tuple=False) -> LongTensor or tuple of LongTensors + + .. note:: + :func:`torch.nonzero(..., as_tuple=False) ` (default) returns a + 2-D tensor where each row is the index for a nonzero value. + + :func:`torch.nonzero(..., as_tuple=True) ` returns a tuple of 1-D + index tensors, allowing for advanced indexing, so ``x[x.nonzero(as_tuple=True)]`` + gives all nonzero values of tensor ``x``. Of the returned tuple, each index tensor + contains nonzero indices for a certain dimension. + + See below for more details on the two behaviors. + + When :attr:`input` is on CUDA, :func:`torch.nonzero() ` causes + host-device synchronization. + + **When** :attr:`as_tuple` **is** ``False`` **(default)**: + + Returns a tensor containing the indices of all non-zero elements of + :attr:`input`. Each row in the result contains the indices of a non-zero + element in :attr:`input`. The result is sorted lexicographically, with + the last index changing the fastest (C-style). + + If :attr:`input` has :math:`n` dimensions, then the resulting indices tensor + :attr:`out` is of size :math:`(z \times n)`, where :math:`z` is the total number of + non-zero elements in the :attr:`input` tensor. + + **When** :attr:`as_tuple` **is** ``True``: + + Returns a tuple of 1-D tensors, one for each dimension in :attr:`input`, + each containing the indices (in that dimension) of all non-zero elements of + :attr:`input` . + + If :attr:`input` has :math:`n` dimensions, then the resulting tuple contains :math:`n` + tensors of size :math:`z`, where :math:`z` is the total number of + non-zero elements in the :attr:`input` tensor. + + As a special case, when :attr:`input` has zero dimensions and a nonzero scalar + value, it is treated as a one-dimensional tensor with one element. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (LongTensor, optional): the output tensor containing indices + + Returns: + LongTensor or tuple of LongTensor: If :attr:`as_tuple` is ``False``, the output + tensor containing indices. If :attr:`as_tuple` is ``True``, one 1-D tensor for + each dimension, containing the indices of each nonzero element along that + dimension. + + Example:: + + >>> torch.nonzero(torch.tensor([1, 1, 1, 0, 1])) + tensor([[ 0], + [ 1], + [ 2], + [ 4]]) + >>> torch.nonzero(torch.tensor([[0.6, 0.0, 0.0, 0.0], + ... [0.0, 0.4, 0.0, 0.0], + ... [0.0, 0.0, 1.2, 0.0], + ... [0.0, 0.0, 0.0,-0.4]])) + tensor([[ 0, 0], + [ 1, 1], + [ 2, 2], + [ 3, 3]]) + >>> torch.nonzero(torch.tensor([1, 1, 1, 0, 1]), as_tuple=True) + (tensor([0, 1, 2, 4]),) + >>> torch.nonzero(torch.tensor([[0.6, 0.0, 0.0, 0.0], + ... [0.0, 0.4, 0.0, 0.0], + ... [0.0, 0.0, 1.2, 0.0], + ... [0.0, 0.0, 0.0,-0.4]]), as_tuple=True) + (tensor([0, 1, 2, 3]), tensor([0, 1, 2, 3])) + >>> torch.nonzero(torch.tensor(5), as_tuple=True) + (tensor([0]),) + """ + +def nonzero_static( + input: Tensor, + *, + size: _int | SymInt, + fill_value: _int = -1, + out: Tensor | None = None, +) -> Tensor: ... +def norm_except_dim(v: Tensor, pow: _int = 2, dim: _int = 0) -> Tensor: ... +@overload +def normal( + mean: Tensor, + std: Tensor, + *, + generator: Generator | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + normal(mean, std, *, generator=None, out=None) -> Tensor + + Returns a tensor of random numbers drawn from separate normal distributions + whose mean and standard deviation are given. + + The :attr:`mean` is a tensor with the mean of + each output element's normal distribution + + The :attr:`std` is a tensor with the standard deviation of + each output element's normal distribution + + The shapes of :attr:`mean` and :attr:`std` don't need to match, but the + total number of elements in each tensor need to be the same. + + .. note:: When the shapes do not match, the shape of :attr:`mean` + is used as the shape for the returned output tensor + + .. note:: When :attr:`std` is a CUDA tensor, this function synchronizes + its device with the CPU. + + Args: + mean (Tensor): the tensor of per-element means + std (Tensor): the tensor of per-element standard deviations + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.normal(mean=torch.arange(1., 11.), std=torch.arange(1, 0, -0.1)) + tensor([ 1.0425, 3.5672, 2.7969, 4.2925, 4.7229, 6.2134, + 8.0505, 8.1408, 9.0563, 10.0566]) + + .. function:: normal(mean=0.0, std, *, out=None) -> Tensor + :noindex: + + Similar to the function above, but the means are shared among all drawn + elements. + + Args: + mean (float, optional): the mean for all distributions + std (Tensor): the tensor of per-element standard deviations + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.normal(mean=0.5, std=torch.arange(1., 6.)) + tensor([-1.2793, -1.0732, -2.0687, 5.1177, -1.2303]) + + .. function:: normal(mean, std=1.0, *, out=None) -> Tensor + :noindex: + + Similar to the function above, but the standard deviations are shared among + all drawn elements. + + Args: + mean (Tensor): the tensor of per-element means + std (float, optional): the standard deviation for all distributions + + Keyword args: + out (Tensor, optional): the output tensor + + Example:: + + >>> torch.normal(mean=torch.arange(1., 6.)) + tensor([ 1.1552, 2.6148, 2.6535, 5.8318, 4.2361]) + + .. function:: normal(mean, std, size, *, out=None) -> Tensor + :noindex: + + Similar to the function above, but the means and standard deviations are shared + among all drawn elements. The resulting tensor has size given by :attr:`size`. + + Args: + mean (float): the mean for all distributions + std (float): the standard deviation for all distributions + size (int...): a sequence of integers defining the shape of the output tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.normal(2, 3, size=(1, 4)) + tensor([[-1.3987, -1.9544, 3.6048, 0.7909]]) + """ + +@overload +def normal( + mean: Tensor, + std: _float = 1, + *, + generator: Generator | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + normal(mean, std, *, generator=None, out=None) -> Tensor + + Returns a tensor of random numbers drawn from separate normal distributions + whose mean and standard deviation are given. + + The :attr:`mean` is a tensor with the mean of + each output element's normal distribution + + The :attr:`std` is a tensor with the standard deviation of + each output element's normal distribution + + The shapes of :attr:`mean` and :attr:`std` don't need to match, but the + total number of elements in each tensor need to be the same. + + .. note:: When the shapes do not match, the shape of :attr:`mean` + is used as the shape for the returned output tensor + + .. note:: When :attr:`std` is a CUDA tensor, this function synchronizes + its device with the CPU. + + Args: + mean (Tensor): the tensor of per-element means + std (Tensor): the tensor of per-element standard deviations + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.normal(mean=torch.arange(1., 11.), std=torch.arange(1, 0, -0.1)) + tensor([ 1.0425, 3.5672, 2.7969, 4.2925, 4.7229, 6.2134, + 8.0505, 8.1408, 9.0563, 10.0566]) + + .. function:: normal(mean=0.0, std, *, out=None) -> Tensor + :noindex: + + Similar to the function above, but the means are shared among all drawn + elements. + + Args: + mean (float, optional): the mean for all distributions + std (Tensor): the tensor of per-element standard deviations + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.normal(mean=0.5, std=torch.arange(1., 6.)) + tensor([-1.2793, -1.0732, -2.0687, 5.1177, -1.2303]) + + .. function:: normal(mean, std=1.0, *, out=None) -> Tensor + :noindex: + + Similar to the function above, but the standard deviations are shared among + all drawn elements. + + Args: + mean (Tensor): the tensor of per-element means + std (float, optional): the standard deviation for all distributions + + Keyword args: + out (Tensor, optional): the output tensor + + Example:: + + >>> torch.normal(mean=torch.arange(1., 6.)) + tensor([ 1.1552, 2.6148, 2.6535, 5.8318, 4.2361]) + + .. function:: normal(mean, std, size, *, out=None) -> Tensor + :noindex: + + Similar to the function above, but the means and standard deviations are shared + among all drawn elements. The resulting tensor has size given by :attr:`size`. + + Args: + mean (float): the mean for all distributions + std (float): the standard deviation for all distributions + size (int...): a sequence of integers defining the shape of the output tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.normal(2, 3, size=(1, 4)) + tensor([[-1.3987, -1.9544, 3.6048, 0.7909]]) + """ + +@overload +def normal( + mean: _float, + std: Tensor, + *, + generator: Generator | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + normal(mean, std, *, generator=None, out=None) -> Tensor + + Returns a tensor of random numbers drawn from separate normal distributions + whose mean and standard deviation are given. + + The :attr:`mean` is a tensor with the mean of + each output element's normal distribution + + The :attr:`std` is a tensor with the standard deviation of + each output element's normal distribution + + The shapes of :attr:`mean` and :attr:`std` don't need to match, but the + total number of elements in each tensor need to be the same. + + .. note:: When the shapes do not match, the shape of :attr:`mean` + is used as the shape for the returned output tensor + + .. note:: When :attr:`std` is a CUDA tensor, this function synchronizes + its device with the CPU. + + Args: + mean (Tensor): the tensor of per-element means + std (Tensor): the tensor of per-element standard deviations + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.normal(mean=torch.arange(1., 11.), std=torch.arange(1, 0, -0.1)) + tensor([ 1.0425, 3.5672, 2.7969, 4.2925, 4.7229, 6.2134, + 8.0505, 8.1408, 9.0563, 10.0566]) + + .. function:: normal(mean=0.0, std, *, out=None) -> Tensor + :noindex: + + Similar to the function above, but the means are shared among all drawn + elements. + + Args: + mean (float, optional): the mean for all distributions + std (Tensor): the tensor of per-element standard deviations + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.normal(mean=0.5, std=torch.arange(1., 6.)) + tensor([-1.2793, -1.0732, -2.0687, 5.1177, -1.2303]) + + .. function:: normal(mean, std=1.0, *, out=None) -> Tensor + :noindex: + + Similar to the function above, but the standard deviations are shared among + all drawn elements. + + Args: + mean (Tensor): the tensor of per-element means + std (float, optional): the standard deviation for all distributions + + Keyword args: + out (Tensor, optional): the output tensor + + Example:: + + >>> torch.normal(mean=torch.arange(1., 6.)) + tensor([ 1.1552, 2.6148, 2.6535, 5.8318, 4.2361]) + + .. function:: normal(mean, std, size, *, out=None) -> Tensor + :noindex: + + Similar to the function above, but the means and standard deviations are shared + among all drawn elements. The resulting tensor has size given by :attr:`size`. + + Args: + mean (float): the mean for all distributions + std (float): the standard deviation for all distributions + size (int...): a sequence of integers defining the shape of the output tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.normal(2, 3, size=(1, 4)) + tensor([[-1.3987, -1.9544, 3.6048, 0.7909]]) + """ + +@overload +def normal( + mean: _float, + std: _float, + size: Sequence[_int | SymInt], + *, + generator: Generator | None = None, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + normal(mean, std, *, generator=None, out=None) -> Tensor + + Returns a tensor of random numbers drawn from separate normal distributions + whose mean and standard deviation are given. + + The :attr:`mean` is a tensor with the mean of + each output element's normal distribution + + The :attr:`std` is a tensor with the standard deviation of + each output element's normal distribution + + The shapes of :attr:`mean` and :attr:`std` don't need to match, but the + total number of elements in each tensor need to be the same. + + .. note:: When the shapes do not match, the shape of :attr:`mean` + is used as the shape for the returned output tensor + + .. note:: When :attr:`std` is a CUDA tensor, this function synchronizes + its device with the CPU. + + Args: + mean (Tensor): the tensor of per-element means + std (Tensor): the tensor of per-element standard deviations + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.normal(mean=torch.arange(1., 11.), std=torch.arange(1, 0, -0.1)) + tensor([ 1.0425, 3.5672, 2.7969, 4.2925, 4.7229, 6.2134, + 8.0505, 8.1408, 9.0563, 10.0566]) + + .. function:: normal(mean=0.0, std, *, out=None) -> Tensor + :noindex: + + Similar to the function above, but the means are shared among all drawn + elements. + + Args: + mean (float, optional): the mean for all distributions + std (Tensor): the tensor of per-element standard deviations + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.normal(mean=0.5, std=torch.arange(1., 6.)) + tensor([-1.2793, -1.0732, -2.0687, 5.1177, -1.2303]) + + .. function:: normal(mean, std=1.0, *, out=None) -> Tensor + :noindex: + + Similar to the function above, but the standard deviations are shared among + all drawn elements. + + Args: + mean (Tensor): the tensor of per-element means + std (float, optional): the standard deviation for all distributions + + Keyword args: + out (Tensor, optional): the output tensor + + Example:: + + >>> torch.normal(mean=torch.arange(1., 6.)) + tensor([ 1.1552, 2.6148, 2.6535, 5.8318, 4.2361]) + + .. function:: normal(mean, std, size, *, out=None) -> Tensor + :noindex: + + Similar to the function above, but the means and standard deviations are shared + among all drawn elements. The resulting tensor has size given by :attr:`size`. + + Args: + mean (float): the mean for all distributions + std (float): the standard deviation for all distributions + size (int...): a sequence of integers defining the shape of the output tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.normal(2, 3, size=(1, 4)) + tensor([[-1.3987, -1.9544, 3.6048, 0.7909]]) + """ + +@overload +def not_equal( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + not_equal(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.ne`. + """ + +@overload +def not_equal( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + not_equal(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.ne`. + """ + +@overload +def nuclear_norm( + input: Tensor, + dim: _int | _size, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: ... +@overload +def nuclear_norm( + input: Tensor, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: ... +def numel(self: Tensor) -> _int: + r""" + numel(input: Tensor) -> int + + Returns the total number of elements in the :attr:`input` tensor. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> a = torch.randn(1, 2, 3, 4, 5) + >>> torch.numel(a) + 120 + >>> a = torch.zeros(4,4) + >>> torch.numel(a) + 16 + """ + +@overload +def ones( + size: Sequence[_int | SymInt], + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + ones(*size, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with the scalar value `1`, with the shape defined + by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.ones(2, 3) + tensor([[ 1., 1., 1.], + [ 1., 1., 1.]]) + + >>> torch.ones(5) + tensor([ 1., 1., 1., 1., 1.]) + """ + +@overload +def ones( + *size: _int | SymInt, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + ones(*size, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with the scalar value `1`, with the shape defined + by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.ones(2, 3) + tensor([[ 1., 1., 1.], + [ 1., 1., 1.]]) + + >>> torch.ones(5) + tensor([ 1., 1., 1., 1., 1.]) + """ + +@overload +def ones( + size: _size, + *, + names: Sequence[str | EllipsisType | None] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + ones(*size, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with the scalar value `1`, with the shape defined + by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.ones(2, 3) + tensor([[ 1., 1., 1.], + [ 1., 1., 1.]]) + + >>> torch.ones(5) + tensor([ 1., 1., 1., 1., 1.]) + """ + +@overload +def ones( + *size: _int, + names: Sequence[str | EllipsisType | None] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + ones(*size, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with the scalar value `1`, with the shape defined + by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword arguments: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.ones(2, 3) + tensor([[ 1., 1., 1.], + [ 1., 1., 1.]]) + + >>> torch.ones(5) + tensor([ 1., 1., 1., 1., 1.]) + """ + +def ones_like( + input: Tensor, + *, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + ones_like(input, *, dtype=None, layout=None, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns a tensor filled with the scalar value `1`, with the same size as + :attr:`input`. ``torch.ones_like(input)`` is equivalent to + ``torch.ones(input.size(), dtype=input.dtype, layout=input.layout, device=input.device)``. + + .. warning:: + As of 0.4, this function does not support an :attr:`out` keyword. As an alternative, + the old ``torch.ones_like(input, out=output)`` is equivalent to + ``torch.ones(input.size(), out=output)``. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + + Keyword arguments: + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + + Example:: + + >>> input = torch.empty(2, 3) + >>> torch.ones_like(input) + tensor([[ 1., 1., 1.], + [ 1., 1., 1.]]) + """ + +def orgqr( + input: Tensor, + input2: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + orgqr(input, tau) -> Tensor + + Alias for :func:`torch.linalg.householder_product`. + """ + +def ormqr( + input: Tensor, + input2: Tensor, + input3: Tensor, + left: _bool = True, + transpose: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + ormqr(input, tau, other, left=True, transpose=False, *, out=None) -> Tensor + + Computes the matrix-matrix multiplication of a product of Householder matrices with a general matrix. + + Multiplies a :math:`m \times n` matrix `C` (given by :attr:`other`) with a matrix `Q`, + where `Q` is represented using Householder reflectors `(input, tau)`. + See `Representation of Orthogonal or Unitary Matrices`_ for further details. + + If :attr:`left` is `True` then `op(Q)` times `C` is computed, otherwise the result is `C` times `op(Q)`. + When :attr:`left` is `True`, the implicit matrix `Q` has size :math:`m \times m`. + It has size :math:`n \times n` otherwise. + If :attr:`transpose` is `True` then `op` is the conjugate transpose operation, otherwise it's a no-op. + + Supports inputs of float, double, cfloat and cdouble dtypes. + Also supports batched inputs, and, if the input is batched, the output is batched with the same dimensions. + + .. seealso:: + :func:`torch.geqrf` can be used to form the Householder representation `(input, tau)` of matrix `Q` + from the QR decomposition. + + .. note:: + This function supports backward but it is only fast when ``(input, tau)`` do not require gradients + and/or ``tau.size(-1)`` is very small. + `` + + Args: + input (Tensor): tensor of shape `(*, mn, k)` where `*` is zero or more batch dimensions + and `mn` equals to `m` or `n` depending on the :attr:`left`. + tau (Tensor): tensor of shape `(*, min(mn, k))` where `*` is zero or more batch dimensions. + other (Tensor): tensor of shape `(*, m, n)` where `*` is zero or more batch dimensions. + left (bool): controls the order of multiplication. + transpose (bool): controls whether the matrix `Q` is conjugate transposed or not. + + Keyword args: + out (Tensor, optional): the output Tensor. Ignored if `None`. Default: `None`. + + .. _Representation of Orthogonal or Unitary Matrices: + https://www.netlib.org/lapack/lug/node128.html + """ + +def outer( + input: Tensor, + vec2: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + outer(input, vec2, *, out=None) -> Tensor + + Outer product of :attr:`input` and :attr:`vec2`. + If :attr:`input` is a vector of size :math:`n` and :attr:`vec2` is a vector of + size :math:`m`, then :attr:`out` must be a matrix of size :math:`(n \times m)`. + + .. note:: This function does not :ref:`broadcast `. + + Args: + input (Tensor): 1-D input vector + vec2 (Tensor): 1-D input vector + + Keyword args: + out (Tensor, optional): optional output matrix + + Example:: + + >>> v1 = torch.arange(1., 5.) + >>> v2 = torch.arange(1., 4.) + >>> torch.outer(v1, v2) + tensor([[ 1., 2., 3.], + [ 2., 4., 6.], + [ 3., 6., 9.], + [ 4., 8., 12.]]) + """ + +def pairwise_distance( + x1: Tensor, + x2: Tensor, + p: _float = 2, + eps: _float = 1e-06, + keepdim: _bool = False, +) -> Tensor: ... +def pdist(input: Tensor, p: _float = 2) -> Tensor: ... +def permute(input: Tensor, dims: _size) -> Tensor: + r""" + permute(input, dims) -> Tensor + + Returns a view of the original tensor :attr:`input` with its dimensions permuted. + + Args: + input (Tensor): the input tensor. + dims (tuple of int): The desired ordering of dimensions + + Example: + >>> x = torch.randn(2, 3, 5) + >>> x.size() + torch.Size([2, 3, 5]) + >>> torch.permute(x, (2, 0, 1)).size() + torch.Size([5, 2, 3]) + """ + +def permute_copy( + input: Tensor, + dims: _size, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.permute`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def pinverse(input: Tensor, rcond: _float = 1e-15) -> Tensor: + r""" + pinverse(input, rcond=1e-15) -> Tensor + + Alias for :func:`torch.linalg.pinv` + """ + +def pixel_shuffle(input: Tensor, upscale_factor: _int) -> Tensor: ... +def pixel_unshuffle(input: Tensor, downscale_factor: _int) -> Tensor: ... +def poisson(input: Tensor, generator: Generator | None = None) -> Tensor: + r""" + poisson(input, generator=None) -> Tensor + + Returns a tensor of the same size as :attr:`input` with each element + sampled from a Poisson distribution with rate parameter given by the corresponding + element in :attr:`input` i.e., + + .. math:: + \text{out}_i \sim \text{Poisson}(\text{input}_i) + + :attr:`input` must be non-negative. + + Args: + input (Tensor): the input tensor containing the rates of the Poisson distribution + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + + Example:: + + >>> rates = torch.rand(4, 4) * 5 # rate parameter between 0 and 5 + >>> torch.poisson(rates) + tensor([[9., 1., 3., 5.], + [8., 6., 6., 0.], + [0., 4., 5., 3.], + [2., 1., 4., 2.]]) + """ + +def poisson_nll_loss( + input: Tensor, + target: Tensor, + log_input: _bool, + full: _bool, + eps: _float, + reduction: _int, +) -> Tensor: ... +def polar( + abs: Tensor, + angle: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + polar(abs, angle, *, out=None) -> Tensor + + Constructs a complex tensor whose elements are Cartesian coordinates + corresponding to the polar coordinates with absolute value :attr:`abs` and angle + :attr:`angle`. + + .. math:: + \text{out} = \text{abs} \cdot \cos(\text{angle}) + \text{abs} \cdot \sin(\text{angle}) \cdot j + + .. note:: + `torch.polar` is similar to + `std::polar `_ + and does not compute the polar decomposition + of a complex tensor like Python's `cmath.polar` and SciPy's `linalg.polar` do. + The behavior of this function is undefined if `abs` is negative or NaN, or if `angle` is + infinite. + + + Args: + abs (Tensor): The absolute value the complex tensor. Must be float or double. + angle (Tensor): The angle of the complex tensor. Must be same dtype as + :attr:`abs`. + + Keyword args: + out (Tensor): If the inputs are ``torch.float32``, must be + ``torch.complex64``. If the inputs are ``torch.float64``, must be + ``torch.complex128``. + + Example:: + + >>> import numpy as np + >>> abs = torch.tensor([1, 2], dtype=torch.float64) + >>> angle = torch.tensor([np.pi / 2, 5 * np.pi / 4], dtype=torch.float64) + >>> z = torch.polar(abs, angle) + >>> z + tensor([(0.0000+1.0000j), (-1.4142-1.4142j)], dtype=torch.complex128) + """ + +def polygamma( + n: _int, + input: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + polygamma(n, input, *, out=None) -> Tensor + + Alias for :func:`torch.special.polygamma`. + """ + +def positive(input: Tensor) -> Tensor: + r""" + positive(input) -> Tensor + + Returns :attr:`input`. + Throws a runtime error if :attr:`input` is a bool tensor. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> t = torch.randn(5) + >>> t + tensor([ 0.0090, -0.2262, -0.0682, -0.2866, 0.3940]) + >>> torch.positive(t) + tensor([ 0.0090, -0.2262, -0.0682, -0.2866, 0.3940]) + """ + +@overload +def pow( + input: Tensor, + exponent: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + pow(input, exponent, *, out=None) -> Tensor + + Takes the power of each element in :attr:`input` with :attr:`exponent` and + returns a tensor with the result. + + :attr:`exponent` can be either a single ``float`` number or a `Tensor` + with the same number of elements as :attr:`input`. + + When :attr:`exponent` is a scalar value, the operation applied is: + + .. math:: + \text{out}_i = x_i ^ \text{exponent} + + When :attr:`exponent` is a tensor, the operation applied is: + + .. math:: + \text{out}_i = x_i ^ {\text{exponent}_i} + + When :attr:`exponent` is a tensor, the shapes of :attr:`input` + and :attr:`exponent` must be :ref:`broadcastable `. + + Args: + input (Tensor): the input tensor. + exponent (float or tensor): the exponent value + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([ 0.4331, 1.2475, 0.6834, -0.2791]) + >>> torch.pow(a, 2) + tensor([ 0.1875, 1.5561, 0.4670, 0.0779]) + >>> exp = torch.arange(1., 5.) + + >>> a = torch.arange(1., 5.) + >>> a + tensor([ 1., 2., 3., 4.]) + >>> exp + tensor([ 1., 2., 3., 4.]) + >>> torch.pow(a, exp) + tensor([ 1., 4., 27., 256.]) + + .. function:: pow(self, exponent, *, out=None) -> Tensor + :noindex: + + :attr:`self` is a scalar ``float`` value, and :attr:`exponent` is a tensor. + The returned tensor :attr:`out` is of the same shape as :attr:`exponent` + + The operation applied is: + + .. math:: + \text{out}_i = \text{self} ^ {\text{exponent}_i} + + Args: + self (float): the scalar base value for the power operation + exponent (Tensor): the exponent tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> exp = torch.arange(1., 5.) + >>> base = 2 + >>> torch.pow(base, exp) + tensor([ 2., 4., 8., 16.]) + """ + +@overload +def pow( + self: Number | _complex, + exponent: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + pow(input, exponent, *, out=None) -> Tensor + + Takes the power of each element in :attr:`input` with :attr:`exponent` and + returns a tensor with the result. + + :attr:`exponent` can be either a single ``float`` number or a `Tensor` + with the same number of elements as :attr:`input`. + + When :attr:`exponent` is a scalar value, the operation applied is: + + .. math:: + \text{out}_i = x_i ^ \text{exponent} + + When :attr:`exponent` is a tensor, the operation applied is: + + .. math:: + \text{out}_i = x_i ^ {\text{exponent}_i} + + When :attr:`exponent` is a tensor, the shapes of :attr:`input` + and :attr:`exponent` must be :ref:`broadcastable `. + + Args: + input (Tensor): the input tensor. + exponent (float or tensor): the exponent value + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([ 0.4331, 1.2475, 0.6834, -0.2791]) + >>> torch.pow(a, 2) + tensor([ 0.1875, 1.5561, 0.4670, 0.0779]) + >>> exp = torch.arange(1., 5.) + + >>> a = torch.arange(1., 5.) + >>> a + tensor([ 1., 2., 3., 4.]) + >>> exp + tensor([ 1., 2., 3., 4.]) + >>> torch.pow(a, exp) + tensor([ 1., 4., 27., 256.]) + + .. function:: pow(self, exponent, *, out=None) -> Tensor + :noindex: + + :attr:`self` is a scalar ``float`` value, and :attr:`exponent` is a tensor. + The returned tensor :attr:`out` is of the same shape as :attr:`exponent` + + The operation applied is: + + .. math:: + \text{out}_i = \text{self} ^ {\text{exponent}_i} + + Args: + self (float): the scalar base value for the power operation + exponent (Tensor): the exponent tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> exp = torch.arange(1., 5.) + >>> base = 2 + >>> torch.pow(base, exp) + tensor([ 2., 4., 8., 16.]) + """ + +@overload +def pow( + input: Tensor, + exponent: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + pow(input, exponent, *, out=None) -> Tensor + + Takes the power of each element in :attr:`input` with :attr:`exponent` and + returns a tensor with the result. + + :attr:`exponent` can be either a single ``float`` number or a `Tensor` + with the same number of elements as :attr:`input`. + + When :attr:`exponent` is a scalar value, the operation applied is: + + .. math:: + \text{out}_i = x_i ^ \text{exponent} + + When :attr:`exponent` is a tensor, the operation applied is: + + .. math:: + \text{out}_i = x_i ^ {\text{exponent}_i} + + When :attr:`exponent` is a tensor, the shapes of :attr:`input` + and :attr:`exponent` must be :ref:`broadcastable `. + + Args: + input (Tensor): the input tensor. + exponent (float or tensor): the exponent value + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([ 0.4331, 1.2475, 0.6834, -0.2791]) + >>> torch.pow(a, 2) + tensor([ 0.1875, 1.5561, 0.4670, 0.0779]) + >>> exp = torch.arange(1., 5.) + + >>> a = torch.arange(1., 5.) + >>> a + tensor([ 1., 2., 3., 4.]) + >>> exp + tensor([ 1., 2., 3., 4.]) + >>> torch.pow(a, exp) + tensor([ 1., 4., 27., 256.]) + + .. function:: pow(self, exponent, *, out=None) -> Tensor + :noindex: + + :attr:`self` is a scalar ``float`` value, and :attr:`exponent` is a tensor. + The returned tensor :attr:`out` is of the same shape as :attr:`exponent` + + The operation applied is: + + .. math:: + \text{out}_i = \text{self} ^ {\text{exponent}_i} + + Args: + self (float): the scalar base value for the power operation + exponent (Tensor): the exponent tensor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> exp = torch.arange(1., 5.) + >>> base = 2 + >>> torch.pow(base, exp) + tensor([ 2., 4., 8., 16.]) + """ + +def prelu(input: Tensor, weight: Tensor) -> Tensor: ... +@overload +def prod(input: Tensor, *, dtype: _dtype | None = None) -> Tensor: + r""" + prod(input: Tensor, *, dtype: Optional[_dtype]) -> Tensor + + Returns the product of all elements in the :attr:`input` tensor. + + Args: + input (Tensor): the input tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[-0.8020, 0.5428, -1.5854]]) + >>> torch.prod(a) + tensor(0.6902) + + .. function:: prod(input, dim, keepdim=False, *, dtype=None) -> Tensor + :noindex: + + Returns the product of each row of the :attr:`input` tensor in the given + dimension :attr:`dim`. + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in + the output tensor having 1 fewer dimension than :attr:`input`. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> a = torch.randn(4, 2) + >>> a + tensor([[ 0.5261, -0.3837], + [ 1.1857, -0.2498], + [-1.1646, 0.0705], + [ 1.1131, -1.0629]]) + >>> torch.prod(a, 1) + tensor([-0.2018, -0.2962, -0.0821, -1.1831]) + """ + +@overload +def prod( + input: Tensor, + dim: _int, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + prod(input: Tensor, *, dtype: Optional[_dtype]) -> Tensor + + Returns the product of all elements in the :attr:`input` tensor. + + Args: + input (Tensor): the input tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[-0.8020, 0.5428, -1.5854]]) + >>> torch.prod(a) + tensor(0.6902) + + .. function:: prod(input, dim, keepdim=False, *, dtype=None) -> Tensor + :noindex: + + Returns the product of each row of the :attr:`input` tensor in the given + dimension :attr:`dim`. + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in + the output tensor having 1 fewer dimension than :attr:`input`. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> a = torch.randn(4, 2) + >>> a + tensor([[ 0.5261, -0.3837], + [ 1.1857, -0.2498], + [-1.1646, 0.0705], + [ 1.1131, -1.0629]]) + >>> torch.prod(a, 1) + tensor([-0.2018, -0.2962, -0.0821, -1.1831]) + """ + +@overload +def prod( + input: Tensor, + dim: str | EllipsisType | None, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + prod(input: Tensor, *, dtype: Optional[_dtype]) -> Tensor + + Returns the product of all elements in the :attr:`input` tensor. + + Args: + input (Tensor): the input tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[-0.8020, 0.5428, -1.5854]]) + >>> torch.prod(a) + tensor(0.6902) + + .. function:: prod(input, dim, keepdim=False, *, dtype=None) -> Tensor + :noindex: + + Returns the product of each row of the :attr:`input` tensor in the given + dimension :attr:`dim`. + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in + the output tensor having 1 fewer dimension than :attr:`input`. + + Args: + input (Tensor): the input tensor. + + dim (int, optional): the dimension to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> a = torch.randn(4, 2) + >>> a + tensor([[ 0.5261, -0.3837], + [ 1.1857, -0.2498], + [-1.1646, 0.0705], + [ 1.1131, -1.0629]]) + >>> torch.prod(a, 1) + tensor([-0.2018, -0.2962, -0.0821, -1.1831]) + """ + +def promote_types(type1: _dtype, type2: _dtype) -> _dtype: + r""" + promote_types(type1, type2) -> dtype + + Returns the :class:`torch.dtype` with the smallest size and scalar kind that is + not smaller nor of lower kind than either `type1` or `type2`. See type promotion + :ref:`documentation ` for more information on the type + promotion logic. + + Args: + type1 (:class:`torch.dtype`) + type2 (:class:`torch.dtype`) + + Example:: + + >>> torch.promote_types(torch.int32, torch.float32) + torch.float32 + >>> torch.promote_types(torch.uint8, torch.long) + torch.long + """ + +def put( + input: Tensor, + index: Tensor, + source: Tensor, + accumulate: _bool = False, +) -> Tensor: ... +def q_per_channel_axis(input: Tensor) -> _int: ... +def q_per_channel_scales(input: Tensor) -> Tensor: ... +def q_per_channel_zero_points(input: Tensor) -> Tensor: ... +def q_scale(input: Tensor) -> _float: ... +def q_zero_point(input: Tensor) -> _int: ... +def qr( + input: Tensor, + some: _bool = True, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.qr: + r""" + qr(input: Tensor, some: bool = True, *, out: Union[Tensor, Tuple[Tensor, ...], List[Tensor], None]) -> (Tensor, Tensor) + + Computes the QR decomposition of a matrix or a batch of matrices :attr:`input`, + and returns a namedtuple (Q, R) of tensors such that :math:`\text{input} = Q R` + with :math:`Q` being an orthogonal matrix or batch of orthogonal matrices and + :math:`R` being an upper triangular matrix or batch of upper triangular matrices. + + If :attr:`some` is ``True``, then this function returns the thin (reduced) QR factorization. + Otherwise, if :attr:`some` is ``False``, this function returns the complete QR factorization. + + .. warning:: + + :func:`torch.qr` is deprecated in favor of :func:`torch.linalg.qr` + and will be removed in a future PyTorch release. The boolean parameter :attr:`some` has been + replaced with a string parameter :attr:`mode`. + + ``Q, R = torch.qr(A)`` should be replaced with + + .. code:: python + + Q, R = torch.linalg.qr(A) + + ``Q, R = torch.qr(A, some=False)`` should be replaced with + + .. code:: python + + Q, R = torch.linalg.qr(A, mode="complete") + + .. warning:: + If you plan to backpropagate through QR, note that the current backward implementation + is only well-defined when the first :math:`\min(input.size(-1), input.size(-2))` + columns of :attr:`input` are linearly independent. + This behavior will probably change once QR supports pivoting. + + .. note:: This function uses LAPACK for CPU inputs and MAGMA for CUDA inputs, + and may produce different (valid) decompositions on different device types + or different platforms. + + Args: + input (Tensor): the input tensor of size :math:`(*, m, n)` where `*` is zero or more + batch dimensions consisting of matrices of dimension :math:`m \times n`. + some (bool, optional): Set to ``True`` for reduced QR decomposition and ``False`` for + complete QR decomposition. If `k = min(m, n)` then: + + * ``some=True`` : returns `(Q, R)` with dimensions (m, k), (k, n) (default) + + * ``'some=False'``: returns `(Q, R)` with dimensions (m, m), (m, n) + + Keyword args: + out (tuple, optional): tuple of `Q` and `R` tensors. + The dimensions of `Q` and `R` are detailed in the description of :attr:`some` above. + + Example:: + + >>> a = torch.tensor([[12., -51, 4], [6, 167, -68], [-4, 24, -41]]) + >>> q, r = torch.qr(a) + >>> q + tensor([[-0.8571, 0.3943, 0.3314], + [-0.4286, -0.9029, -0.0343], + [ 0.2857, -0.1714, 0.9429]]) + >>> r + tensor([[ -14.0000, -21.0000, 14.0000], + [ 0.0000, -175.0000, 70.0000], + [ 0.0000, 0.0000, -35.0000]]) + >>> torch.mm(q, r).round() + tensor([[ 12., -51., 4.], + [ 6., 167., -68.], + [ -4., 24., -41.]]) + >>> torch.mm(q.t(), q).round() + tensor([[ 1., 0., 0.], + [ 0., 1., -0.], + [ 0., -0., 1.]]) + >>> a = torch.randn(3, 4, 5) + >>> q, r = torch.qr(a, some=False) + >>> torch.allclose(torch.matmul(q, r), a) + True + >>> torch.allclose(torch.matmul(q.mT, q), torch.eye(5)) + True + """ + +@overload +def quantile( + input: Tensor, + q: Tensor, + dim: _int | None = None, + keepdim: _bool = False, + *, + interpolation: str = "linear", + out: Tensor | None = None, +) -> Tensor: + r""" + quantile(input, q, dim=None, keepdim=False, *, interpolation='linear', out=None) -> Tensor + + Computes the q-th quantiles of each row of the :attr:`input` tensor along the dimension :attr:`dim`. + + To compute the quantile, we map q in [0, 1] to the range of indices [0, n] to find the location + of the quantile in the sorted input. If the quantile lies between two data points ``a < b`` with + indices ``i`` and ``j`` in the sorted order, result is computed according to the given + :attr:`interpolation` method as follows: + + - ``linear``: ``a + (b - a) * fraction``, where ``fraction`` is the fractional part of the computed quantile index. + - ``lower``: ``a``. + - ``higher``: ``b``. + - ``nearest``: ``a`` or ``b``, whichever's index is closer to the computed quantile index (follows :func:`torch.round`). + - ``midpoint``: ``(a + b) / 2``. + + If :attr:`q` is a 1D tensor, the first dimension of the output represents the quantiles and has size + equal to the size of :attr:`q`, the remaining dimensions are what remains from the reduction. + + .. note:: + By default :attr:`dim` is ``None`` resulting in the :attr:`input` tensor being flattened before computation. + + Args: + input (Tensor): the input tensor. + q (float or Tensor): a scalar or 1D tensor of values in the range [0, 1]. + + dim (int, optional): the dimension to reduce. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword arguments: + interpolation (str, optional): interpolation method to use when the desired quantile lies between two data points. + Can be ``linear``, ``lower``, ``higher``, ``midpoint`` and ``nearest``. + Default is ``linear``. + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(2, 3) + >>> a + tensor([[ 0.0795, -1.2117, 0.9765], + [ 1.1707, 0.6706, 0.4884]]) + >>> q = torch.tensor([0.25, 0.5, 0.75]) + >>> torch.quantile(a, q, dim=1, keepdim=True) + tensor([[[-0.5661], + [ 0.5795]], + + [[ 0.0795], + [ 0.6706]], + + [[ 0.5280], + [ 0.9206]]]) + >>> torch.quantile(a, q, dim=1, keepdim=True).shape + torch.Size([3, 2, 1]) + >>> a = torch.arange(4.) + >>> a + tensor([0., 1., 2., 3.]) + >>> torch.quantile(a, 0.6, interpolation='linear') + tensor(1.8000) + >>> torch.quantile(a, 0.6, interpolation='lower') + tensor(1.) + >>> torch.quantile(a, 0.6, interpolation='higher') + tensor(2.) + >>> torch.quantile(a, 0.6, interpolation='midpoint') + tensor(1.5000) + >>> torch.quantile(a, 0.6, interpolation='nearest') + tensor(2.) + >>> torch.quantile(a, 0.4, interpolation='nearest') + tensor(1.) + """ + +@overload +def quantile( + input: Tensor, + q: _float, + dim: _int | None = None, + keepdim: _bool = False, + *, + interpolation: str = "linear", + out: Tensor | None = None, +) -> Tensor: + r""" + quantile(input, q, dim=None, keepdim=False, *, interpolation='linear', out=None) -> Tensor + + Computes the q-th quantiles of each row of the :attr:`input` tensor along the dimension :attr:`dim`. + + To compute the quantile, we map q in [0, 1] to the range of indices [0, n] to find the location + of the quantile in the sorted input. If the quantile lies between two data points ``a < b`` with + indices ``i`` and ``j`` in the sorted order, result is computed according to the given + :attr:`interpolation` method as follows: + + - ``linear``: ``a + (b - a) * fraction``, where ``fraction`` is the fractional part of the computed quantile index. + - ``lower``: ``a``. + - ``higher``: ``b``. + - ``nearest``: ``a`` or ``b``, whichever's index is closer to the computed quantile index (follows :func:`torch.round`). + - ``midpoint``: ``(a + b) / 2``. + + If :attr:`q` is a 1D tensor, the first dimension of the output represents the quantiles and has size + equal to the size of :attr:`q`, the remaining dimensions are what remains from the reduction. + + .. note:: + By default :attr:`dim` is ``None`` resulting in the :attr:`input` tensor being flattened before computation. + + Args: + input (Tensor): the input tensor. + q (float or Tensor): a scalar or 1D tensor of values in the range [0, 1]. + + dim (int, optional): the dimension to reduce. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword arguments: + interpolation (str, optional): interpolation method to use when the desired quantile lies between two data points. + Can be ``linear``, ``lower``, ``higher``, ``midpoint`` and ``nearest``. + Default is ``linear``. + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(2, 3) + >>> a + tensor([[ 0.0795, -1.2117, 0.9765], + [ 1.1707, 0.6706, 0.4884]]) + >>> q = torch.tensor([0.25, 0.5, 0.75]) + >>> torch.quantile(a, q, dim=1, keepdim=True) + tensor([[[-0.5661], + [ 0.5795]], + + [[ 0.0795], + [ 0.6706]], + + [[ 0.5280], + [ 0.9206]]]) + >>> torch.quantile(a, q, dim=1, keepdim=True).shape + torch.Size([3, 2, 1]) + >>> a = torch.arange(4.) + >>> a + tensor([0., 1., 2., 3.]) + >>> torch.quantile(a, 0.6, interpolation='linear') + tensor(1.8000) + >>> torch.quantile(a, 0.6, interpolation='lower') + tensor(1.) + >>> torch.quantile(a, 0.6, interpolation='higher') + tensor(2.) + >>> torch.quantile(a, 0.6, interpolation='midpoint') + tensor(1.5000) + >>> torch.quantile(a, 0.6, interpolation='nearest') + tensor(2.) + >>> torch.quantile(a, 0.4, interpolation='nearest') + tensor(1.) + """ + +def quantize_per_channel( + input: Tensor, + scales: Tensor, + zero_points: Tensor, + axis: _int, + dtype: _dtype, +) -> Tensor: + r""" + quantize_per_channel(input, scales, zero_points, axis, dtype) -> Tensor + + Converts a float tensor to a per-channel quantized tensor with given scales and zero points. + + Arguments: + input (Tensor): float tensor to quantize + scales (Tensor): float 1D tensor of scales to use, size should match ``input.size(axis)`` + zero_points (int): integer 1D tensor of offset to use, size should match ``input.size(axis)`` + axis (int): dimension on which apply per-channel quantization + dtype (:class:`torch.dtype`): the desired data type of returned tensor. + Has to be one of the quantized dtypes: ``torch.quint8``, ``torch.qint8``, ``torch.qint32`` + + Returns: + Tensor: A newly quantized tensor + + Example:: + + >>> x = torch.tensor([[-1.0, 0.0], [1.0, 2.0]]) + >>> torch.quantize_per_channel(x, torch.tensor([0.1, 0.01]), torch.tensor([10, 0]), 0, torch.quint8) + tensor([[-1., 0.], + [ 1., 2.]], size=(2, 2), dtype=torch.quint8, + quantization_scheme=torch.per_channel_affine, + scale=tensor([0.1000, 0.0100], dtype=torch.float64), + zero_point=tensor([10, 0]), axis=0) + >>> torch.quantize_per_channel(x, torch.tensor([0.1, 0.01]), torch.tensor([10, 0]), 0, torch.quint8).int_repr() + tensor([[ 0, 10], + [100, 200]], dtype=torch.uint8) + """ + +@overload +def quantize_per_tensor( + input: Tensor, + scale: Tensor, + zero_point: Tensor, + dtype: _dtype, +) -> Tensor: + r""" + quantize_per_tensor(input, scale, zero_point, dtype) -> Tensor + + Converts a float tensor to a quantized tensor with given scale and zero point. + + Arguments: + input (Tensor): float tensor or list of tensors to quantize + scale (float or Tensor): scale to apply in quantization formula + zero_point (int or Tensor): offset in integer value that maps to float zero + dtype (:class:`torch.dtype`): the desired data type of returned tensor. + Has to be one of the quantized dtypes: ``torch.quint8``, ``torch.qint8``, ``torch.qint32`` + + Returns: + Tensor: A newly quantized tensor or list of quantized tensors. + + Example:: + + >>> torch.quantize_per_tensor(torch.tensor([-1.0, 0.0, 1.0, 2.0]), 0.1, 10, torch.quint8) + tensor([-1., 0., 1., 2.], size=(4,), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.1, zero_point=10) + >>> torch.quantize_per_tensor(torch.tensor([-1.0, 0.0, 1.0, 2.0]), 0.1, 10, torch.quint8).int_repr() + tensor([ 0, 10, 20, 30], dtype=torch.uint8) + >>> torch.quantize_per_tensor([torch.tensor([-1.0, 0.0]), torch.tensor([-2.0, 2.0])], + >>> torch.tensor([0.1, 0.2]), torch.tensor([10, 20]), torch.quint8) + (tensor([-1., 0.], size=(2,), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.1, zero_point=10), + tensor([-2., 2.], size=(2,), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.2, zero_point=20)) + >>> torch.quantize_per_tensor(torch.tensor([-1.0, 0.0, 1.0, 2.0]), torch.tensor(0.1), torch.tensor(10), torch.quint8) + tensor([-1., 0., 1., 2.], size=(4,), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.10, zero_point=10) + """ + +@overload +def quantize_per_tensor( + input: Tensor, + scale: _float, + zero_point: _int, + dtype: _dtype, +) -> Tensor: + r""" + quantize_per_tensor(input, scale, zero_point, dtype) -> Tensor + + Converts a float tensor to a quantized tensor with given scale and zero point. + + Arguments: + input (Tensor): float tensor or list of tensors to quantize + scale (float or Tensor): scale to apply in quantization formula + zero_point (int or Tensor): offset in integer value that maps to float zero + dtype (:class:`torch.dtype`): the desired data type of returned tensor. + Has to be one of the quantized dtypes: ``torch.quint8``, ``torch.qint8``, ``torch.qint32`` + + Returns: + Tensor: A newly quantized tensor or list of quantized tensors. + + Example:: + + >>> torch.quantize_per_tensor(torch.tensor([-1.0, 0.0, 1.0, 2.0]), 0.1, 10, torch.quint8) + tensor([-1., 0., 1., 2.], size=(4,), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.1, zero_point=10) + >>> torch.quantize_per_tensor(torch.tensor([-1.0, 0.0, 1.0, 2.0]), 0.1, 10, torch.quint8).int_repr() + tensor([ 0, 10, 20, 30], dtype=torch.uint8) + >>> torch.quantize_per_tensor([torch.tensor([-1.0, 0.0]), torch.tensor([-2.0, 2.0])], + >>> torch.tensor([0.1, 0.2]), torch.tensor([10, 20]), torch.quint8) + (tensor([-1., 0.], size=(2,), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.1, zero_point=10), + tensor([-2., 2.], size=(2,), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.2, zero_point=20)) + >>> torch.quantize_per_tensor(torch.tensor([-1.0, 0.0, 1.0, 2.0]), torch.tensor(0.1), torch.tensor(10), torch.quint8) + tensor([-1., 0., 1., 2.], size=(4,), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.10, zero_point=10) + """ + +@overload +def quantize_per_tensor( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + scales: Tensor, + zero_points: Tensor, + dtype: _dtype, +) -> tuple[Tensor, ...]: + r""" + quantize_per_tensor(input, scale, zero_point, dtype) -> Tensor + + Converts a float tensor to a quantized tensor with given scale and zero point. + + Arguments: + input (Tensor): float tensor or list of tensors to quantize + scale (float or Tensor): scale to apply in quantization formula + zero_point (int or Tensor): offset in integer value that maps to float zero + dtype (:class:`torch.dtype`): the desired data type of returned tensor. + Has to be one of the quantized dtypes: ``torch.quint8``, ``torch.qint8``, ``torch.qint32`` + + Returns: + Tensor: A newly quantized tensor or list of quantized tensors. + + Example:: + + >>> torch.quantize_per_tensor(torch.tensor([-1.0, 0.0, 1.0, 2.0]), 0.1, 10, torch.quint8) + tensor([-1., 0., 1., 2.], size=(4,), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.1, zero_point=10) + >>> torch.quantize_per_tensor(torch.tensor([-1.0, 0.0, 1.0, 2.0]), 0.1, 10, torch.quint8).int_repr() + tensor([ 0, 10, 20, 30], dtype=torch.uint8) + >>> torch.quantize_per_tensor([torch.tensor([-1.0, 0.0]), torch.tensor([-2.0, 2.0])], + >>> torch.tensor([0.1, 0.2]), torch.tensor([10, 20]), torch.quint8) + (tensor([-1., 0.], size=(2,), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.1, zero_point=10), + tensor([-2., 2.], size=(2,), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.2, zero_point=20)) + >>> torch.quantize_per_tensor(torch.tensor([-1.0, 0.0, 1.0, 2.0]), torch.tensor(0.1), torch.tensor(10), torch.quint8) + tensor([-1., 0., 1., 2.], size=(4,), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.10, zero_point=10) + """ + +def quantize_per_tensor_dynamic( + input: Tensor, + dtype: _dtype, + reduce_range: _bool, +) -> Tensor: + r""" + quantize_per_tensor_dynamic(input, dtype, reduce_range) -> Tensor + + Converts a float tensor to a quantized tensor with scale and zero_point calculated + dynamically based on the input. + + Arguments: + input (Tensor): float tensor or list of tensors to quantize + dtype (:class:`torch.dtype`): the desired data type of returned tensor. + Has to be one of the quantized dtypes: ``torch.quint8``, ``torch.qint8`` + reduce_range (bool): a flag to indicate whether to reduce the range of quantized + data by 1 bit, it's required to avoid instruction overflow for some hardwares + + Returns: + Tensor: A newly (dynamically) quantized tensor + + Example:: + + >>> t = torch.quantize_per_tensor_dynamic(torch.tensor([-1.0, 0.0, 1.0, 2.0]), torch.quint8, False) + >>> print(t) + tensor([-1., 0., 1., 2.], size=(4,), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.011764705882352941, + zero_point=85) + >>> t.int_repr() + tensor([ 0, 85, 170, 255], dtype=torch.uint8) + """ + +def quantized_batch_norm( + input: Tensor, + weight: Tensor | None, + bias: Tensor | None, + mean: Tensor, + var: Tensor, + eps: _float, + output_scale: _float, + output_zero_point: _int, +) -> Tensor: + r""" + quantized_batch_norm(input, weight=None, bias=None, mean, var, eps, output_scale, output_zero_point) -> Tensor + + Applies batch normalization on a 4D (NCHW) quantized tensor. + + .. math:: + + y = \frac{x - \mathrm{E}[x]}{\sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta + + Arguments: + input (Tensor): quantized tensor + weight (Tensor): float tensor that corresponds to the gamma, size C + bias (Tensor): float tensor that corresponds to the beta, size C + mean (Tensor): float mean value in batch normalization, size C + var (Tensor): float tensor for variance, size C + eps (float): a value added to the denominator for numerical stability. + output_scale (float): output quantized tensor scale + output_zero_point (int): output quantized tensor zero_point + + Returns: + Tensor: A quantized tensor with batch normalization applied. + + Example:: + + >>> qx = torch.quantize_per_tensor(torch.rand(2, 2, 2, 2), 1.5, 3, torch.quint8) + >>> torch.quantized_batch_norm(qx, torch.ones(2), torch.zeros(2), torch.rand(2), torch.rand(2), 0.00001, 0.2, 2) + tensor([[[[-0.2000, -0.2000], + [ 1.6000, -0.2000]], + + [[-0.4000, -0.4000], + [-0.4000, 0.6000]]], + + + [[[-0.2000, -0.2000], + [-0.2000, -0.2000]], + + [[ 0.6000, -0.4000], + [ 0.6000, -0.4000]]]], size=(2, 2, 2, 2), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=0.2, zero_point=2) + """ + +def quantized_gru_cell( + input: Tensor, + hx: Tensor, + w_ih: Tensor, + w_hh: Tensor, + b_ih: Tensor, + b_hh: Tensor, + packed_ih: Tensor, + packed_hh: Tensor, + col_offsets_ih: Tensor, + col_offsets_hh: Tensor, + scale_ih: Number | _complex, + scale_hh: Number | _complex, + zero_point_ih: Number | _complex, + zero_point_hh: Number | _complex, +) -> Tensor: ... +def quantized_lstm_cell( + input: Tensor, + hx: tuple[Tensor, ...] | list[Tensor] | None, + w_ih: Tensor, + w_hh: Tensor, + b_ih: Tensor, + b_hh: Tensor, + packed_ih: Tensor, + packed_hh: Tensor, + col_offsets_ih: Tensor, + col_offsets_hh: Tensor, + scale_ih: Number | _complex, + scale_hh: Number | _complex, + zero_point_ih: Number | _complex, + zero_point_hh: Number | _complex, +) -> tuple[Tensor, Tensor]: ... +def quantized_max_pool1d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size = (), + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: _bool = False, +) -> Tensor: + r""" + quantized_max_pool1d(input, kernel_size, stride=[], padding=0, dilation=1, ceil_mode=False) -> Tensor + + Applies a 1D max pooling over an input quantized tensor composed of several input planes. + + Arguments: + input (Tensor): quantized tensor + kernel_size (list of int): the size of the sliding window + stride (``list of int``, optional): the stride of the sliding window + padding (``list of int``, optional): padding to be added on both sides, must be >= 0 and <= kernel_size / 2 + dilation (``list of int``, optional): The stride between elements within a sliding window, must be > 0. Default 1 + ceil_mode (bool, optional): If True, will use ceil instead of floor to compute the output shape. + Defaults to False. + + + Returns: + Tensor: A quantized tensor with max_pool1d applied. + + Example:: + + >>> qx = torch.quantize_per_tensor(torch.rand(2, 2), 1.5, 3, torch.quint8) + >>> torch.quantized_max_pool1d(qx, [2]) + tensor([[0.0000], + [1.5000]], size=(2, 1), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=1.5, zero_point=3) + """ + +def quantized_max_pool2d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size = (), + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: _bool = False, +) -> Tensor: + r""" + quantized_max_pool2d(input, kernel_size, stride=[], padding=0, dilation=1, ceil_mode=False) -> Tensor + + Applies a 2D max pooling over an input quantized tensor composed of several input planes. + + Arguments: + input (Tensor): quantized tensor + kernel_size (``list of int``): the size of the sliding window + stride (``list of int``, optional): the stride of the sliding window + padding (``list of int``, optional): padding to be added on both sides, must be >= 0 and <= kernel_size / 2 + dilation (``list of int``, optional): The stride between elements within a sliding window, must be > 0. Default 1 + ceil_mode (bool, optional): If True, will use ceil instead of floor to compute the output shape. + Defaults to False. + + + Returns: + Tensor: A quantized tensor with max_pool2d applied. + + Example:: + + >>> qx = torch.quantize_per_tensor(torch.rand(2, 2, 2, 2), 1.5, 3, torch.quint8) + >>> torch.quantized_max_pool2d(qx, [2,2]) + tensor([[[[1.5000]], + + [[1.5000]]], + + + [[[0.0000]], + + [[0.0000]]]], size=(2, 2, 1, 1), dtype=torch.quint8, + quantization_scheme=torch.per_tensor_affine, scale=1.5, zero_point=3) + """ + +def quantized_max_pool3d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size = (), + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: _bool = False, +) -> Tensor: ... +def quantized_rnn_relu_cell( + input: Tensor, + hx: Tensor, + w_ih: Tensor, + w_hh: Tensor, + b_ih: Tensor, + b_hh: Tensor, + packed_ih: Tensor, + packed_hh: Tensor, + col_offsets_ih: Tensor, + col_offsets_hh: Tensor, + scale_ih: Number | _complex, + scale_hh: Number | _complex, + zero_point_ih: Number | _complex, + zero_point_hh: Number | _complex, +) -> Tensor: ... +def quantized_rnn_tanh_cell( + input: Tensor, + hx: Tensor, + w_ih: Tensor, + w_hh: Tensor, + b_ih: Tensor, + b_hh: Tensor, + packed_ih: Tensor, + packed_hh: Tensor, + col_offsets_ih: Tensor, + col_offsets_hh: Tensor, + scale_ih: Number | _complex, + scale_hh: Number | _complex, + zero_point_ih: Number | _complex, + zero_point_hh: Number | _complex, +) -> Tensor: ... +def rad2deg(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + rad2deg(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Returns a new tensor with each of the elements of :attr:`input` + converted from angles in radians to degrees. + + Args: + input (Tensor): the input tensor. + + Keyword arguments: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([[3.142, -3.142], [6.283, -6.283], [1.570, -1.570]]) + >>> torch.rad2deg(a) + tensor([[ 180.0233, -180.0233], + [ 359.9894, -359.9894], + [ 89.9544, -89.9544]]) + """ + +def rad2deg_(input: Tensor) -> Tensor: ... +@overload +def rand( + size: Sequence[_int | SymInt], + *, + generator: Generator | None, + names: Sequence[str | EllipsisType | None] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + rand(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + Returns a tensor filled with random numbers from a uniform distribution + on the interval :math:`[0, 1)` + + The shape of the tensor is defined by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.rand(4) + tensor([ 0.5204, 0.2503, 0.3525, 0.5673]) + >>> torch.rand(2, 3) + tensor([[ 0.8237, 0.5781, 0.6879], + [ 0.3816, 0.7249, 0.0998]]) + """ + +@overload +def rand( + *size: _int | SymInt, + generator: Generator | None, + names: Sequence[str | EllipsisType | None] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + rand(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + Returns a tensor filled with random numbers from a uniform distribution + on the interval :math:`[0, 1)` + + The shape of the tensor is defined by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.rand(4) + tensor([ 0.5204, 0.2503, 0.3525, 0.5673]) + >>> torch.rand(2, 3) + tensor([[ 0.8237, 0.5781, 0.6879], + [ 0.3816, 0.7249, 0.0998]]) + """ + +@overload +def rand( + size: Sequence[_int | SymInt], + *, + generator: Generator | None, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + rand(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + Returns a tensor filled with random numbers from a uniform distribution + on the interval :math:`[0, 1)` + + The shape of the tensor is defined by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.rand(4) + tensor([ 0.5204, 0.2503, 0.3525, 0.5673]) + >>> torch.rand(2, 3) + tensor([[ 0.8237, 0.5781, 0.6879], + [ 0.3816, 0.7249, 0.0998]]) + """ + +@overload +def rand( + *size: _int | SymInt, + generator: Generator | None, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + rand(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + Returns a tensor filled with random numbers from a uniform distribution + on the interval :math:`[0, 1)` + + The shape of the tensor is defined by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.rand(4) + tensor([ 0.5204, 0.2503, 0.3525, 0.5673]) + >>> torch.rand(2, 3) + tensor([[ 0.8237, 0.5781, 0.6879], + [ 0.3816, 0.7249, 0.0998]]) + """ + +@overload +def rand( + size: Sequence[_int | SymInt], + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + rand(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + Returns a tensor filled with random numbers from a uniform distribution + on the interval :math:`[0, 1)` + + The shape of the tensor is defined by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.rand(4) + tensor([ 0.5204, 0.2503, 0.3525, 0.5673]) + >>> torch.rand(2, 3) + tensor([[ 0.8237, 0.5781, 0.6879], + [ 0.3816, 0.7249, 0.0998]]) + """ + +@overload +def rand( + *size: _int | SymInt, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + rand(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + Returns a tensor filled with random numbers from a uniform distribution + on the interval :math:`[0, 1)` + + The shape of the tensor is defined by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.rand(4) + tensor([ 0.5204, 0.2503, 0.3525, 0.5673]) + >>> torch.rand(2, 3) + tensor([[ 0.8237, 0.5781, 0.6879], + [ 0.3816, 0.7249, 0.0998]]) + """ + +@overload +def rand( + size: Sequence[_int | SymInt], + *, + names: Sequence[str | EllipsisType | None] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + rand(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + Returns a tensor filled with random numbers from a uniform distribution + on the interval :math:`[0, 1)` + + The shape of the tensor is defined by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.rand(4) + tensor([ 0.5204, 0.2503, 0.3525, 0.5673]) + >>> torch.rand(2, 3) + tensor([[ 0.8237, 0.5781, 0.6879], + [ 0.3816, 0.7249, 0.0998]]) + """ + +@overload +def rand( + *size: _int | SymInt, + names: Sequence[str | EllipsisType | None] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + rand(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + Returns a tensor filled with random numbers from a uniform distribution + on the interval :math:`[0, 1)` + + The shape of the tensor is defined by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.rand(4) + tensor([ 0.5204, 0.2503, 0.3525, 0.5673]) + >>> torch.rand(2, 3) + tensor([[ 0.8237, 0.5781, 0.6879], + [ 0.3816, 0.7249, 0.0998]]) + """ + +@overload +def rand_like( + input: Tensor, + *, + generator: Generator | None, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + rand_like(input, *, generator=None, dtype=None, layout=None, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns a tensor with the same size as :attr:`input` that is filled with + random numbers from a uniform distribution on the interval :math:`[0, 1)`. + ``torch.rand_like(input)`` is equivalent to + ``torch.rand(input.size(), dtype=input.dtype, layout=input.layout, device=input.device)``. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling. + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + +@overload +def rand_like( + input: Tensor, + *, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + rand_like(input, *, generator=None, dtype=None, layout=None, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns a tensor with the same size as :attr:`input` that is filled with + random numbers from a uniform distribution on the interval :math:`[0, 1)`. + ``torch.rand_like(input)`` is equivalent to + ``torch.rand(input.size(), dtype=input.dtype, layout=input.layout, device=input.device)``. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling. + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + +@overload +def randint( + low: _int, + high: _int, + size: _size, + *, + generator: Generator | None = None, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + pin_memory: _bool = False, +) -> Tensor: + r""" + randint(low=0, high, size, \*, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with random integers generated uniformly + between :attr:`low` (inclusive) and :attr:`high` (exclusive). + + The shape of the tensor is defined by the variable argument :attr:`size`. + + .. note:: + With the global dtype default (``torch.float32``), this function returns + a tensor with dtype ``torch.int64``. + + Args: + low (int, optional): Lowest integer to be drawn from the distribution. Default: 0. + high (int): One above the highest integer to be drawn from the distribution. + size (tuple): a tuple defining the shape of the output tensor. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the desired data type of returned tensor. Default: if ``None``, + this function returns a tensor with dtype ``torch.int64``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.randint(3, 5, (3,)) + tensor([4, 3, 4]) + + + >>> torch.randint(10, (2, 2)) + tensor([[0, 2], + [5, 5]]) + + + >>> torch.randint(3, 10, (2, 2)) + tensor([[4, 5], + [6, 7]]) + """ + +@overload +def randint( + high: _int, + size: _size, + *, + generator: Generator | None = None, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + pin_memory: _bool = False, +) -> Tensor: + r""" + randint(low=0, high, size, \*, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with random integers generated uniformly + between :attr:`low` (inclusive) and :attr:`high` (exclusive). + + The shape of the tensor is defined by the variable argument :attr:`size`. + + .. note:: + With the global dtype default (``torch.float32``), this function returns + a tensor with dtype ``torch.int64``. + + Args: + low (int, optional): Lowest integer to be drawn from the distribution. Default: 0. + high (int): One above the highest integer to be drawn from the distribution. + size (tuple): a tuple defining the shape of the output tensor. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the desired data type of returned tensor. Default: if ``None``, + this function returns a tensor with dtype ``torch.int64``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.randint(3, 5, (3,)) + tensor([4, 3, 4]) + + + >>> torch.randint(10, (2, 2)) + tensor([[0, 2], + [5, 5]]) + + + >>> torch.randint(3, 10, (2, 2)) + tensor([[4, 5], + [6, 7]]) + """ + +@overload +def randint( + high: _int | SymInt, + size: Sequence[_int | SymInt], + *, + generator: Generator | None, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randint(low=0, high, size, \*, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with random integers generated uniformly + between :attr:`low` (inclusive) and :attr:`high` (exclusive). + + The shape of the tensor is defined by the variable argument :attr:`size`. + + .. note:: + With the global dtype default (``torch.float32``), this function returns + a tensor with dtype ``torch.int64``. + + Args: + low (int, optional): Lowest integer to be drawn from the distribution. Default: 0. + high (int): One above the highest integer to be drawn from the distribution. + size (tuple): a tuple defining the shape of the output tensor. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the desired data type of returned tensor. Default: if ``None``, + this function returns a tensor with dtype ``torch.int64``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.randint(3, 5, (3,)) + tensor([4, 3, 4]) + + + >>> torch.randint(10, (2, 2)) + tensor([[0, 2], + [5, 5]]) + + + >>> torch.randint(3, 10, (2, 2)) + tensor([[4, 5], + [6, 7]]) + """ + +@overload +def randint( + high: _int | SymInt, + size: Sequence[_int | SymInt], + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randint(low=0, high, size, \*, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with random integers generated uniformly + between :attr:`low` (inclusive) and :attr:`high` (exclusive). + + The shape of the tensor is defined by the variable argument :attr:`size`. + + .. note:: + With the global dtype default (``torch.float32``), this function returns + a tensor with dtype ``torch.int64``. + + Args: + low (int, optional): Lowest integer to be drawn from the distribution. Default: 0. + high (int): One above the highest integer to be drawn from the distribution. + size (tuple): a tuple defining the shape of the output tensor. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the desired data type of returned tensor. Default: if ``None``, + this function returns a tensor with dtype ``torch.int64``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.randint(3, 5, (3,)) + tensor([4, 3, 4]) + + + >>> torch.randint(10, (2, 2)) + tensor([[0, 2], + [5, 5]]) + + + >>> torch.randint(3, 10, (2, 2)) + tensor([[4, 5], + [6, 7]]) + """ + +@overload +def randint( + low: _int | SymInt, + high: _int | SymInt, + size: Sequence[_int | SymInt], + *, + generator: Generator | None, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randint(low=0, high, size, \*, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with random integers generated uniformly + between :attr:`low` (inclusive) and :attr:`high` (exclusive). + + The shape of the tensor is defined by the variable argument :attr:`size`. + + .. note:: + With the global dtype default (``torch.float32``), this function returns + a tensor with dtype ``torch.int64``. + + Args: + low (int, optional): Lowest integer to be drawn from the distribution. Default: 0. + high (int): One above the highest integer to be drawn from the distribution. + size (tuple): a tuple defining the shape of the output tensor. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the desired data type of returned tensor. Default: if ``None``, + this function returns a tensor with dtype ``torch.int64``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.randint(3, 5, (3,)) + tensor([4, 3, 4]) + + + >>> torch.randint(10, (2, 2)) + tensor([[0, 2], + [5, 5]]) + + + >>> torch.randint(3, 10, (2, 2)) + tensor([[4, 5], + [6, 7]]) + """ + +@overload +def randint( + low: _int | SymInt, + high: _int | SymInt, + size: Sequence[_int | SymInt], + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randint(low=0, high, size, \*, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with random integers generated uniformly + between :attr:`low` (inclusive) and :attr:`high` (exclusive). + + The shape of the tensor is defined by the variable argument :attr:`size`. + + .. note:: + With the global dtype default (``torch.float32``), this function returns + a tensor with dtype ``torch.int64``. + + Args: + low (int, optional): Lowest integer to be drawn from the distribution. Default: 0. + high (int): One above the highest integer to be drawn from the distribution. + size (tuple): a tuple defining the shape of the output tensor. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (torch.dtype, optional): the desired data type of returned tensor. Default: if ``None``, + this function returns a tensor with dtype ``torch.int64``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.randint(3, 5, (3,)) + tensor([4, 3, 4]) + + + >>> torch.randint(10, (2, 2)) + tensor([[0, 2], + [5, 5]]) + + + >>> torch.randint(3, 10, (2, 2)) + tensor([[4, 5], + [6, 7]]) + """ + +@overload +def randint_like( + input: Tensor, + low: _int | SymInt, + high: _int | SymInt, + *, + generator: Generator | None, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randint_like(input, low=0, high, \*, generator=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns a tensor with the same shape as Tensor :attr:`input` filled with + random integers generated uniformly between :attr:`low` (inclusive) and + :attr:`high` (exclusive). + + .. note: + With the global dtype default (``torch.float32``), this function returns + a tensor with dtype ``torch.int64``. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + low (int, optional): Lowest integer to be drawn from the distribution. Default: 0. + high (int): One above the highest integer to be drawn from the distribution. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling. + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + +@overload +def randint_like( + input: Tensor, + low: _int | SymInt, + high: _int | SymInt, + *, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randint_like(input, low=0, high, \*, generator=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns a tensor with the same shape as Tensor :attr:`input` filled with + random integers generated uniformly between :attr:`low` (inclusive) and + :attr:`high` (exclusive). + + .. note: + With the global dtype default (``torch.float32``), this function returns + a tensor with dtype ``torch.int64``. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + low (int, optional): Lowest integer to be drawn from the distribution. Default: 0. + high (int): One above the highest integer to be drawn from the distribution. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling. + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + +@overload +def randint_like( + input: Tensor, + high: Tensor, + *, + generator: Generator | None, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randint_like(input, low=0, high, \*, generator=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns a tensor with the same shape as Tensor :attr:`input` filled with + random integers generated uniformly between :attr:`low` (inclusive) and + :attr:`high` (exclusive). + + .. note: + With the global dtype default (``torch.float32``), this function returns + a tensor with dtype ``torch.int64``. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + low (int, optional): Lowest integer to be drawn from the distribution. Default: 0. + high (int): One above the highest integer to be drawn from the distribution. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling. + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + +@overload +def randint_like( + input: Tensor, + high: Tensor, + *, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randint_like(input, low=0, high, \*, generator=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns a tensor with the same shape as Tensor :attr:`input` filled with + random integers generated uniformly between :attr:`low` (inclusive) and + :attr:`high` (exclusive). + + .. note: + With the global dtype default (``torch.float32``), this function returns + a tensor with dtype ``torch.int64``. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + low (int, optional): Lowest integer to be drawn from the distribution. Default: 0. + high (int): One above the highest integer to be drawn from the distribution. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling. + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + +@overload +def randint_like( + input: Tensor, + high: _int | SymInt, + *, + generator: Generator | None, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randint_like(input, low=0, high, \*, generator=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns a tensor with the same shape as Tensor :attr:`input` filled with + random integers generated uniformly between :attr:`low` (inclusive) and + :attr:`high` (exclusive). + + .. note: + With the global dtype default (``torch.float32``), this function returns + a tensor with dtype ``torch.int64``. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + low (int, optional): Lowest integer to be drawn from the distribution. Default: 0. + high (int): One above the highest integer to be drawn from the distribution. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling. + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + +@overload +def randint_like( + input: Tensor, + high: _int | SymInt, + *, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randint_like(input, low=0, high, \*, generator=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns a tensor with the same shape as Tensor :attr:`input` filled with + random integers generated uniformly between :attr:`low` (inclusive) and + :attr:`high` (exclusive). + + .. note: + With the global dtype default (``torch.float32``), this function returns + a tensor with dtype ``torch.int64``. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + low (int, optional): Lowest integer to be drawn from the distribution. Default: 0. + high (int): One above the highest integer to be drawn from the distribution. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling. + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + +@overload +def randn( + size: Sequence[_int | SymInt], + *, + generator: Generator | None, + names: Sequence[str | EllipsisType | None] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randn(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + + Returns a tensor filled with random numbers from a normal distribution + with mean `0` and variance `1` (also called the standard normal + distribution). + + .. math:: + \text{out}_{i} \sim \mathcal{N}(0, 1) + + For complex dtypes, the tensor is i.i.d. sampled from a `complex normal distribution`_ with zero mean and + unit variance as + + .. math:: + \text{out}_{i} \sim \mathcal{CN}(0, 1) + + This is equivalent to separately sampling the real :math:`(\operatorname{Re})` and imaginary + :math:`(\operatorname{Im})` part of :math:`\text{out}_i` as + + .. math:: + \operatorname{Re}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}),\quad + \operatorname{Im}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}) + + The shape of the tensor is defined by the variable argument :attr:`size`. + + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.randn(4) + tensor([-2.1436, 0.9966, 2.3426, -0.6366]) + >>> torch.randn(2, 3) + tensor([[ 1.5954, 2.8929, -1.0923], + [ 1.1719, -0.4709, -0.1996]]) + + .. _complex normal distribution: https://en.wikipedia.org/wiki/Complex_normal_distribution + """ + +@overload +def randn( + *size: _int | SymInt, + generator: Generator | None, + names: Sequence[str | EllipsisType | None] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randn(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + + Returns a tensor filled with random numbers from a normal distribution + with mean `0` and variance `1` (also called the standard normal + distribution). + + .. math:: + \text{out}_{i} \sim \mathcal{N}(0, 1) + + For complex dtypes, the tensor is i.i.d. sampled from a `complex normal distribution`_ with zero mean and + unit variance as + + .. math:: + \text{out}_{i} \sim \mathcal{CN}(0, 1) + + This is equivalent to separately sampling the real :math:`(\operatorname{Re})` and imaginary + :math:`(\operatorname{Im})` part of :math:`\text{out}_i` as + + .. math:: + \operatorname{Re}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}),\quad + \operatorname{Im}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}) + + The shape of the tensor is defined by the variable argument :attr:`size`. + + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.randn(4) + tensor([-2.1436, 0.9966, 2.3426, -0.6366]) + >>> torch.randn(2, 3) + tensor([[ 1.5954, 2.8929, -1.0923], + [ 1.1719, -0.4709, -0.1996]]) + + .. _complex normal distribution: https://en.wikipedia.org/wiki/Complex_normal_distribution + """ + +@overload +def randn( + size: Sequence[_int | SymInt], + *, + generator: Generator | None, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randn(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + + Returns a tensor filled with random numbers from a normal distribution + with mean `0` and variance `1` (also called the standard normal + distribution). + + .. math:: + \text{out}_{i} \sim \mathcal{N}(0, 1) + + For complex dtypes, the tensor is i.i.d. sampled from a `complex normal distribution`_ with zero mean and + unit variance as + + .. math:: + \text{out}_{i} \sim \mathcal{CN}(0, 1) + + This is equivalent to separately sampling the real :math:`(\operatorname{Re})` and imaginary + :math:`(\operatorname{Im})` part of :math:`\text{out}_i` as + + .. math:: + \operatorname{Re}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}),\quad + \operatorname{Im}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}) + + The shape of the tensor is defined by the variable argument :attr:`size`. + + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.randn(4) + tensor([-2.1436, 0.9966, 2.3426, -0.6366]) + >>> torch.randn(2, 3) + tensor([[ 1.5954, 2.8929, -1.0923], + [ 1.1719, -0.4709, -0.1996]]) + + .. _complex normal distribution: https://en.wikipedia.org/wiki/Complex_normal_distribution + """ + +@overload +def randn( + *size: _int | SymInt, + generator: Generator | None, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randn(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + + Returns a tensor filled with random numbers from a normal distribution + with mean `0` and variance `1` (also called the standard normal + distribution). + + .. math:: + \text{out}_{i} \sim \mathcal{N}(0, 1) + + For complex dtypes, the tensor is i.i.d. sampled from a `complex normal distribution`_ with zero mean and + unit variance as + + .. math:: + \text{out}_{i} \sim \mathcal{CN}(0, 1) + + This is equivalent to separately sampling the real :math:`(\operatorname{Re})` and imaginary + :math:`(\operatorname{Im})` part of :math:`\text{out}_i` as + + .. math:: + \operatorname{Re}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}),\quad + \operatorname{Im}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}) + + The shape of the tensor is defined by the variable argument :attr:`size`. + + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.randn(4) + tensor([-2.1436, 0.9966, 2.3426, -0.6366]) + >>> torch.randn(2, 3) + tensor([[ 1.5954, 2.8929, -1.0923], + [ 1.1719, -0.4709, -0.1996]]) + + .. _complex normal distribution: https://en.wikipedia.org/wiki/Complex_normal_distribution + """ + +@overload +def randn( + size: Sequence[_int | SymInt], + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randn(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + + Returns a tensor filled with random numbers from a normal distribution + with mean `0` and variance `1` (also called the standard normal + distribution). + + .. math:: + \text{out}_{i} \sim \mathcal{N}(0, 1) + + For complex dtypes, the tensor is i.i.d. sampled from a `complex normal distribution`_ with zero mean and + unit variance as + + .. math:: + \text{out}_{i} \sim \mathcal{CN}(0, 1) + + This is equivalent to separately sampling the real :math:`(\operatorname{Re})` and imaginary + :math:`(\operatorname{Im})` part of :math:`\text{out}_i` as + + .. math:: + \operatorname{Re}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}),\quad + \operatorname{Im}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}) + + The shape of the tensor is defined by the variable argument :attr:`size`. + + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.randn(4) + tensor([-2.1436, 0.9966, 2.3426, -0.6366]) + >>> torch.randn(2, 3) + tensor([[ 1.5954, 2.8929, -1.0923], + [ 1.1719, -0.4709, -0.1996]]) + + .. _complex normal distribution: https://en.wikipedia.org/wiki/Complex_normal_distribution + """ + +@overload +def randn( + *size: _int | SymInt, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randn(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + + Returns a tensor filled with random numbers from a normal distribution + with mean `0` and variance `1` (also called the standard normal + distribution). + + .. math:: + \text{out}_{i} \sim \mathcal{N}(0, 1) + + For complex dtypes, the tensor is i.i.d. sampled from a `complex normal distribution`_ with zero mean and + unit variance as + + .. math:: + \text{out}_{i} \sim \mathcal{CN}(0, 1) + + This is equivalent to separately sampling the real :math:`(\operatorname{Re})` and imaginary + :math:`(\operatorname{Im})` part of :math:`\text{out}_i` as + + .. math:: + \operatorname{Re}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}),\quad + \operatorname{Im}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}) + + The shape of the tensor is defined by the variable argument :attr:`size`. + + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.randn(4) + tensor([-2.1436, 0.9966, 2.3426, -0.6366]) + >>> torch.randn(2, 3) + tensor([[ 1.5954, 2.8929, -1.0923], + [ 1.1719, -0.4709, -0.1996]]) + + .. _complex normal distribution: https://en.wikipedia.org/wiki/Complex_normal_distribution + """ + +@overload +def randn( + size: Sequence[_int | SymInt], + *, + names: Sequence[str | EllipsisType | None] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randn(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + + Returns a tensor filled with random numbers from a normal distribution + with mean `0` and variance `1` (also called the standard normal + distribution). + + .. math:: + \text{out}_{i} \sim \mathcal{N}(0, 1) + + For complex dtypes, the tensor is i.i.d. sampled from a `complex normal distribution`_ with zero mean and + unit variance as + + .. math:: + \text{out}_{i} \sim \mathcal{CN}(0, 1) + + This is equivalent to separately sampling the real :math:`(\operatorname{Re})` and imaginary + :math:`(\operatorname{Im})` part of :math:`\text{out}_i` as + + .. math:: + \operatorname{Re}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}),\quad + \operatorname{Im}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}) + + The shape of the tensor is defined by the variable argument :attr:`size`. + + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.randn(4) + tensor([-2.1436, 0.9966, 2.3426, -0.6366]) + >>> torch.randn(2, 3) + tensor([[ 1.5954, 2.8929, -1.0923], + [ 1.1719, -0.4709, -0.1996]]) + + .. _complex normal distribution: https://en.wikipedia.org/wiki/Complex_normal_distribution + """ + +@overload +def randn( + *size: _int | SymInt, + names: Sequence[str | EllipsisType | None] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randn(*size, *, generator=None, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + + Returns a tensor filled with random numbers from a normal distribution + with mean `0` and variance `1` (also called the standard normal + distribution). + + .. math:: + \text{out}_{i} \sim \mathcal{N}(0, 1) + + For complex dtypes, the tensor is i.i.d. sampled from a `complex normal distribution`_ with zero mean and + unit variance as + + .. math:: + \text{out}_{i} \sim \mathcal{CN}(0, 1) + + This is equivalent to separately sampling the real :math:`(\operatorname{Re})` and imaginary + :math:`(\operatorname{Im})` part of :math:`\text{out}_i` as + + .. math:: + \operatorname{Re}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}),\quad + \operatorname{Im}(\text{out}_{i}) \sim \mathcal{N}(0, \frac{1}{2}) + + The shape of the tensor is defined by the variable argument :attr:`size`. + + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.randn(4) + tensor([-2.1436, 0.9966, 2.3426, -0.6366]) + >>> torch.randn(2, 3) + tensor([[ 1.5954, 2.8929, -1.0923], + [ 1.1719, -0.4709, -0.1996]]) + + .. _complex normal distribution: https://en.wikipedia.org/wiki/Complex_normal_distribution + """ + +@overload +def randn_like( + input: Tensor, + *, + generator: Generator | None, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randn_like(input, *, generator=None, dtype=None, layout=None, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns a tensor with the same size as :attr:`input` that is filled with + random numbers from a normal distribution with mean 0 and variance 1. Please refer to :func:`torch.randn` for the + sampling process of complex dtypes. ``torch.randn_like(input)`` is equivalent to + ``torch.randn(input.size(), dtype=input.dtype, layout=input.layout, device=input.device)``. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling. + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + +@overload +def randn_like( + input: Tensor, + *, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randn_like(input, *, generator=None, dtype=None, layout=None, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns a tensor with the same size as :attr:`input` that is filled with + random numbers from a normal distribution with mean 0 and variance 1. Please refer to :func:`torch.randn` for the + sampling process of complex dtypes. ``torch.randn_like(input)`` is equivalent to + ``torch.randn(input.size(), dtype=input.dtype, layout=input.layout, device=input.device)``. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling. + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + +@overload +def randperm( + n: _int | SymInt, + *, + generator: Generator | None, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randperm(n, *, generator=None, out=None, dtype=torch.int64,layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + Returns a random permutation of integers from ``0`` to ``n - 1``. + + Args: + n (int): the upper bound (exclusive) + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: ``torch.int64``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.randperm(4) + tensor([2, 1, 0, 3]) + """ + +@overload +def randperm( + n: _int | SymInt, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + randperm(n, *, generator=None, out=None, dtype=torch.int64,layout=torch.strided, device=None, requires_grad=False, pin_memory=False) -> Tensor + + Returns a random permutation of integers from ``0`` to ``n - 1``. + + Args: + n (int): the upper bound (exclusive) + + Keyword args: + generator (:class:`torch.Generator`, optional): a pseudorandom number generator for sampling + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: ``torch.int64``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> torch.randperm(4) + tensor([2, 1, 0, 3]) + """ + +def range( + start: Number, + end: Number, + step: Number = 1, + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + pin_memory: _bool = False, +) -> Tensor: + r""" + range(start=0, end, step=1, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a 1-D tensor of size :math:`\left\lfloor \frac{\text{end} - \text{start}}{\text{step}} \right\rfloor + 1` + with values from :attr:`start` to :attr:`end` with step :attr:`step`. Step is + the gap between two values in the tensor. + + .. math:: + \text{out}_{i+1} = \text{out}_i + \text{step}. + + .. warning:: + This function is deprecated and will be removed in a future release because its behavior is inconsistent with + Python's range builtin. Instead, use :func:`torch.arange`, which produces values in [start, end). + + Args: + start (float, optional): the starting value for the set of points. Default: ``0``. + end (float): the ending value for the set of points + step (float, optional): the gap between each pair of adjacent points. Default: ``1``. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). If `dtype` is not given, infer the data type from the other input + arguments. If any of `start`, `end`, or `step` are floating-point, the + `dtype` is inferred to be the default dtype, see + :meth:`~torch.get_default_dtype`. Otherwise, the `dtype` is inferred to + be `torch.int64`. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.range(1, 4) + tensor([ 1., 2., 3., 4.]) + >>> torch.range(1, 4, 0.5) + tensor([ 1.0000, 1.5000, 2.0000, 2.5000, 3.0000, 3.5000, 4.0000]) + """ + +def ravel(input: Tensor) -> Tensor: + r""" + ravel(input) -> Tensor + + Return a contiguous flattened tensor. A copy is made only if needed. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> t = torch.tensor([[[1, 2], + ... [3, 4]], + ... [[5, 6], + ... [7, 8]]]) + >>> torch.ravel(t) + tensor([1, 2, 3, 4, 5, 6, 7, 8]) + """ + +def real(input: Tensor) -> Tensor: + r""" + real(input) -> Tensor + + Returns a new tensor containing real values of the :attr:`self` tensor. + The returned tensor and :attr:`self` share the same underlying storage. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> x=torch.randn(4, dtype=torch.cfloat) + >>> x + tensor([(0.3100+0.3553j), (-0.5445-0.7896j), (-1.6492-0.0633j), (-0.0638-0.8119j)]) + >>> x.real + tensor([ 0.3100, -0.5445, -1.6492, -0.0638]) + """ + +def reciprocal(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + reciprocal(input, *, out=None) -> Tensor + + Returns a new tensor with the reciprocal of the elements of :attr:`input` + + .. math:: + \text{out}_{i} = \frac{1}{\text{input}_{i}} + + .. note:: + Unlike NumPy's reciprocal, torch.reciprocal supports integral inputs. Integral + inputs to reciprocal are automatically :ref:`promoted ` to + the default scalar type. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([-0.4595, -2.1219, -1.4314, 0.7298]) + >>> torch.reciprocal(a) + tensor([-2.1763, -0.4713, -0.6986, 1.3702]) + """ + +def reciprocal_(input: Tensor) -> Tensor: ... +def relu(input: Tensor) -> Tensor: ... +def relu_(input: Tensor) -> Tensor: ... +@overload +def remainder( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + remainder(input, other, *, out=None) -> Tensor + + Computes + `Python's modulus operation `_ + entrywise. The result has the same sign as the divisor :attr:`other` and its absolute value + is less than that of :attr:`other`. + + It may also be defined in terms of :func:`torch.div` as + + .. code:: python + + torch.remainder(a, b) == a - a.div(b, rounding_mode="floor") * b + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer and float inputs. + + .. note:: + Complex inputs are not supported. In some cases, it is not mathematically + possible to satisfy the definition of a modulo operation with complex numbers. + See :func:`torch.fmod` for how division by zero is handled. + + .. seealso:: + + :func:`torch.fmod` which implements C++'s `std::fmod `_. + This one is defined in terms of division rounding towards zero. + + Args: + input (Tensor or Scalar): the dividend + other (Tensor or Scalar): the divisor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.remainder(torch.tensor([-3., -2, -1, 1, 2, 3]), 2) + tensor([ 1., 0., 1., 1., 0., 1.]) + >>> torch.remainder(torch.tensor([1, 2, 3, 4, 5]), -1.5) + tensor([ -0.5000, -1.0000, 0.0000, -0.5000, -1.0000 ]) + """ + +@overload +def remainder(self: Number | _complex, other: Tensor) -> Tensor: + r""" + remainder(input, other, *, out=None) -> Tensor + + Computes + `Python's modulus operation `_ + entrywise. The result has the same sign as the divisor :attr:`other` and its absolute value + is less than that of :attr:`other`. + + It may also be defined in terms of :func:`torch.div` as + + .. code:: python + + torch.remainder(a, b) == a - a.div(b, rounding_mode="floor") * b + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer and float inputs. + + .. note:: + Complex inputs are not supported. In some cases, it is not mathematically + possible to satisfy the definition of a modulo operation with complex numbers. + See :func:`torch.fmod` for how division by zero is handled. + + .. seealso:: + + :func:`torch.fmod` which implements C++'s `std::fmod `_. + This one is defined in terms of division rounding towards zero. + + Args: + input (Tensor or Scalar): the dividend + other (Tensor or Scalar): the divisor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.remainder(torch.tensor([-3., -2, -1, 1, 2, 3]), 2) + tensor([ 1., 0., 1., 1., 0., 1.]) + >>> torch.remainder(torch.tensor([1, 2, 3, 4, 5]), -1.5) + tensor([ -0.5000, -1.0000, 0.0000, -0.5000, -1.0000 ]) + """ + +@overload +def remainder( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + remainder(input, other, *, out=None) -> Tensor + + Computes + `Python's modulus operation `_ + entrywise. The result has the same sign as the divisor :attr:`other` and its absolute value + is less than that of :attr:`other`. + + It may also be defined in terms of :func:`torch.div` as + + .. code:: python + + torch.remainder(a, b) == a - a.div(b, rounding_mode="floor") * b + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer and float inputs. + + .. note:: + Complex inputs are not supported. In some cases, it is not mathematically + possible to satisfy the definition of a modulo operation with complex numbers. + See :func:`torch.fmod` for how division by zero is handled. + + .. seealso:: + + :func:`torch.fmod` which implements C++'s `std::fmod `_. + This one is defined in terms of division rounding towards zero. + + Args: + input (Tensor or Scalar): the dividend + other (Tensor or Scalar): the divisor + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.remainder(torch.tensor([-3., -2, -1, 1, 2, 3]), 2) + tensor([ 1., 0., 1., 1., 0., 1.]) + >>> torch.remainder(torch.tensor([1, 2, 3, 4, 5]), -1.5) + tensor([ -0.5000, -1.0000, 0.0000, -0.5000, -1.0000 ]) + """ + +def renorm( + input: Tensor, + p: Number | _complex, + dim: _int, + maxnorm: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + renorm(input, p, dim, maxnorm, *, out=None) -> Tensor + + Returns a tensor where each sub-tensor of :attr:`input` along dimension + :attr:`dim` is normalized such that the `p`-norm of the sub-tensor is lower + than the value :attr:`maxnorm` + + .. note:: If the norm of a row is lower than `maxnorm`, the row is unchanged + + Args: + input (Tensor): the input tensor. + p (float): the power for the norm computation + dim (int): the dimension to slice over to get the sub-tensors + maxnorm (float): the maximum norm to keep each sub-tensor under + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> x = torch.ones(3, 3) + >>> x[1].fill_(2) + tensor([ 2., 2., 2.]) + >>> x[2].fill_(3) + tensor([ 3., 3., 3.]) + >>> x + tensor([[ 1., 1., 1.], + [ 2., 2., 2.], + [ 3., 3., 3.]]) + >>> torch.renorm(x, 1, 0, 5) + tensor([[ 1.0000, 1.0000, 1.0000], + [ 1.6667, 1.6667, 1.6667], + [ 1.6667, 1.6667, 1.6667]]) + """ + +@overload +def repeat_interleave( + input: Tensor, + repeats: Tensor, + dim: _int | None = None, + *, + output_size: _int | SymInt | None = None, +) -> Tensor: + r""" + repeat_interleave(input, repeats, dim=None, *, output_size=None) -> Tensor + + Repeat elements of a tensor. + + .. warning:: + + This is different from :meth:`torch.Tensor.repeat` but similar to ``numpy.repeat``. + + Args: + input (Tensor): the input tensor. + repeats (Tensor or int): The number of repetitions for each element. + repeats is broadcasted to fit the shape of the given axis. + dim (int, optional): The dimension along which to repeat values. + By default, use the flattened input array, and return a flat output + array. + + Keyword args: + output_size (int, optional): Total output size for the given axis + ( e.g. sum of repeats). If given, it will avoid stream synchronization + needed to calculate output shape of the tensor. + + Returns: + Tensor: Repeated tensor which has the same shape as input, except along the given axis. + + Example:: + + >>> x = torch.tensor([1, 2, 3]) + >>> x.repeat_interleave(2) + tensor([1, 1, 2, 2, 3, 3]) + >>> y = torch.tensor([[1, 2], [3, 4]]) + >>> torch.repeat_interleave(y, 2) + tensor([1, 1, 2, 2, 3, 3, 4, 4]) + >>> torch.repeat_interleave(y, 3, dim=1) + tensor([[1, 1, 1, 2, 2, 2], + [3, 3, 3, 4, 4, 4]]) + >>> torch.repeat_interleave(y, torch.tensor([1, 2]), dim=0) + tensor([[1, 2], + [3, 4], + [3, 4]]) + >>> torch.repeat_interleave(y, torch.tensor([1, 2]), dim=0, output_size=3) + tensor([[1, 2], + [3, 4], + [3, 4]]) + + If the `repeats` is `tensor([n1, n2, n3, ...])`, then the output will be + `tensor([0, 0, ..., 1, 1, ..., 2, 2, ..., ...])` where `0` appears `n1` times, + `1` appears `n2` times, `2` appears `n3` times, etc. + + .. function:: repeat_interleave(repeats, *) -> Tensor + :noindex: + + Repeats 0 repeats[0] times, 1 repeats[1] times, 2 repeats[2] times, etc. + + Args: + repeats (Tensor): The number of repetitions for each element. + + Returns: + Tensor: Repeated tensor of size `sum(repeats)`. + + Example:: + + >>> torch.repeat_interleave(torch.tensor([1, 2, 3])) + tensor([0, 1, 1, 2, 2, 2]) + """ + +@overload +def repeat_interleave( + repeats: Tensor, + *, + output_size: _int | SymInt | None = None, +) -> Tensor: + r""" + repeat_interleave(input, repeats, dim=None, *, output_size=None) -> Tensor + + Repeat elements of a tensor. + + .. warning:: + + This is different from :meth:`torch.Tensor.repeat` but similar to ``numpy.repeat``. + + Args: + input (Tensor): the input tensor. + repeats (Tensor or int): The number of repetitions for each element. + repeats is broadcasted to fit the shape of the given axis. + dim (int, optional): The dimension along which to repeat values. + By default, use the flattened input array, and return a flat output + array. + + Keyword args: + output_size (int, optional): Total output size for the given axis + ( e.g. sum of repeats). If given, it will avoid stream synchronization + needed to calculate output shape of the tensor. + + Returns: + Tensor: Repeated tensor which has the same shape as input, except along the given axis. + + Example:: + + >>> x = torch.tensor([1, 2, 3]) + >>> x.repeat_interleave(2) + tensor([1, 1, 2, 2, 3, 3]) + >>> y = torch.tensor([[1, 2], [3, 4]]) + >>> torch.repeat_interleave(y, 2) + tensor([1, 1, 2, 2, 3, 3, 4, 4]) + >>> torch.repeat_interleave(y, 3, dim=1) + tensor([[1, 1, 1, 2, 2, 2], + [3, 3, 3, 4, 4, 4]]) + >>> torch.repeat_interleave(y, torch.tensor([1, 2]), dim=0) + tensor([[1, 2], + [3, 4], + [3, 4]]) + >>> torch.repeat_interleave(y, torch.tensor([1, 2]), dim=0, output_size=3) + tensor([[1, 2], + [3, 4], + [3, 4]]) + + If the `repeats` is `tensor([n1, n2, n3, ...])`, then the output will be + `tensor([0, 0, ..., 1, 1, ..., 2, 2, ..., ...])` where `0` appears `n1` times, + `1` appears `n2` times, `2` appears `n3` times, etc. + + .. function:: repeat_interleave(repeats, *) -> Tensor + :noindex: + + Repeats 0 repeats[0] times, 1 repeats[1] times, 2 repeats[2] times, etc. + + Args: + repeats (Tensor): The number of repetitions for each element. + + Returns: + Tensor: Repeated tensor of size `sum(repeats)`. + + Example:: + + >>> torch.repeat_interleave(torch.tensor([1, 2, 3])) + tensor([0, 1, 1, 2, 2, 2]) + """ + +@overload +def repeat_interleave( + input: Tensor, + repeats: _int | SymInt, + dim: _int | None = None, + *, + output_size: _int | SymInt | None = None, +) -> Tensor: + r""" + repeat_interleave(input, repeats, dim=None, *, output_size=None) -> Tensor + + Repeat elements of a tensor. + + .. warning:: + + This is different from :meth:`torch.Tensor.repeat` but similar to ``numpy.repeat``. + + Args: + input (Tensor): the input tensor. + repeats (Tensor or int): The number of repetitions for each element. + repeats is broadcasted to fit the shape of the given axis. + dim (int, optional): The dimension along which to repeat values. + By default, use the flattened input array, and return a flat output + array. + + Keyword args: + output_size (int, optional): Total output size for the given axis + ( e.g. sum of repeats). If given, it will avoid stream synchronization + needed to calculate output shape of the tensor. + + Returns: + Tensor: Repeated tensor which has the same shape as input, except along the given axis. + + Example:: + + >>> x = torch.tensor([1, 2, 3]) + >>> x.repeat_interleave(2) + tensor([1, 1, 2, 2, 3, 3]) + >>> y = torch.tensor([[1, 2], [3, 4]]) + >>> torch.repeat_interleave(y, 2) + tensor([1, 1, 2, 2, 3, 3, 4, 4]) + >>> torch.repeat_interleave(y, 3, dim=1) + tensor([[1, 1, 1, 2, 2, 2], + [3, 3, 3, 4, 4, 4]]) + >>> torch.repeat_interleave(y, torch.tensor([1, 2]), dim=0) + tensor([[1, 2], + [3, 4], + [3, 4]]) + >>> torch.repeat_interleave(y, torch.tensor([1, 2]), dim=0, output_size=3) + tensor([[1, 2], + [3, 4], + [3, 4]]) + + If the `repeats` is `tensor([n1, n2, n3, ...])`, then the output will be + `tensor([0, 0, ..., 1, 1, ..., 2, 2, ..., ...])` where `0` appears `n1` times, + `1` appears `n2` times, `2` appears `n3` times, etc. + + .. function:: repeat_interleave(repeats, *) -> Tensor + :noindex: + + Repeats 0 repeats[0] times, 1 repeats[1] times, 2 repeats[2] times, etc. + + Args: + repeats (Tensor): The number of repetitions for each element. + + Returns: + Tensor: Repeated tensor of size `sum(repeats)`. + + Example:: + + >>> torch.repeat_interleave(torch.tensor([1, 2, 3])) + tensor([0, 1, 1, 2, 2, 2]) + """ + +def reshape(input: Tensor, shape: Sequence[_int | SymInt]) -> Tensor: + r""" + reshape(input, shape) -> Tensor + + Returns a tensor with the same data and number of elements as :attr:`input`, + but with the specified shape. When possible, the returned tensor will be a view + of :attr:`input`. Otherwise, it will be a copy. Contiguous inputs and inputs + with compatible strides can be reshaped without copying, but you should not + depend on the copying vs. viewing behavior. + + See :meth:`torch.Tensor.view` on when it is possible to return a view. + + A single dimension may be -1, in which case it's inferred from the remaining + dimensions and the number of elements in :attr:`input`. + + Args: + input (Tensor): the tensor to be reshaped + shape (tuple of int): the new shape + + Example:: + + >>> a = torch.arange(4.) + >>> torch.reshape(a, (2, 2)) + tensor([[ 0., 1.], + [ 2., 3.]]) + >>> b = torch.tensor([[0, 1], [2, 3]]) + >>> torch.reshape(b, (-1,)) + tensor([ 0, 1, 2, 3]) + """ + +def resize_as_( + input: Tensor, + the_template: Tensor, + *, + memory_format: memory_format | None = None, +) -> Tensor: ... +def resize_as_sparse_(input: Tensor, the_template: Tensor) -> Tensor: ... +def resolve_conj(input: Tensor) -> Tensor: + r""" + resolve_conj(input) -> Tensor + + Returns a new tensor with materialized conjugation if :attr:`input`'s conjugate bit is set to `True`, + else returns :attr:`input`. The output tensor will always have its conjugate bit set to `False`. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> x = torch.tensor([-1 + 1j, -2 + 2j, 3 - 3j]) + >>> y = x.conj() + >>> y.is_conj() + True + >>> z = y.resolve_conj() + >>> z + tensor([-1 - 1j, -2 - 2j, 3 + 3j]) + >>> z.is_conj() + False + """ + +def resolve_neg(input: Tensor) -> Tensor: + r""" + resolve_neg(input) -> Tensor + + Returns a new tensor with materialized negation if :attr:`input`'s negative bit is set to `True`, + else returns :attr:`input`. The output tensor will always have its negative bit set to `False`. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> x = torch.tensor([-1 + 1j, -2 + 2j, 3 - 3j]) + >>> y = x.conj() + >>> z = y.imag + >>> z.is_neg() + True + >>> out = z.resolve_neg() + >>> out + tensor([-1., -2., 3.]) + >>> out.is_neg() + False + """ + +@overload +def result_type(tensor: Tensor, other: Tensor) -> _dtype: + r""" + result_type(tensor1, tensor2) -> dtype + + Returns the :class:`torch.dtype` that would result from performing an arithmetic + operation on the provided input tensors. See type promotion :ref:`documentation ` + for more information on the type promotion logic. + + Args: + tensor1 (Tensor or Number): an input tensor or number + tensor2 (Tensor or Number): an input tensor or number + + Example:: + + >>> torch.result_type(torch.tensor([1, 2], dtype=torch.int), 1.0) + torch.float32 + >>> torch.result_type(torch.tensor([1, 2], dtype=torch.uint8), torch.tensor(1)) + torch.uint8 + """ + +@overload +def result_type(scalar: Number | _complex, tensor: Tensor) -> _dtype: + r""" + result_type(tensor1, tensor2) -> dtype + + Returns the :class:`torch.dtype` that would result from performing an arithmetic + operation on the provided input tensors. See type promotion :ref:`documentation ` + for more information on the type promotion logic. + + Args: + tensor1 (Tensor or Number): an input tensor or number + tensor2 (Tensor or Number): an input tensor or number + + Example:: + + >>> torch.result_type(torch.tensor([1, 2], dtype=torch.int), 1.0) + torch.float32 + >>> torch.result_type(torch.tensor([1, 2], dtype=torch.uint8), torch.tensor(1)) + torch.uint8 + """ + +@overload +def result_type(tensor: Tensor, other: Number | _complex) -> _dtype: + r""" + result_type(tensor1, tensor2) -> dtype + + Returns the :class:`torch.dtype` that would result from performing an arithmetic + operation on the provided input tensors. See type promotion :ref:`documentation ` + for more information on the type promotion logic. + + Args: + tensor1 (Tensor or Number): an input tensor or number + tensor2 (Tensor or Number): an input tensor or number + + Example:: + + >>> torch.result_type(torch.tensor([1, 2], dtype=torch.int), 1.0) + torch.float32 + >>> torch.result_type(torch.tensor([1, 2], dtype=torch.uint8), torch.tensor(1)) + torch.uint8 + """ + +@overload +def result_type( + scalar1: Number | _complex, + scalar2: Number | _complex, +) -> _dtype: + r""" + result_type(tensor1, tensor2) -> dtype + + Returns the :class:`torch.dtype` that would result from performing an arithmetic + operation on the provided input tensors. See type promotion :ref:`documentation ` + for more information on the type promotion logic. + + Args: + tensor1 (Tensor or Number): an input tensor or number + tensor2 (Tensor or Number): an input tensor or number + + Example:: + + >>> torch.result_type(torch.tensor([1, 2], dtype=torch.int), 1.0) + torch.float32 + >>> torch.result_type(torch.tensor([1, 2], dtype=torch.uint8), torch.tensor(1)) + torch.uint8 + """ + +def rms_norm( + input: Tensor, + normalized_shape: Sequence[_int | SymInt], + weight: Tensor | None = None, + eps: _float | None = None, +) -> Tensor: ... +@overload +def rnn_relu( + data: Tensor, + batch_sizes: Tensor, + hx: Tensor, + params: tuple[Tensor, ...] | list[Tensor] | None, + has_biases: _bool, + num_layers: _int, + dropout: _float, + train: _bool, + bidirectional: _bool, +) -> tuple[Tensor, Tensor]: ... +@overload +def rnn_relu( + input: Tensor, + hx: Tensor, + params: tuple[Tensor, ...] | list[Tensor] | None, + has_biases: _bool, + num_layers: _int, + dropout: _float, + train: _bool, + bidirectional: _bool, + batch_first: _bool, +) -> tuple[Tensor, Tensor]: ... +def rnn_relu_cell( + input: Tensor, + hx: Tensor, + w_ih: Tensor, + w_hh: Tensor, + b_ih: Tensor | None = None, + b_hh: Tensor | None = None, +) -> Tensor: ... +@overload +def rnn_tanh( + data: Tensor, + batch_sizes: Tensor, + hx: Tensor, + params: tuple[Tensor, ...] | list[Tensor] | None, + has_biases: _bool, + num_layers: _int, + dropout: _float, + train: _bool, + bidirectional: _bool, +) -> tuple[Tensor, Tensor]: ... +@overload +def rnn_tanh( + input: Tensor, + hx: Tensor, + params: tuple[Tensor, ...] | list[Tensor] | None, + has_biases: _bool, + num_layers: _int, + dropout: _float, + train: _bool, + bidirectional: _bool, + batch_first: _bool, +) -> tuple[Tensor, Tensor]: ... +def rnn_tanh_cell( + input: Tensor, + hx: Tensor, + w_ih: Tensor, + w_hh: Tensor, + b_ih: Tensor | None = None, + b_hh: Tensor | None = None, +) -> Tensor: ... +def roll( + input: Tensor, + shifts: _int | SymInt | Sequence[_int | SymInt], + dims: _int | _size = (), +) -> Tensor: + r""" + roll(input, shifts, dims=None) -> Tensor + + Roll the tensor :attr:`input` along the given dimension(s). Elements that are + shifted beyond the last position are re-introduced at the first position. If + :attr:`dims` is `None`, the tensor will be flattened before rolling and then + restored to the original shape. + + Args: + input (Tensor): the input tensor. + shifts (int or tuple of ints): The number of places by which the elements + of the tensor are shifted. If shifts is a tuple, dims must be a tuple of + the same size, and each dimension will be rolled by the corresponding + value + dims (int or tuple of ints): Axis along which to roll + + Example:: + + >>> x = torch.tensor([1, 2, 3, 4, 5, 6, 7, 8]).view(4, 2) + >>> x + tensor([[1, 2], + [3, 4], + [5, 6], + [7, 8]]) + >>> torch.roll(x, 1) + tensor([[8, 1], + [2, 3], + [4, 5], + [6, 7]]) + >>> torch.roll(x, 1, 0) + tensor([[7, 8], + [1, 2], + [3, 4], + [5, 6]]) + >>> torch.roll(x, -1, 0) + tensor([[3, 4], + [5, 6], + [7, 8], + [1, 2]]) + >>> torch.roll(x, shifts=(2, 1), dims=(0, 1)) + tensor([[6, 5], + [8, 7], + [2, 1], + [4, 3]]) + """ + +def rot90(input: Tensor, k: _int = 1, dims: _size = (0, 1)) -> Tensor: + r""" + rot90(input, k=1, dims=(0, 1)) -> Tensor + + Rotate an n-D tensor by 90 degrees in the plane specified by dims axis. + Rotation direction is from the first towards the second axis if k > 0, and from the second towards the first for k < 0. + + Args: + input (Tensor): the input tensor. + k (int): number of times to rotate. Default value is 1 + dims (a list or tuple): axis to rotate. Default value is [0, 1] + + Example:: + + >>> x = torch.arange(4).view(2, 2) + >>> x + tensor([[0, 1], + [2, 3]]) + >>> torch.rot90(x, 1, [0, 1]) + tensor([[1, 3], + [0, 2]]) + + >>> x = torch.arange(8).view(2, 2, 2) + >>> x + tensor([[[0, 1], + [2, 3]], + + [[4, 5], + [6, 7]]]) + >>> torch.rot90(x, 1, [1, 2]) + tensor([[[1, 3], + [0, 2]], + + [[5, 7], + [4, 6]]]) + """ + +@overload +def round(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + round(input, *, decimals=0, out=None) -> Tensor + + Rounds elements of :attr:`input` to the nearest integer. + + For integer inputs, follows the array-api convention of returning a + copy of the input tensor. + The return type of output is same as that of input's dtype. + + .. note:: + This function implements the "round half to even" to + break ties when a number is equidistant from two + integers (e.g. `round(2.5)` is 2). + + When the :attr:\`decimals\` argument is specified the + algorithm used is similar to NumPy's `around`. This + algorithm is fast but inexact and it can easily + overflow for low precision dtypes. + Eg. `round(tensor([10000], dtype=torch.float16), decimals=3)` is `inf`. + + .. seealso:: + :func:`torch.ceil`, which rounds up. + :func:`torch.floor`, which rounds down. + :func:`torch.trunc`, which rounds towards zero. + + Args: + input (Tensor): the input tensor. + decimals (int): Number of decimal places to round to (default: 0). + If decimals is negative, it specifies the number of positions + to the left of the decimal point. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.round(torch.tensor((4.7, -2.3, 9.1, -7.7))) + tensor([ 5., -2., 9., -8.]) + + >>> # Values equidistant from two integers are rounded towards the + >>> # the nearest even value (zero is treated as even) + >>> torch.round(torch.tensor([-0.5, 0.5, 1.5, 2.5])) + tensor([-0., 0., 2., 2.]) + + >>> # A positive decimals argument rounds to the to that decimal place + >>> torch.round(torch.tensor([0.1234567]), decimals=3) + tensor([0.1230]) + + >>> # A negative decimals argument rounds to the left of the decimal + >>> torch.round(torch.tensor([1200.1234567]), decimals=-3) + tensor([1000.]) + """ + +@overload +def round( + input: Tensor, + *, + decimals: _int, + out: Tensor | None = None, +) -> Tensor: + r""" + round(input, *, decimals=0, out=None) -> Tensor + + Rounds elements of :attr:`input` to the nearest integer. + + For integer inputs, follows the array-api convention of returning a + copy of the input tensor. + The return type of output is same as that of input's dtype. + + .. note:: + This function implements the "round half to even" to + break ties when a number is equidistant from two + integers (e.g. `round(2.5)` is 2). + + When the :attr:\`decimals\` argument is specified the + algorithm used is similar to NumPy's `around`. This + algorithm is fast but inexact and it can easily + overflow for low precision dtypes. + Eg. `round(tensor([10000], dtype=torch.float16), decimals=3)` is `inf`. + + .. seealso:: + :func:`torch.ceil`, which rounds up. + :func:`torch.floor`, which rounds down. + :func:`torch.trunc`, which rounds towards zero. + + Args: + input (Tensor): the input tensor. + decimals (int): Number of decimal places to round to (default: 0). + If decimals is negative, it specifies the number of positions + to the left of the decimal point. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> torch.round(torch.tensor((4.7, -2.3, 9.1, -7.7))) + tensor([ 5., -2., 9., -8.]) + + >>> # Values equidistant from two integers are rounded towards the + >>> # the nearest even value (zero is treated as even) + >>> torch.round(torch.tensor([-0.5, 0.5, 1.5, 2.5])) + tensor([-0., 0., 2., 2.]) + + >>> # A positive decimals argument rounds to the to that decimal place + >>> torch.round(torch.tensor([0.1234567]), decimals=3) + tensor([0.1230]) + + >>> # A negative decimals argument rounds to the left of the decimal + >>> torch.round(torch.tensor([1200.1234567]), decimals=-3) + tensor([1000.]) + """ + +@overload +def round_(input: Tensor) -> Tensor: ... +@overload +def round_(input: Tensor, *, decimals: _int) -> Tensor: ... +def row_indices_copy(input: Tensor, *, out: Tensor | None = None) -> Tensor: ... +def row_stack( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + row_stack(tensors, *, out=None) -> Tensor + + Alias of :func:`torch.vstack`. + """ + +def rrelu( + input: Tensor, + lower: Number | _complex = 0.125, + upper: Number | _complex = 0.3333333333333333, + training: _bool = False, + generator: Generator | None = None, +) -> Tensor: ... +def rrelu_( + input: Tensor, + lower: Number | _complex = 0.125, + upper: Number | _complex = 0.3333333333333333, + training: _bool = False, + generator: Generator | None = None, +) -> Tensor: ... +def rsqrt(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + rsqrt(input, *, out=None) -> Tensor + + Returns a new tensor with the reciprocal of the square-root of each of + the elements of :attr:`input`. + + .. math:: + \text{out}_{i} = \frac{1}{\sqrt{\text{input}_{i}}} + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([-0.0370, 0.2970, 1.5420, -0.9105]) + >>> torch.rsqrt(a) + tensor([ nan, 1.8351, 0.8053, nan]) + """ + +def rsqrt_(input: Tensor) -> Tensor: ... +@overload +def rsub( + input: Tensor, + other: Tensor, + *, + alpha: Number | _complex = 1, +) -> Tensor: ... +@overload +def rsub( + input: Tensor, + other: Number | _complex, + alpha: Number | _complex = 1, +) -> Tensor: ... +def saddmm( + input: Tensor, + mat1: Tensor, + mat2: Tensor, + *, + beta: Number = 1, + alpha: Number = 1, + out: Tensor | None = None, +) -> Tensor: ... +def scalar_tensor( + s: Number | _complex, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: ... +@overload +def scatter( + input: Tensor, + dim: _int, + index: Tensor, + src: Tensor, + *, + reduce: str, + out: Tensor | None = None, +) -> Tensor: + r""" + scatter(input, dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_` + """ + +@overload +def scatter( + input: Tensor, + dim: _int, + index: Tensor, + src: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + scatter(input, dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_` + """ + +@overload +def scatter( + input: Tensor, + dim: _int, + index: Tensor, + value: Number | _complex, + *, + reduce: str, + out: Tensor | None = None, +) -> Tensor: + r""" + scatter(input, dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_` + """ + +@overload +def scatter( + input: Tensor, + dim: str | EllipsisType | None, + index: Tensor, + src: Tensor, +) -> Tensor: + r""" + scatter(input, dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_` + """ + +@overload +def scatter( + input: Tensor, + dim: _int, + index: Tensor, + value: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + scatter(input, dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_` + """ + +@overload +def scatter( + input: Tensor, + dim: str | EllipsisType | None, + index: Tensor, + value: Number | _complex, +) -> Tensor: + r""" + scatter(input, dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_` + """ + +@overload +def scatter_add( + input: Tensor, + dim: _int, + index: Tensor, + src: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + scatter_add(input, dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_add_` + """ + +@overload +def scatter_add( + input: Tensor, + dim: str | EllipsisType | None, + index: Tensor, + src: Tensor, +) -> Tensor: + r""" + scatter_add(input, dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_add_` + """ + +def scatter_reduce( + input: Tensor, + dim: _int, + index: Tensor, + src: Tensor, + reduce: str, + *, + include_self: _bool = True, + out: Tensor | None = None, +) -> Tensor: + r""" + scatter_reduce(input, dim, index, src, reduce, *, include_self=True) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_reduce_` + """ + +@overload +def searchsorted( + sorted_sequence: Tensor, + input: Tensor, + *, + out_int32: _bool = False, + right: _bool = False, + side: str | None = None, + sorter: Tensor | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + searchsorted(sorted_sequence, values, *, out_int32=False, right=False, side=None, out=None, sorter=None) -> Tensor + + Find the indices from the *innermost* dimension of :attr:`sorted_sequence` such that, if the + corresponding values in :attr:`values` were inserted before the indices, when sorted, the order + of the corresponding *innermost* dimension within :attr:`sorted_sequence` would be preserved. + Return a new tensor with the same size as :attr:`values`. More formally, + the returned index satisfies the following rules: + + .. list-table:: + :widths: 12 10 78 + :header-rows: 1 + + * - :attr:`sorted_sequence` + - :attr:`right` + - *returned index satisfies* + * - 1-D + - False + - ``sorted_sequence[i-1] < values[m][n]...[l][x] <= sorted_sequence[i]`` + * - 1-D + - True + - ``sorted_sequence[i-1] <= values[m][n]...[l][x] < sorted_sequence[i]`` + * - N-D + - False + - ``sorted_sequence[m][n]...[l][i-1] < values[m][n]...[l][x] <= sorted_sequence[m][n]...[l][i]`` + * - N-D + - True + - ``sorted_sequence[m][n]...[l][i-1] <= values[m][n]...[l][x] < sorted_sequence[m][n]...[l][i]`` + + Args: + sorted_sequence (Tensor): N-D or 1-D tensor, containing monotonically increasing sequence on the *innermost* + dimension unless :attr:`sorter` is provided, in which case the sequence does not + need to be sorted + values (Tensor or Scalar): N-D tensor or a Scalar containing the search value(s). + + Keyword args: + out_int32 (bool, optional): indicate the output data type. torch.int32 if True, torch.int64 otherwise. + Default value is False, i.e. default output data type is torch.int64. + right (bool, optional): if False, return the first suitable location that is found. If True, return the + last such index. If no suitable index found, return 0 for non-numerical value + (eg. nan, inf) or the size of *innermost* dimension within :attr:`sorted_sequence` + (one pass the last index of the *innermost* dimension). In other words, if False, + gets the lower bound index for each value in :attr:`values` on the corresponding + *innermost* dimension of the :attr:`sorted_sequence`. If True, gets the upper + bound index instead. Default value is False. :attr:`side` does the same and is + preferred. It will error if :attr:`side` is set to "left" while this is True. + side (str, optional): the same as :attr:`right` but preferred. "left" corresponds to False for :attr:`right` + and "right" corresponds to True for :attr:`right`. It will error if this is set to + "left" while :attr:`right` is True. Default value is None. + out (Tensor, optional): the output tensor, must be the same size as :attr:`values` if provided. + sorter (LongTensor, optional): if provided, a tensor matching the shape of the unsorted + :attr:`sorted_sequence` containing a sequence of indices that sort it in the + ascending order on the innermost dimension + + + Example:: + + >>> sorted_sequence = torch.tensor([[1, 3, 5, 7, 9], [2, 4, 6, 8, 10]]) + >>> sorted_sequence + tensor([[ 1, 3, 5, 7, 9], + [ 2, 4, 6, 8, 10]]) + >>> values = torch.tensor([[3, 6, 9], [3, 6, 9]]) + >>> values + tensor([[3, 6, 9], + [3, 6, 9]]) + >>> torch.searchsorted(sorted_sequence, values) + tensor([[1, 3, 4], + [1, 2, 4]]) + >>> torch.searchsorted(sorted_sequence, values, side='right') + tensor([[2, 3, 5], + [1, 3, 4]]) + + >>> sorted_sequence_1d = torch.tensor([1, 3, 5, 7, 9]) + >>> sorted_sequence_1d + tensor([1, 3, 5, 7, 9]) + >>> torch.searchsorted(sorted_sequence_1d, values) + tensor([[1, 3, 4], + [1, 3, 4]]) + """ + +@overload +def searchsorted( + sorted_sequence: Tensor, + self: Number | _complex, + *, + out_int32: _bool = False, + right: _bool = False, + side: str | None = None, + sorter: Tensor | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + searchsorted(sorted_sequence, values, *, out_int32=False, right=False, side=None, out=None, sorter=None) -> Tensor + + Find the indices from the *innermost* dimension of :attr:`sorted_sequence` such that, if the + corresponding values in :attr:`values` were inserted before the indices, when sorted, the order + of the corresponding *innermost* dimension within :attr:`sorted_sequence` would be preserved. + Return a new tensor with the same size as :attr:`values`. More formally, + the returned index satisfies the following rules: + + .. list-table:: + :widths: 12 10 78 + :header-rows: 1 + + * - :attr:`sorted_sequence` + - :attr:`right` + - *returned index satisfies* + * - 1-D + - False + - ``sorted_sequence[i-1] < values[m][n]...[l][x] <= sorted_sequence[i]`` + * - 1-D + - True + - ``sorted_sequence[i-1] <= values[m][n]...[l][x] < sorted_sequence[i]`` + * - N-D + - False + - ``sorted_sequence[m][n]...[l][i-1] < values[m][n]...[l][x] <= sorted_sequence[m][n]...[l][i]`` + * - N-D + - True + - ``sorted_sequence[m][n]...[l][i-1] <= values[m][n]...[l][x] < sorted_sequence[m][n]...[l][i]`` + + Args: + sorted_sequence (Tensor): N-D or 1-D tensor, containing monotonically increasing sequence on the *innermost* + dimension unless :attr:`sorter` is provided, in which case the sequence does not + need to be sorted + values (Tensor or Scalar): N-D tensor or a Scalar containing the search value(s). + + Keyword args: + out_int32 (bool, optional): indicate the output data type. torch.int32 if True, torch.int64 otherwise. + Default value is False, i.e. default output data type is torch.int64. + right (bool, optional): if False, return the first suitable location that is found. If True, return the + last such index. If no suitable index found, return 0 for non-numerical value + (eg. nan, inf) or the size of *innermost* dimension within :attr:`sorted_sequence` + (one pass the last index of the *innermost* dimension). In other words, if False, + gets the lower bound index for each value in :attr:`values` on the corresponding + *innermost* dimension of the :attr:`sorted_sequence`. If True, gets the upper + bound index instead. Default value is False. :attr:`side` does the same and is + preferred. It will error if :attr:`side` is set to "left" while this is True. + side (str, optional): the same as :attr:`right` but preferred. "left" corresponds to False for :attr:`right` + and "right" corresponds to True for :attr:`right`. It will error if this is set to + "left" while :attr:`right` is True. Default value is None. + out (Tensor, optional): the output tensor, must be the same size as :attr:`values` if provided. + sorter (LongTensor, optional): if provided, a tensor matching the shape of the unsorted + :attr:`sorted_sequence` containing a sequence of indices that sort it in the + ascending order on the innermost dimension + + + Example:: + + >>> sorted_sequence = torch.tensor([[1, 3, 5, 7, 9], [2, 4, 6, 8, 10]]) + >>> sorted_sequence + tensor([[ 1, 3, 5, 7, 9], + [ 2, 4, 6, 8, 10]]) + >>> values = torch.tensor([[3, 6, 9], [3, 6, 9]]) + >>> values + tensor([[3, 6, 9], + [3, 6, 9]]) + >>> torch.searchsorted(sorted_sequence, values) + tensor([[1, 3, 4], + [1, 2, 4]]) + >>> torch.searchsorted(sorted_sequence, values, side='right') + tensor([[2, 3, 5], + [1, 3, 4]]) + + >>> sorted_sequence_1d = torch.tensor([1, 3, 5, 7, 9]) + >>> sorted_sequence_1d + tensor([1, 3, 5, 7, 9]) + >>> torch.searchsorted(sorted_sequence_1d, values) + tensor([[1, 3, 4], + [1, 3, 4]]) + """ + +def segment_reduce( + data: Tensor, + reduce: str, + *, + lengths: Tensor | None = None, + indices: Tensor | None = None, + offsets: Tensor | None = None, + axis: _int = 0, + unsafe: _bool = False, + initial: Number | _complex | None = None, +) -> Tensor: + r""" + segment_reduce(data: Tensor, reduce: str, *, lengths: Tensor | None = None, indices: Tensor | None = None, offsets: Tensor | None = None, axis: _int = 0, unsafe: _bool = False, initial: Number | _complex | None = None) -> Tensor # noqa: B950 + + Perform a segment reduction operation on the input tensor along the specified axis. + + Args: + data (Tensor): The input tensor on which the segment reduction operation will be performed. + reduce (str): The type of reduction operation. Supported values are ``sum``, ``mean``, ``max``, ``min``, ``prod``. + + Keyword args: + lengths (Tensor, optional): Length of each segment. Default: ``None``. + offsets (Tensor, optional): Offset of each segment. Default: ``None``. + axis (int, optional): The axis perform reduction. Default: ``0``. + unsafe (bool, optional): Skip validation If `True`. Default: ``False``. + initial (Number, optional): The initial value for the reduction operation. Default: ``None``. + + Example:: + + >>> data = torch.tensor([[1, 2, 3, 4],[5, 6, 7, 8],[9, 10, 11, 12]], dtype=torch.float32, device='cuda') + >>> lengths = torch.tensor([2, 1], device='cuda') + >>> torch.segment_reduce(data, 'max', lengths=lengths) + tensor([[ 5., 6., 7., 8.], + [ 9., 10., 11., 12.]], device='cuda:0') + """ + +@overload +def select(input: Tensor, dim: _int, index: _int | SymInt) -> Tensor: + r""" + select(input, dim, index) -> Tensor + + Slices the :attr:`input` tensor along the selected dimension at the given index. + This function returns a view of the original tensor with the given dimension removed. + + .. note:: If :attr:`input` is a sparse tensor and returning a view of + the tensor is not possible, a RuntimeError exception is + raised. In this is the case, consider using + :func:`torch.select_copy` function. + + Args: + input (Tensor): the input tensor. + dim (int): the dimension to slice + index (int): the index to select with + + .. note:: + + :meth:`select` is equivalent to slicing. For example, + ``tensor.select(0, index)`` is equivalent to ``tensor[index]`` and + ``tensor.select(2, index)`` is equivalent to ``tensor[:,:,index]``. + """ + +@overload +def select( + input: Tensor, + dim: str | EllipsisType | None, + index: _int, +) -> Tensor: + r""" + select(input, dim, index) -> Tensor + + Slices the :attr:`input` tensor along the selected dimension at the given index. + This function returns a view of the original tensor with the given dimension removed. + + .. note:: If :attr:`input` is a sparse tensor and returning a view of + the tensor is not possible, a RuntimeError exception is + raised. In this is the case, consider using + :func:`torch.select_copy` function. + + Args: + input (Tensor): the input tensor. + dim (int): the dimension to slice + index (int): the index to select with + + .. note:: + + :meth:`select` is equivalent to slicing. For example, + ``tensor.select(0, index)`` is equivalent to ``tensor[index]`` and + ``tensor.select(2, index)`` is equivalent to ``tensor[:,:,index]``. + """ + +def select_copy( + input: Tensor, + dim: _int, + index: _int | SymInt, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.select`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def select_scatter( + input: Tensor, + src: Tensor, + dim: _int, + index: _int | SymInt, +) -> Tensor: + r""" + select_scatter(input, src, dim, index) -> Tensor + + Embeds the values of the :attr:`src` tensor into :attr:`input` at the given index. + This function returns a tensor with fresh storage; it does not create a view. + + + Args: + input (Tensor): the input tensor. + src (Tensor): The tensor to embed into :attr:`input` + dim (int): the dimension to insert the slice into. + index (int): the index to select with + + .. note:: + + :attr:`src` must be of the proper size in order to be embedded + into :attr:`input`. Specifically, it should have the same shape as + ``torch.select(input, dim, index)`` + + Example:: + + >>> a = torch.zeros(2, 2) + >>> b = torch.ones(2) + >>> a.select_scatter(b, 0, 0) + tensor([[1., 1.], + [0., 0.]]) + """ + +def selu(input: Tensor) -> Tensor: ... +def selu_(input: Tensor) -> Tensor: ... +def set_flush_denormal(mode: _bool) -> _bool: + r""" + set_flush_denormal(mode) -> bool + + Disables denormal floating numbers on CPU. + + Returns ``True`` if your system supports flushing denormal numbers and it + successfully configures flush denormal mode. :meth:`~torch.set_flush_denormal` + is supported on x86 architectures supporting SSE3 and AArch64 architecture. + + Args: + mode (bool): Controls whether to enable flush denormal mode or not + + Example:: + + >>> torch.set_flush_denormal(True) + True + >>> torch.tensor([1e-323], dtype=torch.float64) + tensor([ 0.], dtype=torch.float64) + >>> torch.set_flush_denormal(False) + True + >>> torch.tensor([1e-323], dtype=torch.float64) + tensor(9.88131e-324 * + [ 1.0000], dtype=torch.float64) + """ + +def set_num_interop_threads(num: _int) -> None: + r""" + set_num_interop_threads(int) + + Sets the number of threads used for interop parallelism + (e.g. in JIT interpreter) on CPU. + + .. warning:: + Can only be called once and before any inter-op parallel work + is started (e.g. JIT execution). + """ + +def set_num_threads(num: _int) -> None: + r""" + set_num_threads(int) + + Sets the number of threads used for intraop parallelism on CPU. + + .. warning:: + To ensure that the correct number of threads is used, set_num_threads + must be called before running eager, JIT or autograd code. + """ + +def sgn(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + sgn(input, *, out=None) -> Tensor + + This function is an extension of torch.sign() to complex tensors. + It computes a new tensor whose elements have + the same angles as the corresponding elements of :attr:`input` and + absolute values (i.e. magnitudes) of one for complex tensors and + is equivalent to torch.sign() for non-complex tensors. + + .. math:: + \text{out}_{i} = \begin{cases} + 0 & |\text{{input}}_i| == 0 \\ + \frac{{\text{{input}}_i}}{|{\text{{input}}_i}|} & \text{otherwise} + \end{cases} + + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> t = torch.tensor([3+4j, 7-24j, 0, 1+2j]) + >>> t.sgn() + tensor([0.6000+0.8000j, 0.2800-0.9600j, 0.0000+0.0000j, 0.4472+0.8944j]) + """ + +def sigmoid(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + sigmoid(input, *, out=None) -> Tensor + + Alias for :func:`torch.special.expit`. + """ + +def sigmoid_(input: Tensor) -> Tensor: ... +def sign(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + sign(input, *, out=None) -> Tensor + + Returns a new tensor with the signs of the elements of :attr:`input`. + + .. math:: + \text{out}_{i} = \operatorname{sgn}(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([0.7, -1.2, 0., 2.3]) + >>> a + tensor([ 0.7000, -1.2000, 0.0000, 2.3000]) + >>> torch.sign(a) + tensor([ 1., -1., 0., 1.]) + """ + +def signbit(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + signbit(input, *, out=None) -> Tensor + + Tests if each element of :attr:`input` has its sign bit set or not. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([0.7, -1.2, 0., 2.3]) + >>> torch.signbit(a) + tensor([ False, True, False, False]) + >>> a = torch.tensor([-0.0, 0.0]) + >>> torch.signbit(a) + tensor([ True, False]) + + .. note:: + signbit handles signed zeros, so negative zero (-0) returns True. + """ + +def sin(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + sin(input, *, out=None) -> Tensor + + Returns a new tensor with the sine of the elements in the :attr:`input` tensor, + where each value in this input tensor is in radians. + + .. math:: + \text{out}_{i} = \sin(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([-0.5461, 0.1347, -2.7266, -0.2746]) + >>> torch.sin(a) + tensor([-0.5194, 0.1343, -0.4032, -0.2711]) + """ + +def sin_(input: Tensor) -> Tensor: ... +def sinc(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + sinc(input, *, out=None) -> Tensor + + Alias for :func:`torch.special.sinc`. + """ + +def sinc_(input: Tensor) -> Tensor: ... +def sinh(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + sinh(input, *, out=None) -> Tensor + + Returns a new tensor with the hyperbolic sine of the elements of + :attr:`input`. + + .. math:: + \text{out}_{i} = \sinh(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([ 0.5380, -0.8632, -0.1265, 0.9399]) + >>> torch.sinh(a) + tensor([ 0.5644, -0.9744, -0.1268, 1.0845]) + + .. note:: + When :attr:`input` is on the CPU, the implementation of torch.sinh may use + the Sleef library, which rounds very large results to infinity or negative + infinity. See `here `_ for details. + """ + +def sinh_(input: Tensor) -> Tensor: ... +def slice_copy( + input: Tensor, + dim: _int = 0, + start: _int | SymInt | None = None, + end: _int | SymInt | None = None, + step: _int | SymInt = 1, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.slice`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def slice_inverse( + input: Tensor, + src: Tensor, + dim: _int = 0, + start: _int | SymInt | None = None, + end: _int | SymInt | None = None, + step: _int | SymInt = 1, +) -> Tensor: ... +def slice_scatter( + input: Tensor, + src: Tensor, + dim: _int = 0, + start: _int | SymInt | None = None, + end: _int | SymInt | None = None, + step: _int | SymInt = 1, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + slice_scatter(input, src, dim=0, start=None, end=None, step=1) -> Tensor + + Embeds the values of the :attr:`src` tensor into :attr:`input` at the given + dimension. + This function returns a tensor with fresh storage; it does not create a view. + + + Args: + input (Tensor): the input tensor. + src (Tensor): The tensor to embed into :attr:`input` + dim (int): the dimension to insert the slice into + start (Optional[int]): the start index of where to insert the slice + end (Optional[int]): the end index of where to insert the slice + step (int): the how many elements to skip in + + Example:: + + >>> a = torch.zeros(8, 8) + >>> b = torch.ones(2, 8) + >>> a.slice_scatter(b, start=6) + tensor([[0., 0., 0., 0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0., 0., 0., 0.], + [0., 0., 0., 0., 0., 0., 0., 0.], + [1., 1., 1., 1., 1., 1., 1., 1.], + [1., 1., 1., 1., 1., 1., 1., 1.]]) + + >>> b = torch.ones(8, 2) + >>> a.slice_scatter(b, dim=1, start=2, end=6, step=2) + tensor([[0., 0., 1., 0., 1., 0., 0., 0.], + [0., 0., 1., 0., 1., 0., 0., 0.], + [0., 0., 1., 0., 1., 0., 0., 0.], + [0., 0., 1., 0., 1., 0., 0., 0.], + [0., 0., 1., 0., 1., 0., 0., 0.], + [0., 0., 1., 0., 1., 0., 0., 0.], + [0., 0., 1., 0., 1., 0., 0., 0.], + [0., 0., 1., 0., 1., 0., 0., 0.]]) + """ + +def slogdet( + input: Tensor, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.slogdet: + r""" + slogdet(input) -> (Tensor, Tensor) + + Alias for :func:`torch.linalg.slogdet` + """ + +def smm(input: Tensor, mat2: Tensor) -> Tensor: + r""" + smm(input, mat) -> Tensor + + Performs a matrix multiplication of the sparse matrix :attr:`input` + with the dense matrix :attr:`mat`. + + Args: + input (Tensor): a sparse matrix to be matrix multiplied + mat (Tensor): a dense matrix to be matrix multiplied + """ + +@overload +def softmax( + input: Tensor, + dim: _int, + dtype: _dtype | None = None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + softmax(input, dim, *, dtype=None) -> Tensor + + Alias for :func:`torch.nn.functional.softmax`. + """ + +@overload +def softmax( + input: Tensor, + dim: str | EllipsisType | None, + *, + dtype: _dtype | None = None, +) -> Tensor: + r""" + softmax(input, dim, *, dtype=None) -> Tensor + + Alias for :func:`torch.nn.functional.softmax`. + """ + +@overload +def sort( + input: Tensor, + *, + stable: _bool | None, + dim: _int = -1, + descending: _bool = False, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.sort: + r""" + sort(input, dim=-1, descending=False, *, stable=False, out=None) -> (Tensor, LongTensor) + + Sorts the elements of the :attr:`input` tensor along a given dimension + in ascending order by value. + + If :attr:`dim` is not given, the last dimension of the `input` is chosen. + + If :attr:`descending` is ``True`` then the elements are sorted in descending + order by value. + + If :attr:`stable` is ``True`` then the sorting routine becomes stable, preserving + the order of equivalent elements. + + A namedtuple of (values, indices) is returned, where the `values` are the + sorted values and `indices` are the indices of the elements in the original + `input` tensor. + + Args: + input (Tensor): the input tensor. + dim (int, optional): the dimension to sort along + descending (bool, optional): controls the sorting order (ascending or descending) + + Keyword args: + stable (bool, optional): makes the sorting routine stable, which guarantees that the order + of equivalent elements is preserved. + out (tuple, optional): the output tuple of (`Tensor`, `LongTensor`) that can + be optionally given to be used as output buffers + + Example:: + + >>> x = torch.randn(3, 4) + >>> sorted, indices = torch.sort(x) + >>> sorted + tensor([[-0.2162, 0.0608, 0.6719, 2.3332], + [-0.5793, 0.0061, 0.6058, 0.9497], + [-0.5071, 0.3343, 0.9553, 1.0960]]) + >>> indices + tensor([[ 1, 0, 2, 3], + [ 3, 1, 0, 2], + [ 0, 3, 1, 2]]) + + >>> sorted, indices = torch.sort(x, 0) + >>> sorted + tensor([[-0.5071, -0.2162, 0.6719, -0.5793], + [ 0.0608, 0.0061, 0.9497, 0.3343], + [ 0.6058, 0.9553, 1.0960, 2.3332]]) + >>> indices + tensor([[ 2, 0, 0, 1], + [ 0, 1, 1, 2], + [ 1, 2, 2, 0]]) + >>> x = torch.tensor([0, 1] * 9) + >>> x.sort() + torch.return_types.sort( + values=tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + indices=tensor([ 2, 16, 4, 6, 14, 8, 0, 10, 12, 9, 17, 15, 13, 11, 7, 5, 3, 1])) + >>> x.sort(stable=True) + torch.return_types.sort( + values=tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + indices=tensor([ 0, 2, 4, 6, 8, 10, 12, 14, 16, 1, 3, 5, 7, 9, 11, 13, 15, 17])) + """ + +@overload +def sort( + input: Tensor, + dim: _int = -1, + descending: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.sort: + r""" + sort(input, dim=-1, descending=False, *, stable=False, out=None) -> (Tensor, LongTensor) + + Sorts the elements of the :attr:`input` tensor along a given dimension + in ascending order by value. + + If :attr:`dim` is not given, the last dimension of the `input` is chosen. + + If :attr:`descending` is ``True`` then the elements are sorted in descending + order by value. + + If :attr:`stable` is ``True`` then the sorting routine becomes stable, preserving + the order of equivalent elements. + + A namedtuple of (values, indices) is returned, where the `values` are the + sorted values and `indices` are the indices of the elements in the original + `input` tensor. + + Args: + input (Tensor): the input tensor. + dim (int, optional): the dimension to sort along + descending (bool, optional): controls the sorting order (ascending or descending) + + Keyword args: + stable (bool, optional): makes the sorting routine stable, which guarantees that the order + of equivalent elements is preserved. + out (tuple, optional): the output tuple of (`Tensor`, `LongTensor`) that can + be optionally given to be used as output buffers + + Example:: + + >>> x = torch.randn(3, 4) + >>> sorted, indices = torch.sort(x) + >>> sorted + tensor([[-0.2162, 0.0608, 0.6719, 2.3332], + [-0.5793, 0.0061, 0.6058, 0.9497], + [-0.5071, 0.3343, 0.9553, 1.0960]]) + >>> indices + tensor([[ 1, 0, 2, 3], + [ 3, 1, 0, 2], + [ 0, 3, 1, 2]]) + + >>> sorted, indices = torch.sort(x, 0) + >>> sorted + tensor([[-0.5071, -0.2162, 0.6719, -0.5793], + [ 0.0608, 0.0061, 0.9497, 0.3343], + [ 0.6058, 0.9553, 1.0960, 2.3332]]) + >>> indices + tensor([[ 2, 0, 0, 1], + [ 0, 1, 1, 2], + [ 1, 2, 2, 0]]) + >>> x = torch.tensor([0, 1] * 9) + >>> x.sort() + torch.return_types.sort( + values=tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + indices=tensor([ 2, 16, 4, 6, 14, 8, 0, 10, 12, 9, 17, 15, 13, 11, 7, 5, 3, 1])) + >>> x.sort(stable=True) + torch.return_types.sort( + values=tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + indices=tensor([ 0, 2, 4, 6, 8, 10, 12, 14, 16, 1, 3, 5, 7, 9, 11, 13, 15, 17])) + """ + +@overload +def sort( + input: Tensor, + *, + stable: _bool | None, + dim: str | EllipsisType | None, + descending: _bool = False, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.sort: + r""" + sort(input, dim=-1, descending=False, *, stable=False, out=None) -> (Tensor, LongTensor) + + Sorts the elements of the :attr:`input` tensor along a given dimension + in ascending order by value. + + If :attr:`dim` is not given, the last dimension of the `input` is chosen. + + If :attr:`descending` is ``True`` then the elements are sorted in descending + order by value. + + If :attr:`stable` is ``True`` then the sorting routine becomes stable, preserving + the order of equivalent elements. + + A namedtuple of (values, indices) is returned, where the `values` are the + sorted values and `indices` are the indices of the elements in the original + `input` tensor. + + Args: + input (Tensor): the input tensor. + dim (int, optional): the dimension to sort along + descending (bool, optional): controls the sorting order (ascending or descending) + + Keyword args: + stable (bool, optional): makes the sorting routine stable, which guarantees that the order + of equivalent elements is preserved. + out (tuple, optional): the output tuple of (`Tensor`, `LongTensor`) that can + be optionally given to be used as output buffers + + Example:: + + >>> x = torch.randn(3, 4) + >>> sorted, indices = torch.sort(x) + >>> sorted + tensor([[-0.2162, 0.0608, 0.6719, 2.3332], + [-0.5793, 0.0061, 0.6058, 0.9497], + [-0.5071, 0.3343, 0.9553, 1.0960]]) + >>> indices + tensor([[ 1, 0, 2, 3], + [ 3, 1, 0, 2], + [ 0, 3, 1, 2]]) + + >>> sorted, indices = torch.sort(x, 0) + >>> sorted + tensor([[-0.5071, -0.2162, 0.6719, -0.5793], + [ 0.0608, 0.0061, 0.9497, 0.3343], + [ 0.6058, 0.9553, 1.0960, 2.3332]]) + >>> indices + tensor([[ 2, 0, 0, 1], + [ 0, 1, 1, 2], + [ 1, 2, 2, 0]]) + >>> x = torch.tensor([0, 1] * 9) + >>> x.sort() + torch.return_types.sort( + values=tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + indices=tensor([ 2, 16, 4, 6, 14, 8, 0, 10, 12, 9, 17, 15, 13, 11, 7, 5, 3, 1])) + >>> x.sort(stable=True) + torch.return_types.sort( + values=tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + indices=tensor([ 0, 2, 4, 6, 8, 10, 12, 14, 16, 1, 3, 5, 7, 9, 11, 13, 15, 17])) + """ + +@overload +def sort( + input: Tensor, + dim: str | EllipsisType | None, + descending: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.sort: + r""" + sort(input, dim=-1, descending=False, *, stable=False, out=None) -> (Tensor, LongTensor) + + Sorts the elements of the :attr:`input` tensor along a given dimension + in ascending order by value. + + If :attr:`dim` is not given, the last dimension of the `input` is chosen. + + If :attr:`descending` is ``True`` then the elements are sorted in descending + order by value. + + If :attr:`stable` is ``True`` then the sorting routine becomes stable, preserving + the order of equivalent elements. + + A namedtuple of (values, indices) is returned, where the `values` are the + sorted values and `indices` are the indices of the elements in the original + `input` tensor. + + Args: + input (Tensor): the input tensor. + dim (int, optional): the dimension to sort along + descending (bool, optional): controls the sorting order (ascending or descending) + + Keyword args: + stable (bool, optional): makes the sorting routine stable, which guarantees that the order + of equivalent elements is preserved. + out (tuple, optional): the output tuple of (`Tensor`, `LongTensor`) that can + be optionally given to be used as output buffers + + Example:: + + >>> x = torch.randn(3, 4) + >>> sorted, indices = torch.sort(x) + >>> sorted + tensor([[-0.2162, 0.0608, 0.6719, 2.3332], + [-0.5793, 0.0061, 0.6058, 0.9497], + [-0.5071, 0.3343, 0.9553, 1.0960]]) + >>> indices + tensor([[ 1, 0, 2, 3], + [ 3, 1, 0, 2], + [ 0, 3, 1, 2]]) + + >>> sorted, indices = torch.sort(x, 0) + >>> sorted + tensor([[-0.5071, -0.2162, 0.6719, -0.5793], + [ 0.0608, 0.0061, 0.9497, 0.3343], + [ 0.6058, 0.9553, 1.0960, 2.3332]]) + >>> indices + tensor([[ 2, 0, 0, 1], + [ 0, 1, 1, 2], + [ 1, 2, 2, 0]]) + >>> x = torch.tensor([0, 1] * 9) + >>> x.sort() + torch.return_types.sort( + values=tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + indices=tensor([ 2, 16, 4, 6, 14, 8, 0, 10, 12, 9, 17, 15, 13, 11, 7, 5, 3, 1])) + >>> x.sort(stable=True) + torch.return_types.sort( + values=tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1]), + indices=tensor([ 0, 2, 4, 6, 8, 10, 12, 14, 16, 1, 3, 5, 7, 9, 11, 13, 15, 17])) + """ + +def sparse_bsc_tensor( + ccol_indices: Tensor | list, + row_indices: Tensor | list, + values: Tensor | list, + size: _size | None = None, + *, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + check_invariants: _bool | None = None, +) -> Tensor: + r""" + sparse_bsc_tensor(ccol_indices, row_indices, values, size=None, *, dtype=None, device=None, pin_memory=False, requires_grad=False, check_invariants=None) -> Tensor + + Constructs a :ref:`sparse tensor in BSC (Block Compressed Sparse + Column)) ` with specified 2-dimensional blocks at the + given :attr:`ccol_indices` and :attr:`row_indices`. Sparse matrix + multiplication operations in BSC format are typically faster than that + for sparse tensors in COO format. Make you have a look at :ref:`the + note on the data type of the indices `. + + .. note:: + + If the ``device`` argument is not specified the device of the given + :attr:`values` and indices tensor(s) must match. If, however, the + argument is specified the input Tensors will be converted to the + given device and in turn determine the device of the constructed + sparse tensor. + + Args: + ccol_indices (array_like): (B+1)-dimensional array of size + ``(*batchsize, ncolblocks + 1)``. The last element of each + batch is the number of non-zeros. This tensor encodes the + index in values and row_indices depending on where the given + column starts. Each successive number in the tensor subtracted + by the number before it denotes the number of elements in a + given column. + row_indices (array_like): Row block coordinates of each block in + values. (B+1)-dimensional tensor with the same length + as values. + values (array_list): Initial blocks for the tensor. Can be a list, + tuple, NumPy ``ndarray``, and other types that + represents a (1 + 2 + K)-dimensional tensor where ``K`` is the + number of dense dimensions. + size (list, tuple, :class:`torch.Size`, optional): Size of the + sparse tensor: ``(*batchsize, nrows * blocksize[0], ncols * + blocksize[1], *densesize)`` If not provided, the size will be + inferred as the minimum size big enough to hold all non-zero + blocks. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of + returned tensor. Default: if None, infers data type from + :attr:`values`. + device (:class:`torch.device`, optional): the desired device of + returned tensor. Default: if None, uses the current device + for the default tensor type (see + :func:`torch.set_default_device`). :attr:`device` will be + the CPU for CPU tensor types and the current CUDA device for + CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + check_invariants (bool, optional): If sparse tensor invariants are checked. + Default: as returned by :func:`torch.sparse.check_sparse_tensor_invariants.is_enabled`, + initially False. + + Example:: + + >>> ccol_indices = [0, 1, 2] + >>> row_indices = [0, 1] + >>> values = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] + >>> torch.sparse_bsc_tensor(torch.tensor(ccol_indices, dtype=torch.int64), + ... torch.tensor(row_indices, dtype=torch.int64), + ... torch.tensor(values), dtype=torch.double) + tensor(ccol_indices=tensor([0, 1, 2]), + row_indices=tensor([0, 1]), + values=tensor([[[1., 2.], + [3., 4.]], + [[5., 6.], + [7., 8.]]]), size=(2, 2), nnz=2, dtype=torch.float64, + layout=torch.sparse_bsc) + """ + +def sparse_bsr_tensor( + crow_indices: Tensor | list, + col_indices: Tensor | list, + values: Tensor | list, + size: _size | None = None, + *, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + check_invariants: _bool | None = None, +) -> Tensor: + r""" + sparse_bsr_tensor(crow_indices, col_indices, values, size=None, *, dtype=None, device=None, pin_memory=False, requires_grad=False, check_invariants=None) -> Tensor + + Constructs a :ref:`sparse tensor in BSR (Block Compressed Sparse Row)) + ` with specified 2-dimensional blocks at the given + :attr:`crow_indices` and :attr:`col_indices`. Sparse matrix + multiplication operations in BSR format are typically faster than that + for sparse tensors in COO format. Make you have a look at :ref:`the + note on the data type of the indices `. + + .. note:: + + If the ``device`` argument is not specified the device of the given + :attr:`values` and indices tensor(s) must match. If, however, the + argument is specified the input Tensors will be converted to the + given device and in turn determine the device of the constructed + sparse tensor. + + Args: + crow_indices (array_like): (B+1)-dimensional array of size + ``(*batchsize, nrowblocks + 1)``. The last element of each + batch is the number of non-zeros. This tensor encodes the + block index in values and col_indices depending on where the + given row block starts. Each successive number in the tensor + subtracted by the number before it denotes the number of + blocks in a given row. + col_indices (array_like): Column block coordinates of each block + in values. (B+1)-dimensional tensor with the same length as + values. + values (array_list): Initial values for the tensor. Can be a list, + tuple, NumPy ``ndarray``, scalar, and other types that + represents a (1 + 2 + K)-dimensional tensor where ``K`` is the + number of dense dimensions. + size (list, tuple, :class:`torch.Size`, optional): Size of the + sparse tensor: ``(*batchsize, nrows * blocksize[0], ncols * + blocksize[1], *densesize)`` where ``blocksize == + values.shape[1:3]``. If not provided, the size will be + inferred as the minimum size big enough to hold all non-zero + blocks. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of + returned tensor. Default: if None, infers data type from + :attr:`values`. + device (:class:`torch.device`, optional): the desired device of + returned tensor. Default: if None, uses the current device + for the default tensor type (see + :func:`torch.set_default_device`). :attr:`device` will be + the CPU for CPU tensor types and the current CUDA device for + CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + check_invariants (bool, optional): If sparse tensor invariants are checked. + Default: as returned by :func:`torch.sparse.check_sparse_tensor_invariants.is_enabled`, + initially False. + + Example:: + + >>> crow_indices = [0, 1, 2] + >>> col_indices = [0, 1] + >>> values = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]] + >>> torch.sparse_bsr_tensor(torch.tensor(crow_indices, dtype=torch.int64), + ... torch.tensor(col_indices, dtype=torch.int64), + ... torch.tensor(values), dtype=torch.double) + tensor(crow_indices=tensor([0, 1, 2]), + col_indices=tensor([0, 1]), + values=tensor([[[1., 2.], + [3., 4.]], + [[5., 6.], + [7., 8.]]]), size=(2, 2), nnz=2, dtype=torch.float64, + layout=torch.sparse_bsr) + """ + +def sparse_compressed_tensor( + compressed_indices: Tensor | list, + plain_indices: Tensor | list, + values: Tensor | list, + size: _size | None = None, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + check_invariants: _bool | None = None, +) -> Tensor: + r""" + sparse_compressed_tensor(compressed_indices, plain_indices, values, size=None, *, dtype=None, layout=None, device=None, pin_memory=False, requires_grad=False, check_invariants=None) -> Tensor + + Constructs a :ref:`sparse tensor in Compressed Sparse format - CSR, + CSC, BSR, or BSC - ` with specified values at + the given :attr:`compressed_indices` and :attr:`plain_indices`. Sparse + matrix multiplication operations in Compressed Sparse format are + typically faster than that for sparse tensors in COO format. Make you + have a look at :ref:`the note on the data type of the indices + `. + + .. note:: + + If the ``device`` argument is not specified the device of the given + :attr:`values` and indices tensor(s) must match. If, however, the + argument is specified the input Tensors will be converted to the + given device and in turn determine the device of the constructed + sparse tensor. + + Args: + compressed_indices (array_like): (B+1)-dimensional array of size + ``(*batchsize, compressed_dim_size + 1)``. The last element of + each batch is the number of non-zero elements or blocks. This + tensor encodes the index in ``values`` and ``plain_indices`` + depending on where the given compressed dimension (row or + column) starts. Each successive number in the tensor + subtracted by the number before it denotes the number of + elements or blocks in a given compressed dimension. + plain_indices (array_like): Plain dimension (column or row) + coordinates of each element or block in values. (B+1)-dimensional + tensor with the same length as values. + + values (array_list): Initial values for the tensor. Can be a list, + tuple, NumPy ``ndarray``, scalar, and other types. that + represents a (1+K)-dimensional (for CSR and CSC layouts) or + (1+2+K)-dimensional tensor (for BSR and BSC layouts) where + ``K`` is the number of dense dimensions. + size (list, tuple, :class:`torch.Size`, optional): Size of the + sparse tensor: ``(*batchsize, nrows * blocksize[0], ncols * + blocksize[1], *densesize)`` where ``blocksize[0] == + blocksize[1] == 1`` for CSR and CSC formats. If not provided, + the size will be inferred as the minimum size big enough to + hold all non-zero elements or blocks. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of + returned tensor. Default: if None, infers data type from + :attr:`values`. + layout (:class:`torch.layout`, required): the desired layout of + returned tensor: :attr:`torch.sparse_csr`, + :attr:`torch.sparse_csc`, :attr:`torch.sparse_bsr`, or + :attr:`torch.sparse_bsc`. + device (:class:`torch.device`, optional): the desired device of + returned tensor. Default: if None, uses the current device + for the default tensor type (see + :func:`torch.set_default_device`). :attr:`device` will be + the CPU for CPU tensor types and the current CUDA device for + CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + check_invariants (bool, optional): If sparse tensor invariants are checked. + Default: as returned by :func:`torch.sparse.check_sparse_tensor_invariants.is_enabled`, + initially False. + + Example:: + + >>> compressed_indices = [0, 2, 4] + >>> plain_indices = [0, 1, 0, 1] + >>> values = [1, 2, 3, 4] + >>> torch.sparse_compressed_tensor(torch.tensor(compressed_indices, dtype=torch.int64), + ... torch.tensor(plain_indices, dtype=torch.int64), + ... torch.tensor(values), dtype=torch.double, layout=torch.sparse_csr) + tensor(crow_indices=tensor([0, 2, 4]), + col_indices=tensor([0, 1, 0, 1]), + values=tensor([1., 2., 3., 4.]), size=(2, 2), nnz=4, + dtype=torch.float64, layout=torch.sparse_csr) + """ + +def sparse_coo_tensor( + indices: Tensor, + values: Tensor | list, + size: _size | None = None, + *, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + check_invariants: _bool | None = None, + is_coalesced: _bool | None = None, +) -> Tensor: + r""" + sparse_coo_tensor(indices, values, size=None, *, dtype=None, device=None, pin_memory=False, requires_grad=False, check_invariants=None, is_coalesced=None) -> Tensor + + Constructs a :ref:`sparse tensor in COO(rdinate) format + ` with specified values at the given + :attr:`indices`. + + .. note:: + + This function returns an :ref:`uncoalesced tensor + ` when :attr:`is_coalesced` is + unspecified or ``None``. + + .. note:: + + If the ``device`` argument is not specified the device of the given + :attr:`values` and indices tensor(s) must match. If, however, the + argument is specified the input Tensors will be converted to the + given device and in turn determine the device of the constructed + sparse tensor. + + Args: + indices (array_like): Initial data for the tensor. Can be a list, tuple, + NumPy ``ndarray``, scalar, and other types. Will be cast to a :class:`torch.LongTensor` + internally. The indices are the coordinates of the non-zero values in the matrix, and thus + should be two-dimensional where the first dimension is the number of tensor dimensions and + the second dimension is the number of non-zero values. + values (array_like): Initial values for the tensor. Can be a list, tuple, + NumPy ``ndarray``, scalar, and other types. + size (list, tuple, or :class:`torch.Size`, optional): Size of the sparse tensor. If not + provided the size will be inferred as the minimum size big enough to hold all non-zero + elements. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if None, infers data type from :attr:`values`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if None, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + check_invariants (bool, optional): If sparse tensor invariants are checked. + Default: as returned by :func:`torch.sparse.check_sparse_tensor_invariants.is_enabled`, + initially False. + is_coalesced (bool, optional): When``True``, the caller is + responsible for providing tensor indices that correspond to a + coalesced tensor. If the :attr:`check_invariants` flag is + False, no error will be raised if the prerequisites are not + met and this will lead to silently incorrect results. To force + coalescion please use :meth:`coalesce` on the resulting + Tensor. + Default: None: except for trivial cases (e.g. nnz < 2) the + resulting Tensor has is_coalesced set to ``False```. + + Example:: + + >>> i = torch.tensor([[0, 1, 1], + ... [2, 0, 2]]) + >>> v = torch.tensor([3, 4, 5], dtype=torch.float32) + >>> torch.sparse_coo_tensor(i, v, [2, 4]) + tensor(indices=tensor([[0, 1, 1], + [2, 0, 2]]), + values=tensor([3., 4., 5.]), + size=(2, 4), nnz=3, layout=torch.sparse_coo) + + >>> torch.sparse_coo_tensor(i, v) # Shape inference + tensor(indices=tensor([[0, 1, 1], + [2, 0, 2]]), + values=tensor([3., 4., 5.]), + size=(2, 3), nnz=3, layout=torch.sparse_coo) + + >>> torch.sparse_coo_tensor(i, v, [2, 4], + ... dtype=torch.float64, + ... device=torch.device('cuda:0')) + tensor(indices=tensor([[0, 1, 1], + [2, 0, 2]]), + values=tensor([3., 4., 5.]), + device='cuda:0', size=(2, 4), nnz=3, dtype=torch.float64, + layout=torch.sparse_coo) + + # Create an empty sparse tensor with the following invariants: + # 1. sparse_dim + dense_dim = len(SparseTensor.shape) + # 2. SparseTensor._indices().shape = (sparse_dim, nnz) + # 3. SparseTensor._values().shape = (nnz, SparseTensor.shape[sparse_dim:]) + # + # For instance, to create an empty sparse tensor with nnz = 0, dense_dim = 0 and + # sparse_dim = 1 (hence indices is a 2D tensor of shape = (1, 0)) + >>> S = torch.sparse_coo_tensor(torch.empty([1, 0]), [], [1]) + tensor(indices=tensor([], size=(1, 0)), + values=tensor([], size=(0,)), + size=(1,), nnz=0, layout=torch.sparse_coo) + + # and to create an empty sparse tensor with nnz = 0, dense_dim = 1 and + # sparse_dim = 1 + >>> S = torch.sparse_coo_tensor(torch.empty([1, 0]), torch.empty([0, 2]), [1, 2]) + tensor(indices=tensor([], size=(1, 0)), + values=tensor([], size=(0, 2)), + size=(1, 2), nnz=0, layout=torch.sparse_coo) + + .. _torch.sparse: https://pytorch.org/docs/stable/sparse.html + """ + +def sparse_csc_tensor( + ccol_indices: Tensor | list, + row_indices: Tensor | list, + values: Tensor | list, + size: _size | None = None, + *, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + check_invariants: _bool | None = None, +) -> Tensor: + r""" + sparse_csc_tensor(ccol_indices, row_indices, values, size=None, *, dtype=None, device=None, pin_memory=False, requires_grad=False, check_invariants=None) -> Tensor + + Constructs a :ref:`sparse tensor in CSC (Compressed Sparse Column) + ` with specified values at the given + :attr:`ccol_indices` and :attr:`row_indices`. Sparse matrix + multiplication operations in CSC format are typically faster than that + for sparse tensors in COO format. Make you have a look at :ref:`the + note on the data type of the indices `. + + .. note:: + + If the ``device`` argument is not specified the device of the given + :attr:`values` and indices tensor(s) must match. If, however, the + argument is specified the input Tensors will be converted to the + given device and in turn determine the device of the constructed + sparse tensor. + + Args: + ccol_indices (array_like): (B+1)-dimensional array of size + ``(*batchsize, ncols + 1)``. The last element of each batch + is the number of non-zeros. This tensor encodes the index in + values and row_indices depending on where the given column + starts. Each successive number in the tensor subtracted by the + number before it denotes the number of elements in a given + column. + row_indices (array_like): Row coordinates of each element in + values. (B+1)-dimensional tensor with the same length as + values. + values (array_list): Initial values for the tensor. Can be a list, + tuple, NumPy ``ndarray``, scalar, and other types that + represents a (1+K)-dimensional tensor where ``K`` is the number + of dense dimensions. + size (list, tuple, :class:`torch.Size`, optional): Size of the + sparse tensor: ``(*batchsize, nrows, ncols, *densesize)``. If + not provided, the size will be inferred as the minimum size + big enough to hold all non-zero elements. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of + returned tensor. Default: if None, infers data type from + :attr:`values`. + device (:class:`torch.device`, optional): the desired device of + returned tensor. Default: if None, uses the current device + for the default tensor type (see + :func:`torch.set_default_device`). :attr:`device` will be + the CPU for CPU tensor types and the current CUDA device for + CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + check_invariants (bool, optional): If sparse tensor invariants are checked. + Default: as returned by :func:`torch.sparse.check_sparse_tensor_invariants.is_enabled`, + initially False. + + Example:: + + >>> ccol_indices = [0, 2, 4] + >>> row_indices = [0, 1, 0, 1] + >>> values = [1, 2, 3, 4] + >>> torch.sparse_csc_tensor(torch.tensor(ccol_indices, dtype=torch.int64), + ... torch.tensor(row_indices, dtype=torch.int64), + ... torch.tensor(values), dtype=torch.double) + tensor(ccol_indices=tensor([0, 2, 4]), + row_indices=tensor([0, 1, 0, 1]), + values=tensor([1., 2., 3., 4.]), size=(2, 2), nnz=4, + dtype=torch.float64, layout=torch.sparse_csc) + """ + +def sparse_csr_tensor( + crow_indices: Tensor | list, + col_indices: Tensor | list, + values: Tensor | list, + size: _size | None = None, + *, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + check_invariants: _bool | None = None, +) -> Tensor: + r""" + sparse_csr_tensor(crow_indices, col_indices, values, size=None, *, dtype=None, device=None, pin_memory=False, requires_grad=False, check_invariants=None) -> Tensor + + Constructs a :ref:`sparse tensor in CSR (Compressed Sparse Row) ` with specified + values at the given :attr:`crow_indices` and :attr:`col_indices`. Sparse matrix multiplication operations + in CSR format are typically faster than that for sparse tensors in COO format. Make you have a look + at :ref:`the note on the data type of the indices `. + + .. note:: + + If the ``device`` argument is not specified the device of the given + :attr:`values` and indices tensor(s) must match. If, however, the + argument is specified the input Tensors will be converted to the + given device and in turn determine the device of the constructed + sparse tensor. + + Args: + crow_indices (array_like): (B+1)-dimensional array of size + ``(*batchsize, nrows + 1)``. The last element of each batch + is the number of non-zeros. This tensor encodes the index in + values and col_indices depending on where the given row + starts. Each successive number in the tensor subtracted by the + number before it denotes the number of elements in a given + row. + col_indices (array_like): Column coordinates of each element in + values. (B+1)-dimensional tensor with the same length + as values. + values (array_list): Initial values for the tensor. Can be a list, + tuple, NumPy ``ndarray``, scalar, and other types that + represents a (1+K)-dimensional tensor where ``K`` is the number + of dense dimensions. + size (list, tuple, :class:`torch.Size`, optional): Size of the + sparse tensor: ``(*batchsize, nrows, ncols, *densesize)``. If + not provided, the size will be inferred as the minimum size + big enough to hold all non-zero elements. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of + returned tensor. Default: if None, infers data type from + :attr:`values`. + device (:class:`torch.device`, optional): the desired device of + returned tensor. Default: if None, uses the current device + for the default tensor type (see + :func:`torch.set_default_device`). :attr:`device` will be + the CPU for CPU tensor types and the current CUDA device for + CUDA tensor types. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + check_invariants (bool, optional): If sparse tensor invariants are checked. + Default: as returned by :func:`torch.sparse.check_sparse_tensor_invariants.is_enabled`, + initially False. + + Example:: + + >>> crow_indices = [0, 2, 4] + >>> col_indices = [0, 1, 0, 1] + >>> values = [1, 2, 3, 4] + >>> torch.sparse_csr_tensor(torch.tensor(crow_indices, dtype=torch.int64), + ... torch.tensor(col_indices, dtype=torch.int64), + ... torch.tensor(values), dtype=torch.double) + tensor(crow_indices=tensor([0, 2, 4]), + col_indices=tensor([0, 1, 0, 1]), + values=tensor([1., 2., 3., 4.]), size=(2, 2), nnz=4, + dtype=torch.float64, layout=torch.sparse_csr) + """ + +def split_copy( + input: Tensor, + split_size: _int | SymInt, + dim: _int = 0, + *, + out: tuple[Tensor, ...] | list[Tensor] | None = None, +) -> None: + r""" + Performs the same operation as :func:`torch.split`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def split_with_sizes( + input: Tensor, + split_sizes: Sequence[_int | SymInt], + dim: _int = 0, +) -> tuple[Tensor, ...]: ... +def split_with_sizes_copy( + input: Tensor, + split_sizes: Sequence[_int | SymInt], + dim: _int = 0, + *, + out: tuple[Tensor, ...] | list[Tensor] | None = None, +) -> None: + r""" + Performs the same operation as :func:`torch.split_with_sizes`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def spmm(input: Tensor, mat2: Tensor) -> Tensor: ... +def sqrt(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + sqrt(input, *, out=None) -> Tensor + + Returns a new tensor with the square-root of the elements of :attr:`input`. + + .. math:: + \text{out}_{i} = \sqrt{\text{input}_{i}} + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([-2.0755, 1.0226, 0.0831, 0.4806]) + >>> torch.sqrt(a) + tensor([ nan, 1.0112, 0.2883, 0.6933]) + """ + +def sqrt_(input: Tensor) -> Tensor: ... +def square(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + square(input: Tensor, *, out: Optional[Tensor]) -> Tensor + + Returns a new tensor with the square of the elements of :attr:`input`. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([-2.0755, 1.0226, 0.0831, 0.4806]) + >>> torch.square(a) + tensor([ 4.3077, 1.0457, 0.0069, 0.2310]) + """ + +def square_(input: Tensor) -> Tensor: ... +@overload +def squeeze(input: Tensor) -> Tensor: + r""" + squeeze(input: Tensor, dim: Optional[Union[int, List[int]]]) -> Tensor + + Returns a tensor with all specified dimensions of :attr:`input` of size `1` removed. + + For example, if `input` is of shape: + :math:`(A \times 1 \times B \times C \times 1 \times D)` then the `input.squeeze()` + will be of shape: :math:`(A \times B \times C \times D)`. + + When :attr:`dim` is given, a squeeze operation is done only in the given + dimension(s). If `input` is of shape: :math:`(A \times 1 \times B)`, + ``squeeze(input, 0)`` leaves the tensor unchanged, but ``squeeze(input, 1)`` + will squeeze the tensor to the shape :math:`(A \times B)`. + + .. note:: The returned tensor shares the storage with the input tensor, + so changing the contents of one will change the contents of the other. + + .. warning:: If the tensor has a batch dimension of size 1, then `squeeze(input)` + will also remove the batch dimension, which can lead to unexpected + errors. Consider specifying only the dims you wish to be squeezed. + + Args: + input (Tensor): the input tensor. + dim (int or tuple of ints, optional): if given, the input will be squeezed + only in the specified dimensions. + + .. versionchanged:: 2.0 + :attr:`dim` now accepts tuples of dimensions. + + Example:: + + >>> x = torch.zeros(2, 1, 2, 1, 2) + >>> x.size() + torch.Size([2, 1, 2, 1, 2]) + >>> y = torch.squeeze(x) + >>> y.size() + torch.Size([2, 2, 2]) + >>> y = torch.squeeze(x, 0) + >>> y.size() + torch.Size([2, 1, 2, 1, 2]) + >>> y = torch.squeeze(x, 1) + >>> y.size() + torch.Size([2, 2, 1, 2]) + >>> y = torch.squeeze(x, (1, 2, 3)) + torch.Size([2, 2, 2]) + """ + +@overload +def squeeze(input: Tensor, dim: _int) -> Tensor: + r""" + squeeze(input: Tensor, dim: Optional[Union[int, List[int]]]) -> Tensor + + Returns a tensor with all specified dimensions of :attr:`input` of size `1` removed. + + For example, if `input` is of shape: + :math:`(A \times 1 \times B \times C \times 1 \times D)` then the `input.squeeze()` + will be of shape: :math:`(A \times B \times C \times D)`. + + When :attr:`dim` is given, a squeeze operation is done only in the given + dimension(s). If `input` is of shape: :math:`(A \times 1 \times B)`, + ``squeeze(input, 0)`` leaves the tensor unchanged, but ``squeeze(input, 1)`` + will squeeze the tensor to the shape :math:`(A \times B)`. + + .. note:: The returned tensor shares the storage with the input tensor, + so changing the contents of one will change the contents of the other. + + .. warning:: If the tensor has a batch dimension of size 1, then `squeeze(input)` + will also remove the batch dimension, which can lead to unexpected + errors. Consider specifying only the dims you wish to be squeezed. + + Args: + input (Tensor): the input tensor. + dim (int or tuple of ints, optional): if given, the input will be squeezed + only in the specified dimensions. + + .. versionchanged:: 2.0 + :attr:`dim` now accepts tuples of dimensions. + + Example:: + + >>> x = torch.zeros(2, 1, 2, 1, 2) + >>> x.size() + torch.Size([2, 1, 2, 1, 2]) + >>> y = torch.squeeze(x) + >>> y.size() + torch.Size([2, 2, 2]) + >>> y = torch.squeeze(x, 0) + >>> y.size() + torch.Size([2, 1, 2, 1, 2]) + >>> y = torch.squeeze(x, 1) + >>> y.size() + torch.Size([2, 2, 1, 2]) + >>> y = torch.squeeze(x, (1, 2, 3)) + torch.Size([2, 2, 2]) + """ + +@overload +def squeeze(input: Tensor, dim: _size) -> Tensor: + r""" + squeeze(input: Tensor, dim: Optional[Union[int, List[int]]]) -> Tensor + + Returns a tensor with all specified dimensions of :attr:`input` of size `1` removed. + + For example, if `input` is of shape: + :math:`(A \times 1 \times B \times C \times 1 \times D)` then the `input.squeeze()` + will be of shape: :math:`(A \times B \times C \times D)`. + + When :attr:`dim` is given, a squeeze operation is done only in the given + dimension(s). If `input` is of shape: :math:`(A \times 1 \times B)`, + ``squeeze(input, 0)`` leaves the tensor unchanged, but ``squeeze(input, 1)`` + will squeeze the tensor to the shape :math:`(A \times B)`. + + .. note:: The returned tensor shares the storage with the input tensor, + so changing the contents of one will change the contents of the other. + + .. warning:: If the tensor has a batch dimension of size 1, then `squeeze(input)` + will also remove the batch dimension, which can lead to unexpected + errors. Consider specifying only the dims you wish to be squeezed. + + Args: + input (Tensor): the input tensor. + dim (int or tuple of ints, optional): if given, the input will be squeezed + only in the specified dimensions. + + .. versionchanged:: 2.0 + :attr:`dim` now accepts tuples of dimensions. + + Example:: + + >>> x = torch.zeros(2, 1, 2, 1, 2) + >>> x.size() + torch.Size([2, 1, 2, 1, 2]) + >>> y = torch.squeeze(x) + >>> y.size() + torch.Size([2, 2, 2]) + >>> y = torch.squeeze(x, 0) + >>> y.size() + torch.Size([2, 1, 2, 1, 2]) + >>> y = torch.squeeze(x, 1) + >>> y.size() + torch.Size([2, 2, 1, 2]) + >>> y = torch.squeeze(x, (1, 2, 3)) + torch.Size([2, 2, 2]) + """ + +@overload +def squeeze(input: Tensor, dim: str | EllipsisType | None) -> Tensor: + r""" + squeeze(input: Tensor, dim: Optional[Union[int, List[int]]]) -> Tensor + + Returns a tensor with all specified dimensions of :attr:`input` of size `1` removed. + + For example, if `input` is of shape: + :math:`(A \times 1 \times B \times C \times 1 \times D)` then the `input.squeeze()` + will be of shape: :math:`(A \times B \times C \times D)`. + + When :attr:`dim` is given, a squeeze operation is done only in the given + dimension(s). If `input` is of shape: :math:`(A \times 1 \times B)`, + ``squeeze(input, 0)`` leaves the tensor unchanged, but ``squeeze(input, 1)`` + will squeeze the tensor to the shape :math:`(A \times B)`. + + .. note:: The returned tensor shares the storage with the input tensor, + so changing the contents of one will change the contents of the other. + + .. warning:: If the tensor has a batch dimension of size 1, then `squeeze(input)` + will also remove the batch dimension, which can lead to unexpected + errors. Consider specifying only the dims you wish to be squeezed. + + Args: + input (Tensor): the input tensor. + dim (int or tuple of ints, optional): if given, the input will be squeezed + only in the specified dimensions. + + .. versionchanged:: 2.0 + :attr:`dim` now accepts tuples of dimensions. + + Example:: + + >>> x = torch.zeros(2, 1, 2, 1, 2) + >>> x.size() + torch.Size([2, 1, 2, 1, 2]) + >>> y = torch.squeeze(x) + >>> y.size() + torch.Size([2, 2, 2]) + >>> y = torch.squeeze(x, 0) + >>> y.size() + torch.Size([2, 1, 2, 1, 2]) + >>> y = torch.squeeze(x, 1) + >>> y.size() + torch.Size([2, 2, 1, 2]) + >>> y = torch.squeeze(x, (1, 2, 3)) + torch.Size([2, 2, 2]) + """ + +@overload +def squeeze_copy(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + Performs the same operation as :func:`torch.squeeze`, but all output tensors + are freshly created instead of aliasing the input. + """ + +@overload +def squeeze_copy( + input: Tensor, + dim: _int, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.squeeze`, but all output tensors + are freshly created instead of aliasing the input. + """ + +@overload +def squeeze_copy( + input: Tensor, + dim: _size, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.squeeze`, but all output tensors + are freshly created instead of aliasing the input. + """ + +@overload +def sspaddmm( + beta: Number | _complex, + self: Tensor, + alpha: Number | _complex, + mat1: Tensor, + mat2: Tensor, +) -> Tensor: + r""" + sspaddmm(input, mat1, mat2, *, beta=1, alpha=1, out=None) -> Tensor + + Matrix multiplies a sparse tensor :attr:`mat1` with a dense tensor + :attr:`mat2`, then adds the sparse tensor :attr:`input` to the result. + + Note: This function is equivalent to :func:`torch.addmm`, except + :attr:`input` and :attr:`mat1` are sparse. + + Args: + input (Tensor): a sparse matrix to be added + mat1 (Tensor): a sparse matrix to be matrix multiplied + mat2 (Tensor): a dense matrix to be matrix multiplied + + Keyword args: + beta (Number, optional): multiplier for :attr:`mat` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat1 @ mat2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + """ + +@overload +def sspaddmm( + input: Tensor, + mat1: Tensor, + mat2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + sspaddmm(input, mat1, mat2, *, beta=1, alpha=1, out=None) -> Tensor + + Matrix multiplies a sparse tensor :attr:`mat1` with a dense tensor + :attr:`mat2`, then adds the sparse tensor :attr:`input` to the result. + + Note: This function is equivalent to :func:`torch.addmm`, except + :attr:`input` and :attr:`mat1` are sparse. + + Args: + input (Tensor): a sparse matrix to be added + mat1 (Tensor): a sparse matrix to be matrix multiplied + mat2 (Tensor): a dense matrix to be matrix multiplied + + Keyword args: + beta (Number, optional): multiplier for :attr:`mat` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat1 @ mat2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + """ + +@overload +def sspaddmm( + beta: Number | _complex, + self: Tensor, + mat1: Tensor, + mat2: Tensor, +) -> Tensor: + r""" + sspaddmm(input, mat1, mat2, *, beta=1, alpha=1, out=None) -> Tensor + + Matrix multiplies a sparse tensor :attr:`mat1` with a dense tensor + :attr:`mat2`, then adds the sparse tensor :attr:`input` to the result. + + Note: This function is equivalent to :func:`torch.addmm`, except + :attr:`input` and :attr:`mat1` are sparse. + + Args: + input (Tensor): a sparse matrix to be added + mat1 (Tensor): a sparse matrix to be matrix multiplied + mat2 (Tensor): a dense matrix to be matrix multiplied + + Keyword args: + beta (Number, optional): multiplier for :attr:`mat` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat1 @ mat2` (:math:`\alpha`) + out (Tensor, optional): the output tensor. + """ + +def stack( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + dim: _int = 0, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + stack(tensors, dim=0, *, out=None) -> Tensor + + Concatenates a sequence of tensors along a new dimension. + + All tensors need to be of the same size. + + .. seealso:: + + :func:`torch.cat` concatenates the given sequence along an existing dimension. + + Arguments: + tensors (sequence of Tensors): sequence of tensors to concatenate + dim (int, optional): dimension to insert. Has to be between 0 and the number + of dimensions of concatenated tensors (inclusive). Default: 0 + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> x = torch.randn(2, 3) + >>> x + tensor([[ 0.3367, 0.1288, 0.2345], + [ 0.2303, -1.1229, -0.1863]]) + >>> torch.stack((x, x)) # same as torch.stack((x, x), dim=0) + tensor([[[ 0.3367, 0.1288, 0.2345], + [ 0.2303, -1.1229, -0.1863]], + + [[ 0.3367, 0.1288, 0.2345], + [ 0.2303, -1.1229, -0.1863]]]) + >>> torch.stack((x, x)).size() + torch.Size([2, 2, 3]) + >>> torch.stack((x, x), dim=1) + tensor([[[ 0.3367, 0.1288, 0.2345], + [ 0.3367, 0.1288, 0.2345]], + + [[ 0.2303, -1.1229, -0.1863], + [ 0.2303, -1.1229, -0.1863]]]) + >>> torch.stack((x, x), dim=2) + tensor([[[ 0.3367, 0.3367], + [ 0.1288, 0.1288], + [ 0.2345, 0.2345]], + + [[ 0.2303, 0.2303], + [-1.1229, -1.1229], + [-0.1863, -0.1863]]]) + >>> torch.stack((x, x), dim=-1) + tensor([[[ 0.3367, 0.3367], + [ 0.1288, 0.1288], + [ 0.2345, 0.2345]], + + [[ 0.2303, 0.2303], + [-1.1229, -1.1229], + [-0.1863, -0.1863]]]) + """ + +@overload +def std( + input: Tensor, + dim: _int | _size | None, + unbiased: _bool = True, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + std(input, dim=None, *, correction=1, keepdim=False, out=None) -> Tensor + + Calculates the standard deviation over the dimensions specified by :attr:`dim`. + :attr:`dim` can be a single dimension, list of dimensions, or ``None`` to + reduce over all dimensions. + + The standard deviation (:math:`\sigma`) is calculated as + + .. math:: \sigma = \sqrt{\frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2} + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.std(a, dim=1, keepdim=True) + tensor([[1.0311], + [0.7477], + [1.2204], + [0.9087]]) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def std( + input: Tensor, + dim: _int | _size | None = None, + *, + correction: Number | _complex | None = None, + keepdim: _bool = False, + out: Tensor | None = None, +) -> Tensor: + r""" + std(input, dim=None, *, correction=1, keepdim=False, out=None) -> Tensor + + Calculates the standard deviation over the dimensions specified by :attr:`dim`. + :attr:`dim` can be a single dimension, list of dimensions, or ``None`` to + reduce over all dimensions. + + The standard deviation (:math:`\sigma`) is calculated as + + .. math:: \sigma = \sqrt{\frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2} + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.std(a, dim=1, keepdim=True) + tensor([[1.0311], + [0.7477], + [1.2204], + [0.9087]]) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def std(input: Tensor, unbiased: _bool = True) -> Tensor: + r""" + std(input, dim=None, *, correction=1, keepdim=False, out=None) -> Tensor + + Calculates the standard deviation over the dimensions specified by :attr:`dim`. + :attr:`dim` can be a single dimension, list of dimensions, or ``None`` to + reduce over all dimensions. + + The standard deviation (:math:`\sigma`) is calculated as + + .. math:: \sigma = \sqrt{\frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2} + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.std(a, dim=1, keepdim=True) + tensor([[1.0311], + [0.7477], + [1.2204], + [0.9087]]) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def std( + input: Tensor, + dim: Sequence[str | EllipsisType | None], + *, + correction: Number | _complex | None = None, + keepdim: _bool = False, + out: Tensor | None = None, +) -> Tensor: + r""" + std(input, dim=None, *, correction=1, keepdim=False, out=None) -> Tensor + + Calculates the standard deviation over the dimensions specified by :attr:`dim`. + :attr:`dim` can be a single dimension, list of dimensions, or ``None`` to + reduce over all dimensions. + + The standard deviation (:math:`\sigma`) is calculated as + + .. math:: \sigma = \sqrt{\frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2} + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.std(a, dim=1, keepdim=True) + tensor([[1.0311], + [0.7477], + [1.2204], + [0.9087]]) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def std( + input: Tensor, + dim: Sequence[str | EllipsisType | None], + unbiased: _bool = True, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + std(input, dim=None, *, correction=1, keepdim=False, out=None) -> Tensor + + Calculates the standard deviation over the dimensions specified by :attr:`dim`. + :attr:`dim` can be a single dimension, list of dimensions, or ``None`` to + reduce over all dimensions. + + The standard deviation (:math:`\sigma`) is calculated as + + .. math:: \sigma = \sqrt{\frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2} + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.std(a, dim=1, keepdim=True) + tensor([[1.0311], + [0.7477], + [1.2204], + [0.9087]]) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def std_mean( + input: Tensor, + dim: _int | _size | None, + unbiased: _bool = True, + keepdim: _bool = False, +) -> tuple[Tensor, Tensor]: + r""" + std_mean(input, dim=None, *, correction=1, keepdim=False, out=None) -> (Tensor, Tensor) + + Calculates the standard deviation and mean over the dimensions specified by + :attr:`dim`. :attr:`dim` can be a single dimension, list of dimensions, or + ``None`` to reduce over all dimensions. + + The standard deviation (:math:`\sigma`) is calculated as + + .. math:: \sigma = \sqrt{\frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2} + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Returns: + A tuple (std, mean) containing the standard deviation and mean. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.std_mean(a, dim=0, keepdim=True) + (tensor([[1.2620, 1.0028, 1.0957, 0.6038]]), + tensor([[ 0.0645, 0.4485, 0.8707, -0.0665]])) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def std_mean( + input: Tensor, + dim: _int | _size | None = None, + *, + correction: Number | _complex | None = None, + keepdim: _bool = False, +) -> tuple[Tensor, Tensor]: + r""" + std_mean(input, dim=None, *, correction=1, keepdim=False, out=None) -> (Tensor, Tensor) + + Calculates the standard deviation and mean over the dimensions specified by + :attr:`dim`. :attr:`dim` can be a single dimension, list of dimensions, or + ``None`` to reduce over all dimensions. + + The standard deviation (:math:`\sigma`) is calculated as + + .. math:: \sigma = \sqrt{\frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2} + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Returns: + A tuple (std, mean) containing the standard deviation and mean. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.std_mean(a, dim=0, keepdim=True) + (tensor([[1.2620, 1.0028, 1.0957, 0.6038]]), + tensor([[ 0.0645, 0.4485, 0.8707, -0.0665]])) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def std_mean( + input: Tensor, + unbiased: _bool = True, +) -> tuple[Tensor, Tensor]: + r""" + std_mean(input, dim=None, *, correction=1, keepdim=False, out=None) -> (Tensor, Tensor) + + Calculates the standard deviation and mean over the dimensions specified by + :attr:`dim`. :attr:`dim` can be a single dimension, list of dimensions, or + ``None`` to reduce over all dimensions. + + The standard deviation (:math:`\sigma`) is calculated as + + .. math:: \sigma = \sqrt{\frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2} + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Returns: + A tuple (std, mean) containing the standard deviation and mean. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.std_mean(a, dim=0, keepdim=True) + (tensor([[1.2620, 1.0028, 1.0957, 0.6038]]), + tensor([[ 0.0645, 0.4485, 0.8707, -0.0665]])) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def std_mean( + input: Tensor, + dim: Sequence[str | EllipsisType | None], + *, + correction: Number | _complex | None = None, + keepdim: _bool = False, +) -> tuple[Tensor, Tensor]: + r""" + std_mean(input, dim=None, *, correction=1, keepdim=False, out=None) -> (Tensor, Tensor) + + Calculates the standard deviation and mean over the dimensions specified by + :attr:`dim`. :attr:`dim` can be a single dimension, list of dimensions, or + ``None`` to reduce over all dimensions. + + The standard deviation (:math:`\sigma`) is calculated as + + .. math:: \sigma = \sqrt{\frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2} + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Returns: + A tuple (std, mean) containing the standard deviation and mean. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.std_mean(a, dim=0, keepdim=True) + (tensor([[1.2620, 1.0028, 1.0957, 0.6038]]), + tensor([[ 0.0645, 0.4485, 0.8707, -0.0665]])) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def std_mean( + input: Tensor, + dim: Sequence[str | EllipsisType | None], + unbiased: _bool = True, + keepdim: _bool = False, +) -> tuple[Tensor, Tensor]: + r""" + std_mean(input, dim=None, *, correction=1, keepdim=False, out=None) -> (Tensor, Tensor) + + Calculates the standard deviation and mean over the dimensions specified by + :attr:`dim`. :attr:`dim` can be a single dimension, list of dimensions, or + ``None`` to reduce over all dimensions. + + The standard deviation (:math:`\sigma`) is calculated as + + .. math:: \sigma = \sqrt{\frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2} + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Returns: + A tuple (std, mean) containing the standard deviation and mean. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.std_mean(a, dim=0, keepdim=True) + (tensor([[1.2620, 1.0028, 1.0957, 0.6038]]), + tensor([[ 0.0645, 0.4485, 0.8707, -0.0665]])) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def sub( + input: Tensor | Number | _complex, + other: Tensor | Number | _complex, + *, + alpha: Number | _complex | None = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + sub(input, other, *, alpha=1, out=None) -> Tensor + + Subtracts :attr:`other`, scaled by :attr:`alpha`, from :attr:`input`. + + .. math:: + \text{{out}}_i = \text{{input}}_i - \text{{alpha}} \times \text{{other}}_i + + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer, float, and complex inputs. + + Args: + input (Tensor): the input tensor. + other (Tensor or Number): the tensor or number to subtract from :attr:`input`. + + Keyword args: + alpha (Number): the multiplier for :attr:`other`. + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor((1, 2)) + >>> b = torch.tensor((0, 1)) + >>> torch.sub(a, b, alpha=2) + tensor([1, 0]) + """ + +@overload +def sub(self: Tensor, alpha: Number | _complex, other: Tensor) -> Tensor: + r""" + sub(input, other, *, alpha=1, out=None) -> Tensor + + Subtracts :attr:`other`, scaled by :attr:`alpha`, from :attr:`input`. + + .. math:: + \text{{out}}_i = \text{{input}}_i - \text{{alpha}} \times \text{{other}}_i + + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer, float, and complex inputs. + + Args: + input (Tensor): the input tensor. + other (Tensor or Number): the tensor or number to subtract from :attr:`input`. + + Keyword args: + alpha (Number): the multiplier for :attr:`other`. + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor((1, 2)) + >>> b = torch.tensor((0, 1)) + >>> torch.sub(a, b, alpha=2) + tensor([1, 0]) + """ + +@overload +def sub( + self: Tensor, + alpha: Number | _complex, + other: Tensor, + *, + out: Tensor, +) -> Tensor: + r""" + sub(input, other, *, alpha=1, out=None) -> Tensor + + Subtracts :attr:`other`, scaled by :attr:`alpha`, from :attr:`input`. + + .. math:: + \text{{out}}_i = \text{{input}}_i - \text{{alpha}} \times \text{{other}}_i + + + Supports :ref:`broadcasting to a common shape `, + :ref:`type promotion `, and integer, float, and complex inputs. + + Args: + input (Tensor): the input tensor. + other (Tensor or Number): the tensor or number to subtract from :attr:`input`. + + Keyword args: + alpha (Number): the multiplier for :attr:`other`. + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor((1, 2)) + >>> b = torch.tensor((0, 1)) + >>> torch.sub(a, b, alpha=2) + tensor([1, 0]) + """ + +@overload +def subtract( + input: Tensor, + other: Tensor, + *, + alpha: Number | _complex = 1, + out: Tensor | None = None, +) -> Tensor: + r""" + subtract(input, other, *, alpha=1, out=None) -> Tensor + + Alias for :func:`torch.sub`. + """ + +@overload +def subtract( + input: Tensor, + other: Number | _complex, + alpha: Number | _complex = 1, +) -> Tensor: + r""" + subtract(input, other, *, alpha=1, out=None) -> Tensor + + Alias for :func:`torch.sub`. + """ + +@overload +def sum(input: Tensor, *, dtype: _dtype | None = None) -> Tensor: + r""" + sum(input, *, dtype=None) -> Tensor + + Returns the sum of all elements in the :attr:`input` tensor. + + Args: + input (Tensor): the input tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + .. note:: Use the `dtype` argument if you need the result in a specific tensor type. + Otherwise, the result type may be automatically promoted (e.g., from `torch.int32` to `torch.int64`). + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.1133, -0.9567, 0.2958]]) + >>> torch.sum(a) + tensor(-0.5475) + + .. function:: sum(input, dim, keepdim=False, *, dtype=None) -> Tensor + :noindex: + + Returns the sum of each row of the :attr:`input` tensor in the given + dimension :attr:`dim`. If :attr:`dim` is a list of dimensions, + reduce over all of them. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[ 0.0569, -0.2475, 0.0737, -0.3429], + [-0.2993, 0.9138, 0.9337, -1.6864], + [ 0.1132, 0.7892, -0.1003, 0.5688], + [ 0.3637, -0.9906, -0.4752, -1.5197]]) + >>> torch.sum(a, 1) + tensor([-0.4598, -0.1381, 1.3708, -2.6217]) + >>> b = torch.arange(4 * 5 * 6).view(4, 5, 6) + >>> torch.sum(b, (2, 1)) + tensor([ 435., 1335., 2235., 3135.]) + """ + +@overload +def sum( + input: Tensor, + dim: _int | _size | None, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + sum(input, *, dtype=None) -> Tensor + + Returns the sum of all elements in the :attr:`input` tensor. + + Args: + input (Tensor): the input tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + .. note:: Use the `dtype` argument if you need the result in a specific tensor type. + Otherwise, the result type may be automatically promoted (e.g., from `torch.int32` to `torch.int64`). + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.1133, -0.9567, 0.2958]]) + >>> torch.sum(a) + tensor(-0.5475) + + .. function:: sum(input, dim, keepdim=False, *, dtype=None) -> Tensor + :noindex: + + Returns the sum of each row of the :attr:`input` tensor in the given + dimension :attr:`dim`. If :attr:`dim` is a list of dimensions, + reduce over all of them. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[ 0.0569, -0.2475, 0.0737, -0.3429], + [-0.2993, 0.9138, 0.9337, -1.6864], + [ 0.1132, 0.7892, -0.1003, 0.5688], + [ 0.3637, -0.9906, -0.4752, -1.5197]]) + >>> torch.sum(a, 1) + tensor([-0.4598, -0.1381, 1.3708, -2.6217]) + >>> b = torch.arange(4 * 5 * 6).view(4, 5, 6) + >>> torch.sum(b, (2, 1)) + tensor([ 435., 1335., 2235., 3135.]) + """ + +@overload +def sum( + input: Tensor, + dim: Sequence[str | EllipsisType | None], + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + out: Tensor | None = None, +) -> Tensor: + r""" + sum(input, *, dtype=None) -> Tensor + + Returns the sum of all elements in the :attr:`input` tensor. + + Args: + input (Tensor): the input tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + .. note:: Use the `dtype` argument if you need the result in a specific tensor type. + Otherwise, the result type may be automatically promoted (e.g., from `torch.int32` to `torch.int64`). + + Example:: + + >>> a = torch.randn(1, 3) + >>> a + tensor([[ 0.1133, -0.9567, 0.2958]]) + >>> torch.sum(a) + tensor(-0.5475) + + .. function:: sum(input, dim, keepdim=False, *, dtype=None) -> Tensor + :noindex: + + Returns the sum of each row of the :attr:`input` tensor in the given + dimension :attr:`dim`. If :attr:`dim` is a list of dimensions, + reduce over all of them. + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + Example:: + + >>> a = torch.randn(4, 4) + >>> a + tensor([[ 0.0569, -0.2475, 0.0737, -0.3429], + [-0.2993, 0.9138, 0.9337, -1.6864], + [ 0.1132, 0.7892, -0.1003, 0.5688], + [ 0.3637, -0.9906, -0.4752, -1.5197]]) + >>> torch.sum(a, 1) + tensor([-0.4598, -0.1381, 1.3708, -2.6217]) + >>> b = torch.arange(4 * 5 * 6).view(4, 5, 6) + >>> torch.sum(b, (2, 1)) + tensor([ 435., 1335., 2235., 3135.]) + """ + +def svd( + input: Tensor, + some: _bool = True, + compute_uv: _bool = True, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.svd: + r""" + svd(input, some=True, compute_uv=True, *, out=None) -> (Tensor, Tensor, Tensor) + + Computes the singular value decomposition of either a matrix or batch of + matrices :attr:`input`. The singular value decomposition is represented as a + namedtuple `(U, S, V)`, such that :attr:`input` :math:`= U \text{diag}(S) V^{\text{H}}`. + where :math:`V^{\text{H}}` is the transpose of `V` for real inputs, + and the conjugate transpose of `V` for complex inputs. + If :attr:`input` is a batch of matrices, then `U`, `S`, and `V` are also + batched with the same batch dimensions as :attr:`input`. + + If :attr:`some` is `True` (default), the method returns the reduced singular + value decomposition. In this case, if the last two dimensions of :attr:`input` are + `m` and `n`, then the returned `U` and `V` matrices will contain only + `min(n, m)` orthonormal columns. + + If :attr:`compute_uv` is `False`, the returned `U` and `V` will be + zero-filled matrices of shape `(m, m)` and `(n, n)` + respectively, and the same device as :attr:`input`. The argument :attr:`some` + has no effect when :attr:`compute_uv` is `False`. + + Supports :attr:`input` of float, double, cfloat and cdouble data types. + The dtypes of `U` and `V` are the same as :attr:`input`'s. `S` will + always be real-valued, even if :attr:`input` is complex. + + .. warning:: + + :func:`torch.svd` is deprecated in favor of :func:`torch.linalg.svd` + and will be removed in a future PyTorch release. + + ``U, S, V = torch.svd(A, some=some, compute_uv=True)`` (default) should be replaced with + + .. code:: python + + U, S, Vh = torch.linalg.svd(A, full_matrices=not some) + V = Vh.mH + + ``_, S, _ = torch.svd(A, some=some, compute_uv=False)`` should be replaced with + + .. code:: python + + S = torch.linalg.svdvals(A) + + .. note:: Differences with :func:`torch.linalg.svd`: + + * :attr:`some` is the opposite of + :func:`torch.linalg.svd`'s :attr:`full_matrices`. Note that + default value for both is `True`, so the default behavior is + effectively the opposite. + * :func:`torch.svd` returns `V`, whereas :func:`torch.linalg.svd` returns + `Vh`, that is, :math:`V^{\text{H}}`. + * If :attr:`compute_uv` is `False`, :func:`torch.svd` returns zero-filled + tensors for `U` and `Vh`, whereas :func:`torch.linalg.svd` returns + empty tensors. + + .. note:: The singular values are returned in descending order. If :attr:`input` is a batch of matrices, + then the singular values of each matrix in the batch are returned in descending order. + + .. note:: The `S` tensor can only be used to compute gradients if :attr:`compute_uv` is `True`. + + .. note:: When :attr:`some` is `False`, the gradients on `U[..., :, min(m, n):]` + and `V[..., :, min(m, n):]` will be ignored in the backward pass, as those vectors + can be arbitrary bases of the corresponding subspaces. + + .. note:: The implementation of :func:`torch.linalg.svd` on CPU uses LAPACK's routine `?gesdd` + (a divide-and-conquer algorithm) instead of `?gesvd` for speed. Analogously, + on GPU, it uses cuSOLVER's routines `gesvdj` and `gesvdjBatched` on CUDA 10.1.243 + and later, and MAGMA's routine `gesdd` on earlier versions of CUDA. + + .. note:: The returned `U` will not be contiguous. The matrix (or batch of matrices) will + be represented as a column-major matrix (i.e. Fortran-contiguous). + + .. warning:: The gradients with respect to `U` and `V` will only be finite when the input does not + have zero nor repeated singular values. + + .. warning:: If the distance between any two singular values is close to zero, the gradients with respect to + `U` and `V` will be numerically unstable, as they depends on + :math:`\frac{1}{\min_{i \neq j} \sigma_i^2 - \sigma_j^2}`. The same happens when the matrix + has small singular values, as these gradients also depend on `S^{-1}`. + + .. warning:: For complex-valued :attr:`input` the singular value decomposition is not unique, + as `U` and `V` may be multiplied by an arbitrary phase factor :math:`e^{i \phi}` on every column. + The same happens when :attr:`input` has repeated singular values, where one may multiply + the columns of the spanning subspace in `U` and `V` by a rotation matrix + and `the resulting vectors will span the same subspace`_. + Different platforms, like NumPy, or inputs on different device types, + may produce different `U` and `V` tensors. + + Args: + input (Tensor): the input tensor of size `(*, m, n)` where `*` is zero or more + batch dimensions consisting of `(m, n)` matrices. + some (bool, optional): controls whether to compute the reduced or full decomposition, and + consequently, the shape of returned `U` and `V`. Default: `True`. + compute_uv (bool, optional): controls whether to compute `U` and `V`. Default: `True`. + + Keyword args: + out (tuple, optional): the output tuple of tensors + + Example:: + + >>> a = torch.randn(5, 3) + >>> a + tensor([[ 0.2364, -0.7752, 0.6372], + [ 1.7201, 0.7394, -0.0504], + [-0.3371, -1.0584, 0.5296], + [ 0.3550, -0.4022, 1.5569], + [ 0.2445, -0.0158, 1.1414]]) + >>> u, s, v = torch.svd(a) + >>> u + tensor([[ 0.4027, 0.0287, 0.5434], + [-0.1946, 0.8833, 0.3679], + [ 0.4296, -0.2890, 0.5261], + [ 0.6604, 0.2717, -0.2618], + [ 0.4234, 0.2481, -0.4733]]) + >>> s + tensor([2.3289, 2.0315, 0.7806]) + >>> v + tensor([[-0.0199, 0.8766, 0.4809], + [-0.5080, 0.4054, -0.7600], + [ 0.8611, 0.2594, -0.4373]]) + >>> torch.dist(a, torch.mm(torch.mm(u, torch.diag(s)), v.t())) + tensor(8.6531e-07) + >>> a_big = torch.randn(7, 5, 3) + >>> u, s, v = torch.svd(a_big) + >>> torch.dist(a_big, torch.matmul(torch.matmul(u, torch.diag_embed(s)), v.mT)) + tensor(2.6503e-06) + + .. _the resulting vectors will span the same subspace: + (https://en.wikipedia.org/wiki/Singular_value_decomposition#Singular_values,_singular_vectors,_and_their_relation_to_the_SVD) + """ + +def swapaxes(input: Tensor, axis0: _int, axis1: _int) -> Tensor: + r""" + swapaxes(input, axis0, axis1) -> Tensor + + Alias for :func:`torch.transpose`. + + This function is equivalent to NumPy's swapaxes function. + + Examples:: + + >>> x = torch.tensor([[[0,1],[2,3]],[[4,5],[6,7]]]) + >>> x + tensor([[[0, 1], + [2, 3]], + + [[4, 5], + [6, 7]]]) + >>> torch.swapaxes(x, 0, 1) + tensor([[[0, 1], + [4, 5]], + + [[2, 3], + [6, 7]]]) + >>> torch.swapaxes(x, 0, 2) + tensor([[[0, 4], + [2, 6]], + + [[1, 5], + [3, 7]]]) + """ + +def swapdims(input: Tensor, dim0: _int, dim1: _int) -> Tensor: + r""" + swapdims(input, dim0, dim1) -> Tensor + + Alias for :func:`torch.transpose`. + + This function is equivalent to NumPy's swapaxes function. + + Examples:: + + >>> x = torch.tensor([[[0,1],[2,3]],[[4,5],[6,7]]]) + >>> x + tensor([[[0, 1], + [2, 3]], + + [[4, 5], + [6, 7]]]) + >>> torch.swapdims(x, 0, 1) + tensor([[[0, 1], + [4, 5]], + + [[2, 3], + [6, 7]]]) + >>> torch.swapdims(x, 0, 2) + tensor([[[0, 4], + [2, 6]], + + [[1, 5], + [3, 7]]]) + """ + +def sym_constrain_range( + size: Number | _complex, + *, + min: _int | None = None, + max: _int | None = None, +) -> None: ... +def sym_constrain_range_for_size( + size: Number | _complex, + *, + min: _int | None = None, + max: _int | None = None, +) -> None: ... +def t(input: Tensor) -> Tensor: + r""" + t(input) -> Tensor + + Expects :attr:`input` to be <= 2-D tensor and transposes dimensions 0 + and 1. + + 0-D and 1-D tensors are returned as is. When input is a 2-D tensor this + is equivalent to ``transpose(input, 0, 1)``. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> x = torch.randn(()) + >>> x + tensor(0.1995) + >>> torch.t(x) + tensor(0.1995) + >>> x = torch.randn(3) + >>> x + tensor([ 2.4320, -0.4608, 0.7702]) + >>> torch.t(x) + tensor([ 2.4320, -0.4608, 0.7702]) + >>> x = torch.randn(2, 3) + >>> x + tensor([[ 0.4875, 0.9158, -0.5872], + [ 0.3938, -0.6929, 0.6932]]) + >>> torch.t(x) + tensor([[ 0.4875, 0.3938], + [ 0.9158, -0.6929], + [-0.5872, 0.6932]]) + + See also :func:`torch.transpose`. + """ + +def t_copy(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + Performs the same operation as :func:`torch.t`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def take( + input: Tensor, + index: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + take(input, index) -> Tensor + + Returns a new tensor with the elements of :attr:`input` at the given indices. + The input tensor is treated as if it were viewed as a 1-D tensor. The result + takes the same shape as the indices. + + Args: + input (Tensor): the input tensor. + index (LongTensor): the indices into tensor + + Example:: + + >>> src = torch.tensor([[4, 3, 5], + ... [6, 7, 8]]) + >>> torch.take(src, torch.tensor([0, 2, 5])) + tensor([ 4, 5, 8]) + """ + +def take_along_dim( + input: Tensor, + indices: Tensor, + dim: _int | None = None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + take_along_dim(input, indices, dim=None, *, out=None) -> Tensor + + Selects values from :attr:`input` at the 1-dimensional indices from :attr:`indices` along the given :attr:`dim`. + + If :attr:`dim` is None, the input array is treated as if it has been flattened to 1d. + + Functions that return indices along a dimension, like :func:`torch.argmax` and :func:`torch.argsort`, + are designed to work with this function. See the examples below. + + .. note:: + This function is similar to NumPy's `take_along_axis`. + See also :func:`torch.gather`. + + Args: + input (Tensor): the input tensor. + indices (LongTensor): the indices into :attr:`input`. Must have long dtype. + dim (int, optional): dimension to select along. Default: 0 + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> t = torch.tensor([[10, 30, 20], [60, 40, 50]]) + >>> max_idx = torch.argmax(t) + >>> torch.take_along_dim(t, max_idx) + tensor([60]) + >>> sorted_idx = torch.argsort(t, dim=1) + >>> torch.take_along_dim(t, sorted_idx, dim=1) + tensor([[10, 20, 30], + [40, 50, 60]]) + """ + +def tan(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + tan(input, *, out=None) -> Tensor + + Returns a new tensor with the tangent of the elements in the :attr:`input` tensor, + where each value in this input tensor is in radians. + + .. math:: + \text{out}_{i} = \tan(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([-1.2027, -1.7687, 0.4412, -1.3856]) + >>> torch.tan(a) + tensor([-2.5930, 4.9859, 0.4722, -5.3366]) + """ + +def tan_(input: Tensor) -> Tensor: ... +def tanh(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + tanh(input, *, out=None) -> Tensor + + Returns a new tensor with the hyperbolic tangent of the elements + of :attr:`input`. + + .. math:: + \text{out}_{i} = \tanh(\text{input}_{i}) + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([ 0.8986, -0.7279, 1.1745, 0.2611]) + >>> torch.tanh(a) + tensor([ 0.7156, -0.6218, 0.8257, 0.2553]) + """ + +def tanh_(input: Tensor) -> Tensor: ... +def tensor( + data: Any, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + pin_memory: _bool = False, +) -> Tensor: + r""" + tensor(data, *, dtype=None, device=None, requires_grad=False, pin_memory=False) -> Tensor + + Constructs a tensor with no autograd history (also known as a "leaf tensor", see :doc:`/notes/autograd`) by copying :attr:`data`. + + .. warning:: + + When working with tensors prefer using :func:`torch.Tensor.clone`, + :func:`torch.Tensor.detach`, and :func:`torch.Tensor.requires_grad_` for + readability. Letting `t` be a tensor, ``torch.tensor(t)`` is equivalent to + ``t.detach().clone()``, and ``torch.tensor(t, requires_grad=True)`` + is equivalent to ``t.detach().clone().requires_grad_(True)``. + + .. seealso:: + + :func:`torch.as_tensor` preserves autograd history and avoids copies where possible. + :func:`torch.from_numpy` creates a tensor that shares storage with a NumPy array. + + Args: + data (array_like): Initial data for the tensor. Can be a list, tuple, + NumPy ``ndarray``, scalar, and other types. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, infers data type from :attr:`data`. + device (:class:`torch.device`, optional): the device of the constructed tensor. If None and data is a tensor + then the device of data is used. If None and data is not a tensor then + the result tensor is constructed on the current device. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + + Example:: + + >>> torch.tensor([[0.1, 1.2], [2.2, 3.1], [4.9, 5.2]]) + tensor([[ 0.1000, 1.2000], + [ 2.2000, 3.1000], + [ 4.9000, 5.2000]]) + + >>> torch.tensor([0, 1]) # Type inference on data + tensor([ 0, 1]) + + >>> torch.tensor([[0.11111, 0.222222, 0.3333333]], + ... dtype=torch.float64, + ... device=torch.device('cuda:0')) # creates a double tensor on a CUDA device + tensor([[ 0.1111, 0.2222, 0.3333]], dtype=torch.float64, device='cuda:0') + + >>> torch.tensor(3.14159) # Create a zero-dimensional (scalar) tensor + tensor(3.1416) + + >>> torch.tensor([]) # Create an empty tensor (of size (0,)) + tensor([]) + """ + +@overload +def tensor_split( + input: Tensor, + tensor_indices_or_sections: Tensor, + dim: _int = 0, +) -> tuple[Tensor, ...]: + r""" + tensor_split(input, indices_or_sections, dim=0) -> List of Tensors + + Splits a tensor into multiple sub-tensors, all of which are views of :attr:`input`, + along dimension :attr:`dim` according to the indices or number of sections specified + by :attr:`indices_or_sections`. This function is based on NumPy's + :func:`numpy.array_split`. + + Args: + input (Tensor): the tensor to split + indices_or_sections (Tensor, int or list or tuple of ints): + If :attr:`indices_or_sections` is an integer ``n`` or a zero dimensional long tensor + with value ``n``, :attr:`input` is split into ``n`` sections along dimension :attr:`dim`. + If :attr:`input` is divisible by ``n`` along dimension :attr:`dim`, each + section will be of equal size, :code:`input.size(dim) / n`. If :attr:`input` + is not divisible by ``n``, the sizes of the first :code:`int(input.size(dim) % n)` + sections will have size :code:`int(input.size(dim) / n) + 1`, and the rest will + have size :code:`int(input.size(dim) / n)`. + + If :attr:`indices_or_sections` is a list or tuple of ints, or a one-dimensional long + tensor, then :attr:`input` is split along dimension :attr:`dim` at each of the indices + in the list, tuple or tensor. For instance, :code:`indices_or_sections=[2, 3]` and :code:`dim=0` + would result in the tensors :code:`input[:2]`, :code:`input[2:3]`, and :code:`input[3:]`. + + If :attr:`indices_or_sections` is a tensor, it must be a zero-dimensional or one-dimensional + long tensor on the CPU. + + dim (int, optional): dimension along which to split the tensor. Default: ``0`` + + Example:: + + >>> x = torch.arange(8) + >>> torch.tensor_split(x, 3) + (tensor([0, 1, 2]), tensor([3, 4, 5]), tensor([6, 7])) + + >>> x = torch.arange(7) + >>> torch.tensor_split(x, 3) + (tensor([0, 1, 2]), tensor([3, 4]), tensor([5, 6])) + >>> torch.tensor_split(x, (1, 6)) + (tensor([0]), tensor([1, 2, 3, 4, 5]), tensor([6])) + + >>> x = torch.arange(14).reshape(2, 7) + >>> x + tensor([[ 0, 1, 2, 3, 4, 5, 6], + [ 7, 8, 9, 10, 11, 12, 13]]) + >>> torch.tensor_split(x, 3, dim=1) + (tensor([[0, 1, 2], + [7, 8, 9]]), + tensor([[ 3, 4], + [10, 11]]), + tensor([[ 5, 6], + [12, 13]])) + >>> torch.tensor_split(x, (1, 6), dim=1) + (tensor([[0], + [7]]), + tensor([[ 1, 2, 3, 4, 5], + [ 8, 9, 10, 11, 12]]), + tensor([[ 6], + [13]])) + """ + +@overload +def tensor_split( + input: Tensor, + sections: _int | SymInt, + dim: _int = 0, +) -> tuple[Tensor, ...]: + r""" + tensor_split(input, indices_or_sections, dim=0) -> List of Tensors + + Splits a tensor into multiple sub-tensors, all of which are views of :attr:`input`, + along dimension :attr:`dim` according to the indices or number of sections specified + by :attr:`indices_or_sections`. This function is based on NumPy's + :func:`numpy.array_split`. + + Args: + input (Tensor): the tensor to split + indices_or_sections (Tensor, int or list or tuple of ints): + If :attr:`indices_or_sections` is an integer ``n`` or a zero dimensional long tensor + with value ``n``, :attr:`input` is split into ``n`` sections along dimension :attr:`dim`. + If :attr:`input` is divisible by ``n`` along dimension :attr:`dim`, each + section will be of equal size, :code:`input.size(dim) / n`. If :attr:`input` + is not divisible by ``n``, the sizes of the first :code:`int(input.size(dim) % n)` + sections will have size :code:`int(input.size(dim) / n) + 1`, and the rest will + have size :code:`int(input.size(dim) / n)`. + + If :attr:`indices_or_sections` is a list or tuple of ints, or a one-dimensional long + tensor, then :attr:`input` is split along dimension :attr:`dim` at each of the indices + in the list, tuple or tensor. For instance, :code:`indices_or_sections=[2, 3]` and :code:`dim=0` + would result in the tensors :code:`input[:2]`, :code:`input[2:3]`, and :code:`input[3:]`. + + If :attr:`indices_or_sections` is a tensor, it must be a zero-dimensional or one-dimensional + long tensor on the CPU. + + dim (int, optional): dimension along which to split the tensor. Default: ``0`` + + Example:: + + >>> x = torch.arange(8) + >>> torch.tensor_split(x, 3) + (tensor([0, 1, 2]), tensor([3, 4, 5]), tensor([6, 7])) + + >>> x = torch.arange(7) + >>> torch.tensor_split(x, 3) + (tensor([0, 1, 2]), tensor([3, 4]), tensor([5, 6])) + >>> torch.tensor_split(x, (1, 6)) + (tensor([0]), tensor([1, 2, 3, 4, 5]), tensor([6])) + + >>> x = torch.arange(14).reshape(2, 7) + >>> x + tensor([[ 0, 1, 2, 3, 4, 5, 6], + [ 7, 8, 9, 10, 11, 12, 13]]) + >>> torch.tensor_split(x, 3, dim=1) + (tensor([[0, 1, 2], + [7, 8, 9]]), + tensor([[ 3, 4], + [10, 11]]), + tensor([[ 5, 6], + [12, 13]])) + >>> torch.tensor_split(x, (1, 6), dim=1) + (tensor([[0], + [7]]), + tensor([[ 1, 2, 3, 4, 5], + [ 8, 9, 10, 11, 12]]), + tensor([[ 6], + [13]])) + """ + +@overload +def tensor_split( + input: Tensor, + indices: Sequence[_int | SymInt], + dim: _int = 0, +) -> tuple[Tensor, ...]: + r""" + tensor_split(input, indices_or_sections, dim=0) -> List of Tensors + + Splits a tensor into multiple sub-tensors, all of which are views of :attr:`input`, + along dimension :attr:`dim` according to the indices or number of sections specified + by :attr:`indices_or_sections`. This function is based on NumPy's + :func:`numpy.array_split`. + + Args: + input (Tensor): the tensor to split + indices_or_sections (Tensor, int or list or tuple of ints): + If :attr:`indices_or_sections` is an integer ``n`` or a zero dimensional long tensor + with value ``n``, :attr:`input` is split into ``n`` sections along dimension :attr:`dim`. + If :attr:`input` is divisible by ``n`` along dimension :attr:`dim`, each + section will be of equal size, :code:`input.size(dim) / n`. If :attr:`input` + is not divisible by ``n``, the sizes of the first :code:`int(input.size(dim) % n)` + sections will have size :code:`int(input.size(dim) / n) + 1`, and the rest will + have size :code:`int(input.size(dim) / n)`. + + If :attr:`indices_or_sections` is a list or tuple of ints, or a one-dimensional long + tensor, then :attr:`input` is split along dimension :attr:`dim` at each of the indices + in the list, tuple or tensor. For instance, :code:`indices_or_sections=[2, 3]` and :code:`dim=0` + would result in the tensors :code:`input[:2]`, :code:`input[2:3]`, and :code:`input[3:]`. + + If :attr:`indices_or_sections` is a tensor, it must be a zero-dimensional or one-dimensional + long tensor on the CPU. + + dim (int, optional): dimension along which to split the tensor. Default: ``0`` + + Example:: + + >>> x = torch.arange(8) + >>> torch.tensor_split(x, 3) + (tensor([0, 1, 2]), tensor([3, 4, 5]), tensor([6, 7])) + + >>> x = torch.arange(7) + >>> torch.tensor_split(x, 3) + (tensor([0, 1, 2]), tensor([3, 4]), tensor([5, 6])) + >>> torch.tensor_split(x, (1, 6)) + (tensor([0]), tensor([1, 2, 3, 4, 5]), tensor([6])) + + >>> x = torch.arange(14).reshape(2, 7) + >>> x + tensor([[ 0, 1, 2, 3, 4, 5, 6], + [ 7, 8, 9, 10, 11, 12, 13]]) + >>> torch.tensor_split(x, 3, dim=1) + (tensor([[0, 1, 2], + [7, 8, 9]]), + tensor([[ 3, 4], + [10, 11]]), + tensor([[ 5, 6], + [12, 13]])) + >>> torch.tensor_split(x, (1, 6), dim=1) + (tensor([[0], + [7]]), + tensor([[ 1, 2, 3, 4, 5], + [ 8, 9, 10, 11, 12]]), + tensor([[ 6], + [13]])) + """ + +def threshold( + input: Tensor, + threshold: Number | _complex, + value: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: ... +def threshold_( + input: Tensor, + threshold: Number | _complex, + value: Number | _complex, +) -> Tensor: ... +def tile(input: Tensor, dims: Sequence[_int | SymInt]) -> Tensor: + r""" + tile(input, dims) -> Tensor + + Constructs a tensor by repeating the elements of :attr:`input`. + The :attr:`dims` argument specifies the number of repetitions + in each dimension. + + If :attr:`dims` specifies fewer dimensions than :attr:`input` has, then + ones are prepended to :attr:`dims` until all dimensions are specified. + For example, if :attr:`input` has shape (8, 6, 4, 2) and :attr:`dims` + is (2, 2), then :attr:`dims` is treated as (1, 1, 2, 2). + + Analogously, if :attr:`input` has fewer dimensions than :attr:`dims` + specifies, then :attr:`input` is treated as if it were unsqueezed at + dimension zero until it has as many dimensions as :attr:`dims` specifies. + For example, if :attr:`input` has shape (4, 2) and :attr:`dims` + is (3, 3, 2, 2), then :attr:`input` is treated as if it had the + shape (1, 1, 4, 2). + + .. note:: + + This function is similar to NumPy's tile function. + + Args: + input (Tensor): the tensor whose elements to repeat. + dims (tuple): the number of repetitions per dimension. + + Example:: + + >>> x = torch.tensor([1, 2, 3]) + >>> x.tile((2,)) + tensor([1, 2, 3, 1, 2, 3]) + >>> y = torch.tensor([[1, 2], [3, 4]]) + >>> torch.tile(y, (2, 2)) + tensor([[1, 2, 1, 2], + [3, 4, 3, 4], + [1, 2, 1, 2], + [3, 4, 3, 4]]) + """ + +def topk( + input: Tensor, + k: _int | SymInt, + dim: _int = -1, + largest: _bool = True, + sorted: _bool = True, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.topk: + r""" + topk(input, k, dim=None, largest=True, sorted=True, *, out=None) -> (Tensor, LongTensor) + + Returns the :attr:`k` largest elements of the given :attr:`input` tensor along + a given dimension. + + If :attr:`dim` is not given, the last dimension of the `input` is chosen. + + If :attr:`largest` is ``False`` then the `k` smallest elements are returned. + + A namedtuple of `(values, indices)` is returned with the `values` and + `indices` of the largest `k` elements of each row of the `input` tensor in the + given dimension `dim`. + + The boolean option :attr:`sorted` if ``True``, will make sure that the returned + `k` elements are themselves sorted + + .. note:: + When using `torch.topk`, the indices of tied elements are not guaranteed to be stable + and may vary across different invocations. + + Args: + input (Tensor): the input tensor. + k (int): the k in "top-k" + dim (int, optional): the dimension to sort along + largest (bool, optional): controls whether to return largest or + smallest elements + sorted (bool, optional): controls whether to return the elements + in sorted order + + Keyword args: + out (tuple, optional): the output tuple of (Tensor, LongTensor) that can be + optionally given to be used as output buffers + + Example:: + + >>> x = torch.arange(1., 6.) + >>> x + tensor([ 1., 2., 3., 4., 5.]) + >>> torch.topk(x, 3) + torch.return_types.topk(values=tensor([5., 4., 3.]), indices=tensor([4, 3, 2])) + """ + +def trace(input: Tensor) -> Tensor: + r""" + trace(input) -> Tensor + + Returns the sum of the elements of the diagonal of the input 2-D matrix. + + Example:: + + >>> x = torch.arange(1., 10.).view(3, 3) + >>> x + tensor([[ 1., 2., 3.], + [ 4., 5., 6.], + [ 7., 8., 9.]]) + >>> torch.trace(x) + tensor(15.) + """ + +@overload +def transpose(input: Tensor, dim0: _int, dim1: _int) -> Tensor: + r""" + transpose(input, dim0, dim1) -> Tensor + + Returns a tensor that is a transposed version of :attr:`input`. + The given dimensions :attr:`dim0` and :attr:`dim1` are swapped. + + If :attr:`input` is a strided tensor then the resulting :attr:`out` + tensor shares its underlying storage with the :attr:`input` tensor, so + changing the content of one would change the content of the other. + + If :attr:`input` is a :ref:`sparse tensor ` then the + resulting :attr:`out` tensor *does not* share the underlying storage + with the :attr:`input` tensor. + + If :attr:`input` is a :ref:`sparse tensor ` with compressed + layout (SparseCSR, SparseBSR, SparseCSC or SparseBSC) the arguments + :attr:`dim0` and :attr:`dim1` must be both batch dimensions, or must + both be sparse dimensions. The batch dimensions of a sparse tensor are the + dimensions preceding the sparse dimensions. + + .. note:: + Transpositions which interchange the sparse dimensions of a `SparseCSR` + or `SparseCSC` layout tensor will result in the layout changing between + the two options. Transposition of the sparse dimensions of a ` SparseBSR` + or `SparseBSC` layout tensor will likewise generate a result with the + opposite layout. + + + Args: + input (Tensor): the input tensor. + dim0 (int): the first dimension to be transposed + dim1 (int): the second dimension to be transposed + + Example:: + + >>> x = torch.randn(2, 3) + >>> x + tensor([[ 1.0028, -0.9893, 0.5809], + [-0.1669, 0.7299, 0.4942]]) + >>> torch.transpose(x, 0, 1) + tensor([[ 1.0028, -0.1669], + [-0.9893, 0.7299], + [ 0.5809, 0.4942]]) + + See also :func:`torch.t`. + """ + +@overload +def transpose( + input: Tensor, + dim0: str | EllipsisType | None, + dim1: str | EllipsisType | None, +) -> Tensor: + r""" + transpose(input, dim0, dim1) -> Tensor + + Returns a tensor that is a transposed version of :attr:`input`. + The given dimensions :attr:`dim0` and :attr:`dim1` are swapped. + + If :attr:`input` is a strided tensor then the resulting :attr:`out` + tensor shares its underlying storage with the :attr:`input` tensor, so + changing the content of one would change the content of the other. + + If :attr:`input` is a :ref:`sparse tensor ` then the + resulting :attr:`out` tensor *does not* share the underlying storage + with the :attr:`input` tensor. + + If :attr:`input` is a :ref:`sparse tensor ` with compressed + layout (SparseCSR, SparseBSR, SparseCSC or SparseBSC) the arguments + :attr:`dim0` and :attr:`dim1` must be both batch dimensions, or must + both be sparse dimensions. The batch dimensions of a sparse tensor are the + dimensions preceding the sparse dimensions. + + .. note:: + Transpositions which interchange the sparse dimensions of a `SparseCSR` + or `SparseCSC` layout tensor will result in the layout changing between + the two options. Transposition of the sparse dimensions of a ` SparseBSR` + or `SparseBSC` layout tensor will likewise generate a result with the + opposite layout. + + + Args: + input (Tensor): the input tensor. + dim0 (int): the first dimension to be transposed + dim1 (int): the second dimension to be transposed + + Example:: + + >>> x = torch.randn(2, 3) + >>> x + tensor([[ 1.0028, -0.9893, 0.5809], + [-0.1669, 0.7299, 0.4942]]) + >>> torch.transpose(x, 0, 1) + tensor([[ 1.0028, -0.1669], + [-0.9893, 0.7299], + [ 0.5809, 0.4942]]) + + See also :func:`torch.t`. + """ + +def transpose_copy( + input: Tensor, + dim0: _int, + dim1: _int, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.transpose`, but all output tensors + are freshly created instead of aliasing the input. + """ + +@overload +def trapezoid(y: Tensor, x: Tensor, *, dim: _int = -1) -> Tensor: + r""" + trapezoid(y, x=None, *, dx=None, dim=-1) -> Tensor + + Computes the `trapezoidal rule `_ along + :attr:`dim`. By default the spacing between elements is assumed to be 1, but + :attr:`dx` can be used to specify a different constant spacing, and :attr:`x` can be + used to specify arbitrary spacing along :attr:`dim`. Only one of :attr:`x` or :attr:`dx` should be specified. + + + Assuming :attr:`y` is a one-dimensional tensor with elements :math:`{y_0, y_1, ..., y_n}`, + the default computation is + + .. math:: + \begin{aligned} + \sum_{i = 1}^{n} \frac{1}{2} (y_i + y_{i-1}) + \end{aligned} + + When :attr:`dx` is specified the computation becomes + + .. math:: + \begin{aligned} + \sum_{i = 1}^{n} \frac{\Delta x}{2} (y_i + y_{i-1}) + \end{aligned} + + effectively multiplying the result by :attr:`dx`. When :attr:`x` is specified, + assuming :attr:`x` is also a one-dimensional tensor with + elements :math:`{x_0, x_1, ..., x_n}`, the computation becomes + + .. math:: + \begin{aligned} + \sum_{i = 1}^{n} \frac{(x_i - x_{i-1})}{2} (y_i + y_{i-1}) + \end{aligned} + + When :attr:`x` and :attr:`y` have the same size, the computation is as described above and no broadcasting is needed. + The broadcasting behavior of this function is as follows when their sizes are different. For both :attr:`x` + and :attr:`y`, the function computes the difference between consecutive elements along + dimension :attr:`dim`. This effectively creates two tensors, `x_diff` and `y_diff`, that have + the same shape as the original tensors except their lengths along the dimension :attr:`dim` is reduced by 1. + After that, those two tensors are broadcast together to compute final output as part of the trapezoidal rule. + See the examples below for details. + + .. note:: + The trapezoidal rule is a technique for approximating the definite integral of a function + by averaging its left and right Riemann sums. The approximation becomes more accurate as + the resolution of the partition increases. + + Arguments: + y (Tensor): Values to use when computing the trapezoidal rule. + x (Tensor): If specified, defines spacing between values as specified above. + + Keyword arguments: + dx (float): constant spacing between values. If neither :attr:`x` or :attr:`dx` + are specified then this defaults to 1. Effectively multiplies the result by its value. + dim (int): The dimension along which to compute the trapezoidal rule. + The last (inner-most) dimension by default. + + Examples:: + + >>> # Computes the trapezoidal rule in 1D, spacing is implicitly 1 + >>> y = torch.tensor([1, 5, 10]) + >>> torch.trapezoid(y) + tensor(10.5) + + >>> # Computes the same trapezoidal rule directly to verify + >>> (1 + 10 + 10) / 2 + 10.5 + + >>> # Computes the trapezoidal rule in 1D with constant spacing of 2 + >>> # NOTE: the result is the same as before, but multiplied by 2 + >>> torch.trapezoid(y, dx=2) + 21.0 + + >>> # Computes the trapezoidal rule in 1D with arbitrary spacing + >>> x = torch.tensor([1, 3, 6]) + >>> torch.trapezoid(y, x) + 28.5 + + >>> # Computes the same trapezoidal rule directly to verify + >>> ((3 - 1) * (1 + 5) + (6 - 3) * (5 + 10)) / 2 + 28.5 + + >>> # Computes the trapezoidal rule for each row of a 3x3 matrix + >>> y = torch.arange(9).reshape(3, 3) + tensor([[0, 1, 2], + [3, 4, 5], + [6, 7, 8]]) + >>> torch.trapezoid(y) + tensor([ 2., 8., 14.]) + + >>> # Computes the trapezoidal rule for each column of the matrix + >>> torch.trapezoid(y, dim=0) + tensor([ 6., 8., 10.]) + + >>> # Computes the trapezoidal rule for each row of a 3x3 ones matrix + >>> # with the same arbitrary spacing + >>> y = torch.ones(3, 3) + >>> x = torch.tensor([1, 3, 6]) + >>> torch.trapezoid(y, x) + array([5., 5., 5.]) + + >>> # Computes the trapezoidal rule for each row of a 3x3 ones matrix + >>> # with different arbitrary spacing per row + >>> y = torch.ones(3, 3) + >>> x = torch.tensor([[1, 2, 3], [1, 3, 5], [1, 4, 7]]) + >>> torch.trapezoid(y, x) + array([2., 4., 6.]) + """ + +@overload +def trapezoid( + y: Tensor, + *, + dx: Number | _complex = 1, + dim: _int = -1, +) -> Tensor: + r""" + trapezoid(y, x=None, *, dx=None, dim=-1) -> Tensor + + Computes the `trapezoidal rule `_ along + :attr:`dim`. By default the spacing between elements is assumed to be 1, but + :attr:`dx` can be used to specify a different constant spacing, and :attr:`x` can be + used to specify arbitrary spacing along :attr:`dim`. Only one of :attr:`x` or :attr:`dx` should be specified. + + + Assuming :attr:`y` is a one-dimensional tensor with elements :math:`{y_0, y_1, ..., y_n}`, + the default computation is + + .. math:: + \begin{aligned} + \sum_{i = 1}^{n} \frac{1}{2} (y_i + y_{i-1}) + \end{aligned} + + When :attr:`dx` is specified the computation becomes + + .. math:: + \begin{aligned} + \sum_{i = 1}^{n} \frac{\Delta x}{2} (y_i + y_{i-1}) + \end{aligned} + + effectively multiplying the result by :attr:`dx`. When :attr:`x` is specified, + assuming :attr:`x` is also a one-dimensional tensor with + elements :math:`{x_0, x_1, ..., x_n}`, the computation becomes + + .. math:: + \begin{aligned} + \sum_{i = 1}^{n} \frac{(x_i - x_{i-1})}{2} (y_i + y_{i-1}) + \end{aligned} + + When :attr:`x` and :attr:`y` have the same size, the computation is as described above and no broadcasting is needed. + The broadcasting behavior of this function is as follows when their sizes are different. For both :attr:`x` + and :attr:`y`, the function computes the difference between consecutive elements along + dimension :attr:`dim`. This effectively creates two tensors, `x_diff` and `y_diff`, that have + the same shape as the original tensors except their lengths along the dimension :attr:`dim` is reduced by 1. + After that, those two tensors are broadcast together to compute final output as part of the trapezoidal rule. + See the examples below for details. + + .. note:: + The trapezoidal rule is a technique for approximating the definite integral of a function + by averaging its left and right Riemann sums. The approximation becomes more accurate as + the resolution of the partition increases. + + Arguments: + y (Tensor): Values to use when computing the trapezoidal rule. + x (Tensor): If specified, defines spacing between values as specified above. + + Keyword arguments: + dx (float): constant spacing between values. If neither :attr:`x` or :attr:`dx` + are specified then this defaults to 1. Effectively multiplies the result by its value. + dim (int): The dimension along which to compute the trapezoidal rule. + The last (inner-most) dimension by default. + + Examples:: + + >>> # Computes the trapezoidal rule in 1D, spacing is implicitly 1 + >>> y = torch.tensor([1, 5, 10]) + >>> torch.trapezoid(y) + tensor(10.5) + + >>> # Computes the same trapezoidal rule directly to verify + >>> (1 + 10 + 10) / 2 + 10.5 + + >>> # Computes the trapezoidal rule in 1D with constant spacing of 2 + >>> # NOTE: the result is the same as before, but multiplied by 2 + >>> torch.trapezoid(y, dx=2) + 21.0 + + >>> # Computes the trapezoidal rule in 1D with arbitrary spacing + >>> x = torch.tensor([1, 3, 6]) + >>> torch.trapezoid(y, x) + 28.5 + + >>> # Computes the same trapezoidal rule directly to verify + >>> ((3 - 1) * (1 + 5) + (6 - 3) * (5 + 10)) / 2 + 28.5 + + >>> # Computes the trapezoidal rule for each row of a 3x3 matrix + >>> y = torch.arange(9).reshape(3, 3) + tensor([[0, 1, 2], + [3, 4, 5], + [6, 7, 8]]) + >>> torch.trapezoid(y) + tensor([ 2., 8., 14.]) + + >>> # Computes the trapezoidal rule for each column of the matrix + >>> torch.trapezoid(y, dim=0) + tensor([ 6., 8., 10.]) + + >>> # Computes the trapezoidal rule for each row of a 3x3 ones matrix + >>> # with the same arbitrary spacing + >>> y = torch.ones(3, 3) + >>> x = torch.tensor([1, 3, 6]) + >>> torch.trapezoid(y, x) + array([5., 5., 5.]) + + >>> # Computes the trapezoidal rule for each row of a 3x3 ones matrix + >>> # with different arbitrary spacing per row + >>> y = torch.ones(3, 3) + >>> x = torch.tensor([[1, 2, 3], [1, 3, 5], [1, 4, 7]]) + >>> torch.trapezoid(y, x) + array([2., 4., 6.]) + """ + +@overload +def trapz(y: Tensor, *, dx: _float = 1, dim: _int = -1) -> Tensor: + r""" + trapz(y, x, *, dim=-1) -> Tensor + + Alias for :func:`torch.trapezoid`. + """ + +@overload +def trapz(y: Tensor, x: Tensor, *, dim: _int = -1) -> Tensor: + r""" + trapz(y, x, *, dim=-1) -> Tensor + + Alias for :func:`torch.trapezoid`. + """ + +def triangular_solve( + input: Tensor, + A: Tensor, + upper: _bool = True, + transpose: _bool = False, + unitriangular: _bool = False, + *, + out: Tensor | tuple[Tensor, ...] | list[Tensor] | None = None, +) -> torch.return_types.triangular_solve: + r""" + triangular_solve(b, A, upper=True, transpose=False, unitriangular=False, *, out=None) -> (Tensor, Tensor) + + Solves a system of equations with a square upper or lower triangular invertible matrix :math:`A` + and multiple right-hand sides :math:`b`. + + In symbols, it solves :math:`AX = b` and assumes :math:`A` is square upper-triangular + (or lower-triangular if :attr:`upper`\ `= False`) and does not have zeros on the diagonal. + + `torch.triangular_solve(b, A)` can take in 2D inputs `b, A` or inputs that are + batches of 2D matrices. If the inputs are batches, then returns + batched outputs `X` + + If the diagonal of :attr:`A` contains zeros or elements that are very close to zero and + :attr:`unitriangular`\ `= False` (default) or if the input matrix is badly conditioned, + the result may contain `NaN` s. + + Supports input of float, double, cfloat and cdouble data types. + + .. warning:: + + :func:`torch.triangular_solve` is deprecated in favor of :func:`torch.linalg.solve_triangular` + and will be removed in a future PyTorch release. + :func:`torch.linalg.solve_triangular` has its arguments reversed and does not return a + copy of one of the inputs. + + ``X = torch.triangular_solve(B, A).solution`` should be replaced with + + .. code:: python + + X = torch.linalg.solve_triangular(A, B) + + Args: + b (Tensor): multiple right-hand sides of size :math:`(*, m, k)` where + :math:`*` is zero of more batch dimensions + A (Tensor): the input triangular coefficient matrix of size :math:`(*, m, m)` + where :math:`*` is zero or more batch dimensions + upper (bool, optional): whether :math:`A` is upper or lower triangular. Default: ``True``. + transpose (bool, optional): solves `op(A)X = b` where `op(A) = A^T` if this flag is ``True``, + and `op(A) = A` if it is ``False``. Default: ``False``. + unitriangular (bool, optional): whether :math:`A` is unit triangular. + If True, the diagonal elements of :math:`A` are assumed to be + 1 and not referenced from :math:`A`. Default: ``False``. + + Keyword args: + out ((Tensor, Tensor), optional): tuple of two tensors to write + the output to. Ignored if `None`. Default: `None`. + + Returns: + A namedtuple `(solution, cloned_coefficient)` where `cloned_coefficient` + is a clone of :math:`A` and `solution` is the solution :math:`X` to :math:`AX = b` + (or whatever variant of the system of equations, depending on the keyword arguments.) + + Examples:: + + >>> A = torch.randn(2, 2).triu() + >>> A + tensor([[ 1.1527, -1.0753], + [ 0.0000, 0.7986]]) + >>> b = torch.randn(2, 3) + >>> b + tensor([[-0.0210, 2.3513, -1.5492], + [ 1.5429, 0.7403, -1.0243]]) + >>> torch.triangular_solve(b, A) + torch.return_types.triangular_solve( + solution=tensor([[ 1.7841, 2.9046, -2.5405], + [ 1.9320, 0.9270, -1.2826]]), + cloned_coefficient=tensor([[ 1.1527, -1.0753], + [ 0.0000, 0.7986]])) + """ + +def tril( + input: Tensor, + diagonal: _int | SymInt = 0, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + tril(input, diagonal=0, *, out=None) -> Tensor + + Returns the lower triangular part of the matrix (2-D tensor) or batch of matrices + :attr:`input`, the other elements of the result tensor :attr:`out` are set to 0. + + The lower triangular part of the matrix is defined as the elements on and + below the diagonal. + + The argument :attr:`diagonal` controls which diagonal to consider. If + :attr:`diagonal` = 0, all elements on and below the main diagonal are + retained. A positive value includes just as many diagonals above the main + diagonal, and similarly a negative value excludes just as many diagonals below + the main diagonal. The main diagonal are the set of indices + :math:`\lbrace (i, i) \rbrace` for :math:`i \in [0, \min\{d_{1}, d_{2}\} - 1]` where + :math:`d_{1}, d_{2}` are the dimensions of the matrix. + + Args: + input (Tensor): the input tensor. + diagonal (int, optional): the diagonal to consider + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(3, 3) + >>> a + tensor([[-1.0813, -0.8619, 0.7105], + [ 0.0935, 0.1380, 2.2112], + [-0.3409, -0.9828, 0.0289]]) + >>> torch.tril(a) + tensor([[-1.0813, 0.0000, 0.0000], + [ 0.0935, 0.1380, 0.0000], + [-0.3409, -0.9828, 0.0289]]) + + >>> b = torch.randn(4, 6) + >>> b + tensor([[ 1.2219, 0.5653, -0.2521, -0.2345, 1.2544, 0.3461], + [ 0.4785, -0.4477, 0.6049, 0.6368, 0.8775, 0.7145], + [ 1.1502, 3.2716, -1.1243, -0.5413, 0.3615, 0.6864], + [-0.0614, -0.7344, -1.3164, -0.7648, -1.4024, 0.0978]]) + >>> torch.tril(b, diagonal=1) + tensor([[ 1.2219, 0.5653, 0.0000, 0.0000, 0.0000, 0.0000], + [ 0.4785, -0.4477, 0.6049, 0.0000, 0.0000, 0.0000], + [ 1.1502, 3.2716, -1.1243, -0.5413, 0.0000, 0.0000], + [-0.0614, -0.7344, -1.3164, -0.7648, -1.4024, 0.0000]]) + >>> torch.tril(b, diagonal=-1) + tensor([[ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [ 0.4785, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000], + [ 1.1502, 3.2716, 0.0000, 0.0000, 0.0000, 0.0000], + [-0.0614, -0.7344, -1.3164, 0.0000, 0.0000, 0.0000]]) + """ + +def tril_indices( + row: _int, + col: _int, + offset: _int = 0, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + tril_indices(row, col, offset=0, *, dtype=torch.long, device='cpu', layout=torch.strided) -> Tensor + + Returns the indices of the lower triangular part of a :attr:`row`-by- + :attr:`col` matrix in a 2-by-N Tensor, where the first row contains row + coordinates of all indices and the second row contains column coordinates. + Indices are ordered based on rows and then columns. + + The lower triangular part of the matrix is defined as the elements on and + below the diagonal. + + The argument :attr:`offset` controls which diagonal to consider. If + :attr:`offset` = 0, all elements on and below the main diagonal are + retained. A positive value includes just as many diagonals above the main + diagonal, and similarly a negative value excludes just as many diagonals below + the main diagonal. The main diagonal are the set of indices + :math:`\lbrace (i, i) \rbrace` for :math:`i \in [0, \min\{d_{1}, d_{2}\} - 1]` + where :math:`d_{1}, d_{2}` are the dimensions of the matrix. + + .. note:: + When running on CUDA, ``row * col`` must be less than :math:`2^{59}` to + prevent overflow during calculation. + + Args: + row (``int``): number of rows in the 2-D matrix. + col (``int``): number of columns in the 2-D matrix. + offset (``int``): diagonal offset from the main diagonal. + Default: if not provided, 0. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor, + only support ``torch.int``, ``torch.long``. Default: if ``None``, ``torch.long``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + layout (:class:`torch.layout`, optional): currently only support ``torch.strided``. + + Example:: + + >>> a = torch.tril_indices(3, 3) + >>> a + tensor([[0, 1, 1, 2, 2, 2], + [0, 0, 1, 0, 1, 2]]) + + >>> a = torch.tril_indices(4, 3, -1) + >>> a + tensor([[1, 2, 2, 3, 3, 3], + [0, 0, 1, 0, 1, 2]]) + + >>> a = torch.tril_indices(4, 3, 1) + >>> a + tensor([[0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3], + [0, 1, 0, 1, 2, 0, 1, 2, 0, 1, 2]]) + """ + +def triplet_margin_loss( + anchor: Tensor, + positive: Tensor, + negative: Tensor, + margin: _float = 1.0, + p: _float = 2, + eps: _float = 1e-06, + swap: _bool = False, + reduction: _int = 1, +) -> Tensor: ... +def triu( + input: Tensor, + diagonal: _int | SymInt = 0, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + triu(input, diagonal=0, *, out=None) -> Tensor + + Returns the upper triangular part of a matrix (2-D tensor) or batch of matrices + :attr:`input`, the other elements of the result tensor :attr:`out` are set to 0. + + The upper triangular part of the matrix is defined as the elements on and + above the diagonal. + + The argument :attr:`diagonal` controls which diagonal to consider. If + :attr:`diagonal` = 0, all elements on and above the main diagonal are + retained. A positive value excludes just as many diagonals above the main + diagonal, and similarly a negative value includes just as many diagonals below + the main diagonal. The main diagonal are the set of indices + :math:`\lbrace (i, i) \rbrace` for :math:`i \in [0, \min\{d_{1}, d_{2}\} - 1]` where + :math:`d_{1}, d_{2}` are the dimensions of the matrix. + + Args: + input (Tensor): the input tensor. + diagonal (int, optional): the diagonal to consider + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(3, 3) + >>> a + tensor([[ 0.2309, 0.5207, 2.0049], + [ 0.2072, -1.0680, 0.6602], + [ 0.3480, -0.5211, -0.4573]]) + >>> torch.triu(a) + tensor([[ 0.2309, 0.5207, 2.0049], + [ 0.0000, -1.0680, 0.6602], + [ 0.0000, 0.0000, -0.4573]]) + >>> torch.triu(a, diagonal=1) + tensor([[ 0.0000, 0.5207, 2.0049], + [ 0.0000, 0.0000, 0.6602], + [ 0.0000, 0.0000, 0.0000]]) + >>> torch.triu(a, diagonal=-1) + tensor([[ 0.2309, 0.5207, 2.0049], + [ 0.2072, -1.0680, 0.6602], + [ 0.0000, -0.5211, -0.4573]]) + + >>> b = torch.randn(4, 6) + >>> b + tensor([[ 0.5876, -0.0794, -1.8373, 0.6654, 0.2604, 1.5235], + [-0.2447, 0.9556, -1.2919, 1.3378, -0.1768, -1.0857], + [ 0.4333, 0.3146, 0.6576, -1.0432, 0.9348, -0.4410], + [-0.9888, 1.0679, -1.3337, -1.6556, 0.4798, 0.2830]]) + >>> torch.triu(b, diagonal=1) + tensor([[ 0.0000, -0.0794, -1.8373, 0.6654, 0.2604, 1.5235], + [ 0.0000, 0.0000, -1.2919, 1.3378, -0.1768, -1.0857], + [ 0.0000, 0.0000, 0.0000, -1.0432, 0.9348, -0.4410], + [ 0.0000, 0.0000, 0.0000, 0.0000, 0.4798, 0.2830]]) + >>> torch.triu(b, diagonal=-1) + tensor([[ 0.5876, -0.0794, -1.8373, 0.6654, 0.2604, 1.5235], + [-0.2447, 0.9556, -1.2919, 1.3378, -0.1768, -1.0857], + [ 0.0000, 0.3146, 0.6576, -1.0432, 0.9348, -0.4410], + [ 0.0000, 0.0000, -1.3337, -1.6556, 0.4798, 0.2830]]) + """ + +def triu_indices( + row: _int, + col: _int, + offset: _int = 0, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + triu_indices(row, col, offset=0, *, dtype=torch.long, device='cpu', layout=torch.strided) -> Tensor + + Returns the indices of the upper triangular part of a :attr:`row` by + :attr:`col` matrix in a 2-by-N Tensor, where the first row contains row + coordinates of all indices and the second row contains column coordinates. + Indices are ordered based on rows and then columns. + + The upper triangular part of the matrix is defined as the elements on and + above the diagonal. + + The argument :attr:`offset` controls which diagonal to consider. If + :attr:`offset` = 0, all elements on and above the main diagonal are + retained. A positive value excludes just as many diagonals above the main + diagonal, and similarly a negative value includes just as many diagonals below + the main diagonal. The main diagonal are the set of indices + :math:`\lbrace (i, i) \rbrace` for :math:`i \in [0, \min\{d_{1}, d_{2}\} - 1]` + where :math:`d_{1}, d_{2}` are the dimensions of the matrix. + + .. note:: + When running on CUDA, ``row * col`` must be less than :math:`2^{59}` to + prevent overflow during calculation. + + Args: + row (``int``): number of rows in the 2-D matrix. + col (``int``): number of columns in the 2-D matrix. + offset (``int``): diagonal offset from the main diagonal. + Default: if not provided, 0. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor, + only support ``torch.int``, ``torch.long``. Default: if ``None``, ``torch.long``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + layout (:class:`torch.layout`, optional): currently only support ``torch.strided``. + + Example:: + + >>> a = torch.triu_indices(3, 3) + >>> a + tensor([[0, 0, 0, 1, 1, 2], + [0, 1, 2, 1, 2, 2]]) + + >>> a = torch.triu_indices(4, 3, -1) + >>> a + tensor([[0, 0, 0, 1, 1, 1, 2, 2, 3], + [0, 1, 2, 0, 1, 2, 1, 2, 2]]) + + >>> a = torch.triu_indices(4, 3, 1) + >>> a + tensor([[0, 0, 1], + [1, 2, 2]]) + """ + +def true_divide( + input: Tensor | Number, + other: Tensor | Number, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + true_divide(dividend, divisor, *, out) -> Tensor + + Alias for :func:`torch.div` with ``rounding_mode=None``. + """ + +def trunc(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + trunc(input, *, out=None) -> Tensor + + Returns a new tensor with the truncated integer values of + the elements of :attr:`input`. + + For integer inputs, follows the array-api convention of returning a + copy of the input tensor. + + Args: + input (Tensor): the input tensor. + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.randn(4) + >>> a + tensor([ 3.4742, 0.5466, -0.8008, -0.9079]) + >>> torch.trunc(a) + tensor([ 3., 0., -0., -0.]) + """ + +def trunc_(input: Tensor) -> Tensor: ... +@overload +def unbind(input: Tensor, dim: _int = 0) -> tuple[Tensor, ...]: + r""" + unbind(input, dim=0) -> seq + + Removes a tensor dimension. + + Returns a tuple of all slices along a given dimension, already without it. + + Arguments: + input (Tensor): the tensor to unbind + dim (int): dimension to remove + + Example:: + + >>> torch.unbind(torch.tensor([[1, 2, 3], + >>> [4, 5, 6], + >>> [7, 8, 9]])) + (tensor([1, 2, 3]), tensor([4, 5, 6]), tensor([7, 8, 9])) + """ + +@overload +def unbind( + input: Tensor, + dim: str | EllipsisType | None, +) -> tuple[Tensor, ...]: + r""" + unbind(input, dim=0) -> seq + + Removes a tensor dimension. + + Returns a tuple of all slices along a given dimension, already without it. + + Arguments: + input (Tensor): the tensor to unbind + dim (int): dimension to remove + + Example:: + + >>> torch.unbind(torch.tensor([[1, 2, 3], + >>> [4, 5, 6], + >>> [7, 8, 9]])) + (tensor([1, 2, 3]), tensor([4, 5, 6]), tensor([7, 8, 9])) + """ + +def unbind_copy( + input: Tensor, + dim: _int = 0, + *, + out: tuple[Tensor, ...] | list[Tensor] | None = None, +) -> None: + r""" + Performs the same operation as :func:`torch.unbind`, but all output tensors + are freshly created instead of aliasing the input. + """ + +@overload +def unflatten( + input: Tensor, + dim: str | EllipsisType | None, + sizes: Sequence[_int | SymInt], + names: Sequence[str | EllipsisType | None], +) -> Tensor: + r""" + unflatten(input, dim, sizes) -> Tensor + + Expands a dimension of the input tensor over multiple dimensions. + + .. seealso:: + + :func:`torch.flatten` the inverse of this function. It coalesces several dimensions into one. + + Args: + input (Tensor): the input tensor. + dim (int): Dimension to be unflattened, specified as an index into + ``input.shape``. + sizes (Tuple[int]): New shape of the unflattened dimension. + One of its elements can be `-1` in which case the corresponding output + dimension is inferred. Otherwise, the product of ``sizes`` *must* + equal ``input.shape[dim]``. + + Returns: + A View of input with the specified dimension unflattened. + + Examples:: + >>> torch.unflatten(torch.randn(3, 4, 1), 1, (2, 2)).shape + torch.Size([3, 2, 2, 1]) + >>> torch.unflatten(torch.randn(3, 4, 1), 1, (-1, 2)).shape + torch.Size([3, 2, 2, 1]) + >>> torch.unflatten(torch.randn(5, 12, 3), -2, (2, 2, 3, 1, 1)).shape + torch.Size([5, 2, 2, 3, 1, 1, 3]) + """ + +@overload +def unflatten( + input: Tensor, + dim: _int, + sizes: Sequence[_int | SymInt], +) -> Tensor: + r""" + unflatten(input, dim, sizes) -> Tensor + + Expands a dimension of the input tensor over multiple dimensions. + + .. seealso:: + + :func:`torch.flatten` the inverse of this function. It coalesces several dimensions into one. + + Args: + input (Tensor): the input tensor. + dim (int): Dimension to be unflattened, specified as an index into + ``input.shape``. + sizes (Tuple[int]): New shape of the unflattened dimension. + One of its elements can be `-1` in which case the corresponding output + dimension is inferred. Otherwise, the product of ``sizes`` *must* + equal ``input.shape[dim]``. + + Returns: + A View of input with the specified dimension unflattened. + + Examples:: + >>> torch.unflatten(torch.randn(3, 4, 1), 1, (2, 2)).shape + torch.Size([3, 2, 2, 1]) + >>> torch.unflatten(torch.randn(3, 4, 1), 1, (-1, 2)).shape + torch.Size([3, 2, 2, 1]) + >>> torch.unflatten(torch.randn(5, 12, 3), -2, (2, 2, 3, 1, 1)).shape + torch.Size([5, 2, 2, 3, 1, 1, 3]) + """ + +def unfold_copy( + input: Tensor, + dimension: _int, + size: _int, + step: _int, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.unfold`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def unique_dim( + input: Tensor, + dim: _int, + sorted: _bool = True, + return_inverse: _bool = False, + return_counts: _bool = False, +) -> tuple[Tensor, Tensor, Tensor]: ... +def unsafe_chunk( + input: Tensor, + chunks: _int, + dim: _int = 0, +) -> tuple[Tensor, ...]: + r""" + unsafe_chunk(input, chunks, dim=0) -> List of Tensors + + Works like :func:`torch.chunk` but without enforcing the autograd restrictions + on inplace modification of the outputs. + + .. warning:: + This function is safe to use as long as only the input, or only the outputs + are modified inplace after calling this function. It is user's + responsibility to ensure that is the case. If both the input and one or more + of the outputs are modified inplace, gradients computed by autograd will be + silently incorrect. + """ + +def unsafe_split( + input: Tensor, + split_size: _int | SymInt, + dim: _int = 0, +) -> tuple[Tensor, ...]: + r""" + unsafe_split(tensor, split_size_or_sections, dim=0) -> List of Tensors + + Works like :func:`torch.split` but without enforcing the autograd restrictions + on inplace modification of the outputs. + + .. warning:: + This function is safe to use as long as only the input, or only the outputs + are modified inplace after calling this function. It is user's + responsibility to ensure that is the case. If both the input and one or more + of the outputs are modified inplace, gradients computed by autograd will be + silently incorrect. + """ + +def unsafe_split_with_sizes( + input: Tensor, + split_sizes: Sequence[_int | SymInt], + dim: _int = 0, +) -> tuple[Tensor, ...]: ... +def unsqueeze(input: Tensor, dim: _int) -> Tensor: + r""" + unsqueeze(input, dim) -> Tensor + + Returns a new tensor with a dimension of size one inserted at the + specified position. + + The returned tensor shares the same underlying data with this tensor. + + A :attr:`dim` value within the range ``[-input.dim() - 1, input.dim() + 1)`` + can be used. Negative :attr:`dim` will correspond to :meth:`unsqueeze` + applied at :attr:`dim` = ``dim + input.dim() + 1``. + + Args: + input (Tensor): the input tensor. + dim (int): the index at which to insert the singleton dimension + + Example:: + + >>> x = torch.tensor([1, 2, 3, 4]) + >>> torch.unsqueeze(x, 0) + tensor([[ 1, 2, 3, 4]]) + >>> torch.unsqueeze(x, 1) + tensor([[ 1], + [ 2], + [ 3], + [ 4]]) + """ + +def unsqueeze_copy( + input: Tensor, + dim: _int, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.unsqueeze`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def values_copy(input: Tensor, *, out: Tensor | None = None) -> Tensor: + r""" + Performs the same operation as :func:`torch.values`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def vander( + x: Tensor, + N: _int | None = None, + increasing: _bool = False, +) -> Tensor: + r""" + vander(x, N=None, increasing=False) -> Tensor + + Generates a Vandermonde matrix. + + The columns of the output matrix are elementwise powers of the input vector :math:`x^{(N-1)}, x^{(N-2)}, ..., x^0`. + If increasing is True, the order of the columns is reversed :math:`x^0, x^1, ..., x^{(N-1)}`. Such a + matrix with a geometric progression in each row is named for Alexandre-Theophile Vandermonde. + + Arguments: + x (Tensor): 1-D input tensor. + N (int, optional): Number of columns in the output. If N is not specified, + a square array is returned :math:`(N = len(x))`. + increasing (bool, optional): Order of the powers of the columns. If True, + the powers increase from left to right, if False (the default) they are reversed. + + Returns: + Tensor: Vandermonde matrix. If increasing is False, the first column is :math:`x^{(N-1)}`, + the second :math:`x^{(N-2)}` and so forth. If increasing is True, the columns + are :math:`x^0, x^1, ..., x^{(N-1)}`. + + Example:: + + >>> x = torch.tensor([1, 2, 3, 5]) + >>> torch.vander(x) + tensor([[ 1, 1, 1, 1], + [ 8, 4, 2, 1], + [ 27, 9, 3, 1], + [125, 25, 5, 1]]) + >>> torch.vander(x, N=3) + tensor([[ 1, 1, 1], + [ 4, 2, 1], + [ 9, 3, 1], + [25, 5, 1]]) + >>> torch.vander(x, N=3, increasing=True) + tensor([[ 1, 1, 1], + [ 1, 2, 4], + [ 1, 3, 9], + [ 1, 5, 25]]) + """ + +@overload +def var( + input: Tensor, + dim: _int | _size | None, + unbiased: _bool = True, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + var(input, dim=None, *, correction=1, keepdim=False, out=None) -> Tensor + + Calculates the variance over the dimensions specified by :attr:`dim`. :attr:`dim` + can be a single dimension, list of dimensions, or ``None`` to reduce over all + dimensions. + + The variance (:math:`\sigma^2`) is calculated as + + .. math:: \sigma^2 = \frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2 + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.var(a, dim=1, keepdim=True) + tensor([[1.0631], + [0.5590], + [1.4893], + [0.8258]]) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def var( + input: Tensor, + dim: _int | _size | None = None, + *, + correction: Number | _complex | None = None, + keepdim: _bool = False, + out: Tensor | None = None, +) -> Tensor: + r""" + var(input, dim=None, *, correction=1, keepdim=False, out=None) -> Tensor + + Calculates the variance over the dimensions specified by :attr:`dim`. :attr:`dim` + can be a single dimension, list of dimensions, or ``None`` to reduce over all + dimensions. + + The variance (:math:`\sigma^2`) is calculated as + + .. math:: \sigma^2 = \frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2 + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.var(a, dim=1, keepdim=True) + tensor([[1.0631], + [0.5590], + [1.4893], + [0.8258]]) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def var(input: Tensor, unbiased: _bool = True) -> Tensor: + r""" + var(input, dim=None, *, correction=1, keepdim=False, out=None) -> Tensor + + Calculates the variance over the dimensions specified by :attr:`dim`. :attr:`dim` + can be a single dimension, list of dimensions, or ``None`` to reduce over all + dimensions. + + The variance (:math:`\sigma^2`) is calculated as + + .. math:: \sigma^2 = \frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2 + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.var(a, dim=1, keepdim=True) + tensor([[1.0631], + [0.5590], + [1.4893], + [0.8258]]) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def var( + input: Tensor, + dim: Sequence[str | EllipsisType | None], + *, + correction: Number | _complex | None = None, + keepdim: _bool = False, + out: Tensor | None = None, +) -> Tensor: + r""" + var(input, dim=None, *, correction=1, keepdim=False, out=None) -> Tensor + + Calculates the variance over the dimensions specified by :attr:`dim`. :attr:`dim` + can be a single dimension, list of dimensions, or ``None`` to reduce over all + dimensions. + + The variance (:math:`\sigma^2`) is calculated as + + .. math:: \sigma^2 = \frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2 + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.var(a, dim=1, keepdim=True) + tensor([[1.0631], + [0.5590], + [1.4893], + [0.8258]]) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def var( + input: Tensor, + dim: Sequence[str | EllipsisType | None], + unbiased: _bool = True, + keepdim: _bool = False, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + var(input, dim=None, *, correction=1, keepdim=False, out=None) -> Tensor + + Calculates the variance over the dimensions specified by :attr:`dim`. :attr:`dim` + can be a single dimension, list of dimensions, or ``None`` to reduce over all + dimensions. + + The variance (:math:`\sigma^2`) is calculated as + + .. math:: \sigma^2 = \frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2 + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.var(a, dim=1, keepdim=True) + tensor([[1.0631], + [0.5590], + [1.4893], + [0.8258]]) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def var_mean( + input: Tensor, + dim: _int | _size | None, + unbiased: _bool = True, + keepdim: _bool = False, +) -> tuple[Tensor, Tensor]: + r""" + var_mean(input, dim=None, *, correction=1, keepdim=False, out=None) -> (Tensor, Tensor) + + Calculates the variance and mean over the dimensions specified by :attr:`dim`. + :attr:`dim` can be a single dimension, list of dimensions, or ``None`` to + reduce over all dimensions. + + The variance (:math:`\sigma^2`) is calculated as + + .. math:: \sigma^2 = \frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2 + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Returns: + A tuple (var, mean) containing the variance and mean. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.var_mean(a, dim=0, keepdim=True) + (tensor([[1.5926, 1.0056, 1.2005, 0.3646]]), + tensor([[ 0.0645, 0.4485, 0.8707, -0.0665]])) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def var_mean( + input: Tensor, + dim: _int | _size | None = None, + *, + correction: Number | _complex | None = None, + keepdim: _bool = False, +) -> tuple[Tensor, Tensor]: + r""" + var_mean(input, dim=None, *, correction=1, keepdim=False, out=None) -> (Tensor, Tensor) + + Calculates the variance and mean over the dimensions specified by :attr:`dim`. + :attr:`dim` can be a single dimension, list of dimensions, or ``None`` to + reduce over all dimensions. + + The variance (:math:`\sigma^2`) is calculated as + + .. math:: \sigma^2 = \frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2 + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Returns: + A tuple (var, mean) containing the variance and mean. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.var_mean(a, dim=0, keepdim=True) + (tensor([[1.5926, 1.0056, 1.2005, 0.3646]]), + tensor([[ 0.0645, 0.4485, 0.8707, -0.0665]])) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def var_mean( + input: Tensor, + unbiased: _bool = True, +) -> tuple[Tensor, Tensor]: + r""" + var_mean(input, dim=None, *, correction=1, keepdim=False, out=None) -> (Tensor, Tensor) + + Calculates the variance and mean over the dimensions specified by :attr:`dim`. + :attr:`dim` can be a single dimension, list of dimensions, or ``None`` to + reduce over all dimensions. + + The variance (:math:`\sigma^2`) is calculated as + + .. math:: \sigma^2 = \frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2 + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Returns: + A tuple (var, mean) containing the variance and mean. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.var_mean(a, dim=0, keepdim=True) + (tensor([[1.5926, 1.0056, 1.2005, 0.3646]]), + tensor([[ 0.0645, 0.4485, 0.8707, -0.0665]])) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def var_mean( + input: Tensor, + dim: Sequence[str | EllipsisType | None], + *, + correction: Number | _complex | None = None, + keepdim: _bool = False, +) -> tuple[Tensor, Tensor]: + r""" + var_mean(input, dim=None, *, correction=1, keepdim=False, out=None) -> (Tensor, Tensor) + + Calculates the variance and mean over the dimensions specified by :attr:`dim`. + :attr:`dim` can be a single dimension, list of dimensions, or ``None`` to + reduce over all dimensions. + + The variance (:math:`\sigma^2`) is calculated as + + .. math:: \sigma^2 = \frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2 + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Returns: + A tuple (var, mean) containing the variance and mean. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.var_mean(a, dim=0, keepdim=True) + (tensor([[1.5926, 1.0056, 1.2005, 0.3646]]), + tensor([[ 0.0645, 0.4485, 0.8707, -0.0665]])) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +@overload +def var_mean( + input: Tensor, + dim: Sequence[str | EllipsisType | None], + unbiased: _bool = True, + keepdim: _bool = False, +) -> tuple[Tensor, Tensor]: + r""" + var_mean(input, dim=None, *, correction=1, keepdim=False, out=None) -> (Tensor, Tensor) + + Calculates the variance and mean over the dimensions specified by :attr:`dim`. + :attr:`dim` can be a single dimension, list of dimensions, or ``None`` to + reduce over all dimensions. + + The variance (:math:`\sigma^2`) is calculated as + + .. math:: \sigma^2 = \frac{1}{\max(0,~N - \delta N)}\sum_{i=0}^{N-1}(x_i-\bar{x})^2 + + where :math:`x` is the sample set of elements, :math:`\bar{x}` is the + sample mean, :math:`N` is the number of samples and :math:`\delta N` is + the :attr:`correction`. + + + + If :attr:`keepdim` is ``True``, the output tensor is of the same size + as :attr:`input` except in the dimension(s) :attr:`dim` where it is of size 1. + Otherwise, :attr:`dim` is squeezed (see :func:`torch.squeeze`), resulting in the + output tensor having 1 (or ``len(dim)``) fewer dimension(s). + + + Args: + input (Tensor): the input tensor. + + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + If ``None``, all dimensions are reduced. + + + Keyword args: + correction (int): difference between the sample size and sample degrees of freedom. + Defaults to `Bessel's correction`_, ``correction=1``. + + .. versionchanged:: 2.0 + Previously this argument was called ``unbiased`` and was a boolean + with ``True`` corresponding to ``correction=1`` and ``False`` being + ``correction=0``. + + keepdim (bool, optional): whether the output tensor has :attr:`dim` retained or not. Default: ``False``. + + out (Tensor, optional): the output tensor. + + Returns: + A tuple (var, mean) containing the variance and mean. + + Example: + + >>> a = torch.tensor( + ... [[ 0.2035, 1.2959, 1.8101, -0.4644], + ... [ 1.5027, -0.3270, 0.5905, 0.6538], + ... [-1.5745, 1.3330, -0.5596, -0.6548], + ... [ 0.1264, -0.5080, 1.6420, 0.1992]] + ... ) # fmt: skip + >>> torch.var_mean(a, dim=0, keepdim=True) + (tensor([[1.5926, 1.0056, 1.2005, 0.3646]]), + tensor([[ 0.0645, 0.4485, 0.8707, -0.0665]])) + + .. _Bessel's correction: https://en.wikipedia.org/wiki/Bessel%27s_correction + """ + +def vdot( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + vdot(input, other, *, out=None) -> Tensor + + Computes the dot product of two 1D vectors along a dimension. + + In symbols, this function computes + + .. math:: + + \sum_{i=1}^n \overline{x_i}y_i. + + where :math:`\overline{x_i}` denotes the conjugate for complex + vectors, and it is the identity for real vectors. + + .. note:: + + Unlike NumPy's vdot, torch.vdot intentionally only supports computing the dot product + of two 1D tensors with the same number of elements. + + .. seealso:: + + :func:`torch.linalg.vecdot` computes the dot product of two batches of vectors along a dimension. + + Args: + input (Tensor): first tensor in the dot product, must be 1D. Its conjugate is used if it's complex. + other (Tensor): second tensor in the dot product, must be 1D. + + Keyword args: + + .. note:: out (Tensor, optional): the output tensor. + + + Example:: + + >>> torch.vdot(torch.tensor([2, 3]), torch.tensor([2, 1])) + tensor(7) + >>> a = torch.tensor((1 +2j, 3 - 1j)) + >>> b = torch.tensor((2 +1j, 4 - 0j)) + >>> torch.vdot(a, b) + tensor([16.+1.j]) + >>> torch.vdot(b, a) + tensor([16.-1.j]) + """ + +def view_as_complex(input: Tensor) -> Tensor: + r""" + view_as_complex(input) -> Tensor + + Returns a view of :attr:`input` as a complex tensor. For an input complex + tensor of :attr:`size` :math:`m1, m2, \dots, mi, 2`, this function returns a + new complex tensor of :attr:`size` :math:`m1, m2, \dots, mi` where the last + dimension of the input tensor is expected to represent the real and imaginary + components of complex numbers. + + .. warning:: + :func:`view_as_complex` is only supported for tensors with + :class:`torch.dtype` ``torch.float64`` and ``torch.float32``. The input is + expected to have the last dimension of :attr:`size` 2. In addition, the + tensor must have a `stride` of 1 for its last dimension. The strides of all + other dimensions must be even numbers. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> x=torch.randn(4, 2) + >>> x + tensor([[ 1.6116, -0.5772], + [-1.4606, -0.9120], + [ 0.0786, -1.7497], + [-0.6561, -1.6623]]) + >>> torch.view_as_complex(x) + tensor([(1.6116-0.5772j), (-1.4606-0.9120j), (0.0786-1.7497j), (-0.6561-1.6623j)]) + """ + +def view_as_complex_copy( + input: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.view_as_complex`, but all output tensors + are freshly created instead of aliasing the input. + """ + +def view_as_real(input: Tensor) -> Tensor: + r""" + view_as_real(input) -> Tensor + + Returns a view of :attr:`input` as a real tensor. For an input complex tensor of + :attr:`size` :math:`m1, m2, \dots, mi`, this function returns a new + real tensor of size :math:`m1, m2, \dots, mi, 2`, where the last dimension of size 2 + represents the real and imaginary components of complex numbers. + + .. warning:: + :func:`view_as_real` is only supported for tensors with ``complex dtypes``. + + Args: + input (Tensor): the input tensor. + + Example:: + + >>> x=torch.randn(4, dtype=torch.cfloat) + >>> x + tensor([(0.4737-0.3839j), (-0.2098-0.6699j), (0.3470-0.9451j), (-0.5174-1.3136j)]) + >>> torch.view_as_real(x) + tensor([[ 0.4737, -0.3839], + [-0.2098, -0.6699], + [ 0.3470, -0.9451], + [-0.5174, -1.3136]]) + """ + +def view_as_real_copy( + input: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.view_as_real`, but all output tensors + are freshly created instead of aliasing the input. + """ + +@overload +def view_copy( + input: Tensor, + dtype: _dtype, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.view`, but all output tensors + are freshly created instead of aliasing the input. + """ + +@overload +def view_copy( + input: Tensor, + size: Sequence[_int | SymInt], + *, + out: Tensor | None = None, +) -> Tensor: + r""" + Performs the same operation as :func:`torch.view`, but all output tensors + are freshly created instead of aliasing the input. + """ + +@overload +def vsplit(input: Tensor, sections: _int) -> tuple[Tensor, ...]: + r""" + vsplit(input, indices_or_sections) -> List of Tensors + + Splits :attr:`input`, a tensor with two or more dimensions, into multiple tensors + vertically according to :attr:`indices_or_sections`. Each split is a view of + :attr:`input`. + + This is equivalent to calling torch.tensor_split(input, indices_or_sections, dim=0) + (the split dimension is 0), except that if :attr:`indices_or_sections` is an integer + it must evenly divide the split dimension or a runtime error will be thrown. + + This function is based on NumPy's :func:`numpy.vsplit`. + + Args: + input (Tensor): tensor to split. + indices_or_sections (int or list or tuple of ints): See argument in :func:`torch.tensor_split`. + + Example:: + + >>> t = torch.arange(16.0).reshape(4,4) + >>> t + tensor([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [12., 13., 14., 15.]]) + >>> torch.vsplit(t, 2) + (tensor([[0., 1., 2., 3.], + [4., 5., 6., 7.]]), + tensor([[ 8., 9., 10., 11.], + [12., 13., 14., 15.]])) + >>> torch.vsplit(t, [3, 6]) + (tensor([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.]]), + tensor([[12., 13., 14., 15.]]), + tensor([], size=(0, 4))) + """ + +@overload +def vsplit(input: Tensor, indices: _size) -> tuple[Tensor, ...]: + r""" + vsplit(input, indices_or_sections) -> List of Tensors + + Splits :attr:`input`, a tensor with two or more dimensions, into multiple tensors + vertically according to :attr:`indices_or_sections`. Each split is a view of + :attr:`input`. + + This is equivalent to calling torch.tensor_split(input, indices_or_sections, dim=0) + (the split dimension is 0), except that if :attr:`indices_or_sections` is an integer + it must evenly divide the split dimension or a runtime error will be thrown. + + This function is based on NumPy's :func:`numpy.vsplit`. + + Args: + input (Tensor): tensor to split. + indices_or_sections (int or list or tuple of ints): See argument in :func:`torch.tensor_split`. + + Example:: + + >>> t = torch.arange(16.0).reshape(4,4) + >>> t + tensor([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.], + [12., 13., 14., 15.]]) + >>> torch.vsplit(t, 2) + (tensor([[0., 1., 2., 3.], + [4., 5., 6., 7.]]), + tensor([[ 8., 9., 10., 11.], + [12., 13., 14., 15.]])) + >>> torch.vsplit(t, [3, 6]) + (tensor([[ 0., 1., 2., 3.], + [ 4., 5., 6., 7.], + [ 8., 9., 10., 11.]]), + tensor([[12., 13., 14., 15.]]), + tensor([], size=(0, 4))) + """ + +def vstack( + tensors: tuple[Tensor, ...] | list[Tensor] | None, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + vstack(tensors, *, out=None) -> Tensor + + Stack tensors in sequence vertically (row wise). + + This is equivalent to concatenation along the first axis after all 1-D tensors have been reshaped by :func:`torch.atleast_2d`. + + Args: + tensors (sequence of Tensors): sequence of tensors to concatenate + + Keyword args: + out (Tensor, optional): the output tensor. + + Example:: + + >>> a = torch.tensor([1, 2, 3]) + >>> b = torch.tensor([4, 5, 6]) + >>> torch.vstack((a,b)) + tensor([[1, 2, 3], + [4, 5, 6]]) + >>> a = torch.tensor([[1],[2],[3]]) + >>> b = torch.tensor([[4],[5],[6]]) + >>> torch.vstack((a,b)) + tensor([[1], + [2], + [3], + [4], + [5], + [6]]) + """ + +@overload +def where(condition: Tensor) -> tuple[Tensor, ...]: + r""" + where(condition, input, other, *, out=None) -> Tensor + + Return a tensor of elements selected from either :attr:`input` or :attr:`other`, depending on :attr:`condition`. + + The operation is defined as: + + .. math:: + \text{out}_i = \begin{cases} + \text{input}_i & \text{if } \text{condition}_i \\ + \text{other}_i & \text{otherwise} \\ + \end{cases} + + .. note:: + The tensors :attr:`condition`, :attr:`input`, :attr:`other` must be :ref:`broadcastable `. + + Arguments: + condition (BoolTensor): When True (nonzero), yield input, otherwise yield other + input (Tensor or Scalar): value (if :attr:`input` is a scalar) or values selected at indices + where :attr:`condition` is ``True`` + other (Tensor or Scalar): value (if :attr:`other` is a scalar) or values selected at indices + where :attr:`condition` is ``False`` + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + Tensor: A tensor of shape equal to the broadcasted shape of :attr:`condition`, :attr:`input`, :attr:`other` + + Example:: + + >>> x = torch.randn(3, 2) + >>> y = torch.ones(3, 2) + >>> x + tensor([[-0.4620, 0.3139], + [ 0.3898, -0.7197], + [ 0.0478, -0.1657]]) + >>> torch.where(x > 0, 1.0, 0.0) + tensor([[0., 1.], + [1., 0.], + [1., 0.]]) + >>> torch.where(x > 0, x, y) + tensor([[ 1.0000, 0.3139], + [ 0.3898, 1.0000], + [ 0.0478, 1.0000]]) + >>> x = torch.randn(2, 2, dtype=torch.double) + >>> x + tensor([[ 1.0779, 0.0383], + [-0.8785, -1.1089]], dtype=torch.float64) + >>> torch.where(x > 0, x, 0.) + tensor([[1.0779, 0.0383], + [0.0000, 0.0000]], dtype=torch.float64) + + .. function:: where(condition) -> tuple of LongTensor + :noindex: + + ``torch.where(condition)`` is identical to + ``torch.nonzero(condition, as_tuple=True)``. + + .. note:: + See also :func:`torch.nonzero`. + """ + +@overload +def where( + condition: Tensor, + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + where(condition, input, other, *, out=None) -> Tensor + + Return a tensor of elements selected from either :attr:`input` or :attr:`other`, depending on :attr:`condition`. + + The operation is defined as: + + .. math:: + \text{out}_i = \begin{cases} + \text{input}_i & \text{if } \text{condition}_i \\ + \text{other}_i & \text{otherwise} \\ + \end{cases} + + .. note:: + The tensors :attr:`condition`, :attr:`input`, :attr:`other` must be :ref:`broadcastable `. + + Arguments: + condition (BoolTensor): When True (nonzero), yield input, otherwise yield other + input (Tensor or Scalar): value (if :attr:`input` is a scalar) or values selected at indices + where :attr:`condition` is ``True`` + other (Tensor or Scalar): value (if :attr:`other` is a scalar) or values selected at indices + where :attr:`condition` is ``False`` + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + Tensor: A tensor of shape equal to the broadcasted shape of :attr:`condition`, :attr:`input`, :attr:`other` + + Example:: + + >>> x = torch.randn(3, 2) + >>> y = torch.ones(3, 2) + >>> x + tensor([[-0.4620, 0.3139], + [ 0.3898, -0.7197], + [ 0.0478, -0.1657]]) + >>> torch.where(x > 0, 1.0, 0.0) + tensor([[0., 1.], + [1., 0.], + [1., 0.]]) + >>> torch.where(x > 0, x, y) + tensor([[ 1.0000, 0.3139], + [ 0.3898, 1.0000], + [ 0.0478, 1.0000]]) + >>> x = torch.randn(2, 2, dtype=torch.double) + >>> x + tensor([[ 1.0779, 0.0383], + [-0.8785, -1.1089]], dtype=torch.float64) + >>> torch.where(x > 0, x, 0.) + tensor([[1.0779, 0.0383], + [0.0000, 0.0000]], dtype=torch.float64) + + .. function:: where(condition) -> tuple of LongTensor + :noindex: + + ``torch.where(condition)`` is identical to + ``torch.nonzero(condition, as_tuple=True)``. + + .. note:: + See also :func:`torch.nonzero`. + """ + +@overload +def where( + condition: Tensor, + self: Number | _complex, + other: Tensor, +) -> Tensor: + r""" + where(condition, input, other, *, out=None) -> Tensor + + Return a tensor of elements selected from either :attr:`input` or :attr:`other`, depending on :attr:`condition`. + + The operation is defined as: + + .. math:: + \text{out}_i = \begin{cases} + \text{input}_i & \text{if } \text{condition}_i \\ + \text{other}_i & \text{otherwise} \\ + \end{cases} + + .. note:: + The tensors :attr:`condition`, :attr:`input`, :attr:`other` must be :ref:`broadcastable `. + + Arguments: + condition (BoolTensor): When True (nonzero), yield input, otherwise yield other + input (Tensor or Scalar): value (if :attr:`input` is a scalar) or values selected at indices + where :attr:`condition` is ``True`` + other (Tensor or Scalar): value (if :attr:`other` is a scalar) or values selected at indices + where :attr:`condition` is ``False`` + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + Tensor: A tensor of shape equal to the broadcasted shape of :attr:`condition`, :attr:`input`, :attr:`other` + + Example:: + + >>> x = torch.randn(3, 2) + >>> y = torch.ones(3, 2) + >>> x + tensor([[-0.4620, 0.3139], + [ 0.3898, -0.7197], + [ 0.0478, -0.1657]]) + >>> torch.where(x > 0, 1.0, 0.0) + tensor([[0., 1.], + [1., 0.], + [1., 0.]]) + >>> torch.where(x > 0, x, y) + tensor([[ 1.0000, 0.3139], + [ 0.3898, 1.0000], + [ 0.0478, 1.0000]]) + >>> x = torch.randn(2, 2, dtype=torch.double) + >>> x + tensor([[ 1.0779, 0.0383], + [-0.8785, -1.1089]], dtype=torch.float64) + >>> torch.where(x > 0, x, 0.) + tensor([[1.0779, 0.0383], + [0.0000, 0.0000]], dtype=torch.float64) + + .. function:: where(condition) -> tuple of LongTensor + :noindex: + + ``torch.where(condition)`` is identical to + ``torch.nonzero(condition, as_tuple=True)``. + + .. note:: + See also :func:`torch.nonzero`. + """ + +@overload +def where( + condition: Tensor, + input: Tensor, + other: Number | _complex, +) -> Tensor: + r""" + where(condition, input, other, *, out=None) -> Tensor + + Return a tensor of elements selected from either :attr:`input` or :attr:`other`, depending on :attr:`condition`. + + The operation is defined as: + + .. math:: + \text{out}_i = \begin{cases} + \text{input}_i & \text{if } \text{condition}_i \\ + \text{other}_i & \text{otherwise} \\ + \end{cases} + + .. note:: + The tensors :attr:`condition`, :attr:`input`, :attr:`other` must be :ref:`broadcastable `. + + Arguments: + condition (BoolTensor): When True (nonzero), yield input, otherwise yield other + input (Tensor or Scalar): value (if :attr:`input` is a scalar) or values selected at indices + where :attr:`condition` is ``True`` + other (Tensor or Scalar): value (if :attr:`other` is a scalar) or values selected at indices + where :attr:`condition` is ``False`` + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + Tensor: A tensor of shape equal to the broadcasted shape of :attr:`condition`, :attr:`input`, :attr:`other` + + Example:: + + >>> x = torch.randn(3, 2) + >>> y = torch.ones(3, 2) + >>> x + tensor([[-0.4620, 0.3139], + [ 0.3898, -0.7197], + [ 0.0478, -0.1657]]) + >>> torch.where(x > 0, 1.0, 0.0) + tensor([[0., 1.], + [1., 0.], + [1., 0.]]) + >>> torch.where(x > 0, x, y) + tensor([[ 1.0000, 0.3139], + [ 0.3898, 1.0000], + [ 0.0478, 1.0000]]) + >>> x = torch.randn(2, 2, dtype=torch.double) + >>> x + tensor([[ 1.0779, 0.0383], + [-0.8785, -1.1089]], dtype=torch.float64) + >>> torch.where(x > 0, x, 0.) + tensor([[1.0779, 0.0383], + [0.0000, 0.0000]], dtype=torch.float64) + + .. function:: where(condition) -> tuple of LongTensor + :noindex: + + ``torch.where(condition)`` is identical to + ``torch.nonzero(condition, as_tuple=True)``. + + .. note:: + See also :func:`torch.nonzero`. + """ + +@overload +def where( + condition: Tensor, + self: Number | _complex, + other: Number | _complex, +) -> Tensor: + r""" + where(condition, input, other, *, out=None) -> Tensor + + Return a tensor of elements selected from either :attr:`input` or :attr:`other`, depending on :attr:`condition`. + + The operation is defined as: + + .. math:: + \text{out}_i = \begin{cases} + \text{input}_i & \text{if } \text{condition}_i \\ + \text{other}_i & \text{otherwise} \\ + \end{cases} + + .. note:: + The tensors :attr:`condition`, :attr:`input`, :attr:`other` must be :ref:`broadcastable `. + + Arguments: + condition (BoolTensor): When True (nonzero), yield input, otherwise yield other + input (Tensor or Scalar): value (if :attr:`input` is a scalar) or values selected at indices + where :attr:`condition` is ``True`` + other (Tensor or Scalar): value (if :attr:`other` is a scalar) or values selected at indices + where :attr:`condition` is ``False`` + + Keyword args: + out (Tensor, optional): the output tensor. + + Returns: + Tensor: A tensor of shape equal to the broadcasted shape of :attr:`condition`, :attr:`input`, :attr:`other` + + Example:: + + >>> x = torch.randn(3, 2) + >>> y = torch.ones(3, 2) + >>> x + tensor([[-0.4620, 0.3139], + [ 0.3898, -0.7197], + [ 0.0478, -0.1657]]) + >>> torch.where(x > 0, 1.0, 0.0) + tensor([[0., 1.], + [1., 0.], + [1., 0.]]) + >>> torch.where(x > 0, x, y) + tensor([[ 1.0000, 0.3139], + [ 0.3898, 1.0000], + [ 0.0478, 1.0000]]) + >>> x = torch.randn(2, 2, dtype=torch.double) + >>> x + tensor([[ 1.0779, 0.0383], + [-0.8785, -1.1089]], dtype=torch.float64) + >>> torch.where(x > 0, x, 0.) + tensor([[1.0779, 0.0383], + [0.0000, 0.0000]], dtype=torch.float64) + + .. function:: where(condition) -> tuple of LongTensor + :noindex: + + ``torch.where(condition)`` is identical to + ``torch.nonzero(condition, as_tuple=True)``. + + .. note:: + See also :func:`torch.nonzero`. + """ + +@overload +def xlogy( + input: Tensor, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + xlogy(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.special.xlogy`. + """ + +@overload +def xlogy( + self: Number | _complex, + other: Tensor, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + xlogy(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.special.xlogy`. + """ + +@overload +def xlogy( + input: Tensor, + other: Number | _complex, + *, + out: Tensor | None = None, +) -> Tensor: + r""" + xlogy(input, other, *, out=None) -> Tensor + + Alias for :func:`torch.special.xlogy`. + """ + +@overload +def xlogy_(input: Tensor, other: Tensor) -> Tensor: ... +@overload +def xlogy_(input: Tensor, other: Number | _complex) -> Tensor: ... +def zero_(input: Tensor) -> Tensor: ... +@overload +def zeros( + size: Sequence[_int | SymInt], + *, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + zeros(*size, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with the scalar value `0`, with the shape defined + by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.zeros(2, 3) + tensor([[ 0., 0., 0.], + [ 0., 0., 0.]]) + + >>> torch.zeros(5) + tensor([ 0., 0., 0., 0., 0.]) + """ + +@overload +def zeros( + *size: _int | SymInt, + out: Tensor | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + zeros(*size, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with the scalar value `0`, with the shape defined + by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.zeros(2, 3) + tensor([[ 0., 0., 0.], + [ 0., 0., 0.]]) + + >>> torch.zeros(5) + tensor([ 0., 0., 0., 0., 0.]) + """ + +@overload +def zeros( + size: _size, + *, + names: Sequence[str | EllipsisType | None] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + zeros(*size, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with the scalar value `0`, with the shape defined + by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.zeros(2, 3) + tensor([[ 0., 0., 0.], + [ 0., 0., 0.]]) + + >>> torch.zeros(5) + tensor([ 0., 0., 0., 0., 0.]) + """ + +@overload +def zeros( + *size: _int, + names: Sequence[str | EllipsisType | None] | None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + zeros(*size, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + + Returns a tensor filled with the scalar value `0`, with the shape defined + by the variable argument :attr:`size`. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + Can be a variable number of arguments or a collection like a list or tuple. + + Keyword args: + out (Tensor, optional): the output tensor. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + Default: if ``None``, uses a global default (see :func:`torch.set_default_dtype`). + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, uses the current device for the default tensor type + (see :func:`torch.set_default_device`). :attr:`device` will be the CPU + for CPU tensor types and the current CUDA device for CUDA tensor types. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + + Example:: + + >>> torch.zeros(2, 3) + tensor([[ 0., 0., 0.], + [ 0., 0., 0.]]) + + >>> torch.zeros(5) + tensor([ 0., 0., 0., 0., 0.]) + """ + +def zeros_like( + input: Tensor, + *, + memory_format: memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, +) -> Tensor: + r""" + zeros_like(input, *, dtype=None, layout=None, device=None, requires_grad=False, memory_format=torch.preserve_format) -> Tensor + + Returns a tensor filled with the scalar value `0`, with the same size as + :attr:`input`. ``torch.zeros_like(input)`` is equivalent to + ``torch.zeros(input.size(), dtype=input.dtype, layout=input.layout, device=input.device)``. + + .. warning:: + As of 0.4, this function does not support an :attr:`out` keyword. As an alternative, + the old ``torch.zeros_like(input, out=output)`` is equivalent to + ``torch.zeros(input.size(), out=output)``. + + Args: + input (Tensor): the size of :attr:`input` will determine size of the output tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: if ``None``, defaults to the dtype of :attr:`input`. + layout (:class:`torch.layout`, optional): the desired layout of returned tensor. + Default: if ``None``, defaults to the layout of :attr:`input`. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if ``None``, defaults to the device of :attr:`input`. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + + Example:: + + >>> input = torch.empty(2, 3) + >>> torch.zeros_like(input) + tensor([[ 0., 0., 0.], + [ 0., 0., 0.]]) + """ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/__init__.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..392d89ff7185e80f5063f64bc0a9b7cdb5a6e007 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/__init__.pyi @@ -0,0 +1,13003 @@ +# @generated by tools/pyi/gen_pyi.py from torch/_C/__init__.pyi.in +# mypy: disable-error-code="type-arg" +# mypy: allow-untyped-defs +# ruff: noqa: F401 + +from collections.abc import Callable, Iterable, Iterator, Sequence +from enum import Enum, IntEnum +from pathlib import Path +from types import EllipsisType +from typing import ( + Any, + AnyStr, + Generic, + IO, + Literal, + NamedTuple, + overload, + Protocol, + runtime_checkable, + SupportsIndex, + TypeAlias, + TypeVar, +) +from typing_extensions import ParamSpec, Self + +import numpy + +import torch +from torch import inf, SymInt, Tensor +from torch._C import ( + _acc, + _aoti, + _cpu, + _dynamo, + _export, + _functionalization, + _functorch, + _lazy, + _lazy_ts_backend, + _nn, + _onnx, + _VariableFunctions, + _verbose, +) +from torch._prims_common import DeviceLikeType +from torch.autograd.graph import Node as _Node +from torch.cuda import _POOL_HANDLE +from torch.distributed.device_mesh import DeviceMesh +from torch.distributed.tensor._op_schema import OpSchema +from torch.distributed.tensor.placement_types import Placement +from torch.fx.node import Node as FxNode +from torch.package import PackageExporter +from torch.storage import TypedStorage, UntypedStorage +from torch.types import ( + _bool, + _bytes, + _complex, + _device, + _dispatchkey, + _dtype, + _float, + _int, + _layout, + _qscheme, + _size, + _str, + _symsize, + Device, + IntLikeType, + Number, + Storage, +) +from torch.utils._python_dispatch import TorchDispatchMode +from torch.utils.checkpoint import GraphExecGroup + +# This module is defined in torch/csrc/Module.cpp + +K = TypeVar("K") # noqa: PYI001 +T = TypeVar("T") # noqa: PYI001 +S = TypeVar("S", bound=torch.Tensor) # noqa: PYI001 +P = ParamSpec("P") # noqa: PYI001 +R = TypeVar("R", covariant=True) # return value (always covariant) # noqa: PYI001 +T_co = TypeVar("T_co", covariant=True) # noqa: PYI001 + +@runtime_checkable +class _NestedSequence(Protocol[T_co]): + """A protocol for representing nested sequences. + + References:: + `numpy._typing._NestedSequence` + + """ + + def __len__(self, /) -> _int: ... + def __getitem__(self, index: _int, /) -> T_co | _NestedSequence[T_co]: ... + def __contains__(self, x: object, /) -> _bool: ... + def __iter__(self, /) -> Iterator[T_co | _NestedSequence[T_co]]: ... + def __reversed__(self, /) -> Iterator[T_co | _NestedSequence[T_co]]: ... + def count(self, value: Any, /) -> _int: ... + def index(self, value: Any, /) -> _int: ... + +# Defined in torch/csrc/Device.cpp +class device: + type: str # THPDevice_type + index: _int # THPDevice_index + + def __get__(self, instance, owner=None) -> device: ... + + # THPDevice_pynew + @overload + def __init__(self, device: DeviceLikeType) -> None: ... + @overload + def __init__(self, type: str, index: _int) -> None: ... + + # Uncomment if we ever make torch.device a decorator + # def __call__(self, func: T) -> T: ... + + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_val, exc_tb) -> None: ... + def __reduce__(self) -> tuple[Any, ...]: ... # THPDevice_reduce + +# Defined in torch/csrc/Stream.cpp +class Stream: + stream_id: _int # Stream id + device_index: _int + device_type: _int + + device: _device # The device of the stream + + @overload + def __new__( + cls, + device: DeviceLikeType | None = None, + *, + priority: _int = 0, + ) -> Self: ... + @overload + def __new__( + cls, + stream_id: _int, + device_index: _int, + device_type: _int, + *, + priority: _int = 0, + ) -> Self: ... + def query(self) -> _bool: ... + def synchronize(self) -> None: ... + def wait_event(self, event: Event) -> None: ... + def wait_stream(self, other: Stream) -> None: ... + def record_event(self, event: Event | None = None) -> Event: ... + def __hash__(self) -> _int: ... + def __eq__(self, other: object) -> _bool: ... + def __enter__(self) -> Self: ... + def __exit__(self, exc_type, exc_val, exc_tb) -> None: ... + +# Defined in torch/csrc/Event.cpp +class Event: + device: _device # The device of the Event + event_id: _int # The raw event created by device backend + + def __new__( + cls, + device: DeviceLikeType | None = None, + *, + enable_timing: _bool = False, + blocking: _bool = False, + interprocess: _bool = False, + ) -> Self: ... + @classmethod + def from_ipc_handle(cls, device: _device, ipc_handle: bytes) -> Event: ... + def record(self, stream: Stream | None = None) -> None: ... + def wait(self, stream: Stream | None = None) -> None: ... + def query(self) -> _bool: ... + def elapsed_time(self, other: Event) -> _float: ... + def synchronize(self) -> None: ... + def ipc_handle(self) -> bytes: ... + +# Defined in torch/csrc/Size.cpp +class Size(tuple[_int, ...]): + # TODO: __reduce__ + + @overload + def __getitem__(self: Size, key: SupportsIndex, /) -> _int: ... + @overload + def __getitem__(self: Size, key: slice, /) -> Size: ... + # Note: torch.Size does not support adding non-integer tuples. + def __add__(self, other: tuple[_int, ...], /) -> Size: ... # type: ignore[override] + def __radd__(self: Size, other: tuple[_int, ...], /) -> Size: ... + def __mul__(self, other: SupportsIndex, /) -> Size: ... + def __rmul__(self, other: SupportsIndex, /) -> Size: ... + def numel(self: Size, /) -> _int: ... + +# Defined in torch/csrc/Dtype.cpp +class dtype: + # TODO: __reduce__ + is_floating_point: _bool + is_complex: _bool + is_signed: _bool + itemsize: _int + def to_real(self) -> dtype: ... + def to_complex(self) -> dtype: ... + +# Defined in torch/csrc/TypeInfo.cpp +class iinfo: + bits: _int + min: _int + max: _int + dtype: str + + def __init__(self, dtype: _dtype) -> None: ... + +class finfo: + bits: _int + min: _float + max: _float + eps: _float + tiny: _float + smallest_normal: _float + resolution: _float + dtype: str + + @overload + def __init__(self, dtype: _dtype) -> None: ... + @overload + def __init__(self) -> None: ... + +float32: dtype = ... +float: dtype = ... +float64: dtype = ... +double: dtype = ... +float16: dtype = ... +bfloat16: dtype = ... +float8_e4m3fn: dtype = ... +float8_e4m3fnuz: dtype = ... +float8_e5m2: dtype = ... +float8_e5m2fnuz: dtype = ... +float8_e8m0fnu: dtype = ... +float4_e2m1fn_x2: dtype = ... +half: dtype = ... +uint8: dtype = ... +uint16: dtype = ... +uint32: dtype = ... +uint64: dtype = ... +int8: dtype = ... +int16: dtype = ... +short: dtype = ... +int32: dtype = ... +int: dtype = ... +int64: dtype = ... +long: dtype = ... +complex32: dtype = ... +complex64: dtype = ... +chalf: dtype = ... +cfloat: dtype = ... +complex128: dtype = ... +cdouble: dtype = ... +quint8: dtype = ... +qint8: dtype = ... +qint32: dtype = ... +bool: dtype = ... +quint4x2: dtype = ... +quint2x4: dtype = ... +bits1x8: dtype = ... +bits2x4: dtype = ... +bits4x2: dtype = ... +bits8: dtype = ... +bits16: dtype = ... + +# Defined in torch/csrc/Layout.cpp +class layout: ... + +# Defined in torch/csrc/utils/disable_torch_function.cpp +def DisableTorchFunction(): ... +def DisableTorchFunctionSubclass(): ... + +# Defined in torch/csrc/utils/tensor_layouts.cpp +strided: layout = ... +sparse_coo: layout = ... +sparse_csr: layout = ... +sparse_csc: layout = ... +sparse_bsr: layout = ... +sparse_bsc: layout = ... +_mkldnn: layout = ... +jagged: layout = ... + +# Defined in torch/csrc/MemoryFormat.cpp +class memory_format: ... + +# Defined in torch/csrc/utils/tensor_memoryformats.cpp +contiguous_format: memory_format = ... +channels_last: memory_format = ... +channels_last_3d: memory_format = ... +preserve_format: memory_format = ... + +# Defined in torch/csrc/QScheme.cpp +class qscheme: ... + +# Defined in torch/csrc/utils/tensor_qschemes.h +per_tensor_affine: qscheme = ... +per_channel_affine: qscheme = ... +per_tensor_symmetric: qscheme = ... +per_channel_symmetric: qscheme = ... +per_channel_affine_float_qparams: qscheme = ... + +# Defined in torch/csrc/autograd/python_function.cpp +class _FunctionBase: + saved_tensors: tuple[Tensor] + _raw_saved_tensors: tuple[Any] + next_functions: tuple[tuple[Any, _int], ...] + needs_input_grad: tuple[_bool] + metadata: dict + _materialize_non_diff_grads: _bool + # skip adding type hints for the fields that have wrappers defined + # in torch/autograd/function.py + +# Defined in torch/csrc/autograd/python_legacy_variable.cpp +class _LegacyVariableBase(Tensor): # inherits from Tensor to appease mypy + def __init__( + self, + data: Tensor | None = ..., + requires_grad: _bool | None = ..., + volatile: _bool | None = ..., + _grad_fn: _FunctionBase | None = ..., + ) -> None: ... + +# Defined in torch/csrc/jit/python/init.cpp +class IODescriptor: ... +class JITException(Exception): ... + +class Future(Generic[T]): + def __init__(self, devices: list[device]) -> None: ... + def done(self) -> _bool: ... + def value(self) -> T: ... + def wait(self) -> T: ... + def add_done_callback(self, callback: Callable) -> None: ... + def then(self, callback: Callable) -> Future[T]: ... + def set_result(self, result: T) -> None: ... + def _set_unwrap_func(self, callback: Callable) -> None: ... + +class _Await: + def __init__(self) -> None: ... + def fn(self) -> Callable: ... + def args(self) -> tuple[Any, ...]: ... + def is_nowait(self) -> _bool: ... + +def _jit_set_num_profiled_runs(num: _size) -> _size: ... + +# Defined in torch/csrc/jit/passes/mobile_optimizer_type.h +class _MobileOptimizerType: ... + +CONV_BN_FUSION: _MobileOptimizerType +INSERT_FOLD_PREPACK_OPS: _MobileOptimizerType +REMOVE_DROPOUT: _MobileOptimizerType +FUSE_ADD_RELU: _MobileOptimizerType +HOIST_CONV_PACKED_PARAMS: _MobileOptimizerType +VULKAN_AUTOMATIC_GPU_TRANSFER: _MobileOptimizerType + +def fork(*args: Any, **kwargs: Any) -> Future: ... +def wait(fut: Future) -> Any: ... +def _awaitable(*args: Any, **kwargs: Any) -> _Await: ... +def _awaitable_wait(aw: _Await) -> Any: ... +def _awaitable_nowait(x: Any) -> _Await: ... +def _collect_all(futures: list[Future]) -> Future: ... +def _set_print_stack_traces_on_fatal_signal(print: _bool) -> None: ... +def unify_type_list(types: list[JitType]) -> JitType: ... +def _freeze_module( + module: ScriptModule, + preserved_attrs: list[str] = ..., + freeze_interfaces: _bool = True, + preserveParameters: _bool = True, +) -> ScriptModule: ... +def _jit_pass_optimize_frozen_graph(Graph, optimize_numerics: _bool = True) -> None: ... +def _jit_pass_optimize_for_inference( + module: torch.jit.ScriptModule, + other_methods: list[str] = ..., +) -> None: ... +def _jit_pass_fold_frozen_conv_bn(graph: Graph): ... +def _jit_pass_fold_frozen_conv_add_or_sub(graph: Graph): ... +def _jit_pass_fold_frozen_conv_mul_or_div(graph: Graph): ... +def _jit_pass_fuse_frozen_conv_add_relu(graph: Graph): ... +def _jit_pass_concat_frozen_linear(graph: Graph): ... +def _jit_pass_convert_frozen_ops_to_mkldnn(graph: Graph): ... +def _jit_pass_transpose_frozen_linear(graph: Graph): ... +def _jit_pass_remove_dropout(module: torch.jit.ScriptModule): ... +def _is_tracing() -> _bool: ... +def _jit_init() -> _bool: ... +def _jit_flatten(arg: Any) -> tuple[list[Tensor], IODescriptor]: ... +def _jit_unflatten(vars: list[Tensor], desc: IODescriptor) -> Any: ... +def _jit_get_operation(op_name: str) -> tuple[Callable, list[str]]: ... +def _get_operation_overload( + op_name: str, + op_overload_name: str, +) -> tuple[Callable, Callable, list[Any]]: ... +def _get_schema(op_name: str, overload_name: str) -> FunctionSchema: ... +def _jit_pass_optimize_for_mobile( + module: torch.jit.ScriptModule, + optimization_blocklist: set[_MobileOptimizerType], + preserved_methods: list[AnyStr], +) -> torch.jit.ScriptModule: ... +def _clone_module_with_class( + module: torch.jit.ScriptModule, + ignored_methods: list[AnyStr], + ignored_attributes: list[AnyStr], +) -> torch.jit.ScriptModule: ... +def _jit_pass_vulkan_optimize_for_mobile( + module: torch.jit.ScriptModule, + optimization_blocklist: set[_MobileOptimizerType], + preserved_methods: list[AnyStr], +) -> torch.jit.ScriptModule: ... +def _jit_pass_metal_optimize_for_mobile( + module: torch.jit.ScriptModule, + preserved_methods: list[AnyStr], +) -> torch.jit.ScriptModule: ... +def _jit_pass_inline(Graph) -> None: ... +def _jit_pass_constant_propagation(Graph) -> None: ... +def _jit_pass_propagate_shapes_on_graph(Graph) -> None: ... +def _jit_register_decomposition_for_schema(schema: FunctionSchema, Graph) -> None: ... +def _jit_erase_non_input_shape_information(Graph) -> None: ... +def _jit_get_schemas_for_operator(name: str) -> list[FunctionSchema]: ... +def _jit_get_all_schemas() -> list[FunctionSchema]: ... +def _jit_check_alias_annotation( + g: Graph, + args: tuple[Any, ...], + unqualified_op_name: str, +): ... +def _jit_can_fuse_on_cpu() -> _bool: ... +def _jit_can_fuse_on_gpu() -> _bool: ... +def _jit_can_fuse_on_cpu_legacy() -> _bool: ... +def _debug_get_fusion_group_inlining() -> _bool: ... +def _debug_set_fusion_group_inlining(enable: _bool): ... +def _jit_texpr_fuser_enabled() -> _bool: ... +def _jit_nvfuser_enabled() -> _bool: ... +def _jit_llga_enabled() -> _bool: ... +def _jit_set_llga_enabled(enable: _bool): ... +def _llvm_enabled() -> _bool: ... +def _jit_override_can_fuse_on_cpu(override: _bool): ... +def _jit_override_can_fuse_on_gpu(override: _bool): ... +def _jit_override_can_fuse_on_cpu_legacy(override: _bool): ... +def _jit_set_symbolic_shapes_test_mode(override: _bool): ... +def _jit_symbolic_shapes_test_mode_enabled() -> _bool: ... +def _jit_set_texpr_fuser_enabled(enable: _bool): ... +def _jit_set_te_must_use_llvm_cpu(use_llvm: _bool): ... +def _jit_set_nvfuser_enabled(enable: _bool) -> _bool: ... +def _jit_cat_wo_conditionals(optimize_cat: _bool): ... +def _jit_opt_conditionals(opt_conds: _bool): ... +def _jit_pass_canonicalize(graph: Graph, keep_unique_names: _bool = True): ... +def _jit_pass_erase_shape_information(graph: Graph): ... +def _jit_pass_fold_convbn(module: torch.jit.ScriptModule): ... +def _jit_pass_insert_observers( + module: torch.jit.ScriptModule, + method_name: str, + qconfig_dict: dict[str, Any], + inplace: _bool, + quant_type: _int, +): ... +def _jit_pass_insert_quant_dequant( + module: torch.jit.ScriptModule, + method_name: str, + inplace: _bool, + debug: _bool, + quant_type: _int, +): ... +def _jit_pass_insert_quant_dequant_for_ondevice_ptq( + module: torch.jit.ScriptModule, + method_name: str, + inplace: _bool, + debug: _bool, + quant_type: _int, +): ... +def _jit_pass_quant_finalize( + module: torch.jit.ScriptModule, + quant_type: _int, + preserved_attrs: Sequence[str], +): ... +def _jit_pass_quant_finalize_for_ondevice_ptq( + module: torch.jit.ScriptModule, + quant_type: _int, + method_name: str, +): ... +def _jit_pass_insert_observer_method_for_ondevice_ptq( + module: torch.jit.ScriptModule, + method_name: str, + qconfig_dict: dict[str, Any], + inplace: _bool, + quant_type: _int, +): ... +def _jit_set_profiling_executor(profiling_flag: _bool) -> _bool: ... +def _jit_set_profiling_mode(profiling_flag: _bool) -> _bool: ... +def _jit_set_fusion_strategy( + strategy: list[tuple[str, _int]], +) -> list[tuple[str, _int]]: ... +def _jit_try_infer_type(obj: Any) -> InferredType: ... +def _jit_get_trigger_value(trigger_name: str) -> _int: ... + +# Defined in torch/csrc/jit/python/script_init.cpp +ResolutionCallback: TypeAlias = Callable[[str], Callable[..., Any]] + +# Defined in torch/csrc/jit/python/script_init.cpp +# and torch/csrc/jit/python/init.cpp +def _maybe_call_torch_function_for_op_packet( + op_overload_packet: Any, + *args: Any, + **kwargs: Any, +) -> Any: ... +def _check_schema_allow_fake_script_object( + schema: FunctionSchema, + *args: Any, + **kwargs: Any, +) -> _bool: ... +def _create_function_from_graph(qualname: str, graph: Graph) -> ScriptFunction: ... +def _debug_set_autodiff_subgraph_inlining(disabled: _bool) -> None: ... +def _ivalue_tags_match(lhs: ScriptModule, rhs: ScriptModule) -> _bool: ... +def _jit_assert_is_instance(obj: Any, type: JitType): ... +def _jit_clear_class_registry() -> None: ... +def _jit_set_emit_hooks( + ModuleHook: Callable | None, + FunctionHook: Callable | None, +) -> None: ... +def _jit_get_emit_hooks() -> tuple[Callable, Callable]: ... +def _load_for_lite_interpreter( + filename: str | Path, + map_location: DeviceLikeType | None, +): ... +def _load_for_lite_interpreter_from_buffer( + buffer: IO[bytes], + map_location: DeviceLikeType | None, +): ... +def _export_operator_list(module: LiteScriptModule): ... +def _quantize_ondevice_ptq_dynamic(module: LiteScriptModule, method_name: str): ... +def _get_model_bytecode_version(filename: str | Path) -> _int: ... +def _get_model_bytecode_version_from_buffer(buffer: IO[bytes]) -> _int: ... +def _backport_for_mobile( + filename_input: str | Path, + filename_output: str | Path, + to_version: _int, +) -> None: ... +def _backport_for_mobile_from_buffer( + buffer: IO[bytes], + filename_output: str | Path, + to_version: _int, +) -> None: ... +def _backport_for_mobile_to_buffer( + filename_input: str | Path, + to_version: _int, +) -> bytes: ... +def _backport_for_mobile_from_buffer_to_buffer( + buffer: IO[bytes], + to_version: _int, +) -> bytes: ... +def _get_model_ops_and_info(filename: str | Path): ... +def _get_model_ops_and_info_from_buffer(buffer: IO[bytes]): ... +def _get_mobile_model_contained_types(filename: str | Path): ... +def _get_mobile_model_contained_types_from_buffer(buffer: IO[bytes]): ... +def _logging_set_logger(logger: LoggerBase) -> LoggerBase: ... +def _get_graph_executor_optimize(optimize: _bool | None = None) -> _bool: ... +def _set_graph_executor_optimize(optimize: _bool): ... +def _export_opnames(module: ScriptModule) -> list[str]: ... +def _create_function_from_trace( + qualname: str, + func: Callable[..., Any], + input_tuple: tuple[Any, ...], + var_lookup_fn: Callable[[Tensor], str], + strict: _bool, + force_outplace: _bool, + argument_names: list[str], +) -> tuple[Graph, Stack]: ... +def _create_function_from_trace_with_dict( + qualname: str, + func: Callable[..., Any], + input_dict: dict[str, Any], + var_lookup_fn: Callable[[Tensor], str], + strict: _bool, + force_outplace: _bool, + argument_names: list[str], +) -> tuple[Graph, Stack]: ... +def _jit_is_script_object(obj: Any) -> _bool: ... +def _last_executed_optimized_graph() -> Graph: ... +def parse_type_comment(comment: str) -> Decl: ... +def _get_upgraders_map_size() -> _int: ... +def _get_upgraders_entry_map() -> dict[str, str]: ... +def _dump_upgraders_map() -> dict[str, str]: ... +def _test_only_populate_upgraders(content: dict[str, str]) -> None: ... +def _test_only_remove_upgraders(content: dict[str, str]) -> None: ... +def merge_type_from_type_comment( + decl: Decl, + type_annotation_decl: Decl, + is_method: _bool, +) -> Decl: ... +def parse_ir(input: str, parse_tensor_constants: _bool = False) -> Graph: ... +def parse_schema(schema: str) -> FunctionSchema: ... +def get_device(input: Tensor) -> _int: ... +def _resolve_type_from_object( + obj: Any, + range: SourceRange, + rcb: ResolutionCallback, +) -> JitType: ... +def _create_module_with_type(ty: JitType) -> ScriptModule: ... +def _create_object_with_type(ty: ClassType) -> ScriptObject: ... +def _run_emit_module_hook(m: ScriptModule): ... +def _replace_overloaded_method_decl( + overload_decl: Decl, + implementation_def: Def, + new_name: str, +) -> Def: ... +def _jit_pass_lower_all_tuples(graph: Graph) -> None: ... +def _jit_pass_onnx_set_dynamic_input_shape( + graph: Graph, + dynamic_axes: dict[str, dict[_int, str]], + input_names: list[str], +) -> None: ... +def _jit_pass_onnx_graph_shape_type_inference( + graph: Graph, + params_dict: dict[str, IValue], + opset_version: _int, +) -> None: ... +def _jit_pass_onnx_assign_output_shape( + graph: Graph, + tensors: list[Tensor], + desc: IODescriptor, + onnx_shape_inference: _bool, + is_script: _bool, + opset_version: _int, +) -> None: ... +def _jit_pass_onnx_remove_inplace_ops_for_onnx( + graph: Graph, + module: ScriptModule | None = None, +) -> None: ... +def _jit_pass_remove_inplace_ops(graph: Graph) -> None: ... +def _jit_pass_canonicalize_graph_fuser_ops(graph: Graph) -> None: ... +def _jit_pass_peephole( + graph: Graph, + disable_shape_peepholes: _bool = False, +) -> None: ... +def _jit_pass_onnx_autograd_function_process(graph: Graph) -> None: ... +def _jit_pass_fuse_addmm(graph: Graph) -> None: ... +def _jit_pass_onnx_preprocess(graph: Graph) -> None: ... +def _jit_pass_prepare_division_for_onnx(graph: Graph) -> None: ... +def _jit_pass_onnx_remove_print(graph: Graph) -> None: ... +def _jit_pass_onnx_preprocess_caffe2(graph: Graph) -> None: ... +def _jit_pass_onnx_unpack_quantized_weights( + graph: Graph, + paramsDict: dict[str, IValue], +) -> dict[str, IValue]: ... +def _jit_pass_onnx_quantization_insert_permutes( + graph: Graph, + paramsDict: dict[str, IValue], +) -> dict[str, IValue]: ... +def _jit_pass_custom_pattern_based_rewrite_graph( + pattern: str, + fused_node_name: str, + graph: Graph, +) -> None: ... +def _jit_onnx_list_model_parameters( + module: ScriptModule, +) -> tuple[ScriptModule, list[IValue]]: ... +def _jit_pass_erase_number_types(graph: Graph) -> None: ... +def _jit_pass_onnx_lint(graph: Graph) -> None: ... +def _jit_pass_onnx( + graph: Graph, + _jit_pass_onnx: _onnx.OperatorExportTypes, +) -> Graph: ... +def _jit_pass_onnx_scalar_type_analysis( + graph: Graph, + lowprecision_cast: _bool, + opset_version: _int, +) -> None: ... +def _jit_pass_onnx_peephole( + graph: Graph, + opset_version: _int, + fixed_batch_size: _bool, +) -> None: ... +def _jit_pass_dce_allow_deleting_nodes_with_side_effects(graph: Graph) -> None: ... +def _jit_pass_onnx_function_substitution(graph: Graph) -> None: ... +def _jit_pass_onnx_function_extraction( + graph: Graph, + module_names: set[str], + param_names: list[str], +) -> dict[Node, dict[str, str]]: ... +def _jit_pass_onnx_clear_scope_records() -> None: ... +def _jit_pass_onnx_track_scope_attributes( + graph: Graph, + onnx_attrs: dict[str, Any], +) -> None: ... +def _jit_is_onnx_log_enabled() -> _bool: ... +def _jit_set_onnx_log_enabled(enabled: _bool) -> None: ... +def _jit_set_onnx_log_output_stream(stream_name: str) -> None: ... +def _jit_onnx_log(*args: Any) -> None: ... +def _jit_pass_lower_graph(graph: Graph, m: Module) -> tuple[Graph, list[IValue]]: ... +def _jit_pass_inline_fork_wait(graph: Graph) -> None: ... +def _jit_pass_onnx_deduplicate_initializers( + graph: Graph, + params_dict: dict[str, IValue], + is_train: _bool, +) -> dict[str, IValue]: ... +def _jit_pass_onnx_eval_peephole( + graph: Graph, + paramsDict: dict[str, IValue], +) -> dict[str, IValue]: ... +def _jit_pass_onnx_constant_fold( + graph: Graph, + paramsDict: dict[str, IValue], + opset_version: _int, +) -> dict[str, IValue]: ... +def _jit_pass_onnx_eliminate_unused_items( + graph: Graph, + paramsDict: dict[str, IValue], +) -> dict[str, IValue]: ... +def _jit_pass_onnx_cast_all_constant_to_floating(graph: Graph) -> None: ... +def _jit_pass_filter_non_tensor_arguments( + params: dict[str, IValue], +) -> dict[str, Tensor]: ... +def _jit_decay_packed_param_input_types(graph: Graph) -> None: ... +def _jit_pass_onnx_node_shape_type_inference( + n: Node, + paramsDict: dict[str, IValue], + opset_version: _int, +) -> None: ... +def _jit_onnx_convert_pattern_from_subblock( + block: Block, + n: Node, + env: dict[Value, Value], + values_in_env: set[Value], +) -> list[Value]: ... +def _jit_pass_onnx_block( + old_block: Block, + new_block: Block, + operator_export_type: _onnx.OperatorExportTypes, + env: dict[Value, Value], + values_in_env: set[Value], + is_sub_block: _bool, +) -> dict[Value, Value]: ... +def _jit_pass_onnx_assign_scoped_names_for_node_and_value(graph: Graph) -> None: ... +def _jit_pass_fixup_onnx_controlflow_node( + n: Node, + opset_version: _int, +) -> list[Value]: ... +def _jit_onnx_create_full_scope_name(class_name: str, variable_name: str) -> str: ... +def _compile_graph_to_code_table(name: str, graph: Graph) -> IValue: ... +def _generate_upgraders_graph() -> dict[str, Graph]: ... +def _calculate_package_version_based_on_upgraders(val: _bool): ... +def _get_version_calculator_flag() -> _bool: ... +def _jit_script_interface_compile( + name: str, + class_def: ClassDef, + rcb: ResolutionCallback, + is_module: _bool, +): ... +def _jit_script_compile_overload( + qualname: str, + overload_decl: Decl, + implementation_def: Def, + rcb: ResolutionCallback, + implementation_defaults: dict[str, Any], + signature: Any, +): ... +def _jit_script_compile( + qual_name: str, + definition: Def, + rcb: ResolutionCallback, + defaults: dict[str, Any], +): ... +def _jit_script_class_compile( + qual_name: str, + definition: ClassDef, + defaults: dict[str, dict[str, Any]], + rcb: ResolutionCallback, +): ... +def _parse_source_def(src: str) -> Def: ... +def import_ir_module( + cu: CompilationUnit, + filename: str | Path, + map_location: DeviceLikeType | None, + extra_files: dict[str, Any], +) -> ScriptModule: ... +def import_ir_module_from_buffer( + cu: CompilationUnit, + buffer: IO[bytes], + map_location: DeviceLikeType | None, + extra_files: dict[str, Any], +) -> ScriptModule: ... +def _import_ir_module_from_package( + cu: CompilationUnit, + reader: PyTorchFileReader, + storage_context: DeserializationStorageContext, + map_location: DeviceLikeType | None, + ts_id: str, +) -> ScriptModule: ... +def _assign_output_shapes(graph: Graph, inputs: list[Tensor]) -> Graph: ... +def _check_onnx_proto(proto: str) -> None: ... +def _propagate_and_assign_input_shapes( + graph: Graph, + inputs: tuple[Tensor, ...], + param_count_list: list[_int], + with_grad: _bool, + propagate: _bool, +) -> Graph: ... + +# Defined in torch/csrc/jit/runtime/graph_executor.h +class GraphExecutorState: ... + +# Defined in torch/torch/csrc/jit/ir/alias_analysis.h +class AliasDb: ... + +class _InsertPoint: + def __enter__(self) -> None: ... + def __exit__(self, *exc_info: object) -> None: ... + +# Defined in torch/csrc/jit/ir/ir.h +class Use: + @property + def user(self) -> Node: ... + @property + def offset(self) -> _int: ... + def isAfter(self, other: Use) -> _bool: ... + +# Defined in torch/csrc/jit/ir/ir.h +class Value: + def type(self) -> JitType: ... + def setType(self, t: JitType) -> Value: ... + def setTypeAs(self, other: Value) -> Value: ... + def inferTypeFrom(self, t: Tensor) -> None: ... + def debugName(self) -> str: ... + def setDebugName(self, name: str) -> None: ... + def unique(self) -> _int: ... + def offset(self) -> _int: ... + def node(self) -> Node: ... + def uses(self) -> list[Use]: ... + def replaceAllUsesWith(self, val: Value) -> None: ... + def replaceAllUsesAfterNodeWith(self, node: Node, val: Value) -> None: ... + def requires_grad(self) -> _bool: ... + def requiresGrad(self) -> _bool: ... + def copyMetadata(self, other: Value) -> Value: ... + def isCompleteTensor(self) -> _bool: ... + def toIValue(self) -> IValue: ... + +# Defined in torch/csrc/jit/ir/ir.h +class Block: + def inputs(self) -> Iterator[Value]: ... + def outputs(self) -> Iterator[Value]: ... + def nodes(self) -> Iterator[Node]: ... + def paramNode(self) -> Node: ... + def returnNode(self) -> Node: ... + def owningNode(self) -> Node: ... + def registerOutput(self, n: Value) -> _int: ... + def addNode(self, name: str, inputs: Sequence[Value]) -> Node: ... + +# Defined in torch/csrc/jit/ir/ir.h +class Node: + def __getitem__(self, key: str) -> Any: ... + def schema(self) -> str: ... + def input(self) -> Value: ... + def inputs(self) -> Iterator[Value]: ... + def inputsAt(self, idx: _int) -> Value: ... + def inputsSize(self) -> _int: ... + def output(self) -> Value: ... + def outputs(self) -> Iterator[Value]: ... + def outputsAt(self, idx: _int) -> Value: ... + def outputsSize(self) -> _int: ... + def hasMultipleOutputs(self) -> _bool: ... + def blocks(self) -> list[Block]: ... + def addBlock(self) -> Block: ... + def mustBeNone(self) -> _bool: ... + def matches(self, pattern: str) -> _bool: ... + def kind(self) -> str: ... + def kindOf(self, name: str) -> str: ... + def addInput(self, name: str) -> Value: ... + def replaceInput(self, i: _int, newValue: Value) -> Value: ... + def replaceInputWith(self, from_: Value, to: Value) -> None: ... + def replaceAllUsesWith(self, n: Node) -> None: ... + def insertBefore(self, n: Node) -> Node: ... + def insertAfter(self, n: Node) -> Node: ... + def isBefore(self, n: Node) -> _bool: ... + def isAfter(self, n: Node) -> _bool: ... + def moveBefore(self, n: Node) -> None: ... + def moveAfter(self, n: Node) -> None: ... + def removeInput(self, i: _int) -> None: ... + def removeAllInputs(self, i: _int) -> None: ... + def hasUses(self) -> _bool: ... + def eraseOutput(self, i: _int) -> None: ... + def addOutput(self) -> Value: ... + def scopeName(self) -> str: ... + def isNondeterministic(self) -> _bool: ... + def copyAttributes(self, rhs: Node) -> Node: ... + def copyMetadata(self, rhs: Node) -> Node: ... + def hasAttributes(self) -> _bool: ... + def hasAttribute(self, name: str) -> _bool: ... + def removeAttribute(self, attr: str) -> Node: ... + def namedInput(self, name: str) -> Value: ... + def sourceRange(self) -> SourceRange: ... + def owningBlock(self) -> Block: ... + def findNode(self, kind: str, recurse: _bool = True) -> Node: ... + def findAllNodes(self, kind: str, recurse: _bool = True) -> list[Node]: ... + def getModuleHierarchy(self) -> str: ... + def prev(self) -> Node: ... + def destroy(self) -> None: ... + def attributeNames(self) -> list[str]: ... + + # Accessors for attributes as types. + def f(self, name: str) -> _float: ... + def f_(self, name: str, val: _float) -> Node: ... + def fs(self, name: str) -> list[_float]: ... + def fs_(self, name: str, val: list[_float]) -> Node: ... + def c(self, name: str) -> complex: ... + def c_(self, name: str, val: complex) -> Node: ... + def s(self, name: str) -> str: ... + def s_(self, name: str, val: str) -> Node: ... + def ss(self, name: str) -> list[str]: ... + def ss_(self, name: str, val: list[str]) -> Node: ... + def i(self, name: str) -> _int: ... + def i_(self, name: str, val: _int) -> Node: ... + # Cannot define "is" like this because it's a reserved keyword in python. + # def is(self, name: str) -> List[_int]: ... + # def is_(self, name: str, val: List[_int]) -> Node: ... + def g(self, name: str) -> Graph: ... + def g_(self, name: str, val: Graph) -> Node: ... + def gs(self, name: str) -> list[Graph]: ... + def gs_(self, name: str, val: list[Graph]) -> Node: ... + def ival(self, name: str) -> IValue: ... + def ival_(self, name: str, val: IValue) -> Node: ... + def t(self, name: str) -> Tensor: ... + def t_(self, name: str, val: Tensor) -> Node: ... + def ts(self, name: str) -> list[Tensor]: ... + def ts_(self, name: str, val: list[Tensor]) -> Node: ... + def ty(self, name: str) -> JitType: ... + def ty_(self, name: str, val: JitType) -> Node: ... + def tys(self, name: str) -> list[JitType]: ... + def tys_(self, name: str, val: list[JitType]) -> Node: ... + +# Defined in torch/torch/csrc/jit/ir/ir.h +class Graph: + def inputs(self) -> Iterator[Value]: ... + def outputs(self) -> Iterator[Value]: ... + def nodes(self) -> Iterator[Node]: ... + def param_node(self) -> Node: ... + def return_node(self) -> Node: ... + def addInput(self, name: str = "") -> Value: ... + def eraseInput(self, i: _int) -> None: ... + def registerOutput(self, n: Value) -> _int: ... + def eraseOutput(self, i: _int) -> None: ... + def create(self, name: str, args, num_outputs: _int) -> Node: ... + def appendNode(self, n: Node) -> Node: ... + def prependNode(self, n: Node) -> Node: ... + def insertNode(self, n: Node) -> Node: ... + def block(self) -> Block: ... + def lint(self) -> None: ... + def alias_db(self) -> AliasDb: ... + def setInsertPoint(self, n: Block | Node) -> None: ... + def insert_point_guard(self, n: Block | Node) -> _InsertPoint: ... + def insertPoint(self) -> Node: ... + def insertGraph(self, callee: Graph, inputs: list[Value]) -> list[Value]: ... + def makeMultiOutputIntoTuple(self) -> None: ... + def copy(self) -> Graph: ... + +# Defined in torch/aten/src/ATen/core/alias_info.h +class AliasInfo: + is_write: _bool + before_set: set[str] + after_set: set[str] + def __init__( + self, + is_write: _bool, + before_set: set[str], + after_set: set[str], + ) -> None: ... + +# Defined in torch/aten/src/ATen/core/function_schema.h +class Argument: + name: str + type: JitType + default_value: Any | None + def has_default_value(self) -> _bool: ... + kwarg_only: _bool + is_out: _bool + alias_info: AliasInfo | None + is_write: _bool + real_type: JitType + def __init__( + self, + name: str, + type: JitType, + N: _int | None, + defualt_value: Any | None, + kwarg_only: _bool, + alias_info: AliasInfo | None, + ) -> None: ... + +class FunctionSchema: + arguments: list[Argument] + returns: list[Argument] + name: str + overload_name: str + is_mutable: _bool + def __init__( + self, + name: str, + overload_name: str, + arguments: list[Argument], + returns: list[Argument], + is_vararg: _bool, + is_varret: _bool, + ) -> None: ... + def _is_view_op(self) -> _bool: ... + +class _UpgraderEntry: + bumped_at_version: _int + upgrader_name: str + old_schema: str + def __init__( + self, + bumped_at_version: _int, + upgrader_name: str, + old_schema: str, + ) -> None: ... + +class _UpgraderRange: + min_version: _int + max_version: _int + +def _get_max_operator_version() -> _int: ... +def _get_operator_version_map() -> dict[str, list[_UpgraderEntry]]: ... +def _get_upgrader_ranges(name: str) -> list[_UpgraderRange]: ... +def _test_only_add_entry_to_op_version(op_name: str, entry: _UpgraderEntry) -> None: ... +def _test_only_remove_entry_to_op_version(op_name: str) -> None: ... + +# Defined in torch/csrc/jit/python/script_init.cpp +class ScriptModuleSerializer: + def __init__(self, export_writer: PyTorchFileWriter) -> None: ... + def serialize(self, model: ScriptModule, script_module_id: _int) -> None: ... + def write_files(self) -> None: ... + def storage_context(self) -> SerializationStorageContext: ... + +# Defined in torch/csrc/jit/python/script_init.cpp +class SerializationStorageContext: + def __init__(self) -> None: ... + def has_storage(self, storage: Storage) -> _bool: ... + def get_or_add_storage(self, storage: Storage) -> _int: ... + +# Defined in torch/csrc/jit/python/script_init.cpp +class DeserializationStorageContext: + def __init__(self) -> None: ... + def get_storage(self, name: str, dtype: _dtype) -> Tensor: ... + def has_storage(self, name: str) -> _bool: ... + def add_storage(self, name: str, tensor: Tensor) -> _int: ... + +# Defined in torch/csrc/jit/python/script_init.cpp +class ConcreteModuleTypeBuilder: + def __init__(self, obj: Any) -> None: ... + def set_module_dict(self): ... + def set_module_list(self): ... + def set_parameter_list(self): ... + def set_parameter_dict(self): ... + def add_attribute( + self, + name: str, + ty: JitType, + is_param: _bool, + is_buffer: _bool, + ): ... + def add_module(self, name: str, meta: ConcreteModuleType): ... + def add_constant(self, name: str, value: Any): ... + def add_overload(self, method_name: str, overloaded_method_names: list[str]): ... + def add_builtin_function(self, name: str, symbol_name: str): ... + def add_failed_attribute(self, name: str, failure_reason: str): ... + def add_function_attribute( + self, + name: str, + ty: JitType, + func: Callable[..., Any], + ): ... + def add_ignored_attribute(self, name: str): ... + def add_ignored_attributes(self, names: list[str]): ... + def add_forward_hook(self, hook: Callable[..., Any]): ... + def add_forward_pre_hook(self, pre_hook: Callable[..., Any]): ... + +class ConcreteModuleType: + def get_constants(self) -> dict[str, Any]: ... + def equals(self, other: ConcreteModuleType) -> _bool: ... + @staticmethod + def from_jit_type(ty: JitType) -> ConcreteModuleType: ... + +class CallStack: + def __init__(self, name: str, range: SourceRange) -> None: ... + +class ErrorReport: + def __init__(self, range: SourceRange) -> None: ... + def what(self) -> str: ... + @staticmethod + def call_stack() -> str: ... + +class CompilationUnit: + def __init__(self, lang: str = ..., _frames_up: _int = ...) -> None: ... + def find_function(self, name: str) -> ScriptFunction: ... + def __getattr__(self, name: str) -> ScriptFunction: ... + def define( + self, + script: str, + rcb: ResolutionCallback = ..., + _frames_up: _int = ..., + ): ... + def get_interface(self, name: str) -> InterfaceType: ... + def get_functions(self) -> list[ScriptFunction]: ... + def create_function( + self, + name: str, + graph: Graph, + shouldMangle: _bool = ..., + ) -> ScriptFunction: ... + def get_class(self, name: str) -> ClassType: ... + +class ScriptObject: + def setattr(self, name: str, value: Any): ... + def _get_method(self, name: str) -> ScriptMethod: ... + def _type(self) -> ClassType: ... + +class ScriptModule(ScriptObject): + def _method_names(self) -> list[str]: ... + def _get_method(self, name: str) -> ScriptMethod: ... + +class LiteScriptModule: + def __call__(self, *input): ... + def find_method(self, method_name: str): ... + def forward(self, *input) -> list[str]: ... + def run_method(self, method_name: str, *input): ... + +# NOTE: switch to collections.abc.Callable in python 3.9 +class ScriptFunction(Generic[P, R]): + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R: ... + def save(self, filename: str, _extra_files: dict[str, bytes]) -> None: ... + def save_to_buffer(self, _extra_files: dict[str, bytes]) -> bytes: ... + @property + def graph(self) -> Graph: ... + def inlined_graph(self) -> Graph: ... + def schema(self) -> FunctionSchema: ... + def code(self) -> str: ... + def name(self) -> str: ... + @property + def qualified_name(self) -> str: ... + +# NOTE: switch to collections.abc.Callable in python 3.9 +class ScriptMethod(Generic[P, R]): + graph: Graph + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> R: ... + @property + def owner(self) -> ScriptModule: ... + @property + def name(self) -> str: ... + @property + def schema(self) -> FunctionSchema: ... + +class ScriptDict(Generic[K, T]): + def __init__(self, dict: dict[K, T]) -> None: ... + def __len__(self) -> _int: ... + def __contains__(self, key: K) -> _bool: ... + def __getitem__(self, key: K) -> T: ... + def __setitem__(self, key: K, value: T) -> None: ... + def __delitem__(self, key: K) -> None: ... + def __iter__(self) -> Iterator[K]: ... + def items(self) -> Iterator[tuple[K, T]]: ... + def keys(self) -> Iterator[K]: ... + +class ScriptList(Generic[T]): + def __init__(self, list: list[T]) -> None: ... + def __len__(self) -> _int: ... + def __contains__(self, item: T) -> _bool: ... + @overload + def __getitem__(self, idx: _int) -> T: ... + @overload + def __getitem__(self, idx: slice) -> ScriptList[T]: ... + @overload + def __setitem__(self, idx: _int, value: T) -> None: ... + @overload + def __setitem__(self, idx: slice, value: list[T]) -> None: ... + def __delitem__(self, idx: _int) -> None: ... + def __iter__(self) -> Iterator[T]: ... + def count(self, value: T) -> _int: ... + def remove(self, value: T) -> None: ... + def append(self, value: T) -> None: ... + def clear(self) -> None: ... + @overload + def extend(self, values: list[T]) -> None: ... + @overload + def extend(self, values: Iterable[T]) -> None: ... + @overload + def pop(self) -> T: ... + @overload + def pop(self, idx: _int) -> T: ... + +class ModuleDict: + def __init__(self, mod: ScriptModule) -> None: ... + def items(self) -> list[tuple[str, Any]]: ... + +class ParameterDict: + def __init__(self, mod: ScriptModule) -> None: ... + +class BufferDict: + def __init__(self, mod: ScriptModule) -> None: ... + +# Defined in torch/csrc/jit/api/module.h +class Module: ... + +# Defined in torch/csrc/Module.cpp +def _initExtension(shm_manager_path: str) -> None: ... # THPModule_initExtension +def _autograd_init() -> _bool: ... # THPAutograd_initExtension +def _add_docstr(obj: T, doc_obj: str) -> T: ... # THPModule_addDocStr +def _init_names(arg: Sequence[type]) -> None: ... # THPModule_initNames +def _has_distributed() -> _bool: ... # THPModule_hasDistributed +def _set_default_tensor_type(type) -> None: ... # THPModule_setDefaultTensorType +def _set_default_dtype(d: _dtype) -> None: ... # THPModule_setDefaultDtype +def _infer_size(arg1: Size, arg2: Size) -> Size: ... # THPModule_inferSize +def _crash_if_csrc_asan() -> _int: ... # THPModule_crashIfCsrcASAN +def _crash_if_csrc_ubsan() -> _int: ... # THPModule_crashIfCsrcUBSAN +def _crash_if_aten_asan() -> _int: ... # THPModule_crashIfATenASAN +def _show_config() -> str: ... # THPModule_showConfig +def _cxx_flags() -> str: ... # THPModule_cxxFlags +def _parallel_info() -> str: ... # THPModule_parallelInfo +def _get_cpu_capability() -> str: ... # THPModule_getCpuCapability +def _set_backcompat_broadcast_warn( + arg: _bool, +) -> None: ... # THPModule_setBackcompatBroadcastWarn +def _get_backcompat_broadcast_warn() -> ( + _bool +): ... # THPModule_getBackcompatBroadcastWarn +def _set_backcompat_keepdim_warn( + arg: _bool, +) -> None: ... # THPModule_setBackcompatKeepdimWarn +def _get_backcompat_keepdim_warn() -> _bool: ... # THPModule_getBackcompatKeepdimWarn +def get_num_thread() -> _int: ... # THPModule_getNumThreads +def set_num_threads(nthreads: _int) -> None: ... # THPModule_setNumThreads +def get_num_interop_threads() -> _int: ... # THPModule_getNumInteropThreads +def set_num_interop_threads( + nthreads: _int, +) -> None: ... # THPModule_setNumInteropThreads +def _get_cudnn_enabled() -> _bool: ... # THPModule_userEnabledCuDNN +def _set_cudnn_enabled(arg: _bool) -> None: ... # THPModule_setUserEnabledCuDNN +def _get_flash_sdp_enabled() -> _bool: ... # THPModule_userEnabledFusedSDP +def _set_sdp_use_flash(arg: _bool) -> None: ... # THPModule_setSDPUseFlash +def _get_mem_efficient_sdp_enabled() -> _bool: ... # THPModule_userEnabledMathSDP +def _set_sdp_use_mem_efficient( + arg: _bool, +) -> None: ... # THPModule_setSDPUseMemEfficient +def _get_math_sdp_enabled() -> _bool: ... # THPModule_userEnabledMathSDP +def _set_sdp_use_math(arg: _bool) -> None: ... # THPModule_setSDPUseMath +def _get_math_sdp_allow_fp16_bf16_reduction() -> ( + _bool +): ... # THPModule_allowFP16BF16ReductionMathSDP +def _set_math_sdp_allow_fp16_bf16_reduction( + arg: _bool, +) -> None: ... # THPModule_setAllowFP16BF16ReductionMathSDP +def _get_overrideable_sdp_enabled() -> ( + _bool +): ... # THPModule_userEnabledOverrideableSDP +def _set_sdp_use_overrideable( + arg: _bool, +) -> None: ... # THPModule_setSDPUseOverrideable +def _get_sdp_priority_order() -> list[_int]: ... # THPModule_getSDPPriorityOrder +def _set_sdp_priority_order( + arg: list[_int], +) -> None: ... # THPModule_setSDPPriorityOrder +def _get_cudnn_sdp_enabled() -> _bool: ... # THPModule_userEnabledMathSDP +def _set_sdp_use_cudnn(arg: _bool) -> None: ... # THPModule_setSDPUseMath +def _get_mkldnn_enabled() -> _bool: ... # THPModule_userEnabledMkldnn +def _set_mkldnn_enabled(arg: _bool) -> None: ... # THPModule_setUserEnabledMkldnn +def _get_cudnn_benchmark() -> _bool: ... # THPModule_benchmarkCuDNN +def _set_cudnn_benchmark(arg: _bool) -> None: ... # THPModule_setBenchmarkCuDNN +def _get_miopen_immediate() -> _bool: ... # THPModule_userImmediateMiopen +def _set_miopen_immediate(arg: _bool) -> None: ... # THPModule_setUserImmediateMiopen +def _get_cudnn_deterministic() -> _bool: ... # THPModule_deterministicCuDNN +def _set_cudnn_deterministic(arg: _bool) -> None: ... # THPModule_setDeterministicCuDNN +def _get_mkldnn_deterministic() -> _bool: ... # THPModule_deterministicMkldnn +def _set_mkldnn_deterministic( + arg: _bool, +) -> None: ... # THPModule_setDeterministicMkldnn +def _get_onednn_allow_tf32() -> _bool: ... # THPModule_allowTF32OneDNN +def _set_onednn_allow_tf32(arg: _bool) -> None: ... # THPModule_setAllowTF32OneDNN +def _get_deterministic_algorithms() -> _bool: ... # THPModule_deterministicAlgorithms +def _get_deterministic_algorithms_warn_only() -> ( + _bool +): ... # THPModule_deterministicAlgorithmsWarnOnly +def _set_deterministic_algorithms( + mode: _bool, + *, + warn_only: _bool = ..., +) -> None: ... # THPModule_setDeterministicAlgorithms +def _get_deterministic_fill_uninitialized_memory() -> ( + _bool +): ... # THPModule_deterministicFillUninitializedMemory +def _set_deterministic_fill_uninitialized_memory( + arg: _bool, +) -> None: ... # THPModule_setDeterministicFillUninitializedMemory +def _get_nnpack_enabled() -> _bool: ... # THPModule_userEnabledNNPACK +def _set_nnpack_enabled(arg: _bool) -> None: ... # THPModule_setUserEnabledNNPACK +def _get_warnAlways() -> _bool: ... # THPModule_warnAlways +def _set_warnAlways(arg: _bool) -> None: ... # THPModule_setWarnAlways +def _get_cudnn_allow_tf32() -> _bool: ... # THPModule_allowTF32CuDNN +def _set_cudnn_allow_tf32(arg: _bool) -> None: ... # THPModule_setAllowTF32CuDNN +def _get_cublas_allow_tf32() -> _bool: ... # THPModule_allowTF32CuBLAS +def _set_cublas_allow_tf32(arg: _bool) -> None: ... # THPModule_setAllowTF32CuBLAS +def _get_float32_matmul_precision() -> str: ... # THPModule_float32MatmulPrecision +def _set_float32_matmul_precision( + arg: str, +) -> None: ... # THPModule_setFloat32MatmulPrecision +def _get_cublas_allow_fp16_reduced_precision_reduction() -> tuple[ + _bool, _bool +]: ... # THPModule_allowFP16ReductionCuBLAS +def _set_cublas_allow_fp16_reduced_precision_reduction( + arg: _bool, + allow_splitk: _bool = ..., +) -> None: ... # THPModule_setAllowFP16ReductionCuBLAS +def _get_cublas_allow_bf16_reduced_precision_reduction() -> tuple[ + _bool, _bool +]: ... # THPModule_allowBF16ReductionCuBLAS +def _set_cublas_allow_bf16_reduced_precision_reduction( + arg: _bool, + allow_splitk: _bool = ..., +) -> None: ... # THPModule_setAllowBF16ReductionCuBLAS +def _get_cublas_allow_fp16_accumulation() -> ( + _bool +): ... # THPModule_allowFP16AccumulationCuBLAS +def _set_cublas_allow_fp16_accumulation( + arg: _bool, +) -> None: ... # THPModule_setAllowFP16AccumulationCuBLAS +def _get_sm_carveout_experimental() -> _int | None: ... +def _set_sm_carveout_experimental(arg: _int | None) -> None: ... +def _set_conj(x: Tensor, conj: _bool) -> None: ... +def _set_neg(x: Tensor, neg: _bool) -> None: ... +def _set_meta_in_tls_dispatch_include(meta_in_tls: _bool) -> None: ... +def _autocast_supported_devices() -> list[str]: ... +def _meta_in_tls_dispatch_include() -> _bool: ... +def _stash_obj_in_tls(key: str, arg: Any) -> None: ... +def _get_obj_in_tls(key: str) -> Any: ... +def _is_key_in_tls(key: str) -> _bool: ... +def _select_batch_norm_backend(*args, **kwargs) -> BatchNormBackend: ... +def _select_conv_backend(*args, **kwargs) -> ConvBackend: ... +def _conv_determine_backend_memory_format( + input: Tensor, + weight: Tensor, + backend: ConvBackend, +) -> memory_format: ... +def _has_storage(x: Tensor) -> _bool: ... +def _construct_storage_from_data_pointer( + data_ptr: _int, + device: torch.device, + size: _int, +) -> Storage: ... +def _should_allow_numbers_as_tensors(func_name: str) -> _bool: ... +def _group_tensors_by_device_and_dtype( + nested_tensorlists: list[list[Tensor | None]], + with_indices: _bool = False, +) -> dict[ + tuple[torch.device, torch.dtype], + tuple[list[list[Tensor | None]], list[_int]], +]: ... +def _initCrashHandler() -> None: ... +def _set_warn_on_accumulate_grad_stream_mismatch(enabled: _bool) -> None: ... + +# NB: There is no Capsule type in typing, see +# https://github.com/python/cpython/issues/109562 +def _to_dlpack( + data: Tensor, + dl_device: tuple[IntEnum, _int] | None = None, + copy: _bool | None = None, +) -> Any: ... # THPModule_toDLPack +def _to_dlpack_versioned( + data: Tensor, + dl_device: tuple[IntEnum, _int] | None = None, + copy: _bool | None = None, +) -> Any: ... # THPModule_toDLPackVersioned +def _from_dlpack(data: Any) -> Tensor: ... # THPModule_fromDLPack +def _torchDeviceToDLDevice( + device: torch.device, +) -> tuple[_int, _int]: ... # THPModule_torchDeviceToDLDevice +def _dlpack_exchange_api() -> object: ... # THPModule_DLPackExchangeAPI +def _get_cpp_backtrace( + frames_to_skip: _int, + maximum_number_of_frames: _int, +) -> str: ... # THPModule_getCppBacktrace +def set_flush_denormal(arg: _bool) -> _bool: ... # THPModule_setFlushDenormal +def get_default_dtype() -> _dtype: ... # THPModule_getDefaultDtype +def _get_default_device() -> str: ... # THPModule_getDefaultDevice +def _get_qengine() -> _int: ... # THPModule_qEngine +def _set_qengine(qengine: _int) -> None: ... # THPModule_setQEngine +def _supported_qengines() -> list[_int]: ... # THPModule_supportedQEngines +def _is_xnnpack_enabled() -> _bool: ... # THPModule_isEnabledXNNPACK +def _check_sparse_tensor_invariants() -> ( + _bool +): ... # THPModule_checkSparseTensorInvariants +def _set_check_sparse_tensor_invariants( + arg: _bool, +) -> None: ... # THPModule_setCheckSparseTensorInvariants +def _is_default_mobile_cpu_allocator_set() -> ( + _bool +): ... # THPModule_isDefaultMobileCPUAllocatorSet +def _set_default_mobile_cpu_allocator() -> ( + None +): ... # THPModule_setDefaultMobileCPUAllocator +def _unset_default_mobile_cpu_allocator() -> ( + None +): ... # THPModule_unsetDefaultMobileCPUAllocator +def _is_torch_function_enabled() -> _bool: ... # THPModule_isEnabledTorchFunction +def _is_torch_function_all_disabled() -> ( + _bool +): ... # THPModule_isAllDisabledTorchFunction +def _has_torch_function( + args: Iterable[Any], +) -> _bool: ... # THPModule_has_torch_function +def _has_torch_function_unary(Any) -> _bool: ... # THPModule_has_torch_function_unary +def _has_torch_function_variadic( + *args: Any, +) -> _bool: ... # THPModule_has_torch_function_variadic +def _vmapmode_increment_nesting() -> _int: ... # THPModule_vmapmode_increment_nesting +def _vmapmode_decrement_nesting() -> _int: ... # THPModule_vmapmode_decrement_nesting +def _log_api_usage_once(str) -> None: ... # LogAPIUsageOnceFromPython +def _log_api_usage_metadata( + event: str, + metadata_map: dict[str, str], +) -> None: ... # LogAPIUsageMetadataFromPython +def _demangle(str) -> str: ... # c10::demangle +def _disabled_torch_function_impl( + func: Callable, + types: Iterable[type], + args: tuple, + kwargs: dict, +) -> Any: ... # THPModule_disable_torch_function +def _disabled_torch_dispatch_impl( + func: Callable, + types: Iterable[type], + args: tuple, + kwargs: dict, +) -> Any: ... # THPModule_disable_dispatch_function +def _get_linalg_preferred_backend() -> _LinalgBackend: ... +def _set_linalg_preferred_backend(arg: _LinalgBackend): ... +def _get_fp32_precision_getter(backend: str, op: str) -> str: ... +def _set_fp32_precision_setter(backend: str, op: str, value: str) -> str: ... +def _ensureCUDADeviceGuardSet() -> None: ... + +class _LinalgBackend: + Default: _LinalgBackend + Cusolver: _LinalgBackend + Magma: _LinalgBackend + +# mypy error: +# Detected enum "torch._C.BatchNormBackend" in a type stub with zero +# members. There is a chance this is due to a recent change in the semantics +# of enum membership. If so, use `member = value` to mark an enum member, +# instead of `member: type` +class BatchNormBackend(Enum): ... # type: ignore[misc] + +def _get_blas_preferred_backend() -> _BlasBackend: ... +def _set_blas_preferred_backend(arg: _BlasBackend): ... + +class _BlasBackend: + Default: _BlasBackend + Cublas: _BlasBackend + Cublaslt: _BlasBackend + Ck: _BlasBackend + +def _get_rocm_fa_preferred_backend() -> torch._C._ROCmFABackend: ... +def _set_rocm_fa_preferred_backend(arg: torch._C._ROCmFABackend): ... + +class _ROCmFABackend: + Default: _ROCmFABackend + AOTriton: _ROCmFABackend + Ck: _ROCmFABackend + +# mypy error: +# Error (MYPY) [misc] +# Detected enum "torch._C.ConvBackend" in a type stub with zero members. +# There is a chance this is due to a recent change in the semantics of enum +# membership. If so, use `member = value` to mark an enum member, instead of +# `member: type` +class ConvBackend(Enum): ... # type: ignore[misc] + +class Tag(Enum): + core = 0 + cudagraph_unsafe = 1 + data_dependent_output = 2 + dynamic_output_shape = 3 + flexible_layout = 4 + generated = 5 + inplace_view = 6 + maybe_aliasing_or_mutating = 7 + needs_contiguous_strides = 8 + needs_exact_strides = 9 + needs_fixed_stride_order = 10 + nondeterministic_bitwise = 11 + nondeterministic_seeded = 12 + pointwise = 13 + pt2_compliant_tag = 14 + reduction = 15 + view_copy = 16 + +# Defined in `valgrind.h` and `callgrind.h` respectively. +def _valgrind_supported_platform() -> _bool: ... # NVALGRIND +def _valgrind_toggle() -> None: ... # CALLGRIND_TOGGLE_COLLECT +def _valgrind_toggle_and_dump_stats() -> ( + None +): ... # CALLGRIND_TOGGLE_COLLECT and CALLGRIND_DUMP_STATS + +has_openmp: _bool +has_mkl: _bool +_has_kleidiai: _bool +_has_mps: _bool +has_lapack: _bool +_has_cuda: _bool +_has_magma: _bool +_has_xpu: _bool +_has_mkldnn: _bool +_has_mkldnn_acl: _bool +_has_cudnn: _bool +_has_cusparselt: _bool +has_spectral: _bool +_GLIBCXX_USE_CXX11_ABI: _bool +default_generator: Generator + +# Defined in torch/csrc/autograd/init.cpp +def _set_grad_enabled(enabled: _bool) -> None: ... +def is_grad_enabled() -> _bool: ... +def _set_fwd_grad_enabled(enabled: _bool) -> None: ... +def _is_fwd_grad_enabled() -> _bool: ... +def _any_requires_grad(*args, **kwargs) -> _bool: ... +def _any_output_is_alias_to_input_or_output(*args, **kwargs) -> _bool: ... +def is_inference_mode_enabled() -> _bool: ... +@overload +def set_autocast_enabled(device_type: str, enabled: _bool) -> None: ... +@overload +def set_autocast_enabled(enabled: _bool) -> None: ... +@overload +def is_autocast_enabled(device_type: str) -> _bool: ... +@overload +def is_autocast_enabled() -> _bool: ... +def set_autocast_dtype(device_type: str, dtype: _dtype) -> None: ... +def get_autocast_dtype(device_type: str) -> _dtype: ... +def clear_autocast_cache() -> None: ... +def set_autocast_cpu_enabled(enabled: _bool) -> None: ... +def is_autocast_cpu_enabled() -> _bool: ... +def _is_any_autocast_enabled() -> _bool: ... +def _is_autocast_available(device_type: str) -> _bool: ... +def set_autocast_cpu_dtype(dtype: _dtype) -> None: ... +def set_autocast_gpu_dtype(dtype: _dtype) -> None: ... +def get_autocast_cpu_dtype() -> _dtype: ... +def get_autocast_gpu_dtype() -> _dtype: ... +def autocast_increment_nesting() -> _int: ... +def autocast_decrement_nesting() -> _int: ... +def is_autocast_cache_enabled() -> _bool: ... +def set_autocast_cache_enabled(enabled: _bool) -> None: ... +def _increment_version(tensors: Iterable[Tensor]) -> None: ... +def set_anomaly_enabled(enabled: _bool, check_nan: _bool = True) -> None: ... +def is_anomaly_enabled() -> _bool: ... +def is_anomaly_check_nan_enabled() -> _bool: ... +def _is_multithreading_enabled() -> _bool: ... +def _set_multithreading_enabled(enabled: _bool) -> None: ... +def _set_view_replay_enabled(enabled: _bool) -> None: ... +def _is_view_replay_enabled() -> _bool: ... +def _set_graph_exec_group(group: GraphExecGroup | None) -> None: ... +def _get_graph_exec_group() -> GraphExecGroup | None: ... +def _enter_dual_level() -> _int: ... +def _exit_dual_level(level: _int) -> None: ... +def _make_dual(tensor: Tensor, tangent: Tensor, level: _int) -> Tensor: ... +def _unpack_dual(tensor: Tensor, level: _int) -> Tensor: ... +def __set_forward_AD_enabled(enabled: _bool) -> None: ... +def __is_forward_AD_enabled() -> _bool: ... +def _register_default_hooks(pack_hook: Callable, unpack_hook: Callable) -> None: ... +def _reset_default_hooks() -> None: ... +def _is_torch_function_mode_enabled() -> _bool: ... +def _push_on_torch_function_stack(cls: Any) -> None: ... +def _pop_torch_function_stack() -> Any: ... +def _get_function_stack_at(idx: _int) -> Any: ... +def _len_torch_function_stack() -> _int: ... +def _set_torch_dispatch_mode(cls: Any) -> None: ... +def _push_on_torch_dispatch_stack(cls: TorchDispatchMode) -> None: ... +def _pop_torch_dispatch_stack(mode_key: _TorchDispatchModeKey | None = None) -> Any: ... +def _get_dispatch_mode(mode_key: _TorchDispatchModeKey | None) -> Any: ... +def _unset_dispatch_mode(mode: _TorchDispatchModeKey) -> TorchDispatchMode | None: ... +def _set_dispatch_mode(mode: TorchDispatchMode) -> None: ... +def _get_dispatch_stack_at(idx: _int) -> Any: ... +def _len_torch_dispatch_stack() -> _int: ... +def _activate_gpu_trace() -> None: ... + +class _DisableTorchDispatch: + def __init__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +class _EnableTorchFunction: + def __init__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +class _EnablePythonDispatcher: + def __init__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +class _DisablePythonDispatcher: + def __init__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +class _EnablePreDispatch: + def __init__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +class _DisableFuncTorch: + def __init__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +class _DisableAutocast: + def __init__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +class _InferenceMode: + def __init__(self, enabled: _bool) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +def _set_autograd_fallback_mode(mode: str) -> None: ... +def _get_autograd_fallback_mode() -> str: ... + +# Defined in torch/csrc/jit/python/script_init.cpp +class LoggerBase: ... +class NoopLogger(LoggerBase): ... +class LockingLogger(LoggerBase): ... + +class AggregationType(Enum): + SUM = 0 + AVG = 1 + +class FileCheck: + def run(self, test_string: str) -> None: ... + def check(self, test_string: str) -> FileCheck: ... + def check_not(self, test_string: str) -> FileCheck: ... + def check_same(self, test_string: str) -> FileCheck: ... + def check_next(self, test_string: str) -> FileCheck: ... + def check_count( + self, + test_string: str, + count: _int, + exactly: _bool = False, + ) -> FileCheck: ... + def check_dag(self, test_string: str) -> FileCheck: ... + def check_source_highlighted(self, test_string: str) -> FileCheck: ... + def check_regex(self, test_string: str) -> FileCheck: ... + +# Defined in torch/csrc/jit/python/init.cpp +class PyTorchFileReader: + @overload + def __init__(self, name: str) -> None: ... + @overload + def __init__(self, buffer: IO[bytes]) -> None: ... + def get_record(self, name: str) -> bytes: ... + def get_all_records(self) -> list[str]: ... + def serialization_id(self) -> str: ... + +class PyTorchFileWriter: + @overload + def __init__( + self, + name: str, + compute_crc32: _bool = True, + storage_alignment: _int = 64, + ) -> None: ... + @overload + def __init__( + self, + buffer: IO[bytes], + compute_crc32: _bool = True, + storage_alignment: _int = 64, + ) -> None: ... + def write_record( + self, + name: str, + data: Storage | bytes | _int, + size: _int, + ) -> None: ... + def write_end_of_file(self) -> None: ... + def set_min_version(self, version: _int) -> None: ... + def get_all_written_records(self) -> list[str]: ... + def archive_name(self) -> str: ... + def serialization_id(self) -> str: ... + +def _jit_get_inline_everything_mode() -> _bool: ... +def _jit_set_inline_everything_mode(enabled: _bool) -> None: ... +def _jit_get_logging_option() -> str: ... +def _jit_set_logging_option(option: str) -> None: ... +def _jit_set_logging_stream(stream_name: str) -> None: ... +def _jit_pass_cse(Graph) -> _bool: ... +def _jit_pass_dce(Graph) -> None: ... +def _jit_pass_dce_graph(Graph) -> None: ... +def _jit_pass_lint(Graph) -> None: ... +def _register_opaque_type(type_name: str) -> None: ... +def _is_opaque_type_registered(type_name: str) -> _bool: ... + +# Defined in torch/csrc/jit/python/python_custom_class.cpp +def _get_custom_class_python_wrapper(name: str, attr: str) -> Any: ... + +# Defined in torch/csrc/Module.cpp +def _rename_privateuse1_backend(backend: str) -> None: ... +def _get_privateuse1_backend_name() -> str: ... + +# Defined in torch/csrc/Generator.cpp +class Generator: + device: _device + def __init__(self, device: DeviceLikeType | None = None) -> None: ... + def __reduce__( + self, + ) -> tuple[type[Generator], tuple[_device], tuple[_int, _int | None, Tensor]]: ... + def __setstate__(self, state: tuple[_int, _int | None, Tensor]) -> None: ... + def get_state(self) -> Tensor: ... + def set_state(self, _new_state: Tensor) -> Generator: ... + def clone_state(self) -> Generator: ... + def graphsafe_get_state(self) -> Generator: ... + def graphsafe_set_state(self, _new_state: Generator) -> Generator: ... + def set_offset(self, offset: _int) -> Generator: ... + def get_offset(self) -> _int: ... + def manual_seed(self, seed: _int) -> Generator: ... + def seed(self) -> _int: ... + def initial_seed(self) -> _int: ... + +# Defined in torch/csrc/utils/python_dispatch.cpp + +class _DispatchOperatorHandle: + def schema(self) -> FunctionSchema: ... + def debug(self) -> str: ... + def redispatch_boxed(self, keyset: DispatchKeySet, *args, **kwargs) -> Any: ... + +class _DispatchModule: + def reset(self) -> None: ... + def def_(self, schema: str, alias: str = "") -> _DispatchModule: ... + def def_legacy(self, schema: str) -> _DispatchModule: ... + def def_name_t_t( + self, + name: str, + dispatch: str, + debug: str = "default_def_name_t_t", + ) -> _DispatchModule: ... + def def_schema_t_t( + self, + schema: str, + dispatch: str, + alias: str, + debug: str = "default_def_schema_t_t", + ) -> _DispatchModule: ... + def impl_t_t( + self, + name: str, + dispatch: str, + debug: str = "impl_t_t", + ) -> _DispatchModule: ... + def impl_with_aoti_compile( + self, + ns: str, + op_name_with_overload: str, + dispatch: _dispatchkey, + ) -> None: ... + def impl(self, name: str, dispatch: _dispatchkey, func: Callable) -> None: ... + def define(self, schema: str, alias: str = "") -> str: ... + def fallback_fallthrough(self, dispatch: str = "") -> _DispatchModule: ... + def fallback( + self, + dispatch: _dispatchkey, + func: Callable, + with_keyset: _bool = False, + ) -> None: ... + +_after_ADInplaceOrView_keyset: DispatchKeySet +_after_autograd_keyset: DispatchKeySet + +class _SafeKernelFunction: + def call_boxed(self, keyset: DispatchKeySet, *args, **kwargs) -> Any: ... + @property + def op_handle(self) -> _DispatchOperatorHandle: ... + +def _dispatch_library( + kind: str, + name: str, + dispatch: str, + file: str = "", + linenum: Any = 0, +) -> _DispatchModule: ... +def _dispatch_dump(name: str) -> str: ... +def _dispatch_dump_table(name: str) -> str: ... +def _dispatch_check_invariants(name: str) -> None: ... +def _dispatch_check_all_invariants() -> None: ... +def _dispatch_call_boxed(handle: _DispatchOperatorHandle, *args, **kwargs) -> Any: ... +def _dispatch_find_schema_or_throw( + name: str, + overload_name: str, +) -> _DispatchOperatorHandle: ... +def _dispatch_set_report_error_callback( + handle: _DispatchOperatorHandle, + callback: Callable, +) -> None: ... +def _dispatch_has_kernel(name: str) -> _bool: ... +def _dispatch_has_kernel_for_dispatch_key( + name: str, + dispatch: _dispatchkey, +) -> _bool: ... +def _dispatch_has_kernel_for_any_dispatch_key( + name: str, + dispatch_key_set: DispatchKeySet, +) -> _bool: ... +def _dispatch_kernel_for_dispatch_key_is_fallthrough( + name: str, + dispatch: _dispatchkey, +) -> _bool: ... +def _dispatch_has_computed_kernel_for_dispatch_key( + name: str, + dispatch: _dispatchkey, +) -> _bool: ... +def _dispatch_get_computed_kernel_for_dispatch_key( + name: str, + dispatch: _dispatchkey, +) -> _SafeKernelFunction: ... +def _dispatch_find_dangling_impls() -> list[str]: ... +def _dispatch_get_all_op_names() -> list[str]: ... +def _dispatch_tls_set_dispatch_key_excluded( + dispatch: _dispatchkey, + val: _bool, +) -> None: ... +def _dispatch_tls_is_dispatch_key_excluded(dispatch: _dispatchkey) -> _bool: ... +def _dispatch_tls_set_dispatch_key_included( + dispatch: _dispatchkey, + val: _bool, +) -> None: ... +def _dispatch_tls_is_dispatch_key_included(dispatch: _dispatchkey) -> _bool: ... +def _dispatch_isTensorSubclassLike(tensor: Tensor) -> _bool: ... +def _dispatch_key_name(dispatch: _dispatchkey) -> str: ... +def _dispatch_key_for_device(device_type: str) -> str: ... +def _parse_dispatch_key(key: str) -> DispatchKey | None: ... +def _dispatch_key_parse(dispatch: _dispatchkey) -> DispatchKey: ... +def _dispatch_num_backends() -> _int: ... +def _dispatch_pystub(name: str, overload: str) -> tuple[str, str] | None: ... +def _dispatch_is_alias_key(dispatch: _dispatchkey) -> _bool: ... +def _functionality_to_backend_keys(dispatch: _dispatchkey) -> list[DispatchKey]: ... +def _functionalization_reapply_views_tls() -> _bool: ... +def _only_lift_cpu_tensors() -> _bool: ... +def _set_only_lift_cpu_tensors(value: _bool) -> None: ... +def _set_throw_on_mutable_data_ptr(tensor: Tensor) -> None: ... +def _set_warn_deprecated_on_mutable_data_ptr(tensor: Tensor) -> None: ... + +class DispatchKey(Enum): + Undefined = ... + FPGA = ... + MAIA = ... + Vulkan = ... + Metal = ... + MKLDNN = ... + OpenGL = ... + OpenCL = ... + IDEEP = ... + CustomRNGKeyId = ... + MkldnnCPU = ... + Sparse = ... + SparseCsr = ... + NestedTensor = ... + Dense = ... + PythonTLSSnapshot = ... + PreDispatch = ... + PythonDispatcher = ... + Python = ... + FuncTorchDynamicLayerBackMode = ... + ZeroTensor = ... + Conjugate = ... + Negative = ... + BackendSelect = ... + Named = ... + AutogradOther = ... + AutogradFunctionality = ... + AutogradNestedTensor = ... + Tracer = ... + Autocast = ... + AutocastCPU = ... + AutocastCUDA = ... + Batched = ... + VmapMode = ... + FuncTorchGradWrapper = ... + FuncTorchBatched = ... + BatchedNestedTensor = ... + FuncTorchVmapMode = ... + FuncTorchDynamicLayerFrontMode = ... + Functionalize = ... + TESTING_ONLY_GenericWrapper = ... + TESTING_ONLY_GenericMode = ... + ADInplaceOrView = ... + Autograd = ... + CompositeImplicitAutograd = ... + CompositeImplicitAutogradNestedTensor = ... + CompositeExplicitAutograd = ... + CompositeExplicitAutogradNonFunctional = ... + FuncTorchBatchedDecomposition = ... + CPU = ... + CUDA = ... + HIP = ... + XLA = ... + MTIA = ... + MPS = ... + IPU = ... + XPU = ... + HPU = ... + VE = ... + Lazy = ... + Meta = ... + PrivateUse1 = ... + PrivateUse2 = ... + PrivateUse3 = ... + QuantizedCPU = ... + QuantizedCUDA = ... + QuantizedHIP = ... + QuantizedXLA = ... + QuantizedMTIA = ... + QuantizedMPS = ... + QuantizedIPU = ... + QuantizedXPU = ... + QuantizedHPU = ... + QuantizedVE = ... + QuantizedLazy = ... + QuantizedMeta = ... + QuantizedPrivateUse1 = ... + QuantizedPrivateUse2 = ... + QuantizedPrivateUse3 = ... + SparseCPU = ... + SparseCUDA = ... + SparseHIP = ... + SparseXLA = ... + SparseMTIA = ... + SparseMPS = ... + SparseIPU = ... + SparseXPU = ... + SparseHPU = ... + SparseVE = ... + SparseLazy = ... + SparseMeta = ... + SparsePrivateUse1 = ... + SparsePrivateUse2 = ... + SparsePrivateUse3 = ... + SparseCsrCPU = ... + SparseCsrCUDA = ... + SparseCsrHIP = ... + SparseCsrXLA = ... + SparseCsrMTIA = ... + SparseCsrMPS = ... + SparseCsrIPU = ... + SparseCsrXPU = ... + SparseCsrHPU = ... + SparseCsrVE = ... + SparseCsrLazy = ... + SparseCsrMeta = ... + SparseCsrPrivateUse1 = ... + SparseCsrPrivateUse2 = ... + SparseCsrPrivateUse3 = ... + NestedTensorCPU = ... + NestedTensorCUDA = ... + NestedTensorHIP = ... + NestedTensorXLA = ... + NestedTensorMTIA = ... + NestedTensorMPS = ... + NestedTensorIPU = ... + NestedTensorXPU = ... + NestedTensorHPU = ... + NestedTensorVE = ... + NestedTensorLazy = ... + NestedTensorMeta = ... + NestedTensorPrivateUse1 = ... + NestedTensorPrivateUse2 = ... + NestedTensorPrivateUse3 = ... + AutogradCPU = ... + AutogradCUDA = ... + AutogradHIP = ... + AutogradXLA = ... + AutogradMTIA = ... + AutogradMPS = ... + AutogradIPU = ... + AutogradXPU = ... + AutogradHPU = ... + AutogradVE = ... + AutogradLazy = ... + AutogradMeta = ... + AutogradPrivateUse1 = ... + AutogradPrivateUse2 = ... + AutogradPrivateUse3 = ... + +class DispatchKeySet: + def __init__(self, key: DispatchKey) -> None: ... + def __or__(self, other: DispatchKeySet) -> DispatchKeySet: ... + def __sub__(self, other: DispatchKeySet) -> DispatchKeySet: ... + def __and__(self, other: DispatchKeySet) -> DispatchKeySet: ... + def raw_repr(self) -> _int: ... + @staticmethod + def from_raw_repr(raw: _int) -> DispatchKeySet: ... + def highestPriorityTypeId(self) -> DispatchKey: ... + def has(self, k: _dispatchkey) -> _bool: ... + def add(self, k: _dispatchkey) -> DispatchKeySet: ... + def remove(self, k: _dispatchkey) -> DispatchKeySet: ... + +_dispatch_autogradother_backends: DispatchKeySet +_additional_keys_to_prop_for_wrapper_tensors: DispatchKeySet + +def _dispatch_has_backend_fallback(dispatch: _dispatchkey) -> _bool: ... +def _dispatch_keyset_full_after(t: _dispatchkey) -> DispatchKeySet: ... +def _dispatch_keyset_full() -> DispatchKeySet: ... +def _dispatch_keyset_to_string(keyset: DispatchKeySet) -> str: ... +def _dispatch_get_backend_keyset_from_autograd( + dispatch: _dispatchkey, +) -> DispatchKeySet: ... +def _dispatch_keys(tensor: Tensor) -> DispatchKeySet: ... +def _dispatch_tls_local_exclude_set() -> DispatchKeySet: ... +def _dispatch_tls_local_include_set() -> DispatchKeySet: ... +def _dispatch_is_included_in_alias( + dispatch_a: _dispatchkey, + dispatch_b: _dispatchkey, +) -> _bool: ... +def _propagate_xla_data(a: Tensor, b: Tensor) -> None: ... +def _replace_(a: Tensor, b: Tensor) -> None: ... +def _commit_update(a: Tensor) -> None: ... + +class _ExcludeDispatchKeyGuard: + def __init__(self, keyset: DispatchKeySet) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +class _IncludeDispatchKeyGuard: + def __init__(self, k: DispatchKey) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +class _ForceDispatchKeyGuard: + def __init__(self, include: DispatchKeySet, exclude: DispatchKeySet) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +class _PreserveDispatchKeyGuard: + def __init__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +class _AutoDispatchBelowAutograd: + def __init__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +class _AutoDispatchBelowADInplaceOrView: + def __init__(self) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +def _dispatch_print_registrations_for_dispatch_key(dispatch_key: str = "") -> None: ... +def _dispatch_get_registrations_for_dispatch_key( + dispatch_key: str = "", +) -> list[str]: ... +def _are_functorch_transforms_active() -> _bool: ... + +# Define in torch/csrc/autograd/init.cpp +def _set_python_dispatcher(dispatcher: object) -> None: ... +def _get_nested_int(id: _int, coeff: _int) -> SymInt: ... +def _get_constant_bool_symnode(val: _bool) -> Any: ... + +class _TorchDispatchModeKey(Enum): + FAKE = ... + PROXY = ... + FUNCTIONAL = ... + +class _SetExcludeDispatchKeyGuard: + def __init__(self, k: DispatchKey, enabled: _bool) -> None: ... + def __enter__(self): ... + def __exit__(self, *exc_info: object) -> None: ... + +def _get_dtensor_allow_implicit_replication() -> _bool: ... +def _set_dtensor_allow_implicit_replication(value: _bool) -> None: ... + +# Defined in torch/csrc/utils/schema_info.h + +class _SchemaInfo: + def __init__(self, schema: FunctionSchema) -> None: ... + @overload + def is_mutable(self) -> _bool: ... + @overload + def is_mutable(self, name: str) -> _bool: ... + def has_argument(self, name: str) -> _bool: ... + +# Defined in torch/csrc/utils/init.cpp +class BenchmarkConfig: + num_calling_threads: _int + num_worker_threads: _int + num_warmup_iters: _int + num_iters: _int + profiler_output_path: str + +class BenchmarkExecutionStats: + latency_avg_ms: _float + num_iters: _int + +class ThroughputBenchmark: + def __init__(self, module: Any) -> None: ... + def add_input(self, *args: Any, **kwargs: Any) -> None: ... + def run_once(self, *args: Any, **kwargs: Any) -> Any: ... + def benchmark(self, config: BenchmarkConfig) -> BenchmarkExecutionStats: ... + +# Defined in torch/csrc/Storage.cpp +class StorageBase: ... + +# TODO: where +class DoubleTensor(Tensor): ... +class FloatTensor(Tensor): ... +class BFloat16Tensor(Tensor): ... +class LongTensor(Tensor): ... +class IntTensor(Tensor): ... +class ShortTensor(Tensor): ... +class HalfTensor(Tensor): ... +class CharTensor(Tensor): ... +class ByteTensor(Tensor): ... +class BoolTensor(Tensor): ... + +# Defined in torch/csrc/autograd/python_engine.cpp +class _ImperativeEngine: + def queue_callback(self, callback: Callable[[], None]) -> None: ... + def run_backward(self, *args: Any, **kwargs: Any) -> tuple[Tensor, ...]: ... + def is_checkpoint_valid(self) -> _bool: ... + +# Defined in torch/csrc/autograd/python_variable.cpp +class _TensorMeta(type): ... + +_Index: TypeAlias = SupportsIndex | _bool | _int | slice | EllipsisType | Tensor | None | _NestedSequence[_bool | _int | slice | EllipsisType | Tensor | None] # fmt: skip + +# Defined in torch/csrc/autograd/python_variable.cpp +class TensorBase(metaclass=_TensorMeta): + requires_grad: _bool + retains_grad: _bool + shape: Size + data: Tensor + names: list[str] + device: _device + dtype: _dtype + grad_dtype: _dtype | None + layout: _layout + real: Tensor + imag: Tensor + T: Tensor + H: Tensor + mT: Tensor + mH: Tensor + ndim: _int + output_nr: _int + _version: _int + _base: Tensor | None + _cdata: _int + grad_fn: _Node | None + _grad_fn: Any + _grad: Tensor | None + grad: Tensor | None + _backward_hooks: dict[_int, Callable[[Tensor], Tensor | None]] | None + nbytes: _int + itemsize: _int + _has_symbolic_sizes_strides: _bool + + def _view_func_unsafe( + self, + new_base: Tensor, + symint_visitor_fn: Callable[[_int], _int] | None = None, + tensor_visitor_fn: Callable[[Tensor], Tensor] | None = None, + ): ... + def __abs__(self) -> Tensor: ... + def __add__(self, other: Tensor | Number | _complex) -> Tensor: ... + @overload + def __and__(self, other: Tensor) -> Tensor: ... + @overload + def __and__(self, other: Number | _complex) -> Tensor: ... + @overload + def __and__(self, other: Tensor | _int) -> Tensor: ... + def __bool__(self) -> _bool: ... + def __complex__(self) -> _complex: ... + def __contains__(self, item: Any, /) -> _bool: ... + def __div__(self, other: Tensor | Number | _complex) -> Tensor: ... + @overload + def __eq__(self, other: Tensor | Number | _complex) -> Tensor: ... # type: ignore[overload-overlap] + @overload + def __eq__(self, other: object) -> _bool: ... + def __float__(self) -> _float: ... + def __floordiv__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __ge__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __getitem__(self, indices: _Index | tuple[_Index, ...], /) -> Tensor: ... + def __gt__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __iadd__(self, other: Tensor | Number | _complex) -> Tensor: ... # noqa: PYI034 + @overload + def __iand__(self, other: Tensor) -> Tensor: ... + @overload + def __iand__(self, other: Number | _complex) -> Tensor: ... + @overload + def __iand__(self, other: Tensor | _int) -> Tensor: ... + def __idiv__(self, other: Tensor | Number | _complex) -> Tensor: ... # noqa: PYI034 + def __ifloordiv__(self, other: Tensor | Number | _complex) -> Tensor: ... # noqa: PYI034 + @overload + def __ilshift__(self, other: Tensor) -> Tensor: ... + @overload + def __ilshift__(self, other: Number | _complex) -> Tensor: ... + @overload + def __ilshift__(self, other: Tensor | _int) -> Tensor: ... + def __imod__(self, other: Tensor | Number | _complex) -> Tensor: ... # noqa: PYI034 + def __imul__(self, other: Tensor | Number | _complex) -> Tensor: ... # noqa: PYI034 + def __index__(self) -> _int: ... + @overload + def __init__( + self, + *args: Any, + device: DeviceLikeType | None = None, + ) -> None: ... + @overload + def __init__(self, storage: Storage) -> None: ... + @overload + def __init__(self, other: Tensor) -> None: ... + @overload + def __init__( + self, + size: _size, + *, + device: DeviceLikeType | None = None, + ) -> None: ... + def __int__(self) -> _int: ... + def __invert__(self) -> Tensor: ... + @overload + def __ior__(self, other: Tensor) -> Tensor: ... + @overload + def __ior__(self, other: Number | _complex) -> Tensor: ... + @overload + def __ior__(self, other: Tensor | _int) -> Tensor: ... + @overload + def __irshift__(self, other: Tensor) -> Tensor: ... + @overload + def __irshift__(self, other: Number | _complex) -> Tensor: ... + @overload + def __irshift__(self, other: Tensor | _int) -> Tensor: ... + def __isub__(self, other: Tensor | Number | _complex) -> Tensor: ... # noqa: PYI034 + @overload + def __ixor__(self, other: Tensor) -> Tensor: ... + @overload + def __ixor__(self, other: Number | _complex) -> Tensor: ... + @overload + def __ixor__(self, other: Tensor | _int) -> Tensor: ... + def __le__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __long__(self) -> _int: ... + @overload + def __lshift__(self, other: Tensor) -> Tensor: ... + @overload + def __lshift__(self, other: Number | _complex) -> Tensor: ... + @overload + def __lshift__(self, other: Tensor | _int) -> Tensor: ... + def __lt__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __matmul__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __mod__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __mul__(self, other: Tensor | Number | _complex) -> Tensor: ... + @overload + def __ne__(self, other: Tensor | Number | _complex) -> Tensor: ... # type: ignore[overload-overlap] + @overload + def __ne__(self, other: object) -> _bool: ... + def __neg__(self) -> Tensor: ... + def __new__(cls, *args, **kwargs) -> Self: ... + def __nonzero__(self) -> _bool: ... + @overload + def __or__(self, other: Tensor) -> Tensor: ... + @overload + def __or__(self, other: Number | _complex) -> Tensor: ... + @overload + def __or__(self, other: Tensor | _int) -> Tensor: ... + def __pow__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __radd__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __rand__(self, other: Tensor | _int) -> Tensor: ... + def __rfloordiv__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __rmul__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __ror__(self, other: Tensor | _int) -> Tensor: ... + def __rpow__(self, other: Tensor | Number | _complex) -> Tensor: ... # type: ignore[has-type] + @overload + def __rshift__(self, other: Tensor) -> Tensor: ... + @overload + def __rshift__(self, other: Number | _complex) -> Tensor: ... + @overload + def __rshift__(self, other: Tensor | _int) -> Tensor: ... + def __rsub__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __rtruediv__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __rxor__(self, other: Tensor | _int) -> Tensor: ... + def __setitem__( + self, + indices: _Index | tuple[_Index, ...], + value: Tensor | Number, + /, + ) -> None: ... + def __sub__(self, other: Tensor | Number | _complex) -> Tensor: ... + def __truediv__(self, other: Tensor | Number | _complex) -> Tensor: ... + @overload + def __xor__(self, other: Tensor) -> Tensor: ... + @overload + def __xor__(self, other: Number | _complex) -> Tensor: ... + @overload + def __xor__(self, other: Tensor | _int) -> Tensor: ... + def _addmm_activation( + self, + mat1: Tensor, + mat2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + use_gelu: _bool = False, + ) -> Tensor: ... + def _autocast_to_full_precision( + self, + cuda_enabled: _bool, + cpu_enabled: _bool, + ) -> Tensor: ... + def _autocast_to_reduced_precision( + self, + cuda_enabled: _bool, + cpu_enabled: _bool, + cuda_dtype: _dtype, + cpu_dtype: _dtype, + ) -> Tensor: ... + def _coalesced_(self, coalesced: _bool) -> Tensor: ... + def _conj(self) -> Tensor: ... + def _conj_physical(self) -> Tensor: ... + def _dimI(self) -> _int: ... + def _dimV(self) -> _int: ... + @staticmethod + def _dtensor__new__( + cls: type[S], + local_tensor: Tensor, + spec: torch.distributed.tensor._dtensor_spec.DTensorSpec, + requires_grad: _bool, + ) -> S: ... + def _indices(self) -> Tensor: ... + def _is_all_true(self) -> Tensor: ... + def _is_any_true(self) -> Tensor: ... + def _is_view(self) -> _bool: ... + def _is_zerotensor(self) -> _bool: ... + def _lazy_clone(self) -> Tensor: ... + @staticmethod + def _make_subclass( + cls: type[S], + data: Tensor, + require_grad: _bool = False, + dispatch_strides: _bool = False, + dispatch_device: _bool = False, + device_for_backend_keys: _device | None = None, + ) -> S: ... + @staticmethod + def _make_wrapper_subclass( + cls: type[S], + size: Sequence[_int | SymInt], + strides: Sequence[_int | SymInt] | None = None, + storage_offset: _int | SymInt | None = None, + memory_format: torch.memory_format | None = None, + dtype: _dtype | None = None, + layout: _layout = strided, + device: _device | None = None, + pin_memory: _bool = False, + requires_grad: _bool = False, + dispatch_sizes_strides_policy: str | None = None, + dispatch_device: _bool = False, + dispatch_layout: _bool = False, + _extra_dispatch_keys: torch.DispatchKeySet | None = None, + storage_size: _int | SymInt | None = None, + ) -> S: ... + def _neg_view(self) -> Tensor: ... + def _nested_tensor_size(self) -> Tensor: ... + def _nested_tensor_storage_offsets(self) -> Tensor: ... + def _nested_tensor_strides(self) -> Tensor: ... + def _nnz(self) -> _int: ... + def _sparse_mask_projection( + self, + mask: Tensor, + accumulate_matches: _bool = False, + ) -> Tensor: ... + def _to_dense( + self, + dtype: _dtype | None = None, + masked_grad: _bool | None = None, + ) -> Tensor: ... + @overload + def _to_sparse( + self, + *, + layout: _layout | None = None, + blocksize: _int | _size | None = None, + dense_dim: _int | None = None, + ) -> Tensor: ... + @overload + def _to_sparse(self, sparse_dim: _int) -> Tensor: ... + def _to_sparse_bsc( + self, + blocksize: _int | _size, + dense_dim: _int | None = None, + ) -> Tensor: ... + def _to_sparse_bsr( + self, + blocksize: _int | _size, + dense_dim: _int | None = None, + ) -> Tensor: ... + def _to_sparse_csc(self, dense_dim: _int | None = None) -> Tensor: ... + def _to_sparse_csr(self, dense_dim: _int | None = None) -> Tensor: ... + def _values(self) -> Tensor: ... + def abs(self) -> Tensor: + r""" + abs() -> Tensor + + See :func:`torch.abs` + """ + + def abs_(self) -> Tensor: + r""" + abs_() -> Tensor + + In-place version of :meth:`~Tensor.abs` + """ + + def absolute(self) -> Tensor: + r""" + absolute() -> Tensor + + Alias for :func:`abs` + """ + + def absolute_(self) -> Tensor: + r""" + absolute_() -> Tensor + + In-place version of :meth:`~Tensor.absolute` + Alias for :func:`abs_` + """ + + def acos(self) -> Tensor: + r""" + acos() -> Tensor + + See :func:`torch.acos` + """ + + def acos_(self) -> Tensor: + r""" + acos_() -> Tensor + + In-place version of :meth:`~Tensor.acos` + """ + + def acosh(self) -> Tensor: + r""" + acosh() -> Tensor + + See :func:`torch.acosh` + """ + + def acosh_(self) -> Tensor: + r""" + acosh_() -> Tensor + + In-place version of :meth:`~Tensor.acosh` + """ + + def add( + self, + other: Tensor | Number | _complex | torch.SymInt | torch.SymFloat, + *, + alpha: Number | _complex | None = 1, + out: Tensor | None = None, + ) -> Tensor: + r""" + add(other, *, alpha=1) -> Tensor + + Add a scalar or tensor to :attr:`self` tensor. If both :attr:`alpha` + and :attr:`other` are specified, each element of :attr:`other` is scaled by + :attr:`alpha` before being used. + + When :attr:`other` is a tensor, the shape of :attr:`other` must be + :ref:`broadcastable ` with the shape of the underlying + tensor + + See :func:`torch.add` + """ + + def add_( + self, + other: Tensor | Number | _complex | torch.SymInt | torch.SymFloat, + *, + alpha: Number | _complex | None = 1, + ) -> Tensor: + r""" + add_(other, *, alpha=1) -> Tensor + + In-place version of :meth:`~Tensor.add` + """ + + def addbmm( + self, + batch1: Tensor, + batch2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + addbmm(batch1, batch2, *, beta=1, alpha=1) -> Tensor + + See :func:`torch.addbmm` + """ + + def addbmm_( + self, + batch1: Tensor, + batch2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + addbmm_(batch1, batch2, *, beta=1, alpha=1) -> Tensor + + In-place version of :meth:`~Tensor.addbmm` + """ + + def addcdiv( + self, + tensor1: Tensor, + tensor2: Tensor, + *, + value: Number | _complex = 1, + ) -> Tensor: + r""" + addcdiv(tensor1, tensor2, *, value=1) -> Tensor + + See :func:`torch.addcdiv` + """ + + def addcdiv_( + self, + tensor1: Tensor, + tensor2: Tensor, + *, + value: Number | _complex = 1, + ) -> Tensor: + r""" + addcdiv_(tensor1, tensor2, *, value=1) -> Tensor + + In-place version of :meth:`~Tensor.addcdiv` + """ + + def addcmul( + self, + tensor1: Tensor, + tensor2: Tensor, + *, + value: Number | _complex = 1, + ) -> Tensor: + r""" + addcmul(tensor1, tensor2, *, value=1) -> Tensor + + See :func:`torch.addcmul` + """ + + def addcmul_( + self, + tensor1: Tensor, + tensor2: Tensor, + *, + value: Number | _complex = 1, + ) -> Tensor: + r""" + addcmul_(tensor1, tensor2, *, value=1) -> Tensor + + In-place version of :meth:`~Tensor.addcmul` + """ + + def addmm( + self, + mat1: Tensor, + mat2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + addmm(mat1, mat2, *, beta=1, alpha=1) -> Tensor + + See :func:`torch.addmm` + """ + + def addmm_( + self, + mat1: Tensor, + mat2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + addmm_(mat1, mat2, *, beta=1, alpha=1) -> Tensor + + In-place version of :meth:`~Tensor.addmm` + """ + + def addmv( + self, + mat: Tensor, + vec: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + addmv(mat, vec, *, beta=1, alpha=1) -> Tensor + + See :func:`torch.addmv` + """ + + def addmv_( + self, + mat: Tensor, + vec: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + addmv_(mat, vec, *, beta=1, alpha=1) -> Tensor + + In-place version of :meth:`~Tensor.addmv` + """ + + def addr( + self, + vec1: Tensor, + vec2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + addr(vec1, vec2, *, beta=1, alpha=1) -> Tensor + + See :func:`torch.addr` + """ + + def addr_( + self, + vec1: Tensor, + vec2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + addr_(vec1, vec2, *, beta=1, alpha=1) -> Tensor + + In-place version of :meth:`~Tensor.addr` + """ + + def adjoint(self) -> Tensor: + r""" + adjoint() -> Tensor + + Alias for :func:`adjoint` + """ + + def align_as(self, other: Tensor) -> Tensor: + r""" + align_as(other) -> Tensor + + Permutes the dimensions of the :attr:`self` tensor to match the dimension order + in the :attr:`other` tensor, adding size-one dims for any new names. + + This operation is useful for explicit broadcasting by names (see examples). + + All of the dims of :attr:`self` must be named in order to use this method. + The resulting tensor is a view on the original tensor. + + All dimension names of :attr:`self` must be present in ``other.names``. + :attr:`other` may contain named dimensions that are not in ``self.names``; + the output tensor has a size-one dimension for each of those new names. + + To align a tensor to a specific order, use :meth:`~Tensor.align_to`. + + Examples:: + + # Example 1: Applying a mask + >>> mask = torch.randint(2, [127, 128], dtype=torch.bool).refine_names('W', 'H') + >>> imgs = torch.randn(32, 128, 127, 3, names=('N', 'H', 'W', 'C')) + >>> imgs.masked_fill_(mask.align_as(imgs), 0) + + + # Example 2: Applying a per-channel-scale + >>> def scale_channels(input, scale): + >>> scale = scale.refine_names('C') + >>> return input * scale.align_as(input) + + >>> num_channels = 3 + >>> scale = torch.randn(num_channels, names=('C',)) + >>> imgs = torch.rand(32, 128, 128, num_channels, names=('N', 'H', 'W', 'C')) + >>> more_imgs = torch.rand(32, num_channels, 128, 128, names=('N', 'C', 'H', 'W')) + >>> videos = torch.randn(3, num_channels, 128, 128, 128, names=('N', 'C', 'H', 'W', 'D')) + + # scale_channels is agnostic to the dimension order of the input + >>> scale_channels(imgs, scale) + >>> scale_channels(more_imgs, scale) + >>> scale_channels(videos, scale) + + .. warning:: + The named tensor API is experimental and subject to change. + """ + + @overload + def align_to( + self, + order: Sequence[str | EllipsisType | None], + ellipsis_idx: _int, + ) -> Tensor: ... + @overload + def align_to(self, names: Sequence[str | EllipsisType | None]) -> Tensor: ... + @overload + def all(self) -> Tensor: + r""" + all(dim=None, keepdim=False) -> Tensor + + See :func:`torch.all` + """ + + @overload + def all(self, dim: _size | None = None, keepdim: _bool = False) -> Tensor: + r""" + all(dim=None, keepdim=False) -> Tensor + + See :func:`torch.all` + """ + + @overload + def all(self, dim: _int, keepdim: _bool = False) -> Tensor: + r""" + all(dim=None, keepdim=False) -> Tensor + + See :func:`torch.all` + """ + + @overload + def all( + self, + dim: str | EllipsisType | None, + keepdim: _bool = False, + ) -> Tensor: + r""" + all(dim=None, keepdim=False) -> Tensor + + See :func:`torch.all` + """ + + def allclose( + self, + other: Tensor, + rtol: _float = 1e-05, + atol: _float = 1e-08, + equal_nan: _bool = False, + ) -> _bool: + r""" + allclose(other, rtol=1e-05, atol=1e-08, equal_nan=False) -> Tensor + + See :func:`torch.allclose` + """ + + def amax(self, dim: _int | _size = (), keepdim: _bool = False) -> Tensor: + r""" + amax(dim=None, keepdim=False) -> Tensor + + See :func:`torch.amax` + """ + + def amin(self, dim: _int | _size = (), keepdim: _bool = False) -> Tensor: + r""" + amin(dim=None, keepdim=False) -> Tensor + + See :func:`torch.amin` + """ + + def aminmax( + self, + *, + dim: _int | None = None, + keepdim: _bool = False, + ) -> torch.return_types.aminmax: + r""" + aminmax(*, dim=None, keepdim=False) -> (Tensor min, Tensor max) + + See :func:`torch.aminmax` + """ + + def angle(self) -> Tensor: + r""" + angle() -> Tensor + + See :func:`torch.angle` + """ + + @overload + def any(self) -> Tensor: + r""" + any(dim=None, keepdim=False) -> Tensor + + See :func:`torch.any` + """ + + @overload + def any(self, dim: _size | None = None, keepdim: _bool = False) -> Tensor: + r""" + any(dim=None, keepdim=False) -> Tensor + + See :func:`torch.any` + """ + + @overload + def any(self, dim: _int, keepdim: _bool = False) -> Tensor: + r""" + any(dim=None, keepdim=False) -> Tensor + + See :func:`torch.any` + """ + + @overload + def any( + self, + dim: str | EllipsisType | None, + keepdim: _bool = False, + ) -> Tensor: + r""" + any(dim=None, keepdim=False) -> Tensor + + See :func:`torch.any` + """ + + def apply_(self, callable: Callable) -> Tensor: + r""" + apply_(callable) -> Tensor + + Applies the function :attr:`callable` to each element in the tensor, replacing + each element with the value returned by :attr:`callable`. + + .. note:: + + This function only works with CPU tensors and should not be used in code + sections that require high performance. + """ + + def arccos(self) -> Tensor: + r""" + arccos() -> Tensor + + See :func:`torch.arccos` + """ + + def arccos_(self) -> Tensor: + r""" + arccos_() -> Tensor + + In-place version of :meth:`~Tensor.arccos` + """ + + def arccosh(self) -> Tensor: + r""" + acosh() -> Tensor + + See :func:`torch.arccosh` + """ + + def arccosh_(self) -> Tensor: + r""" + acosh_() -> Tensor + + In-place version of :meth:`~Tensor.arccosh` + """ + + def arcsin(self) -> Tensor: + r""" + arcsin() -> Tensor + + See :func:`torch.arcsin` + """ + + def arcsin_(self) -> Tensor: + r""" + arcsin_() -> Tensor + + In-place version of :meth:`~Tensor.arcsin` + """ + + def arcsinh(self) -> Tensor: + r""" + arcsinh() -> Tensor + + See :func:`torch.arcsinh` + """ + + def arcsinh_(self) -> Tensor: + r""" + arcsinh_() -> Tensor + + In-place version of :meth:`~Tensor.arcsinh` + """ + + def arctan(self) -> Tensor: + r""" + arctan() -> Tensor + + See :func:`torch.arctan` + """ + + def arctan2(self, other: Tensor) -> Tensor: + r""" + arctan2(other) -> Tensor + + See :func:`torch.arctan2` + """ + + def arctan2_(self, other: Tensor) -> Tensor: + r""" + atan2_(other) -> Tensor + + In-place version of :meth:`~Tensor.arctan2` + """ + + def arctan_(self) -> Tensor: + r""" + arctan_() -> Tensor + + In-place version of :meth:`~Tensor.arctan` + """ + + def arctanh(self) -> Tensor: + r""" + arctanh() -> Tensor + + See :func:`torch.arctanh` + """ + + def arctanh_(self) -> Tensor: + r""" + arctanh_(other) -> Tensor + + In-place version of :meth:`~Tensor.arctanh` + """ + + def argmax(self, dim: _int | None = None, keepdim: _bool = False) -> Tensor: + r""" + argmax(dim=None, keepdim=False) -> LongTensor + + See :func:`torch.argmax` + """ + + def argmin(self, dim: _int | None = None, keepdim: _bool = False) -> Tensor: + r""" + argmin(dim=None, keepdim=False) -> LongTensor + + See :func:`torch.argmin` + """ + + @overload + def argsort( + self, + *, + stable: _bool, + dim: _int = -1, + descending: _bool = False, + ) -> Tensor: + r""" + argsort(dim=-1, descending=False) -> LongTensor + + See :func:`torch.argsort` + """ + + @overload + def argsort(self, dim: _int = -1, descending: _bool = False) -> Tensor: + r""" + argsort(dim=-1, descending=False) -> LongTensor + + See :func:`torch.argsort` + """ + + @overload + def argsort( + self, + dim: str | EllipsisType | None, + descending: _bool = False, + ) -> Tensor: + r""" + argsort(dim=-1, descending=False) -> LongTensor + + See :func:`torch.argsort` + """ + + def argwhere(self) -> Tensor: + r""" + argwhere() -> Tensor + + See :func:`torch.argwhere` + """ + + def as_strided( + self, + size: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + storage_offset: _int | SymInt | None = None, + ) -> Tensor: + r""" + as_strided(size, stride, storage_offset=None) -> Tensor + + See :func:`torch.as_strided` + """ + + def as_strided_( + self, + size: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + storage_offset: _int | SymInt | None = None, + ) -> Tensor: + r""" + as_strided_(size, stride, storage_offset=None) -> Tensor + + In-place version of :meth:`~Tensor.as_strided` + """ + + def as_strided_scatter( + self, + src: Tensor, + size: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + storage_offset: _int | SymInt | None = None, + ) -> Tensor: + r""" + as_strided_scatter(src, size, stride, storage_offset=None) -> Tensor + + See :func:`torch.as_strided_scatter` + """ + + def as_subclass(self, cls: type[S]) -> S: + r""" + as_subclass(cls) -> Tensor + + Makes a ``cls`` instance with the same data pointer as ``self``. Changes + in the output mirror changes in ``self``, and the output stays attached + to the autograd graph. ``cls`` must be a subclass of ``Tensor``. + """ + + def asin(self) -> Tensor: + r""" + asin() -> Tensor + + See :func:`torch.asin` + """ + + def asin_(self) -> Tensor: + r""" + asin_() -> Tensor + + In-place version of :meth:`~Tensor.asin` + """ + + def asinh(self) -> Tensor: + r""" + asinh() -> Tensor + + See :func:`torch.asinh` + """ + + def asinh_(self) -> Tensor: + r""" + asinh_() -> Tensor + + In-place version of :meth:`~Tensor.asinh` + """ + + def atan(self) -> Tensor: + r""" + atan() -> Tensor + + See :func:`torch.atan` + """ + + def atan2(self, other: Tensor) -> Tensor: + r""" + atan2(other) -> Tensor + + See :func:`torch.atan2` + """ + + def atan2_(self, other: Tensor) -> Tensor: + r""" + atan2_(other) -> Tensor + + In-place version of :meth:`~Tensor.atan2` + """ + + def atan_(self) -> Tensor: + r""" + atan_() -> Tensor + + In-place version of :meth:`~Tensor.atan` + """ + + def atanh(self) -> Tensor: + r""" + atanh() -> Tensor + + See :func:`torch.atanh` + """ + + def atanh_(self) -> Tensor: + r""" + atanh_(other) -> Tensor + + In-place version of :meth:`~Tensor.atanh` + """ + + def baddbmm( + self, + batch1: Tensor, + batch2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + baddbmm(batch1, batch2, *, beta=1, alpha=1) -> Tensor + + See :func:`torch.baddbmm` + """ + + def baddbmm_( + self, + batch1: Tensor, + batch2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + baddbmm_(batch1, batch2, *, beta=1, alpha=1) -> Tensor + + In-place version of :meth:`~Tensor.baddbmm` + """ + + @overload + def bernoulli(self, *, generator: Generator | None = None) -> Tensor: + r""" + bernoulli(*, generator=None) -> Tensor + + Returns a result tensor where each :math:`\texttt{result[i]}` is independently + sampled from :math:`\text{Bernoulli}(\texttt{self[i]})`. :attr:`self` must have + floating point ``dtype``, and the result will have the same ``dtype``. + + See :func:`torch.bernoulli` + """ + + @overload + def bernoulli( + self, + p: _float, + *, + generator: Generator | None = None, + ) -> Tensor: + r""" + bernoulli(*, generator=None) -> Tensor + + Returns a result tensor where each :math:`\texttt{result[i]}` is independently + sampled from :math:`\text{Bernoulli}(\texttt{self[i]})`. :attr:`self` must have + floating point ``dtype``, and the result will have the same ``dtype``. + + See :func:`torch.bernoulli` + """ + + @overload + def bernoulli_( + self, + p: Tensor, + *, + generator: Generator | None = None, + ) -> Tensor: + r""" + bernoulli_(p=0.5, *, generator=None) -> Tensor + + Fills each location of :attr:`self` with an independent sample from + :math:`\text{Bernoulli}(\texttt{p})`. :attr:`self` can have integral + ``dtype``. + + :attr:`p` should either be a scalar or tensor containing probabilities to be + used for drawing the binary random number. + + If it is a tensor, the :math:`\text{i}^{th}` element of :attr:`self` tensor + will be set to a value sampled from + :math:`\text{Bernoulli}(\texttt{p\_tensor[i]})`. In this case `p` must have + floating point ``dtype``. + + See also :meth:`~Tensor.bernoulli` and :func:`torch.bernoulli` + """ + + @overload + def bernoulli_( + self, + p: _float = 0.5, + *, + generator: Generator | None = None, + ) -> Tensor: + r""" + bernoulli_(p=0.5, *, generator=None) -> Tensor + + Fills each location of :attr:`self` with an independent sample from + :math:`\text{Bernoulli}(\texttt{p})`. :attr:`self` can have integral + ``dtype``. + + :attr:`p` should either be a scalar or tensor containing probabilities to be + used for drawing the binary random number. + + If it is a tensor, the :math:`\text{i}^{th}` element of :attr:`self` tensor + will be set to a value sampled from + :math:`\text{Bernoulli}(\texttt{p\_tensor[i]})`. In this case `p` must have + floating point ``dtype``. + + See also :meth:`~Tensor.bernoulli` and :func:`torch.bernoulli` + """ + + def bfloat16(self) -> Tensor: + r""" + bfloat16(memory_format=torch.preserve_format) -> Tensor + ``self.bfloat16()`` is equivalent to ``self.to(torch.bfloat16)``. See :func:`to`. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + def bincount( + self, + weights: Tensor | None = None, + minlength: _int | SymInt = 0, + ) -> Tensor: + r""" + bincount(weights=None, minlength=0) -> Tensor + + See :func:`torch.bincount` + """ + + @overload + def bitwise_and(self, other: Tensor) -> Tensor: + r""" + bitwise_and() -> Tensor + + See :func:`torch.bitwise_and` + """ + + @overload + def bitwise_and(self, other: Number | _complex) -> Tensor: + r""" + bitwise_and() -> Tensor + + See :func:`torch.bitwise_and` + """ + + @overload + def bitwise_and_(self, other: Tensor) -> Tensor: + r""" + bitwise_and_() -> Tensor + + In-place version of :meth:`~Tensor.bitwise_and` + """ + + @overload + def bitwise_and_(self, other: Number | _complex) -> Tensor: + r""" + bitwise_and_() -> Tensor + + In-place version of :meth:`~Tensor.bitwise_and` + """ + + @overload + def bitwise_left_shift(self, other: Tensor) -> Tensor: + r""" + bitwise_left_shift(other) -> Tensor + + See :func:`torch.bitwise_left_shift` + """ + + @overload + def bitwise_left_shift(self, other: Number | _complex) -> Tensor: + r""" + bitwise_left_shift(other) -> Tensor + + See :func:`torch.bitwise_left_shift` + """ + + @overload + def bitwise_left_shift_(self, other: Tensor) -> Tensor: + r""" + bitwise_left_shift_(other) -> Tensor + + In-place version of :meth:`~Tensor.bitwise_left_shift` + """ + + @overload + def bitwise_left_shift_(self, other: Number | _complex) -> Tensor: + r""" + bitwise_left_shift_(other) -> Tensor + + In-place version of :meth:`~Tensor.bitwise_left_shift` + """ + + def bitwise_not(self) -> Tensor: + r""" + bitwise_not() -> Tensor + + See :func:`torch.bitwise_not` + """ + + def bitwise_not_(self) -> Tensor: + r""" + bitwise_not_() -> Tensor + + In-place version of :meth:`~Tensor.bitwise_not` + """ + + @overload + def bitwise_or(self, other: Tensor) -> Tensor: + r""" + bitwise_or() -> Tensor + + See :func:`torch.bitwise_or` + """ + + @overload + def bitwise_or(self, other: Number | _complex) -> Tensor: + r""" + bitwise_or() -> Tensor + + See :func:`torch.bitwise_or` + """ + + @overload + def bitwise_or_(self, other: Tensor) -> Tensor: + r""" + bitwise_or_() -> Tensor + + In-place version of :meth:`~Tensor.bitwise_or` + """ + + @overload + def bitwise_or_(self, other: Number | _complex) -> Tensor: + r""" + bitwise_or_() -> Tensor + + In-place version of :meth:`~Tensor.bitwise_or` + """ + + @overload + def bitwise_right_shift(self, other: Tensor) -> Tensor: + r""" + bitwise_right_shift(other) -> Tensor + + See :func:`torch.bitwise_right_shift` + """ + + @overload + def bitwise_right_shift(self, other: Number | _complex) -> Tensor: + r""" + bitwise_right_shift(other) -> Tensor + + See :func:`torch.bitwise_right_shift` + """ + + @overload + def bitwise_right_shift_(self, other: Tensor) -> Tensor: + r""" + bitwise_right_shift_(other) -> Tensor + + In-place version of :meth:`~Tensor.bitwise_right_shift` + """ + + @overload + def bitwise_right_shift_(self, other: Number | _complex) -> Tensor: + r""" + bitwise_right_shift_(other) -> Tensor + + In-place version of :meth:`~Tensor.bitwise_right_shift` + """ + + @overload + def bitwise_xor(self, other: Tensor) -> Tensor: + r""" + bitwise_xor() -> Tensor + + See :func:`torch.bitwise_xor` + """ + + @overload + def bitwise_xor(self, other: Number | _complex) -> Tensor: + r""" + bitwise_xor() -> Tensor + + See :func:`torch.bitwise_xor` + """ + + @overload + def bitwise_xor_(self, other: Tensor) -> Tensor: + r""" + bitwise_xor_() -> Tensor + + In-place version of :meth:`~Tensor.bitwise_xor` + """ + + @overload + def bitwise_xor_(self, other: Number | _complex) -> Tensor: + r""" + bitwise_xor_() -> Tensor + + In-place version of :meth:`~Tensor.bitwise_xor` + """ + + def bmm(self, mat2: Tensor) -> Tensor: + r""" + bmm(batch2) -> Tensor + + See :func:`torch.bmm` + """ + + def bool(self) -> Tensor: + r""" + bool(memory_format=torch.preserve_format) -> Tensor + + ``self.bool()`` is equivalent to ``self.to(torch.bool)``. See :func:`to`. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + @overload + def broadcast_to(self, size: Sequence[_int | SymInt]) -> Tensor: + r""" + broadcast_to(shape) -> Tensor + + See :func:`torch.broadcast_to`. + """ + + @overload + def broadcast_to(self, *size: _int | SymInt) -> Tensor: + r""" + broadcast_to(shape) -> Tensor + + See :func:`torch.broadcast_to`. + """ + + def byte(self) -> Tensor: + r""" + byte(memory_format=torch.preserve_format) -> Tensor + + ``self.byte()`` is equivalent to ``self.to(torch.uint8)``. See :func:`to`. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + def cauchy_( + self, + median: _float = 0, + sigma: _float = 1, + *, + generator: Generator | None = None, + ) -> Tensor: + r""" + cauchy_(median=0, sigma=1, *, generator=None) -> Tensor + + Fills the tensor with numbers drawn from the Cauchy distribution: + + .. math:: + + f(x) = \dfrac{1}{\pi} \dfrac{\sigma}{(x - \text{median})^2 + \sigma^2} + + .. note:: + Sigma (:math:`\sigma`) is used to denote the scale parameter in Cauchy distribution. + """ + + def ccol_indices(self) -> Tensor: ... + def ceil(self) -> Tensor: + r""" + ceil() -> Tensor + + See :func:`torch.ceil` + """ + + def ceil_(self) -> Tensor: + r""" + ceil_() -> Tensor + + In-place version of :meth:`~Tensor.ceil` + """ + + def chalf(self, *, memory_format: memory_format | None = None) -> Tensor: + r""" + chalf(memory_format=torch.preserve_format) -> Tensor + + ``self.chalf()`` is equivalent to ``self.to(torch.complex32)``. See :func:`to`. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + def char(self) -> Tensor: + r""" + char(memory_format=torch.preserve_format) -> Tensor + + ``self.char()`` is equivalent to ``self.to(torch.int8)``. See :func:`to`. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + def cholesky(self, upper: _bool = False) -> Tensor: + r""" + cholesky(upper=False) -> Tensor + + See :func:`torch.cholesky` + """ + + def cholesky_inverse(self, upper: _bool = False) -> Tensor: + r""" + cholesky_inverse(upper=False) -> Tensor + + See :func:`torch.cholesky_inverse` + """ + + def cholesky_solve(self, input2: Tensor, upper: _bool = False) -> Tensor: + r""" + cholesky_solve(input2, upper=False) -> Tensor + + See :func:`torch.cholesky_solve` + """ + + def chunk(self, chunks: _int, dim: _int = 0) -> tuple[Tensor, ...]: + r""" + chunk(chunks, dim=0) -> List of Tensors + + See :func:`torch.chunk` + """ + + @overload + def clamp( + self, + min: Tensor | None = None, + max: Tensor | None = None, + ) -> Tensor: + r""" + clamp(min=None, max=None) -> Tensor + + See :func:`torch.clamp` + """ + + @overload + def clamp( + self, + min: Number | _complex | None = None, + max: Number | _complex | None = None, + ) -> Tensor: + r""" + clamp(min=None, max=None) -> Tensor + + See :func:`torch.clamp` + """ + + @overload + def clamp_( + self, + min: Tensor | None = None, + max: Tensor | None = None, + ) -> Tensor: + r""" + clamp_(min=None, max=None) -> Tensor + + In-place version of :meth:`~Tensor.clamp` + """ + + @overload + def clamp_( + self, + min: Number | _complex | None = None, + max: Number | _complex | None = None, + ) -> Tensor: + r""" + clamp_(min=None, max=None) -> Tensor + + In-place version of :meth:`~Tensor.clamp` + """ + + @overload + def clamp_max(self, max: Tensor) -> Tensor: ... + @overload + def clamp_max(self, max: Number | _complex) -> Tensor: ... + @overload + def clamp_max_(self, max: Tensor) -> Tensor: ... + @overload + def clamp_max_(self, max: Number | _complex) -> Tensor: ... + @overload + def clamp_min(self, min: Tensor) -> Tensor: ... + @overload + def clamp_min(self, min: Number | _complex) -> Tensor: ... + @overload + def clamp_min_(self, min: Tensor) -> Tensor: ... + @overload + def clamp_min_(self, min: Number | _complex) -> Tensor: ... + @overload + def clip( + self, + min: Tensor | None = None, + max: Tensor | None = None, + ) -> Tensor: + r""" + clip(min=None, max=None) -> Tensor + + Alias for :meth:`~Tensor.clamp`. + """ + + @overload + def clip( + self, + min: Number | _complex | None = None, + max: Number | _complex | None = None, + ) -> Tensor: + r""" + clip(min=None, max=None) -> Tensor + + Alias for :meth:`~Tensor.clamp`. + """ + + @overload + def clip_( + self, + min: Tensor | None = None, + max: Tensor | None = None, + ) -> Tensor: + r""" + clip_(min=None, max=None) -> Tensor + + Alias for :meth:`~Tensor.clamp_`. + """ + + @overload + def clip_( + self, + min: Number | _complex | None = None, + max: Number | _complex | None = None, + ) -> Tensor: + r""" + clip_(min=None, max=None) -> Tensor + + Alias for :meth:`~Tensor.clamp_`. + """ + + def clone(self, *, memory_format: memory_format | None = None) -> Tensor: + r""" + clone(*, memory_format=torch.preserve_format) -> Tensor + + See :func:`torch.clone` + """ + + def coalesce(self) -> Tensor: + r""" + coalesce() -> Tensor + + Returns a coalesced copy of :attr:`self` if :attr:`self` is an + :ref:`uncoalesced tensor `. + + Returns :attr:`self` if :attr:`self` is a coalesced tensor. + + .. warning:: + Throws an error if :attr:`self` is not a sparse COO tensor. + """ + + def col_indices(self) -> Tensor: + r""" + col_indices() -> IntTensor + + Returns the tensor containing the column indices of the :attr:`self` + tensor when :attr:`self` is a sparse CSR tensor of layout ``sparse_csr``. + The ``col_indices`` tensor is strictly of shape (:attr:`self`.nnz()) + and of type ``int32`` or ``int64``. When using MKL routines such as sparse + matrix multiplication, it is necessary to use ``int32`` indexing in order + to avoid downcasting and potentially losing information. + + Example:: + + >>> csr = torch.eye(5,5).to_sparse_csr() + >>> csr.col_indices() + tensor([0, 1, 2, 3, 4], dtype=torch.int32) + """ + + def conj(self) -> Tensor: + r""" + conj() -> Tensor + + See :func:`torch.conj` + """ + + def conj_physical(self) -> Tensor: + r""" + conj_physical() -> Tensor + + See :func:`torch.conj_physical` + """ + + def conj_physical_(self) -> Tensor: + r""" + conj_physical_() -> Tensor + + In-place version of :meth:`~Tensor.conj_physical` + """ + + def contiguous( + self, + memory_format: torch.memory_format = torch.contiguous_format, + ) -> Tensor: + r""" + contiguous(memory_format=torch.contiguous_format) -> Tensor + + Returns a contiguous in memory tensor containing the same data as :attr:`self` tensor. If + :attr:`self` tensor is already in the specified memory format, this function returns the + :attr:`self` tensor. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.contiguous_format``. + """ + + def copy_(self, other: Tensor, non_blocking: _bool = False) -> Tensor: + r""" + copy_(src, non_blocking=False) -> Tensor + + Copies the elements from :attr:`src` into :attr:`self` tensor and returns + :attr:`self`. + + The :attr:`src` tensor must be :ref:`broadcastable ` + with the :attr:`self` tensor. It may be of a different data type or reside on a + different device. + + Args: + src (Tensor): the source tensor to copy from + non_blocking (bool, optional): if ``True`` and this copy is between CPU and GPU, + the copy may occur asynchronously with respect to the host. For other + cases, this argument has no effect. Default: ``False`` + """ + + @overload + def copysign(self, other: Tensor) -> Tensor: + r""" + copysign(other) -> Tensor + + See :func:`torch.copysign` + """ + + @overload + def copysign(self, other: Number | _complex) -> Tensor: + r""" + copysign(other) -> Tensor + + See :func:`torch.copysign` + """ + + @overload + def copysign_(self, other: Tensor) -> Tensor: + r""" + copysign_(other) -> Tensor + + In-place version of :meth:`~Tensor.copysign` + """ + + @overload + def copysign_(self, other: Number | _complex) -> Tensor: + r""" + copysign_(other) -> Tensor + + In-place version of :meth:`~Tensor.copysign` + """ + + def corrcoef(self) -> Tensor: + r""" + corrcoef() -> Tensor + + See :func:`torch.corrcoef` + """ + + def cos(self) -> Tensor: + r""" + cos() -> Tensor + + See :func:`torch.cos` + """ + + def cos_(self) -> Tensor: + r""" + cos_() -> Tensor + + In-place version of :meth:`~Tensor.cos` + """ + + def cosh(self) -> Tensor: + r""" + cosh() -> Tensor + + See :func:`torch.cosh` + """ + + def cosh_(self) -> Tensor: + r""" + cosh_() -> Tensor + + In-place version of :meth:`~Tensor.cosh` + """ + + @overload + def count_nonzero(self, dim: _int | None = None) -> Tensor: + r""" + count_nonzero(dim=None) -> Tensor + + See :func:`torch.count_nonzero` + """ + + @overload + def count_nonzero(self, dim: _size) -> Tensor: + r""" + count_nonzero(dim=None) -> Tensor + + See :func:`torch.count_nonzero` + """ + + @overload + def count_nonzero(self, *dim: _int) -> Tensor: + r""" + count_nonzero(dim=None) -> Tensor + + See :func:`torch.count_nonzero` + """ + + def cov( + self, + *, + correction: _int = 1, + fweights: Tensor | None = None, + aweights: Tensor | None = None, + ) -> Tensor: + r""" + cov(*, correction=1, fweights=None, aweights=None) -> Tensor + + See :func:`torch.cov` + """ + + def cpu( + self, + memory_format: torch.memory_format = torch.preserve_format, + ) -> Tensor: + r""" + cpu(memory_format=torch.preserve_format) -> Tensor + + Returns a copy of this object in CPU memory. + + If this object is already in CPU memory, + then no copy is performed and the original object is returned. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + def cross(self, other: Tensor, dim: _int | None = None) -> Tensor: + r""" + cross(other, dim=None) -> Tensor + + See :func:`torch.cross` + """ + + def crow_indices(self) -> Tensor: + r""" + crow_indices() -> IntTensor + + Returns the tensor containing the compressed row indices of the :attr:`self` + tensor when :attr:`self` is a sparse CSR tensor of layout ``sparse_csr``. + The ``crow_indices`` tensor is strictly of shape (:attr:`self`.size(0) + 1) + and of type ``int32`` or ``int64``. When using MKL routines such as sparse + matrix multiplication, it is necessary to use ``int32`` indexing in order + to avoid downcasting and potentially losing information. + + Example:: + + >>> csr = torch.eye(5,5).to_sparse_csr() + >>> csr.crow_indices() + tensor([0, 1, 2, 3, 4, 5], dtype=torch.int32) + """ + + def cuda( + self, + device: _device | _int | str | None = None, + non_blocking: _bool = False, + memory_format: torch.memory_format = torch.preserve_format, + ) -> Tensor: + r""" + cuda(device=None, non_blocking=False, memory_format=torch.preserve_format) -> Tensor + + Returns a copy of this object in CUDA memory. + + If this object is already in CUDA memory and on the correct device, + then no copy is performed and the original object is returned. + + Args: + device (:class:`torch.device`, optional): The destination GPU device. + Defaults to the current CUDA device. + non_blocking (bool, optional): If ``True`` and the source is in pinned memory, + the copy will be asynchronous with respect to the host. + Otherwise, the argument has no effect. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + @overload + def cummax(self, dim: _int) -> torch.return_types.cummax: + r""" + cummax(dim) -> (Tensor, Tensor) + + See :func:`torch.cummax` + """ + + @overload + def cummax( + self, + dim: str | EllipsisType | None, + ) -> torch.return_types.cummax: + r""" + cummax(dim) -> (Tensor, Tensor) + + See :func:`torch.cummax` + """ + + @overload + def cummin(self, dim: _int) -> torch.return_types.cummin: + r""" + cummin(dim) -> (Tensor, Tensor) + + See :func:`torch.cummin` + """ + + @overload + def cummin( + self, + dim: str | EllipsisType | None, + ) -> torch.return_types.cummin: + r""" + cummin(dim) -> (Tensor, Tensor) + + See :func:`torch.cummin` + """ + + @overload + def cumprod(self, dim: _int, *, dtype: _dtype | None = None) -> Tensor: + r""" + cumprod(dim, dtype=None) -> Tensor + + See :func:`torch.cumprod` + """ + + @overload + def cumprod( + self, + dim: str | EllipsisType | None, + *, + dtype: _dtype | None = None, + ) -> Tensor: + r""" + cumprod(dim, dtype=None) -> Tensor + + See :func:`torch.cumprod` + """ + + @overload + def cumprod_(self, dim: _int, *, dtype: _dtype | None = None) -> Tensor: + r""" + cumprod_(dim, dtype=None) -> Tensor + + In-place version of :meth:`~Tensor.cumprod` + """ + + @overload + def cumprod_( + self, + dim: str | EllipsisType | None, + *, + dtype: _dtype | None = None, + ) -> Tensor: + r""" + cumprod_(dim, dtype=None) -> Tensor + + In-place version of :meth:`~Tensor.cumprod` + """ + + @overload + def cumsum(self, dim: _int, *, dtype: _dtype | None = None) -> Tensor: + r""" + cumsum(dim, dtype=None) -> Tensor + + See :func:`torch.cumsum` + """ + + @overload + def cumsum( + self, + dim: str | EllipsisType | None, + *, + dtype: _dtype | None = None, + ) -> Tensor: + r""" + cumsum(dim, dtype=None) -> Tensor + + See :func:`torch.cumsum` + """ + + @overload + def cumsum_(self, dim: _int, *, dtype: _dtype | None = None) -> Tensor: + r""" + cumsum_(dim, dtype=None) -> Tensor + + In-place version of :meth:`~Tensor.cumsum` + """ + + @overload + def cumsum_( + self, + dim: str | EllipsisType | None, + *, + dtype: _dtype | None = None, + ) -> Tensor: + r""" + cumsum_(dim, dtype=None) -> Tensor + + In-place version of :meth:`~Tensor.cumsum` + """ + + def data_ptr(self) -> _int: + r""" + data_ptr() -> int + + Returns the address of the first element of :attr:`self` tensor. + """ + + def deg2rad(self) -> Tensor: + r""" + deg2rad() -> Tensor + + See :func:`torch.deg2rad` + """ + + def deg2rad_(self) -> Tensor: + r""" + deg2rad_() -> Tensor + + In-place version of :meth:`~Tensor.deg2rad` + """ + + def dense_dim(self) -> _int: + r""" + dense_dim() -> int + + Return the number of dense dimensions in a :ref:`sparse tensor ` :attr:`self`. + + .. note:: + Returns ``len(self.shape)`` if :attr:`self` is not a sparse tensor. + + See also :meth:`Tensor.sparse_dim` and :ref:`hybrid tensors `. + """ + + def dequantize(self) -> Tensor: + r""" + dequantize() -> Tensor + + Given a quantized Tensor, dequantize it and return the dequantized float Tensor. + """ + + def det(self) -> Tensor: + r""" + det() -> Tensor + + See :func:`torch.det` + """ + + def detach(self) -> Tensor: ... + def detach_(self) -> Tensor: ... + def diag(self, diagonal: _int = 0) -> Tensor: + r""" + diag(diagonal=0) -> Tensor + + See :func:`torch.diag` + """ + + def diag_embed( + self, + offset: _int = 0, + dim1: _int = -2, + dim2: _int = -1, + ) -> Tensor: + r""" + diag_embed(offset=0, dim1=-2, dim2=-1) -> Tensor + + See :func:`torch.diag_embed` + """ + + def diagflat(self, offset: _int = 0) -> Tensor: + r""" + diagflat(offset=0) -> Tensor + + See :func:`torch.diagflat` + """ + + @overload + def diagonal( + self, + *, + outdim: str | EllipsisType | None, + dim1: str | EllipsisType | None, + dim2: str | EllipsisType | None, + offset: _int = 0, + ) -> Tensor: + r""" + diagonal(offset=0, dim1=0, dim2=1) -> Tensor + + See :func:`torch.diagonal` + """ + + @overload + def diagonal( + self, + offset: _int = 0, + dim1: _int = 0, + dim2: _int = 1, + ) -> Tensor: + r""" + diagonal(offset=0, dim1=0, dim2=1) -> Tensor + + See :func:`torch.diagonal` + """ + + def diagonal_scatter( + self, + src: Tensor, + offset: _int = 0, + dim1: _int = 0, + dim2: _int = 1, + ) -> Tensor: + r""" + diagonal_scatter(src, offset=0, dim1=0, dim2=1) -> Tensor + + See :func:`torch.diagonal_scatter` + """ + + def diff( + self, + n: _int = 1, + dim: _int = -1, + prepend: Tensor | None = None, + append: Tensor | None = None, + ) -> Tensor: + r""" + diff(n=1, dim=-1, prepend=None, append=None) -> Tensor + + See :func:`torch.diff` + """ + + def digamma(self) -> Tensor: + r""" + digamma() -> Tensor + + See :func:`torch.digamma` + """ + + def digamma_(self) -> Tensor: + r""" + digamma_() -> Tensor + + In-place version of :meth:`~Tensor.digamma` + """ + + def dim(self) -> _int: + r""" + dim() -> int + + Returns the number of dimensions of :attr:`self` tensor. + """ + + def dist(self, other: Tensor, p: Number | _complex = 2) -> Tensor: + r""" + dist(other, p=2) -> Tensor + + See :func:`torch.dist` + """ + + def div( + self, + other: Tensor | Number, + *, + rounding_mode: str | None = None, + ) -> Tensor: + r""" + div(value, *, rounding_mode=None) -> Tensor + + See :func:`torch.div` + """ + + def div_( + self, + other: Tensor | Number, + *, + rounding_mode: str | None = None, + ) -> Tensor: + r""" + div_(value, *, rounding_mode=None) -> Tensor + + In-place version of :meth:`~Tensor.div` + """ + + @overload + def divide(self, other: Tensor) -> Tensor: + r""" + divide(value, *, rounding_mode=None) -> Tensor + + See :func:`torch.divide` + """ + + @overload + def divide(self, other: Tensor, *, rounding_mode: str | None) -> Tensor: + r""" + divide(value, *, rounding_mode=None) -> Tensor + + See :func:`torch.divide` + """ + + @overload + def divide( + self, + other: Number | _complex, + *, + rounding_mode: str | None, + ) -> Tensor: + r""" + divide(value, *, rounding_mode=None) -> Tensor + + See :func:`torch.divide` + """ + + @overload + def divide(self, other: Number | _complex) -> Tensor: + r""" + divide(value, *, rounding_mode=None) -> Tensor + + See :func:`torch.divide` + """ + + @overload + def divide_(self, other: Tensor) -> Tensor: + r""" + divide_(value, *, rounding_mode=None) -> Tensor + + In-place version of :meth:`~Tensor.divide` + """ + + @overload + def divide_(self, other: Tensor, *, rounding_mode: str | None) -> Tensor: + r""" + divide_(value, *, rounding_mode=None) -> Tensor + + In-place version of :meth:`~Tensor.divide` + """ + + @overload + def divide_( + self, + other: Number | _complex, + *, + rounding_mode: str | None, + ) -> Tensor: + r""" + divide_(value, *, rounding_mode=None) -> Tensor + + In-place version of :meth:`~Tensor.divide` + """ + + @overload + def divide_(self, other: Number | _complex) -> Tensor: + r""" + divide_(value, *, rounding_mode=None) -> Tensor + + In-place version of :meth:`~Tensor.divide` + """ + + def dot(self, tensor: Tensor) -> Tensor: + r""" + dot(other) -> Tensor + + See :func:`torch.dot` + """ + + def double(self) -> Tensor: + r""" + double(memory_format=torch.preserve_format) -> Tensor + + ``self.double()`` is equivalent to ``self.to(torch.float64)``. See :func:`to`. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + @overload + def dsplit(self, sections: _int) -> tuple[Tensor, ...]: + r""" + dsplit(split_size_or_sections) -> List of Tensors + + See :func:`torch.dsplit` + """ + + @overload + def dsplit(self, indices: _size) -> tuple[Tensor, ...]: + r""" + dsplit(split_size_or_sections) -> List of Tensors + + See :func:`torch.dsplit` + """ + + @overload + def dsplit(self, *indices: _int) -> tuple[Tensor, ...]: + r""" + dsplit(split_size_or_sections) -> List of Tensors + + See :func:`torch.dsplit` + """ + + def element_size(self) -> _int: + r""" + element_size() -> int + + Returns the size in bytes of an individual element. + + Example:: + + >>> torch.tensor([]).element_size() + 4 + >>> torch.tensor([], dtype=torch.uint8).element_size() + 1 + """ + + @overload + def eq(self, other: Tensor) -> Tensor: + r""" + eq(other) -> Tensor + + See :func:`torch.eq` + """ + + @overload + def eq(self, other: Number | _complex) -> Tensor: + r""" + eq(other) -> Tensor + + See :func:`torch.eq` + """ + + @overload + def eq_(self, other: Tensor) -> Tensor: + r""" + eq_(other) -> Tensor + + In-place version of :meth:`~Tensor.eq` + """ + + @overload + def eq_(self, other: Number | _complex) -> Tensor: + r""" + eq_(other) -> Tensor + + In-place version of :meth:`~Tensor.eq` + """ + + def equal(self, other: Tensor) -> _bool: + r""" + equal(other) -> bool + + See :func:`torch.equal` + """ + + def erf(self) -> Tensor: + r""" + erf() -> Tensor + + See :func:`torch.erf` + """ + + def erf_(self) -> Tensor: + r""" + erf_() -> Tensor + + In-place version of :meth:`~Tensor.erf` + """ + + def erfc(self) -> Tensor: + r""" + erfc() -> Tensor + + See :func:`torch.erfc` + """ + + def erfc_(self) -> Tensor: + r""" + erfc_() -> Tensor + + In-place version of :meth:`~Tensor.erfc` + """ + + def erfinv(self) -> Tensor: + r""" + erfinv() -> Tensor + + See :func:`torch.erfinv` + """ + + def erfinv_(self) -> Tensor: + r""" + erfinv_() -> Tensor + + In-place version of :meth:`~Tensor.erfinv` + """ + + def exp(self) -> Tensor: + r""" + exp() -> Tensor + + See :func:`torch.exp` + """ + + def exp2(self) -> Tensor: + r""" + exp2() -> Tensor + + See :func:`torch.exp2` + """ + + def exp2_(self) -> Tensor: + r""" + exp2_() -> Tensor + + In-place version of :meth:`~Tensor.exp2` + """ + + def exp_(self) -> Tensor: + r""" + exp_() -> Tensor + + In-place version of :meth:`~Tensor.exp` + """ + + @overload + def expand( + self, + size: Sequence[_int | SymInt], + *, + implicit: _bool = False, + ) -> Tensor: + r""" + expand(*sizes) -> Tensor + + Returns a new view of the :attr:`self` tensor with singleton dimensions expanded + to a larger size. + + Passing -1 as the size for a dimension means not changing the size of + that dimension. + + Tensor can be also expanded to a larger number of dimensions, and the + new ones will be appended at the front. For the new dimensions, the + size cannot be set to -1. + + Expanding a tensor does not allocate new memory, but only creates a + new view on the existing tensor where a dimension of size one is + expanded to a larger size by setting the ``stride`` to 0. Any dimension + of size 1 can be expanded to an arbitrary value without allocating new + memory. + + Args: + *sizes (torch.Size or int...): the desired expanded size + + .. warning:: + + More than one element of an expanded tensor may refer to a single + memory location. As a result, in-place operations (especially ones that + are vectorized) may result in incorrect behavior. If you need to write + to the tensors, please clone them first. + + Example:: + + >>> x = torch.tensor([[1], [2], [3]]) + >>> x.size() + torch.Size([3, 1]) + >>> x.expand(3, 4) + tensor([[ 1, 1, 1, 1], + [ 2, 2, 2, 2], + [ 3, 3, 3, 3]]) + >>> x.expand(-1, 4) # -1 means not changing the size of that dimension + tensor([[ 1, 1, 1, 1], + [ 2, 2, 2, 2], + [ 3, 3, 3, 3]]) + """ + + @overload + def expand(self, *size: _int | SymInt, implicit: _bool = False) -> Tensor: + r""" + expand(*sizes) -> Tensor + + Returns a new view of the :attr:`self` tensor with singleton dimensions expanded + to a larger size. + + Passing -1 as the size for a dimension means not changing the size of + that dimension. + + Tensor can be also expanded to a larger number of dimensions, and the + new ones will be appended at the front. For the new dimensions, the + size cannot be set to -1. + + Expanding a tensor does not allocate new memory, but only creates a + new view on the existing tensor where a dimension of size one is + expanded to a larger size by setting the ``stride`` to 0. Any dimension + of size 1 can be expanded to an arbitrary value without allocating new + memory. + + Args: + *sizes (torch.Size or int...): the desired expanded size + + .. warning:: + + More than one element of an expanded tensor may refer to a single + memory location. As a result, in-place operations (especially ones that + are vectorized) may result in incorrect behavior. If you need to write + to the tensors, please clone them first. + + Example:: + + >>> x = torch.tensor([[1], [2], [3]]) + >>> x.size() + torch.Size([3, 1]) + >>> x.expand(3, 4) + tensor([[ 1, 1, 1, 1], + [ 2, 2, 2, 2], + [ 3, 3, 3, 3]]) + >>> x.expand(-1, 4) # -1 means not changing the size of that dimension + tensor([[ 1, 1, 1, 1], + [ 2, 2, 2, 2], + [ 3, 3, 3, 3]]) + """ + + def expand_as(self, other: Tensor) -> Tensor: + r""" + expand_as(other) -> Tensor + + Expand this tensor to the same size as :attr:`other`. + ``self.expand_as(other)`` is equivalent to ``self.expand(other.size())``. + + Please see :meth:`~Tensor.expand` for more information about ``expand``. + + Args: + other (:class:`torch.Tensor`): The result tensor has the same size + as :attr:`other`. + """ + + def expm1(self) -> Tensor: + r""" + expm1() -> Tensor + + See :func:`torch.expm1` + """ + + def expm1_(self) -> Tensor: + r""" + expm1_() -> Tensor + + In-place version of :meth:`~Tensor.expm1` + """ + + def exponential_( + self, + lambd: _float = 1, + *, + generator: Generator | None = None, + ) -> Tensor: + r""" + exponential_(lambd=1, *, generator=None) -> Tensor + + Fills :attr:`self` tensor with elements drawn from the PDF (probability density function): + + .. math:: + + f(x) = \lambda e^{-\lambda x}, x > 0 + + .. note:: + In probability theory, exponential distribution is supported on interval [0, :math:`\inf`) (i.e., :math:`x >= 0`) + implying that zero can be sampled from the exponential distribution. + However, :func:`torch.Tensor.exponential_` does not sample zero, + which means that its actual support is the interval (0, :math:`\inf`). + + Note that :func:`torch.distributions.exponential.Exponential` is supported on the interval [0, :math:`\inf`) and can sample zero. + """ + + @overload + def fill_(self, value: Tensor) -> Tensor: + r""" + fill_(value) -> Tensor + + Fills :attr:`self` tensor with the specified value. + """ + + @overload + def fill_(self, value: Number | _complex) -> Tensor: + r""" + fill_(value) -> Tensor + + Fills :attr:`self` tensor with the specified value. + """ + + def fill_diagonal_( + self, + fill_value: Number | _complex, + wrap: _bool = False, + ) -> Tensor: + r""" + fill_diagonal_(fill_value, wrap=False) -> Tensor + + Fill the main diagonal of a tensor that has at least 2-dimensions. + When dims>2, all dimensions of input must be of equal length. + This function modifies the input tensor in-place, and returns the input tensor. + + Arguments: + fill_value (Scalar): the fill value + wrap (bool, optional): the diagonal 'wrapped' after N columns for tall matrices. Default: ``False`` + + Example:: + + >>> a = torch.zeros(3, 3) + >>> a.fill_diagonal_(5) + tensor([[5., 0., 0.], + [0., 5., 0.], + [0., 0., 5.]]) + >>> b = torch.zeros(7, 3) + >>> b.fill_diagonal_(5) + tensor([[5., 0., 0.], + [0., 5., 0.], + [0., 0., 5.], + [0., 0., 0.], + [0., 0., 0.], + [0., 0., 0.], + [0., 0., 0.]]) + >>> c = torch.zeros(7, 3) + >>> c.fill_diagonal_(5, wrap=True) + tensor([[5., 0., 0.], + [0., 5., 0.], + [0., 0., 5.], + [0., 0., 0.], + [5., 0., 0.], + [0., 5., 0.], + [0., 0., 5.]]) + """ + + def fix(self) -> Tensor: + r""" + fix() -> Tensor + + See :func:`torch.fix`. + """ + + def fix_(self) -> Tensor: + r""" + fix_() -> Tensor + + In-place version of :meth:`~Tensor.fix` + """ + + @overload + def flatten( + self, + start_dim: _int, + end_dim: _int, + out_dim: str | EllipsisType | None, + ) -> Tensor: + r""" + flatten(start_dim=0, end_dim=-1) -> Tensor + + See :func:`torch.flatten` + """ + + @overload + def flatten(self, start_dim: _int = 0, end_dim: _int = -1) -> Tensor: + r""" + flatten(start_dim=0, end_dim=-1) -> Tensor + + See :func:`torch.flatten` + """ + + @overload + def flatten( + self, + start_dim: str | EllipsisType | None, + end_dim: str | EllipsisType | None, + out_dim: str | EllipsisType | None, + ) -> Tensor: + r""" + flatten(start_dim=0, end_dim=-1) -> Tensor + + See :func:`torch.flatten` + """ + + @overload + def flatten( + self, + dims: Sequence[str | EllipsisType | None], + out_dim: str | EllipsisType | None, + ) -> Tensor: + r""" + flatten(start_dim=0, end_dim=-1) -> Tensor + + See :func:`torch.flatten` + """ + + @overload + def flip(self, dims: _size) -> Tensor: + r""" + flip(dims) -> Tensor + + See :func:`torch.flip` + """ + + @overload + def flip(self, *dims: _int) -> Tensor: + r""" + flip(dims) -> Tensor + + See :func:`torch.flip` + """ + + def fliplr(self) -> Tensor: + r""" + fliplr() -> Tensor + + See :func:`torch.fliplr` + """ + + def flipud(self) -> Tensor: + r""" + flipud() -> Tensor + + See :func:`torch.flipud` + """ + + def float(self) -> Tensor: + r""" + float(memory_format=torch.preserve_format) -> Tensor + + ``self.float()`` is equivalent to ``self.to(torch.float32)``. See :func:`to`. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + @overload + def float_power(self, exponent: Tensor) -> Tensor: + r""" + float_power(exponent) -> Tensor + + See :func:`torch.float_power` + """ + + @overload + def float_power(self, exponent: Number | _complex) -> Tensor: + r""" + float_power(exponent) -> Tensor + + See :func:`torch.float_power` + """ + + @overload + def float_power_(self, exponent: Tensor) -> Tensor: + r""" + float_power_(exponent) -> Tensor + + In-place version of :meth:`~Tensor.float_power` + """ + + @overload + def float_power_(self, exponent: Number | _complex) -> Tensor: + r""" + float_power_(exponent) -> Tensor + + In-place version of :meth:`~Tensor.float_power` + """ + + def floor(self) -> Tensor: + r""" + floor() -> Tensor + + See :func:`torch.floor` + """ + + def floor_(self) -> Tensor: + r""" + floor_() -> Tensor + + In-place version of :meth:`~Tensor.floor` + """ + + def floor_divide( + self, + other: Tensor | Number | torch.SymInt | torch.SymFloat, + *, + out: Tensor | None = None, + ) -> Tensor: + r""" + floor_divide(value) -> Tensor + + See :func:`torch.floor_divide` + """ + + def floor_divide_( + self, + other: Tensor | Number | torch.SymInt | torch.SymFloat, + ) -> Tensor: + r""" + floor_divide_(value) -> Tensor + + In-place version of :meth:`~Tensor.floor_divide` + """ + + def fmax(self, other: Tensor) -> Tensor: + r""" + fmax(other) -> Tensor + + See :func:`torch.fmax` + """ + + def fmin(self, other: Tensor) -> Tensor: + r""" + fmin(other) -> Tensor + + See :func:`torch.fmin` + """ + + @overload + def fmod(self, other: Tensor) -> Tensor: + r""" + fmod(divisor) -> Tensor + + See :func:`torch.fmod` + """ + + @overload + def fmod(self, other: Number | _complex) -> Tensor: + r""" + fmod(divisor) -> Tensor + + See :func:`torch.fmod` + """ + + @overload + def fmod_(self, other: Tensor) -> Tensor: + r""" + fmod_(divisor) -> Tensor + + In-place version of :meth:`~Tensor.fmod` + """ + + @overload + def fmod_(self, other: Number | _complex) -> Tensor: + r""" + fmod_(divisor) -> Tensor + + In-place version of :meth:`~Tensor.fmod` + """ + + def frac(self) -> Tensor: + r""" + frac() -> Tensor + + See :func:`torch.frac` + """ + + def frac_(self) -> Tensor: + r""" + frac_() -> Tensor + + In-place version of :meth:`~Tensor.frac` + """ + + def frexp(self) -> torch.return_types.frexp: + r""" + frexp(input) -> (Tensor mantissa, Tensor exponent) + + See :func:`torch.frexp` + """ + + @overload + def gather( + self, + dim: _int, + index: Tensor, + *, + sparse_grad: _bool = False, + ) -> Tensor: + r""" + gather(dim, index) -> Tensor + + See :func:`torch.gather` + """ + + @overload + def gather( + self, + dim: str | EllipsisType | None, + index: Tensor, + *, + sparse_grad: _bool = False, + ) -> Tensor: + r""" + gather(dim, index) -> Tensor + + See :func:`torch.gather` + """ + + def gcd(self, other: Tensor) -> Tensor: + r""" + gcd(other) -> Tensor + + See :func:`torch.gcd` + """ + + def gcd_(self, other: Tensor) -> Tensor: + r""" + gcd_(other) -> Tensor + + In-place version of :meth:`~Tensor.gcd` + """ + + @overload + def ge(self, other: Tensor) -> Tensor: + r""" + ge(other) -> Tensor + + See :func:`torch.ge`. + """ + + @overload + def ge(self, other: Number | _complex) -> Tensor: + r""" + ge(other) -> Tensor + + See :func:`torch.ge`. + """ + + @overload + def ge_(self, other: Tensor) -> Tensor: + r""" + ge_(other) -> Tensor + + In-place version of :meth:`~Tensor.ge`. + """ + + @overload + def ge_(self, other: Number | _complex) -> Tensor: + r""" + ge_(other) -> Tensor + + In-place version of :meth:`~Tensor.ge`. + """ + + def geometric_( + self, + p: _float, + *, + generator: Generator | None = None, + ) -> Tensor: + r""" + geometric_(p, *, generator=None) -> Tensor + + Fills :attr:`self` tensor with elements drawn from the geometric distribution: + + .. math:: + + P(X=k) = (1 - p)^{k - 1} p, k = 1, 2, ... + + .. note:: + :func:`torch.Tensor.geometric_` `k`-th trial is the first success hence draws samples in :math:`\{1, 2, \ldots\}`, whereas + :func:`torch.distributions.geometric.Geometric` :math:`(k+1)`-th trial is the first success + hence draws samples in :math:`\{0, 1, \ldots\}`. + """ + + def geqrf(self) -> torch.return_types.geqrf: + r""" + geqrf() -> (Tensor, Tensor) + + See :func:`torch.geqrf` + """ + + def ger(self, vec2: Tensor) -> Tensor: + r""" + ger(vec2) -> Tensor + + See :func:`torch.ger` + """ + + def get_device(self) -> _int: + r""" + get_device() -> Device ordinal (Integer) + + For CUDA tensors, this function returns the device ordinal of the GPU on which the tensor resides. + For CPU tensors, this function returns `-1`. + + Example:: + + >>> x = torch.randn(3, 4, 5, device='cuda:0') + >>> x.get_device() + 0 + >>> x.cpu().get_device() + -1 + """ + + @overload + def greater(self, other: Tensor) -> Tensor: + r""" + greater(other) -> Tensor + + See :func:`torch.greater`. + """ + + @overload + def greater(self, other: Number | _complex) -> Tensor: + r""" + greater(other) -> Tensor + + See :func:`torch.greater`. + """ + + @overload + def greater_(self, other: Tensor) -> Tensor: + r""" + greater_(other) -> Tensor + + In-place version of :meth:`~Tensor.greater`. + """ + + @overload + def greater_(self, other: Number | _complex) -> Tensor: + r""" + greater_(other) -> Tensor + + In-place version of :meth:`~Tensor.greater`. + """ + + @overload + def greater_equal(self, other: Tensor) -> Tensor: + r""" + greater_equal(other) -> Tensor + + See :func:`torch.greater_equal`. + """ + + @overload + def greater_equal(self, other: Number | _complex) -> Tensor: + r""" + greater_equal(other) -> Tensor + + See :func:`torch.greater_equal`. + """ + + @overload + def greater_equal_(self, other: Tensor) -> Tensor: + r""" + greater_equal_(other) -> Tensor + + In-place version of :meth:`~Tensor.greater_equal`. + """ + + @overload + def greater_equal_(self, other: Number | _complex) -> Tensor: + r""" + greater_equal_(other) -> Tensor + + In-place version of :meth:`~Tensor.greater_equal`. + """ + + @overload + def gt(self, other: Tensor) -> Tensor: + r""" + gt(other) -> Tensor + + See :func:`torch.gt`. + """ + + @overload + def gt(self, other: Number | _complex) -> Tensor: + r""" + gt(other) -> Tensor + + See :func:`torch.gt`. + """ + + @overload + def gt_(self, other: Tensor) -> Tensor: + r""" + gt_(other) -> Tensor + + In-place version of :meth:`~Tensor.gt`. + """ + + @overload + def gt_(self, other: Number | _complex) -> Tensor: + r""" + gt_(other) -> Tensor + + In-place version of :meth:`~Tensor.gt`. + """ + + def half(self) -> Tensor: + r""" + half(memory_format=torch.preserve_format) -> Tensor + + ``self.half()`` is equivalent to ``self.to(torch.float16)``. See :func:`to`. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + def hardshrink(self, lambd: Number | _complex = 0.5) -> Tensor: + r""" + hardshrink(lambd=0.5) -> Tensor + + See :func:`torch.nn.functional.hardshrink` + """ + + def has_names(self) -> _bool: + r""" + Is ``True`` if any of this tensor's dimensions are named. Otherwise, is ``False``. + """ + + @overload + def hash_tensor( + self, + dim: _int | _size = (), + *, + keepdim: _bool = False, + mode: _int = 0, + ) -> Tensor: ... + @overload + def hash_tensor( + self, + *dim: _int, + keepdim: _bool = False, + mode: _int = 0, + ) -> Tensor: ... + def heaviside(self, values: Tensor) -> Tensor: + r""" + heaviside(values) -> Tensor + + See :func:`torch.heaviside` + """ + + def heaviside_(self, values: Tensor) -> Tensor: + r""" + heaviside_(values) -> Tensor + + In-place version of :meth:`~Tensor.heaviside` + """ + + def histc( + self, + bins: _int = 100, + min: Number | _complex = 0, + max: Number | _complex = 0, + ) -> Tensor: + r""" + histc(bins=100, min=0, max=0) -> Tensor + + See :func:`torch.histc` + """ + + @overload + def histogram( + self, + bins: Tensor, + *, + weight: Tensor | None = None, + density: _bool = False, + ) -> torch.return_types.histogram: + r""" + histogram(input, bins, *, range=None, weight=None, density=False) -> (Tensor, Tensor) + + See :func:`torch.histogram` + """ + + @overload + def histogram( + self, + bins: _int = 100, + *, + range: Sequence[_float] | None = None, + weight: Tensor | None = None, + density: _bool = False, + ) -> torch.return_types.histogram: + r""" + histogram(input, bins, *, range=None, weight=None, density=False) -> (Tensor, Tensor) + + See :func:`torch.histogram` + """ + + @overload + def hsplit(self, sections: _int) -> tuple[Tensor, ...]: + r""" + hsplit(split_size_or_sections) -> List of Tensors + + See :func:`torch.hsplit` + """ + + @overload + def hsplit(self, indices: _size) -> tuple[Tensor, ...]: + r""" + hsplit(split_size_or_sections) -> List of Tensors + + See :func:`torch.hsplit` + """ + + @overload + def hsplit(self, *indices: _int) -> tuple[Tensor, ...]: + r""" + hsplit(split_size_or_sections) -> List of Tensors + + See :func:`torch.hsplit` + """ + + def hypot(self, other: Tensor) -> Tensor: + r""" + hypot(other) -> Tensor + + See :func:`torch.hypot` + """ + + def hypot_(self, other: Tensor) -> Tensor: + r""" + hypot_(other) -> Tensor + + In-place version of :meth:`~Tensor.hypot` + """ + + def i0(self) -> Tensor: + r""" + i0() -> Tensor + + See :func:`torch.i0` + """ + + def i0_(self) -> Tensor: + r""" + i0_() -> Tensor + + In-place version of :meth:`~Tensor.i0` + """ + + def igamma(self, other: Tensor) -> Tensor: + r""" + igamma(other) -> Tensor + + See :func:`torch.igamma` + """ + + def igamma_(self, other: Tensor) -> Tensor: + r""" + igamma_(other) -> Tensor + + In-place version of :meth:`~Tensor.igamma` + """ + + def igammac(self, other: Tensor) -> Tensor: + r""" + igammac(other) -> Tensor + See :func:`torch.igammac` + """ + + def igammac_(self, other: Tensor) -> Tensor: + r""" + igammac_(other) -> Tensor + In-place version of :meth:`~Tensor.igammac` + """ + + @overload + def index_add( + self, + dim: _int, + index: Tensor, + source: Tensor, + *, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + index_add(dim, index, source, *, alpha=1) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.index_add_`. + """ + + @overload + def index_add( + self, + dim: str | EllipsisType | None, + index: Tensor, + source: Tensor, + *, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + index_add(dim, index, source, *, alpha=1) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.index_add_`. + """ + + def index_add_( + self, + dim: _int, + index: Tensor, + source: Tensor, + *, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + index_add_(dim, index, source, *, alpha=1) -> Tensor + + Accumulate the elements of :attr:`alpha` times ``source`` into the :attr:`self` + tensor by adding to the indices in the order given in :attr:`index`. For example, + if ``dim == 0``, ``index[i] == j``, and ``alpha=-1``, then the ``i``\ th row of + ``source`` is subtracted from the ``j``\ th row of :attr:`self`. + + The :attr:`dim`\ th dimension of ``source`` must have the same size as the + length of :attr:`index` (which must be a vector), and all other dimensions must + match :attr:`self`, or an error will be raised. + + For a 3-D tensor the output is given as:: + + self[index[i], :, :] += alpha * src[i, :, :] # if dim == 0 + self[:, index[i], :] += alpha * src[:, i, :] # if dim == 1 + self[:, :, index[i]] += alpha * src[:, :, i] # if dim == 2 + + Note: + This operation may behave nondeterministically when given tensors on a CUDA device. See :doc:`/notes/randomness` for more information. + + Args: + dim (int): dimension along which to index + index (Tensor): indices of ``source`` to select from, + should have dtype either `torch.int64` or `torch.int32` + source (Tensor): the tensor containing values to add + + Keyword args: + alpha (Number): the scalar multiplier for ``source`` + + Example:: + + >>> x = torch.ones(5, 3) + >>> t = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=torch.float) + >>> index = torch.tensor([0, 4, 2]) + >>> x.index_add_(0, index, t) + tensor([[ 2., 3., 4.], + [ 1., 1., 1.], + [ 8., 9., 10.], + [ 1., 1., 1.], + [ 5., 6., 7.]]) + >>> x.index_add_(0, index, t, alpha=-1) + tensor([[ 1., 1., 1.], + [ 1., 1., 1.], + [ 1., 1., 1.], + [ 1., 1., 1.], + [ 1., 1., 1.]]) + """ + + @overload + def index_copy(self, dim: _int, index: Tensor, source: Tensor) -> Tensor: + r""" + index_copy(dim, index, tensor2) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.index_copy_`. + """ + + @overload + def index_copy( + self, + dim: str | EllipsisType | None, + index: Tensor, + source: Tensor, + ) -> Tensor: + r""" + index_copy(dim, index, tensor2) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.index_copy_`. + """ + + @overload + def index_copy_(self, dim: _int, index: Tensor, source: Tensor) -> Tensor: + r""" + index_copy_(dim, index, tensor) -> Tensor + + Copies the elements of :attr:`tensor` into the :attr:`self` tensor by selecting + the indices in the order given in :attr:`index`. For example, if ``dim == 0`` + and ``index[i] == j``, then the ``i``\ th row of :attr:`tensor` is copied to the + ``j``\ th row of :attr:`self`. + + The :attr:`dim`\ th dimension of :attr:`tensor` must have the same size as the + length of :attr:`index` (which must be a vector), and all other dimensions must + match :attr:`self`, or an error will be raised. + + .. note:: + If :attr:`index` contains duplicate entries, multiple elements from + :attr:`tensor` will be copied to the same index of :attr:`self`. The result + is nondeterministic since it depends on which copy occurs last. + + Args: + dim (int): dimension along which to index + index (LongTensor): indices of :attr:`tensor` to select from + tensor (Tensor): the tensor containing values to copy + + Example:: + + >>> x = torch.zeros(5, 3) + >>> t = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=torch.float) + >>> index = torch.tensor([0, 4, 2]) + >>> x.index_copy_(0, index, t) + tensor([[ 1., 2., 3.], + [ 0., 0., 0.], + [ 7., 8., 9.], + [ 0., 0., 0.], + [ 4., 5., 6.]]) + """ + + @overload + def index_copy_( + self, + dim: str | EllipsisType | None, + index: Tensor, + source: Tensor, + ) -> Tensor: + r""" + index_copy_(dim, index, tensor) -> Tensor + + Copies the elements of :attr:`tensor` into the :attr:`self` tensor by selecting + the indices in the order given in :attr:`index`. For example, if ``dim == 0`` + and ``index[i] == j``, then the ``i``\ th row of :attr:`tensor` is copied to the + ``j``\ th row of :attr:`self`. + + The :attr:`dim`\ th dimension of :attr:`tensor` must have the same size as the + length of :attr:`index` (which must be a vector), and all other dimensions must + match :attr:`self`, or an error will be raised. + + .. note:: + If :attr:`index` contains duplicate entries, multiple elements from + :attr:`tensor` will be copied to the same index of :attr:`self`. The result + is nondeterministic since it depends on which copy occurs last. + + Args: + dim (int): dimension along which to index + index (LongTensor): indices of :attr:`tensor` to select from + tensor (Tensor): the tensor containing values to copy + + Example:: + + >>> x = torch.zeros(5, 3) + >>> t = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=torch.float) + >>> index = torch.tensor([0, 4, 2]) + >>> x.index_copy_(0, index, t) + tensor([[ 1., 2., 3.], + [ 0., 0., 0.], + [ 7., 8., 9.], + [ 0., 0., 0.], + [ 4., 5., 6.]]) + """ + + @overload + def index_fill(self, dim: _int, index: Tensor, value: Tensor) -> Tensor: + r""" + index_fill(dim, index, value) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.index_fill_`. + """ + + @overload + def index_fill( + self, + dim: str | EllipsisType | None, + index: Tensor, + value: Tensor, + ) -> Tensor: + r""" + index_fill(dim, index, value) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.index_fill_`. + """ + + @overload + def index_fill( + self, + dim: _int, + index: Tensor, + value: Number | _complex, + ) -> Tensor: + r""" + index_fill(dim, index, value) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.index_fill_`. + """ + + @overload + def index_fill( + self, + dim: str | EllipsisType | None, + index: Tensor, + value: Number | _complex, + ) -> Tensor: + r""" + index_fill(dim, index, value) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.index_fill_`. + """ + + @overload + def index_fill_(self, dim: _int, index: Tensor, value: Tensor) -> Tensor: + r""" + index_fill_(dim, index, value) -> Tensor + + Fills the elements of the :attr:`self` tensor with value :attr:`value` by + selecting the indices in the order given in :attr:`index`. + + Args: + dim (int): dimension along which to index + index (LongTensor): indices of :attr:`self` tensor to fill in + value (float): the value to fill with + + Example:: + + >>> x = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=torch.float) + >>> index = torch.tensor([0, 2]) + >>> x.index_fill_(1, index, -1) + tensor([[-1., 2., -1.], + [-1., 5., -1.], + [-1., 8., -1.]]) + """ + + @overload + def index_fill_( + self, + dim: str | EllipsisType | None, + index: Tensor, + value: Tensor, + ) -> Tensor: + r""" + index_fill_(dim, index, value) -> Tensor + + Fills the elements of the :attr:`self` tensor with value :attr:`value` by + selecting the indices in the order given in :attr:`index`. + + Args: + dim (int): dimension along which to index + index (LongTensor): indices of :attr:`self` tensor to fill in + value (float): the value to fill with + + Example:: + + >>> x = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=torch.float) + >>> index = torch.tensor([0, 2]) + >>> x.index_fill_(1, index, -1) + tensor([[-1., 2., -1.], + [-1., 5., -1.], + [-1., 8., -1.]]) + """ + + @overload + def index_fill_( + self, + dim: _int, + index: Tensor, + value: Number | _complex, + ) -> Tensor: + r""" + index_fill_(dim, index, value) -> Tensor + + Fills the elements of the :attr:`self` tensor with value :attr:`value` by + selecting the indices in the order given in :attr:`index`. + + Args: + dim (int): dimension along which to index + index (LongTensor): indices of :attr:`self` tensor to fill in + value (float): the value to fill with + + Example:: + + >>> x = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=torch.float) + >>> index = torch.tensor([0, 2]) + >>> x.index_fill_(1, index, -1) + tensor([[-1., 2., -1.], + [-1., 5., -1.], + [-1., 8., -1.]]) + """ + + @overload + def index_fill_( + self, + dim: str | EllipsisType | None, + index: Tensor, + value: Number | _complex, + ) -> Tensor: + r""" + index_fill_(dim, index, value) -> Tensor + + Fills the elements of the :attr:`self` tensor with value :attr:`value` by + selecting the indices in the order given in :attr:`index`. + + Args: + dim (int): dimension along which to index + index (LongTensor): indices of :attr:`self` tensor to fill in + value (float): the value to fill with + + Example:: + + >>> x = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9]], dtype=torch.float) + >>> index = torch.tensor([0, 2]) + >>> x.index_fill_(1, index, -1) + tensor([[-1., 2., -1.], + [-1., 5., -1.], + [-1., 8., -1.]]) + """ + + def index_put( + self, + indices: tuple[Tensor, ...] | list[Tensor] | None, + values: Tensor, + accumulate: _bool = False, + ) -> Tensor: + r""" + index_put(indices, values, accumulate=False) -> Tensor + + Out-place version of :meth:`~Tensor.index_put_`. + """ + + def index_put_( + self, + indices: tuple[Tensor, ...] | list[Tensor] | None, + values: Tensor, + accumulate: _bool = False, + ) -> Tensor: + r""" + index_put_(indices, values, accumulate=False) -> Tensor + + Puts values from the tensor :attr:`values` into the tensor :attr:`self` using + the indices specified in :attr:`indices` (which is a tuple of Tensors). The + expression ``tensor.index_put_(indices, values)`` is equivalent to + ``tensor[indices] = values``. Returns :attr:`self`. + + If :attr:`accumulate` is ``True``, the elements in :attr:`values` are added to + :attr:`self`. If accumulate is ``False``, the behavior is undefined if indices + contain duplicate elements. + + Args: + indices (tuple of LongTensor): tensors used to index into `self`. + values (Tensor): tensor of same dtype as `self`. + accumulate (bool): whether to accumulate into self + """ + + def index_reduce( + self, + dim: _int, + index: Tensor, + source: Tensor, + reduce: str, + *, + include_self: _bool = True, + ) -> Tensor: ... + def index_reduce_( + self, + dim: _int, + index: Tensor, + source: Tensor, + reduce: str, + *, + include_self: _bool = True, + ) -> Tensor: + r""" + index_reduce_(dim, index, source, reduce, *, include_self=True) -> Tensor + + Accumulate the elements of ``source`` into the :attr:`self` + tensor by accumulating to the indices in the order given in :attr:`index` + using the reduction given by the ``reduce`` argument. For example, if ``dim == 0``, + ``index[i] == j``, ``reduce == prod`` and ``include_self == True`` then the ``i``\ th + row of ``source`` is multiplied by the ``j``\ th row of :attr:`self`. If + :obj:`include_self="True"`, the values in the :attr:`self` tensor are included + in the reduction, otherwise, rows in the :attr:`self` tensor that are accumulated + to are treated as if they were filled with the reduction identities. + + The :attr:`dim`\ th dimension of ``source`` must have the same size as the + length of :attr:`index` (which must be a vector), and all other dimensions must + match :attr:`self`, or an error will be raised. + + For a 3-D tensor with :obj:`reduce="prod"` and :obj:`include_self=True` the + output is given as:: + + self[index[i], :, :] *= src[i, :, :] # if dim == 0 + self[:, index[i], :] *= src[:, i, :] # if dim == 1 + self[:, :, index[i]] *= src[:, :, i] # if dim == 2 + + Note: + This operation may behave nondeterministically when given tensors on a CUDA device. See :doc:`/notes/randomness` for more information. + + .. note:: + + This function only supports floating point tensors. + + .. warning:: + + This function is in beta and may change in the near future. + + Args: + dim (int): dimension along which to index + index (Tensor): indices of ``source`` to select from, + should have dtype either `torch.int64` or `torch.int32` + source (FloatTensor): the tensor containing values to accumulate + reduce (str): the reduction operation to apply + (:obj:`"prod"`, :obj:`"mean"`, :obj:`"amax"`, :obj:`"amin"`) + + Keyword args: + include_self (bool): whether the elements from the ``self`` tensor are + included in the reduction + + Example:: + + >>> x = torch.empty(5, 3).fill_(2) + >>> t = torch.tensor([[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], dtype=torch.float) + >>> index = torch.tensor([0, 4, 2, 0]) + >>> x.index_reduce_(0, index, t, 'prod') + tensor([[20., 44., 72.], + [ 2., 2., 2.], + [14., 16., 18.], + [ 2., 2., 2.], + [ 8., 10., 12.]]) + >>> x = torch.empty(5, 3).fill_(2) + >>> x.index_reduce_(0, index, t, 'prod', include_self=False) + tensor([[10., 22., 36.], + [ 2., 2., 2.], + [ 7., 8., 9.], + [ 2., 2., 2.], + [ 4., 5., 6.]]) + """ + + @overload + def index_select(self, dim: _int, index: Tensor) -> Tensor: + r""" + index_select(dim, index) -> Tensor + + See :func:`torch.index_select` + """ + + @overload + def index_select( + self, + dim: str | EllipsisType | None, + index: Tensor, + ) -> Tensor: + r""" + index_select(dim, index) -> Tensor + + See :func:`torch.index_select` + """ + + def indices(self) -> Tensor: + r""" + indices() -> Tensor + + Return the indices tensor of a :ref:`sparse COO tensor `. + + .. warning:: + Throws an error if :attr:`self` is not a sparse COO tensor. + + See also :meth:`Tensor.values`. + + .. note:: + This method can only be called on a coalesced sparse tensor. See + :meth:`Tensor.coalesce` for details. + """ + + def inner(self, other: Tensor) -> Tensor: + r""" + inner(other) -> Tensor + + See :func:`torch.inner`. + """ + + def int(self) -> Tensor: + r""" + int(memory_format=torch.preserve_format) -> Tensor + + ``self.int()`` is equivalent to ``self.to(torch.int32)``. See :func:`to`. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + def int_repr(self) -> Tensor: + r""" + int_repr() -> Tensor + + Given a quantized Tensor, + ``self.int_repr()`` returns a CPU Tensor with uint8_t as data type that stores the + underlying uint8_t values of the given Tensor. + """ + + def inverse(self) -> Tensor: + r""" + inverse() -> Tensor + + See :func:`torch.inverse` + """ + + def is_coalesced(self) -> _bool: + r""" + is_coalesced() -> bool + + Returns ``True`` if :attr:`self` is a :ref:`sparse COO tensor + ` that is coalesced, ``False`` otherwise. + + .. warning:: + Throws an error if :attr:`self` is not a sparse COO tensor. + + See :meth:`coalesce` and :ref:`uncoalesced tensors `. + """ + + def is_complex(self) -> _bool: + r""" + is_complex() -> bool + + Returns True if the data type of :attr:`self` is a complex data type. + """ + + def is_conj(self) -> _bool: + r""" + is_conj() -> bool + + Returns True if the conjugate bit of :attr:`self` is set to true. + """ + + def is_contiguous( + self, + memory_format: torch.memory_format = torch.contiguous_format, + ) -> _bool: + r""" + is_contiguous(memory_format=torch.contiguous_format) -> bool + + Returns True if :attr:`self` tensor is contiguous in memory in the order specified + by memory format. + + Args: + memory_format (:class:`torch.memory_format`, optional): Specifies memory allocation + order. Default: ``torch.contiguous_format``. + """ + is_cpu: _bool + r"""Is ``True`` if the Tensor is stored on the CPU, ``False`` otherwise.""" + is_cuda: _bool + r"""Is ``True`` if the Tensor is stored on the GPU, ``False`` otherwise.""" + + def is_distributed(self) -> _bool: ... + def is_floating_point(self) -> _bool: + r""" + is_floating_point() -> bool + + Returns True if the data type of :attr:`self` is a floating point data type. + """ + + def is_inference(self) -> _bool: + r""" + is_inference() -> bool + + See :func:`torch.is_inference` + """ + is_ipu: _bool + r"""Is ``True`` if the Tensor is stored on the IPU, ``False`` otherwise.""" + is_leaf: _bool + r"""All Tensors that have :attr:`requires_grad` which is ``False`` will be leaf Tensors by convention. + + For Tensors that have :attr:`requires_grad` which is ``True``, they will be leaf Tensors if they were + created by the user. This means that they are not the result of an operation and so + :attr:`grad_fn` is None. + + Only leaf Tensors will have their :attr:`grad` populated during a call to :func:`backward`. + To get :attr:`grad` populated for non-leaf Tensors, you can use :func:`retain_grad`. + + Example:: + + >>> a = torch.rand(10, requires_grad=True) + >>> a.is_leaf + True + >>> b = torch.rand(10, requires_grad=True).cuda() + >>> b.is_leaf + False + # b was created by the operation that cast a cpu Tensor into a cuda Tensor + >>> c = torch.rand(10, requires_grad=True) + 2 + >>> c.is_leaf + False + # c was created by the addition operation + >>> d = torch.rand(10).cuda() + >>> d.is_leaf + True + # d does not require gradients and so has no operation creating it (that is tracked by the autograd engine) + >>> e = torch.rand(10).cuda().requires_grad_() + >>> e.is_leaf + True + # e requires gradients and has no operations creating it + >>> f = torch.rand(10, requires_grad=True, device="cuda") + >>> f.is_leaf + True + # f requires grad, has no operation creating it""" + is_maia: _bool + is_meta: _bool + r"""Is ``True`` if the Tensor is a meta tensor, ``False`` otherwise. Meta tensors + are like normal tensors, but they carry no data.""" + is_mkldnn: _bool + is_mps: _bool + r"""Is ``True`` if the Tensor is stored on the MPS device, ``False`` otherwise.""" + is_mtia: _bool + def is_neg(self) -> _bool: + r""" + is_neg() -> bool + + Returns True if the negative bit of :attr:`self` is set to true. + """ + is_nested: _bool + def is_nonzero(self) -> _bool: ... + def is_pinned(self, device: DeviceLikeType | None = None) -> _bool: + r""" + Returns true if this tensor resides in pinned memory. + By default, the device pinned memory on will be the current :ref:`accelerator`. + """ + is_quantized: _bool + r"""Is ``True`` if the Tensor is quantized, ``False`` otherwise.""" + + def is_same_size(self, other: Tensor) -> _bool: ... + def is_set_to(self, tensor: Tensor) -> _bool: + r""" + is_set_to(tensor) -> bool + + Returns True if both tensors are pointing to the exact same memory (same + storage, offset, size and stride). + """ + + def is_signed(self) -> _bool: + r""" + is_signed() -> bool + + Returns True if the data type of :attr:`self` is a signed data type. + """ + is_sparse: _bool + r"""Is ``True`` if the Tensor uses sparse COO storage layout, ``False`` otherwise.""" + is_sparse_csr: _bool + r"""Is ``True`` if the Tensor uses sparse CSR storage layout, ``False`` otherwise.""" + is_vulkan: _bool + is_xpu: _bool + r"""Is ``True`` if the Tensor is stored on the XPU, ``False`` otherwise.""" + + def isclose( + self, + other: Tensor, + rtol: _float = 1e-05, + atol: _float = 1e-08, + equal_nan: _bool = False, + ) -> Tensor: + r""" + isclose(other, rtol=1e-05, atol=1e-08, equal_nan=False) -> Tensor + + See :func:`torch.isclose` + """ + + def isfinite(self) -> Tensor: + r""" + isfinite() -> Tensor + + See :func:`torch.isfinite` + """ + + def isinf(self) -> Tensor: + r""" + isinf() -> Tensor + + See :func:`torch.isinf` + """ + + def isnan(self) -> Tensor: + r""" + isnan() -> Tensor + + See :func:`torch.isnan` + """ + + def isneginf(self) -> Tensor: + r""" + isneginf() -> Tensor + + See :func:`torch.isneginf` + """ + + def isposinf(self) -> Tensor: + r""" + isposinf() -> Tensor + + See :func:`torch.isposinf` + """ + + def isreal(self) -> Tensor: + r""" + isreal() -> Tensor + + See :func:`torch.isreal` + """ + + def istft( + self, + n_fft: _int, + hop_length: _int | None = None, + win_length: _int | None = None, + window: Tensor | None = None, + center: _bool = True, + normalized: _bool = False, + onesided: _bool | None = None, + length: _int | None = None, + return_complex: _bool = False, + ) -> Tensor: + r""" + istft(n_fft, hop_length=None, win_length=None, window=None, + center=True, normalized=False, onesided=True, length=None) -> Tensor + + See :func:`torch.istft` + """ + + def item(self) -> Number: + r""" + item() -> number + + Returns the value of this tensor as a standard Python number. This only works + for tensors with one element. For other cases, see :meth:`~Tensor.tolist`. + + This operation is not differentiable. + + Example:: + + >>> x = torch.tensor([1.0]) + >>> x.item() + 1.0 + """ + + def kron(self, other: Tensor) -> Tensor: + r""" + kron(other) -> Tensor + + See :func:`torch.kron` + """ + + @overload + def kthvalue( + self, + k: _int | SymInt, + dim: _int = -1, + keepdim: _bool = False, + ) -> torch.return_types.kthvalue: + r""" + kthvalue(k, dim=None, keepdim=False) -> (Tensor, LongTensor) + + See :func:`torch.kthvalue` + """ + + @overload + def kthvalue( + self, + k: _int | SymInt, + dim: str | EllipsisType | None, + keepdim: _bool = False, + ) -> torch.return_types.kthvalue: + r""" + kthvalue(k, dim=None, keepdim=False) -> (Tensor, LongTensor) + + See :func:`torch.kthvalue` + """ + + def lcm(self, other: Tensor) -> Tensor: + r""" + lcm(other) -> Tensor + + See :func:`torch.lcm` + """ + + def lcm_(self, other: Tensor) -> Tensor: + r""" + lcm_(other) -> Tensor + + In-place version of :meth:`~Tensor.lcm` + """ + + def ldexp(self, other: Tensor) -> Tensor: + r""" + ldexp(other) -> Tensor + + See :func:`torch.ldexp` + """ + + def ldexp_(self, other: Tensor) -> Tensor: + r""" + ldexp_(other) -> Tensor + + In-place version of :meth:`~Tensor.ldexp` + """ + + @overload + def le(self, other: Tensor) -> Tensor: + r""" + le(other) -> Tensor + + See :func:`torch.le`. + """ + + @overload + def le(self, other: Number | _complex) -> Tensor: + r""" + le(other) -> Tensor + + See :func:`torch.le`. + """ + + @overload + def le_(self, other: Tensor) -> Tensor: + r""" + le_(other) -> Tensor + + In-place version of :meth:`~Tensor.le`. + """ + + @overload + def le_(self, other: Number | _complex) -> Tensor: + r""" + le_(other) -> Tensor + + In-place version of :meth:`~Tensor.le`. + """ + + @overload + def lerp(self, end: Tensor, weight: Tensor) -> Tensor: + r""" + lerp(end, weight) -> Tensor + + See :func:`torch.lerp` + """ + + @overload + def lerp(self, end: Tensor, weight: Number | _complex) -> Tensor: + r""" + lerp(end, weight) -> Tensor + + See :func:`torch.lerp` + """ + + @overload + def lerp_(self, end: Tensor, weight: Tensor) -> Tensor: + r""" + lerp_(end, weight) -> Tensor + + In-place version of :meth:`~Tensor.lerp` + """ + + @overload + def lerp_(self, end: Tensor, weight: Number | _complex) -> Tensor: + r""" + lerp_(end, weight) -> Tensor + + In-place version of :meth:`~Tensor.lerp` + """ + + @overload + def less(self, other: Tensor) -> Tensor: + r""" + lt(other) -> Tensor + + See :func:`torch.less`. + """ + + @overload + def less(self, other: Number | _complex) -> Tensor: + r""" + lt(other) -> Tensor + + See :func:`torch.less`. + """ + + @overload + def less_(self, other: Tensor) -> Tensor: + r""" + less_(other) -> Tensor + + In-place version of :meth:`~Tensor.less`. + """ + + @overload + def less_(self, other: Number | _complex) -> Tensor: + r""" + less_(other) -> Tensor + + In-place version of :meth:`~Tensor.less`. + """ + + @overload + def less_equal(self, other: Tensor) -> Tensor: + r""" + less_equal(other) -> Tensor + + See :func:`torch.less_equal`. + """ + + @overload + def less_equal(self, other: Number | _complex) -> Tensor: + r""" + less_equal(other) -> Tensor + + See :func:`torch.less_equal`. + """ + + @overload + def less_equal_(self, other: Tensor) -> Tensor: + r""" + less_equal_(other) -> Tensor + + In-place version of :meth:`~Tensor.less_equal`. + """ + + @overload + def less_equal_(self, other: Number | _complex) -> Tensor: + r""" + less_equal_(other) -> Tensor + + In-place version of :meth:`~Tensor.less_equal`. + """ + + def lgamma(self) -> Tensor: + r""" + lgamma() -> Tensor + + See :func:`torch.lgamma` + """ + + def lgamma_(self) -> Tensor: + r""" + lgamma_() -> Tensor + + In-place version of :meth:`~Tensor.lgamma` + """ + + def log(self) -> Tensor: + r""" + log() -> Tensor + + See :func:`torch.log` + """ + + def log10(self) -> Tensor: + r""" + log10() -> Tensor + + See :func:`torch.log10` + """ + + def log10_(self) -> Tensor: + r""" + log10_() -> Tensor + + In-place version of :meth:`~Tensor.log10` + """ + + def log1p(self) -> Tensor: + r""" + log1p() -> Tensor + + See :func:`torch.log1p` + """ + + def log1p_(self) -> Tensor: + r""" + log1p_() -> Tensor + + In-place version of :meth:`~Tensor.log1p` + """ + + def log2(self) -> Tensor: + r""" + log2() -> Tensor + + See :func:`torch.log2` + """ + + def log2_(self) -> Tensor: + r""" + log2_() -> Tensor + + In-place version of :meth:`~Tensor.log2` + """ + + def log_(self) -> Tensor: + r""" + log_() -> Tensor + + In-place version of :meth:`~Tensor.log` + """ + + def log_normal_( + self, + mean: _float = 1, + std: _float = 2, + *, + generator: Generator | None = None, + ) -> Tensor: + r""" + log_normal_(mean=1, std=2, *, generator=None) + + Fills :attr:`self` tensor with numbers samples from the log-normal distribution + parameterized by the given mean :math:`\mu` and standard deviation + :math:`\sigma`. Note that :attr:`mean` and :attr:`std` are the mean and + standard deviation of the underlying normal distribution, and not of the + returned distribution: + + .. math:: + + f(x) = \dfrac{1}{x \sigma \sqrt{2\pi}}\ e^{-\frac{(\ln x - \mu)^2}{2\sigma^2}} + """ + + @overload + def log_softmax(self, dim: _int, dtype: _dtype | None = None) -> Tensor: ... + @overload + def log_softmax( + self, + dim: str | EllipsisType | None, + *, + dtype: _dtype | None = None, + ) -> Tensor: ... + def logaddexp(self, other: Tensor) -> Tensor: + r""" + logaddexp(other) -> Tensor + + See :func:`torch.logaddexp` + """ + + def logaddexp2(self, other: Tensor) -> Tensor: + r""" + logaddexp2(other) -> Tensor + + See :func:`torch.logaddexp2` + """ + + @overload + def logcumsumexp(self, dim: _int) -> Tensor: + r""" + logcumsumexp(dim) -> Tensor + + See :func:`torch.logcumsumexp` + """ + + @overload + def logcumsumexp(self, dim: str | EllipsisType | None) -> Tensor: + r""" + logcumsumexp(dim) -> Tensor + + See :func:`torch.logcumsumexp` + """ + + def logdet(self) -> Tensor: + r""" + logdet() -> Tensor + + See :func:`torch.logdet` + """ + + def logical_and(self, other: Tensor) -> Tensor: + r""" + logical_and() -> Tensor + + See :func:`torch.logical_and` + """ + + def logical_and_(self, other: Tensor) -> Tensor: + r""" + logical_and_() -> Tensor + + In-place version of :meth:`~Tensor.logical_and` + """ + + def logical_not(self) -> Tensor: + r""" + logical_not() -> Tensor + + See :func:`torch.logical_not` + """ + + def logical_not_(self) -> Tensor: + r""" + logical_not_() -> Tensor + + In-place version of :meth:`~Tensor.logical_not` + """ + + def logical_or(self, other: Tensor) -> Tensor: + r""" + logical_or() -> Tensor + + See :func:`torch.logical_or` + """ + + def logical_or_(self, other: Tensor) -> Tensor: + r""" + logical_or_() -> Tensor + + In-place version of :meth:`~Tensor.logical_or` + """ + + def logical_xor(self, other: Tensor) -> Tensor: + r""" + logical_xor() -> Tensor + + See :func:`torch.logical_xor` + """ + + def logical_xor_(self, other: Tensor) -> Tensor: + r""" + logical_xor_() -> Tensor + + In-place version of :meth:`~Tensor.logical_xor` + """ + + def logit(self, eps: _float | None = None) -> Tensor: + r""" + logit() -> Tensor + + See :func:`torch.logit` + """ + + def logit_(self, eps: _float | None = None) -> Tensor: + r""" + logit_() -> Tensor + + In-place version of :meth:`~Tensor.logit` + """ + + @overload + def logsumexp(self, dim: _int | _size, keepdim: _bool = False) -> Tensor: + r""" + logsumexp(dim, keepdim=False) -> Tensor + + See :func:`torch.logsumexp` + """ + + @overload + def logsumexp( + self, + dim: Sequence[str | EllipsisType | None], + keepdim: _bool = False, + ) -> Tensor: + r""" + logsumexp(dim, keepdim=False) -> Tensor + + See :func:`torch.logsumexp` + """ + + def long(self) -> Tensor: + r""" + long(memory_format=torch.preserve_format) -> Tensor + + ``self.long()`` is equivalent to ``self.to(torch.int64)``. See :func:`to`. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + @overload + def lt(self, other: Tensor) -> Tensor: + r""" + lt(other) -> Tensor + + See :func:`torch.lt`. + """ + + @overload + def lt(self, other: Number | _complex) -> Tensor: + r""" + lt(other) -> Tensor + + See :func:`torch.lt`. + """ + + @overload + def lt_(self, other: Tensor) -> Tensor: + r""" + lt_(other) -> Tensor + + In-place version of :meth:`~Tensor.lt`. + """ + + @overload + def lt_(self, other: Number | _complex) -> Tensor: + r""" + lt_(other) -> Tensor + + In-place version of :meth:`~Tensor.lt`. + """ + + def lu_solve(self, LU_data: Tensor, LU_pivots: Tensor) -> Tensor: + r""" + lu_solve(LU_data, LU_pivots) -> Tensor + + See :func:`torch.lu_solve` + """ + + def map2_(self, x: Tensor, y: Tensor, callable: Callable) -> Tensor: ... + def map_(self, other: Tensor, callable: Callable) -> Tensor: + r""" + map_(tensor, callable) + + Applies :attr:`callable` for each element in :attr:`self` tensor and the given + :attr:`tensor` and stores the results in :attr:`self` tensor. :attr:`self` tensor and + the given :attr:`tensor` must be :ref:`broadcastable `. + + The :attr:`callable` should have the signature:: + + def callable(a, b) -> number + """ + + @overload + def masked_fill(self, mask: Tensor, value: Tensor) -> Tensor: + r""" + masked_fill(mask, value) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.masked_fill_` + """ + + @overload + def masked_fill(self, mask: Tensor, value: Number | _complex) -> Tensor: + r""" + masked_fill(mask, value) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.masked_fill_` + """ + + @overload + def masked_fill_(self, mask: Tensor, value: Tensor) -> Tensor: + r""" + masked_fill_(mask, value) + + Fills elements of :attr:`self` tensor with :attr:`value` where :attr:`mask` is + True. The shape of :attr:`mask` must be + :ref:`broadcastable ` with the shape of the underlying + tensor. + + Args: + mask (BoolTensor): the boolean mask + value (float): the value to fill in with + """ + + @overload + def masked_fill_(self, mask: Tensor, value: Number | _complex) -> Tensor: + r""" + masked_fill_(mask, value) + + Fills elements of :attr:`self` tensor with :attr:`value` where :attr:`mask` is + True. The shape of :attr:`mask` must be + :ref:`broadcastable ` with the shape of the underlying + tensor. + + Args: + mask (BoolTensor): the boolean mask + value (float): the value to fill in with + """ + + def masked_scatter(self, mask: Tensor, source: Tensor) -> Tensor: + r""" + masked_scatter(mask, tensor) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.masked_scatter_` + + .. note:: + + The inputs :attr:`self` and :attr:`mask` + :ref:`broadcast `. + + Example: + + >>> self = torch.tensor([0, 0, 0, 0, 0]) + >>> mask = torch.tensor( + ... [[0, 0, 0, 1, 1], [1, 1, 0, 1, 1]], + ... dtype=torch.bool, + ... ) + >>> source = torch.tensor([[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]) + >>> self.masked_scatter(mask, source) + tensor([[0, 0, 0, 0, 1], + [2, 3, 0, 4, 5]]) + """ + + def masked_scatter_(self, mask: Tensor, source: Tensor) -> Tensor: + r""" + masked_scatter_(mask, source) + + Copies elements from :attr:`source` into :attr:`self` tensor at positions where + the :attr:`mask` is True. Elements from :attr:`source` are copied into :attr:`self` + starting at position 0 of :attr:`source` and continuing in order one-by-one for each + occurrence of :attr:`mask` being True. + The shape of :attr:`mask` must be :ref:`broadcastable ` + with the shape of the underlying tensor. The :attr:`source` should have at least + as many elements as the number of ones in :attr:`mask`. + + Args: + mask (BoolTensor): the boolean mask + source (Tensor): the tensor to copy from + + .. note:: + + The :attr:`mask` operates on the :attr:`self` tensor, not on the given + :attr:`source` tensor. + + Example: + + >>> self = torch.tensor([[0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]) + >>> mask = torch.tensor( + ... [[0, 0, 0, 1, 1], [1, 1, 0, 1, 1]], + ... dtype=torch.bool, + ... ) + >>> source = torch.tensor([[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]) + >>> self.masked_scatter_(mask, source) + tensor([[0, 0, 0, 0, 1], + [2, 3, 0, 4, 5]]) + """ + + def masked_select(self, mask: Tensor) -> Tensor: + r""" + masked_select(mask) -> Tensor + + See :func:`torch.masked_select` + """ + + def matmul(self, other: Tensor) -> Tensor: + r""" + matmul(tensor2) -> Tensor + + See :func:`torch.matmul` + """ + + def matrix_exp(self) -> Tensor: + r""" + matrix_exp() -> Tensor + + See :func:`torch.matrix_exp` + """ + + def matrix_power(self, n: _int) -> Tensor: + r""" + matrix_power(n) -> Tensor + + .. note:: :meth:`~Tensor.matrix_power` is deprecated, use :func:`torch.linalg.matrix_power` instead. + + Alias for :func:`torch.linalg.matrix_power` + """ + + @overload + def max(self) -> Tensor: + r""" + max(dim=None, keepdim=False) -> Tensor or (Tensor, Tensor) + + See :func:`torch.max` + """ + + @overload + def max(self, other: Tensor) -> Tensor: + r""" + max(dim=None, keepdim=False) -> Tensor or (Tensor, Tensor) + + See :func:`torch.max` + """ + + @overload + def max(self, dim: _int, keepdim: _bool = False) -> torch.return_types.max: + r""" + max(dim=None, keepdim=False) -> Tensor or (Tensor, Tensor) + + See :func:`torch.max` + """ + + @overload + def max( + self, + dim: str | EllipsisType | None, + keepdim: _bool = False, + ) -> torch.return_types.max: + r""" + max(dim=None, keepdim=False) -> Tensor or (Tensor, Tensor) + + See :func:`torch.max` + """ + + def maximum(self, other: Tensor) -> Tensor: + r""" + maximum(other) -> Tensor + + See :func:`torch.maximum` + """ + + @overload + def mean(self, *, dtype: _dtype | None = None) -> Tensor: + r""" + mean(dim=None, keepdim=False, *, dtype=None) -> Tensor + + See :func:`torch.mean` + """ + + @overload + def mean( + self, + dim: _int | _size | None, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + ) -> Tensor: + r""" + mean(dim=None, keepdim=False, *, dtype=None) -> Tensor + + See :func:`torch.mean` + """ + + @overload + def mean( + self, + dim: Sequence[str | EllipsisType | None], + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + ) -> Tensor: + r""" + mean(dim=None, keepdim=False, *, dtype=None) -> Tensor + + See :func:`torch.mean` + """ + + @overload + def median(self) -> Tensor: + r""" + median(dim=None, keepdim=False) -> (Tensor, LongTensor) + + See :func:`torch.median` + """ + + @overload + def median( + self, + dim: _int, + keepdim: _bool = False, + ) -> torch.return_types.median: + r""" + median(dim=None, keepdim=False) -> (Tensor, LongTensor) + + See :func:`torch.median` + """ + + @overload + def median( + self, + dim: str | EllipsisType | None, + keepdim: _bool = False, + ) -> torch.return_types.median: + r""" + median(dim=None, keepdim=False) -> (Tensor, LongTensor) + + See :func:`torch.median` + """ + + @overload + def min(self) -> Tensor: + r""" + min(dim=None, keepdim=False) -> Tensor or (Tensor, Tensor) + + See :func:`torch.min` + """ + + @overload + def min(self, other: Tensor) -> Tensor: + r""" + min(dim=None, keepdim=False) -> Tensor or (Tensor, Tensor) + + See :func:`torch.min` + """ + + @overload + def min(self, dim: _int, keepdim: _bool = False) -> torch.return_types.min: + r""" + min(dim=None, keepdim=False) -> Tensor or (Tensor, Tensor) + + See :func:`torch.min` + """ + + @overload + def min( + self, + dim: str | EllipsisType | None, + keepdim: _bool = False, + ) -> torch.return_types.min: + r""" + min(dim=None, keepdim=False) -> Tensor or (Tensor, Tensor) + + See :func:`torch.min` + """ + + def minimum(self, other: Tensor) -> Tensor: + r""" + minimum(other) -> Tensor + + See :func:`torch.minimum` + """ + + def mm(self, mat2: Tensor) -> Tensor: + r""" + mm(mat2) -> Tensor + + See :func:`torch.mm` + """ + + @overload + def mode( + self, + dim: _int = -1, + keepdim: _bool = False, + ) -> torch.return_types.mode: + r""" + mode(dim=None, keepdim=False) -> (Tensor, LongTensor) + + See :func:`torch.mode` + """ + + @overload + def mode( + self, + dim: str | EllipsisType | None, + keepdim: _bool = False, + ) -> torch.return_types.mode: + r""" + mode(dim=None, keepdim=False) -> (Tensor, LongTensor) + + See :func:`torch.mode` + """ + + @overload + def moveaxis(self, source: _int, destination: _int) -> Tensor: + r""" + moveaxis(source, destination) -> Tensor + + See :func:`torch.moveaxis` + """ + + @overload + def moveaxis(self, source: _size, destination: _size) -> Tensor: + r""" + moveaxis(source, destination) -> Tensor + + See :func:`torch.moveaxis` + """ + + @overload + def movedim(self, source: _int, destination: _int) -> Tensor: + r""" + movedim(source, destination) -> Tensor + + See :func:`torch.movedim` + """ + + @overload + def movedim(self, source: _size, destination: _size) -> Tensor: + r""" + movedim(source, destination) -> Tensor + + See :func:`torch.movedim` + """ + + def msort(self) -> Tensor: + r""" + msort() -> Tensor + + See :func:`torch.msort` + """ + + def mul( + self, + other: Tensor | Number | _complex | torch.SymInt | torch.SymFloat, + *, + out: Tensor | None = None, + ) -> Tensor: + r""" + mul(value) -> Tensor + + See :func:`torch.mul`. + """ + + def mul_( + self, + other: Tensor | Number | _complex | torch.SymInt | torch.SymFloat, + ) -> Tensor: + r""" + mul_(value) -> Tensor + + In-place version of :meth:`~Tensor.mul`. + """ + + def multinomial( + self, + num_samples: _int | SymInt, + replacement: _bool = False, + *, + generator: Generator | None = None, + ) -> Tensor: + r""" + multinomial(num_samples, replacement=False, *, generator=None) -> Tensor + + See :func:`torch.multinomial` + """ + + @overload + def multiply(self, other: Tensor) -> Tensor: + r""" + multiply(value) -> Tensor + + See :func:`torch.multiply`. + """ + + @overload + def multiply(self, other: Number | _complex) -> Tensor: + r""" + multiply(value) -> Tensor + + See :func:`torch.multiply`. + """ + + @overload + def multiply_(self, other: Tensor) -> Tensor: + r""" + multiply_(value) -> Tensor + + In-place version of :meth:`~Tensor.multiply`. + """ + + @overload + def multiply_(self, other: Number | _complex) -> Tensor: + r""" + multiply_(value) -> Tensor + + In-place version of :meth:`~Tensor.multiply`. + """ + + def mv(self, vec: Tensor) -> Tensor: + r""" + mv(vec) -> Tensor + + See :func:`torch.mv` + """ + + def mvlgamma(self, p: _int) -> Tensor: + r""" + mvlgamma(p) -> Tensor + + See :func:`torch.mvlgamma` + """ + + def mvlgamma_(self, p: _int) -> Tensor: + r""" + mvlgamma_(p) -> Tensor + + In-place version of :meth:`~Tensor.mvlgamma` + """ + + def nan_to_num( + self, + nan: _float | None = None, + posinf: _float | None = None, + neginf: _float | None = None, + ) -> Tensor: + r""" + nan_to_num(nan=0.0, posinf=None, neginf=None) -> Tensor + + See :func:`torch.nan_to_num`. + """ + + def nan_to_num_( + self, + nan: _float | None = None, + posinf: _float | None = None, + neginf: _float | None = None, + ) -> Tensor: + r""" + nan_to_num_(nan=0.0, posinf=None, neginf=None) -> Tensor + + In-place version of :meth:`~Tensor.nan_to_num`. + """ + + def nanmean( + self, + dim: _int | _size | None = None, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + ) -> Tensor: + r""" + nanmean(dim=None, keepdim=False, *, dtype=None) -> Tensor + + See :func:`torch.nanmean` + """ + + @overload + def nanmedian(self) -> Tensor: + r""" + nanmedian(dim=None, keepdim=False) -> (Tensor, LongTensor) + + See :func:`torch.nanmedian` + """ + + @overload + def nanmedian( + self, + dim: _int, + keepdim: _bool = False, + ) -> torch.return_types.nanmedian: + r""" + nanmedian(dim=None, keepdim=False) -> (Tensor, LongTensor) + + See :func:`torch.nanmedian` + """ + + @overload + def nanmedian( + self, + dim: str | EllipsisType | None, + keepdim: _bool = False, + ) -> torch.return_types.nanmedian: + r""" + nanmedian(dim=None, keepdim=False) -> (Tensor, LongTensor) + + See :func:`torch.nanmedian` + """ + + @overload + def nanquantile( + self, + q: Tensor, + dim: _int | None = None, + keepdim: _bool = False, + *, + interpolation: str = "linear", + ) -> Tensor: + r""" + nanquantile(q, dim=None, keepdim=False, *, interpolation='linear') -> Tensor + + See :func:`torch.nanquantile` + """ + + @overload + def nanquantile( + self, + q: _float, + dim: _int | None = None, + keepdim: _bool = False, + *, + interpolation: str = "linear", + ) -> Tensor: + r""" + nanquantile(q, dim=None, keepdim=False, *, interpolation='linear') -> Tensor + + See :func:`torch.nanquantile` + """ + + def nansum( + self, + dim: _int | _size | None = None, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + ) -> Tensor: + r""" + nansum(dim=None, keepdim=False, dtype=None) -> Tensor + + See :func:`torch.nansum` + """ + + @overload + def narrow(self, dim: _int, start: Tensor, length: _int | SymInt) -> Tensor: + r""" + narrow(dimension, start, length) -> Tensor + + See :func:`torch.narrow`. + """ + + @overload + def narrow( + self, + dim: _int, + start: _int | SymInt, + length: _int | SymInt, + ) -> Tensor: + r""" + narrow(dimension, start, length) -> Tensor + + See :func:`torch.narrow`. + """ + + def narrow_copy( + self, + dim: _int, + start: _int | SymInt, + length: _int | SymInt, + ) -> Tensor: + r""" + narrow_copy(dimension, start, length) -> Tensor + + See :func:`torch.narrow_copy`. + """ + + def ndimension(self) -> _int: + r""" + ndimension() -> int + + Alias for :meth:`~Tensor.dim()` + """ + + @overload + def ne(self, other: Tensor) -> Tensor: + r""" + ne(other) -> Tensor + + See :func:`torch.ne`. + """ + + @overload + def ne(self, other: Number | _complex) -> Tensor: + r""" + ne(other) -> Tensor + + See :func:`torch.ne`. + """ + + @overload + def ne_(self, other: Tensor) -> Tensor: + r""" + ne_(other) -> Tensor + + In-place version of :meth:`~Tensor.ne`. + """ + + @overload + def ne_(self, other: Number | _complex) -> Tensor: + r""" + ne_(other) -> Tensor + + In-place version of :meth:`~Tensor.ne`. + """ + + def neg(self) -> Tensor: + r""" + neg() -> Tensor + + See :func:`torch.neg` + """ + + def neg_(self) -> Tensor: + r""" + neg_() -> Tensor + + In-place version of :meth:`~Tensor.neg` + """ + + def negative(self) -> Tensor: + r""" + negative() -> Tensor + + See :func:`torch.negative` + """ + + def negative_(self) -> Tensor: + r""" + negative_() -> Tensor + + In-place version of :meth:`~Tensor.negative` + """ + + def nelement(self) -> _int: + r""" + nelement() -> int + + Alias for :meth:`~Tensor.numel` + """ + + @overload + def new(cls, *args: Any, device: DeviceLikeType | None = None) -> Self: ... + @overload + def new(cls, storage: Storage) -> Self: ... + @overload + def new(cls, other: Tensor) -> Self: ... + @overload + def new(cls, size: _size, *, device: DeviceLikeType | None = None) -> Self: ... + @overload + def new_empty( + self, + size: Sequence[_int | SymInt], + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, + ) -> Tensor: + r""" + new_empty(size, *, dtype=None, device=None, requires_grad=False, layout=torch.strided, pin_memory=False) -> Tensor + + + Returns a Tensor of size :attr:`size` filled with uninitialized data. + By default, the returned Tensor has the same :class:`torch.dtype` and + :class:`torch.device` as this tensor. + + Args: + size (int...): a list, tuple, or :class:`torch.Size` of integers defining the + shape of the output tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired type of returned tensor. + Default: if None, same :class:`torch.dtype` as this tensor. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if None, same :class:`torch.device` as this tensor. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> tensor = torch.ones(()) + >>> tensor.new_empty((2, 3)) + tensor([[ 5.8182e-18, 4.5765e-41, -1.0545e+30], + [ 3.0949e-41, 4.4842e-44, 0.0000e+00]]) + """ + + @overload + def new_empty( + self, + *size: _int | SymInt, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, + ) -> Tensor: + r""" + new_empty(size, *, dtype=None, device=None, requires_grad=False, layout=torch.strided, pin_memory=False) -> Tensor + + + Returns a Tensor of size :attr:`size` filled with uninitialized data. + By default, the returned Tensor has the same :class:`torch.dtype` and + :class:`torch.device` as this tensor. + + Args: + size (int...): a list, tuple, or :class:`torch.Size` of integers defining the + shape of the output tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired type of returned tensor. + Default: if None, same :class:`torch.dtype` as this tensor. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if None, same :class:`torch.device` as this tensor. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> tensor = torch.ones(()) + >>> tensor.new_empty((2, 3)) + tensor([[ 5.8182e-18, 4.5765e-41, -1.0545e+30], + [ 3.0949e-41, 4.4842e-44, 0.0000e+00]]) + """ + + def new_empty_strided( + self, + size: Sequence[_int | SymInt], + stride: Sequence[_int | SymInt], + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, + ) -> Tensor: + r""" + new_empty_strided(size, stride, dtype=None, device=None, requires_grad=False, layout=torch.strided, pin_memory=False) -> Tensor + + + Returns a Tensor of size :attr:`size` and strides :attr:`stride` filled with + uninitialized data. By default, the returned Tensor has the same + :class:`torch.dtype` and :class:`torch.device` as this tensor. + + Args: + size (int...): a list, tuple, or :class:`torch.Size` of integers defining the + shape of the output tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired type of returned tensor. + Default: if None, same :class:`torch.dtype` as this tensor. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if None, same :class:`torch.device` as this tensor. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> tensor = torch.ones(()) + >>> tensor.new_empty_strided((2, 3), (3, 1)) + tensor([[ 5.8182e-18, 4.5765e-41, -1.0545e+30], + [ 3.0949e-41, 4.4842e-44, 0.0000e+00]]) + """ + + def new_full( + self, + size: Sequence[_int | SymInt], + fill_value: Number | _complex, + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, + ) -> Tensor: + r""" + new_full(size, fill_value, *, dtype=None, device=None, requires_grad=False, layout=torch.strided, pin_memory=False) -> Tensor + + + Returns a Tensor of size :attr:`size` filled with :attr:`fill_value`. + By default, the returned Tensor has the same :class:`torch.dtype` and + :class:`torch.device` as this tensor. + + Args: + fill_value (scalar): the number to fill the output tensor with. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired type of returned tensor. + Default: if None, same :class:`torch.dtype` as this tensor. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if None, same :class:`torch.device` as this tensor. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> tensor = torch.ones((2,), dtype=torch.float64) + >>> tensor.new_full((3, 4), 3.141592) + tensor([[ 3.1416, 3.1416, 3.1416, 3.1416], + [ 3.1416, 3.1416, 3.1416, 3.1416], + [ 3.1416, 3.1416, 3.1416, 3.1416]], dtype=torch.float64) + """ + + @overload + def new_ones( + self, + size: _size, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + pin_memory: _bool = False, + ) -> Tensor: + r""" + new_ones(size, *, dtype=None, device=None, requires_grad=False, layout=torch.strided, pin_memory=False) -> Tensor + + + Returns a Tensor of size :attr:`size` filled with ``1``. + By default, the returned Tensor has the same :class:`torch.dtype` and + :class:`torch.device` as this tensor. + + Args: + size (int...): a list, tuple, or :class:`torch.Size` of integers defining the + shape of the output tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired type of returned tensor. + Default: if None, same :class:`torch.dtype` as this tensor. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if None, same :class:`torch.device` as this tensor. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> tensor = torch.tensor((), dtype=torch.int32) + >>> tensor.new_ones((2, 3)) + tensor([[ 1, 1, 1], + [ 1, 1, 1]], dtype=torch.int32) + """ + + @overload + def new_ones( + self, + size: Sequence[_int | SymInt], + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, + ) -> Tensor: + r""" + new_ones(size, *, dtype=None, device=None, requires_grad=False, layout=torch.strided, pin_memory=False) -> Tensor + + + Returns a Tensor of size :attr:`size` filled with ``1``. + By default, the returned Tensor has the same :class:`torch.dtype` and + :class:`torch.device` as this tensor. + + Args: + size (int...): a list, tuple, or :class:`torch.Size` of integers defining the + shape of the output tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired type of returned tensor. + Default: if None, same :class:`torch.dtype` as this tensor. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if None, same :class:`torch.device` as this tensor. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> tensor = torch.tensor((), dtype=torch.int32) + >>> tensor.new_ones((2, 3)) + tensor([[ 1, 1, 1], + [ 1, 1, 1]], dtype=torch.int32) + """ + + @overload + def new_ones( + self, + *size: _int | SymInt, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, + ) -> Tensor: + r""" + new_ones(size, *, dtype=None, device=None, requires_grad=False, layout=torch.strided, pin_memory=False) -> Tensor + + + Returns a Tensor of size :attr:`size` filled with ``1``. + By default, the returned Tensor has the same :class:`torch.dtype` and + :class:`torch.device` as this tensor. + + Args: + size (int...): a list, tuple, or :class:`torch.Size` of integers defining the + shape of the output tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired type of returned tensor. + Default: if None, same :class:`torch.dtype` as this tensor. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if None, same :class:`torch.device` as this tensor. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> tensor = torch.tensor((), dtype=torch.int32) + >>> tensor.new_ones((2, 3)) + tensor([[ 1, 1, 1], + [ 1, 1, 1]], dtype=torch.int32) + """ + + def new_tensor( + self, + data: Any, + dtype: _dtype | None = None, + device: DeviceLikeType | None = None, + requires_grad: _bool = False, + pin_memory: _bool = False, + ) -> Tensor: + r""" + new_tensor(data, *, dtype=None, device=None, requires_grad=False, layout=torch.strided, pin_memory=False) -> Tensor + + + Returns a new Tensor with :attr:`data` as the tensor data. + By default, the returned Tensor has the same :class:`torch.dtype` and + :class:`torch.device` as this tensor. + + .. warning:: + + :func:`new_tensor` always copies :attr:`data`. If you have a Tensor + ``data`` and want to avoid a copy, use :func:`torch.Tensor.requires_grad_` + or :func:`torch.Tensor.detach`. + If you have a numpy array and want to avoid a copy, use + :func:`torch.from_numpy`. + + .. warning:: + + When data is a tensor `x`, :func:`new_tensor()` reads out 'the data' from whatever it is passed, + and constructs a leaf variable. Therefore ``tensor.new_tensor(x)`` is equivalent to ``x.detach().clone()`` + and ``tensor.new_tensor(x, requires_grad=True)`` is equivalent to ``x.detach().clone().requires_grad_(True)``. + The equivalents using ``detach()`` and ``clone()`` are recommended. + + Args: + data (array_like): The returned Tensor copies :attr:`data`. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired type of returned tensor. + Default: if None, same :class:`torch.dtype` as this tensor. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if None, same :class:`torch.device` as this tensor. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> tensor = torch.ones((2,), dtype=torch.int8) + >>> data = [[0, 1], [2, 3]] + >>> tensor.new_tensor(data) + tensor([[ 0, 1], + [ 2, 3]], dtype=torch.int8) + """ + + @overload + def new_zeros( + self, + size: Sequence[_int | SymInt], + *, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, + ) -> Tensor: + r""" + new_zeros(size, *, dtype=None, device=None, requires_grad=False, layout=torch.strided, pin_memory=False) -> Tensor + + + Returns a Tensor of size :attr:`size` filled with ``0``. + By default, the returned Tensor has the same :class:`torch.dtype` and + :class:`torch.device` as this tensor. + + Args: + size (int...): a list, tuple, or :class:`torch.Size` of integers defining the + shape of the output tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired type of returned tensor. + Default: if None, same :class:`torch.dtype` as this tensor. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if None, same :class:`torch.device` as this tensor. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> tensor = torch.tensor((), dtype=torch.float64) + >>> tensor.new_zeros((2, 3)) + tensor([[ 0., 0., 0.], + [ 0., 0., 0.]], dtype=torch.float64) + """ + + @overload + def new_zeros( + self, + *size: _int | SymInt, + dtype: _dtype | None = None, + layout: _layout | None = None, + device: DeviceLikeType | None = None, + pin_memory: _bool | None = False, + requires_grad: _bool | None = False, + ) -> Tensor: + r""" + new_zeros(size, *, dtype=None, device=None, requires_grad=False, layout=torch.strided, pin_memory=False) -> Tensor + + + Returns a Tensor of size :attr:`size` filled with ``0``. + By default, the returned Tensor has the same :class:`torch.dtype` and + :class:`torch.device` as this tensor. + + Args: + size (int...): a list, tuple, or :class:`torch.Size` of integers defining the + shape of the output tensor. + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired type of returned tensor. + Default: if None, same :class:`torch.dtype` as this tensor. + device (:class:`torch.device`, optional): the desired device of returned tensor. + Default: if None, same :class:`torch.device` as this tensor. + requires_grad (bool, optional): If autograd should record operations on the + returned tensor. Default: ``False``. + layout (:class:`torch.layout`, optional): the desired layout of returned Tensor. + Default: ``torch.strided``. + pin_memory (bool, optional): If set, returned tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> tensor = torch.tensor((), dtype=torch.float64) + >>> tensor.new_zeros((2, 3)) + tensor([[ 0., 0., 0.], + [ 0., 0., 0.]], dtype=torch.float64) + """ + + def nextafter(self, other: Tensor) -> Tensor: + r""" + nextafter(other) -> Tensor + See :func:`torch.nextafter` + """ + + def nextafter_(self, other: Tensor) -> Tensor: + r""" + nextafter_(other) -> Tensor + In-place version of :meth:`~Tensor.nextafter` + """ + + @overload + def nonzero(self, *, as_tuple: Literal[False] = False) -> Tensor: + r""" + nonzero() -> LongTensor + + See :func:`torch.nonzero` + """ + + @overload + def nonzero(self, *, as_tuple: Literal[True]) -> tuple[Tensor, ...]: + r""" + nonzero() -> LongTensor + + See :func:`torch.nonzero` + """ + + def nonzero_static( + self, + *, + size: _int | SymInt, + fill_value: _int = -1, + ) -> Tensor: + r""" + nonzero_static(input, *, size, fill_value=-1) -> Tensor + + Returns a 2-D tensor where each row is the index for a non-zero value. + The returned Tensor has the same `torch.dtype` as `torch.nonzero()`. + + Args: + input (Tensor): the input tensor to count non-zero elements. + + Keyword args: + size (int): the size of non-zero elements expected to be included in the out + tensor. Pad the out tensor with `fill_value` if the `size` is larger + than total number of non-zero elements, truncate out tensor if `size` + is smaller. The size must be a non-negative integer. + fill_value (int, optional): the value to fill the output tensor with when `size` is larger + than the total number of non-zero elements. Default is `-1` to represent + invalid index. + + Example: + + # Example 1: Padding + >>> input_tensor = torch.tensor([[1, 0], [3, 2]]) + >>> static_size = 4 + >>> t = torch.nonzero_static(input_tensor, size=static_size) + tensor([[ 0, 0], + [ 1, 0], + [ 1, 1], + [ -1, -1]], dtype=torch.int64) + + # Example 2: Truncating + >>> input_tensor = torch.tensor([[1, 0], [3, 2]]) + >>> static_size = 2 + >>> t = torch.nonzero_static(input_tensor, size=static_size) + tensor([[ 0, 0], + [ 1, 0]], dtype=torch.int64) + + # Example 3: 0 size + >>> input_tensor = torch.tensor([10]) + >>> static_size = 0 + >>> t = torch.nonzero_static(input_tensor, size=static_size) + tensor([], size=(0, 1), dtype=torch.int64) + + # Example 4: 0 rank input + >>> input_tensor = torch.tensor(10) + >>> static_size = 2 + >>> t = torch.nonzero_static(input_tensor, size=static_size) + tensor([], size=(2, 0), dtype=torch.int64) + """ + + def normal_( + self, + mean: _float = 0, + std: _float = 1, + *, + generator: Generator | None = None, + ) -> Tensor: + r""" + normal_(mean=0, std=1, *, generator=None) -> Tensor + + Fills :attr:`self` tensor with elements samples from the normal distribution + parameterized by :attr:`mean` and :attr:`std`. + """ + + @overload + def not_equal(self, other: Tensor) -> Tensor: + r""" + not_equal(other) -> Tensor + + See :func:`torch.not_equal`. + """ + + @overload + def not_equal(self, other: Number | _complex) -> Tensor: + r""" + not_equal(other) -> Tensor + + See :func:`torch.not_equal`. + """ + + @overload + def not_equal_(self, other: Tensor) -> Tensor: + r""" + not_equal_(other) -> Tensor + + In-place version of :meth:`~Tensor.not_equal`. + """ + + @overload + def not_equal_(self, other: Number | _complex) -> Tensor: + r""" + not_equal_(other) -> Tensor + + In-place version of :meth:`~Tensor.not_equal`. + """ + + def numel(self) -> _int: + r""" + numel() -> int + + See :func:`torch.numel` + """ + + def numpy(self, *, force: _bool = False) -> numpy.ndarray: + r""" + numpy(*, force=False) -> numpy.ndarray + + Returns the tensor as a NumPy :class:`ndarray`. + + If :attr:`force` is ``False`` (the default), the conversion + is performed only if the tensor is on the CPU, does not require grad, + does not have its conjugate bit set, and is a dtype and layout that + NumPy supports. The returned ndarray and the tensor will share their + storage, so changes to the tensor will be reflected in the ndarray + and vice versa. + + If :attr:`force` is ``True`` this is equivalent to + calling ``t.detach().cpu().resolve_conj().resolve_neg().numpy()``. + If the tensor isn't on the CPU or the conjugate or negative bit is set, + the tensor won't share its storage with the returned ndarray. + Setting :attr:`force` to ``True`` can be a useful shorthand. + + Args: + force (bool): if ``True``, the ndarray may be a copy of the tensor + instead of always sharing memory, defaults to ``False``. + """ + + def orgqr(self, input2: Tensor) -> Tensor: + r""" + orgqr(input2) -> Tensor + + See :func:`torch.orgqr` + """ + + def ormqr( + self, + input2: Tensor, + input3: Tensor, + left: _bool = True, + transpose: _bool = False, + ) -> Tensor: + r""" + ormqr(input2, input3, left=True, transpose=False) -> Tensor + + See :func:`torch.ormqr` + """ + + def outer(self, vec2: Tensor) -> Tensor: + r""" + outer(vec2) -> Tensor + + See :func:`torch.outer`. + """ + + @overload + def permute(self, dims: _size) -> Tensor: + r""" + permute(*dims) -> Tensor + + See :func:`torch.permute` + """ + + @overload + def permute(self, *dims: _int) -> Tensor: + r""" + permute(*dims) -> Tensor + + See :func:`torch.permute` + """ + + def pin_memory(self, device: DeviceLikeType | None = None) -> Tensor: + r""" + pin_memory() -> Tensor + + Copies the tensor to pinned memory, if it's not already pinned. + By default, the device pinned memory on will be the current :ref:`accelerator`. + """ + + def pinverse(self, rcond: _float = 1e-15) -> Tensor: + r""" + pinverse() -> Tensor + + See :func:`torch.pinverse` + """ + + def polygamma(self, n: _int) -> Tensor: + r""" + polygamma(n) -> Tensor + + See :func:`torch.polygamma` + """ + + def polygamma_(self, n: _int) -> Tensor: + r""" + polygamma_(n) -> Tensor + + In-place version of :meth:`~Tensor.polygamma` + """ + + def positive(self) -> Tensor: + r""" + positive() -> Tensor + + See :func:`torch.positive` + """ + + @overload + def pow(self, exponent: Tensor) -> Tensor: + r""" + pow(exponent) -> Tensor + + See :func:`torch.pow` + """ + + @overload + def pow(self, exponent: Number | _complex) -> Tensor: + r""" + pow(exponent) -> Tensor + + See :func:`torch.pow` + """ + + @overload + def pow_(self, exponent: Tensor) -> Tensor: + r""" + pow_(exponent) -> Tensor + + In-place version of :meth:`~Tensor.pow` + """ + + @overload + def pow_(self, exponent: Number | _complex) -> Tensor: + r""" + pow_(exponent) -> Tensor + + In-place version of :meth:`~Tensor.pow` + """ + + def prelu(self, weight: Tensor) -> Tensor: ... + @overload + def prod(self, *, dtype: _dtype | None = None) -> Tensor: + r""" + prod(dim=None, keepdim=False, dtype=None) -> Tensor + + See :func:`torch.prod` + """ + + @overload + def prod( + self, + dim: _int, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + ) -> Tensor: + r""" + prod(dim=None, keepdim=False, dtype=None) -> Tensor + + See :func:`torch.prod` + """ + + @overload + def prod( + self, + dim: str | EllipsisType | None, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + ) -> Tensor: + r""" + prod(dim=None, keepdim=False, dtype=None) -> Tensor + + See :func:`torch.prod` + """ + + def put( + self, + index: Tensor, + source: Tensor, + accumulate: _bool = False, + ) -> Tensor: + r""" + put(input, index, source, accumulate=False) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.put_`. + `input` corresponds to `self` in :meth:`torch.Tensor.put_`. + """ + + def put_( + self, + index: Tensor, + source: Tensor, + accumulate: _bool = False, + ) -> Tensor: + r""" + put_(index, source, accumulate=False) -> Tensor + + Copies the elements from :attr:`source` into the positions specified by + :attr:`index`. For the purpose of indexing, the :attr:`self` tensor is treated as if + it were a 1-D tensor. + + :attr:`index` and :attr:`source` need to have the same number of elements, but not necessarily + the same shape. + + If :attr:`accumulate` is ``True``, the elements in :attr:`source` are added to + :attr:`self`. If accumulate is ``False``, the behavior is undefined if :attr:`index` + contain duplicate elements. + + Args: + index (LongTensor): the indices into self + source (Tensor): the tensor containing values to copy from + accumulate (bool, optional): whether to accumulate into self. Default: ``False`` + + Example:: + + >>> src = torch.tensor([[4, 3, 5], + ... [6, 7, 8]]) + >>> src.put_(torch.tensor([1, 3]), torch.tensor([9, 10])) + tensor([[ 4, 9, 5], + [ 10, 7, 8]]) + """ + + def q_per_channel_axis(self) -> _int: + r""" + q_per_channel_axis() -> int + + Given a Tensor quantized by linear (affine) per-channel quantization, + returns the index of dimension on which per-channel quantization is applied. + """ + + def q_per_channel_scales(self) -> Tensor: + r""" + q_per_channel_scales() -> Tensor + + Given a Tensor quantized by linear (affine) per-channel quantization, + returns a Tensor of scales of the underlying quantizer. It has the number of + elements that matches the corresponding dimensions (from q_per_channel_axis) of + the tensor. + """ + + def q_per_channel_zero_points(self) -> Tensor: + r""" + q_per_channel_zero_points() -> Tensor + + Given a Tensor quantized by linear (affine) per-channel quantization, + returns a tensor of zero_points of the underlying quantizer. It has the number of + elements that matches the corresponding dimensions (from q_per_channel_axis) of + the tensor. + """ + + def q_scale(self) -> _float: + r""" + q_scale() -> float + + Given a Tensor quantized by linear(affine) quantization, + returns the scale of the underlying quantizer(). + """ + + def q_zero_point(self) -> _int: + r""" + q_zero_point() -> int + + Given a Tensor quantized by linear(affine) quantization, + returns the zero_point of the underlying quantizer(). + """ + + def qr(self, some: _bool = True) -> torch.return_types.qr: + r""" + qr(some=True) -> (Tensor, Tensor) + + See :func:`torch.qr` + """ + + def qscheme(self) -> _qscheme: + r""" + qscheme() -> torch.qscheme + + Returns the quantization scheme of a given QTensor. + """ + + @overload + def quantile( + self, + q: Tensor, + dim: _int | None = None, + keepdim: _bool = False, + *, + interpolation: str = "linear", + ) -> Tensor: + r""" + quantile(q, dim=None, keepdim=False, *, interpolation='linear') -> Tensor + + See :func:`torch.quantile` + """ + + @overload + def quantile( + self, + q: _float, + dim: _int | None = None, + keepdim: _bool = False, + *, + interpolation: str = "linear", + ) -> Tensor: + r""" + quantile(q, dim=None, keepdim=False, *, interpolation='linear') -> Tensor + + See :func:`torch.quantile` + """ + + def rad2deg(self) -> Tensor: + r""" + rad2deg() -> Tensor + + See :func:`torch.rad2deg` + """ + + def rad2deg_(self) -> Tensor: + r""" + rad2deg_() -> Tensor + + In-place version of :meth:`~Tensor.rad2deg` + """ + + @overload + def random_(self, *, generator: Generator | None = None) -> Tensor: + r""" + random_(from=0, to=None, *, generator=None) -> Tensor + + Fills :attr:`self` tensor with numbers sampled from the discrete uniform + distribution over ``[from, to - 1]``. If not specified, the values are usually + only bounded by :attr:`self` tensor's data type. However, for floating point + types, if unspecified, range will be ``[0, 2^mantissa]`` to ensure that every + value is representable. For example, `torch.tensor(1, dtype=torch.double).random_()` + will be uniform in ``[0, 2^53]``. + """ + + @overload + def random_( + self, + from_: _int, + to: _int | None, + *, + generator: Generator | None = None, + ) -> Tensor: + r""" + random_(from=0, to=None, *, generator=None) -> Tensor + + Fills :attr:`self` tensor with numbers sampled from the discrete uniform + distribution over ``[from, to - 1]``. If not specified, the values are usually + only bounded by :attr:`self` tensor's data type. However, for floating point + types, if unspecified, range will be ``[0, 2^mantissa]`` to ensure that every + value is representable. For example, `torch.tensor(1, dtype=torch.double).random_()` + will be uniform in ``[0, 2^53]``. + """ + + @overload + def random_( + self, + to: _int, + *, + generator: Generator | None = None, + ) -> Tensor: + r""" + random_(from=0, to=None, *, generator=None) -> Tensor + + Fills :attr:`self` tensor with numbers sampled from the discrete uniform + distribution over ``[from, to - 1]``. If not specified, the values are usually + only bounded by :attr:`self` tensor's data type. However, for floating point + types, if unspecified, range will be ``[0, 2^mantissa]`` to ensure that every + value is representable. For example, `torch.tensor(1, dtype=torch.double).random_()` + will be uniform in ``[0, 2^53]``. + """ + + def ravel(self) -> Tensor: + r""" + ravel() -> Tensor + + see :func:`torch.ravel` + """ + + def reciprocal(self) -> Tensor: + r""" + reciprocal() -> Tensor + + See :func:`torch.reciprocal` + """ + + def reciprocal_(self) -> Tensor: + r""" + reciprocal_() -> Tensor + + In-place version of :meth:`~Tensor.reciprocal` + """ + + def record_stream(self, s: Stream) -> None: + r""" + record_stream(stream) + + Marks the tensor as having been used by this stream. When the tensor + is deallocated, ensure the tensor memory is not reused for another tensor + until all work queued on :attr:`stream` at the time of deallocation is + complete. + + .. note:: + + The caching allocator is aware of only the stream where a tensor was + allocated. Due to the awareness, it already correctly manages the life + cycle of tensors on only one stream. But if a tensor is used on a stream + different from the stream of origin, the allocator might reuse the memory + unexpectedly. Calling this method lets the allocator know which streams + have used the tensor. + + .. warning:: + + This method is most suitable for use cases where you are providing a + function that created a tensor on a side stream, and want users to be able + to make use of the tensor without having to think carefully about stream + safety when making use of them. These safety guarantees come at some + performance and predictability cost (analogous to the tradeoff between GC + and manual memory management), so if you are in a situation where + you manage the full lifetime of your tensors, you may consider instead + manually managing CUDA events so that calling this method is not necessary. + In particular, when you call this method, on later allocations the + allocator will poll the recorded stream to see if all operations have + completed yet; you can potentially race with side stream computation and + non-deterministically reuse or fail to reuse memory for an allocation. + + You can safely use tensors allocated on side streams without + :meth:`~Tensor.record_stream`; you must manually ensure that + any non-creation stream uses of a tensor are synced back to the creation + stream before you deallocate the tensor. As the CUDA caching allocator + guarantees that the memory will only be reused with the same creation stream, + this is sufficient to ensure that writes to future reallocations of the + memory will be delayed until non-creation stream uses are done. + (Counterintuitively, you may observe that on the CPU side we have already + reallocated the tensor, even though CUDA kernels on the old tensor are + still in progress. This is fine, because CUDA operations on the new + tensor will appropriately wait for the old operations to complete, as they + are all on the same stream.) + + Concretely, this looks like this:: + + with torch.cuda.stream(s0): + x = torch.zeros(N) + + s1.wait_stream(s0) + with torch.cuda.stream(s1): + y = some_comm_op(x) + + ... some compute on s0 ... + + # synchronize creation stream s0 to side stream s1 + # before deallocating x + s0.wait_stream(s1) + del x + + Note that some discretion is required when deciding when to perform + ``s0.wait_stream(s1)``. In particular, if we were to wait immediately + after ``some_comm_op``, there wouldn't be any point in having the side + stream; it would be equivalent to have run ``some_comm_op`` on ``s0``. + Instead, the synchronization must be placed at some appropriate, later + point in time where you expect the side stream ``s1`` to have finished + work. This location is typically identified via profiling, e.g., using + Chrome traces produced + :meth:`torch.autograd.profiler.profile.export_chrome_trace`. If you + place the wait too early, work on s0 will block until ``s1`` has finished, + preventing further overlapping of communication and computation. If you + place the wait too late, you will use more memory than is strictly + necessary (as you are keeping ``x`` live for longer.) For a concrete + example of how this guidance can be applied in practice, see this post: + `FSDP and CUDACachingAllocator + `_. + """ + + def refine_names( + self, + names: Sequence[str | EllipsisType | None], + ) -> Tensor: ... + def relu(self) -> Tensor: ... + def relu_(self) -> Tensor: ... + @overload + def remainder(self, other: Tensor) -> Tensor: + r""" + remainder(divisor) -> Tensor + + See :func:`torch.remainder` + """ + + @overload + def remainder(self, other: Number | _complex) -> Tensor: + r""" + remainder(divisor) -> Tensor + + See :func:`torch.remainder` + """ + + @overload + def remainder_(self, other: Tensor) -> Tensor: + r""" + remainder_(divisor) -> Tensor + + In-place version of :meth:`~Tensor.remainder` + """ + + @overload + def remainder_(self, other: Number | _complex) -> Tensor: + r""" + remainder_(divisor) -> Tensor + + In-place version of :meth:`~Tensor.remainder` + """ + + def rename( + self, + names: Sequence[str | EllipsisType | None] | None, + ) -> Tensor: ... + def rename_( + self, + names: Sequence[str | EllipsisType | None] | None, + ) -> Tensor: ... + def renorm( + self, + p: Number | _complex, + dim: _int, + maxnorm: Number | _complex, + ) -> Tensor: + r""" + renorm(p, dim, maxnorm) -> Tensor + + See :func:`torch.renorm` + """ + + def renorm_( + self, + p: Number | _complex, + dim: _int, + maxnorm: Number | _complex, + ) -> Tensor: + r""" + renorm_(p, dim, maxnorm) -> Tensor + + In-place version of :meth:`~Tensor.renorm` + """ + + @overload + def repeat(self, repeats: Sequence[_int | SymInt]) -> Tensor: + r""" + repeat(*repeats) -> Tensor + + Repeats this tensor along the specified dimensions. + + Unlike :meth:`~Tensor.expand`, this function copies the tensor's data. + + .. warning:: + + :meth:`~Tensor.repeat` behaves differently from + `numpy.repeat `_, + but is more similar to + `numpy.tile `_. + For the operator similar to `numpy.repeat`, see :func:`torch.repeat_interleave`. + + Args: + repeat (torch.Size, int..., tuple of int or list of int): The number of times to repeat this tensor along each dimension + + Example:: + + >>> x = torch.tensor([1, 2, 3]) + >>> x.repeat(4, 2) + tensor([[ 1, 2, 3, 1, 2, 3], + [ 1, 2, 3, 1, 2, 3], + [ 1, 2, 3, 1, 2, 3], + [ 1, 2, 3, 1, 2, 3]]) + >>> x.repeat(4, 2, 1).size() + torch.Size([4, 2, 3]) + """ + + @overload + def repeat(self, *repeats: _int | SymInt) -> Tensor: + r""" + repeat(*repeats) -> Tensor + + Repeats this tensor along the specified dimensions. + + Unlike :meth:`~Tensor.expand`, this function copies the tensor's data. + + .. warning:: + + :meth:`~Tensor.repeat` behaves differently from + `numpy.repeat `_, + but is more similar to + `numpy.tile `_. + For the operator similar to `numpy.repeat`, see :func:`torch.repeat_interleave`. + + Args: + repeat (torch.Size, int..., tuple of int or list of int): The number of times to repeat this tensor along each dimension + + Example:: + + >>> x = torch.tensor([1, 2, 3]) + >>> x.repeat(4, 2) + tensor([[ 1, 2, 3, 1, 2, 3], + [ 1, 2, 3, 1, 2, 3], + [ 1, 2, 3, 1, 2, 3], + [ 1, 2, 3, 1, 2, 3]]) + >>> x.repeat(4, 2, 1).size() + torch.Size([4, 2, 3]) + """ + + @overload + def repeat_interleave( + self, + repeats: Tensor, + dim: _int | None = None, + *, + output_size: _int | SymInt | None = None, + ) -> Tensor: + r""" + repeat_interleave(repeats, dim=None, *, output_size=None) -> Tensor + + See :func:`torch.repeat_interleave`. + """ + + @overload + def repeat_interleave( + self, + repeats: _int | SymInt, + dim: _int | None = None, + *, + output_size: _int | SymInt | None = None, + ) -> Tensor: + r""" + repeat_interleave(repeats, dim=None, *, output_size=None) -> Tensor + + See :func:`torch.repeat_interleave`. + """ + + def requires_grad_(self, mode: _bool = True) -> Tensor: + r""" + requires_grad_(requires_grad=True) -> Tensor + + Change if autograd should record operations on this tensor: sets this tensor's + :attr:`requires_grad` attribute in-place. Returns this tensor. + + :func:`requires_grad_`'s main use case is to tell autograd to begin recording + operations on a Tensor ``tensor``. If ``tensor`` has ``requires_grad=False`` + (because it was obtained through a DataLoader, or required preprocessing or + initialization), ``tensor.requires_grad_()`` makes it so that autograd will + begin to record operations on ``tensor``. + + Args: + requires_grad (bool): If autograd should record operations on this tensor. + Default: ``True``. + + Example:: + + >>> # Let's say we want to preprocess some saved weights and use + >>> # the result as new weights. + >>> saved_weights = [0.1, 0.2, 0.3, 0.25] + >>> loaded_weights = torch.tensor(saved_weights) + >>> weights = preprocess(loaded_weights) # some function + >>> weights + tensor([-0.5503, 0.4926, -2.1158, -0.8303]) + + >>> # Now, start to record operations done to weights + >>> weights.requires_grad_() + >>> out = weights.pow(2).sum() + >>> out.backward() + >>> weights.grad + tensor([-1.1007, 0.9853, -4.2316, -1.6606]) + """ + + @overload + def reshape(self, shape: Sequence[_int | SymInt]) -> Tensor: + r""" + reshape(*shape) -> Tensor + + Returns a tensor with the same data and number of elements as :attr:`self` + but with the specified shape. This method returns a view if :attr:`shape` is + compatible with the current shape. See :meth:`torch.Tensor.view` on when it is + possible to return a view. + + See :func:`torch.reshape` + + Args: + shape (tuple of ints or int...): the desired shape + """ + + @overload + def reshape(self, *shape: _int | SymInt) -> Tensor: + r""" + reshape(*shape) -> Tensor + + Returns a tensor with the same data and number of elements as :attr:`self` + but with the specified shape. This method returns a view if :attr:`shape` is + compatible with the current shape. See :meth:`torch.Tensor.view` on when it is + possible to return a view. + + See :func:`torch.reshape` + + Args: + shape (tuple of ints or int...): the desired shape + """ + + def reshape_as(self, other: Tensor) -> Tensor: + r""" + reshape_as(other) -> Tensor + + Returns this tensor as the same shape as :attr:`other`. + ``self.reshape_as(other)`` is equivalent to ``self.reshape(other.sizes())``. + This method returns a view if ``other.sizes()`` is compatible with the current + shape. See :meth:`torch.Tensor.view` on when it is possible to return a view. + + Please see :meth:`reshape` for more information about ``reshape``. + + Args: + other (:class:`torch.Tensor`): The result tensor has the same shape + as :attr:`other`. + """ + + @overload + def resize_( + self, + size: Sequence[_int | SymInt], + *, + memory_format: memory_format | None = None, + ) -> Tensor: + r""" + resize_(*sizes, memory_format=torch.contiguous_format) -> Tensor + + Resizes :attr:`self` tensor to the specified size. If the number of elements is + larger than the current storage size, then the underlying storage is resized + to fit the new number of elements. If the number of elements is smaller, the + underlying storage is not changed. Existing elements are preserved but any new + memory is uninitialized. + + .. warning:: + + This is a low-level method. The storage is reinterpreted as C-contiguous, + ignoring the current strides (unless the target size equals the current + size, in which case the tensor is left unchanged). For most purposes, you + will instead want to use :meth:`~Tensor.view()`, which checks for + contiguity, or :meth:`~Tensor.reshape()`, which copies data if needed. To + change the size in-place with custom strides, see :meth:`~Tensor.set_()`. + + .. note:: + + If :func:`torch.use_deterministic_algorithms()` and + :attr:`torch.utils.deterministic.fill_uninitialized_memory` are both set to + ``True``, new elements are initialized to prevent nondeterministic behavior + from using the result as an input to an operation. Floating point and + complex values are set to NaN, and integer values are set to the maximum + value. + + Args: + sizes (torch.Size or int...): the desired size + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + Tensor. Default: ``torch.contiguous_format``. Note that memory format of + :attr:`self` is going to be unaffected if ``self.size()`` matches ``sizes``. + + Example:: + + >>> x = torch.tensor([[1, 2], [3, 4], [5, 6]]) + >>> x.resize_(2, 2) + tensor([[ 1, 2], + [ 3, 4]]) + """ + + @overload + def resize_( + self, + *size: _int | SymInt, + memory_format: memory_format | None = None, + ) -> Tensor: + r""" + resize_(*sizes, memory_format=torch.contiguous_format) -> Tensor + + Resizes :attr:`self` tensor to the specified size. If the number of elements is + larger than the current storage size, then the underlying storage is resized + to fit the new number of elements. If the number of elements is smaller, the + underlying storage is not changed. Existing elements are preserved but any new + memory is uninitialized. + + .. warning:: + + This is a low-level method. The storage is reinterpreted as C-contiguous, + ignoring the current strides (unless the target size equals the current + size, in which case the tensor is left unchanged). For most purposes, you + will instead want to use :meth:`~Tensor.view()`, which checks for + contiguity, or :meth:`~Tensor.reshape()`, which copies data if needed. To + change the size in-place with custom strides, see :meth:`~Tensor.set_()`. + + .. note:: + + If :func:`torch.use_deterministic_algorithms()` and + :attr:`torch.utils.deterministic.fill_uninitialized_memory` are both set to + ``True``, new elements are initialized to prevent nondeterministic behavior + from using the result as an input to an operation. Floating point and + complex values are set to NaN, and integer values are set to the maximum + value. + + Args: + sizes (torch.Size or int...): the desired size + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + Tensor. Default: ``torch.contiguous_format``. Note that memory format of + :attr:`self` is going to be unaffected if ``self.size()`` matches ``sizes``. + + Example:: + + >>> x = torch.tensor([[1, 2], [3, 4], [5, 6]]) + >>> x.resize_(2, 2) + tensor([[ 1, 2], + [ 3, 4]]) + """ + + def resize_as_( + self, + the_template: Tensor, + *, + memory_format: memory_format | None = None, + ) -> Tensor: + r""" + resize_as_(tensor, memory_format=torch.contiguous_format) -> Tensor + + Resizes the :attr:`self` tensor to be the same size as the specified + :attr:`tensor`. This is equivalent to ``self.resize_(tensor.size())``. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + Tensor. Default: ``torch.contiguous_format``. Note that memory format of + :attr:`self` is going to be unaffected if ``self.size()`` matches ``tensor.size()``. + """ + + def resize_as_sparse_(self, the_template: Tensor) -> Tensor: ... + def resolve_conj(self) -> Tensor: + r""" + resolve_conj() -> Tensor + + See :func:`torch.resolve_conj` + """ + + def resolve_neg(self) -> Tensor: + r""" + resolve_neg() -> Tensor + + See :func:`torch.resolve_neg` + """ + + def retain_grad(self) -> None: + r""" + retain_grad() -> None + + Enables this Tensor to have their :attr:`grad` populated during + :func:`backward`. This is a no-op for leaf tensors. + """ + + def roll( + self, + shifts: _int | SymInt | Sequence[_int | SymInt], + dims: _int | _size = (), + ) -> Tensor: + r""" + roll(shifts, dims) -> Tensor + + See :func:`torch.roll` + """ + + def rot90(self, k: _int = 1, dims: _size = (0, 1)) -> Tensor: + r""" + rot90(k, dims) -> Tensor + + See :func:`torch.rot90` + """ + + @overload + def round(self) -> Tensor: + r""" + round(decimals=0) -> Tensor + + See :func:`torch.round` + """ + + @overload + def round(self, *, decimals: _int) -> Tensor: + r""" + round(decimals=0) -> Tensor + + See :func:`torch.round` + """ + + @overload + def round_(self) -> Tensor: + r""" + round_(decimals=0) -> Tensor + + In-place version of :meth:`~Tensor.round` + """ + + @overload + def round_(self, *, decimals: _int) -> Tensor: + r""" + round_(decimals=0) -> Tensor + + In-place version of :meth:`~Tensor.round` + """ + + def row_indices(self) -> Tensor: ... + def rsqrt(self) -> Tensor: + r""" + rsqrt() -> Tensor + + See :func:`torch.rsqrt` + """ + + def rsqrt_(self) -> Tensor: + r""" + rsqrt_() -> Tensor + + In-place version of :meth:`~Tensor.rsqrt` + """ + + @overload + def scatter(self, dim: _int, index: Tensor, src: Tensor) -> Tensor: + r""" + scatter(dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_` + """ + + @overload + def scatter( + self, + dim: _int, + index: Tensor, + src: Tensor, + *, + reduce: str, + ) -> Tensor: + r""" + scatter(dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_` + """ + + @overload + def scatter( + self, + dim: _int, + index: Tensor, + value: Number | _complex, + *, + reduce: str, + ) -> Tensor: + r""" + scatter(dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_` + """ + + @overload + def scatter( + self, + dim: str | EllipsisType | None, + index: Tensor, + src: Tensor, + ) -> Tensor: + r""" + scatter(dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_` + """ + + @overload + def scatter( + self, + dim: _int, + index: Tensor, + value: Number | _complex, + ) -> Tensor: + r""" + scatter(dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_` + """ + + @overload + def scatter( + self, + dim: str | EllipsisType | None, + index: Tensor, + value: Number | _complex, + ) -> Tensor: + r""" + scatter(dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_` + """ + + @overload + def scatter_(self, dim: _int, index: Tensor, src: Tensor) -> Tensor: + r""" + scatter_(dim, index, src, *, reduce=None) -> Tensor + + Writes all values from the tensor :attr:`src` into :attr:`self` at the indices + specified in the :attr:`index` tensor. For each value in :attr:`src`, its output + index is specified by its index in :attr:`src` for ``dimension != dim`` and by + the corresponding value in :attr:`index` for ``dimension = dim``. + + For a 3-D tensor, :attr:`self` is updated as:: + + self[index[i][j][k]][j][k] = src[i][j][k] # if dim == 0 + self[i][index[i][j][k]][k] = src[i][j][k] # if dim == 1 + self[i][j][index[i][j][k]] = src[i][j][k] # if dim == 2 + + This is the reverse operation of the manner described in :meth:`~Tensor.gather`. + + It is also required that + ``index.size(d) <= src.size(d)`` for all dimensions ``d``, and that + ``index.size(d) <= self.size(d)`` for all dimensions ``d != dim``. + Note that ``input`` and ``index`` do not broadcast against each other for NPUs, + so when running on NPUs, :attr:`input` and :attr:`index` must have the same number of dimensions. + Standard broadcasting occurs in all other cases. + + Moreover, as for :meth:`~Tensor.gather`, the values of :attr:`index` must be + between ``0`` and ``self.size(dim) - 1`` inclusive. + + .. warning:: + + When indices are not unique, the behavior is non-deterministic (one of the + values from ``src`` will be picked arbitrarily) and the gradient will be + incorrect (it will be propagated to all locations in the source that + correspond to the same index)! + + .. note:: + + The backward pass is implemented only for ``src.shape == index.shape``. + + Additionally accepts an optional :attr:`reduce` argument that allows + specification of an optional reduction operation, which is applied to all + values in the tensor :attr:`src` into :attr:`self` at the indices + specified in the :attr:`index`. For each value in :attr:`src`, the reduction + operation is applied to an index in :attr:`self` which is specified by + its index in :attr:`src` for ``dimension != dim`` and by the corresponding + value in :attr:`index` for ``dimension = dim``. + + Given a 3-D tensor and reduction using the multiplication operation, :attr:`self` + is updated as:: + + self[index[i][j][k]][j][k] *= src[i][j][k] # if dim == 0 + self[i][index[i][j][k]][k] *= src[i][j][k] # if dim == 1 + self[i][j][index[i][j][k]] *= src[i][j][k] # if dim == 2 + + Reducing with the addition operation is the same as using + :meth:`~torch.Tensor.scatter_add_`. + + .. warning:: + The reduce argument with Tensor ``src`` is deprecated and will be removed in + a future PyTorch release. Please use :meth:`~torch.Tensor.scatter_reduce_` + instead for more reduction options. + + Args: + dim (int): the axis along which to index + index (LongTensor): the indices of elements to scatter, can be either empty + or of the same dimensionality as ``src``. When empty, the operation + returns ``self`` unchanged. + src (Tensor): the source element(s) to scatter. + + Keyword args: + reduce (str, optional): reduction operation to apply, can be either + ``'add'`` or ``'multiply'``. + + Example:: + + >>> src = torch.arange(1, 11).reshape((2, 5)) + >>> src + tensor([[ 1, 2, 3, 4, 5], + [ 6, 7, 8, 9, 10]]) + >>> index = torch.tensor([[0, 1, 2, 0]]) + >>> torch.zeros(3, 5, dtype=src.dtype).scatter_(0, index, src) + tensor([[1, 0, 0, 4, 0], + [0, 2, 0, 0, 0], + [0, 0, 3, 0, 0]]) + >>> index = torch.tensor([[0, 1, 2], [0, 1, 4]]) + >>> torch.zeros(3, 5, dtype=src.dtype).scatter_(1, index, src) + tensor([[1, 2, 3, 0, 0], + [6, 7, 0, 0, 8], + [0, 0, 0, 0, 0]]) + + >>> torch.full((2, 4), 2.).scatter_(1, torch.tensor([[2], [3]]), + ... 1.23, reduce='multiply') + tensor([[2.0000, 2.0000, 2.4600, 2.0000], + [2.0000, 2.0000, 2.0000, 2.4600]]) + >>> torch.full((2, 4), 2.).scatter_(1, torch.tensor([[2], [3]]), + ... 1.23, reduce='add') + tensor([[2.0000, 2.0000, 3.2300, 2.0000], + [2.0000, 2.0000, 2.0000, 3.2300]]) + + .. function:: scatter_(dim, index, value, *, reduce=None) -> Tensor: + :noindex: + + Writes the value from :attr:`value` into :attr:`self` at the indices + specified in the :attr:`index` tensor. This operation is equivalent to the previous version, + with the :attr:`src` tensor filled entirely with :attr:`value`. + + Args: + dim (int): the axis along which to index + index (LongTensor): the indices of elements to scatter, can be either empty + or of the same dimensionality as ``src``. When empty, the operation + returns ``self`` unchanged. + value (Scalar): the value to scatter. + + Keyword args: + reduce (str, optional): reduction operation to apply, can be either + ``'add'`` or ``'multiply'``. + + Example:: + + >>> index = torch.tensor([[0, 1]]) + >>> value = 2 + >>> torch.zeros(3, 5).scatter_(0, index, value) + tensor([[2., 0., 0., 0., 0.], + [0., 2., 0., 0., 0.], + [0., 0., 0., 0., 0.]]) + """ + + @overload + def scatter_( + self, + dim: _int, + index: Tensor, + src: Tensor, + *, + reduce: str, + ) -> Tensor: + r""" + scatter_(dim, index, src, *, reduce=None) -> Tensor + + Writes all values from the tensor :attr:`src` into :attr:`self` at the indices + specified in the :attr:`index` tensor. For each value in :attr:`src`, its output + index is specified by its index in :attr:`src` for ``dimension != dim`` and by + the corresponding value in :attr:`index` for ``dimension = dim``. + + For a 3-D tensor, :attr:`self` is updated as:: + + self[index[i][j][k]][j][k] = src[i][j][k] # if dim == 0 + self[i][index[i][j][k]][k] = src[i][j][k] # if dim == 1 + self[i][j][index[i][j][k]] = src[i][j][k] # if dim == 2 + + This is the reverse operation of the manner described in :meth:`~Tensor.gather`. + + It is also required that + ``index.size(d) <= src.size(d)`` for all dimensions ``d``, and that + ``index.size(d) <= self.size(d)`` for all dimensions ``d != dim``. + Note that ``input`` and ``index`` do not broadcast against each other for NPUs, + so when running on NPUs, :attr:`input` and :attr:`index` must have the same number of dimensions. + Standard broadcasting occurs in all other cases. + + Moreover, as for :meth:`~Tensor.gather`, the values of :attr:`index` must be + between ``0`` and ``self.size(dim) - 1`` inclusive. + + .. warning:: + + When indices are not unique, the behavior is non-deterministic (one of the + values from ``src`` will be picked arbitrarily) and the gradient will be + incorrect (it will be propagated to all locations in the source that + correspond to the same index)! + + .. note:: + + The backward pass is implemented only for ``src.shape == index.shape``. + + Additionally accepts an optional :attr:`reduce` argument that allows + specification of an optional reduction operation, which is applied to all + values in the tensor :attr:`src` into :attr:`self` at the indices + specified in the :attr:`index`. For each value in :attr:`src`, the reduction + operation is applied to an index in :attr:`self` which is specified by + its index in :attr:`src` for ``dimension != dim`` and by the corresponding + value in :attr:`index` for ``dimension = dim``. + + Given a 3-D tensor and reduction using the multiplication operation, :attr:`self` + is updated as:: + + self[index[i][j][k]][j][k] *= src[i][j][k] # if dim == 0 + self[i][index[i][j][k]][k] *= src[i][j][k] # if dim == 1 + self[i][j][index[i][j][k]] *= src[i][j][k] # if dim == 2 + + Reducing with the addition operation is the same as using + :meth:`~torch.Tensor.scatter_add_`. + + .. warning:: + The reduce argument with Tensor ``src`` is deprecated and will be removed in + a future PyTorch release. Please use :meth:`~torch.Tensor.scatter_reduce_` + instead for more reduction options. + + Args: + dim (int): the axis along which to index + index (LongTensor): the indices of elements to scatter, can be either empty + or of the same dimensionality as ``src``. When empty, the operation + returns ``self`` unchanged. + src (Tensor): the source element(s) to scatter. + + Keyword args: + reduce (str, optional): reduction operation to apply, can be either + ``'add'`` or ``'multiply'``. + + Example:: + + >>> src = torch.arange(1, 11).reshape((2, 5)) + >>> src + tensor([[ 1, 2, 3, 4, 5], + [ 6, 7, 8, 9, 10]]) + >>> index = torch.tensor([[0, 1, 2, 0]]) + >>> torch.zeros(3, 5, dtype=src.dtype).scatter_(0, index, src) + tensor([[1, 0, 0, 4, 0], + [0, 2, 0, 0, 0], + [0, 0, 3, 0, 0]]) + >>> index = torch.tensor([[0, 1, 2], [0, 1, 4]]) + >>> torch.zeros(3, 5, dtype=src.dtype).scatter_(1, index, src) + tensor([[1, 2, 3, 0, 0], + [6, 7, 0, 0, 8], + [0, 0, 0, 0, 0]]) + + >>> torch.full((2, 4), 2.).scatter_(1, torch.tensor([[2], [3]]), + ... 1.23, reduce='multiply') + tensor([[2.0000, 2.0000, 2.4600, 2.0000], + [2.0000, 2.0000, 2.0000, 2.4600]]) + >>> torch.full((2, 4), 2.).scatter_(1, torch.tensor([[2], [3]]), + ... 1.23, reduce='add') + tensor([[2.0000, 2.0000, 3.2300, 2.0000], + [2.0000, 2.0000, 2.0000, 3.2300]]) + + .. function:: scatter_(dim, index, value, *, reduce=None) -> Tensor: + :noindex: + + Writes the value from :attr:`value` into :attr:`self` at the indices + specified in the :attr:`index` tensor. This operation is equivalent to the previous version, + with the :attr:`src` tensor filled entirely with :attr:`value`. + + Args: + dim (int): the axis along which to index + index (LongTensor): the indices of elements to scatter, can be either empty + or of the same dimensionality as ``src``. When empty, the operation + returns ``self`` unchanged. + value (Scalar): the value to scatter. + + Keyword args: + reduce (str, optional): reduction operation to apply, can be either + ``'add'`` or ``'multiply'``. + + Example:: + + >>> index = torch.tensor([[0, 1]]) + >>> value = 2 + >>> torch.zeros(3, 5).scatter_(0, index, value) + tensor([[2., 0., 0., 0., 0.], + [0., 2., 0., 0., 0.], + [0., 0., 0., 0., 0.]]) + """ + + @overload + def scatter_( + self, + dim: _int, + index: Tensor, + value: Number | _complex, + *, + reduce: str, + ) -> Tensor: + r""" + scatter_(dim, index, src, *, reduce=None) -> Tensor + + Writes all values from the tensor :attr:`src` into :attr:`self` at the indices + specified in the :attr:`index` tensor. For each value in :attr:`src`, its output + index is specified by its index in :attr:`src` for ``dimension != dim`` and by + the corresponding value in :attr:`index` for ``dimension = dim``. + + For a 3-D tensor, :attr:`self` is updated as:: + + self[index[i][j][k]][j][k] = src[i][j][k] # if dim == 0 + self[i][index[i][j][k]][k] = src[i][j][k] # if dim == 1 + self[i][j][index[i][j][k]] = src[i][j][k] # if dim == 2 + + This is the reverse operation of the manner described in :meth:`~Tensor.gather`. + + It is also required that + ``index.size(d) <= src.size(d)`` for all dimensions ``d``, and that + ``index.size(d) <= self.size(d)`` for all dimensions ``d != dim``. + Note that ``input`` and ``index`` do not broadcast against each other for NPUs, + so when running on NPUs, :attr:`input` and :attr:`index` must have the same number of dimensions. + Standard broadcasting occurs in all other cases. + + Moreover, as for :meth:`~Tensor.gather`, the values of :attr:`index` must be + between ``0`` and ``self.size(dim) - 1`` inclusive. + + .. warning:: + + When indices are not unique, the behavior is non-deterministic (one of the + values from ``src`` will be picked arbitrarily) and the gradient will be + incorrect (it will be propagated to all locations in the source that + correspond to the same index)! + + .. note:: + + The backward pass is implemented only for ``src.shape == index.shape``. + + Additionally accepts an optional :attr:`reduce` argument that allows + specification of an optional reduction operation, which is applied to all + values in the tensor :attr:`src` into :attr:`self` at the indices + specified in the :attr:`index`. For each value in :attr:`src`, the reduction + operation is applied to an index in :attr:`self` which is specified by + its index in :attr:`src` for ``dimension != dim`` and by the corresponding + value in :attr:`index` for ``dimension = dim``. + + Given a 3-D tensor and reduction using the multiplication operation, :attr:`self` + is updated as:: + + self[index[i][j][k]][j][k] *= src[i][j][k] # if dim == 0 + self[i][index[i][j][k]][k] *= src[i][j][k] # if dim == 1 + self[i][j][index[i][j][k]] *= src[i][j][k] # if dim == 2 + + Reducing with the addition operation is the same as using + :meth:`~torch.Tensor.scatter_add_`. + + .. warning:: + The reduce argument with Tensor ``src`` is deprecated and will be removed in + a future PyTorch release. Please use :meth:`~torch.Tensor.scatter_reduce_` + instead for more reduction options. + + Args: + dim (int): the axis along which to index + index (LongTensor): the indices of elements to scatter, can be either empty + or of the same dimensionality as ``src``. When empty, the operation + returns ``self`` unchanged. + src (Tensor): the source element(s) to scatter. + + Keyword args: + reduce (str, optional): reduction operation to apply, can be either + ``'add'`` or ``'multiply'``. + + Example:: + + >>> src = torch.arange(1, 11).reshape((2, 5)) + >>> src + tensor([[ 1, 2, 3, 4, 5], + [ 6, 7, 8, 9, 10]]) + >>> index = torch.tensor([[0, 1, 2, 0]]) + >>> torch.zeros(3, 5, dtype=src.dtype).scatter_(0, index, src) + tensor([[1, 0, 0, 4, 0], + [0, 2, 0, 0, 0], + [0, 0, 3, 0, 0]]) + >>> index = torch.tensor([[0, 1, 2], [0, 1, 4]]) + >>> torch.zeros(3, 5, dtype=src.dtype).scatter_(1, index, src) + tensor([[1, 2, 3, 0, 0], + [6, 7, 0, 0, 8], + [0, 0, 0, 0, 0]]) + + >>> torch.full((2, 4), 2.).scatter_(1, torch.tensor([[2], [3]]), + ... 1.23, reduce='multiply') + tensor([[2.0000, 2.0000, 2.4600, 2.0000], + [2.0000, 2.0000, 2.0000, 2.4600]]) + >>> torch.full((2, 4), 2.).scatter_(1, torch.tensor([[2], [3]]), + ... 1.23, reduce='add') + tensor([[2.0000, 2.0000, 3.2300, 2.0000], + [2.0000, 2.0000, 2.0000, 3.2300]]) + + .. function:: scatter_(dim, index, value, *, reduce=None) -> Tensor: + :noindex: + + Writes the value from :attr:`value` into :attr:`self` at the indices + specified in the :attr:`index` tensor. This operation is equivalent to the previous version, + with the :attr:`src` tensor filled entirely with :attr:`value`. + + Args: + dim (int): the axis along which to index + index (LongTensor): the indices of elements to scatter, can be either empty + or of the same dimensionality as ``src``. When empty, the operation + returns ``self`` unchanged. + value (Scalar): the value to scatter. + + Keyword args: + reduce (str, optional): reduction operation to apply, can be either + ``'add'`` or ``'multiply'``. + + Example:: + + >>> index = torch.tensor([[0, 1]]) + >>> value = 2 + >>> torch.zeros(3, 5).scatter_(0, index, value) + tensor([[2., 0., 0., 0., 0.], + [0., 2., 0., 0., 0.], + [0., 0., 0., 0., 0.]]) + """ + + @overload + def scatter_( + self, + dim: _int, + index: Tensor, + value: Number | _complex, + ) -> Tensor: + r""" + scatter_(dim, index, src, *, reduce=None) -> Tensor + + Writes all values from the tensor :attr:`src` into :attr:`self` at the indices + specified in the :attr:`index` tensor. For each value in :attr:`src`, its output + index is specified by its index in :attr:`src` for ``dimension != dim`` and by + the corresponding value in :attr:`index` for ``dimension = dim``. + + For a 3-D tensor, :attr:`self` is updated as:: + + self[index[i][j][k]][j][k] = src[i][j][k] # if dim == 0 + self[i][index[i][j][k]][k] = src[i][j][k] # if dim == 1 + self[i][j][index[i][j][k]] = src[i][j][k] # if dim == 2 + + This is the reverse operation of the manner described in :meth:`~Tensor.gather`. + + It is also required that + ``index.size(d) <= src.size(d)`` for all dimensions ``d``, and that + ``index.size(d) <= self.size(d)`` for all dimensions ``d != dim``. + Note that ``input`` and ``index`` do not broadcast against each other for NPUs, + so when running on NPUs, :attr:`input` and :attr:`index` must have the same number of dimensions. + Standard broadcasting occurs in all other cases. + + Moreover, as for :meth:`~Tensor.gather`, the values of :attr:`index` must be + between ``0`` and ``self.size(dim) - 1`` inclusive. + + .. warning:: + + When indices are not unique, the behavior is non-deterministic (one of the + values from ``src`` will be picked arbitrarily) and the gradient will be + incorrect (it will be propagated to all locations in the source that + correspond to the same index)! + + .. note:: + + The backward pass is implemented only for ``src.shape == index.shape``. + + Additionally accepts an optional :attr:`reduce` argument that allows + specification of an optional reduction operation, which is applied to all + values in the tensor :attr:`src` into :attr:`self` at the indices + specified in the :attr:`index`. For each value in :attr:`src`, the reduction + operation is applied to an index in :attr:`self` which is specified by + its index in :attr:`src` for ``dimension != dim`` and by the corresponding + value in :attr:`index` for ``dimension = dim``. + + Given a 3-D tensor and reduction using the multiplication operation, :attr:`self` + is updated as:: + + self[index[i][j][k]][j][k] *= src[i][j][k] # if dim == 0 + self[i][index[i][j][k]][k] *= src[i][j][k] # if dim == 1 + self[i][j][index[i][j][k]] *= src[i][j][k] # if dim == 2 + + Reducing with the addition operation is the same as using + :meth:`~torch.Tensor.scatter_add_`. + + .. warning:: + The reduce argument with Tensor ``src`` is deprecated and will be removed in + a future PyTorch release. Please use :meth:`~torch.Tensor.scatter_reduce_` + instead for more reduction options. + + Args: + dim (int): the axis along which to index + index (LongTensor): the indices of elements to scatter, can be either empty + or of the same dimensionality as ``src``. When empty, the operation + returns ``self`` unchanged. + src (Tensor): the source element(s) to scatter. + + Keyword args: + reduce (str, optional): reduction operation to apply, can be either + ``'add'`` or ``'multiply'``. + + Example:: + + >>> src = torch.arange(1, 11).reshape((2, 5)) + >>> src + tensor([[ 1, 2, 3, 4, 5], + [ 6, 7, 8, 9, 10]]) + >>> index = torch.tensor([[0, 1, 2, 0]]) + >>> torch.zeros(3, 5, dtype=src.dtype).scatter_(0, index, src) + tensor([[1, 0, 0, 4, 0], + [0, 2, 0, 0, 0], + [0, 0, 3, 0, 0]]) + >>> index = torch.tensor([[0, 1, 2], [0, 1, 4]]) + >>> torch.zeros(3, 5, dtype=src.dtype).scatter_(1, index, src) + tensor([[1, 2, 3, 0, 0], + [6, 7, 0, 0, 8], + [0, 0, 0, 0, 0]]) + + >>> torch.full((2, 4), 2.).scatter_(1, torch.tensor([[2], [3]]), + ... 1.23, reduce='multiply') + tensor([[2.0000, 2.0000, 2.4600, 2.0000], + [2.0000, 2.0000, 2.0000, 2.4600]]) + >>> torch.full((2, 4), 2.).scatter_(1, torch.tensor([[2], [3]]), + ... 1.23, reduce='add') + tensor([[2.0000, 2.0000, 3.2300, 2.0000], + [2.0000, 2.0000, 2.0000, 3.2300]]) + + .. function:: scatter_(dim, index, value, *, reduce=None) -> Tensor: + :noindex: + + Writes the value from :attr:`value` into :attr:`self` at the indices + specified in the :attr:`index` tensor. This operation is equivalent to the previous version, + with the :attr:`src` tensor filled entirely with :attr:`value`. + + Args: + dim (int): the axis along which to index + index (LongTensor): the indices of elements to scatter, can be either empty + or of the same dimensionality as ``src``. When empty, the operation + returns ``self`` unchanged. + value (Scalar): the value to scatter. + + Keyword args: + reduce (str, optional): reduction operation to apply, can be either + ``'add'`` or ``'multiply'``. + + Example:: + + >>> index = torch.tensor([[0, 1]]) + >>> value = 2 + >>> torch.zeros(3, 5).scatter_(0, index, value) + tensor([[2., 0., 0., 0., 0.], + [0., 2., 0., 0., 0.], + [0., 0., 0., 0., 0.]]) + """ + + @overload + def scatter_add(self, dim: _int, index: Tensor, src: Tensor) -> Tensor: + r""" + scatter_add(dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_add_` + """ + + @overload + def scatter_add( + self, + dim: str | EllipsisType | None, + index: Tensor, + src: Tensor, + ) -> Tensor: + r""" + scatter_add(dim, index, src) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_add_` + """ + + def scatter_add_(self, dim: _int, index: Tensor, src: Tensor) -> Tensor: + r""" + scatter_add_(dim, index, src) -> Tensor + + Adds all values from the tensor :attr:`src` into :attr:`self` at the indices + specified in the :attr:`index` tensor in a similar fashion as + :meth:`~torch.Tensor.scatter_`. For each value in :attr:`src`, it is added to + an index in :attr:`self` which is specified by its index in :attr:`src` + for ``dimension != dim`` and by the corresponding value in :attr:`index` for + ``dimension = dim``. + + For a 3-D tensor, :attr:`self` is updated as:: + + self[index[i][j][k]][j][k] += src[i][j][k] # if dim == 0 + self[i][index[i][j][k]][k] += src[i][j][k] # if dim == 1 + self[i][j][index[i][j][k]] += src[i][j][k] # if dim == 2 + + :attr:`self`, :attr:`index` and :attr:`src` should have same number of + dimensions. It is also required that ``index.size(d) <= src.size(d)`` for all + dimensions ``d``, and that ``index.size(d) <= self.size(d)`` for all dimensions + ``d != dim``. Note that ``index`` and ``src`` do not broadcast. + When :attr:`index` is empty, we always return the original tensor + without further error checking. + + Note: + This operation may behave nondeterministically when given tensors on a CUDA device. See :doc:`/notes/randomness` for more information. + + .. note:: + + The backward pass is implemented only for ``src.shape == index.shape``. + + Args: + dim (int): the axis along which to index + index (LongTensor): the indices of elements to scatter and add, can be + either empty or of the same dimensionality as ``src``. When empty, the + operation returns ``self`` unchanged. + src (Tensor): the source elements to scatter and add + + Example:: + + >>> src = torch.ones((2, 5)) + >>> index = torch.tensor([[0, 1, 2, 0, 0]]) + >>> torch.zeros(3, 5, dtype=src.dtype).scatter_add_(0, index, src) + tensor([[1., 0., 0., 1., 1.], + [0., 1., 0., 0., 0.], + [0., 0., 1., 0., 0.]]) + >>> index = torch.tensor([[0, 1, 2, 0, 0], [0, 1, 2, 2, 2]]) + >>> torch.zeros(3, 5, dtype=src.dtype).scatter_add_(0, index, src) + tensor([[2., 0., 0., 1., 1.], + [0., 2., 0., 0., 0.], + [0., 0., 2., 1., 1.]]) + """ + + def scatter_reduce( + self, + dim: _int, + index: Tensor, + src: Tensor, + reduce: str, + *, + include_self: _bool = True, + ) -> Tensor: + r""" + scatter_reduce(dim, index, src, reduce, *, include_self=True) -> Tensor + + Out-of-place version of :meth:`torch.Tensor.scatter_reduce_` + """ + + def scatter_reduce_( + self, + dim: _int, + index: Tensor, + src: Tensor, + reduce: str, + *, + include_self: _bool = True, + ) -> Tensor: + r""" + scatter_reduce_(dim, index, src, reduce, *, include_self=True) -> Tensor + + Reduces all values from the :attr:`src` tensor to the indices specified in + the :attr:`index` tensor in the :attr:`self` tensor using the applied reduction + defined via the :attr:`reduce` argument (:obj:`"sum"`, :obj:`"prod"`, :obj:`"mean"`, + :obj:`"amax"`, :obj:`"amin"`). For each value in :attr:`src`, it is reduced to an + index in :attr:`self` which is specified by its index in :attr:`src` for + ``dimension != dim`` and by the corresponding value in :attr:`index` for + ``dimension = dim``. If :obj:`include_self="True"`, the values in the :attr:`self` + tensor are included in the reduction. + + :attr:`self`, :attr:`index` and :attr:`src` should all have + the same number of dimensions. It is also required that + ``index.size(d) <= src.size(d)`` for all dimensions ``d``, and that + ``index.size(d) <= self.size(d)`` for all dimensions ``d != dim``. + Note that ``index`` and ``src`` do not broadcast. + + For a 3-D tensor with :obj:`reduce="sum"` and :obj:`include_self=True` the + output is given as:: + + self[index[i][j][k]][j][k] += src[i][j][k] # if dim == 0 + self[i][index[i][j][k]][k] += src[i][j][k] # if dim == 1 + self[i][j][index[i][j][k]] += src[i][j][k] # if dim == 2 + + Note: + This operation may behave nondeterministically when given tensors on a CUDA device. See :doc:`/notes/randomness` for more information. + + .. note:: + + The backward pass is implemented only for ``src.shape == index.shape``. + + .. warning:: + + This function is in beta and may change in the near future. + + Args: + dim (int): the axis along which to index + index (LongTensor): the indices of elements to scatter and reduce. + src (Tensor): the source elements to scatter and reduce + reduce (str): the reduction operation to apply for non-unique indices + (:obj:`"sum"`, :obj:`"prod"`, :obj:`"mean"`, :obj:`"amax"`, :obj:`"amin"`) + include_self (bool): whether elements from the :attr:`self` tensor are + included in the reduction + + Example:: + + >>> src = torch.tensor([1., 2., 3., 4., 5., 6.]) + >>> index = torch.tensor([0, 1, 0, 1, 2, 1]) + >>> input = torch.tensor([1., 2., 3., 4.]) + >>> input.scatter_reduce(0, index, src, reduce="sum") + tensor([5., 14., 8., 4.]) + >>> input.scatter_reduce(0, index, src, reduce="sum", include_self=False) + tensor([4., 12., 5., 4.]) + >>> input2 = torch.tensor([5., 4., 3., 2.]) + >>> input2.scatter_reduce(0, index, src, reduce="amax") + tensor([5., 6., 5., 2.]) + >>> input2.scatter_reduce(0, index, src, reduce="amax", include_self=False) + tensor([3., 6., 5., 2.]) + """ + + @overload + def select(self, dim: _int, index: _int | SymInt) -> Tensor: + r""" + select(dim, index) -> Tensor + + See :func:`torch.select` + """ + + @overload + def select(self, dim: str | EllipsisType | None, index: _int) -> Tensor: + r""" + select(dim, index) -> Tensor + + See :func:`torch.select` + """ + + def select_scatter( + self, + src: Tensor, + dim: _int, + index: _int | SymInt, + ) -> Tensor: + r""" + select_scatter(src, dim, index) -> Tensor + + See :func:`torch.select_scatter` + """ + + @overload + def set_( + self, + source: Storage | TypedStorage | UntypedStorage, + storage_offset: IntLikeType, + size: _symsize, + stride: _symsize, + ) -> Tensor: + r""" + set_(source=None, storage_offset=0, size=None, stride=None) -> Tensor + + Sets the underlying storage, size, and strides. If :attr:`source` is a tensor, + :attr:`self` tensor will share the same storage and have the same size and + strides as :attr:`source`. Changes to elements in one tensor will be reflected + in the other. + + If :attr:`source` is a :class:`~torch.Storage`, the method sets the underlying + storage, offset, size, and stride. + + Args: + source (Tensor or Storage): the tensor or storage to use + storage_offset (int, optional): the offset in the storage + size (torch.Size, optional): the desired size. Defaults to the size of the source. + stride (tuple, optional): the desired stride. Defaults to C-contiguous strides. + """ + + @overload + def set_(self, source: Storage | TypedStorage | UntypedStorage) -> Tensor: + r""" + set_(source=None, storage_offset=0, size=None, stride=None) -> Tensor + + Sets the underlying storage, size, and strides. If :attr:`source` is a tensor, + :attr:`self` tensor will share the same storage and have the same size and + strides as :attr:`source`. Changes to elements in one tensor will be reflected + in the other. + + If :attr:`source` is a :class:`~torch.Storage`, the method sets the underlying + storage, offset, size, and stride. + + Args: + source (Tensor or Storage): the tensor or storage to use + storage_offset (int, optional): the offset in the storage + size (torch.Size, optional): the desired size. Defaults to the size of the source. + stride (tuple, optional): the desired stride. Defaults to C-contiguous strides. + """ + + def sgn(self) -> Tensor: + r""" + sgn() -> Tensor + + See :func:`torch.sgn` + """ + + def sgn_(self) -> Tensor: + r""" + sgn_() -> Tensor + + In-place version of :meth:`~Tensor.sgn` + """ + + def short(self) -> Tensor: + r""" + short(memory_format=torch.preserve_format) -> Tensor + + ``self.short()`` is equivalent to ``self.to(torch.int16)``. See :func:`to`. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + def sigmoid(self) -> Tensor: + r""" + sigmoid() -> Tensor + + See :func:`torch.sigmoid` + """ + + def sigmoid_(self) -> Tensor: + r""" + sigmoid_() -> Tensor + + In-place version of :meth:`~Tensor.sigmoid` + """ + + def sign(self) -> Tensor: + r""" + sign() -> Tensor + + See :func:`torch.sign` + """ + + def sign_(self) -> Tensor: + r""" + sign_() -> Tensor + + In-place version of :meth:`~Tensor.sign` + """ + + def signbit(self) -> Tensor: + r""" + signbit() -> Tensor + + See :func:`torch.signbit` + """ + + def sin(self) -> Tensor: + r""" + sin() -> Tensor + + See :func:`torch.sin` + """ + + def sin_(self) -> Tensor: + r""" + sin_() -> Tensor + + In-place version of :meth:`~Tensor.sin` + """ + + def sinc(self) -> Tensor: + r""" + sinc() -> Tensor + + See :func:`torch.sinc` + """ + + def sinc_(self) -> Tensor: + r""" + sinc_() -> Tensor + + In-place version of :meth:`~Tensor.sinc` + """ + + def sinh(self) -> Tensor: + r""" + sinh() -> Tensor + + See :func:`torch.sinh` + """ + + def sinh_(self) -> Tensor: + r""" + sinh_() -> Tensor + + In-place version of :meth:`~Tensor.sinh` + """ + + @overload + def size(self, dim: None = None) -> Size: + r""" + size(dim=None) -> torch.Size or int + + Returns the size of the :attr:`self` tensor. If ``dim`` is not specified, + the returned value is a :class:`torch.Size`, a subclass of :class:`tuple`. + If ``dim`` is specified, returns an int holding the size of that dimension. + + Args: + dim (int, optional): The dimension for which to retrieve the size. + + Example:: + + >>> t = torch.empty(3, 4, 5) + >>> t.size() + torch.Size([3, 4, 5]) + >>> t.size(dim=1) + 4 + """ + + @overload + def size(self, dim: _int) -> _int: + r""" + size(dim=None) -> torch.Size or int + + Returns the size of the :attr:`self` tensor. If ``dim`` is not specified, + the returned value is a :class:`torch.Size`, a subclass of :class:`tuple`. + If ``dim`` is specified, returns an int holding the size of that dimension. + + Args: + dim (int, optional): The dimension for which to retrieve the size. + + Example:: + + >>> t = torch.empty(3, 4, 5) + >>> t.size() + torch.Size([3, 4, 5]) + >>> t.size(dim=1) + 4 + """ + + def slice_inverse( + self, + src: Tensor, + dim: _int = 0, + start: _int | SymInt | None = None, + end: _int | SymInt | None = None, + step: _int | SymInt = 1, + ) -> Tensor: ... + def slice_scatter( + self, + src: Tensor, + dim: _int = 0, + start: _int | SymInt | None = None, + end: _int | SymInt | None = None, + step: _int | SymInt = 1, + ) -> Tensor: + r""" + slice_scatter(src, dim=0, start=None, end=None, step=1) -> Tensor + + See :func:`torch.slice_scatter` + """ + + def slogdet(self) -> torch.return_types.slogdet: + r""" + slogdet() -> (Tensor, Tensor) + + See :func:`torch.slogdet` + """ + + def smm(self, mat2: Tensor) -> Tensor: + r""" + smm(mat) -> Tensor + + See :func:`torch.smm` + """ + + @overload + def softmax(self, dim: _int, dtype: _dtype | None = None) -> Tensor: + r""" + softmax(dim) -> Tensor + + Alias for :func:`torch.nn.functional.softmax`. + """ + + @overload + def softmax( + self, + dim: str | EllipsisType | None, + *, + dtype: _dtype | None = None, + ) -> Tensor: + r""" + softmax(dim) -> Tensor + + Alias for :func:`torch.nn.functional.softmax`. + """ + + @overload + def sort( + self, + *, + stable: _bool | None, + dim: _int = -1, + descending: _bool = False, + ) -> torch.return_types.sort: + r""" + sort(dim=-1, descending=False) -> (Tensor, LongTensor) + + See :func:`torch.sort` + """ + + @overload + def sort( + self, + dim: _int = -1, + descending: _bool = False, + ) -> torch.return_types.sort: + r""" + sort(dim=-1, descending=False) -> (Tensor, LongTensor) + + See :func:`torch.sort` + """ + + @overload + def sort( + self, + *, + stable: _bool | None, + dim: str | EllipsisType | None, + descending: _bool = False, + ) -> torch.return_types.sort: + r""" + sort(dim=-1, descending=False) -> (Tensor, LongTensor) + + See :func:`torch.sort` + """ + + @overload + def sort( + self, + dim: str | EllipsisType | None, + descending: _bool = False, + ) -> torch.return_types.sort: + r""" + sort(dim=-1, descending=False) -> (Tensor, LongTensor) + + See :func:`torch.sort` + """ + + def sparse_dim(self) -> _int: + r""" + sparse_dim() -> int + + Return the number of sparse dimensions in a :ref:`sparse tensor ` :attr:`self`. + + .. note:: + Returns ``0`` if :attr:`self` is not a sparse tensor. + + See also :meth:`Tensor.dense_dim` and :ref:`hybrid tensors `. + """ + + def sparse_mask(self, mask: Tensor) -> Tensor: + r""" + sparse_mask(mask) -> Tensor + + Returns a new :ref:`sparse tensor ` with values from a + strided tensor :attr:`self` filtered by the indices of the sparse + tensor :attr:`mask`. The values of :attr:`mask` sparse tensor are + ignored. :attr:`self` and :attr:`mask` tensors must have the same + shape. + + .. note:: + + The returned sparse tensor might contain duplicate values if :attr:`mask` + is not coalesced. It is therefore advisable to pass ``mask.coalesce()`` + if such behavior is not desired. + + .. note:: + + The returned sparse tensor has the same indices as the sparse tensor + :attr:`mask`, even when the corresponding values in :attr:`self` are + zeros. + + Args: + mask (Tensor): a sparse tensor whose indices are used as a filter + + Example:: + + >>> nse = 5 + >>> dims = (5, 5, 2, 2) + >>> I = torch.cat([torch.randint(0, dims[0], size=(nse,)), + ... torch.randint(0, dims[1], size=(nse,))], 0).reshape(2, nse) + >>> V = torch.randn(nse, dims[2], dims[3]) + >>> S = torch.sparse_coo_tensor(I, V, dims).coalesce() + >>> D = torch.randn(dims) + >>> D.sparse_mask(S) + tensor(indices=tensor([[0, 0, 0, 2], + [0, 1, 4, 3]]), + values=tensor([[[ 1.6550, 0.2397], + [-0.1611, -0.0779]], + + [[ 0.2326, -1.0558], + [ 1.4711, 1.9678]], + + [[-0.5138, -0.0411], + [ 1.9417, 0.5158]], + + [[ 0.0793, 0.0036], + [-0.2569, -0.1055]]]), + size=(5, 5, 2, 2), nnz=4, layout=torch.sparse_coo) + """ + + def sparse_resize_( + self, + size: _size, + sparse_dim: _int, + dense_dim: _int, + ) -> Tensor: + r""" + sparse_resize_(size, sparse_dim, dense_dim) -> Tensor + + Resizes :attr:`self` :ref:`sparse tensor ` to the desired + size and the number of sparse and dense dimensions. + + .. note:: + If the number of specified elements in :attr:`self` is zero, then + :attr:`size`, :attr:`sparse_dim`, and :attr:`dense_dim` can be any + size and positive integers such that ``len(size) == sparse_dim + + dense_dim``. + + If :attr:`self` specifies one or more elements, however, then each + dimension in :attr:`size` must not be smaller than the corresponding + dimension of :attr:`self`, :attr:`sparse_dim` must equal the number + of sparse dimensions in :attr:`self`, and :attr:`dense_dim` must + equal the number of dense dimensions in :attr:`self`. + + .. warning:: + Throws an error if :attr:`self` is not a sparse tensor. + + Args: + size (torch.Size): the desired size. If :attr:`self` is non-empty + sparse tensor, the desired size cannot be smaller than the + original size. + sparse_dim (int): the number of sparse dimensions + dense_dim (int): the number of dense dimensions + """ + + def sparse_resize_and_clear_( + self, + size: _size, + sparse_dim: _int, + dense_dim: _int, + ) -> Tensor: + r""" + sparse_resize_and_clear_(size, sparse_dim, dense_dim) -> Tensor + + Removes all specified elements from a :ref:`sparse tensor + ` :attr:`self` and resizes :attr:`self` to the desired + size and the number of sparse and dense dimensions. + + .. warning: + Throws an error if :attr:`self` is not a sparse tensor. + + Args: + size (torch.Size): the desired size. + sparse_dim (int): the number of sparse dimensions + dense_dim (int): the number of dense dimensions + """ + + @overload + def split(self, split_size: _int, dim: _int = 0) -> Sequence[Tensor]: ... + @overload + def split( + self, + split_size: tuple[_int, ...], + dim: _int = 0, + ) -> Sequence[Tensor]: ... + def split_with_sizes( + self, + split_sizes: Sequence[_int | SymInt], + dim: _int = 0, + ) -> tuple[Tensor, ...]: ... + def sqrt(self) -> Tensor: + r""" + sqrt() -> Tensor + + See :func:`torch.sqrt` + """ + + def sqrt_(self) -> Tensor: + r""" + sqrt_() -> Tensor + + In-place version of :meth:`~Tensor.sqrt` + """ + + def square(self) -> Tensor: + r""" + square() -> Tensor + + See :func:`torch.square` + """ + + def square_(self) -> Tensor: + r""" + square_() -> Tensor + + In-place version of :meth:`~Tensor.square` + """ + + @overload + def squeeze(self) -> Tensor: + r""" + squeeze(dim=None) -> Tensor + + See :func:`torch.squeeze` + """ + + @overload + def squeeze(self, dim: _int) -> Tensor: + r""" + squeeze(dim=None) -> Tensor + + See :func:`torch.squeeze` + """ + + @overload + def squeeze(self, dim: _size) -> Tensor: + r""" + squeeze(dim=None) -> Tensor + + See :func:`torch.squeeze` + """ + + @overload + def squeeze(self, *dim: _int) -> Tensor: + r""" + squeeze(dim=None) -> Tensor + + See :func:`torch.squeeze` + """ + + @overload + def squeeze(self, dim: str | EllipsisType | None) -> Tensor: + r""" + squeeze(dim=None) -> Tensor + + See :func:`torch.squeeze` + """ + + @overload + def squeeze_(self) -> Tensor: + r""" + squeeze_(dim=None) -> Tensor + + In-place version of :meth:`~Tensor.squeeze` + """ + + @overload + def squeeze_(self, dim: _int) -> Tensor: + r""" + squeeze_(dim=None) -> Tensor + + In-place version of :meth:`~Tensor.squeeze` + """ + + @overload + def squeeze_(self, dim: _size) -> Tensor: + r""" + squeeze_(dim=None) -> Tensor + + In-place version of :meth:`~Tensor.squeeze` + """ + + @overload + def squeeze_(self, *dim: _int) -> Tensor: + r""" + squeeze_(dim=None) -> Tensor + + In-place version of :meth:`~Tensor.squeeze` + """ + + @overload + def squeeze_(self, dim: str | EllipsisType | None) -> Tensor: + r""" + squeeze_(dim=None) -> Tensor + + In-place version of :meth:`~Tensor.squeeze` + """ + + def sspaddmm( + self, + mat1: Tensor, + mat2: Tensor, + *, + beta: Number | _complex = 1, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + sspaddmm(mat1, mat2, *, beta=1, alpha=1) -> Tensor + + See :func:`torch.sspaddmm` + """ + + @overload + def std( + self, + dim: _int | _size | None, + unbiased: _bool = True, + keepdim: _bool = False, + ) -> Tensor: + r""" + std(dim=None, *, correction=1, keepdim=False) -> Tensor + + See :func:`torch.std` + """ + + @overload + def std( + self, + dim: _int | _size | None = None, + *, + correction: Number | _complex | None = None, + keepdim: _bool = False, + ) -> Tensor: + r""" + std(dim=None, *, correction=1, keepdim=False) -> Tensor + + See :func:`torch.std` + """ + + @overload + def std(self, unbiased: _bool = True) -> Tensor: + r""" + std(dim=None, *, correction=1, keepdim=False) -> Tensor + + See :func:`torch.std` + """ + + @overload + def std( + self, + dim: Sequence[str | EllipsisType | None], + unbiased: _bool = True, + keepdim: _bool = False, + ) -> Tensor: + r""" + std(dim=None, *, correction=1, keepdim=False) -> Tensor + + See :func:`torch.std` + """ + + @overload + def std( + self, + dim: Sequence[str | EllipsisType | None], + *, + correction: Number | _complex | None = None, + keepdim: _bool = False, + ) -> Tensor: + r""" + std(dim=None, *, correction=1, keepdim=False) -> Tensor + + See :func:`torch.std` + """ + + def untyped_storage(self) -> UntypedStorage: ... + def storage_offset(self) -> _int | SymInt: + r""" + storage_offset() -> int + + Returns :attr:`self` tensor's offset in the underlying storage in terms of + number of storage elements (not bytes). + + Example:: + + >>> x = torch.tensor([1, 2, 3, 4, 5]) + >>> x.storage_offset() + 0 + >>> x[3:].storage_offset() + 3 + """ + + def storage_type(self) -> Storage: ... + @overload + def stride(self, dim: None = None) -> tuple[_int, ...]: + r""" + stride(dim) -> tuple or int + + Returns the stride of :attr:`self` tensor. + + Stride is the jump necessary to go from one element to the next one in the + specified dimension :attr:`dim`. A tuple of all strides is returned when no + argument is passed in. Otherwise, an integer value is returned as the stride in + the particular dimension :attr:`dim`. + + Args: + dim (int, optional): the desired dimension in which stride is required + + Example:: + + >>> x = torch.tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]) + >>> x.stride() + (5, 1) + >>> x.stride(0) + 5 + >>> x.stride(-1) + 1 + """ + + @overload + def stride(self, dim: _int) -> _int: + r""" + stride(dim) -> tuple or int + + Returns the stride of :attr:`self` tensor. + + Stride is the jump necessary to go from one element to the next one in the + specified dimension :attr:`dim`. A tuple of all strides is returned when no + argument is passed in. Otherwise, an integer value is returned as the stride in + the particular dimension :attr:`dim`. + + Args: + dim (int, optional): the desired dimension in which stride is required + + Example:: + + >>> x = torch.tensor([[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]) + >>> x.stride() + (5, 1) + >>> x.stride(0) + 5 + >>> x.stride(-1) + 1 + """ + + def sub( + self, + other: Tensor | Number | _complex | torch.SymInt | torch.SymFloat, + *, + alpha: Number | _complex | None = 1, + out: Tensor | None = None, + ) -> Tensor: + r""" + sub(other, *, alpha=1) -> Tensor + + See :func:`torch.sub`. + """ + + def sub_( + self, + other: Tensor | Number | _complex | torch.SymInt | torch.SymFloat, + *, + alpha: Number | _complex | None = 1, + ) -> Tensor: + r""" + sub_(other, *, alpha=1) -> Tensor + + In-place version of :meth:`~Tensor.sub` + """ + + @overload + def subtract( + self, + other: Tensor, + *, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + subtract(other, *, alpha=1) -> Tensor + + See :func:`torch.subtract`. + """ + + @overload + def subtract( + self, + other: Number | _complex, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + subtract(other, *, alpha=1) -> Tensor + + See :func:`torch.subtract`. + """ + + @overload + def subtract_( + self, + other: Tensor, + *, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + subtract_(other, *, alpha=1) -> Tensor + + In-place version of :meth:`~Tensor.subtract`. + """ + + @overload + def subtract_( + self, + other: Number | _complex, + alpha: Number | _complex = 1, + ) -> Tensor: + r""" + subtract_(other, *, alpha=1) -> Tensor + + In-place version of :meth:`~Tensor.subtract`. + """ + + @overload + def sum(self, *, dtype: _dtype | None = None) -> Tensor: + r""" + sum(dim=None, keepdim=False, dtype=None) -> Tensor + + See :func:`torch.sum` + """ + + @overload + def sum( + self, + dim: _int | _size | None, + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + ) -> Tensor: + r""" + sum(dim=None, keepdim=False, dtype=None) -> Tensor + + See :func:`torch.sum` + """ + + @overload + def sum( + self, + dim: Sequence[str | EllipsisType | None], + keepdim: _bool = False, + *, + dtype: _dtype | None = None, + ) -> Tensor: + r""" + sum(dim=None, keepdim=False, dtype=None) -> Tensor + + See :func:`torch.sum` + """ + + @overload + def sum_to_size(self, size: Sequence[_int | SymInt]) -> Tensor: + r""" + sum_to_size(*size) -> Tensor + + Sum ``this`` tensor to :attr:`size`. + :attr:`size` must be broadcastable to ``this`` tensor size. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + """ + + @overload + def sum_to_size(self, *size: _int | SymInt) -> Tensor: + r""" + sum_to_size(*size) -> Tensor + + Sum ``this`` tensor to :attr:`size`. + :attr:`size` must be broadcastable to ``this`` tensor size. + + Args: + size (int...): a sequence of integers defining the shape of the output tensor. + """ + + def svd( + self, + some: _bool = True, + compute_uv: _bool = True, + ) -> torch.return_types.svd: + r""" + svd(some=True, compute_uv=True) -> (Tensor, Tensor, Tensor) + + See :func:`torch.svd` + """ + + def swapaxes(self, axis0: _int, axis1: _int) -> Tensor: + r""" + swapaxes(axis0, axis1) -> Tensor + + See :func:`torch.swapaxes` + """ + + def swapaxes_(self, axis0: _int, axis1: _int) -> Tensor: + r""" + swapaxes_(axis0, axis1) -> Tensor + + In-place version of :meth:`~Tensor.swapaxes` + """ + + def swapdims(self, dim0: _int, dim1: _int) -> Tensor: + r""" + swapdims(dim0, dim1) -> Tensor + + See :func:`torch.swapdims` + """ + + def swapdims_(self, dim0: _int, dim1: _int) -> Tensor: + r""" + swapdims_(dim0, dim1) -> Tensor + + In-place version of :meth:`~Tensor.swapdims` + """ + + def t(self) -> Tensor: + r""" + t() -> Tensor + + See :func:`torch.t` + """ + + def t_(self) -> Tensor: + r""" + t_() -> Tensor + + In-place version of :meth:`~Tensor.t` + """ + + def take(self, index: Tensor) -> Tensor: + r""" + take(indices) -> Tensor + + See :func:`torch.take` + """ + + def take_along_dim( + self, + indices: Tensor, + dim: _int | None = None, + ) -> Tensor: + r""" + take_along_dim(indices, dim) -> Tensor + + See :func:`torch.take_along_dim` + """ + + def tan(self) -> Tensor: + r""" + tan() -> Tensor + + See :func:`torch.tan` + """ + + def tan_(self) -> Tensor: + r""" + tan_() -> Tensor + + In-place version of :meth:`~Tensor.tan` + """ + + def tanh(self) -> Tensor: + r""" + tanh() -> Tensor + + See :func:`torch.tanh` + """ + + def tanh_(self) -> Tensor: + r""" + tanh_() -> Tensor + + In-place version of :meth:`~Tensor.tanh` + """ + + @overload + def tensor_split( + self, + indices: Sequence[_int | SymInt], + dim: _int = 0, + ) -> tuple[Tensor, ...]: + r""" + tensor_split(indices_or_sections, dim=0) -> List of Tensors + + See :func:`torch.tensor_split` + """ + + @overload + def tensor_split( + self, + tensor_indices_or_sections: Tensor, + dim: _int = 0, + ) -> tuple[Tensor, ...]: + r""" + tensor_split(indices_or_sections, dim=0) -> List of Tensors + + See :func:`torch.tensor_split` + """ + + @overload + def tensor_split( + self, + sections: _int | SymInt, + dim: _int = 0, + ) -> tuple[Tensor, ...]: + r""" + tensor_split(indices_or_sections, dim=0) -> List of Tensors + + See :func:`torch.tensor_split` + """ + + @overload + def tile(self, dims: Sequence[_int | SymInt]) -> Tensor: + r""" + tile(dims) -> Tensor + + See :func:`torch.tile` + """ + + @overload + def tile(self, *dims: _int | SymInt) -> Tensor: + r""" + tile(dims) -> Tensor + + See :func:`torch.tile` + """ + + @overload + def to( + self, + dtype: _dtype, + non_blocking: _bool = False, + copy: _bool = False, + *, + memory_format: torch.memory_format | None = None, + ) -> Tensor: + r""" + to(*args, **kwargs) -> Tensor + + Performs Tensor dtype and/or device conversion. A :class:`torch.dtype` and :class:`torch.device` are + inferred from the arguments of ``self.to(*args, **kwargs)``. + + .. note:: + + If the ``self`` Tensor already + has the correct :class:`torch.dtype` and :class:`torch.device`, then ``self`` is returned. + Otherwise, the returned tensor is a copy of ``self`` with the desired + :class:`torch.dtype` and :class:`torch.device`. + + .. note:: + + If ``self`` requires gradients (``requires_grad=True``) but the target + ``dtype`` specified is an integer type, the returned tensor will implicitly + set ``requires_grad=False``. This is because only tensors with + floating-point or complex dtypes can require gradients. + + Here are the ways to call ``to``: + + .. method:: to(dtype, non_blocking=False, copy=False, memory_format=torch.preserve_format) -> Tensor + :noindex: + + Returns a Tensor with the specified :attr:`dtype` + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + + .. note:: + + According to `C++ type conversion rules `_, + converting floating point value to integer type will truncate the fractional part. + If the truncated value cannot fit into the target type (e.g., casting ``torch.inf`` to ``torch.long``), + the behavior is undefined and the result may vary across platforms. + + .. method:: to(device=None, dtype=None, non_blocking=False, copy=False, memory_format=torch.preserve_format) -> Tensor + :noindex: + + Returns a Tensor with the specified :attr:`device` and (optional) + :attr:`dtype`. If :attr:`dtype` is ``None`` it is inferred to be ``self.dtype``. + When :attr:`non_blocking` is set to ``True``, the function attempts to perform + the conversion asynchronously with respect to the host, if possible. This + asynchronous behavior applies to both pinned and pageable memory. However, + caution is advised when using this feature. For more information, refer to the + `tutorial on good usage of non_blocking and pin_memory `__. + When :attr:`copy` is set, a new Tensor is created even when the Tensor + already matches the desired conversion. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + + .. method:: to(other, non_blocking=False, copy=False) -> Tensor + :noindex: + + Returns a Tensor with same :class:`torch.dtype` and :class:`torch.device` as + the Tensor :attr:`other`. + When :attr:`non_blocking` is set to ``True``, the function attempts to perform + the conversion asynchronously with respect to the host, if possible. This + asynchronous behavior applies to both pinned and pageable memory. However, + caution is advised when using this feature. For more information, refer to the + `tutorial on good usage of non_blocking and pin_memory `__. + When :attr:`copy` is set, a new Tensor is created even when the Tensor + already matches the desired conversion. + + Example:: + + >>> tensor = torch.randn(2, 2) # Initially dtype=float32, device=cpu + >>> tensor.to(torch.float64) + tensor([[-0.5044, 0.0005], + [ 0.3310, -0.0584]], dtype=torch.float64) + + >>> cuda0 = torch.device('cuda:0') + >>> tensor.to(cuda0) + tensor([[-0.5044, 0.0005], + [ 0.3310, -0.0584]], device='cuda:0') + + >>> tensor.to(cuda0, dtype=torch.float64) + tensor([[-0.5044, 0.0005], + [ 0.3310, -0.0584]], dtype=torch.float64, device='cuda:0') + + >>> other = torch.randn((), dtype=torch.float64, device=cuda0) + >>> tensor.to(other, non_blocking=True) + tensor([[-0.5044, 0.0005], + [ 0.3310, -0.0584]], dtype=torch.float64, device='cuda:0') + """ + + @overload + def to( + self, + device: DeviceLikeType | None = None, + dtype: _dtype | None = None, + non_blocking: _bool = False, + copy: _bool = False, + *, + memory_format: torch.memory_format | None = None, + ) -> Tensor: + r""" + to(*args, **kwargs) -> Tensor + + Performs Tensor dtype and/or device conversion. A :class:`torch.dtype` and :class:`torch.device` are + inferred from the arguments of ``self.to(*args, **kwargs)``. + + .. note:: + + If the ``self`` Tensor already + has the correct :class:`torch.dtype` and :class:`torch.device`, then ``self`` is returned. + Otherwise, the returned tensor is a copy of ``self`` with the desired + :class:`torch.dtype` and :class:`torch.device`. + + .. note:: + + If ``self`` requires gradients (``requires_grad=True``) but the target + ``dtype`` specified is an integer type, the returned tensor will implicitly + set ``requires_grad=False``. This is because only tensors with + floating-point or complex dtypes can require gradients. + + Here are the ways to call ``to``: + + .. method:: to(dtype, non_blocking=False, copy=False, memory_format=torch.preserve_format) -> Tensor + :noindex: + + Returns a Tensor with the specified :attr:`dtype` + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + + .. note:: + + According to `C++ type conversion rules `_, + converting floating point value to integer type will truncate the fractional part. + If the truncated value cannot fit into the target type (e.g., casting ``torch.inf`` to ``torch.long``), + the behavior is undefined and the result may vary across platforms. + + .. method:: to(device=None, dtype=None, non_blocking=False, copy=False, memory_format=torch.preserve_format) -> Tensor + :noindex: + + Returns a Tensor with the specified :attr:`device` and (optional) + :attr:`dtype`. If :attr:`dtype` is ``None`` it is inferred to be ``self.dtype``. + When :attr:`non_blocking` is set to ``True``, the function attempts to perform + the conversion asynchronously with respect to the host, if possible. This + asynchronous behavior applies to both pinned and pageable memory. However, + caution is advised when using this feature. For more information, refer to the + `tutorial on good usage of non_blocking and pin_memory `__. + When :attr:`copy` is set, a new Tensor is created even when the Tensor + already matches the desired conversion. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + + .. method:: to(other, non_blocking=False, copy=False) -> Tensor + :noindex: + + Returns a Tensor with same :class:`torch.dtype` and :class:`torch.device` as + the Tensor :attr:`other`. + When :attr:`non_blocking` is set to ``True``, the function attempts to perform + the conversion asynchronously with respect to the host, if possible. This + asynchronous behavior applies to both pinned and pageable memory. However, + caution is advised when using this feature. For more information, refer to the + `tutorial on good usage of non_blocking and pin_memory `__. + When :attr:`copy` is set, a new Tensor is created even when the Tensor + already matches the desired conversion. + + Example:: + + >>> tensor = torch.randn(2, 2) # Initially dtype=float32, device=cpu + >>> tensor.to(torch.float64) + tensor([[-0.5044, 0.0005], + [ 0.3310, -0.0584]], dtype=torch.float64) + + >>> cuda0 = torch.device('cuda:0') + >>> tensor.to(cuda0) + tensor([[-0.5044, 0.0005], + [ 0.3310, -0.0584]], device='cuda:0') + + >>> tensor.to(cuda0, dtype=torch.float64) + tensor([[-0.5044, 0.0005], + [ 0.3310, -0.0584]], dtype=torch.float64, device='cuda:0') + + >>> other = torch.randn((), dtype=torch.float64, device=cuda0) + >>> tensor.to(other, non_blocking=True) + tensor([[-0.5044, 0.0005], + [ 0.3310, -0.0584]], dtype=torch.float64, device='cuda:0') + """ + + @overload + def to( + self, + other: Tensor, + non_blocking: _bool = False, + copy: _bool = False, + *, + memory_format: torch.memory_format | None = None, + ) -> Tensor: + r""" + to(*args, **kwargs) -> Tensor + + Performs Tensor dtype and/or device conversion. A :class:`torch.dtype` and :class:`torch.device` are + inferred from the arguments of ``self.to(*args, **kwargs)``. + + .. note:: + + If the ``self`` Tensor already + has the correct :class:`torch.dtype` and :class:`torch.device`, then ``self`` is returned. + Otherwise, the returned tensor is a copy of ``self`` with the desired + :class:`torch.dtype` and :class:`torch.device`. + + .. note:: + + If ``self`` requires gradients (``requires_grad=True``) but the target + ``dtype`` specified is an integer type, the returned tensor will implicitly + set ``requires_grad=False``. This is because only tensors with + floating-point or complex dtypes can require gradients. + + Here are the ways to call ``to``: + + .. method:: to(dtype, non_blocking=False, copy=False, memory_format=torch.preserve_format) -> Tensor + :noindex: + + Returns a Tensor with the specified :attr:`dtype` + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + + .. note:: + + According to `C++ type conversion rules `_, + converting floating point value to integer type will truncate the fractional part. + If the truncated value cannot fit into the target type (e.g., casting ``torch.inf`` to ``torch.long``), + the behavior is undefined and the result may vary across platforms. + + .. method:: to(device=None, dtype=None, non_blocking=False, copy=False, memory_format=torch.preserve_format) -> Tensor + :noindex: + + Returns a Tensor with the specified :attr:`device` and (optional) + :attr:`dtype`. If :attr:`dtype` is ``None`` it is inferred to be ``self.dtype``. + When :attr:`non_blocking` is set to ``True``, the function attempts to perform + the conversion asynchronously with respect to the host, if possible. This + asynchronous behavior applies to both pinned and pageable memory. However, + caution is advised when using this feature. For more information, refer to the + `tutorial on good usage of non_blocking and pin_memory `__. + When :attr:`copy` is set, a new Tensor is created even when the Tensor + already matches the desired conversion. + + Args: + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + + .. method:: to(other, non_blocking=False, copy=False) -> Tensor + :noindex: + + Returns a Tensor with same :class:`torch.dtype` and :class:`torch.device` as + the Tensor :attr:`other`. + When :attr:`non_blocking` is set to ``True``, the function attempts to perform + the conversion asynchronously with respect to the host, if possible. This + asynchronous behavior applies to both pinned and pageable memory. However, + caution is advised when using this feature. For more information, refer to the + `tutorial on good usage of non_blocking and pin_memory `__. + When :attr:`copy` is set, a new Tensor is created even when the Tensor + already matches the desired conversion. + + Example:: + + >>> tensor = torch.randn(2, 2) # Initially dtype=float32, device=cpu + >>> tensor.to(torch.float64) + tensor([[-0.5044, 0.0005], + [ 0.3310, -0.0584]], dtype=torch.float64) + + >>> cuda0 = torch.device('cuda:0') + >>> tensor.to(cuda0) + tensor([[-0.5044, 0.0005], + [ 0.3310, -0.0584]], device='cuda:0') + + >>> tensor.to(cuda0, dtype=torch.float64) + tensor([[-0.5044, 0.0005], + [ 0.3310, -0.0584]], dtype=torch.float64, device='cuda:0') + + >>> other = torch.randn((), dtype=torch.float64, device=cuda0) + >>> tensor.to(other, non_blocking=True) + tensor([[-0.5044, 0.0005], + [ 0.3310, -0.0584]], dtype=torch.float64, device='cuda:0') + """ + + def to_dense( + self, + dtype: _dtype | None = None, + *, + masked_grad: _bool | None = None, + ) -> Tensor: + r""" + to_dense(dtype=None, *, masked_grad=True) -> Tensor + + Creates a strided copy of :attr:`self` if :attr:`self` is not a strided tensor, otherwise returns :attr:`self`. + + Keyword args: + {dtype} + masked_grad (bool, optional): If set to ``True`` (default) and + :attr:`self` has a sparse layout then the backward of + :meth:`to_dense` returns ``grad.sparse_mask(self)``. + + Example:: + + >>> s = torch.sparse_coo_tensor( + ... torch.tensor([[1, 1], + ... [0, 2]]), + ... torch.tensor([9, 10]), + ... size=(3, 3)) + >>> s.to_dense() + tensor([[ 0, 0, 0], + [ 9, 0, 10], + [ 0, 0, 0]]) + """ + + def to_mkldnn(self, dtype: _dtype | None = None) -> Tensor: + r""" + to_mkldnn() -> Tensor + Returns a copy of the tensor in ``torch.mkldnn`` layout. + """ + + def to_padded_tensor( + self, + padding: _float, + output_size: Sequence[_int | SymInt] | None = None, + ) -> Tensor: + r""" + to_padded_tensor(padding, output_size=None) -> Tensor + See :func:`to_padded_tensor` + """ + + @overload + def to_sparse( + self, + *, + layout: _layout | None = None, + blocksize: _int | _size | None = None, + dense_dim: _int | None = None, + ) -> Tensor: + r""" + to_sparse(sparseDims) -> Tensor + + Returns a sparse copy of the tensor. PyTorch supports sparse tensors in + :ref:`coordinate format `. + + Args: + sparseDims (int, optional): the number of sparse dimensions to include in the new sparse tensor + + Example:: + + >>> d = torch.tensor([[0, 0, 0], [9, 0, 10], [0, 0, 0]]) + >>> d + tensor([[ 0, 0, 0], + [ 9, 0, 10], + [ 0, 0, 0]]) + >>> d.to_sparse() + tensor(indices=tensor([[1, 1], + [0, 2]]), + values=tensor([ 9, 10]), + size=(3, 3), nnz=2, layout=torch.sparse_coo) + >>> d.to_sparse(1) + tensor(indices=tensor([[1]]), + values=tensor([[ 9, 0, 10]]), + size=(3, 3), nnz=1, layout=torch.sparse_coo) + + .. method:: to_sparse(*, layout=None, blocksize=None, dense_dim=None) -> Tensor + :noindex: + + Returns a sparse tensor with the specified layout and blocksize. If + the :attr:`self` is strided, the number of dense dimensions could be + specified, and a hybrid sparse tensor will be created, with + `dense_dim` dense dimensions and `self.dim() - 2 - dense_dim` batch + dimension. + + .. note:: If the :attr:`self` layout and blocksize parameters match + with the specified layout and blocksize, return + :attr:`self`. Otherwise, return a sparse tensor copy of + :attr:`self`. + + Args: + + layout (:class:`torch.layout`, optional): The desired sparse + layout. One of ``torch.sparse_coo``, ``torch.sparse_csr``, + ``torch.sparse_csc``, ``torch.sparse_bsr``, or + ``torch.sparse_bsc``. Default: if ``None``, + ``torch.sparse_coo``. + + blocksize (list, tuple, :class:`torch.Size`, optional): Block size + of the resulting BSR or BSC tensor. For other layouts, + specifying the block size that is not ``None`` will result in a + RuntimeError exception. A block size must be a tuple of length + two such that its items evenly divide the two sparse dimensions. + + dense_dim (int, optional): Number of dense dimensions of the + resulting CSR, CSC, BSR or BSC tensor. This argument should be + used only if :attr:`self` is a strided tensor, and must be a + value between 0 and dimension of :attr:`self` tensor minus two. + + Example:: + + >>> x = torch.tensor([[1, 0], [0, 0], [2, 3]]) + >>> x.to_sparse(layout=torch.sparse_coo) + tensor(indices=tensor([[0, 2, 2], + [0, 0, 1]]), + values=tensor([1, 2, 3]), + size=(3, 2), nnz=3, layout=torch.sparse_coo) + >>> x.to_sparse(layout=torch.sparse_bsr, blocksize=(1, 2)) + tensor(crow_indices=tensor([0, 1, 1, 2]), + col_indices=tensor([0, 0]), + values=tensor([[[1, 0]], + [[2, 3]]]), size=(3, 2), nnz=2, layout=torch.sparse_bsr) + >>> x.to_sparse(layout=torch.sparse_bsr, blocksize=(2, 1)) + RuntimeError: Tensor size(-2) 3 needs to be divisible by blocksize[0] 2 + >>> x.to_sparse(layout=torch.sparse_csr, blocksize=(3, 1)) + RuntimeError: to_sparse for Strided to SparseCsr conversion does not use specified blocksize + + >>> x = torch.tensor([[[1], [0]], [[0], [0]], [[2], [3]]]) + >>> x.to_sparse(layout=torch.sparse_csr, dense_dim=1) + tensor(crow_indices=tensor([0, 1, 1, 3]), + col_indices=tensor([0, 0, 1]), + values=tensor([[1], + [2], + [3]]), size=(3, 2, 1), nnz=3, layout=torch.sparse_csr) + """ + + @overload + def to_sparse(self, sparse_dim: _int) -> Tensor: + r""" + to_sparse(sparseDims) -> Tensor + + Returns a sparse copy of the tensor. PyTorch supports sparse tensors in + :ref:`coordinate format `. + + Args: + sparseDims (int, optional): the number of sparse dimensions to include in the new sparse tensor + + Example:: + + >>> d = torch.tensor([[0, 0, 0], [9, 0, 10], [0, 0, 0]]) + >>> d + tensor([[ 0, 0, 0], + [ 9, 0, 10], + [ 0, 0, 0]]) + >>> d.to_sparse() + tensor(indices=tensor([[1, 1], + [0, 2]]), + values=tensor([ 9, 10]), + size=(3, 3), nnz=2, layout=torch.sparse_coo) + >>> d.to_sparse(1) + tensor(indices=tensor([[1]]), + values=tensor([[ 9, 0, 10]]), + size=(3, 3), nnz=1, layout=torch.sparse_coo) + + .. method:: to_sparse(*, layout=None, blocksize=None, dense_dim=None) -> Tensor + :noindex: + + Returns a sparse tensor with the specified layout and blocksize. If + the :attr:`self` is strided, the number of dense dimensions could be + specified, and a hybrid sparse tensor will be created, with + `dense_dim` dense dimensions and `self.dim() - 2 - dense_dim` batch + dimension. + + .. note:: If the :attr:`self` layout and blocksize parameters match + with the specified layout and blocksize, return + :attr:`self`. Otherwise, return a sparse tensor copy of + :attr:`self`. + + Args: + + layout (:class:`torch.layout`, optional): The desired sparse + layout. One of ``torch.sparse_coo``, ``torch.sparse_csr``, + ``torch.sparse_csc``, ``torch.sparse_bsr``, or + ``torch.sparse_bsc``. Default: if ``None``, + ``torch.sparse_coo``. + + blocksize (list, tuple, :class:`torch.Size`, optional): Block size + of the resulting BSR or BSC tensor. For other layouts, + specifying the block size that is not ``None`` will result in a + RuntimeError exception. A block size must be a tuple of length + two such that its items evenly divide the two sparse dimensions. + + dense_dim (int, optional): Number of dense dimensions of the + resulting CSR, CSC, BSR or BSC tensor. This argument should be + used only if :attr:`self` is a strided tensor, and must be a + value between 0 and dimension of :attr:`self` tensor minus two. + + Example:: + + >>> x = torch.tensor([[1, 0], [0, 0], [2, 3]]) + >>> x.to_sparse(layout=torch.sparse_coo) + tensor(indices=tensor([[0, 2, 2], + [0, 0, 1]]), + values=tensor([1, 2, 3]), + size=(3, 2), nnz=3, layout=torch.sparse_coo) + >>> x.to_sparse(layout=torch.sparse_bsr, blocksize=(1, 2)) + tensor(crow_indices=tensor([0, 1, 1, 2]), + col_indices=tensor([0, 0]), + values=tensor([[[1, 0]], + [[2, 3]]]), size=(3, 2), nnz=2, layout=torch.sparse_bsr) + >>> x.to_sparse(layout=torch.sparse_bsr, blocksize=(2, 1)) + RuntimeError: Tensor size(-2) 3 needs to be divisible by blocksize[0] 2 + >>> x.to_sparse(layout=torch.sparse_csr, blocksize=(3, 1)) + RuntimeError: to_sparse for Strided to SparseCsr conversion does not use specified blocksize + + >>> x = torch.tensor([[[1], [0]], [[0], [0]], [[2], [3]]]) + >>> x.to_sparse(layout=torch.sparse_csr, dense_dim=1) + tensor(crow_indices=tensor([0, 1, 1, 3]), + col_indices=tensor([0, 0, 1]), + values=tensor([[1], + [2], + [3]]), size=(3, 2, 1), nnz=3, layout=torch.sparse_csr) + """ + + def to_sparse_bsc( + self, + blocksize: _int | _size, + dense_dim: _int | None = None, + ) -> Tensor: + r""" + to_sparse_bsc(blocksize, dense_dim) -> Tensor + + Convert a tensor to a block sparse column (BSC) storage format of + given blocksize. If the :attr:`self` is strided, then the number of + dense dimensions could be specified, and a hybrid BSC tensor will be + created, with `dense_dim` dense dimensions and `self.dim() - 2 - + dense_dim` batch dimension. + + Args: + + blocksize (list, tuple, :class:`torch.Size`, optional): Block size + of the resulting BSC tensor. A block size must be a tuple of + length two such that its items evenly divide the two sparse + dimensions. + + dense_dim (int, optional): Number of dense dimensions of the + resulting BSC tensor. This argument should be used only if + :attr:`self` is a strided tensor, and must be a value between 0 + and dimension of :attr:`self` tensor minus two. + + Example:: + + >>> dense = torch.randn(10, 10) + >>> sparse = dense.to_sparse_csr() + >>> sparse_bsc = sparse.to_sparse_bsc((5, 5)) + >>> sparse_bsc.row_indices() + tensor([0, 1, 0, 1]) + + >>> dense = torch.zeros(4, 3, 1) + >>> dense[0:2, 0] = dense[0:2, 2] = dense[2:4, 1] = 1 + >>> dense.to_sparse_bsc((2, 1), 1) + tensor(ccol_indices=tensor([0, 1, 2, 3]), + row_indices=tensor([0, 1, 0]), + values=tensor([[[[1.]], + + [[1.]]], + + + [[[1.]], + + [[1.]]], + + + [[[1.]], + + [[1.]]]]), size=(4, 3, 1), nnz=3, + layout=torch.sparse_bsc) + """ + + def to_sparse_bsr( + self, + blocksize: _int | _size, + dense_dim: _int | None = None, + ) -> Tensor: + r""" + to_sparse_bsr(blocksize, dense_dim) -> Tensor + + Convert a tensor to a block sparse row (BSR) storage format of given + blocksize. If the :attr:`self` is strided, then the number of dense + dimensions could be specified, and a hybrid BSR tensor will be + created, with `dense_dim` dense dimensions and `self.dim() - 2 - + dense_dim` batch dimension. + + Args: + + blocksize (list, tuple, :class:`torch.Size`, optional): Block size + of the resulting BSR tensor. A block size must be a tuple of + length two such that its items evenly divide the two sparse + dimensions. + + dense_dim (int, optional): Number of dense dimensions of the + resulting BSR tensor. This argument should be used only if + :attr:`self` is a strided tensor, and must be a value between 0 + and dimension of :attr:`self` tensor minus two. + + Example:: + + >>> dense = torch.randn(10, 10) + >>> sparse = dense.to_sparse_csr() + >>> sparse_bsr = sparse.to_sparse_bsr((5, 5)) + >>> sparse_bsr.col_indices() + tensor([0, 1, 0, 1]) + + >>> dense = torch.zeros(4, 3, 1) + >>> dense[0:2, 0] = dense[0:2, 2] = dense[2:4, 1] = 1 + >>> dense.to_sparse_bsr((2, 1), 1) + tensor(crow_indices=tensor([0, 2, 3]), + col_indices=tensor([0, 2, 1]), + values=tensor([[[[1.]], + + [[1.]]], + + + [[[1.]], + + [[1.]]], + + + [[[1.]], + + [[1.]]]]), size=(4, 3, 1), nnz=3, + layout=torch.sparse_bsr) + """ + + def to_sparse_csc(self, dense_dim: _int | None = None) -> Tensor: + r""" + to_sparse_csc() -> Tensor + + Convert a tensor to compressed column storage (CSC) format. Except + for strided tensors, only works with 2D tensors. If the :attr:`self` + is strided, then the number of dense dimensions could be specified, + and a hybrid CSC tensor will be created, with `dense_dim` dense + dimensions and `self.dim() - 2 - dense_dim` batch dimension. + + Args: + + dense_dim (int, optional): Number of dense dimensions of the + resulting CSC tensor. This argument should be used only if + :attr:`self` is a strided tensor, and must be a value between 0 + and dimension of :attr:`self` tensor minus two. + + Example:: + + >>> dense = torch.randn(5, 5) + >>> sparse = dense.to_sparse_csc() + >>> sparse._nnz() + 25 + + >>> dense = torch.zeros(3, 3, 1, 1) + >>> dense[0, 0] = dense[1, 2] = dense[2, 1] = 1 + >>> dense.to_sparse_csc(dense_dim=2) + tensor(ccol_indices=tensor([0, 1, 2, 3]), + row_indices=tensor([0, 2, 1]), + values=tensor([[[1.]], + + [[1.]], + + [[1.]]]), size=(3, 3, 1, 1), nnz=3, + layout=torch.sparse_csc) + """ + + def to_sparse_csr(self, dense_dim: _int | None = None) -> Tensor: + r""" + to_sparse_csr(dense_dim=None) -> Tensor + + Convert a tensor to compressed row storage format (CSR). Except for + strided tensors, only works with 2D tensors. If the :attr:`self` is + strided, then the number of dense dimensions could be specified, and a + hybrid CSR tensor will be created, with `dense_dim` dense dimensions + and `self.dim() - 2 - dense_dim` batch dimension. + + Args: + + dense_dim (int, optional): Number of dense dimensions of the + resulting CSR tensor. This argument should be used only if + :attr:`self` is a strided tensor, and must be a value between 0 + and dimension of :attr:`self` tensor minus two. + + Example:: + + >>> dense = torch.randn(5, 5) + >>> sparse = dense.to_sparse_csr() + >>> sparse._nnz() + 25 + + >>> dense = torch.zeros(3, 3, 1, 1) + >>> dense[0, 0] = dense[1, 2] = dense[2, 1] = 1 + >>> dense.to_sparse_csr(dense_dim=2) + tensor(crow_indices=tensor([0, 1, 2, 3]), + col_indices=tensor([0, 2, 1]), + values=tensor([[[1.]], + + [[1.]], + + [[1.]]]), size=(3, 3, 1, 1), nnz=3, + layout=torch.sparse_csr) + """ + + def tolist(self) -> list: + r""" + tolist() -> list or number + + Returns the tensor as a (nested) list. For scalars, a standard + Python number is returned, just like with :meth:`~Tensor.item`. + Tensors are automatically moved to the CPU first if necessary. + + This operation is not differentiable. + + Examples:: + + >>> a = torch.randn(2, 2) + >>> a.tolist() + [[0.012766935862600803, 0.5415473580360413], + [-0.08909505605697632, 0.7729271650314331]] + >>> a[0,0].tolist() + 0.012766935862600803 + """ + + def topk( + self, + k: _int | SymInt, + dim: _int = -1, + largest: _bool = True, + sorted: _bool = True, + ) -> torch.return_types.topk: + r""" + topk(k, dim=None, largest=True, sorted=True) -> (Tensor, LongTensor) + + See :func:`torch.topk` + """ + + def trace(self) -> Tensor: + r""" + trace() -> Tensor + + See :func:`torch.trace` + """ + + @overload + def transpose(self, dim0: _int, dim1: _int) -> Tensor: + r""" + transpose(dim0, dim1) -> Tensor + + See :func:`torch.transpose` + """ + + @overload + def transpose( + self, + dim0: str | EllipsisType | None, + dim1: str | EllipsisType | None, + ) -> Tensor: + r""" + transpose(dim0, dim1) -> Tensor + + See :func:`torch.transpose` + """ + + def transpose_(self, dim0: _int, dim1: _int) -> Tensor: + r""" + transpose_(dim0, dim1) -> Tensor + + In-place version of :meth:`~Tensor.transpose` + """ + + def triangular_solve( + self, + A: Tensor, + upper: _bool = True, + transpose: _bool = False, + unitriangular: _bool = False, + ) -> torch.return_types.triangular_solve: + r""" + triangular_solve(A, upper=True, transpose=False, unitriangular=False) -> (Tensor, Tensor) + + See :func:`torch.triangular_solve` + """ + + def tril(self, diagonal: _int | SymInt = 0) -> Tensor: + r""" + tril(diagonal=0) -> Tensor + + See :func:`torch.tril` + """ + + def tril_(self, diagonal: _int | SymInt = 0) -> Tensor: + r""" + tril_(diagonal=0) -> Tensor + + In-place version of :meth:`~Tensor.tril` + """ + + def triu(self, diagonal: _int | SymInt = 0) -> Tensor: + r""" + triu(diagonal=0) -> Tensor + + See :func:`torch.triu` + """ + + def triu_(self, diagonal: _int | SymInt = 0) -> Tensor: + r""" + triu_(diagonal=0) -> Tensor + + In-place version of :meth:`~Tensor.triu` + """ + + def true_divide( + self, + other: Tensor | Number | torch.SymInt | torch.SymFloat, + *, + out: Tensor | None = None, + ) -> Tensor: + r""" + true_divide(value) -> Tensor + + See :func:`torch.true_divide` + """ + + def true_divide_( + self, + other: Tensor | Number | torch.SymInt | torch.SymFloat, + ) -> Tensor: + r""" + true_divide_(value) -> Tensor + + In-place version of :meth:`~Tensor.true_divide_` + """ + + def trunc(self) -> Tensor: + r""" + trunc() -> Tensor + + See :func:`torch.trunc` + """ + + def trunc_(self) -> Tensor: + r""" + trunc_() -> Tensor + + In-place version of :meth:`~Tensor.trunc` + """ + + @overload + def type(self, dtype: None = None, non_blocking: _bool = False) -> str: + r""" + type(dtype=None, non_blocking=False, **kwargs) -> str or Tensor + Returns the type if `dtype` is not provided, else casts this object to + the specified type. + + If this is already of the correct type, no copy is performed and the + original object is returned. + + Args: + dtype (dtype or string): The desired type + non_blocking (bool): If ``True``, and the source is in pinned memory + and destination is on the GPU or vice versa, the copy is performed + asynchronously with respect to the host. Otherwise, the argument + has no effect. + **kwargs: For compatibility, may contain the key ``async`` in place of + the ``non_blocking`` argument. The ``async`` arg is deprecated. + """ + + @overload + def type(self, dtype: str | _dtype, non_blocking: _bool = False) -> Tensor: + r""" + type(dtype=None, non_blocking=False, **kwargs) -> str or Tensor + Returns the type if `dtype` is not provided, else casts this object to + the specified type. + + If this is already of the correct type, no copy is performed and the + original object is returned. + + Args: + dtype (dtype or string): The desired type + non_blocking (bool): If ``True``, and the source is in pinned memory + and destination is on the GPU or vice versa, the copy is performed + asynchronously with respect to the host. Otherwise, the argument + has no effect. + **kwargs: For compatibility, may contain the key ``async`` in place of + the ``non_blocking`` argument. The ``async`` arg is deprecated. + """ + + def type_as(self, other: Tensor) -> Tensor: + r""" + type_as(tensor) -> Tensor + + Returns this tensor cast to the type of the given tensor. + + This is a no-op if the tensor is already of the correct type. This is + equivalent to ``self.type(tensor.type())`` + + Args: + tensor (Tensor): the tensor which has the desired type + """ + + @overload + def unbind(self, dim: _int = 0) -> tuple[Tensor, ...]: + r""" + unbind(dim=0) -> seq + + See :func:`torch.unbind` + """ + + @overload + def unbind(self, dim: str | EllipsisType | None) -> tuple[Tensor, ...]: + r""" + unbind(dim=0) -> seq + + See :func:`torch.unbind` + """ + + @overload + def unflatten( + self, + dim: str | EllipsisType | None, + sizes: Sequence[_int | SymInt], + names: Sequence[str | EllipsisType | None], + ) -> Tensor: ... + @overload + def unflatten(self, dim: _int, sizes: Sequence[_int | SymInt]) -> Tensor: ... + def unfold(self, dimension: _int, size: _int, step: _int) -> Tensor: + r""" + unfold(dimension, size, step) -> Tensor + + Returns a view of the original tensor which contains all slices of size :attr:`size` from + :attr:`self` tensor in the dimension :attr:`dimension`. + + Step between two slices is given by :attr:`step`. + + If `sizedim` is the size of dimension :attr:`dimension` for :attr:`self`, the size of + dimension :attr:`dimension` in the returned tensor will be + `(sizedim - size) / step + 1`. + + An additional dimension of size :attr:`size` is appended in the returned tensor. + + Args: + dimension (int): dimension in which unfolding happens + size (int): the size of each slice that is unfolded + step (int): the step between each slice + + Example:: + + >>> x = torch.arange(1., 8) + >>> x + tensor([ 1., 2., 3., 4., 5., 6., 7.]) + >>> x.unfold(0, 2, 1) + tensor([[ 1., 2.], + [ 2., 3.], + [ 3., 4.], + [ 4., 5.], + [ 5., 6.], + [ 6., 7.]]) + >>> x.unfold(0, 2, 2) + tensor([[ 1., 2.], + [ 3., 4.], + [ 5., 6.]]) + """ + + def uniform_( + self, + from_: _float = 0, + to: _float = 1, + *, + generator: Generator | None = None, + ) -> Tensor: + r""" + uniform_(from=0, to=1, *, generator=None) -> Tensor + + Fills :attr:`self` tensor with numbers sampled from the continuous uniform + distribution: + + .. math:: + f(x) = \dfrac{1}{\text{to} - \text{from}} + """ + + def unsafe_chunk(self, chunks: _int, dim: _int = 0) -> tuple[Tensor, ...]: + r""" + unsafe_chunk(chunks, dim=0) -> List of Tensors + + See :func:`torch.unsafe_chunk` + """ + + def unsafe_split( + self, + split_size: _int | SymInt, + dim: _int = 0, + ) -> tuple[Tensor, ...]: + r""" + unsafe_split(split_size, dim=0) -> List of Tensors + + See :func:`torch.unsafe_split` + """ + + def unsafe_split_with_sizes( + self, + split_sizes: Sequence[_int | SymInt], + dim: _int = 0, + ) -> tuple[Tensor, ...]: ... + def unsqueeze(self, dim: _int) -> Tensor: + r""" + unsqueeze(dim) -> Tensor + + See :func:`torch.unsqueeze` + """ + + def unsqueeze_(self, dim: _int) -> Tensor: + r""" + unsqueeze_(dim) -> Tensor + + In-place version of :meth:`~Tensor.unsqueeze` + """ + + def values(self) -> Tensor: + r""" + values() -> Tensor + + Return the values tensor of a :ref:`sparse COO tensor `. + + .. warning:: + Throws an error if :attr:`self` is not a sparse COO tensor. + + See also :meth:`Tensor.indices`. + + .. note:: + This method can only be called on a coalesced sparse tensor. See + :meth:`Tensor.coalesce` for details. + """ + + @overload + def var( + self, + dim: _int | _size | None, + unbiased: _bool = True, + keepdim: _bool = False, + ) -> Tensor: + r""" + var(dim=None, *, correction=1, keepdim=False) -> Tensor + + See :func:`torch.var` + """ + + @overload + def var( + self, + dim: _int | _size | None = None, + *, + correction: Number | _complex | None = None, + keepdim: _bool = False, + ) -> Tensor: + r""" + var(dim=None, *, correction=1, keepdim=False) -> Tensor + + See :func:`torch.var` + """ + + @overload + def var(self, unbiased: _bool = True) -> Tensor: + r""" + var(dim=None, *, correction=1, keepdim=False) -> Tensor + + See :func:`torch.var` + """ + + @overload + def var( + self, + dim: Sequence[str | EllipsisType | None], + unbiased: _bool = True, + keepdim: _bool = False, + ) -> Tensor: + r""" + var(dim=None, *, correction=1, keepdim=False) -> Tensor + + See :func:`torch.var` + """ + + @overload + def var( + self, + dim: Sequence[str | EllipsisType | None], + *, + correction: Number | _complex | None = None, + keepdim: _bool = False, + ) -> Tensor: + r""" + var(dim=None, *, correction=1, keepdim=False) -> Tensor + + See :func:`torch.var` + """ + + def vdot(self, other: Tensor) -> Tensor: + r""" + vdot(other) -> Tensor + + See :func:`torch.vdot` + """ + + @overload + def view(self, dtype: _dtype) -> Tensor: + r""" + view(*shape) -> Tensor + + Returns a new tensor with the same data as the :attr:`self` tensor but of a + different :attr:`shape`. + + The returned tensor shares the same data and must have the same number + of elements, but may have a different size. For a tensor to be viewed, the new + view size must be compatible with its original size and stride, i.e., each new + view dimension must either be a subspace of an original dimension, or only span + across original dimensions :math:`d, d+1, \dots, d+k` that satisfy the following + contiguity-like condition that :math:`\forall i = d, \dots, d+k-1`, + + .. math:: + + \text{stride}[i] = \text{stride}[i+1] \times \text{size}[i+1] + + Otherwise, it will not be possible to view :attr:`self` tensor as :attr:`shape` + without copying it (e.g., via :meth:`contiguous`). When it is unclear whether a + :meth:`view` can be performed, it is advisable to use :meth:`reshape`, which + returns a view if the shapes are compatible, and copies (equivalent to calling + :meth:`contiguous`) otherwise. + + Args: + shape (torch.Size or int...): the desired size + + Example:: + + >>> x = torch.randn(4, 4) + >>> x.size() + torch.Size([4, 4]) + >>> y = x.view(16) + >>> y.size() + torch.Size([16]) + >>> z = x.view(-1, 8) # the size -1 is inferred from other dimensions + >>> z.size() + torch.Size([2, 8]) + + >>> a = torch.randn(1, 2, 3, 4) + >>> a.size() + torch.Size([1, 2, 3, 4]) + >>> b = a.transpose(1, 2) # Swaps 2nd and 3rd dimension + >>> b.size() + torch.Size([1, 3, 2, 4]) + >>> c = a.view(1, 3, 2, 4) # Does not change tensor layout in memory + >>> c.size() + torch.Size([1, 3, 2, 4]) + >>> torch.equal(b, c) + False + + + .. method:: view(dtype) -> Tensor + :noindex: + + Returns a new tensor with the same data as the :attr:`self` tensor but of a + different :attr:`dtype`. + + If the element size of :attr:`dtype` is different than that of ``self.dtype``, + then the size of the last dimension of the output will be scaled + proportionally. For instance, if :attr:`dtype` element size is twice that of + ``self.dtype``, then each pair of elements in the last dimension of + :attr:`self` will be combined, and the size of the last dimension of the output + will be half that of :attr:`self`. If :attr:`dtype` element size is half that + of ``self.dtype``, then each element in the last dimension of :attr:`self` will + be split in two, and the size of the last dimension of the output will be + double that of :attr:`self`. For this to be possible, the following conditions + must be true: + + * ``self.dim()`` must be greater than 0. + * ``self.stride(-1)`` must be 1. + + Additionally, if the element size of :attr:`dtype` is greater than that of + ``self.dtype``, the following conditions must be true as well: + + * ``self.size(-1)`` must be divisible by the ratio between the element + sizes of the dtypes. + * ``self.storage_offset()`` must be divisible by the ratio between the + element sizes of the dtypes. + * The strides of all dimensions, except the last dimension, must be + divisible by the ratio between the element sizes of the dtypes. + + If any of the above conditions are not met, an error is thrown. + + .. warning:: + + This overload is not supported by TorchScript, and using it in a Torchscript + program will cause undefined behavior. + + + Args: + dtype (:class:`torch.dtype`): the desired dtype + + Example:: + + >>> x = torch.randn(4, 4) + >>> x + tensor([[ 0.9482, -0.0310, 1.4999, -0.5316], + [-0.1520, 0.7472, 0.5617, -0.8649], + [-2.4724, -0.0334, -0.2976, -0.8499], + [-0.2109, 1.9913, -0.9607, -0.6123]]) + >>> x.dtype + torch.float32 + + >>> y = x.view(torch.int32) + >>> y + tensor([[ 1064483442, -1124191867, 1069546515, -1089989247], + [-1105482831, 1061112040, 1057999968, -1084397505], + [-1071760287, -1123489973, -1097310419, -1084649136], + [-1101533110, 1073668768, -1082790149, -1088634448]], + dtype=torch.int32) + >>> y[0, 0] = 1000000000 + >>> x + tensor([[ 0.0047, -0.0310, 1.4999, -0.5316], + [-0.1520, 0.7472, 0.5617, -0.8649], + [-2.4724, -0.0334, -0.2976, -0.8499], + [-0.2109, 1.9913, -0.9607, -0.6123]]) + + >>> x.view(torch.cfloat) + tensor([[ 0.0047-0.0310j, 1.4999-0.5316j], + [-0.1520+0.7472j, 0.5617-0.8649j], + [-2.4724-0.0334j, -0.2976-0.8499j], + [-0.2109+1.9913j, -0.9607-0.6123j]]) + >>> x.view(torch.cfloat).size() + torch.Size([4, 2]) + + >>> x.view(torch.uint8) + tensor([[ 0, 202, 154, 59, 182, 243, 253, 188, 185, 252, 191, 63, 240, 22, + 8, 191], + [227, 165, 27, 190, 128, 72, 63, 63, 146, 203, 15, 63, 22, 106, + 93, 191], + [205, 59, 30, 192, 112, 206, 8, 189, 7, 95, 152, 190, 12, 147, + 89, 191], + [ 43, 246, 87, 190, 235, 226, 254, 63, 111, 240, 117, 191, 177, 191, + 28, 191]], dtype=torch.uint8) + >>> x.view(torch.uint8).size() + torch.Size([4, 16]) + """ + + @overload + def view(self, size: Sequence[_int | SymInt]) -> Tensor: + r""" + view(*shape) -> Tensor + + Returns a new tensor with the same data as the :attr:`self` tensor but of a + different :attr:`shape`. + + The returned tensor shares the same data and must have the same number + of elements, but may have a different size. For a tensor to be viewed, the new + view size must be compatible with its original size and stride, i.e., each new + view dimension must either be a subspace of an original dimension, or only span + across original dimensions :math:`d, d+1, \dots, d+k` that satisfy the following + contiguity-like condition that :math:`\forall i = d, \dots, d+k-1`, + + .. math:: + + \text{stride}[i] = \text{stride}[i+1] \times \text{size}[i+1] + + Otherwise, it will not be possible to view :attr:`self` tensor as :attr:`shape` + without copying it (e.g., via :meth:`contiguous`). When it is unclear whether a + :meth:`view` can be performed, it is advisable to use :meth:`reshape`, which + returns a view if the shapes are compatible, and copies (equivalent to calling + :meth:`contiguous`) otherwise. + + Args: + shape (torch.Size or int...): the desired size + + Example:: + + >>> x = torch.randn(4, 4) + >>> x.size() + torch.Size([4, 4]) + >>> y = x.view(16) + >>> y.size() + torch.Size([16]) + >>> z = x.view(-1, 8) # the size -1 is inferred from other dimensions + >>> z.size() + torch.Size([2, 8]) + + >>> a = torch.randn(1, 2, 3, 4) + >>> a.size() + torch.Size([1, 2, 3, 4]) + >>> b = a.transpose(1, 2) # Swaps 2nd and 3rd dimension + >>> b.size() + torch.Size([1, 3, 2, 4]) + >>> c = a.view(1, 3, 2, 4) # Does not change tensor layout in memory + >>> c.size() + torch.Size([1, 3, 2, 4]) + >>> torch.equal(b, c) + False + + + .. method:: view(dtype) -> Tensor + :noindex: + + Returns a new tensor with the same data as the :attr:`self` tensor but of a + different :attr:`dtype`. + + If the element size of :attr:`dtype` is different than that of ``self.dtype``, + then the size of the last dimension of the output will be scaled + proportionally. For instance, if :attr:`dtype` element size is twice that of + ``self.dtype``, then each pair of elements in the last dimension of + :attr:`self` will be combined, and the size of the last dimension of the output + will be half that of :attr:`self`. If :attr:`dtype` element size is half that + of ``self.dtype``, then each element in the last dimension of :attr:`self` will + be split in two, and the size of the last dimension of the output will be + double that of :attr:`self`. For this to be possible, the following conditions + must be true: + + * ``self.dim()`` must be greater than 0. + * ``self.stride(-1)`` must be 1. + + Additionally, if the element size of :attr:`dtype` is greater than that of + ``self.dtype``, the following conditions must be true as well: + + * ``self.size(-1)`` must be divisible by the ratio between the element + sizes of the dtypes. + * ``self.storage_offset()`` must be divisible by the ratio between the + element sizes of the dtypes. + * The strides of all dimensions, except the last dimension, must be + divisible by the ratio between the element sizes of the dtypes. + + If any of the above conditions are not met, an error is thrown. + + .. warning:: + + This overload is not supported by TorchScript, and using it in a Torchscript + program will cause undefined behavior. + + + Args: + dtype (:class:`torch.dtype`): the desired dtype + + Example:: + + >>> x = torch.randn(4, 4) + >>> x + tensor([[ 0.9482, -0.0310, 1.4999, -0.5316], + [-0.1520, 0.7472, 0.5617, -0.8649], + [-2.4724, -0.0334, -0.2976, -0.8499], + [-0.2109, 1.9913, -0.9607, -0.6123]]) + >>> x.dtype + torch.float32 + + >>> y = x.view(torch.int32) + >>> y + tensor([[ 1064483442, -1124191867, 1069546515, -1089989247], + [-1105482831, 1061112040, 1057999968, -1084397505], + [-1071760287, -1123489973, -1097310419, -1084649136], + [-1101533110, 1073668768, -1082790149, -1088634448]], + dtype=torch.int32) + >>> y[0, 0] = 1000000000 + >>> x + tensor([[ 0.0047, -0.0310, 1.4999, -0.5316], + [-0.1520, 0.7472, 0.5617, -0.8649], + [-2.4724, -0.0334, -0.2976, -0.8499], + [-0.2109, 1.9913, -0.9607, -0.6123]]) + + >>> x.view(torch.cfloat) + tensor([[ 0.0047-0.0310j, 1.4999-0.5316j], + [-0.1520+0.7472j, 0.5617-0.8649j], + [-2.4724-0.0334j, -0.2976-0.8499j], + [-0.2109+1.9913j, -0.9607-0.6123j]]) + >>> x.view(torch.cfloat).size() + torch.Size([4, 2]) + + >>> x.view(torch.uint8) + tensor([[ 0, 202, 154, 59, 182, 243, 253, 188, 185, 252, 191, 63, 240, 22, + 8, 191], + [227, 165, 27, 190, 128, 72, 63, 63, 146, 203, 15, 63, 22, 106, + 93, 191], + [205, 59, 30, 192, 112, 206, 8, 189, 7, 95, 152, 190, 12, 147, + 89, 191], + [ 43, 246, 87, 190, 235, 226, 254, 63, 111, 240, 117, 191, 177, 191, + 28, 191]], dtype=torch.uint8) + >>> x.view(torch.uint8).size() + torch.Size([4, 16]) + """ + + @overload + def view(self, *size: _int | SymInt) -> Tensor: + r""" + view(*shape) -> Tensor + + Returns a new tensor with the same data as the :attr:`self` tensor but of a + different :attr:`shape`. + + The returned tensor shares the same data and must have the same number + of elements, but may have a different size. For a tensor to be viewed, the new + view size must be compatible with its original size and stride, i.e., each new + view dimension must either be a subspace of an original dimension, or only span + across original dimensions :math:`d, d+1, \dots, d+k` that satisfy the following + contiguity-like condition that :math:`\forall i = d, \dots, d+k-1`, + + .. math:: + + \text{stride}[i] = \text{stride}[i+1] \times \text{size}[i+1] + + Otherwise, it will not be possible to view :attr:`self` tensor as :attr:`shape` + without copying it (e.g., via :meth:`contiguous`). When it is unclear whether a + :meth:`view` can be performed, it is advisable to use :meth:`reshape`, which + returns a view if the shapes are compatible, and copies (equivalent to calling + :meth:`contiguous`) otherwise. + + Args: + shape (torch.Size or int...): the desired size + + Example:: + + >>> x = torch.randn(4, 4) + >>> x.size() + torch.Size([4, 4]) + >>> y = x.view(16) + >>> y.size() + torch.Size([16]) + >>> z = x.view(-1, 8) # the size -1 is inferred from other dimensions + >>> z.size() + torch.Size([2, 8]) + + >>> a = torch.randn(1, 2, 3, 4) + >>> a.size() + torch.Size([1, 2, 3, 4]) + >>> b = a.transpose(1, 2) # Swaps 2nd and 3rd dimension + >>> b.size() + torch.Size([1, 3, 2, 4]) + >>> c = a.view(1, 3, 2, 4) # Does not change tensor layout in memory + >>> c.size() + torch.Size([1, 3, 2, 4]) + >>> torch.equal(b, c) + False + + + .. method:: view(dtype) -> Tensor + :noindex: + + Returns a new tensor with the same data as the :attr:`self` tensor but of a + different :attr:`dtype`. + + If the element size of :attr:`dtype` is different than that of ``self.dtype``, + then the size of the last dimension of the output will be scaled + proportionally. For instance, if :attr:`dtype` element size is twice that of + ``self.dtype``, then each pair of elements in the last dimension of + :attr:`self` will be combined, and the size of the last dimension of the output + will be half that of :attr:`self`. If :attr:`dtype` element size is half that + of ``self.dtype``, then each element in the last dimension of :attr:`self` will + be split in two, and the size of the last dimension of the output will be + double that of :attr:`self`. For this to be possible, the following conditions + must be true: + + * ``self.dim()`` must be greater than 0. + * ``self.stride(-1)`` must be 1. + + Additionally, if the element size of :attr:`dtype` is greater than that of + ``self.dtype``, the following conditions must be true as well: + + * ``self.size(-1)`` must be divisible by the ratio between the element + sizes of the dtypes. + * ``self.storage_offset()`` must be divisible by the ratio between the + element sizes of the dtypes. + * The strides of all dimensions, except the last dimension, must be + divisible by the ratio between the element sizes of the dtypes. + + If any of the above conditions are not met, an error is thrown. + + .. warning:: + + This overload is not supported by TorchScript, and using it in a Torchscript + program will cause undefined behavior. + + + Args: + dtype (:class:`torch.dtype`): the desired dtype + + Example:: + + >>> x = torch.randn(4, 4) + >>> x + tensor([[ 0.9482, -0.0310, 1.4999, -0.5316], + [-0.1520, 0.7472, 0.5617, -0.8649], + [-2.4724, -0.0334, -0.2976, -0.8499], + [-0.2109, 1.9913, -0.9607, -0.6123]]) + >>> x.dtype + torch.float32 + + >>> y = x.view(torch.int32) + >>> y + tensor([[ 1064483442, -1124191867, 1069546515, -1089989247], + [-1105482831, 1061112040, 1057999968, -1084397505], + [-1071760287, -1123489973, -1097310419, -1084649136], + [-1101533110, 1073668768, -1082790149, -1088634448]], + dtype=torch.int32) + >>> y[0, 0] = 1000000000 + >>> x + tensor([[ 0.0047, -0.0310, 1.4999, -0.5316], + [-0.1520, 0.7472, 0.5617, -0.8649], + [-2.4724, -0.0334, -0.2976, -0.8499], + [-0.2109, 1.9913, -0.9607, -0.6123]]) + + >>> x.view(torch.cfloat) + tensor([[ 0.0047-0.0310j, 1.4999-0.5316j], + [-0.1520+0.7472j, 0.5617-0.8649j], + [-2.4724-0.0334j, -0.2976-0.8499j], + [-0.2109+1.9913j, -0.9607-0.6123j]]) + >>> x.view(torch.cfloat).size() + torch.Size([4, 2]) + + >>> x.view(torch.uint8) + tensor([[ 0, 202, 154, 59, 182, 243, 253, 188, 185, 252, 191, 63, 240, 22, + 8, 191], + [227, 165, 27, 190, 128, 72, 63, 63, 146, 203, 15, 63, 22, 106, + 93, 191], + [205, 59, 30, 192, 112, 206, 8, 189, 7, 95, 152, 190, 12, 147, + 89, 191], + [ 43, 246, 87, 190, 235, 226, 254, 63, 111, 240, 117, 191, 177, 191, + 28, 191]], dtype=torch.uint8) + >>> x.view(torch.uint8).size() + torch.Size([4, 16]) + """ + + def view_as(self, other: Tensor) -> Tensor: + r""" + view_as(other) -> Tensor + + View this tensor as the same size as :attr:`other`. + ``self.view_as(other)`` is equivalent to ``self.view(other.size())``. + + Please see :meth:`~Tensor.view` for more information about ``view``. + + Args: + other (:class:`torch.Tensor`): The result tensor has the same size + as :attr:`other`. + """ + + @overload + def vsplit(self, sections: _int) -> tuple[Tensor, ...]: + r""" + vsplit(split_size_or_sections) -> List of Tensors + + See :func:`torch.vsplit` + """ + + @overload + def vsplit(self, indices: _size) -> tuple[Tensor, ...]: + r""" + vsplit(split_size_or_sections) -> List of Tensors + + See :func:`torch.vsplit` + """ + + @overload + def vsplit(self, *indices: _int) -> tuple[Tensor, ...]: + r""" + vsplit(split_size_or_sections) -> List of Tensors + + See :func:`torch.vsplit` + """ + + @overload + def where(self, condition: Tensor, other: Tensor) -> Tensor: + r""" + where(condition, y) -> Tensor + + ``self.where(condition, y)`` is equivalent to ``torch.where(condition, self, y)``. + See :func:`torch.where` + """ + + @overload + def where(self, condition: Tensor, other: Number | _complex) -> Tensor: + r""" + where(condition, y) -> Tensor + + ``self.where(condition, y)`` is equivalent to ``torch.where(condition, self, y)``. + See :func:`torch.where` + """ + + @overload + def xlogy(self, other: Tensor) -> Tensor: + r""" + xlogy(other) -> Tensor + + See :func:`torch.xlogy` + """ + + @overload + def xlogy(self, other: Number | _complex) -> Tensor: + r""" + xlogy(other) -> Tensor + + See :func:`torch.xlogy` + """ + + @overload + def xlogy_(self, other: Tensor) -> Tensor: + r""" + xlogy_(other) -> Tensor + + In-place version of :meth:`~Tensor.xlogy` + """ + + @overload + def xlogy_(self, other: Number | _complex) -> Tensor: + r""" + xlogy_(other) -> Tensor + + In-place version of :meth:`~Tensor.xlogy` + """ + + def xpu( + self, + device: _device | _int | str | None = None, + non_blocking: _bool = False, + memory_format: torch.memory_format = torch.preserve_format, + ) -> Tensor: + r""" + xpu(device=None, non_blocking=False, memory_format=torch.preserve_format) -> Tensor + + Returns a copy of this object in XPU memory. + + If this object is already in XPU memory and on the correct device, + then no copy is performed and the original object is returned. + + Args: + device (:class:`torch.device`, optional): The destination XPU device. + Defaults to the current XPU device. + non_blocking (bool, optional): If ``True`` and the source is in pinned memory, + the copy will be asynchronous with respect to the host. + Otherwise, the argument has no effect. Default: ``False``. + memory_format (:class:`torch.memory_format`, optional): the desired memory format of + returned Tensor. Default: ``torch.preserve_format``. + """ + + def zero_(self) -> Tensor: + r""" + zero_() -> Tensor + + Fills :attr:`self` tensor with zeros. + """ + +_TensorBase = TensorBase + +def _DTensor_OpSchema_post_init(self: OpSchema) -> None: ... +def _DTensor_OpSchema_recompute_comparison_key(self: OpSchema) -> None: ... +def _DTensor_compute_global_tensor_info( + tensor: Tensor, mesh: DeviceMesh, placements: Sequence[Placement] +) -> tuple[list[_int], list[_int]]: ... +def _get_DTensor_sharding_propagator_cache_stats() -> tuple[_int, _int]: ... +def _clear_DTensor_sharding_propagator_cache() -> None: ... + +# Defined in torch/csrc/multiprocessing/init.cpp +def _multiprocessing_init() -> None: ... +def _set_thread_name(name: str) -> None: ... +def _get_thread_name() -> str: ... + +# Defined in torch/csrc/Module.cpp +def _accelerator_hooks_device_count() -> _int: ... +def _accelerator_hooks_set_current_device(device_index: _int) -> None: ... +def _accelerator_hooks_get_current_device() -> _int: ... +def _accelerator_hooks_exchange_device(device_index: _int) -> _int: ... +def _accelerator_hooks_maybe_exchange_device(device_index: _int) -> _int: ... +def _get_accelerator(check: _bool = False) -> _device: ... +def _storage_Use_Count(storage_ptr: _int) -> _int: ... + +# Defined in torch/csrc/mtia/Module.cpp +def _mtia_init() -> None: ... +def _mtia_isBuilt() -> _bool: ... +def _mtia_isInBadFork() -> _bool: ... +def _mtia_deviceSynchronize() -> None: ... +def _mtia_getCurrentStream(device: _int) -> Stream: ... +def _mtia_getCurrentRawStream(device: _int) -> _int: ... +def _mtia_setCurrentStream(stream: Stream) -> None: ... +def _mtia_getDefaultStream(device: _int) -> Stream: ... +def _mtia_setStream(stream_id: _int, device_index: _int, device_type: _int) -> None: ... +def _mtia_memoryStats(device: _int) -> dict[str, Any]: ... +def _mtia_getDeviceCapability(device: _int) -> tuple[_int, _int]: ... +def _mtia_getDeviceProperties(device: _int) -> dict[str, Any]: ... +def _mtia_emptyCache() -> None: ... +def _mtia_recordMemoryHistory( + enabled: str | None, + stacks: str, + max_entries, +) -> None: ... +def _mtia_memorySnapshot() -> dict[str, Any]: ... +def _mtia_attachOutOfMemoryObserver( + observer: Callable[[_int, _int, _int, _int], None], +) -> None: ... +def _mtia_getDeviceCount() -> _int: ... +def _mtia_resetPeakMemoryStats(device: _int) -> None: ... +def _mtia_graphPoolHandle() -> tuple[_int, _int]: ... + +# Defined in torch/csrc/mtia/Module.cpp +class _MTIAGraph: + def __new__(cls, keep_graph: _bool = ...) -> Self: ... + def capture_begin(self, pool: tuple[_int, _int]) -> None: ... + def capture_end(self) -> None: ... + def instantiate(self) -> None: ... + def replay(self) -> None: ... + def reset(self) -> None: ... + def pool(self) -> tuple[_int, _int]: ... + +# Defined in torch/csrc/mps/Module.cpp +def _mps_deviceSynchronize() -> None: ... +def _mps_get_core_count() -> _int: ... +def _mps_get_default_generator() -> Generator: ... +def _mps_get_name() -> _str: ... +def _mps_emptyCache() -> None: ... +def _mps_setMemoryFraction(fraction: _float) -> None: ... +def _mps_currentAllocatedMemory() -> _int: ... +def _mps_driverAllocatedMemory() -> _int: ... +def _mps_recommendedMaxMemory() -> _int: ... +def _mps_is_available() -> _bool: ... +def _mps_is_on_macos_or_newer(major: _int, minor: _int) -> _bool: ... +def _mps_profilerStartTrace(mode: str, wait_until_completed: _bool) -> None: ... +def _mps_profilerStopTrace() -> None: ... +def _mps_acquireEvent(enable_timing: _bool) -> _int: ... +def _mps_releaseEvent(event_id: _int) -> None: ... +def _mps_recordEvent(event_id: _int) -> None: ... +def _mps_waitForEvent(event_id: _int) -> None: ... +def _mps_synchronizeEvent(event_id: _int) -> None: ... +def _mps_queryEvent(event_id: _int) -> _bool: ... +def _mps_elapsedTimeOfEvents(start_event_id: _int, end_event_id: _int) -> _float: ... +def _mps_isCaptureEnabled() -> _bool: ... +def _mps_isCapturing() -> _bool: ... +def _mps_startCapture(name: str) -> None: ... +def _mps_stopCapture() -> None: ... + +# Defined in torch/csrc/cuda/Module.cpp +def _cuda_getCurrentStream(device: _int) -> tuple: ... +def _cuda_getCurrentRawStream(device: _int) -> _int: ... +def _cuda_getDefaultStream(device: _int) -> tuple: ... +def _cuda_getStreamFromExternal(data_ptr: _int, device_index: _int) -> tuple: ... +def _cuda_getCurrentBlasHandle() -> _int: ... +def _cuda_clearCublasWorkspaces() -> None: ... +def _cuda_setDevice(device: _int) -> None: ... +def _cuda_exchangeDevice(device: _int) -> _int: ... +def _cuda_maybeExchangeDevice(device: _int) -> _int: ... +def _cuda_getDevice() -> _int: ... +def _cuda_getDeviceCount() -> _int: ... +def _cuda_set_sync_debug_mode(warn_level: _int | str) -> None: ... +def _cuda_get_sync_debug_mode() -> _int: ... +def _cuda_sleep(cycles: _int) -> None: ... +def _cuda_busy_wait_for_flag() -> None: ... +def _cuda_clear_flag() -> None: ... +def _cuda_synchronize() -> None: ... +def _cuda_ipc_collect() -> None: ... +def _cuda_getArchFlags() -> str | None: ... +def _cuda_init() -> None: ... +def _cuda_setStream(stream_id: _int, device_index: _int, device_type: _int) -> None: ... +def _cuda_getCompiledVersion() -> _int: ... +def _cuda_cudaHostAllocator() -> _int: ... +def _cuda_cudaCachingAllocator_raw_alloc(size: _int, cuda_stream: _int) -> _int: ... +def _cuda_cudaCachingAllocator_raw_delete(ptr: _int) -> None: ... +def _cuda_cudaCachingAllocator_enable(val: _bool) -> None: ... +def _cuda_beginAllocateToPool(device: _int, mempool_id: tuple[_int, _int]) -> None: ... +def _cuda_beginAllocateCurrentThreadToPool( + device: _int, + mempool_id: tuple[_int, _int], +) -> None: ... +def _cuda_endAllocateToPool(device: _int, mempool_id: tuple[_int, _int]) -> None: ... +def _cuda_beginAllocateCurrentStreamToPool( + device: _int, + mempool_id: tuple[_int, _int], +) -> None: ... +def _cuda_releasePool(device: _int, mempool_id: tuple[_int, _int]) -> None: ... +def _cuda_checkPoolLiveAllocations( + device: _int, + mempool_id: tuple[_int, _int], + expected_live_allocations: set, +) -> _bool: ... +def _cuda_setCheckpointPoolState( + device: _int, + state: _cuda_CUDAAllocator_AllocatorState, + stale_storages: list[_int], + storages_to_add_deleters_to: list[_int], +) -> None: ... +def _cuda_getMemoryFraction(device: _int) -> _float: ... +def _cuda_setMemoryFraction(fraction: _float, device: _int) -> None: ... +def _cuda_emptyCache() -> None: ... +def _cuda_memoryStats(device: _int) -> dict[str, Any]: ... +def _cuda_resetAccumulatedMemoryStats(device: _int) -> None: ... +def _cuda_resetPeakMemoryStats(device: _int) -> None: ... +def _cuda_hostMemoryStats() -> dict[str, Any]: ... +def _cuda_resetAccumulatedHostMemoryStats() -> None: ... +def _cuda_resetPeakHostMemoryStats() -> None: ... +def _cuda_memorySnapshot(mempool_id: tuple[_int, _int] | None) -> dict[str, Any]: ... +def _cuda_setMemoryMetadata(metadata: str) -> None: ... +def _cuda_getMemoryMetadata() -> str: ... +def _cuda_record_memory_history_legacy( + enabled: _bool, + record_context: _bool, + record_context_cpp: _bool, + alloc_trace_max_entries: _int, + alloc_trace_record_context: _bool, + clear_history: _bool, + compile_context: _bool, + global_record_annotations: _bool, +) -> None: ... +def _cuda_record_memory_history( + enabled: str | None, + context: str | None, + stacks: str, + max_entries: _int, + clear_history: _bool, + compile_context: _bool, + global_record_annotations: _bool, +) -> None: ... +def _cuda_isHistoryEnabled() -> _bool: ... +def _cuda_getAllocatorBackend() -> str: ... + +class _cuda_CUDAAllocator_AllocatorState: ... + +def _cuda_getCheckpointState( + device: _int, + mempool: tuple[_int, _int], +) -> _cuda_CUDAAllocator_AllocatorState: ... +def _set_cached_tensors_enabled(enabled: _bool) -> None: ... +def _add_cached_tensor(t: Tensor) -> None: ... +def _remove_cached_tensor(t: Tensor) -> None: ... +def _tensors_data_ptrs_at_indices_equal( + tensors: list[Tensor | _int], + ptrs: list[_int | None], + indices: list[_int], +) -> _bool: ... +def _construct_CUDA_Tensor_From_Storage_And_Metadata( + metadata: dict, + storage: Storage, +) -> Tensor: ... +def _set_storage_access_error_msg(t: Tensor, s: str) -> None: ... +def _set_storage_data_ptr_access_error_msg(storage_ptr: _int, s: str) -> None: ... +def _free_And_Remove_DeleterFn(storage_ptr: _int) -> None: ... +def _has_Standard_Deleter(storage_ptr: _int) -> _bool: ... + +class _cuda_CUDAAllocator: ... + +def _cuda_customAllocator(alloc_fn: _int, free_fn: _int) -> _cuda_CUDAAllocator: ... +def _cuda_changeCurrentAllocator(allocator: _cuda_CUDAAllocator) -> None: ... +def _cuda_getAllocator() -> _cuda_CUDAAllocator: ... +def _cuda_lock_mutex() -> None: ... +def _cuda_unlock_mutex() -> None: ... +def _cuda_canDeviceAccessPeer(device: _int, peer_device: _int) -> _bool: ... +def _cuda_jiterator_compile_and_launch_kernel( + code_string: str, + kernel_name: str, + return_by_ref: _bool, + num_outputs: _int, + tensors: tuple, + kwargs: dict[str, _int | _float | _bool], +) -> Tensor: ... +def _cuda_get_cudnn_benchmark_limit() -> _int: ... +def _cuda_set_cudnn_benchmark_limit(arg: _int) -> None: ... +def _cuda_get_conv_benchmark_empty_cache() -> _bool: ... +def _cudnn_set_conv_benchmark_empty_cache(enable: _bool) -> None: ... +def _nccl_version() -> _int: ... +def _nccl_version_suffix() -> bytes: ... +def _nccl_unique_id() -> bytes: ... +def _nccl_init_rank(nranks: _int, comm_id: bytes, rank: _int) -> object: ... +def _nccl_reduce( + input: Sequence[Tensor], + output: Tensor, + root: _int, + op: _int, + streams: Sequence[_CudaStreamBase] | None, + comms: Sequence[object] | None, +) -> None: ... +def _nccl_all_reduce( + input: Sequence[Tensor], + output: Sequence[Tensor], + op: _int, + streams: Sequence[_CudaStreamBase] | None, + comms: Sequence[object] | None, +) -> None: ... +def _nccl_broadcast( + input: Sequence[Tensor], + root: _int, + streams: Sequence[_CudaStreamBase] | None, + comms: Sequence[object] | None, +) -> None: ... +def _nccl_all_gather( + input: Sequence[Tensor], + output: Sequence[Tensor], + streams: Sequence[_CudaStreamBase] | None, + comms: Sequence[object] | None, +) -> None: ... +def _nccl_reduce_scatter( + input: Sequence[Tensor], + output: Sequence[Tensor], + op: _int, + streams: Sequence[_CudaStreamBase] | None, + comms: Sequence[object] | None, +) -> None: ... +def _rocm_is_backward_pass() -> _bool: ... +def _cuda_tunableop_enable(val: _bool) -> None: ... +def _cuda_tunableop_is_enabled() -> _bool: ... +def _cuda_tunableop_tuning_enable(val: _bool) -> None: ... +def _cuda_tunableop_tuning_is_enabled() -> _bool: ... +def _cuda_tunableop_set_max_tuning_duration(duration: _int) -> None: ... +def _cuda_tunableop_get_max_tuning_duration() -> _int: ... +def _cuda_tunableop_set_max_tuning_iterations(iterations: _int) -> None: ... +def _cuda_tunableop_get_max_tuning_iterations() -> _int: ... +def _cuda_tunableop_set_filename( + filename: str, + insert_device_ordinal: _bool | None, +) -> None: ... +def _cuda_tunableop_get_filename() -> str: ... +def _cuda_tunableop_read_file(filename: str | None) -> _bool: ... +def _cuda_tunableop_get_results() -> tuple[str, str, str, _float]: ... +def _cuda_tunableop_get_validators() -> tuple[str, str]: ... +def _cuda_tunableop_set_rotating_buffer_size(buffer_size: _int) -> None: ... +def _cuda_tunableop_get_rotation_buffer_size() -> _int: ... +def _cuda_tunableop_set_numerical_check_tolerances( + enabled: _bool, atol: _float = 1e-5, rtol: _float = 1e-5 +) -> None: ... + +class _CudaDeviceProperties: + name: str + major: _int + minor: _int + multi_processor_count: _int + total_memory: _int + is_integrated: _int + is_multi_gpu_board: _int + max_threads_per_multi_processor: _int + gcnArchName: str + warp_size: _int + uuid: str + L2_cache_size: _int + clock_rate: _int + memory_clock_rate: _int + memory_bus_width: _int + shared_memory_per_block: _int + +# Functions related to SDPA +class _SDPAParams: + query: Tensor + key: Tensor + value: Tensor + attn_mask: Tensor | None + dropout: _float + is_causal: _bool + enable_gqa: _bool + def __init__( + self, + query: Tensor, + key: Tensor, + value: Tensor, + attn_mask: Tensor | None, + dropout: _float, + is_causal: _bool, + enable_gqa: _bool, + ) -> None: ... + +class _SDPBackend(Enum): + ERROR = -1 + MATH = 0 + FLASH_ATTENTION = 1 + EFFICIENT_ATTENTION = 2 + CUDNN_ATTENTION = 3 + OVERRIDEABLE = 4 + +def _is_flash_attention_available() -> _bool: ... +def _can_use_cudnn_attention(params: _SDPAParams, debug: _bool) -> _bool: ... +def _can_use_flash_attention(params: _SDPAParams, debug: _bool) -> _bool: ... +def _can_use_mem_efficient_attention(params: _SDPAParams, debug: _bool) -> _bool: ... + +# Defined in torch/csrc/cuda/GdsFile.cpp +def _gds_register_buffer(t: Storage) -> None: ... +def _gds_deregister_buffer(t: Storage) -> None: ... +def _gds_register_handle(fd: _int) -> _int: ... +def _gds_deregister_handle(handle: _int) -> None: ... +def _gds_load_storage(handle: _int, s: Storage, offset: _int) -> None: ... +def _gds_save_storage(handle: _int, s: Storage, offset: _int) -> None: ... + +# Defined in torch/csrc/cuda/python_comm.cpp +def _broadcast(tensor: Tensor, devices: list[_int]) -> list[Tensor]: ... +def _broadcast_out(tensor: Tensor, out_tensors: list[Tensor]) -> list[Tensor]: ... +def _broadcast_coalesced( + tensors: list[Tensor], + devices: list[_int], + buffer_size: _int, +) -> list[list[Tensor]]: ... +def _scatter( + tensor: Tensor, + devices: list[_int], + chunk_sizes: list[_int] | None, + dim: _int, + streams: list[Stream] | None, +) -> list[Tensor]: ... +def _scatter_out( + tensor: Tensor, + out_tensors: list[Tensor], + dim: _int, + streams: list[Stream] | None, +) -> list[Tensor]: ... +def _gather( + tensors: list[Tensor], + dim: _int, + destination_index: _int | None, +) -> Tensor: ... +def _gather_out(tensors: list[Tensor], out_tensor: Tensor, dim: _int) -> Tensor: ... + +# Defined in torch/csrc/cuda/Stream.cpp +class _CudaStreamBase(Stream): + stream_id: _int + device_index: _int + device_type: _int + + device: _device + cuda_stream: _int + priority: _int + + def __new__( + cls, + priority: _int = 0, + stream_id: _int = 0, + device_index: _int = 0, + stream_ptr: _int = 0, + ) -> Self: ... + def query(self) -> _bool: ... + def synchronize(self) -> None: ... + def priority_range(self) -> tuple[_int, _int]: ... + +# Defined in torch/csrc/cuda/Event.cpp +class _CudaEventBase: + device: _device + cuda_event: _int + + def __new__( + cls, + enable_timing: _bool = False, + blocking: _bool = False, + interprocess: _bool = False, + external: _bool = False, + ) -> Self: ... + @classmethod + def from_ipc_handle(cls, device: _device, ipc_handle: bytes) -> _CudaEventBase: ... + def record(self, stream: _CudaStreamBase) -> None: ... + def wait(self, stream: _CudaStreamBase) -> None: ... + def query(self) -> _bool: ... + def elapsed_time(self, other: _CudaEventBase) -> _float: ... + def synchronize(self) -> None: ... + def ipc_handle(self) -> bytes: ... + +# Defined in torch/csrc/cuda/Graph.cpp +class _CUDAGraph: + def __new__(cls, keep_graph: _bool = ...) -> Self: ... + def capture_begin( + self, + pool: _POOL_HANDLE | None = ..., + capture_error_mode: str = "global", + ) -> None: ... + def capture_end(self) -> None: ... + def instantiate(self) -> None: ... + def register_generator_state(self, Generator) -> None: ... + def replay(self) -> None: ... + def reset(self) -> None: ... + def pool(self) -> _POOL_HANDLE: ... + def enable_debug_mode(self) -> None: ... + def debug_dump(self, debug_path: str) -> None: ... + def raw_cuda_graph(self) -> _int: ... + def raw_cuda_graph_exec(self) -> _int: ... + +# Defined in torch/csrc/cuda/MemPool.cpp +class _MemPool: + def __init__( + self, + allocator: _cuda_CUDAAllocator | None = None, + is_user_created: _bool = True, + use_on_oom: _bool = False, + no_split: _bool = False, + ) -> None: ... + @property + def id(self) -> tuple[_int, _int]: ... + @property + def allocator(self) -> _cuda_CUDAAllocator | None: ... + def use_count(self) -> _int: ... + +def _cuda_isCurrentStreamCapturing() -> _bool: ... +def _graph_pool_handle() -> tuple[_int, _int]: ... + +# Defined in torch/csrc/xpu/Module.cpp +def _xpu_setDevice(device: _int) -> None: ... +def _xpu_exchangeDevice(device: _int) -> _int: ... +def _xpu_maybeExchangeDevice(device: _int) -> _int: ... +def _xpu_getDevice() -> _int: ... +def _xpu_getDeviceCount() -> _int: ... +def _xpu_getArchFlags() -> str | None: ... +def _xpu_init() -> None: ... +def _xpu_setStream(stream_id: _int, device_index: _int, device_type: _int) -> None: ... +def _xpu_getCurrentStream(device: _int) -> tuple: ... +def _xpu_getCurrentRawStream(device: _int) -> _int: ... +def _xpu_getStreamFromExternal(data_ptr: _int, device_index: _int) -> tuple: ... +def _xpu_synchronize(device: _int) -> None: ... +def _xpu_emptyCache() -> None: ... +def _xpu_memoryStats(device: _int) -> dict[str, Any]: ... +def _xpu_resetAccumulatedMemoryStats(device: _int) -> None: ... +def _xpu_resetPeakMemoryStats(device: _int) -> None: ... +def _xpu_getMemoryInfo(device: _int) -> tuple[_int, _int]: ... +def _xpu_canDeviceAccessPeer(device: _int, peer: _int) -> _bool: ... +def _xpu_getMemoryFraction(device: _int) -> _float: ... +def _xpu_setMemoryFraction(fraction: _float, device: _int) -> None: ... + +class _XpuDeviceProperties: + name: str + platform_name: str + vendor: str + device_id: _int + driver_version: str + version: str + max_compute_units: _int + gpu_eu_count: _int + max_work_group_size: _int + max_num_sub_groups: _int + sub_group_sizes: list[_int] + has_fp16: _bool + has_fp64: _bool + has_atomic64: _bool + has_bfloat16_conversions: _bool + has_subgroup_matrix_multiply_accumulate: _bool + has_subgroup_matrix_multiply_accumulate_tensor_float32: _bool + has_subgroup_2d_block_io: _bool + total_memory: _int + gpu_subslice_count: _int + architecture: _int + type: str + uuid: Any + +class _xpu_XPUAllocator: ... + +def _xpu_customAllocator(alloc_fn: _int, free_fn: _int) -> _xpu_XPUAllocator: ... +def _xpu_changeCurrentAllocator(allocator: _xpu_XPUAllocator) -> None: ... +def _xpu_getAllocator() -> _xpu_XPUAllocator: ... + +# Defined in torch/csrc/xpu/Stream.cpp +class _XpuStreamBase(Stream): + stream_id: _int + device_index: _int + device_type: _int + + device: _device + sycl_queue: _int + priority: _int + + def __new__( + cls, + priority: _int = 0, + stream_id: _int = 0, + device_index: _int = 0, + device_type: _int = 0, + ) -> Self: ... + def query(self) -> _bool: ... + def synchronize(self) -> None: ... + @staticmethod + def priority_range() -> tuple: ... + +# Defined in torch/csrc/xpu/Event.cpp +class _XpuEventBase: + device: _device + sycl_event: _int + + def __new__(cls, enable_timing: _bool = False) -> Self: ... + def record(self, stream: _XpuEventBase) -> None: ... + def wait(self, stream: _XpuStreamBase) -> None: ... + def query(self) -> _bool: ... + def elapsed_time(self, other: _XpuEventBase) -> _float: ... + def synchronize(self) -> None: ... + +# Defined in torch/csrc/DataLoader.cpp +def _set_worker_signal_handlers( + *arg: Any, +) -> None: ... # THPModule_setWorkerSignalHandlers +def _set_worker_pids( + key: _int, + child_pids: tuple[_int, ...], +) -> None: ... # THPModule_setWorkerPIDs +def _remove_worker_pids(loader_id: _int) -> None: ... # THPModule_removeWorkerPIDs +def _error_if_any_worker_fails() -> None: ... # THPModule_errorIfAnyWorkerFails + +# Defined in torch/csrc/DeviceAccelerator.cpp +def _accelerator_getAccelerator() -> _device: ... +def _accelerator_setDeviceIndex(device_index: _int) -> None: ... +def _accelerator_getDeviceIndex() -> _int: ... +def _accelerator_getDeviceCapability(device_index: _int) -> dict[str, Any]: ... +def _accelerator_setStream(Stream) -> None: ... +def _accelerator_getStream(device_index: _int) -> Stream: ... +def _accelerator_synchronizeDevice(device_index: _int) -> None: ... +def _accelerator_exchangeDevice(device_index: _int) -> _int: ... +def _accelerator_maybeExchangeDevice(device_index: _int) -> _int: ... +def _accelerator_isAllocatorInitialized() -> _bool: ... +def _accelerator_emptyCache() -> None: ... +def _accelerator_getDeviceStats(device_index: _int) -> dict[str, Any]: ... +def _accelerator_resetAccumulatedStats(device_index: _int) -> None: ... +def _accelerator_resetPeakStats(device_index: _int) -> None: ... +def _accelerator_getMemoryInfo(device_index: _int) -> tuple[_int, _int]: ... +def _accelerator_setAllocatorSettings(env: str) -> None: ... + +# Defined in torch/csrc/jit/python/python_tracer.cpp +class TracingState: + def push_scope(self, scope_name: str) -> None: ... + def pop_scope(self) -> None: ... + def current_scope(self) -> str: ... + def set_graph(self, graph: Graph) -> None: ... + def graph(self) -> Graph: ... + +def _create_graph_by_tracing( + func: Callable[..., Any], + inputs: Any, + var_name_lookup_fn: Callable[[Tensor], str], + strict: Any, + force_outplace: Any, + self: Any = None, + argument_names: list[str] = ..., +) -> tuple[Graph, Stack]: ... +def _tracer_warn_use_python(): ... +def _get_tracing_state() -> TracingState: ... + +# Defined in torch/csrc/jit/python/python_ir.cpp +# Not actually defined in python_ir.cpp, not sure where they are. +class IValue: ... + +Stack: TypeAlias = list[IValue] + +class JitType: + annotation_str: str + def isSubtypeOf(self, other: JitType) -> _bool: ... + def with_dtype(self, dtype: _dtype) -> JitType: ... + def with_sizes(self, sizes: list[_int | None]) -> JitType: ... + def kind(self) -> str: ... + def scalarType(self) -> str | None: ... + def getElementType(self) -> JitType: ... + def dtype(self) -> _dtype | None: ... + +class InferredType: + def __init__(self, arg: JitType | str) -> None: ... + def type(self) -> JitType: ... + def success(self) -> _bool: ... + def reason(self) -> str: ... + +class Type(JitType): + def str(self) -> _str: ... + def containedTypes(self) -> list[JitType]: ... + def dim(self) -> _int | None: ... + def undefined(self) -> _bool | None: ... + def sizes(self) -> list[_int] | None: ... + def symbol_sizes(self) -> list[_int] | None: ... + def varyingSizes(self) -> list[_int | None] | None: ... + def strides(self) -> list[_int] | None: ... + def contiguous(self) -> Self: ... + def device(self) -> _device | None: ... + def is_interface_type(self) -> _bool: ... + def requires_grad(self) -> _bool: ... + @property + def annotation_string(self) -> _str: ... + +class AnyType(JitType): + @staticmethod + def get() -> AnyType: ... + +class NoneType(JitType): + @staticmethod + def get() -> NoneType: ... + +class BoolType(JitType): + @staticmethod + def get() -> BoolType: ... + +class FloatType(JitType): + @staticmethod + def get() -> FloatType: ... + +class ComplexType(JitType): + @staticmethod + def get() -> ComplexType: ... + +class IntType(JitType): + @staticmethod + def get() -> IntType: ... + +class SymIntType(JitType): + @staticmethod + def get() -> SymIntType: ... + +class SymBoolType(JitType): + @staticmethod + def get() -> SymBoolType: ... + +class NumberType(JitType): + @staticmethod + def get() -> NumberType: ... + +class StringType(JitType): + @staticmethod + def get() -> StringType: ... + +class DeviceObjType(JitType): + @staticmethod + def get() -> DeviceObjType: ... + +class _GeneratorType(JitType): + @staticmethod + def get() -> _GeneratorType: ... + +class StreamObjType(JitType): + @staticmethod + def get() -> StreamObjType: ... + +class ListType(JitType): + def __init__(self, a: JitType) -> None: ... + def getElementType(self) -> JitType: ... + @staticmethod + def ofInts() -> ListType: ... + @staticmethod + def ofTensors() -> ListType: ... + @staticmethod + def ofFloats() -> ListType: ... + @staticmethod + def ofComplexDoubles() -> ListType: ... + @staticmethod + def ofBools() -> ListType: ... + @staticmethod + def ofStrings() -> ListType: ... + +class DictType(JitType): + def __init__(self, key: JitType, value: JitType) -> None: ... + def getKeyType(self) -> JitType: ... + def getValueType(self) -> JitType: ... + +class TupleType(JitType): + def __init__(self, a: list[JitType | None]) -> None: ... + def elements(self) -> list[JitType]: ... + +class UnionType(JitType): + def __init__(self, a: list[JitType]) -> None: ... + +class ClassType(JitType): + def __init__(self, qualified_name: str) -> None: ... + def qualified_name(self) -> str: ... + +class InterfaceType(JitType): + def __init__(self, qualified_name: str) -> None: ... + def getMethod(self, name: str) -> FunctionSchema | None: ... + def getMethodNames(self) -> list[str]: ... + +JitTypeT = TypeVar("JitTypeT", bound=JitType) # noqa: PYI001 + +class OptionalType(JitType, Generic[JitTypeT]): + def __init__(self, a: JitTypeT) -> None: ... + def getElementType(self) -> JitTypeT: ... + @staticmethod + def ofTensor() -> OptionalType: ... + +class FutureType(JitType): + def __init__(self, a: JitType) -> None: ... + def getElementType(self) -> JitType: ... + +class AwaitType(JitType): + def __init__(self, a: JitType) -> None: ... + def getElementType(self) -> JitType: ... + +class RRefType(JitType): + def __init__(self, a: JitType) -> None: ... + +class EnumType(JitType): + def __init__( + self, + qualified_name: str, + value_type: JitType, + enum_names_values: list[Any], + ) -> None: ... + +class TensorType(JitType): + @classmethod + def get(cls) -> TensorType: ... + @classmethod + def getInferred(cls) -> TensorType: ... + def with_sizes(self, other: list[_int | None] | None) -> TensorType: ... + def sizes(self) -> list[_int] | None: ... + def varyingSizes(self) -> list[_int | None] | None: ... + def strides(self) -> list[_int] | None: ... + def device(self) -> _device | None: ... + def dim(self) -> _int: ... + def dtype(self) -> _dtype | None: ... + @staticmethod + def create_from_tensor(t: Tensor) -> TensorType: ... + +# Defined in torch/csrc/jit/python/python_tree_views.cpp +class SourceRange: ... +class TreeView: ... + +class Ident(TreeView): + @property + def name(self) -> str: ... + +class ClassDef(TreeView): ... + +class Def(TreeView): + def name(self) -> Ident: ... + +class Decl(TreeView): ... + +# Defined in torch/csrc/distributed/rpc/init.cpp +def _rpc_init() -> _bool: ... + +# Defined in torch/csrc/distributed/autograd/init.cpp +def _dist_autograd_init() -> _bool: ... + +# Defined in torch/csrc/distributed/c10d/init.cpp +def _c10d_init() -> _bool: ... + +# Defined in torch/csrc/distributed/rpc/testing/init.cpp +def _faulty_agent_init() -> _bool: ... +def _register_py_class_for_device(device: str, cls: Any) -> None: ... + +# Defined in torch/csrc/Module.cpp +def _current_graph_task_id() -> _int: ... +def _current_autograd_node() -> _Node: ... +def _will_engine_execute_node(node: _Node) -> _bool: ... +def _dispatch_key_set(tensor) -> str: ... + +# Defined in torch/csrc/Exceptions.cpp +class AcceleratorError(RuntimeError): ... +class OutOfMemoryError(RuntimeError): ... +class _DistError(RuntimeError): ... +class _DistBackendError(RuntimeError): ... +class _DistStoreError(RuntimeError): ... +class _DistNetworkError(RuntimeError): ... +class _DistQueueEmptyError(_DistStoreError): ... + +# Defined in torch/csrc/profiler/init.cpp +class CapturedTraceback: ... + +def gather_traceback(python: _bool, script: _bool, cpp: _bool) -> CapturedTraceback: ... +def symbolize_tracebacks( + tracebacks: list[CapturedTraceback], +) -> list[dict[str, Any]]: ... +def _load_mobile_module_from_file(filename: str): ... +def _load_mobile_module_from_bytes(bytes_: bytes): ... +def _load_jit_module_from_file(filename: str): ... +def _load_jit_module_from_bytes(bytes_: bytes): ... +def _save_mobile_module(m: LiteScriptModule, filename: str): ... +def _save_jit_module(m: ScriptModule, filename: str, extra_files: dict[str, Any]): ... +def _save_mobile_module_to_bytes(m: LiteScriptModule) -> bytes: ... +def _save_jit_module_to_bytes( + m: ScriptModule, + extra_files: dict[str, Any], +) -> bytes: ... +def _get_module_info_from_flatbuffer(data: bytes): ... +def _jit_resolve_packet(op_name: str, *args, **kwargs) -> str: ... +def _swap_tensor_impl(t1: Tensor, t2: Tensor): ... +def _pickle_save(obj: Any) -> bytes: ... +def _pickle_load_obj(bs: bytes) -> Any: ... + +# Defined in torch/csrc/jit/runtime/static/init.cpp +def _jit_to_static_module(graph_or_module: Graph | ScriptModule) -> Any: ... +def _fuse_to_static_module( + graph_or_module: Graph | ScriptModule, + min_size: _int, +) -> Any: ... + +# Defined in torch/csrc/fx/node.cpp +def _fx_map_aggregate(a: Any, fn: Callable[[Any], Any]) -> Any: ... +def _fx_map_arg(a: Any, fn: Callable[[Any], Any]) -> Any: ... + +class _NodeBase: + _erased: _bool + _prev: FxNode + _next: FxNode + def __init__( + self, + graph: Any, + name: str, + op: str, + target: Any, + return_type: Any, + ) -> None: ... + def _update_args_kwargs(self, args: tuple[Any, ...], kwargs: dict[str, Any]): ... + def _prepend(self, n: FxNode) -> None: ... + def _replace_input_with(self, old_input: FxNode, new_input: FxNode) -> None: ... + def _remove_from_list(self) -> None: ... + def __lt__(self, n: Self) -> _bool: ... + def __gt__(self, n: Self) -> _bool: ... + def __le__(self, n: Self) -> _bool: ... + def __ge__(self, n: Self) -> _bool: ... + +class _NodeIter(Iterator[FxNode]): + def __init__(self, root: FxNode, reversed: _bool) -> None: ... + def __iter__(self) -> Self: ... + def __next__(self) -> FxNode: ... + +# Defined in torch/csrc/inductor/static_cuda_launcher.cpp +class _StaticCudaLauncher: + @staticmethod + def _load_kernel( + cubin_file: str, + func_name: str, + shared_mem_bytes: _int, + device: _int, + ) -> tuple[_int, _int, _int]: ... + @staticmethod + def _launch_kernel( + func: _int, + grid_x: _int, + grid_y: _int, + grid_z: _int, + num_warps: _int, + shared_mem_bytes: _int, + arg_types: str, + args: tuple[Any, ...], + stream: _int, + ) -> None: ... + +# Defined in torch/csrc/cuda/GreenContext.cpp +class GreenContext: + @staticmethod + def create( + num_sms: _int, + device_id: _int, + ) -> GreenContext: ... + def set_context( + self, + ) -> None: ... + def pop_context( + self, + ) -> None: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_aoti.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_aoti.pyi new file mode 100644 index 0000000000000000000000000000000000000000..2f57b5e5e72b1e30d0955bf955c3de22174c70ef --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_aoti.pyi @@ -0,0 +1,164 @@ +from ctypes import c_void_p +from typing import overload, Protocol + +from torch import Tensor + +# Defined in torch/csrc/inductor/aoti_runner/pybind.cpp + +# Tensor to AtenTensorHandle +def unsafe_alloc_void_ptrs_from_tensors(tensors: list[Tensor]) -> list[c_void_p]: ... +def unsafe_alloc_void_ptr_from_tensor(tensor: Tensor) -> c_void_p: ... + +# AtenTensorHandle to Tensor +def alloc_tensors_by_stealing_from_void_ptrs( + handles: list[c_void_p], +) -> list[Tensor]: ... +def alloc_tensor_by_stealing_from_void_ptr( + handle: c_void_p, +) -> Tensor: ... + +class AOTIModelContainerRunner(Protocol): + def run( + self, inputs: list[Tensor], stream_handle: c_void_p = ... + ) -> list[Tensor]: ... + def get_call_spec(self) -> list[str]: ... + def get_constant_names_to_original_fqns(self) -> dict[str, str]: ... + def get_constant_names_to_dtypes(self) -> dict[str, int]: ... + def extract_constants_map(self, use_inactive: bool) -> dict[str, Tensor]: ... + def update_constant_buffer( + self, + tensor_map: dict[str, Tensor], + use_inactive: bool, + validate_full_updates: bool, + user_managed: bool = ..., + ) -> None: ... + def swap_constant_buffer(self) -> None: ... + def free_inactive_constant_buffer(self) -> None: ... + +class AOTIModelContainerRunnerCpu: + def __init__(self, model_so_path: str, num_models: int) -> None: ... + def run( + self, inputs: list[Tensor], stream_handle: c_void_p = ... + ) -> list[Tensor]: ... + def get_call_spec(self) -> list[str]: ... + def get_constant_names_to_original_fqns(self) -> dict[str, str]: ... + def get_constant_names_to_dtypes(self) -> dict[str, int]: ... + def extract_constants_map(self, use_inactive: bool) -> dict[str, Tensor]: ... + def update_constant_buffer( + self, + tensor_map: dict[str, Tensor], + use_inactive: bool, + validate_full_updates: bool, + user_managed: bool = ..., + ) -> None: ... + def swap_constant_buffer(self) -> None: ... + def free_inactive_constant_buffer(self) -> None: ... + +class AOTIModelContainerRunnerCuda: + @overload + def __init__(self, model_so_path: str, num_models: int) -> None: ... + @overload + def __init__( + self, model_so_path: str, num_models: int, device_str: str + ) -> None: ... + @overload + def __init__( + self, model_so_path: str, num_models: int, device_str: str, cubin_dir: str + ) -> None: ... + def run( + self, inputs: list[Tensor], stream_handle: c_void_p = ... + ) -> list[Tensor]: ... + def get_call_spec(self) -> list[str]: ... + def get_constant_names_to_original_fqns(self) -> dict[str, str]: ... + def get_constant_names_to_dtypes(self) -> dict[str, int]: ... + def extract_constants_map(self, use_inactive: bool) -> dict[str, Tensor]: ... + def update_constant_buffer( + self, + tensor_map: dict[str, Tensor], + use_inactive: bool, + validate_full_updates: bool, + user_managed: bool = ..., + ) -> None: ... + def swap_constant_buffer(self) -> None: ... + def free_inactive_constant_buffer(self) -> None: ... + +class AOTIModelContainerRunnerXpu: + @overload + def __init__(self, model_so_path: str, num_models: int) -> None: ... + @overload + def __init__( + self, model_so_path: str, num_models: int, device_str: str + ) -> None: ... + @overload + def __init__( + self, model_so_path: str, num_models: int, device_str: str, kernel_bin_dir: str + ) -> None: ... + def run( + self, inputs: list[Tensor], stream_handle: c_void_p = ... + ) -> list[Tensor]: ... + def get_call_spec(self) -> list[str]: ... + def get_constant_names_to_original_fqns(self) -> dict[str, str]: ... + def get_constant_names_to_dtypes(self) -> dict[str, int]: ... + def extract_constants_map(self, use_inactive: bool) -> dict[str, Tensor]: ... + def update_constant_buffer( + self, + tensor_map: dict[str, Tensor], + use_inactive: bool, + validate_full_updates: bool, + user_managed: bool = ..., + ) -> None: ... + def swap_constant_buffer(self) -> None: ... + def free_inactive_constant_buffer(self) -> None: ... + +class AOTIModelContainerRunnerMps: + def __init__(self, model_so_path: str, num_models: int) -> None: ... + def run( + self, inputs: list[Tensor], stream_handle: c_void_p = ... + ) -> list[Tensor]: ... + def get_call_spec(self) -> list[str]: ... + def get_constant_names_to_original_fqns(self) -> dict[str, str]: ... + def get_constant_names_to_dtypes(self) -> dict[str, int]: ... + def extract_constants_map(self, use_inactive: bool) -> dict[str, Tensor]: ... + def update_constant_buffer( + self, + tensor_map: dict[str, Tensor], + use_inactive: bool, + validate_full_updates: bool, + user_managed: bool = ..., + ) -> None: ... + def swap_constant_buffer(self) -> None: ... + def free_inactive_constant_buffer(self) -> None: ... + +# Defined in torch/csrc/inductor/aoti_package/pybind.cpp +class AOTIModelPackageLoader: + def __init__( + self, + model_package_path: str, + model_name: str, + run_single_threaded: bool, + num_runners: int, + device_index: int, + ) -> None: ... + def get_metadata(self) -> dict[str, str]: ... + def run( + self, inputs: list[Tensor], stream_handle: c_void_p = ... + ) -> list[Tensor]: ... + def boxed_run( + self, inputs: list[Tensor], stream_handle: c_void_p = ... + ) -> list[Tensor]: ... + def get_call_spec(self) -> list[str]: ... + def get_constant_fqns(self) -> list[str]: ... + def load_constants( + self, + constants_map: dict[str, Tensor], + use_inactive: bool, + check_full_update: bool, + user_managed: bool = ..., + ) -> None: ... + def update_constant_buffer( + self, + tensor_map: dict[str, Tensor], + use_inactive: bool, + validate_full_updates: bool, + user_managed: bool = ..., + ) -> None: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_autograd.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_autograd.pyi new file mode 100644 index 0000000000000000000000000000000000000000..1ff5d847b61aa4f29245d0a08a4d316100992f25 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_autograd.pyi @@ -0,0 +1,144 @@ +# mypy: allow-untyped-defs +from collections.abc import Callable +from enum import Enum +from typing import Any + +import torch +from torch._C._profiler import ( + _ProfilerEvent, + ActiveProfilerType, + ProfilerActivity, + ProfilerConfig, +) + +# Defined in torch/csrc/autograd/init.cpp + +class DeviceType(Enum): + CPU = ... + CUDA = ... + XPU = ... + MKLDNN = ... + OPENGL = ... + OPENCL = ... + IDEEP = ... + HIP = ... + FPGA = ... + MAIA = ... + XLA = ... + MTIA = ... + MPS = ... + HPU = ... + Meta = ... + Vulkan = ... + Metal = ... + PrivateUse1 = ... + +class ProfilerEvent: + def cpu_elapsed_us(self, other: ProfilerEvent) -> float: ... + def cpu_memory_usage(self) -> int: ... + def cuda_elapsed_us(self, other: ProfilerEvent) -> float: ... + def privateuse1_elapsed_us(self, other: ProfilerEvent) -> float: ... + def cuda_memory_usage(self) -> int: ... + def device(self) -> int: ... + def handle(self) -> int: ... + def has_cuda(self) -> bool: ... + def is_remote(self) -> bool: ... + def kind(self) -> int: ... + def name(self) -> str: ... + def node_id(self) -> int: ... + def sequence_nr(self) -> int: ... + def shapes(self) -> list[list[int]]: ... + def thread_id(self) -> int: ... + def flops(self) -> float: ... + def is_async(self) -> bool: ... + +class _KinetoEvent: + def name(self) -> str: ... + def overload_name(self) -> str: ... + def device_index(self) -> int: ... + def device_resource_id(self) -> int: ... + def start_ns(self) -> int: ... + def end_ns(self) -> int: ... + def duration_ns(self) -> int: ... + def is_async(self) -> bool: ... + def linked_correlation_id(self) -> int: ... + def shapes(self) -> list[list[int]]: ... + def dtypes(self) -> list[str]: ... + def concrete_inputs(self) -> list[Any]: ... + def kwinputs(self) -> dict[str, Any]: ... + def device_type(self) -> DeviceType: ... + def start_thread_id(self) -> int: ... + def end_thread_id(self) -> int: ... + def correlation_id(self) -> int: ... + def fwd_thread_id(self) -> int: ... + def stack(self) -> list[str]: ... + def scope(self) -> int: ... + def sequence_nr(self) -> int: ... + def flops(self) -> int: ... + def cuda_elapsed_us(self) -> int: ... + def privateuse1_elapsed_us(self) -> int: ... + def is_user_annotation(self) -> bool: ... + def is_hidden_event(self) -> bool: ... + def metadata_json(self) -> str: ... + +class _ProfilerResult: + def events(self) -> list[_KinetoEvent]: ... + def legacy_events(self) -> list[list[ProfilerEvent]]: ... + def save(self, path: str) -> None: ... + def experimental_event_tree(self) -> list[_ProfilerEvent]: ... + def trace_start_ns(self) -> int: ... + +class SavedTensor: ... + +def _enable_profiler( + config: ProfilerConfig, + activities: set[ProfilerActivity], +) -> None: ... +def _prepare_profiler( + config: ProfilerConfig, + activities: set[ProfilerActivity], +) -> None: ... +def _toggle_collection_dynamic( + enable: bool, + activities: set[ProfilerActivity], +) -> None: ... +def _disable_profiler() -> _ProfilerResult: ... +def _profiler_enabled() -> bool: ... +def _add_metadata_json(key: str, value: str) -> None: ... +def _kineto_step() -> None: ... +def _get_current_graph_task_keep_graph() -> bool: ... +def _get_sequence_nr() -> int: ... +def kineto_available() -> bool: ... +def _record_function_with_args_enter(name: str, *args) -> torch.Tensor: ... +def _record_function_with_args_exit(handle: torch.Tensor) -> None: ... +def _supported_activities() -> set[ProfilerActivity]: ... +def _enable_record_function(enable: bool) -> None: ... +def _set_empty_test_observer(is_global: bool, sampling_prob: float) -> None: ... +def _push_saved_tensors_default_hooks( + pack_hook: Callable[[torch.Tensor], Any], + unpack_hook: Callable[[Any], torch.Tensor], +) -> None: ... +def _pop_saved_tensors_default_hooks() -> None: ... +def _top_saved_tensors_default_hooks( + ignore_is_tracing: bool, +) -> tuple[Callable[[torch.Tensor], Any], Callable[[Any], torch.Tensor]]: ... +def _unsafe_set_version_counter( + t: tuple[torch.Tensor, ...], prev_version: tuple[int, ...] +) -> None: ... +def _enable_profiler_legacy(config: ProfilerConfig) -> None: ... +def _disable_profiler_legacy() -> list[list[ProfilerEvent]]: ... +def _profiler_type() -> ActiveProfilerType: ... +def _saved_tensors_hooks_enable() -> None: ... +def _saved_tensors_hooks_disable(message: str, fail_if_non_empty=True) -> None: ... +def _saved_tensors_hooks_get_disabled_error_message() -> str | None: ... +def _saved_tensors_hooks_set_tracing(is_tracing: bool) -> bool: ... + +class CreationMeta(Enum): + DEFAULT = ... + IN_CUSTOM_FUNCTION = ... + MULTI_OUTPUT_NODE = ... + NO_GRAD_MODE = ... + INFERENCE_MODE = ... + +def _set_creation_meta(t: torch.Tensor, creation_meta: CreationMeta) -> None: ... +def _get_creation_meta(t: torch.Tensor) -> CreationMeta: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_cpu.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_cpu.pyi new file mode 100644 index 0000000000000000000000000000000000000000..a667edc721a9c13b26a095fee1ba653c542c09af --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_cpu.pyi @@ -0,0 +1,13 @@ +from torch.types import _bool, _int + +# Defined in torch/csrc/cpu/Module.cpp + +def _is_avx2_supported() -> _bool: ... +def _is_avx512_supported() -> _bool: ... +def _is_avx512_vnni_supported() -> _bool: ... +def _is_avx512_bf16_supported() -> _bool: ... +def _is_amx_tile_supported() -> _bool: ... +def _is_amx_fp16_supported() -> _bool: ... +def _init_amx() -> _bool: ... +def _L1d_cache_size() -> _int: ... +def _L2_cache_size() -> _int: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_cudnn.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_cudnn.pyi new file mode 100644 index 0000000000000000000000000000000000000000..cfea3f956f2a373dfe6895743a49a3c6cec439fa --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_cudnn.pyi @@ -0,0 +1,14 @@ +from enum import IntEnum + +# Defined in torch/csrc/cuda/shared/cudnn.cpp +is_cuda: bool + +def getRuntimeVersion() -> tuple[int, int, int]: ... +def getCompileVersion() -> tuple[int, int, int]: ... +def getVersionInt() -> int: ... + +class RNNMode(IntEnum): + rnn_relu = ... + rnn_tanh = ... + lstm = ... + gru = ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_cusparselt.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_cusparselt.pyi new file mode 100644 index 0000000000000000000000000000000000000000..a1c4bbb217777d50b9a3384da11d9992db5e18f0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_cusparselt.pyi @@ -0,0 +1 @@ +def getVersionInt() -> int: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed.pyi new file mode 100644 index 0000000000000000000000000000000000000000..c8416f60edca35493ad7c1703cf2e27d58975be0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed.pyi @@ -0,0 +1,21 @@ +# This module is defined in torch/csrc/distributed/python_placement.cpp + +class Placement: + def is_partial(self, reduce_op: str | None = None) -> bool: ... + def is_replicate(self) -> bool: ... + def is_shard(self, dim: int | None = None) -> bool: ... + +class Shard(Placement): + dim: int + def __init__(self, dim: int): ... + +class StridedShard(Placement): + dim: int + split_factor: int + def __init__(self, dim: int, *, split_factor: int): ... + +class Replicate(Placement): ... + +class Partial(Placement): + reduce_op: str + def __init__(self, reduce_op: str | None = None): ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed_autograd.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed_autograd.pyi new file mode 100644 index 0000000000000000000000000000000000000000..6e1e39bec2927c046455928ff74bc7de9683e66f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed_autograd.pyi @@ -0,0 +1,26 @@ +from typing import Any + +import torch + +# This module is defined in torch/csrc/distributed/autograd/init.cpp + +class DistAutogradContext: + def _context_id(self) -> int: ... + def _recv_functions(self) -> dict[int, Any]: ... + def _send_functions(self) -> dict[int, Any]: ... + def _known_worker_ids(self) -> set[int]: ... + +def _new_context() -> DistAutogradContext: ... +def _release_context(context_id: int) -> None: ... +def _get_max_id() -> int: ... +def _is_valid_context(worker_id: int) -> bool: ... +def _retrieve_context(context_id: int) -> DistAutogradContext: ... +def _current_context() -> DistAutogradContext: ... +def _init(worker_id: int) -> None: ... +def _get_debug_info() -> dict[str, str]: ... +def backward( + context_id: int, + roots: list[torch.Tensor], + retain_graph: bool = False, +) -> None: ... +def get_gradients(context_id: int) -> dict[torch.Tensor, torch.Tensor]: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed_c10d.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed_c10d.pyi new file mode 100644 index 0000000000000000000000000000000000000000..1bcb2d498ad2d43944082ce447677bb5467433d7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed_c10d.pyi @@ -0,0 +1,889 @@ +# mypy: allow-untyped-defs +# mypy: disable-error-code="type-arg" +from collections.abc import Callable +from datetime import timedelta +from enum import Enum +from typing import Any, Optional, overload, Union + +import torch +from torch import Tensor +from torch._C import ScriptObject +from torch._C._autograd import DeviceType +from torch.distributed.distributed_c10d import GroupName +from torch.futures import Future + +# This module is defined in torch/csrc/distributed/c10d/init.cpp + +_DEFAULT_FIRST_BUCKET_BYTES: int +_DEFAULT_NO_TIMEOUT: timedelta +_DEFAULT_PG_TIMEOUT: timedelta +_DEFAULT_PG_NCCL_TIMEOUT: timedelta + +class BuiltinCommHookType(Enum): + ALLREDUCE = ... + FP16_COMPRESS = ... + +def _register_comm_hook(reducer: Reducer, state: Any, comm_hook: Any): ... +def _register_builtin_comm_hook( + reducer: Reducer, + comm_hook_type: BuiltinCommHookType, +): ... +def _set_global_rank(rank: int) -> None: ... +def _hash_tensors(tensors: list[Tensor]) -> int: ... + +class GradBucket: + def index(self) -> int: ... + def buffer(self) -> Tensor: ... + def gradients(self) -> list[Tensor]: ... + def is_last(self) -> bool: ... + def set_buffer(self, tensor: Tensor) -> None: ... + def parameters(self) -> list[Tensor]: ... + +class Reducer: + def __init__( + self, + params: list[Tensor], + bucket_indices: list[list[int]], + per_bucket_size_limits: list[int], + process_group: ProcessGroup, + expect_sparse_gradients: list[bool] = ..., + bucket_bytes_cap: int = ..., # kDefaultBucketBytesCap in reducer.hpp + find_unused_parameters: bool = ..., + gradient_as_bucket_view: bool = ..., + param_to_name_mapping: dict[int, str] = ..., + first_bucket_types_cap: int = ..., # kDefaultFirstBucketBytes in reducer.hpp + skip_all_reduce_unused_params: bool = ..., + use_python_reducer: bool = ..., + ) -> None: ... + def prepare_for_forward(self) -> None: ... + def prepare_for_backward(self, output: list[Tensor]) -> None: ... + def get_backward_stats(self) -> list[int]: ... + def _install_post_backward_futures(self, futures: list[Future]) -> None: ... + def _rebuild_buckets(self) -> bool: ... + def _get_zeros_like_grad_buckets(self) -> list[GradBucket]: ... + def _push_all_rebuilt_params(self) -> None: ... + def _set_forward_pass_work_handle( + self, + work: Work, + use_static_world_size: bool, + ): ... + def _get_local_used_map(self) -> Tensor: ... + def _set_ddp_runtime_logging_sample_rate(self, sample_rate: int) -> None: ... + def _set_static_graph(self) -> None: ... + def _run_comm_hook(self, bucket: GradBucket) -> Future: ... + def set_logger(self, logger: Logger) -> None: ... + def _remove_autograd_hooks(self) -> None: ... + def _check_reducer_finalized(self) -> None: ... + def _set_sparse_metadata(self, global_unique_ids: dict[str, Tensor]) -> None: ... + def _reset_state(self) -> None: ... + def _update_process_group(self, new_process_group: ProcessGroup) -> None: ... + +class DDPLoggingData: + strs_map: dict[str, str] + ints_map: dict[str, int] + +class Logger: + def __init__(self, reducer: Reducer) -> None: ... + def set_construction_data_and_log( + self, + module_name: str, + device_ids: list[int], + output_device: int, + broadcast_buffers: bool, + has_sync_bn: bool, + static_graph: bool, + ): ... + def set_runtime_stats_and_log(self) -> None: ... + def set_error_and_log(self, error: str) -> None: ... + def _get_ddp_logging_data(self) -> DDPLoggingData: ... + def _set_comm_hook_name(self, comm_hook: str) -> None: ... + def _set_uneven_input_join(self) -> None: ... + def _set_static_graph(self) -> None: ... + +class _WorkerServer: + port: int + + def __init__(self, host_or_file: str, port: int = ...) -> None: ... + def shutdown(self) -> None: ... + +def get_debug_level(): ... +def set_debug_level(): ... +def set_debug_level_from_env(): ... + +class DebugLevel(Enum): + OFF = ... + INFO = ... + DETAIL = ... + +class ReduceOp: + # pyrefly: ignore # unknown-name + def __init__(self, op: RedOpType) -> None: ... + + # pyrefly: ignore # unknown-name + SUM: RedOpType = ... + # pyrefly: ignore # unknown-name + AVG: RedOpType = ... + # pyrefly: ignore # unknown-name + PRODUCT: RedOpType = ... + # pyrefly: ignore # unknown-name + MIN: RedOpType = ... + # pyrefly: ignore # unknown-name + MAX: RedOpType = ... + # pyrefly: ignore # unknown-name + BAND: RedOpType = ... + # pyrefly: ignore # unknown-name + BOR: RedOpType = ... + # pyrefly: ignore # unknown-name + BXOR: RedOpType = ... + # pyrefly: ignore # unknown-name + PREMUL_SUM: RedOpType = ... + # pyrefly: ignore # unknown-name + UNUSED: RedOpType = ... + + # mypy error being ignored: + # Detected enum "torch._C._distributed_c10d.ReduceOp.RedOpType" in a type + # stub with zero members. There is a chance this is due to a recent change + # in the semantics of enum membership. If so, use `member = value` to mark + # an enum member, instead of `member: type` + class RedOpType(Enum): ... # type: ignore[misc] + +class BroadcastOptions: + rootRank: int + rootTensor: int + timeout: timedelta + asyncOp: bool + +class AllreduceOptions: + reduceOp: ReduceOp + timeout: timedelta + asyncOp: bool + sparseIndices: Optional[Tensor] + +class AllreduceCoalescedOptions(AllreduceOptions): ... + +class ReduceOptions: + reduceOp: ReduceOp + rootRank: int + rootTensor: int + timeout: timedelta + asyncOp: bool + +class AllgatherOptions: + timeout: timedelta + asyncOp: bool + +class GatherOptions: + rootRank: int + timeout: timedelta + asyncOp: bool + +class ScatterOptions: + rootRank: int + timeout: timedelta + asyncOp: bool + +class ReduceScatterOptions: + reduceOp: ReduceOp + timeout: timedelta + asyncOp: bool + +class BarrierOptions: + device_ids: list[int] + device: torch.device + timeout: timedelta + asyncOp: bool + +class AllToAllOptions: + timeout: timedelta + asyncOp: bool + +class Store: + def set(self, key: str, value: str): ... + def get(self, key: str) -> bytes: ... + def add(self, key: str, value: int) -> int: ... + def check(self, keys: list[str]) -> bool: ... + def compare_set( + self, + key: str, + expected_value: str, + desired_value: str, + ) -> bytes: ... + def delete_key(self, key: str) -> bool: ... + def multi_get(self, keys: list[str]) -> list[bytes]: ... + def num_keys(self) -> int: ... + def set_timeout(self, timeout: timedelta): ... + @overload + def wait(self, keys: list[str]): ... + @overload + def wait(self, keys: list[str], timeout: timedelta): ... + def queue_pop(self, key: str, block: bool = True) -> bytes: ... + def queue_push(self, key: str, value: Union[bytes, str]) -> None: ... + def queue_len(self, key: str) -> int: ... + def list_keys(self) -> list[str]: ... + +class FileStore(Store): + def __init__(self, path: str, numWorkers: int = ...) -> None: ... + +class HashStore(Store): + def __init__(self) -> None: ... + +class TCPStore(Store): + def __init__( + self, + host_name: str, + port: int, + world_size: int | None = ..., + is_master: bool = ..., + timeout: timedelta = ..., + wait_for_workers: bool = ..., + multi_tenant: bool = ..., + master_listen_fd: int | None = ..., + use_libuv: bool | None = ..., + ) -> None: ... + @property + def host(self) -> str: ... + @property + def port(self) -> int: ... + +class PrefixStore(Store): + def __init__(self, prefix: str, store: Store) -> None: ... + @property + def underlying_store(self) -> Store: ... + +class _ControlCollectives: + def barrier(self, key: str, timeout: timedelta, blocking: bool) -> None: ... + def broadcast_send(self, key: str, data: str, timeout: timedelta) -> None: ... + def broadcast_recv(self, key: str, timeout: timedelta) -> str: ... + def gather_send(self, key: str, data: str, timeout: timedelta) -> None: ... + def gather_recv(self, key: str, timeout: timedelta) -> str: ... + def scatter_send(self, key: str, data: str, timeout: timedelta) -> None: ... + def scatter_recv(self, key: str, timeout: timedelta) -> str: ... + def all_gather(self, key: str, data: str, timeout: timedelta) -> str: ... + def all_sum(self, key: str, data: int, timeout: timedelta) -> int: ... + +class _StoreCollectives(_ControlCollectives): + def __init__(self, store: Store, rank: int, world_size: int) -> None: ... + +class _DistributedBackendOptions: + def __init__(self) -> None: ... + @property + def store(self) -> Store: ... + @store.setter + def store(self, store: Store) -> None: ... + @property + def group_rank(self) -> int: ... + @group_rank.setter + def group_rank(self, rank: int) -> None: ... + @property + def group_size(self) -> int: ... + @group_size.setter + def group_size(self, size: int) -> None: ... + @property + def timeout(self) -> timedelta: ... + @timeout.setter + def timeout(self, timeout: timedelta) -> None: ... + @property + def group_id(self) -> str: ... + @group_id.setter + def group_id(self, group_id: str) -> None: ... + @property + def global_ranks_in_group(self) -> list[int]: ... + @global_ranks_in_group.setter + def global_ranks_in_group(self, ranks: list[int]) -> None: ... + +class Work: + def is_completed(self) -> bool: ... + def is_success(self) -> bool: ... + def exception(self) -> Any: ... + def wait(self, timeout: timedelta = ...) -> bool: ... + def block_current_stream(self) -> None: ... + def get_future(self) -> Future: ... + def source_rank(self) -> int: ... + def _source_rank(self) -> int: ... + def result(self) -> list[Tensor]: ... + def synchronize(self) -> None: ... + def boxed(self) -> ScriptObject: ... + @staticmethod + def unbox(obj: ScriptObject) -> Work: ... + +class Backend: + class Options: + def __init__(self, backend: str, timeout: timedelta = ...) -> None: ... + @property + def backend(self) -> str: ... + @property + def _timeout(self) -> timedelta: ... + @_timeout.setter + def _timeout(self, val: timedelta) -> None: ... + global_ranks_in_group: list[int] + group_name: GroupName + + def __init__( + self, + rank: int, + size: int, + ) -> None: ... + @property + def supports_splitting(self) -> bool: ... + @property + def supports_coalescing(self) -> bool: ... + @property + def supports_time_estimate(self) -> bool: ... + def set_timeout(self, timeout: timedelta) -> None: ... + @property + def options(self) -> Options: ... + def rank(self) -> int: ... + def size(self) -> int: ... + def name(self) -> str: ... + def abort(self) -> None: ... + def shutdown(self) -> None: ... + def eager_connect_single_device(self, device: torch.device | None) -> None: ... + def _set_sequence_number_for_group(self) -> None: ... + def _set_default_timeout(self, timeout: timedelta) -> None: ... + def get_error(self) -> ErrorType: ... + def supports_tensor_alloc(self, device: torch.device) -> bool: ... + def allocate_tensor( + self, + size: int, + *, + dtype: torch.dtype, + device: torch.device, + ) -> Tensor: ... + @property + def mem_allocator(self) -> Any: ... + +class ProcessGroup: + class BackendType(Enum): + UNDEFINED = ... + GLOO = ... + NCCL = ... + UCC = ... + MPI = ... + XCCL = ... + CUSTOM = ... + + def __init__( + self, + store: Store, + rank: int, + size: int, + ) -> None: ... + def rank(self) -> int: ... + def size(self) -> int: ... + def get_group_store(self) -> Store: ... + def split_group( + self, + new_ranks: list[int], + timeout: Optional[timedelta] = None, + opts: Optional[Backend.Options] = None, + group_name: GroupName | None = None, + group_desc: Optional[str] = None, + ) -> Optional[ProcessGroup]: ... + def merge_remote_group( + self, + store: Store, + size: int, + timeout: timedelta, + group_name: GroupName | None = None, + group_desc: Optional[str] = None, + ) -> ProcessGroup: ... + def abort(self) -> None: ... + def set_timeout(self, timeout: timedelta) -> None: ... + def shutdown(self) -> None: ... + @overload + def broadcast( + self, + tensors: list[Tensor], + opts=..., + ) -> Work: ... + @overload + def broadcast( + self, + tensor: Tensor, + root: int, + timeout: timedelta | None = None, + ) -> Work: ... + @overload + def allreduce( + self, + tensors: list[Tensor], + opts: AllreduceOptions = ..., + ) -> Work: ... + @overload + def allreduce( + self, + tensors: list[Tensor], + op=..., + timeout: timedelta | None = None, + ) -> Work: ... + @overload + def allreduce( + self, + tensor: Tensor, + op=..., + timeout: timedelta | None = None, + ) -> Work: ... + def allreduce_coalesced( + self, + tensors: list[Tensor], + opts=..., + ) -> Work: ... + def reduce_scatter_tensor_coalesced( + self, + outputTensors: list[Tensor], + inputTensors: list[Tensor], + opts: ReduceScatterOptions | None = None, + ) -> Work: ... + @overload + def reduce( + self, + tensors: list[Tensor], + opts=..., + ) -> Work: ... + @overload + def reduce( + self, + tensor: Tensor, + root: int, + op=..., + timeout: timedelta | None = None, + ) -> Work: ... + @overload + def allgather( + self, + output_tensors: list[list[Tensor]], + input_tensors: list[Tensor], + opts=..., + ) -> Work: ... + @overload + def allgather( + self, + output_tensors: list[Tensor], + input_tensor: Tensor, + timeout: timedelta | None = None, + ) -> Work: ... + def _allgather_base( + self, + output: Tensor, + input: Tensor, + opts=..., + ) -> Work: ... + def allgather_coalesced( + self, + output_lists: list[list[Tensor]], + input_list: list[Tensor], + opts=..., + ) -> Work: ... + def allgather_into_tensor_coalesced( + self, + output_lists: list[Tensor], + input_list: list[Tensor], + opts=..., + ) -> Work: ... + @overload + def gather( + self, + output_tensors: list[list[Tensor]], + input_tensors: list[Tensor], + opts=..., + ) -> Work: ... + @overload + def gather( + self, + output_tensors: list[Tensor], + input_tensor: Tensor, + root: int, + timeout: timedelta | None = None, + ) -> Work: ... + @overload + def scatter( + self, + output_tensors: list[Tensor], + input_tensors: list[list[Tensor]], + opts=..., + ) -> Work: ... + @overload + def scatter( + self, + output_tensor: Tensor, + input_tensors: list[Tensor], + root: int, + timeout: timedelta | None = None, + ) -> Work: ... + @overload + def reduce_scatter( + self, + output_tensors: list[Tensor], + input_tensors: list[list[Tensor]], + opts=..., + ) -> Work: ... + @overload + def reduce_scatter( + self, + output_tensors: Tensor, + input_tensor: list[Tensor], + op=..., + timeout: timedelta | None = None, + ) -> Work: ... + def _reduce_scatter_base( + self, + outputTensor: Tensor, + inputTensor: Tensor, + opts: ReduceScatterOptions | None, + ) -> Work: ... + @overload + def alltoall_base( + self, + output_tensor: Tensor, + input_tensor: Tensor, + output_split_sizes: list[int], + input_split_sizes: list[int], + opts=..., + ) -> Work: ... + @overload + def alltoall_base( + self, + output: Tensor, + input: Tensor, + output_split_sizes: list[int], + input_split_sizes: list[int], + timeout: timedelta | None = None, + ) -> Work: ... + @overload + def alltoall( + self, + output_tensor: list[Tensor], + input_tensor: list[Tensor], + opts=..., + ) -> Work: ... + @overload + def alltoall( + self, + output: list[Tensor], + input: list[Tensor], + timeout: timedelta | None = None, + ) -> Work: ... + def send( + self, + tensors: list[Tensor], + dstRank: int, + tag: int, + ) -> Work: ... + def recv( + self, + tensors: list[Tensor], + srcRank: int, + tag: int, + ) -> Work: ... + def recv_anysource(self, tensors: list[Tensor], tag: int) -> Work: ... + @overload + def barrier(self, opts=...) -> Work: ... + @overload + def barrier(self, timeout: timedelta | None = None) -> Work: ... + def boxed(self) -> ScriptObject: ... + @staticmethod + def unbox(obj: ScriptObject) -> ProcessGroup: ... + def _start_coalescing(self, device: torch.device) -> None: ... + def _end_coalescing(self, device: torch.device) -> Work: ... + def _get_backend_name(self) -> str: ... + def _backend_id(self, backend_type: BackendType) -> int: ... + @property + def _device_types(self) -> list[torch.device]: ... + def _get_backend(self, device: torch.device) -> Backend: ... + def _set_default_backend(self, backend_type: BackendType) -> None: ... + def _register_backend( + self, + device: torch.device, + backend_type: BackendType, + backend: Backend | None, + ) -> None: ... + def _set_group_name(self, name: GroupName) -> None: ... + def _set_group_desc(self, desc: str) -> None: ... + def name(self) -> str: ... + def _has_hooks(self) -> bool: ... + def _wait_for_pending_works(self) -> None: ... + def _set_sequence_number_for_group(self) -> None: ... + @property + def bound_device_id(self) -> torch.device | None: ... + @bound_device_id.setter + def bound_device_id(self, device: torch.device | None) -> None: ... + @property + def group_name(self) -> GroupName: ... + @property + def group_desc(self) -> str: ... + +class FakeProcessGroup(Backend): + @staticmethod + def _create_internal(rank: int, world_size: int) -> FakeProcessGroup: ... + +class FakeWork(Work): + seq_id: int + def __init__(self) -> None: ... + def wait(self, timeout: timedelta = ...) -> bool: ... + def getFuture(self) -> Future: ... + +class PythonCallbackWork(Work): + def __init__(self, callback: Callable[[timedelta], bool]) -> None: ... + def wait(self, timeout: timedelta = ...) -> bool: ... + def get_future(self) -> Future: ... + +class ProcessGroupGloo(Backend): + class Device: ... + + class Options(Backend.Options): + devices: list[ProcessGroupGloo.Device] + threads: int + + def __init__(self): ... + + def __init__( + self, + store: Store, + rank: int, + size: int, + timeout: timedelta, + ) -> None: ... + @staticmethod + def create_device(hostname="", interface="", lazy_init=None) -> Device: ... + @staticmethod + def create_default_device(lazy_init=None) -> Device: ... + def _set_default_timeout(self, timeout) -> None: ... + @property + def options(self) -> Options: ... # type: ignore[override] + +class _ProcessGroupWrapper(Backend): + def __init__(self, pg: Backend, gloo_pg: ProcessGroupGloo) -> None: ... + wrapped_pg: Backend + +class ErrorType(Enum): + SUCCESS = ... + TIMEOUT = ... + COMM_ERROR = ... + REMOTE_ERROR = ... + +class ProcessGroupNCCL(Backend): + class NCCLConfig: + blocking: int + cga_cluster_size: int + min_ctas: int + max_ctas: int + def unsafe_get_ptr(self) -> int: ... + + class Options(Backend.Options): + config: ProcessGroupNCCL.NCCLConfig + is_high_priority_stream: bool + split_from: ProcessGroupNCCL + split_color: int + + def __init__(self, is_high_priority_stream: bool = False): ... + + def __init__( + self, + store: Store, + rank: int, + size: int, + options: Options, + ) -> None: ... + def _group_start(self) -> None: ... + def _group_end(self) -> None: ... + def _start_time_estimate(self) -> None: ... + def _end_time_estimate(self) -> float: ... + def _set_default_timeout(self, timeout) -> None: ... + def perform_nocolor_split(self, device: torch.device) -> None: ... + def register_mem_pool(self, pool: torch.cuda.MemPool) -> None: ... + def deregister_mem_pool(self, pool: torch.cuda.MemPool) -> None: ... + def comm_split_count(self) -> int: ... + def _add_ephemeral_timeout(self, timeout: timedelta) -> None: ... + def abort(self) -> None: ... + def _is_initialized(self) -> bool: ... + @property + def uid(self) -> int: ... + @property + def options(self) -> Options: ... # type: ignore[override] + @staticmethod + def get_build_nccl_version(self) -> tuple[int, int, int]: ... + @staticmethod + def get_runtime_nccl_version(self) -> tuple[int, int, int]: ... + +class ProcessGroupUCC(Backend): + def __init__( + self, + store: Store, + rank: int, + size: int, + timeout: timedelta, + ) -> None: ... + +class ProcessGroupMPI(Backend): + def __init__( + self, + rank: int, + size: int, + pgComm: int, + ) -> None: ... + @staticmethod + def create(ranks: list[int]) -> ProcessGroupMPI: ... + +def _compute_bucket_assignment_by_size( + tensors: list[Tensor], + bucket_size_limits: list[int], + expect_sparse_gradient: list[bool] = ..., + tensor_indices: list[int] = ..., +) -> tuple[list[list[int]], list[int]]: ... +def _broadcast_coalesced( + process_group: ProcessGroup, + tensors: list[Tensor], + buffer_size: int, + src: int, +): ... +def _test_python_store(store: Store): ... +def _verify_params_across_processes( + process_group: ProcessGroup, + params: list[Tensor], + logger: Logger | None, +): ... +def _make_nccl_premul_sum(factor: float | list[Tensor]) -> ReduceOp: ... +def _register_process_group( + group_name: GroupName, + process_group: ProcessGroup, +) -> None: ... +def _resolve_process_group(group_name: GroupName) -> ProcessGroup: ... +def _register_work(tensor: torch.Tensor, work: Work) -> ProcessGroup: ... +def _get_work_registry_size() -> int: ... +def _set_allow_inflight_collective_as_graph_input( + value: bool, +) -> None: ... +def _allow_inflight_collective_as_graph_input() -> bool: ... +def _unregister_all_process_groups() -> None: ... +def _unregister_process_group(group_name: GroupName) -> None: ... + +# Initializes the device state in CUmodule so that it's able to perform NVSHMEM +# operations. CUmodule is a pointer to a CUDA module, carried by a int64 in +# Python. At C++ interface, it is converted to a uintptr_t. +def _nvshmemx_cumodule_init(module: int) -> None: ... + +# Check if NVSHMEM is available on current system. +def _is_nvshmem_available() -> bool: ... + +class _SymmetricMemory: + @staticmethod + def set_group_info( + group_name: str, + rank: int, + world_size: int, + store: Store, + ) -> None: ... + @staticmethod + def empty_strided_p2p( + size: torch.types._size, + stride: torch.types._size, + dtype: torch.dtype, + device: torch.device, + group_name: str | None = None, + alloc_id: int | None = None, + ) -> torch.Tensor: ... + @staticmethod + def has_multicast_support( + device_type: DeviceType, + device_idx: int, + ) -> bool: ... + # Set Symmetric Memory allocation backend. + @staticmethod + def set_backend(name: str) -> None: ... + @staticmethod + def get_backend(device: torch.device) -> Optional[str]: ... + @staticmethod + def get_mempool_allocator(device: torch.device) -> Any: ... + signal_pad_size: int + @property + def rank(self) -> int: ... + @property + def world_size(self) -> int: ... + @staticmethod + def rendezvous( + tensor: torch.Tensor, group_name: str | None = None + ) -> _SymmetricMemory: ... + def get_buffer( + self, + rank: int, + sizes: torch.types._size, + dtype: torch.dtype, + storage_offset: int | None = 0, + ) -> torch.Tensor: ... + def get_signal_pad( + self, + rank: int, + sizes: torch.types._size = [], + dtype: torch.dtype | None = None, + storage_offset: int | None = 0, + ) -> torch.Tensor: ... + def barrier(self, channel: int = 0, timeout_ms: int = 0) -> None: ... + def put_signal( + self, + dst_rank: int, + channel: int = 0, + timeout_ms: int = 0, + ) -> None: ... + def wait_signal( + self, + src_rank: int, + channel: int = 0, + timeout_ms: int = 0, + ) -> None: ... + def get_remote_tensor( + self, + peer: int, + sizes: torch.types._size, + dtype: torch.dtype, + ) -> torch.Tensor: ... + @staticmethod + def memset32( + tensor: torch.Tensor, offset: int, val: int, count: int = 1 + ) -> torch.Tensor: ... + @staticmethod + def stream_write_value32( + tensor: torch.Tensor, offset: int, val: int + ) -> torch.Tensor: ... + @property + def buffer_ptrs(self) -> list[int]: ... + @property + def buffer_ptrs_dev(self) -> int: ... + @property + def signal_pad_ptrs(self) -> list[int]: ... + @property + def signal_pad_ptrs_dev(self) -> int: ... + @property + def multicast_ptr(self) -> int: ... + @property + def buffer_size(self) -> int: ... + +class ProcessGroupXCCL(Backend): + class Options(Backend.Options): + is_high_priority_stream: bool + + def __init__(self, is_high_priority_stream: bool = False): ... + + def __init__( + self, + store: Store, + rank: int, + size: int, + options: Options, + ) -> None: ... + @property + def options(self) -> Options: ... # type: ignore[override] + +def _set_process_group(pg: ProcessGroup) -> None: ... +def _current_process_group() -> ProcessGroup: ... + +class _Request: + def body(self) -> bytes: ... + def get_param(self, str) -> str: ... + +class _Response: + def set_content(self, content: str | bytes, content_type: str) -> None: ... + def set_status(self, status: int) -> None: ... + +def _register_handler( + name: str, handler: Callable[[_Request, _Response], None] +) -> None: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed_rpc.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed_rpc.pyi new file mode 100644 index 0000000000000000000000000000000000000000..48f636d8524637305cd9c758e9f22428d3b055cc --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed_rpc.pyi @@ -0,0 +1,188 @@ +# mypy: allow-untyped-defs +# mypy: disable-error-code="type-arg" +from datetime import timedelta +from typing import Any, Generic, overload, TypeVar + +import torch +from torch._C import Future +from torch._C._autograd import ProfilerEvent +from torch._C._distributed_c10d import Store +from torch._C._profiler import ProfilerConfig + +# This module is defined in torch/csrc/distributed/rpc/init.cpp + +_DEFAULT_INIT_METHOD: str +_DEFAULT_NUM_WORKER_THREADS: int +_UNSET_RPC_TIMEOUT: float +_DEFAULT_RPC_TIMEOUT_SEC: float + +_T = TypeVar("_T") + +class RpcBackendOptions: + rpc_timeout: float + init_method: str + def __init__( + self, + rpc_timeout: float = ..., + init_method: str = ..., + ) -> None: ... + +class WorkerInfo: + def __init__(self, name: str, worker_id: int) -> None: ... + @property + def name(self) -> str: ... + @property + def id(self) -> int: ... + def __eq__(self, other: object) -> bool: ... + +class RpcAgent: + def join(self, shutdown: bool = False, timeout: float = 0): ... + def sync(self): ... + def shutdown(self): ... + @overload + def get_worker_info(self) -> WorkerInfo: ... + @overload + def get_worker_info(self, workerName: str) -> WorkerInfo: ... + def get_worker_infos(self) -> list[WorkerInfo]: ... + def _get_device_map(self, dst: WorkerInfo) -> dict[torch.device, torch.device]: ... + def get_debug_info(self) -> dict[str, str]: ... + def get_metrics(self) -> dict[str, str]: ... + +class PyRRef(Generic[_T]): + def __init__(self, value: _T, type_hint: Any = None) -> None: ... + def is_owner(self) -> bool: ... + def confirmed_by_owner(self) -> bool: ... + def owner(self) -> WorkerInfo: ... + def owner_name(self) -> str: ... + def to_here(self, timeout: float = ...) -> _T: ... + def local_value(self) -> Any: ... + def rpc_sync(self, timeout: float = ...) -> Any: ... + def rpc_async(self, timeout: float = ...) -> Any: ... + def remote(self, timeout: float = ...) -> Any: ... + def _serialize(self) -> tuple: ... + @staticmethod + def _deserialize(tp: tuple) -> PyRRef: ... + def _get_type(self) -> type[_T]: ... + def _get_future(self) -> Future[_T]: ... + def _get_profiling_future(self) -> Future[_T]: ... + def _set_profiling_future(self, profilingFuture: Future[_T]): ... + +class _TensorPipeRpcBackendOptionsBase(RpcBackendOptions): + num_worker_threads: int + device_maps: dict[str, dict[torch.device, torch.device]] + devices: list[torch.device] + def __init__( + self, + num_worker_threads: int, + _transports: list | None, + _channels: list | None, + rpc_timeout: float = ..., + init_method: str = ..., + device_maps: dict[str, dict[torch.device, torch.device]] = {}, # noqa: B006 + devices: list[torch.device] = [], # noqa: B006 + ) -> None: ... + def _set_device_map( + self, + to: str, + device_map: dict[torch.device, torch.device], + ): ... + +class TensorPipeAgent(RpcAgent): + def __init__( + self, + store: Store, + name: str, + worker_id: int, + world_size: int | None, + opts: _TensorPipeRpcBackendOptionsBase, + reverse_device_maps: dict[str, dict[torch.device, torch.device]], + devices: list[torch.device], + ) -> None: ... + def join(self, shutdown: bool = False, timeout: float = 0): ... + def shutdown(self): ... + @overload + def get_worker_info(self) -> WorkerInfo: ... + @overload + def get_worker_info(self, workerName: str) -> WorkerInfo: ... + @overload + def get_worker_info(self, id: int) -> WorkerInfo: ... + def get_worker_infos(self) -> list[WorkerInfo]: ... + def _get_device_map(self, dst: WorkerInfo) -> dict[torch.device, torch.device]: ... + def _update_group_membership( + self, + worker_info: WorkerInfo, + my_devices: list[torch.device], + reverse_device_map: dict[str, dict[torch.device, torch.device]], + is_join: bool, + ): ... + def _get_backend_options(self) -> _TensorPipeRpcBackendOptionsBase: ... + @property + def is_static_group(self) -> bool: ... + @property + def store(self) -> Store: ... + +def _is_current_rpc_agent_set() -> bool: ... +def _get_current_rpc_agent() -> RpcAgent: ... +def _set_and_start_rpc_agent(agent: RpcAgent): ... +def _reset_current_rpc_agent(): ... +def _delete_all_user_and_unforked_owner_rrefs(timeout: timedelta = ...): ... +def _destroy_rref_context(ignoreRRefLeak: bool): ... +def _rref_context_get_debug_info() -> dict[str, str]: ... +def _cleanup_python_rpc_handler(): ... +def _invoke_rpc_builtin( + dst: WorkerInfo, + opName: str, + rpcTimeoutSeconds: float, + *args: Any, + **kwargs: Any, +): ... +def _invoke_rpc_python_udf( + dst: WorkerInfo, + pickledPythonUDF: str, + tensors: list[torch.Tensor], + rpcTimeoutSeconds: float, + isAsyncExecution: bool, +): ... +def _invoke_rpc_torchscript( + dstWorkerName: str, + qualifiedNameStr: str, + argsTuple: tuple, + kwargsDict: dict, + rpcTimeoutSeconds: float, + isAsyncExecution: bool, +): ... +def _invoke_remote_builtin( + dst: WorkerInfo, + opName: str, + rpcTimeoutSeconds: float, + *args: Any, + **kwargs: Any, +): ... +def _invoke_remote_python_udf( + dst: WorkerInfo, + pickledPythonUDF: str, + tensors: list[torch.Tensor], + rpcTimeoutSeconds: float, + isAsyncExecution: bool, +): ... +def _invoke_remote_torchscript( + dstWorkerName: WorkerInfo, + qualifiedNameStr: str, + rpcTimeoutSeconds: float, + isAsyncExecution: bool, + *args: Any, + **kwargs: Any, +): ... +def get_rpc_timeout() -> float: ... +def enable_gil_profiling(flag: bool): ... +def _set_rpc_timeout(rpcTimeoutSeconds: float): ... + +class RemoteProfilerManager: + @staticmethod + def set_current_profiling_key(key: str): ... + +def _enable_server_process_global_profiler(new_config: ProfilerConfig): ... +def _disable_server_process_global_profiler() -> list[list[list[ProfilerEvent]]]: ... +def _set_profiler_node_id(default_node_id: int): ... +def _enable_jit_rref_pickle(): ... +def _disable_jit_rref_pickle(): ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed_rpc_testing.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed_rpc_testing.pyi new file mode 100644 index 0000000000000000000000000000000000000000..9313281027dbd89ae65ebe65c95a1d0c38593b80 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_distributed_rpc_testing.pyi @@ -0,0 +1,32 @@ +import torch +from torch._C._distributed_c10d import Store +from torch._C._distributed_rpc import _TensorPipeRpcBackendOptionsBase, TensorPipeAgent + +# This module is defined in torch/csrc/distributed/rpc/testing/init.cpp + +class FaultyTensorPipeRpcBackendOptions(_TensorPipeRpcBackendOptionsBase): + def __init__( + self, + num_worker_threads: int, + rpc_timeout: float, + init_method: str, + messages_to_fail: list[str], + messages_to_delay: dict[str, float], + num_fail_sends: int, + ) -> None: ... + num_send_recv_threads: int + messages_to_fail: list[str] + messages_to_delay: dict[str, float] + num_fail_sends: int + +class FaultyTensorPipeAgent(TensorPipeAgent): + def __init__( + self, + store: Store, + name: str, + rank: int, + world_size: int, + options: FaultyTensorPipeRpcBackendOptions, + reverse_device_maps: dict[str, dict[torch.device, torch.device]], + devices: list[torch.device], + ) -> None: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_functionalization.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_functionalization.pyi new file mode 100644 index 0000000000000000000000000000000000000000..4e00df97e2717c1d3c63db3fc42e31507211eb99 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_functionalization.pyi @@ -0,0 +1,16 @@ +from torch import Tensor +from torch.types import _bool + +# Defined in torch/csrc/functionalization/Module.cpp + +class ViewMeta: + has_symbolic_inputs: _bool + +# Returns the list of ViewMeta instances of the given functional tensor. +# +# Although we do have python bindings for their types, we won't +# expose them here, since they should not be used by users. +def get_view_meta_sequence(tensor: Tensor) -> list[ViewMeta]: ... + +# Applies the ViewMeta sequence on top of the given base. +def apply_view_meta_sequence(base: Tensor, sequence: list[ViewMeta]) -> Tensor: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_functions.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_functions.pyi new file mode 100644 index 0000000000000000000000000000000000000000..5b0dee51a71093f1245887aa13c2405d3f2e5720 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_functions.pyi @@ -0,0 +1,19 @@ +from typing import AnyStr, overload + +from torch import Tensor + +class UndefinedGrad: + def __init__(self) -> None: ... + def __call__(self, *inputs: Tensor) -> list[Tensor]: ... + +class DelayedError: + def __init__(self, msg: AnyStr, num_inputs: int) -> None: ... + + # __call__ should really be a higher-kinded type: + # def __call__(self, arg: Tensor) -> Tensor: ... + # def __call__(self, *args: Tensor * num_inputs) -> Tuple[Tensor * num_inputs]: ... + + @overload + def __call__(self, i0: Tensor) -> Tensor: ... + @overload + def __call__(self, *args: Tensor) -> tuple[Tensor, ...]: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_functorch.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_functorch.pyi new file mode 100644 index 0000000000000000000000000000000000000000..a35befcad392d1607bc9b25345fa466829345623 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_functorch.pyi @@ -0,0 +1,89 @@ +# mypy: allow-untyped-defs +from enum import Enum + +from torch import Tensor + +# Defined in torch/csrc/functorch/init.cpp + +def set_inplace_requires_grad_allowed(allowed: bool) -> None: ... +def get_inplace_requires_grad_allowed() -> bool: ... +def _set_dynamic_layer_keys_included(included: bool) -> None: ... +def get_unwrapped(tensor: Tensor) -> Tensor: ... +def is_batchedtensor(tensor: Tensor) -> bool: ... +def is_functionaltensor(tensor: Tensor) -> bool: ... +def is_functorch_wrapped_tensor(tensor: Tensor) -> bool: ... +def is_gradtrackingtensor(tensor: Tensor) -> bool: ... +def is_legacy_batchedtensor(tensor: Tensor) -> bool: ... +def maybe_get_bdim(tensor: Tensor) -> int: ... +def maybe_get_level(tensor: Tensor) -> int: ... +def maybe_current_level() -> int | None: ... +def unwrap_if_dead(tensor: Tensor) -> Tensor: ... +def _unwrap_for_grad(tensor: Tensor, level: int) -> Tensor: ... +def _wrap_for_grad(tensor: Tensor, level: int) -> Tensor: ... +def _unwrap_batched(tensor: Tensor, level: int) -> tuple[Tensor, int | None]: ... +def current_level() -> int: ... +def count_jvp_interpreters() -> int: ... +def _add_batch_dim(tensor: Tensor, bdim: int, level: int) -> Tensor: ... +def _maybe_unsafe_set_level(tensor: Tensor, level: int) -> None: ... +def set_single_level_autograd_function_allowed(allowed: bool) -> None: ... +def get_single_level_autograd_function_allowed() -> bool: ... +def _unwrap_functional_tensor(tensor: Tensor, reapply_views: bool) -> Tensor: ... +def _wrap_functional_tensor(tensor: Tensor, level: int) -> Tensor: ... +def _vmap_increment_nesting(batch_size: int, randomness: str) -> int: ... +def _vmap_decrement_nesting() -> int: ... +def _grad_increment_nesting() -> int: ... +def _grad_decrement_nesting() -> int: ... +def _jvp_increment_nesting() -> int: ... +def _jvp_decrement_nesting() -> int: ... + +# Defined in aten/src/ATen/functorch/Interpreter.h +class TransformType(Enum): + Torch = ... + Vmap = ... + Grad = ... + Jvp = ... + Functionalize = ... + +class RandomnessType(Enum): + Error = ... + Same = ... + Different = ... + +class CInterpreter: + def key(self) -> TransformType: ... + def level(self) -> int: ... + def serialize(self) -> bytes: ... + @staticmethod + def deserialize(bytes) -> CInterpreter: ... + +class CGradInterpreterPtr: + def __init__(self, interpreter: CInterpreter) -> None: ... + def lift(self, Tensor) -> Tensor: ... + def prevGradMode(self) -> bool: ... + +class CJvpInterpreterPtr: + def __init__(self, interpreter: CInterpreter) -> None: ... + def lift(self, Tensor) -> Tensor: ... + def prevFwdGradMode(self) -> bool: ... + +class CFunctionalizeInterpreterPtr: + def __init__(self, interpreter: CInterpreter) -> None: ... + def key(self) -> TransformType: ... + def level(self) -> int: ... + def functionalizeAddBackViews(self) -> bool: ... + +class CVmapInterpreterPtr: + def __init__(self, interpreter: CInterpreter) -> None: ... + def key(self) -> TransformType: ... + def level(self) -> int: ... + def batchSize(self) -> int: ... + def randomness(self) -> RandomnessType: ... + +class DynamicLayer: ... + +def get_dynamic_layer_stack_depth() -> int: ... +def get_interpreter_stack() -> list[CInterpreter]: ... +def peek_interpreter_stack() -> CInterpreter: ... +def pop_dynamic_layer_stack() -> DynamicLayer: ... +def pop_dynamic_layer_stack_and_undo_to_depth(int) -> None: ... +def push_dynamic_layer_stack(dl: DynamicLayer) -> int: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_instruction_counter.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_instruction_counter.pyi new file mode 100644 index 0000000000000000000000000000000000000000..4e3c27567eb228b8763a6d2578db827b1ffbde41 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_instruction_counter.pyi @@ -0,0 +1,4 @@ +# Defined in torch/csrc/instruction_counter/Module.cpp + +def start() -> int: ... +def end(id: int) -> int: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_itt.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_itt.pyi new file mode 100644 index 0000000000000000000000000000000000000000..8a54437f527b994821133532f26598c951631b28 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_itt.pyi @@ -0,0 +1,5 @@ +# Defined in torch/csrc/itt.cpp +def is_available() -> None: ... +def rangePush(message: str) -> None: ... +def rangePop() -> None: ... +def mark(message: str) -> None: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_jit_tree_views.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_jit_tree_views.pyi new file mode 100644 index 0000000000000000000000000000000000000000..cf4cffc05a9c3414a60309da2ee8e1aec3dea2d0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_jit_tree_views.pyi @@ -0,0 +1,202 @@ +from typing import Any, Optional + +# Defined in torch/csrc/jit/python/python_tree_views.cpp + +class SourceRange: + def highlight(self) -> str: ... + @property + def start(self) -> int: ... + @property + def end(self) -> int: ... + +class SourceRangeFactory: + def __init__( + self, + text: str, + filename: Any, + file_lineno: int, + leading_whitespace_chars: int, + ) -> None: ... + def make_range(self, line: int, start_col: int, end_col: int) -> SourceRange: ... + def make_raw_range(self, start: int, end: int) -> SourceRange: ... + @property + def source(self) -> str: ... + +class TreeView: + def range(self) -> SourceRange: ... + def dump(self) -> None: ... + +class Ident(TreeView): + def __init__(self, *args: Any, **kwargs: Any) -> None: ... + @property + def name(self) -> str: ... + +class Param(TreeView): + def __init__(self, type: Optional[Any], name: Ident, kwarg_only: bool) -> None: ... + +class Attribute(TreeView): + def __init__(self, name: Ident, value: Any) -> None: ... + +# Literals +def TrueLiteral(range: SourceRange) -> Any: ... +def FalseLiteral(range: SourceRange) -> Any: ... +def NoneLiteral(range: SourceRange) -> Any: ... + +# Tree nodes +class Stmt(TreeView): + def __init__(self, thing: TreeView) -> None: ... + +class Expr(TreeView): ... + +class Def(TreeView): + def __init__(self, name: Ident, decl: Any, body: list[Stmt]) -> None: ... + def decl(self) -> Any: ... + def name(self) -> Ident: ... + +class Property(TreeView): + def __init__( + self, r: SourceRange, name: Ident, getter: Def, setter: Optional[Def] + ) -> None: ... + def name(self) -> Ident: ... + def getter_name(self) -> str: ... + def setter_name(self) -> Optional[Ident]: ... + +class ClassDef(TreeView): + def __init__( + self, name: Ident, body: list[Stmt], props: list[Property], assigns: list[Any] + ) -> None: ... + +class Decl(TreeView): + def __init__( + self, r: SourceRange, params: list[Param], return_type: Optional[Expr] + ) -> None: ... + +class Delete(Stmt): + def __init__(self, range: SourceRange, targets: list[Expr]) -> None: ... + +class WithItem(Expr): + def __init__( + self, range: SourceRange, target: Expr, var: Optional[Any] + ) -> None: ... + +class Assign(Stmt): + def __init__( + self, lhs: list[Expr], rhs: Expr, type: Optional[Expr] = None + ) -> None: ... + +class AugAssign(Stmt): + def __init__(self, lhs: Expr, kind_str: str, rhs: Expr) -> None: ... + +class Return(Stmt): + def __init__(self, range: SourceRange, value: Optional[Expr]) -> None: ... + +class Raise(Stmt): + def __init__(self, range: SourceRange, expr: Expr) -> None: ... + +class Assert(Stmt): + def __init__(self, range: SourceRange, test: Expr, msg: Optional[Expr]) -> None: ... + +class Pass(Stmt): + def __init__(self, range: SourceRange) -> None: ... + +class Break(Stmt): ... +class Continue(Stmt): ... + +class Dots(Expr, TreeView): + def __init__(self, range: SourceRange) -> None: ... + +class If(Stmt): + def __init__( + self, + range: SourceRange, + cond: Expr, + true_branch: list[Stmt], + false_branch: list[Stmt], + ) -> None: ... + +class While(Stmt): + def __init__(self, range: SourceRange, cond: Expr, body: list[Stmt]) -> None: ... + +class With(Stmt): + def __init__( + self, range: SourceRange, targets: list[WithItem], body: list[Stmt] + ) -> None: ... + +class For(Stmt): + def __init__( + self, + range: SourceRange, + targets: list[Expr], + itrs: list[Expr], + body: list[Stmt], + ) -> None: ... + +class ExprStmt(Stmt): + def __init__(self, expr: Expr) -> None: ... + +class Var(Expr): + def __init__(self, name: Ident) -> None: ... + @property + def name(self) -> str: ... + +class BinOp(Expr): + def __init__(self, kind: str, lhs: Expr, rhs: Expr) -> None: ... + +class UnaryOp(Expr): + def __init__(self, range: SourceRange, kind: str, expr: Expr) -> None: ... + +class Const(Expr): + def __init__(self, range: SourceRange, value: str) -> None: ... + +class StringLiteral(Expr): + def __init__(self, range: SourceRange, value: str) -> None: ... + +class Apply(Expr): + def __init__( + self, expr: Expr, args: list[Expr], kwargs: list[Attribute] + ) -> None: ... + +class Select(Expr): + def __init__(self, expr: Expr, field: Ident) -> None: ... + +class TernaryIf(Expr): + def __init__(self, cond: Expr, true_expr: Expr, false_expr: Expr) -> None: ... + +class ListComp(Expr): + def __init__( + self, range: SourceRange, elt: Expr, target: Expr, iter: Expr + ) -> None: ... + +class DictComp(Expr): + def __init__( + self, range: SourceRange, key: Expr, value: Expr, target: Expr, iter: Expr + ) -> None: ... + +class ListLiteral(Expr): + def __init__(self, range: SourceRange, args: list[Expr]) -> None: ... + +class TupleLiteral(Expr): + def __init__(self, range: SourceRange, args: list[Expr]) -> None: ... + +class DictLiteral(Expr): + def __init__( + self, range: SourceRange, keys: list[Expr], values: list[Expr] + ) -> None: ... + +class Subscript(Expr): + def __init__(self, base: Expr, subscript_exprs: list[Expr]) -> None: ... + +class SliceExpr(Expr): + def __init__( + self, + range: SourceRange, + lower: Optional[Expr], + upper: Optional[Expr], + step: Optional[Expr], + ) -> None: ... + +class Starred(Expr): + def __init__(self, range: SourceRange, expr: Expr) -> None: ... + +class EmptyTypeAnnotation(TreeView): + def __init__(self, range: SourceRange) -> None: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_lazy.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_lazy.pyi new file mode 100644 index 0000000000000000000000000000000000000000..c6b2b89fa3a9e89d889d9248097315d7c9d8635c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_lazy.pyi @@ -0,0 +1,26 @@ +from torch import Tensor + +# defined in torch/csrc/lazy/python/init.cpp +def _mark_step(device: str, devices: list[str], wait: bool) -> None: ... +def _wait_device_ops(devices: list[str]) -> None: ... +def _reset_metrics() -> None: ... +def _counter_names() -> list[str]: ... +def _counter_value(name: str) -> int: ... +def _metrics_report() -> str: ... +def _get_graph_hash(tensors: list[Tensor]) -> str: ... +def _sync_multi( + tensors: list[Tensor], + devices: list[str], + wait: bool = True, + sync_ltc_data: bool = True, +) -> None: ... +def _get_tensor_id(tensor: Tensor) -> int: ... +def _get_tensors_text(tensors: list[Tensor]) -> str: ... +def _get_tensors_dot(tensors: list[Tensor]) -> str: ... +def _get_tensors_backend(tensors: list[Tensor]) -> str: ... +def _get_force_fallback() -> str: ... +def _set_force_fallback(newval: str) -> None: ... +def _clear_ir_cache() -> None: ... +def _dump_ir_cache(filename: str) -> None: ... +def _set_reuse_ir(val: bool) -> None: ... +def _get_default_device_type() -> str: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_lazy_ts_backend.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_lazy_ts_backend.pyi new file mode 100644 index 0000000000000000000000000000000000000000..cd1bef2de069dfc93cd2b6243ab97d550d75f086 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_lazy_ts_backend.pyi @@ -0,0 +1,12 @@ +# mypy: allow-untyped-defs +# defined in torch/csrc/lazy/python/init.cpp + +from typing import Any + +from torch import Tensor + +def _init(): ... +def _get_tensors_ts_device_data_node( + tensors: list[Tensor], +) -> tuple[list[int], list[Any]]: ... +def _run_cached_graph(hash_str: str, graph_inputs: list[Any]) -> list[Tensor]: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_monitor.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_monitor.pyi new file mode 100644 index 0000000000000000000000000000000000000000..82f2a3e4427038dcb0616d7b305c7c5c85c97606 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_monitor.pyi @@ -0,0 +1,58 @@ +# Defined in torch/csrc/monitor/python_init.cpp + +import datetime +from collections.abc import Callable +from enum import Enum +from types import TracebackType + +class Aggregation(Enum): + VALUE = ... + MEAN = ... + COUNT = ... + SUM = ... + MAX = ... + MIN = ... + +class Stat: + name: str + count: int + def __init__( + self, + name: str, + aggregations: list[Aggregation], + window_size: int, + max_samples: int = -1, + ) -> None: ... + def add(self, v: float) -> None: ... + def get(self) -> dict[Aggregation, float]: ... + +class Event: + name: str + timestamp: datetime.datetime + data: dict[str, int | float | bool | str] + def __init__( + self, + name: str, + timestamp: datetime.datetime, + data: dict[str, int | float | bool | str], + ) -> None: ... + +def log_event(e: Event) -> None: ... + +class EventHandlerHandle: ... + +def register_event_handler(handler: Callable[[Event], None]) -> EventHandlerHandle: ... +def unregister_event_handler(handle: EventHandlerHandle) -> None: ... + +class _WaitCounterTracker: + def __enter__(self) -> None: ... + def __exit__( + self, + exc_type: type[BaseException] | None = None, + exc_value: BaseException | None = None, + traceback: TracebackType | None = None, + ) -> None: ... + +class _WaitCounter: + def __init__(self, key: str) -> None: ... + def guard(self) -> _WaitCounterTracker: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_nn.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_nn.pyi new file mode 100644 index 0000000000000000000000000000000000000000..b4e82e847108c9a613a87f26ca89eb79c63f6e9d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_nn.pyi @@ -0,0 +1,341 @@ +# @generated by tools/pyi/gen_pyi.py from torch/_C/_nn.pyi.in +# mypy: disable-error-code="type-arg" + +from collections.abc import Sequence +from typing import Literal, overload + +from torch import memory_format, Tensor +from torch.types import _bool, _device, _dtype, _int, _size + +# Defined in tools/autograd/templates/python_nn_functions.cpp + +def adaptive_avg_pool2d(input: Tensor, output_size: _int | _size) -> Tensor: ... +def adaptive_avg_pool3d(input: Tensor, output_size: _int | _size) -> Tensor: ... +def adaptive_max_pool2d( + input: Tensor, + output_size: _int | _size, +) -> tuple[Tensor, Tensor]: ... +def adaptive_max_pool3d( + input: Tensor, + output_size: _int | _size, +) -> tuple[Tensor, Tensor]: ... +def avg_pool2d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + ceil_mode: bool = False, + count_include_pad: bool = True, + divisor_override: int | None = None, +) -> Tensor: ... +def avg_pool3d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + ceil_mode: bool = False, + count_include_pad: bool = True, + divisor_override: int | None = None, +) -> Tensor: ... +def binary_cross_entropy( + input: Tensor, + target: Tensor, + weight: Tensor | None = None, + reduction: str = ..., +) -> Tensor: ... +def col2im( + input: Tensor, + output_size: _int | _size, + kernel_size: _int | _size, + dilation: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, +) -> Tensor: ... +def cross_entropy_loss( + input: Tensor, + target: Tensor, + weight: Tensor | None = None, + reduction: str = ..., + ignore_index: int = -100, + label_smoothing: float = 0.0, +) -> Tensor: ... +def elu( + input: Tensor, + alpha: float = 1.0, + scale: float = 1.0, + input_scale: float = 1.0, +) -> Tensor: ... +def elu_(input: Tensor, alpha: float = ...) -> Tensor: ... +def fractional_max_pool2d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size, + _random_samples: Tensor, +) -> tuple[Tensor, Tensor]: ... +def fractional_max_pool3d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size, + _random_samples: Tensor, +) -> tuple[Tensor, Tensor]: ... +def gelu(input: Tensor, approximate: str = ...) -> Tensor: ... +def glu(input: Tensor, dim: int = -1) -> Tensor: ... +def hardsigmoid(input: Tensor, *, out: Tensor | None = None) -> Tensor: ... +def hardsigmoid_(input: Tensor) -> Tensor: ... +def hardswish(input: Tensor) -> Tensor: ... +def hardswish_(input: Tensor) -> Tensor: ... +def hardtanh( + input: Tensor, + min_val: float = ..., + max_val: float = ..., + *, + out: Tensor | None = None, +) -> Tensor: ... +def hardtanh_( + input: Tensor, + min_val: float = ..., + max_val: float = ..., +) -> Tensor: ... +def huber_loss( + input: Tensor, + target: Tensor, + reduction: str = ..., + delta: float = 1.0, +) -> Tensor: ... +def im2col( + input: Tensor, + kernel_size: _int | _size, + dilation: _int | _size, + padding: _int | _size, + stride: _int | _size, +) -> Tensor: ... +def l1_loss(input: Tensor, target: Tensor, reduction: str = ...) -> Tensor: ... +def leaky_relu( + input: Tensor, + negative_slope: float = ..., + *, + out: Tensor | None = None, +) -> Tensor: ... +def leaky_relu_(input: Tensor, negative_slope: float = ...) -> Tensor: ... +def linear( + input: Tensor, + weight: Tensor, + bias: Tensor | None = None, +) -> Tensor: ... +def log_sigmoid(input: Tensor) -> Tensor: ... +def max_pool2d_with_indices( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, +) -> tuple[Tensor, Tensor]: ... +def max_pool3d_with_indices( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, +) -> tuple[Tensor, Tensor]: ... +def max_unpool2d( + input: Tensor, + indices: Tensor, + output_size: Sequence[int] | None, +) -> Tensor: ... +def max_unpool3d( + input: Tensor, + indices: Tensor, + output_size: Sequence[int] | None, + stride: _int | _size, + padding: _int | _size, +) -> Tensor: ... +def mish(input: Tensor) -> Tensor: ... +def mish_(input: Tensor) -> Tensor: ... +def mse_loss(input: Tensor, target: Tensor, reduction: str = ...) -> Tensor: ... +def multi_margin_loss( + input: Tensor, + target: Tensor, + p: float = 1.0, + margin: float = 1.0, + weight: Tensor | None = None, + reduction: str = ..., +) -> Tensor: ... +def multilabel_margin_loss( + input: Tensor, + target: Tensor, + reduction: str = ..., +) -> Tensor: ... +def nll_loss_nd( + input: Tensor, + target: Tensor, + weight: Tensor | None = None, + reduction: str = ..., + ignore_index: int = -100, +) -> Tensor: ... +def one_hot(tensor: Tensor, num_classes: int = ...) -> Tensor: ... +def pad( + input: Tensor, + pad: Sequence[int], + mode: str = ..., + value: float | None = None, +) -> Tensor: ... +def relu6(input: Tensor) -> Tensor: ... +def relu6_(input: Tensor) -> Tensor: ... +def scaled_dot_product_attention( + query: Tensor, + key: Tensor, + value: Tensor, + attn_mask: Tensor | None = None, + dropout_p: float = 0.0, + is_causal: bool = False, + scale: float | None = None, + enable_gqa: bool = False, +) -> Tensor: ... +def silu(input: Tensor) -> Tensor: ... +def silu_(input: Tensor) -> Tensor: ... +def smooth_l1_loss( + input: Tensor, + target: Tensor, + reduction: str = ..., + beta: float = 1.0, +) -> Tensor: ... +def soft_margin_loss( + input: Tensor, + target: Tensor, + reduction: str = ..., +) -> Tensor: ... +def softplus( + input: Tensor, + beta: float = ..., + threshold: float = ..., +) -> Tensor: ... +def softshrink(input: Tensor, lambd: float = ...) -> Tensor: ... + +# Defined in aten/src/ATen/native/mkldnn/Linear.cpp +def mkldnn_linear(input: Tensor, weight: Tensor, bias: Tensor | None) -> Tensor: ... + +# Defined at aten/src/ATen/native/mkldnn/MKLDNNConversions.cpp +def mkldnn_reorder_conv2d_weight( + self: Tensor, + padding: list, + stride: list, + dilatation: list, + groups: int, +) -> Tensor: ... +def mkldnn_reorder_conv3d_weight( + self: Tensor, + padding: list, + stride: list, + dilatation: list, + groups: int, +) -> Tensor: ... + +# Defined in aten/src/ATen/native/mkldnn/Prelu.cpp +def mkldnn_prelu(input: Tensor, weight: Tensor) -> Tensor: ... + +# Defined at tools/autograd/templates/python_nn_functions.cpp +@overload +def _parse_to( + device: _device, + dtype: _dtype, + non_blocking: _bool, + copy: _bool, + *, + memory_format: memory_format, +) -> tuple[_device, _dtype, _bool, memory_format]: ... +@overload +def _parse_to( + dtype: _dtype, + non_blocking: _bool, + copy: _bool, + *, + memory_format: memory_format, +) -> tuple[_device, _dtype, _bool, memory_format]: ... +@overload +def _parse_to( + tensor: Tensor, + non_blocking: _bool, + copy: _bool, + *, + memory_format: memory_format, +) -> tuple[_device, _dtype, _bool, memory_format]: ... + +# Defined in aten/src/ATen/native/PackedSequence.cpp +def pad_sequence( + sequences: list[Tensor] | tuple[Tensor, ...], + batch_first: bool = False, + padding_value: float = 0.0, + padding_side: Literal["left", "right"] = "right", +) -> Tensor: ... + +# Upsample functions used by torch.nn.functional.interpolate +def upsample_nearest1d( + input: Tensor, + output_size: Sequence[int] | None, + scale_factors: Sequence[float] | None, +) -> Tensor: ... +def upsample_nearest2d( + input: Tensor, + output_size: Sequence[int] | None, + scale_factors: Sequence[float] | None, +) -> Tensor: ... +def upsample_nearest3d( + input: Tensor, + output_size: Sequence[int] | None, + scale_factors: Sequence[float] | None, +) -> Tensor: ... +def _upsample_nearest_exact1d( + input: Tensor, + output_size: Sequence[int] | None, + scale_factors: Sequence[float] | None, +) -> Tensor: ... +def _upsample_nearest_exact2d( + input: Tensor, + output_size: Sequence[int] | None, + scale_factors: Sequence[float] | None, +) -> Tensor: ... +def _upsample_nearest_exact3d( + input: Tensor, + output_size: Sequence[int] | None, + scale_factors: Sequence[float] | None, +) -> Tensor: ... +def upsample_linear1d( + input: Tensor, + output_size: Sequence[int] | None, + align_corners: bool, + scale_factors: Sequence[float] | None, +) -> Tensor: ... +def _upsample_bilinear2d_aa( + input: Tensor, + output_size: Sequence[int] | None, + align_corners: bool, + scale_factors: Sequence[float] | None, +) -> Tensor: ... +def upsample_bilinear2d( + input: Tensor, + output_size: Sequence[int] | None, + align_corners: bool, + scale_factors: Sequence[float] | None, +) -> Tensor: ... +def upsample_trilinear3d( + input: Tensor, + output_size: Sequence[int] | None, + align_corners: bool, + scale_factors: Sequence[float] | None, +) -> Tensor: ... +def _upsample_bicubic2d_aa( + input: Tensor, + output_size: Sequence[int] | None, + align_corners: bool, + scale_factors: Sequence[float] | None, +) -> Tensor: ... +def upsample_bicubic2d( + input: Tensor, + output_size: Sequence[int] | None, + align_corners: bool, + scale_factors: Sequence[float] | None, +) -> Tensor: ... +def flatten_dense_tensors(tensors: list[Tensor]) -> Tensor: ... +def unflatten_dense_tensors(flat: Tensor, tensors: list[Tensor]) -> list[Tensor]: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_nvtx.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_nvtx.pyi new file mode 100644 index 0000000000000000000000000000000000000000..9b96874c36578ebaba065188d726455ff0b771be --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_nvtx.pyi @@ -0,0 +1,9 @@ +# mypy: allow-untyped-defs +# Defined in torch/csrc/cuda/shared/nvtx.cpp +def rangePushA(message: str) -> int: ... +def rangePop() -> int: ... +def rangeStartA(message: str) -> int: ... +def rangeEnd(int) -> None: ... +def markA(message: str) -> None: ... +def deviceRangeStart(message: str, stream: int) -> object: ... +def deviceRangeEnd(range_handle: object, stream: int) -> None: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_onnx.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_onnx.pyi new file mode 100644 index 0000000000000000000000000000000000000000..349e0b9ad12f0dd9306fde89a40718f26b158f0e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_onnx.pyi @@ -0,0 +1,39 @@ +# Defined in torch/csrc/onnx/init.cpp + +from enum import Enum + +PRODUCER_VERSION: str + +class TensorProtoDataType(Enum): + UNDEFINED = ... + FLOAT = ... + UINT8 = ... + INT8 = ... + UINT16 = ... + INT16 = ... + INT32 = ... + INT64 = ... + STRING = ... + BOOL = ... + FLOAT16 = ... + DOUBLE = ... + UINT32 = ... + UINT64 = ... + COMPLEX64 = ... + COMPLEX128 = ... + BFLOAT16 = ... + FLOAT8E5M2 = ... + FLOAT8E4M3FN = ... + FLOAT8E5M2FNUZ = ... + FLOAT8E4M3FNUZ = ... + +class OperatorExportTypes(Enum): + ONNX = ... + ONNX_ATEN = ... + ONNX_ATEN_FALLBACK = ... + ONNX_FALLTHROUGH = ... + +class TrainingMode(Enum): + EVAL = ... + PRESERVE = ... + TRAINING = ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_profiler.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_profiler.pyi new file mode 100644 index 0000000000000000000000000000000000000000..ae8121e4b71d268a7b9fafc93890856d65e7c7b5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_profiler.pyi @@ -0,0 +1,247 @@ +from enum import Enum +from typing import Literal, TypeAlias + +from torch._C import device, dtype, layout + +# defined in torch/csrc/profiler/python/init.cpp + +class RecordScope(Enum): + FUNCTION = ... + BACKWARD_FUNCTION = ... + TORCHSCRIPT_FUNCTION = ... + KERNEL_FUNCTION_DTYPE = ... + CUSTOM_CLASS = ... + BUILD_FEATURE = ... + LITE_INTERPRETER = ... + USER_SCOPE = ... + STATIC_RUNTIME_OP = ... + STATIC_RUNTIME_MODEL = ... + +class ProfilerState(Enum): + Disabled = ... + CPU = ... + CUDA = ... + NVTX = ... + ITT = ... + PRIVATEUSE1 = ... + KINETO = ... + KINETO_GPU_FALLBACK = ... + KINETO_PRIVATEUSE1_FALLBACK = ... + +class ActiveProfilerType(Enum): + NONE = ... + LEGACY = ... + KINETO = ... + NVTX = ... + ITT = ... + PRIVATEUSE1 = ... + +class ProfilerActivity(Enum): + CPU = ... + CUDA = ... + XPU = ... + MTIA = ... + HPU = ... + PrivateUse1 = ... + +class _EventType(Enum): + TorchOp = ... + Backend = ... + Allocation = ... + OutOfMemory = ... + PyCall = ... + PyCCall = ... + Kineto = ... + +class _ExperimentalConfig: + def __init__( + self, + profiler_metrics: list[str] = ..., + profiler_measure_per_kernel: bool = ..., + verbose: bool = ..., + performance_events: list[str] = ..., + enable_cuda_sync_events: bool = ..., + profile_all_threads: bool = ..., + ) -> None: ... + +class ProfilerConfig: + def __init__( + self, + state: ProfilerState, + report_input_shapes: bool, + profile_memory: bool, + with_stack: bool, + with_flops: bool, + with_modules: bool, + experimental_config: _ExperimentalConfig, + trace_id: str | None = None, + ) -> None: ... + +class _ProfilerEvent: + start_tid: int + start_time_ns: int + children: list[_ProfilerEvent] + + # TODO(robieta): remove in favor of `self.typed` + extra_fields: ( + _ExtraFields_TorchOp + | _ExtraFields_Backend + | _ExtraFields_Allocation + | _ExtraFields_OutOfMemory + | _ExtraFields_PyCall + | _ExtraFields_PyCCall + | _ExtraFields_Kineto + ) + + @property + def typed( + self, + ) -> ( + tuple[Literal[_EventType.TorchOp], _ExtraFields_TorchOp] + | tuple[Literal[_EventType.Backend], _ExtraFields_Backend] + | tuple[Literal[_EventType.Allocation], _ExtraFields_Allocation] + | tuple[Literal[_EventType.OutOfMemory], _ExtraFields_OutOfMemory] + | tuple[Literal[_EventType.PyCall], _ExtraFields_PyCall] + | tuple[Literal[_EventType.PyCCall], _ExtraFields_PyCCall] + | tuple[Literal[_EventType.Kineto], _ExtraFields_Kineto] + ): ... + @property + def name(self) -> str: ... + @property + def tag(self) -> _EventType: ... + @property + def id(self) -> int: ... + @property + def parent(self) -> _ProfilerEvent | None: ... + @property + def correlation_id(self) -> int: ... + @property + def end_time_ns(self) -> int: ... + @property + def duration_time_ns(self) -> int: ... + +class _TensorMetadata: + impl_ptr: int | None + storage_data_ptr: int | None + id: int | None + + @property + def allocation_id(self) -> int | None: ... + @property + def layout(self) -> layout: ... + @property + def device(self) -> device: ... + @property + def dtype(self) -> dtype: ... + @property + def sizes(self) -> list[int]: ... + @property + def strides(self) -> list[int]: ... + +Scalar: TypeAlias = int | float | bool | complex +Input: TypeAlias = _TensorMetadata | list[_TensorMetadata] | Scalar | None + +class _ExtraFields_TorchOp: + name: str + sequence_number: int + allow_tf32_cublas: bool + + @property + def inputs(self) -> list[Input]: ... + @property + def scope(self) -> RecordScope: ... + +class _ExtraFields_Backend: ... + +class _ExtraFields_Allocation: + ptr: int + id: int | None + alloc_size: int + total_allocated: int + total_reserved: int + + @property + def allocation_id(self) -> int | None: ... + @property + def device(self) -> device: ... + +class _ExtraFields_OutOfMemory: ... + +class _PyFrameState: + line_number: int + function_name: str + + @property + def file_name(self) -> str: ... + +class _NNModuleInfo: + @property + def self_ptr(self) -> int: ... + @property + def cls_ptr(self) -> int: ... + @property + def cls_name(self) -> str: ... + @property + def parameters( + self, + ) -> list[tuple[str, _TensorMetadata, _TensorMetadata | None]]: ... + +class _OptimizerInfo: + @property + def parameters( + self, + ) -> list[ + tuple[ + # Parameter + _TensorMetadata, + # + # Gradient (if present during optimizer.step()) + _TensorMetadata | None, + # + # Optimizer state for Parameter as (name, tensor) pairs + list[tuple[str, _TensorMetadata]], + ] + ]: ... + +class _ExtraFields_PyCCall: + @property + def caller(self) -> _PyFrameState: ... + +class _ExtraFields_PyCall: + @property + def callsite(self) -> _PyFrameState: ... + @property + def caller(self) -> _PyFrameState: ... + @property + def module(self) -> _NNModuleInfo | None: ... + @property + def optimizer(self) -> _OptimizerInfo | None: ... + +class _ExtraFields_Kineto: ... + +def _add_execution_trace_observer(output_file_path: str) -> bool: ... +def _remove_execution_trace_observer() -> None: ... +def _enable_execution_trace_observer() -> None: ... +def _disable_execution_trace_observer() -> None: ... +def _set_record_concrete_inputs_enabled_val(val: bool) -> None: ... +def _set_fwd_bwd_enabled_val(val: bool) -> None: ... +def _set_cuda_sync_enabled_val(val: bool) -> None: ... + +class CapturedTraceback: ... + +def gather_traceback(python: bool, script: bool, cpp: bool) -> CapturedTraceback: ... + +# The Dict has name, filename, line +def symbolize_tracebacks( + to_symbolize: list[CapturedTraceback], +) -> list[list[dict[str, str]]]: ... + +class _RecordFunctionFast: + def __init__( + self, + name: str, + input_values: list | tuple | None = None, + keyword_values: dict | None = None, + ) -> None: ... + def __enter__(self) -> None: ... + def __exit__(self, *exc_info: object) -> None: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_verbose.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_verbose.pyi new file mode 100644 index 0000000000000000000000000000000000000000..2388ce2bb8a5edd4c7640c374537e71607e5b72e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C/_verbose.pyi @@ -0,0 +1,3 @@ +# Defined in torch/csrc/utils/verbose.cpp +def mkl_set_verbose(enable: int) -> int: ... +def mkldnn_set_verbose(level: int) -> int: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C_flatbuffer/__init__.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C_flatbuffer/__init__.pyi new file mode 100644 index 0000000000000000000000000000000000000000..38750ed26aa26900591829fb7d51a3e3e1cdeeec --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_C_flatbuffer/__init__.pyi @@ -0,0 +1,11 @@ +# mypy: allow-untyped-defs +from torch._C import LiteScriptModule, ScriptModule + +def _load_mobile_module_from_file(filename: str): ... +def _load_mobile_module_from_bytes(bytes_: bytes): ... +def _load_jit_module_from_file(filename: str): ... +def _load_jit_module_from_bytes(bytes_: bytes): ... +def _save_mobile_module(m: LiteScriptModule, filename: str): ... +def _save_jit_module(m: ScriptModule, filename: str): ... +def _save_mobile_module_to_bytes(m: LiteScriptModule) -> bytes: ... +def _save_jit_module_to_bytes(m: ScriptModule) -> bytes: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_custom_op/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_custom_op/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_custom_op/autograd.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_custom_op/autograd.py new file mode 100644 index 0000000000000000000000000000000000000000..eed665a1a0d6267ad9c62dac94ef8f08b1608216 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_custom_op/autograd.py @@ -0,0 +1,307 @@ +# mypy: allow-untyped-defs +import functools +from collections import namedtuple + +import torch +import torch.utils._pytree as pytree + + +# NOTE [CustomOp autograd kernel indirection] +# We register `inner` as the autograd kernel for this custom_op. +# `inner` either calls the autograd formula registered by the user, +# or goes into an `autograd_not_implemented` kernel. +# +# The reason why this indirection exists is +# so that we can swap out the autograd kernel (the PyTorch dispatcher +# doesn't actually allow us to do this). By default, we want +# the `autograd_not_implemented` behavior, but then the user may come +# and register something that is actually a backward formula +def autograd_kernel_indirection(custom_op): + autograd_fallback = autograd_not_implemented(custom_op) + + def inner(*args, **kwargs): + if custom_op._has_impl("autograd"): + kernel = custom_op._get_impl("autograd").func + return kernel(*args, **kwargs) + # As explained in NOTE ["backward", "save_for_backward", and "autograd"], + # after the user gives us "backward" and "save_for_backward", we generate + # the "autograd" impl. If the user only provided one, then we tell + # the user they've done something wrong. + if custom_op._has_impl("save_for_backward") or custom_op._has_impl("backward"): + missing = ( + "save_for_backward" if custom_op._has_impl("backward") else "backward" + ) + found = "save_for_backward" if missing == "backward" else "backward" + loc = custom_op._get_impl(found).location + raise RuntimeError( + f"We found a '{found}' registration for {custom_op} at " + f"{loc} but were unable to find a '{missing}' registration. " + f"To use the CustomOp API to register a backward formula, " + f"please provide us both a backward function and a " + f"'save for backward' function via `impl_backward` and " + f"`impl_save_for_backward` respectively." + ) + return autograd_fallback(*args, **kwargs) + + return inner + + +# TODO(#101191): Use the actual C++ autograd not implemented fallback, +# or change the default autograd fallback to the autograd not implemented fallback. +def autograd_not_implemented(custom_op): + def kernel(*args, **kwargs): + if torch.is_grad_enabled() and pytree.tree_any( + lambda x: isinstance(x, torch.Tensor) and x.requires_grad, (args, kwargs) + ): + raise RuntimeError("Autograd has not been implemented for operator") + with torch._C._AutoDispatchBelowAutograd(): + return custom_op(*args, **kwargs) + + return kernel + + +def mark_non_differentiable(ctx, output, output_differentiability): + # Output types are restricted to be: + # - Tensor + # - Tensor[] + # - int, bool, Scalar, float + # See _check_can_register_backward + if output_differentiability is not None: + if not isinstance(output, tuple): + tuple_output = (output,) + else: + tuple_output = output # type: ignore[assignment] + assert len(output_differentiability) == len(tuple_output) + non_differentiable_tensors = [] + for idx, (differentiable, out) in enumerate( + zip(output_differentiability, tuple_output) + ): + if isinstance(out, torch.Tensor): + if not differentiable: + non_differentiable_tensors.append(out) + continue + if isinstance(out, list): + if not differentiable: + non_differentiable_tensors.extend(out) + continue + if differentiable: + raise RuntimeError( + f"With output_differentiability={output_differentiability}. " + f"At idx {idx}, we received an object of type {type(out)} that " + f"is not a Tensor, so it cannot have be marked as differentiable in " + f"output_differentiability." + ) + if non_differentiable_tensors: + ctx.mark_non_differentiable(*non_differentiable_tensors) + + +def construct_autograd_kernel( + schema, + output_differentiability, + custom_op, + op_overload, + save_for_backward_fn, + backward_fn, +): + def apply(*args): + flat_args, spec = pytree.tree_flatten(args) + out_spec = None + + def forward(ctx, *flat_args): + ctx.set_materialize_grads(True) + args = pytree.tree_unflatten(list(flat_args), spec) + with torch._C._AutoDispatchBelowAutograd(): + output = op_overload(*args) + + # We use the info about args to give better error messages in backward + args_info = namedtuple_args(schema, pytree.tree_map(type, args)) + + save_for_backward_fn_inputs = namedtuple_args(schema, args) + to_save = save_for_backward_fn(save_for_backward_fn_inputs, output) + + save_pytree_for_backward(ctx, (to_save, args_info)) + mark_non_differentiable(ctx, output, output_differentiability) + + nonlocal out_spec + flat_output, out_spec = pytree.tree_flatten(output) + return tuple(flat_output) + + def backward(ctx, *flat_grad_output): + assert out_spec is not None + grads = pytree.tree_unflatten(list(flat_grad_output), out_spec) + saved, args_info = unpack_saved(ctx) + # There is nothing on the ctx object for now, it is just there so + # that we can add additional things in the future. + inner_ctx = object() + if not isinstance(grads, tuple): + grads = (grads,) + grad_inputs_dict = backward_fn(inner_ctx, saved, *grads) + + # Massage the grad_inputs_dict to a form acceptable by + # autograd.Function. + validate_grad_inputs_dict(grad_inputs_dict, custom_op, args_info) + return grad_inputs_dict_to_flat_tuple(grad_inputs_dict, args_info) + + generated_cls = gen_autograd_function( + custom_op._opname + "_customop", forward, backward + ) + + flat_output = generated_cls.apply(*flat_args) + assert out_spec is not None + return pytree.tree_unflatten(list(flat_output), out_spec) + + return apply + + +def gen_autograd_function(name, forward, backward): + generated_cls = type( + name, + (torch.autograd.Function,), + { + "forward": staticmethod(forward), + "backward": staticmethod(backward), + }, + ) + return generated_cls + + +@functools.lru_cache +def namedtuple_args_cls(schema): + attribs = [arg.name for arg in schema.arguments.flat_all] + name = str(schema.name) + "_args" + # mypy doesn't support dynamic namedtuple name + tuple_cls = namedtuple(name, attribs) # type: ignore[misc] + return tuple_cls + + +def namedtuple_args(schema, args): + assert isinstance(args, tuple) + tuple_cls = namedtuple_args_cls(schema) + return tuple_cls(*args) + + +def validate_grad_inputs_dict(grad_inputs_dict, forward_op, args_info): + def error(what): + backward = forward_op._get_impl("backward") + raise RuntimeError( + f"In the backward function defined for {forward_op} at " + f"{backward.location} using the CustomOp API, {what}" + ) + + if not isinstance(grad_inputs_dict, dict): + error( + f"expected the output of the backward function to be a dict but " + f"got {type(grad_inputs_dict)}" + ) + + expected_keys = { + arg.name + for arg in forward_op._schema.arguments.flat_all + if arg.type.is_tensor_like() + } + actual_keys = grad_inputs_dict.keys() + if expected_keys != actual_keys: + error( + f"expected the returned grad_input dict to have keys " + f"{expected_keys} but got {actual_keys}. The backward " + f"function must return a gradient (can be None) for each arg " + f"to the CustomOp that may be a Tensor or Sequence[Tensor]. " + f"Args declared to be non-Tensor-like types should not appear " + f"in the grad_input dict" + ) + + for name, grad in grad_inputs_dict.items(): + arg_info = getattr(args_info, name) + + if isinstance(arg_info, list): + if not isinstance(grad, (tuple, list)): + error( + f"for input '{name}' expected the grad_input dict to " + f"hold a list of gradients but got object of type " + f"{type(grad)}." + ) + if len(grad) != len(arg_info): + error( + f"for input '{name}' expected the grad_input dict to " + f"hold a list of {len(arg_info)} gradients but got " + f"{len(grad)}" + ) + for idx, (g, info) in enumerate(zip(grad, arg_info)): + if g is None: + continue + if not isinstance(g, torch.Tensor): + error( + f"for input '{name}' expected the grad_input dict to " + f"hold a list of None or Tensor gradients but got " + f"object of {type(g)} at index {idx}" + ) + if not issubclass(info, torch.Tensor): + error( + f"for input '{name}', got a Tensor as the gradient " + f"for the {idx}-th value but expected None because " + f"the {idx}-th value was not a Tensor (it was " + f"type {arg_info}" + ) + continue + + if grad is None: + continue + if not isinstance(grad, torch.Tensor): + error( + f"got object of type {type(grad)} as the gradient for input " + f"'{name}', " + f"but expected the gradient to be either None or a Tensor" + ) + if not issubclass(arg_info, torch.Tensor): + error( + f"got a Tensor as the gradient for input '{name}' but " + f"expected None as the gradient because input '{name}' " + f"was not a Tensor (it was type {arg_info})." + ) + + +def grad_inputs_dict_to_flat_tuple(grad_inputs_dict, args_info): + result = [] + for name, arg_info in args_info._asdict().items(): + if name not in grad_inputs_dict: + result.append(pytree.tree_map(lambda x: None, arg_info)) + continue + result.append(grad_inputs_dict[name]) + return tuple(pytree.tree_leaves(result)) + + +# Saves "stuff" (a pytree) onto the ctx object. Use unpack_saved to unpack it. +# autograd.Function prefers that users use ctx.save_for_backward to +# save Tensors (to avoid reference cycles) and for non-Tensors to go onto the +# ctx object. +def save_pytree_for_backward(ctx, stuff): + flat_stuff, spec = pytree.tree_flatten(stuff) + num_elts = len(flat_stuff) + tensor_idxs = [ + idx for idx, thing in enumerate(flat_stuff) if isinstance(thing, torch.Tensor) + ] + non_tensor_idxs = [ + idx + for idx, thing in enumerate(flat_stuff) + if not isinstance(thing, torch.Tensor) + ] + tensors = [thing for thing in flat_stuff if isinstance(thing, torch.Tensor)] + non_tensors = [thing for thing in flat_stuff if not isinstance(thing, torch.Tensor)] + + ctx.spec = spec + ctx.num_elts = num_elts + ctx.save_for_backward(*tensors) + ctx.tensor_idxs = tensor_idxs + ctx.saved_non_tensors = non_tensors + ctx.non_tensor_idxs = non_tensor_idxs + + +# Inverse operation to save_pytree_for_backward +def unpack_saved(ctx): + flat_stuff = [None] * ctx.num_elts + for tensor, idx in zip(ctx.saved_tensors, ctx.tensor_idxs): + flat_stuff[idx] = tensor + for non_tensor, idx in zip(ctx.saved_non_tensors, ctx.non_tensor_idxs): + flat_stuff[idx] = non_tensor + stuff = pytree.tree_unflatten(flat_stuff, ctx.spec) + return stuff diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_custom_op/impl.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_custom_op/impl.py new file mode 100644 index 0000000000000000000000000000000000000000..1398f808da21f21b004fb87a8fd813ebaa8e69b7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_custom_op/impl.py @@ -0,0 +1,716 @@ +# mypy: allow-untyped-defs +import dataclasses +import functools +import inspect +import sys +import typing +import warnings +import weakref + +import torch +import torch._C as _C +import torch._library.infer_schema +import torch.library as library +from torch._library.infer_schema import infer_schema +from torch.library import get_ctx +from torchgen.model import ( + BaseTy, + BaseType, + FunctionSchema, + ListType, + OperatorName, + SchemaKind, +) + +from .autograd import autograd_kernel_indirection, construct_autograd_kernel + + +""" +torch._custom_op is deprecated. We shipped a production-ready version of it into torch.library. +Please use those APIs instead. +""" + +__all__ = ["custom_op", "CustomOp", "get_ctx"] + + +SUPPORTED_DEVICE_TYPE_TO_KEY = { + "cpu": "CPU", + "cuda": "CUDA", +} + +# We will not let users register CustomOps with anything that could look like +# PyTorch internals to avoid confusion. +RESERVED_NS = { + "prim", + "prims", + "aten", + "at", + "torch", + "pytorch", +} + + +def warn_deprecated(): + warnings.warn( + "torch._custom_op is deprecated and will be removed in PyTorch 2.6, please " + "use the equivalent torch.library API instead.", + DeprecationWarning, + stacklevel=2, + ) + + +def custom_op( + qualname: str, manual_schema: typing.Optional[str] = None +) -> typing.Callable: + r""" + This API is deprecated, please use torch.library.custom_op instead + """ + warn_deprecated() + + def inner(func): + if not inspect.isfunction(func): + raise ValueError( + f"custom_op(...)(func): Expected `func` to be a Python " + f"function, got: {type(func)}" + ) + + ns, name = parse_qualname(qualname) + validate_namespace(ns) + if func.__name__ != name: + raise ValueError( + f"custom_op(qualname='{qualname}', ...)(func): expected `func` " + f"to have name '{name}' but got '{func.__name__}'. " + f"Please either change the name of `func` or the qualname that " + f"is passed to `custom_op`" + ) + + schema = ( + infer_schema(func, mutates_args=()) + if manual_schema is None + else manual_schema + ) + schema_str = f"{name}{schema}" + function_schema = FunctionSchema.parse(schema_str) + validate_schema(function_schema) + if manual_schema is not None: + validate_function_matches_schema(function_schema, func) + + lib = library.Library(ns, "FRAGMENT") + lib.define(schema_str) + ophandle = find_ophandle_or_throw(ns, function_schema.name) + result = CustomOp( + lib, ns, function_schema, name, ophandle, _private_access=True + ) + + result.__name__ = func.__name__ # pyrefly: ignore [bad-assignment] + result.__module__ = func.__module__ + result.__doc__ = func.__doc__ + + library.impl(lib, result._opname, "Autograd")( + autograd_kernel_indirection(weakref.proxy(result)) + ) + + torch._C._dispatch_set_report_error_callback( + ophandle, functools.partial(report_error_callback, weakref.proxy(result)) + ) + + return result + + return inner + + +# Global dictionary holding references to all CustomOp objects +# Yes, it keeps all CustomOps alive (see NOTE [CustomOp lifetime]) +# Used to query the CustomOp associated with a specific C++ dispatcher operator. +# An example usage is FakeTensor: FakeTensor checks if a specific operator +# has an implementation registered via the CustomOp API. +# Indexed by qualname (e.g. aten::foo) +global_registry: dict[str, "CustomOp"] = {} + + +class CustomOp: + r""" + This API is deprecated, please use torch.library.custom_op instead + """ + + def __init__( + self, lib, cpp_ns, schema, operator_name, ophandle, *, _private_access=False + ): + super().__init__() + warn_deprecated() + if not _private_access: + raise RuntimeError( + "The CustomOp constructor is private and we do not guarantee " + "BC for it. Please use custom_op(...) to create a CustomOp object" + ) + name = f"{cpp_ns}::{operator_name}" + self._schema = schema + self._cpp_ns = cpp_ns + self._lib: library.Library = lib + self._ophandle: _C._DispatchOperatorHandle = ophandle + # Has the name of the op, e.g. "foo". We cache here for convenience. + self._opname: str = operator_name + # this is _opname but with namespace. e.g. "custom::foo" + self._qualname: str = name + self.__name__ = None # mypy requires this + # NB: Some of these impls are registered as kernels to DispatchKeys. + # Modifying the _impls dict directly won't do anything in that case. + self._impls: dict[str, typing.Optional[FuncAndLocation]] = {} + # See NOTE [CustomOp autograd kernel indirection] + self._registered_autograd_kernel_indirection = False + + global_registry[self._qualname] = self + + def _register_autograd_kernel_indirection(self): + assert not self._registered_autograd_kernel_indirection + self._lib.impl( + self._opname, autograd_kernel_indirection(weakref.proxy(self)), "Autograd" + ) + self._registered_autograd_kernel_indirection = True + + # Records the impl and the source location in self._impls + # Note that this doesn't cause torch.library to use the impl, that + # needs to be done in a separate self._lib.impl call. + def _register_impl(self, kind, func, stacklevel=2): + if self._has_impl(kind): + func_and_location = self._impls[kind] + assert func_and_location is not None # Pacify mypy + location = func_and_location.location + raise RuntimeError( + f"Attempting to register a {kind} impl for operator {self._qualname} " + f"that already has a {kind} impl registered from Python at " + f"{location}. This is not supported." + ) + frame = inspect.getframeinfo(sys._getframe(stacklevel)) + location = f"{frame.filename}:{frame.lineno}" + self._impls[kind] = FuncAndLocation(func, location) + + def _get_impl(self, kind): + return self._impls[kind] + + def _has_impl(self, kind): + return kind in self._impls + + def _destroy(self): + # NOTE: [CustomOp lifetime] + # A CustomOp, once created, lives forever. The mechanism is that the + # global registry holds a reference to it. However, to make testing + # easier, we want to be able to destroy CustomOp objects. + # CustomOp._destroy does the job, though it leaves the CustomOp + # in a garbage state. + del self._lib + + opnamespace = getattr(torch.ops, self._cpp_ns) + if hasattr(opnamespace, self._opname): + delattr(opnamespace, self._opname) + + del global_registry[self._qualname] + + def __repr__(self): + return f'' + + def __call__(self, *args, **kwargs): + # Bypass torch.ops.* and directly do OperatorHandle::callBoxed. + # Using torch.ops.* is a bit of a pain (it can be slow and it has lifetime + # issues from caching operators that make testing CustomOp difficult). + result = _C._dispatch_call_boxed(self._ophandle, *args, **kwargs) + return result + + def impl( + self, + device_types: typing.Union[str, typing.Iterable[str]], + _stacklevel=2, + ) -> typing.Callable: + r""" + This API is deprecated, please use torch.library.custom_op instead + """ + if isinstance(device_types, str): + device_types = [device_types] + for device_type in device_types: + validate_device_type(device_type) + + def inner(f): + for device_type in set(device_types): + self._check_doesnt_have_library_impl(device_type) + self._register_impl(device_type, f, stacklevel=_stacklevel) + dispatch_key = SUPPORTED_DEVICE_TYPE_TO_KEY[device_type] + library.impl(self._lib, self._opname, dispatch_key)(f) + return f + + return inner + + def _check_doesnt_have_library_impl(self, device_type): + if self._has_impl(device_type): + return + key = SUPPORTED_DEVICE_TYPE_TO_KEY[device_type] + if _C._dispatch_has_computed_kernel_for_dispatch_key(self._qualname, key): + raise RuntimeError( + f"impl(..., device_types={device_type}): the operator {self._qualname} " + f"already has an implementation for this device type via a " + f"pre-existing torch.library or TORCH_LIBRARY registration." + ) + + def impl_factory(self) -> typing.Callable: + r"""Register an implementation for a factory function.""" + + def inner(f): + self._register_impl("factory", f) + library.impl(self._lib, self._opname, "BackendSelect")(f) + return f + + return inner + + def impl_abstract(self, _stacklevel=2) -> typing.Callable: + r""" + This API is deprecated, please use torch.library.custom_op instead + """ + + def inner(f): + self._check_doesnt_have_library_meta_impl() + self._register_impl("abstract", f, stacklevel=_stacklevel) + location = self._get_impl("abstract").location + + qualname = self._qualname + + # Handle DispatchKey.Meta registration + @functools.wraps(f) + def f_with_ctx(*args, **kwargs): + def error_on_ctx(): + raise RuntimeError( + f"Attempted to call get_ctx() for the meta implementation " + f"for {qualname}." + f"You have presumably called get_ctx() because the operator " + f"has a data-dependent output shape; if so, there is no " + f"such meta implementation and this error is the correct " + f"behavior. Otherwise, please remove the call to get_ctx() " + f"in the implementation registered with impl_abstract " + f"at {location}" + ) + + with torch._library.fake_impl.set_ctx_getter(error_on_ctx): + return f(*args, **kwargs) + + self._lib.impl(self._opname, f_with_ctx, "Meta") + return f + + return inner + + def _check_can_register_backward(self): + def error(detail): + raise RuntimeError( + f"Cannot use torch._custom_ops APIs to register backward " + f"formula for {detail}. Got operator " + f"{self._qualname} with schema: {schema}" + ) + + schema = self._schema + if schema.kind() != SchemaKind.functional: + error("non-functional operator") + + rets = schema.returns + if not schema.returns: + error("operator with no returns") + + assert len(rets) > 0 + is_non_mutating_view = any( + r.annotation is not None and not r.annotation.is_write for r in rets + ) + if is_non_mutating_view: + error("operator that returns views") + + # We make assumptions about the schema's return types. + allowed_return_types = { + BaseType(BaseTy.int): "int", + BaseType(BaseTy.SymInt): "SymInt", + BaseType(BaseTy.bool): "bool", + BaseType(BaseTy.float): "float", + BaseType(BaseTy.Tensor): "Tensor", + ListType(BaseType(BaseTy.Tensor), None): "List[Tensor]", + } + for ret in schema.returns: + if ret.type in allowed_return_types: + continue + error( + f"operator with return not in {list(allowed_return_types.values())} (got {ret.type})" + ) + + def _check_doesnt_have_library_autograd_impl(self): + if self._registered_autograd_kernel_indirection: + return + + if _C._dispatch_has_kernel_for_dispatch_key( + self._qualname, "CompositeImplicitAutograd" + ): + raise RuntimeError( + f"impl_backward/impl_save_for_backward: the operator {self._qualname} " + f"already has an implementation for this device type via a " + f"pre-existing registration to DispatchKey::CompositeImplicitAutograd." + f"CompositeImplicitAutograd operators do not need an autograd formula; " + f"instead, the operator will decompose into its constituents and those " + f"can have autograd formulas defined on them." + ) + + # We can improve this by adding "all Autograd keys", but + # realistically people will just be using this API for CPU/CUDA for now. + for key in ["Autograd", "AutogradCPU", "AutogradCUDA"]: + if _C._dispatch_has_kernel_for_dispatch_key(self._qualname, key): + raise RuntimeError( + f"impl_backward/impl_save_for_backward: " + f"the operator {self._qualname} already has an Autograd kernel " + f"registered to DispatchKey::{key} vi a pre-existing " + f"torch.library or TORCH_LIBRARY registration. Please either " + f"remove those registrations or don't use the torch._custom_ops APIs" + ) + + def _check_doesnt_have_library_meta_impl(self): + if self._has_impl("abstract"): + return + + # If the user's operator is CompositeExplicitAutograd, + # allow them to impl_abstract. This is being pragmatic + # (existing custom ops may have CompositeExplicitAutograd + # registration that don't work with Meta kernels, so this + # gives them an escape hatch). + if _C._dispatch_has_kernel_for_dispatch_key( + self._qualname, "CompositeExplicitAutograd" + ) and not _C._dispatch_has_kernel_for_dispatch_key(self._qualname, "Meta"): + return + + # Otherwise, if the user's already has a Meta kernel or their + # op is CompositeImplicitAutograd or some other alias dispatch key, + # raise. + + # Special case for CompositeImplicitAutograd + if _C._dispatch_has_kernel_for_dispatch_key( + self._qualname, "CompositeImplicitAutograd" + ): + raise RuntimeError( + f"impl_abstract(...): the operator {self._qualname} " + f"already has an implementation for this device type via a " + f"pre-existing registration to DispatchKey::CompositeImplicitAutograd." + f"CompositeImplicitAutograd operators do not need an abstract impl; " + f"instead, the operator will decompose into its constituents and those " + f"can have abstract impls defined on them." + ) + + if _C._dispatch_has_kernel_for_dispatch_key(self._qualname, "Meta"): + raise RuntimeError( + f"impl_abstract(...): the operator {self._qualname} " + f"already has an DispatchKey::Meta implementation via a " + f"pre-existing torch.library or TORCH_LIBRARY registration. " + f"Please either remove that registration or don't call impl_abstract." + ) + + # NOTE ["backward", "save_for_backward", and "autograd"] + # As a part of the explicit autograd API, a user must provide us + # a "save_for_backward" function and a "backward" function. + # When both of these have been provided, then we automatically + # construct the "autograd" kernel. + def _register_autograd_kernel(self): + assert self._has_impl("backward") + assert self._has_impl("save_for_backward") + kernel = construct_autograd_kernel( + self._schema, + self._output_differentiability, + self, + get_op(self._qualname), + self._get_impl("save_for_backward").func, + self._get_impl("backward").func, + ) + self._register_impl("autograd", kernel) + + def impl_save_for_backward(self, _stacklevel=2): + r"""Register a function that tells us what to save for backward. + + Please see impl_backward for more details. + """ + + def inner(f): + self._check_can_register_backward() + self._check_doesnt_have_library_autograd_impl() + if not self._registered_autograd_kernel_indirection: + self._register_autograd_kernel_indirection() + self._register_impl("save_for_backward", f, stacklevel=_stacklevel) + if self._has_impl("backward"): + self._register_autograd_kernel() + + return inner + + def impl_backward(self, output_differentiability=None, _stacklevel=2): + r""" + This API is deprecated, please use torch.library.custom_op instead + """ + if output_differentiability is not None: + + def yell(): + raise RuntimeError( + f"impl_backward(output_differentiability): expected " + f"output_differentiability to be a list of bools with " + f"length equal to the number of outputs of this CustomOp " + f"got: {output_differentiability}" + ) + + if not isinstance(output_differentiability, list): + yell() + for diff in output_differentiability: + if not isinstance(diff, bool): + yell() + if len(self._schema.returns) != len(output_differentiability): + yell() + + def inner(f): + self._check_can_register_backward() + self._check_doesnt_have_library_autograd_impl() + if not self._registered_autograd_kernel_indirection: + self._register_autograd_kernel_indirection() + self._register_impl("backward", f, stacklevel=_stacklevel) + self._output_differentiability = output_differentiability + if self._has_impl("save_for_backward"): + self._register_autograd_kernel() + + return inner + + +@dataclasses.dataclass +class FuncAndLocation: + func: typing.Callable + location: str + + +def find_ophandle_or_throw(cpp_ns: str, operator_name: OperatorName): + overload_name = ( + "" if operator_name.overload_name is None else operator_name.overload_name + ) + return _C._dispatch_find_schema_or_throw( + f"{cpp_ns}::{str(operator_name.name)}", overload_name + ) + + +def validate_namespace(ns: str) -> None: + if "." in ns: + raise ValueError( + f'custom_op(..., ns="{ns}"): expected ns to not contain any . (and be a ' + f"valid variable name)" + ) + if ns in RESERVED_NS: + raise ValueError( + f"custom_op(..., ns='{ns}'): '{ns}' is a reserved namespace, " + f"please choose something else. " + ) + + +def validate_schema(schema: FunctionSchema) -> None: + if not torch._library.utils.is_functional_schema(schema): + raise ValueError( + f"custom_op only supports functional operators " + f"(ops that do not mutate any inputs, do not return " + f"views of the inputs, and has at least one return). " + f"Got the following non-functional schema: {schema}" + ) + + # For simplicity: don't allow self arguments + if schema.arguments.self_arg is not None: + raise ValueError( + f"custom_op does not support arguments named 'self'. Please " + f"rename your argument. Got: {schema}" + ) + + +def parse_qualname(qualname: str) -> tuple[str, str]: + names = qualname.split("::", 1) + if len(names) != 2: + raise ValueError( + f"Expected there to be a namespace in {qualname}, i.e. The " + f"operator name should look something like ns::foo" + ) + if "." in names[1]: + raise ValueError( + f"The torch.custom_ops APIs do not handle overloads, " + f"i.e. operator names with '.' in them. " + f"Please name your operator something like ns::foo. " + f"Got: {qualname}" + ) + return names[0], names[1] + + +def validate_device_type(device_type: str) -> None: + if device_type not in SUPPORTED_DEVICE_TYPE_TO_KEY: + raise ValueError( + f"CustomOp.impl(device_types=[{device_type}, ...]): we only support device_type " + f"in {SUPPORTED_DEVICE_TYPE_TO_KEY.keys()}." + ) + + +def supported_param(param: inspect.Parameter) -> bool: + return param.kind in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ) + + +def validate_function_matches_schema( + schema: FunctionSchema, func: typing.Callable +) -> None: + sig = inspect.signature(func) + + if not all(supported_param(p) for _, p in sig.parameters.items()): + raise ValueError( + f"custom_op(..., manual_schema)(func): positional-only args, " + f"varargs, and kwargs are not supported. Please rewrite `func` " + f"to not have them. Got `func` with signature: {sig}" + ) + + if ( + any( + p.annotation is not inspect.Parameter.empty + for _, p in sig.parameters.items() + ) + or sig.return_annotation is not inspect.Signature.empty + ): + raise ValueError( + f"custom_op(..., manual_schema)(func): When passing in a manual " + f"schema, we expect `func` to have no type annotations to avoid " + f"ambiguity. Got `func` with signature: {sig}" + ) + + positional = [ + (name, param) + for name, param in sig.parameters.items() + if param.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD + ] + kwargonly = [ + (name, param) + for name, param in sig.parameters.items() + if param.kind == inspect.Parameter.KEYWORD_ONLY + ] + + def error(): + raise ValueError( + f"custom_op(..., manual_schema)(func): When passing in a manual " + f"schema, we expect `func`'s signature to match `manual_schema` " + f"(aside from type annotations). " + f"func's signature: {sig}, manual_schema: {schema}" + ) + + def error_default_args(): + raise ValueError( + f"custom_op(..., manual_schema)(func): " + f"neither func nor manual_schema should have default " + f"arguments. Got " + f"func's signature: {sig}, manual_schema: {schema}" + ) + + def compare(sig_args, schema_args): + if len(sig_args) != len(schema_args): + error() + for (name, param), arg in zip(sig_args, schema_args): + if name != arg.name: + error() + if param.default is not inspect.Parameter.empty or arg.default is not None: + error_default_args() + + compare(positional, schema.arguments.flat_positional) + compare(kwargonly, schema.arguments.flat_kwarg_only) + + +def report_error_callback(custom_op: typing.Any, key: str) -> None: + if key == "Undefined": + raise NotImplementedError( + f"{custom_op}: There were no Tensor inputs to this operator " + f"(e.g. you passed an empty list of Tensors). If your operator is a " + f"factory function (that is, it takes no Tensors and constructs " + f"a new one), then please use CustomOp.impl_factory to register " + f"an implementation for it" + ) + if key == "Meta": + raise NotImplementedError( + f"{custom_op}: when running with device='Meta' tensors: there is no " + f"abstract impl registered for this CustomOp. Please register one via " + f"CustomOp.impl_abstract to get this CustomOp to work with Meta tensors" + ) + if key in ("CPU", "CUDA"): + device = key.lower() + raise NotImplementedError( + f"{custom_op}: when running with device='{device}' tensors: there is no " + f"{device} impl registered for this CustomOp. Please register one via " + f"CustomOp.impl(device_type='{device}')" + ) + raise NotImplementedError( + f"{custom_op}: No implementation for dispatch key {key}. It is likely " + f"that we have not added this functionality yet, please either open an " + f"issue or if you're feeling adventurous, use the low-level " + f"torch.library API" + ) + + +def custom_op_from_existing(op): + ns = op.namespace + lib = torch.library.Library(ns, "FRAGMENT") + name = op.name().split("::")[-1] + schema_str = str(op._schema) + # CustomOp expects the schema string without the namespace + schema_str = schema_str.rsplit("::", maxsplit=1)[-1] + schema = FunctionSchema.parse(schema_str) + return CustomOp(lib, ns, schema, name, op, _private_access=True) + + +def get_op(qualname): + def error_not_found(): + raise ValueError( + f"Could not find the operator {qualname}. Please make sure you have " + f"already registered the operator and (if registered from C++) " + f"loaded it via torch.ops.load_library." + ) + + ns, name = parse_qualname(qualname) + if not hasattr(torch.ops, ns): + error_not_found() + opnamespace = getattr(torch.ops, ns) + if not hasattr(opnamespace, name): + error_not_found() + packet = getattr(opnamespace, name) + if not hasattr(packet, "default"): + error_not_found() + return packet.default + + +def _find_custom_op(qualname, also_check_torch_library=False): + if qualname in global_registry: + return global_registry[qualname] + if not also_check_torch_library: + raise RuntimeError( + f'Could not find custom op "{qualname}". Did you register it via ' + f"the torch._custom_ops API?" + ) + overload = get_op(qualname) + result = custom_op_from_existing(overload) + return result + + +def get_abstract_impl(qualname): + if qualname not in torch._custom_op.impl.global_registry: + return None + custom_op = torch._custom_op.impl.global_registry[qualname] + if custom_op is None: + return None + if not custom_op._has_impl("abstract"): + return None + return custom_op._get_impl("abstract").func + + +def _custom_op_with_schema(qualname, schema, needs_fixed_stride_order=True): + ns, name = qualname.split("::") + schema_str = f"{name}{schema}" + function_schema = FunctionSchema.parse(schema_str) + validate_schema(function_schema) + tags = [torch._C.Tag.needs_fixed_stride_order] if needs_fixed_stride_order else [] + lib = library.Library(ns, "FRAGMENT") + lib.define(schema_str, tags=tags) + ophandle = find_ophandle_or_throw(ns, function_schema.name) + result = CustomOp(lib, ns, function_schema, name, ophandle, _private_access=True) + result._register_autograd_kernel_indirection() + + torch._C._dispatch_set_report_error_callback( + ophandle, functools.partial(report_error_callback, weakref.proxy(result)) + ) + return get_op(qualname) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..4de5f712e16d64c0293dcd6ce38c41e04130ad36 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/__init__.py @@ -0,0 +1,216 @@ +""" +TorchDynamo is a Python-level JIT compiler designed to make unmodified PyTorch programs faster. +TorchDynamo hooks into the frame evaluation API in CPython (PEP 523) to dynamically modify Python +bytecode right before it is executed. It rewrites Python bytecode in order to extract sequences of +PyTorch operations into an FX Graph which is then just-in-time compiled with a customizable backend. +It creates this FX Graph through bytecode analysis and is designed to mix Python execution with +compiled backends to get the best of both worlds: usability and performance. This allows it to +seamlessly optimize PyTorch programs, including those using modern Python features. +""" + +import torch + +from . import ( + aot_compile, + config, + convert_frame, + eval_frame, + functional_export, + resume_execution, +) +from .backends.registry import list_backends, lookup_backend, register_backend +from .callback import callback_handler, on_compile_end, on_compile_start +from .code_context import code_context +from .convert_frame import replay +from .decorators import ( + allow_in_graph, + assume_constant_result, + disable, + disable_nested_graph_breaks, + disallow_in_graph, + dont_skip_tracing, + error_on_graph_break, + forbid_in_graph, + graph_break, + is_dynamo_disable_recursive, + mark_dynamic, + mark_static, + mark_static_address, + maybe_mark_dynamic, + nonstrict_trace, + patch_dynamo_config, + run, + set_stance, + skip_frame, + step_unsupported, + substitute_in_graph, +) +from .eval_frame import ( + _reset_guarded_backend_cache, + explain, + export, + is_dynamo_supported, + is_inductor_supported, + optimize, + optimize_assert, + OptimizedModule, + reset_code, +) + +# pyrefly: ignore [deprecated] +from .external_utils import is_compiling +from .mutation_guard import GenerationTracker +from .pgo import reset_code_state +from .symbolic_convert import TensorifyState +from .utils import ( + graph_break_reasons, + guard_failures, + orig_code_map, + register_hook_for_recompile_user_context, + reset_frame_count, + reset_recompile_user_contexts, +) + + +# Register polyfill functions +from .polyfills import loader as _ # usort: skip # noqa: F401 + + +__all__ = [ + "allow_in_graph", + "assume_constant_result", + "config", + "disable", + "disable_nested_graph_breaks", + "disallow_in_graph", + "dont_skip_tracing", + "export", + "explain", + "forbid_in_graph", + "graph_break", + "is_compiling", + "is_dynamo_disable_recursive", + "list_backends", + "lookup_backend", + "mark_dynamic", + "maybe_mark_dynamic", + "mark_static", + "mark_static_address", + "nonstrict_trace", + "optimize", + "optimize_assert", + "OptimizedModule", + "patch_dynamo_config", + "register_backend", + "replay", + "reset", + "reset_recompile_user_contexts", + "run", + "error_on_graph_break", + "set_recursion_limit", + "set_stance", + "skip_frame", + "step_unsupported", + "substitute_in_graph", +] + +# allowlist this for weights_only load of NJTs +torch.serialization.add_safe_globals([torch._dynamo.decorators._DimRange]) + +if torch.manual_seed is torch.random.manual_seed: + import torch.jit._builtins + + # Wrap manual_seed with the disable decorator. + # Can't do it at its implementation due to dependency issues. + torch.manual_seed = torch._disable_dynamo(torch.manual_seed) + # Add the new manual_seed to the builtin registry. + torch.jit._builtins._register_builtin(torch.manual_seed, "aten::manual_seed") + + +def reset() -> None: + """ + Clear all compile caches and restore initial state. This function is intended + to reset Dynamo's state *as if* you had started a fresh process invocation, which + makes it good for testing scenarios where you want to behave as if you started + a new process. It does NOT affect any file system caches. + + NB: this does NOT reset logging state. Don't use this to test logging + initialization/reinitialization. + """ + # TODO: https://github.com/pytorch/pytorch/issues/139200 + import logging + + log = logging.getLogger(__name__) + log.info("torch._dynamo.reset") + with convert_frame.compile_lock: + reset_code_caches() + convert_frame.input_codes.clear() + reset_code_state() + convert_frame.output_codes.clear() + orig_code_map.clear() + guard_failures.clear() + graph_break_reasons.clear() + resume_execution.ContinueExecutionCache.cache.clear() + _reset_guarded_backend_cache() + reset_frame_count() + torch._dynamo.compiled_autograd.reset() + convert_frame.FRAME_COUNTER = 0 + convert_frame.FRAME_COMPILE_COUNTER.clear() + callback_handler.clear() + GenerationTracker.clear() + TensorifyState.clear() + torch._dynamo.utils.warn_once_cache.clear() + torch._C._autograd._saved_tensors_hooks_set_tracing(False) + + +def reset_code_caches() -> None: + """ + Clears in-memory code cache, which is what stores compiled products. This + resets less state than :func:`reset` and is mostly only used for testing + purposes. + """ + # TODO: https://github.com/pytorch/pytorch/issues/139200 + import logging + + log = logging.getLogger(__name__) + log.info("torch._dynamo.reset_code_caches") + """Clear compile caches that are keyed by code objects""" + with convert_frame.compile_lock: + reset_code_state() + for weak_code in ( + convert_frame.input_codes.seen + convert_frame.output_codes.seen + ): + code = weak_code() + if code: + reset_code(code) + code_context.clear() + + +def get_recursion_limit() -> int: + """ + Returns the internal dynamo recursion limit set by `torch._dynamo.set_recursion_limit`. + + Returns -1 if no c recursion limit has been set. + """ + return torch._C._dynamo.eval_frame.get_c_recursion_limit() + + +def set_recursion_limit(limit: int) -> None: + """ + Sets an internal dynamo recursion limit. The limit must be >= 1, or -1 to reset + to the default (unset) state. + + This is possibly needed in Python 3.12-3.13 since there is a separate C recursion limit + that is not visible at the Python level. If you are getting RecursionErrors during + Dynamo compilation and `sys.setrecursionlimit()` doesn't help, this function may alleviate + the issue. + + NOTE: this function does NOT call `sys.setrecursionlimit()` - the user is expected to manually + call this if required. This is because the 2 recursion limits are not sync'd up - e.g. in + Python 3.12, functions can be inline-evaluated, which apparently doesn't use up the C stack. + + WARNING: increasing the recursion limit to an arbitrary large value may cause segfaults + due to stack overflows! You can try also try to manually increase the stack size, e.g. + with `$ ulimit -s ...` + """ + torch._C._dynamo.eval_frame.set_c_recursion_limit(limit) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/_trace_wrapped_higher_order_op.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/_trace_wrapped_higher_order_op.py new file mode 100644 index 0000000000000000000000000000000000000000..1de308b8037028ced08f75912e59151ff570096b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/_trace_wrapped_higher_order_op.py @@ -0,0 +1,250 @@ +"""trace_wrapped(*args, fn) is equivalent to fn(*args), but with a twist: +if you make_fx trace through this call, we will not actually trace into fn; instead, +we will directly insert it as a call_function to fn in the graph. +(Unlike make_fx, Dynamo WILL inline into fn.) +You can think of this as a one off allow_in_graph equivalent for proxy tensor tracing. + +Because proxy tensor tracing does not actually run the function, there are +requirements on the behavior of fn. We are still figuring it out, but here is the current state: + +1) fn SHOULD only take a single argument, which must be a tensor +2) fn MUST return a new tensor with the same metadata as the original tensor + (e.g., zeros_like(input) is a permissible implementation of fn). + This is verified via an extra assert that is inserted into the traced graph. +3) fn MAY have side effects, but it MAY NOT perform metadata mutation on other tensors + participating in proxy tensor tracing (it MAY mutate other tensors, it MAY mutate Python state) +These requirements stem from the requirement that we need to continue performing proxy tensor tracing, +which assumes accurate fake tensor metadata, without actually running fn. +In the future, we may allow for a "meta" function associated with fn to allow for more interesting input-output patterns. + +Note that tensors / Python state are allowed to be mutated. +This is relaxed constraint is not always sound, but it is sound for backward tracing with fake +tensors as it takes place in AOTAutograd, as the backward pass is guaranteed not to depend on concrete +tensor values (via fake tensor) or Python state (because the autograd engine doesn't depend on Python). + +The intended use case for this function is to allow AOTAutograd to defer complex +backward hooks to compiled autograd. AOTAutograd performs a make_fx trace which preserves +the function call as is in the graph, and only when we Dynamo through the backward graph in +compiled autograd do we inline into the function. +""" + +from typing import Any, Optional + +import torch +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._higher_order_ops.utils import autograd_not_implemented +from torch._ops import HigherOrderOperator, OpOverload +from torch._subclasses import FakeTensorMode +from torch.fx.experimental._backward_state import BackwardState +from torch.fx.experimental.proxy_tensor import ProxyTorchDispatchMode, track_tensor_tree +from torch.overrides import TorchFunctionMode +from torch.utils._python_dispatch import _get_current_dispatch_mode +from torch.utils._pytree import tree_map_only + + +Tensor = torch.Tensor + + +__all__ = ["trace_wrapped"] + + +@torch.library.custom_op("flex_lib::zeros_and_scatter", mutates_args=()) # type: ignore[misc] +def zeros_and_scatter( + shape: list[int], + indices: list[Tensor], + vals: Tensor, +) -> Tensor: + """Custom Op so that we can register a custom lowering for the new_output + scatter in the backwards pass""" + grad = torch.zeros(shape, device=vals.device, dtype=vals.dtype) + return torch.ops.aten.index_put(grad, indices, vals, accumulate=True) + + +@zeros_and_scatter.register_fake # type: ignore[misc] +def _( + shape: list[int], + indices: list[Tensor], + vals: Tensor, +) -> Tensor: + return vals.new_empty(shape) + + +@zeros_and_scatter.register_vmap # type: ignore[misc] +def _(info, indims, shape, indices, value): # type: ignore[no-untyped-def] + """The batching rule is special in that it returns a tensor that is not batched""" + indices_indims = indims[1] + expanded_indices = [] + for idx, idx_indim in zip(indices, indices_indims): + # The index is not a being batched, we should unsqueeze and expand to val + if idx_indim is None: + expanded_indices.append(idx.expand(value.shape)) + else: + # the index is being part of the vmap batch, it should be the same size as val + assert idx.shape == value.shape + expanded_indices.append(idx) + + out = torch.ops.flex_lib.zeros_and_scatter( + shape, + expanded_indices, + value, + ) + return out, None + + +class ModIndex(torch.autograd.Function): + generate_vmap_rule = True + + @staticmethod + # pyrefly: ignore [bad-override] + def forward(x: Tensor, indices: list[Tensor]) -> Tensor: + return torch.ops.aten.index(x, indices) + + @staticmethod + def setup_context(ctx: Any, inputs: tuple[Any, ...], output: Any) -> None: + x, indices = inputs + ctx.save_for_backward(*indices) + ctx.input_shape = x.shape + + @staticmethod + def backward(ctx, gradOut): # type: ignore[no-untyped-def] + indices = ctx.saved_tensors + return ( + torch.ops.flex_lib.zeros_and_scatter( + ctx.input_shape, + indices, + gradOut, + ), + None, + ) + + @classmethod + @torch._export.wrappers.allow_in_pre_dispatch_graph + def apply(cls, *args, **kwargs): # type: ignore[no-untyped-def] + return super().apply(*args, **kwargs) + + +mod_index = ModIndex.apply + + +class TransformGetItemToIndex(TorchFunctionMode): + # This is needed since we want to support calling + # A[q_idx], where q_idx is a scalar tensor in score_mod. + # Today, when q_idx is a scalar tensor, we implicitly convert it to a python + # scalar and create a view. We do not want that behavior in this case, so we + # use this torchfunctionmode to override that behavior for score_mod + # wherever we're running it. + def __torch_function__( + self, + func: OpOverload, + types: tuple[torch._C._TensorMeta, ...], + args: tuple[object, ...] = (), + kwargs: Optional[dict[str, object]] = None, + ) -> object: + if func is torch.Tensor.__getitem__: + index_args = pytree.tree_leaves(args[1]) + if all(isinstance(x, torch.Tensor) for x in index_args): + return mod_index(args[0], index_args) + return func(*args, **(kwargs or {})) + + +def trace_wrapped(*args: Any, **kwargs: Any) -> Any: + with torch.no_grad(): + return _trace_wrapped_op(*args, **kwargs) + + +class TraceWrapped(HigherOrderOperator): + def __init__(self) -> None: + super().__init__("trace_wrapped") + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + return super().__call__(*args, **kwargs) + + +# TODO(jansel): need to ensure this does not get DCEed +_trace_wrapped_op = TraceWrapped() + + +def _assert_meta( + grad: torch.Tensor, + size: tuple[int, ...], + stride: tuple[int, ...], + dtype: torch.dtype, +) -> torch.Tensor: + assert grad.size() == size, "size mismatch" + assert grad.stride() == stride, "stride mismatch" + assert grad.dtype == dtype, "dtype mismatch" + return grad + + +@_trace_wrapped_op.py_impl(ProxyTorchDispatchMode) +def inner_trace( + mode: ProxyTorchDispatchMode, + *args: Any, + bw_state: Optional[BackwardState] = None, + **kwargs: Any, +) -> Any: + def self_invoke(*args: Any, **dyn_kwargs: Any) -> Any: + with torch.no_grad(): + return _trace_wrapped_op(*args, **dyn_kwargs, **kwargs) + + def unwrap_proxies(x: Any) -> Any: + if isinstance(x, torch.Tensor): + return mode.tracer.unwrap_proxy(x) # type: ignore[union-attr] + if isinstance(x, (list, tuple)): + return type(x)(map(unwrap_proxies, x)) + if x is None: + return None + raise AssertionError(f"unhandled type: {type(x)}") + + proxy_kwargs = {} + if bw_state is not None: + assert isinstance(bw_state, BackwardState) and bw_state.proxy is not None + proxy_kwargs["bw_state"] = bw_state.proxy + out_proxy = mode.tracer.create_proxy( + "call_function", + self_invoke, + unwrap_proxies(args), + proxy_kwargs, + name="trace_wrapped", + ) + + if args[0] is None: + grad = args[1] # module backward hooks + else: + grad = args[0] # other backward hooks + grad = tree_map_only(torch.Tensor, torch.empty_like, grad) + track_tensor_tree(grad, out_proxy, constant=None, tracer=mode.tracer) + return grad + + +@_trace_wrapped_op.py_impl(FakeTensorMode) +def inner_fake(*args: Any, **kwargs: Any) -> None: + raise RuntimeError("This op should never be invoked here") + + +@_trace_wrapped_op.py_impl(DispatchKey.CompositeExplicitAutograd) +def _trace_wrapped_op_dense(*args: Any, fn: Any, **kwargs: Any) -> Any: + mode = _get_current_dispatch_mode() + assert mode is None, "Mode should never be enabled for CPU/CUDA key" + return fn(*args, **kwargs) + + +_trace_wrapped_op.py_impl(DispatchKey.Autograd)( + autograd_not_implemented(_trace_wrapped_op, deferred_error=True) +) + + +@_trace_wrapped_op.py_functionalize_impl +def _trace_wrapped_functionalized(ctx: Any, *args: Any, **kwargs: Any) -> Any: + unwrapped_args = ctx.unwrap_tensors(args) + with ctx.redispatch_to_next(): + return ctx.wrap_tensors(_trace_wrapped_op(*unwrapped_args, **kwargs)) + + +def autograd_function_backward_rewritten(original_backward: Any) -> Any: + def new_backward(ctx: Any, *grads: Any) -> Any: + # pyrefly: ignore [bad-assignment] + grads = [g.contiguous() for g in grads] + return original_backward(ctx, *grads) + + return new_backward diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/aot_compile.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/aot_compile.py new file mode 100644 index 0000000000000000000000000000000000000000..7bc03aff84a20b0a73783eafcadc6b37aa1b56d1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/aot_compile.py @@ -0,0 +1,369 @@ +import dataclasses +import inspect +import io +import logging +import pickle +import types +from collections.abc import Callable +from contextlib import AbstractContextManager, ExitStack +from dataclasses import dataclass +from typing import Any, Optional, TYPE_CHECKING + +import torch +import torch.fx +from torch._dynamo.convert_frame import GraphRuntimeEnv +from torch._dynamo.graph_utils import _graph_device_type +from torch._dynamo.package import SystemInfo + +from . import convert_frame +from .aot_compile_types import ( + BundledAOTAutogradSerializableCallable, + SerializableCallable, +) +from .hooks import Hooks + + +if TYPE_CHECKING: + from .guards import GuardManagerWrapper + from .package import SourceInfo + + +log = logging.getLogger(__name__) + + +def bind_locals( + signature: inspect.Signature, *args: Any, **kwargs: Any +) -> dict[str, Any]: + bound_arguments = signature.bind(*args, **kwargs) + bound_arguments.apply_defaults() + return bound_arguments.arguments + + +@dataclass +class CompileArtifacts: + signature: inspect.Signature + guard_manager: Optional["GuardManagerWrapper"] + guards_state: bytes + backend_id: str + compiled_fn: SerializableCallable + original_code: types.CodeType + runtime_env: GraphRuntimeEnv + source_info: "SourceInfo" + device_type: str + backend_name: str + system_info: SystemInfo = dataclasses.field(default_factory=SystemInfo.current) + + def check_compatibility(self) -> None: + current_system = SystemInfo.current() + current_system.check_compatibility(self.system_info, self.device_type) + + +class AOTCompilePickler(pickle.Pickler): + @classmethod + def _unpickle_cell(cls, val: Any) -> Any: + def _() -> Any: + return val + + assert _.__closure__ is not None + return _.__closure__[0] + + # pyrefly: ignore [bad-override] + def reducer_override(self, obj: Any) -> Any: + if isinstance(obj, type((lambda x: lambda: x)(0).__closure__[0])): # type: ignore[index] # noqa: PLC3002 + return type(self)._unpickle_cell, (obj.cell_contents,) + return NotImplemented + + +@dataclass +class AOTCompiledFunction: + _artifacts: CompileArtifacts + _guard_check_enabled: bool = True + _extra_globals: Optional[dict[str, object]] = None + + def guard_check(self, *args: Any, **kwargs: Any) -> bool: + f_locals: dict[str, Any] = {} + env = self._artifacts.runtime_env + if env.closure: + assert env.bytecode.co_freevars and len(env.closure) == len( + env.bytecode.co_freevars + ) + f_locals = { + name: cell.cell_contents + for name, cell in zip(env.bytecode.co_freevars, env.closure) + } + f_locals.update(bind_locals(self._artifacts.signature, *args, **kwargs)) + assert self._artifacts.guard_manager is not None + return self._artifacts.guard_manager.check(f_locals) + + def __post_init__(self) -> None: + from .package import load_guard_manager, load_guards_state + + self._artifacts.check_compatibility() + + # pyrefly: ignore [read-only] + self.fn = self._artifacts.runtime_env.forward_callable( + self._artifacts.backend_id, + self._artifacts.compiled_fn, + extra_globals=self._extra_globals, + ) + + if self._artifacts.guard_manager is None: + guards_state = load_guards_state(self._artifacts.guards_state) + self._artifacts.guard_manager = load_guard_manager( + guards_state, + self._artifacts.original_code, + self.fn.__globals__, + ) + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + assert self._artifacts.guard_manager is not None + if self._guard_check_enabled and not self.guard_check(*args, **kwargs): + f_locals = bind_locals(self._artifacts.signature, *args, **kwargs) + reason = str(self._artifacts.guard_manager.check_verbose(f_locals)) + raise RuntimeError(f"GuardManager check failed, reason: {reason}") + return self.fn(*args, **kwargs) + + def source_info(self) -> "SourceInfo": + return self._artifacts.source_info + + def save_compiled_function(self, path: str) -> None: + with open(path, "wb") as f: + f.write(type(self).serialize(self)) + + @classmethod + def serialize(cls, fn: "AOTCompiledFunction") -> bytes: + from torch._dynamo.package import SerializedCode + + state = fn._artifacts.__dict__.copy() + state["guard_manager"] = None + state["runtime_env"] = dataclasses.replace( + state["runtime_env"], + bytecode=SerializedCode.from_code_object(state["runtime_env"].bytecode), + ) + compiled_fn = state["compiled_fn"] + state["compiled_fn"] = ( + type(compiled_fn).deserialize_compile_artifacts, + type(compiled_fn).serialize_compile_artifacts(compiled_fn), + ) + state["original_code"] = SerializedCode.from_code_object(state["original_code"]) + buf = io.BytesIO() + pickler = AOTCompilePickler(buf) + pickler.dump(state) + return buf.getvalue() + + @classmethod + def deserialize( + cls, data: bytes, f_globals: Optional[dict[str, object]] = None + ) -> "AOTCompiledFunction": + from torch._dynamo.package import SerializedCode + + state = pickle.loads(data) + state["runtime_env"] = dataclasses.replace( + state["runtime_env"], + bytecode=SerializedCode.to_code_object(state["runtime_env"].bytecode), + ) + deserializer, compiled_fn_state = state["compiled_fn"] + with torch._inductor.config.patch(enable_autograd_for_aot=True): + state["compiled_fn"] = deserializer(compiled_fn_state) + state["original_code"] = SerializedCode.to_code_object(state["original_code"]) + + artifacts = CompileArtifacts(**state) + return cls(artifacts, _extra_globals=f_globals) + + def disable_guard_check(self) -> None: + self._guard_check_enabled = False + + +def aot_compile_fullgraph( + model: Any, + example_inputs: tuple[tuple[Any, ...], dict[str, Any]], + hooks: Hooks, + backend: Callable[[torch.fx.GraphModule, list[torch.Tensor]], SerializableCallable], +) -> AOTCompiledFunction: + from torch._dynamo.guards import CheckFunctionManager + from torch._dynamo.package import SourceInfo + from torch._dynamo.utils import dynamo_timed, get_metrics_context + from torch._guards import TracingContext + + args, kwargs = example_inputs + + with ( + get_metrics_context(), + dynamo_timed("fullgraph_capture"), + torch._functorch.config.patch(strict_autograd_cache=True), + ): + capture_output = convert_frame.fullgraph_capture(model, args, kwargs) + graph_capture_output = capture_output.graph_capture_output + assert graph_capture_output.output_graph is not None + + if not hooks.guard_filter_fn: + from torch._dynamo.types import GuardFilterEntry + + def new_guard_filter_fn( + guard_entries: list[GuardFilterEntry], + ) -> list[bool]: + return [ + ( + not ( + g.is_global + or g.guard_type + in CheckFunctionManager.UNSUPPORTED_SERIALIZATION_GUARD_TYPES + ) + ) + for g in guard_entries + ] + + hooks.guard_filter_fn = new_guard_filter_fn + + fn, _ = convert_frame.get_traced_fn(model) + + backend_input = capture_output.backend_input + assert backend_input is not None + backend_input.graph_module._backend_id = backend_input.backend_id # type: ignore[assignment] + device_type = _graph_device_type(backend_input.graph_module.graph) + assert ( + backend_input.fake_mode.shape_env + is graph_capture_output.output_graph.shape_env + ) + tracing_context = TracingContext(backend_input.fake_mode) + tracing_context.tensor_to_context = backend_input.tensor_to_context + with ( + torch._guards.tracing(tracing_context), + torch._functorch.config.patch( + { + "bundled_autograd_cache": True, + "force_non_lazy_backward_lowering": True, + } + ), + ): + compiled_fn = backend( + backend_input.graph_module, backend_input.example_inputs + ) + # If Inductor backend is used, grab the compiled_fn from PrecompileContext + # TODO: this should be replaced once we make the backend return the SerializableCallable directly. + if isinstance(backend, torch._TorchCompileInductorWrapper): + compiled_fn = BundledAOTAutogradSerializableCallable(compiled_fn) + + if not isinstance(compiled_fn, SerializableCallable): + if hasattr(backend, "compiler_fn"): + compiler_fn = backend.compiler_fn + else: + compiler_fn = backend + raise RuntimeError( + f"Compiled function type {type(compiled_fn)} (produced " + + f"from backend {compiler_fn}) does not implement SerializableCallable." + ) + + check_fn = graph_capture_output.build_guards( + fn.__code__, hooks=hooks, save=True, strict_error=True + ) + + assert check_fn.guards_state is not None + + source_info = SourceInfo(inlined_sources=set()) + for traced_code in graph_capture_output.traced_code: + source_info.add_code(traced_code) + + artifacts = CompileArtifacts( + signature=convert_frame._get_signature(fn), + guard_manager=check_fn.guard_manager, + guards_state=check_fn.guards_state, + backend_id=backend_input.backend_id, + compiled_fn=compiled_fn, + original_code=fn.__code__, + runtime_env=graph_capture_output.get_runtime_env(), + source_info=source_info, + device_type=device_type, + backend_name=getattr(backend, "compiler_name", "unknown"), + ) + aot_compiled_fn = AOTCompiledFunction( + _artifacts=artifacts, _extra_globals=fn.__globals__ + ) + + return aot_compiled_fn + + +@dataclass +class ModelInput: + """ + WIP type: represents a single model input + Which consists of a tuple of arguments and a set of contexts in which to run the model. + + For each ModelInput, we'll compile one full graph of the model, and then use the guards generated + to dispatch between the compiled graphs. + + + """ + + args: tuple[Any] + kwargs: dict[str, Any] + contexts: list[AbstractContextManager[Any]] + + +@dataclass +class AOTCompiledModel: + # Represents a single forward function of a model along with dispatch + # compiled_results is serializable. We require the model to deserialize again. + model: torch.nn.Module + compiled_results: list[AOTCompiledFunction] + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + for result in self.compiled_results: + if result.guard_check(self.model, *args, **kwargs): + return result(self.model, *args, **kwargs) + # All guards failed, just run one of them and throw the guard check error. + return self.compiled_results[0](self.model, *args, **kwargs) + + def serialize(self) -> bytes: + data: list[bytes] = [] + for result in self.compiled_results: + data.append(AOTCompiledFunction.serialize(result)) + return pickle.dumps(data) + + @classmethod + def deserialize(cls, model: torch.nn.Module, data: bytes) -> "AOTCompiledModel": + from torch._dynamo.utils import get_metrics_context + from torch._guards import compile_context, CompileContext + + results: list[bytes] = pickle.loads(data) + compiled_results = [] + for result in results: + with ( + compile_context(CompileContext(convert_frame.get_compile_id({}))), + get_metrics_context(), + ): + compiled_results.append(AOTCompiledFunction.deserialize(result)) + return cls(model, compiled_results) + + +def aot_compile_module( + model: torch.nn.Module, + inputs: list[ModelInput], + hooks: Hooks, + backend: Callable[[torch.fx.GraphModule, list[torch.Tensor]], SerializableCallable], +) -> AOTCompiledModel: + """ + Compiles a single nn.Module with any number of inputs, and returns a compiled forward function. + """ + + def compile_single_graph(model_input: ModelInput) -> AOTCompiledFunction: + example_inputs = (model_input.args, model_input.kwargs) + orig_forward = model.forward + with ExitStack() as stack: + for ctx in model_input.contexts: + stack.enter_context(ctx) + return aot_compile_fullgraph( + orig_forward, + example_inputs, + hooks=hooks, + backend=backend, + ) + + compiled_results = [] + for model_input in inputs: + log.info("Compiling input %s..", model_input) + compiled_results.append(compile_single_graph(model_input)) + + assert len(compiled_results) > 0 + + return AOTCompiledModel(model, compiled_results) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/aot_compile_types.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/aot_compile_types.py new file mode 100644 index 0000000000000000000000000000000000000000..4a6604681bbfbc4a3bc3852ef6cb6defd3800ccb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/aot_compile_types.py @@ -0,0 +1,65 @@ +import abc +import pickle +from typing import Any + +import torch + + +class SerializableCallable(abc.ABC): + @classmethod + @abc.abstractmethod + def serialize_compile_artifacts(cls, fn: Any) -> bytes: + pass + + @classmethod + @abc.abstractmethod + def deserialize_compile_artifacts(cls, data: bytes) -> Any: + pass + + @abc.abstractmethod + def __call__(self, *args: Any, **kwargs: Any) -> Any: + pass + + +class BundledAOTAutogradSerializableCallable(SerializableCallable): + """ + Represents a serializable callable generated by compile_fx. + This class wraps around the compiled function generated by AOTAutograd. + + TODO: Instead of using PrecompileContext to grab it from AOTAutograd, + this object should be what's *returned* by aot_module_simplified. + We'll do that refactor in a later PR. + """ + + def __init__(self, compiled_fn: Any) -> None: + """ + Takes in a BundledAOTAutogradCacheArtifact, which is the serialized form + of a compiled function generated by AOTAutograd. + """ + assert hasattr(compiled_fn, "serialize") + self.compiled_fn = compiled_fn + + def __getattr__(self, attr: Any) -> Any: + return getattr(self.compiled_fn, attr) + + @classmethod + def serialize_compile_artifacts( + cls, fn: "BundledAOTAutogradSerializableCallable" + ) -> bytes: + with torch._functorch.config.patch("bundled_autograd_cache", True): + result = pickle.dumps(fn.compiled_fn.serialize()) + return result + + @classmethod + def deserialize_compile_artifacts(cls, data: bytes) -> Any: + from torch._functorch._aot_autograd.aot_autograd_result import ( + deserialize_bundled_cache_entry, + ) + + entry = pickle.loads(data) + + compiled_fn = deserialize_bundled_cache_entry(entry) + return cls(compiled_fn) + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + return self.compiled_fn(*args, **kwargs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/bytecode_analysis.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/bytecode_analysis.py new file mode 100644 index 0000000000000000000000000000000000000000..c7a982906b3fef2c10db4147b5defe988b8b2e84 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/bytecode_analysis.py @@ -0,0 +1,265 @@ +""" +This module provides utilities for analyzing and optimizing Python bytecode. +Key functionality includes: +- Dead code elimination +- Jump instruction optimization +- Stack size analysis and verification +- Live variable analysis +- Line number propagation and cleanup +- Exception table handling for Python 3.11+ + +The utilities in this module are used to analyze and transform bytecode +for better performance while maintaining correct semantics. +""" + +import bisect +import dataclasses +import dis +import itertools +import sys +from typing import Any, TYPE_CHECKING, Union + + +if TYPE_CHECKING: + # TODO(lucaskabela): consider moving Instruction into this file + # and refactoring in callsite; that way we don't have to guard this import + from .bytecode_transformation import Instruction + +TERMINAL_OPCODES = { + dis.opmap["RETURN_VALUE"], + dis.opmap["JUMP_FORWARD"], + dis.opmap["RAISE_VARARGS"], + # TODO(jansel): double check exception handling +} +TERMINAL_OPCODES.add(dis.opmap["RERAISE"]) +if sys.version_info >= (3, 11): + TERMINAL_OPCODES.add(dis.opmap["JUMP_BACKWARD"]) + TERMINAL_OPCODES.add(dis.opmap["JUMP_FORWARD"]) +else: + TERMINAL_OPCODES.add(dis.opmap["JUMP_ABSOLUTE"]) +# pyrefly: ignore [unsupported-operation] +if (3, 12) <= sys.version_info < (3, 14): + TERMINAL_OPCODES.add(dis.opmap["RETURN_CONST"]) +if sys.version_info >= (3, 13): + TERMINAL_OPCODES.add(dis.opmap["JUMP_BACKWARD_NO_INTERRUPT"]) +JUMP_OPCODES = set(dis.hasjrel + dis.hasjabs) +JUMP_OPNAMES = {dis.opname[opcode] for opcode in JUMP_OPCODES} +HASLOCAL = set(dis.haslocal) +HASFREE = set(dis.hasfree) + +stack_effect = dis.stack_effect + + +def get_indexof(insts: list["Instruction"]) -> dict["Instruction", int]: + """ + Get a mapping from instruction memory address to index in instruction list. + Additionally checks that each instruction only appears once in the list. + """ + indexof = {} + for i, inst in enumerate(insts): + assert inst not in indexof + indexof[inst] = i + return indexof + + +def remove_dead_code(instructions: list["Instruction"]) -> list["Instruction"]: + """Dead code elimination""" + indexof = get_indexof(instructions) + live_code = set() + + def find_live_code(start: int) -> None: + for i in range(start, len(instructions)): + if i in live_code: + return + live_code.add(i) + inst = instructions[i] + if inst.exn_tab_entry: + find_live_code(indexof[inst.exn_tab_entry.target]) + if inst.opcode in JUMP_OPCODES: + assert inst.target is not None + find_live_code(indexof[inst.target]) + if inst.opcode in TERMINAL_OPCODES: + return + + find_live_code(0) + + # change exception table entries if start/end instructions are dead + # assumes that exception table entries have been propagated, + # e.g. with bytecode_transformation.propagate_inst_exn_table_entries, + # and that instructions with an exn_tab_entry lies within its start/end. + if sys.version_info >= (3, 11): + live_idx = sorted(live_code) + for i, inst in enumerate(instructions): + if i in live_code and inst.exn_tab_entry: + # find leftmost live instruction >= start + start_idx = bisect.bisect_left( + live_idx, indexof[inst.exn_tab_entry.start] + ) + assert start_idx < len(live_idx) + # find rightmost live instruction <= end + end_idx = ( + bisect.bisect_right(live_idx, indexof[inst.exn_tab_entry.end]) - 1 + ) + assert end_idx >= 0 + assert live_idx[start_idx] <= i <= live_idx[end_idx] + inst.exn_tab_entry.start = instructions[live_idx[start_idx]] + inst.exn_tab_entry.end = instructions[live_idx[end_idx]] + + return [inst for i, inst in enumerate(instructions) if i in live_code] + + +def remove_pointless_jumps(instructions: list["Instruction"]) -> list["Instruction"]: + """Eliminate jumps to the next instruction""" + pointless_jumps = { + id(a) + for a, b in itertools.pairwise(instructions) + if a.opname == "JUMP_ABSOLUTE" and a.target is b + } + return [inst for inst in instructions if id(inst) not in pointless_jumps] + + +def propagate_line_nums(instructions: list["Instruction"]) -> None: + """Ensure every instruction has line number set in case some are removed""" + cur_line_no = None + + def populate_line_num(inst: "Instruction") -> None: + nonlocal cur_line_no + if inst.starts_line: + cur_line_no = inst.starts_line + + inst.starts_line = cur_line_no + + for inst in instructions: + populate_line_num(inst) + + +def remove_extra_line_nums(instructions: list["Instruction"]) -> None: + """Remove extra starts line properties before packing bytecode""" + + cur_line_no = None + + def remove_line_num(inst: "Instruction") -> None: + nonlocal cur_line_no + if inst.starts_line is None: + return + elif inst.starts_line == cur_line_no: + inst.starts_line = None + else: + cur_line_no = inst.starts_line + + for inst in instructions: + remove_line_num(inst) + + +@dataclasses.dataclass +class ReadsWrites: + reads: set[Any] + writes: set[Any] + visited: set[Any] + + +def livevars_analysis( + instructions: list["Instruction"], instruction: "Instruction" +) -> set[Any]: + indexof = get_indexof(instructions) + must = ReadsWrites(set(), set(), set()) + may = ReadsWrites(set(), set(), set()) + + def walk(state: ReadsWrites, start: int) -> None: + if start in state.visited: + return + state.visited.add(start) + + for i in range(start, len(instructions)): + inst = instructions[i] + if inst.opcode in HASLOCAL or inst.opcode in HASFREE: + if "LOAD" in inst.opname or "DELETE" in inst.opname: + if inst.argval not in must.writes: + state.reads.add(inst.argval) + elif "STORE" in inst.opname: + state.writes.add(inst.argval) + elif inst.opname == "MAKE_CELL": + pass + else: + raise NotImplementedError(f"unhandled {inst.opname}") + if inst.exn_tab_entry: + walk(may, indexof[inst.exn_tab_entry.target]) + if inst.opcode in JUMP_OPCODES: + assert inst.target is not None + walk(may, indexof[inst.target]) + state = may + if inst.opcode in TERMINAL_OPCODES: + return + + walk(must, indexof[instruction]) + return must.reads | may.reads + + +@dataclasses.dataclass +class FixedPointBox: + value: bool = True + + +@dataclasses.dataclass +class StackSize: + low: Union[int, float] + high: Union[int, float] + fixed_point: FixedPointBox + + def zero(self) -> None: + self.low = 0 + self.high = 0 + self.fixed_point.value = False + + def offset_of(self, other: "StackSize", n: int) -> None: + prior = (self.low, self.high) + self.low = min(self.low, other.low + n) + self.high = max(self.high, other.high + n) + if (self.low, self.high) != prior: + self.fixed_point.value = False + + def exn_tab_jump(self, depth: int) -> None: + prior = (self.low, self.high) + self.low = min(self.low, depth) + self.high = max(self.high, depth) + if (self.low, self.high) != prior: + self.fixed_point.value = False + + +def stacksize_analysis(instructions: list["Instruction"]) -> Union[int, float]: + assert instructions + fixed_point = FixedPointBox() + stack_sizes = { + inst: StackSize(float("inf"), float("-inf"), fixed_point) + for inst in instructions + } + stack_sizes[instructions[0]].zero() + + for _ in range(100): + if fixed_point.value: + break + fixed_point.value = True + + for inst, next_inst in zip(instructions, instructions[1:] + [None]): + stack_size = stack_sizes[inst] + if inst.opcode not in TERMINAL_OPCODES: + assert next_inst is not None, f"missing next inst: {inst}" + eff = stack_effect(inst.opcode, inst.arg, jump=False) + stack_sizes[next_inst].offset_of(stack_size, eff) + if inst.opcode in JUMP_OPCODES: + assert inst.target is not None, f"missing target: {inst}" + stack_sizes[inst.target].offset_of( + stack_size, stack_effect(inst.opcode, inst.arg, jump=True) + ) + if inst.exn_tab_entry: + # see https://github.com/python/cpython/blob/3.11/Objects/exception_handling_notes.txt + # on why depth is computed this way. + depth = inst.exn_tab_entry.depth + int(inst.exn_tab_entry.lasti) + 1 + stack_sizes[inst.exn_tab_entry.target].exn_tab_jump(depth) + + low = min(x.low for x in stack_sizes.values()) + high = max(x.high for x in stack_sizes.values()) + + assert fixed_point.value, "failed to reach fixed point" + assert low >= 0 + return high diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/bytecode_transformation.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/bytecode_transformation.py new file mode 100644 index 0000000000000000000000000000000000000000..31c1d243de721e1a4fe5ed01a73a90789f964248 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/bytecode_transformation.py @@ -0,0 +1,1846 @@ +""" +This module provides utilities for analyzing, transforming and manipulating Python bytecode. +It includes functionality for: +- Converting between different bytecode formats and versions +- Virtualizing jumps and managing jump targets +- Handling exception tables and their entries +- Managing instruction offsets and extended arguments +- Providing a clean API for bytecode modification and transformation +- Supporting Python version-specific bytecode features +- Generating bytecode from template functions + +The module is designed to work across different Python versions (3.7+) and handles +version-specific bytecode differences transparently. +""" + +import copy +import dataclasses +import dis +import functools +import itertools +import sys +import types +import uuid +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence +from typing import Any, cast, Optional, TYPE_CHECKING, Union + +from . import config +from .bytecode_analysis import ( + get_indexof, + propagate_line_nums, + remove_extra_line_nums, + stacksize_analysis, +) +from .utils import is_safe_constant + + +if TYPE_CHECKING: + from .output_graph import DynamoTracerOutput + + +@dataclasses.dataclass(slots=True) +class InstructionExnTabEntry: + start: "Instruction" + end: "Instruction" + target: "Instruction" + depth: int + lasti: bool + + def __repr__(self) -> str: + return ( + f"InstructionExnTabEntry(start={self.start.short_inst_repr()}, " + f"end={self.end.short_inst_repr()}, " + f"target={self.target.short_inst_repr()}, " + f"depth={self.depth}, lasti={self.lasti})" + ) + + def __eq__(self, o: object) -> bool: + if not isinstance(o, InstructionExnTabEntry): + return False + return ( + self.start is o.start + and self.end is o.end + and self.target is o.target + and self.depth == o.depth + and self.lasti == o.lasti + ) + + +@dataclasses.dataclass(slots=True) +class Instruction: + """A mutable version of dis.Instruction""" + + opcode: int + opname: str + arg: Optional[int] + argval: Any + offset: Optional[int] = None + starts_line: Optional[int] = None + is_jump_target: bool = False + positions: Optional["dis.Positions"] = None + # extra fields to make modification easier: + target: Optional["Instruction"] = None + exn_tab_entry: Optional[InstructionExnTabEntry] = None + argrepr: Optional[str] = None + + def __hash__(self) -> int: + return id(self) + + def __eq__(self, other: object) -> bool: + return id(self) == id(other) + + def short_inst_repr(self) -> str: + return f"Instruction(opname={self.opname}, offset={self.offset})" + + def copy_positions(self, other: "Instruction") -> None: + self.starts_line = other.starts_line + self.positions = other.positions + + +if sys.version_info >= (3, 13): + + def convert_instruction(i: dis.Instruction) -> Instruction: + return Instruction( + i.opcode, + i.opname, + i.arg, + i.argval, + i.offset, + i.line_number, + i.is_jump_target, + i.positions, + ) + +elif sys.version_info >= (3, 11): + + def convert_instruction(i: dis.Instruction) -> Instruction: + return Instruction( + i.opcode, + i.opname, + i.arg, + i.argval, + i.offset, + i.starts_line, + i.is_jump_target, + i.positions, + ) + +else: + + def convert_instruction(i: dis.Instruction) -> Instruction: + return Instruction( + i.opcode, + i.opname, + i.arg, + i.argval, + i.offset, + i.starts_line, + i.is_jump_target, + None, + ) + + +class _NotProvided: + def __repr__(self) -> str: + return "_NotProvided" + + +if sys.version_info >= (3, 12): + + def inst_has_op_bits(name: str) -> bool: + return name in ("LOAD_ATTR", "LOAD_GLOBAL", "LOAD_SUPER_ATTR") + +elif sys.version_info >= (3, 11): + + def inst_has_op_bits(name: str) -> bool: + return name == "LOAD_GLOBAL" + +else: + + def inst_has_op_bits(name: str): + return False + + +def create_instruction( + name: str, + *, + arg: Optional[int] = None, + argval: Optional[Any] = _NotProvided, + target: Optional[Instruction] = None, +) -> Instruction: + """ + At most one of `arg`, `argval`, and `target` can be not None/_NotProvided. + This is to prevent ambiguity, e.g. does + create_instruction("LOAD_CONST", 5) + mean load the constant at co_consts[5], or load the constant 5? + + If `arg` is not provided, it will be computed during assembly from + `argval` or `target`. + + Bits in the args of instructions LOAD_GLOBAL, LOAD_ATTR (3.12+), and LOAD_SUPER_ATTR + modify the behavior of the instruction. In this case, we allow both `arg` + and `argval` to be set. The value of `arg` here is expected to be the value of + the op bits and the true value of `arg` will be computed during assembly. + If `arg` is not set, the bits are assumed to be 0. + """ + + # allow for instructions with op bits to have both arg and argval specified + if inst_has_op_bits(name): + if target is not None: + raise RuntimeError("target cannot be specified for instruction") + if arg is None: + arg = 0 + else: + cnt = (arg is not None) + (argval is not _NotProvided) + (target is not None) + if cnt > 1: + raise RuntimeError( + "only one of arg, argval, and target can be not None/_NotProvided" + ) + if arg is not None and not isinstance(arg, int): + raise RuntimeError("instruction arg must be int or None") + return Instruction( + opcode=dis.opmap[name], opname=name, arg=arg, argval=argval, target=target + ) + + +# Python 3.11 remaps +def create_jump_absolute(target: Instruction) -> Instruction: + inst = "JUMP_FORWARD" if sys.version_info >= (3, 11) else "JUMP_ABSOLUTE" + return create_instruction(inst, target=target) + + +def is_jump_absolute(target: Instruction) -> bool: + return target.opname in ("JUMP_FORWARD", "JUMP_ABSOLUTE") + + +def create_load_const(val: Any, checked: bool = True) -> Instruction: + """ + In general we should only create `LOAD_CONST` for immutable objects, but + sometimes it's convenient _and safe_ for Dynamo create `LOAD_CONST` for + mutable objects. In such cases, use `checked=False`. + """ + if checked: + assert is_safe_constant(val), f"unsafe constant {val}" + return create_instruction("LOAD_CONST", argval=val) + + +def create_dup_top() -> Instruction: + if sys.version_info >= (3, 11): + return create_instruction("COPY", arg=1) + return create_instruction("DUP_TOP") + + +def create_rot_n(n: int) -> list[Instruction]: + """ + Returns a "simple" sequence of instructions that rotates TOS to the n-th + position in the stack. For Python < 3.11, returns a single ROT_* + instruction. If no such instruction exists, an error is raised and the + caller is expected to generate an equivalent sequence of instructions. + For Python >= 3.11, any rotation can be expressed as a simple sequence of + swaps. + """ + if n <= 1: + # don't rotate + return [] + + if sys.version_info >= (3, 11): + # rotate can be expressed as a sequence of swap operations + # e.g. rotate 3 is equivalent to swap 3, swap 2 + return [create_instruction("SWAP", arg=i) for i in range(n, 1, -1)] + + if n <= 4: + return [create_instruction("ROT_" + ["TWO", "THREE", "FOUR"][n - 2])] + return [create_instruction("ROT_N", arg=n)] + + +def add_push_null( + inst_or_insts: Union[Instruction, list[Instruction]], +) -> list[Instruction]: + """ + Appends or prepends a PUSH_NULL instruction to `inst_or_insts`, + depending on Python version. Used when you know that + `inst_or_insts` generates a callable that will be called. + + NOTE: Assumes `inst_or_insts` is a single instruction or sequence of + instructions that pushes exactly 1 object to the stack that is to + be called. It is important that you include ALL instructions that + construct the callable - not just the first instruction/a prefix. + + Will attempt to use the NULL push bit for instructions + with such bits (LOAD_GLOBAL 3.11+, LOAD_ATTR 3.12+, LOAD_SUPER_ATTR). + In this case, instructions WILL be modified. + """ + if isinstance(inst_or_insts, Instruction): + insts: list[Instruction] = [inst_or_insts] + else: + assert isinstance(inst_or_insts, list) + insts = inst_or_insts + + def inst_has_bit_set(idx: int) -> bool: + assert insts[idx].arg is not None + return insts[idx].arg & 1 == 1 # type: ignore[operator] + + def set_inst_bit(idx: int) -> None: + assert insts[idx].arg is not None + insts[idx].arg |= 1 # type: ignore[operator] + + if sys.version_info >= (3, 13): + # In 3.13, NULL follows the callable + if inst_has_op_bits(insts[-1].opname) and not inst_has_bit_set(-1): + # All insts with op bits have the push_null bit as the last one. + # Only set the bit if it hasn't been set - otherwise, we need + # to add another PUSH_NULL. + set_inst_bit(-1) + else: + insts = insts + [create_instruction("PUSH_NULL")] + elif sys.version_info >= (3, 12): + # LOAD_ATTR/LOAD_SUPER_ATTR at the end + # We assume that `insts` will only load 1 object, so + # LOAD_GLOBAL at the end doesn't need to be checked + if inst_has_op_bits(insts[-1].opname) and not inst_has_bit_set(-1): + set_inst_bit(-1) + elif insts[0].opname == "LOAD_GLOBAL" and not inst_has_bit_set(0): + set_inst_bit(0) + else: + insts = [create_instruction("PUSH_NULL")] + insts + elif sys.version_info >= (3, 11): + # 3.11 introduced NULL preceding callable + if inst_has_op_bits(insts[0].opname) and not inst_has_bit_set(0): + set_inst_bit(0) + else: + insts = [create_instruction("PUSH_NULL")] + insts + return insts + + +def add_push_null_call_function_ex( + inst_or_insts: Union[Instruction, list[Instruction]], +) -> list[Instruction]: + """Like add_push_null, but the low bit of LOAD_ATTR/LOAD_SUPER_ATTR + is not set, due to an expected CALL_FUNCTION_EX instruction. + """ + if isinstance(inst_or_insts, Instruction): + insts: list[Instruction] = [inst_or_insts] + else: + assert isinstance(inst_or_insts, list) + insts = inst_or_insts + + if sys.version_info < (3, 11): + return insts + + idx = -1 if sys.version_info >= (3, 13) else 0 + if insts[idx].opname == "LOAD_GLOBAL": + assert insts[idx].arg is not None + if insts[idx].arg & 1 == 0: # type: ignore[operator] + insts[idx].arg |= 1 # type: ignore[operator] + return insts + + if sys.version_info >= (3, 13): + insts = insts + [create_instruction("PUSH_NULL")] + else: + insts = [create_instruction("PUSH_NULL")] + insts + + return insts + + +def create_call_function(nargs: int, push_null: bool) -> list[Instruction]: + """ + Creates a sequence of instructions that makes a function call. + + `push_null` is used in Python 3.11+ only. It is used in codegen when + a function call is intended to be made with the NULL + fn convention, + and we know that the NULL has not been pushed yet. We will push a + NULL and rotate it to the correct position immediately before making + the function call. + + `push_null` should be True if no NULL is pushed for the callable. + Conversely, `push_null` should be False if a NULL was pushed for the callable. + Prefer using `push_null=False` when possible since we will not need to rotate + NULL to the right place, which is less efficient. + + Generally, you should codegen a function by using `add_push_null` then + `create_call_function` with `push_null=False`. + + Example of when to set push_null False: + + insts = [ + create_instruction("LOAD_GLOBAL", argval="torch"), + create_instruction("LOAD_ATTR", argval="nn"), + create_instruction("LOAD_ATTR", argval="functional"), + create_instruction("LOAD_ATTR", argval="relu"), + ] + insts = add_push_null(insts) + insts.append(create_instruction("LOAD_FAST", argval="x")) + insts.extend(create_call_function(1, False)) + + Example of when to set push_null True: + + insts = [create_instruction("LOAD_FAST", x)] + for should_wrap, wrapper_name in wrappers: + if should_wrap: + insts.extend([ + create_instruction("LOAD_GLOBAL", argval="wrapper1"), + create_instruction("SWAP", arg=2), + *create_call_function(1, True), + ) + """ + if sys.version_info >= (3, 11): + output = [] + if push_null: + output.append(create_instruction("PUSH_NULL")) + # 3.13 swapped NULL and callable + rots = nargs + 1 if sys.version_info >= (3, 13) else nargs + 2 + output.extend(create_rot_n(rots)) + if sys.version_info < (3, 12): + output.append(create_instruction("PRECALL", arg=nargs)) + output.append(create_instruction("CALL", arg=nargs)) + return output + return [create_instruction("CALL_FUNCTION", arg=nargs)] + + +def create_call_function_ex( + has_kwargs: bool, push_null: bool, ignore_314_kwargs_push: bool = False +) -> list[Instruction]: + """ + Assumes that in 3.14+, if has_kwargs=False, there is NOT a NULL + on the TOS for the kwargs. This utility function will add a PUSH_NULL. + + If the caller has already pushed a NULL for the kwargs, then set ignore_314_kwargs_push=True + so we don't push another NULL for the kwargs. + """ + if sys.version_info >= (3, 11): + output = [] + if ( + sys.version_info >= (3, 14) + and not has_kwargs + and not ignore_314_kwargs_push + ): + output.append(create_instruction("PUSH_NULL")) + has_kwargs = True + if push_null: + output.append(create_instruction("PUSH_NULL")) + # 3.13 swapped NULL and callable + # if flags == 1, 2 values popped - otherwise if flags == 0, 1 value + rots = ( + int(has_kwargs) + 2 + if sys.version_info >= (3, 13) + else int(has_kwargs) + 3 + ) + output.extend(create_rot_n(rots)) + output.append(create_instruction("CALL_FUNCTION_EX", arg=int(has_kwargs))) + return output + return [create_instruction("CALL_FUNCTION_EX", arg=int(has_kwargs))] + + +def create_call_method(nargs: int) -> list[Instruction]: + if sys.version_info >= (3, 12): + return [create_instruction("CALL", arg=nargs)] + if sys.version_info >= (3, 11): + return [ + create_instruction("PRECALL", arg=nargs), + create_instruction("CALL", arg=nargs), + ] + return [create_instruction("CALL_METHOD", arg=nargs)] + + +def create_load_method(name: str) -> Instruction: + if sys.version_info >= (3, 12): + # in 3.12, create a LOAD_ATTR instruction with the low bit set + return create_instruction("LOAD_ATTR", arg=1, argval=name) + return create_instruction("LOAD_METHOD", argval=name) + + +def create_setup_with(target: Instruction) -> Instruction: + opname = "BEFORE_WITH" if sys.version_info >= (3, 11) else "SETUP_WITH" + return create_instruction(opname, target=target) + + +def create_swap(n: int) -> list[Instruction]: + if sys.version_info >= (3, 11): + return [create_instruction("SWAP", arg=n)] + # in Python < 3.11, SWAP is a macro that expands to multiple instructions + if n == 1: + return [] + elif n == 2: + return [create_instruction("ROT_TWO")] + elif n == 3: + return [create_instruction("ROT_THREE"), create_instruction("ROT_TWO")] + """ + e.g. swap "a" and "b" in this stack: + 0 a 1 2 3 b + 0 a [1 2 3 b] + 0 a [1 2 3 b] [1 2 3 b] + 0 a [1 2 3 b] [1 2 3 b] -1 + 0 a [1 2 3 b] b + 0 b a [1 2 3 b] + 0 b a [1 2 3 b] [1 2 3 b] + 0 b [1 2 3 b] a [1 2 3 b] + 0 b [1 2 3 b] a [1 2 3 b] -1 + 0 b [1 2 3 a] + 0 b [1 2 3 a] [1 2 3 a] + 0 b [1 2 3 a] [1 2 3 a] reverse + 0 b [a 3 2 1] None + 0 b [a 3 2 1] + 0 b 1 2 3 a + """ + return [ + create_instruction("BUILD_LIST", arg=n - 1), + create_instruction("DUP_TOP"), + create_instruction("LOAD_CONST", argval=-1), + create_binary_subscr(), + create_instruction("ROT_THREE"), + create_instruction("DUP_TOP"), + create_instruction("ROT_THREE"), + create_instruction("LOAD_CONST", argval=-1), + create_instruction("STORE_SUBSCR"), + create_instruction("DUP_TOP"), + create_load_method("reverse"), + *create_call_method(0), + create_instruction("POP_TOP"), + create_instruction("UNPACK_SEQUENCE", arg=n - 1), + ] + + +def create_binary_slice( + start: Optional[int], end: Optional[int], store: bool = False +) -> list[Instruction]: + """ + BINARY_SLICE and STORE_SLICE (if `set` is True) for all Python versions + """ + if sys.version_info >= (3, 14): + subscr_inst = ( + create_instruction("STORE_SUBSCR") if store else create_binary_subscr() + ) + return [ + create_load_const(slice(start, end)), + subscr_inst, + ] + elif sys.version_info >= (3, 12): + inst_name = "STORE_SLICE" if store else "BINARY_SLICE" + return [ + create_load_const(start), + create_load_const(end), + create_instruction(inst_name), + ] + else: + inst_name = "STORE_SUBSCR" if store else "BINARY_SUBSCR" + return [ + create_load_const(start), + create_load_const(end), + create_instruction("BUILD_SLICE", arg=2), + create_instruction(inst_name), + ] + + +def create_copy(i: int) -> list[Instruction]: + if sys.version_info >= (3, 11): + return [create_instruction("COPY", arg=i)] + if i == 1: + return [create_instruction("DUP_TOP")] + # COPY 4 + # 0 1 2 3 + # 3 1 2 0 + # 3 1 2 0 0 + # 0 1 2 0 3 + # 0 1 2 3 0 + return [ + *create_swap(i), + create_dup_top(), + *create_swap(i + 1), + *create_swap(2), + ] + + +# mainly for debugging generated bytecode +def create_print_on_stack(depth: int) -> list[Instruction]: + return [ + *add_push_null(create_instruction("LOAD_CONST", argval=print)), + *create_copy(depth + (2 if sys.version_info >= (3, 11) else 1)), + *create_call_function(1, False), + create_instruction("POP_TOP"), + ] + + +# mainly for debugging generated bytecode +def create_print_value(value: Any) -> list[Instruction]: + return [ + *add_push_null(create_instruction("LOAD_CONST", argval=print)), + create_instruction("LOAD_CONST", argval=value), + *create_call_function(1, False), + create_instruction("POP_TOP"), + ] + + +def create_binary_subscr() -> Instruction: + if sys.version_info < (3, 14): + return create_instruction("BINARY_SUBSCR") + # https://github.com/python/cpython/blob/0e46c0499413bc5f9f8336fe76e2e67cf93f64d8/Include/opcode.h#L36 + return create_instruction("BINARY_OP", arg=26) + + +def create_build_tuple(n: int) -> Instruction: + if sys.version_info >= (3, 14) and n == 0: + return create_load_const(()) + return create_instruction("BUILD_TUPLE", arg=n) + + +def linetable_writer( + first_lineno: int, +) -> tuple[list[int], Callable[[int, int], None], Callable[[int], None]]: + """ + Used to create typing.CodeType.co_linetable + See https://github.com/python/cpython/blob/main/Objects/lnotab_notes.txt + This is the internal format of the line number table for Python 3.10 + """ + assert sys.version_info[:2] == (3, 10) + linetable: list[int] = [] + lineno = first_lineno + lineno_delta = 0 + byteno = 0 + + def _update(byteno_delta: int, lineno_delta: int) -> None: + while byteno_delta != 0 or lineno_delta != 0: + byte_offset = max(0, min(byteno_delta, 254)) + line_offset = max(-127, min(lineno_delta, 127)) + assert byte_offset != 0 or line_offset != 0 + byteno_delta -= byte_offset + lineno_delta -= line_offset + linetable.extend((byte_offset, line_offset & 0xFF)) + + def update(lineno_new: int, byteno_new: int) -> None: + nonlocal lineno, lineno_delta, byteno + byteno_delta = byteno_new - byteno + byteno = byteno_new + _update(byteno_delta, lineno_delta) + lineno_delta = lineno_new - lineno + lineno = lineno_new + + def end(total_bytes: int) -> None: + _update(total_bytes - byteno, lineno_delta) + + return linetable, update, end + + +def encode_varint(n: int) -> list[int]: + """ + 6-bit chunk encoding of an unsigned integer + See https://github.com/python/cpython/blob/3.11/Objects/locations.md + """ + assert n >= 0 + b = [n & 63] + n >>= 6 + while n > 0: + b[-1] |= 64 + b.append(n & 63) + n >>= 6 + return b + + +def linetable_311_writer( + first_lineno: int, +) -> tuple[list[int], Callable[[Optional["dis.Positions"], int], None]]: + """ + Used to create typing.CodeType.co_linetable + See https://github.com/python/cpython/blob/3.11/Objects/locations.md + This is the internal format of the line number table for Python 3.11 + """ + assert sys.version_info >= (3, 11) + linetable = [] + lineno = first_lineno + + def update(positions: Optional["dis.Positions"], inst_size: int) -> None: + nonlocal lineno + lineno_new = positions.lineno if positions else None + + def _update(delta: int, size: int) -> None: + assert 0 < size <= 8 + # first byte - use 13 (no column info) is positions is + # malformed, otherwise use 14 (long form) + other_varints: tuple[int, ...] = () + if ( + positions + and positions.lineno is not None + and positions.end_lineno is not None + and positions.col_offset is not None + and positions.end_col_offset is not None + ): + linetable.append(0b1_1110_000 + size - 1) + # for whatever reason, column offset needs `+ 1` + # https://github.com/python/cpython/blob/1931c2a438c50e6250725c84dff94fc760b9b951/Python/compile.c#L7603 + other_varints = ( + positions.end_lineno - positions.lineno, + positions.col_offset + 1, + positions.end_col_offset + 1, + ) + else: + linetable.append(0b1_1101_000 + size - 1) + # encode signed int + if delta < 0: + delta = ((-delta) << 1) | 1 + else: + delta <<= 1 + # encode unsigned int + linetable.extend(encode_varint(delta)) + for n in other_varints: + linetable.extend(encode_varint(n)) + + if lineno_new is None: + lineno_delta = 0 + else: + lineno_delta = lineno_new - lineno + lineno = lineno_new + while inst_size > 8: + _update(lineno_delta, 8) + inst_size -= 8 + _update(lineno_delta, inst_size) + + return linetable, update + + +@dataclasses.dataclass(slots=True) +class ExceptionTableEntry: + start: int + end: int + target: int + depth: int + lasti: bool + + +def encode_exception_table_varint(n: int) -> list[int]: + """ + Similar to `encode_varint`, but the 6-bit chunks are ordered in reverse. + """ + assert n >= 0 + b = [n & 63] + n >>= 6 + while n > 0: + b.append(n & 63) + n >>= 6 + b.reverse() + for i in range(len(b) - 1): + b[i] |= 64 + return b + + +def decode_exception_table_varint(bytes_iter: Iterator[int]) -> int: + """ + Inverse of `encode_exception_table_varint`. + """ + b = next(bytes_iter) + val = b & 63 + while b & 64: + val <<= 6 + b = next(bytes_iter) + val |= b & 63 + return val + + +def check_exception_table(tab: list[ExceptionTableEntry]) -> None: + """ + Verifies that a list of ExceptionTableEntries will make a well-formed + jump table: entries are non-empty, sorted, and do not overlap. + """ + for i in range(len(tab) - 1): + assert ( + tab[i].start <= tab[i].end + and tab[i].end < tab[i + 1].start + and tab[i + 1].start <= tab[i + 1].end + ) + + +def parse_exception_table(exntab: bytes) -> list[ExceptionTableEntry]: + """ + Parse the exception table according to + https://github.com/python/cpython/blob/3.11/Objects/exception_handling_notes.txt + """ + exntab_iter = iter(exntab) + tab = [] + try: + while True: + start = decode_exception_table_varint(exntab_iter) * 2 + length = decode_exception_table_varint(exntab_iter) * 2 + end = start + length - 2 + target = decode_exception_table_varint(exntab_iter) * 2 + dl = decode_exception_table_varint(exntab_iter) + depth = dl >> 1 + lasti = bool(dl & 1) + tab.append(ExceptionTableEntry(start, end, target, depth, lasti)) + except StopIteration: + check_exception_table(tab) + return tab + + +def assemble_exception_table(tab: list[ExceptionTableEntry]) -> bytes: + """ + Inverse of parse_exception_table - encodes list of exception + table entries into bytes. + """ + b = [] + for entry in tab: + first_entry = encode_exception_table_varint(entry.start // 2) + first_entry[0] |= 1 << 7 + b.extend(first_entry) + length = entry.end - entry.start + 2 + b.extend(encode_exception_table_varint(length // 2)) + b.extend(encode_exception_table_varint(entry.target // 2)) + dl = (entry.depth << 1) + entry.lasti + b.extend(encode_exception_table_varint(dl)) + return bytes(b) + + +def assemble(instructions: list[Instruction], firstlineno: int) -> tuple[bytes, bytes]: + """Do the opposite of dis.get_instructions()""" + code: list[int] = [] + if sys.version_info >= (3, 11): + lnotab, update_lineno = linetable_311_writer(firstlineno) + num_ext = 0 + for i, inst in enumerate(instructions): + if inst.opname == "EXTENDED_ARG": + inst_size = 1 + num_ext += 1 + # copy positions from the actual instruction + for j in (1, 2, 3): + if instructions[i + j].opname != "EXTENDED_ARG": + inst.positions = instructions[i + j].positions + break + else: + inst_size = instruction_size(inst) // 2 + num_ext + num_ext = 0 + update_lineno(inst.positions, inst_size) + num_ext = 0 + arg = inst.arg or 0 + code.extend((inst.opcode, arg & 0xFF)) + for _ in range(instruction_size(inst) // 2 - 1): + code.extend((0, 0)) + else: + lnotab, update_lineno, end = linetable_writer(firstlineno) + + for inst in instructions: + if inst.starts_line is not None: + update_lineno(inst.starts_line, len(code)) + arg = inst.arg or 0 + code.extend((inst.opcode, arg & 0xFF)) + + end(len(code)) + + return bytes(code), bytes(lnotab) + + +def _get_instruction_by_offset( + offset_to_inst: dict[int, Instruction], offset: int +) -> Optional[Instruction]: + """ + Get the instruction located at a given offset, accounting for EXTENDED_ARGs + """ + for n in (0, 2, 4, 6): + if offset_to_inst[offset + n].opcode != dis.EXTENDED_ARG: + return offset_to_inst[offset + n] + return None + + +def virtualize_jumps(instructions: Iterable[Instruction]) -> None: + """Replace jump targets with pointers to make editing easier""" + jump_targets = { + inst.offset: inst for inst in instructions if inst.offset is not None + } + + for inst in instructions: + if inst.opcode in dis.hasjabs or inst.opcode in dis.hasjrel: + inst.target = _get_instruction_by_offset(jump_targets, inst.argval) + + +_REL_JUMPS = set(dis.hasjrel) + + +def flip_jump_direction(instruction: Instruction) -> None: + if sys.version_info < (3, 11): + raise RuntimeError("Cannot flip jump direction in Python < 3.11") + if "FORWARD" in instruction.opname: + instruction.opname = instruction.opname.replace("FORWARD", "BACKWARD") + elif "BACKWARD" in instruction.opname: + instruction.opname = instruction.opname.replace("BACKWARD", "FORWARD") + else: + raise AttributeError("Instruction is not a forward or backward jump") + instruction.opcode = dis.opmap[instruction.opname] + assert instruction.opcode in _REL_JUMPS + + +def _get_instruction_front(instructions: list[Instruction], idx: int) -> Instruction: + """ + i.e. get the first EXTENDED_ARG instruction (if any) when targeting + instructions[idx] with a jump. + """ + target = instructions[idx] + for offset in (1, 2, 3): + if idx >= offset and instructions[idx - offset].opcode == dis.EXTENDED_ARG: + target = instructions[idx - offset] + else: + break + return target + + +def devirtualize_jumps(instructions: list[Instruction]) -> None: + """Fill in args for virtualized jump target after instructions may have moved""" + jumps = set(dis.hasjabs).union(set(dis.hasjrel)) + + # check for negative jump args and fix them + for inst in instructions: + if inst.opcode in jumps: + if inst.opcode not in dis.hasjabs: + assert ( + inst.target is not None + and inst.target.offset is not None + and inst.offset is not None + ) + if inst.target.offset < inst.offset: + if sys.version_info < (3, 11): + raise RuntimeError("Got negative jump offset for Python < 3.11") + # forward jumps become backward + if "FORWARD" in inst.opname: + flip_jump_direction(inst) + else: + # backward jumps become forward + if sys.version_info >= (3, 11) and "BACKWARD" in inst.opname: + flip_jump_direction(inst) + + # jump instruction size may have changed due to flips + update_offsets(instructions) + indexof = get_indexof(instructions) + + # compute jump instruction arg + for inst in instructions: + if inst.opcode in jumps: + assert inst.target is not None + target = _get_instruction_front(instructions, indexof[inst.target]) + if inst.opcode in dis.hasjabs: + if sys.version_info < (3, 11): + # `arg` is expected to be bytecode offset, whereas `offset` is byte offset. + # Divide since bytecode is 2 bytes large. + inst.arg = int(target.offset / 2) + else: + raise RuntimeError("Python 3.11+ should not have absolute jumps") + else: # relative jump + # byte offset between target and next instruction + assert target.offset is not None and inst.offset is not None + inst.arg = abs( + int(target.offset - inst.offset - instruction_size(inst)) + ) + # pyrefly: ignore [unsupported-operation] + inst.arg //= 2 + inst.argval = target.offset + inst.argrepr = f"to {target.offset}" + + +def virtualize_exception_table( + exn_tab_bytes: bytes, instructions: list[Instruction] +) -> None: + """Replace exception table entries with pointers to make editing easier""" + exn_tab = parse_exception_table(exn_tab_bytes) + offset_to_inst = {cast(int, inst.offset): inst for inst in instructions} + offsets = sorted(offset_to_inst.keys()) + end_offset_idx = 0 + exn_tab_iter = iter(exn_tab) + try: + + def step() -> tuple[ExceptionTableEntry, InstructionExnTabEntry]: + nonlocal end_offset_idx + entry = next(exn_tab_iter) + # find rightmost offset <= entry.end, since entry.end may not be + # an actual instruction, e.g. if the end instruction is LOAD_GLOBAL, + # which takes more than 2 bytes, then entry.end points to the end + # of the LOAD_GLOBAL instruction, not the beginning. + while ( + end_offset_idx < len(offsets) and offsets[end_offset_idx] <= entry.end + ): + end_offset_idx += 1 + assert end_offset_idx > 0 + end_offset = offsets[end_offset_idx - 1] + inst_entry = InstructionExnTabEntry( + _get_instruction_by_offset(offset_to_inst, entry.start), # type: ignore[arg-type] + _get_instruction_by_offset(offset_to_inst, end_offset), # type: ignore[arg-type] + _get_instruction_by_offset(offset_to_inst, entry.target), # type: ignore[arg-type] + entry.depth, + entry.lasti, + ) + return entry, inst_entry + + entry, inst_entry = step() + for inst in instructions: + assert inst.offset is not None + while inst.offset > entry.end: + entry, inst_entry = step() + if inst.offset >= entry.start: + inst.exn_tab_entry = copy.copy(inst_entry) + except StopIteration: + pass + + +def compute_exception_table( + instructions: list[Instruction], +) -> list[ExceptionTableEntry]: + """Compute exception table in list format from instructions with exn_tab_entries""" + exn_dict: dict[tuple[int, int], tuple[int, int, bool]] = {} + indexof = get_indexof(instructions) + + for inst in instructions: + if inst.exn_tab_entry: + # account for prefixed EXTENDED_ARGS + start = _get_instruction_front( + instructions, indexof[inst.exn_tab_entry.start] + ).offset + assert start is not None + # point to the last 2 bytes of the end instruction + end = ( + cast(int, inst.exn_tab_entry.end.offset) + + instruction_size(inst.exn_tab_entry.end) + - 2 + ) + assert end is not None + target = _get_instruction_front( + instructions, indexof[inst.exn_tab_entry.target] + ).offset + assert target is not None + key = (start, end) + val = (target, inst.exn_tab_entry.depth, inst.exn_tab_entry.lasti) + if key in exn_dict: + assert exn_dict[key] == val + exn_dict[key] = val + + # Dynamo may construct nested exception table entries for convenience, + # but Python expects exception table entries to not overlap. + # NOTE: below, "keys" refer to old instruction entries' starts and ends, + # and "entries" refer to the generated exception table entries. + + # Sort keys by increasing start, then decreasing end + keys_sorted = sorted(exn_dict.keys(), key=lambda t: (t[0], -t[1])) + # smallest byte that the next exception table entry can start at + nexti = 0 + # stack of current nested keys + key_stack: list[tuple[int, int]] = [] + exn_tab: list[ExceptionTableEntry] = [] + + def pop() -> None: + """ + Pop the key_stack and append an exception table entry if possible. + """ + nonlocal nexti + if key_stack: + key = key_stack.pop() + if nexti <= key[1]: + exn_tab.append( + ExceptionTableEntry(max(key[0], nexti), key[1], *exn_dict[key]) + ) + nexti = key[1] + 2 + + for key in keys_sorted: + # pop keys that are no longer nested over the current key + while key_stack and key_stack[-1][1] < key[0]: + pop() + if key_stack: + # create an entry covering to the current key, if possible + assert key_stack[-1][0] <= key[0] <= key[1] <= key_stack[-1][1] + left = max(nexti, key_stack[-1][0]) + if left < key[0]: + exn_tab.append( + ExceptionTableEntry(left, key[0] - 2, *exn_dict[key_stack[-1]]) + ) + nexti = key[0] + key_stack.append(key) + while key_stack: + pop() + check_exception_table(exn_tab) + return exn_tab + + +def check_inst_exn_tab_entries_nested( + tab: list[InstructionExnTabEntry], indexof: dict[Instruction, int] +) -> None: + """ + Checks `tab` is a properly sorted list of nested InstructionExnTabEntry's, + i.e. no entries partially overlap. + "Properly sorted" means entries are sorted by increasing starts, then + decreasing ends. + """ + entry_stack: list[tuple[int, int]] = [] + for entry in tab: + key = (indexof[entry.start], indexof[entry.end]) + while entry_stack and entry_stack[-1][1] < key[0]: + entry_stack.pop() + if entry_stack: + assert entry_stack[-1][0] <= key[0] <= key[1] <= entry_stack[-1][1] + entry_stack.append(key) + + +def propagate_inst_exn_table_entries(instructions: list[Instruction]) -> None: + """ + Copies exception table entries to all instructions in an entry's range. + Supports nested exception table entries. + """ + indexof = get_indexof(instructions) + entries: dict[tuple[int, int], InstructionExnTabEntry] = {} + for inst in instructions: + if inst.exn_tab_entry: + key = ( + indexof[inst.exn_tab_entry.start], + indexof[inst.exn_tab_entry.end], + ) + if key in entries: + assert inst.exn_tab_entry == entries[key] + entries[key] = inst.exn_tab_entry + sorted_entries = [ + entries[key] for key in sorted(entries.keys(), key=lambda t: (t[0], -t[1])) + ] + check_inst_exn_tab_entries_nested(sorted_entries, indexof) + # Propagation of nested entries works since nested entries come later + # in sorted order. + for entry in sorted_entries: + for i in range(indexof[entry.start], indexof[entry.end] + 1): + instructions[i].exn_tab_entry = copy.copy(entry) + + +def check_inst_exn_tab_entries_valid(instructions: list[Instruction]) -> None: + """ + Checks that exn_tab_entries of instructions are valid. + An entry's start, end, and target must be in instructions. + Instructions with an exn_tab_entry are located within + the entry's start and end instructions. + Instructions do not share exn_tab_entries. + + Implicitly checks for no duplicate instructions. + """ + indexof = get_indexof(instructions) + exn_tab_entry_set = set() + for i, inst in enumerate(instructions): + if inst.exn_tab_entry: + assert sys.version_info >= (3, 11) + assert id(inst.exn_tab_entry) not in exn_tab_entry_set + exn_tab_entry_set.add(id(inst.exn_tab_entry)) + entry = inst.exn_tab_entry + assert entry.start in indexof + assert entry.end in indexof + assert entry.target in indexof + assert indexof[entry.start] <= i <= indexof[entry.end] + + +def strip_extended_args(instructions: list[Instruction]) -> None: + instructions[:] = [i for i in instructions if i.opcode != dis.EXTENDED_ARG] + + +# Overwrites old_inst with a sequence of new instructions. +# This is necessary in order to preserve jump targets to the old +# instruction, exception table entries, and positions. +# Returns the modified sequence of instructions (including the modified +# old instruction!) that can be manipulated elsewhere. +def overwrite_instruction( + old_inst: Instruction, new_insts: list[Instruction] +) -> list[Instruction]: + # update old_inst.exnt_tab_entry.end if necessary + if ( + old_inst.exn_tab_entry + and old_inst.exn_tab_entry.end is old_inst + and len(new_insts) > 1 + ): + old_inst.exn_tab_entry.end = new_insts[-1] + # preserve exception table entries and positions + for inst in new_insts[1:]: + inst.exn_tab_entry = copy.copy(old_inst.exn_tab_entry) + inst.positions = old_inst.positions + # modify old_inst in-place to preserve jump target + old_inst.opcode = new_insts[0].opcode + old_inst.opname = new_insts[0].opname + old_inst.arg = new_insts[0].arg + old_inst.argval = new_insts[0].argval + old_inst.target = new_insts[0].target + return [old_inst] + new_insts[1:] + + +def remove_load_call_method(instructions: list[Instruction]) -> list[Instruction]: + """LOAD_METHOD puts a NULL on the stack which causes issues, so remove it""" + assert sys.version_info < (3, 11) + rewrites = {"LOAD_METHOD": "LOAD_ATTR", "CALL_METHOD": "CALL_FUNCTION"} + for inst in instructions: + if inst.opname in rewrites: + inst.opname = rewrites[inst.opname] + inst.opcode = dis.opmap[inst.opname] + return instructions + + +def remove_jump_if_none(instructions: list[Instruction]) -> None: + new_insts = [] + for inst in instructions: + if "_NONE" in inst.opname: + is_op = create_instruction("IS_OP", arg=int("NOT" in inst.opname)) + # need both argval and arg set correctly now (not later) + is_op.argval = is_op.arg + + if sys.version_info < (3, 12): + jump_op = create_instruction( + ( + "POP_JUMP_FORWARD_IF_TRUE" + if "FORWARD" in inst.opname + else "POP_JUMP_BACKWARD_IF_TRUE" + ), + target=inst.target, + ) + else: + jump_op = create_instruction("POP_JUMP_IF_TRUE", target=inst.target) + + replace_insts = [ + create_instruction("LOAD_CONST", argval=None), + is_op, + jump_op, + ] + new_insts.extend(overwrite_instruction(inst, replace_insts)) + else: + new_insts.append(inst) + instructions[:] = new_insts + + +def remove_binary_store_slice(instructions: list[Instruction]) -> None: + new_insts = [] + for inst in instructions: + new_insts.append(inst) + if inst.opname in ("BINARY_SLICE", "STORE_SLICE"): + # new instruction + if sys.version_info >= (3, 14) and inst.opname == "BINARY_SLICE": + subscr_inst = create_binary_subscr() + else: + subscr_inst = create_instruction(inst.opname.replace("SLICE", "SUBSCR")) + if inst.exn_tab_entry and inst.exn_tab_entry.end is inst: + inst.exn_tab_entry.end = subscr_inst + subscr_inst.exn_tab_entry = copy.copy(inst.exn_tab_entry) + subscr_inst.positions = inst.positions + # modify inst in-place to preserve jump target + inst.opcode = dis.opmap["BUILD_SLICE"] + inst.opname = "BUILD_SLICE" + inst.arg = 2 + inst.argval = 2 + new_insts.append(subscr_inst) + instructions[:] = new_insts + + +FUSED_INSTS = { + "LOAD_FAST_LOAD_FAST": ("LOAD_FAST", "LOAD_FAST"), + "LOAD_FAST_BORROW_LOAD_FAST_BORROW": ("LOAD_FAST_BORROW", "LOAD_FAST_BORROW"), + "STORE_FAST_STORE_FAST": ("STORE_FAST", "STORE_FAST"), + "STORE_FAST_LOAD_FAST": ("STORE_FAST", "LOAD_FAST"), +} + + +def remove_fused_load_store(instructions: list[Instruction]) -> None: + new_insts = [] + for inst in instructions: + if inst.opname in FUSED_INSTS: + inst0, inst1 = FUSED_INSTS[inst.opname] + argval0, argval1 = inst.argval + + replace_insts = [ + create_instruction(inst0, argval=argval0), + create_instruction(inst1, argval=argval1), + ] + new_insts.extend(overwrite_instruction(inst, replace_insts)) + else: + new_insts.append(inst) + instructions[:] = new_insts + + +# adds GRAPH_BREAK_IF_LEAF (not a real instruction) before RETURN_* instructions +# for testing purposes +def add_graph_break_if_leaf_instructions(instructions: list[Instruction]) -> None: + new_insts = [] + for inst in instructions: + if "RETURN" in inst.opname: + replace_insts = [ + create_instruction("NOP", argval="GRAPH_BREAK_IF_LEAF"), + create_instruction(inst.opname, argval=inst.argval), + ] + new_insts.extend(overwrite_instruction(inst, replace_insts)) + else: + new_insts.append(inst) + instructions[:] = new_insts + + +def remove_graph_break_if_leaf_instructions(instructions: list[Instruction]) -> None: + new_insts = [] + for inst, next_inst in itertools.pairwise(instructions): + if ( + inst.opname == "NOP" + and inst.argval == "GRAPH_BREAK_IF_LEAF" + and next_inst.opname.startswith("RETURN") + ): + # remove this instruction and update all other instructions' jump targets + for i in range(len(instructions)): + if instructions[i].target is inst: + instructions[i].target = next_inst + if instructions[i].exn_tab_entry: + # linter is mistakenly complaining that None has no attribute "..." + # but this codepath only runs if instructions[i] is not None + if instructions[i].exn_tab_entry.start is inst: # type: ignore[union-attr] + instructions[i].exn_tab_entry.start = next_inst # type: ignore[union-attr] + if instructions[i].exn_tab_entry.end is inst: # type: ignore[union-attr] + instructions[i].exn_tab_entry.end = next_inst # type: ignore[union-attr] + if instructions[i].exn_tab_entry.target is inst: # type: ignore[union-attr] + instructions[i].exn_tab_entry.target = next_inst # type: ignore[union-attr] + else: + new_insts.append(inst) + new_insts.append(instructions[-1]) + instructions[:] = new_insts + + +def explicit_super(code: types.CodeType, instructions: list[Instruction]) -> None: + """convert super() with no args into explicit arg form""" + cell_and_free = (code.co_cellvars or ()) + (code.co_freevars or ()) + if not len(code.co_varnames): + # A function with no argument cannot contain a valid "super()" call + return + output = [] + for idx, inst in enumerate(instructions): + output.append(inst) + if inst.opname == "LOAD_GLOBAL" and inst.argval == "super": + nexti = instructions[idx + 1] + if nexti.arg == 0 and ( + (sys.version_info >= (3, 12) and nexti.opname == "CALL") + or ( + sys.version_info >= (3, 11) + and sys.version_info < (3, 12) + and nexti.opname == "PRECALL" + ) + or (sys.version_info < (3, 11) and nexti.opname == "CALL_FUNCTION") + ): + assert "__class__" in cell_and_free + output.append(create_instruction("LOAD_DEREF", argval="__class__")) + first_var = code.co_varnames[0] + if first_var in cell_and_free: + output.append(create_instruction("LOAD_DEREF", argval=first_var)) + else: + output.append(create_instruction("LOAD_FAST", argval=first_var)) + nexti.arg = 2 + nexti.argval = 2 + if nexti.opname == "PRECALL": + # also update the following CALL instruction + call_inst = instructions[idx + 2] + call_inst.arg = 2 + call_inst.argval = 2 + + instructions[:] = output + + +def fix_extended_args(instructions: list[Instruction]) -> int: + """Fill in correct argvals for EXTENDED_ARG ops""" + output: list[Instruction] = [] + + def maybe_pop_n(n: int) -> None: + for _ in range(n): + if output and output[-1].opcode == dis.EXTENDED_ARG: + output.pop() + + for inst in instructions: + if inst.opcode == dis.EXTENDED_ARG: + # Leave this instruction alone for now so we never shrink code + inst.arg = 0 + elif inst.arg and inst.arg > 0xFFFFFF: + maybe_pop_n(3) + output.append(create_instruction("EXTENDED_ARG", arg=inst.arg >> 24)) + output.append(create_instruction("EXTENDED_ARG", arg=inst.arg >> 16)) + output.append(create_instruction("EXTENDED_ARG", arg=inst.arg >> 8)) + elif inst.arg and inst.arg > 0xFFFF: + maybe_pop_n(2) + output.append(create_instruction("EXTENDED_ARG", arg=inst.arg >> 16)) + output.append(create_instruction("EXTENDED_ARG", arg=inst.arg >> 8)) + elif inst.arg and inst.arg > 0xFF: + maybe_pop_n(1) + output.append(create_instruction("EXTENDED_ARG", arg=inst.arg >> 8)) + output.append(inst) + + added = len(output) - len(instructions) + assert added >= 0 + instructions[:] = output + return added + + +def instruction_size(inst: Instruction) -> int: + import torch + + if sys.version_info >= (3, 11): + return 2 * (torch._C._dynamo.eval_frame.py_opcode_caches[inst.opcode] + 1) + return 2 + + +def check_offsets(instructions: Sequence[Instruction]) -> None: + offset = 0 + for inst in instructions: + assert inst.offset == offset + offset += instruction_size(inst) + + +def update_offsets(instructions: Sequence[Instruction]) -> None: + offset = 0 + for inst in instructions: + inst.offset = offset + # pyrefly: ignore [unsupported-operation] + offset += instruction_size(inst) + + +def debug_bytes(*args: bytes) -> str: + index = range(max(map(len, args))) + result = [ + " ".join(f"{x:03}" for x in arg) + for arg in [index] + + list(args) + + [[int(a != b) for a, b in zip(args[-1], args[-2])]] + ] + + return "bytes mismatch\n" + "\n".join(result) + + +def debug_checks(code: types.CodeType) -> None: + """Make sure our assembler produces same bytes as we start with""" + dode, _ = transform_code_object(code, lambda x, y: None, safe=True) + assert code.co_code == dode.co_code, debug_bytes(code.co_code, dode.co_code) + assert code.co_lnotab == dode.co_lnotab, debug_bytes(code.co_lnotab, dode.co_lnotab) + + +HAS_LOCAL = set(dis.haslocal) +HAS_NAME = set(dis.hasname) +HAS_FREE = set(dis.hasfree) +HAS_CONST = set(dis.hasconst) + + +def get_const_index(code_options: dict[str, Any], val: Any) -> int: + for i, v in enumerate(code_options["co_consts"]): + # NOTE: stronger comparison is required, since we have + # examples where two values compare equal but have + # different semantic meaning in some cases, e.g. + # 0.0 == -0.0 but have different effects in torch.copysign. + if val is v: + return i + code_options["co_consts"] += (val,) + return len(code_options["co_consts"]) - 1 + + +def fix_vars( + instructions: list[Instruction], + code_options: dict[str, Any], + varname_from_oparg: Optional[Callable[..., Any]] = None, +) -> None: + # compute instruction arg from argval if arg is not provided + names = {name: idx for idx, name in enumerate(code_options["co_names"])} + + def get_name_index(name: str) -> int: + try: + idx = names[name] + except KeyError: + # Add a missing item to co_names + idx = names[name] = len(names) + code_options["co_names"] = (*code_options["co_names"], name) + assert len(code_options["co_names"]) == len(names) + return idx + + if sys.version_info < (3, 11): + assert varname_from_oparg is None + varnames = {name: idx for idx, name in enumerate(code_options["co_varnames"])} + freenames = { + name: idx + for idx, name in enumerate( + code_options["co_cellvars"] + code_options["co_freevars"] + ) + } + else: + assert callable(varname_from_oparg) + allnames = {} + for idx in itertools.count(): + try: + name = varname_from_oparg(idx) + allnames[name] = idx + except IndexError: + break + varnames = {name: allnames[name] for name in code_options["co_varnames"]} + freenames = { + name: allnames[name] + for name in code_options["co_cellvars"] + code_options["co_freevars"] + } + for i in range(len(instructions)): + + def should_compute_arg() -> bool: + # argval is prioritized over arg + return instructions[i].argval is not _NotProvided + + if instructions[i].opname == "LOAD_GLOBAL": + # 3.11 LOAD_GLOBAL requires both arg and argval - see create_instruction + assert instructions[i].argval is not _NotProvided + if sys.version_info >= (3, 11): + assert instructions[i].arg is not None + instructions[i].arg = (get_name_index(instructions[i].argval) << 1) + ( + cast(int, instructions[i].arg) % 2 + ) + else: + instructions[i].arg = get_name_index(instructions[i].argval) + elif instructions[i].opname == "LOAD_ATTR": + # 3.12 LOAD_ATTR requires both arg and argval, like LOAD_GLOBAL + assert instructions[i].argval is not _NotProvided + if sys.version_info >= (3, 12): + assert instructions[i].arg is not None + instructions[i].arg = (get_name_index(instructions[i].argval) << 1) + ( + cast(int, instructions[i].arg) % 2 + ) + else: + instructions[i].arg = get_name_index(instructions[i].argval) + elif instructions[i].opname == "LOAD_SUPER_ATTR": + assert instructions[i].arg is not None + assert instructions[i].argval is not _NotProvided + # Copy low bit, force second bit on for explicit super (the "+ 2") + instructions[i].arg = ( + (get_name_index(instructions[i].argval) << 2) + + (cast(int, instructions[i].arg) % 2) + + 2 + ) + elif instructions[i].opname in FUSED_INSTS: + assert sys.version_info >= (3, 13) + assert isinstance(instructions[i].argval, tuple) + assert len(instructions[i].argval) == 2 + arg_tuple = tuple( + varnames[name] if name in varnames else freenames[name] + for name in instructions[i].argval + ) + instructions[i].arg = (arg_tuple[0] << 4) + (arg_tuple[1] & 15) + elif instructions[i].opcode in HAS_LOCAL: + if should_compute_arg(): + if ( + sys.version_info >= (3, 13) + and instructions[i].argval not in varnames + ): + # instructions like LOAD_FAST used for both local and free vars + instructions[i].arg = freenames[instructions[i].argval] + else: + instructions[i].arg = varnames[instructions[i].argval] + elif instructions[i].opcode in HAS_NAME: + if should_compute_arg(): + instructions[i].arg = get_name_index(instructions[i].argval) + elif instructions[i].opcode in HAS_FREE: + if should_compute_arg(): + instructions[i].arg = freenames[instructions[i].argval] + elif instructions[i].opcode in HAS_CONST: + # NOTE: only update argval if arg is not provided. This assumes + # that any additions to co_consts are appended. + if instructions[i].arg is None: + # cannot use a dictionary since consts may not be hashable + idx = get_const_index(code_options, instructions[i].argval) + assert idx >= 0 + instructions[i].arg = idx + + +def clear_instruction_args(instructions: list[Instruction]) -> None: + # Clear the instruction arg for instructions that have argvals. + # Useful for using dis'd bytecode within generated bytecode. + for inst in instructions: + if ( + inst.argval is not _NotProvided + and ( + inst.opcode in HAS_LOCAL + or inst.opcode in HAS_NAME + or inst.opcode in HAS_FREE + or inst.opcode in HAS_CONST + ) + and inst.opname not in ("LOAD_GLOBAL", "LOAD_ATTR", "LOAD_SUPER_ATTR") + ): + inst.arg = None + + +@functools.lru_cache +def get_code_keys() -> list[str]: + # Python 3.11 changes to code keys are not fully documented. + # See https://github.com/python/cpython/blob/3.11/Objects/clinic/codeobject.c.h#L24 + # for new format. + keys = ["co_argcount"] + keys.append("co_posonlyargcount") + keys.extend( + [ + "co_kwonlyargcount", + "co_nlocals", + "co_stacksize", + "co_flags", + "co_code", + "co_consts", + "co_names", + "co_varnames", + "co_filename", + "co_name", + ] + ) + if sys.version_info >= (3, 11): + keys.append("co_qualname") + keys.append("co_firstlineno") + keys.append("co_linetable") + if sys.version_info >= (3, 11): + # not documented, but introduced in https://github.com/python/cpython/issues/84403 + keys.append("co_exceptiontable") + keys.extend( + [ + "co_freevars", + "co_cellvars", + ] + ) + return keys + + +def transform_code_object( + code: types.CodeType, + transformations: Callable[ + [list[Instruction], dict[str, Any]], Optional["DynamoTracerOutput"] + ], + safe: bool = False, +) -> tuple[types.CodeType, Optional["DynamoTracerOutput"]]: + keys = get_code_keys() + code_options = {k: getattr(code, k) for k in keys} + assert len(code_options["co_varnames"]) == code_options["co_nlocals"] + + instructions = cleaned_instructions(code, safe) + # propagate line nums again for added instructions + propagate_line_nums(instructions) + + tracer_output = transformations(instructions, code_options) + _, bytecode = clean_and_assemble_instructions(instructions, keys, code_options) + return bytecode, tracer_output + + +def clean_and_assemble_instructions( + instructions: list[Instruction], keys: list[str], code_options: dict[str, Any] +) -> tuple[list[Instruction], types.CodeType]: + remove_graph_break_if_leaf_instructions(instructions) + # also implicitly checks for no duplicate instructions + check_inst_exn_tab_entries_valid(instructions) + + code_options["co_nlocals"] = len(code_options["co_varnames"]) + varname_from_oparg = None + if sys.version_info >= (3, 11): + # temporary code object with updated names + tmp_code = types.CodeType(*[code_options[k] for k in keys]) + varname_from_oparg = tmp_code._varname_from_oparg # type: ignore[attr-defined] + fix_vars(instructions, code_options, varname_from_oparg=varname_from_oparg) + + dirty = True + while dirty: + update_offsets(instructions) + devirtualize_jumps(instructions) + # this pass might change offsets, if so we need to try again + dirty = bool(fix_extended_args(instructions)) + + remove_extra_line_nums(instructions) + bytecode, lnotab = assemble(instructions, code_options["co_firstlineno"]) + + code_options["co_linetable"] = lnotab + code_options["co_code"] = bytecode + code_options["co_stacksize"] = stacksize_analysis(instructions) + assert set(keys) - {"co_posonlyargcount"} == set(code_options.keys()) - { + "co_posonlyargcount" + } + if sys.version_info >= (3, 11): + code_options["co_exceptiontable"] = assemble_exception_table( + compute_exception_table(instructions) + ) + + return instructions, types.CodeType(*[code_options[k] for k in keys]) + + +def populate_kw_names_argval(instructions: Sequence[Instruction], consts: Any) -> None: + for inst in instructions: + if inst.opname == "KW_NAMES": + inst.argval = consts[inst.arg] + + +# If safe=True, we do not make any bytecode modifications. +# Mainly used for debugging bytecode_transformation (see debug_checks) +def cleaned_instructions(code: types.CodeType, safe: bool = False) -> list[Instruction]: + instructions = _cached_cleaned_instructions(code, safe) + # We have a lot of code that implicitly mutates the instruction array. We + # could do better here by making the copies explicit when necessary. + return _clone_instructions(instructions) + + +# Copy an instructions array, making sure to remap the individual instruction targets. +def _clone_instructions(instructions: Sequence[Instruction]) -> list[Instruction]: + # This is super hot and this is the fastest way to do this (tried copy.copy + # and dataclasses.replace). + copied = [ + Instruction( + i.opcode, + i.opname, + i.arg, + i.argval, + i.offset, + i.starts_line, + i.is_jump_target, + i.positions, + i.target, + i.exn_tab_entry, + i.argrepr, + ) + for i in instructions + ] + + remap = dict(zip(instructions, copied)) + # Handle `None` in the remapper so we don't need an extra `if`. + remap[None] = None # type: ignore[index, assignment] + + for i in copied: + i.target = remap[i.target] # type: ignore[index] + if entry := i.exn_tab_entry: + i.exn_tab_entry = InstructionExnTabEntry( + remap[entry.start], + remap[entry.end], + remap[entry.target], + entry.depth, + entry.lasti, + ) + return copied + + +@functools.lru_cache +def _cached_cleaned_instructions( + code: types.CodeType, safe: bool = False +) -> Sequence[Instruction]: + instructions = list(map(convert_instruction, dis.get_instructions(code))) + # propagate now in case we remove some instructions + propagate_line_nums(instructions) + check_offsets(instructions) + if sys.version_info >= (3, 11): + populate_kw_names_argval(instructions, code.co_consts) + virtualize_exception_table(code.co_exceptiontable, instructions) + virtualize_jumps(instructions) + strip_extended_args(instructions) + if not safe: + if sys.version_info < (3, 11): + remove_load_call_method(instructions) + if sys.version_info < (3, 12): + explicit_super(code, instructions) + if sys.version_info >= (3, 11): + remove_jump_if_none(instructions) + if sys.version_info >= (3, 12): + remove_binary_store_slice(instructions) + if sys.version_info >= (3, 13): + remove_fused_load_store(instructions) + if config.debug_force_graph_break_on_leaf_return: + add_graph_break_if_leaf_instructions(instructions) + if sys.version_info >= (3, 11): + update_offsets(instructions) + devirtualize_jumps(instructions) + return instructions + + +_unique_id_counter = itertools.count() + + +def unique_id(name: str, with_uuid: bool = False) -> str: + ret = f"{name}_{next(_unique_id_counter)}" + if with_uuid: + ret += f"_{uuid.uuid4()}".replace("-", "_") + return ret + + +def is_generator(code: types.CodeType) -> bool: + co_generator = 0x20 + return (code.co_flags & co_generator) > 0 + + +def bytecode_from_template( + fn: Callable[..., Any], + varname_map: Optional[Mapping[Any, Any]] = None, + noreturn: bool = True, + noprefix: bool = True, +) -> list[Instruction]: + """Generates bytecode from a template function `fn` for use in + dynamo bytecode generation. + + For example, we can generate Python-version-independent bytecode + for looping through a dictionary and copying the values to a new dictionary. + + def template(d1, d2): + for k, v in d1.items(): + d2[k] = v + + + or a try block: + + def template(): + try: + dummy1 + except: + dummy2 + raise + dummy3 + + Args: + fn: a function template to generate bytecode from + varname_map: a mapping of `fn`'s varnames to new names. This + map will be applied to the generated bytecode's varnames. + For example, local variables in `fn` can be replaced with + new names that are generated by `OutputGraph.new_var`. + noreturn: remove all RETURN_* bytecodes and replace them with a jump + to the end of the bytecode. NOTE: any items pushed to the stack + for return WILL remain on the stack! Append a POP_TOP if you don't want + that item to be present. + noprefix: remove prefix bytecodes (all bytecode before the first RESUME, inclusive). + """ + insts = cleaned_instructions(fn.__code__) + clear_instruction_args(insts) + + if noprefix: + for i, inst in enumerate(insts): + if inst.opname == "RESUME": + insts = insts[i + 1 :] + break + + for inst in insts: + # If we don't reset starts_line, then the generated + # bytecode's line number will be based on fn's. + inst.starts_line = None + inst.positions = None + if varname_map and inst.argval in varname_map: + inst.argval = varname_map[inst.argval] + + if noreturn: + if sys.version_info >= (3, 12): + # replace RETURN_CONST with LOAD_CONST RETURN_VALUE + new_insts = [] + for inst in insts: + if inst.opname == "RETURN_CONST": + inst.opcode = dis.opmap["LOAD_CONST"] + inst.opname = "LOAD_CONST" + new_insts.append(inst) + # no need to propagate target/exn table + new_insts.append(create_instruction("RETURN_VALUE")) + else: + new_insts.append(inst) + insts = new_insts + + returns = [] + for inst in insts: + if inst.opname == "RETURN_VALUE": + returns.append(inst) + + if len(returns) == 1 and returns[0] is insts[-1]: + # only 1 return at the end - just pop it + insts.pop(-1) + elif len(returns) > 0: + # create jump target - if the last inst is a return, + # we can replace it with a NOP and make that the jump target. + if insts[-1] is returns[-1]: + insts[-1].opname = "NOP" + insts[-1].opcode = dis.opmap["NOP"] + insts[-1].arg = None + insts[-1].argval = _NotProvided + returns.pop(-1) + else: + insts.append(create_instruction("NOP")) + + # replace returns with jumps + for inst in returns: + # don't replace inst with new instruction + # due to targeting/exn table/etc. + jump_inst = create_jump_absolute(insts[-1]) + inst.opname = jump_inst.opname + inst.opcode = jump_inst.opcode + inst.arg = jump_inst.arg + inst.argval = jump_inst.argval + inst.target = jump_inst.target + + return insts diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/cache_size.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/cache_size.py new file mode 100644 index 0000000000000000000000000000000000000000..d1a46742f37ac87c729a9d3973b6c85c36410716 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/cache_size.py @@ -0,0 +1,187 @@ +import logging +import weakref +from dataclasses import dataclass +from typing import Any, Optional + +from torch._guards import CompileId + +from . import config +from .types import DynamoFrameType + + +log: logging.Logger = logging.getLogger(__name__) +""" +[Note on cache size limit] + +Background - TorchDynamo cache is a linked list. Each cache entry is a +(guard_manager, out_code, next pointer). These are stored on the f_code's co_extra +scratch space. When a frame is invoked, we walk this linked list and run +guard_manager in each cache_entry to decide if the frame needs recompilation. If none +of the guard_manager's returns True, we recompile and add a new entry. To ensure we +don't end up recompiling infinitely, we put limits on the cache size. + +There are two limits +1) recompile_limit +2) accumulated_recompile_limit + + +Earlier we used to have only limit - maximum number of entries in 1 cache line +(which is now represented by (2) above). So, why do we need two limits? Lets try +to understand that. + +In general, we want our cache limit value to be a small number (e.g. 8 or even +lower). This ensures that for frames that cause too many recompilation fall to +eager quickly. However, there is another problem that prevents us from lowering +the value of recompile_limit. This is due to ID_MATCH'd guards. Today, we put +ID_MATCH guards on nn module if there is a graph break. This means we will have +many recompilations for the same code object because the ID_MATCH guard fails +for different instances of the nn module. This is a common pattern in how models +are authored. Therefore, this requires us to keep the recompile_limit high. + +We resolve this by introducing these two limits. The first limit (1) limits the +number of cache entries that have an ID_MATCH'd guard for an nn module instance. +And, (2)nd limit becomes a safeguard mechanism to have a maximum compilations +for a code object. One important question is - what is the limit for the code +object that does not have any ID_MATCH guard? For such code objects, we choose +(1) as the cache size limit. + +Lets take an example to understand how these limits help. Suppose, we have 16 +instances of a nn module and we ID_MATCH on the self object. Further, suppose +the inputs to these functions have varying batch size, leading to one +recompilation. In total, there will be 32 recompilations, and therefore 32 cache +entries on the forward code object. In the older case when we had only 1 limit, +our cache size limit must be >= 32 to capture all these recompilations. Now, +suppose there is a separate function in the same program which is very dynamic +and unsuitable for compilation. Such a function will need to undergo 32 +compilations to burst the cache and fallback to eager. These 32 recompilations +are too many and we want to fallback for these compilation-unfriendly functions +sooner. + +In the new scenario, we can have (1) recompile_limit = 2, (2) +accumulated_recompile_limit = 32. This means that each ID_MATCH'd object can +have maximum of two cache entries, and the maximum number of cache entries +(irrespective of ID_MATCH obj) is 32. This covers the case of forward code +object which has 32 recompilations. For the other function, the one unsuitable +for recompilation, our limit is 2. So, we will burst the cache in just 2 +recompilations. In this manner, these 2 limits help us resolve the tension +mentioned earlier. +""" + + +@dataclass +class CacheSizeRelevantForFrame: + """ + We track the number of cache entries that have same id_match objects as the + given frame. + + TODO(janimesh) - Consider adding a map from tuple_of_match_ids to count - + https://github.com/pytorch/pytorch/pull/107496#discussion_r1304564682 - this + could be useful for debugging as well. + """ + + # Total number of CacheEntry objects in the Dynamo linked list + num_cache_entries: int = 0 + + # Number of CacheEntry objects having same ID_MATCH'd objects as given frame. + num_cache_entries_with_same_id_matched_objs: int = 0 + + def will_compilation_exceed(self, limit: int) -> bool: + # Checks if a compilation will exceed the given limit (that's why >=). + return ( + self.will_compilation_exceed_accumulated_limit() + or self.will_compilation_exceed_specific_limit(limit) + ) + + def will_compilation_exceed_accumulated_limit(self) -> bool: + return self.num_cache_entries >= config.accumulated_recompile_limit + + def will_compilation_exceed_specific_limit(self, limit: int) -> bool: + return self.num_cache_entries_with_same_id_matched_objs >= limit + + +def _get_weakref_from_f_locals( + frame: DynamoFrameType, local_name: str +) -> Optional[weakref.ref[Any]]: + obj = frame.f_locals.get(local_name, None) + weak_id = None + try: + weak_id = weakref.ref(obj) + except TypeError: + pass # cannot weakref bool object + return weak_id + + +def _has_same_id_matched_objs(frame: DynamoFrameType, cache_entry: Any) -> bool: + """ + Checks if the ID_MATCH'd objects saved on cache_entry are same as the ones + in frame.f_locals. + """ + if not cache_entry: + return False + + for ( + local_name, + weakref_from_cache_entry, + ) in cache_entry.guard_manager.id_matched_objs.items(): + if weakref_from_cache_entry() is not None: + weakref_from_frame = _get_weakref_from_f_locals(frame, local_name) + if weakref_from_frame is not weakref_from_cache_entry: + return False + + # Also covers the case where no ID_MATCH objects are saved in frame.f_locals + return True + + +def compute_cache_size( + frame: DynamoFrameType, cache_entry: Any +) -> CacheSizeRelevantForFrame: + # Walk the linked list to calculate the cache size + num_cache_entries = 0 + num_cache_entries_with_same_id_matched_objs = 0 + + while cache_entry: + num_cache_entries += 1 + # Track the number of cache entries having same ID_MATCH'd objects as + # that of frame.f_locals. This will be used later to compare against the + # recompile_limit. + if _has_same_id_matched_objs(frame, cache_entry): + num_cache_entries_with_same_id_matched_objs += 1 + cache_entry = cache_entry.next + + return CacheSizeRelevantForFrame( + num_cache_entries, num_cache_entries_with_same_id_matched_objs + ) + + +def is_recompilation(cache_size: CacheSizeRelevantForFrame) -> bool: + """ + If the frame (earlier parsed by compute_cache_size) has more than 1 cache + entry with same ID_MATCH'd objects, then its a recompilation. + """ + # Note that you can have multiple entries in the cache but still not a + # recompile, e.g., you can have 64 nn module instances, each one having an + # ID_MATCH guard, and each one having just 1 cache entry in the cache. In + # this case, we can have 64 entries in the cache, but no recompilation + # because there is only one entry for each id_matched_obj. + return cache_size.will_compilation_exceed(1) + + +def exceeds_recompile_limit( + cache_size: CacheSizeRelevantForFrame, compile_id: CompileId +) -> tuple[bool, str]: + """ + Checks if we are exceeding the cache size limit. + """ + if cache_size.will_compilation_exceed_accumulated_limit(): + return True, "accumulated_recompile_limit" + if cache_size.will_compilation_exceed_specific_limit(config.recompile_limit): + return True, "recompile_limit" + # NOTE this check is needed in the case that the frame's cache doesn't grow + # and we keep recompiling. This can happen if the guard guard_manager becomes invalidated, + # e.g. due to guarded objects being freed. This technically makes the + # will_compilation_exceed_accumulated_limit check unnecessary, but we will keep the + # check in case we have a better fix in the future. + assert compile_id.frame_compile_id is not None + if compile_id.frame_compile_id >= config.accumulated_recompile_limit: + return True, "accumulated_recompile_limit" + return False, "" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/callback.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/callback.py new file mode 100644 index 0000000000000000000000000000000000000000..25e9f260e34b3b054ba6552ec9d5fc64a61c20fb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/callback.py @@ -0,0 +1,171 @@ +""" +This module provides callback management functionality for TorchDynamo's compilation process. + +It implements a thread-safe system for registering, managing and executing callbacks that run +at the start and end of TorchDynamo compilations. Key features include: + +- Registration and deregistration of compilation callbacks +- Thread-safe callback handling with proper locking mechanisms +- Prevention of duplicate callback execution when configured +- Decorator utilities for easy callback registration +- Context manager for controlled callback lifecycle + +The module centers around the CompilationCallbackHandler class which maintains separate +lists for start and end callbacks, manages their execution order, and ensures thread-safety. +Utility decorators @on_compile_start and @on_compile_end provide a convenient way to +register compilation hooks. + +Example usage: + @on_compile_start + def my_start_callback(): + print("Starting compilation") + + @on_compile_end + def my_end_callback(): + print("Compilation complete") +""" + +import enum +import threading +from collections.abc import Callable, Generator +from contextlib import contextmanager +from dataclasses import dataclass, field # noqa: F811 +from typing import Any + + +class CallbackTrigger(enum.Enum): + # most common case, dynamo attempts to trace a new frame + DYNAMO = 1 + # backward compilation can be deferred to runtime + LAZY_BACKWARD = 2 + # some backends autotune at runtime + TRITON_AUTOTUNING = 3 # Temporarily disabled due to spam + # cudagraphs record at runtime + CUDAGRAPH_RECORDING = 4 + + +@dataclass +class CallbackArgs: + callback_trigger: CallbackTrigger + compile_id: str + + +@dataclass +class CompilationCallbackHandler: + start_callbacks: list[Callable[[CallbackArgs], None]] = field(default_factory=list) + end_callbacks: list[Callable[[CallbackArgs], None]] = field(default_factory=list) + + __pending_callbacks_counter: int = field(default=0, init=False, repr=False) + __pending_callbacks_counter_lock: threading.Lock = field( + default_factory=threading.Lock, init=False, repr=False + ) + + def register_start_callback( + self, callback: Callable[[CallbackArgs], None] + ) -> Callable[[CallbackArgs], None]: + """ + Register a callback function to be called when the compilation starts. + + Args: + - callback (Callable): The callback function to register. + """ + self.start_callbacks.append(callback) + return callback + + def register_end_callback( + self, callback: Callable[[CallbackArgs], None] + ) -> Callable[[CallbackArgs], None]: + """ + Register a callback function to be called when the compilation ends. + + Args: + - callback (Callable): The callback function to register. + """ + self.end_callbacks.append(callback) + return callback + + def remove_start_callback(self, callback: Callable[[CallbackArgs], None]) -> None: + """ + Remove a registered start callback function. + + Args: + - callback (Callable): The callback function to remove. + """ + self.start_callbacks.remove(callback) + + def remove_end_callback(self, callback: Callable[[CallbackArgs], None]) -> None: + """ + Remove a registered end callback function. + + Args: + - callback (Callable): The callback function to remove. + """ + self.end_callbacks.remove(callback) + + def run_start_callbacks(self, args: CallbackArgs) -> None: + """ + Execute all registered start callbacks. + """ + for callback in self.start_callbacks: + callback(args) + + def run_end_callbacks(self, args: CallbackArgs) -> None: + """ + Execute all registered end callbacks. + """ + for callback in self.end_callbacks: + callback(args) + + @contextmanager + def install_callbacks( + self, trigger: CallbackTrigger, compile_id: str + ) -> Generator[None, Any, Any]: + """ + Context manager to install the callbacks and run them when the context is exited. + """ + args = CallbackArgs(trigger, compile_id) + try: + with self.__pending_callbacks_counter_lock: + self.__pending_callbacks_counter += 1 + if self.__pending_callbacks_counter == 1: + self.run_start_callbacks(args) + yield + finally: + with self.__pending_callbacks_counter_lock: + assert self.__pending_callbacks_counter > 0, ( + "Pending callbacks counter cannot become negative." + ) + if self.__pending_callbacks_counter == 1: + self.run_end_callbacks(args) + self.__pending_callbacks_counter -= 1 + + def clear(self) -> None: + """ + Clear all registered callbacks. + """ + self.start_callbacks.clear() + self.end_callbacks.clear() + assert self.__pending_callbacks_counter == 0 + + +callback_handler = CompilationCallbackHandler() + + +def on_compile_start( + callback: Callable[[CallbackArgs], None], +) -> Callable[[CallbackArgs], None]: + """ + Decorator to register a callback function for the start of the compilation. + """ + callback_handler.register_start_callback(callback) + return callback + + +def on_compile_end( + callback: Callable[[CallbackArgs], None], +) -> Callable[[CallbackArgs], None]: + """ + Decorator to register a callback function for the end of the compilation. + """ + callback_handler.register_end_callback(callback) + return callback diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/code_context.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/code_context.py new file mode 100644 index 0000000000000000000000000000000000000000..f2ccb3f0dc90ef8d0f78508bc4a9300997555ebf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/code_context.py @@ -0,0 +1,60 @@ +""" +This module provides thread-safe code context management for TorchDynamo using weak references. + +The CodeContextDict class maintains a mapping between Python code objects and their associated +context data, using weak references to automatically clean up entries when code objects are +garbage collected. This prevents memory leaks while allowing context data to be associated +with code objects throughout their lifecycle. + +Key features: +- Thread-safe context storage and retrieval +- Automatic cleanup using weak references +- Safe context management for Python code objects +- Memory-leak prevention + +Example usage: + code_obj = compile('x = 1', '', 'exec') + + # Store context + context = code_context.get_context(code_obj) + context['metadata'] = {'optimized': True} + + # Retrieve context + if code_context.has_context(code_obj): + ctx = code_context.get_context(code_obj) + # Use context data... + + # Remove context + ctx = code_context.pop_context(code_obj) +""" + +import types +from typing import Any + +from .utils import ExactWeakKeyDictionary + + +class CodeContextDict: + def __init__(self) -> None: + self.code_context: ExactWeakKeyDictionary = ExactWeakKeyDictionary() + + def has_context(self, code: types.CodeType) -> bool: + return code in self.code_context + + def get_context(self, code: types.CodeType) -> dict[str, Any]: + ctx = self.code_context.get(code) + if ctx is None: + ctx = {} + self.code_context[code] = ctx + return ctx + + def pop_context(self, code: types.CodeType) -> dict[str, Any]: + ctx = self.get_context(code) + self.code_context._remove_id(id(code)) + return ctx + + def clear(self) -> None: + self.code_context.clear() + + +code_context: CodeContextDict = CodeContextDict() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/codegen.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/codegen.py new file mode 100644 index 0000000000000000000000000000000000000000..8c19cb8b61e27a473653cc95725b1badb2f87f98 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/codegen.py @@ -0,0 +1,710 @@ +""" +This module provides utilities for generating Python bytecode in PyTorch's Dynamo system. +It includes functionality for: +- Constructing bytecode sequences for Python operations +- Managing stack operations and variable tracking +- Handling graph outputs and their conversions +- Supporting different Python versions (3.11+, 3.12+, 3.13+) +- Converting high-level operations to low-level bytecode instructions +- Managing constant loading and attribute access +- Supporting function creation and closure handling +""" + +import collections +import dataclasses +import re +import sys +import types +from collections import Counter, deque +from collections.abc import Callable, Iterable +from typing import Any, Optional, TYPE_CHECKING, Union + +import torch.nn +from torch.utils._ordered_set import OrderedSet + +from . import config, graph_break_hints, utils +from .bytecode_transformation import ( + add_push_null, + add_push_null_call_function_ex, + create_binary_subscr, + create_build_tuple, + create_call_function, + create_call_function_ex, + create_call_method, + create_dup_top, + create_instruction, + create_load_const, + create_load_method, + create_rot_n, + Instruction, +) +from .exc import IncorrectUsage, unimplemented +from .source import AttrSource, ChainedSource, DictGetItemSource, Source +from .utils import is_safe_constant, rot_n_helper +from .variables.base import ValueMutationExisting, VariableTracker +from .variables.functions import ( + ContextlibContextManagerLocalGeneratorObjectVariable, + LocalGeneratorObjectVariable, +) +from .variables.nn_module import NNModuleVariable +from .variables.tensor import ( + NumpyNdarrayVariable, + SymNodeVariable, + TensorVariable, + UnspecializedPythonVariable, +) +from .variables.torch_function import TensorWithTFOverrideVariable + + +if TYPE_CHECKING: + from torch._dynamo.variables.builder import GraphArg + + from .symbolic_convert import InstructionTranslatorBase + + +@dataclasses.dataclass +class GraphOutputEntry: + index: int + variable: VariableTracker + + +class PyCodegen: + """ + Helper class uses for constructing Python bytecode + """ + + def __init__( + self, + tx: "InstructionTranslatorBase", + root: Optional[torch.nn.Module] = None, + graph_output_var: Optional[str] = None, + tempvars: Optional[dict[Union[VariableTracker, Source], Any]] = None, + overridden_sources: Optional[dict[Source, Source]] = None, + ) -> None: + self.root = root + self.top_of_stack: Optional[Union[VariableTracker, Source]] = None + self.uses: Counter[Union[VariableTracker, Source]] = collections.Counter() + self.graph_outputs: dict[int, GraphOutputEntry] = {} + self._output: list[Instruction] = [] + # This determines which VariableTracker/Source should be stored as + # locals, and maps the VariableTracker/Source to the local variable + # name. Note that it could map to None initially, in which case we'll + # overwrite it to map to real temporary names via `add_cache`. + self.tempvars: dict[Union[VariableTracker, Source], Any] = tempvars or {} + self.tx = tx + self.graph_output_var = graph_output_var + self.code_options = self.tx.output.code_options + self.cell_and_freevars = self.tx.cell_and_freevars + self.new_var = self.tx.output.new_var + self.value_from_source: bool = True + # This serves as a way for codegen to use a different source; we need + # this because sometimes we can't easily modify the original source + # without affecting other components, e.g., guards. + self.overridden_sources: dict[Source, Source] = overridden_sources or {} + + def restore_stack( + self, stack_values: list[Any], *, value_from_source: bool = True + ) -> None: + prev = self.value_from_source + self.value_from_source &= value_from_source + try: + self.foreach(stack_values) + finally: + self.value_from_source = prev + + def graph_output_vars(self) -> list[VariableTracker]: + return [x.variable for x in self.graph_outputs.values()] + + def call_reconstruct( + self, value: Union[VariableTracker, Source, "GraphArg"] + ) -> None: + res = value.reconstruct(self) + assert res is None, f"reconstruct!=None {value}" + + def add_push_null( + self, gen_fn: Callable[[], None], call_function_ex: bool = False + ) -> None: + """ + `gen_fn` generates instructions via PyCodegen methods + that push a single callable to the stack. + + `add_push_null` pushes a NULL to the stack before or after the + instructions generated by `gen_fn`, depending on Python version. + + Will attempt to use the NULL push bit for instructions + with such bits (LOAD_GLOBAL 3.11+, LOAD_ATTR 3.12+, LOAD_SUPER_ATTR). + """ + old_len = len(self._output) + if sys.version_info < (3, 13): + # gen_fn may DUP_TOP instead if TOS is not cleared. + # Will cause problems since NULL will be pushed right + # before the generated instructions in <= 3.12 + self.clear_tos() + gen_fn() + # inplace modify self._output + added_insts = self._output[old_len:] + del self._output[old_len:] + if call_function_ex: + self._output.extend(add_push_null_call_function_ex(added_insts)) + else: + self._output.extend(add_push_null(added_insts)) + if sys.version_info >= (3, 13): + # NULL will be at top of stack + self.clear_tos() + + def __call__( + self, value: Union[VariableTracker, Source, None], allow_cache: bool = True + ) -> None: + """ + Generate code such that top-of-stack (TOS) is set to value. + + `allow_cache` controls the behavior in the following manner. `value` can + either be a VariableTracker or a Source. + + If `value` is a `Source`, `allow_cache` must be True (invariant asserted + below). If the source was reconstructed earlier, we will reuse the + generated code by loading from top of stack or tempvars. + + If `value` is a `VariableTracker`, we have the following cases: + + 1) `allow_cache=True` + a) If the value.source is not None, we will emit the code based on + `value.source` to handle aliasing. + b) If value.source is None (example reconstructing a local list + returned by the compiled function), we will reconstruct the variable + tracker (w/o any source) to emit bytecode that generates a new + python object. + + In both cases of value.source being None or not, if the value was + reconstructed earlier, we will reuse the generated code by loading from + top of stack or tempvars. + + 2) `allow_cache=False` - This is a special case (allow_cache defaults to + True). + a) If the value.source is not None, we reconstruct the variable + tracker and emit a new python object. You might wonder what about + aliasing? The place where we use this config also has the followup + code where the original python object is assigned to this new python + value to handle aliasing (check side_effects.py and search for + allow_cache=False). + + b) If value.source is None, this is not allowed + + Notable effects: + 1. `self.top_of_stack` will be set to `value`, if we don't codegen + `value` based on source. + 2. `self.uses[value]` will increment, unless (a). we codegen via + `top_of_stack` or cached `tempvars`, or (b). `value` has special VT + types like `NNModuleVariable`, etc. + """ + assert value is not None + if isinstance(value, Source): + # If the source needs to be overridden, use the new one. + source = self.overridden_sources.get(value, value) + assert allow_cache is True, "allow_cache must be True for Source" + if self.top_of_stack is value: + self._output.append(create_dup_top()) + return + + if self.tempvars.get(source) is not None: + self._output.append(self.create_load(self.tempvars[source])) + self.top_of_stack = source + return + + self.uses[source] += 1 + try: + self.call_reconstruct(source) + except NotImplementedError: + unimplemented( + gb_type="Reconstruction failure: source.reconstruct not implemented", + context=str(source), + explanation=f"Dynamo has no bytecode reconstruction implemented for {type(source)} variable {source}.", + hints=[*graph_break_hints.DYNAMO_BUG], + ) + if source in self.tempvars: + self._output.append(create_dup_top()) + self.add_cache(source) + self.top_of_stack = source + + return + + assert isinstance(value, VariableTracker) + output = self._output + graph_outputs = self.graph_outputs + + if allow_cache: + if self.top_of_stack is value: + output.append(create_dup_top()) + return + + if self.tempvars.get(value) is not None: + output.append(self.create_load(self.tempvars[value])) + self.top_of_stack = value + return + + if value.is_realized() and isinstance( + value, ContextlibContextManagerLocalGeneratorObjectVariable + ): + raise IncorrectUsage( + "NYI: Returning a @contextmanager object from a torch.compile function" + ) + + # Dynamo normally prefers codegen from source to account for aliasing. + if ( + value.source is not None + and allow_cache + and not ( + value.is_realized() and isinstance(value, LocalGeneratorObjectVariable) + ) + ): + # There's a corner case for export: for instance, if the computation + # graph is just identity on an input tensor, Dynamo would just emit + # a `LOAD_FAST` from the input source, rather than generating an + # identity FX graph. + # + # However, export wants to maximize graph capture; in the case + # above, export _wants to_ obtain an identity FX graph (despite it + # appears unnecessarily expensive for `torch.compile`), so we have + # the following option to override Dynamo's preference for codegen + # from source. Moreover, this option applies recursively, for cases + # like input tensor being returned in a new dictionary. + # + # And why the `ValueMutationExisting` check? Not sure, so leaving it + # to keep the old behavior, as when `value_from_source` was + # introduced. TODO sort out the invariants among side effect, + # codegen and export. + if ( + isinstance(value.mutation_type, ValueMutationExisting) + or self.value_from_source + ): + return self(value.source) + + if value.is_python_constant() and is_safe_constant(value.as_python_constant()): + output.append(self.create_load_const(value.as_python_constant())) + elif isinstance(value, TensorWithTFOverrideVariable): + graph_outputs_key = self.add_graph_output(value) + + self.add_push_null( + lambda: self.load_import_from(utils.__name__, "to_subclass") + ) + self.load_graph_output(graph_outputs[graph_outputs_key].index) + output.append( + self.create_load_global( + value.global_mangled_class_name(self.tx), # type: ignore[arg-type] + add=True, + ) + ) + output.extend(create_call_function(2, False)) + elif ( + isinstance(value, SymNodeVariable) + and value.python_type() is float + and not self.tx.export + ): + # This is a little unusual; force the output convention to be a + # Tensor here. Don't do this for export because this is + # apparently load bearing for export tests (but I am a bit + # doubtful it actually works in the real world) + # NB: It works to add_graph_output on a computed expression + # as_tensor here, because we memoize as_tensor calls on + # SymNodeVariable! + graph_outputs_key = self.add_graph_output( + value.as_tensor(self.tx, torch.float64) + ) + + def gen_fn() -> None: + self.load_graph_output(graph_outputs[graph_outputs_key].index) + output.append(self.create_load_attr("item")) + + self.add_push_null(gen_fn) + output.extend(create_call_function(0, False)) + elif isinstance( + value, + ( + TensorVariable, + SymNodeVariable, + UnspecializedPythonVariable, + NumpyNdarrayVariable, + ), + ): + graph_outputs_key = self.add_graph_output(value) + + if isinstance(value, NumpyNdarrayVariable): + self.add_push_null( + lambda: self.load_import_from(utils.__name__, "to_numpy_helper") + ) + self.load_graph_output(graph_outputs[graph_outputs_key].index) + output.extend(create_call_function(1, False)) + elif isinstance(value, UnspecializedPythonVariable) and value.need_unwrap: + + def gen_fn() -> None: + self.load_graph_output(graph_outputs[graph_outputs_key].index) + output.append(self.create_load_attr("item")) + + self.add_push_null(gen_fn) + output.extend(create_call_function(0, False)) + else: + self.load_graph_output(graph_outputs[graph_outputs_key].index) + elif isinstance(value, NNModuleVariable): + parts = value.module_key.split(".") + if parts[0] in self.code_options["co_varnames"]: + output.append(self.create_load(parts[0])) + parts = parts[1:] + else: + assert self.root is not None + output.append(self.create_load_const_unchecked(self.root)) + for part in parts: + output.append(self.create_load_attr(part)) + else: + self.uses[value] += 1 + try: + self.call_reconstruct(value) + except NotImplementedError: + unimplemented( + gb_type="Reconstruction failure", + context=str(value), + explanation=f"Dynamo has no bytecode reconstruction implemented for sourceless variable {value}.", + hints=[ + "If Dynamo is attempting to trace a return statement and your code is attempting to return a variable " + "that Dynamo cannot reconstruct, then remove it from the return statement.", + *graph_break_hints.CAUSED_BY_EARLIER_GRAPH_BREAK, + "Report an issue to PyTorch if you need reconstrtuction support. Note that objects that don't have " + "reconstruction rules may be fundamentally unreconstructable.", + ], + ) + if allow_cache and value in self.tempvars: + self._output.append(create_dup_top()) + self.add_cache(value) + + self.top_of_stack = value + + def add_graph_output(self, value: VariableTracker) -> int: + graph_outputs_key = id(value.as_proxy()) + if graph_outputs_key not in self.graph_outputs: + self.graph_outputs[graph_outputs_key] = GraphOutputEntry( + len(self.graph_outputs), value + ) + return graph_outputs_key + + def load_graph_output(self, index: int) -> None: + output = self._output + assert self.graph_output_var is not None + output.append(self.create_load(self.graph_output_var)) + output.append(self.create_load_const(index)) + output.append(self.create_binary_subscr()) + + def add_cache(self, value: Union[VariableTracker, Source]) -> None: + var = self.new_var() + self.tempvars[value] = var + self._output.append(self.create_store(var)) + + def foreach(self, items: Iterable[Union[VariableTracker, Source]]) -> None: + for i in items: + self(i) + + def create_binary_subscr(self) -> Instruction: + return create_binary_subscr() + + def setup_globally_cached(self, name: str, value: Any) -> list[Instruction]: + """Store value in a new global""" + name = re.sub(r"[^a-zA-Z0-9_]+", "_", name) + f_globals = self.tx.f_globals + if name in f_globals: + assert id(f_globals[name]) == id(value) + else: + f_globals[name] = value + return [self.create_load_global(name, add=True)] + + def clear_tos(self) -> None: + self.top_of_stack = None + + def append_output(self, inst: Instruction) -> None: + assert isinstance(inst, Instruction) + self._output.append(inst) + self.clear_tos() + + def extend_output(self, insts: list[Instruction]) -> None: + assert all(isinstance(x, Instruction) for x in insts) + self._output.extend(insts) + self.clear_tos() + + def get_instructions(self) -> list[Instruction]: + return self._output + + def create_load(self, name: str) -> Instruction: + assert name in self.code_options["co_varnames"], f"{name} missing" + return create_instruction("LOAD_FAST", argval=name) + + def create_load_closure(self, name: str) -> Instruction: + assert name in self.cell_and_freevars() + inst_name = "LOAD_FAST" if sys.version_info >= (3, 13) else "LOAD_CLOSURE" + return create_instruction(inst_name, argval=name) + + def create_load_deref(self, name: str) -> Instruction: + assert name in self.cell_and_freevars() + return create_instruction("LOAD_DEREF", argval=name) + + def create_store(self, name: str) -> Instruction: + assert name in self.code_options["co_varnames"], f"{name} missing" + return create_instruction("STORE_FAST", argval=name) + + def create_store_deref(self, name: str) -> Instruction: + assert name in self.cell_and_freevars() + return create_instruction("STORE_DEREF", argval=name) + + def create_load_global(self, name: str, add: bool = False) -> Instruction: + if add: + self.tx.output.update_co_names(name) + assert name in self.code_options["co_names"], f"{name} not in co_names" + return create_instruction("LOAD_GLOBAL", argval=name) + + def create_load_const(self, value: Any) -> Instruction: + return create_load_const(value) + + def create_load_const_unchecked(self, value: Any) -> Instruction: + return create_load_const(value, checked=False) + + def load_method(self, name: str) -> None: + self.tx.output.update_co_names(name) + self.append_output(create_load_method(name)) + + def call_method(self, nargs: int) -> None: + self.extend_output(create_call_method(nargs)) + + def create_load_attr(self, name: str) -> Instruction: + if name not in self.code_options["co_names"]: + self.code_options["co_names"] += (name,) + return create_instruction("LOAD_ATTR", argval=name) + + def load_attr(self, name: str) -> None: + self.append_output(self.create_load_attr(name)) + + def create_load_attrs(self, names: str) -> list[Instruction]: + return [self.create_load_attr(name) for name in names.split(".")] + + def create_store_attr(self, name: str) -> Instruction: + if name not in self.code_options["co_names"]: + self.code_options["co_names"] += (name,) + return create_instruction("STORE_ATTR", argval=name) + + def store_attr(self, name: str) -> None: + self.append_output(self.create_store_attr(name)) + + def load_function_name( + self, fn_name: str, push_null: bool, num_on_stack: int = 0 + ) -> list[Instruction]: + """Load the global fn_name on the stack num_on_stack down""" + output = [] + if push_null and sys.version_info >= (3, 11): + output.extend(add_push_null(self.create_load_global(fn_name, add=True))) + if num_on_stack > 0: + output.extend( + [ + *self.rot_n(num_on_stack + 2), + *self.rot_n(num_on_stack + 2), + ] + ) + else: + output.extend( + [ + self.create_load_global(fn_name, add=True), + *self.rot_n(num_on_stack + 1), + ] + ) + return output + + def rot_n(self, n: int) -> list[Instruction]: + try: + return create_rot_n(n) + except AttributeError: + # desired rotate bytecode doesn't exist, generate equivalent bytecode + return [ + create_build_tuple(n), + self.create_load_const_unchecked(rot_n_helper(n)), + *create_rot_n(2), + *create_call_function_ex(False, False), + create_instruction("UNPACK_SEQUENCE", arg=n), + ] + + def pop_top(self) -> None: + self.append_output(create_instruction("POP_TOP")) + + def call_function(self, nargs: int, push_null: bool) -> None: + self.extend_output(create_call_function(nargs, push_null=push_null)) + + def dup_top(self) -> None: + self.append_output(create_dup_top()) + + def store(self, varname: str) -> None: + self.append_output(self.create_store(varname)) + + def load_deref(self, varname: str) -> None: + self.append_output(self.create_load_deref(varname)) + + def make_function_with_closure( + self, + fn_name: str, + code: types.CodeType, + ) -> None: + """Creates a closure with code object `code`. + + Expects the TOS to be the tuple of cells to use for this closure. + TOS will be popped to create the closure. + Args: + - fn_name: name of the function + - code: code object of the function + (does not include the tuple of cells on the TOS) + """ + output = self._output + + output.append(self.create_load_const(code)) + if sys.version_info < (3, 11): + output.append(self.create_load_const(fn_name)) + if sys.version_info >= (3, 13): + output.extend( + [ + create_instruction("MAKE_FUNCTION"), + create_instruction("SET_FUNCTION_ATTRIBUTE", arg=0x08), + ] + ) + else: + output.append(create_instruction("MAKE_FUNCTION", arg=0x08)) + + self.clear_tos() + + def create_load_python_module(self, mod: types.ModuleType) -> Instruction: + """ + Generate a LOAD_GLOBAL instruction to fetch a given python module. + """ + output = self.tx.output + global_scope = output.global_scope + name = re.sub(r"^.*[.]", "", mod.__name__) + if global_scope.get(name, None) is mod: + return self.create_load_global(name, add=True) + prefix = f"___module_{name}" + global_name = self.tx.output.install_global_by_id(prefix, mod) + return self.create_load_global(global_name, add=True) + + def mark_source_temp(self, source: Source) -> None: + """ + Mark a source as a temp variable, so that it can be reused. + """ + if source not in self.tempvars: + self.tempvars[source] = None + + def make_call_generated_code(self, fn_name: str) -> None: + """Call the generated code function stored in fn_name""" + self.extend_output(self.load_function_name(fn_name, True)) + + graphargs = self.tx.output.graphargs + + def extract_nested_sources(source: Source) -> list[Source]: + nested_sources: list[Source] = [] + if isinstance(source, ChainedSource): + nested_sources.append(source.base) + if isinstance(source, DictGetItemSource) and isinstance( + source.index, Source + ): + nested_sources.append(source.index) + return nested_sources + + def collect_temp_sources(sources: deque[Source], codegen: PyCodegen) -> None: + seen_sources: OrderedSet[Source] = OrderedSet() + while sources: + current_source = sources.popleft() + if current_source in seen_sources: + # This source is used at least twice, so it can be reused + codegen.mark_source_temp(current_source) + # Dont trace source further. This prevents us from marking too + # many nodes as temp sources. + continue + seen_sources.add(current_source) + sources.extend(extract_nested_sources(current_source)) + + # Collect all the sources that are used more than once, so that we can + # generate tmp variables in the generated pre-graph bytecode. This + # essentially implements CSE. + collect_temp_sources( + deque([arg.source for arg in graphargs if arg.source is not None]), self + ) + + cm_var = None + if config.record_runtime_overhead: + # Record the pregraph bytecode start + self.add_push_null( + lambda: self.load_import_from( + utils.__name__, "record_pregraph_bytecode_enter" + ) + ) + self.extend_output(create_call_function(0, False)) + cm_var = self.new_var() + self.store(cm_var) + + for arg in graphargs: + if arg.pass_arg_as_tensor: + self.add_push_null( + lambda: self.extend_output( + [ + self.create_load_python_module(torch), + self.create_load_attr("_as_tensor_fullprec"), + ] + ) + ) + self.call_reconstruct(arg) + self.extend_output(create_call_function(1, False)) + else: + self.call_reconstruct(arg) + + if config.record_runtime_overhead: + # Record the pregraph bytecode end + self.add_push_null( + lambda: self.load_import_from( + utils.__name__, "record_pregraph_bytecode_exit" + ) + ) + assert cm_var is not None + self.extend_output([self.create_load(cm_var)]) + self.extend_output(create_call_function(1, False)) + self.pop_top() + + self.extend_output(create_call_function(len(graphargs), False)) + + def create_import_name(self, module_name: str) -> Instruction: + return create_instruction("IMPORT_NAME", argval=module_name) + + def load_import_from(self, module_name: str, object_name: str) -> None: + source = AttrSource(self.tx.import_source(module_name), object_name) + # Note: This approach is somewhat aggressive because typically, a source is marked + # as a tempvar only when it is used more than once. In this case, we're marking it + # as a tempvar without performing that analysis. However, this is a simple solution, + # and in many cases, load imports are reused multiple times. + self.mark_source_temp(source) + self(source) + + def create_call_function_kw( + self, nargs: int, kw_names: Iterable[str], push_null: bool + ) -> list[Instruction]: + if sys.version_info >= (3, 13): + output = create_call_function(nargs, push_null) + assert output[-1].opname == "CALL" + output.insert(-1, self.create_load_const(kw_names)) + output[-1] = create_instruction("CALL_KW", arg=nargs) + return output + elif sys.version_info >= (3, 11): + output = create_call_function(nargs, push_null) + if sys.version_info >= (3, 12): + idx = -1 + expected_inst = "CALL" + else: + idx = -2 + expected_inst = "PRECALL" + assert output[idx].opname == expected_inst + kw_names_inst = create_instruction("KW_NAMES", argval=kw_names) + output.insert(idx, kw_names_inst) + return output + return [ + self.create_load_const(kw_names), + create_instruction("CALL_FUNCTION_KW", arg=nargs), + ] + + def create_delete(self, value: object) -> Instruction: + return create_instruction("DELETE_FAST", argval=value) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/compiled_autograd.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/compiled_autograd.py new file mode 100644 index 0000000000000000000000000000000000000000..5f501a564ada50e6368d5f7f8bcf1c1bd3fc3a72 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/compiled_autograd.py @@ -0,0 +1,1620 @@ +""" +Provides functionality for compiling PyTorch's autograd (automatic differentiation) system. + +This module implements compiled autograd, which traces and optimizes backward pass +computations at runtime. The key components are: + +- AutogradCompilerInstance: Traces and compiles autograd graphs using FX +- Context managers (_enable/_disable): Control when compiled autograd is active +- Utility functions: Support graph manipulation, tensor operations, and hooks + +Compiled autograd can significantly improve backward pass performance by removing +Python overhead and enabling additional optimizations. It works by capturing +backward computations into an FX graph that can be compiled and optimized, +while maintaining the same semantics as eager mode autograd. +""" + +import contextlib +import functools +import itertools +import operator +import time +from collections import Counter, defaultdict +from collections.abc import Callable, Generator, Sequence +from typing import Any, Optional, TYPE_CHECKING, Union + +import torch +import torch.utils._pytree as pytree +from torch._dispatch.python import enable_python_dispatcher +from torch._dynamo.external_utils import ( + call_accumulate_grad, + call_backward, + call_hook, + FakeCompiledAutogradEngine, + unwrap_maybe_dynamic_int, +) +from torch._dynamo.source import GetItemSource, LocalSource +from torch._dynamo.utils import ( + counters, + get_chromium_event_logger, + lazy_format_graph_code, + set_locals_to_steal, +) +from torch._functorch._aot_autograd.runtime_wrappers import ( + AutogradLazyBackwardCompileInfo, + CachedAutogradLazyBackwardCompileInfo, +) +from torch._guards import compile_context, CompileContext, CompileId, Source +from torch._logging import getArtifactLogger, trace_structured +from torch._prims_common import clone_preserve_strides +from torch._subclasses import FakeTensorMode +from torch._subclasses.fake_tensor import FakeTensor +from torch.fx import GraphModule +from torch.fx.experimental._backward_state import BackwardState +from torch.fx.experimental.proxy_tensor import ( + decompose, + disable_autocast_cache, + disable_proxy_modes_tracing, + fetch_object_proxy, + ProxyTorchDispatchMode, + PythonKeyTracer, + track_tensor_tree, +) +from torch.fx.experimental.symbolic_shapes import DimDynamic, ShapeEnv +from torch.fx.traceback import preserve_node_meta, set_stack_trace +from torch.types import FloatLikeType, IntLikeType +from torch.utils._ordered_set import OrderedSet +from torch.utils._traceback import CapturedTraceback + + +if TYPE_CHECKING: + from torch.fx.proxy import Proxy + + +TURN_OFF_MSG = """You can turn off compiled autograd by either: +1. Moving the unsupported autograd call outside of the torch.compile'd region. +2. Wrapping the unsupported autograd call in the torch._dynamo.compiled_autograd._disable() context manager. +3. Setting torch._dynamo.config.compiled_autograd=False for the torch.compile call containing the unsupported autograd call. +4. Setting torch._dynamo.config.compiled_autograd=False at the start of the program.""" + +compiled_autograd_log = getArtifactLogger(__name__, "compiled_autograd") +verbose_log = getArtifactLogger(__name__, "compiled_autograd_verbose") + + +def snapshot_verbose_logging_enabled() -> bool: + return torch._logging._internal.log_state.is_artifact_enabled( + "compiled_autograd_verbose" + ) + + +def snapshot_cudagraph_enabled() -> bool: + return torch._inductor.config.triton.cudagraphs + + +def maybe_clone(x: Optional[torch.Tensor]) -> Optional[torch.Tensor]: + if x is not None: + return clone_preserve_strides(x) + return x + + +def extract_bw_module(CompiledFunction: Any) -> Callable[..., Any]: + if isinstance( + CompiledFunction._lazy_backward_info, AutogradLazyBackwardCompileInfo + ): + return CompiledFunction._lazy_backward_info.bw_module + elif isinstance( + CompiledFunction._lazy_backward_info, CachedAutogradLazyBackwardCompileInfo + ): + with torch._subclasses.fake_tensor.unset_fake_temporarily(): + return CompiledFunction._lazy_backward_info.bw_module_fn() + else: + raise AssertionError( + "Unexpected Lazy Backward Compilation Info Type. Please file an issue." + ) + + +# Note: [Anomaly Mode Semantics in Compiled Autograd] +# In the eager autograd engine, anomaly mode is able to detect NaNs +# after each node. This is useful, because the executed code with +# and without anomaly mode are the same. So assuming determinism, +# a NaN in regular mode should also happen in anomaly mode. +# +# With torch.compile, following eager semantics would require inserting +# runtime asserts to check for NaNs, which could prevent some fusions. +# This results in different code being run with and without anomaly mode. +# So different semantics are needed, this implementation below will check +# for NaNs at the end of the autograd call, instead of after each node +class NaNChecker: + def __init__(self, accumulate_grad: bool) -> None: + self.accumulate_grad = accumulate_grad + self.params_indices: list[int] = [] + self.params_to_check: dict[str, torch.Tensor] = {} + self.output_names: list[str] = [] + + def prep_with_graph(self, graph: torch.fx.Graph) -> None: + inputs_node = next(iter(graph.nodes)) + acc_grad_nodes = graph.find_nodes( + op="call_function", target=call_accumulate_grad + ) + output_nodes = graph.find_nodes(op="output")[0].args[0] + assert self.accumulate_grad == bool( + acc_grad_nodes + ) and self.accumulate_grad == (not output_nodes) + + for node in acc_grad_nodes: + param_node = node.args[0] + # AccumulateGrad always saves a reference to the param + # so Compiled Autograd will always lift the param and + # this should always be true + assert ( + param_node.target is operator.getitem + and param_node.args[0] is inputs_node # type: ignore[possibly-undefined] + and isinstance(param_node.args[1], int) + ) + self.params_indices.append(param_node.args[1]) + + self.output_names = [node.name for node in output_nodes] + + def prep_with_inputs(self, inputs: tuple[torch.Tensor, ...]) -> None: + if not self.accumulate_grad: + # Using .grad, nothing to prep + return + + # Using .backward, we must check existing grads on params if any + for idx in self.params_indices: + grad = inputs[idx].grad + if grad is not None: + assert not torch.isnan(grad).any(), ( + f"Compiled autograd running under anomaly mode with inputs[{idx}] already " + f"having NaN gradient. This is not supported. {TURN_OFF_MSG}" + ) + + self.params_to_check[f"inputs[{idx}]"] = inputs[idx] + + def check(self, out: tuple[torch.Tensor, ...]) -> None: + if self.accumulate_grad: + # Using .backward, graph outputs are empty + assert not out + nan_params: list[str] = [] + for inputs_str, param in self.params_to_check.items(): + assert param.grad is not None # not true for autograd.grad + if torch.isnan(param.grad).any(): + nan_params.append(inputs_str) + + if nan_params: + raise RuntimeError( + f"Compiled Autograd returned NaN gradients for parameters: {','.join(nan_params)}." + ) + else: + # Using .grad, graph outputs are grads + nan_grads: list[str] = [] + for i, grad in enumerate(out): + if torch.isnan(grad).any(): + nan_grads.append(self.output_names[i]) + + if nan_grads: + raise RuntimeError( + f"Compiled Autograd returned NaN gradients for output nodes: {','.join(nan_grads)}." + ) + + +# We lazily bind "functional backward" variants for PyTorch built-in autograd +# nodes to this class. Example: torch._dynamo.compiled_autograd.ops.MulBackward0 +# Each "functional backward" is bound the first time the node's apply_with_saved +# function is called. It's possible to avoid lazy binding and instead bind +# all of this upfront (perhaps at import time) via codegen changes. +class OpNamespace: + def __init__(self) -> None: + self.custom_function_name_counter: Counter[str] = Counter() + + def add( + self, + name: str, + fn: Callable[..., Any], + is_custom_function: bool, + is_traceable: bool, + ) -> str: + if is_custom_function: + name = "CppNode" + name + count = self.custom_function_name_counter[name] + self.custom_function_name_counter[name] += 1 + name = f"{name}{count}" + + assert not hasattr(self, name) + result = Op(name, fn, is_custom_function) + if is_traceable: + setattr(self, name, torch._dynamo.allow_in_graph(result)) + else: + # C++ autograd function was not marked as traceable + # Dynamo can't dry run it at compile time, so must fallback to eager + @torch._dynamo.disable # type: ignore[misc] + def run_non_traceable_cpp_in_eager(*args: Any, **kwargs: Any) -> Any: + return result(*args, **kwargs) + + setattr(self, name, run_non_traceable_cpp_in_eager) + return name + + def get(self, name: str) -> Any: + return getattr(self, name) + + +class Op: + def __init__( + self, name: str, fn: Callable[..., Any], is_custom_function: bool + ) -> None: + self.fn = fn + self.is_custom_function = is_custom_function + self.__name__ = name + self.__module__ = "torch._dynamo.compiled_autograd.ops" + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + return self.fn(*args, **kwargs) + + def __repr__(self) -> str: + return self.__module__ + "." + self.__name__ + + +ops = OpNamespace() + + +_graph_placeholders = ["inputs", "sizes", "scalars", "hooks", "packed_data"] +_impure_targets = OrderedSet( + [ + call_hook, + call_backward, + FakeCompiledAutogradEngine._exec_final_callbacks_stub, + call_accumulate_grad, + ] +) + +COMPILE_COUNTER = itertools.count() + + +def make_compile_context(compiled_autograd_id: int) -> Any: + return compile_context( + CompileContext( + CompileId( + compiled_autograd_id=compiled_autograd_id, + frame_id=None, + frame_compile_id=None, + ) + ) + ) + + +class AutogradCompilerInstance: + def __init__(self, compiler_fn: Callable[..., Any]) -> None: + self.compiler_fn = compiler_fn + self.stack = contextlib.ExitStack() + self.close = self.stack.close + self.shape_env = ShapeEnv() + self.fake_tensor_mode = FakeTensorMode( + allow_fallback_kernels=True, + allow_non_fake_inputs=True, + shape_env=self.shape_env, + ) + self.fx_tracer = PythonKeyTracer() + self.proxy_mode = ProxyTorchDispatchMode(self.fx_tracer, "symbolic") + self.hooks_proxy: Optional[Proxy] = None + + def wrap_fake(self, x: torch.Tensor, source: Optional[Source]) -> FakeTensor: + assert isinstance(x, torch.Tensor) + return self.fake_tensor_mode.from_tensor(x, source=source) + + @staticmethod + def source(name: str, idx: Any) -> GetItemSource: + return GetItemSource(LocalSource(name), idx) + + def begin_capture( + self, + inputs: list[torch.Tensor], + sizes: list[int], + scalars: list[Union[int, float]], + origins: list[list[tuple[int, str]]], + accumulate_grad: bool, + check_nans: bool, + ) -> tuple[str, list[torch.Tensor], list[IntLikeType], list[FloatLikeType]]: + counters["compiled_autograd"]["captures"] += 1 + self.id = next(COMPILE_COUNTER) + self.aot_id_counter: dict[int, int] = defaultdict(int) + self.compile_context = make_compile_context(self.id) + self.compile_context.__enter__() + self.nan_checker = NaNChecker(accumulate_grad) if check_nans else None + self.start_time_ns = time.time_ns() + get_chromium_event_logger().log_event_start( + "compiled_autograd", + self.start_time_ns, + {"graph_id": self.id}, + log_pt2_compile_event=True, + ) + self.fx_tracer.root = torch.nn.Module() + self.fx_tracer.graph = torch.fx.Graph(tracer_cls=PythonKeyTracer) + self.fx_tracer.tensor_attrs = {} + self.symnode_proxy_lookup = {} + ( + args_proxy, + self.sizes_proxy, + self.scalars_proxy, + self.hooks_proxy, + self.packed_data_proxy, + ) = ( + self.fx_tracer.create_proxy("placeholder", name, (), {}) + for name in _graph_placeholders + ) + + self.stack.enter_context(preserve_node_meta()) + inputs_origins, sizes_origins, scalars_origins = origins + + # Turn on PythonDispatcher during initial trace to make it identifiable + # that tracing is happening, which is needed to prevent hashing symints + self.stack.enter_context(enable_python_dispatcher()) + + # tensor inputs to fake tensors + x = inputs[0] # mypy will complain about unbound x + try: + for idx, x in enumerate(inputs): + inputs[idx] = self.wrap_fake(x, self.source("inputs", idx)) + except Exception as e: + raise NotImplementedError( + f"Found tensor of type {type(x)}, which is not supported by FakeTensorMode. {TURN_OFF_MSG}" + ) from e + self.bind_objects_to_proxies(inputs, args_proxy, inputs_origins) + + # size inputs to symints + sym_sizes = [ + self.shape_env.create_unspecified_symint_and_symbol( + val, + self.source("sizes", idx), + DimDynamic.DYNAMIC, + ) + for idx, val in enumerate(sizes) + ] + + # We want to mark every size as dynamic, but since there's no way to + # mark a primitive `int` as dynamic, we need to wrap it in a tensor. + # In the graph, we unwrap it with `unwrap_maybe_dynamic_int` back into a primitive. + proxies = [self.sizes_proxy[i] for i in range(len(sym_sizes))] # type: ignore[index] + for i, symint in enumerate(sym_sizes): + proxies[i] = self.fx_tracer.create_proxy( + "call_function", + unwrap_maybe_dynamic_int, + (proxies[i],), + {}, + ) + self.symnode_proxy_lookup[symint.node] = proxies[i] + proxies = self.bind_objects_to_proxies(sym_sizes, proxies, sizes_origins) + + for idx, val in enumerate(scalars): + source = self.source("scalars", idx) + if isinstance(val, int): + scalars[idx] = self.shape_env.create_unspecified_symint_and_symbol( + val, + source, + DimDynamic.DYNAMIC, + ) + elif isinstance(val, float): + scalars[idx] = self.shape_env.create_symfloatnode( + self.shape_env.create_unspecified_symbol( + val, + source=source, + dynamic_dim=DimDynamic.DYNAMIC, + ), + hint=val, + source=source, + ) + else: + raise AssertionError("Unexpected scalar type: ", type(val)) + self.bind_objects_to_proxies(scalars, self.scalars_proxy, scalars_origins) + for i, symval in enumerate(scalars): + self.symnode_proxy_lookup[symval.node] = self.scalars_proxy[i] # type: ignore[union-attr] + + # TODO(jansel): are all these modes needed? + self.stack.enter_context(decompose({})) + self.stack.enter_context(self.fake_tensor_mode) + self.stack.enter_context(self.proxy_mode) + self.stack.enter_context(disable_autocast_cache()) + # Needed to make sure we don't accidentally specialize any symbols + assert self.fake_tensor_mode.shape_env is not None + env = self.fake_tensor_mode.shape_env + self.stack.enter_context( + torch.fx.experimental.symbolic_shapes._suppress_guards(env) + ) + return ( + str(CompileContext.current_compile_id()), + inputs, + sym_sizes, + scalars, # type: ignore[return-value] + ) + + def log_compile_reasons( + self, + compile_reasons: list[str], + ) -> None: + assert compile_reasons + trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "compiled_autograd_compile_reasons", + "encoding": "json", + }, + payload_fn=lambda: compile_reasons, + ) + + def proxy_call_aot_backward( + self, + pinputs: Sequence[Any], + psaved_tensors: Sequence[torch.Tensor], + saved_tensors: Sequence[torch.Tensor], + pctx: Any, + ctx: Any, + maybe_backward_state_idx: Optional[int], + ) -> Sequence[Any]: + # The AOTBackward call consists of three things: the prologue, the + # backward graph, and the epilogue. + # Our strategy is: + # - allow_in_graph the prologue (in the CA graph and Dynamo graph), + # - copy-paste the backward graph into the CA graph so that CA passes and Dynamo can see it + # - trace directly through the epilogue. Anything that gets baked in is + # constant metadata (for example, metadata about the number of outputs, or removing + # RNG arguments or effect tokens). + # If Dynamo graph capture were better, then we could add a node for the prologue + # into the CA graph and have Dynamo trace into it. + + psymints = [self.to_proxy(e) for e in ctx._get_compiled_autograd_symints()] + + # NOTE: we should only close over constants + CompiledFunction = ctx._forward_cls + bw_module = extract_bw_module(CompiledFunction) + metadata = CompiledFunction.metadata + maybe_subclass_metadata = CompiledFunction.maybe_subclass_metadata + aot_id = CompiledFunction._aot_id + del CompiledFunction + + if torch.is_grad_enabled(): + for output_alias_info in metadata.output_info: + if output_alias_info.requires_grad: + raise RuntimeError( + "torch.compile does not currently support higher order gradients." + ) + + @torch._dynamo.allow_in_graph # type: ignore[misc] + def call_aot_bwd_prologue( + ctx_saved_tensors: Sequence[torch.Tensor], + ctx_symints: Sequence[IntLikeType], + *flat_args: Sequence[Any], + ) -> Any: + out = torch._functorch._aot_autograd.runtime_wrappers._backward_prologue_functional( + ctx_saved_tensors, + ctx_symints, + metadata, + maybe_subclass_metadata, + *flat_args, + ) + return out + + pgrads = self.fx_tracer.create_proxy( + kind="call_function", + target=call_aot_bwd_prologue, + args=( + psaved_tensors, + psymints, + *pinputs, + ), + kwargs={}, + ) + + pbackward_state = None + if maybe_backward_state_idx is not None: + pbackward_state = self.hooks_proxy[maybe_backward_state_idx] # type: ignore[index] + + # Copy-paste the AOT backward graph into the compiled autograd graph + def copy_paste_aot_backward_graph() -> list[torch.Tensor]: + def num_inputs(graph: torch.fx.Graph) -> int: + num_args = 0 + for node in graph.nodes: + if node.op == "placeholder": + num_args += 1 + continue + else: + break + return num_args + + # set up the proxy inputs to bw_module + # the calling convention is: [*symints, *args (primals and tangents), backward_state] + num_args = num_inputs(bw_module.graph) # type: ignore[attr-defined] + pall_args = [ + pgrads[i] for i in range(num_args - int(pbackward_state is not None)) + ] + # replace the symints with our symints + symints = ctx._get_compiled_autograd_symints() + assert len(symints) == len(ctx.symints) + psymints = [self.to_proxy(e) for e in symints] + pall_args[: len(symints)] = psymints + # Add backward_state + if pbackward_state is not None: + pall_args.append(pbackward_state) + + # run over all nodes of the aot_backward graph. + # copy and paste them all into the compiled autograd graph. + args_idx = 0 + value_remap = {} + poutputs: Optional[list[torch.fx.Proxy]] = None + + # names of nodes must appear only once in the fx.Graph + # dedup AOT backwards that appear multiple times + deduped_aot_id = str(aot_id) + if self.aot_id_counter[aot_id]: + deduped_aot_id += f"_{self.aot_id_counter[aot_id]}" + self.aot_id_counter[aot_id] += 1 + + def make_unique(node_name: str) -> str: + # make it both informative and unique + return f"aot{deduped_aot_id}_{node_name}" + + for node in bw_module.graph.nodes: # type: ignore[attr-defined] + if node.op == "placeholder": + ph = pall_args[args_idx].node + ph.name = make_unique(node.name) + value_remap[node] = ph + args_idx += 1 + elif node.op == "output": + assert len(node.args) == 1 + poutputs = [ + torch.fx.Proxy(value_remap[n], self.fx_tracer) + if isinstance(n, torch.fx.Node) + else n + for n in node.args[0] + ] + elif node.op == "get_attr": + name = node.target + qualname = self.fx_tracer.get_fresh_qualname(name) + setattr(self.fx_tracer.root, qualname, getattr(bw_module, name)) + result = self.fx_tracer.create_node("get_attr", qualname, (), {}) + result.name = make_unique(node.name) + value_remap[node] = result + elif node.op == "call_function": + if node.target is torch.ops.aten.view.default: + # this aot bwd graph is being lazily compiled + # we must manually apply the view_to_reshape post grad pass + # since it was already applied to the aot fwd, and baked into the gradients + node.target = torch.ops.aten.reshape.default + result = self.fx_tracer.graph.node_copy( + node, lambda n: value_remap[n] + ) + result.name = make_unique(node.name) + value_remap[node] = result + elif node.op == "call_module": + name = node.target + qualname = self.fx_tracer.get_fresh_qualname(name) + setattr(self.fx_tracer.root, qualname, getattr(bw_module, name)) + result = self.fx_tracer.graph.node_copy( + node, lambda n: value_remap[n] + ) + result.target = qualname + value_remap[node] = result + else: + raise AssertionError("shouldn't get here") + + assert poutputs is not None + + # In general we don't know what the shapes of the outputs are, so allocate + # some dummy sizes for them. + def dummy() -> torch.Tensor: + with disable_proxy_modes_tracing(): + return torch.zeros(0, 0, 0, 0, 123) + + outputs = [ + dummy() if isinstance(o, torch.fx.Proxy) else o for o in poutputs + ] + self.bind_objects_to_proxies(outputs, poutputs) + return outputs + + outputs = copy_paste_aot_backward_graph() + + def proxy_subclass_constructor( + subclass_meta: Any, is_runtime: bool, unwrapped_args: Sequence[Any] + ) -> torch.Tensor: + @torch._dynamo.allow_in_graph # type: ignore[misc] + def make_subclass(*unwrapped_args: Any) -> Any: + return subclass_meta.creation_fn(unwrapped_args, is_runtime=is_runtime) + + punwrapped_args = pytree.tree_map(self.to_proxy, unwrapped_args) + + poutput = self.fx_tracer.create_proxy( + kind="call_function", + target=make_subclass, + args=tuple(punwrapped_args), + kwargs={}, + ) + + output = self.allocate_dummy() + self.bind_objects_to_proxies([output], [poutput]) + return output + + results = torch._functorch._aot_autograd.runtime_wrappers._backward_epilogue_functional( + metadata, + maybe_subclass_metadata, + outputs, + make_subclass_override=proxy_subclass_constructor, + ) + presults = pytree.tree_map(self.to_proxy, results) + return presults + + def proxy_call_backward( + self, + inputs: Sequence[Any], + output_metadatas: Sequence[Any], + saved_tensors: Sequence[torch.Tensor], + backward_idx: int, + ctx: torch.autograd.function.BackwardCFunction, + maybe_backward_state_idx: Optional[int], + ) -> tuple[Optional[torch.Tensor], ...]: + assert self.hooks_proxy is not None + pctx = self.hooks_proxy[backward_idx] # type: ignore[index] + pinputs = self.to_proxy(inputs) + psaved_tensors = self.to_proxy(saved_tensors) + if hasattr(ctx._forward_cls, "_aot_id"): # type: ignore[attr-defined] + # AOT backward + proxies = self.proxy_call_aot_backward( + pinputs, + psaved_tensors, + saved_tensors, + pctx, + ctx, + maybe_backward_state_idx, + ) + else: + proxies = self.fx_tracer.create_proxy( + kind="call_function", + target=call_backward, + args=( + pctx, + psaved_tensors, + *pinputs, + ), + kwargs={}, + ) + assert proxies is not None + + with disable_proxy_modes_tracing(): + # create fake Tensors + grad_ins: list[Optional[torch.Tensor]] = [] + for idx, output_metadata in enumerate(output_metadatas): + if output_metadata is None or proxies[idx] is None: + grad_ins.append(None) + continue + + layout, device, dtype, size = output_metadata + grad_ins.append( + torch.empty(size=size, dtype=dtype, layout=layout, device=device) + ) + self.bind_objects_to_proxies(grad_ins, proxies) + return tuple(grad_ins) + + def call_copy_slices_prologue( + self, + inputs: Sequence[Any], + base_sizes: Sequence[Any], + base_strides: Sequence[Any], + base_storage_offset: Any, + view_sizes: Sequence[Any], + view_strides: Sequence[Any], + view_storage_offset: Any, + ) -> Sequence[torch.Tensor]: + args = ( + inputs, + self.to_proxy(base_sizes), + self.to_proxy(base_strides), + self.to_proxy(base_storage_offset), + self.to_proxy(view_sizes), + self.to_proxy(view_strides), + self.to_proxy(view_storage_offset), + ) + return self.proxy_call(copy_slices_prologue, args, [None] * 3) + + def call_copy_slices_epilogue( + self, + needs_input_grad: Sequence[bool], + result: torch.Tensor, + res: Sequence[Any], + grad_slice: torch.Tensor, + ) -> Sequence[torch.Tensor]: + return self.proxy_call( + copy_slices_epilogue, + (needs_input_grad, result, res, grad_slice), + [None] * len(needs_input_grad), + ) + + def allocate_dummy(self) -> torch.Tensor: + with disable_proxy_modes_tracing(): + # Weird quantity so it's easy to grep + return torch.zeros([0, 123456789]) + + def bind_function( + self, + fn_name: str, + fn: Callable[..., Any], + is_custom_function: bool, + is_traceable: bool, + ) -> str: + """Binds ops.fn_name = fn""" + return ops.add(fn_name, fn, is_custom_function, is_traceable) + + def apply_functional( + self, + fn_name: str, + grads: Sequence[Any], + args: Any, + output_metadata: Sequence[Any], + ) -> Sequence[torch.Tensor]: + """Proxies a call to ops.fn_name(grads, *args) into the graph""" + op = ops.get(fn_name) + return self.proxy_call(op, (grads, *args), output_metadata) + + def proxy_call( + self, fn: Callable[..., Any], args: Any, output_metadata: Sequence[Any] + ) -> Sequence[torch.Tensor]: + """Proxies a call to fn(*args) into the graph""" + proxy_args = pytree.tree_map(lambda e: self.to_proxy(e), args) + proxy_out = self.fx_tracer.create_proxy( + "call_function", fn, args=proxy_args, kwargs={} + ) + result = [self.allocate_dummy() for _ in output_metadata] + self.bind_objects_to_proxies(result, [proxy_out[i] for i in range(len(result))]) + return result + + def validate_outputs( + self, _: Any, outputs: Sequence[Any], args: Any, output_metadata: Sequence[Any] + ) -> Sequence[torch.Tensor]: + """Proxies a call to ops.validate_outputs(outputs, *args) into the graph""" + op = ops.get("validate_outputs") + proxy_args = pytree.tree_map(self.to_proxy, (outputs, *args)) + new_proxy_outputs = self.fx_tracer.create_proxy( + "call_function", op, args=proxy_args, kwargs={} + ) + assert len(output_metadata) == len(outputs) + self.bind_objects_to_proxies(outputs, new_proxy_outputs) + return outputs + + def accumulate(self, old_var: Any, new_var: Any) -> torch.Tensor: + old_var_proxy = self.to_proxy(old_var) + new_var_proxy = self.to_proxy(new_var) + proxy_out = self.fx_tracer.create_proxy( + "call_function", torch.add, args=(old_var_proxy, new_var_proxy), kwargs={} + ) + result = self.allocate_dummy() + self.bind_objects_to_proxies([result], [proxy_out]) + return result + + def accumulate_grad( + self, variable: torch.Tensor, grad: torch.Tensor, has_post_hooks: bool + ) -> None: + self.fx_tracer.create_proxy( + "call_function", + call_accumulate_grad, + args=( + self.to_proxy(variable), + self.to_proxy(grad), + has_post_hooks, + ), + kwargs={}, + ) + + def proxy_call_hook( + self, hook: Callable[..., Any], *args: Any, **kwargs: Any + ) -> torch.fx.Proxy: + return self.fx_tracer.create_proxy( + "call_function", + call_hook, + ( + hook, + *[self.to_proxy(x) for x in args], + ), + kwargs, + ) + + def unpack_hook(self, hook_id: int, data_id: int) -> torch.Tensor: + assert self.hooks_proxy is not None + hook = self.hooks_proxy[hook_id] # type: ignore[index] + data = self.packed_data_proxy[data_id] # type: ignore[index] + proxy = self.proxy_call_hook( + hook, + data, + hook_type="unpack_hook", + ) + out = self.allocate_dummy() + self.bind_objects_to_proxies([out], [proxy]) + return out + + def tensor_pre_hook( + self, inputs: list[torch.Tensor], hook_id: int, i: int + ) -> list[torch.Tensor]: + assert self.hooks_proxy is not None + hook = self.hooks_proxy[hook_id] # type: ignore[index] + proxy = self.proxy_call_hook( + hook, + inputs[i], + hook_type="tensor_pre_hook", + ) + with disable_proxy_modes_tracing(): + inputs[i] = maybe_clone(inputs[i]) # type: ignore[assignment] + self.bind_objects_to_proxies([inputs[i]], [proxy]) + return inputs + + def cpp_tensor_pre_hook( + self, inputs: list[torch.Tensor], hook_id: int, i: int + ) -> list[torch.Tensor]: + proxy = self.fx_tracer.create_proxy( + "call_function", + torch._C._dynamo.compiled_autograd.call_cpp_tensor_pre_hooks, + (hook_id, self.to_proxy(inputs[i])), + {}, + ) + with disable_proxy_modes_tracing(): + inputs[i] = maybe_clone(inputs[i]) # type: ignore[assignment] + self.bind_objects_to_proxies([inputs[i]], [proxy]) + return inputs + + def pre_hook(self, inputs: Sequence[Any], hook_id: int) -> list[torch.Tensor]: + assert self.hooks_proxy is not None + hook = self.hooks_proxy[hook_id] # type: ignore[index] + proxies = self.proxy_call_hook( + hook, + inputs, + hook_type="pre_hook", + ) + with disable_proxy_modes_tracing(): + inputs = [maybe_clone(x) for x in inputs] + self.bind_objects_to_proxies(inputs, proxies) + return inputs + + def post_hook( + self, outputs: list[torch.Tensor], inputs: Sequence[torch.Tensor], hook_id: int + ) -> list[torch.Tensor]: + assert self.hooks_proxy is not None + hook = self.hooks_proxy[hook_id] # type: ignore[index] + proxies = self.proxy_call_hook( + hook, + outputs, + inputs, + hook_type="post_hook", + ) + with disable_proxy_modes_tracing(): + outputs = [maybe_clone(x) for x in outputs] # type: ignore[misc] + self.bind_objects_to_proxies(outputs, proxies) + return outputs + + def post_acc_grad_hook( + self, input: torch.Tensor, hook_id: int + ) -> list[torch.Tensor]: + assert isinstance(input, torch.Tensor) + assert self.hooks_proxy is not None + hook = self.hooks_proxy[hook_id] # type: ignore[index] + proxy = self.proxy_call_hook( + hook, + input, + hook_type="post_acc_grad_hook", + ) + with disable_proxy_modes_tracing(): + res = [maybe_clone(input)] + self.bind_objects_to_proxies(res, [proxy]) + return res # type: ignore[return-value] + + # Note: [Compiled autograd and cudagraphs] + # Eager autograd backward implements scalars as 0-dim tensors, see DivBackward0::other_. + # When compiled autograd traces those nodes, it lifts the scalar tensors, resulting in a graph + # with some cpu 0-dim tensor inputs. To prevent the entire graph from skipping cudagraph, we move the + # scalars tensors to cuda. This works because ATen/prims ops will accept cuda 0-dim tensors too. + def move_graph_nodes_to_cuda(self, graph: torch.fx.Graph) -> list[int]: + to_move: dict[int, torch.fx.Node] = {} + has_cuda_inputs = False + nodes = list(graph.nodes) + assert nodes[0].target == "inputs" + inputs = nodes[0] + inputs_users = list(inputs.users.keys()) + # input access nodes should immediately follow placeholder nodes + first_getitem_idx = len(_graph_placeholders) + assert nodes[first_getitem_idx] == inputs_users[0] + last_getitem_idx = first_getitem_idx + len(inputs_users) - 1 + assert nodes[last_getitem_idx] == inputs_users[-1] + # getitem nodes on inputs + for i, node in enumerate(inputs_users): + if not has_cuda_inputs and node.meta["val"].device.type == "cuda": + has_cuda_inputs = True + continue + + is_cpu = node.meta["val"].device.type == "cpu" + is_scalar = len(node.meta["val"].size()) == 0 + if is_cpu and is_scalar: + node_users = list(node.users.keys()) + # We can only move the cpu scalar if it is not exposed to user code. + if all( + ( + isinstance(user.target, torch._ops.OpOverload) + and user.target.namespace in ("prims", "aten") + ) + or ( + isinstance(user.target, Op) + and not user.target.is_custom_function + ) + for user in node_users + ): + # all users are prims/aten, can move safely + to_move[i] = node + + # only move cpu scalars to cuda if there were cuda activations in this graph, + # this is to handle the case where cudagraphs is enabled on a cpu-only graph + if has_cuda_inputs: + for node in to_move.values(): + verbose_log.debug("Moving node %s from cpu to cuda", node) + node.meta["val"] = node.meta["val"].cuda() + + # return runtime indices we need to move to cuda + return list(to_move.keys()) + + return [] + + def is_sym_node(self, node: Any) -> bool: + return ( + isinstance(node, torch.fx.Node) + and node.op == "call_function" + and node.target + in [torch.ops.aten.sym_size.int, torch.ops.aten.sym_numel.default] + ) + + def dce(self) -> None: + # Most of these removed nodes would have been removed during Dynamo and AOTDispatch + # Remove some of these nodes earlier to improve compilation speed + + # Dynamo guards will error instead of creating aliasing guards unless we unpack them in the graph + unpack_nodes: OrderedSet[torch.fx.Node] = OrderedSet() + i: int | None = None + for i, node in enumerate(self.fx_tracer.graph.find_nodes(op="placeholder")): # noqa: B007 + unpack_nodes.update(node.users.keys()) + assert i == len(_graph_placeholders) - 1 + + def is_impure(node: torch.fx.Node) -> bool: + if node in unpack_nodes or ( + node.op == "call_function" and node.target in _impure_targets + ): + return True + return node.is_impure() + + before = len(self.fx_tracer.graph.nodes) + self.fx_tracer.graph.eliminate_dead_code(is_impure) + after = len(self.fx_tracer.graph.nodes) + verbose_log.debug("DCE removed %d nodes", before - after) + + def remove_unused_sizes(self) -> set[int]: + used_sizes = [] + unused_sizes = [] + + # seek placeholder, should be at nodes[1] + it = iter(self.fx_tracer.graph.nodes) + next(it) + sizes_node = next(it) + assert sizes_node.name == "sizes" + + for getitem_node in sizes_node.users: + assert getitem_node.target is operator.getitem + if getitem_node.users: + used_sizes.append(getitem_node) + else: + # remove from the graph + unused_sizes.append(getitem_node) + + used_sizes_idx: set[int] = set() + for used in used_sizes: + assert isinstance(used.args, tuple) + assert used.args[0] == sizes_node + assert isinstance(used.args[1], int) + next_size_idx = len(used_sizes_idx) + # used later reindex the runtime sizes arg + used_sizes_idx.add(used.args[1]) + # reindex the graph + used.args = (used.args[0], next_size_idx) + + for unused in unused_sizes: + self.fx_tracer.graph.erase_node(unused) + + return used_sizes_idx + + def create_graph_module(self, id: str) -> GraphModule: + return GraphModule(self.fx_tracer.root, self.fx_tracer.graph, id) + + def end_capture(self, outputs: Any) -> tuple[Callable[..., Any], Any]: + self.fx_tracer.create_proxy( + "call_function", + FakeCompiledAutogradEngine._exec_final_callbacks_stub, + (), + {}, + ) + self.stack.close() + self.fx_tracer.create_node( + "output", + "output", + (self.fx_tracer.create_arg(self.to_proxy(outputs)),), + {}, + ) + runtime_inputs_to_move: list[int] = [] + if snapshot_cudagraph_enabled(): + runtime_inputs_to_move = self.move_graph_nodes_to_cuda(self.fx_tracer.graph) + + # We traced using dummy tensors. Delete all the metadata of the dummy tensors. + # It's probably better to refactor this class to use a different tracer + # than the make_fx tracer, but that is a larger change. + for node in self.fx_tracer.graph.nodes: + for field in ["tensor_meta", "example_value", "val"]: + if field in node.meta: + del node.meta[field] + + trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "compiled_autograd_graph_pre_reordering", + "encoding": "string", + }, + payload_fn=lambda: GraphModule( + self.fx_tracer.root, + self.fx_tracer.graph, + f"CompiledAutograd{self.id}PreReordering", + ).print_readable(print_output=False), + ) + self.delay_unpack_hook_nodes() + self.reorder_tensor_pre_hook_nodes() + self.reorder_pre_hook_nodes_to_schedule_asap() + self.reorder_accumulate_grad_nodes() + self.reorder_pre_hook_nodes_to_mimic_eager() + self.reorder_post_acc_grad_hook_nodes() + self.reorder_post_hook_nodes() + # TODO(yf225): work around: remove dead codes like `sym_size` and `sym_numel` which are not used downstream. e.g. + # ``` + # sym_numel_default = torch.ops.aten.sym_numel.default(sum_109); sum_109 = None + # eq_115 = 16 == sym_numel_default; sym_numel_default = eq_115 = None + # sym_size_int_39 = torch.ops.aten.sym_size.int(getitem_112, 1); getitem_112 = None + # eq_116 = 16 == sym_size_int_39; eq_116 = None + # eq_117 = 16 == sym_size_int_39; sym_size_int_39 = eq_117 = None + # ``` + # Proper fix is Richard's Python compiled autograd effort which will avoid calling make_fx and + # should prevent these ops from going into the CA graph. + self.dce() + if self.nan_checker: + self.nan_checker.prep_with_graph(self.fx_tracer.graph) + + # keep only sizes that are actually used in the graph + used_sizes_idx = self.remove_unused_sizes() + + graph = self.create_graph_module(f"CompiledAutograd{self.id}") + set_locals_to_steal(graph, ["inputs"]) + lazy_graph_code = lazy_format_graph_code( + "Compiled autograd graph", + graph, + include_device=True, + include_stride=True, + colored=True, + ) + compiled_autograd_log.info("%s", lazy_graph_code) + verbose_log.debug("%s", lazy_graph_code) + trace_structured( + "compiled_autograd_graph", + payload_fn=lambda: graph.print_readable(print_output=False), + ) + + def runtime_wrapper( + compiled_fn: Callable[..., Any], + inputs: Any, + sizes: Any, + scalars: Any, + hooks: Any, + packed_inputs: Any, + ) -> tuple[Any, Any]: + global in_compiled_autograd_region + try: + in_compiled_autograd_region = True + + if self.nan_checker: + self.nan_checker.prep_with_inputs(inputs) + + filtered_sizes = [] + for idx, integer in enumerate(sizes): + if idx in used_sizes_idx: + # can't create negative size + if integer > 0: + filtered_sizes.append(torch.empty(0, integer)) + torch._dynamo.maybe_mark_dynamic(filtered_sizes[-1], 1) + else: + filtered_sizes.append(integer) + + for i in runtime_inputs_to_move: + inputs[i] = inputs[i].pin_memory().cuda(non_blocking=True) + + with _disable(), make_compile_context(self.id): + out = compiled_fn( + inputs, filtered_sizes, scalars, hooks, packed_inputs + ) + if self.nan_checker: + self.nan_checker.check(out) + return out + finally: + in_compiled_autograd_region = False + + get_chromium_event_logger().log_event_end( + "compiled_autograd", + time.time_ns(), + {"graph_id": self.id}, + self.start_time_ns, + log_pt2_compile_event=True, + ) + self.compile_context.__exit__(None, None, None) + return runtime_wrapper, self.compiler_fn(graph) + + @staticmethod + def get_all_nodes(args: Sequence[Any]) -> list[torch.fx.Node]: + # filter out non-Node args, like None + nodes = [n for n in args if type(n) is torch.fx.Node] + return nodes + + @staticmethod + def is_placeholder(node: torch.fx.Node) -> bool: + if node.op == "placeholder" or ( + node.op == "call_function" + and node.target is operator.getitem + and node.args[0].op == "placeholder" # type: ignore[union-attr, arg-type] + ): + return True + return False + + def reorder_accumulate_grad_nodes(self) -> None: + """ + Usage of AOTAutograd causes all the accumulate_grad_ nodes to get pushed to the end of + the graph. This differs from eager mode, which schedules them as soon as possible. This + pass attempts to reorder the graph to mimic eager behavior. + """ + for node in self.fx_tracer.graph.find_nodes( + op="call_function", target=call_accumulate_grad + ): + param_node, grad_node = node.args[0], node.args[1] + getitem_node = None + if grad_node.target is operator.getitem: + getitem_node = grad_node + grad_node = getitem_node.args[0] + + arg = max([param_node, grad_node]) # last arg + if arg is not node.prev and not self.is_placeholder(arg): + arg.append(node) + if getitem_node is not None: + arg.append(getitem_node) + + def delay_unpack_hook_nodes(self) -> None: + """ + We can delay unpack hooks until they are needed, even later than in the eager autograd engine. + """ + for node in self.fx_tracer.graph.find_nodes( + op="call_function", target=call_hook + ): + if node.kwargs.get("hook_type", None) != "unpack_hook": + continue + + first_user = min(node.users) + first_user.prepend(node) + + def reorder_tensor_pre_hook_nodes(self) -> None: + """ + Usage of AOTAutograd causes all the tensor_pre_hook nodes to get pushed + to the end of the graph. This differs from eager mode, which schedules + them as soon as possible. This pass attempts to reorder the graph to + mimic eager behavior. + """ + for node in self.fx_tracer.graph.find_nodes( + op="call_function", target=call_hook + ): + if node.kwargs.get("hook_type", None) != "tensor_pre_hook": + continue + + getitem_node = node.args[0] + input_node = node.args[1] # tensor_pre_hook handle only one grad tensor + + if input_node is not node.prev and not self.is_placeholder(input_node): + input_node.append(getitem_node) + getitem_node.append(node) + + def reorder_pre_hook_nodes_to_schedule_asap(self) -> None: + """ + In this function, we schedule the pre hooks as soon as possible. This + does not match eager behavior (schedule pre hook right before its + registered node), but it can make acc grad be scheduled properly when + the pre hooks are registered to them. After reordering acc grad node, we + will reorder the pre hooks again to mimic eager behavior. + """ + for node in self.fx_tracer.graph.find_nodes( + op="call_function", target=call_hook + ): + if node.kwargs.get("hook_type", None) != "pre_hook": + continue + + getitem_node = node.args[0] + # pre_hook handle a tuple of grad tensors + input_nodes = self.get_all_nodes(node.args[1]) + + to_remove = [] + to_append = [] + hook_block = [node] # contain the hook and hook args getitem + for n in input_nodes: + if n.op == "call_function" and n.target is operator.getitem: + to_append.append(n.args[0]) + to_remove.append(n) + hook_block.append(n) + for a, b in zip(to_remove, to_append): + input_nodes.remove(a) + input_nodes.append(b) # type: ignore[arg-type] + + arg = max(input_nodes) # last input + if arg is not node.prev and not self.is_placeholder(arg): + arg.append(getitem_node) + for n in hook_block: + getitem_node.append(n) + + def reorder_pre_hook_nodes_to_mimic_eager(self) -> None: + """ + Usage of AOTAutograd causes all the pre_hook nodes to get pushed to the + end of the graph. This differs from eager mode, which schedules them + right before their registered node execution. This pass attempts to + reorder the graph to mimic eager behavior. + """ + pre_hooks = [] + for node in self.fx_tracer.graph.find_nodes( + op="call_function", target=call_hook + ): + if node.kwargs.get("hook_type", None) != "pre_hook": + continue + pre_hooks.append(node) + + for node in reversed(pre_hooks): + hook_getitem_node = node.args[0] + + users = list(node.users.keys()) + if len(users) == 0: + continue + + # users are all getitem ops and they are used by same registered node + assert all( + user.op == "call_function" and user.target is operator.getitem + for user in users + ) + registered_node = next(iter(users[0].users.keys())) + + if registered_node is not node.next: + registered_node.prepend(hook_getitem_node) + registered_node.prepend(node) + for getitem in users: + registered_node.prepend(getitem) + + def reorder_post_acc_grad_hook_nodes(self) -> None: + """ + Usage of AOTAutograd causes all the post_acc_grad_hook nodes to get + pushed to the end of the graph. This differs from eager mode, which + schedules them as soon as possible. This pass attempts to reorder the + graph to mimic eager behavior. + """ + post_acc_grad_hooks = [] + for node in self.fx_tracer.graph.find_nodes( + op="call_function", target=call_hook + ): + if node.kwargs.get("hook_type", None) != "post_acc_grad_hook": + continue + post_acc_grad_hooks.append(node) + + # nodes in post_acc_grad_hooks are in topo order. For hooks registered + # to same node, we should keep their relative order + for node in reversed(post_acc_grad_hooks): + getitem_node = node.args[0] + param_node = node.args[1] # post_acc_grad_hook handle one param + + # find the corresponding acc_grad node + acc_grad_node = None + for n in list(param_node.users.keys()): + if n.op == "call_function" and n.target is call_accumulate_grad: + acc_grad_node = n + break + + assert acc_grad_node is not None, ( + "post_acc_grad_hook must have corresponding acc grad node" + ) + + # append post_acc_grad_hook after acc_grad node + acc_grad_node.append(getitem_node) + getitem_node.append(node) + + def reorder_post_hook_nodes(self) -> None: + """ + Usage of AOTAutograd causes all the post_hook nodes to get pushed to the + end of the graph. This differs from eager mode, which schedules them as + soon as possible. This pass attempts to reorder the graph to mimic eager + behavior. + """ + post_hooks = [] + for node in self.fx_tracer.graph.find_nodes( + op="call_function", target=call_hook + ): + if node.kwargs.get("hook_type", None) != "post_hook": + continue + post_hooks.append(node) + + for node in reversed(post_hooks): + getitem_node = node.args[0] + output_nodes = node.args[1] + input_nodes = node.args[2] + + if len(output_nodes) > 0: + continue + + input_nodes_and_users = [] + input_nodes_and_users.extend(list(input_nodes)) + for input_node in input_nodes: + input_nodes_and_users.extend( + user + for user in list(input_node.users.keys()) + if not ( + user.op == "call_function" + and user.target is call_hook + and node.kwargs.get("hook_type", None) == "post_hook" + ) + ) + + arg = max(input_nodes_and_users) # last input users + if arg.op == "call_function" and arg.target is call_accumulate_grad: + param_node = arg.args[0] + post_acc_grad_hook_node = None + for n in list(param_node.users.keys()): + if ( + n.op == "call_function" + and n.target is call_hook + and n.kwargs.get("hook_type", None) == "post_acc_grad_hook" + ): + post_acc_grad_hook_node = n + + if post_acc_grad_hook_node is not None: + post_acc_grad_hook_node.append(getitem_node) + getitem_node.append(node) + continue + + if arg is not node.prev and not self.is_placeholder(arg): + arg.append(getitem_node) + getitem_node.append(node) + + def to_proxy(self, t: Any) -> Any: + if t is None: + return None + if isinstance(t, list): + return [self.to_proxy(x) for x in t] + if isinstance(t, tuple): + return tuple(self.to_proxy(x) for x in t) + if isinstance(t, (torch.SymInt, torch.SymFloat)): + return self.symnode_proxy_lookup[t.node] + if not isinstance(t, torch.Tensor): + # constant types like device, dtype, str + return t + proxy_tensor = fetch_object_proxy(self.fx_tracer, t) + assert isinstance(proxy_tensor, torch.fx.experimental.proxy_tensor._ProxyTensor) + return proxy_tensor.proxy + + def bind_objects_to_proxies( + self, + objects: Sequence[Any], + proxies: Any, + origins: Optional[list[tuple[int, str]]] = None, + ) -> Sequence[Any]: + if isinstance(proxies, torch.fx.Proxy): + if origins: + assert len(origins) == len(objects) + bound_proxies = [] + for i in range(len(objects)): + nodecall_index, node_name = origins[i] + self.set_node_origin(node_name, nodecall_index, None) + bound_proxies.append(proxies[i]) # type: ignore[index] + proxies = bound_proxies + else: + proxies = [proxies[i] for i in range(len(objects))] # type: ignore[index] + + assert len(objects) == len(proxies) + track_tensor_tree(objects, proxies, constant=None, tracer=self.fx_tracer) + return proxies + + def bind_backward_state(self, index: int) -> BackwardState: + assert self.hooks_proxy is not None + proxy = self.hooks_proxy[index] # type: ignore[index] + bw_state = BackwardState() + track_tensor_tree(bw_state, proxy, constant=None, tracer=self.fx_tracer) + return bw_state + + def set_node_origin( + self, + node_name: str, + nodecall_index: int, + pyobj: Optional[torch.autograd.Function], + ) -> None: + maybe_aot_id = "" + if pyobj is not None: + forward_cls = pyobj._forward_cls # type: ignore[attr-defined] + if hasattr(forward_cls, "_aot_id"): + # backward was created by AOT Dispatcher + if forward_cls._lazy_backward_info is None: + raise RuntimeError( + """This compiled backward function was saved by AOTAutogradCache, which does not support + compiled autograd. Please turn off AOTAutogradCache using `TORCHINDUCTOR_AUTOGRAD_CACHE=0`.""" + ) + maybe_aot_id = forward_cls._aot_id + new_code = f"{node_name}{maybe_aot_id} (NodeCall {nodecall_index})" + raw_stack_trace = CapturedTraceback.extract().format()[-1] + new_stack_trace = raw_stack_trace.replace( + "raw_stack_trace = CapturedTraceback.extract().format()[-1]", new_code + ) + set_stack_trace(new_stack_trace) + + +# state of the autograd engine dispatch, kept in sync by enable/disable context managers +compiled_autograd_enabled = False + +# global flag to check if compiled autograd is enabled but Dynamo stance is "force_eager" +compiled_autograd_enabled_force_eager = False + +# global flag to check if we are processing graphs produced from a compiled autograd graph +in_compiled_autograd_region = False + +active_disable_ctx = False + +depth = 0 + + +@contextlib.contextmanager +def _enable( + compiler_fn: Callable[..., Any], + dynamic: bool = True, + ignore_active_disable_ctx: bool = True, +) -> Generator[None, None, None]: + # The entrypoint to enable CA. + # It is recommended to enable via `torch._dynamo.config.compiled_autograd = True` rather + # than using this context manager directly. If you are torch.compiling the corresponding + # forward pass, make sure they are wrapped under this context as well. + # + # Example: + # def train(model, inputs, target): + # compiled_model = torch.compile(model) + # pred = compiled_model(data) + # loss = compute_loss(pred, target) + # loss.backward() + # + # with _enable(compiler_fn): + # train(model, inputs, target) + # + # Inputs: + # - compiler_fn: The wrapper that will consume the compiled autograd graph, e.g. `torch.compile` + # - dynamic: Whether compiled autograd will treat tensors in the autograd graph (params, activations) as dynamic. + # This doesn't affect the dynamic configuration of the compilation wrapper. + + if not ignore_active_disable_ctx and active_disable_ctx: + yield + else: + if dynamic: + assert type(dynamic) is bool + + from torch._dynamo import eval_frame + + if eval_frame._stance.stance == "force_eager": + # If user explicitly sets Dynamo stance to "force_eager", we want Compiled Autograd + # to fall back to eager as well. + global compiled_autograd_enabled_force_eager + compiled_autograd_enabled_force_eager = True + try: + yield + finally: + compiled_autograd_enabled_force_eager = False + else: + # we need to import this, because user might not have imported it if they directly use this context manager + # we need to lazily import it, because of circular dependencies + if torch.cuda.is_available(): + from torch._inductor import cudagraph_trees # noqa: F401 + + ( + prior_compiler, + prior_dynamic, + ) = torch._C._dynamo.compiled_autograd.set_autograd_compiler( + functools.partial(AutogradCompilerInstance, compiler_fn), dynamic + ) + if snapshot_verbose_logging_enabled(): + torch._C._dynamo.compiled_autograd.set_verbose_logger(verbose_log) # type:ignore[arg-type] + global compiled_autograd_enabled + compiled_autograd_enabled = True + global depth + prior_depth = depth + depth += 1 + try: + with torch.autograd.set_multithreading_enabled(False): + yield + finally: + if not prior_compiler: + compiled_autograd_enabled = False + torch._C._dynamo.compiled_autograd.set_autograd_compiler( + prior_compiler, prior_dynamic + ) + depth -= 1 + assert depth == prior_depth, ( + "Nested Compiled Autograd Contexts must return before their parent context" + ) + + +@contextlib.contextmanager +def _disable() -> Generator[None, None, None]: + ( + prior_compiler, + prior_dynamic, + ) = torch._C._dynamo.compiled_autograd.set_autograd_compiler(None, False) + global compiled_autograd_enabled + compiled_autograd_enabled = False + global active_disable_ctx + if not active_disable_ctx: + active_disable_ctx = True + try: + yield + finally: + if prior_compiler: + compiled_autograd_enabled = True + active_disable_ctx = False + torch._C._dynamo.compiled_autograd.set_autograd_compiler( + prior_compiler, prior_dynamic + ) + + +# return to starting state of a new process +def reset() -> None: + global compiled_autograd_enabled + compiled_autograd_enabled = False + assert not in_compiled_autograd_region + torch._C._dynamo.compiled_autograd.set_autograd_compiler(None, False) + torch._C._dynamo.compiled_autograd.set_verbose_logger(None) + torch._C._dynamo.compiled_autograd.clear_cache() + global COMPILE_COUNTER + COMPILE_COUNTER = itertools.count() + + +# Reimplementation of part of CopySlices::apply in Python. +# The shared code is really similar so we're not going to try to deduplicate. +def copy_slices_prologue( + inputs: Sequence[torch.Tensor], + base_sizes: Sequence[IntLikeType], + base_strides: Sequence[IntLikeType], + base_storage_offset: IntLikeType, + view_sizes: Sequence[IntLikeType], + view_strides: Sequence[IntLikeType], + view_storage_offset: IntLikeType, +) -> list[torch.Tensor]: + grad = inputs[0] + result = grad.new_empty_strided(base_sizes, base_strides) + assert grad is not None + result.copy_(grad) + offset = view_storage_offset - base_storage_offset + grad_slice = result.as_strided(view_sizes, view_strides, offset) + return [result, grad_slice, grad_slice.clone(memory_format=torch.contiguous_format)] + + +# Reimplementation of part of CopySlices::apply in Python. +# The shared code is really similar so we're not going to try to deduplicate. +def copy_slices_epilogue( + needs_input_grad: Sequence[bool], + result: torch.Tensor, + res: Sequence[Optional[torch.Tensor]], + grad_slice: torch.Tensor, +) -> list[Optional[torch.Tensor]]: + grad_inputs: list[Optional[torch.Tensor]] = [None] * len(needs_input_grad) + for i in range(len(needs_input_grad)): + if needs_input_grad[i]: + if res[i] is None: + continue + if i == 0: + to_copy = res[i] + assert to_copy is not None + grad_slice.copy_(to_copy) + grad_inputs[i] = result + else: + grad_inputs[i] = res[i] + return grad_inputs diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/comptime.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/comptime.py new file mode 100644 index 0000000000000000000000000000000000000000..f53c753365b6366100ca557797f416be30810458 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/comptime.py @@ -0,0 +1,443 @@ +""" +This module provides the public comptime interface to TorchDynamo, enabling users to execute +arbitrary Python code during symbolic evaluation of their programs. + +The comptime interface allows inspection and modification of TorchDynamo's compilation +process while it is running. This can be useful for: + +- Debugging compilation issues +- Inspecting intermediate state +- Adding custom guards or graph breaks +- Analyzing symbolic shapes and values + +Example usage: + + import torch + from torch._dynamo.comptime import comptime + + def my_model(x): + # Print the compile-time known information about x + comptime.print(x) + + # Print the current FX graph being constructed + comptime.print_graph() + + # Force a value to be treated as static + if comptime(lambda ctx: ctx.get_local("x").is_dynamic()): + comptime.force_static(x) + + # Add a manual graph break + comptime.graph_break() + +Note: While this API provides significant flexibility, it intentionally avoids +exposing internal implementation details of TorchDynamo to maintain compatibility +across versions. +""" + +import builtins +import dis +import time +import traceback +from collections.abc import Callable, Sequence +from typing import Any, Optional, TextIO, Union + +import torch +from torch._dynamo.symbolic_convert import InstructionTranslatorBase +from torch._dynamo.variables.base import VariableTracker +from torch._subclasses.fake_tensor import FakeTensor +from torch.fx.experimental.symbolic_shapes import free_symbols + +from .exc import unimplemented +from .variables import CellVariable +from .variables.tensor import SymNodeVariable + + +class ComptimeVar: + """ + A ComptimeVar represents a Python value, at some particular point + in time, in the Python code we are symbolically evaluating with + torchdynamo. This must be distinguished from a runtime value, as + at compile-time there are some properties of the variable we + do not know (for example, if the ComptimeVar represents a Tensor, + we only know metadata about the tensor; we do NOT know what the + actual data in the Tensor is.) + """ + + def __init__(self, v: VariableTracker) -> None: + self.__variable = v + + def as_proxy(self) -> Union[VariableTracker, Sequence[VariableTracker]]: + """ + Returns an fx.Proxy (or tuple/list of fx.Proxy) representing + this variable in the FX graph we are assembling to pass + to the user compiler. + + This method only works for variables we actually track in + the FX graph, aka Tensors (and ints, if you are compiling + with dynamic shapes). In particular, if you have a list + or tuple of tensors, you will get a list/tuple of proxies + (not a single proxy representing the entire list/tuple). + """ + return self.__variable.as_proxy() + + def is_proxy(self) -> bool: + """ + Returns True if as_proxy() would succeed. + """ + return self.__variable.is_proxy() + + def as_fake(self) -> Union[FakeTensor, torch.SymInt]: + """ + Returns a "fake" value (either a FakeTensor or a SymInt) + representing the variable in question. This only works + for variables that denote Tensor or int. You can use + this to query metadata; e.g., v.as_fake().size(0) will + tell you the compile-time known size of the tensor. + + WARNING: Do NOT mutate the returned tensor. + """ + return self.__variable.as_proxy().node.meta["example_value"] + + def size(self, dim: Optional[int] = None) -> Union[int, torch.SymInt]: + """ + Returns the size of the tensor (if dim is None) or the size + at the dimension dim. The returned size may be a SymInt. + """ + return self.as_fake().size(dim) # type: ignore[union-attr, return-value] + + def python_type(self) -> type: + """ + Returns what type(v) would have returned for the variable + at compile time. + """ + return self.__variable.python_type() + + def as_python_constant(self) -> Any: + """ + Returns the Python value this variable would have, but only if it is + completely known at compile-time (e.g., it is constant). + + WARNING: Do NOT mutate the returned constant. The returned constant + may or may not correspond to the actual value this variable may take + on at runtime; for example, if the variable in question is a constant + list, we may return a copy of that list. + """ + return self.__variable.as_python_constant() + + def is_python_constant(self) -> bool: + """ + Returns True if as_python_constant would succeed. + """ + return self.__variable.is_python_constant() + + def is_dynamic(self) -> bool: + if isinstance(self.__variable, SymNodeVariable): + fs = free_symbols(self.__variable.sym_num) + return bool(fs) + return False + + def force_static(self) -> None: + """ + Forces that a value is static, inducing a guard on its specific value + """ + if isinstance(self.__variable, SymNodeVariable): + self.__variable.evaluate_expr() + elif self.__variable.is_python_constant(): + # TODO: Maybe complain if this isn't a int/bool/float variable + pass + else: + raise AssertionError( + f"cannot force {self.__variable} ({type(self.__variable)}) static" + ) + + def _i_will_not_complain_if_bc_breaks_VariableTracker(self) -> VariableTracker: + """ + Returns the internal data structure VariableTracker that Dynamo uses + to represent variables at compile time. There are no BC guarantees on + this API and WE RESERVE THE RIGHT TO BREAK YOUR CODE if you rely on + it. + """ + return self.__variable + + def __repr__(self) -> str: + return self.__variable.debug_repr() + + # TODO: API for adding a custom guard + + +class ComptimeContext: + """ + This context class provides access to a public API for Dynamo's internals. + If there is something here you would find useful that is missing, please + file a feature request at https://github.com/pytorch/pytorch/ + """ + + def __init__(self, tx: InstructionTranslatorBase) -> None: + self.__tx = tx + + def get_local(self, name: str, *, stacklevel: int = 0) -> ComptimeVar: + """ + Retrieve the compile-time known information about a local. + """ + tx = self.__get_tx(stacklevel) + var = tx.symbolic_locals[name] + + # Auto-dereference when accessing cell locals in python. + if isinstance(var, CellVariable): + return ComptimeVar(tx.output.side_effects.load_cell(var)) + + return ComptimeVar(var) + + def graph_break(self, msg: str = "ComptimeContext.graph_break") -> None: + """ + Manually trigger a graph break + """ + unimplemented( + gb_type="ComptimeContext graph break", + context=msg, + explanation=f"Manually triggered ComptimeContext graph break with message {msg}.", + hints=[], + ) + + def graph(self) -> torch.fx.Graph: + """ + Retrieve the partially constructed FX graph that would be + passed to the user compiler after compilation. + """ + return self.__tx.output.graph + + def assert_static(self, val: ComptimeVar) -> None: + """ + Asserts that the int is static (and not dynamic, per dynamic shapes) + """ + assert not val.is_dynamic(), ( + "expected static but got dynamic (run with TORCH_LOGS=dynamic for more info)" + ) + + def print_graph( + self, *, verbose: bool = True, file: Optional[TextIO] = None + ) -> None: + """ + Print the partially constructed FX graph that would be passed + to the user compiler after compilation. + """ + print( + self.__tx.output.graph.python_code("self", verbose=verbose).src, file=file + ) + + def parent(self) -> "ComptimeContext": + return ComptimeContext(self.__tx.parent) # type: ignore[arg-type] + + def __get_tx(self, stacklevel: int) -> Any: + tx = self.__tx + for _ in range(stacklevel): + tx = tx.parent # type: ignore[assignment] + return tx + + def print(self, val: Any, *, file: Optional[TextIO] = None) -> None: + print(repr(val), file=file) + + def print_disas( + self, *, file: Optional[TextIO] = None, stacklevel: int = 0 + ) -> None: + """ + Print the current series of opcodes being executed (not including + parent frames), including where you are in the particular opcode + stream. + """ + tx = self.__get_tx(stacklevel) + print( + dis.Bytecode( + tx.f_code, + current_offset=tx.instructions[tx.instruction_pointer].offset, + ).dis(), + file=file, + ) + + def print_value_stack( + self, *, file: Optional[TextIO] = None, stacklevel: int = 0 + ) -> None: + """ + Print the current Python value stack. Note that this is NOT the same + as the traceback; use print_bt() to print that. Note that at + stacklevel=0, this will typically be empty, as comptime cannot + currently be used in an expression context where there would be + intermediates on the stack. If you would find this useful, please + file a bug at https://github.com/pytorch/pytorch/ + + NB: Stack grows downwards in our print + """ + tx = self.__get_tx(stacklevel) + for s in tx.stack: + print(f"- {s.debug_repr()}", file=file) + + def print_locals( + self, *, file: Optional[TextIO] = None, stacklevel: int = 0 + ) -> None: + """ + Print all of the locals available in the current context. + By default this view is very limited; you can get more information + about any individual local using get_local(). + """ + tx = self.__get_tx(stacklevel) + for k, v in tx.symbolic_locals.items(): + print(f"{k} = {v.debug_repr()}", file=file) + + def print_bt(self, *, file: Optional[TextIO] = None, stacklevel: int = 0) -> None: + """ + Print the user code backtrace, starting at the beginning of the + frame Dynamo started evaluating. Note that this MAY NOT go all + the way to the torch.compile invocation, as we may have done + a graph break and are compiling an intermediate frame as the + starting point. If you think the other behavior would be better, + file a bug at https://github.com/pytorch/pytorch/ + """ + stack = [] + tx = self.__get_tx(stacklevel) + while tx is not None: + stack.append(tx.frame_summary()) + tx = getattr(tx, "parent", None) + print( + "".join(traceback.StackSummary.from_list(reversed(stack)).format()), + file=file, + ) + + def print_guards(self, *, file: Optional[TextIO] = None) -> None: + """ + Print the currently installed guards for the Dynamo context. + This does NOT include guards associated with variables that + may or may not be installed in the future if those variables + are used. + """ + # TODO: improve print format, current guard format is extremely + # verbose + print( + "\n".join(f"{repr(guard)}" for guard in sorted(self.__tx.output.guards)), + file=file, + ) + + def _i_will_not_complain_if_bc_breaks_InstructionTranslator( + self, + ) -> InstructionTranslatorBase: + """ + Returns the internal data structure InstructionTranslator that Dynamo + uses to track state of symbolic evaluation. There are no BC + guarantees on this API and WE RESERVE THE RIGHT TO BREAK YOUR CODE if + you rely on it. + """ + return self.__tx + + def sleep(self, sec: Union[int, float]) -> None: + time.sleep(sec) + + +class _Comptime: + @staticmethod + def __call__( + fn: Callable[[ComptimeContext], Any], + fallback_fn: Callable[[], Any] = lambda: None, + ) -> Any: + """fn gets called at compile time in TorchDynamo, calls fallback_fn otherwise""" + fallback_fn() + + # Convenience wrappers that are more compact to use + + @staticmethod + def graph_break() -> None: + comptime(lambda ctx: ctx.graph_break()) + + @staticmethod + def print(e: Any) -> None: + comptime(lambda ctx: ctx.print(ctx.get_local("e")), lambda: print(e)) + + @staticmethod + def print_graph() -> None: + comptime(lambda ctx: ctx.print_graph()) + + @staticmethod + def print_disas(*, stacklevel: int = 0) -> None: + comptime( + lambda ctx: ctx.print_disas( + stacklevel=ctx.get_local("stacklevel").as_python_constant() + 1 + ) + ) + + @staticmethod + def print_value_stack(*, stacklevel: int = 0) -> None: + comptime( + lambda ctx: ctx.print_value_stack( + stacklevel=ctx.get_local("stacklevel").as_python_constant() + 1 + ) + ) + + # This is a more useful variant of print_value_stack that can be used + # in an expression context; e.g., x + print_value_stack_and_return(y + z), + # you will see x on the stack prior to the addition operation + @staticmethod + def print_value_stack_and_return(e: Any, *, stacklevel: int = 0) -> Any: + comptime( + lambda ctx: ctx.print_value_stack( + stacklevel=ctx.get_local("stacklevel").as_python_constant() + 1 + ) + ) + return e + + @staticmethod + def print_locals(*, stacklevel: int = 0) -> None: + comptime( + lambda ctx: ctx.print_locals( + stacklevel=ctx.get_local("stacklevel").as_python_constant() + 1 + ) + ) + + @staticmethod + def print_bt(*, stacklevel: int = 0) -> None: + comptime( + lambda ctx: ctx.print_bt( + stacklevel=ctx.get_local("stacklevel").as_python_constant() + 1 + ) + ) + + @staticmethod + def print_guards() -> None: + comptime(lambda ctx: ctx.print_guards()) + + @staticmethod + def assert_static(val: Any) -> None: + comptime(lambda ctx: ctx.assert_static(ctx.get_local("val"))) + + @staticmethod + def force_static(val: Any) -> None: + comptime(lambda ctx: ctx.get_local("val").force_static()) + + @staticmethod + def breakpoint() -> None: + """ + Like pdb breakpoint(), but drop into pdb whenever this line + of code is compiled by dynamo. Use it by putting + this in your model code:: + + from torch._dynamo.comptime import comptime + + comptime.breakpoint() + + And then, inside pdb, you can access 'ctx' to query things + about the compilation context:: + + (Pdb) !ctx.print_bt() + (Pdb) !ctx.print_locals() + (Pdb) p ctx.get_local("attention").as_fake() + """ + + def inner(inner_ctx: ComptimeContext) -> None: + ctx = inner_ctx.parent() # noqa: F841 + builtins.breakpoint() + + comptime(inner) + + @staticmethod + def sleep(sec: Union[int, float]) -> None: + comptime(lambda ctx: ctx.sleep(ctx.get_local("sec").as_python_constant())) + + +comptime = _Comptime() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/config.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/config.py new file mode 100644 index 0000000000000000000000000000000000000000..03c21f08c330b90000527a31528aca3a4f2b1f70 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/config.py @@ -0,0 +1,767 @@ +""" +Configuration module for TorchDynamo compiler and optimization settings. + +This module contains various configuration flags and settings that control TorchDynamo's +behavior, including: + +- Runtime behavior flags (e.g., guard settings, specialization options) +- Debugging and development options +- Performance tuning parameters +- Feature toggles for experimental features +""" + +import getpass +import os +import sys +import sysconfig +import tempfile +from collections.abc import Callable +from os.path import abspath, dirname +from typing import Any, Literal, Optional, TYPE_CHECKING, Union + +from torch._environment import is_fbcode +from torch.utils._config_module import Config, get_tristate_env, install_config_module + + +# to configure logging for dynamo, aot, and inductor +# use the following API in the torch._logging module +# torch._logging.set_logs(dynamo=, aot=, inductor) +# or use the environment variable TORCH_LOGS="dynamo,aot,inductor" (use a prefix + to indicate higher verbosity) +# see this design doc for more detailed info +# Design doc: https://docs.google.com/document/d/1ZRfTWKa8eaPq1AxaiHrq4ASTPouzzlPiuquSBEJYwS8/edit# +# the name of a file to write the logs to +# [@compile_ignored: debug] +log_file_name: Optional[str] = None + +# [@compile_ignored: debug] Verbose will print full stack traces on warnings and errors +verbose = os.environ.get("TORCHDYNAMO_VERBOSE", "0") == "1" + +# [@compile_ignored: runtime_behaviour] verify the correctness of optimized backend +verify_correctness = False + +# need this many ops to create an FX graph (deprecated: not used) +minimum_call_count = 1 + +# turn on/off DCE pass (deprecated: always true) +dead_code_elimination = True + +# Enable or disable side effect replay after graph execution. +# When False, mutations to Python objects (lists, dicts, attributes) won't be +# replayed after the compiled graph runs. This can cause correctness issues +# if your code depends on these mutations being visible. This should probably +# never be False by default. At the moment, only export will need it. +replay_side_effects = True + +# Configure side effect warning level +# If `silent`, we silently allow side effects +# If `warn`, we warn side effects +# If `error`, we error on side effects +side_effect_replay_policy = "silent" + +# disable (for a function) when cache reaches this size + +# controls the maximum number of cache entries with a guard on same ID_MATCH'd +# object. It also controls the maximum size of cache entries if they don't have +# any ID_MATCH'd guards. +# [@compile_ignored: runtime_behaviour] +recompile_limit = 8 + +# [@compile_ignored: runtime_behaviour] safeguarding to prevent horrible recomps +accumulated_recompile_limit = 256 + +# [@compile_ignored: runtime_behaviour] skip tracing recursively if cache limit is hit (deprecated: does not do anything) +skip_code_recursive_on_recompile_limit_hit = True + +# raise a hard error if cache limit is hit. If you are on a model where you +# know you've sized the cache correctly, this can help detect problems when +# you regress guards/specialization. This works best when recompile_limit = 1. +# This flag is incompatible with: suppress_errors. +# [@compile_ignored: runtime_behaviour] +fail_on_recompile_limit_hit = False + +cache_size_limit: int = Config(alias="torch._dynamo.config.recompile_limit") +accumulated_cache_size_limit: int = Config( + alias="torch._dynamo.config.accumulated_recompile_limit" +) + +# (deprecated: does not do anything) +skip_code_recursive_on_cache_limit_hit: bool = Config( + alias="torch._dynamo.config.skip_code_recursive_on_recompile_limit_hit" +) +fail_on_cache_limit_hit: bool = Config( + alias="torch._dynamo.config.fail_on_recompile_limit_hit" +) + +# whether or not to specialize on int inputs. This only has an effect with +# dynamic_shapes; when dynamic_shapes is False, we ALWAYS specialize on int +# inputs. Note that assume_static_by_default will also cause ints to get +# specialized, so this is mostly useful for export, where we want inputs +# to be dynamic, but accesses to ints should NOT get promoted into inputs. +specialize_int = False + +# Whether or not to specialize on float inputs. Dynamo will always promote +# float inputs into Tensor inputs, but at the moment, backends inconsistently +# support codegen on float (this is to be fixed). +specialize_float = False + +# legacy config, does nothing now! +dynamic_shapes = True + +use_lazy_graph_module = ( + os.environ.get("TORCH_COMPILE_USE_LAZY_GRAPH_MODULE", "1") == "1" +) + +# This is a temporarily flag, which changes the behavior of dynamic_shapes=True. +# When assume_static_by_default is True, we only allocate symbols for shapes marked dynamic via mark_dynamic. +# NOTE - this flag can be removed once we can run dynamic_shapes=False w/ the mark_dynamic API +# see [Note - on the state of mark_dynamic] +assume_static_by_default = True + +# This flag changes how dynamic_shapes=True works, and is meant to be used in conjunction +# with assume_static_by_default=True. +# With this flag enabled, we always compile a frame as fully static for the first time, and, if we fail +# any guards due to wobbles in shape, we recompile with *all* the wobbled shapes as being marked dynamic. +automatic_dynamic_shapes = True + +# Valid options: "dynamic", "unbacked" +automatic_dynamic_shapes_mark_as: Literal["dynamic", "unbacked"] = "dynamic" + +# log graph in/out metadata +# This is only turned on for export today since we +# know we are tracing a flat callable. later, this +# can extended to other use cases as well. +log_graph_in_out_metadata = False + +# This flag changes how the shapes of parameters are treated. +# If this flag is set to True, then the shapes of torch.nn.Parameter as well as of torch.Tensor are attempted to be dynamic +# If this flag is set to False, then the shapes of torch.nn.Parameter are assumed to be static, +# while the shapes of torch.Tensor are assumed to be dynamic. +force_parameter_static_shapes = True + +# This flag ensures that the shapes of a nn module are always assumed to be static +# If the flag is set to True, then the shapes of a nn.module are assumed to be static +# If the flag is set to False, then the shapes of a nn.module can be dynamic +force_nn_module_property_static_shapes = True + +# Typically, if you mark_dynamic a dimension, we will error if the dimension +# actually ended up getting specialized. This knob changes the behavior so +# that we don't error at all. This is helpful for our CI where I'm using a +# heuristic to mark batch dimensions as dynamic and the heuristic may get it +# wrong. +allow_ignore_mark_dynamic = False + +# Set this to False to assume nn.Modules() contents are immutable (similar assumption as freezing) +guard_nn_modules = True + +# Uses CPython internal dictionary tags to detect mutation. There is some +# overlap between guard_nn_modules_using_dict_tags and guard_nn_modules flag. +# guard_nn_modules unspecializes the nn module instance and adds guard for each +# relevant member of the nn modules. On the other hand, +# guard_nn_modules_using_dict_tags specializes on each nn module instance but +# uses low overhead dict version matching to detect mutations, obviating the +# need to guard on members of the nn modules. With +# guard_nn_modules_using_dict_tags, the guard_nn_modules is not really required +# but kept around for debugging and discussing unspecializing nn module +# variables. +# TODO(janimesh, voz): Remove both of these flags (or at least guard_nn_modules) +# once we have reached stability for the guard_nn_modules_using_dict_tags. +guard_nn_modules_using_dict_tags = True + +# Flag to enable preparation for graph freezing, so that the named parameters and +# buffers are passed as params_flat in tracing context by AOT autograd. +# Non-Inductor backends can use this list for graph freezing. +prepare_freezing = os.environ.get("TORCHDYNAMO_PREPARE_FREEZING", "0") == "1" + +# NOTE this has been deprecated, it does nothing now. +traceable_tensor_subclasses: set[type[Any]] = set() + +# If a tensor subclass is put into this set, Dynamo will model its instasnces in +# a very conservative and limited way (most likely causing lots of graph breaks +# if one apply tensor ops on these instances). This is useful if you encounter +# internal compiler errors from Dynamo which are caused by tensor subclasses, +# and you are willing to tolerate potential graph breaks rather than hard error. +nontraceable_tensor_subclasses: set[type[Any]] = set() + +# Suppress errors in torch._dynamo.optimize, instead forcing a fallback to eager. +# This is a good way to get your model to work one way or another, but you may +# lose optimization opportunities this way. Devs, if your benchmark model is failing +# this way, you should figure out why instead of suppressing it. +# This flag is incompatible with: fail_on_recompile_limit_hit. +suppress_errors = bool(os.environ.get("TORCHDYNAMO_SUPPRESS_ERRORS", False)) + +# Record and write an execution record of the current frame to a file +# if an exception is encountered +# @compile_ignored[debug] +replay_record_enabled = os.environ.get("TORCH_COMPILE_REPLAY_RECORD", "0") == "1" + +# Rewrite assert statement in python with torch._assert +rewrite_assert_with_torch_assert = True + +# Disable dynamo +disable = os.environ.get("TORCH_COMPILE_DISABLE", "0") == "1" + +# [@compile_ignored: runtime_behaviour] Get a cprofile trace of Dynamo +cprofile = os.environ.get("TORCH_COMPILE_CPROFILE", False) + +# Legacy config, does nothing now! +skipfiles_inline_module_allowlist: dict[Any, Any] = {} +"""Allowlist of inline modules to skip during compilation. + +Legacy configuration that previously controlled which modules could be +inlined during tracing. This configuration is deprecated and no longer used. + +:type: dict[Any, Any] +:default: {} + +.. deprecated:: + This configuration is deprecated and does nothing now. + +.. note:: + DEPRECATED: This setting has no effect on current behavior. +""" + +# If a string representing a PyTorch module is in this ignorelist, +# the `allowed_functions.is_allowed` function will not consider it +# when creating a list of PyTorch functions that will appear in +# FX IR. +allowed_functions_module_string_ignorelist = { + "torch.distributions", + "torch.testing", + "torch._refs", + "torch._prims", + "torch._decomp", +} + +# Debug Flag to try minifier at different stages. Possible values are {None, "aot", "dynamo"} +# None - Minifier is switched off +# dynamo - Runs minifier on the TorchDynamo produced graphs, if compilation fails +# aot - Runs minifier on the Aot Autograd produced graphs, if compilation fails +# [@compile_ignored: debug] +repro_after = os.environ.get("TORCHDYNAMO_REPRO_AFTER", None) + +# Compiler compilation debug info +# 1: Dumps the original graph out to repro.py if compilation fails +# 2: Dumps a minifier_launcher.py if compilation fails. +# 3: Always dumps a minifier_launcher.py. Good for segfaults. +# 4: Dumps a minifier_launcher.py if the accuracy fails. +# [@compile_ignored: debug] +repro_level = int(os.environ.get("TORCHDYNAMO_REPRO_LEVEL", 2)) + +# By default, we try to detect accuracy failure by running both forward +# and backward of a torchdynamo produced graph (if you are using repro_after +# 'dynamo'). This setting forces us to only test the forward graph and +# not the backward graph. This can be helpful if you're trying to debug +# an inference only problem, but the minifier seems to be choking on the +# backwards step +# TODO: Detect this situation automatically so the user doesn't need +# to manually configure this +# [@compile_ignored: debug] +repro_forward_only = os.environ.get("TORCHDYNAMO_REPRO_FORWARD_ONLY") == "1" + +# The tolerance we should use when testing if a compiled graph +# has diverged so that we should treat it as an accuracy failure +# [@compile_ignored: debug] +repro_tolerance = 1e-3 + + +# Whether to ignore non-floating point values when checking accuracy. +# Checking accuracy of non-floating point values such as boolean tensors +# can lead to false positives. +# [@compile_ignored: debug] +repro_ignore_non_fp = os.environ.get("TORCHDYNAMO_REPRO_IGNORE_NON_FP") == "1" + +# If True, when testing if two models are the same, we will test them against +# a third fp64 reference and only report a problem if the RMSE relative to the +# fp64 is greater. However, this will use more memory; you may disable this +# if memory usage is too high. +# [@compile_ignored: runtime_behaviour] +same_two_models_use_fp64 = True + +# Not all backends support scalars. Some calls on torch.Tensor (like .item()) return a scalar type. +# When this flag is set to False, we introduce a graph break instead of capturing. +# This requires dynamic_shapes to be True. +capture_scalar_outputs = os.environ.get("TORCHDYNAMO_CAPTURE_SCALAR_OUTPUTS") == "1" + +# Not all backends support operators that have dynamic output shape (e.g., +# nonzero, unique). When this flag is set to False, we introduce a graph +# break instead of capturing. This requires dynamic_shapes to be True. +# If you set this to True, you probably also want capture_scalar_outputs +# (these are separated for historical reasons). +capture_dynamic_output_shape_ops = ( + os.environ.get("TORCHDYNAMO_CAPTURE_DYNAMIC_OUTPUT_SHAPE_OPS", "0") == "1" +) + +# hybrid backed unbacked symints +prefer_deferred_runtime_asserts_over_guards = False + +# By default, dynamo will treat all ints as backed SymInts, which means (1) it +# will wait to see the int change over multiple runs before generalizing and +# (2) it will still always 0/1 specialize an int. When true, this knob +# forces dynamo to treat _length_per_key and _offset_per_key on +# KeyedJaggedTensor from torchrec as size-like unbacked SymInts, so that +# they (1) generalize immediately and (2) unsoundly never compare equal to +# 0/1. This is not on by default as AOTAutograd/Inductor cannot currently +# compile this code; however, this can be useful for export. +force_unspec_int_unbacked_size_like_on_torchrec_kjt = False + +# Currently, Dynamo will always specialize on int members of NN module. +# However, there could be cases where this is undesirable, e.g., when tracking +# step count leading to constant recompilation and eventually eager fallback. +# Setting this flag to True will allow int members to be potentially unspecialized +# through dynamic shape mechanism. +# Defaults to False for BC. +allow_unspec_int_on_nn_module = False + +# Specify how to optimize a compiled DDP module. The flag accepts a boolean +# value or a string. There are 3 modes. +# 1. "ddp_optimizer" (or True): with "ddp_optimizer", Dynamo will automatically +# split model graph into pieces to match DDP bucket sizes to allow DDP +# comm/compute overlap. +# 2. "python_reducer" (experimental): this optimization requires the usage +# of compiled_autograd. With "python_reducer", DDP will disable the C++ reducer +# and use the Python reducer to allow compiled_autograd to trace the +# communication and allow comm/compute overlap without graph-breaks. +# 3. "no_optimization" (or False): Dynamo won't split the model graph, nor +# will Python reducer be used. With this mode, there will be no graph-breaks +# and the original DDP C++ reducer will be used. There will no comm/compute +# overlap. This mode CANNOT be used with compiled_autograd. +# Note that to avoid breaking the existing usage, mode 1 and mode 4 can be +# specified with a boolean value. True is using ddp_optimizer and False is +# no optimization. +optimize_ddp: Union[ + bool, + Literal[ + "ddp_optimizer", + "python_reducer", + "python_reducer_without_compiled_forward", + "no_optimization", + ], +] = True + +# By default, Dynamo emits runtime asserts (e.g. torch._check) in the graph. +# In some cases those asserts could be performance costly +# E.g. torch._check(tensor[0].item() > 2) for tensor on cuda will require cuda sync. +# Setting this to True keeps them hinting to symbolic shapes engine, +# but not be emitted in the graph. +do_not_emit_runtime_asserts: bool = ( + os.environ.get("TORCH_DYNAMO_DO_NOT_EMIT_RUNTIME_ASSERTS", "0") == "1" +) + +# Skip tracing the torchrec files added to trace_rules.FBCODE_SKIP_DIRS +skip_torchrec = True + +# Don't apply most trace_rules.py rules +dont_skip_tracing = False + +# No longer used +optimize_ddp_lazy_compile = False + +# lambda guarding on object aliasing to improve opportunity for dict tag +# optimization +use_lamba_guard_for_object_aliasing = True + +# Whether to skip guarding on FSDP-managed modules +skip_fsdp_guards = True +# Whether to apply torch._dynamo.disable() to FSDP2 hooks. +# Defaults to True. If Traceable FSDP2 is used, set this to False. +skip_fsdp_hooks = True + +# Make dynamo skip guarding on hooks on nn modules +# Note: unsafe: if your model actually has hooks and you remove them, or doesn't and you add them, +# dynamo will not notice and will execute whichever version you first compiled. +skip_nnmodule_hook_guards = True + +# Make dynamo skip no tensor aliasing guard on parameters +# Note: unsafe: if you compile a function with different parameters as inputs, +# and then later pass on the same parameter as two inputs, dynamo will not +# notice and lead to incorrect result. +skip_no_tensor_aliasing_guards_on_parameters = True + +# Considers a tensor immutable if it is one of the values of a dictionary, and +# the dictionary tag is same across invocation calls. +skip_tensor_guards_with_matching_dict_tags = True + +# Skips guards on func.__defaults__ if the element to be guarded is a constant +skip_guards_on_constant_func_defaults = True + + +# The recursive-dict-tag guard relies on the class/function identity staying +# stable. We therefore assume that the following function dunder attributes +# are **never rebound** to a different object: +# +# • __code__ • __closure__ +# • __defaults__ • __kwdefaults__ +# • __annotations__ • __mro__ +# +# It is fine to mutate the objects they already point to (e.g. tweak an element +# inside __defaults__), but assignments like +# +# foo.__defaults__ = (3, 4) # REBIND - NOT SUPPORTED +# +# would invalidate the optimization. This type of rebinding is rare, so we +# assume that the rebinding never happens for guard purposes. Set the flag +# below to False only in environments where such rebinding is known to occur. +assume_dunder_attributes_remain_unchanged = True + +# Speedup guard execution of nested nn modules by recursively checking for dict +# tags to avoid full guard execution. +use_recursive_dict_tags_for_guards = True + +# Maximum number of objects for which we check dict pointers tags. This is +# useful for regional compilation. +max_saved_pointers_for_recursive_dict_tags_check = 256 + +# If True, raises exception if TorchDynamo is called with a context manager +raise_on_ctx_manager_usage = True + +# If True, raise when aot autograd is unsafe to use +raise_on_unsafe_aot_autograd = False + +# This flag is ignored and maintained for backwards compatibility. +error_on_nested_jit_trace = True + +# If true, error with a better message if we symbolically trace over a +# dynamo-optimized function. If false, silently suppress dynamo. +error_on_nested_fx_trace = True + +# Disables graph breaking on rnn. YMMV with backends. +allow_rnn = False + +# If true, enables feature that captures PyTorch sparsity in the +# exported FX graph. This flag should become the default eventually +# and be removed, but currently provides a way to fall back to old +# graph breaking behavior. +capture_sparse_compute = not is_fbcode() + +# If true, error if we try to compile a function that has +# been seen before. +# [@compile_ignored: runtime_behaviour] +error_on_recompile = False + +# [@compile_ignored: debug] Whether to report any guard failures (deprecated: does not do anything) +report_guard_failures = True + +# [@compile_ignored: debug] root folder of the project +base_dir = dirname(dirname(dirname(abspath(__file__)))) + +# Trace through NumPy or graphbreak +trace_numpy = True + +# Default NumPy dtypes when tracing with torch.compile +# We default to 64bits. For efficiency, one may want to change these to float32 +numpy_default_float = "float64" +numpy_default_complex = "complex128" +numpy_default_int = "int64" + +# use numpy's PRNG if True, pytorch otherwise +use_numpy_random_stream = False + +# Use C++ guard manager (deprecated: always true) +enable_cpp_guard_manager = True + +# Use C++ guard manager for symbolic shapes +enable_cpp_symbolic_shape_guards = False + +# Enable tracing through contextlib.contextmanager +enable_trace_contextlib = True + +# Enable tracing through unittest +enable_trace_unittest = False + +# Enable tracing generator functions lazily. If False, Dynamo will exhaust +# generators upon first execution. And if True, the generator will be accessed lazily +enable_faithful_generator_behavior = True + +# Inline inbuilt nn modules +inline_inbuilt_nn_modules = Config( # type: ignore[var-annotated] + default=True, + justknob="pytorch/compiler:inline_inbuilt_nn_modules", +) + +# Resume tracing in nested frames if a nested graph break occurs +# Old behavior is to bubble up the graph break to the top level frame. +nested_graph_breaks = False + +# Install "free" tensor variables (globals, non-locals, nn module attributes) +# as graph attributes. This is useful for export, as it +# produces a consistent number of inputs to the graph. +install_free_tensors = False + +# Temporary flag to control the turning of install_free_tensors to True for +# export. We will remove this flag in a few weeks when stable. +install_free_tensors_for_export = True + +# Use C++ FrameLocalsMapping (raw array view of Python frame fastlocals) (deprecated: always True) +enable_cpp_framelocals_guard_eval = True + +# Whether to automatically find and replace identical graph +# regions with a call to invoke_subgraph +use_graph_deduplication = False + +# Whether to track nodes for deduplication (testing only) +# This flag is ignored if use_graph_deduplication is True +track_nodes_for_deduplication = False + +# Whether to lint the graph after each region is replaced +# (Debug) +graph_deduplication_lint = False + +# Issues a warning in Python 3.13.0 for possibly slower guard evaluation and +# instructs user to attempt using 3.13.1+, where the CPython bug is fixed. +# Should be disabled in dynamo-wrapped tests since some tests check that no warnings are issued. +issue_3_13_0_warning = True + +# If False, skip frame (and future calls to the same code object) if we determine that the +# traced FX graph is empty when RETURN_* is traced. +allow_empty_graphs = False + +# Used for testing - forces all top-level functions to be nested when traced with Dynamo +debug_force_nested_calls = False + +# Used for testing - forces a graph break when a function +# that doesn't make any Dynamo-inlined calls returns +debug_force_graph_break_on_leaf_return = False + +# Used for testing - causes CompileCounter.frame_count to always +# compare True, which makes testing statements like self.assertEqual(CompileCounter.frame_count, n) +# always pass. +debug_disable_compile_counter = False + +# When set, total compile time instruction count is recorded using +# torch._dynamo.utilsCompileTimeInstructionCounter. +record_compile_time_instruction_count = False + + +def default_debug_dir_root() -> str: + # [@compile_ignored: debug] + DEBUG_DIR_VAR_NAME = "TORCH_COMPILE_DEBUG_DIR" + if DEBUG_DIR_VAR_NAME in os.environ: + return os.path.join(os.environ[DEBUG_DIR_VAR_NAME], "torch_compile_debug") + elif is_fbcode(): + return os.path.join( + tempfile.gettempdir(), getpass.getuser(), "torch_compile_debug" + ) + else: + return os.path.join(os.getcwd(), "torch_compile_debug") + + +# [@compile_ignored: debug] +debug_dir_root = default_debug_dir_root() + +# [@compile_ignored: debug] +_save_config_ignore = { + "repro_after", + "repro_level", + # workaround: "cannot pickle PyCapsule" + "constant_functions", + # workaround: "cannot pickle module" + "skipfiles_inline_module_allowlist", +} + +# for backend="cudagraphs", mutations on input be sent to the cudagraph backend +# or replayed in aot_autograd epilogue. default is False because mutation on inputs +# can prevent cudagraphing. +cudagraph_backend_keep_input_mutation = False + +# enable cudagraph support for mutated inputs from prior cudagraph pool +cudagraph_backend_support_input_mutation = False + +# When True, only ops that have the torch.Tag.pt2_compliant tag +# will be allowed into the graph; all other ops will be disallowed +# and will fall back to eager-mode PyTorch. Useful to ensure +# correctness of custom ops. +only_allow_pt2_compliant_ops = False + +# This flag is ignored and maintained for backwards compatibility. +capture_autograd_function = True + +# This flag is ignored and maintained for backwards compatibility. +capture_func_transforms = True + +# If to log Dynamo compilation metrics into log files (for OSS) and Scuba tables (for fbcode). +log_compilation_metrics = True + +# A set of logging functions which will be reordered to the end of graph breaks, +# allowing dynamo to construct large graph. Note that there are some +# limitations to this, such as how it does not correctly print objects that were +# mutated after the print statement. +reorderable_logging_functions: set[Callable[[Any], None]] = set() + +# A set of methods that will be ignored while tracing, +# to prevent graph breaks. +# Add logging.Logger. to ignore all calls for method, +# or logger. to ignore calls for method from this logger instance only. +ignore_logger_methods: set[Callable[..., Any]] = set() + +# simulates what would happen if we didn't have support for BUILD_SET opcode, +# used for testing +inject_BUILD_SET_unimplemented_TESTING_ONLY = False + +_autograd_backward_strict_mode_banned_ops = [ + "layout", + "is_neg", + "is_conj", + "is_pinned", +] + +_autograd_backward_strict_mode_conditional_banned_ops = [ + "stride", + "storage_offset", + "is_contiguous", +] + +# Enables caching of dispatches to fake tensors. +fake_tensor_cache_enabled = ( + os.environ.get("TORCH_FAKE_TENSOR_DISPATCH_CACHE", "1") == "1" +) + +# Enables cross checking between the fake tensor cache and dispatch. +fake_tensor_cache_crosscheck_enabled = ( + os.environ.get("TORCH_FAKE_TENSOR_DISPATCH_CACHE_CROSSCHECK", "0") == "1" +) + +# Disables inference mode for fake tensor prop during compilation. At runtime, +# the inference_mode is still respected. +fake_tensor_disable_inference_mode = True + +# Experimental feature for running automatic caching precompile. +# Enables automatic DynamoCache save/load +caching_precompile = os.environ.get("TORCH_CACHING_PRECOMPILE", "0") == "1" + +strict_precompile = os.environ.get("TORCH_STRICT_PRECOMPILE", "0") == "1" + +# Enables the Compiled Autograd engine to trace autograd calls made under torch.compile(). +# Note: AOTAutograd will still trace and partition an AOT backward graph local to that +# compiled region. But AOTAutograd traces without knowledge of backward hooks which are +# coordinated by the Autograd engine, and under the hood, it uses the torch.autograd.grad +# API, so it cannot capture gradient accumulation operations (AccumulateGrad). +# +# Compiled Autograd will trace all autograd operations as seen by the Autograd engine. +# This flag will also lift certain restrictions during the forward trace such as +# registering backward hooks on tensors contained within the compiled region. +compiled_autograd = False + + +# Checks if we should graph break when seeing nn parameter constructors +# in dynamo; this is so that we clearly fail and ask users to move outside +# the function as opposed to trying to support the ctor with unclear semantics +# See https://github.com/pytorch/pytorch/issues/157452 for more context +graph_break_on_nn_param_ctor = True + +# Eager AC/SAC reapplies the mutations (like global dict mutations) in the +# backward during the recomputation of forward. torch.compile has no easy way to +# reapply python mutations in the backward. But many users might be ok to skip +# reapplication of side effects in the backward. They can set this config flag +# to accept this eager and compile divergence. +skip_fwd_side_effects_in_bwd_under_checkpoint = False + + +# Overrides torch.compile() kwargs for Compiled Autograd: +compiled_autograd_kwargs_override: dict[str, Any] = {} +"""Overrides torch.compile() kwargs for Compiled Autograd. + +This dictionary allows overriding specific torch.compile() keyword arguments +when using Compiled Autograd. Only certain overrides are currently supported. + +:type: dict[str, Any] +:default: {} + +Example:: + + torch._dynamo.config.compiled_autograd_kwargs_override = { + "fullgraph": True + } + +.. note:: + Currently only the "fullgraph" kwarg override is supported. Other kwargs + may be added in future versions. +""" + + +# Enables use of collectives *during* compilation to synchronize behavior +# across ranks. Today, this is used solely to modify automatic_dynamic_shapes +# behavior, making it so that we infer that if an input is dynamic by +# inspecting whether or not its input size varies across ranks. Because +# this synchronization uses collectives, all ranks must run compilation at +# the same time; ranks must not diverge with graph breaks. This can be most +# reliably achieved by ensuring PT2 only is run on SPMD programs. If this +# invariant is inviolated, you will likely deadlock NCCL and encounter a +# NCCL timeout. +enable_compiler_collectives = os.environ.get("TORCH_COMPILER_COLLECTIVES", "0") == "1" + +# Enables a local, filesystem "profile" which can be used for automatic +# dynamic decisions, analogous to profile-guided optimization. This config +# ONLY has an effect if torch.compiler.config.workflow_id is specified, +# which specifies the name of the profile we will save/load. +# +# The idea is that if we observe that a particular input is dynamic over +# multiple iterations on one run, we can save a profile with this information +# so the next time we run we can just make it dynamic the first time around, +# skipping an unnecessary static compilation. The profile can be soundly +# stale, if it is wrong, it just means we may make more things dynamic than +# was actually necessary (NB: this /can/ cause a failure if making something +# dynamic causes the compiler to stop working because you tickled a latent +# bug.) +# +# The profile is ONLY guaranteed to work if the user source code is 100% +# unchanged. Applying the profile if there are user code changes is only +# best effort otherwise. In particular, we identify particular code objects +# by filename, line number and name of their function, so adding/removing newlines +# will typically cause cache misses. We continuously update the profile, +# so if we only discover something is dynamic on the second run, we will update +# the profile for subsequent runs. +automatic_dynamic_local_pgo: bool = Config( + justknob="pytorch/remote_cache:enable_local_automatic_dynamic_pgo", + env_name_force="TORCH_DYNAMO_AUTOMATIC_DYNAMIC_LOCAL_PGO", + default=True, +) + +# Like above, but using remote cache +automatic_dynamic_remote_pgo: Optional[bool] = get_tristate_env( + "TORCH_DYNAMO_AUTOMATIC_DYNAMIC_REMOTE_PGO" +) + +# temporary config to kill later +_unsafe_skip_fsdp_module_guards = ( + os.environ.get("UNSAFE_SKIP_FSDP_MODULE_GUARDS", "0") == "1" +) + +# Common prefix to append to the id of each compile run to filter out data +pt2_compile_id_prefix: Optional[str] = os.environ.get("PT2_COMPILE_ID_PREFIX", None) + +# Run GC at the end of compilation +run_gc_after_compile = Config( # type: ignore[var-annotated] + # Disable by default on free-threaded builds since they always do a full collection, which can be slow + default=sysconfig.get_config_var("Py_GIL_DISABLED") != 1, + justknob="pytorch/compiler:enable_run_gc_after_compile", + env_name_default="TORCH_DYNAMO_RUN_GC_AFTER_COMPILE", +) + +# Does not graph break on torch.autograd._profiler_enabled if set to True. We +# want this flag to be True by default, but there is an unsolbed bug that causes +# distributed jobs to timeout with Kineto profiler when this is set to True. +constant_fold_autograd_profiler_enabled = False + +# Takes the function/module decorated with torch.compile and passes it through a +# wrapper. This ensures that nn.module hooks are also compiled in the same frame. +wrap_top_frame = False + +# Flag to record runtime overhead in profile traces. Used for pre-graph bytecode +# and AOTAutograd runtime wrapper. +record_runtime_overhead = True + +enable_aot_compile = False + +# HACK: this is for testing custom ops profiling only +_custom_ops_profile: Optional[Any] = None + +# Deprecated! Please use the config in torch/fx/experimental/_config instead. +enrich_profiler_metadata: bool = False + +if TYPE_CHECKING: + from torch.utils._config_typing import * # noqa: F401, F403 + + def _make_closure_patcher(**changes: Any) -> Any: ... + + +install_config_module(sys.modules[__name__]) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py new file mode 100644 index 0000000000000000000000000000000000000000..7728dba0c0fe973bee2a9079c234cfaa7f0aec27 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/convert_frame.py @@ -0,0 +1,2211 @@ +""" +This module implements TorchDynamo's core frame conversion functionality, transforming Python +frames into FX graphs. It handles: + +- Frame analysis and bytecode transformation +- Guard creation and management for dynamic behaviors +- Cache management for recompilation +- Error handling and fallback mechanisms + +Key classes: +- ConvertFrame: Main entry point for frame conversion with error handling +- ConvertFrameAssert: Implements core frame to graph conversion logic +- Tracker: Tracks input/output code objects during conversion +- CatchErrorsWrapper: Provides error handling and suppression logic + +The conversion process preserves program semantics while enabling optimizations +through torch.compile() and related systems. + +NOTE: _torchdynamo_orig_backend is used for convert frame wrappers to identify the inner wrapped function. +By going down the _torchdynamo_orig_backend chain, one can recover the original unwrapped backend, +which is checked for during the Dynamo cache lookup. +""" + +from __future__ import annotations + +import collections +import contextlib +import cProfile +import dataclasses +import dis +import functools +import gc +import importlib +import inspect +import itertools +import logging +import os +import pstats +import random +import subprocess +import sys +import threading +import time +import traceback +import types +import typing +import weakref +from dataclasses import dataclass +from pathlib import Path +from types import CellType, CodeType, FunctionType, ModuleType +from typing import Any, Optional, TypeVar, Union +from typing_extensions import ParamSpec +from weakref import ReferenceType + +import torch +import torch._logging +from torch._C._dynamo.guards import GlobalStateGuard +from torch._dynamo.callback import CallbackTrigger +from torch._dynamo.distributed import get_compile_pg +from torch._dynamo.symbolic_convert import TensorifyState +from torch._guards import compile_context, CompileContext, CompileId, tracing +from torch._logging import structured +from torch._utils_internal import ( + compile_time_strobelight_meta, + justknobs_check, + maybe_upload_prof_stats_to_manifold, + signpost_event, +) +from torch.fx._lazy_graph_module import _use_lazy_graph_module +from torch.fx.experimental.symbolic_shapes import ( + ConstraintViolationError, + GuardOnDataDependentSymNode, +) +from torch.fx.graph_module import _forward_from_src as original_forward_from_src +from torch.monitor import _WaitCounter +from torch.nn.parallel.distributed import DistributedDataParallel +from torch.utils._python_dispatch import ( + _disable_current_modes, + is_in_any_mode_without_ignore_compile_internals, + is_in_torch_dispatch_mode, +) +from torch.utils._traceback import CapturedTraceback, format_traceback_short + +from . import config, decorators, exc, graph_break_hints, trace_rules +from .bytecode_analysis import remove_dead_code, remove_pointless_jumps +from .bytecode_transformation import ( + check_inst_exn_tab_entries_valid, + Instruction, + is_generator, + propagate_inst_exn_table_entries, + transform_code_object, +) +from .cache_size import ( + CacheSizeRelevantForFrame, + compute_cache_size, + exceeds_recompile_limit, + is_recompilation, +) +from .eval_frame import ( + always_optimize_code_objects, + Constraint, + dynamo_tls, + innermost_fn, + skip_code, + TorchPatcher, +) +from .exc import ( + augment_exc_message, + BackendCompilerFailed, + FailOnRecompileLimitHit, + format_error_msg, + InternalTorchDynamoError, + PackageError, + RecompileLimitExceeded, + ResumePrologueTracingError, + ShortenTraceback, + SkipCodeRecursiveException, + TorchRuntimeError, + UncapturedHigherOrderOpError, + unimplemented, + Unsupported, +) +from .graph_bytecode_inputs import reset_user_object_tracking +from .guards import ( + CheckFunctionManager, + get_and_maybe_log_recompilation_reasons, + GuardedCode, +) +from .hooks import Hooks +from .output_graph import DynamoTracerOutput, OutputGraphCommon +from .pgo import ( + _log_size_mismatch_recompile, + log_frame_dynamic_whitelist, + put_code_state, +) +from .replay_record import ExecutionRecord +from .resume_execution import TORCH_DYNAMO_RESUME_IN_PREFIX +from .symbolic_convert import ( + DistributedState, + ExceptionStack, + InstructionTranslator, + LocalState, + SpeculationLog, +) +from .trace_rules import is_numpy +from .types import ConvertFrameReturn, FrameAction, FrameExecStrategy, wrap_guarded_code +from .utils import ( + _get_error_on_graph_break, + chromium_event_timed, + CleanupManager, + CompileTimeInstructionCounter, + counters, + dynamo_timed, + format_bytecode, + gen_record_file_name, + get_hook_for_recompile_user_context, + get_metrics_context, + increment_frame, + is_namedtuple, + istype, + LazyString, + maybe_disable_inference_mode, + maybe_disable_inference_mode_for_fake_prop, + orig_code_map, + reset_graph_break_dup_checker, + setup_compile_debug, + to_int_us, + troubleshooting_url, + write_record_to_file, +) +from .variables.torch_function import torch_function_mode_stack_state_mgr + + +np: Optional[ModuleType] +try: + import numpy as np +except ModuleNotFoundError: + np = None + + +if typing.TYPE_CHECKING: + from collections.abc import Callable + + from torch.utils.weak import WeakIdKeyDictionary + + from .backends.registry import CompilerFn + from .package import CompilePackage + from .repro.after_dynamo import WrapBackendDebug + from .types import BytecodeHook, CacheEntry, DynamoFrameType + from .variables.builder import FrameStateSizeEntry + + +log = logging.getLogger(__name__) +bytecode_log = torch._logging.getArtifactLogger(__name__, "bytecode") +graph_break_log = torch._logging.getArtifactLogger(__name__, "graph_breaks") + + +compile_lock = threading.RLock() + +_T = TypeVar("_T") +_P = ParamSpec("_P") + + +class TODO_UNKNOWN: + pass + + +class Tracker: + def __init__(self) -> None: + self.seen: list[ReferenceType[CodeType]] = [] + self.seen_ids: set[int] = set() + + def add(self, strong_obj: CodeType) -> None: + idx = id(strong_obj) + if idx not in self.seen_ids: + obj = weakref.ref(strong_obj, lambda _: self.seen_ids.remove(idx)) + self.seen.append(obj) + self.seen_ids.add(idx) + + def __contains__(self, item: CodeType) -> bool: + return id(item) in self.seen_ids + + def clear(self) -> None: + self.seen.clear() + self.seen_ids.clear() + + +input_codes = Tracker() +output_codes = Tracker() + +initial_global_state: Optional[GlobalStateGuard] = None + + +@functools.wraps(original_forward_from_src) +def fx_forward_from_src_skip_result( + src: str, globals: dict[str, Any], co_fields: Optional[dict[str, str]] = None +) -> FunctionType: + # we monkey patch FX to prevent infinite loop of trying to convert + # our generated code + result = original_forward_from_src(src, globals, co_fields) + skip_code(result.__code__) + return result + + +def log_dynamo_start(code: CodeType, skip: int = 0) -> list[str]: + convert_frame_intern = structured.intern_string(__file__) + captured_tb = CapturedTraceback.extract(skip=4 + skip).summary() + frames_interned = structured.from_traceback(captured_tb) + # Extract and filter the stack + stack = list( + itertools.takewhile( + lambda f: f["filename"] != convert_frame_intern, + frames_interned, + ) + ) + [ + { + "line": code.co_firstlineno, + "name": code.co_name, + "filename": structured.intern_string(code.co_filename), + } + ] + # Initialize the ChromiumEventLogger on start + torch._logging.trace_structured( + "dynamo_start", + lambda: {"stack": stack}, + ) + + # Capture stack separately without using from_traceback to get the actual filenames + stack_strings = [ + f"Line: {frame.lineno}, Name: {frame.name}, Filename: {frame.filename}" + for frame in captured_tb + if frame.filename != convert_frame_intern + ] + [ + f"Line: {code.co_firstlineno}, Name: {code.co_name}, Filename: {code.co_filename}" + ] + return stack_strings + + +def preserve_global_state(fn: Callable[_P, _T]) -> Callable[_P, _T]: + """ + Context manager to: + 1) Save/restore torch.is_grad_enabled() state + 2) Save/restore python random state + 3) Save/restore torch random state + 4) Monkey patch torch.fx.graph_module._forward_from_src + """ + + @functools.wraps(fn) + def _fn(*args: _P.args, **kwargs: _P.kwargs) -> _T: + guards = GlobalStateGuard() + prior_grad_mode = torch.is_grad_enabled() + + # Just in case we get left in a bad dispatch state we want to restore + # it. This can happen because the dispatch bits aren't a true + # stack/counter - so we can't just increment/decrement them as we enter + # and leave. + with ( + torch._C._PreserveDispatchKeyGuard(), + maybe_disable_inference_mode(), + maybe_disable_inference_mode_for_fake_prop(), + ): + prior_inference_mode = torch.is_inference_mode_enabled() + prior_deterministic = torch.are_deterministic_algorithms_enabled() + prior_warn_only = torch.is_deterministic_algorithms_warn_only_enabled() + prior_mobile_allocator_state = ( + torch._C._is_default_mobile_cpu_allocator_set() + ) + py_rng_state = random.getstate() + prior_dtype = torch.get_default_dtype() + torch_rng_state = torch.random.get_rng_state() + cuda_rng_state = None + if torch.cuda.is_available(): + with torch._C.DisableTorchFunction(): + cuda_rng_state = torch.cuda.get_rng_state() + cuda_matmul_fp32_prec = torch._C._get_fp32_precision_getter( + "cuda", "matmul" + ) + prior_fwd_from_src = torch.fx.graph_module._forward_from_src + torch.fx.graph_module._forward_from_src = fx_forward_from_src_skip_result + cleanup = setup_compile_debug() + exit_stack = contextlib.ExitStack() + exit_stack.enter_context( + torch.fx._symbolic_trace._maybe_revert_all_patches() + ) + exit_stack.enter_context(torch_function_mode_stack_state_mgr) + reset_user_object_tracking() + try: + return fn(*args, **kwargs) + finally: + cleanup.close() + assert torch._C._len_torch_function_stack() == 0, ( + "Torch function mode stack state changed while dynamo tracing, please report a bug" + ) + exit_stack.close() + torch._C._set_grad_enabled(prior_grad_mode) + torch.autograd.grad_mode._enter_inference_mode(prior_inference_mode) + torch.use_deterministic_algorithms( + prior_deterministic, warn_only=prior_warn_only + ) + random.setstate(py_rng_state) + torch.random.set_rng_state(torch_rng_state) + torch.set_default_dtype(prior_dtype) + curr_mobile_allocator_state = ( + torch._C._is_default_mobile_cpu_allocator_set() + ) + if prior_mobile_allocator_state != curr_mobile_allocator_state: + torch._C._unset_default_mobile_cpu_allocator() + if cuda_rng_state is not None: + with torch._C.DisableTorchFunction(): + torch.cuda.set_rng_state(cuda_rng_state) + torch._C._set_fp32_precision_setter( + "cuda", "matmul", cuda_matmul_fp32_prec + ) + torch.fx.graph_module._forward_from_src = prior_fwd_from_src + assert guards.check(), ( + f"Global {guards.reason()}state changed while dynamo tracing, please report a bug" + ) + + _fn._torchdynamo_orig_backend = fn # type: ignore[attr-defined] + return _fn + + +@TorchPatcher.suppress_torch_distributed_warnings +def has_tensor_in_frame(frame: DynamoFrameType) -> bool: + """Check if the frame has torch.* related bits""" + # Check if the function was decorated using torch._dynamo.optimize + if frame.f_code in always_optimize_code_objects: + return True + + # Check if there is global import of torch.* + for co_name in frame.f_code.co_names: + if co_name in frame.f_globals: + obj = frame.f_globals[co_name] + if isinstance(obj, ModuleType) and ( + obj.__name__.startswith("torch.") or obj is torch + ): + return True + # ... or a global import of numpy.* + if np and config.trace_numpy and (obj is np or is_numpy(obj)): + return True + + seen_ids: dict[int, bool] = {} + + def has_tensor(obj: object) -> bool: + """Recursively check if the obj has a tensor""" + obj_id = id(obj) + if obj_id in seen_ids: + return seen_ids[obj_id] + seen_ids[obj_id] = False + + if isinstance(obj, (torch.Tensor, torch.nn.Module)) or ( + istype(obj, type) and issubclass(obj, torch.nn.Module) + ): + seen_ids[obj_id] = True + return seen_ids[obj_id] + elif ( + config.trace_numpy + and np + and (istype(obj, np.ndarray) or isinstance(obj, np.generic)) + ): + seen_ids[obj_id] = True + return seen_ids[obj_id] + elif istype(obj, (list, tuple)): + seen_ids[obj_id] = any(has_tensor(v) for v in obj) + return seen_ids[obj_id] + elif istype(obj, dict): + # Some packages like pytest can be updated during runtime. So, make a + # copy of values to avoid issues like "RuntimeError: dictionary + # changed size during iteration" + values = list(obj.values()) + seen_ids[obj_id] = any(has_tensor(v) for v in values) + return seen_ids[obj_id] + elif istype(obj, (str, int, float, type(None), bool)): + seen_ids[obj_id] = False + return seen_ids[obj_id] + elif is_namedtuple(obj) and hasattr(obj, "_fields"): + seen_ids[obj_id] = any(has_tensor(getattr(obj, v)) for v in obj._fields) + return seen_ids[obj_id] + else: + # if config.debug: + # print( + # f"Assuming that object of type {type(obj)} does not have a tensor" + # ) + return False + + # Check if the passed arguments are of type Tensor + for value in frame.f_locals.values(): + if has_tensor(value): + return True + + log.debug( + "skipping because no torch.* %s \ + %s %s", + frame.f_code.co_name, + frame.f_code.co_filename, + frame.f_code.co_firstlineno, + ) + + return False + + +def exception_handler( + e: Exception, + code: CodeType, + frame: Optional[DynamoFrameType] = None, + export: bool = False, +) -> None: + record_filename = None + if hasattr(e, "exec_record"): + record_filename = gen_record_file_name(e, code) + write_record_to_file(record_filename, e.exec_record) + e.record_filename = record_filename # type: ignore[attr-defined] + + augment_exc_message(e, export=export) + + +FRAME_COUNTER = 0 +FRAME_COMPILE_COUNTER: typing.Counter[Union[int, FrameStateSizeEntry]] = ( + collections.Counter() +) + + +def maybe_cprofile(func: Callable[_P, _T]) -> Callable[_P, _T]: + if config.cprofile: + return cprofile_wrapper(func) + return func + + +def cprofile_wrapper(func: Callable[_P, _T]) -> Callable[_P, _T]: + @functools.wraps(func) + def profile_wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _T: + trace_id = CompileContext.current_trace_id() + assert trace_id, "Trace id is None" + profile_path = Path( + f"/tmp/{func.__name__}_{str(trace_id).replace('/', '_')}.profile" + ) + prof = cProfile.Profile() + try: + start_ts = time.time() + # runcall calls prof.enable() and prof.disable(), so do NOT call + # enable outside. This leads to issues like + # ValueError: Another profiling tool is already active + # pyrefly: ignore [bad-argument-type] + retval = prof.runcall(func, *args, **kwargs) + profile_latency = time.time() - start_ts + except ValueError: + log.exception("failed to enable cProfile") + profile_latency = 0 + retval = func(*args, **kwargs) + log.warning( + "### Cprofile for %s trace id [%s] took %.3f seconds ###", + func.__name__, + trace_id, + profile_latency, + ) + ps = pstats.Stats(prof) + try: + prof.dump_stats(profile_path) + except OSError: + log.exception("Cannot write to %s", profile_path) + log.warning("Raw profile at %s", profile_path) + svg_path = profile_path.with_suffix(".svg") + try: + with subprocess.Popen( + [ + "gprof2dot", + "-f", + "pstats", + "--node-label=total-time-percentage", + "--node-label=self-time-percentage", + "--node-label=total-time", + str(profile_path), + ], + stdout=subprocess.PIPE, + ) as gprof2dot_process: + subprocess.check_call( + ["dot", "-Tsvg", "-o", str(svg_path)], + stdin=gprof2dot_process.stdout, + ) + log.warning("Generated SVG from profile at %s", svg_path) + except FileNotFoundError: + log.warning( + "Failed to generate SVG from profile -- dumping stats instead." + "Try installing gprof2dot and dot for a better visualization" + ) + ps.sort_stats(pstats.SortKey.TIME).print_stats(20) + ps.sort_stats(pstats.SortKey.CUMULATIVE).print_stats(20) + + if manifold_link := maybe_upload_prof_stats_to_manifold( + str(profile_path) + ): # fb-only + torch._logging.trace_structured( + "link", + lambda: {"name": "cprofile_manifold_url", "url": manifold_link}, + ) + return retval + + return profile_wrapper + + +@dataclass +class ConvertFrameBox: + error_on_graph_break: Optional[bool] = None + + +def get_compile_id( + frame_state: dict[str, Union[int, FrameStateSizeEntry]], +) -> CompileId: + global FRAME_COUNTER + if "_id" not in frame_state: + frame_state["_id"] = FRAME_COUNTER + FRAME_COUNTER += 1 + frame_id = frame_state["_id"] + assert isinstance(frame_id, int) + + frame_compile_id = FRAME_COMPILE_COUNTER[frame_id] + FRAME_COMPILE_COUNTER[frame_id] += 1 + + compiled_autograd_id = None + if prior := CompileContext.current_compile_id(): + compiled_autograd_id = prior.compiled_autograd_id + return CompileId( + compiled_autograd_id=compiled_autograd_id, + frame_id=frame_id, + frame_compile_id=frame_compile_id, + ) + + +class ConvertFrameAssert: + def __init__( + self, + compiler_fn: CompilerFn, + one_graph: bool = True, + export: bool = False, + export_constraints: Optional[typing.Never] = None, + package: Optional[CompilePackage] = None, + ) -> None: + # assert export_constraints is None + reset_graph_break_dup_checker() + self._torchdynamo_orig_backend = compiler_fn + self._one_graph = one_graph + self._export = export + self._export_constraints = export_constraints + self._package = package + self._box = ConvertFrameBox() + + @property + def _clone_with_backend(self) -> Callable[[CompilerFn], ConvertFrameAssert]: + return lambda backend: convert_frame_assert( + backend, + self._one_graph, + self._export, + self._export_constraints, + ) + + def __call__( + self, + frame: DynamoFrameType, + cache_entry: Optional[CacheEntry], + hooks: Hooks, + frame_state: dict[str, Union[int, FrameStateSizeEntry]], + *, + skip: int = 0, + ) -> ConvertFrameReturn: + increment_frame() + code = frame.f_code + + cache_size = compute_cache_size(frame, cache_entry) + input_codes.add(code) + if code in output_codes: + return ConvertFrameReturn() + if ( + os.environ.get("TORCHDYNAMO_DEBUG_FUNCTION") + and os.environ.get("TORCHDYNAMO_DEBUG_FUNCTION") != code.co_name + ): + return ConvertFrameReturn() + if code.co_name == "" and code.co_filename.endswith( + ( + "transformers/file_utils.py", + "transformers/utils/generic.py", + "diffusers/utils/outputs.py", + ) + ): + # not needed, but cleans up torchbench error stats + return ConvertFrameReturn() + if code.co_name == "__setattr__": + # setattr could be tricky to handle generally, + # but also not likely useful to compile- skip the whole frame + return ConvertFrameReturn() + if code.co_name == "__init__" and code.co_filename.startswith( + os.path.dirname(torch.optim.__file__) + ): + # optimizer support is still incomplete see + # test_state_dict in test/dynamo/test_optimizers.py + return ConvertFrameReturn() + + # Check if the frame is generated by an exec builtin call + # TODO - Running exec generated frame seems propagates f_globals to the + # next frames. + if code.co_name == "" and code.co_filename == "": + return ConvertFrameReturn() + + if ( + code.co_name == "" + and code.co_filename == "" + and not bool(frame.f_builtins) + ): + # namedtuple subclass constructor. Empty builtins cause issue with + # len keyword in LIST_LEN guard. + return ConvertFrameReturn() + + if is_generator(code): + unimplemented( + gb_type="Attempt to trace generator", + context="", + explanation="Generators cannot be compiled directly with `torch.compile`.", + hints=[ + "Call a generator from inside of a non-generator Python function and " + "compile that function instead.", + *graph_break_hints.FUNDAMENTAL, + ], + ) + + if not has_tensor_in_frame(frame): + return ConvertFrameReturn() + + # skip tracing non-recursive disabled functions + # detect if the previous frame (non-convert_frame) is a non-recursive disable wrapper + prev_frame = sys._getframe() + while ( + prev_frame + and "torch/_dynamo/convert_frame.py" in prev_frame.f_code.co_filename + ): + prev_frame = prev_frame.f_back # type: ignore[assignment] + if ( + prev_frame + and prev_frame.f_code is decorators._nonrecursive_disable_wrapper_code + ): + return ConvertFrameReturn(apply_to_code=False) + + global initial_global_state + initial_global_state = GlobalStateGuard() + + compile_id = get_compile_id(frame_state) + frame_id = compile_id.frame_id + + signpost_event( + "dynamo", + "_convert_frame_assert._compile", + { + "co_name": code.co_name, + "frame_id": frame_id, + "compile_id": str(compile_id), + "co_filename": code.co_filename, + "co_firstlineno": code.co_firstlineno, + "cache_size": cache_size.num_cache_entries_with_same_id_matched_objs, + "accumulated_cache_size": cache_size.num_cache_entries, + }, + ) + + # Record traced frames, skipping Dynamo generated ones. + if not code.co_name.startswith(TORCH_DYNAMO_RESUME_IN_PREFIX): + info = f"{code.co_name} {code.co_filename}:{code.co_firstlineno}" + dynamo_tls.traced_frame_infos.append(info) + + with compile_context(CompileContext(compile_id)): + result = _compile( + frame.f_code, + frame.f_globals, + frame.f_locals, + frame.f_builtins, + frame.closure, + self._torchdynamo_orig_backend, + self._one_graph, + self._export, + self._export_constraints, + hooks, + cache_entry, + cache_size, + frame, + frame_state=frame_state, + compile_id=compile_id, + skip=skip + 1, + package=self._package, + convert_frame_box=self._box, + ) + + if config.caching_precompile and self._package is not None: + from .package import DynamoCache + + # Record that the dynamo package has changed + DynamoCache.record_package(self._package) + return result + + +def convert_frame_assert( + compiler_fn: CompilerFn, + one_graph: bool = True, + export: bool = False, + export_constraints: Optional[typing.Never] = None, + package: Optional[CompilePackage] = None, +) -> ConvertFrameAssert: + """Fully convert a frame into an FX graph, raising an exception if we fail.""" + return ConvertFrameAssert( + compiler_fn, one_graph, export, export_constraints, package + ) + + +from collections import OrderedDict + +from torch.utils.hooks import RemovableHandle + + +# we have to use `OrderedDict` to make `RemovableHandle` work. +_bytecode_hooks: dict[int, BytecodeHook] = OrderedDict() + + +def register_bytecode_hook(hook: BytecodeHook) -> RemovableHandle: + """Register hooks for bytecode generated by Dynamo. The hook can do some + logging, as well as return a new code object to be used. Please refer + to `BytecodeHook` for the hook signature. + """ + handle = RemovableHandle(_bytecode_hooks) + _bytecode_hooks[handle.id] = hook + return handle + + +# TODO - We want to run preserve_node_meta context manager here, but the CI +# fails (its unclear if the failures were flaky) +# @torch.fx.traceback.preserve_node_meta() +@preserve_global_state +def trace_frame( + code: types.CodeType, + globals: dict[str, object], + locals: dict[str, object], + builtins: dict[str, object], + closure: tuple[CellType], + compiler_fn: CompilerFn, + tf_mode_stack: list[torch.overrides.TorchFunctionMode], + one_graph: bool, + speculation_log: SpeculationLog, + instructions: list[Instruction], + code_options: dict[str, object], + *, + export: bool = False, + export_constraints: Optional[typing.Never] = None, + frame_state: Optional[dict[str, Union[int, FrameStateSizeEntry]]] = None, + distributed_state: Optional[DistributedState] = None, + package: Optional[CompilePackage] = None, +) -> DynamoTracerOutput: + from torch.fx.experimental.validator import bisect, translation_validation_enabled + + speculation_log.restart() # type: ignore[has-type] + exn_vt_stack = ExceptionStack() + tracer = InstructionTranslator( + instructions, + code, + locals, + globals, + builtins, + closure, + tf_mode_stack, + code_options, + compiler_fn, + one_graph, + export, + export_constraints, + frame_state=frame_state, + speculation_log=speculation_log, # type: ignore[has-type] + exn_vt_stack=exn_vt_stack, + distributed_state=distributed_state, # type: ignore[has-type] + package=package, + ) + + def run_tracer() -> None: + try: + tracer.output.mark_bytecode_tracing_start() + with tracing(tracer.output.tracing_context), tracer.set_current_tx(): + tracer.run() + except exc.UnspecializeRestartAnalysis: + speculation_log.clear() # type: ignore[has-type] + raise + except ( + exc.SpeculationRestartAnalysis, + exc.TensorifyScalarRestartAnalysis, + exc.SkipFrame, + ): + raise + except Exception: + if translation_validation_enabled(): + bisect(tracer.output.shape_env) + raise + finally: + tracer.output.call_cleanup_hooks() + tracer.f_locals = {} + + try: + run_tracer() + tracer_output = DynamoTracerOutput(tracer) + output = tracer_output.output_graph + assert output is not None + assert output.output_instructions + instructions[:] = output.output_instructions + code_options.update(output.code_options) + propagate_inst_exn_table_entries(instructions) + check_inst_exn_tab_entries_valid(instructions) + instructions[:] = remove_pointless_jumps(remove_dead_code(instructions)) + except Exception as e: + e._torch_dynamo_tracer_output = DynamoTracerOutput(tracer, error=True) # type: ignore[attr-defined] + raise + return tracer_output + + +@dataclass +class DynamoOutput: + """ + Represents the core data returned from a single dynamo run, including: + - Guards, wrapped inside tracer_output.output_graph.guards + - Generated bytecode + - Other information needed for compilation. + This data structure should capture all the "interesting" information dynamo + produces on the frontend side before it enters user backend. + """ + + tracer_output: DynamoTracerOutput + bytecode: types.CodeType + last_attempt_start_time: Optional[float] + + def build_guards( + self, + code: types.CodeType, + hooks: Optional[Hooks] = None, + save: bool = False, + cache_entry: Optional[CacheEntry] = None, + strict_error: bool = False, + ) -> CheckFunctionManager: + output_graph = self.tracer_output.output_graph + assert output_graph is not None + return CheckFunctionManager( + code, + output_graph, + cache_entry, + hooks.guard_fail_fn if hooks else None, + hooks.guard_filter_fn if hooks else None, + save_guards=save, + strict_error=strict_error, + ) + + def graph_capture_output( + self, argdefs: Optional[tuple[Any, ...]] = None + ) -> GraphCaptureOutput: + output_graph = self.tracer_output.output_graph + assert output_graph is not None + return GraphCaptureOutput( + OutputGraphCommon( + output_graph.dump_guards_state(), + output_graph.import_sources, + output_graph.shape_env, + output_graph.export_metadata, + output_graph.tracked_fakes_id_to_source, + ), + output_graph.import_sources, + output_graph.traced_code, + self.bytecode, + self.tracer_output.closure, + argdefs, + self.tracer_output.f_globals, + ) + + +@dataclass +class BackendInput: + """ + Represents core data structure that dynamo will pass to a backend, including: + - Graph module + - Example inputs + - The FakeTensorMode used for compiling graph. + This data structure should capture all the information dynamo produces + on for the user backend. + """ + + backend_id: str + graph_module: torch.fx.GraphModule + example_inputs: Any + fake_mode: torch._subclasses.fake_tensor.FakeTensorMode + tensor_to_context: WeakIdKeyDictionary + + +@dataclass(frozen=True) +class GraphRuntimeEnv: + bytecode: types.CodeType + import_sources: dict[str, str] + used_globals: dict[str, Any] + closure: Optional[tuple[Any, ...]] + argdefs: Optional[tuple[Any, ...]] + external_refs: set[str] = dataclasses.field(default_factory=set) + + def forward_callable( + self, + backend_id: str, + compiled_fn: Callable[..., Any], + *, + extra_globals: Optional[dict[str, Any]] = None, + ) -> Callable[..., Any]: + import_sources = { + alias: importlib.import_module(module_name) + for alias, module_name in self.import_sources.items() + } + f_globals = { + **import_sources, + **self.used_globals, + **(extra_globals or {}), + backend_id: compiled_fn, + } + + # check that all external references are available + self._check_external_refs(f_globals) + + return types.FunctionType( + self.bytecode, + f_globals, + closure=self.closure, + argdefs=self.argdefs, + ) + + def _check_external_refs(self, f_globals: dict[str, Any]) -> None: + missing_refs = [] + for ref in self.external_refs: + if ref not in f_globals: + missing_refs.append(ref) + + if missing_refs: + raise RuntimeError( + f"Missing required external references: {missing_refs}. " + "Please load AOT compiled function with `f_globals=`" + ) + + +@dataclass +class GraphCaptureOutput: + """ + Minimal version of DynamoOutput + """ + + output_graph: OutputGraphCommon + import_sources: dict[str, str] + traced_code: list[CodeType] + bytecode: CodeType + closure: Optional[tuple[Any, ...]] + argdefs: Optional[tuple[Any, ...]] + f_globals: dict[str, Any] + + def build_guards( + self, + code: types.CodeType, + hooks: Optional[Hooks] = None, + save: bool = False, + cache_entry: Optional[CacheEntry] = None, + strict_error: bool = False, + ) -> CheckFunctionManager: + return CheckFunctionManager( + code, + self.output_graph, + cache_entry, + hooks.guard_fail_fn if hooks else None, + hooks.guard_filter_fn if hooks else None, + save_guards=save, + strict_error=strict_error, + ) + + def get_runtime_env(self) -> GraphRuntimeEnv: + from torch._dynamo.source import get_global_source_name + + used_globals = {} + for ( + source + ) in self.output_graph.export_metadata.graph_input_idx_to_local_source.values(): + global_name = get_global_source_name(source) + if global_name is None: + continue + if global_name in self.f_globals: + used_globals[global_name] = self.f_globals[global_name] + + # Scan bytecode for all external references + external_refs = self._get_external_refs(self.bytecode) + + return GraphRuntimeEnv( + bytecode=self.bytecode, + import_sources=self.import_sources, + used_globals=used_globals, + closure=self.closure, + argdefs=self.argdefs, + external_refs=external_refs, + ) + + @staticmethod + def _get_external_refs(bytecode: types.CodeType) -> set[str]: + import dis + + external_refs: set[str] = set() + + # Get all instructions from the bytecode + for instruction in dis.get_instructions(bytecode): + # LOAD_GLOBAL loads a global variable or a builtin + if instruction.opname == "LOAD_GLOBAL": + if instruction.argval: + external_refs.add(instruction.argval) + # LOAD_NAME loads a name (used in module-level code, less common in functions) + elif instruction.opname == "LOAD_NAME": + if instruction.argval: + external_refs.add(instruction.argval) + + return external_refs + + +@dataclass +class CaptureOutput: + """ + CaptureOutput should represent all the information produced from torch + compiler for a single graph capture. This intends to be consumed by + various compiler frontends so that we can share as much compiler internals + as possible and avoid great divergence between different stacks. + This data structure should eventually contain all the information compiler + produces as more refactors happens to converge different compiler + frontends. + """ + + graph_capture_output: GraphCaptureOutput + # BackendInput can be None when dynamo didn't compile any graph (no tensor op) + backend_input: Optional[BackendInput] + + def forward_callable( + self, + *, + compiled_fn: Optional[Callable[..., Any]] = None, + extra_globals: Optional[dict[str, Any]] = None, + ) -> Callable[..., Any]: + runtime_env = self.graph_capture_output.get_runtime_env() + assert self.backend_input is not None + backend_id = self.backend_input.backend_id + # pyrefly: ignore [not-callable] + compiled_fn = compiled_fn or self.backend_input.graph_module + return runtime_env.forward_callable( + backend_id, compiled_fn, extra_globals=extra_globals + ) + + +def get_traced_fn(mod: Any) -> tuple[FunctionType, Optional[object]]: + """ + Utility function to get the function to trace, and optionally a bound self + object, from a callable (nn.Module, function, or method). + """ + import inspect + + if isinstance(mod, torch.nn.Module): + resolved_forward = mod.forward + if hasattr(resolved_forward, "__self__"): + # pyrefly: ignore [missing-attribute] + resolved_forward = resolved_forward.__func__ + + # Mirrored from NNModuleVariable.call_function: + # https://github.com/pytorch/pytorch/blob/main/torch/_dynamo/variables/nn_module.py#L1035 + if ( + len(mod._forward_pre_hooks) == 0 + and len(mod._forward_hooks) == 0 + and len(torch.nn.modules.module._global_forward_pre_hooks) == 0 + and len(torch.nn.modules.module._global_forward_hooks) == 0 + and len(mod._backward_pre_hooks) == 0 + and len(mod._backward_hooks) == 0 + and len(torch.nn.modules.module._global_backward_pre_hooks) == 0 + and len(torch.nn.modules.module._global_backward_hooks) == 0 + and resolved_forward != torch.nn.Module.forward + ): + # We cannot trace __call__ by default because it will break + # the legacy dynamo export. If we want to revisit this, + # feel free to remove this path and try unittests in + # test_strict_export_v2.py + mod = mod.forward + elif isinstance(mod, torch.fx.GraphModule): + mod = mod._call_impl + else: + mod = mod.__call__ + + if hasattr(mod, "__self__"): + # pyrefly: ignore [missing-attribute] + return mod.__func__, mod.__self__ + elif inspect.isfunction(mod): + return mod, None + else: + raise RuntimeError(f"Unsupported model code type {mod}") + + +def _get_signature(fn: Any) -> inspect.Signature: + return inspect.signature(fn, follow_wrapped=False) + + +def _get_frame( + mod: Any, + args: tuple[Any, ...], + kwargs: Optional[dict[str, Any]] = None, +) -> FrameInfo: + """ + Create a frame to trace, given a model, args, and optional kwargs. + """ + import builtins + + fn, self_opt = get_traced_fn(mod) + if self_opt is not None: + args = (self_opt,) + args + if kwargs is None: + kwargs = {} + + signature = _get_signature(fn) + bound_arguments = signature.bind(*args, **kwargs) + bound_arguments.apply_defaults() + f_locals = bound_arguments.arguments + + closure = fn.__closure__ or () + freevars = fn.__code__.co_freevars + if freevars or closure: + assert len(closure) == len(freevars) + f_locals.update( + {name: cell.cell_contents for name, cell in zip(freevars, closure)} + ) + + return FrameInfo( + fn.__code__, + fn.__globals__, + f_locals, + builtins.__dict__, + closure=fn.__closure__ or (), # type: ignore[arg-type] + argdefs=fn.__defaults__, + ) + + +def fullgraph_capture( + mod: Any, + args: tuple[Any, ...], + kwargs: Optional[dict[str, Any]] = None, + *, + constraints: Optional[list[Constraint]] = None, + _is_export_deprecated_do_not_use: bool = False, +) -> CaptureOutput: + """ + This API captures a full graph for a model, given example inputs to trace with. + + Specifically, it takes a callable (nn.Module, method, or function), args, and + optional kwargs, and returns Dynamo-captured graph along with other important + compile-time information. This serves as the common graph-capture mechanism + for different torch compiler AOT frontends (e.g. AOT precompile, export). + + Note that this API doesn't apply context managers like metrics context, + and the expectation is that the caller will apply them depending + on the use case. + + The CaptureOutput is separated into two parts: + 1. Frontend specific information, which includes: + - guards + - generated bytecode + - other information tracked by OutputGraphCommon. + 2. Backend specific information (indexed by unique backend id) such as: + - fx graph + - example inputs + """ + frame = _get_frame(mod, args, kwargs) + + with compile_context(CompileContext(get_compile_id({}))): + return _fullgraph_capture_frame( + frame, + constraints=constraints, + _is_export_deprecated_do_not_use=_is_export_deprecated_do_not_use, + ) + + +@dataclass +class FrameInfo: + code: types.CodeType + globals: dict[str, object] + locals: dict[str, object] + builtins: dict[str, object] + closure: tuple[CellType] + argdefs: Optional[tuple[Any, ...]] + + +def _fullgraph_capture_frame( + frame: FrameInfo, + *, + constraints: Optional[list[Constraint]] = None, + _is_export_deprecated_do_not_use: bool = False, +) -> CaptureOutput: + from torch._guards import TracingContext + + backend_input: Optional[BackendInput] = None + + def fullgraph_compiler( + gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor] + ) -> torch.fx.GraphModule: + nonlocal backend_input + tracing_context = TracingContext.get() + fake_mode = tracing_context.fake_mode + tensor_to_context = tracing_context.tensor_to_context + assert fake_mode is not None + assert isinstance(gm.meta["backend_id"], str) + backend_input = BackendInput( + gm.meta["backend_id"], gm, example_inputs, fake_mode, tensor_to_context + ) + return gm + + try: + dynamo_output = compile_frame( + frame.code, + frame.globals, + frame.locals, + frame.builtins, + frame.closure, + compiler_fn=fullgraph_compiler, + export=_is_export_deprecated_do_not_use, + export_constraints=constraints, # type: ignore[arg-type] + one_graph=True, + restart_reasons=set(), + ) + # https://github.com/pytorch/pytorch/blob/main/torch/_dynamo/eval_frame.py#L831 + except Unsupported as e: + augment_exc_message(e) + if config.verbose: + raise + # strip internal tracebacks from causes + cur_exn: BaseException = e + while cur_exn.__cause__ is not None: + cur_exn.__cause__.with_traceback(None) + cur_exn = cur_exn.__cause__ + # pyrefly: ignore [invalid-inheritance] + raise e.with_traceback(None) from e.__cause__ # User compiler error + + return CaptureOutput( + dynamo_output.graph_capture_output(frame.argdefs), + backend_input, + ) + + +def compile_frame( # type: ignore[return] + code: types.CodeType, + globals: dict[str, object], + locals: dict[str, object], + builtins: dict[str, object], + closure: tuple[CellType], + compiler_fn: CompilerFn, + one_graph: bool, + restart_reasons: set[str], + *, + export: bool = False, + export_constraints: Optional[typing.Never] = None, + frame_state: Optional[dict[str, Union[int, FrameStateSizeEntry]]] = None, + distributed_state: Optional[DistributedState] = None, + package: Optional[CompilePackage] = None, + # pyrefly: ignore [bad-return] +) -> DynamoOutput: + """ + A helper function taking a frame and backend, then return the generated bytecode + and guards as a common data structure. + This is a shared interface for multiple compiler frontends (e.g. torch.compile, + torch.export) that needs to capture a graph out of python code. + """ + # This is shared across restarts + speculation_log = SpeculationLog() + + def transform( + instructions: list[Instruction], code_options: dict[str, object] + ) -> DynamoTracerOutput: + tf_mode_stack: list[torch.overrides.TorchFunctionMode] = ( + torch.overrides._get_current_function_mode_stack() + ) + tracer_output = trace_frame( + code, + globals, + locals, + builtins, + closure, + compiler_fn, + tf_mode_stack, + one_graph, + speculation_log, + instructions, + code_options, + export=export, + export_constraints=export_constraints, + frame_state=frame_state, + distributed_state=distributed_state, + package=package, + ) + + assert tracer_output is not None + return tracer_output + + last_attempt_start_time = None + for attempt in itertools.count(): + CompileContext.get().attempt = attempt + + try: + with dynamo_timed(f"compile_attempt_{attempt}", log_pt2_compile_event=True): + bytecode, tracer_output = transform_code_object(code, transform) + assert tracer_output is not None + return DynamoOutput( + tracer_output=tracer_output, + bytecode=bytecode, + last_attempt_start_time=last_attempt_start_time, + ) + except exc.RestartAnalysis as e: + if not isinstance(e, exc.TensorifyScalarRestartAnalysis): + TensorifyState.clear() + log.info( + "Restarting analysis due to %s", + LazyString(format_traceback_short, e.__traceback__), + ) + # Clean up the failed tracer output's graph to break reference cycles + failed_tracer_output = getattr(e, "_torch_dynamo_tracer_output", None) + if failed_tracer_output: + failed_tracer_output._cleanup_output_graph() + # If restart reason is None just log the type of the exception + restart_reasons.add(e.restart_reason or str(type(e))) + # We now have a new "last attempt", reset the clock + last_attempt_start_time = time.time() + if attempt > 100: + unimplemented( + gb_type="Excessive RestartAnalysis() calls", + context="", + explanation="Dynamo attempted to trace the same frame 100+ times. " + "Giving up on compiling as the compile time tradeoff is likely not " + "worth the performance gain.", + hints=[], + ) + except exc.SkipFrame as e: + if not isinstance(e, exc.TensorifyScalarRestartAnalysis): + TensorifyState.clear() + # Clean up the failed tracer output's graph to break reference cycles + failed_tracer_output = getattr(e, "_torch_dynamo_tracer_output", None) + if failed_tracer_output: + failed_tracer_output._cleanup_output_graph() + log.debug( # noqa: G200 + "Skipping frame %s %s \ + %s %s", + e, + code.co_name, + code.co_filename, + code.co_firstlineno, + ) + raise + + +def _compile( + code: CodeType, + globals: dict[str, object], + locals: dict[str, object], + builtins: dict[str, object], + closure: tuple[CellType], + compiler_fn: CompilerFn, + one_graph: bool, + export: bool, + export_constraints: Optional[typing.Never], + hooks: Hooks, + cache_entry: Optional[CacheEntry], + cache_size: CacheSizeRelevantForFrame, + frame: Optional[DynamoFrameType] = None, + frame_state: Optional[dict[str, Union[int, FrameStateSizeEntry]]] = None, + *, + compile_id: CompileId, + skip: int = 0, + package: Optional[CompilePackage] = None, + # Can be used to record things for the caller, both + # in the case of normal and exception code paths + convert_frame_box: Optional[ConvertFrameBox] = None, +) -> ConvertFrameReturn: + from torch.fx.experimental.validator import ( + BisectValidationException, + ValidationException, + ) + + # Only nonlocal defs here please! + # Time spent compiling this frame before restarting or failing analysis + dynamo_time_before_restart: float = 0.0 + + @compile_time_strobelight_meta(phase_name="compile_inner") + def compile_inner( + code: CodeType, one_graph: bool, hooks: Hooks + ) -> tuple[ConvertFrameReturn, Optional[DynamoTracerOutput]]: + with contextlib.ExitStack() as stack: + stack.enter_context( + torch._dynamo.callback_handler.install_callbacks( + CallbackTrigger.DYNAMO, str(CompileContext.current_compile_id()) + ) + ) + stack.enter_context(CompileTimeInstructionCounter.record()) + return _compile_inner(code, one_graph, hooks) + + return ( + ConvertFrameReturn(), + None, + ) # dead, but see https://github.com/python/mypy/issues/7577 + + @maybe_cprofile + def _compile_inner( + code: CodeType, + one_graph: bool, + hooks: Hooks, + ) -> tuple[ConvertFrameReturn, DynamoTracerOutput]: + nonlocal dynamo_time_before_restart + last_attempt_start_time = start_time = time.time() + + def log_bytecode( + prefix: str, name: str, filename: str, line_no: int, code: CodeType + ) -> None: + if bytecode_log.isEnabledFor(logging.DEBUG): + bytecode_log.debug( + format_bytecode(prefix, name, filename, line_no, code) + ) + + log_bytecode( + "ORIGINAL BYTECODE", + code.co_name, + code.co_filename, + code.co_firstlineno, + code, + ) + + out_code = None + try: + dynamo_output = compile_frame( + code, + globals, + locals, + builtins, + closure, + compiler_fn, + one_graph, + restart_reasons, + export=export, + export_constraints=export_constraints, + frame_state=frame_state, + distributed_state=distributed_state, + package=package, + ) + except exc.SkipFrame as e: + if one_graph: + log.debug("No graph captured with export/fullgraph=True") + assert e._torch_dynamo_tracer_output is not None + return ConvertFrameReturn(), e._torch_dynamo_tracer_output + + assert distributed_state is None or distributed_state.all_states is not None, ( # type: ignore[has-type] + "compiler collective wasn't run before compilation completed" + ) + out_code = dynamo_output.bytecode + tracer_output = dynamo_output.tracer_output + if dynamo_output.last_attempt_start_time is not None: + last_attempt_start_time = dynamo_output.last_attempt_start_time + + assert out_code is not None + log_bytecode( + "MODIFIED BYTECODE", + code.co_name, + code.co_filename, + code.co_firstlineno, + out_code, + ) + + for idx, hook in enumerate(_bytecode_hooks.values()): + with dynamo_timed(f"bytecode_hooks_{idx}", log_pt2_compile_event=True): + hook_output = hook(code, out_code) + if hook_output is not None: + out_code = hook_output + + orig_code_map[out_code] = code + output_codes.add(out_code) + dynamo_time_before_restart = last_attempt_start_time - start_time + assert tracer_output.output_graph is not None + output = tracer_output.output_graph + + # Tests for new code objects. + # The rationale for these tests can be found in torch/csrc/dynamo/eval_frame.c + # Only test once the code object is created. + # They are not tested during runtime. + + def count_args(code: CodeType) -> int: + import inspect + + return ( + code.co_argcount + + code.co_kwonlyargcount + + bool(code.co_flags & inspect.CO_VARARGS) + + bool(code.co_flags & inspect.CO_VARKEYWORDS) + ) + + assert out_code is not None + + total_argcount_old = count_args(code) + total_argcount_new = count_args(out_code) + msg = "arg mismatch: " + msg += f"old code object has args {code.co_varnames[:total_argcount_old]}, " + msg += f"new code object has args {out_code.co_varnames[:total_argcount_new]}" + assert ( + code.co_varnames[:total_argcount_old] + == out_code.co_varnames[:total_argcount_new] + ), msg + + msg = "free var mismatch: " + msg += f"old code object has free var {code.co_freevars}, " + msg += f"new code object has free var {out_code.co_freevars}" + assert code.co_freevars == out_code.co_freevars, msg + + msg = "cell var mismatch: " + msg += f"old code object has cell var {code.co_cellvars}, " + msg += f"new code object has cell var {out_code.co_cellvars}" + assert code.co_cellvars == out_code.co_cellvars, msg + + # Skipping Dynamo on a frame without any extracted graph. + # This does not affect eager functionality. But this is necessary + # for export for cases where Dynamo-reconstructed bytecode can create + # new function frames, confusing export in thinking that there + # are extra graphs now. + + if output.export and output.is_empty_graph(): + return ConvertFrameReturn(), tracer_output + + assert output.guards is not None + CleanupManager.instance[out_code] = output.cleanups + nonlocal cache_entry + with dynamo_timed("build_guards", log_pt2_compile_event=True): + check_fn = dynamo_output.build_guards( + code, + hooks=hooks, + save=package is not None, + cache_entry=cache_entry, + ) + + if package is not None: + assert check_fn.guards_state is not None + package.add_guarded_code(check_fn.guards_state, out_code) + package.add_inlined_source(output.tracing_context.traced_code) + package.update_device_type(output.current_tracer.graph) + + compile_id_str = str(compile_id) if compile_id is not None else "Unknown" + annotation_str = "Torch-Compiled Region: " + compile_id_str + guarded_code = GuardedCode( + out_code, + check_fn.guard_manager, # type: ignore[arg-type] + compile_id, + annotation_str, + ) + + if not output.is_empty_graph() and hooks.guard_export_fn is not None: + # We should not run the guard_export_fn when Dynamo does not + # generate any graph. This can happen in export when TorchDynamo + # generated bytecode has some reconstruction logic for mutated + # variables which can trigger TorchDynamo on the children frames but + # they are benign and do not generate any new graphs. + hooks.guard_export_fn(output.guards) + + return wrap_guarded_code(guarded_code), tracer_output + + metrics_context = get_metrics_context() + code_context = ( + package.code_context(code) if package is not None else contextlib.nullcontext() + ) + with ( + _use_lazy_graph_module(config.use_lazy_graph_module), + compile_context(CompileContext(compile_id)), + chromium_event_timed( + "dynamo", reset_event_log_on_exit=True, log_pt2_compile_event=True + ), + _WaitCounter("pytorch.wait_counter.entire_forward_compile").guard(), + metrics_context, + dynamo_timed( + "_compile.compile_inner", + phase_name="entire_frame_compile", + dynamo_compile_column_us="dynamo_cumulative_compile_time_us", + ), + code_context, + ): + restart_reasons: set[str] = set() + if compile_pg := get_compile_pg(): + distributed_state = DistributedState(compile_pg, LocalState()) + else: + distributed_state = None + + # Check recompilations + recompile_reason: Optional[str] = None + if is_recompilation(cache_size) and frame: + reasons = get_and_maybe_log_recompilation_reasons( + cache_entry, frame, innermost_fn(compiler_fn) + ) + recompile_reason = ( + "Unable to find recompilation reasons" if not reasons else reasons[0] + ) + # Recheck for recompilation, for when inline_inbuilt_nn_modules is set to False + inline_inbuilt_nn_modules_candidate = False + if not config.inline_inbuilt_nn_modules and frame: + inbuilt_nn_reasons = get_and_maybe_log_recompilation_reasons( + cache_entry, frame, innermost_fn(compiler_fn), skip_logging=True + ) + inbuilt_nn_recompile_reason = ( + None if not inbuilt_nn_reasons else inbuilt_nn_reasons[0] + ) + + if ( + inbuilt_nn_recompile_reason is not None + and "[inline-inbuilt-nn-modules-candidate]" + in inbuilt_nn_recompile_reason + ): + inline_inbuilt_nn_modules_candidate = True + + # Set if the recompile is a candidate for inline_inbuilt_nn_modules + # regardless of whether inline_inbuilt_nn_modules is set or not + metrics_context.update_outer( + { + "recompile_reason": recompile_reason, + "inline_inbuilt_nn_modules_candidate": inline_inbuilt_nn_modules_candidate, + } + ) + + recompile_user_contexts = get_hook_for_recompile_user_context() + if recompile_user_contexts: + # cap each user context to N chars for data retention purposes. N=256 + # is chosen to be large enough to capture the most important info. + user_contexts_msg = { + user_context()[:256] for user_context in recompile_user_contexts + } + metrics_context.set("recompile_user_contexts", user_contexts_msg) + + exceeded, limit_type = exceeds_recompile_limit(cache_size, compile_id) + if exceeded: + + def format_func_info(code: CodeType) -> str: + return f"'{code.co_name}' ({code.co_filename}:{code.co_firstlineno})" + + # NS: Don't add period at the end of string, as it'll be added to URL + # rendering it incorrect + log.warning( + "torch._dynamo hit config.%s (%s)\n" + " function: %s\n" + " last reason: %s\n" + 'To log all recompilation reasons, use TORCH_LOGS="recompiles".\n' + "To diagnose recompilation issues, see %s", + limit_type, + getattr(config, limit_type), + format_func_info(code), + recompile_reason, + troubleshooting_url, + ) + if config.fail_on_recompile_limit_hit: + raise FailOnRecompileLimitHit( + f"{limit_type} reached, because fail_on_recompile_limit_hit = True this is a HARD failure" + ) + elif one_graph: + raise FailOnRecompileLimitHit( + f"{limit_type} reached with fullgraph=True. Excessive recompilations can degrade " + "performance due to the compilation overhead of each recompilation. To monitor " + "recompilations, enable TORCH_LOGS=recompiles. If recompilations are expected, consider " + "increasing torch._dynamo.config.cache_size_limit to an appropriate value." + ) + elif justknobs_check( + "pytorch/compiler:skip_code_recursive_on_recompile_limit_hit" + ): + raise RecompileLimitExceeded(f"{limit_type} reached") + else: + # do not recursively skip frames + unimplemented( + gb_type="Dynamo cache limit exceeded", + context=f"Limit type: {limit_type}", + explanation="Dynamo attempted to recompile the code object too many times, " + f"exceeding the {limit_type} cache size limit." + "Giving up on compiling as the compile time tradeoff is likely not " + "worth the performance gain.", + hints=[], + ) + + log.debug( + "torchdynamo start compiling %s %s:%s, stack (elided %s frames):\n%s", + code.co_name, + code.co_filename, + code.co_firstlineno, + skip + 2, + # -2: omit current frame, omit contextlib decorator + "".join(CapturedTraceback.extract(skip=2 + skip).format()), + ) + # -4: -2 as above, plus trace_structured frames + # + # NB: the frame looks like this: + # + # # handled by skip argument + # torch/_dynamo/convert_frame.py:1069 in catch_errors + # torch/_dynamo/convert_frame.py:910 in _convert_frame + # torch/_dynamo/convert_frame.py:464 in _convert_frame_assert + # torch/_utils_internal.py:70 in wrapper_function + # + # # 2 current frame and context lib + # env/lib/python3.10/contextlib.py:79 in inner + # torch/_dynamo/convert_frame.py:776 in _compile + # + # # 2 extra here + # torch/_logging/_internal.py:1064 in trace_structured + # torch/_dynamo/convert_frame.py:780 in + stack_trace = log_dynamo_start(code, skip) + start_time_ns = time.time_ns() + fail_type: Optional[str] = None + fail_reason: Optional[str] = None + exception_stack_trace: Optional[list[str]] = None + fail_user_frame_filename: Optional[str] = None + fail_user_frame_lineno: Optional[int] = None + torch._dynamo.utils.ReinplaceCounters.clear() + guarded_code = None + tracer_output = None + try: + guarded_code, tracer_output = compile_inner(code, one_graph, hooks) + + # NB: We only put_code_state in success case. Success case here + # does include graph breaks; specifically, if a graph break still + # resulted in a partially compiled graph, we WILL return here. An + # Unsupported exception will only bubble to the top level if we + # are unable to compile the frame at all. In this case, there's + # no point in uploading the code state, because we will always + # fail exactly the same way even without the update. (It's useful + # to upload for graph break though, because this can prevent + # extra graph break compilations.) + put_code_state() + if ( + tracer_output + and (output_graph := tracer_output.output_graph) + and output_graph.has_outputs() + ): + log_frame_dynamic_whitelist(code) + if recompile_reason and "size mismatch at index" in recompile_reason: + _log_size_mismatch_recompile() + + return guarded_code + except Exception as e: + # NB: e's msg is mutated here to add user stack, but we DON'T want + # that stack in the Scuba logged fail_reason. So we grab the fail + # info here and add it to the metrics context below. + fail_type = type(e).__qualname__ + fail_reason = str(e) + exception_stack_trace = [traceback.format_exc()] + exception_handler(e, code, frame, export=export) + # NB: this is the post-mutation exception + torch._logging.trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "dynamo_error", + "encoding": "string", + }, + payload_fn=lambda: traceback.format_exc(), + ) + fail_user_frame_filename, fail_user_frame_lineno = exc.get_exc_message( + e, compile_id + ) + tracer_output = getattr(e, "_torch_dynamo_tracer_output", None) + if isinstance( + e, + ( + Unsupported, + TorchRuntimeError, + BackendCompilerFailed, + AssertionError, + ConstraintViolationError, + GuardOnDataDependentSymNode, + ValidationException, + UncapturedHigherOrderOpError, + BisectValidationException, + ShortenTraceback, + PackageError, + ResumePrologueTracingError, + ), + ): + raise + else: + # Rewrap for clarity + raise InternalTorchDynamoError( + f"{type(e).__qualname__}: {str(e)}" + ).with_traceback(e.__traceback__) from None + finally: + # === WARNING WARNING WARNING === + # If you commit a bug here, it will suppress writing to + # dynamo_compile table, and we will not have telemetry. + # Be extra careful when making changes here! + + if torch._dynamo.config.run_gc_after_compile: + with dynamo_timed("gc", dynamo_compile_column_us="gc_time_us"): + log.info("run_gc_after_compile: running gc") + gc.collect(1) + + output = None + if tracer_output: + output = tracer_output.output_graph + if output: + output.local_scope = {} + # tracer should already be None, keep an extra check here just in case. + if tracer := output.root_tx: + tracer.f_locals = {} + + from .utils import curr_frame + + frame_key = str(curr_frame) + if fail_reason is None and output is not None: + guard_count = len(output.guards) + shape_env_guard_count = len(output.shape_env.guards) + graph_op_count = output.count_calls() + graph_node_count = len(output.graph.nodes) + graph_node_shapes = output.get_graph_sizes_structured() + graph_input_count = len(output.placeholders) + non_compliant_ops = {op.__qualname__ for op in output.non_compliant_ops} + compliant_custom_ops = { + op.__qualname__ for op in output.compliant_custom_ops + } + torch._dynamo.utils.ReinplaceCounters.log() + else: + guard_count = None + shape_env_guard_count = None + graph_op_count = None + graph_node_count = None + graph_node_shapes = {} + graph_input_count = None + non_compliant_ops = set({}) + compliant_custom_ops = set({}) + restart_reasons = set() + # If compilation failed, the entire time is wasted + dynamo_time_before_restart = (time.time_ns() - start_time_ns) / 1e9 + + metrics = { + "frame_key": frame_key, + "co_name": code.co_name, + "co_filename": code.co_filename, + "co_firstlineno": code.co_firstlineno, + "cache_size": cache_size.num_cache_entries_with_same_id_matched_objs, + "accumulated_cache_size": cache_size.num_cache_entries, + "guard_count": guard_count, + "shape_env_guard_count": shape_env_guard_count, + "graph_op_count": graph_op_count, + "graph_node_count": graph_node_count, + "graph_input_count": graph_input_count, + "fail_type": fail_type, + "fail_reason": fail_reason, + "fail_user_frame_filename": fail_user_frame_filename, + "fail_user_frame_lineno": fail_user_frame_lineno, + "non_compliant_ops": non_compliant_ops, + "compliant_custom_ops": compliant_custom_ops, + "restart_reasons": restart_reasons, + "dynamo_time_before_restart_s": dynamo_time_before_restart, + "has_guarded_code": guarded_code is not None, + "specialize_float": config.specialize_float, + "is_forward": True, + "dynamo_compile_time_before_restart_us": to_int_us( + dynamo_time_before_restart + ), + "stack_trace": stack_trace, + "graph_node_shapes": str(graph_node_shapes), + "exception_stack_trace": exception_stack_trace, + } + # TODO: replace with CompileEventLogger.compilation_metrics + # There are some columns here not in PT2 Compile Events + # so we need to slightly change it + metrics_context.update_outer(metrics) + # === END WARNING WARNING WARNING === + + # If tracer is available, then tracer.error_on_graph_break reflects value of + # global symbolic_convert.error_on_graph_break at the time of the graph break - + # symbolic_convert.error_on_graph_break may have been (correctly) changed during cleanup. + # If tracer is unavailable, then fallback to symbolic_convert.error_on_graph_break. + if convert_frame_box: + convert_frame_box.error_on_graph_break = ( + tracer_output.error_on_graph_break + if tracer_output + else _get_error_on_graph_break() + ) + + +class ConvertFrame: + def __init__( + self, + compiler_fn: CompilerFn, + hooks: Hooks, + package: Optional[CompilePackage] = None, + ) -> None: + self._torchdynamo_orig_backend = compiler_fn + self._inner_convert = convert_frame_assert( + compiler_fn, one_graph=False, package=package + ) + self._hooks = hooks + + @property + def _clone_with_backend(self) -> Callable[[WrapBackendDebug], ConvertFrame]: + return lambda backend: convert_frame( + backend, + self._hooks, + ) + + def __call__( + self, + frame: DynamoFrameType, + cache_entry: Optional[CacheEntry], + hooks: Hooks, + frame_state: dict[str, Union[int, FrameStateSizeEntry]], + skip: int = 0, + ) -> ConvertFrameReturn: + input_codes.add(frame.f_code) + counters["frames"]["total"] += 1 + try: + result = self._inner_convert( + frame, cache_entry, hooks, frame_state, skip=skip + 1 + ) + counters["frames"]["ok"] += 1 + return result + except Exception as e: + # Do not allow errors to be suppressed if we're tracing a resume function prologue + if isinstance(e, ResumePrologueTracingError): + raise + + error_on_graph_break = ( + self._inner_convert._box.error_on_graph_break is not None + ) + assert error_on_graph_break is not None + if self._inner_convert._box.error_on_graph_break: + # NOTE we _might_ have to wrap the current in a custom exception + # in order to correctly bubble up to the top-level compile wrapper in + # eval_frame.py. But re-raising seems to work for now because exceptions from tracing + # a nested call that results in a top-level frame compile will be handled by the caller + # as an observed exception - we don't expect that exception to be suppressed. + raise + + # These two exception types are "soft" failure, in the sense that + # we know this is due to something we didn't implement all the + # way, scare the user less about it. That being said, if you + # are trying to understand why a graph break happened, it's still + # important to have this information, so offer it. + # + # NB: NotImplementedError used to be on this list, but actually + # it is impossible for it to reach here, as it is converted into + # InternalTorchDynamoError. This behavior seemed reasonable + # to me (ezyang, Aug 2023) so I kept it, but maybe at some point + # someone wanted these to also get suppressed. If so, you'll + # need to make these exceptions not get wrapped + + # We intentionally don't want to suppress error here. + if isinstance(e, UncapturedHigherOrderOpError): + raise + + soft_fail = isinstance(e, Unsupported) + code = frame.f_code + # This is a soft failure. In the sense, the code path reaches here + # when we do not support graph breaks on bytecodes like LOAD_ATTR, + # BUILD_SET etc. In such case, we can fallback to eager without + # scaring users. + if soft_fail and graph_break_log.isEnabledFor(logging.DEBUG): + # Log this message in the graph break. Also use the string + # "skip: " to tell that the whole frame is falling back to + # eager. + if hasattr(e, "compile_id") and hasattr(e, "real_stack"): + with compile_context(CompileContext(e.compile_id)): # type: ignore[attr-defined] + user_stack = e.real_stack + user_stack_formatted = "".join( + traceback.format_list(user_stack) + ) + frame_info = exc.format_frame_info(code) + user_stack_trace = ( + "Graph break: torch.compile cannot properly resume from this graph break, which results in a skip.\n" + f"torch.compile will skip tracing the frame {frame_info} and fall back to eager.\n" + "The graph break occurred in the following user code:\n" + f"{user_stack_formatted}" + ) + torch._logging.trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "dynamo_graph_break_reason", + "encoding": "string", + }, + payload_fn=lambda: f"{user_stack_trace}\n{traceback.format_exc()}", + ) + graph_break_log.debug( + user_stack_trace, + exc_info=True, + stack_info=config.verbose, + ) + + if not config.suppress_errors and not soft_fail: + raise + + # Suppress the error. NB: It's very important to do the + # suppression logging HERE, where the actual suppression + # happens. Previously it was somewhere else and so it was + # possible to accidentally not log at all. + record_filename = getattr(e, "record_filename", None) + code = frame.f_code + error_msg = format_error_msg(e, code, record_filename, frame) + + if soft_fail: + log.info(error_msg, exc_info=True) + else: + log.warning(error_msg, exc_info=True) + + if isinstance(e, SkipCodeRecursiveException): + return ConvertFrameReturn( + frame_exec_strategy=FrameExecStrategy( + FrameAction.SKIP, FrameAction.SKIP + ) + ) + elif isinstance(e, RecompileLimitExceeded): + return ConvertFrameReturn( + frame_exec_strategy=FrameExecStrategy( + FrameAction.RUN_ONLY, FrameAction.RUN_ONLY + ) + ) + + return ConvertFrameReturn() + + +def convert_frame( + compiler_fn: CompilerFn, + hooks: Hooks, + package: Optional[CompilePackage] = None, +) -> ConvertFrame: + """Try to convert a frame into an FX graph, if error leave frame unmodified""" + return ConvertFrame(compiler_fn, hooks, package=package) + + +# TODO mlazos: add support for same args, or record them +def replay(filename: str) -> None: + from .backends.debugging import eager + + original_replay_val = config.replay_record_enabled + config.replay_record_enabled = False + with open(filename, "rb") as in_file: + record = ExecutionRecord.load(in_file) + record.globals = dict(itertools.chain(record.globals.items(), globals().items())) + + with decorators.error_on_graph_break(False): + try: + _compile( + record.code, + record.globals, + record.locals, + record.builtins, + record.closure, + compiler_fn=eager, + one_graph=False, + export=False, + export_constraints=None, + hooks=Hooks(), + cache_size=CacheSizeRelevantForFrame(0, 0), + cache_entry=None, + frame=None, + frame_state={}, + compile_id=CompileId(frame_id=42, frame_compile_id=999), + ) + finally: + config.replay_record_enabled = original_replay_val + + +def first_real_inst_idx(code: CodeType) -> int: + if sys.version_info < (3, 11): + return 0 + for inst in dis.get_instructions(code): + if inst.opname == "RESUME": + return inst.offset // 2 + raise RuntimeError("RESUME instruction not found in code") + + +class ConvertFrameProtocol(typing.Protocol): + def __call__( + self, + frame: DynamoFrameType, + cache_entry: Optional[CacheEntry], + hooks: Hooks, + frame_state: dict[str, Union[int, FrameStateSizeEntry]], + *, + skip: int = 0, + ) -> ConvertFrameReturn: ... + + +def should_skip_due_to_torch_dispatch_mode() -> bool: + return is_in_any_mode_without_ignore_compile_internals() + + +class CatchErrorsWrapper: + def __init__(self, callback: ConvertFrameProtocol, hooks: Hooks) -> None: + functools.wraps(callback)(self) + self._torchdynamo_orig_backend = callback + self.hooks = hooks + + def __call__( + self, + frame: DynamoFrameType, + cache_entry: Optional[CacheEntry], + frame_state: dict[str, Union[int, FrameStateSizeEntry]], + ) -> ConvertFrameReturn: + assert frame_state is not None + input_codes.add(frame.f_code) + + is_skipfile = trace_rules.check(frame.f_code) + if sys.version_info >= (3, 13): + has_started_execution = frame.f_lasti > first_real_inst_idx(frame.f_code) + else: + has_started_execution = frame.f_lasti >= first_real_inst_idx(frame.f_code) + if ( + # TODO: the first condition is not covered by any test + has_started_execution + or is_skipfile + or config.disable + or ( + should_skip_due_to_torch_dispatch_mode() + and not getattr(self._torchdynamo_orig_backend, "_export", False) + ) + ): + if log.isEnabledFor(logging.DEBUG): + if has_started_execution: + skip_reason = "traced frame already" + elif trace_rules.check(frame.f_code): + skip_reason = "in skipfiles" + elif is_in_torch_dispatch_mode(include_infra_modes=False): + skip_reason = "non-infra torch dispatch mode present, this is not supported today in torch.compile" + else: + skip_reason = "dynamo tracing is disabled" + + log.debug( + "skipping: %s (reason: %s, file: %s)", + frame.f_code.co_name, + skip_reason, + frame.f_code.co_filename, + ) + return ConvertFrameReturn() + + if ( + frame.f_code.co_filename == "" and frame.f_code.co_name == "__new__" + ) or ( + frame.f_code.co_filename.endswith("collections/__init__.py") + and frame.f_code.co_name == "_make" + ): + # nametuple constructor/_make + return ConvertFrameReturn() + if torch._dynamo.utils.get_optimize_ddp_mode() == "ddp_optimizer": + ddp_module = DistributedDataParallel._get_active_ddp_module() + if ddp_module: + with compile_lock: + from torch._dynamo.backends.distributed import DDPOptimizer + + ddp_optimizer = DDPOptimizer( + bucket_bytes_cap=ddp_module.bucket_bytes_cap, + backend_compile_fn=self._torchdynamo_orig_backend._torchdynamo_orig_backend, # type: ignore[attr-defined] + ) + assert hasattr( + self._torchdynamo_orig_backend, "_clone_with_backend" + ), ( + "DDPOptimizer only supports callback fns that know how to clone themselves." + ) + hijacked_callback = ( + self._torchdynamo_orig_backend._clone_with_backend( + ddp_optimizer.compile_fn, + ) + ) + return hijacked_callback( + frame, cache_entry, self.hooks, frame_state + ) + + with compile_lock, _disable_current_modes(): + # skip=1: skip this frame + result = self._torchdynamo_orig_backend( + frame, cache_entry, self.hooks, frame_state, skip=1 + ) + return result + + +def catch_errors_wrapper( + callback: ConvertFrameProtocol, hooks: Hooks +) -> CatchErrorsWrapper: + return CatchErrorsWrapper(callback, hooks) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/create_parameter_op.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/create_parameter_op.py new file mode 100644 index 0000000000000000000000000000000000000000..2a716865c3f48e4355af10b5ff3fcb2268d1ebc0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/create_parameter_op.py @@ -0,0 +1,69 @@ +import threading +from collections.abc import Generator +from contextlib import contextmanager +from typing import Any + +import torch + + +# See [Note: Metadata mutation in proxy tracing] for why sacrificial parameter mutates +# metadata during proxy tracing and we should remove the sacrificial parameter logic. +doc = """ +This is used when dynamo traces torch.nn.Parameter, which normally would not trace properly +with AOTAutograd. We instead create a placeholder torch.nn.Parameter before the graph, which +becomes a graph arg and has no storage backing it. At the point in the graph where the parameter +actually should be created we mutate this sacrificial placeholder into it. This allows gradients +to flow into the parameter as if it were an input to the graph (which is the only thing we are +allowed to compute gradients on). +""".strip() + + +class TracableCreateParameter(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(ctx: Any, tensor: Any, placeholder: Any) -> torch.nn.Parameter: + assert not tensor.requires_grad + return placeholder.set_(tensor) + + @staticmethod + def backward(ctx: Any, *grad_outputs: torch.Tensor) -> tuple[None, torch.Tensor]: + grad = grad_outputs[0] + return None, grad # grad flows to placeholder + + +def tracable_create_parameter( + tensor: torch.Tensor, placeholder: torch.nn.Parameter +) -> torch.nn.Parameter: + with torch.set_grad_enabled(placeholder.requires_grad): + out = TracableCreateParameter.apply(tensor, placeholder) + return out + + +def new_parameter_placeholder( + size: tuple[int, ...], dtype: torch.dtype, device: torch.device, requires_grad: bool +) -> torch.nn.Parameter: + """Create a placeholder to be passed to the above functions""" + result = torch.nn.Parameter( + torch.empty(size, dtype=dtype, device=device), requires_grad=requires_grad + ) + # TODO(jansel): alloc followed by free is inefficient, need a way to allocate an unbacked tensor. + # Allocating a zero tensor would causes assert failures in autograd. + result.untyped_storage().resize_(0) + return result + + +_TLS = threading.local() + + +@contextmanager +def do_not_convert_to_tracable_parameter() -> Generator[bool, None, None]: + old_flag = getattr(_TLS, "convert_tracable_parameter", True) + _TLS.convert_tracable_parameter = False + try: + yield False + finally: + _TLS.convert_tracable_parameter = old_flag + + +def can_convert_to_tracable_parameter() -> bool: + return getattr(_TLS, "convert_tracable_parameter", True) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/current_scope_id.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/current_scope_id.py new file mode 100644 index 0000000000000000000000000000000000000000..74a5f4888c64629f3225118d91b52ba05e000ce0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/current_scope_id.py @@ -0,0 +1,42 @@ +""" +Provides thread-local scope identification for SubgraphTracer instances. + +This module implements a thread-safe mechanism for tracking nested tracing contexts, +which is essential when multiple SubgraphTracer instances are active. The scope ID +helps identify which tracer context is currently active when direct access to the +InstructionTranslator is difficult. + +Key components: +- Thread-local scope ID storage (_current_scope_id) +- Getter function (current_scope_id) to safely access the current scope +- Context manager (enter_new_scope) for managing nested scope transitions + +The scope ID increments when entering a new context and decrements when exiting, +allowing proper tracking of nested tracing operations across different threads. +""" + +import contextlib +import threading +from collections.abc import Generator + + +# Global variable to identify which SubgraphTracer we are in. +# It is sometimes difficult to find an InstructionTranslator to use. +_current_scope_id = threading.local() + + +def current_scope_id() -> int: + global _current_scope_id + if not hasattr(_current_scope_id, "value"): + _current_scope_id.value = 1 + return _current_scope_id.value + + +@contextlib.contextmanager +def enter_new_scope() -> Generator[None, None, None]: + global _current_scope_id + try: + _current_scope_id.value = current_scope_id() + 1 + yield + finally: + _current_scope_id.value = current_scope_id() - 1 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/dce_extra_outputs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/dce_extra_outputs.py new file mode 100644 index 0000000000000000000000000000000000000000..0c9342902ab2ee22417e13b61c471bd001fca02f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/dce_extra_outputs.py @@ -0,0 +1,187 @@ +""" +DCE pass for unused extra outputs in HOP subgraphs. + +When enable_side_effects_with_extra_outputs is True, HOPs like invoke_subgraph, +checkpoint (tag_activation_checkpoint), and autograd.Function (autograd_function_apply) +return all intermediate tensors/symints as extra outputs to support side effects. +However, many of these extra outputs may not actually be used in the parent graph. + +Special handling for autograd_function_apply: +- The forward subgraph MUST return (output, saved_values, ...) where indices 0 and 1 + are always required by the runtime +- Only indices 2+ (extra intermediates) can be removed by DCE + +This pass removes unused extra outputs by: +1. Identifying which outputs of HOP calls are actually used +2. Removing unused outputs from the subgraph's output node +3. Updating the HOP call to reflect the new output arity +4. Updating getitem indices to account for removed outputs +""" + +import collections +import operator + +import torch + + +# HOPs that may have extra outputs that can be DCE'd +_HOPS_WITH_EXTRA_OUTPUTS = { + torch.ops.higher_order.invoke_subgraph, + torch.ops.higher_order.tag_activation_checkpoint, + # torch.ops.higher_order.autograd_function_apply, +} + + +def dce_hop_extra_outputs(gm: torch.fx.GraphModule) -> bool: + """ + Remove unused extra outputs from HOP calls recursively. + + Processes graphs top-down: first DCE the current graph's HOP outputs, + then recursively process nested subgraphs. This ensures that when we + process a nested subgraph, the parent has already removed unused getitems, + so the nested subgraph sees the correct usage information. + + Args: + gm: The GraphModule to optimize + + Returns: + True if any modifications were made, False otherwise + """ + modified = False + + # Group HOP nodes by subgraph name + # Multiple invocations may share the same subgraph, so we need to check + # which indices are used across ALL invocations before removing any + subgraph_to_nodes: dict[str, list[torch.fx.Node]] = collections.defaultdict(list) + + for node in gm.graph.nodes: + if node.op == "call_function" and node.target in _HOPS_WITH_EXTRA_OUTPUTS: + subgraph_attr = node.args[0] + if ( + isinstance(subgraph_attr, torch.fx.Node) + and subgraph_attr.op == "get_attr" + ): + subgraph_name = subgraph_attr.target + assert isinstance(subgraph_name, str) + subgraph_to_nodes[subgraph_name].append(node) + + # STEP 1: DCE this graph's HOP outputs first (top-down) + for subgraph_name, hop_nodes in subgraph_to_nodes.items(): + if _dce_subgraph(gm, subgraph_name, hop_nodes): + modified = True + + if modified: + gm.graph.lint() + gm.recompile() + + # STEP 2: Recursively process nested subgraphs + # After we've removed unused getitems from this graph, nested subgraphs + # will see the correct usage information + for subgraph_name in subgraph_to_nodes: + subgraph = getattr(gm, subgraph_name) + if isinstance(subgraph, torch.fx.GraphModule): + if dce_hop_extra_outputs(subgraph): + modified = True + + return modified + + +def _dce_subgraph( + gm: torch.fx.GraphModule, subgraph_name: str, hop_nodes: list[torch.fx.Node] +) -> bool: + """ + DCE a single subgraph by removing unused output indices. + """ + subgraph = getattr(gm, subgraph_name) + + if not isinstance(subgraph, torch.fx.GraphModule): + return False + + # Collect used indices for THIS subgraph + used_indices: set[int] = set() + + # Check if this is the forward subgraph of autograd_function_apply + # For autograd_function_apply, the fwd subgraph must return (output, saved_values, ...) + # where indices 0 and 1 are ALWAYS required by the runtime + # is_autograd_fwd = any( + # node.target == torch.ops.higher_order.autograd_function_apply + # for node in hop_nodes + # ) + is_autograd_fwd = False + + for hop_node in hop_nodes: + for user in list(hop_node.users): + if user.op == "call_function" and user.target == operator.getitem: + if len(list(user.users)) > 0: + idx = user.args[1] + assert isinstance(idx, int) + used_indices.add(idx) + + output_node = next(n for n in subgraph.graph.nodes if n.op == "output") + old_outputs = list(output_node.args[0]) + + # For autograd_function_apply forward subgraph, indices 0 (output) and 1 (saved_values) + # are ALWAYS used by the runtime, even if not explicitly accessed via getitem + if is_autograd_fwd and len(old_outputs) >= 2: + used_indices.add(0) # output + used_indices.add(1) # saved_values + + # Nothing to DCE if all outputs are used or no outputs are used + if len(used_indices) >= len(old_outputs) or len(used_indices) == 0: + return False + + # Build mapping from old indices to new indices + old_to_new: dict[int, int] = {} + new_outputs = [] + new_idx = 0 + + for old_idx in range(len(old_outputs)): + if old_idx in used_indices: + old_to_new[old_idx] = new_idx + new_outputs.append(old_outputs[old_idx]) + new_idx += 1 + + # Update subgraph output node + # Create a new output node with the filtered outputs + with subgraph.graph.inserting_before(output_node): + new_output_node = subgraph.graph.output(tuple(new_outputs)) + output_node.replace_all_uses_with(new_output_node) + subgraph.graph.erase_node(output_node) + + for hop_node in hop_nodes: + # Update getitem nodes to use new indices + for user in list(hop_node.users): + if user.op == "call_function" and user.target == operator.getitem: + old_idx = user.args[1] + assert isinstance(old_idx, int) + if old_idx not in old_to_new: + assert len(list(user.users)) == 0 + gm.graph.erase_node(user) + continue + + new_idx = old_to_new[old_idx] + # Create a new getitem node with the new index + with gm.graph.inserting_before(user): + new_getitem = gm.graph.call_function( + operator.getitem, args=(user.args[0], new_idx) + ) + # Copy metadata from old node + new_getitem.meta = user.meta.copy() + user.replace_all_uses_with(new_getitem) + gm.graph.erase_node(user) + + # Update example_value metadata on hop_node + if "example_value" in hop_node.meta: + old_example = hop_node.meta["example_value"] + assert isinstance(old_example, (tuple, list)) + new_example = tuple( + old_example[old_idx] + for old_idx in range(len(old_outputs)) + if old_idx in used_indices + ) + hop_node.meta["example_value"] = new_example + + subgraph.graph.lint() + subgraph.recompile() + + return True diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/debug_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/debug_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..2acf517aba92f73aacc5b28602e1cb098902ba69 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/debug_utils.py @@ -0,0 +1,937 @@ +""" +Debug utilities for TorchDynamo compilation and execution. + +This module provides various debugging tools and utilities for TorchDynamo, including: + +- Minification support for reducing test cases while preserving bugs +- Input/output handling via InputReader and InputWriter for reproducible testing +- Accuracy checking between original and compiled models +- Neural network module string conversion via NNModuleToString +- Profiling tools and system information collection +- Buck build system integration for Meta-internal testing + +Key classes: +- InputReader/InputWriter: Handle serialization of model inputs/outputs +- NNModuleToString: Converts nn.Modules to string representations +- BuckTargetWriter: Manages Buck build system integration +""" + +from __future__ import annotations + +import atexit +import copy +import cProfile +import functools +import getpass +import inspect +import itertools +import logging +import os +import re +import subprocess +import sys +import tempfile +import textwrap +from collections import Counter +from importlib import import_module +from typing import Any, Optional, TYPE_CHECKING, TypeVar + +import torch +import torch._prims_common as utils +import torch._subclasses.meta_utils +from torch import Tensor +from torch._dynamo.testing import rand_strided +from torch._inductor.cpp_builder import normalize_path_separator +from torch._prims_common import is_float_dtype +from torch.multiprocessing.reductions import StorageWeakRef +from torch.utils._content_store import ContentStoreReader, ContentStoreWriter + +from . import config +from .utils import clone_inputs, get_debug_dir, warn_once + + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + + from torch.hub import tqdm + from torch.storage import UntypedStorage + + +log = logging.getLogger(__name__) + +T = TypeVar("T") + + +inductor_config = import_module("torch._inductor.config") +use_buck = inductor_config.is_fbcode() + +if use_buck: + import libfb.py.build_info + + +extra_deps = [] +extra_imports = "" +cur_target = "" +if use_buck: + extra_deps = [ + "//caffe2/torch/fb/sparsenn:sparsenn_operators_gpu", + "//caffe2/torch/fb/sparsenn:sparsenn_operators", + "//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu", + "//deeplearning/fbgemm/fbgemm_gpu:sparse_ops", + ] + cur_target = libfb.py.build_info.BuildInfo.get_build_rule().replace("fbcode:", "//") # type: ignore[possibly-undefined] + extra_imports = "\n".join([f'torch.ops.load_library("{x}")' for x in extra_deps]) + + +BUCK_CMD_PREFIX = ["buck2", "run", "@mode/dev-nosan"] + + +class BuckTargetWriter: + def __init__(self, filename: str) -> None: + self.subdir, self.py_file = os.path.split(os.path.abspath(filename)) + self.target = self.py_file.replace(".py", "") + + # Get main_module path from fbcode + self.path = f"{self.subdir.replace('/', '.')}.{self.target}" + self.path = self.path[self.path.find("fbcode.") :] + self.path = self.path[7:] + + # Get cmd line path + tmp = self.subdir + tmp = tmp[tmp.find("fbcode/") :][7:] + self.cmd_line_path = f"//{tmp}:{self.target}" + + def build(self) -> str: + extra_cpp_deps = "\n".join([f' "{x}",' for x in extra_deps]) + return textwrap.dedent( + f""" +load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary") + +python_binary( + name="{self.target}", + srcs = ["{self.py_file}"], + compile = False, + deps = [ + "//caffe2:torch", + "//caffe2:libtorch", + "//caffe2/functorch:functorch", + "//triton:triton", + "{cur_target}", + ], + cpp_deps = [ +{extra_cpp_deps} + ], + main_module = "{self.path}", + par_style = "xar", +) +""" + ) + + def write(self, print_msg: bool = True) -> list[str]: + target_file = os.path.join(self.subdir, "TARGETS") + with open(target_file, "w") as fd: + fd.write(self.build()) + # log.warning("Wrote isolation TARGETS file at %s", target_file) + cmd_split = BUCK_CMD_PREFIX + [self.cmd_line_path] + if print_msg: + log.warning( + "Found an example that reproduces the error. Run this cmd to repro - %s", + " ".join(cmd_split), + ) + return cmd_split + + +def minifier_dir() -> str: + path = os.path.join(get_debug_dir(), "minifier") + if path is None: + path = f"{tempfile.gettempdir()}/minifier_{getpass.getuser()}" + if not os.path.exists(path): + os.makedirs(path, exist_ok=True) + return path + + +MAX_CONSTANT_NUMEL_INLINE = 4 + + +class NNModuleToString: + safe_reprs = [ + torch.nn.Linear, + torch.nn.Conv1d, + torch.nn.Conv2d, + torch.nn.Conv3d, + torch.nn.BatchNorm1d, + torch.nn.BatchNorm2d, + torch.nn.BatchNorm3d, + torch.nn.LayerNorm, + torch.nn.Dropout, + torch.nn.Softmax, + torch.nn.ReLU, + torch.nn.GELU, + torch.nn.Identity, + torch.nn.MaxPool2d, + torch.nn.Embedding, + torch.nn.Tanh, + torch.nn.ConvTranspose1d, + torch.nn.GLU, + torch.nn.LSTM, + torch.nn.Flatten, + torch.nn.AdaptiveAvgPool2d, + ] + + @staticmethod + def can_convert_to_string(gm: torch.fx.GraphModule) -> bool: + cant_convert = set() + for _, module in gm.named_children(): + if type(module) not in NNModuleToString.safe_reprs: + cant_convert.add(module) + + if len(cant_convert) > 0: + log.warning("We have not tested reprs of some modules - %s", cant_convert) + # TODO - Assuming that all modules can be safely repr'd. Check if that assumption is correct. + return True + + @staticmethod + def convert(gm: torch.fx.GraphModule) -> str: + from torch.nn.modules.module import _addindent + + tab = " " * 4 + + model_str = textwrap.dedent( + """ + from torch.nn import * + class Repro(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + """ + ) + + for module_name, module in gm.named_children(): + module_str = f"{module.__repr__()}" + # module should be a core torch.nn.Module, so all parameters + # should be on the same device. + example_param = next(module.parameters(), None) + if example_param is not None and example_param.is_cuda: + module_str = f"{module_str}.cuda()" + model_str += f"{tab * 2}self.{module_name} = {module_str}\n" + + for buffer_name, buffer in gm._buffers.items(): + if buffer is None: + continue + # Serialize full data for small buffers + if buffer.numel() <= MAX_CONSTANT_NUMEL_INLINE: + from torch._tensor_str import PRINT_OPTS + + assert PRINT_OPTS.threshold >= MAX_CONSTANT_NUMEL_INLINE + tensor_str = repr(buffer) + elif torch.is_floating_point(buffer): + tensor_str = f"torch.randn({list(buffer.shape)}, dtype={buffer.dtype})" + else: + tensor_str = ( + f"torch.randint(1, size={list(buffer.shape)}, dtype={buffer.dtype})" + ) + if buffer.is_cuda: + tensor_str = f"{tensor_str}.cuda()" + model_str += ( + f"{tab * 2}self.register_buffer('{buffer_name}', {tensor_str})\n" + ) + + for param_name, param in gm._parameters.items(): + if param is None: + continue + maybe_device = "" + if param.is_cuda: + maybe_device = ', device="cuda"' + tensor_str = f"torch.nn.Parameter(torch.randn({list(param.shape)}, dtype={param.dtype}{maybe_device}))" + model_str += f"{tab * 2}self.{param_name} = {tensor_str}\n" + + # TODO - Keep this code for now. But, I don't think we will need this. + # attrs = dir(gm) + # for attr in attrs: + # if "_tensor_constant" in attr: + # val = getattr(gm, attr) + # model_str += f" {attr} = {val!r}\n" + + model_str += f"{_addindent(gm.code, 4)}\n" + return model_str + + +@functools.cache # subprocess is expensive +def _cuda_system_info_comment() -> str: + if not torch.cuda.is_available(): + return "# torch.cuda.is_available()==False, no GPU info collected\n" + + model_str = "# CUDA Info: \n" + try: + cuda_version_out = subprocess.check_output(["nvcc", "--version"]) + cuda_version_lines = cuda_version_out.decode().split("\n") + comment = "".join([f"# {s} \n" for s in cuda_version_lines if s != ""]) + model_str += f"{comment}\n" + except (FileNotFoundError, subprocess.CalledProcessError): + model_str += "# nvcc not found\n" + + gpu_names = Counter( + torch.cuda.get_device_name(i) for i in range(torch.cuda.device_count()) + ) + + model_str += "# GPU Hardware Info: \n" + for name, count in gpu_names.items(): + model_str += f"# {name} : {count} \n" + model_str += "\n" + return model_str + + +def generate_env_vars_string(*, stable_output: bool = False) -> str: + """ + Generate a string configuration for environment variables related to Dynamo, Inductor, and Triton. + """ + if stable_output: + return "# env var omitted due to stable_output=True" + + allow_list = ["TORCH", "DYNAMO", "INDUCTOR", "TRITON"] + skip_list = ["TRITON_LIBDEVICE_PATH", "TRITON_PTXAS_PATH", "TRITON_LIBCUDA_PATH"] + + def filter(key: str) -> bool: + return any(string in key for string in allow_list) and key not in skip_list + + config_lines = [ + f"os.environ['{key}'] = '{value}'" + for key, value in os.environ.items() + if filter(key) + ] + config_string = "\n".join(config_lines) + return normalize_path_separator(f"""\ +import os +{config_string} + """) + + +def generate_config_string(*, stable_output: bool = False) -> str: + import torch._functorch.config + import torch._inductor.config + + if stable_output: + return "# config omitted due to stable_output=True" + + experimental_config = torch.fx.experimental._config.codegen_config() # type: ignore[attr-defined] + return f"""\ +import torch._dynamo.config +import torch._inductor.config +import torch._functorch.config +import torch.fx.experimental._config +{torch._dynamo.config.codegen_config()} +{torch._inductor.config.codegen_config()} +{torch._functorch.config.codegen_config()} +{experimental_config} +""" + + +def get_minifier_repro_path() -> str: + return os.path.join(minifier_dir(), "minifier_launcher.py") + + +def helper_for_dump_minify(contents: str) -> None: + minified_repro_path = get_minifier_repro_path() + log.warning("Writing minified repro to:\n%s", minified_repro_path) + + if use_buck: + BuckTargetWriter(minified_repro_path).write() + try: + with open(minified_repro_path, "w") as fd: + fd.write(contents) + + except OSError as e: + log.exception("") + raise NotImplementedError(f"Could not write to {minified_repro_path}") from e + + +class AccuracyError(Exception): + pass + + +def clone_inputs_retaining_gradness(example_inputs: Sequence[Any]) -> list[Any]: + """ + This clone inputs is different from utils clone_input. In case of minifier, + all the tensors are leaf tensors while creating a new graph. So, we set the + requires_grad field w/o checking the leafness of the tensor. + """ + cloned_inputs = clone_inputs(example_inputs) + for idx in range(len(example_inputs)): + if isinstance(cloned_inputs[idx], torch.Tensor): + cloned_inputs[idx].requires_grad_(example_inputs[idx].requires_grad) + return cloned_inputs # type: ignore[return-value] + + +def run_fwd_maybe_bwd( + gm: torch.fx.GraphModule, + args: Sequence[Any], + only_fwd: bool = False, + disable_clone: bool = False, +) -> Any: + """ + Runs a forward and possibly backward iteration for a given mod and args. + + When disable_clone is True, we will use args as-is without cloning. + This is higher fidelity but we may destroy the args in the process. + """ + from .testing import collect_results, reduce_to_scalar_loss, requires_bwd_pass + + gm = copy.deepcopy(gm) + if not disable_clone: + args = clone_inputs_retaining_gradness(args) + + if hasattr(gm, "zero_grad"): + gm.zero_grad(True) + + # TorchInductor returned callable expects lists. So, may need a boxed calling convention. + out = gm(args) if getattr(gm, "_boxed_call", False) else gm(*args) + + if only_fwd: + return out + if requires_bwd_pass(out): + loss = reduce_to_scalar_loss(out) + loss.backward() + return collect_results(gm, out, None, args) + + +def same_two_models( + gm: torch.fx.GraphModule, + opt_gm: torch.fx.GraphModule, + example_inputs: Sequence[Any], + only_fwd: bool = False, + *, + require_fp64: bool = False, + ignore_non_fp: bool = False, +) -> bool: + """ + Check two models have same accuracy. + + require_fp64: if True, raise an error if we unable to calculate the fp64 reference + ignore_non_fp: if True, do not compare outputs which are not floating point. This + is mostly useful for the minifier (which wants to avoid quantizing floating point + error into integer/boolean error) + """ + from .utils import same + + ref = run_fwd_maybe_bwd(gm, example_inputs, only_fwd) + + fp64_ref = None + if config.same_two_models_use_fp64: + try: + fp64_model, fp64_examples = cast_to_fp64( + copy.deepcopy(gm), clone_inputs_retaining_gradness(example_inputs) + ) + fp64_ref = run_fwd_maybe_bwd(fp64_model, fp64_examples, only_fwd) + except Exception: + if require_fp64: + raise RuntimeError( # noqa: B904 + "Could not generate fp64 outputs, workaround with torch._dynamo.config.same_two_models_use_fp64 = False" + ) + log.warning("Could not generate fp64 outputs") + + try: + res = run_fwd_maybe_bwd(opt_gm, example_inputs, only_fwd) + except Exception: + # This means that the minified graph is bad/exposes a different problem. + # As we are checking accuracy here, lets log the exception and return True. + log.exception( + "While minifying the program in accuracy minification mode, " + "ran into a runtime exception which is likely an unrelated issue." + " Skipping this graph." + ) + return True + + passing = same( + ref, + res, + fp64_ref, + tol=config.repro_tolerance, + equal_nan=True, + ignore_non_fp=ignore_non_fp, + ) + return passing + + +def cast_dtype_args_to_fp64(model: torch.fx.GraphModule) -> torch.fx.GraphModule: + for node in model.graph.nodes: + if ( + node.op == "call_function" + and node.target is torch.ops.prims.convert_element_type.default + ): + assert len(node.args) == 2 + if is_float_dtype(node.args[1]) and node.args[1] != torch.float64: + node.args = (node.args[0], torch.float64) + if node.op == "call_function": + dtype = node.kwargs.get("dtype") + if dtype is not None and is_float_dtype(dtype): + new_kwargs = dict(node.kwargs) + new_kwargs["dtype"] = torch.float64 + node.kwargs = new_kwargs + + model.graph.lint() + model.recompile() + return model + + +def cast_to( + dtype: torch.dtype, model: torch.fx.GraphModule, inputs: list[Any] +) -> tuple[torch.fx.GraphModule, list[Any]]: + from torch.utils._pytree import tree_map + + model = model.to(dtype) + if dtype == torch.float64: + # If casting to fp64 for accuracy comparison, we need to + # replace dtype arguments embedded in the graph with fp64 + model = cast_dtype_args_to_fp64(model) + + inputs = tree_map( + lambda x: x.to(dtype) + if isinstance(x, torch.Tensor) and x.is_floating_point() + else x, + inputs, + ) + return model, inputs + + +def cast_to_fp64( + model: torch.fx.GraphModule, inputs: list[Any] +) -> tuple[torch.fx.GraphModule, list[Any]]: + return cast_to(torch.float64, model, inputs) + + +def backend_accuracy_fails( + gm: torch.fx.GraphModule, + example_inputs: Sequence[Any], + compiler_fn: Callable[[torch.fx.GraphModule, list[Any]], torch.fx.GraphModule], + only_fwd: bool = False, + *, + require_fp64: bool = False, + ignore_non_fp: bool = False, +) -> bool: + try: + compiled_gm = compiler_fn( + copy.deepcopy(gm), clone_inputs_retaining_gradness(example_inputs) + ) + return not same_two_models( + gm, + compiled_gm, + example_inputs, + only_fwd, + require_fp64=require_fp64, + ignore_non_fp=ignore_non_fp, + ) + except Exception: + # This means that the minified graph is bad/exposes a different problem. + # As we are checking accuracy here, lets log the exception and return False. + log.exception( + "While minifying the program in accuracy minification mode, " + "ran into a runtime exception which is likely an unrelated issue." + " Skipping this graph" + ) + return False + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # +# REPRO SUPPORT CODE +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # + + +# Helper functions for computing what the default values of tensor +# values should be. These all coincide with factory functions, e.g., torch.empty + + +def _stride_or_default( + stride: Optional[torch._prims_common.StrideType], + *, + shape: torch._prims_common.ShapeType, +) -> torch._prims_common.StrideType: + return stride if stride is not None else utils.make_contiguous_strides_for(shape) + + +def _mk_defaulter(d: T) -> Callable[[Optional[T]], T]: + return lambda x: x if x is not None else d + + +_dtype_or_default = _mk_defaulter(torch.float32) +_device_or_default = _mk_defaulter(torch.device("cpu")) +_storage_offset_or_default = _mk_defaulter(0) +_requires_grad_or_default = _mk_defaulter(False) +_is_leaf_or_default = _mk_defaulter(False) + + +class NopInputReader: + def __init__(self) -> None: + self.total = 0 + + def storage( + self, + storage_hash: Optional[str], + nbytes: int, + *, + device: Optional[torch._prims_common.DeviceLikeType] = None, + dtype_hint: Optional[torch.dtype] = None, + ) -> None: + self.total += 1 + + def tensor(self, *args: Any, **kwargs: Any) -> Optional[torch.Tensor]: + pass + + def symint(self, *args: Any, **kwargs: Any) -> Optional[int]: + pass + + +# TODO: Support bundling the entire repro into a zip file for ease of +# transferring around +class InputReader: + def __init__(self, save_dir: Optional[str] = None, *, pbar: Optional[tqdm] = None): + # If None, we will generate random data instead. It's important + # to natively support this use case as it will allow people to + # share repros without including the real data, if the problem + # reproduces even on random data. + if save_dir is None: + log.warning("no save_dir specified, will generate random data") + self.store = ContentStoreReader(save_dir) if save_dir is not None else None + self.args: list[Any] = [] + self.pbar = pbar + + def storage( + self, + storage_hash: Optional[str], + nbytes: int, + *, + device: Optional[torch._prims_common.DeviceLikeType] = None, + dtype_hint: Optional[torch.dtype] = None, + ) -> UntypedStorage: + if self.pbar is not None: + self.pbar.update(1) + device = _device_or_default(device) # type: ignore[arg-type] + dtype_hint = _dtype_or_default(dtype_hint) + if self.store is not None and storage_hash is not None: + try: + storage = self.store.read_storage(storage_hash) + except FileNotFoundError: + pass + else: + if device != storage.device: + log.warning("device mismatch: %s != %s", device, storage.device) + # TODO: transfer it to the right device? But failing this + # way would be very mysterious! Would have been better + # not to store device in the serialized format... + return storage + warn_once(f"could not load {storage_hash}, generating random data instead") + shape = (nbytes // dtype_hint.itemsize,) + stride = _stride_or_default(None, shape=shape) + return rand_strided(shape, stride, dtype_hint, device).untyped_storage() + + def tensor( + self, + storage: UntypedStorage, + shape: torch._prims_common.ShapeType, + stride: Optional[torch._prims_common.StrideType] = None, + *, + storage_offset: Optional[int] = None, + dtype: Optional[torch.dtype] = None, + requires_grad: Optional[bool] = None, + is_leaf: Optional[bool] = None, + **metadata: Any, + ) -> torch.Tensor: + stride = _stride_or_default(stride, shape=shape) + storage_offset = _storage_offset_or_default(storage_offset) + dtype = _dtype_or_default(dtype) + is_leaf = _is_leaf_or_default(is_leaf) + requires_grad = _requires_grad_or_default(requires_grad) + t = torch.tensor( + [], dtype=dtype, device=storage.device, requires_grad=requires_grad + ) + with torch.no_grad(): + t.set_(storage, storage_offset, shape, stride) + if not is_leaf: + # Fake up some autograd history in a very naughty way + with torch.enable_grad(): + t = t.clone(memory_format=torch.preserve_format) + with torch.no_grad(): + t.set_(storage, storage_offset, shape, stride) + assert torch._subclasses.meta_utils.safe_is_leaf(t) == is_leaf + torch._utils.set_tensor_metadata(t, metadata) + self.args.append(t) + return t # for BC + + def symint(self, val: Any) -> Any: + self.args.append(val) + return val # for BC + + +# Here is our writer strategy: +# 1. We will stream all of the inputs to disk +# 2. You can now deterministically randomize the inputs, or reload +# the inputs from disk +# 3. You can YOLO run the script without the inputs, in which case +# we'll fill the inputs with random data and pray. This is the +# legacy behavior, but it's also useful if you want to find out +# if we're so broken even random inputs trigger it +# 4. We could offer an in process "check if the randomized thing +# works too" but this is delicate so we don't do it + + +class InputWriter: + def __init__(self, save_dir: Optional[str], *, stable_hash: bool = False) -> None: + self._lines: list[str] = [] + # TODO: consider ensuring tensor and storage counters line up? + self.storage_counter = itertools.count() + self.save_dir = save_dir + self.store = ( + ContentStoreWriter(save_dir, stable_hash=stable_hash) + if save_dir is not None + else None + ) + self.seen_storages: dict[StorageWeakRef, str] = {} + + def lines(self) -> list[str]: + r = [ + "def load_args(reader):", + ] + r.extend(f" {l}" for l in self._lines) + # In case we need to change the internal format of load_args + # in an FC-breaking way + r.append("load_args._version = 0") + return r + + # Storages are untyped, but we need to initialize them with data if + # we don't have the real data, so we give a hint saying what kind + # of initialization may be appropriate + # + # If we had a FakeTensor, device_hint tells us what device should be + def storage( + self, + untyped_storage: UntypedStorage, + *, + device_hint: Optional[torch._prims_common.DeviceLikeType] = None, + dtype_hint: Optional[torch.dtype] = None, + ) -> str: + ws = StorageWeakRef(untyped_storage) + v = self.seen_storages.get(ws) + if v is not None: + return v + v = f"buf{next(self.storage_counter)}" + maybe_dtype_hint = "" + if _dtype_or_default(None) != _dtype_or_default(dtype_hint): + maybe_dtype_hint = f", dtype_hint={dtype_hint!r}" + # TODO: being optional on device is kind of pointless as the default + # is CPU but most repros we care about are CUDA + maybe_device = "" + device = untyped_storage.device + if device.type == "meta": + assert device_hint is not None + device = device_hint # type: ignore[assignment] + if _device_or_default(None) != device: + maybe_device = f", device={device!r}" + nbytes = untyped_storage.nbytes() + storage_hash = None + if self.store is not None and untyped_storage.device.type != "meta": + storage_hash = self.store.write_storage(untyped_storage) + self._lines.append( + f"{v} = reader.storage({storage_hash!r}, {nbytes!r}{maybe_device}{maybe_dtype_hint})" + ) + self.seen_storages[ws] = v + return v + + def tensor(self, name: str, t: torch.Tensor) -> None: + from torch.fx.experimental.symbolic_shapes import statically_known_true, sym_eq + + storage = self.storage( + t.untyped_storage(), dtype_hint=t.dtype, device_hint=t.device + ) + args = [] + # NB: this is positional, must come first + if not statically_known_true( + sym_eq(_stride_or_default(None, shape=t.shape), t.stride()) + ): + args.append(str(tuple(t.stride()))) + if _dtype_or_default(None) != t.dtype: + args.append(f"dtype={t.dtype!r}") + if not statically_known_true( + _storage_offset_or_default(None) == t.storage_offset() + ): + args.append(f"storage_offset={t.storage_offset()!r}") + tensor_metadata = torch._utils.get_tensor_metadata(t) + if tensor_metadata: + args.extend(f"{k}={v!r}" for k, v in tensor_metadata.items()) + if _requires_grad_or_default(None) != t.requires_grad: + args.append(f"requires_grad={t.requires_grad!r}") + is_leaf = torch._subclasses.meta_utils.safe_is_leaf(t) + if _is_leaf_or_default(None) != is_leaf: + args.append(f"is_leaf={is_leaf!r}") + self._lines.append( + "reader.tensor(" + + ", ".join([storage, str(tuple(t.shape)), *args]) + + f") # {name}" + ) + + def unsupported(self, name: str, arg: Any) -> None: + # NB: Try hard not to /print/ a tensor, that will be very slow + self._lines.append(f"# {name} was unsupported type for dumping: {type(arg)}") + # Best effort dump as much useful stuff we can lol, in case you want + # to repair the repro + if isinstance(arg, (list, tuple)): + self._lines.append('"""') + for i, a in enumerate(arg): + name_i = f"{name}[{i}]" + if isinstance(a, torch.Tensor): + self.tensor(name_i, a) + elif isinstance(a, (int, torch.SymInt)): + self.symint(name_i, a) + else: + self.unsupported(name_i, a) + self._lines.append('"""') + + # write out that the arg was filtered out as it is constant + def const(self, name: str) -> None: + self._lines.append( + f"reader.const({name!r}) # {name}, filtered out during compilation" + ) + + # TODO: this doesn't actually symint atm + def symint(self, name: str, val: Any) -> None: + if isinstance(val, torch.SymInt): + val = val.node.hint + self._lines.append(f"reader.symint({val!r}) # {name}") + + +def aot_graph_input_parser( + func: Callable[[list[Tensor]], list[Tensor]], + device: str = "cuda", + sym_shapes: Optional[dict[str, int]] = None, + default_sym_shape: Optional[int] = None, +) -> dict[str, Any]: + """ + Takes in a function which has been printed with print_readable() and constructs kwargs to run it. + + Handles Tensor inputs, Symints, and a graph module which might have tensor constants. + + Consider a function `forward` defined as follows: + + def forward(self, primals_1: "f32[1001, 6]", primals_2: "f32[s0]", primals_3: "Sym(s0)",): + _tensor_constant0: "i64[4190]" = self._tensor_constant0 + # Further implementation + + kwargs = aot_graph_input_parser(forward) + forward(**kwargs) + """ + + from torch.utils._dtype_abbrs import dtype_abbrs + + dtype_map: dict[str, torch.dtype] = { + value: key for key, value in dtype_abbrs.items() + } + dtype_pattern: str = "|".join(dtype_abbrs.values()) + + # Extracting the source code from the function + source = inspect.getsource(func) + + # Regular expressions + tensor_assignment_regex = rf"(_tensor_constant\d+): \"({dtype_pattern})\[\s*(.*?)\s*\]\" = self\.(_tensor_constant\d+)" + tensor_regex = rf"({dtype_pattern})\[\s*(.*?)\s*\]" + sym_shape_regex = r"Sym\((s\d+)\)" + + class TensorContainer: + "Container for tensors as attributes" + + # Dictionary for tensors from annotations + kwargs: dict[str, Any] = {} + + sym_shapes_dict: dict[str, int] = sym_shapes or {} + + def get_sym_int(symint: str) -> int: + torch._check( + symint in sym_shapes_dict or default_sym_shape is not None, + lambda: f"{symint} not in symbolic_shapes and default sym shape not passed in", + ) + return sym_shapes_dict.get(symint, default_sym_shape) # type: ignore[return-value] + + def gen_tensor(shape: torch._prims_common.ShapeType, dtype: torch.dtype) -> Tensor: + # Resolve symbolic shapes to concrete values + resolved_shape = [] + dynamic_dims = [] + for i, dim in enumerate(shape): + dim = dim.strip() # type: ignore[attr-defined] + if "s" in dim: + s = get_sym_int(dim) + resolved_shape.append(s) + dynamic_dims.append(i) + else: + if dim: + resolved_shape.append(int(dim)) + + constructor = torch.randn if dtype.is_floating_point else torch.zeros + out = constructor(resolved_shape, dtype=dtype, device=device) # type: ignore[call-arg] + for d in dynamic_dims: + torch._dynamo.mark_dynamic(out, d) + return out + + # Parse function annotations for tensor generation + annotations = func.__annotations__ + for param, annotation in annotations.items(): + # Skip 'return' annotation + if param == "return": + continue + + match = re.search(tensor_regex, annotation) + if match: + data_type, shape_str = match.groups() + shape = tuple(shape_str.split(",")) + dtype = dtype_map[data_type] + # pyrefly: ignore [bad-argument-type] + kwargs[param] = gen_tensor(shape, dtype) + + match = re.search(sym_shape_regex, annotation) + if match: + kwargs[param] = get_sym_int(match.group(1)) + + if "self" in inspect.signature(func).parameters: + container = TensorContainer() + kwargs["self"] = container + for match in re.finditer(tensor_assignment_regex, source): + attr_name, data_type, shape_str, _ = match.groups() + shape = tuple(shape_str.split(",")) + dtype = dtype_map[data_type] + # pyrefly: ignore [bad-argument-type] + setattr(container, attr_name, gen_tensor(shape, dtype)) + + return kwargs + + +def profile_to_file(filename: str) -> Callable[[T], T]: + """ + Decorator to cProfile a given function and save the result to disk on process exit. + + Args: + filename: filename to save profile to + """ + prof = cProfile.Profile() + filename = os.path.abspath(os.path.expanduser(filename)) + + def decorator(fn: Any) -> Any: + @functools.wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + prof.enable() + try: + return fn(*args, **kwargs) + finally: + prof.disable() + + return wrapper + + def save_it() -> None: + prof.dump_stats(filename) + sys.stderr.write( + textwrap.dedent( + f"""\ + Wrote profile to {filename}, view with: + + snakeviz {filename} + + """ + ) + ) + + atexit.register(save_it) + return decorator diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/decorators.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/decorators.py new file mode 100644 index 0000000000000000000000000000000000000000..3a9718b045cb67d6900fa966504642a45be90eb1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/decorators.py @@ -0,0 +1,1051 @@ +""" +This module provides decorators and utilities for controlling TorchDynamo's behavior during compilation. +""" + +import functools +import inspect +import weakref +from collections.abc import Callable +from dataclasses import dataclass +from types import TracebackType +from typing import Any, Optional, overload, TYPE_CHECKING, TypeVar, Union +from typing_extensions import ParamSpec + +import torch +from torch.compiler import is_compiling +from torch.utils._contextlib import _DecoratorContextManager +from torch.utils._python_dispatch import is_traceable_wrapper_subclass + +from . import trace_rules, variables +from .comptime import comptime +from .eval_frame import ( + _set_stance, + DisableContext, + DynamoStance, + innermost_fn, + RunOnlyContext, + skip_code, +) +from .exc import IncorrectUsage +from .external_utils import ( + get_nonrecursive_disable_wrapper, + wrap_dunder_call_ctx_manager, +) +from .utils import _get_error_on_graph_break, _set_error_on_graph_break, is_function + + +if TYPE_CHECKING: + from types import FunctionType + + from torch._C._dynamo.eval_frame import ( # noqa: F401 + reset_code, + set_eval_frame, + set_guard_complete_hook, + set_guard_error_hook, + unsupported, + ) + + from .variables import VariableTracker +else: + for name in dir(torch._C._dynamo.eval_frame): + if name.startswith("__"): + continue + globals()[name] = getattr(torch._C._dynamo.eval_frame, name) + + +_P = ParamSpec("_P") +_R = TypeVar("_R") +FuncType = Callable[..., Any] +F = TypeVar("F", bound=FuncType) + + +def run(fn: Optional[Callable[_P, _R]] = None) -> Any: + """Don't do any dynamic compiles, just use prior optimizations""" + if fn is not None: + fn = innermost_fn(fn) + assert callable(fn) + return RunOnlyContext()(fn) + return RunOnlyContext() + + +def disable(fn=None, recursive=True, *, reason=None, wrapping=True): # type: ignore[no-untyped-def] + """ + Decorator to disable TorchDynamo + + If recursive=True, Dynamo is completely skipped on the decorated function + frame as well as the recursively invoked functions. + + If recursive=False, Dynamo skips frames associated with the function code, + but still process recursively invoked frames. + + If reason is provided, it will be printed when Dynamo attempts to trace the disabled function. + """ + if recursive: + if fn is not None: + fn = innermost_fn(fn) + assert callable(fn) + return DisableContext(msg=reason, wrapping=wrapping)(fn) + return DisableContext(msg=reason, wrapping=wrapping) + else: + + def wrap(fn: Callable[_P, _R]) -> Callable[_P, _R]: + fn = innermost_fn(fn) + assert callable(fn) + + nonrecursive_disable_wrapper = get_nonrecursive_disable_wrapper(fn) + nonrecursive_disable_wrapper._torchdynamo_disable = True # type: ignore[attr-defined] + nonrecursive_disable_wrapper._torchdynamo_disable_msg = reason # type: ignore[attr-defined] + nonrecursive_disable_wrapper._torchdynamo_orig_callable = fn # type: ignore[attr-defined] + nonrecursive_disable_wrapper._torchdynamo_disable_recursive = False # type: ignore[attr-defined] + # pyrefly: ignore [bad-return] + return nonrecursive_disable_wrapper + + if fn is None: + return wrap + return wrap(fn) + + +_nonrecursive_disable_wrapper_code = disable(lambda: None, recursive=False).__code__ # type: ignore[attr-defined] +skip_code(_nonrecursive_disable_wrapper_code) + + +def skip(fn: Optional[Callable[_P, _R]] = None) -> Callable[..., Any]: + """ + Skip frames associated with the function code, but still process recursively + invoked frames + """ + if fn is None: + return skip + fn = innermost_fn(fn) + assert callable(fn) + skip_code(fn.__code__) + fn._torchdynamo_disable = True # type: ignore[attr-defined] + return fn + + +class set_stance(_DecoratorContextManager): + """ + Decorator, context manager, function to set the current stance of the compiler. + + Stances documented in corresponding function in torch/compiler/__init__.py + """ + + _dynamo_forbidden = True + + def __init__( + self, + stance: str = "default", + *, + skip_guard_eval_unsafe: bool = False, + force_backend: Union[str, Callable[..., Any], None] = None, + ) -> None: + if force_backend is not None and stance != "default": + raise RuntimeError("non-default stance cannot have force_backend set") + + self.stance = DynamoStance(stance, skip_guard_eval_unsafe, force_backend) + self.prev = _set_stance(self.stance) + + def __call__(self, fn: F) -> F: + _set_stance(self.prev) + wrapper = super().__call__(fn) + # forbid wrapper in graph + wrapper._dynamo_forbidden = True # type: ignore[attr-defined] + return wrapper + + def __enter__(self) -> None: + _set_stance(self.stance) + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + _set_stance(self.prev) + + def clone(self) -> "set_stance": + return self.__class__(self.stance.stance, force_backend=self.stance.backend) + + +def assume_constant_result(fn): # type: ignore[no-untyped-def] + fn._dynamo_marked_constant = True # type: ignore[attr-defined] + return fn + + +def allow_in_graph(fn): # type: ignore[no-untyped-def] + """ + Tells the compiler frontend (Dynamo) to skip symbolic introspection of the function + and instead directly write it to the graph when encountered. + + See :func:`torch.compiler.allow_in_graph`'s docstring for the full documentation + + WARNING: this API can be a footgun, please read the documentation carefully. + """ + if isinstance(fn, (list, tuple)): + return [allow_in_graph(x) for x in fn] + assert callable(fn), "allow_in_graph expects a callable" + if trace_rules.lookup_callable(fn) != variables.TorchInGraphFunctionVariable: + fn_id = id(fn) + trace_rules._disallowed_callable_ids.remove(fn_id) + trace_rules._allowed_callable_ids.add(fn_id) + + # Avoid id reuse which creates subtle bugs. + def deregister() -> None: + trace_rules._allowed_callable_ids.remove(fn_id) + + weakref.finalize(fn, deregister) + return fn + + +def nonstrict_trace(traceable_fn: Callable[_P, _R]) -> Callable[_P, _R]: + # Like `allow_in_graph`, but with the following enhancements/differences: + # + # 1. Supports user-defined class as inputs, as long as the class has been + # registered with pytree. + # 2. Reads to global/captured tensors forces the underlying graph to treat + # those tensors as constant, and we _assume_ they will not be updated. This + # is similar to FX tracing. + # 3. In the resulting Dynamo graph, the call to a `nonstrict_trace`-ed function + # will be represented as a call to `torch._higher_order_ops.flat_apply`, + # which takes in the `nonstrict_trace`-ed function and pytree-flattened + # inputs. + # 4. Only the returned function is traceable, and the original function will + # not be. Moreover, `nonstrict_trace` can be used inside a `torch.compile` + # region. + # + # NOTE: like `allow_in_graph`, aliasing information is neither preserved + # between inputs themselves, nor between inputs and outputs. + assert callable(traceable_fn), "nonstrict_trace expects a callable" + + @functools.wraps(traceable_fn) + def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _R: + return traceable_fn(*args, **kwargs) + + wrapped_id = id(wrapped) + + # This line allows us to reuse much of the `allow_in_graph` impl. + trace_rules._allowed_callable_ids.add(wrapped_id) + + # This line allows us to diverge the impl from `allow_in_graph`. + trace_rules._nonstrict_trace_callable_ids.add(wrapped_id) + + # Avoid id reuse which creates subtle bugs. + def deregister() -> None: + trace_rules._allowed_callable_ids.remove(wrapped_id) + trace_rules._nonstrict_trace_callable_ids.remove(wrapped_id) + + weakref.finalize(wrapped, deregister) + + return wrapped + + +def _disallow_in_graph_helper(throw_if_not_allowed: bool) -> Callable[..., Any]: + def inner(fn: Any) -> Any: + if isinstance(fn, (list, tuple)): + return [disallow_in_graph(x) for x in fn] + assert callable(fn), "disallow_in_graph expects a callable" + if ( + throw_if_not_allowed + and trace_rules.lookup_callable(fn) + != variables.TorchInGraphFunctionVariable + and trace_rules.lookup(fn) != variables.TorchInGraphFunctionVariable + ): + raise IncorrectUsage( + "disallow_in_graph is expected to be used on an already allowed callable (like torch.* ops). " + "Allowed callables means callables that TorchDynamo puts as-is in the extracted graph." + ) + trace_rules._allowed_callable_ids.remove(id(fn)) + trace_rules._nonstrict_trace_callable_ids.remove(id(fn)) + trace_rules._disallowed_callable_ids.add(id(fn)) + return fn + + return inner + + +def disallow_in_graph(fn: Callable[..., Any]) -> Any: + """ + Customize which functions TorchDynamo will exclude in the generated + graph and force a graph break on. + :: + + torch._dynamo.disallow_in_graph(torch.sub) + + + @torch._dynamo.optimize(...) + def fn(a): + x = torch.add(x, 1) + x = torch.sub(x, 1) + x = torch.add(x, 1) + return x + + + fn(...) + + Will break the graph on `torch.sub`, and give two graphs each with a + single `torch.add()` op. + """ + return _disallow_in_graph_helper(throw_if_not_allowed=True)(fn) + + +@_disallow_in_graph_helper(throw_if_not_allowed=False) +def graph_break(msg: str = "") -> None: + """Force a graph break""" + + +# NOTE: primarily used for internal debugging purposes! +@_disallow_in_graph_helper(throw_if_not_allowed=False) +def skip_frame(msg: str = "") -> None: + """Force a skipped frame""" + + +@_disallow_in_graph_helper(throw_if_not_allowed=False) +def step_unsupported(msg: str = "") -> None: + """Force a step unsupported graph break, which results in compiling + the traced FX graph so far, then skipping the rest of the frame. + In order to get expected behavior, there should be at least 2 ops + and a part of the code not contained in any try/with blocks.""" + + +def forbid_in_graph(fn: Any) -> Any: + """ + Customize which functions TorchDynamo will assert are not present while tracing. + + If you want a graph break on this function instead, use disallow_in_graph. + TODO(voz): We now have allow_in_graph, disallow_in_graph, forbid_in_graph - some more robust + documentation would not be amiss. + """ + if isinstance(fn, (list, tuple)): + return [forbid_in_graph(x) for x in fn] + assert callable(fn), "forbid_in_graph applies only to callables" + # pyrefly: ignore [missing-attribute] + fn._dynamo_forbidden = True + return fn + + +def substitute_in_graph( + original_fn: Callable[_P, _R], + *, + can_constant_fold_through: bool = False, + skip_signature_check: bool = False, + # type that is embedded in the Python interpreter + is_embedded_type: bool = False, # internal use only +) -> Callable[[Callable[_P, _R]], Callable[_P, _R]]: + """ + Register a polyfill handler for a function, usually a C function from the C extension, to be + used in place of the original function when inlining the original function in the graph. + + .. note:: + + The polyfill handler is only used when inlining the original function. It is not used when + the original function is called directly. In the eager mode, the decorated function calls + the performant C function rather than the polyfill handler. + + The polyfill handler is a function that will be called in place of the original function when + inlining the original function. The polyfill handler should have the same signature and the same + behavior as the original function. + + Args: + original_fn (callable): The original function, usually a C function, to register a polyfill + handler for. + can_constant_fold_through (bool, optional): Whether the polyfill handler can be constant + folded through. That is, if the polyfill handler is a pure function and its arguments + are constant, the result of the polyfill handler can be constant folded during the + compilation. Defaults to ``False``. + skip_signature_check (bool, optional): Whether to skip the signature check between the + original function and the polyfill handler. Defaults to ``False``. + + Returns: + A decorator that registers the polyfill handler for the original function. + + Example:: + + >>> # xdoctest: +SKIP("conflict with the tests: duplicate polyfill handlers") + >>> import operator + >>> operator.indexOf([1, 2, 3, 4, 5], 3) + 2 + >>> torch.compile(operator.indexOf, fullgraph=True)([1, 2, 3, 4, 5], 3) + Traceback (most recent call last): + ... + torch._dynamo.exc.Unsupported: ... + + >>> @torch.compiler.substitute_in_graph(operator.indexOf) + ... def indexOf(a, b, /): + ... for i, item in enumerate(a): + ... if item is b or item == b: + ... return i + ... raise ValueError("sequence.index(x): x not in sequence") + >>> + >>> torch.compile(operator.indexOf, fullgraph=True)([1, 2, 3, 4, 5], 3) + 2 + """ + if not is_function(original_fn) and not ( + is_embedded_type and inspect.isclass(original_fn) + ): + raise TypeError( + f"substitute_in_graph expects a function but got {type(original_fn)!r}" + ) + if is_embedded_type: + if not inspect.isclass(original_fn): + raise TypeError( + f"substitute_in_graph expects a class but got {type(original_fn)!r}" + ) + + from .variables.builder import ITERTOOLS_POLYFILLED_TYPE_IDS, ITERTOOLS_TYPE_IDS + + if id(original_fn) in ITERTOOLS_TYPE_IDS: + ITERTOOLS_POLYFILLED_TYPE_IDS.add(id(original_fn)) + + def wrapper(traceable_fn: Callable[_P, _R]) -> Callable[_P, _R]: + if not is_function(traceable_fn): + raise TypeError( + f"@substitute_in_graph(...) expects a function but got {type(traceable_fn)!r}" + ) + + if not skip_signature_check: + try: + original_sig = inspect.signature(original_fn) + except ValueError: + pass + else: + traceable_sig = inspect.signature(traceable_fn) + + def sig_ident( + sig: inspect.Signature, + ) -> tuple[tuple[str, ...], set[str], dict[str, Any]]: + # Ignore annotations for parameters and return type + return ( + tuple( + p.name + for p in sig.parameters.values() + if ( + p.kind + not in { + p.KEYWORD_ONLY, + # the name of *args and **kwargs is not important + p.VAR_POSITIONAL, + p.VAR_KEYWORD, + } + ) + ), + { + p.name + for p in sig.parameters.values() + if p.kind == p.KEYWORD_ONLY + }, + { + p.name: p.default + for p in sig.parameters.values() + # the name of *args and **kwargs is not important + if p.kind not in {p.VAR_POSITIONAL, p.VAR_KEYWORD} + }, + ) + + wildcard_sig = inspect.signature(lambda *args, **kwargs: None) + + if ( + sig_ident(original_sig) != sig_ident(traceable_sig) + and sig_ident(original_sig) != sig_ident(wildcard_sig) + and sig_ident(traceable_sig) != sig_ident(wildcard_sig) + ): + raise TypeError( + f"Signature mismatch between {original_fn} and {traceable_fn}: " + f"{original_sig} != {traceable_sig}" + ) + + from torch._dynamo.guards import GuardBuilder + from torch._dynamo.trace_rules import ( + _polyfilled_function_ids, + get_torch_obj_rule_map, + ) + from torch._dynamo.variables import PolyfilledFunctionVariable + from torch._dynamo.variables.builder import VariableBuilder + + id_dispatch_map = VariableBuilder._id_dispatch() + if id(original_fn) in id_dispatch_map: + raise ValueError( + f"Duplicate dispatch rule for {original_fn}: " + "already registered in VariableBuilder's id dispatch map" + ) + + if id(original_fn) in _polyfilled_function_ids: + raise ValueError(f"Duplicate polyfilled object {original_fn}") + + rule_map: dict[Any, type[VariableTracker]] = get_torch_obj_rule_map() + if original_fn in rule_map: + raise ValueError( + f"Duplicate object {original_fn} with different rules: " + f"{PolyfilledFunctionVariable}, {rule_map[original_fn]}" + ) + + polyfill_handlers: dict[Callable[..., Any], FunctionType] + polyfill_handlers = PolyfilledFunctionVariable._get_polyfill_handlers() + if original_fn in polyfill_handlers: + raise ValueError( + f"Duplicate polyfill handlers for {original_fn}: " + f"already handled by {polyfill_handlers[original_fn]}" + ) + + # Need to wrap the function because we may cannot assign __torch_dynamo_polyfill__ to a + # C++ function. + @functools.wraps(traceable_fn) + def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _R: + return original_fn(*args, **kwargs) + + def dispatch_fn( + self: VariableBuilder, value: Callable[_P, _R] + ) -> PolyfilledFunctionVariable: + if inspect.isclass(value): + guard_type = GuardBuilder.CLASS_MATCH + elif inspect.ismodule(value): + guard_type = GuardBuilder.MODULE_MATCH + else: + guard_type = GuardBuilder.ID_MATCH + return PolyfilledFunctionVariable( + value, + source=self.source, + **self.install_guards(guard_type), + ) + + id_dispatch_map[id(original_fn)] = id_dispatch_map[id(wrapped)] = dispatch_fn + _polyfilled_function_ids.add(id(original_fn)) + _polyfilled_function_ids.add(id(wrapped)) + rule_map[original_fn] = rule_map[wrapped] = PolyfilledFunctionVariable + polyfill_handlers[original_fn] = polyfill_handlers[wrapped] = wrapped # type: ignore[assignment] + + wrapped.__torch_dynamo_original__ = original_fn # type: ignore[attr-defined] + wrapped.__torch_dynamo_polyfill__ = traceable_fn # type: ignore[attr-defined] + wrapped.__torch_dynamo_can_constant_fold_through__ = can_constant_fold_through # type: ignore[attr-defined] + + return wrapped # type: ignore[return-value] + + return wrapper + + +# Helper function to flatten a tensor subclass and apply a function to +# all inner tensors that match the outer dim. Used to reduce duplication +# across the various marking APIs. +def _apply_func_to_inner_tensors_of_same_dim( + func: Callable[..., Any], t: object, *args: Any, **kwargs: Any +) -> None: + assert is_traceable_wrapper_subclass(t) + + attrs, _ctx = t.__tensor_flatten__() + assert isinstance(t, torch.Tensor) + for attr in attrs: + inner = getattr(t, attr) + if inner.dim() == t.dim(): + func(inner, *args, **kwargs) + + +@dataclass(frozen=True) +class _DimRange: + """ + This represents an dimension of a tensor and the corresponding + min and max values it can take. Don't create this + class directly; instead, use :func:`mark_dynamic`. + """ + + dim: int + min: int + max: int + + +@forbid_in_graph +def mark_unbacked( + t: Any, + index: Union[int, list[Any], tuple[Any]], + hint_override: Optional[int] = None, + strict: bool = False, + specialize_on: Optional[list[Any]] = None, +) -> None: + """ + Mark a tensor as having an unbacked dimension. This changes the semantics of operations: + - The size of the specified dimension will always be reported as not equal to zero or one. + - Assertions on this index will be turned into runtime asserts. + - Attempting to get the real value of this dimension will raise an exception. + - In effect, this dimension is treated as data-dependent (its value is unknown). + + Args: + t (Any): The tensor to mark as having an unbacked dimension. + index (int or list/tuple of int): The dimension(s) to mark as unbacked. Can be a single integer or a list/tuple of integers. + hint_override (Optional[int], default=None): An optional integer to override the size hint for this dimension. + This is only used by the inductor backend for size hint queries, such as during autotuning. + strict (bool, default=False): If True, an error will be raised if the unbacked dimension is specialized. + By default (strict=False), specialization is allowed and will proceed without error. + specialize_on (Optional[list[Any]], default=None): A list of specialization criteria (e.g., lambdas) for this dimension. + If provided, Dynamo will generate specialized compiled regions for each criterion in addition to a generic trace. + """ + if torch.distributed.is_available() and isinstance( + t, torch.distributed.tensor.DTensor + ): + # apply on inner tensor sizes/strides + mark_unbacked(t._local_tensor, index) + else: + # You could have copied the mark_dynamic behavior but I'm not convinced + # it's what you want + assert not is_traceable_wrapper_subclass(t), "not implemented yet" + + if isinstance(index, int): + if strict: + if not hasattr(t, "_dynamo_strict_unbacked_indices"): + # pyrefly: ignore [missing-attribute] + t._dynamo_strict_unbacked_indices = set() + # pyrefly: ignore [missing-attribute] + t._dynamo_strict_unbacked_indices.add(index) + return + + if not hasattr(t, "_specialized_on"): + # pyrefly: ignore [missing-attribute] + t._specialize_on = {} + + if not hasattr(t, "_dynamo_unbacked_indices"): + # pyrefly: ignore [missing-attribute] + t._dynamo_unbacked_indices = set() + + if not hasattr(t, "_dynamo_hint_overrides"): + # pyrefly: ignore [missing-attribute] + t._dynamo_hint_overrides = {} + + if hint_override: + # pyrefly: ignore [missing-attribute] + t._dynamo_hint_overrides[index] = hint_override + + # FX tracers don't respect @forbid_in_graph and choke on the following error since it passes in proxies: + # TypeError: 'Attribute' object does not support item assignment + # pyrefly: ignore [missing-attribute] + if isinstance(t._specialize_on, dict): + # pyrefly: ignore [missing-attribute] + t._specialize_on[index] = specialize_on if specialize_on is not None else [] + + # pyrefly: ignore [missing-attribute] + t._dynamo_unbacked_indices.add(index) + return + + assert isinstance(index, (list, tuple)) + for i in index: + mark_unbacked(t, i) + + +@forbid_in_graph +def mark_dynamic( + t: Any, + index: Union[int, list[Any], tuple[Any]], + *, + hint_override: Optional[int] = None, + min: Optional[int] = None, + max: Optional[int] = None, + specialize_on: Optional[list[Any]] = None, +) -> None: + """ + Mark a tensor as having a dynamic dim and set corresponding min and max range for the dim. + + [Note - on the state of mark_dynamic] + + The behavior of having a dynamic dimension on a tensor is governed by a few factors: + + 1) torch._dynamo.config dynamic_shapes True or False. + a) dynamic_shapes=True - dynamic_shapes must be True for mark_dynamic to work. + a) dynamic_shapes=False - This config will raise an exception when used in conjunction with + mark_dynamic. We will eventually support this. + + 2) If the dimension is fully constrained - as in, it does not allow more than a single value + in both eager (torch.compile, torch._dynamo.optimize) mode and export mode (torch._dynamo.export), + we will raise an error + + 3) If the dimension is partially constrained - allowing at least 2 values but not the full unbounded + range of shapes, in eager we will pass it through, but export will raise an error. + + 4) Attempts to trace this function will explicitly raise. As such, all calls to mark_dynamic must be made + before torch.compile. + + 5) If hint_override is passed, the hint_override for the specified dimension will replace the provided value + from the first example input as the official size hint. + + 6) If specialize_on is passed in, we will perform a single generic Dynamo trace followed by + multiple specialized compilations in addition to a single generic compilation. NB: For now we only support + per dimension specialization, or in other words we do not generate a cross product of specializations. + At runtime, we will dispatch to a specialized compiled region if the input matches the specialization criteria. + + For example: + mark_dynamic(..., specialize_on=[ + lambda x: x == 8, + lambda x: x == 16 + ]) + + This approach results in one Dynamo trace and two backend compilations. When the input dimension equals 8 or 16 + at runtime, execution will be directed to the specialized compiled region. Performance measurements indicate + 2-8x speedups depending on the specific specialization and model architecture. + + """ + if is_traceable_wrapper_subclass(t): + # default behavior: mirror mark_dynamic() on all inner tensors with same dim as t + # TODO: Make this configurable via a supported public API + _apply_func_to_inner_tensors_of_same_dim( + mark_dynamic, t, index, min=min, max=max + ) + + if isinstance(index, int): + if not hasattr(t, "_dynamo_dynamic_indices"): + # pyrefly: ignore [missing-attribute] + t._dynamo_dynamic_indices = set() + # pyrefly: ignore [missing-attribute] + t._dynamo_dynamic_range = set() + # pyrefly: ignore [missing-attribute] + t._dynamo_hint_overrides = {} + + if not hasattr(t, "_specialize_on"): + # pyrefly: ignore [missing-attribute] + t._specialize_on = {} + + if hint_override: + # pyrefly: ignore [missing-attribute] + t._dynamo_hint_overrides[index] = hint_override + # TODO(voz): Should we bounds check? + # pyrefly: ignore [missing-attribute] + t._dynamo_dynamic_indices.add(index) + t._dynamo_dynamic_range.add(_DimRange(index, min, max)) # type: ignore[arg-type] + + # FX tracers don't respect @forbid_in_graph and choke on the following error since it passes in proxies: + # TypeError: 'Attribute' object does not support item assignment + # pyrefly: ignore [missing-attribute] + if isinstance(t._specialize_on, dict): + t._specialize_on[index] = specialize_on if specialize_on is not None else [] + + return + + assert isinstance(index, (list, tuple)) + for i in index: + mark_dynamic(t, i, min=min, max=max) + mark_dynamic(t, i, min=min, max=max, specialize_on=specialize_on) + + +@forbid_in_graph +def maybe_mark_dynamic(t: Any, index: Union[int, list[Any], tuple[Any]]) -> None: + """ + Mark a tensor as having a dynamic dim, but don't enforce it (i.e., if this + dimension ends up getting specialized, don't error). + """ + if is_traceable_wrapper_subclass(t): + # default behavior: mirror maybe_mark_dynamic() on all inner tensors with same dim as t + # TODO: Make this configurable via a supported public API + _apply_func_to_inner_tensors_of_same_dim(maybe_mark_dynamic, t, index) + + if isinstance(index, int): + if not hasattr(t, "_dynamo_weak_dynamic_indices"): + # pyrefly: ignore [missing-attribute] + t._dynamo_weak_dynamic_indices = set() + # TODO(voz): Should we bounds check? + # pyrefly: ignore [missing-attribute] + t._dynamo_weak_dynamic_indices.add(index) + return + + assert isinstance(index, (list, tuple)) + for i in index: + maybe_mark_dynamic(t, i) + + +def mark_static( + t: Any, index: Optional[Union[int, list[Any], tuple[Any]]] = None +) -> None: + """ + Mark a tensor as having a static dim or mark a nn module class as static. + + For tensors + =========== + This will prevent us from attempting to compile it dynamically + when dynamic=True; this can improve trace-time performance. + + This has lower precedence than mark_dynamic. + + Unlike mark_dynamic, this can be done inside a graph, in which case it + induces specialization on the tensor. + + For nn.Module classes + ===================== + For static nn.Module classes, TorchDynamo assumes that the module instance + attributes will not be modified after compilation. This will ensure that + TorchDynamo keeps integer attributes CONSTANT and not symints. + + From TorchDynamo implementation side, the instances of static-marked + nn.Module class will be converted to UnspecializedBuiltinNNModuleVariable, + which have the same properties. + + Note that we still have to guard on the attributes, because different + instances of the nn.Module can have different values of the attributes. The + key point here is that the attributes are static. + """ + if is_compiling(): + if index is None: + for s in t.size(): + comptime.force_static(s) + else: + comptime.force_static(t.size(index)) + return + + if is_traceable_wrapper_subclass(t): + # default behavior: mirror mark_static() on all inner tensors with same dim as t + # TODO: Make this configurable via a supported public API + _apply_func_to_inner_tensors_of_same_dim(mark_static, t, index) + + # pyrefly: ignore [bad-argument-type] + if not isinstance(t, torch.Tensor) and issubclass(t, torch.nn.Module): + # pyrefly: ignore [missing-attribute] + t._dynamo_marked_static = True + # pyrefly: ignore [bad-return] + return t + + if not isinstance(t, torch.Tensor): + raise TypeError( + f"mark_static expects a tensor/nn.Module class but received {type(t)}" + ) + + if isinstance(index, int): + if not hasattr(t, "_dynamo_static_indices"): + t._dynamo_static_indices = set() # type: ignore[attr-defined] + # TODO(voz): Should we bounds check? + t._dynamo_static_indices.add(index) # type: ignore[attr-defined] + elif index is None: + for i in range(t.dim()): + mark_static(t, i) + else: + assert isinstance(index, (list, tuple)) + for i in index: + mark_static(t, i) + + +@forbid_in_graph +def mark_static_address(t: Any, guard: bool = False) -> None: + """ + Marks an input tensor whose address should be treated as constant across calls to the + same dynamo-compiled function. This indicates to cudagraphs that an extra allocation + is not needed for this input. The data_ptr will be guarded if guard=True, and cause a full + recompile if the data_ptr changes. Note: If this address changes, cudagraphs will re-record + if guard=False. + """ + if not isinstance(t, torch.Tensor): + raise TypeError(f"mark_static_address expects a tensor but received {type(t)}") + + if guard: + t._dynamo_static_input_type = "guarded" # type: ignore[attr-defined] + else: + t._dynamo_static_input_type = "unguarded" # type: ignore[attr-defined] + + +# One day, Dynamo will support tracing into einops directly (no allow_in_graph needed) +# Note that PyTorch supports multiple versions of einops, so when that day comes, +# we still need to be really careful about version matches. +def _allow_in_graph_einops() -> None: + import einops + + try: + # requires einops > 0.6.1, torch >= 2.0 + from einops._torch_specific import ( # type: ignore[attr-defined] # noqa: F401 + _ops_were_registered_in_torchdynamo, + ) + + # einops > 0.6.1 will call the op registration logic as it is imported. + except ImportError: + # einops <= 0.6.1 + allow_in_graph(einops.rearrange) + allow_in_graph(einops.reduce) + if hasattr(einops, "repeat"): + allow_in_graph(einops.repeat) # available since einops 0.2.0 + if hasattr(einops, "einsum"): + allow_in_graph(einops.einsum) # available since einops 0.5.0 + if hasattr(einops, "pack"): + allow_in_graph(einops.pack) # available since einops 0.6.0 + if hasattr(einops, "unpack"): + allow_in_graph(einops.unpack) # available since einops 0.6.0 + + +# Note: this carefully avoids eagerly import einops. +trace_rules.add_module_init_func("einops", _allow_in_graph_einops) + + +# Proxy class for torch._dynamo.config patching - so dynamo can identify context managers/decorators +# created by patch_dynamo_config, compared to ones created by a raw torch._dynamo.config.patch. +class DynamoConfigPatchProxy: + def __init__(self, config_patch: Any) -> None: + self.config_patch = config_patch + + @property + def changes(self) -> dict[str, Any]: + return self.config_patch.changes + + # Decorator implementation that simply sets up `self` as a context manager. + # Placed in external_utils so that we can trace through it. + __call__ = wrap_dunder_call_ctx_manager + + def __enter__(self) -> None: + return self.config_patch.__enter__() + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + return self.config_patch.__exit__(exc_type, exc_val, exc_tb) + + +# Criteria for patchable config: +# - Config values must be constants (i.e. int, float, str, bool, None). +# - in particular, NO list, set, dict. +# - Traceable config patches are only useful for configs that change dynamo behavior +# from symbolic_convert and below. +# - e.g. patching recompile_limit won't really do anything. +# - For patching configs that affect Dynamo behavior above symbolic_convert, +# ensure that Dynamo behaves soundly even if tracing is done with different config. +# - e.g. be careful if patching guard-related configs as configs may have changed +# between guard creation and evaluation. +_allowed_config_patches = ( + "verbose", + "verify_correctness", + "rewrite_assert_with_torch_assert", + "capture_scalar_outputs", + "allow_unspec_int_on_nn_module", + "skip_torchrec", + "dont_skip_tracing", + "nested_graph_breaks", +) + +from . import config + + +for name in _allowed_config_patches: + assert hasattr(config, name), "nonexistent config" +del config + + +def _patch_dynamo_config_check(changes: dict[str, Any]) -> None: + for k, v in changes.items(): + if k not in _allowed_config_patches: + raise ValueError( + f"patch_dynamo_config does not support patching config {k}" + ) + if not torch._dynamo.utils.is_safe_constant(v): + raise ValueError( + f"patch_dynamo_config does not support patching config {k} " + f"with non-safe-constant value {v}" + ) + + +# TODO: also implement nonrecursive patch_dynamo_config/dont_skip_tracing. +# Unlike config.patch, we also need to accept tuple as input in order to +# deal with context manager reconstruction. +def patch_dynamo_config( + arg1: Optional[Union[str, dict[str, Any], tuple[tuple[str, Any], ...]]] = None, + arg2: Any = None, + **kwargs: Any, +) -> DynamoConfigPatchProxy: + """ + A wrapper around torch._dynamo.config.patch that can be traced by Dynamo to + temporarily change config values DURING tracing. + + See _allowed_config_patches for the list of allowed config patches. + + Arguments are the same as with torch._dynamo.config.patch. + + Can be used as a decorator or a context manager. + + User code SHOULD NOT MODIFY the return value of this function. + + WARNING: changing Dynamo config during tracing can lead to unpredictable tracing behavior! + Proceed only as advised! + """ + if isinstance(arg1, tuple): + arg1 = dict(arg1) + config_patch = torch._dynamo.config.patch(arg1, arg2, **kwargs) + _patch_dynamo_config_check(config_patch.changes) + # check for valid patching using config_patch.changes + return DynamoConfigPatchProxy(config_patch) + + +@overload +def dont_skip_tracing(fn: None = None) -> DynamoConfigPatchProxy: ... + + +@overload +def dont_skip_tracing(fn: Callable[_P, _R]) -> Callable[_P, _R]: ... + + +def dont_skip_tracing(fn: Optional[Any] = None) -> Any: + """ + Context manager/decorator to trace into functions intentionally marked by developers to be skipped + when tracing. + + This decorator will also apply to recursively invoked functions. + """ + ctx = patch_dynamo_config(dont_skip_tracing=True) + if fn: + return ctx(fn) + return ctx + + +@overload +def disable_nested_graph_breaks(fn: None = None) -> DynamoConfigPatchProxy: ... + + +@overload +def disable_nested_graph_breaks(fn: Callable[_P, _R]) -> Callable[_P, _R]: ... + + +def disable_nested_graph_breaks(fn: Optional[Any] = None) -> Any: + """ + Context manager/decorator to disable nested graph breaks when tracing + this function and any nested functions. Used when nested graph breaks + is causing problems. + """ + ctx = patch_dynamo_config(nested_graph_breaks=False) + if fn: + return ctx(fn) + return ctx + + +class ErrorOnGraphBreakDecoratorContextManager: + def __init__(self, error_on_graph_break: bool) -> None: + self.error_on_graph_break = error_on_graph_break + + __call__ = wrap_dunder_call_ctx_manager + + def __enter__(self) -> None: + self.prev_error_on_graph_break = _get_error_on_graph_break() + _set_error_on_graph_break(self.error_on_graph_break) + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + _set_error_on_graph_break(self.prev_error_on_graph_break) + + +def error_on_graph_break( + error_on_graph_break: bool, +) -> ErrorOnGraphBreakDecoratorContextManager: + """ + Context manager/decorator to toggle torch.compile's `error_on_graph_break` setting at compile time. + + If `fullgraph` is set, then `error_on_graph_break` does nothing + (i.e. `fullgraph = True` takes higher precedence). If `fullgraph` is False, then + `error_on_graph_break` determines whether `torch.compile` throws an error upon + encountering a graph break, or attempts to continue tracing. + + `error_on_graph_break` can be toggled during compile time with this decorator to allow graph breaks in some + compiled regions but not others. One key difference from `fullgraph` is that `error_on_graph_break = True` + does NOT guarantee that a single graph is captured from the compiled function. + + The default value of torch.compile's `error_on_graph_break` setting is False. + """ + return ErrorOnGraphBreakDecoratorContextManager(error_on_graph_break) + + +def is_dynamo_disable_recursive(method: Callable[[Any], Any]) -> Optional[bool]: + """ + Check if a method is marked as `dynamo_disable` recursively. It returns: + - True if disable(recursive=True) + - False if disable(recursive=False) + - None if method is not a disable decorator + """ + return getattr(method, "_torchdynamo_disable_recursive", None) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/device_interface.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/device_interface.py new file mode 100644 index 0000000000000000000000000000000000000000..00ba284f9b44cef909fb88e5a3341efee7371dc4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/device_interface.py @@ -0,0 +1,612 @@ +""" +Device abstraction layer for TorchDynamo and Inductor backends. + +This module provides a unified interface for different hardware backends (CUDA, XPU, +CPU, MPS, MTIA) through a common device interface. Key components include: + +- DeviceInterface: Base class defining the common API for all device types +- Device-specific implementations: CudaInterface, XpuInterface, CpuInterface, MpsInterface, MtiaInterface +- Device registration system for managing available backends +- Worker APIs for multi-processing scenarios +- Stream and event management across different devices +- Device property caching for worker processes + +The abstraction layer enables device-agnostic code in TorchDynamo while allowing +specialized implementations for each hardware backend's unique features. +""" + +import inspect +import time +from collections import namedtuple +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from typing import Any, Literal, Optional, Union + +import torch + + +get_cuda_stream: Optional[Callable[[int], int]] +if torch.cuda._is_compiled(): + from torch._C import _cuda_getCurrentRawStream as get_cuda_stream +else: + get_cuda_stream = None + +# Recording the device properties in the main process but used in worker process. +caching_worker_device_properties: dict[str, Any] = {} +caching_worker_current_devices: dict[str, int] = {} + + +class DeviceInterface: + """ + This is a simple device runtime interface for Inductor. It enables custom + backends to be integrated with Inductor in a device-agnostic semantic. + """ + + class device: + def __new__(cls, device: torch.types.Device) -> Any: + raise NotImplementedError + + class Event: + def __new__(cls, *args: Any, **kwargs: Any) -> Any: + raise NotImplementedError( + "Event should be inherited from torch.Event, otherwise, it couldn't be captured by dynamo." + ) + + class Stream: + def __new__(cls, *args: Any, **kwargs: Any) -> Any: + raise NotImplementedError( + "Stream should be inherited from torch.Stream, otherwise, it couldn't be captured by dynamo." + ) + + class Worker: + """ + Worker API to query device properties that will work in multi processing + workers that cannot use the GPU APIs (due to processing fork() and + initialization time issues). Properties are recorded in the main process + before we fork the workers. + """ + + @staticmethod + def set_device(device: int) -> None: + raise NotImplementedError + + @staticmethod + def current_device() -> int: + raise NotImplementedError + + @staticmethod + def get_device_properties(device: torch.types.Device = None) -> Any: + raise NotImplementedError + + @staticmethod + def current_device() -> int: + raise NotImplementedError + + @staticmethod + def set_device(device: torch.types.Device) -> None: + raise NotImplementedError + + @staticmethod + def maybe_exchange_device(device: int) -> int: + raise NotImplementedError + + @staticmethod + def exchange_device(device: int) -> int: + raise NotImplementedError + + @staticmethod + def device_count() -> int: + raise NotImplementedError + + @staticmethod + def is_available() -> bool: + raise NotImplementedError + + @staticmethod + def stream(stream: torch.Stream) -> Any: + raise NotImplementedError + + @staticmethod + def current_stream() -> torch.Stream: + raise NotImplementedError + + @staticmethod + def set_stream(stream: torch.Stream) -> None: + raise NotImplementedError + + @staticmethod + def _set_stream_by_id(stream_id: int, device_index: int, device_type: int) -> None: + raise NotImplementedError + + @staticmethod + def get_raw_stream(device_idx: int) -> int: + raise NotImplementedError + + @staticmethod + def synchronize(device: torch.types.Device = None) -> None: + raise NotImplementedError + + @classmethod + def get_device_properties(cls, device: torch.types.Device = None) -> Any: + return cls.Worker.get_device_properties(device) + + @staticmethod + def get_compute_capability(device: torch.types.Device = None) -> Any: + raise NotImplementedError + + @staticmethod + def is_bf16_supported(including_emulation: bool = False) -> bool: + raise NotImplementedError + + @classmethod + def is_dtype_supported( + cls, dtype: torch.dtype, including_emulation: bool = False + ) -> bool: + return dtype != torch.bfloat16 or cls.is_bf16_supported(including_emulation) + + @staticmethod + def memory_allocated(device: torch.types.Device = None) -> int: + raise NotImplementedError + + @staticmethod + def is_triton_capable(device: torch.types.Device = None) -> bool: + """ + Returns True if the device has Triton support, False otherwise, even if + the appropriate Triton backend is not available. + """ + return False + + @classmethod + def raise_if_triton_unavailable(cls, device: torch.types.Device = None) -> None: + """ + Raises a `RuntimeError` with the appropriate human-readable instructions + to resolve the issue if Triton is not available for the given device, or + the default device if `device` is `None`. + + The caller should ensure the presence of the 'triton' package before + calling this method. + """ + if not cls.is_triton_capable(): + raise RuntimeError("This device is not capable of supporting Triton") + + +class DeviceGuard: + """ + This class provides a context manager for device switching. This is a stripped + down version of torch.{device_name}.device. + + The context manager changes the current device to the given device index + on entering the context and restores the original device on exiting. + The device is switched using the provided device interface. + """ + + def __init__( + self, device_interface: type[DeviceInterface], index: Optional[int] + ) -> None: + self.device_interface = device_interface + self.idx = index + self.prev_idx = -1 + + def __enter__(self) -> None: + if self.idx is not None: + self.prev_idx = self.device_interface.exchange_device(self.idx) + + def __exit__(self, type: Any, value: Any, traceback: Any) -> Literal[False]: + if self.idx is not None: + self.idx = self.device_interface.maybe_exchange_device(self.prev_idx) + return False + + +class CudaInterface(DeviceInterface): + device = torch.cuda.device # type: ignore[assignment] + + # register Event and Stream class into the backend interface + # make sure Event and Stream are implemented and inherited from the torch.Event and torch.Stream + Event = torch.cuda.Event # type: ignore[assignment] + Stream = torch.cuda.Stream # type: ignore[assignment] + + # pyrefly: ignore [bad-override] + class Worker: + @staticmethod + def set_device(device: int) -> None: + caching_worker_current_devices["cuda"] = device + + @staticmethod + def current_device() -> int: + if "cuda" in caching_worker_current_devices: + return caching_worker_current_devices["cuda"] + return torch.cuda.current_device() + + @staticmethod + def get_device_properties(device: torch.types.Device = None) -> Any: + if device is not None: + if isinstance(device, str): + device = torch.device(device) + assert device.type == "cuda" + if isinstance(device, torch.device): + device = device.index + if device is None: + device = CudaInterface.Worker.current_device() + + if "cuda" not in caching_worker_device_properties: + device_prop = [ + torch.cuda.get_device_properties(i) + for i in range(torch.cuda.device_count()) + ] + caching_worker_device_properties["cuda"] = device_prop + + return caching_worker_device_properties["cuda"][device] + + current_device = staticmethod(torch.cuda.current_device) + set_device = staticmethod(torch.cuda.set_device) + device_count = staticmethod(torch.cuda.device_count) + stream = staticmethod(torch.cuda.stream) # type: ignore[assignment] + # pyrefly: ignore [bad-override] + current_stream = staticmethod(torch.cuda.current_stream) + set_stream = staticmethod(torch.cuda.set_stream) # type: ignore[assignment] + _set_stream_by_id = staticmethod(torch.cuda._set_stream_by_id) # type: ignore[assignment] + synchronize = staticmethod(torch.cuda.synchronize) + get_device_properties = staticmethod(torch.cuda.get_device_properties) # type: ignore[assignment] + get_raw_stream = staticmethod(get_cuda_stream) # type: ignore[assignment, arg-type] + exchange_device = staticmethod(torch.cuda._exchange_device) # type: ignore[arg-type, has-type] + maybe_exchange_device = staticmethod(torch.cuda._maybe_exchange_device) # type: ignore[arg-type, has-type] + memory_allocated = staticmethod(torch.cuda.memory_allocated) + is_bf16_supported = staticmethod(torch.cuda.is_bf16_supported) # type: ignore[arg-type] + + # Can be mock patched by @patch decorator. + @staticmethod + def is_available() -> bool: + return torch.cuda.is_available() + + @staticmethod + def get_compute_capability(device: torch.types.Device = None) -> Union[int, str]: + if torch.version.hip is None: + major, min = torch.cuda.get_device_capability(device) + return major * 10 + min + else: + return torch.cuda.get_device_properties(device).gcnArchName.split(":", 1)[0] + + @staticmethod + def is_triton_capable(device: torch.types.Device = None) -> bool: + return ( + torch.version.hip is not None + or torch.cuda.get_device_properties(device).major >= 7 + ) + + @staticmethod + def raise_if_triton_unavailable(device: torch.types.Device = None) -> None: + from torch._inductor.exc import GPUTooOldForTriton + + if not CudaInterface.is_triton_capable(device): + device_props = torch.cuda.get_device_properties(device) + raise GPUTooOldForTriton(device_props, inspect.currentframe()) + + import triton.backends + + if torch.version.hip is not None: + if "amd" not in triton.backends.backends: + raise RuntimeError("triton not built with the 'amd' backend") + elif "nvidia" not in triton.backends.backends: + raise RuntimeError("triton not built with the 'nvidia' backend") + + +get_mtia_stream: Optional[Callable[[int], int]] +if torch.mtia._is_compiled(): + from torch._C import _mtia_getCurrentRawStream as get_mtia_stream +else: + get_mtia_stream = None + + +class MtiaInterface(DeviceInterface): + device = torch.mtia.device # type: ignore[assignment] + Event = torch.mtia.Event # type: ignore[assignment] + Stream = torch.mtia.Stream # type: ignore[assignment] + + # pyrefly: ignore [bad-override] + class Worker: + @staticmethod + def set_device(device: int) -> None: + caching_worker_current_devices["mtia"] = device + + @staticmethod + def current_device() -> int: + if "mtia" in caching_worker_current_devices: + return caching_worker_current_devices["mtia"] + return torch.mtia.current_device() + + @staticmethod + def get_device_properties(device: torch.types.Device = None) -> Any: + if device is not None: + if isinstance(device, str): + device = torch.device(device) + assert device.type == "mtia" + if isinstance(device, torch.device): + device = device.index + if device is None: + device = MtiaInterface.Worker.current_device() + + if "mtia" not in caching_worker_device_properties: + device_prop = [ + torch.mtia.get_device_properties(i) + for i in range(torch.mtia.device_count()) + ] + caching_worker_device_properties["mtia"] = device_prop + + return caching_worker_device_properties["mtia"][device] + + current_device = staticmethod(torch.mtia.current_device) + set_device = staticmethod(torch.mtia.set_device) # type: ignore[assignment] + device_count = staticmethod(torch.mtia.device_count) + stream = staticmethod(torch.mtia.stream) # type: ignore[assignment] + # pyrefly: ignore [bad-override] + current_stream = staticmethod(torch.mtia.current_stream) + set_stream = staticmethod(torch.mtia.set_stream) # type: ignore[assignment] + _set_stream_by_id = staticmethod(torch.mtia._set_stream_by_id) # type: ignore[assignment] + synchronize = staticmethod(torch.mtia.synchronize) + get_device_properties = staticmethod(torch.mtia.get_device_properties) # type: ignore[assignment] + get_raw_stream = staticmethod(get_mtia_stream) # type: ignore[assignment, arg-type] + exchange_device = staticmethod(torch.mtia._exchange_device) # type: ignore[arg-type, has-type] + maybe_exchange_device = staticmethod(torch.mtia._maybe_exchange_device) # type: ignore[arg-type, has-type] + memory_allocated = staticmethod(torch.mtia.memory_allocated) # type: ignore[assignment] + is_bf16_supported = staticmethod(torch.mtia.is_bf16_supported) # type: ignore[arg-type] + + # Can be mock patched by @patch decorator. + @staticmethod + def is_available() -> bool: + ret = torch.mtia.is_available() + return ret + + @staticmethod + def get_compute_capability(device: torch.types.Device = None) -> Any: + cc = torch.mtia.get_device_capability(device) + return cc + + @staticmethod + def is_triton_capable(device: torch.types.Device = None) -> bool: + return True + + @staticmethod + def raise_if_triton_unavailable(evice: torch.types.Device = None) -> None: + import triton.backends + + if "mtia" not in triton.backends.backends: + raise RuntimeError("triton not built with the 'mtia' backend") + + +get_xpu_stream: Optional[Callable[[int], int]] +if torch.xpu._is_compiled(): + from torch._C import _xpu_getCurrentRawStream as get_xpu_stream +else: + get_xpu_stream = None + + +class XpuInterface(DeviceInterface): + device = torch.xpu.device # type: ignore[assignment] + Event = torch.xpu.Event # type: ignore[assignment] + Stream = torch.xpu.Stream # type: ignore[assignment] + + # pyrefly: ignore [bad-override] + class Worker: + @staticmethod + def set_device(device: int) -> None: + caching_worker_current_devices["xpu"] = device + + @staticmethod + def current_device() -> int: + if "xpu" in caching_worker_current_devices: + return caching_worker_current_devices["xpu"] + return torch.xpu.current_device() + + @staticmethod + def get_device_properties(device: torch.types.Device = None) -> Any: + if device is not None: + if isinstance(device, str): + device = torch.device(device) + assert device.type == "xpu" + if isinstance(device, torch.device): + device = device.index + if device is None: + device = XpuInterface.Worker.current_device() + + if "xpu" not in caching_worker_device_properties: + device_prop = [ + torch.xpu.get_device_properties(i) + for i in range(torch.xpu.device_count()) + ] + caching_worker_device_properties["xpu"] = device_prop + + return caching_worker_device_properties["xpu"][device] + + current_device = staticmethod(torch.xpu.current_device) + set_device = staticmethod(torch.xpu.set_device) + device_count = staticmethod(torch.xpu.device_count) # type: ignore[has-type] + stream = staticmethod(torch.xpu.stream) # type: ignore[assignment] + # pyrefly: ignore [bad-override] + current_stream = staticmethod(torch.xpu.current_stream) + set_stream = staticmethod(torch.xpu.set_stream) # type: ignore[assignment] + _set_stream_by_id = staticmethod(torch.xpu._set_stream_by_id) # type: ignore[assignment] + synchronize = staticmethod(torch.xpu.synchronize) + get_device_properties = staticmethod(torch.xpu.get_device_properties) # type: ignore[assignment] + get_raw_stream = staticmethod(get_xpu_stream) # type: ignore[assignment, arg-type] + exchange_device = staticmethod(torch.xpu._exchange_device) # type: ignore[arg-type, has-type] + maybe_exchange_device = staticmethod(torch.xpu._maybe_exchange_device) # type: ignore[arg-type, has-type] + memory_allocated = staticmethod(torch.xpu.memory_allocated) + + # Can be mock patched by @patch decorator. + @staticmethod + def is_available() -> bool: + return torch.xpu.is_available() + + @staticmethod + def get_compute_capability(device: torch.types.Device = None) -> Any: + cc = torch.xpu.get_device_capability(device) + return cc + + @staticmethod + def is_bf16_supported(including_emulation: bool = False) -> bool: + return torch.xpu.is_bf16_supported() + + @staticmethod + def is_triton_capable(device: torch.types.Device = None) -> bool: + return True + + @staticmethod + def raise_if_triton_unavailable(device: torch.types.Device = None) -> None: + import triton.backends + + if "intel" not in triton.backends.backends: + raise RuntimeError("triton not built with the 'intel' backend") + + +@dataclass +class CpuDeviceProperties: + multi_processor_count: int + + +class CpuInterface(DeviceInterface): + # pyrefly: ignore [bad-override] + class Event(torch.Event): + def __init__(self, enable_timing: bool = True) -> None: + self.time = 0.0 + + def elapsed_time(self, end_event: Any) -> float: + return (end_event.time - self.time) * 1000 + + def record(self, stream: Any = None) -> None: + self.time = time.perf_counter() + + # pyrefly: ignore [bad-override] + class Worker: + @staticmethod + def get_device_properties( + device: torch.types.Device = None, + ) -> CpuDeviceProperties: + import multiprocessing + + cpu_count = multiprocessing.cpu_count() + return CpuDeviceProperties(cpu_count) + + @staticmethod + def is_available() -> bool: + return True + + @staticmethod + def is_bf16_supported(including_emulation: bool = False) -> bool: + return True + + @staticmethod + def get_compute_capability(device: torch.types.Device = None) -> str: + return "" + + @staticmethod + def get_raw_stream(device_idx: Any) -> int: + return 0 + + @staticmethod + def current_device() -> int: + return 0 + + @staticmethod + def synchronize(device: torch.types.Device = None) -> None: + pass + + @staticmethod + def is_triton_capable(device: torch.types.Device = None) -> bool: + return True + + @staticmethod + def raise_if_triton_unavailable(device: torch.types.Device = None) -> None: + import triton.backends + + if "cpu" not in triton.backends.backends: + raise RuntimeError("triton not built with the 'cpu' backend") + + +class MpsInterface(DeviceInterface): + @staticmethod + def is_bf16_supported(including_emulation: bool = False) -> bool: + return torch.backends.mps.is_macos_or_newer(14, 0) + + @classmethod + def is_dtype_supported( + cls, dtype: torch.dtype, including_emulation: bool = False + ) -> bool: + if dtype in [torch.float64, torch.complex128]: + return False + return dtype != torch.bfloat16 or cls.is_bf16_supported(including_emulation) + + @staticmethod + def is_available() -> bool: + return torch.backends.mps.is_available() + + @staticmethod + def current_device() -> int: + return 0 + + @staticmethod + def get_compute_capability(device: torch.types.Device = None) -> str: + return "" + + @staticmethod + def synchronize(device: torch.types.Device = None) -> None: + torch.mps.synchronize() + + # pyrefly: ignore [bad-override] + class Worker: + @staticmethod + def get_device_properties(device: torch.types.Device = None) -> Any: + return namedtuple("MPSProperties", ["multi_processor_count"])( + torch.backends.mps.get_core_count() # type: ignore[arg-type] + ) + + @staticmethod + def current_device() -> int: + return 0 + + +device_interfaces: dict[str, type[DeviceInterface]] = {} +_device_initialized = False + + +def register_interface_for_device( + device: Union[str, torch.device], device_interface: type[DeviceInterface] +) -> None: + if isinstance(device, torch.device): + device = device.type + device_interfaces[device] = device_interface + + +def get_interface_for_device(device: Union[str, torch.device]) -> type[DeviceInterface]: + if isinstance(device, torch.device): + device = device.type + if not _device_initialized: + init_device_reg() + if device in device_interfaces: + return device_interfaces[device] + raise NotImplementedError(f"No interface for device {device}") + + +def get_registered_device_interfaces() -> Iterable[tuple[str, type[DeviceInterface]]]: + if not _device_initialized: + init_device_reg() + return device_interfaces.items() + + +def init_device_reg() -> None: + global _device_initialized + register_interface_for_device("cuda", CudaInterface) + for i in range(torch.cuda.device_count()): + register_interface_for_device(f"cuda:{i}", CudaInterface) + + register_interface_for_device("xpu", XpuInterface) + for i in range(torch.xpu.device_count()): + register_interface_for_device(f"xpu:{i}", XpuInterface) + + register_interface_for_device("mtia", MtiaInterface) + for i in range(torch.mtia.device_count()): + register_interface_for_device(f"mtia:{i}", MtiaInterface) + + register_interface_for_device("cpu", CpuInterface) + register_interface_for_device("mps", MpsInterface) + + _device_initialized = True diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/distributed.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/distributed.py new file mode 100644 index 0000000000000000000000000000000000000000..490b6330fafa45c871771610849707d26216cccf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/distributed.py @@ -0,0 +1,54 @@ +""" +Manages process groups for distributed compilation in TorchDynamo. + +This module handles the initialization and management of process groups used for +distributed compilation. Key features: + +- Lazy initialization of compilation process groups +- Only creates groups when distributed mode is enabled and available +- Integrates with compiler_collectives configuration setting +- Provides a single global process group for compilation coordination + +The process group is created only when needed and if the distributed environment +is properly initialized, making it safe to import and use this module even in +non-distributed scenarios. +""" + +from typing import Optional + +import torch.distributed as dist + +from . import config + + +_COMPILE_PG: Optional[dist.ProcessGroup] = None +_GUARD_PG: Optional[dist.ProcessGroup] = None + + +def get_compile_pg() -> Optional[dist.ProcessGroup]: + if ( + config.enable_compiler_collectives + and dist.is_available() + and dist.is_initialized() + ): + global _COMPILE_PG + if _COMPILE_PG is None: + # , timeout=datetime.timedelta(seconds=2) + _COMPILE_PG = dist.distributed_c10d._new_group_with_tag( + pg_tag="pt2_compile_pg" + ) + return _COMPILE_PG + + return None + + +# NB: Unlike get_compile_pg, this is only called when guard collectives were +# explicitly requested +def get_guard_pg() -> Optional[dist.ProcessGroup]: + if dist.is_available() and dist.is_initialized(): + global _GUARD_PG + if _GUARD_PG is None: + _GUARD_PG = dist.distributed_c10d._new_group_with_tag(pg_tag="pt2_guard_pg") + return _GUARD_PG + + return None diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py new file mode 100644 index 0000000000000000000000000000000000000000..4a46fb366b0b8fb631ee73057fe3024c105e08ae --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py @@ -0,0 +1,2504 @@ +# mypy: disable-error-code="method-assign" + +""" +This module implements the core frame evaluation handler for TorchDynamo's compilation system. +The eval frame handler intercepts Python bytecode execution at runtime to enable dynamic +compilation and optimization of PyTorch code. + +Key components defined here: +- Frame evaluation handlers that intercept and analyze Python execution frames +- Guards management for tracking dependencies and invalidating compiled code +- Optimization contexts and decorators (optimize, run_once, disable, etc.) +- Export functionality for saving optimized graphs +- Backend compiler integrations and callback management + +Functions in this file are responsible for modifying the eval frame handler at RUNTIME. +Therefore, all functions in this file are hot and performance-critical. Functions that +only execute at compile time should be placed in torch._dynamo.convert_frame. + +The eval frame handler is the core mechanism that enables TorchDynamo to dynamically +intercept, analyze and optimize PyTorch code during execution. It works by registering +a custom frame evaluation function that gets called for every Python frame, allowing +us to detect PyTorch operations and trigger compilation as needed. +""" + +from __future__ import annotations + +import atexit +import contextlib +import functools +import inspect +import logging +import os +import sys +import sysconfig +import textwrap +import threading +import traceback +import types +import unittest +import warnings +import weakref +from collections.abc import Sized +from dataclasses import dataclass +from enum import Enum +from os.path import dirname, join +from typing import Any, NamedTuple, Optional, TYPE_CHECKING, Union +from unittest.mock import patch + +import sympy + +import torch +import torch.fx +import torch.utils._pytree as pytree +import torch.utils.checkpoint +from torch import _guards + +# see discussion at https://github.com/pytorch/pytorch/issues/120699 +from torch._C._dynamo.eval_frame import ( # noqa: F401 + reset_code, + set_code_exec_strategy, + set_eval_frame, + set_guard_complete_hook, + set_guard_error_hook, + set_skip_guard_eval_unsafe, + unsupported, +) +from torch._dispatch.python import enable_python_dispatcher +from torch._dynamo.types import ConvertFrameReturn, FrameAction, FrameExecStrategy +from torch._export.utils import _compiling_state_context +from torch._subclasses.fake_tensor import unset_fake_temporarily +from torch._utils_internal import DISABLE_JUSTKNOBS, justknobs_check, log_export_usage +from torch.export.dynamic_shapes import ( + _combine_args, + _DimHint, + _DimHintType, + _IntWrapper, + _process_dynamic_shapes, + _RelaxedConstraint, + Constraint, +) +from torch.fx import GraphModule, traceback as fx_traceback +from torch.fx.experimental._dynamism import ( + clone_and_convert_to_meta, + track_dynamism_across_examples, +) +from torch.fx.experimental.proxy_tensor import make_fx +from torch.fx.experimental.symbolic_shapes import ( + ConstraintViolationError, + DimDynamic, + ShapeEnv, + StatelessSymbolicContext, +) +from torch.fx.graph import _PyTreeCodeGen, _PyTreeInfo + +from . import config, convert_frame, distributed, external_utils, trace_rules, utils +from .backends.registry import CompilerFn, lookup_backend +from .code_context import code_context +from .exc import ( + CondOpArgsMismatchError, + ShortenTraceback, + Unsupported, + UserError, + UserErrorType, +) +from .hooks import Hooks +from .mutation_guard import install_generation_tagging_init +from .utils import ( + _get_error_on_graph_break, + _set_error_on_graph_break, + common_constant_types, + compile_times, +) + + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Sequence + + from torch._dynamo.package import CompilePackage + from torch._dynamo.repro.after_dynamo import WrapBackendDebug + from torch._subclasses import fake_tensor + from torch.fx.node import Argument, Node, Target + + from .types import ( + CacheEntry, + DynamoCallback, + DynamoFrameType, + GuardFail, + GuardFilterEntry, + ) + + +log = logging.getLogger(__name__) + + +always_optimize_code_objects = utils.ExactWeakKeyDictionary() +null_context = contextlib.nullcontext + + +# See https://github.com/python/typing/pull/240 +class Unset(Enum): + token = 0 + + +cached_backends: dict[int, CompilerFn] = {} + +unset = Unset.token + +_in_optimized_module = False + + +if DISABLE_JUSTKNOBS: + _maybe_set_eval_frame = set_eval_frame +else: + + def _maybe_set_eval_frame(callback: DynamoCallback) -> DynamoCallback: + # A wrapper on set_eval_frame that is guarded by a Justknob. + # Users can disable torchDynamo by setting the JK to False. + if not justknobs_check("pytorch/compiler:enable_compiler_set_eval_frame"): + torch._dynamo.utils.warn_once( + "Dynamo disabled by Justknob: enable_compiler_set_eval_frame, skipping set_eval_frame" + ) + return callback + else: + return set_eval_frame(callback) + + +@dataclass +class DynamoStance: + stance: str = "default" + skip_guard_eval_unsafe: bool = False + backend: Union[str, Callable[..., Any], None] = None + + +_stance = DynamoStance() + + +def _set_stance(stance: DynamoStance) -> DynamoStance: + global _stance + + from torch._C._dynamo.eval_frame import get_eval_frame_callback + + callback = get_eval_frame_callback() + + if callback is not False and callback is not None: + raise RuntimeError("attempted to set_stance in a torch.compile region") + + prior = _stance + _stance = stance + return prior + + +_set_stance._dynamo_forbidden = True # type: ignore[attr-defined] + +_EXAMPLE_INPUTS: Optional[dict[str, list[Any]]] = None + + +def get_example_inputs(key: str) -> list[Any]: + global _EXAMPLE_INPUTS + if _EXAMPLE_INPUTS is None: + _EXAMPLE_INPUTS = {} + + if key not in _EXAMPLE_INPUTS: + _EXAMPLE_INPUTS[key] = [] + + return _EXAMPLE_INPUTS[key] + + +@contextlib.contextmanager +def _set_in_optimized_module(): + # Set in dynamo's OptimizedModule forward, to have better coverage than is_compiling(). + # Prevents graph-breaking forward hooks from being registered & traced. + # TODO(pianpwk): subsume this flag with better is_compiling() coverage + global _in_optimized_module + _old_in_optimized_module = ( + _in_optimized_module # do we need this? can we just set it to False after + ) + _in_optimized_module = True + try: + yield + finally: + _in_optimized_module = _old_in_optimized_module + + +def _is_in_optimized_module() -> bool: + global _in_optimized_module + return _in_optimized_module + + +def _callback_from_stance(callback: DynamoCallback) -> DynamoCallback: + if _stance.stance == "default": + # force_backend + if _stance.backend is not None and callback not in (False, None): + callback = _create_wrapped_callback(get_compiler_fn(_stance.backend)) + + return callback + elif _stance.stance == "eager_then_compile": + if callback not in (False, None): + return _create_delayed_compile_callback(callback, _stance.stance) + return callback + elif _stance.stance == "aot_eager_then_compile": + if callback not in (False, None): + return _create_delayed_compile_callback(callback, _stance.stance) + return callback + elif _stance.stance == "force_eager": + # disable + return None + elif _stance.stance == "eager_on_recompile": + # run mode + return False + elif _stance.stance == "fail_on_recompile": + if callback in (False, None): + return callback + + def fail_callback( + frame: DynamoFrameType, *args: Any, **kwargs: Any + ) -> ConvertFrameReturn: + if trace_rules.check(frame.f_code): + return ConvertFrameReturn() + if not convert_frame.has_tensor_in_frame(frame): + return ConvertFrameReturn() + + from torch._C._dynamo.eval_frame import ( + _debug_get_cache_entry_list, + _debug_get_precompile_entries, + ) + from torch._dynamo.guards import get_and_maybe_log_recompilation_reasons + + message = ( + "Detected recompile when torch.compile stance is 'fail_on_recompile'. " + + f"filename: '{frame.f_code.co_filename}', " + + f"function name: '{frame.f_code.co_name}', " + + f"line number: {frame.f_lineno}" + ) + cache_entries = _debug_get_cache_entry_list(frame.f_code) + if cache_entries: + reasons = get_and_maybe_log_recompilation_reasons( + cache_entries[0], frame, innermost_fn(callback), skip_logging=True + ) + if reasons: + failures = textwrap.indent("\n".join(reasons), "- ") + guard_failure_details = ( + f"triggered by the following guard failure(s):\n{failures}" + ) + message += f"\n{textwrap.indent(guard_failure_details, ' ')}" + precompile_entries = _debug_get_precompile_entries(frame.f_code) + if len(precompile_entries) > 0: + message += "\nFailed on the following precompiled guards: " + for entry in precompile_entries: + message += f"\n{entry.guard_manager}{entry.guard_manager.check_verbose(frame.f_locals)}" # type: ignore[attr-defined] + raise RuntimeError(message) + + # to prevent cache miss due to different backend + fail_callback._torchdynamo_orig_backend = callback # type: ignore[attr-defined] + + return fail_callback + else: + raise RuntimeError(f"invalid torch.compile stance '{_stance}'") + + +def _create_wrapped_callback( + compiler_fn: CompilerFn, +) -> convert_frame.CatchErrorsWrapper: + hooks = Hooks() + return convert_frame.catch_errors_wrapper( + convert_frame.convert_frame( # type: ignore[arg-type] + compiler_fn, + hooks, + ), + hooks, + ) + + +def _get_or_add_example_inputs(frame: DynamoFrameType) -> list[Any]: + key = frame.f_code.co_filename + str(frame.f_code.co_firstlineno) + example_inputs = get_example_inputs(key) + + if len(example_inputs) < 2: + example_inputs.append(clone_and_convert_to_meta(frame.f_locals)) + + return example_inputs + + +def _create_delayed_compile_callback( + callback: DynamoCallback, stance: str +) -> Callable[..., Any]: + def callback_fn(*args: Any, **kwargs: Any) -> convert_frame.ConvertFrameReturn: + frame = args[0] + example_inputs = _get_or_add_example_inputs(frame) + + if len(example_inputs) == 1: + if stance == "eager_then_compile": + return ConvertFrameReturn( + frame_exec_strategy=FrameExecStrategy( + FrameAction.DEFAULT, FrameAction.DEFAULT + ) + ) + elif stance == "aot_eager_then_compile": + aot_eager_fn = get_compiler_fn("aot_eager") + return _create_wrapped_callback(aot_eager_fn)(*args, **kwargs) + + dynamism = track_dynamism_across_examples(example_inputs) + code_context.get_context(frame.f_code)["dynamism"] = dynamism + compiler_fn = callback._torchdynamo_orig_backend._torchdynamo_orig_backend # type: ignore[union-attr] + return _create_wrapped_callback(compiler_fn)(*args, **kwargs) + + # to prevent cache miss due to different backend + callback_fn._torchdynamo_orig_backend = callback # type: ignore[attr-defined] + + return callback_fn + + +def _is_skip_guard_eval_unsafe_stance() -> bool: + return _stance.skip_guard_eval_unsafe + + +def _reset_guarded_backend_cache() -> None: + global cached_backends + for backend in cached_backends.values(): + if hasattr(backend, "reset"): + backend.reset() + cached_backends.clear() + + +DONT_WRAP_FILES = { + # For tracing into fx modules + inspect.getsourcefile(GraphModule), + join(dirname(dirname(__file__)), "onnx/_internal/fx/dynamo_graph_extractor.py"), +} + + +def _debug_get_cache_entry_list( + code: Union[types.CodeType, Callable[..., Any]], +) -> list[CacheEntry]: + """ + Given a code object or a callable object, retrieve the cache entries + stored in this code. + """ + if callable(code): + code = code.__code__ + return torch._C._dynamo.eval_frame._debug_get_cache_entry_list(code) + + +class OptimizedModule(torch.nn.Module): + """ + Wraps the original nn.Module object and later patches its + forward method to optimized self.forward method. + """ + + _torchdynamo_orig_callable: Callable[..., Any] + get_compiler_config: Callable[[], Any] + + _opt_mod_attributes = { + "_orig_mod", + "dynamo_ctx", + "_torchdynamo_orig_callable", + "get_compiler_config", + "forward", + "_forward", + "__dict__", + "named_children_walk", + "_super_module_initialized", + } + + def __init__(self, mod: torch.nn.Module, dynamo_ctx: _TorchDynamoContext) -> None: + # NOTE: this must go first, because attribute reads/writes of `self` + # uses `_orig_mod`, and sometimes users override `Module.__init__` to + # do attribute reads/writes on `self`. + # + # We also can't use regular setattr because `super().__setattr__` will + # complain for module value before `super().__init__()` + object.__setattr__(self, "_orig_mod", mod) + self._super_module_initialized = False + super().__init__() + self._super_module_initialized = True + + # Installs the params/buffer + self._orig_mod = mod # `super().__setattr__` will register this module + self.dynamo_ctx = dynamo_ctx + self._initialize() + self.training = self._orig_mod.training + + def __len__(self) -> int: + # Proxy the len call to the original module + if isinstance(self._orig_mod, Sized): + return len(self._orig_mod) + # Mimic python's default behavior for objects without a length + raise TypeError(f"{type(self._orig_mod).__name__} does not support len()") + + def _initialize(self) -> None: + # Do this stuff in constructor to lower overhead slightly + if isinstance(self.dynamo_ctx, DisableContext): + # No need to check trace rules + self.forward = self.dynamo_ctx(self._orig_mod.__call__) + elif config.wrap_top_frame or ( + isinstance(self._orig_mod.forward, types.MethodType) + and ( + trace_rules.check(self._orig_mod.forward) + or getattr(self._orig_mod, "_is_fsdp_managed_module", False) + ) + ): + # This may be a torch.nn.* instance in trace_rules.py which + # won't trigger a frame evaluation workaround to add an extra + # frame we can capture + self.forward = self.dynamo_ctx(external_utils.wrap_inline(self._orig_mod)) + else: + # Invoke hooks outside of dynamo then pickup the inner frame + self.forward = self.dynamo_ctx(self._orig_mod.__call__) + + if hasattr(self._orig_mod, "_initialize_hook"): + self._forward = self.forward + self.forward = self._call_lazy_check + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + if torch.nn.modules.module._has_any_global_hook(): + warnings.warn( + "Using `torch.compile(module)` when there are global hooks on " + "modules (e.g., from `register_module_forward_hook`); this will" + " cause the hooks to fire an extra time for the " + "`OptimizedModule` created by `torch.compile(module)`. If this " + "causes undesired behavior, please try using `module.compile()`" + ", or use the per-module hooks instead", + stacklevel=2, + ) + with _set_in_optimized_module(): + return super().__call__(*args, **kwargs) + + def _aot_compile(self, inputs: list[torch._dynamo.aot_compile.ModelInput]) -> None: + """ + Experimental: AOT Compile a set of inputs and use that as the forward function + """ + model = self._orig_mod + hooks = self.dynamo_ctx._hooks + assert hooks is not None + if not config.enable_aot_compile: + raise RuntimeError( + "AOT Compile is not enabled, please set torch._dynamo.config.enable_aot_config=True" + ) + if not self.dynamo_ctx.fullgraph: + raise RuntimeError( + "Graph breaks are not supported with aot compile. Please use torch.compile(fullgraph=True)." + ) + + if not callable(self.dynamo_ctx.callback): + raise RuntimeError("aot compile requires a callable dynamo callback.") + + backend = innermost_fn( + self.dynamo_ctx.callback, unaltered_fn_attr="_torchdynamo_orig_backend" + ) + from torch._dynamo.aot_compile import aot_compile_module + + self.forward = aot_compile_module(model, inputs, hooks, backend) + + def _save_aot_compiled_module(self, path: Optional[str] = None) -> bytes: + if not config.enable_aot_compile: + raise RuntimeError( + "AOT Compile is not enabled, please set torch._dynamo.config.enable_aot_config=True" + ) + from torch._dynamo.aot_compile import AOTCompiledModel + + assert isinstance(self.forward, AOTCompiledModel) + result: bytes = self.forward.serialize() + if path is not None: + with open(path, "wb") as f: + f.write(result) + return result + + def _load_aot_compiled_module(self, data: bytes) -> None: + if not config.enable_aot_compile: + raise RuntimeError( + "AOT Compile is not enabled, please set torch._dynamo.config.enable_aot_config=True" + ) + from torch._dynamo.aot_compile import AOTCompiledModel + + compiled_forward = AOTCompiledModel.deserialize(self._orig_mod, data) + assert isinstance(compiled_forward, AOTCompiledModel) + self.forward = compiled_forward + + def __reduce__( + self, + ) -> tuple[type[OptimizedModule], tuple[torch.nn.Module, _TorchDynamoContext]]: + return (self.__class__, (self._orig_mod, self.dynamo_ctx)) + + def __getstate__(self) -> dict[str, Any]: + state = dict(self.__dict__) + state.pop("forward", None) + state.pop("__call__", None) + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__ = state + self._initialize() + + @property + # pyrefly: ignore [bad-override] + def training(self) -> bool: + return self._orig_mod.training + + @training.setter + def training(self, value: bool) -> None: + # Ignore the `training` mutation in `super().__init__()`, since that's + # setting the default on `nn.Module`, but we are mirroring the + # `training` attr in `self._orig_mod`. + if self._super_module_initialized: + self._orig_mod.training = value + + def __getattr__(self, name: str) -> Any: + if name == "_orig_mod": + return self._modules["_orig_mod"] + return getattr(self._orig_mod, name) + + def __setattr__(self, name: str, val: Any) -> None: + # Allow patching over class attributes + if hasattr(type(self), name): + return super().__setattr__(name, val) + + if name in OptimizedModule._opt_mod_attributes: + return super().__setattr__(name, val) + return setattr(self._orig_mod, name, val) + + def __delattr__(self, name: str) -> None: + # This mirrors `__setattr__` + if hasattr(type(self), name): + return super().__delattr__(name) + + if name in OptimizedModule._opt_mod_attributes: + return super().__delattr__(name) + return delattr(self._orig_mod, name) + + def _call_lazy_check(self, *args: Any, **kwargs: Any) -> Any: + if ( + hasattr(self._orig_mod, "_initialize_hook") + and hasattr(self._orig_mod, "_infer_parameters") + and callable(self._orig_mod._infer_parameters) + ): + # In the case of a lazy module, we want to run + # the pre-hooks which initialize it. + # Afterwards, lazy module deletes its pre-hooks + # to avoid treating it as lazy on subsequent recompile. + self._orig_mod._infer_parameters(self._orig_mod, args, kwargs) + return self._forward(*args, **kwargs) + + def __dir__(self) -> list[str]: + orig_mod_attrs = self._orig_mod.__dir__() + return orig_mod_attrs + [ + attr for attr in super().__dir__() if attr not in orig_mod_attrs + ] + + +def remove_from_cache(f: Any) -> None: + """ + Make sure f.__code__ is not cached to force a recompile + """ + if isinstance(f, types.CodeType): + reset_code(f) + elif hasattr(f, "__code__"): + reset_code(f.__code__) + elif hasattr(getattr(f, "forward", None), "__code__"): + reset_code(f.forward.__code__) + else: + from . import reset # type: ignore[attr-defined] + + reset() + log.warning("could not determine __code__ for %s", f) + + +def nothing() -> None: + pass + + +def always_false() -> bool: + return False + + +def innermost_fn( + fn: Callable[..., Any], unaltered_fn_attr: str = "_torchdynamo_orig_callable" +) -> Callable[..., Any]: + """ + In case of nesting of _TorchDynamoContext calls, find the innermost + function. TorchDynamo caches on fn.__code__ object, so its necessary to find + the innermost function to pass on the optimize, run, disable etc. + """ + unaltered_fn = fn + while hasattr(unaltered_fn, unaltered_fn_attr): + unaltered_fn = getattr(unaltered_fn, unaltered_fn_attr) + assert callable(unaltered_fn), ( + f"A callable function is expected, but {type(unaltered_fn)} is provided." + ) + return unaltered_fn + + +def make_set_enable_dynamic(enable: bool) -> Any: + assert isinstance(enable, bool) + if enable: + # Assume everything is dynamic by default + return config._make_closure_patcher(assume_static_by_default=False) + else: + return config._make_closure_patcher( + automatic_dynamic_shapes=False, assume_static_by_default=True + ) + + +# A thread local storage that serves to store information as Dynamo traces +# through a user provided function. +class DynamoTLS(threading.local): + # Each string is a summary of a frame Dynamo attempted to trace, stored in + # temporal order. + traced_frame_infos: list[str] = [] + + +dynamo_tls = DynamoTLS() + + +def clear_dynamo_tls() -> None: + dynamo_tls.traced_frame_infos.clear() + + +@atexit.register +def _log_traced_frames() -> None: + """ + At program exit, log all of the frames Dynamo has attempted to trace from, + excluding the continuation frames generated by Dynamo. + """ + msg = "\n".join(dynamo_tls.traced_frame_infos) + msg = textwrap.indent(msg, " * ") + msg = f"TorchDynamo attempted to trace the following frames: [\n{msg}\n]" + log.info(msg) + + +def guard_collectives_hook(guard_eval_result: bool) -> bool: + import torch.distributed as dist + from torch._dynamo.utils import dynamo_timed + + # guard_eval_result == True ==> cache hit + if pg := distributed.get_guard_pg(): + with dynamo_timed( + "guard_collective", log_pt2_compile_event=False, log_waitcounter=True + ): + log.debug("guard_collective %s", guard_eval_result) + # TODO: a bit awkward to time, this isn't inside of the dynamo compile region + all_results = [None] * pg.size() + dist.all_gather_object(all_results, guard_eval_result, group=pg) + # True = everyone hit, OK to run + # False = someone missed, force recompile everywhere + res = all(all_results) + log.debug("guard_collective %s -> %s", guard_eval_result, res) + return res + return guard_eval_result + + +_not_set = object() + + +class _TorchDynamoContext: + def __init__( + self, + callback: DynamoCallback, + on_enter: Callable[[], Any] = nothing, + backend_ctx_ctor: Callable[ + [], contextlib.AbstractContextManager[Any] + ] = null_context, + patch_fn: Callable[[], Any] = nothing, + first_ctx: bool = False, + *, + fullgraph: bool = False, + error_on_graph_break: Optional[bool] = None, + export: bool = False, + dynamic: Optional[bool] = None, + compiler_config: Optional[Any] = None, + package: Optional[CompilePackage] = None, + hooks: Optional[Hooks] = None, + ) -> None: + super().__init__() + assert callable(callback) or callback is False or callback is None + self.callback: DynamoCallback = callback + self._backend_ctx_ctor = backend_ctx_ctor + self.prior: Union[Unset, DynamoCallback] = unset + self.first_ctx = first_ctx + self.fullgraph = fullgraph + self.error_on_graph_break = error_on_graph_break + self.export = export + self._dynamic = dynamic + self.compiler_config = compiler_config + self.cleanup_fns: list[Callable[[], Any]] = [] + self.enter_exit_hooks = [] + self._package = package + self._hooks = hooks + patch_fn() + + # Save the backends so that we can reset them during torch._dynamo.reset + backend = innermost_fn(callback, unaltered_fn_attr="_torchdynamo_orig_backend") # type: ignore[arg-type] + cached_backends.setdefault(id(backend), backend) # type: ignore[arg-type] + + if dynamic is not None: + self.enter_exit_hooks.append(make_set_enable_dynamic(dynamic)) + + if on_enter is not nothing: + # this case is not common + def call_on_enter() -> Callable[[], None]: + on_enter() + return nothing + + self.enter_exit_hooks.append(call_on_enter) + + if backend_ctx_ctor is not contextlib.nullcontext: + # this case is not common + def call_backend_ctx() -> functools.partial[Optional[bool]]: + ctx = backend_ctx_ctor() + ctx.__enter__() + return functools.partial(ctx.__exit__, None, None, None) + + self.enter_exit_hooks.append(call_backend_ctx) + + def __enter__(self) -> None: + if config.raise_on_ctx_manager_usage: + raise RuntimeError( + "torch._dynamo.optimize(...) is used with a context manager. " + "Please refer to https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html " + "to use torch._dynamo.optimize(...) as an annotation/decorator. " + ) + self.prior = set_eval_frame(None) + self.cleanup_fns = [enter() for enter in self.enter_exit_hooks] + self.prior_skip_guard_eval_unsafe = set_skip_guard_eval_unsafe( + _is_skip_guard_eval_unsafe_stance() + ) + _maybe_set_eval_frame(_callback_from_stance(self.callback)) + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[types.TracebackType], + ) -> Optional[bool]: + assert self.prior is not unset + set_eval_frame(None) + set_skip_guard_eval_unsafe(self.prior_skip_guard_eval_unsafe) + for cleanup in self.cleanup_fns: + cleanup() + self.cleanup_fns.clear() + _maybe_set_eval_frame(_callback_from_stance(self.prior)) + self.prior = unset + return None + + def __call__(self, fn: Any) -> Any: + # public api for compiler config/options + def get_compiler_config() -> Any: + return self.compiler_config + + from .package import DynamoCache + + # If self._package is lazily initialized, we should check the dynamo cache now + if config.caching_precompile: + if self._package is not None and not self._package.is_initialized(): + fn_key = fn.forward if isinstance(fn, torch.nn.Module) else fn + result = DynamoCache.load(fn_key) + if result is None: + # Create a fresh CompilePackage + self._package.initialize(fn_key, None, ignore_inlined_sources=False) + else: + try: + self._package.initialize( + fn_key, result.dynamo, ignore_inlined_sources=False + ) + self._package.install(result.backends) + except RuntimeError: + log.warning( + "Failed to load entry from dynamo cache", exc_info=True + ) + self._package.initialize( + fn_key, None, ignore_inlined_sources=False + ) + + fn = innermost_fn(fn) + + def aot_compile(example_inputs: tuple[tuple[Any, ...], dict[str, Any]]) -> Any: + from torch._dynamo.aot_compile import aot_compile_fullgraph + + if torch._inductor.config.force_disable_caches: + raise RuntimeError( + "Cannot precompile with torch._inductor.config.force_disable_caches=True; caching is required." + ) + + if not self.fullgraph: + raise RuntimeError( + "Graph breaks are not supported with aot compile. Please use torch.compile(fullgraph=True)." + ) + + if not callable(self.callback): + raise RuntimeError("aot compile requires a callable dynamo callback.") + + assert self._hooks is not None + + return aot_compile_fullgraph( + fn, + example_inputs, + hooks=self._hooks, + backend=innermost_fn( + self.callback, unaltered_fn_attr="_torchdynamo_orig_backend" + ), + ) + + # add context containing GraphModule to any GraphModule forward functions + if isinstance(fn, GraphModule): + # add context containing GraphModule to any GraphModule forward functions + code_context.get_context(fn.forward.__code__)["orig_graphmodule"] = ( + weakref.ref(fn) + ) + + # Optimize the forward method of torch.nn.Module object + if isinstance(fn, torch.nn.Module): + mod = fn + new_mod = OptimizedModule(mod, self) + # Save the function pointer to find the original callable while nesting + # of decorators. + new_mod._torchdynamo_orig_callable = mod.forward + + # when compiling torch.nn.Module, + # provide public api OptimizedModule.get_compiler_config() + assert not hasattr(new_mod, "get_compiler_config") + new_mod.get_compiler_config = get_compiler_config + + return new_mod + + if inspect.isclass(fn): + # User has wrapped the class with compile/disable decorator. Apply + # disable to init/call method. + cls_obj = fn + cls_obj.__call__ = self(cls_obj.__call__) + if issubclass(cls_obj, torch.nn.Module): + # NN module variable tracker directly inlines the _call_impl. + cls_obj._call_impl = self(cls_obj._call_impl) + return cls_obj + + assert callable(fn), ( + f"A callable function is expected, but {type(fn)} is provided." + ) + + try: + filename = inspect.getsourcefile(fn) + except TypeError: + filename = None + if config.debug_force_nested_calls: + fn = external_utils.wrap_inline(fn) + elif config.wrap_top_frame or ( + (filename is None or trace_rules.check(fn)) + and ( + getattr(fn, "__name__", "") + not in ["_call_impl", "_wrapped_call_impl", "_lazy_forward"] + ) + and filename not in DONT_WRAP_FILES + ): + # call to a builtin without a frame for us to capture + fn = external_utils.wrap_inline(fn) + + def do_nothing(*arg: Any, **kwargs: Any) -> None: + pass + + callback: Callable[..., Any] = do_nothing + if hasattr(self, "callback"): + callback = self.callback # type: ignore[assignment] + + is_jit_tracing = torch._C._is_tracing + is_fx_symbolic_tracing = torch.fx._symbolic_trace.is_fx_symbolic_tracing + + @functools.wraps(fn) + def compile_wrapper(*args: Any, **kwargs: Any) -> Any: + prior = set_eval_frame(None) + try: + # We shouldn't compile inside kernel invocation. + if tracing_context := torch._guards.TracingContext.try_get(): + if ( + tracing_context.fake_mode is not None + and tracing_context.fake_mode.in_kernel_invocation + ): + return fn(*args, **kwargs) + # Skip nested compile - just inline the function + if is_fx_symbolic_tracing(): + if config.error_on_nested_fx_trace: + raise RuntimeError( + "Detected that you are using FX to symbolically trace " + "a dynamo-optimized function. This is not supported at the moment." + ) + else: + return fn(*args, **kwargs) + + if is_jit_tracing(): + raise RuntimeError( + "Detected that you are using FX to torch.jit.trace " + "a dynamo-optimized function. This is not supported at the moment." + ) + + cleanups = [enter() for enter in self.enter_exit_hooks] + prior_skip_guard_eval_unsafe = set_skip_guard_eval_unsafe( + _is_skip_guard_eval_unsafe_stance() + ) + prior_error_on_graph_break = None + if not self.fullgraph and self.error_on_graph_break is not None: + prior_error_on_graph_break = _get_error_on_graph_break() + _set_error_on_graph_break(self.error_on_graph_break) + + # Ensure that if an assertion occurs after graph pushes + # something onto the DynamicLayerStack then we pop it off (the + # constructed graph code isn't guarded with try/finally). + # + # This used to be a context but putting a `with` here is a noticeable + # perf regression (#126293) + saved_dynamic_layer_stack_depth = ( + torch._C._functorch.get_dynamic_layer_stack_depth() + ) + + _maybe_set_eval_frame(_callback_from_stance(callback)) + + try: + return fn(*args, **kwargs) + except Unsupported as e: + if config.verbose: + raise + # strip internal tracebacks from causes + cur_exn: BaseException = e + while cur_exn.__cause__ is not None: + cur_exn.__cause__.with_traceback(None) + cur_exn = cur_exn.__cause__ + # pyrefly: ignore [invalid-inheritance] + raise e.with_traceback(None) from e.__cause__ # User compiler error + except ShortenTraceback as e: + # Failures in the backend likely don't have useful + # data in the TorchDynamo frames, so we strip them out. + raise e.remove_dynamo_frames() from None # see TORCHDYNAMO_VERBOSE=1 + finally: + # Restore the dynamic layer stack depth if necessary. + set_eval_frame(None) + if prior_error_on_graph_break is not None: + _set_error_on_graph_break(prior_error_on_graph_break) + torch._C._functorch.pop_dynamic_layer_stack_and_undo_to_depth( + saved_dynamic_layer_stack_depth + ) + + set_skip_guard_eval_unsafe(prior_skip_guard_eval_unsafe) + for cleanup in cleanups: + cleanup() + finally: + _maybe_set_eval_frame(prior) + + # hooks to properly handle inlining + if self.error_on_graph_break is not None: + compile_wrapper._torchdynamo_inline = ( # type: ignore[attr-defined] + external_utils.wrap_inline_with_error_on_graph_break( + fn, self.error_on_graph_break + ) + ) + else: + compile_wrapper._torchdynamo_inline = fn # type: ignore[attr-defined] + + # Save the function pointer to find the original callable while nesting + # of decorators. + compile_wrapper._torchdynamo_orig_callable = fn # type: ignore[attr-defined] + + # when compiling user function instead of nn.Module + # provide public api _fn.get_compiler_config() + assert not hasattr(compile_wrapper, "get_compiler_config") + compile_wrapper.get_compiler_config = get_compiler_config # type: ignore[attr-defined] + if torch._dynamo.config.enable_aot_compile: + compile_wrapper.aot_compile = aot_compile # type: ignore[attr-defined] + + # If the function is called using torch._dynamo.optimize decorator, we + # should prevent any type of skipping. + if callback not in (None, False): + if not hasattr(fn, "__code__"): + raise RuntimeError( + textwrap.dedent( + """ + + torch._dynamo.optimize is called on a non function object. + If this is a callable class, please wrap the relevant code into a function and optimize the + wrapper function. + + >> class CallableClass: + >> def __init__(self) -> None: + >> super().__init__() + >> self.relu = torch.nn.ReLU() + >> + >> def __call__(self, x): + >> return self.relu(torch.sin(x)) + >> + >> def print_hello(self): + >> print("Hello world") + >> + >> mod = CallableClass() + + If you want to optimize the __call__ function and other code, wrap that up in a function + + >> def wrapper_fn(x): + >> y = mod(x) + >> return y.sum() + + and then optimize the wrapper_fn + + >> opt_wrapper_fn = torch._dynamo.optimize(wrapper_fn) + """ + ) + ) + always_optimize_code_objects[fn.__code__] = True + + return compile_wrapper + + +class OptimizeContext(_TorchDynamoContext): + def __init__( + self, + callback: DynamoCallback, + backend_ctx_ctor: Callable[[], contextlib.AbstractContextManager[Any]], + first_ctx: bool = False, + *, + fullgraph: bool = False, + error_on_graph_break: Optional[bool] = None, + export: bool = False, + dynamic: Optional[bool] = None, + compiler_config: Optional[Any] = None, + rebuild_ctx: Optional[ + Callable[[], Union[OptimizeContext, _NullDecorator]] + ] = None, + package: Optional[CompilePackage] = None, + hooks: Optional[Hooks] = None, + ) -> None: + def on_enter() -> None: + install_generation_tagging_init() + + super().__init__( + callback=callback, + on_enter=on_enter, + backend_ctx_ctor=backend_ctx_ctor, + patch_fn=TorchPatcher.patch, + first_ctx=first_ctx, + fullgraph=fullgraph, + error_on_graph_break=error_on_graph_break, + export=export, + dynamic=dynamic, + compiler_config=compiler_config, + package=package, + hooks=hooks, + ) + + if config.compiled_autograd: + _dynamic = self._dynamic + if _dynamic is None: + _dynamic = not torch._dynamo.config.assume_static_by_default + + def call_compiled_autograd() -> functools.partial[Optional[bool]]: + assert rebuild_ctx is not None + compiler_fn = rebuild_ctx() + ctx = torch._dynamo.compiled_autograd._enable( + compiler_fn, + # pyrefly: ignore [bad-argument-type] + dynamic=_dynamic, + ignore_active_disable_ctx=False, + ) + ctx.__enter__() + return functools.partial(ctx.__exit__, None, None, None) + + self.enter_exit_hooks.append(call_compiled_autograd) + + def __reduce__( + self, + ) -> tuple[type[OptimizeContext], tuple[Any, ...], dict[str, Any]]: + return ( + self.__class__, + (self.callback, self._backend_ctx_ctor, self.first_ctx), + { + "export": self.export, + "dynamic": self._dynamic, + "compiler_config": self.compiler_config, + }, + ) + + +class RunOnlyContext(_TorchDynamoContext): + def __init__(self) -> None: + # cudagraph trees relies on generation increment + def on_enter() -> None: + torch._dynamo.mutation_guard.GenerationTracker.generation += 1 + + super().__init__(callback=False, on_enter=on_enter) + + def __reduce__(self) -> tuple[type[RunOnlyContext], tuple[Any, ...]]: + return (self.__class__, ()) + + +class DisableContext(_TorchDynamoContext): + def __init__(self, msg: Optional[str] = None, wrapping: bool = True) -> None: + super().__init__(callback=None) + self.msg = msg + self.wrapping = wrapping + + def __call__(self, fn: Callable[..., Any]) -> Callable[..., Any]: + # Earlier this code was in the base class _TorchDynamoContext. But we + # moved it here to have better code organization. For disable, we just + # want the callback to be None. We don't have to check trace_rules or + # create any wrapper. + fn = innermost_fn(fn) + + if isinstance(fn, torch.nn.Module): + mod = fn + new_mod = OptimizedModule(mod, self) + new_mod._torchdynamo_orig_callable = mod.forward + return new_mod + + if isinstance(fn, type): + # User has wrapped the class with compile/disable decorator. Apply + # disable to init/call method. + cls_obj = fn + # Disable on init is useful for reconstruction of bytecodes where we + # want to prevent Dynamo from tracing into the init function. Check + # test_reconstruction in test_model_output.py. + cls_obj.__init__ = self(cls_obj.__init__) # type: ignore[misc] + cls_obj.__call__ = self(cls_obj.__call__) + if issubclass(cls_obj, torch.nn.Module): + # NN module variable tracker directly inlines the _call_impl. Disable it. + # pyrefly: ignore [missing-attribute] + cls_obj._call_impl = self(cls_obj._call_impl) + return cls_obj + + assert callable(fn), ( + f"A callable function is expected, but {type(fn)} is provided." + ) + + def _fn(*args: Any, **kwargs: Any) -> Any: + prior = set_eval_frame(None) + try: + _maybe_set_eval_frame(_callback_from_stance(self.callback)) + try: + if torch.compiler.is_exporting(): + with fx_traceback.annotate( + { + "_torchdynamo_disable": True, + "_torchdynamo_disable_recursive": True, + "_torchdynamo_disable_method": getattr( + fn, "__name__", type(fn).__name__ + ), + } + ): + return fn(*args, **kwargs) + return fn(*args, **kwargs) + finally: + set_eval_frame(None) + finally: + _maybe_set_eval_frame(prior) + + # Under some circumstances (e.g. precompile) we can end up calling @disable + # decorator in generated bytecode and trigger recompile. This is due to the + # fact that the old callback from torch.compile() is still active and under + # this circumstance we will trigger a failure with set_stance("fail_on_recompile"). + # Therefore we want to skip calling into any frame in this case. + if self.wrapping: + _fn = functools.wraps(fn)(_fn) + + _fn._torchdynamo_disable = True # type: ignore[attr-defined] + _fn._torchdynamo_disable_msg = self.msg # type: ignore[attr-defined] + + # Save the function pointer to find the original callable while nesting + # of decorators. + _fn._torchdynamo_orig_callable = fn # type: ignore[attr-defined] + + _fn._torchdynamo_disable_recursive = True # type: ignore[attr-defined] + + return _fn + + def __reduce__(self) -> tuple[type[DisableContext], tuple[Any, ...]]: + return (self.__class__, ()) + + +def _optimize_catch_errors( + compile_fn: convert_frame.ConvertFrameProtocol, + hooks: Hooks, + backend_ctx_ctor: Callable[ + [], contextlib.AbstractContextManager[Any] + ] = null_context, + fullgraph: bool = False, + error_on_graph_break: Optional[bool] = None, + export: bool = False, + dynamic: Optional[bool] = None, + compiler_config: Optional[Any] = None, + rebuild_ctx: Optional[Callable[[], Union[OptimizeContext, _NullDecorator]]] = None, + package: Optional[CompilePackage] = None, +) -> OptimizeContext: + return OptimizeContext( + convert_frame.catch_errors_wrapper(compile_fn, hooks), + backend_ctx_ctor=backend_ctx_ctor, + first_ctx=True, + fullgraph=fullgraph, + error_on_graph_break=error_on_graph_break, + export=export, + dynamic=dynamic, + compiler_config=compiler_config, + rebuild_ctx=rebuild_ctx, + package=package, + hooks=hooks, + ) + + +def get_compiler_fn( + compiler_fn: Union[str, Callable[..., Any], None], +) -> WrapBackendDebug: + from .repro.after_dynamo import wrap_backend_debug + + if compiler_fn is None: + # Special case None to avoid crashing in hasattr + compiler_str = None + elif hasattr(compiler_fn, "compiler_name"): + compiler_str = compiler_fn.compiler_name # type: ignore[union-attr] + assert isinstance(compiler_str, str) + elif isinstance(compiler_fn, str): + compiler_str = compiler_fn + else: + compiler_str = None + compiler_fn = lookup_backend(compiler_fn) # type: ignore[arg-type] + return wrap_backend_debug(compiler_fn, compiler_str) + + +class _NullDecorator(contextlib.nullcontext): # type: ignore[type-arg] + def __call__(self, fn: Callable[..., Any]) -> Callable[..., Any]: + assert callable(fn), ( + f"A callable function is expected, but {type(fn)} is provided." + ) + return fn + + +# Make dynamo graph to have same input/output spec as user code +def argument_names( + f_sig: inspect.Signature, + args: Union[list[Any], tuple[Any, ...]], + kwargs: dict[str, Any], +) -> list[str]: + def signature_to_fullargspec(sig: inspect.Signature) -> inspect.FullArgSpec: + # Get a list of Parameter objects from the Signature object + params = list(sig.parameters.values()) + # Separate positional arguments, keyword-only arguments and varargs/varkw + args = [ + p.name for p in params if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD + ] + kwonlyargs = [ + p.name for p in params if p.kind == inspect.Parameter.KEYWORD_ONLY + ] + varargs = next( + (p.name for p in params if p.kind == inspect.Parameter.VAR_POSITIONAL), + None, + ) + varkw = next( + (p.name for p in params if p.kind == inspect.Parameter.VAR_KEYWORD), + None, + ) + # Get default values for positional arguments and keyword-only arguments + defaults = tuple( + p.default + for p in params + if p.kind == inspect.Parameter.POSITIONAL_OR_KEYWORD + and p.default is not inspect.Parameter.empty + ) + kwonlydefaults = { + p.name: p.default + for p in params + if p.kind == inspect.Parameter.KEYWORD_ONLY + and p.default is not inspect.Parameter.empty + } + # Get annotations for parameters and return value + annotations = {} + if sig.return_annotation: + annotations = {"return": sig.return_annotation} + for parameter in params: + annotations[parameter.name] = parameter.annotation + # Return a FullArgSpec object with the extracted attributes + return inspect.FullArgSpec( + args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations + ) + + fullargspec = signature_to_fullargspec(f_sig) + + # 1. Map `args` 1-to-1 to positional arguments in original signature. + input_strs = fullargspec.args[: len(args)] + + if len(args) > len(fullargspec.args): + # 2. If there are more arguments left in `args`, they map to varargs in original + # signature. Assign names as {varargs}_0, {varargs}_1, ... + assert fullargspec.varargs is not None, "More arguments than expected" + input_strs += [ + f"{fullargspec.varargs}_{i}" for i in range(len(args) - len(input_strs)) + ] + elif len(args) < len(fullargspec.args): + # 3. If there are fewer arguments in `args` than `fullargspec.args`, + # it implies these are arguments either with default values, or provided in + # `kwargs`. The former can be safely ignored. Because Dynamo.export does not + # export them as part of the function signature. The latter will be handled + # in the next step. + for unprovided_arg in fullargspec.args[ + len(args) : -len(fullargspec.defaults or []) + ]: + assert unprovided_arg in kwargs, f"Missing argument {unprovided_arg}" + + # 4. Keyword arguments provided in `kwargs`. + input_strs += list(kwargs.keys()) + + # 5. Keyword-only arguments with default values if not provided are not exported + # as part of the function signature. + for kwonly_arg in fullargspec.kwonlyargs: + kwonlydefaults = fullargspec.kwonlydefaults or {} + assert kwonly_arg in kwargs or kwonly_arg in kwonlydefaults, ( + f"Missing keyword only argument {kwonly_arg}" + ) + + return input_strs + + +def check_if_dynamo_supported() -> None: + if sys.version_info >= (3, 15): + raise RuntimeError("Python 3.15+ not yet supported for torch.compile") + elif sysconfig.get_config_var("Py_GIL_DISABLED") == 1 and sys.version_info < ( + 3, + 13, + 3, + ): + raise RuntimeError( + "torch.compile is not supported on Python < 3.13.3 built with GIL disabled. " + "Please use Python 3.13.3+." + ) + + +def is_dynamo_supported() -> bool: + try: + check_if_dynamo_supported() + return True + except Exception: + return False + + +def check_if_inductor_supported() -> None: + check_if_dynamo_supported() + + +def is_inductor_supported() -> bool: + try: + check_if_inductor_supported() + return True + except Exception: + return False + + +def check_for_incompatible_configs() -> None: + # Some of the configs should be mutually exclusive + assert not (config.suppress_errors and config.fail_on_recompile_limit_hit), ( + "Dynamo configs suppress_error and fail_on_recompile_limit_hit can not both be active at the same time." + ) + + +def optimize(*args: Any, **kwargs: Any) -> Union[OptimizeContext, _NullDecorator]: + def rebuild_ctx() -> Union[OptimizeContext, _NullDecorator]: + ca_kwargs_override = config.compiled_autograd_kwargs_override + if ca_kwargs_override: + # NOTE: The process of translating other `torch.compile` kwargs to `torch._dynamo.optimize` kwargs + # is more complicated, we will add it in the future when needed. + assert set(ca_kwargs_override.keys()) == {"fullgraph"}, ( + f"Only `fullgraph` kwarg override is supported for now, but got {ca_kwargs_override.keys()}" + ) + kwargs["nopython"] = ca_kwargs_override["fullgraph"] + return optimize(*args, **kwargs) + + return _optimize(rebuild_ctx, *args, **kwargs) + + +def _optimize( + rebuild_ctx: Callable[[], Union[OptimizeContext, _NullDecorator]], + backend: Union[str, Callable[..., Any]] = "inductor", + *, + nopython: bool = False, + error_on_graph_break: Optional[bool] = None, + guard_export_fn: Optional[Callable[[_guards.GuardsSet], None]] = None, + guard_fail_fn: Optional[Callable[[GuardFail], None]] = None, + guard_filter_fn: Optional[Callable[[list[GuardFilterEntry]], list[bool]]] = None, + disable: bool = False, + dynamic: Optional[bool] = None, + package: Optional[CompilePackage] = None, +) -> Union[OptimizeContext, _NullDecorator]: + """ + The main entrypoint of TorchDynamo. Do graph capture and call + backend() to optimize extracted graphs. + + Args: + backend: One of the two things: + - Either, a function/callable taking a torch.fx.GraphModule and + example_inputs and returning a python callable that runs the + graph faster. + One can also provide additional context for the backend, like + torch.jit.fuser("fuser2"), by setting the backend_ctx_ctor attribute. + See AOTAutogradMemoryEfficientFusionWithContext for the usage. + - Or, a string backend name in `torch._dynamo.list_backends()` + nopython: If True, graph breaks will be errors and there will + be a single whole-program graph. + error_on_graph_break: If not None, the current `error_on_graph_break` setting is set to the given value. + See `torch._dynamo.error_on_graph_break()` for more details on what `error_on_graph_break` means. + + Unlike `nopython=True` (i.e. `fullgraph=True`), there is no guarantee of a single whole-program graph. + If `nopython` is True, `error_on_graph_break` does nothing. + disable: If True, turn this decorator into a no-op + dynamic: If True, upfront compile as dynamic a kernel as possible. If False, + disable all dynamic shapes support (always specialize). If None, automatically + detect when sizes vary and generate dynamic kernels upon recompile. + + Example Usage:: + + @torch._dynamo.optimize() + def toy_example(a, b): ... + """ + check_if_dynamo_supported() + check_for_incompatible_configs() + # Note: The hooks object could be global instead of passed around, *however* that would make + # for a confusing API usage and plumbing story wherein we nest multiple .optimize calls. + # There is some prior art around this, w/r/t nesting backend calls are enforced to be the same + # compiler, however, this feels onerous for callback and hooks, and it feels better to give our users an + # easier to understand UX at the cost of a little more plumbing on our end. + hooks = Hooks( + guard_export_fn=guard_export_fn, + guard_fail_fn=guard_fail_fn, + guard_filter_fn=guard_filter_fn, + ) + torch._C._log_api_usage_once("torch._dynamo.optimize") + if ( + disable + or os.environ.get("TORCHDYNAMO_DISABLE", "") == "1" + or (not justknobs_check("pytorch/compiler:enable_dynamo")) + ): + return _NullDecorator() + + if nopython and not config.debug_force_graph_break_on_leaf_return: + return optimize_assert( + backend, + dynamic=dynamic, + hooks=hooks, + rebuild_ctx=rebuild_ctx, + package=package, + ) + + backend = get_compiler_fn(backend) + + # Find if backend has any extra context manager + backend_ctx_ctor = getattr(backend, "backend_ctx_ctor", null_context) + + # The backend function is stashed in the callable returned by + # _optimize_catch_errors in the field _torchdynamo_orig_backend. This can + # be used by eval_frame.c to insert a guard on the backend. + + # With CachingPrecompile, instantiate an uninitialized CompilePackage + # which gets initialized by _optimize_catch_errors.__call__ once we have a function + if config.caching_precompile and package is None: + from .package import CompilePackage + + package = CompilePackage(fn=None, dynamo=None, ignore_inlined_sources=False) + + return _optimize_catch_errors( + convert_frame.convert_frame( + backend, + hooks, + package=package, + ), + hooks, + backend_ctx_ctor, + fullgraph=False, + error_on_graph_break=error_on_graph_break + and not config.debug_force_graph_break_on_leaf_return, + dynamic=dynamic, + compiler_config=( + backend.get_compiler_config() + if hasattr(backend, "get_compiler_config") + else None + ), + rebuild_ctx=rebuild_ctx, + package=package, + ) + + +# TODO(voz): Consider making "explain" output alongside a run / part of a run +@patch("torch._dynamo.symbolic_convert.explain", True) +def explain(f: Callable[..., Any], *extra_args: Any, **extra_kwargs: Any) -> Any: + from .backends.debugging import ExplainOutput + + def inner(*args: Any, **kwargs: Any) -> ExplainOutput: + # TODO(voz): Do we want a decorator for this? + from . import reset # type: ignore[attr-defined] + + reset() + + graphs: list[torch.fx.GraphModule] = [] + break_reasons: list[Any] = [] + op_count: int = 0 + ops_per_graph: list[list[Target]] = [] + out_guards: list[_guards.Guard] = [] + + def dynamo_graph_accumulating_compiler( + gm: torch.fx.GraphModule, example_inputs: Any + ) -> Callable[..., Any]: + from .backends.debugging import _explain_graph_detail + + nonlocal graphs + nonlocal op_count + nonlocal ops_per_graph + nonlocal break_reasons + + gm, graphs, op_count, ops_per_graph, break_reasons = _explain_graph_detail( + gm, graphs, op_count, ops_per_graph, break_reasons + ) + + return gm.forward + + def guard_export_print(guards: Iterable[_guards.Guard]) -> None: + nonlocal out_guards + out_guards.extend(guards) + + opt_f = optimize( + dynamo_graph_accumulating_compiler, + nopython=False, + guard_export_fn=guard_export_print, + )(f) + # TODO(voz): We may have instances of `f` that mutate inputs, we should track sideeffects and reject. + opt_f(*args, **kwargs) + + graph_count = len(graphs) + graph_break_count = graph_count - 1 + compile_time = compile_times(repr="str") + + # TODO(voz): Do we want a decorator for this? + reset() + + return ExplainOutput( + graphs, + graph_count, + graph_break_count, + break_reasons, + op_count, + ops_per_graph, + out_guards, + compile_time, + ) + + if extra_args or extra_kwargs: + warnings.warn( + "explain(f, *args, **kwargs) is deprecated, use explain(f)(*args, **kwargs) instead. " + "If you don't migrate, we may break your explain call in the future if your user defined kwargs " + "conflict with future kwargs added to explain(f).", + FutureWarning, + stacklevel=2, + ) + return inner(*extra_args, **extra_kwargs) + else: + return inner + + +class FlattenInputOutputSignature(torch.fx.Transformer): + def __init__( + self, + m: torch.fx.GraphModule, + flat_args: list[Any], + matched_input_elements_positions: list[int], + flat_results: Sequence[Any], + matched_output_elements_positions: list[int], + example_fake_inputs: list[torch.Tensor], + flat_args_dynamic_dims: list[set[int]], + fake_mode: Optional[fake_tensor.FakeTensorMode] = None, + ) -> None: + super().__init__(m) + + assert len(flat_args_dynamic_dims) == len(flat_args) + matched_input_elements_to_fake = { + val: example_fake_inputs[ix] + for ix, val in enumerate(matched_input_elements_positions) + } + + self.new_args = [] + for i in range(len(flat_args)): + arg = super().placeholder(f"arg{i}", (), {}) + if i in matched_input_elements_to_fake: + arg.node.meta["val"] = matched_input_elements_to_fake[i] + else: + # Fill node.meta["val"] with faketensor from the input, + # if it's not found in matched_input_elements_positions + if fake_mode is not None and isinstance(flat_args[i], torch.Tensor): + # TODO(zhxchen17) Also preserve all the user constraints here. + arg.node.meta["val"] = fake_mode.from_tensor( + flat_args[i], + symbolic_context=StatelessSymbolicContext( + dynamic_sizes=[ + ( + DimDynamic.DYNAMIC + if d in flat_args_dynamic_dims[i] + else DimDynamic.STATIC + ) + for d in range(len(flat_args[i].shape)) + ], + constraint_sizes=[None] * len(flat_args[i].shape), + ), + ) + elif isinstance(flat_args[i], _IntWrapper): + arg.node.meta["val"] = flat_args[i].val + else: + arg.node.meta["val"] = flat_args[i] + + self.new_args.append(arg) + self.old_args_gen = (self.new_args[i] for i in matched_input_elements_positions) + self.matched_output_elements_positions = matched_output_elements_positions + self.flat_results = flat_results + + def placeholder( + self, target: Target, args: tuple[Argument, ...], kwargs: dict[str, Any] + ) -> Any: + arg = next(self.old_args_gen) + if "val" in self.current_node.meta: + arg.node.meta["val"] = self.current_node.meta["val"] + if "tensor_dict" in self.current_node.meta: + arg.node.meta["tensor_dict"] = self.current_node.meta["tensor_dict"] + if "example_value" in self.current_node.meta: + # NB: intentionally do not use set_example_value + arg.node.meta["example_value"] = self.current_node.meta["example_value"] + if "unbacked_bindings" in self.current_node.meta: + arg.node.meta["unbacked_bindings"] = self.current_node.meta[ + "unbacked_bindings" + ] + return arg + + def output( + self, target: Target, args: tuple[Argument, ...], kwargs: dict[str, Any] + ) -> Any: + dynamo_result_flat = args[0] + lookup = [*dynamo_result_flat, *self.new_args] # type: ignore[misc] + new_results_flat = [] + for i in range(len(self.flat_results)): + if self.matched_output_elements_positions[i] is not None: + new_results_flat.append( + lookup[self.matched_output_elements_positions[i]] + ) + else: + const_val = self.flat_results[i] + assert isinstance(const_val, tuple(common_constant_types)) + new_results_flat.append(const_val) + return super().output(target, (new_results_flat,), {}) + + def run_node(self, n: Node) -> Any: + self.current_node = n + result_proxy = super().run_node(n) + if "val" in self.current_node.meta: + result_proxy.node.meta["val"] = self.current_node.meta["val"] + if "example_value" in self.current_node.meta: + # NB: intentionally do not use set_example_value + result_proxy.node.meta["example_value"] = self.current_node.meta[ + "example_value" + ] + if "unbacked_bindings" in self.current_node.meta: + result_proxy.node.meta["unbacked_bindings"] = self.current_node.meta[ + "unbacked_bindings" + ] + if self.current_node.op != "output": + result_proxy.node._rename( + getattr(self.current_node, "name", result_proxy.node.name) + ) + return result_proxy + + def transform(self) -> torch.fx.GraphModule: + result_gm = super().transform() + if "dynamo_flat_name_to_original_fqn" in self.module.meta: # type: ignore[operator] + result_gm.meta["dynamo_flat_name_to_original_fqn"] = self.module.meta[ # type: ignore[index] + "dynamo_flat_name_to_original_fqn" # type: ignore[index] + ] + if "dynamo_compile_id" in self.module.meta: # type: ignore[operator] + result_gm.meta["dynamo_compile_id"] = self.module.meta["dynamo_compile_id"] # type: ignore[index] + return result_gm + + +class ExportResult(NamedTuple): + graph_module: torch.fx.GraphModule + guards: _guards.GuardsSet + # NB: Do not add new fields without overriding __iter__; people are + # destructuring so it is BC-breaking + + +# NOTE: this function only supports graphs created by Dynamo's OutputGraph module +def check_signature_rewritable(graph: torch.fx.GraphModule) -> None: + input_errors = [] + for node in graph.graph.find_nodes(op="placeholder"): + # set in OutputGraph._call_user_compiler + assert hasattr(node, "_dynamo_source") + assert hasattr(graph, "_source_to_user_stacks") + + # NOTE: We can safely ignore these type warnings if and only if + # the function is made from OutputGraph (checked in the assertions) + source = node._dynamo_source # type: ignore[attr-defined] + user_stacks = graph._source_to_user_stacks.get(source) # type: ignore[operator, union-attr] + if user_stacks is None: + continue + assert len(user_stacks) > 0 + # In some cases we may not have a useful stack. Look for a + # useful stack + stack = None + for s in user_stacks: + if len(s) == 0: + continue + stack = s + break + if stack is None: + msg = f"{source.name}, a closed over free variable" + else: + tb = "".join(traceback.format_list(stack)) + extra = "" + if len(user_stacks) > 1: + extra = f"(elided {len(user_stacks) - 1} more accesses)" + msg = f"{source.name}, accessed at:\n{tb}{extra}" + # TODO: option to print ALL of the stack traces at once + input_errors.append(msg) + + if input_errors: + raise UserError( + UserErrorType.INVALID_INPUT, + "Cannot export model which references tensors that are neither " + "buffers/parameters/constants nor are direct inputs. For each tensor, if you'd " + "like this tensor to be an explicit input, add it as a dummy argument " + "to the top-level model definition you are exporting; if you would " + "like its value to be embedded as an exported constant, wrap its access " + "in a function marked with @assume_constant_result.\n\n" + + "\n\n".join(input_errors), + ) + + +def check_user_input_output(flat_values: list[Any], error_type: UserErrorType) -> None: + supported_types = [ + torch.Tensor, + torch.SymInt, + torch.SymFloat, + torch.SymBool, + torch._C.ScriptObject, + _IntWrapper, + ] + list(common_constant_types) + + def is_supported_type(val: Any) -> bool: + return isinstance(val, tuple(supported_types)) + + value_type = "input" if error_type == UserErrorType.INVALID_INPUT else "output" + # We only check that the outputs are not None. Inputs can be None. + for v in flat_values: + if not is_supported_type(v): + if error_type == UserErrorType.INVALID_INPUT and v is None: + continue + + raise UserError( + error_type, + f"It looks like one of the {value_type}s with type `{type(v)}` " + "is not supported or pytree-flattenable. \n" + f"Exported graphs {value_type}s can only contain the " + f"following supported types: {supported_types}. \n" + "If you are using a custom class object, " + "please register a pytree_flatten/unflatten function " + "using `torch.utils._pytree.register_pytree_node` or " + "`torch.export.register_dataclass`.", + ) + + +def rewrite_signature( + f_sig: inspect.Signature, + graph: torch.fx.GraphModule, + fake_mode: Optional[fake_tensor.FakeTensorMode], + flat_args: list[Any], + in_spec: pytree.TreeSpec, + example_fake_inputs: list[Any], + graph_captured_input: Iterable[Any], + graph_captured_output: Optional[Iterable[Any]], + dynamo_traced_result: Any, + flat_args_dynamic_dims: list[set[int]], +) -> torch.fx.GraphModule: + orig_args, orig_kwargs = pytree.tree_unflatten(flat_args, in_spec) + + check_user_input_output(flat_args, UserErrorType.INVALID_INPUT) + flat_results_traced, out_spec_traced = pytree.tree_flatten(dynamo_traced_result) + check_user_input_output(flat_results_traced, UserErrorType.INVALID_OUTPUT) + + def check_optional_input_and_error(f_sig: inspect.Signature) -> None: + # Check if function has optional input. + for name, param in f_sig.parameters.items(): + if param.default is not inspect.Parameter.empty: + import torch._dynamo.graph_break_hints as graph_break_hints + from torch._dynamo.exc import unimplemented + + log.error( + "Parameter %s is optional with a default value of %s", + name, + param.default, + ) + unimplemented( + gb_type="rewrite_signature: cannot trace optional function input", + context="", + explanation=f"Parameter {name} is optional with a default value of {param.default}. This is not supported yet.", + hints=[ + *graph_break_hints.SUPPORTABLE, + ], + ) + + def produce_matching( + debug_type: str, sources: Iterable[Any], candidates: Iterable[Any] + ) -> list[Optional[int]]: + matched_elements_positions: list[Optional[int]] = [] + dict_of_source_vals = {} + for i, val in enumerate(sources): + dict_of_source_vals[id(val)] = i + + for val in candidates: + if isinstance(val, tuple(common_constant_types)): + matched_elements_positions.append(None) + elif id(val) not in dict_of_source_vals: + if debug_type == "inputs": + check_optional_input_and_error(f_sig) + raise AssertionError( + f"Unexpectedly found a {type(val)} in the {debug_type}.\n" + 'Please file an issue along with a paste of the logs from TORCH_LOGS="+export"', + ) + else: + matched_elements_positions.append(dict_of_source_vals[id(val)]) + + return matched_elements_positions + + matched_input_elements_positions = produce_matching( + "inputs", flat_args, graph_captured_input + ) + + assert graph_captured_output is not None + matched_output_elements_positions = produce_matching( + "outputs", list(graph_captured_output) + flat_args, flat_results_traced + ) + + new_graph = FlattenInputOutputSignature( + graph, + flat_args, + matched_input_elements_positions, # type: ignore[arg-type] + flat_results_traced, + matched_output_elements_positions, # type: ignore[arg-type] + example_fake_inputs, + flat_args_dynamic_dims, + fake_mode, + ).transform() + + new_graph.graph._codegen = _PyTreeCodeGen( + _PyTreeInfo( + argument_names(f_sig, orig_args, orig_kwargs), + in_spec, + out_spec_traced, + ) + ) + new_graph.recompile() + return new_graph + + +def export( + f: Callable[..., Any], + *extra_args: Any, + aten_graph: bool = False, + pre_dispatch: bool = False, + decomposition_table: Optional[ + dict[torch._ops.OpOverload, Callable[..., Any]] + ] = None, + tracing_mode: str = "symbolic", + dynamic_shapes: Optional[Union[dict[str, Any], tuple[Any], list[Any]]] = None, + specialize_float: bool = True, + assume_static_by_default: bool = False, + same_signature: bool = True, + disable_constraint_solver: bool = False, + prefer_deferred_runtime_asserts_over_guards: bool = False, + _log_export_usage: bool = True, + constraints: Optional[list[Constraint]] = None, + **extra_kwargs: Any, +) -> Callable[..., ExportResult]: + """ + Export an input function f to a format that can be executed outside of PyTorch using the FX graph. + + Args: + f (callable): A PyTorch function to be exported. + + aten_graph (bool): If True, exports a graph with ATen operators. + If False, exports a graph with Python operators. Default is False. + + pre_dispatch (bool): If True, exports a graph with ATen operators, + but before any logic in the PyTorch dispatcher has run. + This can be useful if you want to apply further transformations on a graph before running it + through autograd, autocast, or any other functionalities that are integrated into the dispatcher. + This flag is only valid if aten_graph=True is set. + Default is False. + + decomposition_table (dict): A dictionary that maps operators to their decomposition functions. + Required if aten_graph or tracing_mode is specified. Default is None. + + tracing_mode (str): If "symbolic", turn on dynamic shapes support. Default is "symbolic". + + dynamic_shapes: + An optional argument where the type should either be: + 1) a dict from argument names of ``f`` to their dynamic shape specifications, + 2) a tuple that specifies dynamic shape specifications for each input in original order. + If you are specifying dynamism on keyword args, you will need to pass them in the order that + is defined in the original function signature. + + The dynamic shape of a tensor argument can be specified as either + (1) a dict from dynamic dimension indices to :func:`Dim` types, where it is + not required to include static dimension indices in this dict, but when they are, + they should be mapped to None; or (2) a tuple / list of :func:`Dim` types or None, + where the :func:`Dim` types correspond to dynamic dimensions, and static dimensions + are denoted by None. Arguments that are dicts or tuples / lists of tensors are + recursively specified by using mappings or sequences of contained specifications. + + same_signature (bool): If True, rewrite the returned graph's signature to be the same as f. + + disable_constraint_solver (bool): Whether the dim constraint solver must be disabled. + + Returns: + A function that given args and kwargs, returns a tuple of (graph, guards) + Graph: An FX graph representing the execution of the input PyTorch function with the provided arguments and options. + Guards: The guards we accumulated during tracing f above + + Raises: + AssertionError: If decomposition_table is specified without setting aten_graph=True, + or if graph breaks during tracing in export. + + AssertionError: If Dynamo input and output is not consistent with traced input/output. + + Note - this headerdoc was authored by ChatGPT, with slight modifications by the author. + """ + if config.debug_force_graph_break_on_leaf_return: + raise unittest.SkipTest("Cannot force graph break on export") + + if _log_export_usage: + log_export_usage(event="export.private_api", flags={"_dynamo"}) + + # Deal with "local variable referenced before assignment" + _f = f + _specialize_float = specialize_float + _assume_static_by_default = assume_static_by_default + _constraints = constraints + + def inner(*args: Any, **kwargs: Any) -> ExportResult: + if not _constraints: + combined_args = _combine_args(_f, args, kwargs) + constraints = _process_dynamic_shapes(combined_args, dynamic_shapes) + else: + constraints = _constraints + + f = _f + specialize_float = _specialize_float + assume_static_by_default = _assume_static_by_default + check_if_dynamo_supported() + torch._C._log_api_usage_once("torch._dynamo.export") + if decomposition_table is not None: + assert aten_graph, ( + "Specifying a decomposition_table table or tracing mode is illegal without setting aten_graph=True" + ) + if pre_dispatch: + assert aten_graph, "pre_dispatch=True can only be used when aten_graph=True" + f = innermost_fn(f) + call_to_inspect = f.forward if isinstance(f, torch.nn.Module) else f + original_signature = inspect.signature(call_to_inspect) # type: ignore[arg-type] + graph = None + out_guards = None + graph_captured_input = None + graph_captured_result: Optional[tuple[torch.Tensor, ...]] = None + fake_mode = None + result_traced = None + + def guard_export_print(guards: _guards.GuardsSet) -> None: + nonlocal out_guards + assert out_guards is None, ( + "whole graph export entails exactly one guard export" + ) + out_guards = guards + + example_inputs: list[Any] = [] + + def dynamo_normalization_capturing_compiler( + gm: torch.fx.GraphModule, inner_example_inputs: list[Any] + ) -> Callable[..., Any]: + nonlocal graph + assert graph is None, ( + "Tried to emit a second graph during export. Tracing through 'f' must produce a single graph." + ) + graph = gm + + nonlocal fake_mode, example_inputs + # NB: do NOT pass inner_example_inputs here, we are detecting the + # Dynamo allocated fake mode, which should be DISTINCT from a + # potential outer ambient fake mode which the user provided. + # example_inputs is always the user specified inputs, so they + # would have the wrong fake mode attached to them + fake_mode = _guards.detect_fake_mode() + example_inputs = inner_example_inputs + + def result_capturing_wrapper(*graph_inputs: Any) -> Any: + nonlocal graph_captured_result + nonlocal graph_captured_input + + graph_captured_input = graph_inputs + assert graph is not None + + named_parameters = dict(graph.named_parameters(remove_duplicate=False)) + named_buffers = dict(graph.named_buffers(remove_duplicate=False)) + + ambient_fake_mode = ( + _guards.detect_fake_mode(graph_inputs) + if _guards.detect_fake_mode(graph_inputs) is not None + else fake_mode + ) + + # We reran fake tensor propagation, but we didn't do + # anything with the resulting unbacked SymInts. Drop them + # from the pending list. + # NB: this is wrong if graph_captured_result has + # data-dependent output size! + ignore_fresh_unbacked = null_context() + assert ambient_fake_mode is not None + if shape_env := ambient_fake_mode.shape_env: + ignore_fresh_unbacked = shape_env.ignore_fresh_unbacked_symbols() # type: ignore[assignment] + + with ( + ambient_fake_mode, + enable_python_dispatcher(), + ignore_fresh_unbacked, + ): + params_and_buffers = { + **named_parameters, + **named_buffers, + } + fake_params_buffers = {} + + for name, value in params_and_buffers.items(): + fake_params_buffers[name] = ambient_fake_mode.from_tensor( + value, static_shapes=True + ) + + from torch._export.non_strict_utils import ( + key_path_to_source, + KeyPath, + ) + + def fakify_with_ambient( + path: KeyPath, t: Union[torch.Tensor, _IntWrapper, Any] + ) -> Any: + if isinstance(t, torch.Tensor): + # pyrefly: ignore [missing-attribute] + return ambient_fake_mode.from_tensor(t, static_shapes=True) + elif isinstance(t, _IntWrapper): + if ( + t.dynamism is not None + and isinstance(t.dynamism, _DimHint) + and t.dynamism.type + in ( + _DimHintType.DYNAMIC, + _DimHintType.AUTO, + ) + ): # type: ignore[union-attr] + source = key_path_to_source(path) + symint = ambient_fake_mode.shape_env.create_unspecified_symint_and_symbol( # type: ignore[union-attr] + t.val, source, DimDynamic.DYNAMIC + ) + return symint + else: + return t.val + else: + return t + + fake_graph_inputs = pytree.tree_map_with_path( + fakify_with_ambient, graph_inputs + ) + graph_captured_result = torch.func.functional_call( + graph, + fake_params_buffers, # type: ignore[arg-type] + fake_graph_inputs, # type: ignore[arg-type] + ) + + return graph_captured_result + + return result_capturing_wrapper + + # Note: This is needed by rewrite_signature. We need to put it before + # optimize_assert since user program may mutate the inputs. + flat_args, in_spec = pytree.tree_flatten((args, kwargs)) + + remove_from_cache(f) + constraint_violation_error = None + if tracing_mode != "symbolic": + assume_static_by_default = True + with ( + config.patch( + specialize_int=True, + specialize_float=specialize_float, + assume_static_by_default=assume_static_by_default, + automatic_dynamic_shapes=False, + capture_dynamic_output_shape_ops=True, + capture_scalar_outputs=True, + constant_fold_autograd_profiler_enabled=True, + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, + # install_free_tensors ensures that params and buffers are still + # added as graph attributes, and makes Dynamo emits graphs that + # follow export pytree-able input requirements + install_free_tensors=config.install_free_tensors_for_export, + ), + _compiling_state_context(), + ): + opt_f = optimize_assert( + dynamo_normalization_capturing_compiler, + hooks=Hooks( + guard_export_fn=guard_export_print, + guard_fail_fn=None, + ), + export=True, + export_constraints=constraints, + )(f) + # TODO(voz): We may have instances of `f` that mutate inputs, we should track sideeffects and reject. + try: + result_traced = opt_f(*args, **kwargs) + except ConstraintViolationError as e: + constraint_violation_error = e + remove_from_cache(f) + + if ( + not disable_constraint_solver + and (shape_env := getattr(fake_mode, "shape_env", None)) is not None + and (dim_constraints := shape_env.dim_constraints) is not None + and not isinstance( + call_to_inspect, (torch._ops.OpOverloadPacket, torch._ops.OpOverload) + ) + and not trace_rules.check(call_to_inspect) + ): + dim_constraints.solve() + + forced_specializations = dim_constraints.forced_specializations() + + msg = dim_constraints.prettify_results( + original_signature, + dynamic_shapes, + constraint_violation_error, + forced_specializations, + ) + if constraint_violation_error: + constraint_violation_error.args = ( + constraint_violation_error.args[0] + msg, + ) + else: + if forced_specializations: + constraint_violation_error = ConstraintViolationError(msg) + else: + log.info( + "Summary of dimension constraints:%s", + msg, + ) + + # Error if we have any constraints on static values + + for k in shape_env.var_to_range: + if isinstance(k, sympy.Integer): + constraint_violation_error = ConstraintViolationError( + f"{''.join(traceback.format_list(shape_env.var_to_stack[k]))}\n" + "It appears that you're trying to set a constraint on a " + f"value which we evaluated to have a static value of {k}. " + 'Set TORCH_LOGS="+export" for more information.' + ) + if constraint_violation_error: + raise constraint_violation_error + + if graph is None: + assert same_signature, ( + "Failed to produce a graph during tracing as no tensor operations were found and same_signature is False." + ) + # If the module does not contain any tensor computation, we would create a graph with inputs and outputs. + # To be consistent with the graph traced by dynano, `graph` will have only tensor inputs as placeholders + # and tensor outputs as output nodes. non-tensor inputs and outputs will be added when rewriting signature. + # We will also construct the `example_inputs`, `graph_captured_input`, and `graph_captured_result` corresponding + # to `graph`. + example_inputs = [] + graph_captured_input = () + graph_captured_result = () + fake_mode = torch._subclasses.FakeTensorMode( + shape_env=ShapeEnv(), export=True + ) + if out_guards is None: + out_guards = _guards.GuardsSet() + assert out_guards is not None # suppress mypy error + parameter_names = list(original_signature.parameters.keys()) + fx_graph = torch.fx.Graph() + for i, name in enumerate(parameter_names): + if torch.is_tensor(flat_args[i]): + node = fx_graph.placeholder(name) + node.meta["val"] = fake_mode.from_tensor( + flat_args[i], static_shapes=True + ) + graph_captured_input = graph_captured_input + (flat_args[i],) + example_inputs.append(flat_args[i]) + fx_graph.output(graph_captured_result) + module = torch.nn.Module() + graph = torch.fx.GraphModule(module, fx_graph) + log.info( + "Failed to capture a graph during tracing as no tensor operations were found.:\n\n%s", + graph.print_readable(print_output=False, colored=True), + ) + else: + assert out_guards is not None, "Failed to produce guards during tracing" + assert fake_mode is not None + + log.info( + "Dynamo captured graph:\n\n%s", + graph.print_readable(print_output=False, colored=True), + ) + + # This check need to happened before aten_graph + # because placeholder's _source_node attribute is not preserved by make_fx + if same_signature: + check_signature_rewritable(graph) + + # NB: This is mostly hitting the cache; Dynamo already converted these + example_fake_inputs = [ + fake_mode.from_tensor(t) if isinstance(t, torch.Tensor) else t + for t in example_inputs + ] + + if aten_graph: + # Running graph with interpreter is needed for propagating the stack_trace + def graph_with_interpreter(*args: Any) -> Any: + with torch.fx.traceback.preserve_node_meta(): + return torch.fx.Interpreter(graph).run(*args) # type: ignore[arg-type] + + with unset_fake_temporarily(), enable_python_dispatcher(), fake_mode: + try: + graph = make_fx( + graph_with_interpreter, + decomposition_table=decomposition_table, + tracing_mode="real", + _allow_non_fake_inputs=True, + pre_dispatch=pre_dispatch, + _allow_fake_constant=False, + )(*example_fake_inputs) + except CondOpArgsMismatchError as e: + # Wrap the internal error to the user-facing error + raise UserError( # noqa: B904 + UserErrorType.DYNAMIC_CONTROL_FLOW, + str(e), + case_name="cond_operands", + ) + + assert graph is not None + for node in graph.graph.find_nodes(op="get_attr"): + if isinstance(getattr(graph, node.target), torch.Tensor): # type: ignore[arg-type] + node.meta["val"] = fake_mode.from_tensor( + getattr(graph, node.target), # type: ignore[arg-type] + static_shapes=True, + ) + + if same_signature: + flat_args_dynamic_dims = [ + { + c.dim + for c in (constraints or ()) + if ( + c.t_id == id(x) + and not isinstance(c, _RelaxedConstraint) + and c.constraint_range.vr.lower != c.constraint_range.vr.upper + ) + } + for x in flat_args + ] + graph = rewrite_signature( + original_signature, + graph, + fake_mode, + flat_args, + in_spec, + example_fake_inputs, + graph_captured_input, # type: ignore[arg-type] + graph_captured_result, + result_traced, # type: ignore[possibly-undefined] + flat_args_dynamic_dims, + ) + return ExportResult(graph, out_guards) + + if extra_args or extra_kwargs: + warnings.warn( + "export(f, *args, **kwargs) is deprecated, use export(f)(*args, **kwargs) instead. " + "If you don't migrate, we may break your export call in the future if your user defined kwargs " + "conflict with future kwargs added to export(f).", + FutureWarning, + stacklevel=2, + ) + return inner(*extra_args, **extra_kwargs) # type: ignore[return-value] + else: + return inner + + +def optimize_assert(*args: Any, **kwargs: Any) -> OptimizeContext: + if "rebuild_ctx" in kwargs and kwargs["rebuild_ctx"] is not None: + # called from optimize + rebuild_ctx = kwargs["rebuild_ctx"] + del kwargs["rebuild_ctx"] + else: + + def rebuild_ctx() -> OptimizeContext: + return optimize_assert(*args, **kwargs) + + return _optimize_assert(rebuild_ctx, *args, **kwargs) + + +def _optimize_assert( + rebuild_ctx: Callable[[], OptimizeContext], + backend: Union[str, Callable[..., Any], None], + *, + hooks: Hooks = Hooks(None, None, None), + export: bool = False, + export_constraints: Optional[Any] = None, + dynamic: Optional[bool] = None, + package: Optional[CompilePackage] = None, +) -> OptimizeContext: + """ + Guarantees single-graph capture. + The same as `torch._dynamo.optimize(backend)` but ignores + symbolic_convert.error_on_graph_break setting. + + Used for fullgraph=True and export, since we must always error on graph breaks and ignore + symbolic_convert.error_on_graph_break. Can also be used for testing. + """ + backend = get_compiler_fn(backend) + + # Find if backend has any extra context manager + backend_ctx_ctor = getattr(backend, "backend_ctx_ctor", null_context) + + if config.caching_precompile and package is None: + # Create an uninitialized package that will be set/filled by + # _OptimizeContext.__call__ + # We need to instantiate the object here because the same CompilePackage + # needs to be shared between convert_frame_assert + # and OptimizeContext. + from .package import CompilePackage + + package = CompilePackage(fn=None, dynamo=None, ignore_inlined_sources=False) + + return _optimize_catch_errors( + convert_frame.convert_frame_assert( + backend, + export=export, + export_constraints=export_constraints, + package=package, + ), + hooks, + backend_ctx_ctor, + fullgraph=True, + export=export, + dynamic=dynamic, + rebuild_ctx=rebuild_ctx, + package=package, + ) + + +class TorchPatcher: + @staticmethod + @functools.cache + def patch() -> None: + # A better way to disable the following would be decorate the source + # functions with @torch._disable_dynamo. However, this causes issues + # with torch.deploy internally. + from .decorators import disable + + torch.jit.trace = disable( + torch.jit.trace, reason="tracing into TorchScript not fully supported" + ) + torch.jit.trace_module = disable( + torch.jit.trace_module, + reason="tracing into TorchScript not fully supported", + ) + torch.jit._get_trace_graph = disable( + torch.jit._get_trace_graph, + reason="tracing into TorchScript not fully supported", + ) + torch.fx._symbolic_trace.Tracer.trace = disable( + torch.fx._symbolic_trace.Tracer.trace, + reason="tracing into FX not fully supported", + ) + torch.distributions.Distribution.set_default_validate_args(False) + + from torch.optim import ( + adadelta, + adagrad, + adam, + adamax, + adamw, + asgd, + lbfgs, + nadam, + radam, + rmsprop, + rprop, + sgd, + sparse_adam, + ) + + optimizer_modules = { + adadelta, + adagrad, + adam, + adamax, + adamw, + asgd, + lbfgs, + nadam, + radam, + rmsprop, + rprop, + sgd, + sparse_adam, + } + + for opt_mod in optimizer_modules: + opt_name = opt_mod.__name__.split(".")[-1] + fused_fn_name = f"_fused_{opt_name}" + + if hasattr(opt_mod, fused_fn_name): + setattr( + opt_mod, + fused_fn_name, + disable( + getattr(opt_mod, fused_fn_name), + reason="don't trace into fused optimizer", + ), + ) + + optimizer_classes = [ + opt + for opt in torch.optim.__dict__.values() + if inspect.isclass(opt) and issubclass(opt, torch.optim.Optimizer) + ] + + # Note: we don't support sparsity or tracing through backwards + excluded_optimizer_classes = { + torch.optim.SparseAdam, + torch.optim.LBFGS, + } + + for opt in optimizer_classes: + if opt in excluded_optimizer_classes: + opt.step = disable( + opt.step, reason=f"optimizer {opt} step not supported" + ) + + if hasattr(opt, "_init_group"): + opt._init_group = disable( + opt._init_group, reason=f"optimizer {opt} _init_group not supported" + ) + + @staticmethod + def suppress_torch_distributed_warnings( + fn: Callable[..., Any], + ) -> Callable[..., Any]: + def inner_fn(*args: Any, **kwargs: Any) -> Any: + with torch._logging.hide_warnings( + torch._logging._internal.user_warning_filter + ): + return fn(*args, **kwargs) + + return inner_fn + + +def skip_code(code: types.CodeType) -> None: + set_code_exec_strategy( + code, FrameExecStrategy(FrameAction.SKIP, FrameAction.DEFAULT) + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/exc.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/exc.py new file mode 100644 index 0000000000000000000000000000000000000000..a7bdf1caff2415261c70f9e66da731741b56a3d6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/exc.py @@ -0,0 +1,827 @@ +from __future__ import annotations + + +"""Exception handling and error reporting for TorchDynamo. + +This module provides a comprehensive set of exception classes and utilities for error +handling in TorchDynamo. It includes: + +Base Exceptions: + - TorchDynamoException: Base class for all TorchDynamo-specific exceptions + - Various specialized subclasses for different error scenarios + +User Error Handling: + - UserError: Exceptions for user-facing errors in TorchDynamo usage + - UserErrorType: Enumeration of different categories of user errors + - Formatted error messages with debugging information + +Observed Exceptions: + - Classes for handling exceptions observed during tracing + - Special handling for StopIteration, LookupError, etc. + - Exception state management during compilation + +Error Formatting: + - Stack trace filtering and formatting + - Error message augmentation + - Debugging utilities for error reporting +""" + +import json +import logging +import re +import textwrap +import typing +from enum import auto, Enum +from functools import lru_cache +from pathlib import Path +from traceback import extract_stack, format_exc, format_list, StackSummary +from typing import Any, NoReturn, Optional, TYPE_CHECKING + +import torch._guards +from torch._utils_internal import get_file_path_2 + +from . import config +from .utils import counters + + +if TYPE_CHECKING: + import types + + from torch._guards import CompileId + + from .output_graph import DynamoTracerOutput + from .symbolic_convert import InstructionTranslatorBase + from .types import DynamoFrameType + + +def exportdb_error_message(case_name: str) -> str: + return ( + "For more information about this error, see: " + + "https://pytorch.org/docs/main/generated/exportdb/index.html#" + + case_name.replace("_", "-") + ) + + +log = logging.getLogger(__name__) +graph_breaks_log = torch._logging.getArtifactLogger(__name__, "graph_breaks") + + +class TorchDynamoException(RuntimeError): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._torch_dynamo_tracer_output: Optional[DynamoTracerOutput] = None + + +class InternalTorchDynamoError(TorchDynamoException): + pass + + +class ResumePrologueTracingError(TorchDynamoException): + pass + + +class RestartAnalysis(TorchDynamoException): + restart_reason: Optional[str] + + def __init__(self, *args: Any, restart_reason: Optional[str] = None) -> None: + self.restart_reason = restart_reason + super().__init__(*args) + + +class SpeculationRestartAnalysis(RestartAnalysis): + pass + + +class UnspecializeRestartAnalysis(RestartAnalysis): + pass + + +class CompileCollectiveRestartAnalysis(RestartAnalysis): + pass + + +class TensorifyScalarRestartAnalysis(RestartAnalysis): + pass + + +class SkipFrame(TorchDynamoException): + pass + + +class TorchRuntimeError(TorchDynamoException): + pass + + +class InvalidBackend(TorchDynamoException): + def __init__(self, name: str) -> None: + super().__init__( + f"Invalid backend: {name!r}, see `torch._dynamo.list_backends()` for available backends." + ) + + +class ResetRequired(TorchDynamoException): + def __init__(self) -> None: + super().__init__( + textwrap.dedent( + """ + Must call `torch._dynamo.reset()` before changing backends. Detected two calls to + `torch.compile()` with a different backend compiler arguments. + """ + ) + ) + + +class ShortenTraceback(TorchDynamoException): + def __init__( + self, *args: Any, first_useful_frame: Optional[types.FrameType], **kwargs: Any + ) -> None: + super().__init__(*args, **kwargs) + self.first_useful_frame = first_useful_frame + + def remove_dynamo_frames(self) -> typing.Self: + tb = self.__traceback__ + if self.first_useful_frame is None or tb is None or config.verbose: + return self + while tb.tb_frame is not self.first_useful_frame: + tb = tb.tb_next + assert tb is not None, "internal error, please report a bug" + return self.with_traceback(tb) + + +class BackendCompilerFailed(ShortenTraceback): + def __init__( + self, + backend_fn: Any, + inner_exception: Exception, + first_useful_frame: Optional[types.FrameType], + ) -> None: + self.backend_name = getattr(backend_fn, "__name__", "?") + self.inner_exception = inner_exception + msg = f"backend={self.backend_name!r} raised:\n{type(inner_exception).__name__}: {inner_exception}" + super().__init__(msg, first_useful_frame=first_useful_frame) + + +class Unsupported(TorchDynamoException): + def __init__( + self, + msg: str, + *, + case_name: Optional[str] = None, + real_stack: None | StackSummary = None, + ) -> None: + super().__init__(msg) + if not real_stack: + real_stack = torch._guards.TracingContext.extract_stack() + self.real_stack = real_stack + self.msg = msg + self.category: Optional[str] = None + self.add_to_stats() + self.case_name: Optional[str] = case_name + + def remove_from_stats(self) -> None: + assert self.category is not None + counters[self.category][self.msg] -= 1 + if counters[self.category][self.msg] <= 0: + del counters[self.category][self.msg] + + def add_to_stats(self, category: str = "unimplemented") -> None: + self.category = category + counters[category][self.msg] += 1 + + +class UnknownPropertiesDuringBackwardTrace(Unsupported): + pass + + +class RecompileError(TorchDynamoException): + pass + + +class ArgsMismatchError(Unsupported): + pass + + +class AttributeMutationError(Unsupported): + pass + + +class InfiniteGeneratorError(Unsupported): + # Raised when the number of yielded values is greater than MAX_ITERATOR_LIMIT + pass + + +class SideEffectsError(Unsupported): + pass + + +class CondOpArgsMismatchError(ArgsMismatchError): + """ + Internal error from cond() due to arguments mismatch. + """ + + +class UserErrorType(Enum): + DYNAMIC_CONTROL_FLOW = auto() + ANTI_PATTERN = auto() + STANDARD_LIBRARY = auto() + CONSTRAINT_VIOLATION = auto() + DYNAMIC_DIM = auto() + INVALID_INPUT = auto() + INVALID_OUTPUT = auto() + UNSUPPORTED_ALIASED_MUTATED_DYNAMIC_INPUTS = auto() + + +class UserError(Unsupported): + def __init__( + self, error_type: UserErrorType, msg: str, case_name: Optional[str] = None + ) -> None: + """ + Type of errors that would be valid in Eager, but not supported in TorchDynamo. + The error message should tell user about next actions. + + error_type: Type of user error + msg: Actionable error message + case_name: (Optional) Unique name (snake case) for the usage example in exportdb. + """ + if case_name is not None: + assert isinstance(case_name, str) + if msg.endswith("."): + msg += " " + else: + msg += "\n" + msg += exportdb_error_message(case_name) + super().__init__(msg) + self.error_type = error_type + self.message = msg + + +class SkipCodeRecursiveException(TorchDynamoException): + pass + + +class RecompileLimitExceeded(Unsupported): + pass + + +# debug exception thrown when tracing torch._dynamo.step_unsupported() +class StepUnsupported(TorchDynamoException): + def __init__(self) -> None: + self.real_stack = torch._guards.TracingContext.extract_stack() + + +class UnsafeScriptObjectError(TorchDynamoException): + pass + + +class UncapturedHigherOrderOpError(TorchDynamoException): + def __init__(self, msg: str, real_stack: Optional[StackSummary] = None) -> None: + super().__init__(msg) + self.msg = msg + self.real_stack = ( + real_stack + if real_stack is not None + else torch._guards.TracingContext.extract_stack() + ) + + +class IncorrectUsage(Exception): + pass + + +# TODO: I'm a little uncertain about what error classification we should have +# for this. This is potentially a user error, but regressions in +# specialization in PyTorch proper could also trigger this problem +class FailOnRecompileLimitHit(Exception): + pass + + +class PackageError(TorchDynamoException): + pass + + +class ObservedException(TorchDynamoException): + # An exception observed during the tracing. This exception is used by Dynamo to handle exceptions. + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.real_stack: StackSummary = torch._guards.TracingContext.extract_stack() + + +class ObservedUserStopIteration(ObservedException): + # An UserStopIteration exception observed during the Dynamo tracing (e.g Dynamo tracing __next__) + value: Optional[Any] + + # Reference `StopIteration_init` in CPython + # https://github.com/python/cpython/blob/3.11/Objects/exceptions.c#L568-L584 + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__("unhandled `raise StopIteration`") + if len(args) > 0: + self.value = args[0] + else: + self.value = None + + +class ObservedLookupError(ObservedException): + # A LookupError exception to be raised from inside Dynamo tracing. This can happen on __getitem__ + pass + + +class ObservedIndexError(ObservedLookupError): + # An IndexError exception to be raised from inside Dynamo tracing. This can happen on list __getitem__ + pass + + +class ObservedKeyError(ObservedLookupError): + # A KeyError exception to be raised from inside Dynamo tracing. This can happen on dict __getitem__ + pass + + +class ObservedGeneratorExit(ObservedException): + pass + + +class ObservedAttributeError(ObservedException): + # An AttributeError exception to be raised from inside Dynamo tracing. This can happen on user defined object __getattr__ + pass + + +class ObservedRuntimeError(ObservedException): + # A RuntimeError exception to be raised from inside Dynamo tracing. This can happen on generator.throw(..) method + pass + + +class ObservedNotImplementedError(ObservedException): + pass + + +class ObservedTypeError(ObservedException): + # A TypeError exception to be raised from inside Dynamo tracing. This can happen on generator.send(..) method + pass + + +observed_exception_map = { + StopIteration: ObservedUserStopIteration, + LookupError: ObservedLookupError, + IndexError: ObservedIndexError, + GeneratorExit: ObservedGeneratorExit, + KeyError: ObservedKeyError, + AttributeError: ObservedAttributeError, + RuntimeError: ObservedRuntimeError, + NotImplementedError: ObservedNotImplementedError, + TypeError: ObservedTypeError, +} + + +def get_dynamo_observed_exception(exc_type: type[Exception]) -> type[ObservedException]: + if exc_type not in observed_exception_map: + name = getattr(exc_type, "__name__", str(exc_type)) + observed_exception_map[exc_type] = type( # type: ignore[assignment] + f"Observed{name}Error", (ObservedException,), {} + ) + # pyrefly: ignore [index-error] + return observed_exception_map[exc_type] + + +def raise_observed_exception( + exc_type: type[Exception], + tx: InstructionTranslatorBase, + *, + args: Optional[list[Any]] = None, + kwargs: Optional[dict[str, Any]] = None, +) -> NoReturn: + from .variables import BuiltinVariable + + # CPython here raises an exception. Since there is no python code, we have to manually setup the exception + # stack and raise the exception. + exception_vt = BuiltinVariable(exc_type).call_function(tx, args or [], kwargs or {}) # type: ignore[arg-type] + tx.exn_vt_stack.set_current_exception(exception_vt) # type: ignore[arg-type] + raised_exc = get_dynamo_observed_exception(exc_type) + # Store the original exception arguments for better error messages + if args: + raise raised_exc(*args) + raise raised_exc + + +def handle_observed_exception(tx: Any) -> None: + # This is essentially exception handling code, equivalent of this pseudo code + # + # try: + # ... somebody raising StopIteration + # except StopIteration + # pass + # + # If this was going through the python code, we would have called exception_handler method, but FOR_ITER + # handles the exception completely in CPython. For example for 3.11, the resulting bytecode is + # + # + # 6 46 LOAD_GLOBAL 2 (StopIteration) + # 58 RAISE_VARARGS 1 + # >> 60 PUSH_EXC_INFO + + # 7 62 LOAD_GLOBAL 2 (StopIteration) + # 74 CHECK_EXC_MATCH + # 76 POP_JUMP_FORWARD_IF_FALSE 3 (to 84) + # 78 POP_TOP + + # 8 80 POP_EXCEPT + # + + # Fortunately this translates to a simple pop from the exn_vt_stack + tx.exn_vt_stack.clear_current_exception() + + +# These exceptions are ok to fallback to eager/graph_break. +exceptions_allowed_to_be_fallback = ( + torch._subclasses.fake_tensor.DataDependentOutputException, + torch._subclasses.fake_tensor.DynamicOutputShapeException, + torch._subclasses.fake_tensor.UnsupportedOperatorException, + torch._subclasses.fake_tensor.UnsupportedFakeTensorException, + torch._subclasses.fake_tensor.UnsupportedMutationAliasingException, +) + + +def unimplemented_with_warning( + e: Exception, + code: types.CodeType, + *, + gb_type: str, + context: str, + explanation: str, + hints: list[str], +) -> NoReturn: + # This function calls unimplemented internally and eventually graph breaks + # or falls to eager. unimplemented itself does not print any user warnings, + # i.e., its very silent. This helper function is intended when an error is + # encountered in the torch.compile stack which is worth showing as warning + # to the user. For example, if AOT Autograd backend fails with a fake tensor + # exception, its ok to fallback to eager but not silently. Here, we can use + # this function to log the message and the stack trace. + graph_break_msg = format_error_msg_verbose(e, code) + torch._logging.trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "dynamo_graph_break_reason", + "encoding": "string", + }, + payload_fn=lambda: graph_break_msg, + ) + graph_breaks_log.debug("%s", graph_break_msg) + _unimplemented = unimplemented + # to prevent a graph break registry entry + _unimplemented( + gb_type=gb_type, + context=context, + explanation=explanation, + hints=hints, + from_exc=e, + log_warning=True, + ) + + +def format_graph_break_message( + gb_type: str, + context: str, + explanation: str, + hints: list[str], +) -> str: + explanation = textwrap.indent(explanation, " ").lstrip() + hints_str = "\n".join( + " Hint: " + textwrap.indent(hint, " ").lstrip() for hint in hints + ) + context = textwrap.indent(context, " ").lstrip() + + msg = f"""\ +{gb_type} + Explanation: {explanation} +{hints_str} + + Developer debug context: {context} +""" + return msg + + +@lru_cache(maxsize=1) +def _load_gb_type_to_gb_id_map() -> dict[str, Any]: + """ + Loads the gb_type to gb_id map from the graph break registry from JSON file with caching. + + Includes historical gb_type (mapping behavior of duplicate gb_types with different gb_ids is undefined). + """ + try: + script_dir = Path(__file__).resolve().parent + registry_path = get_file_path_2( + "", str(script_dir), "graph_break_registry.json" + ) + with open(registry_path) as f: + registry = json.load(f) + except Exception: + log.exception("Error accessing the registry file") + registry = {} + + mapping = {} + for k, v in registry.items(): + for entry in v: + mapping[entry["Gb_type"]] = k + + return mapping + + +def get_gbid_documentation_link(gb_type: str) -> Optional[str]: + """ + Retrieves the GBID documentation link for a given graph break type. + + Args: + gb_type: The graph break type to look up. + + Returns: + A string containing the documentation URL if found, otherwise None. + """ + GRAPH_BREAK_SITE_URL = ( + "https://meta-pytorch.github.io/compile-graph-break-site/gb/" # @lint-ignore + ) + + gb_type_to_gb_id_map = _load_gb_type_to_gb_id_map() + + if gb_type in gb_type_to_gb_id_map: + return ( + f"{GRAPH_BREAK_SITE_URL}gb{gb_type_to_gb_id_map[gb_type].lstrip('GB')}.html" + ) + + return None + + +_NOTHING = object() + + +def unimplemented( + *, + gb_type: str, + context: str, + explanation: str, + hints: list[str], + from_exc: Any = _NOTHING, + log_warning: bool = False, +) -> NoReturn: + """ + Called within dynamo to cause a graph break. + Args: + gb_type: Context-free graph break type. It should be a short string without any + information specific to the tracing context (i.e. no dynamically-generated strings) + context: Developer context for the graph break. It can contain tracing context/dynamic strings. + explanation: User-facing context-dependent explanation for the graph break. Can be dynamic. + hints: List of user-facing hints for the graph break. + """ + + msg = format_graph_break_message(gb_type, context, explanation, hints) + + documentation_link = get_gbid_documentation_link(gb_type) + + if documentation_link: + msg += f"\n For more details about this graph break, please visit: {documentation_link}" + + if log_warning: + log.warning(msg) + if from_exc is not _NOTHING: + past_real_stack = None + if hasattr(from_exc, "real_stack"): + past_real_stack = from_exc.real_stack + raise Unsupported(msg, real_stack=past_real_stack) from from_exc + raise Unsupported(msg) + + +# KeyError has special handling for its args +# see https://github.com/python/cpython/blob/3.11/Objects/exceptions.c#L2534 for details +class KeyErrorMsg: + def __init__(self, value: Any) -> None: + self.value = value + + def __str__(self) -> str: + return str(self.value) + + def __repr__(self) -> str: + return self.__str__() + + +def augment_exc_message(exc: Exception, msg: str = "\n", export: bool = False) -> None: + import traceback + + exc.innermost_user_frame_summary = None # type: ignore[attr-defined] + + real_stack = get_real_stack(exc) + if real_stack is not None and len(real_stack) > 0: + exc.innermost_user_frame_summary = real_stack[-1] # type: ignore[attr-defined] + msg += f"\nfrom user code:\n {''.join(traceback.format_list(real_stack))}" + + if config.replay_record_enabled and hasattr(exc, "record_filename"): + msg += ( + f"\nLast frame execution written to {exc.record_filename}. To run only this frame while debugging, run\ + torch._dynamo.replay('{exc.record_filename}').\n" + ) + + if not config.verbose and hasattr(exc, "real_stack"): + msg += ( + "\nSet TORCHDYNAMO_VERBOSE=1 for the internal stack trace " + "(please do this especially if you're reporting a bug to PyTorch). " + 'For even more developer context, set TORCH_LOGS="+dynamo"\n' + ) + + if hasattr(exc, "inner_exception") and hasattr( + exc.inner_exception, "minifier_path" + ): + if hasattr(exc.inner_exception, "buck_command"): + msg += ( + f"\nMinifier script written to {exc.inner_exception.minifier_path}. Run " + f"this buck command to find the smallest traced graph " + f"which reproduces this error: {exc.inner_exception.buck_command}\n" + ) + else: + msg += ( + f"\nMinifier script written to {exc.inner_exception.minifier_path}. Run " + "this script to find the smallest traced graph which reproduces this error.\n" + ) + + old_msg = "" if len(exc.args) == 0 else str(exc.args[0]) + + if isinstance(exc, KeyError): + exc.args = (KeyErrorMsg(old_msg + msg),) + exc.args[1:] + else: + new_msg = old_msg + msg + exc.args = (new_msg,) + exc.args[1:] + + +def get_exc_message( + e: Exception, compile_id: CompileId +) -> tuple[Optional[str], Optional[int]]: + filename = None + lineno = None + if e.innermost_user_frame_summary is not None: # type: ignore[attr-defined] + filename = e.innermost_user_frame_summary.filename # type: ignore[attr-defined] + lineno = e.innermost_user_frame_summary.lineno # type: ignore[attr-defined] + e.compile_id = compile_id # type: ignore[attr-defined] + return filename, lineno + + +def get_stack_above_dynamo() -> StackSummary: + return filter_stack(extract_stack()) + + +def get_real_stack( + exc: Exception, frame: Optional[DynamoFrameType] = None +) -> Optional[StackSummary]: + real_stack = getattr(exc, "real_stack", None) + if real_stack is None: + return None + + # NB: it's possible for real_stack to be []; we still attempt to + # report a stack anyway because the stack_above_dynamo may still + # be useful for debugging + + if frame is not None: + # NB: frame is PyInterpreterFrame on Python 3.11 and later, + # not a TRUE frame object. You can't actually feed it + # to traceback because it doesn't have enough information. + # To solve this problem, we technically should just materialize + # the frame, the same way _PyFrame_GetFrameObject would do + # (but we cannot actually do this, because this populates + # frame_obj field, which default eval frame doesn't like). + # + # Fortunately, in this case, we can hack it: there's no need + # to actually use the truly top frame, we can just extract + # from where we are right now and rely on filter_stack to + # get rid of all the dynamo frames. For ease of testing + # we apply this behavior to ALL Python versions + stack_above_dynamo = get_stack_above_dynamo() + else: + stack_above_dynamo = StackSummary() + + return StackSummary.from_list(stack_above_dynamo + real_stack) + + +# filter out all frames after entering dynamo +def filter_stack(stack: StackSummary) -> StackSummary: + user_stack = StackSummary() + for frame in stack: + if frame.filename is None: + continue + if "convert_frame" in frame.filename: + break + if "eval_frame" in frame.filename or ( + frame.line and "torch._dynamo.optimize(" in frame.line + ): + continue + user_stack.append(frame) + + return user_stack + + +def remove_resume_prefix(name: str) -> Optional[str]: + from .resume_execution import TORCH_DYNAMO_RESUME_IN_PREFIX + + match = re.match(f"{TORCH_DYNAMO_RESUME_IN_PREFIX}_(\\w+)_at_\\d+", name) + if match: + return match.group(1) + return None + + +def collapse_resume_frames(stack: StackSummary) -> StackSummary: + """ + When we graph break, we create a resume function and make a regular Python call + to it, which gets intercepted by Dynamo. This behavior is normally shown in the + traceback, which can be confusing to a user. So we can filter out resume frames + for better traceback clarity. + + Example: + File "..." line 3, in f + + File "..." line 5, in torch_dynamo_resume_in_f_at_80 + + File "..." line 10, in torch_dynamo_resume_in_f_at_120 + + + becomes + File "..." line 10, in f + + """ + + new_stack = StackSummary() + for frame in stack: + if frame.filename is None: + continue + name = remove_resume_prefix(frame.name) + if new_stack and name and new_stack[-1].name == name: + new_stack[-1] = frame + frame.name = name + else: + new_stack.append(frame) + + return new_stack + + +def format_error_msg_verbose( + exc: Exception, + code: types.CodeType, + record_filename: Optional[str] = None, + frame: Optional[DynamoFrameType] = None, +) -> str: + msg = ( + f"WON'T CONVERT {code.co_name} {code.co_filename} line {code.co_firstlineno}\n" + ) + msg += "=" * 10 + " TorchDynamo Stack Trace " + "=" * 10 + "\n" + msg += format_exc() + real_stack = get_real_stack(exc, frame) + if real_stack is not None: + msg += ( + "\n" + + "=" * 10 + + " The above exception occurred while processing the following code " + + "=" * 10 + + "\n\n" + ) + msg += "".join(format_list(real_stack)) + msg += "\n" + msg += "=" * 10 + + return msg + + +def format_frame_info(code: types.CodeType) -> str: + return ( + f"{getattr(code, 'co_name', '')} " + f"({getattr(code, 'co_filename', '')} " + f"line {getattr(code, 'co_firstlineno', 0)})" + ) + + +def format_skip_frame_message(code: Optional[types.CodeType], reason: str) -> str: + if code is not None: + frame_info = format_frame_info(code) + return ( + f"torch.compile intentionally decided to skip the frame {frame_info} and fall back to eager.\n" + f"Reason: {reason}" + ) + else: + return ( + f"torch.compile intentionally decided to skip the frame and fall back to eager.\n" + f"Reason: {reason}" + ) + + +def format_loop_skip_frame_message(code: types.CodeType, frame_summary: str) -> str: + frame_info = format_frame_info(code) + return ( + "Skipping frame because there is a graph break in a for/while loop\n" + f"torch.compile intentionally decided to skip the frame {frame_info} and fall back to eager.\n" + f"Reason: Skipping frame because there is a graph break in a for/while loop.\n" + f"{frame_summary}" + ) + + +def format_error_msg( + exc: Exception, + code: types.CodeType, + record_filename: Optional[str] = None, + frame: Optional[DynamoFrameType] = None, +) -> str: + if config.verbose: + return format_error_msg_verbose(exc, code, record_filename, frame) + return f"WON'T CONVERT {code.co_name} {code.co_filename}\ + line {code.co_firstlineno} \ndue to: \n{format_exc()}" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/external_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/external_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..10422a3e2b82b2054778ca892f05e45958f4e710 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/external_utils.py @@ -0,0 +1,287 @@ +""" +This module contains utility functions that are explicitly allowed to be called during +TorchDynamo compilation. These functions are carefully vetted to ensure they work +correctly within the TorchDynamo tracing and compilation process. + +Key functionality groups: + +- Compilation State: + Functions for checking compilation state (is_compiling) + +- Function Wrapping: + Utilities for wrapping functions (wrap_inline, wrap_numpy) to work with + TorchDynamo compilation + +- Autograd Hooks: + Functions and classes for handling autograd hooks and backward passes + (call_hook, FakeBackwardCFunction, etc.) + +- Tensor Operations: + Utility functions for tensor operations and transformations +""" + +import functools +import warnings +from collections.abc import Callable +from typing import Any, Optional, TYPE_CHECKING, TypeVar, Union +from typing_extensions import deprecated, ParamSpec + +import torch +import torch.utils._pytree as pytree + + +try: + import numpy as np +except ModuleNotFoundError: + np = None # type: ignore[assignment] + +_P = ParamSpec("_P") +_R = TypeVar("_R") + +if TYPE_CHECKING: + # TorchScript does not support `@deprecated` + # This is a workaround to avoid breaking TorchScript + @deprecated( + "`torch._dynamo.external_utils.is_compiling` is deprecated. Use `torch.compiler.is_compiling` instead.", + category=FutureWarning, + ) + def is_compiling() -> bool: + return torch.compiler.is_compiling() + +else: + + def is_compiling() -> bool: + """ + Indicates whether we are tracing/compiling with torch.compile() or torch.export(). + """ + # NOTE: With `@torch.compile(backend="eager")`, torch._dynamo.is_compiling() will get traced + # and return true. torch.compiler.is_compiling() is skipped and will return false. + return torch.compiler.is_compiling() + + +def wrap_inline(fn: Callable[_P, _R]) -> Callable[_P, _R]: + """ + Create an extra frame around fn that is not in skipfiles. + """ + + @functools.wraps(fn) + def inner(*args: _P.args, **kwargs: _P.kwargs) -> _R: + return fn(*args, **kwargs) + + return inner + + +def call_hook( + hook: Callable[..., Optional[torch.Tensor]], *args: Any, **kwargs: Any +) -> torch.Tensor: + """ + Used by compiled autograd to handle hook returning None. + """ + result = hook(*args) + if result is None: + return args[0] + elif kwargs.get("hook_type") == "post_acc_grad_hook": + raise RuntimeError("Tensor post accumulate grad hooks should return None.") + return result + + +def wrap_numpy(f: Callable[_P, _R]) -> Callable[_P, _R]: + r"""Decorator that turns a function from ``np.ndarray``s to ``np.ndarray``s into a function + from ``torch.Tensor``s to ``torch.Tensor``s. + """ + if not np: + return f + + @functools.wraps(f) + def wrap(*args: _P.args, **kwargs: _P.kwargs) -> pytree.PyTree: + args, kwargs = pytree.tree_map_only( + torch.Tensor, lambda x: x.numpy(), (args, kwargs) + ) + # pyrefly: ignore [invalid-param-spec] + out = f(*args, **kwargs) + # pyrefly: ignore [missing-attribute] + return pytree.tree_map_only(np.ndarray, lambda x: torch.as_tensor(x), out) + + return wrap + + +class FakeBackwardCFunction: + def __init__( + self, + real: torch.autograd.function.BackwardCFunction, + saved_tensors: list[torch.Tensor], + ) -> None: + self.real = real + self.saved_tensors = saved_tensors + + def __getattr__(self, name: str) -> Any: + if name == "saved_variables": + warnings.warn( + "'saved_variables' is deprecated; use 'saved_tensors'", + DeprecationWarning, + ) + return self.saved_tensors + + return getattr(self.real, name) + + +def call_backward( + backward_c_function: torch.autograd.function.BackwardCFunction, + saved_tensors: list[torch.Tensor], + *args: Any, +) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]: + fake = FakeBackwardCFunction(backward_c_function, saved_tensors) + grads = fake._forward_cls.backward(fake, *args) # type: ignore[attr-defined] + + if not isinstance(grads, tuple): + grads = (grads,) + + return grads + + +def normalize_as_list(x: Any) -> list[Any]: + if isinstance(x, tuple): + return list(x) + elif isinstance(x, list): + return x + return [x] + + +def untyped_storage_size(x: torch.Tensor) -> int: + return x.untyped_storage().size() + + +class FakeCompiledAutogradEngine: + @staticmethod + def queue_callback( + final_callbacks: list[Callable[[], None]], cb: Callable[[], None] + ) -> None: + final_callbacks.append(cb) + + @staticmethod + def exec_final_callbacks(final_callbacks: list[Callable[[], None]]) -> None: + i = 0 + while i < len(final_callbacks): + cb = final_callbacks[i] + cb() + i += 1 + final_callbacks.clear() + + @staticmethod + def _exec_final_callbacks_stub() -> None: + pass + + +def call_hook_from_backward_state( + *args: Any, bw_state: Any, hook_name: str, **kwargs: Any +) -> Any: + return getattr(bw_state, hook_name)(*args, **kwargs) + + +def call_module_hooks_from_backward_state( + _: Any, result: Any, *args: Any, bw_state: Any, hooks_name: str, module_name: str +) -> Any: + module = getattr(bw_state, module_name) + hooks = getattr(bw_state, hooks_name) + for hook in hooks: + new_result = hook(module, result, *args) + if new_result is not None: + result = new_result + return result + + +# used for torch._dynamo.disable(recursive=False) +def get_nonrecursive_disable_wrapper(fn: Callable[_P, _R]) -> Callable[_P, _R]: + # wrap function to get the right error message + # this function is in external_utils so that convert_frame doesn't skip it. + @functools.wraps(fn) + def nonrecursive_disable_wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: + if torch.compiler.is_exporting(): + raise RuntimeError( + "Non-recursive torch.compiler.disable is not supported with torch.export." + ) + return fn(*args, **kwargs) + + return nonrecursive_disable_wrapper + + +def wrap_dunder_call_ctx_manager(self: Any, func: Callable[_P, _R]) -> Callable[_P, _R]: + """ + Apply self as a ctx manager around a call to func + """ + + # NOTE: do not functools.wraps(func) because we don't ever want this frame to be skipped! + def inner(*args: _P.args, **kwargs: _P.kwargs) -> _R: + with self: + return func(*args, **kwargs) + + return inner + + +# Use only on ints marked dynamic via torch.empty(0, integer) +# Currently only way to mark ints as dynamic: https://github.com/pytorch/pytorch/issues/129623 +def unwrap_maybe_dynamic_int(x: Union[torch.Tensor, int]) -> int: + if isinstance(x, torch.Tensor): + # x.size() is expected to be [0, dynamic_int] + return x.size(1) + return x + + +def call_accumulate_grad( + variable: torch.Tensor, grad: torch.Tensor, has_post_hooks: bool +) -> None: + updated_grad = torch._dynamo.compiled_autograd.ops.AccumulateGrad( # type: ignore[attr-defined] + [grad], variable, variable.grad, has_post_hooks + ) + variable.grad = updated_grad[0] + + +def wrap_inline_with_error_on_graph_break( + fn: Callable[_P, _R], error_on_graph_break: bool +) -> Callable[_P, _R]: + # NB: need multiple definitions in order to prevent `fullgraph` from + # being a freevar of wrapper + # NOTE: do not functools.wraps(fn) because we don't ever want these wrappers to be skipped! + if error_on_graph_break: + + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: + with torch._dynamo.error_on_graph_break(True): + return fn(*args, **kwargs) + + else: + + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: + with torch._dynamo.error_on_graph_break(False): + return fn(*args, **kwargs) + + return wrapper + + +def filter_out_const_values(tup: tuple[Any, ...], masks: list[bool]) -> tuple[Any, ...]: + """ + masks is a list of bools, where True means the corresponding element in tup + is a const value. Filter out the const values. + """ + out = [] + for mask_idx, mask in enumerate(masks): + if not mask: + out.append(tup[mask_idx]) + return tuple(out) + + +def insert_const_values_with_mask( + tup: tuple[Any, ...], masks: list[bool], values: tuple[Any, ...] +) -> tuple[Any, ...]: + """ + masks and values are of same length. For indices where the mask is True, use + the const_values to fill in. + """ + out = [] + idx = 0 + for mask_idx, mask in enumerate(masks): + if mask: + out.append(values[mask_idx]) + else: + out.append(tup[idx]) + idx += 1 + return tuple(out) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/funcname_cache.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/funcname_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..f71cb5c6b02a3fa192bd4b2f35836deef5133410 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/funcname_cache.py @@ -0,0 +1,75 @@ +""" +This module provides functionality for caching and looking up fully qualified function +and class names from Python source files by line number. + +It uses Python's tokenize module to parse source files and tracks function/class +definitions along with their nesting to build fully qualified names (e.g. 'class.method' +or 'module.function'). The results are cached in a two-level dictionary mapping: + + filename -> (line_number -> fully_qualified_name) + +Example usage: + name = get_funcname("myfile.py", 42) # Returns name of function/class at line 42 + clearcache() # Clear the cache if file contents have changed + +The parsing is done lazily when a file is first accessed. Invalid Python files or +IO errors are handled gracefully by returning empty cache entries. +""" + +import tokenize +from typing import Optional + + +cache: dict[str, dict[int, str]] = {} + + +def clearcache() -> None: + cache.clear() + + +def _add_file(filename: str) -> None: + try: + with tokenize.open(filename) as f: + tokens = list(tokenize.generate_tokens(f.readline)) + except (OSError, tokenize.TokenError): + cache[filename] = {} + return + + # NOTE: undefined behavior if file is not valid Python source, + # since tokenize will have undefined behavior. + result: dict[int, str] = {} + # current full funcname, e.g. xxx.yyy.zzz + cur_name = "" + cur_indent = 0 + significant_indents: list[int] = [] + + for i, token in enumerate(tokens): + if token.type == tokenize.INDENT: + cur_indent += 1 + elif token.type == tokenize.DEDENT: + cur_indent -= 1 + # possible end of function or class + if significant_indents and cur_indent == significant_indents[-1]: + significant_indents.pop() + # pop the last name + cur_name = cur_name.rpartition(".")[0] + elif ( + token.type == tokenize.NAME + and i + 1 < len(tokens) + and tokens[i + 1].type == tokenize.NAME + and (token.string == "class" or token.string == "def") + ): + # name of class/function always follows class/def token + significant_indents.append(cur_indent) + if cur_name: + cur_name += "." + cur_name += tokens[i + 1].string + result[token.start[0]] = cur_name + + cache[filename] = result + + +def get_funcname(filename: str, lineno: int) -> Optional[str]: + if filename not in cache: + _add_file(filename) + return cache[filename].get(lineno, None) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/functional_export.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/functional_export.py new file mode 100644 index 0000000000000000000000000000000000000000..19e8007f86fdf7daab6279f94ba83feda2640beb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/functional_export.py @@ -0,0 +1,850 @@ +import inspect +import logging +import sys +import traceback +from collections import namedtuple +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, Optional, TYPE_CHECKING, Union + +import sympy + +import torch +import torch.fx +import torch.utils._pytree as pytree +from torch._dispatch.python import enable_python_dispatcher +from torch._dynamo.convert_frame import CaptureOutput, fullgraph_capture, get_traced_fn +from torch._dynamo.eval_frame import argument_names, check_user_input_output +from torch._dynamo.exc import UserErrorType +from torch._dynamo.utils import dynamo_timed, get_metrics_context +from torch._export.utils import _compiling_state_context +from torch._guards import TracingContext +from torch.export.dynamic_shapes import _RelaxedConstraint, Constraint +from torch.fx import Node +from torch.fx.experimental.proxy_tensor import make_fx +from torch.fx.experimental.symbolic_shapes import ( + ConstraintViolationError, + DimDynamic, + ShapeEnv, + StatelessSymbolicContext, +) +from torch.fx.graph import _ExportCodeGen, _PyTreeCodeGen, _PyTreeInfo +from torch.utils._pytree import TreeSpec + + +if TYPE_CHECKING: + from torch._subclasses.fake_tensor import FakeTensorMode + + +log = logging.getLogger(__name__) + + +def post_process_error_msg( + constraint_violation_error: ConstraintViolationError, + func: Callable[..., Any], + args: Any, + kwargs: Any, +): + """ + Because we trace a different callable, the sources are all messed up. + Manually patch them so the error message looks correct. + """ + from torch.export._unlift import _get_input_paths, _replace_sources + + orig_sig = inspect.signature(func) + flat_input_paths = _get_input_paths((args, kwargs), orig_sig) + if constraint_violation_error.args: + constraint_violation_error.args = ( + _replace_sources(constraint_violation_error.args[0], flat_input_paths), + ) + return constraint_violation_error + + +EXPORT_ROOT_REPLACEMENTS = [ + ("__export_root_", "_"), + ("_export_root.", ""), + ("._export_root", ""), +] + + +def clean_export_root_string(text: str) -> str: + """Generic utility to clean export_root patterns from strings.""" + result = text + for pattern, replacement in EXPORT_ROOT_REPLACEMENTS: + result = result.replace(pattern, replacement) + return result + + +def clean_nn_module_stack_and_source_fn( + graph_module: torch.fx.GraphModule, is_inline_builtin=False +) -> torch.fx.GraphModule: + """ + Clean up nn_module_stack metadata by removing export_root references. + + Removes the _export_root module references from nn_module_stack metadata + in graph nodes, which are artifacts from the export process. Fixes two patterns: + + 1. Keys: Removes "__export_root_" and "__modules['_export_root']_" prefixes + - Normal case: "L__self____export_root_child" -> "L__self__child" + - inline_builtin case: Uses numeric ID strings like "140468831433840" + + 2. Values: Removes "._export_root" and "._modules['_export_root']" from child names + e.g., "L['self']._export_root.child" -> "L['self'].child" + e.g., "L['self']._modules['_export_root'].child" -> "L['self'].child" + + Also removes the root export entry "L__self____export_root" entirely. + + Args: + graph_module: The GraphModule to clean up + is_inline_builtin: If True, keys are numeric ID strings and self references + (L['self']) are filtered out + + Returns: + The cleaned GraphModule (modified in-place) + """ + + def _process_nn_module_stack(nn_module_stack): + if "L__self____export_root" in nn_module_stack: + del nn_module_stack["L__self____export_root"] + + # Clean up remaining entries + cleaned_stack = {} + for key, (child_name, child_class) in nn_module_stack.items(): + # Clean key by removing export_root patterns + clean_key = clean_export_root_string(key) + + # Clean child_name by removing export_root patterns + clean_name = clean_export_root_string(child_name) + + # Skip self reference for inline builtin case + if is_inline_builtin and clean_name == "L['self']": + continue + + cleaned_stack[clean_key] = (clean_name, child_class) + return cleaned_stack + + def _process_source_fn(source_fn_stack): + cleaned_stack = [] + for item in source_fn_stack: + if isinstance(item, tuple) and len(item) == 2: + name, cls = item + if isinstance(name, str): + clean_name = clean_export_root_string(name) + cleaned_stack.append((clean_name, cls)) + else: + cleaned_stack.append(item) + else: + cleaned_stack.append(item) + return cleaned_stack + + for node in graph_module.graph.nodes: + if "nn_module_stack" in node.meta: + node.meta["nn_module_stack"] = _process_nn_module_stack( + node.meta["nn_module_stack"].copy() + ) + + source_fn_stack = node.meta.get("source_fn_stack", None) + if source_fn_stack: + node.meta["source_fn_stack"] = _process_source_fn(source_fn_stack.copy()) + + if "dynamo_flat_name_to_original_fqn" in graph_module.meta: + # Clean up flat name to original fqn mapping + clean_name_to_original_fqn = {} + for flat_name, original_fqn in graph_module.meta[ + "dynamo_flat_name_to_original_fqn" + ].items(): + clean_name_to_original_fqn[clean_export_root_string(flat_name)] = ( + clean_export_root_string(original_fqn) + ) + graph_module.meta["dynamo_flat_name_to_original_fqn"] = ( + clean_name_to_original_fqn + ) + + return graph_module + + +def clean_export_root(graph_module: torch.fx.GraphModule) -> None: + """Remove export_root artifacts from FX graph in-place""" + + # Unlike getattr node, call_module can be invoked multiple times + # In those cases, we should fix all invocations of call_module + clean_named_module_map: dict[str, str] = {} + + # Update get_attr nodes in-place + for node in graph_module.graph.nodes: + if node.op == "get_attr": + old_target = node.target + new_target = clean_export_root_string(old_target) + if new_target != old_target: + node.target = new_target + assert hasattr(graph_module, old_target) + # Move the parameter to the new name + param = torch.fx.graph_module._get_attr(graph_module, old_target) + torch.fx.graph_module._assign_attr(param, graph_module, new_target) + torch.fx.graph_module._del_attr(graph_module, old_target) + # Dynamo will only have one nested level + if node.op == "call_module": + old_target = node.target + assert isinstance(old_target, str) + new_target = clean_export_root_string(old_target) + assert isinstance(new_target, str) + new_name = clean_export_root_string(node.name) + if new_target == old_target: + continue + + # if this module has already been cleaned before, just lookup from map. + if old_target in clean_named_module_map: + node.target = clean_named_module_map[old_target] + node.name = new_name + continue + target = graph_module.get_submodule(old_target) + graph_module.delete_submodule(old_target) + graph_module.add_submodule(new_target, target) + node.target = new_target + node.name = new_name + clean_named_module_map[old_target] = new_target + + +class ModuleToTrace(torch.nn.Module): + def __init__(self, foo: Any, in_spec: Any) -> None: + super().__init__() + self._export_root = foo + self.in_spec = in_spec + + def forward(self, *flat_args: Any) -> "ExportTracerOutput": + args, kwargs = pytree.tree_unflatten(flat_args, self.in_spec) + res = self._export_root(*args, **kwargs) + out_flat, out_spec = pytree.tree_flatten(res) + return ExportTracerOutput(out_flat, out_spec) + + +ExportTracerOutput = namedtuple("ExportTracerOutput", ["flat_args", "out_spec"]) + + +# mypy: disable-error-code="no-untyped-def,var-annotated,assignment,index,operator" +class DynamoGraphTransformer(torch.fx.Transformer): + """Graph transformer for dynamo export that flattens inputs/outputs without complex matching.""" + + def __init__( + self, + module: torch.fx.GraphModule, + flat_inputs: list[Any], + flat_args_dynamic_dims: list[set[int]], + graph_input_order: dict[int, int], + graph_output_map: dict[int, tuple[str, Any]], + fake_mode: Optional[Any] = None, + ) -> None: + super().__init__(module) + + assert len(flat_args_dynamic_dims) == len(flat_inputs) + + self.flat_inputs = flat_inputs + self.flat_args_dynamic_dims = flat_args_dynamic_dims + self.graph_input_order = graph_input_order + self.graph_output_map = graph_output_map + self.fake_mode = fake_mode + + # Get original placeholders and output + self.placeholders = [n for n in module.graph.nodes if n.op == "placeholder"] + self.output_node = next(n for n in module.graph.nodes if n.op == "output") + + # Create new flattened input placeholders + self.new_input_nodes: dict[int, torch.fx.Node] = {} + self._create_flattened_inputs() + + # Iterator for replacing old placeholders + self.old_to_new_mapping = {} + self._create_placeholder_mapping() + + def _create_flattened_inputs(self) -> None: + """Create new placeholder nodes for flattened inputs with proper fake tensors.""" + for i in range(len(self.flat_inputs)): + placeholder = super().placeholder(f"arg_{i}", (), {}) + + # Check if this user input (index i) maps to a graph placeholder + if i in self.graph_input_order: + # graph_input_order[i] gives us which graph placeholder this user input corresponds to + graph_placeholder_idx = self.graph_input_order[i] + if graph_placeholder_idx < len(self.placeholders): + orig_placeholder = self.placeholders[graph_placeholder_idx] + # Copy other metadata but not "val" yet + for key, value in orig_placeholder.meta.items(): + if key != "val": + placeholder.node.meta[key] = value + + # Always ensure we have proper "val" metadata from fake tensor + if self.fake_mode is not None and isinstance( + self.flat_inputs[i], torch.Tensor + ): + placeholder.node.meta["val"] = self.fake_mode.from_tensor( + self.flat_inputs[i], + symbolic_context=StatelessSymbolicContext( + dynamic_sizes=[ + ( + DimDynamic.DYNAMIC + if d in self.flat_args_dynamic_dims[i] + else DimDynamic.STATIC + ) + for d in range(len(self.flat_inputs[i].shape)) + ], + constraint_sizes=[None] * len(self.flat_inputs[i].shape), + ), + ) + elif hasattr(self.flat_inputs[i], "val"): # _IntWrapper case + placeholder.node.meta["val"] = self.flat_inputs[i].val + else: + placeholder.node.meta["val"] = self.flat_inputs[i] + + # pyrefly: ignore [unsupported-operation] + self.new_input_nodes[i] = placeholder + + def _create_placeholder_mapping(self) -> None: + """Create mapping from old placeholders to new ones.""" + # graph_input_order maps: user_input_index -> graph_placeholder_index + # We need to create: old_graph_placeholder -> new_user_input_placeholder + for user_input_idx, graph_placeholder_idx in self.graph_input_order.items(): + if graph_placeholder_idx < len(self.placeholders): + old_placeholder = self.placeholders[graph_placeholder_idx] + new_placeholder = self.new_input_nodes[user_input_idx] + self.old_to_new_mapping[old_placeholder] = new_placeholder + + def placeholder(self, target, args, kwargs) -> Any: + """Replace old placeholders with new flattened ones.""" + # Return the corresponding new placeholder + if self.current_node in self.old_to_new_mapping: + new_arg = self.old_to_new_mapping[self.current_node] + + # Copy over additional metadata from current node, but don't overwrite "val" + for key in ["tensor_dict", "example_value", "unbacked_bindings"]: + if key in self.current_node.meta: + new_arg.node.meta[key] = self.current_node.meta[key] + + # Only copy "val" if we don't already have a good one + if "val" in self.current_node.meta and "val" not in new_arg.node.meta: + new_arg.node.meta["val"] = self.current_node.meta["val"] + + return new_arg + else: + # Shouldn't happen if mapping is correct, but fallback + return super().placeholder(target, args, kwargs) + + def output(self, target, args, kwargs) -> Any: + """Transform output according to graph_output_map.""" + original_outputs = args[0] + + # Build new output list based on graph_output_map + new_outputs = [] + for i in sorted(self.graph_output_map.keys()): + output_type, val = self.graph_output_map[i] + + if output_type == "graph_out": + new_outputs.append(original_outputs[val]) + elif output_type == "input": + input_idx = val.index + new_outputs.append(self.new_input_nodes[input_idx]) + elif output_type == "constant": + new_outputs.append(val) + + return super().output(target, (tuple(new_outputs),), {}) + + def run_node(self, node: Node) -> Any: + """Run node transformation and preserve metadata.""" + self.current_node = node + result = super().run_node(node) + + # Copy important metadata + if hasattr(result, "node") and result.node is not node: + for key in ["val", "example_value", "unbacked_bindings"]: + if key in node.meta: + result.node.meta[key] = node.meta[key] + + # Preserve node names (except output) + if node.op != "output" and hasattr(node, "name"): + result.node._rename(node.name) + + return result + + def transform(self) -> torch.fx.GraphModule: + """Perform the graph transformation and copy module metadata.""" + result_gm = super().transform() + + # Copy module metadata like the original implementation + if hasattr(self.module, "meta"): + # pyrefly: ignore [unsupported-operation] + if "dynamo_flat_name_to_original_fqn" in self.module.meta: + # pyrefly: ignore [index-error] + result_gm.meta["dynamo_flat_name_to_original_fqn"] = self.module.meta[ + # pyrefly: ignore [index-error] + "dynamo_flat_name_to_original_fqn" + ] + # pyrefly: ignore [unsupported-operation] + if "dynamo_compile_id" in self.module.meta: + # pyrefly: ignore [index-error] + result_gm.meta["dynamo_compile_id"] = self.module.meta[ + # pyrefly: ignore [index-error] + "dynamo_compile_id" + ] + + return result_gm + + +def _suggest_or_raise_constraint_violation( + module_to_trace: torch.nn.Module, + orig_callable: Callable, # type: ignore[type-arg] + fake_mode: Optional["FakeTensorMode"], + graph_capture_output: CaptureOutput, + args: Any, + kwargs: Any, + dynamic_shapes: Optional[Union[dict[str, Any], tuple[Any], list[Any]]], +): + constraint_violation_error = None + try: + # Check if we have any constraint violations + fn, _ = get_traced_fn(module_to_trace) + graph_capture_output.graph_capture_output.build_guards(fn.__code__) + except ConstraintViolationError as e: + constraint_violation_error = e + + if ( + (shape_env := getattr(fake_mode, "shape_env", None)) is not None + and (dim_constraints := shape_env.dim_constraints) is not None + and not isinstance( + module_to_trace.forward, + torch._ops.OpOverloadPacket | torch._ops.OpOverload, + ) + ): + dim_constraints.solve() + + forced_specializations = dim_constraints.forced_specializations() + + msg = dim_constraints.prettify_results( + inspect.signature(orig_callable), # type: ignore[attr-defined] + dynamic_shapes, + constraint_violation_error, + forced_specializations, + ) + if constraint_violation_error: + if constraint_violation_error.args: + constraint_violation_error.args = ( + constraint_violation_error.args[0] + msg, + ) + else: + constraint_violation_error.args = (msg,) + else: + if forced_specializations: + constraint_violation_error = ConstraintViolationError(msg) + else: + log.info( + "Summary of dimension constraints:%s", + msg, + ) + + # Error if we have any constraints on static values + + for k in shape_env.var_to_range: + if isinstance(k, sympy.Integer): + constraint_violation_error = ConstraintViolationError( + f"{''.join(traceback.format_list(shape_env.var_to_stack[k]))}\n" + "It appears that you're trying to set a constraint on a " + f"value which we evaluated to have a static value of {k}. " + 'Set TORCH_LOGS="+export" for more information.' + ) + if constraint_violation_error: + constraint_violation_error = post_process_error_msg( + constraint_violation_error, orig_callable, args, kwargs + ) + raise constraint_violation_error + + +def _normalize_shuffle_graph(shuffle_gm: torch.fx.GraphModule) -> None: + shuffle_gm.graph.eliminate_dead_code() + shuffle_gm.recompile() + for name, buffer in list(shuffle_gm.named_buffers()): + delattr(shuffle_gm, name) + setattr(shuffle_gm, name, buffer) + + +@dataclass(frozen=True) +class PyTreeifyOutput: + graph_module: torch.fx.GraphModule + in_spec: TreeSpec + in_shuffle_graph: torch.fx.GraphModule + num_flat_args: int + out_spec: TreeSpec + out_shuffle_graph: torch.fx.GraphModule + root: Optional[torch.nn.Module] = None + + +def pytreeify( + out: CaptureOutput, mod: Any, args: tuple[Any, ...], kwargs: dict[str, Any] +) -> PyTreeifyOutput: + """ + Given a dynamo capture output, return a callable graph module that + contain the following information: + 1. input/output pytree spec + 2. input/output shuffle functions + Input shuffle functions are the converters taking pytree falttened inputs + and reorder them to the calling convention of dynamo raw graph module. + Output shuffle functions are the converters taking the outputs of the + dynamo raw graph module and convert them to the pytree format. + + This function will replay any side effects that happened during the bytecode, + so it is important to check against side effects before calling this function. + """ + assert out.backend_input is not None + backend_input = out.backend_input + + root = None + if isinstance(mod, torch.nn.Module): + args = (mod,) + args + root = mod + elif inspect.ismethod(mod): + args = (mod.__self__,) + args + root = mod.__self__ + + flat_real_args, in_spec = pytree.tree_flatten((args, kwargs)) + torch._dynamo.eval_frame.check_user_input_output( + flat_real_args[1 if root else 0 :], UserErrorType.INVALID_INPUT + ) + f_globals = out.graph_capture_output.f_globals + + class Yield(Exception): + pass + + class InShuffle(torch.nn.Module): + def __init__(self): + super().__init__() + self.mod = mod + self.num_inputs = len(flat_real_args) + self.gm_inputs = None + + def forward(self, *flat_proxy_args): + args, kwargs = pytree.tree_unflatten( + [flat_proxy_args[i] for i in range(self.num_inputs)], in_spec + ) + + def backend_dummy(*example_inputs): + self.gm_inputs = example_inputs + raise Yield + + try: + out.forward_callable( + compiled_fn=backend_dummy, extra_globals=f_globals + )(*args, **kwargs) + except Yield: + assert self.gm_inputs is not None + return self.gm_inputs + raise RuntimeError + + fake_mode = torch._dynamo.utils.detect_fake_mode(flat_real_args) + if fake_mode and fake_mode.shape_env is None: + fake_mode.shape_env = ShapeEnv() + in_shuffle_graph = make_fx( + InShuffle(), tracing_mode="symbolic", proxy_module_inputs=True + )(*flat_real_args) + _normalize_shuffle_graph(in_shuffle_graph) + + output_node = next(iter(reversed(backend_input.graph_module.graph.nodes))) + + class OutShuffle(torch.nn.Module): + def __init__(self): + super().__init__() + self.num_inputs = len(flat_real_args) + + self.num_outputs = len(output_node.args[0]) + self.out_spec: Optional[TreeSpec] = None + + def forward(self, *flat_proxy_args): + args, kwargs = pytree.tree_unflatten( + [flat_proxy_args[i] for i in range(self.num_inputs)], in_spec + ) + + def backend_dummy(*example_inputs): + return [ + flat_proxy_args[self.num_inputs + i] + for i in range(self.num_outputs) + ] + + results = out.forward_callable( + compiled_fn=backend_dummy, extra_globals=f_globals + )(*args, **kwargs) + ret, self.out_spec = pytree.tree_flatten(results) + return ret + + out_shuffle = OutShuffle() + flat_out_shuffle_args = [ + *flat_real_args, + *pytree.tree_map_only( + torch.fx.Node, + lambda x: fake_mode.from_tensor(x.meta["example_value"]) + if fake_mode + else x.meta["example_value"], + output_node.args[0], + ), + ] + fake_mode = torch._dynamo.utils.detect_fake_mode(flat_out_shuffle_args) + if fake_mode and fake_mode.shape_env is None: + fake_mode.shape_env = ShapeEnv() + with enable_python_dispatcher(): + out_shuffle_graph = make_fx( + out_shuffle, tracing_mode="real", proxy_module_inputs=True + )(*flat_out_shuffle_args) + _normalize_shuffle_graph(out_shuffle_graph) + + assert out_shuffle.out_spec is not None + return PyTreeifyOutput( + backend_input.graph_module, + in_spec, + in_shuffle_graph, + len(flat_real_args), + out_shuffle.out_spec, + out_shuffle_graph, + root=root, # type: ignore[arg-type] + ) + + +def normalize_graph_module(gm): + for node in gm.graph.nodes: + if node.op == "placeholder": + node.meta["val"] = node.meta["example_value"] + + +def dynamo_graph_capture_for_export( + mod: Callable[..., Any], + constraints: Optional[list[Constraint]] = None, +) -> Callable[..., Any]: + def inner(*args: Any, **kwargs: Any) -> Any: + assert not torch._dynamo.config.install_free_tensors + with ( + torch._dynamo.config.patch(side_effect_replay_policy="warn"), + get_metrics_context(), + dynamo_timed("fullgraph_capture"), + ): + out = fullgraph_capture( + mod, + args, + kwargs, + constraints=constraints, + ) + + # TODO filter out side effects. + pyt = pytreeify(out, mod, args, kwargs) + + graph_module = pyt.graph_module + tree_leaf_names = [ + graph_module.graph._graph_namespace.create_name(f"_tree_leaf_{i}", None) + for i in range(pyt.num_flat_args) + ] + graph_module.graph._codegen = _ExportCodeGen( + _PyTreeInfo( + # TODO we should be able to use the names from dynamo graph directly. + argument_names(inspect.signature(mod), args, kwargs), + pyt.in_spec, + pyt.out_spec, + ), + pyt.in_shuffle_graph, + pyt.out_shuffle_graph, + tree_leaf_names, + graph_module if isinstance(pyt.root, torch.nn.Module) else pyt.root, + ) # type: ignore[attr-defined] + normalize_graph_module(graph_module) + if pyt.root is not None: + graph_module._parameters = pyt.root._parameters.copy() + graph_module._buffers = pyt.root._buffers.copy() + assert all(not hasattr(graph_module, m) for m in pyt.root._modules) + graph_module._modules.update(pyt.root._modules) + graph_module._non_persistent_buffers_set = ( + pyt.root._non_persistent_buffers_set.copy() + ) + if sys.version_info >= (3, 14): + import annotationlib # added in 3.14 + + annotations = annotationlib.get_annotations(torch.nn.Module) + else: + annotations = getattr(torch.nn.Module, "__annotations__", None) + for name, value in pyt.root.__dict__.items(): + if annotations and name not in annotations: + graph_module.__dict__[name] = value + graph_module._in_spec = pyt.in_spec + graph_module._out_spec = pyt.out_spec + assert not hasattr(graph_module, "_in_shuffle_graph") + assert not hasattr(graph_module, "_out_shuffle_graph") + graph_module._in_shuffle_graph = pyt.in_shuffle_graph + graph_module._out_shuffle_graph = pyt.out_shuffle_graph + delattr(graph_module, "_param_name_to_source") + graph_module.recompile() + graph_module.meta["module_call_specs"] = ( + out.graph_capture_output.output_graph.export_metadata.module_call_spec + ) + assert out.backend_input is not None + graph_module.meta["fake_mode"] = out.backend_input.fake_mode # type: ignore[attr-defined] + graph_module.meta["fake_mode"].allow_non_fake_inputs = True + tracing_context = TracingContext(graph_module.meta["fake_mode"]) + tracing_context.tensor_to_context = out.backend_input.tensor_to_context # type: ignore[attr-defined] + graph_module.meta["tracing_context"] = tracing_context + return graph_module + + return inner + + +def _dynamo_graph_capture_for_export( + mod: Callable[..., Any], + *, + constraints: Optional[list[Constraint]] = None, + dynamic_shapes: Optional[Union[dict[str, Any], tuple[Any], list[Any]]] = None, +) -> Callable[..., torch.fx.GraphModule]: + """ + Improved dynamo graph capture using transformer approach with proper fake tensor handling. + + This function creates a capture instance that handles: + 1. PyTree flattening/unflattening with proper input ordering + 2. Dynamo graph capture with export-specific context + 3. FX graph transformation for export compatibility + 4. Proper fake tensor metadata preservation + 5. Dynamic dimension constraint handling + + Notable improvements over manual approach: + - Uses FX Transformer for cleaner graph manipulation + - Properly handles fake tensor metadata and dynamic dimensions + - Preserves all necessary metadata for export + - More robust error handling and edge case management + + TODO: + 1. Are we actually gonna run the bytecode? + 2. Need to attach guards + """ + + _dynamic_shapes = dynamic_shapes + _constraints = constraints + + def inner(*args: Any, **kwargs: Any) -> torch.fx.GraphModule: + # This sets the is_exporting flag when building guards. + with _compiling_state_context(): + flat_inputs, in_spec = pytree.tree_flatten((args, kwargs)) + check_user_input_output(flat_inputs, UserErrorType.INVALID_INPUT) + module_to_trace = ModuleToTrace(mod, in_spec) + orig_callable = mod.forward if isinstance(mod, torch.nn.Module) else mod + + constraints: Optional[list[Constraint]] = _constraints + dynamic_shapes: Optional[Union[dict[str, Any], tuple[Any], list[Any]]] = ( + _dynamic_shapes + ) + + from . import reset # type: ignore[attr-defined] + + reset() + + dynamo_config_ctx = torch._dynamo.config.patch( + specialize_int=True, + specialize_float=True, + assume_static_by_default=True, + automatic_dynamic_shapes=False, + capture_dynamic_output_shape_ops=True, + capture_scalar_outputs=True, + constant_fold_autograd_profiler_enabled=True, + log_graph_in_out_metadata=True, + # install_free_tensors ensures that params and buffers are still + # added as graph attributes, and makes Dynamo emits graphs that + # follow export pytree-able input requirements In future, if we + # fully rely on bytecode for the runtime, we can turn this flag + # off. + install_free_tensors=torch._dynamo.config.install_free_tensors_for_export, + ) + + with ( + get_metrics_context(), + dynamo_timed("fullgraph_capture"), + dynamo_config_ctx, + ): + out = fullgraph_capture( + module_to_trace, + tuple(flat_inputs), + constraints=_constraints, + _is_export_deprecated_do_not_use=True, + ) + + assert out.graph_capture_output.output_graph is not None + + example_inputs: list[Any] = [] + if out.backend_input is not None: + graph = out.backend_input.graph_module + fake_mode = out.backend_input.fake_mode + example_inputs = out.backend_input.example_inputs + else: + graph = torch.fx.GraphModule(torch.nn.Module(), torch.fx.Graph()) + graph.graph.output(None) + graph.recompile() + fake_mode = None + + _suggest_or_raise_constraint_violation( + module_to_trace, + orig_callable, + fake_mode, + out, + args, + kwargs, + dynamic_shapes, + ) + + # Extract export metadata from the new location + export_metadata = out.graph_capture_output.output_graph.export_metadata + graph_inputs = export_metadata.graph_input_idx_to_local_source + graph_output_map = export_metadata.output_return_type + out_spec = export_metadata.out_spec + module_call_spec = export_metadata.module_call_spec + + # Compute dynamic dimensions for each input based on constraints + flat_args_dynamic_dims = [ + { + c.dim + for c in (constraints or ()) + if ( + c.t_id == id(x) + and not isinstance(c, _RelaxedConstraint) + and c.constraint_range.vr.lower != c.constraint_range.vr.upper + ) + } + for x in flat_inputs + ] + + # Create input order mapping from dynamo's internal order to user order + graph_input_order: dict[int, int] = {} + for inp in graph_inputs: + source = graph_inputs[inp] + assert isinstance(source, torch._dynamo.source.GetItemSource) + graph_input_order[source.index] = len(graph_input_order) + + for real_idx, graph_idx in graph_input_order.items(): + flat_inputs[real_idx] = example_inputs[graph_idx] + + # Use FX transformer to rebuild the graph cleanly + transformed_graph = DynamoGraphTransformer( + graph, + flat_inputs, + flat_args_dynamic_dims, + graph_input_order, + graph_output_map, + fake_mode, + ).transform() + + # Set up PyTree codegen for proper input/output handling + transformed_graph.graph._codegen = _PyTreeCodeGen( + _PyTreeInfo( + argument_names(inspect.signature(orig_callable), args, kwargs), # type: ignore[attr-defined, arg-type] + in_spec, + out_spec, + ) + ) + transformed_graph.recompile() + + clean_nn_module_stack_and_source_fn( + transformed_graph, torch._dynamo.config.inline_inbuilt_nn_modules + ) + clean_export_root(transformed_graph) + + transformed_graph.meta["module_call_specs"] = module_call_spec + transformed_graph.meta["fake_mode"] = fake_mode + + return transformed_graph + + return inner diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_break_hints.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_break_hints.py new file mode 100644 index 0000000000000000000000000000000000000000..5a1da5d1cc6f1ee8588cf3c5201daf0d64042220 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_break_hints.py @@ -0,0 +1,26 @@ +USER_ERROR = [ + "Dynamo has detected that tracing the code will result in an error when running in eager. " + "Please double check that your code doesn't contain a similar error when actually running eager/uncompiled.", +] +DYNAMO_BUG = [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch.", +] +DIFFICULT = [ + "This graph break may be difficult to debug. Please report an issue to PyTorch for assistance.", +] +FUNDAMENTAL = [ + "This graph break is fundamental - it is unlikely that Dynamo will ever be able to trace through " + "your code. Consider finding a workaround.", +] +SUPPORTABLE = [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you " + "encounter this graph break often and it is causing performance issues.", +] +CAUSED_BY_EARLIER_GRAPH_BREAK = [ + "This graph break may have been caused by an earlier graph break. Resolving the earlier graph break may resolve this one.", +] +INFERENCE_MODE = [ + "Avoid using `tensor.is_inference()` and `torch.is_inference_mode_enabled()` in your compile code. " + "This is primarily used in conjunction with `torch.inference_mode`. Consider using `torch.no_grad` instead " + "because `torch.no_grad` leads to same improvements as `inference_mode` when `torch.compile` is used.", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_break_registry.json b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_break_registry.json new file mode 100644 index 0000000000000000000000000000000000000000..3b7f7ca78c802209e88e51fb3ca3e3a4cc05a8c3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_break_registry.json @@ -0,0 +1,3756 @@ +{ + "GB0000": [ + { + "Gb_type": "All __torch_function__ overrides returned NotImplemented due to TypeError from user code", + "Context": "fn={fn}, args={args}, kwargs={kwargs}", + "Explanation": "All __torch_function__ overrides for for function {fn} returned NotImplemented", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0001": [ + { + "Gb_type": "Argument of `as_subclass` must be a non-dispatcher-style tensor subclass", + "Context": "{self}.as_subclass({cls})", + "Explanation": "Currently not supported", + "Hints": [ + "Avoid this call or move it outside `torch.compile` regione", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0002": [ + { + "Gb_type": "Assertion failed on symbolic shapes", + "Context": "str(sym_expr)", + "Explanation": "", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0003": [ + { + "Gb_type": "Attempt to trace generator", + "Context": "", + "Explanation": "Generators cannot be compiled directly with `torch.compile`.", + "Hints": [ + "Call a generator from inside of a non-generator Python function and ", + "compile that function instead.", + "This graph break is fundamental - it is unlikely that Dynamo will ever be able to trace through your code. Consider finding a workaround." + ] + } + ], + "GB0004": [ + { + "Gb_type": "Attempted super().__delattr__() on an object without mutation tracking", + "Context": "call_method {self} {name}", + "Explanation": "Dynamo needs to track mutations on an object before `super().__delattr__` can be used on it. But the object ({self.objvar}) doesn't have attribute mutation tracking enabled.", + "Hints": [ + "Ensure the object is tracked by Dynamo's side effect system.", + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0005": [ + { + "Gb_type": "Attempted to a str() method implemented in C/C++", + "Context": "", + "Explanation": "{type(arg.value)} has a C/C++ based str method. This is not supported.", + "Hints": [ + "Write the str method in Python" + ] + } + ], + "GB0006": [ + { + "Gb_type": "Attempted to call a super() attribute that is not a function or method", + "Context": "call_method {self} {name}", + "Explanation": "Dynamo does not know how to trace the call `super().{name}()` because `super().{name}` is not a function or method attribute.", + "Hints": [ + "Ensure the attribute accessed via `super()` is a standard method or function." + ] + } + ], + "GB0007": [ + { + "Gb_type": "Attempted to call function marked as skipped", + "Context": "module: {module_name}, qualname: {qualname}, skip reason: {reason}", + "Explanation": "explanation", + "Hints": [] + } + ], + "GB0008": [ + { + "Gb_type": "Attempted to inline function marked as skipped", + "Context": "qualname: {fn_qualname}, name: {func.get_name()}, filename: `{func.get_filename()}`, skip reason: {result.reason}", + "Explanation": "Dynamo developers have intentionally marked that the function `{fn_qualname}` should not be traced.", + "Hints": [] + } + ], + "GB0009": [ + { + "Gb_type": "Attempted to inline function marked as skipped (SkipFunctionVariable)", + "Context": "Attempted to inline a SkipFunctionVariable {func}", + "Explanation": "Attempted to inline a function that was previously determined to be marked as intentionally skipped.", + "Hints": [] + } + ], + "GB0010": [ + { + "Gb_type": "Attempted to read a deleted variable", + "Context": "item: {item}, name: {name}", + "Explanation": "", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0011": [ + { + "Gb_type": "Attempted to read undefined local variable", + "Context": "LOAD_FAST {name}", + "Explanation": "Could not find a local variable with name `{name}`", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0012": [ + { + "Gb_type": "Attempted to read undefined local variable (implicit)", + "Context": "LOAD_FAST {name}", + "Explanation": "Could not find an implicit local variable with name `{name}`", + "Hints": [ + "This happens in dict/list comprehensions", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0013": [ + { + "Gb_type": "Attempted to represent unregistered RemovableHandle", + "Context": "", + "Explanation": "Dynamo attempted to build a representation of a torch.utils.hooks.RemovableHandle, which is not supported. This happens because the RemovableHandle was created in another frame.", + "Hints": [] + } + ], + "GB0014": [ + { + "Gb_type": "Attempted to wrap RNN, GRU, or LSTM", + "Context": "str(value)", + "Explanation": "Dynamo does not support RNN, GRU, or LSTM.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0015": [ + { + "Gb_type": "Attempted to wrap sparse Tensor", + "Context": "", + "Explanation": "torch.compile does not support sparse Tensors", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0016": [ + { + "Gb_type": "Attempted to wrap strided NestedTensor", + "Context": "", + "Explanation": "torch.compile does not support strided NestedTensor", + "Hints": [] + } + ], + "GB0017": [ + { + "Gb_type": "Attempted to wrap torch._higher_order_ops.invoke_subgraph", + "Context": "", + "Explanation": "Directly using invoke_subgraph is not supported. Use nested_compile_region", + "Hints": [] + } + ], + "GB0018": [ + { + "Gb_type": "Attempted to wrap unbacked SymInt", + "Context": "", + "Explanation": "Unbacked SymInt input is not supported yet.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0019": [ + { + "Gb_type": "AutogradFunctionContextVariable escaped Dynamo-traced region", + "Context": "", + "Explanation": "We cannot reconstruct a torch.autograd.Function's context object.", + "Hints": [] + } + ], + "GB0020": [ + { + "Gb_type": "BUILD_STRING key conflict", + "Context": "format_string_parts: {format_string_parts}, kwargs: {kwargs}, part.sym_kwargs: {part.sym_kwargs}", + "Explanation": "Failed to build format string due to key conflict", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0021": [ + { + "Gb_type": "BUILD_STRING type error", + "Context": "str(part)", + "Explanation": "Format string part type is not correct - expected constant or format string.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0022": [ + { + "Gb_type": "Bad import result", + "Context": "typestr(value)", + "Explanation": "Import result is not a Python module.", + "Hints": [] + } + ], + "GB0023": [ + { + "Gb_type": "Builtin `operator.*` comparison with constant `self` failed", + "Context": "call_method {self} {name} {args} {kwargs}", + "Explanation": "\"Failed to compare {self} with {other}, \" + f\"because {other} is not a Python constant or its mutation check fails.\"", + "Hints": [] + } + ], + "GB0024": [ + { + "Gb_type": "CLEANUP_THROW with StopIteration", + "Context": "", + "Explanation": "Received StopIteration when handling generator.throw/close. This is not supported.", + "Hints": [] + } + ], + "GB0025": [ + { + "Gb_type": "Call to `torch._dynamo.graph_break()`", + "Context": "Called `torch._dynamo.graph_break()` with args `{args}`, kwargs `{kwargs}`", + "Explanation": "User-inserted graph break. Message: {graph_break_msg}", + "Hints": [ + "Remove the `torch._dynamo.graph_break()` call." + ] + } + ], + "GB0026": [ + { + "Gb_type": "Calling subclass default constructor with more than tensor argument", + "Context": "{self.value}(args={args}, kwargs={kwargs})", + "Explanation": "Currently not supported", + "Hints": [ + "Avoid this constructor call or move it outside ", + "`torch.compile` regione", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0027": [ + { + "Gb_type": "Cannot check Tensor object identity without its fake value", + "Context": "str(fake_tensor)", + "Explanation": "TensorVariable is missing a fake example_value.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0028": [ + { + "Gb_type": "Caught non-Exception value", + "Context": "str(exc_instance)", + "Explanation": "Except expects to receive an object of Exception type but received {exc_instance}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0029": [ + { + "Gb_type": "Compilation of intermediate hooks requires compiled autograd", + "Context": "var_getattr {self} {name}", + "Explanation": "Dynamo must be in compiled_autograd to register hooks.", + "Hints": [] + } + ], + "GB0030": [ + { + "Gb_type": "ComptimeContext graph break", + "Context": "msg", + "Explanation": "Manually triggered ComptimeContext graph break with message {msg}.", + "Hints": [] + } + ], + "GB0031": [ + { + "Gb_type": "Custom __getattribute__ in nn.Module attribute access", + "Context": "var_getattr {self} {name}", + "Explanation": "Dynamo does not support checking key existence on `nn.Module` instances that have a custom `__getattribute__` method defined.", + "Hints": [ + "Avoid defining `__getattribute__` in your module.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0032": [ + { + "Gb_type": "Custom __getattribute__ in nn.Module dict key check", + "Context": "has_key_in_generic_dict {self} {key}", + "Explanation": "Dynamo does not support checking key existence on `nn.Module` instances that have a custom `__getattribute__` method defined.", + "Hints": [ + "Avoid defining `__getattribute__` in your module.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0033": [ + { + "Gb_type": "Data dependent operator", + "Context": "str(cause.func)", + "Explanation": "Operator `{cause.func}` has a non-Tensor output whose value is dependent on the data of Tensor inputs.", + "Hints": [] + } + ], + "GB0034": [ + { + "Gb_type": "Data-dependent assertion failed (cannot compile partial graph)", + "Context": "value: {value}", + "Explanation": "Dynamo has determined when encountering a data-dependent assert failure that it should not compile the partial graph.", + "Hints": [ + "Use `torch._assert()` to raise a hard AssertionError when the check fails. ", + "This error will propagate back the user code ", + "that called the compiled function (i.e. Dynamo will not trace any exception handling).", + "Remove the assert statement.", + "Move the assert statement outside of any context managers in order to graph break with ", + "partial graph compilation (if fullgraph=False).", + "This graph break is fundamental - it is unlikely that Dynamo will ever be able to trace through your code. Consider finding a workaround." + ] + } + ], + "GB0035": [ + { + "Gb_type": "Data-dependent branching with non-constant __bool__", + "Context": "method: {x}, result: {result}", + "Explanation": "Attempted to perform data-dependent branching on a user-defined object with a __bool__ method that did not return a constant.", + "Hints": [] + } + ], + "GB0036": [ + { + "Gb_type": "Dynamic shape operator", + "Context": "str(cause.func)", + "Explanation": "Operator `{cause.func}`'s output shape depends on input Tensor data.", + "Hints": [ + "Enable tracing of dynamic shape operators with ", + "`torch._dynamo.config.capture_dynamic_output_shape_ops = True`" + ] + } + ], + "GB0037": [ + { + "Gb_type": "Dynamic shape operator (no meta kernel)", + "Context": "str(cause.func)", + "Explanation": "Operator `{cause.func}` does not have a meta kernel that supports dynamic output shapes", + "Hints": [ + "Please report an issue to PyTorch" + ] + } + ], + "GB0038": [ + { + "Gb_type": "Dynamic slicing with Tensor arguments", + "Context": "SliceVariable start: {start}, stop: {stop}, step: {step}", + "Explanation": "Creating slices with Tensor arguments is not supported. e.g. `l[:x]`, where `x` is a 1-element tensor.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0039": [ + { + "Gb_type": "Dynamo cache limit exceeded", + "Context": "Limit type: {limit_type}", + "Explanation": "Dynamo attempted to recompile the code object too many times, exceeding the {limit_type} cache size limit.Giving up on compiling as the compile time tradeoff is likely not worth the performance gain.", + "Hints": [] + } + ], + "GB0040": [ + { + "Gb_type": "Encountered aliasing during higher order op tracing", + "Context": "context", + "Explanation": "Higher order ops do not support aliasing. Found in {source_target.name}", + "Hints": [ + "Replace `return input` with `return input.clone()` to avoid aliasing.", + "Consider using the debug context to change user code to avoid aliasing.", + "Please open an issue." + ] + } + ], + "GB0041": [ + { + "Gb_type": "Encountered input mutation during higher order op tracing", + "Context": "context", + "Explanation": "Higher order ops do not support input mutation. Found in {source_target.name}", + "Hints": [ + "Consider using the debug context to change user code to avoid mutation.", + "Please open an issue." + ] + } + ], + "GB0042": [ + { + "Gb_type": "Encountered non user function variable during invoke_subgraph HOP tracing", + "Context": "str(fn_vt)", + "Explanation": "invoke_subgraph does not support non user function variable", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0043": [ + { + "Gb_type": "Encountered non-PT2-compliant op", + "Context": "", + "Explanation": "msg + + err_epilogue", + "Hints": [] + } + ], + "GB0044": [ + { + "Gb_type": "Encountered strided NestedTensor in automatic dynamic dim determination", + "Context": "", + "Explanation": "torch.compile does not support strided NestedTensor", + "Hints": [] + } + ], + "GB0045": [ + { + "Gb_type": "Encountered tensor.is_inference() during tracing", + "Context": "", + "Explanation": "tensor.is_inference() is not supported", + "Hints": [ + "This graph break is fundamental - it is unlikely that Dynamo will ever be able to trace through your code. Consider finding a workaround." + ] + } + ], + "GB0046": [ + { + "Gb_type": "Encountered torch.is_inference_mode_enabled during tracing", + "Context": "", + "Explanation": "torch.is_inference_mode_enabled() is not supported", + "Hints": [ + "This graph break is fundamental - it is unlikely that Dynamo will ever be able to trace through your code. Consider finding a workaround." + ] + } + ], + "GB0047": [ + { + "Gb_type": "Encountered unconverted argument when attempting to inline", + "Context": "func: {func}, arg: {v}", + "Explanation": "An argument to an inlined function was not successfully converted to a VariableTracker.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0048": [ + { + "Gb_type": "Error getting associated real value", + "Context": "call_id {self}", + "Explanation": "Dynamo encountered an error while trying to get the associated real value.", + "Hints": [] + } + ], + "GB0049": [ + { + "Gb_type": "Error when attempting to resolve op packet", + "Context": "", + "Explanation": "str(e)", + "Hints": [] + } + ], + "GB0050": [ + { + "Gb_type": "Exception with bad expected type", + "Context": "str(expected_exc_types)", + "Explanation": "`except ...` has unsupported type {expected_exc_types}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0051": [ + { + "Gb_type": "Exception with non-type expectation", + "Context": "str(expected_type)", + "Explanation": "`except ...` expects a non-type: {expected_type}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0052": [ + { + "Gb_type": "Excessive RestartAnalysis() calls", + "Context": "", + "Explanation": "Dynamo attempted to trace the same frame 100+ times. Giving up on compiling as the compile time tradeoff is likely not worth the performance gain.", + "Hints": [] + } + ], + "GB0053": [ + { + "Gb_type": "FSDP with use_orig_params=False", + "Context": "", + "Explanation": "Dynamo only supports FSDP with use_orig_params=True", + "Hints": [] + } + ], + "GB0054": [ + { + "Gb_type": "Failed to construct Enum variable", + "Context": "value: {value_vt}, allowed enum values: {list(cls_type)}", + "Explanation": "Attempted to construct an Enum value that is non-constant (e.g. int, string) or is not an acceptable value for the Enum. Acceptable values for Enum `{cls_type}`: {list(cls_type)}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0055": [ + { + "Gb_type": "Failed to convert args/kwargs to proxy", + "Context": "call_function args: {typestr(*args)} {typestr(*list(kwargs.values()))}", + "Explanation": "Missing `as_proxy()` implementation for some arg/kwarg.", + "Hints": [] + } + ], + "GB0056": [ + { + "Gb_type": "Failed to mutate tensor data attribute", + "Context": "setattr({obj}, {name}, {val})", + "Explanation": "Dyanmo only supports mutating `.data` of tensor created outside `torch.compile` region", + "Hints": [ + "Don't mutate `.data` on this tensor, or move ", + "the mutation out of `torch.compile` region" + ] + } + ], + "GB0057": [ + { + "Gb_type": "Failed to raise exception", + "Context": "str(exc)", + "Explanation": "Attempted to raise a non-Exception type/value.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0058": [ + { + "Gb_type": "Failed to set tensor attribute", + "Context": "setattr({obj}, {name}, {val})", + "Explanation": "Dyanmo doesn't support setting these tensor attributes", + "Hints": [ + "Don't mutate attribute '{name}' on tensors, or ", + "move the mutation out of `torch.compile` region" + ] + } + ], + "GB0059": [ + { + "Gb_type": "Failed to trace builtin operator", + "Context": "builtin {fn.__name__} {arg_types} {has_kwargs}", + "Explanation": "Dynamo does not know how to trace builtin operator `{fn.__name__}` with argument types {real_arg_types} (has_kwargs {has_kwargs})", + "Hints": [ + "Avoid calling builtin `{fn.__name__}` with argument types {real_arg_types}. ", + "Consider using an equivalent alternative function/method to `{fn.__name__}`.", + "If you are attempting to call a logging function (e.g. `print`), ", + "you can try adding it to `torch._dynamo.config.reorderable_logging_functions`.", + "Please report an issue to PyTorch." + ] + } + ], + "GB0060": [ + { + "Gb_type": "Failed to trace unittest method", + "Context": "function: unittest.TestCase.{name}", + "Explanation": "Dynamo does not know how to trace unittest method `{name}` ", + "Hints": [ + "Avoid calling `TestCase.{name}`. ", + "Please report an issue to PyTorch." + ] + } + ], + "GB0061": [ + { + "Gb_type": "Failed to unpack object for BUILD_LIST_UNPACK", + "Context": "str(seq)", + "Explanation": "{seq} cannot be unpacked into a list for the BUILD_LIST_UNPACK bytecode (`[*x, *y, ...]`).", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0062": [ + { + "Gb_type": "Failed to unpack object for UNPACK_EX", + "Context": "str(seq)", + "Explanation": "{seq} cannot be unpacked into a list for the UNPACK_EX bytecode.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0063": [ + { + "Gb_type": "Failed to unpack object for UNPACK_SEQUENCE", + "Context": "str(seq)", + "Explanation": "{seq} cannot be unpacked into a list for the UNPACK_SEQUENCE bytecode (i.e. `a, b, c = d`).", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0064": [ + { + "Gb_type": "Fake tensor propagation exception", + "Context": "str(e.reason)", + "Explanation": "msg", + "Hints": [] + } + ], + "GB0065": [ + { + "Gb_type": "Graph break in inlined function", + "Context": "", + "Explanation": "Graph breaks in an inlined call are not supported.", + "Hints": [] + } + ], + "GB0066": [ + { + "Gb_type": "Graph break under GenericContextWrappingVariable", + "Context": "Active generic context managers: {self.active_generic_context_managers}", + "Explanation": "Attempted to graph break in an active context manager(s) that doesn't support graph breaking.", + "Hints": [ + "Move the offending context manager(s) to outside the compiled region.", + "This graph break may have been caused by an earlier graph break. Resolving the earlier graph break may resolve this one." + ] + } + ], + "GB0067": [ + { + "Gb_type": "HigherOrderOperator: Mutating a variable not in the current scope (SideEffects)", + "Context": "", + "Explanation": "This is not supported.", + "Hints": [] + } + ], + "GB0068": [ + { + "Gb_type": "Illegal method invocation in strict mode", + "Context": "call_method {self} {name} {args} {kwargs}", + "Explanation": "Dynamo currently does not support this method ({name}) invocation in strict mode.", + "Hints": [] + } + ], + "GB0069": [ + { + "Gb_type": "Import failure", + "Context": "module_name: {module_name}, fromlist: {fromlist}, level={level}", + "Explanation": "Failure when attempting to import.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0070": [ + { + "Gb_type": "Indexing list with non-scalar tensor", + "Context": "call_method {self} {name} {args} {kwargs}", + "Explanation": "Attempted to index list-like object with tensor with > 1 element.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0071": [ + { + "Gb_type": "Inline attempt with __self__", + "Context": "str(func)", + "Explanation": "Attempted to inline a function with the `__self__` attribute. Dynamo is expected to decompose method calls into function calls with a `self` argument.", + "Hints": [] + } + ], + "GB0072": [ + { + "Gb_type": "Inplace op on input tensor", + "Context": "", + "Explanation": "Attempted to trace an inplace view op on input tensor {typestr(self.value)}.", + "Hints": [ + "Ensure you do not modify input tensor in place.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0073": [ + { + "Gb_type": "Invoking an nn.Module inside a HigherOrderOperator", + "Context": "", + "Explanation": "This is not supported.", + "Hints": [] + } + ], + "GB0074": [ + { + "Gb_type": "Invoking an nn.Module inside a higher order operator", + "Context": "Higher order op name: {self.source_target}", + "Explanation": "This is not supported.", + "Hints": [] + } + ], + "GB0075": [ + { + "Gb_type": "LOAD_BUILD_CLASS bytecode not supported", + "Context": "", + "Explanation": "Dynamo does not support tracing classes that are defined in the compiled region.", + "Hints": [ + "Move the class definition out of the compiled region.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0076": [ + { + "Gb_type": "LOAD_FAST_CHECK on uninitialized variable", + "Context": "inst.argval", + "Explanation": "Attempted to load uninitialized local variable {inst.argval}", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0077": [ + { + "Gb_type": "Length mismatch when unpacking object for UNPACK_SEQUENCE", + "Context": "expected length: {inst.argval}, actual: {len(val)}", + "Explanation": "{seq} unpacked to a list for the UNPACK_SEQUENCE bytecode (i.e. `a, b, c = d`) with unexpected length.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0078": [ + { + "Gb_type": "Limitation of `nonstrict_trace", + "Context": "{self}", + "Explanation": "msg", + "Hints": [ + "make sure definition of {fn_name} is outside ", + "`torch.compile` region" + ] + } + ], + "GB0079": [ + { + "Gb_type": "Missing CALL_INTRINSIC_1 handler", + "Context": "CALL_INTRINSIC_1 operand: {inst.argval}", + "Explanation": "No handler implemented for CALL_INTRINSIC_1 {inst.argval} instruction.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0080": [ + { + "Gb_type": "Missing FakeTensor example value", + "Context": "str(node)", + "Explanation": "`FakeTensor` example value was required for {node} but not available.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0081": [ + { + "Gb_type": "Missing attribute when running call_method node", + "Context": "", + "Explanation": "make_error_message(\"attribute not defined\")", + "Hints": [] + } + ], + "GB0082": [ + { + "Gb_type": "Missing bytecode handler", + "Context": "{opname} with args {args}", + "Explanation": "Dynamo does not know how to handle the bytecode instruction `{opname}`.", + "Hints": [ + "Do not trace code that produces the `{opname}` bytecode instruction ", + "(see https://docs.python.org/3/library/dis.html for bytecode semantics).", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0083": [ + { + "Gb_type": "Module-level backwards hooks require compiled autograd.", + "Context": "", + "Explanation": "", + "Hints": [ + "Enable compiled autograd by setting torch._dynamo.config.compiled_autograd = True." + ] + } + ], + "GB0084": [ + { + "Gb_type": "Non-constant attribute given to `super().__delattr__()`", + "Context": "call_method {self} {name}", + "Explanation": "Dynamo requires the attribute name passed to `super().__delattr__(...)` to be a constant (string).", + "Hints": [ + "Ensure the attribute name is a string literal or a constant variable." + ] + } + ], + "GB0085": [ + { + "Gb_type": "Non-function or method in subclass of torch.autograd.Function", + "Context": "call_apply {self} {args} {kwargs}", + "Explanation": "Dynamo requires the `forward` attribute of a `torch.autograd.Function` subclass to be a standard Python function or method. Found type `{type(fn).__name__}` instead.", + "Hints": [ + "Ensure the `forward` method is defined as a regular ", + "function or instance method." + ] + } + ], + "GB0086": [ + { + "Gb_type": "Not a Python constant", + "Context": "guard_as_python_constant {self}", + "Explanation": "Failed to convert {self} into a Python constant.", + "Hints": [] + } + ], + "GB0087": [ + { + "Gb_type": "NotImplementedError/UnsupportedFakeTensorException when running FX node", + "Context": "", + "Explanation": "make_error_message(e)", + "Hints": [] + } + ], + "GB0088": [ + { + "Gb_type": "Observed exception", + "Context": "raised exception {curr_exc.python_type_name()}({curr_exc.args})", + "Explanation": "observed_exn_gb_explanation", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0089": [ + { + "Gb_type": "Observed exception (EXCEPT_HANDLER)", + "Context": "str(raised_exception)", + "Explanation": "observed_exn_gb_explanation + \" This graph break is unexpected.\"", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0090": [ + { + "Gb_type": "Operator does not support running with fake tensors", + "Context": "unsupported operator: {cause.func}", + "Explanation": "", + "Hints": [ + "{import_suggestion}see ", + "https://docs.google.com/document/d/1GgvOe7C8_NVOMLOCwDaYV1mXXyHMXY7ExoewHqooxrs/edit#heading=h.64r4npvq0w0", + " for how to fix" + ] + } + ], + "GB0091": [ + { + "Gb_type": "Read uninitialized cell", + "Context": "str(cellvar)", + "Explanation": "Attempted to read a cell variable that has not been populated yet.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0092": [ + { + "Gb_type": "Reconstruction failure", + "Context": "str(value)", + "Explanation": "Dynamo has no bytecode reconstruction implemented for sourceless variable {value}.", + "Hints": [ + "If Dynamo is attempting to trace a return statement and your code is attempting to return a variable ", + "that Dynamo cannot reconstruct, then remove it from the return statement.", + "Report an issue to PyTorch if you need reconstrtuction support. Note that objects that don't have ", + "reconstruction rules may be fundamentally unreconstructable.", + "This graph break may have been caused by an earlier graph break. Resolving the earlier graph break may resolve this one." + ] + } + ], + "GB0093": [ + { + "Gb_type": "Reconstruction failure: source.reconstruct not implemented", + "Context": "str(source)", + "Explanation": "Dynamo has no bytecode reconstruction implemented for {type(source)} variable {source}.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0094": [ + { + "Gb_type": "SEND with bad type", + "Context": "TOS type: {typestr(tos)}", + "Explanation": "Attempted to SEND with unsupported type {typestr(tos)}.", + "Hints": [] + } + ], + "GB0095": [ + { + "Gb_type": "Set Exception object `__traceback__` attribute to not-`None`", + "Context": "call_setattr {self} {name}", + "Explanation": "Dynamo does not support setting the attribute '__traceback__' on tracked exception objects to anything other than None.", + "Hints": [ + "Avoid setting '__traceback__' on exception objects ", + "within traced code, or set it to None." + ] + } + ], + "GB0096": [ + { + "Gb_type": "Should not compile partial graph (STORE_ATTR)", + "Context": "", + "Explanation": "Dynamo has determined when encountering an unsupported STORE_ATTR instruction (i.e. `obj.attr = val`) that it should not compile the partial graph.", + "Hints": [] + } + ], + "GB0097": [ + { + "Gb_type": "Side effect on existing deque with limited maxlen", + "Context": "", + "Explanation": "This is not supported.", + "Hints": [ + "Don't use a deque with `maxlen` specified." + ] + } + ], + "GB0098": [ + { + "Gb_type": "Skip calling `torch.compiler.disable()`d function", + "Context": "str(self.value)", + "Explanation": "Skip calling function `{self.value}` since it was wrapped with `torch.compiler.disable` (reason: {msg})", + "Hints": [ + "Remove the `torch.compiler.disable` call" + ] + } + ], + "GB0099": [ + { + "Gb_type": "Skip inlining `torch.compiler.disable()`d function", + "Context": "str(func.get_function())", + "Explanation": "Skip inlining function {func.get_function()} since it was wrapped with `torch.compiler.disable` (reason: {msg})", + "Hints": [ + "Remove the `torch.compiler.disable` call" + ] + } + ], + "GB0100": [ + { + "Gb_type": "Storing Tensor hook handle in globals", + "Context": "name", + "Explanation": "This is not supported.", + "Hints": [] + } + ], + "GB0101": [ + { + "Gb_type": "Storing Tensor hook handle in globals (inline call)", + "Context": "inst.argval", + "Explanation": "This is not supported.", + "Hints": [] + } + ], + "GB0102": [ + { + "Gb_type": "Strict mode banned op", + "Context": "var_getattr {self} {name}", + "Explanation": "Getattr invocation '{name}' in strict mode is not supported.", + "Hints": [ + "Remove `{name}` from the list of banned ops by ", + "setting `torch._dynamo.config._autograd_backward_strict_mode_banned_ops`." + ] + } + ], + "GB0103": [ + { + "Gb_type": "Tensor subclass overridden method call", + "Context": "{name}", + "Explanation": "`torch.compile` currently can't trace this", + "Hints": [ + "Avoid calling {name} of tensor subclass in torch.compile region", + "Renaming method `{name}` of type {self.class_type}", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0104": [ + { + "Gb_type": "Tensor with grad_fn()", + "Context": "var_getattr {self} grad_fn", + "Explanation": "Dynamo does not support tracing tensors with a grad_fn directly.", + "Hints": [] + } + ], + "GB0105": [ + { + "Gb_type": "Tensor.numpy() with trace_numpy=False", + "Context": "call_method {self} numpy", + "Explanation": "`Tensor.numpy()` was called, but the `trace_numpy` configuration was manually disabled.", + "Hints": [ + "Set `torch._dynamo.config.trace_numpy = True` to allow ", + "Dynamo to trace through NumPy." + ] + } + ], + "GB0106": [ + { + "Gb_type": "Tensor.numpy() without NumPy installed", + "Context": "call_method {self} numpy", + "Explanation": "`Tensor.numpy()` was called, but the NumPy library is not available in the current environment.", + "Hints": [ + "Ensure NumPy is installed in your Python environment." + ] + } + ], + "GB0107": [ + { + "Gb_type": "Tensor.random_ op", + "Context": "Tensor.{name}(args={args}, kwargs={kwargs})", + "Explanation": "This is currently not supported.", + "Hints": [ + "Use the out-of-place version of this op", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0108": [ + { + "Gb_type": "Tensor.retain_grad() with AOTDispatcher", + "Context": "var_getattr {self} retain_grad", + "Explanation": "`Tensor.retain_grad()` does not work with AOTDispatcher.", + "Hints": [] + } + ], + "GB0109": [ + { + "Gb_type": "Tensor.tolist() with non-integer tensor", + "Context": "call_method {self} to_list", + "Explanation": "Dynamo currently does not support tracing `tolist()` on non-integer tensors.", + "Hints": [ + "Ensure the input tensor to `tolist()` is an integer ", + "type (e.g., int8, int16, int32, int64)." + ] + } + ], + "GB0110": [ + { + "Gb_type": "Tensor.uniform_ op called with `from` keyword", + "Context": "Tensor.{name}(args={args}, kwargs={kwargs})", + "Explanation": "This is currently not supported.", + "Hints": [ + "Avoid using the `from` keyword.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0111": [ + { + "Gb_type": "TypeError from user code", + "Context": "call_function({self.value}, {args}, {kwargs})", + "Explanation": "msg", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0112": [ + { + "Gb_type": "TypeError when making fake tensor call", + "Context": "TypeError {node.target}: {cause}", + "Explanation": "", + "Hints": [] + } + ], + "GB0113": [ + { + "Gb_type": "Unable to resolve super getattr", + "Context": "", + "Explanation": "Dynamo failed to trace attribute `{name}` accessed via `super()` (for type `{self.typevar}` and object `{self.objvar}`) because the resolved attribute type is not supported.", + "Hints": [ + "Ensure the attribute exists in the parent class.", + "Check the arguments passed to `super()`." + ] + } + ], + "GB0114": [ + { + "Gb_type": "Unexpected failure during itertools.accumulate() iteration", + "Context": "call_function {self} {args} {kwargs}", + "Explanation": "Unexpected failure in invoking function during accumulate. Failed running func {func}({item}{acc})", + "Hints": [ + "This graph break may be difficult to debug. Please report an issue to PyTorch for assistance." + ] + } + ], + "GB0115": [ + { + "Gb_type": "Unexpected failure during itertools.groupby() iteration", + "Context": "call_function {self} {args} {kwargs}", + "Explanation": "Unexpected failure in invoking function during groupby", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0116": [ + { + "Gb_type": "Unexpected type in sourceless builder", + "Context": "{value_type.__module__}.{value_type.__qualname__}", + "Explanation": "SourcelessBuilder.create does not know how to wrap {value_type}", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0117": [ + { + "Gb_type": "Unhandled args for method", + "Context": "call_method {self} {name} {args} {kwargs}", + "Explanation": "Dynamo encountered an error while calling the method `{name}`.", + "Hints": [] + } + ], + "GB0118": [ + { + "Gb_type": "Unimplemented next() call", + "Context": "next({self})", + "Explanation": "This abstract method must be implemented", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0119": [ + { + "Gb_type": "Uninitialized nn.Module", + "Context": "typestr(value)", + "Explanation": "Attempted to trace an uninitialized nn.Module of type {typestr(value)}.", + "Hints": [ + "Ensure your nn.Module instance has called `super().__init__()`.", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0120": [ + { + "Gb_type": "Unreachable sub-generator code", + "Context": "", + "Explanation": "Should only be encountered while implementing generator support.", + "Hints": [] + } + ], + "GB0121": [ + { + "Gb_type": "UnspecializedNNModuleVariable missing method", + "Context": "call_method: {self} {name} {args} {kwargs}", + "Explanation": "Dynamo does not support tracing method {name} of nn.Module {self.value}", + "Hints": [ + "Dynamo does not really define unspecialized nn.Module very well.", + "This graph break may be difficult to debug. Please report an issue to PyTorch for assistance." + ] + } + ], + "GB0122": [ + { + "Gb_type": "Unsupported SourceType", + "Context": "MutationType.__init__ {self} {typ}", + "Explanation": "Dynamo does not support the type `{typ}`", + "Hints": [ + "This branch is not supposed to be reachable.", + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0123": [ + { + "Gb_type": "Unsupported Tensor.backward() call", + "Context": "call_method {self} backward {args} {kwargs}", + "Explanation": "Dynamo currently does not support tracing `Tensor.backward()`.", + "Hints": [ + "This graph break is fundamental - it is unlikely that Dynamo will ever be able to trace through your code. Consider finding a workaround." + ] + } + ], + "GB0124": [ + { + "Gb_type": "Unsupported Tensor.item() call with capture_scalar_outputs=False", + "Context": "call_method {self} item {args} {kwargs}", + "Explanation": "Dynamo does not support tracing `Tensor.item()` with config.capture_scalar_outputs=False.", + "Hints": [ + "Set `torch._dynamo.config.capture_scalar_outputs = True` ", + "or `export TORCHDYNAMO_CAPTURE_SCALAR_OUTPUTS=1` ", + "to include these operations in the captured graph." + ] + } + ], + "GB0125": [ + { + "Gb_type": "Unsupported Tensor.requires_grad_() call", + "Context": "call_method {self} requires_grad_", + "Explanation": "Dynamo does not support changes to a Tensor's `requires_grad` through calling `requires_grad_()`.", + "Hints": [] + } + ], + "GB0126": [ + { + "Gb_type": "Unsupported Tensor.resize_() call", + "Context": "call_method {self} resize_ {args} {kwargs}", + "Explanation": "Dynamo currently does not support tracing `Tensor.resize_()`.", + "Hints": [] + } + ], + "GB0127": [ + { + "Gb_type": "Unsupported Tensor.resize_as_() call", + "Context": "call_method {self} resize_as_ {args} {kwargs}", + "Explanation": "Dynamo currently does not support tracing `Tensor.resize_as_()`.", + "Hints": [] + } + ], + "GB0128": [ + { + "Gb_type": "Unsupported Tensor.set_() call", + "Context": "call_method {self} set_ {args} {kwargs}", + "Explanation": "Dynamo currently does not support tracing `Tensor.set_()` overloads that include more than one argument.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0129": [ + { + "Gb_type": "Unsupported Tensor.sparse_resize_() call", + "Context": "call_method {self} sparse_resize_ {args} {kwargs}", + "Explanation": "Dynamo currently does not support tracing `Tensor.sparse_resize_()`.", + "Hints": [] + } + ], + "GB0130": [ + { + "Gb_type": "Unsupported Tensor.sparse_resize_and_clear_() call", + "Context": "call_method {self} sparse_resize_and_clear_ {args} {kwargs}", + "Explanation": "Dynamo currently does not support tracing `Tensor.sparse_resize_and_clear_()`.", + "Hints": [] + } + ], + "GB0131": [ + { + "Gb_type": "Unsupported __setitem__/__setattr__ inline attempt", + "Context": "code name: {code.co_name}, args: {args}", + "Explanation": "Attempted to inline {code.co_name} where first argument (self) is not a user-defined object.", + "Hints": [] + } + ], + "GB0132": [ + { + "Gb_type": "Unsupported `func` in itertools.accumulate", + "Context": "call_function {self} {args} {kwargs}", + "Explanation": "Dynamo does not know how to get the function to use for itertools.accumulate. itertools.accumulate expects the `func` as the second argument or as a keyword argument.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0133": [ + { + "Gb_type": "Unsupported arguments for itertools.accumulate", + "Context": "call_function {self} {args} {kwargs}", + "Explanation": "Dynamo does not know how to trace itertools.accumulate with args: {args} and kwargs: {kwargs}. itertools.accumulate expects an iterable, an optional binary function for accumulation, and an optional initial value to set the starting state.", + "Hints": [ + "Make sure the arguments to itertools.accumulate are correct.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0134": [ + { + "Gb_type": "Unsupported arguments for itertools.groupby", + "Context": "call_function {self} {args} {kwargs}", + "Explanation": "Dynamo does not know how to trace itertools.groupby with args: {args} and kwargs: {kwargs}. itertools.groupby expects an iterable to group and an optional key function to determine groupings.", + "Hints": [ + "Make sure the arguments to itertools.groupby are correct.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0135": [ + { + "Gb_type": "Unsupported attribute assignment on Exception object", + "Context": "call_setattr {self} {name}", + "Explanation": "Dynamo does not support setting the attribute '{name}' on tracked exception objects. Only `__context__`, `__cause__`, `__suppress_context__`, and `__traceback__` are supported.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0136": [ + { + "Gb_type": "Unsupported attribute for range() object", + "Context": "var_getattr {self} {name}", + "Explanation": "Expected attribute to be one of {','.join(fields)} but got {name}", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0137": [ + { + "Gb_type": "Unsupported attribute for slice() object", + "Context": "var_getattr {self} {name}", + "Explanation": "Expected attribute to be one of {','.join(fields)} but got {name}", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0138": [ + { + "Gb_type": "Unsupported autograd.Function context `save_for_backward`", + "Context": "call_method {self} {name}", + "Explanation": "Dynamo requires the `saved_tensors` attribute to be initialized on the `autograd.Function` context object.", + "Hints": [ + "Ensure that the `saved_tensors` attribute is properly ", + "initialized before calling `save_for_backward`. ", + "`save_for_backward` only supported on a newly constructed `torch.autograd.function.FunctionCtx`." + ] + } + ], + "GB0139": [ + { + "Gb_type": "Unsupported autograd.Function context method", + "Context": "call_method {self} {name}", + "Explanation": "Dynamo does not support calling the method `{name}` on `autograd.Function` context objects. Supported methods are `__setattr__`, `save_for_backward` and `mark_non_differentiable`.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0140": [ + { + "Gb_type": "Unsupported autograd.Function method", + "Context": "call_method {self} {name}", + "Explanation": "Dynamo does not support calling the method `{name}` directly on the `torch.autograd.Function` instance. Supported methods include `apply`, `backward`, static methods, and class methods.", + "Hints": [ + "Ensure the method is decorated with `@staticmethod` ", + "or `@classmethod` if it's meant to be called on the class." + ] + } + ], + "GB0141": [ + { + "Gb_type": "Unsupported call_id() without source", + "Context": "call_id {self}", + "Explanation": "call_id() not supported for sourceless TensorVariable.", + "Hints": [] + } + ], + "GB0142": [ + { + "Gb_type": "Unsupported context manager", + "Context": "Attempted SETUP_WITH/BEFORE_WITH/LOAD_SPECIAL on {ctx}", + "Explanation": "Dynamo does not know how to enter a `{ctx.python_type_name()}` context manager.", + "Hints": [ + "Avoid using the unsupported context manager.", + "If the context manager seems like it should be supported (e.g. torch.set_grad_enabled), then ", + "it may be the case that it was created outside the compiled region, which Dynamo does not support. ", + "Supported context managers can cross graph break boundaries only if they are local non-closure ", + "variables, or are intermediate values.", + "File an issue to PyTorch. Simple context managers can potentially be supported, ", + "but note that context managers can't be supported in general" + ] + }, + { + "Gb_type": "Unsupported context manager", + "Context": "Attempted SETUP_WITH/BEFORE_WITH on {ctx}", + "Explanation": "Dynamo does not know how to enter a `{ctx.python_type_name()}` context manager.", + "Hints": [ + "Avoid using the unsupported context manager.", + "If the context manager seems like it should be supported (e.g. torch.set_grad_enabled), then ", + "it may be the case that it was created outside the compiled region, which Dynamo does not support. ", + "Supported context managers can cross graph break boundaries only if they are local non-closure ", + "variables, or are intermediate values.", + "File an issue to PyTorch. Simple context managers can potentially be supported, ", + "but note that context managers can't be supported in general" + ] + } + ], + "GB0143": [ + { + "Gb_type": "Unsupported conversion for slice assignment", + "Context": "call_method {self} {name} {args}", + "Explanation": "Missing dynamo support for converting {value} into a list for slice assignment.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0144": [ + { + "Gb_type": "Unsupported custom jvp", + "Context": "call_apply {self} {args} {kwargs}", + "Explanation": "Dynamo does not support tracing `torch.autograd.Function` subclasses that define a custom `jvp` method.", + "Hints": [ + "Remove the custom `jvp` method if possible.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0145": [ + { + "Gb_type": "Unsupported custom vjp", + "Context": "call_apply {self} {args} {kwargs}", + "Explanation": "Dynamo does not support tracing `torch.autograd.Function` subclasses that define a custom `vjp` method.", + "Hints": [ + "Remove the custom `vjp` method if possible.", + "Use standard `backward` instead if applicable.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0146": [ + { + "Gb_type": "Unsupported event method", + "Context": "str(name)", + "Explanation": "Dynamo doesn't support tracing the {method_name} method. We currently support wait, record, synchronize, and query.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0147": [ + { + "Gb_type": "Unsupported function call", + "Context": "call_function {self} {args} {kwargs}", + "Explanation": "Dynamo does not know how to trace the function `{self.debug_repr()}`", + "Hints": [ + "Avoid calling `{self.debug_repr()}` in your code.", + "Please report an issue to PyTorch." + ] + } + ], + "GB0148": [ + { + "Gb_type": "Unsupported function call (delayed)", + "Context": "source: {self.source}", + "Explanation": "Dynamo determined that a graph break should occur when calling `{self.source.name}`. Reason: {self.msg}", + "Hints": [] + } + ], + "GB0149": [ + { + "Gb_type": "Unsupported functorch tracing attempt", + "Context": "", + "Explanation": "msg", + "Hints": [] + } + ], + "GB0150": [ + { + "Gb_type": "Unsupported hasattr call", + "Context": "call_obj_hasattr {self} {name}", + "Explanation": "Dynamo does not know how to trace the function `{self.debug_repr()}`", + "Hints": [ + "Avoid calling `hasattr({self.__class__.__name__}, {name})` in your code.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0151": [ + { + "Gb_type": "Unsupported inspect call", + "Context": "inspect_parameter_names {self}", + "Explanation": "Dynamo does not know how to trace the function `{self.debug_repr()}`", + "Hints": [] + } + ], + "GB0152": [ + { + "Gb_type": "Unsupported key type for itertools.groupby", + "Context": "call_function {self} {args} {kwargs}", + "Explanation": "Dynamo does not know how to trace itertools.groupby with key type: {str(type(key))}. We only support grouping keys that are constants (int, float, str, etc.)", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0153": [ + { + "Gb_type": "Unsupported key type for nn.Module.__getitem__", + "Context": "call_method: {self} {name} {args} {kwargs}", + "Explanation": "Dynamo does not support getitem on `nn.Module` with non-constant key.", + "Hints": [] + } + ], + "GB0154": [ + { + "Gb_type": "Unsupported kwargs for itertools.accumulate", + "Context": "call_function {self} {args} {kwargs}", + "Explanation": "Expected kwargs: 'initial', 'func', but got {','.join(set(kwargs.keys()) - {'initial', 'func'})}", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0155": [ + { + "Gb_type": "Unsupported kwargs for itertools.groupby", + "Context": "call_function {self} {args} {kwargs}", + "Explanation": "Expected kwargs: 'key', but got {','.join(set(kwargs.keys()) - {'key'})}", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0156": [ + { + "Gb_type": "Unsupported method call", + "Context": "call_method {self} {name} {args} {kwargs}", + "Explanation": "Dynamo does not know how to trace method `{name}` of class `{self.python_type_name()}`", + "Hints": [] + } + ], + "GB0157": [ + { + "Gb_type": "Unsupported ndarray attribute access", + "Context": "var_getattr {self} {name}", + "Explanation": "Dynamo currently does not support tracing `ndarray.{name}`.", + "Hints": [] + } + ], + "GB0158": [ + { + "Gb_type": "Unsupported ndarray method call", + "Context": "call_method {self} {name} {args} {kwargs}", + "Explanation": "`ndarray.{name}()` is not modelled in `torch._numpy`.", + "Hints": [] + } + ], + "GB0159": [ + { + "Gb_type": "Unsupported ndarray.__version__ access", + "Context": "var_getattr {self} {name}", + "Explanation": "Dynamo currently does not support tracing `ndarray.{name}`.", + "Hints": [] + } + ], + "GB0160": [ + { + "Gb_type": "Unsupported next() call", + "Context": "next({self})", + "Explanation": "Dynamo does not know how to trace calling `next()` on variable `{self}`.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0161": [ + { + "Gb_type": "Unsupported nn.Module attribute type", + "Context": "nn.Module subclass: {typestr(base)}, name: {name}, attribute type: {typestr(subobj)}", + "Explanation": "Dynamo does not support tracing nn.Module attributes of type `{typestr(subobj)}`", + "Hints": [ + "Refactor your code so that `{name}` (type `{typestr(subobj)}`) is not an attribute of `{typestr(base)}`", + "Currently supported attribute types are methods, classmethods, staticmethods, ", + "properties, constants, and tensors.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0162": [ + { + "Gb_type": "Unsupported super().__init__() call", + "Context": "call_method {self} {name} {args} {kwargs}", + "Explanation": "Dynamo encountered a super().__init__() call on {objvar} that resolved to a `torch.nn.Module.__init__()` call that we cannot trace.", + "Hints": [ + "This graph break may be difficult to debug. Please report an issue to PyTorch for assistance." + ] + } + ], + "GB0163": [ + { + "Gb_type": "Unsupported tensor subclass attribute access", + "Context": "{name}", + "Explanation": "`torch.compile` currently can't trace this", + "Hints": [ + "Avoid accessing {name} of tensor subclass in torch.compile region", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0164": [ + { + "Gb_type": "Unsupported tensor subclass overridden attribute access", + "Context": "{name}", + "Explanation": "`torch.compile` only support tracing certain types of overridden tensor subclass attributes", + "Hints": [ + "Avoid accessing {name} of tensor subclass in torch.compile region", + "Renaming attribute `{name}` of type {self.class_type}", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0165": [ + { + "Gb_type": "Unsupported torch._C._ImperativeEngine method", + "Context": "call_method {self} {name}", + "Explanation": "Dynamo only supports the `queue_callback` method on a torch._C._ImperativeEngine instance, but found: `{name}`.", + "Hints": [] + } + ], + "GB0166": [ + { + "Gb_type": "Unsupported torch._C._ImperativeEngine.queue_callback()", + "Context": "call_method {self} {name}", + "Explanation": "queue_callback() is only supported when Compiled Autograd is enabled with fullgraph=True.", + "Hints": [] + } + ], + "GB0167": [ + { + "Gb_type": "Variadic function call with bad args/kwargs type", + "Context": "args type: {typestr(argsvars)}, kwargs type: {typestr(kwargsvars)}", + "Explanation": "Expected args to be a list and kwargs to be a dict", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0168": [ + { + "Gb_type": "Variadic function call with bad flags", + "Context": "flags: {inst.argval}", + "Explanation": "Attempted to call a variadic function (CALL_FUNCTION_EX) with bad flags {inst.argval}", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0169": [ + { + "Gb_type": "Write to immutable cell", + "Context": "cellvar: {cellvar}, value: {value}", + "Explanation": "Dynamo doesn't support writing to immutable/sourceless cell variables.", + "Hints": [ + "This graph break may be difficult to debug. Please report an issue to PyTorch for assistance." + ] + } + ], + "GB0170": [ + { + "Gb_type": "Data-dependent branching", + "Context": "attempted to jump with {value}", + "Explanation": "_explanation", + "Hints": [ + "Use `torch.cond` to express dynamic control flow.", + "This graph break is fundamental - it is unlikely that Dynamo will ever be able to trace through your code. Consider finding a workaround." + ] + }, + { + "Gb_type": "Data-dependent branching", + "Context": "attempted to jump with {value}", + "Explanation": "_explanation", + "Hints": [] + }, + { + "Gb_type": "_gb_type", + "Context": "attempted to jump with {value}", + "Explanation": "_explanation", + "Hints": [] + } + ], + "GB0171": [ + { + "Gb_type": "assert with non-string message", + "Context": "str(args)", + "Explanation": "Dynamo only supports asserts with string messages", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0172": [ + { + "Gb_type": "async_op=True for distributed collectives", + "Context": "{self.fn}, args={args}, kwargs={kwargs}", + "Explanation": "`torch.compile` doesn't support `async_op=True for {self.fn}", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0173": [ + { + "Gb_type": "backward_state does not support export", + "Context": "", + "Explanation": "Compiled autograd doesn't work with `torch.export`.", + "Hints": [] + } + ], + "GB0174": [ + { + "Gb_type": "bad args to builtin cast()", + "Context": "got args {args} {kwargs}", + "Explanation": "Dynamo expects exactly 2 args to builtin cast().", + "Hints": [ + "Ensure your call to cast() has exactly 2 arguments." + ] + } + ], + "GB0175": [ + { + "Gb_type": "builtin isinstance() cannot determine type of argument", + "Context": "isinstance({arg}, {isinstance_type_var})", + "Explanation": "Dynamo doesn't have a rule to determine the type of argument {arg}", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + }, + { + "Gb_type": "builtin isinstance() cannot determine type of argument", + "Context": "isinstance({arg}, {isinstance_type})", + "Explanation": "Dynamo doesn't have a rule to determine the type of argument {arg}", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0176": [ + { + "Gb_type": "call_id() without associated real value", + "Context": "call_id {self}", + "Explanation": "Dynamo could not find an associated real value for the tensor.", + "Hints": [] + } + ], + "GB0177": [ + { + "Gb_type": "can't handle functions not implemented in python ", + "Context": "{fn}", + "Explanation": "Dynamo can only handle functions defined in python", + "Hints": [ + "Move usage of this function out of `torch.compile` region", + "Avoid using `tensor.is_inference()` and `torch.is_inference_mode_enabled()` in your compile code. This is primarily used in conjunction with `torch.inference_mode`. Consider using `torch.no_grad` instead because `torch.no_grad` leads to same improvements as `inference_mode` when `torch.compile` is used." + ] + } + ], + "GB0178": [ + { + "Gb_type": "constant fold exception", + "Context": "attempted to run function {fn} with arguments {args}", + "Explanation": "Encountered exception when attempting to constant fold.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0179": [ + { + "Gb_type": "copy.deepcopy()", + "Context": "copy.deepcopy({x})", + "Explanation": "Dynamo does not support copy.deepcopy()", + "Hints": [ + "Avoid calling copy.deepcopy()", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0180": [ + { + "Gb_type": "dataclass fields failure", + "Context": "obj: {obj}; variable type: {type(obj)}", + "Explanation": "Dataclass fields handling fails for {obj}. Expected it to be a user-defined object.", + "Hints": [] + } + ], + "GB0181": [ + { + "Gb_type": "dtype mismatch between tensor and its gradient", + "Context": "tensor dtype: {value.dtype}; grad dtype: {safe_grad(value).dtype}", + "Explanation": "Inconsistent dtype between tensor and its gradient. This can happen in FSDP and crashes meta tensor creation.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0182": [ + { + "Gb_type": "failed to broadcast when attempting Tensor comparison op", + "Context": "{op.__name__}({left}, {right})", + "Explanation": "Dynamo was unable to broad cast the arguments {left}, {right} when attempting to trace the comparison op {op.__name__}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0183": [ + { + "Gb_type": "failed to call dict.fromkeys()", + "Context": "{user_cls.__name__}.fromkeys(): {args} {kwargs}", + "Explanation": "Failed to call {user_cls.__name__}.fromkeys() because arguments could not be automatically converted to a list, or some dict key is not hashable.", + "Hints": [ + "Manually convert the argument to a list.", + "Ensure all keys are hashable." + ] + } + ], + "GB0184": [ + { + "Gb_type": "failed to call str() on user defined object", + "Context": "str(arg)", + "Explanation": "User defined object has no __str__ or __repr__ method", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0185": [ + { + "Gb_type": "failed to convert numpy.ndarray to Tensor", + "Context": "str(value)", + "Explanation": "Exception encountered when attempting to convert numpy.ndarray to Tensor", + "Hints": [] + } + ], + "GB0186": [ + { + "Gb_type": "functools.partial() with non-literal keyword", + "Context": "non-literal keyword: {k}", + "Explanation": "functools.partial() expects literal/string keywords", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0187": [ + { + "Gb_type": "functools.wraps", + "Context": "{fn}", + "Explanation": "`torch.compile` can't trace `functools.wraps` on functions defined outside the compile region", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0188": [ + { + "Gb_type": "getattr with no source", + "Context": "var_getattr {self} {name}", + "Explanation": "Dynamo does not know how to access an attribute on an `nn.Module` instance that lacks a source. This is usually an internal error in Dynamo.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0189": [ + { + "Gb_type": "getattr() on nn.Module with pending mutation", + "Context": "getattr({obj}, {name}, {default})", + "Explanation": "Intentionally graph breaking on getattr() on a nn.Module with a pending mutation", + "Hints": [] + } + ], + "GB0190": [ + { + "Gb_type": "getattr() with non-constant name argument", + "Context": "getattr({obj}, {name_var}, {default})", + "Explanation": "getattr() with non-constant name argument is not supported", + "Hints": [ + "Ensure the name argument of getattr() is a string" + ] + } + ], + "GB0191": [ + { + "Gb_type": "id() with unsupported args", + "Context": "str(args)", + "Explanation": "Dynamo doesn't know how to trace id() call with args {args}", + "Hints": [ + "Supported args are Tensors, and functions/nn.Modules/user-defined objects ", + "from outside the compiled region.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0192": [ + { + "Gb_type": "input iterator to itertools.cycle has too many items", + "Context": "next({self})", + "Explanation": "Has reached internal Dynamo max iterator limit: {MAX_ITERATOR_LIMIT}", + "Hints": [] + } + ], + "GB0193": [ + { + "Gb_type": "invalid call to builtin op handler", + "Context": "invalid args to {self_handler}: {args} {kwargs}", + "Explanation": "Encountered TypeError when trying to handle op {fn.__name__}", + "Hints": [ + "This graph break may be difficult to debug. Please report an issue to PyTorch for assistance." + ] + } + ], + "GB0194": [ + { + "Gb_type": "isinstance() called on user defined object with C extensions", + "Context": "isinstance({arg}, {isinstance_type})", + "Explanation": "User-defined object with C extensions can have torch.Tensor attributes; intentionally graph breaking.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0195": [ + { + "Gb_type": "issubclass() with non-constant arguments", + "Context": "issubclass({left_ty}, {right_ty})", + "Explanation": "issubclass() with non-constant arguments not supported.", + "Hints": [ + "Make sure your arguments are types.", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0196": [ + { + "Gb_type": "key not found in dict", + "Context": "Key {arg.value}", + "Explanation": "msg", + "Hints": [ + "Check if the key exists in the dictionary before accessing it.", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0197": [ + { + "Gb_type": "list elements are pointing to the list itself", + "Context": "", + "Explanation": "Dynamo does not support lists whose items reference to itself", + "Hints": [ + "Avoid using self referential list" + ] + } + ], + "GB0198": [ + { + "Gb_type": "mapping proxy affected by dictionary mutation", + "Context": "Source: {self.source}, Dict mutation detected", + "Explanation": "msg", + "Hints": [ + "Avoid modifying dictionaries that might be referenced by mapping proxy objects", + "Or avoid using the mapping proxy objects after modifying its underlying dictionary" + ] + } + ], + "GB0199": [ + { + "Gb_type": "mapping proxy cannot be reconstructed", + "Context": "Source: {self.source}", + "Explanation": "msg", + "Hints": [ + "Use a mapping proxy constructed in the same `torch.compile` region.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0200": [ + { + "Gb_type": "missing BUILD_SET handler", + "Context": "", + "Explanation": "Missing BUILD_SET bytecode handler (for testing purposes).", + "Hints": [] + } + ], + "GB0201": [ + { + "Gb_type": "namedtuple construction", + "Context": "args={args}, kwargs={kwargs}", + "Explanation": "`torch.compile` only support certain input types for namedtuple", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0202": [ + { + "Gb_type": "non-const argument in nn.Module method", + "Context": "call_method: {self} {name} {args} {kwargs}", + "Explanation": "Dynamo does not support calling method `{name}` of ``nn.Module`` {module} with non-constant arguments.", + "Hints": [] + } + ], + "GB0203": [ + { + "Gb_type": "non-const keys in dict_keys", + "Context": "non-const keys: {[k for k in value if not ConstantVariable.is_literal(k)]}", + "Explanation": "Dynamo expects dict_keys keys to be constants.", + "Hints": [ + "Ensure your dict_keys keys are constants (e.g. int, float, strings)" + ] + } + ], + "GB0204": [ + { + "Gb_type": "non-const keys in mappingproxy", + "Context": "non-const keys: {[k for k in value.keys() if not ConstantVariable.is_literal(k)]}", + "Explanation": "Dynamo expects mappingproxy keys to be constants.", + "Hints": [ + "Ensure your mappingproxy keys are constants (e.g. int, float, strings)" + ] + } + ], + "GB0205": [ + { + "Gb_type": "proxy not set", + "Context": "as_proxy {self}", + "Explanation": "Dynamo requires the autograd.Function context to be initialized with a proxy.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0206": [ + { + "Gb_type": "setattr() on Tensor.requires_grad", + "Context": "setattr({obj}, {name}, {val})", + "Explanation": "setattr() on Tensor.requires_grad not supported. Mutating requires_grad can introduce a new leaf from non-leaf or vice versa in the middle of the graph, which AOTAutograd does not currently know how to handle.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0207": [ + { + "Gb_type": "sort with non-constant keys", + "Context": "str(first_non_constant_key)", + "Explanation": "Cannot perform sort with non-constant key. First non-constant key type: {python_type}. Most notably, we cannot sort with Tensor or SymInt keys, but we can sort ints.", + "Hints": [ + "Use something else as the key." + ] + } + ], + "GB0208": [ + { + "Gb_type": "torch.* op returned non-Tensor", + "Context": "example_value type: {typestr(example_value)}; op: {proxy.node.op}; target: {proxy.node.target}", + "Explanation": "torch.* ops that return a non-Tensor cannot be traced into the Dynamo FX graph output", + "Hints": [] + } + ], + "GB0209": [ + { + "Gb_type": "torch.autograd._unsafe_preserve_version_counter escaped from compiled region", + "Context": "str(self)", + "Explanation": "Dynamo doesn't support compiling a region that returns a torch.autograd._unsafe_preserve_version_counter context manager.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0210": [ + { + "Gb_type": "torch.distributed package is not available!", + "Context": "", + "Explanation": "The PyTorch package doesn't include torch.distributed when building from source.", + "Hints": [ + "Set USE_DISTRIBUTED=1 to enable it when building PyTorch from source." + ] + } + ], + "GB0211": [ + { + "Gb_type": "torch.nn.Module with a non-function custom __getattr__", + "Context": "var_getattr {self} {name}", + "Explanation": "Dynamo detected a nn.Module object with a custom `__getattr__` method, but this method is not a standard Python function (e.g., it might be implemented in C/C++). Dynamo cannot currently trace into such non-standard `__getattr__` methods.", + "Hints": [ + "Avoid using objects with non-standard __getattr__ methods ", + "within the compiled region. If possible, implement ", + "__getattr__ as a standard Python function.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0212": [ + { + "Gb_type": "torch.profiler object escaped from compiled region", + "Context": "str(self)", + "Explanation": "Dynamo doesn't support compiling a region that returns a torch.profiler context manager.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0213": [ + { + "Gb_type": "unimplemented builtin op on tensor arguments", + "Context": "partial tensor op: {self} {args} {kwargs}", + "Explanation": "Dynamo does not know how to trace builtin operator {self.fn} with tensor arguments", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0214": [ + { + "Gb_type": "unsupported SymNode comparison op", + "Context": "{op.__name__}({left}, {right})", + "Explanation": "Dynamo does not support the comparison op {op.__name__} with SymNode arguments {left}, {right}", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0215": [ + { + "Gb_type": "unsupported Tensor comparison op", + "Context": "{op.__name__}({left}, {right})", + "Explanation": "Dynamo does not support the comparison op {op.__name__} with Tensor arguments {left}, {right}", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0216": [ + { + "Gb_type": "unsupported grid type for triton hop check_grid", + "Context": "grid type = {type(grid)}", + "Explanation": "`torch.compile` only supports list-like grid for check_grid", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0217": [ + { + "Gb_type": "unsupported hasattr operation", + "Context": "Class {self.user_cls}", + "Explanation": "msg", + "Hints": [ + "Consider using a regular dictionary instead", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0218": [ + { + "Gb_type": "unsupported index(Tensor)", + "Context": "", + "Explanation": "Dynamo does not support tracing builtin index() on a Tensor", + "Hints": [] + } + ], + "GB0219": [ + { + "Gb_type": "Backend compiler exception", + "Context": "Backend: {name}\nException:{str(e)}\nTraceback:\n{self.root_tx.format_frame_summary()}", + "Explanation": "Backend compiler `{name}` failed with {str(e)}. Adding a graph break.", + "Hints": [ + "Report an issue to the backend compiler repo." + ] + } + ], + "GB0220": [ + { + "Gb_type": "Failed to mutate tensor data attribute to different dtype", + "Context": "setattr({obj}, {name}, {val})", + "Explanation": "Dyanmo only supports mutating `.data` of tensor to a new one with the same dtype", + "Hints": [ + "Don't mutate `.data` on this tensor, or move ", + "the mutation out of `torch.compile` region" + ] + } + ], + "GB0221": [ + { + "Gb_type": "non-generator contextlib.contextmanager", + "Context": "str(self.vt.get_code())", + "Explanation": "Cannot compile function decorated with `@contextlib.contextmanager` that is not a generator, i.e. does not use `yield`", + "Hints": [ + "Use `yield` in the function body instead of `return`.", + "Remove the `@contextlib.contextmanager` decorator." + ] + } + ], + "GB0222": [ + { + "Gb_type": "Attempted to wrap a set with tensors", + "Context": "Python set containing torch.Tensor elements", + "Explanation": "Dynamo cannot trace sets of tensors. To get a stable ordering, Dynamo needs to convert the set into a list and the order might not be stable if the set contains tensors.", + "Hints": [ + "Use a dictionary where the keys are tensors.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0223": [ + { + "Gb_type": "torch.compile call with > 1 args", + "Context": "args={args}, kwargs={kwargs}", + "Explanation": "Attempted to call `torch.compile` with > 1 args. Dynamo does not support this.", + "Hints": [ + "Remove the torch.compile call or its additional args.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0224": [ + { + "Gb_type": "Attempted to call torch in-graph function on only torch.SymInt arguments", + "Context": "fn={self.value}, args={args}, kwargs={kwargs}", + "Explanation": "Attempted to call {str(self.value)} (that should be put in the FX graph) on only torch.SymInt arguments. Dynamo does not support this.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0225": [ + { + "Gb_type": "Attempted to use tensor creation function with requires_grad=True", + "Context": "fn={self.value}, args={args}, kwargs={kwargs}", + "Explanation": "Dynamo does not support this.", + "Hints": [ + "Create the tensor outside the compiled region.", + "Do not set `requires_grad=True`.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0226": [ + { + "Gb_type": "`torch.nn.Parameter()` with unsupported data type", + "Context": "data={data}", + "Explanation": "Called `torch.nn.Parameter()` with non-Tensor argument.", + "Hints": [ + "Ensure the argument to `torch.nn.Parameter()` is a `torch.Tensor`.", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0227": [ + { + "Gb_type": "Attempted to use torch.nn.Parameter constructor with tensor subclass", + "Context": "str(data)", + "Explanation": "Dynamo does not support this.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0228": [ + { + "Gb_type": "`torch.nn.Parameter`: cannot convert to traceable tracable", + "Context": "", + "Explanation": "convert_tracable_parameter is set to False.", + "Hints": [ + "Check usage of context manager: do_not_convert_to_tracable_parameter", + "This graph break may be difficult to debug. Please report an issue to PyTorch for assistance." + ] + } + ], + "GB0229": [ + { + "Gb_type": "Unexpected type of data placeholder op for parameter construction", + "Context": "data_node.op={data_node.op}", + "Explanation": "Data node op should be placeholder or get_attr.", + "Hints": [ + "This graph break may be difficult to debug. Please report an issue to PyTorch for assistance." + ] + } + ], + "GB0230": [ + { + "Gb_type": "Attempted to use torch.use_deterministic_algorithms(warn_only=True)", + "Context": "mode={mode}, warn_only={warn_only}", + "Explanation": "Dynamo does not support this.", + "Hints": [ + "Remove param warn_only in function call torch.use_deterministic_algorithms.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0231": [ + { + "Gb_type": "call `torch.from_numpy` with `torch._dynamo.config.trace_numpy=False`", + "Context": "trace_numpy={config.trace_numpy}", + "Explanation": "Attempted to call `torch.from_numpy` with config `torch._dynamo.config.trace_numpy` set to `False`.", + "Hints": [ + "Change `torch._dynamo.config.trace_numpy` to `True`." + ] + } + ], + "GB0232": [ + { + "Gb_type": "`torch.from_numpy` with NumPy unavailable", + "Context": "", + "Explanation": "Attempted to call `torch.numpy` but NumPy could not be imported.", + "Hints": [ + "Check NumPy version and installation in your environment.", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0233": [ + { + "Gb_type": "Attempted to use strided NestedTensor", + "Context": "layout={layout}", + "Explanation": "Dynamo does not support this.", + "Hints": [ + "Change layout=torch.jagged.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0234": [ + { + "Gb_type": "Attempted to pop from empty torch function mode stack", + "Context": "", + "Explanation": "Called `torch._C._pop_torch_function_stack` when torch function mode stack is empty.", + "Hints": [ + "Do not pop from empty torch function mode stack.", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0235": [ + { + "Gb_type": "`torch.nn.Parameter` with non-constant Tensor attributes", + "Context": "data={data}", + "Explanation": "Dynamo does not support this.", + "Hints": [ + "Ensure the Tensor argument's shape, dtype, and device are correct.", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0236": [ + { + "Gb_type": "Invalid input type for nonstrict_trace-ed function", + "Context": "Encountered input of type <{type_name}>.", + "Explanation": "For `nonstrict_trace`-ed functions, only basic types (e.g., torch.Tensor, int, float) or pytree containers of those are allowed as inputs. The provided argument contains an unsupported type.", + "Hints": [ + "Use one of the following to register the type with pytree:\n", + "* `torch.utils._pytree.register_constant`\n", + "* `torch.utils._pytree.register_dataclass`\n", + "* `torch.utils._pytree.register_pytree_node`" + ] + } + ], + "GB0237": [ + { + "Gb_type": "non-constant `requires_grad` argument to `torch.nn.Parameter`", + "Context": "requires_grad={requires_grad}", + "Explanation": "Dynamo does not support this.", + "Hints": [ + "Change `requires_grad` to be a bool.", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0238": [ + { + "Gb_type": "Input marked with `pytree.register_constant` constructed in the `torch.compile` region", + "Context": "Input={input_spec_vt}, offending type <{type_name}>.", + "Explanation": "Calling a `nonstrict_trace`-ed function with an input that contains an object of type <{type_name}>, which was marked with `pytree.register_constant`. However, the object was constructed _inside_ the `torch.compile` region. This is not supported.", + "Hints": [ + "Construct the object _outside_ the `torch.compile` region, or submit an issue to GitHub.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0239": [ + { + "Gb_type": "Invalid use of pytree_flatten with nonstrict_trace-ed function", + "Context": "Input={input_spec_vt}, offending type <{type_name}>.", + "Explanation": "Calling a `nonstrict_trace`-ed function where one of the inputs has been registered with a `pytree_flatten` that places an object of type <{type_name}> into the context.", + "Hints": [ + "Modifying the `pytree_flatten` to avoid placing the object into the context.", + "Apply one of the following to <{type_name}>:\n", + "* `torch.utils._pytree.register_constant`\n", + "* `torch.utils._pytree.register_dataclass`\n", + "* `torch.utils._pytree.register_pytree_node`", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0240": [ + { + "Gb_type": "Shape mismatch with out= list of tensor variants", + "Context": "fn={self.value}, args={args}, kwargs={kwargs}", + "Explanation": "Shape mismatch when calling {self.value} with `out=`. Provided `out=` shape: {saved_out_shape}. Actual shape: {fake_out.shape}.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0241": [ + { + "Gb_type": "Attempted to call op with non-contiguous `out=` list of tensors", + "Context": "self.value={self.value}, args={args}, kwargs={kwargs}", + "Explanation": "Dynamo does not support this.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0242": [ + { + "Gb_type": "Attempted to call op with non-contiguous `out=` tensor", + "Context": "self.value={self.value}, args={args}, kwargs={kwargs}", + "Explanation": "Dynamo does not support this.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0243": [ + { + "Gb_type": "Attempted to use `torch.nn.modules.utils._ntuple` with unsupported argument type", + "Context": "value={value}", + "Explanation": "Dynamo does not support this.", + "Hints": [ + "Change use of _ntuple with argument as constant or tensor." + ] + } + ], + "GB0244": [ + { + "Gb_type": "Attempted to use `torch.nn.Parameter()` with export", + "Context": "", + "Explanation": "Dynamo does not support this.", + "Hints": [ + "Do not use `torch.nn.Parameter()` with export.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0245": [ + { + "Gb_type": "Attempted to use `nested_tensor` with non-list input", + "Context": "tensor_list={tensor_list}", + "Explanation": "Dynamo does not support this.", + "Hints": [ + "Change `nested_tensor` with list input.", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0246": [ + { + "Gb_type": "Attempted to use `torch.nn.functional.one_hot` with data-dependent output shape", + "Context": "args={args}, kwargs={kwargs}", + "Explanation": "Dynamo does not support this.", + "Hints": [ + "Explicitly set the `num_classes` param of the function call ", + "`torch.nn.functional.one_hot` to something other than -1." + ] + } + ], + "GB0247": [ + { + "Gb_type": "Shape mismatch with out= tensor variant", + "Context": "fn={self.value}, args={args}, kwargs={kwargs}", + "Explanation": "Shape mismatch when calling {self.value} with `out=`. Provided `out=` shape: {saved_out_shapes}. Actual shape: {fake_out.shape}.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0248": [ + { + "Gb_type": "improper torch.get_device_module arguments", + "Context": "args={args}, kwargs={kwargs}", + "Explanation": "torch.get_device_module accepts 1 optional argument `device`", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0249": [ + { + "Gb_type": "bad device argument to torch.accelerator.current_stream", + "Context": "args={args}, kwargs={kwargs}", + "Explanation": "Expected valid string/torch.device argument ('cpu', 'cuda', etc.)", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + }, + { + "Gb_type": "bad device argument to torch.get_device_module", + "Context": "args={args}, kwargs={kwargs}", + "Explanation": "Expected valid string/torch.device argument ('cpu', 'cuda', etc.)", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + }, + { + "Gb_type": "bad device argument to torch.accelerator.current_stream", + "Context": "args={args}, kwargs={kwargs}", + "Explanation": "Expected valid string/torch.device argument ('cpu', 'cuda', etc.)", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + }, + { + "Gb_type": "bad device argument to torch.get_device_module", + "Context": "args={args}, kwargs={kwargs}", + "Explanation": "Expected valid string/torch.device argument ('cpu', 'cuda', etc.)", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0250": [ + { + "Gb_type": "ndarray.astype(object)", + "Context": "call_method {self} {name} {args} {kwargs}", + "Explanation": "`ndarray.astype('O')` or `ndarray.astype(object)` is not supported by torch.compile, as there is no equivalent to object type in torch.Tensor. This will be executed eagerly.", + "Hints": [ + "This graph break is fundamental - it is unlikely that Dynamo will ever be able to trace through your code. Consider finding a workaround." + ] + } + ], + "GB0251": [ + { + "Gb_type": "Unsupported output type for nonstrict_trace-ed function", + "Context": "Function: {fn.__name__}", + "Explanation": "For `nonstrict_trace`-ed functions, only basic types (e.g., torch.Tensor, int, list) are allowed as output. The result of this call contains an unsupported type.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0252": [ + { + "Gb_type": "could not find name in object's mro", + "Context": "name={name}, object type={type(self.value)}, mro={type(self.value).__mro__}", + "Explanation": "Could not find name `{name}` in mro {type(self.value).__mro__}", + "Hints": [ + "Ensure the name `{name}` is defined somewhere in {self.value}'s type hierarchy.", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0253": [ + { + "Gb_type": "call_method on generator", + "Context": "object={self.value}, method={name}, args={args}, kwargs={kwargs}", + "Explanation": "Detected a method call to a user-defined generator object. This is not fully supported.", + "Hints": [ + "Set `torch._dynamo.config.enable_faithful_generator_behavior = False`. Note that this ", + "may cause silent incorrectness, since we will eagerly unpack generators instead of lazily ", + "evaluating them." + ] + } + ], + "GB0254": [ + { + "Gb_type": "non-const setattr name on user-defined object", + "Context": "object={self}, name={name}, value={value}", + "Explanation": "Detected a call to `setattr` of a user-defined object with a non-constant name.", + "Hints": [ + "Ensure that the name is a string." + ] + } + ], + "GB0255": [ + { + "Gb_type": "attempted to call sourceless user-defined object as a method", + "Context": "object={self.value}, function={func}, args={args}, kwargs={kwargs}", + "Explanation": "Dynamo does not support this.", + "Hints": [ + "Ensure the user-defined object {self.value} is constructed outside the compiled region." + ] + } + ], + "GB0256": [ + { + "Gb_type": "User-defined object with non-function __getattr__", + "Context": "object={self.value}, name={name}, getattr_fn={getattr_fn}", + "Explanation": "Found a non-function __getattr__ {getattr_fn} from a user-defined object {self.value} when attempting to getattr `{name}`", + "Hints": [ + "Ensure the object's __getattr__ is a function type." + ] + } + ], + "GB0257": [ + { + "Gb_type": "TypedDict with optional keys", + "Context": "str(self.value)", + "Explanation": "Dyanmo does not support tracing TypedDict with optional keys", + "Hints": [ + "Avoid using TypedDict with optional keys", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0258": [ + { + "Gb_type": "collections.deque() with bad arguments", + "Context": "args={args}, kwargs={kwargs}", + "Explanation": "Detected call to collections.deque() with bad arguments.", + "Hints": [ + "Fix the call to collections.deque().", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0259": [ + { + "Gb_type": "collections.deque() with bad iterable argument", + "Context": "args={args}, kwargs={kwargs}", + "Explanation": "Call to collections.deque() has an iterable argument that Dynamo cannot convert to a list.", + "Hints": [ + "Use a simpler sequence type that Dynamo can convert to a list ", + "(e.g. list, tuple, list iterator, etc.)", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0260": [ + { + "Gb_type": "missing args to functools.partial", + "Context": "", + "Explanation": "functools.partial requires at least one argument", + "Hints": [ + "Fix the functools.partial call.", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0261": [ + { + "Gb_type": "User-defined object method with non-function __func__", + "Context": "object={self.value}, name={name}, method={dynamic_subobj}, method.__self__={dynamic_subobj.__self__}, method.__func__={dynamic_subobj.__func__}", + "Explanation": "Method {dynamic_subobj} (name={name}) of user-defined object {self.value} has a __func__ ({dynamic_subobj.__func__}) that is not a function type.", + "Hints": [ + "Ensure that the method's __func__ is a function type." + ] + } + ], + "GB0262": [ + { + "Gb_type": "unsupported contextlib.* API", + "Context": "{self.value}", + "Explanation": "{self.value} not supported. This may be due to its use of context-specific operations that are not supported in Dynamo yet (i.e. Exception handling)", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0263": [ + { + "Gb_type": "attempted to trace contextlib.contextmanager", + "Context": "args={args}", + "Explanation": "Tracing contextlib.contextmanager is disabled.", + "Hints": [ + "Set torch._dynamo.config.enable_trace_contextlib = True" + ] + } + ], + "GB0264": [ + { + "Gb_type": "Attempted to use `torch.nn.Parameter()` constructor with Dynamo", + "Context": "", + "Explanation": "Dynamo does not support this", + "Hints": [ + "Try to construct `torch.nn.Parameter()` outside the compiled region.", + "If this is not possible, turn `graph_break_on_nn_param_ctor` off", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0265": [ + { + "Gb_type": "FakeScriptObject missing method implementation", + "Context": "value={self.value}, method={name}", + "Explanation": "TorchScript object {self.value} doesn't define the method {name}.", + "Hints": [ + "Ensure the method {name} is implemented in {self.value}.", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0266": [ + { + "Gb_type": "Weird method call on TorchScript object", + "Context": "value={self.value}, method={name}", + "Explanation": "This particular method call ({name}) is not supported (e.g. calling `__setattr__`). Most method calls to TorchScript objects should be supported.", + "Hints": [ + "Avoid calling this method." + ] + } + ], + "GB0267": [ + { + "Gb_type": "Attempted to access non-callable attribute of TorchScript object", + "Context": "value={self.value}, method={name}", + "Explanation": "Attribute accesses of TorchScript objects to non-callable attributes are not supported.", + "Hints": [ + "Use method calls instead of attribute access." + ] + } + ], + "GB0268": [ + { + "Gb_type": "Unsupported kwargs for itertools.product", + "Context": "call_function {self} {args} {kwargs}", + "Explanation": "Expected kwargs: 'repeat', but got {','.join(set(kwargs.keys()) - {'repeat'})}", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0269": [ + { + "Gb_type": "Forced graph break on leaf function", + "Context": "", + "Explanation": "Forced graph break for nested graph break testing purposes", + "Hints": [ + "Set torch._dynamo.config.debug_force_graph_break_on_leaf_return = False" + ] + } + ], + "GB0270": [ + { + "Gb_type": "unimplemented builtin op vars() with no arguments", + "Context": "vars: {self} {args}", + "Explanation": "Dynamo does not know how to trace builtin operator {self.fn} with no arguments", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0271": [ + { + "Gb_type": "Class attribute mutation when the __dict__ was already materialized", + "Context": "str(self.value)", + "Explanation": "Dyanmo does not support tracing mutations on a class when its __dict__ is materialized", + "Hints": [] + } + ], + "GB0272": [ + { + "Gb_type": "Failed to make weakref to User Object when storing by ID", + "Context": "user_objected: {obj}", + "Explanation": "Object does not allow us to make a weakref to it", + "Hints": [] + }, + { + "Gb_type": "Failed to make weakref to User Object", + "Context": "user_objected: {obj}", + "Explanation": "Object does not allow us to make a weakref to it", + "Hints": [] + } + ], + "GB0273": [ + { + "Gb_type": "Keyword args passed to exception constructor", + "Context": "{self} with kwargs {init_kwargs}", + "Explanation": "Dynamo does not know how to handle keyword args passed to an exception constructor", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0274": [ + { + "Gb_type": "Attempted to reconstruct context manager's __enter__ method", + "Context": "str(self.ctx)", + "Explanation": "Attempted to reconstruct context manager {type_str} while tracing `with ...:`", + "Hints": [ + "It is likely there is a graph break while tracing `with ctx:` ", + "but outside the actual `ctx.__enter__()` method. ", + "`torch.compile` does not expect this to happen.", + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0275": [ + { + "Gb_type": "torch._dynamo.step_unsupported() with empty checkpoint", + "Context": "", + "Explanation": "traced torch._dynamo.step_unsupported(), but there is no checkpoint to step_graph_break from. This graph break is used for debugging only.", + "Hints": [ + "Remove the torch._dynamo.step_unsupported() call.", + "Include at least one checkpoint: (1) include at least 2 ops and (2) make sure there is some ", + "line of code that is not in a try/with block, and has an empty Python stack.", + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0276": [ + { + "Gb_type": "Failed to make weakref to User Object", + "Context": "user_object: {value}", + "Explanation": "Object does not allow us to make a weakref to it", + "Hints": [] + } + ], + "GB0277": [ + { + "Gb_type": "Attempted to wrap sparse Tensor with VariableTracker", + "Context": "str(example_value)", + "Explanation": "torch.compile does not support sparse Tensors with VariableTracker", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0278": [ + { + "Gb_type": "Unsupported dict type for fromkeys()", + "Context": "{user_cls.__name__}.fromkeys(): {args} {kwargs}", + "Explanation": "Failed to call {user_cls.__name__}.fromkeys() because {user_cls.__name__} is not any type of dict, OrderedDict, or defaultdict", + "Hints": [ + "Ensure {user_cls.__name__} is a type of dict, OrderedDict, or defaultdict." + ] + } + ], + "GB0279": [ + { + "Gb_type": "torch.fx.traceback.annotate escaped from compiled region", + "Context": "str(self)", + "Explanation": "Dynamo doesn't support graph break on torch.fx.traceback.annotate.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0280": [ + { + "Gb_type": "1-arg super not implemented", + "Context": "", + "Explanation": "Dynamo failed to trace attribute `{name}` accessed via `super()` (for type `{self.typevar}` and object `{self.objvar}`) because one-argument of super() is not supported.", + "Hints": [ + "Use two-argument super(type, object_or_type)." + ] + } + ], + "GB0281": [ + { + "Gb_type": "Invalid or non-const argument in nn.Module __getitem__", + "Context": "call_method: {self} {name} {args} {kwargs}", + "Explanation": "Dynamo does not support calling method `{name}` of ``nn.Module`` {module} with a non-constant or non-(str, int) key.", + "Hints": [ + "Use constant arguments of type str or int for __getitem__" + ] + } + ], + "GB0282": [ + { + "Gb_type": "Placement with custom __getattr__ not supported", + "Context": "{value_type.__name__} with custom __getattr__", + "Explanation": "Dynamo does not support Placement types with custom __getattr__ methods", + "Hints": [ + "Use Placement types without custom __getattr__ methods", + "Move the Placement usage outside the compiled region" + ] + } + ], + "GB0283": [ + { + "Gb_type": "Failed to make weakref to graph-created external object", + "Context": "user_object: {example_value}", + "Explanation": "Object does not allow us to make a weakref to it", + "Hints": [] + } + ], + "GB0284": [ + { + "Gb_type": "cannot resume from torch._dynamo.step_unsupported()", + "Context": "", + "Explanation": "traced torch._dynamo.step_unsupported(), but Dynamo is instructed to error on graph break. This graph break is used for debugging only.", + "Hints": [ + "Remove the torch._dynamo.step_unsupported() call.", + "Make sure fullgraph=False and error_on_graph_break=False.", + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0285": [ + { + "Gb_type": "unsupported arguments to torch.accelerator.current_stream", + "Context": "args={args}, kwargs={kwargs}", + "Explanation": "torch.accelerator.current_stream accepts one optional argument `device`", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0286": [ + { + "Gb_type": "bad device argument to torch.get_device_module", + "Context": "args={args}, kwargs={kwargs}", + "Explanation": "Expected valid string/torch.device argument ('cpu', 'cuda', etc.)", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0287": [ + { + "Gb_type": "unsupported type.__dict__['__annotations__'].__get__ call", + "Context": "call_function {self}, args: {args}, kwargs: {kwargs}", + "Explanation": "`torch.compile` only supports calling type.__dict__['__annotations__'].__get__ on a single constant argument (i.e. a type).", + "Hints": [ + "Make sure your call to type.__dict__['__annotations__'] only has ", + "one positional argument (no keyword arguments).", + "Make sure the argument to type.__dict__['__annotations__'] is a constant ", + "(i.e. type). For example, `object`, `int`, `MyCustomClass`.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0288": [ + { + "Gb_type": "Can't extract message from torch._check()", + "Context": "str(message_vt)", + "Explanation": "The second argument of torch._check() must be a functiondefined within the torch.compile regionthat does not reference a non-local variable.", + "Hints": [ + "Make sure the message function is defined in the torch.compile region.", + "Remove any closure variables, e.g. ", + "remove references to closure variable `x` in `lambda: f'{x} failed check'`", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0289": [ + { + "Gb_type": "unsupported method call on `typing` variable", + "Context": "typing variable: {self.value}, method name: {name}, args: {args}, kwargs: {kwargs}", + "Explanation": "`torch.compile` does not support method call `{name}` on `typing` variable f{self.value}.", + "Hints": [ + "Avoid calling the {name} method on {self.value}.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0290": [ + { + "Gb_type": "attempted to trace numpy.* function as a method", + "Context": "numpy function: {self.value}, args: {args}, kwargs: {kwargs}", + "Explanation": "Tracing numpy.* functions as methods is not supported.", + "Hints": [ + "This graph break may be difficult to debug. Please report an issue to PyTorch for assistance." + ] + } + ], + "GB0291": [ + { + "Gb_type": "logging.Logger method not supported for non-export cases", + "Context": "method: {self.value}.{name}, args: {args}, kwargs: {kwargs}", + "Explanation": "logging.Logger methods are not supported for non-export cases.", + "Hints": [ + "Add the logging method to `torch._dynamo.config.ignore_logger_methods." + ] + } + ], + "GB0292": [ + { + "Gb_type": "constant-like method call with unsupported return type", + "Context": "{self._error_prefix}.{name}(*{args}, **{kwargs}) returned {result}", + "Explanation": "Attempted to call {self._error_prefix}.{name}, got unsupported return value {result}.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0293": [ + { + "Gb_type": "attempted to trace numpy function with config.trace_numpy=False", + "Context": "numpy function: {self.value}, args: {args}, kwargs: {kwargs}", + "Explanation": "Attempted to trace numpy function {self.value} while `torch._dynamo.config.trace_numpy` was set to False.", + "Hints": [ + "Set `torch._dynamo.config.trace_numpy` to True to trace numpy functions." + ] + } + ], + "GB0294": [ + { + "Gb_type": "attempted to trace numpy function unsupported by PyTorch", + "Context": "numpy function: {self.value}, args: {args}, kwargs: {kwargs} (corresponding torch function: {func})", + "Explanation": "Can't find numpy numpy function {self.value} in torch._numpy.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0295": [ + { + "Gb_type": "cannot reconstruct NullVariable in Python < 3.11", + "Context": "", + "Explanation": "Attempted to generate PUSH_NULL instruction in Python < 3.11; where this instruction does not exist.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0296": [ + { + "Gb_type": "attempted to reorder a debugging function that can't actually be reordered", + "Context": "fn: {self.value}, args: {args}, kwargs: {kwargs}", + "Explanation": "`torch.compile` can only reorder functions where the arguments are Tensors, constants, or string formatters.", + "Hints": [ + "Avoid calling the logging function {self.value} with args that are not supported." + ] + } + ], + "GB0297": [ + { + "Gb_type": "random.Random() with improper arguments", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "random.Random() with > 1 arg or with kwargs is not supported.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0298": [ + { + "Gb_type": "attempted to trace torch._numpy.random function with config.use_numpy_random_stream=True", + "Context": "numpy function: {self.value}, args: {args}, kwargs: {kwargs} (corresponding torch function: {func})", + "Explanation": "Attempted to trace {self.value} when `torch._dynamo.config.use_numpy_random_stream` is set to True.", + "Hints": [ + "Set `torch._dynamo.config.use_numpy_random_stream` to False.", + "Avoid calling {self.value}." + ] + } + ], + "GB0299": [ + { + "Gb_type": "constant-like method call with non-constant args", + "Context": "{self._error_prefix}.{name}(*{args}, **{kwargs})", + "Explanation": "Attempted to call {self._error_prefix}.{name} with non-constant args.", + "Hints": [ + "Ensure that the args to the method call are constant (int, str, etc.)." + ] + } + ], + "GB0300": [ + { + "Gb_type": "numpy function that produces a const collection type encountered non-const arguments", + "Context": "numpy function: {self.value}, args: {args}, kwargs: {kwargs} (corresponding torch function: {func})", + "Explanation": "numpy function {self.value} that produces a const collection type (e.g. np.dtype, np.iinfo/np.finfo) received arguments that are not constant.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0301": [ + { + "Gb_type": "HOP: non torch.Tensor leaf", + "Context": "args types: {[type(a.realize()) for a in args]}", + "Explanation": "Expected all leaves to be of torch.Tensor type.", + "Hints": [] + } + ], + "GB0302": [ + { + "Gb_type": "HOP: non-callable variable", + "Context": "arg name: {arg_name}, func_var type: {str(func_var)}", + "Explanation": "{arg_name} should be a callable but is of type {str(func_var)}.", + "Hints": [] + } + ], + "GB0303": [ + { + "Gb_type": "torch.while_loop: improper args/kwargs", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "torch.while_loop expects 4 positional arguments (got {len(args)}) and no keyword arguments (got {len(kwargs)}) Usage: while_loop(cond_fn, body_fn, operands)", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0304": [ + { + "Gb_type": "torch.while_loop: improper additional_inputs", + "Context": "str(additional_inputs)", + "Explanation": "Expected additional_inputs to be a list/tuple but got {additional_inputs.python_type()}", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0305": [ + { + "Gb_type": "invalid set_subgraph_inputs and sub_kwargs settings", + "Context": "set_subgraph_inputs: {set_subgraph_inputs}, sub_kwargs: {sub_kwargs}", + "Explanation": "`sub_kwargs` cannot be used when `set_subgraph_inputs` is not set to 'automatic'.", + "Hints": [ + "Use `set_subgraph_inputs='automatic'` when passing `sub_kwargs`.", + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0306": [ + { + "Gb_type": "unsupported HigherOrderOperator", + "Context": "str(value)", + "Explanation": "Unable to create higher order operator variable for {value.__name__}.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0307": [ + { + "Gb_type": "unsupported HigherOrderOperator function call", + "Context": "str(self.value)", + "Explanation": "Unable to trace calling higher order operator variable for {self.value.__name__}.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0308": [ + { + "Gb_type": "torch.while_loop: unsupported cond_fn return type", + "Context": "str(cond_r)", + "Explanation": "Expected cond_fn to return a scalar tensor or a bool but got {cond_r_meta.shape}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0309": [ + { + "Gb_type": "torch.cond: improper args/kwargs", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "torch.cond expects 4 positional arguments (got {len(args)}) and no keyword arguments (got {len(kwargs)}) Usage: cond(pred, cond_fn, body_fn, operands)", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0310": [ + { + "Gb_type": "torch.cond: improper predicate", + "Context": "str(pred)", + "Explanation": "Expected `pred` to be a bool or a boolean tensor with a single item but got {str(type(pred))} with original python type {str(pred.python_type())}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0311": [ + { + "Gb_type": "torch.cond: improper operands", + "Context": "str(operands)", + "Explanation": "Expected `operands` to be a list/tuple but got {operands.python_type()}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0312": [ + { + "Gb_type": "torch.cond: improper operands contents", + "Context": "str(operands)", + "Explanation": "Expected `operands` to be a list/tuple of pytrees that only consists of tensor leaves.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0313": [ + { + "Gb_type": "torch.cond: differing branch outputs", + "Context": "true_spec: {true_spec.treespec}, false_spec: {false_spec.treespec}, same_spec: {same_spec}", + "Explanation": "Expected branches to return the same pytree structure.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0314": [ + { + "Gb_type": "HOP body output unsupported", + "Context": "non-tensor outputs: {non_tensor_output}", + "Explanation": "HigherOrderOperator body's output must consist of tensors or ints/bools only but got {out.python_type()}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0315": [ + { + "Gb_type": "torch.associative_scan: improper xs", + "Context": "str(xs)", + "Explanation": "Expected xs to be a list/tuple but got {xs.python_type()}", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0316": [ + { + "Gb_type": "torch.associative_scan: improper additional_inputs", + "Context": "str(additional_inputs)", + "Explanation": "Expected additional_inputs to be a list/tuple but got {additional_inputs.python_type()}", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0317": [ + { + "Gb_type": "torch.associative_scan: zero-sized tensor", + "Context": "str(xs_vars[0])", + "Explanation": "associative_scan() operator doesn't support zero-sized tensors during tracing.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0318": [ + { + "Gb_type": "torch.associative_scan: combine_fn improper number of leaves", + "Context": "str(_combine_treespec.as_python_constant())", + "Explanation": "combine_fn needs to produce one pytree for the output but combine_fn produces the pytree {_combine_treespec.as_python_constant()}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0319": [ + { + "Gb_type": "torch.associative_scan: mismatched input/output tree structure", + "Context": "xs: {xs_treespec.as_python_constant()}, output: {_combine_treespec.as_python_constant()}", + "Explanation": "The tree structure of the xs and the outs of the combine_fn are are expected to be identical, but got xs: {xs_treespec.as_python_constant()} vs output: {_combine_treespec.as_python_constant()}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0320": [ + { + "Gb_type": "torch.scan: improper xs", + "Context": "str(xs)", + "Explanation": "Expected xs to be a list/tuple but got {xs.python_type()}", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0321": [ + { + "Gb_type": "torch.scan: improper init", + "Context": "str(init)", + "Explanation": "Expected init to be a list/tuple with at least one element but got {init.python_type()}", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0322": [ + { + "Gb_type": "torch.scan: no init leaves", + "Context": "", + "Explanation": "Expected init leaves.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0323": [ + { + "Gb_type": "torch.scan: improper additional_inputs", + "Context": "str(additional_inputs)", + "Explanation": "Expected additional_inputs to be a list/tuple but got {additional_inputs.python_type()}", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0324": [ + { + "Gb_type": "torch.scan: zero-sized tensor", + "Context": "str(xs_vars[0])", + "Explanation": "associative_scan() operator doesn't support zero-sized tensors during tracing.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0325": [ + { + "Gb_type": "torch.map: kwargs not supported", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "torch.map expects no keyword arguments (got {len(kwargs)})", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0326": [ + { + "Gb_type": "torch.map: improper inputs", + "Context": "str(sample_shape)", + "Explanation": "torch.map doesn't support scalar or non-zero sized tensors during tracing.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0327": [ + { + "Gb_type": "executorch_call_delegate: kwargs not supported", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "executorch_call_delegate expects no keyword arguments (got {len(kwargs)})", + "Hints": [] + } + ], + "GB0328": [ + { + "Gb_type": "torch.func.functional_call capture is disabled", + "Context": "", + "Explanation": "torch.func.functional_call capture is disabled", + "Hints": [ + "Set `torch._dynamo.config.inline_inbuilt_nn_modules=True` to enable." + ] + } + ], + "GB0329": [ + { + "Gb_type": "WrapHigherOrderVariable: kwargs unexpected", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "kwargs should have been flattened into lifted args.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0330": [ + { + "Gb_type": "wrap_with_set_grad_enabled: unexpected kwargs", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "wrap_with_set_grad_enabled expects no keyword arguments (got {len(kwargs)}).", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0331": [ + { + "Gb_type": "wrap_with_set_grad_enabled: non-constant grad_enabled", + "Context": "str(grad_enabled)", + "Explanation": "wrap_with_set_grad_enabled expects grad_enabled argument to be a constant.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0332": [ + { + "Gb_type": "wrap_with_set_grad_enabled: unexpected freevars", + "Context": "str(body_lifted_freevars)", + "Explanation": "wrap_with_set_grad_enabled expects no freevars.", + "Hints": [] + } + ], + "GB0333": [ + { + "Gb_type": "wrap_with_autocast: unexpected kwargs", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "wrap_with_autocast expects no keyword arguments (got {len(kwargs)}).", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0334": [ + { + "Gb_type": "wrap_with_autocast: unexpected freevars", + "Context": "str(body_lifted_freevars)", + "Explanation": "wrap_with_autocast expects no freevars.", + "Hints": [] + } + ], + "GB0335": [ + { + "Gb_type": "hints_wrapper: improper args/kwargs", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "hints_wrapper expects 3 positional arguments (got {len(args)}) and 1 keyword argument (got {len(kwargs)}). Usage: hints_wrapper(body_fn, args, kwargs, hints=...). args is expected to be list/tuple and kwargs is expected to be a dict.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0336": [ + { + "Gb_type": "out_dtype: unexpected kwargs", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "out_dtype expects no keyword arguments (got {len(kwargs)}).", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0337": [ + { + "Gb_type": "strict_mode: unexpected kwargs", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "strict_mode higher order op expects no keyword arguments (got {len(kwargs)}).", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0338": [ + { + "Gb_type": "invoke_subgraph: kwargs unexpected", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "kwargs should have been flattened into lifted args.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0339": [ + { + "Gb_type": "torch.while_loop: infinite loop detected", + "Context": "str(cond_r)", + "Explanation": "Infinite loop detected because while_loop's cond_fn always returns the same value {pred}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0340": [ + { + "Gb_type": "torch.cond: unsupported branch return type", + "Context": "str(ret_val)", + "Explanation": "Expected branches to return a possibly nested pytree of tensors or constant ints.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0341": [ + { + "Gb_type": "torch.associative_scan: improper args", + "Context": "args: {args}", + "Explanation": "torch.associative_scan expects 2 positional arguments (got {len(args)}) Usage: associative_scan(combine_fn, xs)", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0342": [ + { + "Gb_type": "torch.scan: improper combine_fn", + "Context": "str(combine_fn_var)", + "Explanation": "Expected combine_fn to be wrapped as functools.partial in scan user-facing api or a graph module if we're re-exporting but got {combine_fn_var.python_type()}.", + "Hints": [ + "This graph break may be difficult to debug. Please report an issue to PyTorch for assistance." + ] + } + ], + "GB0343": [ + { + "Gb_type": "torch.scan: improper combine_fn number of returns", + "Context": "str(combine_result_vars)", + "Explanation": "Expect combine_fn to return a tuple (next_carry, y) but got {combine_result_vars}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0344": [ + { + "Gb_type": "wrap_with_autocast: expected constant arg", + "Context": "str(args)", + "Explanation": "wrap_with_autocast expects device_type, dtype, enabled, and cache_enabled arguments to be constants.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0345": [ + { + "Gb_type": "strict_mode: improper args", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "strict_mode higher order op expects flat inputs (list/tuple/dict)", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0346": [ + { + "Gb_type": "autograd.Function.apply: non-function or method forward", + "Context": "str(self.fwd_graph)", + "Explanation": "Expected forward function to be a function or method.", + "Hints": [] + } + ], + "GB0347": [ + { + "Gb_type": "autograd.Function.apply: _materialize_non_diff_grads mutation", + "Context": "", + "Explanation": "Mutations to autograd.Function.ctx._materialize_non_diff_grads are not supported.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0348": [ + { + "Gb_type": "autograd.Function.apply: non-function or method backward", + "Context": "str(self.bwd_graph)", + "Explanation": "Expected backward function to be a function or method.", + "Hints": [] + } + ], + "GB0349": [ + { + "Gb_type": "cannot unwrap variable for check_meta_consistency", + "Context": "str(var)", + "Explanation": "Expected {var} to be TensorVariable, SymNodeVariable, or ConstantVariable", + "Hints": [] + } + ], + "GB0350": [ + { + "Gb_type": "torch.cond: unsupported branch return type (constant non-int)", + "Context": "str(ret_val)", + "Explanation": "Constants returned from branches must be ints.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0351": [ + { + "Gb_type": "HOP body taking non-Tensor as input", + "Context": "str(sub_args)", + "Explanation": "{description} with body that accepts non-Tensors as input. Got type {a.python_type()} at index {idx}.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0352": [ + { + "Gb_type": "autograd.Function.apply: non-function or method backward (2)", + "Context": "str(self.bwd_graph)", + "Explanation": "Expected backward function to be a function or method.", + "Hints": [] + } + ], + "GB0353": [ + { + "Gb_type": "rewrite_signature: cannot trace optional function input", + "Context": "", + "Explanation": "Parameter {name} is optional with a default value of {param.default}. This is not supported yet.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0354": [ + { + "Gb_type": "failed to find name in frame builtins", + "Context": "", + "Explanation": "Failed to find name `{argval}` in frame's builtins.", + "Hints": [ + "This is likely to be a Dynamo bug. Please report an issue to PyTorch." + ] + } + ], + "GB0355": [ + { + "Gb_type": "non-single Tensor return unsupported", + "Context": "api: {api}, ret: {ret}", + "Explanation": "{api} over function that returns something other than one Tensor.", + "Hints": [] + } + ], + "GB0356": [ + { + "Gb_type": "failed to handle argument for FlexAttentionBackward HOP", + "Context": "args: {args}, kwargs: {kwargs}", + "Explanation": "Missing Dynamo support for FlexAttentionBackward HOP argument.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0357": [ + { + "Gb_type": "UnspecializedNNModuleVariable wrapped around ScriptModules unsupported", + "Context": "str(value)", + "Explanation": "ScriptModules aren't supported in UnspecializedNNModuleVariable because their .forward function isn't a static member of their type.", + "Hints": [ + "This graph break may be difficult to debug. Please report an issue to PyTorch for assistance." + ] + } + ], + "GB0358": [ + { + "Gb_type": "optimizer: pending mutation on parameter", + "Context": "variable: {variable}, parameter: {p}", + "Explanation": "Pending mutations on a parameter (e.g. due to using closure) require a graph break.", + "Hints": [] + } + ], + "GB0359": [ + { + "Gb_type": "unsupported torch._C._SDPAParams attribute", + "Context": "name: {name}", + "Explanation": "Unable to fetch attribute {name} from torch._C._SDPAParams.", + "Hints": [ + "Dynamo has detected that tracing the code will result in an error when running in eager. Please double check that your code doesn't contain a similar error when actually running eager/uncompiled." + ] + } + ], + "GB0360": [ + { + "Gb_type": "torch.fx.experimental.symbolic_shapes.guard_scalar branch not supported", + "Context": "expr: {expr}", + "Explanation": "Expected `expr` to be a symbolic variable or constant.", + "Hints": [] + } + ], + "GB0361": [ + { + "Gb_type": "triton kernel unsupported feature", + "Context": "", + "Explanation": "Encountered triton kernel unsupported feature: {msg}", + "Hints": [] + } + ], + "GB0362": [ + { + "Gb_type": "Attempted to access attributes/methods on an OpaqueObject", + "Context": "value={self.value}, attr={name}", + "Explanation": "Attribute/method access of OpaqueObjects is not supported.", + "Hints": [ + "Use custom operators instead of direct attribute/method access." + ] + } + ], + "GB0363": [ + { + "Gb_type": "An opaque object was created in the middle of the program.", + "Context": "Opaque object type: {self.value}.", + "Explanation": "Opaque objects cannot be created inside the torch.compile region. They must be created before entering the compiled function.", + "Hints": [ + "Please create the opaque object before calling torch.compile ", + "and pass it in as an argument or as a global variable." + ] + }, + { + "Gb_type": "An opaque object was created in the middle of the program.", + "Context": "Opaque object types: {self.value}", + "Explanation": "Opaque objects cannot be created inside the torch.compile region. They must be created before entering the compiled function.", + "Hints": [ + "Please create the opaque object before calling torch.compile ", + "and pass it in as an argument or as a global variable." + ] + } + ], + "GB0364": [ + { + "Gb_type": "User-defined object with overridden __hash__", + "Context": "hashing object of type={type(obj)} and variable tracker {vt}", + "Explanation": "Found a user-defined object {vt} with overridden __hash__ when attempting to hash it", + "Hints": [ + "Dynamo does not support hashing user-defined objects with overridden __hash__", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0365": [ + { + "Gb_type": "Dynamo cannot determine whether the underlying object is hashable", + "Context": "is_python_hashable {self}", + "Explanation": "Dynamo does not know whether the underlying python object for {self} is hashable", + "Hints": [ + "Consider using a different type of object as the dictionary key instead of {type_self}.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0366": [ + { + "Gb_type": "Dynamo cannot determine the hash of an object", + "Context": "get_python_hash {self}", + "Explanation": "Dynamo does not know the hash of the underlying python object for {self}", + "Hints": [ + "Consider using a different type of object as the dictionary key instead of {self.python_type()}.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0367": [ + { + "Gb_type": "Dynamo cannot determine the equality comparison of an object", + "Context": "is_python_equal {self}", + "Explanation": "Dynamo does not know the equality comparison of the underlying python object for {self}", + "Hints": [ + "Consider using a different type of object as the dictionary key instead of {self.python_type()}.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0368": [ + { + "Gb_type": "Frozen dataclass with __post_init__", + "Context": "dataclass={dataclass_cls.__name__}", + "Explanation": "Cannot reconstruct frozen dataclass with __post_init__ method, as it may have side effects that would be incorrectly replayed.", + "Hints": [ + "Remove the __post_init__ method from the frozen dataclass.", + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ], + "GB0369": [ + { + "Gb_type": "Frozen dataclass with missing field", + "Context": "dataclass={dataclass_cls.__name__}, field={field.name}", + "Explanation": "Cannot reconstruct frozen dataclass: field '{field.name}' was not tracked during tracing.", + "Hints": [ + "It may be possible to write Dynamo tracing rules for this code. Please report an issue to PyTorch if you encounter this graph break often and it is causing performance issues." + ] + } + ] +} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_bytecode_inputs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_bytecode_inputs.py new file mode 100644 index 0000000000000000000000000000000000000000..c00a88bfbaa81a6c200a2a6a4178b2a7d3738b08 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_bytecode_inputs.py @@ -0,0 +1,96 @@ +import weakref +from collections.abc import Callable +from typing import Any + +from torch._dynamo.source import Source + + +PyCodegen = Any + +# This file is to handle types that we don't want to support +# as explicit FX graph inputs. This uses a sidetable which +# we populate in bytecode and is loaded during graph execution + +# We use a dynamo-generated index as a level of indirection +# this allows us to register objects externally in pre-graph bytecode that we want +# to pass to the graph, but not support their types as graph inputs +index_to_bytecode_constructor: dict[int, Callable[[PyCodegen], None]] = {} + +index_to_external_object_weakref: dict[int, weakref.ReferenceType[Any]] = {} + +keep_alive: list[Any] = [] + + +def has_user_objects() -> bool: + return bool(index_to_bytecode_constructor) + + +def stash_graph_created_object(obj: Any) -> Any: + keep_alive.append(obj) + return obj + + +def get_external_object_by_index(index: int) -> Any: + assert index in index_to_external_object_weakref, ( + "Index not registered in index_to_user_object_weakref" + ) + obj = index_to_external_object_weakref[index]() + assert obj is not None, "User object is no longer alive" + return index_to_external_object_weakref[index]() + + +def store_user_object_weakrefs(*args: Any) -> None: + global index_to_external_object_weakref + index_to_external_object_weakref.clear() + index_to_external_object_weakref.update( + {i: weakref.ref(arg) for i, arg in enumerate(args)} + ) + + +def reset_user_object_tracking() -> None: + index_to_bytecode_constructor.clear() + index_to_external_object_weakref.clear() + keep_alive.clear() + + +def register_graph_created_object( + example_value: Any, construct_fn: Callable[[int, PyCodegen], None] +) -> int: + global index_to_bytecode_constructor + global keep_alive + keep_alive.append(example_value) + index = len(index_to_bytecode_constructor) + index_to_bytecode_constructor[index] = lambda cg: construct_fn(index, cg) + try: + index_to_external_object_weakref[index] = weakref.ref(example_value) + except TypeError as e: + from .exc import unimplemented + + unimplemented( + gb_type="Failed to make weakref to graph-created external object", + context=f"user_object: {example_value}", + explanation="Object does not allow us to make a weakref to it", + hints=[], + from_exc=e, + ) + return index + + +# Register a user object to be used in the graph +def register_user_object(value: Any, source: Source) -> int: + global index_to_bytecode_constructor + index = len(index_to_bytecode_constructor) + index_to_bytecode_constructor[index] = lambda cg: cg(source) + try: + index_to_external_object_weakref[index] = weakref.ref(value) + except TypeError as e: + from .exc import unimplemented + + unimplemented( + gb_type="Failed to make weakref to User Object", + context=f"user_object: {value}", + explanation="Object does not allow us to make a weakref to it", + hints=[], + from_exc=e, + ) + return index diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_deduplication.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_deduplication.py new file mode 100644 index 0000000000000000000000000000000000000000..0517fd5c1df8b6a08129ff535d104ba56463dea9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_deduplication.py @@ -0,0 +1,610 @@ +""" +This module implements graph deduplication functionality for TorchDynamo's optimization pipeline. +Graph deduplication identifies identical subgraphs in the computational graph and merges them +to reduce redundancy and improve performance. The process involves analyzing regions of the graph, +identifying structurally equivalent regions, and replacing them with a single shared implementation. +This optimization is particularly effective for models with repeated patterns or similar computational +structures across different parts of the network. +""" + +import logging +import operator +from collections import defaultdict, deque +from collections.abc import Generator, Iterable +from typing import Optional + +import torch +import torch.fx +from torch._dynamo import config +from torch.multiprocessing.reductions import StorageWeakRef +from torch.utils._ordered_set import OrderedSet + +from .graph_region_tracker import Node, Region +from .graph_utils import _detect_cycles, _get_flat_args, _get_flat_args_unique + + +# Represents an index into the region +# to select a node and then +# an index into that node's +# flattened arguments +UsageIndex = tuple[int, int] + +log = logging.getLogger(__name__) + +last_node_to_additional_deps: Optional[dict[Node, OrderedSet[Node]]] = None + + +def apply_graph_deduplication(output_graph) -> dict[str, torch.fx.GraphModule]: # type: ignore[no-untyped-def] + """ + This is the main entry point for applying the graph deduplication pass. \ +Deduplication occurs in two phases: + 1. Subgraph creation: + Subgraph creation works by taking one representative region from each region \ +group and creating a subgraph from it, which will then be used to replace all regions \ +in the group. This is implemented by first copying all nodes of the region to the new \ +subgraph and then finding all inputs which are not within the region and creating placeholders \ +for them. For the outputs, all regions in a region group need to be scanned to ensure the \ +largest set of outputs is found, and then an output node is created which returns \ +a tuple of all outputs. + + 2. Graph replacement: + To replace each region with the extracted subgraph, the node index in the region \ +and argument index within the node's flattened args and kwargs are recorded once during \ +subgraph creation. This allows us to determine which (external to the region) nodes and \ +in which order these nodes are passed as inputs. For the outputs, getitem nodes are created \ +for each output, and all nodes in the region with external outputs are replaced by the proper \ +getitem node. Finally, all original nodes are erased (there should be no uses of these \ +left in the graph). + +The deduplication mutates the output_graph argument in place. + +Returns a mapping of nodes to their subgraph output replacement node to remap outputs +when they are created in output_graph. + """ + + duplicated_region_groups = output_graph.region_tracker.get_identical_regions( + output_graph.graph + ) + node_to_mutated_arg_positions = ( + output_graph.region_tracker.node_to_mutated_arg_positions + ) + node_to_additional_deps = _populate_additional_deps( + output_graph.graph, output_graph.region_tracker.node_to_mutated_arg_positions + ) + + sub_gms: dict[str, torch.fx.GraphModule] = {} + + for region_group in duplicated_region_groups: + inds_with_external_users = _get_all_output_indices(region_group) + region = region_group[0] + ( + subgraph, + external_node_usages, + node_usage_to_tuple_elems, + ind_to_tuple_spec, + ) = _create_subgraph(region, inds_with_external_users) + + # Ignore regions with no args for now, could they possibly be evaluated at compile time? + if not list(external_node_usages): + continue + + sub_gm = torch.fx.GraphModule(output_graph.nn_modules, subgraph) + subgraph_name = output_graph.install_subgraph("subgraph", sub_gm) + sub_gms[subgraph_name] = sub_gm + with output_graph.graph.inserting_before(): + get_subgraph_node = output_graph.graph.create_node( + "get_attr", subgraph_name, (), {} + ) + + for region in region_group: + _replace_region_with_subgraph( + output_graph.graph, + region, + get_subgraph_node, + external_node_usages, + node_usage_to_tuple_elems, + ind_to_tuple_spec, + inds_with_external_users, + subgraph_name, + node_to_additional_deps, + node_to_mutated_arg_positions, + ) + + # This is to expose the updated node_to_additional_deps to tests + global last_node_to_additional_deps + last_node_to_additional_deps = node_to_additional_deps + + _stable_topological_sort( + output_graph.graph, + node_to_additional_deps, + ) + return sub_gms + + +def _replace_region_with_subgraph( + graph: torch.fx.Graph, + region: Region, + get_subgraph_node: Node, + external_node_usages: Iterable[OrderedSet[UsageIndex]], + node_usage_to_tuple_elems: dict[UsageIndex, OrderedSet[int]], + ind_to_tuple_spec: dict[int, dict[tuple[int, ...], int]], + inds_with_external_users: list[int], + subgraph_name: str, + node_to_additional_deps: dict[Node, OrderedSet[Node]], + node_to_mutated_arg_positions: dict[Node, OrderedSet[int]], +) -> None: + sub_args = [] + flattened_getitem_nodes: OrderedSet[Node] = OrderedSet() + for usages in external_node_usages: + usage = next(iter(usages)) + node_ind, usage_ind = usage + node = region[node_ind] + flattened_args_kwargs = _get_flat_args(node, {}) + for user_ind, node_usage_ind in usages: + user = region[user_ind] + if user in node_to_mutated_arg_positions: + if node_usage_ind in node_to_mutated_arg_positions[user]: + log.debug( + "NYI: Failed to substitute region %s due to mutation", region + ) + return + if usage in node_usage_to_tuple_elems: + tuple_elems = [region[i] for i in node_usage_to_tuple_elems[usage]] + flattened_getitem_nodes.update(tuple_elems) + sub_args.extend(tuple_elems) + else: + sub_args.append(flattened_args_kwargs[usage_ind]) + + # Input/Output aliasing not supported in HOPs today + # Note: we should use the nodes in the original graph (the region here) + # because we use the original traced example values for this check + if _has_aliasing( + region, sub_args, inds_with_external_users, flattened_getitem_nodes + ): + return + + invoke_args = (get_subgraph_node, subgraph_name, *sub_args) + + invoke_subgraph_node = graph.create_node( + "call_function", + torch.ops.higher_order.invoke_subgraph, + invoke_args, # type: ignore[arg-type] + {}, + ) + + ind = 0 + flattened_output_nodes: OrderedSet[Node] = OrderedSet() + for external_user_ind in inds_with_external_users: + node = region[external_user_ind] + if _is_tuple_node(node): + tuple_spec = ind_to_tuple_spec[external_user_ind] + flattened_output_nodes.update( + _replace_tuple_outputs( + node, ind, tuple_spec, invoke_subgraph_node, graph + ) + ) + ind += len(tuple_spec) + else: + subgraph_output = graph.create_node( + "call_function", operator.getitem, (invoke_subgraph_node, ind), {} + ) + node.replace_all_uses_with(subgraph_output, propagate_meta=True) + ind += 1 + + # Erase in reverse topological order + for node in reversed(region): + if node in flattened_getitem_nodes: + # Don't erase these, since they will still be used + continue + + if node not in flattened_output_nodes: + graph.erase_node(node) + + # Remove any nodes with additional deps + # This is safe; we've guaranteed that there is + # no input mutation, so all additional deps + # will be internal to the subgraph + node_to_additional_deps.pop(node, None) + for deps in node_to_additional_deps.values(): + try: + deps.remove(node) + deps.add(invoke_subgraph_node) + except KeyError: + pass + + if config.graph_deduplication_lint: + print(_detect_cycles(graph, node_to_additional_deps)) + _stable_topological_sort(graph, node_to_additional_deps) + graph.lint() + + +def _get_external_inputs( + region: Region, +) -> dict[Node, OrderedSet[UsageIndex]]: + external_node_to_usages = defaultdict[Node, OrderedSet[UsageIndex]](OrderedSet) + region_unique = set(region) + for node_ind, node in enumerate(region): + flattened_args_kwargs = _get_flat_args(node, {}) + for arg_ind, in_node in enumerate(flattened_args_kwargs): + if isinstance(in_node, Node) and in_node not in region_unique: + # in_node may occur in multiple nodes' flat_args + # track this so we can check if the arg is mutated + # Previously, we only needed to track one occurrence + # to be able to map that node to a placeholder + external_node_to_usages[in_node].add((node_ind, arg_ind)) + + return external_node_to_usages + + +def _get_all_output_indices(regions: list[Region]) -> list[int]: + # Scan all regions to get the set of all possible output nodes indices in the region + # perhaps we can record this information during region creation for more efficiency? + inds_with_external_users: set[int] = set() + for region in regions: + _get_inds_with_external_users(region, inds_with_external_users) + + return sorted(inds_with_external_users) + + +def _get_inds_with_external_users(region: Region, inds_unique: set[int]) -> None: + for ind, node in enumerate(region): + for user in node.users: + if user not in region: + if ind not in inds_unique: + inds_unique.add(ind) + + +def _create_subgraph( + region: Region, + inds_with_external_users: list[int], +) -> tuple[ + torch.fx.Graph, + list[OrderedSet[UsageIndex]], + dict[UsageIndex, OrderedSet[int]], + dict[int, dict[tuple[int, ...], int]], +]: + subgraph: torch.fx.Graph = torch.fx.Graph() + external_input_to_usages = _get_external_inputs(region) + external_node_usages = list[OrderedSet[UsageIndex]]() + region_to_subgraph_node = {} + flattened_getitem_nodes: OrderedSet[Node] = OrderedSet() + node_usage_to_tuple_elems: dict[UsageIndex, OrderedSet[int]] = {} + + for node, usage_indices in external_input_to_usages.items(): + # We don't handle tuples as inputs today + if _is_tuple_node(node): + # If a node is a tuple we will possibly create multiple placeholders for them + # and track which nodes we won't copy into the subgraph because they are flattened away + # Later, when replacing each region with this subgraph, we will create a getitem node + # externally which will perform the flattening on the outer nodes. + flattened_node_indices = _get_flattened_node_indices(node, region) + for ind in flattened_node_indices: + placeholder = subgraph.placeholder( + f"supgraph_input_{node.name}_flattened_{ind}" + ) + region_to_subgraph_node[region[ind]] = placeholder + flattened_getitem_nodes.add(region[ind]) + node_usage_to_tuple_elems[next(iter(usage_indices))] = ( + flattened_node_indices + ) + else: + placeholder = subgraph.placeholder(f"subgraph_input_{node.name}") + region_to_subgraph_node[node] = placeholder + + external_node_usages.append(usage_indices) + + def map_arg(node: Node) -> Node: + if node in region_to_subgraph_node: + return region_to_subgraph_node[node] + else: + return node + + def copy_to_subgraph(node: Node) -> Node: + subgraph_node = subgraph.node_copy(node, lambda old: map_arg(old)) + region_to_subgraph_node[node] = subgraph_node + return subgraph_node + + output_list = [] + ind_to_tuple_spec = {} + for ind, node in enumerate(region): + if node not in flattened_getitem_nodes: + subgraph_node = copy_to_subgraph(node) + if ind in inds_with_external_users: + # flatten tuple outputs by generating a getitem node tree + if _is_tuple_node(node): + getitem_nodes, ind_to_tuple_spec[ind] = _create_getitem_nodes( + node, subgraph_node, subgraph + ) + output_list.extend(getitem_nodes) + else: + output_list.append(subgraph_node) + + subgraph.output(tuple(output_list)) + + return subgraph, external_node_usages, node_usage_to_tuple_elems, ind_to_tuple_spec + + +def _stable_topological_sort_impl( + graph: torch.fx.Graph, + node_to_additional_deps: dict[Node, OrderedSet[Node]], + do_sort: bool = True, +) -> bool: + # Nodes are in exactly one of these four collections: + + # - Nodes in `pending` are waiting to be processed (in reverse order): + pending = list(reversed(graph.nodes)) + + # - Nodes in `ready` have been processed and are already in the correct + # order. + ready = OrderedSet[Node]() + + # - `waiting` is a mapping from a dependency to nodes which depend on that + # dependency. + waiting = defaultdict(list) + + # - `outputs` are always at the end of the graph + outputs = OrderedSet[Node]() + + # The cursor indicates the last processed node so we can add new nodes + # after it. + cursor = None + while pending: + node = pending.pop() + + if node.target == "output": + outputs.add(node) + assert not node.users, "output nodes should have no users" + continue + + waiting_for = [ + x + for x in _get_flat_args_unique(node, node_to_additional_deps) + if x not in ready + ] + if waiting_for: + # We have unprocessed input nodes. Might as well wait for the last + # arg so an already sorted list will only recheck this node once. + waiting[waiting_for[-1]].append(node) + else: + ready.add(node) + if cursor and cursor.next is not node and do_sort: + cursor.append(node) + cursor = node + # Mark the nodes that have been waiting for this node to finish as + # ready to check again. + pending.extend(reversed(waiting.pop(node, ()))) + + ready.update(outputs) + return not waiting and len(ready) == len(graph.nodes) + + +def _stable_topological_sort( + graph: torch.fx.Graph, + node_to_additional_deps: dict[Node, OrderedSet[Node]], +) -> None: + assert _stable_topological_sort_impl(graph, node_to_additional_deps) + + +def _has_cycle( + graph: torch.fx.Graph, + node_to_additional_deps: dict[Node, OrderedSet[Node]], +) -> bool: + return not _stable_topological_sort_impl( + graph, node_to_additional_deps, do_sort=False + ) + + +def _populate_additional_deps( + graph: torch.fx.Graph, node_to_mutated_arg_positions: dict[Node, OrderedSet[int]] +) -> dict[Node, OrderedSet[Node]]: + node_to_additional_deps: dict[Node, OrderedSet[Node]] = defaultdict(OrderedSet) + _add_mutation_dependencies(node_to_mutated_arg_positions, node_to_additional_deps) + _add_global_state_dependencies(graph, node_to_additional_deps) + return node_to_additional_deps + + +def _add_global_state_dependencies( + graph: torch.fx.Graph, node_to_additional_deps: dict[Node, OrderedSet[Node]] +) -> None: + import torch.amp + + all_nodes = list(graph.nodes) + + # These are targets of the nodes which need to stay in the same relative place in the graph + global_state_targets = {torch.amp._enter_autocast, torch.amp._exit_autocast} + all_nodes_dep_on: list[Node] = [] + + def prev_cur_nodes( + all_nodes: list[Node], + ) -> Generator[tuple[list[Node], Node], None, None]: + prev_nodes: list[Node] = [] + next_nodes = list(reversed(all_nodes)) + + while next_nodes: + cur_node = next_nodes.pop() + yield prev_nodes, cur_node + prev_nodes.append(cur_node) + + for prev_nodes, cur_node in prev_cur_nodes(all_nodes): + args_unique = _get_flat_args_unique(cur_node, {}) + new_deps = [n for n in all_nodes_dep_on if n not in args_unique] + + if new_deps: + additional_deps = node_to_additional_deps[cur_node] + additional_deps.update(new_deps) + + if cur_node.target in global_state_targets: + additional_deps = node_to_additional_deps[cur_node] + additional_deps.update(n for n in prev_nodes if n not in args_unique) + all_nodes_dep_on.append(cur_node) + + +def _add_mutation_dependencies( + node_to_mutated_arg_positions: dict[Node, OrderedSet[int]], + node_to_additional_deps: dict[Node, OrderedSet[Node]], +) -> None: + for node, indices in node_to_mutated_arg_positions.items(): + flat_args_kwargs = _get_flat_args(node, {}) + + # for all mutated args, + # add dependency on usages which occur after node to ensure + # node will always be ordered before them + # also add node as a dependency on usages which + # occur before node to ensure node is ordered after them + for index in indices: + mutated_arg = flat_args_kwargs[index] + for user in mutated_arg.users: + if user is node: + continue + # pyrefly: ignore # unsupported-operation + elif user < node: + node_to_additional_deps[node].add(user) + # pyrefly: ignore # unsupported-operation + elif user > node: + node_to_additional_deps[user].add(node) + + +def _has_aliasing( + region: Region, + inputs: list[Node], + inds_with_external_users: list[int], + flattened_getitem_nodes: OrderedSet[Node], +) -> bool: + input_storages: dict[StorageWeakRef, Node] = dict() + for node in inputs: + if node in flattened_getitem_nodes: + continue + example_value = node.meta["example_value"] + if isinstance(example_value, torch.Tensor): + storage = StorageWeakRef(example_value._typed_storage()) + if storage in input_storages: + # input-input aliasing + log.debug( + "NYI: Failed to substitute region %s due to input-output aliasing detected at nodes %s, %s", + region, + input_storages[storage], + node, + ) + return True + input_storages[storage] = node + output_storages: dict[StorageWeakRef, Node] = dict() + for i in inds_with_external_users: + out_node = region[i] + if out_node in flattened_getitem_nodes: + continue + if out_node: + example_value = out_node.meta["example_value"] + assert not isinstance(example_value, list) + if isinstance(example_value, torch.Tensor): + storage = StorageWeakRef(example_value._typed_storage()) + if storage in output_storages: + # output-output aliasing + log.debug( + "NYI: Failed to substitute region %s due to output-output aliasing detected at nodes %s, %s", + region, + output_storages[storage], + out_node, + ) + return True + output_storages[storage] = out_node + intersected_storages = input_storages.keys() & output_storages.keys() + if len(intersected_storages) > 0: + # input-output aliasing + aliased = [ + (input_storages[s], output_storages[s]) for s in intersected_storages + ] + aliased = ", ".join([f"{i} and {o}" for i, o in aliased]) + log.debug( + "NYI: Failed to substitute region %s due to input-output aliasing detected at nodes %s", + region, + aliased, + ) + return True + return False + + +def _is_tuple_node(node: Node) -> bool: + return isinstance(node.meta["example_value"], tuple) + + +def _get_children_getitems(node: Node) -> Generator[Node, None, None]: + for user in node.users: + if user.target is operator.getitem and isinstance(user.args[1], int): + yield user + + +def _get_flattened_node_indices(node: Node, region: Region) -> OrderedSet[int]: + """Returns an ordered set of indices, each representing a node in the region which will be flattened""" + flattened_node_to_ind = {n: i for i, n in enumerate(region)} + node_indices: OrderedSet[int] = OrderedSet() + queue = deque(_get_children_getitems(node)) + while queue: + cur_node = queue.popleft() + if any(user in region for user in cur_node.users): + node_indices.add(flattened_node_to_ind[cur_node]) + for child in _get_children_getitems(cur_node): + queue.append(child) + return node_indices + + +def _create_getitem_nodes( + node: Node, subgraph_tuple_node: Node, subgraph: torch.fx.Graph +) -> tuple[list[Node], dict[tuple[int, ...], int]]: + tup = node.meta["example_value"] + assert isinstance(tup, tuple), "_get_getitem_children expects tuple" + + getitem_nodes: list[Node] = [] + queue = deque([(e, (i,), subgraph_tuple_node) for i, e in enumerate(tup)]) + path_to_output_index = {} + + while queue: + cur_elem, path, parent = queue.popleft() + + with subgraph.inserting_after(parent): + new_getitem_node = subgraph.create_node( + "call_function", operator.getitem, (parent, path[-1]), {} + ) + new_getitem_node.meta["example_value"] = cur_elem + + path_to_output_index[path] = len(getitem_nodes) + getitem_nodes.append(new_getitem_node) + + if isinstance(cur_elem, tuple): + queue.extend( + [(e, path + (i,), new_getitem_node) for i, e in enumerate(cur_elem)] # type: ignore[arg-type,misc] + ) + + return getitem_nodes, path_to_output_index # type: ignore[return-value] + + +def _replace_tuple_outputs( + node: Node, + output_index: int, + tuple_spec: dict[tuple[int, ...], int], + invoke_subgraph_node: Node, + graph: torch.fx.Graph, +) -> OrderedSet[Node]: + assert _is_tuple_node(node), "_replace_tuple_outputs expects a tuple node" + + queue = deque((c, (c.args[1],)) for c in _get_children_getitems(node)) + erased_nodes: OrderedSet[Node] = OrderedSet() + while queue: + cur_node, path = queue.pop() + + for c in _get_children_getitems(cur_node): + queue.append((c, path + (c.args[1],))) # type: ignore[return-value, arg-type] + + with graph.inserting_after(invoke_subgraph_node): + subgraph_output = graph.create_node( + "call_function", + operator.getitem, + (invoke_subgraph_node, output_index + tuple_spec[path]), # type: ignore[index] + {}, + ) + cur_node.replace_all_uses_with(subgraph_output, propagate_meta=True) + graph.erase_node(cur_node) + erased_nodes.add(cur_node) + + graph.erase_node(node) + erased_nodes.add(node) + return erased_nodes diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_region_tracker.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_region_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..61f92dd06e22ce638ffec04bf2f88d9ce2a32db3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_region_tracker.py @@ -0,0 +1,502 @@ +""" +This module provides functionality for tracking and managing regions in computational graphs. +It supports graph optimization by identifying and grouping similar regions based on their +structure and behavior. The module implements algorithms for: + +1. Tracking nodes and their relationships in the computational graph +2. Identifying identical or similar regions across the graph +3. Managing graph regions for optimization purposes +4. Supporting deduplication and other graph transformation passes + +The core functionality revolves around the GraphRegionTracker class which maintains +mappings between nodes and their duplicates, enabling efficient graph analysis and +optimization operations. +""" + +from __future__ import annotations + +import copyreg +import io +import logging +import math +import operator +import pickle +from collections import defaultdict, deque +from dataclasses import fields +from typing import Any, Optional, TYPE_CHECKING, TypeVar + +import torch._logging +import torch.fx +from torch._subclasses.fake_tensor import FakeTensor +from torch.utils._ordered_set import OrderedSet +from torch.utils._pytree import tree_flatten + +from .graph_utils import _get_flat_args_unique + + +T = TypeVar("T") + + +if TYPE_CHECKING: + from collections.abc import Callable + + from .symbolic_convert import InstructionTranslatorBase + + +Node = torch.fx.Node +Region = list[Node] +IdenticalNodes = list[Node] +GlobalStateKey = tuple[ + bool, + bool, + int, + tuple[bool, bool], + tuple[bool, bool], + torch.dtype, + bool, + bool, + bool, + bool, +] + +log = logging.getLogger(__name__) +graph_expansion_log = torch._logging.getArtifactLogger( + __name__, "graph_region_expansion" +) + + +def debug_log(msg: str, *args) -> None: # type: ignore[no-untyped-def] + graph_expansion_log.debug(msg, *args) + + +def _extract_tensor_metadata_for_node_hash( + x: torch.Tensor, +) -> tuple[Callable[[T], T], tuple[Any, ...]]: + from torch._inductor.codecache import _ident, extract_tensor_metadata_for_cache_key + + out = [] + metadata = extract_tensor_metadata_for_cache_key(x) + for field in fields(metadata): + out.append(getattr(metadata, field.name)) + + return (_ident, tuple(out)) + + +class NodeHashException(Exception): + pass + + +class InputPickler(pickle.Pickler): + def __init__(self) -> None: + from torch._inductor.codecache import _ident + + stream = io.BytesIO() + self._stream = stream + super().__init__(stream) + self.dispatch_table = copyreg.dispatch_table.copy() + self.dispatch_table.update( + { + FakeTensor: _extract_tensor_metadata_for_node_hash, + torch.SymInt: lambda x: (_ident, (str(x),)), + torch.SymBool: lambda x: (_ident, (str(x),)), + torch.SymFloat: lambda x: (_ident, (str(x),)), + } + ) + self.fast = True + + def dumps(self, obj: Any) -> bytes: + """ + Pickle an object and return a byte string. + """ + try: + self.dump(obj) + return self._stream.getvalue() + except (TypeError, AttributeError) as e: + raise NodeHashException from e + finally: + self._stream.seek(0) + self._stream.truncate(0) + + +def _extract_args(arg: Any) -> Any: + if isinstance(arg, Node): + return arg.meta.get("example_value") + elif isinstance(arg, (torch.Tensor, int)): + return arg + else: + return None + + +def _normalize_args( + node: Node, +) -> tuple[tuple[str, ...], tuple[Optional[Any], ...]]: + flat_args, _ = tree_flatten(node.args) + sorted_kwargs = sorted(node.kwargs.items(), key=operator.itemgetter(0)) + sorted_keys = tuple(sorted(node.kwargs.keys())) + flat_kwargs, _ = tree_flatten(sorted_kwargs) + all_args = flat_args + flat_kwargs + return (sorted_keys, tuple(_extract_args(arg) for arg in all_args)) + + +def _sort_with_ref_region( + index_to_rank: dict[int, int], regions: list[list[Any]] +) -> None: + # sort topologically + # we need to handle edge cases where some nodes have no dependencies + # so first we map each node to its ranking + ref_region = regions[0] + sorted_indices = sorted(range(len(ref_region)), key=lambda i: index_to_rank[i]) + for region in regions: + region[:] = [region[i] for i in sorted_indices] + + +def get_global_state_key() -> GlobalStateKey: + return ( + torch.is_grad_enabled(), + torch.is_inference_mode_enabled(), + torch.get_num_threads(), + torch._C._get_cublas_allow_fp16_reduced_precision_reduction(), + torch._C._get_cublas_allow_bf16_reduced_precision_reduction(), + torch.get_default_dtype(), + torch.are_deterministic_algorithms_enabled(), + torch._C._get_cublas_allow_tf32(), + torch.is_deterministic_algorithms_warn_only_enabled(), + torch._C._autograd._saved_tensors_hooks_is_enabled(), # type: ignore[attr-defined] + ) + + +# This is typical BFS with the caveat +# that a node's children need to be explicitly +# added with the add_children() method +# The flow is yield a node and check if it's valid for all regions +# if not valid, discard and continue onto the next node +# Note: this iterates backward through the graph by looking at args/kwargs +# of a node +class BackwardBfsArgIter: + def __init__(self, origin: Node) -> None: + self._cur: Optional[Node] = origin + self._queue: deque[Optional[Node]] = deque() + + @staticmethod + def create(origin: Node) -> BackwardBfsArgIter: + it = BackwardBfsArgIter(origin) + it.add_children(origin) + # pop the origin node, since it is the origin of + # the region and does not need to be considered for addition + assert it.next() + return it + + def next(self) -> Optional[Node]: + ret = self._cur + if not self._queue: + self._cur = None + else: + self._cur = self._queue.popleft() + return ret + + def peek(self) -> Optional[Node]: + return self._cur + + def add_children(self, node: Node) -> None: + flat_args = _get_flat_args_unique(node, {}) + for arg in flat_args: + if isinstance(arg, Node): + self._append(arg) + + def _append(self, arg: Node) -> None: + if self._cur is None: + self._cur = arg + else: + self._queue.append(arg) + + def __str__(self) -> str: + return f"BackwardBfsArgIter(cur={self._cur}, queue={self._queue})" + + +class GraphRegionTracker: + """ + GraphRegionTracker tracks each node added to the output graph and generates a key based on the source location, + instruction pointer, input shapes, and global state at the time the node is inserted into the graph. Nodes with + the same key are grouped together in a list of identical nodes (the value of node_to_duplicates). + + hash_to_duplicates: Dict[str, IdenticalNodes] - A dictionary mapping the key to a list of identical nodes + node_to_duplicates: Dict[Node, IdenticalNodes] - A dictionary mapping a node to the list of identical nodes it belongs to + input_pickler: InputPickler - An instance of InputPickler used to generate a node hash + """ + + def __init__(self) -> None: + self.hash_to_duplicates: dict[str, IdenticalNodes] = defaultdict(list) + self.node_to_duplicates: dict[Node, IdenticalNodes] = {} + # Note: position is in flattened args/kwargs list + self.node_to_mutated_arg_positions: dict[Node, OrderedSet[int]] = {} + self.input_pickler = InputPickler() + + def _hash_node( + self, filename: str, lineno: int, instruction_pointer: Optional[int], node: Node + ) -> str: + from torch._inductor.codecache import sha256_hash + + key = ( + get_global_state_key(), + filename, + lineno, + instruction_pointer, + _normalize_args(node), + ) + return sha256_hash(self.input_pickler.dumps(key)) + + def _is_identical(self, n0: Node, n1: Node) -> bool: + return ( + n0 in self.node_to_duplicates + and n1 in self.node_to_duplicates + and self.node_to_duplicates[n0] is self.node_to_duplicates[n1] + and n0 is not n1 + ) + + def track_node(self, tx: InstructionTranslatorBase, node: Node) -> None: + """ + The main entry point for tracking a node. This function will hash the node argument and group + nodes with the same hash together. It updates the hash_to_duplicates and node_to_duplicates dictionaries + to track the new node. + """ + try: + if ( + node not in self.node_to_duplicates + ): # don't allow nodes to be added twice + duplicates = self.hash_to_duplicates[ + self._hash_node( + tx.f_code.co_filename, tx.lineno, tx.instruction_pointer, node + ) + ] + duplicates.append(node) + self.node_to_duplicates[node] = duplicates + except NodeHashException as e: + log.debug("Unable to hash node %s with exception %s", node, e) # noqa: G200 + + def track_node_mutations( + self, + node: Node, + flat_args_kwargs: list[Any], + id_to_initial_version: dict[int, int], + ) -> None: + """ + This function tracks which argument positions are mutated by the given node. Subgraph HOP does not support + input mutations today so we will skip regions which have inputs that are mutated. + """ + mutated_arg_positions = OrderedSet[int]() + for i, arg in enumerate(flat_args_kwargs): + val_id = id(arg) + if ( + val_id in id_to_initial_version + and id_to_initial_version[val_id] != arg._version + ): + mutated_arg_positions.add(i) + + if mutated_arg_positions: + self.node_to_mutated_arg_positions[node] = mutated_arg_positions + + def add_node_mutation( + self, + node: Node, + arg_pos: int, + ) -> None: + if node in self.node_to_mutated_arg_positions: + self.node_to_mutated_arg_positions[node].add(arg_pos) + else: + self.node_to_mutated_arg_positions[node] = OrderedSet([arg_pos]) + + def get_identical_regions(self, graph: torch.fx.Graph) -> list[list[Region]]: + """ + This function is responsible for extracting the largest regions of identical nodes from the given graph. + **Note**: This function assumes the nodes that have been tracked with track_node are in the provided graph argument. + + The algorithm proceeds as follows: + The nodes tracked via track_node above are organized into region groups. The initial region groups look like this: + [[IdenticalNode1], [IdenticalNode2], [IdenticalNode3]] and each sublist is called a region. For each region group + (starting at the topologically latest region group), the inner regions are gradually expanded one node at time from + the flattened args and kwargs of the node in each region provided that for all regions in the group, the nodes being + added are also identical (ie have the same key computed by track_node). This is checked by verifying that the two + nodes have the same identical node list in node_to_duplicates. + """ + topological_ranking = {node: i for i, node in enumerate(graph.nodes)} + region_groups_with_rank = [] + # needed to detect if replacing a region will create cycles + node_to_recursive_ancestors = _populate_recursive_ancestor_map(graph) + + # Create region groups; a region group is a group + # of regions that are all identical. In this initial state + # each region in the group is a single node, and we discard + # groups that are only a single region. + # We track the topological ranking to start with groups later in the graph + # the reason for this is that we will necessarily create the largest groups first. + for group in self.hash_to_duplicates.values(): + if len(group) > 1: + region_group = [] + min_rank = math.inf + # pyrefly: ignore [bad-assignment] + for node in group: + # some nodes aren't in the topo ranking? + if node in topological_ranking: + min_rank = min(min_rank, topological_ranking[node]) + region_group.append([node]) + + if len(region_group) > 1: + region_groups_with_rank.append((region_group, min_rank)) + + region_groups_with_rank.sort(key=lambda rg: -rg[1]) + region_groups = [rg for rg, _ in region_groups_with_rank] + + # We start from regions later in the graph and expand them earlier + # as a result, we will create the largest regions first and they won't + # overlap. + seen_nodes: set[Node] = set() + for region_group in region_groups: + fully_expand_region_group( + region_group, + seen_nodes, + node_to_recursive_ancestors, + self._is_identical, + ) + # sort topologically + # we need to handle edge cases where some nodes have no dependencies + # so first we map each node to its ranking, + ref_region = region_group[0] + index_to_rank = { + index: topological_ranking[n] for index, n in enumerate(ref_region) + } + _sort_with_ref_region(index_to_rank, region_group) + + return [ + region_group for region_group in region_groups if len(region_group[0]) > 1 + ] + + def __str__(self) -> str: + return f"GraphRegionTracker(hash_to_duplicates={self.hash_to_duplicates}, node_to_duplicates={self.node_to_duplicates})" + + +class RegionWrapper: + """Holds state for regions e.g. ancestors and new candidate nodes for consideration""" + + def __init__( + self, region: Region, node_to_recursive_ancestors: dict[Node, set[Node]] + ) -> None: + assert len(region) == 1, "all regions should start with one node" + node = region[0] + self.node_to_recursive_ancestors = node_to_recursive_ancestors + self.iter = BackwardBfsArgIter.create(node) + self.nodes_unique = OrderedSet([node]) + self.ancestors = set(node_to_recursive_ancestors[node]) + self.region = region + + def next_candidate(self) -> Optional[Node]: + return self.iter.next() + + def will_inclusion_create_cycle(self, node: Node) -> bool: + external_users = [user for user in node.users if user not in self.nodes_unique] + for user in external_users: + if user in self.ancestors: + return True + + return False + + def add(self, node: Node) -> None: + self.nodes_unique.add(node) + self.region.append(node) + self.iter.add_children(node) + self.ancestors.update(self.node_to_recursive_ancestors[node]) + + +def fully_expand_region_group( + regions: list[Region], + seen_nodes: set[Node], + node_to_recursive_ancestors: dict[Node, set[Node]], + is_identical_fn: Callable[[Node, Node], bool], +) -> None: + debug_log("--------------------------------------------------") + debug_log("expanding new region group: %s", regions) + + # All regions should start with 1 node + assert all(len(region) == 1 for region in regions) + region_wrappers = [ + RegionWrapper(region, node_to_recursive_ancestors) for region in regions + ] + + nodes_to_add = OrderedSet[Node]() + current_node = region_wrappers[0].next_candidate() + + # No children + if current_node is None: + return + + # Loop incrementally adding new nodes to each region + # regions are only expanded if the node to add is valid + # for ALL regions + while current_node: + add_to_all_regions = not region_wrappers[0].will_inclusion_create_cycle( + current_node + ) + nodes_to_add.clear() + nodes_to_add.add(current_node) + for region_wrapper in region_wrappers[1:]: + candidate = region_wrapper.next_candidate() + + debug_log("--------------------") + debug_log( + "considering candidate: %s, cur_node: %s", candidate, current_node + ) + + if not candidate or not add_to_all_regions: + add_to_all_regions = False + continue + + debug_log( + "candidate in previously claimed nodes?: %s", candidate in seen_nodes + ) + debug_log("is_identical: %s", is_identical_fn(candidate, current_node)) + + add_to_all_regions &= ( + candidate not in seen_nodes + and candidate not in nodes_to_add + and candidate.op != "placeholder" + and candidate.op != "get_attr" + and is_identical_fn(candidate, current_node) + and not region_wrapper.will_inclusion_create_cycle(candidate) + ) + nodes_to_add.add(candidate) + + debug_log(f"add_to_all_regions: {add_to_all_regions}") + debug_log("--------------------") + + if add_to_all_regions: + assert len(region_wrappers) == len(nodes_to_add), ( + "Number of nodes to add must equal the number of regions" + ) + for region_wrapper, node in zip(region_wrappers, nodes_to_add): + region_wrapper.add(node) + debug_log("adding %s's children", node) + debug_log("%s %s", node.args, list(node.kwargs.items())) + seen_nodes.add(node) + + current_node = region_wrappers[0].next_candidate() + + # Ensure regions are sorted in topological order + for region in regions: + region.reverse() + + debug_log("end expand new region group: %s", regions) + debug_log("--------------------------------------------------") + + +def _populate_recursive_ancestor_map(graph: torch.fx.Graph) -> dict[Node, set[Node]]: + node_to_recursive_ancestors: dict[Node, set[Node]] = {} + for node in graph.nodes: + node_to_recursive_ancestors[node] = set() + for node in graph.nodes: + all_args = _get_flat_args_unique(node, {}) + for arg in all_args: + if isinstance(arg, Node): + node_to_recursive_ancestors[node].update( + node_to_recursive_ancestors[arg] + ) + node_to_recursive_ancestors[node].add(arg) + return node_to_recursive_ancestors diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a7429e83174b88b65170a0e99e50033ff9a308e1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/graph_utils.py @@ -0,0 +1,116 @@ +from collections import deque +from typing import Any, Optional + +import torch +from torch.fx import Graph, map_arg, Node +from torch.utils._ordered_set import OrderedSet +from torch.utils._pytree import tree_flatten + + +# flattens with support for slices +# Note: a better way to do this would +# be register/unregister slices as pytree nodes +# but there is no unregister API in the pytorch +# pytree impl +def _get_flat_args( + node: Node, node_to_additional_deps: dict[Node, OrderedSet[Node]] +) -> list[Node]: + args = list[Any]() + map_arg((node.args, node.kwargs), args.append) + if node in node_to_additional_deps: + args.extend(node_to_additional_deps[node]) + return args + + +def _get_flat_args_unique( + node: Node, node_to_additional_deps: dict[Node, OrderedSet[Node]] +) -> OrderedSet[Node]: + args = OrderedSet[Node]() + map_arg((node.args, node.kwargs), args.add) + if node in node_to_additional_deps: + args.update(node_to_additional_deps[node]) + return args + + +def _detect_cycles( + graph: Graph, node_to_additional_deps: dict[Node, OrderedSet[Node]] +) -> str: + current_path: deque[Node] = deque() + current_path_set: set[Node] = set() + pending: deque[tuple[Node, Node]] = deque() + + def add_to_current_path(node: Node) -> None: + current_path.append(node) + current_path_set.add(node) + + def pop_current_path() -> None: + node = current_path.pop() + current_path_set.remove(node) + + def current_path_head() -> Node: + return current_path[-1] + + for origin in graph.find_nodes(op="output"): + current_path.clear() + current_path_set.clear() + add_to_current_path(origin) + for child in _get_flat_args_unique(origin, node_to_additional_deps): + pending.append((child, origin)) + + while pending: + cur_node, parent = pending.pop() + + # handle backtracking + while current_path and current_path_head() != parent: + pop_current_path() + + if not isinstance(cur_node, Node): + continue + + if cur_node in current_path_set: + current_path.append(cur_node) + return f"cycle detected in path: {current_path}" + + add_to_current_path(cur_node) + + for child in _get_flat_args_unique(cur_node, node_to_additional_deps): + pending.append((child, cur_node)) + + return "no cycle detected" + + +def _graph_device_type(graph: Optional[Graph]) -> str: + if graph is None: + return "cpu" + + def _device_type(x: Any) -> str: + if isinstance(x, torch.device): + return x.type + if isinstance(x, torch.Tensor): + return x.device.type + return "cpu" + + def _flatten_meta(node: Node, key: str) -> list[Any]: + if key not in node.meta: + return [] + flat, _ = tree_flatten(node.meta[key]) + return flat + + for node in graph.nodes: + for key in ("val", "example_value"): + for obj in _flatten_meta(node, key): + return _device_type(obj) + + # Check for device conversions + if node.op == "call_method": + for gpu in ["cuda", "xpu"]: + if node.target == gpu: + return gpu + if node.target == "to" and gpu in node.args: + return gpu + + # Check args/kwargs for non-CPU device specs + flat_args, _ = tree_flatten((node.args, node.kwargs)) + for obj in flat_args: + return _device_type(obj) + return "cpu" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/guards.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/guards.py new file mode 100644 index 0000000000000000000000000000000000000000..67fe4ff7d1fbd9a14edd9661eeb26e20072774a1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/guards.py @@ -0,0 +1,4645 @@ +""" +Core guard system for Dynamo that detects when compiled code needs to be recompiled due to +changes in program state. Guards are conditions that must remain true for previously-compiled +code to be valid for reuse. + +This module provides the infrastructure for creating, managing and checking guards, including: +- Guard creation and composition +- Guard state management and invalidation +- Guard checking and failure handling +- Utilities for guard optimization and debugging +- Integration with Dynamo's compilation caching + +The guard system is critical for Dynamo's ability to efficiently reuse compiled code while +maintaining correctness by detecting when recompilation is necessary due to changes in +program state, tensor properties, or control flow. +""" + +from __future__ import annotations + +import ast +import builtins +import collections +import dataclasses +import enum +import functools +import importlib +import inspect +import io +import logging +import math +import pickle +import sys +import textwrap +import traceback +import types +import warnings +import weakref +from contextlib import contextmanager +from copy import deepcopy +from inspect import currentframe +from typing import Any, NoReturn, Optional, TYPE_CHECKING, Union + + +try: + from typing import LiteralString +except ImportError: + from typing_extensions import LiteralString + +from typing_extensions import TypeAliasType, TypeVar +from weakref import ReferenceType + +import torch +import torch.overrides +import torch.utils._device +from torch._C._dynamo.eval_frame import code_framelocals_names +from torch._C._dynamo.guards import ( + check_obj_id, + check_type_id, + ClosureGuardAccessor, + CodeGuardAccessor, + dict_version, + DictGetItemGuardAccessor, + DictGuardManager, + FuncDefaultsGuardAccessor, + FuncKwDefaultsGuardAccessor, + GetAttrGuardAccessor, + GetGenericDictGuardAccessor, + GuardAccessor, + GuardDebugInfo, + GuardManager, + install_no_tensor_aliasing_guard, + install_object_aliasing_guard, + install_storage_overlapping_guard, + install_symbolic_shape_guard, + LeafGuard, + profile_guard_manager, + RelationalGuard, + RootGuardManager, + TupleGetItemGuardAccessor, + TypeDictGuardAccessor, + TypeGuardAccessor, + TypeMROGuardAccessor, +) +from torch._dynamo.source import ( + get_global_source_name, + get_local_source_name, + IndexedSource, + is_from_flatten_script_object_source, + is_from_local_source, + is_from_optimizer_source, + is_from_skip_guard_source, + is_from_unspecialized_builtin_nn_module_source, + TensorProperty, + TensorPropertySource, +) +from torch._dynamo.utils import CompileEventLogger, get_metrics_context +from torch._guards import ( + CompileContext, + CompileId, + DuplicateInputs, + Guard, + GuardBuilderBase, + GuardEnvExpr, + GuardSource, + Source, + StorageOverlap, +) +from torch._inductor.utils import IndentedBuffer +from torch._library.opaque_object import is_opaque_value_type +from torch._logging import structured +from torch._utils_internal import justknobs_check +from torch.fx.experimental.symbolic_shapes import ( + _CppShapeGuardsHelper, + _ShapeGuardsHelper, + EqualityConstraint, + is_symbolic, + SYMPY_INTERP, +) +from torch.utils import _pytree as pytree +from torch.utils._ordered_set import OrderedSet +from torch.utils._traceback import format_frame, report_compile_source_on_error +from torch.utils.weak import TensorWeakRef + +from . import config, convert_frame, exc +from .eval_frame import set_guard_error_hook +from .source import ( + AttrProxySource, + AttrSource, + CallFunctionNoArgsSource, + CallMethodItemSource, + ChainedSource, + ClosureSource, + CodeSource, + CollectionsSource, + ConstantSource, + ConstDictKeySource, + CurrentStreamSource, + DataclassFieldsSource, + DefaultsSource, + DictGetItemSource, + DictSubclassGetItemSource, + DynamicScalarSource, + FlattenScriptObjectSource, + FloatTensorSource, + FSDPNNModuleSource, + GenericAttrSource, + GetItemSource, + GlobalSource, + GlobalStateSource, + GlobalWeakRefSource, + GradSource, + ListGetItemSource, + LocalSource, + NamedTupleFieldsSource, + NNModuleSource, + NonSerializableSetGetItemSource, + NumpyTensorSource, + OptimizerSource, + ScriptObjectQualifiedNameSource, + ShapeEnvSource, + SubclassAttrListSource, + TorchFunctionModeStackSource, + TorchSource, + TupleIteratorGetItemSource, + TypeDictSource, + TypeMROSource, + TypeSource, + UnspecializedBuiltinNNModuleSource, + UnspecializedNNModuleSource, + UnspecializedParamBufferSource, + WeakRefCallSource, +) +from .types import ( # noqa: F401 + CacheEntry, + DynamoFrameType, + ExtraState, + GuardedCode, + GuardFail, + GuardFilterEntry, + GuardFn, +) +from .utils import ( + builtin_dict_keys, + common_constant_types, + dataclass_fields, + dict_keys, + get_current_stream, + get_custom_getattr, + get_torch_function_mode_stack, + get_torch_function_mode_stack_at, + guard_failures, + istype, + key_is_id, + key_to_id, + normalize_range_iter, + orig_code_map, + tensor_always_has_static_shape, + tuple_iterator_getitem, + tuple_iterator_len, + unpatched_nn_module_getattr, + verify_guard_fn_signature, +) + + +if TYPE_CHECKING: + from collections.abc import Callable + + +guard_manager_testing_hook_fn: Optional[Callable[[Any, Any, Any], Any]] = None + +try: + import numpy as np +except ModuleNotFoundError: + np = None # type: ignore[assignment] + + +if TYPE_CHECKING: + from collections.abc import Generator, KeysView, Sequence + + from sympy import Symbol + + from torch._C import DispatchKeySet + from torch._dynamo.output_graph import OutputGraphCommon, OutputGraphGuardsState + +T = TypeVar("T") +log = logging.getLogger(__name__) +guards_log = torch._logging.getArtifactLogger(__name__, "guards") +recompiles_log = torch._logging.getArtifactLogger(__name__, "recompiles") +recompiles_verbose_log = torch._logging.getArtifactLogger( + __name__, "recompiles_verbose" +) +verbose_guards_log = torch._logging.getArtifactLogger(__name__, "verbose_guards") + + +dunder_attrs_assumed_constants = ( + "__defaults__", + "__kwdefaults__", + "__code__", + "__closure__", + "__annotations__", + "__func__", + "__mro__", +) + + +def get_framelocals_idx(code: types.CodeType, var_name: str) -> int: + # Refer to index in the frame's localsplus directly. + # NOTE: name order for a code object doesn't change. + # NOTE: we need to find the LAST matching index because <= 3.10 contains + # duplicate names in the case of cells: a name can be both local and cell + # and will take up 2 slots of the frame's localsplus. The correct behavior + # is to refer to the cell, which has a higher index. + framelocals_names_reversed = code_framelocals_names_reversed_cached(code) + framelocals_idx = ( + len(framelocals_names_reversed) - framelocals_names_reversed.index(var_name) - 1 + ) + return framelocals_idx + + +class IndentedBufferWithPrefix(IndentedBuffer): + def prefix(self) -> str: + return "| " * (self._indent * self.tabwidth) + + def writeline(self, line: str, skip_prefix: bool = False) -> None: # type: ignore[override] + if skip_prefix: + super().writeline(line) + else: + super().writeline("+- " + line) + + +class GuardManagerWrapper: + """ + A helper class that contains the root guard manager. An instance of this + class is stored in the Dynamo cache entry, so that the cache entry can + access the RootGuardManager stored in the "root" attribute and directly call + the check_nopybind from C++. + """ + + def __init__(self, root: Optional[RootGuardManager] = None) -> None: + if root is None: + self.root = RootGuardManager() + else: + self.root = root + + self.diff_guard_root: Optional[RootGuardManager] = None + self.closure_vars: Optional[dict[str, Any]] = None + self.args: Optional[list[str]] = None + self.code_parts: list[str] = [] + self.verbose_code_parts: Optional[list[str]] = None + self.global_scope: Optional[dict[str, Any]] = None + self.guard_fail_fn: Optional[Callable[[GuardFail], None]] = None + self.cache_entry: Optional[CacheEntry] = None + self.extra_state: Optional[ExtraState] = None + self.id_matched_objs: dict[str, ReferenceType[object]] = {} + self.no_tensor_aliasing_sources: list[str] = [] + + self.printed_relational_guards: set[RelationalGuard] = set() + + self.diff_guard_sources: OrderedSet[str] = OrderedSet() + + @contextmanager + def _preserve_printed_relational_guards(self) -> Generator[None, None, None]: + self.printed_relational_guards = set() + try: + yield + finally: + self.printed_relational_guards = set() + + # TODO: clarify what fn and attributes guard manager has to get the right things here + def collect_diff_guard_sources(self) -> OrderedSet[str]: + # At the time of finalize, we have only marked guard managers with + # TENSOR_MATCH guards as diff guard managers. So, we do a tree traversal + # and collect all the nodes in the tree (branches) that lead to tensor + # guards. + + # After a recompilation, some of guard managers will have a fail_count > + # 0, so we collect them as well. Later on, we accumulate the diff guard + # sources for all the guard managers. + + def visit_dict_manager(node: DictGuardManager) -> bool: + is_diff_guard_node = ( + node.get_source() in self.diff_guard_sources or node.fail_count() > 0 + ) + for _idx, (key_mgr, val_mgr) in sorted( + node.get_key_value_managers().items() + ): + is_diff_guard_node |= visit(key_mgr) | visit(val_mgr) + + if is_diff_guard_node: + self.diff_guard_sources.add(node.get_source()) + + return is_diff_guard_node + + def visit_manager(node: GuardManager) -> bool: + assert not isinstance(node, DictGuardManager) + + is_diff_guard_node = ( + node.get_source() in self.diff_guard_sources or node.fail_count() > 0 + ) + for child_mgr in node.get_child_managers(): + is_diff_guard_node |= visit(child_mgr) + + if is_diff_guard_node: + self.diff_guard_sources.add(node.get_source()) + + return is_diff_guard_node + + def visit(node: GuardManager) -> bool: + if node is None: + return False + if isinstance(node, DictGuardManager): + return visit_dict_manager(node) + return visit_manager(node) + + visit(self.root) + + return self.diff_guard_sources + + def finalize(self) -> None: + if config.use_recursive_dict_tags_for_guards and justknobs_check( + "pytorch/compiler:use_recursive_dict_tags_for_guards" + ): + self.find_tag_safe_roots() + self.prepare_diff_guard_manager() + + def prepare_diff_guard_manager(self) -> None: + self.collect_diff_guard_sources() + self.populate_diff_guard_manager() + + def find_tag_safe_roots(self) -> None: + """ + Identify ``tag safe nodes`` and ``tag safe roots`` within a guard tree. + + ----------------------------------------------------------------------- + tag safe node + ----------------------------------------------------------------------- + A *tag safe node* is a ``GuardManager`` whose guarded value satisfies one + of the following conditions: + + 1. Immutable value - The value is intrinsically immutable according to + ``is_immutable_object``. Tensors are considered immutable. To ensure + that symbolic guards run, we also check that the GuardManager has no + accessors. + + 2. Nested tag safe dictionary - The value is a ``dict`` whose keys and + values are all tag safe nodes (checked recursively). Such dictionaries + allow entire nested structures to be skipped once their identity tag + matches. + + 3. Pure ``nn.Module`` - The value is an ``nn.Module`` whose sole + accessor is ``GetGenericDictGuardAccessor``—i.e., it only exposes its + ``__dict__`` and nothing else that could mutate between runs. + + For every tag safe node, verifying the identity/tag of just the top-level + dictionary is enough to guarantee the entire subtree is unchanged, enabling + a *fast-path* guard check. + + ----------------------------------------------------------------------- + tag safe root + ----------------------------------------------------------------------- + A ``tag safe root`` is a tag safe node whose parent is not tag safe. + These boundary nodes mark the points where guard evaluation can safely + prune traversal: if a tag-safe root's dictionary tag matches, the entire + subtree beneath it is skipped. + + One strong requirement for tag safe root is for the guarded object to + support weakref. Refer to more details in the Recursive dict tag + matching note. In short, we need to save the weakref of the object on + first invocation, and check if it is still valid in later iterations, to + apply recursive dict tag optimizations. `dict` objects do NOT support + weakref. Therefore, as of now, we only mark nn module related guard + managers as tag safe roots. + + Algorithm + --------- + The search runs in post-order traversal + + 1. Visit leaves and classify them as tag safe or not. + 2. Propagate tag-safety upward: a parent dictionary becomes tag safe only if + all of its children are already tag-safe. + 3. Propagate tag-safe-rootness upward: if the whole subtree is tag safe, + the current node becomes the new tag safe root, otherwise propagate the + subtree tag safe roots. + 4. Collect every tag safe node and, by inspecting parent tags, label the + subset that are tag safe roots. + """ + + def check_tag_safety( + node: GuardManager, accepted_accessors: tuple[type[GuardAccessor], ...] + ) -> bool: + accessors = node.get_accessors() + child_mgrs = node.get_child_managers() + return all( + isinstance(accessor, accepted_accessors) and mgr.is_tag_safe() + for accessor, mgr in zip(accessors, child_mgrs) + ) + + def visit_dict_manager(node: DictGuardManager) -> list[GuardManager]: + # Just recurse through the key and value dict managers and check if + # all of them are tag safe nodes. + assert issubclass(node.get_type_of_guarded_value(), dict) + + tag_safe_roots = [] + is_subtree_tag_safe = True + + # Recurse to get the tag safe roots from subtree. + for _idx, (key_mgr, val_mgr) in sorted( + node.get_key_value_managers().items() + ): + if key_mgr is not None: + visit(key_mgr) + if val_mgr is not None: + tag_safe_roots.extend(visit(val_mgr)) + + for key_mgr, val_mgr in node.get_key_value_managers().values(): + if key_mgr: + is_subtree_tag_safe &= key_mgr.is_tag_safe() + + if val_mgr: + is_subtree_tag_safe &= val_mgr.is_tag_safe() + + if is_subtree_tag_safe: + node.mark_tag_safe() + return tag_safe_roots + + def visit_manager(node: GuardManager) -> list[GuardManager]: + assert not isinstance(node, DictGuardManager) + + # Collect the subtree tag safe roots + tag_safe_roots = [] + for child_mgr in node.get_child_managers(): + tag_safe_roots.extend(visit(child_mgr)) + + if node.is_guarded_value_immutable(): + # If the node guards a tensor, mark it tag safe only if there + # are no accessors. Presence of accessors means presence of + # symbolic shape guards. + if issubclass(node.get_type_of_guarded_value(), torch.Tensor): + if node.has_no_accessors() and not node.has_object_aliasing_guard(): + node.mark_tag_safe() + else: + node.mark_tag_safe() + elif issubclass(node.get_type_of_guarded_value(), dict): + accessors = node.get_accessors() + child_mgrs = node.get_child_managers() + is_subtree_tag_safe = all( + isinstance(accessor, DictGetItemGuardAccessor) and mgr.is_tag_safe() + for accessor, mgr in zip(accessors, child_mgrs) + ) + if is_subtree_tag_safe: + node.mark_tag_safe() + elif issubclass(node.get_type_of_guarded_value(), torch.nn.Module): + is_subtree_tag_safe = check_tag_safety( + node, (GetGenericDictGuardAccessor, TypeGuardAccessor) + ) + if is_subtree_tag_safe: + node.mark_tag_safe() + # Return the current node as tag safe root, discarding the + # subtree tag safe roots. + return [ + node, + ] + elif ( + node.get_type_of_guarded_value() + in ( + types.FunctionType, + types.MethodType, + staticmethod, + classmethod, + ) + and config.assume_dunder_attributes_remain_unchanged + ): + # Assumption: callers will not reassignthe attributes + # func.__code__, func.__closure__, func.__defaults__, or func.__kwdefaults__. + # Mutating the objects those attributes point to is fine; + # rebinding the attribute itself is not. + # Example ─ allowed: foo.__defaults__[0].bar = 99 + # forbidden: foo.__defaults__ = (3, 4) + is_subtree_tag_safe = check_tag_safety( + node, + ( + CodeGuardAccessor, + ClosureGuardAccessor, + FuncDefaultsGuardAccessor, + FuncKwDefaultsGuardAccessor, + GetAttrGuardAccessor, + ), + ) + + for accessor in node.get_accessors(): + if isinstance(accessor, GetAttrGuardAccessor): + is_subtree_tag_safe &= ( + accessor.get_attr_name() in dunder_attrs_assumed_constants + ) + + if is_subtree_tag_safe: + node.mark_tag_safe() + elif issubclass(node.get_type_of_guarded_value(), types.CellType): + is_subtree_tag_safe = check_tag_safety(node, (GetAttrGuardAccessor,)) + + is_subtree_tag_safe &= all( + isinstance(accessor, GetAttrGuardAccessor) + and accessor.get_attr_name() == "cell_contents" + for accessor in node.get_accessors() + ) + if is_subtree_tag_safe: + node.mark_tag_safe() + elif ( + issubclass(node.get_type_of_guarded_value(), tuple) + and node.get_source().endswith(dunder_attrs_assumed_constants) + and config.assume_dunder_attributes_remain_unchanged + ): + # We trust tuples obtained from a function's __closure__ or + # __defaults__. Any *other* tuple-valued attribute can be + # silently replaced—for example: + # + # foo.bar = (1, 2) # original + # foo.bar = (3, 4) # rebinding that our dict-tag optimisation won't see + # + # Therefore only tuples from __closure__ / __defaults__ participate in the + # recursive-dict-tag optimization; all others are ignored. + is_subtree_tag_safe = check_tag_safety( + node, (TupleGetItemGuardAccessor,) + ) + if is_subtree_tag_safe: + node.mark_tag_safe() + elif issubclass(node.get_type_of_guarded_value(), type): + is_subtree_tag_safe = check_tag_safety( + node, (TypeDictGuardAccessor, TypeMROGuardAccessor) + ) + if is_subtree_tag_safe: + node.mark_tag_safe() + + return tag_safe_roots + + def visit(node: GuardManager) -> list[GuardManager]: + if node is None: + return [] + if isinstance(node, DictGuardManager): + return visit_dict_manager(node) + return visit_manager(node) + + tag_safe_roots = visit(self.root) + for node in tag_safe_roots: + if issubclass(node.get_type_of_guarded_value(), torch.nn.Module): + node.mark_tag_safe_root() + + def populate_diff_guard_manager(self) -> None: + self.diff_guard_root = self.clone_with_chosen_sources(self.diff_guard_sources) + + # Ensure that that C++ side points to the updated diff guard manager. + # When a new GuardManagerWrapper is created, it does not have a + # cache_entry attribute, so it relies on the CacheEntry constructor to + # set the diff_guard_root in C++. But once it is saved in the Dynamo + # cache, C++ side adds a cache_entry attribute. On recompiles, this + # cache_entry is visible, so we update the C++ side to point to the + # update guard manager. + if self.cache_entry: + self.cache_entry.update_diff_guard_root_manager() + + def clone_with_chosen_sources( + self, chosen_sources: OrderedSet[str] + ) -> RootGuardManager: + def filter_fn(node_mgr: GuardManager) -> bool: + return node_mgr.get_source() in chosen_sources + + return self.root.clone_manager(filter_fn) + + def get_guard_lines(self, guard: LeafGuard) -> list[str]: + guard_name = guard.__class__.__name__ + parts = guard.verbose_code_parts() + parts = [guard_name + ": " + part for part in parts] + return parts + + def get_manager_line( + self, guard_manager: GuardManager, accessor_str: Optional[str] = None + ) -> str: + source = guard_manager.get_source() + t = guard_manager.__class__.__name__ + s = t + ": source=" + source + if accessor_str: + s += ", " + accessor_str + s += f", type={guard_manager.get_type_of_guarded_value()}" + s += f", tag_safe=({guard_manager.is_tag_safe()}, {guard_manager.is_tag_safe_root()})" + return s + + def construct_dict_manager_string( + self, mgr: DictGuardManager, body: IndentedBufferWithPrefix + ) -> None: + for idx, (key_mgr, val_mgr) in sorted(mgr.get_key_value_managers().items()): + body.writeline(f"KeyValueManager pair at index={idx}") + with body.indent(): + if key_mgr: + body.writeline(f"KeyManager: {self.get_manager_line(key_mgr)}") + self.construct_manager_string(key_mgr, body) + + if val_mgr: + body.writeline(f"ValueManager: {self.get_manager_line(val_mgr)}") + self.construct_manager_string(val_mgr, body) + + def construct_manager_string( + self, mgr: GuardManager, body: IndentedBufferWithPrefix + ) -> None: + with body.indent(): + for guard in mgr.get_leaf_guards(): + if isinstance(guard, RelationalGuard): + if guard not in self.printed_relational_guards: + self.printed_relational_guards.add(guard) + # pyrefly: ignore [bad-argument-type] + body.writelines(self.get_guard_lines(guard)) + else: + body.writelines( + [ + guard.__class__.__name__, + ] + ) + else: + body.writelines(self.get_guard_lines(guard)) + + # This works for both DictGuardManager and SubclassedDictGuardManager + if isinstance(mgr, DictGuardManager): + self.construct_dict_manager_string(mgr, body) + + # General case of GuardManager/RootGuardManager + for accessor, child_mgr in zip( + mgr.get_accessors(), mgr.get_child_managers() + ): + body.writeline( + self.get_manager_line(child_mgr, f"accessed_by={accessor.repr()}") + ) + self.construct_manager_string(child_mgr, body) + + def __str__(self) -> str: + with self._preserve_printed_relational_guards(): + body = IndentedBufferWithPrefix() + body.tabwidth = 1 + body.writeline("", skip_prefix=True) + body.writeline("TREE_GUARD_MANAGER:", skip_prefix=True) + body.writeline("RootGuardManager") + self.construct_manager_string(self.root, body) + if hasattr(self.root, "get_epilogue_lambda_guards"): + for guard in self.root.get_epilogue_lambda_guards(): + body.writelines(self.get_guard_lines(guard)) + return body.getvalue() + + def check(self, x: Any) -> bool: + # Only needed for debugging purposes. + return self.root.check(x) + + def check_verbose(self, x: Any) -> GuardDebugInfo: + # Only needed for debugging purposes. + return self.root.check_verbose(x) + + def populate_code_parts_for_debugging(self) -> None: + # This should be called when the guard manager is fully populated + relational_guards_seen = set() + + def get_code_parts(leaf_guard: LeafGuard) -> list[str]: + code_parts = [] + for verbose_code_part in leaf_guard.verbose_code_parts(): + code_part = verbose_code_part.split("#")[0].rstrip() + code_parts.append(code_part) + return code_parts + + def visit(mgr: GuardManager) -> None: + nonlocal relational_guards_seen + for guard in mgr.get_leaf_guards(): + if isinstance(guard, RelationalGuard): + if guard not in relational_guards_seen: + # pyrefly: ignore [bad-argument-type] + self.code_parts.extend(get_code_parts(guard)) + relational_guards_seen.add(guard) + else: + self.code_parts.extend(get_code_parts(guard)) + + for child_mgr in mgr.get_child_managers(): + visit(child_mgr) + + visit(self.root) + + +def from_numpy(a: Any) -> torch.Tensor: + # If not numpy array, piggy back on e.g. tensor guards to check type + # Re-enable torch function since we disable it on leaf guards + # we need it to properly construct the tensor if a default device is set + with torch.overrides._enable_torch_function(): + # pyrefly: ignore [missing-attribute] + return torch.as_tensor(a) if isinstance(a, (np.generic, np.ndarray)) else a + + +# For user stack printing +@functools.cache +def uninteresting_files() -> set[str]: + import torch._dynamo.external_utils + import torch._dynamo.polyfills + + mods = [torch._dynamo.external_utils, torch._dynamo.polyfills] + + from torch._dynamo.polyfills.loader import POLYFILLED_MODULES + + # pyrefly: ignore [bad-argument-type] + mods.extend(POLYFILLED_MODULES) + + return {inspect.getfile(m) for m in mods} + + +_CLOSURE_VARS: Optional[dict[str, object]] = None + + +def _get_closure_vars() -> dict[str, object]: + global _CLOSURE_VARS + if _CLOSURE_VARS is None: + _CLOSURE_VARS = { + "___check_type_id": check_type_id, + "___check_obj_id": check_obj_id, + "___odict_getitem": collections.OrderedDict.__getitem__, + "___key_to_id": key_to_id, + "___dict_version": dict_version, + "___dict_contains": lambda a, b: dict.__contains__(b, a), + "___tuple_iterator_len": tuple_iterator_len, + "___normalize_range_iter": normalize_range_iter, + "___tuple_iterator_getitem": tuple_iterator_getitem, + "___dataclass_fields": dataclass_fields, + "___namedtuple_fields": lambda x: x._fields, + "___get_torch_function_mode_stack_at": get_torch_function_mode_stack_at, + "___get_current_stream": get_current_stream, + "__math_isnan": math.isnan, + "__numpy_isnan": None if np is None else np.isnan, + "inf": float("inf"), + "__load_module": importlib.import_module, + "utils_device": torch.utils._device, + "device": torch.device, + "___from_numpy": from_numpy, + "___as_tensor": torch._as_tensor_fullprec, + "torch": torch, + "inspect": inspect, + } + return _CLOSURE_VARS + + +def _ast_unparse(node: ast.AST) -> str: + return ast.unparse(node).replace("\n", "") + + +strip_function_call = torch._C._dynamo.strip_function_call + + +def get_verbose_code_part(code_part: str, guard: Optional[Guard]) -> str: + extra = "" + if guard is not None: + if guard.user_stack: + for fs in reversed(guard.user_stack): + if fs.filename not in uninteresting_files(): + extra = f" # {format_frame(fs, line=True)}" + if len(extra) > 1024: + # For fx graphs, the line can be very long in case of + # torch.stack ops, where many inputs are set to None + # after the operation. This increases the size of the + # guards log file. In such cases, do not print the line + # contents. + extra = f" # {format_frame(fs)}" + break + elif guard.stack: + summary = guard.stack.summary() + if len(summary) > 0: + extra = f" # {format_frame(summary[-1])}" + else: + extra = " # " + return f"{code_part:<60}{extra}" + + +def get_verbose_code_parts( + code_parts: Union[str, list[str]], + guard: Optional[Guard], + recompile_hint: Optional[str] = None, +) -> list[str]: + if not isinstance(code_parts, list): + code_parts = [code_parts] + + verbose_code_parts = [ + get_verbose_code_part(code_part, guard) for code_part in code_parts + ] + if recompile_hint: + verbose_code_parts = [ + f"{part} (HINT: {recompile_hint})" for part in verbose_code_parts + ] + + return verbose_code_parts + + +def convert_int_to_concrete_values(dim: Any) -> Optional[int]: + if dim is None: + return None + if not is_symbolic(dim): + return dim + else: + assert isinstance(dim, torch.SymInt) + return dim.node.maybe_as_int() + + +def convert_to_concrete_values(size_or_stride: list[Any]) -> list[Optional[int]]: + return [convert_int_to_concrete_values(dim) for dim in size_or_stride] + + +def get_tensor_guard_code_part( + value: torch.Tensor, + name: str, + sizes: list[Optional[int]], + strides: list[Optional[int]], + pytype: type, + dispatch_keys: DispatchKeySet, +) -> str: + dispatch_key = ( + dispatch_keys | torch._C._dispatch_tls_local_include_set() + ) - torch._C._dispatch_tls_local_exclude_set() + dtype = value.dtype + device_index = value.device.index + requires_grad = value.requires_grad + guard_str = ( + f"check_tensor({name}, {pytype.__qualname__}, {dispatch_key}, {dtype}, " + f"device={device_index}, requires_grad={requires_grad}, size={sizes}, stride={strides})" + ) + return guard_str + + +def get_key_index(dct: dict[Any, Any], key: Any) -> int: + # Ensure that we call dict.keys and not value.keys (which can call + # overridden keys method). In the C++ guards, we relied on PyDict_Next + # to traverse the dictionary, which uses the internal data structure and + # does not call the overridden keys method. + return list(builtin_dict_keys(dct)).index(key) + + +def get_key_index_source(source: Any, index: Any) -> str: + return f"list(dict.keys({source}))[{index}]" + + +def raise_local_type_error(obj: Any) -> NoReturn: + raise TypeError( + f"Type {type(obj)} for object {obj} cannot be saved " + + "into torch.compile() package since it's defined in local scope. " + + "Please define the class at global scope (top level of a module)." + ) + + +def should_optimize_getattr_on_nn_module(value: Any) -> bool: + # If inline_inbuilt_nn_modules flag is True, Dynamo has already traced + # through the __getattr__, and therefore it is always safe to optimize + # getattr on nn modules. + return isinstance(value, torch.nn.Module) and ( + config.inline_inbuilt_nn_modules + or get_custom_getattr(value) is unpatched_nn_module_getattr + ) + + +@dataclasses.dataclass(frozen=True) +class NNModuleAttrAccessorInfo: + # Represents where is the attr name is present in the nn module attribute + # access + + # Tells that the attribute can be accessed via __dict__ + present_in_generic_dict: bool = False + + # Either the actual name or _parameters/_buffers/_modules + l1_key: Optional[str] = None + + # Actual parameter/buffer/submodule name + l2_key: Optional[str] = None + + +def getitem_on_dict_manager( + source: Union[DictGetItemSource, DictSubclassGetItemSource], + base_guard_manager: DictGuardManager, + base_example_value: Any, + example_value: Any, + guard_manager_enum: GuardManagerType, +) -> GuardManager: + base_source_name = source.base.name + if isinstance(source.index, ConstDictKeySource): + index = source.index.index + else: + assert isinstance(base_example_value, dict) + index = get_key_index(base_example_value, source.index) + + key_source = get_key_index_source(base_source_name, index) + + # Ensure that we call dict.keys and not value.keys (which can call + # overridden keys method). In the C++ guards, we relied on PyDict_Next + # to traverse the dictionary, which uses the internal data structure and + # does not call the overridden keys method. + key_example_value = list(builtin_dict_keys(base_example_value))[index] + if isinstance(key_example_value, (int, str)): + value_source = f"{base_source_name}[{key_example_value!r}]" + else: + value_source = f"{base_source_name}[{key_source}]" + if not isinstance(source.index, ConstDictKeySource): + # We have to insert a key manager guard here + # TODO - source debug string is probably wrong here. + base_guard_manager.get_key_manager( + index=index, + source=key_source, + example_value=source.index, + guard_manager_enum=GuardManagerType.GUARD_MANAGER, + ).add_equals_match_guard( + source.index, [f"{key_source} == {key_example_value!r}"] + ) + + return base_guard_manager.get_value_manager( + index=index, + source=value_source, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + + +def match_on_id_for_tensor(guard: Guard) -> bool: + source = guard.originating_source + # For numpy tensors, always use TENSOR_MATCH because __from_numpy leads + # to a new tensor every time and therefore id differs. + if isinstance(source, NumpyTensorSource): + return False + + if guard.is_specialized_nn_module(): + return True + + return source.is_dict_key() and not isinstance(source, GradSource) + + +# The ready to eval generated code (possibly multiple parts) for a guard, plus +# the original guard object that created it for provenance +@dataclasses.dataclass +class GuardCodeList: + code_list: list[str] + guard: Guard + + +class GuardManagerType(enum.Enum): + GUARD_MANAGER = 1 + DICT_GUARD_MANAGER = 2 + + +@functools.cache +def code_framelocals_names_reversed_cached(code: types.CodeType) -> list[str]: + return list(reversed(code_framelocals_names(code))) + + +class GuardBuilder(GuardBuilderBase): + def __init__( + self, + f_code: types.CodeType, + id_ref: Callable[[object, str], int], + source_ref: Callable[[Source], str], + lookup_weakrefs: Callable[[object], Optional[weakref.ref[object]]], + local_scope: dict[str, object], + global_scope: dict[str, object], + guard_manager: GuardManagerWrapper, + check_fn_manager: CheckFunctionManager, + save_guards: bool = False, + runtime_global_scope: Optional[dict[str, object]] = None, + source_get_cache: Optional[dict[str, Any]] = None, + ) -> None: + self.f_code = f_code + self.id_ref = id_ref + self.source_ref = source_ref + self.lookup_weakrefs = lookup_weakrefs + self.scope: dict[str, dict[str, object]] = {"L": local_scope, "G": global_scope} + self.src_get_value_cache: weakref.WeakKeyDictionary[Source, object] = ( + weakref.WeakKeyDictionary() + ) + self.runtime_global_scope = runtime_global_scope or global_scope + self.source_get_cache = source_get_cache or {} + self.scope["__builtins__"] = builtins.__dict__.copy() + for ( + name, + package_module, + ) in torch.package.package_importer._package_imported_modules.items(): + name = name.replace(">", "_").replace("<", "_").replace(".", "_dot_") + # Write the package module into the scope so that we can import it + self.scope["__builtins__"][name] = package_module + # Write the demangled name to the scope so that we can use it + self.scope[name] = package_module + self.guard_manager = guard_manager + + self.argnames: list[str] = [] + # Code is python expression strings generated for each guard + self.code: list[GuardCodeList] = [] + # shape_env_code is only used by builder and is used for + # shape env code. This exists only because we need to make sure + # shape env guards get run after tensor match guards (since the + # tensor match guards make sure we actually have tensors) + self.shape_env_code: list[GuardCodeList] = [] + + # Collect the guard managers and debug info to insert no tensor aliasing + # guards. + self.no_tensor_aliasing_names: list[str] = [] + self.no_tensor_aliasing_guard_managers: list[GuardManager] = [] + + self.check_fn_manager: CheckFunctionManager = check_fn_manager + + self.guard_tree_values: dict[int, Any] = {} + self.save_guards = save_guards + + # Collect the ids of dicts which need key order guarding. source_name is + # not sufficient because for nn modules, we can have different sources + # to access the same object - self._module["param"] is same as + # self.param. + self.key_order_guarded_dict_ids = set() + assert self.check_fn_manager.output_graph is not None + for source in self.check_fn_manager.output_graph.guard_on_key_order: + dict_obj = self.get(source) + if self.save_guards: + self.source_get_cache[source.name] = dict_obj + self.key_order_guarded_dict_ids.add(id(dict_obj)) + + # Keep track of weak references of objects with ID_MATCH guard. This + # info is stored alongside optimized_code and guard_manager and is used to + # limit the number of cache entries with same ID_MATCH'd object. + self.id_matched_objs: dict[str, ReferenceType[object]] = {} + + # Save the guard managers to avoid repeatedly traversing sources. + self._cached_guard_managers: dict[str, GuardManager] = {} + self._cached_duplicate_input_guards: set[tuple[str, str]] = set() + self.object_aliasing_guard_codes: list[tuple[str, str]] = [] + self.guard_nn_modules = config.guard_nn_modules and justknobs_check( + "pytorch/compiler:guard_nn_modules" + ) + self.already_added_code_parts: OrderedSet[str] = OrderedSet() + + def guard_on_dict_keys_and_ignore_order( + self, example_value: dict[Any, Any], guard: Guard + ) -> None: + dict_mgr = self.get_guard_manager(guard) + if isinstance(dict_mgr, DictGuardManager): + raise NotImplementedError( + "Not expecting a DictGuardManager. Seems like Dynamo incorrectly " + f"added the dict to tx.output.guard_on_key_order for {guard.name}" + ) + + # Iterate over the dicts and install a dict_getitem_manager. + dict_source = guard.originating_source.name + + # Ensure that we call dict.keys and not value.keys (which can call + # overridden keys method). In the C++ guards, we relied on PyDict_Next + # to traverse the dictionary, which uses the internal data structure and + # does not call the overridden keys method. + for key in builtin_dict_keys(example_value): + value = example_value[key] + value_source = DictGetItemSource(guard.originating_source, index=key) + guard_manager_enum = self.get_guard_manager_type( + value_source, example_value + ) + dict_mgr.dict_getitem_manager( + key=key, + source=f"{dict_source}[{key!r}]", + example_value=value, + guard_manager_enum=guard_manager_enum, + ) + + def guard_on_dict_keys_and_order(self, value: dict[Any, Any], guard: Guard) -> None: + # Add key managers for the DictGuardManager. Then add either an + # ID_MATCH or EQUALS_MATCH guard on the key. + dict_mgr = self.get_guard_manager(guard) + if not isinstance(dict_mgr, DictGuardManager): + raise NotImplementedError( + "Expecting a DictGuardManager. Seems like Dynamo forgot " + f"to set the right guard manager enum for {guard.name}" + ) + assert isinstance(dict_mgr, DictGuardManager) + + # Ensure that we call dict.keys and not value.keys (which can call + # overridden keys method). In the C++ guards, we relied on PyDict_Next + # to traverse the dictionary, which uses the internal data structure and + # does not call the overridden keys method. + for idx, key in enumerate(builtin_dict_keys(value)): + key_source = get_key_index_source(guard.name, idx) + key_manager = dict_mgr.get_key_manager( + index=idx, + source=key_source, + example_value=key, + guard_manager_enum=GuardManagerType.GUARD_MANAGER, + ) + if key_is_id(key): + # Install ID_MATCH guard + id_val = self.id_ref(key, key_source) + key_manager.add_id_match_guard( + id_val, + get_verbose_code_parts( + f"__check_obj_id({key_source}, {id_val})", guard + ), + ) + else: + # Install EQUALS_MATCH guard + key_manager.add_equals_match_guard( + key, get_verbose_code_parts(f"{key_source} == {key!r}", guard) + ) + + @staticmethod + def _get_generic_dict_manager_example_value(example_value: Any) -> Optional[Any]: + # due to a bug in 3.13.0 (introduced by https://github.com/python/cpython/pull/116115, + # reported in https://github.com/python/cpython/issues/125608, + # fixed by https://github.com/python/cpython/pull/125611), we cannot take + # advantage of __dict__ versions to speed up guard checks. + if ( + config.issue_3_13_0_warning + and sys.version_info >= (3, 13) + and sys.version_info < (3, 13, 1) + ): + warnings.warn( + "Guards may run slower on Python 3.13.0. Consider upgrading to Python 3.13.1+.", + RuntimeWarning, + ) + return None + return example_value + + def getattr_on_nn_module( + self, + source: AttrSource, + base_guard_manager: GuardManager, + base_example_value: Any, + example_value: Any, + base_source_name: str, + source_name: str, + guard_manager_enum: GuardManagerType, + ) -> GuardManager: + """ + This tries to avoid calling the expensive nn module custom getattr method by + checking if the attribute is accessible via __dict__. For attributes that + are not accessible via __dict__ (like descriptors), we fallback to + PyObject_GetAttr. + + There are two cases that we optimize for + 1) attributes present directly in __dict__, e.g training. + 2) parameters/buffers/modules - they can be accessed via _parameters, + _buffers, _modules keys in __dict__. For example, mod.linear can be + accessed as mod.__dict__["_parameters"]["linear"] + + The most common and expensive case for nn module guards is of type + mod.submod1.submod2.submod3.training. We avoid the python getattr of nn + modules by going through the __dict__. + """ + + def getitem_on_dict_mgr( + mgr: GuardManager, + key: Any, + source_name: str, + base_example_value: Any, + example_value: Any, + guard_manager_enum: GuardManagerType, + ) -> GuardManager: + if isinstance(mgr, DictGuardManager): + # Case where the user code relies on key order, e.g., + # named_parameters + index = get_key_index(base_example_value, key) + + # Install the key manager and add equals match guard + key_source = f"list(dict.keys({source_name}))[{index!r}]" + mgr.get_key_manager( + index=index, + source=key_source, + example_value=key, + guard_manager_enum=GuardManagerType.GUARD_MANAGER, + ).add_equals_match_guard(key, [f"{key_source} == {key!r}"]) + + # Install the value manager + return mgr.get_value_manager( + index=index, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + else: + return mgr.dict_getitem_manager( + key=key, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + + attr_name = source.member + mod_dict = base_example_value.__dict__ + + all_class_attribute_names: set[str] = set() + for x in inspect.getmro(base_example_value.__class__): + all_class_attribute_names.update(x.__dict__.keys()) + + accessor_info = NNModuleAttrAccessorInfo(False, None, None) + + if attr_name in mod_dict: + accessor_info = NNModuleAttrAccessorInfo(True, attr_name, None) + elif "_parameters" in mod_dict and attr_name in mod_dict["_parameters"]: + accessor_info = NNModuleAttrAccessorInfo(True, "_parameters", attr_name) + elif "_buffers" in mod_dict and attr_name in mod_dict["_buffers"]: + accessor_info = NNModuleAttrAccessorInfo(True, "_buffers", attr_name) + elif ( + attr_name not in all_class_attribute_names + and "_modules" in mod_dict + and attr_name in mod_dict["_modules"] + ): + # Check test_attr_precedence test - instance attributes always take precedence unless its an nn.Module. + accessor_info = NNModuleAttrAccessorInfo(True, "_modules", attr_name) + + if not accessor_info.present_in_generic_dict: + # The attribute can be accessed by __getattribute__ call, so rely on + # PyObject_GetAttr + return base_guard_manager.getattr_manager( + attr=source.member, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + else: + assert accessor_info.l1_key + l1_key = accessor_info.l1_key + l2_key = accessor_info.l2_key + + # Set source strings for debug info + mod_dict_source = f"{base_source_name}.__dict__" + l1_source_name = l2_source_name = None + l1_value = l2_value = None + l1_guard_manager_enum = l2_guard_manager_enum = None + if l2_key: + l1_source = AttrSource(source.base, l1_key) + l1_source_name = l1_source.name + l1_value = mod_dict[l1_key] + # do not guard on key order for _parameters etc unless the user code + # actually needs the key order (e.g. calling named_parameters) + l1_guard_manager_enum = self.get_guard_manager_type(l1_source, l1_value) + + l2_source_name = source_name + l2_value = example_value + l2_guard_manager_enum = self.get_guard_manager_type( + source, example_value + ) + else: + l1_source_name = source_name + l1_value = example_value + l1_guard_manager_enum = self.get_guard_manager_type( + source, example_value + ) + + # Get __dict__ accessor. No need to guard on dict key order, so use base + # Guard Manager + mod_generic_dict_manager = base_guard_manager.get_generic_dict_manager( + source=mod_dict_source, + example_value=self._get_generic_dict_manager_example_value(mod_dict), + guard_manager_enum=GuardManagerType.GUARD_MANAGER, + ) + + l1_mgr = getitem_on_dict_mgr( + mgr=mod_generic_dict_manager, + key=l1_key, + source_name=l1_source_name, + base_example_value=mod_dict, + example_value=l1_value, + guard_manager_enum=l1_guard_manager_enum, + ) + + if l2_key: + assert l2_source_name is not None and l2_guard_manager_enum is not None + return getitem_on_dict_mgr( + mgr=l1_mgr, + key=l2_key, + source_name=l2_source_name, + base_example_value=l1_value, + example_value=l2_value, + guard_manager_enum=l2_guard_manager_enum, + ) + return l1_mgr + + def requires_key_order_guarding(self, source: Source) -> bool: + source_name = source.name + if source_name == "": + return False + obj_id = id(self.get(source)) + return obj_id in self.key_order_guarded_dict_ids + + def get_guard_manager_type( + self, + source: Source, + example_value: Optional[ + Union[KeysView[Any], set[Any], frozenset[Any], dict[Any, Any]] + ], + ) -> GuardManagerType: + guard_manager_enum = GuardManagerType.GUARD_MANAGER + if self.requires_key_order_guarding(source): + # Fix this if condition + if isinstance(example_value, dict_keys): + guard_manager_enum = GuardManagerType.DICT_GUARD_MANAGER + elif isinstance(example_value, (set, frozenset)): + # we don't need to guard on key order for set/frozenset + # but the if above will be true for these types as set is + # implemented using a dict in Dynamo + guard_manager_enum = GuardManagerType.GUARD_MANAGER + else: + assert isinstance(example_value, dict) + guard_manager_enum = GuardManagerType.DICT_GUARD_MANAGER + return guard_manager_enum + + def manager_guards_on_keys(self, mgr_enum: GuardManagerType) -> bool: + return mgr_enum == GuardManagerType.DICT_GUARD_MANAGER + + def get_global_guard_manager(self) -> GuardManager: + return self.guard_manager.root.globals_dict_manager( + f_globals=self.runtime_global_scope, + source="G", + example_value=self.scope["G"], + guard_manager_enum=GuardManagerType.GUARD_MANAGER, + ) + + def get_guard_manager_from_source(self, source: Source) -> GuardManager: + root_guard_manager = self.guard_manager.root + + example_value = None + source_name = source.name + + if source_name != "" and source_name in self._cached_guard_managers: + return self._cached_guard_managers[source_name] + + if source_name != "": + example_value = self.get(source) + self.guard_tree_values[id(example_value)] = example_value + + guard_manager_enum = self.get_guard_manager_type(source, example_value) + + # Get base manager related information + base_source_name = None + base_example_value = None + base_guard_manager = None + base_guard_manager_enum = GuardManagerType.GUARD_MANAGER + if isinstance(source, ChainedSource): + base_source_name = source.base.name + base_example_value = self.get(source.base) + base_guard_manager = self.get_guard_manager_from_source(source.base) + base_guard_manager_enum = self.get_guard_manager_type( + source.base, base_example_value + ) + + # Use istype instead of isinstance to check for exact type of source. + if istype(source, LocalSource): + framelocals_idx = get_framelocals_idx(self.f_code, source.local_name) + out = root_guard_manager.framelocals_manager( + key=(source.local_name, framelocals_idx), + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, GlobalSource): + # Global manager accepts a dict but it is not a DictGuardManager + # because globals dict is big and we typically guard on a very + # selected items on globals. + out = self.get_global_guard_manager().dict_getitem_manager( + key=source.global_name, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, GlobalWeakRefSource): + out = self.get_global_guard_manager().global_weakref_manager( + global_name=source.global_name, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, GlobalStateSource): + # Don't do anything here. We guard on global state completely in + # C++. So just return the root mgr. + return root_guard_manager + elif istype(source, ShapeEnvSource): + return root_guard_manager + elif istype(source, TypeSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.type_manager( + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, TypeDictSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.type_dict_manager( + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, TypeMROSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.type_mro_manager( + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype( + source, + ( + OptimizerSource, + NNModuleSource, + UnspecializedNNModuleSource, + UnspecializedBuiltinNNModuleSource, + FSDPNNModuleSource, + ), + ): + assert base_guard_manager # to make mypy happy + out = base_guard_manager + elif istype(source, TorchSource): + out = root_guard_manager.lambda_manager( + python_lambda=lambda _: torch, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, CollectionsSource): + out = root_guard_manager.lambda_manager( + python_lambda=lambda _: collections, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, TorchFunctionModeStackSource): + out = root_guard_manager.lambda_manager( + python_lambda=lambda _: get_torch_function_mode_stack_at( + source._get_index() + ), + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, CurrentStreamSource): + out = root_guard_manager.lambda_manager( + python_lambda=lambda _: get_current_stream(source.device), + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, GradSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.grad_manager( + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, GenericAttrSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.generic_getattr_manager( + attr=source.member, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, (AttrSource, UnspecializedParamBufferSource)): + assert base_guard_manager # to make mypy happy + assert isinstance(source, AttrSource) + if should_optimize_getattr_on_nn_module(base_example_value): + assert base_source_name + out = self.getattr_on_nn_module( + source, + base_guard_manager, + base_example_value, + example_value, + base_source_name, + source_name, + guard_manager_enum, + ) + else: + out = base_guard_manager.getattr_manager( + attr=source.member, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, (DictGetItemSource, DictSubclassGetItemSource)): + assert base_guard_manager # to make mypy happy + assert isinstance(base_example_value, (dict, collections.OrderedDict)) + assert isinstance(source, (DictGetItemSource, DictSubclassGetItemSource)) + if isinstance(base_guard_manager, DictGuardManager): + assert self.manager_guards_on_keys(base_guard_manager_enum) + out = getitem_on_dict_manager( + source, + base_guard_manager, + base_example_value, + example_value, + guard_manager_enum, + ) + else: + if isinstance(source.index, ConstDictKeySource): + raise RuntimeError( + "Expecting clean index here. Likely Dynamo forgot to mark" + " a dict as guard_on_key_order" + ) + out = base_guard_manager.dict_getitem_manager( + key=source.index, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, TensorPropertySource): + out = getattr( + base_guard_manager, + f"tensor_property_{source.prop.name.lower()}_manager", + )( + idx=source.idx, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, IndexedSource): + assert base_guard_manager # to make mypy happy + + out = base_guard_manager.indexed_manager( + idx=source.idx, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, ListGetItemSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.list_getitem_manager( + key=source.index, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, GetItemSource): + assert base_guard_manager # to make mypy happy + assert not isinstance( + base_example_value, (dict, collections.OrderedDict) + ), "Use DictGetItemSource" + if isinstance(base_example_value, list) and not source.index_is_slice: + out = base_guard_manager.list_getitem_manager( + key=source.index, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif isinstance(base_example_value, tuple) and not source.index_is_slice: + out = base_guard_manager.tuple_getitem_manager( + key=source.index, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + else: + index = source.index + if source.index_is_slice: + index = source.unpack_slice() + out = base_guard_manager.getitem_manager( + key=index, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, DefaultsSource): + assert base_guard_manager # to make mypy happy + assert base_source_name + assert callable(base_example_value) + if not source.is_kw: + out = base_guard_manager.func_defaults_manager( + source=base_source_name, + example_value=base_example_value.__defaults__, + guard_manager_enum=GuardManagerType.GUARD_MANAGER, + ).getitem_manager( + key=source.idx_key, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + else: + # kwdefauts is a dict, so use a DictGuardManager + kwdefaults = base_example_value.__kwdefaults__ + assert base_source_name is not None + kw_source = base_source_name + ".__kwdefaults__" + + # kwdefaults is a dict. No need to guard on dict order. + dict_mgr = base_guard_manager.func_kwdefaults_manager( + source=kw_source, + example_value=kwdefaults, + guard_manager_enum=GuardManagerType.GUARD_MANAGER, + ) + assert not isinstance(dict_mgr, DictGuardManager) + + out = dict_mgr.dict_getitem_manager( + key=source.idx_key, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, NumpyTensorSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.lambda_manager( + python_lambda=from_numpy, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, SubclassAttrListSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.lambda_manager( + python_lambda=lambda x: x.__tensor_flatten__()[0], + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, FlattenScriptObjectSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.lambda_manager( + python_lambda=lambda x: x.__obj_flatten__(), + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, ScriptObjectQualifiedNameSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.lambda_manager( + python_lambda=lambda x: x._type().qualified_name(), + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, AttrProxySource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.lambda_manager( + python_lambda=lambda x: x.get_base(), + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, CallMethodItemSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.lambda_manager( + python_lambda=lambda x: x.item(), + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, FloatTensorSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.lambda_manager( + python_lambda=lambda x: torch._as_tensor_fullprec(x), + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, TupleIteratorGetItemSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.tuple_iterator_getitem_manager( + index=source.index, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif isinstance(source, ConstDictKeySource): + if not isinstance(base_guard_manager, DictGuardManager): + raise AssertionError( + "ConstDictKeySource can only work on DictGuardManager" + ) + out = base_guard_manager.get_key_manager( + index=source.index, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, NonSerializableSetGetItemSource): + assert base_guard_manager + out = base_guard_manager.set_getitem_manager( + index=source.index, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, WeakRefCallSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.weakref_call_manager( + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, CallFunctionNoArgsSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.call_function_no_args_manager( + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, DataclassFieldsSource): + assert base_guard_manager + out = base_guard_manager.lambda_manager( + python_lambda=lambda x: dataclass_fields(x), + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, NamedTupleFieldsSource): + assert base_guard_manager + out = base_guard_manager.lambda_manager( + python_lambda=lambda x: x._fields, + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, CodeSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.code_manager( + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, ClosureSource): + assert base_guard_manager # to make mypy happy + out = base_guard_manager.closure_manager( + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + elif istype(source, DynamicScalarSource): + assert base_guard_manager + out = base_guard_manager.lambda_manager( + python_lambda=lambda x: int(x), + source=source_name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + else: + raise AssertionError( + f"missing guard manager builder {source} - {source.name}" + ) + + self._cached_guard_managers[source.name] = out + return out + + def get_guard_manager(self, guard: Guard) -> GuardManager: + return self.get_guard_manager_from_source(guard.originating_source) + + def add_python_lambda_leaf_guard_to_root( + self, + code_parts: list[str], + verbose_code_parts: list[str], + closure_vars: Optional[dict[str, object]] = None, + is_epilogue: bool = True, + ) -> None: + if closure_vars is None: + closure_vars = _get_closure_vars() + # Adds a lambda leaf guard to the root guard manager. It wraps the + # code_parts in a function object which is then passed on to the leaf + # guard. + make_guard_fn_args = ", ".join(closure_vars.keys()) + _guard_body, pycode = build_guard_function(code_parts, make_guard_fn_args) + out: dict[str, Any] = {} + globals_for_guard_fn = {"G": self.scope["G"]} + guards_log.debug("Python shape guard function:\n%s", pycode) + exec(pycode, globals_for_guard_fn, out) + guard_fn = out["___make_guard_fn"](*closure_vars.values()) + if is_epilogue: + # Epilogue guards are run after all the other guards have finished. + # If epilogue guards contain a getattr or getitem access, one of the + # other guards would fail preventing the epilogue guards to run. + self.guard_manager.root.add_epilogue_lambda_guard( + guard_fn, verbose_code_parts + ) + else: + self.guard_manager.root.add_lambda_guard(guard_fn, verbose_code_parts) + + # Warning: use this with care! This lets you access what the current + # value of the value you are guarding on is. You probably don't want + # to actually durably save this value though (because it's specific + # to this frame!) Instead, you should be reading out some property + # (like its type) which is what you permanently install into the + # guard code. + def get( + self, + guard_or_source: Guard | Source, + closure_vars: Optional[dict[str, Any]] = None, + ) -> Any: + name = guard_or_source.name + if isinstance(guard_or_source, Source): + src = guard_or_source + else: + src = guard_or_source.originating_source + if self.source_get_cache: + if name in self.source_get_cache: + return self.source_get_cache[name] + if closure_vars is None: + closure_vars = _get_closure_vars() + ret = src.get_value(self.scope, closure_vars, self.src_get_value_cache) + if self.save_guards and ".__closure__" in name: + self.source_get_cache[name] = ret + return ret + + # Registers the usage of the source name referenced by the + # string (or stored in the Guard) as being guarded upon. It's important + # to call this before generating some code that makes use of 'guard', + # because without this call, we won't actually bind the variable + # you reference in the actual guard closure (oops!) + def arg_ref(self, guard: Union[str, Guard]) -> str: + name: str + if isinstance(guard, str): + name = guard + else: + name = guard.name + base = strip_function_call(name) + if base not in self.argnames: + is_valid = torch._C._dynamo.is_valid_var_name(base) + if is_valid: + if is_valid == 2: + log.warning("invalid var name: %s", guard) + self.argnames.append(base) + + return name + + def _guard_on_attribute( + self, + guard: Guard, + attr_name: str, + guard_fn: Callable[[GuardBuilderBase, Guard], Any], + ) -> None: + if attr_name == "__code__": + attr_source = CodeSource(guard.originating_source) + else: + attr_source = AttrSource(guard.originating_source, attr_name) # type: ignore[assignment] + # Copy the stack info + new_guard = Guard( + attr_source, guard_fn, stack=guard.stack, user_stack=guard.user_stack + ) + new_guard.create(self) + + # Note: the order of the guards in this file matters since we sort guards on the same object by lineno + def HASATTR(self, guard: Guard) -> None: + source = guard.originating_source + if isinstance(source, NNModuleSource): + source = source.base + if isinstance(source, CodeSource): + # No need to guard that a function has a __code__ attribute + return + assert isinstance(source, AttrSource), f"invalid source {guard.name}" + base_source = source.base + base = base_source.name + attr = source.member + + ref = self.arg_ref(base) + val = hasattr(self.get(base_source), attr) + code = None + if val: + code = f"hasattr({ref}, {attr!r})" + else: + code = f"not hasattr({ref}, {attr!r})" + + if code in self.already_added_code_parts: + return + + self._set_guard_export_info( + guard, [code], provided_guarded_object=self.get(base_source) + ) + + base_manager = self.get_guard_manager_from_source(base_source) + if val: + # Just install a getattr manager. GetAttrGuardAccessor itself + # acts as hasattr guard. + example_value = self.get(source) + base_example_value = self.get(base_source) + guard_manager_enum = self.get_guard_manager_type(source, example_value) + + # if the base value is nn.Module, check if we can speedup the + # guard by going through __dict__ attrs. + if should_optimize_getattr_on_nn_module(base_example_value): + self.getattr_on_nn_module( + source, + base_manager, + base_example_value, + example_value, + base, + source.name, + guard_manager_enum, + ) + else: + base_manager.getattr_manager( + attr=attr, + source=guard.name, + example_value=example_value, + guard_manager_enum=guard_manager_enum, + ) + else: + base_manager.add_no_hasattr_guard(attr, get_verbose_code_parts(code, guard)) + self.already_added_code_parts.add(code) + + def NOT_PRESENT_IN_GENERIC_DICT( + self, guard: Guard, attr: Optional[Any] = None + ) -> None: + assert attr is not None + ref = self.arg_ref(guard) + val = self.get(guard) + + base_manager = self.get_guard_manager(guard) + + code = f"not ___dict_contains({attr!r}, {ref}.__dict__)" + if code in self.already_added_code_parts: + return + + mod_dict_source = f"{guard.name}.__dict__" + mod_generic_dict_manager = base_manager.get_generic_dict_manager( + source=mod_dict_source, + example_value=self._get_generic_dict_manager_example_value(val.__dict__), + guard_manager_enum=GuardManagerType.GUARD_MANAGER, + ) + + mod_generic_dict_manager.add_dict_contains_guard( + False, attr, get_verbose_code_parts(code, guard) + ) + self.already_added_code_parts.add(code) + + def TYPE_MATCH(self, guard: Guard) -> None: + # ___check_type_id is same as `id(type(x)) == y` + value = self.get(guard) + if isinstance(value, torch._subclasses.FakeTensor) and value.pytype: + t = value.pytype + else: + t = type(value) + + if t.__qualname__ != t.__name__: + # Type match guards must be local scope, this is + # raised in self.serialize_guards + guard._unserializable = True + + obj_id = self.id_ref(t, f"type({guard.name})") + type_repr = repr(t) + code = f"___check_type_id({self.arg_ref(guard)}, {obj_id}), type={type_repr}" + self._set_guard_export_info(guard, [code]) + + self.get_guard_manager(guard).add_type_match_guard( + obj_id, get_verbose_code_parts(code, guard) + ) + + def DICT_VERSION(self, guard: Guard) -> None: + # ___check_dict_version is same as `dict_version(x) == y` + ref = self.arg_ref(guard) + val = self.get(guard) + version = dict_version(self.get(guard)) + code = f"___dict_version({ref}) == {version}" + self._set_guard_export_info(guard, [code]) + + # TODO(anijain2305) - Delete this when DictGuardManager uses tags + # for dicts. + self.get_guard_manager(guard).add_dict_version_guard( + val, get_verbose_code_parts(code, guard) + ) + + def DICT_CONTAINS(self, guard: Guard, key: str, invert: bool) -> None: + dict_ref = self.arg_ref(guard) + + maybe_not = "not " if invert else "" + code = f"{maybe_not}___dict_contains({key!r}, {dict_ref})" + if code in self.already_added_code_parts: + return + self._set_guard_export_info(guard, [code]) + + self.get_guard_manager(guard).add_dict_contains_guard( + not invert, key, get_verbose_code_parts(code, guard) + ) + self.already_added_code_parts.add(code) + + def SET_CONTAINS(self, guard: Guard, key: Any, invert: bool) -> None: + set_ref = self.arg_ref(guard) + item = key + contains = not invert # install_dict_contains_guard inverts "contains" + + code = f"set.__contains__({set_ref}, {item!r})" + if code in self.already_added_code_parts: + return + + self._set_guard_export_info(guard, [code]) + + self.get_guard_manager(guard).add_set_contains_guard( + contains, item, get_verbose_code_parts(code, guard) + ) + self.already_added_code_parts.add(code) + + def BOOL_MATCH(self, guard: Guard) -> None: + # checks val == True or val == False + ref = self.arg_ref(guard) + val = self.get(guard) + assert istype(val, bool) + code = [f"{ref} == {val!r}"] + self._set_guard_export_info(guard, code) + + if val: + self.get_guard_manager(guard).add_true_match_guard( + get_verbose_code_parts(code, guard) + ) + else: + self.get_guard_manager(guard).add_false_match_guard( + get_verbose_code_parts(code, guard) + ) + + def NONE_MATCH(self, guard: Guard) -> None: + # checks `val is None` + ref = self.arg_ref(guard) + val = self.get(guard) + assert val is None + code = [f"{ref} is None"] + self._set_guard_export_info(guard, code) + + self.get_guard_manager(guard).add_none_match_guard( + get_verbose_code_parts(code, guard) + ) + + def ID_MATCH(self, guard: Guard, recompile_hint: Optional[str] = None) -> None: + # TODO - Run a CI with the following uncommented to find the remaining places + # val = self.get(guard) + # if inspect.isclass(val): + # raise AssertionError(f"{guard.name} is a class, use CLASS_MATCH guard") + # if inspect.ismodule(val): + # raise AssertionError(f"{guard.name} is a module, use MODULE_MATCH guard") + return self.id_match_unchecked(guard, recompile_hint) + + def id_match_unchecked( + self, guard: Guard, recompile_hint: Optional[str] = None + ) -> None: + # ___check_obj_id is same as `id(x) == y` + if isinstance(guard.originating_source, TypeSource): + # optional optimization to produce cleaner/faster guard code + return self.TYPE_MATCH( + Guard(guard.originating_source.base, GuardBuilder.TYPE_MATCH) # type: ignore[arg-type] + ) + + ref = self.arg_ref(guard) + val = self.get(guard) + id_val = self.id_ref(val, guard.name) + try: + type_repr = repr(val) + except Exception: + # During deepcopy reconstruction or other state transitions, + # objects may be in an incomplete state where repr() fails + type_repr = f"<{type(val).__name__}>" + code = f"___check_obj_id({ref}, {id_val}), type={type_repr}" + self._set_guard_export_info(guard, [code], provided_func_name="ID_MATCH") + self.get_guard_manager(guard).add_id_match_guard( + id_val, get_verbose_code_parts(code, guard, recompile_hint) + ) + + # Keep track of ID_MATCH'd objects. This will be used to modify the + # cache size logic + if isinstance(guard.originating_source, LocalSource): + # TODO(anijain2305) - This is currently restricted to nn.Module objects + # because many other ID_MATCH'd objects fail - like DeviceMesh. + # Increase the scope of ID_MATCH'd objects. + if isinstance(val, torch.nn.Module): + local_name = guard.originating_source.local_name + weak_id = self.lookup_weakrefs(val) + if weak_id is not None: + self.id_matched_objs[local_name] = weak_id + + def NOT_NONE_MATCH(self, guard: Guard, value: Optional[Any] = None) -> None: + ref = self.arg_ref(guard) + val = self.get(guard) + assert isinstance(val, torch.Tensor) + code = f"{ref} is not None" + self._set_guard_export_info(guard, [code]) + + self.get_guard_manager(guard).add_not_none_guard( + get_verbose_code_parts(code, guard) + ) + + def DISPATCH_KEY_SET_MATCH(self, guard: Guard) -> None: + ref = self.arg_ref(guard) + val = self.get(guard) + assert isinstance(val, torch._C.DispatchKeySet) + code_parts = f"{ref}.raw_repr() == {val!r}.raw_repr()" + + self.get_guard_manager(guard).add_dispatch_key_set_guard( + val, get_verbose_code_parts(code_parts, guard) + ) + + def DUAL_LEVEL(self, guard: Guard) -> None: + # Invalidate dual level if current dual level is different than the one + # in the fx graph + assert self.check_fn_manager.output_graph is not None + dual_level = self.check_fn_manager.output_graph.dual_level + code = [f"torch.autograd.forward_ad._current_level == {dual_level}"] + self._set_guard_export_info(guard, code) + self.guard_manager.root.add_dual_level_match_guard( + dual_level, get_verbose_code_parts(code, guard) + ) + + def FUNCTORCH_STACK_MATCH(self, guard: Guard) -> None: + # Invalidate functorch code if current level is different than + # the one when FX graph was generated + assert self.check_fn_manager.output_graph is not None + cis = self.check_fn_manager.output_graph.functorch_layers + states = [ci.get_state() for ci in cis] + code = [f"torch._functorch.pyfunctorch.compare_functorch_state({states})"] + self._set_guard_export_info(guard, code) + + # TODO(anijain2305) - Consider this moving this guard to C++ + compare_fn = torch._functorch.pyfunctorch.compare_functorch_state + + def fn(x: Any) -> bool: + return compare_fn(states) + + self.guard_manager.root.add_lambda_guard( + fn, get_verbose_code_parts(code, guard) + ) + + def AUTOGRAD_SAVED_TENSORS_HOOKS(self, guard: Guard) -> None: + get_hooks = torch._functorch._aot_autograd.utils.top_saved_tensors_hooks + are_inline_hooks = ( + torch._functorch._aot_autograd.utils.saved_tensors_hooks_are_inlineable + ) + + def hooks_ids_fn( + hooks: tuple[Callable[[torch.Tensor], Any], Callable[[Any], torch.Tensor]], + ) -> Optional[tuple[int, ...]]: + if not are_inline_hooks(hooks): + return None + + return tuple(map(id, hooks)) + + guard_hooks_ids = hooks_ids_fn(get_hooks()) + + code = [ + f"torch._functorch.aot_autograd.utils.top_saved_tensors_hooks ids == {guard_hooks_ids}" + ] + self._set_guard_export_info(guard, code) + + def fn(x: Any) -> bool: + return guard_hooks_ids == hooks_ids_fn(get_hooks()) + + self.guard_manager.root.add_lambda_guard( + fn, get_verbose_code_parts(code, guard) + ) + + def TENSOR_SUBCLASS_METADATA_MATCH(self, guard: Guard) -> None: + value = self.get(guard) + original_metadata = deepcopy(self.get(guard).__tensor_flatten__()[1]) + if hasattr(value, "__metadata_guard__"): + verify_guard_fn_signature(value) + cls = type(value) + + def metadata_checker(x: Any) -> bool: + return cls.__metadata_guard__( + original_metadata, x.__tensor_flatten__()[1] + ) + + else: + + def metadata_checker(x: Any) -> bool: + return x.__tensor_flatten__()[1] == original_metadata + + global_name = f"___check_metadata_{id(metadata_checker)}_c{CompileContext.current_compile_id()}" + self.get_guard_manager(guard).add_lambda_guard( + metadata_checker, get_verbose_code_parts(global_name, guard) + ) + + def DTENSOR_SPEC_MATCH(self, guard: Guard) -> None: + # Copied from DTensor __metadata_guard__ + # TODO - Consider moving this to C++ if stable + value = deepcopy(self.get(guard)) + + def guard_fn(x: Any) -> bool: + return x._check_equals(value, skip_shapes=True) + + code = f"__dtensor_spec_{id(guard_fn)}" + self.get_guard_manager(guard).add_lambda_guard( + guard_fn, get_verbose_code_parts(code, guard) + ) + + def EQUALS_MATCH(self, guard: Guard, recompile_hint: Optional[str] = None) -> None: + ref = self.arg_ref(guard) + val = self.get(guard) + if np: + np_types: tuple[type[Any], ...] = ( + np.int8, + np.int16, + np.int32, + np.int64, + np.uint8, + np.uint16, + np.uint32, + np.uint64, + np.float16, + np.float32, + np.float64, + ) + else: + np_types = () + + ok_mutable_types = (list, set) + + ok_types = tuple( + common_constant_types + | { + type, + tuple, + frozenset, + slice, + range, + dict_keys, + torch.Size, + torch.Stream, + torch.cuda.streams.Stream, + *np_types, + *ok_mutable_types, + } + ) + + if torch.distributed.is_available(): + from torch.distributed.device_mesh import DeviceMesh + from torch.distributed.tensor.placement_types import ( + _StridedShard, + Partial, + Replicate, + Shard, + ) + + ok_types = ok_types + ( + Shard, + Replicate, + Partial, + DeviceMesh, + _StridedShard, + ) + + from torch.export.dynamic_shapes import _IntWrapper + + ok_types = ok_types + (_IntWrapper,) + + import torch.utils._pytree as pytree + + assert ( + isinstance(val, ok_types) + or pytree.is_constant_class(type(val)) + or is_opaque_value_type(type(val)) + ), f"Unexpected type {type(val)}" + + # Special case for nan because float("nan") == float("nan") evaluates to False + if istype(val, float) and math.isnan(val): + code = [f"(type({ref}) is float and __math_isnan({ref}))"] + self._set_guard_export_info(guard, code) + + self.get_guard_manager(guard).add_float_is_nan_guard( + get_verbose_code_parts(code, guard), + ) + return + + # Python math library doesn't support complex nan, so we need to use numpy + # pyrefly: ignore [missing-attribute] + if istype(val, complex) and np.isnan(val): + code = [f"(type({ref}) is complex and __numpy_isnan({ref}))"] + self._set_guard_export_info(guard, code) + + self.get_guard_manager(guard).add_complex_is_nan_guard( + get_verbose_code_parts(code, guard), + ) + return + + # Construct a debug string to put into the c++ equals match guard. + code = [f"{ref} == {val!r}"] + if istype(val, ok_mutable_types): + # C++ guards perform a pointer equality check to speedup guards, but the assumption is that the object + # is immutable. For a few corner cases like sets and lists, we make a deepcopy to purposefully fail the + # pointer equality check. + val = deepcopy(val) + + verbose_code_parts = get_verbose_code_parts(code, guard) + if recompile_hint: + verbose_code_parts = [ + f"{part} (HINT: {recompile_hint})" for part in verbose_code_parts + ] + + self.get_guard_manager(guard).add_equals_match_guard(val, verbose_code_parts) + self._set_guard_export_info(guard, code) + return + + def CONSTANT_MATCH(self, guard: Guard) -> None: + val = self.get(guard) + if istype(val, bool): + self.BOOL_MATCH(guard) + elif val is None: + self.NONE_MATCH(guard) + elif istype(val, types.CodeType): + self.ID_MATCH(guard) + else: + self.EQUALS_MATCH(guard) + + def NN_MODULE(self, guard: Guard) -> None: + # don't support this in serialization because it uses unsupported ID_MATCH + self.ID_MATCH(guard, "[inline-inbuilt-nn-modules-candidate]") + val = self.get(guard) + if hasattr(val, "training"): + assert istype(val.training, bool) + if not self.guard_nn_modules: + # If guard_nn_modules is true, we will guard on the right set of guards + self._guard_on_attribute(guard, "training", GuardBuilder.CONSTANT_MATCH) # type: ignore[arg-type] + else: + exc.unimplemented( + gb_type="Attempted to guard on uninitialized nn.Module", + context="", + explanation="Attempted to setup an NN_MODULE guard on uninitialized " + f"nn.Module subclass `{type(val)}`.", + hints=[ + "Ensure the `nn.Module` subclass instance has called `super().__init__()`.", + ], + ) + + def FUNCTION_MATCH(self, guard: Guard) -> None: + """things like torch.add and user defined functions""" + # don't support this in serialization because it uses unsupported ID_MATCH + return self.ID_MATCH(guard) + + def CLASS_MATCH(self, guard: Guard) -> None: + """Equals ID_MATCH on classes - better readability than directly calling ID_MATCH""" + val = self.get(guard) + if not inspect.isclass(val): + raise AssertionError( + f"{guard.name} is not a class, but CLASS_MATCH is used" + ) + self.id_match_unchecked(guard) + + def MODULE_MATCH(self, guard: Guard) -> None: + """Equals ID_MATCH on modules - better readability than directly calling ID_MATCH""" + val = self.get(guard) + if not inspect.ismodule(val): + raise AssertionError( + f"{guard.name} is not a module, but MODULE_MATCH is used" + ) + self.id_match_unchecked(guard) + + def CLOSURE_MATCH(self, guard: Guard) -> None: + """matches a closure by __code__ id.""" + # don't support this in serialization because it uses unsupported FUNCTION_MATCH + val = self.get(guard) + # Strictly only want user-defined functions + if type(val) is types.FunctionType and hasattr(val, "__code__"): + self._guard_on_attribute(guard, "__code__", GuardBuilder.HASATTR) # type: ignore[arg-type] + self._guard_on_attribute(guard, "__code__", GuardBuilder.CONSTANT_MATCH) # type: ignore[arg-type] + else: + self.FUNCTION_MATCH(guard) + + def BUILTIN_MATCH(self, guard: Guard) -> None: + if self.save_guards: + # Record which builtin variables are used for pruning later. + if isinstance(guard.originating_source, DictGetItemSource): + self.check_fn_manager.used_builtin_vars.add( + guard.originating_source.index + ) + return self.id_match_unchecked(guard) + + def SEQUENCE_LENGTH(self, guard: Guard) -> None: + # This guard is used to check length of PySequence objects like list, + # tuple, collections.deque etc + ref = self.arg_ref(guard) + value = self.get(guard) + + if not isinstance(value, dict): + # C++ DICT_LENGTH checks for type + self.TYPE_MATCH(guard) + + code = [] + if len(value) == 0: + code.append(f"not {ref}") + else: + code.append(f"len({ref}) == {len(value)}") + + self._set_guard_export_info(guard, code) + if isinstance(value, dict): + self.get_guard_manager(guard).add_dict_length_check_guard( + len(value), get_verbose_code_parts(code, guard) + ) + else: + self.get_guard_manager(guard).add_length_check_guard( + len(value), get_verbose_code_parts(code, guard) + ) + + def TUPLE_ITERATOR_LEN(self, guard: Guard) -> None: + ref = self.arg_ref(guard) + value = self.get(guard) + t = type(value) + + code = [] + code.append(f"___tuple_iterator_len({ref}) == {tuple_iterator_len(value)}") + self._set_guard_export_info(guard, code) + + t = type(value) + obj_id = self.id_ref(t, f"type({guard.name})") + + self.get_guard_manager(guard).add_tuple_iterator_length_guard( + tuple_iterator_len(value), obj_id, get_verbose_code_parts(code, guard) + ) + + def RANGE_ITERATOR_MATCH(self, guard: Guard) -> None: + ref = self.arg_ref(guard) + value = self.get(guard) + t = type(value) + + code = [] + normalized_range_iter = normalize_range_iter(value) + code.append(f"___normalize_range_iter({ref}) == {normalized_range_iter}") + self._set_guard_export_info(guard, code) + + t = type(value) + obj_id = self.id_ref(t, f"type({guard.name})") + + start, stop, step = normalized_range_iter + self.get_guard_manager(guard).add_range_iterator_match_guard( + start, stop, step, obj_id, get_verbose_code_parts(code, guard) + ) + + # TODO(voz): Deduplicate w/ AOTAutograd dupe input guards + def DUPLICATE_INPUT(self, guard: Guard, source_b: Source) -> None: + if is_from_skip_guard_source( + guard.originating_source + ) or is_from_skip_guard_source(source_b): + return + + if self.save_guards: + if name := get_local_source_name(source_b): + self.check_fn_manager.additional_used_local_vars.add(name) + if name := get_global_source_name(source_b): + self.check_fn_manager.additional_used_global_vars.add(name) + + ref_a = self.arg_ref(guard) + ref_b = self.arg_ref(source_b.name) + + if is_from_optimizer_source( + guard.originating_source + ) or is_from_optimizer_source(source_b): + return + + # Check that the guard has not been inserted already + key = (ref_a, ref_b) + if key in self._cached_duplicate_input_guards: + return + + self._cached_duplicate_input_guards.add((ref_a, ref_b)) + self._cached_duplicate_input_guards.add((ref_b, ref_a)) + + code = [f"{ref_b} is {ref_a}"] + self._set_guard_export_info(guard, code) + + if config.use_lamba_guard_for_object_aliasing: + # Save the code part so that we can install a lambda guard at the + # end. Read the Note - On Lambda guarding of object aliasing - to + # get more information. + code_part = code[0] + verbose_code_part = get_verbose_code_parts(code_part, guard)[0] + self.object_aliasing_guard_codes.append((code_part, verbose_code_part)) + else: + install_object_aliasing_guard( + self.get_guard_manager(guard), + self.get_guard_manager_from_source(source_b), + get_verbose_code_parts(code, guard), + ) + + def WEAKREF_ALIVE(self, guard: Guard) -> None: + code = [f"{self.arg_ref(guard)} is not None"] + + self._set_guard_export_info(guard, code) + self.get_guard_manager(guard).add_not_none_guard( + get_verbose_code_parts(code, guard) + ) + + def MAPPING_KEYS_CHECK(self, guard: Guard) -> None: + """Guard on the key order of types.MappingProxyType object""" + ref = self.arg_ref(guard) + value = self.get(guard) + + code = [] + code.append(f"list({ref}.keys()) == {list(value.keys())}") + self._set_guard_export_info(guard, code) + self.get_guard_manager(guard).add_mapping_keys_guard(value, code) + + def DICT_KEYS_MATCH(self, guard: Guard) -> None: + """Insert guard to check that the keys of a dict are same""" + ref = self.arg_ref(guard) + value = self.get(guard) + + if value is torch.utils._pytree.SUPPORTED_NODES: + # For SUPPORTED_NODES, we can guard on the dictionary version (PEP509). + self.DICT_VERSION(guard) + return + + self.SEQUENCE_LENGTH(guard) + + code = [] + # Ensure that we call dict.keys and not value.keys (which can call + # overridden keys method). In the C++ guards, we relied on PyDict_Next + # to traverse the dictionary, which uses the internal data structure and + # does not call the overridden keys method. + code.append(f"list(dict.keys({ref})) == {list(builtin_dict_keys(value))!r}") + self._set_guard_export_info(guard, code) + + if self.requires_key_order_guarding(guard.originating_source): + self.guard_on_dict_keys_and_order(value, guard) + else: + self.guard_on_dict_keys_and_ignore_order(value, guard) + + def EMPTY_NN_MODULE_HOOKS_DICT(self, guard: Guard) -> None: + """Special guard to skip guards on empty hooks. This is controlled by skip_nnmodule_hook_guards""" + if config.skip_nnmodule_hook_guards: + # This is unsafe if you add/remove a hook on nn module variable + return + self.SEQUENCE_LENGTH(guard) + + def GRAD_MODE(self, guard: Guard) -> None: + pass # we always guard on this via GlobalStateGuard() + + def DETERMINISTIC_ALGORITHMS(self, guard: Guard) -> None: + pass # we always guard on this via GlobalStateGuard() + + def FSDP_TRAINING_STATE(self, guard: Guard) -> None: + pass # we always guard on this via GlobalStateGuard() + + def GLOBAL_STATE(self, guard: Guard) -> None: + output_graph = self.check_fn_manager.output_graph + assert output_graph is not None + global_state = output_graph.global_state_guard + self.check_fn_manager.global_state = global_state + self.guard_manager.root.add_global_state_guard( + global_state, ["___check_global_state()"] + ) + + def TORCH_FUNCTION_STATE(self, guard: Guard) -> None: + assert self.check_fn_manager.torch_function_mode_stack is not None + self.check_fn_manager.torch_function_mode_stack_check_fn = ( + make_torch_function_mode_stack_guard( + self.check_fn_manager.torch_function_mode_stack + ) + ) + self.guard_manager.root.add_torch_function_mode_stack_guard( + self.check_fn_manager.torch_function_mode_stack, + ["___check_torch_function_mode_stack()"], + ) + + def DEFAULT_DEVICE(self, guard: Guard) -> None: + """Guard on CURRENT_DEVICE per torch.utils._device""" + assert guard.source is GuardSource.GLOBAL + + assert self.check_fn_manager.output_graph is not None + code = [ + f"utils_device.CURRENT_DEVICE == {self.check_fn_manager.output_graph.current_device!r}" + ] + self._set_guard_export_info(guard, code) + + self.get_guard_manager(guard).add_default_device_guard( + get_verbose_code_parts(code, guard) + ) + + def SHAPE_ENV(self, guard: Guard) -> None: + from torch._dynamo.output_graph import OutputGraphCommon + + assert guard.name == "" + output_graph = self.check_fn_manager.output_graph + assert output_graph is not None + if self.check_fn_manager.shape_code_parts is not None: + shape_code_parts = self.check_fn_manager.shape_code_parts + python_code_parts = shape_code_parts.python_code_parts + verbose_code_parts = shape_code_parts.verbose_code_parts + if shape_code_parts.cpp_code_parts is not None: + cpp_code_parts = shape_code_parts.cpp_code_parts + python_fallback = shape_code_parts.python_fallback + else: + # Let's handle ShapeEnv guards. To do this, we will resolve + # shape variables to sources from tracked_fakes. This must happen after + # tensor checks. + # NB: self.output_graph can be None in the debug_nops tests + assert isinstance(output_graph, OutputGraphCommon) + assert output_graph.shape_env is not None + fs = output_graph.shape_env.tracked_fakes or [] + input_contexts = [a.symbolic_context for a in fs] + + def get_sources(t_id: int, dim: int) -> list[Source]: + # Looks up base sources mapped to a tensor id and uses them to create + # sources for the corresponding tensor dimension. + return [ + TensorPropertySource(source, TensorProperty.SIZE, dim) + # pyrefly: ignore [missing-attribute] + for source in output_graph.tracked_fakes_id_to_source[t_id] + ] + + if output_graph.export_constraints: + names: dict[str, tuple[int, int]] = {} + source_pairs: list[tuple[Source, Source]] = [] + derived_equalities: list[ # type: ignore[type-arg] + tuple[Source, Union[Source, Symbol], Callable] + ] = [] + phantom_symbols: dict[str, Symbol] = {} + relaxed_sources: set[Source] = set() + for constraint in output_graph.export_constraints: # type: ignore[attr-defined] + if constraint.t_id in output_graph.tracked_fakes_id_to_source: + torch.export.dynamic_shapes._process_equalities( + constraint, + get_sources, + output_graph.shape_env, + names, + source_pairs, + derived_equalities, + phantom_symbols, + relaxed_sources, + ) + else: + log.warning("Untracked tensor used in export constraints") + equalities_inputs = EqualityConstraint( + source_pairs=source_pairs, + derived_equalities=derived_equalities, + phantom_symbols=list(phantom_symbols.values()), + relaxed_sources=relaxed_sources, + warn_only=False, + ) + else: + equalities_inputs = None + + def _get_code_parts(langs: tuple[str, ...]) -> list[_ShapeGuardsHelper]: + # pyrefly: ignore [missing-attribute] + return output_graph.shape_env.produce_guards_verbose( + [a.fake for a in fs], # type: ignore[misc] + [a.source for a in fs], + input_contexts=input_contexts, # type: ignore[arg-type] + equalities_inputs=equalities_inputs, + source_ref=self.source_ref, + # Export keeps static. + # pyrefly: ignore [missing-attribute] + ignore_static=(not output_graph.export), + langs=langs, + ) + + if config.enable_cpp_symbolic_shape_guards: + try: + # For exporting we need the python code parts + python_code_parts, verbose_code_parts, cpp_code_parts = ( + _get_code_parts(("python", "verbose_python", "cpp")) # type: ignore[assignment] + ) + python_fallback = False + except OverflowError: + # Cannot use int64_t + python_fallback = True + python_code_parts, verbose_code_parts = _get_code_parts( + ("python", "verbose_python") + ) + else: + python_fallback = True + python_code_parts, verbose_code_parts = _get_code_parts( + ("python", "verbose_python") + ) + + # When exporting, we may work with the shape constraints some more in + # postprocessing, so don't freeze yet + if not output_graph.export: + output_graph.shape_env.freeze() + + if self.save_guards: + # For SHAPE_ENV we want to skip serializing the entire ShapeEnv so instead + # we directly serialize the generated code here. + maybe_cpp_code_parts = locals().get("cpp_code_parts") + assert maybe_cpp_code_parts is None or isinstance( + maybe_cpp_code_parts, _CppShapeGuardsHelper + ) + maybe_shape_env_sources = ( + [] + if maybe_cpp_code_parts is None + else list(maybe_cpp_code_parts.source_to_symbol.keys()) + ) + self.check_fn_manager.shape_code_parts = ShapeCodeParts( + python_code_parts=python_code_parts, + verbose_code_parts=verbose_code_parts, + cpp_code_parts=maybe_cpp_code_parts, + python_fallback=python_fallback, + shape_env_sources=maybe_shape_env_sources, + ) + + for code in python_code_parts.exprs: + self._set_guard_export_info(guard, [code]) + + # Make ShapeEnv guards available for testing. + if compile_context := CompileContext.try_get(): + compile_context.shape_env_guards.extend(verbose_code_parts.exprs) + + int_source_to_symbol = [] + float_source_to_symbol = [] + + if not python_fallback: + assert cpp_code_parts # type: ignore[possibly-undefined] + code_parts, source_to_symbol = ( + # pyrefly: ignore [unbound-name] + cpp_code_parts.exprs, + # pyrefly: ignore [unbound-name, missing-attribute] + cpp_code_parts.source_to_symbol, + ) + + if not code_parts: + return + + for source, symbol in source_to_symbol.items(): + if isinstance(source, ConstantSource): + python_fallback = True + else: + example_value = self.get( + source, + closure_vars={**SYMPY_INTERP, **_get_closure_vars()}, + ) + if isinstance(example_value, int): + int_source_to_symbol.append((source, symbol)) + elif isinstance(example_value, float): + float_source_to_symbol.append((source, symbol)) + else: + # SymInts/SymFloats go through python guard as we only support + # int64_t/double in C++ guards for now. + python_fallback = True + + if not python_fallback: + import ctypes + + from torch._inductor.codecache import CppCodeCache + + assert cpp_code_parts # type: ignore[possibly-undefined] + code_parts, source_to_symbol = ( + # pyrefly: ignore [unbound-name] + cpp_code_parts.exprs, + # pyrefly: ignore [unbound-name, missing-attribute] + cpp_code_parts.source_to_symbol, + ) + + source_to_symbol = dict(int_source_to_symbol + float_source_to_symbol) + try: + guard_managers = [ + self.get_guard_manager_from_source(IndexedSource(source, i)) + for i, source in enumerate(source_to_symbol) + ] + + int_symbols_str = ", ".join( + f"{symbol} = int_values[{i}]" + for i, (_, symbol) in enumerate(int_source_to_symbol) + ) + float_symbols_str = ", ".join( + f"{symbol} = float_values[{i}]" + for i, (_, symbol) in enumerate(float_source_to_symbol) + ) + + if int_symbols_str: + int_symbols_str = f"int64_t {int_symbols_str};" + if float_symbols_str: + float_symbols_str = f"double {float_symbols_str};" + + func_str = textwrap.dedent( + f""" + #include + #include + #include + #include + + #if defined(_MSC_VER) + # define EXTERN_DLL_EXPORT extern "C" __declspec(dllexport) + #else + # define EXTERN_DLL_EXPORT extern "C" + #endif + + EXTERN_DLL_EXPORT int8_t guard(int64_t *int_values, double *float_values) {{ + {int_symbols_str} + {float_symbols_str} + return ({") && (".join(code_parts)}); + }} + """ + ) + guards_log.debug( + "C++ shape guard function: %s %s", + func_str, + verbose_code_parts.exprs, + ) + clib = CppCodeCache.load(func_str) + cguard = ctypes.cast(clib.guard, ctypes.c_void_p).value + assert cguard + except torch._inductor.exc.InvalidCxxCompiler: + # No valid C++ compiler to compile the shape guard + pass + else: + install_symbolic_shape_guard( + guard_managers, + len(int_source_to_symbol), + len(float_source_to_symbol), + cguard, + clib, + verbose_code_parts.exprs, + ) + return + + # Install all the symbolic guards in one python lambda guard. These are run + # at the very end of the RootGuardManager via epilogue guards. + # TODO(anijain2305,williamwen42) - Consider moving this to C++. + if python_code_parts.exprs: + self.add_python_lambda_leaf_guard_to_root( + python_code_parts.exprs, + verbose_code_parts.exprs, + closure_vars={**SYMPY_INTERP, **_get_closure_vars()}, + ) + + def TENSOR_MATCH(self, guard: Guard, value: Optional[Any] = None) -> None: + if config._unsafe_skip_fsdp_module_guards and guard.is_fsdp_module(): + return + # For tensors that are part of the Dynamo extracted Fx graph module, an + # ID_MATCH suffices. Once we turn on inline_inbuilt_nn_modules, these + # will be lifted as inputs and have a TENSOR_MATCH guard. + if match_on_id_for_tensor(guard): + self.ID_MATCH(guard) + else: + if isinstance(value, TensorWeakRef): + value = value() + + value = value if value is not None else self.get(guard) + + pytype = type(value) + dispatch_keys = torch._C._dispatch_keys(value) + if isinstance(value, torch._subclasses.FakeTensor): + if value.pytype is not None: + pytype = value.pytype + if value.dispatch_keys is not None: + dispatch_keys = value.dispatch_keys + + assert isinstance(value, torch.Tensor) + + if config.log_compilation_metrics and isinstance(value, torch.nn.Parameter): + metrics_context = get_metrics_context() + if metrics_context.in_progress(): + metrics_context.increment("param_numel", value.numel()) + metrics_context.increment("param_bytes", value.nbytes) + metrics_context.increment("param_count", 1) + + tensor_name = self.arg_ref(guard) + # [Note - On Export Tensor Guards] + # + # In eager mode, tensor guards are evaluated through C++, in guards.cpp + # see [Note - On Eager Tensor Guards] for more info. + # + # In export mode, we instead maintain parallel logic between C++ and python + # here, with an exception of checking the dispatch key - with the idea that a dispatch key + # is an entirely runtime notion that would make no sense to keep in an exported graph. + # + # Now, this idea is okay, but to paraphrase @ezyang, this mental model is sufficient for now, although + # not entirely true. + # For example, suppose one of the input tensors had the negative dispatch key. + # You should end up with a graph that is specialized for tensors that have a negative dispatch key. + # If you allow a Tensor that does NOT have this bit set, you will accidentally run it "as if" it were negated. + # Now, negative key only shows up for complex numbers, and most likely, the exported to target doesn't + # support this feature at all, but the point stands that :some: tensor state only shows up on dispatch key. + # TODO(voz): Either populate a dispatch_key check into the guards, or error on users passing in an unsupported + # subset of keys during export. + # + # The list of tensor fields and calls we care about can be found in `terms` below. + # TODO(voz): We are missing storage offset in all our tensor guards? + code: list[str] = [] + assert self.check_fn_manager.output_graph is not None + if self.check_fn_manager.output_graph.export: + self.TYPE_MATCH(guard) + terms = [ + "dtype", + "device", + "requires_grad", + "ndimension", + ] + + for term in terms: + term_src = AttrSource(guard.originating_source, term) + if term == "ndimension": + term = "ndimension()" + term_src = CallFunctionNoArgsSource(term_src) + real_value = self.get(term_src) + if istype(real_value, (torch.device, torch.dtype)): + # copy pasted from EQUALS_MATCH + code.append(f"str({tensor_name}.{term}) == {str(real_value)!r}") + else: + code.append(f"{tensor_name}.{term} == {real_value}") + else: + guard_manager = self.get_guard_manager(guard) + + # skip_no_tensor_aliasing_guards_on_parameters bring + # unsoundness. If you compile a function with two different + # parameters, but later on you pass on same tensor as two + # different outputs (aliasing), Dynamo will not detect this. + # But we deliberately take this soundness hit because this + # usecase is quite rare and there is substantial reduction in + # guard overhead. + # For numpy tensors, since those are ephemeral, we don't have to + # insert aliasing guards on them + if not ( + config.skip_no_tensor_aliasing_guards_on_parameters + and ( + istype(value, torch.nn.Parameter) + or is_from_unspecialized_builtin_nn_module_source( + guard.originating_source + ) + ) + ) and not isinstance(guard.originating_source, NumpyTensorSource): + # Keep track of all the tensor guard managers to insert + # NoAliasing check at the end. + self.no_tensor_aliasing_names.append(tensor_name) + self.no_tensor_aliasing_guard_managers.append(guard_manager) + + output_graph = self.check_fn_manager.output_graph + metadata = output_graph.input_source_to_sizes_strides[ + guard.originating_source + ] + size = convert_to_concrete_values(metadata["size"]) + stride = convert_to_concrete_values(metadata["stride"]) + + verbose_code_parts = get_verbose_code_parts( + get_tensor_guard_code_part( + value, + tensor_name, + size, + stride, + pytype, + dispatch_keys, + ), + guard, + ) + guard_manager.add_tensor_match_guard( + value, + size, # type: ignore[arg-type] + stride, # type: ignore[arg-type] + tensor_name, + verbose_code_parts, + pytype, + dispatch_keys, + ) + + # We consider TENSOR_MATCH guard to be important enough to be + # included in diff guard manager by default. + if not isinstance(value, torch.nn.Parameter): + self.guard_manager.diff_guard_sources.add(guard.name) + + # A frame is valid for reuse with dynamic dimensions if the new + # (user-requested) dynamic dimensions are a subset of the old + # (already compiled) dynamic dimensions. + # + # It's a little non-obvious why you'd want this: in particular, + # if an already compiled frame matches all of the guards, why + # not just use it, why force a recompile? + # + # We force it for two reasons: + # + # - The user *required* us to compile with a new dynamic dimension, + # we should not ignore that and serve up the old, specialized + # frame. Listen to the user! + # + # - In fact, we are obligated to *raise an error* if we fail to + # make the requested dimension dynamic. If we don't + # recompile, we can't tell if that dimension can actually be + # made dynamic. + # + # If the new dynamic dims are a subset of the old, we already know + # we can make them dynamic (since we made them dynamic in old). + # This is slightly unsound, because maybe your input size is + # [s0, s0, s1] and so you can do it dynamic if you say dynamic + # dims {0, 1, 2} but you can't if you only do {0, 2} (because now + # the second s0 is specialized). But we're not entirely sure if + # this is a good idea anyway lol... (if you want to try removing + # this logic, be my guest! -- ezyang 2024) + # + assert guard.source is not None + static, _reason = tensor_always_has_static_shape( + value, is_tensor=True, tensor_source=guard.originating_source + ) + + if not static: + if hasattr(value, "_dynamo_dynamic_indices"): + dynamic_indices = value._dynamo_dynamic_indices + code_part = f"(({tensor_name}._dynamo_dynamic_indices.issubset({dynamic_indices})) if hasattr({tensor_name}, '_dynamo_dynamic_indices') else True)" # noqa: B950 + code.append(code_part) + self.get_guard_manager(guard).add_dynamic_indices_guard( + dynamic_indices, get_verbose_code_parts(code_part, guard) + ) + # In the case of us not having any dynamic dimension indices, we compiled the frame with no chance of + # raising for this specific tensor - and any inputs with more dynamic user directives specified must be recompiled. + else: + code_part = ( + f"hasattr({tensor_name}, '_dynamo_dynamic_indices') == False" + ) + code.append(code_part) + self.get_guard_manager(guard).add_no_hasattr_guard( + "_dynamo_dynamic_indices", + get_verbose_code_parts(code_part, guard), + ) + if len(code) > 0: + self._set_guard_export_info(guard, code) + + # A util that in the case of export, adds data onto guards + def _set_guard_export_info( + self, + guard: Guard, + code_list: list[str], + provided_guarded_object: Optional[Any] = None, + provided_func_name: Optional[str] = None, + ) -> None: + # WARNING: It is important that cur_frame/caller do NOT stay in + # the current frame, because they will keep things live longer + # than they should. See TestMisc.test_release_module_memory + cur_frame = currentframe() + assert cur_frame is not None + caller = cur_frame.f_back + del cur_frame + assert caller is not None + func_name = provided_func_name or caller.f_code.co_name + del caller + # We use func_name for export, so might as well get a nice defensive check out of it + assert func_name in self.__class__.__dict__, ( + f"_produce_guard_code must be called from inside GuardedCode. Called from {func_name}" + ) + + # Not all guards have names, some can be installed globally (see asserts on HAS_GRAD) + if provided_guarded_object is None: + name = guard.name + guarded_object = None if not name else self.get(guard) + else: + guarded_object = provided_guarded_object + + guarded_object_type = ( + weakref.ref(type(guarded_object)) if guarded_object is not None else None + ) + obj_ref = None + # Not necessary to have weakref for Enum type, but there is a bug that + # makes hasattr(guarded_object.__class__, "__weakref__") return True. + supports_weakref = ( + getattr(guarded_object.__class__, "__weakrefoffset__", 0) != 0 + ) + # See D64140537 for why we are checking for tuple. + if supports_weakref and not isinstance( + guarded_object, (enum.Enum, tuple, weakref.ProxyTypes) + ): + obj_ref = weakref.ref(guarded_object) + + guard.set_export_info( + func_name, + guarded_object_type, + code_list, + obj_ref, + ) + + +# Common Sub-Expression Elimination for Python expressions. +# +# There are 2 steps to this pass: +# 1. Count the frequency of each sub-expression (i.e. inner +# node in the AST tree) +# +# 2. Replace those that occur more than once by a fresh variable 'v'. +# 'v' will be defined in the 'preface' list (output argument to +# 'NodeTransformer') +# +# NB: the use of 'ast.unparse' while visiting the nodes makes this pass +# quadratic on the depth of the tree. +# +# NB: this pass creates a new variable for each AST node that is repeated +# more than 'USE_THRESHOLD'. e.g. if 'a.b.c.d' is used 10 times, 'a.b.c' +# and 'a.b' are also used 10 times. So, there will be a new variable for +# each of them. +class PyExprCSEPass: + # Maximum number of times a given expression can be used without being + # replaced by a fresh variable. + USE_THRESHOLD = 1 + + # Ad-Hoc: AST nodes this pass focuses on. + ALLOWED_NODE_TYPES = (ast.Attribute, ast.Call, ast.Subscript) + + @dataclasses.dataclass + class Config: + expr_count: dict[str, int] + expr_to_name: dict[str, str] + + class ExprCounter(ast.NodeVisitor): + def __init__(self, config: PyExprCSEPass.Config) -> None: + self._config = config + + def visit(self, node: ast.AST) -> None: + if isinstance(node, PyExprCSEPass.ALLOWED_NODE_TYPES): + self._config.expr_count[_ast_unparse(node)] += 1 + super().visit(node) + + class Replacer(ast.NodeTransformer): + def __init__( + self, + config: PyExprCSEPass.Config, + gen_name: Callable[[], str], + ) -> None: + super().__init__() + self._config = config + self._gen_name = gen_name + self.preface: list[str] = [] + + def visit(self, node: ast.AST) -> Any: + if isinstance(node, PyExprCSEPass.ALLOWED_NODE_TYPES): + expr = _ast_unparse(node) + + # Replacement only occurs if a given expression is used more + # than once. + if self._config.expr_count[expr] > PyExprCSEPass.USE_THRESHOLD: + if expr not in self._config.expr_to_name: + # Parent 'visit' is called so that we CSE the inner expressions first. + # + # The resulting expression is used as right-hand-side of the variable + # assignment. i.e. we are CSE-ing the children before the parents. + # + # Indexing still uses the old 'node', since that's what was counted + # by the 'NodeVisitor'. + node_ = super().visit(node) + expr_ = _ast_unparse(node_) + var_name = self._gen_name() + self.preface.append(f"{var_name} = {expr_}") + self._config.expr_to_name[expr] = var_name + else: + var_name = self._config.expr_to_name[expr] + return ast.Name(var_name, ast.Load()) + + return super().visit(node) + + def __init__(self) -> None: + self._counter = 0 + self._config = self.Config( + expr_count=collections.defaultdict(lambda: 0), expr_to_name={} + ) + + def _new_var(self, prefix: str = "_var") -> str: + name = f"{prefix}{self._counter}" + self._counter += 1 + return name + + def count(self, exprs: list[str]) -> None: + counter = self.ExprCounter(self._config) + for e in exprs: + try: + counter.visit(ast.parse(e)) + except SyntaxError as ex: + log.exception("Failed to visit expr at line %s.\n%s", ex.lineno, e) + raise + + def replace(self, expr: str) -> tuple[list[str], str]: + replacer = self.Replacer(self._config, self._new_var) + new_node = replacer.visit(ast.parse(expr)) + return replacer.preface, _ast_unparse(new_node) + + +def must_add_nn_module_guards(guard: Guard) -> bool: + # For config.guard_nn_modules=False, we can skip all the guards that + # originate from inside of nn module except for a few categories. + return ( + # Guard for defaults + isinstance(guard.originating_source, DefaultsSource) + # Guard using dict tags if the config flag is set + or ( + config.guard_nn_modules_using_dict_tags + and guard.create_fn is GuardBuilder.NN_MODULE + ) + ) + + +class DeletedGuardManagerWrapper(GuardManagerWrapper): + def __init__(self, reason: str) -> None: + super().__init__() + self.invalidation_reason = reason + + def populate_diff_guard_manager(self) -> None: + self.diff_guard_root = None + + +@dataclasses.dataclass +class ShapeCodeParts: + python_code_parts: _ShapeGuardsHelper + verbose_code_parts: _ShapeGuardsHelper + cpp_code_parts: Optional[_CppShapeGuardsHelper] + python_fallback: bool + shape_env_sources: list[Source] + + +@dataclasses.dataclass +class GuardsState: + output_graph: OutputGraphGuardsState + shape_code_parts: Optional[ShapeCodeParts] + source_get_cache: Optional[dict[str, Any]] = None + + +class _Missing: + def __init__(self, reason: Optional[str] = None) -> None: + self._reason = reason + + def __repr__(self) -> str: + return f"_Missing({self._reason})" + + def __str__(self) -> str: + return f"_Missing({self._reason})" + + # Sometimes _Missing object is used as the callable with functools.partial, + # so we add a dummy __call__ here to bypass TypeError from partial(). + def __call__(self, *args: Any, **kwargs: Any) -> Any: + return _Missing() + + +@functools.cache +def _get_unsupported_types() -> tuple[type, ...]: + # We only do ID_MATCH on C objects which is already banned from guards serialization. + ret: tuple[type, ...] = ( + types.CodeType, + torch._C.Stream, + weakref.ReferenceType, + ) + try: + ret += (torch._C._distributed_c10d.ProcessGroup,) + except AttributeError: + pass + return ret + + +class GuardsStatePickler(pickle.Pickler): + def __init__( + self, + guard_tree_values: dict[int, Any], + empty_values: dict[int, Any], + missing_values: dict[int, Any], + *args: Any, + **kwargs: Any, + ) -> None: + super().__init__(*args, **kwargs) + self.fake_mode = torch._subclasses.FakeTensorMode() + self.tensor_converter = torch._subclasses.fake_tensor.FakeTensorConverter() + self.guard_tree_values = guard_tree_values + self.empty_values = empty_values + self.missing_values = missing_values + + @classmethod + def _unpickle_module(cls, state: Any) -> torch.nn.Module: + mod = torch.nn.Module() + mod.__setstate__(state) + return mod + + @classmethod + def _unpickle_tensor( + cls, + meta_tensor: torch.Tensor, + device: torch.device, + pytype: type, + dispatch_keys_raw: int, + grad: torch.Tensor, + ) -> torch.Tensor: + fake_mode = torch._subclasses.FakeTensorMode() + tensor_converter = torch._subclasses.fake_tensor.FakeTensorConverter() + ret = tensor_converter.from_meta_and_device( + fake_mode, + meta_tensor, + device, + pytype, + torch._C.DispatchKeySet.from_raw_repr(dispatch_keys_raw), + ) + ret.grad = grad + return ret + + @classmethod + def _unpickle_traceable_wrapper_subclass( + cls, + meta_tensor: torch.Tensor, + device: torch.device, + pytype: type, + dispatch_keys_raw: int, + ctx: Any, + inner_data: list[tuple[str, Callable[..., Any], tuple[Any, ...]]], + ) -> torch.Tensor: + # Unpickle the inner tensor components. These could also be subclass instances. + inner_tensors = {} + for attr, unpickle_func, unpickle_func_args in inner_data: + inner_tensors[attr] = unpickle_func(*unpickle_func_args) + + outer_size, outer_stride = meta_tensor.shape, meta_tensor.stride() + out = type(meta_tensor).__tensor_unflatten__( # type: ignore[attr-defined] + inner_tensors, ctx, outer_size, outer_stride + ) + out.pytype = pytype + out.dispatch_keys = torch._C.DispatchKeySet.from_raw_repr(dispatch_keys_raw) + return out + + @classmethod + def _unpickle_python_module(cls, alias: str) -> types.ModuleType: + return importlib.import_module(alias) + + @classmethod + def _unpickle_dispatch_key_set(cls, raw_repr: int) -> torch._C.DispatchKeySet: + return torch._C.DispatchKeySet.from_raw_repr(raw_repr) + + @classmethod + def _unpickle_functorch_interpreter( + cls, json: bytes + ) -> torch._C._functorch.CInterpreter: + return torch._C._functorch.CInterpreter.deserialize(json) + + @classmethod + def _unpickle_mapping_proxy( + cls, d: dict[Any, Any] + ) -> types.MappingProxyType[Any, Any]: + return types.MappingProxyType(d) + + @classmethod + def _unpickle_dict_keys(cls, elems: list[Any]) -> Any: + return dict.fromkeys(elems).keys() + + @classmethod + def _unpickle_fsdp_module_type( + cls, original_type: type[torch.nn.Module] + ) -> type[torch.nn.Module]: + return torch.distributed.fsdp._fully_shard._fully_shard.get_cls_to_fsdp_cls()[ + original_type + ] + + @classmethod + def _unpickle_ddp_module( + cls, state: dict[str, Any] + ) -> torch.nn.parallel.DistributedDataParallel: + ty = torch.nn.parallel.DistributedDataParallel + ddp = ty.__new__(ty) + torch.nn.Module.__setstate__(ddp, state) + return ddp + + @classmethod + def _unpickle_c_op(cls, name: str) -> Any: + return getattr(torch.ops._C, name) + + @classmethod + def _unpickle_bound_method(cls, func: Any, base: Any) -> Any: + return types.MethodType(func, base) + + @staticmethod + def _unpickle_sdp_backend(name: str) -> torch.nn.attention.SDPBackend: + # Reconstruct from the Python-facing enum namespace + return getattr(torch.nn.attention.SDPBackend, name) + + @classmethod + def _unpickle_cell(cls, val: Any) -> Any: + def _() -> Any: + return val + + assert _.__closure__ is not None + return _.__closure__[0] + + # pyrefly: ignore [bad-override] + def reducer_override( + self, obj: Any + ) -> Union[tuple[Callable[..., Any], tuple[Any, ...]], Any]: + import sympy + + if id(obj) in self.empty_values: + return type(obj).__new__, (type(obj),) + + if id(obj) in self.missing_values: + return _Missing, ("missing values",) + + if isinstance(obj, torch.Tensor) and obj.device.type != "meta": + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + if id(obj) not in self.guard_tree_values: + return _Missing, ("tensor guard tree",) + + if is_traceable_wrapper_subclass(obj): + # inner_data is a list of tuples of: + # (inner attr name, unpickle func, tuple of func inputs) + # This supports traceable wrapper subclass inner tensors. + inner_data = [] + attrs, ctx = obj.__tensor_flatten__() + # recursively call for inner tensor components + for attr in attrs: + inner = getattr(obj, attr) + if isinstance(inner, torch.Tensor): + self.guard_tree_values[id(inner)] = inner + func, args_tuple = self.reducer_override(inner) + inner_data.append((attr, func, args_tuple)) + + return type(self)._unpickle_traceable_wrapper_subclass, ( + torch.empty_like(obj, device="meta"), + obj.device, + type(obj), + torch._C._dispatch_keys(obj).raw_repr(), + ctx, + inner_data, + ) + + return type(self)._unpickle_tensor, ( + torch.empty_like(obj, device="meta", requires_grad=obj.requires_grad), + obj.device, + type(obj), + torch._C._dispatch_keys(obj).raw_repr(), + obj.grad, + ) + + elif isinstance(obj, torch.nn.Module): + if id(obj) not in self.guard_tree_values: + return _Missing, ("module guard tree",) + + # DDP module is a special case because it tries to restore unneeded + # data in custom __setstate__. We cannot skip ddp module because it + # is often a toplevel module. + if isinstance(obj, torch.nn.parallel.DistributedDataParallel): + return type(self)._unpickle_ddp_module, (obj.__getstate__(),) + + if type(obj).__qualname__ == type(obj).__name__: + return NotImplemented + if obj.__class__.__getstate__ == torch.nn.Module.__getstate__: + return type(self)._unpickle_module, (obj.__getstate__(),) + + elif inspect.ismodule(obj): + return type(self)._unpickle_python_module, (obj.__name__,) + + elif isinstance(obj, torch._C.DispatchKeySet): + return type(self)._unpickle_dispatch_key_set, (obj.raw_repr(),) + + elif isinstance(obj, torch._C._functorch.CInterpreter): + return type(self)._unpickle_functorch_interpreter, (obj.serialize(),) + + elif ( + inspect.isclass(obj) + and issubclass(obj, sympy.Function) + and hasattr(obj, "_torch_handler_name") + ): + assert hasattr(obj, "_torch_unpickler") + return obj._torch_unpickler, (obj._torch_handler_name,) + + elif isinstance(obj, torch.SymInt): + raise RuntimeError(f"Cannot serialize SymInt {obj} (node: {obj.node})") + + elif isinstance(obj, types.MappingProxyType): + return type(self)._unpickle_mapping_proxy, (obj.copy(),) + + elif isinstance(obj, torch._dynamo.utils.dict_keys): + return type(self)._unpickle_dict_keys, (list(obj),) + + elif isinstance( + obj, torch._ops.OpOverloadPacket + ) and obj._qualified_op_name.startswith("_C::"): + return type(self)._unpickle_c_op, (obj.__name__,) + + elif ( + obj.__class__.__module__ == "builtins" + and obj.__class__.__name__ == "PyCapsule" + ): + # Skipping PyCapsule since there isn't much to be guarded about them. + return _Missing, ("capsule",) + + elif isinstance(obj, _get_unsupported_types()): + return _Missing, ("unsupported",) + + elif inspect.isfunction(obj): + if obj.__code__.co_flags & inspect.CO_NESTED: + return _Missing, ("nested function",) + if obj.__module__ in sys.modules: + f = sys.modules[obj.__module__] + for name in obj.__qualname__.split("."): + f = getattr(f, name, None) # type: ignore[assignment] + if f is not obj: + return _Missing, ("fqn mismatch",) + elif inspect.ismethod(obj): + func = obj.__func__ + method_self = obj.__self__ + inner_func = getattr(method_self, func.__name__) + if inspect.ismethod(inner_func): + inner_func = inner_func.__func__ + if func is not inner_func: + return type(self)._unpickle_bound_method, (func, method_self) + + elif isinstance(obj, type((lambda x: lambda: x)(0).__closure__[0])): # type: ignore[index] # noqa: PLC3002 + return type(self)._unpickle_cell, (obj.cell_contents,) + + if hasattr(torch.distributed, "distributed_c10d") and isinstance( + obj, torch.distributed.distributed_c10d.Work + ): + if id(obj) not in self.guard_tree_values: + return _Missing, ("distributed_c10d.Work",) + + if isinstance(obj, torch.nn.attention.SDPBackend): + return type(self)._unpickle_sdp_backend, (obj.name,) + + if type(obj).__qualname__ != type(obj).__name__: + raise torch._dynamo.exc.PackageError( + f"Type {type(obj)} for object {obj} cannot be saved " + + "into torch.compile() package since it's defined in local scope. " + + "Please define the class at global scope (top level of a module)." + ) + + if ( + inspect.isclass(obj) + and hasattr(torch.distributed, "fsdp") + and issubclass(obj, torch.distributed.fsdp._fully_shard.FSDPModule) + ): + if obj is not torch.distributed.fsdp._fully_shard.FSDPModule: + original_type = obj.__mro__[2] + assert issubclass(original_type, torch.nn.Module) + assert ( + original_type + in torch.distributed.fsdp._fully_shard._fully_shard.get_cls_to_fsdp_cls() + ) + return type(self)._unpickle_fsdp_module_type, (original_type,) + + return NotImplemented + + +def pickle_guards_state(state: GuardsState, guard_tree_values: dict[int, Any]) -> bytes: + buf = io.BytesIO() + empty_values = {} + missing_values = {} + + leaves = pytree.tree_leaves(state.output_graph.local_scope) + for leaf in leaves: + if inspect.ismethod(leaf) and hasattr(leaf, "__self__"): + base = leaf.__self__ + if id(base) not in guard_tree_values: + try: + type(base).__new__(type(base)) + empty_values[id(base)] = base + except: # noqa: E722, B001 + pass + elif id(leaf) not in guard_tree_values: + # TODO See if we have lift this branch as the first one. + # Prune more objects in pytree hierarchy. + missing_values[id(leaf)] = leaf + pickler = GuardsStatePickler(guard_tree_values, empty_values, missing_values, buf) + try: + pickler.dump(state) + except AttributeError as e: + raise torch._dynamo.exc.PackageError(str(e)) from e + return buf.getvalue() + + +# NB: Naively, you'd expect this to only be a function that produces +# the callable that constitutes the guard. However, there is some +# delicate handling for invalidating this check function when the +# locals/globals get invalidated, so there's some extra state +# we have to hold in this manager class. +class CheckFunctionManager: + def __init__( + self, + f_code: types.CodeType, + output_graph: OutputGraphCommon, + cache_entry: Optional[CacheEntry] = None, + guard_fail_fn: Optional[Callable[[GuardFail], None]] = None, + guard_filter_fn: Optional[ + Callable[[list[GuardFilterEntry]], list[bool]] + ] = None, + shape_code_parts: Optional[ShapeCodeParts] = None, + runtime_global_scope: Optional[dict[str, Any]] = None, + save_guards: bool = False, + strict_error: bool = False, + source_get_cache: Optional[dict[str, Any]] = None, + ): + guards = output_graph.guards if output_graph else None + self._weakrefs: dict[int, ReferenceType[object]] = {} + + existing_diff_guard_sources = ( + update_diff_guard_managers_for_existing_cache_entries(cache_entry) + ) + self.output_graph: Optional[OutputGraphCommon] = output_graph + assert self.output_graph is not None + + # Only used for serialization. + self.shape_code_parts = shape_code_parts + + # NB: Until we trace device contexts, we need to use the stack recorded at the beginning of tracing + # in case a set default device call was made in the graph. + self.torch_function_mode_stack = ( + output_graph.torch_function_mode_stack if output_graph else None + ) + self.used_builtin_vars: OrderedSet[str] = OrderedSet() + self.additional_used_local_vars: OrderedSet[str] = OrderedSet() + self.additional_used_global_vars: OrderedSet[str] = OrderedSet() + self.runtime_global_scope = runtime_global_scope + self.global_state: Optional[torch._C._dynamo.guards.GlobalStateGuard] = None + self.torch_function_mode_stack_check_fn: Optional[Callable[[], bool]] = None + + if not justknobs_check("pytorch/compiler:guard_nn_modules"): + log.warning("guard_nn_modules is turned off using justknobs killswitch") + + # TODO Be more explicit about the behavior for the users. + if torch._dynamo.config.caching_precompile: + _guard_filter_fn = guard_filter_fn or (lambda gs: [True for g in gs]) + + def guard_filter_fn(guards: list[GuardFilterEntry]) -> list[bool]: + ret = [] + for keep, g in zip(_guard_filter_fn(guards), guards): + if not keep: + ret.append(False) + elif ( + g.guard_type + in ( + "ID_MATCH", + "CLOSURE_MATCH", + "WEAKREF_ALIVE", + "DICT_VERSION", + ) + or "ID_MATCH" in g.derived_guard_types + or "DICT_VERSION" in g.derived_guard_types + ): + log.warning( + "%s guard on %s is dropped with caching_precompile=True.", + g.guard_type, + g.orig_guard.name, + ) + ret.append(False) + else: + ret.append(True) + return ret + + sorted_guards = sorted(guards or (), key=Guard.sort_key) + + if guard_filter_fn: + # If we're filtering guards, we need to build it an extra time first + # because filtering depends on the builder/guard_manager results + builder, guard_manager = self.build_guards( + sorted_guards, + existing_diff_guard_sources, + f_code, + output_graph, + False, + source_get_cache=source_get_cache, + ) + + def make_guard_filter_entry(guard: Guard) -> GuardFilterEntry: + MISSING = object() + name = strip_local_scope(guard.name) + if name == "": + has_value = False + value = MISSING + else: + try: + # Guard evaluation is expected to fail when we guard on + # things like "not hasattr(x, 'foo')". In cases like this, + # we don't have a well defined value because such thing + # doesn't exist. + value = builder.get(guard) + has_value = True + except: # noqa: B001,E722 + value = MISSING + has_value = False + is_global = get_global_source_name(guard.originating_source) is not None + return GuardFilterEntry( + name=name, + has_value=has_value, + value=value, + guard_type=guard.create_fn_name(), + derived_guard_types=( + tuple(guard.guard_types) if guard.guard_types else () + ), + is_global=is_global, + orig_guard=guard, + ) + + filter_results = guard_filter_fn( + [make_guard_filter_entry(guard) for guard in sorted_guards] + ) + assert len(filter_results) == len(sorted_guards) + assert all(type(x) is bool for x in filter_results) + sorted_guards = [ + guard for i, guard in enumerate(sorted_guards) if filter_results[i] + ] + + # Redo the guards because filtering relies on the results from the last guard builder. + builder, guard_manager = self.build_guards( + sorted_guards, + existing_diff_guard_sources, + f_code, + output_graph, + save_guards, + source_get_cache=source_get_cache, + ) + + self.guard_manager = guard_manager + self.compile_check_fn(builder, sorted_guards, guard_fail_fn) + + # Keep track of weak references of objects with ID_MATCH guard. This + # info is stored alongside optimized_code and guard_manager and is used to + # limit the number of cache entries with same ID_MATCH'd object. + # TODO(anijain2305) - Currently this information is stored as an attr on + # the guard_manager itself to avoid changing CacheEntry data structure in + # eval_frame.c. In future, we should probably replace guard_manager with a + # queryable data structure such that this information is already present + # in some form. + self.guard_manager.id_matched_objs = builder.id_matched_objs + + guards_log.debug("%s", self.guard_manager) + self.guard_manager.id_matched_objs = builder.id_matched_objs + + # Check that the guard returns True. False means that we will always + # recompile. + # TODO(anijain2305, ydwu4) - Skipping export because of following test + # python -s test/dynamo/test_export.py -k test_export_with_symbool_inputs + latency = 0.0 + + if not output_graph.skip_guards_check and not output_graph.export: + if not self.guard_manager.check(output_graph.local_scope): + reasons = get_guard_fail_reason_helper( + self.guard_manager, + output_graph.local_scope, + CompileContext.current_compile_id(), + backend=None, # no need to set this because we are trying to find the offending guard entry + ) + raise AssertionError( + "Guard failed on the same frame it was created. This is a bug - please create an issue." + f"Guard fail reason: {reasons}" + ) + + if guard_manager_testing_hook_fn is not None: + guard_manager_testing_hook_fn( + self.guard_manager, output_graph.local_scope, builder + ) + + # NB for developers: n_iters is chosen to be 1 to prevent excessive + # increase in compile time. We first do a cache flush to measure the + # guard latency more accurately. This cache flush is expensive. + # Note - If you are working on a guard optimization, it might be a + # good idea to increase this number for more stability during + # development. + latency = profile_guard_manager( + self.guard_manager.root, output_graph.local_scope, 1 + ) + guards_log.debug("Guard eval latency = %s us", f"{latency:.2f}") + # Note: We use `increment_toplevel` instead of `compilation_metric` + # here. This is because, in scenarios where `torch._dynamo.reset` + # is invoked, the same frame ID and compile ID may be reused during + # a new compilation cycle. This behavior causes issues with + # `compilation_metric`, as it expects the metric field to be empty. + # Ideally, we would overwrite the existing entry in such cases, but + # we currently lack an API to support overwriting metrics. However, + # since these situations are rare and typically impractical to + # account for, we simply increment at the toplevel instead. + CompileEventLogger.increment_toplevel("guard_latency_us", int(latency)) + + self.guards_state: Optional[bytes] = None + if save_guards: + from torch._dynamo.output_graph import OutputGraphCommon + + assert isinstance(self.output_graph, OutputGraphCommon) + try: + self.guards_state = self.serialize_guards( + builder, sorted_guards, self.output_graph + ) + except exc.PackageError as e: + if torch._dynamo.config.strict_precompile or strict_error: + raise e + self.output_graph.bypass_package( + f"Guard evaluation failed: {str(e)}", + traceback=traceback.format_exc().split("\n"), + ) + + # TODO: don't do the string rep, do something more structured here + torch._logging.trace_structured( + "dynamo_cpp_guards_str", + payload_fn=lambda: f"{self.guard_manager}\nGuard latency = {latency:.2f} us", + ) + # NB - We have to very careful of cleaning up here. Because of the + # invalidate function, we can create a weakref finalizer that keeps + # `self` alive for very long. Sometimes by mistake, we can run + # invalidate for a type/object (check id_ref method) that Python can + # leak by design, preventing us from calling the finalizer. In that + # case, the `self` will be alive even though the cache entry will be + # deleted (check invalidate method), which can cause a memory leak, + # e.g., not setting output_graph = None can keep hold of nn_modules. + self._weakrefs.clear() + self.output_graph = None + + UNSUPPORTED_SERIALIZATION_GUARD_TYPES: tuple[LiteralString, ...] = ( + "DICT_VERSION", + "NN_MODULE", + "ID_MATCH", + "FUNCTION_MATCH", + "CLASS_MATCH", + "MODULE_MATCH", + "CLOSURE_MATCH", + "WEAKREF_ALIVE", + ) + + def serialize_guards( + self, + builder: GuardBuilder, + sorted_guards: list[Guard], + output_graph: OutputGraphCommon, + ) -> bytes: + # We check whether our list of guards are serializable here + for guard in sorted_guards: + guard_type = guard.create_fn_name() + derived_guard_types = tuple(guard.guard_types) if guard.guard_types else () + # BUILTIN_MATCH calls TYPE_MATCH sometimes, so we need to check both for + # a chance that the guard is unserializable + if guard_type in ("TYPE_MATCH", "BUILTIN_MATCH"): + if guard._unserializable: + # Only call builder.get again if we know we're going to throw + obj = builder.get(guard) + raise_local_type_error(obj) + elif ( + guard_type in CheckFunctionManager.UNSUPPORTED_SERIALIZATION_GUARD_TYPES + ): + raise torch._dynamo.exc.PackageError( + f"{guard_type} guard cannot be serialized." + ) + elif failed := next( + ( + i + for i in derived_guard_types + if i in CheckFunctionManager.UNSUPPORTED_SERIALIZATION_GUARD_TYPES + ), + None, + ): + # Just raise the first failed guard name + raise torch._dynamo.exc.PackageError( + f"{failed} guard cannot be serialized." + ) + + builtins_dict_name = output_graph.name_of_builtins_dict_key_in_fglobals or "" + used_global_vars = set() + used_local_vars = set() + + def prune_variable(source: Source) -> None: + if name := get_global_source_name(source): + assert isinstance(name, str) + # Leave out the builtins dict key, as we will special handle + # it later because the guarded code rarely use the entire + # builtin dict in the common case. + if name != builtins_dict_name: + used_global_vars.add(name) + elif name := get_local_source_name(source): + assert isinstance(name, str) + used_local_vars.add(name) + + output_graph_guards_state = output_graph.dump_guards_state() + # Only serialize the global variables that are actually used in guards. + for guard in sorted_guards: + if isinstance(guard.originating_source, ShapeEnvSource): + assert self.shape_code_parts + for source in self.shape_code_parts.shape_env_sources: + prune_variable(source) + else: + prune_variable(guard.originating_source) + + for source in output_graph.guard_on_key_order: + prune_variable(source) + + def normalize_create_fn(x: Callable[..., None]) -> Callable[..., None]: + if isinstance(x, functools.partial): + + def _ref(x: Any) -> Any: + if isinstance(x, (TensorWeakRef, weakref.ref)): + return x() + return x + + new_args = tuple(_ref(a) for a in x.args) + new_keywords = {k: _ref(v) for k, v in x.keywords.items()} + return functools.partial(x.func, *new_args, **new_keywords) + + return x + + global_scope_state = { + k: v + for k, v in output_graph_guards_state.global_scope.items() + if k in used_global_vars or k in self.additional_used_global_vars + } + global_scope_state[builtins_dict_name] = { + k: v + for k, v in output_graph_guards_state.global_scope[ + builtins_dict_name + ].items() # type: ignore[attr-defined] + if k in self.used_builtin_vars + } + output_graph_guards_state = dataclasses.replace( + output_graph_guards_state, + local_scope={ + k: v + for k, v in output_graph_guards_state.local_scope.items() + if k in used_local_vars or k in self.additional_used_local_vars + }, + global_scope=global_scope_state, + _guards=torch._guards.GuardsSet( + OrderedSet( + dataclasses.replace( + guard, + obj_weakref=None, + guarded_class_weakref=None, + create_fn=normalize_create_fn(guard.create_fn), + ) + for guard in sorted_guards + ) + ), + input_source_to_sizes_strides=pytree.tree_map( + convert_int_to_concrete_values, + output_graph_guards_state.input_source_to_sizes_strides, + ), + skip_guards_check=True, + ) + guards_state = GuardsState( + output_graph=output_graph_guards_state, + shape_code_parts=self.shape_code_parts, + source_get_cache=builder.source_get_cache, + ) + + return pickle_guards_state(guards_state, builder.guard_tree_values) + + def build_guards( + self, + sorted_guards: list[Guard], + existing_diff_guard_sources: OrderedSet[str], + f_code: types.CodeType, + output_graph: OutputGraphGuardsState, + save_guards: bool, + source_get_cache: Optional[dict[str, Any]] = None, + ) -> tuple[GuardBuilder, GuardManagerWrapper]: + guard_manager = GuardManagerWrapper() + guard_manager.diff_guard_sources = existing_diff_guard_sources + + w_builder = None + + def source_ref(source: Source) -> str: + guard_source = source.guard_source + if guard_source is GuardSource.CONSTANT: + # No need to track constants + return source.name + assert w_builder + r_builder = w_builder() + assert r_builder is not None + return r_builder.arg_ref(source.name) + + builder = GuardBuilder( + f_code, + self.id_ref, + source_ref, + self.lookup_weakrefs, + output_graph.local_scope, + output_graph.global_scope, + guard_manager, + self, + save_guards, + runtime_global_scope=self.runtime_global_scope, + source_get_cache=source_get_cache, + ) + + # Break retain cycle. See test_release_scope_memory + def cleanup_builder(weak_b: weakref.ref[GuardBuilder]) -> None: + b = weak_b() + if b: + b.scope = None # type: ignore[assignment] + + # Break retain cycle. See test_release_input_memory + w_builder = weakref.ref(builder, cleanup_builder) + + guard_on_nn_modules = config.guard_nn_modules and justknobs_check( + "pytorch/compiler:guard_nn_modules" + ) + + for guard in sorted_guards: + if ( + not guard_on_nn_modules + and guard.is_specialized_nn_module() + # Default func args must be guarded on. + # TODO: we could make use of 'DefaultsSource' and offer a .guard.is_defaults() API + and "__defaults__" not in guard.name + and "__kwdefaults__" not in guard.name + and (config.skip_nnmodule_hook_guards or "hooks" not in guard.name) + ): + continue + + guard.create(builder) + return builder, guard_manager + + def compile_check_fn( + self, + builder: GuardBuilder, + guards_out: list[Guard], + guard_fail_fn: Optional[Callable[[GuardFail], None]], + ) -> None: + # see parallel handling of ".0" / "___implicit0" in _eval_frame.c + largs = builder.argnames + largs += ["**___kwargs_ignored"] + + guards_log.debug("GUARDS:") + + code_parts = [] + verbose_code_parts = [] + structured_guard_fns: list[Callable[[], dict[str, Any]]] = [] + + # Add compile id info in the guard manager for debugging purpose + self.guard_manager.root.attach_compile_id( + str(CompileContext.current_compile_id()) + ) + + # Clear references to torch_function modes held in the list + self.torch_function_mode_stack = None + + def add_code_part( + code_part: str, guard: Optional[Guard], log_only: bool = False + ) -> None: + verbose_code_part = get_verbose_code_part(code_part, guard) + guards_log.debug("%s", verbose_code_part) + + structured_guard_fns.append( + lambda: { + "code": code_part, + "stack": ( + structured.from_traceback(guard.stack.summary()) + if guard and guard.stack + else None + ), + "user_stack": ( + structured.from_traceback(guard.user_stack) + if guard and guard.user_stack + else None + ), + } + ) + + if verbose_guards_log.isEnabledFor(logging.DEBUG): + maybe_stack = "" + maybe_user_stack = "" + if guard is not None: + if guard.stack: + maybe_stack = f"\nStack:\n{''.join(guard.stack.format())}" + if guard.user_stack: + maybe_user_stack = ( + f"\nUser stack:\n{''.join(guard.user_stack.format())}" + ) + verbose_guards_log.debug( + "Guard: %s%s%s", + code_part, + maybe_stack, + maybe_user_stack, + ) + + if not log_only: + code_parts.append(code_part) + verbose_code_parts.append(verbose_code_part) + + seen = set() + for gcl in builder.code: + for code in gcl.code_list: + if code not in seen: + # If Cpp guard manager is enabled, we don't need to add to + # code_parts. + add_code_part(code, gcl.guard, True) + seen.add(code) + + no_tensor_aliasing_names = builder.no_tensor_aliasing_names + check_tensors_fn = None + check_tensors_verbose_fn = None + + if len(no_tensor_aliasing_names) > 1: + # Install tensor aliasing guard. TENSOR_MATCH guards are already + # installed for cpp guard manager. + install_no_tensor_aliasing_guard( + builder.no_tensor_aliasing_guard_managers, + no_tensor_aliasing_names, + ["check_no_aliasing(" + ", ".join(no_tensor_aliasing_names) + ")"], + ) + + # Note - On Lambda guarding of object aliasing + # We previously installed object-aliasing guards as relational guards, + # but that undermined the recursive-dict guard optimization: placing the + # aliasing guard at a leaf prevented the parent dict node from + # qualifying as a recursive-dict guard root. Because aliasing guards are + # rare, we now emit them as epilogue guards via a small Python lambda. + # This repeats the access in Python—adding a bit of work—but the + # overhead is outweighed by the gains from enabling recursive-dict guard + # optimization. + if ( + config.use_lamba_guard_for_object_aliasing + and builder.object_aliasing_guard_codes + ): + aliasing_code_parts, aliasing_verbose_code_parts = map( + list, zip(*builder.object_aliasing_guard_codes) + ) + builder.add_python_lambda_leaf_guard_to_root( + aliasing_code_parts, aliasing_verbose_code_parts + ) + + aotautograd_guards: list[GuardEnvExpr] = ( + self.output_graph.aotautograd_guards if self.output_graph else [] + ) + + # TODO(anijain2305) - There is a duplicate logic in Dynamo to find + # aliased input tensors. So most probably we don't need this here. + # Revisit. + for guard in aotautograd_guards: + if isinstance(guard, DuplicateInputs): + source_a = guard.input_source_a + source_b = guard.input_source_b + code_part = f"{source_a.name} is {source_b.name}" + install_object_aliasing_guard( + builder.get_guard_manager_from_source(source_a), + builder.get_guard_manager_from_source(source_b), + [code_part], + ) + add_code_part(code_part, None, True) + elif isinstance(guard, StorageOverlap): + overlapping_guard_managers = [ + builder.get_guard_manager_from_source(s) + for s in guard.overlapping_sources + ] + non_overlapping_guard_managers = [ + builder.get_guard_manager_from_source(s) + for s in guard.non_overlapping_sources + ] + code_part = ( + """check_overlapping(""" + f"""overlapping=[{", ".join(s.name for s in guard.overlapping_sources)}], """ + f"""non_overlapping=[{", ".join(s.name for s in guard.non_overlapping_sources)}])""" + ) + install_storage_overlapping_guard( + overlapping_guard_managers, + non_overlapping_guard_managers, + [code_part], + ) + add_code_part(code_part, None, True) + else: + raise RuntimeError(f"Unknown GuardEnvExpr: {guard}") + + # TODO: the "guard" here is actually just the top level SHAPE_ENV + # which is useless. Get ShapeEnv to pass in more provenance. + for gcl in builder.shape_env_code: + for code in gcl.code_list: + # Shape env guards are already added for CPP guard manager in + # SHAPE_ENV implementation. + add_code_part(code, gcl.guard, True) + + # OK, all done generating guards + if structured_guard_fns: + torch._logging.trace_structured( + "dynamo_guards", payload_fn=lambda: [f() for f in structured_guard_fns] + ) + + if convert_frame.initial_global_state is None: + # we should only hit this case in NopTests() + check_global_state = convert_frame.GlobalStateGuard().check + else: + check_global_state = getattr(self.global_state, "check", None) + closure_vars = { + "___check_tensors": check_tensors_fn, + "___check_tensors_verbose": check_tensors_verbose_fn, + "___check_global_state": check_global_state, + "___check_torch_function_mode_stack": self.torch_function_mode_stack_check_fn, + **SYMPY_INTERP, + **_get_closure_vars(), + } + + self.guard_manager.finalize() + + globals_for_guard_fn = {"G": builder.scope["G"]} + # Guard manager construction is complete. Ensure we did not miss to + # insert a guard in cpp guard manager. + assert len(code_parts) == 0 + + self.guard_manager.closure_vars = closure_vars + self.guard_manager.args = largs + self.guard_manager.populate_code_parts_for_debugging() + self.guard_manager.verbose_code_parts = verbose_code_parts + # Grab only G, but preserve "G" because guards access it as "G" + self.guard_manager.global_scope = globals_for_guard_fn + self.guard_manager.guard_fail_fn = guard_fail_fn + # will be populated by a non-owning reference to CacheEntry/ExtraState + # when the CacheEntry is constructed + self.guard_manager.cache_entry = None + self.guard_manager.extra_state = None + self.guard_manager.no_tensor_aliasing_sources = no_tensor_aliasing_names + + def invalidate(self, obj_str: str) -> None: + # Some tests reveal that CheckFunctionManager has no attribute + # guard_manager, but this case should not be of any concern. + # This case doesn't seem easy to repro. + if ( + hasattr(self, "guard_manager") + and not isinstance(self.guard_manager, DeletedGuardManagerWrapper) + and (cache_entry := self.guard_manager.cache_entry) is not None + and (extra_state := self.guard_manager.extra_state) is not None + ): + assert isinstance(cache_entry, CacheEntry) + + assert isinstance(extra_state, ExtraState) + reason = f"Cache line invalidated because {obj_str} got deallocated" + deleted_guard_manager = DeletedGuardManagerWrapper(reason) + + extra_state.invalidate(cache_entry, deleted_guard_manager) + self.guard_manager = deleted_guard_manager + + def id_ref(self, obj: object, obj_str: str) -> int: + """add a weakref, return the id""" + try: + if id(obj) not in self._weakrefs: + # We will clear the _weakrefs dict at the end of __init__ + # function, which will delete the callbacks as well. Therefore, + # we are using a finalizer which is kept alive. + self._weakrefs[id(obj)] = weakref.ref(obj) + weakref.finalize( + obj, functools.partial(self.invalidate, obj_str=obj_str) + ) + except TypeError: + pass # cannot weakref bool object + return id(obj) + + def lookup_weakrefs(self, obj: object) -> Optional[weakref.ref[object]]: + """Lookup the _weakrefs created in id_ref function for ID_MATCH'd objects""" + if id(obj) in self._weakrefs: + return self._weakrefs[id(obj)] + return None + + +def build_guard_function(code_parts: list[str], closure_args: str) -> tuple[str, str]: + from torch._inductor.utils import IndentedBuffer + + csepass = PyExprCSEPass() + try: + csepass.count(code_parts) + + def replace(expr: str) -> tuple[list[str], str]: + return csepass.replace(expr) + + except RecursionError: + # If we hit recursion limits during CSE analysis, fall back to a no-op replace function + # This can happen with extremely complex guard expressions + def replace(expr: str) -> tuple[list[str], str]: + return [], expr + + # Generate the inner body of the guard function. + # i.e. if-chain of the guard expressions. + guard_body = IndentedBuffer() + for expr in code_parts: + preface, expr = replace(expr) + guard_body.writelines(preface) + guard_body.writeline(f"if not ({expr}):") + with guard_body.indent(): + guard_body.writeline("return False") + + # Wrap the inner body into the actual guard function. + guard = IndentedBuffer() + guard.writeline("def guard(L):") + with guard.indent(): + guard.splice(guard_body) + guard.writeline("return True") + + # Wrap the whole guard function into another function + # with the closure variables. + make_guard_fn = IndentedBuffer() + make_guard_fn.writeline(f"def ___make_guard_fn({closure_args}):") + with make_guard_fn.indent(): + make_guard_fn.splice(guard) + make_guard_fn.writeline("return guard") + + return guard_body.getvalue(), make_guard_fn.getvalue() + + +def is_recompiles_enabled() -> bool: + return torch._logging._internal.log_state.is_artifact_enabled("recompiles") + + +def is_recompiles_verbose_enabled() -> bool: + return torch._logging._internal.log_state.is_artifact_enabled("recompiles_verbose") + + +# this will only be used if cpp guards are disabled +def make_torch_function_mode_stack_guard( + initial_stack: list[torch.overrides.TorchFunctionMode], +) -> Callable[[], bool]: + types = [type(x) for x in initial_stack] + + def check_torch_function_mode_stack() -> bool: + cur_stack = get_torch_function_mode_stack() + + if len(cur_stack) != len(types): + return False + + for ty, mode in zip(types, cur_stack): + if ty is not type(mode): + return False + + return True + + return check_torch_function_mode_stack + + +Scope = TypeAliasType("Scope", dict[str, object]) + + +def recompilation_reason_for_no_tensor_aliasing_guard( + guard_manager: GuardManagerWrapper, scope: Scope +) -> list[str]: + assert guard_manager.global_scope is not None + global_scope = dict(guard_manager.global_scope) + ids_to_source = collections.defaultdict(list) + for tensor_source in guard_manager.no_tensor_aliasing_sources: + global_scope["__compile_source__"] = tensor_source + tensor_id = id(eval(tensor_source, global_scope, scope)) + ids_to_source[tensor_id].append(tensor_source) + + duplicate_tensors = [ + f"{ids_to_source[key]}" for key in ids_to_source if len(ids_to_source[key]) > 1 + ] + + reason = ", ".join(duplicate_tensors) + return [f"Duplicate tensors found: {reason}"] + + +def strip_local_scope(s: str) -> str: + """ + Replace occurrences of L[...] with just the inner content. + Handles both single and double quotes. + + This is to generate user friendly recompilation messages. + """ + import re + + pattern = r"L\[\s*['\"](.*?)['\"]\s*\]" + return re.sub(pattern, r"\1", s) + + +def get_guard_fail_reason_helper( + guard_manager: GuardManagerWrapper, + f_locals: dict[str, object], + compile_id: Optional[CompileId], + backend: Optional[Callable], +) -> str: + """ + Return the reason why `guard_manager` failed. + Updates `guard_failures` with the generated reason. + Only the first failed check of guard_manager is reported. + """ + assert guard_manager.global_scope is not None + assert guard_manager.closure_vars is not None + scope = {"L": f_locals, "G": guard_manager.global_scope["G"]} + scope.update(guard_manager.closure_vars) + reasons: list[str] = [] + + cache_entry_backend = None + if guard_manager.cache_entry: + cache_entry_backend = guard_manager.cache_entry.backend + + no_tensor_aliasing_check_failed = False + + verbose_code_parts: list[str] = [] + guard_debug_info = guard_manager.check_verbose(f_locals) + # For test_export_with_map_cond, the check_verbose fail even without the + # C++ guard manager. We need to fix the issue to remove the comment. + # assert not guard_debug_info.result + if not guard_debug_info.result: + verbose_code_parts = guard_debug_info.verbose_code_parts + # verbose_code_parts is either the actual reason (e.g. in case of + # TENSOR_MATCH) or it could be a list of verbose_code_part that we + # passed to the leaf guard at construction time. If its a list, we + # walk through this list and find the guard that failed. This is + # very important for symbolic shape guards which are currently + # installed as a lambda guard and can encompass a long list of code_parts. + + if len(verbose_code_parts) == 1: + if "Duplicate tensor found" in verbose_code_parts[0]: + no_tensor_aliasing_check_failed = True + else: + reasons = verbose_code_parts + verbose_code_parts = [] + elif cache_entry_backend != backend: + # None of the guard entries failed - a backend match issue + reason = ( + "BACKEND_MATCH failure: torch.compile detected different backend callables." + " If this is unexpected, wrap your backend in functools.partial (or reuse the" + " same cached backend) to avoid creating a new backend function each time." + " More details: https://github.com/pytorch/pytorch/issues/168373" + ) + reasons.append(reason) + else: + # Unexpected recompilation - points to a bug + reason = ( + "Unexpected recompilation: runtime guards failed even though they passed" + " during recompilation-reason analysis." + " Please open an issue with a minimal repro:" + " https://github.com/pytorch/pytorch" + ) + reasons.append(reason) + + if no_tensor_aliasing_check_failed: + reasons = recompilation_reason_for_no_tensor_aliasing_guard( + guard_manager, scope + ) + else: + for part in verbose_code_parts: + global_scope = dict(guard_manager.global_scope) + global_scope["__compile_source__"] = part + with report_compile_source_on_error(): + try: + fail_reason = eval(part, global_scope, scope) + except Exception: + if is_recompiles_verbose_enabled(): + continue + else: + raise + # Only ___check_tensors knows how to return a fancy fail reason; + # for everything else we just report the code that failed + + if isinstance(fail_reason, bool) and not fail_reason: + fail_reason = part + if isinstance(fail_reason, str): + reasons.append(fail_reason) + if not is_recompiles_verbose_enabled(): + break + + reason_str = f"{compile_id}: " + "; ".join(reasons) + return strip_local_scope(reason_str) + + +def get_guard_fail_reason( + guard_manager: GuardManagerWrapper, + code: types.CodeType, + f_locals: dict[str, object], + compile_id: CompileId, + backend: Callable, + skip_logging: bool = False, +) -> str: + if isinstance(guard_manager, DeletedGuardManagerWrapper): + return f"{compile_id}: {guard_manager.invalidation_reason}" + reason_str = get_guard_fail_reason_helper( + guard_manager, f_locals, compile_id, backend + ) + if skip_logging: + return reason_str + guard_failures[orig_code_map[code]].append(reason_str) + + try: + if guard_manager.guard_fail_fn is not None: + guard_manager.guard_fail_fn( + GuardFail(reason_str or "unknown reason", orig_code_map[code]) + ) + except Exception: + log.exception( + "Failure in guard_fail_fn callback - raising here will cause a NULL Error on guard eval", + ) + + return reason_str + + +def get_and_maybe_log_recompilation_reasons( + cache_entry: Optional[CacheEntry], + frame: DynamoFrameType, + backend: Callable, + skip_logging: bool = False, +) -> list[str]: + """ + Return the list of guard failure reasons using cache_entry. + Logs the recompilation reason if `recompiles` logging is enabled. + Raises a RecompileError if `config.error_on_recompile` is enabled. + """ + reasons = [] + while cache_entry is not None: + reason = get_guard_fail_reason( + cache_entry.guard_manager, + cache_entry.code, + frame.f_locals, + cache_entry.compile_id, + backend, + skip_logging, + ) + if reason: + reasons.append(reason) + cache_entry = cache_entry.next + + code = frame.f_code + + if skip_logging: + return reasons + # at least one of "recompiles" or "recompiles_verbose" is enabled + do_recompiles_log = is_recompiles_enabled() or is_recompiles_verbose_enabled() + + if do_recompiles_log or config.error_on_recompile: + if is_recompiles_verbose_enabled(): + failures = "\n\n".join( + f"guard {i} failures:\n" + textwrap.indent(reason, "- ") + for i, reason in enumerate(reasons) + ) + else: + failures = textwrap.indent("\n".join(reasons), "- ") + guard_failure_details = ( + f"triggered by the following guard failure(s):\n{failures}" + ) + message = ( + f"Recompiling function {code.co_name} in {code.co_filename}:{code.co_firstlineno}\n" + f"{textwrap.indent(guard_failure_details, ' ')}" + ) + if do_recompiles_log: + if is_recompiles_verbose_enabled(): + recompiles_verbose_log.debug(message) + else: + recompiles_log.debug(message) + if config.error_on_recompile: + raise exc.RecompileError(message) + + torch._logging.trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "recompile_reasons", + "encoding": "json", + }, + payload_fn=lambda: reasons, + ) + + return reasons + + +def update_diff_guard_managers_for_existing_cache_entries( + cache_entry: Optional[CacheEntry], +) -> OrderedSet[str]: + first_cache_entry = cache_entry + + # On the first pass, go through the cache entries and accumulate the diff + # guard sources. Different guard managers can fail with different sources. + # So, we collect all of them first. + acc_diff_guard_sources: OrderedSet[str] = OrderedSet() + while cache_entry is not None: + acc_diff_guard_sources.update( + cache_entry.guard_manager.collect_diff_guard_sources() + ) + cache_entry = cache_entry.next # type: ignore[assignment] + + # On the second pass, set the diff_guard_sources for each cache line to the + # accumulated value. And the re-populate the diff guard manager. + cache_entry = first_cache_entry + while cache_entry is not None: + cache_entry.guard_manager.diff_guard_sources = acc_diff_guard_sources + cache_entry.guard_manager.populate_diff_guard_manager() + cache_entry = cache_entry.next # type: ignore[assignment] + + # return the accumulated sources to set up the new cache line. + return acc_diff_guard_sources + + +def guard_error_hook( + guard_manager: GuardFn, + code: types.CodeType, + f_locals: dict[str, object], + index: int, + last: bool, +) -> None: + print( + f"ERROR RUNNING GUARDS {code.co_name} {code.co_filename}:{code.co_firstlineno}" + ) + print("lambda " + ", ".join(guard_manager.args) + ":") + print(" ", " and\n ".join(guard_manager.code_parts)) + + print(guard_manager) + + local_scope = {"L": f_locals, **guard_manager.closure_vars} + for guard in guard_manager.code_parts: + try: + eval(guard, guard_manager.global_scope, local_scope) + except: # noqa: B001,E722 + print(f"Malformed guard:\n{guard}") + + +set_guard_error_hook(guard_error_hook) + + +def unique(seq: Sequence[T]) -> Generator[T, None, None]: + seen = set() + for x in seq: + if x not in seen: + yield x + seen.add(x) + + +def make_dupe_guard( + obj_source: Source, dupe_source: Source +) -> Optional[functools.partial[Any]]: + # Note - we may end up in a situation where we invoke something like + # def fn(x, y) + # with fn(x, x) + # Prior to the addition of tracking to all relevant objects, we would handle this just fine by + # eagerly re-entering VB and rewrapping inputs, correctly creating graphargs and placeholders. However, + # with tracking on inputs, duplicate inputs or aliased relationships may end up getting erased here - + # In the fn(x, x) example call above look like a graph with a single input. + # In order to ensure that we do not reuse fn(x, x) for fn(x, y), we create a duplicate input guard. + + # Note - we may not have a source, that is fine, it just means we had an object that is safe to have + # leave unsourced - like a local list created and discharged entirely within a local scope. + if dupe_source and dupe_source != obj_source: + ser_source_is_local = is_from_local_source(dupe_source) + source_is_local = is_from_local_source(obj_source) + if is_from_flatten_script_object_source( + dupe_source + ) or is_from_flatten_script_object_source(obj_source): + raise exc.UnsafeScriptObjectError( + f"{obj_source.name} is aliasing {dupe_source.name}. This is not supported." + f" Please do a clone for corresponding input." + ) + + # Note - both must be local, or global, or we will run afoul of a lack of merging in how we currently + # reconcile guards builder scopes in compile_check_fn. This technically means we miss a guard here, + # so maybe we should do this refactor before we land this... + # TODO(voz): Combine local and global guard builders. + if ser_source_is_local == source_is_local: + # Note - this is a little aggressive - these being duplicate input does not always matter. + # However, this should always be a sound guard to add here. + return functools.partial(GuardBuilder.DUPLICATE_INPUT, source_b=dupe_source) + return None + + +def install_guard(*guards: Guard, skip: int = 0) -> None: + """ + Add dynamo guards to the current tracing context. + + Args: + guards: guard(s) to add + skip: number of stack frames to ignore for debug stack trace + """ + from torch._guards import TracingContext + + collect_debug_stack = guards_log.isEnabledFor( + logging.DEBUG + ) or verbose_guards_log.isEnabledFor(logging.DEBUG) + add = TracingContext.get().guards_context.dynamo_guards.add + for guard in guards: + assert isinstance(guard, Guard) + if is_from_skip_guard_source(guard.originating_source): + continue + add(guard, collect_debug_stack=collect_debug_stack, skip=skip + 1) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/hooks.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/hooks.py new file mode 100644 index 0000000000000000000000000000000000000000..4f47a80d1ae0a1185fdf32cc017190e91cd42fb3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/hooks.py @@ -0,0 +1,25 @@ +"""Hook system for Dynamo's guard functionality. + +This module provides a way to register callback functions that are triggered during +guard-related operations. + +The Hooks class manages two types of hook functions: +- guard_export_fn: Called when guards need to be exported, taking a GuardsSet as input +- guard_fail_fn: Called when a guard check fails, taking a GuardFail object as input +These hooks enable customization of guard export and failure handling behaviors. +""" + +import dataclasses +from collections.abc import Callable +from typing import Optional + +from torch._guards import GuardsSet + +from .types import GuardFail, GuardFilterEntry + + +@dataclasses.dataclass +class Hooks: + guard_export_fn: Optional[Callable[[GuardsSet], None]] = None + guard_fail_fn: Optional[Callable[[GuardFail], None]] = None + guard_filter_fn: Optional[Callable[[list[GuardFilterEntry]], list[bool]]] = None diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/logging.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/logging.py new file mode 100644 index 0000000000000000000000000000000000000000..74862962adaa10f294408fd8dbe172dcdc71d743 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/logging.py @@ -0,0 +1,73 @@ +"""Logging utilities for Dynamo and Inductor. + +This module provides specialized logging functionality including: +- Step-based logging that prepends step numbers to log messages +- Progress bar management for compilation phases +- Centralized logger management for Dynamo and Inductor components + +The logging system helps track the progress of compilation phases and provides structured +logging output for debugging and monitoring. +""" + +import itertools +import logging +from collections.abc import Callable +from typing import Any + +from torch.hub import _Faketqdm, tqdm + + +# Disable progress bar by default, not in dynamo config because otherwise get a circular import +disable_progress = True + + +# Return all loggers that torchdynamo/torchinductor is responsible for +def get_loggers() -> list[logging.Logger]: + return [ + logging.getLogger("torch.fx.experimental.symbolic_shapes"), + logging.getLogger("torch._dynamo"), + logging.getLogger("torch._inductor"), + ] + + +# Creates a logging function that logs a message with a step # prepended. +# get_step_logger should be lazily called (i.e. at runtime, not at module-load time) +# so that step numbers are initialized properly. e.g.: + +# @functools.cache +# def _step_logger(): +# return get_step_logger(logging.getLogger(...)) + +# def fn(): +# _step_logger()(logging.INFO, "msg") + +_step_counter = itertools.count(1) + +# Update num_steps if more phases are added: Dynamo, AOT, Backend +# This is very inductor centric +# _inductor.utils.has_triton() gives a circular import error here + +if not disable_progress: + try: + import triton # noqa: F401 + + num_steps = 3 + except ImportError: + num_steps = 2 + pbar = tqdm(total=num_steps, desc="torch.compile()", delay=0) + + +def get_step_logger(logger: logging.Logger) -> Callable[..., None]: + if not disable_progress: + pbar.update(1) + if not isinstance(pbar, _Faketqdm): + pbar.set_postfix_str(f"{logger.name}") + + step = next(_step_counter) + + def log(level: int, msg: str, **kwargs: Any) -> None: + if "stacklevel" not in kwargs: + kwargs["stacklevel"] = 2 + logger.log(level, "Step %s: %s", step, msg, **kwargs) + + return log diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/metrics_context.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/metrics_context.py new file mode 100644 index 0000000000000000000000000000000000000000..bc341f10897c658aadb964ff21e8b12425ba092e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/metrics_context.py @@ -0,0 +1,251 @@ +"""Metrics collection and management system for Dynamo. + +This module provides context managers for gathering and reporting metrics during +compilation and runtime. + +It includes two main components: +- MetricsContext: A context manager for collecting metrics during compilation, supporting + nested contexts and various metric types (counters, sets, key-value pairs) +- RuntimeMetricsContext: A specialized context for runtime metrics collection that doesn't + require explicit context management + +The metrics system enables comprehensive monitoring and analysis of both compilation and +execution performance. +""" + +from __future__ import annotations + +import heapq +import logging +import time +from collections.abc import Callable +from typing import Any, Optional, TYPE_CHECKING, TypeAlias +from typing_extensions import Self + + +if TYPE_CHECKING: + from collections.abc import Iterator + +from torch.utils._traceback import CapturedTraceback + + +log = logging.getLogger(__name__) + + +class TopN: + """ + Helper to record a list of metrics, keeping only the top N "most expensive" elements. + """ + + def __init__(self, at_most: int = 25): + self.at_most = at_most + self.heap: list[tuple[int, Any]] = [] + + def add(self, key: Any, val: int) -> None: + # Push if we haven't reached the max size, else push and pop the smallest + fn = heapq.heappush if len(self.heap) < self.at_most else heapq.heappushpop + fn(self.heap, (val, key)) + + def __len__(self) -> int: + return len(self.heap) + + def __iter__(self) -> Iterator[tuple[Any, int]]: + return ((key, val) for val, key in sorted(self.heap, reverse=True)) + + +OnExitType: TypeAlias = Callable[ + [int, int, dict[str, Any], Optional[type[BaseException]], Optional[BaseException]], + None, +] + + +class MetricsContext: + def __init__(self, on_exit: OnExitType): + """ + Use this class as a contextmanager to create a context under which to accumulate + a set of metrics, e.g., metrics gathered during a compilation. On exit of the + contextmanager, call the provided 'on_exit' function and pass a dictionary of + all metrics set during the lifetime of the contextmanager. + """ + self._on_exit = on_exit + self._metrics: dict[str, Any] = {} + self._start_time_ns: int = 0 + self._level: int = 0 + self._edits: list[tuple[CapturedTraceback, set[str]]] = [] + + def __enter__(self) -> Self: + """ + Initialize metrics recording. + """ + if self._level == 0: + # In case of recursion, track at the outermost context. + self._metrics = {} + self._start_time_ns = time.time_ns() + + self._level += 1 + return self + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_value: Optional[BaseException], + _traceback: Any, + ) -> None: + """ + At exit, call the provided on_exit function. + """ + self._level -= 1 + assert self._level >= 0 + if self._level == 0: + try: + end_time_ns = time.time_ns() + self._on_exit( + self._start_time_ns, end_time_ns, self._metrics, exc_type, exc_value + ) + except Exception: + log.exception("Unexpected exception logging compilation metrics") + + def in_progress(self) -> bool: + """ + True if we've entered the context. + """ + return self._level > 0 + + def increment(self, metric: str, value: int) -> None: + """ + Increment a metric by a given amount. + """ + if self._level == 0: + raise RuntimeError(f"Cannot increment {metric} outside of a MetricsContext") + if metric not in self._metrics: + self._metrics[metric] = 0 + self._metrics[metric] += value + + def _render_edits(self, pred: set[str]) -> str: + return "\n\n" + "\n\n".join( + "Previous Traceback:\n" + "".join(e.format()) + for e, k in self._edits + if k & pred + ) + + def set(self, metric: str, value: Any, overwrite: bool = False) -> None: + """ + Set a metric to a given value. Raises if the metric has been assigned previously + in the current context. + """ + if self._level == 0: + raise RuntimeError(f"Cannot set {metric} outside of a MetricsContext") + if metric in self._metrics and not overwrite: + raise RuntimeError( + self._render_edits({metric}) + + f"\n\nRuntimeError: Metric '{metric}' has already been set in the current context " + "(see above for current and previous traceback)." + ) + self._edits.append((CapturedTraceback.extract(skip=1), {metric})) + self._metrics[metric] = value + + def set_key_value(self, metric: str, key: str, value: Any) -> None: + """ + Treats a give metric as a dictionary and set the k and value within it. + Note that the metric must be a dictionary or not present. + + We allow this to be called multiple times (i.e. for features, it's not uncommon + for them to be used multiple times within a single compilation). + """ + if self._level == 0: + raise RuntimeError(f"Cannot set {metric} outside of a MetricsContext") + if metric not in self._metrics: + self._metrics[metric] = {} + self._metrics[metric][key] = value + + def update(self, values: dict[str, Any], overwrite: bool = False) -> None: + """ + Set multiple metrics directly. This method does NOT increment. Raises if any + metric has been assigned previously in the current context and overwrite is + not set to True. + """ + if self._level == 0: + raise RuntimeError("Cannot update metrics outside of a MetricsContext") + existing = self._metrics.keys() & values.keys() + if existing and not overwrite: + raise RuntimeError( + self._render_edits(set(values.keys())) + + f"\n\nRuntimeError: Metric(s) {existing} have already been set in the current context. " + "(see above for current and previous traceback)." + ) + self._edits.append((CapturedTraceback.extract(skip=1), set(values.keys()))) + self._metrics.update(values) + + def update_outer(self, values: dict[str, Any]) -> None: + """ + Update, but only when at the outermost context. + """ + if self._level == 0: + raise RuntimeError("Cannot update metrics outside of a MetricsContext") + if self._level == 1: + self.update(values) + + def add_to_set(self, metric: str, value: Any) -> None: + """ + Records a metric as a set() of values. + """ + if self._level == 0: + raise RuntimeError(f"Cannot add {metric} outside of a MetricsContext") + if metric not in self._metrics: + self._metrics[metric] = set() + self._metrics[metric].add(value) + + def add_top_n(self, metric: str, key: Any, val: int) -> None: + """ + Records a metric as a TopN set of values. + """ + if self._level == 0: + return + if metric not in self._metrics: + self._metrics[metric] = TopN() + self._metrics[metric].add(key, val) + + +class RuntimeMetricsContext: + def __init__(self, on_exit: OnExitType): + """ + Similar to MetricsContext, but used to gather the runtime metrics that are + decoupled from compilation, where there's not a natural place to insert a + context manager. + """ + self._on_exit = on_exit + self._metrics: dict[str, Any] = {} + self._start_time_ns: int = 0 + + def increment( + self, metric: str, value: int, extra: Optional[dict[str, Any]] = None + ) -> None: + """ + Increment a metric by a given amount. + """ + if not self._metrics: + # Start timing on the first entry + self._start_time_ns = time.time_ns() + if metric not in self._metrics: + self._metrics[metric] = 0 + self._metrics[metric] += value + + if extra: + for k, v in extra.items(): + if k not in self._metrics and v is not None: + self._metrics[k] = v + + def finish(self) -> None: + """ + Call the on_exit function with the metrics gathered so far and reset. + """ + if self._metrics: + try: + end_time_ns = time.time_ns() + self._on_exit( + self._start_time_ns, end_time_ns, self._metrics, None, None + ) + except Exception: + log.exception("Unexpected exception logging runtime metrics") + finally: + self._metrics = {} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/mutation_guard.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/mutation_guard.py new file mode 100644 index 0000000000000000000000000000000000000000..0467ea1ba1164f63a4ec9c77be28ad5d1fb3e88e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/mutation_guard.py @@ -0,0 +1,160 @@ +"""Mutation tracking and dynamic module detection system for Dynamo. + +This module provides mechanisms to track and respond to mutations in PyTorch modules +and detect dynamically created or modified modules. + +Key components: +- MutationTracker: Tracks mutations to objects and invalidates associated cached code +- GenerationTracker: Tracks module creation timing to identify dynamic instances +- Patching system for nn.Module to detect mutations and dynamic creation + +The system ensures that Dynamo's optimizations remain valid by detecting and responding +to runtime changes in module state and structure. +""" + +import functools +import weakref +from collections.abc import MutableMapping +from typing import Any + +import torch.nn +from torch.nn import Module + +from . import config +from .utils import ExactWeakKeyDictionary, nn_module_has_global_hooks + + +unpatched_nn_module_init = torch.nn.Module.__init__ + + +class MutationTracker: + db: ExactWeakKeyDictionary = ExactWeakKeyDictionary() + + def __init__(self) -> None: + self.mutation_count: int = 0 + self.watchers: list[weakref.ReferenceType[Any]] = [] + + def on_mutation(self, name: str) -> None: + self.mutation_count += 1 + tmp = self.watchers + self.watchers = [] + for ref in tmp: + guarded = ref() + if guarded is not None: + guarded.invalidate(ref) + + def track(self, guarded_code: Any) -> None: + self.watchers.append(weakref.ref(guarded_code)) + + +def watch(obj: Any, guarded_code: Any) -> None: + """invalidate guarded_code when obj is mutated""" + ensure_patched(type(obj)) + + if obj not in MutationTracker.db: + MutationTracker.db[obj] = MutationTracker() + tracker = MutationTracker.db[obj] + tracker.track(guarded_code) + + +def ensure_patched(cls: Any) -> None: + if getattr(cls, "___needs_mutation_patch", True): + cls.___needs_mutation_patch = False + original_setattr = cls.__setattr__ + + @functools.wraps(original_setattr) + def custom_setattr(self: Any, key: str, value: Any) -> None: + try: + MutationTracker.db[self].on_mutation(key) + except KeyError: + pass + return original_setattr(self, key, value) + + cls.__setattr__ = custom_setattr + + +class GenerationTracker: + generation: int = 0 + dynamic_classes: ExactWeakKeyDictionary = ExactWeakKeyDictionary() + generation_values: ExactWeakKeyDictionary = ExactWeakKeyDictionary() + + @classmethod + def tag(cls, obj: Any) -> None: + cls.generation_values[obj] = cls.generation + + @staticmethod + def mark_class_dynamic(cls: type[torch.nn.Module]) -> None: + assert issubclass(cls, torch.nn.Module) + GenerationTracker.dynamic_classes[cls] = True + + @classmethod + def get_generation_value(cls, obj: Any) -> int: + if obj not in cls.generation_values: + return -1 + return cls.generation_values[obj] + + @classmethod + def check(cls, obj: Any) -> bool: + return ( + obj in cls.generation_values + and cls.generation_values[obj] == cls.generation + ) + + @classmethod + def clear(cls) -> None: + cls.generation = 0 + cls.dynamic_classes = ExactWeakKeyDictionary() + cls.generation_values = ExactWeakKeyDictionary() + + +def is_dynamic_nn_module(obj: Any, is_export: bool) -> bool: + """Check for nn.Modules() created dynamically or mutated""" + if isinstance(obj, torch.nn.Module) and ( + "forward" in obj.__dict__ or isinstance(obj, (dict, MutableMapping)) + ): + # A monkey patched `.forward` indicates something wacky is going on + # Similarly a nn module also subclassed as a dict is unusual. + return True + if hasattr(obj, "torchdynamo_force_dynamic"): + return obj.torchdynamo_force_dynamic + if ( + isinstance(obj, torch.nn.Module) + and config.inline_inbuilt_nn_modules + and (not is_export or config.install_free_tensors) + ): + return True + + if isinstance(obj, torch.nn.Module) and nn_module_has_global_hooks(): + return True + dyn = GenerationTracker.dynamic_classes.get(type(obj)) or GenerationTracker.check( + obj + ) + return dyn + + +def install_generation_tagging_init() -> None: + """ + Monkey patch torch.nn.Module.__init__ and torch.nn.Module.__setstate__ + so we can detect nn.Module instances created dynamically inside forward methods. + """ + + if getattr(Module, "___needs_generation_tag_patch", True): + init = Module.__init__ + + def patched_init(self: Module, *args: Any, **kwargs: Any) -> None: + init(self, *args, **kwargs) + GenerationTracker.tag(self) + + Module.__init__ = patched_init # type: ignore[method-assign] + + setstate = Module.__setstate__ + + def patched_setstate(self: Module, state: Any) -> None: + setstate(self, state) + GenerationTracker.tag(self) + + Module.__setstate__ = patched_setstate # type: ignore[method-assign] + + Module.___needs_generation_tag_patch = False # type: ignore[attr-defined] + + GenerationTracker.generation += 1 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/output_graph.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/output_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..37d6dd4328c8a93224ce8b037138994ee76f6760 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/output_graph.py @@ -0,0 +1,3895 @@ +""" +Core graph building functionality for PyTorch's Dynamo system. This module contains +the essential components for constructing and managing FX graphs during compilation: + +- OutputGraph: Manages the overall graph construction and compilation process. It owns + a SubgraphTracer and handles graph compilation, execution, and state management. + OutputGraph also manages features like graph deduplication, symbolic shape handling, + and tracking of side effects. + +- SubgraphTracer: Handles the actual FX graph construction by tracing Python code. + It supports advanced features like higher-order operators through nested tracers, + lifting of free variables, and handling of symbolic shapes. + +The module supports key Dynamo features including: +- Higher-order operators through nested SubgraphTracers +- Graph deduplication for optimization +- Symbolic shape handling and propagation +- Side effect tracking and management +- Guard insertion and management +""" + +import collections +import contextlib +import copy +import functools +import inspect +import itertools +import logging +import operator +import re +import sys +import traceback +import warnings +import weakref +from collections.abc import Callable, Generator, Sequence +from dataclasses import dataclass, field as dc_field +from types import CodeType +from typing import Any, cast, Optional, TYPE_CHECKING, Union +from typing_extensions import ParamSpec, TypeVar + +import sympy + +import torch._guards +import torch._logging +import torch.distributed as dist +import torch.nn +import torch.utils._pytree as pytree +from torch import fx, Tensor +from torch._C._dynamo import guards +from torch._dynamo.exc import ShortenTraceback, TensorifyScalarRestartAnalysis +from torch._guards import ( + CompileContext, + CompileId, + GlobalContextCheckpointState, + Source, + tracing, + TracingContext, +) +from torch._library.opaque_object import is_opaque_type +from torch._subclasses.fake_tensor import FakeTensor +from torch._utils_internal import signpost_event +from torch.export.dynamic_shapes import _ConstraintTarget +from torch.fx._lazy_graph_module import _make_graph_module # type: ignore[attr-defined] +from torch.fx.experimental._backward_state import BackwardState +from torch.fx.experimental.symbolic_shapes import ( + free_symbols, + guard_scalar, + is_symbolic, + ShapeEnv, + Specialization, + uninteresting_files, +) +from torch.fx.node import Target +from torch.fx.passes.runtime_assert import insert_deferred_runtime_asserts +from torch.utils._ordered_set import OrderedSet +from torch.utils._python_dispatch import is_traceable_wrapper_subclass + +from . import config, exc, logging as torchdynamo_logging, variables +from .backends.registry import CompiledFn, CompilerFn +from .bytecode_transformation import ( + create_binary_slice, + create_binary_subscr, + create_build_tuple, + create_call_function, + create_dup_top, + create_instruction, + create_load_const, + create_rot_n, + create_swap, + Instruction, + unique_id, +) +from .code_context import code_context +from .codegen import PyCodegen +from .current_scope_id import enter_new_scope +from .device_interface import get_interface_for_device +from .exc import ( + BackendCompilerFailed, + exceptions_allowed_to_be_fallback, + SkipFrame, + unimplemented, + unimplemented_with_warning, +) +from .graph_bytecode_inputs import has_user_objects, index_to_bytecode_constructor +from .graph_deduplication import apply_graph_deduplication +from .graph_region_tracker import GraphRegionTracker +from .guards import GuardBuilder, install_guard +from .mutation_guard import is_dynamic_nn_module +from .side_effects import AttributeMutationExisting, SideEffects, ValueMutationExisting +from .source import ( + _get_source_debug_name, + AttrSource, + BackwardStateSource, + ConstantSource, + GetItemSource, + GlobalStateSource, + is_constant_source, + is_from_local_source, + LocalSource, + NumpyTensorSource, + ParamBufferSource, + ShapeEnvSource, + SyntheticLocalSource, + TensorProperty, + TensorPropertySource, +) +from .utils import ( + _extract_tensor_dict, + checkpoint_params, + CleanupHook, + clone_inputs, + count_calls, + counters, + dynamo_timed, + get_instruction_source_311, + get_locals_to_steal, + get_static_address_type, + get_unique_name_wrt, + graph_break_reasons, + increment_op_count, + istype, + lazy_format_graph_code, + LazyString, + nn_module_proxy, + same, + set_example_value, +) +from .variables.base import VariableTracker +from .variables.builder import ( + BackwardStateGraphArg, + GraphArg, + TrackedFake, + wrap_fx_proxy, +) +from .variables.ctx_manager import ContextWrappingVariable +from .variables.lists import BaseListVariable +from .variables.misc import NullVariable +from .variables.nn_module import NNModuleVariable +from .variables.tensor import ( + NumpyNdarrayVariable, + SymNodeVariable, + UnspecializedPythonVariable, +) +from .variables.torch_function import TensorWithTFOverrideVariable +from .variables.user_defined import UserDefinedDictVariable + + +if TYPE_CHECKING: + from torch._dynamo.package import CompilePackage + from torch._dynamo.symbolic_convert import InstructionTranslatorBase + from torch.multiprocessing.reductions import StorageWeakRef + +log = logging.getLogger(__name__) +graph_tabular_log = torch._logging.getArtifactLogger(__name__, "graph") +graph_code_log = torch._logging.getArtifactLogger(__name__, "graph_code") +graph_sizes_log = torch._logging.getArtifactLogger(__name__, "graph_sizes") +trace_call_log = torch._logging.getArtifactLogger(__name__, "trace_call") + +RootGuardManager = guards.RootGuardManager + + +# Capture fn pointer at import time +# This is to guard against trying to mark the iterated tensors +# as static in case user overrides fn ptr +og_module_named_buffers_fn_ptr = torch.nn.Module.named_buffers +og_module_named_parameters_fn_ptr = torch.nn.Module.named_parameters + + +@dataclass(frozen=True) +class VariableTrackerCacheKey: + vt_id: int + # Two different source can point to the same object. However, Dynamo handles + # globals and local source differently when it comes to guards and possibly + # some other parts as well. So, cache also relies on the source. + source: Source + + +@dataclass(frozen=True) +class AliasingInfo: + has_aliasing: bool + msg: str + + +@dataclass(frozen=True) +class MutationInfo: + has_mutation: bool + msg: str + + +class VariableTrackerCache: + def __init__(self) -> None: + self.cache: dict[VariableTrackerCacheKey, VariableTracker] = {} + + def lookup(self, value: Any, source: Source) -> Optional[VariableTracker]: + key = VariableTrackerCacheKey(id(value), source) + if key not in self.cache: + return None + return self.cache[key] + + def add(self, value: Any, source: Source, vt: VariableTracker) -> None: + key = VariableTrackerCacheKey(id(value), source) + self.cache[key] = vt + + def clone(self) -> "VariableTrackerCache": + # Needed for copy and restore graph state + new_cache = VariableTrackerCache() + new_cache.cache.update(self.cache) + return new_cache + + def clear(self) -> None: + self.cache.clear() + + +@functools.cache +def _step_logger() -> Any: + return torchdynamo_logging.get_step_logger(log) + + +@dataclass +class GraphCompileReason: + """Stores why a given output graph was compiled; i.e. what caused the graph break.""" + + reason: str + user_stack: list[traceback.FrameSummary] + + # Indicates if this was a graph break reason due to graph break. + graph_break: bool = True + + def __post_init__(self) -> None: + if self.graph_break: + graph_break_reasons.append(self) + + +def _get_gen_rand_values_fn(random_calls: Any) -> Callable[[], list[Any]]: + def _gen_rand_values() -> list[Any]: + return [fn(*args, **kwargs) for fn, args, kwargs in random_calls] + + return _gen_rand_values + + +class FakeRootModule(torch.nn.Module): + """Trick the constructor of fx.GraphModule""" + + def __init__(self, nn_modules: dict[str, torch.nn.Module]): + super().__init__() + for k, v in nn_modules.items(): + setattr(self, k, v) + + def __repr__(self) -> str: + return "FakeRootModule(...)" + + def add_nn_modules(self, nn_modules: dict[str, torch.nn.Module]) -> None: + for k, v in nn_modules.items(): + setattr(self, k, v) + + +class WrapperBackend: + def __init__(self, backend: CompilerFn) -> None: + self.backend: CompilerFn = backend + + def __call__( + self, gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor] + ) -> CompiledFn: + self.restore = checkpoint_params(gm) + self.gm = gm + copy_gm = copy.deepcopy(self.gm) + self.candidate = self.backend(copy_gm, example_inputs) + + if self.candidate is None or self.candidate is self.gm.forward: + return self.gm.forward + + if not config.verify_correctness: + return self.candidate + + # if verify_correctness=True + try: + correct = self.gm.forward(*clone_inputs(example_inputs)) + result = self.candidate(*clone_inputs(example_inputs)) + + # TODO: replace `same` function with the one in testing + if same(correct, result): + return self.candidate + + raise RuntimeError(f"incorrect results of backend {self}") + + except Exception: + log.exception("error in verify_correctness") + raise + finally: + self.restore() + + +Scope = dict[str, object] + + +@dataclass +class OutputGraphGuardsState: + """ + A base class containing fields that are considered "persistent" when we + want to save all the important state for reconstrucing guards in a different + process. Normally we don't need to add states here, but we may have to when + the information is needed to serialize the guards, so the fields here are + supposed to be serializable as a requirement. + """ + + local_scope: Scope + global_scope: Scope + # This records the initial torch function mode stack for guarding + torch_function_mode_stack: list[torch.overrides.TorchFunctionMode] + guard_on_key_order: set[Source] + # Map from graph input's `Source` to sizes / strides metadata + input_source_to_sizes_strides: dict[Source, dict[str, Any]] + dual_level: int + functorch_layers: list[torch._functorch.pyfunctorch.FuncTorchInterpreter] + current_device: Optional[torch.device] + global_state_guard: torch._C._dynamo.guards.GlobalStateGuard + _guards: torch._guards.GuardsSet + _aotautograd_guards: list[torch._guards.GuardEnvExpr] + + # Whether or not the guards should be checked for correctness + + export: bool = False + skip_guards_check: bool = False + export_constraints: bool = False + name_of_builtins_dict_key_in_fglobals: Optional[str] = None + + @property + def shape_env(self) -> ShapeEnv: + raise AssertionError(f"shape_env shouldn't be accessed from {type(self)}") + + @property + def guards(self) -> torch._guards.GuardsSet: + return self._guards + + @property + def aotautograd_guards(self) -> list[torch._guards.GuardEnvExpr]: + return self._aotautograd_guards + + def dump_guards_state(self) -> "OutputGraphGuardsState": + # Dump a serializable version of self without extras + return OutputGraphGuardsState( + local_scope=self.local_scope, + global_scope=self.global_scope, + torch_function_mode_stack=self.torch_function_mode_stack, + guard_on_key_order=self.guard_on_key_order, + input_source_to_sizes_strides=self.input_source_to_sizes_strides, + dual_level=self.dual_level, + functorch_layers=self.functorch_layers, + current_device=self.current_device, + global_state_guard=self.global_state_guard, + name_of_builtins_dict_key_in_fglobals=self.name_of_builtins_dict_key_in_fglobals, + export=self.export, + export_constraints=self.export_constraints, + _guards=self.guards, + _aotautograd_guards=self.aotautograd_guards, + skip_guards_check=self.skip_guards_check, + ) + + +@dataclass +class StackLocalsMetadata: + """ + Stores metadata for a frame's stack and locals for the purposes of building resume functions + """ + + num_stack: int = 0 # number of stack elements, minus removed NULLs + locals_names: dict[str, int] = dc_field( + default_factory=dict + ) # order of locals codegen'd to the stack + stack_null_idxes: list[int] = dc_field(default_factory=list) + locals_null_keys: list[str] = dc_field(default_factory=list) + stack_ctx_args: list[tuple[int, tuple[Any, ...]]] = dc_field(default_factory=list) + stack_ctx_idxes_orig: list[int] = dc_field(default_factory=list) + locals_ctx_args: list[tuple[str, tuple[Any, ...]]] = dc_field(default_factory=list) + + +# TODO we should expand this to make it work for atribtrary in/out +@dataclass +class ExportMetaData: + # maps graph input index to its' source which is later + # used in export to map to correct user input. In its' flat form, + # just looks like GetItem(base=LocalSource("foo", idx=0)) + graph_input_idx_to_local_source: dict[int, Source] = dc_field(default_factory=dict) + # maps user output idx to what type of output it is. There are 3 options: + # 1) graph out + # 2) user input + # 3) constants + output_return_type: dict[int, tuple[str, Any]] = dc_field(default_factory=dict) + # output spec of the traced function + out_spec: Union[torch.utils._pytree.TreeSpec, torch.utils._pytree.LeafSpec] = ( + torch.utils._pytree._LEAF_SPEC + ) + module_call_spec: dict[ + str, + dict[str, Union[torch.utils._pytree.TreeSpec, torch.utils._pytree.LeafSpec]], + ] = dc_field(default_factory=dict) + + +def get_builtins_dict(global_scope: Scope) -> dict[str, Any]: + # f_globals["__builtins__"] can be a dict or a module. This is an + # implementation detail - + # https://docs.python.org/3/library/builtins.html. + + # This makes guarding on any builtin messy because the guard check_fn + # has to check if the __builtins__ is a module or dict, and then access + # by either using getattr or getitem respectively. + + # To solve this problem, we insert a new entry in f_globals which points + # to the builtins __dict__ and then we guard any builtin on this dict. + # To avoid any collision with the pre-existing keys, we use the + # install_global to give us a unique dict key. + + f_builtins = global_scope["__builtins__"] + if not isinstance(f_builtins, dict): + f_builtins = f_builtins.__dict__ + return f_builtins + + +class OutputGraphCommon(OutputGraphGuardsState): + """ + A minimal interface for full graph capture. It is intended to be + the target of any tracer that feeds into backends. + + Currently dynamo's OutputGraph is the only known implementation + of this interface, used by (aot) precompile and (strict) export. + Importantly, that implementation also contains many other fields + that are using during tracing but not included in this interface + because they are not used once tracing is complete. + + It should be safe to assume that (caching) precompile also uses + this interface. + + In the future, we want make_fx, used by (non-strict) export, to + also implement this interface. + + The serializable part of this interface is OutputGraphGuardsState. + We do not need to serialize other parts; however it will pay to + be disciplined about what those other parts are, especially since + we want other tracers to be able to meaningfully implement them, + and we should generally try to cut them down when possible. + """ + + def __init__( + self, + output_graph_guards_state: OutputGraphGuardsState, + import_sources: Optional[dict[str, str]] = None, + shape_env: Optional[ShapeEnv] = None, + export_metadata: Optional[ExportMetaData] = None, + tracked_fakes_id_to_source: Optional[dict[int, list[Source]]] = None, + ): + super().__init__( + output_graph_guards_state.local_scope, + output_graph_guards_state.global_scope, + output_graph_guards_state.torch_function_mode_stack, + output_graph_guards_state.guard_on_key_order, + output_graph_guards_state.input_source_to_sizes_strides, + output_graph_guards_state.dual_level, + output_graph_guards_state.functorch_layers, + output_graph_guards_state.current_device, + output_graph_guards_state.global_state_guard, + output_graph_guards_state._guards, + output_graph_guards_state._aotautograd_guards, + output_graph_guards_state.export, + output_graph_guards_state.skip_guards_check, + output_graph_guards_state.export_constraints, + output_graph_guards_state.name_of_builtins_dict_key_in_fglobals, + ) + + self.import_sources = import_sources or {} + # The following fields are currently known to be used by clients. + # In particular, we need: + # - shape_env, for building guards + # - export_metadata, for un/flattening inputs and outputs + # - tracked_fakes_id_to_source, for processing tensor dim constraints + self._shape_env = shape_env or ShapeEnv() # private for inheritance + self.export_metadata = export_metadata or ExportMetaData() + self.tracked_fakes_id_to_source: dict[int, list[Source]] = ( + tracked_fakes_id_to_source or {} + ) + + @property + def shape_env(self) -> ShapeEnv: + return self._shape_env + + def bypass_package(self, reason: str = "", **kwargs: Any) -> None: + # NOTE: currently there are no tests for this but it is reachable + # when building guards, so technically necessary to include here. + # It is unclear whether we should include packaging altogether. + raise NotImplementedError + + +class OutputGraph(OutputGraphCommon): + """ + Wrapper class to hold outputs of InstructionTranslator. Mainly the + generated fx.Graph. + + OutputGraph is 1:1 with a frame being processed. Each frame is associated + with some root InstructionTranslator. When user code calls a function, + we construct a InliningInstructionTranslator that continues to write into + the root InstructionTranslator's OutputGraph. + """ + + side_effects: SideEffects + + def __init__( + self, + code_options: dict[str, Any], + compiler_fn: Optional[CompilerFn], + root_tx: "InstructionTranslatorBase", + export: bool, + export_constraints: Sequence[_ConstraintTarget], + frame_state: Any, + local_scope: Scope, + global_scope: Scope, + f_code: CodeType, + torch_function_mode_stack: list[torch.overrides.TorchFunctionMode], + package: Optional["CompilePackage"], + one_graph: bool = False, + ) -> None: + OutputGraphGuardsState.__init__( + self, + local_scope, + global_scope, + torch_function_mode_stack, + guard_on_key_order=set(), + input_source_to_sizes_strides={}, + dual_level=torch.autograd.forward_ad._current_level, + functorch_layers=torch._functorch.pyfunctorch.retrieve_all_functorch_interpreters(), + current_device=torch.utils._device.CURRENT_DEVICE, + # initial_global_state is only None during NopTest. + global_state_guard=torch._dynamo.convert_frame.initial_global_state + or torch._C._dynamo.guards.GlobalStateGuard(), + # These are set by @property instead, just initialize them as blank + _guards=torch._guards.GuardsSet(), + _aotautograd_guards=[], + ) + self.tracers = [SubgraphTracer(self, is_export=export)] + # Map from graph input's `Source` to its `VariableTracker` to + # de-duplicate graph inputs by source and reuse the tracker + self.input_source_to_var: dict[Source, VariableTracker] = {} + self.export = export + self.export_constraints = export_constraints # type: ignore[assignment] + self.frame_state = frame_state + self.cleanup_hooks: list[Callable[[], Any]] = [] + # compile_id is an id number for the current torch.compile + self.compile_id: int = next(_compile_id_counter) + # Set of globals installed via install_global* APIs + self.installed_globals: set[str] = set() + + # TODO: maybe should just pass the entire f_code in here? Not + # sure... + self.co_fields = { + "co_name": f_code.co_name, + "co_filename": f_code.co_filename, + "co_firstlineno": f_code.co_firstlineno, + } + + self.region_tracker = GraphRegionTracker() + + # tracked_fakes says where any tensor that was wrapped to fake came + # from. It is similar to GraphArg, in that all GraphArgs will get + # will get added to TrackedFakes, but TrackedFakes also contains + # GraphArgs that got pruned, and things like Tensor attributes which + # aren't explicit graph inputs. Used by shape guard + self.tracked_fakes: list[TrackedFake] = [] + + shape_env = ShapeEnv( + # Reference Cycle! + # Share a reference to the list of TrackedFake. + # + # ShapeEnv needs this in order to be able to reproduce the call + # to produce_guards at an arbitrary time point. That is because + # TrackedFake instances may have its metadata changed throughout + # the program execution. + tracked_fakes=self.tracked_fakes, + # We want to allow capture scalar outputs and allow_dynamic_output_shape_ops when fullgraph=True + allow_scalar_outputs=one_graph or config.capture_scalar_outputs, + allow_dynamic_output_shape_ops=one_graph + or config.capture_dynamic_output_shape_ops, + prefer_deferred_runtime_asserts_over_guards=config.prefer_deferred_runtime_asserts_over_guards, + co_fields=self.co_fields, + ) + + # In export mode, we force the shape_env to strictly disallow any constraining + # of the user marked dynamic dims + import torch._functorch.config as _config + + with _config.patch(fake_tensor_allow_unsafe_data_ptr_access=False): + fake_mode = torch._subclasses.FakeTensorMode( + shape_env=shape_env, + # TODO (tmanlaibaatar) Remove this once we always lift params and buffers + allow_non_fake_inputs=bool(self.export), + export=self.export, + ) + self.tracing_context: TracingContext = TracingContext(fake_mode) + self.tracing_context.traced_code.append(f_code) + self.traced_code = self.tracing_context.traced_code + self.dynamo_compile_id: Optional[CompileId] = ( + CompileContext.current_compile_id() + ) + self.init_ambient_guards() + + # Map each tensor id to a list of sources. This is necessary because + # tensor ids cannot be recovered from tracked fakes (in general). + # We use this map to interpret (i.e., check for violations of) constraints, + # specifically equality constraints, which have shared tensor ids in them. + # This map should also be generally useful, e.g., for (de)serialization. + self.tracked_fakes_id_to_source: dict[int, list[Source]] = ( + collections.defaultdict(list) + ) + # Stores the full fqn of a param or buffer to the relevant source. + self.param_name_to_source: Optional[dict[str, Source]] = {} + self.side_effects = SideEffects(self) + # Cached variable trackers. This makes symbolic analysis of LOAD_GLOBAL + # and LOAD_ATTR for same python objects free. + self.variable_tracker_cache = VariableTrackerCache() + self.unique_var_id = itertools.count() + self.code_options: dict[str, Any] = dict(code_options) + self.output_instructions: list[Instruction] = [] + # used to track nodes that are added between calls of copy_graphstate + # and restore_graphstate + self.timestamp = 0 + + # A list of register_finalizer_fns to apply to the output graph module + self.register_finalizer_fns: list[Callable[[fx.GraphModule], None]] = [] + + # Not checkpointed + self.compiler_fn: Optional[CompilerFn] = compiler_fn + self.root_tx = root_tx + + self.package = package + # Given a source, what are the user stacks of all locations that + # accessed it? + # + # For efficiency, we only populate this: + # - During export, and + # - If the source could potentially lead to a spurious export input + # + # Feel free to populate this more frequently if other use-cases arise, + # but be aware that we have to generate full stacks for each + # recording! + self.source_to_user_stacks: dict[Source, list[traceback.StackSummary]] = {} + + self._current_tx: list[InstructionTranslatorBase] = [] + self.cleanups: list[CleanupHook] = [] + self.should_exit = False + self.unspec_variable_map: dict[str, UnspecializedPythonVariable] = {} + + # This returns false if TF Overall (both mode and subclass) is disabled OR that TF Mode stack is empty + self.torch_function_mode_enabled = torch._C._is_torch_function_mode_enabled() + + # Tracks if the output graph has a user defined allowed function in the + # graph. This is used later to determine if we should fallback to eager + # for certain exceptions. THe idea is that if the user has applied + # allow_in_graph, they would like to see the error instead of falling + # back for backend errors. + self.has_user_defined_allowed_in_graph = False + + # Tracks a list of called ops that were not tagged with "pt2_compliant_tag". + # This information is useful for logging. + self.non_compliant_ops: set[torch._ops.OpOverload] = set({}) + + # Tracks a list of called custom ops that were tagged with "pt2_compliant_tag". + # This information is useful for logging. + self.compliant_custom_ops: set[torch._ops.OpOverload] = set({}) + + # We save the global torch state here to be restored in case of graph + # breaks. The relevant issue is seen here + # https://github.com/pytorch/pytorch/pull/100570#issuecomment-1543427086 + # where inlining of a function changes the global state (because of the + # presence of torch.no_grad) and there is a graph break. + self.save_global_state() + + # Tracks the original FQNs of the constant tensors from the original graph, + # i.e. buffers and parameters. + self.dynamo_flat_name_to_original_fqn: dict[str, str] = {} + + # All calls to random() are replaced with a single call to __gen_rand_values + # functions that returns a tuple of random values for each original call. + # random_calls tracks calls to random() and random_values_var stores the name of + # the variable that stores __gen_rand_values results. + self.random_calls: list[ + tuple[Callable[..., object], tuple[object, ...], dict[str, object]] + ] = [] + self.random_values_var: Any = None + + # Bytecode to insert right before we call the graph + self.pregraph_bytecode: list[Instruction] = [] + + # Use to pass values to backward hooks when using compiled autograd + self.backward_state: dict[str, VariableTracker] = {} + self.backward_state_proxy: Optional[torch.fx.Proxy] = None + self.backward_state_var: Optional[str] = None + + # pyrefly: ignore [bad-override] + self.name_of_builtins_dict_key_in_fglobals: str = ( + self.install_builtins_dict_in_fglobals() + ) + + self.compiler_trace_stack = contextlib.ExitStack() + + # These are the ambient, currently-global saved_tensor_hooks stashed in autograd, + # that are set for the entire duration of the compiled region. + # This is an invariant today because we graph break on the saved_tensor_hook + # context manager inside a compiled region + self.saved_tensors_hooks_subgraph_names: Optional[list[str]] = ( + self.maybe_install_saved_tensors_hooks_subgraphs() + ) + + # mangled alias -> module fqn name + self.import_sources: dict[str, str] = {} + + self.export_metadata = ExportMetaData() + + # Set of inlined unspecialized modules names to generate the + # dynamo_flat_name_to_original_fqn mapping. + self.used_inlined_inbuilt_modules_names: OrderedSet[str] = OrderedSet() + + def mark_bytecode_tracing_start(self) -> None: + self.compiler_trace_stack.enter_context( + dynamo_timed( + "bytecode_tracing", + log_pt2_compile_event=True, + ) + ) + + def mark_bytecode_tracing_stop(self) -> None: + self.compiler_trace_stack.close() + + def install_builtins_dict_in_fglobals(self) -> str: + f_builtins = get_builtins_dict(self.global_scope) + return self.install_global("__builtins_dict__", f_builtins) + + def add_backward_state_hook( + self, hook: VariableTracker, prefix: str = "hook" + ) -> tuple[str, torch.fx.Proxy]: + name = f"{prefix}{len(self.backward_state)}" + assert name not in self.backward_state + self.backward_state[name] = hook + return name, self.get_backward_state_proxy() + + def get_backward_state_proxy(self) -> torch.fx.Proxy: + if self.backward_state_proxy is None: + if self.export: + unimplemented( + gb_type="backward_state does not support export", + context="", + explanation="Compiled autograd doesn't work with `torch.export`.", + hints=[], + ) + example_value = BackwardState() + self.backward_state_proxy = self.root_tracer.create_graph_input( + "dynamo_backward_state", + type(example_value), + example_value, + source=BackwardStateSource(), + ) + self.backward_state_proxy.node.meta["grapharg"] = BackwardStateGraphArg() + self.backward_state_var = self.new_var() + return self.backward_state_proxy + + # This gets its own helper function so guards DEBUG logs are more informative + def init_ambient_guards(self) -> None: + # Register a SHAPE_ENV guard to make sure we setup shape guards + # that show up in ShapeEnv + self.guards.add(ShapeEnvSource().make_guard(GuardBuilder.SHAPE_ENV)) + + self.guards.add( + GlobalStateSource().make_guard(GuardBuilder.DETERMINISTIC_ALGORITHMS) + ) + + self.guards.add(GlobalStateSource().make_guard(GuardBuilder.GRAD_MODE)) + + self.guards.add(GlobalStateSource().make_guard(GuardBuilder.DEFAULT_DEVICE)) + + self.guards.add(GlobalStateSource().make_guard(GuardBuilder.GLOBAL_STATE)) + self.guards.add( + GlobalStateSource().make_guard(GuardBuilder.TORCH_FUNCTION_STATE) + ) + + ci = torch._C._functorch.peek_interpreter_stack() + if ci is not None: + self.guards.add( + GlobalStateSource().make_guard(GuardBuilder.FUNCTORCH_STACK_MATCH) + ) + if not torch._dynamo.compiled_autograd.in_compiled_autograd_region: + self.guards.add( + GlobalStateSource().make_guard( + GuardBuilder.AUTOGRAD_SAVED_TENSORS_HOOKS + ) + ) + + def maybe_install_saved_tensors_hooks_subgraphs(self) -> Optional[list[str]]: + if torch._dynamo.compiled_autograd.in_compiled_autograd_region: + return None + + get_hooks = torch._functorch._aot_autograd.utils.top_saved_tensors_hooks + are_inline_hooks = ( + torch._functorch._aot_autograd.utils.saved_tensors_hooks_are_inlineable + ) + hooks = get_hooks() + if not are_inline_hooks(hooks): + return None + + # If GraphModule provided by user contains fx.wrap, + # We can only rely on user provided cache hash in this case. + # If user did not provide cache hash - then we always bypass cache. + + pack_gm, unpack_gm = hooks + pack_subgraph_name = self.install_subgraph( + "saved_tensors_hooks_pack", + torch.fx.GraphModule(self.nn_modules, pack_gm.graph), + ) + unpack_subgraph_name = self.install_subgraph( + "saved_tensors_hooks_unpack", + torch.fx.GraphModule(self.nn_modules, unpack_gm.graph), + ) + assert pack_subgraph_name == "saved_tensors_hooks_pack_0" + assert unpack_subgraph_name == "saved_tensors_hooks_unpack_0" + return [pack_subgraph_name, unpack_subgraph_name] + + def synthetic_graph_input( + self, fn: Callable[..., Any], args: tuple[Any, ...] + ) -> VariableTracker: + """ + call fn(*args) before the graph runs and turn the result into a fake input. + """ + example_value = fn(*args) + varname = self.new_var() + cg = PyCodegen(self.root_tx) + cg.add_push_null( + lambda: cg.load_import_from( + fn.__module__, + fn.__name__, + ) + ) + cg.foreach(map(variables.ConstantVariable.create, args)) + cg.call_function(len(args), False) + cg.store(varname) + self.pregraph_bytecode.extend(cg.get_instructions()) + source = SyntheticLocalSource(varname) + result = VariableTracker.build(self.root_tx, example_value, source) + # Realize the VT because we will delete the guards on it in the next line. + result = result.realize() + TracingContext.get().guards_context.dynamo_guards.remove_guards_with_source( + source + ) + return result + + def add_cleanup_hook(self, fn: Callable[[], Any]) -> None: + self.cleanup_hooks.append(fn) + + def call_cleanup_hooks(self) -> None: + for hook in reversed(self.cleanup_hooks): + hook() + self.cleanup_hooks.clear() + + @property + def root_tracer(self) -> "SubgraphTracer": + return self.tracers[0] + + @property + def current_tracer(self) -> "SubgraphTracer": + return self.tracers[-1] + + def is_root_tracer(self) -> bool: + # Helper to tell if we are inside the higher order operator tracing. + return len(self.tracers) == 1 + + @property + def graph(self) -> torch.fx.Graph: + return self.current_tracer.graph + + # TODO(rzou): can delete after we refactor speculate_subgraph to use nested GraphTracer. + @graph.setter + def graph(self, value: torch.fx.Graph) -> None: + self.current_tracer.graph = value + + @property + def input_name_to_proxy(self) -> dict[str, fx.Proxy]: + return self.current_tracer.input_name_to_proxy + + @property + def real_value_cache(self) -> dict[fx.Node, torch.Tensor]: + return self.current_tracer.real_value_cache + + @property + def bound_symbols(self) -> dict[sympy.Symbol, Union[torch.fx.Proxy, "LazyProxy"]]: + return self.current_tracer.bound_symbols + + # If you are here, and you're looking for create_graph_input, + # to avoid ambiguity, please call one of the following: + # - self.current_tracer.create_graph_input + # - self.root_tracer.create_graph_input + # See NOTE [HigherOrderOperator tracing design] for more context. + + def create_proxy(self, *args: Any, **kwargs: Any) -> torch.fx.Proxy: + return self.current_tracer.create_proxy(*args, **kwargs) + + def create_node(self, *args: Any, **kwargs: Any) -> torch.fx.Node: + return self.current_tracer.create_node(*args, **kwargs) + + def remove_node(self, *args: Any, **kwargs: Any) -> None: + return self.current_tracer.remove_node(*args, **kwargs) + + @contextlib.contextmanager + def subtracer( + self, + source_target: Optional[Target], + prior_tracer: "SubgraphTracer", + description: Optional[str] = None, + ) -> Generator[fx.Tracer, None, None]: + new_scope_ctx = enter_new_scope() + try: + if prior_tracer: + # Lineage MUST stay preserved + assert prior_tracer.parent is self.current_tracer + new_scope_ctx.__enter__() + tracer = ( + prior_tracer + if prior_tracer + else SubgraphTracer( + self, + parent=self.current_tracer, + source_target=source_target, + is_export=self.current_tracer.is_export, + description=description, + ) + ) + self.tracers.append(tracer) + yield tracer + finally: + new_scope_ctx.__exit__(None, None, None) + self.tracers.pop() + + @property + def output(self) -> "OutputGraph": + return self + + @property + def fake_mode(self) -> torch._subclasses.FakeTensorMode: + assert self.tracing_context.fake_mode is not None + return self.tracing_context.fake_mode + + @property + def shape_env(self) -> ShapeEnv: + assert self.tracing_context.fake_mode is not None + assert self.tracing_context.fake_mode.shape_env is not None + return self.tracing_context.fake_mode.shape_env + + @property + def guards(self) -> torch._guards.GuardsSet: + return self.tracing_context.guards_context.dynamo_guards + + @property + def nn_modules(self) -> dict[str, Any]: + return self.tracing_context.module_context.nn_modules + + @property + def aotautograd_guards(self) -> list[torch._guards.GuardEnvExpr]: + return self.tracing_context.guards_context.aotautograd_guards + + def save_global_state( + self, out: Optional[dict[str, tuple[Callable[..., Any], bool]]] = None + ) -> None: + """ + Saves to out if it is provided. Else saves to the tracing context's global_state. + """ + global_state = cast( + dict[str, tuple[Callable[..., Any], bool]], + ( + out + if out is not None + else self.tracing_context.global_context.global_state + ), + ) + + global_state["grad_enabled"] = (torch.set_grad_enabled, torch.is_grad_enabled()) + + global_state["autocast_enabled"] = ( + functools.partial(torch.set_autocast_enabled, "cuda"), + torch.is_autocast_enabled("cuda"), + ) + global_state["autocast_cpu_enabled"] = ( + functools.partial(torch.set_autocast_enabled, "cpu"), + torch.is_autocast_enabled("cpu"), + ) + global_state["autocast_gpu_dtype"] = ( # type:ignore[assignment] + functools.partial(torch.set_autocast_dtype, "cuda"), + torch.get_autocast_dtype("cuda"), + ) + global_state["autocast_cpu_dtype"] = ( # type:ignore[assignment] + functools.partial(torch.set_autocast_dtype, "cpu"), + torch.get_autocast_dtype("cpu"), + ) + global_state["autocast_cache_enabled"] = ( + torch.set_autocast_cache_enabled, + torch.is_autocast_cache_enabled(), + ) + + def push_tx(self, tx: "InstructionTranslatorBase") -> None: + self._current_tx.append(tx) + + def pop_tx(self) -> "InstructionTranslatorBase": + return self._current_tx.pop() + + @property + def current_tx(self) -> "InstructionTranslatorBase": + return self.root_tx if not self._current_tx else self._current_tx[-1] + + def count_calls(self) -> int: + return count_calls(self.graph) + + def is_empty_graph(self) -> bool: + return len(list(self.graph.nodes)) == 0 + + def has_outputs(self) -> bool: + return len([x for x in self.graph.nodes if x.op == "output"]) > 0 + + def get_submodule(self, keys: str) -> Union[torch.nn.Module, Any]: + assert keys + obj: Union[torch.nn.Module, dict[str, torch.nn.Module]] = self.nn_modules + for k in keys.split("."): + if isinstance(obj, dict): + obj = obj[k] + else: + obj = getattr(obj, k) + return obj + + def new_var(self, name: str = "tmp") -> str: + existing = set(self.code_options["co_varnames"]) + # In common case, this will be O(1) + while True: + var = f"{name}_{next(self.unique_var_id)}" + if var not in existing: + self.code_options["co_varnames"] += (var,) + return var + + def update_co_names(self, name: str) -> None: + """Ensure self.code_options.co_names contains name""" + if name not in self.code_options["co_names"]: + self.code_options["co_names"] += (name,) + + @staticmethod + def module_key_name(*names: Any) -> str: + # create a new unique name + name = "_".join(map(str, names)) + # Strip _buffers[..]/_parameters[..]/_modules[..] names + name = re.sub( + r"\._(?:modules|parameters|buffers)\[(['\"])([^'\"\]]+)\1\]", r".\2", name + ) + # Replace getattr(a, b) with a.b + name = re.sub( + r"getattr\(\s*([^,]+?)\s*,\s*(['\"])([^'\"]+)\2\s*\)", r"\1.\3", name + ) + # Strip the guard lookup L/G access + name = re.sub(r"^[GL]\['?(.*?)'?\]$", r"\1", name) + # e.g. replace abc.xyz[123].qkv with abc.xyz_123.qkv + name = re.sub(r"\[(\d+)\]", r"_\g<1>", name) + # e.g. replace abc.xyz_123.qkv with abc_xyz_123_qkv + name = re.sub(r"[^a-zA-Z0-9]", "_", name) + + if not name or not name[0].isalpha(): + name = "sub" + name + + return name + + def register_static_attr_and_return_proxy( + self, attr_prefix: str, attr_value: Any + ) -> fx.Proxy: + # Check if the module already exists, if it does, return the already + # added proxy. This is important for executorch tests. + if isinstance(attr_value, torch.nn.Module): + for name, mod in self.nn_modules.items(): + if mod is attr_value: + proxy = self.create_proxy("get_attr", name, (), {}) + return proxy + + attr_name = get_unique_name_wrt(attr_prefix, self.nn_modules) + # TODO `nn_modules` has been historically overloaded to store a lot more + # than just nn module objects, fix that. + self.nn_modules[attr_name] = attr_value + proxy = self.create_proxy("get_attr", attr_name, (), {}) + set_example_value(proxy.node, attr_value) + return proxy + + def register_attr_or_module( + self, + target: Union[torch.nn.Module, torch.Tensor, Any], + *names: Any, + **options: Any, + ) -> VariableTracker: + if is_dynamic_nn_module(target, self.export): + # Instead of returning UnspecializedNNModuleVariable, call + # VariableTracker.build so that it is tracked for mutation. + return VariableTracker.build(self.current_tx, target, **options) + + options = dict(options) + assert "source" in options + source = options["source"] + assert not isinstance(source, ParamBufferSource) + + if isinstance(target, torch.Tensor): + tracer = self.current_tracer + if not self.is_root_tracer(): + # For higher order ops, we don't want to insert the get_attr in + # innermost graph. Instead, we want to raise the params/buffers + # as inputs to the higher-order graph, and register them as + # get_attrs in the root tracer. + + # Note that Dynamo will still call lift_tracked_freevar_to_input + # when these inputs are encountered for the inner graph. The + # only difference is what happens at the root tracer for + # nn.Parameters vs free inputs. The free inputs are registered + # as placeholders in the root graph, whereas the nn.Parameters + # are registered as get_attr nodes in the root graph. + tracer = self.root_tracer + + def wrap_name(module_key: str) -> VariableTracker: + assert self.param_name_to_source is not None + self.param_name_to_source[module_key] = source + + # Check if the attr has already been registered. This can happen + # when two different sources point to the same tensor. + assert self.root_tx is not None + if target in self.root_tx.output.side_effects: + return self.root_tx.output.side_effects[target] + + if get_static_address_type(target) == "guarded" and not isinstance( + source, NumpyTensorSource + ): + install_guard(source.make_guard(GuardBuilder.ID_MATCH)) + elif not is_constant_source(source): + install_guard(source.make_guard(GuardBuilder.TENSOR_MATCH)) + + vt = wrap_fx_proxy( + self.root_tx, + tracer.create_proxy("get_attr", module_key, (), {}), + example_value=target, + **options, + ) + + # Track the object so to avoid duplicate registration in case of + # different sources pointing to the same tensor object. + vt = self.root_tx.output.side_effects.track_object_existing(target, vt) + + assert "tensor_dict" not in vt.as_proxy().node.meta + # pyrefly: ignore [bad-argument-type] + vt.as_proxy().node.meta["tensor_dict"] = _extract_tensor_dict(target) + + return vt + + elif isinstance(target, torch.nn.Module): + assert isinstance(target, torch.nn.Module) + + if source: + install_guard(source.make_guard(GuardBuilder.NN_MODULE)) + + def wrap_name(module_key: str) -> VariableTracker: + # pyrefly: ignore [bad-argument-type] + return NNModuleVariable(type(target), module_key, target, **options) + + else: + # This is Dynamo created graph module, e.g., graph module coming + # from higher order ops. NNModuleVariable tracker can't be + # sourceless, so let's return a unspecializedNNModule variable + # tracker. + def wrap_name(module_key: str) -> VariableTracker: + # pyrefly: ignore[bad-argument-type] + return variables.UnspecializedNNModuleVariable(target, **options) + + elif isinstance(target, (torch.SymInt, torch.SymFloat)): + # HACKY CODE REGION BEGIN + # WE ARE PIGGYBACKING ON EXISTING INFRA TO REGISTER ATTRS + # This ultimately gets written to self.nn_modules, which is unfortunate + # Attrs that are tenors and symints and such need to be migrated to have their + # own storage + # alas, this is like this for now + + def wrap_name(module_key: str) -> VariableTracker: + return SymNodeVariable.create( + self, + self.create_proxy("get_attr", module_key, (), {}), + sym_num=target, + **options, + ) + + # HACKY CODE REGION END + else: + + def wrap_name(module_key: str) -> VariableTracker: + self.output.update_co_names(module_key) + self.global_scope[module_key] = target + return VariableTracker.build( + self, # type: ignore[arg-type] + target, + ConstantSource(source_name=module_key), + ) + + for k, v in self.nn_modules.items(): + if v is target: + # it already exists + return wrap_name(k) + + name = OutputGraph.module_key_name(*names) + name = get_unique_name_wrt(name, self.nn_modules, self.global_scope) + self.nn_modules[name] = target + if isinstance(target, torch.nn.Module): + + def register_leaf_name(leaf_name: str) -> None: + assert self.param_name_to_source is not None + new_source = ParamBufferSource(source, leaf_name) + new_name = f"{name}.{leaf_name}" + self.param_name_to_source[new_name] = new_source + if isinstance(source, LocalSource): + self.dynamo_flat_name_to_original_fqn[ + OutputGraph.module_key_name(new_source.name) + ] = leaf_name + + # annoying, but there are cases when we do not have parameters + # see test_nn_moduledict_contains + if hasattr(target, "_parameters"): + for leaf_name, _ in target.named_parameters(): + register_leaf_name(leaf_name) + if hasattr(target, "_buffers"): + for leaf_name, _ in target.named_buffers(): + register_leaf_name(leaf_name) + + return wrap_name(name) + + def handle_aliases_for_stolen_lists( + self, tx: "InstructionTranslatorBase" + ) -> tuple[list[Instruction], dict[Source, Source]]: + # If list inputs are stolen, but still needed after the function call, create aliases to keep them alive + maybe_gm = self.local_scope.get("self") + stolen_list_names = get_locals_to_steal(maybe_gm) + if not stolen_list_names: + return [], {} + + alias_insts = [] + needs_alias: dict[str, list[VariableTracker]] = {} + + queue = [ + *tx.stack, + *tx.symbolic_locals.values(), + *self.side_effects.store_attr_mutations.keys(), + ] + + while queue: + x = queue.pop() + if isinstance(x, BaseListVariable): + assert isinstance(x.items, list) + queue += x.items + continue + + if not ( + ( + x not in self.side_effects.store_attr_mutations + or isinstance(x.mutation_type, AttributeMutationExisting) + ) + and isinstance(x.source, GetItemSource) + and isinstance(x.source.base, LocalSource) + and x.source.base.local_name in stolen_list_names + ): + continue + + stolen_name = x.source.base.local_name + if stolen_name not in needs_alias: + needs_alias[stolen_name] = [] + needs_alias[stolen_name].append(x) + + visited = {} + overridden_sources: dict[Source, Source] = {} + for arg in self.graphargs: + if not ( + isinstance(arg._example, list) + and isinstance(arg.source, LocalSource) + and arg.source.local_name in needs_alias + ): + continue + + # arg is a list that will be cleared by the compiled function + list_name = arg.source.local_name + assert list_name in self.code_options["co_varnames"] + for x in needs_alias[list_name]: + # Skip if already handled. + if x.source in overridden_sources: + continue + + # A small codegen optimization because we might have different + # VariableTrackers that share the same source. + assert x.source is not None + list_idx = x.source.index # type: ignore[attr-defined] + if list_idx not in visited: + alias_name = self.new_var( + f"{list_name}_ref" + ) # self.new_var already adds unique id suffix + + visited[list_idx] = alias_name + # bytecode of `alias_name = list_name[list_idx]` + alias_insts.extend( + [ + create_instruction("LOAD_FAST", argval=list_name), + create_load_const(list_idx), + create_binary_subscr(), + create_instruction("STORE_FAST", argval=alias_name), + ] + ) + + # operate on alias, handled by suffix codegen + assert x.source is not None + old_source = x.source + overridden_sources[old_source] = LocalSource(visited[list_idx]) + + # NOTE: we need `overridden_sources` because (1) we want to codegen for + # these list items to use the new local source, but (2) we want to avoid + # updating `source` in place because that might break invariants in + # other parts of Dynamo like guards. + return alias_insts, overridden_sources + + def _get_stack_values_to_restore( + self, tx: "InstructionTranslatorBase", stack_pops: int + ) -> tuple[list[VariableTracker], StackLocalsMetadata]: + """ + Gets the stack + locals values belonging to tx that need to be restored. + + Also prunes dead tx locals and realizes all VTs in the tx's stack. + + NullVariables in stack/locals will NOT be restored, unless they are the top `stack_pops` + elements of the stack - it is expected that the next instruction to run will pop the top + `stack_pops` elements of the stack, so we should codegen NULLs. + + Returns: + - stack_values: stack and locals values that need to be restored + - meta: locations of NULLs and ContextWrappingVariables in the stack/locals + (ignores the top `stack_pops` values on the stack) + """ + tx.prune_dead_locals() + + stack_values = [] + meta = StackLocalsMetadata() + + # realize any unrealized tensor VTs in case they + # need to be added to self.nn_modules as attributes + for i, value in enumerate(tx.stack): + variables.LazyVariableTracker.realize_all(value) + # ignore top `stack_pops` values on the stack + if len(tx.stack) - i <= stack_pops: + stack_values.append(value) + continue + if isinstance(value, NullVariable): + meta.stack_null_idxes.append(i) + else: + stack_values.append(value) + if isinstance(value, ContextWrappingVariable): + target_values = ( + () if value.target_values is None else tuple(value.target_values) + ) + # NOTE: track index in stack after NULLs have been removed + meta.stack_ctx_args.append((len(stack_values) - 1, target_values)) + meta.stack_ctx_idxes_orig.append(i) + + meta.num_stack = len(stack_values) + + cell_and_freevars = set(tx.cellvars() + tx.freevars()) + + # NB: Typically (i.e., for graph compile from RETURN_VALUE), + # symbolic_locals will be empty at this point, as prune_dead_locals + # will clear out all of symbolic_locals because RETURN_VALUE is the + # last instruction and no more locals are used. The fanciness here + # is only needed for partial graphs. + # NOTE: All cell and free variables are represented as CellVariable, + # so checks for NULLs and context managers in the case of codegen'ing resume + # functions will not be performed on them. This is expected behavior. + for k, v in tx.symbolic_locals.items(): + # Note! this explicitly uses .local_name for matching + # Failure to do so will cause spurious registrations in val_to_names. + # This will in turn result in spurious variables showing up in the graph. + # This was very tricky to debug. For an example, dump the graph at call_user_compiler + # while running test_subgraphs.py + # Do not include top-frame unmodified locals here - otherwise, the compiled graph may + # erroneously include them as part of the return. We manually codegen them afterward. + if ( + isinstance(v.source, LocalSource) + and v.source.local_name == k + and tx is self.root_tx + ): + continue + # Do not load cell/free vars + if k in cell_and_freevars: + continue + # Do not load variable if it is NULL. + if sys.version_info >= (3, 12): + # NOTE: do not use isinstance, since it realizes lazy VT's + # Continuation function will load the NULL for v. + if type.__instancecheck__(NullVariable, v): + meta.locals_null_keys.append(k) + continue + else: + # A variable should never be NULL in < 3.12 + assert not type.__instancecheck__(NullVariable, v) + meta.locals_names[k] = len(meta.locals_names) + if isinstance(v, ContextWrappingVariable): + target_values = ( + () if v.target_values is None else tuple(v.target_values) + ) + meta.locals_ctx_args.append((k, target_values)) + stack_values.append(v) + + return stack_values, meta + + def compile_subgraph( + self, + tx: "InstructionTranslatorBase", + reason: GraphCompileReason, + partial_convert: bool = False, + stack_pops: int = 0, + ) -> list[StackLocalsMetadata]: + """ + Compiles the current subgraph, with inputs w.r.t. self.root_tx, and codegens: + - Call the compiled subgraph + - Apply side effects + - Codegen stack and locals + - Store the locals + + Python does not allow NULL to be an arg to a function, so we do not codegen NULLs on the stack, + unless the value is one of the top `stack_pops` values on the stack (these values are expected to be + popped immediately after this generated code. The prologue of the resume function is expected to restore + any dropped NULLs. + + Returns stack indices and locals keys where we dropped NULLs, and where we found inactive context manager objects. + """ + + assert self.root_tx is not None + + if not config.nested_graph_breaks: + # expect to only compile 1 frame + assert self.root_tx is tx + + # bytecode tracing has finished. Pop the context manager for dynamo_timed + self.mark_bytecode_tracing_stop() + + self.partial_convert = partial_convert + self.compile_subgraph_reason = reason + self.should_exit = True + + log.debug("COMPILING GRAPH due to %s", reason) + + # prefix instructions (Python 3.11+) + prefix_insts: list[Instruction] = [] + if sys.version_info >= (3, 11): + for inst in self.root_tx.prefix_insts: + if inst.opname == "COPY_FREE_VARS": + prefix_insts.append( + create_instruction( + "COPY_FREE_VARS", + arg=len(self.root_tx.code_options["co_freevars"]), + ) + ) + else: + prefix_insts.append(copy.copy(inst)) + + # stack values and restore vars for each frame are pushed in reverse order + # i.e. last element corresponds to root frame (1), + # first element corresponds to current frame (N) + all_stack_values = [] + all_stack_locals_metas = [] + cur_tx: Optional[InstructionTranslatorBase] = tx + while cur_tx is not None: + # this should have been checked by the caller + assert all(block.can_restore() for block in cur_tx.block_stack) + + stack_values, meta = self._get_stack_values_to_restore( + cur_tx, stack_pops if cur_tx is tx else 0 + ) + all_stack_values.append(stack_values) + all_stack_locals_metas.append(meta) + + # Exit from all context manager variables to make sure global state is restored + for block in reversed(cur_tx.block_stack): + block.exit(cur_tx, is_graph_break=reason.graph_break) + + cur_tx = cur_tx.parent + + # "Garbage collect the heap". + self.side_effects.prune_dead_object_new(tx) + + self.add_output_instructions(prefix_insts) + + assert not (self.pregraph_bytecode and self.export), ( + "export does not support pregraph_bytecode" + ) + self.add_output_instructions(self.pregraph_bytecode) + + alias_insts, overridden_sources = self.handle_aliases_for_stolen_lists( + self.root_tx + ) + self.add_output_instructions(alias_insts) + + self.cleanup_graph() + + # Use nn.Module "proxies" in the constructed GraphModule so that + # the resulting GM does not hold additional strong references to the original modules. + # This prevents a strong ref cycle where Dynamo created code holds on to references + # to modules that also have Dynamo code cache invalidation checks. + # When cache invalidation runs, the generated GM will be invalidated, which also deletes + # the proxies. + nn_modules_proxies = { + name: nn_module_proxy(mod) for name, mod in self.nn_modules.items() + } + root = FakeRootModule(nn_modules_proxies) + + from .decorators import disable + + # to handle random calls + if len(self.random_calls) > 0: + random_calls_instructions = [] + self.random_values_var = self.new_var("random_values") + rand_fn = disable( + _get_gen_rand_values_fn(self.random_calls), + reason="do not trace into Dynamo rng recovery function", + ) + rand_fn_name = self.install_global("__gen_rand_values", rand_fn) + codegen = PyCodegen( + self.root_tx, root, overridden_sources=overridden_sources + ) + random_calls_instructions.extend( + codegen.load_function_name(rand_fn_name, True) + ) + random_calls_instructions.extend(create_call_function(0, False)) + random_calls_instructions.append( + codegen.create_store(self.random_values_var), + ) + self.add_output_instructions(random_calls_instructions) + + # Codegen stack convention before the unsupported instruction + # NOTE: in these comment blocks, "locals" EXCLUDE free and cell vars. + # NOTE: stack/locals/cells must be codegen'd BEFORE the unsupported instruction, since the latter + # can arbitrarily mutate the former. + # [frame N cells, .., frame 1 cells], + # [ + # frame N locals, + # frame N-1 stack + locals, + # ..., + # frame 1 stack + locals, + # ], frame N stack + + # see symbolic_convert.py for + # codegen stack convention after the unsupported instruction + # NOTE: cells will be loaded into continuation functions directly by symbolic_convert + + # this determines the order that values are codegen'd to the stack + stack_values_flat = [val for vals in all_stack_values for val in vals] + stored_graph_output_var = False + graph_output_var = None + + # call compiled fx graph and codegen all values - stack and locals + if ( + self.root_tx is tx # single frame + and stack_values_flat + and all( + not isinstance( + v, + ( + UnspecializedPythonVariable, + NumpyNdarrayVariable, + TensorWithTFOverrideVariable, + ), + ) + and not (isinstance(v, SymNodeVariable) and v.python_type() is float) + for v in stack_values_flat + ) + and all(x.is_tensor() for x in stack_values_flat) + and len(set(stack_values_flat)) == len(stack_values_flat) + and self.side_effects.is_empty() + and not tx.debug_locals + and not self.backward_state + and not all_stack_locals_metas[-1].stack_null_idxes + and not all_stack_locals_metas[-1].locals_null_keys + ): + # optimization to generate better code in a common case + + # codegen cells + # no side effects, so no new cells created - no need to call side_effects.codegen_save_tempvars + cell_cg = PyCodegen(self.root_tx) + self.codegen_cells(tx, cell_cg) + self.add_output_instructions( + [ + # load in reverse since UNPACK_SEQUENCE will reverse + *self.compile_and_call_fx_graph( + tx, list(reversed(stack_values_flat)), root + ), + *cell_cg.get_instructions(), + *create_swap(2), + create_instruction("UNPACK_SEQUENCE", arg=len(stack_values_flat)), + ] + ) + # function output will be moved to the correct places below + else: + graph_output_var = self.new_var("graph_out") + # load stack values in a flat manner - we will codegen bytecode to place them correctly + # according to our convention above + pass1 = PyCodegen( + self.root_tx, + root, + graph_output_var, + overridden_sources=overridden_sources, + ) + self.codegen_suffix(tx, stack_values_flat, pass1) + + # Use `pass1.uses` to selectively cache multi-user variables into a + # temporary local source. This (a). speeds up loading VTs with long + # chained source, and (b). avoids redundantly saving single-user VT + # into a temporary local. + tempvars = {} # type: ignore[var-annotated] + for val, count in pass1.uses.items(): + # If it's already a local source, no need to cache it + if count > 1 and not istype(val, (SyntheticLocalSource, LocalSource)): + tempvars[val] = None + pass2 = PyCodegen( + self.root_tx, + root, + graph_output_var, + tempvars=tempvars, + overridden_sources=overridden_sources, + ) + self.codegen_suffix(tx, stack_values_flat, pass2) + + if ( + torch._dynamo.config.log_graph_in_out_metadata + and stack_values_flat + and len(stack_values_flat) == 1 + ): + vt = stack_values_flat[0] + if ( + isinstance(vt, torch._dynamo.variables.NamedTupleVariable) + and vt.tuple_cls + is torch._dynamo.functional_export.ExportTracerOutput + ): + flat_returns = vt.items[0] + out_spec = vt.items[1] + assert isinstance( + flat_returns, torch._dynamo.variables.ListVariable + ) + + vt_to_graph_out_idx: dict[VariableTracker, int] = {} + for value in pass2.graph_outputs.values(): + assert isinstance(value, torch._dynamo.codegen.GraphOutputEntry) + variable: VariableTracker = value.variable + vt_to_graph_out_idx[variable] = value.index + + for idx, vt in enumerate(flat_returns.items): + if vt in vt_to_graph_out_idx: + self.export_metadata.output_return_type[idx] = ( + "graph_out", + vt_to_graph_out_idx[vt], + ) + elif ( + vt.source is not None + and (source := getattr(vt.source, "base", None)) # type: ignore[assignment] + and source.is_input + ): + self.export_metadata.output_return_type[idx] = ( + "input", + vt.source, + ) + elif vt.is_python_constant(): + self.export_metadata.output_return_type[idx] = ( + "constant", + vt.as_python_constant(), + ) + else: + assert f"Encountered unrecognized type {vt} at output {idx}" # noqa: PLW0129 + + self.export_metadata.out_spec = out_spec.as_python_constant() + + output = [] + if count_calls(self.graph) != 0 or len(pass2.graph_outputs) != 0: + output.extend( + self.compile_and_call_fx_graph(tx, pass2.graph_output_vars(), root) + ) + + if len(pass2.graph_outputs) != 0: + output.append(pass2.create_store(graph_output_var)) + stored_graph_output_var = True + else: + output.append(create_instruction("POP_TOP")) + else: + # NB: Important to run compiler collective even when there is + # a graph break + self.run_compiler_collective() + self.add_output_instructions(output + pass2.get_instructions()) + + # store all stack and locals for each frame + # current state of the stack: + # all cells, + # *(frame N stack), *(frame N locals), + # ..., + # *(frame 1 stack), *(frame 1 locals) + + self.add_output_instructions( + [ + create_instruction( + "BUILD_LIST", + arg=len(stack_values_flat) - all_stack_locals_metas[0].num_stack, + ), + ] + ) + + # current state of the stack: + # all cells, + # *(frame N stack), [ + # *(frame N locals), + # *(frame N-1 stack), *(frame N-1 locals), + # ... + # *(frame 1 stack), *(frame 1 locals), + # ] + # iterate current frame (N) to root frame (1) + # sliding window over frame stack/locals + start_idx = 0 + end_idx = 0 + for i, meta in enumerate(all_stack_locals_metas): + # do not pack frame N's stack into the value list + n_vals = len(meta.locals_names) + if i != 0: + n_vals += meta.num_stack + if n_vals == 0: + self.add_output_instructions( + [ + create_instruction("BUILD_LIST", arg=0), + *create_swap(2), + ] + ) + # [], stack_values_flat + else: + end_idx += n_vals + self.add_output_instructions( + [ + create_dup_top(), + *create_binary_slice(start_idx, end_idx), + *create_swap(2), + ] + ) + start_idx += n_vals + # stack_values_flat[x:y], stack_values_flat + + # add root frame's unmodified locals here + if i == len(all_stack_locals_metas) - 1: + root_cg = PyCodegen(self.root_tx) + unmodified_locals_names: dict[str, int] = {} + for k, v in self.root_tx.symbolic_locals.items(): + if isinstance(v.source, LocalSource) and v.source.local_name == k: + root_cg.append_output(root_cg.create_load(k)) + unmodified_locals_names[k] = len(meta.locals_names) + len( + unmodified_locals_names + ) + self.add_output_instructions( + root_cg.get_instructions() + + [ + create_instruction( + "BUILD_LIST", arg=len(unmodified_locals_names) + ), + # arg=2 because we already swapped the locals list back + create_instruction("LIST_EXTEND", arg=2), + ] + ) + meta.locals_names.update(unmodified_locals_names) + + # *(frame N stack), metas[0] stack + locals, ..., metas[i] stack + locals, stack_values_flat + + # current state of the stack: + # all cells, + # *(frame N stack), + # frame N locals, + # frame N-1 stack, frame N-1 locals, + # ... + # frame 1 stack, frame 1 locals, + # stack_values_flat + # + + self.add_output_instructions( + [ + create_instruction("POP_TOP"), + create_instruction("BUILD_LIST", arg=len(all_stack_locals_metas)), + *create_rot_n(all_stack_locals_metas[0].num_stack + 1), + ] + ) + + # final state of the stack before running the unsupported bytecode: + # all cells, + # [ + # [frame N locals], + # [frame N-1 stack + locals], + # ..., + # [frame 1 stack + locals], + # ], *(frame N stack) + + if graph_output_var and stored_graph_output_var: + self.add_output_instructions( + [create_instruction("DELETE_FAST", argval=graph_output_var)] + ) + + if torch._dynamo.config.side_effect_replay_policy in ["warn", "error"]: + from torch.export._trace import _ExportModuleSpecTrackerDict + + potential_side_effects = [] + for var in self.side_effects._get_modified_vars(): + if hasattr(var, "mutation_type"): + mut_type = var.mutation_type + # Make sure to skip codegen specific mutations + if isinstance( + mut_type, (AttributeMutationExisting, ValueMutationExisting) + ): + if isinstance(var, UserDefinedDictVariable) and isinstance( + var.value, _ExportModuleSpecTrackerDict + ): + for k, v in var.items.items(): + specs = {} + for k_spec, val in v.items.items(): + specs[k_spec.vt.as_python_constant()] = ( + val.as_python_constant() + ) + assert ["in_spec", "out_spec"] == list(specs.keys()) + self.export_metadata.module_call_spec[ + k.vt.as_python_constant() + ] = specs + # export uses tracepoint pass to dump submodule inp/out spec + # into global state, so we filter it here + if not ( + isinstance(var, UserDefinedDictVariable) + and isinstance(var.value, _ExportModuleSpecTrackerDict) + ): + potential_side_effects.append(var) + side_effect_refs = [ + _get_source_debug_name(var.source) for var in potential_side_effects + ] + + if side_effect_refs: + if torch._dynamo.config.side_effect_replay_policy == "warn": + warnings.warn( + f"While compiling, we found certain side effects happened in the model.forward. " + f"Here are the list of potential sources you can double check: {side_effect_refs}" + ) + else: + raise RuntimeError( + f"While compiling, we found certain side effects happened in the model.forward. " + f"Here are the list of potential sources you can double check: {side_effect_refs}" + ) + + return all_stack_locals_metas + + def codegen_cells(self, tx: "InstructionTranslatorBase", cg: PyCodegen) -> None: + # no need to codegen if reason.graph_break is False (since we won't resume) + if self.compile_subgraph_reason.graph_break: + tx_cnt = 0 + cur_tx: Optional[InstructionTranslatorBase] = tx + while cur_tx is not None: + # NOTE: we generate cells in the same order as resume_execution.py: sorted freevars + cellvars + # Emitting `LOAD_FAST/LOAD_CLOSURE` with names in `co_freevars` + # requires that in the generated bytecode, these cells would keep + # their original local names, which we ensure via + # `CellVariable.local_name`. + freevars = tuple(sorted(cur_tx.cell_and_freevars())) + for cell in freevars: + if cur_tx is self.root_tx: # root frame + cg.append_output(cg.create_load_closure(cell)) + else: # nested frame + assert cur_tx.post_prune_cell_and_freevars + cg(cur_tx.post_prune_cell_and_freevars[cell]) + cg.append_output(create_build_tuple(len(freevars))) + cur_tx = cur_tx.parent + tx_cnt += 1 + cg.append_output(create_instruction("BUILD_LIST", arg=tx_cnt)) + else: + cg.append_output(create_instruction("BUILD_LIST", arg=0)) + + def codegen_suffix( + self, + tx: "InstructionTranslatorBase", + stack_values: list[VariableTracker], + cg: PyCodegen, + ) -> None: + # NOTE: `codegen_save_tempvars` must run first to update `source` fields + # for variables with `AttributeMutationNew`, as they don't implement + # `reconstruct` themselves. + self.side_effects.codegen_save_tempvars(cg) + if self.backward_state: + assert not self.export + for name, val in self.backward_state.items(): + cg(val) + assert self.backward_state_var is not None + cg.append_output(cg.create_load(self.backward_state_var)) + cg.store_attr(name) + if config.replay_side_effects: + self.side_effects.codegen_hooks(cg) + + # TODO get debug_locals working for nested graph breaks + # Return variables used for logging at the end + for debug_var, args in tx.debug_locals: + cg.add_push_null(lambda: cg(debug_var)) + for arg in args: + cg(arg) + cg.extend_output(create_call_function(len(args), False)) + cg.extend_output([create_instruction("POP_TOP")]) + + # codegen cells before we apply side effects + self.codegen_cells(tx, cg) + + cg.restore_stack(stack_values, value_from_source=not tx.export) + if config.replay_side_effects: + self.side_effects.codegen_update_mutated(cg) + + def cleanup_graph(self) -> None: + """ + Remove "creation_timestamp" from node meta + + Remove this pattern from the graph: + torch._C._set_grad_enabled(False) + torch._C._set_grad_enabled(True) + """ + assert self.should_exit + nodes = list(self.graph.nodes) + for node in nodes: + node.meta.pop("creation_timestamp", None) + + grad_enabled = torch.is_grad_enabled() + for node1, node2 in itertools.pairwise(nodes): + if ( + node1.target is torch._C._set_grad_enabled + and tuple(node1.args) == (not grad_enabled,) + and not node1._erased + ): + grad_enabled = node1.args[0] + if ( + node2.target is torch._C._set_grad_enabled + and tuple(node2.args) == (not grad_enabled,) + and not node2._erased + ): + grad_enabled = node2.args[0] + self.graph.erase_node(node1) + self.graph.erase_node(node2) + + def bypass_package(self, reason: str = "", **kwargs: Any) -> None: + """ + Do not save this output graph to the CompilePackage + """ + if not self.package: + return + if torch._dynamo.config.strict_precompile: + raise torch._dynamo.exc.PackageError( + "Detected a package bypass: %s", reason + ) + log.warning("Detected a package bypass: %s", reason) + torch._logging.trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "precompile_cache_bypass", + "encoding": "json", + }, + payload_fn=lambda: { + # precede with underscore so it always appear first in JSON in tlparse + "_reason": reason, + **kwargs, + }, + ) + self.package.bypass_current_entry() + self.package = None + + def get_graph_sizes_structured(self) -> dict[str, list[Union[int, str]]]: + ret: dict[str, list[Union[int, str]]] = {} + for node in self.graph.nodes: + example_value = node.meta.get("example_value", None) + if isinstance(example_value, torch._subclasses.FakeTensor): + size = example_value.size() + ret[node.name] = [s if isinstance(s, int) else repr(s) for s in size] + return ret + + def get_graph_sizes(self, name: str) -> str: + graph_sizes_str = "TRACED GRAPH TENSOR SIZES\n" + graph_sizes_str += f"===== {name} =====\n" + for node in self.graph.nodes: + example_value = node.meta.get("example_value", None) + if isinstance(example_value, torch._subclasses.FakeTensor): + size = example_value.size() + graph_sizes_str += f"{node.name}: {tuple(size)}\n" + concrete_size = [] + has_symint = False + for sz in size: + if isinstance(sz, int): + concrete_size.append(sz) + elif isinstance(sz, torch.SymInt): + has_symint = True + concrete_size.append(sz.node.hint) + else: + break + else: + if has_symint: + graph_sizes_str += ( + f"{node.name} (concrete): {tuple(concrete_size)}\n" + ) + return graph_sizes_str + + @contextlib.contextmanager + def restore_global_state(self) -> Any: + """ + Momentarily restores the global state to what it was prior to tracing the current output + """ + prior_global_state = self.tracing_context.global_context.copy_graphstate() + current_global_state: dict[str, tuple[Any, bool]] = {} + self.save_global_state(out=current_global_state) + try: + # Set to state prior to tracing the graph + self.tracing_context.global_context.restore_graphstate(prior_global_state) + yield + finally: + # Reset to state at the current time (e.g. before calling the user compiler) + self.tracing_context.global_context.restore_graphstate( + GlobalContextCheckpointState(current_global_state) + ) + + def run_compiler_collective(self) -> None: + tx = self.root_tx + assert tx is not None + if (ds := tx.distributed_state) is not None and ds.all_states is None: + compile_pg = ds.compile_pg + + log.info("compiler_collective %s", ds.local_state) + torch._logging.trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "compiler_collective", + "encoding": "string", + }, + payload_fn=lambda: ds.local_state.render(), + ) + device_types = compile_pg._device_types + assert len(device_types) == 1, ( + "Expect only one device type but got {}".format("+".join(device_types)) + ) + with ( + get_interface_for_device(device_types.pop()).device( # type: ignore[attr-defined] + compile_pg.rank() % torch.accelerator.device_count() + ), + dynamo_timed("compiler_collective", log_pt2_compile_event=True), + ): + all_states: list[Any] = [None] * compile_pg.size() + + dist.all_gather_object(all_states, ds.local_state, group=compile_pg) + + ds.all_states = all_states + # Clear speculation log, because are tracing may diverge due to + # this information from the compiler collective + tx.speculation_log.clear() + raise exc.CompileCollectiveRestartAnalysis + + def compile_and_call_fx_graph( + self, + tx: "InstructionTranslatorBase", + rv: list[VariableTracker], + root: FakeRootModule, + ) -> list[Instruction]: + """ + Generate code from self.graph and return the Instruction()s to + call that generated code. + + Code is generated w.r.t. self.root_tx. + tx is only used for preserving GraphModule metadata + """ + with torch._guards.TracingContext.clear_frame(): + from .decorators import disable + + assert self.should_exit + + self.run_compiler_collective() + if count_calls(self.graph) == 0 and len(rv) == 0: + return [] + + name = unique_id("__compiled_fn", with_uuid=True) + + assert isinstance(rv, list) + assert isinstance(root, FakeRootModule) + + output_node = self.create_node( + "output", + "output", + (self.current_tracer.create_arg(tuple(x.as_proxy() for x in rv)),), + {}, + ) + sub_gms = self.dedup_pass() + root.add_nn_modules(sub_gms) # type: ignore[arg-type] + + self.current_tracer._maybe_preserve_original_meta(tx, output_node) + if not config.do_not_emit_runtime_asserts: + # There is a rare scenario where codegen_suffix adds a new entry + # to self.nn_modules while `root` knows only about the + # nn_modules at the time of its creation. This causes failures + # while creating the graph module because self.graph and root + # are out of sync. This only happens for `get_attr` nodes, so + # here we clean up the get_attr nodes that are unused. + for attr in dir(root): + subgraph = getattr(root, attr) + if isinstance(subgraph, fx.GraphModule): + insert_deferred_runtime_asserts( + subgraph, + self.shape_env, + name, + export=self.export, + ) + self.remove_unused_get_attr_nodes() + insert_deferred_runtime_asserts( + fx.GraphModule(root, self.graph), + self.shape_env, + name, + export=self.export, + ) + # NB: deferred runtime asserts can keep graphargs live, so make sure + # those are inserted before pruning + self.remove_unused_graphargs() + ncalls = count_calls(self.graph) + counters["stats"]["calls_captured"] += ncalls + + self.remove_tensorify_specialized_graphargs() + + # free a bit of memory + self.real_value_cache.clear() + + gm = _make_graph_module(root, self.graph) + + from .dce_extra_outputs import dce_hop_extra_outputs + + dce_hop_extra_outputs(gm) + + # Saved tensors hooks are not used by the graph. + # GraphModule by default only copies used in the graph submodules. + # Copying them into the result graph manually. + if self.saved_tensors_hooks_subgraph_names: + for subgraph_name in self.saved_tensors_hooks_subgraph_names: + setattr(gm, subgraph_name, getattr(root, subgraph_name)) + + for register_finalizer in self.register_finalizer_fns: + register_finalizer(gm) + + if next(gm.parameters(), None) is not None: + # If dynamo produces a graph with parameters, skip package stuff + # Bypass output graph + self.bypass_package( + "Graph contains named parameters: either inline_inbuilt_nn_modules=False or there are static addresses.", + inline_builtin_nn_modules=torch._dynamo.config.inline_inbuilt_nn_modules, + gm=gm.print_readable( + print_output=False, include_stride=True, include_device=True + ), + ) + + if self.package is not None: + gm._backend_id = name + + gm.compile_subgraph_reason = self.compile_subgraph_reason + gm.meta["dynamo_flat_name_to_original_fqn"] = ( + self.dynamo_flat_name_to_original_fqn.copy() + ) + gm.meta["dynamo_compile_id"] = self.dynamo_compile_id + gm.meta["backend_id"] = name + + graph_code_log.debug( + "%s", + lazy_format_graph_code( + name, gm, include_stride=True, include_device=True, colored=True + ), + ) + torch._logging.trace_structured( + "dynamo_output_graph", + lambda: {"sizes": self.get_graph_sizes_structured()}, + payload_fn=lambda: gm.print_readable( + print_output=False, include_stride=True, include_device=True + ), + ) + self.call_cleanup_hooks() + old_fake_mode = self.tracing_context.fake_mode + assert old_fake_mode is not None + if not self.export: + import torch._functorch.config as _config + + with _config.patch(fake_tensor_allow_unsafe_data_ptr_access=False): + # TODO(voz): The way export uses gm, and fake tensors, is not supported with us resetting + + # Why create a new FakeTensorMode? + # + # The reason this needs to be done is because when we do Dynamo tracing, fake + # tensors can have their metadata mutated. Thus, the fake tensor we allocated + # for any given tensor may no longer be valid for the beginning trace of the + # graph. Nor is it convenient to "clone" the input tensors before mutating them, + # since you have to preserve aliasing. So we just reconstruct the FakeTensorMode + # from scratch when we go to AOTAutograd. But the ShapeEnv must be preserved as + # Dynamo made decisions about what is dynamic or not / guards from the user code + # that is not in graph. + backend_fake_mode = torch._subclasses.FakeTensorMode( + shape_env=old_fake_mode.shape_env, + ) + # TODO(voz): Ostensibly, this should be scoped and + # restore back to old_fake_mode, but doing so currently violates + # a lot of fake_tensor ownership assumptions and runs afoul of detect_fake_mode + self.tracing_context.fake_mode = backend_fake_mode + + with self.restore_global_state(): + compiled_fn = self.call_user_compiler(gm, self.example_inputs()) + + from torch.fx._lazy_graph_module import _LazyGraphModule + + if isinstance(compiled_fn, _LazyGraphModule) or ( + isinstance(getattr(compiled_fn, "__self__", None), _LazyGraphModule) + and compiled_fn.__name__ == "_lazy_forward" # type: ignore[attr-defined] + ): + # Since dynamo will run the forward method for the GraphModule shortly + # anyways, it does not hurt to do the real recompilation here if + # this is a _LazyGraphModule. This makes it easier for dynamo to + # optimize a _LazyGraphModule. + + lazy_gm = ( + compiled_fn + if isinstance(compiled_fn, _LazyGraphModule) + else compiled_fn.__self__ # type: ignore[attr-defined] + ) + + _LazyGraphModule.force_recompile(lazy_gm) + + if not isinstance(compiled_fn, _LazyGraphModule): + # replace compiled_fn with the real forward method + compiled_fn = lazy_gm.forward + + if self.package is not None: + self.package.add_backend_id(name, compiled_fn) + + compiled_fn = disable( + compiled_fn, reason="do not trace Dynamo-compiled graph" + ) + + counters["stats"]["unique_graphs"] += 1 + assert old_fake_mode.shape_env is not None + if specializations := old_fake_mode.shape_env.specializations: + specialization_guards = [] + specialization_cache: dict[Specialization, Callable[[Any], Any]] = {} + sources = [a.source for a in self.graphargs] + for specialization in specializations: + source_index = sources.index(specialization.source) + check_fn_source = inspect.getsource(specialization.check_fn).strip() + # Required because the LABDA_GUARD API requires a root guard manager + unused_root_guard_manager = RootGuardManager() + check_fn = guards.LAMBDA_GUARD( # type: ignore[attr-defined] + unused_root_guard_manager, + specialization.check_fn, + [check_fn_source], + ) + + log.debug( + "Compiling backend specialized graph with specialization=%s", + check_fn_source, + ) + + specialization_guards.append( + ( + functools.partial( + lambda idx, args, check_fn=check_fn: check_fn( + args[idx] + ), + source_index, + ), + specialization, + ) + ) + + @torch._dynamo.disable(reason="do not trace Dynamo-compiled graph") # type: ignore[misc] + def specialized_dispatch(*args: Any, **kwargs: Any) -> Any: + for check_fn, specialization in specialization_guards: + if check_fn(args): + if specialization in specialization_cache: + return specialization_cache[specialization]( + *args, **kwargs + ) + + with self.shape_env.patch_source_specialization( + specialization.source, specialization.check_fn + ): + # Modify gm so AOTAutogradCache key changes per specialization + gm.meta["specialization"] = specialization + example_inputs: list[Tensor] = list(args) + with tracing(self.tracing_context): + specialization_cache[specialization] = ( + self.call_user_compiler(gm, example_inputs) + ) + + return specialization_cache[specialization](*args, **kwargs) + return compiled_fn(*args, **kwargs) + + # This is safe because we pre-process name to be unique + self.install_global_unsafe(name, specialized_dispatch) + else: + # This is safe because we pre-process name to be unique + self.install_global_unsafe(name, compiled_fn) + + assert self.root_tx is not None + cg = PyCodegen(self.root_tx) + + if has_user_objects(): + # NB: This is where we store possible user objects before running the graph + # index_to_user_object_weakref is the function used in the graph to translate + # the dynamo-generated index into the actual object passed to the compiled function. + # We generate bytecode to store all user objects at the proper index in the below + # call. + cg.add_push_null( + lambda: cg.load_import_from( + torch._dynamo.graph_bytecode_inputs.__name__, + "store_user_object_weakrefs", + ) + ) + + tmp_vars = [] + for constructor in index_to_bytecode_constructor.values(): + constructor(cg) + var_name = ( + self.new_var() + ) # keep alive any user objects for the rest of the frame + # TODO: we could omit this for objects we create but shouldn't be too much overhead for now + cg.store(var_name) + tmp_vars.append(var_name) + + for var_name in tmp_vars: + cg.append_output(cg.create_load(var_name)) + + cg.call_function(len(index_to_bytecode_constructor), False) + cg.pop_top() + + for idx, arg in enumerate(self.graphargs): + self.export_metadata.graph_input_idx_to_local_source[idx] = arg.source + + cg.make_call_generated_code(name) + return cg.get_instructions() + + @property + def placeholders(self) -> list[fx.Node]: + return self.graph.find_nodes(op="placeholder") + + @property + def graphargs(self) -> list[GraphArg]: + return [node.meta["grapharg"] for node in self.placeholders] + + def call_user_compiler( + self, gm: fx.GraphModule, example_inputs: list[Tensor] + ) -> CompiledFn: + with dynamo_timed( + "OutputGraph.call_user_compiler", + phase_name="backend_compile", + log_pt2_compile_event=True, + log_waitcounter=True, + waitcounter_name_override="compile_aot_autograd", + dynamo_compile_column_us="aot_autograd_cumulative_compile_time_us", + ): + return self._call_user_compiler(gm, example_inputs) + + def _call_user_compiler( + self, gm: fx.GraphModule, example_inputs: list[Tensor] + ) -> CompiledFn: + assert self.compiler_fn is not None + tot = 0 + placeholders = [] + for node in gm.graph.nodes: + if node.op in ("call_function", "call_method", "call_module"): + tot += 1 + if node.op == "placeholder": + placeholders.append(node) + increment_op_count(tot) + for pl in placeholders: + if not hasattr(pl, "_dynamo_source"): + arg = pl.meta["grapharg"] + # TODO: Why isn't this stored in meta :think: + # NOTE: can't move these into meta: https://github.com/pytorch/pytorch/issues/141640 + pl._dynamo_source = arg.source + + # NOTE: can't move these into meta: https://github.com/pytorch/pytorch/issues/141640 + gm._param_name_to_source = self.param_name_to_source # type: ignore[assignment] + gm._source_to_user_stacks = self.source_to_user_stacks # type: ignore[assignment] + + name = ( + self.compiler_fn.__name__ + if hasattr(self.compiler_fn, "__name__") + else "" + ) + try: + _step_logger()(logging.INFO, f"calling compiler function {name}") + compiler_fn = self.compiler_fn + if config.verify_correctness: + compiler_fn = WrapperBackend(compiler_fn) + compiled_fn = compiler_fn(gm, example_inputs) + _step_logger()(logging.INFO, f"done compiler function {name}") + assert callable(compiled_fn), "compiler_fn did not return callable" + except (TensorifyScalarRestartAnalysis, ShortenTraceback): + raise + except exceptions_allowed_to_be_fallback as e: + if self.has_user_defined_allowed_in_graph: + raise BackendCompilerFailed( + self.compiler_fn, e, inspect.currentframe() + ).with_traceback(e.__traceback__) from None + unimplemented_with_warning( + e, + self.root_tx.f_code, + gb_type="Backend compiler exception", + context=f"Backend: {name}\nException:{str(e)}\nTraceback:\n{self.root_tx.format_frame_summary()}", + explanation=f"Backend compiler `{name}` failed with {str(e)}. Adding a graph break.", + hints=[ + "Report an issue to the backend compiler repo.", + ], + ) + except SkipFrame as e: + # The backend compiler has requested that we skip the frame, instead of + # aborting execution. + raise e + except Exception as e: + raise BackendCompilerFailed( + self.compiler_fn, e, inspect.currentframe() + ).with_traceback(e.__traceback__) from None + + signpost_event( + "dynamo", + "OutputGraph.call_user_compiler", + { + **self.co_fields, + "op_count": tot, + "node_count": len(gm.graph.nodes), + "input_count": len(placeholders), + }, + ) + + # pyrefly: ignore [unbound-name] + return compiled_fn + + def dedup_pass(self) -> dict[str, torch.fx.GraphModule]: + if torch._dynamo.config.use_graph_deduplication: + return apply_graph_deduplication(self) + else: + return {} + + def install_subgraph(self, name: str, sub_gm: torch.fx.GraphModule) -> str: + next_name = get_unique_name_wrt(name, self.nn_modules, requires_suffix=True) + sub_gm.__name__ = next_name # type: ignore[assignment] + sub_gm.torchdynamo_force_dynamic = False # type: ignore[assignment] + # This graph module is not present in the user space, so it can't be + # accessed by a source. Set source=None. + self.register_attr_or_module(sub_gm, next_name, source=None) + return next_name + + def example_inputs(self) -> list[torch.Tensor]: + result = [arg.example for arg in self.graphargs] + return result + + def remove_unused_get_attr_nodes(self) -> None: + for node in sorted(self.graph.find_nodes(op="get_attr"), reverse=True): + if len(list(node.users)) == 0: + self.remove_node(node) + + def remove_unused_graphargs(self) -> None: + # NB: It's OK to drop GraphArg for symbols that ended up being + # specialized iff they are not used in runtime assertions. You don't + # even have to make a guard for it, because ShapeEnv produce_guards + # operates on tracked_fakes, which never gets pruned. + # That being said, you'll get marginally better generated + # guard code if you promote the guard into a Dynamo guard (since that + # allows for the guard to be done using C++ guards.) If we get + # ShapeEnv guards to go into C++ guards, this will stop being a thing + # though! + + assert self.should_exit + + # Miniature DCE pass, but only for obviously trivial operations + def is_static_true(b_node: fx.node.Argument) -> bool: + if b_node is True: + return True + if not isinstance(b_node, fx.Node): + return False + b = b_node.meta.get("example_value") + if b is None: + return False + if b is True: + return True + if ( + isinstance(b, torch.SymBool) + and (r := b.node.maybe_as_bool()) is not None + ): + return r + # TODO: We can also technically remove all cases when the input + # doesn't have unbacked inputs, since it's all in the ShapeEnv + return False + + def is_symnode_arg(a: fx.node.Argument) -> bool: + from torch.fx.experimental.sym_node import SymTypes + + if isinstance(a, (int, float, bool)): + return True + if isinstance(a, fx.Node): + return isinstance(a.meta.get("example_value"), SymTypes) + return False + + # NB: We assume that you cannot do mutations on int/float/bool, + # because they are immutable types, and therefore is always safe to + # DCE. + def is_symnode_compute_node(node: fx.Node) -> bool: + from torch.fx.experimental.sym_node import SymTypes + + if node.op != "call_function": + return False + # TODO: I don't think it's possible to have a bare int/float here? + if not isinstance(node.meta.get("example_value"), SymTypes): + return False + # TODO: This will bail here if you ever end up with a more complicated + # computation function, like sum(list_of_ints), even though it + # should be DCE'able + if not all(is_symnode_arg(a) for a in node.args): + return False + if not all(is_symnode_arg(a) for a in node.kwargs.values()): + return False + return True + + from torch.fx.experimental.symbolic_shapes import is_accessor_node + + for node in reversed(list(self.graph.nodes)): + if len(list(node.users)) == 0: + if ( + node.op == "get_attr" + or (node.op == "call_function" and node.target is operator.getitem) + or ( + node.op == "call_function" + and node.target is torch._check + and is_static_true(node.args[0]) + ) + or is_symnode_compute_node(node) + or is_accessor_node(node) + ): + self.remove_node(node) + + def placeholder_binds_symbol(node: fx.Node) -> Optional[sympy.Symbol]: + arg = node.meta["grapharg"] + example = arg.example + if isinstance(example, torch.SymInt) and isinstance( + example.node.expr, sympy.Symbol + ): + return example.node.expr + return None + + def remove_unused(node: fx.Node) -> None: + log.debug("REMOVE UNUSED GRAPHARG %s", node.meta["grapharg"].source.name) + # I'm not really sure why you need to delete these from the + # node since the node is going to get removed + del node.meta["grapharg"] + self.remove_node(node) + self.real_value_cache.pop(node, None) + + used_symbols: set[sympy.Symbol] = set() + + def update_used_symbols( + used_symbols: set[sympy.Symbol], fake: Union[torch.SymInt, torch.Tensor] + ) -> None: + used_symbols |= free_symbols(fake) + + recheck_placeholders = [] + for node in self.placeholders: + binds_symbol = placeholder_binds_symbol(node) is not None + # Don't delete symbol bindings yet + if binds_symbol: + if not node.users: + recheck_placeholders.append(node) + else: + if not node.users and not isinstance( + node.meta["grapharg"], BackwardStateGraphArg + ): + remove_unused(node) + else: + # Register the free symbols as uses + arg = node.meta["grapharg"] + if isinstance(arg, BackwardStateGraphArg): + continue + if isinstance(node.meta["grapharg"].example, torch.ScriptObject): + real_script_obj = node.meta["grapharg"].example + fake_script_obj = node.meta["grapharg"].example_strong_ref + if not torch._library.fake_class_registry.tracing_with_real( + real_script_obj + ): + flat_dict = dict(real_script_obj.__obj_flatten__()) # type: ignore[attr-defined] + for attr in flat_dict: + fake_attr_val = getattr( + fake_script_obj.wrapped_obj, attr + ) + pytree.tree_map_only( + (torch.SymInt, torch.Tensor), + lambda t: update_used_symbols(used_symbols, t), + fake_attr_val, + ) + continue + if is_opaque_type(type(node.meta["grapharg"].example)): + continue + fake = ( + arg.fake_tensor if arg.fake_tensor is not None else arg.example + ) + update_used_symbols(used_symbols, fake) + + # After removing unused graphargs, prune unused binds_symbol + for node in recheck_placeholders: + symbol = placeholder_binds_symbol(node) + if symbol is not None: + if symbol not in used_symbols: + remove_unused(node) + else: + # Make sure we delete later occurrences of the same symbol + used_symbols.remove(symbol) + + def remove_tensorify_specialized_graphargs(self) -> None: + # This is a pretty interesting function. Basically we have this problem + # where our compiler tends to choke when we have unused inputs. The way + # we support dynamic float arguments is by doing a joint fx pass and + # tensorifying away as many symfloats as we can. For the remaining symfloats + # we have no choice but to specialize... HOWEVER at that point in time + # we can no longer remove graph inputs. So our sledgehammer solution is to + # save the state of what inputs we should have specialized in dynamo and + # restart analysis. This function incorporates this "view from the future" + # state and specializes inputs that we know we won't be able to tensorify + # away in the joint pass. In principle we shouldn't choke on unused inputs + # and so this shouldn't be necessary. In practice CUDA graphs choke on + # unused inputs so we need this for now. + + # Import here to prevent circular import + from torch._dynamo.symbolic_convert import TensorifyState + + for node in self.graph.nodes: + example_value = node.meta.get("example_value") + if ( + isinstance(example_value, FakeTensor) + and example_value.item_memo is not None + and hasattr(example_value.item_memo.node._expr, "name") + and all(u.target == "item" for u in node.users) + and TensorifyState.should_specialize( + # We use _expr instead of expr b/c we want the symbol not the replacement + example_value.item_memo.node._expr.name + ) + ): + for u in list(node.users): + u.replace_all_uses_with(guard_scalar(example_value.item_memo)) + self.remove_node(u) + self.remove_node(node) + + def add_output_instructions(self, prefix: list[Instruction]) -> None: + """ + We call this on the creation of a new compiled subgraph that is inserted + before user code. + """ + self.output_instructions.extend(prefix) + self.should_exit = True + + def install_global_unsafe(self, name: str, value: Any) -> None: + """ + WARNING: prefer the safer `install_global_by_id/install_global`. + torch.compile instances should be independent of each other; + one footgun is to have one instance depend on the existence of + a global installed by another instance. This can happen if we mangle + a global the same way across both instances. + """ + assert name not in self.installed_globals + self.installed_globals.add(name) + self.cleanups.append(CleanupHook.create(self.global_scope, name, value)) + + def install_global_by_id(self, prefix: str, value: Any) -> str: + """ + Installs a global if it hasn't been installed already. + This is determined by (prefix, id(value)) pair. + + Returns the name of the newly installed global. + """ + # NB: need self.compile_id to distinguish this global + # from another global created in a different torch.compile instance + name = f"{prefix}_{id(value)}_c{self.compile_id}" + if name in self.installed_globals: + return name + self.install_global_unsafe(name, value) + return name + + def install_global(self, prefix: str, value: Any) -> str: + """ + Installs a global, generating a unique name for it. + + Returns the name of the newly installed global. + """ + # NB: unique_id is unique, even across torch.compile instances + name = unique_id(prefix) + self.install_global_unsafe(name, value) + return name + + def cleanup(self) -> None: + # There is a reference cycle between tracer and OutputGraph, causing + # some of the tensor objects to be held alive for longer than necessary. + self.root_tx = None # type: ignore[assignment] + self.nn_modules.clear() + self.used_inlined_inbuilt_modules_names.clear() + self.param_name_to_source = None + + for node in self.graph.nodes: + if "grapharg" in node.meta: + del node.meta["grapharg"] + self.real_value_cache.clear() + self.input_name_to_proxy.clear() + self.side_effects.clear() + self.variable_tracker_cache.clear() + self.register_finalizer_fns.clear() + self.dynamo_flat_name_to_original_fqn.clear() + self.tracing_context.clear() + self.input_source_to_var.clear() + self.unspec_variable_map.clear() + self.backward_state.clear() + + def add_graph_finalizer( + self, register_finalizer: Callable[[fx.GraphModule], None] + ) -> None: + self.register_finalizer_fns.append(register_finalizer) + + def example_value_from_input_node(self, node: torch.fx.Node) -> Any: + """Extract the non-fake example tensor""" + if node.op == "placeholder": + return node.meta["grapharg"].example + assert node.op == "get_attr" + return self.nn_modules[node.target] # type: ignore[index] + + def add_fqn_info_for_inlined_modules( + self, inlined_module: torch.nn.Module, source: Source + ) -> None: + name = OutputGraph.module_key_name(source.name) + name = get_unique_name_wrt( + name, self.used_inlined_inbuilt_modules_names, self.global_scope + ) + self.used_inlined_inbuilt_modules_names.add(name) + + def register_leaf_name(leaf_name: str) -> None: + assert self.param_name_to_source is not None + new_source = ParamBufferSource(source, leaf_name) + new_name = f"{name}.{leaf_name}" + self.param_name_to_source[new_name] = new_source + if isinstance(source, LocalSource): + self.dynamo_flat_name_to_original_fqn[ + OutputGraph.module_key_name(new_source.name) + ] = leaf_name + + # annoying, but there are cases when we do not have parameters + # see test_nn_moduledict_contains + if hasattr(inlined_module, "_parameters"): + if ( + callable(inlined_module.named_parameters) + and inlined_module.named_parameters.__func__ # type: ignore[attr-defined] + is og_module_named_parameters_fn_ptr + ): + for leaf_name, _ in inlined_module.named_parameters(): + register_leaf_name(leaf_name) + if hasattr(inlined_module, "_buffers"): + if ( + callable(inlined_module.named_buffers) + and inlined_module.named_buffers.__func__ # type: ignore[attr-defined] + is og_module_named_buffers_fn_ptr + ): + for leaf_name, _ in inlined_module.named_buffers(): + register_leaf_name(leaf_name) + + +class DynamoTracerOutput: + error_on_graph_break: bool + is_tracing_resume_prologue: bool + output_graph: Optional[OutputGraph] + # output_graph_for_cleanup is set even when there's an error, to allow + # cleanup of graph nodes to break reference cycles + output_graph_for_cleanup: Optional[OutputGraph] + closure: Optional[tuple[Any, ...]] + f_globals: dict[str, Any] + + def __init__( + self, tracer: "InstructionTranslatorBase", error: Optional[Any] = None + ) -> None: + self.error_on_graph_break = tracer.error_on_graph_break + self.is_tracing_resume_prologue = tracer.is_tracing_resume_prologue + self.closure = tracer.closure + self.f_globals = tracer.f_globals + self.output_graph_for_cleanup = tracer.output + if error: + self.output_graph = None + else: + self.output_graph = tracer.output + + def _cleanup_output_graph(self) -> None: + output_graph = self.output_graph_for_cleanup + if output_graph: + for tracer in output_graph.tracers: + tracer.graph._clear_nodes() + # Also clear tracked_fakes to break FakeTensorMode → ShapeEnv → TrackedFake → FakeTensor cycle + if ( + output_graph.tracing_context.fake_mode + and output_graph.tracing_context.fake_mode.shape_env + ): + output_graph.tracing_context.fake_mode.shape_env.tracked_fakes = None + + +err_epilogue = ( + "With the current config, we will graph break " + "(and fall back to eager-mode PyTorch) on all ops " + "that have do not have the 'pt2_compliant_tag'. " + "Please see the following doc for how to mark this op as PT2 compliant " + "https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html" +) + + +def check_pt2_compliant_op( + output_graph: OutputGraph, kind: str, target: Any, args: Any, kwargs: Any +) -> None: + if kind != "call_function": + return + + def encountered_compliant_op(target: torch._ops.OpOverload) -> None: + if target.namespace in {"prim", "prims", "aten"}: + return + output_graph.compliant_custom_ops.add(target) + + def encountered_non_compliant_op(target: torch._ops.OpOverload, msg: str) -> None: + output_graph.non_compliant_ops.add(target) + if config.only_allow_pt2_compliant_ops: + unimplemented( + gb_type="Encountered non-PT2-compliant op", + context="", + explanation=msg + " " + err_epilogue, + hints=[], + ) + + if isinstance(target, torch._ops.OpOverload): + if torch.Tag.pt2_compliant_tag in target.tags: + encountered_compliant_op(target) + return + encountered_non_compliant_op( + target, + f"Encountered the torch.ops.OpOverload {target} that is not PT2 compliant.", + ) + return + + if isinstance(target, torch._ops.OpOverloadPacket): + overloads = tuple(target.overloads()) + # Optimization: Overload resolution is expensive. + # If there's only one overload, we know what it will resolve to. + if len(overloads) == 1: + op = getattr(target, overloads[0]) + if torch.Tag.pt2_compliant_tag in op.tags: + encountered_compliant_op(op) + return + encountered_non_compliant_op( + op, + f"Encountered the non-overloaded " + f"torch.ops.OpOverloadPacket {target} " + f"that is not PT2 compliant. ", + ) + return + + args, kwargs = torch._dynamo.utils.get_fake_values_from_nodes( + output_graph.current_tx, (args, kwargs), False + ) + try: + overload = torch._C._jit_resolve_packet( + target._qualified_op_name, *args, **kwargs + ) + except RuntimeError as e: + unimplemented( + gb_type="Error when attempting to resolve op packet", + context="", + explanation=str(e), + hints=[], + ) + + # pyrefly: ignore [unbound-name] + op = getattr(target, overload) + if torch.Tag.pt2_compliant_tag in op.tags: + encountered_compliant_op(op) + else: + encountered_non_compliant_op( + op, + f"Encountered the torch.ops.OpOverloadPacket {target} " + # pyrefly: ignore [unbound-name] + f"which resolves to the overload ({overload}) that is " + f"not PT2 compliant.", + ) + + +_compile_id_counter = itertools.count() + +P = ParamSpec("P") +R = TypeVar("R") + + +class LazyProxy: + def __init__( + self, + tracer: "SubgraphTracer", + fn: Callable[P, R], + *args: P.args, + **kwargs: P.kwargs, + ) -> None: + self.tracer = tracer + # pyrefly: ignore [invalid-type-var] + self.fn = fn + self.args = args + self.kwargs = kwargs + + def __call__(self) -> Any: + return self.fn(*self.args, **self.kwargs) + + +class SubgraphTracer(fx.Tracer): + """ + Holds an FX graph that is being traced. OutputGraph owns a SubgraphTracer + and the separation of responsibilities is that SubgraphTracer is + responsible for building the graph while OutputGraph is responsible for + compiling and executing the graph. + """ + + def __init__( + self, + output_graph: "OutputGraph", + parent: Optional["SubgraphTracer"] = None, + is_export: bool = False, + source_target: Optional[Target] = None, + description: Optional[str] = None, + ) -> None: + super().__init__() + self.output_graph = weakref.proxy(output_graph) + self.graph = torch.fx.Graph() + + # See note [Export inputs must be explicitly passed in] + self.is_export = is_export + # Map from graph input name to its placeholder proxy object, where the + # map's keys give all current placeholder node names and can be used to + # create unique node names + self.input_name_to_proxy: dict[str, fx.Proxy] = {} + # Node => computed real value (see utils.get_real_value) + self.real_value_cache: dict[fx.Node, torch.Tensor] = {} + + # SubgraphTracers can be nested. See NOTE [HigherOrderOperator tracing design] + self.parent = parent + self.source_target = source_target + self.description = description + # A dict mapping previously free variables (Proxy objects) + # to new Proxy objects that wrap inputs to this subgraph. + # + # This dict maps proxies in outer graphs to placeholders in current graph. + # It serves two purposes: + # - Proxies are associated with VariableTrackers. If we see + # the same VariableTracker twice (and it is a free variable), + # then we want to use the same Proxy in the current subgraph to + # record the tracing. + # - If we are tracing a HigherOrderOperator's body_fn, then we + # need to keep track of what free variables were lifted so we can + # rewrite the HigherOrderOperator call using the traced body_fn. + # Dicts maintain the order of args for the HigherOrderOperator call. + self.lifted_freevars: dict[fx.Proxy, fx.Proxy] = {} + + # map basic symbols (unbacked and unbacked) to their bound proxies. + # There are only two cases where bound_symbols will be recorded: + # 1. when we create_graph_input for a backed SymInt that's basic symbol + # 2. when we track_produced_symints for intermediate results + # bound_symbols always map the symbol to the proxy whose + # tracer is the current tracer that's readily accessible in current tracer's graph. + self.bound_symbols: dict[sympy.Symbol, Union[torch.fx.Proxy, LazyProxy]] = {} + + # Maps _DynamicScalar object ids to allocated SymInt nodes, for symbol reuse + self.dynamic_scalar_nodes: dict[int, torch.SymInt] = {} + + self.prev_inst = None + # True if we want to allow externally visible side-effects (doesn't throw error on their existence) + # during this tracer's tracing. This is currently only used by experimental AC out-of-tree + # via torch._dynamo.utils._disable_side_effect_safety_checks_for_current_subtracer. + # Note: Externally visible side-effects are allowed if this flag OR the above flag is True. + self.unsafe_allow_externally_visible_side_effects = False + self.traced_with_externally_visible_side_effects = False + # True if we want to allow side effects by returning them as extra outputs from the subgraph. + # This is set when enable_side_effects_in_hop=True for HOPs like invoke_subgraph + # and checkpoint (when skip_fwd_side_effects_in_bwd_under_checkpoint config is True). + self.allow_side_effects_in_hop = False + + # True if this tracer is currently tracing (reconstructing) into a Python generator + self.is_reconstructing_generator = False + + self.debug_level: int = parent.debug_level + 1 if parent is not None else 0 + + self._cur_code = None + self._orig_gm_meta: Optional[list[Any]] = None + self._orig_gm_lineno_map: Optional[dict[int, Optional[int]]] = None + self._orig_gm_firstlineno: Optional[int] = None + # Each SubgraphTracer is associated with a source target, which indicates + # which operator this subgraph is attached to. We compute a source_fn_stack + # based on the source target. For the root tracer, it's set to []. + # This is useful for debugging and transforming the exported graph. + if self.parent is None: + self.source_fn_stack: list[Any] = [] + else: + self.source_fn_stack = self.parent.source_fn_stack + [ + (self.graph._target_to_str(source_target), source_target) + ] + + # This is used to create a unique name for the placeholder + self._used_names: OrderedSet[str] = OrderedSet() + # Stores the versions of the input tensors at the time they are inserted + # as placeholders in the graph. This is used to track input mutation. + self._input_versions_at_beginning: list[int] = [] + if torch.is_inference_mode_enabled(): + raise RuntimeError( + "Inference mode is supposed to be disabled during compilation. Please open an issue." + ) + + self.tracked_tensor_or_symint_vt: OrderedSet[VariableTracker] = OrderedSet() + + def record_tensor_or_symint_vt(self, vt: VariableTracker): + self.tracked_tensor_or_symint_vt.add(vt) + + # preserve original meta if it is available + def _maybe_preserve_original_meta( + self, tx: "InstructionTranslatorBase", node: fx.Node + ) -> None: + if ( + self._orig_gm_meta + and self._orig_gm_lineno_map + and self._orig_gm_firstlineno + ): + lineno = tx.current_instruction.starts_line + node_idx = None + if lineno is not None: + node_idx = self._orig_gm_lineno_map.get( + lineno - self._orig_gm_firstlineno, None + ) + if node_idx is not None: + meta = self._orig_gm_meta[node_idx] + for field in fx.proxy._COPY_META_FIELDS: + if field in meta: + node.meta[field] = meta[field] + if "stack_trace" in meta: + node.meta["stack_trace"] = meta["stack_trace"] + + def create_proxy( + self, + kind: str, + target: Any, + args: Any, + kwargs: Any, + name: Optional[str] = None, + type_expr: Optional[Any] = None, + proxy_factory_fn: Optional[Callable[[fx.Node], fx.Proxy]] = None, + ) -> fx.Proxy: + # NOTE: [Nested SubgraphTracer and free_variable handling] + # -------------------------------------------------------- + # Read NOTE [HigherOrderOperator tracing design] first. + # + # Let's say we're in the middle of introspecting the body of a possibly + # nested HigherOrderOperator, and we see a free variable. + # + # There are two cases: + # 1. We see a free variable that is already tracked by Dynamo. + # 2. We see a free variable that has not been tracked by Dynamo + # + # In case 1, we call `maybe_lift_tracked_freevar_to_input` (below) + # which will lift the freevar to be an input of this subgraph + # and also recursively lift it to be an input on the parent(s). + # + # In case 2, before the call to `create_proxy`, the InstructionTranslator + # will see the freevar when it gets loaded by Python bytecode. + # E.g. for Python 3.11 the bytecodes that may do this are LOAD_DEREF or + # LOAD_GLOBAL. + # There, the InstructionTranslator asks Dynamo to begin tracking the + # freevar by building a new Variable. + # Building a new Variable automatically lifts the freevar to be an + # input of the root SubgraphTracer. + # + # The implications for the code below are: + # - We will always be in Case 1 when we get to this code. + # - Any "free variable" we encounter here is guaranteed to already be + # bound, that is, it is either a graph input of the root graph, or + # some local variable of the root graph or a subgraph. + # - The additional work we need to do here is *only* that we need to + # lift this free variable into inputs (recursively) of each nested + # higher-order-op subgraph until we hit the subgraph where the free + # variable is bound + if self.parent is not None: + flat_args, tree_spec = pytree.tree_flatten((args, kwargs)) + new_flat_args = [] + for arg in flat_args: + maybe_new_arg = self.maybe_lift_tracked_freevar_to_input(arg) + new_flat_args.append(maybe_new_arg) + + args, kwargs = pytree.tree_unflatten(new_flat_args, tree_spec) + + rv = super().create_proxy( + kind, + target, + args, + kwargs, + name, + type_expr, + proxy_factory_fn, # type: ignore[arg-type] + ) + + # append stack trace to fx node + tx = self.output_graph.current_tx + + # log detailed location of line of code in 3.11 + if sys.version_info >= (3, 11) and kind in ( + "call_function", + "call_method", + "call_module", + ): + cur_inst = tx.current_instruction + if ( + cur_inst is not self.prev_inst + and cur_inst.positions is not None + and cur_inst.positions.lineno is not None + ): + tx_code = tx.f_code + header = tx.get_line_of_code_header(lineno=cur_inst.positions.lineno) + + def get_trace_call_log_str() -> str: + line = get_instruction_source_311(tx_code, cur_inst).rstrip() + return f"TRACE FX call {rv.node.name} from {header}\n{line}" + + trace_call_log.debug("%s", LazyString(get_trace_call_log_str)) + self.prev_inst = cur_inst + + # update reference to original meta if we're tracing a new code object + is_retracing = False + if tx.f_code is not self._cur_code: + orig_graphmodule_maybe = code_context.get_context(tx.f_code).get( + "orig_graphmodule", lambda: None + )() + if isinstance(orig_graphmodule_maybe, torch.fx.GraphModule): + is_retracing = True + self._orig_gm_meta = [ + nd.meta for nd in orig_graphmodule_maybe.graph.nodes + ] + self._orig_gm_lineno_map = orig_graphmodule_maybe._lineno_map + self._orig_gm_firstlineno = ( + orig_graphmodule_maybe.forward.__code__.co_firstlineno + ) + else: + self._orig_gm_meta = None + self._orig_gm_lineno_map = None + self._orig_gm_firstlineno = None + nn_module_stack = tx.nn_module_stack + if nn_module_stack: + rv.node.meta["nn_module_stack"] = nn_module_stack.copy() + + if kind in {"call_function", "call_method"}: + stack = (rv.node.name, target) + if nn_module_stack: + # Current codebase assumes that the nn_module_stack has the + # builtin modules in the stack. + current_nn_module = list(rv.node.meta["nn_module_stack"].values())[-1][ + 1 + ] + if current_nn_module.__module__.startswith( + ("torch.nn.modules", "torch.ao.") + ) and not current_nn_module.__module__.startswith( + "torch.nn.modules.container" + ): + stack = (rv.node.name, current_nn_module) + + rv.node.meta["source_fn_stack"] = self.source_fn_stack + [stack] + elif kind == "call_module": + if self.parent is not None: + # TODO can remove once inline_inbuilt_nn_modules is always True + unimplemented( + gb_type="Invoking an nn.Module inside a higher order operator", + context=f"Higher order op name: {self.source_target}", + explanation="This is not supported.", + hints=[], + ) + # For modules we store the class + rv.node.meta["source_fn_stack"] = self.source_fn_stack + [ + ( + rv.node.name, + next( + ty + for k, (_, ty) in rv.node.meta["nn_module_stack"].items() + if k.split("@")[0] == target + ), + ) + ] + + self._maybe_preserve_original_meta(tx, rv.node) + + if not is_retracing: + if "nn_module_stack" not in rv.node.meta: + nn_module_stack = tx.nn_module_stack + if nn_module_stack: + rv.node.meta["nn_module_stack"] = nn_module_stack.copy() + + if "source_fn_stack" not in rv.node.meta: + if kind in {"call_function", "call_method"}: + rv.node.meta["source_fn_stack"] = self.source_fn_stack + [ + (rv.node.name, target) + ] + elif kind == "call_module": + if self.parent is not None: + # TODO can remove once inline_inbuilt_nn_modules is always True + unimplemented( + gb_type="Invoking an nn.Module inside a HigherOrderOperator", + context="", + explanation="This is not supported.", + hints=[], + ) + # For modules we store the class + rv.node.meta["source_fn_stack"] = self.source_fn_stack + [ + ( + rv.node.name, + rv.node.meta["nn_module_stack"][target][1], + ) + ] + + if "stack_trace" not in rv.node.meta: + frame_summaries: list[traceback.FrameSummary] = [] + while tx: + # Avoid frame summaries from inside the torch/nn/modules. This ensures that we keep the stack trace of + # the user code. + if not tx.is_co_filename_from_nn_modules(): + frame_summaries.append(tx.frame_summary()) + tx = getattr(tx, "parent", None) + + filtered_frame_summaries = [ + frame + for frame in frame_summaries + if frame.filename not in uninteresting_files() + ] + + # Reverse the frame_summaries, such that the innermost frame is at the last + filtered_frame_summaries.reverse() + + # official from_list stub doesn't have new-style type + msgs = traceback.StackSummary.from_list(filtered_frame_summaries).format() + rv.node.stack_trace = "".join(msgs) + + if ( + torch._dynamo.config.use_graph_deduplication + or torch._dynamo.config.track_nodes_for_deduplication + ): + self.output_graph.region_tracker.track_node( + self.output_graph.current_tx, rv.node + ) + return rv + + def create_node( + self, + op: str, + target: Target, + args: Any = None, + kwargs: Any = None, + name: Optional[str] = None, + type_expr: Optional[Any] = None, + ) -> fx.Node: + check_pt2_compliant_op(self.output_graph, op, target, args, kwargs) + if self.parent is not None: + flat_args = pytree.arg_tree_leaves(*args, **kwargs) + for arg in flat_args: + if not isinstance(arg, torch.fx.Node): + continue + assert arg.graph == self.graph, ( + "create_node using arg not from this SubgraphTracer" + ) + + node = super().create_node(op, target, args, kwargs, name, type_expr) + node.meta["creation_timestamp"] = self.output_graph.timestamp + self._used_names.add(node.name) + return node + + # Note: we did not override erase_node since + # we call self.graph.erase_node elsewhere + def remove_node(self, node: fx.Node) -> None: + if len(node.users) > 0: + user_graph_nodes: list[torch.fx.Node] = [] + for user in node.users: + # For the case where user.graph == self.graph, that is a real bug and will raise + # properly. + if user.graph != self.graph: + # This is a nested graph, which needs to be deleted. + # If we do not do this, we will raise on attempting to remove this. + # As we only get here during restoration cleanup, this is sound. + user_graph_nodes.extend(reversed(list(user.graph.nodes))) + for other_graph_node in user_graph_nodes: + other_graph_node.graph.erase_node(other_graph_node) + self.graph.erase_node(node) + self.input_name_to_proxy.pop(node.name, None) + + # when before=True, we will insert this input before the most recent + # inserted proxy. This is a hack to get around an ordering problem, + # where we first insert a tensor argument, and then insert bindings + # for SymInts that may occur in the tensor argument. + # Remove this if https://github.com/pytorch/pytorch/issues/99007 gets + # fixed. + def create_graph_input( + self, + name: str, + type_expr: Any, + example_value: Any, + before: bool = False, + source: Optional[Source] = None, + ) -> fx.Proxy: + if isinstance(example_value, torch.Tensor): + self._input_versions_at_beginning.append(example_value._version) + log.debug( + "create_graph_input %s %s %s at debug_level %s before=%s", + name, + source.name if source is not None else "(none)", + example_value, + self.debug_level, + before, + ) + if source is None: + assert self.parent is not None, ( + f"you are required to provide a source for inputs {name} example_val {example_value} on the root tracer" + ) + + # Note [Export inputs must be explicitly passed in] + # In eager, we are generally OK with adding graph inputs whenever we + # want, because we take care of writing the bytecode that knows how + # to source all the inputs. + # + # In export, this is bad, because you want a self-contained export + # object which only depends on the inputs you explicitly passed to it. + # So we are a bit more strict about what sources can become inputs + # in export + if self.is_export and self.parent is None: + assert source is not None + if not is_from_local_source(source, only_allow_input=True): + self.output_graph.source_to_user_stacks.setdefault(source, []).append( + TracingContext.extract_stack() + ) + + # _used_names contains the names of all the nodes in the graph, + # including intermediates. This ensures that we do not have a name + # collision. + name = get_unique_name_wrt(name, self._used_names) + if self.input_name_to_proxy: + prev_name = next(reversed(self.input_name_to_proxy)) + node = self.input_name_to_proxy[prev_name].node + if before: + ctx = self.graph.inserting_before(node) + else: + ctx = self.graph.inserting_after(node) + else: + ctx = self.graph.inserting_before(None) + with ctx: + proxy = self.create_proxy("placeholder", name, (), {}, type_expr=type_expr) + set_example_value(proxy.node, example_value) + if self.input_name_to_proxy and before: + k, v = self.input_name_to_proxy.popitem() + self.input_name_to_proxy[name] = proxy + self.input_name_to_proxy[k] = v + else: + self.input_name_to_proxy[name] = proxy + + # For placeholder nodes, `name` is passed as a str to the target, + # and then torch.fx decides the node.name. So, record the `target` + # name as well in the _used_names to prevent any collision. + self._used_names.add(name) + + # NOTE: [Auto lift basic free symbols when create_graph_input] + # There are two sources of basic symbols: + # + # - They can come from inputs, e.g. when an input tensor is specified as dynamic. We handle + # this case by intercepting at create_graph_input. Whenever we call create_graph_input, we + # try to also lift the basic symbols in example values as graph input. + # + # 1. When create_graph_input for a tensor that has symbolic shapes, + # we look for basic symbols in its size and stride, we check if the symbol is bound + # in current graph (i.e. bound_symbols), it it's not bound, we'll create a placeholder + # for it then recursively check its parent, creates ph if not bound at parent until. + # reachting the top-level, where we require a source is attached to the proxy. + # + # 2. When create_graph_input for a tensor that contains compound exprs, + # for example, if an input to subgraph takes size [s1+s2//8], we'll look for the + # the free basic symbols in the sizes and lift all of them following 1. + # + # 3. When create_graph_input for a symint. The following invariants hold: + # a. if symint's expr is a basic symbol, we only lift it once. + # b. if symint's expr is compuned, we lift the expr as a single input. We won't lift The basic symbols + # in the compuned expr are NOT lifted. Because if the basic symbols are used inside the subgraph + # they will be lifted according to 3.a + # + # - They can come from intermediate results: + # For example, data-dependent operators such as t.item(), t.nonzero(), where basic symbols + # might be created. For this purpose, we track the basic symbols of intermediate results + # immediately after they're created at wrap_fx_proxy with track_produced_symints. Notice + # that for basic symbols that're already tracked by create_graph_input, we won't track it again. + # + # Also see NOTE: [Export inputs must be explicitly passed in] + is_strict_export = self.is_export + is_non_strict_export = torch.compiler.is_compiling() + if not is_strict_export and not is_non_strict_export: + if isinstance(example_value, torch.Tensor): + self._lift_basic_symbols(example_value, source) + elif isinstance(example_value, (list, tuple)): + for i, e in enumerate(example_value): + if not isinstance(e, torch.Tensor): + continue + + e_source = None + if source: + e_source = GetItemSource( + base=source, index=i, index_is_slice=False + ) + + self._lift_basic_symbols(e, e_source) + + # Bound the symbol to ph if example_value is a SymInt with basic symbol. + if isinstance(example_value, torch.SymInt) and isinstance( + example_value.node.expr, sympy.Symbol + ): + self.bound_symbols[example_value.node.expr] = proxy + return proxy + + # See NOTE: [Nested SubgraphTracer and free_variable handling] for more details + def lift_tracked_freevar_to_input( + self, proxy: fx.Proxy + ) -> Union[LazyProxy, fx.Proxy]: + # You're doing something wrong if we are the root SubgraphTracer because + # Dynamo adds tensors to graph inputs before creating a proxy for them. + assert self.parent is not None, ( + "lift_tracked_freevar_to_input should not be called on root SubgraphTracer" + ) + + example_value = proxy.node.meta["example_value"] + + # To avoid lifting the same symbol twice, we check whether basic symbols has been tracked. + # For example, the basic symbols may have already been lifted for current subgraph when + # we automatically lift basic symbols in the sizes/strides of a tensor t. + # Suppose parent graph calls sz = t.size()[0], it creates + # a proxy in parent and the subgraph accesses sz via closure. sz's proxy is not tracked + # in current sub-tracer so we may lift the same symbol twice. + if ( + isinstance(example_value, torch.SymInt) + and example_value.node.expr in self.bound_symbols + ): + return self.bound_symbols[example_value.node.expr] + + # Proxies are associated with VariableTracker. + # It is possible that we've already lifted the Proxy to be an input. + # If that is the case, just return the already lifted Proxy. + if proxy in self.lifted_freevars: + return self.lifted_freevars[proxy] + + # We first lift proxy to parent's graph then lift to current graph's input + # so that when we bind symints of the sizes in current graph, those symints + # would already be lifted as inputs to parent graph. + if proxy.tracer != self.parent: + self.parent.lift_tracked_freevar_to_input(proxy) + + example_value = proxy.node.meta["example_value"] + new_proxy = self.create_graph_input( + proxy.node.name, type(example_value), example_value + ) + self.lifted_freevars[proxy] = new_proxy + return new_proxy + + def maybe_lift_tracked_freevar_to_input(self, arg: Any) -> Any: + """ + If arg is a free variable, then lift it to be an input. + Returns the new lifted arg (if arg was a freevar), else the + original arg. + """ + if not isinstance(arg, torch.fx.Proxy): + # Note: arg can be a python built-in slice type e.g. + # x[:max_seq] is represented as get_item(t, (slice(None, max_seq, None))) + # we need to also look into the slice variable itself to lift the + # proxies there. + if isinstance(arg, slice): + return slice( + *( + self.maybe_lift_tracked_freevar_to_input(sub_arg) + for sub_arg in (arg.start, arg.stop, arg.step) + ) + ) + else: + return arg + elif arg.tracer == self: + return arg + return self.lift_tracked_freevar_to_input(arg) + + # See NOTE: [Auto lift basic free symbols when create_graph_input] for overall design + # You MUST call this API every time when creating a proxy in wrap_fx_proxy for a call + # that produced symints or tensors with unbacked symint shapes. + # This function is used to track the symints with its proxies created during + # dynamo tracing so that subgraph knows how to bind a symbol input with parent's proxy. + # LazyProxy are created for tensor shapes that're unbacked so that we don't create proxies + # for symbols that're not going to be used, the LazyProxy will be turned into a proxy + # when it's lifted as input to subgraph. + def track_produced_symints( + self, example_value: Any, e_proxy: Union[LazyProxy, torch.fx.Proxy] + ) -> None: + # When binding the symbols in an example_value, we bind the symbols + # to the proxy's associated Tracer instead of current tracer. + # This is because: + # 1. We may be calling wrap_tensors during speculate_subgraph because + # the variables are lazily realized. The proxy are top-level phs but + # current tracer is a subtracer. + # 2. For autograd.Function, we trace the backward graph with a new tracer + # whose parent is the forward tracer, but we're using all the proxies created + # in forward tracer to trace the backward. + # For example, forward calls save_for_backward for a input tensor t. + # Backward calls t.tolist(). In this case, all the proxies that backward tracer + # sees are from parent tracer (i.e. the forward tracer). (e.g. t[0].item()) + # See test_validate_outputs_unbacked for repro on 2. + tracer = e_proxy.tracer + assert isinstance(tracer, SubgraphTracer) + + def need_bind(s: Any) -> bool: + from torch.fx.experimental.symbolic_shapes import is_symbolic + + return ( + is_symbolic(s) + and isinstance(s.node.expr, sympy.Symbol) + and s.node.expr not in self.bound_symbols + ) + + def _proxy_with_example_value( + example_value: Any, *args: Any, **kwargs: Any + ) -> fx.Proxy: + # We need to insert proxy for creating sym_size/sym_stride/sym_storage right after e_proxy + nonlocal e_proxy + e_proxy = e_proxy() if isinstance(e_proxy, LazyProxy) else e_proxy + assert isinstance(e_proxy, torch.fx.Proxy) + with tracer.graph.inserting_after(e_proxy.node): + proxy = tracer.create_proxy(*args, **kwargs) + set_example_value(proxy.node, example_value) + return proxy + + if isinstance(example_value, torch.Tensor): + for i, s in enumerate(example_value.size()): + if need_bind(s): + log.debug( + "track_produced_symints %s for %s.size()[%s] at debug_level %s", + s, + e_proxy, + i, + tracer.debug_level, + ) + lazy_proxy = LazyProxy( + tracer, + _proxy_with_example_value, + s, + "call_function", + torch.ops.aten.sym_size.int, + (e_proxy, i), + {}, + type_expr=type(s), + ) + self.track_produced_symints(s, lazy_proxy) + + storage_offset = example_value.storage_offset() + if need_bind(storage_offset): + log.debug( + "track_produced_symints %s for %s.storage_offset() at debug_level %s", + storage_offset, + e_proxy, + tracer.debug_level, + ) + lazy_proxy = LazyProxy( + tracer, + _proxy_with_example_value, + storage_offset, + "call_function", + torch.ops.aten.sym_storage_offset, + (e_proxy,), + {}, + type_expr=type(storage_offset), + ) + self.track_produced_symints(storage_offset, lazy_proxy) + + if example_value.layout is torch.strided: + for i, s in enumerate(example_value.stride()): + if need_bind(s): + log.debug( + "track_produced_symints %s for %s.stride()[%s] at debug_level %s", + s, + e_proxy, + i, + tracer.debug_level, + ) + lazy_proxy = LazyProxy( + tracer, + _proxy_with_example_value, + s, + "call_function", + torch.ops.aten.sym_stride.int, + (e_proxy, i), + {}, + type_expr=type(s), + ) + self.track_produced_symints(s, lazy_proxy) + + elif example_value.layout is torch.sparse_coo: + self.track_produced_symints(example_value._indices(), e_proxy) + self.track_produced_symints(example_value._values(), e_proxy) + elif example_value.layout in {torch.sparse_csr, torch.sparse_bsr}: + self.track_produced_symints(example_value.crow_indices(), e_proxy) + self.track_produced_symints(example_value.col_indices(), e_proxy) + elif example_value.layout in {torch.sparse_csc, torch.sparse_bsc}: + self.track_produced_symints(example_value.ccol_indices(), e_proxy) + self.track_produced_symints(example_value.row_indices(), e_proxy) + if is_traceable_wrapper_subclass(example_value): + attrs, ctx = example_value.__tensor_flatten__() + for attr in attrs: + inner_t = getattr(example_value, attr) + self.track_produced_symints(inner_t, getattr(e_proxy, attr)) + elif isinstance(example_value, torch.SymInt): + if need_bind(example_value): + expr = example_value.node.expr + tracer.bound_symbols[expr] = e_proxy + + # See Note [Auto lift basic free symbols when create_graph_input] + def _lift_basic_symbols( + self, example_value: Union[torch.SymInt, torch.Tensor], src: Optional[Source] + ) -> None: + # The before arg is for inserting symints in the sizes/strides of a tensor + # before the tensor. This ordering ensures that when we look at the tensor's + # symbols, they're already lifted/tracked. E.g. this assumption is used + # in insert_deferred_runtime_asserts. + def _lift_symbols_in_symint( + s: Union[int, torch.SymInt], + source: Optional[Source], + before: bool = False, + ) -> None: + if not is_symbolic(s): + return + + assert isinstance(s, torch.SymInt) + self_to_be_bound = self.lookup_unbound_symbols(s) + if len(self_to_be_bound) == 0: + return + + # For subgraph + if self.parent is not None: + # Recursively lift symbols in symint until top-level. + self.parent._lift_basic_symbols(s, source) + for s0 in self_to_be_bound: + parent_proxy = self.parent.bound_symbols[s0] + example_val = parent_proxy.node.meta["example_value"] # type: ignore[union-attr] + assert isinstance(example_val, torch.SymInt) + ph = self.create_graph_input( + str(s0), + type(example_val), + example_val, + before=before, + source=source, + ) + log.debug( + "_lift_symbols_in_symint %s from %s at debug_level %s", + s0, + source.name if source is not None else "subgraph inputs", + self.debug_level, + ) + self.lifted_freevars[parent_proxy] = ph # type: ignore[index] + # For root_tracer: + else: + assert len(self_to_be_bound) == 1, ( + f"For root tracer, we only expect to bind basic symbols (compound symbols " + f"should be cached before) but got unbound symbols {self_to_be_bound} in {s}" + ) + assert source is not None, ( + f"Source of '{s}' is None when lifting it to input of top-level. If it's an unbacked symbol, " + "this could be because it's not tracked with lazy_bind_unbacked_symbols. " + f"Otherwise, should provide a source when create_graph_input for `{s}` at root tracer." + ) + s0 = next(iter(self_to_be_bound)) + ph = self.create_graph_input( + str(s0), + type(s), + s, + before=before, + source=source, + ) + log.debug( + "_lift_symbols_in_symint %s from %s at debug_level %s", + s, + source.name if source is not None else "subgraph inputs", + self.debug_level, + ) + ph.node.meta["grapharg"] = GraphArg( + source, + s, + pass_arg_as_tensor=False, + fake_tensor=None, + is_tensor=False, + ) + + if isinstance(example_value, torch.Tensor): + for i, s in enumerate(example_value.size()): + _lift_symbols_in_symint( + s, + ( + TensorPropertySource(src, TensorProperty.SIZE, i) + if src is not None + else None + ), + before=True, + ) + if example_value.layout is torch.strided: + for i, s in enumerate(example_value.stride()): + _lift_symbols_in_symint( + s, + ( + TensorPropertySource(src, TensorProperty.STRIDE, i) + if src is not None + else None + ), + before=True, + ) + _lift_symbols_in_symint( + example_value.storage_offset(), + ( + TensorPropertySource(src, TensorProperty.STORAGE_OFFSET) + if src is not None + else None + ), + before=True, + ) + elif example_value.layout is torch.sparse_coo: + self._lift_basic_symbols(example_value._indices(), src) + self._lift_basic_symbols(example_value._values(), src) + elif example_value.layout in {torch.sparse_csr, torch.sparse_bsr}: + self._lift_basic_symbols(example_value.crow_indices(), src) + self._lift_basic_symbols(example_value.col_indices(), src) + elif example_value.layout in {torch.sparse_csc, torch.sparse_bsc}: + self._lift_basic_symbols(example_value.ccol_indices(), src) + self._lift_basic_symbols(example_value.row_indices(), src) + if is_traceable_wrapper_subclass(example_value): + attrs, ctx = example_value.__tensor_flatten__() + for attr in attrs: + inner_t = getattr(example_value, attr) + self._lift_basic_symbols( + inner_t, AttrSource(src, attr) if src is not None else None + ) + elif isinstance(example_value, torch.SymInt): + _lift_symbols_in_symint( + example_value, + src, + ) + + # Lookup the proxy in current tracer for each symbol in expressions of s, + # See Note [Auto lift basic free symbols when create_graph_input] + def lookup_unbound_symbols(self, s: torch.SymInt) -> list[sympy.Symbol]: + free_symbols = s.node.expr.free_symbols + if len(free_symbols) == 0: + return [] + + to_be_bound = [] + for s0 in free_symbols: + if s0 not in self.bound_symbols: + to_be_bound.append(s0) + continue + + proxy = self.bound_symbols[s0] + if isinstance(proxy, LazyProxy): + proxy = proxy() + self.bound_symbols[s0] = proxy + assert isinstance(proxy, torch.fx.Proxy) and proxy.tracer is self, ( + f"The proxy of symbol {s0} doesn't belong to current tracer." + ) + # Sort the symbols so that we can have a deterministic lifting order + return sorted(to_be_bound, key=lambda s: s.name) + + def has_input_mutation(self) -> MutationInfo: + input_versions_at_beginning = self._input_versions_at_beginning + input_nodes = [] + + input_versions_at_end = [] + for node in self.graph.nodes: + if node.op == "placeholder": + example_value = node.meta["example_value"] + if isinstance(example_value, torch.Tensor): + input_versions_at_end.append(example_value._version) + input_nodes.append(node) + else: + break + + mutated_inputs = [ + i + for i, (v1, v2) in enumerate( + zip(input_versions_at_beginning, input_versions_at_end) + ) + if v1 != v2 + ] + + if mutated_inputs: + mutated_nodes = [input_nodes[i] for i in mutated_inputs] + msg = f"Input mutation detected at {mutated_nodes}" + return MutationInfo(True, msg) + + return MutationInfo(False, "") + + def has_aliasing(self) -> AliasingInfo: + from torch._dynamo.variables.higher_order_ops import get_tensor_storages + from torch._higher_order_ops.utils import _collect_fake_inputs + + input_storages: dict[StorageWeakRef, torch.fx.Node] = dict() + + for node in self.graph.nodes: + if node.op == "placeholder": + example_value = _collect_fake_inputs([node])[0] + if isinstance(example_value, torch.Tensor): + for storage in get_tensor_storages(example_value): + if storage in input_storages: + # input-input aliasing + msg = f"Input-to-input aliasing detected at nodes {input_storages[storage]} and {node}" + return AliasingInfo(True, msg) + input_storages[storage] = node + else: + break + + output_storages: dict[StorageWeakRef, torch.fx.Node] = dict() + out_nodes = self.graph.find_nodes(op="output")[0] + for out_node in pytree.tree_leaves(out_nodes.args[0]): + if out_node: + example_value = _collect_fake_inputs([out_node])[0] + assert not isinstance(example_value, list) + if isinstance(example_value, torch.Tensor): + for storage in get_tensor_storages(example_value): + if storage in output_storages: + # output-output aliasing + msg = f"Output-to-output aliasing detected at nodes {output_storages[storage]} and {out_node}" + return AliasingInfo(True, msg) + output_storages[storage] = out_node + + intersected_storages = input_storages.keys() & output_storages.keys() + if len(intersected_storages) > 0: + # input-output aliasing + aliased = [ + (input_storages[s], output_storages[s]) for s in intersected_storages + ] + aliased = ", ".join([f"{i} and {o}" for i, o in aliased]) + msg = f"Input-to-output aliasing detected at nodes {aliased}" + return AliasingInfo(True, msg) + + return AliasingInfo(False, "") + + +# NOTE: [HigherOrderOperator tracing design] +# Ignoring HigherOrderOperators for a moment, +# OutputGraph represents the graph being built by Dynamo that may be compiled +# and executed. It holds a root SubgraphTracer where the FX graph is built. +# +# HigherOrderOperators are operators that take functions as their arguments. +# When Dynamo encounters a HigherOrderOperator, then it attempts to introspect +# the function passed to it (call this the "body function"), capture it into a +# GraphModule, and rewrite the call to the HigherOrderOperator to use the +# GraphModule. +# +# The way we handle the capture of body functions is through having +# (possibly nested) SubgraphTracers, one per body function. +# +# Mechanically, we do the introspection by: +# - Creating a new SubgraphTracer via OutputGraph.subtracer +# - Executing the body function. +# This constructs the graph of the body function in the new SubgraphTracer +# while modifying the state of the OutputGraph. For example: +# - the OutputGraph can receive new GraphArgs (if we discover any new +# untracked Tensors) +# - side effects from the body function get accumulated into +# OutputGraph.side_effects +# - guards produced by the body function get accumulated into OutputGraph.guards +# +# The traced function has some special properties that make it easier for us +# to transform later down the line: +# - we lift all free variables to being inputs. +# +# If the introspection fails (due to the existence of graph breaks), then +# we roll back the current OutputGraph state and graph break on the +# HigherOrderOperator. diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/package.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/package.py new file mode 100644 index 0000000000000000000000000000000000000000..e178a8e6a7380a89fe848f0f28bd6c009a82d0fa --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/package.py @@ -0,0 +1,1157 @@ +""" +This module provides the infrastructure for creating and managing compile package +for torch.compile. We mainly have two abstractions here: + - CompilePackage: Overarching data structure for store and lookup a list of compiled codes. + - CodeCacheEntry: Data structure for a single code being compiled by torch.compile. +The caching behavior is always under user control explicitly so that a stronger guarantee can +be provided about cache hit for a specific compiled model. Users can load the compile package +from a different process or host. +""" + +import abc +import ast +import contextlib +import dataclasses +import functools +import hashlib +import importlib +import inspect +import json +import logging +import os +import pickle +import platform +import shutil +import sys +import types +from collections.abc import Callable, Generator, Iterator +from contextlib import nullcontext +from typing import Any, NewType, Optional, TYPE_CHECKING +from typing_extensions import Never + +import torch +from torch._dynamo.exc import PackageError +from torch._dynamo.graph_utils import _graph_device_type + +from .bytecode_transformation import get_code_keys +from .utils import counters, dynamo_timed, increment_frame + + +logger = logging.getLogger(__name__) + + +if TYPE_CHECKING: + from .guards import GuardManagerWrapper, GuardsState + + +@dataclasses.dataclass(frozen=True) +class SerializedCode: + co_argcount: int + co_posonlyargcount: int + co_kwonlyargcount: int + co_nlocals: int + co_stacksize: int + co_flags: int + co_code: bytes + co_consts: tuple[Any, ...] + co_names: tuple[str, ...] + co_varnames: tuple[str, ...] + co_filename: str + co_name: str + co_firstlineno: int + co_cellvars: tuple[str, ...] + co_freevars: tuple[str, ...] + co_linetable: Optional[bytes] = None + co_qualname: Optional[str] = None + co_exceptiontable: Optional[bytes] = None + co_lnotab: Optional[str] = None + + @classmethod + @functools.cache + def from_code_object(cls, code: types.CodeType) -> "SerializedCode": + kwargs = {key: getattr(code, key) for key in get_code_keys()} + kwargs["co_consts"] = tuple( + cls.from_code_object(c) if isinstance(c, types.CodeType) else c + for c in kwargs["co_consts"] + ) + return cls(**kwargs) + + @classmethod + @functools.cache + def to_code_object(cls, serialized_code: "SerializedCode") -> types.CodeType: + kwargs = {key: getattr(serialized_code, key) for key in get_code_keys()} + kwargs["co_consts"] = tuple( + cls.to_code_object(c) if isinstance(c, SerializedCode) else c + for c in kwargs["co_consts"] + ) + return types.CodeType( + *kwargs.values(), + ) + + +@dataclasses.dataclass +class _GuardedCodeCacheEntry: + """ + Contains the serializable information associated with a single compilation in dynamo. + To restore an execution of compiled code, we will need to serialize the following data: + - Dynamo bytecode for mapping Python inputs/outputs. + - Dynamo guards. + """ + + guards_state: bytes + dynamo_code: SerializedCode + + +def load_guards_state(guards_state: bytes) -> Any: + try: + import torch.distributed.fsdp._fully_shard._fully_shard as _fully_shard + + ctx = _fully_shard.disable_fsdp_module_new_init() + except ImportError: + ctx = nullcontext() # type: ignore[assignment] + with ctx: + return pickle.loads(guards_state) + + +def load_guard_manager( + guards_state: "GuardsState", + target_code: types.CodeType, + runtime_global_scope: Any, +) -> "GuardManagerWrapper": + from .output_graph import OutputGraphCommon + + return torch._dynamo.guards.CheckFunctionManager( + target_code, + OutputGraphCommon(guards_state.output_graph), + shape_code_parts=guards_state.shape_code_parts, + runtime_global_scope=runtime_global_scope, + source_get_cache=guards_state.source_get_cache, + ).guard_manager + + +_BackendId = NewType("_BackendId", str) # __compiled_fn +_FunctionId = NewType("_FunctionId", str) # __resume_at + + +@dataclasses.dataclass(frozen=True) +class InlinedSource: + module: str + firstlineno: int + lastlineno: int + checksum: str + content: str + + +@functools.cache +def _get_module_content(module: types.ModuleType) -> str: + return inspect.getsource(module) + + +@dataclasses.dataclass +class SourceInfo: + inlined_sources: set[InlinedSource] + + def add_code(self, code: types.CodeType) -> None: + module = inspect.getmodule(code) + if module is None: + return + sourcelines, firstlineno = inspect.getsourcelines(code) + lastlineno = firstlineno + len(sourcelines) + source = "".join(sourcelines) + assert source == "".join(_get_sourcelines(module, firstlineno, lastlineno)) + self.inlined_sources.add( + InlinedSource( + module=module.__name__, + firstlineno=firstlineno, + lastlineno=lastlineno, + checksum=_hash_source(source), + content=_get_module_content(module), + ) + ) + + +@dataclasses.dataclass +class _DynamoCodeCacheEntry: + """ + Contains the serializable information associated with a single code object + in dynamo. To restore an execution of compiled code, we will need the following + ingredients: + 1. The "original" code object, which serves as the entry point for eager + execution, i.e. the code only executed when there's no cache entry hit. + 2. The python module name this code object belongs to, for identifying the + enclosing global scope to inject compiled and resume functions. + 3. A list of function names that pointing to this code object. There could be + multiple function objects pointing to the same code such as recursive functions. + 4. A list of guarded code that eval frame dispatches to. + 5. A list of imported module objects unioned from all compiled branches. + 6. A list of "backends" (compiled fx graph) unioned from all compield branches. + 7. A string path used to access the original code object users defined. + A code object can be accessed by "{python_module}.{function_name}.{code_source}" . + 8. A boolean flag indicating whether the function is installed to global scope. + 9. A boolean flag indicating whether the function has a compile id. + 10. Whether or not this code entry was bypassed + """ + + python_code: SerializedCode + python_module: str + function_names: list[_FunctionId] + guarded_codes: list[_GuardedCodeCacheEntry] + import_sources: dict[str, str] + backend_ids: list[_BackendId] + code_source: Optional[str] + install_to_global: bool + has_compile_id: bool = False + bypassed: bool = False + + +def _lookup_code(entry: _DynamoCodeCacheEntry) -> types.CodeType: + assert len(entry.function_names) == 1 + fn: Any = sys.modules[entry.python_module] + parts = entry.function_names[0].split(".") + for part in parts: + fn = getattr(fn, part) + if entry.code_source: + parts = entry.code_source.split(".") + for part in parts: + if part.endswith("]"): + index_begin = part.rfind("[") + assert isinstance(index_begin, int) and index_begin >= 0 + attr = getattr(fn, part[:index_begin], None) + if attr is None: + raise PackageError(f"Cannot find source for code entry {entry}") + fn = attr[ast.literal_eval(part[index_begin + 1 : -1])] + else: + fn = getattr(fn, part) + else: + raise PackageError(f"Cannot find source for code entry {entry}") + assert isinstance(fn, types.CodeType) + return fn + + +def _raise_resolution_error(code: types.CodeType, scope: Any) -> Never: + raise PackageError( + f"Cannot resolve a fully qualified name for {code}. Lookup scope: {scope}" + ) + + +def _get_code_source(code: types.CodeType) -> tuple[str, str]: + """ + Given a code object, return a fully qualified name which will be used as + a serialized handle to access the code object from the new process. + This is normally a straightforward process, but there are some corner cases: + 1. When a function is defined with decorator, then this function will be captured + inside a closure with the wrapper object. + 2. When a function is defined as a nested function, then the code object will be + stored on the co_consts field of the parent code object by Python compiler. + This function handles all of the corner cases above. + """ + + module = inspect.getmodule(code) + if module is None: + raise PackageError(f"Cannot find module for code {code}") + + toplevel: Any = module + if sys.version_info >= (3, 11): + parts = code.co_qualname.split(".") + + for part in parts: + if not hasattr(toplevel, part): + _raise_resolution_error(code, toplevel) + toplevel = getattr(toplevel, part) + if inspect.isfunction(toplevel): + break + seen = set() + + def _find_code_source(obj: Any) -> Optional[str]: + nonlocal toplevel + nonlocal seen + if obj in seen: + return None + + seen.add(obj) + + if inspect.iscode(obj): + if obj is code: + return "" + + for i, const in enumerate(obj.co_consts): + if (res := _find_code_source(const)) is not None: + return f".co_consts[{i}]{res}" + + if inspect.isfunction(obj): + if (res := _find_code_source(obj.__code__)) is not None: + toplevel = obj + return f".__code__{res}" + if obj.__closure__ is not None: + for i, cell in enumerate(obj.__closure__): + try: + cell_contents = cell.cell_contents + except ValueError: + continue + if not ( + inspect.isfunction(cell_contents) + or inspect.iscode(cell_contents) + ): + continue + if (res := _find_code_source(cell_contents)) is not None: + toplevel = obj + return f".__closure__[{i}].cell_contents{res}" + + if sys.version_info < (3, 11): + if inspect.ismodule(obj): + for value in obj.__dict__.values(): + if not (inspect.isfunction(value) or inspect.isclass(value)): + continue + if (res := _find_code_source(value)) is not None: + return res + + if inspect.isclass(obj): + for name, value in obj.__dict__.items(): + value = getattr(obj, name) + if not (inspect.isfunction(value) or inspect.isclass(value)): + continue + if (res := _find_code_source(value)) is not None: + if value.__name__ != name: + _raise_resolution_error(code, toplevel) + return res + return None + + code_source = _find_code_source(toplevel) + if code_source is None: + _raise_resolution_error(code, toplevel) + # pyrefly: ignore [missing-attribute] + return toplevel.__qualname__, code_source.strip(".") + + +@dataclasses.dataclass(frozen=True) +class SystemInfo: + """ + System information including Python, PyTorch, and GPU details. + This information is used to ensure compiled artifacts can only be loaded + with compatible system configurations. + """ + + python_version: str + torch_version: str + toolkit_version: Optional[str] + triton_version: Optional[tuple[int, int]] + gpu_name: Optional[str] + CHECK_GPUS = ("cuda", "xpu") + + @classmethod + def current(cls) -> "SystemInfo": + """Create a SystemInfo instance with current system information.""" + # Get GPU name if CUDA or XPU is available + gpu_name = None + from torch.utils._triton import get_triton_version + + gpu_name, toolkit_version = None, None + for device_type in cls.CHECK_GPUS: + if getattr(torch, device_type).is_available(): + try: + gpu_name = getattr(torch, device_type).get_device_name() + toolkit_version = getattr(torch.version, device_type) + break + except Exception: + pass + + return cls( + python_version=platform.python_version(), + torch_version=torch.__version__, + toolkit_version=toolkit_version, + triton_version=get_triton_version((0, 0)), + gpu_name=gpu_name, + ) + + def check_compatibility( + self, other: "SystemInfo", device_type: str = "cpu" + ) -> None: + """ + Check if this SystemInfo is compatible with another SystemInfo. + Raises RuntimeError if incompatible. + """ + if self.python_version != other.python_version: + raise RuntimeError( + f"Compile package was created with a different Python version: {self.python_version}" + ) + + if self.torch_version != other.torch_version: + raise RuntimeError( + f"Compile package was created with a different PyTorch version: {self.torch_version}" + ) + if device_type in self.CHECK_GPUS: + if not getattr(torch, device_type).is_available(): + raise RuntimeError(f"{device_type} is not available") + + if self.toolkit_version != other.toolkit_version: + raise RuntimeError( + f"Compile package was created with a different toolkit version: {self.toolkit_version}" + ) + + if ( + other.triton_version != (0, 0) + and self.triton_version != other.triton_version + ): + raise RuntimeError( + f"Compile package was created with a different Triton version: {self.triton_version}" + ) + + # Check GPU name if CUDA/XPU was used + if other.gpu_name is not None and self.gpu_name != other.gpu_name: + raise RuntimeError( + f"Compile package was created with different GPU: " + f"cached={self.gpu_name}, current={other.gpu_name}" + ) + + +@dataclasses.dataclass +class _DynamoCacheEntry: + codes: list[_DynamoCodeCacheEntry] + source_info: SourceInfo + device_type: str + system_info: SystemInfo = dataclasses.field(default_factory=SystemInfo.current) + fn_name: Optional[str] = None + fn_first_lineno: Optional[str] = None + + @property + def backend_ids(self) -> set[_BackendId]: + return {backend_id for code in self.codes for backend_id in code.backend_ids} + + def check_versions(self) -> None: + """Check if the current system is compatible with the system used to create this cache entry.""" + current_system_info = SystemInfo.current() + self.system_info.check_compatibility(current_system_info, self.device_type) + + def debug_info(self) -> dict[str, Any]: + assert len(self.codes) > 0 + return { + "num_codes": str(len(self.codes)), + "fn_name": self.fn_name, + "fn_first_lineno": self.fn_first_lineno, + "device_type": self.device_type, + "backend_ids": list(self.backend_ids), + } + + +from torch.compiler._cache import ( + CacheArtifact, + CacheArtifactFactory, + CacheArtifactManager, +) + + +@CacheArtifactFactory.register +class PrecompileCacheArtifact(CacheArtifact): + def populate_cache(self) -> None: + DynamoCache._write_to_local_cache(self.content, self.key) + + @staticmethod + def type() -> str: + return "precompile" + + +@dataclasses.dataclass +class PrecompileCacheEntry: + """ + A full cache entry for caching precompile, for a toplevel torch.compile. + Consists of a _DynamoCacheEntry, which contains all the dynamo related contents, + and a set of backends content. In general, the backend content here will always + be of type precompile_context.BackendCacheArtifact + """ + + dynamo: _DynamoCacheEntry + backends: dict[_BackendId, Any] + + @staticmethod + def from_cache_entry( + cache_entry: _DynamoCacheEntry, backends: dict[_BackendId, Any] + ) -> Optional["PrecompileCacheEntry"]: + backend_content: dict[_BackendId, Any] = {} + + for code in cache_entry.codes: + for backend_id in code.backend_ids: + if backend_id not in backends: + logger.warning("Backend not found") + debug_str = json.dumps( + { + "entry": cache_entry.debug_info(), + "missing_backend": backend_id, + } + ) + torch._logging.trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "dynamo_cache_bypass", + "encoding": "json", + }, + payload_fn=lambda: debug_str, + expect_trace_id=False, + ) + code.bypassed = True + break + else: + backend_content[backend_id] = backends[backend_id] + + return PrecompileCacheEntry(dynamo=cache_entry, backends=backend_content) + + +def _hash_source(source: str) -> str: + sha256_hash = hashlib.sha256() + sha256_hash.update(source.encode()) + return sha256_hash.hexdigest() + + +def _get_sourcelines( + m: types.ModuleType, firstlineno: int, lastlineno: int +) -> list[str]: + return inspect.getsourcelines(m)[0][firstlineno - 1 : lastlineno - 1] + + +def _hash_sourcelines(m: types.ModuleType, firstlineno: int, lastlineno: int) -> str: + return _hash_source("".join(_get_sourcelines(m, firstlineno, lastlineno))) + + +def _compile_frame_context( + code: types.CodeType, +) -> contextlib.AbstractContextManager[None]: + from torch._dynamo.convert_frame import get_compile_id, log_dynamo_start + from torch._guards import compile_context, CompileContext + + # Each code represents a new compile frame + # recompiles on the same frame are all saved + # under the same cache entry, so we don't have recompile ids + # i.e. If cold start had 0/0, 0/1, 1/0, 1/1, these would be + # collapsed into 0/0, 1/0 on warm. + @contextlib.contextmanager + def _ctx() -> Iterator[None]: + increment_frame() + compile_id = get_compile_id(frame_state={}) + with ( + compile_context(CompileContext(compile_id)), + dynamo_timed( + "_compile.compile_inner", + phase_name="entire_frame_compile", + dynamo_compile_column_us="dynamo_cumulative_compile_time_us", + # TODO: save all relevant compilation metrics + metadata={ + "frame_key": str(torch._dynamo.utils.curr_frame), + "co_name": code.co_name, + "co_filename": code.co_filename, + "co_firstlineno": code.co_firstlineno, + }, + ), + ): + log_dynamo_start(code) + yield + + return _ctx() + + +class CompilePackage: + """ + CompilePackage is considered a low level component and should not be directly exposed to + end users. It has the following interface: + + 1. `CompilePackage.__init__()` which optionally takes previously serialized dynamo states. + a. when `dynamo` argument is None, it will construct a brand new CompilePackage object. + b. when `dynamo` argument is not None, it will load a pre-compiled dynamo state. + 2. `package.save()` which dumps the dynamo and backend states to a DynamoCacheEntry object. + 3. `package.install(backends) which will handle all the side-effectful global scope + updates with compiled functions and resume functions. + """ + + def __init__( + self, + fn: Optional[Callable[..., Any]], + dynamo: Optional[_DynamoCacheEntry] = None, + ignore_inlined_sources: bool = False, + ) -> None: + self._innermost_fn = None + self._codes: dict[types.CodeType, _DynamoCodeCacheEntry] = {} + + self._current_entry: Optional[_DynamoCodeCacheEntry] = None + self._installed_globals: dict[types.ModuleType, list[str]] = {} + # device_type that model compiled with. + self._device_type = "cpu" + + # For debugging/testing purpose only. + self._cached_backends: dict[_BackendId, Any] = {} + self._source_info: SourceInfo = SourceInfo(inlined_sources=set()) + self._resume_codes: set[types.CodeType] = set() + self._initialized = False + if fn is not None: + self.initialize(fn, dynamo, ignore_inlined_sources) + self.uninstall() + self.validate() + + def is_initialized(self) -> bool: + return self._initialized + + def initialize( + self, + fn: Any, + dynamo: Optional[_DynamoCacheEntry] = None, + ignore_inlined_sources: bool = False, + ) -> None: + from .eval_frame import innermost_fn + + assert not self._initialized + self._source_info = SourceInfo(inlined_sources=set()) + self._innermost_fn = innermost_fn(fn) # type: ignore[assignment] + assert self._innermost_fn is not None + if dynamo is not None: + assert isinstance(dynamo, _DynamoCacheEntry) + dynamo.check_versions() + if not ignore_inlined_sources: + for code in dynamo.source_info.inlined_sources: + m = importlib.import_module(code.module) + checksum = _hash_sourcelines(m, code.firstlineno, code.lastlineno) + if checksum != code.checksum: + raise RuntimeError( + f"Source code changes detected for {code.module} (line {code.firstlineno} - line {code.lastlineno})" + ) + + # pyrefly: ignore [bad-assignment] + self._source_info = dynamo.source_info + + main, *codes = dynamo.codes + # pyrefly: ignore [bad-assignment] + self._codes = {self._innermost_fn.__code__: main} + for code in codes: + self._codes[SerializedCode.to_code_object(code.python_code)] = code + else: + self._add_function( + self._innermost_fn.__code__, self._innermost_fn.__module__ + ) + # pyrefly: ignore [bad-assignment] + self._initialized = True + + def _add_function( + self, + python_code: types.CodeType, + python_module: str, + function_name: Optional[_FunctionId] = None, + code_source: Optional[str] = None, + install_to_global: bool = False, + ) -> None: + if python_code not in self._codes: + code = _DynamoCodeCacheEntry( + python_code=SerializedCode.from_code_object(python_code), + python_module=python_module, + function_names=[], + guarded_codes=[], + import_sources={}, + backend_ids=[], + code_source=code_source, + install_to_global=install_to_global, + ) + self._codes[python_code] = code + else: + code = self._codes[python_code] + assert code.python_module == python_module + assert code.install_to_global == install_to_global + assert code.code_source == code_source + + if function_name is not None: + code.function_names.append(function_name) + + @property + def cached_backends(self) -> dict[_BackendId, Any]: + return self._cached_backends + + @functools.cached_property + def source_id(self) -> str: + assert self._innermost_fn is not None + return CompilePackage.source_id_from_fn(self._innermost_fn) + + def _add_user_function(self, code: types.CodeType) -> None: + function_name, code_source = _get_code_source(code) + module = inspect.getmodule(code) + if module is None: + raise PackageError(f"Cannot find module for code {code}") + self._add_function( + code, + module.__name__, + function_name=_FunctionId(function_name), + code_source=code_source, + ) + + @contextlib.contextmanager + def code_context(self, code: types.CodeType) -> Generator[None, None, None]: + assert self._current_entry is None + + # Sometimes user code cannot be inlined in dynamo resulting in extra user code + # being compiled. We should record these as when they are actually invoked. + if code not in self._codes: + self._add_user_function(code) + + entry = self._codes[code] + self._current_entry = entry + try: + yield + finally: + entry.has_compile_id = True + self._current_entry = None + + def add_guarded_code( + self, + guards_state: bytes, + dynamo_code: types.CodeType, + ) -> None: + assert self._current_entry is not None + if self._current_entry.bypassed: + return + guarded_code_entry = _GuardedCodeCacheEntry( + guards_state=guards_state, + dynamo_code=SerializedCode.from_code_object(dynamo_code), + ) + self._current_entry.guarded_codes.append(guarded_code_entry) + + def add_inlined_source(self, sources: list[types.CodeType]) -> None: + assert self._current_entry is not None + if self._current_entry.bypassed: + return + for code in sources: + if code in self._resume_codes: + continue + self._source_info.add_code(code) + + def update_device_type(self, graph: Optional[torch.fx.Graph]) -> None: + self._device_type = _graph_device_type(graph) + + def bypass_current_entry(self) -> None: + assert self._current_entry is not None + self._current_entry.bypassed = True + + def add_resume_function( + self, + python_code: types.CodeType, + python_module: str, + function_name: Optional[str], + ) -> None: + self._add_function( + python_code, + python_module, + function_name=_FunctionId(function_name) if function_name else None, + install_to_global=True, + ) + self._resume_codes.add(python_code) + + def add_import_source(self, alias: str, module_name: str) -> None: + assert self._current_entry is not None + self._current_entry.import_sources[alias] = module_name + + def add_backend_id(self, backend_id: str, backend: Optional[Any] = None) -> None: + assert self._current_entry is not None + assert backend_id.startswith("__compiled_fn_") # sanity check + backend_id = _BackendId(backend_id) + self._current_entry.backend_ids.append(backend_id) + if backend is not None: + self._cached_backends[backend_id] = backend + + def validate(self) -> None: + assert self._current_entry is None + assert self._innermost_fn is not None + assert self._initialized + assert next(iter(self._codes)) is self._innermost_fn.__code__ + + def _install_global(self, module: types.ModuleType, name: str, value: Any) -> None: + module.__dict__[name] = value + self._installed_globals.setdefault(module, []).append(name) + + def uninstall(self) -> None: + from torch._C._dynamo.eval_frame import _reset_precompile_entries + + assert self._innermost_fn is not None + for module, names in self._installed_globals.items(): + for name in names: + module.__dict__.pop(name) + + # pyrefly: ignore [bad-assignment] + self._installed_globals = {} + + _reset_precompile_entries(self._innermost_fn.__code__) + + def install(self, backends: dict[_BackendId, Any]) -> None: + """ + Sync the package states to the compiled function. This includes the following actions: + 1. Clean up the previously installed states. + 2. Install the compiled functions to global scopes. + 3. Install the precompiled cache entries to ExtraStates on the code object. + """ + from torch._C._dynamo.eval_frame import _load_precompile_entry + + from .output_graph import get_builtins_dict + + self.uninstall() + for code, entry in self._codes.items(): + context = ( + _compile_frame_context(code) + if entry.has_compile_id + else contextlib.nullcontext() + ) + with context: + module = sys.modules[entry.python_module] + for alias, module_name in entry.import_sources.items(): + self._install_global( + module, alias, importlib.import_module(module_name) + ) + target_code = code + if entry.install_to_global: + for function_name in entry.function_names: + fn = types.FunctionType(code, module.__dict__, function_name) + self._install_global(module, function_name, fn) + if entry.code_source: + target_code = _lookup_code(entry) + + if entry.bypassed: + # If the entry is bypassed, do not install backends + # or guarded codes. + continue + + for backend_id in entry.backend_ids: + if backend_id not in backends: + raise RuntimeError( + f"Backend {backend_id} is not found in the given backends" + ) + with dynamo_timed( + "after_deserialization", phase_name="backend_compile" + ): + backend = backends[backend_id].after_deserialization() + self._install_global( + module, + backend_id, + torch._dynamo.disable(backend), + ) + + if len(entry.guarded_codes) == 0: + # Dynamo generates empty graph for trivial functions, should just skip them + # in these cases. + torch._dynamo.eval_frame.skip_code(target_code) + + for guarded_code in entry.guarded_codes: + with dynamo_timed("precompile_load_guards"): + guards_state = load_guards_state(guarded_code.guards_state) + runtime_global_scope = sys.modules[entry.python_module].__dict__ + # The installed builtins dict might be absent from the runtime + # while loading guards. Populate it if it's missing. + if ( + builtin_dict_name + := guards_state.output_graph.name_of_builtins_dict_key_in_fglobals + ): + builtins_dict = get_builtins_dict(runtime_global_scope) + if builtin_dict_name in runtime_global_scope: + assert ( + runtime_global_scope[builtin_dict_name] is builtins_dict + ) + else: + runtime_global_scope[builtin_dict_name] = builtins_dict + assert isinstance(guards_state, torch._dynamo.guards.GuardsState) + with dynamo_timed("precompile_build_guards"): + guard_manager = load_guard_manager( + guards_state, target_code, runtime_global_scope + ) + _load_precompile_entry( + target_code, + guard_manager, + SerializedCode.to_code_object(guarded_code.dynamo_code), + ) + + def cache_entry(self) -> _DynamoCacheEntry: + self.validate() + assert self._innermost_fn is not None + return _DynamoCacheEntry( + codes=list(self._codes.values()), + source_info=self._source_info, + device_type=self._device_type, + fn_name=self._innermost_fn.__qualname__, + fn_first_lineno=self._innermost_fn.__code__.co_firstlineno, + ) + + @staticmethod + def source_id_from_fn(fn: Callable[..., Any]) -> str: + from .eval_frame import innermost_fn + + innermost_fn_ = innermost_fn(fn) + + sha256_hash = hashlib.sha256() + sha256_hash.update(innermost_fn_.__qualname__.encode()) + sha256_hash.update(str(innermost_fn_.__code__.co_firstlineno).encode()) + return sha256_hash.hexdigest() + + +_Backends = dict[_BackendId, Any] + + +class DynamoStore(abc.ABC): + """ + A DynamoStore tracks active CompilePackages, and provides methods to store and retrieve them. + + This is an abstract base class for different storage implementations. + """ + + def record_package(self, package: CompilePackage) -> None: + """ + Records a package to PrecompileContext, so that it can be serialized later. + """ + from torch._dynamo.precompile_context import PrecompileContext + + cache_entry = package.cache_entry() + PrecompileContext.record_dynamo_cache_entry( + cache_entry=cache_entry, key=package.source_id + ) + + def record_eager_backend(self, backend_id: _BackendId, backend: Any) -> None: + """ + Records eager fx graphs to PrecompileContext for testing purposes. + """ + from torch._dynamo.precompile_context import ( + EagerCacheArtifact, + PrecompileContext, + ) + + result = EagerCacheArtifact(key=backend_id, content=backend) + PrecompileContext.record_artifact(result) + + @abc.abstractmethod + def clear(self) -> None: ... + + @abc.abstractmethod + def write( + self, + cache_entry: PrecompileCacheEntry, + path: str, + ) -> None: + """ + Abstract method to write dynamo cache entry and backends to storage. + + Args: + dynamo: The dynamo cache entry to write + backends: Dictionary of backend content to write + path: Path or key to identify where to write the data + """ + ... + + def save_cache_entry(self, cache_entry: _DynamoCacheEntry, key: str) -> None: + """ + Saves a package to a given path. Grabs backends from PrecompileContext. + """ + from torch._dynamo.precompile_context import ( + BackendCacheArtifact, + PrecompileContext, + ) + + backend_content: _Backends = {} + for backend_id in cache_entry.backend_ids: + serialized_backend = PrecompileContext.serialize_artifact_by_key(backend_id) + if serialized_backend is None: + raise RuntimeError( + f"Backend {backend_id} is not found in the given backends" + ) + assert isinstance(serialized_backend, BackendCacheArtifact) + backend_content[backend_id] = serialized_backend + + entry = PrecompileCacheEntry(cache_entry, backend_content) + + self.write(entry, key) + + def save_package(self, package: CompilePackage, key: str) -> None: + """ + Saves a package to a given path. Grabs backends from PrecompileContext. + """ + self.record_package(package) + cache_entry = package.cache_entry() + self.save_cache_entry(cache_entry, key) + + @abc.abstractmethod + def read(self, path: str) -> PrecompileCacheEntry: + """ + Abstract method to read dynamo cache entry and backends from storage. + + Args: + path: Path or key to identify where to read the data from + + Returns: + A tuple containing (dynamo_cache_entry, backend_content) + """ + ... + + def load_cache_entry(self, key: str) -> PrecompileCacheEntry: + from torch._dynamo.precompile_context import ( + BackendCacheArtifact, + PrecompileContext, + ) + + precompile_entry = self.read(key) + for backend in precompile_entry.backends.values(): + assert isinstance(backend, BackendCacheArtifact) + PrecompileContext.record_artifact(backend) + + return precompile_entry + + def load_package( + self, fn: Any, key: str + ) -> tuple[CompilePackage, dict[_BackendId, Any]]: + """ + Loads a package from a given path and returns it plus a list of deserialized backends + """ + entry = self.load_cache_entry(key) + package = CompilePackage(fn, entry.dynamo) + return package, entry.backends + + +class InMemoryDynamoStore(DynamoStore): + """ + A DynamoStore implementation that keeps state about CompilePackages in memory. + """ + + def __init__(self) -> None: + self.packages: dict[str, PrecompileCacheEntry] = {} + + def clear(self) -> None: + self.packages.clear() + + def write( + self, + entry: PrecompileCacheEntry, + path: str, + ) -> None: + """ + Store the dynamo cache entry and backends in memory instead of writing to disk. + """ + self.packages[path] = entry + + def read(self, path: str) -> PrecompileCacheEntry: + """ + Read dynamo cache entry and backends from memory. + """ + if path not in self.packages: + raise RuntimeError(f"No package found with key {path}") + + return self.packages[path] + + +class DiskDynamoStore(DynamoStore): + """ + A DynamoStore implementation that keeps state about CompilePackages on disk. + """ + + def __init__(self, path_prefix: str = ""): + """ + Initialize a DiskDynamoStore with a path prefix. + + Args: + path_prefix: Prefix directory for where to put CompilePackages on disk + """ + self._path_prefix = path_prefix + + def path_prefix(self) -> str: + return self._path_prefix + + def clear(self) -> None: + """ + Clear all CompilePackages from disk. + """ + if self.path_prefix(): + shutil.rmtree(self.path_prefix(), ignore_errors=True) + + def write( + self, + entry: PrecompileCacheEntry, + path: str, + ) -> None: + """ + Write dynamo cache entry and backends to disk. + """ + try: + pickled_content: bytes = pickle.dumps(entry) + CacheArtifactManager.record_artifact( + PrecompileCacheArtifact.type(), path, pickled_content + ) + self._write_to_local_cache(pickled_content, path) + except Exception as e: + raise RuntimeError(f"Failed to save package to {path}: {e}") from e + + def _write_to_local_cache(self, pickled_content: bytes, path: str) -> None: + from torch._inductor.codecache import write_atomic + + path = os.path.join(self.path_prefix(), path) if self.path_prefix() else path + try: + os.makedirs(path, exist_ok=True) + write_atomic(os.path.join(path, "entry"), pickled_content) + except Exception as e: + raise RuntimeError(f"Failed to save package to {path}: {e}") from e + + def read(self, path: str) -> PrecompileCacheEntry: + """ + Read dynamo cache entry and backends from disk. + """ + path = os.path.join(self.path_prefix(), path) if self.path_prefix() else path + try: + with open(os.path.join(path, "entry"), "rb") as f: + pickled_content = f.read() + entry = pickle.loads(pickled_content) + return entry + except Exception as e: + raise RuntimeError(f"Failed to load package from path {path}: {e}") from e + + +class DiskDynamoCache(DiskDynamoStore): + """ + Special DiskDynamoStore which adds some helper functions for automatically + tracking paths of packages + """ + + def save(self, package: CompilePackage) -> None: + """ + Saves a package to a given path. Grabs backends from PrecompileContext. + """ + key = package.source_id + logger.info("Saving CompilePackage for %s", package.source_id) + super().save_package(package, key) + + def load(self, fn: Callable[..., Any]) -> Optional[PrecompileCacheEntry]: + """ + Loads a package from a given path and returns it plus a list of deserialized backends + """ + key = CompilePackage.source_id_from_fn(fn) + logger.info("Loading CompilePackage for %s", key) + path = os.path.join(self.path_prefix(), key) + if os.path.exists(path): + try: + result = super().load_cache_entry(key) + counters["dynamo_cache"]["dynamo_cache_hit"] += 1 + return result + except Exception: + counters["dynamo_cache"]["dynamo_cache_error"] += 1 + logger.warning("Failed to load package from path %s", exc_info=True) + return None + logger.info("No package found for %s", key) + counters["dynamo_cache"]["dynamo_cache_miss"] += 1 + return None + + def load_and_install_package( + self, fn: Callable[..., Any] + ) -> Optional[CompilePackage]: + """ + Load directly into a package and install backends + """ + results = self.load(fn) + if results is None: + return None + else: + package = CompilePackage(fn, results.dynamo) + package.install(results.backends) + return package + + def path_prefix(self) -> str: + return os.path.join(cache_dir(), "dynamo") + + +def cache_dir() -> str: + from torch._inductor.runtime.cache_dir_utils import cache_dir + + return cache_dir() + + +DynamoCache = DiskDynamoCache(os.path.join(cache_dir(), "dynamo")) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/pgo.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/pgo.py new file mode 100644 index 0000000000000000000000000000000000000000..58cb5d2a521e632fde3302ea3b1c62cc50825bfd --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/pgo.py @@ -0,0 +1,1004 @@ +""" +Profile Guided Optimization (PGO) implementation for Dynamo. + +This module provides functionality for caching and managing code state profiles +that guide optimization decisions in Dynamo. It implements both local and remote +caching mechanisms for storing profile information across runs, handles profile +merging across distributed ranks, and manages the lifecycle of profile data +during compilation. The profiles track dynamic vs static properties of tensors +and help Dynamo make better specialization decisions. +""" + +from __future__ import annotations + +import base64 +import copy +import dataclasses +import enum +import functools +import logging +import os +import pickle +import re +import zlib +from collections import defaultdict +from typing import Optional, TYPE_CHECKING, TypeVar, Union +from typing_extensions import override, Self + +import torch._dynamo.config +import torch._utils_internal +import torch.compiler.config +import torch.distributed as dist +from torch._dynamo.utils import ( + CompileEventLogger, + dynamo_timed, + set_feature_use, + warn_once, +) +from torch._environment import is_fbcode +from torch._logging._internal import trace_structured_artifact +from torch.compiler._cache import ( + CacheArtifact, + CacheArtifactFactory, + CacheArtifactManager, +) +from torch.utils._ordered_set import OrderedSet + + +if TYPE_CHECKING: + import types + + from torch._dynamo.symbolic_convert import InstructionTranslator + from torch._inductor.remote_cache import JsonDataTy, RemoteCache + + +class ReservedWorkflowIdUserError(ValueError): + pass + + +log = logging.getLogger(__name__) + +LOCK_TIMEOUT = 10 + +# How does in memory representation work? Concretely, this module is +# responsible for holding GLOBAL state representing the state it holds, no +# other copies permitted. So we retire frame_state entirely and store it +# here. This should be reset when Dynamo is reset. We never GC information +# (similar to how the filesystem doesn't get cleaned up except by tmp +# cleaner), so the expectation is the information is relatively cheap and we +# don't mind leaking it. + + +# How exactly did we design the cache key? Here are some of the questions: +# +# - JOB_ID: Do we have a unique identifier for the "training run" (such that +# it stays the same if we're running the same code, and changes if we're +# running something different). +# +# - RANK: Are we sharing the cache across ranks, or does each rank get +# an individual cache? +# +# We choose to require job_id for PGO cache. This is to prevent +# situations where unrelated invocations of PyTorch unpredictably cause +# changes to each other's behavior. With a job_id, at least you know there +# is some "state" associated with it. (State dict might be another way to +# tell if a run is related or not.) You can opt-in to YOLO everything +# aliases everything by passing a shared job_id for all your invocations. +# +# We choose to NOT share PGO cache across ranks. With no RANK_SHARING, there +# is never contention between runs, so we can leisurely update a bundle with +# information we need. Because we are grouped by job_id, we can have a single +# consolidated bundle for everything (or not; maybe worry about O(n^2) IO if +# we updated every compile--let's just instrument this.) Can even take a +# filelock for extra safety (expect no contention); expect 50ns overhead from +# uncontended filelock. +# +# If we did share ranks, everyone is storming to modify the same cache files. +# We can do this by having folks atomic write to a CAS-store and then having +# readers do on-the-fly merging (this can be implemented in remote using +# prefix iteration). As an optional optimization, one rank can be elected to +# handling bundling post facto (ideally, this is done async, after quiescence, +# without compiler collective need to wait for everyone to finish writing +# their bits.) Not sure how you can avoid a listdir because if some rank shows +# up with some new entries we need to pull them in ASAP (unless you want to +# delay bundling). +# +# But compiler collectives fill a similar niche: compilers chat with each +# other so rank 0 has collected everything. So elect rank 0 only to write the +# bundle. Don't even need CAS-store atomic write; just one rank writing an +# updating bundles. The point is that use compiler collectives to share +# profiles across ranks, but use the PGO cache to persist profiles per rank +# across attempts. No need to have one mechanism to do everything. + + +@functools.cache +def _hash_containing_file(filepath: str) -> str: + # if the file does not exists we consider filepath to be the hash. + if not os.path.exists(filepath): + return filepath + + with open(filepath, "rb") as file: + content = file.read() + crc32_value = zlib.crc32(content) + hash = format(crc32_value & 0xFFFFFFFF, "08x") + return hash + + +@dataclasses.dataclass(frozen=True) +class CodeId: + filename: str + firstlineno: int + name: str + # When a job restart, the code can be copied to a different path than the previous attempt. In that case + # self.filename will have a different value, we do not want to consider those differences. Instead we + # hash the content of the file and use it as an identifier of the file. + # + # self.filename is kept in the object to give readable information/pointer to the actual file, in a local + # code state it will refer to the first seen file path. + file_hash: str + + # Exclude file name. + def __eq__(self, other: object) -> bool: + if not isinstance(other, CodeId): + return False + return ( + self.file_hash == other.file_hash + and self.firstlineno == other.firstlineno + and self.name == other.name + ) + + # Ensure if two CodeIds are the same, then they have the same hash by excluding filename. + def __hash__(self) -> int: + return hash((self.file_hash, self.name, self.firstlineno)) + + def __str__(self) -> str: + return f"hash({self.file_hash}){self.filename}:{self.firstlineno}:{self.name}" + + @staticmethod + def make(code: types.CodeType) -> CodeId: + return CodeId( + code.co_filename, + code.co_firstlineno, + code.co_name, + _hash_containing_file(code.co_filename), + ) + + +@dataclasses.dataclass +class CodeState: + automatic_dynamic: defaultdict[str, FrameStateSizeEntry] = dataclasses.field( + # pyrefly: ignore [unbound-name] + default_factory=lambda: defaultdict(FrameStateSizeEntry) + ) + + +_INIT_CODE_STATE: Optional[defaultdict[CodeId, CodeState]] = None +_CODE_STATE: Optional[defaultdict[CodeId, CodeState]] = None +_LOGGED_DYNAMIC_ALLOWLIST: bool = False +_KNOWN_DYNAMIC_SOURCES: set[str] = set() + + +@dataclasses.dataclass(frozen=True) +class InferStride: + """ + Denotes the quantity stride[dim] * size[dim], which is what the stride would + be for the next physical dimension that results in a contiguous layout. + + For example, given size = [2, 3], stride = [3, 1], we can replace this with + stride = [InferStride(1), 1], because InferStride(1) = stride[1] * size[1] = 1 * 3 = 3 + + Indirecting the representation in this way is important for the join operation + on strides as if we join [2, 3][3, 1] and [2, 4][4, 1], + we don't want [2, None][None, 1] which would get eventually symbolized into + [2, s0][s1, 1] (notice that the relationship between s0 and s1 is broken). + If we instead rewrite the expressions as InferStride so we have [2, 3][InferStride(1), 1] + and [2, 4][InferStride(1), 1] we now join to [2, None][InferStride(1), 1] will + result in [2, s0][s0, 1], as desired. + """ + + dim: int + + +_T = TypeVar("_T") + + +class AutoUnset(enum.Enum): + """ + The identity element of our semilattice, a generic "don't know" element that + is always subsumed when we get more information. + """ + + token = 0 + + +auto_unset = AutoUnset.token + + +class AutoDynamic(enum.Enum): + """ + The top element of our (bounded) semilattice, whenever you merge this with + any other element you always get it again + """ + + token = 0 + + +auto_dynamic = AutoDynamic.token + + +@dataclasses.dataclass +class FrameStateSizeEntry: + scalar: Union[int, AutoDynamic, AutoUnset] = dataclasses.field(default=auto_unset) + # NB: We don't have cases where we have a known dimensionality but + # we know NOTHING about the individual sizes + size: Union[AutoDynamic, AutoUnset, tuple[Union[int, AutoDynamic], ...]] = ( + dataclasses.field(default=auto_unset) + ) + stride: Union[ + AutoDynamic, AutoUnset, tuple[Union[int, AutoDynamic, InferStride], ...] + ] = dataclasses.field(default=auto_unset) + + def render(self) -> str: + # Special cases + def render_single(s: Union[int, AutoDynamic, AutoUnset, InferStride]) -> str: + if s is auto_dynamic: + return "?" + elif s is auto_unset: + # This basically shouldn't happen, this is for debugging + return "auto unset" + elif isinstance(s, InferStride): + return f"S({s.dim})" + else: + return str(s) + + def render_tuple(ss: tuple[Union[int, AutoDynamic, InferStride], ...]) -> str: + return "[" + ", ".join(render_single(s) for s in ss) + "]" + + # Common cases + if self.size is auto_dynamic and self.stride is auto_dynamic: + if self.scalar is auto_dynamic: + return "fully dynamic scalar or tensor" + else: + return f"scalar {self.scalar}" + elif self.scalar is auto_dynamic: + if isinstance(self.size, tuple) and isinstance(self.stride, tuple): + return f"tensor size={render_tuple(self.size)} stride={render_tuple(self.stride)}" + + # Fallback + return f"unusual {repr(self)}" + + def __post_init__(self) -> None: + assert not isinstance(self.scalar, torch.SymInt), self.scalar + if isinstance(self.size, tuple): + for s in self.size: + assert not isinstance(s, torch.SymInt), s + if isinstance(self.stride, tuple): + for s1 in self.stride: + assert not isinstance(s1, torch.SymInt), s1 + + def is_size_dynamic(self, dim: int) -> bool: + if self.size is auto_dynamic: + return True + if self.size is auto_unset: + return False + return self.size[dim] is auto_dynamic + + def is_stride_dynamic(self, dim: int) -> bool: + # At the moment, dynamic strides is a bit buggy. Good test case + # here is `PYTORCH_TEST_WITH_DYNAMO=1 python test/test_autograd.py + # TestAutograd.test_gradcheck_jacobian_mismatch` + # + # This if statement preserves historical behavior, which is that we + # ONLY make strides dynamic if the size is exactly static everywhere. + # We could potentially relax this but in general we should be very + # careful about when to infer dynamic strides. + # + # Actually, the existing algorithm is already somewhat problematic. + # Suppose a tensor that is sometimes: + # f32[2, 3, 5][15, 5, 1] and other times + # f32[2, 3, 5][5, 10, 1] (specifically, dim 0 and 1 are physically transposed). + # If we infer strides should be (DYNAMIC, DYNAMIC, 1). But this is + # silly: we really should have just guarded on dim order. + if not ( + isinstance(self.size, tuple) and all(type(s) is int for s in self.size) + ): + return False + if self.stride is auto_dynamic: + return True + if self.stride is auto_unset: + return False + return self.stride[dim] is auto_dynamic + + @staticmethod + def _munge_symint(xs: tuple[int, ...]) -> tuple[Union[AutoDynamic, int], ...]: + return tuple(auto_dynamic if isinstance(x, torch.SymInt) else x for x in xs) + + @classmethod + def make_scalar(cls, x: int) -> FrameStateSizeEntry: + return FrameStateSizeEntry(scalar=x, size=auto_dynamic, stride=auto_dynamic) + + @classmethod + def make_tensor( + cls, size: tuple[int, ...], stride: tuple[int, ...] + ) -> FrameStateSizeEntry: + return FrameStateSizeEntry( + scalar=auto_dynamic, + size=cls._munge_symint(size), + stride=cls._munge_symint(stride), + ) + + @classmethod + def make_size(cls, size: tuple[int, ...]) -> FrameStateSizeEntry: + return FrameStateSizeEntry( + scalar=auto_unset, + size=cls._munge_symint(size), + stride=auto_unset, + ) + + @staticmethod + def _merge_atom(x: _T, y: _T) -> Union[AutoDynamic, _T]: + if x is auto_unset: + return y + if y is auto_unset: + return x + if x is auto_dynamic or y is auto_dynamic or x != y: + return auto_dynamic + return x + + @classmethod + def _merge_atom_tup( + cls, + xs: Union[AutoDynamic, AutoUnset, tuple[_T, ...]], + ys: Union[AutoDynamic, AutoUnset, tuple[_T, ...]], + ) -> Union[AutoDynamic, AutoUnset, tuple[Union[AutoDynamic, _T], ...]]: + if xs is auto_unset: + return ys + if ys is auto_unset: + return xs + if xs is auto_dynamic or ys is auto_dynamic: + return auto_dynamic + if len(xs) != len(ys): + return auto_dynamic + return tuple(cls._merge_atom(x, y) for x, y in zip(xs, ys)) + + def __ior__(self, other: Self) -> Self: + self.scalar = self._merge_atom(self.scalar, other.scalar) + self.size = self._merge_atom_tup(self.size, other.size) + self.stride = self._merge_atom_tup(self.stride, other.stride) + return self + + +def update_automatic_dynamic( + tx: InstructionTranslator, + name: str, + entry: FrameStateSizeEntry, + *, + is_unspecialized_nn_module: bool = False, +) -> FrameStateSizeEntry: + code_id = CodeId.make(tx.f_code) + frame_state = get_code_state()[code_id] + if torch._dynamo.config.automatic_dynamic_shapes: + is_update = name in frame_state.automatic_dynamic + mut_entry = frame_state.automatic_dynamic[name] + old_entry = copy.copy(mut_entry) + mut_entry |= entry + + # Do some logs (damn, I spend more code logging than I do actually doing + # the updates lol) + if is_update and old_entry.scalar != mut_entry.scalar: + log.debug( + "automatic dynamic int %s val %s != %s", + name, + entry.scalar, + old_entry.scalar, + ) + CompileEventLogger.instant( + "automatic_dynamic", + { + "name": name, + "dim_changed": "scalar", + "reason": "scalar change", + "cached": str(old_entry.scalar), + "new": str(entry.scalar), + }, + ) + if is_unspecialized_nn_module: + log.info( + "%s is converted to a symbolic integer. It is an attribute of a " + "user defined nn module class. If you wish to keep it static, you can " + "mark the nn module class as `torch._dynamo.mark_static`.", + name, + ) + + def log_tup( + tup_name: str, short_reason: str, long_reason: str, i: Optional[int] = None + ) -> None: + entry_tup = ( + getattr(entry, tup_name) if i is None else getattr(entry, tup_name)[i] + ) + old_entry_tup = ( + getattr(old_entry, tup_name) + if i is None + else getattr(old_entry, tup_name)[i] + ) + log.debug( + "automatic dynamic %s %s %s %s != %s", + tup_name, + name, + short_reason, + # NB: We used to only report len(...) here for dim mismatch + entry_tup, + old_entry_tup, + ) + CompileEventLogger.instant( + "automatic_dynamic", + { + "name": name, + "dim_changed": "all" if i is None else i, + "reason": long_reason, + "cached": str(old_entry_tup), + "new": str(entry_tup), + }, + ) + + if is_update and old_entry.size != mut_entry.size: + if isinstance(old_entry.size, tuple) and isinstance(entry.size, tuple): + if len(old_entry.size) != len(entry.size): + log_tup("size", "dim", "dimensionality change") + else: + for i in range(len(entry.size)): + if old_entry.size[i] != entry.size[i]: + log_tup("size", f"size({i})", "size change", i) + else: + log_tup("size", "other", "other") + + if is_update and old_entry.stride != mut_entry.stride: + if isinstance(old_entry.stride, tuple) and isinstance(entry.stride, tuple): + if len(old_entry.stride) != len(entry.stride): + log_tup("stride", "dim", "dimensionality change") + else: + for i in range(len(entry.stride)): + if old_entry.stride[i] != entry.stride[i]: + log_tup("stride", f"stride({i})", "stride change", i) + else: + log_tup("stride", "other", "other") + else: + old_entry = frame_state.automatic_dynamic[name] + log.debug( + "automatic dynamic is off, overwriting int %s val %s -> %s", + name, + old_entry.scalar, + entry.scalar, + ) + frame_state.automatic_dynamic[name] = entry + mut_entry = entry + + return mut_entry + + +def process_automatic_dynamic( + tx: InstructionTranslator, + name: str, + entry: FrameStateSizeEntry, + *, + is_unspecialized_nn_module: bool = False, +) -> FrameStateSizeEntry: + if (st := tx.distributed_state) is None: + return update_automatic_dynamic( + tx, + name, + entry, + is_unspecialized_nn_module=is_unspecialized_nn_module, + ) + elif st.all_states is None: + # Preflight, always pretend as if it's static. The point here + # is we want to get through the preflight quickly, and static + # will run faster. The preexisting frame state will get + # applied anyway after we do compiler collectives. + # TODO: I'm not sure if we should just bong the entire pgo + # state here, it kind of depends if we're going to have other + # things that talk in compiler collective. Also, the PGO + # state, if we've already inferred something is automatic + # dynamic, will have lost the actual input sizes, which might + # be useful for debugging purposes (e.g., observing 0/1 + # specialization). Bonging the entire PGO state here would + # let us delete this logic here; the compiler collective + # would just directly update_automatic_dynamic + st.local_state.automatic_dynamic[name] = entry + return entry + else: + # Apply the updates. NB: all_states includes the local state + # too. + res = None + for sub_state in st.all_states: + if name in sub_state.automatic_dynamic: + res = update_automatic_dynamic( + tx, + name, + sub_state.automatic_dynamic[name], + is_unspecialized_nn_module=is_unspecialized_nn_module, + ) + assert res is not None + return res + + +def format_cache_key(key: str) -> str: + # NB: We always use global rank for keys, even though they are overkill + # for local only cache + rank = None + if dist.is_available() and dist.is_initialized(): + rank = dist.get_rank() + + tag = torch.compiler.config.cache_key_tag + return f"{key}:{rank}:{tag}" + + +def get_cache_key() -> Optional[str]: + # TODO: info versions of these logs that log only once + if torch.compiler.config.force_disable_caches: + warn_once( + "dynamo_pgo force disabled by torch.compiler.config.force_disable_caches" + ) + return None + + # NB: We namespace the cache keys so that only user-specified job id + # can alias with each other. + if (r := torch.compiler.config.job_id) is not None: + if r.startswith("mast:"): + raise ReservedWorkflowIdUserError( + "torch.compiler.config.job_id with prefix 'mast:' is reserved for " + "automatically generated job id associated with a specific MAST job " + "name and version." + ) + return format_cache_key(r) + + if (name_version := torch._utils_internal.get_mast_job_name_version()) is not None: + mast_job_name, mast_job_version = name_version + return format_cache_key(f"mast:{mast_job_name}:{mast_job_version}") + + return None + + +def get_extra_cache_key(sticky_key: str) -> Optional[str]: + if torch.compiler.config.force_disable_caches: + warn_once( + "dynamo_pgo force disabled by torch.compiler.config.force_disable_caches" + ) + return None + + return format_cache_key(sticky_key) + + +# This solely controls local PGO +def code_state_path(cache_key: str) -> Optional[str]: + if not torch._dynamo.config.automatic_dynamic_local_pgo: + log.debug("automatic_dynamic_local_pgo not enabled") + return None + + from torch._inductor.runtime.runtime_utils import cache_dir + + code_state_key = re.sub(r'[<>:"/\\|?*]', "_", f"code_state_{cache_key}.pkl") + return os.path.join(cache_dir(), "dynamo", code_state_key) + + +def should_use_remote_dynamo_pgo_cache() -> bool: + if torch.compiler.config.force_disable_caches: + return False + + if (r := torch._dynamo.config.automatic_dynamic_remote_pgo) is not None: + return r + + if not is_fbcode(): + return False + + if torch._utils_internal.is_fb_unit_test(): + return False + + try: + from torch._inductor.fb.remote_cache import REMOTE_CACHE_VERSION + except ModuleNotFoundError: + return False + + return REMOTE_CACHE_VERSION >= torch._utils_internal.justknobs_getval_int( + "pytorch/remote_cache:dynamo_pgo_version" + ) + + +def get_remote_cache() -> Optional[RemoteCache[JsonDataTy]]: + from torch._inductor.remote_cache import create_cache + + if not should_use_remote_dynamo_pgo_cache(): + return None + + return create_cache( + "dynamo-pgo", + is_fbcode(), + "FbRemoteDynamoPGOCache", + "RemoteDynamoPGOCache", + ) + + +def _collect_dynamic_sources(code_state: CodeState) -> OrderedSet[str]: + dynamic_sources: OrderedSet[str] = OrderedSet() + for src, fs in code_state.automatic_dynamic.items(): + dynamic = False + if isinstance(fs.size, tuple): + dynamic = auto_dynamic in fs.size # type: ignore[operator] + elif fs.scalar == auto_dynamic: + dynamic = True + if dynamic: + dynamic_sources.add(src) + return dynamic_sources + + +def _collect_missing_sources(all_sources: OrderedSet[str]) -> OrderedSet[str]: + from torch._dynamo.variables.builder import is_dynamic_source + + global _KNOWN_DYNAMIC_SOURCES + missing_sources: OrderedSet[str] = OrderedSet() + for src in all_sources: + if src in _KNOWN_DYNAMIC_SOURCES: + continue + elif is_dynamic_source(src): + _KNOWN_DYNAMIC_SOURCES.add(src) + continue + missing_sources.add(src) + return missing_sources + + +def log_frame_dynamic_whitelist(f_code: types.CodeType) -> None: + global _KNOWN_DYNAMIC_SOURCES + code_id = CodeId.make(f_code) + frame_state = get_code_state()[code_id] + all_dynamic_sources = _collect_dynamic_sources(frame_state) + frame_whitelist = ",".join(all_dynamic_sources) + missing_whitelist = ",".join(_collect_missing_sources(all_dynamic_sources)) + if frame_whitelist: + with dynamo_timed(name := "pgo.dynamic_whitelist", log_pt2_compile_event=True): + CompileEventLogger.pt2_compile( + name, + recompile_dynamic_whitelist=frame_whitelist, + missing_dynamic_whitelist=missing_whitelist, + ) + + +def _log_size_mismatch_recompile() -> None: + global _LOGGED_DYNAMIC_ALLOWLIST + if not _LOGGED_DYNAMIC_ALLOWLIST: + torch._utils_internal.add_mlhub_insight( + category="dynamic_shapes_analysis", + insight="Dynamic shape recompilation detected", + insight_description="PGO detected a recompilation due to dynamic shapes. \ + Please follow the instruction from the action link to reduce \ + recompilation overhead.", + ) + # add mlhub insight only once per rank + _LOGGED_DYNAMIC_ALLOWLIST = True + + +def render_code_state(cs: defaultdict[CodeId, CodeState]) -> str: + code_state_str = "\n".join( + f"{k}:\n" + + "\n".join( + f" {src}: {fs.render()}" for src, fs in v.automatic_dynamic.items() + ) + for k, v in cs.items() + ) + dynamic_sources: OrderedSet[str] = OrderedSet() + for state in cs.values(): + dynamic_sources.update(_collect_dynamic_sources(state)) + if dynamic_sources: + code_state_str += ( + "\n\nPGO detected a recompilation due to dynamic shapes. " + "To reduce shape recompilations by compiling dynamically to start, " + f'set environment variable TORCH_COMPILE_DYNAMIC_SOURCES="{",".join(dynamic_sources)}"' + ) + return code_state_str + + +@CacheArtifactFactory.register +class PGOCacheArtifact(CacheArtifact): + @override + def populate_cache(self) -> None: + meta = write_local_impl( + self._rewrite_cache_key_for_mega_cache(self.key), self.content + ) + assert meta is not None + + @override + @staticmethod + def type() -> str: + return "pgo" + + @staticmethod + def _rewrite_cache_key_for_mega_cache(original_key: str) -> str: + """ + The PGO cache artifact key for a MAST job contains the job name and the version. + When we want to use the cache artifact on a different MAST job, we need to + update the key to use the new MAST job's name and version. + """ + if not original_key.startswith("mast:"): + # if original_key is overridden, then dont change it + return original_key + if (new_key := get_cache_key()) is not None: + return new_key + return original_key + + +def hit(key: str, ty: str) -> defaultdict[CodeId, CodeState]: + global _INIT_CODE_STATE + assert isinstance(_CODE_STATE, defaultdict) + log.info("get_code_state %s hit %s, %d entries", key, ty, len(_CODE_STATE)) + trace_structured_artifact( + f"get_{ty}_code_state", + "string", + lambda: render_code_state(_CODE_STATE), # type: ignore[arg-type] + ) + set_feature_use("pgo", True) + _INIT_CODE_STATE = copy.deepcopy(_CODE_STATE) + return _CODE_STATE + + +def get_local_code_state(cache_key: str) -> Optional[defaultdict[CodeId, CodeState]]: + global _CODE_STATE + path = code_state_path(cache_key) + if path is not None and os.path.exists(path): + with dynamo_timed( + name := "pgo.get_local_code_state", log_pt2_compile_event=True + ): + CompileEventLogger.pt2_compile(name, cache_key=cache_key) + # Read lock not necessary as we always write atomically write to + # the actual location + with open(path, "rb") as f: + try: + content = f.read() + _CODE_STATE = pickle.loads(content) + CompileEventLogger.pt2_compile(name, cache_size_bytes=f.tell()) + except Exception: + log.warning( + "get_code_state failed while reading %s", path, exc_info=True + ) + else: + CacheArtifactManager.record_artifact( + PGOCacheArtifact.type(), cache_key, content + ) + return hit(path, "local") + return None + + +def lookup_remote_cache_entry( + remote_cache: RemoteCache[JsonDataTy], + cache_key: str, + event_name: Optional[str] = None, +) -> Optional[defaultdict[CodeId, CodeState]]: + code_state = None + try: + cache_data = remote_cache.get(cache_key) + except Exception: + log.warning("get_code_state failed remote read on %s", cache_key, exc_info=True) + else: + if cache_data is not None: + try: + assert isinstance(cache_data, dict) + data = cache_data["data"] + assert isinstance(data, str) + payload = base64.b64decode(data) + if event_name is not None: + CompileEventLogger.pt2_compile( + event_name, cache_size_bytes=len(payload) + ) + code_state = pickle.loads(payload) + except Exception: + log.warning( + "get_code_state failed parsing remote result on %s", + cache_key, + exc_info=True, + ) + else: + CacheArtifactManager.record_artifact( + PGOCacheArtifact.type(), cache_key, payload + ) + else: + log.info("get_code_state remote miss on %s", cache_key) + return code_state + + +def get_remote_code_state(cache_key: str) -> Optional[defaultdict[CodeId, CodeState]]: + global _CODE_STATE + remote_cache = get_remote_cache() + if remote_cache is not None: + with dynamo_timed( + name := "pgo.get_remote_code_state", + log_pt2_compile_event=True, + dynamo_compile_column_us="pgo_get_remote_code_state_time_us", + ): + CompileEventLogger.pt2_compile(name, cache_key=cache_key) + code_state = lookup_remote_cache_entry(remote_cache, cache_key, name) + if code_state is not None: + _CODE_STATE = code_state + return hit(cache_key, "remote") + return None + + +def get_extra_remote_code_state(cache_key: str) -> None: + """ + Reads an additional PGO profile from the given cache key, and merges it with the default PGO profile. + """ + global _CODE_STATE + assert _CODE_STATE is not None + + remote_cache = get_remote_cache() + if remote_cache is not None: + with dynamo_timed( + name := "pgo.get_extra_remote_code_state", + log_pt2_compile_event=True, + dynamo_compile_column_us="pgo_get_remote_code_state_time_us", + ): + CompileEventLogger.pt2_compile(name, cache_key=cache_key) + code_state = lookup_remote_cache_entry(remote_cache, cache_key) + log.info( + "get_extra_code_state %s hit, %d entries", + cache_key, + len(code_state) if code_state is not None else 0, + ) + if code_state is not None: + assert not _CODE_STATE + _CODE_STATE = code_state + # log to tlparse + trace_structured_artifact( + "get_extra_remote_code_state", + "string", + lambda: render_code_state(code_state), + ) + + +def get_code_state() -> defaultdict[CodeId, CodeState]: + global _CODE_STATE, _INIT_CODE_STATE + if _CODE_STATE is not None: + return _CODE_STATE + + # Initialize it (even if we don't look up profile) + _CODE_STATE = defaultdict(CodeState) + + cache_key = get_cache_key() + if cache_key is None: + return _CODE_STATE + + # Attempt local + local_code_state = get_local_code_state(cache_key) + + # Attempt remote + if local_code_state is None: + get_remote_code_state(cache_key) + + # Attempt additional remote if neither local/default remote succeeded + if ( + not _CODE_STATE + and (sticky_read := torch.compiler.config.pgo_extra_read_key) is not None + ): + extra_read_key = get_extra_cache_key(sticky_read) + if extra_read_key is not None: + get_extra_remote_code_state(extra_read_key) + + log.info("get_code_state using default") + + assert _CODE_STATE is not None + return _CODE_STATE + + +def put_code_state() -> None: + if _CODE_STATE is None: + log.info("put_code_state: never initialized, will not write") + return + + if _CODE_STATE == _INIT_CODE_STATE: + log.info("put_code_state: no change, skipping") + return + + cache_key = get_cache_key() + if cache_key is None: + log.info("put_code_state: no cache key, skipping") + return + + put_local_code_state(cache_key) + put_remote_code_state(cache_key) + if (sticky_write := torch.compiler.config.pgo_extra_write_key) is not None: + extra_write_key = get_extra_cache_key(sticky_write) + if extra_write_key is not None: + put_remote_code_state(extra_write_key) + + +def write_local_impl(cache_key: str, pickled_code: bytes) -> Optional[tuple[str, int]]: + path = code_state_path(cache_key) + + if path is None: + return None + + # If the user isn't misusing our API, we should have exclusive access to + # this directory. But it's not too hard + + tmp_path = path + ".tmp" + lock_path = path + ".lock" + # We /mostly/ don't need the lock but the tmp file could be clobbered + # TODO: use a safe tempfile create to eliminate lock + from torch.utils._filelock import FileLock + + os.makedirs(os.path.dirname(path), exist_ok=True) + + with FileLock(lock_path, timeout=LOCK_TIMEOUT): + with open(tmp_path, "wb") as f: + f.write(pickled_code) + size = f.tell() + os.replace(tmp_path, path) + return path, size + + +def put_local_code_state(cache_key: str) -> None: + with dynamo_timed(name := "pgo.put_local_code_state", log_pt2_compile_event=True): + CompileEventLogger.pt2_compile(name, cache_key=cache_key) + assert _CODE_STATE is not None + + pickled_code = pickle.dumps(_CODE_STATE) + + CacheArtifactManager.record_artifact( + PGOCacheArtifact.type(), cache_key, pickled_code + ) + + meta = write_local_impl(cache_key, pickled_code) + if meta is None: + log.info("put_code_state: local cache disabled") + return + path, size = meta + + CompileEventLogger.pt2_compile(name, cache_size_bytes=size) + log.info("put_code_state: wrote local %s, %d entries", path, len(_CODE_STATE)) + trace_structured_artifact( + "put_local_code_state", + "string", + lambda: render_code_state(_CODE_STATE), + ) + + +def put_remote_code_state(cache_key: str, extra_code_state: bool = False) -> None: + event_name = ( + "put_remote_code_state" + if not extra_code_state + else "put_extra_remote_code_state" + ) + with dynamo_timed( + name := f"pgo.{event_name}", + log_pt2_compile_event=True, + dynamo_compile_column_us="pgo_put_remote_code_state_time_us", + ): + CompileEventLogger.pt2_compile(name, cache_key=cache_key) + assert _CODE_STATE is not None + + remote_cache = get_remote_cache() + + if remote_cache is None: + log.info("%s: remote cache disabled", event_name) + return + + content = pickle.dumps(_CODE_STATE) + CompileEventLogger.pt2_compile(name, cache_size_bytes=len(content)) + cache_data: JsonDataTy = { + "data": base64.b64encode(content).decode("ascii"), + } + remote_cache.put(cache_key, cache_data) + log.info( + "%s: wrote remote %s, %d entries", event_name, cache_key, len(_CODE_STATE) + ) + # TODO: don't log this multiple times + trace_structured_artifact( + event_name, + "string", + lambda: render_code_state(_CODE_STATE), + ) + + +# NB: this does NOT reset the cached code state on disk +def reset_code_state() -> None: + global _CODE_STATE, _INIT_CODE_STATE, _LOGGED_DYNAMIC_ALLOWLIST + _CODE_STATE = None + _INIT_CODE_STATE = None + _LOGGED_DYNAMIC_ALLOWLIST = False diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/precompile_context.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/precompile_context.py new file mode 100644 index 0000000000000000000000000000000000000000..bae360041b58c430f104f6d786f52f1b3a2eed9a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/precompile_context.py @@ -0,0 +1,231 @@ +import copy +import json +import logging +from abc import abstractmethod +from collections import defaultdict +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any, Generic, Optional, TypeVar + +import torch +from torch._dynamo.package import ( + _BackendId, + _DynamoCacheEntry, + DynamoCache, + PrecompileCacheEntry, +) + + +""" +Classes and implementations related to precompile +""" + +T = TypeVar("T") +logger = logging.getLogger(__name__) + + +@dataclass +class BackendCacheArtifact(Generic[T]): + """ + Represents a single serializable backend artifact from a dynamo backend. + Each BackendCacheArtifact has a key associated with it along with some + serializable content. + + Example implementation: + + class MyPrecompileCacheArtifact(PrecompileCacheArtifact[MySerializableType]): + my_field: int + + def after_deserialization(self) -> MySerializableType: + result = pickle.loads(self.content) + # Do some extra work post deserialization + result.my_post_deserialization_function(self.my_field) + return result + """ + + key: str + content: Any + + @abstractmethod + def after_deserialization(self) -> T: + """ + Code to be run after reading raw byte contents from disk. + Generally converts self.content from raw bytes back into its original form. + """ + ... + + def edit_contents(self, edit_fn: Callable[..., Any]) -> None: + """ + Edit the contents of the artifact. + """ + self.content = edit_fn(self.content) + + +class EagerCacheArtifact(BackendCacheArtifact[Any]): + def after_deserialization(self) -> Any: + return self.content + + +class BypassDynamoCacheEntry(Exception): + pass + + +class PrecompileContext: + """ + PrecompileContext is a special CacheArtifactManager for handling precompilation + It uses the same interface as CacheArtifactManager, but handles deserialization differently: instead + of placing each artifact into respective caches, it will stitch all the cache artifacts for a single key + together and place it into a global Precompile Cache. + + PrecompileContext has two main portions: dynamo_cache_entries and backend_cache_artifacts. + When saving, PrecompileContext.serialize() will serialize all dynamo cache entries along with any PrecompileCacheArtifacts that + are needed to save those dynamo cache entries. + + The following artifact types are supported by PrecompileContext: + - BundledAOTAutogradCacheArtifact + + """ + + # Protected by the compile_lock + # _backend_artifacts_by_key organizes results by the key of each artifact. + # Each object here must be serializable + _backend_artifacts_by_key: dict[_BackendId, BackendCacheArtifact[Any]] = {} + + # On call to `serialize()`, all cache artifacts in _dynamo_cache_entries are converted + # into DynamoCacheArtifacts and added to _new_cache_artifacts for serialization + _dynamo_cache_entries: dict[str, _DynamoCacheEntry] = {} + + @classmethod + def clear(cls) -> None: + cls._backend_artifacts_by_key.clear() + cls._dynamo_cache_entries.clear() + + @classmethod + def record_artifact( + cls, + artifact: BackendCacheArtifact[Any], + ) -> None: + """ + Records a backend artifact to be used with dynamo cache entries + """ + # Temporarily disable all dispatch modes (including FakeTensorMode) during + # deepcopy to avoid issues with cloning fake tensors (e.g., device mesh + # with meta tensors that fail when cloning due to device mismatches) + from torch.utils._mode_utils import no_dispatch + + with no_dispatch(): + cls._backend_artifacts_by_key[_BackendId(artifact.key)] = copy.deepcopy( + artifact + ) + + @classmethod + def record_dynamo_cache_entry( + cls, cache_entry: _DynamoCacheEntry, key: str + ) -> None: + cls._dynamo_cache_entries[key] = cache_entry + + @classmethod + def edit_artifact(cls, key: str, edit_fn: Callable[..., Any]) -> None: + """ + Edit the content of an existing artifact + """ + assert key in cls._backend_artifacts_by_key, f"Key {key} not found in artifacts" + artifact = cls._backend_artifacts_by_key[_BackendId(key)] + artifact.edit_contents(edit_fn) + + @classmethod + def serialize_artifact_by_key(cls, key: str) -> Optional[BackendCacheArtifact[Any]]: + """ + Return the backend cache artifact with the associated key + """ + return cls._backend_artifacts_by_key.get(_BackendId(key), None) + + @staticmethod + def dump_debug_info( + dynamo_entries: dict[str, _DynamoCacheEntry], + backend_artifacts: dict[_BackendId, BackendCacheArtifact[Any]], + ) -> dict[str, Any]: + """ + Return a JSON serializable debug dump of all entries in the precompile context + Called in serialize before serialization, and in populate_caches after deserialization + """ + # Print debug information + debug_info: defaultdict[str, list[Any]] = defaultdict(list) + for key, cache_entry in dynamo_entries.items(): + info = cache_entry.debug_info() + info["key"] = key + debug_info["dynamo"].append(info) + + for artifact in backend_artifacts.values(): + debug_info["backends"].append(artifact.key) + + return debug_info + + @classmethod + def save_to_dynamo_cache(cls) -> dict[str, Any]: + precompile_cache_entries, debug_info = cls.create_cache_entries() + for key, entry in precompile_cache_entries.items(): + DynamoCache.write(entry, key) + return debug_info + + @classmethod + def create_cache_entries( + cls, + ) -> tuple[dict[str, PrecompileCacheEntry], dict[str, Any]]: + """ + Grabs all the cache entries in the precompile context and + stitches them together into full PrecompileCacheEntries. + """ + dynamo_entries = cls._dynamo_cache_entries + backend_artifacts = cls._backend_artifacts_by_key + + num_artifacts = len(dynamo_entries) + + debug_info = PrecompileContext.dump_debug_info( + dynamo_entries, backend_artifacts + ) + debug_str = json.dumps( + { + "num_entries": num_artifacts, + "artifacts": debug_info, + }, + ) + torch._logging.trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "dynamo_cache_entries", + "encoding": "json", + }, + payload_fn=lambda: debug_str, + expect_trace_id=False, + ) + + precompile_cache_entries = {} + + for key, cache_entry in dynamo_entries.items(): + try: + result = PrecompileCacheEntry.from_cache_entry( + cache_entry, backend_artifacts + ) + if result is not None: + precompile_cache_entries[key] = result + except Exception as e: + logger.warning("Failed to create cache entry %s", key, exc_info=True) + + error = e + data = json.dumps( + { + "key": key, + "error": str(error), + } + ) + torch._logging.trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "dynamo_cache_exception", + "encoding": "json", + }, + payload_fn=lambda: data, + ) + continue + return precompile_cache_entries, debug_info diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/profiler.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..3c0a943eb7a05b08b4209fdaf071b7c37f70fc4b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/profiler.py @@ -0,0 +1,177 @@ +""" +Dynamo profiling implementation. + +This module provides profiling functionality for Dynamo, including: +- ProfileMetrics: Class for collecting and aggregating performance metrics like + execution time, operator counts, and fusion statistics +- ProfileResult: Class for analyzing and reporting profiling results +- Utilities for tracking missed/uncaptured operations +- Functions for instrumenting FX graphs with profiling capabilities + +The profiler helps measure and optimize the performance of Dynamo-compiled code +by tracking both captured and total operations, timing, and graph statistics. +""" + +from __future__ import annotations + +import dataclasses +import os +from typing import Any +from typing_extensions import Self + +import torch + +from .utils import print_once + + +@dataclasses.dataclass +class ProfileMetrics: + microseconds: float = 0.0 + operators: int = 0 + fusions: int = 0 + graphs: int = 0 + + def __iadd__(self, other: Self) -> Self: + self.microseconds += other.microseconds + self.operators += other.operators + self.fusions += other.fusions + return self + + def __add__(self, other: ProfileMetrics) -> ProfileMetrics: + assert isinstance(other, ProfileMetrics) + return ProfileMetrics( + self.microseconds + other.microseconds, + self.operators + other.operators, + self.fusions + other.fusions, + ) + + def __truediv__(self, other: Any) -> ProfileMetrics: + if isinstance(other, int): + other = ProfileMetrics(other, other, other) + return ProfileMetrics( + # pyrefly: ignore [no-matching-overload] + self.microseconds / max(1, other.microseconds), + # pyrefly: ignore [bad-argument-type] + self.operators / max(1, other.operators), + # pyrefly: ignore [bad-argument-type] + self.fusions / max(1, other.fusions), + ) + + def __str__(self) -> str: + return f"{self.operators:4.0%} ops {self.microseconds:4.0%} time" + + def tocsv(self) -> list[float]: + return [self.operators, self.microseconds] + + +class ProfileResult: + def __init__( + self, captured: ProfileMetrics, total: ProfileMetrics, unique_graphs: int + ) -> None: + self.captured: ProfileMetrics = captured or ProfileMetrics() + self.total: ProfileMetrics = total or ProfileMetrics() + self.unique_graphs: int = unique_graphs + + def __iadd__(self, other: Self) -> Self: + self.captured += other.captured + self.total += other.total + self.unique_graphs += other.unique_graphs + return self + + def percent(self) -> ProfileMetrics: + return self.captured / self.total + + def __str__(self) -> str: + return ( + f"{self.unique_graphs:2} graphs {self.captured.graphs:2} graph calls " + f"{self.captured.operators:4}/{self.total.operators:4} = " + + str(self.percent()) + ) + + def tocsv(self) -> list[Any]: + return [ + self.unique_graphs, + self.captured.graphs, + self.captured.operators, + self.total.operators, + ] + self.percent().tocsv() + + +def should_print_missing() -> bool: + return os.environ.get("TORCHDYNAMO_PRINT_MISSING") == "1" + + +def print_missing(stack: list[str]) -> None: + if any("/torch/autograd/profiler.py" in x for x in stack): + return + stack = [ + x for x in stack if ("> ".join(stack[-3:])) + + +class Profiler: + unique_graphs: int = 0 + + def __init__(self) -> None: + self.prof = torch.profiler.profile( + activities=[torch.profiler.ProfilerActivity.CPU], + with_stack=should_print_missing(), + ) + + def results(self) -> ProfileResult: + captured_regions = 0 + captured_ops = 0 + captured_microseconds = 0 + total_ops = 0 + total_microseconds = 0 + + last_op_end_time = -1 + captured_region_end_time = -1 + events = sorted(self.prof.events(), key=lambda x: x.time_range.start) + for e in events: + if e.name == "TORCHDYNAMO": + captured_region_end_time = e.time_range.end + captured_regions += 1 + # ignore `handle = torch.zeros(1)` in record_function.__init__() + total_ops -= 1 + elif e.time_range.start >= last_op_end_time: + last_op_end_time = e.time_range.end + if e.time_range.end <= captured_region_end_time: + captured_ops += 1 + captured_microseconds += e.time_range.elapsed_us() + elif should_print_missing(): + print_missing(e.stack) + total_ops += 1 + total_microseconds += e.time_range.elapsed_us() + else: + pass # ops recursively called from other ops (ignored) + + unique_graphs = Profiler.unique_graphs + Profiler.unique_graphs = 0 + # we counted one extra op that is part of the profiler setup code + total_ops -= 1 + + return ProfileResult( + captured=ProfileMetrics( + microseconds=captured_microseconds, + operators=captured_ops, + fusions=captured_ops - captured_regions, + graphs=captured_regions, + ), + total=ProfileMetrics( + microseconds=total_microseconds, + operators=total_ops, + fusions=total_ops - 1, + ), + unique_graphs=unique_graphs, + ) + + +def fx_insert_profiling(gm: torch.fx.GraphModule, example_inputs: list[Any]) -> Any: + def _wrapped(*args: Any) -> Any: + with torch.profiler.record_function("TORCHDYNAMO"): + return gm.forward(*args) + + Profiler.unique_graphs += 1 + return _wrapped diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/replay_record.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/replay_record.py new file mode 100644 index 0000000000000000000000000000000000000000..5d01217fdbb6139dddf203a931599c4de4b532c6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/replay_record.py @@ -0,0 +1,130 @@ +""" +Python execution state recording and replay functionality. + +This module provides mechanisms for capturing and replaying Python execution state: + +- ModuleRecord: Tracks module access patterns and attribute usage +- DummyModule: Lightweight module substitute for replay +- ExecutionRecord: Manages execution context including globals, locals and builtins +- ExecutionRecorder: Records variable states and module access during execution + +The module enables serialization and reproduction of Python execution environments, +particularly useful for debugging and testing frameworks that need to capture +and recreate specific program states. +""" + +import dataclasses +from dataclasses import field +from io import BufferedReader, BufferedWriter +from types import CellType, CodeType, ModuleType +from typing import Any, IO, Union +from typing_extensions import Self + +from torch.utils._import_utils import import_dill + + +dill = import_dill() + + +@dataclasses.dataclass +class ModuleRecord: + module: ModuleType + accessed_attrs: dict[str, Any] = field(default_factory=dict) + + +@dataclasses.dataclass +class DummyModule: + name: str + is_torch: bool = False + value: object = None + + @property + def __name__(self) -> str: + return self.name + + +@dataclasses.dataclass +class ExecutionRecord: + code: CodeType + closure: tuple[CellType] + globals: dict[str, Any] = field(default_factory=dict) + locals: dict[str, Any] = field(default_factory=dict) + builtins: dict[str, Any] = field(default_factory=dict) + code_options: dict[str, Any] = field(default_factory=dict) + + def dump(self, f: Union[IO[str], BufferedWriter]) -> None: + assert dill is not None, "replay_record requires `pip install dill`" + dill.dump(self, f) + + @classmethod + def load(cls, f: Union[IO[bytes], BufferedReader]) -> Self: + assert dill is not None, "replay_record requires `pip install dill`" + return dill.load(f) + + +@dataclasses.dataclass +class ExecutionRecorder: + LOCAL_MOD_PREFIX = "___local_mod_" + + code: CodeType + closure: tuple[CellType] + globals: dict[str, Any] = field(default_factory=dict) + locals: dict[str, Any] = field(default_factory=dict) + builtins: dict[str, Any] = field(default_factory=dict) + code_options: dict[str, Any] = field(default_factory=dict) + name_to_modrec: dict[str, ModuleRecord] = field(default_factory=dict) + + def add_local_var(self, name: str, var: Any) -> None: + if isinstance(var, ModuleType): + self.locals[name] = self._add_mod(var) + else: + self.locals[name] = var + + def add_global_var(self, name: str, var: Any) -> None: + if isinstance(var, ModuleType): + self.globals[name] = self._add_mod(var) + else: + self.globals[name] = var + + def add_local_mod(self, name: str, mod: ModuleType) -> None: + assert isinstance(mod, ModuleType) + self.add_global_var(name, mod) + + def record_module_access(self, mod: ModuleType, name: str, val: Any) -> None: + if isinstance(val, ModuleType): + self.name_to_modrec[mod.__name__].accessed_attrs[name] = self._add_mod(val) + return + + if mod.__name__ in self.name_to_modrec: + self.name_to_modrec[mod.__name__].accessed_attrs[name] = val + + def get_record(self) -> ExecutionRecord: + return ExecutionRecord( + self.code, + self.closure, + ExecutionRecorder._resolve_modules(self.globals), + ExecutionRecorder._resolve_modules(self.locals), + self.builtins.copy(), + self.code_options.copy(), + ) + + def _add_mod(self, mod: ModuleType) -> ModuleRecord: + if mod.__name__ not in self.name_to_modrec: + self.name_to_modrec[mod.__name__] = ModuleRecord(mod) + + return self.name_to_modrec[mod.__name__] + + @classmethod + def _resolve_modules(cls, vars: dict[str, Any]) -> dict[str, Any]: + def resolve_module(var: Any) -> Any: + if not isinstance(var, ModuleRecord): + return var + + dummy_mod = DummyModule(var.module.__name__) + for attr_name, attr_value in var.accessed_attrs.items(): + attr_value = resolve_module(attr_value) + dummy_mod.__setattr__(attr_name, attr_value) + + return dummy_mod + + return {k: resolve_module(v) for k, v in vars.items()} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/resume_execution.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/resume_execution.py new file mode 100644 index 0000000000000000000000000000000000000000..cc119844e762bf843d1cecb5810e654f31b014c0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/resume_execution.py @@ -0,0 +1,746 @@ +""" +This module provides functionality for resuming Python execution at specific points in code, +primarily used by PyTorch Dynamo for control flow handling and optimization. It implements +bytecode transformation and execution state management to enable: + +- Resuming execution at arbitrary points in Python bytecode +- Managing context managers and their state across execution boundaries +- Transforming and generating new code objects with preserved execution state +- Supporting Python 3.11+ exception handling and block management +- Restoring torch function mode stacks and other execution context + +The module is critical for PyTorch Dynamo's ability to optimize code while preserving +Python semantics and execution state. +""" + +import copy +import dataclasses +import sys +import types +from collections.abc import Callable, Iterable +from contextlib import AbstractContextManager +from typing import Any, cast, Optional + +from .bytecode_transformation import ( + add_push_null, + bytecode_from_template, + create_binary_subscr, + create_call_function, + create_call_function_ex, + create_instruction, + create_jump_absolute, + create_load_const, + Instruction, + overwrite_instruction, + transform_code_object, + unique_id, +) +from .utils import ExactWeakKeyDictionary + + +# taken from code.h in cpython +CO_OPTIMIZED = 0x0001 +CO_NEWLOCALS = 0x0002 +CO_VARARGS = 0x0004 +CO_VARKEYWORDS = 0x0008 +CO_NESTED = 0x0010 +CO_GENERATOR = 0x0020 +CO_NOFREE = 0x0040 +CO_COROUTINE = 0x0080 +CO_ITERABLE_COROUTINE = 0x0100 +CO_ASYNC_GENERATOR = 0x0200 + +# trace_rules.py import this constant for consistency +TORCH_DYNAMO_RESUME_IN_PREFIX = "torch_dynamo_resume_in" +IS_TRACING_RESUME_PROLOGUE_VARNAME = "__is_tracing_resume_prologue" + + +# If is_resume - this codegen is for a resume function +def _initial_push_null(insts: list[Instruction]) -> None: + if sys.version_info >= (3, 11): + insts.append(create_instruction("PUSH_NULL")) + if sys.version_info < (3, 13): + insts.append(create_instruction("SWAP", arg=2)) + + +# Generates bytecode from template and splits the code where LOAD_FAST dummy is present. +def _bytecode_from_template_with_split( + template: Callable[..., Any], + stack_index: int, + varname_map: Optional[dict[str, Any]] = None, +) -> tuple[list[Instruction], list[Instruction]]: + template_code = bytecode_from_template(template, varname_map=varname_map) + template_code.append(create_instruction("POP_TOP")) + + # adjust exception table entry depth + for inst in template_code: + if inst.exn_tab_entry: + inst.exn_tab_entry.depth += stack_index + + # search for LOAD_FAST dummy and replace it with 2 NOPs (we can break up the bytecode between them) + dummy_idx, dummy_inst = next( + ( + (i, inst) + for i, inst in enumerate(template_code) + if inst.opname in ("LOAD_FAST", "LOAD_FAST_BORROW") + and inst.argval == "dummy" + ), + (None, None), + ) + assert dummy_idx is not None and dummy_inst is not None + + # replace LOAD_FAST dummy with first NOP marking exception area + overwrite_instruction(dummy_inst, [create_instruction("NOP")]) + + # POP_TOP follows LOAD_FAST dummy - replace with NOP marking end of exception area + assert template_code[dummy_idx + 1].opname == "POP_TOP" + overwrite_instruction(template_code[dummy_idx + 1], [create_instruction("NOP")]) + + return template_code[: dummy_idx + 1], template_code[dummy_idx + 1 :] + + +def _try_except_tf_mode_template(dummy: Any, stack_var_name: Any) -> None: + # NOTE: Make sure this name matches what is generated by symbolic_convert:import_source + # on torch._dynamo.utils. + # pyrefly: ignore [unknown-name] + global __import_torch_dot__dynamo_dot_utils + try: + dummy + except: # noqa: E722, B001 + __import_torch_dot__dynamo_dot_utils.set_torch_function_mode_stack( # type: ignore[name-defined] + stack_var_name + ) + raise + + +@dataclasses.dataclass(frozen=True) +class ReenterWith: + stack_index: int + target_values: Optional[tuple[Any, ...]] = None + + def try_except_torch_function_mode( + self, code_options: dict[str, Any], cleanup: list[Instruction] + ) -> list[Instruction]: + """ + Codegen based off of: + try: + (rest) + except: + (restore previous tf mode stack) + raise + """ + from .variables.torch_function import get_prev_stack_var_name + + setup_try_except, epilogue = _bytecode_from_template_with_split( + _try_except_tf_mode_template, + self.stack_index, + varname_map={"stack_var_name": get_prev_stack_var_name()}, + ) + cleanup[:] = epilogue + cleanup + + return setup_try_except + + # If we do not want to destroy the stack, we can do the same thing as a + # `SETUP_WITH` block, only that we store the context manager in a local_symbol + def try_finally( + self, code_options: dict[str, Any], cleanup: list[Instruction] + ) -> list[Instruction]: + """ + Codegen based off of: + load args + enter context + try: + (rest) + finally: + exit context + """ + # NOTE: we assume that TOS is a context manager CLASS! + load_args = [] + if self.target_values: + load_args = [create_load_const(val) for val in self.target_values] + ctx_name = unique_id(f"___context_manager_{self.stack_index}") + if ctx_name not in code_options["co_varnames"]: + code_options["co_varnames"] += (ctx_name,) + for name in ["__enter__", "__exit__"]: + if name not in code_options["co_names"]: + code_options["co_names"] += (name,) + + create_ctx: list[Instruction] = [] + _initial_push_null(create_ctx) + create_ctx.extend( + [ + *load_args, + *create_call_function(len(load_args), False), + create_instruction("STORE_FAST", argval=ctx_name), + ] + ) + + def _template(ctx: AbstractContextManager[Any], dummy: Any) -> None: + ctx.__enter__() + try: + dummy + finally: + ctx.__exit__(None, None, None) + + setup_try_finally, epilogue = _bytecode_from_template_with_split( + _template, self.stack_index, varname_map={"ctx": ctx_name} + ) + cleanup[:] = epilogue + cleanup + return create_ctx + setup_try_finally + + def __call__( + self, code_options: dict[str, Any], cleanup: list[Instruction] + ) -> tuple[list[Instruction], Optional[Instruction]]: + """ + Codegen based off of: + with ctx(args): + (rest) + """ + # NOTE: we assume that TOS is a context manager CLASS! + load_args = [] + if self.target_values: + load_args = [create_load_const(val) for val in self.target_values] + + create_ctx: list[Instruction] = [] + # Do not push NULL in Python 3.14+ since the NULL should be on the symbolic stack. + if sys.version_info < (3, 14): + _initial_push_null(create_ctx) + create_ctx.extend( + [ + *load_args, + *create_call_function(len(load_args), False), + ] + ) + + def _template(ctx: AbstractContextManager[Any], dummy: Any) -> None: + with ctx: + dummy + + setup_with, epilogue = _bytecode_from_template_with_split( + _template, self.stack_index + ) + cleanup[:] = epilogue + cleanup + + load_fast_ctx_inst = next( + ( + inst + for inst in setup_with + if inst.opname in ("LOAD_FAST", "LOAD_FAST_BORROW") + and inst.argval == "ctx" + ), + None, + ) + assert load_fast_ctx_inst is not None + # ctx already loaded on stack before the template - no need to LOAD_FAST + overwrite_instruction(load_fast_ctx_inst, [create_instruction("NOP")]) + + # 3.11+ only + push_exc_info_gen = ( + inst for inst in epilogue if inst.opname == "PUSH_EXC_INFO" + ) + push_exc_info_inst = next(push_exc_info_gen, None) + # expect only 1 PUSH_EXC_INFO in epilogue + assert next(push_exc_info_gen, None) is None + + return create_ctx + setup_with, push_exc_info_inst + + +@dataclasses.dataclass +class ResumeFunctionMetadata: + code: types.CodeType + instructions: list[Instruction] = dataclasses.field(default_factory=list) + # Python 3.11+ fields + # NOTE: Python 3.11 removed blocks, but for our purposes, a "block" consists + # of instructions of all exception table entries that have the same target. + + # map from PUSH_EXC_INFO's in the prefix to original block target offset + prefix_block_target_offset_remap: list[int] = dataclasses.field( + default_factory=list + ) + # per-offset map from new block target offsets to original block target offsets + block_target_offset_remap: dict[tuple[int, int], dict[int, int]] = ( + dataclasses.field(default_factory=dict) + ) + + +def _filter_iter( + l1: Iterable[Any], + l2: Iterable[Any], + cond: Callable[[Any, Any], bool], +) -> list[Any]: + """ + Two-pointer conditional filter. + e.g. _filter_iter(insts, sorted_offsets, lambda i, o: i.offset == o) + returns the instructions with offsets in sorted_offsets + """ + it = iter(l2) + res: list[Instruction] = [] + try: + cur = next(it) + for val in l1: + if cond(val, cur): + res.append(val) + cur = next(it) + except StopIteration: + pass + return res + + +def _load_tuple_and_call(tup: tuple[Any, ...]) -> list[Instruction]: + insts: list[Instruction] = [] + _initial_push_null(insts) + insts.extend(create_load_const(val) for val in tup) + insts.extend(create_call_function(len(tup), False)) + return insts + + +class ContinueExecutionCache: + cache = ExactWeakKeyDictionary() + generated_code_metadata = ExactWeakKeyDictionary() + + @classmethod + def lookup( + cls, code: types.CodeType, lineno: int, init_offset: int, *key: Any + ) -> types.CodeType: + if code not in cls.cache: + cls.cache[code] = {} + key = tuple(key) + if key not in cls.cache[code]: + cls.cache[code][key] = cls.generate(code, lineno, init_offset, *key) + return cls.cache[code][key] + + @classmethod + def generate( + cls, + code: types.CodeType, + lineno: int, + init_offset: int, + resume_offset: int, + setup_fn_target_offsets: tuple[int, ...], # only used in Python 3.11+ + nstack: int, + argnames: tuple[str, ...], + argnames_null: tuple[str, ...], + setup_fns: tuple[ReenterWith, ...], + handle_inactive_ctx: bool, + stack_ctx_vars: tuple[tuple[int, tuple[Any, ...]], ...], + argnames_ctx_vars: tuple[tuple[str, tuple[Any, ...]], ...], + null_idxes: tuple[int, ...], + # mainly used to ensure distinct code objects per stack trace, + # which prevents excessive recompilation of inner frames + nested_code_objs: tuple[types.CodeType], + # Are we currently graph breaking on an instruction that doesn't push + # its result to the stack? If so, and we are not the leaf resume, then we need to pop + # the result of calling the next resume function. + pop_nested_resume_result: bool, + ) -> types.CodeType: + assert resume_offset is not None + assert not ( + code.co_flags + & (CO_GENERATOR | CO_COROUTINE | CO_ITERABLE_COROUTINE | CO_ASYNC_GENERATOR) + ) + assert code.co_flags & CO_OPTIMIZED + if code in ContinueExecutionCache.generated_code_metadata: + return cls.generate_based_on_original_code_object( + code, + lineno, + init_offset, + resume_offset, + setup_fn_target_offsets, + nstack, + argnames, + argnames_null, + setup_fns, + handle_inactive_ctx, + stack_ctx_vars, + argnames_ctx_vars, + null_idxes, + nested_code_objs, + pop_nested_resume_result, + ) + + is_py311_plus = sys.version_info >= (3, 11) + meta = ResumeFunctionMetadata(code) + + def update( + instructions: list[Instruction], code_options: dict[str, Any] + ) -> None: + meta.instructions = copy.deepcopy(instructions) + + args = ["__nested_resume_fns", "__nested_frame_values"] + args += [f"___stack{i}" for i in range(nstack)] + args.extend(v for v in argnames if v not in args) + freevars = tuple(code_options["co_cellvars"] or []) + tuple( + code_options["co_freevars"] or [] + ) + freevars = tuple(sorted(freevars)) + code_options["co_name"] = ( + f"{TORCH_DYNAMO_RESUME_IN_PREFIX}_{code_options['co_name']}_at_{lineno}" + ) + if is_py311_plus: + qualified_path = code_options["co_qualname"].rsplit(".", maxsplit=1) + if len(qualified_path) == 1: + code_options["co_qualname"] = code_options["co_name"] + else: + assert len(qualified_path) == 2 + module_name, co_name = qualified_path + code_options["co_qualname"] = ( + f"{module_name}.{TORCH_DYNAMO_RESUME_IN_PREFIX}_{co_name}_at_{lineno}" + ) + code_options["co_firstlineno"] = lineno + code_options["co_cellvars"] = () + code_options["co_freevars"] = freevars + code_options["co_argcount"] = len(args) + code_options["co_posonlyargcount"] = 0 + code_options["co_kwonlyargcount"] = 0 + code_options["co_varnames"] = tuple( + args + + [v for v in argnames_null if v not in args] + + [v for v in code_options["co_varnames"] if v not in args] + + [IS_TRACING_RESUME_PROLOGUE_VARNAME] + ) + code_options["co_flags"] = code_options["co_flags"] & ~( + CO_VARARGS | CO_VARKEYWORDS + ) + target = next(i for i in instructions if i.offset == resume_offset) + + prefix = [] + if is_py311_plus: + if freevars: + prefix.append( + create_instruction("COPY_FREE_VARS", arg=len(freevars)) + ) + prefix.append(create_instruction("RESUME", arg=0)) + + # Set is_tracing_resume_prologue to prevent graph breaks. + # This doesn't really do anything at runtime, but dynamo will trace this + # and will know that we're in a resume function prologue. + prefix.extend( + [ + create_instruction("LOAD_CONST", argval=True), + create_instruction( + "STORE_FAST", argval=IS_TRACING_RESUME_PROLOGUE_VARNAME + ), + ] + ) + + cleanup: list[Instruction] = [] + hooks = {fn.stack_index: fn for fn in setup_fns} + hook_target_offsets = { + fn.stack_index: setup_fn_target_offsets[i] + for i, fn in enumerate(setup_fns) + } + offset_to_inst = {inst.offset: inst for inst in instructions} + # map old hook targets to new targets generated by the hook + old_hook_target_remap = {} + stack_i = 0 + null_i = 0 + stack_ctx_vars_d = dict(stack_ctx_vars) # type: ignore[var-annotated,arg-type] + for i in range(nstack + len(null_idxes)): + if null_i < len(null_idxes) and null_idxes[null_i] == i: + prefix.append(create_instruction("PUSH_NULL")) + null_i += 1 + else: + prefix.append( + create_instruction("LOAD_FAST", argval=f"___stack{stack_i}") + ) + if handle_inactive_ctx and stack_i in stack_ctx_vars_d: + # NOTE: we assume that current stack var is a context manager CLASS! + # Load args for context variable and construct it + prefix.extend(_load_tuple_and_call(stack_ctx_vars_d[stack_i])) + stack_i += 1 + + if i in hooks: + hook = hooks.pop(i) + hook_insts, exn_target = hook(code_options, cleanup) + prefix.extend(hook_insts) + if is_py311_plus: + hook_target_offset = hook_target_offsets.pop(i) + old_hook_target = offset_to_inst[hook_target_offset] + meta.prefix_block_target_offset_remap.append(hook_target_offset) + old_hook_target_remap[old_hook_target] = exn_target + + if is_py311_plus: + # reverse the mapping since targets of later/nested contexts are inserted + # into the mapping later, but show up earlier in the prefix. + meta.prefix_block_target_offset_remap = list( + reversed(meta.prefix_block_target_offset_remap) + ) + + assert not hooks + + # NOTE: we assume that local var is a context manager CLASS! + # initialize inactive context vars in argnames + if handle_inactive_ctx: + for name, vals in argnames_ctx_vars: + prefix.append(create_instruction("LOAD_FAST", argval=name)) + prefix.extend(_load_tuple_and_call(vals)) + prefix.append(create_instruction("STORE_FAST", argval=name)) + + # 3.12+: store NULL into variables that were NULL + if argnames_null: + assert sys.version_info >= (3, 12) + for v in argnames_null: + assert v not in args + prefix.extend( + [ + create_instruction("PUSH_NULL"), + create_instruction("STORE_FAST", argval=v), + ] + ) + + # Call nested resume function + if nested_code_objs: + prefix.extend( + [ + # set up __nested_resume_fns[-1] call + *add_push_null( + [ + create_instruction( + "LOAD_FAST", argval="__nested_resume_fns" + ), + create_instruction("LOAD_CONST", argval=-1), + create_binary_subscr(), + ] + ), + # del __nested_resume_fns[-1] + create_instruction("LOAD_FAST", argval="__nested_resume_fns"), + create_instruction("LOAD_CONST", argval=-1), + create_instruction("DELETE_SUBSCR"), + # load [__nested_resume_fns, __nested_frame_values] + create_instruction("LOAD_FAST", argval="__nested_resume_fns"), + create_instruction("LOAD_FAST", argval="__nested_frame_values"), + create_instruction("BUILD_LIST", arg=2), + # load __nested_frame_values[-1] + create_instruction("LOAD_FAST", argval="__nested_frame_values"), + create_instruction("LOAD_CONST", argval=-1), + create_binary_subscr(), + # create [ + # __nested_resume_fns, + # __nested_frame_values, + # *__nested_frame_values[-1], + # ] + create_instruction("LIST_EXTEND", arg=1), + # del __nested_frame_values[-1] + create_instruction("LOAD_FAST", argval="__nested_frame_values"), + create_instruction("LOAD_CONST", argval=-1), + create_instruction("DELETE_SUBSCR"), + # delete __nested values + create_instruction("DELETE_FAST", argval="__nested_resume_fns"), + create_instruction( + "DELETE_FAST", argval="__nested_frame_values" + ), + # Set is_tracing_resume_prologue back to allow graph breaks + # in the nested resume + create_instruction("LOAD_CONST", argval=False), + create_instruction( + "STORE_FAST", argval=IS_TRACING_RESUME_PROLOGUE_VARNAME + ), + # finish the call + *create_call_function_ex(False, False), + ] + ) + if pop_nested_resume_result: + # pop the result of calling the nested resume function + prefix.append(create_instruction("POP_TOP")) + else: + # Set is_tracing_resume_prologue back to allow graph breaks after the jump + prefix.extend( + [ + create_instruction("LOAD_CONST", argval=False), + create_instruction( + "STORE_FAST", argval=IS_TRACING_RESUME_PROLOGUE_VARNAME + ), + ] + ) + + prefix.append(create_jump_absolute(target)) + + # because the line number table monotonically increases from co_firstlineno + # remove starts_line for any instructions before the graph break instruction + # this will ensure the instructions after the break have the correct line numbers + for inst in instructions: + if inst.offset == target.offset: + break + inst.starts_line = None + if sys.version_info >= (3, 11): + inst.positions = None + + if cleanup: + prefix.extend(cleanup) + prefix.extend(cls.unreachable_codes(code_options)) + + # remap original instructions' exception table entries + if old_hook_target_remap: + # pyrefly: ignore [unbound-name] + assert is_py311_plus + for inst in instructions: + if ( + inst.exn_tab_entry + and inst.exn_tab_entry.target in old_hook_target_remap + ): + inst.exn_tab_entry.target = old_hook_target_remap[ # type: ignore[assignment] + inst.exn_tab_entry.target + ] + + # TODO(jansel): add dead code elimination here + instructions[:] = prefix + instructions + + new_code, _ = transform_code_object(code, update) + ContinueExecutionCache.generated_code_metadata[new_code] = meta + return new_code + + @staticmethod + def unreachable_codes(code_options: dict[str, Any]) -> list[Instruction]: + """Codegen a `raise None` to make analysis work for unreachable code""" + return [ + create_load_const(None), + create_instruction("RAISE_VARARGS", arg=1), + ] + + @classmethod + def generate_based_on_original_code_object( + cls, + code: types.CodeType, + lineno: int, + init_offset: int, + resume_offset: int, + setup_fn_target_offsets: tuple[int, ...], + *args: Any, + ) -> types.CodeType: + """ + This handles the case of generating a resume into code generated + to resume something else. We want to always generate starting + from the original code object so that if control flow paths + converge we only generated 1 resume function (rather than 2^n + resume functions). + """ + + meta: ResumeFunctionMetadata = ContinueExecutionCache.generated_code_metadata[ + code + ] + + def find_orig_offset(cur_offset: int) -> int: + orig_offset = -1 + + def find_orig_offset_transform( + instructions: list[Instruction], code_options: dict[str, Any] + ) -> None: + nonlocal orig_offset + (target,) = (i for i in instructions if i.offset == cur_offset) + # match the functions starting at the last instruction as we have added a prefix + new_target_tuple = tuple( + i2 + for i1, i2 in zip( + reversed(instructions), reversed(meta.instructions) + ) + if i1 is target + ) + + if not new_target_tuple: + # Instruction with cur_offset in instructions was not found + # in the original code - orig_offset left as -1. + # Caller expected to handle this case. + return + + assert len(new_target_tuple) == 1 + new_target = new_target_tuple[0] + + assert target.opcode == new_target.opcode + assert new_target.offset is not None + orig_offset = new_target.offset + + transform_code_object(code, find_orig_offset_transform) + return orig_offset + + orig_init_offset = find_orig_offset(init_offset) + # It is fine if the initial instruction is not found in the original code; + # this means we graph broke in the prefix, which only happens with nested graph breaks. + # We should not be running into ambiguous graph break issues here. + orig_resume_offset = find_orig_offset(resume_offset) + assert orig_resume_offset > -1, ( + "resume instruction not found in original code - this is a bug." + ) + + if sys.version_info >= (3, 11): + # setup_fn_target_offsets currently contains the target offset of + # each setup_fn, based on `code`. When we codegen the resume function + # based on the original code object, `meta.code`, the offsets in + # setup_fn_target_offsets must be based on `meta.code` instead. + offset_key = (orig_init_offset, orig_resume_offset) + # NOTE: we key by offset_key since the same resume function may graph + # break in multiple places and we need different block_target_offset_remap's + # for each graph break location. Keying by orig_resume_offset may not be enough + # if 2 graph breaks on different initial offsets resume on the same instruction + # (although this is rare and not tested anywhere). + if offset_key not in meta.block_target_offset_remap: + block_target_offset_remap = meta.block_target_offset_remap[ + offset_key + ] = {} + + def remap_block_offsets( + instructions: list[Instruction], code_options: dict[str, Any] + ) -> None: + # NOTE: each prefix block generates exactly one PUSH_EXC_INFO, + # so we can tell which block a prefix PUSH_EXC_INFO belongs to, + # by counting. Then we can use meta.prefix_block_target_offset_remap + # to determine where in the original code the PUSH_EXC_INFO offset + # replaced. + prefix_blocks: list[Instruction] = [] + for inst in instructions: + # NOTE meta.prefix_block_target_offset_remap is based off of how we codegen'd + # context managers at the prefix/prologue of the resume function. It is the same for + # every graph break in the same resume function, so we do not need to recompute + # for each graph break (unlike for meta.block_target_offset_remap) + if len(prefix_blocks) == len( + meta.prefix_block_target_offset_remap + ): + break + if inst.opname == "PUSH_EXC_INFO": + prefix_blocks.append(inst) + + # remap block target offsets for blocks generated in the resume prefix + for inst, o in zip( + prefix_blocks, meta.prefix_block_target_offset_remap + ): + block_target_offset_remap[cast(int, inst.offset)] = o + + # current bytecode targets are after the prefix PUSH_EXC_INFO's + cur_start_offset = ( + cast(int, prefix_blocks[-1].offset) if prefix_blocks else -1 + ) + # get the remaining block target offsets of the current bytecode + cur_inst_offsets = sorted( + n for n in setup_fn_target_offsets if n > cur_start_offset + ) + targets = _filter_iter( + instructions, cur_inst_offsets, lambda inst, o: inst.offset == o + ) + # The original code and resume code should have matching suffixes. + # Match the post-prefix block target offsets of the current resume code + # and the original code. + orig_targets = reversed( + _filter_iter( + zip(reversed(instructions), reversed(meta.instructions)), + reversed(targets), + lambda v1, v2: v1[0] is v2, + ) + ) + for orig, cur in zip(orig_targets, targets): + block_target_offset_remap[cur.offset] = orig[1].offset + + transform_code_object(code, remap_block_offsets) + + # if offset_key or offset is not in setup_fn_target_offsets, it is an error + # that needs to be fixed + setup_fn_target_offsets = tuple( + meta.block_target_offset_remap[offset_key][n] + for n in setup_fn_target_offsets + ) + return ContinueExecutionCache.lookup( + meta.code, + lineno, + orig_init_offset, + orig_resume_offset, + setup_fn_target_offsets, + *args, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/side_effects.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/side_effects.py new file mode 100644 index 0000000000000000000000000000000000000000..18ab5a54d734a6f134e9034585475e1896eca3ea --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/side_effects.py @@ -0,0 +1,1234 @@ +""" +Side effect tracking and management for TorchDynamo's compilation system. + +This module provides infrastructure for tracking and managing side effects that occur +during symbolic execution, including: + +- Tracking mutations to objects, attributes, and variables +- Managing context changes (cell variables, global namespace modifications) +- Handling aliasing and object identity preservation +- Managing stack frame state and local variable changes +- Tracking function calls with side effects + +Key classes: +- SideEffects: Main container for tracking all side effects during execution +- MutableSideEffects: Specialization for mutable object tracking +- AttributeMutation/ValueMutation: Track specific types of mutations +- Various specialized side effect classes for different scenarios + +The side effect system ensures that mutations performed during symbolic execution +are properly replayed during runtime, maintaining the correctness of compiled code +while enabling optimizations where safe. +""" + +import collections +import contextlib +import inspect +import warnings +import weakref +from collections.abc import Generator, MutableMapping +from types import CellType +from typing import Any, Optional, TYPE_CHECKING + +import torch.nn +from torch._dynamo.variables.misc import AutogradFunctionContextVariable + +from . import graph_break_hints, utils, variables +from .bytecode_transformation import ( + bytecode_from_template, + create_call_function, + create_call_method, + create_instruction, +) +from .codegen import PyCodegen +from .exc import SideEffectsError, unimplemented +from .source import GlobalSource, LocalCellSource, Source, TempLocalSource +from .utils import is_frozen_dataclass, nn_module_new, object_new +from .variables.base import ( + AttributeMutation, + AttributeMutationExisting, + AttributeMutationNew, + is_side_effect_safe, + ValueMutationExisting, + ValueMutationNew, + VariableTracker, +) +from .variables.user_defined import FrozenDataClassVariable + + +if TYPE_CHECKING: + from torch._dynamo.output_graph import OutputGraph + from torch._dynamo.symbolic_convert import InstructionTranslatorBase + from torch._dynamo.variables.lists import ListVariable + + +def _manual_dict_setitem( + dict_from: dict[Any, Any], dict_to: dict[Any, Any], mro_index: int +) -> None: + # Carefully calls the dict or OrderedDict `clear` or `__setitem__`. We have + # to be careful because we don't want to trigger the user defined object + # setitem or clear. The mro_index is used to find the dict/OrderedDict from + # the class mro. + dict_class = type(dict_to).__mro__[mro_index] + dict_class.clear(dict_to) # type: ignore[attr-defined] + for k, v in dict_from.items(): + dict_class.__setitem__(dict_to, k, v) # type: ignore[index] + + +def _manual_list_update(list_from: list[Any], list_to: list[Any]) -> None: + list.clear(list_to) + list.extend(list_to, list_from) + + +class SideEffects: + """ + Maintain records of mutations and provide methods to apply them during code generation. + + Handles tracking and applying side effects during PyTorch Dynamo compilation, + maintaining Python semantics by managing mutations, attribute modifications, + and other side effects that occur during program execution. + + Key responsibilities: + - Tracks mutations to Python objects, lists, and dictionaries that need to be + applied after an FX graph is run. + - Manages attribute modifications and deletions + - Handles tensor hooks and backward pass state + - Tracks cell variable mutations and global variable changes + - Ensures correct ordering and application of side effects after graph execution + + This ensures that optimized code behaves identically to the original Python code with + respect to object mutations and other side effects. + """ + + id_to_variable: dict[int, VariableTracker] + store_attr_mutations: dict[VariableTracker, dict[str, VariableTracker]] + keepalive: list[Any] + + def __init__( + self, + output_graph: "OutputGraph", + id_to_variable: Optional[dict[int, VariableTracker]] = None, + store_attr_mutations: Optional[ + dict[VariableTracker, dict[str, VariableTracker]] + ] = None, + keepalive: Optional[list[Any]] = None, + save_for_backward: Optional[ + list[tuple[AutogradFunctionContextVariable, list[VariableTracker]]] + ] = None, + tensor_hooks: Optional[ + dict[ + int, + tuple[ + "variables.TensorVariable", + VariableTracker, + "variables.RemovableHandleVariable", + str, + ], + ] + ] = None, + ) -> None: + super().__init__() + self.output_graph_weakref = weakref.ref(output_graph) + self.id_to_variable = id_to_variable or {} + self.store_attr_mutations = store_attr_mutations or {} + self.keepalive = keepalive or [] + self.save_for_backward = save_for_backward or [] + self.tensor_hooks = tensor_hooks or {} + # Used by MappingProxyVariable to graph break in case of any mutated + # dict + self._has_existing_dict_mutation = False + # Track Compiled Autograd final callbacks that must be called at the end of Compiled Autograd backward graph. + # Only applicable if this graph is created from Dynamo tracing in Compiled Autograd. + self.ca_final_callbacks_var: Optional[ListVariable] = None + + # Tracks VariableTracker objects whose mutations can be skipped. + # For normal mutated variables, Dynamo generates code to replay/reconstruct + # the mutations after graph execution. However, variables in this set have + # their mutations ignored - the mutations happen during + # execution but don't need to be replayed in the generated code. + # Used for temporary mutations in contexts like torch.func.functional_call, + # where module parameters/buffers are modified but later restored. + self.ignore_mutation_on_these_variables: set[VariableTracker] = set() + + def ignore_mutations_on(self, var: VariableTracker) -> None: + """Mutations to this variable will be executed but not not tracked, + typically used for temporary mutations that are later restored.""" + self.ignore_mutation_on_these_variables.add(var) + + def stop_ignoring_mutations_on(self, var: VariableTracker) -> None: + """Remove a variable from the skip mutation set, restoring normal mutation tracking.""" + if var in self.ignore_mutation_on_these_variables: + self.ignore_mutation_on_these_variables.remove(var) + + def __eq__(self, other: object) -> bool: + assert isinstance(other, SideEffects) + # NB: do NOT test keepalive + return ( + self.id_to_variable == other.id_to_variable + and self.store_attr_mutations == other.store_attr_mutations + and self.save_for_backward == other.save_for_backward + and self.tensor_hooks == other.tensor_hooks + ) + + def diff(self, other: "SideEffects") -> Optional[str]: + if self.id_to_variable != other.id_to_variable: + sk_itv = self.id_to_variable.keys() + ok_itv = other.id_to_variable.keys() + if sk_itv != ok_itv: + return f"id_to_variable keys: {sk_itv} != {ok_itv}" + # Feel free to augment this with more fancy diffing logic + # if needed for debugging + return "id_to_variable: unknown diff" + elif self.store_attr_mutations != other.store_attr_mutations: + sk_sam = self.store_attr_mutations.keys() + ok_sam = other.store_attr_mutations.keys() + if sk_sam != ok_sam: + return f"store_attr_mutations keys: {sk_sam} != {ok_sam}" + return "store_attr_mutations: unknown diff" + elif self.save_for_backward != other.save_for_backward: + return "save_for_backward" + elif self.tensor_hooks != other.tensor_hooks: + return "tensor_hooks" + else: + return None + + def clone(self) -> "SideEffects": + """Create a shallow copy""" + ref = self.output_graph_weakref() + assert ref is not None + return self.__class__( + output_graph=ref, + id_to_variable=dict(self.id_to_variable), + store_attr_mutations={ + k: dict(v) for k, v in self.store_attr_mutations.items() + }, + keepalive=list(self.keepalive), + save_for_backward=self.save_for_backward, + tensor_hooks=self.tensor_hooks, + ) + + def __contains__(self, item: Any) -> bool: + return id(item) in self.id_to_variable + + def __getitem__(self, item: Any) -> VariableTracker: + return self.id_to_variable[id(item)] + + def should_allow_externally_visible_side_effects_in_subtracer(self) -> bool: + output_graph = self.output_graph_weakref() + return bool( + output_graph + and output_graph.current_tx.output.current_tracer.unsafe_allow_externally_visible_side_effects + ) + + def should_allow_side_effects_in_hop(self) -> bool: + output_graph = self.output_graph_weakref() + return bool( + output_graph + and output_graph.current_tx.output.current_tracer.allow_side_effects_in_hop + ) + + def is_reconstructing_generator(self) -> bool: + output_graph = self.output_graph_weakref() + + return bool( + output_graph + and output_graph.current_tx.output.current_tracer.is_reconstructing_generator + ) + + def check_allowed_side_effect(self, item: VariableTracker) -> bool: + from torch._dynamo.variables.misc import AutogradFunctionContextVariable + + # People do things like self.dim = dim inside autograd.Function. + # These are benign. + if isinstance(item, AutogradFunctionContextVariable): + return True + if self.should_allow_externally_visible_side_effects_in_subtracer(): + return True + if self.should_allow_side_effects_in_hop(): + return True + if self.is_reconstructing_generator(): + # This is missing the case where one mutates a tensor. See + # test_generator.py::test_reconstruct_generator_tensor_mutation + raise SideEffectsError( + "Cannot reconstruct a generator with variable mutations. " + "Dynamo needs to fully exhaust the generator, which may cause " + "unintended variable modifications." + ) + assert item.mutation_type is not None + if not is_side_effect_safe(item.mutation_type): + # TODO plumb HOP information here + unimplemented( + gb_type="HigherOrderOperator: Mutating a variable not in the current scope (SideEffects)", + context="", + explanation="This is not supported.", + hints=[], + ) + return False + + def store_attr( + self, item: VariableTracker, name: str, value: VariableTracker + ) -> None: + assert self.is_attribute_mutation(item) + self.check_allowed_side_effect(item) + if item not in self.store_attr_mutations: + self.store_attr_mutations[item] = {} + self.store_attr_mutations[item][name] = value + + def load_attr( + self, + item: VariableTracker, + name: str, + deleted_ok: bool = False, + check: bool = False, + ) -> VariableTracker: + if check: + assert self.is_attribute_mutation(item) + result = self.store_attr_mutations[item][name] + if not deleted_ok and isinstance(result, variables.DeletedVariable): + unimplemented( + gb_type="Attempted to read a deleted variable", + context=f"item: {item}, name: {name}", + explanation="", + hints=[*graph_break_hints.USER_ERROR], + ) + return result + + def store_cell(self, cellvar: VariableTracker, value: VariableTracker) -> None: + if cellvar.is_immutable(): + unimplemented( + gb_type="Write to immutable cell", + context=f"cellvar: {cellvar}, value: {value}", + explanation="Dynamo doesn't support writing to immutable/sourceless cell variables.", + hints=[*graph_break_hints.DIFFICULT], + ) + assert isinstance(cellvar, variables.CellVariable) + assert isinstance(value, variables.VariableTracker) + self.store_attr(cellvar, "cell_contents", value) + + def load_cell(self, cellvar: VariableTracker) -> VariableTracker: + assert isinstance(cellvar, variables.CellVariable) + if self.has_pending_mutation_of_attr(cellvar, "cell_contents"): + return self.load_attr(cellvar, "cell_contents", check=False) + if cellvar.pre_existing_contents: + return cellvar.pre_existing_contents + unimplemented( + gb_type="Read uninitialized cell", + context=str(cellvar), + explanation="Attempted to read a cell variable that has not been populated yet.", + hints=[*graph_break_hints.USER_ERROR], + ) + + def load_global(self, gvar: VariableTracker, name: str) -> VariableTracker: + assert isinstance(gvar, variables.VariableTracker) + return self.load_attr(gvar, name) + + def store_global( + self, gvar: VariableTracker, name: str, value: VariableTracker + ) -> None: + assert isinstance(gvar, variables.VariableTracker) + assert isinstance(value, variables.VariableTracker) + self.store_attr(gvar, name, value) + + @staticmethod + def cls_supports_mutation_side_effects(cls: type) -> bool: + return inspect.getattr_static(cls, "__getattribute__", None) in ( + object.__getattribute__, + dict.__getattribute__, + set.__getattribute__, + frozenset.__getattribute__, + int.__getattribute__, + str.__getattribute__, + list.__getattribute__, + tuple.__getattribute__, + BaseException.__getattribute__, + ) + + def is_attribute_mutation(self, item: VariableTracker) -> bool: + return isinstance(item.mutation_type, AttributeMutation) + + def has_pending_mutation(self, item: VariableTracker) -> bool: + return self.is_attribute_mutation(item) and bool( + self.store_attr_mutations.get(item) + ) + + def has_pending_mutation_of_attr(self, item: VariableTracker, name: str) -> bool: + return self.is_attribute_mutation( + item + ) and name in self.store_attr_mutations.get(item, ()) + + def is_modified(self, item: VariableTracker) -> bool: + if item.is_immutable(): + return False + if isinstance(item.mutation_type, (AttributeMutationNew, ValueMutationNew)): + return True + + if isinstance(item, variables.UserDefinedObjectVariable): + # Checks if the underlying dict or tuple vt has been modified + return item in self.store_attr_mutations or item.is_underlying_vt_modified( + self + ) + + if self.is_attribute_mutation(item): + return item in self.store_attr_mutations + assert item.mutation_type is not None + return item.mutation_type.is_modified # type: ignore[attr-defined] + + def _track_obj( + self, + item: Any, + variable: VariableTracker, + mutation_type_cls: type = ValueMutationExisting, + ) -> VariableTracker: + """Start tracking an existing or new variable for mutation""" + if id(item) in self.id_to_variable: + raise AssertionError( + f"{variable} is already tracked for mutation. This could be " + "because you are not using VariableBuilder to construct " + "the variable tracker. " + f"Source of new object: {variable.source}. " + f"Source of previously tracked object: {self.id_to_variable[id(item)].source}." + ) + + variable.mutation_type = mutation_type_cls() + self.id_to_variable[id(item)] = variable + self.keepalive.append(item) + return variable + + track_mutable = _track_obj + + def track_object_existing( + self, + item: Any, + variable: VariableTracker, + ) -> VariableTracker: + return self._track_obj( + item, + variable, + mutation_type_cls=AttributeMutationExisting, + ) + + def track_object_new( + self, + cls_source: Source, + user_cls: Any, + variable_cls: Any, + options: dict[str, Any], + ) -> VariableTracker: + if user_cls is torch.autograd.function.FunctionCtx: + with warnings.catch_warnings(record=True): + obj = torch.autograd.Function() + else: + obj = object_new(user_cls) + variable = variable_cls( + obj, + mutation_type=AttributeMutationNew(cls_source), + **options, + ) + self.id_to_variable[id(obj)] = variable + self.keepalive.append(obj) + return variable + + def get_variable_cls(self, user_cls: type) -> type: + from torch.overrides import TorchFunctionMode + + from .variables.ctx_manager import GenericContextWrappingVariable + from .variables.torch_function import TorchFunctionModeVariable + from .variables.user_defined import is_forbidden_context_manager + + variable_cls: type[variables.UserDefinedObjectVariable] = ( + variables.UserDefinedObjectVariable + ) + if issubclass( + user_cls, TorchFunctionMode + ) and TorchFunctionModeVariable.is_supported_torch_function_mode(user_cls): + variable_cls = TorchFunctionModeVariable + elif ( + hasattr(user_cls, "__enter__") + and hasattr(user_cls, "__exit__") + and not is_forbidden_context_manager(user_cls) + ): + variable_cls = GenericContextWrappingVariable + elif issubclass(user_cls, torch.nn.Module): + variable_cls = variables.UnspecializedNNModuleVariable + elif issubclass(user_cls, (dict, collections.OrderedDict)): + variable_cls = variables.UserDefinedDictVariable + elif issubclass(user_cls, (set, frozenset)): + variable_cls = variables.UserDefinedSetVariable + elif issubclass(user_cls, tuple): + variable_cls = variables.UserDefinedTupleVariable + elif issubclass(user_cls, list): + variable_cls = variables.UserDefinedListVariable + elif issubclass(user_cls, MutableMapping): + variable_cls = variables.MutableMappingVariable + elif is_frozen_dataclass(user_cls): + variable_cls = FrozenDataClassVariable + elif issubclass(user_cls, BaseException): + variable_cls = variables.UserDefinedExceptionObjectVariable + assert issubclass(variable_cls, variables.UserDefinedObjectVariable) + return variable_cls + + def get_example_value( + self, + base_cls_vt: VariableTracker, + cls_vt: VariableTracker, + init_args: list[VariableTracker], + ) -> Any: + user_cls = cls_vt.value # type: ignore[attr-defined] + if issubclass(user_cls, torch.nn.Module): + # TODO(anijain2305) - Is it possible to remove this specialization? + obj = nn_module_new(user_cls) + else: + if isinstance(base_cls_vt, variables.BuiltinVariable): + base_cls = base_cls_vt.fn + elif isinstance(base_cls_vt, variables.UserDefinedClassVariable): + base_cls = base_cls_vt.value + else: + raise RuntimeError(f"Unexpected base_cls_vt {base_cls_vt}") + + assert variables.UserDefinedClassVariable.is_supported_new_method( + base_cls.__new__ + ) + # TODO(anijain2305) - Consider adding get_example_value method to + # each VT to get an example value for all args. As we expand the + # scope to other __new__ methods, we might need to call __new__ with + # init_args (like functools.partial) + # init_args = [arg.get_example_value() for arg in init_args] + # obj = base_cls.__new__(user_cls, *init_args) + + obj = base_cls.__new__(user_cls) + return obj + + def track_new_user_defined_object( + self, + base_cls_vt: VariableTracker, + cls_vt: VariableTracker, + init_args: list[VariableTracker], + ) -> VariableTracker: + """ + Creates a UserDefinedObjectVariable (or its subclass) variable tracker + and mark it for attribute mutation tracking. + + Also records the variable trackers to call __new__ method on + reconstruction. Roughly, the reconstruction looks like this + base_cls_vt.__new__(user_cls, *init_args) + """ + cls_source = cls_vt.source + user_cls = cls_vt.value # type: ignore[attr-defined] + variable_cls = self.get_variable_cls(user_cls) + obj = self.get_example_value(base_cls_vt, cls_vt, init_args) + + variable = variable_cls( + obj, + cls_source=cls_vt.source, + base_cls_vt=base_cls_vt, + init_args=init_args, + mutation_type=AttributeMutationNew(cls_source), + ) + self.id_to_variable[id(obj)] = variable + self.keepalive.append(obj) + return variable + + def track_cell_new( + self, + ) -> VariableTracker: + obj = object() + variable = variables.CellVariable( + mutation_type=AttributeMutationNew(), + ) + self.id_to_variable[id(obj)] = variable + self.keepalive.append(obj) + return variable + + def track_cell_existing( + self, source: Optional[Source], cell: CellType, contents: VariableTracker + ) -> VariableTracker: + variable = variables.CellVariable( + # We don't support mutation to cell without source because we need + # source to properly codegen the mutations. + mutation_type=None if source is None else AttributeMutationExisting(), + pre_existing_contents=contents, + source=source, + ) + self.id_to_variable[id(cell)] = variable + self.keepalive.append(cell) + return variable + + def track_global_existing(self, source: Source, item: Any) -> VariableTracker: + variable = variables.NewGlobalVariable( + mutation_type=AttributeMutationExisting(), + source=source, + ) + self.id_to_variable[id(item)] = variable + self.keepalive.append(item) + return variable + + def track_save_for_backward( + self, ctx: VariableTracker, args: list[VariableTracker] + ) -> None: + assert isinstance(ctx, variables.AutogradFunctionContextVariable) + self.save_for_backward.append((ctx, args)) + + def track_runahead_tensor_and_symvar_side_effects( + self, other: "SideEffects" + ) -> None: + # In higher order ops we want to keep track of tensors seen in the + # speculate_subgraph so that we don't lift them again as a new input in + # other speculate_subgraph or in the root tracer. + for other_item in other.keepalive: + other_id = id(other_item) + other_variable = other.id_to_variable[other_id] + if other_id not in self.id_to_variable and isinstance( + other_variable, (variables.TensorVariable, variables.SymNodeVariable) + ): + self.track_object_existing(other_item, other_variable) + + def prune_dead_object_new(self, tx: "InstructionTranslatorBase") -> None: + # Avoid VT cycles from e.g., recursive function. + visited: set[VariableTracker] = set() + live_new_objects: set[VariableTracker] = set() + + def visit(var: VariableTracker) -> None: + if var in visited: + return + visited.add(var) + # Object may have been mutated, store this mutation. + if isinstance(var.mutation_type, AttributeMutationNew): + live_new_objects.add(var) + # It's possible that we have mutated the value of this variable + # to be another one. The new value is in store_attr_mutations. + # Also recurse through the new value to detect alive AttributeMutationNew. + if var in self.store_attr_mutations: + VariableTracker.visit( + visit, # noqa: F821 + self.store_attr_mutations[var], + ) + + def is_live(var: VariableTracker) -> bool: + if isinstance(var.mutation_type, AttributeMutationNew): + return var in live_new_objects + return True + + pre_existing_vars = [ + var + for var in self.id_to_variable.values() + if not isinstance(var.mutation_type, AttributeMutationNew) + ] + + # The only live side effects come from returns (tx.stack), any intermediates + # during a graph break (tx.symbolic_locals), and mutation on pre-existing variables. + # Recursively visit Variables and see if any of them have been mutated. + init_live_vars = [] + # gather stack/symbolic_locals for all tx's up the chain + cur_tx: Optional[InstructionTranslatorBase] = tx + while cur_tx is not None: + init_live_vars.extend([cur_tx.stack, cur_tx.symbolic_locals]) + if cur_tx.parent is not None: + # for non-root tx'es, also keep the cells/freevars alive so they get codegen'd properly + # TODO see if we could prune dead cells - cell pruning information needs to be forwarded + # to the resume function creation as well. + assert cur_tx.post_prune_cell_and_freevars is not None + init_live_vars.append(cur_tx.post_prune_cell_and_freevars) + cur_tx = cur_tx.parent + VariableTracker.visit( + visit, + # TODO track from all possible sources. + init_live_vars + + [ + pre_existing_vars, + tx.output.backward_state, + self.tensor_hooks, + ], + ) + # Manually release the self-referential function, which indirectly + # captures certain `VariableTracker` and affects parts of PT test/logic + # that are sensitive to when certain objects get released. + del visit + + # NB: cell variable handling.is tricky. + # cell variables must stay alive if any NestedUserFunctionVariable + # are live. "visit"-ing the NestedUserFunctionVariable visits + # the .closures field, from which we will see if we need to keep + # any mutations to cell variables alive. + + self.id_to_variable = { + k: v for k, v in self.id_to_variable.items() if is_live(v) + } + self.store_attr_mutations = { + k: v for k, v in self.store_attr_mutations.items() if is_live(k) + } + + def mutation(self, var: VariableTracker) -> None: + if var in self.ignore_mutation_on_these_variables: + return + + self.check_allowed_side_effect(var) + if isinstance(var.mutation_type, ValueMutationExisting): + var.mutation_type.is_modified = True + if ( + var.source + and isinstance(var, variables.ConstDictVariable) + and not isinstance(var, variables.SetVariable) + ): + self._has_existing_dict_mutation = True + + def has_existing_dict_mutation(self) -> bool: + return self._has_existing_dict_mutation + + def _get_modified_vars(self) -> list[VariableTracker]: + return [var for var in self.id_to_variable.values() if self.is_modified(var)] + + def codegen_save_tempvars(self, cg: PyCodegen) -> None: + # We must codegen modified VT to their source by default, so that + # mutation and aliasing are properly accounted for. + # + # Since newly constructed objects don't have a source, we manually + # codegen their construction and store them to a newly assigned local + # source. Note that `ValueMutationNew` isn't tracked by SideEffects. + for var in self._get_modified_vars(): + if not isinstance(var.mutation_type, AttributeMutationNew): + assert var.source is not None + continue + + if isinstance(var, variables.CellVariable): + # Cells created in the root frame are created either by + # `MAKE_CELL` or by them being in `co_cellvars`, so we only emit + # `make_cell` for the non-root-frame cells here. + # TODO generalize this so we never need to call `make_cell`. + if var.local_name is None: + cg.add_push_null( + lambda: cg.load_import_from(utils.__name__, "make_cell") + ) + cg.extend_output(create_call_function(0, False)) + cg.add_cache(var) + var.source = TempLocalSource(cg.tempvars[var]) # type: ignore[attr-defined] + elif var.source is None: + # pyrefly: ignore [bad-assignment] + var.source = LocalCellSource(var.local_name) + elif var.is_tensor(): + # NOTE: for historical reasons we never assigned local sources + # to newly constructed tensor object, so we keep it that way. + # They are always loaded from output of the fx graph, so one can + # think of it as having a "OutputGraphSource" for codegen + # purposes. + # + # However, tensor subclass objects are different, because the + # reconstruction logic in `PyCodegen` loads the data tensor from + # graph output and then calls `as_subclass`, meaning we must + # assign a source to it to ensure we only reconstruct one + # subclass instance. + if isinstance( + var, variables.torch_function.TensorWithTFOverrideVariable + ): + # Don't codegen from temp source assigned from the 1st pass. + cg(var, allow_cache=False) + cg.add_cache(var) + # `add_cache` generates STORE and consumes TOS, but we never + # cleared it. TODO move this call into `add_cache` + cg.clear_tos() + var.source = TempLocalSource(cg.tempvars[var]) + elif isinstance(var, variables.AutogradFunctionContextVariable): + unimplemented( + gb_type="AutogradFunctionContextVariable escaped Dynamo-traced region", + context="", + explanation="We cannot reconstruct a torch.autograd.Function's context object.", + hints=[], + ) + else: + # Reconstruct the bytecode for + # base_cls.__new__(user_cls, *args) + if isinstance(var, variables.UserDefinedObjectVariable): + + def load_new_method() -> None: + # pyrefly: ignore [missing-attribute] + assert var.base_cls_vt is not None + cg(var.base_cls_vt) # type: ignore[attr-defined] + cg.extend_output([cg.create_load_attr("__new__")]) + + cg.add_push_null(load_new_method) + else: + cg.add_push_null( + lambda: cg.load_import_from(utils.__name__, "object_new") + ) + assert var.mutation_type.cls_source is not None + cg(var.mutation_type.cls_source) + + # Generate the args to the __new__ method + for arg in var.init_args: # type: ignore[attr-defined] + cg(arg) + + # Call the __new__ method + cg.extend_output(create_call_function(1 + len(var.init_args), False)) # type: ignore[attr-defined] + + cg.add_cache(var) + var.source = TempLocalSource(cg.tempvars[var]) + + for ctx, args in self.save_for_backward: + cg(ctx.source) + cg.load_method("save_for_backward") + for arg in args: + cg(arg) + cg.extend_output( + [ + *create_call_method(len(args)), + create_instruction("POP_TOP"), + ] + ) + + def register_hook( + self, + tensor: "variables.TensorVariable", + hook: VariableTracker, + handle: "variables.RemovableHandleVariable", + name: str, + ) -> None: + assert tensor.is_tensor() + assert isinstance(hook, variables.VariableTracker) + assert ( + isinstance(handle, variables.RemovableHandleVariable) + and handle.is_mutable() + ) + assert hasattr(torch.Tensor, name) + idx = len(self.tensor_hooks.keys()) + # duplicate index possible because of self.remove_hook() + while idx in self.tensor_hooks: + idx += 1 + self.tensor_hooks[idx] = (tensor, hook, handle, name) + assert not handle.idx + handle.idx = idx + + def remove_hook(self, idx: int) -> None: + del self.tensor_hooks[idx] + + def codegen_hooks(self, cg: PyCodegen) -> None: + for ( + tensor, + hook, + handle, + name, + ) in self.tensor_hooks.values(): + # Note: [On tensor.register_hook] + # + # register_hook on a tensor, AKA backward hooks, have slightly nuanced differences in how they are implemented + # when it comes to hooks on objects with sources (inputs, params) vs objects without sources (intermediaries). + # + # For tensors with a source, we bypass direct inclusion of register_hook calls in the graph. + # Instead, these are tracked and stashed as a global variable, enabling their association with tensors in + # the residuals. During dynamo's frame creation, these hooks are invoked seamlessly on known reconstructible/fetch-able + # tensors. Because a source indicates knowledge of this object outside the torch compile region, and + # because we are running residuals firmly before .backward() can be run, it is sound to invoke + # `register_hook` on a known tensor. + # + # For tensors without a source, we support a limited subset of hooks. Global functions only, and + # compiled_autograd must be enabled or we will graph break. + # + # Handling the Handle: When a user retains the register_hook result in a handle, we intercept the + # STORE_FAST operation to record the user-designated local variable name. This ensures the reconstructed + # bytecode retains this name. If no handle is defined, we simply pop the generated value to keep the + # stack intact. + # + # Dynamo Tensor Hooks Workflow: + # - Functions passed to register_hook are lifted globally. + # - For tensors with sources: + # - In the "side_effects" phase of codegen, we iterate over tensors with hooks to: + # - Generate the tensor. + # - Issue a register_hook call on the tensor, linking to the globally stored function. + # - Incorporate a handle if one was established in the eager phase. + # - For tensors without sources: + # - We don't generate any instructions for registering a hook. + # - Handles from intermediary hooks are NYI. + # - We produce a call function that utilizes the trace_wrapped higher order op, closing over it. + # - We then manually insert the call function above into the graph. + # - The handle's exact user-specified name, "user_code_variable_name", is discerned and associated during STORE_FAST. + assert tensor.source, "Hooks on non input tensors NYI - should not get here" + + def gen_fn() -> None: + cg(tensor) + cg.extend_output([cg.create_load_attr(name)]) + + cg.add_push_null(gen_fn) + cg(hook) + cg.extend_output(create_call_function(1, False)) + + # Adding the handle to the cache means RemovableHandleVariable().reconstruct() will + # be associated with the return value of register_hook(). This consumes the top of stack. + cg.add_cache(handle) + + def get_ca_final_callbacks_var(self) -> "variables.ListVariable": + from .variables.base import ValueMutationNew + + if self.ca_final_callbacks_var is None: + self.ca_final_callbacks_var = variables.ListVariable( + [], mutation_type=ValueMutationNew() + ) + + return self.ca_final_callbacks_var + + def codegen_update_mutated(self, cg: PyCodegen) -> None: + suffixes = [] + for var in self._get_modified_vars(): + if isinstance(var, variables.ListVariable): + # old[:] = new + cg(var, allow_cache=False) # Don't codegen via source + cg(var.source) # type: ignore[attr-defined] + cg.extend_output( + [ + cg.create_load_const(None), + cg.create_load_const(None), + create_instruction("BUILD_SLICE", arg=2), + ] + ) + suffixes.append([create_instruction("STORE_SUBSCR")]) + elif isinstance(var, variables.lists.DequeVariable): + # For limited maxlen, the order of operations matter for side + # effect, but we currently don't track the order, so no support. + if not var.maxlen.is_constant_none(): + unimplemented( + gb_type="Side effect on existing deque with limited maxlen", + context="", + explanation="This is not supported.", + hints=[ + "Don't use a deque with `maxlen` specified.", + ], + ) + + # old.extend(new), this runs last + cg(var.source) + cg.load_method("extend") + cg(var, allow_cache=False) # Don't codegen via source + suffixes.append( + [ + *create_call_method(1), + create_instruction("POP_TOP"), + ] + ) + + # old.clear(), this runs first + cg(var.source) + cg.load_method("clear") + suffixes.append( + [ + *create_call_method(0), + create_instruction("POP_TOP"), + ] + ) + + elif isinstance(var, variables.ConstDictVariable): + # Reconstruct works as follow: + # (1) Skip codegen if there are no new items + # (2) codegen(...) each pair of key/value + # (3) create a new dictionary with the pairs of key/values above + # (4) clear the original dictionary + # + only if a key was removed from the input dict + # (5) update the original dictionary with the dict created in (2) + + if var.has_new_items(): + cg(var.source) # type: ignore[attr-defined] + cg.load_method("update") + cg(var, allow_cache=False) # Don't codegen via source + + if var.should_reconstruct_all: + cg(var.source) # type: ignore[attr-defined] + cg.load_method("clear") + + suffixes.append( + [ + *create_call_method(1), # update + create_instruction("POP_TOP"), + ] + ) + + if var.should_reconstruct_all: + # clear will appear before "update" as the suffixes are + # applied in reverse order. + suffixes.append( + [ + *create_call_method(0), # clear + create_instruction("POP_TOP"), + ] + ) + + elif isinstance( + var, variables.torch_function.TorchFunctionModeStackVariable + ): + # Needed in the finally block for stack restoration + cg.add_push_null( + lambda: cg.load_import_from( + utils.__name__, "get_torch_function_mode_stack" + ) + ) + cg.call_function(0, False) + name = variables.torch_function.get_prev_stack_var_name() + cg.code_options["co_varnames"] += (name,) + cg.append_output(create_instruction("STORE_FAST", argval=name)) + cg.add_push_null( + lambda: cg.load_import_from( + utils.__name__, "set_torch_function_mode_stack" + ) + ) + + cg.foreach(var.symbolic_stack) + cg.append_output( + create_instruction("BUILD_LIST", arg=len(var.symbolic_stack)) + ) + cg.call_function(1, False) + cg.append_output(create_instruction("POP_TOP")) + + elif isinstance(var, variables.CellVariable) and var.local_name is not None: + # Emit more readable and performant bytecode. + # TODO generalize this for cells created during inlining. + if var in self.store_attr_mutations: + contents_var = self.load_cell(var) + cg(contents_var) + suffixes.append([cg.create_store_deref(var.local_name)]) + + elif self.is_attribute_mutation(var): + if isinstance( + var, + variables.UserDefinedDictVariable, + # pyrefly: ignore [bad-argument-type] + ) and self.is_modified(var._dict_vt): + # Do dict related update manually here. The store_attr + # mutations will be applied later. + varname_map = {} + for name in _manual_dict_setitem.__code__.co_varnames: + varname_map[name] = cg.tx.output.new_var() + + try: + mro_index = type(var.value).__mro__.index( + collections.OrderedDict + ) + except ValueError: + mro_index = type(var.value).__mro__.index(dict) + + cg.extend_output( + [ + create_instruction("LOAD_CONST", argval=mro_index), + create_instruction( + "STORE_FAST", argval=varname_map["mro_index"] + ), + ] + ) + + cg(var.source) # type: ignore[attr-defined] + cg.extend_output( + [ + create_instruction( + "STORE_FAST", argval=varname_map["dict_to"] + ) + ] + ) + + # pyrefly: ignore [bad-argument-type] + cg(var._dict_vt, allow_cache=False) # Don't codegen via source + cg.extend_output( + [ + create_instruction( + "STORE_FAST", argval=varname_map["dict_from"] + ) + ] + ) + + dict_update_insts = bytecode_from_template( + _manual_dict_setitem, varname_map=varname_map + ) + + suffixes.append( + [ + *dict_update_insts, + create_instruction("POP_TOP"), + ] + ) + elif isinstance( + var, + variables.UserDefinedListVariable, + # pyrefly: ignore [bad-argument-type] + ) and self.is_modified(var._list_vt): + # Update the list to the updated items. Be careful in + # calling the list methods and not the overridden methods. + varname_map = {} + for name in _manual_list_update.__code__.co_varnames: + varname_map[name] = cg.tx.output.new_var() + + cg(var.source) # type: ignore[attr-defined] + cg.extend_output( + [ + create_instruction( + "STORE_FAST", argval=varname_map["list_to"] + ) + ] + ) + + # pyrefly: ignore [bad-argument-type] + cg(var._list_vt, allow_cache=False) # Don't codegen via source + cg.extend_output( + [ + create_instruction( + "STORE_FAST", argval=varname_map["list_from"] + ) + ] + ) + + list_update_insts = bytecode_from_template( + _manual_list_update, varname_map=varname_map + ) + + suffixes.append( + [ + *list_update_insts, + create_instruction("POP_TOP"), + ] + ) + + # Applying mutations involves two steps: 1) Push all + # reconstructed objects onto the stack. 2) Call STORE_ATTR to + # apply the mutations. + # + # Dynamo must ensure that mutations are applied in the same + # order as in the original program. Therefore, two reverse + # operations occur below. + # + # The first reverse operation concerns `suffixes`. We apply + # suffixes in reverse order due to the way Python handles the + # stack. In Step 1, we push all reconstructed objects onto the + # stack, but the item at the top of the stack refers to the last + # attribute in the mutation order. If not fixed, this will apply + # the mutations of attributes in the reverse order. To account + # for this reversal, we iterate through the mutable attributes + # in reverse order. + for name, value in reversed( + self.store_attr_mutations.get(var, {}).items() + ): + if isinstance(var, variables.NewGlobalVariable): + cg.tx.output.update_co_names(name) + cg(value) + assert isinstance(var.source, GlobalSource) # type: ignore[attr-defined] + suffixes.append( + [create_instruction("STORE_GLOBAL", argval=name)] + ) + elif isinstance(value, variables.DeletedVariable): + if isinstance( + var.mutation_type, AttributeMutationExisting + ) and hasattr(getattr(var, "value", None), name): + cg.tx.output.update_co_names(name) + cg(var.source) + suffixes.append( + [create_instruction("DELETE_ATTR", argval=name)] + ) + elif isinstance( + var, variables.UserDefinedObjectVariable + ) and var.should_skip_descriptor_setter(name): + cg.add_push_null( + lambda: cg.load_import_from( + utils.__name__, "object_setattr_ignore_descriptor" + ) + ) + cg(var.source) # type: ignore[attr-defined] + cg(variables.ConstantVariable(name)) + cg(value) + suffixes.append( + [ + *create_call_function(3, False), + create_instruction("POP_TOP"), + ] + ) + elif ( + isinstance(var, variables.UserDefinedObjectVariable) + and var.needs_slow_setattr() + ): + # __setattr__ is defined on this object, so call object.__setattr__ directly + cg.load_import_from("builtins", "object") + cg.load_method("__setattr__") + cg(var.source) # type: ignore[attr-defined] + cg(variables.ConstantVariable(name)) + cg(value) + suffixes.append( + [*create_call_method(3), create_instruction("POP_TOP")] + ) + else: + cg.tx.output.update_co_names(name) + cg(value) + cg(var) + suffixes.append([create_instruction("STORE_ATTR", argval=name)]) + elif isinstance(var, variables.ListIteratorVariable): + for _ in range(var.index): + cg.add_push_null( + lambda: cg.load_import_from(utils.__name__, "iter_next") + ) + cg(var.source) # type: ignore[attr-defined] + cg.call_function(1, False) + cg.pop_top() + elif isinstance(var, variables.RandomVariable): + # set correct random seed state + def gen_fn() -> None: + cg(var.source) # type: ignore[attr-defined] + cg.load_attr("setstate") + + cg.add_push_null(gen_fn) + cg(var.wrap_state(var.random.getstate())) + + suffixes.append( + [ + *create_call_function(1, False), # setstate + create_instruction("POP_TOP"), + ] + ) + else: + raise AssertionError(type(var)) + + # do all the actual mutations at the very end to handle dependencies + for suffix in reversed(suffixes): + cg.extend_output(suffix) + + def is_empty(self) -> bool: + return not ( + any(map(self.is_modified, self.id_to_variable.values())) + or self.tensor_hooks + or self.save_for_backward + or self.tensor_hooks + ) + + def clear(self) -> None: + self.keepalive.clear() + self.id_to_variable.clear() + + +@contextlib.contextmanager +def allow_side_effects_in_hop( + tx: "InstructionTranslatorBase", +) -> Generator[None, None, None]: + """Context manager to temporarily allow side effects with extra outputs. + + This is used for special cases (like FSDP functions) that need to perform + side effects even when the general policy is to disallow them. + """ + orig_val = tx.output.current_tracer.allow_side_effects_in_hop + try: + tx.output.current_tracer.allow_side_effects_in_hop = True + yield + finally: + tx.output.current_tracer.allow_side_effects_in_hop = orig_val + + +@contextlib.contextmanager +def allow_externally_visible_side_effects_in_subtracer( + tx: "InstructionTranslatorBase", +) -> Generator[None, None, None]: + orig_val = tx.output.current_tracer.unsafe_allow_externally_visible_side_effects + try: + tx.output.current_tracer.unsafe_allow_externally_visible_side_effects = True + tx.output.current_tracer.traced_with_externally_visible_side_effects = True + yield + finally: + tx.output.current_tracer.unsafe_allow_externally_visible_side_effects = orig_val + + +@contextlib.contextmanager +def disallow_side_effects_in_generator( + tx: "InstructionTranslatorBase", +) -> Generator[None, None, None]: + orig_val = tx.output.current_tracer.is_reconstructing_generator + try: + tx.output.current_tracer.is_reconstructing_generator = True + yield + finally: + tx.output.current_tracer.is_reconstructing_generator = orig_val diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/source.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/source.py new file mode 100644 index 0000000000000000000000000000000000000000..dd3386f765cfebf5e5107065bdb1ed0141ff341a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/source.py @@ -0,0 +1,1300 @@ +""" +This module provides Source classes that track the origins of values in PyTorch Dynamo. +Sources represent where values come from (e.g. local variables, globals, attributes) and +are used for guard generation and code reconstruction during compilation. + +The module includes specialized sources for: +- Local variables and synthetic locals +- Global variables and constants +- Object attributes and method calls +- NN module specialization (specialized vs unspecialized) +- Random values and tensor properties +- Default argument handling +- FSDP (Fully Sharded Data Parallel) modules + +Sources play a key role in Dynamo's guard system by tracking value origins for +guard generation, and in code reconstruction by providing methods to rebuild +the code needed to recreate values. +""" + +import dataclasses +import enum +import functools +from collections.abc import Callable +from typing import Any, Optional, TYPE_CHECKING, Union + +from torch import device as device_type +from torch._guards import ( + ChainedSource, + dataclass_with_cached_hash, + Guard, + GuardSource, + Source, +) + +from . import utils +from .bytecode_transformation import ( + create_binary_subscr, + create_build_tuple, + create_call_function, +) + + +if TYPE_CHECKING: + from .codegen import PyCodegen + +# It shouldn't be supported to construct an NNModuleVariable inside an FSDP module, +# so those cases are omitted intentionally + +# represents nn.Modules tracked with NNModuleVariable (specialized is implicit in the variable name) +_GUARD_SOURCE_SPECIALIZED_NN_MODULE = { + GuardSource.LOCAL: GuardSource.LOCAL_SPECIALIZED_NN_MODULE, + GuardSource.GLOBAL: GuardSource.GLOBAL_SPECIALIZED_NN_MODULE, + GuardSource.LOCAL_SPECIALIZED_NN_MODULE: GuardSource.LOCAL_SPECIALIZED_NN_MODULE, + GuardSource.GLOBAL_SPECIALIZED_NN_MODULE: GuardSource.GLOBAL_SPECIALIZED_NN_MODULE, + # Just to ensure that guard_source() works + GuardSource.LOCAL_UNSPECIALIZED_NN_MODULE: GuardSource.LOCAL_UNSPECIALIZED_NN_MODULE, + GuardSource.GLOBAL_UNSPECIALIZED_NN_MODULE: GuardSource.GLOBAL_UNSPECIALIZED_NN_MODULE, + GuardSource.LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE: GuardSource.LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + GuardSource.GLOBAL_UNSPECIALIZED_BUILTIN_NN_MODULE: GuardSource.GLOBAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + GuardSource.LOCAL_FSDP_MODULE: GuardSource.LOCAL_FSDP_MODULE, + GuardSource.GLOBAL_FSDP_MODULE: GuardSource.GLOBAL_FSDP_MODULE, +} + +# represents nn.Modules tracked with UnspecializedNNModuleVariable +_GUARD_SOURCE_UNSPECIALIZED_NN_MODULE = { + GuardSource.LOCAL: GuardSource.LOCAL_UNSPECIALIZED_NN_MODULE, + GuardSource.GLOBAL: GuardSource.GLOBAL_UNSPECIALIZED_NN_MODULE, + GuardSource.LOCAL_UNSPECIALIZED_NN_MODULE: GuardSource.LOCAL_UNSPECIALIZED_NN_MODULE, + GuardSource.GLOBAL_UNSPECIALIZED_NN_MODULE: GuardSource.GLOBAL_UNSPECIALIZED_NN_MODULE, + # this happens for an UnspecializedNNModule submodule on a NNModuleVariable + GuardSource.LOCAL_SPECIALIZED_NN_MODULE: GuardSource.LOCAL_UNSPECIALIZED_NN_MODULE, + GuardSource.GLOBAL_SPECIALIZED_NN_MODULE: GuardSource.GLOBAL_UNSPECIALIZED_NN_MODULE, + # Just to ensure that guard_source() works + GuardSource.LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE: GuardSource.LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + GuardSource.GLOBAL_UNSPECIALIZED_BUILTIN_NN_MODULE: GuardSource.GLOBAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + GuardSource.LOCAL_FSDP_MODULE: GuardSource.LOCAL_FSDP_MODULE, + GuardSource.GLOBAL_FSDP_MODULE: GuardSource.GLOBAL_FSDP_MODULE, +} + +# represents nn.Modules tracked with UnspecializedBuiltinNNModuleVariable +_GUARD_SOURCE_UNSPECIALIZED_BUILTIN_NN_MODULE = { + GuardSource.LOCAL: GuardSource.LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + GuardSource.GLOBAL: GuardSource.GLOBAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + GuardSource.LOCAL_UNSPECIALIZED_NN_MODULE: GuardSource.LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + GuardSource.GLOBAL_UNSPECIALIZED_NN_MODULE: GuardSource.GLOBAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + GuardSource.LOCAL_SPECIALIZED_NN_MODULE: GuardSource.LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + GuardSource.GLOBAL_SPECIALIZED_NN_MODULE: GuardSource.GLOBAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + # Just to ensure that guard_source() works + GuardSource.LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE: GuardSource.LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + GuardSource.GLOBAL_UNSPECIALIZED_BUILTIN_NN_MODULE: GuardSource.GLOBAL_UNSPECIALIZED_BUILTIN_NN_MODULE, + GuardSource.LOCAL_FSDP_MODULE: GuardSource.LOCAL_FSDP_MODULE, + GuardSource.GLOBAL_FSDP_MODULE: GuardSource.GLOBAL_FSDP_MODULE, +} + +_GUARD_SOURCE_FSDP_MODULE = { + GuardSource.LOCAL: GuardSource.LOCAL_FSDP_MODULE, + GuardSource.GLOBAL: GuardSource.GLOBAL_FSDP_MODULE, + GuardSource.LOCAL_SPECIALIZED_NN_MODULE: GuardSource.LOCAL_FSDP_MODULE, + GuardSource.GLOBAL_SPECIALIZED_NN_MODULE: GuardSource.GLOBAL_FSDP_MODULE, + GuardSource.LOCAL_FSDP_MODULE: GuardSource.LOCAL_FSDP_MODULE, + GuardSource.GLOBAL_FSDP_MODULE: GuardSource.GLOBAL_FSDP_MODULE, + GuardSource.LOCAL_UNSPECIALIZED_NN_MODULE: GuardSource.LOCAL_FSDP_MODULE, + GuardSource.GLOBAL_UNSPECIALIZED_NN_MODULE: GuardSource.GLOBAL_FSDP_MODULE, + GuardSource.LOCAL_UNSPECIALIZED_BUILTIN_NN_MODULE: GuardSource.LOCAL_FSDP_MODULE, + GuardSource.GLOBAL_UNSPECIALIZED_BUILTIN_NN_MODULE: GuardSource.GLOBAL_FSDP_MODULE, +} + + +def is_constant_source(source: Source) -> bool: + if isinstance(source, ConstantSource): + return True + try: + if source.guard_source == GuardSource.CONSTANT: + return True + except NotImplementedError: + pass + + return False + + +def _get_source_debug_name(source: Optional[Source]) -> str: + if source is None: + return "" + else: + try: + return source.name + except NotImplementedError: + return "" + + +def _esc_str(s: Any, apply_repr: bool = False) -> str: + """ + Escapes curly brackets for format strings. + e.g. "frozenset({0})" becomes "frozenset({{0}})". + This is used by _name_template for example, because it's + expected to return a format string, but we may wish to include + strings that should not be accidentally formatted. + """ + if apply_repr: + s = repr(s) + else: + s = str(s) + return s.replace("{", "{{").replace("}", "}}") + + +@dataclass_with_cached_hash(frozen=True) +class LocalSource(Source): + local_name: str + + # Whether this local is an input to the root frame. + is_input: bool = False + + # Whether we know this input is dynamic (based on example_inputs) + # For non tensors, we simply look at the first index of the tuple + dynamism: Optional[frozenset[str]] = None + + # Whether the item at this source is the _content_ of a cell that is + # dereferenced from the root frame, i.e., it's a part of the `co_cellvars` + # or `co_freevars`. + is_derefed_cell_contents: bool = False + + def reconstruct(self, codegen: "PyCodegen") -> None: + if self.is_derefed_cell_contents: + codegen.load_deref(self.local_name) + else: + codegen.append_output(codegen.create_load(self.local_name)) + + @property + def guard_source(self) -> GuardSource: + return GuardSource.LOCAL + + @functools.cached_property + def _name_template(self) -> str: + return f"L[{_esc_str(self.local_name, apply_repr=True)}]" + + +@dataclass_with_cached_hash(frozen=True) +class TempLocalSource(Source): + # like LocalSource, but cannot be guarded on + local_name: str + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.append_output(codegen.create_load(self.local_name)) + + @property + def guard_source(self) -> GuardSource: + return GuardSource.TEMP_LOCAL + + @property + def _name_template(self) -> str: + raise NotImplementedError( + "Cannot create guard on TempLocalSource - this is an internal Dynamo bug. Please file an issue on GitHub." + ) + + +@dataclass_with_cached_hash(frozen=True) +class SyntheticLocalSource(Source): + local_name: str + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.append_output(codegen.create_load(self.local_name)) + + @property + def guard_source(self) -> GuardSource: + return GuardSource.SYNTHETIC_LOCAL + + @functools.cached_property + def _name_template(self) -> str: + return f"SYNTHETIC_LOCAL[{_esc_str(self.local_name, apply_repr=True)}]" + + +@dataclass_with_cached_hash(frozen=True) +class RandomValueSource(Source): + random_call_index: int + + @property + def guard_source(self) -> GuardSource: + return GuardSource.RANDOM_VALUE + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.append_output(codegen.create_load(codegen.tx.output.random_values_var)) + codegen.append_output(codegen.create_load_const(self.random_call_index)) + codegen.append_output(create_binary_subscr()) + + @functools.cached_property + def _name_template(self) -> str: + return f"random_value_{_esc_str(self.random_call_index)}" + + +@dataclass_with_cached_hash(frozen=True) +class GlobalSource(Source): + global_name: str + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.append_output(codegen.create_load_global(self.global_name, add=True)) + + @property + def guard_source(self) -> GuardSource: + return GuardSource.GLOBAL + + @functools.cached_property + def _name_template(self) -> str: + return f"G[{_esc_str(self.global_name, apply_repr=True)}]" + + +@dataclass_with_cached_hash(frozen=True) +class GlobalWeakRefSource(Source): + global_name: str + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.add_push_null( + lambda: codegen.append_output( + codegen.create_load_global(self.global_name, add=True) + ) + ) + codegen.extend_output(create_call_function(0, False)) + + @property + def guard_source(self) -> GuardSource: + return GuardSource.GLOBAL + + @functools.cached_property + def _name_template(self) -> str: + return f"G[{_esc_str(self.global_name, apply_repr=True)}]()" + + +@dataclass_with_cached_hash(frozen=True) +class WeakRefCallSource(ChainedSource): + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.add_push_null(lambda: codegen(self.base)) + codegen.extend_output(create_call_function(0, False)) + + @property + def _name_template(self) -> str: + return "{0}()" + + +@dataclass_with_cached_hash(frozen=True) +class CallFunctionNoArgsSource(WeakRefCallSource): + pass + + +@dataclass_with_cached_hash(frozen=True) +class AttrSource(ChainedSource): + member: str + + def __post_init__(self) -> None: + assert self.base, "Can't construct an AttrSource without a valid base source" + if "." in self.member: + member_parts = self.member.split(".") + object.__setattr__( + self, "base", AttrSource(self.base, ".".join(member_parts[:-1])) + ) + object.__setattr__(self, "member", member_parts[-1]) + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + codegen.extend_output(codegen.create_load_attrs(self.member)) + + @functools.cached_property + def _name_template(self) -> str: + if not self.member.isidentifier(): + return f"getattr({{0}}, {_esc_str(self.member, apply_repr=True)})" + return f"{{0}}.{_esc_str(self.member)}" + + +@dataclass_with_cached_hash(frozen=True) +class GenericAttrSource(ChainedSource): + member: str + + def __post_init__(self) -> None: + assert self.base, "Can't construct an AttrSource without a valid base source" + if "." in self.member: + member_parts = self.member.split(".") + object.__setattr__( + self, "base", AttrSource(self.base, ".".join(member_parts[:-1])) + ) + object.__setattr__(self, "member", member_parts[-1]) + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + codegen.extend_output(codegen.create_load_attrs(self.member)) + + @functools.cached_property + def _name_template(self) -> str: + return ( + f"object.__getattribute__({{0}}, {_esc_str(self.member, apply_repr=True)})" + ) + + +# Represents obj.__dict__ where obj is a type object +@dataclass_with_cached_hash(frozen=True) +class TypeDictSource(ChainedSource): + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + codegen.extend_output(codegen.create_load_attrs("__dict__")) + + @property + def _name_template(self) -> str: + # type(ob).__dict__ can return a proxy of the dict. But in the C++ + # guard accessor, we are use type->tp_dict which is a dict. So, + # forcefully pass a dict object to ensure that the GuardManager + # registers that its working on a dict object. + return "dict({0}.__dict__)" + + +# Represents obj.__mro__ where object is type object +@dataclass_with_cached_hash(frozen=True) +class TypeMROSource(ChainedSource): + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + codegen.extend_output(codegen.create_load_attrs("__mro__")) + + @property + def _name_template(self) -> str: + return "{0}.__mro__" + + +@dataclass_with_cached_hash(frozen=True) +class LocalCellSource(Source): + """ + Conceptually, this class is `LocalSource` for cell objects implicitly + generated by Python (e.g., captured variables). + """ + + local_name: str + + def reconstruct(self, codegen: "PyCodegen") -> None: + # Although `LOAD_FAST` and `LOAD_CLOSURE` have the same semantics, + # Dynamo's bytecode transformation differentiates them slightly, so we + # always emit `LOAD_CLOSURE` here. + codegen.append_output(codegen.create_load_closure(self.local_name)) + + # All the other methods are intentionally unimplemented because e.g., a + # local cell object should never be used for guards. + + +# Represents obj.__code__ where object is type object +@dataclass_with_cached_hash(frozen=True) +class CodeSource(ChainedSource): + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + codegen.extend_output(codegen.create_load_attrs("__code__")) + + @property + def _name_template(self) -> str: + return "{0}.__code__" + + +# Represents obj.__closure__ where object is type object +@dataclass_with_cached_hash(frozen=True) +class ClosureSource(ChainedSource): + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + codegen.extend_output(codegen.create_load_attrs("__closure__")) + + @property + def _name_template(self) -> str: + return "{0}.__closure__" + + +# Represents tensor.grad source. It could be represented by AttrSource as well. +# But, we could access grad field on tensor directly in C++ without going +# through the Python bytecodes. Therefore, we use a separate source for grad +# field. +@dataclass_with_cached_hash(frozen=True) +class GradSource(ChainedSource): + member: str = "grad" + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + codegen.extend_output(codegen.create_load_attrs(self.member)) + + @functools.cached_property + def _name_template(self) -> str: + return f"{{0}}.{_esc_str(self.member)}" + + +@dataclass_with_cached_hash(frozen=True) +class ParamBufferSource(AttrSource): + @functools.cached_property + def guard_source(self) -> GuardSource: + return _GUARD_SOURCE_SPECIALIZED_NN_MODULE[self.base.guard_source] + + +# Special AttrSource to differentiate module._buffers or module._parameters +@dataclass_with_cached_hash(frozen=True) +class UnspecializedParamBufferSource(AttrSource): + pass + + +# This source is intended to be used in places where a source is needed but it is expected +# that the symbol will be simplified out later on. Symbols with ephemeral sources are +# prioritized to be simplified out when e.g. compared against a symbol without an ephemeral +# source. Guarding on this source is an error. +# +# Example: During subclass view fake-ification, any close-over ViewFunc state should be +# symbolicized / fake-ified to avoid invalid specialization during view replay. This source +# is useful for symbols utilized in the middle of the view chain that are not expected to be +# present within the final view shape metadata. +@dataclass_with_cached_hash(frozen=True) +class EphemeralSource(Source): + desc: Optional[str] = None + + @property + def guard_source(self) -> GuardSource: + return GuardSource.EPHEMERAL + + @functools.cached_property + def _name_template(self) -> str: + desc = ": " + self.desc if self.desc is not None else "" + return f"" + + def make_guard(self, fn: Callable[..., Any]) -> Guard: + raise NotImplementedError + + def is_ephemeral(self) -> bool: + return True + + +@dataclass_with_cached_hash(frozen=True) +class SkipGuardSource(ChainedSource): + def reconstruct(self, codegen: "PyCodegen") -> None: + self.base.reconstruct(codegen) + + @property + def _name_template(self) -> str: + return "{0}" + + +class TensorProperty(enum.Enum): + SIZE = 0 + STRIDE = 1 + STORAGE_OFFSET = 2 + + def method_name(self) -> str: + if self is TensorProperty.SIZE: + return "size" + elif self is TensorProperty.STRIDE: + return "stride" + elif self is TensorProperty.STORAGE_OFFSET: + return "storage_offset" + else: + raise AssertionError(f"unhandled {_esc_str(self)}") + + +@dataclass_with_cached_hash(frozen=True) +class TensorPropertySource(ChainedSource): + prop: TensorProperty + idx: Optional[int] = None # None for STORAGE_OFFSET + + def __post_init__(self) -> None: + assert self.base is not None + if self.prop is TensorProperty.STORAGE_OFFSET: + assert self.idx is None + else: + assert self.idx is not None + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.add_push_null( + lambda: codegen.load_import_from( + utils.__name__, f"call_{_esc_str(self.prop.method_name())}" + ) + ) + codegen(self.base) + + if self.idx is not None: + codegen.append_output(codegen.create_load_const(self.idx)) + codegen.extend_output( + create_call_function(2 if self.idx is not None else 1, False) + ) + + @functools.cached_property + def _name_template(self) -> str: + if self.prop is TensorProperty.SIZE: + return f"{{0}}.size()[{_esc_str(self.idx)}]" + elif self.prop is TensorProperty.STRIDE: + return f"{{0}}.stride()[{_esc_str(self.idx)}]" + elif self.prop is TensorProperty.STORAGE_OFFSET: + assert self.idx is None + return "{0}.storage_offset()" + else: + raise AssertionError(f"unhandled {_esc_str(self.prop)}") + + +@dataclass_with_cached_hash(frozen=True) +class IndexedSource(ChainedSource): + idx: int + + def __post_init__(self) -> None: + assert self.base is not None + + def reconstruct(self, codegen: "PyCodegen") -> None: + raise NotImplementedError + + @functools.cached_property + def _name_template(self) -> str: + return f"({_esc_str(self.idx)}, {{0}})" + + +@dataclass_with_cached_hash(frozen=True) +class NegateSource(ChainedSource): + def __post_init__(self) -> None: + assert self.base is not None + + def reconstruct(self, codegen: "PyCodegen") -> None: + raise NotImplementedError + + @property + def _name_template(self) -> str: + # NB: use method call so that function stripping regexes work + return "{0}.__neg__()" + + +@dataclass_with_cached_hash(frozen=True) +class ConvertIntSource(ChainedSource): + def __post_init__(self) -> None: + assert self.base is not None + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + + @property + def _name_template(self) -> str: + return "cast_symbool_to_symint_guardless({0})" + + +@dataclass_with_cached_hash(frozen=True) +class DynamicScalarSource(ChainedSource): + is_int: bool + + def __post_init__(self) -> None: + assert self.base is not None + + def reconstruct(self, codegen: "PyCodegen") -> None: + # Integer casting at reconstruction helps reduce the amount of DynamicInts returned + # to the user, in favor of plain ints. + # For example, a compiled region that only does int arithmetic could return a + # DynamicInt without the casting here. + codegen.add_push_null(lambda: codegen.load_import_from("builtins", "int")) + codegen(self.base) + codegen.extend_output(create_call_function(1, False)) + + @property + def _name_template(self) -> str: + return "int({0})" + + +@dataclass_with_cached_hash(frozen=True) +class FlattenScriptObjectSource(ChainedSource): + def __post_init__(self) -> None: + assert self.base is not None + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + + @property + def _name_template(self) -> str: + return "{0}.__obj_flatten__()" + + +@dataclass_with_cached_hash(frozen=True) +class ScriptObjectQualifiedNameSource(ChainedSource): + def __post_init__(self) -> None: + assert self.base is not None + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + + @property + def _name_template(self) -> str: + return "{0}._type().qualified_name()" + + +class AttrProxySource(ChainedSource): + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + + @property + def _name_template(self) -> str: + return "{0}.get_base()" + + +@dataclass_with_cached_hash(frozen=True) +class DefaultsSource(ChainedSource): + idx_key: Union[int, str] + is_kw: bool = False + field: str = dataclasses.field(init=False, repr=False, compare=False) + _name: str = dataclasses.field(init=False, repr=False, compare=False) + + def __post_init__(self) -> None: + assert self.base, ( + "Base must be a valid source in order to properly track and guard this Defaults to its origin." + ) + if self.is_kw: + assert isinstance(self.idx_key, str) + object.__setattr__(self, "field", "__kwdefaults__") + object.__setattr__( + self, + "_name", + f"{{0}}.{_esc_str(self.field)}['{_esc_str(self.idx_key)}']", + ) + else: + assert isinstance(self.idx_key, int) + object.__setattr__(self, "field", "__defaults__") + object.__setattr__( + self, "_name", f"{{0}}.{_esc_str(self.field)}[{_esc_str(self.idx_key)}]" + ) + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + codegen.extend_output(codegen.create_load_attrs(self.field)) + codegen.append_output(codegen.create_load_const(self.idx_key)) + codegen.append_output(create_binary_subscr()) + + @functools.cached_property + def _name_template(self) -> str: + return self._name + + +@dataclass_with_cached_hash(frozen=True) +class GetItemSource(ChainedSource): + index: Any + index_is_slice: bool = False + + def __post_init__(self) -> None: + assert self.base is not None + if isinstance(self.index, slice): + # store the hashable version of the slice so the whole GetItemSource is hashable + super().__setattr__("index", self.index.__reduce__()) + super().__setattr__("index_is_slice", True) + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + if self.index_is_slice: + codegen.append_output(codegen.create_load_const(self.unpack_slice())) + else: + codegen.append_output(codegen.create_load_const(self.index)) + codegen.append_output(create_binary_subscr()) + + def unpack_slice(self) -> slice: + assert self.index_is_slice + slice_class, slice_args = self.index + return slice_class(*slice_args) + + @functools.cached_property + def _name_template(self) -> str: + # Index can be of following types + # 1) index is a slice - example 1:4 + # 2) index is a constant - example string, integer + assert not isinstance(self.index, Source) + if self.index_is_slice: + return f"{{0}}[{_esc_str(self.unpack_slice(), apply_repr=True)}]" + else: + return f"{{0}}[{_esc_str(self.index, apply_repr=True)}]" + + +@dataclass_with_cached_hash(frozen=True) +class ConstDictKeySource(ChainedSource): + index: Any + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.add_push_null( + lambda: codegen.load_import_from(utils.__name__, "dict_keys_getitem") + ) + codegen(self.base) + codegen.append_output(codegen.create_load_const(self.index)) + codegen.extend_output(create_call_function(2, False)) + + @functools.cached_property + def _name_template(self) -> str: + # The list creation will be CSE'd by PyExprCSEPass + return f"list(dict.keys({{0}}))[{_esc_str(self.index, apply_repr=True)}]" + + def is_dict_key(self) -> bool: + return True + + +@dataclass_with_cached_hash(frozen=True) +class NonSerializableSetGetItemSource(ChainedSource): + index: int + + def __post_init__(self) -> None: + from .variables import ConstantVariable + + assert ConstantVariable.is_literal(self.index) + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.add_push_null( + lambda: codegen.load_import_from(utils.__name__, "set_getitem") + ) + codegen(self.base) + codegen.append_output(codegen.create_load_const(self.index)) + codegen.extend_output(create_call_function(2, False)) + + @functools.cached_property + def _name_template(self) -> str: + # set ordering might not be stable + return f"list({{0}})[{_esc_str(self.index, apply_repr=True)}]" + + def is_dict_key(self) -> bool: + return False + + +# Used to access an item from the dictionary +@dataclass_with_cached_hash(frozen=True) +class DictGetItemSource(ChainedSource): + # Key to access in the dictionary. It can be one of the following types + # 1) ConstDictKeySource + # 2) constant - like string, integer + index: Any + + def __post_init__(self) -> None: + from .variables import ConstantVariable + + assert isinstance( + self.index, ConstDictKeySource + ) or ConstantVariable.is_literal(self.index) + + def reconstruct(self, codegen: "PyCodegen") -> None: + # Load dict + codegen(self.base) + + # Load key + if isinstance(self.index, Source): + codegen(self.index) + else: + codegen.append_output(codegen.create_load_const(self.index)) + codegen.append_output(create_binary_subscr()) + + @functools.cached_property + def _name_template(self) -> str: + if isinstance(self.index, ConstDictKeySource): + return f"{{0}}[{_esc_str(self.index.name)}]" + else: + return f"{{0}}[{_esc_str(self.index, apply_repr=True)}]" + + +# Same as DictGetItemSource but used for dict.__getitem__ calls to ensure that +# torch.compile does not run the overridden __getitem__ method +@dataclass_with_cached_hash(frozen=True) +class DictSubclassGetItemSource(ChainedSource): + # Key to access in the dictionary. It can be one of the following types + # 1) ConstDictKeySource + # 2) constant - like string, integer + index: Any + + def __post_init__(self) -> None: + from .variables import ConstantVariable + + assert isinstance( + self.index, ConstDictKeySource + ) or ConstantVariable.is_literal(self.index) + + def reconstruct(self, codegen: "PyCodegen") -> None: + # reconstruct dict.__getitem__(dct, key) + + # Load dict.__getitem__ + codegen.add_push_null( + lambda: codegen.load_import_from(utils.__name__, "dict_getitem") + ) + + # Load dict + codegen(self.base) + + # Load key + if isinstance(self.index, Source): + codegen(self.index) + else: + codegen.append_output(codegen.create_load_const(self.index)) + + codegen.extend_output(create_call_function(2, False)) + + @functools.cached_property + def _name_template(self) -> str: + if isinstance(self.index, ConstDictKeySource): + return f"dict.__getitem__({{0}}, {_esc_str(self.index.name)})" + else: + return f"{{0}}[{_esc_str(self.index, apply_repr=True)}]" + + +@dataclass_with_cached_hash(frozen=True) +class ListGetItemSource(GetItemSource): + """ + Same as GetItemSource with reconstruct and name overridden to be list specific. + """ + + def reconstruct(self, codegen: "PyCodegen") -> None: + # Reconstruct list.__getitem__(lst, index) to avoid any side effects + # from possibly overridden __getitem__. + + # Load list.__getitem__ + codegen.add_push_null( + lambda: codegen.load_import_from(utils.__name__, "list_getitem") + ) + + # Load the list + codegen(self.base) + + # Load the index + if self.index_is_slice: + raise RuntimeError( + "List[slice] is a temporary object and should not have a source" + ) + else: + codegen.append_output(codegen.create_load_const(self.index)) + + codegen.extend_output(create_call_function(2, False)) + + @functools.cached_property + def _name_template(self) -> str: + # Index can be of following types + # 1) index is a slice - example 1:4 + # 2) index is a constant - example string, integer + assert not isinstance(self.index, Source) + if self.index_is_slice: + raise RuntimeError( + "List[slice] is a temporary object and should not have a source" + ) + else: + return f"list.__getitem__({{0}}, {_esc_str(self.index, apply_repr=True)})" + + +@dataclass_with_cached_hash(frozen=True) +class TupleIteratorGetItemSource(GetItemSource): + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.add_push_null( + lambda: codegen.load_import_from(utils.__name__, "tuple_iterator_getitem") + ) + codegen(self.base) + codegen.append_output(codegen.create_load_const(self.index)) + codegen.extend_output(create_call_function(2, False)) + + @functools.cached_property + def _name_template(self) -> str: + return ( + f"___tuple_iterator_getitem({{0}}, {_esc_str(self.index, apply_repr=True)})" + ) + + +@dataclass_with_cached_hash(frozen=True) +class NamedTupleFieldsSource(ChainedSource): + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + codegen.extend_output(codegen.create_load_attrs("_fields")) + + @property + def _name_template(self) -> str: + return "___namedtuple_fields({0})" + + +@dataclass_with_cached_hash(frozen=True) +class DataclassFieldsSource(ChainedSource): + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.add_push_null( + lambda: codegen.load_import_from(utils.__name__, "dataclass_fields") + ) + codegen(self.base) + codegen.extend_output(create_call_function(1, False)) + + @property + def _name_template(self) -> str: + return "___dataclass_fields({0})" + + +@dataclass_with_cached_hash(frozen=True) +class TypeSource(ChainedSource): + def __post_init__(self) -> None: + assert self.base is not None + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.add_push_null(lambda: codegen.load_import_from("builtins", "type")) + codegen(self.base) + codegen.extend_output(create_call_function(1, False)) + + @property + def _name_template(self) -> str: + return "type({0})" + + +@dataclass_with_cached_hash(frozen=True) +class OptimizerSource(ChainedSource): + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + + @property + def _name_template(self) -> str: + return "{0}" + + +@dataclass_with_cached_hash(frozen=True) +class NNModuleSource(ChainedSource): + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen(self.base) + + @functools.cached_property + def guard_source(self) -> GuardSource: + return _GUARD_SOURCE_SPECIALIZED_NN_MODULE[self.base.guard_source] + + @property + def _name_template(self) -> str: + return "{0}" + + +@dataclass_with_cached_hash(frozen=True) +class UnspecializedNNModuleSource(NNModuleSource): + @functools.cached_property + def guard_source(self) -> GuardSource: + return _GUARD_SOURCE_UNSPECIALIZED_NN_MODULE[self.base.guard_source] + + +@dataclass_with_cached_hash(frozen=True) +class UnspecializedBuiltinNNModuleSource(UnspecializedNNModuleSource): + @functools.cached_property + def guard_source(self) -> GuardSource: + return _GUARD_SOURCE_UNSPECIALIZED_BUILTIN_NN_MODULE[self.base.guard_source] + + +@dataclass_with_cached_hash(frozen=True) +class FSDPNNModuleSource(NNModuleSource): + @functools.cached_property + def guard_source(self) -> GuardSource: + return _GUARD_SOURCE_FSDP_MODULE[self.base.guard_source] + + +@dataclass_with_cached_hash(frozen=True) +class GlobalStateSource(Source): + @property + def _name_template(self) -> str: + return "" + + @property + def guard_source(self) -> GuardSource: + return GuardSource.GLOBAL + + +@dataclass_with_cached_hash(frozen=True) +class TorchSource(Source): + """Points to the actual `torch` module - used instead of GlobalSource + in case the user has overridden `torch` in their local namespace""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + from .guards import GuardBuilder, install_guard + + install_guard(self.make_guard(GuardBuilder.ID_MATCH)) + + @property + def _name_template(self) -> str: + return "__import__('torch')" + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.extend_output( + [ + codegen.create_load_const(0), # level + create_build_tuple(0), # fromlist + codegen.create_import_name("torch"), + ] + ) + + @property + def guard_source(self) -> GuardSource: + return GuardSource.GLOBAL + + +@dataclass_with_cached_hash(frozen=True) +class CollectionsSource(Source): + """Points to the actual `collections` module - used instead of GlobalSource + in case the user has overridden `collections` in their local namespace""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + from .guards import GuardBuilder, install_guard + + install_guard(self.make_guard(GuardBuilder.ID_MATCH)) + + @property + def _name_template(self) -> str: + return "__import__('collections')" + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.extend_output( + [ + codegen.create_load_const(0), # level + create_build_tuple(0), # fromlist + codegen.create_import_name("collections"), + ] + ) + + @property + def guard_source(self) -> GuardSource: + return GuardSource.GLOBAL + + +@dataclass_with_cached_hash(frozen=True) +class TorchFunctionModeStackSource(Source): + ind: int + + @functools.cached_property + def _name_template(self) -> str: + return f"___get_torch_function_mode_stack_at({_esc_str(self._get_index())})" + + def _get_index(self) -> int: + from .variables.torch_function import TorchFunctionModeStackVariable + + return TorchFunctionModeStackVariable.get_mode_index(self.ind) + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.add_push_null( + lambda: codegen.load_import_from( + utils.__name__, "get_torch_function_mode_stack_at" + ) + ) + codegen.extend_output([codegen.create_load_const(self._get_index())]) + codegen.extend_output(create_call_function(1, False)) + + @property + def guard_source(self) -> GuardSource: + return GuardSource.GLOBAL + + +@dataclass_with_cached_hash(frozen=True) +class ConstantSource(Source): + source_name: str + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.append_output(codegen.create_load_global(self.source_name, add=False)) + + @property + def guard_source(self) -> GuardSource: + return GuardSource.CONSTANT + + @functools.cached_property + def _name_template(self) -> str: + return self.source_name + + def make_guard(self, fn: Any) -> Any: + raise NotImplementedError + + +@dataclass_with_cached_hash(frozen=True) +class NumpyTensorSource(ChainedSource): + @property + def _name_template(self) -> str: + return "___from_numpy({0})" + + def reconstruct(self, codegen: "PyCodegen") -> None: + codegen.add_push_null(lambda: codegen.load_import_from("torch", "as_tensor")) + codegen(self.base) + codegen.extend_output(create_call_function(1, False)) + + +@dataclass_with_cached_hash(frozen=True) +class SubclassAttrListSource(ChainedSource): + @property + def _name_template(self) -> str: + return "{0}.__tensor_flatten__()[0]" + + +# NB: We don't expect you to actually ever generate guards against this +# source, it is ephemeral +@dataclass_with_cached_hash(frozen=True) +class FloatTensorSource(ChainedSource): + @property + def _name_template(self) -> str: + return "___as_tensor({0})" + + +@dataclass_with_cached_hash(frozen=True) +class CallMethodItemSource(ChainedSource): + @property + def _name_template(self) -> str: + return "{0}.item()" + + +# This is a synthetic source that is associated with the singleton +# shape env guard we always register for all frames. We get the actual +# guard contents from the ambient ShapeEnv +@dataclass_with_cached_hash(frozen=True) +class ShapeEnvSource(Source): + @property + def _name_template(self) -> str: + return "" + + @property + def guard_source(self) -> GuardSource: + return GuardSource.SHAPE_ENV + + +@dataclass_with_cached_hash(frozen=True) +class CurrentStreamSource(Source): + device: device_type + + @functools.cached_property + def _name_template(self) -> str: + return f"___get_current_stream(torch.device('{_esc_str(self.device.type)}', {_esc_str(self.device.index)}))" + + def reconstruct(self, codegen: "PyCodegen") -> None: + num_args = 1 + codegen.add_push_null( + lambda: codegen.load_import_from(utils.__name__, "get_current_stream") + ) + codegen.add_push_null(lambda: codegen.load_import_from("torch", "device")) + codegen.extend_output([codegen.create_load_const(self.device.type)]) + if self.device.index is not None: + num_args += 1 + codegen.extend_output([codegen.create_load_const(self.device.index)]) + codegen.extend_output(create_call_function(num_args, False)) + codegen.extend_output(create_call_function(1, False)) + + @property + def guard_source(self) -> GuardSource: + return GuardSource.GLOBAL + + +@dataclass_with_cached_hash(frozen=True) +class BackwardStateSource(Source): + @property + def _name_template(self) -> str: + return "" + + @property + def guard_source(self) -> GuardSource: + return GuardSource.BACKWARD_STATE + + +def get_local_source_name( + source: Source, *, only_allow_input: bool = False +) -> Optional[str]: + if isinstance(source, ChainedSource): + return get_local_source_name(source.base, only_allow_input=only_allow_input) + if not isinstance(source, LocalSource): + return None + if only_allow_input and not source.is_input: + return None + return source.local_name + + +def is_from_local_source(source: Source, *, only_allow_input: bool = False) -> bool: + return get_local_source_name(source, only_allow_input=only_allow_input) is not None + + +def is_from_global_source(source: Source) -> bool: + return get_global_source_name(source) is not None + + +def get_global_source_name(source: Source) -> Optional[str]: + if isinstance(source, ChainedSource): + return get_global_source_name(source.base) + if not isinstance(source, GlobalSource): + return None + return source.global_name + + +def is_from_nonlocal_source(source: Source) -> bool: + if isinstance(source, ChainedSource): + return is_from_nonlocal_source(source.base) + return ( + isinstance(source, LocalSource) + and source.is_derefed_cell_contents + and not source.is_input + ) + + +def is_from_closure_source(source: Source) -> bool: + if isinstance(source, ClosureSource): + return True + if isinstance(source, ChainedSource): + return is_from_closure_source(source.base) + return False + + +def is_from_source(source: Source, target: Source) -> bool: + if isinstance(source, ChainedSource): + return is_from_source(source.base, target) + return source == target + + +@functools.lru_cache +def is_from_unspecialized_nn_module_source(source: Source) -> bool: + if isinstance(source, UnspecializedNNModuleSource): + return True + if isinstance(source, ChainedSource): + return is_from_unspecialized_nn_module_source(source.base) + return False + + +@functools.lru_cache +def is_from_unspecialized_builtin_nn_module_source(source: Source) -> bool: + if isinstance(source, UnspecializedBuiltinNNModuleSource): + return True + if isinstance(source, ChainedSource): + return is_from_unspecialized_builtin_nn_module_source(source.base) + return False + + +@functools.lru_cache +def is_from_unspecialized_param_buffer_source(source: Source) -> bool: + if isinstance(source, UnspecializedParamBufferSource): + return True + if isinstance(source, ChainedSource): + return is_from_unspecialized_param_buffer_source(source.base) + return False + + +@functools.lru_cache +def is_from_flatten_script_object_source(source: Source) -> bool: + if isinstance(source, FlattenScriptObjectSource): + return True + elif isinstance(source, ChainedSource): + return is_from_flatten_script_object_source(source.base) + return False + + +@functools.lru_cache +def is_from_optimizer_source(source: Source) -> bool: + if isinstance(source, OptimizerSource): + return True + if isinstance(source, ChainedSource): + return is_from_optimizer_source(source.base) + return False + + +# TODO: can probably write a generic "test this on everything in the chain" +# helper +@functools.lru_cache +def is_from_defaults(source: Source) -> bool: + if isinstance(source, DefaultsSource): + return True + + # Accessed with func.__kwdefaults__["foo"] + if ( + isinstance(source, DictGetItemSource) + and isinstance(source.base, AttrSource) + and source.base.member == "__kwdefaults__" + ): + return True + + # Accessed with func.__defaults__[0] + if ( + isinstance(source, GetItemSource) + and isinstance(source.base, AttrSource) + and source.base.member == "__defaults__" + ): + return True + + if isinstance(source, ChainedSource): + return is_from_defaults(source.base) + return False + + +@functools.lru_cache +def is_from_skip_guard_source(source: Source) -> bool: + if isinstance(source, SkipGuardSource): + return True + + if isinstance(source, ChainedSource): + return is_from_skip_guard_source(source.base) + + return False diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py new file mode 100644 index 0000000000000000000000000000000000000000..4dca58f63615ef6403740802b849fae490cf04af --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/symbolic_convert.py @@ -0,0 +1,5294 @@ +""" +Core module responsible for converting Python bytecode into TorchDynamo's symbolic execution format. + +This module implements the bytecode-level tracing system that allows TorchDynamo to analyze +and transform Python code. It converts Python bytecode instructions into a symbolic format +that tracks the flow of tensors and other values through the program. + +Key components: +- InstructionTranslatorBase: Base class for converting bytecode to symbolic execution +- InstructionTranslator: Main translator for function bytecode +- InliningInstructionTranslator: Handles inlining of called functions +- SpeculationLog: Manages state for speculative execution and rollback + +The symbolic conversion process handles: +- Control flow (loops, conditionals, etc.) +- Function inlining and call stack management +- Tracking of program values and side effects +- Graph breaks and resumption points +- Exception handling and stack frame management + +This is a core part of TorchDynamo's tracing system that enables ahead-of-time +optimization of PyTorch programs. +""" + +from __future__ import annotations + +import collections +import collections.abc +import contextlib +import copy +import dataclasses +import dis +import functools +import importlib +import inspect +import itertools +import linecache +import logging +import operator +import re +import sys +import threading +import traceback +import types +import weakref +from collections import deque +from traceback import StackSummary +from typing import Any, cast, NoReturn, Optional, TYPE_CHECKING, TypeAlias, Union +from typing_extensions import TypeIs + +import torch +import torch._logging +from torch._dynamo.exc import ObservedException, TensorifyScalarRestartAnalysis +from torch._guards import tracing, TracingContext +from torch._logging.structured import dump_file +from torch.fx.experimental.symbolic_shapes import guard_bool +from torch.utils._functools import cache_method + +from . import ( + config, + exc, + graph_break_hints, + logging as torchdynamo_logging, + trace_rules, + variables, +) +from .bytecode_analysis import ( + get_indexof, + JUMP_OPNAMES, + livevars_analysis, + propagate_line_nums, +) +from .bytecode_transformation import ( + cleaned_instructions, + create_binary_slice, + create_call_function, + create_call_function_ex, + create_copy, + create_dup_top, + create_instruction, + create_jump_absolute, + create_rot_n, + create_swap, + get_code_keys, + Instruction, + is_generator, + is_jump_absolute, + unique_id, +) +from .code_context import code_context +from .codegen import PyCodegen +from .exc import ( + ArgsMismatchError, + BackendCompilerFailed, + collapse_resume_frames, + format_graph_break_message, + format_loop_skip_frame_message, + format_skip_frame_message, + get_stack_above_dynamo, + ResumePrologueTracingError, + StepUnsupported, + unimplemented, + Unsupported, +) +from .funcname_cache import get_funcname +from .guards import GuardBuilder, install_guard +from .output_graph import GraphCompileReason, OutputGraph, StackLocalsMetadata +from .polyfills import impl_CONTAINS_OP_fallback +from .replay_record import DummyModule, ExecutionRecorder +from .resume_execution import ( + ContinueExecutionCache, + IS_TRACING_RESUME_PROLOGUE_VARNAME, + ReenterWith, +) +from .source import ( + AttrSource, + DictGetItemSource, + GlobalSource, + GlobalWeakRefSource, + LocalCellSource, + LocalSource, + SkipGuardSource, + Source, +) +from .trace_rules import is_builtin_constant, is_forbidden +from .utils import ( + _get_error_on_graph_break, + counters, + get_fake_value, + get_instruction_source_311, + get_metrics_context, + graph_break_dup_warning_checker, + istype, + LazyString, + proxy_args_kwargs, +) +from .variables.base import typestr, ValueMutationNew, VariableTracker +from .variables.builder import FrameStateSizeEntry, VariableBuilder, wrap_fx_proxy +from .variables.builtin import BuiltinVariable +from .variables.constant import ConstantVariable +from .variables.ctx_manager import ( + ContextWrappingVariable, + GenericContextWrappingVariable, + WithEnterFunctionVariable, + WithExitFunctionVariable, +) +from .variables.dicts import ConstDictVariable, SetVariable +from .variables.functions import ( + BaseUserFunctionVariable, + LocalGeneratorFunctionVariable, + LocalGeneratorObjectVariable, + NestedUserFunctionVariable, + SkipFunctionVariable, + UserFunctionVariable, + UserMethodVariable, +) +from .variables.iter import MAX_ITERATOR_LIMIT +from .variables.lazy import LazyVariableTracker +from .variables.lists import ( + BaseListVariable, + IteratorVariable, + ListIteratorVariable, + ListVariable, + SliceVariable, + TupleVariable, +) +from .variables.misc import ( + CellVariable, + ExceptionVariable, + GetAttrVariable, + NullVariable, + PythonModuleVariable, + UnknownVariable, +) +from .variables.nn_module import NNModuleVariable, UnspecializedNNModuleVariable +from .variables.streams import SymbolicStreamState +from .variables.tensor import supported_comparison_ops, SymNodeVariable +from .variables.torch_function import ( + SymbolicTorchFunctionState, + TorchFunctionModeVariable, +) +from .variables.user_defined import ( + RemovableHandleVariable, + UserDefinedClassVariable, + UserDefinedExceptionClassVariable, + UserDefinedExceptionObjectVariable, + UserDefinedObjectVariable, +) + + +if TYPE_CHECKING: + from collections.abc import Callable, Generator, Sequence + + from torch._subclasses.fake_tensor import FakeTensorMode + + from .package import CompilePackage + +log = logging.getLogger(__name__) +graph_break_log = torch._logging.getArtifactLogger(__name__, "graph_breaks") +trace_call_log = torch._logging.getArtifactLogger(__name__, "trace_call") +trace_source_log = torch._logging.getArtifactLogger(__name__, "trace_source") +trace_bytecode_log = torch._logging.getArtifactLogger(__name__, "trace_bytecode") +tls = threading.local() +compare_op_handlers: dict[str, Any] = { + k: BuiltinVariable(v).call_function for k, v in supported_comparison_ops.items() +} +handle_contains = BuiltinVariable(operator.contains).call_function +handle_not = BuiltinVariable(operator.not_).call_function +compare_op_handlers["in"] = lambda tx, args, _: handle_contains( + tx, [*reversed(args)], {} +) +compare_op_handlers["not in"] = lambda tx, args, _: handle_not( + tx, [handle_contains(tx, [*reversed(args)], {})], {} +) + +PT2_ISSUE_TRACKER_URL = "https://github.com/pytorch/pytorch/issues/new?&labels=oncall%3A+pt2&projects=&template=pt2-bug-report.yml" + +ExceptionVals: TypeAlias = Union[ + variables.ExceptionVariable, + UserDefinedExceptionClassVariable, + UserDefinedExceptionObjectVariable, +] + + +@functools.cache +def _import_module(name: str) -> types.ModuleType: + """ + Import the named module and cache the result. importlib.import_module() + seems to do some filesystem checking to validate the name so not caching + this can be slow. + """ + return importlib.import_module(name) + + +@dataclasses.dataclass +class SpeculationEntry: + filename: str + lineno: int + instruction_pointer: int + inst: Instruction # for debugging only + _failed: bool = False + error_on_graph_break: Optional[bool] = None + reason: Optional[GraphCompileReason] = None + + def fail_and_restart_analysis(self, error_on_graph_break: bool) -> None: + """ + Start tracing of the current frame over again, and don't take this branch. + """ + self._failed = True + self.error_on_graph_break = error_on_graph_break + if self.reason is not None: + restart_reason = self.reason.reason + else: + restart_reason = "Unknown fail_and_restart_analysis" + raise exc.SpeculationRestartAnalysis(restart_reason=restart_reason) + + def failed(self, tx: InstructionTranslatorBase) -> bool: + if self._failed: + assert self.error_on_graph_break is not None + tx.error_on_graph_break = self.error_on_graph_break + return True + return False + + +@dataclasses.dataclass +class SpeculationLog: + """ + SpeculationLog replaces the prior copy_graphstate/restore_graphstate + checkpointing. Rather than saving/restoring state, we restart the + dynamo conversion process over from the beginning -- but when we + hit the start of the speculation that failed, we instead generate + a graph break. + """ + + entries: list[SpeculationEntry] = dataclasses.field(default_factory=list) + index: int = 0 + + def restart(self) -> None: + self.index = 0 + + def clear(self) -> None: + self.entries.clear() + self.index = 0 + + def next( + self, filename: str, lineno: int, instruction_pointer: int, inst: Instruction + ) -> SpeculationEntry: + """ + Lookup or create a SpeculationEntry() that is shared across + RestartAnalysis calls. Args are used only for debug checks. + """ + if len(self.entries) == self.index: + self.entries.append( + SpeculationEntry(filename, lineno, instruction_pointer, inst) + ) + entry = self.entries[self.index] + prev_entry_msg = "" + if self.index != 0: + prev_entry = self.entries[self.index - 1] + prev_entry_msg = ( + f"Previous instruction: {prev_entry.filename}:{prev_entry.lineno}" + f"({prev_entry.inst.opname} @ {prev_entry.instruction_pointer})\n" + ) + if not ( + entry.instruction_pointer == instruction_pointer + and entry.filename == filename + and entry.lineno == lineno + ): + raise SpeculationLogDivergence( + f""" +SpeculationLog diverged at index {self.index} (log had {len(self.entries)} entries): +- Expected: {entry.filename}:{entry.lineno} ({entry.inst.opname} at ip={entry.instruction_pointer}) +- Actual: {filename}:{lineno} ({inst.opname} at ip={instruction_pointer}) +{prev_entry_msg} +There are two usual reasons why this may have occurred: +- When Dynamo analysis restarted, the second run took a different path than + the first. If this occurred, the previous instruction is the critical instruction that + behaved differently. +- Speculation entries are only added under certain conditions (as seen in + step()), e.g., there must exist operators in the graph; those conditions may + have changed on restart. + +If this divergence was intentional, clear the speculation log before restarting (do NOT +do this for graph breaks, you will infinite loop). + +Otherwise, please submit a bug report, ideally including the contents of TORCH_LOGS=+dynamo +""" + ) + self.index += 1 + return entry + + +@dataclasses.dataclass +class LocalState: + automatic_dynamic: dict[str, FrameStateSizeEntry] = dataclasses.field( + default_factory=dict + ) + + def render(self) -> str: + return "\n".join( + f"{k}: {v.render()}" for k, v in self.automatic_dynamic.items() + ) + + +# Mutable box that is shared across restarts +@dataclasses.dataclass +class DistributedState: + compile_pg: Any + local_state: LocalState + all_states: Optional[list[LocalState]] = None + + +class TensorifyState: + # These are the set of string symfloats names (eg. "zf0") that we collect + # from the tensorify_python_scalars.py joint fx pass to inform us about + # which float inputs we should specialize when we restart analysis. + force_specializations: set[str] = set() + + @classmethod + def specialize(cls, index: str) -> None: + cls.force_specializations.add(index) + + @classmethod + def should_specialize(cls, index: str) -> bool: + return index in cls.force_specializations + + @classmethod + def clear(cls) -> None: + cls.force_specializations.clear() + + @classmethod + def empty(cls) -> bool: + return len(cls.force_specializations) == 0 + + +@functools.cache +def _step_logger() -> Callable[..., None]: + return torchdynamo_logging.get_step_logger(log) + + +@contextlib.contextmanager +def save_and_restart_speculation_log( + tx: InstructionTranslatorBase, +) -> Generator[None, None, None]: + # When reconstructing a generator after a graph break, we advance it until + # it is fully exhausted. This process adds new entries to the speculation + # log that were not previously observed. Without temporarily clearing the + # speculation log, this could lead to a divergence error. + + entries = tx.speculation_log.entries + index = tx.speculation_log.index + try: + tx.speculation_log.entries = [] + tx.speculation_log.index = 0 + yield + finally: + tx.speculation_log.entries = entries + tx.speculation_log.index = index + + +@contextlib.contextmanager +def temporarely_allow_writes_to_output_graph( + tx: InstructionTranslatorBase, +) -> Generator[None, None, None]: + try: + tmp = tx.output.should_exit + tx.output.should_exit = False + yield + finally: + tx.output.should_exit = tmp + + +@dataclasses.dataclass +class BlockStackEntry: + # Current instruction that pushes something to block_stack + inst: Instruction + target: Instruction + stack_index: int + with_context: Optional[ + Union[ContextWrappingVariable, GenericContextWrappingVariable] + ] = None + + def can_restore(self) -> bool: + return self.with_context is not None + + def resume_fn(self) -> ReenterWith: + assert self.stack_index is not None + if ( + self.with_context + and hasattr(self.with_context, "target_values") + and self.with_context.target_values + ): + return ReenterWith( + self.stack_index - 1, tuple(self.with_context.target_values) + ) + else: + return ReenterWith(self.stack_index - 1) + + def exit( + self, tx: InstructionTranslatorBase, is_graph_break: bool + ) -> VariableTracker | None: + assert self.with_context is not None + if ( + is_graph_break and self.with_context.exit_on_graph_break() + ) or not is_graph_break: + return self.with_context.exit(tx) # type: ignore[arg-type] + return None + + +class SpeculationLogDivergence(AssertionError): + pass + + +class ReturnValueOp(Exception): + pass + + +class YieldValueOp(Exception): + """ + Signal to the symbolic tracer to stop and return control flow to the + caller + """ + + +def stack_op(fn: Callable[..., object]) -> Callable[..., Any]: + nargs = len(inspect.signature(fn).parameters) + fn_var = BuiltinVariable(fn) + + @functools.wraps(fn) + def impl(self: InstructionTranslator, inst: Instruction) -> None: + self.push(fn_var.call_function(self, self.popn(nargs), {})) + + return impl + + +def is_stdlib(mod: object) -> bool: + if not isinstance(mod, types.ModuleType): + return False + return mod.__name__.split(".")[0] in sys.stdlib_module_names + + +@functools.cache +def get_assert_bytecode_sequence(with_msg: bool) -> list[str]: + if with_msg: + + def fn(x: Any) -> None: + assert x, "msg" + else: + + def fn(x: Any) -> None: + assert x + + insts = [inst.opname for inst in dis.get_instructions(fn)] + + # expect to find POP_JUMP_[FORWARD_]IF_TRUE + begin_idx = next(i for i, inst in enumerate(insts) if inst.startswith("POP_JUMP")) + end_idx = insts.index("RAISE_VARARGS") + + return insts[begin_idx + 1 : end_idx + 1] + + +def _detect_and_normalize_assert_statement( + self: InstructionTranslatorBase, + truth_fn: Callable[[object], bool], + push: bool, +) -> bool: + # Detect if this jump instruction is assert and normalize the assert + # by pushing dummy error message when nothing is given. + # + # Python 3.9-3.13 assertion is in following format (minus small differences) + # 18 POP_JUMP_IF_TRUE 28 + # 20 LOAD_ASSERTION_ERROR + # 22 LOAD_CONST 3 ('Assert message') -> optional instruction + # 24 CALL_FUNCTION 1 -> optional instruction + # 26 RAISE_VARARGS + + if (truth_fn is not operator.truth) or push: + return False + + assert isinstance(self.instruction_pointer, int) + current_instruction_pointer = self.instruction_pointer + + for with_msg in (False, True): + assert_insts = get_assert_bytecode_sequence(with_msg) + cur_insts = self.instructions[ + current_instruction_pointer : current_instruction_pointer + + len(assert_insts) + ] + cur_insts = [inst.opname for inst in cur_insts] + if cur_insts == assert_insts: + if with_msg: + load_const_idx = assert_insts.index("LOAD_CONST") + error_msg = self.instructions[ + current_instruction_pointer + load_const_idx + ].argval + else: + error_msg = "assertion error" + self.push(ConstantVariable.create(error_msg)) + return True + + return False + + +explain = False + + +# [NOTE] graph break handling in symbolic_convert +# There are 4 possible graph break cases that InstructionTranslatorBase handles: +# 1. Regular graph breaks from CALL, BINARY_SUBSCR, etc. (implemented by break_graph_if_unsupported) +# 2. Data-dependent condition graph breaks (implemented by generic_jump) +# 4. All other unhandled graph breaks - unsupported step graph breaks (implemented in InstructionTranslatorBase.step) +# +# Graph breaks are handled in the following manner: +# 1. The Unsupported exception is caught. If we cannot compile a partial graph (should_compile_partial_graph() is False), +# then propagate the exception upward. For unsupported step graph breaks, the condition to abort partial compilation is +# more restrictive (see InstructionTranslatorBase.step). +# 2. If the Unsupported exception escapes symbolic_convert.py, then we are done. +# Otherwise, we want to attempt partial compilation. +# Log the graph break via log_graph_break. If we're handling a data-dependent graph break (type 2.), then we can immediately +# codegen the compiled graph and resume function and we're done. This is because the jump instruction we graph break on is +# limited in how it can manipulate Python state (say, in comparison, to CALL, which can modify Python state arbitrarily). +# Otherwise, we need to restart compilation. We need to restart because by processing the unsupported instruction, +# we may have modified the VariableTrackers, and we need all of our VariableTrackers to be in the state BEFORE tracing the +# unsupported instruction. +# 3. During the first compilation, we updated a speculation log, indicating points in the code that we can resume from. +# On the second compilation, we will stop tracing at the first speculation log that fails. Then we compile the partial +# graph and resume function. +# +# Logging invariants: +# 1. No logs need to be made if Unsupported escapes symbolic_convert.py. Python's default exception printing will +# print out all of the necessary information and no partial compilation will be attempted. +# 2. log_graph_break should be called as soon as Unsupported is caught and we determined we want to partial compile. +# This always happens on the first compilation, NOT the restart handling this graph +# 3. Any compile_subgraph call should be preceded immediately by a log in the form of "... triggered compile". + + +def generic_jump( + truth_fn: Callable[[object], bool], push: bool +) -> Callable[[InstructionTranslatorBase, Instruction], None]: + # graph break message fields for data dependent branching + _gb_type = "Data-dependent branching" + _explanation = ( + "Detected data-dependent branching (e.g. `if my_tensor.sum() > 0:`). " + "Dynamo does not support tracing dynamic control flow." + ) + _hints = [ + *graph_break_hints.FUNDAMENTAL, + "Use `torch.cond` to express dynamic control flow.", + ] + + def jump_graph_break( + self: InstructionTranslatorBase, + inst: Instruction, + value: VariableTracker, + extra_msg: str = "", + ) -> None: + assert self.should_compile_partial_graph() + self.log_graph_break( + self.code_options, + reason=format_graph_break_message( + gb_type=_gb_type, + context=f"attempted to jump with {value}", + explanation=_explanation, + hints=_hints, + ), + ) + # compile a partial subgraph prefix then jump into user code + if self.maybe_has_backedge(): + msg = format_loop_skip_frame_message( + self.f_code, + "".join(traceback.format_list([self.frame_summary()])), + ) + log.info(msg) + raise exc.SkipFrame(msg) + + self.push(value) + log.debug("generic_jump triggered compile") + all_stack_locals_metadata = self.output.compile_subgraph( + self, + reason=GraphCompileReason( + f"generic_jump {typestr(value)}{extra_msg}", [self.frame_summary()] + ), + stack_pops=1, + ) + self.pop() + + if_next = self.create_call_resume_at( + self.next_instruction, + all_stack_locals_metadata, + ) + if push: + self.push(value) + assert inst.target is not None + if_jump = self.create_call_resume_at( + inst.target, + all_stack_locals_metadata, + ) + + if sys.version_info >= (3, 13): + # 3.13 requires stack[-1] to be bool type + self.output.add_output_instructions([create_instruction("TO_BOOL")]) + + jump_inst = create_instruction(inst.opname, target=if_jump[0]) + jump_inst.copy_positions(inst) + self.output.add_output_instructions([jump_inst] + if_next + if_jump) + + def inner(self: InstructionTranslatorBase, inst: Instruction) -> None: + value: VariableTracker = self.pop() + if ( + config.rewrite_assert_with_torch_assert + and _detect_and_normalize_assert_statement(self, truth_fn, push) + ): + error_msg: VariableTracker = self.pop() + # Skip over things like `assert True` + if value.is_python_constant(): + if bool(value.as_python_constant()): + return self.jump(inst) + elif self.should_compile_partial_graph(): + jump_graph_break(self, inst, value) + else: + unimplemented( + gb_type="Data-dependent assertion failed (cannot compile partial graph)", + context=f"value: {value}", + explanation="Dynamo has determined when encountering a data-dependent assert failure " + "that it should not compile the partial graph.", + hints=[ + *graph_break_hints.FUNDAMENTAL, + "Use `torch._assert()` to raise a hard AssertionError when the check fails. " + "This error will propagate back the user code " + "that called the compiled function (i.e. Dynamo will not trace any exception handling).", + "Remove the assert statement.", + "Move the assert statement outside of any context managers in order to graph break with " + "partial graph compilation (if fullgraph=False).", + ], + ) + + # TODO maybe should respect DtoH sync intention of users later?? + # Manually insert torch._assert_async instead of python assert and jump over + # assert related instructions as we don't need them anymore. + + # if we see Tensor as assert statement, no need to call scalar_tensor + if value.is_tensor(): + self.output.create_proxy( + "call_function", + torch._assert_async, + *proxy_args_kwargs((value, error_msg), {}), + ) + self.jump(inst) + return + + if isinstance(value, SymNodeVariable): + # if the assertion is normal shape expression. + # just install guard and bail out. + sym_expr = value.sym_num + if not isinstance(sym_expr, torch.SymBool): + sym_expr = sym_expr != 0 + + result = torch.fx.experimental.symbolic_shapes.expect_true(sym_expr) + if not result: + unimplemented( + gb_type="Assertion failed on symbolic shapes", + context=str(sym_expr), + explanation="", + hints=[*graph_break_hints.USER_ERROR], + ) + self.jump(inst) + return + + scalar_to_tensor_proxy = self.output.create_proxy( + "call_function", torch.scalar_tensor, *proxy_args_kwargs((value,), {}) + ) + + scalar_to_tensor = wrap_fx_proxy( + self, + scalar_to_tensor_proxy, + example_value=get_fake_value(scalar_to_tensor_proxy.node, self), + ) + + self.output.create_proxy( + "call_function", + torch._assert_async, + *proxy_args_kwargs((scalar_to_tensor, error_msg), {}), + ) + self.jump(inst) + return + + if value.is_python_constant(): + # ConstDictVariable is optimized to be very lazy about insertion of + # guards, so we have to manually insert a SEQUENCE_LENGTH guard + # here. + if isinstance(value, ConstDictVariable) and value.source: + install_guard(value.source.make_guard(GuardBuilder.SEQUENCE_LENGTH)) + if truth_fn(value.as_python_constant()): + if push: + self.push(value) + self.jump(inst) + elif value.is_tensor() and self.should_compile_partial_graph(): + jump_graph_break(self, inst, value) + elif isinstance(value, NNModuleVariable): + # Equivalent of "self.nn_module is not None" + mod = self.output.get_submodule(value.module_key) + if truth_fn(mod): + if push: + self.push(value) + self.jump(inst) + elif isinstance(value, UserDefinedObjectVariable): + try: + x = value.var_getattr(self, "__bool__") # type: ignore[arg-type] + except exc.ObservedAttributeError: + exc.handle_observed_exception(self) + # if __bool__ is missing, trying __len__ to infer a truth value. + try: + x = value.var_getattr(self, "__len__") # type: ignore[arg-type] + except exc.ObservedAttributeError: + exc.handle_observed_exception(self) + x = None + + # __bool__ or __len__ is function + if isinstance(x, UserMethodVariable): + result = x.call_function(self, [], {}) # type: ignore[arg-type, assignment] + method_name = getattr(getattr(x, "fn", None), "__name__", None) + if result.is_python_constant(): + result_value = result.as_python_constant() + if method_name == "__bool__" and not isinstance(result_value, bool): + msg = variables.ConstantVariable.create( + f"__bool__ should return bool, returned {type(result_value).__name__}" + ) + exc.raise_observed_exception(TypeError, self, args=[msg]) + if isinstance(result_value, (bool, int)) and truth_fn(result_value): + if push: + self.push(value) + self.jump(inst) + elif isinstance(result, SymNodeVariable): + if result.evaluate_expr(): + if push: + self.push(value) + self.jump(inst) + else: + unimplemented( + gb_type="Data-dependent branching with non-constant __bool__", + context=f"method: {x}, result: {result}", + explanation="Attempted to perform data-dependent branching on a user-defined " + "object with a __bool__ method that did not return a constant.", + hints=[], + ) + # __bool__ or __len__ is non-function or not existed in the user defined object + else: + if truth_fn(True): + if push: + self.push(value) + self.jump(inst) + elif not value.is_tensor() and value.has_unpack_var_sequence(self): + if truth_fn(len(value.unpack_var_sequence(self))): + if push: + self.push(value) + self.jump(inst) + elif isinstance(value, SymNodeVariable): + try: + # if the user is branching on a SymBool, guard on it + # if the user has code like: + # if size: + # ... + # then they are just testing truthiness: guard that the expr != 0 + if isinstance(value.sym_num, torch.SymBool): + eval_result = value.evaluate_expr(self.output) + else: + eval_result = guard_bool(value.sym_num != 0) + except exc.UserError as e: + if self.should_compile_partial_graph(): + return jump_graph_break(self, inst, value, extra_msg=f"\n{e}") + raise + if truth_fn(eval_result): + if push: + self.push(value) + self.jump(inst) + elif isinstance(value, variables.BackwardHookVariable): + if truth_fn(True): + if push: + self.push(value) + self.jump(inst) + else: + from .source import is_constant_source + + if value.source is not None and is_constant_source(value.source): + if truth_fn(value.get_real_value()): # type: ignore[attr-defined] + if push: + self.push(value) + self.jump(inst) + else: + unimplemented( + gb_type="Data-dependent branching", + context=f"attempted to jump with {value}", + explanation=_explanation, + hints=[ + *graph_break_hints.FUNDAMENTAL, + "Use `torch.cond` to express dynamic control flow.", + ], + ) + + return inner + + +# NOTE: for the purposes of nested graph breaks, break_graph_if_unsupported only works on instructions +# with 0 or 1 outputs. If you wish to support bytecodes with 2+ outputs, either rewrite the instruction +# into a sequence of simpler instructions, or file an issue for consultation. +# There is an additional requirement that if the instruction causes a function call, e.g. STORE_ATTR, +# nothing should happen to the result of the function call. +def break_graph_if_unsupported( + *, push: bool, msg_prefix: str +) -> Callable[ + [Callable[..., None]], Callable[[InstructionTranslatorBase, Instruction], None] +]: + def decorator( + inner_fn: Callable[..., None], + ) -> Callable[[InstructionTranslatorBase, Instruction], None]: + @functools.wraps(inner_fn) + def wrapper(self: InstructionTranslatorBase, inst: Instruction) -> None: + prev_push = self.current_instruction_push + self.current_instruction_push = push + speculation = self.speculate() + if speculation.failed(self): + # no need to restore current_instruction_push if speculation failed + assert speculation.reason is not None + return handle_graph_break(self, inst, speculation.reason) + try: + return inner_fn(self, inst) + except Unsupported as excp: + if self.active_generic_context_managers: + # raise original graph break if fullgraph/error_on_graph_break=True + if self.one_graph or self.error_on_graph_break: + raise + + # We don't support graph break under GenericContextWrappingVariable, + # If there is, we roll back to the checkpoint and fall back. + excp.remove_from_stats() + unimplemented( + gb_type="Graph break under GenericContextWrappingVariable", + context=f"Active generic context managers: {self.active_generic_context_managers}", + explanation="Attempted to graph break in an active context manager(s) that doesn't support graph breaking.", + hints=[ + "Move the offending context manager(s) to outside the compiled region.", + *graph_break_hints.CAUSED_BY_EARLIER_GRAPH_BREAK, + ], + from_exc=excp, + ) + + if isinstance(excp, exc.UncapturedHigherOrderOpError): + raise + + if not self.should_compile_partial_graph(): + raise + + self.log_graph_break( + self.code_options, + reason=f"{msg_prefix}:\n\n{str(excp)}", + user_stack=excp.real_stack, + ) + + if self.maybe_has_backedge(): + msg = format_loop_skip_frame_message( + self.f_code, + "".join(traceback.format_list([self.frame_summary()])), + ) + log.info(msg) + raise exc.SkipFrame(msg) from excp + + excp.remove_from_stats() + excp.add_to_stats("graph_break") + speculation.reason = GraphCompileReason(excp.msg, excp.real_stack) + finally: + self.current_instruction_push = prev_push + speculation.fail_and_restart_analysis(self.error_on_graph_break) + + def handle_graph_break( + self: InstructionTranslatorBase, + inst: Instruction, + reason: GraphCompileReason, + ) -> None: + if ( + sys.version_info >= (3, 11) + and sys.version_info < (3, 12) + and inst.opname == "CALL" + ): + # stack effect for PRECALL + CALL is split between the two instructions + stack_effect = dis.stack_effect( + dis.opmap["PRECALL"], inst.arg + ) + dis.stack_effect(dis.opmap["CALL"], inst.arg) + else: + stack_effect = dis.stack_effect(inst.opcode, inst.arg) + + log.debug("%s triggered compile", inst.opname) + all_stack_locals_metadata = self.output.compile_subgraph( + self, reason=reason, stack_pops=int(push) - stack_effect + ) + cg = PyCodegen(self.output.root_tx) + cleanup: list[Instruction] = [] + # Reconstruct the context variable CLASS in the block stack + for b in self.block_stack: + # Don't exit any modes we have entered, + # output bytecode will mutate the tf mode stack accordingly + if isinstance(b.with_context, TorchFunctionModeVariable): + cg.extend_output( + b.resume_fn().try_except_torch_function_mode( + cg.code_options, cleanup + ) + ) + continue + assert b.with_context is not None + assert isinstance(b.with_context, (ContextWrappingVariable)) + b.with_context.reconstruct_type(cg) + cg.extend_output(b.resume_fn().try_finally(cg.code_options, cleanup)) + self.output.add_output_instructions(cg.get_instructions()) + del cg + + if sys.version_info >= (3, 11) and inst.opname == "CALL": + kw_names = ( + self.kw_names.as_python_constant() + if self.kw_names is not None + else () + ) + if len(kw_names) > 0: + # KW_NAMES no longer used in 3.13 + assert sys.version_info < (3, 13) + self.output.add_output_instructions( + [create_instruction("KW_NAMES", argval=kw_names)] + ) + assert inst.arg is not None + call_insts = create_call_function(inst.arg, False) + call_insts[-1].copy_positions(inst) + self.output.add_output_instructions(call_insts) + else: + # copy instruction, but without exception table data + assert inst.target is None + inst_copy = copy.copy(inst) + inst_copy.exn_tab_entry = None + self.output.add_output_instructions([inst_copy]) + + self.output.add_output_instructions(cleanup) + + self.popn(int(push) - stack_effect) + if push: + self.push(UnknownVariable()) + self.output.add_output_instructions( + self.create_call_resume_at( + self.next_instruction, + all_stack_locals_metadata, + ) + ) + + return wrapper + + return decorator + + +class BytecodeDispatchTableMeta(type): + """Installs a `cls.dispatch_table` on every subclass to speed up calls to self.OPCODE()""" + + def __init__(cls: type, name: str, bases: Any, dct: Any) -> None: + super().__init__(name, bases, dct) # type: ignore[misc] + + def _missing(opname: str, *args: Any) -> None: + unimplemented( + gb_type="Missing bytecode handler", + context=f"{opname} with args {args}", + explanation=f"Dynamo does not know how to handle the bytecode instruction `{opname}`.", + hints=[ + f"Do not trace code that produces the `{opname}` bytecode instruction " + "(see https://docs.python.org/3/library/dis.html for bytecode semantics).", + *graph_break_hints.SUPPORTABLE, + ], + ) + + dispatch_table = { + op: getattr(cls, opname, functools.partial(_missing, opname)) + for opname, op in dis.opmap.items() + } + # pyrefly: ignore [missing-attribute] + cls.dispatch_table = [dispatch_table.get(i) for i in range(2**8)] + + +@dataclasses.dataclass +class ExceptionStack: + """ + Exception stack that it is shared among all InstructionTranslator instances + """ + + # Exception handling in CPython is a bit confusing and some of the bytecode + # have a slightly different behavior than what is documented. While reading + # the documentation, is important to notice that the terms "current exception" + # and "stack" sometimes refers to a C variable with the same name and the + # exception stack, respectively. + # + # The lifetime of an exception is (Python 3.11+): + # + tx._raise_exception_variable(...) := sets the current_exception variable + # + PUSH_EXC_INFO := pushes the current_exception to the *exception stack* + # + POP_EXCEPT := pops TOS from the *exception stack* + + _exc_stack: list[ExceptionVals] = dataclasses.field(default_factory=list) + _current_exception: Optional[ExceptionVals] = dataclasses.field(default=None) + + def clear_current_exception(self) -> None: + self._current_exception = None + + def set_current_exception(self, val: ExceptionVals) -> None: + self._set_context_and_break_context_reference_cycle(val) + self._current_exception = val + + def move_current_exception_to_stack(self) -> None: + assert self._current_exception is not None + self.append(self._current_exception) + self.clear_current_exception() + + def get_current_exception(self) -> ExceptionVals: + assert self._current_exception is not None + return self._current_exception + + def _set_context_recursive( + self, val: ExceptionVals, prev_idx: int + ) -> ExceptionVals: + if (ctx := val.__context__) and type(ctx) is not ConstantVariable: # type: ignore[union-attr] + return val + if len(self._exc_stack) + prev_idx > 0: + prev = self._exc_stack[prev_idx] + self._set_context_recursive(prev, prev_idx - 1) + val.set_context(prev) # type: ignore[union-attr, arg-type] + return val + + def _break_context_reference_cycle(self, val: ExceptionVals) -> None: + # See test_exceptions::test_raise_does_not_create_context_chain_cycle + # Based on https://github.com/python/cpython/blob/e635bf2e49797ecb976ce45a67fce2201a25ca68/Python/errors.c#L207-L228 + # As noted on CPython, this is O(chain length) but the context chains + # are usually very small + o = slow_o = val + slow_update_toggle = False # floyd's algorithm for detecting cycle + while True: + context = o.__context__ # type: ignore[union-attr] + if type(context) is ConstantVariable: # context not set + break + + if context is val: + o.set_context(ConstantVariable(None)) # type: ignore[union-attr, arg-type] + break + + o = context # type: ignore[assignment] + if o is slow_o: + # pre-existing cycle - all exceptions on the path were + # visited and checked + break + + if slow_update_toggle: + # visited all exceptions + slow_o = slow_o.__context__ # type: ignore[union-attr, assignment] + slow_update_toggle = not slow_update_toggle + + def _set_context_and_break_context_reference_cycle( + self, val: ExceptionVals + ) -> None: + # set Exception.__context__ + self._set_context_recursive(val, len(self._exc_stack) - 1) + self._break_context_reference_cycle(val) + + def pop(self) -> ExceptionVals: + return self._exc_stack.pop() + + def append(self, val: ExceptionVals) -> None: + self._exc_stack.append(val) + + def __len__(self) -> int: + return len(self._exc_stack) + + def __getitem__(self, index: int) -> ExceptionVals: + return self._exc_stack[index] + + def __str__(self) -> str: + return f"{self._exc_stack=} - {self._current_exception=}" + + __repr__ = __str__ + + +class InstructionTranslatorBase( + metaclass=BytecodeDispatchTableMeta, +): + output: OutputGraph + symbolic_locals: dict[str, VariableTracker] + symbolic_globals: dict[str, VariableTracker] + symbolic_torch_function_state: SymbolicTorchFunctionState + symbolic_stream_state: SymbolicStreamState + post_prune_cell_and_freevars: Optional[dict[str, VariableTracker]] + stack: list[VariableTracker] + instruction_pointer: Optional[int] + current_instruction: Instruction + current_instruction_push: bool + block_stack: list[BlockStackEntry] + lineno: int + kw_names: Optional[ConstantVariable] + accept_prefix_inst: bool + prefix_insts: list[Instruction] + inline_depth: int + inconsistent_side_effects: bool + current_speculation: Optional[SpeculationEntry] + dispatch_table: list[Any] + exn_vt_stack: ExceptionStack + exec_recorder: Optional[ExecutionRecorder] + strict_checks_fn: Optional[Callable[[VariableTracker], bool]] + start_point: Optional[int] + is_leaf_tracer: bool + parent: Optional[InstructionTranslatorBase] + debug_locals: list[tuple[VariableTracker, list[VariableTracker]]] + package: Optional[CompilePackage] + latest_bytecode_queue: deque[str] + # Store the latest bytecode before graph_break() call by user + + def mark_inconsistent_side_effects(self) -> None: + """ + InstructionTranslator has encountered instructions which may cause + dynamo to see a different version of history from eager + See: https://github.com/pytorch/pytorch/issues/110765 + """ + self.inconsistent_side_effects = True + + def maybe_has_backedge(self) -> bool: + # This function employs a heuristic. It does not reliably detect a backedge. + # The heuristic is straightforward: starting from the current instruction and + # continuing to the end, if any jump instruction targets an instruction before + # the current one, there might be a backedge. + + # Python 3.12 introduced changes to bytecode that group common paths in + # blockstacks (with or try...else) and allow for early returns. Consequently, + # there can be multiple RETURN_VALUE instructions. Another heuristic is to + # halt detection upon encountering the first RETURN_VALUE or RETURN_CONST. + + # These heuristics can result in both false positives and negatives, but + # in either case, the Dynamo code remains valid. For false positives + # (where an edge is incorrectly marked as a backedge), Dynamo will + # perform a SkipFrame instead of potentially applying optimizations. For + # false negatives (where an edge that should be marked as a backedge + # isn't), multiple graphs may be generated if there's a break in the + # graph during a for loop. In general, its better to have fewer false + # negatives so that Dynamo does not skip the whole frame. + + # If any parent tx has a backedge, then return True + cur_tx: Optional[InstructionTranslatorBase] = self + while cur_tx is not None: + cur_offset = cur_tx.current_instruction.offset + assert cur_tx.instruction_pointer is not None + for inst in cur_tx.instructions[cur_tx.instruction_pointer :]: + if inst.opname in ("RETURN_VALUE", "RETURN_CONST"): + break + if inst.opname in JUMP_OPNAMES: + jump_offset = inst.argval + if jump_offset < cur_offset: + return True + cur_tx = cur_tx.parent + return False + + def cellvars(self) -> list[str]: + return self.code_options["co_cellvars"] + + def freevars(self) -> list[str]: + return self.code_options["co_freevars"] + + def cell_and_freevars(self) -> list[str]: + if not hasattr(self, "_cell_and_freevars"): + self._cell_and_freevars = self.cellvars() + self.freevars() + return self._cell_and_freevars + + def prune_dead_locals(self) -> None: + # keep cell and freevar references alive + self.post_prune_cell_and_freevars = { + k: v + for k, v in self.symbolic_locals.items() + if k in self.cell_and_freevars() + } + # Only keep the locals that must remain on the stack. + reads = livevars_analysis(self.instructions, self.current_instruction) + self.symbolic_locals = { + k: v for k, v in self.symbolic_locals.items() if k in reads + } + + def call_function( + self, + fn: VariableTracker, + args: list[VariableTracker], + kwargs: dict[str, VariableTracker], + ) -> None: + assert isinstance(fn, VariableTracker) + assert isinstance(args, list) + assert isinstance(kwargs, dict) + assert all( + isinstance(x, VariableTracker) + for x in itertools.chain(args, kwargs.values()) + ) + inner_fn = None + if hasattr(fn, "value"): + inner_fn = fn.value + if hasattr(fn, "fn"): + inner_fn = fn.fn + if inner_fn and callable(inner_fn) and is_forbidden(inner_fn): + raise AssertionError(f"Attempt to trace forbidden callable {inner_fn}") + self.push(fn.call_function(self, args, kwargs)) # type: ignore[arg-type] + + def inline_generator_function( + self, fn: VariableTracker, args: Sequence[Any], kwargs: dict[str, Any] + ) -> Any: + """ + Redirect the call to the generator "call_function" + """ + if not isinstance(fn, LocalGeneratorFunctionVariable): + fn = LocalGeneratorFunctionVariable(fn) # type: ignore[arg-type] + return fn.call_function(self, args, kwargs) # type: ignore[arg-type] + + def inline_user_function_return( + self, fn: VariableTracker, args: Sequence[Any], kwargs: dict[str, Any] + ) -> Any: + """ + A call to some user defined function by inlining it. + """ + self.is_leaf_tracer = False + if config.enable_faithful_generator_behavior and is_generator(fn.get_code()): # type: ignore[attr-defined] + return self.inline_generator_function(fn, args, kwargs) + else: + return InliningInstructionTranslator.inline_call(self, fn, args, kwargs) + + def get_line_of_code_header(self, lineno: Optional[int] = None) -> str: + if lineno is None: + lineno = self.lineno + inline_depth_str = ( + f" (inline depth: {self.inline_depth})" if self.inline_depth > 0 else "" + ) + funcname = get_funcname(self.f_code.co_filename, lineno) + funcname_str = "" if funcname is None else f" ({funcname})" + return f"{self.f_code.co_filename}:{lineno} in {self.f_code.co_name}{funcname_str}{inline_depth_str}" + + def get_log_starts_line_log_str(self) -> str: + log_str = f"TRACE starts_line {self.get_line_of_code_header()}\n" + line = linecache.getline(self.f_code.co_filename, self.lineno).rstrip() + log_str += f" {line}" + return log_str + + def starts_line(self, lineno: int) -> None: + if self.lineno == lineno: + return + self.lineno = lineno + TracingContext.set_current_loc( + self.f_code.co_filename, lineno, self.f_code.co_name + ) + + if self.is_trace_source_log_enabled: + trace_source_log.debug("%s", LazyString(self.get_log_starts_line_log_str)) + + def step(self) -> bool: + """Process exactly one instruction, return False we should exit""" + self.error_on_graph_break = _get_error_on_graph_break() + + ip = self.instruction_pointer + if ip is None: + return False + self.current_instruction = inst = self.instructions[ip] + self.instruction_pointer = ip + 1 + + if inst.starts_line: + self.starts_line(inst.starts_line) + + if ( + not self.stack + and self.should_compile_partial_graph() + and self.is_non_empty_graph() + ): + self.current_speculation = self.speculate() + if self.current_speculation.failed(self): + self.step_graph_break(inst) + return False + + if self.is_trace_bytecode_log_enabled: + trace_bytecode_log.debug( + "TRACE %s %s %s", inst.opname, inst.argval, repr(self.stack) + ) + + # Store the latest 20 bytecode execution for the process, + # Used repr for byte processing and limiting the length to 2048 + if config.verbose: + try: + stack_repr = repr(self.stack) + except ValueError: + # Handle large integers that exceed sys.int_info.str_digits_check_threshold + stack_repr = "" + self.latest_bytecode_queue.append( + f"TRACE {inst.opname} {repr(inst.argval)} {stack_repr}" + ) + + self.update_block_stack(inst) + + try: + self.dispatch_table[inst.opcode](self, inst) + return not self.output.should_exit + except TensorifyScalarRestartAnalysis: + raise + except exc.ObservedException as e: + self.exception_handler(e) + return True + except (ReturnValueOp, YieldValueOp): + return False + except (Unsupported, StepUnsupported) as e: + # More restrictive condition than should_compile_partial_graph: + # if this condition is true, then we SHOULD NOT attempt to find + # a previous checkpoint to resume from and try to resume - we should + # immediately error out. + # The condition is more restrictive because, it may be possible to resume significantly earlier + # in the code (the most recent speculation point). This happens, for example, in the case + # of a graph break in a try block. + if ( + self.one_graph + or self.error_on_graph_break + or self.is_tracing_resume_prologue + ): + if isinstance(e, StepUnsupported): + unimplemented( + gb_type="cannot resume from torch._dynamo.step_unsupported()", + context="", + explanation="traced torch._dynamo.step_unsupported(), but Dynamo is instructed " + "to error on graph break. This graph break is used for debugging only.", + hints=[ + "Remove the torch._dynamo.step_unsupported() call.", + "Make sure fullgraph=False and error_on_graph_break=False.", + *graph_break_hints.DYNAMO_BUG, + ], + ) + raise + if self.current_speculation is None: + log.debug("empty checkpoint - cannot resume from graph break") + if isinstance(e, StepUnsupported): + unimplemented( + gb_type="torch._dynamo.step_unsupported() with empty checkpoint", + context="", + explanation="traced torch._dynamo.step_unsupported(), but there is no checkpoint " + "to step_graph_break from. This graph break is used for debugging only.", + hints=[ + "Remove the torch._dynamo.step_unsupported() call.", + "Include at least one checkpoint: (1) include at least 2 ops and (2) make sure there is some " + "line of code that is not in a try/with block, and has an empty Python stack.", + *graph_break_hints.DYNAMO_BUG, + ], + ) + raise + reason = ( + "Encountered graph break that we cannot resume from. " + "Compiling up to the previous resumable state, " + "then skipping the rest of the function. " + f"Graph break encountered:\n{str(e)}" + ) + self.log_graph_break( + self.code_options, + reason=reason, + user_stack=e.real_stack, + ) + + self.current_speculation.fail_and_restart_analysis(self.error_on_graph_break) + return False + + if sys.version_info >= (3, 11): + + def update_block_stack(self, inst: Instruction) -> None: + # 3.11+ no longer uses a block stack, but we still keep track of one + # so that we know which contexts are currently active. + # For our purposes, all exception table entries with the same target + # are considered to be part of the same "block". + # NOTE: we only keep track of with blocks that are not contained in try blocks. + # This is because we will not create continuation functions on graph breaks in try blocks, + # but we may for with blocks. We do not push blocks here since + # with blocks are pushed when handling BEFORE_WITH. + entry = inst.exn_tab_entry + if entry: + # Detect when we have exited the top with block. + # The with blocks on the block stack are not enclosed in try + # blocks, so a with block's cleanup code should be in the + # previous with block (if any). + if ( + len(self.block_stack) >= 2 + and entry.target is not self.block_stack[-1].target + and entry.target is self.block_stack[-2].target + ): + # exit the current block + self.block_stack.pop() + else: + # no longer in any block + # It is possible for NOPs to be between two instructions + # in the same block, but the NOPs are not covered by an + # exception table entry. In this case, assume that we + # are still in the same block. + # In 3.12+, JUMP_BACKWARD might also not be covered by + # an exception table entry, so we also assume that we + # are still in the same block. It is probably safe to do + # this in 3.11, even though we haven't encountered this case before. + # In 3.14+, NOT_TAKEN might also not be covered by an exn table entry. + if self.block_stack and inst.opname not in ( + "NOP", + "JUMP_BACKWARD", + "NOT_TAKEN", + ): + # If we really escape from a block and the current + # instruction is not in another block, then there + # should be no other nested blocks that we are in. + assert len(self.block_stack) == 1 + self.block_stack.pop() + + else: + + def update_block_stack(self, inst: Instruction) -> None: + pass + + @property + def next_instruction(self) -> Instruction: + assert self.instruction_pointer is not None + return self.instructions[self.instruction_pointer] + + def step_graph_break(self, continue_inst: Instruction) -> None: + # generate code from checkpoint + assert not self.output.output_instructions + assert self.current_speculation is not None + # NOTE: adding an assert here since it seems like the only place + # where we call step_graph_break right now is when the stack is empty, + # so let's enforce that for now. + assert not self.stack + # NOTE: if we support non-empty self.stack in the future, the `stack_pops` argument + # below should be set to the stack length to ensure that the stack is codegen'd + # for the rest of the function. + log.debug("step triggered compile") + all_stack_locals_metadata = self.output.compile_subgraph( + self, + partial_convert=True, + reason=GraphCompileReason("step_unsupported", [self.frame_summary()]), + ) + # current frame state + # cells, + # [ + # frame N locals, + # frame N-1 stack + locals, + # ..., + # frame 1 stack + locals, + # ], + if self.parent: + from .eval_frame import skip_code + + # nested graph break + assert config.nested_graph_breaks + cg = PyCodegen(self.output.root_tx) + + # codegen cells and frame values only for frame N + cg.extend_output( + [ + *create_copy(2), + cg.create_load_const(0), + cg.create_binary_subscr(), + create_instruction("BUILD_LIST", arg=1), + *create_copy(2), + cg.create_load_const(0), + cg.create_binary_subscr(), + create_instruction("BUILD_LIST", arg=1), + ] + ) + # No need to fix stack, since stack is assumed to be empty here. + # Do NOT handle_inactive_ctx because we will be skipping this resume code. + leaf_resume_code, leaf_resume_name = self.create_resume( + 0, continue_inst, all_stack_locals_metadata[0], [], cg, True, False + ) + skip_code(leaf_resume_code) + + # current frame state + # cells, + # [ + # frame N locals, + # frame N-1 stack + locals, + # ..., + # frame 1 stack + locals, + # ], [frame N cells], [frame N locals], + self.codegen_call_resume([leaf_resume_code], [leaf_resume_name], cg) + + # current frame state + # cells, + # [ + # frame N locals, + # frame N-1 stack + locals, + # ..., + # frame 1 stack + locals, + # ], leaf_resume result + + # pop frame N cells and locals + cg.extend_output( + [ + *create_copy(2), + cg.create_load_const(0), + create_instruction("DELETE_SUBSCR"), + *create_copy(3), + cg.create_load_const(0), + create_instruction("DELETE_SUBSCR"), + ] + ) + + # add the leaf_resume result to frame N-1 stack + num_stack = all_stack_locals_metadata[1].num_stack + cg.extend_output( + [ + create_instruction("BUILD_LIST", arg=1), + *create_copy(2), + cg.create_load_const(0), + cg.create_binary_subscr(), + *create_binary_slice(num_stack, num_stack, True), + ] + ) + self.parent.push(UnknownVariable()) + all_stack_locals_metadata[1].num_stack += 1 + + # current frame state + # cells, frame_values + # extract frame N-1 stack to stack + cg.extend_output( + [ + create_dup_top(), + cg.create_load_const(0), + cg.create_binary_subscr(), + *create_binary_slice(0, num_stack + 1), + ] + ) + + # current frame state + # cells, frame_values, frame N-1 stack + leaf_resume result + # remove frame N-1 stack from frame_values + cg.extend_output( + # frame_values[0] = frame_values[0][num_stack + 1:] + [ + *create_copy(2), + cg.create_load_const(0), + cg.create_binary_subscr(), + create_dup_top(), + *create_binary_slice(num_stack + 1, None), + *create_swap(2), + cg.create_load_const(0), + create_instruction("STORE_SUBSCR"), + ] + ) + + # current frame state + # cells, frame_values, frame N-1 stack + leaf_resume result + # unpack the stack (need to unpack twice since UNPACK_SEQUENCE unpacks in reverse order) + cg.extend_output( + [ + create_instruction("UNPACK_SEQUENCE", arg=num_stack + 1), + create_instruction("BUILD_LIST", arg=num_stack + 1), + create_instruction("UNPACK_SEQUENCE", arg=num_stack + 1), + ] + ) + + # call the remaining resume functions + # current frame state + # [frame N-1 cells, ..., frame 1 cells], + # [ + # frame N-1 locals, + # frame N-2 stack + locals, + # ..., + # frame 1 stack + locals, + # ], *(frame N-1 stack), leaf_resume result + self.output.add_output_instructions( + cg.get_instructions() + + self.parent.create_call_resume_at( + self.parent.next_instruction, all_stack_locals_metadata[1:] + ) + ) + else: + # pop cells + self.output.add_output_instructions( + [ + *create_swap(2), + create_instruction("POP_TOP"), + ] + ) + # load locals from frame values + cg = PyCodegen(self.output.root_tx) + self.output.add_output_instructions( + [ + cg.create_load_const(-1), + cg.create_binary_subscr(), + ] + ) + for local, idx in all_stack_locals_metadata[-1].locals_names.items(): + self.output.add_output_instructions( + [ + create_dup_top(), + cg.create_load_const(idx), + cg.create_binary_subscr(), + cg.create_store(local), + ] + ) + self.output.add_output_instructions( + [ + create_instruction("POP_TOP"), + create_jump_absolute(continue_inst), + *self.instructions, + ] + ) + + def run_ctx_mgr(self) -> Any: + # NB: Don't push the top level frame summary; set_current_loc will + # take care of it. However, DO make sure we attach real_stack to + # exceptions + return TracingContext.current_frame(None) + + def run(self) -> None: + with self.run_ctx_mgr(): + dump_file(self.f_code.co_filename) + try: + self.output.push_tx(self) + self.start_point = self.instruction_pointer + try: + while self.step(): + pass + except Exception as e: + if self.is_tracing_resume_prologue: + raise ResumePrologueTracingError( + "Error while tracing through a Dynamo-generated resume function prologue. " + "Errors are not allowed when tracing resume function prologues.\n" + f"{type(e).__qualname__}: {str(e)}" + ).with_traceback(e.__traceback__) from None + raise + except TensorifyScalarRestartAnalysis: + raise + except BackendCompilerFailed: + raise + except RuntimeError as e: + if hasattr(e, "msg") and "Data-dependent" in e.msg: + readable_graph = torch.fx.GraphModule( + self.output.nn_modules, self.output.graph + ).print_readable( + print_output=False, include_stride=True, include_device=True + ) + e.partial_fx_graph = readable_graph # type: ignore[attr-defined] + raise + + raise + except Exception as e: + if self.exec_recorder: + e.exec_record = self.exec_recorder.get_record() # type: ignore[attr-defined] + + raise + finally: + self.output.pop_tx() + # Cleanup the outputGraph to delete the held tensors. We perform the + # cleanup only for InstructionTranslator and not + # InliningInstructionTranslator. The InliningInstructionTranslator + # mutates the output object and is restored to original state if + # there was an exception. + if isinstance(self, InstructionTranslator): + self.output.cleanup() + + # Note that this call maybe redundant if compile_subgraph is + # called. This is ok, because calling exit stack close() + # twice is not an issue (second stop is a no op). + self.output.mark_bytecode_tracing_stop() + + def push(self, val: Optional[VariableTracker]) -> None: + assert val is None or isinstance(val, VariableTracker), ( + f"push expects VariableTracker, got {typestr(val)}" + ) + self.stack.append(val) # type: ignore[arg-type] + + def push_many(self, vals: list[VariableTracker]) -> None: + for val in vals: + self.push(val) + + def pop(self) -> VariableTracker: + return self.stack.pop() + + def popn(self, n: int) -> list[VariableTracker]: + return [*reversed([self.pop() for _ in range(n)])] + + def LOAD_FAST(self, inst: Instruction) -> None: + name = inst.argval + if self.exec_recorder and name in self.f_locals: + self.exec_recorder.add_local_var(name, self.f_locals[name]) + + try: + self.push(self.symbolic_locals[name].unwrap()) + except KeyError: + if name.startswith("."): + try: + # This happens in dict/list comprehensions + new_name = name.replace(".", "implicit") + self.push(self.symbolic_locals[new_name]) + except KeyError: + unimplemented( + gb_type="Attempted to read undefined local variable (implicit)", + context=f"LOAD_FAST {name}", + explanation=f"Could not find an implicit local variable with name `{name}`", + hints=[ + "This happens in dict/list comprehensions", + *graph_break_hints.USER_ERROR, + ], + ) + else: + unimplemented( + gb_type="Attempted to read undefined local variable", + context=f"LOAD_FAST {name}", + explanation=f"Could not find a local variable with name `{name}`", + hints=[*graph_break_hints.USER_ERROR], + ) + + # for continuation functions + if name.startswith("__stack"): + self.symbolic_locals.pop(name) + + def LOAD_DEREF(self, inst: Instruction) -> None: + assert inst.argval in self.cell_and_freevars() + cell = self.symbolic_locals[inst.argval] + contents_var = self.output.side_effects.load_cell(cell) + self.push(contents_var) + + if self.exec_recorder and inst.argval in self.f_locals: + self.exec_recorder.add_local_var(inst.argval, self.f_locals[inst.argval]) + + def STORE_FAST(self, inst: Instruction) -> None: + name = inst.argval + loaded_vt = self.pop() + loaded_vt.set_name_hint(name) + self.symbolic_locals[name] = loaded_vt + if name == IS_TRACING_RESUME_PROLOGUE_VARNAME: + val = loaded_vt.as_python_constant() + assert type(val) is bool + self.is_tracing_resume_prologue = val + + def DELETE_FAST(self, inst: Instruction) -> None: + del self.symbolic_locals[inst.argval] + + def STORE_DEREF(self, inst: Instruction) -> None: # type: ignore[override] + assert inst.argval in self.cell_and_freevars() + cell = self.symbolic_locals[inst.argval] + val = self.pop() + self.output.side_effects.store_cell(cell, val) + + assert isinstance(cell, CellVariable) # tame mypy + if cell.local_name is not None: + val.set_name_hint(cell.local_name) # type: ignore[attr-defined] + + LOAD_CLOSURE = LOAD_FAST + + def _load_const(self, inst: Instruction) -> VariableTracker: + i = inst.arg + if i is None: + return ConstantVariable.create(value=inst.argval) # type: ignore[return-value] + val = self._constants_cache[i] + if not val: + self._constants_cache[i] = ConstantVariable.create(value=inst.argval) # type: ignore[call-overload] + val = self._constants_cache[i] + assert val is not None + return val + + def LOAD_CONST(self, inst: Instruction) -> None: + self.push(self._load_const(inst)) + + def _load_global(self, inst: Instruction) -> None: + name = inst.argval + + if self.exec_recorder: + if name in self.f_globals: + self.exec_recorder.add_global_var(name, self.f_globals[name]) + else: + assert name in self.f_builtins + self.exec_recorder.builtins[name] = self.f_builtins[name] + + if name not in self.f_globals: + return self.load_builtin(inst) + + if name in self.symbolic_globals: + variable = self.output.side_effects[self.symbolic_globals[name]] + self.push(self.output.side_effects.load_global(variable, name)) + return + + value = self.f_globals[name] + self.push(VariableTracker.build(self, value, GlobalSource(name))) + + @functools.cached_property + def nn_modules_globals_vt(self) -> VariableTracker: + module_name = "torch.nn.modules.module" + module_source = self.import_source(module_name) + fglobals_value = _import_module(module_name) + return VariableTracker.build(self, fglobals_value, module_source) + + def LOAD_GLOBAL(self, inst: Instruction) -> None: + assert inst.arg is not None + if sys.version_info >= (3, 11) and sys.version_info < (3, 13) and inst.arg % 2: + self.PUSH_NULL(inst) + self._load_global(inst) + if sys.version_info >= (3, 13) and inst.arg % 2: + self.PUSH_NULL(inst) + + def STORE_GLOBAL(self, inst: Instruction) -> None: + value = self.pop() + name = inst.argval + source = GlobalSource(name) + if name not in self.symbolic_globals: + self.symbolic_globals[name] = object() # type: ignore[assignment] # sentinel object + variable = self.output.side_effects.track_global_existing( + source, self.symbolic_globals[name] + ) + if isinstance(value, RemovableHandleVariable): + unimplemented( + gb_type="Storing Tensor hook handle in globals", + context=name, + explanation="This is not supported.", + hints=[], + ) + self.output.side_effects.store_global(variable, name, value) + + # Cache note: This cache only exists for the duration of this + # InstructionTranslator - so it should be safe to do. + @cache_method + def import_source(self, module_name: str) -> GlobalSource: + """Create an alias to a module for use in guards""" + if "torch_package" in module_name: + value = torch.package.package_importer._package_imported_modules[ + module_name + ] + alias = ( + module_name.replace(">", "_").replace("<", "_").replace(".", "_dot_") + ) + else: + value = _import_module(module_name) + alias = f"__import_{module_name.replace('.', '_dot_')}" + + if self.package is not None: + self.package.add_import_source(alias, module_name) + self.output.import_sources[alias] = module_name + f_globals = self.output.global_scope + assert alias not in f_globals or f_globals[alias] is value + f_globals[alias] = value + self.output.update_co_names(alias) + return GlobalSource(alias) + + def resolve_name(self, name: str, package: str, level: int) -> str: + """ + Copied from the Cpython implementation of __import__ + Resolve a relative module name to an absolute one. + https://github.com/python/cpython/blob/5a094f0255eea1db58fb2cf14c200971e64ec36e/Lib/importlib/_bootstrap.py#L902 + """ + bits = package.rsplit(".", level - 1) + if len(bits) < level: + raise ImportError("attempted relative import beyond top-level package") + base = bits[0] + return f"{base}.{name}" if name else base + + def calc_package(self) -> str: + """ + Copied from the Cpython implementation of __import__ + https://github.com/python/cpython/blob/5a094f0255eea1db58fb2cf14c200971e64ec36e/Lib/importlib/_bootstrap.py#L1090 + """ + package = self.f_globals.get("__package__") + spec = self.f_globals.get("__spec__") + if package is not None: + if spec is not None and package != spec.parent: + log.warning( + "__package__ != __spec__.parent (%r != %r)", + package, + spec.parent, + stacklevel=3, + ) + return package + elif spec is not None: + return spec.parent + else: + log.warning( + "can't resolve package from __spec__ or __package__, " + "falling back on __name__ and __path__", + stacklevel=3, + ) + package = self.f_globals["__name__"] + if "__path__" not in self.f_globals: + package = package.rpartition(".")[0] + return package + + def IMPORT_NAME(self, inst: Instruction) -> None: + level, fromlist = self.popn(2) + level = level.as_python_constant() + fromlist = fromlist.as_python_constant() + module_name = inst.argval + + # Are we replaying? if so, load recorded module + recorded_name = ( + f"{ExecutionRecorder.LOCAL_MOD_PREFIX}_{level}_{fromlist}_{module_name}" + ) + if recorded_name in self.f_globals: + value = self.f_globals[recorded_name] + source = GlobalSource(recorded_name) + else: + try: + value = __import__( + module_name, + fromlist=fromlist, + level=level, + globals=self.f_globals, + ) + except ImportError: + unimplemented( + gb_type="Import failure", + context=f"module_name: {module_name}, fromlist: {fromlist}, level={level}", + explanation="Failure when attempting to import.", + hints=[*graph_break_hints.USER_ERROR], + ) + + if level != 0: + pkg = self.calc_package() + module_name = self.resolve_name(module_name, pkg, level) + + # For __import__, when the name variable is of the form package.module, + # normally, the top-level package (the name up till the first dot) is + # returned, not the module named by module_name. However, when a + # non-empty fromlist argument is given, the module named by name is + # returned. Therefore, we set the source correctly here. + if not fromlist: + top_level_module_name = module_name.partition(".")[0] + source = self.import_source(top_level_module_name) + else: + source = self.import_source(module_name) + + if self.exec_recorder: + # pyrefly: ignore [unbound-name] + self.exec_recorder.add_local_mod(recorded_name, value) + + # pyrefly: ignore [unbound-name] + if istype(value, (types.ModuleType, DummyModule)): + # pyrefly: ignore [unbound-name] + self.push(PythonModuleVariable(value, source=source)) + else: + unimplemented( + gb_type="Bad import result", + # pyrefly: ignore [unbound-name] + context=typestr(value), + explanation="Import result is not a Python module.", + hints=[], + ) + + # fb internal 3.12 opcode + EAGER_IMPORT_NAME = IMPORT_NAME + + def IMPORT_FROM(self, inst: Instruction) -> None: + self.DUP_TOP(inst) + self._load_attr(inst.argval) + + # Cache note: This cache only exists for the duration of this + # InstructionTranslator - so it should be safe to do. + @cache_method + def load_builtin_from_argval(self, argval: Any) -> VariableTracker: + if argval not in self.f_builtins: + unimplemented( + gb_type="failed to find name in frame builtins", + context="", + explanation=f"Failed to find name `{argval}` in frame's builtins.", + hints=[ + *graph_break_hints.DYNAMO_BUG, + ], + ) + val = self.f_builtins[argval] + + if callable(val): + builtins_source = GlobalSource( + self.output.name_of_builtins_dict_key_in_fglobals + ) + var_source = DictGetItemSource(builtins_source, argval) + return VariableTracker.build(self, val, var_source) + else: + assert is_builtin_constant(val) + return ConstantVariable.create(value=val) + + def load_builtin(self, inst: Instruction) -> None: + self.push(self.load_builtin_from_argval(inst.argval)) + + def jump(self, inst: Instruction) -> None: + assert self.instruction_pointer is not None + assert self.start_point is not None + assert inst.target is not None + get_metrics_context().increment( + "ir_count", self.instruction_pointer - self.start_point + ) + self.instruction_pointer = self.indexof[inst.target] + self.start_point = self.instruction_pointer + + JUMP_FORWARD = jump + JUMP_ABSOLUTE = jump + + POP_JUMP_IF_FALSE = generic_jump(operator.not_, False) + POP_JUMP_IF_TRUE = generic_jump(operator.truth, False) + JUMP_IF_FALSE_OR_POP = generic_jump(operator.not_, True) + JUMP_IF_TRUE_OR_POP = generic_jump(operator.truth, True) + + def SETUP_LOOP(self, inst: Instruction) -> None: + # only exists in python<=3.7 + assert inst.target is not None + self.block_stack.append(BlockStackEntry(inst, inst.target, len(self.stack))) + + def SETUP_EXCEPT(self, inst: Instruction) -> None: + # only exists in python<=3.7 + assert inst.target is not None + self.block_stack.append(BlockStackEntry(inst, inst.target, len(self.stack))) + + def POP_BLOCK(self, inst: Instruction) -> None: + self.block_stack.pop() + + def SETUP_WITH(self, inst: Instruction) -> None: + self.setup_or_before_with(inst) + + def SETUP_FINALLY(self, inst: Instruction) -> None: + assert inst.target is not None + self.block_stack.append(BlockStackEntry(inst, inst.target, len(self.stack))) + + def BEGIN_FINALLY(self, inst: Instruction) -> None: + self.push(None) + + def WITH_CLEANUP_START(self, inst: Instruction) -> None: + exit, exc = self.popn(2) + assert exc is None + self.push(exc) + # pyrefly: ignore [bad-argument-type] + self.push(exit.call_function(self, [ConstantVariable.create(None)] * 3, {})) + + def WITH_CLEANUP_FINISH(self, inst: Instruction) -> None: + self.popn(2) + self.push(None) + + def FOR_ITER(self, inst: Instruction) -> None: + it = self.pop().realize() + self.push(it) + try: + val = it.next_variable(self) + self.push(val) + except (StopIteration, exc.ObservedUserStopIteration) as e: + if isinstance(e, exc.ObservedUserStopIteration): + exc.handle_observed_exception(self) + + if sys.version_info >= (3, 12): + # CPython 3.12 actually jumps to the instruction after the END_FOR + # and performs the action of END_FOR as part of FOR_ITER. We jump + # to the END_FOR and run it, so we need to make sure 2 values are + # on the stack for it to pop. + self.push(ConstantVariable.create(None)) + else: + # pop the iterator in Python < 3.12 + self.pop() + self.jump(inst) + + def _create_exception_type(self, val: VariableTracker) -> VariableTracker: + if isinstance( + val, (variables.BuiltinVariable, UserDefinedExceptionClassVariable) + ): + # Create the instance of the exception type + # https://github.com/python/cpython/blob/3.11/Python/ceval.c#L6547-L6549 + val = val.call_function(self, [], {}) # type: ignore[arg-type] + return val + + def _raise_exception_variable(self, val: VariableTracker) -> NoReturn: + # User can raise exception in 2 ways + # 1) raise exception type - raise NotImplementedError + # 2) raise exception instance - raise NotImplementedError("foo") + + # 1) when user raises exception type + val = self._create_exception_type(val) + + # Handle https://peps.python.org/pep-0479/ + # CPython 3.12+ has a specific bytecode instruction (CALL_INTRINSIC_1 3) for this + if ( + is_generator(self.f_code) + and isinstance(val, variables.ExceptionVariable) + and val.exc_type is StopIteration + ): + val = variables.BuiltinVariable(RuntimeError).call_function(self, [], {}) # type: ignore[arg-type] + + # Save the exception in a global data structure + self.exn_vt_stack.set_current_exception(val) # type: ignore[arg-type] + + # 2) when user raises exception instance + if self._isinstance_exception(val): + observed_exception_type = exc.get_dynamo_observed_exception(val.exc_type) # type: ignore[attr-defined, union-attr] + raise observed_exception_type(f"raised exception {val}") + unimplemented( + gb_type="Failed to raise exception", + context=str(exc), + explanation="Attempted to raise a non-Exception type/value.", + hints=[*graph_break_hints.USER_ERROR], + ) + + def RAISE_VARARGS(self, inst: Instruction) -> None: + if inst.arg == 0: + if not len(self.exn_vt_stack): + msg = ConstantVariable("No active exception to reraise") + exc.raise_observed_exception(RuntimeError, self, args=[msg]) + + # re-raise the previous exception. Here CPython refers to the exception + # on top of the exception stack + assert len(self.exn_vt_stack) + val = self.exn_vt_stack[-1] + assert self._isinstance_exception(val), val + self._raise_exception_variable(val) + elif inst.arg == 1: + # raise TOS + val = self.stack[-1] # type: ignore[assignment] + self._raise_exception_variable(val) + else: + # raise .. from ... + from_vt = self.pop() + val = self.pop() # type: ignore[assignment] + try: + self._raise_exception_variable(val) + finally: + # Update __cause__/__suppress_context__ in the raised exception + curr_exc = self.exn_vt_stack.get_current_exception() + cause = self._create_exception_type(from_vt) + curr_exc.call_setattr(self, ConstantVariable("__cause__"), cause) # type: ignore[arg-type, union-attr, assignment] + + def CLEANUP_THROW(self, inst: Instruction) -> None: + # https://github.com/python/cpython/pull/96010 + tos = self.stack[-1] + assert isinstance(tos, ExceptionVariable) + if tos.exc_type is StopIteration: + unimplemented( + gb_type="CLEANUP_THROW with StopIteration", + context="", + explanation="Received StopIteration when handling generator.throw/close. This is not supported.", + hints=[], + ) + else: + self.RERAISE(inst) + + def RERAISE(self, inst: Instruction) -> None: + # https://docs.python.org/3/library/dis.html#opcode-RERAISE + # Re-raises the exception currently on top of the stack. If oparg is + # non-zero, pops an additional value from the stack which is used to + # set f_lasti of the current frame. + + if sys.version_info >= (3, 11): + # RERAISE is currently supported in a narrow case of `raise ... from None` + val = self.pop() + if inst.argval: + # RERAISE 1 + _ = self.pop() + self._raise_exception_variable(val) + else: + # RERAISE 0 + self.push(val) + self._raise_exception_variable(val) + else: + _exc = self.pop() + val = self.pop() + _tb = self.pop() + self._raise_exception_variable(val) + + def _isinstance_exception(self, val: VariableTracker) -> TypeIs[ExceptionVals]: + return isinstance( + val, + ( + variables.ExceptionVariable, + UserDefinedExceptionClassVariable, + UserDefinedExceptionObjectVariable, + ), + ) + + def WITH_EXCEPT_START(self, inst: Instruction) -> None: + args: list[VariableTracker] = [] + if sys.version_info >= (3, 11): + fn_loc = 4 if sys.version_info < (3, 14) else 5 + # At the top of the stack are 4 values: + # - TOP = exc_info() + # - SECOND = previous exception + # - THIRD: lasti of exception in exc_info() + # - FOURTH: the context.__exit__ bound method + # We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). + # Then we push the __exit__ return value. + # In Python 3.14+, there is a NULL placed between the context.__exit__ bound method and the lasti, + # that is, fn is now the 5th from TOS. + assert len(self.stack) >= fn_loc + fn = self.stack[-fn_loc] + val = self.stack[-1] + assert self._isinstance_exception(val) + typ = BuiltinVariable(val.exc_type) # type: ignore[attr-defined, union-attr] + tb = ConstantVariable(None) + if sys.version_info >= (3, 14): + if not isinstance(self.stack[-4], NullVariable): + args.append(self.stack[-4]) + else: + assert len(self.stack) >= 7 + fn = self.stack[-7] + val = self.stack[-2] + assert self._isinstance_exception(val) + typ = BuiltinVariable(val.exc_type) # type: ignore[attr-defined] + tb = ConstantVariable(None) + + args += [typ, val, tb] + self.call_function(fn, args, {}) + + def exception_handler(self, raised_exception: ObservedException) -> None: + observed_exn_gb_explanation = ( + "Dynamo found no exception handler at the top-level compiled function " + "when encountering an exception. Exception will propagate outside the compiled region." + ) + + def bubble_exception_to_interpreter() -> None: + # Bubble the exception to the interpreter + curr_exc = self.exn_vt_stack.get_current_exception() + dynamo_exc = exc.get_dynamo_observed_exception(curr_exc.python_type()) + assert isinstance(raised_exception, dynamo_exc) # sanity check + unimplemented( + gb_type="Observed exception", + context=f"raised exception {curr_exc.python_type_name()}({curr_exc.args})", # type: ignore[union-attr] + explanation=observed_exn_gb_explanation, + hints=[ + *graph_break_hints.USER_ERROR, + *graph_break_hints.SUPPORTABLE, + ], + from_exc=raised_exception, + ) + + if sys.version_info >= (3, 11): + exn_tab_entry = self.current_instruction.exn_tab_entry + if exn_tab_entry: + # Implementation is based on https://github.com/python/cpython/blob/3.11/Objects/exception_handling_notes.txt + + # 1) pop values from the stack until it matches the stack depth + # for the handler + while len(self.stack) > exn_tab_entry.depth: + self.pop() + + # 2) if 'lasti' is true, then push the offset that the exception was raised at + if exn_tab_entry.lasti: + self.push( + variables.ConstantVariable(self.current_instruction.offset) + ) + + # 3) push the exception to the stack + self.push(self.exn_vt_stack.get_current_exception()) + + # 4) jump to the handler + self.jump(exn_tab_entry) # type: ignore[arg-type] + else: + # No handler found. Bubble the exception to the parent + # instruction translator. We use special exception for this. + self.stack.clear() + if type(self) is InstructionTranslator: + bubble_exception_to_interpreter() + raise raised_exception + else: + if len(self.block_stack): + # base implementation - https://github.com/python/cpython/blob/3.10/Python/ceval.c#L4455 + + block_stack_entry = self.block_stack.pop() + + while block_stack_entry.inst.opname == "EXCEPT_HANDLER": + # TODO(anijain2305) - This is not tested .. unable to create a testcase + # https://github.com/python/cpython/blob/3.10/Python/ceval.c#L1456 + self.popn(3) + self.exn_vt_stack.pop() + if len(self.block_stack) == 0: + # No handler found in this frame. Bubble the exception to the parent + # instruction translator. + self.stack.clear() + if type(self) is InstructionTranslator: + unimplemented( + gb_type="Observed exception (EXCEPT_HANDLER)", + context=str(raised_exception), + explanation=observed_exn_gb_explanation + + " This graph break is unexpected.", + hints=[*graph_break_hints.DYNAMO_BUG], + ) + + raise raised_exception + block_stack_entry = self.block_stack.pop() + + exception_var = self.exn_vt_stack.get_current_exception() + self.exn_vt_stack.move_current_exception_to_stack() + + # 1) pop values from the stack until it matches the stack depth + # for the handler + while len(self.stack) > block_stack_entry.stack_index: + self.pop() + + # Push a dummy block stack entry of EXCEPT_HANDLER + # https://github.com/python/cpython/blob/3.10/Python/ceval.c#L1456 + except_handler_inst = Instruction(1e6, "EXCEPT_HANDLER", None, 0) + self.block_stack.append( + BlockStackEntry(except_handler_inst, None, len(self.stack)) + ) + + # Push old exception + if len(self.exn_vt_stack) >= 2: + old_exception = self.exn_vt_stack[-2] + + # Push the old exception on to stack - tb, value, type + # Traceback is currently mapped to UnknownVariable + self.push(variables.UnknownVariable()) + self.push(old_exception) + self.push(variables.BuiltinVariable(old_exception.exc_type)) + else: + # Push empty exception tb, value, type + self.push(variables.ConstantVariable(None)) + self.push(variables.ConstantVariable(None)) + self.push(variables.ConstantVariable(None)) + + # Push new exception - tb, val, type + # Traceback is currently mapped to UnknownVariable + self.push(variables.UnknownVariable()) + self.push(exception_var) + self.push(variables.BuiltinVariable(exception_var.exc_type)) + + # Jump to target + self.jump(block_stack_entry) + else: + # No handler found. Bubble the exception to the parent + # instruction translator. We use special exception for this. + self.stack.clear() + if type(self) is InstructionTranslator: + bubble_exception_to_interpreter() + raise raised_exception + + def PUSH_EXC_INFO(self, inst: Instruction) -> None: + # https://docs.python.org/3/library/dis.html#opcode-PUSH_EXC_INFO + # Pops a value from the stack. Pushes the current exception to the top + # of the stack. Pushes the value originally popped back to the stack. + # + # The behavior of this opcode in CPython is a bit different than what it + # is described. It pops a value from the stack, pushes the top of the + # exception stack to the interpreter stack and moves the + # "current exception" to the exception stack. + # + # As an example, suppose the stack is in the following state: + # + stack = [..., ConstantVariable(1), ConstantVariable(2)] + # + current_exception = TypeError + # + exception_stack = [ValueError] + # + # After PUSH_EXC_INFO is executed + # + stack = [..., ConstantVariable(1), ValueError, ConstantVariable(2)] + # + current_exception = None + # + exception_stack = [ValueError, TypeError] + + val = self.pop() + if len(self.exn_vt_stack) == 0: + prev_exc: VariableTracker = ConstantVariable(None) + else: + prev_exc = self.exn_vt_stack[-1] + self.push(prev_exc) + self.push(val) + self.exn_vt_stack.move_current_exception_to_stack() + + def POP_EXCEPT(self, inst: Instruction) -> None: + if sys.version_info >= (3, 11): + _ = self.pop() + # This exception is handled and therefore we can clear the error indicator + assert len(self.exn_vt_stack) + self.exn_vt_stack.pop() + else: + assert len(self.block_stack) > 0 + if self.block_stack[-1].inst.opname != "EXCEPT_HANDLER": + raise AssertionError( + "Bug in Dynamo tracing of exception handling." + "Top of the block stack is not EXCEPT_HANDLER." + ) + self.block_stack.pop() + + self.popn(3) + + # This exception is handled and therefore we can clear the error indicator + assert len(self.exn_vt_stack) + self.exn_vt_stack.pop() + + def check_if_exc_matches(self) -> bool: + assert len(self.stack) >= 2 + expected_exc_types = self.pop() + if sys.version_info >= (3, 11): + # CHECK_EXC_MATCH (which is used from 3.11 onwards) does not pop. + # This is the description from the disassembly doc + # + # Performs exception matching for ``except``. Tests whether the ``STACK[-2]`` + # is an exception matching ``STACK[-1]``. Pops ``STACK[-1]`` and pushes the boolean + # result of the test. + exc_instance = self.stack[-1] + else: + # This is used prior to 3.11 via opcode JUMP_IF_NOT_EXC_MATCH + # There is no documentation but here is the code pointer that does 2 pops + # https://github.com/python/cpython/blob/3.10/Python/ceval.c#L3650-L3665 + exc_instance = self.stack.pop() + + # Users can check exception in 3 ways + # 1) except NotImplementedError --> BuiltinVariable + # 2) except CustomException --> UserDefinedExceptionClassVariable + # 3) except (NotImplementedError, AttributeError) -> TupleVariable + + if not isinstance( + expected_exc_types, + ( + BuiltinVariable, + TupleVariable, + UserDefinedExceptionClassVariable, + UserDefinedExceptionObjectVariable, + ), + ): + unimplemented( + gb_type="Exception with bad expected type", + context=str(expected_exc_types), + explanation=f"`except ...` has unsupported type {expected_exc_types}.", + hints=[*graph_break_hints.USER_ERROR], + ) + + if sys.version_info >= (3, 11): + if not self._isinstance_exception(exc_instance): + unimplemented( + gb_type="Caught non-Exception value", + context=str(exc_instance), + explanation=f"Except expects to receive an object of Exception type but received {exc_instance}.", + hints=[*graph_break_hints.USER_ERROR], + ) + + if isinstance(expected_exc_types, TupleVariable): + expected_types = expected_exc_types.items + else: + expected_types = [ + expected_exc_types, + ] + + for expected_type in expected_types: + if not isinstance( + expected_type, + ( + BuiltinVariable, + UserDefinedExceptionObjectVariable, + UserDefinedExceptionClassVariable, + ), + ): + unimplemented( + gb_type="Exception with non-type expectation", + context=str(expected_type), + explanation=f"`except ...` expects a non-type: {expected_type}.", + hints=[*graph_break_hints.USER_ERROR], + ) + if self._isinstance_exception(exc_instance) and issubclass( + exc_instance.exc_type, # type: ignore[union-attr] + expected_type.fn, # type: ignore[attr-defined] + ): + return True + elif isinstance(exc_instance, variables.BuiltinVariable) and issubclass( + exc_instance.fn, + # pyrefly: ignore [missing-attribute] + expected_type.fn, + ): + return True + + return False + + def CHECK_EXC_MATCH(self, inst: Instruction) -> None: + self.push(variables.ConstantVariable(self.check_if_exc_matches())) + + def JUMP_IF_NOT_EXC_MATCH(self, inst: Instruction) -> None: + if not self.check_if_exc_matches(): + self.jump(inst) + + def COMPARE_OP(self, inst: Instruction) -> None: + if inst.argval == "exception match": + self.CHECK_EXC_MATCH(inst) + else: + self.push(compare_op_handlers[inst.argval](self, self.popn(2), {})) + + def GET_ITER(self, inst: Instruction) -> None: + self.call_function(BuiltinVariable(iter), [self.pop()], {}) + + @break_graph_if_unsupported( + push=True, + msg_prefix="Encountered graph break when attempting to trace CALL_FUNCTION: a call to a regular function, e.g. f(x, y)", + ) + def CALL_FUNCTION(self, inst: Instruction) -> None: + args = self.popn(inst.argval) + fn = self.pop() + self.call_function(fn, args, {}) + + @break_graph_if_unsupported( + push=True, + msg_prefix="Encountered graph break when attempting to trace CALL_FUNCTION_EX: a variadic function call, e.g. f(*args)", + ) + def CALL_FUNCTION_EX(self, inst: Instruction) -> None: + kwargsvars: VariableTracker + if inst.argval == 0: + kwargsvars = ConstDictVariable({}) + argsvars = self.pop() + elif inst.argval == 1 or sys.version_info >= (3, 14): + # Python 3.14+ removed the argval and replaced it with a possibly NULL kwargs + kwargsvars = self.pop() + if isinstance(kwargsvars, NullVariable): + kwargsvars = ConstDictVariable({}) + argsvars = self.pop() + else: + unimplemented( + gb_type="Variadic function call with bad flags", + context=f"flags: {inst.argval}", + explanation=f"Attempted to call a variadic function (CALL_FUNCTION_EX) with bad flags {inst.argval}", + hints=[*graph_break_hints.DYNAMO_BUG], + ) + + if sys.version_info >= (3, 13): + # 3.13 swapped null and callable + null = self.pop() + assert isinstance(null, NullVariable) + + fn = self.pop() + + if sys.version_info >= (3, 11) and sys.version_info < (3, 13): + null = self.pop() + assert isinstance(null, NullVariable) + + if not isinstance( + # pyrefly: ignore [unbound-name] + argsvars, + BaseListVariable, + # pyrefly: ignore [unbound-name] + ) and argsvars.has_force_unpack_var_sequence(self): + # pyrefly: ignore [unbound-name] + argsvars = TupleVariable(argsvars.force_unpack_var_sequence(self)) + + # Unpack for cases like fn(**obj) where obj is a map + # pyrefly: ignore [unbound-name] + if isinstance(kwargsvars, UserDefinedObjectVariable): + kwargsvars = BuiltinVariable.call_custom_dict(self, dict, kwargsvars) # type: ignore[arg-type] + + # pyrefly: ignore [unbound-name] + if not isinstance(argsvars, BaseListVariable) or not isinstance( + # pyrefly: ignore [unbound-name] + kwargsvars, + ConstDictVariable, + ): + unimplemented( + gb_type="Variadic function call with bad args/kwargs type", + # pyrefly: ignore [unbound-name] + context=f"args type: {typestr(argsvars)}, kwargs type: {typestr(kwargsvars)}", + explanation="Expected args to be a list and kwargs to be a dict", + hints=[*graph_break_hints.USER_ERROR], + ) + + # Map to a dictionary of str -> VariableTracker + # pyrefly: ignore [unbound-name, missing-attribute] + kwargsvars = kwargsvars.keys_as_python_constant() + # pyrefly: ignore [unbound-name, missing-attribute] + self.call_function(fn, argsvars.items, kwargsvars) + + @break_graph_if_unsupported( + push=True, + msg_prefix="Encountered graph break when attempting to trace CALL_FUNCTION_KW: " + "a function call with keyword arguments, e.g. f(x=True)", + ) + def CALL_FUNCTION_KW(self, inst: Instruction) -> None: + argnames = self.pop() + args = self.popn(inst.argval) + fn = self.pop() + assert isinstance(argnames, TupleVariable) and argnames.is_python_constant() + argnames = argnames.as_python_constant() + args, kwargs_list = args[: -len(argnames)], args[-len(argnames) :] + kwargs = dict(zip(argnames, kwargs_list)) + assert len(kwargs) == len(argnames) + self.call_function(fn, args, kwargs) + + def LOAD_METHOD_SUPER(self, inst: Instruction) -> None: + self.CALL_FUNCTION(dataclasses.replace(inst, argval=2)) + arg = inst.argval[0] + argval = self.code_options["co_names"][arg] + if sys.version_info < (3, 11): + self._load_attr(argval) + else: + self.LOAD_METHOD(dataclasses.replace(inst, argval=argval)) + + def LOAD_ATTR_SUPER(self, inst: Instruction) -> None: + self.CALL_FUNCTION(dataclasses.replace(inst, argval=2)) + arg = inst.argval[0] + argval = self.code_options["co_names"][arg] + self._load_attr(argval) + + def LOAD_METHOD(self, inst: Instruction) -> None: + self._load_attr(inst.argval) + obj = self.pop() + if sys.version_info >= (3, 13): + self.push(obj) + self.PUSH_NULL(inst) + elif sys.version_info >= (3, 11): + # always follow the NULL + fn convention, since if obj + # is actually a method, self is already bound to it, so it + # doesn't need to be passed in as an arg. + self.PUSH_NULL(inst) + self.push(obj) + else: + self.push(obj) + self.push(None) + + def CALL_METHOD(self, inst: Instruction) -> None: + args = self.popn(inst.argval) + dummy = self.pop() + assert dummy is None + fn = self.pop() + self.call_function(fn, args, {}) + + def _load_attr(self, attr: Any) -> None: + obj = self.pop() + result = BuiltinVariable(getattr).call_function( + self, # type: ignore[arg-type] + [obj, ConstantVariable.create(attr)], + {}, + ) + self.push(result) + + def LOAD_ATTR(self, inst: Instruction) -> None: + if sys.version_info >= (3, 12): + # pyrefly: ignore [unsupported-operation] + if inst.arg % 2: + self.LOAD_METHOD(inst) + return + self._load_attr(inst.argval) + + @break_graph_if_unsupported( + push=False, + msg_prefix="Encountered graph break when attempting to trace STORE_ATTR: storing an object's attribute, e.g. x.attr = y", + ) + def STORE_ATTR(self, inst: Instruction) -> None: + val, obj = self.popn(2) + BuiltinVariable(setattr).call_function( + self, # type: ignore[arg-type] + [obj, ConstantVariable.create(inst.argval), val], + {}, + ) + + def DELETE_ATTR(self, inst: Instruction) -> None: + obj = self.pop() + BuiltinVariable(delattr).call_function( + self, # type: ignore[arg-type] + [obj, ConstantVariable.create(inst.argval)], + {}, + ) + + @staticmethod + def codegen_return_with_pops( + inst: Instruction, num_stack: int + ) -> list[Instruction]: + """ + Debug CPython expects the stack to be empty after the return. + Calling compile_subgraph will push cells and frame values to TOS. + This function will pop those 2 values from the stack before actually returning. + + Expects the stack to be: + cells, frame values, current frame stack (0 or 1 values) + + Pops cells and frame values, leaving the current frame stack as TOS. + A return instruction is included. + """ + insts = [] + # NOTE: Debug CPython expects the stack to be empty after the return. + # Expect the current stack to be in the state + # cells, frame values, current frame stack (0 or 1 values) + assert num_stack <= 1 + if num_stack == 1: + insts.extend(create_swap(3)) + return_inst = ( + create_instruction("RETURN_VALUE") + if inst.opname == "RETURN_VALUE" + else create_instruction("RETURN_CONST", argval=inst.argval) + ) + insts.extend( + [create_instruction("POP_TOP"), create_instruction("POP_TOP"), return_inst] + ) + return insts + + def create_resume( + self, + idx: int, + resume_inst: Instruction, + meta: StackLocalsMetadata, + resume_codes: list[types.CodeType], + cg: PyCodegen, + is_leaf: bool, + handle_inactive_ctx: bool, + ) -> tuple[types.CodeType, str]: + """ + Creates the resume function for the frame corresponding to `self`. + + Expects the TOS to be: + [frame N cells, ..., frame 1 cells], + [ + frame N stack + locals, + ..., + frame 1 stack + locals + ] + + Some additional codegen may happen to prepare the frame stack + locals values for the generated resume function: + - inactive context variables in the stack and locals will be replaced by their types + - if the frame is a leaf frame, prune dead locals + + Regardless of codegen, the stack will be left in the same state as before. + + Args: + - idx: depth of this frame: 0 corresponds to the leaf frame (frame N), N-1 to the root frame (frame 1). + - resume_inst: the instruction that this frame should resume at + - meta: metadata for this frame returned from OutputGraph.compile_subgraph + - resume_codes: nested resume code objects generated from previous create_resume calls. + - cg: codegen object to output to + - is_leaf: True if `self` corresponds to the leaf frame. + - handle_inactive_ctx: If True, handles inactive context variables as described above. This is necessary + iff the resume function is traced + """ + # Handle inactive context variables. + # The resume function assumes that context variables are the class, NOT the object. + # e.g. torch.set_grad_enabled(True) will be reconstructed as torch.set_grad_enabled + # NOTE: if the unsupported instruction modifies the inactive context variable, it may + # result in silent incorrectness! + if handle_inactive_ctx: + for (j, _), j_orig in zip(meta.stack_ctx_args, meta.stack_ctx_idxes_orig): + # Replace the stack var with the context class + ctx = cast(ContextWrappingVariable, self.stack[j_orig]) + # frames[idx][j] = reconstructed_ctx + cg.append_output(create_dup_top()) + ctx.reconstruct_type(cg) + cg.extend_output( + [ + *create_swap(2), + cg.create_load_const(idx), + cg.create_binary_subscr(), + cg.create_load_const(j), + create_instruction("STORE_SUBSCR"), + ] + ) + + for name, _ in meta.locals_ctx_args: + # Replace the local with the context class + ctx = cast(ContextWrappingVariable, self.symbolic_locals[name]) + # frames[idx][meta.num_stack +meta.locals_names[name]] = reconstructed_ctx + cg.append_output(create_dup_top()) + ctx.reconstruct_type(cg) + cg.extend_output( + [ + *create_swap(2), + cg.create_load_const(idx), + cg.create_binary_subscr(), + cg.create_load_const(meta.num_stack + meta.locals_names[name]), + create_instruction("STORE_SUBSCR"), + ] + ) + + # If the resume instruction is a jump absolute, then resume + # at the target instead. This handles the case where we + # graph break again in a nested function before jump-resuming + # this frame. + if is_jump_absolute(resume_inst): + assert resume_inst.target + resume_inst = resume_inst.target + + resume_name = unique_id(f"__resume_at_{resume_inst.offset}") + + # More locals may have been pruned in the current/leaf frame + # after the unsupported instruction (e.g. branch). + # There should not be any pruning in the other frames since + # the current instruction there should be a CALL. + if is_leaf: + reads = livevars_analysis(self.instructions, resume_inst) + all_argnames = tuple( + k + for k in self.symbolic_locals + if k in reads and k not in self.cell_and_freevars() + ) + argnames_null_set = set(meta.locals_null_keys) + argnames = tuple(k for k in all_argnames if k not in argnames_null_set) + argnames_null = tuple(k for k in all_argnames if k in argnames_null_set) + + # codegen filter for current frame's locals + # current stack state: frames + cg.extend_output( + [ + create_dup_top(), + cg.create_load_const(idx), + cg.create_binary_subscr(), + create_dup_top(), + ] + ) + for arg in argnames: + # current stack state: frames, frames[i], *(prev locals), frames[i] + cg.extend_output( + [ + create_dup_top(), + cg.create_load_const(meta.num_stack + meta.locals_names[arg]), + cg.create_binary_subscr(), + *create_swap(2), + ], + ) + # current stack state: frames, frames[i], *(frame i live locals), frames[i] + cg.extend_output( + [ + create_instruction("POP_TOP"), + create_instruction("BUILD_LIST", arg=len(argnames)), + *create_swap(2), + # frames, frames i live locals, frames[i] + *create_binary_slice(meta.num_stack, None, True), + # frames[i][num_stack:] = frame i live locals + ] + ) + # current stack state: frames + else: + argnames = tuple(meta.locals_names.keys()) + argnames_null = tuple(meta.locals_null_keys) + + if sys.version_info < (3, 12): + assert len(argnames_null) == 0, "variables should not be NULL in < 3.12" + + # compile_subgraph did not codegen any NULLs, + # so we should not count NullVariables + stack_len = len(self.stack) - len(meta.stack_null_idxes) + + assert self.current_instruction.offset is not None + + new_code: types.CodeType = ContinueExecutionCache.lookup( + self.f_code, + self.lineno, + self.current_instruction.offset, + resume_inst.offset, + tuple(b.target.offset for b in self.block_stack), + stack_len, + argnames, + argnames_null, + tuple(b.resume_fn() for b in self.block_stack), + handle_inactive_ctx, + tuple(meta.stack_ctx_args), + tuple(meta.locals_ctx_args), + tuple(meta.stack_null_idxes), + tuple(resume_codes), + not self.current_instruction_push, + ) + + # Add original GraphModule context to the resume function to handle + # the case of a graph break while tracing a GraphModule + orig_graphmodule_maybe = code_context.get_context(self.f_code).get( + "orig_graphmodule", lambda: None + )() + if orig_graphmodule_maybe is not None: + code_context.get_context(new_code)["orig_graphmodule"] = weakref.ref( + orig_graphmodule_maybe + ) + + # add resume function to the global scope + if new_code.co_freevars: + # expose code object for debugging purposes + self.output.install_global_unsafe(resume_name, new_code) + package_name = None + else: + # This is safe: we pre-generate a unique name + self.output.install_global_unsafe( + resume_name, + types.FunctionType(new_code, self.f_globals, resume_name), + ) + package_name = resume_name + + if self.package is not None: + self.package.add_resume_function( + new_code, self.f_globals["__name__"], package_name + ) + + counters["resumes"][new_code.co_name] += 1 + + return new_code, resume_name + + def create_call_resume_at( + self, + inst: Instruction, + all_stack_locals_metadata: list[StackLocalsMetadata], + ) -> list[Instruction]: + """ + Codegen all resume function(s) from the frame stack starting at `self`, call them, + and return the result. + Assumes that the unsupported instruction has already been run. + + Expects the TOS to be: + [ + frame N locals, + frame N-1 stack + locals, + ..., + frame 1 stack + locals + ], *(frame N stack (post-unsupported instruction)) + + Leaves the result of calling the resume functions on the stack and returns it + (empty stack after return). + + Args: + - inst: the instruction of the current (deepest) frame to resume at + - all_stack_locals_metadata: metadata returned from OutputGraph.compile_subgraph - contains + metadata such as local names, NULL positions, stack length, etc. + """ + + self.instruction_pointer = None + + cg = PyCodegen(self.output.root_tx) + + # NOTE: We do not need to codegen frames whose resume instruction is RETURN_VALUE + # We could also do something similar for RETURN_CONST, but a lot more code is necessary + # since we would need to track RETURN_CONST values and inject the constant in the right places. + + # Filter out tx'es that are resuming on RETURN_*. + txes: list[InstructionTranslatorBase] = [] + idxes: list[int] = [] + resume_insts: list[Instruction] = [] + cur_tx: Optional[InstructionTranslatorBase] = self + idx = 0 + while cur_tx is not None: + if cur_tx is self: + resume_inst = inst + else: + resume_inst = cur_tx.next_instruction + if resume_inst.opname != "RETURN_VALUE": + txes.append(cur_tx) + idxes.append(idx) + resume_insts.append(resume_inst) + + cur_tx = cur_tx.parent + idx += 1 + + current_num_stack = len(self.stack) - len( + all_stack_locals_metadata[0].stack_null_idxes + ) + + # Every tx is returning - no need to call a resume function. + if not txes: + # Pop everything but TOS, then return the TOS. + # Frame N's stack must have length >= 1 since it's about to RETURN_VALUE. + # Frame N actually should have stack length == 1, because debug CPython expects + # empty stacks after return, but there is no guarantee written down anywhere. + assert current_num_stack >= 1 + cg.extend_output(create_swap(current_num_stack + 2)) + for _ in range(current_num_stack + 1): + cg.append_output(create_instruction("POP_TOP")) + cg.append_output(create_instruction("RETURN_VALUE")) + + return cg.get_instructions() + + # Let frame k be the deepest frame where the resume function is not RETURN_VALUE + # - If k == N, then the frame N stack is prepended to the frame N locals. + # - If k != N, then frame N's TOS is added to frame k's stack. + + # Rearrange the TOS to be compatible with create_resume and codegen_call_resume: + # [ + # frame N stack + locals, + # ..., + # frame 1 stack + locals + # ] + + # create the stack values that should be moved + if txes[0] is self: + # Frame N is non-returning, pack all of frame N's stack to + # be moved to frame N's frame values + cg.append_output(create_instruction("BUILD_LIST", arg=current_num_stack)) + # frame N stack is not yet on the frame N's frame values + stack_insert_idx = 0 + all_stack_locals_metadata[0].num_stack = current_num_stack + else: + # Frame N is returning. Let frame k be the deepest non-returning frame. + # Add frame N's TOS to frame k's stack. + # pop frame N stack except TOS + cg.extend_output(create_swap(current_num_stack)) + for _ in range(current_num_stack - 1): + cg.append_output(create_instruction("POP_TOP")) + cg.append_output(create_instruction("BUILD_LIST", arg=1)) + # frame k stack is already on frame k's frame values + stack_insert_idx = all_stack_locals_metadata[idxes[0]].num_stack + all_stack_locals_metadata[idxes[0]].num_stack += 1 + txes[0].push(UnknownVariable()) + + # move the predetermined stack value(s) to the deepest non-returning frame + cg.extend_output( + [ + *create_copy(2), + # frame_values, return_const, frame_values + cg.create_load_const(idxes[0]), + cg.create_binary_subscr(), + *create_binary_slice(stack_insert_idx, stack_insert_idx, True), + # frame_values[idxes[0]][stack_insert_idx:stack_insert_idx] = frame N stack/[return_const/TOS] + # frame_values left on top of stack + ] + ) + + # filter out frame values of skipped tx'es + filter_insts = [] + for idx in idxes: + filter_insts.extend( + [ + create_dup_top(), + cg.create_load_const(idx), + cg.create_binary_subscr(), + *create_swap(2), + ] + ) + # TOS: cells, frame_values[idxes[0]], ..., frame_values[idxes[...]], frame_values + filter_insts.extend( + [ + create_instruction("POP_TOP"), + create_instruction("BUILD_LIST", arg=len(idxes)), + ] + ) + # TOS: cells, filtered frame_values + + cg.extend_output(filter_insts) + # filter out cells of skipped tx'es using the same instructions in filter_insts, + # but with cells as TOS instead of frame values + cg.extend_output( + [ + *create_swap(2), + *copy.deepcopy(filter_insts), + *create_swap(2), + ] + ) + # TOS: filtered cells, filtered frame_values + + resume_codes: list[types.CodeType] = [] + resume_names = [] + for i, cur_tx in enumerate(txes): + resume_code, resume_name = cur_tx.create_resume( + i, + resume_insts[i], + all_stack_locals_metadata[idxes[i]], + resume_codes, + cg, + cur_tx is self, + True, + ) + resume_codes.append(resume_code) + resume_names.append(resume_name) + + self.codegen_call_resume(resume_codes, resume_names, cg) + cg.append_output(create_instruction("RETURN_VALUE")) + + return cg.get_instructions() + + @staticmethod + def codegen_call_resume( + resume_codes: list[types.CodeType], resume_names: list[str], cg: PyCodegen + ) -> None: + """ + Calls the provided resume functions. + + Expects the TOS to be in the state: + [frame N cells, ..., frame 1 cells], + [ + frame N stack + locals, + frame N-1 stack + locals, + ..., + frame 1 stack + locals + ] + + Pops the cells and frame values, leaving the result of calling the resume functions on TOS. + + Args: + - resume_codes: list of resume function code objects to call + - resume_names: list of the corresponding names of the resume functions + - cg: PyCodegen object to output instructions to + """ + # NOTE: We will load cells as we load resume functions + + # load resume functions except the root's + cg.extend_output(create_copy(2)) + for i, (name, code) in enumerate(zip(resume_names, resume_codes)): + if i == len(resume_names) - 1: + break + # stack: cells, frames, *(resume 1, ...), cells + if code.co_freevars: + cg.extend_output( + [ + create_dup_top(), + cg.create_load_const(i), + cg.create_binary_subscr(), + ] + ) + cg.make_function_with_closure(name, code) + else: + cg.extend_output(cg.load_function_name(name, False, 0)) + cg.extend_output(create_swap(2)) + cg.extend_output( + [ + create_instruction("POP_TOP"), + create_instruction("BUILD_LIST", arg=len(resume_codes) - 1), + ] + ) + + # stack: cells, frames, [resume 1, ..., resume N - 1] + # load root resume function + cg.extend_output(create_swap(3)) + if resume_codes[-1].co_freevars: + cg.extend_output( + [ + cg.create_load_const(-1), + cg.create_binary_subscr(), + ] + ) + cg.make_function_with_closure(resume_names[-1], resume_codes[-1]) + cg.extend_output( + [ + *create_rot_n(3), + ] + ) + else: + cg.extend_output( + [ + create_instruction("POP_TOP"), + *cg.load_function_name(resume_names[-1], False), + *create_rot_n(3), + ] + ) + + # resume 1, [resume N, ..., resume 2], frames + + # load top level-frame; final stack state should be: + # first resume function (+ NULL), + # [ + # [resume N, ..., resume 2], + # [ + # frame N stack + locals, + # ..., + # frame 2 stack + locals, + # ], *(frame 1 stack + locals) + # ] + cg.extend_output( + [ + create_dup_top(), + create_dup_top(), + # frames, frames, frames + cg.create_load_const(-1), + cg.create_binary_subscr(), + # frames, frames, frames[-1] + *create_swap(2), + # frames, frames[-1], frames + cg.create_load_const(-1), + create_instruction("DELETE_SUBSCR"), + ] + ) + + # TOS: resume 1, remaining resumes, frames (popped), frame 1 stack + locals + cg.extend_output( + [ + *create_rot_n(3), + create_instruction("BUILD_LIST", arg=2), + *create_swap(2), + # [resumes, frames (popped)], frame 1 stack + locals + create_instruction("LIST_EXTEND", arg=1), + ] + ) + + # TOS: resume 1, [remaining resumes, frames, *(frame 1 stack + locals)] + cg.extend_output(create_call_function_ex(False, True)) + + def should_compile_partial_graph(self) -> bool: + if sys.version_info >= (3, 11): + # Do not compile if current instruction's block is not the top with block + entry = self.current_instruction.exn_tab_entry + if entry and ( + not self.block_stack or entry.target is not self.block_stack[-1].target + ): + return False + return ( + all(b.can_restore() for b in self.block_stack) + and not self.one_graph + and not self.error_on_graph_break + and not self.is_tracing_resume_prologue + and not self.active_generic_context_managers + # Do not allow nested graph breaks in HOPs + and self.output.current_tracer.parent is None + ) + + @break_graph_if_unsupported( + push=False, + msg_prefix="Encountered graph break when attempting to trace STORE_SUBSCR: trying to store subscript, e.g. x[key] = y", + ) + def STORE_SUBSCR(self, inst: Instruction) -> None: + val, obj, key = self.popn(3) + obj.call_method(self, "__setitem__", [key, val], {}) + + def DELETE_SUBSCR(self, inst: Instruction) -> None: + obj, key = self.popn(2) + obj.call_method(self, "__delitem__", [key], {}) + + def BUILD_TUPLE(self, inst: Instruction) -> None: + items = self.popn(inst.argval) + self.push(TupleVariable(items)) + + def BUILD_SLICE(self, inst: Instruction) -> None: + items = self.popn(inst.argval) + self.push(SliceVariable(items, tx=self)) # type: ignore[arg-type] + + def BUILD_LIST(self, inst: Instruction) -> None: + items = self.popn(inst.argval) + self.push(ListVariable(items, mutation_type=ValueMutationNew())) + + def BUILD_SET(self, inst: Instruction) -> None: + if config.inject_BUILD_SET_unimplemented_TESTING_ONLY: + unimplemented( + gb_type="missing BUILD_SET handler", + context="", + explanation="Missing BUILD_SET bytecode handler (for testing purposes).", + hints=[], + ) + items = self.popn(inst.argval) + new_set = SetVariable(items, mutation_type=ValueMutationNew()) + self.push(new_set) + + def BUILD_LIST_UNPACK(self, inst: Instruction, cls: type = ListVariable) -> None: + seqs = self.popn(inst.argval) + items = [] + for seq in seqs: + try: + items.extend(seq.force_unpack_var_sequence(self)) + except NotImplementedError: + unimplemented( + gb_type="Failed to unpack object for BUILD_LIST_UNPACK", + context=str(seq), + explanation=f"{seq} cannot be unpacked into a list for the BUILD_LIST_UNPACK " + "bytecode (`[*x, *y, ...]`).", + hints=[*graph_break_hints.USER_ERROR], + ) + self.push(cls(items, mutation_type=ValueMutationNew())) + + def BUILD_TUPLE_UNPACK(self, inst: Instruction) -> None: + self.BUILD_LIST_UNPACK(inst, cls=TupleVariable) + + BUILD_TUPLE_UNPACK_WITH_CALL = BUILD_TUPLE_UNPACK + + def BUILD_MAP(self, inst: Instruction) -> None: + items = self.popn(inst.argval * 2) + d = dict(zip(items[::2], items[1::2])) + self.push(ConstDictVariable(d, mutation_type=ValueMutationNew())) + + def BUILD_MAP_UNPACK(self, inst: Instruction) -> None: + items = self.popn(inst.argval) + # ensure everything is a dict + items = [BuiltinVariable(dict).call_function(self, [x], {}) for x in items] # type: ignore[arg-type] + result: dict[Any, Any] = {} + for x in items: + assert isinstance(x, ConstDictVariable) + result.update(x.items) + self.push( + ConstDictVariable( + result, + mutation_type=ValueMutationNew(), + ) + ) + + BUILD_MAP_UNPACK_WITH_CALL = BUILD_MAP_UNPACK + + def BUILD_CONST_KEY_MAP(self, inst: Instruction) -> None: + keys = self.pop() + values = self.popn(inst.argval) + assert isinstance(keys, TupleVariable) + assert keys.is_python_constant() + + keys = keys.force_unpack_var_sequence(self) + assert len(keys) == len(values) + + self.push( + ConstDictVariable( + dict(zip(keys, values)), + mutation_type=ValueMutationNew(), + ) + ) + + def MAP_ADD(self, inst: Instruction) -> None: + k, v = self.popn(2) + assert inst.argval > 0 + assert inst.arg is not None + obj = self.stack[-inst.arg].realize() + assert isinstance(obj, ConstDictVariable) + obj.call_method(self, "__setitem__", (k, v), {}) # type: ignore[arg-type] + + def SET_ADD(self, inst: Instruction) -> None: + v = self.pop() + assert inst.argval > 0 + assert inst.arg is not None + obj = self.stack[-inst.arg] + assert isinstance(obj, SetVariable) + assert obj.is_mutable() + obj.call_method(self, "add", [v], {}) # type: ignore[arg-type] + + def SET_UPDATE(self, inst: Instruction) -> None: + v = self.pop() + assert inst.argval > 0 + assert inst.arg is not None + obj = self.stack[-inst.arg] + assert isinstance(obj, SetVariable) + assert obj.is_mutable() + obj.call_method(self, "update", [v], {}) # type: ignore[arg-type] + + def LIST_APPEND(self, inst: Instruction) -> None: + v = self.pop() + assert inst.argval > 0 + assert inst.arg is not None + obj = self.stack[-inst.arg].realize() + assert isinstance(obj, ListVariable) + assert obj.is_mutable() + self.output.side_effects.mutation(obj) + obj.items.append(v) + + def MAKE_FUNCTION(self, inst: Instruction) -> None: + flags = inst.arg + if sys.version_info < (3, 11): + fn_name = self.pop() + code = self.pop() + if sys.version_info >= (3, 11): + # MAKE_FUNCTION behavior actually changed in 3.11, see + # https://github.com/python/cpython/pull/93189/ + assert hasattr(code.value, "co_qualname") # type: ignore[attr-defined] + fn_name = ConstantVariable.create(value=code.value.co_qualname) # type: ignore[attr-defined] + defaults = None + closure = None + annotations = None + kwdefaults = None + + if sys.version_info < (3, 13): + # in 3.13, this is handled in SET_FUNCTION_ATTRIBUTE + if flags is not None: + if flags & 0x08: + closure = self.pop() + if flags & 0x04: + annotations = self.pop() + if flags & 0x02: + kwdefaults = self.pop() + if flags & 0x01: + defaults = self.pop() + + self.push( + NestedUserFunctionVariable( + fn_name, + code, + self.f_globals, + defaults, + kwdefaults, + annotations, + closure, + ) + ) + + def UNPACK_SEQUENCE(self, inst: Instruction) -> None: + seq = self.pop() + if seq.is_tensor(): + val = seq.unpack_var_sequence(self, idxes=range(inst.argval)) # type: ignore[arg-type] + elif isinstance(seq, GetAttrVariable) and seq.obj.is_tensor(): + # x, y = a.shape + proxy = getattr(seq.obj.as_proxy(), seq.name) + val = [wrap_fx_proxy(self, proxy[i]) for i in range(inst.argval)] + elif seq.has_force_unpack_var_sequence(self): + val = seq.force_unpack_var_sequence(self) + else: + unimplemented( + gb_type="Failed to unpack object for UNPACK_SEQUENCE", + context=str(seq), + explanation=f"{seq} cannot be unpacked into a list for the UNPACK_SEQUENCE bytecode " + "(i.e. `a, b, c = d`).", + hints=[*graph_break_hints.USER_ERROR], + ) + # pyrefly: ignore [unbound-name] + if len(val) != inst.argval: + unimplemented( + gb_type="Length mismatch when unpacking object for UNPACK_SEQUENCE", + # pyrefly: ignore [unbound-name] + context=f"expected length: {inst.argval}, actual: {len(val)}", + explanation=f"{seq} unpacked to a list for the UNPACK_SEQUENCE bytecode " + "(i.e. `a, b, c = d`) with unexpected length.", + hints=[*graph_break_hints.DYNAMO_BUG], + ) + # pyrefly: ignore [unbound-name] + for i in reversed(val): + self.push(i) + + def UNPACK_EX(self, inst: Instruction) -> None: + assert 0 <= inst.argval <= 0xFFFF + prefix = inst.argval & 0xFF # low byte + suffix = inst.argval >> 8 # high byte + seq = self.pop() + if seq.has_force_unpack_var_sequence(self): + vals = list(seq.force_unpack_var_sequence(self)) + assert len(vals) >= prefix + suffix + vals_prefix = vals[:prefix] + vals_list = vals[prefix : len(vals) - suffix] + vals_suffix = vals[len(vals) - suffix :] + for item in reversed(vals_suffix): + self.push(item) + self.push(TupleVariable(vals_list)) + for item in reversed(vals_prefix): + self.push(item) + else: + unimplemented( + gb_type="Failed to unpack object for UNPACK_EX", + context=str(seq), + explanation=f"{seq} cannot be unpacked into a list for the UNPACK_EX bytecode.", + hints=[*graph_break_hints.USER_ERROR], + ) + + @break_graph_if_unsupported( + push=False, msg_prefix="Encountered intentional debugging graph break" + ) + def graph_break_on_leaf_function(self, inst: Instruction) -> None: + if self.is_leaf_tracer: + unimplemented( + gb_type="Forced graph break on leaf function", + context="", + explanation="Forced graph break for nested graph break testing purposes", + hints=[ + "Set torch._dynamo.config.debug_force_graph_break_on_leaf_return = False", + ], + ) + + def NOP(self, inst: Instruction) -> None: + # Dynamo-specific testing behavior + if inst.argval == "GRAPH_BREAK_IF_LEAF": + self.graph_break_on_leaf_function(inst) + + def POP_TOP(self, inst: Instruction) -> None: + self.pop() + + def ROT_TWO(self, inst: Instruction) -> None: + a = self.pop() + b = self.pop() + self.push(a) + self.push(b) + + def ROT_THREE(self, inst: Instruction) -> None: + a = self.pop() + b = self.pop() + c = self.pop() + self.push(a) + self.push(c) + self.push(b) + + def ROT_FOUR(self, inst: Instruction) -> None: + a = self.pop() + b = self.pop() + c = self.pop() + d = self.pop() + self.push(a) + self.push(d) + self.push(c) + self.push(b) + + def DUP_TOP(self, inst: Instruction) -> None: + a = self.pop() + self.push(a) + self.push(a) + + def DUP_TOP_TWO(self, inst: Instruction) -> None: + a = self.pop() + b = self.pop() + self.push(b) + self.push(a) + self.push(b) + self.push(a) + + def _convert_value(self, value: VariableTracker, flag: int) -> VariableTracker: + if flag == 1: + return BuiltinVariable(str).call_function(self, [value], {}) # type: ignore[arg-type] + elif flag == 2: + return BuiltinVariable(repr).call_function(self, [value], {}) # type: ignore[arg-type] + elif flag == 3: + return BuiltinVariable(ascii).call_function(self, [value], {}) # type: ignore[arg-type] + return value + + def _format_value(self, fmt_spec: VariableTracker, flags: int) -> None: + value = self.pop() + if isinstance(value, SymNodeVariable): + from torch._dynamo.variables.lazy import ( + LazySymNodeFormatString, + LazyVariableTracker, + ) + + value = LazyVariableTracker.create( + LazySymNodeFormatString(value, fmt_spec), source=value.source + ) + self.push(value) + return + + value = self._convert_value(value, flags & 0x03) + + fmt_var = ConstantVariable.create("{:" + fmt_spec.as_python_constant() + "}") + + self.call_function(BuiltinVariable(str.format), [fmt_var, value], {}) + + def FORMAT_VALUE(self, inst: Instruction) -> None: + flags = inst.arg + assert flags is not None + if (flags & 0x04) == 0x04: + fmt_spec = self.pop() + else: + fmt_spec = ConstantVariable.create("") + + return self._format_value(fmt_spec, flags) + + def BUILD_STRING(self, inst: Instruction) -> None: + format_string_parts: list[str] = [] + args: list[VariableTracker] = [] + kwargs: dict[str, VariableTracker] = {} + assert inst.arg is not None + for part in self.popn(inst.arg): + if part.is_python_constant(): + format_string_parts.append("{}") + args.append(part) + elif isinstance(part, variables.StringFormatVariable): + format_string_parts.append(part.format_string) + args.extend(part.sym_args) + if set(kwargs.keys()) & set(part.sym_kwargs.keys()): + unimplemented( + gb_type="BUILD_STRING key conflict", + context=f"format_string_parts: {format_string_parts}, kwargs: {kwargs}, part.sym_kwargs: {part.sym_kwargs}", + explanation="Failed to build format string due to key conflict", + hints=[*graph_break_hints.USER_ERROR], + ) + kwargs.update(part.sym_kwargs) + else: + unimplemented( + gb_type="BUILD_STRING type error", + context=str(part), + explanation="Format string part type is not correct - expected constant or format string.", + hints=[*graph_break_hints.USER_ERROR], + ) + self.push( + variables.StringFormatVariable.create( + "".join(format_string_parts), args, kwargs + ) + ) + + def IS_OP(self, inst: Instruction) -> None: + assert inst.argval == 0 or inst.argval == 1 + if inst.argval == 0: + new_argval = "is" + else: + new_argval = "is not" + new_inst = create_instruction("COMPARE_OP", argval=new_argval) + self.COMPARE_OP(new_inst) + + def CONTAINS_OP(self, inst: Instruction) -> None: + assert inst.argval == 0 or inst.argval == 1 + left, right = self.popn(2) + op = inst.argval + try: + self.push(right.call_method(self, "__contains__", [left], {})) + except ( + # right.__contains__ can raise TypeError + exc.ObservedTypeError, + # Ideally we should only capture TypeError here but some VTs don't + # implement hasattr(vt, "__contains__") entirely + Unsupported, + ) as excp: # object doesn't support __contains__ + # Use __iter__ as fallback + if isinstance(excp, Unsupported): + excp.remove_from_stats() + self.push( + self.inline_user_function_return( + VariableTracker.build(self, impl_CONTAINS_OP_fallback), + [left, right], + {}, + ) + ) + if op == 1: + self.UNARY_NOT(inst) + + def LIST_EXTEND(self, inst: Instruction) -> None: + v = self.pop() + assert inst.argval > 0 + assert inst.arg is not None + obj = self.stack[-inst.arg] + assert isinstance(obj, ListVariable) + assert obj.is_mutable() + obj.call_method(self, "extend", [v], {}) # type: ignore[arg-type] + + def LIST_TO_TUPLE(self, inst: Instruction) -> None: + self.push(BuiltinVariable(tuple).call_function(self, [self.pop()], {})) # type: ignore[arg-type] + + def STOPITERATION_ERROR(self, inst: Instruction) -> None: + # wrap the generator body in a try: ... except StopIteration: ... which + # converts the StopIteration into a RuntimeError + # https://peps.python.org/pep-0479/ + # https://github.com/python/cpython/pull/99006 + # https://github.com/python/cpython/commit/28187141cc34063ef857976ddbca87ba09a882c2 + val = self.stack[-1] + assert self._isinstance_exception(val) + if val.exc_type is StopIteration: # type: ignore[union-attr] + new_val = variables.BuiltinVariable(RuntimeError).call_function( + self, # type: ignore[arg-type] + [ConstantVariable("generator raised StopIteration")], + {}, + ) + new_val.call_setattr(self, ConstantVariable("__context__"), val) # type: ignore[attr-defined] + new_val.call_setattr(self, ConstantVariable("__cause__"), val) # type: ignore[attr-defined] + self.stack[-1] = new_val + + def DICT_MERGE(self, inst: Instruction) -> None: + v = self.pop() + assert inst.argval > 0 + assert inst.arg is not None + obj = self.stack[-inst.arg].realize() + assert isinstance(obj, ConstDictVariable) + assert obj.is_mutable() + obj.call_method(self, "update", [v], {}) # type: ignore[arg-type] + + DICT_UPDATE = DICT_MERGE + + def GEN_START(self, inst: Instruction) -> None: + self.pop() + + def GET_LEN(self, inst: Instruction) -> None: + tos = self.stack[-1] + if tos.is_python_constant(): + self.push(ConstantVariable.create(len(tos.as_python_constant()))) + else: + self.push(tos.call_method(self, "__len__", [], {})) + + def MATCH_MAPPING(self, inst: Instruction) -> None: + tos = self.stack[-1] + assert isinstance(tos, ConstDictVariable) + if isinstance(tos.items, collections.abc.Mapping): + self.push(ConstantVariable.create(True)) + else: + self.push(ConstantVariable.create(False)) + + def MATCH_SEQUENCE(self, inst: Instruction) -> None: + tos = self.stack[-1] + assert tos.is_python_constant() + tos_value = tos.as_python_constant() + if isinstance(tos_value, collections.abc.Sequence) and not isinstance( + tos_value, (str, bytes, bytearray) + ): + self.push(ConstantVariable.create(True)) + else: + self.push(ConstantVariable.create(False)) + + def MATCH_KEYS(self, inst: Instruction) -> None: + tos = self.stack[-1] + assert isinstance(tos, TupleVariable) + keys = tos.unpack_var_sequence(self) # type: ignore[arg-type] + tos1 = self.stack[-2] + assert isinstance(tos1, ConstDictVariable) + + if all(k in tos1 for k in keys): # type: ignore[attr-defined] + self.push(TupleVariable([tos1.getitem_const(self, k) for k in keys])) # type: ignore[attr-defined,arg-type] + if sys.version_info < (3, 11): + self.push(ConstantVariable.create(True)) + else: + self.push(ConstantVariable.create(None)) + if sys.version_info < (3, 11): + self.push(ConstantVariable.create(False)) + + def LOAD_ASSERTION_ERROR(self, inst: Instruction) -> None: + self.push(self.load_builtin_from_argval("AssertionError")) + + def LOAD_BUILD_CLASS(self, inst: Instruction) -> None: + self.push(self.load_builtin_from_argval("__build_class__")) + + UNARY_POSITIVE = stack_op(operator.pos) + UNARY_NEGATIVE = stack_op(operator.neg) + UNARY_NOT = stack_op(operator.not_) + UNARY_INVERT = stack_op(operator.invert) + + BINARY_POWER = stack_op(operator.pow) + BINARY_MULTIPLY = stack_op(operator.mul) + BINARY_MATRIX_MULTIPLY = stack_op(operator.matmul) + BINARY_FLOOR_DIVIDE = stack_op(operator.floordiv) + BINARY_TRUE_DIVIDE = stack_op(operator.truediv) + BINARY_MODULO = stack_op(operator.mod) + BINARY_REMAINDER = stack_op(operator.mod) + BINARY_ADD = stack_op(operator.add) + BINARY_SUBTRACT = stack_op(operator.sub) + BINARY_SUBSCR = break_graph_if_unsupported( + push=True, + msg_prefix="Encountered graph break when attempting to trace BINARY_SUBSCR: a binary subscript, e.g. x[attr]", + )(stack_op(operator.getitem)) + BINARY_LSHIFT = stack_op(operator.lshift) + BINARY_RSHIFT = stack_op(operator.rshift) + BINARY_AND = stack_op(operator.and_) + BINARY_OR = stack_op(operator.or_) + BINARY_XOR = stack_op(operator.xor) + + INPLACE_POWER = stack_op(operator.ipow) + INPLACE_MULTIPLY = stack_op(operator.imul) + INPLACE_MATRIX_MULTIPLY = stack_op(operator.imatmul) + INPLACE_FLOOR_DIVIDE = stack_op(operator.ifloordiv) + INPLACE_TRUE_DIVIDE = stack_op(operator.itruediv) + INPLACE_MODULO = stack_op(operator.imod) + INPLACE_REMAINDER = stack_op(operator.imod) + INPLACE_ADD = stack_op(operator.iadd) + INPLACE_SUBTRACT = stack_op(operator.isub) + INPLACE_LSHIFT = stack_op(operator.ilshift) + INPLACE_RSHIFT = stack_op(operator.irshift) + INPLACE_AND = stack_op(operator.iand) + INPLACE_XOR = stack_op(operator.ixor) + INPLACE_OR = stack_op(operator.ior) + + # 3.11 opcodes + def RESUME(self, inst: Instruction) -> None: + if inst.arg == 0: + self.append_prefix_inst(inst) + self.accept_prefix_inst = False + else: + assert not self.accept_prefix_inst + + if sys.version_info >= (3, 11): + + def BINARY_OP(self, inst: Instruction) -> None: + assert inst.arg is not None + return _binary_op_lookup[inst.arg](self, inst) + + def PRECALL(self, inst: Instruction) -> None: + pass + + def KW_NAMES(self, inst: Instruction) -> None: + kw_names = self.code_options["co_consts"][inst.arg] + assert isinstance(kw_names, tuple) + for name in kw_names: + assert isinstance(name, str) + assert self.kw_names is None + self.kw_names = ConstantVariable.create(value=kw_names) # type: ignore[assignment] + + def PUSH_NULL(self, inst: Instruction) -> None: + self.push(NullVariable()) + + def _call(self, inst: Instruction, call_kw: bool = False) -> None: + # see https://docs.python.org/3.11/library/dis.html#opcode-CALL + # for convention + if call_kw: + # TOS is kw_names for CALL_KW instruction + assert sys.version_info >= (3, 13) + kw_names = self.pop() + assert isinstance(kw_names, TupleVariable) and kw_names.is_python_constant() + kw_names = kw_names.as_python_constant() + else: + kw_names = self.kw_names.value if self.kw_names else () + + assert inst.arg is not None + contents = self.popn(inst.arg + 2) + if sys.version_info >= (3, 13): + # NULL and callable swapped + fn = contents[0] + args = [] if isinstance(contents[1], NullVariable) else [contents[1]] + else: + if isinstance(contents[0], NullVariable): + fn = contents[1] + args = [] + else: + fn = contents[0] + args = [contents[1]] + + if kw_names: + # pyrefly: ignore [bad-argument-type] + args = args + contents[2 : -len(kw_names)] + # pyrefly: ignore [bad-argument-type] + kwargs_list = contents[-len(kw_names) :] + # pyrefly: ignore [no-matching-overload] + kwargs = dict(zip(kw_names, kwargs_list)) + # pyrefly: ignore [bad-argument-type] + assert len(kwargs) == len(kw_names) + else: + args = args + contents[2:] + kwargs = {} + + try: + # if call_function fails, need to set kw_names to None, otherwise + # a subsequent call may have self.kw_names set to an old value + self.call_function(fn, args, kwargs) + finally: + self.kw_names = None + + @break_graph_if_unsupported( + push=True, + msg_prefix="Encountered graph break when attempting to trace CALL: a function call, e.g. f(x, y)", + ) + def CALL(self, inst: Instruction) -> None: + self._call(inst) + + def COPY(self, inst: Instruction) -> None: + assert inst.arg is not None + self.push(self.stack[-inst.arg]) + + def SWAP(self, inst: Instruction) -> None: + assert inst.arg is not None + self.stack[-1], self.stack[-inst.arg] = self.stack[-inst.arg], self.stack[-1] + + JUMP_BACKWARD = jump + JUMP_BACKWARD_NO_INTERRUPT = jump + + POP_JUMP_FORWARD_IF_TRUE = generic_jump(operator.truth, False) + POP_JUMP_BACKWARD_IF_TRUE = generic_jump(operator.truth, False) + POP_JUMP_FORWARD_IF_FALSE = generic_jump(operator.not_, False) + POP_JUMP_BACKWARD_IF_FALSE = generic_jump(operator.not_, False) + + def CACHE(self, inst: Instruction) -> None: + pass + + def BEFORE_WITH(self, inst: Instruction) -> None: + self.setup_or_before_with(inst) + + def enter_ctx( + self, + ctx: Union[ContextWrappingVariable, GenericContextWrappingVariable], + inst: Instruction, + ) -> VariableTracker: + if ( + isinstance(ctx, GenericContextWrappingVariable) + and not ctx.supports_graph_breaks() + ): + self.active_generic_context_managers.append(ctx) + + if sys.version_info >= (3, 11): + # See update_block_stack/create_resume for block stack details. + # Only push a block if the current instruction's block is a + # with block that is not nested in a try block - that is, the current + # instruction's block target is the same as the top block's target. + if inst.exn_tab_entry and ( + not self.block_stack + or inst.exn_tab_entry.target is not self.block_stack[-1].target + ): + target = None + else: + assert self.next_instruction.exn_tab_entry is not None + target = self.next_instruction.exn_tab_entry.target + else: + target = inst.target + + if target: + if isinstance(self, InstructionTranslator) or config.nested_graph_breaks: + self.block_stack.append( + BlockStackEntry(inst, target, len(self.stack), ctx) + ) + else: + self.block_stack.append(BlockStackEntry(inst, target, len(self.stack))) + + return ctx.enter(self) # type: ignore[arg-type] + + @staticmethod + def unsupported_ctx_graph_break(ctx: VariableTracker) -> NoReturn: + unimplemented( + gb_type="Unsupported context manager", + context=f"Attempted SETUP_WITH/BEFORE_WITH/LOAD_SPECIAL on {ctx}", + explanation=f"Dynamo does not know how to enter a `{ctx.python_type_name()}` context manager.", + hints=[ + "Avoid using the unsupported context manager.", + "If the context manager seems like it should be supported (e.g. torch.set_grad_enabled), then " + "it may be the case that it was created outside the compiled region, which Dynamo does not support. " + "Supported context managers can cross graph break boundaries only if they are local non-closure " + "variables, or are intermediate values.", + "File an issue to PyTorch. Simple context managers can potentially be supported, " + "but note that context managers can't be supported in general", + ], + ) + + def setup_or_before_with(self, inst: Instruction) -> None: + ctx = self.pop() + if not isinstance( + ctx, (ContextWrappingVariable, GenericContextWrappingVariable) + ): + self.unsupported_ctx_graph_break(ctx) + + # Need this redundant check for mypy + assert isinstance( + ctx, (ContextWrappingVariable, GenericContextWrappingVariable) + ) + + self.push(WithExitFunctionVariable(ctx, inst.target)) + self.push(self.enter_ctx(ctx, inst)) + + def append_prefix_inst(self, inst: Instruction) -> None: + assert self.accept_prefix_inst + self.prefix_insts.append(inst) + + def MAKE_CELL(self, inst: Instruction) -> None: + if sys.version_info >= (3, 12) and not self.accept_prefix_inst: + # In 3.12+, MAKE_CELL is not longer necessarily a prefix instruction. + # It can be generated by inlined comprehensions. + assert isinstance(self.symbolic_locals[inst.argval], NullVariable) + self.symbolic_locals[inst.argval] = ( + self.output.side_effects.track_cell_new() + ) + else: + self.append_prefix_inst(inst) + + def COPY_FREE_VARS(self, inst: Instruction) -> None: + self.append_prefix_inst(inst) + + def RETURN_GENERATOR(self, inst: Instruction) -> None: + self.append_prefix_inst(inst) + + # 3.12 opcodes + # BINARY/STORE_SLICE opcodes are broken down into + # BUILD_SLICE 2 and BINARY/STORE_SUBSCR + + def END_FOR(self, inst: Instruction) -> None: + if sys.version_info >= (3, 13): + self.pop() + else: + self.popn(2) + + def LOAD_FAST_CHECK(self, inst: Instruction) -> None: + if istype(self.symbolic_locals.get(inst.argval, None), NullVariable): + unimplemented( + gb_type="LOAD_FAST_CHECK on uninitialized variable", + context=inst.argval, + explanation=f"Attempted to load uninitialized local variable {inst.argval}", + hints=[*graph_break_hints.USER_ERROR], + ) + self.LOAD_FAST(inst) + + def LOAD_FAST_AND_CLEAR(self, inst: Instruction) -> None: + if inst.argval not in self.symbolic_locals: + self.push(NullVariable()) + else: + self.LOAD_FAST(inst) + self.symbolic_locals[inst.argval] = NullVariable() + + def LOAD_SUPER_ATTR(self, inst: Instruction) -> None: + self.CALL_FUNCTION(dataclasses.replace(inst, argval=2)) + assert inst.arg is not None + if inst.arg & 1: + self.LOAD_METHOD(inst) + else: + self._load_attr(inst.argval) + + def CALL_INTRINSIC_1(self, inst: Instruction) -> None: + if inst.argval == 3: + # INTRINSIC_STOPITERATION_ERROR + self.STOPITERATION_ERROR(inst) + elif inst.argval == 5: + # INTRINSIC_UNARY_POSITIVE + self.UNARY_POSITIVE(inst) + elif inst.argval == 6: + # INTRINSIC_LIST_TO_TUPLE + self.push(TupleVariable(self.pop().force_unpack_var_sequence(self))) + else: + unimplemented( + gb_type="Missing CALL_INTRINSIC_1 handler", + context=f"CALL_INTRINSIC_1 operand: {inst.argval}", + explanation=f"No handler implemented for CALL_INTRINSIC_1 {inst.argval} instruction.", + hints=[*graph_break_hints.SUPPORTABLE], + ) + + def END_SEND(self, inst: Instruction) -> None: + tos = self.pop() + self.pop() + self.push(tos) + + # 3.13 opcodes + # fused instructions LOAD_FAST_LOAD_FAST, STORE_FAST_STORE_FAST, STORE_FAST_LOAD_FAST + # are broken down. + @break_graph_if_unsupported( + push=True, + msg_prefix="Encountered graph break when attempting to trace CALL_KW: " + "a function call with keyword arguments, e.g. f(x=True)", + ) + def CALL_KW(self, inst: Instruction) -> None: + self._call(inst, call_kw=True) + + def TO_BOOL(self, inst: Instruction) -> None: + # TO_BOOL only precedes a conditional jump or UNARY_NOT (see compile.c in CPython) + # So we can skip this instruction as long as we remember to codegen a TO_BOOL + # before conditional jumps/UNARY_NOT. + assert self.next_instruction.opname in ( + "POP_JUMP_IF_TRUE", + "POP_JUMP_IF_FALSE", + "UNARY_NOT", + ) + + def SET_FUNCTION_ATTRIBUTE(self, inst: Instruction) -> None: + flags = inst.arg + assert flags is not None + fn = self.pop() + assert isinstance(fn, NestedUserFunctionVariable) + attr = self.pop() + + if flags & 0x10: + assert sys.version_info >= (3, 14) + + # maybe use Format.VALUE_WITH_FAKE_GLOBALS instead? + # https://docs.python.org/3/library/annotationlib.html#annotationlib.Format.VALUE_WITH_FAKE_GLOBALS + attr = attr.call_function(self, [ConstantVariable.create(1)], {}) + fn.annotations = attr + elif flags & 0x08: + fn.closure = attr + elif flags & 0x04: + fn.annotations = attr + elif flags & 0x02: + fn.kwdefaults = attr + elif flags & 0x01: + fn.defaults = attr + + self.push(fn) + + def CONVERT_VALUE(self, inst: Instruction) -> None: + self.push(self._convert_value(self.pop(), inst.argval)) + + def FORMAT_SIMPLE(self, inst: Instruction) -> None: + self._format_value(ConstantVariable.create(""), 0) + + def FORMAT_WITH_SPEC(self, inst: Instruction) -> None: + self._format_value(self.pop(), 0) + + # 3.14 opcodes + LOAD_FAST_BORROW = LOAD_FAST + NOT_TAKEN = NOP + POP_ITER = POP_TOP + + # See + # https://github.com/python/cpython/blob/805e3368d6d07e58430654d1365283924fdf4143/Python/ceval.c#L559 + # for the LOAD_SPECIAL table - make sure it matches for Python 3.14+ + _load_special_names = ( + "__enter__", + "__exit__", + "__aenter__", + "__aexit__", + ) + + def LOAD_SPECIAL(self, inst: Instruction) -> None: + assert isinstance(inst.arg, int), "expected LOAD_SPECIAL arg to be set to int" + attr = self._load_special_names[inst.arg] + if attr in ("__enter__", "__exit__"): + ctx = self.pop() + if not isinstance( + ctx, (ContextWrappingVariable, GenericContextWrappingVariable) + ): + self.unsupported_ctx_graph_break(ctx) + + # Need this redundant check for mypy + assert isinstance( + ctx, (ContextWrappingVariable, GenericContextWrappingVariable) + ) + if attr == "__enter__": + self.push(WithEnterFunctionVariable(ctx)) + self.PUSH_NULL(inst) + else: + # WithExitFunctionVariable doesn't really do anything with target for 3.11+ + self.push(WithExitFunctionVariable(ctx, None)) + self.PUSH_NULL(inst) + else: + # Implementation is similar to LOAD_METHOD for 3.13+ + self._load_attr(attr) + obj = self.pop() + self.push(obj) + self.PUSH_NULL(inst) + + def LOAD_SMALL_INT(self, inst: Instruction) -> None: + self.push(ConstantVariable.create(inst.argval)) + + # See + # https://github.com/python/cpython/blob/7519ac294fc5c4fd7fb9cb8dc0edc960688cf887/Python/pylifecycle.c#L814 + # for the common constants - make sure it matches for Python 3.14+. + # The common constants are all attributes of `builtins`. + _common_constants = ( + "AssertionError", + "NotImplementedError", + "tuple", + "all", + "any", + ) + + def LOAD_COMMON_CONSTANT(self, inst: Instruction) -> None: + assert isinstance(inst.arg, int), ( + "expected LOAD_COMMON_CONSTANT arg to be set to int" + ) + self.push(self.load_builtin_from_argval(self._common_constants[inst.arg])) + + def is_non_empty_graph(self) -> bool: + if self.output.count_calls() > 1: + # perf optimization only + self.is_non_empty_graph = lambda: True # type: ignore[method-assign] + return True + return False + + def format_frame_summary( + self, additional_stack_frames: Optional[list[Any]] = None + ) -> str: + if additional_stack_frames is None: + additional_stack_frames = [] + return "".join( + traceback.format_list( + [self.frame_summary()] + list(reversed(additional_stack_frames)) + ) + ) + + def frame_summary(self) -> traceback.FrameSummary: + return traceback.FrameSummary( + getattr(self.f_code, "co_filename", ""), + self.lineno, + getattr(self.f_code, "co_name", ""), + lookup_line=False, + ) + + def is_co_filename_from_nn_modules(self) -> bool: + filename = getattr(self.f_code, "co_filename", "") + nn_modules_pattern = re.compile(r".*torch/nn/modules.*") + return nn_modules_pattern.match(filename) is not None + + def store_global_weakref_by_id(self, prefix: str, value: Any) -> str: + global_name = self.output.install_global_by_id(prefix, weakref.ref(value)) + install_guard( + GlobalWeakRefSource(global_name).make_guard(GuardBuilder.WEAKREF_ALIVE) + ) + return global_name + + @property + def fake_mode(self) -> Optional[FakeTensorMode]: + return self.output.tracing_context.fake_mode + + @contextlib.contextmanager + def strict_translation_mode( + self, check_fn: Callable[[VariableTracker], bool] + ) -> Any: + """ + Strict mode is enabled on a per-VariableTracker level depending on the return value of check_fn(node). + """ + prior = self.strict_checks_fn + self.strict_checks_fn = check_fn + try: + yield + finally: + self.strict_checks_fn = prior + + def speculate(self) -> SpeculationEntry: + assert self.instruction_pointer is not None + assert self.instruction_pointer > 0 + return self.speculation_log.next( + self.f_code.co_filename, + self.lineno, + self.instruction_pointer - 1, + self.instructions[self.instruction_pointer - 1], + ) + + def _make_frame_loc( + self, filename: str, lineno: Optional[int], fallback_lineno: int + ) -> tuple[str, int]: + if lineno is None or lineno < 0: + return (filename, fallback_lineno) + return (filename, lineno) + + def _get_frame_loc_chain( + self, frame_loc: tuple[str, int] + ) -> tuple[tuple[str, int], ...]: + frame_loc_chain_list: list[tuple[str, int]] = [] + + if config.nested_graph_breaks: + current_tx: Optional[InstructionTranslatorBase] = self.parent + while current_tx is not None: + parent_frame_loc = self._make_frame_loc( + current_tx.f_code.co_filename, + current_tx.lineno, + current_tx.f_code.co_firstlineno, + ) + frame_loc_chain_list.append(parent_frame_loc) + current_tx = current_tx.parent + + frame_loc_chain_list.reverse() + frame_loc_chain_list.append(frame_loc) + return tuple(frame_loc_chain_list) + + def log_graph_break( + self, + code_options: dict[str, Any], + reason: str = "", + user_stack: Optional[StackSummary] = None, + ) -> None: + if user_stack is None: + user_stack = torch._guards.TracingContext.extract_stack() + + try: + if config.nested_graph_breaks and self.parent is not None: + frame_loc = self._make_frame_loc( + self.f_code.co_filename, + self.lineno, + self.f_code.co_firstlineno, + ) + else: + frame_loc = self._make_frame_loc( + user_stack[-1].filename, + user_stack[-1].lineno, + 0, + ) + except IndexError: + # first instruction + frame_loc = ( + code_options["co_filename"], + code_options["co_firstlineno"], + ) + frame_loc_chain = self._get_frame_loc_chain(frame_loc) + stack_above_dynamo_formatted = "" + if config.verbose: + stack_above_dynamo = get_stack_above_dynamo() + stack_above_dynamo_formatted = "".join( + traceback.format_list(stack_above_dynamo) + ) + else: + user_stack = get_stack_above_dynamo() + user_stack # type: ignore[assignment] + # pyrefly: ignore [bad-argument-type] + user_stack = collapse_resume_frames(user_stack) + user_stack_formatted = "".join(traceback.format_list(user_stack)) + user_stack_trace = ( + f"Graph break in user code at {frame_loc[0]}:{frame_loc[1]}\n" + f"Graph Break Reason: {reason}\n" + "User code traceback:\n" + ) + + if config.verbose: + user_stack_trace += ( + f"{stack_above_dynamo_formatted}\n" + "========== most recent `torch.compile` tracing attempt started here ==========\n\n" + f"{user_stack_formatted}\n" + "NOTE: the most recent `torch.compile` tracing attempt might not be where you applied `torch.compile`! " + "This is due to how graph breaks are implemented - the optimized code object returned by Dynamo will call another " + "Dynamo-generated resume function and tracing is re-enabled by calling the resume function as a normal Python " + "function, which Dynamo intercepts as a top-level frame.\n" + ) + else: + user_stack_trace += str(user_stack_formatted) + + torch._logging.trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "dynamo_graph_break_reason", + "encoding": "string", + }, + payload_fn=lambda: f"{user_stack_trace}\n{traceback.format_exc()}", + ) + + # torch._dynamo.explain() formats this a little nicer, and presents a slightly + # more actionable user code pointer + if ( + graph_break_log.isEnabledFor(logging.DEBUG) + and not explain + and graph_break_dup_warning_checker.add(frame_loc_chain) # type: ignore[arg-type] + ): + # This log line MUST contain the string "Graph break in user code", + # This log line is exercised from + # python test/dynamo/test_exc.py -k test_graph_break_log + if config.verbose: + user_stack_trace += ( + "\nMost recent bytecode instructions traced (max 20):\n" + ) + user_stack_trace += "\n".join(self.latest_bytecode_queue) + "\n" + + graph_break_log.debug( + user_stack_trace, + ) + else: + # This log line MUST not contain the string "Graph break in user code", + # exercised by + # python test/dynamo/test_misc.py -k test_duplicate_graph_break_log + graph_break_log.debug( + "Graph break (user stack suppressed due to duplicate graph break) in user code at %s:%s\nGraph Break Reason: %s", + frame_loc[0], + frame_loc[1], + reason, + ) + + def __init__( + self, + output: OutputGraph, + instructions: list[Instruction], + f_locals: dict[str, Any], + f_globals: dict[str, Any], + f_builtins: dict[str, Any], + code_options: dict[str, Any], + symbolic_locals: dict[str, VariableTracker], + symbolic_globals: dict[str, VariableTracker], + symbolic_torch_function_state: SymbolicTorchFunctionState, + symbolic_stream_state: SymbolicStreamState, + f_code: types.CodeType, + export: bool, + inline_depth: int, + speculation_log: SpeculationLog, + exn_vt_stack: ExceptionStack, + distributed_state: Optional[DistributedState], + # This determines whether to use the execution recorder. + closure: Optional[tuple[types.CellType]] = None, + package: Optional[CompilePackage] = None, + ) -> None: + super().__init__() + self.speculation_log = speculation_log + self.distributed_state = distributed_state + + # Mutable state checkpointed by copy_graphstate() + self.output = output + self.symbolic_locals = symbolic_locals + self.symbolic_globals = symbolic_globals + self.symbolic_torch_function_state = symbolic_torch_function_state + self.symbolic_stream_state = symbolic_stream_state + # used to keep cell/freevars alive after pruning symbolic_locals (prune_dead_locals) + # in order to generate any nested closures + self.post_prune_cell_and_freevars = None + self.stack: list[VariableTracker] = [] + self.instruction_pointer = 0 + self.start_point = None + self.current_instruction = create_instruction("NOP") + self.current_instruction_push = True + self.block_stack = [] + # states before SETUP_WITH for checkpointing and fallback + self.active_generic_context_managers: list[GenericContextWrappingVariable] = [] + self.lineno = -1 + self.kw_names = None + self.accept_prefix_inst = True + self.prefix_insts = [] + self.exn_vt_stack = exn_vt_stack + self.latest_bytecode_queue = deque(maxlen=20) + + # Properties of the input/output code + self.instructions: list[Instruction] = instructions + self.indexof: dict[Instruction, int] = get_indexof(self.instructions) + self.f_locals: dict[str, Any] = ( + f_locals # needed for recording accessed locals for replay + ) + self.f_globals: dict[str, Any] = f_globals + self.f_builtins: dict[str, Any] = f_builtins + self.code_options: dict[str, Any] = code_options + self.f_code: types.CodeType = f_code + self.closure = closure + + # Execution record for replaying errors + if closure is not None and config.replay_record_enabled: + self.exec_recorder = ExecutionRecorder( + code=f_code, closure=closure, code_options=code_options + ) + else: + self.exec_recorder = None + # Stack of module being parsed, current nn.module is at the end of ordered dict. + # The first field of tuple is the fully qualified name of current module + # in original hierarchy. The second field is the type of current nn.module + self.nn_module_stack: dict[str, tuple[str, type[Any]]] = {} + self.num_calls: dict[str, int] = {} + # Flag to indicate whether tracing is used for export. + self.export = export + # NOTE: one_graph is used for export/fullgraph=True to always force errors on graph breaks. + # To toggle erroring/resuming on graph breaks during fullgraph=False compile, self.error_on_graph_break + # is used instead. Every step(), its value is updated to the global tls.error_on_graph_break. + # We mirror this value since cleanup may (correctly) inadvertently change tls.error_on_graph_break. + # This assumes that we cannot both trace a change to tls.error_on_graph_break and graph break on + # the same instruction. + self.one_graph = False + self.error_on_graph_break = False + # Also do not graph break when tracing resume function prologues + self.is_tracing_resume_prologue = False + + self.current_speculation = None + + self.strict_checks_fn = None + + self.is_leaf_tracer = True + self.parent = None + self.debug_locals = [] + + self.package = package + + from .resume_execution import ( + CO_ASYNC_GENERATOR, + CO_COROUTINE, + CO_GENERATOR, + CO_ITERABLE_COROUTINE, + ) + + if f_code.co_flags & ( + CO_GENERATOR | CO_COROUTINE | CO_ITERABLE_COROUTINE | CO_ASYNC_GENERATOR + ): + self.push(BuiltinVariable(None)) + + self.inline_depth = inline_depth + self.inconsistent_side_effects = False + self._constants_cache: list[ + Optional[Union[ConstantVariable, SliceVariable]] + ] = [None] * len(f_code.co_consts) + + self.is_trace_bytecode_log_enabled: Optional[bool] = ( + trace_bytecode_log.isEnabledFor(logging.DEBUG) + ) + self.is_trace_source_log_enabled: Optional[bool] = ( + trace_source_log.isEnabledFor(logging.DEBUG) + ) + linecache.lazycache(f_code.co_filename, f_globals) + + +class InstructionTranslator(InstructionTranslatorBase): + @staticmethod + def current_tx() -> InstructionTranslator: + return tls.current_tx + + @contextlib.contextmanager + def set_current_tx(self) -> Any: + prior = getattr(tls, "current_tx", None) + tls.current_tx = self + try: + yield + finally: + tls.current_tx = prior + + def __init__( + self, + instructions: list[Instruction], + f_code: types.CodeType, + f_locals: dict[str, Any], + f_globals: dict[str, Any], + f_builtins: dict[str, Any], + closure: Optional[tuple[Any, ...]], + torch_function_mode_stack: Any, + code_options: dict[str, Any], + compiler_fn: Any, + one_graph: bool, + export: bool, + export_constraints: Any, + frame_state: Any, + speculation_log: SpeculationLog, + exn_vt_stack: ExceptionStack, + distributed_state: Optional[DistributedState], + package: Optional[CompilePackage], + ) -> None: + _step_logger()( + logging.INFO, + f"torchdynamo start tracing {f_code.co_name} {code_options['co_filename']}:{code_options['co_firstlineno']}", + ) + super().__init__( + output=OutputGraph( + code_options, + compiler_fn, + self, + export, + export_constraints, + frame_state, + local_scope=f_locals, + global_scope=f_globals, + f_code=f_code, + torch_function_mode_stack=torch_function_mode_stack, + one_graph=one_graph, + package=package, + ), + instructions=instructions, + f_locals=f_locals, + f_globals=f_globals, + f_builtins=f_builtins, + closure=closure, + code_options=code_options, + symbolic_locals={}, # set below + # A global var is inserted only after a STORE_GLOBAL happens to it + symbolic_globals={}, + symbolic_torch_function_state=None, # type: ignore[arg-type] # set below + symbolic_stream_state=None, # type: ignore[arg-type] # set below + f_code=f_code, + export=export, + inline_depth=0, + speculation_log=speculation_log, + exn_vt_stack=exn_vt_stack, + distributed_state=distributed_state, + package=package, + ) + + self._throw_if_in_functorch() + + # as soon as we create the tracing context we should keep it active, so any calls + # into dynamo apis can rely on finding it + with tracing(self.output.tracing_context), self.set_current_tx(): + self.one_graph: bool = one_graph + self.export = export + if self.export: + assert self.one_graph, ( + "Export without one graph - something has gone wrong." + ) + + self.symbolic_locals = {} + # Populate `symbolic_locals` with non-cell variables. + cell_and_freevars: set[str] = set(self.cell_and_freevars()) + + dynamism = code_context.get_context(f_code).get("dynamism", None) + for name, value in f_locals.items(): + if name not in cell_and_freevars: + local_dynamism = None + if dynamism: + local_dynamism = frozenset(dynamism.get(name, {}).items()) + var = LazyVariableTracker.create( + value, + LocalSource( + name, + is_input=True, + dynamism=local_dynamism, + ), + ) + self.symbolic_locals[name] = var + + # Populate `symbolic_locals` with cells created by this frame, + # effectively implementing the `MAKE_CELL` instructions. + side_effects = self.output.side_effects + for name in self.cellvars(): + if name in f_locals: + # This models cells that are also function inputs. + value = f_locals[name] + # NOTE: root frame inputs that are captured by a nested + # function become special cell objects -- they exist in + # `f_locals` as contents of the cells, rather than the cells + # objects themselves. + # + # In Dynamo, we choose to represent such input cell objects + # as newly created (rather than pre-existing) cell objects, + # because + # + # 1. The reason for representing a pre-existing cell object + # is to emit guard or codegen mutations. However, local + # cells should never be used for guards. Moreover, at this + # point these input cell objects should've never been + # accessed by anyone else, since Dynamo intercepts the frame + # right after its evaluation starts, i.e., right after these + # cell objects are created. So they should have no external + # reference, meaning no mutation needs to be propagated. + # + # 2. This conveniently allows codegen to prune away + # mutations to these cells, unless they escape the frame. + contents_source = LocalSource( + name, is_input=True, is_derefed_cell_contents=True + ) + contents_var: VariableTracker = LazyVariableTracker.create( + value, contents_source + ) + cell_var = side_effects.track_cell_new() + side_effects.store_cell(cell_var, contents_var) + else: + cell_var = side_effects.track_cell_new() + cell_var.local_name = name # type: ignore[attr-defined] + self.symbolic_locals[name] = cell_var + + # Populate `symbolic_locals` with cells captured by this frame, + # effectively implementing the `COPY_FREE_VARS` instruction. + assert closure is not None + for name, cell in zip(self.freevars(), closure): + cell_source = LocalCellSource(name) + contents_source = LocalSource(name, is_derefed_cell_contents=True) + try: + contents_var = LazyVariableTracker.create( + cell.cell_contents, contents_source + ) + except ValueError: + # Cell has not yet been assigned + contents_var = variables.DeletedVariable() + cell_var = side_effects.track_cell_existing( + cell_source, cell, contents_var + ) + cell_var.local_name = name # type: ignore[attr-defined] + self.symbolic_locals[name] = cell_var + + self.symbolic_torch_function_state = SymbolicTorchFunctionState( + torch_function_mode_stack + ) + + self.symbolic_stream_state = SymbolicStreamState() + + if export: + # export gets confused if we never realize unused inputs + # in export mode just eagerly realize everything + self.symbolic_locals = variables.LazyVariableTracker.realize_all( + self.symbolic_locals + ) + + def _throw_if_in_functorch(self) -> None: + # Fallback to eager in case of a graph break inside vmap + eager = torch._dynamo.lookup_backend("eager") + compiler_fn = inspect.getattr_static( + self.output.compiler_fn, "compiler_fn", self.output.compiler_fn + ) + ci = torch._C._functorch.peek_interpreter_stack() + forbidden_keys = ( + torch._C._functorch.TransformType.Vmap, + torch._C._functorch.TransformType.Grad, + torch._C._functorch.TransformType.Jvp, + ) + + if ci is not None and ci.key() in forbidden_keys and compiler_fn is not eager: + name = ci.key().name.lower() + msg = ( + "If you are reaching here, it means dynamo failed for one of the following reasons:\n" + # Calling a torch.compiled function + f"- Calling torch.func.{name}(compiled_fn) function from eager mode is not supported. " + f"Ensure that torch.func.{name} is also wrapped within a torch.compile function. " + "For more information, see PyTorch issue #128711.\n" + # if it reaches here, it means Dynamo failed to inline a functorch function + f"- torch.func.{name}(fn) requires the function to be inlined by dynamo" + ) + unimplemented( + gb_type="Unsupported functorch tracing attempt", + context="", + explanation=msg, + hints=[], + ) + + def get_example_value(self, source: Source) -> Any: + if isinstance(source, LocalSource): + return self.f_locals[source.local_name] + if isinstance(source, GlobalSource): + return self.f_globals[source.global_name] + raise KeyError + + def symbolic_locals_contain_module_class(self) -> bool: + for v in self.symbolic_locals.values(): + if isinstance(v, UserDefinedClassVariable) and issubclass( + v.as_python_constant(), torch.nn.Module + ): + return True + return False + + def replace_tos_if_return_is_generator(self) -> None: + if ( + len(self.stack) + and (tos := self.stack[-1]) + and isinstance(tos, LocalGeneratorObjectVariable) + ): + self.stack[-1] = ListIteratorVariable( + tos.force_unpack_var_sequence(self), + mutation_type=ValueMutationNew(), + ) + + def _return(self, inst: Instruction) -> None: + self.replace_tos_if_return_is_generator() + assert self.instruction_pointer is not None + assert self.start_point is not None + get_metrics_context().increment( + "ir_count", self.instruction_pointer - self.start_point + ) + + if ( + not config.allow_empty_graphs + and self.output.count_calls() == 0 + and not self.inconsistent_side_effects + and not self.symbolic_locals_contain_module_class() + and not self.export + and not self.one_graph + and not self.error_on_graph_break + and not self.is_tracing_resume_prologue + ): + raise exc.SkipFrame( + format_skip_frame_message(self.f_code, "no content in function call") + ) + self.instruction_pointer = None + _step_logger()( + logging.INFO, + f"torchdynamo done tracing {self.f_code.co_name} ({inst.opname})", + ) + log.debug("return triggered compile") + all_stack_locals_metadata = self.output.compile_subgraph( + self, + reason=GraphCompileReason( + "return_value", [self.frame_summary()], graph_break=False + ), + # the value to be returned + stack_pops=1 if inst.opname == "RETURN_VALUE" else 0, + ) + # check that our stack/locals meta are correct: + # we should only be tracing 1 frame, and there should not be any NULLs on the stack + assert len(all_stack_locals_metadata) == 1 + assert not all_stack_locals_metadata[0].stack_null_idxes + self.output.add_output_instructions( + self.codegen_return_with_pops(inst, all_stack_locals_metadata[0].num_stack) + ) + raise ReturnValueOp + + def RETURN_VALUE(self, inst: Instruction) -> None: + self._return(inst) + + def RETURN_CONST(self, inst: Instruction) -> None: + self._return(inst) + + +if sys.version_info >= (3, 11): + _binary_op_lookup = [ + getattr( + InstructionTranslator, + opname[3:] if "INPLACE" in opname else f"BINARY_{opname[3:]}", + ) + for opname, _ in dis._nb_ops # type: ignore[attr-defined] + ] + + +class InliningInstructionTranslator(InstructionTranslatorBase): + """Trace and inline a called method""" + + symbolic_result: Optional[VariableTracker] + # pyrefly: ignore [bad-override] + parent: InstructionTranslatorBase + + @classmethod + def inline_call(cls, parent: Any, func: Any, args: Any, kwargs: Any) -> Any: + tracer = cls.build_inline_tracer(parent, func, args, kwargs) + return tracer.inline_call_() + + @staticmethod + def check_inlineable(func: Any) -> trace_rules.SkipResult: + if func.has_self(): + unimplemented( + gb_type="Inline attempt with __self__", + context=str(func), + explanation="Attempted to inline a function with the `__self__` attribute. " + "Dynamo is expected to decompose method calls into function calls with a `self` argument.", + hints=[], + ) + + if isinstance(func, UserFunctionVariable) and inspect.getattr_static( + func.get_function(), "_torchdynamo_disable", False + ): + msg = inspect.getattr_static( + func.get_function(), "_torchdynamo_disable_msg", None + ) + unimplemented( + gb_type="Skip inlining `torch.compiler.disable()`d function", + context=str(func.get_function()), + explanation=f"Skip inlining function {func.get_function()} since it was wrapped " + f"with `torch.compiler.disable` (reason: {msg})", + hints=[ + "Remove the `torch.compiler.disable` call", + ], + ) + + result = trace_rules.check_verbose(func, is_inlined_call=True) + if result.skipped: + from torch._dynamo.variables.misc import produce_trampoline_autograd_apply + + # _origin marks this as coming from an internal dynamo known function that is safe to + # trace through. + if ( + hasattr(getattr(func, "fn", None), "_origin") + # pyrefly: ignore [missing-attribute] + and func.fn._origin is produce_trampoline_autograd_apply + ): + # Known sound + return trace_rules.SkipResult( + False, "allowlist in dynamo known function" + ) + fn_qualname = func.fn.__qualname__ if hasattr(func, "fn") else "" + hints = [ + f"Avoid calling the function `{fn_qualname}`.", + ] + if "_dynamo" not in func.get_filename(): + hints += [ + f"Apply `@torch._dynamo.dont_skip_tracing` to the function `{fn_qualname}` " + "to force tracing into the function. " + "More graph breaks may occur as a result of attempting to trace into the function.", + "Please file an issue to PyTorch.", + ] + unimplemented( + gb_type="Attempted to inline function marked as skipped", + context=f"qualname: {fn_qualname}, name: {func.get_name()}, " + f"filename: `{func.get_filename()}`, skip reason: {result.reason}", + explanation=f"Dynamo developers have intentionally marked that the function `{fn_qualname}` " + "should not be traced.", + hints=hints, + ) + + return result + + @staticmethod + def build_inline_tracer( + parent: Any, + func: VariableTracker, + args: list[VariableTracker], + kwargs: Any, + ) -> InliningInstructionTranslator: + assert isinstance( + func, + ( + UserFunctionVariable, + NestedUserFunctionVariable, + LocalGeneratorFunctionVariable, + LocalGeneratorObjectVariable, + ), + ) + code: types.CodeType = func.get_code() + result = None + tracing_ctx = parent.output.tracing_context + + # Check if we have already identified this function to be inline-able. + # The exception is dont_skip_tracing flag which affects the inline + # behavior. If the flag is True, don't rely on previous results. + if not config.dont_skip_tracing and tracing_ctx: + if previous_result := tracing_ctx.previously_inlined_functions.get( + code, None + ): + result = previous_result + + if result is None: + if isinstance(func, SkipFunctionVariable): + unimplemented( + gb_type="Attempted to inline function marked as skipped (SkipFunctionVariable)", + context=f"Attempted to inline a SkipFunctionVariable {func}", + explanation=( + "Attempted to inline a function that was previously determined to be marked as intentionally skipped." + ), + hints=[], + ) + result = InliningInstructionTranslator.check_inlineable(func) + assert result.skipped is False + + if not config.dont_skip_tracing and tracing_ctx: + tracing_ctx.previously_inlined_functions[code] = result + + try: + # pyrefly: ignore [missing-attribute] + sub_locals = func.bind_args(parent, args, kwargs) + except TypeError as e: + # Wrap the general TypeError during bind_args() to the internal ArgsMismatchError with detailed info + raise ArgsMismatchError( # noqa: B904 + "{reason}.\n func = {func}, args = {args}, kwargs = {kwargs}".format( + reason=str(e), + # pyrefly: ignore [missing-attribute] + func=f"'{func.get_name()}' {func.get_filename()}:{func.get_code().co_firstlineno}", + args=[arg.python_type() for arg in args], + kwargs=kwargs, + ), + ) + + for v in itertools.chain(sub_locals.values()): + if not isinstance(v, VariableTracker): + unimplemented( + gb_type="Encountered unconverted argument when attempting to inline", + context=f"func: {func}, arg: {v}", + explanation="An argument to an inlined function was not successfully converted to a VariableTracker.", + hints=[*graph_break_hints.DYNAMO_BUG], + ) + + if code.co_name in ("__setitem__", "__setattr__") and not ( + args and isinstance(args[0], variables.UserDefinedObjectVariable) + ): + unimplemented( + gb_type="Unsupported __setitem__/__setattr__ inline attempt", + context=f"code name: {code.co_name}, args: {args}", + explanation=f"Attempted to inline {code.co_name} where first argument (self) is not a user-defined object.", + hints=[], + ) + + suffix = "" + # TODO: mlazos, add support for enabling multiple artifact logs + # with a single alias + if torch._logging._internal.log_state.is_artifact_enabled("bytecode"): + suffix = f"\n{dis.Bytecode(code).dis()}" + if sys.version_info >= (3, 11): + cur_inst = parent.current_instruction + parent_code = parent.f_code + + def get_trace_call_log_str() -> str: + header = parent.get_line_of_code_header( + lineno=cur_inst.positions.lineno + ) + line = get_instruction_source_311(parent_code, cur_inst).rstrip() + return f"TRACE inlined call {code.co_name} from {header}\n{line}" + + trace_call_log.debug("%s", LazyString(get_trace_call_log_str)) + log.debug("INLINING %s%s, %s", code, suffix, result.reason) + + # Detect inline GraphModule calls in order to propagate node metadata, + # by checking if the first argument (self) is a variable tracking a GraphModule. + if args and isinstance(args[0], NNModuleVariable): + module = parent.output.get_submodule(args[0].module_key) + if isinstance(module, torch.fx.GraphModule): + # The inline call might not actually be a call to `forward`, + # but it is enough to add a context for `forward` in case it is called. + code_context.get_context(module.forward.__code__)[ + "orig_graphmodule" + ] = weakref.ref(module) + # When we have inline_nn_module turned on, modules resolve to UnspecializedNNModuleVariable + if args and isinstance(args[0], UnspecializedNNModuleVariable): + module = args[0].value + if isinstance(module, torch.fx.GraphModule): + # The inline call might not actually be a call to `forward`, + # but it is enough to add a context for `forward` in case it is called. + code_context.get_context(module.forward.__code__)[ + "orig_graphmodule" + ] = weakref.ref(module) + + assert not isinstance(func, SkipFunctionVariable) + tracer: InliningInstructionTranslator + if is_generator(code): + tracer = InliningGeneratorInstructionTranslator( + parent, + code, + sub_locals, + parent.symbolic_globals, + parent.symbolic_torch_function_state, + parent.symbolic_stream_state, + func, + ) + else: + tracer = InliningInstructionTranslator( + parent, + code, + sub_locals, + parent.symbolic_globals, + parent.symbolic_torch_function_state, + parent.symbolic_stream_state, + func, + ) + return tracer + + def inline_call_(self) -> VariableTracker: + parent = self.parent + code = self.f_code + + strict_ctx: Any = contextlib.nullcontext() + if parent.strict_checks_fn: + strict_ctx = self.strict_translation_mode(parent.strict_checks_fn) + try: + with strict_ctx: + self.run() + except exc.ObservedException as e: + msg = f"Observed exception DURING INLING {code} : {e}" + log.debug(msg) + # bubble up the exception to the parent frame. + raise + except exc.SkipFrame as e: + msg = f"SKIPPED INLINING {code}: {e}" + log.debug(msg) + raise Unsupported(msg) from e + except Exception: + log.debug("FAILED INLINING %s", code) + raise + finally: + parent.error_on_graph_break = self.error_on_graph_break + + if self.output.should_exit: + # graph break + return ConstantVariable.create(None) # return dummy variable + + assert self.symbolic_result is not None + + if self.f_globals is parent.f_globals: + # Merge symbolic_globals back if parent and child are in the same namespace + parent.symbolic_globals.update(self.symbolic_globals) + + parent.inconsistent_side_effects |= self.inconsistent_side_effects + + log.debug("DONE INLINING %s", code) + self.output.tracing_context.traced_code.append(code) + + if config.enable_faithful_generator_behavior or ( + isinstance(self, InliningGeneratorInstructionTranslator) + and self.is_generator_from_ctx_manager + ): + if ( + is_generator(code) + and isinstance(self, InliningGeneratorInstructionTranslator) + and self.generator_exhausted + ): + assert isinstance(self, InliningGeneratorInstructionTranslator) + # When the generator returns None, we raise StopIteration + args = [] + if not self.symbolic_result.is_constant_none(): + args = [self.symbolic_result] + exc.raise_observed_exception(StopIteration, self, args=args) + else: + return self.symbolic_result + else: + if is_generator(code): + assert isinstance(self, InliningGeneratorInstructionTranslator) + assert self.symbolic_result.is_constant_none() + return ListIteratorVariable( + self.generated_items, + mutation_type=ValueMutationNew(), + ) + else: + return self.symbolic_result + + def __init__( + self, + parent: InstructionTranslatorBase, + code: types.CodeType, + symbolic_locals: dict[str, VariableTracker], + symbolic_globals: dict[str, VariableTracker], + symbolic_torch_function_state: SymbolicTorchFunctionState, + symbolic_stream_state: SymbolicStreamState, + funcvar: BaseUserFunctionVariable | LocalGeneratorObjectVariable, + ) -> None: + f_globals = funcvar.get_globals() + f_builtins = f_globals["__builtins__"] + if not isinstance(f_builtins, dict): + f_builtins = f_builtins.__dict__ + + # Get the cached instructions. These instructions are safe to cache + # because we dont mutate them in transform_code_object (those + # instructions are for the top most Instruction translator). Also, we + # have to be careful about not using _cached_cleaned_instructions here + # because that function is global, while we want the cache to be + # alive only during a compmilation. + tracing_ctx = parent.output.tracing_context + instructions = None + if tracing_ctx: + if tracing_ctx.previously_cleaned_instructions.get(code): + instructions = tracing_ctx.previously_cleaned_instructions[code] + + if instructions is None: + instructions = cleaned_instructions(code) + propagate_line_nums(instructions) + if tracing_ctx: + tracing_ctx.previously_cleaned_instructions[code] = instructions + + super().__init__( + output=parent.output, + f_locals={}, + f_globals=f_globals, + f_builtins=f_builtins, + symbolic_locals=symbolic_locals, + symbolic_globals=symbolic_globals, + symbolic_torch_function_state=symbolic_torch_function_state, + symbolic_stream_state=symbolic_stream_state, + instructions=instructions, + code_options={k: getattr(code, k) for k in get_code_keys()}, + f_code=code, + export=parent.export, + inline_depth=parent.inline_depth + 1, + speculation_log=parent.speculation_log, + exn_vt_stack=parent.exn_vt_stack, + distributed_state=parent.distributed_state, + package=parent.package, + ) + self.funcvar = funcvar + self.parent = parent + self.num_calls = parent.num_calls + self.symbolic_result = None + self.nn_module_stack = parent.nn_module_stack.copy() + self.one_graph = parent.one_graph + + @property + def fake_mode(self) -> Optional[FakeTensorMode]: + return self.parent.fake_mode + + def run_ctx_mgr(self) -> Any: + return TracingContext.current_frame(self.parent.frame_summary()) + + def should_compile_partial_graph(self) -> bool: + if config.nested_graph_breaks: + if not self.funcvar.should_allow_nested_graph_breaks(): + return False + if not self.parent.should_compile_partial_graph(): + return False + return super().should_compile_partial_graph() + return False # inlining functions is all-or-nothing + + def create_call_resume_at( + self, + inst: Instruction, + all_stack_locals_metadata: list[StackLocalsMetadata], + ) -> list[Instruction]: + if config.nested_graph_breaks: + return super().create_call_resume_at(inst, all_stack_locals_metadata) + unimplemented( + gb_type="Graph break in inlined function", + context="", + explanation="Graph breaks in an inlined call are not supported.", + hints=[], + ) + + def RETURN_VALUE(self, inst: Instruction) -> None: + self.symbolic_result = self.pop() # type: ignore[assignment] + self.instruction_pointer = None + raise ReturnValueOp + + def RETURN_CONST(self, inst: Instruction) -> None: + self.symbolic_result = self._load_const(inst) + self.instruction_pointer = None + raise ReturnValueOp + + def get_globals_source_and_value( + self, name: str + ) -> tuple[Any, VariableTracker, Source]: + # NamedTuple's `__new__` has a fake global scope that's not an actual + # module. TODO generalize the check for other non-importable cases. + # https://github.com/python/cpython/blob/8421b03b16a4852a527256cb7cdce2ab2d318548/Lib/collections/__init__.py#L441-L447 + if "__name__" in self.f_globals and not self.f_globals["__name__"].startswith( + "namedtuple_" + ): + module_name = self.f_globals["__name__"] + module_source = self.import_source(module_name) + if "torch_package" in module_name: + fglobals_value = ( + torch.package.package_importer._package_imported_modules[ + module_name + ] + ) # type: ignore[assignment] + else: + fglobals_value = _import_module(module_name) + # Dont use lazy vt because we will do a setattr afterwards + fglobals_vt = VariableBuilder(self, module_source)(fglobals_value) + global_source = AttrSource(module_source, name) + else: + globals_name = self.output.install_global_by_id( + "___unnamed_scope", self.f_globals + ) + globals_source = GlobalSource(globals_name) + fglobals_value = self.f_globals # type: ignore[assignment] + # Dont use lazy vt because we will do a setattr afterwards + fglobals_vt = VariableBuilder(self, globals_source)(fglobals_value) + global_source = DictGetItemSource(globals_source, name) # type: ignore[assignment] + + if is_stdlib(fglobals_value): + # Users don't inplace mutate a stdlib attribute (like inspect, + # collections), skip guards that originate from the stdlib modules. + global_source = SkipGuardSource(global_source) # type: ignore[assignment] + + return fglobals_value, fglobals_vt, global_source + + def _load_global(self, inst: Instruction) -> None: + name = inst.argval + if name not in self.f_globals: + return self.load_builtin(inst) + + if self.output.global_scope is self.f_globals: + # If the global scope matches that of the root frame, use handler in + # root frame instruction translator, to enforce consistency. + super()._load_global(inst) + else: + _, fglobals_vt, global_source = self.get_globals_source_and_value(name) + if self.output.side_effects.has_pending_mutation_of_attr(fglobals_vt, name): + self.push(self.output.side_effects.load_attr(fglobals_vt, name)) + else: + value = self.f_globals[name] + self.push(VariableTracker.build(self, value, global_source)) + + def STORE_GLOBAL(self, inst: Instruction) -> None: + if self.output.global_scope is self.f_globals: + # If the global scope matches that of the root frame, use handler in + # root frame instruction translator, to enforce consistency. + super().STORE_GLOBAL(inst) + else: + value = self.pop() + if isinstance(value, RemovableHandleVariable): + unimplemented( + gb_type="Storing Tensor hook handle in globals (inline call)", + context=inst.argval, + explanation="This is not supported.", + hints=[], + ) + name = inst.argval + _fglobals_value, fglobals_vt, _ = self.get_globals_source_and_value(name) + self.output.side_effects.store_attr(fglobals_vt, name, value) + + +class InliningGeneratorInstructionTranslator(InliningInstructionTranslator): + generated_items: list[VariableTracker] + # Flag whether or not the InlineGenerator should consume the entire iterator + + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.generated_items = [] + self.generator_exhausted = False + self.is_generator_from_ctx_manager = False + + def should_compile_partial_graph(self) -> bool: + # resuming on graph break on inlined generator not supported + return False + + def YIELD_VALUE(self, inst: Instruction) -> None: + top = self.pop() + self.generated_items.append(top) + if len(self.generated_items) > MAX_ITERATOR_LIMIT: + raise exc.InfiniteGeneratorError( + "Too many yield values in generator. Maybe you are inlining an infinite generator. " + f"If not, please report a bug at {PT2_ISSUE_TRACKER_URL}", + ) + self.push(ConstantVariable.create(None)) + if ( + config.enable_faithful_generator_behavior + or self.is_generator_from_ctx_manager + ): + self.symbolic_result = top + # Stop tracing + raise YieldValueOp + + def GET_YIELD_FROM_ITER(self, inst: Instruction) -> None: + tos = self.stack[-1] + if not isinstance(tos, ListIteratorVariable): + self.pop() + res = BuiltinVariable(iter).call_function(self, [tos], {}) # type: ignore[arg-type] + self.push(res) + + def RETURN_VALUE(self, inst: Instruction) -> None: + self.generator_exhausted = True + return super().RETURN_VALUE(inst) + + def RETURN_CONST(self, inst: Instruction) -> None: + self.generator_exhausted = True + return super().RETURN_CONST(inst) + + def YIELD_FROM(self, inst: Instruction) -> None: + assert len(self.stack) >= 2 + val = self.pop() + tos = self.stack[-1] + if not val.is_constant_none(): + # invoke send + # Unreachable code - if you hit this, you are implementing generator support and have + # lifted the `unimplemented("generator")` in frame conversion. This codepath handles + # subgenerator and lines up with this line in Python 3.10 + # https://github.com/python/cpython/blob/3.10/Python/ceval.c#L2599 + unimplemented( + gb_type="Unreachable sub-generator code", + context="", + explanation="Should only be encountered while implementing generator support.", + hints=[], + ) + + try: + val = tos.next_variable(self) + except (StopIteration, exc.ObservedUserStopIteration) as ex: + if isinstance(ex, exc.ObservedUserStopIteration): + exc.handle_observed_exception(self) + + # The iterator is exhausted. Stop the loop and return. + self.pop() + self.push(ConstantVariable.create(ex.value)) + else: + # Repeat the YIELD_FROM instruction in the next eval loop + assert ( + isinstance(self.instruction_pointer, int) + and self.instruction_pointer > 0 + ) + self.instruction_pointer -= 1 + + self.push(val) + # Add the value to yield into generated_items and replace the top of the stack with None + self.YIELD_VALUE(inst) + + def SEND(self, inst: Instruction) -> None: + assert len(self.stack) >= 2 + val = self.pop() + tos = self.stack[-1] + if isinstance(tos, (IteratorVariable, LocalGeneratorObjectVariable)) or ( + isinstance(tos, UserDefinedObjectVariable) + and isinstance(tos.value, collections.abc.Iterator) + ): + if val.is_constant_none(): + try: + val = tos.next_variable(self) # type: ignore[arg-type] + except (StopIteration, exc.ObservedUserStopIteration) as ex: + # To implement SEND, we have to look at the implementation + # when the iterator returns StopIteration. This translates to this code + # 3.11: https://github.com/python/cpython/blob/3.11/Python/ceval.c#L2613-L2619 + # 3.12: https://github.com/python/cpython/blob/3.12/Python/bytecodes.c#L863-L866 + # The implementation is different in 3.11 and 3.12. In 3.12, we rely + # on END_SEND to clean up. In 3.11, SEND does the cleanup as well. + if sys.version_info < (3, 12): + self.pop() # Python 3.12 uses new opcode END_SEND + self.push(ConstantVariable.create(ex.value)) + self.jump(inst) + else: + self.push(val) + else: + # invoke send + # Unreachable code - if you hit this, you are implementing generator support and have + # lifted the `unimplemented("generator")` in frame conversion. This codepath handles + # subgenerator and lines up with this line in Python 3.11 + # https://github.com/python/cpython/blob/3.11/Python/ceval.c#L2597 + unimplemented( + gb_type="Unreachable sub-generator code", + context="", + explanation="Should only be encountered while implementing generator support.", + hints=[], + ) + else: + unimplemented( + gb_type="SEND with bad type", + context=f"TOS type: {typestr(tos)}", + explanation=f"Attempted to SEND with unsupported type {typestr(tos)}.", + hints=[], + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/tensor_version_op.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/tensor_version_op.py new file mode 100644 index 0000000000000000000000000000000000000000..8709c5618d8594422a7793c07130c2d5b284f313 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/tensor_version_op.py @@ -0,0 +1,70 @@ +"""This module implements tensor version operations for Dynamo tracing. + +It provides primitives for handling tensor versioning during tracing, particularly in the +context of functionalization where version operations are handled eagerly on fake tensors. + +When we functionalize _tensor_version + _unsafe_set_version_counter, the ops disappear from +the traced graph. We run them eagerly on the fake tensors used for tracing, in order to get +past asserts that would fail in autograd. + +Why is this ok? +1) Versions on functional tensors do not make any sense since you cannot mutate a functional + tensor. +2) The whole point of version munging is to trick autograd into doing what we want, and after + AotAutograd there is no longer any need for these ops. + +Note this is similar to how no_grad is handled. +""" + +from contextlib import AbstractContextManager +from typing import Any + +import torch +from torch import SymInt +from torch._prims import _make_prim, RETURN_TYPE +from torch._subclasses import FakeTensorMode +from torch._subclasses.functional_tensor import FunctionalTensorMode + + +_tensor_version = _make_prim( + schema="_tensor_version(Tensor self) -> SymInt", + return_type=RETURN_TYPE.NEW, + meta=torch.ops.aten._version.default, + impl_aten=torch.ops.aten._version.default, + doc="Tracable unbacked SymInt version of torch.Tensor._version", +) + + +@_tensor_version.py_impl(FakeTensorMode) # type: ignore[misc] +def _tensor_version_fake(fake_mode: FakeTensorMode, self_tensor: Any) -> SymInt: + """ + The initial dynamo capture of _tensor_version + _unsafe_set_version_counter turns the + `._version` into an unbacked SymInt so that we don't need to specialize on the `._version` + of input tensors to the graph. + """ + assert fake_mode.shape_env is not None + return fake_mode.shape_env.create_unbacked_symint() + + +_unsafe_set_version_counter = _make_prim( + schema="_unsafe_set_version_counter(Tensor[] tensors, SymInt[] versions) -> ()", + return_type=RETURN_TYPE.NEW, + meta=lambda self, version: None, + impl_aten=torch._C._autograd._unsafe_set_version_counter, + doc="Tracable+SymInt version of torch._C._autograd._unsafe_set_version_counter", +) +torch.fx.node.has_side_effect(_unsafe_set_version_counter) + + +@_tensor_version.py_impl(FunctionalTensorMode) # type: ignore[misc] +def _tensor_version_functional(mode: FunctionalTensorMode, self: Any) -> int: + return self._version + + +@_unsafe_set_version_counter.py_impl(FunctionalTensorMode) # type: ignore[misc] +def _unsafe_set_version_counter_functional( + ctx: AbstractContextManager[Any], + tensors: tuple[torch.Tensor, ...], + versions: tuple[int, ...], +) -> None: + torch._C._autograd._unsafe_set_version_counter(tensors, versions) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/test_case.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/test_case.py new file mode 100644 index 0000000000000000000000000000000000000000..ad2637b3b124bdd227b190d1c473959dac444434 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/test_case.py @@ -0,0 +1,242 @@ +"""Testing utilities for Dynamo, providing a specialized TestCase class and test running functionality. + +This module extends PyTorch's testing framework with Dynamo-specific testing capabilities. +It includes: +- A custom TestCase class that handles Dynamo-specific setup/teardown +- Test running utilities with dependency checking +- Automatic reset of Dynamo state between tests +- Proper handling of gradient mode state +""" + +import contextlib +import importlib +import inspect +import logging +import os +import re +import sys +import unittest +from collections.abc import Callable +from typing import Any, Union + +import torch +import torch.testing +from torch._dynamo import polyfills +from torch._logging._internal import trace_log +from torch.testing._internal.common_utils import ( # type: ignore[attr-defined] + IS_WINDOWS, + skipIfTorchDynamo, + TEST_WITH_CROSSREF, + TEST_WITH_TORCHDYNAMO, + TestCase as TorchTestCase, +) + +from . import config, reset, utils + + +log = logging.getLogger(__name__) + + +def run_tests(needs: Union[str, tuple[str, ...]] = ()) -> None: + from torch.testing._internal.common_utils import run_tests + + if TEST_WITH_TORCHDYNAMO or TEST_WITH_CROSSREF: + return # skip testing + + if ( + not torch.xpu.is_available() + and IS_WINDOWS + and os.environ.get("TORCHINDUCTOR_WINDOWS_TESTS", "0") == "0" + ): + return + + if isinstance(needs, str): + needs = (needs,) + for need in needs: + if need == "cuda": + if not torch.cuda.is_available(): + return + else: + try: + importlib.import_module(need) + except ImportError: + return + run_tests() + + +class TestCase(TorchTestCase): + _exit_stack: contextlib.ExitStack + + @classmethod + def tearDownClass(cls) -> None: + cls._exit_stack.close() + super().tearDownClass() + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls._exit_stack = contextlib.ExitStack() # type: ignore[attr-defined] + cls._exit_stack.enter_context( # type: ignore[attr-defined] + config.patch( + raise_on_ctx_manager_usage=True, + suppress_errors=False, + log_compilation_metrics=False, + ), + ) + + def setUp(self) -> None: + self._prior_is_grad_enabled = torch.is_grad_enabled() + super().setUp() + reset() + utils.counters.clear() + self.handler = logging.NullHandler() + trace_log.addHandler(self.handler) + + def tearDown(self) -> None: + trace_log.removeHandler(self.handler) + for k, v in utils.counters.items(): + print(k, v.most_common()) + reset() + utils.counters.clear() + super().tearDown() + if self._prior_is_grad_enabled is not torch.is_grad_enabled(): + log.warning("Running test changed grad mode") + torch.set_grad_enabled(self._prior_is_grad_enabled) + + def assertEqual(self, x: Any, y: Any, *args: Any, **kwargs: Any) -> None: # type: ignore[override] + if ( + config.debug_disable_compile_counter + and isinstance(x, utils.CompileCounterInt) + or isinstance(y, utils.CompileCounterInt) + ): + return + return super().assertEqual(x, y, *args, **kwargs) + + # assertExpectedInline might also need to be disabled for wrapped nested + # graph break tests + + +# NB: multiple inheritance with LoggingTestCase is possible - this should be fine +# since there is no overlap in overridden methods. +class TestCaseWithNestedGraphBreaks(TestCase): + def setUp(self) -> None: + super().setUp() + self.prev_nested_graph_breaks = torch._dynamo.config.nested_graph_breaks + # pyrefly: ignore [bad-assignment] + torch._dynamo.config.nested_graph_breaks = True + + def tearDown(self) -> None: + super().tearDown() + # pyrefly: ignore [bad-assignment] + torch._dynamo.config.nested_graph_breaks = self.prev_nested_graph_breaks + + +@skipIfTorchDynamo("Not a suitable dynamo wrapped test") +class CPythonTestCase(TestCase): + """ + Test class for CPython tests located in "test/dynamo/CPython/Py_version/*". + + This class enables specific features that are disabled by default, such as + tracing through unittest methods. + """ + + _stack: contextlib.ExitStack + dynamo_strict_nopython = True + + # Restore original unittest methods to simplify tracing CPython test cases. + assertEqual = unittest.TestCase.assertEqual # type: ignore[assignment] + assertNotEqual = unittest.TestCase.assertNotEqual # type: ignore[assignment] + assertTrue = unittest.TestCase.assertTrue + assertFalse = unittest.TestCase.assertFalse + assertIs = unittest.TestCase.assertIs + assertIsNot = unittest.TestCase.assertIsNot + assertIsNone = unittest.TestCase.assertIsNone + assertIsNotNone = unittest.TestCase.assertIsNotNone + assertIn = unittest.TestCase.assertIn + assertNotIn = unittest.TestCase.assertNotIn + assertIsInstance = unittest.TestCase.assertIsInstance + assertNotIsInstance = unittest.TestCase.assertNotIsInstance + assertAlmostEqual = unittest.TestCase.assertAlmostEqual + assertNotAlmostEqual = unittest.TestCase.assertNotAlmostEqual + assertGreater = unittest.TestCase.assertGreater + assertGreaterEqual = unittest.TestCase.assertGreaterEqual + assertLess = unittest.TestCase.assertLess + assertLessEqual = unittest.TestCase.assertLessEqual + assertRegex = unittest.TestCase.assertRegex + assertNotRegex = unittest.TestCase.assertNotRegex + assertCountEqual = unittest.TestCase.assertCountEqual + assertMultiLineEqual = polyfills.assert_multi_line_equal + assertSequenceEqual = polyfills.assert_sequence_equal + assertListEqual = unittest.TestCase.assertListEqual + assertTupleEqual = unittest.TestCase.assertTupleEqual + assertSetEqual = unittest.TestCase.assertSetEqual + assertDictEqual = polyfills.assert_dict_equal + # pyrefly: ignore [bad-override] + assertRaises = unittest.TestCase.assertRaises + # pyrefly: ignore [bad-override] + assertRaisesRegex = unittest.TestCase.assertRaisesRegex + assertWarns = unittest.TestCase.assertWarns + assertWarnsRegex = unittest.TestCase.assertWarnsRegex + assertLogs = unittest.TestCase.assertLogs + fail = unittest.TestCase.fail + failureException = unittest.TestCase.failureException + + def compile_fn( + self, + fn: Callable[..., Any], + backend: Union[str, Callable[..., Any]], + nopython: bool, + ) -> Callable[..., Any]: + # We want to compile only the test function, excluding any setup code + # from unittest + + method = getattr(self, self._testMethodName) + method = torch._dynamo.optimize(backend, error_on_graph_break=nopython)(method) + + setattr(self, self._testMethodName, method) + return fn + + def _dynamo_test_key(self) -> str: + suffix = super()._dynamo_test_key() + test_cls = self.__class__ + test_file = inspect.getfile(test_cls).split(os.sep)[-1].split(".")[0] + py_ver = re.search(r"/([\d_]+)/", inspect.getfile(test_cls)) + if py_ver: + py_ver = py_ver.group().strip(os.sep).replace("_", "") # type: ignore[assignment] + else: + return suffix + return f"CPython{py_ver}-{test_file}-{suffix}" + + @classmethod + def tearDownClass(cls) -> None: + cls._stack.close() + super().tearDownClass() + + @classmethod + def setUpClass(cls) -> None: + # Skip test if python versions doesn't match + prefix = os.path.join("dynamo", "cpython") + os.path.sep + regex = re.escape(prefix) + r"\d_\d{2}" + search_path = inspect.getfile(cls) + m = re.search(regex, search_path) + if m: + test_py_ver = tuple(map(int, m.group().removeprefix(prefix).split("_"))) + py_ver = sys.version_info[:2] + if py_ver != test_py_ver: + expected = ".".join(map(str, test_py_ver)) + got = ".".join(map(str, py_ver)) + raise unittest.SkipTest( + f"Test requires Python {expected} but got Python {got}" + ) + else: + raise unittest.SkipTest( + f"Test requires a specific Python version but not found in path {inspect.getfile(cls)}" + ) + + super().setUpClass() + cls._stack = contextlib.ExitStack() # type: ignore[attr-defined] + cls._stack.enter_context( # type: ignore[attr-defined] + config.patch( + enable_trace_unittest=True, + ), + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/test_dont_skip_tracing_functions.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/test_dont_skip_tracing_functions.py new file mode 100644 index 0000000000000000000000000000000000000000..1edce5ff857fb2c0e35f4ac5debc42291a9d073e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/test_dont_skip_tracing_functions.py @@ -0,0 +1,40 @@ +""" +Functions used to test torch._dynamo.dont_skip_tracing. +This file is located in torch/_dynamo so that it is skipped by trace rules. +There is a special rule in trace_rules that doesn't skip this file when +dont_skip_tracing is active. +""" + +import torch + + +def f1(x: torch.Tensor) -> torch.Tensor: + return x + 1 + + +def f2(x: torch.Tensor) -> torch.Tensor: + return x + 1 + + +def f3(x: torch.Tensor) -> torch.Tensor: + return f2(x) + + +def f4(x: torch.Tensor) -> torch.Tensor: + x = f5(x, 1) + x = torch._dynamo.dont_skip_tracing(f6)(x) + x = f5(x, 8) + return x + + +def f5(x: torch.Tensor, n: int) -> torch.Tensor: + if torch.compiler.is_compiling(): + return x + n + return x + + +def f6(x: torch.Tensor) -> torch.Tensor: + x = f5(x, 2) + torch._dynamo.graph_break() + x = f5(x, 4) + return x diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/test_minifier_common.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/test_minifier_common.py new file mode 100644 index 0000000000000000000000000000000000000000..07c0c172342ef714eeee6fb7fca88c368d47f47c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/test_minifier_common.py @@ -0,0 +1,323 @@ +"""Common utilities for testing Dynamo's minifier functionality. + +This module provides the base infrastructure for running minification tests in Dynamo. +It includes: +- MinifierTestResult: A dataclass for storing and processing minifier test results +- MinifierTestBase: A base test class with utilities for: + - Running tests in isolated environments + - Managing temporary directories and configurations + - Executing minifier launcher scripts + - Running and validating reproduction scripts + - Supporting both compile-time and runtime error testing + +The minifier helps reduce failing Dynamo compilations to minimal reproductions. +""" + +import dataclasses +import io +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile +import traceback +from collections.abc import Sequence +from typing import Any, Optional, Union +from unittest.mock import patch + +import torch +import torch._dynamo +import torch._dynamo.test_case +from torch._dynamo.trace_rules import _as_posix_path +from torch.utils._traceback import report_compile_source_on_error + + +@dataclasses.dataclass +class MinifierTestResult: + minifier_code: str + repro_code: str + + def _get_module(self, t: str) -> str: + match = re.search(r"class Repro\(torch\.nn\.Module\):\s+([ ].*\n| *\n)+", t) + assert match is not None, "failed to find module" + r = match.group(0) + r = re.sub(r"\s+$", "\n", r, flags=re.MULTILINE) + r = re.sub(r"\n{3,}", "\n\n", r) + return r.strip() + + def get_exported_program_path(self) -> Optional[str]: + # Extract the exported program file path from AOTI minifier's repro.py + # Regular expression pattern to match the file path + pattern = r'torch\.export\.load\(\s*["\'](.*?)["\']\s*\)' + # Search for the pattern in the text + match = re.search(pattern, self.repro_code) + # Extract and print the file path if a match is found + if match: + file_path = match.group(1) + return file_path + return None + + def minifier_module(self) -> str: + return self._get_module(self.minifier_code) + + def repro_module(self) -> str: + return self._get_module(self.repro_code) + + +class MinifierTestBase(torch._dynamo.test_case.TestCase): + DEBUG_DIR = tempfile.mkdtemp() + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + if not os.path.exists(cls.DEBUG_DIR): + cls.DEBUG_DIR = tempfile.mkdtemp() + cls._exit_stack.enter_context( # type: ignore[attr-defined] + torch._dynamo.config.patch(debug_dir_root=cls.DEBUG_DIR) + ) + # These configurations make new process startup slower. Disable them + # for the minification tests to speed them up. + cls._exit_stack.enter_context( # type: ignore[attr-defined] + torch._inductor.config.patch( + { + # https://github.com/pytorch/pytorch/issues/100376 + "pattern_matcher": False, + # multiprocess compilation takes a long time to warmup + "compile_threads": 1, + # https://github.com/pytorch/pytorch/issues/100378 + "cpp.vec_isa_ok": False, + } + ) + ) + + @classmethod + def tearDownClass(cls) -> None: + if os.getenv("PYTORCH_KEEP_TMPDIR", "0") != "1": + shutil.rmtree(cls.DEBUG_DIR) + else: + print(f"test_minifier_common tmpdir kept at: {cls.DEBUG_DIR}") + cls._exit_stack.close() # type: ignore[attr-defined] + + def _gen_codegen_fn_patch_code(self, device: str, bug_type: str) -> str: + assert bug_type in ("compile_error", "runtime_error", "accuracy") + return f"""\ +{torch._dynamo.config.codegen_config()} +{torch._inductor.config.codegen_config()} +torch._inductor.config.{"cpp" if device == "cpu" else "triton"}.inject_relu_bug_TESTING_ONLY = {bug_type!r} +""" + + def _maybe_subprocess_run( + self, args: Sequence[Any], *, isolate: bool, cwd: Optional[str] = None + ) -> subprocess.CompletedProcess[bytes]: + from torch._inductor.cpp_builder import normalize_path_separator + + if not isolate: + assert len(args) >= 2, args + assert args[0] == "python3", args + if args[1] == "-c": + assert len(args) == 3, args + code = args[2] + args = ["-c"] + else: + assert len(args) >= 2, args + with open(args[1]) as f: + # Need normalize path of the code. + code = normalize_path_separator(f.read()) + args = args[1:] + + # WARNING: This is not a perfect simulation of running + # the program out of tree. We only interpose on things we KNOW we + # need to handle for tests. If you need more stuff, you will + # need to augment this appropriately. + + # NB: Can't use save_config because that will omit some fields, + # but we must save and reset ALL fields + dynamo_config = torch._dynamo.config.get_config_copy() + inductor_config = torch._inductor.config.get_config_copy() + try: + stderr = io.StringIO() + log_handler = logging.StreamHandler(stderr) + log = logging.getLogger("torch._dynamo") + log.addHandler(log_handler) + try: + prev_cwd = _as_posix_path(os.getcwd()) + if cwd is not None: + cwd = _as_posix_path(cwd) + os.chdir(cwd) + with patch("sys.argv", args), report_compile_source_on_error(): + exec(code, {"__name__": "__main__", "__compile_source__": code}) + rc = 0 + except Exception: + rc = 1 + traceback.print_exc(file=stderr) + finally: + log.removeHandler(log_handler) + if cwd is not None: + os.chdir(prev_cwd) # type: ignore[possibly-undefined] + # Make sure we don't leave buggy compiled frames lying + # around + torch._dynamo.reset() + finally: + torch._dynamo.config.load_config(dynamo_config) + torch._inductor.config.load_config(inductor_config) + + # TODO: return a more appropriate data structure here + return subprocess.CompletedProcess( + args, + rc, + b"", + stderr.getvalue().encode("utf-8"), + ) + else: + if cwd is not None: + cwd = _as_posix_path(cwd) + return subprocess.run(args, capture_output=True, cwd=cwd, check=False) + + # Run `code` in a separate python process. + # Returns the completed process state and the directory containing the + # minifier launcher script, if `code` outputted it. + def _run_test_code( + self, code: str, *, isolate: bool + ) -> tuple[subprocess.CompletedProcess[bytes], Union[str, Any]]: + proc = self._maybe_subprocess_run( + ["python3", "-c", code], isolate=isolate, cwd=self.DEBUG_DIR + ) + + print("test stdout:", proc.stdout.decode("utf-8")) + print("test stderr:", proc.stderr.decode("utf-8")) + repro_dir_match = re.search( + r"(\S+)minifier_launcher.py", proc.stderr.decode("utf-8") + ) + if repro_dir_match is not None: + return proc, repro_dir_match.group(1) + return proc, None + + # Runs the minifier launcher script in `repro_dir` + def _run_minifier_launcher( + self, + repro_dir: str, + isolate: bool, + *, + minifier_args: Sequence[Any] = (), + repro_after: Optional[str] = None, + ) -> tuple[subprocess.CompletedProcess[bytes], str]: + self.assertIsNotNone(repro_dir) + launch_file = _as_posix_path(os.path.join(repro_dir, "minifier_launcher.py")) + with open(launch_file) as f: + launch_code = f.read() + + self.assertTrue(os.path.exists(launch_file)) + + args = ["python3", launch_file, "minify", *minifier_args] + if not isolate and repro_after != "aot_inductor": + # AOTI minifier doesn't have --no-isolate flag. + # Everything in AOTI minifier is in no-isolate mode. + args.append("--no-isolate") + launch_proc = self._maybe_subprocess_run(args, isolate=isolate, cwd=repro_dir) + print("minifier stdout:", launch_proc.stdout.decode("utf-8")) + stderr = launch_proc.stderr.decode("utf-8") + print("minifier stderr:", stderr) + + self.assertNotIn("Input graph did not fail the tester", stderr) + + return launch_proc, launch_code + + # Runs the repro script in `repro_dir` + def _run_repro( + self, repro_dir: str, *, isolate: bool = True + ) -> tuple[subprocess.CompletedProcess[bytes], str]: + self.assertIsNotNone(repro_dir) + repro_file = _as_posix_path(os.path.join(repro_dir, "repro.py")) + with open(repro_file) as f: + repro_code = f.read() + + self.assertTrue(os.path.exists(repro_file)) + + repro_proc = self._maybe_subprocess_run( + ["python3", repro_file], isolate=isolate, cwd=repro_dir + ) + print("repro stdout:", repro_proc.stdout.decode("utf-8")) + print("repro stderr:", repro_proc.stderr.decode("utf-8")) + return repro_proc, repro_code + + # Template for testing code. + # `run_code` is the code to run for the test case. + # `patch_code` is the code to be patched in every generated file; usually + # just use this to turn on bugs via the config + def _gen_test_code(self, run_code: str, repro_after: str, repro_level: int) -> str: + repro_after_line = "" + if repro_after == "aot_inductor": + repro_after_line = ( + "torch._inductor.config.aot_inductor.dump_aoti_minifier = True" + ) + elif repro_after: + repro_after_line = f"""\ +torch._dynamo.config.repro_after = "{repro_after}" + """ + return f"""\ +import torch +import torch._dynamo +import torch._inductor +{_as_posix_path(torch._dynamo.config.codegen_config())} +{_as_posix_path(torch._inductor.config.codegen_config())} +{repro_after_line} +torch._dynamo.config.repro_level = {repro_level} +torch._inductor.config.aot_inductor.repro_level = {repro_level} +torch._dynamo.config.debug_dir_root = "{_as_posix_path(self.DEBUG_DIR)}" +{run_code} +""" + + # Runs a full minifier test. + # Minifier tests generally consist of 3 stages: + # 1. Run the problematic code + # 2. Run the generated minifier launcher script + # 3. Run the generated repro script + # + # If possible, you should run the test with isolate=False; use + # isolate=True only if the bug you're testing would otherwise + # crash the process + def _run_full_test( + self, + run_code: str, + repro_after: str, + expected_error: Optional[str], + *, + isolate: bool, + minifier_args: Sequence[Any] = (), + ) -> Optional[MinifierTestResult]: + if isolate: + repro_level = 3 + elif expected_error is None or expected_error == "AccuracyError": + repro_level = 4 + else: + repro_level = 2 + test_code = self._gen_test_code(run_code, repro_after, repro_level) + print("running test", file=sys.stderr) + test_proc, repro_dir = self._run_test_code(test_code, isolate=isolate) + if expected_error is None: + # Just check that there was no error + self.assertEqual(test_proc.returncode, 0) + + self.assertIsNone(repro_dir) + return None + # NB: Intentionally do not test return code; we only care about + # actually generating the repro, we don't have to crash + + self.assertIn(expected_error, test_proc.stderr.decode("utf-8")) + + self.assertIsNotNone(repro_dir) + print("running minifier", file=sys.stderr) + _minifier_proc, minifier_code = self._run_minifier_launcher( + repro_dir, + isolate=isolate, + minifier_args=minifier_args, + repro_after=repro_after, + ) + print("running repro", file=sys.stderr) + repro_proc, repro_code = self._run_repro(repro_dir, isolate=isolate) + + self.assertIn(expected_error, repro_proc.stderr.decode("utf-8")) + self.assertNotEqual(repro_proc.returncode, 0) + return MinifierTestResult(minifier_code=minifier_code, repro_code=repro_code) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/testing.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..4d11cc0cf210168f7f47224bc80a12d53d51d0ad --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/testing.py @@ -0,0 +1,583 @@ +"""Testing utilities and infrastructure for Dynamo. + +This module provides a comprehensive set of testing utilities including: +- Test result collection and validation +- Graph manipulation and comparison tools +- Test case management and execution helpers +- Specialized test decorators for different Python versions and features +- RNG state management +- Compilation counting and monitoring +- Debug utilities for bytecode transformation + +The utilities in this module are used across Dynamo's test suite to ensure +consistent testing patterns and proper test isolation. +""" + +import contextlib +import dis +import functools +import logging +import os.path +import random +import re +import sys +import types +import unittest +from collections.abc import Callable, Sequence +from typing import Any, Optional, overload, TypeVar, Union +from typing_extensions import ParamSpec +from unittest.mock import patch + +import torch +from torch import fx +from torch._dynamo.backends.debugging import aot_eager +from torch._dynamo.output_graph import OutputGraph + +from . import config, eval_frame, optimize_assert, reset +from .bytecode_transformation import ( + create_instruction, + debug_checks, + is_generator, + transform_code_object, +) +from .guards import CheckFunctionManager, CompileId, GuardedCode +from .types import ConvertFrameReturn, DynamoFrameType, wrap_guarded_code +from .utils import CompileCounterInt, same + + +np: Optional[types.ModuleType] = None +try: + import numpy as np +except ModuleNotFoundError: + np = None + + +unsupported = eval_frame.unsupported +three = 3 + +log = logging.getLogger(__name__) + +_P = ParamSpec("_P") + + +def clone_me(x: Optional[torch.Tensor]) -> Optional[torch.Tensor]: + if x is None: + return None + return x.detach().clone().requires_grad_(x.requires_grad) + + +def remove_optimized_module_prefix(name: str) -> str: + return re.sub(r"^_orig_mod[.]", "", name) + + +def extract_graph_and_tracker(fn, *args, **kwargs): # type: ignore[no-untyped-def] + from torch._dynamo.symbolic_convert import InstructionTranslator + + gm = None + region_tracker = None + + def extract_graph_backend(_gm, *args, **kwargs): # type: ignore[no-untyped-def] + nonlocal gm + nonlocal region_tracker + gm = _gm + region_tracker = InstructionTranslator.current_tx().output.region_tracker + return _gm + + torch.compile(backend=extract_graph_backend, fullgraph=True)(fn)(*args, **kwargs) + return gm.graph, region_tracker # type: ignore[union-attr] + + +def extract_graph(fn, *args, **kwargs): # type: ignore[no-untyped-def] + backend = AotEagerAndRecordGraphs() + result = torch.compile(backend=backend)(fn)(*args, **kwargs) + return result, backend.graphs, backend.fw_graphs, backend.bw_graphs + + +def collect_results( + model: torch.nn.Module, prediction: Any, loss: Any, example_inputs: Any +) -> list[Any]: + results = [] + results.append(prediction) + results.append(loss) + # if isinstance(loss, torch.Tensor) and loss.item() > 1: + # log.warning( + # f"High loss value alert - {loss:.2f}. Can result in unstable gradients." + # ) + + grads = {} + params = {} + for name, param in model.named_parameters(): + if isinstance(model, eval_frame.OptimizedModule): + name = remove_optimized_module_prefix(name) + param_copy = param + grad = param.grad + # Treat None and zero grad as same + if param.grad is None: + grad = torch.zeros_like(param) + grads[name + ".grad"] = grad + params[name] = param_copy + results.append(grads) + results.append(params) + buffers = {} + for name, buffer in model.named_buffers(): + if isinstance(model, eval_frame.OptimizedModule): + name = remove_optimized_module_prefix(name) + buffers[name] = buffer + results.append(buffers) + for example in example_inputs: + if isinstance(example, (tuple, list)): + results.extend(inp.grad for inp in example if isinstance(inp, torch.Tensor)) + else: + if isinstance(example, torch.Tensor): + results.append(example.grad) + return results + + +def requires_bwd_pass(out: Any) -> bool: + if isinstance(out, torch.Tensor): + return out.requires_grad + elif isinstance(out, (list, tuple)): + return any(requires_bwd_pass(x) for x in out) + elif out is None: + return False + elif isinstance(out, int): + return False + raise NotImplementedError("Don't know how to reduce", type(out)) + + +@overload +def reduce_to_scalar_loss(out: torch.Tensor) -> torch.Tensor: ... + + +@overload +def reduce_to_scalar_loss( + out: Union[list[Any], tuple[Any, ...], dict[Any, Any]], +) -> float: ... + + +def reduce_to_scalar_loss(out: Any) -> Union[torch.Tensor, float]: + """Reduce the output of a model to get scalar loss""" + if isinstance(out, torch.Tensor): + # Mean does not work on integer tensors + return out.sum() / out.numel() + elif isinstance(out, (list, tuple)): + return sum(reduce_to_scalar_loss(x) for x in out) / len(out) + elif type(out).__name__ in ( + "MaskedLMOutput", + "Seq2SeqLMOutput", + "CausalLMOutputWithCrossAttentions", + ): + return reduce_to_scalar_loss(out.logits) + elif type(out).__name__ == "SquashedNormal": + return out.mean.sum() + elif isinstance(out, dict): + return sum(reduce_to_scalar_loss(value) for value in out.values()) / len( + out.keys() + ) + raise NotImplementedError("Don't know how to reduce", type(out)) + + +def debug_dir() -> str: + path = os.path.join(os.path.dirname(__file__), "../debug") + if not os.path.exists(path): + os.mkdir(path) + return path + + +def debug_dump(name: str, code: types.CodeType, extra: str = "") -> None: + with open(os.path.join(debug_dir(), name), "w") as fd: + fd.write( + f"{dis.Bytecode(code).info()}\n\n{dis.Bytecode(code).dis()}\n\n{extra}\n" + ) + + +def debug_insert_nops( + frame: DynamoFrameType, cache_size: int, hooks: Any, _: Any, *, skip: int = 0 +) -> ConvertFrameReturn: + """used to debug jump updates""" + + def insert_nops(instructions: list[Any], code_options: Any) -> None: + instructions.insert(0, create_instruction("NOP")) + instructions.insert(0, create_instruction("NOP")) + + metrics_context = torch._dynamo.utils.get_metrics_context() + with torch._dynamo.utils.dynamo_timed("debug_insert_nops"), metrics_context: + if is_generator(frame.f_code): + return ConvertFrameReturn() + + debug_checks(frame.f_code) + code, _ = transform_code_object(frame.f_code, insert_nops) + graph = OutputGraph( + code_options={}, + compiler_fn=None, + root_tx=None, # type: ignore[arg-type] + export=False, + export_constraints=[], + frame_state={"_id": 0}, + # TODO: shouldn't this be f_locals/f_globals from frame? + local_scope=locals(), + global_scope=globals(), + f_code=frame.f_code, + torch_function_mode_stack=[], + package=None, + ) + + return wrap_guarded_code( + GuardedCode( + code, + CheckFunctionManager(frame.f_code, graph).guard_manager, # type: ignore[arg-type] + CompileId(frame_id=0, frame_compile_id=0), + ) + ) + + +class CompileCounter: + def __init__(self) -> None: + self.frame_count: Union[int, CompileCounterInt] = 0 + self.clear() + + def __call__( + self, gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor] + ) -> Callable[..., Any]: + self.frame_count += 1 + for node in gm.graph.nodes: + if "call" in node.op: + self.op_count += 1 + return gm.forward + + def clear(self) -> None: + if config.debug_disable_compile_counter: + self.frame_count = CompileCounterInt(0) + else: + self.frame_count = 0 + self.op_count = 0 + + +class CompileCounterWithBackend: + def __init__(self, backend: str) -> None: + self.frame_count: Union[int, CompileCounterInt] = 0 + self.backend = backend + self.graphs: list[torch.fx.GraphModule] = [] + self.clear() + + def __call__( + self, gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor] + ) -> Callable[..., Any]: + from .backends.registry import lookup_backend + + self.frame_count += 1 + for node in gm.graph.nodes: + if "call" in node.op: + self.op_count += 1 + self.graphs.append(gm) + return lookup_backend(self.backend)(gm, example_inputs) + + def clear(self) -> None: + if config.debug_disable_compile_counter: + self.frame_count = CompileCounterInt(0) + else: + self.frame_count = 0 + self.op_count = 0 + self.graphs = [] + + +# Equivalent to backend="eager", but also records graphs that +# we can assert on +class EagerAndRecordGraphs: + def __init__(self) -> None: + self.graphs: list[torch.fx.GraphModule] = [] + + def __call__( + self, gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor] + ) -> Callable[..., Any]: + self.graphs.append(gm) + return gm.forward + + +class AotEagerAndRecordGraphs: + def __init__(self) -> None: + self.graphs: list[torch.fx.GraphModule] = [] + self.fw_graphs: list[torch.fx.GraphModule] = [] + self.bw_graphs: list[torch.fx.GraphModule] = [] + + def __call__( + self, gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor] + ) -> Callable[..., Any]: + self.graphs.append(gm) + + def fw_compiler( + gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor] + ) -> Callable[..., Any]: + self.fw_graphs.append(gm) + return gm.forward + + def bw_compiler( + gm: torch.fx.GraphModule, example_inputs: list[torch.Tensor] + ) -> Callable[..., Any]: + self.bw_graphs.append(gm) + return gm.forward + + return aot_eager( + gm, + example_inputs, + fw_compiler=fw_compiler, + bw_compiler=bw_compiler, + ) + + +class InductorAndRecordGraphs: + def __init__(self) -> None: + self.graphs: list[torch.fx.GraphModule] = [] + self.inductor_graphs: list[torch.fx.GraphModule] = [] + + def __call__(self, gm, example_inputs): # type: ignore[no-untyped-def] + import torch._inductor.compile_fx as compile_fx_mod + + self.graphs.append(gm) + + old_compile_fx_inner = compile_fx_mod._compile_fx_inner + + def patched(*args, **kwargs): # type: ignore[no-untyped-def] + self.inductor_graphs.append(args[0]) + return old_compile_fx_inner(*args, **kwargs) + + with patch.object(compile_fx_mod, "_compile_fx_inner", new=patched): + return compile_fx_mod.compile_fx(gm, example_inputs) + + +def strip_comment(code: str) -> str: + return re.sub(r"(?m)^ *#.*\n?", "", code) + + +def remove_trailing_space(code: str) -> str: + return "\n".join([line.rstrip() for line in code.split("\n")]) + + +def _squash_blank_lines(code: str) -> str: + lines = code.split("\n") + result: list[str] = [] + saw_blank = False + for line in lines: + if line.strip() == "": + if saw_blank: + continue + saw_blank = True + else: + saw_blank = False + result.append(line) + return "\n".join(result) + + +def normalize_gm(gm_str: str) -> str: + # strip comments as comments have path to files which may differ from + # system to system. + stripped = strip_comment(gm_str) + no_trailing = remove_trailing_space(stripped) + return _squash_blank_lines(no_trailing) + + +def empty_line_normalizer(code: str) -> str: + """ + Normalize code: remove empty lines. + """ + normal_code = re.sub(r"[\r\n]+", "\n", code) + return normal_code + + +def standard_test( + self: Any, + fn: Callable[..., Any], + nargs: int, + expected_ops: Optional[int] = None, + expected_ops_dynamic: Optional[int] = None, + expected_frame_count: int = 1, +) -> None: + if not config.assume_static_by_default and expected_ops_dynamic is not None: + expected_ops = expected_ops_dynamic + + actual = CompileCounter() + + args1 = [torch.randn(10, 10) for _ in range(nargs)] + args2 = [torch.randn(10, 10) for _ in range(nargs)] + correct1 = fn(*args1) + correct2 = fn(*args2) + reset() + opt_fn = optimize_assert(actual)(fn) + val1a = opt_fn(*args1) + val2a = opt_fn(*args2) + val1b = opt_fn(*args1) + val2b = opt_fn(*args2) + reset() + self.assertTrue(same(val1a, correct1)) + self.assertTrue(same(val1b, correct1)) + self.assertTrue(same(val2a, correct2)) + self.assertTrue(same(val2b, correct2)) + self.assertEqual(actual.frame_count, expected_frame_count) + if expected_ops is not None: + self.assertEqual(actual.op_count, expected_ops) + + +def dummy_fx_compile( + gm: fx.GraphModule, example_inputs: list[torch.Tensor] +) -> Callable[..., Any]: + return gm.forward + + +def format_speedup( + speedup: float, + pvalue: float, + is_correct: bool = True, + pvalue_threshold: float = 0.1, +) -> str: + if not is_correct: + return "ERROR" + if pvalue > pvalue_threshold: + return f"{speedup:.3f}x SAME" + return f"{speedup:.3f}x p={pvalue:.2f}" + + +def rand_strided( + size: Sequence[int], + stride: Sequence[int], + dtype: torch.dtype = torch.float32, + device: Union[str, torch.device] = "cpu", + extra_size: int = 0, +) -> torch.Tensor: + needed_size = extra_size + if all(s > 0 for s in size): + # only need to allocate if all sizes are non-zero + needed_size += ( + sum((shape - 1) * stride for shape, stride in zip(size, stride)) + 1 + ) + if dtype.is_floating_point: + if dtype.itemsize == 1: + """ + normal distribution kernel is not implemented for fp8.. + Workaround that by creating a fp16 tensor and then cast. + """ + buffer = torch.randn(needed_size, dtype=torch.float16, device=device).to( + dtype=dtype + ) + else: + buffer = torch.randn(needed_size, dtype=dtype, device=device) + else: + buffer = torch.zeros(size=[needed_size], dtype=dtype, device=device) + return torch.as_strided(buffer, size, stride) + + +_T = TypeVar("_T") + + +def check_dynamic_shape_capture() -> bool: + # This also mirrors config from `test/dynamo/test_dynamic_shapes.py:make_dynamic_cls` + return not config.assume_static_by_default + + +def _make_fn_with_patches(fn: Callable[_P, _T], *patches: Any) -> Callable[_P, _T]: + @functools.wraps(fn) + def _fn(*args: _P.args, **kwargs: _P.kwargs) -> _T: + with contextlib.ExitStack() as stack: + for module, attr, val in patches: + stack.enter_context(patch.object(module, attr, val)) + + return fn(*args, **kwargs) + + return _fn + + +def make_test_cls_with_patches( + cls: type, + cls_prefix: str, + fn_suffix: str, + *patches: Any, + xfail_prop: Optional[str] = None, + decorator: Callable[[Callable[..., Any]], Callable[..., Any]] = lambda x: x, +) -> type: + DummyTestClass = type(f"{cls_prefix}{cls.__name__}", cls.__bases__, {}) + DummyTestClass.__qualname__ = DummyTestClass.__name__ + + for name in dir(cls): + if name.startswith("test_"): + fn = getattr(cls, name) + if not callable(fn): + setattr(DummyTestClass, name, getattr(cls, name)) + continue + new_name = f"{name}{fn_suffix}" + new_fn = _make_fn_with_patches(fn, *patches) + new_fn.__name__ = new_name + if xfail_prop is not None and hasattr(fn, xfail_prop): + new_fn = unittest.expectedFailure(new_fn) + setattr(DummyTestClass, new_name, decorator(new_fn)) + # NB: Doesn't handle slots correctly, but whatever + elif not hasattr(DummyTestClass, name): + setattr(DummyTestClass, name, getattr(cls, name)) + + return DummyTestClass + + +# test Python 3.11+ specific features +def skipIfNotPy311(fn: Callable[_P, _T]) -> Callable[_P, _T]: + if sys.version_info >= (3, 11): + return fn + # pyrefly: ignore [bad-return, bad-argument-type] + return unittest.skip(fn) + + +def skipIfNotPy312(fn: Callable[_P, _T]) -> Callable[_P, _T]: + if sys.version_info >= (3, 12): + return fn + return unittest.skip("Requires Python 3.12+")(fn) + + +def skipIfOnlyNotPy312(fn: Callable[_P, _T]) -> Callable[_P, _T]: + if sys.version_info >= (3, 13) or sys.version_info < (3, 12): + return unittest.skip("Requires Python 3.12")(fn) + return fn + + +def xfailIfPy312(fn: Callable[_P, _T]) -> Callable[_P, _T]: + if sys.version_info >= (3, 12): + return unittest.expectedFailure(fn) + return fn + + +def skipIfPy312(fn: Callable[_P, _T]) -> Callable[_P, _T]: + if sys.version_info >= (3, 12): + return unittest.skip("Not supported in Python 3.12+")(fn) + return fn + + +# Controls tests generated in test/inductor/test_torchinductor_dynamic_shapes.py +# and test/dynamo/test_dynamic_shapes.py +def expectedFailureDynamic(fn: Callable[_P, _T]) -> Callable[_P, _T]: + fn._expected_failure_dynamic = True # type: ignore[attr-defined] + return fn + + +# Controls tests generated in test/inductor/test_torchinductor_codegen_dynamic_shapes.py +def expectedFailureCodegenDynamic(fn: Callable[_P, _T]) -> Callable[_P, _T]: + fn._expected_failure_codegen_dynamic = True # type: ignore[attr-defined] + return fn + + +# Controls test generated in test/inductor/test_cpp_wrapper.py +def expectedFailureDynamicWrapper(fn: Callable[_P, _T]) -> Callable[_P, _T]: + fn._expected_failure_dynamic_wrapper = True # type: ignore[attr-defined] + return fn + + +def reset_rng_state(use_xla: bool = False) -> None: + torch.manual_seed(1337) + random.seed(1337) + if np: + np.random.seed(1337) + if use_xla: + import torch_xla.core.xla_model as xm + + xm.set_rng_state(1337, str(xm.xla_device())) + + +def _skipped_function_for_test_reconstruct( + f: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs +) -> _T: + return f(*args, **kwargs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/trace_rules.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/trace_rules.py new file mode 100644 index 0000000000000000000000000000000000000000..7376ad5fe48ab5bba4a8ea7e0edeedfc9b3d447e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/trace_rules.py @@ -0,0 +1,4045 @@ +""" +Tracing rules and policies for TorchDynamo compilation decisions. + +This module defines the rules that govern what code TorchDynamo should trace and compile +versus what should be executed eagerly. It contains functions and classes that determine: + +- Which modules, functions, and objects should be skipped during tracing +- Which parts of the code should cause graph breaks +- How to handle different Python libraries and third-party packages +- Rules for determining when to inline functions vs calling them eagerly + +Key components: +- Skip rules: Functions that return True if an object should be skipped during tracing +- Inlining rules: Policies for when to inline function calls during compilation +- Library-specific handling: Special cases for popular Python packages +- Performance heuristics: Rules that balance compilation overhead vs runtime benefits + +These rules are critical for TorchDynamo's ability to automatically determine +compilation boundaries and optimize PyTorch programs effectively. +""" + +import abc +import builtins +import copy +import dataclasses +import functools +import importlib +import inspect +import linecache +import operator +import os +import random +import re +import sys +import traceback +import types +import unittest +from collections import defaultdict +from collections.abc import Callable +from pathlib import Path +from typing import Any, cast, Optional, Union + +import torch +import torch._inductor.test_operators +import torch.distributed +import torch.utils._content_store +from torch._environment import is_fbcode +from torch.utils import _config_module + +from . import config +from .resume_execution import TORCH_DYNAMO_RESUME_IN_PREFIX +from .utils import ( + getfile, + hashable, + is_lru_cache_wrapped_function, + NP_SUPPORTED_MODULES, + unwrap_if_wrapper, +) +from .variables import ( + BuiltinVariable, + FunctionalCallVariable, + FunctorchHigherOrderVariable, + LocalGeneratorFunctionVariable, + LocalGeneratorObjectVariable, + NestedUserFunctionVariable, + PolyfilledFunctionVariable, + PyTreeGetNodeTypeFunctionVariable, + PyTreeTreeIsLeafFunctionVariable, + ReparametrizeModuleCallVariable, + SkipFunctionVariable, + TorchInGraphFunctionVariable, + UserFunctionVariable, + UserMethodVariable, +) +from .variables.base import VariableTracker + + +np: Optional[types.ModuleType] = None +try: + import numpy as np +except ModuleNotFoundError: + pass + + +""" +A note on skip/inline rules: + +Dynamo consults this file to determine whether function should be inlined or skipped. + +A skip applies at the frame boundary, meaning dynamo either triggers a graph break +at the beginning of the frame or attempts to trace/inline the whole frame. When skipping +a frame, recursively called frames are still traced by dynamo unless also skipped. + +Skipfiles (skipped at the file level instead of function level) still apply on a +frame-by-frame boundary as dynamo traces, but apply to all functions in that file. + +@skip is a helper decorator that can be applied to your function to cause it to be +included here. + +Dynamo skip/inline rules & priorities are defined as follows: +* Inline is the default behavior and will be used unless explicitly skipped. +* Dynamo has two SKIPLIST: BUILTIN_SKIPLIST and THIRDPARTY_SKIPLIST. + * BUILTIN_SKIPLIST contains builtin python modules, such as abc, collections, etc. + * THIRDPARTY_SKIPLIST contains common third party libraries, such as numpy, pandas, etc. +* Functions in these two SKIPLISTs are always skipped, except: + * They have explicitly defined rule in `manual_torch_name_rule_map`; + * The corresponding python module has been put into MOD_INLINELIST. +* PyTorch(torch) is in the BUILTIN_SKIPLIST by default, but there are many cases + where we want inline the functions under torch namespace. + We should specify inline for the functions in `manual_torch_name_rule_map` or + put the corresponding python module into MOD_INLINELIST to make dynamo inline them. +* If you call functions under skipped modules/files, Dynamo will wrap these functions + as SkipFunctionVariable. There are a few functions(e.g, collections.OrderedDict) that + we have special handling at SkipFunctionVariable.call_function. + +Overall: *_INLINELIST has precedence over *_SKIPLIST has precedence over DEFAULT (inline) + +To figure out what the behavior is, check the following list in order: +* `manual_torch_name_rule_map` (Inline if YES) +* MOD_INLINELIST (Inline if YES) +* BUILTIN_SKIPLIST & THIRDPARTY_SKIPLIST (Skip if YES) +* MOD_SKIPLIST (Skip if YES) +* Inline by default + +In general, if you want to force inline a function or module, please consider adding +the function's python module to MOD_INLINELIST first. +Use the `manual_torch_name_rule_map` only when there are other functions under the same module that +you don't want to inline them. +""" + +""" +Map of function objects to their tracing rules (Dynamo variables). +* TorchInGraphFunctionVariable: The functions should be put into the FX graph or can be constant folded. E.g., + - torch.add: should be put into the FX graph. + - torch.is_floating_point: constant folded. +* SkipFunctionVariable: The objects should be skipped from tracing. +* UserFunctionVariable: The functions should be inlined. + +For developers: If you add/remove a torch level API, it may trigger failures from +test/dynamo/test_trace_rules.py:test_torch_name_rule_map_updated. To fix the failures: +If you are adding a new torch level API or Dynamo implementation: +* Add the name with the corresponding tracing rule to this map + if you are adding a new in graph function or Dynamo implementation for an existing function. +* Remove the object name from test/dynamo/test_trace_rules.ignored_c_binding_in_graph_function_names if it's there. + +If you are removing an existing torch level API: +* Remove the entry represented the API from this map or test/dynamo/test_trace_rules.ignored_c_binding_in_graph_function_names + depends on where it is. + + +""" +manual_torch_name_rule_map: dict[ + str, + Union[ + type[TorchInGraphFunctionVariable], + type[SkipFunctionVariable], + type[UserFunctionVariable], + ], +] = { + "torch.onnx.is_in_onnx_export": TorchInGraphFunctionVariable, + "torch.onnx.operators.shape_as_tensor": TorchInGraphFunctionVariable, + "torch.overrides.is_tensor_like": TorchInGraphFunctionVariable, + "torch.jit.is_scripting": TorchInGraphFunctionVariable, + "torch.jit.is_tracing": TorchInGraphFunctionVariable, + "torch.jit.annotate": TorchInGraphFunctionVariable, + "torch.distributed.is_available": TorchInGraphFunctionVariable, + "torch.distributed.is_initialized": TorchInGraphFunctionVariable, + "torch.distributed.get_rank": TorchInGraphFunctionVariable, + "torch.distributed.get_world_size": TorchInGraphFunctionVariable, + "torch.distributed.tensor._api.DTensor#from_local": TorchInGraphFunctionVariable, + "torch.distributed.distributed_c10d._get_group_size_by_name": TorchInGraphFunctionVariable, + "torch.distributed.distributed_c10d._resolve_group_name_by_ranks_and_tag": TorchInGraphFunctionVariable, + "torch.distributed.distributed_c10d._get_group_tag": TorchInGraphFunctionVariable, + "torch.distributed.distributed_c10d.get_process_group_ranks": TorchInGraphFunctionVariable, + "torch._utils.is_compiling": TorchInGraphFunctionVariable, + "torch.fx._symbolic_trace.is_fx_tracing": TorchInGraphFunctionVariable, + "torch.fx._symbolic_trace.is_fx_symbolic_tracing": TorchInGraphFunctionVariable, + "torch._dynamo.external_utils.is_compiling": TorchInGraphFunctionVariable, + "torch._dynamo.utils._disable_side_effect_safety_checks_for_current_subtracer": UserFunctionVariable, + "torch.compiler.is_compiling": TorchInGraphFunctionVariable, + "torch.compiler.is_dynamo_compiling": TorchInGraphFunctionVariable, + "torch.compiler.is_exporting": TorchInGraphFunctionVariable, + "torch._dynamo.eval_frame._is_in_optimized_module": TorchInGraphFunctionVariable, + "torch._C._to_dlpack": SkipFunctionVariable, + "torch.to_dlpack": SkipFunctionVariable, + "torch._check": TorchInGraphFunctionVariable, + # We graph break on RNG state setters or getters like + # `torch.get_rng_state` or `torch.set_rng_state`. These functions + # are not aten operations and therefore they are completely ignored + # by the AOT dispatcher. As a result, the AOT graph does not have + # these setter or getter functions, producing an incorrect graph + # when it comes to rng states. + "torch.default_generator#get_state": SkipFunctionVariable, + "torch._C.Generator#get_state": SkipFunctionVariable, + "torch.get_rng_state": SkipFunctionVariable, + "torch.cuda.get_rng_state": SkipFunctionVariable, + "torch.default_generator#set_state": SkipFunctionVariable, + "torch._C.Generator#set_state": SkipFunctionVariable, + "torch.set_rng_state": SkipFunctionVariable, + "torch.cuda.set_rng_state": SkipFunctionVariable, + # https://github.com/pytorch/pytorch/issues/107187 + "torch.manual_seed": SkipFunctionVariable, + # https://github.com/pytorch/pytorch/issues/93501 + "torch.nn.utils.rnn.pack_padded_sequence": SkipFunctionVariable, + "torch.nn.Parameter": TorchInGraphFunctionVariable, + "torch.nn.Buffer": TorchInGraphFunctionVariable, + "torch._nested_tensor_from_mask": SkipFunctionVariable, + "torch.nested._internal.nested_tensor.nested_from_padded": TorchInGraphFunctionVariable, + "torch.nested.nested_tensor_from_jagged": UserFunctionVariable, + "torch.nested.nested_tensor_from_padded": UserFunctionVariable, + # torch.fx map utils + "torch.fx.node.map_aggregate": UserFunctionVariable, + "torch.fx.node.map_arg": UserFunctionVariable, + "torch.fx.immutable_collections._no_mutation": UserFunctionVariable, + "torch.fx.immutable_collections._immutable_list_flatten": UserFunctionVariable, + "torch.fx.immutable_collections._immutable_list_unflatten": UserFunctionVariable, + "torch.fx.immutable_collections._immutable_dict_flatten": UserFunctionVariable, + "torch.fx.immutable_collections._immutable_dict_unflatten": UserFunctionVariable, + # symbol operators implemented in Python + "torch.sym_not": TorchInGraphFunctionVariable, + "torch.sym_float": TorchInGraphFunctionVariable, + "torch.sym_int": TorchInGraphFunctionVariable, + "torch.sym_max": TorchInGraphFunctionVariable, + "torch.sym_min": TorchInGraphFunctionVariable, + "torch.sym_sqrt": TorchInGraphFunctionVariable, + "torch.sym_ite": TorchInGraphFunctionVariable, + "torch.sym_sum": TorchInGraphFunctionVariable, + "torch.sym_fresh_size": UserFunctionVariable, + "torch.Tensor#_make_wrapper_subclass": SkipFunctionVariable, + "torch.Tensor#__init__": SkipFunctionVariable, + "torch.Tensor#split": TorchInGraphFunctionVariable, + "torch.cuda.set_device": SkipFunctionVariable, + "torch.cuda.current_device": TorchInGraphFunctionVariable, + "torch._C.autocast_decrement_nesting": SkipFunctionVariable, + "torch._C.autocast_increment_nesting": SkipFunctionVariable, + "torch.autograd.grad": SkipFunctionVariable, + "torch.autograd.backward": SkipFunctionVariable, + "torch._C.clear_autocast_cache": SkipFunctionVariable, + "torch.distributions.constraints.is_dependent": SkipFunctionVariable, + "torch.jit.isinstance": SkipFunctionVariable, + "torch._C.set_anomaly_enabled": SkipFunctionVariable, + "torch._C.set_autocast_cache_enabled": SkipFunctionVariable, + "torch._C.set_autocast_cpu_dtype": SkipFunctionVariable, + "torch._C.set_autocast_cpu_enabled": SkipFunctionVariable, + "torch._C.set_autocast_enabled": SkipFunctionVariable, + "torch._C.set_autocast_gpu_dtype": SkipFunctionVariable, + "torch._C.set_autocast_ipu_dtype": SkipFunctionVariable, + "torch._C.set_autocast_ipu_enabled": SkipFunctionVariable, + "torch._C.set_autocast_xla_dtype": SkipFunctionVariable, + "torch._C.set_autocast_xla_enabled": SkipFunctionVariable, + "torch.resize_as_": SkipFunctionVariable, + "torch._functorch.predispatch._add_batch_dim": TorchInGraphFunctionVariable, + "torch._functorch.predispatch._remove_batch_dim": TorchInGraphFunctionVariable, + "torch.resize_as_sparse_": SkipFunctionVariable, + "torch.get_default_device": TorchInGraphFunctionVariable, + # functorch/vmap + "torch._functorch.vmap._check_int_or_none": UserFunctionVariable, + "torch._functorch.vmap._check_out_dims_is_int_or_int_pytree": UserFunctionVariable, + "torch._functorch.vmap._check_randomness_arg": UserFunctionVariable, + "torch._functorch.vmap._chunked_vmap": UserFunctionVariable, + "torch._functorch.vmap._concat_chunked_outputs": UserFunctionVariable, + "torch._functorch.vmap._create_batched_inputs": UserFunctionVariable, + "torch._functorch.vmap._flat_vmap": UserFunctionVariable, + "torch._functorch.vmap._flatten_chunks_output": UserFunctionVariable, + "torch._functorch.vmap._get_chunked_inputs": UserFunctionVariable, + "torch._functorch.vmap._get_name": UserFunctionVariable, + "torch._functorch.vmap._maybe_remove_batch_dim": UserFunctionVariable, + "torch._functorch.vmap._num_outputs": UserFunctionVariable, + "torch._functorch.vmap._process_batched_inputs": UserFunctionVariable, + "torch._functorch.vmap._unwrap_batched": UserFunctionVariable, + "torch._functorch.vmap._validate_and_get_batch_size": UserFunctionVariable, + "torch._functorch.vmap.doesnt_support_saved_tensors_hooks": UserFunctionVariable, + "torch._functorch.vmap.get_chunk_sizes": UserFunctionVariable, + # lazy_load_decompositions uses a lock that is not supported yet in dynamo + # "torch._functorch.vmap.lazy_load_decompositions": UserFunctionVariable, + "torch._functorch.vmap.restore_vmap": UserFunctionVariable, + "torch._functorch.apis.vmap": UserFunctionVariable, + "torch._functorch.vmap.unwrap_batched": UserFunctionVariable, + "torch._functorch.vmap.vmap_impl": FunctorchHigherOrderVariable, + "torch._functorch.vmap.wrap_batched": UserFunctionVariable, + # functorch/grad + "torch._functorch.eager_transforms.grad_impl": FunctorchHigherOrderVariable, + "torch._functorch.apis.grad_and_value": UserFunctionVariable, + "torch._functorch.eager_transforms._as_tuple": UserFunctionVariable, + "torch._functorch.eager_transforms._check_unique_non_empty": UserFunctionVariable, + "torch._functorch.eager_transforms._create_differentiable": UserFunctionVariable, + "torch._functorch.eager_transforms._slice_argnums": UserFunctionVariable, + "torch._functorch.eager_transforms._undo_create_differentiable": UserFunctionVariable, + "torch._functorch.eager_transforms._validate_and_wrap_argnum": UserFunctionVariable, + "torch._functorch.eager_transforms._validate_and_wrap_argnums": UserFunctionVariable, + "torch._functorch.eager_transforms._wrap_all_tensors": UserFunctionVariable, + "torch._functorch.eager_transforms._wrap_tensor_for_grad": UserFunctionVariable, + # functorch/jacrev + "torch._functorch.eager_transforms.jacrev": FunctorchHigherOrderVariable, + "torch._functorch.eager_transforms.error_if_complex": UserFunctionVariable, + "torch._functorch.eager_transforms._chunked_standard_basis_for_": UserFunctionVariable, + "torch._functorch.eager_transforms._safe_zero_index": UserFunctionVariable, + # functorch/vjp + "torch._functorch.eager_transforms.vjp": FunctorchHigherOrderVariable, + "torch._functorch.eager_transforms._vjp_with_argnums": UserFunctionVariable, + "torch._functorch.eager_transforms.assert_non_empty_tensor_output": UserFunctionVariable, + # functorch/jvp + "torch._functorch.eager_transforms._jvp_with_argnums": UserFunctionVariable, + "torch._functorch.eager_transforms.jvp": FunctorchHigherOrderVariable, + "torch._functorch.eager_transforms._replace_args": UserFunctionVariable, + "torch._functorch.eager_transforms.safe_unpack_dual": UserFunctionVariable, + "torch._functorch.eager_transforms.assert_non_empty_list_of_tensors": UserFunctionVariable, + "torch._functorch.eager_transforms.assert_output_is_tensor_or_tensors": UserFunctionVariable, + "torch.autograd.forward_ad.enter_dual_level": UserFunctionVariable, + "torch.autograd.forward_ad.exit_dual_level": UserFunctionVariable, + "torch.autograd.forward_ad.make_dual": UserFunctionVariable, + "torch.autograd.forward_ad.unpack_dual": UserFunctionVariable, + # functorch/linearize + "torch._functorch.eager_transforms.linearize": FunctorchHigherOrderVariable, + # functorch/jacfwd + "torch._functorch.eager_transforms.jacfwd": FunctorchHigherOrderVariable, + "torch._functorch.eager_transforms._construct_standard_basis_for": UserFunctionVariable, + "torch._functorch.eager_transforms.safe_unflatten": UserFunctionVariable, + # functorch/hessian + "torch._functorch.eager_transforms.hessian": FunctorchHigherOrderVariable, + # functional_call + "torch._functorch.functional_call.functional_call": FunctionalCallVariable, + "torch.nn.utils.stateless._groupby_tensor": TorchInGraphFunctionVariable, + "torch.nn.utils.stateless._reparametrize_module": ReparametrizeModuleCallVariable, + # functorch/deprecated + "torch._functorch.deprecated.jvp": UserFunctionVariable, + "torch._functorch.deprecated.hessian": UserFunctionVariable, + "torch._functorch.deprecated.jacfwd": UserFunctionVariable, + "torch._functorch.deprecated.jacrev": UserFunctionVariable, + "torch._functorch.deprecated.grad": UserFunctionVariable, + "torch._functorch.deprecated.grad_and_value": UserFunctionVariable, + "torch._functorch.deprecated.vjp": UserFunctionVariable, + # functorch/C++ bindings + "torch._C._functorch._wrap_for_grad": TorchInGraphFunctionVariable, + "torch._C._functorch._unwrap_for_grad": TorchInGraphFunctionVariable, + "torch._C._functorch._unwrap_batched": TorchInGraphFunctionVariable, + "torch._C._functorch.current_level": TorchInGraphFunctionVariable, + "torch._C._functorch.maybe_current_level": TorchInGraphFunctionVariable, + "torch._C._functorch.is_batchedtensor": TorchInGraphFunctionVariable, + "torch._C._functorch.peek_interpreter_stack": TorchInGraphFunctionVariable, + "torch._C._functorch.unwrap_if_dead": TorchInGraphFunctionVariable, + "torch._functorch.predispatch._vmap_increment_nesting": TorchInGraphFunctionVariable, + "torch._functorch.predispatch._vmap_decrement_nesting": TorchInGraphFunctionVariable, + # everything else + "torch._functorch.pyfunctorch.coerce_cinterpreter": TorchInGraphFunctionVariable, + "torch._higher_order_ops.triton_kernel_wrap.do_prune_configs": UserFunctionVariable, + "torch._higher_order_ops.foreach_map.foreach_map": UserFunctionVariable, + "torch._constrain_as_size": UserFunctionVariable, + "torch._tensor._convert": UserFunctionVariable, + "torch.jit._unwrap_optional": UserFunctionVariable, + "torch.backends.mha.get_fastpath_enabled": UserFunctionVariable, + "torch._dynamo.dont_skip_tracing": UserFunctionVariable, + "torch._dynamo.mark_static": UserFunctionVariable, + "torch._dynamo.nonstrict_trace": UserFunctionVariable, + "torch._dynamo.patch_dynamo_config": UserFunctionVariable, + "torch._dynamo.error_on_graph_break": UserFunctionVariable, + "torch.fx.experimental.symbolic_shapes.guard_size_oblivious": TorchInGraphFunctionVariable, + "torch.fx.experimental.symbolic_shapes.guard_or_true": TorchInGraphFunctionVariable, + "torch.fx.experimental.symbolic_shapes.guard_or_false": TorchInGraphFunctionVariable, + "torch.fx.experimental.symbolic_shapes.statically_known_true": TorchInGraphFunctionVariable, + "torch.fx.experimental.symbolic_shapes.statically_known_false": TorchInGraphFunctionVariable, + "torch.fx.experimental.symbolic_shapes.sym_and": TorchInGraphFunctionVariable, + "torch.fx.experimental.symbolic_shapes.sym_or": TorchInGraphFunctionVariable, + "torch.fx.experimental.symbolic_shapes.guard_scalar": TorchInGraphFunctionVariable, + "torch.fx.experimental.symbolic_shapes.has_static_value": TorchInGraphFunctionVariable, + "torch.cuda._get_device_properties": TorchInGraphFunctionVariable, + "torch.utils.hooks.BackwardHook": TorchInGraphFunctionVariable, + "torch.set_default_device": UserFunctionVariable, + "torch.sparse_bsc_tensor": SkipFunctionVariable, + "torch.sparse_bsr_tensor": SkipFunctionVariable, + "torch.sparse_csc_tensor": SkipFunctionVariable, + "torch.sparse_csr_tensor": SkipFunctionVariable, + "torch.sparse_compressed_tensor": SkipFunctionVariable, + "torch._C._autograd._unsafe_set_version_counter": TorchInGraphFunctionVariable, + "torch.xpu.get_rng_state": SkipFunctionVariable, + "torch.xpu.set_rng_state": SkipFunctionVariable, + # avoid skipping user defined modules in distributed unit tests + "torch/testing/_internal/common_fsdp.py#forward": UserFunctionVariable, + f"torch/testing/_internal/common_fsdp.py#{TORCH_DYNAMO_RESUME_IN_PREFIX}": UserFunctionVariable, + "torch/testing/_internal/distributed/_tensor/common_dtensor.py#forward": UserFunctionVariable, + f"torch/testing/_internal/distributed/_tensor/common_dtensor.py#{TORCH_DYNAMO_RESUME_IN_PREFIX}": UserFunctionVariable, + "torch/testing/_internal/common_distributed.py#forward": UserFunctionVariable, + f"torch/testing/_internal/common_distributed.py#{TORCH_DYNAMO_RESUME_IN_PREFIX}": UserFunctionVariable, + "torch.utils._pytree._get_node_type": PyTreeGetNodeTypeFunctionVariable, + "torch.utils._pytree.tree_is_leaf": PyTreeTreeIsLeafFunctionVariable, +} + + +# In graph functions (including constant folding) that are C bindings +torch_c_binding_in_graph_functions = dict.fromkeys( + [ + "math.acos", + "math.acosh", + "math.asin", + "math.asinh", + "math.atan", + "math.atan2", + "math.atanh", + "math.ceil", + "math.comb", + "math.copysign", + "math.cos", + "math.cosh", + "math.degrees", + "math.dist", + "math.erf", + "math.erfc", + "math.exp", + "math.expm1", + "math.fabs", + "math.factorial", + "math.floor", + "math.fmod", + "math.frexp", + "math.fsum", + "math.gamma", + "math.gcd", + "math.hypot", + "math.isclose", + "math.isfinite", + "math.isinf", + "math.isnan", + "math.isqrt", + "math.lcm", + "math.ldexp", + "math.lgamma", + "math.log", + "math.log10", + "math.log1p", + "math.log2", + "math.modf", + "math.nextafter", + "math.perm", + "math.pow", + "math.prod", + "math.radians", + "math.remainder", + "math.sin", + "math.sinh", + "math.tan", + "math.tanh", + "math.trunc", + "math.ulp", + "torch._adaptive_avg_pool2d", + "torch._adaptive_avg_pool3d", + "torch._add_batch_dim", + "torch._add_relu_", + "torch._add_relu", + "torch._addmm_activation", + "torch._aminmax", + "torch._amp_foreach_non_finite_check_and_unscale_", + "torch._amp_update_scale_", + "torch._assert_async", + "torch._assert_tensor_metadata", + "torch._batch_norm_impl_index", + "torch._C._accelerator_getAccelerator", + "torch._C._accelerator_getDeviceIndex", + "torch._C._accelerator_getStream", + "torch._C._accelerator_setAllocatorSettings", + "torch._C._accelerator_setStream", + "torch._C._accelerator_synchronizeDevice", + "torch._C._activate_gpu_trace", + "torch._C._add_cached_tensor", + "torch._C._add_docstr", + "torch._C._are_functorch_transforms_active", + "torch._C._autograd_init", + "torch._C._awaitable_nowait", + "torch._C._awaitable_wait", + "torch._C._awaitable", + "torch._C._backport_for_mobile_from_buffer_to_buffer", + "torch._C._backport_for_mobile_from_buffer", + "torch._C._backport_for_mobile_to_buffer", + "torch._C._backport_for_mobile", + "torch._C._broadcast_coalesced", + "torch._C._broadcast_out", + "torch._C._broadcast", + "torch._C._c10d_init", + "torch._C._calculate_package_version_based_on_upgraders", + "torch._C._can_use_flash_attention", + "torch._C._can_use_mem_efficient_attention", + "torch._C._can_use_cudnn_attention", + "torch._C._check_onnx_proto", + "torch._C._check_sparse_tensor_invariants", + "torch._C._collect_all", + "torch._C._commit_update", + "torch._C._compile_graph_to_code_table", + "torch._C._construct_CUDA_Tensor_From_Storage_And_Metadata", + "torch._C._construct_storage_from_data_pointer", + "torch._C._conv_determine_backend_memory_format", + "torch._C._cpu._is_avx2_supported", + "torch._C._cpu._is_avx512_supported", + "torch._C._cpu._is_avx512_vnni_supported", + "torch._C._cpu._is_avx512_bf16_supported", + "torch._C._cpu._is_amx_tile_supported", + "torch._C._cpu._is_amx_fp16_supported", + "torch._C._cpu._init_amx", + "torch._C._crash_if_aten_asan", + "torch._C._crash_if_csrc_asan", + "torch._C._crash_if_csrc_ubsan", + "torch._C._crash_if_debug_asserts_fail", + "torch._C._crash_if_vptr_ubsan", + "torch._C._create_function_from_graph", + "torch._C._create_function_from_trace_with_dict", + "torch._C._create_function_from_trace", + "torch._C._create_graph_by_tracing", + "torch._C._create_module_with_type", + "torch._C._create_object_with_type", + "torch._C._cuda_attach_out_of_memory_observer", + "torch._C._cuda_beginAllocateCurrentStreamToPool", + "torch._C._cuda_canDeviceAccessPeer", + "torch._C._cuda_changeCurrentAllocator", + "torch._C._cuda_checkPoolLiveAllocations", + "torch._C._cuda_clearCublasWorkspaces", + "torch._C._cuda_cudaCachingAllocator_raw_alloc", + "torch._C._cuda_cudaCachingAllocator_raw_delete", + "torch._C._cuda_cudaHostAllocator", + "torch._C._cuda_customAllocator", + "torch._C._cuda_emptyCache", + "torch._C._cuda_endAllocateToPool", + "torch._C._cuda_exchangeDevice", + "torch._C._cuda_get_conv_benchmark_empty_cache", + "torch._C._cuda_get_cudnn_benchmark_limit", + "torch._C._cuda_get_sync_debug_mode", + "torch._C._cuda_getAllocator", + "torch._C._cuda_getAllocatorBackend", + "torch._C._cuda_getArchFlags", + "torch._C._cuda_getCheckpointState", + "torch._C._cuda_getCompiledVersion", + "torch._C._cuda_getCurrentBlasHandle", + "torch._C._cuda_getCurrentRawStream", + "torch._C._cuda_getCurrentStream", + "torch._C._cuda_getDefaultStream", + "torch._C._cuda_getDevice", + "torch._C._cuda_getDeviceCount", + "torch._C._cuda_hasPrimaryContext", + "torch._C._cuda_hostMemoryStats", + "torch._C._cuda_init", + "torch._C._cuda_ipc_collect", + "torch._C._cuda_isCurrentStreamCapturing", + "torch._C._cuda_isHistoryEnabled", + "torch._C._cuda_isInBadFork", + "torch._C._cuda_jiterator_compile_and_launch_kernel", + "torch._C._cuda_lock_mutex", + "torch._C._cuda_maybeExchangeDevice", + "torch._C._cuda_memorySnapshot", + "torch._C._cuda_memoryStats", + "torch._C._cuda_record_memory_history_legacy", + "torch._C._cuda_record_memory_history", + "torch._C._cuda_releasePool", + "torch._C._cuda_resetAccumulatedHostMemoryStats", + "torch._C._cuda_resetAccumulatedMemoryStats", + "torch._C._cuda_resetPeakHostMemoryStats", + "torch._C._cuda_resetPeakMemoryStats", + "torch._C._cuda_set_cudnn_benchmark_limit", + "torch._C._cuda_set_sync_debug_mode", + "torch._C._cuda_setCheckpointPoolState", + "torch._C._cuda_setDevice", + "torch._C._cuda_setMemoryFraction", + "torch._C._cuda_setStream", + "torch._C._cuda_sleep", + "torch._C._cuda_synchronize", + "torch._C._cuda_unlock_mutex", + "torch._C._cudnn_set_conv_benchmark_empty_cache", + "torch._C._cudnn.getCompileVersion", + "torch._C._cudnn.getRuntimeVersion", + "torch._C._cudnn.getVersionInt", + "torch._C._current_autograd_node", + "torch._C._current_graph_task_execution_order", + "torch._C._current_graph_task_id", + "torch._C._cxx_flags", + "torch._C._debug_get_fusion_group_inlining", + "torch._C._debug_only_are_vmap_fallback_warnings_enabled", + "torch._C._debug_only_display_vmap_fallback_warnings", + "torch._C._debug_set_autodiff_subgraph_inlining", + "torch._C._debug_set_fusion_group_inlining", + "torch._C._demangle", + "torch._C._disabled_torch_dispatch_impl", + "torch._C._dispatch_call_boxed", + "torch._C._dispatch_check_all_invariants", + "torch._C._dispatch_check_invariants", + "torch._C._dispatch_dump_table", + "torch._C._dispatch_dump", + "torch._C._dispatch_find_dangling_impls", + "torch._C._dispatch_find_schema_or_throw", + "torch._C._dispatch_get_all_op_names", + "torch._C._dispatch_get_backend_keyset_from_autograd", + "torch._C._dispatch_get_registrations_for_dispatch_key", + "torch._C._dispatch_has_backend_fallback", + "torch._C._dispatch_has_computed_kernel_for_dispatch_key", + "torch._C._dispatch_has_kernel_for_any_dispatch_key", + "torch._C._dispatch_has_kernel_for_dispatch_key", + "torch._C._dispatch_has_kernel", + "torch._C._dispatch_is_alias_key", + "torch._C._dispatch_is_included_in_alias", + "torch._C._dispatch_isTensorSubclassLike", + "torch._C._dispatch_key_for_device", + "torch._C._dispatch_key_name", + "torch._C._dispatch_key_parse", + "torch._C._dispatch_key_set", + "torch._C._dispatch_keys", + "torch._C._dispatch_keyset_full_after", + "torch._C._dispatch_keyset_full", + "torch._C._dispatch_keyset_to_string", + "torch._C._dispatch_library", + "torch._C._dispatch_num_backends", + "torch._C._dispatch_print_registrations_for_dispatch_key", + "torch._C._dispatch_pystub", + "torch._C._dispatch_set_report_error_callback", + "torch._C._dispatch_tls_is_dispatch_key_excluded", + "torch._C._dispatch_tls_is_dispatch_key_included", + "torch._C._dispatch_tls_local_exclude_set", + "torch._C._dispatch_tls_local_include_set", + "torch._C._dispatch_tls_set_dispatch_key_excluded", + "torch._C._dispatch_tls_set_dispatch_key_included", + "torch._C._dist_autograd_init", + "torch._C._dump_local_tls_set", + "torch._C._dump_upgraders_map", + "torch._C._enable_mobile_interface_call_export", + "torch._C._enter_dual_level", + "torch._C._error_if_any_worker_fails", + "torch._C._exit_dual_level", + "torch._C._export_operator_list", + "torch._C._export_opnames", + "torch._C._faulty_agent_init", + "torch._C._fft.fft_fft", + "torch._C._fft.fft_fft2", + "torch._C._fft.fft_fftfreq", + "torch._C._fft.fft_fftn", + "torch._C._fft.fft_fftshift", + "torch._C._fft.fft_hfft", + "torch._C._fft.fft_hfft2", + "torch._C._fft.fft_hfftn", + "torch._C._fft.fft_ifft", + "torch._C._fft.fft_ifft2", + "torch._C._fft.fft_ifftn", + "torch._C._fft.fft_ifftshift", + "torch._C._fft.fft_ihfft", + "torch._C._fft.fft_ihfft2", + "torch._C._fft.fft_ihfftn", + "torch._C._fft.fft_irfft", + "torch._C._fft.fft_irfft2", + "torch._C._fft.fft_irfftn", + "torch._C._fft.fft_rfft", + "torch._C._fft.fft_rfft2", + "torch._C._fft.fft_rfftfreq", + "torch._C._fft.fft_rfftn", + "torch._C._free_And_Remove_DeleterFn", + "torch._C._freeze_module", + "torch._C._from_dlpack", + "torch._C._functionality_to_backend_keys", + "torch._C._functionalization_reapply_views_tls", + "torch._C._fuse_to_static_module", + "torch._C._gather_out", + "torch._C._gather", + "torch._C._generate_upgraders_graph", + "torch._C._get_autograd_fallback_mode", + "torch._C._get_backcompat_broadcast_warn", + "torch._C._get_backcompat_keepdim_warn", + "torch._C._get_blas_preferred_backend", + "torch._C._get_caught_jit_exception_class_name", + "torch._C._get_caught_jit_exception_original_msg", + "torch._C._get_constant_bool_symnode", + "torch._C._get_cpp_backtrace", + "torch._C._get_cpu_capability", + "torch._C._get_cublas_allow_bf16_reduced_precision_reduction", + "torch._C._get_cublas_allow_fp16_reduced_precision_reduction", + "torch._C._get_cublas_allow_tf32", + "torch._C._get_cudnn_allow_tf32", + "torch._C._get_cudnn_benchmark", + "torch._C._get_miopen_immediate", + "torch._C._get_cudnn_deterministic", + "torch._C._get_cudnn_enabled", + "torch._C._get_custom_class_python_wrapper", + "torch._C._get_default_device", + "torch._C._get_deterministic_algorithms_warn_only", + "torch._C._get_deterministic_algorithms", + "torch._C._get_deterministic_fill_uninitialized_memory", + "torch._C._get_dispatch_mode", + "torch._C._get_dispatch_stack_at", + "torch._C._get_file_format", + "torch._C._get_flash_sdp_enabled", + "torch._C._get_float32_matmul_precision", + "torch._C._get_function_stack_at", + "torch._C._get_graph_executor_optimize", + "torch._C._get_linalg_preferred_backend", + "torch._C._get_rocm_fa_preferred_backend", + "torch._C._get_math_sdp_enabled", + "torch._C._get_math_sdp_allow_fp16_bf16_reduction", + "torch._C._get_max_operator_version", + "torch._C._get_mem_efficient_sdp_enabled", + "torch._C._get_mkldnn_enabled", + "torch._C._get_cudnn_sdp_enabled", + "torch._C._get_overrideable_sdp_enabled", + "torch._C._set_sdp_use_cudnn", + "torch._C._get_mobile_model_contained_types_from_buffer", + "torch._C._get_mobile_model_contained_types", + "torch._C._get_model_bytecode_version_from_buffer", + "torch._C._get_model_bytecode_version", + "torch._C._get_model_extra_files_from_buffer", + "torch._C._get_model_extra_files", + "torch._C._get_model_ops_and_info_from_buffer", + "torch._C._get_model_ops_and_info", + "torch._C._get_module_info_from_flatbuffer", + "torch._C._get_nnpack_enabled", + "torch._C._get_obj_in_tls", + "torch._C._get_operation_overload", + "torch._C._get_operator_version_map", + "torch._C._get_privateuse1_backend_name", + "torch._C._get_qengine", + "torch._C._get_schema", + "torch._C._get_sm_carveout_experimental", + "torch._C._get_nested_int", + "torch._C._get_tensor_metadata", + "torch._C._get_tracing_state", + "torch._C._get_upgrader_ranges", + "torch._C._get_upgraders_entry_map", + "torch._C._get_upgraders_map_size", + "torch._C._get_value_trace", + "torch._C._get_version_calculator_flag", + "torch._C._get_warnAlways", + "torch._C._graph_pool_handle", + "torch._C._group_tensors_by_device_and_dtype", + "torch._C._hack_do_not_use_clone_module_with_class", + "torch._C._has_distributed", + "torch._C._has_Standard_Deleter", + "torch._C._has_storage", + "torch._C._has_tensorexpr_cpp_tests", + "torch._C._run_tensorexpr_cpp_tests", + "torch._C._has_torch_function_unary", + "torch._C._has_torch_function_variadic", + "torch._C._has_torch_function", + "torch._C._import_ir_module_from_package", + "torch._C._increment_version", + "torch._C._infer_size", + "torch._C._init_names", + "torch._C._initExtension", + "torch._C._is_alias_of", + "torch._C._is_any_autocast_enabled", + "torch._C._is_cached_tensor", + "torch._C._is_flash_attention_available", + "torch._C._is_fwd_grad_enabled", + "torch._C._is_key_in_tls", + "torch._C._is_multithreading_enabled", + "torch._C._is_torch_function_enabled", + "torch._C._is_torch_function_mode_enabled", + "torch._C._is_torch_function_all_disabled", + "torch._C._is_tracing", + "torch._C._is_view_replay_enabled", + "torch._C._is_xnnpack_enabled", + "torch._C._itt.is_available", + "torch._C._itt.mark", + "torch._C._itt.rangePop", + "torch._C._itt.rangePush", + "torch._C._ivalue_debug_python_object", + "torch._C._ivalue_tags_match", + "torch._C._jit_assert_is_instance", + "torch._C._jit_can_fuse_on_cpu_legacy", + "torch._C._jit_can_fuse_on_cpu", + "torch._C._jit_can_fuse_on_gpu", + "torch._C._jit_cat_wo_conditionals", + "torch._C._jit_check_alias_annotation", + "torch._C._jit_clear_class_registry", + "torch._C._jit_debug_fuser_num_cached_kernel_specs", + "torch._C._jit_debug_module_iterators", + "torch._C._jit_decay_packed_param_input_types", + "torch._C._jit_decomposition_graph_for_node", + "torch._C._jit_differentiate", + "torch._C._jit_erase_non_input_shape_information", + "torch._C._jit_flatten", + "torch._C._jit_fuser_get_fused_kernel_code", + "torch._C._jit_get_all_schemas", + "torch._C._jit_get_custom_class_schemas", + "torch._C._jit_get_emit_hooks", + "torch._C._jit_get_inline_everything_mode", + "torch._C._jit_get_logging_option", + "torch._C._jit_get_num_profiled_runs", + "torch._C._jit_get_operation", + "torch._C._jit_get_schemas_for_operator", + "torch._C._jit_get_te_cuda_pointwise_block_count", + "torch._C._jit_get_te_cuda_pointwise_block_size", + "torch._C._jit_get_te_cuda_pointwise_loop_levels", + "torch._C._jit_get_te_generate_block_code", + "torch._C._jit_get_te_must_use_llvm_cpu", + "torch._C._jit_get_tracer_state_warn", + "torch._C._jit_has_cpp_tests", + "torch._C._jit_init", + "torch._C._jit_interpret_graph", + "torch._C._jit_is_onnx_log_enabled", + "torch._C._jit_is_script_object", + "torch._C._jit_llga_enabled", + "torch._C._jit_nvfuser_can_be_enabled", + "torch._C._jit_nvfuser_clear_comparison_callback", + "torch._C._jit_nvfuser_enabled", + "torch._C._jit_nvfuser_horizontal_mode", + "torch._C._jit_nvfuser_set_comparison_callback", + "torch._C._jit_nvfuser_single_node_mode", + "torch._C._jit_object_is_non_holding", + "torch._C._jit_onnx_convert_pattern_from_subblock", + "torch._C._jit_onnx_create_full_scope_name", + "torch._C._jit_onnx_list_model_parameters", + "torch._C._jit_onnx_log", + "torch._C._jit_opt_conditionals", + "torch._C._jit_override_can_fuse_on_cpu_legacy", + "torch._C._jit_override_can_fuse_on_cpu", + "torch._C._jit_override_can_fuse_on_gpu", + "torch._C._jit_pass_autocast", + "torch._C._jit_pass_batch_mm", + "torch._C._jit_pass_canonicalize_graph_fuser_ops", + "torch._C._jit_pass_canonicalize", + "torch._C._jit_pass_complete_shape_analysis", + "torch._C._jit_pass_concat_frozen_linear", + "torch._C._jit_pass_constant_loop_unrolling", + "torch._C._jit_pass_constant_pooling", + "torch._C._jit_pass_constant_propagation_immutable_types", + "torch._C._jit_pass_constant_propagation", + "torch._C._jit_pass_convert_frozen_ops_to_mkldnn", + "torch._C._jit_pass_create_autodiff_subgraphs", + "torch._C._jit_pass_create_functional_graphs", + "torch._C._jit_pass_cse", + "torch._C._jit_pass_custom_pattern_based_rewrite_graph", + "torch._C._jit_pass_custom_pattern_based_rewrite", + "torch._C._jit_pass_dbr_quant_remove_redundant_aliases", + "torch._C._jit_pass_dce_allow_deleting_nodes_with_side_effects", + "torch._C._jit_pass_dce", + "torch._C._jit_pass_decompose_ops", + "torch._C._jit_pass_dedup_module_uses", + "torch._C._jit_pass_erase_number_types", + "torch._C._jit_pass_erase_shape_information", + "torch._C._jit_pass_filter_non_tensor_arguments", + "torch._C._jit_pass_fixup_onnx_controlflow_node", + "torch._C._jit_pass_fold_convbn", + "torch._C._jit_pass_fold_frozen_conv_add_or_sub", + "torch._C._jit_pass_fold_frozen_conv_bn", + "torch._C._jit_pass_fold_frozen_conv_mul_or_div", + "torch._C._jit_pass_fold_frozen_linear_bn", + "torch._C._jit_pass_fold_prepacking_ops", + "torch._C._jit_pass_functional_to_inplace_activation", + "torch._C._jit_pass_fuse_add_relu", + "torch._C._jit_pass_fuse_addmm", + "torch._C._jit_pass_fuse_clamp_w_prepacked_linear_conv", + "torch._C._jit_pass_fuse_frozen_conv_add_relu", + "torch._C._jit_pass_fuse_linear", + "torch._C._jit_pass_fuse_quantized_add_relu", + "torch._C._jit_pass_fuse_tensorexprs", + "torch._C._jit_pass_fuse", + "torch._C._jit_pass_inline_fork_wait", + "torch._C._jit_pass_inline_functional_graphs", + "torch._C._jit_pass_inline", + "torch._C._jit_pass_inplace_to_functional_activation", + "torch._C._jit_pass_insert_observer_method_for_ondevice_ptq", + "torch._C._jit_pass_insert_observers", + "torch._C._jit_pass_insert_prepack_unpack", + "torch._C._jit_pass_insert_prepacked_ops", + "torch._C._jit_pass_insert_quant_dequant_for_ondevice_ptq", + "torch._C._jit_pass_insert_quant_dequant", + "torch._C._jit_pass_integer_value_refinement", + "torch._C._jit_pass_lint", + "torch._C._jit_pass_loop_unrolling", + "torch._C._jit_pass_lower_all_tuples", + "torch._C._jit_pass_lower_graph", + "torch._C._jit_pass_metal_fold_prepacking_ops", + "torch._C._jit_pass_metal_fuse_clamp_w_prepacked_conv", + "torch._C._jit_pass_metal_insert_prepacked_ops", + "torch._C._jit_pass_metal_optimize_for_mobile", + "torch._C._jit_pass_onnx_assign_output_shape", + "torch._C._jit_pass_onnx_assign_scoped_names_for_node_and_value", + "torch._C._jit_pass_onnx_autograd_function_process", + "torch._C._jit_pass_onnx_block", + "torch._C._jit_pass_onnx_cast_all_constant_to_floating", + "torch._C._jit_pass_onnx_clear_scope_records", + "torch._C._jit_pass_onnx_constant_fold", + "torch._C._jit_pass_onnx_deduplicate_initializers", + "torch._C._jit_pass_onnx_eliminate_unused_items", + "torch._C._jit_pass_onnx_eval_peephole", + "torch._C._jit_pass_onnx_function_extraction", + "torch._C._jit_pass_onnx_function_substitution", + "torch._C._jit_pass_onnx_graph_shape_type_inference", + "torch._C._jit_pass_onnx_lint", + "torch._C._jit_pass_onnx_node_shape_type_inference", + "torch._C._jit_pass_onnx_peephole", + "torch._C._jit_pass_onnx_preprocess_caffe2", + "torch._C._jit_pass_onnx_preprocess", + "torch._C._jit_pass_onnx_quantization_insert_permutes", + "torch._C._jit_pass_onnx_remove_inplace_ops_for_onnx", + "torch._C._jit_pass_onnx_remove_print", + "torch._C._jit_pass_onnx_scalar_type_analysis", + "torch._C._jit_pass_onnx_set_dynamic_input_shape", + "torch._C._jit_pass_onnx_track_scope_attributes", + "torch._C._jit_pass_onnx_unpack_quantized_weights", + "torch._C._jit_pass_onnx", + "torch._C._jit_pass_optimize_for_inference", + "torch._C._jit_pass_optimize_for_mobile", + "torch._C._jit_pass_optimize_frozen_graph", + "torch._C._jit_pass_pattern_based_rewrite", + "torch._C._jit_pass_peephole_list_idioms", + "torch._C._jit_pass_peephole", + "torch._C._jit_pass_prepare_division_for_onnx", + "torch._C._jit_pass_propagate_device", + "torch._C._jit_pass_propagate_dtype", + "torch._C._jit_pass_propagate_shapes_on_graph_and_build_compute", + "torch._C._jit_pass_propagate_shapes_on_graph", + "torch._C._jit_pass_quant_finalize_for_ondevice_ptq", + "torch._C._jit_pass_quant_finalize", + "torch._C._jit_pass_quant_fusion", + "torch._C._jit_pass_refine_integer_values", + "torch._C._jit_pass_refine_tuple_types", + "torch._C._jit_pass_remove_dropout", + "torch._C._jit_pass_remove_expands", + "torch._C._jit_pass_remove_inplace_ops", + "torch._C._jit_pass_remove_mutation", + "torch._C._jit_pass_replace_old_ops_with_upgraders", + "torch._C._jit_pass_replicate_dequantize", + "torch._C._jit_pass_run_decompositions", + "torch._C._jit_pass_specialize_autogradzero", + "torch._C._jit_pass_swap_functional_linear", + "torch._C._jit_pass_transform_conv1d_to_conv2d", + "torch._C._jit_pass_transpose_frozen_linear", + "torch._C._jit_pass_vulkan_fold_prepacking_ops", + "torch._C._jit_pass_vulkan_fuse_clamp_w_prepacked_conv", + "torch._C._jit_pass_vulkan_insert_prepacked_ops", + "torch._C._jit_pass_vulkan_optimize_for_mobile", + "torch._C._jit_register_decomposition_for_schema", + "torch._C._jit_register_shape_compute_graph_for_node", + "torch._C._jit_resolve_packet", + "torch._C._jit_run_cpp_tests", + "torch._C._jit_script_class_compile", + "torch._C._jit_script_compile_overload", + "torch._C._jit_script_compile", + "torch._C._jit_script_interface_compile", + "torch._C._jit_set_autocast_mode", + "torch._C._jit_set_bailout_depth", + "torch._C._jit_set_emit_hooks", + "torch._C._jit_set_fusion_strategy", + "torch._C._jit_set_inline_everything_mode", + "torch._C._jit_set_llga_enabled", + "torch._C._jit_set_logging_option", + "torch._C._jit_set_logging_stream", + "torch._C._jit_set_num_profiled_runs", + "torch._C._jit_set_nvfuser_enabled", + "torch._C._jit_set_nvfuser_guard_mode", + "torch._C._jit_set_nvfuser_horizontal_mode", + "torch._C._jit_set_nvfuser_single_node_mode", + "torch._C._jit_set_nvfuser_skip_node_kind", + "torch._C._jit_set_onnx_log_enabled", + "torch._C._jit_set_onnx_log_output_stream", + "torch._C._jit_set_profiling_executor", + "torch._C._jit_set_profiling_mode", + "torch._C._jit_set_symbolic_shapes_test_mode", + "torch._C._jit_set_te_cuda_pointwise_block_count", + "torch._C._jit_set_te_cuda_pointwise_block_size", + "torch._C._jit_set_te_cuda_pointwise_loop_levels", + "torch._C._jit_set_te_generate_block_code", + "torch._C._jit_set_te_must_use_llvm_cpu", + "torch._C._jit_set_texpr_dynamic_shape_enabled", + "torch._C._jit_set_texpr_fuser_enabled", + "torch._C._jit_set_texpr_reductions_enabled", + "torch._C._jit_set_tracer_state_warn", + "torch._C._jit_set_utf8_decoding_ignore", + "torch._C._jit_shape_compute_graph_for_node", + "torch._C._jit_symbolic_shapes_test_mode_enabled", + "torch._C._jit_texpr_dynamic_shape_enabled", + "torch._C._jit_texpr_fallback_allowed", + "torch._C._jit_texpr_fuser_enabled", + "torch._C._jit_texpr_reductions_enabled", + "torch._C._jit_texpr_set_fallback_allowed", + "torch._C._jit_to_backend_selective", + "torch._C._jit_to_backend", + "torch._C._jit_to_static_module", + "torch._C._jit_trace_graph", + "torch._C._jit_trace_module", + "torch._C._jit_tree_views.FalseLiteral", + "torch._C._jit_tree_views.NoneLiteral", + "torch._C._jit_tree_views.TrueLiteral", + "torch._C._jit_try_infer_type", + "torch._C._jit_unflatten", + "torch._C._last_executed_optimized_graph", + "torch._C._len_torch_dispatch_stack", + "torch._C._len_torch_function_stack", + "torch._C._linalg._linalg_eigvals", + "torch._C._linalg.linalg_cholesky_ex", + "torch._C._linalg.linalg_cholesky", + "torch._C._linalg.linalg_cond", + "torch._C._linalg.linalg_cross", + "torch._C._linalg.linalg_det", + "torch._C._linalg.linalg_diagonal", + "torch._C._linalg.linalg_eig", + "torch._C._linalg.linalg_eigh", + "torch._C._linalg.linalg_eigvals", + "torch._C._linalg.linalg_eigvalsh", + "torch._C._linalg.linalg_householder_product", + "torch._C._linalg.linalg_inv_ex", + "torch._C._linalg.linalg_inv", + "torch._C._linalg.linalg_ldl_factor_ex", + "torch._C._linalg.linalg_ldl_factor", + "torch._C._linalg.linalg_ldl_solve", + "torch._C._linalg.linalg_lstsq", + "torch._C._linalg.linalg_lu_factor_ex", + "torch._C._linalg.linalg_lu_factor", + "torch._C._linalg.linalg_lu_solve", + "torch._C._linalg.linalg_lu", + "torch._C._linalg.linalg_matmul", + "torch._C._linalg.linalg_matrix_exp", + "torch._C._linalg.linalg_matrix_norm", + "torch._C._linalg.linalg_matrix_power", + "torch._C._linalg.linalg_matrix_rank", + "torch._C._linalg.linalg_multi_dot", + "torch._C._linalg.linalg_norm", + "torch._C._linalg.linalg_pinv", + "torch._C._linalg.linalg_qr", + "torch._C._linalg.linalg_slogdet", + "torch._C._linalg.linalg_solve_ex", + "torch._C._linalg.linalg_solve_triangular", + "torch._C._linalg.linalg_solve", + "torch._C._linalg.linalg_svd", + "torch._C._linalg.linalg_svdvals", + "torch._C._linalg.linalg_tensorinv", + "torch._C._linalg.linalg_tensorsolve", + "torch._C._linalg.linalg_vander", + "torch._C._linalg.linalg_vecdot", + "torch._C._linalg.linalg_vector_norm", + "torch._C._llvm_enabled", + "torch._C._load_for_lite_interpreter_from_buffer", + "torch._C._load_for_lite_interpreter", + "torch._C._load_jit_module_from_bytes", + "torch._C._load_jit_module_from_file", + "torch._C._load_mobile_module_from_bytes", + "torch._C._load_mobile_module_from_file", + "torch._C._log_api_usage_metadata", + "torch._C._log_api_usage_once", + "torch._C._logging_set_logger", + "torch._C._meta_in_tls_dispatch_include", + "torch._C._mps_acquireEvent", + "torch._C._mps_currentAllocatedMemory", + "torch._C._mps_deviceSynchronize", + "torch._C._mps_driverAllocatedMemory", + "torch._C._mps_recommendedMaxMemory", + "torch._C._mps_elapsedTimeOfEvents", + "torch._C._mps_emptyCache", + "torch._C._mps_get_default_generator", + "torch._C._mps_is_available", + "torch._C._mps_is_in_bad_fork", + "torch._C._mps_is_on_macos_13_or_newer", + "torch._C._mps_profilerStartTrace", + "torch._C._mps_profilerStopTrace", + "torch._C._mps_queryEvent", + "torch._C._mps_recordEvent", + "torch._C._mps_releaseEvent", + "torch._C._mps_setMemoryFraction", + "torch._C._mps_synchronizeEvent", + "torch._C._mps_waitForEvent", + "torch._C._multiprocessing_init", + "torch._C._nccl_all_gather", + "torch._C._nccl_all_reduce", + "torch._C._nccl_broadcast", + "torch._C._nccl_init_rank", + "torch._C._nccl_reduce_scatter", + "torch._C._nccl_reduce", + "torch._C._nccl_unique_id", + "torch._C._nccl_version_suffix", + "torch._C._nccl_version", + "torch._C._nested.nested_tensor", + "torch._C._nested.nested_to_padded_tensor", + "torch._C._new_symbolic_shape_symbol", + "torch._C._nn_module_to_mobile", + "torch._C._nn._conv_depthwise2d", + "torch._C._nn._pad_circular", + "torch._C._nn._pad_enum", + "torch._C._nn._parse_to", + "torch._C._nn._test_ambiguous_defaults", + "torch._C._nn._test_optional_filled_intlist", + "torch._C._nn._test_optional_floatlist", + "torch._C._nn._test_optional_intlist", + "torch._C._nn._test_string_default", + "torch._C._nn._test_warn_in_autograd", + "torch._C._nn._upsample_bicubic2d_aa", + "torch._C._nn._upsample_bilinear2d_aa", + "torch._C._nn._upsample_nearest_exact1d", + "torch._C._nn._upsample_nearest_exact2d", + "torch._C._nn._upsample_nearest_exact3d", + "torch._C._nn.adaptive_avg_pool2d", + "torch._C._nn.adaptive_avg_pool3d", + "torch._C._nn.adaptive_max_pool2d", + "torch._C._nn.adaptive_max_pool3d", + "torch._C._nn.avg_pool2d", + "torch._C._nn.avg_pool3d", + "torch._C._nn.binary_cross_entropy", + "torch._C._nn.col2im", + "torch._C._nn.conv_depthwise3d", + "torch._C._nn.cross_entropy_loss", + "torch._C._nn.elu_", + "torch._C._nn.elu", + "torch._C._nn.flatten_dense_tensors", + "torch._C._nn.fractional_max_pool2d", + "torch._C._nn.fractional_max_pool3d", + "torch._C._nn.gelu_", + "torch._C._nn.gelu", + "torch._C._nn.glu", + "torch._C._nn.hardsigmoid_", + "torch._C._nn.hardsigmoid", + "torch._C._nn.hardswish_", + "torch._C._nn.hardswish", + "torch._C._nn.hardtanh_", + "torch._C._nn.hardtanh", + "torch._C._nn.huber_loss", + "torch._C._nn.im2col", + "torch._C._nn.l1_loss", + "torch._C._nn.leaky_relu_", + "torch._C._nn.leaky_relu", + "torch._C._nn.linear", + "torch._C._nn.log_sigmoid", + "torch._C._nn.max_pool2d_with_indices", + "torch._C._nn.max_pool3d_with_indices", + "torch._C._nn.max_unpool2d", + "torch._C._nn.max_unpool3d", + "torch._C._nn.mish_", + "torch._C._nn.mish", + "torch._C._nn.mkldnn_linear", + "torch._C._nn.mkldnn_reorder_conv2d_weight", + "torch._C._nn.mkldnn_reorder_conv3d_weight", + "torch._C._nn.mse_loss", + "torch._C._nn.multi_margin_loss", + "torch._C._nn.multilabel_margin_loss", + "torch._C._nn.nll_loss_nd", + "torch._C._nn.nll_loss", + "torch._C._nn.nll_loss2d", + "torch._C._nn.one_hot", + "torch._C._nn.pad_sequence", + "torch._C._nn.pad", + "torch._C._nn.reflection_pad1d", + "torch._C._nn.reflection_pad2d", + "torch._C._nn.reflection_pad3d", + "torch._C._nn.relu6_", + "torch._C._nn.relu6", + "torch._C._nn.replication_pad1d", + "torch._C._nn.replication_pad2d", + "torch._C._nn.replication_pad3d", + "torch._C._nn.rrelu_with_noise_", + "torch._C._nn.rrelu_with_noise", + "torch._C._nn.scaled_dot_product_attention", + "torch._C._nn.silu_", + "torch._C._nn.silu", + "torch._C._nn.slow_conv_dilated2d", + "torch._C._nn.slow_conv_dilated3d", + "torch._C._nn.slow_conv_transpose2d", + "torch._C._nn.slow_conv_transpose3d", + "torch._C._nn.slow_conv3d", + "torch._C._nn.smooth_l1_loss", + "torch._C._nn.soft_margin_loss", + "torch._C._nn.softplus", + "torch._C._nn.softshrink", + "torch._C._nn.thnn_conv2d", + "torch._C._nn.unflatten_dense_tensors", + "torch._C._nn.upsample_bicubic2d", + "torch._C._nn.upsample_bilinear2d", + "torch._C._nn.upsample_linear1d", + "torch._C._nn.upsample_nearest1d", + "torch._C._nn.upsample_nearest2d", + "torch._C._nn.upsample_nearest3d", + "torch._C._nn.upsample_trilinear3d", + "torch._C._non_sym_sizes", + "torch._C._overlaps", + "torch._C._parallel_info", + "torch._C._parse_dispatch_key", + "torch._C._parse_source_def", + "torch._C._pop_torch_dispatch_stack", + "torch._C._pop_torch_function_stack", + "torch._C._propagate_and_assign_input_shapes", + "torch._C._propagate_shapes", + "torch._C._propagate_xla_data", + "torch._C._push_on_torch_dispatch_stack", + "torch._C._push_on_torch_function_stack", + "torch._C._quantize_ondevice_ptq_dynamic", + "torch._C._register_py_class_for_device", + "torch._C._remove_cached_tensor", + "torch._C._remove_worker_pids", + "torch._C._rename_privateuse1_backend", + "torch._C._replace_", + "torch._C._replace_overloaded_method_decl", + "torch._C._resolve_type_from_object", + "torch._C._resolve_type", + "torch._C._rocm_is_backward_pass", + "torch._C._rpc_init", + "torch._C._run_emit_module_hook", + "torch._C._save_jit_module_to_bytes", + "torch._C._save_jit_module", + "torch._C._save_mobile_module_to_bytes", + "torch._C._save_mobile_module", + "torch._C._save_parameters", + "torch._C._scatter_out", + "torch._C._scatter", + "torch._C._select_conv_backend", + "torch._C._select_batch_norm_backend", + "torch._C._set_autograd_fallback_mode", + "torch._C._set_backcompat_broadcast_warn", + "torch._C._set_backcompat_keepdim_warn", + "torch._C._set_blas_preferred_backend", + "torch._C._set_cached_tensors_enabled", + "torch._C._set_check_sparse_tensor_invariants", + "torch._C._set_conj", + "torch._C._set_cublas_allow_bf16_reduced_precision_reduction", + "torch._C._set_cublas_allow_fp16_reduced_precision_reduction", + "torch._C._set_cublas_allow_tf32", + "torch._C._set_cudnn_allow_tf32", + "torch._C._set_cudnn_benchmark", + "torch._C._set_cudnn_deterministic", + "torch._C._set_cudnn_enabled", + "torch._C._set_default_dtype", + "torch._C._set_default_mobile_cpu_allocator", + "torch._C._set_default_tensor_type", + "torch._C._set_deterministic_algorithms", + "torch._C._set_deterministic_fill_uninitialized_memory", + "torch._C._set_dispatch_mode", + "torch._C._set_float32_matmul_precision", + "torch._C._set_fwd_grad_enabled", + "torch._C._set_grad_enabled", + "torch._C._set_graph_executor_optimize", + "torch._C._set_linalg_preferred_backend", + "torch._C._set_rocm_fa_preferred_backend", + "torch._C._set_meta_in_tls_dispatch_include", + "torch._C._set_mkldnn_enabled", + "torch._C._set_multithreading_enabled", + "torch._C._set_neg", + "torch._C._set_nnpack_enabled", + "torch._C._set_print_stack_traces_on_fatal_signal", + "torch._C._set_qengine", + "torch._C._set_sdp_use_flash", + "torch._C._set_sdp_use_math", + "torch._C._set_math_sdp_allow_fp16_bf16_reduction", + "torch._C._set_sdp_use_mem_efficient", + "torch._C._set_sdp_use_overrideable", + "torch._C._set_should_use_format_with_string_table", + "torch._C._set_sm_carveout_experimental", + "torch._C._set_storage_access_error_msg", + "torch._C._set_tensor_metadata", + "torch._C._set_tracing_state", + "torch._C._set_value_trace", + "torch._C._set_view_replay_enabled", + "torch._C._set_warnAlways", + "torch._C._set_worker_pids", + "torch._C._set_worker_signal_handlers", + "torch._C._should_allow_numbers_as_tensors", + "torch._C._show_config", + "torch._C._sparse._sparse_addmm", + "torch._C._sparse._sparse_log_softmax", + "torch._C._sparse._sparse_mm_reduce_impl", + "torch._C._sparse._sparse_mm", + "torch._C._sparse._sparse_softmax", + "torch._C._sparse._spdiags", + "torch._C._sparse.sparse_sampled_addmm", + "torch._C._special.special_airy_ai", + "torch._C._special.special_bessel_j0", + "torch._C._special.special_bessel_j1", + "torch._C._special.special_bessel_y0", + "torch._C._special.special_bessel_y1", + "torch._C._special.special_chebyshev_polynomial_t", + "torch._C._special.special_chebyshev_polynomial_u", + "torch._C._special.special_chebyshev_polynomial_v", + "torch._C._special.special_chebyshev_polynomial_w", + "torch._C._special.special_digamma", + "torch._C._special.special_entr", + "torch._C._special.special_erf", + "torch._C._special.special_erfc", + "torch._C._special.special_erfcx", + "torch._C._special.special_erfinv", + "torch._C._special.special_exp2", + "torch._C._special.special_expit", + "torch._C._special.special_expm1", + "torch._C._special.special_gammainc", + "torch._C._special.special_gammaincc", + "torch._C._special.special_gammaln", + "torch._C._special.special_hermite_polynomial_h", + "torch._C._special.special_hermite_polynomial_he", + "torch._C._special.special_i0", + "torch._C._special.special_i0e", + "torch._C._special.special_i1", + "torch._C._special.special_i1e", + "torch._C._special.special_laguerre_polynomial_l", + "torch._C._special.special_legendre_polynomial_p", + "torch._C._special.special_log_ndtr", + "torch._C._special.special_log_softmax", + "torch._C._special.special_log1p", + "torch._C._special.special_logit", + "torch._C._special.special_logsumexp", + "torch._C._special.special_modified_bessel_i0", + "torch._C._special.special_modified_bessel_i1", + "torch._C._special.special_modified_bessel_k0", + "torch._C._special.special_modified_bessel_k1", + "torch._C._special.special_multigammaln", + "torch._C._special.special_ndtr", + "torch._C._special.special_ndtri", + "torch._C._special.special_polygamma", + "torch._C._special.special_psi", + "torch._C._special.special_round", + "torch._C._special.special_scaled_modified_bessel_k0", + "torch._C._special.special_scaled_modified_bessel_k1", + "torch._C._special.special_shifted_chebyshev_polynomial_t", + "torch._C._special.special_shifted_chebyshev_polynomial_u", + "torch._C._special.special_shifted_chebyshev_polynomial_v", + "torch._C._special.special_shifted_chebyshev_polynomial_w", + "torch._C._special.special_sinc", + "torch._C._special.special_softmax", + "torch._C._special.special_spherical_bessel_j0", + "torch._C._special.special_xlog1py", + "torch._C._special.special_xlogy", + "torch._C._special.special_zeta", + "torch._C._stash_obj_in_tls", + "torch._C._storage_id", + "torch._C._storage_Use_Count", + "torch._C._supported_qengines", + "torch._C._te.abs", + "torch._C._te.acos", + "torch._C._te.annotate_input_shapes", + "torch._C._te.asin", + "torch._C._te.atan", + "torch._C._te.atan2", + "torch._C._te.ceil", + "torch._C._te.Compute", + "torch._C._te.Compute2", + "torch._C._te.construct_codegen", + "torch._C._te.cos", + "torch._C._te.cosh", + "torch._C._te.erf", + "torch._C._te.erfc", + "torch._C._te.exp", + "torch._C._te.expm1", + "torch._C._te.fixup_missing_shape_info", + "torch._C._te.floor", + "torch._C._te.fmod", + "torch._C._te.frac", + "torch._C._te.ifThenElse", + "torch._C._te.is_graph_compilable", + "torch._C._te.isnan", + "torch._C._te.lgamma", + "torch._C._te.log", + "torch._C._te.log10", + "torch._C._te.log1p", + "torch._C._te.log2", + "torch._C._te.lower", + "torch._C._te.make_shapes_symbolic", + "torch._C._te.pow", + "torch._C._te.Reduce", + "torch._C._te.remainder", + "torch._C._te.remove_graph_output", + "torch._C._te.remove_unused_self_argument", + "torch._C._te.replace_list_output_with_tuple", + "torch._C._te.round", + "torch._C._te.rsqrt", + "torch._C._te.sigmoid", + "torch._C._te.simplify", + "torch._C._te.sin", + "torch._C._te.sinh", + "torch._C._te.sqrt", + "torch._C._te.tan", + "torch._C._te.tanh", + "torch._C._te.trim_graph", + "torch._C._te.trunc", + "torch._C._tensor_impl_raw_handle", + "torch._C._test_only_add_entry_to_op_version_map", + "torch._C._test_only_populate_upgraders", + "torch._C._test_only_remove_entry_to_op_version_map", + "torch._C._test_only_remove_upgraders", + "torch._C._to_functionality_key", + "torch._C._tracer_set_force_outplace", + "torch._C._tracer_set_get_unique_name_fn", + "torch._C._tracer_warn_use_python", + "torch._C._unset_default_mobile_cpu_allocator", + "torch._C._unset_dispatch_mode", + "torch._C._valgrind_supported_platform", + "torch._C._valgrind_toggle_and_dump_stats", + "torch._C._valgrind_toggle", + "torch._C._verbose.mkl_set_verbose", + "torch._C._verbose.mkldnn_set_verbose", + "torch._C._vmapmode_decrement_nesting", + "torch._C._vmapmode_increment_nesting", + "torch._C._warn_deprecation", + "torch._C._warn", + "torch._C._will_engine_execute_node", + "torch._C._wrap_tensor_impl", + "torch._C._xpu_emptyCache", + "torch._C._xpu_getArchFlags", + "torch._C._xpu_getCurrentStream", + "torch._C._xpu_getCurrentRawStream", + "torch._C._xpu_getDeviceCount", + "torch._C._xpu_getDevice", + "torch._C._xpu_getMemoryInfo", + "torch._C._xpu_getStreamFromExternal", + "torch._C._xpu_isInBadFork", + "torch._C._xpu_init", + "torch._C._xpu_memoryStats", + "torch._C._xpu_resetAccumulatedMemoryStats", + "torch._C._xpu_resetPeakMemoryStats", + "torch._C._xpu_setStream", + "torch._C._xpu_synchronize", + "torch._C.fork", + "torch._C.get_autocast_cpu_dtype", + "torch._C.get_autocast_dtype", + "torch._C.get_autocast_gpu_dtype", + "torch._C.get_autocast_ipu_dtype", + "torch._C.get_autocast_xla_dtype", + "torch._C.get_default_dtype", + "torch._C.get_num_interop_threads", + "torch._C.get_num_threads", + "torch._C.import_ir_module_from_buffer", + "torch._C.import_ir_module", + "torch._C.init_num_threads", + "torch._C.is_anomaly_check_nan_enabled", + "torch._C.is_anomaly_enabled", + "torch._C.is_autocast_cache_enabled", + "torch._C.is_autocast_cpu_enabled", + "torch._C.is_autocast_enabled", + "torch._C.is_autocast_ipu_enabled", + "torch._C.is_autocast_xla_enabled", + "torch._C.is_grad_enabled", + "torch._C.is_inference_mode_enabled", + "torch._C.merge_type_from_type_comment", + "torch._C.parse_ir", + "torch._C.parse_schema", + "torch._C.parse_type_comment", + "torch._C.read_vitals", + "torch._C.set_vital", + "torch._C.unify_type_list", + "torch._C.vitals_enabled", + "torch._C.wait", + "torch._cast_Byte", + "torch._cast_Char", + "torch._cast_Double", + "torch._cast_Float", + "torch._cast_Half", + "torch._cast_Int", + "torch._cast_Long", + "torch._cast_Short", + "torch._choose_qparams_per_tensor", + "torch._chunk_cat", + "torch._coalesce", + "torch._compute_linear_combination", + "torch._conj_copy", + "torch._conj_physical", + "torch._conj", + "torch._convert_indices_from_coo_to_csr", + "torch._convert_indices_from_csr_to_coo", + "torch._convert_weight_to_int4pack", + "torch._convert_weight_to_int4pack_for_cpu", + "torch._convolution_mode", + "torch._convolution", + "torch._copy_from_and_resize", + "torch._copy_from", + "torch._cslt_compress", + "torch._cslt_sparse_mm", + "torch._ctc_loss", + "torch._cudnn_ctc_loss", + "torch._cudnn_init_dropout_state", + "torch._cudnn_rnn_flatten_weight", + "torch._cudnn_rnn", + "torch._cufft_clear_plan_cache", + "torch._cufft_get_plan_cache_max_size", + "torch._cufft_get_plan_cache_size", + "torch._cufft_set_plan_cache_max_size", + "torch._cummax_helper", + "torch._cummin_helper", + "torch._debug_has_internal_overlap", + "torch._dim_arange", + "torch._dirichlet_grad", + "torch._disable_functionalization", + "torch._dyn_quant_matmul_4bit", + "torch._dyn_quant_pack_4bit_weight", + "torch._efficientzerotensor", + "torch._embedding_bag_forward_only", + "torch._embedding_bag", + "torch._empty_affine_quantized", + "torch._empty_per_channel_affine_quantized", + "torch._enable_functionalization", + "torch._euclidean_dist", + "torch._fake_quantize_learnable_per_channel_affine", + "torch._fake_quantize_learnable_per_tensor_affine", + "torch._fake_quantize_per_tensor_affine_cachemask_tensor_qparams", + "torch._fft_c2c", + "torch._fft_c2r", + "torch._fft_r2c", + "torch._fill_mem_eff_dropout_mask_", + "torch._foobar", + "torch._foreach_abs_", + "torch._foreach_abs", + "torch._foreach_acos_", + "torch._foreach_acos", + "torch._foreach_add_", + "torch._foreach_add", + "torch._foreach_addcdiv_", + "torch._foreach_addcdiv", + "torch._foreach_addcmul_", + "torch._foreach_addcmul", + "torch._foreach_asin_", + "torch._foreach_asin", + "torch._foreach_atan_", + "torch._foreach_atan", + "torch._foreach_ceil_", + "torch._foreach_ceil", + "torch._foreach_clamp_max_", + "torch._foreach_clamp_max", + "torch._foreach_clamp_min_", + "torch._foreach_clamp_min", + "torch._foreach_copy_", + "torch._foreach_cos_", + "torch._foreach_cos", + "torch._foreach_cosh_", + "torch._foreach_cosh", + "torch._foreach_div_", + "torch._foreach_div", + "torch._foreach_erf_", + "torch._foreach_erf", + "torch._foreach_erfc_", + "torch._foreach_erfc", + "torch._foreach_exp_", + "torch._foreach_exp", + "torch._foreach_expm1_", + "torch._foreach_expm1", + "torch._foreach_floor_", + "torch._foreach_floor", + "torch._foreach_frac_", + "torch._foreach_frac", + "torch._foreach_lerp_", + "torch._foreach_lerp", + "torch._foreach_lgamma_", + "torch._foreach_lgamma", + "torch._foreach_log_", + "torch._foreach_log", + "torch._foreach_log10_", + "torch._foreach_log10", + "torch._foreach_log1p_", + "torch._foreach_log1p", + "torch._foreach_log2_", + "torch._foreach_log2", + "torch._foreach_maximum_", + "torch._foreach_maximum", + "torch._foreach_minimum_", + "torch._foreach_minimum", + "torch._foreach_mul_", + "torch._foreach_mul", + "torch._foreach_neg_", + "torch._foreach_neg", + "torch._foreach_norm", + "torch._foreach_pow_", + "torch._foreach_pow", + "torch._foreach_reciprocal_", + "torch._foreach_reciprocal", + "torch._foreach_round_", + "torch._foreach_round", + "torch._foreach_sigmoid_", + "torch._foreach_sigmoid", + "torch._foreach_rsqrt_", + "torch._foreach_rsqrt", + "torch._foreach_sign_", + "torch._foreach_sign", + "torch._foreach_sin_", + "torch._foreach_sin", + "torch._foreach_sinh_", + "torch._foreach_sinh", + "torch._foreach_sqrt_", + "torch._foreach_sqrt", + "torch._foreach_sub_", + "torch._foreach_sub", + "torch._foreach_tan_", + "torch._foreach_tan", + "torch._foreach_tanh_", + "torch._foreach_tanh", + "torch._foreach_trunc_", + "torch._foreach_trunc", + "torch._foreach_zero_", + "torch._freeze_functional_tensor", + "torch._from_functional_tensor", + "torch._functional_assert_async", + "torch._functional_sym_constrain_range_for_size", + "torch._functional_sym_constrain_range", + "torch._functionalize_are_all_mutations_hidden_from_autograd", + "torch._functionalize_commit_update", + "torch._functionalize_enable_reapply_views", + "torch._functionalize_has_data_mutation", + "torch._functionalize_has_metadata_mutation", + "torch._functionalize_is_multi_output_view", + "torch._functionalize_mark_mutation_hidden_from_autograd", + "torch._functionalize_replace", + "torch._functionalize_sync", + "torch._functionalize_was_storage_changed", + "torch._fused_adam_", + "torch._fused_adamw_", + "torch._fused_dropout", + "torch._fused_moving_avg_obs_fq_helper", + "torch._fused_sdp_choice", + "torch._fw_primal_copy", + "torch._grid_sampler_2d_cpu_fallback", + "torch._grouped_mm", + "torch._has_compatible_shallow_copy_type", + "torch._histogramdd_bin_edges", + "torch._histogramdd_from_bin_cts", + "torch._histogramdd_from_bin_tensors", + "torch._index_put_impl_", + "torch._indices_copy", + "torch._int_mm", + "torch._is_all_true", + "torch._is_any_true", + "torch._is_functional_tensor", + "torch._is_zerotensor", + "torch._linalg_check_errors", + "torch._linalg_det", + "torch._linalg_eigh", + "torch._linalg_eigvals", + "torch._linalg_slogdet", + "torch._linalg_solve_ex", + "torch._linalg_svd", + "torch._log_softmax_backward_data", + "torch._log_softmax", + "torch._logcumsumexp", + "torch._lstm_mps", + "torch._lu_with_info", + "torch._make_dep_token", + "torch._make_dual_copy", + "torch._make_dual", + "torch._make_per_channel_quantized_tensor", + "torch._make_per_tensor_quantized_tensor", + "torch._masked_scale", + "torch._masked_softmax", + "torch._mirror_autograd_meta_to", + "torch._mixed_dtypes_linear", + "torch._mkldnn_reshape", + "torch._mkldnn_transpose_", + "torch._mkldnn_transpose", + "torch._mps_convolution_transpose", + "torch._mps_convolution", + "torch._native_batch_norm_legit_no_training", + "torch._native_batch_norm_legit", + "torch._native_multi_head_attention", + "torch._neg_view_copy", + "torch._neg_view", + "torch._nested_from_padded_and_nested_example", + "torch._nested_from_padded_tensor", + "torch._nested_tensor_from_mask_left_aligned", + "torch._nested_tensor_from_tensor_list", + "torch._nested_tensor_softmax_with_shape", + "torch._nested_view_from_buffer_copy", + "torch._nested_view_from_buffer", + "torch._nnpack_available", + "torch._nnpack_spatial_convolution", + "torch._pack_padded_sequence", + "torch._pad_packed_sequence", + "torch._pin_memory", + "torch._prelu_kernel", + "torch._propagate_xla_data", + "torch._remove_batch_dim", + "torch._reshape_alias_copy", + "torch._reshape_from_tensor", + "torch._resize_output_", + "torch._rowwise_prune", + "torch._sample_dirichlet", + "torch._saturate_weight_to_fp16", + "torch._scaled_dot_product_attention_math", + "torch._scaled_dot_product_efficient_attention", + "torch._scaled_dot_product_flash_attention", + "torch._scaled_dot_product_flash_attention_for_cpu", + "torch._scaled_dot_product_cudnn_attention", + "torch._scaled_mm", + "torch._scaled_grouped_mm", + "torch._shape_as_tensor", + "torch._sobol_engine_draw", + "torch._sobol_engine_ff_", + "torch._sobol_engine_initialize_state_", + "torch._sobol_engine_scramble_", + "torch._softmax_backward_data", + "torch._softmax", + "torch._sparse_broadcast_to_copy", + "torch._sparse_broadcast_to", + "torch._sparse_csr_prod", + "torch._sparse_csr_sum", + "torch._sparse_log_softmax_backward_data", + "torch._sparse_semi_structured_addmm", + "torch._sparse_semi_structured_linear", + "torch._sparse_semi_structured_mm", + "torch._sparse_softmax_backward_data", + "torch._sparse_sparse_matmul", + "torch._sparse_sum", + "torch._stack", + "torch._standard_gamma_grad", + "torch._standard_gamma", + "torch._test_autograd_multiple_dispatch_view_copy", + "torch._test_autograd_multiple_dispatch_view", + "torch._test_autograd_multiple_dispatch", + "torch._test_check_tensor", + "torch._test_functorch_fallback", + "torch._test_serialization_subcmul", + "torch._to_cpu", + "torch._to_functional_tensor", + "torch._to_sparse_semi_structured", + "torch._transform_bias_rescale_qkv", + "torch._transformer_encoder_layer_fwd", + "torch._trilinear", + "torch._triton_multi_head_attention", + "torch._triton_scaled_dot_attention", + "torch._unique", + "torch._unique2", + "torch._unpack_dual", + "torch._unsafe_index_put", + "torch._unsafe_index", + "torch._unsafe_masked_index_put_accumulate", + "torch._unsafe_masked_index", + "torch._use_cudnn_ctc_loss", + "torch._use_cudnn_rnn_flatten_weight", + "torch._values_copy", + "torch._weight_int4pack_mm", + "torch._weight_int4pack_mm_for_cpu", + "torch._weight_int4pack_mm_with_scales_and_zeros", + "torch._weight_int8pack_mm", + "torch._weight_norm_interface", + "torch._weight_norm", + "torch.abs_", + "torch.abs", + "torch.absolute", + "torch.acos_", + "torch.acos", + "torch.acosh_", + "torch.acosh", + "torch.adaptive_avg_pool1d", + "torch.adaptive_max_pool1d", + "torch.add", + "torch.addbmm", + "torch.addcdiv", + "torch.addcmul", + "torch.addmm", + "torch.addmv_", + "torch.addmv", + "torch.addr", + "torch.adjoint", + "torch.affine_grid_generator", + "torch.alias_copy", + "torch.all", + "torch.allclose", + "torch.alpha_dropout_", + "torch.alpha_dropout", + "torch.amax", + "torch.amin", + "torch.aminmax", + "torch.angle", + "torch.any", + "torch.arange", + "torch.arccos_", + "torch.arccos", + "torch.arccosh_", + "torch.arccosh", + "torch.arcsin_", + "torch.arcsin", + "torch.arcsinh_", + "torch.arcsinh", + "torch.arctan_", + "torch.arctan", + "torch.arctan2", + "torch.arctanh_", + "torch.arctanh", + "torch.argmax", + "torch.argmin", + "torch.argsort", + "torch.argwhere", + "torch.as_strided_", + "torch.as_strided_copy", + "torch.as_strided_scatter", + "torch.as_strided", + "torch.as_tensor", + "torch.asarray", + "torch.asin_", + "torch.asin", + "torch.asinh_", + "torch.asinh", + "torch.atan_", + "torch.atan", + "torch.atan2", + "torch.atanh_", + "torch.atanh", + "torch.avg_pool1d", + "torch.baddbmm", + "torch.bartlett_window", + "torch.batch_norm_backward_elemt", + "torch.batch_norm_backward_reduce", + "torch.batch_norm_elemt", + "torch.batch_norm_gather_stats_with_counts", + "torch.batch_norm_gather_stats", + "torch.batch_norm_stats", + "torch.batch_norm_update_stats", + "torch.batch_norm", + "torch.bernoulli", + "torch.bilinear", + "torch.binary_cross_entropy_with_logits", + "torch.bincount", + "torch.binomial", + "torch.bitwise_and", + "torch.bitwise_left_shift", + "torch.bitwise_not", + "torch.bitwise_or", + "torch.bitwise_right_shift", + "torch.bitwise_xor", + "torch.blackman_window", + "torch.bmm", + "torch.broadcast_to", + "torch.bucketize", + "torch.can_cast", + "torch.cat", + "torch.ccol_indices_copy", + "torch.ceil_", + "torch.ceil", + "torch.celu_", + "torch.celu", + "torch.channel_shuffle", + "torch.cholesky_inverse", + "torch.cholesky_solve", + "torch.cholesky", + "torch.choose_qparams_optimized", + "torch.chunk", + "torch.clamp_", + "torch.clamp_max_", + "torch.clamp_max", + "torch.clamp_min_", + "torch.clamp_min", + "torch.clamp", + "torch.clip_", + "torch.clip", + "torch.clone", + "torch.col_indices_copy", + "torch.column_stack", + "torch.combinations", + "torch.complex", + "torch.concat", + "torch.concatenate", + "torch.conj_physical_", + "torch.conj_physical", + "torch.conj", + "torch.constant_pad_nd", + "torch.conv_tbc", + "torch.conv_transpose1d", + "torch.conv_transpose2d", + "torch.conv_transpose3d", + "torch.conv1d", + "torch.conv2d", + "torch.conv3d", + "torch.convolution", + "torch.copysign", + "torch.corrcoef", + "torch.cos_", + "torch.cos", + "torch.cosh_", + "torch.cosh", + "torch.cosine_embedding_loss", + "torch.cosine_similarity", + "torch.count_nonzero", + "torch.cov", + "torch.cross", + "torch.crow_indices_copy", + "torch.ctc_loss", + "torch.cudnn_affine_grid_generator", + "torch.cudnn_batch_norm", + "torch.cudnn_convolution_add_relu", + "torch.cudnn_convolution_relu", + "torch.cudnn_convolution_transpose", + "torch.cudnn_convolution", + "torch.cudnn_grid_sampler", + "torch.cudnn_is_acceptable", + "torch.cummax", + "torch.cummin", + "torch.cumprod", + "torch.cumsum", + "torch.cumulative_trapezoid", + "torch.deg2rad_", + "torch.deg2rad", + "torch.dequantize", + "torch.det", + "torch.detach_", + "torch.detach_copy", + "torch.detach", + "torch.diag_embed", + "torch.diag", + "torch.diagflat", + "torch.diagonal_copy", + "torch.diagonal_scatter", + "torch.diagonal", + "torch.diff", + "torch.digamma", + "torch.dist", + "torch.div", + "torch.divide", + "torch.dot", + "torch.dropout_", + "torch.dropout", + "torch.dsmm", + "torch.dsplit", + "torch.dstack", + "torch.embedding_bag", + "torch.embedding_renorm_", + "torch.embedding", + "torch.empty_like", + "torch.empty_permuted", + "torch.empty_quantized", + "torch.empty_strided", + "torch.empty", + "torch.eq", + "torch.equal", + "torch.erf_", + "torch.erf", + "torch.erfc_", + "torch.erfc", + "torch.erfinv", + "torch.exp_", + "torch.exp", + "torch.exp2_", + "torch.exp2", + "torch.expand_copy", + "torch.expm1_", + "torch.expm1", + "torch.eye", + "torch.fake_quantize_per_channel_affine", + "torch.fake_quantize_per_tensor_affine", + "torch.fbgemm_linear_fp16_weight_fp32_activation", + "torch.fbgemm_linear_fp16_weight", + "torch.fbgemm_linear_int8_weight_fp32_activation", + "torch.fbgemm_linear_int8_weight", + "torch.fbgemm_linear_quantize_weight", + "torch.fbgemm_pack_gemm_matrix_fp16", + "torch.fbgemm_pack_quantized_matrix", + "torch.feature_alpha_dropout_", + "torch.feature_alpha_dropout", + "torch.feature_dropout_", + "torch.feature_dropout", + "torch.fill_", + "torch.fill", + "torch.fix_", + "torch.fix", + "torch.flatten", + "torch.flip", + "torch.fliplr", + "torch.flipud", + "torch.float_power", + "torch.floor_", + "torch.floor_divide", + "torch.floor", + "torch.fmax", + "torch.fmin", + "torch.fmod", + "torch.frac_", + "torch.frac", + "torch.frexp", + "torch.frobenius_norm", + "torch.from_file", + "torch.from_numpy", + "torch.frombuffer", + "torch.full_like", + "torch.full", + "torch.fused_moving_avg_obs_fake_quant", + "torch.gather", + "torch.gcd_", + "torch.gcd", + "torch.ge", + "torch.geqrf", + "torch.ger", + "torch.get_device", + "torch.get_device_module", + "torch.gradient", + "torch.greater_equal", + "torch.greater", + "torch.grid_sampler_2d", + "torch.grid_sampler_3d", + "torch.grid_sampler", + "torch.group_norm", + "torch.gru_cell", + "torch.gru", + "torch.gt", + "torch.hamming_window", + "torch.hann_window", + "torch.hardshrink", + "torch.hash_tensor", + "torch.heaviside", + "torch.hinge_embedding_loss", + "torch.histc", + "torch.histogram", + "torch.histogramdd", + "torch.hsmm", + "torch.hsplit", + "torch.hspmm", + "torch.hstack", + "torch.hypot", + "torch.i0_", + "torch.i0", + "torch.igamma", + "torch.igammac", + "torch.imag", + "torch.index_add", + "torch.index_copy", + "torch.index_fill", + "torch.index_put_", + "torch.index_put", + "torch.index_reduce", + "torch.index_select", + "torch.indices_copy", + "torch.inner", + "torch.instance_norm", + "torch.int_repr", + "torch.inverse", + "torch.is_complex", + "torch.is_conj", + "torch.is_distributed", + "torch.is_floating_point", + "torch.is_inference", + "torch.is_neg", + "torch.is_nonzero", + "torch.is_same_size", + "torch.is_signed", + "torch.is_vulkan_available", + "torch.isclose", + "torch.isfinite", + "torch.isin", + "torch.isinf", + "torch.isnan", + "torch.isneginf", + "torch.isposinf", + "torch.isreal", + "torch.istft", + "torch.kaiser_window", + "torch.kl_div", + "torch.kron", + "torch.kthvalue", + "torch.layer_norm", + "torch.lcm_", + "torch.lcm", + "torch.ldexp_", + "torch.ldexp", + "torch.le", + "torch.lerp", + "torch.less_equal", + "torch.less", + "torch.lgamma", + "torch.linspace", + "torch.log_", + "torch.log_softmax", + "torch.log", + "torch.log10_", + "torch.log10", + "torch.log1p_", + "torch.log1p", + "torch.log2_", + "torch.log2", + "torch.logaddexp", + "torch.logaddexp2", + "torch.logcumsumexp", + "torch.logdet", + "torch.logical_and", + "torch.logical_not", + "torch.logical_or", + "torch.logical_xor", + "torch.logit_", + "torch.logit", + "torch.logspace", + "torch.logsumexp", + "torch.lstm_cell", + "torch.lstm", + "torch.lt", + "torch.lu_solve", + "torch.lu_unpack", + "torch.margin_ranking_loss", + "torch.masked_fill", + "torch.masked_scatter", + "torch.masked_select", + "torch.matmul", + "torch.matrix_exp", + "torch.matrix_power", + "torch.max_pool1d_with_indices", + "torch.max_pool1d", + "torch.max_pool2d", + "torch.max_pool3d", + "torch.max", + "torch.maximum", + "torch.mean", + "torch.median", + "torch.min", + "torch.minimum", + "torch.miopen_batch_norm", + "torch.miopen_convolution_add_relu", + "torch.miopen_convolution_relu", + "torch.miopen_convolution_transpose", + "torch.miopen_convolution", + "torch.miopen_depthwise_convolution", + "torch.miopen_rnn", + "torch.mkldnn_adaptive_avg_pool2d", + "torch.mkldnn_convolution", + "torch.mkldnn_linear_backward_weights", + "torch.mkldnn_max_pool2d", + "torch.mkldnn_max_pool3d", + "torch.mkldnn_rnn_layer", + "torch.mm", + "torch.mode", + "torch.moveaxis", + "torch.movedim", + "torch.msort", + "torch.mul", + "torch.multinomial", + "torch.multiply", + "torch.mv", + "torch.mvlgamma", + "torch.nan_to_num_", + "torch.nan_to_num", + "torch.nanmean", + "torch.nanmedian", + "torch.nanquantile", + "torch.nansum", + "torch.narrow_copy", + "torch.narrow", + "torch.native_batch_norm", + "torch.native_channel_shuffle", + "torch.native_dropout", + "torch.native_group_norm", + "torch.native_layer_norm", + "torch.native_norm", + "torch.ne", + "torch.neg_", + "torch.neg", + "torch.negative_", + "torch.negative", + "torch.nextafter", + "torch.nonzero_static", + "torch.nonzero", + "torch.norm_except_dim", + "torch.normal", + "torch.not_equal", + "torch.nuclear_norm", + "torch.numel", + "torch.ones_like", + "torch.ones", + "torch.orgqr", + "torch.ormqr", + "torch.outer", + "torch.pairwise_distance", + "torch.pdist", + "torch.permute_copy", + "torch.permute", + "torch.pinverse", + "torch.pixel_shuffle", + "torch.pixel_unshuffle", + "torch.poisson_nll_loss", + "torch.poisson", + "torch.polar", + "torch.polygamma", + "torch.positive", + "torch.pow", + "torch.prelu", + "torch._print", + "torch.prod", + "torch.promote_types", + "torch.put", + "torch.q_per_channel_axis", + "torch.q_per_channel_scales", + "torch.q_per_channel_zero_points", + "torch.q_scale", + "torch.q_zero_point", + "torch.qr", + "torch.quantile", + "torch.quantize_per_channel", + "torch.quantize_per_tensor_dynamic", + "torch.quantize_per_tensor", + "torch.quantized_batch_norm", + "torch.quantized_gru_cell", + "torch.quantized_lstm_cell", + "torch.quantized_max_pool1d", + "torch.quantized_max_pool2d", + "torch.quantized_max_pool3d", + "torch.quantized_rnn_relu_cell", + "torch.quantized_rnn_tanh_cell", + "torch.rad2deg_", + "torch.rad2deg", + "torch.rand_like", + "torch.rand", + "torch.randint_like", + "torch.randint", + "torch.randn_like", + "torch.randn", + "torch.randperm", + "torch.range", + "torch.ravel", + "torch.real", + "torch.reciprocal_", + "torch.reciprocal", + "torch.relu_", + "torch.relu", + "torch.remainder", + "torch.renorm", + "torch.repeat_interleave", + "torch.reshape", + "torch.resolve_conj", + "torch.resolve_neg", + "torch.result_type", + "torch.rms_norm", + "torch.rnn_relu_cell", + "torch.rnn_relu", + "torch.rnn_tanh_cell", + "torch.rnn_tanh", + "torch.roll", + "torch.rot90", + "torch.round_", + "torch.round", + "torch.row_indices_copy", + "torch.row_stack", + "torch.rrelu_", + "torch.rrelu", + "torch.rsqrt_", + "torch.rsqrt", + "torch.rsub", + "torch.saddmm", + "torch.scalar_tensor", + "torch.scatter_add", + "torch.scatter_reduce", + "torch.scatter", + "torch.searchsorted", + "torch.segment_reduce", + "torch.select_copy", + "torch.select_scatter", + "torch.select", + "torch.selu_", + "torch.selu", + "torch.sgn", + "torch.sigmoid_", + "torch.sigmoid", + "torch.sign", + "torch.signal.windows.windows.sqrt", + "torch.signbit", + "torch.sin_", + "torch.sin", + "torch.sinc_", + "torch.sinc", + "torch.sinh_", + "torch.sinh", + "torch.slice_copy", + "torch.slice_scatter", + "torch.slogdet", + "torch.smm", + "torch.softmax", + "torch.sort", + "torch.split_copy", + "torch.split_with_sizes_copy", + "torch.split_with_sizes", + "torch.spmm", + "torch.sqrt_", + "torch.sqrt", + "torch.square_", + "torch.square", + "torch.squeeze_copy", + "torch.squeeze", + "torch.sspaddmm", + "torch.stack", + "torch.std_mean", + "torch.std", + "torch.sub", + "torch.subtract", + "torch.sum", + "torch.svd", + "torch.swapaxes", + "torch.swapdims", + "torch.sym_constrain_range_for_size", + "torch.sym_constrain_range", + "torch.t_copy", + "torch.t", + "torch.take_along_dim", + "torch.take", + "torch.tan_", + "torch.tan", + "torch.tanh_", + "torch.tanh", + "torch.tensor_split", + "torch.tensor", + "torch.threshold_", + "torch.threshold", + "torch.tile", + "torch.topk", + "torch.trace", + "torch.transpose_copy", + "torch.transpose", + "torch.trapezoid", + "torch.trapz", + "torch.triangular_solve", + "torch.tril_indices", + "torch.tril", + "torch.triplet_margin_loss", + "torch.triu_indices", + "torch.triu", + "torch.true_divide", + "torch.trunc_", + "torch.trunc", + "torch.unbind_copy", + "torch.unbind", + "torch.unflatten", + "torch.unfold_copy", + "torch.unsafe_chunk", + "torch.unsafe_split_with_sizes", + "torch.unsafe_split", + "torch.unsqueeze_copy", + "torch.unsqueeze", + "torch.values_copy", + "torch.vander", + "torch.var_mean", + "torch.var", + "torch.vdot", + "torch.view_as_complex_copy", + "torch.view_as_complex", + "torch.view_as_real_copy", + "torch.view_as_real", + "torch.view_copy", + "torch.vsplit", + "torch.vstack", + "torch.where", + "torch.xlogy_", + "torch.xlogy", + "torch.zero_", + "torch.zeros", + "torch.zeros_like", + "torch._fused_sgd_", + "torch.slice_inverse", + "torch._assert_scalar", + "torch._functional_assert_scalar", + "torch.xpu._get_device_properties", + ], + TorchInGraphFunctionVariable, +) + + +if sys.version_info >= (3, 11): + torch_c_binding_in_graph_functions["math.exp2"] = TorchInGraphFunctionVariable + torch_c_binding_in_graph_functions["math.cbrt"] = TorchInGraphFunctionVariable + +if sys.version_info >= (3, 13): + torch_c_binding_in_graph_functions["math.fma"] = TorchInGraphFunctionVariable + +# In graph functions (including constant folding) that are not C bindings +# NOTE: [Cacheability of in-graph torch functions] +# Functions in this list have the property that graphs containing them are safe to cache/serialize. +# serialize given only the information in the graph. I.e, either: +# - Your function does not access or close over global state, or +# - Your function closes over global state, but this state is guarded by dynamo, either +# through constant folding or other mechanisms +# If your function needs a custom special handler (via @register on TorchInGraphFunctionVariable), +# or captures global state, please add it to manual_torch_name_rule_map instead +torch_non_c_binding_in_graph_functions = dict.fromkeys( + [ + "torch.__future__.get_overwrite_module_params_on_conversion", + "torch.__future__.set_overwrite_module_params_on_conversion", + "torch.__getattr__", + "torch._assert", + "torch._check_index", + "torch._check_is_size", + "torch._check_not_implemented", + "torch._check_tensor_all_with", + "torch._check_tensor_all", + "torch._check_type", + "torch._check_value", + "torch._check_with", + "torch._compile._disable_dynamo", + "torch._functorch.apis.chunk_vmap", + "torch._functorch.batch_norm_replacement.batch_norm_without_running_stats", + "torch._functorch.batch_norm_replacement.replace_all_batch_norm_modules_", + "torch._functorch.deprecated.combine_state_for_ensemble", + "torch._functorch.deprecated.functionalize", + "torch._functorch.deprecated.get_warning", + "torch._functorch.deprecated.make_functional_with_buffers", + "torch._functorch.deprecated.make_functional", + "torch._functorch.deprecated.setup_docs", + "torch._functorch.deprecated.warn_deprecated", + "torch._functorch.eager_transforms._any_differentiable", + "torch._functorch.eager_transforms._autograd_grad", + "torch._functorch.eager_transforms._set_tensor_requires_grad", + "torch._functorch.eager_transforms._is_differentiable", + "torch._functorch.eager_transforms._maybe_unwrap_functional_tensor", + "torch._functorch.eager_transforms._maybe_wrap_functional_tensor", + "torch._functorch.eager_transforms._unwrap_all_tensors_from_functional", + "torch._functorch.eager_transforms._wrap_all_tensors_to_functional", + "torch._functorch.eager_transforms.assert_flat_tuple_of_tensors", + "torch._functorch.eager_transforms.functionalize", + "torch._functorch.eager_transforms.lazy_dynamo_disable", + "torch._functorch.eager_transforms.noop", + "torch._functorch.utils.enable_single_level_autograd_function", + "torch._functorch.utils.exposed_in", + "torch._functorch.utils.unwrap_dead_wrappers", + "torch._functorch.predispatch.lazy_load_decompositions", + "torch._functorch.predispatch._vmap_increment_nesting", + "torch._functorch.predispatch._vmap_decrement_nesting", + "torch._functorch.predispatch._add_batch_dim", + "torch._functorch.predispatch._remove_batch_dim", + "torch._guards.compile_context", + "torch._guards.detect_fake_mode", + "torch._guards.tracing", + "torch._higher_order_ops.map._has_potential_branch_input_alias", + "torch._higher_order_ops.map._has_potential_branch_input_mutation", + "torch._higher_order_ops.map._stack_pytree", + "torch._higher_order_ops.map._unstack_pytree", + "torch._higher_order_ops.map.create_fw_bw_graph", + "torch._higher_order_ops.map.map_autograd", + "torch._higher_order_ops.map.map_dense", + "torch._higher_order_ops.map.map_fake_tensor_mode", + "torch._higher_order_ops.map.map_functionalize", + "torch._higher_order_ops.map.map_proxy_torch_dispatch_mode", + "torch._higher_order_ops.map.map_wrapper", + "torch._higher_order_ops.map.trace_map", + "torch._higher_order_ops.out_dtype.elementwise_dtypes", + "torch._higher_order_ops.out_dtype.is_int_mm", + "torch._higher_order_ops.out_dtype.out_dtype_dense", + "torch._higher_order_ops.out_dtype.out_dtype_fake_tensor_mode", + "torch._higher_order_ops.out_dtype.out_dtype_fallback", + "torch._higher_order_ops.out_dtype.out_dtype_func", + "torch._higher_order_ops.out_dtype.out_dtype_proxy", + "torch._higher_order_ops.out_dtype.trace_out_dtype", + "torch._higher_order_ops.utils.autograd_not_implemented_inner", + "torch._higher_order_ops.utils.autograd_not_implemented", + "torch._linalg_utils._symeig", + "torch._linalg_utils.basis", + "torch._linalg_utils.bform", + "torch._linalg_utils.eig", + "torch._linalg_utils.get_floating_dtype", + "torch._linalg_utils.is_sparse", + "torch._linalg_utils.lstsq", + "torch._linalg_utils.matmul", + "torch._linalg_utils.matrix_rank", + "torch._linalg_utils.qform", + "torch._linalg_utils.solve", + "torch._linalg_utils.symeig", + "torch._load_global_deps", + "torch._lowrank._svd_lowrank", + "torch._lowrank.get_approximate_basis", + "torch._lowrank.pca_lowrank", + "torch._lowrank.svd_lowrank", + "torch._preload_cuda_deps", + "torch._register_device_module", + "torch._utils._dummy_type", + "torch._utils._flatten_dense_tensors", + "torch._utils._unflatten_dense_tensors", + "torch._weights_only_unpickler._get_allowed_globals", + "torch._weights_only_unpickler.load", + "torch.accelerator.current_accelerator", + "torch.accelerator.current_device_index", + "torch.accelerator.current_stream", + "torch.accelerator.device_count", + "torch.accelerator.is_available", + "torch.accelerator.set_stream", + "torch.accelerator.synchronize", + "torch.align_tensors", + "torch.amp.autocast_mode._enter_autocast", + "torch.amp.autocast_mode._exit_autocast", + "torch.amp.autocast_mode.autocast_decorator", + "torch.amp.autocast_mode.custom_bwd", + "torch.amp.autocast_mode.custom_fwd", + "torch.are_deterministic_algorithms_enabled", + "torch.atleast_1d", + "torch.atleast_2d", + "torch.atleast_3d", + "torch.autograd._calculate_shape", + "torch.autograd._is_checkpoint_valid", + "torch.autograd._profiler_enabled", + "torch.autograd._make_grads", + "torch.autograd._register_py_tensor_class_for_device", + "torch.autograd._tensor_or_tensors_to_tuple", + "torch.autograd.forward_ad._maybe_load_decompositions", + "torch.autograd.function._iter_filter", + "torch.autograd.function._iter_jit_values", + "torch.autograd.function._iter_None_tensors", + "torch.autograd.function._iter_tensors_permissive", + "torch.autograd.function._iter_tensors", + "torch.autograd.function._jit_unwrap_structured", + "torch.autograd.function._map_tensor_data", + "torch.autograd.function._nested_map", + "torch.autograd.function._unflatten", + "torch.autograd.function.once_differentiable", + "torch.autograd.function.traceable", + "torch.autograd.functional._as_tuple_nocheck", + "torch.autograd.functional._as_tuple", + "torch.autograd.functional._autograd_grad", + "torch.autograd.functional._check_requires_grad", + "torch.autograd.functional._construct_standard_basis_for", + "torch.autograd.functional._fill_in_zeros", + "torch.autograd.functional._grad_postprocess", + "torch.autograd.functional._grad_preprocess", + "torch.autograd.functional._jacfwd", + "torch.autograd.functional._tuple_postprocess", + "torch.autograd.functional._validate_v", + "torch.autograd.functional.hessian", + "torch.autograd.functional.hvp", + "torch.autograd.functional.jacobian", + "torch.autograd.functional.jvp", + "torch.autograd.functional.vhp", + "torch.autograd.functional.vjp", + "torch.autograd.grad_mode._enter_inference_mode", + "torch.autograd.grad_mode._exit_inference_mode", + "torch.autograd.graph._get_sid", + "torch.autograd.graph._get_tid", + "torch.autograd.graph.allow_mutation_on_saved_tensors", + "torch.autograd.graph.get_gradient_edge", + "torch.autograd.graph.increment_version", + "torch.autograd.graph.register_multi_grad_hook", + "torch.autograd.variable", + "torch.backends.__allow_nonbracketed_mutation", + "torch.backends.cpu.get_cpu_capability", + "torch.backends.cuda.can_use_efficient_attention", + "torch.backends.cuda.can_use_flash_attention", + "torch.backends.cuda.can_use_cudnn_attention", + "torch.backends.cuda.enable_flash_sdp", + "torch.backends.cuda.enable_math_sdp", + "torch.backends.cuda.allow_fp16_bf16_reduction_math_sdp", + "torch.backends.cuda.enable_mem_efficient_sdp", + "torch.backends.cuda.flash_sdp_enabled", + "torch.backends.cuda.is_built", + "torch.backends.cuda.is_flash_attention_available", + "torch.backends.cuda.math_sdp_enabled", + "torch.backends.cuda.fp16_bf16_reduction_math_sdp_allowed", + "torch.backends.cuda.mem_efficient_sdp_enabled", + "torch.backends.cuda.cudnn_sdp_enabled", + "torch.backends.cuda.enable_cudnn_sdp", + "torch.backends.cuda.preferred_blas_library", + "torch.backends.cuda.preferred_linalg_library", + "torch.backends.cuda.preferred_rocm_fa_library", + "torch.backends.cuda.sdp_kernel", + "torch.backends.cudnn._init", + "torch.backends.cudnn.flags", + "torch.backends.cudnn.is_acceptable", + "torch.backends.cudnn.is_available", + "torch.backends.cudnn.set_flags", + "torch.backends.cudnn.version", + "torch.backends.disable_global_flags", + "torch.backends.flags_frozen", + "torch.backends.mkl.is_available", + "torch.backends.mkldnn.flags", + "torch.backends.mkldnn.is_available", + "torch.backends.mkldnn.set_flags", + "torch.backends.mps._init", + "torch.backends.mps.is_available", + "torch.backends.mps.is_built", + "torch.backends.mps.is_macos13_or_newer", + "torch.backends.openmp.is_available", + "torch.backends.quantized._get_qengine_id", + "torch.backends.quantized._get_qengine_str", + "torch.block_diag", + "torch.broadcast_tensors", + "torch.cartesian_prod", + "torch.cdist", + "torch.chain_matmul", + "torch.compile", + "torch.compiled_with_cxx11_abi", + "torch._C._cpu._is_avx2_supported", + "torch._C._cpu._is_avx512_supported", + "torch._C._cpu._is_avx512_vnni_supported", + "torch._C._cpu._is_avx512_bf16_supported", + "torch._C._cpu._is_amx_tile_supported", + "torch._C._cpu._is_amx_fp16_supported", + "torch.cpu._init_amx", + "torch.cpu.current_device", + "torch.cpu.current_stream", + "torch.cpu.device_count", + "torch.cpu.is_available", + "torch.cpu.set_device", + "torch.cpu.stream", + "torch.cpu.synchronize", + "torch.cuda._check_capability", + "torch.cuda._check_cubins", + "torch.cuda._device_count_amdsmi", + "torch.cuda._device_count_nvml", + "torch.cuda._get_amdsmi_handler", + "torch.cuda._get_amdsmi_device_index", + "torch.cuda._get_device", + "torch.cuda._get_generator", + "torch.cuda._get_nvml_device_index", + "torch.cuda._get_pynvml_handler", + "torch.cuda._get_rng_state_offset", + "torch.cuda._is_compiled", + "torch.cuda._lazy_call", + "torch.cuda._lazy_init", + "torch.cuda._memory_viz._block_extra_legacy", + "torch.cuda._memory_viz._block_extra", + "torch.cuda._memory_viz._format_size", + "torch.cuda._memory_viz._format_viz", + "torch.cuda._memory_viz._frame_filter", + "torch.cuda._memory_viz._frame_fmt", + "torch.cuda._memory_viz._frames_fmt", + "torch.cuda._memory_viz._profile_to_snapshot", + "torch.cuda._memory_viz._report_free", + "torch.cuda._memory_viz._write_blocks", + "torch.cuda._memory_viz.calc_active", + "torch.cuda._memory_viz.compare", + "torch.cuda._memory_viz.format_flamegraph", + "torch.cuda._memory_viz.memory", + "torch.cuda._memory_viz.profile_plot", + "torch.cuda._memory_viz.segment_plot", + "torch.cuda._memory_viz.segments", + "torch.cuda._memory_viz.segsum", + "torch.cuda._memory_viz.trace_plot", + "torch.cuda._memory_viz.trace", + "torch.cuda._nvml_based_avail", + "torch.cuda._parse_visible_devices", + "torch.cuda._raw_device_count_amdsmi", + "torch.cuda._raw_device_count_nvml", + "torch.cuda._raw_device_uuid_amdsmi", + "torch.cuda._raw_device_uuid_nvml", + "torch.cuda._register_triton_kernels", + "torch.cuda._set_rng_state_offset", + "torch.cuda._set_stream_by_id", + "torch.cuda._sleep", + "torch.cuda._busy_wait_for_flag", + "torch.cuda._clear_flag", + "torch.cuda._transform_uuid_to_ordinals", + "torch.cuda._utils._get_device_index", + "torch.cuda.amp.autocast_mode._cast", + "torch.cuda.amp.autocast_mode.custom_bwd", + "torch.cuda.amp.autocast_mode.custom_fwd", + "torch.cuda.amp.common.amp_definitely_not_available", + "torch.amp.grad_scaler._refresh_per_optimizer_state", + "torch.cuda.can_device_access_peer", + "torch.cuda.check_error", + "torch.cuda.clock_rate", + "torch.cuda.cudart", + "torch.cuda.current_blas_handle", + "torch.cuda.current_stream", + "torch.cuda.default_stream", + "torch.cuda.device_count", + "torch.cuda.device_memory_used", + "torch.cuda.get_arch_list", + "torch.cuda.get_device_capability", + "torch.cuda.get_device_name", + "torch.cuda.get_device_properties", + "torch.cuda.get_gencode_flags", + "torch.cuda.get_sync_debug_mode", + "torch.cuda.graphs.graph_pool_handle", + "torch.cuda.graphs.is_current_stream_capturing", + "torch.cuda.graphs.make_graphed_callables", + "torch.cuda.init", + "torch.cuda.ipc_collect", + "torch.cuda.is_available", + "torch.cuda.is_bf16_supported", + "torch.cuda.is_initialized", + "torch.cuda.jiterator._create_jit_fn", + "torch.cuda.jiterator._create_multi_output_jit_fn", + "torch.cuda.memory_usage", + "torch.cuda.memory._dump_snapshot", + "torch.cuda.memory._free_mutex", + "torch.cuda.memory._get_current_allocator", + "torch.cuda.memory._host_allocator", + "torch.cuda.memory._record_memory_history_impl", + "torch.cuda.memory._record_memory_history_legacy", + "torch.cuda.memory._record_memory_history", + "torch.cuda.memory._save_memory_usage", + "torch.cuda.memory._save_segment_usage", + "torch.cuda.memory._set_allocator_settings", + "torch.cuda.memory._snapshot", + "torch.cuda.memory.caching_allocator_alloc", + "torch.cuda.memory.caching_allocator_delete", + "torch.cuda.memory.caching_allocator_enable", + "torch.cuda.memory.change_current_allocator", + "torch.cuda.memory.empty_cache", + "torch.cuda.memory.get_allocator_backend", + "torch.cuda.memory.get_per_process_memory_fraction", + "torch.cuda.memory.host_memory_stats_as_nested_dict", + "torch.cuda.memory.host_memory_stats", + "torch.cuda.memory.list_gpu_processes", + "torch.cuda.memory.max_memory_allocated", + "torch.cuda.memory.max_memory_cached", + "torch.cuda.memory.max_memory_reserved", + "torch.cuda.memory.mem_get_info", + "torch.cuda.memory.memory_allocated", + "torch.cuda.memory.memory_cached", + "torch.cuda.memory.memory_reserved", + "torch.cuda.memory.memory_snapshot", + "torch.cuda.memory.memory_stats_as_nested_dict", + "torch.cuda.memory.memory_stats", + "torch.cuda.memory.memory_summary", + "torch.cuda.memory.reset_accumulated_host_memory_stats", + "torch.cuda.memory.reset_accumulated_memory_stats", + "torch.cuda.memory.reset_max_memory_allocated", + "torch.cuda.memory.reset_max_memory_cached", + "torch.cuda.memory.reset_peak_host_memory_stats", + "torch.cuda.memory.reset_peak_memory_stats", + "torch.cuda.memory.set_per_process_memory_fraction", + "torch.cuda.nccl._check_sequence_type", + "torch.cuda.nccl.all_gather", + "torch.cuda.nccl.all_reduce", + "torch.cuda.nccl.broadcast", + "torch.cuda.nccl.init_rank", + "torch.cuda.nccl.is_available", + "torch.cuda.nccl.reduce_scatter", + "torch.cuda.nccl.reduce", + "torch.cuda.nccl.unique_id", + "torch.cuda.nccl.version", + "torch.cuda.nvtx.mark", + "torch.cuda.nvtx.range_end", + "torch.cuda.nvtx.range_pop", + "torch.cuda.nvtx.range_push", + "torch.cuda.nvtx.range_start", + "torch.cuda.nvtx.range", + "torch.cuda.power_draw", + "torch.cuda.profiler.init", + "torch.cuda.profiler.profile", + "torch.cuda.profiler.start", + "torch.cuda.profiler.stop", + "torch.cuda.random.get_rng_state_all", + "torch.cuda.random.initial_seed", + "torch.cuda.random.manual_seed_all", + "torch.cuda.random.manual_seed", + "torch.cuda.random.seed_all", + "torch.cuda.random.seed", + "torch.cuda.random.set_rng_state_all", + "torch.cuda.set_stream", + "torch.cuda.set_sync_debug_mode", + "torch.cuda.stream", + "torch.cuda.temperature", + "torch.cuda.utilization", + "torch.einsum", + "torch.functional._check_list_size", + "torch.functional._consecutive_return_counts", + "torch.functional._consecutive_return_inverse_false", + "torch.functional._consecutive_return_inverse_true", + "torch.functional._consecutive_return_inverse", + "torch.functional._consecutive_return_output", + "torch.functional._lu_impl", + "torch.functional._lu_no_infos", + "torch.functional._lu_with_infos", + "torch.functional._meshgrid", + "torch.functional._return_counts", + "torch.functional._return_inverse_false", + "torch.functional._return_inverse_true", + "torch.functional._return_inverse", + "torch.functional._return_output", + "torch.functional._unique_consecutive_impl", + "torch.functional._unique_impl", + "torch.functional._unravel_index", + "torch.functional.broadcast_shapes", + "torch.functional.lu", + "torch.functional.unique", + "torch.functional.unravel_index", + "torch.futures.collect_all", + "torch.futures.wait_all", + "torch.fx.experimental.const_fold.split_const_subgraphs", + "torch.fx.experimental.proxy_tensor.make_fx", + "torch.get_deterministic_debug_mode", + "torch.get_float32_matmul_precision", + "torch.is_deterministic_algorithms_warn_only_enabled", + "torch.is_storage", + "torch.is_tensor", + "torch.is_warn_always_enabled", + "torch.masked._ops._any", + "torch.masked._ops._apply_docstring_templates", + "torch.masked._ops._canonical_dim", + "torch.masked._ops._combine_input_and_mask", + "torch.masked._ops._generate_docstring", + "torch.masked._ops._input_mask", + "torch.masked._ops._output_mask", + "torch.masked._ops._reduction_identity", + "torch.masked._ops._sparse_coo_flatten_indices", + "torch.masked._ops._sparse_coo_scatter_reduction_helper", + "torch.masked._ops._sparse_coo_where", + "torch.masked._ops._sparse_csr_segment_reduction_helper", + "torch.masked._ops._sparse_csr_where", + "torch.masked._ops._std_var", + "torch.masked._ops._where", + "torch.masked._ops.amax", + "torch.masked._ops.amin", + "torch.masked._ops.argmax", + "torch.masked._ops.argmin", + "torch.masked._ops.corresponding_real_dtype", + "torch.masked._ops.cumprod", + "torch.masked._ops.cumsum", + "torch.masked._ops.log_softmax", + "torch.masked._ops.logaddexp", + "torch.masked._ops.logsumexp", + "torch.masked._ops.mean", + "torch.masked._ops.median", + "torch.masked._ops.norm", + "torch.masked._ops.normalize", + "torch.masked._ops.prod", + "torch.masked._ops.softmax", + "torch.masked._ops.softmin", + "torch.masked._ops.std", + "torch.masked._ops.sum", + "torch.masked._ops.var", + "torch.meshgrid", + "torch.mps._get_default_mps_generator", + "torch.mps.current_allocated_memory", + "torch.mps.driver_allocated_memory", + "torch.mps.empty_cache", + "torch.mps.get_rng_state", + "torch.mps.manual_seed", + "torch.mps.profiler.profile", + "torch.mps.profiler.start", + "torch.mps.profiler.stop", + "torch.mps.seed", + "torch.mps.set_per_process_memory_fraction", + "torch.mps.set_rng_state", + "torch.mps.synchronize", + "torch.nested._internal.nested_tensor.buffer_from_jagged", + "torch.nested._internal.nested_tensor.get_tensor_symint", + "torch.nested._internal.nested_tensor.is_expandable_to", + "torch.nested._internal.nested_tensor.jagged_from_list", + "torch.nested._internal.nested_tensor.jagged_from_tensor_and_lengths", + "torch.nested._internal.nested_tensor.nested_view_from_values_offsets", + "torch.nested._internal.nested_tensor.nested_view_from_values_offsets_lengths", + "torch.nested.as_nested_tensor", + "torch.nested.narrow", + "torch.nested.nested_tensor", + "torch.nn._reduction.get_enum", + "torch.nn._reduction.legacy_get_enum", + "torch.nn._reduction.legacy_get_string", + "torch.nn.factory_kwargs", + "torch.nn.functional.adaptive_avg_pool2d", + "torch.nn.functional.adaptive_avg_pool3d", + "torch.nn.functional.adaptive_max_pool1d_with_indices", + "torch.nn.functional.adaptive_max_pool1d", + "torch.nn.functional.adaptive_max_pool2d_with_indices", + "torch.nn.functional.adaptive_max_pool2d", + "torch.nn.functional.adaptive_max_pool3d_with_indices", + "torch.nn.functional.adaptive_max_pool3d", + "torch.nn.functional.affine_grid", + "torch.nn.functional.alpha_dropout", + "torch.nn.functional.assert_int_or_pair", + "torch.nn.functional.batch_norm", + "torch.nn.functional.binary_cross_entropy_with_logits", + "torch.nn.functional.binary_cross_entropy", + "torch.nn.functional.celu", + "torch.nn.functional.cosine_embedding_loss", + "torch.nn.functional.cross_entropy", + "torch.nn.functional.ctc_loss", + "torch.nn.functional.dropout", + "torch.nn.functional.dropout1d", + "torch.nn.functional.dropout2d", + "torch.nn.functional.dropout3d", + "torch.nn.functional.elu", + "torch.nn.functional.embedding_bag", + "torch.nn.functional.embedding", + "torch.nn.functional.feature_alpha_dropout", + "torch.nn.functional.fold", + "torch.nn.functional.fractional_max_pool2d_with_indices", + "torch.nn.functional.fractional_max_pool2d", + "torch.nn.functional.fractional_max_pool3d_with_indices", + "torch.nn.functional.fractional_max_pool3d", + "torch.nn.functional.gaussian_nll_loss", + "torch.nn.functional.glu", + "torch.nn.functional.grid_sample", + "torch.nn.functional.group_norm", + "torch.nn.functional.gumbel_softmax", + "torch.nn.functional.hardsigmoid", + "torch.nn.functional.hardswish", + "torch.nn.functional.hardtanh", + "torch.nn.functional.hinge_embedding_loss", + "torch.nn.functional.huber_loss", + "torch.nn.functional.instance_norm", + "torch.nn.functional.interpolate", + "torch.nn.functional.kl_div", + "torch.nn.functional.l1_loss", + "torch.nn.functional.layer_norm", + "torch.nn.functional.leaky_relu", + "torch.nn.functional.local_response_norm", + "torch.nn.functional.log_softmax", + "torch.nn.functional.lp_pool1d", + "torch.nn.functional.lp_pool2d", + "torch.nn.functional.margin_ranking_loss", + "torch.nn.functional.max_pool1d_with_indices", + "torch.nn.functional.max_pool1d", + "torch.nn.functional.max_pool2d_with_indices", + "torch.nn.functional.max_pool2d", + "torch.nn.functional.max_pool3d_with_indices", + "torch.nn.functional.max_pool3d", + "torch.nn.functional.max_unpool1d", + "torch.nn.functional.max_unpool2d", + "torch.nn.functional.max_unpool3d", + "torch.nn.functional.mish", + "torch.nn.functional.mse_loss", + "torch.nn.functional.multi_head_attention_forward", + "torch.nn.functional.multi_margin_loss", + "torch.nn.functional.multilabel_margin_loss", + "torch.nn.functional.multilabel_soft_margin_loss", + "torch.nn.functional.nll_loss", + "torch.nn.functional.normalize", + "torch.nn.functional.poisson_nll_loss", + "torch.nn.functional.relu", + "torch.nn.functional.relu6", + "torch.nn.functional.rrelu", + "torch.nn.functional.selu", + "torch.nn.functional.sigmoid", + "torch.nn.functional.silu", + "torch.nn.functional.smooth_l1_loss", + "torch.nn.functional.soft_margin_loss", + "torch.nn.functional.softmax", + "torch.nn.functional.softmin", + "torch.nn.functional.softsign", + "torch.nn.functional.tanh", + "torch.nn.functional.tanhshrink", + "torch.nn.functional.triplet_margin_loss", + "torch.nn.functional.unfold", + "torch.nn.functional.upsample_bilinear", + "torch.nn.functional.upsample_nearest", + "torch.nn.functional.upsample", + "torch.nn.grad._pair", + "torch.nn.grad._single", + "torch.nn.grad._triple", + "torch.nn.grad.conv1d_input", + "torch.nn.grad.conv1d_weight", + "torch.nn.grad.conv2d_input", + "torch.nn.grad.conv2d_weight", + "torch.nn.grad.conv3d_input", + "torch.nn.grad.conv3d_weight", + "torch.nn.modules.activation._is_make_fx_tracing", + "torch.nn.modules.utils._list_with_default", + "torch.nn.modules.utils._ntuple", + "torch.nn.modules.utils._quadruple", + "torch.nn.modules.utils._reverse_repeat_tuple", + "torch.nn.modules.utils.consume_prefix_in_state_dict_if_present", + "torch.nn.parameter.is_lazy", + "torch.norm", + "torch.quantization.default_eval_fn", + "torch.random._seed_custom_device", + "torch.random.fork_rng", + "torch.random.initial_seed", + "torch.random.seed", + "torch.return_types.pytree_register_structseq", + "torch.set_default_dtype", + "torch.set_default_tensor_type", + "torch.set_deterministic_debug_mode", + "torch.set_float32_matmul_precision", + "torch.set_warn_always", + "torch.signal.windows.windows._add_docstr", + "torch.signal.windows.windows._window_function_checks", + "torch.signal.windows.windows.bartlett", + "torch.signal.windows.windows.blackman", + "torch.signal.windows.windows.cosine", + "torch.signal.windows.windows.exponential", + "torch.signal.windows.windows.gaussian", + "torch.signal.windows.windows.general_cosine", + "torch.signal.windows.windows.general_hamming", + "torch.signal.windows.windows.hamming", + "torch.signal.windows.windows.hann", + "torch.signal.windows.windows.kaiser", + "torch.signal.windows.windows.merge_dicts", + "torch.signal.windows.windows.nuttall", + "torch.signal.windows.windows.parse_kwargs", + "torch.sparse.semi_structured.to_sparse_semi_structured", + "torch.sparse.sum", + "torch.split", + "torch.stft", + "torch.sym_float", + "torch.sym_int", + "torch.sym_ite", + "torch.sym_max", + "torch.sym_min", + "torch.sym_not", + "torch.tensordot", + "torch.unique_consecutive", + "torch.use_deterministic_algorithms", + "torch.xpu._get_device", + "torch.xpu._get_generator", + "torch.xpu._get_rng_state_offset", + "torch.xpu._is_compiled", + "torch.xpu._lazy_call", + "torch.xpu._lazy_init", + "torch.xpu._set_rng_state_offset", + "torch.xpu._set_stream_by_id", + "torch.xpu._utils._get_device_index", + "torch.xpu.current_device", + "torch.xpu.current_stream", + "torch.xpu.device_count", + "torch.xpu.get_arch_list", + "torch.xpu.get_device_capability", + "torch.xpu.get_device_name", + "torch.xpu.get_device_properties", + "torch.xpu.get_gencode_flags", + "torch.xpu.get_stream_from_external", + "torch.xpu.init", + "torch.xpu.is_available", + "torch.xpu.is_bf16_supported", + "torch.xpu.is_initialized", + "torch.xpu.memory.empty_cache", + "torch.xpu.memory.max_memory_allocated", + "torch.xpu.memory.max_memory_reserved", + "torch.xpu.memory.mem_get_info", + "torch.xpu.memory.memory_allocated", + "torch.xpu.memory.memory_reserved", + "torch.xpu.memory.memory_stats_as_nested_dict", + "torch.xpu.memory.memory_stats", + "torch.xpu.memory.reset_accumulated_memory_stats", + "torch.xpu.memory.reset_peak_memory_stats", + "torch.xpu.random.initial_seed", + "torch.xpu.random.seed_all", + "torch.xpu.random.seed", + "torch.xpu.set_stream", + "torch.xpu.stream", + "torch.xpu.synchronize", + ], + TorchInGraphFunctionVariable, +) + + +torch_name_rule_map = [ + manual_torch_name_rule_map, + torch_c_binding_in_graph_functions, + torch_non_c_binding_in_graph_functions, +] + + +""" +Generate the torch object - Dynamo tracing rule (the wrapping variable) map. +""" + + +@functools.cache +def get_torch_obj_rule_map() -> dict[Any, type["VariableTracker"]]: + d: dict[Any, type[VariableTracker]] = {} + for m in torch_name_rule_map: + for k, v in m.items(): # type: ignore[attr-defined] + if ".py#" not in k: + obj = load_object(k) + else: + torch_dir = _module_dir(torch) + if torch_dir is None: + continue + obj = torch_dir + k[len("torch/") :] + if obj is not None: + if is_lru_cache_wrapped_function(obj): + obj = obj.__wrapped__ + if obj in d and d[obj] != v: + raise AssertionError( + f"Duplicate torch object {obj} with different rules: {v}, {d[obj]}" + ) + else: + d[obj] = v + return d + + +def _load_obj_from_str(fully_qualified_name: str) -> Any: + module, obj_name = fully_qualified_name.rsplit(".", maxsplit=1) + return getattr(importlib.import_module(module), obj_name) + + +""" +Load string represented torch objects. +""" + + +def load_object(name: str) -> Any: + try: + x = name.split("#") + if len(x) == 2: + obj = _load_obj_from_str(x[0]) + val = getattr(obj, x[1]) + else: + assert len(x) == 1, f"Invalid obj name {name}" + val = _load_obj_from_str(x[0]) + val = unwrap_if_wrapper(val) + except (AttributeError, ImportError): + val = None + return val + + +""" +Get all torch.Tensor methods which are allowed to be in graph functions. +""" + + +@functools.cache +def get_tensor_method() -> frozenset[Any]: + disallowed_tensor_methods = {"__new__", "_make_wrapper_subclass", "_make_subclass"} + s = set() + for name in dir(torch.Tensor): + method = getattr(torch.Tensor, name) + if ( + isinstance( + method, + ( + types.MethodDescriptorType, + types.WrapperDescriptorType, + types.BuiltinFunctionType, + ), + ) + and name not in disallowed_tensor_methods + ): + s.add(method) + + # mlazos: these are functions which we handle specially in TensorVariable + s.add(torch.Tensor.__contains__) # type: ignore[arg-type] + s.add(torch.Tensor.register_hook) # type: ignore[arg-type] + return frozenset(s) + + +""" +Return if a torch object is ATen op or torch.Tensor method. +""" + + +def is_aten_op_or_tensor_method(obj: Any) -> bool: + return obj in get_tensor_method() or isinstance( + obj, + (torch._ops.OpOverloadPacket, torch._ops.OpOverload), + ) + + +class FunctionIdSet: + """ + Track a set of `id()`s of objects which are either allowed or not + allowed to go into the generated FX graph. Use to test for torch.*, + numpy.*, builtins.*, etc. + + Support user modification to permit customization of what can be + added to the graph and what will cause a graph break. + """ + + function_ids: Optional[set[int]] = None + function_names: Optional[dict[int, str]] = None + + def __init__( + self, lazy_initializer: Callable[[], Union[dict[int, str], set[int]]] + ) -> None: + self.lazy_initializer = lazy_initializer + + def __call__(self) -> set[int]: + if self.function_ids is None: + value = self.lazy_initializer() + if isinstance(value, dict): + self.function_ids = set(value.keys()) + self.function_names = value + else: + assert isinstance(value, set) + self.function_ids = value + return self.function_ids + + def get_name(self, idx: int, default: str) -> str: + self() # lazy init + assert self.function_names is not None + return self.function_names.get(idx, default) + + def add(self, idx: int) -> None: + function_ids = self() # lazy init + function_ids.add(idx) + + def remove(self, idx: int) -> None: + function_ids = self() + if idx in function_ids: + function_ids.remove(idx) + + def __contains__(self, idx: int) -> bool: + return idx in self() + + +@FunctionIdSet +def _allowed_callable_ids() -> dict[int, str]: + rv: dict[int, str] = {} + return rv + + +@FunctionIdSet +def _disallowed_callable_ids() -> dict[int, str]: + rv: dict[int, str] = {} + return rv + + +@FunctionIdSet +def _nonstrict_trace_callable_ids() -> dict[int, str]: + rv: dict[int, str] = {} + return rv + + +@FunctionIdSet +def _builtin_function_ids() -> dict[int, str]: + # See also torch/_dynamo/polyfills/loader.py, which removes items in _builtin_function_ids + rv = { + id(v): f"builtins.{k}" + for k, v in builtins.__dict__.items() + if not k.startswith("_") and callable(v) + } + rv.update( + { + id(v): f"operator.{k}" + for k, v in operator.__dict__.items() + if not k.startswith("_") and callable(v) + } + ) + rv.update( + { + id(cast): "typing.cast", + id(copy.deepcopy): "copy.deepcopy", + } + ) + return rv + + +@FunctionIdSet +def _polyfilled_function_ids() -> set[int]: + # See also @torch._dynamo.decorators.substitute_in_graph(...), which adds items in _polyfilled_function_ids + return set() + + +@FunctionIdSet +def _numpy_function_ids() -> dict[int, str]: + unsupported_funcs = { + "seed", + "ranf", + "get_bit_generator", + "RandomState", + "set_bit_generator", + "sample", + } + + def is_supported(k: str, v: Any, mod: Any) -> bool: + if not callable(v): + return False + if not getattr(v, "__module__", None): + return True + if v.__module__ == mod.__name__: + return True + if ( + v.__module__ == "numpy.random.mtrand" + and mod.__name__ == "numpy.random" + and k not in unsupported_funcs + ): + return True + return False + + rv = {} + for mod in NP_SUPPORTED_MODULES: + for k, v in mod.__dict__.items(): + if is_supported(k, v, mod): + rv[id(v)] = f"{mod.__name__}.{k}" + return rv + + +@FunctionIdSet +def _builtin_constant_ids() -> dict[int, str]: + """ + Collects constant builtins by eliminating callable items. + """ + rv = { + id(v): f"builtins.{k}" + for k, v in builtins.__dict__.items() + if not k.startswith("_") and not callable(v) + } + return rv + + +_lazy_module_init: dict[str, list[Callable[[], None]]] = defaultdict(list) + + +def add_module_init_func(name: str, init_func: Callable[[], None]) -> None: + """Register a module without eagerly importing it""" + # If the module is already imported, eagerly run init + assert "." not in name, f"Expected a root module name, but got {name}" + assert name not in _lazy_module_init + _lazy_module_init[name].append(init_func) + + +def _maybe_init_lazy_module(obj: object) -> None: + module = getattr(obj, "__module__", None) + if module is None: + return + + base_module = module.split(".")[0] + init_funcs = _lazy_module_init.pop(base_module, None) + if init_funcs is not None: + for fn in init_funcs: + fn() + + +def is_callable_allowed(obj: Any) -> bool: + _maybe_init_lazy_module(obj) + return id(obj) in _allowed_callable_ids + + +def is_nonstrict_trace_callable(obj: Any) -> bool: + _maybe_init_lazy_module(obj) + return id(obj) in _nonstrict_trace_callable_ids + + +def is_callable_disallowed(obj: Any) -> bool: + _maybe_init_lazy_module(obj) + return id(obj) in _disallowed_callable_ids + + +def is_forbidden(obj: Any) -> bool: + _maybe_init_lazy_module(obj) + return inspect.getattr_static(obj, "_dynamo_forbidden", False) + + +def is_builtin_callable(obj: Any) -> bool: + # See also torch/_dynamo/polyfills/loader.py, which removes items in _builtin_function_ids + return id(obj) in _builtin_function_ids + + +def is_builtin_constant(obj: Any) -> bool: + return id(obj) in _builtin_constant_ids + + +def is_polyfilled_callable(obj: Any) -> bool: + # See also @torch._dynamo.decorators.substitute_in_graph(...), which adds items in _polyfilled_function_ids + return id(obj) in _polyfilled_function_ids + + +def is_numpy(obj: Any) -> bool: + if np is None: + return False + return isinstance(obj, (np.ndarray, np.generic)) or id(obj) in _numpy_function_ids + + +def is_numpy_dtype(obj: Any) -> bool: + if np is None: + return False + return isinstance(obj, np.dtype) + + +def is_numpy_type_info(obj: Any) -> bool: + if np is None: + return False + return isinstance(obj, (np.finfo, np.iinfo)) + + +BUILTIN_SKIPLIST = ( + abc, + copy, + random, + traceback, + linecache, +) + +# third party libraries skiplist is defined by str, because users may not use these libraries. +# we should use lazy import & skip in the future. +THIRDPARTY_SKIPLIST = ( + "fx2trt_oss", + "hypothesis", + "networkx", + "numpy", + "onnx", + "onnxruntime", + "onnx_tf", + "pandas", + "sklearn", + "tabulate", + "tensorflow", + "tensorrt", + "torch2trt", + "tqdm", + "tree", + "tvm", + "xarray", +) + + +def _as_posix_path(path: str) -> str: + posix_path = Path(os.path.normpath(path)).as_posix() + # os.path.normpath and pathlib.Path remove trailing slash, so we need to add it back + if path.endswith((os.path.sep, "/")): + posix_path += "/" + return posix_path + + +def _strip_init_py(s: str) -> str: + suffix = "__init__.py" + s = s.removesuffix(suffix) + return _as_posix_path(s) + + +def _module_dir(m: types.ModuleType) -> Optional[str]: + # Protect against a module not exporting __file__ - this can happen for + # frozen modules, for example. + file = getattr(m, "__file__", None) + return file and _strip_init_py(file) + + +# These are legacy workarounds, don't add new modules to this list. +# Please use the MOD_INLINELIST instead to force inline functions under particular modules. +# +# NB: The only thing that is different about MOD_INLINELIST and LEGACY_MOD_INLINELIST +# is the behavior of a function f2 in the module when called by a function f1 +# in a module in MOD_SKIPLIST (see MOD_SKIPLIST for more details) +# +# LEGACY_MOD_INLINELIST is the same thing as Dynamo's behavior on a module that +# is not in any *_INLINELIST or *_SKIPLIST. +# That being said, we prefer people to add things to MOD_INLINELIST over +# LEGACY_MOD_INLINELIST because it is less likely to break existing tests. +LEGACY_MOD_INLINELIST = { + "torch._dynamo.external_utils", + "torch._export.db.examples", + "torch._export.wrappers", + "torch._functorch.apis", + "torch._functorch.deprecated", + "torch.nn.attention.flex_attention", + "torch.ao.quantization.stubs", + "torch.ao.quantization.pt2e.export_utils", + "torch.ao.quantization.pt2e.qat_utils", + "torch.ao.quantization.pt2e.representation.rewrite", + "torch.ao.quantization.pt2e.utils", + "torch.ao.quantization.quantizer.xnnpack_quantizer", + "torch.export.unflatten", +} + +if torch.distributed.is_available(): + LEGACY_MOD_INLINELIST |= { + "torch.distributed.tensor._api", + "torch.distributed.tensor.device_mesh", + "torch.distributed.device_mesh", + "torch.distributed.algorithms._checkpoint.checkpoint_wrapper", + "torch.distributed.tensor.parallel._data_parallel_utils", + "torch.distributed.tensor.parallel._utils", + "torch.distributed.tensor.parallel.style", + # we have to add replicate to LEGACY_MOD_INLINELIST to ensure + # the forward_hook won't be ignored. + "torch.distributed._composable.replicate", + } + if not config.skip_fsdp_hooks: + LEGACY_MOD_INLINELIST.add("torch.distributed.fsdp._fully_shard") + +# Force inline functions under these modules, even they are in *_SKIPLIST. +# We are using python module name instead of file or directory object to avoid circular dependency. +# Please keep this sorted alphabetically. +# +# Btw, it is not "ideal" for something to be in MOD_INLINELIST. If Dynamo +# fully supports a module, then the ideal case is that it is not in +# any *_INLINELIST or *_SKIPLIST: then, the behavior of Dynamo is that +# it will always inline into functions in the module. +MOD_INLINELIST = [ + "torch._decomp", + "torch._dynamo._trace_wrapped_higher_order_op", + "torch._dynamo.compiled_autograd", + "torch._dynamo.comptime", + "torch._dynamo.polyfills", + "torch._dynamo.test_case", + "torch._export.non_strict_utils", + "torch._functorch._aot_autograd.subclass_parametrization", + "torch._functorch.autograd_function", + "torch._functorch.eager_transforms", + "torch._functorch.functional_call", + "torch._functorch.pyfunctorch", + "torch._functorch.vmap", + "torch._inductor.test_operators", + "torch._library.autograd", + "torch._library.custom_ops", + "torch._ops", + "torch._prims", + "torch._refs", + "torch._tensor", + "torch.amp.autocast_mode", + "torch.ao.nn", + "torch.autograd.function", + "torch.backends.cuda", + "torch.cuda.amp.autocast_mode", + "torch.distributions", + "torch.export._tree_utils", + "torch.export._unlift", + "torch.export._wrapper_utils", + "torch.fx._pytree", + "torch.fx._symbolic_trace", + "torch.fx.experimental.proxy_tensor", + "torch.fx.passes.shape_prop", + "torch.fx.traceback", + "torch.nn", + "torch.overrides", + "torch.random", + "torch.return_types", + "torch.sparse", + "torch.testing", + "torch.utils._content_store", + "torch.utils._contextlib", + "torch.utils._cxx_pytree", + "torch.utils._device", + "torch.utils._foreach_utils", + "torch.utils._python_dispatch", + "torch.utils._pytree", + "torch.utils.hooks", +] +assert sorted(set(MOD_INLINELIST)) == MOD_INLINELIST +MOD_INLINELIST = set(MOD_INLINELIST) + + +if torch.distributed.is_available(): + MOD_INLINELIST.add("torch.distributed") + if not config.skip_fsdp_hooks: + MOD_INLINELIST.add("torch.distributed.fsdp._fully_shard") + + +# By default, all functions under these modules are skipped. +# All the other knobs +# (torch_name_rule_map, MOD_INLINELIST, LEGACY_MOD_INLINELIST) +# take precedence over this list; e.g. if a function is in +# MOD_INLINELIST and MOD_SKIPLIST, then it will be inlined. +# See "A note on skip/inline rules" for more details. +# +# The skip is NOT recursive. If a function f1 in a module in MOD_SKIPLIST +# calls out to another function f2 in some other module, then Dynamo's +# behavior (skip/inline) depends on what we've marked f2 as: +# - if f2 is a function in a module in MOD_SKIPLIST, then we skip f2 +# - if f2 is a function in a module in MOD_INLINELIST, then we skip f2 +# - if f2 is a function in a module in LEGACY_MOD_INLINELIST, then we inline f2 +# - if f2 is a function in a module not in any *_LIST, then we inline f2 +MOD_SKIPLIST = [ + "torch._VF", + "torch.__future__", + "torch.__init__", + "torch._awaits", + "torch._classes", + "torch._compile", + "torch._custom_op", + "torch._custom_ops", + "torch._decomp", + "torch._dispatch", + "torch._dynamo", + "torch._export", + "torch._functorch", + "torch._guards", + "torch._higher_order_ops.effects", + "torch._higher_order_ops.torchbind", + "torch._higher_order_ops.wrap", + "torch._inductor", + "torch._jit_internal", + "torch._lazy", + "torch._library", + "torch._linalg_utils", + "torch._lobpcg", + "torch._logging", + "torch._lowrank", + "torch._meta_registrations", + "torch._namedtensor_internals", + "torch._numpy", + "torch._ops", + "torch._prims", + "torch._prims_common", + "torch._python_dispatcher", + "torch._refs", + "torch._strobelight", + "torch._subclasses", + "torch._tensor", + "torch._tensor_str", + "torch._thread_safe_fork", + "torch._utils", + "torch._utils_internal", + "torch._vmap_internals", + "torch._weights_only_unpickler", + "torch.accelerator", + "torch.amp", + "torch.ao", + "torch.autograd", + "torch.backends", + "torch.compiler", + "torch.contrib", + "torch.cpu", + "torch.cuda", + "torch.distributed", + "torch.distributions", + "torch.export", + "torch.fb", + "torch.fft", + "torch.functional", + "torch.futures", + "torch.fx", + "torch.hub", + "torch.jit", + "torch.library", + "torch.linalg", + "torch.masked", + "torch.monitor", + "torch.mps", + "torch.mtia", + "torch.multiprocessing", + "torch.nested", + "torch.nn", + "torch.onnx", + "torch.overrides", + "torch.package", + "torch.profiler", + "torch.quantization", + "torch.quasirandom", + "torch.random", + "torch.serialization", + "torch.signal", + "torch.sparse", + "torch.special", + "torch.storage", + "torch.testing", + "torch.types", + "torch.utils", + "torch.xpu", +] + +assert sorted(set(MOD_SKIPLIST)) == MOD_SKIPLIST +MOD_SKIPLIST = set(MOD_SKIPLIST) + + +@functools.cache +def get_legacy_mod_inlinelist() -> set[str]: + torch_dir = _module_dir(torch) + if torch_dir is None: + return set() + inlinelist = { + _as_posix_path(torch_dir + m[len("torch.") :].replace(".", "/")) + for m in LEGACY_MOD_INLINELIST + } + return inlinelist + + +@functools.cache +def get_mod_inlinelist() -> set[str]: + torch_dir = _module_dir(torch) + if torch_dir is None: + return set() + inlinelist = { + _as_posix_path(torch_dir + m[len("torch.") :].replace(".", "/")) + for m in MOD_INLINELIST + } + return inlinelist + + +@functools.cache +def get_mod_skiplist() -> set[str]: + torch_dir = _module_dir(torch) + if torch_dir is None: + return set() + skiplist = { + _as_posix_path(torch_dir + m[len("torch.") :].replace(".", "/")) + for m in MOD_SKIPLIST + } + return skiplist + + +# skip some standard python builtin libs +SKIP_DIRS = [ + "", + _as_posix_path(_config_module.__file__), + "triton/backends", +] +SKIP_DIRS.extend(map(_as_posix_path, filter(None, map(_module_dir, BUILTIN_SKIPLIST)))) + +SKIP_DIRS_RE = re.compile(r"match nothing^") + +# Skip fbcode paths(including torch.package paths) containing +# one of the following strings. +FBCODE_SKIP_DIRS: set[str] = set() + +FBCODE_SKIP_DIRS_RE = re.compile(f".*({'|'.join(map(re.escape, FBCODE_SKIP_DIRS))})") + +# Remove this after fbcode is fully migrated to tracing through torchrec. +FBCODE_SKIP_TORCHREC_DIRS = { + "torchrec/distributed", + "torchrec/fb/distributed", + "caffe2/torch/fb/sparsenn/pooled_embeddings_modules.py", +} + +FBCODE_SKIP_TORCHREC_DIRS_RE = re.compile( + f".*({'|'.join(re.escape(_as_posix_path(d)) for d in FBCODE_SKIP_TORCHREC_DIRS)})" +) + +# TODO(yanboliang, anijain2305) - There are a few concerns that we should +# resolve +# 1) Audit if torchrec/distributed is even required in FBCODE_SKIPS_DIR +# 2) To inline just one file but skip others in a directory, we could use +# manual_torch_name_rule_map but this one is hard because FBCODE can add unusual +# names like torch_package. +# So, this is a stop gap solution till then. +FBCODE_INLINE_FILES_IN_SKIPPED_DIRS = { + "torchrec/distributed/types.py", +} +FBCODE_INLINE_FILES_IN_SKIPPED_DIRS_RE = re.compile( + f".*({'|'.join(re.escape(_as_posix_path(d)) for d in FBCODE_INLINE_FILES_IN_SKIPPED_DIRS)})" +) + +# torch.optim is a special case, +# we usually want to inline it, but the directory +# structure does not match the module structure +# and we want to skip the functions in optim/lr_scheduler.py +# this has precedence over all other rules in check_file +FORCE_SKIP_FILES = {f"{_module_dir(torch)}optim/lr_scheduler.py"} + + +def _recompile_re() -> None: + global SKIP_DIRS_RE + SKIP_DIRS_RE = re.compile( + rf"^[^\s<]*({'|'.join(re.escape(_as_posix_path(d)) for d in SKIP_DIRS)})" + ) + + +def add(import_name: str) -> None: + if isinstance(import_name, types.ModuleType): + return add(import_name.__name__) + assert isinstance(import_name, str) + from importlib.util import find_spec + + module_spec = find_spec(import_name) + if not module_spec: + return + origin = module_spec.origin + if origin is None: + return + SKIP_DIRS.append(_strip_init_py(origin)) + _recompile_re() + + +@dataclasses.dataclass +class SkipResult: + skipped: bool + reason: Optional[str] + + +def check_file(filename: Optional[str], is_inlined_call: bool = False) -> SkipResult: + """Should skip this file?""" + if filename is None: + return SkipResult(True, "filename is None") + filename = _as_posix_path(filename) + if filename in FORCE_SKIP_FILES: + return SkipResult(True, "FORCE_SKIP_FILES") + + if any(filename.startswith(d) for d in get_legacy_mod_inlinelist()): + return SkipResult( + False, + "LEGACY_MOD_INLINELIST", + ) + if is_inlined_call and is_torch_inline_allowed(filename): + return SkipResult( + False, + "MOD_INLINELIST", + ) + if ( + is_fbcode() + and FBCODE_SKIP_DIRS + and bool(FBCODE_SKIP_DIRS_RE.match(filename)) + and not bool(FBCODE_INLINE_FILES_IN_SKIPPED_DIRS_RE.match(filename)) + ): + return SkipResult( + True, + "FBCODE_SKIP_DIRS", + ) + + if ( + is_fbcode() + and config.skip_torchrec + and FBCODE_SKIP_TORCHREC_DIRS + and bool(FBCODE_SKIP_TORCHREC_DIRS_RE.match(filename)) + and not bool(FBCODE_INLINE_FILES_IN_SKIPPED_DIRS_RE.match(filename)) + ): + return SkipResult(True, "FBCODE_SKIP_TORCHREC_DIRS") + + unittest_dir = _module_dir(unittest) + if ( + unittest_dir is not None + and filename.startswith(unittest_dir) + and not torch._dynamo.config.enable_trace_unittest + ): + return SkipResult(True, "unittest") + + if bool(SKIP_DIRS_RE.match(filename)): + return SkipResult(True, "SKIP_DIRS") + + if any(filename.startswith(d) for d in get_mod_skiplist()): + return SkipResult(True, "MOD_SKIPLIST") + return SkipResult(False, "inlined by default") + + +@dataclasses.dataclass +class FunctionInfo: + py_obj: Optional[object] + name: Optional[str] + filename: str + code: Optional[types.CodeType] + + +""" +This is the main entry point to determine whether an object (function) should be inlined or skipped. +Let's illustrate the logic with an example: + @torch.compile + def f1(x, y): + ...... + f2(x, y) + ...... + + def f2(x, y): + ...... + f3(x, y) + ...... + + def f3(x, y): + ...... + +There are mainly three call sites of check/check_verbose: +* The compile region entrance (like function f1), the corresponding code is located at eval_frame.py. +* When tracing the recursively called functions (like function f2 and f3). + * Dynamo decides inline/skip every time it encounters a new recursively function call, and the call site + is in InliningInstructionTranslator.check_inlineable of symbolic_convert.py. + * If f2 is skipped by Dynamo, when evaluating the frame of f3, Dynamo need the inline/skip check again + and the call site is in catch_errors_wrapper.catch_errors of convert_frame.py. +* For global variables and function arguments, Dynamo needs to decide if they are wrapped as SkipFunctionVariable in builder.py. + +`is_inlined_call` is used to indicate if the current function call is inlined (f2 is inlined call if it passes check) +or not (f3 is not inlined call if f2 is skipped). Inside of the `check_verbose` function, there are more rules +to be checked if this `is_inlined_call`. +The reason to have this flag is that if the upper level function call (e.g, f2) is skipped, +we don't want to inline the lower level function call (e.g, f3) by default. +""" + + +def check_verbose(obj: Any, is_inlined_call: bool = False) -> SkipResult: + if isinstance( + obj, + ( + UserFunctionVariable, + UserMethodVariable, + NestedUserFunctionVariable, + LocalGeneratorFunctionVariable, + LocalGeneratorObjectVariable, + ), + ): + try: + py_obj = obj.get_function() + except NotImplementedError: + py_obj = None + fi = FunctionInfo(py_obj, obj.get_name(), obj.get_filename(), obj.get_code()) + elif isinstance(obj, types.CodeType): + fi = FunctionInfo(None, obj.co_name, obj.co_filename, obj) + elif isinstance(obj, (types.FunctionType, types.MethodType)): + filename = getfile(obj) + assert filename is not None + fi = FunctionInfo( + obj, + obj.__name__, + filename, + obj.__code__, # type: ignore[union-attr] # FIXME Add MethodType.__code__ to typeshed + ) + else: + filename = getfile(obj) + assert filename is not None + fi = FunctionInfo(obj, None, filename, None) + + # Consulte the central trace rules defined in torch._dynamo.trace_rules. + reasons: set[str] = set() + rule = lookup_inner(fi.py_obj, fi.name, fi.filename, is_inlined_call, reasons) + assert rule is not None + if issubclass( + rule, + ( + UserFunctionVariable, + LocalGeneratorFunctionVariable, + PolyfilledFunctionVariable, + ), + ): + return SkipResult( + False, + f"inlined according trace_rules.lookup {reasons.pop()}", + ) + elif issubclass(rule, TorchInGraphFunctionVariable): + return SkipResult( + False, + f"registered in torch_obj_rule {reasons.pop()}", + ) + else: + assert rule == SkipFunctionVariable, rule + return SkipResult( + True, + f"skipped according trace_rules.lookup {reasons.pop()}", + ) + + +def check(obj: Any, is_inlined_call: bool = False) -> bool: + return check_verbose(obj, is_inlined_call).skipped + + +# skip common third party libs +for _name in THIRDPARTY_SKIPLIST: + add(_name) + +_recompile_re() + + +def is_torch_inline_allowed(filename: str) -> bool: + return any(filename.startswith(d) for d in get_mod_inlinelist()) + + +@functools.cache +def dynamo_dir() -> Optional[str]: + import torch._dynamo + + return _module_dir(torch._dynamo) + + +def is_torch(filename: str) -> bool: + dynamo_path = dynamo_dir() + if dynamo_path is not None and filename.startswith(dynamo_path): + return False + torch_path = _module_dir(torch) + return torch_path is not None and filename.startswith(torch_path) + + +""" +Main entry point for looking up the trace rule (the Dynamo variable) for a given callable object. +""" + + +def lookup_callable(obj: Callable[..., Any]) -> Optional[type[VariableTracker]]: + if not hashable(obj): + return None + # Custom allow/disallow in graph takes precedence over the general lookup. + if is_callable_disallowed(obj): + return SkipFunctionVariable + if is_callable_allowed(obj): + return TorchInGraphFunctionVariable + if is_polyfilled_callable(obj): + return PolyfilledFunctionVariable + if is_builtin_callable(obj): + return BuiltinVariable + return None + + +""" +Main entry point for looking up the trace rule (the Dynamo variable) for a given function object. +E.g, the lookup result of `torch.sin` is `TorchInGraphFunctionVariable`. +""" + + +def lookup(obj: Any) -> Optional[type[VariableTracker]]: + return lookup_inner(obj) + + +# also takes config.dont_skip_tracing into account +def lookup_inner( + obj: Any, + name: Optional[str] = None, + filename: Optional[str] = None, + is_direct_call: bool = True, + reasons: Union[None, set[str]] = None, +) -> Optional[type[VariableTracker]]: + result = _lookup_inner( + obj, + name=name, + filename=filename, + is_direct_call=is_direct_call, + reasons=reasons, + ) + # There are still some modules we should absolutely NOT trace into - e.g. most of torch._dynamo, + # as this can result in really weird tracing behaviors. + # Note that if a torch._dynamo function is already not skipped (e.g. functions in external_utils.py), + # then this branch does not apply. + if config.dont_skip_tracing and result is SkipFunctionVariable: + if filename is None: + filename = getfile(obj) + assert filename is not None + filename = _as_posix_path(filename) + torch_dir = _module_dir(torch) + if torch_dir is not None: + dynamo_path = _as_posix_path(torch_dir) + "_dynamo" + if filename.startswith(dynamo_path) and not filename.endswith( + "test_dont_skip_tracing_functions.py" + ): + return SkipFunctionVariable + if reasons is not None: + reasons.add( + "Attempted skip but we are ignoring skips due to torch._dynamo.config.dont_skip_tracing" + ) + return UserFunctionVariable + return result + + +def _lookup_inner( + obj: Any, + name: Optional[str] = None, + filename: Optional[str] = None, + is_direct_call: bool = True, + reasons: Optional[set[str]] = None, +) -> Optional[type[VariableTracker]]: + # Step 1: lookup obj's tracing rule in `torch_name_rule_map`. + # The rules defined in `torch_name_rule_map` mainly includes two parts: + # - Manually defined rules for any functions. + # - The list of torch in graph functions. + try: + can_hash = hashable(obj) + except Exception: + can_hash = False + if not can_hash: + if reasons is not None: + reasons.add("obj is not hashable") + return None + if obj is not None: + if is_aten_op_or_tensor_method(obj): + return TorchInGraphFunctionVariable + rule = get_torch_obj_rule_map().get(obj, None) + if rule is not None: + if reasons is not None: + reasons.add("get_torch_obj_rule_map") + return rule + elif name is not None and filename is not None and not is_direct_call: + if name.startswith(TORCH_DYNAMO_RESUME_IN_PREFIX): + rule = get_torch_obj_rule_map().get( + filename + "#" + TORCH_DYNAMO_RESUME_IN_PREFIX, None + ) + else: + rule = get_torch_obj_rule_map().get(filename + "#" + name, None) + if rule is not None: + if reasons is not None: + reasons.add("get_torch_obj_rule_map") + return rule + elif name == "": + if reasons is not None: + reasons.add("inlining frame from list comprehension") + return UserFunctionVariable + + # Step 2: lookup obj's tracing rule by function name. + if is_direct_call: + if name == "patched_init": + if reasons is not None: + reasons.add("func name is patched_init") + return SkipFunctionVariable + elif name == "__torch_function__" or ( + obj and getattr(obj, "__name__", None) == "__torch_function__" + ): + if reasons is not None: + reasons.add("func name is __torch_function__") + return UserFunctionVariable + + if not is_direct_call: + if name == "__getattr__": + # is_direct_call = False indicates that this is the top-level frame + # being traced (i.e., it is not inlined and not called from + # InliningInstructionTranslator). Tracing __getattr__ at the top + # level is unlikely because we inline it for + # UserDefinedObjectVariable. This scenario occurs only for + # UnspecializedNNModuleVariable, where Dynamo directly calls + # __getattr__ during trace time, generating LOAD_ATTR bytecode + # without going through the underlying __getattr__ data structures. + # When this optimized bytecode is executed, Dynamo is triggered + # again on the __getattr__ call. Therefore, we skip Dynamo tracing + # in this case. + if reasons is not None: + reasons.add( + "Tracing __getattr__ as the top level frame, unsuitable for tracing." + ) + return SkipFunctionVariable + + # Step 3: lookup obj's tracing rule by filename. + if filename is None: + filename = getfile(obj) + + skip_result = check_file(filename, is_direct_call) + if reasons is not None and skip_result.reason is not None: + reasons.add(skip_result.reason) + if skip_result.skipped: + return SkipFunctionVariable + else: + return UserFunctionVariable + + +def clear_lru_cache() -> None: + torch._dynamo.trace_rules.get_torch_obj_rule_map.cache_clear() + torch._dynamo.trace_rules.get_tensor_method.cache_clear() + torch._dynamo.trace_rules.get_legacy_mod_inlinelist.cache_clear() + torch._dynamo.trace_rules.get_mod_inlinelist.cache_clear() + torch._dynamo.trace_rules.dynamo_dir.cache_clear() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/types.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/types.py new file mode 100644 index 0000000000000000000000000000000000000000..8236d8b229be24a253373cc5bd74358d79eeeb44 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/types.py @@ -0,0 +1,140 @@ +"""This module contains the core type definitions and protocols used throughout Dynamo. + +The types defined here fall into several categories: +- Guard related types (GuardFn, GuardFail, GuardedCode): Used for tracking and managing guards that protect compiled code +- Frame and cache types (FrameState, CacheEntry): Used for managing interpreter frame state and caching +- Callback protocols (DynamoCallbackFn): Define the interface for frame evaluation callbacks +- Hook protocols (DynamoGuardHook, ProfilerStartHook, ProfilerEndHook, BytecodeHook): Define various hook points for + instrumentation and customization + +These types provide the foundational interfaces that enable Dynamo's dynamic compilation and optimization system, +ensuring type safety and clear contracts between different components of the system. +""" + +import dataclasses +import types +from collections.abc import Callable +from typing import Any, NamedTuple, Optional, Protocol, Union + +# CacheEntry has a `guard_manager` field for the guard, and a `code` field for the code object. +from torch._C._dynamo.eval_frame import ( + _CacheEntry as CacheEntry, + _ExtraState as ExtraState, + _FrameAction as FrameAction, + _FrameExecStrategy as FrameExecStrategy, + _PyInterpreterFrame as DynamoFrameType, +) +from torch._guards import CompileId, Guard + + +# We use a dict to store additional data per frame. +FrameState = dict[Any, Any] + + +class GuardFail(NamedTuple): + # A string repr of the piece of failed guard code we eval-ed + reason: str + # A code object where we failed a guard + orig_code: types.CodeType + + +@dataclasses.dataclass(frozen=True) +class GuardFilterEntry: + name: str + has_value: bool + value: object + guard_type: str + derived_guard_types: tuple[str, ...] + is_global: bool + orig_guard: Guard + + +class GuardFn(Protocol): + closure_vars: dict[str, object] + args: list[str] + code_parts: list[str] + verbose_code_parts: list[str] + global_scope: dict[str, object] + guard_fail_fn: Optional[Callable[[GuardFail], None]] + cache_entry: Optional[CacheEntry] + extra_state: Optional[ExtraState] + + # maps locals of user function to bool + def __call__(self, f_locals: dict[str, object]) -> bool: ... + + +@dataclasses.dataclass +class GuardedCode: + code: types.CodeType + guard_manager: GuardFn + compile_id: CompileId + trace_annotation: str = "Unknown" + + +@dataclasses.dataclass +class ConvertFrameReturn: + # default return is no compiled code (i.e. `return None`): + # strategy is to skip non-recursively, for all future intercepted frames too + + # eval frame execution strategy for this frame + frame_exec_strategy: FrameExecStrategy = dataclasses.field( + default_factory=lambda: FrameExecStrategy(FrameAction.SKIP, FrameAction.DEFAULT) + ) + # also apply frame_exec strategy to future frames with same code + apply_to_code: bool = True + guarded_code: Optional[GuardedCode] = None + + +def wrap_guarded_code(guarded_code: GuardedCode) -> ConvertFrameReturn: + return ConvertFrameReturn( + frame_exec_strategy=FrameExecStrategy(FrameAction.DEFAULT, FrameAction.DEFAULT), + guarded_code=guarded_code, + ) + + +class DynamoCallbackFn(Protocol): + def __call__( + self, + frame: DynamoFrameType, + cache_entry: Optional[CacheEntry], + frame_state: FrameState, + ) -> ConvertFrameReturn: ... + + +DynamoCallback = Union[DynamoCallbackFn, None, bool] + + +class DynamoGuardHook(Protocol): + def __call__( + self, + guard_manager: GuardFn, + code: types.CodeType, + f_locals: dict[str, object], + index: int, + last: bool, + ) -> None: ... + + +class DynamoGuardCompleteHook(Protocol): + def __call__( + self, + cache_hit: bool, + ) -> bool: ... + + +class ProfilerStartHook(Protocol): + def __call__( + self, + name: str, + # TODO(whc) how do I annotate a _RecordFunction here? + ) -> Any: ... + + +class ProfilerEndHook(Protocol): + def __call__(self, record: Any) -> None: ... + + +class BytecodeHook(Protocol): + def __call__( + self, code: types.CodeType, new_code: types.CodeType + ) -> Optional[types.CodeType]: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1a4f0394846f9b79f0b578b4b7a77d05b2446d30 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_dynamo/utils.py @@ -0,0 +1,5018 @@ +""" +Utility functions and classes used throughout the TorchDynamo system. + +This module contains a collection of helper utilities used by various parts of Dynamo for: +- Performance metrics collection and reporting +- Compilation timing and debugging +- Graph manipulation and tensor operations +- Runtime guards and checks +- Common data structure operations +- Testing and development tools + +This is an internal module that provides shared functionality used across the Dynamo codebase. +""" + +from __future__ import annotations + +import atexit +import collections +import contextlib +import copy +import dataclasses +import datetime +import dis +import enum +import functools +import gc +import importlib +import inspect +import itertools +import json +import linecache +import logging +import math +import operator +import os +import re +import sys +import textwrap +import threading +import time +import traceback +import types +import typing +import uuid +import warnings +import weakref +from collections import Counter, OrderedDict +from contextlib import AbstractContextManager, contextmanager +from dataclasses import is_dataclass +from functools import lru_cache +from types import CodeType, MethodWrapperType +from typing import ( + Any, + cast, + ClassVar, + Generic, + Literal, + Optional, + overload, + TypeAlias, + TypeGuard, + TypeVar, + Union, +) +from typing_extensions import ParamSpec, TypeIs + +import torch +import torch._functorch.config +import torch.fx.experimental.symbolic_shapes +import torch.utils._pytree as pytree +from torch import fx +from torch._C import ( + _instruction_counter, + _len_torch_function_stack, + _pop_torch_function_stack, + _push_on_torch_function_stack, +) +from torch._dispatch.python import enable_python_dispatcher +from torch._dynamo.metrics_context import MetricsContext, RuntimeMetricsContext +from torch._guards import CompileId, Source, TracingContext +from torch._subclasses.meta_utils import is_sparse_compressed +from torch._utils_internal import ( + justknobs_check, + log_chromium_event_internal, + log_compilation_event, + record_chromium_event_internal, + signpost_event, +) +from torch.fx._utils import _format_graph_code, lazy_format_graph_code +from torch.monitor import _WaitCounter +from torch.nn.modules.lazy import LazyModuleMixin +from torch.utils._python_dispatch import is_traceable_wrapper_subclass +from torch.utils._triton import has_triton, has_triton_package +from torch.utils.hooks import RemovableHandle + +from .graph_utils import _get_flat_args + + +if typing.TYPE_CHECKING: + from collections.abc import ( + Callable, + Container, + Generator, + ItemsView, + Iterable, + Iterator, + KeysView, + Mapping, + Sequence, + ValuesView, + ) + + from torch._dynamo.replay_record import ExecutionRecord + from torch._dynamo.symbolic_convert import ( + InstructionTranslator, + InstructionTranslatorBase, + ) + from torch._dynamo.variables.base import VariableTracker + from torch._prims_common import DeviceLikeType + from torch._subclasses import FakeTensorMode + + +try: + import numpy as np +except ModuleNotFoundError: + np = None # type: ignore[assignment] + +try: + import torch._logging + import torch._numpy as tnp + from torch._guards import detect_fake_mode # noqa: F401 + from torch._logging import LazyString + + from . import config + + # NOTE: Make sure `NP_SUPPORTED_MODULES` and `NP_TO_TNP_MODULE` are in sync. + if np: + NP_SUPPORTED_MODULES: tuple[types.ModuleType, ...] = ( + np, + np.fft, + np.linalg, + np.random, + ) + + NP_TO_TNP_MODULE = { + np: tnp, + np.fft: tnp.fft, + np.linalg: tnp.linalg, + np.random: tnp.random, + } + else: + NP_SUPPORTED_MODULES = () + + NP_TO_TNP_MODULE = {} + from torch._subclasses.fake_tensor import FakeTensor, is_fake, maybe_get_fake_mode +except ImportError: + pass + + +T = TypeVar("T") +R = TypeVar("R") +_P = ParamSpec("_P") + +unpatched_nn_module_getattr = torch.nn.Module.__getattr__ +unpatched_nn_module_call = torch.nn.Module.__call__ +unpatched_nn_module_call_impl = torch.nn.Module._call_impl + +counters: collections.defaultdict[str, Counter[str]] = collections.defaultdict( + collections.Counter +) +optimus_scuba_log: dict[str, Any] = {} +troubleshooting_url = ( + "https://pytorch.org/docs/main/compile/programming_model.recompilation.html" +) +nnmodule_doc_url = "https://pytorch.org/docs/main/torch.compiler_nn_module.html" +nnmodule_doc_url_msg = f"See {nnmodule_doc_url} for more information and limitations." +log = logging.getLogger(__name__) + +# profiling compilation time by function +compilation_time_metrics: dict[str, list[float]] = {} + +# This supports calculate_time_spent(), which reports cumulative times +# across the process for any "phase" populated by dynamo_timed. Reset if +# reset_frame_count() is called. +cumulative_time_spent_ns: dict[str, float] = collections.defaultdict(float) + +timer_counter = itertools.count() + + +# Abstraction on top of counters. +class ReInplaceTrigger(enum.Enum): + AUTO_FUNC_V1 = 1 + AUTO_FUNC_V2 = 2 + TRITON_OPS = 3 + + +class ReinplaceCounters: + _values: collections.defaultdict[str, int] = collections.defaultdict(int) + + # Track sizes of known not re-inplaced tensors (exclude dynamic shapes). + @classmethod + def add_missed_bytes(cls, trigger: ReInplaceTrigger, bytes: int) -> None: + if bytes != 0: + cls._values[f"missed_bytes_{trigger.name}"] += bytes + + # Track number of not re-inplaced tensors. + @classmethod + def add_missed_opportunities(cls, trigger: ReInplaceTrigger, count: int) -> None: + if count != 0: + cls._values[f"missed_tensors_{trigger}"] += count + + @classmethod + def clear(cls) -> None: + cls._values.clear() + + @classmethod + def get_total_missed(cls) -> int: + sum = 0 + for trigger in ReInplaceTrigger: + sum += cls._values.get(f"missed_tensors_{trigger}", 0) + return sum + + @classmethod + def get_total_missed_bytes(cls) -> int: + sum = 0 + for trigger in ReInplaceTrigger: + sum += cls._values.get(f"missed_bytes_{trigger.name}", 0) + return sum + + @classmethod + def log(cls) -> None: + # if not empty log. + if cls._values: + signpost_event("inductor", "reinplace_counters", cls._values) + + +def tabulate( + rows: Union[list[tuple[str, Any]], list[list[Any]]], + headers: Union[tuple[str, ...], list[str]], +) -> str: + try: + import tabulate + + return tabulate.tabulate(rows, headers=headers) + except ImportError: + return "\n".join( + ", ".join(map(str, row)) for row in itertools.chain([headers], rows) + ) + + +curr_frame = 0 + + +# Note: Called for you by dynamo - you almost never ever want to invoke this yourself. +def increment_frame() -> None: + global curr_frame + curr_frame = curr_frame + 1 + + +# Note: Called for you by dynamo - you almost never ever want to invoke this yourself. +def reset_frame_count() -> None: + global curr_frame + cumulative_time_spent_ns.clear() + compilation_time_metrics.clear() + curr_frame = 0 + + +_recompile_user_contexts: Optional[list[Callable[[], str]]] = None + + +def register_hook_for_recompile_user_context(hook: Callable[[], str]) -> None: + """ + Register a hook to be called when a recompile is triggered. The hook + should return a string describing user contexts that are not available + to the compiler, such as the current training epoch. This is useful for + debugging and data analysis for recompile. For data retention purposes, + the user context string is capped at 256 characters. + """ + global _recompile_user_contexts + if _recompile_user_contexts is None: + _recompile_user_contexts = [] + _recompile_user_contexts.append(hook) + + +def get_hook_for_recompile_user_context() -> Optional[list[Callable[[], str]]]: + return _recompile_user_contexts + + +def reset_recompile_user_contexts() -> None: + """Clear any registered recompile user-context hooks (test helper).""" + global _recompile_user_contexts + _recompile_user_contexts = None + + +op_count = 0 + + +def increment_op_count(cnt: int) -> None: + global op_count + op_count += cnt + + +# Get the total time in seconds for each "phase" +# For example, {'entire_frame_compile':8.574629999999999, 'backend_compile':5.26806} +def calculate_time_spent() -> dict[str, float]: + total_by_key = {} + for phase, timing in cumulative_time_spent_ns.items(): + # pyrefly: ignore [unsupported-operation] + total_by_key[phase] = timing / 1e9 + + total_by_key["total_wall_time"] = total_by_key.get( + "entire_frame_compile", 0 + ) + total_by_key.get("entire_backward_compile", 0) + # pyrefly: ignore [bad-return] + return total_by_key + + +# Print a report of time spent so far +# Ex: +# TIMING: +# entire_frame_compile:8.574629999999999 +# backend_compile:5.26806 +def print_time_report() -> None: + total_by_key = calculate_time_spent() + + out = "TIMING:" + for key, value in total_by_key.items(): + out = f"{out} {key}:{round(value, 5)}" + + print(out) + + +# Use the following singleton to capture and log CompilationMetrics. Entering the context +# manager allocates a new record to be logged when it exits. (You should not need to use +# this directly unless you introduce a new code path where compilation metrics would be +# gathered). While compiling, use the setters or timer in MetricsContext to update fields +# in the current context. For example: +# +# To set a single field once (use overwrite=True to overwrite): +# get_metrics_context().set("metric_name", value) +# +# To set multiple fields at once (use overwrite=True to overwrite): +# get_metrics_context().update({"name1": val1, "name2": val2}) +# +# To increment an integer field: +# get_metrics_context().increment("metric_name", value) +# +# To record execution time, MetricsContext works with dynamo_timed: +# def foo(...): +# # Updates the "metric_us" field. +# with dynamo_timed("metric", dynamo_compile_column_us="metric_us") +# ... +# +_METRICS_CONTEXT: MetricsContext +_RUNTIME_METRICS_CONTEXT: RuntimeMetricsContext + + +def get_metrics_context() -> MetricsContext: + return _METRICS_CONTEXT + + +def get_runtime_metrics_context() -> RuntimeMetricsContext: + return _RUNTIME_METRICS_CONTEXT + + +class CompileEventLogLevel(enum.Enum): + """ + Enum that loosely corresponds with a "log level" of a given event. + + CHROMIUM_EVENT: Logs only to tlparse. + COMPILE_EVENT: Logs to tlparse + PT2 Compile Events + COMPILATION_METRIC: Logs to tlparse, PT2 Compile Events, and dynamo_compile + """ + + CHROMIUM = 1 + PT2_COMPILE = 2 + COMPILATION_METRIC = 3 + + +class CompileEventLogger: + """ + Helper class for representing adding metadata(i.e. columns) to various compile events. + Use CompileEventLogger to add event data to: + - Chromium events + - PT2 Compile Events + - CompilationMetrics + + This should be used in conjunction with dynamo_timed() and metrics contexts, which create + timed spans and events. CompileEventLogger uses three log levels (described in CompileEventLogLevel), + where each log level logs to all sources below it in the hierarchy. + + Example usages: + - I want to log to an existing chromium event within dynamo timed: + with dynamo_timed("my_event"): + CompileEventLogger.chromium("my_event", foo=bar) + + - I want to log my event to both chromium + pt2_compile_events: + with dynamo_timed("my_event", log_pt2_compile_event=True): + CompileEventLogger.pt2_compile("my_event", foo=bar) + + - I want to add information to dynamo events and dynamo_compile + CompileEventLogger.compilation_metric(foo=bar) + """ + + @staticmethod + def log_instant_event( + event_name: str, + metadata: dict[str, Any], + time_ns: Optional[int] = None, + log_level: CompileEventLogLevel = CompileEventLogLevel.CHROMIUM, + ) -> None: + if time_ns is None: + time_ns = time.time_ns() + chromium_log = get_chromium_event_logger() + if log_level == CompileEventLogLevel.CHROMIUM: + log_pt2_compile_event = False + elif log_level == CompileEventLogLevel.PT2_COMPILE: + log_pt2_compile_event = True + else: + raise RuntimeError( + "Cannot log instant event at COMPILATION_METRIC level. Please choose one of CHROMIUM_EVENT or COMPILE_EVENT" + ) + chromium_log.log_instant_event( + event_name, time_ns, metadata, log_pt2_compile_event + ) + + @staticmethod + def add_data( + event_name: str, + log_level: CompileEventLogLevel, + overwrite: bool = False, + **metadata: object, + ) -> None: + """ + Centralized API for adding data to various events + Log an event to a toplevel "dynamo" event or metrics context + depending on log level. + """ + chromium_log = get_chromium_event_logger() + pt2_compile_substack = chromium_log.get_pt2_compile_substack() + + if log_level == CompileEventLogLevel.CHROMIUM: + chromium_log.add_event_data(event_name, **metadata) + elif log_level == CompileEventLogLevel.PT2_COMPILE: + pt2_compile_substack = chromium_log.get_pt2_compile_substack() + if event_name not in pt2_compile_substack: + raise RuntimeError( + "Error: specified log level PT2_COMPILE, but the event %s" + " is not logged to pt2_compile_events. Make sure the event is active and you passed " + "log_pt2_compile_event=True to dynamo_timed", + event_name, + ) + chromium_log.add_event_data(event_name, **metadata) + else: + assert log_level == CompileEventLogLevel.COMPILATION_METRIC + top_event = chromium_log.get_outermost_event() + + if event_name != top_event: + raise RuntimeError( + "Log level is COMPILATION_METRIC, but event_name isn't the toplevel event. " + "CompilationMetrics must be logged to the toplevel event. Consider using `log_toplevel_event_data` directly." + ) + metrics_context = get_metrics_context() + if not metrics_context.in_progress(): + raise RuntimeError( + "No metrics context is in progress. Please only call this function within a metrics context." + ) + + # TODO: should we assert that the keys of metadata are in CompilationMetrics? + metrics_context.update(metadata, overwrite) + chromium_log.add_event_data(event_name, **metadata) + + @staticmethod + def add_toplevel( + log_level: CompileEventLogLevel, overwrite: bool = False, **metadata: object + ) -> None: + """ + Syntactic sugar for logging to the toplevel event + """ + top_event = get_chromium_event_logger().get_outermost_event() + if top_event is None: + raise RuntimeError( + "No toplevel event active. Please only call this function within a dynamo_timed context." + ) + CompileEventLogger.add_data(top_event, log_level, overwrite, **metadata) + + @staticmethod + def increment( + event_name: str, log_level: CompileEventLogLevel, key: str, value: int + ) -> None: + """ + Increments an existing field, or adds it + """ + chromium_log = get_chromium_event_logger() + if ( + log_level == CompileEventLogLevel.CHROMIUM + or log_level == CompileEventLogLevel.PT2_COMPILE + ): + chromium_log.increment(event_name, key, value) + else: + assert log_level == CompileEventLogLevel.COMPILATION_METRIC + top_event = chromium_log.get_outermost_event() + if event_name != top_event: + raise RuntimeError( + "Log level is COMPILATION_METRIC, but event_name isn't the toplevel event. " + "CompilationMetrics must be logged to the toplevel event. Consider using `increment_toplevel` directly." + ) + + metrics_context = get_metrics_context() + if not metrics_context.in_progress(): + raise RuntimeError( + "No metrics context is in progress. Please only call this function within a metrics context/dynamo_timed." + ) + + metrics_context.increment(key, value) + chromium_log.increment(event_name, key, value) + + @staticmethod + def increment_toplevel( + key: str, + value: int = 1, + log_level: CompileEventLogLevel = CompileEventLogLevel.COMPILATION_METRIC, + ) -> None: + """ + Increments a value on the toplevel metric. By default, logs to metric. + """ + chromium_log = get_chromium_event_logger() + top_event = chromium_log.get_outermost_event() + if top_event is None: + raise RuntimeError( + "No toplevel event active. Please only call this function within a metrics context/dynamo_timed." + ) + CompileEventLogger.increment(top_event, log_level, key, value) + + @staticmethod + def add_to_set( + event_name: str, log_level: CompileEventLogLevel, key: str, value: Any + ) -> None: + """ + Add metadata to a set of values with key . Creates a set if it doesn't exist. + """ + chromium_log = get_chromium_event_logger() + if ( + log_level == CompileEventLogLevel.CHROMIUM + or log_level == CompileEventLogLevel.PT2_COMPILE + ): + chromium_log.add_to_set(event_name, key, value) + else: + assert log_level == CompileEventLogLevel.COMPILATION_METRIC + top_event = chromium_log.get_outermost_event() + if event_name != top_event: + raise RuntimeError( + "Log level is COMPILATION_METRIC, but event_name isn't the toplevel event. " + "CompilationMetrics must be logged to the toplevel event. Consider using `add_to_set_metric` directly." + ) + + metrics_context = get_metrics_context() + if not metrics_context.in_progress(): + raise RuntimeError( + "No metrics context is in progress. Please only call this function within a metrics context/dynamo_timed." + ) + + metrics_context.add_to_set(key, value) + chromium_log.add_to_set(event_name, key, value) + + @staticmethod + def add_to_set_toplevel( + key: str, + value: Any, + log_level: CompileEventLogLevel = CompileEventLogLevel.COMPILATION_METRIC, + ) -> None: + """ + Same as add to set, just does it automatically to the toplevel event instead of having to explicitly name it. + Defaults to COMPILATION_METRIC log level. + """ + chromium_log = get_chromium_event_logger() + top_event = chromium_log.get_outermost_event() + if top_event is None: + raise RuntimeError( + "No toplevel event active. Please only call this function within a metrics context/dynamo_timed." + ) + CompileEventLogger.add_to_set(top_event, log_level, key, value) + + # Helper functions that are syntactic sugar + + @staticmethod + def chromium(event_name: str, **metadata: object) -> None: + """ + Add to in chromium. Each key/value of metadata will appear in the chromium trace. + should be the name of a timed event span passed to `dynamo_timed`. + """ + CompileEventLogger.add_data( + event_name, CompileEventLogLevel.CHROMIUM, overwrite=False, **metadata + ) + + @staticmethod + def pt2_compile(event_name: str, **metadata: object) -> None: + """ + Add to in chromium and PT2 Compile Events. + Each key/value of metadata will appear in the chromium trace. Each kwarg name becomes + a column in PT2 Compile Events, with the corresponding kwarg value. + should be the name of a timed event span passed to `dynamo_timed`, + with log_to_pt2_compile_events=True. + """ + CompileEventLogger.add_data( + event_name, CompileEventLogLevel.PT2_COMPILE, overwrite=False, **metadata + ) + + @staticmethod + def compilation_metric(overwrite: bool = False, **metadata: object) -> None: + """ + Add to the CompilationMetrics context. Also logs to PT2 Compile Events + and chromium. + Each key/value of metadata will appear in the chromium trace. Each kwarg name becomes + a column in PT2 Compile Events and Dynamo Compile, with the corresponding kwarg value. + """ + CompileEventLogger.add_toplevel( + CompileEventLogLevel.COMPILATION_METRIC, overwrite, **metadata + ) + + @staticmethod + def instant( + event_name: str, metadata: dict[str, Any], time_ns: Optional[int] = None + ) -> None: + """ + Log an instant event to chromium logs with name at time . The `args` field in + Perfetto will point to metadata. should be a value obtained from time.time_ns(). + """ + CompileEventLogger.log_instant_event( + event_name, metadata, time_ns, CompileEventLogLevel.CHROMIUM + ) + + @staticmethod + def try_add_pt2_compile(event_name: str, **metadata: object) -> None: + """ + Adds to an existing pt2_compile event, but silently returns if the event doesn't exist + or ChromiumEventLogger is not initialized. + This function is syntactic sugar for chromium_event_logger().try_add_event_data. + """ + if not chromium_event_log_active(): + return + chromium_log = get_chromium_event_logger() + chromium_log.try_add_event_data(event_name, **metadata) + + @staticmethod + def try_(method_fn: Callable[_P, Any], *args: _P.args, **kwargs: _P.kwargs) -> None: + """ + Special function that quietly runs a given method, returning if CHROMIUM_EVENT_LOG is None or metrics context is not set + """ + if not chromium_event_log_active(): + return + metrics_context = get_metrics_context() + if not metrics_context.in_progress(): + return + method_fn(*args, **kwargs) + + +_dynamo_timed_tls = threading.local() + + +@contextmanager +def dynamo_timed( + key: str, + # TODO(masneral): Deprecate this param. + phase_name: Optional[str] = None, + log_pt2_compile_event: bool = False, + metadata: Optional[dict[str, object]] = None, + dynamo_compile_column_us: Optional[str] = None, + compile_id: Optional[CompileId] = None, + is_backward: Optional[bool] = None, + log_waitcounter: bool = False, + waitcounter_name_override: Optional[str] = None, +) -> Generator[Any, None, None]: + """ + dynamo_timed is a context manager + By wrapping a function in dynamo_timed, we can get a few things: + + 1) Optionally log timings to pt2_compile_events. + 2) Optionally log timings to CompilationMetrics (dynamo_compile). + 3) Optionally log chromium events. + 4) Optionally increment a WaitCounter. + 5) Store a record in compilation_time_metrics + For example: + + def _foo(...): + with dynamo_timed("_foo"): + ... + + Would show up as an entry in our timing dict: + OrderedDict([('_foo', [0.083690, 0.23949, 3.1425e-05])]) + This is extremely useful for granular debugging. + + Although it is tempting to use dynamo_timed as a decorator, please do not. + In its decorator form it makes cProfile traces less useful as dynamo_timed + suddenly becomes a bottleneck for lots of function calls (as only one parent + pointer is recorded). + + Params: + - key: key into compile_time_metrics. If phase_name is not provided, this is + also the event name used for pt2_compile_events logs and chromium events. + - phase_name: Optional override for the event name. + - log_pt2_compile_event: Whether to log a pt2 compile event internally. + - metadata: Extra metadata to put in pt2_compile_events. + - dynamo_compile_column_us: If provided, updates the specified CompilationMetrics + field to be logged to dyname_compile column. We expect all columns to be _us; + therefore, the field name must end with "_us". + - compile_id: In the typical case, this parameter should not be needed. Use to + supply the compile_id for those cases where we want to log a compile_id where + it's not naturally available, e.g., for runtime autotuning. + - is_backward: Specify forward/backward directly when not available in a + CompileContext, e.g., during runtime autotuning. + that support it. + - log_waitcounter: If set, we'll log a waitcounter of the form "pytorch.dynamo_timed.{key}" + """ + if phase_name: + event_name = phase_name + fn_name = key + else: + event_name = key + fn_name = None + + if key not in compilation_time_metrics: + compilation_time_metrics[key] = [] + + metrics = compilation_time_metrics[key] + event_metadata = {} + if metadata: + event_metadata.update(metadata) + if fn_name: + event_metadata.update({"fn_name": fn_name}) + if is_backward is not None: + event_metadata.update({"is_backward": is_backward}) + + chromium_log: ChromiumEventLogger = get_chromium_event_logger() + start_ns = time.time_ns() + chromium_log.log_event_start( + event_name, start_ns, event_metadata, log_pt2_compile_event, compile_id + ) + + cx_mgrs: list[typing.Any] = [ + torch.profiler.record_function(f"{key} (dynamo_timed)") + ] + if log_waitcounter: + wc_name = waitcounter_name_override if waitcounter_name_override else key + cx_mgrs.append(_WaitCounter(f"pytorch.wait_counter.{wc_name}").guard()) + + is_compile_time = torch._guards.CompileContext.current_compile_id() is not None + if dynamo_compile_column_us: + # We're standardizing on microseconds for dynamo_compile timings. + assert dynamo_compile_column_us.endswith("_us") + + # Track nested dynamo_timed calls that update CompilationMetrics so we can + # bump a total duration only for the outermost metric. + if not hasattr(_dynamo_timed_tls, "depth"): + _dynamo_timed_tls.depth = 0 + _dynamo_timed_tls.depth += 1 + + # The corresponding WaitCounters that we bump for all overheads + if _dynamo_timed_tls.depth == 1: + cx_mgrs.append(_WaitCounter("pytorch.wait_counter.dynamo_compile").guard()) + if not is_compile_time: + runtime_wc = "pytorch.wait_counter.compile_runtime_overheads" + cx_mgrs.append(_WaitCounter(runtime_wc).guard()) + + try: + with contextlib.ExitStack() as stack: + for cx in cx_mgrs: + stack.enter_context(cx) + yield + finally: + end_ns = time.time_ns() + time_spent_ns = end_ns - start_ns + metrics.append(time_spent_ns / 1e9) + chromium_log.log_event_end( + event_name, end_ns, {}, start_ns, log_pt2_compile_event, compile_id + ) + if dynamo_compile_column_us: + # TODO: the events that we capture in calculate_time_spent() seem a little + # arbitrary. Currently, it's only those fields that are present in + # CompilationMetrics (but note that we accumulate by the associated event + # name, not the field name in CompilationMetrics). Do we want to keep it + # this way? + cumulative_time_spent_ns[event_name] += time_spent_ns + + # Bump the total duration for every outer event. + _dynamo_timed_tls.depth -= 1 + is_outer_event = _dynamo_timed_tls.depth == 0 + + duration_us = time_spent_ns // 1000 + if is_compile_time: + metrics_context = get_metrics_context() + if metrics_context.in_progress(): + metrics_context.increment(dynamo_compile_column_us, duration_us) + if is_outer_event: + metrics_context.increment("duration_us", duration_us) + else: + runtime_context = get_runtime_metrics_context() + runtime_context.increment(dynamo_compile_column_us, duration_us) + if is_outer_event: + extra = { + "compile_id": compile_id, + "is_runtime": True, + "is_forward": not is_backward, + } + runtime_context.increment("duration_us", duration_us, extra) + + +@overload +def compile_times(repr: Literal["str"], aggregate: bool = False) -> str: ... + + +@overload +# pyrefly: ignore [inconsistent-overload] +def compile_times( + repr: Literal["csv"], aggregate: bool = False +) -> tuple[list[str], list[object]]: ... + + +def compile_times( # type: ignore[misc] + repr: str = "str", aggregate: bool = False +) -> Union[str, None, tuple[list[str], list[str]]]: + """ + Get metrics about torchdynamo frontend/backend compilation times. + + Accumulates information from functions tagged with `dynamo_timed`. + + repr='str' returns a printable string for user interaction, and 'csv' + returns headers, rows which can be logged for output + + aggregate causes values from multiple compilations (e.g. split graphs) + to be accumulated into one value. If false, expect more than one value + per metric. + """ + + def fmt_fn(values: list[float], item_fn: Callable[[float], str] = str) -> str: + if aggregate: + return item_fn(sum(values)) + return ", ".join(map(item_fn, values)) + + if repr == "str": + rows = [ + (k, fmt_fn(compilation_time_metrics[k], item_fn=lambda x: f"{x:.4f}")) + for k in compilation_time_metrics + ] + out = "TorchDynamo compilation metrics:\n" + out += tabulate(rows, headers=("Function", "Runtimes (s)")) + return out + elif repr == "csv": + values = [ + fmt_fn(v, item_fn=lambda x: f"{x:.6f}") + for v in compilation_time_metrics.values() + ] + headers = list(compilation_time_metrics.keys()) + return headers, values + return None + + +@atexit.register +def dump_compile_times() -> None: + log.info(compile_times(repr="str", aggregate=True)) + + +tensortype_to_dtype = { + torch.FloatTensor: (torch.float32, torch.float), + torch.DoubleTensor: (torch.float64, torch.double), + torch.HalfTensor: (torch.float16, torch.half), + torch.BFloat16Tensor: (torch.bfloat16,), + torch.ByteTensor: (torch.uint8,), + torch.CharTensor: (torch.int8,), + torch.LongTensor: (torch.int64, torch.long), + torch.IntTensor: (torch.int32, torch.int), + torch.ShortTensor: (torch.int16, torch.short), + torch.BoolTensor: (torch.bool,), +} + + +class DuplicateWarningChecker: + def __init__(self, maxsize: int = 4096) -> None: + self.maxsize = maxsize + self.reset() + + def reset(self) -> None: + self.set: OrderedDict[Any, Any] = OrderedDict() + + def add(self, key: Union[str, tuple[object, object]]) -> bool: + if key in self.set: + self.set.move_to_end(key, last=True) + if not config.verbose: + return False + else: + self.set[key] = None + while len(self.set) > self.maxsize: + self.set.popitem(last=False) + return True + + +graph_break_dup_warning_checker = DuplicateWarningChecker() + + +def setup_compile_debug() -> contextlib.ExitStack: + compile_debug = os.environ.get("TORCH_COMPILE_DEBUG", "0") == "1" + + if compile_debug: + return add_file_handler() + + return contextlib.ExitStack() + + +def reset_graph_break_dup_checker() -> None: + graph_break_dup_warning_checker.reset() + + +# Matches ANSI escape sequences (CSI) +ANSI_ESCAPE_PATTERN = re.compile( + r""" + \x1B # ESC + \[ # [ + [0-?]* # Parameter bytes + [ -/]* # Intermediate bytes + [@-~] # Final byte + """, + re.VERBOSE, +) + + +class StripAnsiFormatter(logging.Formatter): + """Logging formatter that strips ANSI escape codes.""" + + def format(self, record): + msg = super().format(record) + return ANSI_ESCAPE_PATTERN.sub("", msg) + + +def add_file_handler() -> contextlib.ExitStack: + log_path = os.path.join(get_debug_dir(), "torchdynamo") + os.makedirs(log_path, exist_ok=True) + + log_file_handler = logging.FileHandler(os.path.join(log_path, "debug.log")) + log_file_handler.setFormatter(StripAnsiFormatter("%(message)s")) + logger = logging.getLogger("torch._dynamo") + logger.addHandler(log_file_handler) + + exitstack = contextlib.ExitStack() + exitstack.callback(lambda: logger.removeHandler(log_file_handler)) + return exitstack + + +def setup_log_file() -> contextlib.ExitStack: + exitstack = contextlib.ExitStack() + if config.log_file_name is not None: + log_file_handler = logging.FileHandler(config.log_file_name) + for logger in torch._logging._internal.get_loggers(): + logger.addHandler(log_file_handler) + exitstack.callback(lambda: logger.removeHandler(log_file_handler)) + return exitstack + + return exitstack + + +def gen_record_file_name(exc: Exception, code: CodeType) -> str: + return f"{get_debug_dir()}/error_recordings/\ +{code.co_name}_{type(exc).__name__}_{code.co_firstlineno}.rec" + + +def write_record_to_file(filename: str, exec_record: ExecutionRecord) -> None: + try: + if os.path.exists(filename): + log.warning( + "Unable to write execution record %s; file already exists.", filename + ) + else: + os.makedirs(os.path.dirname(filename), exist_ok=True) + with open(filename, "wb") as f: + exec_record.dump(f) + except Exception: + log.exception("Unable to write execution record %s", filename) + + +def count_calls(g: fx.Graph) -> int: + c = 0 + for n in g.nodes: + if "call" in n.op: + c += 1 + return c + + +def identity(x: T) -> T: + return x + + +def hashable(x: Any) -> bool: + try: + hash(x) + return True + except TypeError: + return False + # cannot hash writable memoryview object + except ValueError: + return False + + +def nothing(*args: Any, **kwargs: Any) -> None: + pass + + +class ExactWeakKeyDictionary: + """Similar to weakref.WeakKeyDictionary, but use `is`/`id` rather than `==` to compare equality""" + + def __init__(self) -> None: + self.values: dict[int, Any] = {} + self.refs: dict[int, weakref.ReferenceType[Any]] = {} + + def __getitem__(self, key: Any) -> Any: + return self.values[id(key)] + + def get(self, key: Any, default: Any = None) -> Any: + return self.values.get(id(key), default) + + def __contains__(self, key: Any) -> bool: + return id(key) in self.values + + def __setitem__(self, key: Any, value: Any) -> None: + idx = id(key) + if idx not in self.refs: + self.refs[idx] = weakref.ref(key, lambda ref: self._remove_id(idx)) + self.values[idx] = value + + def _remove_id(self, idx: int) -> None: + if idx in self.values: + del self.values[idx] + if idx in self.refs: + del self.refs[idx] + + def clear(self) -> None: + self.refs.clear() + self.values.clear() + + +@overload +def istype(obj: object, allowed_types: type[T]) -> TypeIs[T]: ... + + +@overload +def istype( + obj: object, allowed_types: tuple[type[list[T]], type[tuple[T, ...]]] +) -> TypeIs[T]: ... + + +@overload +def istype(obj: object, allowed_types: Iterable[type]) -> bool: ... + + +def istype(obj: object, allowed_types: Any) -> bool: + """isinstance() without subclasses""" + if isinstance(allowed_types, (tuple, list, set)): + return type(obj) in allowed_types + return type(obj) is allowed_types + + +if sys.version_info >= (3, 12): + # Some typing classes moved to C in 3.12, + # which no longer have the _Final mixin. + # Check for consistency e.g. here: + # https://github.com/python/cpython/blob/f2b82b3b3b1f8c7a81e84df35ee921e44517cf32/Lib/typing.py#L32 + _builtin_final_typing_classes = ( + typing.ParamSpecArgs, + typing.ParamSpecKwargs, + typing.ParamSpec, + typing.TypeVar, + typing.TypeVarTuple, + typing.TypeAliasType, + ) + + +if sys.version_info >= (3, 14): + _builtin_final_typing_classes += (typing.Union,) + + +def is_typing(value: Any) -> bool: + # _Final catches most of typing classes: + # - Any + # - Callable + # - Union (Python < 3.14) + # ... + # + # NB: we intentionally ignore classes that inherit from Generic, since they + # can be used as both TypingVariable as well as UserDefinedClassVariable. + if sys.version_info >= (3, 12) and isinstance(value, _builtin_final_typing_classes): + return True + return ( + isinstance(value, typing._Final) # type: ignore[attr-defined] + or value is typing.Generic + or value is typing.Union + ) + + +def is_numpy_int_type(value: Any) -> bool: + if not np: + return False + + return istype( + value, + ( + np.int8, + np.int16, + np.int32, + np.int64, + np.uint8, + np.uint16, + np.uint32, + np.uint64, + ), + ) + + +def is_numpy_float_type(value: Any) -> bool: + if not np: + return False + + return istype( + value, + ( + np.float16, + np.float32, + np.float64, + ), + ) + + +@overload +def is_lru_cache_wrapped_function( + value: Callable[..., T], +) -> TypeGuard[functools._lru_cache_wrapper[T]]: ... + + +@overload +def is_lru_cache_wrapped_function( + value: Any, +) -> TypeGuard[functools._lru_cache_wrapper[Any]]: ... + + +def is_lru_cache_wrapped_function( + value: Any, +) -> bool: + return isinstance(value, functools._lru_cache_wrapper) and is_function( + inspect.getattr_static(value, "__wrapped__") + ) + + +_FuncTypes: TypeAlias = Union[ + types.FunctionType, + types.BuiltinFunctionType, + types.MethodDescriptorType, + types.WrapperDescriptorType, +] + + +def is_function_or_wrapper( + value: Any, +) -> TypeIs[Union[_FuncTypes, torch._ops.OpOverloadPacket, torch._ops.OpOverload]]: + return is_function(value) or isinstance( + value, (torch._ops.OpOverloadPacket, torch._ops.OpOverload) + ) + + +def is_function( + value: Any, +) -> TypeIs[_FuncTypes]: + return isinstance( + value, + ( + types.FunctionType, + types.BuiltinFunctionType, + types.MethodDescriptorType, + types.WrapperDescriptorType, + ), + ) + + +cmp_name_to_op_mapping = { + "__eq__": operator.eq, + "__ne__": operator.ne, + "__lt__": operator.lt, + "__le__": operator.le, + "__gt__": operator.gt, + "__ge__": operator.ge, +} + + +cmp_name_to_op_str_mapping = { + "__eq__": "==", + "__ne__": "!=", + "__lt__": "<", + "__le__": "<=", + "__gt__": ">", + "__ge__": ">=", +} + + +def is_wrapper_or_member_descriptor( + value: Any, +) -> TypeIs[ + Union[ + types.GetSetDescriptorType, + types.MethodDescriptorType, + types.WrapperDescriptorType, + types.MemberDescriptorType, + types.MethodWrapperType, + ] +]: + return isinstance( + value, + ( + # set up by PyGetSetDef + types.GetSetDescriptorType, + # set by PyMethodDef, e.g. list.append + types.MethodDescriptorType, + # slots - list.__add__ + types.WrapperDescriptorType, + # set up by PyMemberDef + types.MemberDescriptorType, + # wrapper over C functions + types.MethodWrapperType, + ), + ) + + +def unwrap_if_wrapper(fn: Any) -> Any: + return unwrap_with_attr_name_if_wrapper(fn)[0] + + +def unwrap_with_attr_name_if_wrapper(fn: Any) -> tuple[Any, Optional[str]]: + # TODO(anijain2305) - Investigate if we can get rid of this function + # unpack @torch._dynamo.optimize()(fn) wrapped function + if is_function(fn) and inspect.getattr_static(fn, "_torchdynamo_inline", False): + fn = inspect.getattr_static(fn, "_torchdynamo_inline", fn) + attr_name = "_torchdynamo_inline" + else: + attr_name = None + return fn, attr_name + + +def is_numpy_ndarray(value: Any) -> TypeGuard[np.ndarray]: # type: ignore[type-arg] + if not np: + return False + + return istype(value, np.ndarray) + + +def istensor(obj: Any) -> bool: + """Check of obj is a tensor""" + tensor_list: tuple[type, ...] = ( + torch.Tensor, + torch.nn.Parameter, + *config.traceable_tensor_subclasses, + ) + tensor_list = tensor_list + (torch._subclasses.FakeTensor,) + return istype(obj, tensor_list) + + +def is_lazy_module(mod: Any) -> bool: + return isinstance(mod, LazyModuleMixin) + + +@functools.lru_cache(4096) +def print_once(*args: Any) -> None: + print(*args) + + +def make_cell(val: Any = None) -> types.CellType: + """Some black magic to create a cell object that usually only exists in a closure""" + x = val + + def f() -> Any: + return x + + assert f.__closure__ is not None and len(f.__closure__) == 1 + return f.__closure__[0] + + +def proxy_args_kwargs(args: Any, kwargs: Any) -> tuple[tuple[Any, ...], dict[str, Any]]: + try: + proxy_args = tuple(arg.as_proxy() for arg in args) + proxy_kwargs = {key: arg.as_proxy() for key, arg in kwargs.items()} + return proxy_args, proxy_kwargs + except NotImplementedError as e: + from .exc import unimplemented + from .variables.base import typestr + + unimplemented( + gb_type="Failed to convert args/kwargs to proxy", + context=f"call_function args: {typestr(*args)} {typestr(*list(kwargs.values()))}", + explanation="Missing `as_proxy()` implementation for some arg/kwarg.", + hints=[], + from_exc=e, + ) + + +def to_int_ms(v: Optional[float]) -> Optional[int]: + return None if v is None else int(v * 1000) + + +# float64 timestamp has a quarter microsecond precision in 2024, so while +# this is suboptimal we shouldn't meaningfully lose precision +def to_int_us(v: Optional[float]) -> Optional[int]: + return None if v is None else int(v * 1_000_000) + + +# Version field added to every log. Increment to make it easier to distinguish new +# vs. old entries when you make a substantive change to how the logs are populated. +LOG_FORMAT_VERSION = 3 + + +@dataclasses.dataclass +class CompilationMetrics: + compile_id: Optional[str] = None + frame_key: Optional[str] = None + co_name: Optional[str] = None + co_filename: Optional[str] = None + co_firstlineno: Optional[int] = None + cache_size: Optional[int] = None + accumulated_cache_size: Optional[int] = None + guard_count: Optional[int] = None + shape_env_guard_count: Optional[int] = None + graph_op_count: Optional[int] = None + graph_node_count: Optional[int] = None + graph_input_count: Optional[int] = None + start_time: Optional[float] = None + entire_frame_compile_time_s: Optional[float] = None + backend_compile_time_s: Optional[float] = None + inductor_compile_time_s: Optional[float] = None + code_gen_time_s: Optional[float] = None + fail_type: Optional[str] = None + fail_reason: Optional[str] = None + fail_user_frame_filename: Optional[str] = None + fail_user_frame_lineno: Optional[int] = None + non_compliant_ops: Optional[set[str]] = None + compliant_custom_ops: Optional[set[str]] = None + restart_reasons: Optional[set[str]] = None + dynamo_time_before_restart_s: Optional[float] = None + stack_trace: Optional[list[str]] = None + exception_stack_trace: Optional[list[str]] = None + graph_node_shapes: Optional[str] = None + # Sometimes, we will finish analyzing a frame but conclude we don't want + # to install any guarded code. True means we actually decided to install + # a compiled frame + has_guarded_code: Optional[bool] = None + remote_cache_time_saved_s: Optional[float] = None + structured_logging_overhead_s: Optional[float] = None + config_suppress_errors: Optional[bool] = None + config_inline_inbuilt_nn_modules: Optional[bool] = None + specialize_float: Optional[bool] = None + dynamo_config: Optional[str] = None + compiler_config: Optional[str] = None + is_forward: Optional[bool] = None + num_triton_bundles: Optional[int] = None + remote_fx_graph_cache_get_time_ms: Optional[int] = None + remote_fx_graph_cache_put_time_ms: Optional[int] = None + start_time_us: Optional[int] = None + duration_us: Optional[int] = None + dynamo_cumulative_compile_time_us: Optional[int] = None + aot_autograd_cumulative_compile_time_us: Optional[int] = None + inductor_cumulative_compile_time_us: Optional[int] = None + inductor_code_gen_cumulative_compile_time_us: Optional[int] = None + triton_compile_time_us: Optional[int] = None + runtime_cudagraphify_time_us: Optional[int] = None + runtime_triton_autotune_time_us: Optional[int] = None + dynamo_compile_time_before_restart_us: Optional[int] = None + distributed_ephemeral_timeout_us: Optional[int] = None + structured_logging_overhead_us: Optional[int] = None + remote_fx_graph_cache_get_time_us: Optional[int] = None + remote_fx_graph_cache_put_time_us: Optional[int] = None + backward_cumulative_compile_time_us: Optional[int] = None + end_time_us: Optional[int] = None + pre_grad_pass_time_us: Optional[int] = None + post_grad_pass_time_us: Optional[int] = None + joint_graph_pass_time_us: Optional[int] = None + log_format_version: int = LOG_FORMAT_VERSION + inductor_config: Optional[str] = None + remote_cache_version: Optional[int] = None + inductor_fx_remote_cache_hit_count: Optional[int] = None + inductor_fx_remote_cache_miss_count: Optional[int] = None + inductor_fx_remote_cache_backend_type: Optional[str] = None + inductor_fx_remote_cache_hit_keys: Optional[str] = None + inductor_fx_remote_cache_miss_keys: Optional[str] = None + cuda_version: Optional[str] = None + triton_version: Optional[str] = None + feature_usage: Optional[dict[str, bool]] = None + compile_time_autotune_time_us: Optional[int] = None + is_runtime: Optional[bool] = False + gc_time_us: Optional[int] = None + tensorify_float_attempt: Optional[bool] = None + tensorify_float_success: Optional[bool] = None + tensorify_float_failure: Optional[set[str]] = None + guard_latency_us: Optional[float] = None + recompile_reason: Optional[str] = None + num_graph_breaks: Optional[int] = None + triton_kernel_compile_times_us: Optional[str] = None + ir_count: Optional[int] = None + cudagraph_skip_reason: Optional[str] = None + python_version: Optional[str] = None + pgo_put_remote_code_state_time_us: Optional[int] = None + pgo_get_remote_code_state_time_us: Optional[int] = None + # The number of elements within parameters. This is classically what people + # think of when they think of parameters in a ML model. + param_numel: Optional[int] = None + # The number of elements counted by bytes - i.e. a float32 is 4 bytes + # per element. + param_bytes: Optional[int] = None + # The number of parameters counted by fields. This is mostly a proxy for + # the number of distinct type of params. + param_count: Optional[int] = None + recompile_user_contexts: Optional[set[str]] = None + inline_inbuilt_nn_modules_candidate: Optional[bool] = False + pytorch_version: Optional[str] = None + inductor_provenance: Optional[set[str]] = None + + @classmethod + def create(cls, metrics: dict[str, Any]) -> CompilationMetrics: + """ + Factory method to create a CompilationMetrics from a dict of fields. + Includes the logic to add legacy fields and any pre-processing, e.g., + we transform some fields to comma-separated strings for scuba logging. + """ + + def us_to_s(metric: Optional[int]) -> Optional[float]: + return metric / 1e6 if metric is not None else None + + def us_to_ms(metric: Optional[int]) -> Optional[int]: + return metric // 1000 if metric is not None else None + + def collection_to_str(metric: Optional[Any]) -> Optional[str]: + def safe_str(item: Any) -> str: + try: + return str(item) + except Exception: + return "" + + if metric is None: + return None + + if not isinstance(metric, (set, list)): + return "" + + return ",".join(safe_str(item) for item in sorted(metric)) + + def collection_to_json_str(metric: Optional[Any]) -> Optional[str]: + if metric is None: + return None + try: + return json.dumps(list(metric)) + except Exception: + return "" + + # TODO: The following are legacy fields, populated from the fields that replace + # them. Remove these when we decide we can really deprecate them. + legacy_metrics = { + "start_time": us_to_s(metrics.get("start_time_us")), + "entire_frame_compile_time_s": us_to_s( + metrics.get("dynamo_cumulative_compile_time_us") + ), + "backend_compile_time_s": us_to_s( + metrics.get("aot_autograd_cumulative_compile_time_us") + ), + "inductor_compile_time_s": us_to_s( + metrics.get("inductor_cumulative_compile_time_us") + ), + "code_gen_time_s": us_to_s( + metrics.get("inductor_code_gen_cumulative_compile_time_us") + ), + "remote_cache_time_saved_s": us_to_s( + metrics.get("distributed_ephemeral_timeout_us") + ), + "remote_fx_graph_cache_get_time_ms": us_to_ms( + metrics.get("remote_fx_graph_cache_get_time_us") + ), + "remote_fx_graph_cache_put_time_ms": us_to_ms( + metrics.get("remote_fx_graph_cache_put_time_us") + ), + "structured_logging_overhead_s": us_to_s( + metrics.get("structured_logging_overhead_us") + ), + } + + all_metrics = {**legacy_metrics, **metrics} + + # Processing before logging: + all_metrics["inductor_fx_remote_cache_hit_keys"] = collection_to_str( + all_metrics.get("inductor_fx_remote_cache_hit_keys") + ) + all_metrics["inductor_fx_remote_cache_miss_keys"] = collection_to_str( + all_metrics.get("inductor_fx_remote_cache_miss_keys") + ) + all_metrics["triton_kernel_compile_times_us"] = collection_to_json_str( + all_metrics.get("triton_kernel_compile_times_us") + ) + compile_id = all_metrics.get("compile_id") + all_metrics["compile_id"] = str(compile_id) if compile_id else None + + # pyrefly: ignore [bad-argument-type] + return cls(**all_metrics) + + +DEFAULT_COMPILATION_METRICS_LIMIT = 64 + + +_compilation_metrics: collections.deque[CompilationMetrics] = collections.deque( + maxlen=DEFAULT_COMPILATION_METRICS_LIMIT +) + + +def add_compilation_metrics_to_chromium(c: CompilationMetrics) -> None: + """ + These are the common fields in CompilationMetrics that existed before + metrics_context, and aren't set by MetricsContext.set(). We add the subset + of them that make sense in `dynamo`/toplevel events in PT2 Compile Events + directly. + + If you're tempted to add to this list, consider using CompileEventLogger.compilation_metric() + instead, which will automatically also add it to tlparse and PT2 Compile Events. + TODO: Get rid of this function and replace it with CompileEventLogger directly instead. + """ + event_logger = get_chromium_event_logger() + event_name = event_logger.get_outermost_event() + if not event_name: + return + event_logger.add_event_data( + event_name=event_name, + frame_key=c.frame_key, + co_name=c.co_name, + co_filename=c.co_filename, + co_firstlineno=c.co_firstlineno, + cache_size=c.cache_size, + accumulated_cache_size=c.accumulated_cache_size, + guard_count=c.guard_count, + shape_env_guard_count=c.shape_env_guard_count, + graph_op_count=c.graph_op_count, + graph_node_count=c.graph_node_count, + graph_input_count=c.graph_input_count, + fail_type=c.fail_type, + fail_reason=c.fail_reason, + fail_user_frame_filename=c.fail_user_frame_filename, + fail_user_frame_lineno=c.fail_user_frame_lineno, + # Sets aren't JSON serializable + non_compliant_ops=( + list(c.non_compliant_ops) if c.non_compliant_ops is not None else None + ), + compliant_custom_ops=( + list(c.compliant_custom_ops) if c.compliant_custom_ops is not None else None + ), + restart_reasons=( + list(c.restart_reasons) if c.restart_reasons is not None else None + ), + dynamo_time_before_restart_s=c.dynamo_time_before_restart_s, + has_guarded_code=c.has_guarded_code, + dynamo_config=c.dynamo_config, + ) + + +def _get_dynamo_config_for_logging() -> Optional[str]: + def clean_for_json(d: dict[str, Any]) -> dict[str, Any]: + blocklist = { + "TYPE_CHECKING", + "log_file_name", + "verbose", + "repro_after", + "repro_level", + "repro_forward_only", + "repro_tolerance", + "repro_ignore_non_fp", + "same_two_models_use_fp64", + "base_dir", + "debug_dir_root", + "_save_config_ignore", + "log_compilation_metrics", + "inject_BUILD_SET_unimplemented_TESTING_ONLY", + "_autograd_backward_strict_mode_banned_ops", + "reorderable_logging_functions", + "ignore_logger_methods", + "traceable_tensor_subclasses", + "nontraceable_tensor_subclasses", + "_custom_ops_profile", + } + + return { + key: sorted(value) if isinstance(value, set) else value + for key, value in d.items() + if key not in blocklist + } + + config_dict = clean_for_json(config.get_config_copy()) + return json.dumps(config_dict, sort_keys=True) + + +def _compiler_config_for_logging() -> Optional[str]: + def clean_for_json(d: dict[str, Any]) -> dict[str, Any]: + blocklist = { + "TYPE_CHECKING", + } + + return { + key: sorted(value) if isinstance(value, set) else value + for key, value in d.items() + if key not in blocklist + } + + if not torch.compiler.config: + return None + + try: + compiler_config_copy = torch.compiler.config.get_config_copy() # type: ignore[attr-defined] + except (TypeError, AttributeError): + return "Compiler Config cannot be pickled" + + config_dict = clean_for_json(compiler_config_copy) + return json.dumps(config_dict, sort_keys=True) + + +def _scrubbed_inductor_config_for_logging() -> Optional[str]: + """ + Method to parse and scrub uninteresting configs from inductor config + """ + + # TypeSafeSerializer for json.dumps() + # Skips complex types as values in config dict + class TypeSafeSerializer(json.JSONEncoder): + def default(self, o: Any) -> Any: + try: + return super().default(o) + except Exception: + return "Value is not JSON serializable" + + keys_to_scrub: set[Any] = set() + inductor_conf_str = None + inductor_config_copy = None + + if torch._inductor.config: + try: + inductor_config_copy = torch._inductor.config.get_config_copy() + except (TypeError, AttributeError, RuntimeError, AssertionError): + inductor_conf_str = "Inductor Config cannot be pickled" + + if inductor_config_copy is not None: + try: + for key, val in inductor_config_copy.items(): + if not isinstance(key, str): + keys_to_scrub.add(key) + # Convert set() to list for json.dumps() + if isinstance(val, set): + inductor_config_copy[key] = list(val) + # Evict unwanted keys + for key in keys_to_scrub: + del inductor_config_copy[key] + # Stringify Inductor config + inductor_conf_str = json.dumps( + inductor_config_copy, + cls=TypeSafeSerializer, + skipkeys=True, + sort_keys=True, + ) + except Exception: + # Don't crash because of runtime logging errors + inductor_conf_str = "Inductor Config is not JSON serializable" + return inductor_conf_str + + +def record_compilation_metrics( + start_time_ns: int, + end_time_ns: int, + metrics: dict[str, Any], + exc_type: Optional[type[BaseException]], + exc_value: Optional[BaseException], +) -> None: + if torch._inductor.utils.should_use_remote_fx_graph_cache(): + try: + from torch._inductor.fb.remote_cache import REMOTE_CACHE_VERSION + + remote_cache_version = REMOTE_CACHE_VERSION + inductor_fx_remote_cache_backend_type = "_ManifoldCache" + except ModuleNotFoundError: + remote_cache_version = None + inductor_fx_remote_cache_backend_type = None + else: + inductor_fx_remote_cache_backend_type = None + remote_cache_version = None + + # Populate the compile_id from the metrics context if it's set. Otherwise, + # look for it in the current compile context. + compile_id = metrics.get("compile_id") + if not compile_id: + compile_id = torch._guards.CompileContext.current_compile_id() + + common_metrics = { + "compile_id": compile_id, + "start_time_us": start_time_ns // 1000, + "end_time_us": end_time_ns // 1000, + "fail_type": exc_type.__qualname__ if exc_type else None, + "fail_reason": str(exc_value) if exc_value else None, + "structured_logging_overhead_us": to_int_us( + torch._logging.get_structured_logging_overhead() + ), + "dynamo_config": _get_dynamo_config_for_logging(), + "config_suppress_errors": config.suppress_errors, + "config_inline_inbuilt_nn_modules": config.inline_inbuilt_nn_modules, + "inductor_config": _scrubbed_inductor_config_for_logging(), + "compiler_config": _compiler_config_for_logging(), + "cuda_version": torch.version.cuda, + "triton_version": triton.__version__ if has_triton() else "", + "remote_cache_version": remote_cache_version, + "inductor_fx_remote_cache_backend_type": inductor_fx_remote_cache_backend_type, + "python_version": sys.version, + "pytorch_version": torch.__version__, + } + + compilation_metrics = CompilationMetrics.create({**common_metrics, **metrics}) + _compilation_metrics.append(compilation_metrics) + + name = "compilation_metrics" + if compilation_metrics.is_forward is False: + name = "bwd_" + name + if compilation_metrics.is_runtime is True: + name = name + "_runtime" + + torch._logging.trace_structured( + name, + lambda: { + k: list(v) if isinstance(v, set) else v + for k, v in dataclasses.asdict(compilation_metrics).items() + }, + # NB: Because compilation metrics *includes* the logging overhead time, + # we can't both *measure* the logging overhead of compilation metrics + # without making it inconsistent with compilation metrics itself, so + # we ignore the (hopefully small) time spent logging compilation metrics + record_logging_overhead=False, + # These may be runtime logs, e.g., runtime autotunning, so we provide + # the CompileId from the compilation metrics in case it's not available + # in the current trace. + compile_id=compile_id, + ) + + # If there's a chromium event in flight, add the CompilationMetrics to it. + add_compilation_metrics_to_chromium(compilation_metrics) + + # Finally log the compilation metrics. + if config.log_compilation_metrics: + log_compilation_event(compilation_metrics) + + +# record_compilation_metrics is called by the singleton MetricsContext exit handler. +_METRICS_CONTEXT = MetricsContext(on_exit=record_compilation_metrics) +_RUNTIME_METRICS_CONTEXT = RuntimeMetricsContext(on_exit=record_compilation_metrics) + + +def set_compilation_metrics_limit(new_size: int) -> None: + global _compilation_metrics + while len(_compilation_metrics) > new_size: + _compilation_metrics.popleft() + new_deque = collections.deque(_compilation_metrics, maxlen=new_size) + _compilation_metrics = new_deque + + +def clear_compilation_metrics() -> None: + global _compilation_metrics + _compilation_metrics.clear() + + +def get_compilation_metrics() -> list[CompilationMetrics]: + return list(_compilation_metrics) + + +class ChromiumEventLogger: + """Logs chromium events to structured logs. tlparse will concatenate these into a perfetto UI link. + + See https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview#heading=h.yr4qxyxotyw for + a specification of the Chromium Event JSON format. + """ + + def get_stack(self) -> list[str]: + """ + The main event stack, with every chromium event. + Logged to tlparse. + """ + if hasattr(self.tls, "stack"): + return self.tls.stack + else: + self.tls.stack = [] + return self.tls.stack + + def get_outermost_event(self) -> Optional[str]: + """ + Get the outermost event name (i.e. the longest running event) + or None if the stack is empty. + """ + stack = self.get_stack() + return stack[0] if stack else None + + def get_pt2_compile_substack(self) -> list[str]: + """ + A smaller subset of the main stack that gets used to log + PT2 Compile Events internally. + """ + if hasattr(self.tls, "pt2_compile_substack"): + return self.tls.pt2_compile_substack + else: + self.tls.pt2_compile_substack = [] + return self.tls.pt2_compile_substack + + def get_event_data(self) -> dict[str, Any]: + if not hasattr(self.tls, "event_data"): + self.tls.event_data = {} + return self.tls.event_data + + def __init__(self) -> None: + self.tls = threading.local() + + from . import config + + # Generate a unique id for this logger, which we can use in scuba to filter down + # to a single python run. + if config.pt2_compile_id_prefix: + self.id_ = f"{config.pt2_compile_id_prefix}-{uuid.uuid4()}" + else: + self.id_ = str(uuid.uuid4()) + + # TODO: log to init/id tlparse after I add support for it + log.info("ChromiumEventLogger initialized with id %s", self.id_) + + def try_add_event_data(self, event_name: str, **kwargs: Any) -> None: + """ + Same as add_event_data, but will silently not log if the event isn't in the stack. + """ + if event_name not in self.get_stack(): + return + self.add_event_data(event_name, **kwargs) + + def add_event_data( + self, + event_name: str, + **kwargs: Any, + ) -> None: + """ + Adds additional metadata info to an in-progress event + This metadata is recorded in the END event + """ + if event_name not in self.get_stack(): + raise RuntimeError( + f"Event {repr(event_name)} not in {self.get_stack()}. " + "Cannot add metadata to events that aren't in progress. " + "Please make sure the event has started and hasn't ended." + ) + event_data = self.get_event_data() + if event_name not in event_data: + event_data[event_name] = {} + event_data[event_name].update(kwargs) + + def increment(self, event_name: str, key: str, value: int) -> None: + """ + Increment an integer event data field by the given amount + """ + if event_name not in self.get_stack(): + raise RuntimeError( + f"Event {repr(event_name)} not in {self.get_stack()}. " + "Cannot add metadata to events that aren't in progress. " + "Please make sure the event has started and hasn't ended." + ) + + event_data = self.get_event_data() + if event_name not in event_data: + event_data[event_name] = {} + if key not in event_data[event_name]: + event_data[event_name][key] = 0 + event_data[event_name][key] += value + + def add_to_set( + self, + event_name: str, + key: str, + value: Any, + ) -> None: + """ + Add a value to a set within a event_name's metadata if it exists + """ + if event_name not in self.get_stack(): + raise RuntimeError( + f"Event {repr(event_name)} not in {self.get_stack()}. " + "Cannot add metadata to events that aren't in progress. " + "Please make sure the event has started and hasn't ended." + ) + event_data = self.get_event_data() + if event_name not in event_data: + event_data[event_name] = {} + if key not in event_data[event_name]: + event_data[event_name][key] = set() + event_data[event_name][key].add(value) + + def log_event_start( + self, + event_name: str, + time_ns: int, + metadata: dict[str, Any], + log_pt2_compile_event: bool = False, + compile_id: Optional[CompileId] = None, + ) -> None: + """ + Logs the start of a single event. + :param str event_name Name of event to appear in trace + :param time_ns Timestamp in nanoseconds + :param metadata: Any extra metadata associated with this event + :param log_pt2_compile_event: If True, log to pt2_compile_events + :param compile_id: Explicit compile_id (rather than using the current context) + """ + compile_id = compile_id or torch._guards.CompileContext.current_compile_id() + metadata["compile_id"] = str(compile_id) + self._log_timed_event( + event_name, + time_ns, + "B", + metadata, + ) + self.get_stack().append(event_name) + # Add metadata from start event + self.add_event_data(event_name, **metadata) + if log_pt2_compile_event: + self.get_pt2_compile_substack().append(event_name) + + def reset(self) -> None: + # We this on every compile in case a compile crashes or restarts and we haven't + # cleared the stack. + stack = self.get_stack() + substack = self.get_pt2_compile_substack() + stack.clear() + substack.clear() + event_data = self.get_event_data() + event_data.clear() + + def log_event_end( + self, + event_name: str, + time_ns: int, + metadata: dict[str, Any], + start_time_ns: int, + log_pt2_compile_event: bool, + compile_id: Optional[CompileId] = None, + ) -> None: + """ + Logs the end of a single event. This function should only be + called after log_event_start with the same event_name. + :param event_name: Name of event to appear in trace + :param time_ns: Timestamp in nanoseconds + :param metadata: Any extra metadata associated with this event + :param start_time_ns: The start time timestamp in nanoseconds + :param log_pt_compile_event: If True, log to pt2_compile_events + :param compile_id: Explicit compile_id (rather than using the current context) + """ + compile_id = compile_id or torch._guards.CompileContext.current_compile_id() + metadata["compile_id"] = str(compile_id) + + # Grab metadata collected during event span + all_event_data = self.get_event_data() + if event_name in all_event_data: + event_metadata = all_event_data[event_name] + del all_event_data[event_name] + else: + event_metadata = {} + # Add the passed in metadata + event_metadata.update(metadata) + + event = self._log_timed_event( + event_name, + time_ns, + "E", + event_metadata, + ) + + def pop_stack(stack: list[str]) -> None: + while event_name != stack[-1]: + # If the event isn't the most recent one to end, pop + # off the stack until it is. + # Since event_name in self.stack, this pop is always safe + log.warning( + "ChromiumEventLogger: Detected overlapping events, fixing stack" + ) + stack.pop() + + event_stack = self.get_stack() + # These stack health checks currently never happen, + # but they're written this way to future proof any weird event + # overlaps in the future. + if event_name not in event_stack: + # Something went wrong, we never called start on this event, + # or it was skipped due to overlapping events below + log.warning("ChromiumEventLogger: Start event not in stack, ignoring") + return + + pop_stack(event_stack) + + if log_pt2_compile_event: + pt2_compile_substack = self.get_pt2_compile_substack() + pop_stack(pt2_compile_substack) + log_chromium_event_internal( + event, pt2_compile_substack, self.id_, start_time_ns + ) + # Pop actual event off of stack + pt2_compile_substack.pop() + + # Finally pop the actual event off the stack + event_stack.pop() + + def _log_timed_event( + self, + event_name: str, + time_ns: int, + phase: str, + metadata: Optional[dict[str, Any]] = None, + ) -> dict[str, Any]: + """ + Logs a timed event in chromium format. See log_event_start, log_event_end, etc. + """ + event = { + "name": event_name, + "ts": time_ns / 1000, # Chromium events are in micro seconds + "args": metadata, + "ph": phase, + # These categories are needed in all chromium traces + "cat": "dynamo_timed", + "tid": 0, + "pid": 0, # pid should be specified on all logs, we don't personally care about the actual process id + } + torch._logging.trace_structured( + "chromium_event", + payload_fn=lambda: event, + suppress_context=False, + expect_trace_id=False, # Not every chromium event will have a trace_id + ) + record_chromium_event_internal(event) + return event + + def log_instant_event( + self, + event_name: str, + time_ns: int, + metadata: Optional[dict[str, Any]] = None, + # By default, an instant event isn't logged internally, only to structured logging. + log_pt2_compile_event: bool = False, + ) -> None: + """ + Log an instant event with no associated duration. + :param str event_name: Name of event to appear in trace + :param int time_ns Timestamp in nanoseconds + :param Optional[Dict[str, Any]] metadata: Any extra metadata associated with this event + :param str cname optional color for the arrow in the trace + """ + if metadata is None: + metadata = {} + compile_id = str(torch._guards.CompileContext.current_compile_id()) + metadata["compile_id"] = compile_id + event = { + "name": event_name, + "ts": time_ns / 1000, + "args": metadata, + "ph": "i", + # These categories are needed in all chromium traces + "cat": "dynamo_timed", + "tid": 0, + "pid": 0, + "s": "p", # We use "process" level instant events so they all appear on the same row in the trace. + } + torch._logging.trace_structured( + "chromium_event", + payload_fn=lambda: event, + suppress_context=False, + expect_trace_id=True, + ) + if log_pt2_compile_event: + # Log an instant event with the same start and end time + log_chromium_event_internal( + event, self.get_pt2_compile_substack(), self.id_, time_ns + ) + + +CHROMIUM_EVENT_LOG: Optional[ChromiumEventLogger] = None + + +def get_chromium_event_logger() -> ChromiumEventLogger: + global CHROMIUM_EVENT_LOG + if CHROMIUM_EVENT_LOG is None: + CHROMIUM_EVENT_LOG = ChromiumEventLogger() + return CHROMIUM_EVENT_LOG + + +def chromium_event_log_active() -> bool: + global CHROMIUM_EVENT_LOG + return CHROMIUM_EVENT_LOG is not None + + +@contextmanager +def chromium_event_timed( + event_name: str, + reset_event_log_on_exit: bool = False, + log_pt2_compile_event: bool = False, +) -> Generator[Any, None, None]: + """ + Context manager that creates a chromium start and end event. Chromium event + logging is integrated with dynamo_timed, so you probably want to use that + instead. Use this context manager only if you want to avoid dynamo_timed. + """ + chromium_event_log = get_chromium_event_logger() + chromium_start_time = time.time_ns() + chromium_event_log.log_event_start( + event_name, + chromium_start_time, + {}, + log_pt2_compile_event, + ) + try: + yield + finally: + chromium_event_log.log_event_end( + event_name, + time.time_ns(), + {}, + chromium_start_time, + log_pt2_compile_event, + ) + if reset_event_log_on_exit: + chromium_event_log.reset() + + +@dataclasses.dataclass +class CleanupHook: + """Remove a global variable when hook is called""" + + scope: dict[str, Any] + name: str + + def __call__(self, *args: Any) -> None: + # Make sure we're not shutting down + if CleanupManager is not None: + CleanupManager.count -= 1 + del self.scope[self.name] + + @staticmethod + def create(scope: dict[str, Any], name: str, val: Any) -> CleanupHook: + assert name not in scope + CleanupManager.count += 1 + scope[name] = val + return CleanupHook(scope, name) + + +class CleanupManager(ExactWeakKeyDictionary): + count = 0 + instance: ClassVar[CleanupManager] + + def _remove_id(self, idx: int) -> None: + for hook in self.values[idx]: + hook() + super()._remove_id(idx) + + +CleanupManager.instance = CleanupManager() + + +def clone_tensor(x: torch.Tensor) -> torch.Tensor: + """Clone the tensor and its gradient""" + y = x.clone().requires_grad_(x.requires_grad) + if x.is_leaf and x.grad is not None: + y.grad = x.grad.clone() + return y + + +def clone_input( + x: torch.Tensor, *, dtype: Optional[torch.dtype] = None +) -> torch.Tensor: + """copy while preserving strides""" + # TODO: this is questionable + if is_fake(x): + # this func fails on fake tensors in __torch_dispatch__ + return x + + def torch_clone(x: torch.Tensor) -> torch.Tensor: + y = torch.clone(x) + if x.is_leaf: + y.requires_grad_(x.requires_grad) + if x.is_leaf and x.grad is not None: + y.grad = clone_input(x.grad, dtype=dtype) + if hasattr(x, "_dynamo_dynamic_indices"): + y._dynamo_dynamic_indices = x._dynamo_dynamic_indices.copy() # type: ignore[attr-defined] + return y + + with torch.no_grad(): + if x.device.type == "xla": + # Access data_ptr() for a xla tensor will cause crash + return torch_clone(x) + + # Handle sparse storage (no stride). + if x.layout is torch.sparse_coo: + return torch.sparse_coo_tensor( + torch_clone(x._indices()), + torch_clone(x._values()), + x.shape, + is_coalesced=x.is_coalesced(), + ) + elif is_sparse_compressed(x): + if x.layout in {torch.sparse_csr, torch.sparse_bsr}: + compressed_indices = x.crow_indices() + plain_indices = x.col_indices() + else: + compressed_indices = x.ccol_indices() + plain_indices = x.row_indices() + return torch.sparse_compressed_tensor( + torch_clone(compressed_indices), + torch_clone(plain_indices), + torch_clone(x.values()), + x.shape, + layout=x.layout, + ) + elif is_traceable_wrapper_subclass(x): + # Questionable - but this is required to not fail executorch related + # torchao tests. + return torch_clone(x) + + needed_size = sum( + (shape - 1) * stride for shape, stride in zip(x.size(), x.stride()) + ) + if x.is_quantized: + result = torch.empty_quantized((needed_size + 32,), x) + else: + result = torch.empty( + needed_size + 32, dtype=dtype or x.dtype, device=x.device + ) + cache_line_offset = ( + (x.data_ptr() - result.data_ptr()) % 32 + ) // x.element_size() + result.as_strided_(x.size(), x.stride(), cache_line_offset) + try: + result.copy_(x.clone()) + if x.is_leaf: + result.requires_grad_(x.requires_grad) + if x.is_leaf and x.grad is not None: + result.grad = clone_input(x.grad, dtype=dtype) + except RuntimeError: + # RuntimeError: unsupported operation: more than one element of the written-to + # tensor refers to a single memory location. Please clone() the tensor before + # performing the operation. + return torch_clone(x) + if hasattr(x, "_dynamo_dynamic_indices"): + result._dynamo_dynamic_indices = x._dynamo_dynamic_indices.copy() # type: ignore[attr-defined] + return result + + +@overload +def clone_inputs( + example_inputs: dict[str, Union[T, tuple[T, ...]]], +) -> dict[str, list[T]]: ... + + +@overload +def clone_inputs(example_inputs: Sequence[T]) -> list[T]: ... + + +def clone_inputs(example_inputs: Any) -> Any: + res: Union[dict[str, Any], list[Any]] + if type(example_inputs) is dict: + res = dict(example_inputs) + for key, value in res.items(): + if isinstance(value, tuple): + res[key] = clone_inputs(value) + else: + assert isinstance(value, torch.Tensor), type(value) + res[key] = clone_input(value) + return res + + res = list(example_inputs) + for i in range(len(res)): + if isinstance(res[i], torch.Tensor): + res[i] = clone_input(res[i]) + return res + + +def skip_frame_if_in_functorch_mode(val: torch.Tensor) -> None: + try: + val.data_ptr() # will throw for functorch tensors + except RuntimeError as e: + from .exc import format_skip_frame_message, SkipFrame + + # This will be GradTrackingTensor/BatchedTensor/etc + functorch_subclass_name = re.sub(r"\(.*", "", repr(val)) + raise SkipFrame( + format_skip_frame_message( + None, + f"torch.compile cannot be run in context: {functorch_subclass_name}", + ) + ) from e + + +@contextmanager +def preserve_rng_state() -> Generator[None, None, None]: + disable_functorch = torch._C._DisableFuncTorch + disable_current_modes = torch.utils._python_dispatch._disable_current_modes + with disable_current_modes(), disable_functorch(): + rng_state = torch.clone(torch.random.get_rng_state()) + skip_frame_if_in_functorch_mode(rng_state) + if torch.cuda.is_available(): + cuda_rng_state = torch.clone(torch.cuda.get_rng_state()) + try: + yield + finally: + with torch.utils._python_dispatch._disable_current_modes(): + torch.random.set_rng_state(rng_state) + if torch.cuda.is_available(): + torch.cuda.set_rng_state(cuda_rng_state) # type: ignore[possibly-undefined] + + +def is_jit_model( + model0: Any, +) -> TypeIs[ + Union[ + torch.jit._trace.TopLevelTracedModule, + torch.jit._script.RecursiveScriptModule, + # pyrefly: ignore [invalid-param-spec] + torch.jit.ScriptFunction[Any, Any], + torch.jit.ScriptModule, + ] +]: + return isinstance( + model0, + ( + torch.jit._trace.TopLevelTracedModule, + torch.jit._script.RecursiveScriptModule, + torch.jit.ScriptFunction, + torch.jit.ScriptModule, + ), + ) + + +def torchscript(model: Any, example_inputs: Any, verbose: bool = False) -> Any: + if is_jit_model(model): + # already done? + return model + + try: + return torch.jit.trace(model, example_inputs) + except Exception: + try: + return torch.jit.script(model) + except Exception: + if verbose: + log.exception("jit error") + else: + log.error("Both torch.jit.trace and torch.jit.script failed") + return None + + +def getfile(obj: Any) -> Optional[str]: + try: + return inspect.getfile(obj) + except (TypeError, OSError): + return None + + +def is_namedtuple(obj: Any) -> bool: + """Test if an object is a namedtuple or a torch.return_types.* quasi-namedtuple""" + return is_namedtuple_cls(type(obj)) + + +def is_namedtuple_cls(cls: Any) -> bool: + """Test if an object is a namedtuple or a (torch.return_types|torch.autograd.forward_ad).* quasi-namedtuple""" + try: + if issubclass(cls, tuple): + module = getattr(cls, "__module__", None) + if module in ("torch.return_types", "torch.autograd.forward_ad"): + return True + if isinstance(getattr(cls, "_fields", None), tuple) and callable( + getattr(cls, "_make", None) + ): + # The subclassing style namedtuple can have an extra base `typing.Generic` + bases = tuple(t for t in cls.__bases__ if t is not Generic) + if bases == (tuple,): + # This is a namedtuple type directly created by `collections.namedtuple(...)` + return True + if bases and any( + ( + # Subclass of namedtuple + is_namedtuple_cls(t) + # For subclasses of namedtuple, the __new__ method should not be customized + and cls.__new__ is t.__new__ + ) + for t in bases + ): + return True + except TypeError: + pass + return False + + +@functools.lru_cache(1) +def namedtuple_fields(cls: type) -> tuple[str, ...]: + """Get the fields of a namedtuple or a torch.return_types.* quasi-namedtuple""" + if cls is slice: + return ("start", "stop", "step") + + assert issubclass(cls, tuple) + if hasattr(cls, "_fields"): + # normal namedtuples + return cls._fields + + @dataclasses.dataclass + class Marker: + index: int + + # frustrating ones e.g. torch.return_types.max + assert cls.__module__ == "torch.return_types" + obj = cls(map(Marker, range(cls.n_fields))) # type: ignore[attr-defined] + fields: dict[str, int] = {} + for name in dir(obj): + if name[0] != "_" and isinstance(getattr(obj, name), Marker): + fields[name] = getattr(obj, name).index + assert len(fields) == cls.n_fields # type: ignore[attr-defined] + return tuple(sorted(fields, key=fields.get)) # type: ignore[arg-type] + + +def checkpoint_params(gm: torch.fx.GraphModule) -> Callable[[], None]: + with torch.no_grad(): + rng_state = torch.clone(torch.random.get_rng_state()) + if torch.cuda.is_available(): + cuda_rng_state = torch.clone(torch.cuda.get_rng_state()) + saved_state = [ + (param, param._version, torch.clone(param)) + # pyrefly: ignore [bad-argument-type] + for param in itertools.chain(gm.parameters(), gm.buffers()) + ] + + def restore() -> None: + with torch.no_grad(): + torch.random.set_rng_state(rng_state) + if torch.cuda.is_available(): + torch.cuda.set_rng_state(cuda_rng_state) + for param, version, original_value in saved_state: + if param._version != version: + param.copy_(original_value) + + return restore + + +def timed( + model: Any, example_inputs: Iterable[Any], times: int = 1 +) -> tuple[Any, float]: + if torch.cuda.is_available(): + synchronize = torch.cuda.synchronize + else: + synchronize = nothing + + synchronize() + gc.collect() + torch.manual_seed(1337) + t0 = time.perf_counter() + for _ in range(times): + result = model(*example_inputs) + synchronize() + t1 = time.perf_counter() + return result, t1 - t0 # type: ignore[possibly-undefined] + + +def check_is_cuda(gm: torch.fx.GraphModule, example_inputs: Iterable[Any]) -> bool: + return all(x.is_cuda for x in itertools.chain(example_inputs, gm.parameters(True))) + + +@lru_cache(32) +def rot_n_helper(n: int) -> Callable[..., Any]: + assert n > 1 + vars = [f"v{i}" for i in range(n)] + rotated = reversed(vars[-1:] + vars[:-1]) + fn = eval(f"lambda {','.join(vars)}: ({','.join(rotated)})") + fn.__name__ = f"rot_{n}_helper" + return fn + + +common_constant_types: set[type] = { + int, + float, + complex, + bool, + str, + bytes, + type(None), + Ellipsis.__class__, + NotImplemented.__class__, + types.CodeType, + # Commonly used immutable types from torch. + torch.device, + torch.dtype, + torch.memory_format, + torch.layout, + torch.finfo, + torch.iinfo, + torch.nn.attention.SDPBackend, + torch.cuda._CudaDeviceProperties, +} + +if has_triton_package(): + import triton + + common_constant_types.add(triton.language.dtype) + +""" + Difference between is_safe_constant and common_constant_types. + * common_constant_types: Constants would be wrapped by VariableBuilder.wrap_literal + as ConstantVariable. + * is_safe_constant: Constants can be loaded by LOAD_CONST bytecode. +""" + + +def is_safe_constant(v: Any) -> bool: + if istype(v, (tuple, frozenset)): + return all(map(is_safe_constant, v)) + return isinstance( + v, + ( + enum.Enum, + type, + torch.Size, + typing._GenericAlias, # type: ignore[attr-defined] + types.GenericAlias, + ), + ) or istype( + v, + common_constant_types | {slice}, + ) + + +@functools.cache +def common_constants() -> set[int]: + return { + # We zero-one specialize shapes, so specialize these constants + # too + 0, + 1, + } + + +def is_torch_sym(value: Any) -> TypeGuard[Union[torch.SymBool, torch.SymInt]]: + return isinstance(value, (torch.SymBool, torch.SymInt)) and not isinstance( + value.node, torch.nested._internal.nested_int.NestedIntNode + ) + + +def is_int_specialization_case(value: Any, source: Any) -> bool: + from .source import is_from_defaults + + return not TracingContext.get().force_unspec_int_unbacked_size_like and ( + # Assume integers from global variables want to be specialized + not source.guard_source.is_local() + # Assume that integers that came from NN modules want to be + # specialized (as we don't expect users to be changing the + # NN modules on the fly), unless explicitly disabled + or ( + source.guard_source.is_specialized_nn_module() + and not config.allow_unspec_int_on_nn_module + ) + or ( + source.guard_source.is_unspecialized_builtin_nn_module() + and not config.allow_unspec_int_on_nn_module + ) + or ( + source.guard_source.is_unspecialized_nn_module() + and not config.allow_unspec_int_on_nn_module + ) + or is_from_defaults(source) + # TODO: Delete this condition when rollout is done. NB: this + # condition never evaluates True in open source + or ( + not justknobs_check("pytorch/dynamo:enable_unspecialize_zero_one_plain_int") + and value in common_constants() + ) + ) + + +def specialize_symnode(arg: Any) -> Any: + from .variables import ConstantVariable, LazyVariableTracker, SymNodeVariable + + # Guard and specialize + if isinstance(arg, LazyVariableTracker) and not arg.is_realized(): + # Find if the arg would be realized as SymNodeVariable later on. If yes, + # realize it and specialize. Else return the arg. + + source = arg.original_source() + value = arg.original_value() + + is_symnode_vt = is_torch_sym(value) or ( + not config.specialize_int + and type(value) is int + and not is_int_specialization_case(value, source) + ) + + if not is_symnode_vt: + return arg + + if isinstance(arg, SymNodeVariable): + return ConstantVariable.create(arg.evaluate_expr()) + return arg + + +def guard_if_dyn(arg: Any) -> Any: + from .variables import VariableTracker + + arg = specialize_symnode(arg) + + if isinstance(arg, VariableTracker) and arg.is_python_constant(): + return arg.as_python_constant() + + return arg + + +def check_constant_args(args: Iterable[Any], kwargs: Mapping[Any, Any]) -> bool: + return all(x.is_python_constant() for x in itertools.chain(args, kwargs.values())) + + +def check_unspec_python_args(args: Iterable[Any], kwargs: Mapping[Any, Any]) -> bool: + from .variables import VariableTracker + from .variables.tensor import UnspecializedPythonVariable + + unspec_count = 0 + for x in itertools.chain(args, kwargs.values()): + if isinstance(x, UnspecializedPythonVariable): + unspec_count += 1 + elif not (isinstance(x, VariableTracker) and x.is_python_constant()): + return False + return unspec_count > 0 + + +def check_unspec_or_constant_args( + args: Iterable[Any], kwargs: Mapping[Any, Any] +) -> bool: + # A fused version of: + # return check_constant_args(args, kwargs) or check_unspec_python_args(args, kwargs) + from .variables.tensor import UnspecializedPythonVariable + + for x in itertools.chain(args, kwargs.values()): + if not (x.is_python_constant() or isinstance(x, UnspecializedPythonVariable)): + return False + return True + + +def check_numpy_ndarray_args(args: Iterable[Any], kwargs: Mapping[Any, Any]) -> bool: + from .variables.tensor import NumpyNdarrayVariable + + return any( + isinstance(x, NumpyNdarrayVariable) + for x in itertools.chain(args, kwargs.values()) + ) + + +dict_keys: type[KeysView[Any]] = type({}.keys()) +dict_values: type[ValuesView[Any]] = type({}.values()) +dict_items: type[ItemsView[Any, Any]] = type({}.items()) +odict_values: type[ValuesView[Any]] = type(OrderedDict().values()) +tuple_iterator: type[Iterator[Any]] = type(iter(())) +range_iterator: type[Iterator[Any]] = type(iter(range(0))) +tuple_iterator_len = tuple_iterator.__length_hint__ # type: ignore[attr-defined] +object_new = object.__new__ +dict_new = dict.__new__ +dict_methods = { + method + for method in itertools.chain(dict.__dict__.values(), OrderedDict.__dict__.values()) + if callable(method) +} +set_methods = {method for method in set.__dict__.values() if callable(method)} +frozenset_methods = { + method for method in frozenset.__dict__.values() if callable(method) +} + +tuple_new = tuple.__new__ +tuple_methods = {method for method in tuple.__dict__.values() if callable(method)} +list_methods = {method for method in list.__dict__.values() if callable(method)} +list_getitem = list.__getitem__ + +str_methods = {method for method in str.__dict__.values() if callable(method)} + +K = TypeVar("K") +V = TypeVar("V") + + +def builtin_dict_keys(d: dict[K, V]) -> KeysView[K]: + # Avoids overridden keys method of the dictionary + assert isinstance(d, dict) + return dict.keys(d) + + +def get_items_from_dict(obj: dict[K, V]) -> Iterable[tuple[K, Union[V, Any]]]: + # Get items without calling the user defined __getitem__ or keys method. + assert isinstance(obj, dict) + if istype(obj, (dict, OrderedDict)): + return obj.items() + elif isinstance(obj, OrderedDict): + # pyrefly: ignore [bad-argument-type] + return [(k, OrderedDict.__getitem__(obj, k)) for k in OrderedDict.keys(obj)] + else: + # pyrefly: ignore [bad-argument-type] + return [(k, dict.__getitem__(obj, k)) for k in dict.keys(obj)] + + +def nn_module_new(cls: Any) -> Any: + obj = object_new(cls) + # pyrefly: ignore [bad-argument-type] + torch.nn.Module.__init__(obj) + return obj + + +def product(it: Iterable[T]) -> int: + return functools.reduce(operator.mul, it, 1) + + +def tuple_iterator_getitem(it: Any, index: int) -> Any: + _, (obj,), start = it.__reduce__() + return obj[start + index] + + +def dataclass_fields(cls: Any) -> Any: + return torch._dynamo.disable(dataclasses.fields)(cls) + + +iter_next = next + + +def normalize_range_iter(range_iter: Any) -> tuple[int, int, int]: + _, (range_obj,), maybe_idx = range_iter.__reduce__() + # In 3.12+, `maybe_idx` could be None, and `range_obj.start` would've been + # already incremented by the current index. + # The index (maybe_idx) is the number of steps taken so far. To get the + # correct start value, one must add (maybe_idx * step) to the original + # start. See: + # https://github.com/python/cpython/blob/ea77feecbba389916af8f90b2fc77f07910a2963/Objects/rangeobject.c#L885-L899 + start = range_obj.start + (maybe_idx or 0) * range_obj.step + stop = range_obj.stop + step = range_obj.step + return (start, stop, step) + + +def to_subclass(t: Any, cls: type) -> Any: + return t.as_subclass(cls) + + +dict_getitem = dict.__getitem__ + + +@torch.fx.wrap +def dict_keys_getitem(d: dict[Any, Any], n: int) -> Any: + # Call dict(d) to prevent calling overridden __iter__/keys + dict_class = dict + if isinstance(d, OrderedDict): + dict_class = OrderedDict + # pyrefly: ignore [bad-argument-type] + return next(itertools.islice(dict_class.keys(d), n, n + 1)) + + +def set_getitem(s: set[T], n: int) -> T: + # Set ordering might not be stable + return list(s)[n] + + +def enum_repr(value: Any, local: bool) -> str: + # enum class can override __str__ method. Use __class__ and name attribute + # to extract the class name and key name. + name = value.__class__.__name__ + val = value.name + scope = "L" if local else "G" + local_name = f'{scope}["{name}"].{val}' + return local_name + + +def set_example_value(node: torch.fx.Node, example_value: Any) -> None: + # NB: example_value is a bit of a misnomer, because this is always a fake + # tensor of some sort. Furthermore, these example values serve as the + # runtime state of Dynamo tracing, which means if metadata mutation + # occurs, the example_value gets directly updated (so you can't rely on + # this to accurately reflect what the state of the value was at the time + # the program was traced). + node.meta["example_value"] = example_value + fake_mode = TracingContext.get().fake_mode + assert fake_mode is not None + shape_env = fake_mode.shape_env + if ( + symbol_to_path + := torch.fx.experimental.symbolic_shapes.compute_unbacked_bindings( + shape_env, example_value + ) + ): + node.meta["unbacked_bindings"] = symbol_to_path + + +def _get_fake_tensor(vt: VariableTracker) -> Any: + fake_tensor = vt.as_proxy().node.meta.get("example_value") + if not is_fake(fake_tensor): + from . import graph_break_hints + from .exc import unimplemented + + unimplemented( + gb_type="Cannot check Tensor object identity without its fake value", + context=str(fake_tensor), + explanation="TensorVariable is missing a fake example_value.", + hints=[*graph_break_hints.DYNAMO_BUG], + ) + return fake_tensor + + +def slice_length(s: slice, seq_len: int) -> int: + start, stop, step = s.indices(seq_len) + return max(0, (stop - start + (step - (1 if step > 0 else -1))) // step) + + +def raise_args_mismatch( + tx: InstructionTranslatorBase, + name: str, + expect: str = "", + actual: str = "", +) -> None: + from torch._dynamo.exc import raise_observed_exception + from torch._dynamo.variables import ConstantVariable + + msg_str = ( + f"wrong number of arguments or keyword arguments for {name}() call.\n" + f" Expect: {expect}\n" + f" Actual: {actual}" + ) + + raise_observed_exception( + TypeError, + tx, + args=[ConstantVariable(msg_str)], + ) + + +def iter_contains( + items: Iterable[Any], + search: Any, + tx: InstructionTranslator, + check_tensor_identity: bool = False, +) -> Any: + from .variables import BuiltinVariable, ConstantVariable + + if search.is_python_constant(): + found_const = any( + x.is_python_constant() + and x.as_python_constant() == search.as_python_constant() + for x in items + ) + return ConstantVariable.create(found_const) + + must_check_tensor_id = False + if check_tensor_identity and search.is_tensor(): + must_check_tensor_id = True + # Match of Tensor means match of FakeTensor + search = _get_fake_tensor(search) + + found: Optional[VariableTracker] = None + for x in items: + if must_check_tensor_id: + if x.is_tensor(): + if search is _get_fake_tensor(x): # Object equivalence + return ConstantVariable.create(True) + else: + check = BuiltinVariable(operator.eq).call_function(tx, [x, search], {}) + if found is None: + found = check + else: + found = BuiltinVariable(operator.or_).call_function( + tx, [check, found], {} + ) + if found is None: + found = ConstantVariable.create(False) + return found + + +def key_is_id( + k: Any, +) -> TypeIs[Union[torch.Tensor, torch.nn.Module, MethodWrapperType]]: + """Returns whether it indexes dictionaries using its id""" + return isinstance(k, (torch.Tensor, torch.nn.Module, MethodWrapperType)) + + +def key_to_id(value: Any) -> list[Any]: + return [id(k) if key_is_id(k) else k for k in value] + + +def const_repr(x: Any, *, local: Any) -> str: + from .trace_rules import is_builtin_callable + + if isinstance(x, (list, tuple)): + elems_repr = ",".join(const_repr(s, local=local) for s in x) + if isinstance(x, list): + return f"[{elems_repr}]" + else: + assert isinstance(x, tuple) + if len(x) == 1: + return f"({elems_repr},)" + else: + return f"({elems_repr})" + elif isinstance(x, enum.Enum): + # To workaround repr(Enum) returning invalid global reference before python 3.11 + # by calling enum_repr and removing quotes to render enum in guard code. + return enum_repr(x, local=local).replace("'", "") + elif is_builtin_callable(x): + return x.__name__ + elif isinstance(x, type): + + def fullname(o: Any) -> str: + klass = o.__class__ + module = klass.__module__ + if module == "builtins": + return klass.__qualname__ # avoid outputs like 'builtins.str' + return module + "." + klass.__qualname__ + + return fullname(x) + else: + return f"{x!r}" + + +def dict_keys_repr(const_keys: Any, *, local: Any) -> str: + keys_str = ",".join(const_repr(s, local=local) for s in const_keys) + return "[" + keys_str + "]" + + +GLOBAL_KEY_PREFIX = "__dict_key" + + +from torch._subclasses import UnsupportedFakeTensorException # noqa: F401 + + +def get_safe_global_name(tx: InstructionTranslatorBase, root: str, obj: Any) -> str: + # The global_mangled_class_name should be different for different + # invocations of torch.compile. Otherwise, we can run into a situation + # where multiple torch.compile invocations reuse the same global name, + # but the global's lifetime is tied to the first invocation (and + # may be deleted when the first torch.compile invocation is deleted) + # We mangle it based off of the output_graph's id. + return f"{root}_{id(obj)}_c{tx.output.compile_id}" + + +def is_in(item: T, *containers: Container[T]) -> bool: + for container in containers: + if item in container: + return True + return False + + +def get_unique_name_wrt( + prefix: str, *containers: Any, requires_suffix: bool = False +) -> str: + """ + Return a name that starts with `prefix` and is not in any of the + `containers` (e.g., map, set). + """ + if not requires_suffix and not is_in(prefix, *containers): + return prefix + + for i in itertools.count(): + candidate = f"{prefix}_{i}" + if not is_in(candidate, *containers): + return candidate + + raise AssertionError("unreachable") + + +def wrap_fake_exception(fn: Callable[[], Any]) -> Any: + try: + return fn() + except UnsupportedFakeTensorException as e: + from .exc import unimplemented + + msg = f"Encountered exception ({e.reason}) during fake tensor propagation." + log.warning(msg) + unimplemented( + gb_type="Fake tensor propagation exception", + context=str(e.reason), + explanation=msg, + hints=[], + from_exc=e, + ) + + +def deepcopy_to_fake_tensor( + obj: Any, fake_mode: torch._subclasses.fake_tensor.FakeTensorMode +) -> Any: + with torch._subclasses.fake_tensor.FakeCopyMode(fake_mode): + return wrap_fake_exception(lambda: copy.deepcopy(obj)) + + +def rmse(ref: torch.Tensor, res: torch.Tensor) -> torch.Tensor: + """ + Calculate root mean squared error + """ + return torch.sqrt(torch.mean(torch.square(ref - res))) + + +def bitwise_same(ref: Any, res: Any, equal_nan: bool = False) -> bool: + return same( + ref, + res, + tol=0.0, + equal_nan=equal_nan, + ) + + +def same( + ref: Any, + res: Any, + fp64_ref: Any = None, + cos_similarity: bool = False, + tol: float = 1e-4, + equal_nan: bool = False, + exact_dtype: bool = True, + relax_numpy_equality: bool = False, + ignore_non_fp: bool = False, + log_error: Callable[..., None] = log.error, + use_larger_multiplier_for_smaller_tensor: bool = False, + force_max_multiplier: bool = False, +) -> bool: + """Check correctness to see if ref and res match""" + if fp64_ref is None: + fp64_ref = ref + if isinstance( + ref, (list, tuple, collections.deque, torch.nn.ParameterList, torch.Size) + ): + assert isinstance(res, (list, tuple, collections.deque)), ( + f"type mismatch {type(ref)} {type(res)}" + ) + if len(ref) != len(res): + log_error("Length mismatch") + return False + return len(ref) == len(res) and all( + same( + ai, + bi, + fp64_refi, + cos_similarity, + tol, + equal_nan, + exact_dtype, + relax_numpy_equality, + ignore_non_fp, + log_error=log_error, + use_larger_multiplier_for_smaller_tensor=use_larger_multiplier_for_smaller_tensor, + force_max_multiplier=force_max_multiplier, + ) + for ai, bi, fp64_refi in zip(ref, res, fp64_ref) + ) + elif type(ref).__name__ == "QuestionAnsweringModelOutput": + # This skips checking accuracy for start_logits/end_logits. + # Tentatively, start_logits/end_logits appear to be very prone to + # inaccuracies and is somewhat subsumed by checking the loss. + return same( + ref.loss, + res.loss, + fp64_ref.loss, + cos_similarity, + tol, + equal_nan, + exact_dtype, + relax_numpy_equality, + ignore_non_fp, + log_error=log_error, + use_larger_multiplier_for_smaller_tensor=use_larger_multiplier_for_smaller_tensor, + force_max_multiplier=force_max_multiplier, + ) + elif isinstance(ref, dict): + assert isinstance(res, dict) + assert set(ref.keys()) == set(res.keys()), ( + f"keys mismatch {set(ref.keys())} == {set(res.keys())}" + ) + for k in sorted(ref.keys()): + if not ( + same( + ref[k], + res[k], + fp64_ref[k], + cos_similarity=cos_similarity, + tol=tol, + equal_nan=equal_nan, + exact_dtype=exact_dtype, + relax_numpy_equality=relax_numpy_equality, + ignore_non_fp=ignore_non_fp, + log_error=log_error, + use_larger_multiplier_for_smaller_tensor=use_larger_multiplier_for_smaller_tensor, + force_max_multiplier=force_max_multiplier, + ) + ): + log_error("Accuracy failed for key name %s", k) + return False + return True + elif isinstance(ref, set): + assert isinstance(res, set) + assert set(ref) == set(res), f"elements mismatch {set(ref)} == {set(res)}" + return True + elif isinstance(ref, (torch.Tensor, float)): + assert not isinstance(ref, torch._subclasses.FakeTensor) + assert not isinstance(res, torch._subclasses.FakeTensor) + + def to_tensor(t: Any) -> torch.Tensor: + return t if isinstance(t, torch.Tensor) else torch.tensor(t) + + ref, res, fp64_ref = (to_tensor(val) for val in (ref, res, fp64_ref)) + + if ref.is_sparse: + assert res.is_sparse + ref = ref.to_dense() + res = res.to_dense() + assert isinstance(res, torch.Tensor), f"type mismatch {type(ref)} {type(res)}" + if exact_dtype: + if ref.dtype != res.dtype: + log_error("dtype mismatch %s, %s", ref.dtype, res.dtype) + return False + if ref.dtype == torch.bool: + if ignore_non_fp: + return True + # triton stores bool as int8, so add this for more accurate checking + r = torch.allclose( + ref.to(dtype=torch.uint8), + res.to(dtype=torch.uint8), + atol=tol, + rtol=tol, + equal_nan=equal_nan, + ) + if not r: + log_error("Accuracy failed: uint8 tensor did not match") + return r + + if cos_similarity: + ref = ref.flatten().to(torch.float32) + res = res.flatten().to(torch.float32) + if torch.allclose(ref, res, atol=tol, rtol=tol, equal_nan=True): + # early exit that handles zero/nan better + # cosine_similarity(zeros(10), zeros(10), dim=0) is 0 + return True + score = torch.nn.functional.cosine_similarity(ref, res, dim=0, eps=1e-6) + if score < 0.99: + log.warning("Similarity score=%s", score.detach().cpu().item()) + return bool(score >= 0.99) + else: + if not exact_dtype: + ref = ref.to(res.dtype) + + # First try usual allclose + if torch.allclose(ref, res, atol=tol, rtol=tol, equal_nan=equal_nan): + return True + + # Check error from fp64 version + if fp64_ref.dtype == torch.float64: + # Fix a corner case that res and fp64_ref does not contains NaN and match (with loose tolerance) + # while the ref contains NaN. In this case, RMSE should not match any ways. + # But res is 'BETTER' than ref so we count it pass. + # + # This happens for Super_SloMo when loop ordering after fusion is enabled: + # https://gist.github.com/shunting314/11f235c70f7db0d52718d26f4a701cab + loose_tol = 1e-2 * 4 + if ( + not fp64_ref.isnan().any() + and not res.isnan().any() + and ref.isnan().any() + and torch.allclose( + fp64_ref.to(dtype=res.dtype), + res, + atol=loose_tol, + rtol=loose_tol, + equal_nan=equal_nan, + ) + ): + return True + ref_error = rmse(fp64_ref, ref).item() + # ref unable to produce this with stable numerics in this precision, ignore + if math.isnan(ref_error): + log.warning( + "Found nan in reference. Consider running in higher precision." + ) + + res_error = rmse(fp64_ref, res).item() + + def get_multiplier() -> float: + # In some particular cases, we expect high difference in results. + # At the moment one of this cases is inductor freezing bfloat16 convolution const folding. + # In case of it the res_error is at least one order of magnitude higher. + if force_max_multiplier: + return 10.0 + # In the case of using AMP (Automatic Mixed Precision), certain models have + # failed the benchmark's correctness check. However, the end-to-end model's + # accuracy when comparing AMP with FP32 is within a difference of less than 0.1%. + # Thus, it's possible that the correctness check failures for these models are + # false alarms. We use multiplier of 3 instead of 2 to avoid these false alarms. + multiplier = ( + 3.0 if res.dtype in (torch.float16, torch.bfloat16) else 2.0 + ) + + if use_larger_multiplier_for_smaller_tensor and ( + fp64_ref.numel() <= 10 + ): + multiplier = 10.0 + elif use_larger_multiplier_for_smaller_tensor and ( + fp64_ref.numel() <= 500 + ): + multiplier = 8.0 + elif ( + fp64_ref.numel() < 1000 + or (ref.ndim == 4 and ref.shape[-1] == ref.shape[-2] == 1) + # large tol means a benchmark has been specified as REQUIRE_HIGHER_TOLERANCE + or tol >= 2 * 1e-2 + ): + # In the presence of noise, noise might dominate our error + # metric for smaller tensors. + # Similarly, for 1x1 kernels, there seems to be high noise with amp. + multiplier = 3.0 + return multiplier + + multiplier = get_multiplier() + + passes_test = res_error <= (multiplier * ref_error + tol / 10.0) + if ( + not passes_test + and equal_nan + and math.isnan(ref_error) + and math.isnan(res_error) + # Some unit test for the accuracy minifier relies on + # returning false in this case. + and not torch._inductor.config.cpp.inject_relu_bug_TESTING_ONLY + ): + passes_test = True + if not passes_test: + log_error( + "RMSE (res-fp64): %.5f, (ref-fp64): %.5f and shape=%s. res.dtype: %s, multiplier: %f, tol: %f" + ", use_larger_multiplier_for_smaller_tensor: %d", + res_error, + ref_error, + res.size(), + res.dtype, + multiplier, + tol, + use_larger_multiplier_for_smaller_tensor, + ) + return passes_test + + if ignore_non_fp: + return True + + log_error("Accuracy failed: allclose not within tol=%s", tol) + return False + elif isinstance(ref, (str, int, type(None), bool, torch.device)): + if ignore_non_fp: + return True + r = ref == res + if not r: + log_error("Accuracy failed (%s): %s != %s", type(ref), ref, res) + return r + elif is_numpy_int_type(ref) or is_numpy_float_type(ref): + if relax_numpy_equality and not ( + is_numpy_int_type(res) or is_numpy_float_type(res) + ): + ref = ref.item() + r = (type(ref) is type(res)) and (ref == res) + if not r: + log_error("Accuracy failed (numpy): %s != %s", ref, res) + return r + elif is_numpy_ndarray(ref): + return (type(ref) is type(res)) and same( + torch.as_tensor(ref), + torch.as_tensor(res), + fp64_ref, + cos_similarity=cos_similarity, + tol=tol, + equal_nan=equal_nan, + exact_dtype=exact_dtype, + relax_numpy_equality=relax_numpy_equality, + ignore_non_fp=ignore_non_fp, + log_error=log_error, + use_larger_multiplier_for_smaller_tensor=use_larger_multiplier_for_smaller_tensor, + ) + elif type(ref).__name__ in ( + "MaskedLMOutput", + "Seq2SeqLMOutput", + "CausalLMOutputWithCrossAttentions", + "LongformerMaskedLMOutput", + "Instances", + "SquashedNormal", + "Boxes", + "Normal", + "TanhTransform", + "Foo", + "Variable", + ): + assert type(ref) is type(res) + return all( + same( + getattr(ref, key), + getattr(res, key), + getattr(fp64_ref, key), + cos_similarity=cos_similarity, + tol=tol, + equal_nan=equal_nan, + exact_dtype=exact_dtype, + relax_numpy_equality=relax_numpy_equality, + ignore_non_fp=ignore_non_fp, + log_error=log_error, + use_larger_multiplier_for_smaller_tensor=use_larger_multiplier_for_smaller_tensor, + ) + for key in ref.__dict__ + ) + else: + raise RuntimeError(f"unsupported type: {type(ref).__name__}") + + +def format_func_info(code: CodeType) -> str: + short_filename = code.co_filename.split("/")[-1] + return f"'{code.co_name}' ({short_filename}:{code.co_firstlineno})" + + +@contextlib.contextmanager +def disable_cache_limit() -> Generator[None, None, None]: + prior = config.recompile_limit + # pyrefly: ignore [bad-assignment] + config.recompile_limit = sys.maxsize + prior_acc_limit = config.accumulated_recompile_limit + # pyrefly: ignore [bad-assignment] + config.accumulated_recompile_limit = sys.maxsize + + try: + yield + finally: + config.recompile_limit = prior + config.accumulated_recompile_limit = prior_acc_limit + + +# map from transformed code back to original user code +orig_code_map = ExactWeakKeyDictionary() + +# keep a record of code_obj -> list of guard failure reasons for logging +guard_failures: collections.defaultdict[Any, list[Any]] = collections.defaultdict(list) + +# Keep a record of graph break reasons for logging +graph_break_reasons: list[torch._dynamo.output_graph.GraphCompileReason] = [] + +# keep record of compiled code, if we are in "error if recompile" +# to track code that dynamo has compiled previously +seen_code_map = ExactWeakKeyDictionary() + + +# return same dir unless user changes config between calls +@functools.cache +def _get_debug_dir(root_dir: str) -> str: + dir_name = ( + "run_" + + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f") + # use pid to avoid conflicts among ranks + + "-pid_" + + str(os.getpid()) + ) + return os.path.join(root_dir, dir_name) + + +def get_debug_dir() -> str: + debug_root = config.debug_dir_root + return _get_debug_dir(debug_root) + + +def extract_fake_example_value(node: torch.fx.Node, required: bool = True) -> Any: + if "example_value" in node.meta and is_fake(node.meta["example_value"]): + return node.meta["example_value"] + elif required: + from torch._dynamo.exc import unimplemented + + from . import graph_break_hints + + unimplemented( + gb_type="Missing FakeTensor example value", + context=str(node), + explanation=f"`FakeTensor` example value was required for {node} but not available.", + hints=[*graph_break_hints.DYNAMO_BUG], + ) + else: + return None + + +def ensure_graph_fake(e: Any, tx: InstructionTranslatorBase) -> Any: + assert maybe_get_fake_mode(e) is tx.fake_mode + return e + + +def get_fake_values_from_nodes( + tx: InstructionTranslatorBase, nodes: Any, allow_non_graph_fake: bool +) -> Any: + def visit(n: torch.fx.Node) -> Any: + if n.op == "call_function" and "example_value" not in n.meta: + # fake tensor validity is checked inside get_fake_value using + # ensure_graph_fake + return get_fake_value(n, tx, allow_non_graph_fake) + + elif n.op == "get_attr" and "example_value" not in n.meta: + assert n.target in tx.output.nn_modules + gm = tx.output.nn_modules[n.target] # type: ignore[index] + assert isinstance(gm, torch.fx.GraphModule) + return gm + + out = n.meta["example_value"] + if not allow_non_graph_fake and isinstance(out, torch.Tensor): + return ensure_graph_fake(out, tx) + return out + + return torch.fx.node.map_arg(nodes, visit) + + +def get_concrete_sizes_from_symints( + msg: str, fake_mode: Optional[FakeTensorMode] +) -> str: + """ + Replace symbolic size expressions (like 's0', 's94') in error messages + with their concrete runtime values for better readability. + + Example: "size (s94)" -> "size (s94: hint= 10)" if s94's value is 10. + """ + import re + + from sympy.core.numbers import Integer + + if fake_mode is None: + return msg + + pattern = r"\(s(\d+)\)" + assert fake_mode.shape_env is not None + shape_env = fake_mode.shape_env + var_to_val = shape_env.var_to_val + + def replace_sym(match): + sym_name = f"s{match.group(1)}" + val = next( + (v for k, v in var_to_val.items() if k.name == sym_name), + None, + ) + if isinstance(val, (int, Integer)): + return f"({sym_name}: hint = {str(val)})" + return match.group(0) + + msg = re.sub(pattern, replace_sym, msg) + return msg + + +def get_fake_value( + node: torch.fx.Node, + tx: InstructionTranslatorBase, + allow_non_graph_fake: bool = False, +) -> Any: + """ + Run the computation represented by `node` using fake tensors and return the result. + + allow_non_graph_fake: whether to allow the return result to be: + 1. non-fake or 2. fake that is not created by this instance of Dynamo. + If `True`, you must be prepared to deal with such return values, ideally + by further wrapping them as this graph's fakes. + """ + from torch.utils._sympy.value_ranges import ValueRangeError + + from .exc import ( + TorchRuntimeError, + unimplemented, + Unsupported, + UserError, + UserErrorType, + ) + + op = node.op + + # FX Node should always return the same fake value + if "example_value" in node.meta and is_fake(node.meta["example_value"]): + return node.meta["example_value"] + + args, kwargs = get_fake_values_from_nodes( + tx, (node.args, node.kwargs), allow_non_graph_fake + ) + + if ( + torch._dynamo.config.use_graph_deduplication + or torch._dynamo.config.track_nodes_for_deduplication + ): + flat_args_kwargs = get_fake_values_from_nodes( + tx, _get_flat_args(node, {}), allow_non_graph_fake + ) + id_to_initial_version = { + id(arg): arg._version for arg in flat_args_kwargs if is_fake(arg) + } + else: + flat_args_kwargs = [] + id_to_initial_version = {} + + nnmodule = None + fake_mode = tx.fake_mode + assert fake_mode is not None + if op == "call_method" and len(args) > 0 and isinstance(args[0], torch.nn.Module): + # If the first argument is nn.Module, should copy to fake mode. + args = (deepcopy_to_fake_tensor(args[0], fake_mode),) + tuple(args[1:]) + + if op == "call_module": + nnmodule = tx.output.nn_modules[node.target] # type: ignore[index] + + if is_lazy_module(nnmodule) and hasattr(nnmodule, "_initialize_hook"): + # In the case of a lazy module, we want to run + # the pre-hooks which initialize it. + # Afterwards, lazy module deletes its pre-hooks + # to avoid treating it as lazy on subsequent recompile. + nnmodule._infer_parameters(nnmodule, args) + + # no matter it's lazy module or not, we should copy to fake mode. + nnmodule = deepcopy_to_fake_tensor(nnmodule, fake_mode) + + if node.name in ["interpolate", "is_integer", "wrapped_gradient"] or any( + isinstance(a, complex) for a in args + ): + # We need to specialize symfloats for now. Eventually we should do a tensorify pass in dynamo. + args = tuple( + ( + float(arg) + if isinstance(arg, torch.SymFloat) and arg.node.hint is not None + else arg + ) + for arg in args + ) + + try: + with fake_mode, enable_python_dispatcher(): + ret_val = wrap_fake_exception( + lambda: run_node(tx.output, node, args, kwargs, nnmodule) + ) + except Unsupported: + raise + except RuntimeError as e: + cause: BaseException = e + if e.__cause__ is not None: + cause = e.__cause__ + + if isinstance( + cause, torch._subclasses.fake_tensor.DataDependentOutputException + ): + # capture_scalar_outputs only works for these ops right now + # see torch/_subclasses/fake_impls.py + if cause.func in ( + torch.ops.aten.item.default, + torch.ops.aten._local_scalar_dense.default, + ): + # does this actually get triggered? + hints = [ + "Enable tracing of data-dependent output operators with " + "`torch._dynamo.config.capture_scalar_outputs = True`", + ] + else: + hints = [ + "Consider wrapping the operator into a PyTorch-understood custom operator " + "(see https://pytorch.org/tutorials/advanced/custom_ops_landing_page.html)", + ] + unimplemented( + gb_type="Data dependent operator", + context=str(cause.func), + explanation=f"Operator `{cause.func}` has a non-Tensor output " + "whose value is dependent on the data of Tensor inputs.", + hints=hints, + ) + elif isinstance( + cause, torch._subclasses.fake_tensor.DynamicOutputShapeException + ): + if not torch._dynamo.config.capture_dynamic_output_shape_ops: + unimplemented( + gb_type="Dynamic shape operator", + context=str(cause.func), + explanation=f"Operator `{cause.func}`'s output shape depends on input Tensor data.", + hints=[ + "Enable tracing of dynamic shape operators with " + "`torch._dynamo.config.capture_dynamic_output_shape_ops = True`", + ], + ) + else: + unimplemented( + gb_type="Dynamic shape operator (no meta kernel)", + context=str(cause.func), + explanation=f"Operator `{cause.func}` does not have a meta kernel that supports dynamic output shapes", + hints=[ + "Please report an issue to PyTorch", + ], + ) + elif isinstance( + cause, torch._subclasses.fake_tensor.UnsupportedOperatorException + ): + op = cause.func # type: ignore[assignment] + import_suggestion = "" + if isinstance(op, torch._ops.OpOverload): + maybe_pystub = torch._C._dispatch_pystub( + op._schema.name, op._schema.overload_name + ) + if maybe_pystub is not None: + module, ctx = maybe_pystub + import_suggestion = ( + f"It's possible that the support was implemented in " + f"module `{module}` and you may need to `import {module}`" + f"({ctx}), otherwise " + ) + unimplemented( + gb_type="Operator does not support running with fake tensors", + context=f"unsupported operator: {cause.func}", + explanation="", + hints=[ + f"{import_suggestion}see " + "https://docs.google.com/document/d/1GgvOe7C8_NVOMLOCwDaYV1mXXyHMXY7ExoewHqooxrs/edit#heading=h.64r4npvq0w0" + " for how to fix", + ], + ) + elif isinstance( + cause, torch.fx.experimental.symbolic_shapes.GuardOnDataDependentSymNode + ): + raise UserError( # noqa: B904 + UserErrorType.CONSTRAINT_VIOLATION, + str(cause), + case_name="constrain_as_size_example", + ) + elif isinstance(cause, ValueRangeError): + raise UserError(UserErrorType.CONSTRAINT_VIOLATION, e.args[0]) from e + elif isinstance(cause, TypeError) and "argument" in str(cause): + unimplemented( + gb_type="TypeError when making fake tensor call", + context=f"TypeError {node.target}: {cause}", + explanation="", + hints=[], + ) + msg = get_concrete_sizes_from_symints(str(e), fake_mode) + raise TorchRuntimeError(msg).with_traceback(e.__traceback__) from None + + if not allow_non_graph_fake: + _ = pytree.tree_map_only( + torch.Tensor, functools.partial(ensure_graph_fake, tx=tx), ret_val + ) + + if ( + torch._dynamo.config.use_graph_deduplication + or torch._dynamo.config.track_nodes_for_deduplication + ): + tx.output.region_tracker.track_node_mutations( + node, + flat_args_kwargs, + id_to_initial_version, + ) + + return ret_val + + +_current_node = threading.local() + + +def get_current_node() -> Optional[torch.fx.Node]: + return getattr(_current_node, "value", None) + + +@contextmanager +def set_current_node(node: torch.fx.Node) -> Generator[None, None, None]: + old = get_current_node() + _current_node.value = node + try: + yield + finally: + _current_node.value = old + + +def run_node( + tracer: Any, node: torch.fx.Node, args: Any, kwargs: Any, nnmodule: Any +) -> Any: + """ + Runs a given node, with the given args and kwargs. + + Behavior is dictated by a node's op. + + run_node is useful for extracting real values out of nodes. + See get_real_value for more info on common usage. + + Note: The tracer arg is only used for 'get_attr' ops + Note: The nnmodule arg is only used for 'call_module' ops + + Nodes that are not call_function, call_method, call_module, or get_attr will + raise an AssertionError. + """ + op = node.op + + with set_current_node(node): + + def make_error_message(e: Any) -> str: + return ( + f"Dynamo failed to run FX node with fake tensors: {op} {node.target}(*{args}, **{kwargs}): got " + + repr(e) + ) + + from .exc import Unsupported + + try: + if op == "call_function": + return node.target(*args, **kwargs) # type: ignore[operator] + elif op == "call_method": + if not hasattr(args[0], node.target): # type: ignore[arg-type] + from .exc import unimplemented + + unimplemented( + gb_type="Missing attribute when running call_method node", + context="", + explanation=make_error_message("attribute not defined"), + hints=[], + ) + return getattr(args[0], node.target)(*args[1:], **kwargs) # type: ignore[arg-type] + elif op == "call_module": + assert nnmodule is not None + return nnmodule(*args, **kwargs) + elif op == "get_attr": + return tracer.output_graph.get_submodule(node.target) + elif op == "placeholder": + assert "example_value" in node.meta + return node.meta["example_value"] + + except (NotImplementedError, UnsupportedFakeTensorException) as e: + # NB: mimic how wrap_fake_exception does it + from .exc import unimplemented + + hints = [] + if isinstance(e, NotImplementedError): + hints = [ + "If the op is a PyTorch op, please file an issue to PyTorch.", + ] + + unimplemented( + gb_type="NotImplementedError/UnsupportedFakeTensorException when running FX node", + context="", + explanation=make_error_message(e), + hints=hints, + from_exc=e, + ) + except Unsupported: + raise + except Exception as e: + raise RuntimeError(make_error_message(e)).with_traceback( + e.__traceback__ + ) from e + + raise AssertionError(op) + + +def get_real_value(node: torch.fx.Node, tracer: Any) -> Any: + """ + Run the actual computation represented by `node` and return the result. + This will execute any dependent nodes in the graph as well. + """ + from .exc import TorchRuntimeError + + cache = tracer.real_value_cache + if node in cache: + return cache[node] + + op = node.op + args, kwargs = torch.fx.node.map_arg( # type: ignore[misc] + (node.args, node.kwargs), + lambda n: get_real_value(n, tracer), + ) + + if op == "placeholder" and "grapharg" in node.meta: + return node.meta["grapharg"].example + + if op == "call_module": + nn_module = tracer.output_graph.nn_modules[node.target] + if not is_lazy_module(nn_module): + nn_module = copy.deepcopy(nn_module) + else: + # In the case of a lazy module, we want to run + # the pre-hooks which initialize it + nn_module(*args, **kwargs) + else: + nn_module = None + + try: + real_value = run_node(tracer, node, args, kwargs, nn_module) + cache[node] = real_value + except RuntimeError as e: + raise TorchRuntimeError(str(e)).with_traceback(e.__traceback__) from None + return real_value + + +def assert_no_fake_params_or_buffers(gm: torch.fx.GraphModule) -> None: + from torch._subclasses.fake_tensor import FakeTensorConfig, is_fake + + def stack_or_hint(t: Any) -> str: + if FakeTensorConfig.debug: + import traceback + + return f"FAKE TENSOR CREATION TRACEBACK: \n {traceback.format_list(t._debug_trace)}" + else: + return "Enable TORCH_FAKE_TENSOR_DEBUG=1 to get creation stack traces on fake tensors." + + for name, buffer in gm.named_buffers(): + assert not is_fake(buffer), ( + f"Unexpected fake buffer {name} {stack_or_hint(buffer)}" + ) + for name, param in gm.named_parameters(): + assert not is_fake(param), ( + f"Unexpected fake param {name} {stack_or_hint(param)}" + ) + + +def fqn(obj: Any) -> str: + """ + Returns the fully qualified name of the object. + """ + return f"{obj.__module__}.{obj.__qualname__}" + + +def ifdynstaticdefault(count1: Any, count2: Any) -> Any: + if torch._dynamo.config.assume_static_by_default: + return count1 + else: + return count2 + + +def import_submodule(mod: types.ModuleType) -> None: + """ + Ensure all the files in a given submodule are imported + """ + for filename in sorted(os.listdir(os.path.dirname(cast(str, mod.__file__)))): + if filename.endswith(".py") and filename[0] != "_": + importlib.import_module(f"{mod.__name__}.{filename[:-3]}") + + +def object_has_getattribute(value: Any) -> bool: + return class_has_getattribute(type(value)) + + +def object_setattr_ignore_descriptor(obj: Any, name: str, value: Any) -> None: + # https://github.com/python/cpython/blob/3.11/Objects/object.c#L1286-L1335 + d = object.__getattribute__(obj, "__dict__") + d[name] = value + + +def class_has_getattribute(cls: type) -> bool: + try: + if isinstance( + inspect.getattr_static(cls, "__getattribute__"), + types.FunctionType, + ): + return True + except AttributeError: + pass + return False + + +def get_custom_getattr( + value: Any, ignore_nn_module_getattr: bool = False +) -> Optional[Any]: + try: + getattr_fn = inspect.getattr_static(type(value), "__getattr__") + except AttributeError: + getattr_fn = None + if ignore_nn_module_getattr and getattr_fn is torch.nn.Module.__getattr__: + # ignore this case of getattr + getattr_fn = None + return getattr_fn + + +class TensorStaticReason(enum.Enum): + PARAMETER = 2 + NOT_TENSOR = 4 + NN_MODULE_PROPERTY = 5 + + +def tensor_static_reason_to_message(reason: TensorStaticReason) -> str: + if reason == TensorStaticReason.PARAMETER: + return "mark_dynamic on parameter, parameters are always static today." + if reason == TensorStaticReason.NOT_TENSOR: + return "mark_dynamic on a non tensor, how did this happen?" + if reason == TensorStaticReason.NN_MODULE_PROPERTY: + return "tensor is static because it is nn module associated." + raise AssertionError(f"Illegal reason {reason}") + + +def tensor_always_has_static_shape( + tensor: Union[torch.Tensor, Any], + is_tensor: bool, + tensor_source: Source, +) -> tuple[bool, Optional[TensorStaticReason]]: + """ + Given a tensor, source, and is_tensor flag, determine if a shape should be static. + + Args: + tensor - the real tensor to evaluate, parameters force a static shape. + is_tensor - internal dynamo check, essentially "is_tensor": target_cls is TensorVariable, + tensors not in a TensorVariable for whatever reason are forced static. + + Returns a tuple, where the first element is the bool of whether or not this tensor should have a static shape. + The second element is a TensorStaticReason, useful for passing to tensor_static_reason_to_message if needed. + """ + from .source import is_from_unspecialized_param_buffer_source + + if ( + tensor_source.guard_source.is_specialized_nn_module() + or tensor_source.guard_source.is_unspecialized_builtin_nn_module() + ) and config.force_nn_module_property_static_shapes: + return True, TensorStaticReason.NN_MODULE_PROPERTY + + if ( + type(tensor) is torch.nn.Parameter + or is_from_unspecialized_param_buffer_source(tensor_source) + ) and config.force_parameter_static_shapes: + return True, TensorStaticReason.PARAMETER + if not is_tensor: + return True, TensorStaticReason.NOT_TENSOR + return False, None + + +def lazy_format_graph_tabular(fn_name: str, gm: torch.fx.GraphModule) -> Any: + def inner() -> str: + try: + from tabulate import tabulate # TODO: Check that this is installed + except ImportError: + return ( + "Tabulate module missing, please install tabulate to log the graph in tabular format, logging code instead:\n" + + str(lazy_format_graph_code(fn_name, gm)) + ) + + node_specs = [ + [n.op, n.name, n.target, n.args, n.kwargs] for n in gm.graph.nodes + ] + graph_str = tabulate( + node_specs, headers=["opcode", "name", "target", "args", "kwargs"] + ) + return _format_graph_code(fn_name, gm.forward.__code__.co_filename, graph_str) + + return LazyString(inner) + + +def format_bytecode( + prefix: str, name: str, filename: str, line_no: int, code: Any +) -> str: + return f"{prefix} {name} {filename} line {line_no} \n{dis.Bytecode(code).dis()}\n" + + +forward_hook_names = ["_forward_pre_hooks", "_forward_hooks"] +backward_hook_names = ["_backward_pre_hooks", "_backward_hooks"] +state_dict_hook_names = [ + "_state_dict_pre_hooks", + "_state_dict_hooks", + "_load_state_dict_pre_hooks", + "_load_state_dict_post_hooks", +] +all_hook_names = forward_hook_names + backward_hook_names + state_dict_hook_names + + +def nn_module_has_global_hooks() -> bool: + # This is limited to backward hooks for now because NNModuleVariable + # supports fwd hooks underneath. + return bool( + len(torch.nn.modules.module._global_backward_hooks) + or len(torch.nn.modules.module._global_backward_pre_hooks) + ) + + +def nn_module_get_all_hooks( + mod: torch.nn.Module, + check_forward_hooks: bool = False, + check_backward_hooks: bool = False, + check_state_dict_hooks: bool = False, +) -> list[Any]: + """ + Sometimes its useful to differentiate between types of hooks such as forward/backward/pre + hooks executed during module.__call__, and state_dict hooks which are executed separately. + """ + hook_dicts_to_check = [] + check_all_hooks = ( + not check_forward_hooks + and not check_backward_hooks + and not check_state_dict_hooks + ) + if check_forward_hooks or check_all_hooks: + hook_dicts_to_check.extend(forward_hook_names) + if check_backward_hooks or check_all_hooks: + hook_dicts_to_check.extend(backward_hook_names) + if check_state_dict_hooks: + hook_dicts_to_check.extend(state_dict_hook_names) + + all_hooks = [] + for hook_dict_name in hook_dicts_to_check: + hooks = getattr(mod, hook_dict_name, []) + for hook_name in hooks: + hook = hooks[hook_name] + + all_hooks.append(hook) + return all_hooks + + +def nnmodule_has_hooks( + mod: torch.nn.Module, + check_forward_hooks: bool = False, + check_backward_hooks: bool = False, + check_state_dict_hooks: bool = False, +) -> bool: + """ + Helper function to check if a module has any hooks attached to it. + """ + hooks = nn_module_get_all_hooks( + mod, + check_forward_hooks=check_forward_hooks, + check_backward_hooks=check_backward_hooks, + check_state_dict_hooks=check_state_dict_hooks, + ) + return bool(hooks) + + +def to_numpy_helper(value: Any) -> Any: + """Convert tensor and tnp.ndarray to numpy.ndarray.""" + if is_fake(value): + return value + if isinstance(value, tnp.ndarray): + return to_numpy_helper(value.tensor) + elif isinstance(value, torch.Tensor): + return value.numpy(force=True) + elif isinstance(value, (tuple, list)): + return type(value)(to_numpy_helper(obj) for obj in value) + else: + return value + + +def numpy_to_tensor(value: Any) -> Any: + """Convert tnp.ndarray to tensor, leave other types intact. If a list/tuple, loop through it to convert.""" + assert np is not None + if isinstance(value, np.ndarray): + return torch.as_tensor(value) + if isinstance(value, tnp.ndarray): + return value.tensor + elif isinstance(value, (tuple, list)): + return type(value)(numpy_to_tensor(obj) for obj in value) + else: + return value + + +class numpy_to_tensor_wrapper(Generic[_P, R]): + def __init__(self, f: Callable[_P, R]) -> None: + self.f = f + self.__name__ = "wrapped_" + self.f.__name__ + + def __repr__(self) -> str: + return f">" + + def __call__(self, *args: _P.args, **kwargs: _P.kwargs) -> Any: + out = self.f(*args, **kwargs) + return numpy_to_tensor(out) + + +def numpy_attr_wrapper(obj: Any, name: str) -> Any: + if isinstance(obj, tnp.ndarray): + out = getattr(obj, name) + return numpy_to_tensor(out) + elif isinstance(obj, torch.Tensor): + out = getattr(tnp.ndarray(obj), name) + return numpy_to_tensor(out) + + +class numpy_method_wrapper: + """Convert obj from torch.Tensor to tnp.ndarray and call method. Then convert result back to torch.Tensor.""" + + def __init__(self, method: str) -> None: + self.method = method + self.__name__ = "wrapped_" + self.method + + def __repr__(self) -> str: + return f">" + + def __call__(self, *args: Any, **kwargs: Any) -> Any: + obj = args[0] + if isinstance(obj, torch.Tensor): + obj = tnp.ndarray(obj) + method_callable = getattr(obj, self.method) + out = method_callable(*args[1:], **kwargs) + return numpy_to_tensor(out) + + +class numpy_operator_wrapper(Generic[_P, R]): + """Implements dunder methods for tnp.ndarray via functions from the operator library""" + + def __init__(self, op: Callable[..., Any]) -> None: + self.op = op + self.__name__ = f"wrapped_{op.__name__}" + + def __repr__(self) -> str: + return f">" + + def __call__(self, *args: _P.args, **kwargs: _P.kwargs) -> Any: + assert not kwargs + + # pyrefly: ignore [bad-assignment] + args = ( + tnp.ndarray(arg) if isinstance(arg, torch.Tensor) else arg for arg in args + ) + out = self.op(*args) + return numpy_to_tensor(out) + + +def defake(x: Any) -> Any: + if not isinstance(x, FakeTensor): + return x + size: torch._prims_common.ShapeType + stride: torch._prims_common.StrideType + if x._has_symbolic_sizes_strides: + size = [] + for s in x.size(): + if isinstance(s, torch.SymInt): + size.append(s.node.shape_env.size_hint(s.node.expr)) + else: + size.append(s) + stride = [] + for s in x.stride(): + if isinstance(s, torch.SymInt): + stride.append(s.node.shape_env.size_hint(s.node.expr)) + else: + stride.append(s) + else: + size = x.size() + stride = x.stride() + y = torch.empty_strided( + size, + stride, + dtype=x.dtype, + device=x.device, + requires_grad=x.requires_grad, + ) + y.zero_() + return y + + +def _disable_side_effect_safety_checks_for_current_subtracer( + fn: Callable[_P, R], *args: _P.args, **kwargs: _P.kwargs +) -> R: + return fn(*args, **kwargs) + + +def is_utils_checkpoint(obj: Any) -> bool: + # Lazy import to avoid circular dependencies + import torch.utils.checkpoint + + return obj is torch.utils.checkpoint.checkpoint + + +def is_invoke_subgraph(obj: Any) -> bool: + from torch._higher_order_ops.invoke_subgraph import invoke_subgraph_placeholder + + return obj is invoke_subgraph_placeholder + + +def build_invoke_subgraph_variable(**options: Any) -> Any: + from .variables.higher_order_ops import TorchHigherOrderOperatorVariable + + return TorchHigherOrderOperatorVariable.make( + torch._higher_order_ops.invoke_subgraph, + **options, + ) + + +def build_checkpoint_variable(**options: Any) -> Any: + import torch._higher_order_ops.wrap as higher_order_ops + + from .variables.higher_order_ops import TorchHigherOrderOperatorVariable + + # TODO - This is a temporary situation where we have two versions of + # checkpointing implementation. We will converge on one and remove the other. + activation_checkpoint_op: torch._ops.HigherOrderOperator = ( + higher_order_ops.tag_activation_checkpoint + ) + if torch._functorch.config.functionalize_rng_ops: + activation_checkpoint_op = higher_order_ops.wrap_activation_checkpoint + + return TorchHigherOrderOperatorVariable.make( + activation_checkpoint_op, + **options, + ) + + +def is_compile_supported(device_type: DeviceLikeType) -> Any: + from .eval_frame import is_dynamo_supported + + type = torch.device(device_type).type + compile_supported = is_dynamo_supported() + if type == "cpu": + pass + elif type in ["cuda", "xpu", "mtia"] and compile_supported: + compile_supported = has_triton() + else: + compile_supported = False + return compile_supported + + +# The following 3.11 source code functions are adapted from +# https://github.com/python/cpython/blob/v3.11.4/Lib/traceback.py +# in order to output source code corresponding to bytecode in 3.11+. +# We need our own versions since we want to support multiline expressions. +def _fix_offset(str: str, offset: int) -> int: + """ + Convert byte offset `offset` of `str` into character offset. + Byte offset is used for 3.11+ instruction column data. + Takes things like unicode characters into consideration. + + Unchanged from CPython implementation. + """ + as_utf8 = str.encode("utf-8") + return len(as_utf8[:offset].decode("utf-8", errors="replace")) + + +@dataclasses.dataclass +class _Anchors: + # inclusive + left_end_lineno: int + left_end_offset: int + right_start_lineno: int + # exclusive + right_start_offset: int + + +def _extract_anchors_from_expr(segment: str) -> Optional[_Anchors]: + """ + Given source code `segment` corresponding to a bytecode + instruction, determine: + - for binary ops, the location of the binary op + - for indexing, the location of the brackets. + `segment` is expected to be a valid Python expression + """ + assert sys.version_info >= (3, 11) + + import ast + + try: + # Without brackets, `segment` is parsed as a statement. + # We expect an expression, so wrap `segment` in + # brackets to handle multi-line expressions. + tree = ast.parse("(\n" + segment + "\n)") + except SyntaxError: + return None + + if len(tree.body) != 1: + return None + + lines = segment.split("\n") + + # get character index given byte offset + def normalize(lineno: int, offset: int) -> int: + return _fix_offset(lines[lineno], offset) + + # Gets the next valid character index in `lines`, if + # the current location is not valid. Handles empty lines. + def next_valid_char(lineno: int, col: int) -> tuple[int, int]: + while lineno < len(lines) and col >= len(lines[lineno]): + col = 0 + lineno += 1 + assert lineno < len(lines) and col < len(lines[lineno]) + return lineno, col + + # Get the next valid character index in `lines`. + def increment(lineno: int, col: int) -> tuple[int, int]: + col += 1 + lineno, col = next_valid_char(lineno, col) + assert lineno < len(lines) and col < len(lines[lineno]) + return lineno, col + + # Get the next valid character at least on the next line + def nextline(lineno: int, col: int) -> tuple[int, int]: + col = 0 + lineno += 1 + lineno, col = next_valid_char(lineno, col) + assert lineno < len(lines) and col < len(lines[lineno]) + return lineno, col + + statement = tree.body[0] + if isinstance(statement, ast.Expr): + expr = statement.value + if isinstance(expr, ast.BinOp): + # ast gives locations for BinOp subexpressions, e.g. + # ( left_expr ) + ( right_expr ) + # left^^^^^ right^^^^^ + # -2 since end_lineno is 1-indexed and because we added an extra + # bracket to `segment` when calling ast.parse + cur_lineno = cast(int, expr.left.end_lineno) - 2 + assert expr.left.end_col_offset is not None + cur_col = normalize(cur_lineno, expr.left.end_col_offset) + cur_lineno, cur_col = next_valid_char(cur_lineno, cur_col) + + # Heuristic to find the operator character. + # The original CPython implementation did not look for ), \, or #, + # leading to incorrect anchor location, e.g. + # (x) + (y) + # ~~^~~~~~~ + while (ch := lines[cur_lineno][cur_col]).isspace() or ch in ")\\#": + if ch in "\\#": + cur_lineno, cur_col = nextline(cur_lineno, cur_col) + else: + cur_lineno, cur_col = increment(cur_lineno, cur_col) + + # binary op is 1 or 2 characters long, on the same line + right_col = cur_col + 1 + if ( + right_col < len(lines[cur_lineno]) + and not (ch := lines[cur_lineno][right_col]).isspace() + and ch not in "\\#" + ): + right_col += 1 + # right_col can be invalid since it is exclusive + + return _Anchors(cur_lineno, cur_col, cur_lineno, right_col) + elif isinstance(expr, ast.Subscript): + # ast gives locations for value and slice subexpressions, e.g. + # ( value_expr ) [ slice_expr ] + # value^^^^^ slice^^^^^ + # subscript^^^^^^^^^^^^^^^^^^^^ + # find left bracket (first '[' after value) + left_lineno = cast(int, expr.value.end_lineno) - 2 + assert expr.value.end_col_offset is not None + left_col = normalize(left_lineno, expr.value.end_col_offset) + left_lineno, left_col = next_valid_char(left_lineno, left_col) + while lines[left_lineno][left_col] != "[": + left_lineno, left_col = increment(left_lineno, left_col) + # find right bracket (final character of expression) + right_lineno = cast(int, expr.end_lineno) - 2 + assert expr.end_col_offset is not None + right_col = normalize(right_lineno, expr.end_col_offset) + return _Anchors(left_lineno, left_col, right_lineno, right_col) + elif isinstance(expr, ast.Call): + # ( func_expr ) (args, kwargs) + # func^^^^^ + # call^^^^^^^^^^^^^^^^^^^^^^^^ + # find left bracket (first '(' after func) + left_lineno = cast(int, expr.func.end_lineno) - 2 + assert expr.func.end_col_offset is not None + left_col = normalize(left_lineno, expr.func.end_col_offset) + left_lineno, left_col = next_valid_char(left_lineno, left_col) + while lines[left_lineno][left_col] != "(": + left_lineno, left_col = increment(left_lineno, left_col) + # find right bracket (final character of expression) + right_lineno = cast(int, expr.end_lineno) - 2 + assert expr.end_col_offset is not None + right_col = normalize(right_lineno, expr.end_col_offset) + return _Anchors(left_lineno, left_col, right_lineno, right_col) + + return None + + +def get_instruction_source_311(code: types.CodeType, inst: dis.Instruction) -> str: + """ + Python 3.11+ only. Returns lines of source code (from code object `code`) + corresponding to `inst`'s location data, and underlines relevant code to `inst`. + + Example: CALL on `g`: + f(g( + ^^ + h(x))) + ^^^^^ + + We need our own implementation in < 3.13 since `format_frame_summary` in + Python's `traceback` module doesn't handle multi-line expressions + (and their anchor extraction code is not completely correct). + """ + if sys.version_info >= (3, 13): + # multiline traceback implemented in 3.13+ + frame_summary = traceback.FrameSummary( + code.co_filename, + inst.positions.lineno, + code.co_name, + end_lineno=inst.positions.end_lineno, + colno=inst.positions.col_offset, + end_colno=inst.positions.end_col_offset, + ) + result = traceback.format_list([frame_summary])[0] + # remove first line containing filename info + result = "\n".join(result.splitlines()[1:]) + # indent lines with original indentation + orig_lines = [ + linecache.getline(code.co_filename, lineno).rstrip() + for lineno in range(inst.positions.lineno, inst.positions.end_lineno + 1) + ] + orig_lines_dedent = textwrap.dedent("\n".join(orig_lines)).splitlines() + indent_len = len(orig_lines[0]) - len(orig_lines_dedent[0]) + indent = orig_lines[0][:indent_len] + result = textwrap.indent(textwrap.dedent(result), indent) + return result + + assert inst.positions is not None + if inst.positions.lineno is None: + return "" + # The rstrip + "\n" pattern is used throughout this function to handle + # linecache.getline errors. Error lines are treated as empty strings "", but we want + # to treat them as blank lines "\n". + first_line = linecache.getline(code.co_filename, inst.positions.lineno).rstrip() + if inst.positions.end_lineno is None: + return first_line + if inst.positions.col_offset is None or inst.positions.end_col_offset is None: + return first_line + + # character index of the start of the instruction + start_offset = _fix_offset(first_line, inst.positions.col_offset) + # character index of the end of the instruction + # compute later since end may be a different line + end_offset = None + # expression corresponding to the instruction so we can get anchors + segment = "" + # underline markers to be printed - start with `~` marker and replace with `^` later + markers = [] + + # Compute segment and initial markers + if inst.positions.end_lineno == inst.positions.lineno: + end_offset = _fix_offset(first_line, inst.positions.end_col_offset) + segment = first_line[start_offset:end_offset] + markers.append(" " * start_offset + "~" * (end_offset - start_offset)) + else: + segment = first_line[start_offset:] + "\n" + markers.append(" " * start_offset + "~" * (len(first_line) - start_offset)) + last_line = linecache.getline( + code.co_filename, inst.positions.end_lineno + ).rstrip() + end_offset = _fix_offset(last_line, inst.positions.end_col_offset) + for lineno in range(inst.positions.lineno + 1, inst.positions.end_lineno): + line = linecache.getline(code.co_filename, lineno).rstrip() + segment += line + "\n" + # don't underline leading spaces + num_spaces = len(line) - len(line.lstrip()) + markers.append(" " * num_spaces + "~" * (len(line) - num_spaces)) + segment += last_line[:end_offset] + num_spaces = len(last_line) - len(last_line.lstrip()) + markers.append(" " * num_spaces + "~" * (end_offset - num_spaces)) + + anchors: Optional[_Anchors] = None + try: + anchors = _extract_anchors_from_expr(segment) + except AssertionError: + pass + + # replace `~` markers with `^` where necessary + if anchors is None: + markers = [marker.replace("~", "^") for marker in markers] + else: + # make markers mutable + mutable_markers: list[list[str]] = [list(marker) for marker in markers] + + # anchor positions do not take start_offset into account + if anchors.left_end_lineno == 0: + anchors.left_end_offset += start_offset + if anchors.right_start_lineno == 0: + anchors.right_start_offset += start_offset + + # Turn `~`` markers between anchors to `^` + for lineno in range(len(markers)): + for col in range(len(mutable_markers[lineno])): + if lineno < anchors.left_end_lineno: + continue + if lineno == anchors.left_end_lineno and col < anchors.left_end_offset: + continue + if ( + lineno == anchors.right_start_lineno + and col >= anchors.right_start_offset + ): + continue + if lineno > anchors.right_start_lineno: + continue + if mutable_markers[lineno][col] == "~": + mutable_markers[lineno][col] = "^" + + # make markers into strings again + markers = ["".join(marker) for marker in mutable_markers] + + result = "" + for i in range(len(markers)): + result += ( + linecache.getline(code.co_filename, inst.positions.lineno + i).rstrip() + + "\n" + ) + result += markers[i] + "\n" + return result + + +def get_static_address_type(t: Any) -> Any: + if isinstance(t, torch.Tensor): + return getattr(t, "_dynamo_static_input_type", None) + + return None + + +def is_rng_state_getter_or_setter(value: Any) -> bool: + getters = ( + # The following two functions are not identical, so don't remove anyone! + torch._C.Generator.get_state, + torch.default_generator.get_state, + torch.get_rng_state, + torch.cuda.get_rng_state, + ) + setters = ( + torch._C.Generator.set_state, + torch.default_generator.set_state, + torch.set_rng_state, + torch.cuda.set_rng_state, + ) + return value in (*setters, *getters) + + +def is_tensor_base_attr_getter(value: Any) -> bool: + return ( + isinstance(value, types.MethodWrapperType) + and value.__name__ == "__get__" + and value.__self__.__objclass__ is torch._C._TensorBase # type: ignore[attr-defined] + ) + + +def is_tensor_getset_descriptor(name: str) -> bool: + try: + attr = inspect.getattr_static(torch.Tensor, name) + return type(attr) is types.GetSetDescriptorType + except AttributeError: + return False + + +def is_torch_function_object(value: Any) -> bool: + return hasattr(value, "__torch_function__") + + +def has_torch_function(vt: VariableTracker) -> bool: + # This emulates + # https://github.com/pytorch/pytorch/blob/8d81806211bc3c0ee6c2ef235017bacf1d775a85/torch/csrc/utils/disable_torch_function.cpp#L315-L323 + from torch._dynamo.variables import UserDefinedObjectVariable + from torch._dynamo.variables.torch_function import TensorWithTFOverrideVariable + + # Note on lazy vars: The value will either be realized or not throughout the course of execution + # if the value has a torch function, it will eventually be realized so we can realize it here + # if the value does not have a torch function, it may or may not be realized + # if it is realized it will be used and guards will be installed properly + # if it is not used, guards won't be installed, and it doesn't matter + # if the value has a torch function or not, so we should *not* realize it. + # NB: We technically know that if is_realized is False, LazyVariableTracker has the peek_value method + # but mypy does not unfortunately + if vt.is_realized() or ( + hasattr(vt, "peek_value") and hasattr(vt.peek_value(), "__torch_function__") + ): + func = None + if isinstance(vt, TensorWithTFOverrideVariable): + func = getattr(vt.class_type, "__torch_function__", None) + + elif isinstance(vt, UserDefinedObjectVariable): + func = getattr(vt.value, "__torch_function__", None) + + return func not in (None, torch._C._disabled_torch_function_impl) + + return False + + +# see note [Tensor Fakification and Symbol Caching] +def to_fake_tensor( + t: torch.Tensor, fake_mode: torch._subclasses.fake_tensor.FakeTensorMode +) -> Any: + symbolic_context = None + source = None + if tracing_context := torch._guards.TracingContext.try_get(): + if t in tracing_context.tensor_to_context: + symbolic_context = tracing_context.tensor_to_context[t] + source = symbolic_context.tensor_source + + return fake_mode.from_tensor( + t, static_shapes=False, symbolic_context=symbolic_context, source=source + ) + + +# NB: this works for both classes and instances +def is_frozen_dataclass(value: Any) -> bool: + return ( + not object_has_getattribute(value) + and not class_has_getattribute(value) + and is_dataclass(value) + and hasattr(value, "__dataclass_params__") + and hasattr(value.__dataclass_params__, "frozen") + and value.__dataclass_params__.frozen + ) + + +def get_first_attr(obj: Any, *attrs: str) -> Any: + """ + Return the first available attribute or throw an exception if none is present. + """ + for attr in attrs: + if hasattr(obj, attr): + return getattr(obj, attr) + + raise AssertionError(f"{obj} does not has any of the attributes: {attrs}") + + +@contextlib.contextmanager +def maybe_enable_compiled_autograd( + should_enable: bool, fullgraph: bool = True, dynamic: bool = True +) -> Generator[Any, None, None]: + if not should_enable: + yield + else: + + def compiler_fn(gm: Any) -> Any: + def inner_compiler(gm_: Any, example_inputs_: Any) -> Any: + torch._dynamo.utils.counters["compiled_autograd"]["compiles"] += 1 + return torch._inductor.compile(gm_, example_inputs_) + + return torch.compile( + gm, backend=inner_compiler, fullgraph=fullgraph, dynamic=dynamic + ) + + with torch._dynamo.compiled_autograd._enable(compiler_fn) as ctx: + yield ctx + + +def invalid_removeable_handle() -> RemovableHandle: + # need a subclass so weakref works + class Invalid(dict): # type: ignore[type-arg] + pass + + return RemovableHandle(Invalid()) + + +# Returns a "proxy" (new object with the same class and dict) for (non-GraphModule) nn.Module's. +# Attribute changes to the original object/proxy will be reflected in the other. +# This is useful for cases where we want a keep-alive reference to a module without increasing +# its reference count. +def nn_module_proxy(mod: Any) -> Any: + if not isinstance(mod, torch.nn.Module): + return mod + if isinstance(mod, torch.fx.GraphModule): + # Dynamo-generated GM's shouldn't contain user-created GM's + return mod + proxy = mod.__class__.__new__(mod.__class__) + proxy.__dict__ = mod.__dict__ + return proxy + + +class GmWrapper(torch.nn.Module): + def __init__( + self, gm: torch.fx.GraphModule, unflatten_fn: Callable[[list[Any]], Any] + ) -> None: + super().__init__() + self.gm = gm + self.unflatten_fn = unflatten_fn + + def forward(self, *args: Any) -> Any: + # pyrefly: ignore [annotation-mismatch] + args: list[Any] = list(args) + return self.gm(*self.unflatten_fn(args)) + + +def flatten_graph_inputs( + gm: torch.fx.GraphModule, inputs: Any, compile_gm: Callable[[Any, Any], Any] +) -> Callable[..., Any]: + """ + Mutate inputs so that they are flat and wrap gm such that it + accepts those inputs. This is needed for graphs that take + bumpy inputs. + """ + inputs_idx_to_clear = [ + i + for i, node in enumerate(gm.graph.nodes) + if node.op == "placeholder" and node.meta.get("steal_arg", False) + ] + + if torch._dynamo.compiled_autograd.in_compiled_autograd_region: + # fast path, avoid pytree overhead + # compiled autograd inputs are always a list of tensors, maybe followed by symints + assert inputs_idx_to_clear == [0] + assert isinstance(inputs[0], list) + boxed_inputs_count = len(inputs[0]) + + def flatten_fn(args: Any) -> Any: + return args[0] + list(args[1:]) + + def unflatten_fn(flat_args: Any) -> Any: + return (flat_args[:boxed_inputs_count], *flat_args[boxed_inputs_count:]) + + compiled_fn = compile_gm(GmWrapper(gm, unflatten_fn), flatten_fn(inputs)) + else: + # slow path, don't know inputs structure + flat_inputs, spec = pytree.tree_flatten(inputs) + unflatten_fn = functools.partial(pytree.tree_unflatten, treespec=spec) + compiled_fn = compile_gm(GmWrapper(gm, unflatten_fn), flat_inputs) + # note this doesn't check the spec, assuming it is the same + flatten_fn = pytree.arg_tree_leaves + + def wrapper(*args: Any) -> Any: + flat_args = flatten_fn(args) + + # flat_args is a new list, so we need to clear references from the old list + for i in inputs_idx_to_clear: + args[i].clear() + + # this call is boxed to avoid increasing refcount until we reach aot_module_simplified forward + return compiled_fn(flat_args) + + return wrapper + + +def get_locals_to_steal(maybe_gm: Any) -> list[Any]: + if not isinstance(maybe_gm, torch.fx.GraphModule) or not hasattr(maybe_gm, "meta"): + return [] + return maybe_gm.meta.get("locals_to_steal", []) + + +def set_locals_to_steal(gm: torch.fx.GraphModule, locals_to_steal: list[Any]) -> None: + gm.meta["locals_to_steal"] = locals_to_steal + + +class Lit: + def __init__(self, s: str) -> None: + self.s = s + + def __repr__(self) -> str: + return self.s + + +warn_once_cache: set[str] = set() + + +def warn_once(msg: str, stacklevel: int = 1) -> None: + # Dynamo causes all warnings.warn (in user code and in Dynamo code) to print all the time. + # https://github.com/pytorch/pytorch/issues/128427. + # warn_once is a workaround: if the msg has been warned on before, then we will not + # warn again. + # NB: it's totally ok to store a cache of all the strings: this is what warnings.warn does as well. + if msg in warn_once_cache: + return + warn_once_cache.add(msg) + warnings.warn(msg, stacklevel=stacklevel + 1) + + +def strip_color_from_string(text: str) -> str: + # This regular expression matches ANSI escape codes + ansi_escape = re.compile(r"\x1B[@-_][0-?]*[ -/]*[@-~]") + return ansi_escape.sub("", text) + + +@contextlib.contextmanager +def _disable_saved_tensors_hooks_during_tracing() -> Generator[None, None, None]: + # See NOTE: [Deferring tensor pack/unpack hooks until runtime] + try: + prior = torch._C._autograd._saved_tensors_hooks_set_tracing(True) + yield + finally: + torch._C._autograd._saved_tensors_hooks_set_tracing(prior) + + +def is_parameter_freezing() -> bool: + return torch._inductor.config.freezing and not torch.is_grad_enabled() + + +def get_torch_function_mode_stack() -> list[Any]: + return [ + get_torch_function_mode_stack_at(i) for i in range(_len_torch_function_stack()) + ] + + +def get_torch_function_mode_stack_at(ind: int) -> Any: + assert ind < _len_torch_function_stack() and ind >= 0 + return torch._C._get_function_stack_at(ind) + + +def set_torch_function_mode_stack(stack: list[Any]) -> None: + for _ in range(_len_torch_function_stack()): + _pop_torch_function_stack() + + for mode in stack: + _push_on_torch_function_stack(mode) + + +def clear_torch_function_mode_stack() -> None: + for _ in range(_len_torch_function_stack()): + _pop_torch_function_stack() + + +def get_current_stream(device: torch.device) -> torch.Stream: + return torch.accelerator.current_stream(device) + + +# call from C dynamo in order to inspect values in pdb +def _breakpoint_for_c_dynamo(*args: Any) -> None: + breakpoint() + + +def verify_guard_fn_signature(value: Any) -> None: + fn = value.__metadata_guard__ + sig = inspect.signature(fn) + if len(sig.parameters) != 2: + from .exc import InternalTorchDynamoError + + raise InternalTorchDynamoError( + "Tensor subclass method __metadata_guard__ must take exactly two subclass metadata arguments" + ) + if fn.__self__ != value.__class__: + from .exc import InternalTorchDynamoError + + raise InternalTorchDynamoError( + "Tensor subclass method __metadata_guard__ must be a classmethod" + ) + + +def does_not_override_dict_iter_methods(user_cls: Any) -> bool: + return ( + user_cls.items in (dict.items, OrderedDict.items) + and user_cls.values in (dict.values, OrderedDict.values) + and user_cls.keys in (dict.keys, OrderedDict.keys) + and user_cls.__iter__ in (dict.__iter__, OrderedDict.__iter__) + ) + + +# Helper functions below are to prevent TorchDynamo to prevent tracing of +# __torch_function__ calls triggered on tensor properties in the pre graph +# bytecode. +@torch._disable_dynamo +def call_size(x: Any, i: int) -> int: + return x.size(i) + + +@torch._disable_dynamo +def call_stride(x: Any, i: int) -> int: + return x.stride(i) + + +@torch._disable_dynamo +def call_storage_offset(x: Any) -> int: + return x.storage_offset() + + +# Helper function to extract relevant parts of a tensor's __dict__ to store in node meta. +# To avoid ref cycles, it's important that no tensors are present here, so leave those out. +def _extract_tensor_dict(t: torch.Tensor) -> dict[str, Any]: + KEYS_TO_COPY = [ + "_dynamo_static_input_type", + "tag", + ] + + tensor_dict = { + key: copy.copy(t.__dict__[key]) for key in KEYS_TO_COPY if key in t.__dict__ + } + + return tensor_dict + + +def build_stream(args: tuple[Any], kwargs: dict[Any, Any]) -> torch.Stream: + return torch._C.Stream(*args, **kwargs) + + +def build_event(args: tuple[Any], kwargs: dict[Any, Any]) -> torch.Event: + return torch._C.Event(*args, **kwargs) + + +class CompileTimeInstructionCounter: + _counter: int = 0 + _id: int = -1 + _depth = 0 + + @classmethod + def start(cls) -> None: + cls._depth = cls._depth + 1 + if cls._depth == 1: + cls._id = _instruction_counter.start() + + @classmethod + def end(cls) -> None: + cls._depth = cls._depth - 1 + if cls._depth == 0: + cls._counter += _instruction_counter.end(cls._id) + cls._id = -1 + + @classmethod + def clear(cls) -> None: + cls._counter = 0 + + @classmethod + def value(cls) -> int: + return cls._counter + + @classmethod + @contextmanager + def record(cls) -> Generator[None, None, None]: + try: + if config.record_compile_time_instruction_count: + cls.start() + yield + finally: + if config.record_compile_time_instruction_count: + cls.end() + + +class CompileCounterInt(int): + def __add__(self, other: Any) -> CompileCounterInt: + return CompileCounterInt(super().__add__(other)) + + +def set_feature_use(feature: str, usage: bool) -> None: + """ + Records whether we are using a feature + Generally a feature is a JK. + """ + # Note that sometimes (tests etc...) we're not in a context which we can record into + if get_metrics_context().in_progress(): + get_metrics_context().set_key_value("feature_usage", feature, usage) + + +_ddp_optimization_mode: tuple[str, ...] = ( + "ddp_optimizer", + "python_reducer", # experimental mode + "python_reducer_without_compiled_forward", + "no_optimization", +) + + +def get_optimize_ddp_mode() -> str: + optimize_ddp = config.optimize_ddp + if isinstance(optimize_ddp, bool): + mode = "ddp_optimizer" if optimize_ddp else "no_optimization" + elif isinstance(optimize_ddp, str): + mode = optimize_ddp + else: + raise ValueError( + f"Invalid dynamo config optimize_ddp type {type(optimize_ddp)=}" + ) + + assert mode in _ddp_optimization_mode, ( + f"Invalid dynamo config optimize_ddp value {mode=}" + ) + return mode + + +@contextmanager +def maybe_disable_inference_mode() -> Generator[None, None, None]: + """ + Disables torch.inference_mode for the compilation (still on at runtime). + This simplifies the compile stack where we can assume that inference_mode + will always be off. + + Since inference_mode is equivalent to no_grad + some optimizations (version + counts etc), we turn on no_grad here. The other optimizations are not + relevant to torch.compile. + """ + is_inference_mode_on = ( + config.fake_tensor_disable_inference_mode and torch.is_inference_mode_enabled() + ) + if is_inference_mode_on: + with ( + torch.inference_mode(False), + torch.no_grad(), + ): + yield + else: + yield + + +@contextmanager +def maybe_disable_inference_mode_for_fake_prop() -> Generator[None, None, None]: + """ + Turns off tracking of inference_mode for fake tensor propagation. With this + context manager, when a real tensor is converted to fake tensor, the fake + tensor looses its inference-ness. + """ + if config.fake_tensor_disable_inference_mode: + with torch._subclasses.meta_utils.disable_inference_mode_for_fake_prop(): + yield + else: + yield + + +def is_node_meta_valid(node: Optional[torch.fx.Node]) -> bool: + return node is None or "example_value" in node.meta or "val" in node.meta + + +# If True, enforce fullgraph=True - raise errors on graph break +_error_on_graph_break = False + + +def _get_error_on_graph_break() -> bool: + return _error_on_graph_break + + +def _set_error_on_graph_break(value: bool) -> None: + global _error_on_graph_break + _error_on_graph_break = value + + +@torch._disable_dynamo +def record_pregraph_bytecode_enter() -> AbstractContextManager[None]: + cm: AbstractContextManager[None] = ( + torch._C._profiler._RecordFunctionFast("Pregraph bytecode") + if torch.autograd.profiler._is_profiler_enabled + else contextlib.nullcontext() + ) + cm.__enter__() + return cm + + +@torch._disable_dynamo +def record_pregraph_bytecode_exit(cm: AbstractContextManager[None]) -> None: + cm.__exit__(None, None, None) + + +# Returns a set of code objects present traced in the current TracingContext, or None +# if there is no current TracingContext. +def get_traced_code() -> Optional[list[CodeType]]: + from torch._guards import TracingContext + + return TracingContext.get_traced_code() + + +def raise_on_overridden_hash(obj: Any, vt: VariableTracker) -> None: + from . import graph_break_hints + from .exc import unimplemented + + is_overridden = type(obj).__dict__.get("__hash__", False) + + if is_overridden: + unimplemented( + gb_type="User-defined object with overridden __hash__", + context=f"hashing object of type={type(obj)} and variable tracker {vt}", + explanation=f"Found a user-defined object {vt} with overridden __hash__ when attempting to hash it", + hints=[ + "Dynamo does not support hashing user-defined objects with overridden __hash__", + *graph_break_hints.SUPPORTABLE, + ], + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..7a0a87dab19dca380b6b9e6de94a267dedca86fd --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/__init__.py @@ -0,0 +1,188 @@ +# mypy: allow-untyped-defs +import copy +import dataclasses +import functools +import io +import json +import logging +import os +import re +import sys +import types +import warnings +import weakref +import zipfile +from collections import OrderedDict +from contextlib import contextmanager +from functools import lru_cache + +from typing import Any, Optional, TYPE_CHECKING, Union +from collections.abc import Callable +from unittest.mock import patch + +import torch +import torch.fx +import torch.utils._pytree as pytree + +from torch._dispatch.python import enable_python_dispatcher +from torch._guards import compile_context +from torch._utils_internal import log_export_usage +from torch.export._tree_utils import reorder_kwargs +from torch.export.graph_signature import ( + ArgumentSpec, + ConstantArgument, + ExportGraphSignature, + InputKind, + InputSpec, + OutputKind, + OutputSpec, + SymIntArgument, + SymBoolArgument, + SymFloatArgument, + TensorArgument, +) +from torch.fx import traceback as fx_traceback +from torch.fx._compatibility import compatibility +from torch.fx.experimental.proxy_tensor import make_fx +from torch.fx.graph import _PyTreeCodeGen, _PyTreeInfo + +from .wrappers import _wrap_submodules +from .utils import _materialize_cpp_cia_ops +from . import config + +if TYPE_CHECKING: + from torch._C._aoti import AOTIModelContainerRunner + +log = logging.getLogger(__name__) + +@dataclasses.dataclass +class ExportDynamoConfig: + """ + Manage Export-specific configurations of Dynamo. + """ + allow_rnn: bool = True + + +# We only want to print this once to avoid flooding logs in workflows where aot_compile_warning +# is called multiple times. +@lru_cache +def aot_compile_warning(): + + log.warning("+============================+") + log.warning("| !!! WARNING !!! |") + log.warning("+============================+") + log.warning( + "torch._export.aot_compile()/torch._export.aot_load() is being deprecated, please switch to " + "directly calling torch._inductor.aoti_compile_and_package(torch.export.export())/" + "torch._inductor.aoti_load_package() instead.") + + +def aot_compile( + f: Callable, + args: tuple[Any, ...], + kwargs: Optional[dict[str, Any]] = None, + *, + dynamic_shapes: Optional[dict[str, Any]] = None, + options: Optional[dict[str, Any]] = None, + remove_runtime_assertions: bool = False, + disable_constraint_solver: bool = False, + same_signature: bool = True, +) -> Union[list[Any], str]: + """ + Note: this function is not stable yet + + Traces either an nn.Module's forward function or just a callable with PyTorch + operations inside, generates executable cpp code from the program, and returns + the path to the generated shared library + + Args: + f: the `nn.Module` or callable to trace. + + args: example positional inputs. + + kwargs: optional example keyword inputs. + + dynamic_shapes: Should either be: + 1) a dict from argument names of ``f`` to their dynamic shape specifications, + 2) a tuple that specifies dynamic shape specifications for each input in original order. + If you are specifying dynamism on keyword args, you will need to pass them in the order that + is defined in the original function signature. + + The dynamic shape of a tensor argument can be specified as either + (1) a dict from dynamic dimension indices to :func:`Dim` types, where it is + not required to include static dimension indices in this dict, but when they are, + they should be mapped to None; or (2) a tuple / list of :func:`Dim` types or None, + where the :func:`Dim` types correspond to dynamic dimensions, and static dimensions + are denoted by None. Arguments that are dicts or tuples / lists of tensors are + recursively specified by using mappings or sequences of contained specifications. + + options: A dictionary of options to control inductor + + disable_constraint_solver: Whether the dim constraint solver must be disabled. + + Returns: + Path to the generated shared library + """ + from torch.export._trace import _export_to_torch_ir + from torch._inductor.decomposition import select_decomp_table + from torch._inductor import config as inductor_config + + aot_compile_warning() + + if inductor_config.is_predispatch: + gm = torch.export._trace._export(f, args, kwargs, dynamic_shapes, pre_dispatch=True).module() + else: + # We want to export to Torch IR here to utilize the pre_grad passes in + # inductor, which run on Torch IR. + with torch._export.config.patch(use_new_tracer_experimental=True): + gm = _export_to_torch_ir( + f, + args, + kwargs, + dynamic_shapes, + disable_constraint_solver=disable_constraint_solver, + same_signature=same_signature, + # Disabling this flag, because instead we can rely on the mapping + # dynamo_flat_name_to_original_fqn which is coming from Dynamo. + restore_fqn=False, + ) + + with torch.no_grad(): + so_path = torch._inductor.aot_compile(gm, args, kwargs, options=options) # type: ignore[arg-type] + + assert isinstance(so_path, (str, list)) + return so_path + +def aot_load(so_path: str, device: str) -> Callable: + """ + Loads a shared library generated by aot_compile and returns a callable + + Args: + so_path: Path to the shared library + + Returns: + A callable + """ + aot_compile_warning() + + if device == "cpu": + runner: AOTIModelContainerRunner = torch._C._aoti.AOTIModelContainerRunnerCpu(so_path, 1) + elif device == "cuda" or device.startswith("cuda:"): + runner = torch._C._aoti.AOTIModelContainerRunnerCuda(so_path, 1, device) + elif device == "xpu" or device.startswith("xpu:"): + runner = torch._C._aoti.AOTIModelContainerRunnerXpu(so_path, 1, device) + elif device == "mps" or device.startswith("mps:"): + runner = torch._C._aoti.AOTIModelContainerRunnerMps(so_path, 1) + else: + raise RuntimeError("Unsupported device " + device) + + def optimized(*args, **kwargs): + call_spec = runner.get_call_spec() + in_spec = pytree.treespec_loads(call_spec[0]) + out_spec = pytree.treespec_loads(call_spec[1]) + flat_inputs = pytree.tree_flatten((args, reorder_kwargs(kwargs, in_spec)))[0] + flat_inputs = [x for x in flat_inputs if isinstance(x, torch.Tensor)] + flat_outputs = runner.run(flat_inputs) + return pytree.tree_unflatten(flat_outputs, out_spec) + + return optimized diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/converter.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/converter.py new file mode 100644 index 0000000000000000000000000000000000000000..58de4fd20c953318eb683fe103b3eac3637a077e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/converter.py @@ -0,0 +1,1613 @@ +# mypy: allow-untyped-defs +import builtins +import logging +import operator +import typing +import warnings +from collections.abc import Callable, Sequence +from contextlib import contextmanager +from typing import Any, Optional, Union + +import torch +import torch.export._trace +from torch import _C +from torch._export.passes.replace_quantized_ops_with_standard_ops_pass import ( + replace_quantized_ops_with_standard_ops, +) +from torch.export.dynamic_shapes import _tree_map_with_path, Dim +from torch.export.exported_program import ExportedProgram +from torch.export.graph_signature import ( + ConstantArgument, + CustomObjArgument, + InputKind, + InputSpec, + OutputKind, + OutputSpec, + TensorArgument, +) +from torch.fx import subgraph_rewriter + + +log = logging.getLogger(__name__) + + +def _get_param_count_list(method_graph, args_params): + param_count_list = [] + for input_, arg_params_ in zip(method_graph.inputs(), args_params): + if "PackedParams" in str(input_.type()): + in_vars, _ = torch.jit._flatten(arg_params_) + param_count_list.append(len(in_vars)) + else: + param_count_list.append(arg_params_ is not None) + + return param_count_list + + +def _trace_and_get_graph_from_model(model, args): + # A basic sanity check: make sure the state_dict keys are the same + # before and after running the model. Fail fast! + orig_state_dict_keys = torch.jit._unique_state_dict(model).keys() + + # Disable Autocast cache because it replaces kernel's weight and bias + # by (undesired) constants. + # No perf impact for when there are reused weights since https://github.com/pytorch/pytorch/pull/85665 + prev_autocast_cache_enabled = torch.is_autocast_cache_enabled() + torch.set_autocast_cache_enabled(False) + trace_graph, torch_out, _inputs_states = torch.jit._get_trace_graph( + model, + args, + strict=False, + _force_outplace=False, + _return_inputs_states=True, + ) + torch.set_autocast_cache_enabled(prev_autocast_cache_enabled) + + if orig_state_dict_keys != torch.jit._unique_state_dict(model).keys(): + raise RuntimeError( + "state_dict changed after running the tracer; " + "something weird is happening in your model!" + ) + + return trace_graph, torch_out + + +def _create_jit_graph( + model: Union[torch.nn.Module, torch.jit.ScriptFunction], args: Sequence[Any] +) -> tuple[torch.Graph, list["_C.IValue"], Any, Optional[torch.ScriptModule]]: + if isinstance(model, (torch.jit.ScriptFunction, torch.jit.ScriptModule)): + flattened_args = tuple(torch.jit._flatten(tuple(args))[0]) + torch_out = None + + if isinstance(model, torch.jit.ScriptModule): + try: + graph = model.forward.graph # type: ignore[attr-defined] + except AttributeError as e: + raise RuntimeError("'forward' method must be a script method") from e + _C._jit_pass_onnx_function_substitution(graph) + freezed_module = _C._freeze_module( + typing.cast(_C.ScriptModule, model._c), preserveParameters=True + ) + module, params = _C._jit_onnx_list_model_parameters(freezed_module) + method_graph = module._get_method("forward").graph + args_params = tuple(args) + tuple(params) + param_count_list = _get_param_count_list(method_graph, args_params) + in_vars, _ = torch.jit._flatten(args_params) + graph = _C._propagate_and_assign_input_shapes( + method_graph, tuple(in_vars), param_count_list, False, False + ) + return graph, params, torch_out, module + + # torch.jit.ScriptFunction + params = [] + graph = model.graph + _C._jit_pass_onnx_function_substitution(graph) + param_count_list = _get_param_count_list(graph, args) + graph = _C._propagate_and_assign_input_shapes( + graph, flattened_args, param_count_list, False, False + ) + return graph, params, torch_out, None + + graph, torch_out = _trace_and_get_graph_from_model(model, args) + _C._jit_pass_onnx_lint(graph) + state_dict = torch.jit._unique_state_dict(model) + params = list(state_dict.values()) + graph_inputs = list(graph.inputs()) + user_input_num = len(graph_inputs) - len(state_dict) + param_names = list(state_dict.keys()) + for i, inp in enumerate(graph_inputs): + if i >= user_input_num: + inp.setDebugName(param_names[i - user_input_num]) + _C._jit_pass_onnx_function_substitution(graph) + return graph, params, torch_out, None + + +def list_add(a, b): + return a + b + + +def list_append(container, element): + return container + [element] + + +def execute_subgraph_from_prim_loop( + subgraph, iter_idx, len_loop_local_arguments, *args, **kwargs +): + """ + subgraph: GraphModule from sub-block. + iter_idx: The index of interaction. + len_loop_local_arguments: The number of loop local arguments in args. + """ + + # Loop local variables. TS graph create those as inputs because their values + # are updated inside the loop. + loop_local_args = args[:len_loop_local_arguments] + # Global variables that are not passed in as inputs to the loop sub-blocks + # but are directly used. Most of time, their values are not updated, but + # the only exception is when there are some operations that perform inplace + # updates. + global_args = args[len_loop_local_arguments:] + return subgraph(*global_args, iter_idx, *loop_local_args, **kwargs) + + +def inplace_optimize_sym_size_div(gm: torch.fx.GraphModule): + def pattern(im, dim, scale): + sym_size_int = torch.ops.aten.sym_size.int(im, dim) + scalar_tensor = torch.ops.aten.scalar_tensor(sym_size_int) + div_scalar_mode = torch.ops.aten.div.Scalar_mode( + scalar_tensor, scale, rounding_mode="trunc" + ) + int_tensor = torch.ops.aten.Int.Tensor(div_scalar_mode) + return int_tensor + + def replacement(im, dim, scale): + sym_size_int = torch.ops.aten.sym_size.int(im, dim) + return sym_size_int // scale + + subgraph_rewriter.replace_pattern(gm, pattern, replacement) + + +def is_valid_for_codegen(name): + if len(name) == 0: + raise RuntimeError("Empty argument name for codegen") + if name[0].isdigit(): + return False + return True + + +def normalize_name(name: str, prefix: str = "rename") -> str: + name = name.replace(".", "_") + if is_valid_for_codegen(name): + return name + return f"{prefix}_{name}" + + +def ir_name_to_func_name(name: str) -> str: + """prim::If -> convert_prim_If""" + name_list = name.split("::") + return "convert_" + "_".join(name_list) + + +def get_node_as_placeholder_or_get_attr(fx_graph, name, is_top_level_graph): + if is_top_level_graph: + return fx_graph.get_attr(name) + return fx_graph.placeholder(name) + + +_TORCH_DTYPE_TO_ENUM = { + torch.uint8: 0, + torch.int8: 1, + torch.int16: 2, + torch.int32: 3, + torch.int64: 4, + torch.float16: 5, + torch.float32: 6, + torch.float64: 7, + torch.complex32: 8, + torch.complex64: 9, + torch.complex128: 10, + torch.bool: 11, + torch.qint8: 12, + torch.quint8: 13, + torch.bfloat16: 15, +} + +_TORCH_ENUM_TO_DTYPE = {value: key for key, value in _TORCH_DTYPE_TO_ENUM.items()} + + +def get_dtype_as_int(tensor): + """ + prim::dtype has the signature "Tensor a) -> int", where it gets the dtype of + the tensor and returns the integer corresponding to this dtype based on the + enum in ScalarType.h + """ + dtype = tensor.dtype + if dtype not in _TORCH_DTYPE_TO_ENUM: + raise RuntimeError(f"Unsupported dtype {dtype}") + return _TORCH_DTYPE_TO_ENUM[dtype] + + +# Those operators will be automatically populated to a instance method +# of TS2FXGraphConverter with name convert__(). +# Please check __init__ for method population implementations. +kind_to_standard_operators: dict[str, Callable[..., Any]] = { + "prim::max": builtins.max, + "prim::min": builtins.min, + "prim::TupleIndex": operator.getitem, + "aten::__is__": operator.is_, + "aten::__isnot__": operator.is_not, + "aten::__not__": operator.not_, + "aten::__contains__": operator.contains, + "prim::dtype": get_dtype_as_int, + "aten::len": len, + # Mapping from specialized op to its symbolic counterpart. + # They currently do not have any other overrides. + "aten::numel": torch.ops.aten.sym_numel, + "aten::size": torch.ops.aten.sym_size, + "aten::storage_offset": torch.ops.aten.sym_storage_offset, + "aten::stride": torch.ops.aten.sym_stride, +} + + +def get_ir_value_parent_name_and_attr_name(node): + irv_parent_name, irv_name = node.input().debugName(), node.output().debugName() + attr_name = node.s("name") + return irv_name, irv_parent_name, attr_name + + +def construct_fqn(ir, ref_map, name_map): + name_list = [] + while ir in ref_map: + name_list.append(name_map[ir]) + ir = ref_map[ir] + return ".".join(reversed(name_list)) + + +def get_block_to_lifted_attrs( + graph: torch._C.Graph, +) -> tuple[dict[torch._C.Block, set[str]], dict[str, str]]: + """ + Perform two passes to get a mapping of blocks to a set of FQNs of its lifted attributes. + When a graph has control flow, the graph will be divided into multiple blocks. We want to convert + each block to a graph which will be passed into torch.cond. A restriction for torch.cond is that model + parameters/buffers are expected to be lifted as inputs to the subgraphs. Before converting the model, + we will run this pass which will: + 1. Figure out which params/buffers are used within blocks through tracing the GetAttr calls. + 2. Process the graph bottom up to find the lifted attributes of each block by taking the union + of the attributes used in the current block, and the lifted attributes of all its child blocks. + + Returns: + A mapping of blocks to a set of FQNs of its lifted attributes, and a + mapping of node names to the FQNs of its lifted attributes. + """ + + # A map from a block to its expected to be lifted arguments. + blocks_to_lifted_attrs: dict[torch._C.Block, set[str]] = {} + + # Reference map stores the input (i.e., src) and output (i.e., dest) IR of a + # GetAttr node. By traversing this reference map, we can figure out the + # full IR aliasing pass and figure out the FQN of an attribute. + # E.g., %2 = GetAttr(linear)[%1] --> node_to_parent_map["%2"] = "%1" + node_to_parent_map: dict[str, str] = {} + + # Used for reconstructing the FQN of an attribute based on the reference map. + # In nutshell, for each GetAttr call, GetAttr(input IR, attribute name) -> output IR + # This name map stores which attribute name is called for a src IR --> dest IR action. + # E.g., %2 = GetAttr(linear)[%1] --> node_to_attr_name["%2"] = "linear" + node_to_attr_name: dict[str, str] = {} + + def _dfs_get_attr_dependency(entry): + """ + First DFS path to construct reference map and name map. + """ + for node in entry.nodes(): + if node.kind() == "prim::GetAttr": + ( + irv_name, + irv_parent_name, + attr_name, + ) = get_ir_value_parent_name_and_attr_name(node) + node_to_parent_map[irv_name] = irv_parent_name + node_to_attr_name[irv_name] = attr_name + for block in node.blocks(): + _dfs_get_attr_dependency(block) + + def _map_blocks_to_lifted_attrs(entry): + """ + Walk the graph in a bottom-up fashion to build the expected to be + lifted arguments for each block. + """ + arguments: set[str] = set() + for node in entry.nodes(): + for block in node.blocks(): + # Recursively build. + arguments = arguments.union(_map_blocks_to_lifted_attrs(block)) + if node.kind() == "prim::GetAttr": + irv_name = node.output().debugName() + # Skip for intermediate GetAttr, which will anyway not result a FQN. + # E.g., node_to_parent_name: {"%3": "%2", "%2": "%1"} + # node_to_attr_name: {"%3": "weight", "%2": "linear", "%1": "self"} + # There is only one FQN %3-->%2-->%1: self.linear.weight + # %2-->%1 is not a FQN: self.linear + if irv_name not in set(node_to_parent_map.values()): + arguments.add( + construct_fqn(irv_name, node_to_parent_map, node_to_attr_name) + ) + if not isinstance(entry, torch._C.Graph): # Skip the top level. + blocks_to_lifted_attrs[entry] = arguments + return arguments + + _dfs_get_attr_dependency(graph) + _map_blocks_to_lifted_attrs(graph) + + return blocks_to_lifted_attrs, node_to_attr_name + + +def get_attribute_fqn_from_ts_node( + name_to_attribute_fqn: dict[str, str], node: torch._C.Node +) -> str: + def get_attr(name: str): + if name in name_to_attribute_fqn: + return name_to_attribute_fqn[name] + else: + raise ValueError(f"Attribute {name} not found") + + if node.kind() == "prim::SetAttr": + input_name = next(node.inputs()).debugName() + elif node.kind() == "prim::GetAttr": + input_name = node.input().debugName() + else: + raise RuntimeError( + f"Unexpected node kind when getting attribute fqn. node: {node} " + ) + + attr_name = node.s("name") + root_attr_name = get_attr(input_name) + attr_fqn = f"{root_attr_name}.{attr_name}" if root_attr_name else attr_name + + return attr_fqn + + +def get_op_overload(node: torch._C.Node): + schema_str = node.schema() + assert schema_str != "(no schema)", f"got empty schema for {node}" + schema: torch._C.FunctionSchema = torch._C.parse_schema(schema_str) + ns, op_name = str(schema.name).split("::") + override = schema.overload_name + + try: + op_overload_mod = getattr(torch.ops, ns) + op_overload_packet = getattr(op_overload_mod, op_name) + if override: + op_overload = getattr(op_overload_packet, override) + else: + op_overload = op_overload_packet.default + except Exception as e: + raise RuntimeError( + f"Unable to find operator {node.kind()} with schema {node.schema()}" + ) from e + + return op_overload + + +class TS2FXGraphConverter: + def __init__( + self, + ts_graph: Union[torch._C.Graph, torch._C.Block], + name_to_param: dict[str, torch.Tensor], + name_to_buffer: dict[str, torch.Tensor], + blocks_to_lifted_attrs: dict[torch._C.Block, set[str]], + name_to_non_tensor_attribute: dict[str, Any], + name_to_constant: dict[str, Any], + name_to_attribute_fqn: dict[str, str], + ): + self.ts_graph = ts_graph + # Mapping of parameter FQN to actual parameter value + self.name_to_param = name_to_param + # Mapping of buffer FQN to actual buffer value + self.name_to_buffer = name_to_buffer + + self.fx_graph: torch.fx.Graph = torch.fx.Graph() + self.input_specs: list[InputSpec] = [] + self.output_specs: list[OutputSpec] = [] + + # Mapping of TS node name to converted FX node + self.name_to_node: dict[ + str, Union[torch.fx.Node, list[torch.fx.Node], dict[Any, torch.fx.Node]] + ] = {} + # Mapping of TS node name to constant value (int, str, TorchBind obj, + # tensor constants ...) + self.name_to_constant: dict[str, Any] = name_to_constant + + # Mapping from torchscript node output name to attribute fully qualified name + self.name_to_attribute_fqn: dict[str, str] = name_to_attribute_fqn + + # Mapping from fully qualified name to real values or a fx graph node + # During convert, this represents the current value of a non-tensor attribute + # One use case is: + # def forward(self, x): + # c1 = self.count + # self.count += 1 + # c2 = self.count + # return x + c1 + c2 + self.name_to_non_tensor_attribute_node: dict[str, Any] = {} + + # Mapping from fully qualified name to initial real values inputs + # We separate it from self.name_to_non_tensor_attribute_node since + # we need initial real value input when we construct fx.GraphModule + self.name_to_non_tensor_attribute: dict[str, Any] = name_to_non_tensor_attribute + + self.subgraphs: dict[str, torch.fx.GraphModule] = {} + + # Mapping of block to list of attributes that need to be lifted for each + # block + self.blocks_to_lifted_attrs = blocks_to_lifted_attrs + + # Populate methods for the standard operators. + for k in kind_to_standard_operators: + handler_func_name = ir_name_to_func_name(k) + # Create an indirect function call: + # convert__ --> lambda node: _convert_standard_operator(node) + setattr( + self, + handler_func_name, + lambda node: self._convert_standard_operators(node), + ) + + # This stores a list of return results that do not appear in the original TS + # graph's outputs. The reason we maintain this is because some operations in the sub-block + # might have inplace updates to the variable defined in the parent fx graph. After + # the execution of that sub-block, the variable defined in the parent fx graph also + # needs to be updated. + self.name_update_from_subblock_to_parent: set[str] = set() + + def _is_get_attr_node(self, fqn): + return ( + fqn in self.name_to_buffer + or fqn in self.name_to_param + or ( + fqn in self.name_to_constant + and isinstance(self.name_to_constant[fqn], torch.ScriptObject) + ) + ) + + def _convert_block_to_subgraph(self, node: torch._C.Node, arguments: list[str]): + subgraph_nodes, subgraph_converters = [], [] + for block in node.blocks(): + subgraph_converter = TS2FXGraphConverter( + block, + self.name_to_param, + self.name_to_buffer, + self.blocks_to_lifted_attrs, + {}, + self.name_to_constant, + self.name_to_attribute_fqn, + ) + + for block_arg in arguments: + normalized_block_arg_name = normalize_name(block_arg) + placeholder_node = subgraph_converter.fx_graph.placeholder( + normalized_block_arg_name + ) + subgraph_converter.name_to_node[block_arg] = placeholder_node + + subgraph = subgraph_converter.convert() + subgraph_name = self.add_subgraph(subgraph) + subgraph_nodes.append(self.fx_graph.get_attr(subgraph_name)) + subgraph_converters.append(subgraph_converter) + return subgraph_nodes, subgraph_converters + + def _identify_inputs_as_arguments(self, entry): + """ + Identify inputs from the innermost sub-block. This is needed + for nested sub-blocks when the input is hidden in the nested sub-block. + E.g., example IR of input is hidden in the nested sub-block. + Graph[x.1] + %1 = ... + Block[] + Block[x.1] + %2 = x.1 ... + """ + arguments: set[str] = set() + for block in entry.blocks(): + for block_node in block.nodes(): + for block_node_in in block_node.inputs(): + if ( + block_node_in.debugName() in self.name_to_node + and block_node_in.debugName() not in self.name_to_attribute_fqn + ): + arguments.add(block_node_in.debugName()) + arguments = arguments.union( + self._identify_inputs_as_arguments(block_node) + ) + return arguments + + def is_top_level_graph(self): + return isinstance(self.ts_graph, torch._C.Graph) + + def add_subgraph(self, subgraph) -> str: + name = f"subgraph_{len(self.subgraphs)}" + self.subgraphs[name] = subgraph + return name + + def get_args_kwargs(self, node: torch._C.Node, schema): + args = [] + kwargs = {} + for input, schema_arg in zip(node.inputs(), schema.arguments): + if schema_arg.kwarg_only: + kwargs[schema_arg.name] = self.get_fx_value_by_ir_value(input) + else: + args.append(self.get_fx_value_by_ir_value(input)) + + return tuple(args), kwargs + + def get_fx_value_by_ir_value(self, value: torch._C.Value): + value_name = value.debugName() + + if value_name in self.name_to_node: + input_node = self.name_to_node[value_name] + return input_node + elif value_name in self.name_to_constant: + if isinstance(self.name_to_constant[value_name], torch.ScriptObject): + return self.fx_graph.get_attr(value_name) + return self.name_to_constant[value_name] + elif value_name in self.name_to_attribute_fqn: + return self.get_fx_value_by_fqn(self.name_to_attribute_fqn[value_name]) + else: + raise ValueError(f"Input {value_name} not found") + + def get_fx_value_by_fqn(self, name): + if name in self.name_to_node: + fx_node = self.name_to_node[name] + elif name in self.name_to_constant: + fx_node = self.name_to_constant[name] + elif name in self.name_to_non_tensor_attribute_node: + fx_node = self.name_to_non_tensor_attribute_node[name] + elif name in self.name_to_non_tensor_attribute: + fx_node = self.name_to_non_tensor_attribute[name] + else: + raise ValueError(f"Attribute {name} not found") + return fx_node + + def convert(self) -> torch.fx.GraphModule: + self.convert_graph_inputs() + + for node in self.ts_graph.nodes(): + self.convert_node(node) + + self.convert_graph_outputs() + + # Pass parameter and buffer to the root for lookup. + gm = torch.fx.GraphModule( + { + **self.subgraphs, + **self.name_to_param, + **self.name_to_buffer, + **self.name_to_non_tensor_attribute, + **self.name_to_constant, + }, + self.fx_graph, + ) + + inplace_optimize_sym_size_div(gm) + + gm.graph.lint() + + return gm + + def convert_graph_inputs(self): + for graph_input in self.ts_graph.inputs(): + name = graph_input.debugName() + + if name in self.name_to_param: + normalized_name = normalize_name(name) + self.input_specs.append( + InputSpec( + InputKind.PARAMETER, + arg=TensorArgument(name=normalized_name), + target=name, + ) + ) + fx_node = get_node_as_placeholder_or_get_attr( + self.fx_graph, name, self.is_top_level_graph() + ) + elif name in self.name_to_buffer: + normalized_name = normalize_name(name) + self.input_specs.append( + InputSpec( + InputKind.BUFFER, + arg=TensorArgument(name=normalized_name), + target=name, + persistent=True, + ) + ) + fx_node = get_node_as_placeholder_or_get_attr( + self.fx_graph, name, self.is_top_level_graph() + ) + elif name in self.name_to_constant: + assert isinstance(self.name_to_constant[name], torch.ScriptObject), ( + "Input conversion only handles ScriptObject" + ) + normalized_name = normalize_name(name) + self.input_specs.append( + InputSpec( + InputKind.CUSTOM_OBJ, + arg=CustomObjArgument( + name=normalized_name, class_fqn=normalized_name + ), + target=name, + persistent=False, + ) + ) + fx_node = get_node_as_placeholder_or_get_attr( + self.fx_graph, name, self.is_top_level_graph() + ) + elif isinstance(graph_input.type(), torch.ClassType): + # Directly skip inputs that are ScriptObject but not used in the graph. + continue + else: + normalized_name = normalize_name(name, prefix="input") + self.input_specs.append( + InputSpec( + InputKind.USER_INPUT, + arg=TensorArgument(name=normalized_name), + target=name, + ) + ) + fx_node = self.fx_graph.placeholder(normalized_name) + + self.name_to_node[name] = fx_node + + def convert_aten_Float(self, node: torch._C.Node): + def to_float_tensor(t): + return t.to(dtype=torch.float).item() + + inp_list = [self.get_fx_value_by_ir_value(inp) for inp in node.inputs()] # noqa: C416 + fx_node = self.fx_graph.call_function( + to_float_tensor, + tuple(inp_list), + ) + self.name_to_node[node.output().debugName()] = fx_node + + def convert_aten_tensor(self, node: torch._C.Node): + """aten::tensor creates a constant tensor ad-hoc --> GetAttr""" + args, kwargs = self.get_args_kwargs(node, torch.ops.aten.tensor.default._schema) + + for k in kwargs: + if k == "requires_grad": + kwargs[k] = bool(kwargs[k]) # 0 -> False, 1 -> True + + to_tensor = ( + torch.tensor + if all(isinstance(a, int) for a in args) + else torch._refs.tensor + ) + + def target(*args, **kwargs): + if "dtype" in kwargs and kwargs["dtype"] is not None: + kwargs["dtype"] = _TORCH_ENUM_TO_DTYPE[kwargs["dtype"]] + return to_tensor(*args, **kwargs) + + # def to_dynamic_tensor(*args, **kwargs): + # if "dtype" in kwargs and kwargs["dtype"] is not None: + # kwargs["dtype"] = _TORCH_ENUM_TO_DTYPE[kwargs["dtype"]] + # return torch._refs.tensor(*args, **kwargs) + + output_name = node.output().debugName() + fx_node = self.fx_graph.call_function(target, args, kwargs) + self.name_to_node[output_name] = fx_node + + def convert_aten_append(self, node: torch._C.Node): + # special handle python list append: "aten::append.t(t[](a!) self, t(c -> *) el) -> t[](a!)" + + # inplace append to the list!! This is kinda crazy, as we are inplace mutating the list + # This makes the converter "non-functional", and the result depends on the order of the nodes being converter + # In a sense, the converter now becomes an stateful interpreter + warnings.warn( + "Converting aten::append.t, which is a inplace mutation of the list. " + "This makes the converter non-functional: the result depends on the order of the append nodes being converter!", + stacklevel=2, + ) + + args = tuple(self.get_fx_value_by_ir_value(inp) for inp in node.inputs()) + fx_node = self.fx_graph.call_function(list_append, args) + self.name_to_node[node.output().debugName()] = fx_node + + # inplace mutate arg[0], which is the python list + self.name_to_node[node.inputsAt(0).debugName()] = fx_node + + # Variables that need to be updated to parent module. + if not self.is_top_level_graph() and args[0].op == "placeholder": + self.name_update_from_subblock_to_parent.add(node.inputsAt(0).debugName()) + + def convert_prim_Constant(self, node: torch._C.Node): + name = node.output().debugName() + + value: Any = None + if node.hasAttribute("value"): + constant_kind = node.kindOf("value") + if constant_kind == "i": + value = node.i("value") + elif constant_kind == "f": + value = node.f("value") + elif constant_kind == "s": + value = node.s("value") + elif constant_kind == "t": + alias_name = ( + f"lifted_tensor_{name}" # Follow naming convention from EP tracing. + ) + fx_node = self.fx_graph.get_attr(alias_name) + self.name_to_node[name] = fx_node + name, value = alias_name, node.t("value") + elif constant_kind == "ival": + value = node.ival("value") + else: + raise ValueError(f"Unsupported constant type: {node.kindOf('value')}") + else: + value = None + + self.name_to_constant[name] = value + + def convert_prim_CallMethod(self, node: torch._C.Node): + inp_list = [self.get_fx_value_by_ir_value(inp) for inp in node.inputs()] # noqa: C416 + fx_node = self.fx_graph.call_method( + node.s("name"), + tuple(inp_list), + ) + self.name_to_node[node.output().debugName()] = fx_node + + def convert_prim_device(self, node: torch._C.Node): + input_type = node.input().type() + if input_type.isSubtypeOf(torch._C.TensorType.get()): + device = input_type.device() # type: ignore[attr-defined] + output_name = node.output().debugName() + self.name_to_constant[output_name] = device + else: + raise ValueError(f"Unsupported JitType ({input_type}) when get device") + + def convert_prim_GetAttr(self, node: torch._C.Node): + # Build fully qualified name + attr_fqn = get_attribute_fqn_from_ts_node(self.name_to_attribute_fqn, node) + output_name = node.output().debugName() + self.name_to_attribute_fqn[output_name] = attr_fqn + + if self.is_top_level_graph(): + if self._is_get_attr_node(attr_fqn): + # We insert a get_attr node due to two reasons. + # First, ts graph does not lift tensor constants as input nodes. So + # tensor constants may be ignored by in convert_graph_inputs(). + # Second, attr_fqn may have been written to via SetAttr. Two + # GetAttr may give different values. + self.name_to_node[output_name] = self.fx_graph.get_attr(attr_fqn) + else: + if attr_fqn not in self.name_to_non_tensor_attribute_node: + self.name_to_non_tensor_attribute_node[attr_fqn] = ( + self.name_to_non_tensor_attribute[attr_fqn] + ) + self.name_to_node[output_name] = self.name_to_non_tensor_attribute_node[ + attr_fqn + ] + else: + # Special support for if blocks which do not allow SetAttr TorchScript + # node and get_attr FX Graph Node. + if self._is_get_attr_node(attr_fqn): + self.name_to_node[output_name] = self.name_to_node[attr_fqn] + + def convert_prim_SetAttr(self, node: torch._C.Node): + attr_fqn = get_attribute_fqn_from_ts_node(self.name_to_attribute_fqn, node) + attr_value = tuple(node.inputs())[1] + ts_graph_tensor_input = self.get_fx_value_by_ir_value(attr_value) + if self._is_get_attr_node(attr_fqn): + fx_attr_node = self.fx_graph.get_attr(attr_fqn) + self.fx_graph.call_function( + torch.Tensor.copy_, (fx_attr_node, ts_graph_tensor_input) + ) + else: + self.name_to_non_tensor_attribute_node[attr_fqn] = ts_graph_tensor_input + + def convert_call_function_op(self, node: torch._C.Node): + target = get_op_overload(node) + + args, kwargs = self.get_args_kwargs(node, target._schema) + + fx_node = self.fx_graph.call_function(target, args, kwargs) + + # TODO: convert sourceRange() into stack_trace + # fx_node.meta["stack_trace"] = node.sourceRange() + + if node.outputsSize() == 1: + output_name = node.output().debugName() + self.name_to_node[output_name] = fx_node + else: + for i, outp in enumerate(node.outputs()): + output_name = outp.debugName() + next_fx_node = self.fx_graph.call_function( + operator.getitem, (fx_node, i) + ) + self.name_to_node[output_name] = next_fx_node + + def convert_prim_TupleConstruct(self, node: torch._C.Node): + self._convert_prim_iterator(node) + + def convert_prim_ListConstruct(self, node: torch._C.Node): + self._convert_prim_iterator(node) + + def _convert_prim_iterator(self, node: torch._C.Node): + output_list = [self.get_fx_value_by_ir_value(inp) for inp in node.inputs()] + + output_name = node.output().debugName() + self.name_to_node[output_name] = output_list + + def convert_prim_DictConstruct(self, node: torch._C.Node): + output_dict = {} + k, v = None, None + for i, inp in enumerate(node.inputs()): + # We assume key value are stored in pair in the DictConstruct. + # The first element is the key and the following is the value. + if i % 2 == 0: + k = self.get_fx_value_by_ir_value(inp) + else: + v = self.get_fx_value_by_ir_value(inp) + assert k is not None and v is not None, ( + "DictConstruct has an empty key value pair." + ) + output_dict[k] = v + k, v = None, None + + assert k is None and v is None, ( + "DictConstruct has an odd number of elements (violating our assumption)." + ) + + output_name = node.output().debugName() + self.name_to_node[output_name] = output_dict + + def convert_prim_ListUnpack(self, node: torch._C.Node): + self._convert_prim_unpack_iterator(node) + + def convert_prim_TupleUnpack(self, node: torch._C.Node): + self._convert_prim_unpack_iterator(node) + + def _convert_prim_unpack_iterator(self, node: torch._C.Node): + # Single input and multiple outputs for unpacking. + for i, outp in enumerate(node.outputs()): + outp_name = outp.debugName() + inp = self.get_fx_value_by_ir_value(node.input()) + fx_node = self.fx_graph.call_function(operator.getitem, (inp, i)) + self.name_to_node[outp_name] = fx_node + + def convert_aten_Int(self, node: torch._C.Node): + # converts aten::Int as aten._to_copy + aten::_local_scalar_dense + target = torch.ops.aten._to_copy.default + args = tuple(self.get_fx_value_by_ir_value(input) for input in node.inputs()) + to_copy_node = self.fx_graph.call_function(target, args, {"dtype": torch.int32}) + + fx_node = self.fx_graph.call_function( + torch.ops.aten._local_scalar_dense.default, (to_copy_node,) + ) + + # TODO: convert sourceRange() into stack_trace + # fx_node.meta["stack_trace"] = node.sourceRange() + + output_name = node.output().debugName() + self.name_to_node[output_name] = fx_node + + def convert_prim_NumToTensor(self, node: torch._C.Node): + # Converts prim::NumToTensor as aten.scalar_tensor. + # prim::NumToTensor IRs are currently triggered by: + # .size() https://github.com/pytorch/pytorch/blob/main/torch/csrc/jit/frontend/tracer.cpp#L950 + # .numel() https://github.com/pytorch/pytorch/blob/main/torch/csrc/jit/frontend/tracer.cpp#L971 + # For both of those APIs, torch.jit.trace implicitly sets the output tensor type + # to be LongTensor. + target = torch.ops.aten.scalar_tensor + args = tuple(self.get_fx_value_by_ir_value(input) for input in node.inputs()) + + fx_node = self.fx_graph.call_function(target, args, {"dtype": torch.long}) + output_name = node.output().debugName() + self.name_to_node[output_name] = fx_node + + def convert_prim_CreateObject(self, node: torch._C.Node): + output_name = node.output().debugName() + self.name_to_attribute_fqn[output_name] = "" + + def convert_aten__convolution(self, node: torch._C.Node): + # converts aten::_convolution as aten.convolution, since aten::_convolution + # doesn't have a meta function + target = torch.ops.aten.convolution.default + args, kwargs = self.get_args_kwargs(node, target._schema) + + fx_node = self.fx_graph.call_function(target, args, kwargs) + + output_name = node.output().debugName() + self.name_to_node[output_name] = fx_node + + def convert_aten_div(self, node: torch._C.Node): + target = get_op_overload(node) + schema = target._schema + + args, kwargs = self.get_args_kwargs(node, schema) + + # converts aten::div.Tensor_mode(x, tensor_constant) + # as aten.div.Scalar_mode(x, tensor_constant.item()) + if schema.overload_name == "Tensor_mode": + arg1_name = args[1].name + if arg1_name in self.name_to_constant and isinstance( + self.name_to_constant[arg1_name], torch.Tensor + ): + tensor_constant = self.name_to_constant[arg1_name] + if tensor_constant.numel() == 1: + updated_args = list(args) + updated_args[1] = self.name_to_constant[arg1_name].item() + + fx_node = self.fx_graph.call_function( + torch.ops.aten.div.Scalar_mode, + tuple(updated_args), + kwargs, + ) + + # TODO: convert sourceRange() into stack_trace + # fx_node.meta["stack_trace"] = node.sourceRange() + + output_name = node.output().debugName() + self.name_to_node[output_name] = fx_node + return + + self.convert_call_function_op(node) + + def convert_aten___getitem__(self, node: torch._C.Node): + input_container, index = tuple( + self.get_fx_value_by_ir_value(input) for input in node.inputs() + ) + fx_node = self.fx_graph.call_function( + operator.getitem, (input_container, index) + ) + output_name = node.output().debugName() + self.name_to_node[output_name] = fx_node + + def convert_aten_to(self, node: torch._C.Node): + target = get_op_overload(node) + args, _kwargs = self.get_args_kwargs(node, target._schema) + + # special handle aten.to.dtype and aten.to.prim_dtype followed by inplace_mutation_op + # coz aten.to + inplace_mutation_op pattern would trigger + # "cannot mutate tensors with frozen storage" functionalization error. + # To work around the issue, we override the copy to be True, so that the output + # is for sure not an alias of input + if target is torch.ops.aten.to.dtype or target is torch.ops.aten.to.prim_dtype: + user_nodes = [use.user for use in node.output().uses()] + user_targets = [ + get_op_overload(user_node) + for user_node in user_nodes + if user_node.schema() != "(no schema)" + ] + has_mutable_target = any( + target._schema.is_mutable for target in user_targets + ) + + if has_mutable_target: + assert len(args) >= 4 + new_args = list(args) + new_args[3] = True # copy, override to True + fx_node = self.fx_graph.call_function( + torch.ops.aten.to.dtype, tuple(new_args) + ) + # temp hack to work around the issue https://github.com/pytorch/pytorch/issues/131679 + # When this issue is fixed, the clone node would be no longer needed + clone_node = self.fx_graph.call_function( + torch.ops.aten.clone.default, (fx_node,) + ) + output_name = node.output().debugName() + self.name_to_node[output_name] = clone_node + return + + self.convert_call_function_op(node) + + def convert_aten_add(self, node: torch._C.Node): + if node.schema() == "(no schema)": + if isinstance(node.inputsAt(0).type(), torch.ListType) and isinstance( + node.inputsAt(1).type(), torch.ListType + ): + target = torch.ops.aten.add.t + else: + raise RuntimeError(f"unable to determined the target for {node}") + else: + target = get_op_overload(node) + + if target is torch.ops.aten.add.t: + # special handle python list/tuple add: "aten::add.t(t[] a, t[] b) -> t[]" for + # RuntimeError: aten::add() Expected a value of type 'List[t]' for argument 'a' but instead found type 'immutable_list'. + args, _kwargs = self.get_args_kwargs(node, target._schema) + output_name = node.output().debugName() + self.name_to_node[output_name] = self.fx_graph.call_function(list_add, args) + else: + self.convert_call_function_op(node) + + def _check_prim_loop_support(self, node): + inputs = list(node.inputs()) + + # TODO: (1/N) stage. + if inputs[0].debugName() not in self.name_to_constant: + raise RuntimeError( + "prim::Loop currently cannot run with dynamic value of number of iterations." + ) + + # Make sure the condition is not updated in the subblock. + subblock = next(node.blocks()) + condition_output_name = next(subblock.outputs()).debugName() + for node in subblock.nodes(): + if ( + node.outputsSize() == 1 + and node.output().debugName() == condition_output_name + ): + raise RuntimeError( + "prim::Loop currently cannot run with dynamic value of condition." + ) + if node.outputsSize() >= 2: + for outp in node.outputs(): + if outp.debugName() == condition_output_name: + raise RuntimeError( + "prim::Loop currently cannot run with dynamic value of condition." + ) + + def convert_prim_Loop(self, node: torch._C.Node): + inputs = list(node.inputs()) + self._check_prim_loop_support(node) + + num_iterations = self.get_fx_value_by_ir_value(inputs[0]) + + # Find inputs. + loop_local_arguments = [inp.debugName() for inp in inputs[2:]] + + global_arguments = self._identify_inputs_as_arguments(node) + + # Lift parameters as inputs. + for block in node.blocks(): + global_arguments = global_arguments.union( + self.blocks_to_lifted_attrs[block] + ) + + global_arguments = list(global_arguments) + + subgraph_nodes, subgraph_converters = self._convert_block_to_subgraph( + node, global_arguments + ) + + assert len(subgraph_nodes) == 1 + subgraph_converter = subgraph_converters[0] + if not self.is_top_level_graph(): + self.name_update_from_subblock_to_parent = ( + self.name_update_from_subblock_to_parent.union( + subgraph_converter.name_update_from_subblock_to_parent + ) + ) + + fx_block_args = [ + self.get_fx_value_by_fqn(name) + for name in loop_local_arguments + global_arguments + ] + for iter_idx in range(num_iterations): + loop_node = self.fx_graph.call_function( + execute_subgraph_from_prim_loop, + # Check execute_node function for the expected arguments order. + ( + subgraph_nodes[0], + iter_idx, + len(loop_local_arguments), + *fx_block_args, + ), + {}, + ) + + # Update the value of loop local variables. + if node.outputsSize() >= 1: + for i, outp in enumerate(node.outputs()): + output_name = outp.debugName() + self.name_to_node[output_name] = self.fx_graph.call_function( + operator.getitem, + ( + loop_node, + i + 1, + ), # + 1 because the 0th element is the condition. + ) + fx_block_args[i] = self.name_to_node[output_name] + + # Update the value of global variables, whose values are modified inplace. + + for i, name in enumerate( + subgraph_converter.name_update_from_subblock_to_parent + ): + self.name_to_node[name] = self.fx_graph.call_function( + operator.getitem, + ( + loop_node, + i + node.outputsSize() + 1, + ), # + 1 because the 0th element is the condition. + ) + global_argument_index = global_arguments.index(name) + fx_block_args[i + node.outputsSize() + global_argument_index] = ( + self.name_to_node[name] + ) + + def _check_set_attr_in_if_block(self, if_node: torch._C.Node): + for block in if_node.blocks(): + for node in block.nodes(): + if node.kind() == "prim::SetAttr": + raise RuntimeError( + "During converting prim::If to torch.cond, found prim::SetAttr op" + " which is not supported yet. Please file an issue if you come " + "across this error." + ) + + def convert_prim_If(self, node: torch._C.Node): + self._check_set_attr_in_if_block(node) + + inputs = list(node.inputs()) + assert len(inputs) == 1 + predicate = self.get_fx_value_by_ir_value(inputs[0]) + + # Find inputs. + arguments = self._identify_inputs_as_arguments(node) + + # Lift parameters as inputs. + for block in node.blocks(): + arguments = arguments.union(self.blocks_to_lifted_attrs[block]) + + arguments = list(arguments) + subgraph_nodes, _ = self._convert_block_to_subgraph(node, arguments) + + assert len(subgraph_nodes) == 2 + + fx_block_args = [self.get_fx_value_by_fqn(name) for name in arguments] + + args = ( + predicate, + subgraph_nodes[0], + subgraph_nodes[1], + tuple(fx_block_args), + ) + + cond_node = self.fx_graph.call_function(torch.cond, args, {}) + + # prim::If can also have zero output. + if node.outputsSize() == 1: + output_name = node.output().debugName() + self.name_to_node[output_name] = cond_node + elif node.outputsSize() > 1: + for i, output in enumerate(node.outputs()): + output_name = output.debugName() + getitem = self.fx_graph.call_function(operator.getitem, (cond_node, i)) + self.name_to_node[output_name] = getitem + + def convert_aten_Bool(self, node: torch._C.Node): + self._convert_as_noop(node) + + def convert_prim_Enter(self, node: torch._C.Node): + # export generally treats prim::Enter as noop + # The only context manager export supports is aten::enable_grad. + # Unfortunately, TorchScript does not support aten::enable_grad yet. + # TODO: support aten::enable_grad in both TorchScript and Converter. + return + + def convert_prim_Exit(self, node: torch._C.Node): + # export treats prim::Exit as noop + return + + def _convert_as_noop(self, node: torch._C.Node): + # Converts the node as a no-op by mapping its output node as arg[0] + + target = get_op_overload(node) + schema = target._schema + + args, _kwargs = self.get_args_kwargs(node, schema) + + output_name = node.output().debugName() + self.name_to_node[output_name] = args[0] + + def convert_profiler__record_function_exit(self, node: torch._C.Node): + # _record_function_exit has side effect so we keep it in fx.graph + # currently, _record_function_enter_new and _record_function_exit are + # discarded during `retrace_as_exported_program`. + target = torch.ops.profiler._record_function_exit + args = tuple(self.get_fx_value_by_ir_value(input) for input in node.inputs()) + self.fx_graph.call_function(target, args) + + def convert_prim_tolist(self, node: torch._C.Node): + # prim::tolist cannot be supported by `_convert_standard_operators` + # since it requires call_method instead of call_function. + target = "tolist" + args = (self.get_fx_value_by_ir_value(next(node.inputs())),) + fx_node = self.fx_graph.call_method(target, args) + output_name = node.output().debugName() + self.name_to_node[output_name] = fx_node + + def convert_prim_Uninitialized(self, node: torch._C.Node): + # `prim::Uninitialized` is inserted by the compiler when it can prove + # the value will never be used. It can be introduced by exceptions, + # breaks, continues, and returns. + # So we add a dummy constant to the graph. + output_name = node.output().debugName() + self.name_to_constant[output_name] = torch.Tensor() + + def _convert_standard_operators(self, node: torch._C.Node): + target = kind_to_standard_operators[node.kind()] + args = tuple(self.get_fx_value_by_ir_value(input) for input in node.inputs()) + fx_node = self.fx_graph.call_function(target, args) + output_name = node.output().debugName() + self.name_to_node[output_name] = fx_node + + def convert_node(self, node: torch._C.Node): + node_kind = node.kind() + + # Get handler based on namespace and operator name. + # Provide a default node handler as well in case we don't find + # matching converter for that. + handler_func_name = ir_name_to_func_name(node_kind) + handler_func = getattr(self, handler_func_name, self.convert_call_function_op) + + # str calls print function implemented in CPP. To avoid repeating + # the entire logic here, we simply keep first line from node string (getting rid + # of sub-blocks IR prints). + node_str = "".join(str(node).split("\n")[:1]) + log.debug("[%s] converts [%s]", handler_func.__name__, node_str) + try: + handler_func(node) + except Exception as e: + raise RuntimeError(f"TS2EPConverter failed for node {node_kind}") from e + + def convert_graph_outputs(self): + args = [] + outp_name_list = [outp.debugName() for outp in self.ts_graph.outputs()] + list( + self.name_update_from_subblock_to_parent + ) + for output_name in outp_name_list: + if output_name in self.name_to_node: + fx_node = self.name_to_node[output_name] + # TODO: Revisit this later after HigherOrderOp design changes. + # Currently, we cannot directly return input as output. + if ( + not self.is_top_level_graph() + and isinstance(fx_node, torch.fx.Node) + and fx_node.op == "placeholder" + ): + fx_node = self.fx_graph.call_function(torch.clone, (fx_node,)) + args.append(fx_node) + self.output_specs.append( + OutputSpec( + OutputKind.USER_OUTPUT, + arg=TensorArgument(name=output_name), + target=output_name, + ) + ) + elif output_name in self.name_to_constant: + args.append(self.name_to_constant[output_name]) + self.output_specs.append( + OutputSpec( + OutputKind.USER_OUTPUT, + arg=ConstantArgument( + name=output_name, value=self.name_to_constant[output_name] + ), + target=output_name, + ) + ) + else: + raise ValueError(f"Output {output_name} not found") + + if len(args) == 0: + # Sub-block of prim::If can have zero output. + self.fx_graph.output([]) + elif len(args) == 1: + self.fx_graph.output( + args[0] + ) # Get rid of an extra list wrapped around final output. + elif len(args) > 1: + self.fx_graph.output( + args + ) # For prim::Loop and prim::If with multiple outputs. + else: + # Sub-block of prim::Loop can have multiple outputs. + self.fx_graph.output(args) + + +class ExplainTS2FXGraphConverter(TS2FXGraphConverter): + """ + Run TS2FXGraphConverter in an explain mode. It collects all failed operators conversions + and provide that information to users. In order to collect all failed conversions, it + also mocks some internal attributes (e.g., name_to_node). + """ + + class _DictMock(dict): + def __init__(self, dict_data, mock_value): + super().__init__(dict_data) + self.mock_value = mock_value + + def __getitem__(self, key): + # If the original dictionary has the key, return its value. + # Otherwise, return the mock value. + if not super().__contains__(key): + return self.mock_value + return super().__getitem__(key) + + def __contains__(self, key): + return True + + def __init__( + self, + ts_graph: Union[torch._C.Graph, torch._C.Block], + name_to_param: dict[str, torch.Tensor], + name_to_buffer: dict[str, torch.Tensor], + blocks_to_lifted_attrs: dict[torch._C.Block, set[str]], + name_to_non_tensor_attribute: dict[str, Any], + name_to_constant: dict[str, Any], + name_to_attribute_fqn: dict[str, str], + ): + super().__init__( + ts_graph, + name_to_param, + name_to_buffer, + blocks_to_lifted_attrs, + name_to_non_tensor_attribute, + name_to_constant, + name_to_attribute_fqn, + ) + + # Data to keep track of unsupported nodes. + self.unsupported_node_list: list[torch._C.Node] = [] + + # Add mock to needed attributes. + self.name_to_node = ExplainTS2FXGraphConverter._DictMock( + self.name_to_node, + # Dummy node. + torch.fx.Node( + None, # type: ignore[arg-type] + "mock", + "call_function", + lambda: None, + (), + {}, + ), + ) + + def explain(self): + self.convert_graph_inputs() + for node in self.ts_graph.nodes(): + self.convert_node(node) + self.convert_graph_outputs() + + def convert_node(self, node): + try: + super().convert_node(node) + except Exception: + self.unsupported_node_list.append(node) + + +@contextmanager +def disable_logging(log): + disabled = log.disabled + log.disabled = True + try: + yield + finally: + log.disabled = disabled + + +class TS2EPConverter: + # TorchScript model to ExportedProgram converter + def __init__( + self, + ts_model: Union[torch.jit.ScriptModule, torch.jit.ScriptFunction], + sample_args: tuple[Any, ...], + sample_kwargs: Optional[dict[str, Any]] = None, + ): + self.ts_model = ts_model + self.ts_graph, self.params, _, _ = _create_jit_graph(ts_model, sample_args) + + self.sample_args = sample_args + self.sample_kwargs = sample_kwargs + + self.name_to_param: dict[str, torch.Tensor] = {} + self.name_to_buffer: dict[str, torch.Tensor] = {} + param_list = ( + list(self.ts_model.parameters()) + if not isinstance(self.ts_model, torch._C.ScriptFunction) + else [] + ) + if not isinstance(self.ts_model, torch._C.ScriptFunction): + for k, tensor in self.ts_model.state_dict().items(): # type: ignore[union-attr] + # Check if tensor belongs to any parameter. + if any( + (tensor == param).all() + for param in param_list + if tensor.shape == param.shape + ): + self.name_to_param[k] = tensor + else: + self.name_to_buffer[k] = tensor + + self.name_to_non_tensor_attributes: dict[str, Any] = {} + self.name_to_constant: dict[str, Any] = {} + + self.lift_get_attr() + + def convert(self) -> ExportedProgram: + log.info( + """ +TS2EPConverter logging starts from here. + +INFO: (TORCH_LOGS="export" ) + * Log TorchScript IR. + +DEBUG: (TORCH_LOGS="+export" ), additionally + * Log conversion IR by IR in a format of [] converts []. + """ + ) + log.info("TorchScript graph\n\n%s\n", self.ts_graph) + + blocks_to_lifted_attrs, name_to_attribute_fqn = get_block_to_lifted_attrs( + self.ts_graph + ) + + graph_converter = TS2FXGraphConverter( + self.ts_graph, + self.name_to_param, + self.name_to_buffer, + blocks_to_lifted_attrs, + self.name_to_non_tensor_attributes, + self.name_to_constant, + name_to_attribute_fqn, + ) + gm = graph_converter.convert() + + # Post-processing step to deal with quantized operators. + replace_quantized_ops_with_standard_ops(gm) + log.info("GraphModule: %s", gm.print_readable(print_output=False)) + + ep = self.retrace_as_exported_program( + gm, + graph_converter.name_to_constant, + ) + log.info("%s", ep) + + # Post-processing step to ensure ExportedProgram has the same state_dict as + # the original TorchScript model. Throw warnings for additionally populated + # state_dict entries. + if not isinstance(self.ts_model, torch._C.ScriptFunction): + for k, tensor in self.ts_model.state_dict().items(): # type: ignore[union-attr] + if k not in ep.state_dict: + warnings.warn( + f"Manually populate {k} into state_dict ExportedProgram, but it is never used by the ExportedProgram.", + stacklevel=2, + ) + ep.state_dict[k] = tensor + + return ep + + @disable_logging(log) + def explain(self, print_output=True): + blocks_to_lifted_attrs, name_to_attribute_fqn = get_block_to_lifted_attrs( + self.ts_graph + ) + + graph_converter = ExplainTS2FXGraphConverter( + self.ts_graph, + self.name_to_param, + self.name_to_buffer, + blocks_to_lifted_attrs, + self.name_to_non_tensor_attributes, + self.name_to_constant, + name_to_attribute_fqn, + ) + graph_converter.explain() + if len(graph_converter.unsupported_node_list) > 0: + explain_str = "Unsupported nodes are found in the following list:" + for i, n in enumerate(graph_converter.unsupported_node_list): + node_str = "".join(str(n).split("\n")[:1]) + explain_str += f"\n\n {i}. {n.kind()} [{node_str}]" + else: + explain_str = "Success!" + if print_output: + print(explain_str) + return explain_str + + def retrace_as_exported_program( + self, + gm: torch.fx.GraphModule, + name_to_constant: dict[str, Any], + ): + dynamic_shapes = _tree_map_with_path( + lambda path, x: ( + [Dim.AUTO] * x.dim() if isinstance(x, torch.Tensor) else None + ), + self.sample_args, + ) + + # TODO: adjust input orders to match GraphSignature convention + ep = torch.export._trace._export( + gm, + self.sample_args, + dynamic_shapes=dynamic_shapes, + strict=False, + pre_dispatch=True, + ) + + # Post-processing to make sure the ExportedProgram states are correct. + # Because during conversion, we set tensor constants as GetAttr, + # retracing cannot recognize them as tensor constants but instead + # treat them as buffers. We need to set them again here. + ep._constants.update( + { + k: v + for k, v in name_to_constant.items() + if isinstance(v, (torch.Tensor, torch.ScriptObject)) + } + ) + for k in name_to_constant: + ep.state_dict.pop(k, None) + + for spec in ep.graph_signature.input_specs: + # Mark as constant tensors for erroneously traced buffers. + if spec.kind == InputKind.BUFFER and spec.target in name_to_constant: + assert isinstance(name_to_constant[spec.target], torch.Tensor), ( + f"{type(name_to_constant[spec.target])} has been erroneously marked as buffer" + ) + spec.kind = InputKind.CONSTANT_TENSOR + spec.persistent = None + ep.verifier().check(ep) + + return ep + + def lift_get_attr(self): + # This function lifts multiple data types. + + # 1. Tensor constants attributes (e.g., self.data = torch.tensor([2,3])) + # to buffers. Currently, when there are tensor constants, export + # would error and ask users to register tensor constants as buffers. + # Since it is hard to manually do so for TorchScript models + # (e.g., source code is missing), this function automatically + # lifts tensor constants to be buffers. + + # 2. ScriptObbject to constant. It will then be converted to getattr in + # in the fx graph. + # + # This function should happen in TS2EPConverter instead of + # TS2FXGraphConverter since it gets attributes from self.ts_model + # which is not accessible in TS2FXGraphConverter. It is similar to where + # we collect self.name_to_param and self.name_to_buffer. + name_to_attribute_fqn: dict[str, str] = {} + + def get_attr(fqn: str): + name = fqn.split(".") + v = self.ts_model + for n in name: + v = getattr(v, n) + return v + + def get_fqn(node: torch._C.Node): + attr_name = node.s("name") + input_name = node.input().debugName() + root_attr_name = name_to_attribute_fqn[input_name] + attr_fqn = f"{root_attr_name}.{attr_name}" if root_attr_name else attr_name + return attr_fqn + + def _dfs_get_attr(block): + for node in block.nodes(): + if node.kind() == "prim::CreateObject": + output_name = node.output().debugName() + name_to_attribute_fqn[output_name] = "" + + if node.kind() == "prim::GetAttr": + attr_fqn = get_fqn(node) + value = get_attr(attr_fqn) + output_name = node.output().debugName() + name_to_attribute_fqn[output_name] = attr_fqn + if isinstance(value, torch.Tensor): + if attr_fqn not in self.name_to_buffer: + # Lift tensor constants to be a buffer + self.name_to_buffer[attr_fqn] = value + elif isinstance(value, torch.ScriptObject): + if attr_fqn not in self.name_to_constant: + self.name_to_constant[attr_fqn] = value + else: + self.name_to_non_tensor_attributes[attr_fqn] = value + + for subblock in node.blocks(): + _dfs_get_attr(subblock) + + _dfs_get_attr(self.ts_graph) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/non_strict_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/non_strict_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..4b674ec25e8618176d4f3378264addaf2bc05acf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/non_strict_utils.py @@ -0,0 +1,1142 @@ +# mypy: allow-untyped-defs +import builtins +import contextlib +import functools +import inspect +import logging +import math +import sys +from collections import defaultdict +from collections.abc import Callable, Sequence +from contextlib import contextmanager +from typing import Any, Optional, TYPE_CHECKING, Union + +import torch +import torch.utils._pytree as pytree +from torch._dynamo.source import ( + AttrSource, + GetItemSource, + LocalSource, + TensorProperty, + TensorPropertySource, +) +from torch._dynamo.variables.builder import TrackedFake +from torch._export.passes.lift_constants_pass import ConstantAttrMap +from torch._export.utils import _fakify_params_buffers +from torch._guards import Source +from torch._library.fake_class_registry import FakeScriptObject +from torch._library.opaque_object import is_opaque_type +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.export import Constraint +from torch.export.dynamic_shapes import ( + _check_dynamic_shapes, + _combine_args, + _DimHint, + _DimHintType, + _IntWrapper, + _process_dynamic_shapes, + _RelaxedConstraint, + _tree_map_with_path, +) +from torch.export.graph_signature import CustomObjArgument +from torch.fx.experimental import _config as config +from torch.fx.experimental.symbolic_shapes import ( + _find_user_code_frame, + _suggest_fixes_for_data_dependent_error_non_strict, + ConstraintViolationError, + DimDynamic, + EqualityConstraint, + GuardOnDataDependentSymNode, + RelaxedUnspecConstraint, + ShapeEnv, + StatelessSymbolicContext, + SymIntSymbolicContext, + ValueRanges, +) +from torch.utils._pytree import ( + GetAttrKey, + KeyPath, + MappingKey, + SequenceKey, + tree_map_with_path, +) +from torch.utils._sympy.numbers import int_oo + + +if TYPE_CHECKING: + from sympy import Symbol + + +log = logging.getLogger(__name__) + + +class _KeyPath: + """ + Wraps `KeyPath` to aid `isinstance` checks. + """ + + def __init__(self, kp: KeyPath): + self.kp = kp + + +class _KeyPathTrie: + """ + Builds a trie of `KeyPath` prefixes mapping to `Source` leaves. + """ + + def __init__(self): + self.root = {} + + def add(self, kp: KeyPath, src: Source): + assert len(kp) > 0 + *path, leaf = kp + node = self.root + for k in path: + if k not in node: + node[k] = {} + node = node[k] + node[leaf] = src + + def get(self, kp: KeyPath) -> tuple[Source, KeyPath]: + node = self.root + while not isinstance(node, Source): + assert len(kp) > 0 + k, *kp = kp # type: ignore[assignment] + node = node[k] + # pyrefly: ignore [bad-return] + return node, kp + + +def make_sourced_prefixes(nn_module, args, kwargs) -> _KeyPathTrie: + kp_args, kp_kwargs = tree_map_with_path( + lambda kp, _: _KeyPath(kp), + (tuple(None for _ in args), {k: None for k in kwargs}), # noqa: C420 + ) + kp_combined_args = _combine_args(nn_module, kp_args, kp_kwargs) + + sourced_prefixes = _KeyPathTrie() + for name, struct in kp_combined_args.items(): + src = LocalSource(name) + + if isinstance(struct, _KeyPath): + sourced_prefixes.add(struct.kp, src) + elif isinstance(struct, tuple): + for i, prefix in enumerate(struct): + assert isinstance(prefix, _KeyPath) + sourced_prefixes.add(prefix.kp, GetItemSource(src, i)) + elif isinstance(struct, dict): + for k, prefix in struct.items(): + assert isinstance(prefix, _KeyPath) + sourced_prefixes.add(prefix.kp, GetItemSource(src, k)) + + return sourced_prefixes + + +def key_path_to_source( + kp: KeyPath, sourced_prefixes: Optional[_KeyPathTrie] = None +) -> Source: + """ + Given a key path, return the source for the key path. + """ + if sourced_prefixes is None: + source: Source = LocalSource("args") + else: + source, kp = sourced_prefixes.get(kp) + + for k in kp: + if isinstance(k, SequenceKey): + source = GetItemSource(source, k.idx) + elif isinstance(k, MappingKey): + source = GetItemSource(source, k.key) + elif isinstance(k, GetAttrKey): + source = AttrSource(source, k.name) + else: + raise ValueError(f"Unknown KeyEntry {k}") + + return source + + +def _is_constant_argument(t): + return t is None or isinstance(t, (float, bool, str)) + + +def fakify( + mode: FakeTensorMode, + kp: KeyPath, + t: Any, + t_constraints: dict[int, dict[int, Constraint]], + sources: dict[tuple[int, int], list[Source]], + sourced_prefixes: Optional[_KeyPathTrie] = None, +): + source = key_path_to_source(kp, sourced_prefixes=sourced_prefixes) + if ( + _is_constant_argument(t) + or isinstance(t, (torch.ScriptObject, torch.nn.Module)) + or is_opaque_type(type(t)) + ): + return t + + if isinstance(t, _IntWrapper): + if t.dynamism is not None and t.dynamism.type in ( # type: ignore[union-attr] + _DimHintType.DYNAMIC, + _DimHintType.AUTO, + ): + symint = mode.shape_env.create_unspecified_symint_and_symbol( # type: ignore[union-attr] + t.val, source, DimDynamic.DYNAMIC + ) + context = ( + SymIntSymbolicContext( + constraint=RelaxedUnspecConstraint(warn_only=False) + ) + if t.dynamism.type == _DimHintType.DYNAMIC # type: ignore[union-attr] + else None + ) + mode.shape_env.tracked_fakes.append( # type: ignore[union-attr] + TrackedFake(symint, source, context) + ) + return symint + else: + return t.val + + if not isinstance(t, torch.Tensor): + raise ValueError( + f"Unsupported input type {type(t)}. " + "Export only supports pytree containers of basic types (Tensor, int, float, ...) as input. " + "To register a custom dataclass, use torch.export.register_dataclass. " + "To register a custom container type, use torch.utils._pytree.register_pytree_node. " + "To register a constant input, use torch.utils._pytree.register_constant" + ) + + # Create symbolic context (handles subclass recursion internally) + symbolic_context = _create_symbolic_context_for_tensor( + t, source, t_constraints, sources, mode + ) + + fake = mode.from_tensor(t, source=source, symbolic_context=symbolic_context) + mode.shape_env.tracked_fakes.append(TrackedFake(fake, source, symbolic_context)) # type: ignore[union-attr] + return fake + + +def _create_symbolic_context_for_tensor(t, source, t_constraints, sources, mode): + """Helper function to create symbolic context for a tensor.""" + from torch._dynamo.source import AttrSource + from torch.fx.experimental.symbolic_shapes import ( + DimDynamic, + RelaxedUnspecConstraint, + SubclassSymbolicContext, + ) + from torch.utils._python_dispatch import is_traceable_wrapper_subclass + + # Common dynamic dimension logic for both regular tensors and subclasses + n_dims = len(t.shape) + dynamic_sizes = [] + constraint_sizes = [None] * n_dims + + for i in range(n_dims): + if i in getattr(t, "_dynamo_weak_dynamic_indices", {}): + dynamic_sizes.append(DimDynamic.DYNAMIC) + elif i in getattr(t, "_dynamo_dynamic_indices", {}): + # bit annoying, but we need to replicate process in _dynamo/variables/builder.py + # where a RelaxedUnspecConstraint is created for Dim.DYNAMIC, so constraint violations + # are raised when specializing. + dynamic_sizes.append(DimDynamic.DYNAMIC) + constraint_sizes[i] = RelaxedUnspecConstraint(warn_only=False) # type: ignore[call-overload] + else: + dynamic_sizes.append(DimDynamic.STATIC) + + # Handle nested subclasses + if is_traceable_wrapper_subclass(t): + # Get inner contexts recursively + inner_contexts = {} + attrs, _ = type(t).__tensor_flatten__(t) + + # Propagate outer tensor constraints to inner tensors if not already present + for attr in attrs: + inner_tensor = getattr(t, attr) + inner_source = AttrSource(source, attr) + inner_contexts[attr] = _create_symbolic_context_for_tensor( + inner_tensor, inner_source, t_constraints, sources, mode + ) + + symbolic_context = SubclassSymbolicContext( + dynamic_sizes=dynamic_sizes, + constraint_sizes=constraint_sizes, # type: ignore[arg-type] + view_base_context=None, + tensor_source=source, + shape_env_to_source_to_symbol_cache={}, + inner_contexts=inner_contexts, + ) + else: + symbolic_context: StatelessSymbolicContext = ( # type: ignore[no-redef] + StatelessSymbolicContext( + dynamic_sizes=dynamic_sizes, + constraint_sizes=constraint_sizes, # type: ignore[arg-type] + ) + ) + + # Apply constraints (common logic) + t_id = id(t) + assert mode.shape_env is not None + if t_id in t_constraints: + for i, constraint in t_constraints[t_id].items(): + src = TensorPropertySource(base=source, prop=TensorProperty.SIZE, idx=i) + sources[(t_id, i)].append(src) + if isinstance(constraint, _RelaxedConstraint): + continue + symbolic_context.constraint_sizes[i] = constraint.constraint_range + mode.shape_env.source_name_to_debug_name[src.name] = constraint.name # type: ignore[assignment] + + return symbolic_context + + +def _is_unbacked_symint(symbol): + if not isinstance(symbol, torch.SymInt): + return False + + return symbol.node.shape_env.is_unbacked_symint(symbol.node.expr) + + +def _tensor_min_max(*args, real_callable, tensor_callable, **kwargs): + """ + This logic is replicated from dynamo/variables/builtin.py + """ + if len(args) == 2 and not kwargs: + arg1, arg2 = args + + # Case 1: Both are tensors + if isinstance(arg1, torch.Tensor) and isinstance(arg2, torch.Tensor): + return tensor_callable(arg1, arg2) + + # Case 2: One tensor, one scalar + elif isinstance(arg1, torch.Tensor) or isinstance(arg2, torch.Tensor): + if not isinstance(arg1, torch.Tensor): + arg1, arg2 = arg2, arg1 + + if isinstance(arg2, (int, float)): + kwarg = {"min" if tensor_callable is torch.maximum else "max": arg2} + return torch.clamp(arg1, **kwarg) # type: ignore[call-overload] + else: + return real_callable(arg1, arg2) + + # Case 3: SymInts + elif isinstance(arg1, torch.SymInt) or isinstance(arg2, torch.SymInt): + return ( + torch.sym_max(arg1, arg2) + if tensor_callable is torch.maximum + else torch.sym_min(arg1, arg2) + ) + + # Fallback + else: + return real_callable(arg1, arg2) + + # Single iterable argument handling + if len(args) == 1 and not kwargs: + iterable = args[0] + + if isinstance(iterable, torch.Tensor): + return tensor_callable(iterable) + try: + iterator = iter(iterable) + except TypeError: + pass + else: + items = list(iterator) + if not items: + raise ValueError(f"{real_callable.__name__}() arg is an empty sequence") + + return functools.reduce( + lambda a, b: _tensor_min_max( + a, b, real_callable=real_callable, tensor_callable=tensor_callable + ), + items, + ) + + # Fallback to original callable + return real_callable(*args, **kwargs) + + +@contextmanager +def _override_builtin_ops(): + original_max = builtins.max + original_min = builtins.min + original_pow = math.pow + + # pyrefly: ignore [bad-assignment] + builtins.max = functools.partial( + _tensor_min_max, real_callable=original_max, tensor_callable=torch.maximum + ) + + # pyrefly: ignore [bad-assignment] + builtins.min = functools.partial( + _tensor_min_max, real_callable=original_min, tensor_callable=torch.minimum + ) + + math.pow = lambda x, y: x**y # type: ignore[operator] + + try: + yield + finally: + builtins.max = original_max + builtins.min = original_min + math.pow = original_pow + + +def make_fake_inputs( + nn_module, + args, + kwargs, + dynamic_shapes, + prefer_deferred_runtime_asserts_over_guards=False, +): + """ + Given an nn module, example inputs, and constraints, return a new fake mode, + fake inputs created in that mode whose dynamic shape dimensions are constrained + by the given ranges, and sources for pairs of dynamic shape dimensions that are + constrained to be equal. + """ + # TODO(avik): refactor Dynamo to avoid duplication of the following code + # between non-strict and strict. + # Specifically, here (non-strict) we do the following pre-tracing steps: + # - Fakify inputs. + # - Process input shape equalities. + # In strict, these steps are spread across multiple files: + # - output_graph.py fakifies inputs. + # - [post-tracing] guards.py processes input shape equalities. + import torch._functorch.config as _config + + # Map ints to a wrapper structure to help us mark it as dynamic, if it is + # dynamic. We will unwrap ints in fakify later. + args, kwargs = pytree.tree_map_only(int, lambda a: _IntWrapper(a), (args, kwargs)) + + combined_args = _combine_args(nn_module, args, kwargs) + _check_dynamic_shapes(combined_args, dynamic_shapes) + constraints = _process_dynamic_shapes(combined_args, dynamic_shapes) + t_constraints: dict[int, dict[int, Constraint]] = defaultdict(dict) + for constraint in constraints: + t_constraints[constraint.t_id][constraint.dim] = constraint + + context = torch._guards.TracingContext.try_get() + if context is not None: + # This occurs when we are exporting within dynamo. There already exists + # a toplevel TracingContext with a fake mode, so we do not want to + # create another fake mode. + fake_mode = context.fake_mode + assert fake_mode is not None + else: + if isinstance(nn_module.forward, functools.partial): + # functools handles nesting by itself, no need to recurse + code = nn_module.forward.func.__code__ + elif ( + sys.version_info >= (3, 14) + and (fwd := getattr(nn_module.forward, "__func__", None)) + and isinstance(fwd, functools.partial) + ): + # functools.partial is now a method descriptor: + # https://docs.python.org/3/whatsnew/3.14.html#changes-in-the-python-api + code = fwd.func.__code__ + else: + code = nn_module.forward.__code__ + co_fields = { + "co_name": code.co_name, + "co_filename": code.co_filename, + "co_firstlineno": code.co_firstlineno, + } + with _config.patch(fake_tensor_allow_unsafe_data_ptr_access=False): + fake_mode = FakeTensorMode( + shape_env=ShapeEnv( + tracked_fakes=[], + co_fields=co_fields, + prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards, + trace_asserts=True, + ), + allow_non_fake_inputs=True, + export=True, + ) + if fake_mode.shape_env is None or fake_mode.shape_env.tracked_fakes is None: + raise ValueError( + "Detected fake_mode does not have a shape_env with tracked fakes. " + "If you constructed the module under a FakeTensorMode, " + "please initialize it like: FakeTensorMode(shape_env=ShapeEnv(tracked_fakes=[]))" + ) + + with fake_mode: + original_signature = inspect.signature(nn_module.forward) + sources: dict[tuple[int, int], list[Source]] = defaultdict(list) + sourced_prefixes = make_sourced_prefixes(nn_module, args, kwargs) + fake_args, fake_kwargs = tree_map_with_path( + lambda kp, val: fakify( + fake_mode, + kp, + val, + t_constraints, + sources, + sourced_prefixes=sourced_prefixes, + ), + (args, kwargs), + ) + + names: dict[str, tuple[int, int]] = {} + source_pairs: list[tuple[Source, Source]] = [] + derived_equalities: list[tuple[Source, Union[Source, Symbol], Callable]] = [] + phantom_symbols: dict[str, Symbol] = {} + relaxed_sources: set[Source] = set() + for constraint in constraints: + torch.export.dynamic_shapes._process_equalities( + constraint, + lambda t_id, dim: sources[(t_id, dim)], + fake_mode.shape_env, + names, + source_pairs, + derived_equalities, + phantom_symbols, + relaxed_sources, + ) + + equalities_inputs = EqualityConstraint( + source_pairs=source_pairs, + derived_equalities=derived_equalities, + phantom_symbols=list(phantom_symbols.values()), + relaxed_sources=relaxed_sources, + warn_only=False, + ) + return ( + fake_mode, + fake_args, + fake_kwargs, + equalities_inputs, + original_signature, + dynamic_shapes, + ) + + +def _flatten_dynamic_shapes( + combined_args: dict[str, Any], + dynamic_shapes: Union[dict[str, Any], tuple[Any], list[Any]], +) -> list[Any]: + flat_shapes = [] + + def _tree_map_helper(path, t, shape): + nonlocal flat_shapes + flat_shapes.append(shape) + + _tree_map_with_path(_tree_map_helper, combined_args, dynamic_shapes) + return flat_shapes + + +def _clean_dynamic_markers(tensor: torch.Tensor) -> None: + for attr in [ + "_dynamo_weak_dynamic_indices", + "_dynamo_dynamic_indices", + "_dynamo_dynamic_range", + "_dynamo_static_indices", + "_dynamo_unbacked_indices", + ]: + if hasattr(tensor, attr): + delattr(tensor, attr) + + +def produce_guards_and_solve_constraints( + fake_mode: FakeTensorMode, + gm: torch.fx.GraphModule, + dynamic_shapes: Union[dict[str, Any], tuple[Any], list[Any], None], + equalities_inputs: EqualityConstraint, + original_signature: inspect.Signature, +): + """ + Given a fake mode, sources pairs corresponding to equal dynamic shape dimensions, + and a graph module, produce guards on the fake mode's shape env (raising constraint + violations if any), solve (to suggest simplifications or fixes). + Dynamo already performs this, so this is for non-strict mode. + + Additional inputs: + equalities_inputs: the equality constraints to use for guards + original_signature: the signature of the forward method + """ + shape_env = fake_mode.shape_env + assert shape_env is not None + assert shape_env.tracked_fakes is not None + + placeholders = [tf.fake for tf in shape_env.tracked_fakes] + sources = [tf.source for tf in shape_env.tracked_fakes] + input_contexts = [tf.symbolic_context for tf in shape_env.tracked_fakes] + constraint_violation_error = None + try: + shape_env.produce_guards( + placeholders, + sources, + input_contexts=input_contexts, + equalities_inputs=equalities_inputs, + ignore_static=False, + ) + except ConstraintViolationError as e: + constraint_violation_error = e + + shape_env.frozen = True + dim_constraints = shape_env.dim_constraints + if dim_constraints is None: + # Expected when shape_env.produce_guards throws an early constraint violation error. + # There is nothing to solve for in this case. + # TODO(avik): Maybe record the constraint violation error instead and replay later? + assert constraint_violation_error + raise constraint_violation_error + dim_constraints.solve() + forced_specializations = dim_constraints.forced_specializations() + + msg = dim_constraints.prettify_results( + original_signature, + dynamic_shapes, # type: ignore[arg-type] + constraint_violation_error, + forced_specializations, # type: ignore[arg-type] + ) + + if constraint_violation_error: + if constraint_violation_error.args: + constraint_violation_error.args = ( + constraint_violation_error.args[0] + msg, + ) + else: + constraint_violation_error.args = (msg,) + elif forced_specializations: + constraint_violation_error = ConstraintViolationError(msg) + if constraint_violation_error: + raise constraint_violation_error + + +def is_int(x: object) -> bool: + return isinstance(x, int) or (isinstance(x, torch.SymInt) and x.node.expr.is_number) + + +def _constrain_user_specified_dimhint_range( + symint: torch.SymInt, + hint: int, + dim: _DimHint, + range_constraints, + shape_env, + keypath: KeyPath, + i: Optional[int] = None, +) -> Optional[str]: + trace_vr = ( + range_constraints[symint.node.expr] + if not is_int(symint) + else ValueRanges(int(symint), int(symint)) + ) + + # warn on 0/1 specialization for Dim.AUTO; not an actual error + if dim.type == _DimHintType.AUTO and trace_vr.is_singleton() and hint in (0, 1): + pathstr = f"inputs{pytree.keystr(keypath)}" + if i is not None: + pathstr += f".shape[{i}]" + msg = ( + f"dimension {pathstr} 0/1 specialized; Dim.AUTO was specified along " + + f"with a sample input with hint = {hint}." + ) + log.warning(msg) + + try: + user_vr = ValueRanges( + lower=0 if dim.min is None else dim.min, + upper=int_oo if dim.max is None else dim.max, + ) + if is_int(symint): + out_vr = trace_vr & user_vr + else: + range_constraints[symint.node.expr] &= user_vr + shape_env.var_to_range[symint.node._expr] &= user_vr + out_vr = range_constraints[symint.node.expr] + + # check for Dim.DYNAMIC specializations; special case error message on 0/1 + if dim.type == _DimHintType.DYNAMIC and out_vr.is_singleton(): + path = f"inputs{pytree.keystr(keypath)}" + if i is not None: + path += f".shape[{i}]" + if ( + trace_vr.is_singleton() + and hint in (0, 1) + and not torch.fx.experimental._config.backed_size_oblivious + ): + msg = ( + f"- Received user-specified dim hint Dim.DYNAMIC(min={dim.min}, max={dim.max}), " + f"but export 0/1 specialized due to hint of {hint} for dimension {path}." + ) + else: + msg = ( + f"- Received user-specified dim hint Dim.DYNAMIC(min={dim.min}, max={dim.max}), " + f"but tracing inferred a static shape of {out_vr.lower} for dimension {path}." + ) + return msg + + except torch.utils._sympy.value_ranges.ValueRangeError: + path = f"inputs{pytree.keystr(keypath)}" + if i is not None: + path += f".shape[{i}]" + msg = ( + f"- Received user-specified min/max range of [{dim.min}, {dim.max}], " + f"conflicting with the inferred min/max range of [{trace_vr.lower}, {trace_vr.upper}], " + f"for {path}." + ) + return msg + + return None + + +def make_constraints( + fake_mode: FakeTensorMode, + gm: torch.fx.GraphModule, + combined_args: dict[str, Any], + dynamic_shapes: Union[dict[str, Any], tuple[Any], list[Any], None], + num_lifted_inputs: int, +): + """ + Given a fake mode's shape env and user-specified dynamic shapes, + return the resulting range constraints and equality constraints. + + Additional args: + num_lifted_inputs: the number of non-user-input placeholder nodes in the graph + (used only to enumerate the user-input nodes) + """ + + shape_env = fake_mode.shape_env + assert shape_env is not None + inline_constraints = gm.meta.get("inline_constraints", []) + range_constraints = defaultdict(lambda: ValueRanges(0, int_oo)) | inline_constraints + if not dynamic_shapes: + return dict(range_constraints) + + # clean up dynamic markers from tensors + flat_paths, flat_args = zip(*pytree.tree_flatten_with_path(combined_args)[0]) + for arg in flat_args: + if isinstance(arg, torch.Tensor): + _clean_dynamic_markers(arg) + + # get individual dynamic shapes spec for each input + if not isinstance(dynamic_shapes, dict): + assert isinstance(dynamic_shapes, (tuple, list)) + combined_args = type(dynamic_shapes)(combined_args.values()) # type: ignore[assignment, misc] + flat_dynamic_shapes = _flatten_dynamic_shapes(combined_args, dynamic_shapes) + + # check number of shapes vs. number of inputs + num_placeholders = [node.op == "placeholder" for node in gm.graph.nodes].count(True) + assert len(flat_dynamic_shapes) == num_placeholders - num_lifted_inputs + + free_symbols = set() + range_violations = [] + for input_index, node in enumerate(gm.graph.nodes): + meta_val = node.meta.get("val") + + if ( + input_index < num_lifted_inputs + or node.op != "placeholder" + or meta_val is None + ): + continue + + elif _is_constant_argument(meta_val) or isinstance(meta_val, CustomObjArgument): + continue + + shape_spec = flat_dynamic_shapes[input_index - num_lifted_inputs] + keypath = flat_paths[input_index - num_lifted_inputs] + flat_arg = flat_args[input_index - num_lifted_inputs] + + if isinstance(meta_val, int) or ( + isinstance(meta_val, torch.SymInt) and meta_val.node.expr.is_number + ): + pass + + elif isinstance(meta_val, torch.SymInt): + if shape_spec is not None and isinstance(shape_spec, _DimHint): + hint = flat_arg + range_constraints[meta_val.node.expr] &= shape_env.bound_sympy( + meta_val.node._expr + ) + violation = _constrain_user_specified_dimhint_range( + meta_val, + hint, + shape_spec, + range_constraints, + shape_env, + keypath, + None, + ) + if violation: + range_violations.append(violation) + else: + raise RuntimeError("nyi") + free_symbols.update(meta_val.node.expr.free_symbols) + + elif isinstance(meta_val, torch.Tensor): + for i, d in enumerate(node.meta["val"].shape): + dim = None + if isinstance(shape_spec, (list, tuple)): + dim = shape_spec[i] + elif isinstance(shape_spec, dict): + dim = shape_spec.get(i) + if not is_int(d): + # Compute the range constraint for the symbolic expression corresponding + # to this shape dimension and store it. + if dim is None or isinstance(dim, _DimHint): + range_constraints[d.node.expr] &= shape_env.bound_sympy( + d.node.expr + ) + else: + range_constraints[d.node.expr] &= ValueRanges( + lower=dim.min, upper=dim.max + ) + + free_symbols.update(d.node.expr.free_symbols) + + # check user-specified min/max range for DimHints; + # we might want to do this even if model tracing inferred a static dimension. + if isinstance(dim, _DimHint): + hint = flat_arg.shape[i] + violation = _constrain_user_specified_dimhint_range( + d, hint, dim, range_constraints, shape_env, keypath, i + ) + if violation: + range_violations.append(violation) + else: + raise RuntimeError(f"Unfamiliar meta val: {meta_val}") + + if range_violations: + prefix = "Found the following conflicts between user-specified ranges and inferred ranges from model tracing:\n" + raise ValueError(prefix + "\n".join(range_violations)) + + for symbol in free_symbols: + if symbol not in range_constraints: + # Placeholders can have symbolic shapes that are derived expressions. + # The above code will record direct range constraints for them + # so that we can do runtime assertions. In addition, for serde checks + # we want to record range constraints for their root symbols. + range_constraints[symbol] = shape_env.var_to_range[symbol] + + return dict(range_constraints) + + +def _gather_constant_attrs(m: torch.nn.Module) -> ConstantAttrMap: + """Search the module hierarchy, gathering up all tensor and ScriptObject constants. + + Returns a dictionary mapping hash(value) to the name of the constant. We + have to abuse `hash` here unfortunately, see: [ScriptObject hash]. + """ + constants = ConstantAttrMap() + buffers_parameters = set(m.buffers()) + buffers_parameters.update(m.parameters()) + + def inner(m: torch.nn.Module, prefix_atoms: list[str], constants): + for k, v in m.__dict__.items(): + if isinstance( + v, + ( + torch.Tensor, + torch.ScriptObject, + FakeScriptObject, + ), + ): + if v in buffers_parameters: + # filter out buffers and parameters, leaving only constants + continue + + fqn = ".".join(prefix_atoms + [k]) + constants.add(v, fqn) + for k, v in m.named_children(): + inner(v, prefix_atoms + [k], constants) + + inner(m, [], constants) + return constants + + +def _get_graph_inputs_of_type_nn_module( + args: Optional[tuple[tuple[Any], dict[Any, Any]]], +) -> set[type[torch.nn.Module]]: + if args is None: + return set() + module_types = set() + for arg in pytree.tree_leaves(args): + if isinstance(arg, torch.nn.Module): + module_types.add(type(arg)) + return module_types + + +def _enter_enable_graph_inputs_of_type_nn_module( + module_types: set[type[torch.nn.Module]], +) -> None: + for t in module_types: + torch._export.utils.register_module_as_pytree_input_node(t) + + +def _exit_enable_graph_inputs_of_type_nn_module( + module_types: set[type[torch.nn.Module]], +) -> None: + for t in module_types: + torch._export.utils.deregister_module_as_pytree_input_node(t) + + +@contextlib.contextmanager +def _enable_graph_inputs_of_type_nn_module( + args: Optional[tuple[tuple[Any], dict[Any, Any]]], +): + if args is None: + yield + return + + module_types = _get_graph_inputs_of_type_nn_module(args) + _enter_enable_graph_inputs_of_type_nn_module(module_types) + try: + yield + finally: + _exit_enable_graph_inputs_of_type_nn_module(module_types) + + +@contextlib.contextmanager +def _fakify_module_inputs( + args: tuple[Any], + kwargs: dict[Any, Any], + fake_mode: torch._subclasses.fake_tensor.FakeTensorMode, +): + # This context manager is used to fakify module inputs. + # Inputs: + # args, kwargs: the args and kwargs containing module inputs that haven't been fakified. + # fake_mode: the fake mode to be used for fakifying script objects. It's the same mode that fakify input tensors. + + ctxs = [_enable_graph_inputs_of_type_nn_module((args, kwargs))] + for arg in pytree.tree_leaves((args, kwargs)): + if isinstance(arg, torch.nn.Module): + fake_params_buffers = _fakify_params_buffers(fake_mode, arg) + ctxs.append( + torch.nn.utils.stateless._reparametrize_module( + arg, + fake_params_buffers, + tie_weights=True, + strict=True, + stack_weights=True, + ) + ) + with contextlib.ExitStack() as stack: + for ctx in ctxs: + stack.enter_context(ctx) + yield + + +@contextlib.contextmanager +def _fakify_script_objects( + mod: torch.nn.Module, + args: Sequence[Any], + kwargs: dict[Any, Any], + fake_mode: Optional[torch._subclasses.fake_tensor.FakeTensorMode], +): + # This context manager is used to fakify script objects into FakeScriptObject. + # Inputs: + # mod: the module to be exported, it (and its recursive submodules)'s script object attrs haven't been fakified. + # args, kwargs: the args and kwargs inputs for mod, script object inputs haven't been fakified. + # fake_mode: the fake mode to be used for fakifying script objects. It's the same mode that fakify input tensors. + # + # Returns: + # mod: the patched module, its (and its recursive submodules) script object attrs have been fakified. + # fake_args, fake_kwargs: new fakified args and kwargs. + # Script object inputs have been fakified. Don't touch the tensors. + # fake_constant_attrs: a new map from FakeScriptObject to the fqn of the original script object. + # fake_to_real: a mapping between FakeScriptObject and the original script object in order to un-do the patching. + + constant_attrs: ConstantAttrMap = _gather_constant_attrs(mod) + assert not any( + isinstance(obj, FakeScriptObject) for obj in constant_attrs.values() + ), "Mod shouldn't contain any FakeScriptObject." + assert not pytree.tree_any( + lambda obj: isinstance(obj, FakeScriptObject), (args, kwargs) + ), "args and kwargs shouldn't contain any FakeScriptObject." + + patched_attr = {} + fake_constant_attrs = ConstantAttrMap() + fake_to_real = {} + + def _maybe_fakify_obj(obj): + fake_obj = torch._library.fake_class_registry.maybe_to_fake_obj(fake_mode, obj) + fake_to_real[fake_obj] = obj + return fake_obj + + def _leaf_mod_and_attr( + mod: torch.nn.Module, attr_fqn: str + ) -> tuple[torch.nn.Module, str]: + *prefix_attr, last_attr = attr_fqn.split(".") + cur_mod = mod + for attr in prefix_attr: + cur_mod = getattr(cur_mod, attr) + return cur_mod, last_attr + + try: + for obj, fqns in constant_attrs.items(): + if torch._library.fake_class_registry._is_script_object( + obj + ) or is_opaque_type(obj): + fake_script_obj = _maybe_fakify_obj(obj) + for fqn in fqns: + cur_mod, attr = _leaf_mod_and_attr(mod, fqn) + assert obj is getattr(cur_mod, attr) + setattr(cur_mod, attr, fake_script_obj) + fake_constant_attrs.add(fake_script_obj, fqn) + patched_attr[fqn] = obj + else: + for fqn in fqns: + fake_constant_attrs.add(obj, fqn) + + fake_args, fake_kwargs = pytree.tree_map_only( + torch.ScriptObject, _maybe_fakify_obj, (args, kwargs) + ) + yield (mod, fake_args, fake_kwargs, fake_constant_attrs, fake_to_real) + finally: + for fqn, orig_obj in patched_attr.items(): + cur_mod, attr = _leaf_mod_and_attr(mod, fqn) + setattr(cur_mod, attr, orig_obj) + + +class _NonStrictTorchFunctionHandler(torch.overrides.TorchFunctionMode): + """ + 1. Handles data-dependent errors raised by torch function calls in non-strict. + + Any data-dependent error is due to some condition on unbacked symints + that cannot be resolved. A mechanical way of fixing the error is to use + a torch._check() call to assert either that condition or its negation. + The handler suggests these options as code and points to the location + of the torch function call that raised the error as part of the error + message shown to the user, who can then simply select and copy-paste + a suggested fix at that location. + + NOTE: Not all data-dependent errors are raised by torch function calls. + In particular, conditions on unbacked symints can appear outside such + calls, and as such are not handled here. + + 2. Overrides torch functions that are known to cause problems in non-strict. + + Certain Python features, such as indexing/slicing, cannot be intercepted + in non-strict. Likewise, certain legacy ops, such as distributed collectives, + may need to be mapped to other ops. When there is special handling in Dynamo + for such things, tracing can fail in non-strict (while succeeding in strict). + Fortunately, redirecting to other torch functions can often fix such issues. + + 3. Handles line-of-code logging for each torch function call in non-strict. + + Usage: TORCHEXPORT_EXTENDED_DEBUG_CURRENT_LOC=1 TORCH_LOGS="+export" ... + """ + + def _override(self, func, args, kwargs): + if torch.distributed.is_available(): + from torch.distributed._functional_collectives import ( + REDUCE_OP_TO_STR, + traceable_collective_remaps, + ) + + if func in traceable_collective_remaps: + # Redirect to a corresponding functional collective, following Dynamo. + # See torch/distributed/_functional_collectives.py for details. + # The following is an adaptation of CollectiveFunctionRewriteVariable. + mapped_func = traceable_collective_remaps[func] + signature = inspect.signature(func) + kwargs = dict(signature.bind(*args, **kwargs).arguments) + args = () + if func in ( + torch.distributed.all_reduce, + torch.distributed.reduce_scatter_tensor, + torch.distributed._reduce_scatter_base, + ): + if "op" in kwargs: + kwargs["op"] = REDUCE_OP_TO_STR[kwargs["op"]] + return mapped_func, args, kwargs + if func is torch.tensor: + # Redirect to Python implementation of torch.tensor for data with symints. + # NOTE(avik): We don't unconditionally redirect to this implementation + # because it has some known incompletenesses, e.g., it doesn't support + # empty data. See https://github.com/pytorch/pytorch/issues/143216 + if any( + isinstance(a, (torch.SymInt, torch.SymFloat, torch.SymBool)) + for a in pytree.tree_flatten(args[0])[0] + ): + return torch._refs.tensor, args, kwargs + if func.__name__ == "__getitem__" and isinstance(args[0], torch.Tensor): + + def rewrite(dim, item): + # Redirect to torch.select for indexing. + if item is None: + return dim + 1, (torch.unsqueeze, [dim]) + if isinstance(item, (int, torch.SymInt)): + return dim, (torch.select, [dim, item]) + # Redirect to torch.ops.aten.slice for slicing. + if isinstance(item, slice): + step = item.step or 1 + if item.start is None and item.stop is None and step == 1: + # no-op + return dim + 1, (lambda t: t, []) + return dim + 1, ( + torch.ops.aten.slice, + [dim, item.start, item.stop, step], + ) + # Otherwise do nothing. + + items = list(args[1]) if isinstance(args[1], tuple) else [args[1]] + + has_symint = False + index_ellipsis = None + t = args[0] + n_none_slices = t.ndim + 1 + for i, item in enumerate(items): + if isinstance(item, torch.SymInt) or ( + isinstance(item, slice) + and any( + isinstance(s, torch.SymInt) + for s in (item.start, item.stop, item.step) + ) + ): + has_symint = True + if item is Ellipsis: + index_ellipsis = i + if item is not None: + n_none_slices -= 1 + + # only rewrite when there are symints + if has_symint: + if index_ellipsis is not None: + none_slices = [slice(None)] * n_none_slices + items[index_ellipsis : index_ellipsis + 1] = none_slices + + dim = 0 + # Sequence rewrites. + sequence = [] + for item in items: + if (r := rewrite(dim, item)) is None: + return func, args, kwargs + dim, call_spec = r + sequence.append(call_spec) + + def run(): + # Run sequence. + # pyrefly: ignore [index-error] + t = args[0] + for _method, _args in sequence: + t = _method(t, *_args) + return t + + return run, [], {} + + return func, args, kwargs + + def __torch_function__(self, func, types, args=(), kwargs=None): + kwargs = kwargs or {} + if torch.compiler.is_dynamo_compiling(): + return func(*args, **kwargs) + + if log.isEnabledFor(logging.DEBUG) and config.extended_debug_current_loc: + frame = _find_user_code_frame() + if frame is not None: + log.debug( + "%s called at %s:%s in %s", + func.__qualname__, + frame.f_code.co_filename, + frame.f_lineno, + frame.f_code.co_name, + ) + + func, args, kwargs = self._override(func, args, kwargs) + try: + return func(*args, **kwargs) + except GuardOnDataDependentSymNode as e: + _suggest_fixes_for_data_dependent_error_non_strict(e) + raise diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/pass_base.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/pass_base.py new file mode 100644 index 0000000000000000000000000000000000000000..b2548bb61d69d4cb55c35c38c8507b1c516c4ad7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/pass_base.py @@ -0,0 +1,491 @@ +# mypy: allow-untyped-defs +import operator +import traceback +import typing +from collections.abc import Callable +from contextlib import nullcontext +from typing import Any, Optional, Union + +import torch +from torch import fx +from torch._dispatch.python import enable_python_dispatcher +from torch._export.pass_infra.node_metadata import NodeMetadata +from torch._export.pass_infra.proxy_value import ProxyValue +from torch._higher_order_ops.map import _unstack_pytree +from torch._subclasses import FakeTensor, UnsupportedFakeTensorException +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx import traceback as fx_traceback +from torch.fx.experimental.proxy_tensor import PythonKeyTracer +from torch.fx.experimental.symbolic_shapes import ( + compute_unbacked_bindings, + PropagateUnbackedSymInts, +) +from torch.fx.graph import CodeGen +from torch.fx.passes.infra.pass_base import PassBase, PassResult +from torch.fx.passes.shape_prop import _extract_tensor_metadata, TensorMetadata +from torch.utils import _pytree as pytree + + +__all__ = ["_ExportPassBaseDeprecatedDoNotUse"] + + +Argument = Any +Value = Any +Fn = Callable[..., Any] +PassType = Callable[[torch.fx.GraphModule], Optional[PassResult]] + + +_TORCH_SYM_OPS: set[Callable] = { + torch.sym_int, + torch.sym_float, + torch.sym_ite, + torch.sym_max, + torch.sym_min, + torch.sym_not, + torch.sym_sqrt, +} + + +class ExportPassBaseError(RuntimeError): + pass + + +class _ExportPassBaseDeprecatedDoNotUse(PassBase): + """ + Interpreter-based pass class to help users maintain the IR spec while writing + transformations. + """ + + @staticmethod + def _create_dummy_node_metadata(): + return NodeMetadata({"stack_trace": "".join(traceback.format_stack(limit=1))}) + + class ExportTracer(PythonKeyTracer): + def __init__( + self, callback: "_ExportPassBaseDeprecatedDoNotUse", codegen: CodeGen + ) -> None: + super().__init__() + self.callback = callback + self.root = torch.nn.Module() + self.graph = torch.fx.Graph() + self.graph.set_codegen(codegen) + self.tensor_attrs: dict[str, torch.Tensor] = {} # type: ignore[assignment] + self.fake_tensor_mode: Optional[FakeTensorMode] = None + self.submodules: dict[torch.nn.Module, str] = {} + + def trace(self) -> None: # type: ignore[override] + raise ExportPassBaseError("ExportTracer doesn't support trace().") + + def create_arg(self, a: Argument) -> torch.fx.Node: + if isinstance(a, torch.nn.Module): + if a not in self.submodules: + name_submodule = f"submodule_{len(self.submodules)}" + self.root.add_module(name_submodule, a) + self.submodules[a] = name_submodule + elif isinstance(a, FakeTensor): + if not hasattr(a, "constant") or a.constant is None: + raise ExportPassBaseError(f"Cannot add {a} to graph.") + a = a.constant + node = super().create_arg(a) + if ( + isinstance(a, torch.Tensor) + and isinstance(node, torch.fx.Node) + and node.op == "get_attr" + ): + self.set_metadata(node, a) + self.callback.on_attr(ProxyValue(a, node)) + return node + + def set_metadata( + self, + node: torch.fx.Node, + value: Argument, + ) -> None: + # propagate the fake tensor or sym nodes + def make_val( + x: Argument, + ) -> Union[ + FakeTensor, + torch.SymInt, + torch.SymFloat, + torch.SymBool, + int, + float, + bool, + str, + None, + ]: + if isinstance(x, FakeTensor): + return x + elif isinstance(x, torch.Tensor): + if x.is_quantized: + # TODO (tmanlaibaatar) properly support Quantized FakeTensor + x = torch.dequantize(x) + + try: + assert self.fake_tensor_mode is not None + # TODO we should allocate static shapes + # for param/buffer values + if isinstance(x, torch.nn.Parameter): + fake_tensor = self.fake_tensor_mode.from_tensor( + x, static_shapes=True + ) + else: + fake_tensor = self.fake_tensor_mode.from_tensor(x) + except UnsupportedFakeTensorException: + # TODO: This is just a workaround to get over the + # x.as_subclass error + print( + "Fakeifying a Tensor subclass is not supported \ + right now. Instead a TensorMetadata is used." + ) + fake_tensor = None + return fake_tensor + elif isinstance( + x, + ( + torch.SymInt, + torch.SymFloat, + torch.SymBool, + int, + float, + bool, + str, + ), + ): + return x + else: + return None + + node.meta["val"] = pytree.tree_map(make_val, value) + + # Set the tensor_metadata for values that do not have a corresponding FakeTensor + def make_tensor_meta(x: Argument) -> Optional[TensorMetadata]: + if not isinstance(x, FakeTensor) and isinstance(x, torch.Tensor): + if x.is_quantized: + # TODO (tmanlaibaatar) properly support Quantized FakeTensor + x = torch.dequantize(x) + + try: + assert self.fake_tensor_mode is not None + _ = self.fake_tensor_mode.from_tensor(x) + tensor_meta = None + except UnsupportedFakeTensorException: + # TODO: This is just a workaround to get over the + # x.as_subclass error + tensor_meta = _extract_tensor_metadata(x) + return tensor_meta + else: + return None + + node.meta["tensor_meta"] = pytree.tree_map(make_tensor_meta, value) + + class ExportInterpreter(fx.Interpreter): + def __init__( + self, callback: "_ExportPassBaseDeprecatedDoNotUse", gm: fx.GraphModule + ) -> None: + super().__init__(gm) + self.callback = callback + self.node: torch.fx.Node = next(iter(gm.graph.nodes)) + + # pyrefly: ignore [bad-override] + def placeholder( + self, + target: str, # type: ignore[override] + args: tuple[Argument, ...], + kwargs: dict[str, Argument], + ) -> ProxyValue: + arg = super().placeholder(target, args, kwargs) + return self.callback.placeholder(target, arg, NodeMetadata(self.node.meta)) + + def output( + self, + target: torch.fx.node.Target, + args: tuple[Argument, ...], + kwargs: dict[str, Argument], + ) -> ProxyValue: + return self.callback.output(args[0], NodeMetadata(self.node.meta)).data # type: ignore[return-value] + + def call_function( + self, + target: torch.fx.node.Target, + args: tuple[Argument, ...], + kwargs: dict[str, Argument], + ) -> ProxyValue: + meta = NodeMetadata(self.node.meta) + + if target is operator.getitem: + value, key = args + return self.callback.call_getitem(value, key, meta) + elif getattr(target, "__module__", None) in { + "_operator", + "builtins", + "math", + }: + assert callable(target) + return self.callback.call_sym(target, args, meta) + elif target in _TORCH_SYM_OPS: + assert callable(target) + return self.callback.call_sym(target, args, meta) + elif isinstance( + target, (torch._ops.OpOverload, torch._ops.OpOverloadPacket) + ): + return self.callback.call_operator( + target, + args, + kwargs, + meta, + ) + elif target is torch.ops.higher_order.cond: + pred, true_fn, false_fn, inputs = args + return self.callback.call_cond(pred, true_fn, false_fn, inputs, meta) + elif target is torch.ops.higher_order.map_impl: + f, mapped_args, operands = args # type: ignore[assignment] + return self.callback.call_map(f, mapped_args, operands, meta) + # For other unregistered HigherOrderOps, just interpret them blindly + elif isinstance(target, torch._ops.HigherOrderOperator): + return self.callback._fx( + "call_function", + target, + args, + kwargs, + meta, + ) + else: + raise ExportPassBaseError(f"Unsupported target type: {target}") + + def get_attr( # type: ignore[override] + self, + target: str, + args: tuple[Argument, ...], + kwargs: dict[str, Argument], + ) -> Argument: + return super().get_attr(target, args, kwargs) + + def call_module( + self, + target: torch.fx.node.Target, + args: tuple[Argument, ...], + kwargs: dict[str, Argument], + ) -> None: + raise ExportPassBaseError("call_module is not supported.") + + def call_method( # type: ignore[override] + self, + target: str, + args: tuple[Argument, ...], + kwargs: dict[str, Argument], + ) -> None: + raise ExportPassBaseError("call_method is not supported.") + + def run_node(self, n: torch.fx.Node) -> Argument: + self.node = n + self.callback.node_debug_str = n.format_node() + return super().run_node(n) + + def __init__(self) -> None: + self.interpreter = PropagateUnbackedSymInts( + torch.fx.GraphModule(torch.nn.Module(), torch.fx.Graph()) + ) + self.tracer = self.ExportTracer(self, CodeGen()) + self.fake_tensor_mode: Optional[FakeTensorMode] = None + self._initialized = True + self.node_debug_str: typing.Optional[str] = None + + def _fx( + self, + kind: str, + target: torch.fx.node.Target, + args: tuple[Argument, ...], + kwargs: dict[str, Argument], + meta: NodeMetadata, + ) -> ProxyValue: + args_data, kwargs_data = pytree.tree_map_only( + ProxyValue, lambda x: x.data, (args, kwargs) + ) + res_data = getattr(self.interpreter, kind)(target, args_data, kwargs_data) + args_proxy, kwargs_proxy = pytree.tree_map_only( + ProxyValue, lambda x: x.proxy, (args, kwargs) + ) + + name = None + if isinstance(target, torch._ops.OpOverload): + name = self.tracer.graph._target_to_str(target.overloadpacket.__name__) + + res_proxy = self.tracer.create_proxy( + kind, target, args_proxy, kwargs_proxy, name=name + ) + res_proxy.node.meta.update(meta.data) + if self.fake_tensor_mode and (shape_env := self.fake_tensor_mode.shape_env): + if symbol_to_path := compute_unbacked_bindings(shape_env, res_data): + res_proxy.node.meta["unbacked_bindings"] = symbol_to_path + self.tracer.set_metadata(res_proxy.node, res_data) + return ProxyValue(res_data, res_proxy) + + def inputs(self, graph_module: torch.fx.GraphModule) -> list[Argument]: + # TODO(angelayi): Update this with what we decide to do for metadata in + # the exported graph module + if (args := graph_module.meta.get("args", None)) is not None: + return list(args) + + def extract_input(node: torch.fx.Node) -> Optional[FakeTensor]: + if "val" in node.meta: + fake = node.meta["val"] + if hasattr(fake, "constant") and fake.constant is not None: + return fake.constant + return fake + elif tensor_meta := node.meta.get("tensor_meta"): + assert self.fake_tensor_mode is not None + return FakeTensor( + self.fake_tensor_mode, + torch.empty( + tensor_meta.shape, + dtype=tensor_meta.dtype, + device="meta", + requires_grad=tensor_meta.requires_grad, + memory_format=tensor_meta.memory_format, + ), + torch.device("cpu"), + ) + elif len(node.users) == 0: + return None + raise ExportPassBaseError( + f"Cannot construct an input for graph module: {graph_module}.", + ) + + return [ + extract_input(node) + for node in graph_module.graph.nodes + if node.op == "placeholder" + ] + + def on_attr(self, attr: ProxyValue) -> None: + pass + + def placeholder(self, name: str, arg: Argument, meta: NodeMetadata) -> ProxyValue: + arg_proxy = self.tracer.create_proxy("placeholder", name, (), {}) + arg_proxy.node.meta = meta.data + self.tracer.set_metadata(arg_proxy.node, arg) + return ProxyValue(arg, arg_proxy) + + def call_operator( + self, + op, + args: tuple[Argument, ...], + kwargs: dict[str, Argument], + meta: NodeMetadata, + ) -> ProxyValue: + return self._fx("call_function", op, args, kwargs, meta) + + def call_sym( + self, + target: Fn, + args: tuple[Argument, ...], + meta: NodeMetadata, + ) -> ProxyValue: + return self._fx("call_function", target, args, {}, meta) + + def call_cond( + self, + pred: ProxyValue, + true_fn: torch.fx.GraphModule, + false_fn: torch.fx.GraphModule, + inputs: list[Argument], + meta: NodeMetadata, + ) -> ProxyValue: + true_branch = self.call_submodule(true_fn, tuple(inputs)) + false_branch = self.call_submodule(false_fn, tuple(inputs)) + assert true_branch is not None + assert false_branch is not None + return self._fx( + "call_function", + torch.ops.higher_order.cond, + (pred, true_branch.graph_module, false_branch.graph_module, list(inputs)), + {}, + meta, + ) + + def call_map( + self, + f: torch.fx.GraphModule, + mapped_args: list[ProxyValue], + operands: list[ProxyValue], + meta: NodeMetadata, + ) -> ProxyValue: + xs = _unstack_pytree([arg.data for arg in mapped_args])[0] + f_branch = self.call_submodule(f, tuple(xs + [arg.data for arg in operands])) + assert f_branch is not None + return self._fx( + "call_function", + torch.ops.higher_order.map_impl, + (f_branch.graph_module, mapped_args, operands), + {}, + meta, + ) + + def call_getitem( + self, value: ProxyValue, key: int, meta: NodeMetadata + ) -> ProxyValue: + return self._fx("call_function", operator.getitem, (value, key), {}, meta) + + def output(self, results: list[Argument], meta: NodeMetadata) -> ProxyValue: + return self._fx("output", "output", (results,), {}, meta) + + def call_submodule( + self, graph_module: fx.GraphModule, inputs: tuple[Argument, ...] + ) -> PassResult: + prev_tracer, self.tracer = ( + self.tracer, + self.ExportTracer(self, graph_module.graph._codegen), + ) + self.tracer.fake_tensor_mode = prev_tracer.fake_tensor_mode + interpreter = self.ExportInterpreter(self, graph_module) + # pyrefly: ignore [bad-assignment] + prev_interpreter, self.interpreter = ( + self.interpreter, + torch.fx.Interpreter( # type: ignore[assignment] + torch.fx.GraphModule(torch.nn.Module(), torch.fx.Graph()) + ), + ) + inputs_data = pytree.tree_map_only(ProxyValue, lambda x: x.data, inputs) + with fx_traceback.preserve_node_meta(): + interpreter.run(*inputs_data) + + new_graph_module = torch.fx.GraphModule(self.tracer.root, self.tracer.graph) + + self.tracer = prev_tracer + self.interpreter = prev_interpreter + return PassResult( + new_graph_module, + True, + ) + + def call(self, graph_module: fx.GraphModule) -> PassResult: + if not getattr(self, "_initialized", False): + raise ExportPassBaseError( + "ExportPass is not initialized with __init__().", + ) + + inputs = self.inputs(graph_module) + + fake_tensor_mode = None + for i in inputs: + if isinstance(i, FakeTensor): + assert fake_tensor_mode is None or fake_tensor_mode is i.fake_mode, ( + "Multiple fake tensor mode detected." + ) + fake_tensor_mode = i.fake_mode + if fake_tensor_mode is None: + self.tracer.fake_tensor_mode = FakeTensorMode(allow_non_fake_inputs=True) + fake_tensor_mode = nullcontext() # type: ignore[assignment] + dispatcher_mode = nullcontext() # type: ignore[assignment] + else: + fake_tensor_mode.allow_non_fake_inputs = True + self.tracer.fake_tensor_mode = fake_tensor_mode + dispatcher_mode = enable_python_dispatcher() # type: ignore[assignment] + self.fake_tensor_mode = self.tracer.fake_tensor_mode + + with fake_tensor_mode, dispatcher_mode: # type: ignore[assignment, union-attr] + result = self.call_submodule(graph_module, tuple(inputs)) + + return result diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/tools.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/tools.py new file mode 100644 index 0000000000000000000000000000000000000000..b254fd62e3b2dc379e7d443d12f35f6fa61a5ee3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/tools.py @@ -0,0 +1,148 @@ +# mypy: allow-untyped-defs +import logging +import warnings +from collections.abc import Iterable +from typing import Any, Optional + +import torch +import torch.export +import torch.export._trace +from torch._utils_internal import log_export_usage + + +log = logging.getLogger(__name__) + +__all__ = ["report_exportability"] + + +def _generate_inputs_for_submodules( + model: torch.nn.Module, + target_submodules: Iterable[str], + args: tuple[Any, ...], + kwargs: Optional[dict[str, Any]] = None, +) -> dict[str, tuple[Any, Any]]: + """ + Generate inputs for targeting submdoules in the given model. Note that if two submodules refer to the same obj, this + function doesn't work. + + Args: + model: root model. + inputs: inputs to the root model. + target_submodules: submodules that we want to generate inputs for. + + Returns: + A dict that maps from submodule name to its inputs. + """ + kwargs = kwargs or {} + + handles = [] + results = {} + submodule_to_names = {mod: name for name, mod in model.named_modules()} + + def pre_forward(module, module_args, module_kwargs): + results[submodule_to_names[module]] = (module_args, module_kwargs) + + try: + for name, mod in model.named_modules(): + if name in target_submodules: + handles.append( + mod.register_forward_pre_hook(pre_forward, with_kwargs=True) + ) + model(*args, **kwargs) + except Exception as e: + warnings.warn( + f"Failed to generate submodule inputs because of the following error:\n{e}", + stacklevel=2, + ) + finally: + for h in handles: + h.remove() + return results + + +def report_exportability( + mod: torch.nn.Module, + args: tuple[Any, ...], + kwargs: Optional[dict[str, Any]] = None, + *, + strict: bool = True, + pre_dispatch: bool = False, +) -> dict[str, Optional[Exception]]: + """ + Report exportability issues for a module in one-shot. + + Args: + mod: root module. + args: args to the root module. + kwargs: kwargs to the root module. + Returns: + A dict that maps from submodule name to the exception that was raised when trying to export it. + `None` means the module is exportable without issue. + Sample output: + { + '': UnsupportedOperatorException(func=), + 'submod_1': UnsupportedOperatorException(func=), + 'submod_2': None + } + """ + + log_export_usage(event="export.report_exportability") + + kwargs = kwargs or {} + + all_submod_names = [name for name, _ in mod.named_modules() if name != ""] + submod_inputs = _generate_inputs_for_submodules(mod, all_submod_names, args, kwargs) + + tried_module_types = set() + report: dict[str, Optional[Exception]] = {} + + def try_export(module, module_name, args, kwargs): + nonlocal submod_inputs, report, strict, pre_dispatch, tried_module_types + + if type(module) in tried_module_types: + return + tried_module_types.add(type(module)) + + if args is not None or kwargs is not None: + try: + torch.export._trace._export( + module, + args, + kwargs, + strict=strict, + pre_dispatch=pre_dispatch, + ) + report[module_name] = None + log.info("Successfully exported `%s`", module_name) + return + except Exception as e: + short_msg = repr(e).split("\n")[0] + log.warning( + "Failed exporting `%s` with exception: %s", module_name, short_msg + ) + report[module_name] = e + + for name, submod in module.named_children(): + sub_module_name = name if module_name == "" else f"{module_name}.{name}" + + submod_args, submod_kwargs = submod_inputs.get( + sub_module_name, (None, None) + ) + + try_export(submod, sub_module_name, submod_args, submod_kwargs) + + return + + try_export(mod, "", args, kwargs) + + unique_issues = set() + for exception in report.values(): + if exception is not None: + key = repr(exception).split("\\n")[0] + unique_issues.add(key) + + log.warning("Found %d export issues:", len(unique_issues)) + for issue in unique_issues: + log.warning(issue) + + return report diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..50a921a936d7dd6e9c7622c54b142094aa6b078e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/utils.py @@ -0,0 +1,1613 @@ +# mypy: allow-untyped-defs +import ast +import copy +import dataclasses +import functools +import inspect +import json +import math +import operator +import re +from collections import defaultdict +from collections.abc import Callable, Iterable +from contextlib import contextmanager +from inspect import ismethod, Parameter +from typing import Any, Optional, TYPE_CHECKING, Union + +import torch +from torch._guards import detect_fake_mode +from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode +from torch._subclasses.functional_tensor import FunctionalTensor +from torch.fx._utils import first_call_function_nn_module_stack +from torch.fx.experimental.proxy_tensor import PreDispatchTorchFunctionMode +from torch.fx.passes.runtime_assert import insert_deferred_runtime_asserts + + +if TYPE_CHECKING: + import sympy + + from torch._export.passes.lift_constants_pass import ConstantAttrMap + from torch._ops import OperatorBase + from torch.export import ExportedProgram + from torch.export.graph_signature import ExportGraphSignature + +from torch.export.graph_signature import CustomObjArgument, InputKind, OutputKind +from torch.fx._pytree import ( + _deregister_pytree_flatten_spec, + register_pytree_flatten_spec, +) +from torch.utils._pytree import ( + _deregister_pytree_node, + _register_pytree_node, + Context, + FlattenFunc, + FromDumpableContextFn, + GetAttrKey, + KeyPath, + keystr, + MappingKey, + SequenceKey, + ToDumpableContextFn, + tree_flatten_with_path, + UnflattenFunc, +) + + +placeholder_prefixes = { + InputKind.USER_INPUT: "", + InputKind.PARAMETER: "p_", + InputKind.BUFFER: "b_", + InputKind.CONSTANT_TENSOR: "c_", + InputKind.CUSTOM_OBJ: "obj_", + InputKind.TOKEN: "token", +} + +_DISABLE_ATEN_TO_ASSERTION_PASS = False + + +def _collect_and_set_constant_attrs( + graph_signature, constants, mod +) -> "ConstantAttrMap": + # the exported module will store constants & non-persistent buffers such that + # retracing treats them as persistent buffers, so we inform the constants lifting pass + # and overwrite the new graph signature using the previous program. This is intended to only be used + # in run_decompositions where we still have access to original EP. + from torch._export.passes.lift_constants_pass import ConstantAttrMap + + constant_attrs = ConstantAttrMap() + non_persistent_buffers = { + spec.target + for spec in graph_signature.input_specs + if spec.kind == InputKind.BUFFER and not spec.persistent + } + for name, value in constants.items(): + if name in non_persistent_buffers: + continue + # recursive getattr + _mod = mod + *atoms, attr = name.split(".") + for atom in atoms: + _mod = getattr(_mod, atom) + # remove as buffer, reassign as constant/non-persistent buffer + _mod._buffers.pop(attr, None) + setattr(_mod, attr, value) + constant_attrs.add(value, name) + return constant_attrs + + +def _register_constants_as_buffers( + mod: torch.fx.GraphModule, state_dict, non_persistent_buffers +): + # TODO some annoying circular dependency issue + from torch.export.unflatten import _assign_attr, _AttrKind + + temp_registered_constants = set() + + for node in mod.graph.nodes: + if node.op == "get_attr": + target = torch.fx.graph_module._get_attr(mod, node.target) + if isinstance(target, torch.Tensor): + # Make sure we also check if the original buffer is + # non persistent as well. + if (node.target not in state_dict) and ( + node.target not in non_persistent_buffers + ): + torch.fx.graph_module._del_attr(mod, node.target) + _assign_attr(target, mod, node.target, _AttrKind.BUFFER, False) + temp_registered_constants.add(node.target) + + mod.recompile() + + return temp_registered_constants + + +def _override_graph_signature_for_temp_registered_constants( + sig: "ExportGraphSignature", temp_registered_constants +): + for spec in sig.input_specs: + if spec.target in temp_registered_constants: + spec.kind = InputKind.CONSTANT_TENSOR + spec.persistent = None + + for spec in sig.output_specs: + if ( + spec.kind == OutputKind.BUFFER_MUTATION + and spec.target in temp_registered_constants + ): + raise RuntimeError( + f"Constant {spec.target} is mutated in the forward method. Pls register it as buffer" + ) + + return sig + + +def _overwrite_signature_for_non_persistent_buffers( + old_sig: "ExportGraphSignature", new_sig: "ExportGraphSignature" +): + # overwrite signature for non-persistent buffers + non_persistent_buffers = { + spec.target + for spec in old_sig.input_specs + if spec.kind == InputKind.BUFFER and not spec.persistent + } + + for spec in new_sig.input_specs: + if spec.kind == InputKind.BUFFER and spec.target in non_persistent_buffers: + spec.persistent = False + return new_sig + + +def _collect_param_buffer_metadata(mod: torch.fx.GraphModule) -> dict[str, Any]: + """ + Param/buffer metadata needs to be saved before lowering to aten IR + because aten IR lifts them, as a result, automatic preservation doesn't work. + This is intended to be called on the strict mode tracing right before lowering to + aten IR OR run_decomposition pass. + """ + params_buffers_to_node_meta = {} + + def _getattr(model: torch.fx.GraphModule, attr_name: str): + *prefix, field = attr_name.split(".") + t = model + for item in prefix: + t = getattr(t, item, None) # type: ignore[assignment] + assert t is not None + + return getattr(t, field) + + for node in mod.graph.nodes: + target = node.target + meta = node.meta + if node.op == "call_module": + submodule = _getattr(mod, target) + if isinstance(submodule, torch.nn.Module): + for name, _ in submodule.named_parameters( + recurse=True, remove_duplicate=False + ): + params_buffers_to_node_meta[target + "." + name] = meta + + for name, _ in submodule.named_buffers( + recurse=True, remove_duplicate=False + ): + params_buffers_to_node_meta[target + "." + name] = meta + + if node.op == "get_attr": + submodule = _getattr(mod, target) + if not isinstance(submodule, torch.fx.GraphModule): + params_buffers_to_node_meta[target] = meta + + # If the call_function uses param as input, we also need to update params' meta + # with this call_function node's meta. + # This is basically the same flow as torch.fx.traceback.preserve_meta() + if node.op == "call_function" and not isinstance( + node.target, torch._ops.HigherOrderOperator + ): + for arg in node._input_nodes: + if arg.op == "get_attr": + for entry in torch.fx.proxy._COPY_META_FIELDS: + # the custom field should not be copied + if entry == "custom": + continue + if entry in meta: + params_buffers_to_node_meta[arg.target][entry] = meta[entry] + + return params_buffers_to_node_meta + + +def _maybe_find_pre_dispatch_tf_mode_for_export(): + if not torch._C._is_torch_function_mode_enabled(): + return None + + torch_function_mode_stack = torch.overrides._get_current_function_mode_stack() + + pre_dispatch_tf_modes = [ + mode + for mode in torch_function_mode_stack + if isinstance(mode, PreDispatchTorchFunctionMode) + ] + + assert len(pre_dispatch_tf_modes) <= 1, ( + f"Expected only one PreDispatchTorchFunctionMode, found {len(pre_dispatch_tf_modes)}" + ) + + if len(pre_dispatch_tf_modes) == 0: + return None + + mode = pre_dispatch_tf_modes[0] + return mode + + +def _populate_param_buffer_metadata_to_new_gm( + params_buffers_to_node_meta: dict[str, Any], + gm: torch.fx.GraphModule, + new_sig: "ExportGraphSignature", +) -> None: + """ + Given that we collected param'buffer metadata before, we put them back in + newly traced graph module + """ + # Don't copy over nn_module_stack, stack_trace metadata for params/buffers nodes + for metadata in params_buffers_to_node_meta.values(): + metadata.pop("nn_module_stack", None) + metadata.pop("stack_trace", None) + + for node in gm.graph.nodes: + if node.op == "placeholder": + if node.target in new_sig.inputs_to_parameters: + param_name = new_sig.inputs_to_parameters[node.target] + if param_name in params_buffers_to_node_meta: + for k, v in params_buffers_to_node_meta[param_name].items(): + node.meta[k] = v + if node.target in new_sig.inputs_to_buffers: + buffer_name = new_sig.inputs_to_buffers[node.target] + if buffer_name in params_buffers_to_node_meta: + for k, v in params_buffers_to_node_meta[buffer_name].items(): + node.meta[k] = v + + +def _get_shape_env_from_gm(gm: torch.fx.GraphModule): + vals = [ + node.meta["val"] + for node in gm.graph.nodes + if node.meta.get("val", None) is not None + ] + + fake_mode = _detect_fake_mode_from_gm(gm) + if fake_mode is not None: + return fake_mode.shape_env + for v in vals: + if isinstance(v, torch.SymInt): + return v.node.shape_env + + +def _rename_without_collisions( + name_map: dict[str, str], + find_available: dict[str, int], + used_names: set[str], + orig_name: str, + name: str, + is_placeholder: bool = False, +): + """ + Renames nodes to avoid name collisions, with suffixing. + name_map: map from original name to new name + find_available: map prefix to available suffix + used_names: cache of used names + orig_name: mapping key + name: candidate name (potentially suffixed, e.g. mul_2) + is_placeholder: if the node is a placeholder, avoid detecting suffix + """ + match = re.match(r"(.*)_(\d+)", name) + key = name + + if match and not is_placeholder: + prefix, n = match.group(1), match.group(2) + key = prefix + + new_name = name + if new_name in used_names: + new_name = f"{key}_{find_available[key] + 1}" + + match = re.match(r"(.*)_(\d+)", new_name) + if match: + prefix, n = match.group(1), match.group(2) + if int(n) > find_available[prefix]: + find_available[prefix] = int(n) + + name_map[orig_name] = new_name + used_names.add(new_name) + + return name_map[orig_name] + + +def get_keystr(key_path: KeyPath) -> str: + """For a given index into the flat_args, return a human readable string + describing how to access it, e.g. "*args["foo"][0].bar" + """ + # Prefix the keypath with "*args" or "**kwargs" to make it clearer where + # the arguments come from. Ultimately we ought to serialize the + # original arg names for the best error message here. + args_kwargs_key_path = key_path[0] + assert isinstance(args_kwargs_key_path, SequenceKey) + if args_kwargs_key_path.idx == 0: + return f"*args{keystr(key_path[1:])}" + else: + kwarg_key = key_path[1] + assert isinstance(kwarg_key, (GetAttrKey, MappingKey)) + name = str(kwarg_key)[1:-1] # get rid of the enclosed [] + return f"{name}{keystr(key_path[2:])}" + + +def _check_symint( + symint: Union[int, torch.SymInt], + arg: int, + range_constraints, + unification_map, + keypath: KeyPath, + i: Optional[int] = None, +) -> None: + from torch.export.dynamic_shapes import _IntWrapper + + if ( + isinstance(arg, torch.SymInt) + and not arg.node.expr.is_number + or isinstance(arg, _IntWrapper) + ): + # This can happen when, say, arg is a fake tensor. + # We do not run checks on symbolic shapes of fake inputs as + # such checks can affect the shape env. + return + + import sympy + + from torch._export.passes.add_runtime_assertions_for_constraints_pass import ( + _convert_range_to_int, + ) + from torch.utils._sympy.solve import try_solve + + if isinstance(symint, torch.SymInt) and len(symint.node.expr.free_symbols) == 1: + symbol = next(iter(symint.node.expr.free_symbols)) + if symbol in unification_map: + existing_dim = symint.node.expr.subs(unification_map) + if arg != existing_dim: + path = get_keystr(keypath) + if i is not None: + path += f".shape[{i}]" + raise RuntimeError( + f"Expected input at {path} to be equal to {existing_dim}, but got {arg}", + ) + else: + if isinstance(symint.node.expr, sympy.Symbol): + # Short cut for try_solve below. Also useful in cases where + # sympy.Eq(symint.node.expr, arg) would evaluate to False + # purely because symbol is constrained to be size-like, + # e.g., when symint.node.expr = symbol and arg = 0. + unification_map[symbol] = int(arg) + else: + solution = try_solve(sympy.Eq(symint.node.expr, arg), symbol) + if solution is None: + path = get_keystr(keypath) + if i is not None: + path += f".shape[{i}]" + raise RuntimeError( # noqa: B904 + f"Expected input {path} = {arg} to be " + f"of the form {symint.node.expr}, where {symbol} is an integer" + ) + else: + unification_map[symbol] = int(solution[1]) + + if symint.node.expr in range_constraints: + min_val, max_val = _convert_range_to_int( + range_constraints[symint.node.expr] + ) + # NOTE: we allow dimensions to be 0/1 at runtime + if min_val > 2: + if arg < min_val: + path = get_keystr(keypath) + if i is not None: + path += f".shape[{i}]" + raise RuntimeError( + f"Expected input at {path} to be >= {min_val}, but got {arg}", + ) + if max_val < math.inf: + if arg > max_val: + path = get_keystr(keypath) + if i is not None: + path += f".shape[{i}]" + raise RuntimeError( + f"Expected input at {path} to be <= {max_val}, but got {arg}", + ) + elif isinstance(symint, torch.SymInt) and not symint.node.expr.is_number: + # this means we deferred a guard from export analysis to runtime, let this pass + # we'll add a runtime assert checking equality to this replacement expression + pass + elif arg != int(symint): + path = get_keystr(keypath) + if i is not None: + path += f".shape[{i}]" + raise RuntimeError( + f"Expected input at {path} to be equal to {symint}, but got {arg}. " + "If you meant for this dimension to be dynamic, please re-export and specify dynamic_shapes " + "(e.g. with Dim.DYNAMIC)" + ) + + +def _check_input_constraints_for_graph( + input_placeholders: list[torch.fx.Node], flat_args_with_path, range_constraints +) -> None: + if len(flat_args_with_path) != len(input_placeholders): + raise RuntimeError( + "Unexpected number of inputs " + f"(expected {len(input_placeholders)}, got {len(flat_args_with_path)})" + ) + # NOTE: export already guarantees that the same symbol is used in metadata + # for all InputDims related by equality constraints, so we can just unify + # symbols with given input dimension values to check equality constraints. + unification_map: dict[sympy.Symbol, Any] = {} + for (key_path, arg), node in zip(flat_args_with_path, input_placeholders): + node_val = node.meta.get("val") + if isinstance(node_val, FakeTensor): + if not isinstance(arg, torch.Tensor): + raise RuntimeError( + f"Expected input at {get_keystr(key_path)} to be a tensor, but got {type(arg)}", + ) + + if len(node_val.shape) != len(arg.shape): + raise RuntimeError( + f"Unexpected number of dimensions in input at {get_keystr(key_path)}.shape " + f"(expected {node_val.shape}, got {arg.shape})" + ) + + for j, (arg_dim, node_dim) in enumerate(zip(arg.shape, node_val.shape)): + _check_symint( + node_dim, arg_dim, range_constraints, unification_map, key_path, j + ) + + elif isinstance(node_val, (int, float, str)): + if type(arg) is not type(node_val) or arg != node_val: + raise RuntimeError( + f"Expected input at {get_keystr(key_path)} to be equal to {node_val}, but got {arg}", + ) + elif isinstance(node_val, torch.SymInt): + _check_symint( + node_val, + arg, + range_constraints, + unification_map, + key_path, + None, + ) + + +def register_dataclass_as_pytree_node( + cls: type[Any], + flatten_fn: Optional[FlattenFunc] = None, + unflatten_fn: Optional[UnflattenFunc] = None, + *, + serialized_type_name: Optional[str] = None, + to_dumpable_context: Optional[ToDumpableContextFn] = None, + from_dumpable_context: Optional[FromDumpableContextFn] = None, + return_none_fields: bool = False, +) -> None: + assert dataclasses.is_dataclass(cls), ( + f"Only dataclasses can be registered with this function: {cls}" + ) + + @torch._dynamo.dont_skip_tracing + def default_flatten_fn(obj: Any) -> tuple[list[Any], Context]: + flattened = [] + flat_names = [] + none_names = [] + for f in dataclasses.fields(obj): + name, val = f.name, getattr(obj, f.name) + if val is not None or return_none_fields: + flattened.append(val) + flat_names.append(name) + else: + none_names.append(name) + return flattened, [flat_names, none_names] + + @torch._dynamo.dont_skip_tracing + def default_unflatten_fn(values: Iterable[Any], context: Context) -> Any: + flat_names, none_names = context + return cls(**dict(zip(flat_names, values)), **dict.fromkeys(none_names)) + + @torch._dynamo.dont_skip_tracing + def default_flatten_fn_with_keys(obj: Any) -> tuple[list[Any], Context]: + flattened, (flat_names, _none_names) = flatten_fn(obj) # type: ignore[misc] + return [(MappingKey(k), v) for k, v in zip(flat_names, flattened)], flat_names + + flatten_fn = flatten_fn if flatten_fn is not None else default_flatten_fn + unflatten_fn = unflatten_fn if unflatten_fn is not None else default_unflatten_fn + + if (to_dumpable_context is None) ^ (from_dumpable_context is None): + raise ValueError( + f"Both to_dumpable_context and from_dumpable_context for {cls} must " + "be None or registered." + ) + + _register_pytree_node( + cls, + flatten_fn, + unflatten_fn, + serialized_type_name=serialized_type_name, + flatten_with_keys_fn=default_flatten_fn_with_keys, + to_dumpable_context=to_dumpable_context, + from_dumpable_context=from_dumpable_context, + ) + + +def is_param(program: "ExportedProgram", node: torch.fx.Node) -> bool: + """ + Checks if the given node is a parameter within the exported program + """ + + return node.name in program.graph_signature.inputs_to_parameters + + +def get_param( + program: "ExportedProgram", + node: torch.fx.Node, +) -> Optional[torch.nn.Parameter]: + """ + Returns the parameter associated with the given node in the exported program. + Returns None if the node is not a parameter within the exported program + """ + + if is_param(program, node): + parameter_name = program.graph_signature.inputs_to_parameters[node.name] + return program.state_dict[parameter_name] + + return None + + +def is_buffer(program: "ExportedProgram", node: torch.fx.Node) -> bool: + """ + Checks if the given node is a buffer within the exported program + """ + + return node.name in program.graph_signature.inputs_to_buffers + + +def get_buffer( + program: "ExportedProgram", + node: torch.fx.Node, +) -> Optional[torch.Tensor]: + """ + Returns the buffer associated with the given node in the exported program. + Returns None if the node is not a buffer within the exported program + """ + + if is_buffer(program, node): + buffer_name = program.graph_signature.inputs_to_buffers[node.name] + if buffer_name in program.graph_signature.non_persistent_buffers: + return program.constants[buffer_name] + else: + return program.state_dict[buffer_name] + + return None + + +def is_lifted_tensor_constant( + program: "ExportedProgram", + node: torch.fx.Node, +) -> bool: + """ + Checks if the given node is a lifted tensor constant within the exported program + """ + + return node.name in program.graph_signature.inputs_to_lifted_tensor_constants + + +def get_lifted_tensor_constant( + program: "ExportedProgram", + node: torch.fx.Node, +) -> Optional[torch.Tensor]: + """ + Returns the lifted tensor constant associated with the given node in the exported program. + Returns None if the node is not a lifted tensor constant within the exported program + """ + + if is_lifted_tensor_constant(program, node): + lifted_tensor_name = program.graph_signature.inputs_to_lifted_tensor_constants[ + node.name + ] + return program.constants[lifted_tensor_name] + + return None + + +def sequential_split( + gm: torch.fx.GraphModule, + node_call_back: Callable[[torch.fx.Node], Union[torch.fx.Node, bool]], +) -> torch.fx.GraphModule: + """ + sequential_split creates a new graph module that splits the input graph module into multiple submodules + based on the node_call_back. It doesn't mutate the input graph module. The node_call_back should return + True if the node is a delimiter. Delimiter will be the first node in the next submodule. + """ + from torch.fx.passes.split_module import split_module + + split_map = {} + split_id = 0 + for node in gm.graph.nodes: + if node_call_back(node): + split_id += 1 + split_map[node] = split_id + + new_gm = split_module( + gm, + gm, + lambda node: split_map[node], + keep_original_order=True, + keep_original_node_name=True, + ) + # Keep the codegen from original graph module to preserve e.g. pytree info. + new_gm.graph._codegen = gm.graph._codegen + new_gm.recompile() + return new_gm + + +def nodes_filter(nodes: list[torch.fx.Node], node_call_back) -> list[torch.fx.Node]: + """Returns the nodes that match the node_call_back as a list.""" + return [node for node in nodes if node_call_back(node)] + + +@contextmanager +def _disable_aten_to_metadata_assertions(): + global _DISABLE_ATEN_TO_ASSERTION_PASS + orig_val = _DISABLE_ATEN_TO_ASSERTION_PASS + _DISABLE_ATEN_TO_ASSERTION_PASS = True + try: + yield + finally: + _DISABLE_ATEN_TO_ASSERTION_PASS = orig_val + + +def _insert_aten_to_metadata_assert_pass(gm: torch.fx.GraphModule) -> None: + from torch._export.passes._node_metadata_hook import ( + _node_metadata_hook, + _set_node_metadata_hook, + ) + + if _DISABLE_ATEN_TO_ASSERTION_PASS: + return + + aten_to_variants = [ + torch.ops.aten.to.device, + torch.ops.aten.to.dtype, + torch.ops.aten.to.dtype_layout, + ] + for node in gm.graph.nodes: + if node.target in aten_to_variants: + if ( + node.prev.target is torch.ops.aten._assert_tensor_metadata.default + and node.args[0] == node.prev.args[0] + ): + # skip if already guarded + continue + + if (tensor_val := node.args[0].meta.get("val")) is not None: + with ( + gm.graph.inserting_before(node), + _set_node_metadata_hook( + gm, + functools.partial( + _node_metadata_hook, + metadata={ + "stack_trace": node.meta.get("stack_trace"), + "nn_module_stack": node.meta.get("nn_module_stack"), + }, + ), + ), + ): + gm.graph.call_function( + torch.ops.aten._assert_tensor_metadata.default, + args=(node.args[0],), + kwargs={ + "dtype": tensor_val.dtype, + "device": tensor_val.device, + "layout": tensor_val.layout, + }, + ) + + +def apply_runtime_assertion_pass(gm: torch.fx.GraphModule, graph_signature): + from torch._export.passes._node_metadata_hook import ( + _node_metadata_hook, + _set_node_metadata_hook, + ) + from torch._functorch._aot_autograd.input_output_analysis import _graph_output_names + + if not torch._dynamo.config.do_not_emit_runtime_asserts: + stack_trace = ( + 'File "torch/fx/passes/runtime_assert.py", line 24, ' + "in insert_deferred_runtime_asserts" + ) + with _set_node_metadata_hook( + gm, + functools.partial( + _node_metadata_hook, metadata={"stack_trace": stack_trace} + ), + ): + shape_env = _get_shape_env_from_gm(gm) + if shape_env: + insert_deferred_runtime_asserts( + gm, + shape_env, + f"exported program: {first_call_function_nn_module_stack(gm.graph)}", + export=True, + ) + + # insert runtime assertions for aten.to nodes + _insert_aten_to_metadata_assert_pass(gm) + + # update output specs + gm.recompile() + graph_signature.user_outputs = _graph_output_names(gm) + return gm, graph_signature + + +def nodes_first( + nodes: list[torch.fx.Node], node_call_back=None +) -> Optional[torch.fx.Node]: + """ + Returns the first node that matches the node_call_back. If no node matches, returns None. + When node_call_back is None, returns the first node in the node list. + """ + ret = nodes_filter(nodes, node_call_back if node_call_back else lambda node: True) + if len(ret) > 0: + return ret[0] + return None + + +def nodes_count(nodes: list[torch.fx.Node], node_call_back) -> int: + """Returns the number of nodes that match the node_call_back.""" + return len(nodes_filter(nodes, node_call_back)) + + +def nodes_map(nodes: list[torch.fx.Node], node_call_back) -> list[torch.fx.Node]: + """ + Sequentially visit the nodes list and invoke node_call_back on each element. + Returns the nodes list after the node_call_back is invoked on each element. + """ + for node in nodes: + node_call_back(node) + return nodes + + +def node_replace_(old_node: torch.fx.Node, new_node: torch.fx.Node) -> None: + """ + Replace all uses of old_node with new_node. + """ + old_node.replace_all_uses_with(new_node) + old_node.users.clear() + old_node.graph.erase_node(old_node) + + +def _update_gm_meta_if_possible(gm: torch.fx.GraphModule, mod: torch.nn.Module) -> None: + if ( + isinstance(mod, torch.fx.GraphModule) + and hasattr(mod, "meta") + and "custom" in mod.meta + ): + gm.meta.update({"custom": mod.meta["custom"]}) + + +def node_inline_(call_mod_node: torch.fx.Node) -> Optional[torch.fx.GraphModule]: + """ + Inline the submodule of the given node into the parent module. + Note: we only support the case where submodule takes tensors inputs. + """ + assert call_mod_node.op == "call_module" + gm = call_mod_node.graph.owning_module + assert gm is not None + + assert isinstance(call_mod_node.target, str) + sub_gm = getattr(gm, call_mod_node.target) + + phs = (node for node in sub_gm.graph.nodes if node.op == "placeholder") + body = ( + node for node in sub_gm.graph.nodes if node.op not in ("placeholder", "output") + ) + output = [node for node in sub_gm.graph.nodes if node.op == "output"] + + for ph, arg in zip(phs, call_mod_node.args): + assert isinstance(arg, torch.fx.Node) + node_replace_(ph, arg) + + with gm.graph.inserting_before(call_mod_node): + for node in body: + new_node = gm.graph.node_copy(node) + if node.op == "get_attr": + new_target_name = new_node.target + if hasattr(gm, new_target_name): + # Loop through and find the "submod_{i}" that have no name collision + i = 1 + new_target_name = f"submod_{i}" + while hasattr(gm, new_target_name): + i += 1 + new_target_name = f"submod_{i}" + new_node.target = new_target_name + setattr(gm, new_node.target, getattr(sub_gm, node.target)) + node_replace_(node, new_node) + + if len(output) > 0: + assert len(output) == 1 and len(output[0].args) == 1 + new_output = output[0].args[0] + + if isinstance(new_output, torch.fx.Node): + # Clear the users of the output node and set + # the users to be the users of original call_module node. + new_output.users.clear() + node_replace_(call_mod_node, new_output) + elif isinstance(new_output, (list, tuple)): + # Pop subgraph output node from users. + for node in new_output: + node.users.pop(output[0]) + + # Inline the get_item calls for the output node. + get_item_users = nodes_filter( + list(call_mod_node.users.keys()), + lambda node: node.op == "call_function" + and node.target is operator.getitem, + ) + # get_item_node.args[1] is the idx referring to new_output[idx] + nodes_map( + get_item_users, + lambda get_item_node: node_replace_( + get_item_node, + new_output[get_item_node.args[1]], + ), + ) + call_mod_node.graph.erase_node(call_mod_node) + else: + raise NotImplementedError( + f"Unsupported output type {type(new_output)}. Expect it to be a Node or a list/tuple of Nodes." + ) + else: + call_mod_node.graph.erase_node(call_mod_node) + + gm.delete_all_unused_submodules() + gm.recompile() + return gm + + +def _get_torch_jit_trace_forward_signature(mod: torch.nn.Module) -> inspect.Signature: + """ + Get source code and parse argument names using AST. The function returns + a signature of the forward() function. + + # TODO: Directly provide inspect.signature compatible TS-d module. + """ + ast_mod = ast.parse(mod.code) # type: ignore[call-overload] + ast_func_def: ast.FunctionDef = ast_mod.body[0] + + # FIXME(jiashenc): TorchScript should only allow positional or keywords arguments. + arg_type_map = {"args": Parameter.POSITIONAL_OR_KEYWORD} + + # Traverse all argument types in AST tree and create associated parameters. + param_list = [] + for arg_type, param_type in arg_type_map.items(): + arg_name_list = [a.arg for a in getattr(ast_func_def.args, arg_type)] + for arg_name in arg_name_list: + if arg_name == "self": + continue # Skip self argument. + param_list.append(inspect.Parameter(arg_name, param_type)) + + return inspect.Signature(parameters=param_list) + + +def _bind_signature_to_inputs(mod, fake_args, fake_kwargs): + if isinstance(mod, (torch.jit.ScriptModule, torch.jit.TracedModule)): + sig = _get_torch_jit_trace_forward_signature(mod) + + # Sanity check for placeholder names coming from TorchScript. + assert len(sig.parameters) == len(fake_args) + len(fake_kwargs), ( + "Arguments other than POSITIONAL_OR_KEYWORD kinds in forward() " + "are not supported in _get_torch_jit_trace_forward_signature" + ) + else: + sig = inspect.signature(mod.forward) + + # Rather than binding both fake_args and fake_kwargs to sig names, we + # (partially) bind only fake_args, while reusing fake_kwarg names. This + # ensures that fake_kwargs do not get reordered, which is important to + # match flattened user inputs. + return {**sig.bind_partial(*fake_args).arguments, **fake_kwargs} + + +def _build_cache(name, find_available, used_names): + used_names.add(name) + match = re.match(r"(.*)_(\d+)", name) + if match: + prefix, n = match.group(1), match.group(2) + if int(n) > find_available[prefix]: + find_available[prefix] = int(n) + + +def _name_hoo_subgraph_placeholders(gm: torch.fx.GraphModule) -> None: + """ + Propagate placeholder names from the top-level graph into HigherOrderOp subgraphs, + and handle collisions with non-placeholders by count suffixing. + Different HOO subgraph types have different input schemas, so we first enumerate them + and gather the top-level named placeholder nodes. + """ + + # gather all HOO subgraphs and their top-level named placeholder nodes + subgraph_ph_tuples: list[tuple[torch.fx.GraphModule, list[torch.fx.Node]]] = [] + for node in gm.graph.nodes: + if node.op == "call_function" and isinstance( + node.target, torch._ops.HigherOrderOperator + ): + # HOO subgraphs have varying input schemas, so we enumerate them there + if node.target._name == "cond": + _, true_graph, false_graph, cond_args = node._args + subgraph_ph_tuples.append((getattr(gm, true_graph.target), cond_args)) + subgraph_ph_tuples.append((getattr(gm, false_graph.target), cond_args)) + elif node.target._name == "wrap_with_set_grad_enabled": + subgraph, phs = node._args[1], node._args[2:] + subgraph_ph_tuples.append((getattr(gm, subgraph.target), phs)) + elif node.target._name == "map_impl": + body_graph, array, args = node._args + subgraph_ph_tuples.append( + (getattr(gm, body_graph.target), array + args) + ) + + # propagate names + for subgraph, hoo_phs in subgraph_ph_tuples: + name_map: dict[str, str] = {} + find_available: dict[str, int] = defaultdict(int) + used_names: set[str] = set() + for i, node in enumerate(subgraph.graph.nodes): + if i < len(hoo_phs): # placeholder, retain name + name_map[node.name] = hoo_phs[i].name + node.name = node.target = hoo_phs[i].name + _build_cache(node.name, find_available, used_names) + else: # non-placeholder, check for collisions + node.name = _rename_without_collisions( + name_map, find_available, used_names, node.name, node.name + ) + + # recurse and recompile + _name_hoo_subgraph_placeholders(subgraph) + subgraph.recompile() + + +def _assign_new_node_names( + gm: torch.fx.GraphModule, + name_map: dict[str, str], + custom_meta: dict[str, Any], +) -> None: + """ + Assign new names to all nodes, in the graph module, from name map. + """ + for node in gm.graph.nodes: + if node.op == "placeholder": + assert node.name in name_map + node.name = node.target = name_map[node.name] + if node.name in custom_meta: + if node.meta.get("custom") is None: + node.meta["custom"] = {} + else: + # Assert if any existing key has different value + for k, v in node.meta["custom"].items(): + if ( + k in custom_meta[node.name] + and v != custom_meta[node.name][k] + ): + raise AssertionError( + f"Mismatch in custom metadata for key {k}. Value in " + f"node.meta is {v} and value in custom_meta is {custom_meta[node.name][k]}." + ) + node.meta["custom"].update(custom_meta[node.name]) + # if the constant obj is an input, we also need to update meta["val"] + # because this is created before the placeholder naming pass + if isinstance(node.meta["val"], CustomObjArgument): + node.meta["val"].name = node.name + elif node.name in name_map: + node.name = name_map[node.name] + + +def placeholder_naming_pass( + gm: torch.fx.GraphModule, + export_graph_signature: "ExportGraphSignature", + mod: torch.nn.Module, + fake_args, + fake_kwargs, + fake_params_buffers, + constants: dict[str, Any], +) -> None: + """ + This pass is run at the end of _export_non_strict() to assign better placeholder node names: + - User inputs: + These follow the signature of mod.forward(), e.g. forward(x, y) produces nodes x, y. + For nested inputs from dictionaries, lists, tuples, or dataclasses, + the names are a concatenation of the path to the tensor. + e.g. x = { + 'a': torch.randn(), + 'b': [torch.randn(), torch.randn()] + } + produces nodes x_a, x_b_0, x_b_1. + - Parameters/buffers/constants/custom objects: + These follow the FQN of the object, prefixed by "p", "b", "c", "obj" respectively. + e.g. self.bar.l0.weight produces "p_bar_l0_weight". + - Effect tokens: + These are named token, token_1, ... + """ + + custom_meta: dict[str, Any] = {} + if isinstance(mod, torch.fx.GraphModule): + for node in mod.graph.nodes: + if "custom" in node.meta: + custom_meta[node.name] = node.meta["custom"] + + def _strip_name(x): + if x.startswith("L__self___"): + x = x[len("L__self___") :] + elif x.startswith("self_"): + x = x[len("self_") :] + x = re.sub(r"[^a-zA-Z0-9]", "_", x) + return x + + def _extract_pytree_key(x): + if isinstance(x, MappingKey): + x = re.sub(r"[^a-zA-Z0-9]", "_", str(x.key)) + return x + elif isinstance(x, SequenceKey): + return str(x.idx) + elif isinstance(x, GetAttrKey): + return x.name + else: + raise RuntimeError(f"Pytree key of type {type(x)} not handled for {x}") + + name_map: dict[str, str] = {} + find_available: dict[str, int] = defaultdict(int) + used_names: set[str] = set() + + # map user input names with mod.forward() signature + combined_args = _bind_signature_to_inputs(mod, fake_args, fake_kwargs) + + flat_args_with_path, _ = tree_flatten_with_path(combined_args) + user_input_names = [ + spec.arg.name + for spec in export_graph_signature.input_specs + if spec.kind == InputKind.USER_INPUT + ] + + # use pytree path to name nested user inputs + for (arg_path, _arg), user_input_name in zip(flat_args_with_path, user_input_names): + if user_input_name: + _rename_without_collisions( + name_map, + find_available, + used_names, + user_input_name, + placeholder_prefixes[InputKind.USER_INPUT] + + "_".join(_extract_pytree_key(x).lower() for x in arg_path), + is_placeholder=True, + ) + + # use graph signature input specs to map param/buffer/constant names + # name effect tokens as token, token_1, ... (these aren't visible to user) + for spec in export_graph_signature.input_specs: + if spec.kind == InputKind.USER_INPUT: + continue + if spec.kind == InputKind.TOKEN: + base_name = "" + else: + base_name = _strip_name(spec.target).lower() + base_name = re.sub(r"[^a-zA-Z0-9]", "_", base_name) + + _rename_without_collisions( + name_map, + find_available, + used_names, + spec.arg.name, + placeholder_prefixes[spec.kind] + base_name, + is_placeholder=True, + ) + if base_name in custom_meta: + # the keys in custom_meta are node names from `mod`, + # which is the base_name here. + # we need the re-mapped name for lookup later + custom_meta[name_map[spec.arg.name]] = custom_meta[base_name] + del custom_meta[base_name] + + # handle naming collisions with call_function/get_attr inputs. + # here, we want to prioritize user input names over call_function names + # e.g. not have forward(self, mul): lead to a placeholder node called mul_13, + # so we increment the suffix of call_function nodes as needed + for node in gm.graph.nodes: + if node.op == "placeholder": + continue + _rename_without_collisions( + name_map, find_available, used_names, node.name, node.name + ) + + # assign new node names + _assign_new_node_names(gm, name_map, custom_meta) + + # propagate names to higher order op subgraphs + _name_hoo_subgraph_placeholders(gm) + + # re-generate graph module code + gm.recompile() + + # modify graph signature (input specs, output specs, user input mutations) + for spec in export_graph_signature.input_specs: + assert spec.arg.name in name_map + spec.arg.name = name_map[spec.arg.name] + if ( # handle targets for custom objects + spec.kind == InputKind.CUSTOM_OBJ and spec.target in name_map + ): + # pyrefly: ignore [index-error] + spec.target = name_map[spec.target][4:] # strip obj_ prefix + + for spec in export_graph_signature.output_specs: + if spec.arg.name in name_map: + spec.arg.name = name_map[spec.arg.name] + if spec.kind == OutputKind.USER_INPUT_MUTATION and spec.target in name_map: + # pyrefly: ignore [index-error] + spec.target = name_map[spec.target] + + # rename keys in constants dict for custom objects + for name in list(constants.keys()): + constant = constants[name] + if name in name_map and not isinstance( + constant, torch.Tensor + ): # rename custom objects with generic names + new_name = name_map[name] + if ( + new_name != name + and re.match(r"arg(\d+)_1", name) + and new_name != placeholder_prefixes[InputKind.CUSTOM_OBJ] + name + ): + constants[new_name] = constant + del constants[name] + + +def remove_proxy_from_state_dict(state_dict: dict, in_place: bool) -> dict: + """ + If `in_place` is false, return a new copy of `state_dict` with "proxy" removed from `v.__dict__`. + `v` is the values in the dictionary. + If `in_place` is true, modify `state_dict` in place. + """ + if in_place: + for k, v in state_dict.items(): + if hasattr(v, "proxy"): + delattr(state_dict[k], "proxy") + return state_dict + else: + new_state_dict = {} + for k, v in state_dict.items(): + if hasattr(v, "proxy"): + new_state_dict[k] = v.detach().clone() + else: + new_state_dict[k] = v + return new_state_dict + + +def _detect_fake_mode_from_gm( + gm: torch.fx.GraphModule, +) -> Optional[torch._subclasses.fake_tensor.FakeTensorMode]: + """ + For a given graph module, we look at the "val" of placeholder nodes to find the fake inputs. + Additionally, if gm doesn't have placeholders, we further look at the "example_value" or "val" of other nodes. + If no fake mode is found, we return None for fake_mode. + """ + + fake_inps: list[torch.Tensor] = [] + fake_vals: list[torch.Tensor] = [] + for node in gm.graph.nodes: + if node.op == "placeholder" and "val" in node.meta: + fake_val = node.meta["val"] + if fake_val is not None and isinstance(fake_val, torch.Tensor): + fake_inps.append(fake_val) + elif len(fake_inps) == 0 and ( + "example_value" in node.meta or "val" in node.meta + ): + fake_val = None + if "example_value" in node.meta: + fake_val = node.meta["example_value"] + elif "val" in node.meta: + fake_val = node.meta["val"] + if fake_val is not None and isinstance(fake_val, torch.Tensor): + fake_vals.append(fake_val) + + return detect_fake_mode(fake_inps + fake_vals) + + +@contextmanager +def _disable_load_state_dict_hooks(mod: torch.nn.Module): + state_dict_hooks: dict[int, Callable] = dict(mod._state_dict_hooks) + state_dict_pre_hooks: dict[int, Callable] = dict(mod._state_dict_pre_hooks) + mod._state_dict_hooks.clear() + mod._state_dict_pre_hooks.clear() + try: + yield + finally: + mod._state_dict_hooks = state_dict_hooks + mod._state_dict_pre_hooks = state_dict_pre_hooks + + +def _is_cia_op(op: "OperatorBase") -> bool: + return ( + torch._C._dispatch_has_kernel_for_dispatch_key( + op.name(), torch._C.DispatchKey.CompositeImplicitAutograd + ) + or torch._C.DispatchKey.CompositeImplicitAutograd in op.py_kernels + ) + + +def _is_preservable_cia_op(op: "OperatorBase") -> bool: + return _check_valid_to_preserve(op) and _is_cia_op(op) + + +def _is_aten_op(op: "OperatorBase") -> bool: + return op.name().split("::")[0] == "aten" + + +def _is_custom_op(op: "OperatorBase") -> bool: + return not _is_aten_op(op) + + +# We can't cache this because custom op registry API in python can still +# add entries to the C++ dispatcher. +def _materialize_cpp_cia_ops() -> None: + """ + Utility function to query C++ dispatcher to get the all + possible CIA ops and populate them into torch.ops namespace + """ + cia_ops = torch._C._dispatch_get_registrations_for_dispatch_key( + "CompositeImplicitAutograd" + ) + + # Materialize all CIA ops + for op in cia_ops: + namespace, op_name = tuple(op.split("::")) + split_list = op_name.split(".") + # Sometime overload could be missing + assert len(split_list) == 1 or len(split_list) == 2 + op_name = split_list[0] + op_overload_name = "default" + if len(split_list) == 2: + op_overload_name = split_list[1] + + _ = getattr(getattr(getattr(torch.ops, namespace), op_name), op_overload_name) + + +def _special_op_to_preserve_cia(*args, **kwargs): + """ + This is an special marker that tells our infra that we shouldn't decompose this op. + """ + return NotImplemented + + +# Our strategy for deciding if we can preserve a op is following: +# 1. The op should be known statically that it is functional +# 2. If it is maybe aliasing, we decompose because we must know if an op +# is mutating or aliasing. +def _check_valid_to_preserve(op_overload: "OperatorBase"): + from torch._decomp import _should_decompose_because_unsafe_op + + if _should_decompose_because_unsafe_op(op_overload): + return False + if op_overload in FunctionalTensor.metadata_fns: + return False + + if not hasattr(op_overload, "_schema"): + return False + + alias_info = len( + [i for i in op_overload._schema.arguments if i.alias_info is not None] + ) + + is_mutating_or_aliasing = alias_info != 0 or op_overload._schema.is_mutable + + if is_mutating_or_aliasing: + return False + + if not torch._C._dispatch_has_kernel(op_overload.name()): + return False + + return True + + +@functools.lru_cache(maxsize=1) +def _collect_all_valid_cia_ops_for_aten_namespace() -> set["OperatorBase"]: + return _collect_all_valid_cia_ops_for_namespace(torch.ops.aten) + + +def _collect_all_valid_cia_ops_for_namespace( + op_namespace: torch._ops._OpNamespace, +) -> set["OperatorBase"]: + # Step 1: Materialize all ops from C++ dispatcher + _materialize_cpp_cia_ops() + + # Step 2: Query all ops from python dispatcher + cia_ops = set() + for op in op_namespace: + op_packet = getattr(op_namespace, op) + for overload in op_packet.overloads(): + op_overload = getattr(op_packet, overload) + if _is_preservable_cia_op(op_overload): + cia_ops.add(op_overload) + return cia_ops + + +def _collect_all_valid_cia_ops() -> set["OperatorBase"]: + """ + This is an util function that gets the all CIA functional ops. + + The algorithm is in 2 steps: + 1. We first query C++ dispatcher to get the list of CIA ops + and then we call getattr on torch.ops.aten to lazily populate + them. + + 2. Sometimes, handful of ops have CIA registered in python dispatcher + but not on the C++ side, these can't be caught at the first step. + So we walk again to get the final list. + + Note that the output of this function should never be modified + """ + cia_ops = set() + for op_namespace_name in torch.ops._dir: + # The reason we split here is because aten ops are safe to cache. + if op_namespace_name != "aten": + assert hasattr(torch.ops, op_namespace_name) + op_namespace = getattr(torch.ops, op_namespace_name) + if isinstance(op_namespace, torch._ops._OpNamespace): + cia_ops |= _collect_all_valid_cia_ops_for_namespace(op_namespace) + else: + cia_ops |= _collect_all_valid_cia_ops_for_aten_namespace() + return cia_ops + + +def _get_decomp_for_cia(op: "OperatorBase"): + # [NOTE] Separating out func.decompose + # Ideally we should be able to just register func.decompose but + # we can't as this decomp is gonna be registered to the py_impl. + # As a result it will infinitely recurse. So we first check if the op + # has py_impl entry for CIA and if it is we use that first. If not, + # we register C++ query to py_impl. + dk = torch._C.DispatchKey.CompositeImplicitAutograd + if dk in op.py_kernels and not isinstance(op.py_kernels[dk], torch._C.DispatchKey): + return op.py_kernels[dk] + + def _special_op_to_decompose_cia(*args, **kwargs): + kernel = kwargs["kernel"] + del kwargs["kernel"] + # Can't call kernel.decompose due to infinite recursion as + # we register this kernel to py_impl directly + dk = torch._C.DispatchKey.CompositeImplicitAutograd + if torch._C._dispatch_has_kernel_for_dispatch_key( + kernel.name(), torch._C.DispatchKey.CompositeImplicitAutograd + ): + return kernel._op_dk(dk, *args, **kwargs) + else: + raise AssertionError( + f"Expected {kernel} to have CompositeImplicitAutograd kernel" + ) + + return functools.partial(_special_op_to_decompose_cia, kernel=op) + + +@contextmanager +def _compiling_state_context(): + old_compiling_flag = torch.compiler._is_compiling_flag + old_exporting_flag = torch.compiler._is_exporting_flag + try: + torch.compiler._is_compiling_flag = True + torch.compiler._is_exporting_flag = True + yield + finally: + torch.compiler._is_compiling_flag = old_compiling_flag + torch.compiler._is_exporting_flag = old_exporting_flag + + +def _fakify_params_buffers( + fake_mode: FakeTensorMode, + mod: torch.nn.Module, +) -> dict[str, Union[torch.Tensor, torch.nn.Parameter]]: + params_buffers = { + **dict(mod.named_parameters(remove_duplicate=False)), + **dict(mod.named_buffers(remove_duplicate=False)), + } + + faked_params_buffers = {} + memo: dict[int, FakeTensor] = {} + for key, value in params_buffers.items(): + if id(value) in memo: + fake_tensor = memo[id(value)] + else: + fake_tensor = fake_mode.from_tensor(value, static_shapes=True) + memo[id(value)] = fake_tensor + faked_params_buffers[key] = fake_tensor + return faked_params_buffers # type: ignore[return-value] + + +def register_module_as_pytree_input_node(cls: type[torch.nn.Module]) -> None: + """ + Registers a module as a valid input type for :func:`torch.export.export`. + + Args: + mod: the module instance + serialized_type_name: The serialized name for the module. This is + required if you want to serialize the pytree TreeSpec containing this + module. + + Example:: + + import torch + + + class Module(torch.nn.Module): + def __init__(self): + super().__init__() + self.linear = torch.nn.Linear(3, 3) + + def forward(self, x): + return self.linear(x) + + + torch._export.utils.register_module_as_pytree_node(InputDataClass) + + + class Mod(torch.nn.Module): + def forward(self, x, m): + return m(x) + x + + + ep = torch.export.export(Mod(), (torch.randn(3), Module())) + print(ep) + + """ + assert issubclass(cls, torch.nn.Module) + + import weakref + + class PrototypeModule(weakref.ref): + def __init__(self, m, *args, **kwargs): + super().__init__(m, *args, **kwargs) # type: ignore[call-arg] + assert isinstance(m, torch.nn.Module) + assert not hasattr(self, "_proto_cls") + self._proto_cls = cls + + def __eq__(self, other): + return self._proto_cls == other._proto_cls + + def __deepcopy__(self, memo): + return PrototypeModule(self()) + + def default_flatten_fn(obj: Any) -> tuple[list[Any], Context]: + named_parameters = dict(obj.named_parameters()) + named_buffers = dict(obj.named_buffers()) + params_buffers = {**named_parameters, **named_buffers} + return list(params_buffers.values()), [ + list(params_buffers.keys()), + PrototypeModule(obj), + ] + + def default_unflatten_fn(values: Iterable[Any], context: Context) -> Any: + flat_names, ref = context + if ref is None or ref() is None: + raise RuntimeError("Module has been garbage collected") + obj = ref() + assert flatten_fn is not None + flattened, _ = flatten_fn(obj) + + # NOTE: This helper function will replicate an nn.Module in the exactly same + # structure to be used together with _reparameterize_module. This will + # create a clone of the module with the new parameters and buffers without + # affecting the original module. + def copy_module(mod: torch.nn.Module): + ret = copy.copy(mod) + ret.__dict__ = {copy.copy(k): copy.copy(v) for k, v in mod.__dict__.items()} + for name, child in ret.named_children(): + setattr(ret, name, copy_module(child)) + return ret + + if any(v is not o for v, o in zip(values, flattened)): + with torch.nn.utils.stateless._reparametrize_module( + obj, dict(zip(flat_names, values)), tie_weights=True, strict=True + ): + ret = copy_module(obj) + else: + ret = obj + return ret + + def default_flatten_fn_with_keys(obj: Any) -> tuple[list[Any], Context]: + flattened, [flat_names, *args] = flatten_fn(obj) # type: ignore[misc] + return [(MappingKey(k), v) for k, v in zip(flat_names, flattened)], [ + flat_names, + *args, + ] + + flatten_fn = default_flatten_fn + unflatten_fn = default_unflatten_fn + + serialized_type_name = cls.__module__ + "." + cls.__qualname__ + + def to_dumpable_context(context): + keys, *_ = context + return json.dumps([keys, *([None] * len(_))]) + + def from_dumpable_context(dumpable): + s = json.loads(dumpable) + s[1] = PrototypeModule(torch.nn.Module()) + return s + + _register_pytree_node( + cls, + flatten_fn, + unflatten_fn, + serialized_type_name=serialized_type_name, + flatten_with_keys_fn=default_flatten_fn_with_keys, + to_dumpable_context=to_dumpable_context, + from_dumpable_context=from_dumpable_context, + ) + + def default_flatten_fn_spec(obj, spec) -> list[Any]: + flats, context = flatten_fn(obj) + assert context == spec.context + return flats + + register_pytree_flatten_spec( + cls, + default_flatten_fn_spec, + ) + + +def deregister_module_as_pytree_input_node(cls: type[torch.nn.Module]) -> None: + _deregister_pytree_node(cls) + _deregister_pytree_flatten_spec(cls) + + +def _sync_state(src, dst): + assert isinstance( + src, + torch.nn.Module, + ), f"Expected {src} to be a nn.Module" + assert isinstance( + dst, + torch.nn.Module, + ), f"Expected {dst} to be a nn.Module" + # Share state (params, buffers) between modules. + # This ensures that state mutations are visible across them. + # Since tensor constants are not mutable, copying (without sharing) is OK. + # Also, primitive constants are specialized, so copying (without sharing) is OK. + dst._parameters = src._parameters + dst._buffers = src._buffers + + +def sync_state(*wrapped_method_modules): + """ + Sync state between exported modules corresponding to wrapped methods. + This might be necessary after serializing/deserializing due to copying. + """ + if wrapped_method_modules: + m, *other_ms = wrapped_method_modules + for other_m in other_ms: + _sync_state(m, other_m) + + +class _WrappedMethod(torch.nn.Module): + def __init__(self, method): + super().__init__() + # share state of method's self module + _sync_state(method.__self__, self) + # redirect forward to method + self.forward = method + + +def wrap_method(method): + """ + Wrap a method as a module so that it can be exported. + The wrapped module's forward points to the method, and + the method's original module state is shared. + """ + assert ismethod( + method, + ), f"Expected {method} to be a method" + return _WrappedMethod(method) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/wrappers.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/wrappers.py new file mode 100644 index 0000000000000000000000000000000000000000..e0231694039370ba614dfcc28ac4642179456ede --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_export/wrappers.py @@ -0,0 +1,337 @@ +# mypy: allow-untyped-defs +import inspect +from contextlib import contextmanager +from functools import wraps + +import torch +import torch._custom_ops +from torch._C import DispatchKey +from torch._export.utils import _maybe_find_pre_dispatch_tf_mode_for_export +from torch._higher_order_ops.flat_apply import ( + _ConstantFunction, + flat_apply, + to_graphable, +) +from torch._higher_order_ops.strict_mode import strict_mode +from torch._higher_order_ops.utils import autograd_not_implemented +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ( + PreDispatchTorchFunctionMode, + ProxyTorchDispatchMode, + track_tensor_tree, +) +from torch.utils import _pytree as pytree +from torch.utils._python_dispatch import is_traceable_wrapper_subclass_type + + +class ExportTracepoint(HigherOrderOperator): + def __init__(self): + super().__init__("_export_tracepoint") + + def __call__(self, *args, **kwargs): + return super().__call__(*args, **kwargs) + + +_export_tracepoint = ExportTracepoint() + + +@_export_tracepoint.py_impl(ProxyTorchDispatchMode) +def export_tracepoint_dispatch_mode(mode, *args, **kwargs): + p_args, p_kwargs = pytree.tree_map(mode.tracer.unwrap_proxy, (args, kwargs)) + proxy = mode.tracer.create_proxy( + "call_function", _export_tracepoint, p_args, p_kwargs + ) + return track_tensor_tree(args, proxy, constant=None, tracer=mode.tracer) + + +@_export_tracepoint.py_impl(FakeTensorMode) +def export_tracepoint_fake_tensor_mode(mode, *args, **kwargs): + with mode: + return args + + +@_export_tracepoint.py_functionalize_impl +def export_tracepoint_functional(ctx, *args, **kwargs): + unwrapped_args = ctx.unwrap_tensors(args) + unwrapped_kwargs = ctx.unwrap_tensors(kwargs) + + with ctx.redispatch_to_next(): + _export_tracepoint(*unwrapped_args, **unwrapped_kwargs) + return args + + +_export_tracepoint.py_impl(DispatchKey.Autograd)( + autograd_not_implemented(_export_tracepoint, deferred_error=True) +) + + +@_export_tracepoint.py_impl(DispatchKey.CPU) +def export_tracepoint_cpu(*args, **kwargs): + return args + + +def _wrap_submodule(mod, path, module_call_specs): + assert isinstance(mod, torch.nn.Module) + assert path != "" + submodule = torch.fx.graph_module._get_attr(mod, path) + + def update_module_call_signatures(path, in_spec, out_spec): + if path in module_call_specs: + assert module_call_specs[path]["in_spec"] == in_spec + assert module_call_specs[path]["out_spec"] == out_spec + module_call_specs[path] = {"in_spec": in_spec, "out_spec": out_spec} + + def check_flattened(flat_args): + for a in flat_args: + if not (isinstance(a, (torch.Tensor, str, int, float, bool)) or a is None): + raise AssertionError( + f"Only Tensors or scalars are supported as pytree flattened inputs, got: {a}" + ) + + def pre_hook(module, args, kwargs): + flat_args, in_spec = pytree.tree_flatten((args, kwargs)) + check_flattened(flat_args) + flat_args = _export_tracepoint(*flat_args, kind="module_call_inputs", path=path) + args, kwargs = pytree.tree_unflatten(flat_args, in_spec) + return args, kwargs + + def post_hook(module, args, kwargs, res): + _, in_spec = pytree.tree_flatten((args, kwargs)) + flat_res, out_spec = pytree.tree_flatten(res) + check_flattened(flat_res) + flat_res = _export_tracepoint(*flat_res, kind="module_call_outputs", path=path) + update_module_call_signatures(path, in_spec, out_spec) + return pytree.tree_unflatten(flat_res, out_spec) + + pre_handle = submodule.register_forward_pre_hook(pre_hook, with_kwargs=True) + post_handle = submodule.register_forward_hook(post_hook, with_kwargs=True) + return pre_handle, post_handle + + +@contextmanager +def _wrap_submodules(f, preserve_signature, module_call_signatures): + handles = [] + + try: + for path in preserve_signature: + handles.extend(_wrap_submodule(f, path, module_call_signatures)) + yield + finally: + for handle in handles: + handle.remove() + + +def _mark_strict_experimental(cls): + def call(self, *args): + return strict_mode(self, args) + + cls.__call__ = call + return cls + + +def _register_func_spec_proxy_in_tracer(tracer, name, spec): + """ + This is a wrapper utility method on top of tracer to cache the + already registered subclass spec attribute. This is useful because + Subclass.__init__ will be same for each subclass. By default, fx will + create multiple attributes/proxies for given attribute. + """ + fx_name = name + "0" + if hasattr(tracer.root, fx_name): + assert getattr(tracer.root, fx_name) == spec + return tracer.create_proxy("get_attr", fx_name, (), {}) + + qualname = tracer.get_fresh_qualname(name) + setattr(tracer.root, qualname, spec) + return tracer.create_proxy("get_attr", qualname, (), {}) + + +def _emit_flat_apply_call( + *, + tracer, + spec_name: str, + const_target_for_apply, + graphable_args, + track_value, + call_spec_cache_key: str, +): + # Flatten to graphable form and record the spec on the FX root + flat_args, in_spec = to_graphable(graphable_args) + qualname = tracer.get_fresh_qualname(spec_name) # type: ignore[union-attr] + setattr(tracer.root, qualname, in_spec) # type: ignore[union-attr] + spec_proxy = tracer.create_proxy("get_attr", qualname, (), {}) + + # Reuse/cached ConstantFunction spec on the root + _, func_spec = pytree.tree_flatten(_ConstantFunction(const_target_for_apply)) + func_spec_proxy = _register_func_spec_proxy_in_tracer( + tracer, f"{call_spec_cache_key}_const_func_spec", func_spec + ) + + # Map runtime args -> proxies (always via tracer.unwrap_proxy now) + flat_proxy_args = pytree.tree_map(tracer.unwrap_proxy, flat_args) + + # Emit flat_apply and track result structure + out_proxy = tracer.create_proxy( + "call_function", flat_apply, (func_spec_proxy, spec_proxy, *flat_proxy_args), {} + ) + track_tensor_tree(track_value, out_proxy, constant=None, tracer=tracer) + + +def _is_init(fn): + return callable(fn) and fn.__name__ == "__init__" + + +def mark_subclass_constructor_exportable_experimental(constructor_subclass): + """ + Experimental decorator that makes subclass to be traceable in export + with pre-dispatch IR. To make your subclass traceble in export, you need to: + 1. Implement __init__ method for your subclass (Look at DTensor implementation) + 2. Decorate your __init__ method with _mark_constructor_exportable_experimental + 3. Put torch._dynamo_disable decorator to prevent dynamo from peeking into its' impl + + Example: + + class FooTensor(torch.Tensor): + @staticmethod + def __new__(cls, elem, *, requires_grad=False): + # ... + return torch.Tensor._make_subclass(cls, elem, requires_grad=requires_grad) + + @torch._dynamo_disable + @mark_subclass_constructor_exportable_experimental + def __init__(self, elem, ...): + # ... + """ + if not _is_init(constructor_subclass): + raise RuntimeError( + f"torch._export.wrappers.mark_constructor_exportable_experimental can only be applied on subclass tensor.__init__" + f"But, you are adding it on {constructor_subclass.__name__} which is not supported. " + f"If __init__ doesn't exist on your subclass, please add it. Look at DTensor.__init__ implementation for example" + ) + + def wrapper(*args, **kwargs): + constructor_subclass(*args, **kwargs) + + if not torch.compiler.is_exporting(): + return + + if not is_traceable_wrapper_subclass_type(type(args[0])): + assert constructor_subclass.__qualname__.endswith("__init__") + obj_name = constructor_subclass.__qualname__[: -len("__init__")] + raise RuntimeError( + f"Can't intercept {obj_name} in export because this object is not a traceable " + f"tensor subclass. Please look at DTensor.__init__ implementation as an example of proper usage of this API." + ) + + mode = _maybe_find_pre_dispatch_tf_mode_for_export() + if mode is None: + return + + assert isinstance(mode, PreDispatchTorchFunctionMode) + + tracer = mode.tracer + subclass = args[0] + graphable = (tuple(args[1:]), kwargs) + + spec_name = "_".join(constructor_subclass.__qualname__.lower().split(".")) + call_spec_cache_key = type(subclass).__name__.lower() + + _emit_flat_apply_call( + tracer=tracer, + spec_name=spec_name, + const_target_for_apply=type(subclass), + graphable_args=graphable, + track_value=subclass, # track the constructed subclass instance + call_spec_cache_key=call_spec_cache_key, + ) + return + + return wrapper + + +def allow_in_pre_dispatch_graph(func): + """ + Experimental decorator that adds user function to export pre-dispatch graph. Note that + we only support custom autograd function/subclass constructors today. To use this function: + 1. For subclasses: + 1. refer to instructions in mark_subclass_constructor_exportable_experimental + 2. Define apply method on your custom autograd function and apply this decorator. + + Example: + + class MyCoolCustomAutogradFunc(autograd.Function): + @classmethod + @torch._export.wrappers.allow_in_pre_dispatch_graph + def apply(cls, *args, **kwargs): + return super(MyCoolCustomAutogradFunc, cls).apply(*args, **kwargs) + + """ + if _is_init(func): + return mark_subclass_constructor_exportable_experimental(func) + + if not (_is_init(func) or func.__name__ == "apply"): + raise RuntimeError( + f"torch._export.wrappers.allow_in_pre_dispatch_graph can only be applied on subclass tensor.__init_ " + f"or custom_autograd_function.apply. " + f"But, you are adding it on {func.__name__} which is not supported. " + f"If __init__ doesn't exist on your subclass, please add it. Look at DTensor.__init__ implementation for example. " + f"If you are adding it on custom autograd function, please add it on apply method. " + f"If anything else, file an issue on github and we may consider extending our support. " + ) + + @wraps(func) + def wrapper(*args, **kwargs): + if not torch.compiler.is_exporting(): + return func(*args, **kwargs) + + if not inspect.isclass(args[0]): + return func(*args, **kwargs) + + if not issubclass(args[0], torch.autograd.Function): + return func(*args, **kwargs) + + from torch._ops import _get_dispatch_mode_pre_dispatch + + mode = _get_dispatch_mode_pre_dispatch(torch._C._TorchDispatchModeKey.PROXY) + if mode is None: + return func(*args, **kwargs) + + # Sometimes custom autograd functions can call into HOPs that don't have proxy impl + # at PreDispatch level, so we just dispatch it below to get the concrete result. + include_to_set = torch._C._dispatch_tls_local_include_set().remove( + torch._C.DispatchKey.PreDispatch + ) + exclude_to_set = ( + torch._C._dispatch_tls_local_exclude_set() + | torch._C.DispatchKeySet(torch._C.DispatchKey.PreDispatch) + ) + + with torch._C._ForceDispatchKeyGuard(include_to_set, exclude_to_set): + out = func(*args, **kwargs) + + assert mode.pre_dispatch, "Should only do this in predispatch" + tracer = mode.tracer + + function_cls_name = f"{args[0].__module__}.{args[0].__qualname__}" + graphable = ((function_cls_name, *args[1:]), kwargs) + + from torch.export.custom_ops import ( + _call_custom_autograd_function_in_pre_dispatch, + ) + + spec_name = "_".join(function_cls_name.split(".")) + call_spec_cache_key = type( + _call_custom_autograd_function_in_pre_dispatch + ).__name__.lower() + _emit_flat_apply_call( + tracer=tracer, + spec_name=spec_name, + const_target_for_apply=_call_custom_autograd_function_in_pre_dispatch, + graphable_args=graphable, + track_value=out, + call_spec_cache_key=call_spec_cache_key, + ) + return out + + return wrapper diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..10a55772ab58b21573a6eba0356ddd3080164ac7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/aot_autograd.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/aot_autograd.py new file mode 100644 index 0000000000000000000000000000000000000000..9fdebe6396d4b2bcd40ea940ede65e07ce6c2e96 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/aot_autograd.py @@ -0,0 +1,1749 @@ +# mypy: ignore-errors + +import contextlib +import itertools +from collections.abc import Callable +from contextlib import nullcontext +from functools import wraps +from typing import Any, Optional +from unittest.mock import patch + +import torch +import torch._dynamo.logging +import torch.nn as nn +import torch.utils._pytree as pytree +import torch.utils.dlpack +from torch import Tensor +from torch._decomp.decompositions_for_rng import PhiloxStateTracker, rng_decompositions +from torch._dispatch.python import enable_python_dispatcher +from torch._dynamo import compiled_autograd +from torch._dynamo.utils import ( + CompileEventLogger, + dynamo_timed, + preserve_rng_state, + set_feature_use, +) +from torch._guards import detect_fake_mode +from torch._inductor.cudagraph_utils import BoxedDeviceIndex +from torch._inductor.utils import BoxedBool +from torch._subclasses import FakeTensor, FakeTensorMode +from torch.export._tree_utils import reorder_kwargs +from torch.fx.experimental.proxy_tensor import make_fx +from torch.fx.experimental.symbolic_shapes import ShapeEnv + + +static_inputs_log = torch._logging.getArtifactLogger( + __name__, "cudagraph_static_inputs" +) +from . import config +from ._aot_autograd.autograd_cache import ( # noqa: F401 + AOTAutogradCache, + autograd_cache_key, + should_use_local_autograd_cache, + should_use_remote_autograd_cache, +) +from ._aot_autograd.collect_metadata_analysis import ( # noqa: F401 + run_functionalized_fw_and_collect_metadata, +) +from ._aot_autograd.descriptors import ( + AOTInput, + BufferAOTInput, + ParamAOTInput, + PlainAOTInput, +) +from ._aot_autograd.frontend_utils import ( + _detect_attribute_assignment, + _try_get_metadata_from_dynamo, + construct_fake_mode, + process_inputs, +) +from ._aot_autograd.functional_utils import ( # noqa: F401 + _check_if_mutation_can_be_in_graph, + are_all_mutations_hidden_from_autograd, + are_all_mutations_under_no_grad_or_inference_mode, + assert_functional_graph, + from_fun, + gen_alias_from_base, + has_data_mutation, + has_metadata_mutation, + is_fun, + sync_functional_tensor, + to_fun, +) +from ._aot_autograd.graph_capture_wrappers import ( # noqa: F401 + aot_dispatch_subclass, + create_functional_call, + create_functionalized_fn, + create_functionalized_rng_ops_wrapper, + create_joint, + fn_input_mutations_to_outputs, + fn_prepped_for_autograd, +) +from ._aot_autograd.graph_compile import ( # noqa: F401 + aot_stage1_graph_capture, + aot_stage2_compile, + aot_stage2_export, +) +from ._aot_autograd.input_output_analysis import ( # noqa: F401 + compute_overlapping_inputs, + create_graph_signature, + create_synthetic_base_metadata, + remove_dupe_metadata, +) +from ._aot_autograd.logging_utils import ( # noqa: F401 + callback_set, + describe_input, + format_guard_bug_msg, + get_aot_compilation_context, + get_aot_graph_name, + get_graph_being_compiled, + graph_being_compiled, + model_name, + nth_graph, + set_model_name, + setup_stacktrace_preservation_hooks, + track_graph_compiling, +) +from ._aot_autograd.runtime_wrappers import ( # noqa: F401 + AOTDedupeWrapper, + AOTSyntheticBaseWrapper, + SerializableCompiledFunction, +) +from ._aot_autograd.schemas import ( # noqa: F401 + AOTConfig, + AOTDispatchCompiler, + AOTGraphCapture, + AOTState, + BackwardSignature, + FakifiedFlatArgs, + FQN, + GraphInputName, + GraphOutputName, + GraphSignature, + InputAliasInfo, + JointWithDescriptors, + MutationType, + OutputAliasInfo, + OutputType, + SerializableAOTDispatchCompiler, + SubclassCreationMeta, + SubclassMeta, + TensorAlias, + ViewAndMutationMeta, +) +from ._aot_autograd.subclass_utils import ( # noqa: F401 + requires_subclass_dispatch, + unwrap_tensor_subclasses, + unwrap_tensor_subclasses_with_indices_to_original, + wrap_tensor_subclasses, + wrap_tensor_subclasses_maybe_joint, +) +from ._aot_autograd.utils import ( # noqa: F401 + _get_autocast_states, + _get_symint_hints, + call_func_at_runtime_with_args, + create_tree_flattened_fn, + KNOWN_TYPES, + make_boxed_compiler, + make_boxed_func, + maybe_to_fresh_input, + normalize_as_list, + partial_flatten_asdict, + root_module_when_exporting_non_strict, + simple_wraps, + strict_zip, +) +from .partitioners import default_partition + + +zip = strict_zip + +# This global counter increments every time we compile a graph with +# AOTAutograd. You can use this to correlate runtime error messages +# with compile time (e.g., if you get an error at runtime saying +# compiled graph 3 failed, you can set a breakpoint at compile time +# for this graph number to investigate further at compile time.) +# +# NB: this is different from get_aot_compilation_context, which tracks +# each underlying graph that is compiled. In contrast, AOT_COUNTER +# corresponds to top-level invocations of aot_module/aot_function; +# one counter is allocated per entire compiled block (but this block +# may involve compiling multiple subgraphs; e.g., for forwards/backwards) +AOT_COUNTER = itertools.count() + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# AOT Autograd contains a pretty non-trivial amount of logic to handle edge cases around aliasing and mutation +# that are external to the graph (they show up as side effects in some way when you run the graph). +# +# Take a look at `test_aotdispatch.py TestAOTAutograd.test_input_mutation*` tests for some examples functions +# and what they're compiled graphs looks like. +# Below is a very long comment detailing several edge cases, and showing how AOT Autograd handles them. +# +# Note [AOT Autograd: input data mutations] +# +# If we compile a function that mutates inputs, then those input mutations are real side effects +# that a user expects to see after running the compiled graph. +# However, the graph that we want to send to a backend needs to be *entirely* functional. +# The way we reconcile this difference is that we remove the mutations completely from the graph that we compile +# but we update the graph to return (updated_inputs, user_outputs). +# In the epilogue that runs after the compiled graph is executed, we copy the updated inputs back to the originals. +# +# Example: original user code: +# def f(x): +# x.mul_(2) +# out = x.mul(3) +# return out +# +# After AOT Autograd compiles, we end up with a: +# (a) compiled graph +# (b) autograd.Function.forward() method, that executes the compiled graph +# (c) wrapper function, that calls the autograd.Function.forward() and performs the epilogue +# +# The output of (a, b, c) are all written below. +# +# def compiled_forward_graph(x): +# x_updated = x.mul(2) +# out = x_updated.mul(3) +# return x_updated, out +# +# # x_updated gets a gradient in the compiled backward +# def compiled_backward_graph(grad_x_updated, grad_out): +# grad_x = ... +# return grad_x +# +# def autograd.Function.forward(x): +# x_updated, out = compiled_forward_graph(x) +# return x_updated, out +# +# def compiled_wrapper(x): +# x_updated, out = autograd.Function.apply(x) +# x.copy_(x_updated) +# return out +# +# Another important thing to note is that updated inputs (due to data mutations) *do* participate +# in the compiled backward graph! Since the compiled forward graph gets N extra outputs +# (due to updated inputs showing up as graph outputs), +# The compiled backward gets an additional N inputs. +# That way, during the x.copy_(x_updated) bit in the epilogue, gradients will flow from the updated input +# back to the original input. + + +# Note [AOT Autograd: input metadata mutations] +# +# For the same reason as input mutations, we also don't put input metadata mutations in the graph. +# Instead, we return the updated version of the input (a view), and mutate the input's metadata outside of the graph +# +# Example: original user code: +# def f(x): +# x.t_() +# out = x.mul(3) +# return out +# +# AOT Autograd output (compiled graph, autograd.Function.forward(), wrapper function): +# def compiled_forward_graph(x): +# x_updated = x.t() +# out = x_updated.mul(3) +# return x_updated, out +# +# # x_updated does *not* get a gradient in the compiled backward +# def compiled_backward_graph(grad_out): +# grad_x = ... +# return grad_x +# +# def autograd.Function.forward(x): +# x_updated, out = compiled_forward_graph(x) +# return x_updated, out +# +# def compiled_wrapper(x): +# x_updated, out = autograd.Function.apply(x) +# x.as_strided_(x_updated) +# return out + + +# Note [AOT Autograd: outputs aliasing inputs or intermediates!] +# +# AOT Autograd needs special handling for outputs that alias graph inputs or intermediates! +# Why? +# (1) autograd.Function.forward() has a limitation, where views that returned in the forward cannot later be mutated. +# (2) views don't need to be compiled in the graph anyway - it's cheap to generate them outside of the compiled graph, +# in an epilogue. +# For outputs that alias inputs, we do the following: +# (a) *still* return the aliased output as a graph output +# (b) In the AOT Autograd wrapper/epilogue, we don't return that aliased output. Instead, we use it to regenerate the output. +# +# For outputs that alias *intermediates*, we do the following: +# (a) Return the output in the compiled forward, **and** return it's ._base (a graph intermediates) as an output in the forward +# (b) Use (output, graph_intermediate) to regenerate the alias, and return that to the user (instead of the compiled fw output). +# You might wonder why we return the aliased output directly in the graph (and making the graph compute it), +# only to not return it and instead generate a fresh alias off of the intermediate, +# instead of (say) just storing metadata about the size/stride of the output somewhere to generate the alias. There are two reasons: +# (1) Getting the actual alias tensor allows us to use view-replay to generate the alias, instead of an as_strided() call +# (2) Inductor (and other backends) are free to change the memory format of graph outputs, if it results in better performance. +# This can result in problems if a user later tries to .view() that output expecting it to have one set of strides, +# when it has a different set of strides. +# By including the view op directly in the graph, inductor takes that into account when deciding what memory format +# the graph intermediate should be. +# +# Another important thing to note is how our traced backward() graph handles aliases. +# (this applies to outputs aliasing inputs, outputs aliasing intermediates, +# *and* updated inputs returned in the compiled forward due to metadata-only mutations). +# Any outputs that alias (either inputs or intermediates) do NOT participate in the compiled backward graph +# It would be wasteful to include them in the compiled backward(), because we regenerate them eagerly +# at the end of the forward. +# +# Example: original user code: +# def f(x): +# out1 = x.t() +# intermediate = x.mul(2) +# out2 = intermediate.view(-1) +# return out1, out2 +# +# AOT Autograd output (compiled graph, autograd.Function.forward(), wrapper function): +# def compiled_forward_graph(x): +# out1 = x.t() +# intermediate = x.mul(2) +# out2 = intermediate.view(-1) +# # the compiled graph also returns the intermediate +# return out1, out2, intermediate +# +# # intermediate gets a gradient in the compiled backward. +# # both output aliases (out1 and out2) do not. +# def compiled_backward_graph(grad_intermediate): +# grad_x = ... +# return grad_x +# +# def autograd.Function.forward(x): +# out1, out2, intermediate = compiled_forward_graph(x) +# return out1, out2, intermediate +# +# def compiled_wrapper(x): +# out1, out2, intermediate = autograd.Function.apply(x) +# # regenerate out1 from the input +# out1_regenerated = out1._view_func(x) +# # regenerate out1 from the intermediate +# out2_regenerated = out2._view_func(intermediate) +# return out1_regenerated, out2_regenerated + + +# Note [AOT Autograd: mutations to inputs that alias other inputs] +# +# Another edge case that is (only partially) handled today is when an input is mutated, but itself aliases another input. +# AOT Autograd needs to **ensure** that functionalization knows that the two inputs are aliased to each other. +# That way, when the aliased input is accessed later in the graph, functionalization knows to "update" the alias +# given the mutation that occurred. +# +# This is handled by updating the calling convention: we create a "synthetic base" that becomes a new input +# in the compiled function, and we regenerate the original (aliased) inputs directly off of the base +# inside of the compiled function. +# +# This logic is fully encapsulated in aot_wrapper_synthetic_base() +# +# Example: original user code: +# def f(x, x_view): +# x.mul_(2) +# out = x * x_view +# return out +# f(x, x.view(-1)) +# +# AOT Autograd output (compiled graph, autograd.Function.forward(), wrapper function): +# def compiled_forward_graph(base) +# x = generate_x(base) +# x_view = generate_x_view(base) +# x_updated = x.mul(2) +# x_view_updated = x_updated.view(-1) +# out = x_updated * x_view_updated +# return x_updated, out +# +# # The calling convention change from (aliases) -> (base) happens +# # *outside* of the autograd.Function.forward(). +# # That means the forward() only has 1 input (base), +# # and the backward() only has 1 output (grad_base) +# def compiled_backward_graph(grad_out): +# grad_base = ... +# return grad_base +# +# def autograd.Function.forward(base): +# x_updated, out = compiled_forward_graph(base) +# return x_updated, out +# +# # The compiled wrapper is where we create synthetic bases. +# # The info on which inputs are mutated is also tracked *before* synthetic base creation. +# def compiled_wrapper(x, x_view): +# base = merge_view_inputs(x, x_view) +# x_updated, out = autograd.Function.apply(base) +# # x and x_view are aliased in eager mode, so this mutation to x will automatically affect x_view. +# x.copy_(x_updated) +# return out + + +# Note [AOT Autograd: Views to avoid tangents aliasing inputs] +# +# We view every forward output when creating out tangent tensors to handle the problematic +# case in which a subclass does extra aliasing between graph outputs/inputs in a way that +# is not visible above the subclass. +# +# Ordinarily, when constructing the joint function that we want to trace in AOTAutograd, +# we're guaranteed that the tangent tensors that we pass +# into the joint are distinct tensors from the primals. This is because when +# decide which forward outputs to create tangents for, we only create tangents +# for forward outputs that are not aliases of inputs (See Note +# [AOT Autograd: outputs aliasing inputs or intermediates!]). +# +# However, when wrapper tensor subclasses enter the picture, it is possible +# to have an output of the forward that is a subclass that is not an +# input / alias of an input, but one of its inner tensors is an alias! +# NestedTensor is an example: Performing an out-of-place pointwise op on a +# NestedTensor constructs a fresh NestedTensor that holds onto the input's +# offsets tensor directly. +# +# Having tangent tensors that are the same as the (primal) forward inputs, +# can cause problems during tracing as make_fx() will specialize on our +# duplicate inputs: If we passed in the same tensor for primals_1 and +# tangents_1 during tracing, make_fx() will happily sub out all usages of +# tangents_1 with primals_1 in the graph, which is not what we want. +# +# To work around this, we view every forward output when creating out tangent +# tensors so that tangents can never be the same as forward inputs even if +# forward inputs alias forward outputs. + +# Note [Side-Effectful Tokens in AOTAutograd] +# +# We allow some some side-effectful operators in +# the post-AOTAutograd (functional) graph, such as prints and torchbind operations. +# To ensure that these side-effects are compatible to future graph passes that +# assume that the graph is functional, we will thread "effect tokens" to show +# data dependence between these side-effectful operators. Practically speaking, +# effect tokens are just dummy values (torch.tensor([])). The graph would look +# like the following: +# +# def gm(self, token0, reader): +# token1, frame = with_token(ordered_effect_op, (reader,), token0) +# frame = frame * 2 +# token2, frame2 = with_token(ordered_effect_op, (reader,), token1) +# frame2 = frame2 * 2 +# return token2, frame, frame2 +# +# We will pass the token as an input to the graph, thread it through +# side-effectful operators using the `with_effects` high order operator, and then +# return the updated token as an output. +# So the signature of the graph input would look something like +# (*tokens, *params_buffers, *user_inputs), and the signature of the graph +# output would look something like (*tokens, *outputs). +# +# However, Inductor does not want the concept of tokens in the final generated +# code's input and output. Since changing the graph signature inside of inductor +# is difficult, after generating the forward graph, we will run a pass to +# remove the tokens from the inputgenerate the following graph for Inductor, where +# the tokens are created and sunk within the graph, rather than as inputs and +# outputs: +# +# def gm(self, reader): +# token0 = torch.ops.prims._make_token() +# token1, frame = with_token(ordered_effect_op, (reader,), token0) +# frame = frame * 2 +# token2, frame2 = with_token(ordered_effect_op, (reader,), token1) +# frame2 = frame2 * 2 +# sink_token = torch.ops.prims._sink_tokens([token2]) +# return frame, frame2 + +# +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + +aot_autograd_decompositions = {} + + +def create_aot_state( + stack: contextlib.ExitStack, + flat_fn, + fake_flat_args: FakifiedFlatArgs, + flat_args_descs: list[AOTInput], + aot_config: AOTConfig, + fake_mode: FakeTensorMode, + shape_env: Optional[ShapeEnv], +) -> AOTState: + """ + Traces the forward and backward graphs of the attr:`flat_fn` to generate a + joint graph. The joint graph is an Fx graph with Aten ops. Please refer to + the tracing mechanism to understand the graph capturing details. + + The joint graph is then passed through attr:`partition_fn` to isolate the + forward and backward portions, which are then respectively compiled via the + provided attr:`fw_compiler` and attr:`bw_compiler`. + + The resulting compiled forward and backward graphs are then wrapped up in a + ``torch.autograd.Function`` object. + + The calling convention here is that the first aot_config.num_params_buffers + inputs in flat_args are parameters and buffers, and the rest are inputs. + + We use this to assume that parameters/buffer's shapes don't change. + """ + + # Old name for now to avoid messing with stats. Also, note this is pushed + # on the stack, so it extends BEYOND this function + stack.enter_context( + dynamo_timed("create_aot_dispatcher_function", log_pt2_compile_event=True) + ) + + # This is the main entry point. + # TODO: Chillee argues that dynamo itself should pass in fake tensors to + # the list of arguments when compiling; at the moment we do not do this + + if aot_config.decompositions is None: + aot_config.decompositions = {} + + aot_config.decompositions = { + **aot_autograd_decompositions, + **aot_config.decompositions, + } + + if config.functionalize_rng_ops: + # Update the decompositions with functionalized random decompositions + aot_config.decompositions = { + **rng_decompositions, + **aot_config.decompositions, + } + + # Check flat_args to see if they're already fake. If so, use that fake + # mode instead. + + python_dispatcher_mode = ( + enable_python_dispatcher() if shape_env is not None else nullcontext() + ) + + # See NOTE: [Deferring tensor pack/unpack hooks until runtime] + # If any saved tensor hooks are active, we **don't** want to trace them. + # Instead, we'll let them run at runtime, around the custom autograd.Function + # that we generate in torch.compile. + stack.enter_context(torch.autograd.set_multithreading_enabled(False)) + stack.enter_context(preserve_rng_state()) + stack.enter_context(fake_mode) + stack.enter_context(python_dispatcher_mode) + stack.enter_context(PhiloxStateTracker()) + stack.enter_context( + torch._dynamo.utils._disable_saved_tensors_hooks_during_tracing() + ) + + from torch._library.fake_class_registry import FakeScriptObject, maybe_to_fake_obj + from torch._library.opaque_object import is_opaque_type + + # Tracing may mutate the states the fake script object, + # so we need to duplicate the fake script objects so that subsequent tracing + # won't be affected. + def _dup_fake_script_obj(fake_flat_args): + return [ + maybe_to_fake_obj(detect_fake_mode(fake_flat_args), arg.real_obj) + if isinstance(arg, FakeScriptObject) or is_opaque_type(type(arg)) + else arg + for arg in fake_flat_args + ] + + needs_autograd = any( + x.requires_grad for x in fake_flat_args if isinstance(x, Tensor) + ) + + with enable_python_dispatcher(): + # Patch set_rng_state as set_rng_state with fake tensors is + # nonsensical. This does not affect the collection of metadata. + with patch("torch.cuda.set_rng_state", lambda *args: None): + mod = root_module_when_exporting_non_strict(flat_fn) + if mod is not None: + ctx = _detect_attribute_assignment(mod) + else: + ctx = nullcontext() + + if torch._functorch.config.fake_tensor_propagate_real_tensors: + # Running dynamo_timed causes fake tensor issues when + # propagate real tensor is switched on. + dynamo_timed_ctx = nullcontext() + else: + dynamo_timed_ctx = dynamo_timed( + "aot_collect_metadata", log_pt2_compile_event=True + ) + + with dynamo_timed_ctx, ctx: + fw_metadata = run_functionalized_fw_and_collect_metadata( + flat_fn, + flat_args_descs=flat_args_descs, + static_input_indices=aot_config.static_input_indices, + keep_input_mutations=aot_config.keep_inference_input_mutations, + is_train=needs_autograd, + pre_dispatch=aot_config.pre_dispatch, + )(*_dup_fake_script_obj(fake_flat_args)) + + req_subclass_dispatch = requires_subclass_dispatch( + fake_flat_args, fw_metadata + ) + CompileEventLogger.try_add_pt2_compile( + "backend_compile", requires_subclass_dispatch=req_subclass_dispatch + ) + + output_and_mutation_safe = not any( + x.requires_grad + # view-type operations preserve requires_grad even in no_grad. + # Do not count aliases of inputs with requires_grad as reason to make a training graph, + # as AOTAutograd will perform view-replay to regenerate the view outputs at runtime, + # setting their grad_fn properly. + and not ( + x.output_type in (OutputType.alias_of_input, OutputType.is_input) + and fw_metadata.input_info[x.base_idx].requires_grad + ) + for x in fw_metadata.output_info + ) and not any( + x.requires_grad + and x.mutates_data + and not x.mutations_under_no_grad_or_inference_mode + and not x.mutations_hidden_from_autograd + for x in fw_metadata.input_info + ) + + if needs_autograd and output_and_mutation_safe: + # We realized that none of the outputs require grad, + # and none of the inputs that require grad are mutated. + # so we actually have an inference graph. + needs_autograd = False + # A bit silly: right now in the subclass codepath, our ViewAndMutationMeta + # changes depending on whether we pass in is_train / keep_input_mutations, + # so we're forced to recompute the metadata. + # TODO: refactor the subclass path of run_functionalized_fw_and_collect_metadata + # so that this is unnecessary. + if req_subclass_dispatch: + fw_metadata = run_functionalized_fw_and_collect_metadata( + flat_fn, + flat_args_descs=flat_args_descs, + keep_input_mutations=aot_config.keep_inference_input_mutations, + is_train=False, + pre_dispatch=aot_config.pre_dispatch, + static_input_indices=aot_config.static_input_indices, + )(*fake_flat_args) + else: + fw_metadata = ViewAndMutationMeta( + input_info=fw_metadata.input_info, + output_info=fw_metadata.output_info, + num_intermediate_bases=fw_metadata.num_intermediate_bases, + keep_input_mutations=aot_config.keep_inference_input_mutations, + traced_tangents=fw_metadata.traced_tangents, + traced_tangents_descs=fw_metadata.traced_tangents_descs, + subclass_inp_meta=fw_metadata.subclass_inp_meta, + subclass_fw_graph_out_meta=fw_metadata.subclass_fw_graph_out_meta, + subclass_tangent_meta=fw_metadata.subclass_tangent_meta, + is_train=False, + tokens=fw_metadata.tokens, + static_input_indices=fw_metadata.static_input_indices, + ) + + if fw_metadata.num_intermediate_bases > 0: + assert not req_subclass_dispatch, f"""\ +torch.compile is currently being used with tensor subclass inputs. +We are attempting to a compile a graph with two graph outputs +that alias one another, specifically output indices: + + {[i for i, x in enumerate(fw_metadata.output_info) if x.output_type == OutputType.alias_of_intermediate]} + +ANY output aliasing (even for regular tensors) is currently unsupported if +there are any subclass outputs. If you run into this, please file a github +issue""" + + if aot_config.is_export: + # aot_export: ban input metadata mutations for now to keep shared code paths simpler. + # Keeping .resize_() in the graph will require some work + # Allowing it but keeping the graph functional will require some calling convention changes. + if len([x for x in fw_metadata.input_info if x.mutates_metadata]) != 0: + raise RuntimeError( + f"""\ +Found an input that received a metadata mutation, through e.g. a call to `.resize_()` or `.transpose_()`. +This is currently banned in the aot_export workflow. If you need this functionality, please file a github issue. + +fw_metadata={str(fw_metadata)}""" + ) + # In export, banning data mutations on inputs that require grad for now. + # This should be rare, and is tricky to get right. When we trace the backward, + # we currently trace with autograd.grad instead of .backward(), which makes it difficult + # to ensure that we run autograd all the way through the input **before** it saw the mutation. + if ( + len( + [ + x + for x in fw_metadata.input_info + if x.requires_grad and x.mutates_data + ] + ) + != 0 + and aot_config.export_trace_joint + ): + raise RuntimeError( + f"""\ +Found a graph input that requires gradients, and received a mutation. +This is currently banned in the aot_export workflow. If you need this functionality, please file a github issue. + +fw_metadata={str(fw_metadata)}""" + ) + if req_subclass_dispatch: + raise RuntimeError( + """\ +aot_export is not currently supported with traceable tensor subclass. +If you need this feature, please comment on """ + ) + + # Need to decide on a strategy for functionalized RNG: toggling via global config seems bad, + # and turning it on will require a non-trivial calling convention change for any export runtime. + if config.functionalize_rng_ops: + raise RuntimeError( + """\ +Functionalized RNG is not currently supported in the aot_export workflow. Please file a github issue, +or otherwise set torch._functorch.config.functionalize_rng_ops = False.""" + ) + + return AOTState( + needs_autograd=needs_autograd, + flat_args=_dup_fake_script_obj(fake_flat_args), + flat_args_descs=flat_args_descs, + fw_metadata=fw_metadata, + # Packaging this just for later use + aot_config=aot_config, + stack=stack, + ) + + +def aot_function( + fn: Callable, + fw_compiler: Callable, + bw_compiler: Optional[Callable] = None, + partition_fn: Callable = default_partition, + decompositions: Optional[dict] = None, + num_params_buffers: int = 0, + keep_inference_input_mutations: bool = False, + inference_compiler: Optional[Callable] = None, + *, + # Whether or not to trace with dynamic shapes + dynamic=False, + enable_log=True, + disable_functionalization=False, +) -> Callable: + """ + Traces the forward and backward graph of :attr:`fn` using torch dispatch + mechanism, and then compiles the generated forward and backward graphs + through :attr:`fw_compiler` and :attr:`bw_compiler`. + + :func:`aot_function` traces the forward and backward graph ahead of time, + and generates a joint forward and backward graph. :attr:`partition_fn` is + then used to separate out forward and backward graphs. The partitioner + function can be used to perform optimizations such as recomputation. One can + set `decompositions` dictionary to decompose the operators into a sequence + of core or simpler operators supported by the backend compilers. + + .. warning:: + This API is experimental and likely to change. + + Args: + fn (Callable): A Python function that takes one or more arguments. Must + return one or more Tensors. + fw_compiler (Callable): A Python function that accepts an Fx graph with + Aten ops and input args, and returns a Callable that semantically is + equivalent to the input Fx graph. + bw_compiler (Optional[Callable]): A Python function that accepts an + Fx graph with Aten ops and input args, and returns a Callable that + semantically is equivalent to the input Fx graph. Default: None + (when None, it defaults to the :attr:`fw_compiler`) + partition_fn (Callable): A Python function that takes a joint forward + and backward graph, and partitions it into separate forward and + backward graphs. + decompositions (Dict): A dictionary to define the decomposition of + larger Aten ops into simpler or core Aten ops. + inference_compiler (Optional[Callable]): A Python function that accepts an + Fx graph with Aten ops and input args, and returns a Callable that + semantically is equivalent to the input Fx graph. inference_compiler is invoked + if no autograd is needed. Default: None + (when None, it defaults to the :attr:`fw_compiler`) + Returns: + Returns a ``Callable`` that retains the eager behavior of the original + :attr:`fn`, but with forward and backward graph compiled via + :attr:`fw_compile` and :attr:`bw_compile`. + + A simple example usage of :func:`aot_function` is as follows. This example + will print the forward and backward graphs of the function ``fn`` + + >>> fn = lambda x: x.sin().cos() + >>> def print_compile_fn(fx_module, args): + >>> print(fx_module) + >>> return fx_module + >>> aot_fn = aot_function(fn, print_compile_fn) + >>> x = torch.randn(4, 5, requires_grad=True) + >>> aot_fn(x) + """ + + aot_config = AOTConfig( + fw_compiler=None, + bw_compiler=None, + inference_compiler=None, + partition_fn=None, + decompositions=decompositions, + num_params_buffers=num_params_buffers, + aot_id=next(AOT_COUNTER), + keep_inference_input_mutations=keep_inference_input_mutations, + dynamic_shapes=dynamic, + aot_autograd_arg_pos_to_source=None, + is_export=False, + no_tangents=False, + enable_log=enable_log, + disable_functionalization=disable_functionalization, + ) + cached_res = None + + @wraps(fn) + def returned_function(*args, **kwargs): + nonlocal cached_res + # Now flatten the tensor args + flat_args = pytree.arg_tree_leaves(*args, **kwargs) + + # Compile the function and save it in the cache + if cached_res is None: + flat_fn, out_spec = create_tree_flattened_fn(fn, args, kwargs) + (fake_mode, shape_env) = construct_fake_mode(flat_args, aot_config) + fake_flat_args: FakifiedFlatArgs = process_inputs( + flat_args, aot_config, fake_mode, shape_env + ) + # TODO: We actually could use the pytree path to make better descs. + # Also, the descs here are bad if you do aot_module. + fake_flat_args_descs = [ + PlainAOTInput(i) for i in range(len(fake_flat_args)) + ] + with contextlib.ExitStack() as stack: + aot_state = create_aot_state( + stack, + flat_fn, + fake_flat_args, + fake_flat_args_descs, + aot_config, + fake_mode, + shape_env, + ) + aot_graph_capture = aot_stage1_graph_capture(aot_state, flat_fn) + compiled_fn, _ = aot_stage2_compile( + aot_state, + aot_graph_capture, + partition_fn, + fw_compiler, + bw_compiler, + inference_compiler, + ) + cached_res = (compiled_fn, out_spec) + + cached_fn, out_spec = cached_res + out = cached_fn(flat_args) + return out_spec.unflatten(out) + + return returned_function + + +def aot_module(mod: nn.Module, *args, **kwargs) -> nn.Module: + """ + Traces the forward and backward graph of :attr:`mod` using torch dispatch + tracing mechanism. It is wrapper function, that underneath uses + :func:`aot_function` to perform tracing and compilation. + + :func:`aot_module` lifts the parameters and buffers of ``nn.Module`` as inputs + to a new callable which is then compiled through :func:`aot_function`. + + .. warning:: + This API is experimental and likely to change. + + Args: + mod (Callable): A ``nn.Module`` module. + args : args to be passed to :func:`aot_function` + kwargs : kwargs to be passed to :func:`aot_function` + + Returns: + Returns a ``nn.Module`` that retains the eager behavior of the original + :attr:`mod`, but with forward and backward graph compiled. + + """ + # See Note: [Fake Modules and AOTAutograd] + torch._dynamo.utils.assert_no_fake_params_or_buffers(mod) + + def functional_call(named_params, named_buffers, *args, **kwargs): + params_and_buffers = {**named_params, **named_buffers} + return torch.func.functional_call(mod, params_and_buffers, args, kwargs) + + named_params = dict(mod.named_parameters(remove_duplicate=False)) + named_buffers = dict(mod.named_buffers(remove_duplicate=False)) + num_params_buffers = len(named_params) + len(named_buffers) + compiled_f = aot_function( + functional_call, *args, num_params_buffers=num_params_buffers, **kwargs + ) + + class AOTModule(nn.Module): + def __init__(self) -> None: + super().__init__() + self.orig_module = mod + + def forward(self, *args, **kwargs): + return compiled_f( + named_params, + named_buffers, + *args, + **kwargs, + ) + + return AOTModule() + + +def prepare_aot_module_simplified( + mod: nn.Module, + args, + kwargs, + decompositions: dict, + keep_inference_input_mutations, + boxed_forward_device_index: BoxedDeviceIndex, + ignore_shape_env: bool, + flatten: bool, + *, + force_non_lazy_backward_lowering: bool = False, + disable_functionalization: bool = False, + _record_nn_module_stack: bool = False, +): + if not flatten: + assert kwargs is None + elif kwargs is None: + kwargs = {} + + # TODO: There's something a bit suspicious here; typically simplified + # module shouldn't actually have any parameters... + params = dict(mod.named_parameters(remove_duplicate=False)) + buffers = dict(dict(mod.named_buffers(remove_duplicate=False))) + + params_flat, params_spec = list(params.values()), list(params.keys()) + params_len = len(params_flat) + + buffers_flat, buffers_spec = list(buffers.values()), list(buffers.keys()) + buffers_len = len(buffers_flat) + + params_buffers = {**params, **buffers} + params_buffers_flat = params_flat + buffers_flat + params_buffers_spec = params_spec + buffers_spec + + # Take a break to figure what we're doing with the module + + # NB: This doesn't change the in/out convention, except adding the + # parameters as explicit arguments + functional_call = create_functional_call( + mod, + params_buffers_spec, + params_len + buffers_len, + strict_out_tuple=not flatten, + # We need this for export to run ModuleStackTracer + # instead of PythonKeyTracer + store_orig_mod=_record_nn_module_stack, + ) + + full_args = [*params_flat, *buffers_flat, *args] + in_spec, out_spec = None, None + if flatten: + functional_call, out_spec = create_tree_flattened_fn( + functional_call, full_args, kwargs + ) + full_args, in_spec = pytree.tree_flatten((full_args, kwargs)) + + del kwargs + + # OK, set up the descs + + full_args_descs = [] + full_args_descs.extend(ParamAOTInput(fqn) for fqn in params_spec) + full_args_descs.extend(BufferAOTInput(fqn) for fqn in buffers_spec) + # TODO: it would be better to put pytree information in here + full_args_descs.extend( + PlainAOTInput(i) for i in range(len(full_args) - len(full_args_descs)) + ) + + # TODO: These tracing_context fields should become unnecessary once we + # always maintain sources on all arguments + if tracing_context := torch._guards.TracingContext.try_get(): + # NB: TracingContext misnames this, the "params" here also contains + # buffers + tracing_context.params_flat = params_buffers_flat + ( + tracing_context.params_flat_unwrap_subclasses, + tracing_context.params_unwrapped_to_flat_index, + ) = unwrap_tensor_subclasses_with_indices_to_original(params_buffers_flat) + + # TODO: Might be nice to hold on to the Dynamo source here in full_args_descs! + ( + aot_autograd_arg_pos_to_source, + static_input_indices, + ) = _try_get_metadata_from_dynamo( + mod, params_buffers.keys(), len(full_args), full_args_descs + ) + + dynamic_shapes = False + for x in full_args: + if isinstance(x, FakeTensor): + dynamic_shapes = x.fake_mode.shape_env is not None + break + + aot_config = AOTConfig( + fw_compiler=None, + bw_compiler=None, + inference_compiler=None, + partition_fn=None, + decompositions=decompositions, + num_params_buffers=params_len + buffers_len, + aot_id=next(AOT_COUNTER), + keep_inference_input_mutations=keep_inference_input_mutations, + dynamic_shapes=dynamic_shapes, + aot_autograd_arg_pos_to_source=aot_autograd_arg_pos_to_source, + static_input_indices=static_input_indices, + is_export=False, + no_tangents=False, + cache_info=None, + ignore_shape_env=ignore_shape_env, + precompile_backend_id=getattr(mod, "_backend_id", None), + force_non_lazy_backward_lowering=force_non_lazy_backward_lowering, + disable_functionalization=False, + ) + fake_mode, shape_env = construct_fake_mode(full_args, aot_config) + # NB: full_args_descs not needed here, fake_flat_args is 1:1 with full_args + fake_flat_args = process_inputs( + full_args, aot_config, fake_mode, shape_env, ignore_shape_env + ) + + return ( + functional_call, + params_buffers_flat, + params_spec, + buffers_spec, + fake_flat_args, + full_args_descs, + aot_config, + fake_mode, + shape_env, + in_spec, + out_spec, + ) + + +def aot_module_simplified( + mod: nn.Module, + args, + fw_compiler: AOTDispatchCompiler, + bw_compiler: Optional[AOTDispatchCompiler] = None, + partition_fn: Callable = default_partition, + decompositions: Optional[dict] = None, + keep_inference_input_mutations=False, + inference_compiler: Optional[AOTDispatchCompiler] = None, + # TODO: This doesn't seem to be used in any nontrivial way, check if it's + # actually needed + cudagraphs: Optional[BoxedBool] = None, + boxed_forward_device_index: Optional[BoxedDeviceIndex] = None, + ignore_shape_env: bool = False, + disable_functionalization: bool = False, +) -> nn.Module: + """ + This is the simplified or low overhead version of aot_module. For frontends + like TorchDynamo, the input functions/modules to AOT are static and have + unpacked inputs/outputs. This gives us an opportunity to remove the + (1) pytree overhead to parse inputs/outputs, + (2) AOT Autograd cache, + (3) Reading of params/buffers in every forward call + + :func:`aot_module_simplified` removes these overheads. + """ + + if cudagraphs is None: + cudagraphs = BoxedBool(torch._inductor.config.triton.cudagraphs) + + with contextlib.ExitStack() as stack: + ( + functional_call, + params_buffers_flat, + _params_spec, + _buffers_spec, + fake_flat_args, + full_args_descs, + aot_config, + fake_mode, + shape_env, + _in_spec, + _out_spec, + ) = prepare_aot_module_simplified( + mod, + args, + None, + decompositions, + keep_inference_input_mutations, + boxed_forward_device_index, + ignore_shape_env, + flatten=False, + force_non_lazy_backward_lowering=config.force_non_lazy_backward_lowering, + disable_functionalization=disable_functionalization, + ) + + compiled_fn = None + + if isinstance(fw_compiler, SerializableAOTDispatchCompiler): + local = should_use_local_autograd_cache() + remote = should_use_remote_autograd_cache() + if local or remote: + set_feature_use("aot_autograd_remote_cache", remote) + compiled_fn = AOTAutogradCache.try_load( + mod, + fake_flat_args, + aot_config, + cudagraphs, + boxed_forward_device_index, + local, + remote, + ) + + if compiled_fn is None: + stack.enter_context(compiled_autograd._disable()) + aot_state = create_aot_state( + stack, + functional_call, + fake_flat_args, + full_args_descs, + aot_config, + fake_mode, + shape_env, + ) + aot_graph_capture = aot_stage1_graph_capture(aot_state, functional_call) + compiled_fn, _ = aot_stage2_compile( + aot_state, + aot_graph_capture, + partition_fn, + fw_compiler, + bw_compiler, + inference_compiler, + ) + + if isinstance(mod, torch._dynamo.utils.GmWrapper): + # This function is called by the flatten_graph_inputs wrapper, which boxes + # the inputs so that they can be freed before the end of this scope. + # For overhead reasons, this is not the default wrapper, see comment: + # https://github.com/pytorch/pytorch/pull/122535/files#r1560096481 + @simple_wraps(compiled_fn) + def forward(runtime_args: list[Any]): + flat_args = [] + flat_args.extend(params_buffers_flat) + flat_args.extend(runtime_args) + runtime_args.clear() + return compiled_fn(flat_args) + + else: + # TODO: There is something deeply wrong here; compiled_fn running with + # the boxed calling convention, but aot_module_simplified somehow + # historically returned a function that was not the boxed calling + # convention. This should get fixed... + # NB: GraphModule/nn.Module rely on the non-boxed calling convention here + @simple_wraps(compiled_fn) + def forward(*runtime_args: tuple[Any]): + full_args = [] + full_args.extend(params_buffers_flat) + full_args.extend(runtime_args) + return compiled_fn(full_args) + + # Just for convenience + forward.zero_grad = mod.zero_grad + forward.named_parameters = mod.named_parameters + forward.named_buffers = mod.named_buffers + + # Add a serialize function + def grab_serialize_fn(fn): + if isinstance(fn, SerializableCompiledFunction): + return fn.serialize_fn + elif hasattr(fn, "__wrapped__"): + return grab_serialize_fn(fn.__wrapped__) + else: + return None + + forward.serialize = grab_serialize_fn(forward) # type: ignore[attr-defined] + return forward + + +def boxed_nop_preserve_node_meta(fx_g, example_inputs): + def run(args): + with torch.fx.traceback.preserve_node_meta(): + return torch.fx.Interpreter(fx_g).boxed_run(args) + + run._boxed_call = True + return run + + +def aot_export_joint_with_descriptors( + stack: contextlib.ExitStack, + mod: nn.Module, + args, + kwargs=None, + *, + decompositions: Optional[dict] = None, + keep_inference_input_mutations=False, + ignore_shape_env=False, + disable_functionalization=False, + _record_nn_module_stack=False, +) -> JointWithDescriptors: + """ + This API captures the joint graph for an nn.Module. However, unlike + aot_export_joint_simple or aot_export_module(trace_joint=True), the + calling convention of the produced joint graph follows no fixed positional + schema; for example, you cannot rely on the second argument of the traced + joint graph to correspond to the second argument of the module you traced. + However, the inputs and outputs of the traced graph are schematized + with **descriptors**, annotated on meta['desc'] on the placeholder and + return FX nodes, which you can use to determine the meaning of arguments. + + The major benefit of using this export rather than aot_export_joint_simple + is that we have feature parity with all situations that torch.compile + supports (via aot_module_simplified), including handling for more + complicated cases such as multiple differentiable outputs, input mutations + that must be handled outside of the graph, tensor subclasses, etc. + + What can you do with one of these joint graphs with descriptors? The + motivating use case (autoparallel) involves taking the joint graph, doing + optimizations on it, and then turning it back into a callable so it can be + torch.compile'd at a later point in time. This cannot be done as a + traditional torch.compile joint graph pass for two reasons: + + 1. The sharding of parameters must be decided before parameter + initialization / checkpoint load, far before torch.compile would + ordinarily run. + + 2. We need to change the meaning of parameters (e.g., we might replace + a replicated parameter with a sharded version of it, changing its + input size). torch.compile is ordinarily semantics preserving, and + not allowed to change the meaning of inputs. + + Some descriptors can be quite exotic, so we recommend thinking carefully + if there is a safe fallback you can apply to descriptors you don't understand. + For example, you should have some way to handle not finding a particular + input exactly as is in the final FX graph inputs. + + Note: When using this API, you must create and enter an ExitStack context + manager, which will be passed into this function. This context manager + must remain active if you call the compile function to finish compilation. + (TODO: We may relax this requirement by having AOTAutograd keep track of + how to reconstruct all the context managers at a later point in time.) + + NB: You're not obligated to do a /full/ compile in stage2; instead you can + leave the forward/backward compilers unspecified in which case the + partitioned FX graphs will directly run. The overall autograd Function + can be allowed in graph so you can reprocess it in the context of a + (potentially larger) compiled region later. + + NB: These APIs do NOT hit cache, as we only ever cache the final compile results, + not the intermediate export result. + + NB: If the passed nn.Module has parameters and buffers on it, we will + generate extra implicit parameter/buffer arguments and assign ParamAOTInput + and BufferAOTInput descriptors to them. However, if you generate the input + nn.Module from a mechanism like Dynamo, you will NOT get these descriptors + (because Dynamo will already have taken care of lifting the parameters/buffers + into arguments!) In that case, it would be necessary to analyze the Sources + of the inputs to determine if inputs are parameters and their FQNs. + """ + + ( + functional_call, + _params_buffers_flat, + params_spec, + buffers_spec, + fake_flat_args, + full_args_descs, + aot_config, + fake_mode, + shape_env, + in_spec, + out_spec, + ) = prepare_aot_module_simplified( + mod, + args, + kwargs, + # In contrast, decompositions are needed at this stage. + decompositions, + keep_inference_input_mutations, + None, + ignore_shape_env, + flatten=True, + # Without this, we will attempt to "compile" the backward lazily + # at runtime, but this is pointless because it's just boxed_nop, + # it's trivial. But this will get Inductor confused about scoping + # Metric(s) {'is_forward'} have already been set in the current + # context. + force_non_lazy_backward_lowering=True, + disable_functionalization=disable_functionalization, + _record_nn_module_stack=_record_nn_module_stack, + ) + + # TODO: Maybe this should be in create_aot_state? Not sure, that would + # increase its scope + stack.enter_context(compiled_autograd._disable()) + + aot_state = create_aot_state( + stack, + functional_call, + fake_flat_args, + full_args_descs, + aot_config, + fake_mode, + shape_env, + ) + + # NB: no cache lookup! + aot_graph_capture = aot_stage1_graph_capture(aot_state, functional_call) + + assert out_spec.spec is not None + + return JointWithDescriptors( + _aot_state=aot_state, + _aot_graph_capture=aot_graph_capture, + params_spec=params_spec, + buffers_spec=buffers_spec, + in_spec=in_spec, + out_spec=out_spec.spec, + ) + + +def aot_compile_joint_with_descriptors( + jd: JointWithDescriptors, + *, + partition_fn: Callable = default_partition, + fw_compiler: Optional[AOTDispatchCompiler] = boxed_nop_preserve_node_meta, + bw_compiler: Optional[AOTDispatchCompiler] = boxed_nop_preserve_node_meta, +) -> callable: + """ + Companion function for aot_export_joint_with_descriptors which compiles the joint + graph into a callable function that follows a standard calling convention. + params_flat all are arguments. + + Note: We do NOT instantiate the module; this gives you the flexibility to subclass it and + customize its behavior without having to worry about FQN rebinding. + + TODO: Consider if we should allow_in_graph the result by default. + """ + compiled_fn, _ = aot_stage2_compile( + jd._aot_state, + jd._aot_graph_capture, + partition_fn, + fw_compiler, + bw_compiler, + ) + + # Cribbed from torch/export/pt2_archive/_package.py + @simple_wraps(compiled_fn) + @torch._dynamo.nonstrict_trace # allow recursive compilation + def unflattened_compiled_fn(*args, **kwargs): + flat_inputs = pytree.tree_flatten((args, reorder_kwargs(kwargs, jd.in_spec)))[0] + # TODO: do I need to filter? I hope not! + flat_outputs = compiled_fn(flat_inputs) + return pytree.tree_unflatten(flat_outputs, jd.out_spec) + + return unflattened_compiled_fn + + +def aot_export_module( + mod: nn.Module, + args, + *, + decompositions: Optional[dict] = None, + # If true, we'll return a joint forward-backward graph, + # As well as metadata on the loss + gradients in the backward. + trace_joint: bool, + # If trace_joint is True, we expect your module to return a scalar loss. + # Your module can return multiple outputs, so you must specify which output the loss is. + output_loss_index: Optional[int] = None, + pre_dispatch: bool = False, + # If None, will be inferred from inputs and mod.graph.nodes if mod is a graph module, but the inferred result might be wrong. + dynamic_shapes: Optional[bool] = None, + kwargs=None, +) -> tuple[torch.fx.GraphModule, GraphSignature]: + """ + This function takes in a module, and returns: + (1) an FX graph that can be exported + (2) some metadata about the graph + + If `trace_joint=True` we will return a joint graph of the forward + backward. + + The traced FX graph will have the following properties compared to the original module: + (1) Inputs and outputs to the module will be pytree-flattened + (2) Parameters and buffers on the module will be lifted into graph inputs, + graph_inputs = (*parameters, *buffers, *user_inputs) + (3) The graph will be fully functionalized + (4) Any input mutations will be converted into additional outputs in the graph, + meaning whoever calls this graph is responsible for applying the mutations + back to the original inputs. + (5) If is_joint is provided the graph will return parameter gradients in addition to user outputs. + The graph output will look like: + graph_outputs = (*updated_inputs, *user_outputs, *param_gradients) + + There are also several restrictions on what modules can use this API. In particular: + (1) If trace_joint is specified, we expect the loss function to be **fused** + into the module forward. One of the outputs to the forward must be a scalar loss, + which is specified with `output_loss_index`. + All other outputs to the forward are presumed to not require gradients. + (2) This API cannot capture optimizers (although in theory we could build an API for this). + (3) Metadata mutations on params/buffers/inputs are banned. + (4) Data mutations on anything that requires gradients are banned (parameters) + (5) If an input is mutated, it is not allowed to alias any other inputs. + (6) Parameters must not be duplicated. + """ + if pre_dispatch and trace_joint: + raise RuntimeError("pre_dispatch is not supported when trace_joint is True.") + named_parameters = dict(mod.named_parameters(remove_duplicate=False)) + named_buffers = dict(mod.named_buffers(remove_duplicate=False)) + + params_and_buffers = { + **dict(named_parameters), + **dict(named_buffers), + } + params_and_buffers_flat, params_spec = pytree.tree_flatten(params_and_buffers) + params_and_buffers_flat = tuple(params_and_buffers_flat) + params_len = len(params_and_buffers_flat) + + kwargs = kwargs or {} + + functional_call = create_functional_call( + mod, params_spec, params_len, store_orig_mod=True + ) + + num_fw_outs = None + + if trace_joint: + # This helper effectively just adds some extra asserts about what the backward will look like: + # Outputs must include a scalar loss, that we compute gradients w.r.t. + # We don't compute gradients w.r.t. anything else: so just in case we detach() + # and other output tensors. + def fn_to_trace(*args): + nonlocal num_fw_outs + out = functional_call(*args) + if output_loss_index is None: + raise RuntimeError( + """\ +If trace_joint=Trueit is required that one of your forward outputs must be a scalar loss. +You must specify the which (index) output is the loss with output_loss_index.""" + ) + if isinstance(out, (torch.Tensor)): + out = (out,) + if not isinstance(out, (tuple, list)): + raise RuntimeError( + f"Expected forward output to be either a tensor or a list/tuple of tensors. found {type(out)}" + ) + + for i, o in enumerate(out): + # We only want to create a backward graph w.r.t. the loss that the user passed in. + # This implies that every other output should not require gradients. + # Instead of making this an error (and forcing the user to detach all other outputs + # of their forward), + # we'll automatically detach them here. + if o.requires_grad and i != output_loss_index: + raise RuntimeError( + f"""\ +Found an output of the forward that requires gradients, that was not the scalar loss. +We require all outputs to the forward that are not the scalar loss to not require gradient, +because we will only compute a backward graph against the scalar loss. +You can fix this by calling .detach() on each of your forward outputs that is not the loss. +You specified that output index {output_loss_index} is the loss, but we found that +the output at index {i} requires gradients.""" + ) + out_loss = out[output_loss_index] + num_fw_outs = len(out) + if not out_loss.requires_grad: + raise RuntimeError( + f"""\ +The output at index {output_loss_index} was marked as the loss, but it does not require gradients""" + ) + if out_loss.numel() != 1: + raise RuntimeError( + f"""\ +We require the output marked as the loss (at index {output_loss_index}) to be a scalar, but it has shape {out_loss.shape}""" + ) + return out + + ctx = nullcontext + else: + # Run under no_grad, so our tracing machinery only traces an inference graph. + # However if pre_dispatch=True, we want to correctly trace set_grad_enabled calls for training. + ctx = nullcontext if pre_dispatch else torch.no_grad + fn_to_trace = functional_call + + full_args = [] + # First, the params + # NB: It is REQUIRED that parameters come first, Inductor infers "fixed" + # parameters by looking at the difference in parameter count outside + # and inside AOTAutograd, and assumes the prefix of arguments are fixed + # arguments + full_args.extend(params_and_buffers_flat) + # Next, the input args + full_args.extend(args) + + with ctx(): + fx_g, metadata, in_spec, out_spec = _aot_export_function( + fn_to_trace, + full_args, + decompositions=decompositions, + num_params_buffers=params_len, + no_tangents=True, + pre_dispatch=pre_dispatch, + dynamic_shapes=dynamic_shapes, + trace_joint=trace_joint, + kwargs=kwargs, + ) + + # TODO: subsume this path with the aot_stage2_graph_capture path + if trace_joint: + + @wraps(functional_call) + def flattened_joint(*args): + # The idea here is that the joint graph that AOTAutograd creates has some strict properties: + # (1) It accepts two arguments (primals, tangents), and pytree_flattens them + # (2) It returns a tuple of (fw_outs, gradients) + # This is a very useful convention for anyone who wants to partition the joint graph + # into a separate forward and backward graph. + # However, + # (1) for people exporting a single joint graph, it would be preferable not to have + # any pytrees in the graph. + # (2) We are guaranteed in the aot_export_module case that the forward outputs a loss, + # and there are therefore no tangents that are needed to run the joint graph. + # (3) AOTAutograd creates a grad_input for every input in the forward, + # including None's for inputs that are not grad-requiring tensors. + # we don't want these in our export graph. + # and there are therefore no tangents that are needed to run the joint graph. + # This function "fixes" both of the above by removing any tangent inputs, + # and removing pytrees from the original FX graph. + fake_tangents = [ + None + for _ in range( + metadata.num_outputs + metadata.num_mutated_inp_runtime_indices + ) + ] + fw_outs, gradients = fx_g(args, fake_tangents) + assert len(gradients) == len(args) + output_gradients = [] + for a, grad in zip(args, gradients): + if isinstance(a, torch.Tensor) and a.requires_grad: + assert grad is not None, """\ +Found a parameter that did not receive a gradient. +"This is most likely a bug, but if this needs to be supported please comment on this Github issue: +https://github.com/pytorch/pytorch/issues/101192 +""" + output_gradients.append(grad) + else: + assert grad is None + return *fw_outs, *output_gradients + + fx_g = make_fx(flattened_joint, record_module_stack=True)(*full_args) + + user_args_flat = pytree.arg_tree_leaves(*args, **kwargs) + return fx_g, create_graph_signature( + fx_g, + metadata, + in_spec, + out_spec, + user_args_flat=user_args_flat, + params_and_buffers_flat=params_and_buffers_flat, + param_names=list(named_parameters.keys()), + buffer_names=list(named_buffers.keys()), + trace_joint=trace_joint, + num_user_fw_outs=num_fw_outs, + loss_index=output_loss_index, + ) + + +def aot_export_joint_simple( + func: Callable, + args, + *, + trace_joint: bool, + # It looks like the main consequence of this API is that for dynamic shapes, + # it will assume that params/buffers are static. + # With the new inferred dynamic shapes API, maybe this doesn't matter? + num_params_buffers: int = 0, + decompositions: Optional[dict] = None, +) -> torch.fx.GraphModule: + """ + A simplified version of export. Used by higher order operators. + + This function makes a high-level "no calling convention changes" guarantee: + - If no inputs require grad (so we export an inference graph), + there are *no* calling convention change between the exported graph, and "func". + - If at least one input requires grad (so we trace out and export a joint fw-bw graph), + Then if you were partition the graph into a separate forward and backward graph, + The forward graph will have no calling convention changes compared to "func". + + The above also relies on some strong restrictions around which functions this API accepts: + (1) `args` cannot contain any pytrees (they must have been pytree_flattened already) + (2) `func` cannot mutate any inputs + (3) The outputs of `func` cannot alias any inputs. + + Note: this function is only lightly tested today. It will probably be tested more heavily by higher order ops. + """ + if trace_joint: + ctx = nullcontext + else: + # Run under no_grad, so our tracing machinery only traces an inference graph. + ctx = torch.no_grad + + with ctx(): + fx_g, metadata, in_spec, out_spec = _aot_export_function( + func, + args, + decompositions=decompositions, + trace_joint=trace_joint, + ) + in_spec, _kw_in_spec = in_spec.children() + # At this point, we can just directly return the (joint or inference graph) that we traced. + # First though: a bunch of assertions to make sure that our graph doesn't require + # any calling convention changes compared to the original function. + # These restrictions are *in addition to* the general restrictions on export. + + # No input mutations + if ( + len([x for x in metadata.input_info if x.mutates_data or x.mutates_metadata]) + != 0 + ): + raise RuntimeError( + f"aot_export_joint_simple does not support input mutations. {str(metadata)}" + ) + # No output aliasing + if ( + len([x for x in metadata.output_info if x.output_type != OutputType.non_alias]) + != 0 + ): + raise RuntimeError( + f"aot_export_joint_simple does not support outputs that alias inputs. {str(metadata)}" + ) + # No pytrees + if in_spec.is_leaf(): + raise RuntimeError( + f"aot_export_joint_simple requires inputs to be a single list/tuple. in_spec={str(in_spec)}" + ) + if not all(child.is_leaf() for child in in_spec.children()): + raise RuntimeError( + f"aot_export_joint_simple requires individual inputs not to be pytrees. in_spec={str(in_spec)}" + ) + if out_spec.is_leaf(): + raise RuntimeError( + f"aot_export_joint_simple requires outputs to be a single list/tuple. out_spec={str(out_spec)}" + ) + if not all(child.is_leaf() for child in out_spec.children()): + raise RuntimeError( + f"aot_export_joint_simple requires individual outputs not to be pytrees. out_spec={str(out_spec)}" + ) + # TODO: we might have to temporarily patch config.functionalize_rng + # so that it doesn't run when we're exporting a higher order op. + + if config.debug_assert: + # Smoke test that after partitioning, we can run the forward without any calling convention changes. + fw_module, _bw_module = aot_config.default_partition( # noqa: F821 + fx_g, + args, + num_fwd_outputs=len(fw_metadata.output_infos), # noqa: F821 + ) + # Attempt to run the fw_module with the original user inputs + fake_mode = detect_fake_mode(args) + if fake_mode is None: + fake_mode = FakeTensorMode() + with fake_mode: + fw_module(*args) + return fx_g + + +# Private for now because we aren't providing a contract on what to return +# for joint graphs (we could when there's a clearer use case) +# In the future, we may need to add more export API's that provide their own strong guarantees. +# This is meant as a general helper function for handling various export-y use cases. +def _aot_export_function( + func: Callable, + args, + *, + num_params_buffers: int = 0, + decompositions: Optional[dict] = None, + # If we're exporting a joint graph and we don't want any tangent inputs in the graph + # (because we are backpropping through a scalar 1 loss), + # we need to explicitly specify not to include tangents in the graph. + # It's not enough just to check that our tangent is a scalar, since we also + # need to know if it is a 1 (no need to make it a graph input), or something else + # (requiring it to be a graph input). + # We don't know this info at trace time though, so we need to make it an explicit config. + no_tangents: bool = False, + pre_dispatch: bool = False, + # If None, `dynamic_shapes` will be inferred from inputs, but the inferred result might be wrong. + dynamic_shapes: Optional[bool] = None, + keep_input_mutations: bool = False, + # Under export, configures whether we are getting inference or training IR + trace_joint: bool = False, + kwargs=None, +) -> tuple[torch.fx.GraphModule, ViewAndMutationMeta, pytree.TreeSpec, pytree.TreeSpec]: + kwargs = kwargs or {} + + flat_fn, out_spec = create_tree_flattened_fn(func, args, kwargs) + flat_args, in_spec = pytree.tree_flatten((args, kwargs)) + + fake_mode = None + if dynamic_shapes is None: + # Try to infer `dynamic_shapes from inputs and graph nodes + fake_mode = detect_fake_mode(flat_args) + if ( + fake_mode is None + and hasattr(func, "_orig_mod") + and isinstance(func._orig_mod, torch.fx.GraphModule) + ): + vals = [ + node.meta["val"] + for node in func._orig_mod.graph.nodes + if "val" in node.meta + ] + fake_mode = detect_fake_mode(vals) + dynamic_shapes = fake_mode is not None and fake_mode.shape_env is not None + + # The export use case doesn't care about several bits of AOTConfig + # (1) compilers (we just export the graph) + # (2) partitioners (export is only full graph, user can partition themselves) + aot_config = AOTConfig( + fw_compiler=None, + bw_compiler=None, + inference_compiler=None, + partition_fn=None, + decompositions=decompositions, + num_params_buffers=num_params_buffers, + aot_id=next(AOT_COUNTER), + # For now there's no use case involving keeping input mutations in the graph + # (which we can only do in the inference case anyway). + # We can add this later if we need to. + keep_inference_input_mutations=keep_input_mutations, + dynamic_shapes=dynamic_shapes, + aot_autograd_arg_pos_to_source=None, + is_export=True, + no_tangents=no_tangents, + pre_dispatch=pre_dispatch, + export_trace_joint=trace_joint, + ) + if fake_mode is None: + fake_mode, shape_env = construct_fake_mode(flat_args, aot_config) + else: + shape_env = fake_mode.shape_env + fake_flat_args = process_inputs(flat_args, aot_config, fake_mode, shape_env) + # TODO: Improve the descs here with pytree information + fake_flat_args_descs = [PlainAOTInput(i) for i in range(len(fake_flat_args))] + + with contextlib.ExitStack() as stack: + aot_state = create_aot_state( + stack, + flat_fn, + fake_flat_args, + fake_flat_args_descs, + aot_config, + fake_mode, + shape_env, + ) + aot_graph_capture = aot_stage1_graph_capture(aot_state, flat_fn) + fx_g, meta = aot_stage2_export(aot_state, aot_graph_capture) + + return fx_g, meta, in_spec, out_spec.spec + + +compiled_function = aot_function +compiled_module = aot_module diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/apis.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/apis.py new file mode 100644 index 0000000000000000000000000000000000000000..1faa767d4d05c53381b65d54d1b7b715b8bb77bf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/apis.py @@ -0,0 +1,456 @@ +# mypy: allow-untyped-defs +# NOTE: We allow Dynamo to see this file (via torch/_dynamo/trace_rules.py) so that it can +# trace through functorch transforms. +# Currently, we can't allow Dynamo to see `eager_transforms.py`/`vmap.py` as that break a lot of thing +# and there isn't a mechanism to selectively expose only some functions (eg. grad) from a file +# to Dynamo. +import functools + +from torch._functorch.utils import argnums_t, exposed_in +from torch._functorch.vmap import ( + _check_out_dims_is_int_or_int_pytree, + _check_randomness_arg, + _chunked_vmap, + _process_batched_inputs, + Callable, + in_dims_t, + out_dims_t, + vmap_impl, +) + + +# vmap(func)(inputs) wraps all Tensor inputs to be batched in BatchedTensors, +# sends those into func, and then unwraps the output BatchedTensors. Operations +# on BatchedTensors perform the batched operations that the user is asking for. +# +# vmap's randomness behavior differs from JAX's, which would require a PRNG key +# to be passed everywhere. + + +@exposed_in("torch.func") +def vmap( + func: Callable, + in_dims: in_dims_t = 0, + out_dims: out_dims_t = 0, + randomness: str = "error", + *, + chunk_size=None, +) -> Callable: + """ + vmap is the vectorizing map; ``vmap(func)`` returns a new function that + maps ``func`` over some dimension of the inputs. Semantically, vmap + pushes the map into PyTorch operations called by ``func``, effectively + vectorizing those operations. + + vmap is useful for handling batch dimensions: one can write a function + ``func`` that runs on examples and then lift it to a function that can + take batches of examples with ``vmap(func)``. vmap can also be used to + compute batched gradients when composed with autograd. + + .. note:: + :func:`torch.vmap` is aliased to :func:`torch.func.vmap` for + convenience. Use whichever one you'd like. + + Args: + func (function): A Python function that takes one or more arguments. + Must return one or more Tensors. + in_dims (int or nested structure): Specifies which dimension of the + inputs should be mapped over. ``in_dims`` should have a + structure like the inputs. If the ``in_dim`` for a particular + input is None, then that indicates there is no map dimension. + Default: 0. + out_dims (int or Tuple[int]): Specifies where the mapped dimension + should appear in the outputs. If ``out_dims`` is a Tuple, then + it should have one element per output. Default: 0. + randomness (str): Specifies whether the randomness in this + vmap should be the same or different across batches. If 'different', + the randomness for each batch will be different. If 'same', the + randomness will be the same across batches. If 'error', any calls to + random functions will error. Default: 'error'. WARNING: this flag + only applies to random PyTorch operations and does not apply to + Python's random module or numpy randomness. + chunk_size (None or int): If None (default), apply a single vmap over inputs. + If not None, then compute the vmap :attr:`chunk_size` samples at a time. + Note that :attr:`chunk_size=1` is equivalent to computing the vmap with a for-loop. + If you run into memory issues computing the vmap, please try a non-None chunk_size. + + Returns: + Returns a new "batched" function. It takes the same inputs as + ``func``, except each input has an extra dimension at the index + specified by ``in_dims``. It takes returns the same outputs as + ``func``, except each output has an extra dimension at the index + specified by ``out_dims``. + + .. warning: + :func:`vmap` works best with functional-style code. Please do not + perform any side-effects in ``func``, with the exception of + in-place PyTorch operations. Examples of side-effects include mutating + Python data structures and assigning values to variables not captured + in ``func``. + + One example of using :func:`vmap` is to compute batched dot products. PyTorch + doesn't provide a batched ``torch.dot`` API; instead of unsuccessfully + rummaging through docs, use :func:`vmap` to construct a new function. + + >>> torch.dot # [D], [D] -> [] + >>> batched_dot = torch.func.vmap(torch.dot) # [N, D], [N, D] -> [N] + >>> x, y = torch.randn(2, 5), torch.randn(2, 5) + >>> batched_dot(x, y) + + :func:`vmap` can be helpful in hiding batch dimensions, leading to a simpler + model authoring experience. + + >>> batch_size, feature_size = 3, 5 + >>> weights = torch.randn(feature_size, requires_grad=True) + >>> + >>> def model(feature_vec): + >>> # Very simple linear model with activation + >>> return feature_vec.dot(weights).relu() + >>> + >>> examples = torch.randn(batch_size, feature_size) + >>> result = torch.vmap(model)(examples) + + :func:`vmap` can also help vectorize computations that were previously difficult + or impossible to batch. One example is higher-order gradient computation. + The PyTorch autograd engine computes vjps (vector-Jacobian products). + Computing a full Jacobian matrix for some function f: R^N -> R^N usually + requires N calls to ``autograd.grad``, one per Jacobian row. Using :func:`vmap`, + we can vectorize the whole computation, computing the Jacobian in a single + call to ``autograd.grad``. + + >>> # Setup + >>> N = 5 + >>> f = lambda x: x**2 + >>> x = torch.randn(N, requires_grad=True) + >>> y = f(x) + >>> I_N = torch.eye(N) + >>> + >>> # Sequential approach + >>> jacobian_rows = [torch.autograd.grad(y, x, v, retain_graph=True)[0] + >>> for v in I_N.unbind()] + >>> jacobian = torch.stack(jacobian_rows) + >>> + >>> # vectorized gradient computation + >>> def get_vjp(v): + >>> return torch.autograd.grad(y, x, v) + >>> jacobian = torch.vmap(get_vjp)(I_N) + + :func:`vmap` can also be nested, producing an output with multiple batched dimensions + + >>> torch.dot # [D], [D] -> [] + >>> batched_dot = torch.vmap( + ... torch.vmap(torch.dot) + ... ) # [N1, N0, D], [N1, N0, D] -> [N1, N0] + >>> x, y = torch.randn(2, 3, 5), torch.randn(2, 3, 5) + >>> batched_dot(x, y) # tensor of size [2, 3] + + If the inputs are not batched along the first dimension, ``in_dims`` specifies + the dimension that each inputs are batched along as + + >>> torch.dot # [N], [N] -> [] + >>> batched_dot = torch.vmap(torch.dot, in_dims=1) # [N, D], [N, D] -> [D] + >>> x, y = torch.randn(2, 5), torch.randn(2, 5) + >>> batched_dot( + ... x, y + ... ) # output is [5] instead of [2] if batched along the 0th dimension + + If there are multiple inputs each of which is batched along different dimensions, + ``in_dims`` must be a tuple with the batch dimension for each input as + + >>> torch.dot # [D], [D] -> [] + >>> batched_dot = torch.vmap(torch.dot, in_dims=(0, None)) # [N, D], [D] -> [N] + >>> x, y = torch.randn(2, 5), torch.randn(5) + >>> batched_dot( + ... x, y + ... ) # second arg doesn't have a batch dim because in_dim[1] was None + + If the input is a Python struct, ``in_dims`` must be a tuple containing a struct + matching the shape of the input: + + >>> f = lambda dict: torch.dot(dict["x"], dict["y"]) + >>> x, y = torch.randn(2, 5), torch.randn(5) + >>> input = {"x": x, "y": y} + >>> batched_dot = torch.vmap(f, in_dims=({"x": 0, "y": None},)) + >>> batched_dot(input) + + By default, the output is batched along the first dimension. However, it can be batched + along any dimension by using ``out_dims`` + + >>> f = lambda x: x**2 + >>> x = torch.randn(2, 5) + >>> batched_pow = torch.vmap(f, out_dims=1) + >>> batched_pow(x) # [5, 2] + + For any function that uses kwargs, the returned function will not batch the kwargs but will + accept kwargs + + >>> x = torch.randn([2, 5]) + >>> def fn(x, scale=4.): + >>> return x * scale + >>> + >>> batched_pow = torch.vmap(fn) + >>> assert torch.allclose(batched_pow(x), x * 4) + >>> batched_pow(x, scale=x) # scale is not batched, output has shape [2, 2, 5] + + .. note:: + vmap does not provide general autobatching or handle variable-length + sequences out of the box. + """ + from torch.compiler import is_compiling + + _check_randomness_arg(randomness) + if not (chunk_size is None or chunk_size > 0): + raise ValueError( + f"vmap: chunk_size should be None or greater than 0. (got {chunk_size})" + ) + + def wrapped(*args, **kwargs): + return vmap_impl( + func, in_dims, out_dims, randomness, chunk_size, *args, **kwargs + ) + + if not is_compiling(): + wrapped = functools.wraps(func)(wrapped) + + return wrapped + + +def chunk_vmap( + func: Callable, + in_dims: in_dims_t = 0, + out_dims: out_dims_t = 0, + randomness: str = "error", + chunks=2, +) -> Callable: + """ + chunk_vmap is the vectorizing map (vmap) using chunks of input data. It is a mix of vmap (which vectorizes + everything) and map (which executes things sequentially). ``chunk_vmap`` vectorizes the input with number of + chunks at a time. For more details about vectorizing map, see :func:`vmap`. + + .. note:: + Please use :func:`vmap` with ``chunk_size`` argument instead of this API. + + Args: + func (function): A Python function that takes one or more arguments. + Must return one or more Tensors. + in_dims (int or nested structure): Specifies which dimension of the + inputs should be mapped over. ``in_dims`` should have a + structure like the inputs. If the ``in_dim`` for a particular + input is None, then that indicates there is no map dimension. + Default: 0. + out_dims (int or Tuple[int]): Specifies where the mapped dimension + should appear in the outputs. If ``out_dims`` is a Tuple, then + it should have one element per output. Default: 0. + randomness (str): Specifies whether the randomness in this + vmap should be the same or different across batches. If 'different', + the randomness for each batch will be different. If 'same', the + randomness will be the same across batches. If 'error', any calls to + random functions will error. Default: 'error'. WARNING: this flag + only applies to random PyTorch operations and does not apply to + Python's random module or numpy randomness. + chunks (int): Number of chunks to use to split the input data. Default is 2. + If equals to 1 then :func:`vmap` is called. + + Returns: + Returns a new "batched" function. It takes the same inputs as + ``func``, except each input has an extra dimension at the index + specified by ``in_dims``. It takes returns the same outputs as + ``func``, except each output has an extra dimension at the index + specified by ``out_dims``. + """ + _check_randomness_arg(randomness) + + if chunks == 1: + return vmap(func, in_dims=in_dims, out_dims=out_dims, randomness=randomness) + + def _get_chunk_flat_args(flat_args_, flat_in_dims_, chunks_): + flat_args_chunks = tuple( + t.chunk(chunks_, dim=in_dim) + if in_dim is not None + else [ + t, + ] + * chunks_ + for t, in_dim in zip(flat_args_, flat_in_dims_) + ) + # transpose chunk dim and flatten structure + # chunks_flat_args is a list of flatten args + chunks_flat_args = zip(*flat_args_chunks) + return chunks_flat_args + + @functools.wraps(func) + def wrapped_with_chunks(*args, **kwargs): + _check_out_dims_is_int_or_int_pytree(out_dims, func) + _, flat_in_dims, flat_args, args_spec = _process_batched_inputs( + in_dims, args, func + ) + # Chunk flat arguments + chunks_flat_args = _get_chunk_flat_args(flat_args, flat_in_dims, chunks) + + # Apply vmap on chunks + return _chunked_vmap( + func, + flat_in_dims, + chunks_flat_args, + args_spec, + out_dims, + randomness, + **kwargs, + ) + + return wrapped_with_chunks + + +@exposed_in("torch.func") +def grad(func: Callable, argnums: argnums_t = 0, has_aux: bool = False) -> Callable: + """``grad`` operator helps computing gradients of ``func`` with respect to the + input(s) specified by ``argnums``. This operator can be nested to + compute higher-order gradients. + + Args: + func (Callable): A Python function that takes one or more arguments. + Must return a single-element Tensor. If specified ``has_aux`` equals ``True``, + function can return a tuple of single-element Tensor and other auxiliary objects: + ``(output, aux)``. + argnums (int or Tuple[int]): Specifies arguments to compute gradients with respect to. + ``argnums`` can be single integer or tuple of integers. Default: 0. + has_aux (bool): Flag indicating that ``func`` returns a tensor and other + auxiliary objects: ``(output, aux)``. Default: False. + + Returns: + Function to compute gradients with respect to its inputs. By default, the output of + the function is the gradient tensor(s) with respect to the first argument. + If specified ``has_aux`` equals ``True``, tuple of gradients and output auxiliary objects + is returned. If ``argnums`` is a tuple of integers, a tuple of output gradients with + respect to each ``argnums`` value is returned. + + Example of using ``grad``: + + >>> # xdoctest: +SKIP + >>> from torch.func import grad + >>> x = torch.randn([]) + >>> cos_x = grad(lambda x: torch.sin(x))(x) + >>> assert torch.allclose(cos_x, x.cos()) + >>> + >>> # Second-order gradients + >>> neg_sin_x = grad(grad(lambda x: torch.sin(x)))(x) + >>> assert torch.allclose(neg_sin_x, -x.sin()) + + When composed with ``vmap``, ``grad`` can be used to compute per-sample-gradients: + + >>> # xdoctest: +SKIP + >>> from torch.func import grad, vmap + >>> batch_size, feature_size = 3, 5 + >>> + >>> def model(weights, feature_vec): + >>> # Very simple linear model with activation + >>> assert feature_vec.dim() == 1 + >>> return feature_vec.dot(weights).relu() + >>> + >>> def compute_loss(weights, example, target): + >>> y = model(weights, example) + >>> return ((y - target) ** 2).mean() # MSELoss + >>> + >>> weights = torch.randn(feature_size, requires_grad=True) + >>> examples = torch.randn(batch_size, feature_size) + >>> targets = torch.randn(batch_size) + >>> inputs = (weights, examples, targets) + >>> grad_weight_per_example = vmap(grad(compute_loss), in_dims=(None, 0, 0))( + ... *inputs + ... ) + + Example of using ``grad`` with ``has_aux`` and ``argnums``: + + >>> # xdoctest: +SKIP + >>> from torch.func import grad + >>> def my_loss_func(y, y_pred): + >>> loss_per_sample = (0.5 * y_pred - y) ** 2 + >>> loss = loss_per_sample.mean() + >>> return loss, (y_pred, loss_per_sample) + >>> + >>> fn = grad(my_loss_func, argnums=(0, 1), has_aux=True) + >>> y_true = torch.rand(4) + >>> y_preds = torch.rand(4, requires_grad=True) + >>> out = fn(y_true, y_preds) + >>> # > output is ((grads w.r.t y_true, grads w.r.t y_preds), (y_pred, loss_per_sample)) + + .. note:: + Using PyTorch ``torch.no_grad`` together with ``grad``. + + Case 1: Using ``torch.no_grad`` inside a function: + + >>> # xdoctest: +SKIP + >>> def f(x): + >>> with torch.no_grad(): + >>> c = x ** 2 + >>> return x - c + + In this case, ``grad(f)(x)`` will respect the inner ``torch.no_grad``. + + Case 2: Using ``grad`` inside ``torch.no_grad`` context manager: + + >>> # xdoctest: +SKIP + >>> with torch.no_grad(): + >>> grad(f)(x) + + In this case, ``grad`` will respect the inner ``torch.no_grad``, but not the + outer one. This is because ``grad`` is a "function transform": its result + should not depend on the result of a context manager outside of ``f``. + + """ + # To avoid cyclical dependency. + import torch._functorch.eager_transforms as eager_transforms + from torch.compiler import is_compiling + + def wrapper(*args, **kwargs): + return eager_transforms.grad_impl(func, argnums, has_aux, args, kwargs) + + if not is_compiling(): + wrapper = functools.wraps(func)(wrapper) + + return wrapper + + +@exposed_in("torch.func") +def grad_and_value( + func: Callable, argnums: argnums_t = 0, has_aux: bool = False +) -> Callable: + """ + Returns a function to compute a tuple of the gradient and primal, or + forward, computation. + + Args: + func (Callable): A Python function that takes one or more arguments. + Must return a single-element Tensor. If specified ``has_aux`` + equals ``True``, function can return a tuple of single-element + Tensor and other auxiliary objects: ``(output, aux)``. + argnums (int or Tuple[int]): Specifies arguments to compute gradients + with respect to. ``argnums`` can be single integer or tuple of + integers. Default: 0. + has_aux (bool): Flag indicating that ``func`` returns a tensor and + other auxiliary objects: ``(output, aux)``. Default: False. + + Returns: + Function to compute a tuple of gradients with respect to its inputs + and the forward computation. By default, the output of the function is + a tuple of the gradient tensor(s) with respect to the first argument + and the primal computation. If specified ``has_aux`` equals + ``True``, tuple of gradients and tuple of the forward computation with + output auxiliary objects is returned. If ``argnums`` is a tuple of + integers, a tuple of a tuple of the output gradients with respect to + each ``argnums`` value and the forward computation is returned. + + See :func:`grad` for examples + """ + from torch._functorch import eager_transforms + from torch.compiler import is_compiling + + def wrapper(*args, **kwargs): + return eager_transforms.grad_and_value_impl( + func, argnums, has_aux, args, kwargs + ) + + if not is_compiling(): + wrapper = functools.wraps(func)(wrapper) + + return wrapper diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/autograd_function.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/autograd_function.py new file mode 100644 index 0000000000000000000000000000000000000000..ca7376cf9620c209e427ad6d814a8caaccc98264 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/autograd_function.py @@ -0,0 +1,787 @@ +# mypy: allow-untyped-defs +from typing import NamedTuple + +import torch +import torch.utils._pytree as pytree +from torch._C._functorch import ( + _unwrap_for_grad, + _wrap_for_grad, + current_level, + TransformType, +) +from torch._functorch.apis import vmap +from torch._functorch.utils import enable_single_level_autograd_function +from torch._functorch.vmap import ( + _add_batch_dim, + _broadcast_to_and_flatten, + restore_vmap, + unwrap_batched, + wrap_batched, +) +from torch._ops import HigherOrderOperator +from torch.autograd.forward_ad import _set_fwd_grad_enabled + + +# autograd.Function technically runs before the regular PyTorch dispatcher. +# This is how features like autocast and torch_dispatch (e.g. PythonTLSSnapshot) +# work with it. One day we might decide to change this, but until then, +# we need to give the illusion that autograd.Function runs before those things. +# +# We do this by using creating a custom HigherOrderOperator that only functorch +# dispatches specially. +class CustomFunctionHigherOrderOperator(HigherOrderOperator): + def __init__(self) -> None: + super().__init__("custom_function_call") + + def __call__(self, autograd_function, *args, **kwargs): + # When custom_function_call is done dispatching through functorch, + # it should just invoke the autograd.Function. This is consistent + # with the autograd.Function behavior of being invoked before the + # PyTorch dispatcher. + # + # This will lead us into trouble later down the line, but this is + # pre-existing. There is an invariant that a function traced by + # make_fx should have the same behavior when provided the same + # Tensor. However, make_fx sees autograd.Function as a composite + # (because autograd.Function happens before the Python dispatch key) + # and only traces the forward pass. + if torch._C._are_functorch_transforms_active(): + return super().__call__(autograd_function, *args, **kwargs) + return autograd_function.apply(*args, **kwargs) + + +# "custom_function_call" +# This is the mechanism for an autograd.Function that works with functorch transforms. +# It wraps an autograd.Function; interactions with functorch transforms are defined +# via PyDispatcher and HigherOrderOperator rather than through the traditional PyTorch +# dispatcher. +custom_function_call = CustomFunctionHigherOrderOperator() + + +# The grad rule for custom_function_call is to construct a new _SingleLevelFunction +# (autograd.Function that only works with a single layer (level) of functorch) that: +# - unwraps the inputs +# - redispatches to custom_function_call +# - wraps the outputs +# and whose backward pass calls the original autograd.Function's backward. +# +# Why do we need to redispatch to custom_function_call? +# ----------------------------------------------------- +# This is consistent with how ATen operators work with functorch's grad transform: +# they always redispatch to the original operator. +# Consider torch.sin, and let's say we do grad0(grad1(torch.sin))(x) +# +# grad1 will: +# - set up the autograd graph +# - unwrap the inputs +# - redispatch to at::sin (*) +# - rewrap the outputs on the return +# +# On the redispatch in (*), grad0 will: +# - set up the autograd graph +# - unwrap the inputs +# - redispatch to at::sin +# - rewrap the outputs on the return +# +# To "set up the autograd graph", we generate a _SingleLevelFunction +# and apply it. +@custom_function_call.py_impl(TransformType.Grad) +@custom_function_call.py_impl(TransformType.Jvp) +def custom_function_call_grad(interpreter, autograd_function, *operands): + Generated = generate_single_level_function(interpreter, autograd_function) + with enable_single_level_autograd_function(): + flat_out = Generated.apply(*operands) + return flat_out + + +def generate_single_level_function(interpreter, autograd_function): + level = interpreter.level() + + def forward(*operands): + unwrapped_operands = pytree.tree_map_only( + torch.Tensor, lambda x: _unwrap_for_grad(x, level), operands + ) + # Both enable_grad() and _set_fwd_grad_enabled() are necessary no matter + # the transform. _SingleLevelFunction will turn off both fwd and bwd + # gradient computation and we need to turn it back on here. + with torch.enable_grad(), _set_fwd_grad_enabled(True), interpreter.lower(): + unwrapped_output = custom_function_call( + autograd_function, *unwrapped_operands + ) + + # See NOTE [mark_dirty object identity check] + def wrap_fn(output): + return _wrap_for_grad(output, level) + + return wrap_outputs_maintaining_identity( + unwrapped_output, unwrapped_operands, operands, wrap_fn + ) + + def setup_context(ctx, inputs, output): + return autograd_function.setup_context(ctx, inputs, output) + + # backward is only used if the transform is TransformType.Grad + def backward(ctx, *grads): + result = autograd_function.backward(ctx, *grads) + return result + + # jvp is only used if the transform is TransformType.Jvp + def jvp(ctx, *tangents): + result = autograd_function.jvp(ctx, *tangents) + return result + + # This is the sequence of magic words to dynamically generate a Subclass with + # a given name. A Tensor's .grad_fn field has a class name that is the original + # autograd.Function's name + Backward, so we do this to generate some + # meaningful name. + name = f"{autograd_function.__name__}Generated" + Generated = type( + name, + (torch.autograd.function._SingleLevelFunction,), + { + "forward": staticmethod(forward), + "backward": staticmethod(backward), + "jvp": staticmethod(jvp), + "setup_context": staticmethod(setup_context), + }, + ) + return Generated + + +# wrap_outputs_maintaining_identity handles outputs from the vmap, +# backward (vjp), and jvp staticmethod. The way it distinguishes +# between the vmap case and the {backward, jvp} case is if the out_dims +# are specified or not. +# +# NB: we cannot use out_dims=None as the deciding factor. This because +# out_dims=None can still happen in the vmap staticmethod! What the +# user is saying in that case is that their output does not have a +# dimension that is being vmapped over, which is valid. +NO_OUT_DIMS = "not specified" + + +# NOTE [mark_dirty object identity check] +# autograd.Function's ctx.mark_dirty expect a returned input +# to have the same object identity as the input. +# Mode-only functorch will greatly simplify this logic. +def wrap_outputs_maintaining_identity( + outputs, unwrapped_inputs, orig_inputs, wrap_fn, out_dims=NO_OUT_DIMS +): + flat_unwrapped_inputs = pytree.arg_tree_leaves(*unwrapped_inputs) + flat_orig_inputs = pytree.arg_tree_leaves(*orig_inputs) + + unwrapped_input_to_orig_input = { + id(unwrapped): orig + for unwrapped, orig in zip(flat_unwrapped_inputs, flat_orig_inputs) + } + + flat_outputs, spec = pytree.tree_flatten(outputs) + result = [] + + out_dims_specified = out_dims != NO_OUT_DIMS + + if out_dims_specified: + flat_out_dims = _broadcast_to_and_flatten(out_dims, spec) + # _broadcast_to_and_flatten returns None if it is unable to broadcast. + # TODO: update following link from master to stable once that's out + if flat_out_dims is None: + raise RuntimeError( + f"The autograd.Function's vmap staticmethod returned an " + f"incompatible (output, out_dims) tuple. " + f"Expected out_dims={out_dims} " + f"to be compatible with the structure of `output`. " + f"out_dims has structure {pytree.tree_flatten(out_dims)[1]} " + f"but output has structure {spec}. " + f"For more details, please see " + f"https://pytorch.org/docs/main/notes/extending.func.html" + ) + + for i, output in enumerate(flat_outputs): + if not isinstance(output, torch.Tensor): + result.append(output) + continue + if id(output) in unwrapped_input_to_orig_input: + result.append(unwrapped_input_to_orig_input[id(output)]) + continue + if out_dims_specified: + result.append(wrap_fn(output, flat_out_dims[i])) # type: ignore[possibly-undefined, index] + else: + result.append(wrap_fn(output)) + + return pytree.tree_unflatten(result, spec) + + +# NOTE: [functorch vjp and autograd interaction] +# There's an edge case with the functorch vjp and autograd interaction +# that will eventually be fixed by mode-only functorch. +# The TL;DR is that there's no way to unwrap a dead GradTensorWrapper, +# so we (the framework) need to do it manually. Regular PyTorch operators +# automatically do so this is consistent. +# +# class MyExp(torch.autograd.Function): +# @staticmethod +# def forward(x): +# return x.exp() +# +# @staticmethod +# def setup_context(ctx, inputs, output): +# y = output +# ctx.save_for_backward(y) +# +# @staticmethod +# def backward(gy): +# y, = ctx.saved_tensors() +# return MyMul.apply(gy, y) +# +# x = torch.randn([], requires_grad=True) +# gy = torch.randn([], requires_grad=True) +# _, vjp_fn = vjp(MySin.apply, x) +# result = vjp_fn(gy) +# +# MyMul is an autograd.Function that is not shown here. +# It saves a `y` for backward (since gy requires grad). +# +# in vjp_fn(gy), we get: +# > MyMul.apply(gy, GradTensorWrapper(y, level=dead)) +# Because the y that is saved for backward by MyExp is a GradTensorWrapper +# but is now dead since we are outside the vjp context. +# +# PyTorch dispatcher operations, upon seeing a dead GradTensorWrapper, +# will automatically unwrap the GradTensorWrapper when applied. +# But since autograd.Function technically sits above the regular PyTorch +# dispatcher, it doesn't get this treatment. So we manually do +# the unwrapping to be consistent with regular PyTorch dispatcher operations. + + +class VmapInfo(NamedTuple): + batch_size: int + randomness: str + + +def has_overridden_vmap_rule(autograd_function): + return autograd_function.vmap is not torch.autograd.Function.vmap + + +def validate_vmap_returns_tuple_of_two_elements(result): + base_error_msg = ( + "Expected the vmap staticmethod to have two returns, an output " + "and out_dims with pytree structure compatible with the output. " + ) + if not isinstance(result, tuple): + raise RuntimeError(base_error_msg + f"Got a {type(result)} instead") + if not len(result) == 2: + raise RuntimeError(base_error_msg + f"Got {len(result)} returns instead") + + +@custom_function_call.py_impl(TransformType.Vmap) +def custom_function_call_vmap(interpreter, autograd_function, *operands, **kwargs): + if any( + isinstance(val, torch.Tensor) + for val in torch.utils._pytree.tree_flatten(kwargs)[0] + ): + raise NotImplementedError( + f"Run vmap on autograd.Function with kwarg-only Tensor args. " + f"Please do not pass kwarg-only Tensors to autograd.Function. " + f"Got: {kwargs}" + ) + + if autograd_function.generate_vmap_rule: + if has_overridden_vmap_rule(autograd_function): + # TODO: Update link to stable once that's out + # https://github.com/pytorch/pytorch/issues/92029 + raise RuntimeError( + f"You tried to vmap over {autograd_function.__name__}, but " + f"it has both generate_vmap_rule=True and an overridden vmap " + f"staticmethod. Please set generate_vmap_rule=False or delete " + f"the overridden vmap staticmethod to avoid ambiguity. " + f"For more details, please see " + f"https://pytorch.org/docs/main/notes/extending.func.html" + ) + return custom_function_call_vmap_generate_rule( + interpreter, autograd_function, *operands + ) + + if not has_overridden_vmap_rule(autograd_function): + # TODO: Update link to stable once that's out + # https://github.com/pytorch/pytorch/issues/92029 + raise RuntimeError( + f"You tried to vmap over {autograd_function.__name__}, but " + f"it does not have vmap support. Please override and implement the " + f"vmap staticmethod or set generate_vmap_rule=True. " + f"For more details, please see " + f"https://pytorch.org/docs/main/notes/extending.func.html" + ) + + return custom_function_call_vmap_helper( + interpreter, autograd_function.vmap, autograd_function, *operands, **kwargs + ) + + +def custom_function_call_vmap_helper( + interpreter, vmap_function, op, *operands, **kwargs +): + current_level = interpreter.level() + info = VmapInfo( + batch_size=interpreter.batch_size(), + randomness=interpreter.randomness(), + ) + # We're either in the autograd.Function case (vmap staticmethod) + # or the torch.library.register_vmap case. + autograd_function_case = isinstance(op, torch.autograd.function.FunctionMeta) + + def lower_to_next(): + if autograd_function_case: + return interpreter.lower() + else: + return torch._C._ExcludeDispatchKeyGuard( + torch._C.DispatchKeySet(torch._C.DispatchKey.FuncTorchBatched) + ) + + unwrapped_operands, in_dims = unwrap_batched(operands, current_level) + # If none of the tensors are batched at the current level, then we skip the + # current level. This saves the user from needing to handle this case in + # their vmap staticmethod (and is consistent with our C++ batching rule API) + if pytree.tree_all(lambda dim: dim is None, in_dims): + with lower_to_next(): + if autograd_function_case: + return custom_function_call(op, *operands) + else: + return op(*operands, **kwargs) + + with lower_to_next(): + result = vmap_function(info, in_dims, *unwrapped_operands, **kwargs) + validate_vmap_returns_tuple_of_two_elements(result) + unwrapped_output, out_dims = result + + # See NOTE [mark_dirty object identity check] + def wrap_fn(output, out_dim): + return ( + output + if out_dim is None + else _add_batch_dim(output, out_dim, current_level) + ) + + return wrap_outputs_maintaining_identity( + unwrapped_output, unwrapped_operands, operands, wrap_fn, out_dims=out_dims + ) + + +def unpack_outputs(outputs): + out_dims = outputs[-1] + if isinstance(out_dims, tuple): + outputs = outputs[:-1] + else: + outputs = outputs[0] + return outputs, out_dims + + +def custom_function_call_vmap_generate_rule(interpreter, autograd_function, *operands): + unwrapped_operands, in_dims = unwrap_batched(operands, interpreter.level()) + vmapped_function = vmapify_autograd_function( + autograd_function, in_dims, interpreter.batch_size(), interpreter.randomness() + ) + with interpreter.lower(): + outputs = custom_function_call(vmapped_function, *unwrapped_operands) + + assert isinstance(outputs, tuple) + outputs, out_dims = unpack_outputs(outputs) + return wrap_batched(outputs, out_dims, interpreter.level()) + + +@custom_function_call.py_impl(TransformType.Functionalize) +def custom_function_call_functionalize( + interpreter, autograd_function, generate_vmap_rule, *operands +): + raise RuntimeError("NYI: Functionalize rule for custom_function_call") + + +def vmapify_autograd_function(autograd_function, in_dims, batch_size, randomness): + def forward(*operands): + outputs, out_dims = restore_vmap( + autograd_function.forward, in_dims, batch_size, randomness + )(*operands) + if isinstance(outputs, torch.Tensor): + return outputs, out_dims + else: + return *outputs, out_dims + + def setup_context(ctx, inputs, outputs): + outputs, out_dims = unpack_outputs(outputs) + key = id(Generated) + + def inner(inputs, outputs): + # wrapped_ctx.save_for_backward will: + # - unwrap batchedtensors into (tensor, bdim) + # - save_for_backward(*unwrapped_tensors) + # - assign the bdims to wrapped_ctx._pt_saved_tensors_bdims + wrapped_ctx = CtxCustomSave(ctx, current_level()) + autograd_function.setup_context(wrapped_ctx, inputs, outputs) + + # input_shapes are used for reductify later to reduce expanded gradients + # to the correct shape. + # See NOTE: [Why can't we rely on autograd to reduce expanded gradients?] + # for more details + input_shapes = tuple( + inp.shape if isinstance(inp, torch.Tensor) else None for inp in inputs + ) + if not hasattr(ctx, "_pt_input_shapes"): + ctx._pt_input_shapes = {} + ctx._pt_input_shapes.update({key: input_shapes}) + + if not hasattr(ctx, "_pt_saved_tensors_bdims_stack"): + ctx._pt_saved_tensors_bdims_stack = {} + ctx._pt_saved_tensors_bdims_stack.update( + {key: (wrapped_ctx._pt_saved_tensors_bdims)} + ) + + # See NOTE: [Why do we need to run setup_context under a vmap?] + restore_vmap( + inner, + (in_dims, out_dims), + batch_size, + randomness, + )(inputs, outputs) + + if not hasattr(ctx, "_pt_out_dims"): + ctx._pt_out_dims = {} + ctx._pt_out_dims.update({key: out_dims}) + + def jvp(ctx, *tangents): + key = id(Generated) + + def jvp_no_context(saved_tensors, tangents): + wrapped_ctx = CtxWithSavedTensors(ctx, saved_tensors) + return autograd_function.jvp(wrapped_ctx, *tangents) + + tangent_in_dims = get_tangents_in_dims(in_dims, tangents) + out_tangents, out_tangents_dims = restore_vmap( + jvp_no_context, + (ctx._pt_saved_tensors_bdims_stack[key], tangent_in_dims), + batch_size, + randomness, + )(ctx.saved_tensors, tangents) + + result = reductify( + out_tangents, out_tangents_dims, ctx._pt_out_dims[key], batch_size + ) + if isinstance(result, torch.Tensor): + return result, None + else: + return *result, None + + def backward(ctx, *grad_outputs): + key = id(Generated) + grad_outputs_ = grad_outputs[:-1] + grad_outputs_in_dims = ctx._pt_out_dims[key] + + if not isinstance(grad_outputs_in_dims, tuple): + grad_outputs_in_dims = (grad_outputs_in_dims,) + + grad_outputs_in_dims = tuple( + in_dim if grad_output is not None else None + for grad_output, in_dim in zip(grad_outputs_, grad_outputs_in_dims) + ) + + def backward_no_context(inputs): + saved_tensors, grad_outputs = inputs + wrapped_ctx = CtxWithSavedTensors(ctx, saved_tensors) + return autograd_function.backward(wrapped_ctx, *grad_outputs) + + grad_ins, grad_ins_dims = restore_vmap( + backward_no_context, + ((ctx._pt_saved_tensors_bdims_stack[key], grad_outputs_in_dims),), + batch_size, + randomness, + )((ctx.saved_tensors, grad_outputs_)) + result = reductify( + grad_ins, grad_ins_dims, in_dims, batch_size, ctx._pt_input_shapes[key] + ) + return result + + name = f"Vmapped{autograd_function.__name__}" + Generated = type( + name, + (torch.autograd.Function,), + { + "forward": staticmethod(forward), + "backward": staticmethod(backward), + "jvp": staticmethod(jvp), + "setup_context": staticmethod(setup_context), + "generate_vmap_rule": True, + }, + ) + + return Generated + + +# tangents might be None, so we need to replace +# the corresponding in_dims with None. +def get_tangents_in_dims(input_dims, tangents): + flat_in_dims, spec = pytree.tree_flatten(input_dims) + flat_tangents = pytree.arg_tree_leaves(*tangents) + result = [ + None if tangent is None else in_dim + for in_dim, tangent in zip(flat_in_dims, flat_tangents) + ] + return pytree.tree_unflatten(result, spec) + + +# NOTE: [Why do we need to run setup_context under a vmap?] +# Consider the following autograd.Function +# +# class Sum(torch.autograd.Function): +# @staticmethod +# def forward(x): +# return x.sum() +# @staticmethod +# def setup_context(ctx, inputs, outputs): +# ctx.x_shape = inputs[0] +# @staticmethod +# def backward(ctx, gy): +# return gy.expand(ctx.x_shape) +# +# x = torch.randn(B, 4) +# in_dims = 0 +# vmap(Sum.apply, in_dims)(x) +# +# Let's assume for a moment that we didn't vmap setup_context in VmappedSum: +# +# class VmappedSum(torch.autograd.Function): +# @staticmethod +# def forward(x): +# return vmap(Sum.forward, in_dims)(x) +# +# @staticmethod +# def setup_context(ctx, inputs, outputs): +# Sum.setup_context(ctx, inputs, outputs) +# +# @staticmethod +# def backward(ctx, gy): +# def backward_no_context(gy): +# return gy.expand(ctx.x_shape) +# +# dims = (0,) +# gx = vmap(backward_no_context, dims)(gy) +# return gx +# +# We end up saving [B, 4] as x_shape. In the backward, gy has shape [B], +# and we're doing: +# +# def backward_no_context(gy): +# return gy.expand([B, 4]) +# +# gx = vmap(backward_no_context, dims)(gy: "Tensor[B]") +# +# This gives us the wrong result (gx has shape [B, B, 4], but it should +# have shape [4]). Performing vmap over setup_context means the shape +# saved has shape [4] and leads to a correct result shape for gx. + + +# Wraps a ctx object. Forwards all attr accesses to the underlying object +# except for the attrs in _pt_attrs +class WrappedCtx: + _pt_reserved_attrs: tuple[str, ...] = ("_pt_reserved_attrs", "_pt_inner_ctx") + + def __init__(self, ctx): + if not isinstance(ctx, WrappedCtx): + reserved_attrs = type(self)._pt_reserved_attrs + for name in reserved_attrs: + if not hasattr(ctx, name): + continue + raise RuntimeError( + f"PyTorch reserves the {reserved_attrs} field on ctx. " + "Please name your fields on ctx something else to avoid name " + "collision." + ) + self._pt_inner_ctx = ctx + + def __getattr__(self, name): + return getattr(self._pt_inner_ctx, name) + + def __setattr__(self, name, value): + if name in type(self)._pt_reserved_attrs: + self.__dict__[name] = value + return + return setattr(self._pt_inner_ctx, name, value) + + +# Wraps ctx to create a new ctx object that overrides saved_tensors. +class CtxWithSavedTensors(WrappedCtx): + _pt_reserved_attrs = ("_pt_new_saved_tensors", *WrappedCtx._pt_reserved_attrs) + + def __init__(self, ctx, new_saved_tensors): + super().__init__(ctx) + self._pt_new_saved_tensors = new_saved_tensors + + @property + def saved_tensors(self): + return self._pt_new_saved_tensors + + +class CtxCustomSave(WrappedCtx): + _pt_reserved_attrs = ( + "_pt_saved_tensors_bdims", + "_pt_current_level", + *WrappedCtx._pt_reserved_attrs, + ) + + def __init__(self, ctx, current_level): + super().__init__(ctx) + self._pt_saved_tensors_bdims = () + self._pt_current_level = current_level + + def save_for_backward(self, *tensors): + unwrapped_tensors, bdims = unwrap_batched(tensors, self._pt_current_level) + self._pt_inner_ctx.save_for_backward(*unwrapped_tensors) + self._pt_saved_tensors_bdims = bdims + + def save_for_forward(self, *tensors): + unwrapped_tensors, bdims = unwrap_batched(tensors, self._pt_current_level) + self._pt_inner_ctx.save_for_forward(*unwrapped_tensors) + self._pt_saved_tensors_bdims = bdims + + +def reductify( + grad_input, + grad_input_bdim, + input_bdim, + batch_size, + target_shape_without_bdim_to_reduce_to=None, +): + if not isinstance(grad_input, tuple): + grad_input = (grad_input,) + if not isinstance(grad_input_bdim, tuple): + grad_input_bdim = (grad_input_bdim,) + if not isinstance(input_bdim, tuple): + input_bdim = (input_bdim,) + + if target_shape_without_bdim_to_reduce_to is None: + target_shape_without_bdim_to_reduce_to = len(grad_input) * (None,) + result = tuple( + reductify_leaf(gi, gi_bdim, i_bdim, batch_size, maybe_ishape) + for gi, gi_bdim, i_bdim, maybe_ishape in zip( + grad_input, + grad_input_bdim, + input_bdim, + target_shape_without_bdim_to_reduce_to, + ) + ) + return result + + +def reductify_leaf( + grad_input, + grad_input_bdim, + input_bdim, + batch_size, + target_shape_without_bdim_to_reduce_to=None, +): + if grad_input is None: + return None + + if grad_input_bdim is None and input_bdim is None: + return grad_input + + if grad_input_bdim is not None and input_bdim is None: + return grad_input.sum(grad_input_bdim) + + # NOTE: [Why can't we rely on autograd to reduce expanded gradients?] + # For reverse-mode AD, + # given a grad_input and input, it is valid for the user to return a + # grad_input that has a broadcasted shape when compared to the input. + # In this situation, autograd automatically reduces the grad_input to + # the shape of the input. + # + # However, when input_bdim is not None, we have problems. + # + # [example 1] + # grad_input: Tensor[3, 4], input: Tensor[B, 4] + # We can expand grad_input to Tensor[B, 3, 4], but that isn't broadcastable + # from [B, 4]. + # + # [example 2] + # grad_input: Tensor[3, B, 4], input: Tensor[B, 4] + # We can swizzle grad_input to Tensor[B, 3, 4], but that isn't broadcastable + # from [B, 4]. + # + # This means that we need to also reduce the grad_input to the shape of the + # input. This behavior is controlled by the `target_shape_without_bdim_to_reduce_to` flag; + # if not-None then we do the reducing manually, otherwise, we do not do a reduction. + assert input_bdim is not None + + if grad_input_bdim is None: + grad_input = grad_input.unsqueeze(input_bdim) + new_shape = list(grad_input.shape) + new_shape[input_bdim] = batch_size + grad_input = grad_input.expand(new_shape) + grad_input_bdim = input_bdim + + if target_shape_without_bdim_to_reduce_to is not None: + return vmap( + torch.Tensor.sum_to_size, + in_dims=(grad_input_bdim, None), + out_dims=input_bdim, + )(grad_input, target_shape_without_bdim_to_reduce_to) + + if input_bdim != grad_input_bdim: + grad_input = grad_input.movedim(grad_input_bdim, input_bdim) + return grad_input + + +def autograd_function_forward_rewritten(original_forward, original_setup_context): + def new_forward(ctx, *args, **kwargs): + output = original_forward(*args, **kwargs) + original_setup_context(ctx, args, output) + return output + + return new_forward + + +class AutogradFunctionApply(HigherOrderOperator): + def __init__(self) -> None: + super().__init__("autograd_function_apply") + + def __call__(self, fwd, bwd, *fwd_args, **fwd_kwargs): + saved_values = None + args_tensor_mask = fwd_kwargs["args_tensor_mask"] + non_differentiable_idx = fwd_kwargs["non_differentiable_idx"] + length_of_tensor_args = sum(args_tensor_mask) + # Filter out the original tensor args from fwd_args, + # lifted freevars should not be args of ApplyTemplate.apply + # since we don't need to calculate the gradients of them. + new_fwd_args = fwd_args[:length_of_tensor_args] + + class ApplyTemplate(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(ctx, *args): + nonlocal saved_values + + # The Interpreter here is required to propagate metadata + # from the dynamo graph body to the local_map graph body. + # This is required for fx_traceback.annotate for work. + output, saved_values = torch.fx.Interpreter(fwd).run(None, *fwd_args) + + # If users call ctx.mark_non_differentiable() in the original fwd function. + if len(non_differentiable_idx) > 0: + non_differentiable_output = [] + for i, x in enumerate(output): + if i in non_differentiable_idx: + non_differentiable_output.append(x) + ctx.mark_non_differentiable(*non_differentiable_output) + + return output + + @staticmethod + def backward(ctx, *grad): + # The Interpreter here is required to propagate metadata + # from the dynamo graph body to the local_map graph body. + # This is required for fx_traceback.annotate for work. + + # pyrefly: ignore [not-iterable] + return torch.fx.Interpreter(bwd).run(None, *grad, *saved_values) + + return ApplyTemplate.apply(*new_fwd_args) + + +autograd_function_apply = AutogradFunctionApply() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/batch_norm_replacement.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/batch_norm_replacement.py new file mode 100644 index 0000000000000000000000000000000000000000..77aa9b9c2d7c78647d2c850b550754e43c4fe592 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/batch_norm_replacement.py @@ -0,0 +1,27 @@ +import torch.nn as nn +from torch._functorch.utils import exposed_in + + +def batch_norm_without_running_stats(module: nn.Module) -> None: + if ( + isinstance(module, nn.modules.batchnorm._BatchNorm) + and module.track_running_stats + ): + module.running_mean = None + module.running_var = None + module.num_batches_tracked = None + module.track_running_stats = False + + +@exposed_in("torch.func") +def replace_all_batch_norm_modules_(root: nn.Module) -> nn.Module: + """ + In place updates :attr:`root` by setting the ``running_mean`` and ``running_var`` to be None and + setting track_running_stats to be False for any nn.BatchNorm module in :attr:`root` + """ + # base case + batch_norm_without_running_stats(root) + + for obj in root.modules(): + batch_norm_without_running_stats(obj) + return root diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/benchmark_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/benchmark_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..596f1e7c00dc555759d07af9e46797ac70069b9d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/benchmark_utils.py @@ -0,0 +1,235 @@ +# mypy: ignore-errors + +import contextlib +import json +import operator +import os +import time + +import torch +from torch.profiler import profile, ProfilerActivity + + +def synchronize(): + pass + + +def dump_chrome_trace( + f, + input, + trace_filename, + optimize_ctx, + activities, + num_runs=1, + devices=None, + kwargs_for_f=None, + kwargs_for_profiler=None, +): + """ + Output the chrome trace of running f(input, **kwargs_for_f) with [optimize_ctx] + [num_runs] times to [trace_filename]. + + [activities] are the activities that the profiler will record, e.g. ProfilerActivity.CUDA. + Return total runtime without the profiler + + Outputs to trace_filename + """ + + if devices is None: + devices = ["cuda"] + + global synchronize + if devices != ["cpu"] and torch.cuda.is_available(): + synchronize = torch.cuda.synchronize + + if kwargs_for_f is None: + kwargs_for_f = {} + if kwargs_for_profiler is None: + kwargs_for_profiler = {} + + with optimize_ctx: + torch.manual_seed(1337) + for _ in range(5): # warmup runs + f(input, **kwargs_for_f) + synchronize() + torch.manual_seed(1337) + t0 = time.perf_counter() + for _ in range(num_runs): + f(input, **kwargs_for_f) + synchronize() + t1 = time.perf_counter() + timing = t1 - t0 + + with profile(activities=activities, **kwargs_for_profiler) as prof: + with optimize_ctx: + synchronize() + torch.manual_seed(1337) + for _ in range(num_runs): + f(input, **kwargs_for_f) + synchronize() + prof.export_chrome_trace(trace_filename) + + return timing + + +def get_chrome_trace_events(filename): + with open(filename) as f: + data = json.load(f) + events = data["traceEvents"] + return events + + +def is_gpu_compute_event(event): + global gpu_pids + return ( + "pid" in event + and event["pid"] in gpu_pids + and "ph" in event + and event["ph"] == "X" + ) + + +def get_sorted_gpu_events(events): + sorted_gpu_events = [] + for event in events: + if not is_gpu_compute_event(event): + continue + sorted_gpu_events.append(event) + return sorted(sorted_gpu_events, key=operator.itemgetter("ts")) + + +def get_duration(sorted_gpu_events): + if len(sorted_gpu_events) == 0: + return 0 + event = sorted_gpu_events[0] + current_end_time = event["ts"] + event["dur"] + total_duration = event["dur"] + for event in sorted_gpu_events[1:]: + start_time = max(event["ts"], current_end_time) + end_time = event["ts"] + event["dur"] + total_duration = total_duration + max(end_time - start_time, 0) + current_end_time = max(current_end_time, end_time) + return total_duration + + +def get_sorted_gpu_mm_conv_events(events): + def is_mm_conv_event(event): + return "name" in event and ( + "gemm" in event["name"] + or "conv" in event["name"] + or "cutlass" in event["name"] + or "wgrad" in event["name"] + ) + + gpu_events = get_sorted_gpu_events(events) + sorted_events = [] + for event in gpu_events: + if not is_mm_conv_event(event): + continue + sorted_events.append(event) + return sorted_events + + +gpu_pids = [] + + +def compute_utilization(filename: str, total_length: float): + """ + Process the chrome traces outputs by the pytorch profiler to compute GPU Utilization + and percent of times spent on matmul and convolution + + Args: + filename(str): Name of chrome traces file produced by pytorch profiler + + total_length(float): total length of the process without profiler in second + + Return: + tuple: (GPU Utilization, percent of time spent on matmul and convolution) + """ + events = get_chrome_trace_events(filename) + + # get pids of GPU events + global gpu_pids + gpu_pids = [] + for event in events: + if "name" not in event: + continue + if event["name"] == "process_labels" and "GPU" in event["args"]["labels"]: + gpu_pids.append(event["pid"]) + + total_length = total_length * 1e6 + sorted_gpu_events = get_sorted_gpu_events(events) + utilization = get_duration(sorted_gpu_events) / total_length + + sorted_gpu_mm_conv_events = get_sorted_gpu_mm_conv_events(events) + mm_conv_utilization = get_duration(sorted_gpu_mm_conv_events) / total_length + + return utilization, mm_conv_utilization + + +def benchmark_utilization( + f, + input, + trace_folder, + optimize_ctx=None, + trace_file_name="tmp_chrome_trace", + num_runs=1, +): + """ + Benchmark the GPU Utilization and percent of time spent on matmul and convolution operations of + running f(input, **kwargs_for_f) with [optimize_ctx] [num_runs] times. + It will produce a chrome trace file in trace_folder/trace_file_name.json + + Example: + + ``` + def f(a): + return a.sum() + + + a = torch.rand(2**20, device="cuda") + utilization, mm_conv_utilization = benchmark_utilization( + f, a, "tmp", trace_file_name="tmp_chrome_trace" + ) + ``` + + Args: + f: function to benchmark + + input: input to :attr:`f` + + trace_folder: name of the folder to store the chrome trace + + optimize_ctx: the context in which f will run + + trace_file_name: name of the dumped chrome trace file, default to "tmp_chrome_trace" + + num_runs: number of times to run f, excluding the warm-up runs, default to 1. + + Return: + tuple: (GPU Utilization, percent of time spent on matmul and convolution) + + """ + isExist = os.path.exists(trace_folder) + if not isExist: + os.makedirs(trace_folder) + print("create folder " + trace_folder) + + if optimize_ctx is None: + optimize_ctx = contextlib.nullcontext() + + chrome_trace_file_name = os.path.join(trace_folder, trace_file_name + ".json") + total_length = dump_chrome_trace( + f, + input, + chrome_trace_file_name, + optimize_ctx, + [ProfilerActivity.CUDA], + num_runs=num_runs, + devices=["cuda"], + ) + utilization, mm_conv_utilization = compute_utilization( + chrome_trace_file_name, total_length + ) + + return utilization, mm_conv_utilization diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/compile_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/compile_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..49a1adacab6ef0fbfeec19aeb8e6604836ce02f9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/compile_utils.py @@ -0,0 +1,212 @@ +# mypy: ignore-errors + + +import operator +from collections.abc import Callable + +import sympy + +import torch +import torch.fx as fx +from torch.fx.experimental.symbolic_shapes import free_unbacked_symbols +from torch.multiprocessing.reductions import StorageWeakRef +from torch.utils import _pytree as pytree +from torch.utils._pytree import tree_flatten + + +aten = torch.ops.aten + + +def get_aten_target(node: fx.Node) -> Callable: + if hasattr(node.target, "overloadpacket"): + return node.target.overloadpacket + return node.target + + +rand_ops = [ + aten.dropout, + aten._fused_dropout, + aten._standard_gamma, + aten.bernoulli, + aten.multinomial, + aten.native_dropout, + aten.normal, + aten.poisson, + aten.binomial, + aten.rrelu, + aten.rand_like, + aten.rand, + aten.randint, + aten.randn, + aten.randperm, +] + + +# return a new copy of torch.fx.graph.Graph with CSE applied to the input graph +def fx_graph_cse(fx_g: torch.fx.graph.Graph): + new_graph = fx.Graph() + env = {} # map from node in the old graph to node in the new graph + hash_env = {} # map from hash to a node in the new graph + token_map = {} # map from hash to token + + from torch._inductor.pattern_matcher import ( + compute_mutation_region_ids, + same_mutation_regions, + ) + + compute_mutation_region_ids(fx_g) # type: ignore[arg-type] + + # Make a set of separate storages returned from the output, which will be preserved + # when pruning. This prevents us from deduplicating returned tensors which have + # experienced identical operations, but are separate data structures in eager mode. + output_node: fx.Node = list(fx_g.nodes)[-1] + assert output_node.op == "output" + + def checkable_node(node: fx.Node) -> bool: + """We can evaluate only nodes that represent tensors with defined storage.""" + if "val" not in node.meta or not isinstance(node.meta["val"], torch.Tensor): + return False + + try: + node.meta["val"].untyped_storage() + except NotImplementedError: + return False + + return True + + output_storages = { + StorageWeakRef(n.meta["val"].untyped_storage()) + for n in output_node.all_input_nodes + if checkable_node(n) + } + nodes_that_alias_outputs = { + n + for n in fx_g.nodes + if checkable_node(n) + and StorageWeakRef(n.meta["val"].untyped_storage()) in output_storages + } + + for n in fx_g.nodes: + # The placeholder, output, and get_attr nodes are copied to the new graph without change + # do not CSE away random operations + if ( + n.op == "placeholder" + or n.op == "output" + or n.op == "get_attr" + or get_aten_target(n) in rand_ops + # aten.empty is non-deterministic, so don't CSE it. + # Also, aten.empty is almost always fusible into its consumer, + # so it's not worth CSEing. + or get_aten_target(n) is aten.empty + or n in nodes_that_alias_outputs + # This CSE pass currently doesn't handle re-propagation of unbacked + # meta where it'll sometimes eliminate a _local_scalar_dense but not + # replace the meta of downstream users. eg. one bug we've seen is: + # + # _local_scalar_dense_11: "Sym(u14)" = torch.ops.aten._local_scalar_dense.default(select_10); + # sym_sum_2: "Sym(u19 + u20 + u21)" = torch.sym_sum((_local_scalar_dense_11, _local_scalar_dense_12, _local_scalar_dense_13)) # noqa: B950 + # + # Notice how _local_scalar_dense_11 is u14 but sym_sum_2's meta is incorrectly the old + # pre-cse value of u19. + or ( + "val" in n.meta + and isinstance(n.meta["val"], sympy.Symbol) + and free_unbacked_symbols(n.meta["val"]) + ) + ): + new_node = new_graph.node_copy(n, lambda x: env[x]) + env[n] = new_node + else: # n.op == 'call_function', should never see n.op == 'call_module' or 'call_method' + # substitute args and kwargs members to their mapping in env if exists + # specs can be used to reconstruct nested list/dictionaries + def substitute(arg_list): + arg_list, spec = tree_flatten(arg_list) + for i in range(len(arg_list)): + v = arg_list[i] + if isinstance(v, torch.fx.node.Node) and v in env: + arg_list[i] = env[v] + if isinstance(v, (torch.SymBool, torch.SymInt, torch.SymFloat)): + arg_list[i] = v.node + return tuple(arg_list), spec + + args, args_spec = substitute(n.args) + kwargs, kwargs_spec = substitute(n.kwargs) + + # each token corresponds to a unique node + # nodes with the same token can be substituted + token = { + "target": n.target, + "args": args, + "args_spec": args_spec, + "kwargs": kwargs, + "kwargs_spec": kwargs_spec, + } + + # hash substituted args to a number, do not hash specs because specs are not hashable + # We need to add type into hash to avoid situations like: + # hash((primals_2, 1.0)) == hash((primals_2, 1)) + hash_arg = hash( + (tuple((a, type(a)) for a in args), tuple((a, type(a)) for a in kwargs)) + ) + hash_val = (n.target, hash_arg) + + # check if a node has a substitute and can be eliminated + hash_val_in_hash_env = hash_val in hash_env + overwrite_due_to_mutation = False + if hash_val_in_hash_env and token_map[hash_val] == token: + duplicate_n_prev = hash_env[hash_val] + if same_mutation_regions(n, duplicate_n_prev): + env[n] = duplicate_n_prev + continue + else: + # any futures duplicates should replace with n, not duplicate_n_prev + overwrite_due_to_mutation = True + + new_node = new_graph.node_copy(n, lambda x: env[x]) + env[n] = new_node + if overwrite_due_to_mutation or not hash_val_in_hash_env: + hash_env[hash_val] = new_node + token_map[hash_val] = token + + return new_graph + + +def raise_getitems(gm: fx.GraphModule) -> fx.GraphModule: + # Pre-create a list of nodes to iterate over, as modifying the node order + # during the loop can lead to infinite loops if not handled properly. + getitem_nodes = list( + gm.graph.find_nodes(op="call_function", target=operator.getitem) + ) + + # loop through getitem nodes in the graph and raise them to the parent node + # in reverse order to preserve their original relative order + for node in reversed(getitem_nodes): + assert len(node.all_input_nodes) == 1 + parent = node.all_input_nodes[0] + parent.append(node) + + gm.recompile() + return gm + + +def strip_overloads(gm): + """ + Modifies the target of graph nodes in :attr:`gm` to strip overloads. + + Args: + gm(fx.GraphModule): The input Fx graph module to be modified + """ + for node in gm.graph.nodes: + if isinstance(node.target, torch._ops.OpOverload): + node.target = node.target.overloadpacket + gm.recompile() + + +def get_placeholders(graph): + return graph.find_nodes(op="placeholder") + + +def get_outputs(graph): + for node in graph.find_nodes(op="output"): + return pytree.tree_leaves(node.args[0]) + raise AssertionError("No output node found") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/compilers.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/compilers.py new file mode 100644 index 0000000000000000000000000000000000000000..88954a636f915924e98bc6056a62a406f416d849 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/compilers.py @@ -0,0 +1,443 @@ +# mypy: ignore-errors + +import copy +import logging +import os +import pickle +import random +from collections.abc import Callable +from contextlib import contextmanager +from functools import partial +from typing import Union + +import sympy + +import torch +import torch.fx as fx +import torch.nn as nn +import torch.utils._pytree as pytree +from torch import SymInt +from torch._decomp import get_decompositions +from torch.fx.experimental.symbolic_shapes import bind_symbols + +from .aot_autograd import aot_function, aot_module, make_boxed_compiler +from .compile_utils import strip_overloads +from .partitioners import ( + default_partition, + draw_graph, + min_cut_rematerialization_partition, +) + + +log = logging.getLogger(__name__) + + +# These canonicalization are needed here (and not decompositions), as the ops +# we're trying to canonicalize to CompositeImplicitAutograd. +def _canonicalize(fx_g): + for node in fx_g.graph.find_nodes( + op="call_function", target=torch.ops.aten._to_copy + ): + node.target = torch.ops.aten.to + fx_g.recompile() + return fx_g + + +@contextmanager +def _disable_jit_autocast(): + old_jit_autocast_flag = torch._C._jit_set_autocast_mode(False) + try: + yield + finally: + torch._C._jit_set_autocast_mode(old_jit_autocast_flag) + + +@make_boxed_compiler +def ts_compile(fx_g: fx.GraphModule, inps) -> Callable: + """ + Compiles the :attr:`fx_g` with Torchscript compiler. + + .. warning:: + This API is experimental and likely to change. + + Args: + fx_g(fx.GraphModule): The input Fx graph module to be compiled. + + Returns: + Torch scripted model. + """ + + with _disable_jit_autocast(): + strip_overloads(fx_g) + + for node in fx_g.graph.find_nodes( + op="call_function", target=torch.ops.aten._to_copy + ): + if len(node.args) == 1 and len(node.kwargs) == 1 and "dtype" in node.kwargs: + node.target = torch.ops.aten.to + + for node in fx_g.graph.nodes: + new_kwargs = {} + for k, v in node.kwargs.items(): + if isinstance(v, torch.device): + v = v.type + new_kwargs[k] = v + node.kwargs = new_kwargs + + fx_g.graph.lint() + + fx_g.recompile() + + f = torch.jit.script(fx_g) + + torch._C._jit_pass_remove_mutation(f.graph) + + f = torch.jit.freeze(f.eval()) + f = torch.jit.optimize_for_inference(f) + if not any(isinstance(t, torch._subclasses.FakeTensor) for t in inps): + f(*inps) + return f + + +def _draw_graph_compile(fx_g, _, name, clear_meta=True): + print(fx_g.code) + draw_graph(fx_g, name, clear_meta=clear_meta) + return fx_g + + +def draw_graph_compile(name): + return make_boxed_compiler(partial(_draw_graph_compile, name=name)) + + +@make_boxed_compiler +def nop(fx_g: fx.GraphModule, _) -> Callable: + """ + Returns the :attr:`fx_g` Fx graph module as it is. This is a no-op compiler + and can be used to check accuracy. + + .. warning:: + This API is experimental and likely to change. + + """ + return fx_g + + +class DebugInterpreter(fx.Interpreter): + def run(self, *args): + self.symbol_mapping = bind_symbols(self.module, *args) + super().run(*args) + + def run_node(self, n): + def subst_symint(ni): + if not isinstance(ni, SymInt): + return ni + r = sympy.expand(ni.node.expr.xreplace(self.symbol_mapping)) + assert r.is_number, r + return int(r) + + def subst_symint_tuple(nis): + return tuple(subst_symint(ni) for ni in nis) + + def check_significant_strides(a, b): + if subst_symint(a.numel()) > 0: + for idx in range(a.ndim): + if ( + subst_symint(a.stride(idx)) != b.stride(idx) + and subst_symint(a.size(idx)) > 1 + ): + return False + return True + + def check(nv, rv, desc): + assert callable(desc) + assert nv.dtype == rv.dtype, f"{desc()}: {nv.dtype} != {rv.dtype}" + assert subst_symint_tuple(nv.size()) == rv.size(), ( + f"{desc()}: {nv.size()} aka {subst_symint_tuple(nv.size())} != {rv.size()}" + ) + same_strides = check_significant_strides(nv, rv) + assert same_strides, ( + f"{desc()}: {nv.stride()} aka {subst_symint_tuple(nv.stride())} != {rv.stride()}" + ) + + r = super().run_node(n) + if "val" in n.meta: + n_vals, _n_spec = pytree.tree_flatten(n.meta["val"]) + r_vals, _r_spec = pytree.tree_flatten(r) + # TODO: There is some sort of problem where we record that an + # operator returned a tuple/list, and then later it turns out the + # real version of the operator returned a list/tuple. Need to + # figure out what's actually going on here, the error itself is + # harmless enough as we only getitem out the outputs. + # assert n_spec == r_spec, f"{n_spec} != {r_spec}" + assert len(n_vals) == len(r_vals), f"{len(n_vals)} != {len(r_vals)}" + for i, nv, rv in zip(range(len(n_vals)), n_vals, r_vals): + if not isinstance(rv, torch.Tensor): + continue + check(nv, rv, lambda: f"output {i} where {self.symbol_mapping}") + return r + + +@make_boxed_compiler +def debug_nop(fx_g: fx.GraphModule, _) -> Callable: + """ + Returns a (slow) interpreter over the FX graph module that also checks + various debugging properties (e.g., that tracing strides matched real + strides.) + """ + return DebugInterpreter(fx_g).run + + +@make_boxed_compiler +def simple_ts_compile(fx_g, _): + strip_overloads(fx_g) + f = torch.jit.script(fx_g) + f = torch.jit.freeze(f.eval()) + return f + + +def nnc_jit(f): + return aot_function(f, simple_ts_compile) + + +aten = torch.ops.aten +default_decompositions = { + aten.detach, + aten.gelu_backward, + aten.leaky_relu_backward, + aten.sigmoid_backward, + aten.threshold_backward, + aten.hardtanh_backward, + aten.hardsigmoid_backward, + aten.hardswish_backward, + aten.tanh_backward, + aten.silu_backward, + aten.elu_backward, + aten.cudnn_batch_norm, + aten.cudnn_batch_norm_backward, + aten.masked_fill.Scalar, + aten.masked_fill.Tensor, + aten.elu, + aten.leaky_relu, + aten.hardtanh, + aten.hardswish, + aten.hardsigmoid, + aten.conj_physical, + aten.is_same_size, +} + +default_decompositions = get_decompositions(default_decompositions) + + +@make_boxed_compiler +def print_compile(fx_g, _): + print(fx_g.code) + return fx_g + + +def memory_efficient_fusion( + fn: Union[Callable, nn.Module], + **kwargs, +): + """ + Wrapper function over :func:`aot_function` and :func:`aot_module` to perform + memory efficient fusion. It uses the + :func:`min_cut_rematerialization_partition` partitioner to perform efficient + recomputation. It uses NVFuser to compile the generated forward and backward + graphs. + + .. warning:: + This API is experimental and likely to change. + + Args: + fn (Union[Callable, nn.Module]): A Python function or a ``nn.Module`` + that takes one or more arguments. Must return one or more Tensors. + **kwargs: Any other overrides you want to make to the settings + + Returns: + Returns a ``Callable`` or ``nn.Module`` that retains the eager behavior + of the original :attr:`fn`, but whose forward and backward graphs have + gone through recomputation optimizations, and the graphs have been + compiled with nvfuser. + + """ + config = { + "fw_compiler": ts_compile, + "bw_compiler": ts_compile, + "partition_fn": min_cut_rematerialization_partition, + "decompositions": default_decompositions, + } + config.update(kwargs) + if isinstance(fn, torch.nn.Module): + return aot_module(fn, **config) + else: + return aot_function(fn, **config) + + +def debug_compile(fx_g, inps): + fx_g.to_folder("foo") + print( + f""" +############################################################## +# To minimize FX graph, copy and paste the below and run it # +############################################################## + +import torch +import torch.fx as fx +from functorch.compile import minifier, check_nvfuser_subprocess, check_nvfuser_correctness_subprocess + +inps = {[(i.shape, i.dtype) for i in inps]} +inps = [torch.ones(shape, dtype=dtype, device='cuda') for (shape, dtype) in inps] +from foo import FxModule +mod = FxModule().cuda() + +with torch.jit.fuser("fuser2"): + # check_nvfuser_subprocess can be replaced with check_nvfuser_correctness_subprocess + minifier(fx.symbolic_trace(mod), inps, check_nvfuser_subprocess) +""" + ) + from foo import FxModule + + FxModule().cuda()(*inps) + + return ts_compile(fx_g, inps) + + +graph_index = 0 + + +def get_inputs(input_data_path): + """ + Return a random input for the given inputs meta generated from _save_fx_default. + """ + inputs = [] + with open(input_data_path, "rb") as f: + inputs_meta = pickle.load(f) + inputs = [] + for meta in inputs_meta: + if len(meta) == 1: + type = meta + input = type(random.rand()) + else: + type, shape, _stride, dtype, device = meta + if dtype in { + torch.int, + torch.int32, + torch.int64, + torch.bool, + torch.int, + torch.uint8, + int, + float, + }: + input = torch.randint(0, 1, shape, dtype=dtype, device=device) + else: + input = torch.rand(shape, dtype=dtype, device=device) + inputs.append(input) + return inputs + + +def _save_fx_default(current_name, folder_name, dump_example_input, gm, example_inputs): + """ + The forward, backward, and joint computation graph will be stored in + {folder_name}/{current_name}/{current_name}_forward_{graph_index}, + {folder_name}/{current_name}/{current_name}_backward_{graph_index}, and + {folder_name}/{current_name}/{current_name}_joint_{graph_index} respectively. + The input shape of the graphs will be stored in the .input files. + These files can be loaded with pickle, + and is a list of format (type, shape, stride, dtype, device). + In the case of type = int or float, it is just (type,). + For joint graph input, it is a nested list [[],[]] + where the two inner lists have the same format. + If dump_example_input is True, example_inputs will be stored in .pt file. + Since each function might produce multiple graphs, + the graph_index is used to distinguish difference graphs + """ + from functorch.compile import aot_module_simplified + + def get_input_meta(args): + input_meta = [] + if len(args) > 0 and isinstance(args[0], tuple): # joint input + input_meta += get_input_meta(args[0]) + input_meta += get_input_meta(args[1]) + return input_meta + for arg in args: + if type(arg) is int or type(arg) is float: + input_meta.append((type(arg),)) + else: + input_meta.append( + (type(arg), arg.shape, arg.stride(), arg.dtype, arg.device) + ) + return input_meta + + def graph_saver_helper(gm_to_save, args, type_name): + global graph_index + if len(gm_to_save.graph.nodes) == 0: + log.log( + logging.WARNING, + "No nodes in graph {%s}_{%s}_{%s}.", + current_name, + type_name, + graph_index, + ) + return + + gm = copy.deepcopy(gm_to_save) + gm.graph.set_codegen(torch.fx.graph.CodeGen()) # remove codegen + gm.recompile() + + input_meta = get_input_meta(args) + + os.makedirs(f"{folder_name}/{current_name}", exist_ok=True) + gm.to_folder( + f"{folder_name}/{current_name}/{current_name}_{type_name}_{graph_index}" + ) + with open( + f"{folder_name}/{current_name}/{current_name}_{type_name}_{graph_index}/{current_name}_{type_name}_{graph_index}.input" + ) as f: + pickle.dump(input_meta, f) + if dump_example_input: + torch.save( + args, + f"{folder_name}/{current_name}/{current_name}_{type_name}_{graph_index}/{current_name}_{type_name}_{graph_index}.pt", # noqa: B950 + ) # noqa: E501 + + def graph_saver_forward(gm, fw_args): + graph_saver_helper(gm, fw_args, "forward") + return gm + + def graph_saver_backward(gm, bw_args): + graph_saver_helper(gm, bw_args, "backward") + global graph_index + graph_index += 1 + return gm + + def graph_saver_joint(gm, joint_args): + graph_saver_helper(gm, joint_args, "joint") + return default_partition(gm, joint_args) + + return aot_module_simplified( + gm, + example_inputs, + fw_compiler=graph_saver_forward, + bw_compiler=graph_saver_backward, + partition_fn=graph_saver_joint, + decompositions=default_decompositions, + ) + + +# WARNING: This isn't tested anywhere!! +def graph_dumper_aot(current_name, folder_name, dump_example_input=False): + """ + Dump the forward, backward, and joint computation graph. + Example Usage: + save_fx_func = graph_dumper_aot(current_name, folder_name, dump_example_input = False) + optimize_ctx = torchdynamo.optimize( + save_fx_func + ) + with torch.enable_grad(): + with optimize_ctx: + result = forward_and_backward_pass(model, example_inputs) + """ + global graph_index + graph_index = 0 + return partial(_save_fx_default, current_name, folder_name, dump_example_input) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/config.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/config.py new file mode 100644 index 0000000000000000000000000000000000000000..f53a44c80ecf24b12cb07fe3ae83aca693d730f4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/config.py @@ -0,0 +1,414 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from collections.abc import Callable + + +""" +Global flags for aot autograd +""" + +import os +import sys +from typing import Literal, Optional, TYPE_CHECKING + +from torch.utils._config_module import Config, install_config_module + + +# [@compile_ignored: debug] +_save_config_ignore = [ + # callable not serializable + "joint_custom_pass", +] + + +# Converts torch rng ops to their functional philox rng equivalents. Note that +# we functionalize only CUDA rng ops today. +functionalize_rng_ops = False + +# can be useful for debugging if we are incorrectly creating meta fake tensors +fake_tensor_allow_meta = os.environ.get("FAKE_ALLOW_META", "1") != "0" + +# Enables optional asserts in hotpath code to check for errors. If +# you are seeing weird accuracy problems, try turning this on. +# This is currently off by default as it will harm tracing time, +# but it is on by default for aot_eager. +debug_assert = False + +debug_partitioner = os.environ.get("AOT_PARTITIONER_DEBUG", "0") != "0" + +# See # NOTE [Export custom triton op] +decompose_custom_triton_ops = True + +static_weight_shapes = True + +# See https://github.com/pytorch/pytorch/issues/141881 +# Tells partitioner that parameters are free to save for backward. +treat_parameters_as_free_to_save = True + +# Applies CSE to the graph before partitioning +cse = True + +from torch._environment import is_fbcode + + +enable_autograd_cache: bool = Config( + justknob="pytorch/remote_cache:enable_local_autograd_cache", + env_name_force="TORCHINDUCTOR_AUTOGRAD_CACHE", + default=True, +) + +autograd_cache_allow_custom_autograd_functions: bool = Config( + env_name_force="TORCHINDUCTOR_AUTOGRAD_CACHE_ALLOW_CUSTOM_AUTOGRAD", default=False +) + +# For now, this is just for enabling unit testing in test_aot_autograd_cache.py +# We will either make this the default with AOTAutogradCache, or +# we'll just use it in the precompile flow. So there's no +# need to add env vars or make it configurable +bundled_autograd_cache: bool = False + +# Whether or not to normalize placeholder names in graphs +# from dynaom in AOTAutogradCache +autograd_cache_normalize_inputs = not is_fbcode() + + +def remote_autograd_cache_default() -> Optional[bool]: + if os.environ.get("TORCHINDUCTOR_AUTOGRAD_REMOTE_CACHE") == "1": + return True + if os.environ.get("TORCHINDUCTOR_AUTOGRAD_REMOTE_CACHE") == "0": + return False + return None + + +enable_remote_autograd_cache = remote_autograd_cache_default() + + +# When AOTAutograd regenerates aliased graph outputs, +# attempt to use functionalization's view-replay logic +# before falling back to the autograd engine's view replay or as_strided. +# This can have some perf implications +# (although for many models this will not matter). +# (1) If you have many view ops chained together, replaying all of them +# at runtime can have more overhead compared to a single as_strided call +# (2) If you are doing training, AsStridedBackward is quite slow, +# and the individual view op backward formulas will likely be faster. +# (3) Some backends like XLA do not support as_strided + +# Temporary hack: disable this flag for internal +# (needed to fix an internal issue while avoiding bumping XLA pin) +# eventually: either default this config to false completely +# once XLA pin update works, +# or default config to true and fix relevant bugs + + +# View replay is currently not compatible with AOTAutogradCache, since +# FunctionalTensors are not serializable. We'll need to make them +# serializable before enabling warm cache with this config turned on. +view_replay_for_aliased_outputs = not is_fbcode() + +# Restricts the amount of computation AOTAutograd can do. +# NB: We have essentially disabled this heuristic now. However, this is kept +# here for now in case it's useful. Setting it low can artificially reduce the +# amount of recomputation AOTAutograd performs, although not in any kind of +# principled way. +max_dist_from_bw = 1000 + + +# Bans recomputation of nodes that are reading from nodes that is far before +# the current node +ban_recompute_used_far_apart = True +# Breaks up long chain of fusible ops, as otherwise we can have an arbitrarily +# long chain of recomputation in the backwards pass. +ban_recompute_long_fusible_chains = True +# Bans recomputation of nodes that must be materialized in the backwards pass +# (used by a non-fusible node) +ban_recompute_materialized_backward = True +# Chooses to ban recomputation of nodes based off an allowlist. Setting it to +# False changes it to use a denylist. Main change is on operators like +# sort/pool/stuff that isn't cheap enough to be fusible for free but also isn't +# that expensive +ban_recompute_not_in_allowlist = True +# Chooses to ban recomputation of reductions. This is generally a good idea, as +# the result of reductions is generally very small but recomputing reductions in +# a fusion can be expensive. +ban_recompute_reductions = True +# Prevents the partitioner from ever saving views (i.e. always recompute them). +# Generally a good idea since views are free to recompute. +recompute_views = False +# Set this flag to enable considering non-built-in ops, including triton and custom +# ops, for recomputation during the knapsack optimization solver. +is_non_builtin_to_include = False + +# Rematerialize AC nodes for graphs with forward+loss+backward in one graph. +# This optimization minimizes activation checkpoint node lifetimes by computing them +# just-in-time. For AC nodes only used in backward, they are deferred to backward region +# instead of being computed and saved in forward. This reduces peak memory usage. +# Note: This only applies to forward+loss+backward graphs where torch.autograd.grad is allowed +# in the graph. Joint graphs (standard AOTAutograd) use the partitioner instead. +remat_using_tags_for_fwd_loss_bwd_graph = True + +# By default, the partitioner is purely trying to optimize for runtime (although +# it should always use less memory than eager) +# This knob controls the partitioner to make that tradeoff for you, choosing the +# fastest option that saves less activations than the memory budget. +# Specifically, 0.0 corresponds to the activation memory from applying +# activation checkpointing to the full compiled region, and 1.0 corresponds to +# the activation memory from the default runtime-optimized strategy. So, 0.4 +# would result in a strategy that saves 40% of the activations compared to the +# default strategy. +# It solves a 0-1 knapsack to find the minimum recompute necessary to stay below +# the activation memory budget. +# NOTE: This *cannot* be treated as +activation_memory_budget = 1.0 + +# This controls how we estimate the runtime when deciding what the cheapest +# operators to recompute are. The 3 options are +# "flops": Bases it off of the flop count provided by torch.utils.flop_counter +# "profile": Benchmarks each operator to come up with a runtime +# "testing": Returns 1 for everything +activation_memory_budget_runtime_estimator = "flops" + +# This controls the solver used for the 0-1 knapsack. By default we use a +# quantized DP solution ("dp"). The other approaches are a "greedy", a "ilp" +# (which has a scipy dependency) and "dp_knapsack_sliding_hirschberg", which +# used memory-efficient quantized DP solution +activation_memory_budget_solver = "dp" + +# This dumps out a SVG visualization of the expected runtime vs. activation +# memory tradeoffs for all memory budget values from 0 to 1 in increments of +# 0.5. See an example here: +# https://github.com/pytorch/pytorch/pull/126320#discussion_r1625104015 +visualize_memory_budget_pareto = ( + os.environ.get("PARTITIONER_MEMORY_BUDGET_PARETO", "0") == "1" +) + +# This controls the directory in which to dump the SVG plot with the pareto +# frontier of the activation checkpointing memory-vs-runtime tradeoffs. +memory_budget_pareto_dir = os.environ.get("PARTITIONER_MEMORY_BUDGET_PARETO_DIR") + +# Sets all of the ban_recompute heuristics to False except ban_recompute_reductions +# Generally, this will probably result in some memory improvement, but at the +# cost of some performance +aggressive_recomputation = False + +# activation offloading enablement (testing purpose) +enable_activation_offloading = False + +# activation offloading with separate CUDA stream +activation_offload_separate_stream = False + +# activation offloading wait sinking when using separate stream (fwd graph) +activation_offload_sink_wait = False + +# activation reloading with prefetching when using separate streams (bwd graph) +activation_reload_prefetch = False + +# If FakeTensor.data_ptr() should error. +# This option is independent of AOTAutograd and torch.compile, but our policy +# is to turn it off during torch.compile. +fake_tensor_allow_unsafe_data_ptr_access = True + +# Unlifts effect tokens from the inputs/outputs in the traced graph and instead +# inserts make_token/sink_token calls in the graph to create tokens and then +# sink them at the end. Note that this means the graph is no longer functional +# which may lead to silent errors unless the backend knows how to handle the +# tokens. +unlift_effect_tokens = False + +# NOTE: [The default layout constraint for custom operators.] +# This must be the name of one of the layout constraint tags +# (that is, one of {"needs_fixed_stride_order", "flexible_layout"}), +# If the custom op does not have a layout constraint tag already +# then we assume the following applies. +# +# This config is respected by Inductor and we recommend other backends also +# respect it. +# This config is in torch._functorch and not torch._inductor because it affects +# ProxyTensor tracing. +custom_op_default_layout_constraint: Literal[ + "needs_exact_strides", "needs_fixed_stride_order", "flexible_layout" +] = "needs_exact_strides" + + +# Run aot eager decomp partition with CrossRefFakeMode +# options = False, "all", "custom_ops" +fake_tensor_crossref = False + +# This mode specifies that we should also keep track of the real +# tensor along with the fake tensor, and do real compute. While +# seemingly this eliminates the whole point of fake tensors, there are +# two obvious use cases for it: +# +# 1. When users call item()/other data dependent operations, +# if we propagate_real_tensors we are able to determine what +# the true value is and keep going. +# +# 2. It can be useful for testing, when you want to see if the fake +# and real tensors agree with each other. (Note that there are +# currently known inaccuracies in how we clone real tensors, that +# would have to be tightened up for this to be useful in this +# case.) +# +# Note that fake tensors are typically understood to be cheap to store +# indefinitely, so we tend to hold on to them longer than we would +# hold onto the real tensors. So we also support you explicitly +# deallocating the real tensor associated with a fake tensor, at which +# point we will stop propagating real tensors. +# +# One more thing: when you provide a real tensor to fakeify, we will +# clone it, so that we can safely perform mutations on it if necessary. +# This will increase live memory usage. This could potentially be +# optimized by using COW. We also currently do not faithfully +# maintain autograd metadata on the real tensor; this is fine because +# AOTAutograd will only use the fake tensor to determine leafness/etc +# of tensors in question. +fake_tensor_propagate_real_tensors = False + +# AOTDispatcher traces out a backward graph at the time of the forward pass. +# This flags controls whether or not that backward graph gets autocast behavior +# applied to it. +# +# The options are either: +# - "same_as_forward". We assume that the backward of the torch.compile'ed region +# will be run under the same autocast context manager that the region was run +# under. This is equivalent to running the following code in eager: +# +# with torch.amp.autocast(...): +# y = region(x) +# ... +# z.backward() +# +# - "off". We assume that the backward of the torch.compile'd region will +# not be run under any autocast context managers. +# This is equivalent to running the following code in eager: +# +# with torch.amp.autocast(...): +# y = region(x) +# ... +# z.backward() +# +# - or a list of kwargs dicts that represent an autocast context manager to turn +# on during the backward pass. +# +# e.g. [{"device_type": "cuda"}] is equivalent to running the following code in eager: +# +# y = region(x) +# ... +# with torch.amp.autocast(device="cuda"): +# z.backward() +backward_pass_autocast = "same_as_forward" + +# This controls whether we collect donated buffer. This flag must be set +# False if a user wants to retain_graph=True for backward. +donated_buffer = not is_fbcode() + +# Controls the default graph output format used by draw_graph +# Supported formats are defined here https://graphviz.org/docs/outputs/ +torch_compile_graph_format = os.environ.get("TORCH_COMPILE_GRAPH_FORMAT", "svg") + +# Valid only if fake_tensor_propagate_real_tensors = True; if a fake-real +# kernel mismatch is detected, bypasses by making a fake kernel from the +# real tensor outputs. +generate_fake_kernels_from_real_mismatches = False + +# When there are device mismatches in FakeTensor device propagation, +# prefer a specific device type over others. This is particularly useful +# in full compiled mode where intermediate tensors with device mismatches +# represent only logical differences during compilation - these intermediate +# tensors will never physically materialize in the binary execution, so the +# device mismatch is not a real runtime concern. Enabling this allows the +# compiler to proceed with compilation by choosing the preferred device type +# for consistency. For example, set to "mtia" to prefer MTIA devices over +# CPU, or "cuda" to prefer CUDA devices over CPU. +fake_tensor_prefer_device_type: Optional[str] = None + +# CUDAGraph save run_with_rng functionalization. +# TODO: turn on by default +graphsafe_rng_functionalization = True + +# Whether or not to eagerly compile the backward +# used by AOT compile and other settings +# TODO: once AOT compile calls aot autograd directly instead of +# through compile_fx, we can remove this +force_non_lazy_backward_lowering = False + +# only for testing, used to turn functionalization off in AOTDispatcher +_test_disable_functionalization = True + +# Error on BypassAOTAutogradCache instead of just a warning +# Used for tests +strict_autograd_cache = False + +# Note [Recomputing collectives in the partitioner] +# The purpose of this config is as follows: +# - We have many passes in the compiler (min-cut partitioning, DCE, etc) +# which can reorder or ,delete duplicate nodes in the graph +# - If any of these passes reorder/delete/duplicate a collective +# in a setting where the compiler is being run independently on multiple +# ranks, we run the risk that the compiler will make a different decision on +# different ranks, resulting in a NCCL hang when using torch.compile +# To handle this, we will (by default) ensure that collectives are not modified +# by the compiler. +# +# A few examples: +# - don't dead-code-eliminate collectives +# (in case they are dead on rank i but not rank j) +# - don't recompute collectives in partitioning +# (in case we recompute on rank i but not rank j) +# +# Today this flag **must** be set to false, but eventually +# we want the option to set it to true. +# In order to potentially optimize collectives, we'll need the compiler +# to broadcast information across ranks at compile time to ensure +# that any decisions on collectives are made consistently. +unsafe_allow_optimization_of_collectives = False + +# See Note [AOTAutograd Tangent Subclassness for mutated inputs] +# TODO(ivankobzarev): Remove this config, being able to deduce it compile time. +disable_guess_zero_tangent_for_mutated_input_subclass = False + +# See Note [Tangents memory format] +# By default tangents strideness is guessed to be contiguous, +# At runtime non contiguous tangents will be coerced to be contiguous. +# This config changes this guess for tangents strides to be the same as outputs. +# TODO(ivankobzarev): Remove this config once extra memory usage is investigated. +guess_tangent_strides_as_outputs = False + +# This is a temporary config to ensure all ranks take the same decision in the partitioner +# it will untimately be removed once we share size_hints across ranks through compiler collectives +_sync_decision_cross_ranks = False + +# By default apply inlined saved_tensors_hooks only for "donated" buffers. +# "donated" buffers are invisible to the user, they are intermediates of the forward graph. +# Applying saved tensors hooks for memory optimizations only for intermediates +# guarantees that original saved tensors could be deallocated. +# This config enables saved_tensors_hooks are applied for **all** saved tensors, +# that could include inputs, parameters, outputs. +# "donated" - applied only to saved intermediates of the graph +# "no_static" - applied to all saved but not "static" +# (this includes parameters and user marked as static) +# "all" - no filtering, everything saved for backward. +saved_tensors_hooks_filtering_mode = "donated" + + +# This callback is invoked on the joint graph before partitioning +joint_custom_pass: Callable = None # type: ignore[assignment] + +# Note [Selective Decomposition] +# This config allows selective decomposition of certain operators in the graph. +# When True, it does NOT decompose any nodes, except those nodes that users explicitly +# annotated with regional inductor compile. Please read torch.fx.passes.regional_inductor +# on to explicitly annotate. This is currently only used by inductor lite mode. +selective_decompose: bool = False + + +if TYPE_CHECKING: + from torch.utils._config_typing import * # noqa: F401, F403 + + +# adds patch, save_config, invalid config checks, etc +install_config_module(sys.modules[__name__]) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/deprecated.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/deprecated.py new file mode 100644 index 0000000000000000000000000000000000000000..f9f2764e59b4ee96b248b946b6a21363bddec12e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/deprecated.py @@ -0,0 +1,173 @@ +# mypy: allow-untyped-defs +""" +The APIs in this file are exposed as `functorch.*`. They are thin wrappers +around the torch.func.* APIs that have deprecation warnings -- we're trying +to move people to the torch.func.* equivalents. + +NB: We don't use *args, **kwargs in the signatures because that changes the +documentation. +""" + +import textwrap +import warnings +from collections.abc import Callable +from typing import Any, Optional, Union + +import torch._functorch.apis as apis +import torch._functorch.eager_transforms as _impl +import torch._functorch.make_functional as _nn_impl +import torch.nn as nn +from torch._functorch.eager_transforms import argnums_t +from torch._functorch.vmap import in_dims_t, out_dims_t + + +def get_warning(api, new_api=None, replace_newlines=False): + if new_api is None: + new_api = f"torch.func.{api}" + warning = ( + f"We've integrated functorch into PyTorch. As the final step of the \n" + f"integration, `functorch.{api}` is deprecated as of PyTorch \n" + f"2.0 and will be deleted in a future version of PyTorch >= 2.3. \n" + f"Please use `{new_api}` instead; see the PyTorch 2.0 release notes \n" + f"and/or the `torch.func` migration guide for more details \n" + f"https://pytorch.org/docs/main/func.migrating.html" + ) + if replace_newlines: + warning = warning.replace("\n", "") + return warning + + +def warn_deprecated(api, new_api=None): + warning = get_warning(api, new_api, replace_newlines=True) + warnings.warn(warning, FutureWarning, stacklevel=3) + + +def setup_docs(functorch_api, torch_func_api=None, new_api_name=None): + api_name = functorch_api.__name__ + if torch_func_api is None: + torch_func_api = getattr(_impl, api_name) + # See https://docs.python.org/3/using/cmdline.html#cmdoption-OO + if torch_func_api.__doc__ is None: + return + + warning = get_warning(api_name, new_api_name) + warning_note = "\n.. warning::\n\n" + textwrap.indent(warning, " ") + warning_note = textwrap.indent(warning_note, " ") + functorch_api.__doc__ = torch_func_api.__doc__ + warning_note + + +def vmap( + func: Callable, + in_dims: in_dims_t = 0, + out_dims: out_dims_t = 0, + randomness: str = "error", + *, + chunk_size=None, +) -> Callable: + warn_deprecated("vmap", "torch.vmap") + return apis.vmap(func, in_dims, out_dims, randomness, chunk_size=chunk_size) + + +def grad(func: Callable, argnums: argnums_t = 0, has_aux: bool = False) -> Callable: + warn_deprecated("grad") + return apis.grad(func, argnums, has_aux) + + +def grad_and_value( + func: Callable, argnums: argnums_t = 0, has_aux: bool = False +) -> Callable: + warn_deprecated("grad_and_value") + return apis.grad_and_value(func, argnums, has_aux) + + +def vjp(func: Callable, *primals, has_aux: bool = False): + warn_deprecated("vjp") + return _impl.vjp(func, *primals, has_aux=has_aux) + + +def jvp( + func: Callable, + primals: Any, + tangents: Any, + *, + strict: bool = False, + has_aux: bool = False, +): + warn_deprecated("jvp") + return _impl.jvp(func, primals, tangents, strict=strict, has_aux=has_aux) + + +def jacrev( + func: Callable, + argnums: Union[int, tuple[int, ...]] = 0, + *, + has_aux=False, + chunk_size: Optional[int] = None, + _preallocate_and_copy=False, +): + warn_deprecated("jacrev") + return _impl.jacrev( + func, + argnums, + has_aux=has_aux, + chunk_size=chunk_size, + _preallocate_and_copy=_preallocate_and_copy, + ) + + +def jacfwd( + func: Callable, + argnums: argnums_t = 0, + has_aux: bool = False, + *, + randomness: str = "error", +): + warn_deprecated("jacfwd") + return _impl.jacfwd(func, argnums, has_aux, randomness=randomness) + + +def hessian(func, argnums=0): + warn_deprecated("hessian") + return _impl.hessian(func, argnums=argnums) + + +def functionalize(func: Callable, *, remove: str = "mutations") -> Callable: + warn_deprecated("functionalize") + return _impl.functionalize(func, remove=remove) + + +def make_functional(model: nn.Module, disable_autograd_tracking: bool = False): + warn_deprecated("make_functional", "torch.func.functional_call") + return _nn_impl.make_functional(model, disable_autograd_tracking) + + +def make_functional_with_buffers( + model: nn.Module, disable_autograd_tracking: bool = False +): + warn_deprecated("make_functional_with_buffers", "torch.func.functional_call") + return _nn_impl.make_functional_with_buffers(model, disable_autograd_tracking) + + +def combine_state_for_ensemble(models): + warn_deprecated("combine_state_for_ensemble", "torch.func.stack_module_state") + return _nn_impl.combine_state_for_ensemble(models) + + +setup_docs(vmap, apis.vmap, "torch.vmap") +setup_docs(grad, apis.grad) +setup_docs(grad_and_value, apis.grad_and_value) +setup_docs(vjp) +setup_docs(jvp) +setup_docs(jacrev) +setup_docs(jacfwd) +setup_docs(hessian) +setup_docs(functionalize) +setup_docs(make_functional, _nn_impl.make_functional, "torch.func.functional_call") +setup_docs( + make_functional_with_buffers, _nn_impl.make_functional, "torch.func.functional_call" +) +setup_docs( + combine_state_for_ensemble, + _nn_impl.combine_state_for_ensemble, + "torch.func.stack_module_state", +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/eager_transforms.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/eager_transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..046ee0e75b2be71ebc936ad36d2d719fc36e2d7c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/eager_transforms.py @@ -0,0 +1,1817 @@ +# mypy: ignore-errors + +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import contextlib +from collections.abc import Callable +from functools import partial, wraps +from typing import Any, Optional, Union + +import torch +import torch.autograd.forward_ad as fwAD +from torch._C._functorch import ( + _assert_wrapped_functional, + _func_decrement_nesting, + _func_increment_nesting, + _grad_decrement_nesting, + _grad_increment_nesting, + _jvp_decrement_nesting, + _jvp_increment_nesting, + _propagate_functional_input_mutation, + _unwrap_for_grad, + _unwrap_functional_tensor, + _wrap_for_grad, + _wrap_functional_tensor, + get_inplace_requires_grad_allowed, + get_unwrapped, + is_functorch_wrapped_tensor, + set_inplace_requires_grad_allowed, +) +from torch._functorch.utils import argnums_t, exposed_in +from torch._subclasses.functional_tensor import FunctionalTensor +from torch.fx.experimental import const_fold +from torch.fx.experimental.proxy_tensor import make_fx +from torch.utils import _pytree as pytree +from torch.utils._pytree import ( + tree_flatten, + tree_map, + tree_map_, + tree_map_only, + tree_unflatten, + treespec_pprint, +) + +from .apis import vmap +from .vmap import doesnt_support_saved_tensors_hooks, get_chunk_sizes + + +def lazy_dynamo_disallow(func): + import torch._dynamo + + return torch._dynamo.disallow_in_graph(func) + + +@contextlib.contextmanager +def enable_inplace_requires_grad(enabled): + prev_state = get_inplace_requires_grad_allowed() + set_inplace_requires_grad_allowed(enabled) + try: + yield + finally: + set_inplace_requires_grad_allowed(prev_state) + + +def _set_tensor_requires_grad(x): + # avoid graph-break on x.requires_grad_() + # https://github.com/pytorch/pytorch/pull/110053 + return x.requires_grad_() + + +def _create_differentiable(inps, level=None): + def create_differentiable(x): + if isinstance(x, torch.Tensor): + with enable_inplace_requires_grad(True): + return _set_tensor_requires_grad(x) + raise ValueError(f"Thing passed to transform API must be Tensor, got {type(x)}") + + return tree_map(create_differentiable, inps) + + +def _undo_create_differentiable(inps, level=None): + def unwrap_tensors(x): + if isinstance(x, torch.Tensor): + return _unwrap_for_grad(x, level) + # TODO: Remove the following hack for namedtuples + if isinstance(x, tuple): + return tree_map(unwrap_tensors, tuple(x)) + + raise RuntimeError(f"Expected tensors, got unsupported type {type(x)}") + + return tree_map(unwrap_tensors, inps) + + +def _is_differentiable(maybe_tensor): + if not isinstance(maybe_tensor, torch.Tensor): + return False + return maybe_tensor.requires_grad + + +def _any_differentiable(tensor_or_tuple_of_tensors): + flat_args, _ = tree_unflatten(tensor_or_tuple_of_tensors) + return any(tuple(map(_is_differentiable, flat_args))) + + +def _wrap_tensor_for_grad(maybe_tensor, level): + if not isinstance(maybe_tensor, torch.Tensor): + return maybe_tensor + return _wrap_for_grad(maybe_tensor, level) + + +def _wrap_all_tensors(tensor_pytree, level): + return tree_map(partial(_wrap_tensor_for_grad, level=level), tensor_pytree) + + +def _as_tuple(val): + if isinstance(val, tuple): + return val + return (val,) + + +# Version of autograd.grad that handles outputs that don't depend on inputs + + +def _autograd_grad( + outputs, inputs, grad_outputs=None, retain_graph=False, create_graph=True +): + if grad_outputs is None: + diff_outputs = tuple(out for out in outputs if out.requires_grad) + else: + result = tuple( + (out, go) for out, go in zip(outputs, grad_outputs) if out.requires_grad + ) + if len(result) == 0: + diff_outputs, grad_outputs = (), () + else: + diff_outputs, grad_outputs = zip(*result) + if len(diff_outputs) == 0: + return tuple(torch.zeros_like(inp) for inp in inputs) + with torch._dynamo.compiled_autograd._disable(): + grad_inputs = torch.autograd.grad( + diff_outputs, + inputs, + grad_outputs, + retain_graph=retain_graph, + create_graph=create_graph, + allow_unused=True, + ) + grad_inputs = tuple( + torch.zeros_like(inp) if gi is None else gi + for gi, inp in zip(grad_inputs, inputs) + ) + return grad_inputs + + +# NOTE [grad and vjp interaction with no_grad] +# +# def f(x): +# with torch.no_grad(): +# c = x ** 2 +# return x - c +# +# The thing to consider is if enable_grad is on/off before grad gets called. +# +# Case 1: enable_grad is on. +# grad(f)(x) +# In this case, `grad` should respect the inner torch.no_grad. +# +# Case 2: enable_grad is off +# with torch.no_grad(): +# grad(f)(x) +# In this case, `grad` should respect the inner torch.no_grad, but not the +# outer one. This is because `grad` is a "function transform": its result +# should not depend on the result of a context manager outside of `f`. +# +# This gives us the following desired behavior: +# - (nested) grad transforms must obey torch.no_grad inside them +# - (nested) grad transforms should not obey torch.no_grad outside them +# +# To achieve this behavior, upon entering grad/vjp: +# - we save the current ("previous") is_grad_enabled (*) +# - we unconditionally enable grad. +# +# Inside DynamicLayerBackFallback, when we're temporarily popping `grad` layer +# off the stack: +# - if grad_mode is disabled, then we do nothing. (there is a torch.no_grad +# active, all subsequent grad transforms must obey it). +# - if grad_mode is enabled, and the previous is_grad_enabled (*) is False, +# then we temporarily restore the previous `is_grad_enabled`. This is +# because we're crossing the boundary from a `grad` outside the +# no_grad to a `grad` inside the no_grad. +# +# NB: vjp has some interesting behavior because the vjp's callable can be called +# under a different grad_mode than the forward computation... +# +# NB: forward-mode AD: forward-mode AD doesn't respect torch.no_grad, but +# it respects c10::AutoFwGradMode. We've implemented the same logic for +# our jvp transform (it will have special handling if FwGradMode is disabled). + + +# How do we increment and decrement the nesting? I don't think we can. +@exposed_in("torch.func") +def vjp(func: Callable, *primals, has_aux: bool = False): + """ + Standing for the vector-Jacobian product, returns a tuple containing the + results of ``func`` applied to ``primals`` and a function that, when + given ``cotangents``, computes the reverse-mode Jacobian of ``func`` with + respect to ``primals`` times ``cotangents``. + + Args: + func (Callable): A Python function that takes one or more arguments. Must + return one or more Tensors. + primals (Tensors): Positional arguments to ``func`` that must all be + Tensors. The returned function will also be computing the + derivative with respect to these arguments + has_aux (bool): Flag indicating that ``func`` returns a + ``(output, aux)`` tuple where the first element is the output of + the function to be differentiated and the second element is + other auxiliary objects that will not be differentiated. + Default: False. + + Returns: + Returns a ``(output, vjp_fn)`` tuple containing the output of ``func`` + applied to ``primals`` and a function that computes the vjp of + ``func`` with respect to all ``primals`` using the cotangents passed + to the returned function. If ``has_aux is True``, then instead returns a + ``(output, vjp_fn, aux)`` tuple. + The returned ``vjp_fn`` function will return a tuple of each VJP. + + When used in simple cases, :func:`vjp` behaves the same as :func:`grad` + + >>> x = torch.randn([5]) + >>> f = lambda x: x.sin().sum() + >>> (_, vjpfunc) = torch.func.vjp(f, x) + >>> grad = vjpfunc(torch.tensor(1.0))[0] + >>> assert torch.allclose(grad, torch.func.grad(f)(x)) + + However, :func:`vjp` can support functions with multiple outputs by + passing in the cotangents for each of the outputs + + >>> x = torch.randn([5]) + >>> f = lambda x: (x.sin(), x.cos()) + >>> (_, vjpfunc) = torch.func.vjp(f, x) + >>> vjps = vjpfunc((torch.ones([5]), torch.ones([5]))) + >>> assert torch.allclose(vjps[0], x.cos() + -x.sin()) + + :func:`vjp` can even support outputs being Python structs + + >>> x = torch.randn([5]) + >>> f = lambda x: {"first": x.sin(), "second": x.cos()} + >>> (_, vjpfunc) = torch.func.vjp(f, x) + >>> cotangents = {"first": torch.ones([5]), "second": torch.ones([5])} + >>> vjps = vjpfunc(cotangents) + >>> assert torch.allclose(vjps[0], x.cos() + -x.sin()) + + The function returned by :func:`vjp` will compute the partials with + respect to each of the ``primals`` + + >>> x, y = torch.randn([5, 4]), torch.randn([4, 5]) + >>> (_, vjpfunc) = torch.func.vjp(torch.matmul, x, y) + >>> cotangents = torch.randn([5, 5]) + >>> vjps = vjpfunc(cotangents) + >>> assert len(vjps) == 2 + >>> assert torch.allclose(vjps[0], torch.matmul(cotangents, y.transpose(0, 1))) + >>> assert torch.allclose(vjps[1], torch.matmul(x.transpose(0, 1), cotangents)) + + ``primals`` are the positional arguments for ``f``. All kwargs use their + default value + + >>> x = torch.randn([5]) + >>> def f(x, scale=4.): + >>> return x * scale + >>> + >>> (_, vjpfunc) = torch.func.vjp(f, x) + >>> vjps = vjpfunc(torch.ones_like(x)) + >>> assert torch.allclose(vjps[0], torch.full(x.shape, 4.0)) + + .. note:: + Using PyTorch ``torch.no_grad`` together with ``vjp``. + Case 1: Using ``torch.no_grad`` inside a function: + + >>> def f(x): + >>> with torch.no_grad(): + >>> c = x ** 2 + >>> return x - c + + In this case, ``vjp(f)(x)`` will respect the inner ``torch.no_grad``. + + Case 2: Using ``vjp`` inside ``torch.no_grad`` context manager: + + >>> # xdoctest: +SKIP(failing) + >>> with torch.no_grad(): + >>> vjp(f)(x) + + In this case, ``vjp`` will respect the inner ``torch.no_grad``, but not the + outer one. This is because ``vjp`` is a "function transform": its result + should not depend on the result of a context manager outside of ``f``. + """ + return _vjp_with_argnums(func, *primals, has_aux=has_aux) + + +@contextlib.contextmanager +def grad_increment_nesting(): + try: + grad_level = _grad_increment_nesting() + yield grad_level + finally: + _grad_decrement_nesting() + + +def enter_jvp_nesting(): + global JVP_NESTING + jvp_level = _jvp_increment_nesting() + JVP_NESTING += 1 + return jvp_level + + +def exit_jvp_nesting(): + global JVP_NESTING + _jvp_decrement_nesting() + JVP_NESTING -= 1 + + +@contextlib.contextmanager +def jvp_increment_nesting(): + try: + yield enter_jvp_nesting() + finally: + exit_jvp_nesting() + + +@doesnt_support_saved_tensors_hooks +def _vjp_with_argnums( + func: Callable, *primals, argnums: Optional[argnums_t] = None, has_aux: bool = False +): + # This is the same function as vjp but also accepts an argnums argument + # All args are the same as vjp except for the added argument + # argnums (Optional[int or tuple[int,...]]): Optional, specifies the argument(s) to compute gradients with respect to. + # If None, computes the gradients with respect to all inputs (used for vjp). Default: None + # + # WARN: Users should NOT call this function directly and should just be calling vjp. + # It is only separated so that inputs passed to jacrev but not differentiated get the correct wrappers. + # + # NOTE: All error messages are produced as if vjp was being called, even if this was called by jacrev + # + # Returns the same two elements as :func:`vjp` but the function returned, vjp_fn, returns a tuple of VJPs + # for only the primal elements given by argnums. + with grad_increment_nesting() as level: + # See NOTE [grad and vjp interaction with no_grad] + with torch.enable_grad(): + primals = _wrap_all_tensors(primals, level) + if argnums is None: + diff_primals = _create_differentiable(primals, level) + else: + diff_primals = _slice_argnums(primals, argnums, as_tuple=False) + tree_map_(partial(_create_differentiable, level=level), diff_primals) + primals_out = func(*primals) + + if has_aux: + if not (isinstance(primals_out, tuple) and len(primals_out) == 2): + raise RuntimeError( + "vjp(f, *primals): output of function f should be a tuple: (output, aux) " + "if has_aux is True" + ) + primals_out, aux = primals_out + aux = _undo_create_differentiable(aux, level) + + flat_primals_out, primals_out_spec = tree_flatten(primals_out) + assert_non_empty_tensor_output(flat_primals_out, "vjp(f, *primals)") + flat_diff_primals, primals_spec = tree_flatten(diff_primals) + results = _undo_create_differentiable(primals_out, level) + + for primal_out in flat_primals_out: + assert isinstance(primal_out, torch.Tensor) + if primal_out.is_floating_point() or primal_out.is_complex(): + continue + raise RuntimeError( + "vjp(f, ...): All outputs of f must be " + "floating-point or complex Tensors, got Tensor " + f"with dtype {primal_out.dtype}" + ) + + def wrapper(cotangents, retain_graph=True, create_graph=None): + if create_graph is None: + create_graph = torch.is_grad_enabled() + flat_cotangents, cotangents_spec = tree_flatten(cotangents) + if primals_out_spec != cotangents_spec: + raise RuntimeError( + f"Expected pytree structure of cotangents to be the same " + f"as pytree structure of outputs to the function. " + f"cotangents: {treespec_pprint(cotangents_spec)}, " + f"primal output: {treespec_pprint(primals_out_spec)}" + ) + result = _autograd_grad( + flat_primals_out, + flat_diff_primals, + flat_cotangents, + retain_graph=retain_graph, + create_graph=create_graph, + ) + return tree_unflatten(result, primals_spec) + + if has_aux: + return results, wrapper, aux + else: + return results, wrapper + + +def _safe_zero_index(x): + assert len(x) == 1 + return x[0] + + +# jacrev and jacfwd don't support complex functions +# Helper function to throw appropriate error. +def error_if_complex(func_name, args, is_input): + flat_args = pytree.tree_leaves(args) + for idx, arg in enumerate(flat_args): + if isinstance(arg, torch.Tensor) and arg.dtype.is_complex: + input_or_output = "inputs" if is_input else "outputs" + err_msg = ( + f"{func_name}: Expected all {input_or_output} " + f"to be real but received complex tensor at flattened input idx: {idx}" + ) + raise RuntimeError(err_msg) + + +@exposed_in("torch.func") +def jacrev( + func: Callable, + argnums: Union[int, tuple[int, ...]] = 0, + *, + has_aux=False, + chunk_size: Optional[int] = None, + _preallocate_and_copy=False, +): + """ + Computes the Jacobian of ``func`` with respect to the arg(s) at index + ``argnum`` using reverse mode autodiff + + .. note:: + Using :attr:`chunk_size=1` is equivalent to computing the jacobian + row-by-row with a for-loop i.e. the constraints of :func:`vmap` are + not applicable. + + Args: + func (function): A Python function that takes one or more arguments, + one of which must be a Tensor, and returns one or more Tensors + argnums (int or tuple[int, ...]): Optional, integer or tuple of integers, + saying which arguments to get the Jacobian with respect to. + Default: 0. + has_aux (bool): Flag indicating that ``func`` returns a + ``(output, aux)`` tuple where the first element is the output of + the function to be differentiated and the second element is + auxiliary objects that will not be differentiated. + Default: False. + chunk_size (None or int): If None (default), use the maximum chunk size + (equivalent to doing a single vmap over vjp to compute the jacobian). + If 1, then compute the jacobian row-by-row with a for-loop. + If not None, then compute the jacobian :attr:`chunk_size` rows at a time + (equivalent to doing multiple vmap over vjp). If you run into memory issues computing + the jacobian, please try to specify a non-None chunk_size. + + Returns: + Returns a function that takes in the same inputs as ``func`` and + returns the Jacobian of ``func`` with respect to the arg(s) at + ``argnums``. If ``has_aux is True``, then the returned function + instead returns a ``(jacobian, aux)`` tuple where ``jacobian`` + is the Jacobian and ``aux`` is auxiliary objects returned by ``func``. + + A basic usage with a pointwise, unary operation will give a diagonal array + as the Jacobian + + >>> from torch.func import jacrev + >>> x = torch.randn(5) + >>> jacobian = jacrev(torch.sin)(x) + >>> expected = torch.diag(torch.cos(x)) + >>> assert torch.allclose(jacobian, expected) + + If you would like to compute the output of the function as well as the + jacobian of the function, use the ``has_aux`` flag to return the output + as an auxiliary object: + + >>> from torch.func import jacrev + >>> x = torch.randn(5) + >>> + >>> def f(x): + >>> return x.sin() + >>> + >>> def g(x): + >>> result = f(x) + >>> return result, result + >>> + >>> jacobian_f, f_x = jacrev(g, has_aux=True)(x) + >>> assert torch.allclose(f_x, f(x)) + + :func:`jacrev` can be composed with vmap to produce batched + Jacobians: + + >>> from torch.func import jacrev, vmap + >>> x = torch.randn(64, 5) + >>> jacobian = vmap(jacrev(torch.sin))(x) + >>> assert jacobian.shape == (64, 5, 5) + + Additionally, :func:`jacrev` can be composed with itself to produce + Hessians + + >>> from torch.func import jacrev + >>> def f(x): + >>> return x.sin().sum() + >>> + >>> x = torch.randn(5) + >>> hessian = jacrev(jacrev(f))(x) + >>> assert torch.allclose(hessian, torch.diag(-x.sin())) + + By default, :func:`jacrev` computes the Jacobian with respect to the first + input. However, it can compute the Jacboian with respect to a different + argument by using ``argnums``: + + >>> from torch.func import jacrev + >>> def f(x, y): + >>> return x + y ** 2 + >>> + >>> x, y = torch.randn(5), torch.randn(5) + >>> jacobian = jacrev(f, argnums=1)(x, y) + >>> expected = torch.diag(2 * y) + >>> assert torch.allclose(jacobian, expected) + + Additionally, passing a tuple to ``argnums`` will compute the Jacobian + with respect to multiple arguments + + >>> from torch.func import jacrev + >>> def f(x, y): + >>> return x + y ** 2 + >>> + >>> x, y = torch.randn(5), torch.randn(5) + >>> jacobian = jacrev(f, argnums=(0, 1))(x, y) + >>> expectedX = torch.diag(torch.ones_like(x)) + >>> expectedY = torch.diag(2 * y) + >>> assert torch.allclose(jacobian[0], expectedX) + >>> assert torch.allclose(jacobian[1], expectedY) + + .. note:: + Using PyTorch ``torch.no_grad`` together with ``jacrev``. + Case 1: Using ``torch.no_grad`` inside a function: + + >>> def f(x): + >>> with torch.no_grad(): + >>> c = x ** 2 + >>> return x - c + + In this case, ``jacrev(f)(x)`` will respect the inner ``torch.no_grad``. + + Case 2: Using ``jacrev`` inside ``torch.no_grad`` context manager: + + >>> with torch.no_grad(): + >>> jacrev(f)(x) + + In this case, ``jacrev`` will respect the inner ``torch.no_grad``, but not the + outer one. This is because ``jacrev`` is a "function transform": its result + should not depend on the result of a context manager outside of ``f``. + """ + if not (chunk_size is None or chunk_size > 0): + raise ValueError("jacrev: `chunk_size` should be greater than 0.") + + @wraps(func) + def wrapper_fn(*args): + error_if_complex("jacrev", args, is_input=True) + vjp_out = _vjp_with_argnums(func, *args, argnums=argnums, has_aux=has_aux) + if has_aux: + output, vjp_fn, aux = vjp_out + else: + output, vjp_fn = vjp_out + + # See NOTE: [Computing jacobian with vmap and vjp for multiple outputs] + flat_output, output_spec = tree_flatten(output) + + error_if_complex("jacrev", flat_output, is_input=False) + + # NB: vjp already checks that all outputs are tensors + # Step 1: Construct grad_outputs by splitting the standard basis + flat_output_numels = tuple(out.numel() for out in flat_output) + + primals = _slice_argnums(args, argnums) + flat_primals, primals_spec = tree_flatten(primals) + + def compute_jacobian_stacked(): + # Helper function to compute chunked Jacobian + # The intermediate chunked calculation are only + # scoped at this function level. + chunked_results = [] + for flat_basis_chunk in _chunked_standard_basis_for_( + flat_output, flat_output_numels, chunk_size=chunk_size + ): + if chunk_size == 1: + # sanity check. + for t in flat_basis_chunk: + assert t.size(0) == 1 + + flat_basis_chunk = tree_map( + lambda t: torch.squeeze(t, 0), flat_basis_chunk + ) + + basis = tree_unflatten(flat_basis_chunk, output_spec) + + if chunk_size == 1: + # Behaviour with `chunk_size=1` is same as `for-loop` + # i.e. user shouldn't deal with the limitations of vmap. + chunked_result = vjp_fn(basis) + else: # chunk_size is None or chunk_size != 1 + chunked_result = vmap(vjp_fn)(basis) + + flat_results = pytree.tree_leaves(chunked_result) + + if chunk_size == 1: + flat_results = tree_map( + lambda t: torch.unsqueeze(t, 0), flat_results + ) + + chunked_results.append(flat_results) + + if len(chunked_results) == 1: + # Short-circuit if we used a single chunk + return chunked_results[0] + + # Concatenate chunks. + flat_results = [] + # Iterate and concat the jacobians of different + # inputs. + for idx in range(len(flat_primals)): + r = tuple(r_[idx] for r_ in chunked_results) + flat_results.append(torch.cat(r, 0)) + + return flat_results + + def compute_jacobian_preallocate_and_copy(): + # Helper function to compute chunked Jacobian + # The intermediate chunked calculation are only + # scoped at this function level. + out_vec_size = sum(flat_output_numels) + + # Don't pre-allocate if we have a single chunk. + if not (chunk_size is None or chunk_size >= out_vec_size): + stacked_results = [ + primal.new_zeros(out_vec_size, *primal.shape) + for primal in flat_primals + ] + + for idx, flat_basis_chunk in enumerate( + _chunked_standard_basis_for_( + flat_output, flat_output_numels, chunk_size=chunk_size + ) + ): + if chunk_size == 1: + # sanity check. + for t in flat_basis_chunk: + assert t.size(0) == 1 + + flat_basis_chunk = [torch.squeeze(t, 0) for t in flat_basis_chunk] + + basis = tree_unflatten(flat_basis_chunk, output_spec) + + if chunk_size == 1: + # Behaviour with `chunk_size=1` is same as `for-loop` + # i.e. user shouldn't deal with the limitations of vmap. + chunked_result = vjp_fn(basis) + else: # chunk_size is None or chunk_size != 1 + chunked_result = vmap(vjp_fn)(basis) + + flat_results = pytree.tree_leaves(chunked_result) + + # Short-circuit if we have a single chunk. + if chunk_size is None or chunk_size >= out_vec_size: + if chunk_size == 1: # and out_vec_size == 1 + # Since we squeezed the output dim + flat_results = tree_map( + lambda t: torch.unsqueeze(t, 0), flat_results + ) + return flat_results + + for r, sr in zip(flat_results, stacked_results): + sr[idx * chunk_size : (idx + 1) * chunk_size].copy_(r) + + return stacked_results + + if _preallocate_and_copy: + flat_jacobians_per_input = compute_jacobian_preallocate_and_copy() + else: + flat_jacobians_per_input = compute_jacobian_stacked() + + # Step 2: The returned jacobian is one big tensor per input. In this step, + # we split each Tensor by output. + flat_jacobians_per_input = [ + result.split(flat_output_numels, dim=0) + for result in flat_jacobians_per_input + ] + flat_input_flat_output = [ + tuple( + split.view(out.shape + primal.shape) + for split, out in zip(splits, flat_output) + ) + for splits, primal in zip(flat_jacobians_per_input, flat_primals) + ] + + # Step 3: Right now, `jacobian` is a List[List[Tensor]]. + # The outer List corresponds to the number of primals, + # the inner List corresponds to the number of outputs. + # We need to: + # a. Exchange the order of the outer List and inner List + # b. tree_unflatten the inner Lists (which correspond to the primals) + # c. handle the argnums=int case + # d. tree_unflatten the outer List (which corresponds to the outputs) + flat_output_flat_input = tuple(zip(*flat_input_flat_output)) + + flat_output_input = tuple( + tree_unflatten(flat_input, primals_spec) + for flat_input in flat_output_flat_input + ) + + if isinstance(argnums, int): + flat_output_input = tuple( + _safe_zero_index(flat_input) for flat_input in flat_output_input + ) + output_input = tree_unflatten(flat_output_input, output_spec) + if has_aux: + return output_input, aux + return output_input + + return wrapper_fn + + +# NOTE: [Computing jacobian with vmap and vjp for multiple outputs] +# +# Let's consider f(x) = (x**2, x.sum()) and let x = torch.randn(3). +# It turns out we can compute the jacobian of this function with a single +# call to autograd.grad by using vmap over the correct grad_outputs. +# +# Firstly, one way to compute the jacobian is to stack x**2 and x.sum() +# into a 4D vector. E.g., use g(x) = torch.stack([x**2, x.sum()]) +# +# To get the first row of the jacobian, we call +# >>> autograd.grad(g(x), x, grad_outputs=torch.tensor([1, 0, 0, 0])) +# To get the 2nd row of the jacobian, we call +# >>> autograd.grad(g(x), x, grad_outputs=torch.tensor([0, 1, 0, 0])) +# and so on. +# +# Using vmap, we can vectorize all 4 of these computations into one by +# passing the standard basis for R^4 as the grad_output. +# vmap(partial(autograd.grad, g(x), x))(torch.eye(4)). +# +# Now, how do we compute the jacobian *without stacking the output*? +# We can just split the standard basis across the outputs. So to +# compute the jacobian of f(x), we'd use +# >>> autograd.grad(f(x), x, grad_outputs=_construct_standard_basis_for(...)) +# The grad_outputs looks like the following: +# ( torch.tensor([[1, 0, 0], +# [0, 1, 0], +# [0, 0, 1], +# [0, 0, 0]]), +# torch.tensor([[0], +# [0], +# [0], +# [1]]) ) +# +# But we're not done yet! +# >>> vmap(partial(autograd.grad(f(x), x, grad_outputs=...))) +# returns a Tensor of shape [4, 3]. We have to remember to split the +# jacobian of shape [4, 3] into two: +# - one of shape [3, 3] for the first output +# - one of shape [ 3] for the second output + + +def _chunked_standard_basis_for_(tensors, tensor_numels, chunk_size=None): + # This function: + # - constructs a N=sum(tensor_numels) standard basis. i.e. an NxN identity matrix. + # - Splits the identity matrix into chunks with each chunk size determined by `tensor_numels`. + # - Each chunk corresponds to one tensor. The chunk has the same dtype and + # device as the tensor + # + # For example, with tensor_numels = [1, 2, 1], this function returns: + # ( tensor([[1], tensor([[0, 0], tensor([[0], + # [0], [1, 0], [0], + # [0], [0, 1], [0], + # [0]]) , [0, 0]]) , [1]]) ) + # + # Precondition: tensor_numels == tuple(tensor.numel() for tensor in tensors) + # Precondition: tensors always has at least one element. + # + # See NOTE: [Computing jacobian with vmap and grad for multiple tensors] + # for context behind this function. + # NOTE: Argument `chunk_size` is used to generate chunked basis instead of + # one huge basis matrix. `chunk_size` dictates the maximum size of the + # basis matrix along dim=0. + assert len(tensors) == len(tensor_numels) + assert len(tensors) > 0 + assert chunk_size is None or chunk_size > 0 + total_numel = sum(tensor_numels) + if chunk_size and chunk_size < total_numel: + chunk_numels = get_chunk_sizes(total_numel, chunk_size) + else: # chunk_size is None or chunk_size >= total_numel + chunk_size = total_numel + chunk_numels = [total_numel] + + diag_start_indices = ( + 0, + *torch.tensor(tensor_numels).cumsum(dim=0)[:-1].neg().unbind(), + ) + + for chunk_idx, total_numel in enumerate(chunk_numels): + chunks = tuple( + tensor.new_zeros(total_numel, tensor_numel) + for tensor, tensor_numel in zip(tensors, tensor_numels) + ) + + for chunk, diag_start_idx in zip(chunks, diag_start_indices): + chunk.diagonal(diag_start_idx + chunk_idx * chunk_size).fill_(1) + chunks = tuple( + chunk.view(total_numel, *tensor.shape) + for chunk, tensor in zip(chunks, tensors) + ) + yield chunks + + +def _construct_standard_basis_for(tensors, tensor_numels): + for basis in _chunked_standard_basis_for_(tensors, tensor_numels, chunk_size=None): + return basis + + +def _validate_and_wrap_argnum(argnum, num_args): + if not isinstance(argnum, int): + raise RuntimeError(f"argnum must be int, got: {type(argnum)}") + if argnum >= 0 and argnum < num_args: + return argnum + if argnum < 0 and argnum >= -num_args: + return argnum + num_args + raise RuntimeError(f"Got argnum={argnum}, but only {num_args} positional inputs") + + +def _check_unique_non_empty(argnums): + if isinstance(argnums, tuple): + if len(argnums) == 0: + raise RuntimeError("argnums must be non-empty") + if len(set(argnums)) != len(argnums): + raise RuntimeError(f"argnums elements must be unique, got {argnums}") + + +def _replace_args(old_args, new_args, argnums): + if isinstance(argnums, int): + if len(new_args) != 1: + raise RuntimeError( + f"new_args should be of size 1, was of size {len(new_args)}" + ) + return tuple( + new_args[0] if i == argnums else old_args[i] for i in range(len(old_args)) + ) + if isinstance(argnums, tuple): + if len(new_args) != len(argnums): + raise RuntimeError( + "new_args should have the same size as argnums. " + f"Argnums size {len(argnums)}, new_args size {len(new_args)}" + ) + + def get_right_elem(i): + return new_args[argnums.index(i)] if i in argnums else old_args[i] + + return tuple(get_right_elem(i) for i in range(len(old_args))) + raise RuntimeError(f"argnums must be int or Tuple[int, ...], got: {type(argnums)}") + + +def _validate_and_wrap_argnums(argnums, num_args): + if isinstance(argnums, int): + return _validate_and_wrap_argnum(argnums, num_args) + if isinstance(argnums, tuple): + return tuple(_validate_and_wrap_argnum(argnum, num_args) for argnum in argnums) + raise AssertionError("Should never get here") + + +def _slice_argnums(args, argnums, as_tuple=True): + if not isinstance(argnums, int) and not isinstance(argnums, tuple): + raise RuntimeError( + f"argnums must be int or Tuple[int, ...], got: {type(argnums)}" + ) + argnums = _validate_and_wrap_argnums(argnums, len(args)) + _check_unique_non_empty(argnums) + if isinstance(argnums, int): + if as_tuple: + return (args[argnums],) + else: + return args[argnums] + return tuple(args[i] for i in argnums) + + +JVP_NESTING = 0 + + +def assert_flat_tuple_of_tensors(elts: Any, api: str, argname: str) -> None: + if not isinstance(elts, tuple): + raise RuntimeError( + f"{api}: Expected {argname} to be a tuple of Tensors, got {type(elts)}" + ) + for elt in elts: + if isinstance(elt, torch.Tensor): + continue + raise RuntimeError( + f"{api}: Expected {argname} to be a tuple of Tensors, got " + f"a tuple with an element of type {type(elt)}" + ) + if len(elts) == 0: + raise RuntimeError( + f"{api}: Expected {argname} to be a non-empty tuple of Tensors." + ) + + +def assert_non_empty_tensor_output(output: list[Any], api: str) -> None: + if (len(output) == 1 and output[0] is None) or len(output) < 1: + raise RuntimeError( + f"{api}: Expected f to be a function that has non-empty output (got output = {output})" + ) + for o in output: + if not isinstance(o, torch.Tensor): + raise RuntimeError( + f"{api}: expected f(*primals) to return only tensors" + f", got unsupported type {type(o)}" + ) + + +def assert_output_is_tensor_or_tensors(output: Any, api: str) -> None: + if isinstance(output, torch.Tensor): + return + if not isinstance(output, tuple): + raise RuntimeError( + f"{api}: Expected output of f to be a Tensor or Tensors, got {type(output)}" + ) + if len(output) == 0: + raise RuntimeError( + f"{api}: Expected output of f to be a non-empty tuple of Tensors." + ) + for out in output: + if isinstance(out, torch.Tensor): + continue + raise RuntimeError( + f"{api}: Expected output of f to be a Tensor or Tensors, got " + f"{type(out)} as an output" + ) + + +def assert_non_empty_list_of_tensors( + output: list[torch.Tensor], api: str, argname: str +) -> None: + if len(output) == 0: + raise RuntimeError(f"{api}: Expected {argname} to contain at least one Tensor.") + for out in output: + if isinstance(out, torch.Tensor): + continue + raise RuntimeError( + f"{api}: Expected {argname} to only contain Tensors, got {type(out)}" + ) + + +jvp_str = "jvp(f, primals, tangents)" + + +def safe_unpack_dual(dual, strict): + if not isinstance(dual, torch.Tensor): + raise RuntimeError( + f"{jvp_str}: expected f(*args) to return only tensors" + f", got unsupported type {type(dual)}" + ) + + primal, tangent = fwAD.unpack_dual(dual) + if tangent is None: + if strict: + raise RuntimeError( + "jvp(f, primals, tangents, strict=True): " + "The output of f is independent of " + "the inputs. This is not allowed with strict=True." + ) + tangent = torch.zeros_like(primal) + return primal, tangent + + +@exposed_in("torch.func") +def jvp( + func: Callable, + primals: Any, + tangents: Any, + *, + strict: bool = False, + has_aux: bool = False, +): + """ + Standing for the Jacobian-vector product, returns a tuple containing + the output of `func(*primals)` and the "Jacobian of ``func`` evaluated at + ``primals``" times ``tangents``. This is also known as forward-mode autodiff. + + Args: + func (function): A Python function that takes one or more arguments, + one of which must be a Tensor, and returns one or more Tensors + primals (Tensors): Positional arguments to ``func`` that must all be + Tensors. The returned function will also be computing the + derivative with respect to these arguments + tangents (Tensors): The "vector" for which Jacobian-vector-product is + computed. Must be the same structure and sizes as the inputs to + ``func``. + has_aux (bool): Flag indicating that ``func`` returns a + ``(output, aux)`` tuple where the first element is the output of + the function to be differentiated and the second element is + other auxiliary objects that will not be differentiated. + Default: False. + + Returns: + Returns a ``(output, jvp_out)`` tuple containing the output of ``func`` + evaluated at ``primals`` and the Jacobian-vector product. + If ``has_aux is True``, then instead returns a ``(output, jvp_out, aux)`` tuple. + + .. note:: + You may see this API error out with "forward-mode AD not implemented + for operator X". If so, please file a bug report and we will prioritize it. + + jvp is useful when you wish to compute gradients of a function R^1 -> R^N + + >>> from torch.func import jvp + >>> x = torch.randn([]) + >>> f = lambda x: x * torch.tensor([1.0, 2.0, 3]) + >>> value, grad = jvp(f, (x,), (torch.tensor(1.0),)) + >>> assert torch.allclose(value, f(x)) + >>> assert torch.allclose(grad, torch.tensor([1.0, 2, 3])) + + :func:`jvp` can support functions with multiple inputs by passing in the + tangents for each of the inputs + + >>> from torch.func import jvp + >>> x = torch.randn(5) + >>> y = torch.randn(5) + >>> f = lambda x, y: (x * y) + >>> _, output = jvp(f, (x, y), (torch.ones(5), torch.ones(5))) + >>> assert torch.allclose(output, x + y) + + """ + + return _jvp_with_argnums( + func, primals, tangents, argnums=None, strict=strict, has_aux=has_aux + ) + + +def _jvp_with_argnums( + func: Callable, + primals: Any, + tangents: Any, + argnums: Optional[argnums_t], + *, + strict: bool = False, + has_aux: bool, +): + # This is the same function as jvp but also accepts an argnums argument + # Most args are the same as jvp except for the added argument + # argnums (int or tuple[int, ...]): Optional, specifies the argument(s) to compute gradients with respect to. + # If None, computes the gradients with respect to all inputs (used for jvp). Default: None + # Because of this, tangents must be of length argnums and matches up to the corresponding primal whose index is + # given by argnums + # + # WARN: Users should NOT call this function directly and should just be calling jvp. + # It is only separated so that inputs passed to jacfwd but not differentiated get the correct wrappers. + # + # NOTE: All error messages are produced as if jvp was being called, even if this was called by jacfwd + # + # Returns the same two elements as :func:`jvp` but the returned tuple, ``jvp_out``, only has JVPs with respect to + # the primals given by argnums + if not isinstance(primals, tuple): + raise RuntimeError( + f"{jvp_str}: Expected primals to be a tuple. " + f"E.g. it should be valid to call f(*primals)." + ) + diff_args = primals if argnums is None else _slice_argnums(primals, argnums) + flat_primals, primals_spec = tree_flatten(diff_args) + flat_tangents, tangents_spec = tree_flatten(tangents) + if primals_spec != tangents_spec: + raise RuntimeError( + f"{jvp_str}: Expected primals and tangents to have the same python " + f"structure. For example, if primals is a tuple of 3 tensors, " + f"tangents also must be. Got primals with structure {primals_spec} " + f"and tangents with structure {tangents_spec}" + ) + assert_non_empty_list_of_tensors(flat_primals, jvp_str, "primals") + assert_non_empty_list_of_tensors(flat_tangents, jvp_str, "tangents") + + global JVP_NESTING + + with jvp_increment_nesting() as level: + with fwAD._set_fwd_grad_enabled(True): + ctx = fwAD.dual_level if JVP_NESTING == 1 else contextlib.nullcontext + with ctx(): + flat_duals = tuple( + fwAD.make_dual(p, t) for p, t in zip(flat_primals, flat_tangents) + ) + duals = tree_unflatten(flat_duals, primals_spec) + if argnums is not None: + primals = _wrap_all_tensors(primals, level) + duals = _replace_args(primals, duals, argnums) + result_duals = func(*duals) + if has_aux: + if not (isinstance(result_duals, tuple) and len(result_duals) == 2): + raise RuntimeError( + f"{jvp_str}: output of function f should be a tuple: (output, aux) " + "if has_aux is True" + ) + result_duals, aux = result_duals + aux = _undo_create_differentiable(aux, level) + + result_duals, spec = tree_flatten(result_duals) + assert_non_empty_tensor_output(result_duals, jvp_str) + + primals_out, tangents_out = zip( + *[safe_unpack_dual(dual, strict) for dual in result_duals] + ) + primals_out = tree_map( + partial(_undo_create_differentiable, level=level), primals_out + ) + tangents_out = tree_map( + partial(_undo_create_differentiable, level=level), tangents_out + ) + + primals_out_unflatten = tree_unflatten(primals_out, spec) + tangents_out_unflatten = tree_unflatten(tangents_out, spec) + if has_aux: + return primals_out_unflatten, tangents_out_unflatten, aux + + return primals_out_unflatten, tangents_out_unflatten + + +def safe_unflatten(tensor, dim, shape): + if len(shape) == 0: + assert tensor.shape[dim] == 1 + return tensor.squeeze(dim) + return tensor.unflatten(dim, shape) + + +@exposed_in("torch.func") +def jacfwd( + func: Callable, + argnums: argnums_t = 0, + has_aux: bool = False, + *, + randomness: str = "error", +): + """ + Computes the Jacobian of ``func`` with respect to the arg(s) at index + ``argnum`` using forward-mode autodiff + + Args: + func (function): A Python function that takes one or more arguments, + one of which must be a Tensor, and returns one or more Tensors + argnums (int or tuple[int, ...]): Optional, integer or tuple of integers, + saying which arguments to get the Jacobian with respect to. + Default: 0. + has_aux (bool): Flag indicating that ``func`` returns a + ``(output, aux)`` tuple where the first element is the output of + the function to be differentiated and the second element is + auxiliary objects that will not be differentiated. + Default: False. + randomness(str): Flag indicating what type of randomness to use. + See :func:`vmap` for more detail. Allowed: "different", "same", "error". + Default: "error" + + Returns: + Returns a function that takes in the same inputs as ``func`` and + returns the Jacobian of ``func`` with respect to the arg(s) at + ``argnums``. If ``has_aux is True``, then the returned function + instead returns a ``(jacobian, aux)`` tuple where ``jacobian`` + is the Jacobian and ``aux`` is auxiliary objects returned by ``func``. + + .. note:: + You may see this API error out with "forward-mode AD not implemented + for operator X". If so, please file a bug report and we will prioritize it. + An alternative is to use :func:`jacrev`, which has better operator coverage. + + A basic usage with a pointwise, unary operation will give a diagonal array + as the Jacobian + + >>> from torch.func import jacfwd + >>> x = torch.randn(5) + >>> jacobian = jacfwd(torch.sin)(x) + >>> expected = torch.diag(torch.cos(x)) + >>> assert torch.allclose(jacobian, expected) + + :func:`jacfwd` can be composed with vmap to produce batched + Jacobians: + + >>> from torch.func import jacfwd, vmap + >>> x = torch.randn(64, 5) + >>> jacobian = vmap(jacfwd(torch.sin))(x) + >>> assert jacobian.shape == (64, 5, 5) + + If you would like to compute the output of the function as well as the + jacobian of the function, use the ``has_aux`` flag to return the output + as an auxiliary object: + + >>> from torch.func import jacfwd + >>> x = torch.randn(5) + >>> + >>> def f(x): + >>> return x.sin() + >>> + >>> def g(x): + >>> result = f(x) + >>> return result, result + >>> + >>> jacobian_f, f_x = jacfwd(g, has_aux=True)(x) + >>> assert torch.allclose(f_x, f(x)) + + Additionally, :func:`jacrev` can be composed with itself or :func:`jacrev` + to produce Hessians + + >>> from torch.func import jacfwd, jacrev + >>> def f(x): + >>> return x.sin().sum() + >>> + >>> x = torch.randn(5) + >>> hessian = jacfwd(jacrev(f))(x) + >>> assert torch.allclose(hessian, torch.diag(-x.sin())) + + By default, :func:`jacfwd` computes the Jacobian with respect to the first + input. However, it can compute the Jacboian with respect to a different + argument by using ``argnums``: + + >>> from torch.func import jacfwd + >>> def f(x, y): + >>> return x + y ** 2 + >>> + >>> x, y = torch.randn(5), torch.randn(5) + >>> jacobian = jacfwd(f, argnums=1)(x, y) + >>> expected = torch.diag(2 * y) + >>> assert torch.allclose(jacobian, expected) + + Additionally, passing a tuple to ``argnums`` will compute the Jacobian + with respect to multiple arguments + + >>> from torch.func import jacfwd + >>> def f(x, y): + >>> return x + y ** 2 + >>> + >>> x, y = torch.randn(5), torch.randn(5) + >>> jacobian = jacfwd(f, argnums=(0, 1))(x, y) + >>> expectedX = torch.diag(torch.ones_like(x)) + >>> expectedY = torch.diag(2 * y) + >>> assert torch.allclose(jacobian[0], expectedX) + >>> assert torch.allclose(jacobian[1], expectedY) + + """ + + @wraps(func) + def wrapper_fn(*args): + error_if_complex("jacfwd", args, is_input=True) + primals = args if argnums is None else _slice_argnums(args, argnums) + flat_primals, primals_spec = tree_flatten(primals) + flat_primals_numels = tuple(p.numel() for p in flat_primals) + flat_basis = _construct_standard_basis_for(flat_primals, flat_primals_numels) + basis = tree_unflatten(flat_basis, primals_spec) + + def push_jvp(basis): + output = _jvp_with_argnums( + func, args, basis, argnums=argnums, has_aux=has_aux + ) + # output[0] is the output of `func(*args)` + error_if_complex("jacfwd", output[0], is_input=False) + if has_aux: + _, jvp_out, aux = output + return jvp_out, aux + _, jvp_out = output + return jvp_out + + results = vmap(push_jvp, randomness=randomness)(basis) + if has_aux: + results, aux = results + # aux is in the standard basis format, e.g. NxN matrix + # We need to fetch the first element as original `func` output + flat_aux, aux_spec = tree_flatten(aux) + flat_aux = [value[0] for value in flat_aux] + aux = tree_unflatten(flat_aux, aux_spec) + + jac_outs, spec = tree_flatten(results) + # Most probably below output check can never raise an error + # as jvp should test the output before + # assert_non_empty_output(jac_outs, 'jacfwd(f, ...)(*args)') + + jac_outs_ins = tuple( + tuple( + safe_unflatten(jac_out_in, -1, primal.shape) + for primal, jac_out_in in zip( + flat_primals, + jac_out.movedim(0, -1).split(flat_primals_numels, dim=-1), + ) + ) + for jac_out in jac_outs + ) + jac_outs_ins = tuple( + tree_unflatten(jac_ins, primals_spec) for jac_ins in jac_outs_ins + ) + + if isinstance(argnums, int): + jac_outs_ins = tuple(jac_ins[0] for jac_ins in jac_outs_ins) + if has_aux: + return tree_unflatten(jac_outs_ins, spec), aux + return tree_unflatten(jac_outs_ins, spec) + + return wrapper_fn + + +@exposed_in("torch.func") +def hessian(func, argnums=0): + """ + Computes the Hessian of ``func`` with respect to the arg(s) at index + ``argnum`` via a forward-over-reverse strategy. + + The forward-over-reverse strategy (composing ``jacfwd(jacrev(func))``) is + a good default for good performance. It is possible to compute Hessians + through other compositions of :func:`jacfwd` and :func:`jacrev` like + ``jacfwd(jacfwd(func))`` or ``jacrev(jacrev(func))``. + + Args: + func (function): A Python function that takes one or more arguments, + one of which must be a Tensor, and returns one or more Tensors + argnums (int or tuple[int, ...]): Optional, integer or tuple of integers, + saying which arguments to get the Hessian with respect to. + Default: 0. + + Returns: + Returns a function that takes in the same inputs as ``func`` and + returns the Hessian of ``func`` with respect to the arg(s) at + ``argnums``. + + .. note:: + You may see this API error out with "forward-mode AD not implemented + for operator X". If so, please file a bug report and we will prioritize it. + An alternative is to use ``jacrev(jacrev(func))``, which has better + operator coverage. + + A basic usage with a R^N -> R^1 function gives a N x N Hessian: + + >>> from torch.func import hessian + >>> def f(x): + >>> return x.sin().sum() + >>> + >>> x = torch.randn(5) + >>> hess = hessian(f)(x) # equivalent to jacfwd(jacrev(f))(x) + >>> assert torch.allclose(hess, torch.diag(-x.sin())) + + """ + return jacfwd(jacrev(func, argnums), argnums) + + +@doesnt_support_saved_tensors_hooks +def grad_and_value_impl(func, argnums, has_aux, args, kwargs) -> Callable: + with grad_increment_nesting() as level: + output, aux, grad_input = None, None, None + # See NOTE [grad and vjp interaction with no_grad] + with torch.enable_grad(): + args = _wrap_all_tensors(args, level) + kwargs = _wrap_all_tensors(kwargs, level) + diff_args = _slice_argnums(args, argnums, as_tuple=False) + tree_map_(partial(_create_differentiable, level=level), diff_args) + + output = func(*args, **kwargs) + if has_aux: + if not (isinstance(output, tuple) and len(output) == 2): + raise RuntimeError( + "grad_and_value(f)(*args): output of function f should be a tuple: (output, aux) " + "if has_aux is True" + ) + output, aux = output + + if not isinstance(output, torch.Tensor): + raise RuntimeError( + "grad_and_value(f)(*args): Expected f(*args) " + f"to return a Tensor, got {type(output)}" + ) + if output.dim() != 0: + raise RuntimeError( + "grad_and_value(f)(*args): Expected f(*args) " + "to return a scalar Tensor, got tensor with " + f"{output.dim()} dims. Maybe you wanted to " + "use the vjp or jacrev APIs instead?" + ) + + flat_diff_args, spec = tree_flatten(diff_args) + + # NB: need create_graph so that backward pass isn't run in no_grad mode + flat_outputs = _as_tuple(output) + flat_grad_input = _autograd_grad( + flat_outputs, flat_diff_args, create_graph=True + ) + grad_input = tree_unflatten(flat_grad_input, spec) + + grad_input = _undo_create_differentiable(grad_input, level) + output = _undo_create_differentiable(output, level) + if has_aux: + aux = _undo_create_differentiable(aux, level) + + if has_aux: + return grad_input, (output, aux) + return grad_input, output + + +def grad_impl(func: Callable, argnums: argnums_t, has_aux: bool, args, kwargs): + results = grad_and_value_impl(func, argnums, has_aux, args, kwargs) + if has_aux: + grad, (_, aux) = results + return grad, aux + grad, _ = results + return grad + + +def _maybe_wrap_functional_tensor( + maybe_tensor, level, *, _python_functionalize: bool = False +): + if not isinstance(maybe_tensor, torch.Tensor): + return maybe_tensor + wrapped = _wrap_functional_tensor(maybe_tensor, level) + _assert_wrapped_functional(maybe_tensor, wrapped) + if _python_functionalize: + out = FunctionalTensor(wrapped) + torch._mirror_autograd_meta_to(maybe_tensor, out) + return out + return wrapped + + +def _wrap_all_tensors_to_functional( + tensor_pytree, level, *, _python_functionalize: bool = False +): + return tree_map( + partial( + lambda x: _maybe_wrap_functional_tensor( + x, level, _python_functionalize=_python_functionalize + ) + ), + tensor_pytree, + ) + + +def _maybe_unwrap_functional_tensor(maybe_tensor, *, reapply_views: bool): + if not isinstance(maybe_tensor, torch.Tensor): + return maybe_tensor + if isinstance(maybe_tensor, FunctionalTensor): + maybe_tensor = maybe_tensor.elem + + if not torch._is_functional_tensor(maybe_tensor): + # If it's not a functional tensor, just return it. + # This can happen if we functionalize a fn that returns a global, + # which was never wrapped properly. + return maybe_tensor + # Sync any pending updates on the output tensor + torch._sync(maybe_tensor) + return _unwrap_functional_tensor(maybe_tensor, reapply_views) + + +def _unwrap_all_tensors_from_functional(tensor_pytree, *, reapply_views: bool): + return tree_map( + lambda t: _maybe_unwrap_functional_tensor(t, reapply_views=reapply_views), + tensor_pytree, + ) + + +@exposed_in("torch.func") +def functionalize(func: Callable, *, remove: str = "mutations") -> Callable: + """ + functionalize is a transform that can be used to remove (intermediate) + mutations and aliasing from a function, while preserving the function's + semantics. + + ``functionalize(func)`` returns a new function with the same semantics + as ``func``, but with all intermediate mutations removed. + Every inplace operation performed on an intermediate tensor: + ``intermediate.foo_()`` + gets replaced by its out-of-place equivalent: + ``intermediate_updated = intermediate.foo()``. + + functionalize is useful for shipping a pytorch program off to + backends or compilers that aren't able to easily represent + mutations or aliasing operators. + + Args: + func (Callable): A Python function that takes one or more arguments. + remove (str): An optional string argument, that takes on either + the value 'mutations' or 'mutations_and_views'. + If 'mutations' is passed in then all mutating operators + will be replaced with their non-mutating equivalents. + If 'mutations_and_views' is passed in, then additionally, all aliasing + operators will be replaced with their non-aliasing equivalents. + Default: 'mutations'. + + Returns: + Returns a new "functionalized" function. It takes the same inputs as + ``func``, and has the same behavior, but any mutations + (and optionally aliasing) performed on intermediate tensors + in the function will be removed. + + functionalize will also remove mutations (and views) that were performed on function inputs. + However to preserve semantics, functionalize will "fix up" the mutations after + the transform has finished running, by detecting if any tensor inputs "should have" + been mutated, and copying the new data back to the inputs if necessary. + + + Example:: + + >>> # xdoctest: +SKIP + >>> import torch + >>> from torch.fx.experimental.proxy_tensor import make_fx + >>> from torch.func import functionalize + >>> + >>> # A function that uses mutations and views, but only on intermediate tensors. + >>> def f(a): + ... b = a + 1 + ... c = b.view(-1) + ... c.add_(1) + ... return b + ... + >>> inpt = torch.randn(2) + >>> + >>> out1 = f(inpt) + >>> out2 = functionalize(f)(inpt) + >>> + >>> # semantics are the same (outputs are equivalent) + >>> print(torch.allclose(out1, out2)) + True + >>> + >>> f_traced = make_fx(f)(inpt) + >>> f_no_mutations_traced = make_fx(functionalize(f))(inpt) + >>> f_no_mutations_and_views_traced = make_fx(functionalize(f, remove='mutations_and_views'))(inpt) + >>> + >>> print(f_traced.code) + + + + def forward(self, a_1): + add = torch.ops.aten.add(a_1, 1); a_1 = None + view = torch.ops.aten.view(add, [-1]) + add_ = torch.ops.aten.add_(view, 1); view = None + return add + + >>> print(f_no_mutations_traced.code) + + + + def forward(self, a_1): + add = torch.ops.aten.add(a_1, 1); a_1 = None + view = torch.ops.aten.view(add, [-1]); add = None + add_1 = torch.ops.aten.add(view, 1); view = None + view_1 = torch.ops.aten.view(add_1, [2]); add_1 = None + return view_1 + + >>> print(f_no_mutations_and_views_traced.code) + + + + def forward(self, a_1): + add = torch.ops.aten.add(a_1, 1); a_1 = None + view_copy = torch.ops.aten.view_copy(add, [-1]); add = None + add_1 = torch.ops.aten.add(view_copy, 1); view_copy = None + view_copy_1 = torch.ops.aten.view_copy(add_1, [2]); add_1 = None + return view_copy_1 + + + >>> # A function that mutates its input tensor + >>> def f(a): + ... b = a.view(-1) + ... b.add_(1) + ... return a + ... + >>> f_no_mutations_and_views_traced = make_fx(functionalize(f, remove='mutations_and_views'))(inpt) + >>> # + >>> # All mutations and views have been removed, + >>> # but there is an extra copy_ in the graph to correctly apply the mutation to the input + >>> # after the function has completed. + >>> print(f_no_mutations_and_views_traced.code) + + + + def forward(self, a_1): + view_copy = torch.ops.aten.view_copy(a_1, [-1]) + add = torch.ops.aten.add(view_copy, 1); view_copy = None + view_copy_1 = torch.ops.aten.view_copy(add, [2]); add = None + copy_ = torch.ops.aten.copy_(a_1, view_copy_1); a_1 = None + return view_copy_1 + + + There are a few "failure modes" for functionalize that are worth calling out: + (1) Like other torch.func transforms, `functionalize()` doesn't work with functions + that directly use `.backward()`. The same is true for torch.autograd.grad. + If you want to use autograd, you can compute gradients directly + with `functionalize(grad(f))`. + (2) Like other torch.func transforms, `functionalize()` doesn't work with global state. + If you call `functionalize(f)` on a function that takes views / mutations of + non-local state, functionalization will simply no-op and pass the view/mutation + calls directly to the backend. + One way to work around this is to ensure that any non-local state creation + is wrapped into a larger function, which you then call functionalize on. + (3) `resize_()` has some limitations: functionalize will only work on programs + that use resize_()` as long as the tensor being resized is not a view. + (4) `as_strided()` has some limitations: functionalize will not work on + `as_strided()` calls that result in tensors with overlapping memory. + + + Finally, a helpful mental model for understanding functionalization is that + most user pytorch programs are writing with the public torch API. + When executed, torch operators are generally decomposed into + our internal C++ "ATen" API. + The logic for functionalization happens entirely at the level of ATen. + Functionalization knows how to take every aliasing operator in ATen, + and map it to its non-aliasing equivalent + (e.g. ``tensor.view({-1})`` -> ``at::view_copy(tensor, {-1})``), + and how to take every mutating operator in ATen, + and map it to its non-mutating equivalent + (e.g. ``tensor.add_(1)`` -> ``at::add(tensor, -1)``), + while tracking aliases and mutations out-of-line to know when to fix things up. + Information about which ATen operators are aliasing or mutating all comes from + https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/native/native_functions.yaml. + """ + if remove == "mutations": + reapply_views = True + elif remove == "mutations_and_views": + reapply_views = False + else: + raise RuntimeError( + f"functionalize(f, remove='mutations'): received invalid argument for remove={remove}." + " Valid options are:\n" + " remove='mutations': all inplace and out= operators will be removed from the program, and replaced" + " with their out-of-place equivalents.\n" + " remove='mutations_and_views': In addition to the above, all aliasing operators {view} will be" + " replaced with their non-aliasing counterparts, {view}_copy.\n" + ) + + @wraps(func) + def wrapped(*args, **kwargs): + try: + func_level = _func_increment_nesting(reapply_views) + func_args = _wrap_all_tensors_to_functional(args, func_level) + func_kwargs = _wrap_all_tensors_to_functional(kwargs, func_level) + + flattened_unwrapped_args = pytree.arg_tree_leaves(*args) + flattened_wrapped_args = pytree.arg_tree_leaves(*func_args) + flattened_unwrapped_kwargs = pytree.arg_tree_leaves(**kwargs) + flattened_wrapped_kwargs = pytree.arg_tree_leaves(**func_kwargs) + + func_outputs = func(*func_args, **func_kwargs) + outputs = _unwrap_all_tensors_from_functional( + func_outputs, reapply_views=reapply_views + ) + + for a in flattened_wrapped_args + flattened_wrapped_kwargs: + if isinstance(a, torch.Tensor): + # Call sync_() on the inputs, to ensure that any pending mutations have been applied. + torch._sync(a) + + # And if any mutations were applied to the inputs, we need to propagate them back to the user. + for unwrapped, wrapped in zip( + flattened_unwrapped_args, flattened_wrapped_args + ): + if isinstance(unwrapped, torch.Tensor) and isinstance( + wrapped, torch.Tensor + ): + _propagate_functional_input_mutation(unwrapped, wrapped) + for unwrapped, wrapped in zip( + flattened_unwrapped_kwargs, flattened_wrapped_kwargs + ): + if isinstance(unwrapped, torch.Tensor) and isinstance( + wrapped, torch.Tensor + ): + _propagate_functional_input_mutation(unwrapped, wrapped) + + return outputs + finally: + _func_decrement_nesting() + + return wrapped + + +@exposed_in("torch.func") +def linearize(func: Callable, *primals) -> tuple[Any, Callable]: + """ + Returns the value of ``func`` at ``primals`` and linear approximation + at ``primals``. + + Args: + func (Callable): A Python function that takes one or more arguments. + primals (Tensors): Positional arguments to ``func`` that must all be + Tensors. These are the values at which the function is linearly approximated. + + Returns: + Returns a ``(output, jvp_fn)`` tuple containing the output of ``func`` + applied to ``primals`` and a function that computes the jvp of + ``func`` evaluated at ``primals``. + + linearize is useful if jvp is to be computed multiple times at ``primals``. However, + to achieve this, linearize saves intermediate computation and has higher memory requirements + than directly applying `jvp`. So, if all the ``tangents`` are known, it maybe more efficient + to compute vmap(jvp) instead of using linearize. + + .. note:: + linearize evaluates ``func`` twice. Please file an issue for an implementation + with a single evaluation. + + Example:: + + >>> import torch + >>> from torch.func import linearize + >>> def fn(x): + ... return x.sin() + ... + >>> output, jvp_fn = linearize(fn, torch.zeros(3, 3)) + >>> jvp_fn(torch.ones(3, 3)) + tensor([[1., 1., 1.], + [1., 1., 1.], + [1., 1., 1.]]) + >>> + + """ + # Note: We evaluate `fn` twice. + # Once for returning the output and other while + # tracing the graph. + # If this becomes a bottle-neck, we should update + # make_fx such that it also returns the output. + + output = func(*primals) + _, output_spec = tree_flatten(output) + + flat_primals, primals_argspec = tree_flatten(primals) + + # tangents for tracing + flat_tangents = tuple(p.new_empty(()).expand_as(p) for p in flat_primals) + + # function to trace + def trace_fn(flat_tangents): + with fwAD.dual_level(): + flat_duals = tuple( + fwAD.make_dual(p, t) for p, t in zip(flat_primals, flat_tangents) + ) + duals = tree_unflatten(flat_duals, primals_argspec) + output = func(*duals) + tangents = tree_map_only( + torch.Tensor, lambda dual: safe_unpack_dual(dual, False)[1], output + ) + + return tangents + + jvp_graph = lazy_dynamo_disallow(make_fx)(trace_fn)(flat_tangents) + const_folded_jvp_graph = lazy_dynamo_disallow(const_fold.split_const_subgraphs)( + jvp_graph + ) + + # Hold only the meta-data regarding the primals. + flat_primals_shape = tuple(p.shape for p in flat_primals) + flat_primals_device = tuple(p.device for p in flat_primals) + flat_primals_dtype = tuple(p.dtype for p in flat_primals) + + def forward_ad_checks(flat_tangents): + for idx, t in enumerate(flat_tangents): + if t.shape != flat_primals_shape[idx]: + msg = ( + f"tangent:{idx} with shape {t.shape} in flattened " + f"pytree doesn't match the shape {flat_primals_shape[idx]} " + "of the corresponding primal." + ) + raise RuntimeError(msg) + + if t.device != flat_primals_device[idx]: + msg = ( + f"tangent:{idx} with device {t.device} in flattened " + f"pytree doesn't match the device {flat_primals_device[idx]} " + "of the corresponding primal." + ) + raise RuntimeError(msg) + + if t.dtype != flat_primals_dtype[idx]: + msg = ( + f"tangent:{idx} with dtype {t.dtype} in flattened " + f"pytree doesn't match the dtype {flat_primals_dtype[idx]} " + "of the corresponding primal." + ) + raise RuntimeError(msg) + + # jvp_fn : callable to return + # It takes care of checking the argspec of tangents, + # calling the folded fx graph and unflattening fx graph output + def jvp_fn(*tangents): + flat_tangents, tangent_argspec = tree_flatten(tangents) + if tangent_argspec != primals_argspec: + raise RuntimeError( + f"Expected the tangents {tangent_argspec} to have " + f"the same argspec as the primals {primals_argspec}" + ) + + forward_ad_checks(flat_tangents) + + flat_output = const_folded_jvp_graph(*flat_tangents) + # const folded graph can return flat output, + # so transform output. + return tree_unflatten(flat_output, output_spec) + + return output, jvp_fn + + +@exposed_in("torch.func") +def debug_unwrap(tensor: torch.Tensor, *, recurse=True) -> torch.Tensor: + """Unwraps a functorch tensor (e.g. BatchedTensor, GradTrackingTensor) to its underlying tensor. + + This function should only be used in a debug setting (e.g. trying to print the + value of a Tensor in a debugger). Otherwise, using the result of function + inside of a function being transformed will lead to undefined behavior. + """ + if not is_functorch_wrapped_tensor(tensor): + return tensor + result = get_unwrapped(tensor) + if recurse: + return debug_unwrap(result) + return result diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/functional_call.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/functional_call.py new file mode 100644 index 0000000000000000000000000000000000000000..8e2f943d3e4479c0a44d2102db13f541fe7bcd5e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/functional_call.py @@ -0,0 +1,263 @@ +# mypy: allow-untyped-defs +from collections.abc import Sequence +from typing import Any, Optional, Union + +import torch +import torch.nn as nn +from torch import Tensor +from torch._functorch.utils import exposed_in + + +@exposed_in("torch.func") +def functional_call( + module: "torch.nn.Module", + parameter_and_buffer_dicts: Union[dict[str, Tensor], Sequence[dict[str, Tensor]]], + args: Optional[Union[Any, tuple]] = None, + kwargs: Optional[dict[str, Any]] = None, + *, + tie_weights: bool = True, + strict: bool = False, +): + r"""Performs a functional call on the module by replacing the module parameters + and buffers with the provided ones. + + .. note:: If the module has active parametrizations, passing a value in the + :attr:`parameter_and_buffer_dicts` argument with the name set to the regular parameter + name will completely disable the parametrization. + If you want to apply the parametrization function to the value passed + please set the key as ``{submodule_name}.parametrizations.{parameter_name}.original``. + + .. note:: If the module performs in-place operations on parameters/buffers, these will be reflected + in the ``parameter_and_buffer_dicts`` input. + + + Example:: + + >>> a = {'foo': torch.zeros(())} + >>> # xdoctest: +SKIP + >>> mod = Foo() # does self.foo = self.foo + 1 + >>> print(mod.foo) # tensor(0.) + >>> functional_call(mod, a, torch.ones(())) + >>> print(mod.foo) # tensor(0.) + >>> print(a['foo']) # tensor(1.) + + .. note:: If the module has tied weights, whether or not functional_call respects the tying is determined by the + tie_weights flag. + + Example:: + + >>> a = {'foo': torch.zeros(())} + >>> # xdoctest: +SKIP + >>> mod = Foo() # has both self.foo and self.foo_tied which are tied. Returns x + self.foo + self.foo_tied + >>> print(mod.foo) # tensor(1.) + >>> mod(torch.zeros(())) # tensor(2.) + >>> functional_call(mod, a, torch.zeros(())) # tensor(0.) since it will change self.foo_tied too + >>> functional_call(mod, a, torch.zeros(()), tie_weights=False) # tensor(1.)--self.foo_tied is not updated + >>> new_a = {'foo': torch.zeros(()), 'foo_tied': torch.zeros(())} + >>> functional_call(mod, new_a, torch.zeros()) # tensor(0.) + + An example of passing multiple dictionaries + + .. code-block:: python + + a = ( + {"weight": torch.ones(1, 1)}, + {"buffer": torch.zeros(1)}, + ) # two separate dictionaries + mod = nn.Bar(1, 1) # return self.weight @ x + self.buffer + print(mod.weight) # tensor(...) + print(mod.buffer) # tensor(...) + x = torch.randn((1, 1)) + print(x) + functional_call(mod, a, x) # same as x + print(mod.weight) # same as before functional_call + + + And here is an example of applying the grad transform over the parameters + of a model. + + .. code-block:: python + + import torch + import torch.nn as nn + from torch.func import functional_call, grad + + x = torch.randn(4, 3) + t = torch.randn(4, 3) + model = nn.Linear(3, 3) + + + def compute_loss(params, x, t): + y = functional_call(model, params, x) + return nn.functional.mse_loss(y, t) + + + grad_weights = grad(compute_loss)(dict(model.named_parameters()), x, t) + + .. note:: If the user does not need grad tracking outside of grad transforms, they can detach all of the + parameters for better performance and memory usage + + Example:: + + >>> detached_params = {k: v.detach() for k, v in model.named_parameters()} + >>> grad_weights = grad(compute_loss)(detached_params, x, t) + >>> grad_weights.grad_fn # None--it's not tracking gradients outside of grad + + This means that the user cannot call ``grad_weight.backward()``. However, if they don't need autograd tracking + outside of the transforms, this will result in less memory usage and faster speeds. + + Args: + module (torch.nn.Module): the module to call + parameters_and_buffer_dicts (Dict[str, Tensor] or tuple of Dict[str, Tensor]): the parameters that will be used in + the module call. If given a tuple of dictionaries, they must have distinct keys so that all dictionaries can + be used together + args (Any or tuple): arguments to be passed to the module call. If not a tuple, considered a single argument. + kwargs (dict): keyword arguments to be passed to the module call + tie_weights (bool, optional): If True, then parameters and buffers tied in the original model will be treated as + tied in the reparameterized version. Therefore, if True and different values are passed for the tied + parameters and buffers, it will error. If False, it will not respect the originally tied parameters and + buffers unless the values passed for both weights are the same. Default: True. + strict (bool, optional): If True, then the parameters and buffers passed in must match the parameters and + buffers in the original module. Therefore, if True and there are any missing or unexpected keys, it will + error. Default: False. + + Returns: + Any: the result of calling ``module``. + """ + if isinstance(parameter_and_buffer_dicts, dict): + parameters_and_buffers = parameter_and_buffer_dicts + elif isinstance(parameter_and_buffer_dicts, Sequence): + if not all(isinstance(d, dict) for d in parameter_and_buffer_dicts): + raise ValueError( + "Expected all elements of parameter_and_buffer_dicts to be dictionaries" + ) + all_keys = [k for d in parameter_and_buffer_dicts for k in d] + all_keys_counter: dict[str, int] = {} + for k in all_keys: + v = all_keys_counter.get(k, 0) + all_keys_counter[k] = v + 1 + repeated_keys = [key for key, n in all_keys_counter.items() if n > 1] + if len(repeated_keys) > 0: + raise ValueError( + f"{repeated_keys} appeared in multiple dictionaries; behavior of functional call is ambiguous" + ) + parameters_and_buffers = { + k: v for d in parameter_and_buffer_dicts for k, v in d.items() + } + else: + raise ValueError( + f"Expected parameter_and_buffer_dicts to be a dict, or a list/tuple of dicts, " + f"but got {type(parameter_and_buffer_dicts)}" + ) + + return nn.utils.stateless._functional_call( + module, + parameters_and_buffers, + args, + kwargs, + tie_weights=tie_weights, + strict=strict, + ) + + +@exposed_in("torch.func") +def stack_module_state( + models: Union[Sequence[nn.Module], nn.ModuleList], +) -> tuple[dict[str, Any], dict[str, Any]]: + """stack_module_state(models) -> params, buffers + + Prepares a list of torch.nn.Modules for ensembling with :func:`vmap`. + + Given a list of ``M`` ``nn.Modules`` of the same class, returns two dictionaries + that stack all of their parameters and buffers together, indexed by name. + The stacked parameters are optimizable (i.e. they are new leaf nodes in the + autograd history that are unrelated to the original parameters and can be + passed directly to an optimizer). + + Here's an example of how to ensemble over a very simple model: + + .. code-block:: python + + num_models = 5 + batch_size = 64 + in_features, out_features = 3, 3 + models = [torch.nn.Linear(in_features, out_features) for i in range(num_models)] + data = torch.randn(batch_size, 3) + + + def wrapper(params, buffers, data): + return torch.func.functional_call(models[0], (params, buffers), data) + + + params, buffers = stack_module_state(models) + output = vmap(wrapper, (0, 0, None))(params, buffers, data) + + assert output.shape == (num_models, batch_size, out_features) + + When there's submodules, this follows state dict naming conventions + + .. code-block:: python + + import torch.nn as nn + + + class Foo(nn.Module): + def __init__(self, in_features, out_features): + super().__init__() + hidden = 4 + self.l1 = nn.Linear(in_features, hidden) + self.l2 = nn.Linear(hidden, out_features) + + def forward(self, x): + return self.l2(self.l1(x)) + + + num_models = 5 + in_features, out_features = 3, 3 + models = [Foo(in_features, out_features) for i in range(num_models)] + params, buffers = stack_module_state(models) + print(list(params.keys())) # "l1.weight", "l1.bias", "l2.weight", "l2.bias" + + .. warning:: + All of the modules being stacked together must be the same (except for + the values of their parameters/buffers). For example, they should be in the + same mode (training vs eval). + """ + if len(models) == 0: + raise RuntimeError("stack_module_state: Expected at least one model, got 0.") + if not (all(m.training for m in models) or all(not m.training for m in models)): + raise RuntimeError( + "stack_module_state: Expected all models to have the same training/eval mode." + ) + model0_typ = type(models[0]) + if not all(type(m) is model0_typ for m in models): + raise RuntimeError( + "stack_module_state: Expected all models to be of the same class." + ) + all_params = [dict(model.named_parameters()) for model in models] + params = { + k: construct_stacked_leaf(tuple(params[k] for params in all_params), k) + for k in all_params[0] + } + all_buffers = [dict(model.named_buffers()) for model in models] + buffers = { + k: construct_stacked_leaf(tuple(buffers[k] for buffers in all_buffers), k) + for k in all_buffers[0] + } + + return params, buffers + + +def construct_stacked_leaf( + tensors: Union[tuple[Tensor, ...], list[Tensor]], name: str +) -> Tensor: + all_requires_grad = all(t.requires_grad for t in tensors) + none_requires_grad = all(not t.requires_grad for t in tensors) + if not all_requires_grad and not none_requires_grad: + raise RuntimeError( + f"Expected {name} from each model to have the same .requires_grad" + ) + result = torch.stack(tensors) + if all_requires_grad: + result = result.detach().requires_grad_() + return result diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/fx_minifier.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/fx_minifier.py new file mode 100644 index 0000000000000000000000000000000000000000..60609ad95e68b171efa4298fc70d0890bc24e510 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/fx_minifier.py @@ -0,0 +1,501 @@ +# mypy: ignore-errors + +import copy +import math +import os +import sys +from collections.abc import Callable +from dataclasses import dataclass +from functools import partial, wraps + +import torch +import torch.fx as fx +from torch.hub import tqdm +from torch.multiprocessing.reductions import StorageWeakRef +from torch.utils._content_store import ContentStoreWriter + +from .compile_utils import get_outputs, get_placeholders + + +is_tuple = object() + + +@dataclass +class LoadTensorMeta: + size: list[int] + stride: list[int] + dtype: torch.dtype + device: torch.device + + +class ConcreteProp(torch.fx.Interpreter): + def __init__(self, mod, *, writer=None, skip_offload=False): + super().__init__(mod) + self.writer = writer + self.skip_offload = skip_offload + self.seen_storages = set() + + def run_node(self, n): + self.pbar.update(1) + r = super().run_node(n) + name = n.name + + if isinstance(r, torch.Tensor): + if self.writer is None: + n.meta["concrete_value"] = r + else: + if StorageWeakRef(r.untyped_storage()) in self.seen_storages: + # Refuse to offload tensors which alias other live + # tensors, because this will violate operator contracts + n.meta["concrete_value"] = None + else: + if not self.skip_offload: + self.writer.write_tensor(os.path.join("eager", name), r) + n.meta["concrete_value"] = LoadTensorMeta( + r.size(), r.stride(), r.dtype, r.device + ) + self.seen_storages.add(StorageWeakRef(r.untyped_storage())) + else: + n.meta["concrete_value"] = is_tuple + + return r + + def propagate(self, *args): + with tqdm( + desc="Saving intermediates for delta debugging", + total=len(self.module.graph.nodes), + disable=self.writer is None, + ) as pbar: + self.pbar = pbar + r = super().run(*args) + if not self.skip_offload: + pbar.set_description( + "Saved! To skip next time, run with --skip-saving-eager-intermediates" + ) + return r + + +def is_load_tensor_node(node): + return ( + node.op == "call_function" + and node.target is torch.ops.debugprims.load_tensor.default + ) + + +# inplace modifies node/inps +def _convert_node_to_placeholder(graph, node, inps): + if node.op == "output" or node.op == "placeholder": + return False + + if is_load_tensor_node(node): + return False + + concrete_val = node.meta.get("concrete_value", None) + + if isinstance(concrete_val, torch.Tensor): + node.op = "placeholder" + node.target = node.name + node.args = () + node.kwargs = {} + + inps.append(concrete_val) + return True + + elif concrete_val is None: + return False + + elif concrete_val is is_tuple: + r = False + for tuple_user in list(node.users): + r = _convert_node_to_placeholder(graph, tuple_user, inps) or r + # NB: We must not erase the node at this point, because + # we are iterating over the nodes and this would change + # the iteration order + # graph.erase_node(node) + return r + + elif isinstance(concrete_val, LoadTensorMeta): + node.op = "call_function" + node.target = torch.ops.debugprims.load_tensor.default + node.args = ( + os.path.join("eager", node.name), + concrete_val.size, + concrete_val.stride, + ) + node.kwargs = { + "device": concrete_val.device, + "dtype": concrete_val.dtype, + } + return True + + return False + + +def create_minified_hlo_graph(minified_fx_graph, inputs): + """ + Takes minified FX graph as primary input, and ports it to HLO via StableHLO + Provides minified HLO graph as output, and archive them to local directory + """ + hlo_dir = f"{os.getcwd()}/hlo_files" + os.makedirs(hlo_dir, exists_ok=True) + + from torch_xla.stablehlo import save_torch_model_as_stablehlo + + save_torch_model_as_stablehlo(minified_fx_graph, inputs, hlo_dir) + + +def dump_state(fx_g, inps): + print( + f""" +# Working Repro with {len(fx_g.graph.nodes)} nodes +inps = {[(i.shape, i.dtype, i.device.type) for i in inps]} +inps = [torch.zeros(())] + [torch.ones(shape, dtype=dtype, device=device) for (shape, dtype, device) in inps] +{fx_g.code} +""" + ) + + +def is_power_of_two(n): + if n == 0: + return False + return (n & (n - 1)) == 0 + + +@dataclass +class ReproState: + graph: fx.Graph + inps: list[torch.Tensor] + + def __post_init__(self): + ph_nodes = get_placeholders(self.graph) + assert len(ph_nodes) == len(self.inps) + + +def minifier( + fail_f: fx.GraphModule, + inps, + module_fails, + dump_state: Callable = dump_state, + *, + save_dir=None, + offload_to_disk=False, + skip_offload=False, + skip_sanity=False, + max_granularity=None, +): + """ + Minimizes a FX graph with given inputs, such that the resulting FX graph still returns True for module_fails. + + Does 2 main strategies: + 1. Truncates suffix: Removes some suffix from the graph and sets a new output. + 2. Delta Debugging: Tries replacing half of the graph with inputs. If fails, + tries replacing quarter of the graph, etc. + + >>> # xdoctest: +SKIP(failing) + >>> failing_function = fx.symbolic_trace(f) + >>> minimize(failing_function, [torch.randn(5)], lambda fx_g, inps: fx_g(*inps)) + + note: module_fails returns True if it fails. + """ + assert isinstance(inps, (tuple, list)) + + failing_graph = fail_f.graph + cur_size = len(failing_graph.nodes) + + if max_granularity is not None and not is_power_of_two(max_granularity): + raise RuntimeError(f"max_granularity {max_granularity} not power of two") + + num_queries = 0 + + def deepcopy_fx_graph(fx_graph): + return fx.GraphModule(fail_f, copy.deepcopy(fx_graph)).graph + + def graph_fails(graph, inps): + nonlocal num_queries + graph = copy.deepcopy(graph) + num_queries += 1 + mod = fx.GraphModule(fail_f, graph) + mod.graph.lint() + return module_fails(mod, inps) + + writer = None + if offload_to_disk: + writer = ContentStoreWriter(save_dir) + + ConcreteProp(fail_f, writer=writer, skip_offload=skip_offload).propagate(*inps) + if not skip_sanity and not graph_fails(failing_graph, inps): + raise RuntimeError("Input graph did not fail the tester") + print(f"Started off with {cur_size} nodes", file=sys.stderr) + + def _register_strategy(strategy: Callable, name: str): + @wraps(strategy) + def new_func(old_state: ReproState, granularity=1): + print(file=sys.stderr) + print( + f"Strategy: {name} (G: {granularity}) " + f"({len(old_state.graph.nodes)} nodes, {len(old_state.inps)} inputs)", + file=sys.stderr, + ) + new_state = strategy( + deepcopy_fx_graph(old_state.graph), list(old_state.inps), granularity + ) + if new_state is not None: + new_nodes = len(new_state.graph.nodes) + old_nodes = len(old_state.graph.nodes) + new_inps = len(new_state.inps) + old_inps = len(old_state.inps) + new_outs = len(get_outputs(new_state.graph)) + old_outs = len(get_outputs(old_state.graph)) + progress_made = False + if new_nodes < old_nodes: + progress_made = True + print( + f"SUCCESS: Went from {old_nodes} to {new_nodes} nodes", + file=sys.stderr, + ) + if new_inps > old_inps: + progress_made = True + print( + f"SUCCESS: Went from {old_inps} to {new_inps} inputs", + file=sys.stderr, + ) + if new_outs < old_outs: + progress_made = True + print( + f"SUCCESS: Went from {old_outs} to {new_outs} outputs", + file=sys.stderr, + ) + + if not progress_made: + raise RuntimeError("Success raised but no progress made?") + + if not graph_fails(new_state.graph, new_state.inps): + print( + "WARNING: Something went wrong, not applying this minification", + file=sys.stderr, + ) + return None + return new_state + else: + print(f"FAIL: {name}", file=sys.stderr) + return None + + return new_func + + def register_strategy(name: str): + return partial(_register_strategy, name=name) + + @register_strategy("Truncate suffix") + def remove_suffix(cur_graph, cur_inps, granularity): + tested = set() + new_graph = fx.Graph() + env = {} + for idx, node in enumerate(cur_graph.nodes): + new_node = new_graph.node_copy(node, lambda x: env[x]) + if node.op not in ["placeholder", "output"]: + # If idx is divisible by (granularity * 2), it would have been checked already. + if ( + idx % granularity == 0 + and (idx % (granularity * 2) != 0) + and idx not in tested + ): + output_node = new_graph.output((new_node,)) + if len(new_graph.nodes) < len(cur_graph.nodes) and graph_fails( + new_graph, cur_inps + ): + return ReproState(new_graph, cur_inps) + else: + tested.add(idx) + new_graph.erase_node(output_node) + env[node] = new_node + return None + + @register_strategy("Remove outputs") + def remove_outputs(cur_graph, cur_inps, granularity): + granularity = max(1, granularity // 2) + for idx, node in enumerate(cur_graph.nodes): + node.idx = idx + if node.op == "output": + output = node + break + + if isinstance(output.args[0], fx.Node): + return None + + output_args = sorted( + output.args[0], key=lambda x: x.idx if isinstance(x, fx.Node) else int(1e9) + ) + if len(output_args) == 1: + return None + + for idx in range(0, len(output_args), granularity): + output.args = (output_args[:idx] + output_args[idx + granularity :],) + if graph_fails(cur_graph, cur_inps): + return ReproState(cur_graph, cur_inps) + return None + + def remove_unused_inputs_unchecked(cur_state: ReproState): + cur_graph = cur_state.graph + cur_inps = cur_state.inps + ph_nodes = get_placeholders(cur_graph) + assert len(ph_nodes) == len(cur_inps) + + new_inps = [] + for idx in range(len(ph_nodes)): + if len(ph_nodes[idx].users) == 0: + cur_graph.erase_node(ph_nodes[idx]) + else: + new_inps.append(cur_inps[idx]) + if len(new_inps) < len(cur_inps): + return ReproState(cur_graph, new_inps) + return None + + def remove_unused_inputs_checked(cur_state: ReproState): + new_state = remove_unused_inputs_unchecked(cur_state) + if new_state is not None and graph_fails(new_state.graph, new_state.inps): + return new_state + return None + + def _remove_unused_wrapper(cur_graph, cur_inps, granularity): + return remove_unused_inputs_checked(ReproState(cur_graph, cur_inps)) + + remove_unused_inputs = register_strategy("Remove unused inputs")( + _remove_unused_wrapper + ) + + @register_strategy("Eliminate dead code") + def eliminate_dead_code(cur_graph, cur_inps, granularity): + if cur_graph.eliminate_dead_code() and graph_fails(cur_graph, cur_inps): + return ReproState(cur_graph, cur_inps) + return None + + def _consolidate_placeholders(cur_graph, inps): + new_graph = fx.Graph() + env = {} + seen_non_placeholder = False + + # Move all placeholders to the front; also, if any load_tensor + # is at the front, convert it into an input (because it can be live + # all the time) + for node in cur_graph.nodes: + if node.op == "placeholder": + new_node = new_graph.node_copy(node, lambda x: env[x]) + env[node] = new_node + elif not seen_non_placeholder and is_load_tensor_node(node): + new_node = new_graph.placeholder(node.name) + env[node] = new_node + inps.append( + torch.ops.debugprims.load_tensor.default(*node.args, **node.kwargs) + ) + else: + seen_non_placeholder = True + + # Move everyone else + for node in cur_graph.nodes: + if node not in env: + new_node = new_graph.node_copy(node, lambda x: env[x]) + env[node] = new_node + return new_graph + + @register_strategy("Delta Debugging") + def delta_debugging(cur_graph: fx.Graph, cur_inps, granularity): + num_nodes = len(cur_graph.nodes) + for start_range in range(0, num_nodes, granularity): + is_removing = False + new_graph = deepcopy_fx_graph(cur_graph) + new_inps = cur_inps[:] + end_range = min(num_nodes, start_range + granularity) + for idx in range(start_range, end_range): + new_node = list(new_graph.nodes)[idx] + if _convert_node_to_placeholder(new_graph, new_node, new_inps): + is_removing = True + if not is_removing: + continue + new_graph.eliminate_dead_code() + new_graph = _consolidate_placeholders(new_graph, new_inps) + new_state = remove_unused_inputs_unchecked(ReproState(new_graph, new_inps)) + if new_state is None: + new_state = ReproState(new_graph, new_inps) + if graph_fails(new_state.graph, new_state.inps): + return ReproState(new_state.graph, new_state.inps) + + return None + + @register_strategy("Consolidate Inputs") + def consolidate_inputs(cur_graph, cur_inps, granularity): + old_len = len(cur_inps) + cur_graph = _consolidate_placeholders(cur_graph, cur_inps) + if len(cur_inps) > old_len and graph_fails(cur_graph, cur_inps): + return ReproState(cur_graph, cur_inps) + return None + + failing_state = ReproState(failing_graph, inps) + + def try_granularity(failing_state, granularity, use_non_granular): + print(f"Trying granularity {granularity}", file=sys.stderr) + + strategies = [] + num_nodes = len(failing_state.graph.nodes) + num_outputs = len(get_outputs(failing_state.graph)) + if num_outputs > num_nodes // 2: + strategies += [remove_outputs] + + if use_non_granular: + strategies += [ + eliminate_dead_code, + remove_unused_inputs, + consolidate_inputs, + ] + + strategies += [remove_suffix, delta_debugging] + + for strategy in strategies: + new_state = strategy(failing_state, granularity) + if new_state is not None: + return new_state + return None + + while True: + dump_state(fx.GraphModule(fail_f, failing_state.graph), failing_state.inps) + granularity = int(2 ** (math.floor(math.log2(len(failing_state.graph.nodes))))) + if max_granularity is not None: + granularity = min(max_granularity, granularity) + new_state = try_granularity(failing_state, granularity, use_non_granular=True) + if new_state is not None: + failing_state = new_state + continue + + granularity //= 2 + has_progress = False + while granularity >= 1: + new_state = try_granularity( + failing_state, granularity, use_non_granular=False + ) + if new_state is not None: + failing_state = new_state + has_progress = True + break + granularity //= 2 + if has_progress: + continue + + new_state = remove_outputs(failing_state, 1) + if new_state is not None: + failing_state = new_state + continue + + break + + if not graph_fails(failing_state.graph, failing_state.inps): + raise RuntimeError("Uh oh, something went wrong :( Final graph is not failing") + + print(f"Made {num_queries} queries", file=sys.stderr) + failing_fx = fx.GraphModule(fail_f, failing_state.graph) + + # If XLA debugging environment is enabled, create minified HLO graph as well + if "XLA_HLO_DEBUG" in os.environ: + create_minified_hlo_graph(failing_fx, failing_state.inps) + + dump_state(failing_fx, failing_state.inps) + print("Wrote minimal repro out to repro.py", file=sys.stderr) + return failing_fx, failing_state.inps diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/make_functional.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/make_functional.py new file mode 100644 index 0000000000000000000000000000000000000000..03ecb1c4840e44a685c47f5bec26e8961ba9461e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/make_functional.py @@ -0,0 +1,615 @@ +# mypy: allow-untyped-defs +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import copy +from collections.abc import Callable, Iterable, Sequence +from typing import Any, NoReturn, Union + +import torch +import torch.nn as nn +from torch import Tensor +from torch.nn.utils._named_member_accessor import NamedMemberAccessor + + +# Utilities to make nn.Module "functional" +# In particular the goal is to be able to provide a function that takes as input +# the parameters and evaluate the nn.Module using fixed inputs. + + +def raise_parameter_tying_error() -> NoReturn: + raise RuntimeError( + "make_functional(module): we don't yet support models that " + "do parameter tying (also sometimes known as weight sharing). " + "Please try to rewrite your model by replacing all instances of the " + "tied parameter with another and/or comment your support in " + "https://github.com/pytorch/functorch/issues/446" + ) + + +def create_names_map( + named_params: Union[dict[str, Tensor], Iterable[tuple[str, Tensor]]], + tied_named_params: Union[dict[str, Tensor], Iterable[tuple[str, Tensor]]], +) -> dict[str, list[str]]: + """ + named_params is a dictionary of tensors: {'A': A, 'B': B} + tied_named_params is another dictionary of tensors {'A': A, 'B': B, 'B_tied': B} + with potentially tied (or 'duplicated') tensors + + This function creates a mapping from the names in named_params to the + names in tied_named_params: {'A': ['A'], 'B': ['B', 'B_tied']}. + """ + # pyrefly: ignore [no-matching-overload] + named_params = dict(named_params) + # pyrefly: ignore [no-matching-overload] + tied_named_params = dict(tied_named_params) + + tensors_dict_keys = set(named_params.keys()) + tied_tensors_dict_keys = set(tied_named_params.keys()) + assert tensors_dict_keys.issubset(tied_tensors_dict_keys) + + tensor_to_mapping: dict[Tensor, tuple[str, list[str]]] = {} + for key, tensor in named_params.items(): + # pyrefly: ignore [unsupported-operation] + tensor_to_mapping[tensor] = (key, []) + for key, tensor in tied_named_params.items(): + assert tensor in tensor_to_mapping + # pyrefly: ignore [bad-argument-type] + tensor_to_mapping[tensor][1].append(key) + return dict(tensor_to_mapping.values()) + + +def _extract_members( + mod: nn.Module, + named_members: Callable[..., Iterable[tuple[str, Tensor]]], + subclass: Callable[[Tensor], Tensor], +) -> tuple[tuple[Tensor, ...], tuple[str, ...], dict[str, list[str]]]: + all_named_members = tuple(named_members(remove_duplicate=False)) + unique_named_members = tuple(named_members(remove_duplicate=True)) + names_map = create_names_map(unique_named_members, all_named_members) + + # Remove all the members in the model + memo = {} + accessor = NamedMemberAccessor(mod) + for name, p in all_named_members: + if p not in memo: + memo[p] = subclass(torch.empty_like(p, device="meta")) + replacement = memo[p] + accessor.set_tensor(name, replacement) + + if len(unique_named_members) == 0: + names, params = (), () + else: + names, params = zip(*unique_named_members) # type: ignore[assignment] + return params, names, names_map + + +def extract_weights( + mod: nn.Module, +) -> tuple[tuple[Tensor, ...], tuple[str, ...], dict[str, list[str]]]: + """ + This function removes all the Parameters from the model and + return them as a tuple as well as their original attribute names. + The weights must be re-loaded with `load_weights` before the model + can be used again. + Note that this function modifies the model in place and after this + call, mod.parameters() will be empty. + """ + return _extract_members(mod, mod.named_parameters, nn.Parameter) + + +def extract_buffers( + mod: nn.Module, +) -> tuple[tuple[Tensor, ...], tuple[str, ...], dict[str, list[str]]]: + return _extract_members(mod, mod.named_buffers, lambda x: x) + + +def load_weights( + mod: nn.Module, + names: Sequence[str], + params: Sequence[Tensor], + as_params: bool = False, +) -> None: + """ + Reload a set of weights so that `mod` can be used again to perform a forward pass. + Note that the `params` are regular Tensors (that can have history) and so are left + as Tensors. This means that mod.parameters() will still be empty after this call. + """ + accessor = NamedMemberAccessor(mod) + if as_params: + params = [nn.Parameter(p) for p in params] + accessor.set_tensors(names, params) + + +def _swap_state( + mod: nn.Module, names_map: dict[str, list[str]], elems: Iterable[Tensor] +) -> list[Tensor]: + result: list[Tensor] = [] + accessor = NamedMemberAccessor(mod) + for (_, attr_names), elem in zip(names_map.items(), elems): + for i, attr_name in enumerate(attr_names): + if i == 0: + result.append(accessor.swap_tensor(attr_name, elem)) + else: + accessor.set_tensor(attr_name, elem) + return result + + +def load_buffers( + mod: nn.Module, + names: Sequence[str], + buffers: Sequence[Tensor], + as_params: bool = False, +) -> None: + accessor = NamedMemberAccessor(mod) + accessor.set_tensors(names, buffers) + + +def load_state( + model: nn.Module, + weights: Sequence[Tensor], + weight_names: Sequence[str], + buffers: Sequence[Tensor] = (), + buffer_names: Sequence[str] = (), +) -> nn.Module: + """load_state(model, weights, weight_names, buffers=(), buffer_names=()) -> model + + load_state takes `weights` and `buffers` and assigns them to the model. + This is the inverse operation of `make_functional_deprecated_v1`. + """ + assert len(weight_names) == len(weights) + load_weights(model, weight_names, weights) + if len(buffers) > 0: + assert len(buffer_names) == len(buffers) + load_buffers(model, buffer_names, buffers) + return model + + +def make_functional_deprecated_v1(model: nn.Module): + """make_functional_deprecated_v1(model) -> weights, func, weight_names + + Given an nn.Module, make_functional_deprecated_v1 extracts the state (weights) + and returns a functional version of the model, `func`. This makes + it so that it is possible use transforms over the parameters of + `model`. + + `func` can be invoked as follows: + ``` + x = torch.randn(4, 3) + model = nn.Linear(3, 3) + weights, func, _ = make_functional_deprecated_v1(model) + func(weights, (x,)) + ``` + + And here is an example of applying the grad transform: + ``` + x = torch.randn(4, 3) + model = nn.Linear(3, 3) + weights, _, func = make_functional_deprecated_v1(model) + grad_weights = grad(func)(weights, (x,)) + ``` + + To put the state back into a model, use `load_state`. + """ + buffers = list(model.buffers()) + if len(buffers) > 0: + raise RuntimeError( + "make_functional_deprecated_v1(model): `model` has buffers. Please use " + "make_functional_with_buffers_deprecated_v1(model) instead." + ) + weights, descriptors, _ = extract_weights(model) + + def fun(weights, data): + mutable_model = copy.deepcopy(model) + load_weights(mutable_model, descriptors, weights) + return mutable_model(*data) + + return weights, fun, descriptors + + +def make_functional_with_buffers_deprecated_v1(model: nn.Module): + """make_functional_with_buffers_deprecated_v1(model) -> weights, buffers, func, weight_names, buffer_names + + Given an nn.Module, make_functional_with_buffers_deprecated_v1 extracts the state (weights and buffers) + and returns a functional version of the model, `func`. + + `func` can be invoked as follows: + ``` + x = torch.randn(4, 3) + model = nn.Linear(3, 3) + weights, buffers, func, _, _ = make_functional_with_buffers_deprecated_v1(model) + func(weights, buffers, (x,)) + ``` + + And here is an example of applying the grad transform: + ``` + x = torch.randn(4, 3) + model = nn.Linear(3, 3) + weights, buffers, func, _, _ = make_functional_with_buffers_deprecated_v1(model) + func(weights, buffers, (x,)) + grad_weights = grad(func)(weights, buffers, (x,)) + ``` + + To put the state back into a model, use `load_state`. + """ + weights, weight_descriptors, _ = extract_weights(model) + buffers, buf_descriptors, _ = extract_buffers(model) + + def fun(weights, buffers, data): + mutable_model = copy.deepcopy(model) + load_weights(mutable_model, weight_descriptors, weights) + load_buffers(mutable_model, buf_descriptors, buffers) + return mutable_model(*data) + + return weights, buffers, fun, weight_descriptors, buf_descriptors + + +class FunctionalModuleWithBuffers(nn.Module): + """ + This is the callable object returned by :func:`make_functional_with_buffers`. + """ + + def __init__( + self, + stateless_model: nn.Module, + param_names: tuple[str, ...], + buffer_names: tuple[str, ...], + param_names_map: dict[str, list[str]], + buffer_names_map: dict[str, list[str]], + ) -> None: + super().__init__() + self.stateless_model = stateless_model + self.param_names = param_names + self.buffer_names = buffer_names + + self.all_names_map = dict(param_names_map) + self.all_names_map.update(buffer_names_map) + + @staticmethod + def _create_from( + model: nn.Module, disable_autograd_tracking: bool = False + ) -> tuple["FunctionalModuleWithBuffers", tuple[Tensor, ...], tuple[Tensor, ...]]: + # TODO: We don't need to copy the model to create a stateless copy + model_copy = copy.deepcopy(model) + params, param_names, param_names_map = extract_weights(model_copy) + buffers, buffer_names, buffer_names_map = extract_buffers(model_copy) + if disable_autograd_tracking: + for param in params: + param.requires_grad_(False) + return ( + FunctionalModuleWithBuffers( + model_copy, param_names, buffer_names, param_names_map, buffer_names_map + ), + params, + buffers, + ) + + def forward( + self, params: Iterable[Tensor], buffers: Iterable[Tensor], *args, **kwargs + ) -> Any: + # Temporarily load the state back onto self.stateless_model + old_state = _swap_state( + self.stateless_model, + self.all_names_map, + tuple(params) + tuple(buffers), + ) + try: + return self.stateless_model(*args, **kwargs) + finally: + # Remove the loaded state on self.stateless_model + _swap_state(self.stateless_model, self.all_names_map, old_state) + + +class FunctionalModule(nn.Module): + """ + This is the callable object returned by :func:`make_functional`. + """ + + def __init__( + self, + stateless_model: nn.Module, + param_names: tuple[str, ...], + names_map: dict[str, list[str]], + ) -> None: + super().__init__() + self.stateless_model = stateless_model + self.param_names = param_names + self.names_map = names_map + + @staticmethod + def _create_from( + model: nn.Module, disable_autograd_tracking: bool = False + ) -> tuple["FunctionalModule", tuple[Tensor, ...]]: + # TODO: We don't need to copy the model to create a stateless copy + model_copy = copy.deepcopy(model) + params, param_names, names_map = extract_weights(model_copy) + if disable_autograd_tracking: + for param in params: + param.requires_grad_(False) + return FunctionalModule(model_copy, param_names, names_map), params + + def forward(self, params: Iterable[Tensor], *args, **kwargs) -> Any: + # Temporarily load the state back onto self.stateless_model + old_state = _swap_state(self.stateless_model, self.names_map, params) + try: + return self.stateless_model(*args, **kwargs) + finally: + # Remove the loaded state on self.stateless_model + _swap_state(self.stateless_model, self.names_map, old_state) + + +def make_functional( + model: nn.Module, disable_autograd_tracking: bool = False +) -> tuple[FunctionalModule, tuple[Tensor, ...]]: + """make_functional(model, disable_autograd_tracking=False) -> func, params + + Given a ``torch.nn.Module``, :func:`make_functional` extracts the state + (params) and returns a functional version of the model, ``func``. This + makes it so that it is possible use transforms over the parameters of + ``model``. + + ``func`` can be invoked as follows: + + .. code-block:: python + + import torch + import torch.nn as nn + from functorch import make_functional + + x = torch.randn(4, 3) + model = nn.Linear(3, 3) + func, params = make_functional(model) + func(params, x) + + And here is an example of applying the grad transform over the parameters + of a model. + + .. code-block:: python + + import torch + import torch.nn as nn + from functorch import make_functional, grad + + x = torch.randn(4, 3) + t = torch.randn(4, 3) + model = nn.Linear(3, 3) + func, params = make_functional(model) + + + def compute_loss(params, x, t): + y = func(params, x) + return nn.functional.mse_loss(y, t) + + + grad_weights = grad(compute_loss)(params, x, t) + + If the model has any buffers, please use :func:`make_functional_with_buffers` instead. + + Args: + model (torch.nn.Module): Input model. + disable_autograd_tracking (bool): Flag to disable gradients tracking for output parameters. + The returned params are unrelated to the set of params from the original model. If False (default), + the params will have ``requires_grad=True`` on them (aka they will be trackable with regular + PyTorch autograd), matching the requires_grad-ness of the params from the original model. + Otherwise, the returned params will have ``requires_grad=False``. Default, False. + If you plan on using regular PyTorch autograd (e.g., if you want to call ``.backward()`` or + ``torch.autograd.grad()``, then set ``disable_autograd_tracking=False``. + Otherwise, if you're only planning on using functorch's gradient transforms, + then please set ``disable_autograd_tracking=True`` to avoid unnecessarily tracking + history with PyTorch autograd. + + """ + buffers = list(model.buffers()) + if len(buffers) > 0: + raise RuntimeError( + "make_functional(model): `model` has buffers. Please use " + "make_functional_with_buffers(model) instead." + ) + return FunctionalModule._create_from( + model, disable_autograd_tracking=disable_autograd_tracking + ) + + +def make_functional_with_buffers( + model: nn.Module, disable_autograd_tracking: bool = False +) -> tuple[FunctionalModuleWithBuffers, tuple[Tensor, ...], tuple[Tensor, ...]]: + """make_functional_with_buffers(model, disable_autograd_tracking=False) -> func, params, buffers + + Given a ``torch.nn.Module``, make_functional_with_buffers extracts the + state (params and buffers) and returns a functional version of the model + ``func`` that can be invoked like a function. + + ``func`` can be invoked as follows: + + .. code-block:: python + + import torch + import torch.nn as nn + from functorch import make_functional_with_buffers + + x = torch.randn(4, 3) + model = nn.Linear(3, 3) + func, params, buffers = make_functional_with_buffers(model) + func(params, buffers, x) + + And here is an example of applying the grad transform over the parameters + of a model: + + .. code-block:: python + + import torch + import torch.nn as nn + from functorch import make_functional_with_buffers, grad + + x = torch.randn(4, 3) + t = torch.randn(4, 3) + model = nn.Linear(3, 3) + func, params, buffers = make_functional_with_buffers(model) + + + def compute_loss(params, buffers, x, t): + y = func(params, buffers, x) + return nn.functional.mse_loss(y, t) + + + grad_weights = grad(compute_loss)(params, buffers, x, t) + + Args: + model (torch.nn.Module): Input model. + disable_autograd_tracking (bool): Flag to disable gradients tracking for output parameters. + The returned params are unrelated to the set of params from the original model. If False (default), + the params will have ``requires_grad=True`` on them (aka they will be trackable with regular + PyTorch autograd), matching the requires_grad-ness of the params from the original model. + Otherwise, the returned params will have ``requires_grad=False``. Default, False. + If you plan on using regular PyTorch autograd (e.g., if you want to call ``.backward()`` or + ``torch.autograd.grad()``, then set ``disable_autograd_tracking=False``. + Otherwise, if you're only planning on using functorch's gradient transforms, + then please set ``disable_autograd_tracking=True`` to avoid unnecessarily tracking + history with PyTorch autograd. + + """ + return FunctionalModuleWithBuffers._create_from( + model, disable_autograd_tracking=disable_autograd_tracking + ) + + +def transpose_stack( + tuple_of_tuple_of_tensors: tuple[tuple[Tensor, ...], ...], +) -> tuple[Tensor, ...]: + tuple_of_tuple_of_tensors = tuple(zip(*tuple_of_tuple_of_tensors)) + results = tuple( + torch.stack(shards).detach() for shards in tuple_of_tuple_of_tensors + ) + return results + + +def combine_state_for_ensemble( + models: Sequence[nn.Module], +) -> tuple[FunctionalModuleWithBuffers, tuple[Tensor, ...], tuple[Tensor, ...]]: + """combine_state_for_ensemble(models) -> func, params, buffers + + Prepares a list of torch.nn.Modules for ensembling with :func:`vmap`. + + Given a list of ``M`` ``nn.Modules`` of the same class, stacks all of their + parameters and buffers together to make ``params`` and ``buffers``. + Each parameter and buffer in the result will have an additional dimension + of size ``M``. + + :func:`combine_state_for_ensemble` also returns ``func``, a functional + version of one of the models in :attr:`models`. One cannot directly run + ``func(params, buffers, *args, **kwargs)`` directly, you probably want to + use ``vmap(func, ...)(params, buffers, *args, **kwargs)`` + + Here's an example of how to ensemble over a very simple model: + + .. code-block:: python + + num_models = 5 + batch_size = 64 + in_features, out_features = 3, 3 + models = [torch.nn.Linear(in_features, out_features) for i in range(num_models)] + data = torch.randn(batch_size, 3) + + fmodel, params, buffers = combine_state_for_ensemble(models) + output = vmap(fmodel, (0, 0, None))(params, buffers, data) + + assert output.shape == (num_models, batch_size, out_features) + + .. warning:: + All of the modules being stacked together must be the same (except for + the values of their parameters/buffers). For example, they should be in the + same mode (training vs eval). + + This API is subject to change -- we're investigating better ways to + create ensembles and would love your feedback how to improve this. + """ + if len(models) == 0: + raise RuntimeError( + "combine_state_for_ensemble: Expected at least one model, got 0." + ) + if not (all(m.training for m in models) or all(not m.training for m in models)): + raise RuntimeError( + "combine_state_for_ensemble: Expected all models to " + "have the same training/eval mode." + ) + model0_typ = type(models[0]) + if not all(type(m) is model0_typ for m in models): + raise RuntimeError( + "combine_state_for_ensemble: Expected all models to be of the same class." + ) + funcs, params, buffers = zip( + *[make_functional_with_buffers(model) for model in models] + ) + params = transpose_stack(params) + buffers = transpose_stack(buffers) + return funcs[0], params, buffers + + +def functional_init( + model_class: type[nn.Module], + ensemble_shape: Union[tuple[()], tuple[int, ...]] = (), + device: torch.types.Device = "cpu", +): + def wrapped(*args, **kwargs): + if len(ensemble_shape) >= 2: + raise ValueError("NYI: ensemble_shape with more than 1 element") + if len(ensemble_shape) == 0: + model = model_class(*args, **kwargs).to(device) + return make_functional_deprecated_v1(model) + num_models = ensemble_shape[0] # type: ignore[misc] + if num_models <= 0: + raise ValueError(f"num_models {num_models} should be > 0") + # NB: Not very efficient, more of a POC + models = tuple( + model_class(*args, **kwargs).to(device) for _ in range(num_models) + ) + _, fn, names = make_functional_deprecated_v1(model_class(*args, **kwargs)) + weights = tuple(make_functional_deprecated_v1(model)[0] for model in models) + weights = tuple(zip(*weights)) + weights = tuple(torch.stack(shards).detach() for shards in weights) + return weights, fn, names + + return wrapped + + +def functional_init_with_buffers( + model_class: type[nn.Module], + ensemble_shape: Union[tuple[()], tuple[int, ...]] = (), + device: torch.types.Device = "cpu", +): + def wrapped(*args, **kwargs): + if len(ensemble_shape) >= 2: + raise ValueError("NYI: ensemble_shape with more than 1 element") + if len(ensemble_shape) == 0: + model = model_class(*args, **kwargs).to(device) + return make_functional_deprecated_v1(model) + num_models = ensemble_shape[0] # type: ignore[misc] + if num_models <= 0: + raise ValueError(f"num_models {num_models} should be > 0") + # NB: Not very efficient, more of a POC + models = tuple( + model_class(*args, **kwargs).to(device) for _ in range(num_models) + ) + ( + _, + _, + fn, + weight_names, + buffer_names, + ) = make_functional_with_buffers_deprecated_v1(model_class(*args, **kwargs)) + weights, buffers = zip( + *tuple( + make_functional_with_buffers_deprecated_v1(model)[:2] + for model in models + ) + ) + weights = tuple(zip(*weights)) + weights = tuple(torch.stack(shards).detach() for shards in weights) + buffers = tuple(zip(*buffers)) + buffers = tuple(torch.stack(shards).detach() for shards in buffers) + return weights, buffers, fn, weight_names, buffer_names + + return wrapped diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/partitioners.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/partitioners.py new file mode 100644 index 0000000000000000000000000000000000000000..a3cd74644928fac7b36591468fde49173b242d11 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/partitioners.py @@ -0,0 +1,3130 @@ +# mypy: allow-untyped-defs +import copy +import functools +import hashlib +import heapq +import itertools +import logging +import math +import operator +import os +import os.path +import re +import warnings +from collections import defaultdict +from collections.abc import Callable +from dataclasses import dataclass, replace +from typing import Any, Optional, TYPE_CHECKING, Union + +import torch +import torch._inductor.inductor_prims +import torch.distributed +import torch.fx as fx +import torch.utils._pytree as pytree +from torch._dynamo.utils import counters, is_node_meta_valid +from torch._functorch._activation_checkpointing.ac_logging_utils import ( + create_structured_trace_for_min_cut_info, +) +from torch._inductor import config as inductor_config +from torch._library.utils import is_builtin +from torch._logging import trace_structured +from torch._subclasses.fake_tensor import extract_tensor_metadata +from torch.fx.experimental._backward_state import BackwardState +from torch.fx.experimental.proxy_tensor import is_sym_node, py_sym_types +from torch.fx.experimental.sym_node import magic_methods, method_to_operator +from torch.fx.experimental.symbolic_shapes import ( + find_symbol_binding_fx_nodes, + free_symbols, + hint_int, + is_symbol_binding_fx_node, + statically_known_false, + statically_known_true, +) +from torch.fx.passes import graph_drawer +from torch.utils._ordered_set import OrderedSet +from torch.utils.checkpoint import CheckpointPolicy + +from . import config +from ._activation_checkpointing.graph_info_provider import GraphInfoProvider +from ._activation_checkpointing.knapsack import ( + dp_knapsack, + dp_knapsack_sliding_hirschberg, + greedy_knapsack, + ilp_knapsack, +) +from ._activation_checkpointing.knapsack_evaluator import KnapsackEvaluator +from ._aot_autograd.descriptors import AOTOutput, SavedForBackwardsAOTOutput +from ._aot_autograd.functional_utils import _is_functional_graph +from ._aot_autograd.logging_utils import get_aot_graph_name +from ._aot_autograd.utils import get_cuda_generator_meta_val, is_with_effects +from .compile_utils import fx_graph_cse, get_aten_target, raise_getitems + + +if TYPE_CHECKING: + import sympy + + +AOT_PARTITIONER_DEBUG: bool = config.debug_partitioner +log: logging.Logger = logging.getLogger(__name__) + +aten = torch.ops.aten +prims = torch.ops.prims + + +@dataclass +class OpTypes: + """Class for keeping track of different operator categories""" + + fusible_ops: OrderedSet[Callable] + compute_intensive_ops: OrderedSet[Callable] + random_ops: OrderedSet[Callable] + view_ops: OrderedSet[Callable] + recomputable_ops: OrderedSet[Callable] + + def is_fusible(self, node: fx.Node): + return get_aten_target(node) in self.fusible_ops + + def is_compute_intensive(self, node: fx.Node): + return get_aten_target(node) in self.compute_intensive_ops + + def is_random(self, node: fx.Node): + return get_aten_target(node) in self.random_ops + + def is_view(self, node: fx.Node): + return get_aten_target(node) in self.view_ops + + def is_recomputable(self, node: fx.Node): + return get_aten_target(node) in self.recomputable_ops + + +@dataclass +class NodeInfo: + # Be careful about iterating over these explicitly, as their order may not + # be deterministic + inputs: list[fx.Node] + _required_fw_nodes: OrderedSet[fx.Node] + required_bw_nodes: OrderedSet[fx.Node] + unclaimed_nodes: OrderedSet[fx.Node] + fw_order: dict[fx.Node, int] + # Effectively maps to which of our primals are parameters + static_lifetime_input_nodes: OrderedSet[fx.Node] + + @functools.cached_property + def required_fw_nodes(self) -> list[fx.Node]: + return sorted( + (n for n in self._required_fw_nodes), key=lambda n: self.fw_order[n] + ) + + def is_required_fw(self, n: fx.Node) -> bool: + return n in self._required_fw_nodes + + def is_required_bw(self, n: fx.Node) -> bool: + return n in self.required_bw_nodes + + def is_unclaimed(self, n: fx.Node) -> bool: + return n in self.unclaimed_nodes + + def get_fw_order(self, n: fx.Node) -> int: + assert n in self._required_fw_nodes, f"Node {n} not in fw nodes!" + return self.fw_order[n] + + +@dataclass +class MinCutOptions: + ban_if_used_far_apart: bool + ban_if_long_fusible_chains: bool + ban_if_materialized_backward: bool + ban_if_not_in_allowlist: bool + ban_if_reduction: bool + + +def must_recompute(node: fx.Node) -> bool: + return node.meta.get("recompute", None) in [ + CheckpointPolicy.MUST_RECOMPUTE, + CheckpointPolicy.PREFER_RECOMPUTE, + ] + + +def has_recomputable_ops(fx_g: fx.GraphModule) -> bool: + for node in fx_g.graph.nodes: + if must_recompute(node): + return True + return False + + +def has_recomputable_rng_ops(fx_g: fx.GraphModule) -> bool: + for node in fx_g.graph.nodes: + if ( + must_recompute(node) + and hasattr(node.target, "tags") + and torch.Tag.nondeterministic_seeded in node.target.tags + ): + return True + return False + + +def sym_node_size(node: fx.Node) -> int: + if isinstance(node.meta["val"], (torch.SymInt, torch.SymBool)): + return 1 + assert isinstance(node.meta["val"], torch.SymFloat) + return 4 + + +class InvalidNodeBase: + def __repr__(self): + return "Invalid Node" + + +# Run DCE while overriding the definition of is_impure_node +def is_not_collective(node): + return getattr(node.target, "namespace", None) != "_c10d_functional" + + +InvalidNode = InvalidNodeBase() + + +def _extract_graph_with_inputs_outputs( + joint_graph: fx.Graph, + inputs: list[fx.Node], + outputs: list[fx.Node], + outputs_descs: list[AOTOutput], + subgraph: Optional[str] = None, + ignore_must_be_in_fw_bw: bool = False, +) -> fx.Graph: + """ + Given a graph, extracts out a subgraph that takes the specified nodes as + inputs and returns the specified outputs. + + This includes specifying non-placeholder nodes as inputs. + + The general strategy is to initialize all inputs with proxies as we + encounter them, and trace through the graph, only keeping values which take + in valid proxies. Then, all dead code is eliminated. + """ + new_graph = fx.Graph() + env = {} + + # Add new placeholder nodes in the order specified by the inputs + for node in inputs: + new_node = new_graph.placeholder(node.name) + # Can't use node_copy here as we may be turning previous call_function into placeholders + new_node.meta = node.meta + # pyrefly: ignore [unsupported-operation] + env[node] = new_node + + for node in joint_graph.nodes: + if not ignore_must_be_in_fw_bw: + if ( + _must_be_in_backward(node) + and subgraph != "backward" + and node not in inputs + ): + env[node] = InvalidNode # type: ignore[assignment] + continue + + if ( + _must_be_in_forward(node) + and subgraph != "forward" + and node not in inputs + ): + env[node] = InvalidNode # type: ignore[assignment] + continue + + if node in env: + # Node must be one of our inputs. (Any member of env which wasn't an + # input to start must have been created by this loop and won't be in + # joint_graph.nodes). + continue + elif node.op == "placeholder": + env[node] = InvalidNode # type: ignore[assignment] + elif node.op == "call_function": + all_args = pytree.arg_tree_leaves(*node.args, **node.kwargs) + all_args = [ + isinstance(env[x], InvalidNodeBase) + for x in all_args + if isinstance(x, fx.Node) + ] + if any(all_args): + env[node] = InvalidNode # type: ignore[assignment] + continue + # pyrefly: ignore [unsupported-operation, bad-argument-type] + env[node] = new_graph.node_copy(node, lambda x: env[x]) + elif node.op == "get_attr": + # pyrefly: ignore [unsupported-operation, bad-argument-type] + env[node] = new_graph.node_copy(node, lambda x: env[x]) + elif node.op == "output": + pass + output_values = [] + for x in outputs: + if isinstance(x, fx.Node): + if x not in env: + raise RuntimeError(f"Node {x} couldn't be found in env") + assert not isinstance(env[x], InvalidNodeBase), ( + f"Node {x} was invalid, but is output" + ) + output_values.append(env[x]) + else: + output_values.append(x) + out = new_graph.output(tuple(output_values)) + out.meta["desc"] = outputs_descs + + new_graph.eliminate_dead_code() + new_graph.lint() + return new_graph + + +def _is_primal(node: fx.Node) -> bool: + return ( + node.op == "placeholder" + and "tangents" not in str(node.target) + and not _is_bwd_seed_offset(node) + and not _is_fwd_seed_offset(node) + ) + + +def _is_tangent(node: fx.Node) -> bool: + return node.op == "placeholder" and "tangents" in str(node.target) + + +def is_non_builtin_to_include(node: fx.Node) -> bool: + return config.is_non_builtin_to_include and ( + (isinstance(node.target, torch._ops.OpOverload) and not is_builtin(node.target)) + or node.target == torch.ops.higher_order.triton_kernel_wrapper_functional + ) + + +def _is_bwd_seed_offset(node: fx.Node) -> bool: + return node.op == "placeholder" and ( + "bwd_seed" in str(node.target) or "bwd_base_offset" in str(node.target) + ) + + +def _is_fwd_seed_offset(node: fx.Node) -> bool: + return node.op == "placeholder" and ( + "fwd_seed" in str(node.target) or "fwd_base_offset" in str(node.target) + ) + + +def _is_backward_state(node: fx.Node) -> bool: + return node.op == "placeholder" and isinstance(node.meta.get("val"), BackwardState) + + +def _has_tag_is_backward(node: fx.Node) -> bool: + return node.meta.get("partitioner_tag", None) == "is_backward" + + +def _has_tag_is_forward(node: fx.Node) -> bool: + return node.meta.get("partitioner_tag", None) == "is_forward" + + +def _has_tag_must_be_in_forward(node: fx.Node) -> bool: + return node.meta.get("partitioner_tag", None) == "must_be_in_forward" + + +def _has_tag_must_be_in_backward(node: fx.Node) -> bool: + return node.meta.get("partitioner_tag", None) == "must_be_in_backward" + + +def _must_be_in_forward(node: fx.Node) -> bool: + if _has_tag_must_be_in_forward(node): + return True + is_mutable = is_with_effects(node) or ( + isinstance(node.target, torch._ops.OpOverload) + and node.target._schema.is_mutable + ) + return ( + not _has_tag_is_backward(node) + and not _has_tag_must_be_in_backward(node) + and is_mutable + ) + + +def _must_be_in_backward(node: fx.Node) -> bool: + if _has_tag_must_be_in_backward(node): + return True + is_mutable = is_with_effects(node) or ( + isinstance(node.target, torch._ops.OpOverload) + and node.target._schema.is_mutable + ) + return _has_tag_is_backward(node) and is_mutable + + +def _extract_fwd_bwd_outputs( + joint_module: fx.GraphModule, *, num_fwd_outputs +) -> tuple[list[fx.Node], list[fx.Node], list[AOTOutput], list[AOTOutput]]: + outputs = pytree.arg_tree_leaves( + *(node.args for node in joint_module.graph.find_nodes(op="output")) + ) + outputs_descs = pytree.arg_tree_leaves( + next(iter(joint_module.graph.find_nodes(op="output"))).meta.get( + "desc", [None] * len(outputs) + ) + ) + fwd_outputs = outputs[:num_fwd_outputs] + bwd_outputs = outputs[num_fwd_outputs:] + fwd_outputs_descs = outputs_descs[:num_fwd_outputs] + bwd_outputs_descs = outputs_descs[num_fwd_outputs:] + return fwd_outputs, bwd_outputs, fwd_outputs_descs, bwd_outputs_descs + + +def _remove_by_name(saved_values: list[fx.Node], name: str): + for saved_value in saved_values: + if saved_value.name == name: + saved_values.remove(saved_value) + break + + +def find_first_sym_node( + fwd_module_outputs: Union[list[fx.Node], tuple[fx.Node, ...]], +) -> int: + idx = len(fwd_module_outputs) + for i in range(len(fwd_module_outputs) - 1, -1, -1): + if not is_sym_node(fwd_module_outputs[i]): + idx = i + 1 + break + return idx + + +def calculate_quantization_scaling( + graph: torch.fx.Graph, + node: torch.fx.Node, + max: float = 57344.0, + min: float = 1e-12, + position: int = 0, +): + with graph.inserting_after(node): + abs_node = graph.call_function( + torch.ops.aten.abs.default, + args=(node,), + ) + abs_node.meta["val"] = torch.ops.aten.abs.default(node.meta["val"]) + abs_node.meta["tensor_meta"] = extract_tensor_metadata(abs_node.meta["val"]) + with graph.inserting_after(abs_node): + amax_node = graph.call_function( + torch.ops.aten.amax.default, + args=(abs_node, [-1], True), + ) + amax_node.meta["val"] = torch.ops.aten.amax.default( + abs_node.meta["val"], [-1], True + ) + amax_node.meta["tensor_meta"] = extract_tensor_metadata(amax_node.meta["val"]) + with graph.inserting_after(amax_node): + amax_64_node = graph.call_function( + torch.ops.prims.convert_element_type.default, + args=(amax_node, torch.float64), + ) + amax_64_node.meta["val"] = torch.ops.prims.convert_element_type.default( + amax_node.meta["val"], torch.float64 + ) + amax_64_node.meta["tensor_meta"] = extract_tensor_metadata( + amax_64_node.meta["val"] + ) + with graph.inserting_after(amax_64_node): + clamp_min_node = graph.call_function( + torch.ops.aten.clamp_min.default, + args=(amax_64_node, min), + ) + clamp_min_node.meta["val"] = torch.ops.aten.clamp_min.default( + amax_64_node.meta["val"], min + ) + clamp_min_node.meta["tensor_meta"] = extract_tensor_metadata( + clamp_min_node.meta["val"] + ) + with graph.inserting_after(clamp_min_node): + reciprocal_node = graph.call_function( + torch.ops.aten.reciprocal.default, + args=(clamp_min_node,), + ) + reciprocal_node.meta["val"] = torch.ops.aten.reciprocal.default( + clamp_min_node.meta["val"] + ) + reciprocal_node.meta["tensor_meta"] = extract_tensor_metadata( + reciprocal_node.meta["val"] + ) + with graph.inserting_after(reciprocal_node): + mul_node = graph.call_function( + torch.ops.aten.mul.Tensor, + args=(reciprocal_node, max), + ) + mul_node.meta["val"] = torch.ops.aten.mul.Tensor( + reciprocal_node.meta["val"], max + ) + mul_node.meta["tensor_meta"] = extract_tensor_metadata(mul_node.meta["val"]) + with graph.inserting_after(mul_node): + scale_node = graph.call_function( + torch.ops.prims.convert_element_type.default, + args=(mul_node, torch.float32), + name=f"fp8_scale_pos_{position}_{node.name}", + ) + scale_node.meta["val"] = torch.ops.prims.convert_element_type.default( + mul_node.meta["val"], torch.float32 + ) + scale_node.meta["tensor_meta"] = extract_tensor_metadata(scale_node.meta["val"]) + return scale_node + + +def perform_quantization( + graph: torch.fx.Graph, + node: torch.fx.Node, + scale_node: torch.fx.Node, + quant_type: torch.dtype, + clamp_min: float, + clamp_max: float, + position: int, +) -> torch.fx.Node: + with graph.inserting_after(scale_node): + target_node_32 = graph.call_function( + torch.ops.prims.convert_element_type.default, + args=(node, torch.float32), + ) + target_node_32.meta["val"] = torch.ops.prims.convert_element_type.default( + node.meta["val"], torch.float32 + ) + target_node_32.meta["tensor_meta"] = extract_tensor_metadata( + target_node_32.meta["val"] + ) + with graph.inserting_after(target_node_32): + scaled_target_node = graph.call_function( + torch.ops.aten.mul.Tensor, + args=(target_node_32, scale_node), + ) + scaled_target_node.meta["val"] = torch.ops.aten.mul.Tensor( + target_node_32.meta["val"], scale_node.meta["val"] + ) + scaled_target_node.meta["tensor_meta"] = extract_tensor_metadata( + scaled_target_node.meta["val"] + ) + with graph.inserting_after(scaled_target_node): + clamp_min_scaled_node = graph.call_function( + torch.ops.aten.clamp_min.default, + args=(scaled_target_node, clamp_min), + ) + clamp_min_scaled_node.meta["val"] = torch.ops.aten.clamp_min.default( + scaled_target_node.meta["val"], clamp_min + ) + clamp_min_scaled_node.meta["tensor_meta"] = extract_tensor_metadata( + clamp_min_scaled_node.meta["val"] + ) + with graph.inserting_after(clamp_min_scaled_node): + clamp_max_scaled_node = graph.call_function( + torch.ops.aten.clamp_max.default, + args=(clamp_min_scaled_node, clamp_max), + ) + clamp_max_scaled_node.meta["val"] = torch.ops.aten.clamp_max.default( + clamp_min_scaled_node.meta["val"], clamp_max + ) + clamp_max_scaled_node.meta["tensor_meta"] = extract_tensor_metadata( + clamp_max_scaled_node.meta["val"] + ) + with graph.inserting_after(clamp_max_scaled_node): + quant_activation_node = graph.call_function( + torch.ops.prims.convert_element_type.default, + args=(clamp_max_scaled_node, quant_type), + name=f"fp8_quant_pos_{position}_{node.name}", + ) + quant_activation_node.meta["val"] = ( + torch.ops.prims.convert_element_type.default( + clamp_max_scaled_node.meta["val"], quant_type + ) + ) + quant_activation_node.meta["tensor_meta"] = extract_tensor_metadata( + quant_activation_node.meta["val"] + ) + return quant_activation_node + + +def calculate_tensor_size(tensor: torch.Tensor) -> float: + """ + Calculate the size of a PyTorch tensor in megabytes (MB). + + Args: + tensor (torch.Tensor): Input tensor + + Returns: + float: Memory size in MB + """ + # Get number of elements and size per element + num_elements = tensor.numel() + element_size = tensor.element_size() + + return (num_elements * element_size) / (1024 * 1024) + + +def get_allowed_dtypes() -> list[torch.dtype]: + allowed_dtypes = torch._inductor.config.post_grad_fusion_options[ + "activation_quantization_aten_pass" + ].get("allowed_dtypes", "torch.bfloat16") + allowed_dtypes = [ + getattr(torch, dtype.split(".")[-1]) for dtype in allowed_dtypes.split(";") + ] + return allowed_dtypes + + +def should_quantize(node: torch.fx.Node) -> bool: + allowed_dtypes = get_allowed_dtypes() + if not is_node_meta_valid(node) or node.meta["val"].dtype not in allowed_dtypes: + return False + size_threshold = torch._inductor.config.post_grad_fusion_options[ + "activation_quantization_aten_pass" + ].get("size_in_mb", 100) + # calculate the size of the node + size_in_mb = calculate_tensor_size(node.meta["val"]) + if not torch._inductor.config.post_grad_fusion_options[ + "activation_quantization_aten_pass" + ].get("skip_dynamo_guards", False): + return size_in_mb >= size_threshold + else: + # case 1: we always quantize tensors with dynamic shapes + if torch._inductor.config.post_grad_fusion_options[ + "activation_quantization_aten_pass" + ].get("quantize_dynamic_shape", False): + return statically_known_true( + size_in_mb >= size_threshold + ) or not statically_known_false(size_in_mb >= size_threshold) + else: + # case 2: we always not quantize tensors with dynamic shapes + return statically_known_true(size_in_mb >= size_threshold) + + +def get_quant_type() -> torch.dtype: + quant_type = torch._inductor.config.post_grad_fusion_options[ + "activation_quantization_aten_pass" + ].get("quant_type", "torch.float8_e5m2") + + return getattr(torch, quant_type.split(".")[-1]) + + +def calculate_range(dtype: torch.dtype) -> tuple: + """ + Calculate the range of values for a given torch.dtype. + Args: + dtype (torch.dtype): The input dtype. + Returns: + tuple: A tuple containing the minimum and maximum values. + """ + info = torch.finfo(dtype) + return info.min, info.max + + +def quantize_activation_fw(graph: torch.fx.Graph) -> None: + output = graph.find_nodes(op="output")[0] + fwd_outputs = output.args[0] + quant_type = get_quant_type() + clamp_min, clamp_max = calculate_range(quant_type) + position_to_quant = dict() + tensor_scale_nodes, sym_scale_nodes = [], [] + for position, node in enumerate(fwd_outputs): + # check if the activation node is the node saved for quantization + if node.meta.get("saved_for_quantization", False): + # case: use scaling + if torch._inductor.config.post_grad_fusion_options[ + "activation_quantization_aten_pass" + ].get("use_scaling", True): + # calculating the scale + scale_node = calculate_quantization_scaling( + graph, node, clamp_max, 1e-12, position + ) + + # converting to fp8 + quant_node = perform_quantization( + graph, node, scale_node, quant_type, clamp_min, clamp_max, position + ) + if not is_sym_node(scale_node): + tensor_scale_nodes.append(scale_node) + else: + sym_scale_nodes.append(scale_node) + else: + # case: do not use scaling + with graph.inserting_after(node): + quant_node = graph.call_function( + torch.ops.prims.convert_element_type.default, + args=(node, quant_type), + name=f"fp8_quant_pos_{position}_{node.name}", + ) + quant_node.meta["val"] = ( + torch.ops.prims.convert_element_type.default( + node.meta["val"], quant_type + ) + ) + quant_node.meta["tensor_meta"] = extract_tensor_metadata( + quant_node.meta["val"] + ) + + position_to_quant[position] = quant_node + + # Use position-based lookup for building output + # only update the return node args, and remain all other users unchanged + output_updated_args = [ + position_to_quant.get(i, node) for i, node in enumerate(fwd_outputs) + ] + # add the scale nodes to the output find the first sym_node in the output + # pyrefly: ignore [bad-argument-type] + idx = find_first_sym_node(output_updated_args) + scale_nodes = tensor_scale_nodes + sym_scale_nodes + if scale_nodes: + output_updated_args = ( + output_updated_args[:idx] + scale_nodes + output_updated_args[idx:] + ) + + output.update_arg(0, tuple(output_updated_args)) + counters["inductor"]["activation_quantization_fwd_aten_pass"] += 1 + + +def quantize_activation_bw(graph: torch.fx.Graph) -> None: + bw_inputs = [node for node in graph.nodes if node.op == "placeholder"] + activation_node = None + for node in bw_inputs: + if node.meta.get("saved_for_quantization", False): + node.meta.pop("saved_for_quantization") + dequant_type = node.meta.pop("dequant_type") + # dequantize the node + if torch._inductor.config.post_grad_fusion_options[ + "activation_quantization_aten_pass" + ].get("use_scaling", False): + # case: use scaling + with graph.inserting_after(node): + # find corresponding scale node + scale_name = "fp8_scale_" + node.name.replace("fp8_quant_", "") + scale_node = next( + bwd_input + for bwd_input in bw_inputs + if bwd_input.name == scale_name + ) + with graph.inserting_after(scale_node): + activation_node = graph.call_function( + torch.ops.prims.convert_element_type.default, + args=(node, dequant_type), + ) + activation_node.meta["val"] = ( + torch.ops.prims.convert_element_type.default( + node.meta["val"], dequant_type + ) + ) + activation_node.meta["tensor_meta"] = extract_tensor_metadata( + activation_node.meta["val"] + ) + with graph.inserting_after(activation_node): + divided_target_node_32 = graph.call_function( + torch.ops.aten.div.Tensor, + args=(activation_node, scale_node), + ) + divided_target_node_32.meta["val"] = torch.ops.aten.div.Tensor( + activation_node.meta["val"], scale_node.meta["val"] + ) + divided_target_node_32.meta["tensor_meta"] = ( + extract_tensor_metadata(divided_target_node_32.meta["val"]) + ) + with graph.inserting_after(divided_target_node_32): + dequant_node = graph.call_function( + torch.ops.prims.convert_element_type.default, + args=(divided_target_node_32, dequant_type), + ) + dequant_node.meta["val"] = ( + torch.ops.prims.convert_element_type.default( + divided_target_node_32.meta["val"], dequant_type + ) + ) + dequant_node.meta["tensor_meta"] = extract_tensor_metadata( + dequant_node.meta["val"] + ) + else: + with graph.inserting_after(node): + dequant_node = graph.call_function( + torch.ops.prims.convert_element_type.default, + args=(node, dequant_type), + name="dequant_" + str(node.name), + ) + dequant_node.meta["val"] = ( + torch.ops.prims.convert_element_type.default( + node.meta["val"], dequant_type + ) + ) + dequant_node.meta["tensor_meta"] = extract_tensor_metadata( + dequant_node.meta["val"] + ) + # find the users of the node and replace them with the new node except the dequant_node + for user in list(node.users.keys()): + if user != dequant_node and user != activation_node: + user.replace_input_with(node, dequant_node) + + counters["inductor"]["activation_quantization_bwd_aten_pass"] += 1 + + +def perform_fp8_activation_quantization( + fwd_module: fx.GraphModule, + bwd_module: fx.GraphModule, + bwd_module_inputs: dict[str, fx.Node], +) -> None: + trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "before_activation_quantization_fwd_aten_pass", + "encoding": "string", + }, + payload_fn=lambda: fwd_module.print_readable( + print_output=False, include_stride=True, include_device=True + ), + ) + + quantize_activation_fw(fwd_module.graph) + + trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "after_activation_quantization_fwd_aten_pass", + "encoding": "string", + }, + payload_fn=lambda: fwd_module.print_readable( + print_output=False, include_stride=True, include_device=True + ), + ) + + trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "before_activation_quantization_bwd_aten_pass", + "encoding": "string", + }, + payload_fn=lambda: bwd_module.print_readable( + print_output=False, include_stride=True, include_device=True + ), + ) + + quant_fwd_module_outputs = fwd_module.graph.find_nodes(op="output")[0].args[0] + # update the corresponding bwd_inputs due to the fwd_outputs quantization + for fwd_node in quant_fwd_module_outputs: + if "fp8_quant_" in fwd_node.name: + bwd_input = bwd_module_inputs[ + re.sub(r"^fp8_quant_pos_\d+_", "", fwd_node.name) + ] + with bwd_module.graph.inserting_after(bwd_input): + quant_bwd_input = bwd_module.graph.placeholder(name=fwd_node.name) + dequant_type = bwd_input.meta["dequant_type"] + quant_bwd_input.meta.update(fwd_node.meta) + quant_bwd_input.meta["saved_for_quantization"] = True + quant_bwd_input.meta["dequant_type"] = dequant_type + bwd_input.replace_all_uses_with(quant_bwd_input) + bwd_module.graph.erase_node(bwd_input) + # update the bwd_inputs if quantization with scaling is used + if torch._inductor.config.post_grad_fusion_options[ + "activation_quantization_aten_pass" + ].get("use_scaling", True): + quant_bwd_module_inputs = list(bwd_module.graph.find_nodes(op="placeholder")) + # update the corresponding bwd input nodes find the last non-tangent node + bwd_input_loc = quant_bwd_module_inputs[-1] + for bw_input in reversed(quant_bwd_module_inputs): + if not _is_tangent(bw_input): + bwd_input_loc = bw_input + break + + scaled_fwd_module_outputs = fwd_module.graph.find_nodes(op="output")[0].args[0] + for fwd_node in scaled_fwd_module_outputs: + if "fp8_scale_" in fwd_node.name: + # fwd node is a scale node + with bwd_module.graph.inserting_after(bwd_input_loc): + scale_bwd_input = bwd_module.graph.placeholder(name=fwd_node.name) + scale_bwd_input.meta.update(fwd_node.meta) + bwd_input_loc = scale_bwd_input + + quantize_activation_bw(bwd_module.graph) + + trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "after_activation_quantization_bwd_aten_pass", + "encoding": "string", + }, + payload_fn=lambda: bwd_module.print_readable( + print_output=False, include_stride=True, include_device=True + ), + ) + + +def enable_activation_quantization( + saved_values: list[fx.Node], + fwd_module: fx.GraphModule, + bwd_module: fx.GraphModule, + static_lifetime_input_nodes: Optional[OrderedSet[fx.Node]] = None, +) -> None: + if ( + inductor_config.post_grad_fusion_options.get( + "activation_quantization_aten_pass", None + ) + is None + ): + return + + static_input_names = ( + [node.name for node in static_lifetime_input_nodes] + if static_lifetime_input_nodes + else [] + ) + saved_values_names = {node.name: node for node in saved_values} + if torch._inductor.config.post_grad_fusion_options[ + "activation_quantization_aten_pass" + ].get("exclude_primals", False): + saved_values_names = { + node.name: node for node in saved_values if "primals" not in node.name + } + fwd_module_outputs = fwd_module.graph.find_nodes(op="output")[0].args[0] + bwd_module_inputs = { + node.name: node for node in bwd_module.graph.find_nodes(op="placeholder") + } + should_perform_fp8_quant = False + for node in fwd_module_outputs: + if node.name in saved_values_names and should_quantize(node): + if node.name in static_input_names: + log.debug("Skipping quantization of static input %s: ", node.name) + continue + node.meta["saved_for_quantization"] = True + node.meta["dequant_type"] = node.meta["val"].dtype + # some of the fwd outputs and bwd inputs are not share the same object + bwd_module_inputs[node.name].meta["saved_for_quantization"] = True + bwd_module_inputs[node.name].meta["dequant_type"] = node.meta["val"].dtype + should_perform_fp8_quant = True + + if should_perform_fp8_quant: + perform_fp8_activation_quantization(fwd_module, bwd_module, bwd_module_inputs) + + +def _extract_fwd_bwd_modules( + joint_module: fx.GraphModule, + saved_values: list[fx.Node], + saved_sym_nodes: list[fx.Node], + *, + num_fwd_outputs: int, + static_lifetime_input_nodes: Optional[OrderedSet[fx.Node]] = None, +) -> tuple[fx.GraphModule, fx.GraphModule]: + fwd_outputs, bwd_outputs, fwd_outputs_descs, bwd_outputs_descs = ( + _extract_fwd_bwd_outputs(joint_module, num_fwd_outputs=num_fwd_outputs) + ) + placeholders = joint_module.graph.find_nodes(op="placeholder") + primal_inputs = [*filter(_is_primal, placeholders)] + tangent_inputs = [*filter(_is_tangent, placeholders)] + fwd_seed_offset_inputs = [*filter(_is_fwd_seed_offset, placeholders)] + bwd_seed_offset_inputs = [*filter(_is_bwd_seed_offset, placeholders)] + backward_state_inputs = [*filter(_is_backward_state, placeholders)] + + bwd_graph = _extract_graph_with_inputs_outputs( + joint_module.graph, + saved_sym_nodes + saved_values + tangent_inputs + bwd_seed_offset_inputs, + bwd_outputs, + bwd_outputs_descs, + "backward", + ) + + distributed_enabled = torch.distributed.is_available() + + for node in bwd_graph.find_nodes(op="placeholder"): + # This is to filter out saved values that don't actually end up being used by the backwards pass + if not node.users: + _remove_by_name(saved_values, node.name) + _remove_by_name(saved_sym_nodes, node.name) + # wait_tensor is a bit special: if we have a "dead activation" that is not used in the bw, + # but this dead activation is actually a collective, + # then the collective will generally by followed by a wait_tensor() call. + # we need to peak one node further to see if this wait_tensor is dead as well. + elif distributed_enabled and all( + n.target is torch.ops._c10d_functional.wait_tensor.default + and len(n.users) == 0 + for n in node.users + ): + _remove_by_name(saved_values, node.name) + _remove_by_name(saved_sym_nodes, node.name) + elif _is_backward_state(node): + # BackwardState is saved directly + _remove_by_name(saved_values, node.name) + assert backward_state_inputs + + # Now that we have the finalized list of saved values, we need to ensure + # we propagate all symbols which are referenced by backwards inputs. + # These are not directly used in the graph but are required for downstream + # sizevar assignment + saved_symbols: OrderedSet[sympy.Symbol] = OrderedSet() + saved_sym_nodes_binding = [] + saved_sym_nodes_derived = [] + + # Some symbols may already be bound in the directly saved_sym_nodes, + # keep track of them so we don't re-bind them + for node in saved_sym_nodes: + symbol = is_symbol_binding_fx_node(node) + if symbol: + saved_symbols.add(symbol) + saved_sym_nodes_binding.append(node) + else: + saved_sym_nodes_derived.append(node) + + # Now go through all of the prospective backward inputs and track any + # other symbols we need to bind + symbol_bindings = find_symbol_binding_fx_nodes(joint_module.graph) + for node in itertools.chain(saved_sym_nodes_derived, saved_values, tangent_inputs): + if "val" not in node.meta: + continue + new_symbols = free_symbols(node.meta["val"]) - saved_symbols + # NB: Deterministic order please! + for s in sorted(new_symbols, key=lambda s: s.name): + # NB: For well formed graphs, the symbol should always be present, + # but we also have ways to produce ill-formed graphs, e.g., direct + # make_fx usages, so don't choke in this case + if s not in symbol_bindings: + continue + saved_sym_nodes_binding.append(symbol_bindings[s]) + saved_symbols |= new_symbols + + # Update saved_sym_nodes that are now reordered to have all bindings at + # front. This can also be used later on to figure out the position of saved + # sym nodes in the output of fwd graph. + saved_sym_nodes.clear() + saved_sym_nodes.extend(saved_sym_nodes_binding + saved_sym_nodes_derived) + + # Now, we re-generate the fwd/bwd graphs. + # NB: This might increase compilation time, but I doubt it matters + fwd_graph = _extract_graph_with_inputs_outputs( + joint_module.graph, + primal_inputs + fwd_seed_offset_inputs, + fwd_outputs + saved_values + saved_sym_nodes, + fwd_outputs_descs + + [ + SavedForBackwardsAOTOutput(i) + for i in range(len(saved_values) + len(saved_sym_nodes)) + ], + "forward", + ) + bwd_graph = _extract_graph_with_inputs_outputs( + joint_module.graph, + saved_sym_nodes + + saved_values + + tangent_inputs + + bwd_seed_offset_inputs + + backward_state_inputs, + bwd_outputs, + bwd_outputs_descs, + "backward", + ) + + fwd_module = fx._lazy_graph_module._make_graph_module(joint_module, fwd_graph) + bwd_module = fx._lazy_graph_module._make_graph_module(joint_module, bwd_graph) + enable_activation_quantization( + saved_values, fwd_module, bwd_module, static_lifetime_input_nodes + ) + return fwd_module, bwd_module + + +def default_partition( + joint_module: fx.GraphModule, + _joint_inputs, + *, + num_fwd_outputs, + static_lifetime_input_indices: Optional[list[int]] = None, + static_lifetime_input_nodes: Optional[OrderedSet[fx.Node]] = None, +) -> tuple[fx.GraphModule, fx.GraphModule]: + """ + Partitions the :attr:`joint_module` in a manner that closely resembles the + behavior observed in the original ``.forward()`` and ``.backward()`` of the + callable, i.e., the resulting forward graph contains those operators that + are executed in the original ``.forward()`` callable passed to + :func:`aot_function`. + + The default partitioner collects the operators that are between the forward + inputs and the forward outputs. This helps in finding the tensors which have + to be stashed for the backward pass. These stashed tensors become the output + of the generated forward graph. The remaining operators are then placed in + the backward graph. + + .. warning:: + This API is experimental and likely to change. + + Args: + joint_module(fx.GraphModule): The joint forward and backward graph. This + is the result of AOT Autograd tracing. + + Returns: + Returns the generated forward and backward Fx graph modules. + """ + # Respect the original placement of ops rather than rely on dataflow. + forward_nodes = [] + last_node = None + for node in joint_module.graph.nodes: + if _has_tag_is_forward(node) or _is_primal(node) or _is_fwd_seed_offset(node): + last_node = node + assert last_node is not None + for node in joint_module.graph.nodes: + if not _is_tangent(node): + forward_nodes.append(node) + if node is last_node: + break + forward_node_names = OrderedSet( + node.name for node in forward_nodes if node.op != "output" + ) + graph_has_recomputable_ops = has_recomputable_ops(joint_module) + graph_has_recomputable_rng_ops = has_recomputable_rng_ops(joint_module) + if graph_has_recomputable_ops: + if _is_functional_graph(joint_module.graph)[0] is not None: + # Fall-back to previous behavior to avoid bc-breaking, although can + # eventually flip the switch to make this a hard error. + warnings.warn( + "Trying to unsafely apply AC to a non-functional graph with the " + "default partitioner. Falling back to min-cut partitioner." + ) + return min_cut_rematerialization_partition( + joint_module, + _joint_inputs, + num_fwd_outputs=num_fwd_outputs, + static_lifetime_input_indices=static_lifetime_input_indices, + ) + + joint_module = cleanup_recompute_tags(joint_module, is_default_partition=True) + + if not config.unsafe_allow_optimization_of_collectives: + force_save_collectives(joint_module) + + force_save_bw_mutation_src(joint_module) + + if static_lifetime_input_indices is None: + static_lifetime_input_indices = [] + node_info = classify_nodes( + joint_module, static_lifetime_input_indices, num_fwd_outputs + ) + + saved_values = [] + saved_sym_nodes = [] + + distributed_enabled = torch.distributed.is_available() + + def is_tensor(node): + return "tensor_meta" in node.meta or isinstance( + node.meta.get("val"), torch._subclasses.FakeTensor + ) + + def is_multi_output(node): + return ( + all(user.target == operator.getitem for user in node.users) + and len(node.users) > 0 + ) + + def is_impure(node): + # wait tensor is an "impure" op according to DCE's definition of impure + # (see is_impure in torch/fx/node.py), but it survives past + # functionalization and can be safely dup'd and reordered under the + # assumption SPMD. + return ( + node.is_impure(impure_random=False) + and node.op + not in ( + "placeholder", + "output", + ) + and ( + not distributed_enabled + or node.target is not torch.ops._c10d_functional.wait_tensor.default + ) + ) + + for node in joint_module.graph.nodes: + if node.name not in forward_node_names: + continue + if node.op == "get_attr" and node.name in ( + k for k, v in joint_module.named_modules() + ): + continue + if node.target is torch.ops.aten._assert_scalar.default: + continue + if is_sym_node(node): + # Symints must be kept separate from tensors so that PythonFunction only calls + # save_for_backward on tensors and stashes symints in autograd .ctx + saved_sym_nodes.append(node) + continue + if is_multi_output(node): + # Must be ordered before MUST_SAVE tags to avoid saving tuples marked MUST_SAVE. + continue + if node.meta.get("recompute") == CheckpointPolicy.MUST_SAVE: + saved_values.append(node) + continue + if is_impure(node): + assert not graph_has_recomputable_ops, ( + "Trying to apply AC on a graph with impure op", + node, + node.target, + ) + saved_values.append(node) + continue + assert is_tensor(node) or node.op != "call_function", ( + f"Expected {node} to be a tensor" + ) + backward_usages = [n for n in node.users if n.name not in forward_node_names] + if all(is_sym_node(n) for n in backward_usages): + # If we have a tensor in the forward, where only its sizes/strides are needed in the backward, + # and not the actual tensor data, + # then it will be a lot cheaper to save only the sizes/strides, and not the actual tensor. + # + # Note that saving the tensor could also cause compilation problems: + # If the user mutated an input in the forward and uses its sizes/strides in the backward, + # then we would be obligated to clone the input before saving it to appease autograd. + # (This is how we originally found this bug). + saved_sym_nodes.extend(backward_usages) + continue + if not must_recompute(node): + saved_values.append(node) + + saved_values = list(dict.fromkeys(saved_values).keys()) + saved_sym_nodes = list(dict.fromkeys(saved_sym_nodes).keys()) + + if config._sync_decision_cross_ranks: + saved_values = _sync_decision_cross_ranks(joint_module.graph, saved_values) + + if static_lifetime_input_nodes is None: + static_lifetime_input_nodes = node_info.static_lifetime_input_nodes + fw_module, bw_module = _extract_fwd_bwd_modules( + joint_module, + saved_values, + saved_sym_nodes=saved_sym_nodes, + num_fwd_outputs=num_fwd_outputs, + static_lifetime_input_nodes=static_lifetime_input_nodes, + ) + + # Run DCE while overriding the definition of is_impure_node + fw_module.graph.eliminate_dead_code(is_impure_node=is_not_collective) + bw_module.graph.eliminate_dead_code(is_impure_node=is_not_collective) + + if graph_has_recomputable_ops: + if graph_has_recomputable_rng_ops: + fw_module, bw_module = functionalize_rng_ops( + joint_module, fw_module, bw_module, len(saved_sym_nodes) + ) + bw_module = reordering_to_mimic_autograd_engine(bw_module) + + # raise all getitem ops to as early as possible + # this is helpful for memory, especially in the case of aot_eager backend + fw_module = raise_getitems(fw_module) + bw_module = raise_getitems(bw_module) + + fw_module = thread_graphsafe_rng_from_hops(fw_module, is_backward=False) + if len(node_info.required_bw_nodes) > 0: + bw_module = thread_graphsafe_rng_from_hops(bw_module, is_backward=True) + + return fw_module, bw_module + + +INT_INF = int(1e6) + + +def _tensor_nbytes(numel: int, dtype) -> int: + return numel * dtype.itemsize + + +def _size_of(node: fx.Node) -> int: + def object_nbytes(x) -> int: + if not isinstance(x, torch.Tensor): + return 0 + return _tensor_nbytes(hint_int(x.numel(), fallback=4096), x.dtype) + + if "val" in node.meta: + val = node.meta["val"] + if isinstance(val, py_sym_types): + return 1 + # NB: The fallback values here are meaningless, maybe we should respect + # torch._inductor.config.unbacked_symint_fallback (but this is a + # layering violation) + elif isinstance(val, (list, tuple)): + return sum(object_nbytes(n) for n in val) + elif isinstance(val, dict): + return sum(object_nbytes(n) for _, n in val.items()) + elif isinstance(val, torch.Tensor): + return object_nbytes(val) + + raise RuntimeError(f"Unknown metadata type {type(val)} on node {node}") + if node.op == "get_attr" or node.target is torch.ops.aten._assert_scalar.default: + return 0 + raise RuntimeError( + f"Node {node} didn't have `val` metadata; we should always have `val` metadata on the nodes." + ) + + +# Used for some investigative purposes +def _count_ops(graph: fx.Graph): + from collections import defaultdict + + cnt: dict[str, int] = defaultdict(int) + for node in graph.nodes: + if node.op == "call_function": + cnt[node.target.__name__] += 1 + log.info("%s", sorted(cnt.items(), key=operator.itemgetter(1), reverse=True)) + + +@functools.cache +def pointwise_ops(): + ops = [] + for attr_name in dir(torch.ops.aten): + opoverloadpacket = getattr(torch.ops.aten, attr_name) + if not isinstance(opoverloadpacket, torch._ops.OpOverloadPacket): + continue + + for overload in opoverloadpacket.overloads(): + op_overload = getattr(opoverloadpacket, overload) + if torch.Tag.pointwise in op_overload.tags: + # currently aot autograd uses packet not overload + ops.append(opoverloadpacket) + break + + return ops + + +def sort_depths(args, depth_map: dict[fx.Node, int]) -> list[tuple[fx.Node, int]]: + arg_depths = { + arg: depth_map[arg] for arg in args if isinstance(arg, torch.fx.node.Node) + } + return sorted(arg_depths.items(), key=operator.itemgetter(1), reverse=True) + + +def reordering_to_mimic_autograd_engine(gm: fx.GraphModule) -> fx.GraphModule: + """ + This pass finds the first bwd node in the graph (by looking at users of + tangents) and then reorders the graph by walking from this node to all the + way to the end of the graph. At each op in this traversal, we insert this op + in a new graph and try to bring only the relevant subgraph from the other + non-bwd edges relevant for this op. This closely mimics the behavior of + autograd engine. + + Why is this pass required in the first place? + + This is an artifact of how partitioners work today. The starting point of + partitioner is a joint graph, which is fwd and then bwd graph. In the case + of checkpointing, we keep portions of fwd graph in their original place in + the joint graph, while obtaining a bwd graph. As a result, the resulting bwd + graph has copies of recomputed fwd subgraphs followed by the original bwd + graph. If we run this naively, this leads to bad memory footprint, because + the fwd subgraphs are live for way longer duration than necessary. This pass + reorders the operations such that we prioritize the ops for the original bwd + graph while only realizing those ops from the fwd graph that are necessary + at any given point in the graph. + """ + + new_graph = fx.Graph() + env: dict[fx.Node, fx.Node] = {} + + # Add new placeholder nodes in the order specified by the inputs + for node in gm.graph.find_nodes(op="placeholder"): + env[node] = new_graph.node_copy(node, lambda x: env[x]) + + order = {node: idx for idx, node in enumerate(gm.graph.nodes)} + + def insert_node_in_graph(node): + cur_nodes = [node] + insertable_nodes: OrderedSet[fx.Node] = OrderedSet() + while len(cur_nodes) > 0: + node = cur_nodes.pop() + if node in insertable_nodes or node in env: + continue + insertable_nodes.add(node) + + # Bias traversal towards the nodes that have higher depth - prioritizes + # critical path first. + cur_nodes += node.all_input_nodes + + # pyrefly: ignore [bad-assignment] + insertable_nodes = sorted(insertable_nodes, key=lambda n: order[n]) + for node in insertable_nodes: + env[node] = new_graph.node_copy(node, lambda x: env[x]) + + # Find first bwd node in the graph + tangent_inputs = list(filter(_is_tangent, gm.graph.nodes)) + first_node_in_bwd = None + minimum_order = math.inf + for tangent in tangent_inputs: + for user in tangent.users: + if order[user] < minimum_order: + minimum_order = order[user] + first_node_in_bwd = user + + # If gradInp does not depend upon gradOut, we may not find any nodes in the "backwards pass" + if first_node_in_bwd is None: + return gm + + # Build the graph op-by-op by starting from the node all the way to the end + # copy_ can be not using tangents at all, we must copy it. + for node in list(gm.graph.nodes)[: order[first_node_in_bwd]]: + if node.op == "call_function" and node.target is torch.ops.aten.copy_.default: + insert_node_in_graph(node) + + for node in list(gm.graph.nodes)[order[first_node_in_bwd] :]: + insert_node_in_graph(node) + + # The output node is already built by the traversal. + new_gm = torch.fx.GraphModule(gm, new_graph) + return new_gm + + +def apply_graphsafe_rng_functionalization( + fw_module: torch.fx.GraphModule, + bw_module: torch.fx.GraphModule, + fw_node: torch.fx.Node, + bw_node: torch.fx.Node, + device: torch.device, + rng_count: int, + last_fwd_input: torch.fx.Node, + last_bwd_input: torch.fx.Node, +): + """ + Note [CUDA Graph Safe RNG Functionalization] + + CUDA Graph capture doesn't work with get_rng_state and set_rng_state because these functions operate on CPU values, + while CUDA Graph RNG capture uses on-device CUDA tensors. To solve this, we use graphsafe_set_state with a + CUDA Generator registered to the CUDA Graph before capture begins. graphsafe_set_state updates the generator's pointer + to reference a different GeneratorImpl, ensuring subsequent calls are correctly forwarded to the desired generator + (and its cuda-tensor RNG state during graph capture). + + For each RNG operation's forward/backward pair: + + - We create two generators initialized with identical values + - Each forward and backward call advances its respective generator equally + - This keeps generators synchronized so forward and backward operations use matching RNG values + + When forward is called multiple times before backward (causing desynchronization): + + - We save the forward RNG state + - We update the backward Generator's state before executing backward + + Before each CUDA Graph replay, replay_prologue updates captured RNG pointers with current states, ensuring backward Generator + changes are reflected during replay. + + This function modifies both forward and backward computation graphs by: + + Creating RNG state placeholders for both passes + Updating the forward node to use graph-safe RNG state + Updating the backward node to use graph-safe RNG state + + For more details: https://github.com/pytorch/pytorch/issues/113541 + """ + device_idx = device.index + assert device_idx is not None + fw_graph = fw_module.graph + bw_graph = bw_module.graph + graphsafe_run_with_rng_state = torch._prims.rng_prims.graphsafe_run_with_rng_state + + # Handle forward pass + + # Note: [Generator arguments in AOTDispatcher] + # Generator arguments in AOTDispatcher are added to support graphsafe rng + # functionalization. See note above [CUDA Graph Safe RNG Functionalization] + with fw_module.graph.inserting_after(last_fwd_input): + fwd_rng_state = fw_module.graph.placeholder(f"fwd_rng_state_{rng_count}") + fwd_rng_state.meta["val"] = get_cuda_generator_meta_val(device_idx) + last_fwd_input = fwd_rng_state + + # Handle backward pass + with bw_module.graph.inserting_after(last_bwd_input): + bwd_rng_state = bw_module.graph.placeholder(f"bwd_rng_state_{rng_count}") + # as above, clone so that meta val generator will not contain tensors + bwd_rng_state.meta["val"] = get_cuda_generator_meta_val(device_idx) + last_bwd_input = bwd_rng_state + + # Update forward node + fw_kwargs = dict(fw_node.kwargs) + fw_kwargs["rng_state"] = fwd_rng_state + with fw_module.graph.inserting_after(fw_node): + functional_fw_node = fw_graph.create_node( + "call_function", + graphsafe_run_with_rng_state, + args=(fw_node.target, *fw_node.args), # type: ignore[arg-type] + kwargs=fw_kwargs, + ) + fw_node.replace_all_uses_with(functional_fw_node) + fw_graph.erase_node(fw_node) + + # Update backward node + bwd_kwargs = dict(bw_node.kwargs) + bwd_kwargs["rng_state"] = bwd_rng_state + with bw_graph.inserting_before(bw_node): + rng_output = bw_graph.create_node( + "call_function", + graphsafe_run_with_rng_state, + args=(bw_node.target, *bw_node.args), # type: ignore[arg-type] + kwargs=bwd_kwargs, + ) + bw_node.replace_all_uses_with(rng_output) + bw_graph.erase_node(bw_node) + + return last_fwd_input, last_bwd_input + + +def functionalize_rng_ops( + joint_module: fx.GraphModule, + fw_module: fx.GraphModule, + bw_module: fx.GraphModule, + num_sym_nodes: int, +) -> tuple[fx.GraphModule, fx.GraphModule]: + # During user-driven activation checkpointing, we have to ensure that a rng + # op in fwd yields the same output as the recomputed rng op in the bwd. To + # do this, we use functionalize wrappers to wrap the random ops and share + # rng state between the fwd and bwd graphs. + + # There are 3 main steps to do this + # Step 1 - Construct a mapping of rng node between the fwd and its counterpart in bwd. + # Step 2 - Modify the fwd pass such that + # 1) Replace rand with run_and_save_rng_state wrapper + # 2) Replace the users of the original op with the output[1] of this op. + # 3) Collect all the rng_state - output[0] of each op, and make them + # output nodes. Special care needs to be taken here because fwd outputs + # has symints at the very end. + # Step 3 - Modify the bwd pass such that + # 1) Add the input nodes just before the tangents for the stashed rng states + # 2) Replace rand with run_with_save_rng_state wrappers + # 3) Use the stashed states as inputs to these ops + + # Unique id to generate name + uid = itertools.count() + + def get_rng_ops(gmod): + random_nodes = {} + for node in gmod.graph.nodes: + if ( + node.op == "call_function" + and hasattr(node.target, "tags") + and torch.Tag.nondeterministic_seeded in node.target.tags + ): + random_nodes[node.name] = node + return random_nodes + + def get_device(node) -> Optional[torch.device]: + """ + Check the example value of the node outputs to find the device type. + """ + if "val" not in node.meta: + return None + + candidates = node.meta["val"] + if not isinstance(candidates, tuple): + candidates = (candidates,) + + for candidate in candidates: + if isinstance(candidate, torch.Tensor): + if candidate.device.type == "cuda": + return candidate.device + + return torch.device("cpu") + + def get_sample_rng_state(device: Optional[torch.device]): + from torch._guards import detect_fake_mode # noqa: F401 + + fake_mode = detect_fake_mode() + assert fake_mode is not None + with fake_mode: + if device is not None and device.type == "cuda": + return fake_mode.from_tensor(torch.cuda.get_rng_state()) + return fake_mode.from_tensor(torch.get_rng_state()) + + # Step 1 - Construct a mapping of rng node between the fwd and its counterpart in bwd. + joint_graph_rng_ops = get_rng_ops(joint_module) + fw_graph_rng_ops = get_rng_ops(fw_module) + bw_graph_rng_ops = get_rng_ops(bw_module) + recomputable_rng_ops_map = {} + for node in joint_module.graph.nodes: + if ( + must_recompute(node) + and hasattr(node.target, "tags") + and torch.Tag.nondeterministic_seeded in node.target.tags + ): + base_node = joint_graph_rng_ops[node.name] + fw_node = fw_graph_rng_ops[node.name] + bw_node = bw_graph_rng_ops[node.name] + recomputable_rng_ops_map[base_node] = {"fwd": fw_node, "bwd": bw_node} + + run_and_save_rng = torch._prims.rng_prims.run_and_save_rng_state + run_with_rng_state = torch._prims.rng_prims.run_with_rng_state + + bw_tangent_start_node = None + for node in bw_module.graph.find_nodes(op="placeholder"): + if "tangent" in node.name: + bw_tangent_start_node = node + break + if bw_tangent_start_node is None: + raise RuntimeError( + "Couldn't find tangent node in graph inputs. This is unexpected, please file a bug if you see this" + ) + + fw_rng_state_outputs = [] + + last_fwd_input = next(reversed(fw_module.graph.find_nodes(op="placeholder"))) + last_bwd_input = next(reversed(bw_module.graph.find_nodes(op="placeholder"))) + + devices = OrderedSet( + get_device(node_pair["fwd"]) for node_pair in recomputable_rng_ops_map.values() + ) + # pyrefly: ignore [unbound-name] + devices.discard(torch.device("cpu")) + # multiple cuda devices won't work with cudagraphs anyway, + # fallback to non graphsafe rng checkpointing + multi_cuda_devices = len(devices) > 1 + + # this changes numerics, so if fallback_random is set we will not use it + # pyrefly: ignore [unbound-name] + ind_config = torch._inductor.config + use_rng_graphsafe_rng_functionalization = ( + config.graphsafe_rng_functionalization + and not multi_cuda_devices + and ( + not ind_config.fallback_random + or ind_config.test_configs.graphsafe_rng_func_ignores_fallback_random + ) + ) + + for rng_count, node_pair in enumerate(recomputable_rng_ops_map.values()): + # Step 2 - Modify the fwd pass such that + fw_node = node_pair["fwd"] + bw_node = node_pair["bwd"] + device = get_device(fw_node) + + fw_graph = fw_module.graph + bw_graph = bw_module.graph + + if ( + use_rng_graphsafe_rng_functionalization + and device is not None + and device.type == "cuda" + ): + last_fwd_input, last_bwd_input = apply_graphsafe_rng_functionalization( + fw_module, + bw_module, + fw_node, + bw_node, + device, + rng_count, + last_fwd_input, + last_bwd_input, + ) + else: + with fw_graph.inserting_before(fw_node): + functional_fw_node = fw_graph.create_node( + "call_function", + run_and_save_rng, + args=(fw_node.target, *fw_node.args), + kwargs=fw_node.kwargs, + ) + state = fw_graph.create_node( + "call_function", + operator.getitem, + args=(functional_fw_node, 0), + kwargs={}, + ) + state.meta["val"] = get_sample_rng_state(device) + + rng_output = fw_graph.create_node( + "call_function", + operator.getitem, + args=( + functional_fw_node, + 1, + ), + kwargs={}, + ) + # Copy the meta data from the original node + rng_output.meta = copy.copy(fw_node.meta) + + fw_node.replace_all_uses_with(rng_output) + fw_graph.erase_node(fw_node) + fw_rng_state_outputs.append(state) + + # Step 3 - Modify the bwd pass such that + with bw_graph.inserting_before(bw_tangent_start_node): + state_name = f"rng_state_output_{next(uid)}" + bw_rng_state_node = bw_graph.placeholder(state_name) + bw_rng_state_node.meta["val"] = get_sample_rng_state(device) + + with bw_graph.inserting_before(bw_node): + rng_output = bw_graph.create_node( + "call_function", + run_with_rng_state, + args=(bw_rng_state_node, bw_node.target, *bw_node.args), + kwargs=bw_node.kwargs, + ) + + bw_node.replace_all_uses_with(rng_output) + bw_graph.erase_node(bw_node) + + # Add the rng states in the output of the fwd graph. AOT Autograd assumes + # that symints are at the end of forward graph outputs. So, insert the new + # rng states accordingly. + if fw_rng_state_outputs: + fw_output_node = next(iter(fw_module.graph.find_nodes(op="output"))) + fw_outputs = fw_output_node.args[0] + sym_node_start_idx = len(fw_outputs) - num_sym_nodes + outputs = ( + fw_outputs[:sym_node_start_idx] + + tuple(fw_rng_state_outputs) + + fw_outputs[sym_node_start_idx:] + ) + fw_module.graph.output(outputs) + fw_module.graph.erase_node(fw_output_node) + fw_module.recompile() + bw_module.recompile() + return fw_module, bw_module + + +def force_save_collectives(joint_module: fx.GraphModule) -> None: + """ + By default, the partitioner is not allowed to recompute collectives + unless they come from a user-annotated AC region. + See Note [Recomputing collectives in the partitioner] + """ + for node in joint_module.graph.nodes: + if ( + isinstance(node.target, torch._ops.OpOverload) + and node.target.namespace == "_c10d_functional" + and not must_recompute(node) + ): + node.meta["recompute"] = CheckpointPolicy.MUST_SAVE + + +def force_save_bw_mutation_src(joint_module: fx.GraphModule) -> None: + # If we have mutations of the same primal in forward and backward, + # We must not recompute the source of mutation to not apply twice. + has_mutation_in_bw: OrderedSet[torch.fx.Node] = OrderedSet() + for node in reversed(joint_module.graph.nodes): + if node.op == "output": + continue + + is_copy_ = node.target is torch.ops.aten.copy_.default + if is_copy_: + if _has_tag_must_be_in_backward(node): + has_mutation_in_bw.add(node.args[0]) + + if _has_tag_must_be_in_forward(node) and node.args[0] in has_mutation_in_bw: + node.args[1].meta["recompute"] = CheckpointPolicy.MUST_SAVE + else: + # We use invariant of aotdispatch joint graph, + # That we emit copy_ only in the end of it. + # We do not want to iterate through all the joint graph, + # so break at the first non-output, non-copy_ node. + break + + +def is_getitem_of_multi_output(node): + if node.target != operator.getitem: + return False + parent = node.args[0] + return "tensor_meta" not in parent.meta and node.op == "call_function" + + +def cleanup_recompute_tags( + joint_module: fx.GraphModule, *, is_default_partition: bool +) -> fx.GraphModule: + """ + If there are two consecutive checkpointed blocks with no operator in + between, we would still want to stash the tensor at the boundary of + checkpointed blocks. The following pass makes the last output node + non-recomputable to allow for that. + """ + for node in joint_module.graph.nodes: + if must_recompute(node): + for user in node.users: + if ( + must_recompute(user) + and "ac_graph_id" in user.meta + and "ac_graph_id" in node.meta + and user.meta["ac_graph_id"] > node.meta["ac_graph_id"] + ): + node.meta["recompute"] = CheckpointPolicy.MUST_SAVE + if node.meta.get("has_backward_hook", False) and not any( + must_recompute(user) for user in node.users + ): + # If node is AC region output and has a backward hook on it, we intentionally choose to save it. + # This is to work around circular dependencies in Traceable FSDP2+AC. + # Example: + # ``` + # out = fully_shard(utils.checkpoint(module))(x) + # norm_out = layer_norm(out) + # ``` + # Here there is a circular dependency: + # 1. In backward, grad_input of layer_norm aka. `out_grad` is actually dependent on `out`. + # 2. `out` depends on `out`'s backward hook created by FSDP2 (which does all-gather for `module` weights) + # in order to be recomputed. + # 3. `out`'s backward hook, as is the case for all eager backward hooks, depends on `out_grad` + # -> circular dependency with (1)! + # + # Solution: check whether `out` has a backward hook, and if so, intentionally save `out` + # in forward graph outputs. With this, we can break the above circular dependency. + node.meta["recompute"] = CheckpointPolicy.MUST_SAVE + elif ( + "ac_graph_id" not in node.meta + and any(must_recompute(user) for user in node.users) + and not ( + # Avoid saving getitem nodes which are not labeled with "ac_graph_id" + is_getitem_of_multi_output(node) and "ac_graph_id" in node.args[0].meta + ) + and is_default_partition + ): + # This node is not part of the AC region and a user is marked as recompute. + # This means it's an input to the AC region and we should save it. + # For ease of landing, gate this to default partitioner only, but we should think + # about flipping the switch in general as well. + node.meta["recompute"] = CheckpointPolicy.MUST_SAVE + return joint_module + + +def solve_min_cut( + joint_graph: fx.Graph, + node_info: NodeInfo, + min_cut_options: MinCutOptions, + dont_ban: Optional[OrderedSet[fx.Node]] = None, +): + if dont_ban is None: + dont_ban = OrderedSet() + op_types = get_default_op_list() + + if AOT_PARTITIONER_DEBUG: + joint_module_ops = OrderedSet( + str(node.target._overloadpacket) + for node in joint_graph.nodes + if node.op == "call_function" and hasattr(node.target, "_overloadpacket") + ) + ops_ignored = joint_module_ops - OrderedSet( + str(i) for i in op_types.recomputable_ops + ) + log.info("Ops banned from re-materialization: %s", ops_ignored) + + def can_fuse_into_auto_functionalized(a, b): + if b.target != torch.ops.higher_order.auto_functionalized: + return False + mutable_op = b.args[0] + ( + mutable_arg_names, + _, + ) = torch._higher_order_ops.auto_functionalize.get_mutable_args(mutable_op) + for name in mutable_arg_names: + arg = b.kwargs[name] + if a is arg: + return True + if isinstance(arg, list): + if a in arg: + return True + return False + + def can_fuse_into_triton_kernel_wrapper_functional(a, b): + if b.target != torch.ops.higher_order.triton_kernel_wrapper_functional: + return False + mutable_arg_names = b.kwargs["tensors_to_clone"] + for name in mutable_arg_names: + arg = b.kwargs["kwargs"][name] + if a is arg: + return True + return False + + def is_fusible(a, b): + # We can perform "memory fusion" into a cat, but cat cannot be a + # producer to a fusion + if get_aten_target(b) == aten.cat: + return True + if can_fuse_into_auto_functionalized(a, b): + return True + if can_fuse_into_triton_kernel_wrapper_functional(a, b): + return True + if ( + a.target is operator.getitem + and a.args[0].target + is torch.ops.higher_order.triton_kernel_wrapper_functional + ): + # if a is the output of a user triton kernel, + # then (by default) we will not be able to fuse b into it + return False + return op_types.is_fusible(a) and op_types.is_fusible(b) + + try: + import networkx as nx + except ImportError as e: + raise RuntimeError( + "Need networkx installed to perform smart recomputation heuristics" + ) from e + + def is_materialized_backwards(node): + if op_types.is_view(node): + return False + cur_nodes = OrderedSet([node]) + while len(cur_nodes) > 0: + cur = cur_nodes.pop() + for user in cur.users: + if not node_info.is_required_fw(user) and not is_fusible(cur, user): + return True + if op_types.is_view(user): + cur_nodes.add(user) + + return False + + def should_ban_recomputation(node): + if node.op != "call_function": + return False + if node.target is operator.getitem: + return False + if node.meta.get("recompute", None) == CheckpointPolicy.MUST_SAVE: + return True + if config.recompute_views and op_types.is_view(node): + return False + if node.target in [aten.lift_fresh_copy.default, aten.lift_fresh.default]: + return False + + if min_cut_options.ban_if_not_in_allowlist: + if not op_types.is_recomputable(node): + return True + else: + if ( + op_types.is_random(node) + or op_types.is_compute_intensive(node) + or is_non_builtin_to_include(node) + ): + return True + + # If a node *must* be materialized in the backwards pass, then we + # should never recompute it. This is a pretty subtle point. In + # general, the assumption we make is that recomputing a node in the + # backwards pass is "free". However, if a node must be materialized + # in the backwards pass, then recomputing it is never free. + if min_cut_options.ban_if_materialized_backward and is_materialized_backwards( + node + ): + log.debug("materialized backwards: %s %s", node, tuple(node.users)) + return True + + # Arbitrary hack that sometimes seems to help things. The above + # modification appears to have made this heuristic a lot less critical + # for performance. + # NB: As of PR #121692, this hack no longer seems necessary. + if node.dist_from_bw < 1000 and node.dist_from_bw > config.max_dist_from_bw: + return True + + # If the output of an op is 4x smaller (arbitrary choice), + # then we don't allow recomputation. The idea here is that for + # things like reductions, saving the output of the reduction is very + # cheap/small, and it makes sure we don't do things like recompute + # normalizations in the backwards. + if min_cut_options.ban_if_reduction: + input_tensors_size = sum( + _size_of(i) for i in node.args if isinstance(i, fx.Node) + ) + output_size = _size_of(node) + return output_size * 4 < input_tensors_size + return False + + def is_materialized(node): + if node.op == "placeholder": + return True + + return not all(is_fusible(node, user) for user in node.users) + + def get_node_weight(node, static_lifetime_input_nodes) -> float: + if ( + config.treat_parameters_as_free_to_save + and node in static_lifetime_input_nodes + ): + return 0 + mem_sz = _size_of(node) + if config.recompute_views and op_types.is_view(node): + # If `config.recompute_views=True`, we don't save views. This is generally + # a good idea since views are free to recompute, and it makes it a bit simpler + # to analyze. + # NB: If they're not free to recompute (e.g. nested tensors)... I + # think we should modify checks for view_ops to `is_view` and check + # that. Basically, with nested tensors, `aten.view` is not a "view + # op". + return math.inf + + if isinstance(node.meta["val"], py_sym_types): + # We never want to save symfloats + if not isinstance(node.meta["val"], torch.SymInt): + return INT_INF + + # Heuristic to bias towards nodes closer to the backwards pass + # Complete guess about current value + mem_sz = int(mem_sz * (1.1 ** max(min(node.dist_from_bw, 100), 1))) + if is_materialized(node): + return mem_sz + else: + return mem_sz * 2 + + nx_graph = nx.DiGraph() + banned_nodes: OrderedSet[fx.Node] = OrderedSet() + + def ban_recomputation_if_allowed(node): + if op_types.is_view(node): + return False + if node in dont_ban: + # collectives are *always* banned from recompute, overriding `dont_ban` + # (in particular, the activation memory budget logic is not allowed to recompute collectives) + is_collective = ( + isinstance(node.target, torch._ops.OpOverload) + and node.target.namespace == "_c10d_functional" + ) + if config.unsafe_allow_optimization_of_collectives or not is_collective: + return False + # This bans recomputation of the node unless we've been forced not to by + # user annotation + if must_recompute(node): + return False + + if "val" in node.meta and isinstance(node.meta["val"], torch.SymFloat): + return False + banned_nodes.add(node) + # A node will only ever be recomputed if there is a path from an + # ancestor of this node to the backwards path through this node that + # doesn't go through any saved value. If this node is saved, then that + # condition is not possible. + nx_graph.add_edge("source", node.name + "_in", capacity=math.inf) + return True + + for node in joint_graph.nodes: + if node.op == "output": + continue + + if node in node_info.required_bw_nodes: + if node not in node_info.inputs: + nx_graph.add_edge(node.name + "_in", "sink", capacity=math.inf) + continue + # If someone saves a input for backward as-is and backward + # returns that tensor as-is as a grad input, then the node x would + # be both a required_bw_node and an input. In this case we + # (1) connect x_in to the source, (2) x_out to the sink, and + # (3) assign the proper weight to the x_in-x_out edge, so that + # x would be part of cut nodes. A case where this happens is if + # NestedTensor saves a offset tensor as part of the singleton int + # in sizes. + nx_graph.add_edge(node.name + "_out", "sink", capacity=math.inf) + + if must_recompute(node): + # If user explicitly says they want to recompute a node, we honor it + # by adding an inf-capacity edge from X_in to the sink. + # This way, X_in node is guaranteed to be part of the subgraph that contains "sink" + # after the cut, thus guaranteeing that X op will be recomputed. + nx_graph.add_edge(node.name + "_in", "sink", capacity=math.inf) + continue + + if _is_primal(node) or _is_fwd_seed_offset(node): + ban_recomputation_if_allowed(node) + + # If a node can't be recomputed (too expensive or involves randomness), + # we prevent it from being recomputed by adding an inf edge to the source + # We only need to ban nodes in the fw pass, as those are the only ones that would be recomputed. + if node_info.is_required_fw(node) and should_ban_recomputation(node): + ban_recomputation_if_allowed(node) + + # Checks if a node is actually a tuple. Can be simplified to just an isinstance check if we always use faketensors. + is_non_tensor_node = ( + "val" not in node.meta and "tensor_meta" not in node.meta + ) or ("val" in node.meta and not isinstance(node.meta["val"], torch.Tensor)) + + if is_sym_node(node): + weight = float(sym_node_size(node)) + elif is_non_tensor_node: + weight = ( + 0.0 if isinstance(node.meta.get("val"), BackwardState) else math.inf + ) + else: + weight = get_node_weight(node, node_info.static_lifetime_input_nodes) + # Creates the weights on the "node" edge + nx_graph.add_edge(node.name + "_in", node.name + "_out", capacity=weight) + for user in node.users: + nx_graph.add_edge(node.name + "_out", user.name + "_in", capacity=math.inf) + + # todo(chilli): This is the most questionable of the 3 heuristics for banning recompute. + # Some example models to look at where this helps perf: poolformer_m36, + # mixer_b16_224, cait_m36_384 + + # The "rough" idea here is that if you have some node that is used by both a + # node nearby downstream as well as a node far downstream, if we recompute + # both of the downstream nodes, we're unlikely to be able to fuse both + # downstream nodes together. + + # Thus, we shouldn't aim to recompute far downstream nodes that depend on + # this node. That intuition of "far downstream" is captured by whether + # there's an unfusible op along the chain somewhere + + # It could probably be improved by properly analyzing what's going on in the + # backwards pass instead of only relying on whether it's unfusible in the + # forwards. + + def find_first_unfusible(start_nodes: list[fx.Node], max_range: int) -> int: + """ + Finds the first unfusible node in the chain of nodes starting from + `start_nodes` and returns its position. + """ + sorted_nodes: list[tuple[int, fx.Node, bool]] = [] + for n in start_nodes: + heapq.heappush(sorted_nodes, (node_info.get_fw_order(n), n, True)) + + while len(sorted_nodes) > 0: + _, node, node_is_fusible = heapq.heappop(sorted_nodes) + if not node_is_fusible: + return node_info.get_fw_order(node) + for user in node.users: + if node_info.is_required_fw(user): + if node_info.get_fw_order(user) > max_range: + continue + val: tuple[int, fx.Node, bool] = ( + node_info.get_fw_order(user), + user, + is_fusible(node, user), + ) + if val not in sorted_nodes: + heapq.heappush(sorted_nodes, val) + return max_range + + if min_cut_options.ban_if_used_far_apart: + for used_node in node_info.required_fw_nodes: + orders = [ + node_info.get_fw_order(user) + for user in used_node.users + if node_info.is_required_fw(user) + ] + fw_users = [ + user for user in used_node.users if node_info.is_required_fw(user) + ] + if len(orders) > 0: + first_unfusible_use = find_first_unfusible(fw_users, max(orders)) + for user in tuple(used_node.users): + if ( + node_info.is_required_fw(user) + and node_info.get_fw_order(user) > first_unfusible_use + and is_fusible(used_node, user) + ): + if user in banned_nodes: + continue + log.info( + "used above/below fusible %s:(%s) -> %s -> %s:(%s)", + used_node, + node_info.get_fw_order(used_node), + first_unfusible_use, + user, + node_info.get_fw_order(user), + ) + ban_recomputation_if_allowed(user) + + # This heuristic is fairly straightforward. The idea is that although it is + # cheap to recompute bandwidth-bound ops, we don't want to end up in a situation + # where we have a long chain of pointwise ops from the beginning to the end + # of the model (like say, residual connections) + + # todo: I'm not totally sure why this heuristic matters. It's possible that this is + # working around Inductor fusion decisions, or that it's a patch over + # suboptimal partitioning decisions + + # Some models it improves perf on are cait_m36_384, mixer_b16_224, poolformer_m36 + + if min_cut_options.ban_if_long_fusible_chains: + visited: OrderedSet[fx.Node] = OrderedSet() + for start_node in joint_graph.nodes: + if not node_info.is_required_fw(start_node): + continue + fusible: list[tuple[int, fx.Node]] = [ + (node_info.get_fw_order(start_node), start_node) + ] + start_order = node_info.get_fw_order(start_node) + while len(fusible) > 0: + _, cur = heapq.heappop(fusible) + if cur in visited: + continue + visited.add(cur) + # 100 is arbitrary choice to try and prevent degenerate cases + if ( + node_info.get_fw_order(cur) > start_order + 100 + and len(fusible) == 0 + ): + log.info( + "too long %s %s %s %s", + cur, + start_node, + node_info.get_fw_order(cur), + node_info.get_fw_order(start_node), + ) + ban_recomputation_if_allowed(cur) + break + + for user in cur.users: + if ( + node_info.is_required_fw(user) + and is_fusible(cur, user) + and user not in banned_nodes + ): + heapq.heappush(fusible, (node_info.get_fw_order(user), user)) + + try: + cut_value, partition = nx.minimum_cut(nx_graph, "source", "sink") + except Exception: + log.info("Failed to compute min-cut on following graph:") + log.info("\n".join(nx.readwrite.edgelist.generate_edgelist(nx_graph))) + visualize_min_cut_graph(nx_graph) + raise + + reachable, non_reachable = partition + cutset: OrderedSet[tuple[str, str]] = OrderedSet() + for u, nbrs in ((n, nx_graph[n]) for n in reachable): + cutset.update((u, v) for v in nbrs if v in non_reachable) + + cut_nodes: OrderedSet[str] = OrderedSet() + for node_in, node_out in cutset: + assert node_in[:-3] == node_out[:-4] + node_name = node_in[:-3] + cut_nodes.add(node_name) + + name_to_node = get_name_to_node(joint_graph) + # To make this stuff deterministic + node_idx = {node: idx for idx, node in enumerate(joint_graph.nodes)} + saved_values = sorted( + (name_to_node[node] for node in cut_nodes), key=lambda x: node_idx[x] + ) + return saved_values, banned_nodes + + +def visualize_min_cut_graph(nx_graph): + import networkx as nx + import pydot + + dot_format = nx.nx_pydot.to_pydot(nx_graph).to_string() + dot_graph = pydot.graph_from_dot_data(dot_format)[0] # type: ignore[index] + for edge in dot_graph.get_edges(): + weight = nx_graph[edge.get_source()][edge.get_destination()]["capacity"] + # Set edge label to weight + edge.set_label(str(weight)) # type: ignore[union-attr] + # Color edges with weight 'inf' as red + if weight == float("inf"): + edge.set_color("red") # type: ignore[union-attr] + log.info("Visualizing the failed graph to min_cut_failed.svg") + dot_graph.write_svg("min_cut_failed.svg") # type: ignore[union-attr] + + +def get_default_op_list() -> OpTypes: + default_recomputable_ops: list[Callable] = [ + aten.add, + aten.sub, + aten.div, + aten.atan2, + aten.mul, + aten.max, + aten.min, + aten.pow, + aten.remainder, + aten.fmod, + aten.__and__, + aten.__or__, + aten.__xor__, + aten.__lshift__, + aten.__rshift__, + aten.eq, + aten.ne, + aten.ge, + aten.gt, + aten.le, + aten.lt, + aten.abs, + aten.bitwise_not, + aten.ceil, + aten.floor, + aten.frac, + aten.neg, + aten.relu, + aten.round, + aten.silu, + aten.trunc, + aten.log, + aten.log10, + aten.log1p, + aten.log2, + aten.lgamma, + aten.exp, + aten.expm1, + aten.erf, + aten.erfc, + aten.cos, + aten.acos, + aten.cosh, + aten.sin, + aten.asin, + aten.sinh, + aten.tan, + aten.atan, + aten.tanh, + aten.atanh, + aten.sqrt, + aten.rsqrt, + aten.reciprocal, + aten.sigmoid, + aten.softplus, + aten.threshold, + aten.threshold_backward, + aten.clamp, + aten.where, + aten.lerp, + aten.addcmul, + aten.gelu, + aten.gelu_backward, + aten.sum, + aten.mean, + aten._grad_sum_to_size, + aten.sum_to_size, + aten.amax, + aten.to, + aten.type_as, + operator.getitem, + aten.squeeze, + aten.unsqueeze, + aten.rsub, + aten._to_copy, + ] # noqa: E501,B950 + recomputable_view_ops = [aten.squeeze, aten.unsqueeze, aten.alias] + recomputable_view_ops += [ + aten.view, + aten.slice, + aten.t, + prims.broadcast_in_dim, + aten.expand, + aten.as_strided, + aten.permute, + aten.select, + aten.split, + ] + view_ops = recomputable_view_ops + default_recomputable_ops += [ + prims.div, + prims.convert_element_type, + aten.clone, + aten._to_copy, + aten.full_like, + prims.var, + prims.sum, + aten.var, + aten.std, + prims.broadcast_in_dim, + aten.select, + aten._unsafe_view, + aten.view, + aten.expand, + aten.slice, + aten.reshape, + aten.broadcast_tensors, + aten.scalar_tensor, + aten.ones, + aten.new_zeros, + aten.lift_fresh_copy, + aten.arange, + aten.triu, + aten.var_mean, + aten.isinf, + aten.any, + aten.full, + aten.as_strided, + aten.zeros, + aten.empty, + aten.empty_like, + aten.argmax, + aten.maximum, + prims.iota, + prims._low_memory_max_pool_offsets_to_indices, + ] # noqa: E501,B950 + # Natalia said that we should allow recomputing indexing :) + default_recomputable_ops += [aten.index, aten.gather] + default_recomputable_ops += view_ops + + default_recomputable_ops += pointwise_ops() + + default_recomputable_ops += [ + aten.zeros_like, + ] + + default_recomputable_ops += [method_to_operator(m) for m in magic_methods] + recomputable_ops = OrderedSet(default_recomputable_ops) + + random_ops = OrderedSet[Callable[..., Any]]( + [aten.native_dropout, aten.rand_like, aten.randn_like] + ) + compute_intensive_ops = [ + aten.mm, + aten.convolution, + aten.convolution_backward, + aten.bmm, + aten.addmm, + aten._scaled_dot_product_flash_attention, + aten._scaled_dot_product_efficient_attention, + aten._flash_attention_forward, + aten._efficient_attention_forward, + aten.upsample_bilinear2d, + aten._scaled_mm, + ] # noqa: E501,B950 + + fusible_ops = recomputable_ops | random_ops + return OpTypes( + fusible_ops, + OrderedSet(compute_intensive_ops), + random_ops, + OrderedSet(view_ops), + recomputable_ops, + ) + + +def get_name_to_node(graph: fx.Graph): + name_to_node = {} + for node in graph.nodes: + name_to_node[node.name] = node + return name_to_node + + +def _optimize_runtime_with_given_memory( + joint_graph: fx.Graph, + memory: list[float], + runtimes: list[float], + max_memory: float, + node_info: NodeInfo, + all_recomputable_banned_nodes: list[fx.Node], +) -> tuple[float, list[int], list[int]]: + SOLVER = config.activation_memory_budget_solver + if SOLVER == "greedy": + return greedy_knapsack(memory, runtimes, max_memory) + elif SOLVER == "ilp": + return ilp_knapsack(memory, runtimes, max_memory) + elif SOLVER == "dp": + return dp_knapsack(memory, runtimes, max_memory) + elif SOLVER == "dp_knapsack_sliding_hirschberg": + return dp_knapsack_sliding_hirschberg(memory, runtimes, max_memory) + elif SOLVER == "dynamic_memory_budget_dp": + log.warning( + "dynamic_memory_budget_dp is an experimental solver. " + "It does not guarantee performance improvements. " + "Additionally, it is not guaranteed to be stable." + ) + graph_info_provider = GraphInfoProvider.inialize_from_graph( + joint_graph=joint_graph, + all_recomputable_banned_nodes=all_recomputable_banned_nodes, + recorded_knapsack_input_memories=memory, + recorded_knapsack_input_runtimes=runtimes, + ) + return dp_knapsack( + memory, + runtimes, + KnapsackEvaluator( + graph_info_provider=graph_info_provider, + ).get_knee_point_memory_budget( + knapsack_algo=dp_knapsack, + max_mem_budget=max_memory, + ), + ) + elif callable(SOLVER): + saved_node_idx, recomp_node_idx = SOLVER( + memory, joint_graph, max_memory, node_info, all_recomputable_banned_nodes + ) + return (0.0, saved_node_idx, recomp_node_idx) + else: + raise RuntimeError(f"Not aware of memory budget knapsack solver: {SOLVER}") + + +from torch.utils._mode_utils import no_dispatch + + +# replace symbols in size and strides with their hints without guarding. +def _remove_symbols_without_guarding(x: torch.Tensor, fallback: int) -> torch.Tensor: + shape = list(x.shape) + + def realize_symbol(d): + return hint_int(d, fallback=fallback) + + shape = [realize_symbol(s) for s in shape] + stride = [realize_symbol(s) for s in x.stride()] + return x.new_empty_strided(shape, stride=stride) + + +def estimate_runtime(node): + RUNTIME_MODE = config.activation_memory_budget_runtime_estimator + + def materialize_arg(x): + if isinstance(x, fx.Node) and isinstance(x.meta["val"], torch.Tensor): + return _remove_symbols_without_guarding(x.meta["val"], fallback=4096) + elif isinstance(x, fx.Node) and isinstance(x.meta["val"], torch.SymInt): + return hint_int(x.meta["val"], fallback=4096) + elif isinstance(x, fx.Node) and isinstance(x.meta["val"], torch.SymFloat): + return 1.0 + elif isinstance(x, fx.Node) and isinstance(x.meta["val"], torch.SymBool): + return True + else: + return x + + if RUNTIME_MODE == "testing": + return 1 + + elif RUNTIME_MODE == "profile": + with no_dispatch(): + from torch._inductor.runtime.benchmarking import benchmarker + + args, kwargs = pytree.tree_map(materialize_arg, (node.args, node.kwargs)) + ms = benchmarker.benchmark_gpu(lambda: node.target(*args, **kwargs)) + return ms + + elif RUNTIME_MODE == "flops": + # todo(chilli): Normalize this to also return ms + from torch.utils.flop_counter import FlopCounterMode + + args, kwargs = pytree.tree_map(materialize_arg, (node.args, node.kwargs)) + with FlopCounterMode(display=False) as mode: + node.target(*args, **kwargs) + counted_flops = mode.get_total_flops() + return max(counted_flops, 1) + else: + raise RuntimeError(f"Not aware of runtime estimator: {RUNTIME_MODE}") + + +def choose_saved_values_set( + joint_graph: fx.Graph, + node_info: NodeInfo, + memory_budget=1, +) -> list[fx.Node]: + if memory_budget > 1 or memory_budget < 0: + raise RuntimeError( + f"The valid ranges for memory budget are 0 <= m <= 1. The provided value is {memory_budget}" + ) + min_cut_options = MinCutOptions( + ban_if_used_far_apart=config.ban_recompute_used_far_apart, + ban_if_long_fusible_chains=config.ban_recompute_long_fusible_chains, + ban_if_materialized_backward=config.ban_recompute_materialized_backward, + ban_if_not_in_allowlist=config.ban_recompute_not_in_allowlist, + ban_if_reduction=config.ban_recompute_reductions, + ) + + if config.aggressive_recomputation: + min_cut_options = replace( + min_cut_options, + ban_if_used_far_apart=False, + ban_if_long_fusible_chains=False, + ban_if_materialized_backward=False, + ban_if_not_in_allowlist=False, + ) + if memory_budget == 0: + return node_info.inputs + + runtime_optimized_saved_values, _ = solve_min_cut( + joint_graph, + node_info, + min_cut_options, + ) + # return runtime_optimized_saved_values + if memory_budget == 1: + return runtime_optimized_saved_values + + def estimate_activations_size(saved_values: list[fx.Node]) -> float: + return sum(map(_size_of, saved_values)) / 1e9 + + min_act_size = estimate_activations_size(node_info.inputs) + max_act_size = estimate_activations_size(runtime_optimized_saved_values) + # The optimized choice is smaller than the inputs anyways + if max_act_size <= min_act_size: + return runtime_optimized_saved_values + + def get_normalized_size(sz): + return (sz / 1e9) / (max_act_size - min_act_size) + + def get_mem_ratio(activations: list[fx.Node]): + return (estimate_activations_size(activations) - min_act_size) / ( + max_act_size - min_act_size + ) + + more_aggressive_options = replace( + min_cut_options, + ban_if_used_far_apart=False, + ban_if_long_fusible_chains=False, + ban_if_materialized_backward=False, + ) + more_aggressive_saved_values, _ = solve_min_cut( + joint_graph, node_info, more_aggressive_options + ) + if get_mem_ratio(more_aggressive_saved_values) < memory_budget: + return more_aggressive_saved_values + + aggressive_options = replace( + more_aggressive_options, + ban_if_not_in_allowlist=False, + ) + aggressive_recomputation_saved_values, banned_nodes = solve_min_cut( + joint_graph, node_info, aggressive_options + ) + + if get_mem_ratio(aggressive_recomputation_saved_values) < memory_budget: + return aggressive_recomputation_saved_values + + from torch._inductor.fx_utils import get_node_storage + + input_storages = OrderedSet(get_node_storage(node) for node in node_info.inputs) + + def get_recomputable_banned_nodes( + banned_nodes: OrderedSet[fx.Node], + ) -> list[fx.Node]: + return [ + i + for i in banned_nodes + if ( + # Only allow recomputing nodes that are actually required for BW + i.dist_from_bw < int(1e9) # type: ignore[attr-defined] + and ( + get_node_storage(i) not in input_storages + or is_non_builtin_to_include(i) + ) + ) + ] + + recomputable_banned_nodes = get_recomputable_banned_nodes(banned_nodes) + must_save_nodes = [ + i + for i in recomputable_banned_nodes + if i.meta.get("recompute", False) == CheckpointPolicy.MUST_SAVE + ] + recomputable_banned_nodes = [ + i for i in recomputable_banned_nodes if i not in must_save_nodes + ] + + # default: runtime_optimized_saved_values + # more aggressive: more_aggressive_saved_values + # full aggressive: aggressive_recomputation_saved_values + + all_recomputable_banned_nodes = sorted( + recomputable_banned_nodes, key=_size_of, reverse=True + ) + if len(all_recomputable_banned_nodes) == 0: + return node_info.inputs + must_save_nodes + memories_banned_nodes = [ + get_normalized_size(_size_of(i)) for i in all_recomputable_banned_nodes + ] + runtimes_banned_nodes = [ + estimate_runtime(node) for node in all_recomputable_banned_nodes + ] + from torch.utils._mode_utils import no_dispatch + + def get_saved_values_knapsack(memory_budget, node_info, joint_graph): + with no_dispatch(): + ( + expected_runtime, + saved_node_idxs, + recomputable_node_idxs, + ) = _optimize_runtime_with_given_memory( + joint_graph, + memories_banned_nodes, + runtimes_banned_nodes, + max(memory_budget, 0), + node_info, + all_recomputable_banned_nodes, + ) + dont_ban: OrderedSet[fx.Node] = OrderedSet() + for idx in recomputable_node_idxs: + # if idx in all_recomputable_banned_nodes: + try: + dont_ban.add(all_recomputable_banned_nodes[idx]) + except BaseException: # noqa: B036 + pass + + assert dont_ban.issubset(all_recomputable_banned_nodes) + + saved_values, _ = solve_min_cut( + joint_graph, + node_info, + aggressive_options, + dont_ban, + ) + if AOT_PARTITIONER_DEBUG: + create_structured_trace_for_min_cut_info( + joint_graph=joint_graph, + all_recomputable_banned_nodes=all_recomputable_banned_nodes, + saved_node_idxs=saved_node_idxs, + recomputable_node_idxs=recomputable_node_idxs, + expected_runtime=expected_runtime, + memories_banned_nodes=[ + _size_of(i) for i in all_recomputable_banned_nodes + ], + normalized_memories_banned_nodes=memories_banned_nodes, + runtimes_banned_nodes=runtimes_banned_nodes, + min_cut_saved_values=saved_values, + ) + return saved_values, expected_runtime + + if config.visualize_memory_budget_pareto: + + def estimate_for_budget(b): + saved_values, expected_runtime = get_saved_values_knapsack( + b, node_info=node_info, joint_graph=joint_graph + ) + return ( + b, + sum(runtimes_banned_nodes) - expected_runtime, + get_mem_ratio(saved_values), + ) + + options = [estimate_for_budget(0.0), estimate_for_budget(1.0)] + + if options[0][1:] != options[1][1:]: + bisects = [(options[0], options[1])] + while bisects: + lhs, rhs = bisects.pop() + if rhs[0] - lhs[0] < 1e-3: + options.append(lhs) + options.append(rhs) + continue + mid = estimate_for_budget((lhs[0] + rhs[0]) / 2) + if mid[1:] != lhs[1:]: + bisects.append((lhs, mid)) + if mid[1:] != rhs[1:]: + bisects.append((mid, rhs)) + options.sort() + + import matplotlib.pyplot as plt + + x_values = [item[2] for item in options] + y_values = [item[1] for item in options] + + # Plotting the values with updated axis labels and chart title + plt.figure(figsize=(10, 6)) + plt.plot(x_values, y_values, marker="o") + + # Adding labels for each point + for i, txt in enumerate(x_values): + plt.annotate( + f"{txt:.4f}", + (txt, y_values[i]), + textcoords="offset points", + xytext=(0, 10), + ha="center", + ) + + plt.xlabel("Memory Budget") + plt.ylabel("Runtime of Recomputed Components") + plt.title("Pareto Frontier of Memory Budget vs. Recomputation Runtime") + plt.grid(True) + fig = plt.gcf() + plt.show() + fig_dir = os.getcwd() + if config.memory_budget_pareto_dir is not None: + fig_dir = config.memory_budget_pareto_dir + os.makedirs(fig_dir, exist_ok=True) + rank_suffix = "" + if torch.distributed.is_available() and torch.distributed.is_initialized(): + rank_suffix = f"_rank_{torch.distributed.get_rank()}" + fig_name = os.path.join( + fig_dir, f"memory_budget_pareto{rank_suffix}_{get_aot_graph_name()}.svg" + ) + fig.savefig(fig_name) + log.warning("Generated Pareto frontier curve at %s", fig_name) + + # todo(chilli): Estimated doesn't align exactly with actual - actual is + # usually less memory than estimated. i'm guessing (actually quite + # unsure about this) that's because estimated is just only including + # tensors we actually banned from recompute, but there may be other + # tensors that we choose to save. + + return get_saved_values_knapsack( + memory_budget=memory_budget, node_info=node_info, joint_graph=joint_graph + )[0] + + +def _sync_decision_cross_ranks( + joint_graph: torch.fx.Graph, saved_values: list[torch.fx.Node] +): + # use the same policy across different GPUs + from torch._subclasses.fake_tensor import unset_fake_temporarily + + def has_collectives(joint_graph): + for node in joint_graph.nodes: + if isinstance( + node.target, torch._ops.OpOverload + ) and node.target.namespace in {"_c10d_functional", "c10d_functional"}: + return True + return False + + def has_same_nodes(joint_graph): + # proxy to check if the graph is the same across different GPUs. + # We only consider the name and order of nodes. A more robust way + # would be to check the hash of the whole graph (disregarding input shapes), + # this is a reasonable first-order approximation. + node_str = "/".join(x.name for x in joint_graph.nodes) + inputs = hashlib.sha256(node_str.encode("utf-8")).hexdigest() + all_inputs = [None for _ in range(torch.distributed.get_world_size())] + with no_dispatch(), unset_fake_temporarily(): + # TODO: maybe use a different process group? + torch.distributed.all_gather_object(all_inputs, inputs) + return all(all_inputs[0] == x for x in all_inputs) + + if ( + torch.distributed.is_available() + and torch.distributed.is_initialized() + and torch.distributed.get_world_size() > 1 + and has_collectives(joint_graph) + and has_same_nodes(joint_graph) + ): + with no_dispatch(), unset_fake_temporarily(): + objects = [[x.name for x in saved_values]] + saved_ops_names_all_ranks: list[list[str]] = [ + [] for _ in range(torch.distributed.get_world_size()) + ] + torch.distributed.all_gather_object(saved_ops_names_all_ranks, objects[0]) + name_to_node = get_name_to_node(joint_graph) + saved_sizes: list[int] = [] + saved_ops_with_sizes: dict[str, int] = {} + + for idx, saved_ops_names in enumerate(saved_ops_names_all_ranks): + saved_nodes = [name_to_node[op_name] for op_name in saved_ops_names] + saved_size = 0 + for node in saved_nodes: + size_of_node = _size_of(node) + saved_size += size_of_node + if idx == torch.distributed.get_rank(): + saved_ops_with_sizes[node.name] = size_of_node + saved_ops_with_sizes["total size"] = saved_size + saved_sizes.append(saved_size) + + saved_sizes_tensor = torch.tensor( + saved_sizes, + device=torch.distributed.distributed_c10d._get_object_coll_device(), + ) + torch.distributed.all_reduce( + saved_sizes_tensor, op=torch.distributed.distributed_c10d.ReduceOp.MAX + ) + + picked_rank_idx = int(torch.argmin(saved_sizes_tensor).item()) + sync_decision_cross_ranks_str = f"picked_rank_idx={picked_rank_idx}, saved_nodes of current rank={saved_ops_with_sizes}" + trace_structured( + "artifact", + metadata_fn=lambda: { + "name": "aot_joint_graph_sync_decision_cross_ranks", + "encoding": "string", + }, + payload_fn=lambda: sync_decision_cross_ranks_str, + ) + + saved_values = [ + name_to_node[n] for n in saved_ops_names_all_ranks[picked_rank_idx] + ] + + return saved_values + + +def thread_graphsafe_rng_from_hops(module, is_backward): + """ + Graph-safe RNG lets torch.compile use CUDA Graphs for graphs with RNG ops. + For graphs without HOPs, the partitioner adds placeholder nodes + fwd_rng_state_* and bw_rng_state_* to the forward and backward graphs. At + runtime, the AOTDispatcher retrieves these RNG states and passes them to the + compiled graphs. + + This works well for no-HOP graphs. With HOPs, the partitioner runs + recursively: it first partitions the HOP (producing forward/backward HOP + subgraphs) and then stitches them back into the outer joint graph. For HOPs + that contain RNG ops, the outer joint graph now includes HOP subgraph + modules with extra RNG placeholders. We must thread these placeholders + through the outer module partitioned forward and backward graphs—this + function does exactly that. It collects the RNG placeholder nodes from the + HOPs and creates corresponding placeholders in the outer forward and + backward graphs. + + There is a catch: for a short period, the joint graph is in a “bad” state. + The HOP subgraphs expect additional inputs (because of the new + placeholders), but the outer graph call sites don't yet provide them. We + can't fix this in the joint graph because the joint graph's input signature + is fixed (primals, tangents). As a compromise, we keep the joint graph in + somewhat of a bad state for some time and, once the outer forward and + backward graphs are partitioned, insert the corresponding RNG placeholders + and wire up the calls. + """ + + rng_count = 0 + rng_string = "bwd_rng_state" if is_backward else "fwd_rng_state" + last_input = next(reversed(module.graph.find_nodes(op="placeholder"))) + for hop_node in module.graph.find_nodes( + op="call_function", target=torch.ops.higher_order.invoke_subgraph + ): + subgraph = getattr(module, hop_node.args[0].target) + if isinstance(subgraph, fx.GraphModule): + new_rng_inputs = [] + for placeholder_node in subgraph.graph.find_nodes(op="placeholder"): + if rng_string in placeholder_node.name: + # Found a rng state placeholder in the hop graph, lets add + # the corresponding node in the outer graph + with module.graph.inserting_after(last_input): + rng_state = module.graph.placeholder( + f"{rng_string}_{rng_count}" + ) + rng_count += 1 + rng_state.meta["val"] = placeholder_node.meta["val"] + last_input = rng_state + new_rng_inputs.append(rng_state) + + if new_rng_inputs: + # Pass on the new args that include the new_rng_inputs + with module.graph.inserting_after(hop_node): + new_hop_node_with_fixed_args = module.graph.create_node( + "call_function", + torch.ops.higher_order.invoke_subgraph, + (*hop_node.args, *new_rng_inputs), # type: ignore[arg-type] + {}, + ) + hop_node.replace_all_uses_with( + new_hop_node_with_fixed_args, propagate_meta=True + ) + + # Setup the eager_input_vals + eager_vals = hop_node.meta.get("eager_input_vals") + if eager_vals: + eager_args, eager_kwargs = eager_vals + new_eager_args = ( + *eager_args, + *[inp.meta["val"] for inp in new_rng_inputs], + ) + new_hop_node_with_fixed_args.meta["eager_input_vals"] = ( + new_eager_args, + eager_kwargs, + ) + module.graph.erase_node(hop_node) + + return module + + +def classify_nodes(joint_module, static_lifetime_input_indices, num_fwd_outputs): + name_to_node = get_name_to_node(joint_module.graph) + required_bw_nodes: OrderedSet[fx.Node] = OrderedSet() + for node in joint_module.graph.nodes: + if node.op == "placeholder" and "tangents" in node.target: + required_bw_nodes.add(node) + elif _must_be_in_backward(node): + required_bw_nodes.add(node) + + if node in required_bw_nodes: + required_bw_nodes.update(node.users) + + primal_inputs = list(filter(_is_primal, joint_module.graph.nodes)) + fwd_seed_offset_inputs = list(filter(_is_fwd_seed_offset, joint_module.graph.nodes)) + inputs = primal_inputs + fwd_seed_offset_inputs + fwd_outputs, bwd_outputs, fwd_outputs_descs, bwd_outputs_descs = ( + _extract_fwd_bwd_outputs(joint_module, num_fwd_outputs=num_fwd_outputs) + ) + required_bw_nodes.update( + o for o in bwd_outputs if o is not None and o.op != "output" + ) + forward_only_graph = _extract_graph_with_inputs_outputs( + joint_module.graph, inputs, fwd_outputs, fwd_outputs_descs, "forward" + ) + required_fw_nodes: OrderedSet[fx.Node] = OrderedSet( + name_to_node[node.name] + for node in forward_only_graph.nodes + if node.op != "output" + ) + unclaimed_nodes: OrderedSet[fx.Node] = OrderedSet( + node + for node in joint_module.graph.nodes + if node not in required_fw_nodes and node not in required_bw_nodes + ) + static_lifetime_input_nodes = OrderedSet( + p for i, p in enumerate(primal_inputs) if i in static_lifetime_input_indices + ) + fw_cnt = 0 + fw_order = {} + for node in joint_module.graph.nodes: + if node in required_fw_nodes: + fw_order[node] = fw_cnt + fw_cnt += 1 + return NodeInfo( + inputs, + required_fw_nodes, + required_bw_nodes, + unclaimed_nodes, + fw_order, + static_lifetime_input_nodes, + ) + + +def min_cut_rematerialization_partition( + joint_module: fx.GraphModule, + _joint_inputs, + compiler="inductor", + *, + num_fwd_outputs, + static_lifetime_input_indices: Optional[list[int]] = None, +) -> tuple[fx.GraphModule, fx.GraphModule]: + """ + Partitions the joint graph such that the backward recomputes the forward. + Recomputing helps in trading off memory bandwidth with computation. + + To create the fwd and bwd graph, we copy the joint graph, manually set the + outputs to just original forward or backward outputs. And then we run the + resulting graphs through dead code elimination. + + .. warning:: + This API is experimental and likely to change. + + Args: + joint_module(fx.GraphModule): The joint forward and backward graph. This + is the result of AOT Autograd tracing. + _joint_inputs: The inputs to the joint graph. This is unused. + compiler: This option determines the default set of recomputable ops. + Currently, there are two options: ``nvfuser`` and ``inductor``. + recomputable_ops: This is an optional set of recomputable ops. If this + is not None, then this set of ops will be used instead of the + default set of ops. + num_fwd_outputs: The number of outputs from the forward graph. + + Returns: + Returns the generated forward and backward Fx graph modules. + """ + + joint_module.graph.eliminate_dead_code() + joint_module.recompile() + + fx_g = joint_module.graph + + # add the CSE pass + if config.cse: + cse_graph = fx_graph_cse(fx_g) + joint_module.graph = cse_graph + joint_graph = joint_module.graph + + graph_has_recomputable_ops = has_recomputable_ops(joint_module) + graph_has_recomputable_rng_ops = has_recomputable_rng_ops(joint_module) + if graph_has_recomputable_ops: + joint_module = cleanup_recompute_tags(joint_module, is_default_partition=False) + if not config.unsafe_allow_optimization_of_collectives: + force_save_collectives(joint_module) + force_save_bw_mutation_src(joint_module) + + if static_lifetime_input_indices is None: + static_lifetime_input_indices = [] + node_info = classify_nodes( + joint_module, static_lifetime_input_indices, num_fwd_outputs + ) + + # networkx blows up on graphs with no required backward nodes + # Since there's nothing to partition anyway, and the default partitioner can "handle" + # this case, send our graph over to the default partitioner. + if len(node_info.required_bw_nodes) == 0: + return default_partition( + joint_module, + _joint_inputs, + num_fwd_outputs=num_fwd_outputs, + static_lifetime_input_indices=static_lifetime_input_indices, + static_lifetime_input_nodes=node_info.static_lifetime_input_nodes, + ) + + for node in reversed(joint_module.graph.nodes): + if node.op == "output": + node.dist_from_bw = int(1e9) + elif not node_info.is_required_fw(node): + node.dist_from_bw = 0 + else: + node.dist_from_bw = int(1e9) + for user in node.users: + node.dist_from_bw = min(node.dist_from_bw, user.dist_from_bw + 1) + + memory_budget = config.activation_memory_budget + for node in joint_graph.nodes: + if isinstance(node.meta.get("memory_budget", None), float): + memory_budget = node.meta["memory_budget"] + break + saved_values = choose_saved_values_set( + joint_graph, + node_info, + memory_budget=memory_budget, + ) + # pyrefly: ignore [unbound-name] + if config._sync_decision_cross_ranks: + saved_values = _sync_decision_cross_ranks(joint_graph, saved_values) + # save_for_backward on tensors and stashes symints in autograd .ctx + saved_sym_nodes = list(filter(is_sym_node, saved_values)) + saved_values = list(filter(lambda n: not is_sym_node(n), saved_values)) + + # NB: saved_sym_nodes will be mutated to reflect the actual saved symbols + fw_module, bw_module = _extract_fwd_bwd_modules( + joint_module, + saved_values, + # pyrefly: ignore [bad-argument-type] + saved_sym_nodes=saved_sym_nodes, + num_fwd_outputs=num_fwd_outputs, + static_lifetime_input_nodes=node_info.static_lifetime_input_nodes, + ) + if graph_has_recomputable_ops: + if graph_has_recomputable_rng_ops: + fw_module, bw_module = functionalize_rng_ops( + joint_module, fw_module, bw_module, len(saved_sym_nodes) + ) + bw_module = reordering_to_mimic_autograd_engine(bw_module) + + # pyrefly: ignore [unbound-name] + if config.enable_activation_offloading: + from ._activation_offloading.activation_offloading import ( + enable_activation_offloading, + ) + + enable_activation_offloading( + fw_module, + bw_module, + num_fwd_outputs, + node_info.static_lifetime_input_nodes, + ) + + # raise all getitem ops to as early as possible + # this is helpful for memory, especially in the case of aot_eager backend + fw_module = raise_getitems(fw_module) + bw_module = raise_getitems(bw_module) + + fw_module = thread_graphsafe_rng_from_hops(fw_module, is_backward=False) + bw_module = thread_graphsafe_rng_from_hops(bw_module, is_backward=True) + + if AOT_PARTITIONER_DEBUG: + # Calculate sorted sizes of saved values + sorted_sizes = sorted([(_size_of(i), str(i)) for i in saved_values]) + + # Log total theoretical activations stored + total_activations_size_gb = sum(_size_of(i) for i in saved_values) / 1e9 + log.info("Theoretical Activations Stored: %.2f GB", total_activations_size_gb) + + # Log theoretical per activation storage sizes + log.info("Theoretical Per Activation Storage Sizes: %s", sorted_sizes) + fw_module_nodes = OrderedSet( + node.name for node in fw_module.graph.nodes if node.op == "call_function" + ) + bw_module_nodes = OrderedSet( + node.name for node in bw_module.graph.nodes if node.op == "call_function" + ) + remat_nodes = fw_module_nodes & bw_module_nodes + + counts: dict[str, int] = defaultdict(int) + for node in fw_module.graph.nodes: + if node.name in remat_nodes and hasattr(node.target, "_overloadpacket"): + counts[str(node.target._overloadpacket)] += 1 + log.info( + "# remat/fw/bw: %d/%d/%d", + len(remat_nodes), + len(fw_module_nodes), + len(bw_module_nodes), + ) + rematerialized_ops = sorted( + counts.items(), key=operator.itemgetter(1), reverse=True + ) + log.info("Count of Ops Rematerialized: %s", rematerialized_ops) + return fw_module, bw_module + + +def draw_graph( + traced: torch.fx.GraphModule, + fname: str, + figname: str = "fx_graph", + clear_meta: bool = True, + prog: Optional[Union[str, list[str]]] = None, + parse_stack_trace: bool = False, + dot_graph_shape: Optional[str] = None, +) -> None: + if clear_meta: + new_graph = copy.deepcopy(traced.graph) + traced = fx.GraphModule(traced, new_graph) + for node in traced.graph.nodes: + node.meta = {} + base, ext = os.path.splitext(fname) + if not ext: + ext = "." + config.torch_compile_graph_format + log.info("Writing FX graph to file: %s%s", base, ext) + g = graph_drawer.FxGraphDrawer( + traced, + figname, + parse_stack_trace=parse_stack_trace, + dot_graph_shape=dot_graph_shape, + ) + x = g.get_main_dot_graph() + write_method = getattr(x, "write_" + ext.lstrip(".")) + fname = f"{base}{ext}" + if prog is None: + write_method(fname) + else: + write_method(fname, prog=prog) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/predispatch.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/predispatch.py new file mode 100644 index 0000000000000000000000000000000000000000..aca329be3eb68850359815d597b7f56ac4a6c272 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/predispatch.py @@ -0,0 +1,159 @@ +# mypy: ignore-errors + +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +""" +This module contains pre-dispatch wrappers for functorch operations +that enable proper tracing in PT2 non-strict export/compile fx graph. +""" + +import torch +from torch._C._functorch import ( + _add_batch_dim as _add_batch_dim_impl, + _remove_batch_dim as _remove_batch_dim_impl, + _vmap_decrement_nesting as _vmap_decrement_nesting_impl, + _vmap_increment_nesting as _vmap_increment_nesting_impl, +) + + +def _add_batch_dim(self, batch_dim, level): + """ + Thin wrapper around torch._C._add_batch_dim that is used to proxy in + PT2 export/compile fx graph + """ + from torch._export.utils import _maybe_find_pre_dispatch_tf_mode_for_export + + mode = _maybe_find_pre_dispatch_tf_mode_for_export() + batch_dim = self.ndim + batch_dim if batch_dim < 0 else batch_dim + + if mode: + return torch.overrides.handle_torch_function( + _add_batch_dim, (self,), self, batch_dim, level + ) + + res = _add_batch_dim_impl(self, batch_dim, level) + return res + + +def _remove_batch_dim(self, level, batch_size, out_dim): + """ + Thin wrapper around torch._C._remove_batch_dim that is used to proxy in + PT2 export/compile fx graph + """ + from torch._export.utils import _maybe_find_pre_dispatch_tf_mode_for_export + + mode = _maybe_find_pre_dispatch_tf_mode_for_export() + + if mode: + return torch.overrides.handle_torch_function( + _remove_batch_dim, (self,), self, level, batch_size, out_dim + ) + + res = _remove_batch_dim_impl(self, level, batch_size, out_dim) + return res + + +def _vmap_increment_nesting(batch_size, randomness): + """ + Thin wrapper around torch._C._vmap_increment_nesting that is used + to proxy in export/compile graph + """ + from torch._export.utils import _maybe_find_pre_dispatch_tf_mode_for_export + + mode = _maybe_find_pre_dispatch_tf_mode_for_export() + + if mode: + return torch.overrides.handle_torch_function( + _vmap_increment_nesting, (batch_size,), batch_size, randomness + ) + res = _vmap_increment_nesting_impl(batch_size, randomness) + return res + + +def _vmap_decrement_nesting(): + """ + Thin wrapper around torch._C._vmap_increment_nesting that is used + to proxy in export/compile graph + """ + from torch._export.utils import _maybe_find_pre_dispatch_tf_mode_for_export + + mode = _maybe_find_pre_dispatch_tf_mode_for_export() + + if mode: + return torch.overrides.handle_torch_function( + _vmap_decrement_nesting, + (), + ) + return _vmap_decrement_nesting_impl() + + +# Global variables for lazy_load_decompositions +DECOMPOSITIONS_LOADED = False +DECOMPOSITIONS_LOCK = None # Will be initialized when needed +VMAP_DECOMPOSITIONS_LIB = None + + +def lazy_load_decompositions(): + """ + Lazy loading of vmap decompositions with pre-dispatch support. + """ + from torch._export.utils import _maybe_find_pre_dispatch_tf_mode_for_export + + mode = _maybe_find_pre_dispatch_tf_mode_for_export() + + if mode: + return torch.overrides.handle_torch_function(lazy_load_decompositions, ()) + + global DECOMPOSITIONS_LOADED, DECOMPOSITIONS_LOCK, VMAP_DECOMPOSITIONS_LIB + + if DECOMPOSITIONS_LOADED: + return + + # Initialize lock if needed + if DECOMPOSITIONS_LOCK is None: + import threading + + DECOMPOSITIONS_LOCK = threading.Lock() + + with DECOMPOSITIONS_LOCK: + if DECOMPOSITIONS_LOADED: + return + + import os + + if not (os.environ.get("PYTORCH_JIT", "1") == "1" and __debug__): + DECOMPOSITIONS_LOADED = True + return + + # use an alternate way to register an operator into the decomposition table + # _register_jit_decomposition doesn't work for some operators, e.g. addr, + # because the Tensor types generated cannot be unioned by torchscript + # decomp should be type OpOverload + VMAP_DECOMPOSITIONS_LIB = torch.library.Library( + "aten", "IMPL", "FuncTorchBatched" + ) + + from torch._decomp import decomposition_table + + def _register_python_decomposition_vmap(decomp): + if decomp in decomposition_table: + VMAP_DECOMPOSITIONS_LIB.impl(decomp, decomposition_table[decomp]) + else: + raise RuntimeError(f"could not find decomposition for {decomp}") + + _register_python_decomposition_vmap(torch.ops.aten.mse_loss_backward.default) + _register_python_decomposition_vmap( + torch.ops.aten.smooth_l1_loss_backward.default + ) + _register_python_decomposition_vmap(torch.ops.aten.huber_loss_backward.default) + _register_python_decomposition_vmap(torch.ops.aten.nll_loss_forward.default) + _register_python_decomposition_vmap(torch.ops.aten.nll_loss2d_forward.default) + _register_python_decomposition_vmap(torch.ops.aten.nll_loss_backward.default) + _register_python_decomposition_vmap(torch.ops.aten.nll_loss2d_backward.default) + _register_python_decomposition_vmap(torch.ops.aten.addr.default) + + DECOMPOSITIONS_LOADED = True diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/pyfunctorch.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/pyfunctorch.py new file mode 100644 index 0000000000000000000000000000000000000000..b76cd191c3cc9480225b3017a130e32d2760b59c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/pyfunctorch.py @@ -0,0 +1,316 @@ +# mypy: allow-untyped-defs +import contextlib +from abc import ABC, abstractmethod +from functools import cached_property +from typing import Any + +import torch +import torch.utils._pytree as pytree +from torch._C._functorch import ( + CFunctionalizeInterpreterPtr, + CGradInterpreterPtr, + CInterpreter, + CJvpInterpreterPtr, + CVmapInterpreterPtr, + pop_dynamic_layer_stack, + push_dynamic_layer_stack, + RandomnessType, + TransformType, +) +from torch.autograd.forward_ad import _set_fwd_grad_enabled + + +""" +This file contains the functorch integration with PyDispatcher. + +PyDispatcher does not understand functorch's DynamicLayerStack dispatching +logic because it is entirely implemented in C++ in the fallbacks for two +dispatch keys, FuncTorchDynamicLayer{Front, Back}Mode (PyDispatcher is unable +to directly reuse C++ boxed fallbacks). + +Instead of trying to hammer PyDispatcher into understanding those fallbacks, +we re-implement the logic of peeking the top of the stack for an interpreter, +selecting the interpreter to dispatch on, etc, in Python. This leads to a +simpler design. + +The main difference between C++ functorch and PyDispatcher's functorch logic +is that: +- C++ functorch needs to manually tweak dispatch keys to ping-pong between + DynamicLayerFrontMode and DynamicLayerBackMode. +- PyDispatcher's functorch logic pops an Interpreter from the top of the stack + and asks it to execute the rule associated with the Interpreter. + +In C++ we do the ping-pong because e.g. vmap rules are associated with the +batched DispatchKey, but in PyDispatcher we are able to avoid this by asking +the user to register a batching rule directly to a transform that an +interpreter then invokes. +""" + + +# FuncTorchInterpreter is the Python version of Interpreter (recall that +# the DynamicLayerStack is a stack of interpreters). +# It is a wrapper around the actual C++ Interpreter object. +# +# Keep the methods in sync with aten/src/ATen/functorch/Interpreter.h +class FuncTorchInterpreter(ABC): + def __init__(self, cptr: Any): + self._cptr = cptr + + # Process an operation. eg for vmap, this is invoking a batching rule. + # Conceptually this is analogous to Interpreter::process in C++ + @abstractmethod + def process(self, op, args, kwargs): + pass + + # lower an operation from this Interpreter to the next Interpreter on the stack. + # Concretely, this involves temporarily popping the current Interpreter. + # Conceptually this is analogous to Interpreter::sendToNextInterpreter in C++ + def lower(self): + return temporarily_pop_interpreter_stack() + + def level(self): + return self._cptr.level() + + def key(self): + return self._cptr.key() + + def get_state(self): + raise NotImplementedError + + def check_state(self, state): + return state == self.get_state() + + def __getstate__(self): + state = self.__dict__.copy() + state.pop("_cptr", None) + return state + + +@contextlib.contextmanager +def temporarily_pop_interpreter_stack(): + try: + saved = pop_dynamic_layer_stack() + yield + finally: + push_dynamic_layer_stack(saved) + + +@contextlib.contextmanager +def temporarily_clear_interpreter_stack(): + stack = [] + try: + while torch._C._functorch.peek_interpreter_stack() is not None: + stack.append(pop_dynamic_layer_stack()) + yield list(stack) + finally: + while stack: + push_dynamic_layer_stack(stack.pop()) + + +@contextlib.contextmanager +def temporarily_restore_interpreter_stack(stack): + pushed = [] + try: + for s in reversed(stack): + push_dynamic_layer_stack(s) + pushed.append(s) + yield + finally: + for _ in reversed(pushed): + # TODO: would be nice to assert that the layers are the same, but + # Python object identity is not preserved + pop_dynamic_layer_stack() + + +class VmapInterpreter(FuncTorchInterpreter): + def __init__(self, cdata: CInterpreter): + assert cdata.key() == TransformType.Vmap + # NOTE: [Interpreter cdata vs cptr] + # cdata is a generic CInterpreter. We wrap it in a CVmapInterpreterPtr + # so that we can access methods specific to the vmap interpreter + self._cdata = cdata + + @cached_property + # pyrefly: ignore [bad-override] + def _cptr(self): + return CVmapInterpreterPtr(self._cdata) + + def process(self, op, args, kwargs): + kernel = op.functorch_table[TransformType.Vmap] + return kernel(self, *args, **kwargs) + + def batch_size(self): + return self._cptr.batchSize() + + def randomness(self): + typ = self._cptr.randomness() + if typ == RandomnessType.Error: + return "error" + elif typ == RandomnessType.Same: + return "same" + elif typ == RandomnessType.Different: + return "different" + raise RuntimeError(f"Unknown RandomnessType: {typ}") + + def get_state(self): + return (self.key().name, self.level(), self.randomness()) + + +@contextlib.contextmanager +def nested(*contexts): + with contextlib.ExitStack() as stack: + for ctx in contexts: + stack.enter_context(ctx) + yield contexts + + +class GradInterpreter(FuncTorchInterpreter): + def __init__(self, cdata: CInterpreter): + assert cdata.key() == TransformType.Grad + # See NOTE: [Interpreter cdata vs cptr] + self._cdata = cdata + + @cached_property + # pyrefly: ignore [bad-override] + def _cptr(self): + return CGradInterpreterPtr(self._cdata) + + def lift(self, args, kwargs): + args, kwargs = pytree.tree_map_only( + torch.Tensor, self._cptr.lift, [args, kwargs] + ) + return args, kwargs + + def process(self, op, args, kwargs): + kernel = op.functorch_table[TransformType.Grad] + args, kwargs = self.lift(args, kwargs) + return kernel(self, *args, **kwargs) + + # GradInterpreter has custom lower because of the no_grad interaction + # See NOTE [grad and vjp interaction with no_grad] + # This logic is mirrored from C++ GradInterpreterPtr::sendToNextInterpreter + def lower(self): + prev_grad_mode = self.prev_grad_mode() + if not prev_grad_mode: + return nested(torch.no_grad(), super().lower()) + return super().lower() + + def prev_grad_mode(self): + return self._cptr.prevGradMode() + + def get_state(self): + return (self.key().name, self.level(), self.prev_grad_mode()) + + +class JvpInterpreter(FuncTorchInterpreter): + def __init__(self, cdata: CInterpreter): + assert cdata.key() == TransformType.Jvp + # See NOTE: [Interpreter cdata vs cptr] + self._cdata = cdata + + @cached_property + # pyrefly: ignore [bad-override] + def _cptr(self): + return CJvpInterpreterPtr(self._cdata) + + def lift(self, args, kwargs): + args, kwargs = pytree.tree_map_only( + torch.Tensor, self._cptr.lift, [args, kwargs] + ) + return args, kwargs + + def process(self, op, args, kwargs): + kernel = op.functorch_table[TransformType.Jvp] + args, kwargs = self.lift(args, kwargs) + return kernel(self, *args, **kwargs) + + # Jvp has custom lower because of the no_fwd_grad interaction + # See NOTE [grad and vjp interaction with no_grad] for related info. + # This logic is mirrored from C++ JvpInterpreterPtr::sendToNextInterpreter + def lower(self): + prev_fwd_grad_mode = self.prev_fwd_grad_mode() + if not prev_fwd_grad_mode: + return nested(_set_fwd_grad_enabled(False), super().lower()) + return super().lower() + + def prev_fwd_grad_mode(self): + return self._cptr.prevFwdGradMode() + + def get_state(self): + return (self.key().name, self.level(), self.prev_fwd_grad_mode()) + + +class FunctionalizeInterpreter(FuncTorchInterpreter): + def __init__(self, cdata: CInterpreter): + assert cdata.key() == TransformType.Functionalize + self._cdata = cdata + + @cached_property + # pyrefly: ignore [bad-override] + def _cptr(self): + return CFunctionalizeInterpreterPtr(self._cdata) + + def process(self, op, args, kwargs): + kernel = op.functorch_table[TransformType.Functionalize] + return kernel(self, *args, **kwargs) + + def functionalize_add_back_views(self): + return self._cptr.functionalizeAddBackViews() + + def get_state(self): + return (self.key().name, self.level()) + + +def coerce_cinterpreter(cinterpreter: CInterpreter) -> FuncTorchInterpreter: + key = cinterpreter.key() + if key == TransformType.Grad: + return GradInterpreter(cinterpreter) + if key == TransformType.Vmap: + return VmapInterpreter(cinterpreter) + if key == TransformType.Jvp: + return JvpInterpreter(cinterpreter) + if key == TransformType.Functionalize: + return FunctionalizeInterpreter(cinterpreter) + raise RuntimeError(f"NYI: PyDispatcher has not implemented support for {key}") + + +def retrieve_current_functorch_interpreter() -> FuncTorchInterpreter: + interpreter = torch._C._functorch.peek_interpreter_stack() + assert interpreter is not None + return coerce_cinterpreter(interpreter) + + +def retrieve_all_functorch_interpreters() -> list[FuncTorchInterpreter]: + cis = torch._C._functorch.get_interpreter_stack() + if cis is None: + return [] + return [coerce_cinterpreter(ci) for ci in cis] + + +def compare_functorch_state(states: list[tuple[Any, ...]]) -> bool: + # There are four possible cases covered here: + # 1. Current stack empty AND stack when generated not empty -> Invalidate + # 2. Current stack not empty AND stack when generated empty -> Invalidate + # 3. Current stack and generated stack empty -> Valid FX graph + # 4. Current stack and generated stack not empty -> Valid if both states match + peek = torch._C._functorch.peek_interpreter_stack() + if (peek is None and len(states) != 0) or (peek is not None and len(states) == 0): + return False + + cis = retrieve_all_functorch_interpreters() + return len(cis) == len(states) and all( + ci.check_state(state) for ci, state in zip(cis, states) + ) + + +def dispatch_functorch(op, args, kwargs): + interpreter = retrieve_current_functorch_interpreter() + # In traditional PyTorch operators, DispatchKey::FuncTorchTensorWrapper's + # unwrap_dead_tensors fallback handles unwrapping dead tensor wrappers. + # PyDispatcher sidesteps the PyTorch dispatcher when dealing with functorch + # transforms, so we manually unwrap the dead tensors here. + # This logic won't need to exist when we have mode-only functorch. + args, kwargs = pytree.tree_map_only( + torch.Tensor, torch._C._functorch.unwrap_if_dead, (args, kwargs) + ) + return interpreter.process(op, args, kwargs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/python_key.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/python_key.py new file mode 100644 index 0000000000000000000000000000000000000000..557334f68928a057a6a9e036c904c8c0bd3231c1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/python_key.py @@ -0,0 +1,15 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +__all__ = ["make_fx", "dispatch_trace", "PythonKeyTracer", "pythonkey_decompose"] +from torch.fx.experimental.proxy_tensor import ( + decompose, + dispatch_trace, + make_fx, + PythonKeyTracer, +) + + +pythonkey_decompose = decompose diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/pytree_hacks.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/pytree_hacks.py new file mode 100644 index 0000000000000000000000000000000000000000..96dea7ad100705ae53139aa5ae729fd2206182af --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/pytree_hacks.py @@ -0,0 +1,23 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import warnings + +# TODO: remove this file when the migration of the pytree utility is done +from torch.utils._pytree import tree_map_, treespec_pprint + + +__all__ = ["tree_map_", "treespec_pprint"] + + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "`torch._functorch.pytree_hacks` is deprecated and will be removed in a future release. " + "Please `use torch.utils._pytree` instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/top_operators_github_usage.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/top_operators_github_usage.py new file mode 100644 index 0000000000000000000000000000000000000000..171c6fc6c1e018d4809b9fe7a4ab2152701a11ea --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/top_operators_github_usage.py @@ -0,0 +1,630 @@ +# mypy: ignore-errors + +""" +From https://docs.google.com/spreadsheets/d/12R3nCOLskxPYjjiNkdqy4OdQ65eQp_htebXGODsjSeA/edit#gid=0 +Try to keep this list in sync with that. +""" + +import operator + + +top_torch = [ + ("t", 6837449), + ("tensor", 585786), + ("mode", 462182), + ("cat", 394818), + ("max", 368038), + ("zeros", 329495), + ("load", 327756), + ("no_grad", 294694), + ("save", 265130), + ("from_numpy", 243063), + ("manual_seed", 165044), + ("ones", 153696), + ("randn", 150796), + ("stack", 133358), + ("sum", 130772), + ("arange", 98087), + ("rand", 94715), + ("mean", 88546), + ("exp", 73883), + ("zeros_like", 72831), + ("min", 72248), + ("sigmoid", 66798), + ("log", 62135), + ("matmul", 47811), + ("clamp", 45304), + ("sqrt", 44911), + ("abs", 43535), + ("tanh", 42793), + ("empty", 40311), + ("argmax", 38435), + ("bmm", 33984), + ("pow", 33571), + ("norm", 31125), + ("mm", 30995), + ("is_tensor", 29546), + ("ones_like", 29512), + ("nonzero", 28681), + ("full", 28373), + ("unsqueeze", 27911), + ("where", 26585), + ("randperm", 26450), + ("eye", 24342), + ("mul", 23236), + ("topk", 22537), + ("as_tensor", 21967), + ("sort", 21412), + ("squeeze", 20863), + ("randint", 20771), + ("linspace", 20041), + ("add", 19201), + ("transpose", 18663), + ("split", 18325), + ("gather", 17904), + ("set_grad_enabled", 16013), + ("sin", 15669), + ("cos", 15562), + ("div", 15513), + ("index_select", 14866), + ("multinomial", 14331), + ("flatten", 14267), + ("isnan", 14170), + ("randn_like", 13096), + ("eq", 12680), + ("einsum", 12480), + ("round", 12367), + ("floor", 11628), + ("allclose", 11000), + ("reshape", 10605), + ("diag", 10167), + ("chunk", 9581), + ("std", 9379), + ("set_default_tensor_type", 9281), + ("triu", 8559), + ("meshgrid", 8292), + ("set_num_threads", 8126), + ("unique", 7964), + ("full_like", 7780), + ("tril", 7538), + ("dot", 7275), + ("sign", 6943), + ("equal", 6916), + ("normal", 6750), + ("cumsum", 6556), + ("dist", 6058), + ("isfinite", 6030), + ("gt", 5935), + ("set_printoptions", 5888), + ("range", 5491), + ("empty_like", 5351), + ("flip", 5342), + ("masked_select", 5341), + ("bernoulli", 5262), + ("atan", 5253), + ("var", 5247), + ("prod", 5200), + ("erf", 5088), + ("inverse", 5072), + ("addmm", 4854), + ("logsumexp", 4582), + ("fft", 4436), + ("lt", 4421), + ("log2", 4316), + ("enable_grad", 4238), + ("rand_like", 4187), + ("argsort", 3972), + ("seed", 3932), + ("mv", 3547), + ("ger", 3309), + ("ge", 3248), + ("atan2", 3210), + ("ceil", 3202), + ("ne", 3075), + ("bincount", 3063), + ("acos", 3055), + ("rsqrt", 3031), + ("svd", 3029), + ("numel", 3003), + ("log1p", 2840), + ("unbind", 2808), + ("le", 2714), + ("isinf", 2707), + ("cross", 2646), + ("set_default_dtype", 2536), + ("argmin", 2535), + ("sparse_coo_tensor", 2489), + ("log10", 2304), + ("kthvalue", 2192), + ("set_rng_state", 2158), + ("get_rng_state", 1996), + ("get_default_dtype", 1879), + ("det", 1868), + ("qr", 1864), + ("histc", 1852), + ("symeig", 1832), + ("trace", 1801), + ("median", 1795), + ("addcmul", 1751), + ("remainder", 1717), + ("baddbmm", 1693), + ("lgamma", 1665), + ("repeat_interleave", 1598), + ("fmod", 1576), + ("reciprocal", 1575), + ("tan", 1560), + ("initial_seed", 1532), + ("take", 1529), + ("stft", 1487), + ("get_num_threads", 1477), + ("real", 1459), + ("cholesky", 1406), + ("quantize_per_tensor", 1392), + ("diag_embed", 1364), + ("lerp", 1363), + ("asin", 1345), + ("eig", 1333), + ("trunc", 1290), + ("diagonal", 1287), + ("cosh", 1279), + ("rfft", 1269), + ("cumprod", 1260), + ("addr", 1211), + ("roll", 1198), + ("narrow", 1188), + ("digamma", 1172), + ("square", 1163), + ("sinh", 1131), + ("logspace", 1084), + ("broadcast_tensors", 1070), + ("irfft", 1013), + ("frac", 997), + ("hann_window", 994), + ("solve", 989), + ("logdet", 977), + ("expm1", 968), + ("cdist", 946), + ("addmv", 903), + ("randint_like", 888), + ("tensordot", 888), + ("ifft", 877), + ("true_divide", 854), + ("erfinv", 830), + ("addcdiv", 819), + ("addbmm", 813), + ("renorm", 781), + ("pinverse", 753), + ("isclose", 740), + ("erfc", 729), + ("is_storage", 725), + ("triangular_solve", 723), + ("rot90", 709), + ("logical_not", 686), + ("geqrf", 681), + ("slogdet", 677), + ("lu", 665), + ("hamming_window", 659), + ("orgqr", 651), + ("ormqr", 622), + ("is_floating_point", 602), + ("diagflat", 562), + ("cholesky_solve", 559), + ("tril_indices", 552), + ("chain_matmul", 551), + ("triu_indices", 548), + ("angle", 522), + ("poisson", 505), + ("matrix_power", 485), + ("unique_consecutive", 471), + ("quantize_per_channel", 465), + ("std_mean", 458), + ("bartlett_window", 447), + ("var_mean", 428), + ("lstsq", 421), + ("logical_and", 419), + ("mvlgamma", 411), + ("blackman_window", 400), + ("bitwise_not", 395), + ("cholesky_inverse", 388), + ("as_strided", 384), + ("floor_divide", 353), + ("cartesian_prod", 321), + ("lu_solve", 317), + ("set_flush_denormal", 310), + ("empty_strided", 283), + ("logical_xor", 282), + ("polygamma", 282), + ("logical_or", 280), + ("set_num_interop_threads", 278), + ("combinations", 274), + ("trapz", 270), + ("matrix_rank", 260), + ("lu_unpack", 255), + ("result_type", 244), + ("conj", 231), + ("cummax", 230), + ("lobpcg", 229), + ("bitwise_xor", 217), + ("promote_types", 213), + ("get_num_interop_threads", 211), + ("cummin", 205), + ("bitwise_and", 198), + ("dequantize", 192), + ("bitwise_or", 191), + ("imag", 191), + ("can_cast", 184), + ("istft", 180), + ("compiled_with_cxx11_abi", 159), + ("is_complex", 151), + ("block_diag", 136), + ("pca_lowrank", 124), + ("absolute", 122), + ("svd_lowrank", 108), + ("neg", 2), +] + +top_nn_functional = [ + ("nn.functional.softmax", 10522), + ("nn.functional.relu", 8572), + ("nn.functional.interpolate", 7277), + ("nn.functional.pad", 5207), + ("nn.functional.log_softmax", 4699), + ("nn.functional.normalize", 2338), + ("nn.functional.cross_entropy", 2083), + ("nn.functional.grid_sample", 1970), + ("nn.functional.one_hot", 1967), + ("nn.functional.mse_loss", 1920), + ("nn.functional.conv2d", 1593), + ("nn.functional.dropout", 1516), + ("nn.functional.softplus", 1385), + ("nn.functional.sigmoid", 1128), + ("nn.functional.linear", 1036), + ("nn.functional.gelu", 930), + ("nn.functional.avg_pool2d", 899), + ("nn.functional.max_pool2d", 876), + ("nn.functional.nll_loss", 863), + ("nn.functional.embedding", 737), + ("nn.functional.tanh", 664), + ("nn.functional.leaky_relu", 640), + ("nn.functional.adaptive_avg_pool2d", 633), + ("nn.functional.cosine_similarity", 627), + ("nn.functional.unfold", 609), + ("nn.functional.conv1d", 596), + ("nn.functional.binary_cross_entropy_with_logits", 591), + ("nn.functional.l1_loss", 571), + ("nn.functional.binary_cross_entropy", 492), + ("nn.functional.elu", 416), + ("nn.functional.batch_norm", 413), + ("nn.functional.upsample", 413), + ("nn.functional.fold", 305), + ("nn.functional.affine_grid", 298), + ("nn.functional.max_pool1d", 297), + ("nn.functional.torch", 294), + ("nn.functional.threshold", 263), + ("nn.functional.smooth_l1_loss", 262), + ("nn.functional.pairwise_distance", 253), + ("nn.functional.logsigmoid", 243), + ("nn.functional.adaptive_max_pool2d", 235), + ("nn.functional.relu6", 213), + ("nn.functional.pixel_shuffle", 209), + ("nn.functional.avg_pool3d", 203), + ("nn.functional.bilinear", 203), + ("nn.functional.conv_transpose2d", 201), + ("nn.functional.gumbel_softmax", 197), + ("nn.functional.max_unpool2d", 196), + ("nn.functional.kl_div", 191), + ("nn.functional.hardtanh", 189), + ("nn.functional.ctc_loss", 185), + ("nn.functional.layer_norm", 178), + ("nn.functional.conv3d", 172), + ("nn.functional.max_unpool3d", 167), + ("nn.functional.hardshrink", 165), + ("nn.functional.hardswish", 156), + ("nn.functional.selu", 156), + ("nn.functional.glu", 155), + ("nn.functional.assert_int_or_pair", 150), + ("nn.functional.hardsigmoid", 146), + ("nn.functional.upsample_bilinear", 146), + ("nn.functional.max_pool3d", 140), + ("nn.functional.adaptive_avg_pool3d", 139), + ("nn.functional.instance_norm", 124), + ("nn.functional.embedding_bag", 122), + ("nn.functional.upsample_nearest", 110), + ("nn.functional.avg_pool1d", 105), + ("nn.functional.prelu", 102), + ("nn.functional.celu", 92), + ("nn.functional.dropout2d", 86), + ("nn.functional.hinge_embedding_loss", 82), + ("nn.functional.softsign", 81), + ("nn.functional.max_unpool1d", 74), + ("nn.functional.silu", 74), + ("nn.functional.softshrink", 70), + ("nn.functional.leaky_relu_", 68), + ("nn.functional.softmin", 67), + ("nn.functional.channel_shuffle", 66), + ("nn.functional.multilabel_margin_loss", 66), + ("nn.functional.dropout3d", 65), + ("nn.functional.multi_margin_loss", 65), + ("nn.functional.lp_pool2d", 64), + ("nn.functional.conv_transpose1d", 62), + ("nn.functional.triplet_margin_loss", 62), + ("nn.functional.tanhshrink", 61), + ("nn.functional.adaptive_max_pool1d", 59), + ("nn.functional.cosine_embedding_loss", 58), + ("nn.functional.multi_head_attention_forward", 58), + ("nn.functional.max_pool1d_with_indices", 53), + ("nn.functional.poisson_nll_loss", 53), + ("nn.functional.margin_ranking_loss", 52), + ("nn.functional.soft_margin_loss", 52), + ("nn.functional.adaptive_max_pool3d", 51), + ("nn.functional.group_norm", 51), + ("nn.functional.local_response_norm", 51), + ("nn.functional.multilabel_soft_margin_loss", 51), + ("nn.functional.relu_", 50), + ("nn.functional.alpha_dropout", 49), + ("nn.functional.feature_alpha_dropout", 49), + ("nn.functional.lp_pool1d", 49), + ("nn.functional.adaptive_max_pool1d_with_indices", 48), + ("nn.functional.adaptive_max_pool2d_with_indices", 48), + ("nn.functional.adaptive_max_pool3d_with_indices", 48), + ("nn.functional.fractional_max_pool2d", 48), + ("nn.functional.fractional_max_pool2d_with_indices", 48), + ("nn.functional.fractional_max_pool3d", 48), + ("nn.functional.fractional_max_pool3d_with_indices", 48), + ("nn.functional.max_pool2d_with_indices", 48), + ("nn.functional.max_pool3d_with_indices", 48), + ("nn.functional.handle_torch_function", 47), + ("nn.functional.has_torch_function", 47), + ("nn.functional.adaptive_avg_pool1d", 43), + ("nn.functional.pdist", 43), + ("nn.functional.rrelu_", 37), + ("nn.functional.elu_", 34), + ("nn.functional.boolean_dispatch", 33), + ("nn.functional.hardtanh_", 26), + ("nn.functional.triplet_margin_with_distance_loss", 23), + ("nn.functional.selu_", 20), + ("nn.functional.pixel_unshuffle", 19), + ("nn.functional.conv_transpose3d", 18), + ("nn.functional.gaussian_nll_loss", 15), + ("nn.functional.has_torch_function_unary", 15), + ("nn.functional.has_torch_function_variadic", 15), + ("nn.functional.celu_", 13), + ("nn.functional.huber_loss", 7), + ("nn.functional.mish", 4), + ("nn.functional.threshold_", 3), + ("nn.functional.grad", 2), + ("nn.functional.conv_tbc", 1), + ("nn.functional.math", 1), +] + +top_nn_module = [ + ("nn.Module", 927129, None), + ("nn.Linear", 530688, "nn.functional.linear"), + ("nn.Sequential", 384968, None), + ("nn.Conv2d", 383320, "nn.functional.conv2d"), + ("nn.ReLU", 318877, "nn.functional.relu"), + ("nn.BatchNorm2d", 233265, "nn.functional.batch_norm"), + ("nn.Dropout", 179268, "nn.functional.dropout"), + ("nn.ModuleList", 171225, None), + ("nn.Parameter", 153291, None), + ("nn.CrossEntropyLoss", 152696, "nn.functional.cross_entropy"), + ("nn.MaxPool2d", 138619, "nn.functional.max_pool2d"), + ("nn.Embedding", 111844, "nn.functional.embedding"), + ("nn.DataParallel", 104238, None), + ("nn.MSELoss", 82954, "nn.functional.mse_loss"), + ("nn.Sigmoid", 75810, "nn.functional.sigmoid"), + ("nn.LeakyReLU", 65632, "nn.functional.leaky_relu"), + ("nn.BatchNorm1d", 65374, "nn.functional.batch_norm"), + ("nn.Softmax", 65114, "nn.functional.softmax"), + ("nn.Tanh", 59445, "nn.functional.tanh"), + ("nn.AdaptiveAvgPool2d", 59071, "nn.functional.adaptive_avg_pool2d"), + ("nn.AvgPool2d", 58377, "nn.functional.avg_pool2d"), + ("nn.ConvTranspose2d", 57524, "nn.functional.conv_transpose2d"), + ("nn.LSTM", 57411, None), + ("nn.Conv1d", 41108, "nn.functional.conv1d"), + ("nn.LayerNorm", 36089, "nn.functional.layer_norm"), + ("nn.BCELoss", 34005, "nn.functional.binary_cross_entropy"), + ("nn.Upsample", 32527, "nn.functional.interpolate"), + ("nn.BCEWithLogitsLoss", 29944, "nn.functional.binary_cross_entropy_with_logits"), + ("nn.GRU", 25421, None), + ("nn.Dropout2d", 23512, "nn.functional.dropout2d"), + ("nn.LogSoftmax", 22897, "nn.functional.log_softmax"), + ("nn.L1Loss", 22778, "nn.functional.l1_loss"), + ("nn.GroupNorm", 22183, "nn.functional.group_norm"), + ("nn.NLLLoss", 21751, "nn.functional.nll_loss"), + ("nn.Conv3d", 20874, "nn.functional.conv3d"), + ("nn.Identity", 17911, None), + ("nn.InstanceNorm2d", 16426, "nn.functional.instance_norm"), + ("nn.BatchNorm3d", 16378, "nn.functional.batch_norm"), + ("nn.PReLU", 13472, "nn.functional.prelu"), + ("nn.ReLU6", 12622, "nn.functional.relu6"), + ("nn.ELU", 12508, "nn.functional.elu"), + ("nn.LSTMCell", 10885, None), + ("nn.Flatten", 10384, "torch.flatten"), + ("nn.ModuleDict", 10255, None), + ("nn.ReflectionPad2d", 9954, "nn.functional.pad"), + ("nn.MaxPool3d", 9526, "nn.functional.max_pool3d"), + ("nn.MaxPool1d", 9154, "nn.functional.max_pool1d"), + ("nn.RNN", 9154, None), + ("nn.ZeroPad2d", 8847, "nn.functional.pad"), + ("nn.ParameterList", 7702, None), + ("nn.SyncBatchNorm", 6814, None), + ("nn.PixelShuffle", 6571, "nn.functional.pixel_shuffle"), + ("nn.SmoothL1Loss", 6517, "nn.functional.smooth_l1_loss"), + ("nn.Hardswish", 6458, "nn.functional.hardswish"), + ("nn.AdaptiveMaxPool2d", 6071, "nn.functional.adaptive_max_pool2d"), + ("nn.SELU", 6043, "nn.functional.selu"), + ("nn.ConvTranspose3d", 6039, "nn.functional.conv_transpose3d"), + ("nn.GRUCell", 5840, None), + ("nn.ReplicationPad2d", 5600, "nn.functional.pad"), + ("nn.KLDivLoss", 5541, "nn.functional.kl_div"), + ("nn.ConvTranspose1d", 5183, "nn.functional.conv_transpose1d"), + ("nn.Softplus", 5120, "nn.functional.softplus"), + ("nn.SiLU", 4895, "nn.functional.silu"), + ("nn.AvgPool3d", 4523, "nn.functional.avg_pool3d"), + ("nn.CosineSimilarity", 4058, "nn.functional.cosine_similarity"), + ("nn.GELU", 3932, "nn.functional.gelu"), + ("nn.UpsamplingBilinear2d", 3673, "nn.functional.interpolate"), + ("nn.InstanceNorm1d", 3658, "nn.functional.instance_norm"), + ("nn.Transformer", 3604, None), + ("nn.MultiheadAttention", 3435, "nn.functional.multi_head_attention_forward"), + ("nn.AvgPool1d", 3195, "nn.functional.avg_pool1d"), + ("nn.Dropout3d", 2964, "nn.functional.dropout3d"), + ("nn.AdaptiveAvgPool3d", 2915, "nn.functional.adaptive_avg_pool3d"), + ("nn.InstanceNorm3d", 2893, "nn.functional.instance_norm"), + ("nn.Hardtanh", 2613, "nn.functional.hardtanh"), + ("nn.MarginRankingLoss", 2568, "nn.functional.margin_ranking_loss"), + ("nn.GLU", 2526, "nn.functional.glu"), + ("nn.AdaptiveAvgPool1d", 2481, "nn.functional.adaptive_avg_pool1d"), + ("nn.EmbeddingBag", 2344, "nn.functional.embedding_bag"), + ("nn.TransformerEncoderLayer", 2292, None), + ("nn.TransformerEncoder", 2091, None), + ("nn.MaxUnpool2d", 2031, "nn.functional.max_unpool2d"), + ("nn.UpsamplingNearest2d", 2004, "nn.functional.interpolate"), + ("nn.ConstantPad1d", 1904, "nn.functional.pad"), + ("nn.ConstantPad2d", 1791, "nn.functional.pad"), + ("nn.CTCLoss", 1789, "nn.functional.ctc_loss"), + ("nn.AdaptiveMaxPool1d", 1713, "nn.functional.adaptive_max_pool1d"), + ("nn.AdaptiveLogSoftmaxWithLoss", 1665, None), + ("nn.Bilinear", 1664, "nn.functional.bilinear"), + ("nn.RNNCell", 1653, None), + ("nn.MultiLabelSoftMarginLoss", 1624, "nn.functional.multilabel_soft_margin_loss"), + ("nn.Unfold", 1452, "nn.functional.unfold"), + ("nn.RReLU", 1431, "nn.functional.rrelu"), + ("nn.CosineEmbeddingLoss", 1357, "nn.functional.cosine_embedding_loss"), + ("nn.LocalResponseNorm", 1331, "nn.functional.local_response_norm"), + ("nn.Softmax2d", 1300, "nn.functional.softmax"), + ("nn.PairwiseDistance", 1241, "nn.functional.pairwise_distance"), + ("nn.LogSigmoid", 1235, "nn.functional.logsigmoid"), + ("nn.TripletMarginLoss", 1230, "nn.functional.triplet_margin_loss"), + ("nn.RNNBase", 1133, None), + ("nn.Threshold", 1043, "nn.functional.threshold"), + ("nn.AdaptiveMaxPool3d", 1025, "nn.functional.adaptive_max_pool3d"), + ("nn.CELU", 1018, "nn.functional.celu"), + ("nn.NLLLoss2d", 966, "nn.functional.nll_loss"), + ("nn.Softsign", 877, "nn.functional.softsign"), + ("nn.ReplicationPad1d", 862, "nn.functional.pad"), + ("nn.SoftMarginLoss", 856, "nn.functional.soft_margin_loss"), + ("nn.ParameterDict", 742, None), + ("nn.ReflectionPad1d", 731, "nn.functional.pad"), + ("nn.Softshrink", 713, "nn.functional.softshrink"), + ("nn.AlphaDropout", 710, "nn.functional.alpha_dropout"), + ("nn.Tanhshrink", 681, "nn.functional.tanhshrink"), + ("nn.PoissonNLLLoss", 676, "nn.functional.poisson_nll_loss"), + ("nn.MaxUnpool3d", 660, "nn.functional.max_unpool3d"), + ("nn.Fold", 630, "nn.functional.fold"), + ("nn.MultiMarginLoss", 622, "nn.functional.multi_margin_loss"), + ("nn.TransformerDecoderLayer", 614, None), + ("nn.TransformerDecoder", 607, None), + ("nn.Hardshrink", 592, "nn.functional.hardshrink"), + ("nn.ConstantPad3d", 582, "nn.functional.pad"), + ("nn.MultiLabelMarginLoss", 580, "nn.functional.multilabel_margin_loss"), + ("nn.LPPool2d", 550, "nn.functional.lp_pool2d"), + ("nn.Softmin", 537, "nn.functional.softmin"), + ("nn.MaxUnpool1d", 518, "nn.functional.max_unpool1d"), + ("nn.FractionalMaxPool2d", 484, "nn.functional.fractional_max_pool2d"), + ("nn.Hardsigmoid", 477, "nn.functional.hardsigmoid"), + ("nn.ReplicationPad3d", 470, "nn.functional.pad"), + ("nn.HingeEmbeddingLoss", 442, "nn.functional.hinge_embedding_loss"), + ("nn.LPPool1d", 386, "nn.functional.lp_pool1d"), + ("nn.FractionalMaxPool3d", 252, "nn.functional.fractional_max_pool3d"), + ("nn.Container", 217, None), + ("nn.Unflatten", 206, "nn.functional.unflatten"), + ("nn.FeatureAlphaDropout", 136, "nn.functional.feature_alpha_dropout"), + ( + "nn.TripletMarginWithDistanceLoss", + 107, + "nn.functional.triplet_margin_with_distance_loss", + ), + ("nn.ChannelShuffle", 90, "nn.functional.channel_shuffle"), + ("nn.RNNCellBase", 88, None), + ("nn.LazyLinear", 81, "nn.functional.linear"), + ("nn.UninitializedParameter", 60, None), + ("nn.CrossMapLRN2d", 59, None), + ("nn.GaussianNLLLoss", 55, "nn.functional.gaussian_nll_loss"), + ("nn.PixelUnshuffle", 45, "nn.functional.pixel_unshuffle"), + ("nn.Mish", 31, "nn.functional.mish"), + ("nn.ReflectionPad3d", 22, "nn.functional.pad"), + ("nn.HuberLoss", 18, "nn.functional.huber_loss"), + ("nn.LazyConv2d", 15, None), + ("nn.LazyConv1d", 9, None), + ("nn.LazyConv3d", 8, None), + ("nn.LazyConvTranspose1d", 8, None), + ("nn.LazyConvTranspose2d", 8, None), + ("nn.LazyConvTranspose3d", 8, None), + ("nn.LazyBatchNorm1d", 3, None), + ("nn.LazyBatchNorm2d", 3, None), + ("nn.LazyBatchNorm3d", 3, None), + ("nn.UninitializedBuffer", 3, None), +] + +# No rankings because these are a little hard to get rankings for +method_only_ops = [ + "bfloat16", + "bool", + "byte", + "char", + "contiguous", + "cpu", + "cuda", + "detach", + "double", + "expand", + "expand_as", + "float", + "get_device", + "half", + "hardshrink", + "index_add", + "index_copy", + "index_fill", + "index_put", + "int", + "is_contiguous", + "is_pinned", + "is_set_to", + "is_shared", + "is_signed", + "item", + "long", + "masked_scatter", + "masked_fill", + "narrow_copy", + "numpy", + "pin_memory", + "repeat", + "reshape_as", + "select", + "short", + "storage_offset", + "sum_to_size", + "to", + "to_mkldnn", + "tolist", + "type", + "type_as", + "unfold", + "view", + "view_as", +] + + +def get_nn_functional_top_list(): + top_nn_functional_ = dict(top_nn_functional) + for _, count, functional_name in top_nn_module: + if functional_name is None: + continue + if functional_name == "torch.flatten": + continue + if functional_name not in top_nn_functional_: + top_nn_functional_[functional_name] = count + else: + top_nn_functional_[functional_name] += count + + top_nn_functional_ = list(top_nn_functional_.items()) + top_nn_functional_.sort(key=operator.itemgetter(1), reverse=True) + return top_nn_functional_ + + +usage_count = dict(get_nn_functional_top_list()) +usage_count.update(top_torch) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a2790a0fdd743171270dfd9e7826bb2f8dac6842 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/utils.py @@ -0,0 +1,40 @@ +import contextlib +from collections.abc import Generator +from typing import Any, Union + +import torch +from torch._C._functorch import ( + get_single_level_autograd_function_allowed, + set_single_level_autograd_function_allowed, + unwrap_if_dead, +) +from torch.utils._exposed_in import exposed_in + + +__all__ = [ + "exposed_in", + "argnums_t", + "enable_single_level_autograd_function", + "unwrap_dead_wrappers", +] + + +@contextlib.contextmanager +def enable_single_level_autograd_function() -> Generator[None, None, None]: + try: + prev_state = get_single_level_autograd_function_allowed() + set_single_level_autograd_function_allowed(True) + yield + finally: + set_single_level_autograd_function_allowed(prev_state) + + +def unwrap_dead_wrappers(args: tuple[Any, ...]) -> tuple[Any, ...]: + # NB: doesn't use tree_map_only for performance reasons + result = tuple( + unwrap_if_dead(arg) if isinstance(arg, torch.Tensor) else arg for arg in args + ) + return result + + +argnums_t = Union[int, tuple[int, ...]] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/vmap.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/vmap.py new file mode 100644 index 0000000000000000000000000000000000000000..465be67e41fa40a8b1febbe885e950719599a34a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_functorch/vmap.py @@ -0,0 +1,488 @@ +# mypy: ignore-errors + +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import contextlib +import functools +import itertools +from collections.abc import Callable +from functools import partial +from typing import Any, Optional, Union + +import torch +from torch import Tensor +from torch._C._functorch import is_batchedtensor +from torch._functorch.predispatch import ( + _add_batch_dim, + _remove_batch_dim, + _vmap_decrement_nesting, + _vmap_increment_nesting, + lazy_load_decompositions, +) +from torch.utils._pytree import ( + _broadcast_to_and_flatten, + tree_flatten, + tree_map_, + tree_unflatten, + TreeSpec, +) + + +in_dims_t = Union[int, tuple] +out_dims_t = Union[int, tuple[int, ...]] + + +def doesnt_support_saved_tensors_hooks(f): + message = ( + "torch.func.{grad, vjp, jacrev, hessian} don't yet support saved tensor hooks. " + "Please open an issue with your use case." + ) + + @functools.wraps(f) + def fn(*args, **kwargs): + with torch.autograd.graph.disable_saved_tensors_hooks(message): + return f(*args, **kwargs) + + return fn + + +# Checks that all args-to-be-batched have the same batch dim size +def _validate_and_get_batch_size( + flat_in_dims: list[Optional[int]], flat_args: list +) -> int: + batch_sizes = [ + arg.size(in_dim) + for in_dim, arg in zip(flat_in_dims, flat_args) + if in_dim is not None + ] + if len(batch_sizes) == 0: + raise ValueError("vmap: Expected at least one Tensor to vmap over") + if batch_sizes and any(size != batch_sizes[0] for size in batch_sizes): + raise ValueError( + f"vmap: Expected all tensors to have the same size in the mapped " + f"dimension, got sizes {batch_sizes} for the mapped dimension" + ) + return batch_sizes[0] + + +def _num_outputs(batched_outputs: Union[Tensor, tuple[Tensor, ...]]) -> int: + if isinstance(batched_outputs, tuple): + return len(batched_outputs) + return 1 + + +# If value is a tuple, check it has length `num_elements`. +# If value is not a tuple, make a tuple with `value` repeated `num_elements` times + + +def _as_tuple( + value: Any, num_elements: int, error_message_lambda: Callable[[], str] +) -> tuple: + if not isinstance(value, tuple): + return (value,) * num_elements + if len(value) != num_elements: + raise ValueError(error_message_lambda()) + return value + + +def _process_batched_inputs( + in_dims: in_dims_t, args: tuple, func: Callable +) -> tuple[int, list[Any], list[Any], TreeSpec]: + if not isinstance(in_dims, int) and not isinstance(in_dims, tuple): + raise ValueError( + f"vmap({_get_name(func)}, in_dims={in_dims}, ...)(): " + f"expected `in_dims` to be int or a (potentially nested) tuple " + f"matching the structure of inputs, got: {type(in_dims)}." + ) + if len(args) == 0: + raise ValueError( + f"vmap({_get_name(func)})(): got no inputs. Maybe you forgot to add " + f"inputs, or you are trying to vmap over a function with no inputs. " + f"The latter is unsupported." + ) + + flat_args, args_spec = tree_flatten(args) + flat_in_dims = _broadcast_to_and_flatten(in_dims, args_spec) + if flat_in_dims is None: + raise ValueError( + f"vmap({_get_name(func)}, in_dims={in_dims}, ...)(): " + f"in_dims is not compatible with the structure of `inputs`. " + f"in_dims has structure {tree_flatten(in_dims)[1]} but inputs " + f"has structure {args_spec}." + ) + + for i, (arg, in_dim) in enumerate(zip(flat_args, flat_in_dims)): + if not isinstance(in_dim, int) and in_dim is not None: + raise ValueError( + f"vmap({_get_name(func)}, in_dims={in_dims}, ...)(): " + f"Got in_dim={in_dim} for an input but in_dim must be either " + f"an integer dimension or None." + ) + if isinstance(in_dim, int) and not isinstance(arg, Tensor): + raise ValueError( + f"vmap({_get_name(func)}, in_dims={in_dims}, ...)(): " + f"Got in_dim={in_dim} for an input but the input is of type " + f"{type(arg)}. We cannot vmap over non-Tensor arguments, " + f"please use None as the respective in_dim" + ) + if in_dim is not None and (in_dim < -arg.dim() or in_dim >= arg.dim()): + raise ValueError( + f"vmap({_get_name(func)}, in_dims={in_dims}, ...)(): " + f"Got in_dim={in_dim} for some input, but that input is a Tensor " + f"of dimensionality {arg.dim()} so expected in_dim to satisfy " + f"-{arg.dim()} <= in_dim < {arg.dim()}." + ) + if in_dim is not None and in_dim < 0: + flat_in_dims[i] = in_dim % arg.dim() + + return ( + _validate_and_get_batch_size(flat_in_dims, flat_args), + flat_in_dims, + flat_args, + args_spec, + ) + + +# Creates BatchedTensors for every Tensor in arg that should be batched. +# Returns the (potentially) batched arguments and the batch_size. + + +def _create_batched_inputs( + flat_in_dims: list[Any], flat_args: list[Any], vmap_level: int, args_spec +) -> tuple: + # See NOTE [Ignored _remove_batch_dim, _add_batch_dim] + batched_inputs = [ + arg if in_dim is None else _add_batch_dim(arg, in_dim, vmap_level) + for in_dim, arg in zip(flat_in_dims, flat_args) + ] + return tree_unflatten(batched_inputs, args_spec) + + +def _maybe_remove_batch_dim(name, batched_output, vmap_level, batch_size, out_dim): + if out_dim is None: + if isinstance(batched_output, torch.Tensor) and is_batchedtensor( + batched_output + ): + raise ValueError( + f"vmap({name}, ...): `{name}` can not return a " + f"BatchedTensor when out_dim is None" + ) + return batched_output + + # out_dim is non None + if not isinstance(batched_output, torch.Tensor): + raise ValueError( + f"vmap({name}, ...): `{name}` must only return " + f"Tensors, got type {type(batched_output)}. " + "Did you mean to set out_dims= to None for output?" + ) + + return _remove_batch_dim(batched_output, vmap_level, batch_size, out_dim) + + +# Undos the batching (and any batch dimensions) associated with the `vmap_level`. +def _unwrap_batched( + batched_outputs: Union[Tensor, tuple[Tensor, ...]], + out_dims: out_dims_t, + vmap_level: int, + batch_size: int, + func: Callable, +) -> tuple: + flat_batched_outputs, output_spec = tree_flatten(batched_outputs) + + def incompatible_error(): + raise ValueError( + f"vmap({_get_name(func)}, ..., out_dims={out_dims})(): " + f"out_dims is not compatible with the structure of `outputs`. " + f"out_dims has structure {tree_flatten(out_dims)[1]} but outputs " + f"has structure {output_spec}." + ) + + if isinstance(batched_outputs, torch.Tensor): + # Some weird edge case requires us to spell out the following + # see test_out_dims_edge_case + if isinstance(out_dims, int): + flat_out_dims = [out_dims] + elif isinstance(out_dims, tuple) and len(out_dims) == 1: + flat_out_dims = out_dims + elif out_dims is None: + flat_out_dims = [out_dims] + else: + incompatible_error() + else: + flat_out_dims = _broadcast_to_and_flatten(out_dims, output_spec) + if flat_out_dims is None: + incompatible_error() + + flat_outputs = [ + _maybe_remove_batch_dim( + _get_name(func), batched_output, vmap_level, batch_size, out_dim + ) + for batched_output, out_dim in zip(flat_batched_outputs, flat_out_dims) + ] + return tree_unflatten(flat_outputs, output_spec) + + +def _check_int_or_none(x, func, out_dims): + if isinstance(x, int): + return + if x is None: + return + raise ValueError( + f"vmap({_get_name(func)}, ..., out_dims={out_dims}): `out_dims` must be " + f"an int, None or a python collection of ints representing where in the outputs the " + f"vmapped dimension should appear." + ) + + +def _check_out_dims_is_int_or_int_pytree(out_dims: out_dims_t, func: Callable) -> None: + if isinstance(out_dims, int): + return + tree_map_(partial(_check_int_or_none, func=func, out_dims=out_dims), out_dims) + + +def _get_name(func: Callable): + if hasattr(func, "__name__"): + return func.__name__ + + if isinstance(func, functools.partial): + return f"functools.partial({_get_name(func.func)}, ...)" + + # Not all callables have __name__, in fact, only static functions/methods + # do. A callable created via nn.Module, to name one example, doesn't have a + # __name__. + return repr(func) + + +def vmap_impl(func, in_dims, out_dims, randomness, chunk_size, *args, **kwargs): + lazy_load_decompositions() + _check_out_dims_is_int_or_int_pytree(out_dims, func) + batch_size, flat_in_dims, flat_args, args_spec = _process_batched_inputs( + in_dims, args, func + ) + + if chunk_size is not None: + chunks_flat_args = _get_chunked_inputs( + flat_args, flat_in_dims, batch_size, chunk_size + ) + return _chunked_vmap( + func, + flat_in_dims, + chunks_flat_args, + args_spec, + out_dims, + randomness, + **kwargs, + ) + + # If chunk_size is not specified. + return _flat_vmap( + func, + batch_size, + flat_in_dims, + flat_args, + args_spec, + out_dims, + randomness, + **kwargs, + ) + + +def get_chunk_sizes(total_elems, chunk_size): + n_chunks = total_elems // chunk_size + chunk_sizes = [chunk_size] * n_chunks + # remainder chunk + remainder = total_elems % chunk_size + if remainder != 0: + chunk_sizes.append(remainder) + return chunk_sizes + + +def _get_chunked_inputs(flat_args, flat_in_dims, batch_size, chunk_size): + split_idxs = (batch_size,) + if chunk_size is not None: + chunk_sizes = get_chunk_sizes(batch_size, chunk_size) + split_idxs = tuple(itertools.accumulate(chunk_sizes)) + + flat_args_chunks = tuple( + ( + t.tensor_split(split_idxs, dim=in_dim) + if in_dim is not None + else [ + t, + ] + * len(split_idxs) + ) + for t, in_dim in zip(flat_args, flat_in_dims) + ) + + # transpose chunk dim and flatten structure + # chunks_flat_args is a list of flatten args + chunks_flat_args = zip(*flat_args_chunks) + return chunks_flat_args + + +def _flatten_chunks_output(chunks_output_): + # chunks_output is a list of chunked outputs + # flatten chunked outputs: + flat_chunks_output = [] + arg_spec = None + for output in chunks_output_: + flat_output, arg_specs = tree_flatten(output) + flat_chunks_output.append(flat_output) + if arg_spec is None: + arg_spec = arg_specs + + # transpose chunk dim and flatten structure + # flat_output_chunks is flat list of chunks + flat_output_chunks = list(zip(*flat_chunks_output)) + return flat_output_chunks, arg_spec + + +def _concat_chunked_outputs(out_dims, arg_spec, flat_output_chunks): + # concat chunks on out_dim + flat_out_dims = _broadcast_to_and_flatten(out_dims, arg_spec) + assert len(flat_out_dims) == len(flat_output_chunks) + flat_output = [] + for idx, out_dim in enumerate(flat_out_dims): + flat_output.append(torch.cat(flat_output_chunks[idx], dim=out_dim)) + # release tensors + flat_output_chunks[idx] = None + + return flat_output + + +# Applies vmap on chunked_input and returns concatenated output over the chunks. +def _chunked_vmap( + func, flat_in_dims, chunks_flat_args, args_spec, out_dims, randomness, **kwargs +): + chunks_output = [] + rs = torch.get_rng_state() if randomness == "same" else None + for flat_args in chunks_flat_args: + batch_size = _validate_and_get_batch_size(flat_in_dims, flat_args) + + # The way we compute split the input in `_get_chunked_inputs`, + # we may get a tensor with `0` batch-size. We skip any computation + # in that case. + # Eg. + # >>> chunk_size = 1 + # >>> batch_size = 6 + # >>> t = torch.zeros(batch_size, 1) + # >>> t.tensor_split([1, 2, 3, 4, 5, 6]) + # (tensor([[0.]]), tensor([[0.]]), tensor([[0.]]), tensor([[0.]]), + # tensor([[0.]]), tensor([[0.]]), tensor([], size=(0, 1))) + if batch_size == 0: + continue + + if rs is not None: + torch.set_rng_state(rs) + chunks_output.append( + _flat_vmap( + func, + batch_size, + flat_in_dims, + flat_args, + args_spec, + out_dims, + randomness, + **kwargs, + ) + ) + + flat_output_chunks, arg_spec = _flatten_chunks_output(chunks_output) + + # chunked output tensors are held by both `flat_output_chunks` and `chunks_output`. + # eagerly remove the reference from `chunks_output`. + del chunks_output + + # concat chunks on out_dim + flat_output = _concat_chunked_outputs(out_dims, arg_spec, flat_output_chunks) + + # finally unflatten the output + return tree_unflatten(flat_output, arg_spec) + + +# Vmap refactored helper functions: +def _check_randomness_arg(randomness): + if randomness not in ["error", "different", "same"]: + raise RuntimeError( + f"Only allowed values for randomness are 'error', 'different', or 'same'. Got {randomness}" + ) + + +@contextlib.contextmanager +def vmap_increment_nesting(batch_size, randomness): + try: + vmap_level = _vmap_increment_nesting(batch_size, randomness) + yield vmap_level + finally: + _vmap_decrement_nesting() + + +def _flat_vmap( + func, batch_size, flat_in_dims, flat_args, args_spec, out_dims, randomness, **kwargs +): + with vmap_increment_nesting(batch_size, randomness) as vmap_level: + batched_inputs = _create_batched_inputs( + flat_in_dims, flat_args, vmap_level, args_spec + ) + batched_outputs = func(*batched_inputs, **kwargs) + return _unwrap_batched(batched_outputs, out_dims, vmap_level, batch_size, func) + + +# `restore_vmap` is a private helper function. It is vmap but has the following +# differences: +# - instead of returning outputs, it returns an (outputs, out_dims) tuple. +# out_dims is a pytree of same shape as outputs and contains Optional[int] +# specifying where the vmapped dimension, if it exists, is in the corresponding output. +# - does no validation on in_dims or inputs (vmap expects at least one Tensor to be vmapped). +# restore_vmap allows for no inputs to have the vmap dimension +# - does no validation on outputs (vmap expects only Tensor outputs) +# restore_vmap allows for return of arbitrary outputs (not just Tensors) +# +# The TL;DR is that restore_vmap is more general than vmap and has a slightly +# different API. The relaxations are so that we can "pause" vmap in the middle +# of its execution and then "restore" it later (this is what we do in +# the generate_vmap_rule=True implementation of autograd.Function). +# +# restore_vmap can be technically used in the implementation of vmap, but doing +# that refactor is a bit technically challenging because: +# - vmap couples the tensor-wrapping code with error checking +# - vmap's tensor unwrapping code is in C++; we would need to rewrite part of it +# in python because it overlaps with unwrap_batched +def restore_vmap(func, in_dims, batch_size, randomness): + def inner(*args, **kwargs): + with vmap_increment_nesting(batch_size, randomness) as vmap_level: + batched_inputs = wrap_batched(args, in_dims, vmap_level) + batched_outputs = func(*batched_inputs, **kwargs) + return unwrap_batched(batched_outputs, vmap_level) + + return inner + + +def wrap_batched(args, bdims, level): + flat_args, spec = tree_flatten(args) + flat_bdims = _broadcast_to_and_flatten(bdims, spec) + assert flat_bdims is not None + result = _create_batched_inputs(flat_bdims, flat_args, level, spec) + return result + + +def unwrap_batched(args, level): + flat_args, spec = tree_flatten(args) + if len(flat_args) == 0: + return args, () + result = [ + ( + torch._C._functorch._unwrap_batched(arg, level) + if isinstance(arg, torch.Tensor) + else (arg, None) + ) + for arg in flat_args + ] + output, bdims = zip(*result) + return tree_unflatten(output, spec), tree_unflatten(bdims, spec) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d1d5c567dced423645179389745dac08fa0d13f7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/__init__.py @@ -0,0 +1,82 @@ +from torch._higher_order_ops._invoke_quant import ( + invoke_quant, + invoke_quant_packed, + InvokeQuant, +) +from torch._higher_order_ops.aoti_call_delegate import aoti_call_delegate +from torch._higher_order_ops.associative_scan import associative_scan +from torch._higher_order_ops.auto_functionalize import ( + auto_functionalized, + auto_functionalized_v2, +) +from torch._higher_order_ops.base_hop import BaseHOP +from torch._higher_order_ops.cond import cond +from torch._higher_order_ops.effects import with_effects +from torch._higher_order_ops.executorch_call_delegate import executorch_call_delegate +from torch._higher_order_ops.flat_apply import flat_apply +from torch._higher_order_ops.flex_attention import ( + flex_attention, + flex_attention_backward, +) +from torch._higher_order_ops.foreach_map import _foreach_map, foreach_map +from torch._higher_order_ops.hints_wrap import hints_wrapper +from torch._higher_order_ops.invoke_subgraph import invoke_subgraph +from torch._higher_order_ops.local_map import local_map_hop +from torch._higher_order_ops.map import map +from torch._higher_order_ops.out_dtype import out_dtype +from torch._higher_order_ops.print import print +from torch._higher_order_ops.run_const_graph import run_const_graph +from torch._higher_order_ops.scan import scan +from torch._higher_order_ops.strict_mode import strict_mode +from torch._higher_order_ops.torchbind import call_torchbind +from torch._higher_order_ops.while_loop import ( + while_loop, + while_loop_stack_output_op as while_loop_stack_output, +) +from torch._higher_order_ops.wrap import ( + dynamo_bypassing_wrapper, + inductor_compiled_code, + tag_activation_checkpoint, + wrap_activation_checkpoint, + wrap_with_autocast, + wrap_with_set_grad_enabled, +) + + +__all__ = [ + "cond", + "while_loop", + "invoke_subgraph", + "scan", + "map", + "flex_attention", + "flex_attention_backward", + "hints_wrapper", + "BaseHOP", + "flat_apply", + "foreach_map", + "_foreach_map", + "with_effects", + "tag_activation_checkpoint", + "auto_functionalized", + "auto_functionalized_v2", + "associative_scan", + "out_dtype", + "executorch_call_delegate", + "call_torchbind", + "run_const_graph", + "InvokeQuant", + "invoke_quant", + "invoke_quant_packed", + "wrap_with_set_grad_enabled", + "wrap_with_autocast", + "wrap_activation_checkpoint", + "dynamo_bypassing_wrapper", + "strict_mode", + "aoti_call_delegate", + "map", + "while_loop_stack_output", + "local_map_hop", + "print", + "inductor_compiled_code", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/_invoke_quant.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/_invoke_quant.py new file mode 100644 index 0000000000000000000000000000000000000000..b7a9fb94b93e222e224590f7795395725f52e749 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/_invoke_quant.py @@ -0,0 +1,62 @@ +# mypy: allow-untyped-defs +# need to fix prim_hop_base type annotations first + +import dataclasses +from typing import Optional + +import torch +from torch._higher_order_ops.base_hop import BaseHOP, FunctionWithNoFreeVars + + +class InvokeQuantTracer(BaseHOP): + def __init__(self) -> None: + super().__init__("invoke_quant_packed") + + def __call__(self, subgraph, *operands, scheme=None, quant_options=None): + subgraph = FunctionWithNoFreeVars(subgraph) + return super().__call__( + subgraph, *operands, scheme=scheme, quant_options=quant_options + ) + + +invoke_quant_packed = InvokeQuantTracer() + + +class InvokeQuantUnpacked(BaseHOP): + def __init__(self) -> None: + super().__init__("invoke_quant") + + +invoke_quant = InvokeQuantUnpacked() + + +@dataclasses.dataclass(frozen=True, repr=True) +class InvokeQuant: + """ + Invoke a quantization function that will be preserved as a single operator. Preservation + as a single operator aids in pattern matching and custom lowerings. + + The operation appears as: + torch.ops.higher_order.invoke_quant(subgraph, *args, scheme=scheme) + + Args: + codegen_low_precision: Use observed subgraph dtypes for codegen instead of + upcasting to fp32. Can improve performance for prologue fusion but + requires careful testing of numerics. + """ + + codegen_low_precision: bool = True + + def __call__( + self, + *args, + scheme: Optional[str] = None, + **kwargs, + ): + if not torch.compiler.is_compiling(): + return args[0](*args[1:], **kwargs) + + if scheme is not None: + kwargs["scheme"] = scheme + + return invoke_quant_packed(*args, **kwargs, quant_options=self) # type: ignore[call-arg] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/aoti_call_delegate.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/aoti_call_delegate.py new file mode 100644 index 0000000000000000000000000000000000000000..bb2c62de7617aa51bddbf0590ea61cb70a758e63 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/aoti_call_delegate.py @@ -0,0 +1,164 @@ +# mypy: allow-untyped-defs + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import annotations + +import torch +import torch.utils._pytree as pytree +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode +from torch.fx.experimental.proxy_tensor import ( + disable_proxy_modes_tracing, + ProxyTorchDispatchMode, + track_tensor_tree, +) + + +AOTI_LOWERED_MODULE = "AOTInductorEPModule/AOTInductorRunnerWrapper" + + +class AOTICallDelegate(HigherOrderOperator): + """aoti_call_delegate is a HOP for calling AOTInductor lowered submodule in ExportedProgram. + + It has the following signature: + aoti_call_delegate( + lowered_module: Union[AOTInductorEPModule, AOTInductorRunnerWrapper] + original_gm:fx.GraphModule, + weight_args: List[Tensor], + input_args: List[Tensor], + ) -> outputs: List[Tensor] + + where, + - lowered_module is the AOTInductor lowered submodule, backed by compiled .so file, supporting real tensor inputs + - original_gm is the stateless version of the original GraphModule before lowering, allowing FakeTensor propagation + - weight_args is the list of weights in original GraphModule, including parameters and buffers + - input_args is the list of flatten inputs + """ + + def __init__(self) -> None: + super().__init__("aoti_call_delegate") + + def __call__( + self, + lowered_module: AOTI_LOWERED_MODULE, # type: ignore[valid-type] + original_gm: torch.fx.GraphModule, + weight_args: list[torch.Tensor], + input_args: list[torch.Tensor], + ) -> list[torch.Tensor]: + return super().__call__(lowered_module, original_gm, weight_args, input_args) + + +aoti_call_delegate = AOTICallDelegate() +aoti_call_delegate.fallthrough(torch._C.DispatchKey.PythonDispatcher) +aoti_call_delegate.fallthrough(torch._C.DispatchKey.PythonTLSSnapshot) +aoti_call_delegate.fallthrough(torch._C.DispatchKey.ADInplaceOrView) +aoti_call_delegate.fallthrough(torch._C.DispatchKey.AutocastCPU) + + +@aoti_call_delegate.py_impl(torch._C.DispatchKey.CompositeExplicitAutograd) +def call_delegate_cpu( + lowered_module: AOTI_LOWERED_MODULE, # type: ignore[valid-type] + original_gm: torch.fx.GraphModule, + weight_args: list[torch.Tensor], + input_args: list[torch.Tensor], +) -> list[torch.Tensor]: + # FX creates this immutable_dict/list concept. Get rid of this. + map_types: dict[type, type] = { + torch.fx.immutable_collections.immutable_dict: dict, + torch.fx.immutable_collections.immutable_list: list, + } + new_args = pytree.tree_map_only( + tuple(map_types.keys()), + lambda a: map_types[type(a)](a), + weight_args + input_args, + lambda a: isinstance(a, tuple(map_types.keys())), + ) + has_fake_args = any(isinstance(arg, FakeTensor) for arg in new_args) + if has_fake_args: + # use stateless original_gm for tracing with fake tensors + fake_out = original_gm(*new_args) + return fake_out + else: + # use AOTI Runner for real tensors + new_input_args = new_args[len(weight_args) :] + if type(lowered_module).__name__ == "AOTInductorRunnerWrapper": + return lowered_module(*new_input_args) # type: ignore[misc] + elif type(lowered_module).__name__ == "AOTInductorEPModule": + return lowered_module(new_input_args) # type: ignore[misc] + else: + raise RuntimeError( + f"Unexpected lowered_module type: {type(lowered_module)}." + ) + + +def trace_aoti_call_delegate( + proxy_mode, func_overload, lowered_module, original_gm, weight_args, input_args +): + proxy_mode.tracer.root.register_module("lowered_module", lowered_module) + proxy_mode.tracer.root.register_module("original_gm", original_gm) + + node_args = (lowered_module, original_gm, weight_args, input_args) + proxy_args = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, node_args) + + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", func_overload, proxy_args, {}, name="aoti_call_delegate" + ) + with disable_proxy_modes_tracing(): + out = call_delegate_cpu(lowered_module, original_gm, weight_args, input_args) + + return track_tensor_tree(out, out_proxy, constant=None, tracer=proxy_mode.tracer) + + +@aoti_call_delegate.py_impl(ProxyTorchDispatchMode) +def call_delegate_proxy_torch_dispatch_mode( + mode: ProxyTorchDispatchMode, + lowered_module: AOTI_LOWERED_MODULE, # type: ignore[valid-type] + original_gm: torch.fx.GraphModule, + weight_args: list[torch.Tensor], + input_args: list[torch.Tensor], +): + res = trace_aoti_call_delegate( + mode, aoti_call_delegate, lowered_module, original_gm, weight_args, input_args + ) + return res + + +@aoti_call_delegate.py_impl(FakeTensorMode) +def call_delegate_fake_tensor_mode( + mode: FakeTensorMode, + lowered_module: AOTI_LOWERED_MODULE, # type: ignore[valid-type] + original_gm: torch.fx.GraphModule, + weight_args: list[torch.Tensor], + input_args: list[torch.Tensor], +) -> list[torch.Tensor]: + with mode: + return call_delegate_cpu(lowered_module, original_gm, weight_args, input_args) + + +@aoti_call_delegate.py_functionalize_impl +def call_delegate_functionalize( + ctx, + lowered_module: AOTI_LOWERED_MODULE, # type: ignore[valid-type] + original_gm: torch.fx.GraphModule, + weight_args: list[torch.Tensor], + input_args: list[torch.Tensor], +): + unwrapped_weight_args = tuple( + ctx.unwrap_tensors(weight_arg) for weight_arg in weight_args + ) + unwrapped_input_args = tuple( + ctx.unwrap_tensors(input_arg) for input_arg in input_args + ) + with ctx.redispatch_to_next(): + res = aoti_call_delegate( + lowered_module, + original_gm, + unwrapped_weight_args, # type: ignore[arg-type] + unwrapped_input_args, # type: ignore[arg-type] + ) + return ctx.wrap_tensors(res) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/associative_scan.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/associative_scan.py new file mode 100644 index 0000000000000000000000000000000000000000..7cc2e3007cdffb7bce19fae9bf45f6b53b345476 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/associative_scan.py @@ -0,0 +1,912 @@ +# mypy: allow-untyped-defs +import functools +import itertools +from collections.abc import Callable +from typing import Any + +import torch +import torch._prims_common as utils +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._higher_order_ops.utils import ( + _maybe_compile_and_run_fn, + _maybe_run_with_interpreter, + check_input_alias_and_mutation_return_outputs, + check_meta_consistency, + create_bw_fn, + first_slice_copy, + first_slice_copy_with_grad, + materialize_as_graph, + reenter_make_fx, + save_tensors_and_symints_for_backward, + saved_tensors_and_symints, + split_into_chunks, + unique_graph_id, + validate_subgraph_args_types, +) +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ( + disable_proxy_modes_tracing, + ProxyTorchDispatchMode, + track_tensor_tree, +) + + +aten = torch._ops.ops.aten + + +def wrap_combine_fn_flat(*args, combine_fn, spec, num_leaves): + assert len(args) == 2 * num_leaves, ( + f"Combin_fn received wrong number of arguments, expected {2 * num_leaves}, but got {len(args)}" + ) + lhs = pytree.tree_unflatten(args[:num_leaves], spec) + rhs = pytree.tree_unflatten(args[num_leaves:], spec) + return combine_fn(lhs, rhs) + + +def _interleave(a, b, dim=0): + # https://stackoverflow.com/questions/60869537/how-can-i-interleave-5-pytorch-tensors + if b_trunc := (a.shape[dim] == b.shape[dim] + 1): + pad = ( + [0] * ((b.ndim - dim - 1) * 2 + 1) + + [1] + + [0] * (b.ndim * 2 - ((b.ndim - dim - 1) * 2 + 2)) + ) + b = torch.nn.functional.pad(b, pad) + + stacked = torch.stack([a, b], dim=dim + 1) + interleaved = torch.flatten(stacked, start_dim=dim, end_dim=dim + 1) + # pyrefly: ignore [unbound-name] + if b_trunc: + # TODO: find torch alternative for slice_along dim for torch.jit.script to work + interleaved = aten.slice(interleaved, dim, 0, b.shape[dim] + a.shape[dim] - 1) + return interleaved + + +def safe_map(f, *args): + args = list(map(list, args)) + n = len(args[0]) + for arg in args[1:]: + if len(arg) != n: + raise ValueError("length mismatch: {list(map(len, args))}") + + def nf(a): + return f(*a) + + return list(map(nf, zip(*args))) + + +class AssociativeScanOp(HigherOrderOperator): + def __init__(self): + super().__init__("associative_scan") + + def __call__(self, combine_fn, xs, additional_inputs): + # There is currently an issue that the ScanOp is sometimes called with + # the additional_inputs being a list. See https://github.com/pytorch/pytorch/issues/145785 + # Once this issue is resolved, the assertion should only allow tuples + # and the tuple cast should be removed + assert isinstance(additional_inputs, (tuple, list)), ( + "additional_inputs must be a tuple." + ) + additional_inputs = ( + tuple(additional_inputs) + if isinstance(additional_inputs, list) + else additional_inputs + ) + validate_subgraph_args_types(additional_inputs) + return super().__call__(combine_fn, xs, additional_inputs) + + # pyrefly: ignore [bad-override] + def gen_schema(self, combine_fn, xs, additional_inputs): + from torch._higher_order_ops.schema import HopSchemaGenerator + from torch._higher_order_ops.utils import materialize_as_graph + + # For associative scan, we need two copies of xs for the combine function + # The combine function takes two elements and returns one element + xs_slice1 = [first_slice_copy(x) for x in xs] + xs_slice2 = [first_slice_copy(x) for x in xs] + all_inputs = tuple(xs_slice1 + xs_slice2 + list(additional_inputs)) + + combine_gm: torch.fx.GraphModule = materialize_as_graph(combine_fn, all_inputs) + ( + _, + _, + _, + mutated_inputs, + outputs, + ) = check_input_alias_and_mutation_return_outputs(combine_gm) + if len(mutated_inputs) > 0: + raise RuntimeError( + "For associative_scan, combine_fn cannot have in-place mutations but found " + f"{mutated_inputs}-th inputs are mutated." + ) + + schema_gen = HopSchemaGenerator(self) + schema_gen.add_arg("combine_fn", combine_gm) + + for idx, x in enumerate(xs): + schema_gen.add_arg(f"xs{idx}", x) + + for idx, arg in enumerate(additional_inputs): + schema_gen.add_arg( + f"additional_input{idx}", + arg, + ) + + for out in outputs: + schema_gen.add_output(out) + + schema_gen.add_schema_tree_spec(combine_fn, xs, additional_inputs) + return schema_gen.gen_schema() + + +associative_scan_op = AssociativeScanOp() + + +def associative_scan( + combine_fn: Callable[[pytree.PyTree, pytree.PyTree], pytree.PyTree], + xs: pytree.PyTree, + dim: int, + reverse: bool = False, + combine_mode: str = "pointwise", +) -> torch.Tensor: + r""" + Performs an inclusive scan with an associative combine function. + + .. warning:: + `torch.associative_scan` is a prototype feature in PyTorch. It currently + does not support autograd and you may run into miscompiles. + Read more about feature classification at: + https://pytorch.org/blog/pytorch-feature-classification-changes/#prototype + + This operator requires runtime code generation and so requires support for + ``torch.compile``. Further, only CUDA device codegen is supported at the moment. + + Args: + combine_fn (Callable): A binary callable with type ``(Tensor, Tensor) -> Tensor``, + or if input is a pytree ``(pytree, pytree) -> pytree``. + This function must be pure, i.e., no lifted arguments are supported at the moment, + satisfy the associative property and have no side-effects. + xs (torch.Tensor): The input tensor, or nested pytree of tensors. + All inputs are expected to have the same shape. + dim (int): the dimension to scan over + reverse (bool): A boolean stating if the scan should be reversed with respect to ``dim``, default ``False``. + combine_mode (str): A string indicating whether the ``combine_fn`` is ``pointwise`` or ``generic``, default ``pointwise``. + If ``combine_mode=pointwise``, ``combine_fn`` must be pure, may only contain pointwise operations + and ``xs`` must be CUDA tensors. + In all other cases ``combine_mode=generic`` should be used. + Note: ``combine_mode=pointwise`` is more efficient than ``combine_mode=generic``. + + + Example:: + + def add(x: torch.Tensor, y: torch.Tensor): + return x + y + + + cumsum = associative_scan(add, x, dim) + + """ + # TODO: Support lifted arguments in inductor for associative_scan + # TODO: Support autograd for cases with lifted arguments for combine_mode=pointwise + + # The reason we flatten xs before calling into dynamo is that + # we want to create a consistent input ordering for combine_fn + # and we also want to the input ordering matches the output ordering. + leaves_xs_orig, spec_xs = pytree.tree_flatten(xs) + + def _validate_input(cfn, lxs, d, r, cm): + # Basic arguments check + if not callable(cfn): + raise ValueError(f"Combine_fn must be a callable, but got {cfn}") + if not isinstance(d, int): + raise ValueError("Dim must be an int, but got " + str(type(d))) + if not isinstance(r, bool): + raise RuntimeError("Reverse must be a bool, but got " + str(type(r))) + if cm not in ["pointwise", "generic"]: + raise ValueError( + f"Combine_mode must either 'pointwise' or 'generic', but got {cm}" + ) + if cm == "pointwise" and not all(l.device.type in ("cuda", "xpu") for l in lxs): + raise ValueError( + "For combine_mode='pointwise', all input tensors need to be on CUDA or XPU" + ) + + # Checks for xs + if len(lxs) == 0: + raise ValueError("Expected at least 1 xs leaf") + if any(not isinstance(x, torch.Tensor) for x in lxs): + raise ValueError("xs leaves must be a Tensor") + if any(x.is_sparse for x in lxs): + raise ValueError( + "xs leaves must dense Tensors, consider using `to_dense()`" + ) + if any(x.ndim <= d for x in lxs): + raise ValueError( + "All xs leaves must at least have 'dim' number of dimensions and scan dimension > 0" + ) + if any(x.shape[d] == 0 for x in lxs): + raise ValueError( + "All xs leaves must at least have 'dim' number of dimensions and scan dimension > 0" + ) + + ndim = leaves_xs_orig[0].ndim + dim = utils.canonicalize_dim(ndim, dim) + + _validate_input(combine_fn, leaves_xs_orig, dim, reverse, combine_mode) + + # Move scan dim to 0 and always perform scan on dim 0 + leaves_xs = [torch.movedim(elem, dim, 0) for elem in leaves_xs_orig] + + if reverse: + leaves_xs = [torch.flip(elem, [0]) for elem in leaves_xs] + + if combine_mode == "generic": + # The generic_associative_scan implementation calls the combine_fn with a `batch` along the scan dimension + # For example, consider: + # def add(x: torch.Tensor, y: torch.Tensor): + # return x + y + # leaves = torch.tensor([[0.0, 1.0, 2.0, 3.0] + # [0.0, 1.0, 2.0, 3.0]]) + # which has shape 2 x 4; + # dim = 1; + # In the first iteration of `_scan` the combine_fn gets invoked with + # combine_fn([torch.tensor([[0.0, 2.0], + # [0.0, 2.0]])], + # [torch.tensor([[1.0, 3.0], + # [1.0, 3.0]])]) + # The arguments are of shape 2 x 2, but can be evaluated in parallel along the scan dimension. + combine_fn = functools.partial( + wrap_combine_fn_flat, + combine_fn=torch.vmap( + combine_fn, + in_dims=( + pytree.tree_unflatten([0] * len(leaves_xs), spec_xs), + pytree.tree_unflatten([0] * len(leaves_xs), spec_xs), + ), + out_dims=0, + ), + spec=spec_xs, + num_leaves=len(leaves_xs), + ) + out = generic_associative_scan(combine_fn, leaves_xs, additional_inputs=()) + out = pytree.tree_unflatten(out, spec_xs) + else: + combine_fn = functools.partial( + wrap_combine_fn_flat, + combine_fn=combine_fn, + spec=spec_xs, + num_leaves=len(leaves_xs), + ) + + def run_flattened_associative_scan(combine_fn, leaves_xs): + return associative_scan_op(combine_fn, leaves_xs, additional_inputs=()) + + out = _maybe_compile_and_run_fn( + run_flattened_associative_scan, + combine_fn, + leaves_xs, + ) + + if reverse: + out = pytree.tree_map(lambda elem: elem.flip([0]), out) + + out = pytree.tree_map(lambda elem: torch.movedim(elem, 0, dim), out) + + return out + + +def generic_associative_scan(operator, leaves, dim=0, additional_inputs=()): + r""" + This function performs the associative_scan operation. + The algorithm works by recursively collecting neighbours of ``leaves`` and subsequently + applying the ``operator`` on all pairs in parallel along ``dim``. + The results of the recursive calls are later combined. + + Args: + operator (Callable): A binary callable with type ``(Tensor, Tensor) -> Tensor``, + or if input is a pytree ``(pytree, pytree) -> pytree``. + This function must be pure, pointwise, and satisfy the associative property. + leaves (torch.Tensor): A list of torch.Tensors converted from the pytree of + ``xs`` provided to ``associative_scan``. + All inputs are expected to have the same shape. + dim (int): the dimension to scan over + additional_inputs (Tuple of tensors): A tuple of lifted parameters from the global scope. + This parameter will be populated internally. + + Example:: + + def add(x: torch.Tensor, y: torch.Tensor): + return x + y + + leaves = torch.tensor([0.0, 1.0, 2.0, 3.0]) + + First iteration of _scan -> + # odd_elems -> apply operator on all neighbours + # odd_elems = operator([torch.tensor([0.0, 2.0])], + # [torch.tensor([1.0, 3.0])]) + odd_elems = torch.tensor([1.0, 5.0]) + Second iteration of _scan -> + # odd_elems = operator([torch.tensor([1.0])], + # [torch.tensor([5.0])]) + odd_elems = torch.tensor([6.0]) + # even_elems -> apply operator on all odd_elems and + # every second element of ``elems``, starting from the second element. + # even_elems is expanded with the first element of ``elems`` + even_elems = [1.0] + # Merges odd_elems and even_elems + res = torch.tensor([1.0, 6.0]) + # even_elems -> apply operator on all odd_elems and + # every second element of ``elems``, starting from the second element. + # even_elems is expanded with the first element of ``elems`` + even_elems = [0.0, 3.0] + # Merges odd_elems and even_elems + res = torch.tensor([0.0, 1.0, 3.0, 6.0]) + + """ + + def call_operator(*args): + return pytree.tree_leaves(operator(*args)) + + def _scan(elems): + """Perform the actual recursive scan on ``elems``.""" + num_elems = elems[0].shape[dim] + + if num_elems < 2: + return elems + + reduced_elems = call_operator( + *[aten.slice(elem, dim, 0, -1, 2) for elem in elems], + *[aten.slice(elem, dim, 1, None, 2) for elem in elems], + *additional_inputs, + ) + + # Recursively compute scan for partially reduced tensors. + odd_elems = _scan(reduced_elems) + + if num_elems % 2 == 0: + even_elems = call_operator( + *[aten.slice(e, dim, 0, -1) for e in odd_elems], + *[aten.slice(e, dim, 2, None, 2) for e in elems], + *additional_inputs, + ) + else: + even_elems = call_operator( + *odd_elems, + *[aten.slice(e, dim, 2, None, 2) for e in elems], + *additional_inputs, + ) + + # The first element of a scan is the same as the first element + # of the original `elems`. + even_elems = [ + torch.cat([aten.slice(elem, dim, 0, 1), result], dim=dim) + if result.shape.numel() > 0 and elem.shape[dim] > 0 + else result + if result.shape.numel() > 0 + else aten.slice( + elem, dim, 0, 1 + ) # Jax allows/ignores concat with 0-dim, Pytorch does not + for (elem, result) in zip(elems, even_elems) + ] + + return list( + safe_map(functools.partial(_interleave, dim=dim), even_elems, odd_elems) + ) + + scans = _scan(leaves) + + return scans + + +def trace_associative_scan( + proxy_mode, + func_overload, + combine_fn: Callable, + xs: list[torch.Tensor], + additional_inputs: tuple[torch.Tensor], +): + from torch._dynamo.utils import clone_input + + with disable_proxy_modes_tracing(): + sample_xs = [first_slice_copy(x) for x in itertools.chain(xs, xs)] + sample_additional_inputs = [ + clone_input(x) if isinstance(x, torch.Tensor) else x + for x in additional_inputs + ] + combine_graph = reenter_make_fx(combine_fn)( + *sample_xs, *sample_additional_inputs + ) + + outputs = None + for node in combine_graph.graph.nodes: + if node.op == "output": + assert outputs is None + assert len(node.args) == 1 + outputs = node.args[0] + + assert outputs is not None + outputs = pytree.tree_leaves(outputs) + assert len(outputs) == len(xs), ( + f"expected combine_fn to return {len(xs)} results but got {len(outputs)}" + ) + + xs_fake_tensors: list[torch.Tensor | torch.SymInt | int] = [ + first_slice_copy(x) for x in xs + ] + output_fake_tensors: list[torch.Tensor | torch.SymInt | int] = [ + c.meta["val"] for c in outputs + ] + check_meta_consistency( + xs_fake_tensors, output_fake_tensors, "init", "carry", include_contiguity=False + ) + + _, combine_graph_name = unique_graph_id( + proxy_mode, prefix="associative_scan_combine_graph" + ) + + proxy_mode.tracer.root.register_module(combine_graph_name, combine_graph) + + args = (combine_graph, xs, additional_inputs) + proxy_args = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, args) + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", func_overload, proxy_args, {}, name="associative_scan" + ) + + with disable_proxy_modes_tracing(): + out = tuple(aten.clone(x) for x in xs) + + return track_tensor_tree(out, out_proxy, constant=None, tracer=proxy_mode.tracer) + + +@associative_scan_op.py_impl(DispatchKey.CompositeExplicitAutograd) +def associative_scan_op_dense(combine_fn, xs, additional_inputs): + return generic_associative_scan(combine_fn, xs, additional_inputs=additional_inputs) + + +class AssociativeScanAutogradOp(torch.autograd.Function): + r""" associative_scan + Example:: + xs = torch.arange(1, 5) = [1, 2, 3, 4] + + def combine_fn(a: torch.Tensor, b: torch.Tensor): + return a * b + + ys = associative_scan(comine_fn, xs), + which can be unpacked as: + ys0 = xs0 = 1 + ys1 = combine_fn(ys0, xs1) = combine_fn(1, 2) = 2 + ... + ysT = combine_fn(ys(T-1), xsT) = combine_fn(6, 4) = 24 + ys = [1, 2, 6, 24] + + This creates a recursive data dependency structure where each output yst + depends on all prior inputs xs0 through xst. The dependency can be visualized as: + + Level 0 (Input): xs0 xs1 xs2 xs3 xs4 + \ / | | | + \ / | | | + Level 1: ys1 ───────┘ | | + \ / | + \ / | + Level 2: ys2 ────────┘ | + \ / + \ / + Level 3: ys3 ────────────┘ + \ + \ + Level 4: ys4 + + + We could get the following backward gradient graph: + + + Level 0 (output): g_xs0 g_xs1 g_xs2 g_xs3 g_xs4 + \ / | | | + \ / | | | + Level 1: gl_ys1 ─> g_ys1 ──────┘ | | + \ / | + \ / | + Level 2: gl_ys2 ─> g_ys2 ────────┘ | + \ / + \ / + Level 3: gl_ys3 ─> g_ys3 ────────────┘ + \ + \ + Level 4: gl_ys4 ─> g_ys4, + + where gl_y1 is the gradient of the loss with respect to ys1 and the input of backward. + + To calculate the gradients of the inputs, the chain rule suggests: + + g_xs0 = g_ys1 + g_xs1 = g_ys1 * bw(ys0, xs1) = g_ys1 * bwxs01 + g_xs2 = g_ys2 * bw(ys1, xs2) = g_ys2 * bwxs12 + g_xs3 = g_ys3 * bw(ys2, xs3) = g_ys3 * bwxs23 + g_xs4 = g_ys4 * bw(ys3, xs4) = g_ys4 * bwxs34 + + Notice the bw(...) is just the single step bw (instantaneous gradients), whose formula can be computed from combine_fn. + For example bw(ys3, xs4) (also abbreviated with bwxs34) computes the gradients ∂/∂xs4 combine_fn(ys3, xs4). + Similarly, bw(ys4, ys3) (also abbreviated with bwys43) computes the gradients ∂/∂ys3 combine_fn(ys3, xs4). + + Let's break down how to calculate g_ys by recursively substituting the unknowns: + + g_ys1 = gl_ys1 + g_ys2 * bw(ys2, ys1) + = gl_ys1 + (gl_ys2 + g_ys3 * bw(ys3, ys2)) * bw(ys2, ys1) + = gl_ys1 + gl_ys2 * bw(ys2, ys1) + g_ys3 * bw(ys3, ys2) * bw(y2, y1) + = gl_ys1 + gl_ys2 * bw(ys2, ys1) + gl_ys3 * bw(ys3, ys2) * bw(y2, y1) \ + + g_ys4 * bw(ys4, ys3) * bw(ys3, ys2) * bw(ys2, ys1) + = gl_ys1 + gl_ys2 * bw(ys2, ys1) + gl_ys3 * bw(ys3, ys2) * bw(y2, y1) \ + + gl_ys4 * bw(ys4, ys3) * bw(ys3, ys2) * bw(ys2, ys1) + + Let's do the same for all the g_ys: + g_ys2 = gl_ys2 + gl_ys3 * bw(ys3, ys2) + gl_y4 * bw(ys4, ys3) * bw(ys3, ys2) + g_ys3 = gl_ys3 + gl_ys4 * bw(ys4, ys3) + g_ys4 = gl_ys4 + + Notice that the above can be re-written as columnwise multiplication of y_mat and gl_ys: + + g_ys1 1, bwys21, bwys321, bwys4321 gl_ys1 + g_ys2 = 0, 1 , bwys321, bwys4321 . gl_ys2 + g_ys3 0, 0 , 1 , bwys4321 gl_ys3 + g_ys4 0, 0 , 0 , 1 gl_ys4, + + where bwys21 is an abbreviation for bw(ys2, ys1), + bwys321 is an abbreviation for bw(ys3, ys2) * bw(ys2, ys1) so on and so forth. + + We could effectively compute the upper triangular matrix y_mat with: + cumprod([1, bwys21, bwys32, bwys43]) then masking out the values as needed. + Thus, only [1, bwys21, bwys32, bwys43] are required to compute the y_mat. + + + References: https://justintchiu.com/blog/pscan_diff/ + + NOTE: [associative_scan autograd implementation] + + The forward of associative_scan can be computed with the following steps: + + 1.) Compute the forward output of the associative_scan + ys = associative_scan(combine_fn, xs, additional_inputs) + + The backward of associative_scan can be computed with the following steps: + + 2.) Prepare the backward graph + We prepare the backward graph to be used in the backward function. + We utilize ``create_bw_fn`` to generate the joint function: + combine_fn_bw = create_bw_fn(combine_fn, operands) + where operands = [ys{t-1}, xst, additional_inputs] + + 3.) Materialize the ``combine_fn_bw`` + This is required because torch.compile and torch.autograd.grad + cannot trace through the joint backward function dynamically. + + 4.) Compute the single step bw (instantaneous gradients) at every step t + bwys{t-1}, bwxst = combine_fn_bw(ys{t-1}, xst, 1.) + Here we pass 1 as the upstream gradient to obtain the local partial derivatives. + + This gives: + bwys = [bw(ys1, ys0), bw(ys2, ys1), ..., bw(ysT, ys{T-1})] + bwxs = [bw(ys1, xs0), bw(ys2, xs1), ..., bw(ys{T-1}, xsT)] + + 5.) Compute the gradient transition matrix y_mat + + As shown in the example above, each input xst affects all later outputs ysi for i ≥ t. + According to the chain rule, each such path contributes a product of local gradients g_ysk. + + For example: + ∂ysT/∂xst = ∂ysT/∂ys{T-1} * ∂ys{T-1}/∂ys{T-2} * ... * ∂ys{t+1}/∂yst * ∂yst/∂xst + = bw(ysT, ys{T-1}) * bw(ys{T-1}, ys{T-2}) * ... * bw(ys{t+1}, yst) * bw(ys{t-1}, xst) + + This motivates the use of a cumulative product over bwys to compute all such paths efficiently. + + We now construct the matrix of gradient transition paths: + + 5.1 Repeat g_y values to form the base matrix + y_mat = [[1, bwys21, bwys32, bwys43], + [1, bwys21, bwys32, bwys43], + [1, bwys21, bwys32, bwys43], + [1, bwys21, bwys32, bwys43]] + + 5.2 Mask the lower triangle (inclusive) with 1s + y_mat = [[1, bwys21, bwys32, bwys43], + [1, 1 , bwys32, bwys43], + [1, 1 , 1 , bwys43], + [1, 1 , 1 , 1 ]] + + 5.3 Apply cumulative product row-wise + y_mat = cumprod(y_mat, dim=1) + Resulting in: + y_mat = [[1, bwys21, bwys32 * bwys21, bwys43 * bwys32 * bwys21], + [1, 1 , bwys32 , bwys43 * bwys32 ], + [1, 1 , 1 , bwys43 ], + [1, 1 , 1 , 1 ]] + + 5.4 Zero out the lower triangle (exclusive) + Final y_mat: + y_mat = [[1, bwys21, bwys32 * bwys21, bwys43 * bwys32 * bwys21], + [0, 1 , bwys32 , bwys43 * bwys32 ], + [0, 0 , 1 , bwys43 ], + [0, 0 , 0 , 1 ]] + + 6.) Scale the y_mat with the upstream gradients gl_ys + scaled_y_mat = y_mat * gl_ys + Each entry now holds the full contribution of ∂L/∂ysj to ∂L/∂xsi via the path through ysj. + + 7.) Reduce the scaled_y_mat with a row-wise sum + summed_y_mat = scaled_y_mat.sum(dim=1) + This accumulates all downstream contributions for each xst. + + 8.) Scale with the instantaneous input gradients bwxs + g_xs = summed_y_mat * bwxs + + This gives the final input gradients: + g_xs = [∂L/∂xs0, ∂L/∂xs1, ..., ∂L/∂xsT] + + NOTE: [scan partial grad handling] + If any element of xs or of the outputs does not require gradients + (i.e., requires_grad=False), then the corresponding gradients will be returned + as tensors of zeros with the same shape as the element. + """ + + @staticmethod + # pyrefly: ignore [bad-override] + def forward( + ctx, + combine_fn, + num_xs, + num_additional_inputs, + *operands, + ): + ctx._num_xs = num_xs + ctx._num_additional_inputs = num_additional_inputs + ctx._combine_fn = combine_fn + xs, additional_inputs = split_into_chunks( + operands, [num_xs, num_additional_inputs] + ) + + scan_length = xs[0].shape[0] + ctx._scan_length = scan_length + + # We snapshot the dispatch keys in forward for materializing the + # the bw_graph in backward. + ctx._fw_include_key_set = torch._C._dispatch_tls_local_include_set() + ctx._fw_exclude_key_set = torch._C._dispatch_tls_local_exclude_set() + + with torch._C._AutoDispatchBelowAutograd(): + # 1.) Compute the forward output of the associative_scan + ys = associative_scan_op(combine_fn, xs, additional_inputs) + save_tensors_and_symints_for_backward(ctx, list(operands) + list(ys)) + + return (*ys,) + + @staticmethod + def backward(ctx, *gl_ys): + r""" + This function computes the gradients of the scan operation. + For a detailed description see the document above. + + Args: + flat_grads (torch.Tensor): The tensor of upstream gradients, or a nested pytree of tensors. + E.g.: Gradient of the loss with respect to the forward output ys + """ + + # The backward of associative_scan is always performed on the first dimension + dim = 0 + scan_length = ctx._scan_length + num_xs = ctx._num_xs + num_additional_inputs = ctx._num_additional_inputs + + # Extract the inputs to the forward path and outputs from the forward path + flat_args = saved_tensors_and_symints(ctx) + xs, additional_inputs, outs = split_into_chunks( + flat_args, [num_xs, num_additional_inputs, num_xs] + ) + ndim = outs[0].ndim + + # First_slice_copy does not keep the original requires_grad flag, + # but we need it here in order to compute the correcte gradients + xs_slices = first_slice_copy_with_grad(itertools.chain(xs, xs)) + + # Construct the operands from the forward, fw_operands + # and the operands for a single event t of the forward, fw_operands_slice + fw_operands = (*xs, *additional_inputs) + fw_operands_slice = (*xs_slices, *additional_inputs) + + # 2.) Prepare the backward graph + combine_fn_bw = create_bw_fn(ctx._combine_fn, fw_operands_slice) + + # 3.) Materialize the ``combine_fn_bw`` + # TODO: we need to materialize the bw graphs because dynamo is unable to + # trace through the joint function when torch.compile torch.autograd.grad. + combine_fn_bw_gm = materialize_as_graph( + combine_fn_bw, + ( + *fw_operands_slice, + *[first_slice_copy(o) for o in outs], + ), + ctx._fw_include_key_set, + ctx._fw_exclude_key_set, + force_enable_grad=True, + ) + + # vmap joint graph over scan dimension to compute the individual + # gradients for each time slice ``t`` in parallel. + # This computation can be parallelized, as these are just the instantaneous gradients and not the full chain-rule + mapped_combine_fn_bw_gm = torch.vmap(combine_fn_bw_gm, 0, 0) + + # 4.) Compute the single step bw (instantaneous gradients) at every step ``t`` + # Use a ones_like tensor in order not to scale the bwyst and bwxst, + # with the upstream gradients yet. + # Note: All bwyst and bwxst are computed in parallel, thus the tensors bwys and bwxs are the result. + dummy_upstream_grad = (torch.ones_like(x) for x in gl_ys) + grads = mapped_combine_fn_bw_gm( + *(o.roll(1, dim) for o in outs), *fw_operands, *dummy_upstream_grad + ) + bwys, bwxs = split_into_chunks(grads, [num_xs, num_xs]) + + def compute_y_mat(bwys: torch.Tensor) -> torch.Tensor: + # Prepare a ones and a zeros helper mask in order to easily compute the y_mat + def compute_helper_tril_mask(diagonal): + def expand_masks(mask): + for _ in range(ndim - 1): + mask = mask.unsqueeze(-1) + return mask + + tril_mask = torch.tril( + torch.ones( + scan_length, scan_length, device=bwys.device, dtype=torch.bool + ), + diagonal=diagonal, + ) + tril_mask = expand_masks(tril_mask) + tril_mask = tril_mask.expand(-1, -1, *bwys.shape[1:]) + return tril_mask + + # The ones mask is used to fill the main diagonal and all elements below it with 1s + ones_mask = compute_helper_tril_mask(0) + + # The zero mask is used to set all elements below the main diagonal to 0 + zeros_mask = compute_helper_tril_mask(-1) + + # 5.1) Repeat the elements of bwys to form the square matrix + y_mat = bwys.unsqueeze(dim).repeat_interleave(scan_length, dim) + + # 5.2) Fill the lower triangular part, including the diagonal, + # of the h_mat with 1s. I.e., use the ones_mask to fill with 1s. + y_mat.masked_fill_(ones_mask, 1.0) + + # 5.3) Compute the cumulative products across dim + 1 + y_mat = y_mat.cumprod(dim=dim + 1) + + # 5.4) Replace the elements we filled with 1s before with 0s + y_mat.masked_fill_(zeros_mask, 0.0) + + return y_mat + + def compute_grad(bwxs, bwys, gl_ys): + # Set the first gradient component of bwxs to 1.0, per definition. + torch.select(bwxs, dim, 0).fill_(1.0) + + # 5.) Compute the gradient transition matrix + y_mat = compute_y_mat(bwys) + + # 6.) scale the y_mat with the upstream gradients gl_ys + scaled_y_mat = y_mat * gl_ys + + # 7.) Reduce the y_mat with sum along the columns to get the total contributions for xs_t + summed_y_mat = scaled_y_mat.sum(dim + 1) + + # 8.) Scale with the bwxs to obtain the final gradients g_xs + g_xs = summed_y_mat * bwxs + + return g_xs + + # Stack all leaves of the gradients along the first dimension. + # This is useful as later the gradients of those leaves can be computed in parallel. + bwxs_stacked_leaves = torch.stack(bwxs) + bwys_stacked_leaves = torch.stack(bwys) + gl_ys_stacked_leaves = torch.stack(gl_ys) + + # The compute_grad function is parallelized across all individual leaves of xs + # as these gradients can be computed independently from each other + # TODO: torch.vmap may create composability issues + compute_grad_mapped = torch.vmap(compute_grad, 0, 0) + + g_xs = compute_grad_mapped( + bwxs_stacked_leaves, bwys_stacked_leaves, gl_ys_stacked_leaves + ) + + # TODO: Currently the gradients for the additional_inputs are not computed properly + return *[None] * 3, *g_xs, *[None] * num_additional_inputs + + +@associative_scan_op.py_autograd_impl +def associative_scan_autograd(combine_fn, xs, additional_inputs): + num_xs = len(xs) + num_additional_inputs = len(additional_inputs) + + if num_additional_inputs > 0: + raise RuntimeError( + "Associative_scan does currently not support gradients for lifted parameters!" + ) + + flat_out = AssociativeScanAutogradOp.apply( + combine_fn, + num_xs, + num_additional_inputs, + *(tuple(xs) + tuple(additional_inputs)), + ) + return (*flat_out,) + + +@associative_scan_op.py_impl(ProxyTorchDispatchMode) +def associative_scan_proxy_mode(mode, combine_fn, xs, additional_inputs): + return trace_associative_scan( + mode, associative_scan_op, combine_fn, xs, additional_inputs + ) + + +@associative_scan_op.py_impl(FakeTensorMode) +def assoiciative_scan_fake_tensor_mode(mode, combine_fn, xs, additional_inputs): + with mode: + return tuple(x.clone() for x in xs) + + +@associative_scan_op.py_functionalize_impl +def associative_scan_functionalize(ctx, combine_fn, xs, additional_inputs): + from torch._higher_order_ops.utils import _check_alias_and_mutation + + unwrapped_xs = ctx.unwrap_tensors(xs) + unwrapped_additional_inputs = ctx.unwrap_tensors(additional_inputs) + with ctx.redispatch_to_next(): + functional_combine_fn = ctx.functionalize( + _maybe_run_with_interpreter(combine_fn) + ) + pre_dispatch = hasattr(ctx, "mode") and ctx.mode.pre_dispatch + sample_unwrapped_xs_sliced = [ + first_slice_copy(inp) for inp in itertools.chain(unwrapped_xs, unwrapped_xs) + ] + sample_inputs = list( + itertools.chain( + sample_unwrapped_xs_sliced, + unwrapped_additional_inputs, + ) + ) + _check_alias_and_mutation( + combine_fn, sample_inputs, "associative_scan", pre_dispatch + ) + ret = associative_scan_op( + functional_combine_fn, + unwrapped_xs, + unwrapped_additional_inputs, + ) + return ctx.wrap_tensors(ret) + + +def _fake_associative_scan(combine_fn, xs, dim, reverse=False): + inp_leaves, spec = pytree.tree_flatten(xs) + result_flat: list[Any] = [] + num_leaves = len(inp_leaves) + op = reversed if reverse else lambda x: x + + for ind in op(range(inp_leaves[0].size(dim))): + r = [ + inp_leaves[leave_ind][(slice(None),) * dim + (ind,)] + for leave_ind in range(num_leaves) + ] + if (ind > 0 and not reverse) or ( + ind < (inp_leaves[0].size(dim) - 1) and reverse + ): + r = combine_fn( + pytree.tree_unflatten(result_flat[-1], spec), + pytree.tree_unflatten(r, spec), + ) + r_flat, _ = pytree.tree_flatten(r) + result_flat.append(r_flat) + + results = [ + torch.stack([e[leave_ind] for e in op(result_flat)], dim) + for leave_ind in range(num_leaves) + ] + return pytree.tree_unflatten(results, spec) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/auto_functionalize.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/auto_functionalize.py new file mode 100644 index 0000000000000000000000000000000000000000..3f93036836eecdd47d408452d476f2e1b8fd1b19 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/auto_functionalize.py @@ -0,0 +1,1021 @@ +# mypy: allow-untyped-defs +import warnings +from abc import ABC, abstractmethod +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Any, get_args, Optional, Union + +import torch +import torch._library.utils as library_utils +import torch.utils._pytree as pytree +from torch import Tensor +from torch._C import DispatchKey +from torch._higher_order_ops.utils import ( + _has_gen_schema, + call_op, + HopInstance, + HopSchema, + materialize_callable_in_args, + unique_graph_id, +) +from torch._ops import HigherOrderOperator, OperatorBase, OpOverload +from torch._prims_common import clone_preserve_strides +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ( + disable_proxy_modes_tracing, + ProxyTorchDispatchMode, + track_tensor_tree, +) + + +class SchemaHolder: + def __init__(self, schema: torch.FunctionSchema): + self.schema = schema + + def __eq__(self, other): + return self.schema == other.schema + + def __hash__(self) -> int: + return hash(self.schema) + + @classmethod + def from_tree_spec(cls, tree_spec: pytree.TreeSpec): + assert tree_spec is not None + return cls(pytree.tree_unflatten([], tree_spec).schema) + + +# register_constant allows us to get a tree_spec from pytree.tree_flatten(SchemaHolder(FunctionSchema)). +# The tree_spec is proxable in the graph and we can get back the schema via +# schema = pytree.tree_unflatten([], tree_spec).schema +pytree.register_constant(SchemaHolder) + + +def get_base(tensor): + if torch.is_inference_mode_enabled(): + return tensor._inference_mode_base + else: + return tensor._base + + +class ViewInfo(ABC): + base_index: int + + def __init__(self, base_index): + self.base_index = base_index + + @abstractmethod + def regenerate_view(self, bases_list: list[Tensor]): + pass + + +@dataclass +class AsStridedViewInfo(ViewInfo): + size: Sequence[Union[int, torch.SymInt]] + stride: Sequence[Union[int, torch.SymInt]] + storage_offset: int + + def __init__(self, base_index, size, stride, storage_offset): + super().__init__(base_index) + self.size = size + self.stride = stride + self.storage_offset = storage_offset + + def regenerate_view(self, bases_list: list[Tensor]): + return torch.as_strided( + bases_list[self.base_index], + self.size, + self.stride, + self.storage_offset, + ) + + +@dataclass +class SliceViewInfo(ViewInfo): + dim: Union[int, torch.SymInt] + start: Union[int, torch.SymInt] + end: Union[int, torch.SymInt] + + def __init__(self, base_index, dim, start, end): + super().__init__(base_index) + self.dim = dim + self.start = start + self.end = end + + def regenerate_view(self, bases_list: list[Tensor]): + return torch.ops.aten.slice.Tensor( + bases_list[self.base_index], self.dim, self.start, self.end + ) + + +@dataclass +class AliasViewInfo(ViewInfo): + def __init__(self, base_index): + super().__init__(base_index) + + def regenerate_view(self, bases_list: list[Tensor]): + return torch.ops.aten.alias.default(bases_list[self.base_index]) + + +@dataclass +class NotView(ViewInfo): + def __init__(self, base_index): + super().__init__(base_index) + + def regenerate_view(self, bases_list: list[Tensor]): + return bases_list[self.base_index] + + +def is_alias(base, tensor): + from torch.fx.experimental.symbolic_shapes import statically_known_true, sym_eq + + return all( + statically_known_true(a) + for a in [ + sym_eq(base.storage_offset(), tensor.storage_offset()), + sym_eq(base.stride(), tensor.stride()), + sym_eq(base.size(), tensor.size()), + ] + ) + + +# return None or (dim, start, end) +def try_use_slice(base, tensor): + from torch.fx.experimental.symbolic_shapes import statically_known_true, sym_eq + + # This condition should never be triggered. + if is_alias(base, tensor): + return (0, 0, base.size()[0]) + + # TODO is there cases can we use slice even if stride or len(sizes) are not equal? + if not statically_known_true(sym_eq(tensor.stride(), base.stride())): + return None + if not statically_known_true(sym_eq(len(tensor.size()), len(base.size()))): + return None + + dim = None + count = 0 + for i in range(len(tensor.size())): + if base.size()[i] != tensor.size()[i]: + dim = i + count = count + 1 + if count != 1: + return None + + if tensor.storage_offset() % tensor.stride()[dim] != 0: + return None + start = tensor.storage_offset() // tensor.stride()[dim] + end = start + tensor.size()[dim] + return (dim, start, end) + + +def write_view_information_to_args( + mutable_arg_names: list[str], + mutable_arg_types: list[torch.Type], + kwargs: dict[str, Any], + arg_to_base_index: dict[str, Any], +): + """ + This function writes the view information into kwargs. It reads mutable_args from kwargs. + and uses arg_to_base_index and tensor information to write ViewInfo into kwargs. + mutable_arg_names: mutable custom operator arg names. + mutable_arg_types: mutable custom operator arg types. + kwargs: the original custom operator args. + arg_to_base_index: maps mutable_arg_name to int | [int] that refers to the base tensor that + corresponds to the input tensor + """ + + def write_single_view(prefix: str, tensor: Tensor, base_index: int): + assert f"{prefix}_base_index" not in kwargs + assert f"{prefix}_size" not in kwargs + assert f"{prefix}_stride" not in kwargs + assert f"{prefix}_storage_offset" not in kwargs + + assert f"{prefix}_slice_dim" not in kwargs + assert f"{prefix}_slice_start" not in kwargs + assert f"{prefix}_slice_end" not in kwargs + + def use_as_strided(tensor): + kwargs[f"{prefix}_size"] = tensor.size() + kwargs[f"{prefix}_stride"] = tensor.stride() + kwargs[f"{prefix}_storage_offset"] = tensor.storage_offset() + + def use_slice(dim, start, end): + kwargs[f"{prefix}_slice_dim"] = dim + kwargs[f"{prefix}_slice_start"] = start + kwargs[f"{prefix}_slice_end"] = end + + def use_alias(): + kwargs[f"{prefix}_alias"] = True + + # The start if the function + if tensor is None: + kwargs[f"{prefix}_base_index"] = None + else: + base = get_base(tensor) + kwargs[f"{prefix}_base_index"] = base_index + if base is None: + # no need to add anything else other than _base_index + return + elif is_alias(base, tensor): + use_alias() + elif (slice_info := try_use_slice(base, tensor)) is not None: + use_slice(*slice_info) + else: + use_as_strided(tensor) + + for arg_name, arg_type in zip(mutable_arg_names, mutable_arg_types): + arg = kwargs[arg_name] + if library_utils.is_tensorlist_like_type(arg_type): + if arg is None: + kwargs[f"_{arg_name}_length"] = None + else: + kwargs[f"_{arg_name}_length"] = len(arg) + for i, elem in enumerate(arg): + write_single_view( + f"_{arg_name}_{i}", elem, arg_to_base_index[arg_name][i] + ) + + elif library_utils.is_tensor_like_type(arg_type): + write_single_view( + f"_{arg_name}", + kwargs[arg_name], + arg_to_base_index.get(arg_name), # type: ignore[arg-type] + ) + else: + raise RuntimeError(f"Unsupported type {arg_type}") + + +# Returns a dict of arg_name -> ViewInfo | [ViewInfo] +def read_view_information_from_args( + mutable_arg_names: list[str], + mutable_arg_types: list[torch.Type], + kwargs: dict[str, Any], + all_bases: list[Tensor], +): + """ + This reads the view information added by `write_view_information_to_args` from kwargs, pop them, + and returns a dict arg_name -> ViewInfo | [ViewInfo](if the input is list). that maps each mutable arg + to its view information. + mutable_arg_names: mutable custom operator arg names. + mutable_arg_types: mutable custom operator arg types. + kwargs : args of auto_functionalize(custom_op, kwargs) + """ + + def get_arg(name): + return kwargs.pop(name) + + def read_single_view(prefix): + base_index = get_arg(f"{prefix}_base_index") + if base_index is None: + return None + elif f"{prefix}_alias" in kwargs: + get_arg(f"{prefix}_alias") + return AliasViewInfo(base_index) + elif f"{prefix}_storage_offset" in kwargs: + # The view is regenerated using as_strided. + size = get_arg(f"{prefix}_size") + stride = get_arg(f"{prefix}_stride") + storage_offset = get_arg(f"{prefix}_storage_offset") + return AsStridedViewInfo(base_index, size, stride, storage_offset) + elif f"{prefix}_slice_dim" in kwargs: + dim = get_arg(f"{prefix}_slice_dim") + start = get_arg(f"{prefix}_slice_start") + end = get_arg(f"{prefix}_slice_end") + return SliceViewInfo(base_index, dim, start, end) + else: + # This means that the argument is the base tensor + return NotView(base_index) + + args_view_info: dict[str, Any] = {} + for arg_name, arg_type in zip(mutable_arg_names, mutable_arg_types): + if library_utils.is_tensorlist_like_type(arg_type): + length = get_arg(f"_{arg_name}_length") + if length is None: + # The whole list is None. + args_view_info[arg_name] = None + else: + args_view_info[arg_name] = [ + read_single_view(f"_{arg_name}_{i}") for i in range(length) + ] + + elif library_utils.is_tensor_like_type(arg_type): + args_view_info[arg_name] = read_single_view(f"_{arg_name}") + else: + raise RuntimeError(f"Unsupported type {arg_type}") + return args_view_info + + +# NOTE: [auto-functionalizing custom ops] +# Users may wish to torch.compile custom ops that mutate their inputs. +# torch.compile will automatically support this op without anyone needing +# to provide a functionalization kernel for it. Here's how. +# +# Let's say we have a hypothetical mylib::sin_(Tensor(a!) x) -> () +# op. First, when FakeTensor sees this op: +# - If the schema says it returns nothing, we can generate a trivial +# FakeTensor rule for it (that returns nothing). +# - Otherwise, the user needs to provide a FakeTensor impl (fake impl) +# +# Next, when Python FunctionalTensor sees the op, it will functionalize +# it by emitting a call to an auto_functionalize(op, ["x"], {"x": ...}) +# HOP and replacing the mutated inputs with corresponding outputs of this HOP. +# This HOP effectively runs the functional version of the op when +# called: it clones inputs that will be mutated, runs the op, and +# then returns (output, Tensors with the new values) +# +# auto_functionalize_v2 is an improved version of auto_functionalize that better handle +# re-inplacing views. + + +class AutoFunctionalized(HigherOrderOperator): + """auto_functionalized(_mutable_op, **kwargs) + + This HOP runs a "functional" version of _mutable_op. + + Concretely, it looks at all the arguments that are mutable through + _mutable_op's operator schema, clones those kwargs, runs + `out = _mutable_op(**kwargs)` with the cloned values, and then returns the + operator output concatenated with the cloned values that were mutated. + + We have some restrictions on `_mutable_op`. + See `can_auto_functionalize` for the restrictions. We can likely lift + many of these if users request it. + + The reason why _mutable_op is prefixed with an + underscore is to prevent collisions with kwarg names in **kwargs. + """ + + def __init__(self) -> None: + super().__init__("auto_functionalized", cacheable=True) + + def __call__( + self, + /, + _mutable_op: OpOverload, + **kwargs: Any, + ) -> tuple[Any, tuple[Tensor, ...]]: + assert can_auto_functionalize(_mutable_op) + assert isinstance(kwargs, dict) + return super().__call__(_mutable_op, **kwargs) + + +auto_functionalized = AutoFunctionalized() +auto_functionalized.__module__ = "torch.ops.higher_order" + +auto_functionalized.fallthrough(DispatchKey.AutogradCPU) +auto_functionalized.fallthrough(DispatchKey.AutogradCUDA) + + +_MutableOpType = Union[OpOverload, HigherOrderOperator] + + +class AutoFunctionalizedV2(HigherOrderOperator): + """auto_functionalized_v2(_mutable_op, **kwargs) + + This HOP runs a "functional" version of _mutable_op. + Unlike AutoFunctionalized, this version is improved to better handle + view tensors. This version is only used in non export mode. + """ + + def __init__(self) -> None: + super().__init__("auto_functionalized_v2", cacheable=True) + + def __call__( + self, + /, + _mutable_op: _MutableOpType, + **kwargs: Any, + ) -> tuple[Any, tuple[Tensor, ...]]: + _op_to_check: Optional[Union[OpOverload, HopInstance]] = None + if isinstance(_mutable_op, HigherOrderOperator): + _op_to_check = HopInstance( + _mutable_op, + SchemaHolder.from_tree_spec(kwargs.get("_op_schema")).schema, # type: ignore[arg-type] + ) + else: + _op_to_check = _mutable_op + + assert _op_to_check is not None + assert can_auto_functionalize(_op_to_check) + assert isinstance(kwargs, dict) + return super().__call__(_mutable_op, **kwargs) + + +auto_functionalized_v2 = AutoFunctionalizedV2() +auto_functionalized_v2.__module__ = "torch.ops.higher_order" + +auto_functionalized_v2.fallthrough(DispatchKey.AutogradCPU) +auto_functionalized_v2.fallthrough(DispatchKey.AutogradCUDA) + + +def can_auto_functionalize( + op: Union[OperatorBase, HopInstance], +) -> bool: + if isinstance(op, HopInstance): + # HOPs that implement gen_schema and schema is not functional are auto_functionalizable. + if not _has_gen_schema(op._op): + return False + + else: + if not isinstance(op, OpOverload): + return False + + if torch._library.utils.is_builtin(op): + # We control the built-ins. These may (in rare cases) + # do input metadata mutation (which we have banned on custom ops) + return False + + schema = op._schema + if not schema.is_mutable: + return False + schema = op._schema + + for arg in schema.arguments: + if arg.alias_info is None: + continue + if not arg.alias_info.is_write: + continue + if torch._library.utils.is_tensor_like_type(arg.type): + continue + if torch._library.utils.is_tensorlist_like_type(arg.type): + continue + return False + + if len(schema.returns) == 1 and isinstance(schema.returns[0].type, torch.NoneType): + # Skip schema returns -> None + return True + if isinstance(op, OpOverload): + # The returns of OpOverload must not alias anything + for ret in schema.returns: + if ret.alias_info is None and type(ret.type) is torch.TensorType: + continue + # Not yet supported: List[Tensor] return. + return False + if torch._C._dispatch_has_kernel_for_dispatch_key(op.name(), "Functionalize"): + return False + return True + + +def get_mutable_args_from_schema( + schema: torch.FunctionSchema, +) -> tuple[list[str], list[torch.Type]]: + """ + Returns the list of argument names that get mutated according to the + schema and their types. + """ + mutable_args_names = [ + arg.name + for arg in schema.arguments + if arg.alias_info is not None and arg.alias_info.is_write + ] + + mutable_args_types = [ + arg.type + for arg in schema.arguments + if arg.alias_info is not None and arg.alias_info.is_write + ] + return mutable_args_names, mutable_args_types # type: ignore[return-value] + + +def get_mutable_args(op: OpOverload) -> tuple[list[str], list[torch.Type]]: + return get_mutable_args_from_schema(op._schema) + + +def do_auto_functionalize( + mode: "torch._subclasses.functional_tensor.FunctionalTensorMode", + op: OpOverload, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> Any: + """Functionalizes a call to op(*args, **kwargs) by emitting a call to + `outs = auto_functionalized(op, normalized_kwargs)` + and replacing the mutated (args, kwargs) with the corresponding outputs. + + The normalized_kwargs are just the (args, kwargs), but all in kwarg form. + This makes handling easier for the auto_functionalized HOP. + """ + from torch._subclasses.functional_tensor import PythonFunctionalizeAPI + + ctx = PythonFunctionalizeAPI(mode=mode) + + # All of the (args, kwargs), but all as kwargs. The names for the + # args come from the schema. This makes it easier for us to work with them. + normalized_kwargs = {} + schema = op._schema + for idx, arg in enumerate(schema.arguments): + # NB: torch_dispatch kwargs are the args defined as kwarg-only in the schema + if arg.name in kwargs: + normalized_kwargs[arg.name] = kwargs[arg.name] + elif idx < len(args): + # if its out of bounds we don't need to do anything + # as it means the optional arg was passed with its default + # value + normalized_kwargs[arg.name] = args[idx] + else: + normalized_kwargs[arg.name] = arg.default_value + + unwrapped_kwargs = ctx.unwrap_tensors(normalized_kwargs) # type: ignore[arg-type] + if "self" in unwrapped_kwargs or "self_" in unwrapped_kwargs: + warnings.warn( + "Using `self` or `self_` as an argument in the definition of custom ops may lead to ambiguous parsing. " + "Please consider using a different name for this argument to avoid potential issues.", + stacklevel=2, + ) + with ctx.redispatch_to_next(): + unwrapped_outs = auto_functionalized( + op, + **unwrapped_kwargs, # type: ignore[arg-type] + ) + + # List of the name of args that get mutated (according to the schema) + mutable_args_names, _ = get_mutable_args(op) + + unwrapped_actual_out: Union[Any, tuple[Any]] = unwrapped_outs[ + : -len(mutable_args_names) + ] + unwrapped_mutable_out = unwrapped_outs[-len(mutable_args_names) :] + + if len(op._schema.returns) == 0: + assert unwrapped_actual_out[0] is None + unwrapped_actual_out = None + elif len(op._schema.returns) == 1: + assert len(unwrapped_actual_out) == 1 + unwrapped_actual_out = unwrapped_actual_out[0] + else: + assert len(unwrapped_actual_out) == len(op._schema.returns) + + for name, unwrapped_out in zip(mutable_args_names, unwrapped_mutable_out): + # Can be None if input was `Tensor(a!)?` + if unwrapped_out is None: + continue + + # We only handle Tensor or List[Tensor] here for now. + def sync_update(o, orig_arg): + ctx.replace(orig_arg, o) + ctx.commit_update(orig_arg) + ctx.sync(orig_arg) + + orig_arg = normalized_kwargs[name] + + if isinstance(unwrapped_out, torch.Tensor): + sync_update(unwrapped_out, orig_arg) + elif isinstance(unwrapped_out, list) and all( + isinstance(o, torch.Tensor) for o in unwrapped_out + ): + assert len(orig_arg) == len(unwrapped_out) + for orig_a, o in zip(orig_arg, unwrapped_out): + sync_update(o, orig_a) + else: + raise RuntimeError( + f"unsupported type for auto-functionalization: {unwrapped_out}" + ) + + return ctx.wrap_tensors(unwrapped_actual_out) # type: ignore[arg-type] + + +# Wrapper for GraphModule that applies functionalization during execution to enable +# epilogue graph inlining and better fusion opportunities in subgraphs +# When tracing this wrapper, we'll get a graph module with epilogue. +# +# We want to hash it according to the original graph module, so that when we go +# from Functional mode -> fake mode for multiple invoke_subgraph calls that share, +# the same inner graph module, we can hit the cache. +class FunctionalCallableWithEpilogue: + def __init__(self, orig_callable: Callable): + self.orig_callable = orig_callable + + def __call__(self, *args, **kwargs): + # We call torch.func.functionalize. This allows us to inline the epilogue graph. + # Inlining has the benefit of allowing easiser fusion inside subgraph. + # Though the epilogue graph contains copy_, it is OK because inductor can handle it + # and this is also how we have been supporting top-level graph input mutation. + return tuple(torch.func.functionalize(self.orig_callable)(*args, **kwargs)) + + def __hash__(self): + return id(self.orig_callable) + + +def do_auto_functionalize_v2( + mode: "torch._subclasses.functional_tensor.FunctionalTensorMode", + op: Union[OpOverload, HopInstance], + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> Any: + from torch._subclasses.functional_tensor import PythonFunctionalizeAPI + + ctx = PythonFunctionalizeAPI(mode=mode) + + # All of the (args, kwargs), but all as kwargs. The names for the + # args come from the schema. This makes it easier for us to work with them. + normalized_kwargs = {} + + schema = op._schema + # pyrefly: ignore [bad-assignment] + op = op._op if isinstance(op, HopInstance) else op + assert isinstance(op, get_args(_MutableOpType)) + + def _functionalize_callable(arg: Any): + if callable(arg): + return FunctionalCallableWithEpilogue(arg) + return arg + + args, kwargs = pytree.tree_map(_functionalize_callable, (args, kwargs)) + + for idx, arg in enumerate(schema.arguments): + # NB: torch_dispatch kwargs are the args defined as kwarg-only in the schema + if arg.name in kwargs: + normalized_kwargs[arg.name] = kwargs[arg.name] + elif idx < len(args): + # if its out of bounds we don't need to do anything + # as it means the optional arg was passed with its default + # value + normalized_kwargs[arg.name] = args[idx] + else: + normalized_kwargs[arg.name] = arg.default_value + + # List of the name of args that get mutated (according to the schema) + mutable_args_names, mutable_args_types = get_mutable_args_from_schema(schema) + + # A list of all bases of mutable args without duplication + all_bases = [] + all_bases_addresses: list[int] = [] + + # Map arg_name to the index of its base in all_bases. + arg_to_base_index: dict[str, Any] = {} + + def update_dict(tensor, arg_name, index=None): + base = tensor if get_base(tensor) is None else get_base(tensor) + + def set_result(base_index): + if index is None: + arg_to_base_index[arg_name] = base_index + else: + arg_to_base_index[arg_name][index] = base_index + + if not all_bases_addresses.__contains__(base._cdata): + all_bases_addresses.append(base._cdata) + all_bases.append(base) + set_result(len(all_bases) - 1) + else: + set_result(all_bases_addresses.index(base._cdata)) + + for arg_name in mutable_args_names: + arg = normalized_kwargs[arg_name] + if arg is None: + continue + + if isinstance(arg, list): + arg_to_base_index[arg_name] = {} + for i, tensor in enumerate(arg): + if tensor is None: + arg_to_base_index[arg_name].append(None) + continue + + update_dict(tensor, arg_name, i) + + else: + update_dict(arg, arg_name) + + # add view_meta for each args into unwrapped_kwargs. + write_view_information_to_args( + mutable_args_names, + mutable_args_types, + normalized_kwargs, + arg_to_base_index, + ) + + # remove mutated args from the kwargs (its a function of _all_bases now) + for arg_name in mutable_args_names: + del normalized_kwargs[arg_name] # type: ignore[arg-type] + + unwrapped_kwargs = ctx.unwrap_tensors(normalized_kwargs) # type: ignore[arg-type] + if "self" in unwrapped_kwargs or "self_" in unwrapped_kwargs: + warnings.warn( + "Using `self` or `self_` as an argument in the definition of custom ops may lead to ambiguous parsing. " + "Please consider using a different name for this argument to avoid potential issues.", + stacklevel=2, + ) + all_basis_unwrapped = ctx.unwrap_tensors(all_bases) + + assert "_all_bases" not in unwrapped_kwargs, (op, unwrapped_kwargs) + auto_func_kwargs = dict(unwrapped_kwargs, _all_bases=all_basis_unwrapped) + if isinstance(op, HigherOrderOperator): + assert "_ops_schema" not in unwrapped_kwargs, (op, unwrapped_kwargs) + # We pass in the tree_spec of tree_flatten(SchemaHolder) to make it proxable + auto_func_kwargs.update( + {"_op_schema": pytree.tree_flatten(SchemaHolder(schema))[1]} + ) + + with ctx.redispatch_to_next(): + unwrapped_outs = auto_functionalized_v2( + op, + **auto_func_kwargs, # type: ignore[arg-type] + ) + + unwrapped_actual_out: Union[Any, tuple[Any]] = ( + unwrapped_outs if len(all_bases) == 0 else unwrapped_outs[: -len(all_bases)] + ) + + unwrapped_mutable_out = ( + [] if len(all_bases) == 0 else unwrapped_outs[-len(all_bases) :] + ) + + if isinstance(op, HigherOrderOperator): + assert len(schema.returns) > 0, ( + f"hop is expected to return at least one output {schema}." + ) + assert len(unwrapped_actual_out) == len(schema.returns) + else: + if len(schema.returns) == 0: + assert unwrapped_actual_out[0] is None + unwrapped_actual_out = None + elif len(schema.returns) == 1: + assert len(unwrapped_actual_out) == 1 + unwrapped_actual_out = unwrapped_actual_out[0] + else: + assert len(unwrapped_actual_out) == len(schema.returns) + + for orig_arg, unwrapped_out in zip(all_bases, unwrapped_mutable_out): + # Can be None if input was `Tensor(a!)?` + if unwrapped_out is None: + continue + + # We only handle Tensor or List[Tensor] here for now. + def sync_update(o, orig_arg): + ctx.replace(orig_arg, o) + ctx.commit_update(orig_arg) + ctx.sync(orig_arg) + + if isinstance(unwrapped_out, torch.Tensor): + sync_update(unwrapped_out, orig_arg) + elif isinstance(unwrapped_out, list) and all( + isinstance(o, torch.Tensor) for o in unwrapped_out + ): + assert len(orig_arg) == len(unwrapped_out) + for orig_a, o in zip(orig_arg, unwrapped_out): + sync_update(o, orig_a) + else: + raise RuntimeError( + f"unsupported type for auto-functionalization: {unwrapped_out}" + ) + + return ctx.wrap_tensors(unwrapped_actual_out) # type: ignore[arg-type] + + +# auto_functionalize functions +@auto_functionalized.py_impl(DispatchKey.CompositeExplicitAutograd) +def auto_functionalized_dense( + _mutable_op: OpOverload, + _only_clone_these_tensors: Optional[tuple[str, ...]] = None, + **kwargs: Any, +) -> tuple[Any, tuple[Tensor, ...]]: + new_kwargs = dict(**kwargs) + result = [] + + _mutable_args_names, _ = get_mutable_args(_mutable_op) + for name in _mutable_args_names: + if ( + _only_clone_these_tensors is not None + and name not in _only_clone_these_tensors + ): + new_kwargs[name] = kwargs[name] + else: + new_kwargs[name] = ( + [clone_preserve_strides(x) for x in kwargs[name]] + if kwargs[name] is not None and isinstance(kwargs[name], list) + else ( + clone_preserve_strides(kwargs[name]) + if kwargs[name] is not None + else None + ) + ) + result.append(new_kwargs[name]) + out = _mutable_op(**new_kwargs) + + if isinstance(out, tuple): + return (*out, *result) # type: ignore[return-value] + else: + return (out, *result) # type: ignore[return-value] + + +@auto_functionalized.py_impl(FakeTensorMode) +def auto_functionalized_fake( + mode, + _mutable_op: OpOverload, + **kwargs: Any, +) -> tuple[Any, tuple[Tensor, ...]]: + with mode: + result = auto_functionalized_dense( + _mutable_op, _only_clone_these_tensors=None, **kwargs + ) + return result + + +@auto_functionalized.py_impl(ProxyTorchDispatchMode) +def auto_functionalized_proxy( + mode, + _mutable_op: OpOverload, + **kwargs: Any, +) -> tuple[Any, tuple[Tensor, ...]]: + with disable_proxy_modes_tracing(): + out = auto_functionalized(_mutable_op, **kwargs) + + proxy_kwargs = pytree.tree_map(mode.tracer.unwrap_proxy, kwargs) + out_proxy = mode.tracer.create_proxy( + "call_function", + auto_functionalized, + (_mutable_op,), + proxy_kwargs, + ) + result = track_tensor_tree(out, out_proxy, constant=None, tracer=mode.tracer) + return result + + +@auto_functionalized.py_functionalize_impl +def auto_functionalized_func(ctx, _mutable_op, **kwargs): + unwrapped_kwargs = ctx.unwrap_tensors(kwargs) + with ctx.redispatch_to_next(): + result = auto_functionalized(_mutable_op, **unwrapped_kwargs) + return ctx.wrap_tensors(result) + + +# auto_functionalized_v2 functions +@auto_functionalized_v2.py_impl(DispatchKey.CompositeExplicitAutograd) +def auto_functionalized_v2_dense( + _mutable_op: _MutableOpType, + _only_clone_these_bases: Optional[tuple[int, ...]] = None, + **kwargs: Any, +) -> tuple[Any, tuple[Tensor, ...]]: + _all_bases: list[Tensor] = kwargs.pop("_all_bases", []) + if _only_clone_these_bases is None: + _only_clone_these_bases = tuple(range(len(_all_bases))) + + if isinstance(_mutable_op, OpOverload): + schema: torch._C.FunctionSchema = _mutable_op._schema + else: + schema = pytree.tree_unflatten([], kwargs.pop("_op_schema")).schema + + if isinstance(_mutable_op, OpOverload): + _callable_op: Union[HopInstance, OpOverload] = _mutable_op + else: + assert isinstance(schema, HopSchema) + _callable_op = HopInstance(_mutable_op, schema) + + op_kwargs_new, all_bases_new = _generate_new_op_kwargs_from_bases( + schema, + kwargs, + _all_bases, + _only_clone_these_bases, + ) + + out = call_op( + _callable_op, + tuple(), + op_kwargs_new, + ) + + if isinstance(out, tuple): + return (*out, *all_bases_new) # type: ignore[return-value] + else: + return (out, *all_bases_new) # type: ignore[return-value] + + +def _generate_new_op_kwargs_from_bases( + schema, kwargs, all_bases, _only_clone_these_bases +): + mutable_args_names, mutable_args_types = get_mutable_args_from_schema(schema) + args_view_info = read_view_information_from_args( + mutable_args_names, mutable_args_types, kwargs, all_bases + ) + + def maybe_copy(i, t): + if t is None: + return None + if i in _only_clone_these_bases: + return clone_preserve_strides(t) + else: + return t + + all_bases_new = [maybe_copy(i, t) for i, t in enumerate(all_bases)] + + # create new args + new_kwargs = dict(**kwargs) + + # re-generate all inputs from all_bases_new using args_view_info and add them to new_kwargs. + for arg_name in mutable_args_names: + if args_view_info[arg_name] is None: + new_kwargs[arg_name] = None + elif isinstance(args_view_info[arg_name], list): + new_kwargs[arg_name] = [] + for i, elem in enumerate(args_view_info[arg_name]): + if elem is None: + new_kwargs[arg_name].append(None) + else: + view_info = args_view_info[arg_name][i] + new_kwargs[arg_name].append( + view_info.regenerate_view(all_bases_new) + ) + else: + new_kwargs[arg_name] = args_view_info[arg_name].regenerate_view( + all_bases_new + ) + + return new_kwargs, all_bases_new + + +@auto_functionalized_v2.py_impl(FakeTensorMode) +def auto_functionalized_v2_fake( + mode, + _mutable_op: _MutableOpType, + **kwargs: dict[str, Any], +) -> tuple[Any, tuple[Tensor, ...]]: + with mode: + result = auto_functionalized_v2_dense( + _mutable_op, _only_clone_these_bases=None, **kwargs + ) + return result + + +@auto_functionalized_v2.py_impl(ProxyTorchDispatchMode) +def auto_functionalized_v2_proxy( + mode, + _mutable_op: _MutableOpType, + **kwargs: Any, +) -> tuple[Any, tuple[Tensor, ...]]: + if isinstance(_mutable_op, HigherOrderOperator): + # Note [materialize callable inputs as graph] + # Below code materializes the callable inputs to the hop as graph modules. + # kwargs may contain general callables, that are not proxable e.g. FunctionWithNoFreeVars + # this could happen when we auto_functionalize the backward of the hop, + # where backward fn is a callablle that wraps forward graph module. + # This function materialize the callable args according to the schema of the hop. + + # We cannot materialize the callables in kwargs directly because the inputs to callable + # vary from hops to hop. To make the materialiation process generic to all hops, + # we trace a function that wraps the hop and let each hop itself figure out how to trace + # its callable inputs. Then we look at the schema of the traced hop node and replace the + # callable in original kwarg with the traced subgraphs. + # + # Specifically, we first trace a wrapped_fn that calls into the hop. Then we look for the + # hop node in the traced graph and graph module inputs to the hop. Finally, we replace the + # original kwarg's callable with the graph module. + all_bases = kwargs.get("_all_bases", []) + _only_clone_these_bases = kwargs.get("_only_clone_these_bases") + if _only_clone_these_bases is None: + _only_clone_these_bases = tuple(range(len(all_bases))) + + schema = pytree.tree_unflatten([], kwargs.get("_op_schema")).schema # type: ignore[arg-type] + new_kwargs, _ = _generate_new_op_kwargs_from_bases( + schema, + {k: v for k, v in kwargs.items() if k not in ("_all_bases", "_op_schema")}, + all_bases, + _only_clone_these_bases, + ) + + _, materialized_kwargs = materialize_callable_in_args( + HopInstance(_mutable_op, schema), tuple(), new_kwargs + ) + + # Only replace the callabes in kwargs with the materialized subgraphs. + # The rest of the kwargs are kept unchanged. + for k, v in kwargs.items(): + if callable(v): + assert k in materialized_kwargs and isinstance( + materialized_kwargs[k], torch.fx.GraphModule + ) + kwargs[k] = materialized_kwargs[k] + + with disable_proxy_modes_tracing(): + out = auto_functionalized_v2(_mutable_op, **kwargs) + + proxy_kwargs = pytree.tree_map(mode.tracer.unwrap_proxy, kwargs) + + if isinstance(_mutable_op, HigherOrderOperator): + + def _maybe_register_subgraph(val: Any): + if isinstance(val, torch.fx.GraphModule): + _, graph_name = unique_graph_id( + mode, prefix="auto_functionalized_subgraph" + ) + mode.tracer.root.register_module(graph_name, val) + return val + return val + + proxy_kwargs = pytree.tree_map(_maybe_register_subgraph, proxy_kwargs) + + out_proxy = mode.tracer.create_proxy( + "call_function", + auto_functionalized_v2, + (_mutable_op,), + proxy_kwargs, + ) + result = track_tensor_tree(out, out_proxy, constant=None, tracer=mode.tracer) + return result + + +@auto_functionalized_v2.py_functionalize_impl +def auto_functionalized_v2_func(ctx, _mutable_op, **kwargs): + unwrapped_kwargs = ctx.unwrap_tensors(kwargs) + with ctx.redispatch_to_next(): + result = auto_functionalized_v2(_mutable_op, **unwrapped_kwargs) + return ctx.wrap_tensors(result) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/base_hop.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/base_hop.py new file mode 100644 index 0000000000000000000000000000000000000000..a3e0cc52f8b3af1f4d23e907ace1155318ceb3e0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/base_hop.py @@ -0,0 +1,276 @@ +# mypy: allow-untyped-defs + +import abc + +import torch +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._dispatch.python import suspend_functionalization +from torch._higher_order_ops.auto_functionalize import FunctionalCallableWithEpilogue +from torch._higher_order_ops.utils import ( + check_input_alias_and_mutation_return_outputs, + HopInstance, + materialize_as_graph, + reenter_make_fx, +) +from torch._ops import HigherOrderOperator +from torch._subclasses import FakeTensorMode +from torch._subclasses.functional_tensor import disable_functional_mode +from torch.fx.experimental.proxy_tensor import ( + disable_proxy_modes_tracing, + ProxyTorchDispatchMode, + track_tensor_tree, +) + + +class BaseHOP(HigherOrderOperator, abc.ABC): + """ + This is the "Base" HOP implementation for a HOP that looks like: + + call_subgraph_hop(subgraph, *operands, **kwargs) + + That is: + 1) the HOP stays alive until Inductor + 2) the HOP's semantics are subgraph(*operands) + 3) kwargs may be some config options but aren't passed directly to the subgraph. + + To use this, please subclass this class and override methods as necessary: + ``` + class InvokeQuant(BaseHOP): + def __init__(self): + return super().__init__("invoke_quant") + + + invoke_quant = InvokeQuant() + + + def g(x): + return x.sin().cos() + + + @torch.compile(backend="aot_eager") + def f(x): + return invoke_quant(g, x, scheme="nf4") + ``` + + NOTE: don't subclass BaseHOP out of tree! That is not allowed. All + usages must be in tree. + """ + + def __init__(self, hop_name) -> None: + super().__init__(hop_name) + + # Set up the registrations + # If you want to override any of these, override them in your subclass. + self.py_autograd_impl(self._call_Autograd) + self.py_functionalize_impl(self._call_Functionalize) + self.py_impl(ProxyTorchDispatchMode)(self._call_ProxyTorchDispatchMode) + self.py_impl(FakeTensorMode)(self._call_FakeTensorMode) + self.py_impl(DispatchKey.CompositeExplicitAutograd)( + self._call_CompositeExplicitAutograd + ) + + def __call__(self, subgraph, *operands, **kwargs): + if not isinstance( + subgraph, + ( + torch.fx.GraphModule, + FunctionWithNoFreeVars, + FunctionalCallableWithEpilogue, + ), + ): + raise RuntimeError( + f"{self._name}: when calling this API without torch.compile, " + f"we require that the subgraph be a torch.fx.GraphModule (or " + f"a function we know doesn't have free variables)." + ) + return super().__call__(subgraph, *operands, **kwargs) + + def _call_Autograd(self, subgraph, *operands, **kwargs): + if isinstance(subgraph, torch.fx.GraphModule): + pass + + # We assume the subgraph doesn't mutate inputs and there is no aliasing. + # In the PT2 stack, this is Dynamo's responsibility to figure out. + return BaseHOPFunction.apply(self, subgraph, kwargs, *operands) + + def _call_CompositeExplicitAutograd(self, subgraph, *operands, **kwargs): + from torch.utils._python_dispatch import _get_current_dispatch_mode + + mode = _get_current_dispatch_mode() + assert mode is None, "Mode should never be enabled for CPU/CUDA key" + return subgraph(*operands) + + def _call_ProxyTorchDispatchMode(self, proxy_mode, subgraph, *operands, **kwargs): + traced_graph = reenter_make_fx(subgraph)(*operands) + assert isinstance(proxy_mode.tracer, torch.fx.Tracer) + qualname = proxy_mode.tracer.get_fresh_qualname("subgraph") + proxy_mode.tracer.root.register_module(qualname, traced_graph) + + node_args = (traced_graph, *operands) + proxy_args = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, node_args) # type: ignore[attr-defined] + proxy_kwargs = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, kwargs) # type: ignore[attr-defined] + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", self, proxy_args, proxy_kwargs + ) + + out = self(subgraph, *operands, **kwargs) + return track_tensor_tree( + out, + out_proxy, + constant=None, + tracer=proxy_mode.tracer, # type: ignore[arg-type] + ) + + def _call_FakeTensorMode(self, mode, subgraph, *operands, **kwargs): + # TODO: this should probably route through FakeTensorMode to reuse caching + with mode: + return subgraph(*operands) + + # NOTE [Support input mutation of hops] + # To support input mutation, hop's subgraph must be functionalized because many inductor passes are + # applied to subgraph recursively and only work on functional graph. However, we could inline an + # epilogue graph (i.e. the copy_) into the subgraph because this is how input mutation + # is implemented in the top-level graph when no hop is presented. All passes must have been and will be + # aware of the epilogue graph. + # + # Since we've supported input mutation for custom op with auto_functionalized, we share the infra for hops + # The plan is: + # 1. In hop's Functionalization key, it calls do_auto_functionalize_v2 if subgraph mutates input + # 2. In do_auto_functionalize_v2: + # a. we functionalize the callables in hop's argument. This is to make the subgraphs functional so we + # could recursively run passes on them. Also the epilogue graph is inlined at the end. + # b. we call auto_functionalized_v2 and pass in an additional schema in order to properly invoke + # the hop with normalized kwargs. + # 3. In inductor, we decompose the auto_functionalized hop by callilng into the dense implementation, which + # copies the mutated inputs to the hop if necessary and call the hop. + # After these steps, the rest of the inductor stack knows how to fuse the copy_ in subgraph with other ops. + def _call_Functionalize(self, ctx, subgraph, *operands, **kwargs): + from torch._higher_order_ops.auto_functionalize import ( + can_auto_functionalize, + do_auto_functionalize_v2, + ) + + # invoke_quant has non-proxable argument of type InvokeQuant that + # we cannot generate schema for. + if self is not torch.ops.higher_order.invoke_quant_packed: + hop_instance = HopInstance.create(self, subgraph, *operands, **kwargs) + if can_auto_functionalize(hop_instance): + return do_auto_functionalize_v2( + ctx.mode, hop_instance, (subgraph, *operands), kwargs + ) + + unwrapped_operands = ctx.unwrap_tensors(operands) + with ctx.redispatch_to_next(): + # We assume the subgraph doesn't mutate inputs and there is no aliasing. + # In the PT2 stack, this is Dynamo's responsibility to figure out. + functionalized_subgraph = FunctionWithNoFreeVars( + ctx.functionalize(subgraph) + ) + out = self(functionalized_subgraph, *unwrapped_operands, **kwargs) + return ctx.wrap_tensors(out) + + # pyrefly: ignore [bad-override] + def gen_schema(self, subgraph, *operands, **kwargs): + from .schema import HopSchemaGenerator + + subgraph = materialize_as_graph(subgraph, operands) + ( + inp_inp_alias, + inp_out_alias, + out_out_alias, + mutated_inp_idx, + output, + ) = check_input_alias_and_mutation_return_outputs(subgraph) + + if not ( + len(inp_inp_alias) == 0 + and len(inp_out_alias) == 0 + and len(out_out_alias) == 0 + ): + # TODO: turn this into an error. + # test_foreach_map_backward_binary_foreach_map_addrecip_op fails the alias test. + import warnings + + warnings.warn( + "Aliasing is not supported for HOP subgraph.\n" + f"{subgraph.print_readable(print_output=False)}\n" + f"Alias info: inp-inp alias: {inp_inp_alias}, inp-out alias: {inp_out_alias}, out-out alias{out_out_alias}" + f"This may lead to silent incorrectness.", + stacklevel=2, + ) + + schema_gen = HopSchemaGenerator(self) + schema_gen.add_arg("subgraph", subgraph) + for idx, arg in enumerate(operands): + schema_gen.add_arg(f"arg{idx}", arg, is_mutated=idx in mutated_inp_idx) + + for name, arg in kwargs.items(): + schema_gen.add_arg(name, arg, default_value=arg, kw_only=True) + + for out in output: + schema_gen.add_output(out) + + return schema_gen.gen_schema() + + +class BaseHOPFunction(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(ctx, hop, subgraph, kwargs, *operands): + ctx.hop = hop + ctx.operands = operands + ctx.subgraph = subgraph + ctx.kwargs = kwargs + + with torch._C._AutoDispatchBelowAutograd(): + return hop(subgraph, *operands, **kwargs) + + @staticmethod + def backward(ctx, *grad_outputs): + subgraph = ctx.subgraph + operands = ctx.operands + kwargs = ctx.kwargs + + # TODO: Something special needs to happen with min cut partitioner + with ( + suspend_functionalization(), + disable_functional_mode(), + torch.enable_grad(), + ): + with disable_proxy_modes_tracing(): + from .invoke_subgraph import create_fw_bw_graph + from .utils import _from_fun + + fw_inputs = pytree.tree_map(_from_fun, operands) + ( + _, + joint_graph, + _, + ) = create_fw_bw_graph(subgraph, fw_inputs, grad_outputs) + + # The joint graph returns (*grad_inputs, *fwd_outputs). + # We only need the grad_inputs. + def bwd_fn(*args): + operands = args[: -len(grad_outputs)] + grad_outs = args[-len(grad_outputs) :] + result = joint_graph(*operands, *grad_outs) + grad_inputs = result[: -len(grad_outputs)] + return grad_inputs + + return ( + None, + None, + None, + *ctx.hop( + FunctionWithNoFreeVars(bwd_fn), *operands, *grad_outputs, **kwargs + ), + ) + + +class FunctionWithNoFreeVars: + def __init__(self, fn): + self.fn = fn + + def __call__(self, *args, **kwargs): + return self.fn(*args, **kwargs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/cond.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/cond.py new file mode 100644 index 0000000000000000000000000000000000000000..f2d3c96a5cbfd463103dfe2b040625a1fa188645 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/cond.py @@ -0,0 +1,725 @@ +# mypy: allow-untyped-decorators +# mypy: allow-untyped-defs +import contextlib +import functools +import logging +import warnings +from collections.abc import Callable +from typing import Any, Optional, Union + +import torch +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._C._functorch import ( + _add_batch_dim, + get_unwrapped, + is_batchedtensor, + maybe_get_bdim, +) +from torch._functorch.utils import exposed_in +from torch._higher_order_ops.utils import ( + _maybe_run_with_interpreter, + check_input_alias_and_mutation_return_outputs, + create_bw_fn, + fill_none_with_masks, + filter_with_masks, + materialize_as_graph, + reenter_make_fx, + save_tensors_and_symints_for_backward, + saved_tensors_and_symints, + unique_graph_id, + validate_subgraph_args_types, +) +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode +from torch.fx.experimental.proxy_tensor import ProxyTorchDispatchMode, track_tensor_tree +from torch.utils._python_dispatch import _get_current_dispatch_mode + + +log = logging.getLogger(__name__) + +""" +We're going to define a `cond_op` operation. +In order to do this, we need implementations for each of the dispatch keys. +""" + + +class CondOp(HigherOrderOperator): + def __init__(self): + super().__init__("cond") + + def __call__(self, pred, true_fn, false_fn, operands): + validate_subgraph_args_types(operands) + return super().__call__(pred, true_fn, false_fn, operands) + + # pyrefly: ignore [bad-override] + def gen_schema(self, pred, true_fn, false_fn, operands): + from torch._higher_order_ops.schema import HopSchemaGenerator + from torch._higher_order_ops.utils import materialize_as_graph + + then_gm: torch.fx.GraphModule = materialize_as_graph(true_fn, operands) + else_gm: torch.fx.GraphModule = materialize_as_graph(false_fn, operands) + ( + _, + _, + _, + then_mutated_inputs, + then_outputs, + ) = check_input_alias_and_mutation_return_outputs(then_gm) + ( + _, + _, + _, + else_mutated_inputs, + else_outputs, + ) = check_input_alias_and_mutation_return_outputs(else_gm) + mutated_inputs = set(then_mutated_inputs) | set(else_mutated_inputs) + + schema_gen = HopSchemaGenerator(self) + schema_gen.add_arg("pred", pred) + schema_gen.add_arg("true_fn", then_gm) + schema_gen.add_arg("false_fn", else_gm) + for idx, arg in enumerate(operands): + schema_gen.add_arg(f"operand{idx}", arg, is_mutated=idx in mutated_inputs) + + for out in then_outputs: + schema_gen.add_output(out) + schema_gen.add_schema_tree_spec(pred, true_fn, false_fn, operands) + return schema_gen.gen_schema() + + +cond_op = CondOp() + + +@exposed_in("torch") +def cond( + pred: Union[bool, int, float, torch.Tensor], + true_fn: Callable, + false_fn: Callable, + operands: Union[tuple, list] = (), +) -> Any: + r""" + Conditionally applies `true_fn` or `false_fn`. + + .. warning:: + `torch.cond` is a prototype feature in PyTorch. It has limited support for input and output types. + Please look forward to a more stable implementation in a future version of PyTorch. + Read more about feature classification at: https://pytorch.org/blog/pytorch-feature-classification-changes/#prototype + + `cond` is structured control flow operator. That is, it is like a Python if-statement, + but has restrictions on `true_fn`, `false_fn`, and `operands` that enable it to be + capturable using torch.compile and torch.export. + + Assuming the constraints on `cond`'s arguments are met, `cond` is equivalent to the following:: + + def cond(pred, true_branch, false_branch, operands): + if pred: + return true_branch(*operands) + else: + return false_branch(*operands) + + Args: + pred (Union[bool, torch.Tensor]): A boolean expression or a tensor with one element, + indicating which branch function to apply. + + true_fn (Callable): A callable function (a -> b) that is within the + scope that is being traced. + + false_fn (Callable): A callable function (a -> b) that is within the + scope that is being traced. The true branch and false branch must + have consistent input and outputs, meaning the inputs have to be + the same, and the outputs have to be the same type and shape. Int + output is also allowed. We'll make the output dynamic by turning it + into a symint. + + operands (Tuple of possibly nested dict/list/tuple of torch.Tensor): A tuple of inputs to the + true/false functions. It can be empty if true_fn/false_fn doesn't require input. Defaults to (). + + Example:: + + def true_fn(x: torch.Tensor): + return x.cos() + + + def false_fn(x: torch.Tensor): + return x.sin() + + + return cond(x.shape[0] > 4, true_fn, false_fn, (x,)) + + Restrictions: + - The conditional statement (aka `pred`) must meet one of the following constraints: + + - It's a `torch.Tensor` with only one element, and torch.bool dtype + + - It's a boolean expression, e.g. `x.shape[0] > 10` or `x.dim() > 1 and x.shape[1] > 10` + + - The branch function (aka `true_fn`/`false_fn`) must meet all of the following constraints: + + - The function signature must match with operands. + + - The function must return a tensor with the same metadata, e.g. shape, + dtype, etc. + + - The function cannot have in-place mutations on inputs or global variables. + (Note: in-place tensor operations such as `add_` for intermediate results + are allowed in a branch) + + """ + if torch.compiler.is_dynamo_compiling(): + return cond_op(pred, true_fn, false_fn, operands) + + if isinstance(pred, (bool, int, float)): + # This is the non-strict export case. Strict export and torch.compile are + # handled above in dynamo. + if torch.compiler.is_compiling(): + warnings.warn( + "Pred is a Python constant. When used with torch.cond, it specializes on one of the branches." + " If you want torch.cond to preserve two branches, please make the predicate a boolean tensor or a SymBool.", + UserWarning, + stacklevel=2, + ) + # This is the eager case. We can just run the true or false branch. + if pred: + return true_fn(*operands) + else: + return false_fn(*operands) + + def _validate_input(pred, true_fn, false_fn, operands): + if not isinstance(pred, (bool, torch.Tensor, torch.SymBool)): + raise RuntimeError(f"Expected pred to be bool or tensor, but got {pred}.") + + if isinstance(pred, torch.Tensor) and pred.numel() != 1: + raise RuntimeError( + f"Expected pred to be bool or single-element tensor, but got {pred}." + ) + + if not callable(true_fn) or not callable(false_fn): + raise RuntimeError("Expect both branches to be callable.") + + if not isinstance(operands, (tuple, list)) or pytree.tree_any( + lambda t: not isinstance(t, torch.Tensor), operands + ): + raise RuntimeError( + "Expect operands to be a tuple of possibly nested dict/list/tuple that only " + f"consists of tensor leaves, but got {operands}." + ) + + _validate_input(pred, true_fn, false_fn, operands) + + if not torch._dynamo.is_dynamo_supported(): + raise RuntimeError("torch.cond requires dynamo support.") + + # Dynamo is expecting a callable with "__code__" attribute. + # We cannot directly pass cond_op to it. So we wrap it in a dummy function. + def _cond_op_wrapper(*args, **kwargs): + return cond_op(*args, **kwargs) + + from torch._higher_order_ops.utils import setup_compilation_env + + with setup_compilation_env() as backend: + return torch.compile(_cond_op_wrapper, backend=backend, fullgraph=True)( + pred, true_fn, false_fn, operands + ) + + +def trace_cond(proxy_mode, func_overload, pred, true_fn, false_fn, operands): + assert isinstance(operands, (list, tuple)), ( + f"Cond operands must be a list or tuple of tensors and SymInts {operands}" + ) + + true_graph = reenter_make_fx(true_fn)(*operands) + false_graph = reenter_make_fx(false_fn)(*operands) + + true_outs = [] + false_outs = [] + for node in true_graph.graph.nodes: + if node.op == "output": + true_outs.extend(node.args) + + for node in false_graph.graph.nodes: + if node.op == "output": + false_outs.extend(node.args) + + flat_true_outs = pytree.arg_tree_leaves(*true_outs) + flat_false_outs = pytree.arg_tree_leaves(*false_outs) + if len(flat_true_outs) != len(flat_false_outs): + raise torch._dynamo.exc.CondOpArgsMismatchError( + f"Expected to return same number of outputs but got:" + f"\n true branch returns {len(flat_true_outs)} item(s)" + f"\n false branch returns {len(flat_false_outs)} item(s)" + ) + + i, true_name = unique_graph_id(proxy_mode, prefix="true_graph") + + false_name = f"false_graph_{i}" + assert not hasattr(proxy_mode.tracer.root, false_name) + + proxy_mode.tracer.root.register_module(true_name, true_graph) + proxy_mode.tracer.root.register_module(false_name, false_graph) + + args = (pred, true_graph, false_graph, operands) + + proxy_args = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, args) + + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", func_overload, proxy_args, {} + ) + + out = func_overload(pred, true_graph, false_graph, operands) + + return track_tensor_tree(out, out_proxy, constant=None, tracer=proxy_mode.tracer) + + +@cond_op.py_impl(DispatchKey.CompositeExplicitAutograd) +def cond_op_dense(pred, true_fn, false_fn, operands): + assert all(isinstance(o, (torch.Tensor, int)) for o in operands), ( + f"Dense implementation operands must be a list of tensors and ints {operands}" + ) + mode = _get_current_dispatch_mode() + assert mode is None, "Mode should never be enabled for CPU/CUDA key" + if pred: + return true_fn(*operands) + else: + return false_fn(*operands) + + +class CondAutogradOp(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward( + ctx, + pred, + true_fn, + false_fn, + *operands, + ): + ctx._pred = pred + ctx._true_bw_fn = create_bw_fn( + true_fn, + operands, + ) + ctx._false_bw_fn = create_bw_fn( + false_fn, + operands, + ) + # We snapshot the dispatch keys in forward for materializing the + # the bw_graph in backward. + ctx._fw_include_key_set = torch._C._dispatch_tls_local_include_set() + ctx._fw_exclude_key_set = torch._C._dispatch_tls_local_exclude_set() + save_tensors_and_symints_for_backward(ctx, operands) + + with torch._C._AutoDispatchBelowAutograd(): + return cond_op(pred, true_fn, false_fn, operands) + + @staticmethod + def backward(ctx, *flat_grads): + operands = saved_tensors_and_symints(ctx) + args = operands + flat_grads + # TODO: we need to materialize the bw graphs because dynamo is unable to + # trace through the joint function when torch.compile torch.autograd.grad. + + grads_tensor_masks = [] + + def create_fn_remove_none(fn): + @functools.wraps(fn) + def wrapped(*args): + nonlocal grads_tensor_masks + + true_outputs = fn(*args) + grads_tensor_masks = [ + bool(isinstance(out, torch.Tensor)) for out in true_outputs + ] + return filter_with_masks(true_outputs, grads_tensor_masks) + + return wrapped + + true_bw_gm = materialize_as_graph( + create_fn_remove_none(ctx._true_bw_fn), + args, + ctx._fw_include_key_set, + ctx._fw_exclude_key_set, + force_enable_grad=True, + ) + false_bw_gm = materialize_as_graph( + create_fn_remove_none(ctx._false_bw_fn), + args, + ctx._fw_include_key_set, + ctx._fw_exclude_key_set, + force_enable_grad=True, + ) + grads = cond_op( + ctx._pred, + true_bw_gm, + false_bw_gm, + args, + ) + return None, None, None, *fill_none_with_masks(grads, grads_tensor_masks) + + +# Note: +# As long as one of the tensors in pred or operands requires grad, +# all the output would require grad with backward fn set to be the CondAutogradOp. +# This is consistent with autograd.Function's semantic. +@cond_op.py_autograd_impl +def cond_autograd(pred, true_fn, false_fn, operands): + return CondAutogradOp.apply( + pred, + true_fn, + false_fn, + *operands, + ) + + +@cond_op.py_impl(ProxyTorchDispatchMode) +def inner(mode, pred, true_fn, false_fn, operands): + return trace_cond(mode, cond_op, pred, true_fn, false_fn, operands) + + +@cond_op.py_impl(FakeTensorMode) +def cond_fake_tensor_mode(mode, pred, true_fn, false_fn, operands): + # Ignore here, because if you've gotten here but you're not manually + # tracing the inner graphs, that means that you intend to reuse the graph + # directly. Which means the old unbacked symbol bindings are appropriate. + # This strategy will not work if unbacked symbols can escape. + ignore_fresh_unbacked = contextlib.nullcontext() + if mode.shape_env: + ignore_fresh_unbacked = mode.shape_env.ignore_fresh_unbacked_symbols() + + with mode, ignore_fresh_unbacked: + flat_true_outs, true_out_spec = pytree.tree_flatten(true_fn(*operands)) + flat_false_outs, false_out_spec = pytree.tree_flatten(false_fn(*operands)) + if true_out_spec != false_out_spec: + raise RuntimeError( + "Unmatched output spec from torch.cond branches: " + f"true branch tree_spec {true_out_spec} vs false branch tree_spec {false_out_spec}." + ) + + merged_outs = [] + for true_out, false_out in zip(flat_true_outs, flat_false_outs): + merged_outs.append(_merge_output(true_out, false_out, mode)) + return pytree.tree_unflatten(merged_outs, true_out_spec) + + +def check_tensor_meta_match( + t1: torch.Tensor, t2: torch.Tensor, attr_names: tuple[str, ...], msg_prefix: str +) -> None: + def _get_attr_maybe_call(t: torch.Tensor, attr_name: str) -> Any: + attr = getattr(t, attr_name) + if callable(attr): + return attr() + return attr + + for attr_name in attr_names: + lattr = _get_attr_maybe_call(t1, attr_name) + rattr = _get_attr_maybe_call(t2, attr_name) + torch._check( + lattr == rattr, + lambda: f"{msg_prefix} expected same {attr_name} but got {lattr} and {rattr}.", + ) + + +def _merge_output( + a: Optional[Union[torch.Tensor, int]], + b: Optional[Union[torch.Tensor, int]], + mode: FakeTensorMode, +): + from torch.fx.experimental.symbolic_shapes import ( + has_free_unbacked_symbols, + SymIntEqByExpr, + ) + + if a is None or b is None: + assert a is None and b is None, (a, b) + return None + + def min_max(s0, s1): + def _bound(s0, lower_bound: bool): + if isinstance(s0, int): + return s0 + r = mode.shape_env.var_to_range.get( # type: ignore[union-attr] + s0.node.expr, + torch.utils._sympy.value_ranges.ValueRanges.unknown(), + ) + return r.lower if lower_bound else r.upper + + return min(_bound(s0, True), _bound(s1, True)), max( + _bound(s0, False), _bound(s1, False) + ) + + if type(a) is int and type(b) is int: + if a == b: + return a + assert mode.shape_env is not None + merged_out = mode.shape_env.create_unbacked_symint() + mode.shape_env.constrain_symbol_range(merged_out.node.expr, *min_max(a, b)) + return merged_out + + assert type(a) is FakeTensor and type(b) is FakeTensor, (a, type(a), b, type(b)) + + # Note: we don't check size, stride because + # they'll be merged with unbacked symints if they differ. + _meta_to_check = { + "dtype", + "device", + "layout", + "dim", + "is_quantized", + "is_conj", + "is_sparse", + "storage_offset", + } + check_tensor_meta_match( + a, + b, + tuple(_meta_to_check), + msg_prefix="When merging two branches' output in torch.cond, ", + ) + # NYI + assert not a.is_quantized and not b.is_quantized + assert not a.is_sparse and not b.is_sparse + assert not a.is_conj() and not b.is_conj() + + """ + Step 1: create unbacked symints for sizes that are different + along the same axis. For example: + a.size is [s0, 4, s0, 5, 4, 5] + b.size is [s1, 4, s2, 8, 4, 7] + merged_size will be [u0, 4, u1, u2, 4, u3], where + u0 has range [min(s0, s1), max(s0, s1)] + u1 has range [min(s0, s2), max(s0, s2)] + u2 has range [5, 8] + u3 has range [5, 7] + """ + merged_size: list[Union[int, torch.SymInt]] = [] + + def _has_unbacked_symbols(s: Union[int, torch.SymInt]) -> bool: + if isinstance(s, int): + return False + else: + return has_free_unbacked_symbols(s.node.expr) + + for s0, s1 in zip(a.size(), b.size()): + # If there are unbacked symbols leaked out of true_branch or false_branch + # we need to merge them with a new unbacked symbol and track in parent graph. + if ( + not _has_unbacked_symbols(s0) + and not _has_unbacked_symbols(s1) + and SymIntEqByExpr(s0) == SymIntEqByExpr(s1) + ): + merged_size.append(s0) + else: + assert mode.shape_env is not None + new_size = mode.shape_env.create_unbacked_symint() + mode.shape_env.constrain_symbol_range(new_size.node.expr, *min_max(s0, s1)) + merged_size.append(new_size) + + """ + This follows the logic in symbolic_shapes._compute_symbolic_stride + Step 2: Since tensor stride is an accumulative multiplication of the sizes, which is a permutated + (due to view ops) non-descending sequence. + + Case 1: No size is 1. In this case, strides have unique values. + For example, suppose we have a tensor with: + size [3, 4, 3, 5, 4, 5], + stride (1200, 300, 1, 12, 3, 60), + merged_size [u0, u1, u2, u3, u4, u5]. + + We visit the strides in ascending order: 1, 3, 12, 60, 300, 1200. In each step, we check whether + the current stride is bounded or not and bound next stride by setting. + stride_expr[next_stride] = current_stride_expr * current_size_expr + 1st round: + current_stride is 1, current_size is 3, so next_stride is 1 * 3 = 3, + current_stride_expr is set to 1, current_size_expr is u2, so stride_expr[3] is therefore 1 * u2 = u2 + 2nd round: + current_stride is 3, current_size is 4, so next_stride is 3 * 4 = 12, + current_stride_expr is stride_expr[3] i.e. u2, current_size_expr is u4, so stride_expr[12] = u2 * u4 + ... + + Case 2: At least one dimension has size 1, which can produce duplicates in strides. + In this case, theoretically, we cannot uniquely determine the expr of strides because + the accessing stride_expr with same key in different order causes the final stride expression + to be different. + + Suppose we have: + size: (3, 1) + stride: (1, 1) + merged_size: (u0, u1) + + The stride expr could either be (u1, 1) or (1, u0) depending on whether we start with u1 or u0. + For this reason, we try to break tie by sorting via descending index so we always get (u1, 1). + + Note that backend might optimize the strides anyway so this is usually not a problem as long + as two branches matches. See relevant discussions in https://github.com/pytorch/pytorch/issues/142024. + + Case 3: Dim has 0 stride. 0 stride doesn't participate in the accumulative multiplication of + sizes. So they're always treated as constant even if their corresponding size is turned into unbacked symint. + + Suppose we have: + size: (3, 3) + stride: (0, 1) + merged_size: (u0, u1) + + The merged stride would be (0, 1) + """ + + def _bound_stride( + a_ex_size: torch.Size, + b_ex_size: torch.Size, + a_ex_stride: tuple[int, ...], + b_ex_stride: tuple[int, ...], + merged_size: list[Union[int, torch.SymInt]], + ) -> list[Union[int, torch.SymInt]]: + from torch._inductor.ir import get_stride_order + + a_sorted_stride_idx = get_stride_order(a_ex_stride, mode.shape_env) + b_sorted_stride_idx = get_stride_order(b_ex_stride, mode.shape_env) + + a_stride_li: list[Optional[tuple[Union[int, torch.SymInt], int]]] = [ + None + ] * len(a_ex_stride) + b_stride_li: list[Optional[tuple[Union[int, torch.SymInt], int]]] = [ + None + ] * len(b_ex_stride) + for i, idx in enumerate(a_sorted_stride_idx): + a_stride_li[idx] = (a_ex_stride[i], -i) + for i, idx in enumerate(b_sorted_stride_idx): + b_stride_li[idx] = (b_ex_stride[i], -i) + + for a_pair, b_pair in zip(a_stride_li, b_stride_li): + assert a_pair is not None and b_pair is not None + _, a_idx = a_pair + _, b_idx = b_pair + + if a_idx != b_idx: + raise RuntimeError( + f"The sorted order of strides of the two branches' output doesn't match." + f"this indicates the contiguousness of the two branches are different. " + f"True branch has stride {a_ex_stride} but false branch has stride {b_ex_stride}." + f"Consider using contiguous() to make the two branches have the same contiguousness." + ) + + def _maybe_expr(s: Union[int, torch.SymInt]): + if isinstance(s, int): + return s + return s.node.expr + + a_stride_expr: dict[Any, Union[int, torch.SymInt]] = {} + b_stride_expr: dict[Any, Union[int, torch.SymInt]] = {} + merged_strides: list[Union[int, torch.SymInt]] = [None] * len(a_ex_stride) # type: ignore[list-item] + for a_pair, b_pair in zip(a_stride_li, b_stride_li): + assert a_pair is not None and b_pair is not None + a_val, neg_i = a_pair + b_val, _ = b_pair + + i = -neg_i + if a_val == 0: + assert b_val == 0, (a_val, b_val) + merged_strides[i] = 0 + continue + + if _maybe_expr(a_val) in a_stride_expr: + a_expr = a_stride_expr[_maybe_expr(a_val)] + assert b_stride_expr[_maybe_expr(b_val)] == a_expr, ( + f"a_stride_expr:{a_stride_expr}, b_stride_expr:{b_stride_expr}" + ) + merged_strides[i] = a_expr + else: + if a_val == 1: + assert b_val == 1 + a_stride_expr[_maybe_expr(a_val)] = 1 + b_stride_expr[_maybe_expr(b_val)] = 1 + merged_strides[i] = 1 + else: + # If we cannot find the expr of a_val in a_stride_expr, it means + # the strides is not a simple accumulative multiplication of sizes. + # In this case, we cannot determine the expr of strides from the new + # shapes so we error out and hint users to call contiguous(). + raise RuntimeError( + f"It seems one of cond's output stride is not a simple accumulative multiplication of sizes. " + f"This could be because cond returns a slice of a tensor, which is not dense in memory. " + f"True branch has size {a_ex_size}, stride {a_ex_stride} and false branch has size {b_ex_size} " + f"stride {b_ex_stride}. Hint: can call t.contiguous(). " + ) + nxt_merged_stride_expr = merged_strides[i] * merged_size[i] + a_stride_expr[_maybe_expr(a_val * a_ex_size[i])] = nxt_merged_stride_expr + b_stride_expr[_maybe_expr(b_val * b_ex_size[i])] = nxt_merged_stride_expr + return merged_strides + + merged_stride: list[Union[int, torch.SymInt]] = _bound_stride( + a.size(), b.size(), a.stride(), b.stride(), merged_size + ) + + with mode: + return torch.empty_strided( + merged_size, merged_stride, dtype=a.dtype, device=a.device + ) + + +@cond_op.py_functionalize_impl +def cond_func(ctx, pred, true_fn, false_fn, inputs): + from torch._higher_order_ops.utils import _check_alias_and_mutation + + unwrapped_inputs = ctx.unwrap_tensors(inputs) + unwrapped_pred = ctx.unwrap_tensors(pred) + with ctx.redispatch_to_next(): + functional_true = ctx.functionalize(_maybe_run_with_interpreter(true_fn)) + functional_false = ctx.functionalize(_maybe_run_with_interpreter(false_fn)) + pre_dispatch = hasattr(ctx, "mode") and ctx.mode.pre_dispatch + for branch, branch_name in [(true_fn, "cond_true"), (false_fn, "cond_false")]: + _check_alias_and_mutation( + branch, unwrapped_inputs, branch_name, pre_dispatch + ) + + cond_return = cond_op( + unwrapped_pred, functional_true, functional_false, unwrapped_inputs + ) + return ctx.wrap_tensors(cond_return) + + +@cond_op.py_impl(torch._C._functorch.TransformType.Vmap) +def cond_batch_rule(interpreter, pred, true_fn, false_fn, inputs): + assert isinstance(inputs, (list, tuple)), ( + "Cond inputs must be a list or tuple of tensors" + ) + assert all(isinstance(i, torch.Tensor) for i in inputs), ( + "Cond inputs must be a list of tensors" + ) + + pred_is_batched = isinstance(pred, torch.Tensor) and is_batchedtensor(pred) + pred_ = get_unwrapped(pred) if pred_is_batched else pred + + # unbatched tensors are not vmapped + tensors, in_dims = zip( + *[ + (get_unwrapped(t), maybe_get_bdim(t)) if is_batchedtensor(t) else (t, None) + for t in inputs + ] + ) + + if pred_is_batched: + # prepend "pred" and vmap everything + tensors = (pred_,) + tensors + in_dims = (0,) + in_dims + + def fn(p, *args): + t = true_fn(*args) + f = false_fn(*args) + return torch.where(p, t[0], f[0]) + + with interpreter.lower(): + result = torch.vmap(fn, in_dims=in_dims)(*tensors) + + else: + # predicate is known at this stage and it is a boolean expression or a + # tensor with one element. + true_fn = torch.vmap(true_fn, in_dims=in_dims) + false_fn = torch.vmap(false_fn, in_dims=in_dims) + + with interpreter.lower(): + result = cond_op(pred, true_fn, false_fn, tensors) + + if not isinstance(result, tuple): + result = (result,) + lvl = interpreter.level() + return tuple(_add_batch_dim(r, 0, lvl) for r in result) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/effects.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/effects.py new file mode 100644 index 0000000000000000000000000000000000000000..96d7872048ec8bba4101aa8a8bd8cdc61b3f1bfb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/effects.py @@ -0,0 +1,292 @@ +# mypy: allow-untyped-defs +from typing import Any, Optional, Union + +import torch +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._higher_order_ops.print import print as hop_print +from torch._higher_order_ops.torchbind import call_torchbind +from torch._library.custom_ops import CustomOpDef +from torch._library.effects import EffectType +from torch._library.utils import RegistrationHandle +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ( + disable_proxy_modes_tracing, + ProxyTorchDispatchMode, + track_tensor_tree, +) + + +_op_identifier = Union[ + str, + "torch._ops.OpOverload", + "torch._library.custom_ops.CustomOpDef", + "torch._ops.HigherOrderOperator", +] +OpType = Union["torch._ops.HigherOrderOperator", "torch._ops.OpOverload"] + +_EffectType = EffectType + + +def _get_op_qualname(op: _op_identifier) -> str: + """Convert an op identifier to a qualified string key.""" + if isinstance(op, torch._ops.OpOverload): + return op._name + elif isinstance(op, torch._ops.HigherOrderOperator): + return f"{op.namespace}::{op.name()}" + elif isinstance(op, CustomOpDef): + return op._qualname + elif isinstance(op, str): + return op + + raise ValueError(f"Invalid operator input {op}") + + +def _register_effectful_op( + op: _op_identifier, effect: Optional[EffectType] +) -> RegistrationHandle: + qualname = _get_op_qualname(op) + entry = torch._library.simple_registry.singleton.find(qualname) + handle = entry.effect.register(effect) + return handle + + +def _get_effect(op: _op_identifier) -> Optional[_EffectType]: + qualname = _get_op_qualname(op) + entry = torch._library.simple_registry.singleton.find(qualname) + return entry.effect.effect + + +_register_effectful_op("aten::_print", _EffectType.ORDERED) +_register_effectful_op("profiler::_record_function_exit._RecordFunction", None) +_register_effectful_op(call_torchbind, _EffectType.ORDERED) +_register_effectful_op(hop_print, _EffectType.ORDERED) + + +class WithEffects(HigherOrderOperator): + """ + with_effects(token, op, args, kwargs) -> (new_token, op_results) + + This HOP helps ensure ordering between side effectful ops like prints or ops + using torchbind objects. This is needed to ensure a traced graph from + AOTAutograd is functional so that future optimization passes do not reorder + these operators. This is done through threading "effect tokens" through the + graph to enforce data dependence between side effectful ops. + + The tokens are basically dummy values (torch.tensor([])). We create a token + per "effect type", which are enumerated in the _EffectType enum. + """ + + def __init__(self) -> None: + super().__init__("with_effects") + + def __call__( + self, + token, + op: OpType, + *args: tuple[Any, ...], + **kwargs: dict[str, Any], + ) -> tuple[Any, ...]: + assert isinstance(op, (torch._ops.HigherOrderOperator, torch._ops.OpOverload)) + assert not has_aliasing(op), "Ops with aliasing is not supported" + assert isinstance(kwargs, dict) + return super().__call__(token, op, *args, **kwargs) + + +with_effects = WithEffects() + + +def has_aliasing(op: OpType): + # NOT FOR PUBLIC USE + if isinstance(op, torch._ops.HigherOrderOperator): + return False + + for arg in op._schema.arguments: + if arg.alias_info is not None: + return True + for arg in op._schema.returns: + if arg.alias_info is not None: + return True + return False + + +def has_effects(op) -> bool: + return ( + isinstance(op, (torch._ops.HigherOrderOperator, torch._ops.OpOverload)) + and not has_aliasing(op) + and _get_effect(op) is not None + ) + + +def new_token_tensor() -> torch.Tensor: + return torch.tensor([]) + + +@with_effects.py_impl(DispatchKey.CompositeExplicitAutograd) +def with_effects_dense( + token: torch.Tensor, + op: torch._ops.OpOverload, + *args: tuple[Any, ...], + **kwargs: dict[str, Any], +) -> tuple[torch.Tensor, ...]: + out = op(*args, **kwargs) + new_token = new_token_tensor() + # [NOTE: with_effects return type] + # Note that we should only do *out for tuple type, but not list type. + # This is to match the schema of the op. + # For tuple output, the length of schema output is the same as the length of out. + # For list output, the length of schema output is 1 (e.g. Tensor[]) regardless of the + # length of the list. + if isinstance(out, tuple): + return (new_token, *out) + return (new_token, out) + + +@with_effects.py_impl(FakeTensorMode) +def with_effects_fake( + mode, + token: torch.Tensor, + op: torch._ops.OpOverload, + *args: tuple[Any, ...], + **kwargs: dict[str, Any], +) -> tuple[torch.Tensor, ...]: + with mode: + result = with_effects_dense(token, op, *args, **kwargs) + return result + + +@with_effects.py_impl(ProxyTorchDispatchMode) +def with_effects_proxy( + mode, + token: torch.Tensor, + op: torch._ops.OpOverload, + *args: tuple[Any, ...], + **kwargs: dict[str, Any], +) -> tuple[torch.Tensor, ...]: + with disable_proxy_modes_tracing(): + out = with_effects(token, op, *args, **kwargs) + + proxy_token = mode.tracer.unwrap_proxy(token) + proxy_args = pytree.tree_map(mode.tracer.unwrap_proxy, args) + proxy_kwargs = pytree.tree_map(mode.tracer.unwrap_proxy, kwargs) + + from torch.fx.node import has_side_effect + + # To avoid the being DCEed by graph.eliminate_dead_code if they. + # don't have output or their outputs are not used. + has_side_effect(op) + + out_proxy = mode.tracer.create_proxy( + "call_function", + with_effects, + (proxy_token, op, *proxy_args), + proxy_kwargs, + ) + result = track_tensor_tree(out, out_proxy, constant=None, tracer=mode.tracer) + return result + + +with_effects.fallthrough(DispatchKey.AutogradCPU) +with_effects.fallthrough(DispatchKey.AutogradCUDA) + + +def _get_schema(op, args, kwargs: Optional[dict] = None) -> torch.FunctionSchema: + if isinstance(op, torch._ops.OpOverload): + return op._schema + elif op == call_torchbind: + return getattr(args[0], args[1]).schema + elif op == hop_print: + # hop_print currently expects (format_str, *kwargs) as its arguments + extra_kwargs = kwargs or {} + return op.gen_schema(*args, **extra_kwargs) + else: + raise RuntimeError(f"Unable to get schema for op {op}") + + +def handle_effects( + allow_token_discovery: bool, + tokens: dict[_EffectType, torch.Tensor], + op: OpType, + args: tuple[Any, ...], + kwargs: dict[str, Any], +) -> Any: + """ + Args: + allow_token_discovery: Whether or not we are discovering tokens. If this + is true, we will create a token for every side effect type seen that + does not have a token assigned yet. If this is false, the tokens + should've all been created ahead of time, so we will error if there is + no token mapping to every effect type. + + tokens: Map of effect type to tokens. This is to chain operators of the + same effects together so that they do not get reordered in later + optimization passes. + """ + + # Get a token. We can't do `tokens.get(op, torch.tensor([]))` because + # this will create an empty tensor during proxy mode tracing if the token + # doesn't exist. But the tokens should always exist during proxy mode tracing. + key = _get_effect(op) + assert key is not None + if key not in tokens: + assert allow_token_discovery, ( + f"Could not find a token for effect {key} which came from the function {op}" + ) + proxy_tensor_mode = torch._C._get_dispatch_mode( + torch._C._TorchDispatchModeKey.PROXY + ) + if proxy_tensor_mode is not None: + # If we discovered a new token during tracing, we are in backward. + # Then we patch the graph, adding additional tangents_token as input to the joint graph. + tracer = proxy_tensor_mode.tracer + + from torch.fx.experimental.proxy_tensor import ( + disable_proxy_modes_tracing, + track_tensor_tree, + ) + + with disable_proxy_modes_tracing(): + token_tensor = new_token_tensor() + + token_proxy = proxy_tensor_mode.tracer.create_proxy( + "placeholder", "tangents_token", (), {}, name="tangents_token" + ) + track_tensor_tree(token_tensor, token_proxy, constant=None, tracer=tracer) + + tokens[key] = token_tensor + else: + tokens[key] = new_token_tensor() + + token = tokens[key] + + from torch._subclasses.functional_tensor import PythonFunctionalizeAPI + + ctx = PythonFunctionalizeAPI() + + unwrapped_token = ctx.unwrap_tensors([token])[0] + unwrapped_args = ctx.unwrap_tensors(args) + unwrapped_kwargs = ctx.unwrap_tensors(kwargs) # type: ignore[arg-type] + with ctx.redispatch_to_next(): + (new_token, *unwrapped_outs) = with_effects( + unwrapped_token, op, *unwrapped_args, **unwrapped_kwargs + ) + + schema = _get_schema(op, unwrapped_args, unwrapped_kwargs) + if len(schema.returns) == 0: + assert unwrapped_outs[0] is None + unwrapped_outs = None # type: ignore[assignment] + elif len(schema.returns) == 1: + assert len(unwrapped_outs) == 1 + unwrapped_outs = unwrapped_outs[0] + else: + assert len(unwrapped_outs) == len(schema.returns) + + # Add the newly created token into the tokens map for a following call to + # use this token. + wrapped_token = ctx.wrap_tensors(new_token) + assert isinstance(wrapped_token, torch.Tensor) + tokens[key] = wrapped_token + + # pyrefly: ignore [bad-argument-type] + return ctx.wrap_tensors(unwrapped_outs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/executorch_call_delegate.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/executorch_call_delegate.py new file mode 100644 index 0000000000000000000000000000000000000000..3274502b943cd655564bf08be4c87e33e48def92 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/executorch_call_delegate.py @@ -0,0 +1,178 @@ +# mypy: allow-untyped-defs + +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-strict + +from __future__ import annotations + +from typing import Any, cast + +import torch +import torch.utils._pytree as pytree +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ( + disable_proxy_modes_tracing, + get_proxy_slot, + ProxyTorchDispatchMode, + track_tensor_tree, +) +from torch.utils._pytree import tree_flatten + + +class ExecutorchCallDelegate(HigherOrderOperator): + def __init__(self): + super().__init__("executorch_call_delegate") + + def __call__(self, lowered_module, *args): + return super().__call__(lowered_module, *args) + + +executorch_call_delegate = ExecutorchCallDelegate() +executorch_call_delegate.fallthrough(torch._C.DispatchKey.PythonDispatcher) +executorch_call_delegate.fallthrough(torch._C.DispatchKey.PythonTLSSnapshot) +executorch_call_delegate.fallthrough(torch._C.DispatchKey.ADInplaceOrView) +executorch_call_delegate.fallthrough(torch._C.DispatchKey.AutocastCPU) + +LOWERED_BACKEND_MODULE_TYPE = "LoweredBackendModule" + + +# pyre-ignore +def trace_call_delegate(proxy_mode, func_overload, lowered_module, *args): + # pyre-ignore + def _unwrap_proxy(e): + if not isinstance(e, (torch.Tensor, torch.SymInt, torch.SymFloat)): + return e + return get_proxy_slot( + cast(torch.Tensor, e), + proxy_mode.tracer, + e, + lambda e: e.proxy, # type: ignore[attr-defined] + ) + + if not is_lowered_module(lowered_module): + raise ValueError( + "executorch_call_delegate()'s first argument must be a LoweredBackendModule" + ) + + with disable_proxy_modes_tracing(): + out = call_delegate_cpu(lowered_module, *args) + + get_lowered_module_name(proxy_mode.tracer.root, lowered_module) + + node_args = (lowered_module, *args) + proxy_args = pytree.tree_map(_unwrap_proxy, node_args) + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", func_overload, proxy_args, {}, name="executorch_call_delegate" + ) + return track_tensor_tree(out, out_proxy, constant=None, tracer=proxy_mode.tracer) + + +@executorch_call_delegate.py_impl(torch._C.DispatchKey.CompositeExplicitAutograd) +# pyre-ignore +def call_delegate_cpu(lowered_module, *args): + # FX creates this immutable_dict/list concept. Get rid of this. + map_types: dict[type, type] = { + torch.fx.immutable_collections.immutable_dict: dict, + torch.fx.immutable_collections.immutable_list: list, + } + new_args = pytree.tree_map_only( + tuple(map_types.keys()), + lambda a: map_types[type(a)](a), + args, + lambda a: isinstance(a, tuple(map_types.keys())), + ) + return lowered_module.original_module.module()(*new_args) + + +@executorch_call_delegate.py_autograd_impl +# pyre-ignore +def call_delegate_autograd(lowered_module, *args): + # TODO: support autograd + flat_operands, _ = tree_flatten([lowered_module, *args]) + requires_grad = any( + f.requires_grad for f in flat_operands if isinstance(f, torch.Tensor) + ) + + with torch._C._ExcludeDispatchKeyGuard( + torch._C.DispatchKeySet(torch._C.DispatchKey.AutogradCPU) + ): + res = executorch_call_delegate(lowered_module, *args) + + if requires_grad: + # Create aliases of the output that has requires_grad=True. We need + # at least one of the inputs to err_fn to require grad so that the + # output will have a grad_fn. + + # pyre-ignore + def fake_requires_grad(var): + if var is not None: + var = var.detach() + if torch.is_floating_point(var) or torch.is_complex(var): + var.requires_grad = True + return var + + return pytree.tree_map_only(torch.Tensor, fake_requires_grad, res) + + return res + + +@executorch_call_delegate.py_impl(ProxyTorchDispatchMode) +# pyre-ignore +def call_delegate_proxy_torch_dispatch_mode(mode, lowered_module, *args): + res = trace_call_delegate(mode, executorch_call_delegate, lowered_module, *args) + return res + + +@executorch_call_delegate.py_impl(FakeTensorMode) +# pyre-ignore +def call_delegate_fake_tensor_mode(mode, lowered_module, *args): + with mode: + return call_delegate_cpu(lowered_module, *args) + + +@executorch_call_delegate.py_functionalize_impl +# pyre-ignore +def call_delegate_functionalize(ctx, lowered_module, *args): + unwrapped_args = tuple(ctx.unwrap_tensors(arg) for arg in args) + with ctx.redispatch_to_next(): + res = executorch_call_delegate(lowered_module, *unwrapped_args) + return ctx.wrap_tensors(res) + + +# pyre-ignore: Missing parameter annotation [2]: Parameter `obj` must have a type other than `Any`.Pyre +def is_lowered_module(obj: Any) -> bool: + """ + This function is added to avoid using isinstance(obj, + LoweredBackendModule) as it will import LoweredBackendModule, which may + cause a circular import. + """ + return type(obj).__name__ == LOWERED_BACKEND_MODULE_TYPE + + +def get_lowered_module_name( + root: torch.nn.Module, + # pyre-ignore: Undefined or invalid type [11]: Annotation `LoweredBackendModule` is not defined as a type. + lowered_module: LOWERED_BACKEND_MODULE_TYPE, # type: ignore[valid-type] +) -> str: + """ + Adds the given lowered_module into the given root module and returns the + name of the module added. + """ + # Find a qualifying name for the lowered submodule + qualname = None + i = 0 + while True: + qualname = f"lowered_module_{i}" + if not hasattr(root, qualname): + break + i += 1 + assert qualname is not None + + root.add_module(qualname, lowered_module) + return qualname diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/flat_apply.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/flat_apply.py new file mode 100644 index 0000000000000000000000000000000000000000..8e1fefbcd5c4d92ee1e65a10b50e59e6f14a4951 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/flat_apply.py @@ -0,0 +1,164 @@ +import typing +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Generic, overload, TypeAlias, TypeVar +from typing_extensions import ParamSpec, TypeIs, TypeVarTuple, Unpack + +import torch +import torch.fx.node +import torch.utils._pytree as pytree +from torch._library.opaque_object import is_opaque_type +from torch._ops import HigherOrderOperator + + +_R = TypeVar("_R") +_P = ParamSpec("_P") +_Ts = TypeVarTuple("_Ts") + + +def is_graphable(val: object) -> TypeIs[torch.fx.node.BaseArgumentTypes]: + """Definition: a graphable type is a type that that is an acceptable input/output type to a FX node.""" + return isinstance(val, torch.fx.node.base_types) or is_opaque_type(type(val)) + + +def is_graphable_type(typ: type[object]) -> bool: + """Return whether the given type is graphable""" + return issubclass(typ, torch.fx.node.base_types) or is_opaque_type(typ) + + +def to_graphable(stuff: pytree.PyTree) -> tuple[list[object], pytree.TreeSpec]: + """Flattens stuff into a flat list of graphable types.""" + # We can consider preserving things like List[int] to improve + # perf and readability (right now that is all flattened out) + flat_args, spec = pytree.tree_flatten(stuff) + for arg in flat_args: + if not is_graphable(arg): + raise RuntimeError( + f"Expected all pytree.tree_leaves of (args, kwargs) to be graphable types, but found " + f"non-fx-graphable type {type(arg)}. If this type is meant to be constant, mark it as " + f"via pytree.register_constant; otherwise, register it as a pytree." + ) + return flat_args, spec + + +def from_graphable( + flat_args: tuple[Unpack[_Ts]], spec: pytree.TreeSpec +) -> pytree.PyTree: + """The inverse of to_graphable.""" + stuff = pytree.tree_unflatten(flat_args, spec) + return stuff + + +def func_to_graphable( + func: Callable[..., object], +) -> tuple[list[object], pytree.TreeSpec]: + """ + Pack and flatten a function type into graphable types. + This is useful for legalizing the function argument of `flat_apply`. + """ + return pytree.tree_flatten(_ConstantFunction(func)) + + +@dataclass(frozen=True) +class _ConstantFunction(Generic[_P, _R]): + func: Callable[_P, _R] + + def __call__(self, *args: _P.args, **kwargs: _P.kwargs) -> _R: + return self.func(*args, **kwargs) + + +pytree.register_constant(_ConstantFunction) + + +_OpTypes = ( + torch._ops.OpOverload | torch._ops.OpOverloadPacket | torch._ops.HigherOrderOperator +) +_op_types = typing.get_args(_OpTypes) + + +_Base: TypeAlias = torch.fx.node.BaseArgumentTypes +# pyrefly bug: pyrefly is complaining: Expected a type form, got instance of `Literal['_FXOutput'] +# pyrefly: ignore[not-a-type] +_FXOutput = _Base | Sequence["_FXOutput"] + + +class FlatApply(HigherOrderOperator): + def __init__(self) -> None: + super().__init__("flat_apply") + + def __call__( + self, + func: _OpTypes | pytree.TreeSpec, + in_spec: pytree.TreeSpec, + *flat_args: tuple[Unpack[_Ts]], + **_unused: object, + ) -> object: + """ + Functions that take in non-graphable types cannot directly be put into FX graph. + + Given func(*args, **kwargs), if all of the non-graphable types are pytrees, + then we're able to store a call to flat_apply(func, in_spec, *flat_args) in the FX graph. + + The semantics of flat_apply(func, in_spec, *flat_args) are roughly equivalent to: + + >>> def flat_apply_impl(func, in_spec, *flat_args): + >>> args, kwargs = pytree.tree_unflatten(flat_args, in_spec) + >>> output = func(*args, **kwargs) + >>> return output + + flat_apply supports the following two cases: + - an input type is a container type (e.g. of tensors) registered as a pytree. + We'll tree_flatten the input type and store the spec. + - an input type is a constant type (i.e. torch.compile will specialize on it) + registered with pytree.register_constant. The constant type goes directly + into the spec. + + """ + assert isinstance(func, _op_types) or pytree._is_constant_holder(func) + assert len(_unused) == 0 + return impl(func, in_spec, flat_args) + + +@overload +def _is_valid_output(x: tuple[object, ...]) -> TypeIs[tuple[_FXOutput, ...]]: ... + + +@overload +def _is_valid_output(x: Sequence[object]) -> TypeIs[Sequence[_FXOutput]]: ... + + +def _is_valid_output(x: object) -> bool: + if isinstance(x, (tuple, list)): + return all(map(_is_valid_output, x)) + return is_graphable(x) + + +def impl( + func: _OpTypes | pytree.TreeSpec, + in_spec: pytree.TreeSpec, + flat_args: tuple[Unpack[_Ts]], +) -> _FXOutput: + if isinstance(func, pytree.TreeSpec): + # assume _ConstantFunction + func = pytree._retrieve_constant(func) + assert isinstance(func, _ConstantFunction) + + # pyrefly: ignore[bad-argument-type] # pyrefly bug? + args, kwargs = from_graphable(flat_args, in_spec) + out = func(*args, **kwargs) + + # Right now, all outputs must either be graphable or lists/tuples of graphables. + # + # TODO: The following can be updated to support non-graphable outputs and pytrees. + # For non-graphable constant outputs: the assumption would be that they are constant + # (every time the function runs those MUST be the same) + # For pytree outputs: + # I'm not sure if we need to return (flat_output, spec) or just (flat_output,): + # in the latter case the tracers need to carry out the output specs + # (they need to know how to reconstruct the object from just the flat_output). + + assert _is_valid_output(out) + return out + + +flat_apply = FlatApply() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/flex_attention.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/flex_attention.py new file mode 100644 index 0000000000000000000000000000000000000000..ade9cfb3d5689c0097a6c4391925cd105d708f1c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/flex_attention.py @@ -0,0 +1,1306 @@ +import math +from collections.abc import Callable, Sequence +from typing import Any, Optional, Union + +import torch +import torch.utils._pytree as pytree +from torch import Tensor +from torch._C import DispatchKey +from torch._higher_order_ops.utils import ( + _has_potential_branch_input_mutation, + _maybe_reenter_make_fx, + autograd_not_implemented, + has_user_subclass, + redirect_to_mode, + reenter_make_fx, + register_fake, + save_tensors_and_symints_for_backward, + saved_tensors_and_symints, + UnsupportedAliasMutationException, + validate_subgraph_args_types, +) +from torch._ops import HigherOrderOperator +from torch._subclasses import FakeTensor +from torch._subclasses.functional_tensor import FunctionalTensor +from torch.fx.experimental.proxy_tensor import ( + make_fx, + ProxyTorchDispatchMode, + track_tensor_tree, +) +from torch.fx.graph_module import GraphModule +from torch.utils.checkpoint import _CachedTorchDispatchMode, _CachingTorchDispatchMode + + +# Duplicate of _inductor/kernel/flex_attention.py to avoid circular import +def _construct_strides( + sizes: Sequence[int], + fill_order: Sequence[int], +) -> Sequence[int]: + """From a list of sizes and a fill order, construct the strides of the permuted tensor.""" + # Initialize strides + assert len(sizes) == len(fill_order), ( + "Length of sizes must match the length of the fill order" + ) + strides = [0] * len(sizes) + + # Start with stride 1 for the innermost dimension + current_stride = 1 + + # Iterate through the fill order populating strides + for dim in fill_order: + strides[dim] = current_stride + current_stride *= sizes[dim] + + return strides + + +def _permute_strides(out: torch.Tensor, query_strides: tuple[int, ...]) -> torch.Tensor: + """ + Create a new tensor with the same data and shape as the input, + but with strides permuted based on the input tensor's stride order. + + Args: + out (torch.Tensor): The output tensor of attention. + query_strides (List[int]): The stride order of the input query tensor + + Returns: + torch.Tensor: A new tensor with same shape and data as the input, + but with strides permuted based on the query tensor's stride order. + """ + from torch._inductor.ir import get_fill_order + + fill_order = get_fill_order(query_strides) + assert out.storage_offset() == 0, "Only support storage_offset == 0" + out_strides = _construct_strides(out.shape, fill_order) + new_out = out.new_empty(out.shape).as_strided(out.shape, out_strides) + new_out.copy_(out) + return new_out + + +class FlexAttentionHOP(HigherOrderOperator): + def __init__(self) -> None: + super().__init__("flex_attention", cacheable=True) + + def __call__( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + score_mod: Callable, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple = (), + mask_mod_other_buffers: tuple = (), + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + validate_subgraph_args_types(score_mod_other_buffers + mask_mod_other_buffers) + return super().__call__( + query, + key, + value, + score_mod, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ) + + +flex_attention = FlexAttentionHOP() + + +class FlexAttentionBackwardHOP(HigherOrderOperator): + def __init__(self) -> None: + super().__init__("flex_attention_backward", cacheable=True) + + def __call__( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + out: torch.Tensor, + logsumexp: torch.Tensor, + grad_out: torch.Tensor, + grad_logsumexp: torch.Tensor, + fw_graph: Union[Callable, GraphModule], + joint_graph: GraphModule, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple = (), + mask_mod_other_buffers: tuple = (), + ) -> tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, tuple[Optional[torch.Tensor], ...] + ]: + validate_subgraph_args_types(score_mod_other_buffers + mask_mod_other_buffers) + + return super().__call__( + query, + key, + value, + out, + logsumexp, + grad_out, + grad_logsumexp, + fw_graph, + joint_graph, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ) + + +flex_attention_backward = FlexAttentionBackwardHOP() + + +def _math_attention_inner( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + score_mod: Callable, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple = (), + mask_mod_other_buffers: tuple = (), +) -> tuple[torch.Tensor, torch.Tensor]: + from torch._dynamo._trace_wrapped_higher_order_op import TransformGetItemToIndex + + working_precision = torch.float64 if query.dtype == torch.float64 else torch.float32 + + scores = query.to(working_precision) @ key.to(working_precision).transpose(-2, -1) + + b = torch.arange(0, scores.size(0), device=scores.device) + h = torch.arange(0, scores.size(1), device=scores.device) + m = torch.arange(0, scores.size(2), device=scores.device) + n = torch.arange(0, scores.size(3), device=scores.device) + + captured_buffers_in_dim = (None,) * len(score_mod_other_buffers) + from torch.nn.attention.flex_attention import _vmap_for_bhqkv + + # first input is score + score_mod = _vmap_for_bhqkv(score_mod, prefix=(0,), suffix=captured_buffers_in_dim) + + mask_mod = block_mask[-1] + mask_mod_in_dim_buffers = (None,) * len(mask_mod_other_buffers) + mask_mod = _vmap_for_bhqkv(mask_mod, prefix=(), suffix=mask_mod_in_dim_buffers) + + with TransformGetItemToIndex(): + scores = (scores * scale).to(working_precision) + post_mod_scores = torch.where( + mask_mod(b, h, m, n, *mask_mod_other_buffers), + score_mod(scores, b, h, m, n, *score_mod_other_buffers), + torch.tensor(-float("inf"), dtype=working_precision, device=scores.device), + ) + + return scores, post_mod_scores + + +def math_attention( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + score_mod: Callable, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple = (), + mask_mod_other_buffers: tuple = (), +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Eager implementation + + This implementation uses vmap to vectorize the score_mod function over the batch, head, m, and n dimensions. + We then apply the vectorized score_mod function to the scores matrix. Each wrap of vmap applies one of the + batch, head, m, or n dimensions. We need to apply vmap 4 times to vectorized over all 4 dimensions. + + Args: + query: The query tensor + key: The key tensor + value: The value tensor + score_mod: The score_mod function + other_buffers: Other buffers that are passed to the score_mod function + """ + # broadcast query & key along head dim for GQA + G = query.size(1) // key.size(1) + value = torch.repeat_interleave(value, G, dim=1) + key = torch.repeat_interleave(key, G, dim=1) + + Bq, Bkv = query.size(0), key.size(0) + if not ((Bq == Bkv) or (Bq > 1 and Bkv == 1)): + raise RuntimeError(f"Bq and Bkv must broadcast. Got Bq={Bq} and Bkv={Bkv}") + + key = key.expand((Bq, *key.size()[1:])) + value = value.expand((Bq, *value.size()[1:])) + + _, post_mod_scores = _math_attention_inner( + query, + key, + value, + score_mod, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ) + + # Set fully masked rows' sumexp to 0.0 + logsumexp = post_mod_scores.logsumexp(dim=-1) + masked_rows = torch.all(post_mod_scores == -float("inf"), dim=-1) + logsumexp = torch.where(masked_rows, -float("inf"), logsumexp) + + # working precision will be used so no need to cast to fp32 + max_scores = torch.max(post_mod_scores, dim=-1)[0] + + post_mod_scores = torch._safe_softmax(post_mod_scores, dim=-1) + + # NB: kernel computes in ln2 space, we always convert back at the top level op, so + # for math impl we divide by log(2) because we will multiply by log(2) + + return ( + post_mod_scores.to(query.dtype) @ value, + logsumexp / math.log(2), + max_scores / math.log(2), + ) + + +@flex_attention.py_impl(DispatchKey.CompositeExplicitAutograd) +def sdpa_dense( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + score_mod: Callable, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple = (), + mask_mod_other_buffers: tuple = (), +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + out, lse, max_scores = math_attention( + query, + key, + value, + score_mod, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ) + out = _permute_strides(out, query.stride()) + return out, lse, max_scores + + +def trace_flex_attention( + proxy_mode: ProxyTorchDispatchMode, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + score_mod: Callable, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple = (), + mask_mod_other_buffers: tuple = (), +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Traces the flex_attention operator with the given score_mod function and other_buffers. + + Trace SDPA will call make_fx with "fake" example vals and then trace the score_mod function + This will produce a GraphModule that will be stored on the root tracer as "sdpa_score". We + access this graph module in inductor to inline the score_mod function to the triton template. + """ + from torch._dynamo._trace_wrapped_higher_order_op import TransformGetItemToIndex + + example_out = flex_attention( + query, + key, + value, + score_mod, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ) + example_vals = [query.new_zeros((), requires_grad=query.requires_grad)] + [ + query.new_zeros((), dtype=torch.int) for _ in range(4) + ] + mask_example_vals = [query.new_zeros((), dtype=torch.int) for _ in range(4)] + mask_mod = block_mask[-1] + with TransformGetItemToIndex(): + score_graph = reenter_make_fx(score_mod)( + *example_vals, *score_mod_other_buffers + ) + mask_graph = reenter_make_fx(mask_mod)( + *mask_example_vals, *mask_mod_other_buffers + ) + assert isinstance(proxy_mode.tracer, torch.fx.Tracer) + block_mask = block_mask[:-1] + (mask_graph,) + qualname = proxy_mode.tracer.get_fresh_qualname("sdpa_score") + proxy_mode.tracer.root.register_module(qualname, score_graph) + mask_qualname = proxy_mode.tracer.get_fresh_qualname("sdpa_mask") + proxy_mode.tracer.root.register_module(mask_qualname, mask_graph) + node_args = ( + query, + key, + value, + score_graph, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ) + # pyrefly: ignore [missing-attribute] + proxy_args = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, node_args) + with torch.fx.experimental.proxy_tensor.set_original_aten_op(flex_attention): + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", flex_attention, proxy_args, {} + ) + return track_tensor_tree( + example_out, + out_proxy, + constant=None, + # pyrefly: ignore [bad-argument-type] + tracer=proxy_mode.tracer, + ) + + +@flex_attention.py_impl(ProxyTorchDispatchMode) +def flex_attention_proxy_torch_dispatch_mode( + mode: ProxyTorchDispatchMode, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + score_mod: Callable, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple = (), + mask_mod_other_buffers: tuple = (), +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + assert mode is not None, "Mode should always be enabled for python fallback key" + return trace_flex_attention( + mode, + query, + key, + value, + score_mod, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ) + + +@flex_attention.py_functionalize_impl +def flex_attention_functionalize( + ctx: torch._subclasses.functional_tensor.BaseFunctionalizeAPI, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + score_mod: Callable, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple = (), + mask_mod_other_buffers: tuple = (), +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Defines the functionalization rules for the flex_attention operator. + + Write now we are unwrapping each tensor and then redispatching to the next, however we want to + guard against any mutations in the score_mod function, to the other_buffers since those + are free variables. + """ + from torch._dynamo._trace_wrapped_higher_order_op import TransformGetItemToIndex + + if has_user_subclass( + ( + query, + key, + value, + score_mod, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ), + allowed_subclasses=(FakeTensor, FunctionalTensor), + ): + return NotImplemented + + query_unwrapped = ctx.unwrap_tensors(query) + key_unwrapped = ctx.unwrap_tensors(key) + value_unwrapped = ctx.unwrap_tensors(value) + block_mask_unwrapped = ctx.unwrap_tensors(block_mask) + score_mod_other_buffers_unwrapped = ctx.unwrap_tensors(score_mod_other_buffers) + mask_mod_other_buffers_unwrapped = ctx.unwrap_tensors(mask_mod_other_buffers) + + # Appease the mypy overlords + assert isinstance(query_unwrapped, torch.Tensor) + assert isinstance(key_unwrapped, torch.Tensor) + assert isinstance(value_unwrapped, torch.Tensor) + assert isinstance(block_mask_unwrapped, tuple) + assert isinstance(score_mod_other_buffers_unwrapped, tuple) + assert isinstance(mask_mod_other_buffers_unwrapped, tuple) + + example_vals = ( + [query_unwrapped.new_zeros(())] + + [query_unwrapped.new_zeros((), dtype=torch.int) for _ in range(4)] + + list(score_mod_other_buffers_unwrapped) + ) + with ctx.redispatch_to_next(): + functional_score_mod = ctx.functionalize(score_mod) + pre_dispatch = hasattr(ctx, "mode") and ctx.mode.pre_dispatch + with TransformGetItemToIndex(): + # TODO: So far only the input mutations are checked + # In the other HOPs, also aliases are checked which is + # omitted here + mutates = _has_potential_branch_input_mutation( + score_mod, example_vals, pre_dispatch + ) + # The only care about mutations of existing buffers since we can't replay these. + # However, we can just error if anything is detected + if mutates: + raise UnsupportedAliasMutationException("Mutations detected in score_mod") + + out = flex_attention( + query_unwrapped, + key_unwrapped, + value_unwrapped, + functional_score_mod, + block_mask_unwrapped, + scale, + kernel_options, + score_mod_other_buffers_unwrapped, + mask_mod_other_buffers_unwrapped, + ) + return ctx.wrap_tensors(out) # type: ignore[return-value, arg-type] + + +@register_fake(flex_attention) +def flex_attention_fake_impl( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + score_mod: Callable, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple = (), + mask_mod_other_buffers: tuple = (), +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if has_user_subclass( + ( + query, + key, + value, + score_mod, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ), + allowed_subclasses=(FakeTensor,), + ): + return NotImplemented + + v_head_dim = value.size(-1) + batch_size, num_heads, seq_len_q, _q_head_dim = query.shape + logsumexp = query.new_empty(batch_size, num_heads, seq_len_q, dtype=torch.float32) + max_scores = query.new_empty(batch_size, num_heads, seq_len_q, dtype=torch.float32) + out_shape = (batch_size, num_heads, seq_len_q, v_head_dim) + out = query.new_empty(out_shape) + out = _permute_strides(out, query.stride()) + return out, logsumexp, max_scores + + +# Registers dispatches for SAC +redirect_to_mode(flex_attention, _CachingTorchDispatchMode) +redirect_to_mode(flex_attention, _CachedTorchDispatchMode) + + +# ---------------------------- Autograd Implementation ---------------------------- +def create_fw_bw_graph( + score_mod: Callable, + index_values: tuple[Tensor, Tensor, Tensor, Tensor, Tensor], + other_buffers: tuple[Tensor, ...], +) -> tuple[Callable, Callable]: + # See Note:[HOP create fw_bw graph] + + # All of these imports need to be here in order to avoid circular dependencies + from torch._dispatch.python import suspend_functionalization + from torch._functorch.aot_autograd import AOTConfig, create_joint + from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode + from torch._subclasses.functional_tensor import disable_functional_mode + from torch.fx.experimental.proxy_tensor import disable_proxy_modes_tracing + + dummy_aot_config = AOTConfig( + fw_compiler=None, # type: ignore[arg-type] + bw_compiler=None, # type: ignore[arg-type] + partition_fn=None, # type: ignore[arg-type] + decompositions={}, + num_params_buffers=0, + aot_id=0, + keep_inference_input_mutations=False, + ) + + with suspend_functionalization(), disable_functional_mode(): + with disable_proxy_modes_tracing(): + + def _from_fun( + t: Union[Tensor, torch.SymInt, int], + ) -> Union[Tensor, torch.SymInt, int]: + if isinstance(t, torch.Tensor): + return torch.empty_strided( + t.size(), + t.stride(), + device=t.device, + dtype=t.dtype, + requires_grad=t.requires_grad, + ) + return t + + # If someone runs this hop under the default compiler backend ("eager") + # Then this path will be run with the actual user inputs. We convert them + # to fake tensors in order to not perform any actual compute. + from torch._guards import detect_fake_mode + + fake_mode = detect_fake_mode(index_values) + if fake_mode is None: + fake_mode = FakeTensorMode(allow_non_fake_inputs=True) + + with fake_mode: + unwrapped_score_mod_indexes = pytree.tree_map(_from_fun, index_values) + unwrapped_other_buffers = pytree.tree_map(_from_fun, other_buffers) + + assert all( + isinstance(t, (FakeTensor, int, torch.SymInt)) + for t in unwrapped_score_mod_indexes + unwrapped_other_buffers + ) + + example_flat_out = pytree.tree_map( + _from_fun, + score_mod(*unwrapped_score_mod_indexes, *unwrapped_other_buffers), + ) + if not isinstance(example_flat_out, torch.Tensor): + raise RuntimeError( + "Expected output of score_mod to be a tensor." + f"Got type {type(example_flat_out)}." + ) + example_grad = _from_fun(example_flat_out) + + def joint_f( + score: Tensor, + b: Tensor, + h: Tensor, + m: Tensor, + n: Tensor, + example_grad: Tensor, + *other_buffers: tuple[Tensor, ...], + ) -> tuple[Tensor, ...]: + def fw_with_masks( + *args: tuple[Tensor, ...], + ) -> tuple[tuple[Tensor], tuple[bool]]: + fw_out = score_mod(*args) + out_requires_grad = fw_out.requires_grad + return ((fw_out,), (out_requires_grad,)) + + joint = create_joint(fw_with_masks, aot_config=dummy_aot_config) + args = [score, b, h, m, n] + list(other_buffers) + optional_grad = [example_grad] if example_grad.requires_grad else [] + _, grads = joint(args, optional_grad) + + return grads + + joint_graph = make_fx(joint_f)( + *unwrapped_score_mod_indexes, example_grad, *unwrapped_other_buffers + ) + return score_mod, joint_graph + + +class FlexAttentionAutogradOp(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward( + ctx: Any, + query: Tensor, + key: Tensor, + value: Tensor, + fw_graph: Callable, + joint_graph: Callable, + block_mask: tuple[Any, ...], + scale: float, + kernel_options: dict[str, Any], + mask_mod_other_buffers: tuple[Any, ...], + *score_mod_other_buffers: tuple[Any, ...], + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + any_buffer_requires_grad = any( + buffer.requires_grad + for buffer in mask_mod_other_buffers + if isinstance(buffer, torch.Tensor) + ) + assert not any_buffer_requires_grad, ( + "Captured buffers from mask mod that require grad are not supported." + ) + ctx._fw_graph = fw_graph + ctx._joint_graph = joint_graph + ctx._mask_graph = block_mask[-1] + ctx.scale = scale + ctx.kernel_options = kernel_options + ctx._score_mod_other_buffers_len = len(score_mod_other_buffers) + with torch._C._AutoDispatchBelowAutograd(): + out, logsumexp, max_scores = flex_attention( + query, + key, + value, + fw_graph, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ) + # no grads for you sir + ctx.mark_non_differentiable(max_scores) + save_tensors_and_symints_for_backward( + ctx, + ( + query, + key, + value, + out, + logsumexp, + max_scores, + *block_mask[:-1], + *score_mod_other_buffers, + *mask_mod_other_buffers, + ), + ) + return out, logsumexp, max_scores + + @staticmethod + def backward( # type: ignore[override] + ctx: Any, + grad_out: Tensor, + grad_logsumexp: Tensor, + grad_max_scores: Tensor, + ) -> tuple[Optional[Tensor], ...]: + fw_args = saved_tensors_and_symints(ctx) + ( + query, + key, + value, + out, + logsumexp, + max_scores, + query_lengths, + kv_lengths, + kv_num_blocks, + kv_indices, + full_kv_num_blocks, + full_kv_indices, + q_num_blocks, + q_indices, + full_q_num_blocks, + full_q_indices, + Q_BLOCK_SIZE, + KV_BLOCK_SIZE, + *other_buffers, + ) = fw_args + fw_graph = ctx._fw_graph + joint_graph = ctx._joint_graph + mask_graph = ctx._mask_graph + scale = ctx.scale + kernel_options = ctx.kernel_options + score_mod_other_buffers = tuple( + other_buffers[: ctx._score_mod_other_buffers_len] + ) + mask_mod_other_buffers = tuple( + other_buffers[ctx._score_mod_other_buffers_len :] + ) + # We have asserted that mask_mod_other_buffers do not require grad, + # but score_mod_other_buffers can require grad. + none_grads = [None] * 6 + ( + grad_query, + grad_key, + grad_value, + grad_score_mod_captured, + ) = flex_attention_backward( + query, + key, + value, + out, + logsumexp, + grad_out, + grad_logsumexp, + fw_graph, + joint_graph, + ( + query_lengths, + kv_lengths, + kv_num_blocks, + kv_indices, + full_kv_num_blocks, + full_kv_indices, + q_num_blocks, + q_indices, + full_q_num_blocks, + full_q_indices, + Q_BLOCK_SIZE, + KV_BLOCK_SIZE, + mask_graph, + ), + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ) + return grad_query, grad_key, grad_value, *none_grads, *grad_score_mod_captured + + +# TODO: Rework DispatchKey.Autograd to py_autograd_impl +@flex_attention.py_impl(DispatchKey.Autograd) +def flex_attention_autograd( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + score_mod: Callable, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple[Tensor, ...] = (), + mask_mod_other_buffers: tuple[Tensor, ...] = (), +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + from torch._dynamo._trace_wrapped_higher_order_op import TransformGetItemToIndex + + with TransformGetItemToIndex(): + input_requires_grad = any( + isinstance(t, torch.Tensor) and t.requires_grad + for t in (query, key, value, *score_mod_other_buffers) + ) + if torch.is_grad_enabled() and input_requires_grad: + if block_mask[7] is None: + raise RuntimeError( + "BlockMask q_indices is None. Backward pass requires q_indices to be computed. " + "Please create the BlockMask with compute_q_blocks=True" + ) + example_vals = ( + query.new_zeros((), requires_grad=input_requires_grad), + query.new_zeros((), dtype=torch.int), + query.new_zeros((), dtype=torch.int), + query.new_zeros((), dtype=torch.int), + query.new_zeros((), dtype=torch.int), + ) + fw_graph, bw_graph = create_fw_bw_graph( + score_mod, example_vals, score_mod_other_buffers + ) + else: + fw_graph, bw_graph = score_mod, None + out, logsumexp, max_scores = FlexAttentionAutogradOp.apply( + query, + key, + value, + fw_graph, + bw_graph, + block_mask, + scale, + kernel_options, + mask_mod_other_buffers, + *score_mod_other_buffers, + ) + return out, logsumexp, max_scores + + +# ---------------------------- Backward HOP Implementation ---------------------------- + + +@flex_attention_backward.py_impl(DispatchKey.CompositeExplicitAutograd) +def sdpa_dense_backward( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + out: torch.Tensor, + logsumexp: torch.Tensor, + grad_out: torch.Tensor, + grad_logsumexp: torch.Tensor, + fw_graph: Callable, # GraphModule type hint? + joint_graph: Callable, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple, + mask_mod_other_buffers: tuple, +) -> tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, tuple[Optional[torch.Tensor], ...] +]: + from torch._dynamo._trace_wrapped_higher_order_op import TransformGetItemToIndex + + Bq, Hq, seq_len_q, qk_head_dim = query.shape + Bkv, Hkv, seq_len_kv, v_head_dim = value.shape + + # Get outputs before calling repeat interleave and permute to input stride orders + actual_grad_query = query.new_empty((Bq, Hq, seq_len_q, qk_head_dim)) + actual_grad_query = _permute_strides(actual_grad_query, query.stride()) + + actual_grad_key = key.new_empty((Bq, Hkv, seq_len_kv, qk_head_dim)) + actual_grad_key = _permute_strides(actual_grad_key, key.stride()) + + actual_grad_value = value.new_empty((Bq, Hkv, seq_len_kv, v_head_dim)) + actual_grad_value = _permute_strides(actual_grad_value, value.stride()) + + def _maybe_new_buffer( + buffer: Union[torch.Tensor, torch.SymInt, int], + ) -> Optional[Union[torch.Tensor, torch.SymInt, int]]: + if isinstance(buffer, torch.Tensor): + return ( + torch.empty_like(buffer, memory_format=torch.contiguous_format) + if buffer.requires_grad + else None + ) + return buffer + + actual_grad_score_mod_captured = [ + _maybe_new_buffer(buffer) for buffer in score_mod_other_buffers + ] + + Bq, Bkv = query.size(0), key.size(0) + if not ((Bq == Bkv) or (Bq > 1 and Bkv == 1)): + raise RuntimeError(f"Bq and Bkv must broadcast. Got Bq={Bq} and Bkv={Bkv}") + + key = key.expand((Bq, *key.size()[1:])) + value = value.expand((Bq, *value.size()[1:])) + + G = query.size(1) // key.size(1) + key = torch.repeat_interleave(key, G, dim=1) + value = torch.repeat_interleave(value, G, dim=1) + + # We're undoing the log -> log2 change of base in the forwards + logsumexp = logsumexp * math.log(2) + # The backwards formula for the log -> log2 change of base in the forwards + grad_logsumexp = grad_logsumexp / math.log(2) + scores, post_mod_scores = _math_attention_inner( + query, + key, + value, + fw_graph, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ) + masked_out_rows = logsumexp == -float("inf") + softmax_scores = torch.exp(post_mod_scores - logsumexp.unsqueeze(-1)) + softmax_scores = torch.where(masked_out_rows.unsqueeze(-1), 0, softmax_scores) + + grad_value = softmax_scores.to(query.dtype).transpose(-2, -1) @ grad_out + + grad_softmax_scores = grad_out.to(dtype=softmax_scores.dtype) @ value.to( + dtype=softmax_scores.dtype + ).transpose(-2, -1) + + sum_scores = torch.sum( + out.to(dtype=softmax_scores.dtype) * grad_out.to(dtype=softmax_scores.dtype), + -1, + keepdim=True, + ) + grad_score_mod = softmax_scores * ( + grad_softmax_scores - sum_scores + grad_logsumexp.unsqueeze(-1) + ) + + b = torch.arange(0, scores.size(0), device=scores.device) + h = torch.arange(0, scores.size(1), device=scores.device) + m = torch.arange(0, scores.size(2), device=scores.device) + n = torch.arange(0, scores.size(3), device=scores.device) + + mask_graph = block_mask[-1] + # Gradient of the inline score_mod function, with respect to the scores + captured_buffers_in_dim = (None,) * len(score_mod_other_buffers) + out_dims = [0, None, None, None, None] + [None] * len(score_mod_other_buffers) + from torch.nn.attention.flex_attention import _vmap_for_bhqkv + + # inputs are [score, b, h, q_idx, kv_idx, gradOut, ...] + # score and gradOut are "fully" batched + joint_score_mod = _vmap_for_bhqkv( + joint_graph, + prefix=(0,), + suffix=(0,) + captured_buffers_in_dim, + out_dims=out_dims, + ) + with TransformGetItemToIndex(): + grad_scores, _, _, _, _, *grad_score_mod_captured = joint_score_mod( + scores, b, h, m, n, grad_score_mod, *score_mod_other_buffers + ) + grad_scores = grad_scores * scale + grad_scores = grad_scores.to(query.dtype) + + mask_mod = _vmap_for_bhqkv( + mask_graph, prefix=(), suffix=(None,) * len(mask_mod_other_buffers) + ) + with TransformGetItemToIndex(): + mask_scores = mask_mod(b, h, m, n, *mask_mod_other_buffers) + grad_scores = torch.where( + mask_scores, grad_scores, torch.tensor(0, dtype=query.dtype) + ) + + grad_query = grad_scores @ key + grad_key = grad_scores.transpose(-2, -1) @ query + + # Reduce DK, DV along broadcasted heads. + grad_key = grad_key.view( + grad_key.size(0), -1, G, grad_key.size(-2), grad_key.size(-1) + ) + grad_value = grad_value.view( + grad_value.size(0), -1, G, grad_value.size(-2), grad_value.size(-1) + ) + + grad_key = torch.sum(grad_key, 2, keepdim=False) + grad_value = torch.sum(grad_value, 2, keepdim=False) + + # Fill to correctly strided outputs + actual_grad_query.copy_(grad_query) + actual_grad_key.copy_(grad_key) + actual_grad_value.copy_(grad_value) + + if Bq != Bkv: + assert Bq > 1 and Bkv == 1, ( + f"Bq and Bkv must broadcast. Got Bq={Bq} and Bkv={Bkv}" + ) + + actual_grad_key = torch.sum(actual_grad_key, 0, keepdim=True) + actual_grad_value = torch.sum(actual_grad_value, 0, keepdim=True) + + score_mod_other_buffer_grads = [ + actual_grad.copy_(grad) if isinstance(actual_grad, torch.Tensor) else None + for actual_grad, grad in zip( + actual_grad_score_mod_captured, grad_score_mod_captured + ) + ] + + return ( + actual_grad_query, + actual_grad_key, + actual_grad_value, + tuple(score_mod_other_buffer_grads), + ) + + +def trace_flex_attention_backward( + proxy_mode: ProxyTorchDispatchMode, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + out: torch.Tensor, + logsumexp: torch.Tensor, + grad_out: torch.Tensor, + grad_logsumexp: torch.Tensor, + fw_graph: Union[Callable, GraphModule], + joint_graph: GraphModule, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple = (), + mask_mod_other_buffers: tuple = (), +) -> tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, tuple[Optional[torch.Tensor], ...] +]: + """We already have the forward graph and joint graph from the forward pass, so we create a proxy attach both graphs""" + from torch._dynamo._trace_wrapped_higher_order_op import TransformGetItemToIndex + + example_out = flex_attention_backward( + query, + key, + value, + out, + logsumexp, + grad_out, + grad_logsumexp, + fw_graph, + joint_graph, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ) + + requires_grad = any(pytree.tree_map(lambda x: x.requires_grad, (query, key))) + fw_example_vals = [query.new_zeros((), requires_grad=requires_grad)] + [ + query.new_zeros((), dtype=torch.int) for _ in range(4) + ] + bw_example_vals = fw_example_vals + [query.new_zeros(())] + mask_example_vals = [query.new_zeros((), dtype=torch.int) for _ in range(4)] + mask_graph = block_mask[-1] + with TransformGetItemToIndex(): + # There's no active make_fx during the compiled autograd graph's initial capture + fw_graph = _maybe_reenter_make_fx(fw_graph)( + *fw_example_vals, *score_mod_other_buffers + ) + joint_graph = _maybe_reenter_make_fx(joint_graph)( + *bw_example_vals, *score_mod_other_buffers + ) + mask_graph = _maybe_reenter_make_fx(mask_graph)( + *mask_example_vals, *mask_mod_other_buffers + ) + assert isinstance(proxy_mode.tracer, torch.fx.Tracer) + block_mask = block_mask[:-1] + (mask_graph,) + + qualname = proxy_mode.tracer.get_fresh_qualname("fw_graph") + proxy_mode.tracer.root.register_module(qualname, fw_graph) # type: ignore[arg-type] + qualname = proxy_mode.tracer.get_fresh_qualname("joint_graph") + proxy_mode.tracer.root.register_module(qualname, joint_graph) + qualname = proxy_mode.tracer.get_fresh_qualname("mask_graph") + proxy_mode.tracer.root.register_module(qualname, mask_graph) + + node_args = ( + query, + key, + value, + out, + logsumexp, + grad_out, + grad_logsumexp, + fw_graph, + joint_graph, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ) + # pyrefly: ignore [missing-attribute] + proxy_args = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, node_args) + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", + flex_attention_backward, + proxy_args, + {}, + name="flex_attention_backward", + ) + return track_tensor_tree( + example_out, + out_proxy, + constant=None, + # pyrefly: ignore [bad-argument-type] + tracer=proxy_mode.tracer, + ) + + +@flex_attention_backward.py_impl(ProxyTorchDispatchMode) +def flex_attention_backward_proxy_torch_dispatch_mode( + mode: ProxyTorchDispatchMode, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + out: torch.Tensor, + logsumexp: torch.Tensor, + grad_out: torch.Tensor, + grad_logsumexp: torch.Tensor, + fw_graph: Union[Callable, GraphModule], + joint_graph: GraphModule, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple = (), + mask_mod_other_buffers: tuple = (), +) -> tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, tuple[Optional[torch.Tensor], ...] +]: + assert mode is not None, "Mode should always be enabled for python fallback key" + with torch.fx.experimental.proxy_tensor.set_original_aten_op( + flex_attention_backward + ): + return trace_flex_attention_backward( + mode, + query, + key, + value, + out, + logsumexp, + grad_out, + grad_logsumexp, + fw_graph, + joint_graph, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ) + + +@flex_attention_backward.py_functionalize_impl +def flex_attention_backward_functionalize( + ctx: torch._subclasses.functional_tensor.BaseFunctionalizeAPI, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + out: torch.Tensor, + logsumexp: torch.Tensor, + grad_out: torch.Tensor, + grad_logsumexp: torch.Tensor, + fw_graph: Union[Callable, GraphModule], + joint_graph: GraphModule, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple = (), + mask_mod_other_buffers: tuple = (), +) -> tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, tuple[Optional[torch.Tensor], ...] +]: + """Defines the functionalization rules for the flex_attention operator. + + Write now we are unwrapping each tensor and then redispatching to the next, + since we know that the forward score mod function is assured to be free of mutations + to the other_buffers, we skip that mutate check and go straight to redispatching. + """ + + if has_user_subclass( + ( + query, + key, + value, + out, + logsumexp, + grad_out, + grad_logsumexp, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ), + allowed_subclasses=(FakeTensor, FunctionalTensor), + ): + return NotImplemented + query_unwrapped = ctx.unwrap_tensors(query) + key_unwrapped = ctx.unwrap_tensors(key) + value_unwrapped = ctx.unwrap_tensors(value) + out_unwrapped = ctx.unwrap_tensors(out) + logsumexp_unwrapped = ctx.unwrap_tensors(logsumexp) + grad_out_unwrapped = ctx.unwrap_tensors(grad_out) + grad_logsumexp_unwrapped = ctx.unwrap_tensors(grad_logsumexp) + block_mask_unwrapped = ctx.unwrap_tensors(block_mask) + score_mod_other_buffers_unwrapped = ctx.unwrap_tensors(score_mod_other_buffers) + mask_mod_other_buffers_unwrapped = ctx.unwrap_tensors(mask_mod_other_buffers) + + # Appease the mypy overlords + assert isinstance(query_unwrapped, torch.Tensor) + assert isinstance(key_unwrapped, torch.Tensor) + assert isinstance(value_unwrapped, torch.Tensor) + assert isinstance(out_unwrapped, torch.Tensor) + assert isinstance(logsumexp_unwrapped, torch.Tensor) + assert isinstance(grad_out_unwrapped, torch.Tensor) + assert isinstance(grad_logsumexp_unwrapped, torch.Tensor) + assert isinstance(block_mask_unwrapped, tuple) + assert isinstance(score_mod_other_buffers_unwrapped, tuple) + assert isinstance(mask_mod_other_buffers_unwrapped, tuple) + + with ctx.redispatch_to_next(): + functional_fw_graph = ctx.functionalize(fw_graph) + functional_joint_graph = ctx.functionalize(joint_graph) + + ( + grad_query, + grad_key, + grad_value, + grad_score_mod_captured, + ) = flex_attention_backward( + query_unwrapped, + key_unwrapped, + value_unwrapped, + out_unwrapped, + logsumexp_unwrapped, + grad_out_unwrapped, + grad_logsumexp_unwrapped, + functional_fw_graph, # type: ignore[arg-type] + functional_joint_graph, # type: ignore[arg-type] + block_mask_unwrapped, + scale, + kernel_options, + score_mod_other_buffers_unwrapped, + mask_mod_other_buffers_unwrapped, + ) + + return ctx.wrap_tensors((grad_query, grad_key, grad_value, grad_score_mod_captured)) # type: ignore[return-value,arg-type] + + +@register_fake(flex_attention_backward) +def flex_attention_backward_fake_tensor_mode( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + out: torch.Tensor, + logsumexp: torch.Tensor, + grad_out: torch.Tensor, + grad_logsumexp: torch.Tensor, + fw_graph: Union[Callable, GraphModule], + joint_graph: GraphModule, + block_mask: tuple, + scale: float, + kernel_options: dict[str, Any], + score_mod_other_buffers: tuple = (), + mask_mod_other_buffers: tuple = (), +) -> tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, tuple[Optional[torch.Tensor], ...] +]: + if has_user_subclass( + ( + query, + key, + value, + out, + logsumexp, + grad_out, + grad_logsumexp, + block_mask, + scale, + kernel_options, + score_mod_other_buffers, + mask_mod_other_buffers, + ), + allowed_subclasses=(FakeTensor,), + ): + return NotImplemented + Bq, _, _, qk_head_dim = query.shape + Bkv, Hkv, seq_len_kv, v_head_dim = value.shape + + grad_query = torch.empty_like(query) + # zeros_and_scatter creates a contiguous zeros tensor -> contiguous_format + grad_score_mod_captured = tuple( + ( + torch.empty_like(buffer, memory_format=torch.contiguous_format) + if isinstance(buffer, torch.Tensor) + else None + ) + for buffer in score_mod_other_buffers + ) + + broadcasted_grad_key = key.new_empty((Bq, Hkv, seq_len_kv, qk_head_dim)) + broadcasted_grad_key = _permute_strides(broadcasted_grad_key, key.stride()) + + broadcasted_grad_value = value.new_empty((Bq, Hkv, seq_len_kv, v_head_dim)) + broadcasted_grad_value = _permute_strides(broadcasted_grad_value, value.stride()) + + if Bq > 1 and Bkv == 1: + grad_key = torch.sum(broadcasted_grad_key, dim=0, keepdim=True) + grad_value = torch.sum(broadcasted_grad_value, dim=0, keepdim=True) + else: + grad_key = broadcasted_grad_key + grad_value = broadcasted_grad_value + + return grad_query, grad_key, grad_value, grad_score_mod_captured + + +flex_attention_backward.py_autograd_impl( + autograd_not_implemented(flex_attention_backward, deferred_error=True) +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/foreach_map.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/foreach_map.py new file mode 100644 index 0000000000000000000000000000000000000000..0d02515d555d3be737011bd4cab32393acbe5b07 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/foreach_map.py @@ -0,0 +1,24 @@ +# mypy: allow-untyped-decorators +# mypy: allow-untyped-defs +from collections.abc import Callable +from typing import Any + +from torch._higher_order_ops.base_hop import BaseHOP, FunctionWithNoFreeVars + + +class ForeachMap(BaseHOP): + def __init__(self): + super().__init__("foreach_map") + + def __call__(self, fn, *operands, **kwargs): # type: ignore[override] + fn = FunctionWithNoFreeVars(fn) + return super().__call__(fn, *operands, **kwargs) + + +_foreach_map = ForeachMap() + + +def foreach_map(op: Callable, *operands: Any, **kwargs: dict[str, Any]): + from torch._dynamo.polyfills import foreach_map_fn + + return _foreach_map(foreach_map_fn, op, *operands, **kwargs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/hints_wrap.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/hints_wrap.py new file mode 100644 index 0000000000000000000000000000000000000000..583623393a0a15c81f6181421fc0ee59b538a1f8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/hints_wrap.py @@ -0,0 +1,141 @@ +# mypy: allow-untyped-defs +import torch +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._higher_order_ops.utils import ( + autograd_not_implemented, + reenter_make_fx, + unique_graph_id, +) +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ProxyTorchDispatchMode, track_tensor_tree + + +# used for wrapping a function/op with context hints +class HintsWrapper(HigherOrderOperator): + def __init__(self): + super().__init__("hints_wrapper") + + def __call__(self, body_fn, args, kwargs, hints): + r""" + Call implementation of hints_wrapper + + Args: + body_fn (Callable): A callable function that is within the scope + that is being traced. + + args (Tuple of torch.Tensor/int/float/bool): A tuple of inputs to + body_fn. + + kwargs (dict): Keyword argument to the body_fn. + + hints (dict): A dict of context hints which could be passed to + backend compiler. + """ + if not isinstance(args, tuple): + args = tuple(args) + + if not all(isinstance(t, (torch.Tensor, int, float, bool)) for t in args): + raise RuntimeError( + f"args must be a tuple of tensors, ints, floats, or bools, got {args}" + ) + + if not isinstance(kwargs, dict): + raise RuntimeError(f"kwargs must be a dict, got {type(kwargs)}") + + if len(kwargs) > 0: + raise RuntimeError( + f"kwargs except for hints are not supported, got {kwargs}" + ) + + if not isinstance(hints, dict): + raise RuntimeError(f"hints must be a dict, got {type(hints)}") + + for k, v in hints.items(): + if not isinstance(k, str): + raise RuntimeError(f"hints key must be a str, got {k}.") + + if not isinstance(v, (int, float, bool, str)): + raise RuntimeError( + "hints must be a dict containing int, float, bool or str " + f"value, got value {v} for key {k}." + ) + + return super().__call__(body_fn, args, kwargs, hints) + + +hints_wrapper = HintsWrapper() + + +@hints_wrapper.py_impl(DispatchKey.CompositeExplicitAutograd) +def hints_wrapper_dense(body_fn, args, kwargs, hints): + return body_fn(*args, **kwargs) + + +hints_wrapper.py_autograd_impl( + autograd_not_implemented(hints_wrapper, deferred_error=True) +) + + +@hints_wrapper.py_impl(FakeTensorMode) +def hints_wrapper_fake_tensor_mode(mode, body_func, args, kwargs, hints): + flat_args = pytree.tree_leaves(args) + with mode: + return body_func(*flat_args, **kwargs) + + +@hints_wrapper.py_functionalize_impl +def hints_wrapper_functionalize(ctx, body_fn, args, kwargs, hints): + from torch._higher_order_ops.utils import _check_alias_and_mutation + + unwrapped_args = ctx.unwrap_tensors(args) + unwrapped_kwargs = ctx.unwrap_tensors(kwargs) + unwrapped_hints = ctx.unwrap_tensors(hints) + with ctx.redispatch_to_next(): + functional_body_fn = ctx.functionalize(body_fn) + pre_dispatch = hasattr(ctx, "mode") and ctx.mode.pre_dispatch + _check_alias_and_mutation( + body_fn, unwrapped_args, "hints_wrapper", pre_dispatch + ) + + outputs = hints_wrapper( + functional_body_fn, + unwrapped_args, + unwrapped_kwargs, + unwrapped_hints, + ) + return ctx.wrap_tensors(outputs) + + +def trace_hints_wrapper(proxy_mode, hints_wrapper, body_fn, args, kwargs, hints): + flat_args = tuple(pytree.tree_leaves(args)) + body_graph = reenter_make_fx(body_fn)(*flat_args, **kwargs) + + _, body_graph_name = unique_graph_id(proxy_mode, prefix="hints_wrapper_body_graph") + proxy_mode.tracer.root.register_module(body_graph_name, body_graph) + + new_args: tuple = (body_graph, flat_args, {}) + # merge hints into kwargs + new_kwargs = {} + new_kwargs["hints"] = hints + + proxy_args = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, new_args) + proxy_kwargs = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, new_kwargs) + + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", hints_wrapper, proxy_args, proxy_kwargs, name="hints_wrapper" + ) + + out = body_fn(*flat_args, **kwargs) + return track_tensor_tree(out, out_proxy, constant=None, tracer=proxy_mode.tracer) + + +@hints_wrapper.py_impl(ProxyTorchDispatchMode) +def inner(proxy_mode, body_fn, args, kwargs, hints): + if proxy_mode.enable_tracing: + return trace_hints_wrapper( + proxy_mode, hints_wrapper, body_fn, args, kwargs, hints + ) + else: + return hints_wrapper(body_fn, args, kwargs, hints) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/invoke_subgraph.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/invoke_subgraph.py new file mode 100644 index 0000000000000000000000000000000000000000..8eb3901ab07342d9c59397d0c4a65b298c9cb9ef --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/invoke_subgraph.py @@ -0,0 +1,776 @@ +# mypy: allow-untyped-defs + +import contextlib +from contextlib import nullcontext +from dataclasses import dataclass, field +from typing import Any, Optional, TYPE_CHECKING, Union + +import torch +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._dispatch.python import suspend_functionalization +from torch._higher_order_ops.utils import ( + _from_fun, + _maybe_reenter_make_fx, + _set_compilation_env, + clone_outputs_aliasing_inputs, + FunctionalizeCtxWrapper, + get_dummy_aot_autograd_config, + HopInstance, + prepare_fw_with_masks, + reenter_make_fx, + register_fake, + save_tensors_and_symints_for_backward, + saved_tensors_and_symints, +) +from torch._ops import HigherOrderOperator +from torch._subclasses.functional_tensor import disable_functional_mode +from torch.fx.experimental.proxy_tensor import ( + _temp_remove_metadata_torch_function_mode, + _temp_remove_pre_dispatch_torch_function_mode, + disable_proxy_modes_tracing, + ProxyTorchDispatchMode, + track_tensor_tree, +) +from torch.fx.graph_module import GraphModule +from torch.fx.passes.runtime_assert import insert_deferred_runtime_asserts + + +if TYPE_CHECKING: + from collections.abc import Callable + + +invoke_subgraph_counter = 0 + + +# During the tracing of the joint graph, we construct this information. This is +# used to filter out grad_outs/tangents in the `backward` method of +# InvokeSubgraphAutogradOp. +@dataclass +class OutputMetadata: + num_fw_outs: Optional[int] = None + indexes_with_symint: set[int] = field(default_factory=set) + indexes_with_no_grad: set[int] = field(default_factory=set) + + +class InvokeSubgraphHOP(HigherOrderOperator): + def __init__(self) -> None: + # Invoke subgraph does not have any state, it is just a wrapper over a + # subgraph, so we can safely cache the HOP. + super().__init__("invoke_subgraph", cacheable=True) + # This is used by the fake tensor cache key validator to extract the + # subgraph and iterate over the nodes to find if all nodes are fake + # tensor cacheable. + self.subgraph_indexes = [ + 0, + ] + + # identifier is setup by upper part of the stack. This helps us in + # identifying two invoke_subgraph calls have same subgraph. + def __call__( + self, + subgraph: Union[GraphModule, FunctionalizeCtxWrapper], + identifier: Optional[str], + *operands, + ): + assert identifier is None or isinstance(identifier, str), ( + "identifier must be a None or a string" + ) + + assert all( + isinstance(o, (torch.Tensor, int, torch.SymInt, torch.Generator)) + for o in operands + if o is not None + ), ( + f"invoke_subgraph operands must be a list of tensors/ints/SymInts/Generator {operands}" + ) + + return super().__call__(subgraph, identifier, *operands) + + # pyrefly: ignore [bad-override] + def gen_schema(self, subgraph, identifier, *operands): + from torch._higher_order_ops.schema import HopSchemaGenerator + from torch._higher_order_ops.utils import ( + check_input_alias_and_mutation_return_outputs, + materialize_as_graph, + ) + + gm: torch.fx.GraphModule = materialize_as_graph(subgraph, operands) + + schema_gen = HopSchemaGenerator(self) + schema_gen.add_arg("subgraph", gm) + schema_gen.add_arg("identifier", identifier) + ( + _, + _, + _, + mutated_inputs, + outputs, + ) = check_input_alias_and_mutation_return_outputs(gm) + for idx, arg in enumerate(operands): + schema_gen.add_arg(f"arg{idx}", arg, is_mutated=idx in mutated_inputs) + for out in outputs: + schema_gen.add_output(out) + + return schema_gen.gen_schema() + + +invoke_subgraph = InvokeSubgraphHOP() + + +def invoke_subgraph_placeholder(func, *args, **kwargs): + if torch.compiler.is_dynamo_compiling(): + # This is just a placeholder for Dynamo to replace with invoke_subgraph + raise RuntimeError("invoke_subgraph should not be called directly in Dynamo") + + if torch.compiler.is_compiling(): + # For non-strict export tracing, we still want to go through Dynamo + from torch._dynamo.backends.debugging import ( + make_eager_backend_with_torch_function_mode, + ) + + def _invoke_subgraph_placeholder_wrapper(func, args): + return invoke_subgraph_placeholder(func, *args) + + with ( + _set_compilation_env(), + torch._dynamo.utils.disable_cache_limit(), + _temp_remove_pre_dispatch_torch_function_mode(), + ): + with _temp_remove_metadata_torch_function_mode() as metadata_mode: + if metadata_mode: + backend: Union[str, Callable[..., Any]] = ( + make_eager_backend_with_torch_function_mode(metadata_mode) + ) + else: + backend = "eager" + + return torch.compile( + _invoke_subgraph_placeholder_wrapper, + backend=backend, + fullgraph=True, + )(func, args) + + return func(*args, **kwargs) + + +def mark_compile_region(fn=None): + """ + This wrapper instructs torch.compile to compile the wrapped region once and + reuse the compiled artifact, instead of the usual way of aggressively + inlining the function. + + Under the hood, it tells TorchDynamo to use InvokeSubgraph HOP for the + region. For PyTorch eager, this is a no-op. + """ + + def wrap(func): + def inner(*args, **kwargs): + # Get the innermost function to avoid nested compile regions + inner_func = func + while hasattr(inner_func, "__marked_compile_region_fn__"): + inner_func = inner_func.__marked_compile_region_fn__ + return invoke_subgraph_placeholder(inner_func, *args, **kwargs) + + inner.__marked_compile_region_fn__ = func # type: ignore[attr-defined] + + return inner + + if fn: + return wrap(fn) + else: + return wrap + + +def get_invoke_subgraph_cache(): + cache = None + if tracing_ctx := torch._guards.TracingContext.try_get(): + cache = tracing_ctx.hop_dispatch_set_cache.get_cache(invoke_subgraph) + return cache + + +# TODO (@anijain2305) - Delete this function when base_hop uses invoke_subgraph infra +def trace_joint_graph(fn, fw_inputs, fw_outputs): + """ + Naively trace out a joint graph. This simplifies the reconstruction of joint + graph in the min-cut partitioner later on. + """ + from torch._functorch.aot_autograd import create_joint + + dummy_aot_config = get_dummy_aot_autograd_config() + + # This joint_fn is inserted as the backward graph as is. This simplifies the + # min-cut partitioner work later on. + # Input signature - (*primals, *tangents) + # Output signature - (*grads, *fw_outs) + # The output signature is deliberately kept grads first and fw_outs second. + # Having grads first makes the min-cut partitioner HOP graph stitching + # easier. + def joint_fn(*primals_and_tangents): + primals = primals_and_tangents[: len(fw_inputs)] + tangents = primals_and_tangents[len(fw_inputs) :] + + fw_outs, grads = create_joint( + prepare_fw_with_masks(fn), aot_config=dummy_aot_config + )(primals, tangents) + + maybe_clone = clone_outputs_aliasing_inputs(primals_and_tangents) + + # return signature is deliberately kept (*grads, *fw_outs). This + # simplifies partitioning work later on. + return pytree.tree_map(maybe_clone, tuple(grads + list(fw_outs))) + + primals = list(fw_inputs) + # This assumes that the tangent strides match fw_outputs strides. Check the + # InvokeSubgraphAutogradOp backward op for the contiguous call. + tangents = [_from_fun(out) for out in fw_outputs] + + joint_operands = primals + tangents + + return _maybe_reenter_make_fx(joint_fn)(*joint_operands) + + +# TODO (@anijain2305) - Delete this function when base_hop uses invoke_subgraph infra +def create_fw_bw_graph(subgraph, operands, grad_outputs=None): + with suspend_functionalization(), disable_functional_mode(): + with disable_proxy_modes_tracing(): + # args are functional tensors, generate some example tensors + fw_inputs = pytree.tree_map(_from_fun, operands) + + from torch._guards import detect_fake_mode + + fake_mode = detect_fake_mode(fw_inputs) + context = ( + nullcontext() + if fake_mode is None or fake_mode.shape_env is None + else fake_mode.shape_env.ignore_fresh_unbacked_symbols() + ) + + with context: + fw_outs = pytree.tree_map(_from_fun, subgraph(*fw_inputs)) + + num_fw_outs = len(fw_outs) + + # Collect the indexes of none in the output to check that the grad + # is None at the corresponding index in the backward. This check is + # performed in the autograd.Function - InvokeSubgraphAutogradOp. + # Also collect the indexes of no_grad in the output to filter out + # the grad_outs in the `backward` method. + output_metadata = OutputMetadata() + + output_metadata.num_fw_outs = num_fw_outs + for idx, fw_out in enumerate(fw_outs): + if isinstance(fw_out, torch.SymInt): + output_metadata.indexes_with_symint.add(idx) + elif not fw_out.requires_grad: + output_metadata.indexes_with_no_grad.add(idx) + + if grad_outputs is None: + # Infer grad_outputs to be the same properties as the fw_outputs + # if they're not passed in + # Although fw_outs are equivalent to grad_outputs for tracing + # purposes, we have to carefully handle the None and fw_out that do + # not have require_grad. At those indexes, we will have None in the + # backward graph. + grad_outputs = fw_outs + grad_outputs = [grad for grad in grad_outputs if grad is not None] + grad_outputs = [grad for grad in grad_outputs if grad.requires_grad] + + # Force grad_out to be contiguous. This is because at runtime, + # grad_out could have different strides than fw_outs. So, we + # force the grad_outs to be contiguous for both tracing and + # runtime. + grad_outputs = [grad.contiguous() for grad in grad_outputs] + + if any( + not isinstance(out, torch.Tensor) + for out in grad_outputs + if out is not None + ): + raise RuntimeError( + "Expect outputs of invoke_subgraph to only contains tensors or None. " + f"Got types {[type(out) for out in grad_outputs]}." + ) + + # Trace the forward subgraph + fw_graph = _maybe_reenter_make_fx(subgraph)(*fw_inputs) + + # Trace the joint graph and assign it to the bwd graph + bw_graph = trace_joint_graph( + subgraph, + fw_inputs, + grad_outputs, + ) + return fw_graph, bw_graph, output_metadata + + +def get_output_metadata(subgraph, *operands): + """ + Extract metadata about the subgraph outputs WITHOUT executing the subgraph. + This avoids running side-effectful operations twice (once here, once in forward). + We analyze the graph structure statically to extract metadata. + """ + # Unwrap FunctionalizeCtxWrapper if present + if isinstance(subgraph, FunctionalizeCtxWrapper): + subgraph = subgraph.subgraph + + # If not a GraphModule, fall back to execution-based metadata extraction + if not isinstance(subgraph, torch.fx.GraphModule): + return _get_output_metadata_by_execution(subgraph, *operands) + + output_metadata = OutputMetadata() + + # Extract output arguments from the output node + # The output node has args=(output_values,) where output_values is a tuple/list + output_node = next(reversed(subgraph.graph.find_nodes(op="output"))) + output_metadata.num_fw_outs = len(output_node.args[0]) + + for idx, output_arg in enumerate(output_node.args[0]): + if not isinstance(output_arg, torch.fx.Node): + if isinstance(output_arg, int): + output_metadata.indexes_with_symint.add(idx) + output_metadata.indexes_with_no_grad.add(idx) + continue + + # Check node metadata for type information + if output_arg.meta.get("val") is None: + # If we don't have complete metadata for all outputs, fall back to execution + # This is important for correctness (e.g., detecting SymInts) even though it + # runs side-effectful operations + return _get_output_metadata_by_execution(subgraph, *operands) + + val = output_arg.meta["val"] + if isinstance(val, torch.SymInt): + output_metadata.indexes_with_symint.add(idx) + output_metadata.indexes_with_no_grad.add(idx) + elif isinstance(val, torch.Tensor): + # Check if tensor requires grad from metadata + if hasattr(val, "requires_grad") and not val.requires_grad: + output_metadata.indexes_with_no_grad.add(idx) + else: + # Non-tensor, non-symint (shouldn't happen but be safe) + output_metadata.indexes_with_no_grad.add(idx) + + return output_metadata + + +def _get_output_metadata_by_execution(subgraph, *operands): + """ + Fallback: Extract metadata by executing the subgraph. + This should only be used when static analysis fails. + WARNING: This will run side-effectful operations! + """ + + with suspend_functionalization(), disable_functional_mode(): + with disable_proxy_modes_tracing(): + # args are functional tensors, generate some example tensors + fw_inputs = pytree.tree_map(_from_fun, operands) + + from torch._guards import detect_fake_mode + + fake_mode = detect_fake_mode(fw_inputs) + context = ( + nullcontext() + if fake_mode is None or fake_mode.shape_env is None + else fake_mode.shape_env.ignore_fresh_unbacked_symbols() + ) + + with context: + fw_outs = pytree.tree_map(_from_fun, subgraph(*fw_inputs)) + + num_fw_outs = len(fw_outs) + + output_metadata = OutputMetadata() + output_metadata.num_fw_outs = num_fw_outs + + for idx, fw_out in enumerate(fw_outs): + if isinstance(fw_out, torch.SymInt): + output_metadata.indexes_with_symint.add(idx) + elif not fw_out.requires_grad: + output_metadata.indexes_with_no_grad.add(idx) + + return output_metadata + + +def trace_joint_graph_as_bwd( + subgraph, num_primals, joint_operands, include_key_set, exclude_key_set +): + """ + Naively trace out a joint graph. This simplifies the reconstruction of joint + graph in the min-cut partitioner later on. + """ + from torch._functorch.aot_autograd import create_joint + + dummy_aot_config = get_dummy_aot_autograd_config() + + if isinstance(subgraph, torch.fx.GraphModule): + + def graph_with_interpreter(*args): + # Running graph with interpreter is needed for propagating the stack_trace + with torch.fx.traceback.preserve_node_meta(): + return torch.fx.Interpreter(subgraph).run(*args) + + fn = graph_with_interpreter + else: + fn = subgraph + + # This joint_fn is inserted as the backward graph as is. This simplifies the + # min-cut partitioner work later on. + # Input signature - (*primals, *tangents) + # Output signature - (*grads, *fw_outs) + # The output signature is deliberately kept grads first and fw_outs second. + # Having grads first makes the min-cut partitioner HOP graph stitching + # easier. + def joint_fn(*primals_and_tangents): + primals = primals_and_tangents[:num_primals] + tangents = primals_and_tangents[num_primals:] + + fw_outs, grads = create_joint( + prepare_fw_with_masks(fn), aot_config=dummy_aot_config + )(primals, tangents) + + maybe_clone = clone_outputs_aliasing_inputs(primals_and_tangents) + + # return signature is deliberately kept (*grads, *fw_outs). This + # simplifies partitioning work later on. + return pytree.tree_map(maybe_clone, tuple(grads + list(fw_outs))) + + with suspend_functionalization(), disable_functional_mode(): + with disable_proxy_modes_tracing(): + joint_operands = [_from_fun(arg) for arg in joint_operands] + with contextlib.ExitStack() as stack: + stack.enter_context( + torch._C._ForceDispatchKeyGuard(include_key_set, exclude_key_set), + ) + with torch.enable_grad(): + return _maybe_reenter_make_fx(joint_fn)(*joint_operands) + + +class InvokeSubgraphAutogradOp(torch.autograd.Function): + """ + Saves the subgraph, i.e. original callable, in the forward method. And then + traces out a joint graph in the backward. This delaying of tracing in + backward, also called as lazy backward, ensures that the assumptions about + the grad_out strides and tensor-subclass-ness are already accounted for. + """ + + @staticmethod + # pyrefly: ignore [bad-override] + def forward( + ctx, + subgraph, + identifier, + output_metadata, + *operands, + ): + # We want to delay the backward graph construction until the backward. + # So in forward, we just run the fw callable as is. And save all the + # information necessary to construct the backward graph in the ctx. + ctx._subgraph = subgraph + ctx._identifier = identifier + ctx._output_metadata = output_metadata + # We snapshot the dispatch keys in forward for materializing the + # the bw_graph in backward. + ctx._fw_include_key_set = torch._C._dispatch_tls_local_include_set() + ctx._fw_exclude_key_set = torch._C._dispatch_tls_local_exclude_set() + + save_tensors_and_symints_for_backward(ctx, operands) + + with torch._C._AutoDispatchBelowAutograd(): + out = invoke_subgraph( + subgraph, + f"fw_{identifier}", + *operands, + ) + + # Check that int (coming from symint) is at expected indexes. + for idx, o in enumerate(out): + if isinstance(o, int): + assert idx in output_metadata.indexes_with_symint + + return out + + @staticmethod + def backward( + ctx, + *grad_outs, + ): + from torch._dynamo.utils import dynamo_timed + + subgraph = ctx._subgraph + identifier = ctx._identifier + output_metadata = ctx._output_metadata + primals = saved_tensors_and_symints(ctx) + + # Filter out grads that are None or do not require_grad. This was + # the assumption we made during the tracing of joint_graph. + filtered_grad_outs = [] + for idx, o in enumerate(grad_outs): + if o is None: + assert idx in output_metadata.indexes_with_symint + elif idx in output_metadata.indexes_with_no_grad: + # Deliberately skip over the grad_outs which we know should be + # None because the corresponding fwd_out does not require_grad. + pass + else: + filtered_grad_outs.append(o) + filtered_grad_outs = tuple(filtered_grad_outs) + + # Important note - Even though the forward graph can be same for + # different invoke_subgraphs, the backward graph can be different + # because the tangent strides can be different. So, here we cache on + # tangent_metadata in addition to identifier + from torch._guards import detect_fake_mode + from torch._subclasses._fake_tensor_utils import _CacheKeyState + from torch._subclasses.fake_tensor import extract_tensor_metadata + + fake_mode = detect_fake_mode(primals + filtered_grad_outs) + assert fake_mode is not None, "fake_mode should be enabled for HOPs" + state = _CacheKeyState(fake_mode.shape_env) + + tangent_metadata: list[object] = [] + for tangent in filtered_grad_outs: + metadata = extract_tensor_metadata(tangent) + metadata._flatten_into(tangent_metadata, fake_mode, state) + # pyrefly: ignore [bad-assignment] + tangent_metadata = tuple(tangent_metadata) + + # bw_graph is a joint graph with signature (*primals_and_tangents) and + # returns (*grads_and_fw_outs). To get the grads, we use the num_fw_outs + # to extract the grads. + primals_and_tangents = primals + filtered_grad_outs + + # Check if we have already traced the bwd subgraph. + bw_graph = None + suffix = None + invoke_subgraph_cache = get_invoke_subgraph_cache() + cache_hit = False + if invoke_subgraph_cache: + bw_graph, suffix = invoke_subgraph_cache.get_lazy_bwd_entry( + identifier, tangent_metadata + ) + cache_hit = bw_graph is not None + + if bw_graph is None: + assert suffix is None + with dynamo_timed( + "invoke_subgraph_trace_joint_graph", log_pt2_compile_event=True + ): + bw_graph = trace_joint_graph_as_bwd( + subgraph, + len(primals), + primals_and_tangents, + ctx._fw_include_key_set, + ctx._fw_exclude_key_set, + ) + + if invoke_subgraph_cache and not cache_hit: + suffix = invoke_subgraph_cache.add_lazy_bwd_entry( + identifier, tangent_metadata, bw_graph + ) + + grads = invoke_subgraph( + bw_graph, f"bw_{identifier}_{suffix}", *primals_and_tangents + )[: -output_metadata.num_fw_outs] + return None, None, None, *grads + + +@invoke_subgraph.py_autograd_impl +def _(subgraph, identifier, *operands): + # Check if we have already traced the subgraph. + invoke_subgraph_cache = get_invoke_subgraph_cache() + if invoke_subgraph_cache: + if saved_autograd_fn := invoke_subgraph_cache.get_autograd_key_entry( + identifier + ): + return saved_autograd_fn(*operands) + + output_metadata = get_output_metadata(subgraph, *operands) + + def autograd_fn_callable(*args): + return InvokeSubgraphAutogradOp.apply( + subgraph, identifier, output_metadata, *args + ) + + # Save the autograd_fn_callable in the dispatch set cache. + if invoke_subgraph_cache: + invoke_subgraph_cache.add_autograd_key_entry(identifier, autograd_fn_callable) + + return autograd_fn_callable(*operands) + + +@invoke_subgraph.py_impl(DispatchKey.CompositeExplicitAutograd) +def _(subgraph, identifier, *operands): + from torch.utils._python_dispatch import _get_current_dispatch_mode + + mode = _get_current_dispatch_mode() + assert mode is None, "Mode should never be enabled for CPU/CUDA key" + if getattr(subgraph, "_boxed_call", False): + return subgraph(list(operands)) + else: + return subgraph(*operands) + + +@invoke_subgraph.py_functionalize_impl +def _(ctx, subgraph, identifier, *operands): + from torch._higher_order_ops.auto_functionalize import ( + can_auto_functionalize, + do_auto_functionalize_v2, + ) + + # (in the functionalization metadata phase) Capture tokens before + tokens_before = dict(ctx.mode._tokens) + + # Check if this subgraph has effects stored in the cache + invoke_subgraph_cache = get_invoke_subgraph_cache() + effects = None + if invoke_subgraph_cache: + effects = invoke_subgraph_cache.get_effects(identifier) + + if effects: + assert len(effects) == 1, "Multiple effects within a subgraph NYI" + tokens = ctx.mode._tokens + effects = next(iter(effects)) + token_input = tokens[effects] + + operands = (token_input, *operands) + + def wrap_subgraph(subgraph): + def wrapped_subgraph(token, *args): + res = subgraph(*args) + return ctx.unwrap_tensors(ctx.mode._tokens[effects]), *res + + return wrapped_subgraph + + subgraph = wrap_subgraph(subgraph) + + unwrapped_operands = ctx.unwrap_tensors(operands) + + hop_instance = HopInstance.create(invoke_subgraph, subgraph, identifier, *operands) + if can_auto_functionalize(hop_instance): + # NOTE: [auto_functionalize x invoke_subgraph caching] + # We call auto_functionalized_v2 to support input mutation of invoke_subgraph. + # See NOTE [Support input mutation of hops] for the overall design. + # + # invoke_subgraph is special because of its identifier based caching mechanism. + # In invoke_subgraph's functionalization key implementation, we create a new + # identifier because the subgraph is replaced by FunctionWithNoFreeVars in a + # functional + epilogue form. + assert isinstance(identifier, str), identifier + return do_auto_functionalize_v2( + ctx.mode, + hop_instance, + (subgraph, "auto_functionalized_" + identifier, *operands), + {}, + ) + + with ctx.redispatch_to_next(): + # NB: There is an assumption that subgraph does not mutate inputs and + # there is no aliasing. Its Dynamo responsibility to prevent formation + # of invoke_subgraph ops if input aliasing/mutation is detected. + functionalized_subgraph = FunctionalizeCtxWrapper(ctx, subgraph) + out = invoke_subgraph(functionalized_subgraph, identifier, *unwrapped_operands) + + if effects: + (new_token, *out) = out + ctx.mode._tokens[effects] = new_token + + # (in the functionalization metadata phase) Capture tokens after and see if + # there are any differences (there are new effects or the token value for an + # effect type has changed) + tokens_after = dict(ctx.mode._tokens) + discovered_effects = set() + for effect_type, token in tokens_after.items(): + if effect_type not in tokens_before or tokens_before[effect_type] is not token: + discovered_effects.add(effect_type) + + if discovered_effects: + assert ctx.mode._allow_token_discovery, ( + f"Number of tokens changed by {len(discovered_effects)} when tracing subgraph {subgraph}." + ) + # Store discovered effects in the cache by identifier + if invoke_subgraph_cache: + invoke_subgraph_cache.add_effects(identifier, discovered_effects) + + return ctx.wrap_tensors(out) + + +# Register the hop fake fn. This will be called in the fake_tensor _dispatch_impl. +@register_fake(invoke_subgraph) +def _(subgraph, identifier, *operands): + from torch._dynamo.utils import dynamo_timed + + with dynamo_timed("invoke_subgraph_fake_tensor", log_pt2_compile_event=True): + return subgraph(*operands) + + +@invoke_subgraph.py_impl(ProxyTorchDispatchMode) +def _(proxy_mode: ProxyTorchDispatchMode, subgraph, identifier, *operands): + # Check if we have already traced the subgraph. + graph = None + invoke_subgraph_cache = get_invoke_subgraph_cache() + if invoke_subgraph_cache: + graph = invoke_subgraph_cache.get_proxy_dispatch_entry(identifier) + + if graph is None: + from torch._dynamo.utils import dynamo_timed + + with dynamo_timed("invoke_subgraph_proxy_tensor", log_pt2_compile_event=True): + graph = reenter_make_fx(subgraph)(*operands) + + from torch._guards import detect_fake_mode + + fake_mode = detect_fake_mode(operands) + assert fake_mode is not None and fake_mode.shape_env is not None + insert_deferred_runtime_asserts( + graph, + fake_mode.shape_env, + "invoke_subgraph_proxy_torch_dispatch_mode", + export=True, + ) + graph.recompile() + + assert isinstance(proxy_mode.tracer, torch.fx.Tracer) + if invoke_subgraph_cache: + invoke_subgraph_cache.add_proxy_dispatch_entry(identifier, graph) + + node_args = (graph, identifier, *operands) + + def _unwrap_proxy(arg): + if isinstance(arg, torch.fx.GraphModule): + # NOTE: [invoke_subgraph proxy_mode x auto_functionalize] + # Previously, we assumed that `invoke_subgraph` would always be traced with the same tracer. + # This allowed us to cache modules by their identifiers, assuming they were already registered. + # + # However, this assumption no longer holds when we auto-functionalize `invoke_subgraph`. + # auto_functionalize functionalizes the subgraph and wrap it with `FunctionWithNoFreeVars`. + # In the proxy mode implementation of `auto_functionalized_v2`, we need to materialize `FunctionWithNoFreeVars` + # input as a graph module. To do this, we re-trace the `invoke_subgraph` hop, which starts a new sub-tracer + # (see NOTE [materialize callable inputs as graph]). # When the new sub-tracer traces the `invoke_subgraph` + # with a previously cached identifier, the corresponding graph module might not + # exist as a submodule in the new tracer's root. Therefore, we register it as a submodule below. + # + # The alternative is to give a new identifier when we re-trace the invoke_subgraph but this will increase + # the compilatoin time, which defeats the purpose of caching. + registered_before = False + for ( + _, + submod, + ) in proxy_mode.tracer.root.named_modules(): # type: ignore[union-attr] + if arg is submod: + registered_before = True + + if not registered_before: + qualname = proxy_mode.tracer.get_fresh_qualname("repeated_subgraph") # type: ignore[union-attr] + proxy_mode.tracer.root.register_module(qualname, arg) # type: ignore[union-attr] + return proxy_mode.tracer.unwrap_proxy(arg) # type: ignore[union-attr] + + proxy_args = pytree.tree_map(_unwrap_proxy, node_args) # type: ignore[union-attr] + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", invoke_subgraph, proxy_args, {} + ) + + example_out = invoke_subgraph(graph, identifier, *operands) + return track_tensor_tree( + example_out, out_proxy, constant=None, tracer=proxy_mode.tracer + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/local_map.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/local_map.py new file mode 100644 index 0000000000000000000000000000000000000000..1d4ad631ea102cd66d9766f2c7c0352e82c69e54 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/local_map.py @@ -0,0 +1,582 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. +# +# This source code is licensed under the BSD license found in the +# LICENSE file in the root directory of this source tree. + +# NOTE: this file may be removed once we move to a dynamo frontend + +import contextlib +import functools +from collections.abc import Callable, Generator, Sequence +from contextlib import contextmanager +from typing import Any, Optional, TypeAlias + +import torch +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._higher_order_ops.utils import ( + clone_outputs_aliasing_inputs, + redirect_to_mode, + save_tensors_and_symints_for_backward, + saved_tensors_and_symints, +) +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode +from torch._subclasses.functional_tensor import FunctionalTensor +from torch.fx import GraphModule +from torch.fx.experimental.proxy_tensor import ProxyTorchDispatchMode, track_tensor_tree +from torch.utils.checkpoint import _CachedTorchDispatchMode, _CachingTorchDispatchMode + + +# Proxy the HOP instead of inlining into it +# And trace it with local shapes for AP +_DEFER_INLINING = False + +GraphArg: TypeAlias = tuple[torch.Tensor, int, torch.SymInt, None] + + +@contextmanager +def defer_inlining() -> Generator[None, None, None]: + global _DEFER_INLINING + prior = _DEFER_INLINING + try: + _DEFER_INLINING = True + yield + finally: + _DEFER_INLINING = prior + + +# Used to unwrap tensors classes like FunctionalTensor and Parameter +def _new_tensor( + t: Any, + new_shape: Optional[Sequence[int]] = None, + new_stride: Optional[Sequence[int]] = None, +) -> Any: + if isinstance(t, torch.Tensor): + assert type(t) in (FunctionalTensor, FakeTensor, torch.Tensor), ( + f"No subclasses support for now, found {type(t)}" + ) + return torch.empty_strided( + t.size() if new_shape is None else new_shape, + t.stride() if new_stride is None else new_stride, + device=t.device, + dtype=t.dtype, + requires_grad=t.requires_grad, + ) + return t + + +# Autoparallel specific, we want to treat plain tensors as DTensors +def _redistribute( + args: Any, + all_placements: tuple[Any], + mesh: Any, + shape_stride_fn: Callable[[torch.Tensor, Any, Any], tuple[list[int], list[int]]], +) -> GraphArg: + from torch._dispatch.python import suspend_functionalization + from torch._guards import detect_fake_mode + from torch._subclasses.functional_tensor import disable_functional_mode + from torch.fx.experimental.proxy_tensor import disable_proxy_modes_tracing + + with ( + suspend_functionalization(), + disable_functional_mode(), + disable_proxy_modes_tracing(), + ): + fake_mode = detect_fake_mode(args) + assert fake_mode is not None, ( + "defer_inlining() is only supported for FakeTensors" + ) + + with fake_mode: + new_args = list(pytree.tree_map(_new_tensor, args)) + for i, (tensor, placements) in enumerate(zip(new_args, all_placements)): + if tensor is None: + # Sometimes gradients can be None + continue + + new_shape, new_stride = shape_stride_fn( + tensor, + mesh, + placements, + ) + new_args[i] = _new_tensor( + tensor, new_shape=new_shape, new_stride=new_stride + ) + + new_args = tuple(new_args) + assert all( + isinstance(t, (FakeTensor, int, torch.SymInt, type(None))) + for t in new_args + ), f"Unexpected element in {args=}" + + return new_args + + +def redistribute_fw_inputs( + global_args: Any, all_placements: Any, mesh: Any, _: Optional[int] = None +) -> GraphArg: + assert len(global_args) == len(all_placements) + return _redistribute( + global_args, + all_placements, + mesh, + torch.distributed.tensor._utils.compute_local_tensor_info, + ) + + +def redistribute_fw_outputs( + local_outs: Any, all_placements: Any, mesh: Any, num_activations: int +) -> GraphArg: + assert len(local_outs) == len(all_placements) + num_activations + num_fw_outs = len(local_outs) - num_activations + assert num_fw_outs > 0 + outs, activations = local_outs[:num_fw_outs], local_outs[num_fw_outs:] + return ( + *_redistribute( + outs, + all_placements, + mesh, + torch.distributed.tensor._utils.compute_global_tensor_info, + ), + *activations, + ) + + +def redistribute_bw_inputs( + global_args: Any, all_placements: Any, mesh: Any, num_activations: int +) -> GraphArg: + assert len(global_args) == len(all_placements) + num_activations + activations, inputs = global_args[:num_activations], global_args[num_activations:] + assert len(inputs) > 0 + local_inputs = _redistribute( + inputs, + all_placements, + mesh, + torch.distributed.tensor._utils.compute_local_tensor_info, + ) + return ( + *activations, + *local_inputs, + ) + + +def redistribute_bw_outputs( + local_outs: Any, all_placements: Any, mesh: Any, _: Optional[int] = None +) -> GraphArg: + assert len(local_outs) == len(all_placements) + return _redistribute( + local_outs, + all_placements, + mesh, + torch.distributed.tensor._utils.compute_global_tensor_info, + ) + + +class LocalMapHOP(HigherOrderOperator): + def __init__(self) -> None: + super().__init__("local_map_hop") + + def __call__(self, gm: GraphModule, *args: Any, **kwargs: Any) -> Any: + return super().__call__(gm, *args, **kwargs) + + +local_map_hop = LocalMapHOP() + +# Registers dispatches for SAC +redirect_to_mode(local_map_hop, _CachingTorchDispatchMode) +redirect_to_mode(local_map_hop, _CachedTorchDispatchMode) + + +def create_hop_fw_bw( + fw_gm: GraphModule, + *_args: Any, +) -> tuple[GraphModule, GraphModule, int, int, set[int]]: + """ + Traces a joint, applies passes and partitions it + """ + # Keeping these imports here + # Avoid circular dependencies once we upstream with dynamo frontend + from torch._dispatch.python import suspend_functionalization + from torch._functorch.aot_autograd import AOTConfig, create_joint + from torch._guards import detect_fake_mode + from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode + from torch._subclasses.functional_tensor import disable_functional_mode + from torch.fx.experimental.proxy_tensor import disable_proxy_modes_tracing, make_fx + + local_map_kwargs = fw_gm.meta["local_map_kwargs"] # type: ignore[attr-defined] + assert "in_placements" in local_map_kwargs + assert "out_placements" in local_map_kwargs + assert "device_mesh" in local_map_kwargs + assert len(local_map_kwargs["in_placements"]) == len(_args) + + dummy_aot_config = AOTConfig( + fw_compiler=None, # type: ignore[arg-type] + bw_compiler=None, # type: ignore[arg-type] + partition_fn=None, # type: ignore[arg-type] + decompositions={}, + num_params_buffers=0, + aot_id=0, + keep_inference_input_mutations=False, + ) + + with suspend_functionalization(), disable_functional_mode(): + with disable_proxy_modes_tracing(): + # If someone runs this hop under the default compiler backend ("eager") + # Then this path will be run with the actual user inputs. We convert them + # to fake tensors in order to not perform any actual compute. + + fake_mode = detect_fake_mode(_args) + if fake_mode is None: + fake_mode = FakeTensorMode(allow_non_fake_inputs=True) + + with fake_mode: + fw_inputs = redistribute_fw_inputs( + _args, + local_map_kwargs["in_placements"], + local_map_kwargs["device_mesh"], + ) + assert len(fw_inputs) == len(local_map_kwargs["in_placements"]) + + assert all( + isinstance(t, (FakeTensor, int, torch.SymInt)) for t in fw_inputs + ), f"Unexpected element in {fw_inputs=}" + + ctx = ( + fake_mode.shape_env.ignore_fresh_unbacked_symbols + if fake_mode.shape_env is not None + else contextlib.nullcontext + ) + with ctx(): + fw_outs = fw_gm(*fw_inputs) + + example_grads = pytree.tree_map( + _new_tensor, + fw_outs, + ) + if not isinstance(example_grads, (list, tuple)): + example_grads = [example_grads] + + num_fw_inputs = len(fw_inputs) + num_fw_outputs = len(example_grads) + + def joint_f( + *primals_and_tangents: list[torch.Tensor], + ) -> Any: + primals = primals_and_tangents[:num_fw_inputs] + tangents = primals_and_tangents[num_fw_inputs:] + + def prepare_fw_with_masks( + fw_gm: torch.fx.GraphModule, + ) -> Callable[..., Any]: + def fw_with_masks(*args: Any) -> tuple[tuple[Any], list[bool]]: + # The Interpreter here is required to propagate metadata + # from the dynamo graph body to the local_map graph body. + # This is required for fx_traceback.annotate for work. + fw_out = torch.fx.Interpreter(fw_gm).run(*args) + assert isinstance(fw_out, tuple), ( + "Dynamo traced submodule should return tuple" + ) + return fw_out, [ + bool(isinstance(ret, torch.Tensor) and ret.requires_grad) + for ret in fw_out + ] + + return fw_with_masks + + fw_outs, grads = create_joint( + prepare_fw_with_masks(fw_gm), aot_config=dummy_aot_config + )(primals, tangents) + from torch.fx.experimental.symbolic_shapes import has_free_unbacked_symbols + + assert not has_free_unbacked_symbols((*fw_outs, *grads)), ( + "Unbacked symints leaking outside of the joint graph is not yet supported." + ) + + maybe_clone = clone_outputs_aliasing_inputs(primals_and_tangents) + # put grads first to work with existing hop utils + return pytree.tree_map(maybe_clone, (*grads, *fw_outs)) + + filtered_grads_idx = set() + for i, example_grad in enumerate(example_grads): + # Filter out grads that are None or do not require_grad. + # The AOTAutograd utils we rely on force this assumption. + # We must also filter the runtime tangents too. + if example_grad is not None and ( + isinstance(example_grad, torch.Tensor) and example_grad.requires_grad + ): + filtered_grads_idx.add(i) + + primals_and_tangents = [ + *fw_inputs, + *[example_grads[i] for i in filtered_grads_idx], + ] + joint_hop_gm = make_fx(joint_f)(*primals_and_tangents) + from torch._functorch._aot_autograd.graph_capture import ( + copy_fwd_metadata_to_bw_nodes, + ) + + copy_fwd_metadata_to_bw_nodes(joint_hop_gm) + + from torch._functorch._aot_autograd.graph_compile import prepare_for_partitioner + from torch._inductor.compile_fx import partition_fn + + # Match partitioner convention + prepped_joint_hop_gm = prepare_for_partitioner( + joint_hop_gm, num_fw_inputs, num_fw_outputs + ) + with disable_proxy_modes_tracing(): + # Also runs joint passes + new_fw_gm, new_bw_gm = partition_fn( + prepped_joint_hop_gm, + [], + num_fwd_outputs=num_fw_outputs, + static_lifetime_input_indices=[], + ) + + # Fix tags because min-cut does not respect fw/bw boundary, breaking + # default partitioner's assumptions. + for node in new_fw_gm.graph.nodes: + node.meta["partitioner_tag"] = "is_forward" + for node in new_bw_gm.graph.nodes: + node.meta["partitioner_tag"] = "is_backward" + + # Propagate meta onto fw/bw graphs, later will be set on proxied nodes + new_fw_gm.meta["local_map_kwargs"] = local_map_kwargs + new_bw_gm.meta["local_map_kwargs"] = {**local_map_kwargs} + # Okay because Autoparallel assumes same sharding between param and grads + new_bw_gm.meta["local_map_kwargs"]["in_placements"] = tuple( + [local_map_kwargs["out_placements"][i] for i in filtered_grads_idx] + ) + new_bw_gm.meta["local_map_kwargs"]["out_placements"] = local_map_kwargs[ + "in_placements" + ] + + # Validate Forward + fw_kwargs = new_fw_gm.meta["local_map_kwargs"] + expected_fw_inputs = len(fw_kwargs["in_placements"]) + expected_fw_outputs = len(fw_kwargs["out_placements"]) + actual_fw_inputs = len(new_fw_gm.graph.find_nodes(op="placeholder")) + actual_fw_outputs = num_fw_outputs + assert expected_fw_inputs == actual_fw_inputs + assert expected_fw_outputs == actual_fw_outputs + + # Validate Activations + assert len(new_fw_gm.graph.find_nodes(op="output")) == 1 + num_activations = ( + len(new_fw_gm.graph.find_nodes(op="output")[0].args[0]) - num_fw_outputs + ) + # tensors first, then symints + assert num_activations >= 0 + + # Validate Backward + bw_kwargs = new_bw_gm.meta["local_map_kwargs"] + expected_bw_inputs = len(bw_kwargs["in_placements"]) + expected_bw_outputs = len(bw_kwargs["out_placements"]) + actual_bw_inputs = ( + len(new_bw_gm.graph.find_nodes(op="placeholder")) - num_activations + ) + assert actual_bw_inputs > 0 + assert expected_fw_inputs + expected_bw_inputs == len(primals_and_tangents) + assert actual_fw_inputs + actual_bw_inputs == len(primals_and_tangents) + assert len(new_bw_gm.graph.find_nodes(op="output")) == 1 + actual_bw_outputs = len(new_bw_gm.graph.find_nodes(op="output")[0].args[0]) + assert expected_bw_inputs == actual_bw_inputs + assert expected_bw_outputs == actual_bw_outputs + + new_fw_gm.meta["num_activations"] = num_activations + new_fw_gm.meta["is_backward"] = False + new_bw_gm.meta["num_activations"] = num_activations + new_bw_gm.meta["is_backward"] = True + + return new_fw_gm, new_bw_gm, num_fw_inputs, num_fw_outputs, filtered_grads_idx + + +class LocalMapAutogradOp(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward( + ctx: Any, + fw_gm: GraphModule, + bw_gm: GraphModule, + num_fw_ins: int, + num_fw_outs: int, + filtered_grads_idx: set[int], + *args: Any, + **kwargs: Any, + ) -> tuple[Optional[torch.Tensor], ...]: + from torch._functorch._aot_autograd.schemas import MemoryFormatMeta + + ctx.bw_gm = bw_gm + ctx.num_fw_ins = num_fw_ins + ctx.filtered_grads_idx = filtered_grads_idx + + with torch._C._AutoDispatchBelowAutograd(): + fw_outs_with_saved_activations = local_map_hop(fw_gm, *args, **kwargs) + + fw_outs = fw_outs_with_saved_activations[:num_fw_outs] + saved_activations = fw_outs_with_saved_activations[num_fw_outs:] + save_tensors_and_symints_for_backward(ctx, saved_activations) + + ctx.expected_tangent_metadata = { + i: MemoryFormatMeta.from_tensor(fw_outs[i]) for i in filtered_grads_idx + } + return fw_outs + + @staticmethod + def backward( + ctx: Any, *_grads: tuple[torch.Tensor] + ) -> tuple[Optional[torch.Tensor], ...]: + from torch._functorch._aot_autograd.runtime_wrappers import ( + coerce_to_expected_memory_format, + ) + + assert ctx.pos == sorted(ctx.pos), ( + "Interleaving saved tensor activations and symints is not expected from min-cut partitioner." + ) + ctx.pos = list( + reversed(ctx.pos) + ) # make saved_tensors_and_symints return symints first + saved_activations = saved_tensors_and_symints(ctx) + with torch._C._AutoDispatchBelowAutograd(): + # Filter out grads that are None or do not require_grad. + # The AOTAutograd utils we rely on force this assumption. + grads = [_grads[i] for i in ctx.filtered_grads_idx] + assert len(grads) == len(ctx.expected_tangent_metadata), ( + f"{len(grads)=} vs {len(ctx.expected_tangent_metadata)}" + ) + + for i, meta in ctx.expected_tangent_metadata.items(): + # pyrefly: ignore [bad-argument-type] + grads[i] = coerce_to_expected_memory_format(grads[i], meta) + + grad_ins = local_map_hop(ctx.bw_gm, *saved_activations, *grads) + if len(grad_ins) != ctx.num_fw_ins: + raise RuntimeError( + f"Expected {ctx.num_fw_ins} grad_ins, got {len(grad_ins)}" + ) + return None, None, None, None, None, *grad_ins + + +@local_map_hop.py_impl(torch._C.DispatchKey.Autograd) +def autograd_key( + fw_gm: GraphModule, + *args: Any, + **kwargs: Any, +) -> Any: + local_map_kwargs = fw_gm.meta["local_map_kwargs"] # type: ignore[attr-defined] + assert local_map_kwargs.get("in_grad_placements", None) is None, ( + "local_map in_grad_placements are not yet supported." + ) + if _DEFER_INLINING: + fw_gm, bw_gm, num_fw_ins, num_fw_outs, filtered_grads_idx = create_hop_fw_bw( + fw_gm, *args + ) + return LocalMapAutogradOp.apply( + fw_gm, bw_gm, num_fw_ins, num_fw_outs, filtered_grads_idx, *args, **kwargs + ) + + # TODO: get rid of this when we can install as a subgraph + return torch.fx.Interpreter(fw_gm).run(*args, **kwargs) + + +@local_map_hop.py_functionalize_impl +def functional_mode_key( + ctx: Any, gm: GraphModule, *args: Any, **kwargs: Any +) -> tuple[torch.Tensor]: + assert not kwargs + + unwrapped_inputs = ctx.unwrap_tensors(args) + with ctx.redispatch_to_next(): + out = local_map_hop(gm, *unwrapped_inputs) + return ctx.wrap_tensors(out) + + +@local_map_hop.py_impl(FakeTensorMode) +def fake_mode_key( + mode: FakeTensorMode, + gm: GraphModule, + *args: Any, + **kwargs: Any, +) -> GraphArg: + with mode: + if not _DEFER_INLINING: + return gm(*args, **kwargs) + + # otherwise, we need to convert to local shapes for AP + is_backward = gm.meta["is_backward"] + redistribute_inputs = ( + redistribute_bw_inputs if is_backward else redistribute_fw_inputs + ) + local_args = redistribute_inputs( + args, + gm.meta["local_map_kwargs"]["in_placements"], + gm.meta["local_map_kwargs"]["device_mesh"], + gm.meta["num_activations"], + ) + local_outs = gm(*local_args) + redistribute_outputs = ( + redistribute_bw_outputs if is_backward else redistribute_fw_outputs + ) + global_outs = redistribute_outputs( + local_outs, + gm.meta["local_map_kwargs"]["out_placements"], + gm.meta["local_map_kwargs"]["device_mesh"], + gm.meta["num_activations"], + ) + return global_outs + + +def proxy_mode_key_common( + call_hop: Callable[..., Any], + proxy_mode: ProxyTorchDispatchMode, + gm: GraphModule, + *args: Any, + **kwargs: Any, +) -> tuple[torch.Tensor]: + assert proxy_mode is not None, ( + "Mode should always be enabled for python fallback key" + ) + assert len(kwargs) == 0 + + example_out = call_hop(*args, **kwargs) + proxy_args = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, args) # type: ignore[union-attr] + + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", call_hop, proxy_args, {} + ) + + # extract local_map args, post-dispatch operates on GraphModules + assert gm.meta["local_map_kwargs"] + local_map_kwargs = gm.meta["local_map_kwargs"] + + # propagate local_map args to the call_function node + out_proxy.node.meta["local_map_kwargs"] = local_map_kwargs + + return track_tensor_tree( + example_out, out_proxy, constant=None, tracer=proxy_mode.tracer + ) + + +@local_map_hop.py_impl(ProxyTorchDispatchMode) +def proxy_mode_key( + proxy_mode: ProxyTorchDispatchMode, + gm: GraphModule, + *args: Any, + **kwargs: Any, +) -> tuple[torch.Tensor]: + # TODO: get rid of this when we can install as a subgraph + def call_local_map(*_args: Any, **_kwargs: Any) -> Any: + return functools.partial(local_map_hop, gm)(*_args, **_kwargs) + + return proxy_mode_key_common(call_local_map, proxy_mode, gm, *args, **kwargs) + + +# Running HOP in eager with real tensors +@local_map_hop.py_impl(DispatchKey.CompositeExplicitAutograd) +def real_impl( + gm: GraphModule, + *args: Any, + **kwargs: Any, +) -> tuple[torch.Tensor]: + return gm(*args, **kwargs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/map.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/map.py new file mode 100644 index 0000000000000000000000000000000000000000..dd30c2efd67ee86170d003f4cd9c88df730671c2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/map.py @@ -0,0 +1,283 @@ +# mypy: allow-untyped-defs +import functools +from collections.abc import Callable +from typing import Union +from typing_extensions import TypeVarTuple + +import torch +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._dispatch.python import suspend_functionalization +from torch._higher_order_ops.utils import _maybe_run_with_interpreter, reenter_make_fx +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensorMode +from torch._subclasses.functional_tensor import disable_functional_mode +from torch.fx.experimental.proxy_tensor import ( + disable_proxy_modes_tracing, + ProxyTorchDispatchMode, + track_tensor_tree, +) + +from .utils import ( + _from_fun, + _stack_pytree, + _unstack_pytree, + create_bw_fn, + fill_none_with_masks, + filter_with_masks, + materialize_as_graph, + save_tensors_and_symints_for_backward, + saved_tensors_and_symints, + split_into_chunks, +) + + +class MapImpl(HigherOrderOperator): + def __init__(self): + super().__init__("map_impl") + + def __call__(self, *args, **kwargs): + return super().__call__(*args, **kwargs) + + +map_impl = MapImpl() + + +def map( + f: Callable[[pytree.PyTree, tuple[pytree.PyTree, ...]], pytree.PyTree], + xs: Union[pytree.PyTree, torch.Tensor], + *args: TypeVarTuple, +): + r""" + Performs a map of f with xs. Intuitively, you can think of the semantic being: + + out = [] + for idx in len(xs.size(0)): + xs_sliced = xs.select(0, idx) + out.append(f(xs_sliced, *args)) + torch.stack(out) + + .. warning:: + `torch._higher_order_ops.map` is a prototype feature in PyTorch. It currently + does not support autograd and you may run into miscompiles. + Read more about feature classification at: + https://pytorch.org/blog/pytorch-feature-classification-changes/#prototype + + + Args: + f (Callable): a callable that takes an input x, that could either be a single Tensor + or a nested dict, list of tensors and some additional inputs + xs: the inputs that're to be mapped over. We'll iterate over the first dim of each x + and perform f on each slice. + + *args: additional arguments provided to each step of f. They could also be omitted and + map is able to automatically figure out the read dependency. + + Return: + the stacked output for each step of f + + Example: + + def f(xs): + return xs[0] + xs[1] + const1 + const2 + + xs = [torch.randn(2, 3), torch.randn(2, 3)] + const1 = torch.randn(2, 3) + const2 = torch.randn(2, 3) + # returns a tensor of shape [2, 2, 3] + torch._higher_order_ops.map(f, xs) + + """ + flat_xs, xs_spec = pytree.tree_flatten(xs) + flat_args, args_spec = pytree.tree_flatten(args) + if not all(isinstance(t, torch.Tensor) for t in flat_xs): + raise RuntimeError(f"Mapped xs can only consist of tensors. Got xs {flat_xs}.") + + shapes = [xs.shape for xs in flat_xs] + leading_dim_size = shapes[0][0] + if leading_dim_size == 0: + raise RuntimeError("Leading dimensions of mapped xs cannot be 0.") + + if any(cur_shape[0] != leading_dim_size for cur_shape in shapes): + raise RuntimeError( + f"Leading dimensions of mapped xs must be consistent. Got shapes {shapes}." + ) + + def run_flattened_map(f, flat_xs, flat_args): + def wrapped_fn(*flat_args, f, xs_tree_spec, args_tree_spec, num_xs): + xs = pytree.tree_unflatten(flat_args[:num_xs], xs_tree_spec) + args = pytree.tree_unflatten(flat_args[num_xs:], args_tree_spec) + return f(xs, *args) + + inner_f = functools.partial( + wrapped_fn, + f=f, + xs_tree_spec=xs_spec, + args_tree_spec=args_spec, + num_xs=len(flat_xs), + ) + return map_impl(inner_f, flat_xs, flat_args) + + from torch._higher_order_ops.utils import _maybe_compile_and_run_fn + + return _maybe_compile_and_run_fn(run_flattened_map, f, flat_xs, flat_args) + + +class MapAutogradOp(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(ctx, f, num_mapped_args, *flat_args): + ctx._f = f + ctx._num_mapped_args = num_mapped_args + ctx._num_pos_args = len(flat_args) - num_mapped_args + + # We snapshot the dispatch keys in forward for materializing the + # the bw_graph in backward. + ctx._fw_include_key_set = torch._C._dispatch_tls_local_include_set() + ctx._fw_exclude_key_set = torch._C._dispatch_tls_local_exclude_set() + save_tensors_and_symints_for_backward(ctx, flat_args) + with torch._C._AutoDispatchBelowAutograd(): + return ( + *map_impl(f, flat_args[:num_mapped_args], flat_args[num_mapped_args:]), + ) + + @staticmethod + def backward(ctx, *flat_grads): + fw_args = saved_tensors_and_symints(ctx) + num_mapped_args = ctx._num_mapped_args + num_pos_args = ctx._num_pos_args + num_grads = len(flat_grads) + + fw_mapped_args, pos_args = split_into_chunks( + fw_args, + [ + num_mapped_args, + num_pos_args, + ], + ) + + bw_f = create_bw_fn(ctx._f, fw_args) + + grads_tensor_masks = [] + + # Create a wrapper around thefor the bw_f + def bw_f_wrapper(*args): + nonlocal grads_tensor_masks + + # Dissect args and re-order them for the ``ctx._bw_f`` + # args provided to the wrapper are composed of [*fw_mapped_args, *flat_grads, *pos_args] + # The content of ``bw_f_tangents`` are the upstream gradients, i.e. flat_grads + # The content of ``bw_f_primals`` are the fw_args, i.e., [*fw_mapped_args, *pos_args] + # The bw_f requires *bw_f_primals, *bw_f_tangents + fw_m_args, bw_f_tangents, pos_args = split_into_chunks( + args, [num_mapped_args, num_grads, num_pos_args] + ) + bw_f_primals = *fw_m_args, *pos_args + gradients = bw_f(*bw_f_primals, *bw_f_tangents) + grads_tensor_masks = [ + True if isinstance(out, torch.Tensor) else out for out in gradients + ] + return filter_with_masks(gradients, grads_tensor_masks) + + def construct_args_single_step_bw(): + unwrapped_mapped_xs = pytree.tree_map(_from_fun, fw_mapped_args) + example_xs = _unstack_pytree(unwrapped_mapped_xs)[0] + unwrapped_grads = pytree.tree_map(_from_fun, flat_grads) + example_grads = _unstack_pytree(unwrapped_grads)[0] + example_pos_args = [ + _from_fun(arg) if isinstance(arg, torch.Tensor) else arg + for arg in pos_args + ] + return *example_xs, *example_grads, *example_pos_args + + with suspend_functionalization(), disable_functional_mode(): + with disable_proxy_modes_tracing(): + args_single_step_bw = construct_args_single_step_bw() + + # TODO: we need to materialize the bw graphs because dynamo is unable to + # trace through the joint function when torch.compile torch.autograd.grad. + fn_bw_gm = materialize_as_graph( + bw_f_wrapper, + args_single_step_bw, + ctx._fw_include_key_set, + ctx._fw_exclude_key_set, + force_enable_grad=True, + ) + + grads = map_impl(fn_bw_gm, fw_mapped_args + flat_grads, pos_args) + + return None, None, *fill_none_with_masks(grads, grads_tensor_masks) + + +def trace_map(proxy_mode, func_overload, f, xs, pos_args): + with disable_proxy_modes_tracing(): + example_input = _unstack_pytree(xs)[0] + + body_graph = f + + body_graph = reenter_make_fx(body_graph)(*example_input, *pos_args) + + next_name = proxy_mode.tracer.get_fresh_qualname("body_graph_") + + proxy_mode.tracer.root.register_module(next_name, body_graph) + + fake_outs = map_impl(body_graph, xs, pos_args) + + node_args = (body_graph, list(xs), list(pos_args)) + proxy_args = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, node_args) + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", func_overload, proxy_args, {}, name="map_impl" + ) + return track_tensor_tree( + fake_outs, out_proxy, constant=None, tracer=proxy_mode.tracer + ) + + +@map_impl.py_impl(DispatchKey.CompositeExplicitAutograd) +def map_dense(f, xs, pos_args): + pytrees = [f(*inp, *pos_args) for inp in _unstack_pytree(xs)] + return _stack_pytree(pytrees) + + +@map_impl.py_autograd_impl +def map_autograd(f, xs, pos_args): + num_mapped_args = len(xs) + flat_out = MapAutogradOp.apply(f, num_mapped_args, *xs, *pos_args) + return flat_out + + +@map_impl.py_impl(ProxyTorchDispatchMode) +def map_proxy_torch_dispatch_mode(mode, f, xs, args): + return trace_map(mode, map_impl, f, xs, args) + + +@map_impl.py_impl(FakeTensorMode) +def map_fake_tensor_mode(mode, f, xs, args): + with mode: + return map_dense(f, xs, args) + + +@map_impl.py_functionalize_impl +def map_functionalize(ctx, f, xs, pos_args): + from torch._higher_order_ops.utils import _check_alias_and_mutation + + unwrapped_xs = ctx.unwrap_tensors(xs) + unwrapped_args = ctx.unwrap_tensors(pos_args) + wrapped_fn = ctx.functionalize(_maybe_run_with_interpreter(f)) + + with ctx.redispatch_to_next(): + example_inputs = (*_unstack_pytree(unwrapped_xs)[0], *unwrapped_args) + pre_dispatch = hasattr(ctx, "mode") and ctx.mode.pre_dispatch + _check_alias_and_mutation(f, example_inputs, "map", pre_dispatch) + map_return = map_impl(wrapped_fn, unwrapped_xs, unwrapped_args) + return ctx.wrap_tensors(map_return) + + +def _fake_map(f, x, *args): + from functorch.experimental.control_flow import _stack_pytree, _unstack_pytree + + x_pytrees = _unstack_pytree(x) + zs = [] + for xp in x_pytrees: + zs.append(f(xp, *args)) + return _stack_pytree(zs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/out_dtype.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/out_dtype.py new file mode 100644 index 0000000000000000000000000000000000000000..5f6e409fb215e3961104a37c4013b769a7560824 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/out_dtype.py @@ -0,0 +1,163 @@ +# mypy: allow-untyped-defs + +import torch +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._higher_order_ops.utils import autograd_not_implemented +from torch._ops import HigherOrderOperator +from torch._prims_common import elementwise_dtypes, ELEMENTWISE_TYPE_PROMOTION_KIND +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ( + disable_proxy_modes_tracing, + maybe_handle_decomp, + ProxyTorchDispatchMode, + track_tensor_tree, +) + + +# TODO to figure out a more generic approach +ALLOWABLE_OPS = [ + torch.ops.aten.linear.default, + torch.ops.aten.mm.default, + torch.ops.aten.conv2d.default, + torch.ops.aten.convolution.default, + torch.ops.aten.mul.Tensor, + torch.ops.aten.mul.Scalar, + torch.ops.aten.div.Tensor, + torch.ops.aten.div.Scalar, +] + + +class OutDtypeOperator(HigherOrderOperator): + """ + The out_dtype operator takes an existing ATen functional operator, an + `out_dtype` argument, and arguments to the original operator, and executes + the original operator and returns a Tensor with the `out_dtype` precision. + This operator does not mandate a compute precision so it allows the + representation to not be opinionated about the exact implementation. + + The general implementation for all operators will be the following: + 1. Promote inputs dtypes based on default PyTorch dtype promotion rules, + using the dtypes of all input Tensors/Scalars and the `out_dtype` + arugument. + 2. Execute the operator + 3. Cast the output to `out_dtype` + """ + + def __init__(self) -> None: + super().__init__("out_dtype") + + def __call__(self, op, output_dtype, *args): + if not isinstance(op, torch._ops.OpOverload): + raise ValueError("out_dtype's first argument must be an OpOverload") + if op._schema.is_mutable: + raise ValueError( + "out_dtype's first argument needs to be a functional operator" + ) + if not ( + len(op._schema.returns) == 1 + and isinstance(op._schema.returns[0].type, torch.TensorType) + ): + raise ValueError( + "out_dtype's can only apply to ops that return a single tensor" + f"Instead got {[r.type for r in op._schema.returns]}" + ) + + if op not in ALLOWABLE_OPS: + raise ValueError( + f"out_dtype only allows the following operators: {ALLOWABLE_OPS}." + ) + + res = super().__call__(op, output_dtype, *args) + + return res + + +out_dtype = OutDtypeOperator() + + +def trace_out_dtype(proxy_mode, func_overload, op, output_dtype, *args): + # NB: Long-term we should put the decomposition logic into + # ProxyTorchDispatchMode so that people do not need to call maybe_handle_decomp + # in all HigherOrderOp proxy implementations. + r = maybe_handle_decomp(proxy_mode, func_overload, (op, output_dtype, *args), {}) + if r is not NotImplemented: + return r + + with disable_proxy_modes_tracing(): + # This is a simplified implementation of this operator just for tracing. + # Actual implementation may also first promote the arguments + out = op(*args).to(dtype=output_dtype) + + node_args = (op, output_dtype, *args) + proxy_args = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, node_args) + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", func_overload, proxy_args, {}, name="out_dtype" + ) + return track_tensor_tree(out, out_proxy, constant=None, tracer=proxy_mode.tracer) + + +@out_dtype.py_impl(DispatchKey.CompositeExplicitAutograd) +def out_dtype_dense(op: torch._ops.OpOverload, output_dtype: torch.dtype, *args): + if is_int_mm(op, output_dtype, args): + return torch._int_mm(*args) + return out_dtype_fallback(op, output_dtype, *args) + + +def is_int_mm(op, output_dtype, args): + return ( + op is torch.ops.aten.mm.default + and output_dtype == torch.int32 + and len(args) == 2 + and args[0].dtype == torch.int8 + and args[1].dtype == torch.int8 + and (args[0].is_cuda or args[0].is_xpu) + and (args[1].is_cuda or args[1].is_xpu) + ) + + +def out_dtype_fallback(op, output_dtype, *args): + flat_inputs = pytree.arg_tree_leaves(*args) + [torch.ones(1, dtype=output_dtype)] + promote_dtype: torch.dtype = elementwise_dtypes( + *flat_inputs, + type_promotion_kind=ELEMENTWISE_TYPE_PROMOTION_KIND.DEFAULT, + )[0] + + casted_args = pytree.tree_map_only( + torch.Tensor, lambda arg: arg.to(dtype=promote_dtype), args + ) + res = op(*casted_args).to(dtype=output_dtype) + return res + + +out_dtype.py_autograd_impl(autograd_not_implemented(out_dtype, deferred_error=True)) + + +@out_dtype.py_impl(ProxyTorchDispatchMode) +def out_dtype_proxy( + mode: ProxyTorchDispatchMode, + op: torch._ops.OpOverload, + output_dtype: torch.dtype, + *args, +): + return trace_out_dtype(mode, out_dtype, op, output_dtype, *args) + + +@out_dtype.py_impl(FakeTensorMode) +def out_dtype_fake_tensor_mode( + mode: FakeTensorMode, + op: torch._ops.OpOverload, + output_dtype: torch.dtype, + *args, +): + with mode: + return out_dtype_dense(op, output_dtype, *args) + + +@out_dtype.py_functionalize_impl +def out_dtype_func(ctx, op, output_dtype, *args): + unwrapped_args = tuple(ctx.unwrap_tensors(arg) for arg in args) + + with ctx.redispatch_to_next(): + res = out_dtype(op, output_dtype, *unwrapped_args) + return ctx.wrap_tensors(res) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/partitioner.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/partitioner.py new file mode 100644 index 0000000000000000000000000000000000000000..2a21601aa9d9df0a8e1b57fa58b4690127de5f5e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/partitioner.py @@ -0,0 +1,365 @@ +import logging +from collections.abc import Callable +from typing import Any, Union + +import torch +from torch._higher_order_ops.utils import create_bw_fn, materialize_as_graph + + +logger: logging.Logger = logging.getLogger(__name__) + +logger.setLevel(logging.DEBUG) + + +def _find_hop_subgraph_outputs(gm: torch.fx.GraphModule) -> tuple[torch.fx.Node]: + output_node_args = gm.graph.find_nodes(op="output")[0].args + assert isinstance(output_node_args, tuple) + return output_node_args[0] + + +def is_complex_expr(expr: Any) -> bool: + return not expr.is_symbol and not expr.is_constant() + + +class HopPartitionedGraph: + def __init__( + self, + fw_gm: torch.fx.GraphModule, + bw_gm: torch.fx.GraphModule, + n_fw_outputs: int, + n_intermediates: int, + no_complex_exprs_at_boundary: bool, + ): + self.fw_gm = fw_gm + self.bw_gm = bw_gm + self.n_fw_outputs = n_fw_outputs + self.n_intermediates = n_intermediates + self.no_complex_exprs_at_boundary = no_complex_exprs_at_boundary + self._reorder_fw_output() + self._check_partition_boundary() + + def _check_partition_boundary(self) -> None: + """check partitioned graph is in valid state.""" + invalid_reasons = [] + fw_outputs = _find_hop_subgraph_outputs(self.fw_gm) + for i, out in enumerate(fw_outputs): + if "val" not in out.meta: + invalid_reasons.append(f"fw_gm output[{i}] doesn't have a 'val' meta.") + elif not isinstance(out.meta["val"], (torch.SymInt, torch.Tensor)): + invalid_reasons.append( + f"fw_gm output[{i}] is of type {type(out.meta['val'])} but only SymInt or Tensor are allowed." + ) + + elif ( + isinstance(out.meta["val"], torch.SymInt) + and is_complex_expr(out.meta["val"].node.expr) + and self.no_complex_exprs_at_boundary + ): + invalid_reasons.append( + f"fw_gm output[{i}] must be of type SymInt with basic symbols or " + f"Tensor but got {type(out.meta['val'])} {out.meta['val']}" + ) + + if len(fw_outputs) != self.n_fw_outputs + self.n_intermediates: + invalid_reasons.append( + f"len(fw_outputs) ({len(fw_outputs)}) != n_fw_outputs ({self.n_fw_outputs}) + n_intermediates ({self.n_intermediates})" # noqa: B950 + ) + + bw_phs = list(self.bw_gm.graph.find_nodes(op="placeholder")) + + if len(fw_outputs) != len(bw_phs): + invalid_reasons.append( + f"Expect number of fw_gm's output to be the same as bw_gm's input but " + f"fw_gm has {len(fw_outputs)} outputs, bw_gm takes {len(bw_phs)} inputs." + ) + + original_forward_outputs = fw_outputs[: self.n_fw_outputs] + fw_intermediates = fw_outputs[self.n_fw_outputs :] + + bw_intermediates = bw_phs[: -self.n_fw_outputs] + bw_grads = bw_phs[-self.n_fw_outputs :] + + def _match_size_or_expr( + val1: Union[torch.SymInt, torch.Tensor], + val2: Union[torch.SymInt, torch.Tensor], + ) -> bool: + if type(val1) is not type(val2): + return False + + if isinstance(val1, torch.SymInt) and isinstance(val2, torch.SymInt): + return val1.node.expr == val2.node.expr + elif isinstance(val1, torch.Tensor) and isinstance(val2, torch.Tensor): + return val1.size() == val2.size() + + return False + + for fw, bw in zip(fw_intermediates, bw_intermediates): + if fw.name != bw.name or not _match_size_or_expr( + fw.meta["val"], bw.meta["val"] + ): + invalid_reasons.append("fw intermediates don't match bw intermediates") + + for fw_out, bw_grad in zip(original_forward_outputs, bw_grads): + if not _match_size_or_expr(fw_out.meta["val"], bw_grad.meta["val"]): + invalid_reasons.append("fw outputs don't match bw gradients") + + if len(invalid_reasons) > 0: + newline = "\n" + raise RuntimeError( + "Invalid HopPartitionedGraph. Reasons:\n", + f"{newline.join(invalid_reasons)}", + ) + + def _reorder_fw_output(self) -> None: + """ + Before the pass, fw_gm returns (*fw_outputs, *intermediates1) + and bw_gm takes (*intermediates2, *grad_fw_outputs) as input. + intermediates1 and intermediates2 share the same node names but + they might be in different order. E.g. this could happen if there + are inputs that contain symints. + + To simplify downstream processing, this graph pass normalizes the output of fw_gm + to be consistent with the bacwkard inputs: + + fw_gm: + - input: fw_args + - output: (*fw_outputs, *intermediates) + + bw_gm: + - input: (*intermediates, *grad_fw_outputs) + - output: grad_fw_args + + Example: + + def fw_gm(x, y, z): + a, b, c = f(x), g(y), k(z) + return a, b, c, f_tmp, g_tmp, k_tmp + + , where a, b, c are fw_outputs, f_tmp, g_tmp, k_tmp are intermediates + + The corresponding bw_gm has the following signature: + + def bw_gm(f_tmp, g_tmp, k_tmp, grad_a, grad_b, grac): + return grad_x, grad_y, grad_z + """ + fw_gm_output_nodes = _find_hop_subgraph_outputs(self.fw_gm) + fw_outputs_nodes = fw_gm_output_nodes[: self.n_fw_outputs] + fw_intermediates_nodes = fw_gm_output_nodes[self.n_fw_outputs :] + if len(fw_intermediates_nodes) > 0: + fw_intermediates_name_to_node = {n.name: n for n in fw_intermediates_nodes} + + # First n_intermediates placeholders + bw_names: list[str] = [ + ph.name + for ph in list(self.bw_gm.graph.find_nodes(op="placeholder"))[ + : self.n_intermediates + ] + ] + new_fw_outputs = list(fw_outputs_nodes) + [ + fw_intermediates_name_to_node[name] for name in bw_names + ] + + output_node = self.fw_gm.graph.find_nodes(op="output")[0] + output_node.args = (tuple(new_fw_outputs),) + + self.fw_gm.graph.lint() + self.fw_gm.recompile() + + +class HopJointGraph: + def __init__( + self, + joint_gm: torch.fx.GraphModule, + n_primals: int, + n_fw_outputs: int, + *, + functionalized: bool, + ): + self.joint_gm = joint_gm + self.n_primals = n_primals + self.n_fw_outputs = n_fw_outputs + self.functionalized = functionalized + + self._rename_phs() + self._remove_redundant_sym_size_ops() + + def _rename_phs(self) -> None: + """ + Rename the placeholders for joint_gm so that the partitioner + could recognize which inputs are primals and which are tangents. + """ + self.n_tangents = 0 + for i, ph in enumerate(self.joint_gm.graph.find_nodes(op="placeholder")): + if i < self.n_primals: + ph.target = f"primals_{i}" + ph.name = f"primals_{i}" + else: + self.n_tangents += 1 + ph.target = f"tangents_{i - self.n_primals}" + ph.name = f"tangents_{i - self.n_primals}" + + self.joint_gm.graph.lint() + self.joint_gm.compile() + + def _remove_redundant_sym_size_ops(self) -> None: + """ + Deletes torch.ops.sym_size.int operators whose output is a + corresponding placeholder that holds the same symbol, and replace all usage + of the sym_size node to be directly using the placeholders. + + This is to make sure all basic symbols come from inputs. + """ + placeholder_exprs = {} + for node in self.joint_gm.graph.nodes: + if ( + isinstance(node, torch.fx.Node) + and node.op == "placeholder" + and hasattr(node, "meta") + and "val" in node.meta + ): + val = node.meta["val"] + if isinstance(val, torch.SymInt): + placeholder_exprs[val.node.expr] = node + + nodes_to_remove = [] + for node in self.joint_gm.graph.find_nodes( + op="call_function", target=torch.ops.aten.sym_size.int + ): + assert hasattr(node, "meta") and "val" in node.meta, node + val = node.meta["val"] + expr = val.node.expr + if expr in placeholder_exprs: + placeholder_node = placeholder_exprs[expr] + node.replace_all_uses_with(placeholder_node) + nodes_to_remove.append(node) + + for node in nodes_to_remove: + self.joint_gm.graph.erase_node(node) + + self.joint_gm.graph.lint() + self.joint_gm.recompile() + + def _mark_complex_exprs_as_must_recompute(self) -> None: + """ + For control flow operators such as scan, we don't want to + have symint in the partitioning boundaries because otherwise we would need to support stacking + the symints up, which causes more entropy in the stack. + + By marking the recompute polify for complex nodes as MUST_RECOMPUTE, the partitioning boundary + no longer contains complex expressions. + + Note that this pass doesn't exclude basic symbols from partitioning boundary + and it's up to the downstream to decide whether to return the basic symbol + or have a separate graph pass to remove them. + """ + + from torch._functorch.partitioners import CheckpointPolicy + + for n in ( + node for node in self.joint_gm.graph.nodes if node.op == "call_function" + ): + if "val" not in n.meta: + continue + val = n.meta["val"] + if isinstance(val, torch.SymInt) and is_complex_expr(val.node.expr): + assert n.meta.get("recompute", None) is None + + n.meta["recompute"] = CheckpointPolicy.MUST_RECOMPUTE + + self.joint_gm.graph.lint() + self.joint_gm.recompile() + + def partition( + self, partition_fn: Callable, always_recompute_complex_exprs: bool + ) -> HopPartitionedGraph: + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "before min_cut_partition:\n%s", + self.joint_gm.print_readable(print_output=False), + ) + + if always_recompute_complex_exprs: + self._mark_complex_exprs_as_must_recompute() + + fw_gm, bw_gm = partition_fn( + self.joint_gm, None, num_fwd_outputs=self.n_fw_outputs + ) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug("after partition_fn:") + logger.debug("fw_gm:\n%s", fw_gm.print_readable(print_output=False)) + logger.debug("bw_gm:\n%s", bw_gm.print_readable(print_output=False)) + + n_intermediates = len(_find_hop_subgraph_outputs(fw_gm)) - self.n_fw_outputs + + return HopPartitionedGraph( + fw_gm, + bw_gm, + self.n_fw_outputs, + n_intermediates, + always_recompute_complex_exprs, + ) + + +def create_hop_joint_graph( + fw_fn: Callable, + fw_args: tuple[Union[torch.Tensor, torch.SymInt], ...], + functionalize: bool, +) -> HopJointGraph: + fw_gm = materialize_as_graph(fw_fn, fw_args, force_enable_grad=True) + fw_gm_output_nodes = _find_hop_subgraph_outputs(fw_gm) + + assert all( + isinstance(n, torch.fx.Node) and "val" in n.meta for n in fw_gm_output_nodes + ) + fw_gm_output_vals = tuple(n.meta["val"] for n in fw_gm_output_nodes) # type: ignore[arg-type] + + assert all(isinstance(val, torch.Tensor) for val in fw_gm_output_vals) + example_grads = tuple(torch.zeros_like(val) for val in fw_gm_output_vals) + + joint_fn = create_bw_fn(fw_fn, fw_args, return_fw_outputs=True) + joint_gm = materialize_as_graph( + joint_fn, fw_args + example_grads, force_enable_grad=True + ) + if functionalize: + # Need to first trace out the joint_fn with autograd info on + # then functionalize the graph otherwise the grad information is lost + joint_gm = materialize_as_graph( + torch.func.functionalize(joint_gm, remove="mutations_and_views"), + fw_args + example_grads, + ) + + return HopJointGraph( + joint_gm, + len(fw_args), + len(fw_gm_output_nodes), + functionalized=functionalize, + ) + + +class HopGraphMinCutPartitioner: + @staticmethod + def create_partitioned_graph( + fw_fn: Callable, + fw_args: tuple[Union[torch.Tensor, torch.SymInt], ...], + *, + always_recompute_complex_exprs: bool = False, + ) -> HopPartitionedGraph: + """ + Inputs: + - fw_fn: the forward function that we'll use to create a joint graph and partition + - fw_args: the flat_args to fw_fn + - always_recompute_complex_exprs: when set to True, the bw_gm will do a re-compute + for inputs that are complex expressions such that the partitioning boundary + only consists of basic symbols and tensors. + + Returns a HopPartitionedGraph + """ + from torch._functorch.partitioners import min_cut_rematerialization_partition + + joint_graph: HopJointGraph = create_hop_joint_graph( + fw_fn, fw_args, functionalize=True + ) + return joint_graph.partition( + min_cut_rematerialization_partition, always_recompute_complex_exprs + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/print.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/print.py new file mode 100644 index 0000000000000000000000000000000000000000..889c2a4ca7836e2fca2971af3c96f5a6c5cf1d2f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/print.py @@ -0,0 +1,93 @@ +import builtins + +import torch +import torch.utils._pytree as pytree +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ProxyTorchDispatchMode + + +class Print(HigherOrderOperator): + """ + print(format_str, **kwargs) -> None + + This Higher Order Operator (HOP) provides a functional version of print for use in PyTorch graphs. + It enables format printing with named arguments, e.g., torch._higher_order_ops.print("moo {x} {y}", x=1, y=2). + + This HOP enables printing without causing graph break. + """ + + def __init__(self) -> None: + super().__init__("print") + + def __call__(self, format_str: str, **kwargs: object) -> None: + assert isinstance(format_str, str) + return super().__call__(format_str, **kwargs) + + # pyrefly: ignore [bad-override] + def gen_schema(self, format_str: str, **kwargs: object) -> torch.FunctionSchema: + from torch._higher_order_ops.schema import HopSchemaGenerator + + schema_gen = HopSchemaGenerator(self) + schema_gen.add_arg("format_str", format_str[0]) + + # Add each kwarg as a keyword-only argument + for key, value in kwargs.items(): + schema_gen.add_arg(key, value, kw_only=True) + + schema_gen.add_schema_tree_spec(format_str, **kwargs) + + return schema_gen.gen_schema() + + +print = Print() + + +@print.py_impl(ProxyTorchDispatchMode) +# pyre-ignore +def print_proxy_torch_dispatch_mode( + mode: ProxyTorchDispatchMode, format_str: str, **kwargs: object +) -> None: + proxy_kwargs = pytree.tree_map(mode.tracer.unwrap_proxy, kwargs) # type: ignore[union-attr] # noqa: F841 + mode.tracer.create_proxy("call_function", print, (format_str,), proxy_kwargs) + + +@print.py_impl(FakeTensorMode) +# pyre-ignore +def print_fake_tensor_mode(mode, format_str: str, **kwargs: object): + return None + + +@print.py_impl(torch._C.DispatchKey.CompositeExplicitAutograd) +# pyre-ignore +def print_impl(format_str: str, **kwargs: object) -> None: + # Ensure all immutable_dict/list in kwargs are converted to regular dict/list + map_types: dict[type, type] = { + torch.fx.immutable_collections.immutable_dict: dict, + torch.fx.immutable_collections.immutable_list: list, + } + new_kwargs = pytree.tree_map_only( + tuple(map_types.keys()), + lambda a: map_types[type(a)](a), + kwargs, + lambda a: isinstance(a, tuple(map_types.keys())), + ) + # Use built-in print to avoid recursion with the HOP print + builtins.print(format_str.format(**new_kwargs)) + + +print.fallthrough(torch._C.DispatchKey.AutogradCPU) +print.fallthrough(torch._C.DispatchKey.AutogradCUDA) + + +@print.py_functionalize_impl +def print_func(ctx, format_str: str, **kwargs: object): + from torch._higher_order_ops.effects import handle_effects + + return handle_effects( + ctx.mode._allow_token_discovery, + ctx.mode._tokens, + print, # type: ignore[arg-type] + (format_str,), + kwargs, # type: ignore[arg-type] + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/run_const_graph.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/run_const_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..ed7c5278f5fe67589119414840db6e0da439ff9c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/run_const_graph.py @@ -0,0 +1,74 @@ +from typing import Any, TYPE_CHECKING + +import torch +from torch._C import DispatchKey +from torch._higher_order_ops.utils import autograd_not_implemented +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensorMode + + +if TYPE_CHECKING: + from torch._subclasses.functional_tensor import BaseFunctionalizeAPI + +from torch.fx.experimental.proxy_tensor import ProxyTorchDispatchMode, track_tensor_tree +from torch.utils import _pytree as pytree + + +class RunConstGraph(HigherOrderOperator): + def __init__(self) -> None: + super().__init__("run_const_graph") + + def __call__(self, graph: torch.fx.GraphModule, args: tuple[object, ...]) -> object: + return super().__call__(graph, args) + + +run_const_graph = RunConstGraph() + + +@run_const_graph.py_impl(ProxyTorchDispatchMode) +def run_const_graph_dispatch_mode( + mode: ProxyTorchDispatchMode, graph: torch.fx.GraphModule, args: tuple[object, ...] +) -> object: + const_gm, weights = graph, args + p_args = pytree.tree_map(mode.tracer.unwrap_proxy, (graph, args)) # type: ignore[union-attr] + assert isinstance(const_gm, torch.fx.GraphModule) + assert not hasattr(mode.tracer.root, "_const_graph") # type: ignore[union-attr] + mode.tracer.root.register_module("_const_graph", const_gm) # type: ignore[union-attr] + + proxy = mode.tracer.create_proxy("call_function", run_const_graph, p_args, {}) + + out = const_gm(*weights) + return track_tensor_tree(out, proxy, constant=None, tracer=mode.tracer) + + +@run_const_graph.py_functionalize_impl +def run_const_graph_functional( + ctx: "BaseFunctionalizeAPI", graph: torch.fx.GraphModule, args: tuple[Any, ...] +) -> Any: + unwrapped_args = ctx.unwrap_tensors(args) + + with ctx.redispatch_to_next(): + out = run_const_graph(graph, unwrapped_args) + return ctx.wrap_tensors(out) # type: ignore[arg-type] + + +run_const_graph.py_autograd_impl( + autograd_not_implemented(run_const_graph, deferred_error=True) +) + + +@run_const_graph.py_impl(FakeTensorMode) +def run_const_graph_fake_tensor_mode( + mode: FakeTensorMode, graph: torch.fx.GraphModule, args: tuple[object, ...] +) -> object: + assert isinstance(graph, torch.fx.GraphModule) + with mode: + return graph(*args) + + +@run_const_graph.py_impl(DispatchKey.CPU) +def run_const_graph_cpu( + graph: torch.fx.GraphModule, args: tuple[object, ...] +) -> object: + assert isinstance(graph, torch.fx.GraphModule) + return graph(*args) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/scan.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/scan.py new file mode 100644 index 0000000000000000000000000000000000000000..852339d11ece57e0c20e714799b0aeecb40ae12d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/scan.py @@ -0,0 +1,965 @@ +# mypy: allow-untyped-defs +import enum +import functools +import itertools +import logging +from collections.abc import Callable +from typing import Any + +import torch +import torch._prims_common as utils +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._higher_order_ops.partitioner import ( + _find_hop_subgraph_outputs, + HopGraphMinCutPartitioner, + HopPartitionedGraph, +) +from torch._higher_order_ops.utils import ( + _maybe_compile_and_run_fn, + check_input_alias_and_mutation_return_outputs, + check_meta_consistency, + fill_none_with_masks, + filter_with_masks, + first_slice_copy, + get_tensor_mask, + mask_list, + materialize_as_graph, + reenter_make_fx, + split_into_chunks, + unique_graph_id, + validate_subgraph_args_types, +) +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ( + disable_proxy_modes_tracing, + ProxyTorchDispatchMode, + track_tensor_tree, +) +from torch.utils._python_dispatch import _get_current_dispatch_mode + + +logger: logging.Logger = logging.getLogger(__name__) +aten = torch._ops.ops.aten + + +def wrap_combine_fn_flat( + *args, combine_fn, spec_init, spec_xs, num_init_leaves, num_inp_leaves +): + assert len(args) == (num_init_leaves + num_inp_leaves), ( + f"combine_fn received wrong number of arguments, expected {num_init_leaves + num_inp_leaves}, but got {len(args)}" + ) + carry = pytree.tree_unflatten(args[:num_init_leaves], spec_init) + xs = pytree.tree_unflatten(args[num_init_leaves:], spec_xs) + return combine_fn(carry, xs) + + +def _extract_carry_and_out(flat_out: list[Any], num_carry: int): + return split_into_chunks(flat_out, [num_carry, len(flat_out) - num_carry]) + + +# We also do a clone with contiguous_format. This is to be consistent with +# eager semantic of scan, which stacks the outputs. The result is contiguous +# as a result of the stack operation. +def stack_y(y: torch.Tensor, scan_length: int) -> torch.Tensor: + return ( + y.unsqueeze(0) + .repeat(*([scan_length] + [1] * y.ndim)) + .clone(memory_format=torch.contiguous_format) + ) + + +def call_operator(operator, *args): + return pytree.tree_leaves(operator(*args)) + + +def scan( + combine_fn: Callable[ + [pytree.PyTree, pytree.PyTree], tuple[pytree.PyTree, pytree.PyTree] + ], + init: pytree.PyTree, + xs: pytree.PyTree, + *, + dim: int = 0, + reverse: bool = False, +) -> tuple[pytree.PyTree, pytree.PyTree]: + r""" + Performs an inclusive scan with a combine function. + + .. warning:: + `torch.scan` is a prototype feature in PyTorch. It currently + does not support autograd and you may run into miscompiles. + Read more about feature classification at: + https://pytorch.org/blog/pytorch-feature-classification-changes/#prototype + + Args: + combine_fn (Callable): A binary callable with type ``(Tensor, Tensor) -> (Tensor, Tensor)``, + or if xs is a pytree ``(pytree, pytree) -> (pytree, pytree)``. + The first input to ``combine_fn`` is the previous or initial scan carry + and the second input element to ``combine_fn`` is a slice of the input along dim. + The first output element of ``combine_fn`` is the next scan carry + and the second output of ``combine_fn`` represents a slice of the output. + This function must be pure, i.e., no lifted arguments are supported at the moment + and may not have any side effects. + init (torch.Tensor or pytree with tensor leaves): The initial scan carry, a tensor, or nested pytree of tensors. + The ``init`` is expected to have the same pytree structure as the first output element (i.e. carry) + of ``combine_fn``. + xs (torch.Tensor or pytree with tensor leaves): The input tensor, or nested pytree of tensors. + + Kwargs: + dim (int): the dimension to scan over, default 0. + reverse (bool): A boolean stating if the scan should be reversed with respect to ``dim``, default ``False``. + + Returns: + final_carry (torch.Tensor or pytree with tensor leaves), + the final carry of the scan operation with same pytree structure as init. + out (torch.Tensor or pytree with tensor leaves), + each tensor leaf is a stacked output along first dim, where each slice is the output of a scan iteration. + + Restrictions: + - The combine_fn shouldn't have any aliasing between input-input, input-output, and output-output. E.g. return a view + or the same tensor as input is not supported. As a workaround, can clone the output to avoid aliasing. + + - The combine_fn shouldn't mutate any inputs. We'll remove the mutation restriction for inference soon. Please file an issue + if you input mutation support for training is needed. + + - The combine_fn's init carry should match the next_carry in pytree structure and in tensor metadata. + + Example:: + + def add(x: torch.Tensor, y: torch.Tensor): + next_carry = y = x + y + # clone the output to avoid output-output aliasing + return next_carry, y.clone() + + + i0 = torch.zeros(1) + xs = torch.arange(5) + # returns torch.tensor([10.]), torch.tensor([[0], [1.], [3.], [6.], [10.]]) + last_carry, cumsum = scan(add, init=i0, xs=xs) + + + """ + # The reason we flatten init and xs before calling into dynamo is that + # we want to create a consistent input ordering for combine_fn + # and we also want to the input ordering matches the output ordering. + leaves_init, spec_init = pytree.tree_flatten(init) + leaves_xs_orig, spec_xs = pytree.tree_flatten(xs) + + # Shortcut if no xs is provided + if len(leaves_xs_orig) == 0: + return init, [] + + def _validate_input(cfn, lxs, linit, d, r): + # Basic arguments check + if not callable(cfn): + raise RuntimeError(f"Combine_fn must be a callable, but got {cfn}") + if not isinstance(d, int): + raise RuntimeError("Dim must be an int, but got " + str(type(d))) + if not isinstance(r, bool): + raise RuntimeError("Reverse must be a bool, but got " + str(type(r))) + + # Checks for init + if len(linit) == 0: + raise RuntimeError("scan() operator requires init leaves.") + for x in linit: + if not isinstance(x, torch.Tensor): + raise RuntimeError(f"All init leaves must be a Tensor but got {x}") + + # Checks for xs + for x in lxs: + if not isinstance(x, torch.Tensor): + raise RuntimeError(f"All xs leaves must be a Tensor but got {x}") + if any(x.ndim <= d for x in lxs): + raise RuntimeError( + "All xs leaves must at least have 'dim' number of dimensions and scan dimension > 0" + ) + if any(x.shape[d] == 0 for x in lxs): + raise RuntimeError( + "All xs leaves must at least have 'dim' number of dimensions and scan dimension > 0" + ) + + ndim = leaves_xs_orig[0].ndim + dim = utils.canonicalize_dim(ndim, dim) + + _validate_input(combine_fn, leaves_xs_orig, leaves_init, dim, reverse) + + # Move scan dim to 0 and always perform scan on dim 0 + leaves_xs = [] + for elem in leaves_xs_orig: + leaves_xs.append(torch.movedim(elem, dim, 0) if dim != 0 else elem) + + if reverse: + leaves_xs = [torch.flip(elem, [0]) for elem in leaves_xs] + + # TODO: Support _inductor lowering + # TODO: Unify handling of pytrees for control flow ops, such as cond, while_loop, etc. + + combine_fn = functools.partial( + wrap_combine_fn_flat, + combine_fn=combine_fn, + spec_init=spec_init, + spec_xs=spec_xs, + num_init_leaves=len(leaves_init), + num_inp_leaves=len(leaves_xs), + ) + + def run_flattened_scan(combine_fn, leaves_init, leaves_xs): + return scan_op(combine_fn, leaves_init, leaves_xs, additional_inputs=()) + + carry, out = _maybe_compile_and_run_fn( + run_flattened_scan, + combine_fn, + leaves_init, + leaves_xs, + ) + + if reverse: + out = pytree.tree_map(lambda elem: elem.flip([0]), out) + + return carry, out + + +class ScanOp(HigherOrderOperator): + def __init__(self): + super().__init__("scan") + + def __call__(self, combine_fn, init, xs, additional_inputs): + # There is currently an issue that the ScanOp is sometimes called with + # the additional_inputs being a list. See https://github.com/pytorch/pytorch/issues/145785 + # Once this issue is resolved, the assertion should only allow tuples + # and the tuple cast should be removed + assert isinstance(additional_inputs, (tuple, list)), ( + "additional_inputs must be a tuple." + ) + additional_inputs = ( + tuple(additional_inputs) + if isinstance(additional_inputs, list) + else additional_inputs + ) + validate_subgraph_args_types(additional_inputs) + return super().__call__(combine_fn, init, xs, additional_inputs) + + # pyrefly: ignore [bad-override] + def gen_schema(self, combine_fn, init, xs, additional_inputs): + from torch._higher_order_ops.schema import HopSchemaGenerator + from torch._higher_order_ops.utils import materialize_as_graph + + all_inputs = tuple( + list(init) + [first_slice_copy(x) for x in xs] + list(additional_inputs) + ) + + combine_gm: torch.fx.GraphModule = materialize_as_graph(combine_fn, all_inputs) + + ( + _, + _, + _, + mutated_inputs, + outputs, + ) = check_input_alias_and_mutation_return_outputs(combine_gm) + if len(mutated_inputs) > 0: + raise RuntimeError( + "For scan, combine_fn cannot have in-place mutations but found " + f"{mutated_inputs}-th inputs are mutated." + ) + + schema_gen = HopSchemaGenerator(self) + schema_gen.add_arg("combine_fn", combine_gm) + + for idx, arg in enumerate(init): + schema_gen.add_arg(f"init{idx}", arg) + + for idx, arg in enumerate(xs): + schema_gen.add_arg(f"xs{idx}", arg) + + for idx, arg in enumerate(additional_inputs): + schema_gen.add_arg(f"additional_input{idx}", arg) + + for out in outputs: + schema_gen.add_output(out) + + schema_gen.add_schema_tree_spec(combine_fn, init, xs, additional_inputs) + return schema_gen.gen_schema() + + +scan_op = ScanOp() + + +def generic_scan(operator, init, xs, dim=0, additional_inputs=()): + def _scan(init, xs): + """Perform scan on `elems` using `elems_init.""" + carry = init + if len(xs) == 0: + return carry, [] + + num_elems = xs[0].shape[dim] + ind = 0 + + # Compute dummy shapes for the pre-allocation + num_init_leaves = len(init) + dummy_carry, dummy_out = _extract_carry_and_out( + call_operator( + operator, + *carry, + *[first_slice_copy(elem, dim) for elem in xs], + *additional_inputs, + ), + num_init_leaves, + ) + + out_tensor_mask = get_tensor_mask(dummy_out) + dummy_out_masked = mask_list(out_tensor_mask, dummy_out) + + # Pre-allocate + # outs -> Output matrix + # idxs -> Index matrix for scatter_ + # out: (num_elems, M, N, ...) + # idx: (1, M, N) + outs = [ + torch.zeros( + [num_elems] + list(e.size()), + dtype=e.dtype, + device=e.device, + ) + for i, e in enumerate(dummy_out_masked) + ] + idxs = [ + torch.ones_like(e, dtype=torch.int64).unsqueeze(0) + for i, e in enumerate(dummy_out_masked) + ] + + def store_out_in_outs(out, ind): + # Store the intermediate out in the outs matrix + for o, x, idx in zip(outs, out, idxs): + # o: (num_elems, M, N ...) + # x: (M, N, ...) -> (1, M, N) + # ind * idx: (1, M, N,) with values to be ind + # essentially: o[ind][n][k] = x[0][n][k] + o.scatter_(0, ind * idx, x.unsqueeze(0)) + + for i in range(num_elems): + ind = i + carry, out = _extract_carry_and_out( + call_operator( + operator, + *carry, + *[elem.select(dim, ind) for elem in xs], + *additional_inputs, + ), + num_init_leaves, + ) + + # Store the inits in the outs matrix. + store_out_in_outs(mask_list(out_tensor_mask, out), ind) + + # Expand outs with None depending on the tensor mask of the output + outs_expanded = [outs.pop(0) if out_m else None for out_m in out_tensor_mask] + + return (*carry, *outs_expanded) + + scans = _scan(init, xs) + return scans + + +def trace_scan( + proxy_mode, + func_overload, + combine_fn: Callable, + init: list[torch.Tensor], + xs: list[torch.Tensor], + additional_inputs: tuple[torch.Tensor], +): + from torch._dynamo.utils import clone_input + + with disable_proxy_modes_tracing(): + sample_inits = [clone_input(x_init) for x_init in init] + sample_inputs = [first_slice_copy(x) for x in xs] + sample_additional_inputs = [ + clone_input(x) if isinstance(x, torch.Tensor) else x + for x in additional_inputs + ] + combine_graph = reenter_make_fx(combine_fn)( + *sample_inits, *sample_inputs, *sample_additional_inputs + ) + + outputs = None + for node in combine_graph.graph.nodes: + if node.op == "output": + assert outputs is None + assert len(node.args) == 1 + outputs = node.args[0] + + assert outputs is not None + + carry, output = _extract_carry_and_out(outputs, len(init)) + init_fake_tensors: list[torch.Tensor | torch.SymInt | int] = [ + i.clone() for i in init + ] + carry_fake_tensors: list[torch.Tensor | torch.SymInt | int] = [ + c.meta["val"] for c in carry + ] + check_meta_consistency( + init_fake_tensors, carry_fake_tensors, "init", "carry", include_contiguity=False + ) + + _, combine_graph_name = unique_graph_id(proxy_mode, prefix="scan_combine_graph") + + proxy_mode.tracer.root.register_module(combine_graph_name, combine_graph) + + args = (combine_graph, init, xs, additional_inputs) + proxy_args = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, args) + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", func_overload, proxy_args, {}, name="scan" + ) + + with disable_proxy_modes_tracing(): + scan_length = xs[0].shape[0] + fake_carry, fake_outputs = _extract_carry_and_out( + [o.meta["val"] for o in outputs], len(init) + ) + out = ( + *fake_carry, + *(stack_y(t, scan_length) for t in fake_outputs), + ) + + return track_tensor_tree(out, out_proxy, constant=None, tracer=proxy_mode.tracer) + + +@scan_op.py_impl(DispatchKey.CompositeExplicitAutograd) +def scan_op_dense(combine_fn, init, xs, additional_inputs): + mode = _get_current_dispatch_mode() + assert mode is None, "Mode should never be enabled for CPU/CUDA key" + return generic_scan(combine_fn, init, xs, additional_inputs=additional_inputs) + + +class ScanAutogradOp(torch.autograd.Function): + """ + NOTE: [scan partial grad handling] + If any element of init, of xs, of the outputs or of the additional_inputs does not require gradients, + i.e., requires_grad=False, there will be still gradients returned for those elements, + but those gradients will be a tensor filled with zeros of the same shape as the element itself. + + A special case are additional_inputs that are not tensors. Such inputs can occur for example with symbolic tracing, + where the shape symbol (SymInt) becomes an additional_input. + For such cases, we compute a ``additional_inputs_tensor_mask``, which is True for elements of additional_inputs + that are tensors and False otherwise. Gradients of additional_inputs are only accumulated if this mask is True, + otherwise, the value of initial_g_additional_inputs is passed, which is None for non-Tensor values. + """ + + @staticmethod + # pyrefly: ignore [bad-override] + def forward( + ctx, + hop_partitioned_graph, + n_init, + n_xs, + n_additional_inputs, + *operands, + ): + init, xs, additional_inputs = split_into_chunks( + operands, [n_init, n_xs, n_additional_inputs] + ) + ctx._scan_impl = ScanAutogradImpl( + hop_partitioned_graph, init, xs, additional_inputs + ) + with torch._C._AutoDispatchBelowAutograd(): + return ctx._scan_impl.call_forward() + + @staticmethod + def backward(ctx, *grad_fw_outputs): + return ( + None, + None, + None, + None, + *ctx._scan_impl.call_backward(*grad_fw_outputs), + ) + + +class ScanForwardIntermediatesHandlingPolicy(enum.Enum): + """ + Partitioner can add interemdiates to the output of original graph. + These intermediates fall into 4 categories and we want to have different policies for handling them by + modifying the graph: + + CLONE: we clone the intermediate when it is a carried input (i.e. init). In this case, this carry will be + replaced with new values at each forward step so we need to clone the carry as part of return (i.e. ys) + so as to remove the aliasing and that each step's intermediate will be stacked together and saved in bacwkard. + + REMOVE_XS: we remove the intermediate from output when it is part of xs. Since xs is read-only, in this case, + we can directly save them for backward to use. + + REMOVE_ADDITIONAL_INPUTS: we remove the intermediate from output when it is part of additinonal_inputs. additional_inputs + are also read-only in each step, we can directly save them for bacwkard to use. We differentiate XS and ADDITIONAL_INPUTS + so that we could have different treatment for them in backward. In backward, we need to put xs intermediates in carry but + put additional_inputs as backward scan's additional_inputs. + + KEEP: this corresponds to a real intermediate tensor operations' output. It varies at each forward step, we could just keep + it as part of ys. + + """ + + KEEP = 0 + CLONE = 1 + REMOVE_XS = 2 + REMOVE_ADDITIONAL_INPUTS = 3 + + +class ScanAutogradImpl: + """ + Wraps over partitioned graph and encapsulates scan-specific implementation details + """ + + def __init__( + self, hop_partitioned_graph: HopPartitionedGraph, init, xs, additional_inputs + ): + self.hop_partitioned_graph = hop_partitioned_graph + self.init = init + self.xs = xs + self.additional_inputs = additional_inputs + self.forward_intermediates_handling_policies: list[ + ScanForwardIntermediatesHandlingPolicy + ] = [] + self.saved_fw_xs: list[Any] = [] + self.saved_fw_additional_inputs: list[Any] = [] + self.saved_intermediates: list[Any] = [] + self.fw_spec = pytree.tree_flatten((init, xs, additional_inputs))[1] + self._optimize_forward_intermediates() + + def _insert_clone( + self, need_copy_node: torch.fx.Node, output_node: torch.fx.Node + ) -> torch.fx.Node: + graph: torch.fx.Graph = output_node.graph + with graph.inserting_before(output_node): + clone_node = graph.call_function( + torch.ops.aten.clone.default, + args=(need_copy_node,), + ) + clone_node.meta = ( + need_copy_node.meta.copy() if hasattr(need_copy_node, "meta") else {} + ) + return clone_node + + def _optimize_forward_intermediates(self): + """ + We optimize the forward intermediates by categorize forward intermediates into categories + and construct a ScanForwardIntermediatesHandlingPolicy for them + + """ + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "Need remove aliasing in fw_gm:\n%s", + self.hop_partitioned_graph.fw_gm.print_readable(print_output=False), + ) + + fw_gm = self.hop_partitioned_graph.fw_gm + fw_all_outputs = _find_hop_subgraph_outputs(fw_gm) + phs = list(fw_gm.graph.find_nodes(op="placeholder")) + fw_outputs = fw_all_outputs[: self.hop_partitioned_graph.n_fw_outputs] + fw_intermediates = fw_all_outputs[self.hop_partitioned_graph.n_fw_outputs :] + + init_phs, xs_phs, additional_inputs_phs = pytree.tree_unflatten( + phs, self.fw_spec + ) + init_node_set, xs_node_set, addi_node_set = ( + set(init_phs), + set(xs_phs), + set(additional_inputs_phs), + ) + + assert len(self.forward_intermediates_handling_policies) == 0 + assert len(self.saved_fw_xs) == 0 + assert len(self.saved_fw_additional_inputs) == 0 + intermediate_idx_to_ph_idx = {} + ph_idx = {ph: i for i, ph in enumerate(phs)} + for i, out in enumerate(fw_intermediates): + if out in init_node_set: + self.forward_intermediates_handling_policies.append( + ScanForwardIntermediatesHandlingPolicy.CLONE + ) + intermediate_idx_to_ph_idx[i] = ph_idx[out] + elif out in xs_node_set: + self.forward_intermediates_handling_policies.append( + ScanForwardIntermediatesHandlingPolicy.REMOVE_XS + ) + intermediate_idx_to_ph_idx[i] = ph_idx[out] + elif out in addi_node_set: + self.forward_intermediates_handling_policies.append( + ScanForwardIntermediatesHandlingPolicy.REMOVE_ADDITIONAL_INPUTS + ) + intermediate_idx_to_ph_idx[i] = ph_idx[out] + else: + self.forward_intermediates_handling_policies.append( + ScanForwardIntermediatesHandlingPolicy.KEEP + ) + + new_output_node = [] + real_graph_inputs = ( + list(self.init) + list(self.xs) + list(self.additional_inputs) + ) + fw_output_node = next(iter(fw_gm.graph.find_nodes(op="output"))) + for intermediate_idx, (node, policy) in enumerate( + zip(fw_intermediates, self.forward_intermediates_handling_policies) + ): + if policy == ScanForwardIntermediatesHandlingPolicy.CLONE: + new_output_node.append(self._insert_clone(node, fw_output_node)) + elif policy == ScanForwardIntermediatesHandlingPolicy.REMOVE_XS: + assert intermediate_idx in intermediate_idx_to_ph_idx + inp_idx = intermediate_idx_to_ph_idx[intermediate_idx] + self.saved_fw_xs.append(real_graph_inputs[inp_idx]) + elif ( + policy + == ScanForwardIntermediatesHandlingPolicy.REMOVE_ADDITIONAL_INPUTS + ): + assert intermediate_idx in intermediate_idx_to_ph_idx + inp_idx = intermediate_idx_to_ph_idx[intermediate_idx] + self.saved_fw_additional_inputs.append(real_graph_inputs[inp_idx]) + else: + new_output_node.append(node) + + fw_output_node.args = (tuple(fw_outputs) + tuple(new_output_node),) + fw_gm.graph.lint() + fw_gm.recompile() + + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "after removing aliasing:\n%s", fw_gm.print_readable(print_output=False) + ) + + def call_forward(self): + fw_outputs_and_intermediates: tuple[Any] = scan_op( + self.hop_partitioned_graph.fw_gm, self.init, self.xs, self.additional_inputs + ) # type: ignore[return-type] + fw_outs = fw_outputs_and_intermediates[ + : self.hop_partitioned_graph.n_fw_outputs + ] + saved_intermediates = fw_outputs_and_intermediates[ + self.hop_partitioned_graph.n_fw_outputs : + ] + assert len(self.saved_intermediates) == 0 + self.saved_intermediates.extend(saved_intermediates) + return tuple(fw_outs) + + def call_backward(self, *grad_fw_outputs): + """ + Recall that fw_outputs = (*carry, *ys), bw_gm takes in (*fw_intermediates, *grad_carry, *grad_ys) + and returns (*grad_init, *grad_xs, *grad_additional_inputs) + The bacwkard is a reversed scan that can be constructed as follows: + + grad_additonal_inputs = torch.zeros_like(additional_inputs) + bw_init = (grad_carry, grad_additional_inputs) + bw_xs = (fw_intermediates, grad_ys) + grad_init, grad_additional_inputs, grad_xs = scan( + combine_fn, + bw_init, + bw_xs, + reverse = True + ) + , where combine_fn is defined as follows: + + def combine_fn(bw_init, bw_xs): + grad_carry, grad_additional_inputs = bw_init + fw_intermediates, grad_y = bw_xs + nxt_grad_carry, grad_x, nxt_grad_additional_inputs = bw_gm(*fw_intermediates, *grad_carry, *grad_y) + return (nxt_grad_carry, grad_additional_inputs + nxt_grad_additional_inputs), grad_x + + Note that grad_additional_inputs is accumulated with add, grad_carry is carried over to next iteration and + grad_x is the ys output, which will be stacked together after the loop and will have the same shape as xs. + """ + fw_policy = self.forward_intermediates_handling_policies + saved_intermediates = self.saved_intermediates + saved_fw_xs = self.saved_fw_xs + saved_fw_additional_inputs = self.saved_fw_additional_inputs + + n_carry = len(self.init) + + grad_carry, grad_ys = grad_fw_outputs[:n_carry], grad_fw_outputs[n_carry:] + additional_inputs_tensor_masks = [ + bool(isinstance(t, torch.Tensor)) for t in self.additional_inputs + ] + grad_additional_inputs = [ + torch.zeros_like(t) + for t in filter_with_masks( + self.additional_inputs, additional_inputs_tensor_masks + ) + ] + + bw_init = [grad_carry, grad_additional_inputs] + bw_xs = [ + grad_ys, + saved_fw_xs, + saved_intermediates, + ] + bw_additional_inputs = saved_fw_additional_inputs + + _, flat_spec = pytree.tree_flatten((bw_init, bw_xs, bw_additional_inputs)) + + grad_spec = None + + def bw_single_step_wrapper(*args): + bw_init, bw_xs, bw_additional_inputs = pytree.tree_unflatten( + args, flat_spec + ) + grad_carry, grad_additional_inputs = bw_init + grad_y, saved_fw_xs, saved_intermediates = bw_xs + saved_fw_additional_inputs = bw_additional_inputs + + fw_intermediates = [] + xs_it = iter(saved_fw_xs) + carry_it = iter(saved_intermediates) + addi_it = iter(saved_fw_additional_inputs) + for policy in fw_policy: + if policy in ( + ScanForwardIntermediatesHandlingPolicy.CLONE, + ScanForwardIntermediatesHandlingPolicy.KEEP, + ): + fw_intermediates.append(next(carry_it)) + elif policy == ScanForwardIntermediatesHandlingPolicy.REMOVE_XS: + fw_intermediates.append(next(xs_it)) + elif ( + policy + == ScanForwardIntermediatesHandlingPolicy.REMOVE_ADDITIONAL_INPUTS + ): + fw_intermediates.append(next(addi_it)) + else: + raise RuntimeError(f"Unknown policy: {policy}") + + grad_fw_outputs = (*grad_carry, *grad_y) + + flat_out = self.hop_partitioned_graph.bw_gm( + *fw_intermediates, + *grad_fw_outputs, + ) + + next_grad_carry, grad_xs, grad_addi = split_into_chunks( + flat_out, # type: ignore[arg-type] + [len(self.init), len(self.xs), len(self.additional_inputs)], + ) + + nonlocal grad_spec + flat_grads, grad_spec = pytree.tree_flatten( + ( + next_grad_carry, + [ + prev + cur + for prev, cur in zip( + grad_additional_inputs, + filter_with_masks( + grad_addi, additional_inputs_tensor_masks + ), + ) + ], + grad_xs, + ) + ) + return flat_grads + + single_step_bw_xs = pytree.tree_map(lambda t: t[0], bw_xs) + bw_single_step_gm = materialize_as_graph( + bw_single_step_wrapper, + tuple( + pytree.tree_flatten((bw_init, single_step_bw_xs, bw_additional_inputs))[ + 0 + ] + ), + ) + + flat_grads = scan_op( + bw_single_step_gm, + pytree.tree_flatten(bw_init)[0], + # TODO: torch.flip copies the tensor, we should optimize it away + [torch.flip(x, (0,)) for x in pytree.tree_flatten(bw_xs)[0]], + pytree.tree_flatten(bw_additional_inputs)[0], + ) + assert grad_spec is not None + grad_init, grad_additional_inputs, grad_xs = pytree.tree_unflatten( + flat_grads, grad_spec + ) + return ( + *grad_init, + *[torch.flip(elem, (0,)) for elem in grad_xs], + *fill_none_with_masks( + grad_additional_inputs, additional_inputs_tensor_masks + ), + ) + + +@scan_op.py_autograd_impl +def scan_autograd(combine_fn, init, xs, additional_inputs): + with disable_proxy_modes_tracing(): + hop_partitioned_graph: HopPartitionedGraph = ( + HopGraphMinCutPartitioner.create_partitioned_graph( + combine_fn, + (*init, *[x[0] for x in xs], *additional_inputs), + always_recompute_complex_exprs=True, + ) + ) + + return ScanAutogradOp.apply( + hop_partitioned_graph, + len(init), + len(xs), + len(additional_inputs), + *init, + *xs, + *additional_inputs, + ) + + +@scan_op.py_impl(ProxyTorchDispatchMode) +def scan_proxy_mode(mode, combine_fn, init, xs, additional_inputs): + return trace_scan(mode, scan_op, combine_fn, init, xs, additional_inputs) + + +@scan_op.py_impl(FakeTensorMode) +def scan_fake_tensor_mode(mode, combine_fn, init, xs, additional_inputs): + with mode: + scan_length = xs[0].shape[0] + carry, outputs = _extract_carry_and_out( + combine_fn( + *init, + *[first_slice_copy(inp) for inp in xs], + *additional_inputs, + ), + len(init), + ) + out = ( + *carry, + *(stack_y(t, scan_length) for t in outputs), + ) + return out + + +@scan_op.py_functionalize_impl +def scan_functionalize(ctx, combine_fn, init, xs, additional_inputs): + from torch._higher_order_ops.utils import ( + _check_alias_and_mutation, + _maybe_run_with_interpreter, + ) + + unwrapped_xs = ctx.unwrap_tensors(xs) + unwrapped_init = ctx.unwrap_tensors(init) + unwrapped_additional_inputs = ctx.unwrap_tensors(additional_inputs) + + with ctx.redispatch_to_next(): + functional_combine_fn = ctx.functionalize( + _maybe_run_with_interpreter(combine_fn) + ) + sample_unwrapped_xs_sliced = [first_slice_copy(inp) for inp in unwrapped_xs] + sample_inputs = list( + itertools.chain( + unwrapped_init, + sample_unwrapped_xs_sliced, + unwrapped_additional_inputs, + ) + ) + pre_dispatch = hasattr(ctx, "mode") and ctx.mode.pre_dispatch + _check_alias_and_mutation(combine_fn, sample_inputs, "scan", pre_dispatch) + ret = scan_op( + functional_combine_fn, + unwrapped_init, + unwrapped_xs, + unwrapped_additional_inputs, + ) + return ctx.wrap_tensors(ret) + + +@scan_op.py_impl(torch._C._functorch.TransformType.Vmap) +def scan_batch_rule(interpreter, combine_fn, init, xs, additional_inputs): + from torch._functorch.vmap import restore_vmap, unwrap_batched, wrap_batched + + unbatched_args, in_dims = unwrap_batched( + (init, xs, additional_inputs), interpreter.level() + ) + # move to last dim to not interfere with scan's batching + unbatched_init, unbatched_xs, unbatched_additional_inputs = pytree.tree_map( + lambda x, bdim: x.movedim(bdim, -1) if bdim is not None else x, + unbatched_args, + in_dims, + ) + after_move_dims = tuple( + pytree.tree_flatten( + pytree.tree_map(lambda x: -1 if x is not None else None, in_dims) + )[0] + ) + + with interpreter.lower(): + out_dims = None + + def wrapper(*args): + nonlocal out_dims + outputs, per_slice_out_dims = restore_vmap( + combine_fn, + after_move_dims, + interpreter.batch_size(), + interpreter.randomness(), + )(*args) + # Note: outputs are not batched, we just move the batch dim to the end + # this is to avoid it interfering with scan's batching + outputs = tuple( + pytree.tree_map( + lambda out, out_bdim: out.movedim(out_bdim, -1) + if out_bdim is not None + else out, + outputs, + per_slice_out_dims, + ) + ) + out_dims = tuple( + pytree.tree_map( + lambda out_bdim: -1 if out_bdim is not None else None, + per_slice_out_dims, + ) + ) + return outputs + + unwrapped_out = scan_op( + wrapper, unbatched_init, unbatched_xs, unbatched_additional_inputs + ) + + assert out_dims is not None + batched_out = wrap_batched(unwrapped_out, out_dims, interpreter.level()) + return batched_out + + +# dense implementation for scan. Used for testing only. +def _fake_scan(combine_fn, init, xs=None, dim=0, reverse=False): + carry_leaves, carry_spec = pytree.tree_flatten(init) + inp_leaves, inp_spec = pytree.tree_flatten(xs) + if xs is None or len(inp_leaves) == 0: + return init, [] + result_flat = [] + carry = carry_leaves + op = reversed if reverse else lambda x: x + + dummy_carry, dummy_out = combine_fn( + pytree.tree_unflatten(carry, carry_spec), + pytree.tree_unflatten( + [first_slice_copy(elem, dim) for elem in inp_leaves], + inp_spec, + ), + ) + dummy_out_leaves, dummy_out_spec = pytree.tree_flatten(dummy_out) + num_leaves = len(dummy_out_leaves) + + for ind in op(range(inp_leaves[0].size(dim))): + xs = [elem.select(dim, ind) for elem in inp_leaves] + + carry, y = combine_fn( + pytree.tree_unflatten(carry, carry_spec), + pytree.tree_unflatten(xs, inp_spec), + ) + carry, _ = pytree.tree_flatten(carry) + y, _ = pytree.tree_flatten(y) + result_flat.append(y) + + results = [ + torch.stack([e[leave_ind] for e in op(result_flat)]) + for leave_ind in range(num_leaves) + ] + return ( + pytree.tree_unflatten(carry, carry_spec), + pytree.tree_unflatten(results, dummy_out_spec), + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/schema.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/schema.py new file mode 100644 index 0000000000000000000000000000000000000000..46dc11573a781db389fb8a6d385d9c24afc01b58 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/schema.py @@ -0,0 +1,308 @@ +import copy +from dataclasses import dataclass +from typing import Any, Optional + +import torch +import torch.utils._pytree as pytree +from torch.fx.node import Target + + +# Below is an implementation of generating FunctionSchema from example values. +# This is helpful for generating FunctionSchema for HigherOrderOperator, where +# we don't have a function to inspect and each call of the higher order operator +# would have different schema. +@dataclass(frozen=True) +class HopArgumentInfo: + # Could give a name to the operand by default it's empty string. + name: str + example_value: Any + # Provide an default_value + default_value: Any + # Whether this argument gets mutated in the hop subgraph. + # For output, this should always be False + is_mutated: bool + kw_only: bool + + +class HopArgumentInfoGen: + @staticmethod + def from_example( + example_value: Any, + *, + name: str = "", + default_value: Optional[Any] = None, + is_mutated: bool = False, + kw_only: bool = False, + ) -> HopArgumentInfo: + if default_value is not None: + assert type(example_value) is type(default_value), ( + f"example_value type {type(example_value)} doesn't match default_value type: {type(default_value)}" + ) + + return HopArgumentInfo( + name=name, + example_value=example_value, + default_value=default_value, + is_mutated=is_mutated, + kw_only=kw_only, + ) + + +class CTypeGen: + convert_to_base_ty = { + int: torch._C.IntType.get(), + float: torch._C.FloatType.get(), + str: torch._C.StringType.get(), + bool: torch._C.BoolType.get(), + } + + # should return torch._C.JitType but that annotation is busted + @staticmethod + def from_example(obj: Any) -> Any: + import torch + + if isinstance(obj, torch.fx.GraphModule): + return torch._C.AnyType.get() + elif isinstance(obj, torch.SymInt): + return torch._C.SymIntType.get() + elif isinstance(obj, torch.SymBool): + return torch._C.SymBoolType.get() + return torch._C._jit_try_infer_type(obj).type() + + +class CArgumentGen: + @staticmethod + def from_hop_argument_info( + arg_idx: int, arg_info: HopArgumentInfo, is_output: bool = False + ) -> Any: + typ = CTypeGen.from_example(arg_info.example_value) + if is_output: + return torch._C.Argument("", typ, None, None, False, None) + + alias_set = set({f"alias::a{arg_idx}"}) if arg_info.is_mutated else set() + alias_info = torch._C._AliasInfo(arg_info.is_mutated, alias_set, alias_set) # type: ignore[attr-defined] + return torch._C.Argument( + arg_info.name, + typ, + None, + arg_info.default_value, + arg_info.kw_only, + alias_info, + ) + + +class HopSchemaGenerator: + def __init__(self, hop: torch._ops.HigherOrderOperator): + self.arg_infos: list[HopArgumentInfo] = [] + self.example_outputs: list[Any] = [] + self.schema_tree_spec: Optional[pytree.TreeSpec] = None + self.hop = hop + + def add_arg( + self, + name: str, + example_value: Any, + default_value: Optional[Any] = None, + is_mutated: bool = False, + kw_only: bool = False, + ) -> None: + if callable(example_value): + assert isinstance( + example_value, (torch.fx.GraphModule, torch._ops.OperatorBase) + ), ( + "Expect callable to be a GraphModule or an. Please call materialize_as_graph first " + f"to turn callable arguments {example_value} into a GraphModule." + ) + _, flat_spec = pytree.tree_flatten(example_value) + if not flat_spec.is_leaf(): + raise RuntimeError( + f"example_value {example_value} is not a leaf node. " + "Please only add flattened inputs to the hop schema. " + "If you need some structure in the arguments, please" + "add_arg for flattened args one by one then " + "call add_schema_tree_spec to register the original pytree " + " spec of the args." + ) + + arg_info = HopArgumentInfoGen.from_example( + example_value=example_value, + name=name, + default_value=default_value, + is_mutated=is_mutated, + kw_only=kw_only, + ) + self.arg_infos.append(arg_info) + + def add_output(self, output: Any) -> None: + self.example_outputs.append(output) + + def add_schema_tree_spec(self, *args: Any, **kwargs: Any) -> None: + """schema tree spec is the tree spec from flattening all inputs to the hop with pytree.tree_flatten + Since torch.FunctionSchema only have proper mutation/alias support for flattened inputs, we need + to store the tree spec in order to reconstruct the inputs to the hop. + """ + self.schema_tree_spec = pytree.tree_flatten((args, kwargs))[1] + + def gen_schema(self) -> torch._C.FunctionSchema: + for i, arg_info in enumerate(self.arg_infos): + arg_spec = pytree.tree_flatten(arg_info.example_value)[1] + if not arg_spec.is_leaf() and self.schema_tree_spec is None: + raise RuntimeError( + f"example_value of arg_infos[{i}] is {arg_info.example_value}, which is not a leaf node. " + "Please call add_schema_tree_spec to add a schema tree spec first. " + "Or consider changing the hop's signature to only take flattened arguments." + ) + + return CFunctionSchemaGen.from_hop_argument_info( + str(self.hop), + self.arg_infos, + HopArgumentInfoGen.from_example(tuple(self.example_outputs), name="out"), + self.schema_tree_spec, + ) + + +class CFunctionSchemaGen: + """ + Note: [HigherOrderOperator schema generation] + Each invocation of a HigherOrderOperator will have a different schema. + For example, the schema of torch.cond varies depending on the true_fn and + false_fn. So we need a way to generate the schema for each invocation of a HOP. + + We want to enforce the following invariants for HOP's schema: + 1. Flattened inputs. There should be no pytree structure in it. + 2. Flattened outputs. Note even if the hop returns a single value, it should be wrapped as a tuple. + 3. No aliasing. This includes inp-inp aliasing, inp-out aliasing and out-out aliasing. + + By enforcing these invariants, we could make HOP's schema meets the requirement of schema parser + and makes hop easier to handle downstream. For example, suppose we have an invoke_quant_test HOP: + + class GraphModule(torch.nn.Module): + def forward(self, l_x_, l_y_): + subgraph_0 = self.subgraph_0 + invoke_quant_test = torch.ops.higher_order.invoke_quant_test(subgraph_0, l_x_, l_y_, scheme = 'nf4'); + + class subgraph_0(torch.nn.Module): + def forward(self, l_x_, l_y_): + add_ = l_x_.add_(1) + matmul = l_x_ @ l_y_ + sin = matmul.sin() + child = sin.cos() + child_1 = l_x_ + l_y_ + child_2 = l_x_ - l_y_ + child_3 = l_x_ @ l_y_ + return (child, child_1, child_2, child_3) + + By encoding the inputs of hop into a list of HopArgumentInfo and output as a single HopArgumentInfo, + we would get the following schema: + invoke_quant_test(Any arg0, Tensor(!) arg1, Tensor arg2, str scheme="\\"nf4\\"") -> (Tensor, Tensor, Tensor, Tensor) + """ + + @staticmethod + def from_hop_argument_info( + op_name: str, + inp_argument_info: list[HopArgumentInfo], + out_argument_info: HopArgumentInfo, + schema_tree_spec: Optional[pytree.TreeSpec], + ) -> Any: + args = [] + for i, arg_info in enumerate(inp_argument_info): + args.append(CArgumentGen.from_hop_argument_info(i, arg_info)) + + # NOTE: we want the output to always be a single argument with torch._C.TupleType. + assert isinstance(out_argument_info.example_value, tuple), ( + f"expect out_argument_info's example_value to be a tuple but got {out_argument_info.example_value}" + ) + assert not out_argument_info.is_mutated, ( + "out_argument_info.is_mutated should always be set to False." + ) + rets = None + if len(out_argument_info.example_value) == 1: + rets = [CArgumentGen.from_hop_argument_info(0, out_argument_info, True)] + else: + rets = [ + CArgumentGen.from_hop_argument_info( + i, + HopArgumentInfoGen.from_example( + name=f"out{i}", + example_value=val, + default_value=None, + is_mutated=False, + ), + is_output=True, + ) + for i, val in enumerate(out_argument_info.example_value) + ] + + return HopSchema( + op_name, + "", + args, + rets, + False, + False, + schema_tree_spec, + ) + + +class HopSchema(torch._C.FunctionSchema): + def __init__( + self, + name: str, + overload_name: str, + arguments: list[torch._C.Argument], + returns: list[torch._C.Argument], + is_vararg: bool, + is_varret: bool, + schema_tree_spec: Optional[pytree.TreeSpec], + ): + self.tree_spec = schema_tree_spec + self.is_vararg = is_vararg + self.is_varret = is_varret + super().__init__( + name, + overload_name, + arguments, + returns, + self.is_vararg, + self.is_varret, + ) + + def __deepcopy__(self, memo: Any) -> "HopSchema": + # Need to additionally copy the tree_spec since + # it's not a member of torch._C.FunctionSchema + return HopSchema( + self.name, + self.overload_name, + self.arguments, + self.returns, + self.is_vararg, + self.is_varret, + copy.deepcopy(self.tree_spec), + ) + + +def find_hop_schema( + gm: torch.fx.GraphModule, target: Target +) -> list[torch._C.FunctionSchema]: + schemas = [] + for node in gm.graph.find_nodes(op="call_function", target=target): + + def _get_example_value(node: torch.fx.Node) -> Any: + if node.op == "get_attr": + assert isinstance(node.target, str) + return getattr(gm, node.target) + else: + return ( + node.meta["example_value"] + if "example_value" in node.meta + else node.meta["val"] + ) + + fake_args, fake_kwargs = pytree.tree_map_only( + torch.fx.Node, + _get_example_value, + (node.args, node.kwargs), + ) + schema = node.target.gen_schema(*fake_args, **fake_kwargs) + schemas.append(schema) + return schemas diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/strict_mode.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/strict_mode.py new file mode 100644 index 0000000000000000000000000000000000000000..f5875ded5a994fb037d559a796e64a852c4d1cc6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/strict_mode.py @@ -0,0 +1,116 @@ +# mypy: allow-untyped-defs +from typing import Any, TYPE_CHECKING, Union + +import torch +import torch._subclasses.functional_tensor +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._functorch.utils import exposed_in +from torch._higher_order_ops.utils import _set_compilation_env, autograd_not_implemented +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ( + _temp_remove_metadata_torch_function_mode, + _temp_remove_pre_dispatch_torch_function_mode, + disable_proxy_modes_tracing, + make_fx, + ProxyTorchDispatchMode, + track_tensor_tree, +) +from torch.utils._python_dispatch import _get_current_dispatch_mode + + +if TYPE_CHECKING: + from collections.abc import Callable + + +@exposed_in("torch") +def strict_mode(callable, operands): + from torch._dynamo.backends.debugging import ( + make_eager_backend_with_torch_function_modes, + ) + + if torch.compiler.is_dynamo_compiling(): + return strict_mode_op(callable, operands) + + with _set_compilation_env(): + with _temp_remove_metadata_torch_function_mode() as metadata_mode: + with _temp_remove_pre_dispatch_torch_function_mode() as predispatch_mode: + modes = [metadata_mode, predispatch_mode] + modes = [mode for mode in modes if mode is not None] + if modes: + backend: Union[str, Callable[..., Any]] = ( + make_eager_backend_with_torch_function_modes(modes) + ) + else: + backend = "eager" + with torch._dynamo.utils.disable_cache_limit(): + return torch.compile( + strict_mode_op, backend=backend, fullgraph=True + )(callable, operands) + + +class StrictMode(HigherOrderOperator): + def __init__(self): + super().__init__("strict_mode") + + def __call__(self, callable, operands): + return super().__call__(callable, operands) + + +strict_mode_op = StrictMode() + + +@strict_mode_op.py_impl(DispatchKey.CompositeExplicitAutograd) +def strict_mode_op_dense(callable, operands): + mode = _get_current_dispatch_mode() + assert mode is None, "Mode should never be enabled for CPU/CUDA key" + return callable(*operands) + + +strict_mode_op.py_autograd_impl( + autograd_not_implemented(strict_mode_op, deferred_error=True) +) + + +@strict_mode_op.py_impl(ProxyTorchDispatchMode) +def inner(mode, callable, operands): + return trace_strict_mode(mode, strict_mode_op, callable, operands) + + +def trace_strict_mode(mode, strict_mode_op, callable, operands): + pre_dispatch = getattr(mode, "pre_dispatch", False) + + with disable_proxy_modes_tracing(): + graph = make_fx(callable, pre_dispatch=pre_dispatch)(*operands) + + graph_name = mode.tracer.get_fresh_qualname("strict_graph_") + mode.tracer.root.register_module(graph_name, graph) + + args = (graph, operands) + + proxy_args = pytree.tree_map(mode.tracer.unwrap_proxy, args) + + out_proxy = mode.tracer.create_proxy( + "call_function", strict_mode_op, proxy_args, {}, name="strict_mode" + ) + + out = graph(*operands) + return track_tensor_tree(out, out_proxy, constant=None, tracer=mode.tracer) + + +@strict_mode_op.py_impl(FakeTensorMode) +def strict_mode_fake_tensor_mode(mode, callable, operands): + with mode: + true_outs = callable(*operands) + return true_outs + + +@strict_mode_op.py_functionalize_impl +def strict_mode_func(ctx, callable, inputs): + unwrapped_inputs = ctx.unwrap_tensors(inputs) + with ctx.redispatch_to_next(): + functional_callable = ctx.functionalize(callable) + + cond_return = strict_mode_op(functional_callable, unwrapped_inputs) + return ctx.wrap_tensors(cond_return) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/torchbind.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/torchbind.py new file mode 100644 index 0000000000000000000000000000000000000000..c10e674b7ac0cc70953b7f99f6afcc192bd7d78b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/torchbind.py @@ -0,0 +1,164 @@ +# mypy: allow-untyped-defs +import logging +from contextlib import contextmanager + +import torch +from torch._C import DispatchKey # @manual +from torch._functorch._aot_autograd.utils import KNOWN_TYPES +from torch._higher_order_ops.utils import autograd_not_implemented +from torch._library.fake_class_registry import ( + _is_script_object, + _ns_and_class_name, + FakeScriptObject, +) +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ProxyTorchDispatchMode, track_tensor_tree +from torch.fx.node import has_side_effect +from torch.utils import _pytree as pytree + + +log = logging.getLogger(__name__) + + +# The call_torchbind operator represents a method invocation on a torchbind +# object. The calling convention is: +# call_torchbind(self: ScriptObject, method_name: str, *method_args, **method_kwargs) +# We do not expect users to write this operator directly. Instead it will be +# emitted by Dynamo when tracing encounters a torchbind object. +class CallTorchBind(HigherOrderOperator): + def __init__(self): + super().__init__("call_torchbind") + + def __call__(self, obj, method, *args, **kwargs): + return super().__call__(obj, method, *args, **kwargs) + + @staticmethod + def schema(obj, method) -> torch.FunctionSchema: + """ + Returns the schema of ``CallTorchbind.__call__``. + """ + assert isinstance(obj, torch._inductor.ir.TorchBindObject) + val = obj.get_real_obj() + schema = val._get_method(method).schema + schema_str = str(schema) + new_schema_str = f"call_torchbind({str(schema.arguments[0].real_type)} {schema.arguments[0].name}," + first_comma_index = schema_str.find(",") + if first_comma_index == -1: + # If no comma is found, find the last closing parenthesis + first_comma_index = schema_str.rfind(") ->") + new_schema_str = new_schema_str + " str method" + schema_str[first_comma_index:] + new_schema = torch._C.parse_schema(new_schema_str) + return new_schema + + +call_torchbind = CallTorchBind() + +# Register this operator as side-effectful with FX. +# TODO: this is not really sufficient. While passes (hopefully) check +# Node.is_impure() and make good decisions, we also assume we can execute the +# graph as many times as we want without changing behavior, which is NOT true of +# ops that mutate torchbind object state. +has_side_effect(call_torchbind) + +_orig_scriptmethod_call = torch.ScriptMethod.__call__ + + +def torchbind_method_redispatch(self, *args, **kwargs): + if _is_script_object(self.raw_owner): + return call_torchbind(self.raw_owner, self.name, *args, **kwargs) + return _orig_scriptmethod_call(self, *args, **kwargs) + + +@contextmanager +def enable_torchbind_tracing(): + """Context manager that acts as a feature flag to enable torchbind tracing + behavior. Once torchbind tracing has been stabilized, we can remove this and + turn it always on. + """ + try: + KNOWN_TYPES.append(torch.ScriptObject) + torch.ScriptMethod.__call__ = torchbind_method_redispatch # type: ignore[method-assign] + yield + finally: + assert KNOWN_TYPES.pop() is torch.ScriptObject, ( + "Someone else messed with KNOWN_TYPES during tracing, exploding." + ) + torch.ScriptMethod.__call__ = _orig_scriptmethod_call # type: ignore[method-assign] + + +@call_torchbind.py_impl(DispatchKey.CompositeExplicitAutograd) +def call_torchbind_impl(obj, method, *args, **kwargs): + if isinstance(obj, torch.ScriptObject): + return _orig_scriptmethod_call(getattr(obj, method), *args, **kwargs) + elif isinstance(obj, FakeScriptObject): + return getattr(obj.wrapped_obj, method)(*args, **kwargs) + else: + raise RuntimeError(f"Unsupported first arg type {type(obj)} for call_torchbind") + + +@call_torchbind.py_impl(ProxyTorchDispatchMode) +def inner(mode, *args, **kwargs): + proxy_args = pytree.tree_map(mode.tracer.unwrap_proxy, args) + proxy_kwargs = pytree.tree_map(mode.tracer.unwrap_proxy, kwargs) + + out_proxy = mode.tracer.create_proxy( + "call_function", + call_torchbind, + proxy_args, + proxy_kwargs, + ) + out = call_torchbind(*args, **kwargs) + + obj, method, *_rest_args = args + if isinstance(obj, torch.ScriptObject): + ns, class_name = _ns_and_class_name( + obj._type().qualified_name() # type: ignore[attr-defined] + ) + log.warning( + "Tracing torchbind method %s.%s with real ScriptObject. This may" + " cause the original object being mutated. If this is not intended," + ' You can register a fake class with torch._library.register_fake_class("%s::%s").', + class_name, + method, + ns, + class_name, + ) + + ret = track_tensor_tree(out, out_proxy, constant=None, tracer=mode.tracer) + if "val" not in out_proxy.node.meta: + assert out is None or isinstance(out, (int, float, bool)), ( + "Currently, only these constant dtypes are supported to be returned from torchbind methods." + ) + out_proxy.node.meta["val"] = out + return ret + + +# When tracing with fake script object, the call_torchbind op will return a fake tensor +# When tracing with real script object, the call_torchbind op may return a real tensor, +# we need to convert it to fake tensor manually. Dynamic shape is supported. +@call_torchbind.py_impl(FakeTensorMode) +def call_torchbind_fake(mode, *args, **kwargs): + with mode: + out = call_torchbind_impl(*args, **kwargs) + return pytree.tree_map_only( + torch.Tensor, + lambda x: mode.from_tensor(x, static_shapes=True) + if not isinstance(x, torch._subclasses.fake_tensor.FakeTensor) + else x, + out, + ) + + +call_torchbind.py_autograd_impl( + autograd_not_implemented(call_torchbind, deferred_error=True) +) + + +@call_torchbind.py_functionalize_impl +def call_torchbind_func(ctx, *args, **kwargs): + from torch._higher_order_ops.effects import handle_effects + + return handle_effects( + ctx.mode._allow_token_discovery, ctx.mode._tokens, call_torchbind, args, kwargs + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/triton_kernel_wrap.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/triton_kernel_wrap.py new file mode 100644 index 0000000000000000000000000000000000000000..628c889f6cbc7da3021f2f5f7ace70aa30a270fb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/triton_kernel_wrap.py @@ -0,0 +1,2117 @@ +import collections +import copy +import dataclasses +import functools +import inspect +import itertools +import logging +import operator +import threading +from collections import defaultdict +from collections.abc import Callable, Sequence +from typing import Any, Optional, TYPE_CHECKING, Union +from typing_extensions import Never + +import sympy + +import torch.fx as fx +import torch.utils._pytree as pytree +from torch import SymInt, Tensor +from torch._C import DispatchKey +from torch._higher_order_ops.utils import redirect_to_mode +from torch._ops import HigherOrderOperator +from torch._prims_common import clone_preserve_strides +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ( + disable_proxy_modes_tracing, + ProxyTorchDispatchMode, + track_tensor_tree, +) +from torch.fx.experimental.symbolic_shapes import guard_scalar +from torch.types import IntLikeType +from torch.utils.checkpoint import _CachedTorchDispatchMode, _CachingTorchDispatchMode + + +if TYPE_CHECKING: + from triton._C.libtriton.ir import ( + module as TritonIRModule, + operation as TritonIROperation, + ) + + from torch._dynamo.symbolic_convert import InstructionTranslator + from torch._dynamo.variables.constant import ConstantVariable + from torch._dynamo.variables.functions import TritonKernelVariable + from torch._subclasses.functional_tensor import BaseFunctionalizeAPI + from torch.fx.proxy import Proxy + from torch.utils._triton import has_triton + + TritonMetaParamsType = dict[str, int] + TritonGridTupleType = tuple[Union[int, sympy.Expr, SymInt], ...] + TritonGridCallableType = Callable[[TritonMetaParamsType], tuple[int, ...]] + TritonGridType = Union[TritonGridTupleType, TritonGridCallableType] + + if has_triton(): + from triton.runtime.autotuner import Autotuner, Config as TritonConfig + from triton.runtime.jit import JITFunction + else: + + class Autotuner: # type: ignore[no-redef] + pass + + class JITFunction: # type: ignore[no-redef] + pass + + TritonKernelType = Union[Autotuner, JITFunction] + # mypy specifically complains that TritonAutotunerType is not a valid type if Autotuner is not inside of a Union. + TritonAutotunerType = Union[Autotuner] + +log = logging.getLogger("torch._dynamo") + +# e.g. for a host-side Triton TMA API call ``create_2d_tma_descriptor(ptr, 50, 60, 32, 15, 4)``, +# the metadata will look like ``("experimental", ([50, 60], [32, 15], 4))`` +TMAExperimentalMetadata = tuple[ + str, # type of TMA (should be "experimental") + tuple[ + list[IntLikeType], # dims + list[IntLikeType], # block_dims + IntLikeType, # element_size + ], +] + +# e.g. for host-side Triton TMA API call ``TensorDescriptor.from_tensor(ptr, [32, 64])`` +# the metadata will look like ``("stable", ([32, 64],))`` +TMAStableMetadata = tuple[ + str, # type of TMA ("experimental" or "stable") + tuple[list[IntLikeType],], # block_shape +] + + +def create_tma_experimental_metadata( + dims: list[IntLikeType], + block_dims: list[IntLikeType], + element_size: IntLikeType, +) -> TMAExperimentalMetadata: + return ("experimental", (dims, block_dims, element_size)) + + +def maybe_unpack_tma_experimental_metadata( + tma_meta: Union[TMAExperimentalMetadata, TMAStableMetadata], +) -> Optional[tuple[list[IntLikeType], list[IntLikeType], IntLikeType]]: + if not tma_meta or len(tma_meta) != 2: + return None + if tma_meta[0] == "experimental": + return tma_meta[1] # type: ignore[return-value] + return None + + +def create_tma_stable_metadata( + block_shape: list[IntLikeType], +) -> TMAStableMetadata: + return ("stable", (block_shape,)) + + +def maybe_unpack_tma_stable_metadata( + tma_meta: Union[TMAExperimentalMetadata, TMAStableMetadata], +) -> Optional[tuple[list[IntLikeType]]]: + if not tma_meta or len(tma_meta) != 2: + return None + if tma_meta[0] == "stable": + return tma_meta[1] # type: ignore[return-value] + return None + + +# TMADescriptorMetadata maps kernel parameter names to the metadata that allows +# reconstructing TMA descriptors from the underlying tensors (passed as kernel +# arguments in the fx graph, instead of the TMA descriptors). +# +# Since there are two TMA APIs (the old "experimental" API and the new "stable" API), +# each entry in the dict is a tuple that starts with a string, either "experimental" +# or "stable". The second entry in the tuple is another tuple, with data that depends +# on the API type (see TMAExperimentalMetadata and TMAStableMetadata above). +# +# These are stored as raw tuples (instead of classes) for ease of serialization. +TMADescriptorMetadata = dict[ + str, # kernel parameter name + Union[TMAExperimentalMetadata, TMAStableMetadata], +] + + +############################################################################### +# Kernel Side Table + + +# We cannot put Triton Kernels into the FX graph as the graph nodes +# do not support arbitrary functions. +# Use a side table. +# We use two dicts so that fetching both the kernel and id are O(1) +class KernelSideTable: + id_to_kernel: dict[int, "TritonKernelType"] = {} + kernel_to_id: dict["TritonKernelType", int] = {} + constant_args: dict[int, dict[str, Any]] = {} + lock = threading.Lock() + + # Returns index on the table + def add_kernel(self, kernel: "TritonKernelType") -> int: + with self.lock: + if kernel in self.kernel_to_id: + return self.kernel_to_id[kernel] + + idx = len(self.id_to_kernel) + self.id_to_kernel[idx] = kernel + self.kernel_to_id[kernel] = idx + return idx + + # Returns the triton kernel at the given index + def get_kernel(self, idx: int) -> "TritonKernelType": + # No need to lock here as fetching from dict is atomic + assert idx in self.id_to_kernel + return self.id_to_kernel[idx] + + # Not every constant arg can be added to the graph. Use this side table + # for constant args. + def add_constant_args(self, args: dict[str, Any]) -> int: + with self.lock: + idx = len(self.constant_args) + self.constant_args[idx] = args + return idx + + # Returns the constant args + def get_constant_args(self, idx: int) -> dict[str, Any]: + # No need to lock here as fetching from dict is atomic + assert idx in self.constant_args + return self.constant_args[idx] + + # Resets the table (only meant to be used in unit tests) + # This is only safe assuming single threaded execution + def reset_table(self) -> None: + self.id_to_kernel = {} + self.kernel_to_id = {} + self.constant_args = {} + + +kernel_side_table = KernelSideTable() + + +############################################################################### +# Mutation Tracker + + +@dataclasses.dataclass(frozen=True) +class Param: + idx: int + + +@dataclasses.dataclass(frozen=True) +class Intermediate: + idx: int + + def fake(self) -> bool: + return self.idx < 0 + + +@dataclasses.dataclass(frozen=True) +class Op: + name: str + fn_call_name: Optional[str] + args: list[Union[Param, Intermediate]] + ret: Intermediate = dataclasses.field(repr=False) + # used for scf.yield: see [Note: scf.yield fix-up] + sub_idx: Optional[int] = None + # used for tt.elementwise_inline_asm + # `is_pure = True` assumes the asm block has no side-effects + is_pure: bool = False + + def __post_init__(self) -> None: + if self.name == "tt.call": + assert self.fn_call_name is not None + else: + assert self.fn_call_name is None + + +def generate_ttir( + kernel: "TritonKernelType", + kwargs: dict[str, Any], + tma_descriptor_metadata: TMADescriptorMetadata, +) -> tuple["TritonIRModule", list[str]]: + """ + Uses Triton's internal code generation to create TTIR + """ + import sympy + import triton + import triton.runtime.jit + from triton.compiler.compiler import ASTSource + from triton.runtime.autotuner import Autotuner + from triton.runtime.jit import JITFunction + + from torch._inductor.utils import ( + get_triton_attrs_descriptor_version, + triton_version_uses_attrs_dict, + TritonAttrsDescriptorVersion, + ) + from torch.utils._triton import has_triton_tensor_descriptor_host_tma + + triton_version = get_triton_attrs_descriptor_version() + + import torch._inductor.ir + from torch._subclasses.fake_tensor import FakeTensor + + if isinstance(kernel, Autotuner): + if len(kernel.configs) > 0: + # If we are autotuning, then it doesn't matter which version gets + # picked for tracing purposes, so lets pick the first one + kwargs = {**kwargs, **kernel.configs[0].kwargs} + kernel = kernel.fn + + assert isinstance(kernel, JITFunction) + + # pyrefly: ignore # missing-attribute + context = triton._C.libtriton.ir.context() + target = triton.runtime.driver.active.get_current_target() + backend = triton.compiler.compiler.make_backend(target) + options = backend.parse_options({}) + + # ignore backend-specific kwargs same way as in the native Triton code + # https://github.com/triton-lang/triton/blob/a6bb57d6285e723c58e87dd7cba263db6efff789/python/triton/runtime/jit.py#L594-L596 + # why this is important for user-defined Triton kernels on AMD: https://github.com/pytorch/pytorch/issues/140800 + for name in list(kwargs): + if name not in kernel.arg_names and name in options.__dict__: + kwargs.pop(name) + + if len(kwargs) != len(kernel.arg_names): + raise ValueError( + "Incorrect number of arguments passed to kernel: " + f"passed {list(kwargs.keys())}, expected {kernel.arg_names}." + ) + + # Replace all SymExprs with a regular value for TTIR generation + # Replace all FakeTensor/TensorBox with real tensors + # These replacements are needed for triton's type, key and config functions + ordered_args: dict[str, Any] = {} + for name in kernel.arg_names: + a = kwargs[name] + if isinstance(a, (torch.SymInt, torch.SymFloat, torch.SymBool, sympy.Expr)): + ordered_args[name] = 2 + elif ( + stable_meta := maybe_unpack_tma_stable_metadata( + # pyrefly: ignore [bad-argument-type] + tma_descriptor_metadata.get(name, None) + ) + ) is not None: + from triton.tools.tensor_descriptor import TensorDescriptor + + block_shape = stable_meta[0] + with torch._C._DisableTorchDispatch(): + # need 16-byte aligned strides + elements_per_dim = max(1, 16 // a.dtype.itemsize) + base_tensor = torch.empty( + [elements_per_dim] * len(block_shape), dtype=a.dtype + ) + # pyrefly: ignore # bad-argument-type + ordered_args[name] = TensorDescriptor.from_tensor(base_tensor, block_shape) + elif isinstance(a, (FakeTensor, torch._inductor.ir.TensorBox)): + with torch._C._DisableTorchDispatch(): + ordered_args[name] = torch.empty(2, dtype=a.dtype) + else: + ordered_args[name] = a + + def is_stable_tensor_descriptor_arg(arg: Any) -> bool: + if has_triton_tensor_descriptor_host_tma(): + from triton.tools.tensor_descriptor import TensorDescriptor + + if isinstance(arg, TensorDescriptor): + return True + return False + + def is_tensor_like_arg(arg: Any) -> bool: + if isinstance(arg, Tensor) or is_stable_tensor_descriptor_arg(arg): + return True + return False + + # Note: one would expect that each input to the triton kernel maps to + # one input parameter in the TTIR. This is _not_ true for TMA descriptors: + # one TMA descriptor gets converted into: + # * one TMA descriptor input + # * N strides, for a rank-N tensor + # * N sizes, for a rank-N tensor + # To account for this, we inject some fake arg names as placeholders for + # the stride and size parameters. + def get_tensor_names(name: str, arg: Any) -> list[str]: + if isinstance(arg, Tensor): + return [name] + if is_stable_tensor_descriptor_arg(arg): + stable_meta = maybe_unpack_tma_stable_metadata( + tma_descriptor_metadata[name] + ) + assert stable_meta is not None + block_shape = stable_meta[0] + tensor_rank = len(block_shape) + names = [name] + names.extend(name + f" STRIDE PLACEHOLDER {i}" for i in range(tensor_rank)) + names.extend(name + f" SIZE PLACEHOLDER {i}" for i in range(tensor_rank)) + return names + return [] + + ordered_tensor_names = list( + itertools.chain.from_iterable( + get_tensor_names(name, arg) for name, arg in ordered_args.items() + ) + ) + + def _get_specialization(args): # type: ignore[no-untyped-def] + # Support multiple triton versions. + # This code basically copies JITFunction.run() logic to get the attrs to construct an ASTSource. + if triton_version == TritonAttrsDescriptorVersion.V1_COMPILER: + return kernel._get_config(*args) + elif triton_version in { + TritonAttrsDescriptorVersion.V2_BACKENDS, + TritonAttrsDescriptorVersion.V3_BACKENDS_TUPLE, + }: + from triton.backends.compiler import AttrsDescriptor # noqa: F401 + + target = triton.runtime.driver.active.get_current_target() + backend_ = triton.compiler.compiler.make_backend(target) + # pyrefly: ignore # missing-attribute + return backend_.get_attrs_descriptor(args, kernel.params) + else: + assert ( + get_triton_attrs_descriptor_version() + == TritonAttrsDescriptorVersion.V4_DICT + ) + # specialize_impl switched to create_specialize_impl in https://github.com/triton-lang/triton/pull/6099 + if hasattr(triton.runtime.jit, "create_specialize_impl"): + try: + # Latest versions of Triton take specialize_extra as an arg to create_specialize_impl + specialize_impl = triton.runtime.jit.create_specialize_impl( + specialize_extra=backend.get_arg_specialization # pyrefly: ignore [missing-attribute] + ) + except TypeError: # Unknown arg `specialize_extra` + # Older versions of Triton take specialize_extra as an arg to specialize_impl + specialize_impl = functools.partial( + # pyrefly: ignore # missing-argument + triton.runtime.jit.create_specialize_impl(), + specialize_extra=backend.get_arg_specialization, # pyrefly: ignore [missing-attribute] + ) + # create_specialize_impl is removed in https://github.com/triton-lang/triton/pull/7771 + # switch to native_specialize_impl instead + elif hasattr(triton.runtime.jit, "native_specialize_impl"): + from triton.backends import BaseBackend + from triton.runtime.jit import native_specialize_impl + + def _native_specialize_impl( + arg: Any, + is_const: bool = False, + specialize_value: bool = True, + align: bool = True, + ) -> Callable: + return native_specialize_impl( + BaseBackend, arg, is_const, specialize_value, align + ) + + specialize_impl = _native_specialize_impl + else: + from triton.runtime.jit import specialize_impl as specialize_impl_orig + + specialize_impl = functools.partial( + specialize_impl_orig, + specialize_extra=backend.get_arg_specialization, # pyrefly: ignore [missing-attribute] + ) + + from triton._utils import find_paths_if, get_iterable_path + + # logic is copied from: binder = create_function_from_signature(self.signature, self.params, backend) + attrvals = [] + for arg, kp in zip(args, kernel.params): + if kp.is_constexpr: + attrvals.append(arg) + else: + spec = specialize_impl( + arg, + is_const=kp.is_const, + specialize_value=not kp.do_not_specialize, + align=not kp.do_not_specialize_on_alignment, + ) + # pyrefly: ignore [unsupported-operation] + attrvals.append(spec[1]) + + attrs = find_paths_if(attrvals, lambda _, x: isinstance(x, str)) + attrs = { + k: backend.parse_attr(get_iterable_path(attrvals, k)) for k in attrs + } + return attrs + + specialization = _get_specialization(ordered_args.values()) + constants = { + name: arg for name, arg in ordered_args.items() if not is_tensor_like_arg(arg) + } + + if (mangle_type := getattr(triton.runtime.jit, "mangle_type", None)) is not None: + + def get_signature_value(idx: int, arg: Any) -> str: + if kernel.params[idx].is_constexpr: + return "constexpr" + # pyrefly: ignore [not-callable] + return mangle_type(arg) + + else: + + def get_signature_value(idx: int, arg: Any) -> str: + return kernel._type_of(kernel.key_of(arg)) + + if triton_version_uses_attrs_dict(): + # In newer versions of Triton, the signature includes constexpr args + signature = { + name: get_signature_value(i, arg) + for i, (name, arg) in enumerate(ordered_args.items()) + } + else: + # In older versions of Triton, the signature does not include constexpr args + constexprs = [p.num for p in kernel.params if p.is_constexpr] + signature = { + name: get_signature_value(i, arg) + for i, (name, arg) in enumerate(ordered_args.items()) + if i not in constexprs + } + + # pyrefly: ignore # missing-attribute + triton._C.libtriton.ir.load_dialects(context) + backend.load_dialects(context) + + src = ASTSource(kernel, signature, constants, specialization) + + # Triton changes ASTSource.make_ir to take 3/4 arguments. Handle + # backward compatibility here. + make_ir_sig_params = len(inspect.signature(src.make_ir).parameters) + get_codegen_implementation_sig_params = len( + # pyrefly: ignore # missing-attribute + inspect.signature(backend.get_codegen_implementation).parameters + ) + if make_ir_sig_params == 2: + # pyrefly: ignore # missing-argument + ttir_module = src.make_ir(options, context) + elif make_ir_sig_params == 3: + # pyrefly: ignore # missing-attribute + codegen_fns = backend.get_codegen_implementation() + # pyrefly: ignore # missing-argument + ttir_module = src.make_ir(options, codegen_fns, context) + elif make_ir_sig_params == 4: + codegen_args = [options] if get_codegen_implementation_sig_params == 1 else [] + # pyrefly: ignore # missing-attribute + codegen_fns = backend.get_codegen_implementation(*codegen_args) + module_map = backend.get_module_map() + # pyrefly: ignore[missing-argument,bad-argument-type] + ttir_module = src.make_ir(options, codegen_fns, module_map, context) + else: + codegen_args = [options] if get_codegen_implementation_sig_params == 1 else [] + # pyrefly: ignore # missing-attribute + codegen_fns = backend.get_codegen_implementation(*codegen_args) + module_map = backend.get_module_map() + # pyrefly: ignore # bad-argument-count + ttir_module = src.make_ir(target, options, codegen_fns, module_map, context) + if not ttir_module.verify(): + raise RuntimeError("Verification for TTIR module has failed") + + return ttir_module, ordered_tensor_names + + +def ttir_to_functions( + ttir_module: "TritonIRModule", +) -> dict[str, dict[Intermediate, list[Op]]]: + """ + Walk the `ttir_module` bottom up to mine the `functions` from + the structured MLIR entities representing the Triton kernel + (mlir::Operation, mlir::Block, mlir::Region). + """ + functions: dict[str, dict[Intermediate, list[Op]]] = {} + + # block id --> op result (Intermediate) --> one or more ops + op_stack: dict[int, dict[Intermediate, list[Op]]] = defaultdict( + lambda: defaultdict(list) + ) + region_id_to_block_ids: dict[int, list[int]] = defaultdict(list) + block_id_to_block_arg_ids: dict[int, list[int]] = {} + replacements: dict[int, Union[Intermediate, Param]] = {} + reindex_map: dict[int, int] = {} + next_fake_intermediate = 0 + + def reindex(idx: int) -> int: + if idx not in reindex_map: + reindex_map[idx] = len(reindex_map) + return reindex_map[idx] + + def mlir_to_functions(op: "TritonIROperation") -> None: + name: str = op.get_name() + if name == "builtin.module": + # this wraps all tt.func ops + return + + operand_ids: list[int] = [ + reindex(op.get_operand(i).id()) for i in range(op.get_num_operands()) + ] + result_ids: list[int] = [ + reindex(op.get_result(i).id()) for i in range(op.get_num_results()) + ] + + child_block_ids: list[int] = [] + for i in [op.get_region(i).id() for i in range(op.get_num_regions())]: + # as the walk is bottom-up, the region_id_to_block_ids[i] + # must be populated by the time we process the enclosing op + child_block_ids.extend(region_id_to_block_ids[i]) + + parent_block_id = -1 + parent_block = op.get_block() + if parent_block is not None: + parent_block_id = parent_block.id() + if parent_block_id not in block_id_to_block_arg_ids: + block_id_to_block_arg_ids[parent_block_id] = [] + for i in range(parent_block.get_num_arguments()): + block_id_to_block_arg_ids[parent_block_id].append( + reindex(parent_block.get_argument(i).id()), + ) + # the region info is collected via ops' parent blocks to be + # used later when the region's encloding op is traversed + parent_region = parent_block.get_parent() + if parent_region is not None: + region_id_to_block_ids[parent_region.id()].append(parent_block_id) + + nonlocal next_fake_intermediate + + if name == "tt.func": + # for function ops: gather and inline + # the ops from all child blocks + fn_ops = defaultdict(list) + for child_block_id in child_block_ids: + for result, block_fn_ops in op_stack.pop(child_block_id).items(): + for block_fn_op in block_fn_ops: + fn_ops[result].append(block_fn_op) + + # replace the corresponding Intermediates in the + # child op args with the function args (Params) + for i, idx in enumerate(block_id_to_block_arg_ids[child_block_ids[0]]): + replacements[idx] = Param(i) + + for fn_op_list in fn_ops.values(): + for fn_op in fn_op_list: + for i in range(len(fn_op.args)): + arg = fn_op.args[i] + seen = set() # to break cycles + # there can be transitive replacements, but likely + # no cycles (we keep the `seen` set just in case) + while ( + isinstance(arg, Intermediate) + and arg.idx in replacements + and arg.idx not in seen + ): + seen.add(arg.idx) + arg = fn_op.args[i] = replacements[arg.idx] + + # next function capture starts + # with empty replacements + replacements.clear() + + fn_name = op.get_str_attr("sym_name") + functions[fn_name] = fn_ops + elif child_block_ids: + if name in {"scf.if", "scf.for", "scf.while", "tt.reduce", "tt.scan"}: + # for blocked ops: inline the enclosed ops into + # the parent block + rewire the last op in each + # child block to return the block result + return_ops = [] + for block_id in child_block_ids: + if name == "scf.for": + # example: + # %result = scf.for %iv = %lb to %ub step %step iter_args(%arg = %init) -> (i32) ... + # block args: 2 (%iv, %arg) + # op operands: 4 (%lb, %ub, %step, %init) + # `%arg` is mapping to `%init` + for i, idx in enumerate(block_id_to_block_arg_ids[block_id]): + if i == 0: + next_fake_intermediate -= 1 + replacements[idx] = Intermediate(next_fake_intermediate) + else: + replacements[idx] = Intermediate(operand_ids[i + 2]) + elif name == "scf.while": + # example: + # %3:3 = scf.while (%arg2 = %1, %arg3 = %2, %arg4 = %c0_i32_8) ... + # block args: 3 (%arg2, %arg3, %arg4) + # op operands: 3 (%1, %2, %c0_i32_8) + # `%arg2` is mapping to `%1`, `%arg3` is mapping to `%2`, ... + for i, idx in enumerate(block_id_to_block_arg_ids[block_id]): + replacements[idx] = Intermediate(operand_ids[i]) + elif name == "scf.if": + # the scf block args are ignored by the pass. but, as they + # may be used as operands of the ops inside the block + # (and nested blocks inlined in the current block by now), + # they are replaced by new fake Intermediates to avoid "this + # operand is not returned by any other op in the fn" error + # in the downstream analysis + for idx in block_id_to_block_arg_ids[block_id]: + next_fake_intermediate -= 1 + replacements[idx] = Intermediate(next_fake_intermediate) + else: + assert name in ("tt.reduce", "tt.scan") + # wire the block arguments to the op arguments + num_operands = len(operand_ids) + block_arg_ids = block_id_to_block_arg_ids[block_id] + assert len(block_arg_ids) == 2 * num_operands, ( + f"{name} is expected to have twice as " + "many block arguments as op arguments: " + f"{operand_ids=}, {block_arg_ids=}." + ) + for i, idx in enumerate(block_arg_ids): + # for a tt.reduce/tt.scan op with N arguments, the block + # arguments comprise N reduced values followed by + # N current values corresponding to the N op args + replacements[idx] = Intermediate( + operand_ids[i % num_operands] + ) + + if block_id in op_stack: + block_ops = op_stack.pop(block_id) + if not block_ops: + continue + last_ret, last_ops = block_ops.popitem() + if all( + op.name + in ("scf.yield", "tt.reduce.return", "tt.scan.return") + for op in last_ops + ): + # if last_ops are all return ops, treat them separately + return_ops.extend(last_ops) + else: + # otherwise, return last_ops to the block + block_ops[last_ret] = last_ops + for op_result, child_ops in block_ops.items(): + op_stack[parent_block_id][op_result].extend(child_ops) + + scf_results = [Intermediate(idx) for idx in result_ids] + + if return_ops and all( + (op.name == "scf.yield" and len(result_ids) == len(op.args)) + for op in return_ops + ): + # [Note: scf.yield fix-up] + # + # TL;DR: if our scf.yield takes N args, then we'll create N scf.yield ops to handle each of the + # args. + # + # **Context**: + # During mutation analysis, the analysis pass will identify mutating ops (e.g. tt.store) + # and then DFS upwards towards the parameters of the function. Specifically, the analysis pass + # looks at the mutated arg in tt.store; then looks for its source ops; and then recurses on the + # arguments to each of the source ops. + # + # In the case of scf.if/scf.for, we may have multiple return ops, each passed as an arg + # to scf.yield: + # + # %18:2 = scf.if %... -> (!tt.ptr, !tt.ptr) { + # ... + # scf.yield %1, %2 + # } else { + # scf.yield %3, %4 + # } + # + # And for each of the returns of the scf.if, we'd naively assign the source op of each of the + # return values to be the scf.yields. But the scf.yields take _all_ the returns as arguments. + # Therefore, if _any_ of the return values of the scf.if are mutated, then the analysis pass + # would mark _all_ of the yield args as mutated. + # + # **Solution**: + # For the purposes of this analysis pass, we create N yield ops - one for each + # return-val/yield-arg. In the example above, we'll have two scf.yield's for each branch of the + # scf.if. + + for return_op in return_ops: + for i, (scf_result, yield_arg) in enumerate( + zip(scf_results, return_op.args) + ): + sub_yield_op = Op( + return_op.name, + return_op.fn_call_name, + [yield_arg], + return_op.ret, + sub_idx=i, + ) + op_stack[parent_block_id][scf_result].append(sub_yield_op) + + else: + for scf_result in scf_results: + for return_op in return_ops: + op_stack[parent_block_id][scf_result].append(return_op) + else: + raise RuntimeError( + f"Unknown blocked function: {name}. Can't capture the TTIR." + ) + else: + callee = None + if name == "tt.call": + callee = op.get_flat_symbol_ref_attr("callee") + args: list[Union[Param, Intermediate]] = [ + Intermediate(operand) for operand in operand_ids + ] + block_ops = op_stack[parent_block_id] + + is_pure = False + # Handle the case for tt.elementwise_inline_asm to set `is_pure` for mutation analysis + if name == "tt.elementwise_inline_asm": + is_pure = op.get_bool_attr("pure") + + if result_ids: + for result_id in result_ids: + res = Intermediate(result_id) + block_ops[res].append(Op(name, callee, args, res, is_pure=is_pure)) + else: + next_fake_intermediate -= 1 + fake_res = Intermediate(next_fake_intermediate) + block_ops[fake_res].append( + Op(name, callee, args, fake_res, is_pure=is_pure) + ) + + ttir_module.walk(mlir_to_functions) + + return functions + + +class MemoizeWithCycleCheck: + fn: Callable[..., Any] + cache: dict[tuple[Any], Any] + + def __init__(self, fn: Callable[..., Any]) -> None: + self.fn = fn + self.reset() + + def __call__( + self, + functions: dict[str, dict[Intermediate, list[Op]]], + fn_name: str, + *args: Any, + ) -> list[bool]: + key: tuple[Any, ...] = (fn_name, *args) + if key not in self.cache: + self.cache[key] = None + self.cache[key] = self.fn(functions, fn_name, *args) + if self.cache[key] is None: + raise RuntimeError("Recursion is not supported") + return self.cache[key] + + def reset(self) -> None: + self.cache = {} + + +@MemoizeWithCycleCheck +def get_tma_stores( + functions: dict[str, dict[Intermediate, list[Op]]], fn_name: str +) -> set[Union[Intermediate, Param]]: + """ + Identifies all intermediates and parameters that are written to by a + `tt.experimental_descriptor_store`. It tracks only the specific values + written to via experimental_descriptor_store and the input values to + `tt.reinterpret_tensor_descriptor` used to construct the direct inputs + to tt.experimental_descriptor_store - not any recursive values + used to construct those values. + + For example: for + tt.reinterpret_tensor_descriptor(Intermediate(idx=0), ...) + Intermediate(idx=1) = tt.experimental_descriptor_store(Intermediate(idx=0), ...) + this function will return [Intermediate(idx=0), Intermediate(idx=1)], + + However + Intermediate(idx=4) = arith.addptr(Intermediate(idx=2), Intermediate(idx=3)) + Intermediate(idx=5) = tt.experimental_descriptor_store(Intermediate(idx=4), ...) + tt.experimental_descriptor_store(Intermediate(idx=5), ...) + this function will mark only idx=4 and idx=5 (but not idx=2 or idx=3) + + If an intermediate/parameter is passed into a function and is written to + via experimental_descriptor_store within that function, the argument to the + function will also be marked. + """ + + result: set[Union[Intermediate, Param]] = set() + + ops = functions[fn_name] + for op_list in ops.values(): + for op in op_list: + if op.name == "tt.call": + assert op.fn_call_name in functions + # pyrefly: ignore [bad-argument-type] + tma_stores = get_tma_stores(functions, op.fn_call_name) + for i, inp in enumerate(op.args): + if Param(idx=i) in tma_stores: + result.add(inp) + elif op.name == "tt.experimental_descriptor_store": + assert len(op.args) >= 1 + result.add(op.args[0]) + elif op.name == "tt.descriptor_store": + assert len(op.args) >= 1 + result.add(op.args[0]) + + for val in list(result): + if val in ops: + if not isinstance(val, Intermediate): + continue + for op in ops[val]: + if op.name == "tt.reinterpret_tensor_descriptor": + assert len(op.args) >= 1 + result.add(op.args[0]) + + return result + + +@MemoizeWithCycleCheck +def analyze_kernel_mutations( + functions: dict[str, dict[Intermediate, list[Op]]], fn_name: str, num_args: int +) -> list[bool]: + """ + Analyzes the graph to detect all sinks from a predefined list of sinks + by using triton's MemWrite trait list. NOTE: What if triton exposed this? + From each sink, it traverses the CFG backwards to identify all the input + pointers that are mutated. + """ + # Name of mutation op to mutated parameter indices + # List from Triton Github include/triton/Dialect/Triton/IR/TritonOps.td + # All the OPs that have MemWrite trait. + # What if Triton exposed this? + MUTATION_OPS = { + "tt.store": [0], + "tt.atomic_cas": [0], + "tt.atomic_rmw": [0], + "tt.experimental_descriptor_store": [0], + "tt.experimental_tensormap_create": [0], + "tt.descriptor_store": [0], + } + # Ops that we want to bail out on + UNKNOWN_OPS = {"tt.elementwise_inline_asm"} + + stack: list[Union[Param, Intermediate]] = [] + visited = set() + ops = functions[fn_name] + tma_stores = get_tma_stores(functions, fn_name) + + for op_list in ops.values(): + for op in op_list: + # If we encounter an operation with effects that cannot be reliably analyzed + # (e.g. `tt.elementwise_inline_asm`), we assume it does not mutate any input parameters. + if op.name in UNKNOWN_OPS: + if op.name == "tt.elementwise_inline_asm" and op.is_pure: + continue + raise RuntimeError( + f"ttir analysis hit an op we do not know how to analyze: {op.name}" + ) + + if op.name == "tt.experimental_tensormap_create": + # Note: this is how we implement experimental_descriptor_store mutation analysis. + # for on-device TMA. + # experimental_tensormap_store(a, b, ...) stores b to the location specified + # by descriptor in the memory of a. + # To track this, we first find all the intermediates/params to which we store via + # experimental_tensormap_store (get_tma_stores, called above). Then, during this + # analysis we wait to find the corresponding experimental_tensormap_create (if it + # exists), at which point we will mark the global_ptr as mutated (as done below). + assert len(op.args) >= 2 + if op.args[0] in tma_stores: + stack.append(op.args[1]) + + if op.name == "tt.call": + assert op.fn_call_name in functions + mutations = analyze_kernel_mutations( + functions, + # pyrefly: ignore [bad-argument-type] + op.fn_call_name, + len(op.args), + ) + stack.extend(arg for arg, mutated in zip(op.args, mutations) if mutated) + else: + stack.extend(op.args[idx] for idx in MUTATION_OPS.get(op.name, [])) + + # The following is an iterative DFS algorithm + mutated = [False] * num_args + while stack: + arg = stack.pop() + if arg in visited: + continue + + visited.add(arg) + + if isinstance(arg, Param): + if arg.idx >= num_args: + # This is an argument defined in the kernel, not passed in + continue + mutated[arg.idx] = True + elif isinstance(arg, Intermediate) and not arg.fake(): + for op in ops[arg]: + # Skip arguments to load + if op.name != "tt.load": + stack.extend(op.args) + return mutated + + +def identify_mutated_tensors( + kernel: "TritonKernelType", + kwargs: dict[str, Any], + tma_descriptor_metadata: TMADescriptorMetadata, +) -> list[str]: + """ + Given a triton kernel and the arguments for this kernel, this function + 1) Retrieves the TTIR converted version of the kernel from Triton's API. + 2) Parses the TTIR and creates a control flow graph + 3) Analyzes the graph to detect all input tensor mutations + """ + + ttir_module = None + functions = None + try: + ttir_module, ordered_tensor_names = generate_ttir( + kernel, kwargs, tma_descriptor_metadata + ) + + # extract functions from TTIR using MLIR bindings exposed by Triton code + functions = ttir_to_functions(ttir_module) + + assert functions is not None + kernel_name = next(iter(functions.keys())) + # Triton codegen modifies the name + # pyrefly: ignore [missing-attribute] + assert kernel.fn.__name__ in kernel_name + # Reset the cache between top level invocations + # The cache for analyze kernel mutations is mainly used for cycle + # detection, so each top level invocation needs a clean cache + analyze_kernel_mutations.reset() + get_tma_stores.reset() + mutations = analyze_kernel_mutations( + functions, kernel_name, len(ordered_tensor_names) + ) + + return [ + ordered_tensor_names[i] for i, mutated in enumerate(mutations) if mutated + ] + except Exception: + log.warning( + "Encountered an exception in identify_mutated_tensors, assuming every input is mutated", + exc_info=True, + ) + if ttir_module is not None: + log.debug("TTIR:\n%s", str(ttir_module)) + if functions is not None: + log.debug("functions:") + for name, fn in functions.items(): + log.debug("===\t%s\t===", name) + for ret, ops in fn.items(): + log.debug("%s\t=>\t%s", ret, ops) + return [key for key, value in kwargs.items() if isinstance(value, Tensor)] + + +############################################################################### +# Triton Kernel Wrappers + + +# Used for wrapping a Triton Kernel +class TritonKernelWrapperMutation(HigherOrderOperator): + def __init__(self) -> None: + super().__init__("triton_kernel_wrapper_mutation", cacheable=True) + + def __call__( + self, + kernel_idx: int, + constant_args_idx: int, + grid: list["TritonGridType"], + tma_descriptor_metadata: TMADescriptorMetadata, + kwargs: dict[str, Any], + ) -> Any: + return super().__call__( + kernel_idx=kernel_idx, + constant_args_idx=constant_args_idx, + grid=grid, + tma_descriptor_metadata=tma_descriptor_metadata, + kwargs=kwargs, + ) + + +triton_kernel_wrapper_mutation = TritonKernelWrapperMutation() + + +# Used for wrapping a Triton Kernel in a functional manner +class TritonKernelWrapperFunctional(HigherOrderOperator): + def __init__(self) -> None: + super().__init__("triton_kernel_wrapper_functional", cacheable=True) + + def __call__( + self, + kernel_idx: int, + constant_args_idx: int, + grid: list["TritonGridType"], + tma_descriptor_metadata: TMADescriptorMetadata, + kwargs: dict[str, Any], + tensors_to_clone: list[str], + ) -> dict[str, Any]: + return super().__call__( + kernel_idx=kernel_idx, + constant_args_idx=constant_args_idx, + grid=grid, + tma_descriptor_metadata=tma_descriptor_metadata, + kwargs=kwargs, + tensors_to_clone=tensors_to_clone, + ) + + +triton_kernel_wrapper_functional = TritonKernelWrapperFunctional() + + +@triton_kernel_wrapper_mutation.py_impl(DispatchKey.CompositeExplicitAutograd) +def triton_kernel_wrapper_mutation_dense( + *, + kernel_idx: int, + constant_args_idx: int, + grid: list["TritonGridType"], + tma_descriptor_metadata: TMADescriptorMetadata, + kwargs: dict[str, Any], +) -> None: + from torch._inductor.codegen.wrapper import user_defined_kernel_grid_fn_code + + kernel = kernel_side_table.get_kernel(kernel_idx) + constant_args = kernel_side_table.get_constant_args(constant_args_idx) + + if len(grid) == 1: + grid_fn = grid[0] + else: + fn_name, code = user_defined_kernel_grid_fn_code( + # pyrefly: ignore [missing-attribute] + kernel.fn.__name__, + # pyrefly: ignore [missing-attribute] + kernel.configs, + grid, + ) + namespace: dict[str, Any] = {} + exec(code, namespace) + grid_fn = namespace[fn_name] + + if tma_descriptor_metadata: + # as we need to launch the kernel here, we "unwrap" the + # tma_descriptor_metadata, create the TMA descriptors + # from it, and replace the tensors in the kwargs by the + # corresponding TMA descriptors before launching + kwargs = kwargs.copy() + for k, v in tma_descriptor_metadata.items(): + tensor = kwargs[k] + if (exp_meta := maybe_unpack_tma_experimental_metadata(v)) is not None: + from triton.tools.experimental_descriptor import ( # noqa: F401 + create_1d_tma_descriptor, + create_2d_tma_descriptor, + ) + + dims, block_dims, element_size = exp_meta + create_tma_descriptor = ( + create_1d_tma_descriptor + if len(dims) == 1 + else create_2d_tma_descriptor + ) + kwargs[k] = create_tma_descriptor( + tensor.data_ptr(), + *dims, + *block_dims, + element_size, + ) + else: + stable_meta = maybe_unpack_tma_stable_metadata(v) + assert stable_meta is not None + from triton.tools.tensor_descriptor import TensorDescriptor + + block_shape = stable_meta[0] + # pyrefly: ignore # bad-argument-type + kwargs[k] = TensorDescriptor.from_tensor(tensor, block_shape) + + # move as many positional arguments from dicts to args as we + # can to circumvent the bug with the kwargs and pre_/post_hook: + # https://github.com/triton-lang/triton/issues/5082 + # TODO: remove this when the Triton issue above is fixed + args = [] + # copy kwargs and constant_args here to + # avoid mutating the original inputs + kwargs = kwargs.copy() + constant_args = constant_args.copy() + # pyrefly: ignore [missing-attribute] + for name in kernel.arg_names: + if name in kwargs: + args.append(kwargs.pop(name)) + elif name in constant_args: + args.append(constant_args.pop(name)) + else: + break + + # pyrefly: ignore [index-error] + kernel[grid_fn](*args, **kwargs, **constant_args) + + +@triton_kernel_wrapper_mutation.py_impl(FakeTensorMode) +def triton_kernel_wrapper_mutation_fake_tensor_mode( + mode: FakeTensorMode, + *, + kernel_idx: int, + constant_args_idx: int, + grid: list["TritonGridType"], + tma_descriptor_metadata: TMADescriptorMetadata, + kwargs: dict[str, Any], +) -> None: + with mode: + return None + + +@triton_kernel_wrapper_mutation.py_impl(DispatchKey.Meta) +def _( + *, + kernel_idx: int, + constant_args_idx: int, + grid: list["TritonGridType"], + tma_descriptor_metadata: TMADescriptorMetadata, + kwargs: dict[str, Any], +) -> None: + return None + + +def trace_triton_kernel_wrapper( + proxy_mode: ProxyTorchDispatchMode, + func_overload: Callable[..., Any], + node_args: dict[str, Any], +) -> Optional[dict[str, Any]]: + with disable_proxy_modes_tracing(): + out = func_overload(**node_args) + + proxy_args = pytree.tree_map( + proxy_mode.tracer.unwrap_proxy, # type: ignore[union-attr] + node_args, + ) + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", + func_overload, + (), + proxy_args, + name=func_overload.__name__ + "_proxy", + ) + + ret = track_tensor_tree(out, out_proxy, constant=None, tracer=proxy_mode.tracer) + return ret + + +@triton_kernel_wrapper_mutation.py_impl(ProxyTorchDispatchMode) +def triton_kernel_wrapper_mutation_proxy_torch_dispatch_mode( + mode: ProxyTorchDispatchMode, + *, + kernel_idx: int, + constant_args_idx: int, + grid: list["TritonGridType"], + tma_descriptor_metadata: TMADescriptorMetadata, + kwargs: dict[str, Any], +) -> None: + trace_triton_kernel_wrapper( + mode, + triton_kernel_wrapper_mutation, + { + "kernel_idx": kernel_idx, + "constant_args_idx": constant_args_idx, + "grid": grid, + "tma_descriptor_metadata": tma_descriptor_metadata, + "kwargs": kwargs, + }, + ) + + return None + + +def get_mutated_tensors( + kernel_idx: int, + constant_args_idx: int, + kwargs: dict[str, Any], + tma_descriptor_metadata: TMADescriptorMetadata, +) -> list[str]: + kernel = kernel_side_table.get_kernel(kernel_idx) + constant_args = kernel_side_table.get_constant_args(constant_args_idx) + return identify_mutated_tensors( + kernel, {**kwargs, **constant_args}, tma_descriptor_metadata + ) + + +@triton_kernel_wrapper_mutation.py_functionalize_impl +def triton_kernel_wrapper_mutation_functionalize( + ctx: "BaseFunctionalizeAPI", + kernel_idx: int, + constant_args_idx: int, + grid: list["TritonGridType"], + tma_descriptor_metadata: TMADescriptorMetadata, + kwargs: dict[str, Any], +) -> None: + unwrapped_kwargs = ctx.unwrap_tensors(kwargs) # type: ignore[arg-type] + # TODO(oulgen): Preexisting bug, if two kernel inputs are views of each + # other, and one gets mutated in kernel, and later another gets mutated, + # they are no longer equal. Fix this by graph breaking on this condition + # earlier in dynamo. + tensors_to_clone = get_mutated_tensors( + kernel_idx, constant_args_idx, unwrapped_kwargs, tma_descriptor_metadata + ) + with ctx.redispatch_to_next(): + unwrapped_outputs = triton_kernel_wrapper_functional( + kernel_idx=kernel_idx, + constant_args_idx=constant_args_idx, + grid=grid, + tma_descriptor_metadata=tma_descriptor_metadata, + kwargs=unwrapped_kwargs, + tensors_to_clone=tensors_to_clone, + ) + + assert set(unwrapped_outputs.keys()).issubset(set(kwargs.keys())) + for key, output_arg in unwrapped_outputs.items(): + if not isinstance(output_arg, Tensor): + continue + input_arg = kwargs[key] + assert isinstance(input_arg, Tensor) + + ctx.replace(input_arg, output_arg) + # indicate that above replace is hidden from autograd + ctx.mark_mutation_hidden_from_autograd(input_arg) + ctx.commit_update(input_arg) + ctx.sync(input_arg) + return None + + +@triton_kernel_wrapper_functional.py_impl(DispatchKey.CompositeExplicitAutograd) +def triton_kernel_wrapper_functional_dense( + *, + kernel_idx: int, + constant_args_idx: int, + grid: list["TritonGridType"], + tma_descriptor_metadata: TMADescriptorMetadata, + kwargs: dict[str, Any], + tensors_to_clone: list[str], +) -> dict[str, Any]: + # TODO(oulgen): For performance reasons, we want to ensure that these + # `clone_preserve_strides` calls are never executed at runtime + # (inductor should always optimize them away). + # Requires https://github.com/pytorch/pytorch/issues/109240 + kwargs = { + key: (clone_preserve_strides(val) if key in tensors_to_clone else val) + for key, val in kwargs.items() + } + triton_kernel_wrapper_mutation( + kernel_idx=kernel_idx, + constant_args_idx=constant_args_idx, + grid=grid, + tma_descriptor_metadata=tma_descriptor_metadata, + kwargs=kwargs, + ) + return {key: val for key, val in kwargs.items() if key in tensors_to_clone} + + +@triton_kernel_wrapper_functional.py_impl(FakeTensorMode) +def triton_kernel_wrapper_functional_fake_tensor_mode( + mode: FakeTensorMode, + *, + kernel_idx: int, + constant_args_idx: int, + grid: list["TritonGridType"], + tma_descriptor_metadata: TMADescriptorMetadata, + kwargs: dict[str, Any], + tensors_to_clone: list[str], +) -> dict[str, Any]: + # TODO(oulgen): For performance reasons, we want to ensure that these + # `clone_preserve_strides` calls are never executed at runtime + # (inductor should always optimize them away). + # Requires https://github.com/pytorch/pytorch/issues/109240 + with mode: + return { + key: clone_preserve_strides(val) + for key, val in kwargs.items() + if key in tensors_to_clone + } + + +@triton_kernel_wrapper_functional.py_impl(ProxyTorchDispatchMode) +def triton_kernel_wrapper_functional_proxy_torch_dispatch_mode( + mode: ProxyTorchDispatchMode, + *, + kernel_idx: int, + constant_args_idx: int, + grid: list["TritonGridType"], + tma_descriptor_metadata: TMADescriptorMetadata, + kwargs: dict[str, Any], + tensors_to_clone: list[str], +) -> dict[str, Any]: + ret = trace_triton_kernel_wrapper( + mode, + triton_kernel_wrapper_functional, + { + "kernel_idx": kernel_idx, + "constant_args_idx": constant_args_idx, + "grid": grid, + "tma_descriptor_metadata": tma_descriptor_metadata, + "kwargs": kwargs, + "tensors_to_clone": tensors_to_clone, + }, + ) + assert ret is not None + return ret + + +@triton_kernel_wrapper_functional.py_functionalize_impl +def triton_kernel_wrapper_functional_functionalize( + ctx: "BaseFunctionalizeAPI", + kernel_idx: int, + constant_args_idx: int, + grid: list["TritonGridType"], + tma_descriptor_metadata: TMADescriptorMetadata, + kwargs: dict[str, Any], + tensors_to_clone: list[str], +) -> dict[str, Any]: + unwrapped_kwargs = ctx.unwrap_tensors(kwargs) # type: ignore[arg-type] + with ctx.redispatch_to_next(): + outputs = triton_kernel_wrapper_functional( + kernel_idx=kernel_idx, + constant_args_idx=constant_args_idx, + grid=grid, + tma_descriptor_metadata=tma_descriptor_metadata, + kwargs=unwrapped_kwargs, + tensors_to_clone=tensors_to_clone, + ) + return ctx.wrap_tensors(outputs) # type: ignore[return-value,arg-type] + + +triton_kernel_wrapper_mutation.fallthrough(DispatchKey.PythonDispatcher) # type: ignore[attr-defined] +triton_kernel_wrapper_mutation.fallthrough(DispatchKey.PythonTLSSnapshot) # type: ignore[attr-defined] +triton_kernel_wrapper_mutation.fallthrough(DispatchKey.ADInplaceOrView) +triton_kernel_wrapper_mutation.fallthrough(DispatchKey.BackendSelect) +triton_kernel_wrapper_mutation.fallthrough(DispatchKey.AutocastCPU) # type: ignore[attr-defined] +triton_kernel_wrapper_mutation.fallthrough(DispatchKey.AutocastCUDA) # type: ignore[attr-defined] +triton_kernel_wrapper_mutation.fallthrough(DispatchKey.AutogradCUDA) +triton_kernel_wrapper_mutation.fallthrough(DispatchKey.AutogradCPU) + +triton_kernel_wrapper_functional.fallthrough(DispatchKey.PythonDispatcher) # type: ignore[attr-defined] +triton_kernel_wrapper_functional.fallthrough(DispatchKey.PythonTLSSnapshot) # type: ignore[attr-defined] +triton_kernel_wrapper_functional.fallthrough(DispatchKey.ADInplaceOrView) +triton_kernel_wrapper_functional.fallthrough(DispatchKey.BackendSelect) +triton_kernel_wrapper_functional.fallthrough(DispatchKey.AutocastCPU) # type: ignore[attr-defined] +triton_kernel_wrapper_functional.fallthrough(DispatchKey.AutocastCUDA) # type: ignore[attr-defined] +triton_kernel_wrapper_functional.fallthrough(DispatchKey.AutogradCUDA) +triton_kernel_wrapper_functional.fallthrough(DispatchKey.AutogradCUDA) +triton_kernel_wrapper_functional.fallthrough(DispatchKey.AutogradCPU) + +# Adds SAC support for triton ops +redirect_to_mode(triton_kernel_wrapper_mutation, _CachingTorchDispatchMode) +redirect_to_mode(triton_kernel_wrapper_mutation, _CachedTorchDispatchMode) + +############################################################################### +# The "TritonHOPifier": a class that transforms a call to a triton kernel into +# a call to the triton_kernel_wrapper_mutation HOP. + + +class TritonHOPifier: + """Orchestrator for converting a user-defined triton kernel into a call + to the triton_kernel_wrapper_mutation HOP. + + It has two main use cases. + + 1. When Dynamo sees a triton kernel, it wraps it into a TritonKernelVariable + and uses the TritonHOPifier to convert calls to the TritonKernelVariable + into a call to the HOP. + + 2. In order to capture a user-defined triton kernel while performing + tracing (via make_fx or non-strict export), a user must annotate their + triton kernel with the `wrap_triton` decorator. The decorator uses + TritonHOPifier to convert calls to the triton kernel into a call + to the HOP (which can then be traced). + + Because Dynamo has its own calling conventions for e.g. invoking a user-defined function + TritonHOPifier is an abstract class that can be overridden by its subclasses. + """ + + def raise_unsupported(self, msg: str) -> Never: + raise NotImplementedError("abstract method") + + def is_callable(self, maybe_callable: Any) -> bool: + raise NotImplementedError("abstract method") + + def get_value(self, val: Any) -> Any: + raise NotImplementedError("abstract method") + + def call_grid( # type: ignore[no-untyped-def] + self, + grid, + meta, + tx, + ) -> Union[tuple[Union[int, sympy.Expr, SymInt], ...], tuple["Proxy", ...]]: + raise NotImplementedError("abstract method") + + def wrap_user_defined_obj( + self, + user_obj: Any, + tx: Optional["InstructionTranslator"], + variable: Optional[ + Union["TritonKernelVariable", "TraceableTritonKernelWrapper"] + ], + name: str, + ) -> Any: + raise NotImplementedError("abstract method") + + def call_user_defined_fn( + self, + user_fn: Callable[..., Any], + args: list, + kwargs: dict, + tx: Optional["InstructionTranslator"], + variable: Optional[ + Union["TritonKernelVariable", "TraceableTritonKernelWrapper"] + ], + ) -> Any: + raise NotImplementedError("abstract method") + + def maybe_unpack_configs( + self, configs: list["TritonConfig"], tx: Optional["InstructionTranslator"] + ) -> list["TritonConfig"]: + raise NotImplementedError("abstract method") + + def maybe_unpack_heuristic_result(self, result: Any) -> Any: + raise NotImplementedError("abstract method") + + @staticmethod + def do_prune_configs( # type: ignore[no-untyped-def] + autotuner: "TritonAutotunerType", + early_config_prune: Optional[Callable], + perf_model: Optional[Callable], + top_k: float, + configs: list, + named_args: dict, + kwargs: dict, + ) -> list["TritonConfig"]: + # Reimplement autotuner.prune_configs(...) here + # see: https://github.com/triton-lang/triton/blob/e57b46897191b3b3061c78d0d60e58e94be565b6/python/triton/runtime/autotuner.py # noqa: E501,B950 + # We do this to avoid calling prune_configs, which in turn calls early_config_prune and perf_model + # These are both user-defined functions which can contain side effects, so we want to sandbox them in Dynamo + + if early_config_prune: + configs = early_config_prune(configs, named_args, **kwargs) + + if perf_model: + # we assert top_k is a float before calling this + if isinstance(top_k, float) and top_k <= 1.0: + top_k = int(len(configs) * top_k) + elif not isinstance(top_k, int): + """ + Slice index must be an integer, SupportsIndex or None + """ + raise TypeError( + "Error while pruning configs, top_k must be either 1) a float <= 1.0 or 2) an int" + ) + if len(configs) > top_k: + est_timing = [ + ( + config, + float( + perf_model(**named_args, **kwargs, **config.all_kwargs()) + ), + ) + for config in configs + ] + configs = [ + config[0] + for config in sorted(est_timing, key=operator.itemgetter(1))[:top_k] + ] + return configs + + def call_HOP( # type: ignore[no-untyped-def] + self, + variable, + grids, + combined_args: dict[str, Any], + tx, + ) -> Optional["ConstantVariable"]: + raise NotImplementedError("abstract method") + + def check_grid( # type: ignore[no-untyped-def] + self, grid + ) -> Union[tuple[Union[int, sympy.Expr, SymInt], ...], tuple["Proxy", ...]]: + raise NotImplementedError("abstract method") + + def init_variable( + self, + variable: Union["TraceableTritonKernelWrapper", "TritonKernelVariable"], + kernel: "TritonKernelType", + kernel_idx: Optional[int], + grid: Optional["TritonGridType"], + ) -> None: + from triton.runtime.autotuner import Autotuner + + assert kernel is not None + + variable.kernel = kernel + variable.kernel_idx = kernel_side_table.add_kernel(kernel) + + assert kernel_idx is None or variable.kernel_idx == kernel_idx + + # pyrefly: ignore [bad-assignment] + variable.grid = grid + + if isinstance(kernel, Autotuner): + import torch + import torch._dynamo + + # We only support configs, keys, and restore_value arguments + # of triton.autotune. Make sure other arguments are defaulted. + defaults = inspect.signature(Autotuner.__init__).parameters + # Newer version of triton change attribute name from warmup to num_warmup and rep to num_rep. + # The call to get_first_attr is to maintain backward-compatibility. + + def defaults_ok( + attr: str, alternates: tuple[str, ...], values: tuple[Any, ...] + ) -> bool: + if attr not in defaults: + return True + value = torch._dynamo.utils.get_first_attr(kernel, attr, *alternates) + if value == defaults[attr].default: + return True + return value in values + + if ( + not torch._inductor.config.unsafe_ignore_unsupported_triton_autotune_args + and ( + not defaults_ok("num_warmups", ("warmup",), (25, None)) + or not defaults_ok("num_reps", ("rep",), (100, None)) + or not defaults_ok("use_cuda_graph", (), (False,)) + ) + ): + self.raise_unsupported( + "Only configs, keys, restore_value, and reset_to_zero are supported for triton.autotune" + ) + if ( + not torch._inductor.config.unsafe_ignore_unsupported_triton_autotune_args + and ( + # pre_hook requires running arbitrary code at runtime, which we cannot handle at this time + # https://github.com/pytorch/pytorch/issues/139059 + # we can't support pre_hook or post_hook in user defined triton kernels at the moment, + # as they require the ability to execute code at runtime (AOTI can't support this) + ( + hasattr(kernel, "user_defined_pre_hook") + and kernel.user_defined_pre_hook is not False + ) + or ( + hasattr(kernel, "user_defined_post_hook") + and kernel.user_defined_post_hook is not False + ) + or ( + # Check Config passed to autotuner in configs + any(cfg.pre_hook is not None for cfg in kernel.configs) + ) + ) + ): + self.raise_unsupported( + "pre_hook and post_hook are not supported in triton.Autotune or triton.Config" + ) + + def call_getitem( + self, + variable: Union["TritonKernelVariable", "TraceableTritonKernelWrapper"], + args: Sequence[Any], + ) -> Union["TritonKernelVariable", "TraceableTritonKernelWrapper"]: + # __getitem__ should only be called if we don't already have a grid + # Only grid needs to be passed + if variable.grid is not None or len(args) != 1: + self.raise_unsupported( + "Triton kernels should be called with only a single grid" + ) + + return type(variable)( + kernel=variable.kernel, + kernel_idx=variable.kernel_idx, + grid=args[0], + ) + + def call_run( + self, + variable: Union["TritonKernelVariable", "TraceableTritonKernelWrapper"], + args: Sequence[Any], + kwargs: dict[str, Any], + tx: Optional["InstructionTranslator"], + ) -> Optional["ConstantVariable"]: + if "grid" not in kwargs: + self.raise_unsupported("Triton kernel requires to be called with a grid") + grid = kwargs.pop("grid") + kwargs.pop("warmup", None) + # rewrite kernel.run(*args, grid=grid) to kernel[grid](*args) + return self.call_triton_kernel( + type(variable)( + kernel=variable.kernel, kernel_idx=variable.kernel_idx, grid=grid + ), + args, + kwargs, + tx, + ) + + def call_triton_kernel( + self, + variable: Union["TritonKernelVariable", "TraceableTritonKernelWrapper"], + args: Sequence[Any], + kwargs: dict[str, Any], + tx: Optional["InstructionTranslator"], + ) -> Optional["ConstantVariable"]: + from triton import JITFunction + from triton.runtime.autotuner import autotune, Autotuner, Config, Heuristics + + # Check if num_ctas is in kwargs + if "num_ctas" in kwargs: + self.raise_unsupported( + "Passing num_ctas directly to the Triton kernel is not supported. " + "Please use a Config in @triton.autotune instead." + ) + + # Make sure the kernel has a grid + if variable.grid is None: + self.raise_unsupported("Triton kernels should always be called with a grid") + + # raise an exception if there are multiple @triton.autotune decorators + iter_kernel = variable.kernel + autotuner_count = 0 + while not isinstance(iter_kernel, JITFunction): + if isinstance(iter_kernel, Autotuner): + autotuner_count += 1 + if autotuner_count > 1: + self.raise_unsupported( + "Passing multiple @triton.autotune decorators is not supported. " + "Please use a single @triton.autotune decorator instead." + ) + # pyrefly: ignore # missing-attribute + iter_kernel = iter_kernel.fn + + # Process the @triton.heuristics decorator: + # - We know there is only 1 autotuner decorator here + # - We can apply the heuristic to all triton.Configs in the order that the decorators appear + # This way, when the config is selected, the heuristics have already been applied. + # - Decorators that appear *before* the autotuner are already processed correctly + if isinstance(variable.kernel, Autotuner) and isinstance( + variable.kernel.fn, Heuristics + ): + # unwrap the heuristics decorator, we don't need it anymore + # variable.kernel ==> Autotuner + # variable.kernel.fn ==> Heuristics + # ... + # There can be arbitrarily many heuristics wrappers here! + # ... + # variable.kernel.fn ==> JITFunction + + # Copy the configs, we are going to be modifying them + new_configs = copy.deepcopy(variable.kernel.configs) + + named_args = dict(zip(variable.kernel.arg_names, args)) + + # Iterate through all of the heuristics wrappers that come after the autotune wrapper + iter_kernel = variable.kernel.fn + while isinstance(iter_kernel, Heuristics): + # For each config, apply the heuristic fn(s) + for config_idx in range(len(new_configs)): + for kwarg_key, heuristic_fn in iter_kernel.values.items(): + # Run heuristics on the combined configs + kwargs + heuristic_result = self.call_user_defined_fn( + heuristic_fn, + [ + { + **named_args, + **kwargs, + **new_configs[config_idx].__dict__["kwargs"], + }, + ], + {}, + tx, + variable, + ) + + # Update the kwargs in each config + # maybe_unpack_heuristic_result raises unsupported if the value is non-constant + new_configs[config_idx].__dict__["kwargs"][kwarg_key] = ( + self.maybe_unpack_heuristic_result(heuristic_result) + ) + + iter_kernel = iter_kernel.fn + assert isinstance(iter_kernel, JITFunction) + prune_configs_by = { + "perf_model": variable.kernel.perf_model, + "early_config_prune": variable.kernel.early_config_prune, + "configs_top_k": variable.kernel.configs_top_k, + } + new_kernel = autotune( + configs=new_configs, key=[], prune_configs_by=prune_configs_by + )(iter_kernel) + # create a new variable to contain the new (wrapped) kernel; + # skip kernel_idx to get a new record in the kernel side table + new_var = type(variable)(new_kernel, None, variable.grid) + return self.call_triton_kernel(new_var, args, kwargs, tx) + + SPECIAL_CONFIG_NAMES = { + "num_warps", + "num_stages", + "num_ctas", + "num_consumer_groups", + "num_buffers_warp_spec", + "num_cpu_threads", + } + + # move special config names to configs out of kwargs + special_kwargs = {} + for name in SPECIAL_CONFIG_NAMES: + if name in kwargs: + # remove special kwargs from `kwargs` + val = kwargs.pop(name) + special_kwargs[name] = self.get_value(val) + + if special_kwargs: + if isinstance(variable.kernel, Autotuner): + # if there is Autotuner already, set + # special kwargs to each of its configs + new_configs = copy.deepcopy(variable.kernel.configs) + for config in new_configs: + config.__dict__.update(special_kwargs) + prune_configs_by = { + "perf_model": variable.kernel.perf_model, + "early_config_prune": variable.kernel.early_config_prune, + "configs_top_k": variable.kernel.configs_top_k, + } + + new_kernel = autotune( + configs=new_configs, key=[], prune_configs_by=prune_configs_by + )(variable.kernel.fn) + else: + # if there is no Autotuner, wrap the kernel into a + # new one with a single config with special kwargs + new_config = Config(kwargs={}, **special_kwargs) + + new_kernel = autotune(configs=[new_config], key=[])(variable.kernel) + + # create a new variable to contain the new (wrapped) kernel; + # skip kernel_idx to get a new record in the kernel side table + new_var = type(variable)(new_kernel, None, variable.grid) + return self.call_triton_kernel(new_var, args, kwargs, tx) + + if isinstance(variable.kernel, Autotuner): + special_param_names = [] + for name in SPECIAL_CONFIG_NAMES: + if name in variable.kernel.fn.arg_names: + special_param_names.append(name) + + if special_param_names: + # If the Triton kernel has SPECIAL_CONFIG_NAMES in parameters, those should + # be passed from the kernel configs: the behavior of Triton runtime is that + # those values get folded into the kernel arguments iff there are parameters + # with the same name. Normally the values of those parameters are defined + # outside the `kwargs` part of the autotuning configs. Here we move them to + # the `kwargs` part (if they're absent there) to facilitate passing them as + # arguments to the kernel downstream. + updated = False + new_configs = copy.deepcopy(variable.kernel.configs) + for config in new_configs: + for name in special_param_names: + if name not in config.__dict__["kwargs"]: + assert name in config.__dict__, ( + f"{name} must be in autotuning configs to be used as a kernel parameter" + ) + config.__dict__["kwargs"][name] = config.__dict__[name] + updated = True + + if updated: + prune_configs_by = { + "perf_model": variable.kernel.perf_model, + "early_config_prune": variable.kernel.early_config_prune, + "configs_top_k": variable.kernel.configs_top_k, + } + + new_kernel = autotune( + configs=new_configs, prune_configs_by=prune_configs_by, key=[] + )(variable.kernel.fn) + new_var = type(variable)(new_kernel, None, variable.grid) + return self.call_triton_kernel(new_var, args, kwargs, tx) + + # These are the default values in upstream Triton + # see: https://github.com/triton-lang/triton/blob/e57b46897191b3b3061c78d0d60e58e94be565b6/python/triton/runtime/autotuner.py # noqa: E501,B950 + default_perf_model = None + default_early_config_prune = None + + # run prune_configs_by + if isinstance(variable.kernel, Autotuner) and ( + variable.kernel.perf_model != default_perf_model + or variable.kernel.early_config_prune != default_early_config_prune + ): + # Prune the configs + named_args = dict(zip(variable.kernel.arg_names, args)) + + # The source information is important here so the guards are installed correctly + + wrapped_early_configs_prune = self.wrap_user_defined_obj( + variable.kernel.early_config_prune, + tx, + variable, + "early_config_prune", + ) + + wrapped_perf_model = self.wrap_user_defined_obj( + variable.kernel.perf_model, tx, variable, "perf_model" + ) + + wrapped_configs_top_k = self.wrap_user_defined_obj( + variable.kernel.configs_top_k, tx, variable, "configs_top_k" + ) + + wrapped_configs = self.wrap_user_defined_obj( + variable.kernel.configs, tx, variable, "configs" + ) + + pruned_configs = self.call_user_defined_fn( + self.do_prune_configs, + [ + variable, + wrapped_early_configs_prune, + wrapped_perf_model, + wrapped_configs_top_k, + wrapped_configs, + named_args, + kwargs, + ], + {}, + tx, + variable, + ) + + pruned_configs = self.maybe_unpack_configs(pruned_configs, tx) + + # after pruning the configs, create a new autotuner object with + # these configs and recurse. + new_kernel = autotune(configs=pruned_configs, key=[])(variable.kernel.fn) + # create a new variable to contain the new (wrapped) kernel; + # skip kernel_idx to get a new record in the kernel side table + new_var = type(variable)(new_kernel, None, variable.grid) + return self.call_triton_kernel(new_var, args, kwargs, tx) + + # Both for grid's meta as well as for the kernel, we need combined + # args and kwargs combined and normalized + # pyrefly: ignore # missing-attribute + combined_args_raw = {**dict(zip(variable.kernel.arg_names, args)), **kwargs} + + # precompute the grid for the kernel + configs = ( + [config.kwargs for config in variable.kernel.configs] + if isinstance(variable.kernel, Autotuner) + else [{}] + ) + grids = [] + for config_args in configs: + # If the grid is a function, then lets execute it and convert it to + # a list + grid = variable.grid + assert grid is not None + if self.is_callable(grid): + # Populate the special "meta" argument to call the grid function + meta = {**combined_args_raw, **config_args} + grid = self.call_grid(grid, meta, tx) # type: ignore[arg-type] + grids.append(self.check_grid(grid)) + + for i in range(len(grids)): + if not isinstance(grids[i], tuple): + self.raise_unsupported("Only tuple grids are supported") + # inductor expects all grids to be 3-tuple so lets make it + if len(grids[i]) == 1: + grids[i] = (grids[i][0], 1, 1) + elif len(grids[i]) == 2: + grids[i] = (grids[i][0], grids[i][1], 1) + elif len(grids[i]) > 3: + self.raise_unsupported("Grid can have at most rank 3") + + assert len(grids) != 0 + if isinstance(variable.kernel, JITFunction): + constexprs = [p.num for p in variable.kernel.params if p.is_constexpr] + arg_names = [p.name for p in variable.kernel.params] + else: + # If we are looking at an @triton.autotune decorator, the nested function should be a JITFunction + # This is because we don't support @triton.heuristics or nested @triton.autotune decorators yet + assert isinstance(variable.kernel, Autotuner) + constexprs = [p.num for p in variable.kernel.fn.params if p.is_constexpr] + arg_names = [p.name for p in variable.kernel.fn.params] + + for idx, arg_name in enumerate(arg_names): + if idx in constexprs: + if arg_name in combined_args_raw: + # [Note: Specialize tl.constexpr args in user-defined triton kernels] + # This arg is marked as tl.constexpr. That means that triton will recompile every time + # this value changes. + # https://github.com/pytorch/pytorch/issues/136504 + # One option is to correctly pass the symints in so that the symbolic expressions are defined + # when the triton code is being executed. + # But since triton will have to recompile either way, we instead just specialize on the value. + # + # Depending on the type of `variable` we might expect different types for the symbolic args: + # either SymNodeVariables (for TritonKernelVariables) or SymInts (TracingTritonKernelWrapper) + combined_args_raw[arg_name] = variable.specialize_symbolic( + combined_args_raw[arg_name] + ) + return self.call_HOP(variable, grids, combined_args_raw, tx) + + +############################################################################### +# Helpers for wrap_triton API that makes a user-defined triton kernel traceable into +# a graph via make_fx or non-strict export (coming soon) + + +class TracingTritonHOPifier(TritonHOPifier): + def raise_unsupported(self, msg: str) -> Never: + raise RuntimeError(msg) + + def is_callable(self, maybe_callable: Any) -> bool: + return callable(maybe_callable) + + def get_value(self, val: Any) -> Any: + return val + + def call_grid( + self, + grid: "TritonGridCallableType", + meta: "TritonMetaParamsType", + tx: None, + ) -> tuple[Union[int, sympy.Expr, SymInt], ...]: + assert tx is None + assert isinstance(meta, dict) + assert callable(grid) + return grid(meta) + + def wrap_user_defined_obj( + self, + user_obj: Any, + tx: Optional["InstructionTranslator"], + variable: Optional[ + Union["TritonKernelVariable", "TraceableTritonKernelWrapper"] + ], + name: str, + ) -> Any: + assert tx is None + return user_obj + + def call_user_defined_fn( + self, + user_fn: Callable[..., Any], + args: list, + kwargs: dict, + tx: Optional["InstructionTranslator"], + variable: Optional[ + Union["TritonKernelVariable", "TraceableTritonKernelWrapper"] + ], + ) -> Any: + assert isinstance(args, list) + assert isinstance(kwargs, dict) + assert callable(user_fn) + return user_fn(*args, **kwargs) + + def maybe_unpack_configs( + self, configs: list["TritonConfig"], tx: Optional["InstructionTranslator"] + ) -> list["TritonConfig"]: + assert isinstance(configs, list) + return configs + + def maybe_unpack_heuristic_result(self, result: Any) -> Any: + return result + + def check_grid( + self, + grid: "TritonGridType", + ) -> tuple[Union[int, sympy.Expr, SymInt], ...]: + if not isinstance(grid, collections.abc.Sequence): + raise RuntimeError( + "wrap_triton can only handle grids that resolve to Sequence[int]." + ) + # normalize to tuple + return tuple(grid) + + def store_non_graphable_args( + self, + combined_args: dict[str, Any], + ) -> tuple[dict, int]: + """ + Some args cannot be stored in the FX graph. + Put them in the side table. + """ + + def is_graphable(val: Any) -> bool: + return isinstance(val, (fx.node.base_types, fx.Node)) + + non_graphable_args = { + k: v for k, v in combined_args.items() if not is_graphable(v) + } + graphable_args = {k: v for k, v in combined_args.items() if is_graphable(v)} + + constant_args_idx = kernel_side_table.add_constant_args(non_graphable_args) + + return graphable_args, constant_args_idx + + def call_HOP( + self, + variable: "TraceableTritonKernelWrapper", + grids: list["TritonGridTupleType"], + combined_args: dict[str, Any], + tx: None, + ) -> None: + assert tx is None + assert isinstance(variable, TraceableTritonKernelWrapper) + + graphable_args, constant_args_idx = self.store_non_graphable_args(combined_args) + + assert isinstance(variable.kernel_idx, int) + return triton_kernel_wrapper_mutation( + kernel_idx=variable.kernel_idx, + constant_args_idx=constant_args_idx, + grid=grids, # type: ignore[arg-type] + # TMA descriptor capturing not yet + # supported in non-dynamo tracing + tma_descriptor_metadata={}, + kwargs=graphable_args, + ) + + +tracing_triton_hopifier_singleton = TracingTritonHOPifier() + + +class TraceableTritonKernelWrapper: + kernel: "TritonKernelType" + kernel_idx: Optional[int] + grid: Optional["TritonGridType"] + + def __init__( + self, + kernel: "TritonKernelType", + kernel_idx: Optional[int], + grid: Optional["TritonGridType"], + ) -> None: + # pyrefly: ignore # bad-assignment + self.kernel = None + self.grid = None + tracing_triton_hopifier_singleton.init_variable(self, kernel, kernel_idx, grid) + assert self.kernel is not None + + def __getitem__(self, *args: Sequence[Any]) -> "TraceableTritonKernelWrapper": + return tracing_triton_hopifier_singleton.call_getitem(self, args) # type: ignore[return-value] + + def run(self, *args: Sequence[Any], **kwargs: dict[str, Any]) -> Any: + from torch._library.triton import is_wrap_triton_enabled + + if is_wrap_triton_enabled(): + return tracing_triton_hopifier_singleton.call_run(self, args, kwargs, None) + else: + assert self.kernel is not None + # pyrefly: ignore [missing-attribute] + return self.kernel.run(*args, **kwargs) + + def __call__(self, *args: Sequence[Any], **kwargs: dict[str, Any]) -> Any: + from torch._library.triton import is_wrap_triton_enabled + + if is_wrap_triton_enabled(): + return tracing_triton_hopifier_singleton.call_triton_kernel( + self, args, kwargs, None + ) + else: + assert self.kernel is not None + # pyrefly: ignore [index-error] + return self.kernel[self.grid](*args, **kwargs) + + def specialize_symbolic(self, arg: Sequence[Any]) -> Any: + import torch + + # See [Note: Specialize tl.constexpr args in user-defined triton kernels] + if isinstance(arg, (torch.SymInt, torch.SymBool, torch.SymFloat)): + return guard_scalar(arg) + return arg diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..fad19b1d5ffae24abd656922ade56cfb0331d4bc --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/utils.py @@ -0,0 +1,1255 @@ +# mypy: allow-untyped-defs +import contextlib +import functools +from collections.abc import Callable, Iterable, Sequence +from contextlib import AbstractContextManager, contextmanager, ExitStack, nullcontext +from dataclasses import dataclass +from typing import Any, Optional, overload, TypeVar, Union + +import torch +import torch.fx.traceback as fx_traceback +import torch.utils._pytree as pytree +from torch._dispatch.python import suspend_functionalization +from torch._guards import detect_fake_mode +from torch._higher_order_ops.schema import HopSchema +from torch._ops import HigherOrderOperator, OperatorBase, OpOverload +from torch._subclasses.fake_tensor import FakeTensor +from torch._subclasses.functional_tensor import ( + disable_functional_mode, + FunctionalTensor, +) +from torch.fx.experimental.proxy_tensor import ( + _temp_remove_metadata_torch_function_mode, + disable_proxy_modes_tracing, + make_fx, +) +from torch.fx.passes.runtime_assert import insert_deferred_runtime_asserts +from torch.fx.passes.shape_prop import _extract_tensor_metadata, TensorMetadata +from torch.multiprocessing.reductions import StorageWeakRef + + +@dataclass +class UnsupportedAliasMutationException(RuntimeError): + reason: str + + +def autograd_not_implemented_inner( + operator: OperatorBase, delayed_error: bool, *args: Any, **kwargs: Any +) -> Any: + """If autograd is enabled and any of the arguments require grad this will either + raise an error or return a DelayedError depending on the value of delayed. + + Args: + operator: The Operator to call with the *args and **kwargs with + op_name: The name of the Operator + delayed_error: If True, return a DelayedError instead of raising an error + args: The flattened operands to the Operator + kwargs: The keyword arguments to the Operator + + Raises: + RuntimeError: If autograd is enabled and any of the arguments to the Operator + """ + with torch._C._AutoDispatchBelowAutograd(): + result = operator(*args, **kwargs) + flat_operands = pytree.arg_tree_leaves(*args) + if torch.is_grad_enabled() and any( + f.requires_grad for f in flat_operands if isinstance(f, torch.Tensor) + ): + if delayed_error: + err_fn = torch._C._functions.DelayedError( + f"Autograd not implemented for {str(operator)}", + 1, + ) + + def fake_requires_grad(tensor): + if torch.is_floating_point(tensor) or torch.is_complex(tensor): + tensor = tensor.detach() + tensor.requires_grad = True + return tensor + + return pytree.tree_map_only( + torch.Tensor, lambda x: err_fn(fake_requires_grad(x)), result + ) + else: + raise RuntimeError(f"Autograd not implemented for {str(operator)}") + return result + + +def autograd_not_implemented(op: OperatorBase, deferred_error: bool) -> Callable: + def inner(*args, **kwargs): + return autograd_not_implemented_inner(op, deferred_error, *args, **kwargs) + + return inner + + +def _maybe_run_with_interpreter(fn): + maybe_interpreted_fn = fn + if isinstance(fn, torch.fx.GraphModule) and fx_traceback.has_preserved_node_meta(): + # Running graph with interpreter is needed for propagating the stack_trace + def graph_with_interpreter(*args): + with fx_traceback.preserve_node_meta(): + return torch.fx.Interpreter(fn).run(*args) + + maybe_interpreted_fn = graph_with_interpreter + return maybe_interpreted_fn + + +def _maybe_compile_and_run_fn(fn, *args): + if not torch.compiler.is_dynamo_compiling(): + with setup_compilation_env() as backend: # type: ignore[attr-defined] + return torch.compile(fn, backend=backend, fullgraph=True)(*args) + else: + return fn(*args) + + +def reenter_make_fx(fn): + from torch.fx.experimental.proxy_tensor import _CURRENT_MAKE_FX_TRACER + + @functools.wraps(fn) + def wrapped(*args): + assert _CURRENT_MAKE_FX_TRACER is not None, ( + "Cannot reenter make_fx when we're not under a make_fx tracing session" + ) + gm = _CURRENT_MAKE_FX_TRACER.trace_subgraph( + _maybe_run_with_interpreter(fn), *args + ) + return gm + + return wrapped + + +def _maybe_reenter_make_fx(fn): + from torch.fx.experimental.proxy_tensor import _CURRENT_MAKE_FX_TRACER + + if _CURRENT_MAKE_FX_TRACER is not None: + return reenter_make_fx(fn) + else: + + def _maybe_make_fx_with_fake_mode(fn): + @functools.wraps(fn) + def wrapped(*args): + from torch._guards import detect_fake_mode + + fake_mode = detect_fake_mode(args) + if fake_mode is None: + # we creaeta a fake_mode here to make sure we could + # trace the graph with data-dependent calls e.g. .item() + return make_fx(fn, tracing_mode="fake")(*args) + # Tracing with real if all inputs have been fakfied + return make_fx(fn)(*args) + + return wrapped + + return _maybe_make_fx_with_fake_mode(fn) + + +def check_meta_consistency( + lhs_list: list[Union[torch.Tensor, torch.SymInt, int]], + rhs_list: list[Union[torch.Tensor, torch.SymInt, int]], + lhs_name: str, + rhs_name: str, + include_contiguity: bool = True, +) -> None: + def diff_meta_pairs( + lhs_list: list[Union[torch.Tensor, torch.SymInt, int]], + rhs_list: list[Union[torch.Tensor, torch.SymInt, int]], + ) -> list[str]: + def diff_meta( + lhs: Union[torch.Tensor, torch.SymInt, int], + rhs: Union[torch.Tensor, torch.SymInt, int], + ) -> str: + if isinstance(lhs, torch.Tensor) and isinstance(rhs, torch.Tensor): + return ", ".join( + diff_tensor_meta( + _extract_tensor_metadata( + lhs, include_contiguity=include_contiguity + ), + _extract_tensor_metadata( + rhs, include_contiguity=include_contiguity + ), + check_grad=False, + ) + ) + else: + + def _both_int_types(lhs, rhs): + return isinstance(lhs, (int, torch.SymInt)) and isinstance( + rhs, (int, torch.SymInt) + ) + + def _both_tensor(lhs, rhs): + return isinstance(lhs, torch.Tensor) and isinstance( + rhs, torch.Tensor + ) + + if not _both_int_types(lhs, rhs) and not _both_tensor(lhs, rhs): + return f"type: {lhs} vs {rhs}" + + return "" + + # Manually check the device of lhs and rhs as this field is currently not part of TensorMetadata + def diff_device( + lhs: Union[torch.Tensor, torch.SymInt, int], + rhs: Union[torch.Tensor, torch.SymInt, int], + ) -> str: + if isinstance(lhs, torch.Tensor) and isinstance(rhs, torch.Tensor): + if ( + rhs.device.type == lhs.device.type + and rhs.device.index == lhs.device.index + ): + return "" + else: + return "device" + return "" + + if len(lhs_list) != len(rhs_list): + raise torch._dynamo.exc.UncapturedHigherOrderOpError( + f"Expected {lhs_name} and {rhs_name} to have same number of outputs but got lhs:{lhs_list} and rhs:{rhs_list}" + ) + all_diffs = [] + for i, (lhs, rhs) in enumerate(zip(lhs_list, rhs_list)): + if diff := diff_meta(lhs, rhs): + all_diffs.append( + f"pair[{i}] differ in {diff}, where lhs is {lhs} and rhs is {rhs}" + ) + if diff := diff_device(lhs, rhs): + all_diffs.append( + f"pair[{i}] differ in {diff}, where lhs is {lhs} and rhs is {rhs}" + ) + return all_diffs + + if all_diffs := diff_meta_pairs(lhs_list, rhs_list): + diff_str = "\n".join(all_diffs) + raise torch._dynamo.exc.UncapturedHigherOrderOpError( + f"Expected {lhs_name} and {rhs_name} to have same metadata but found:\n{diff_str}" + ) + + +@contextmanager +def setup_compilation_env(): + """ + Context manager that sets up proper environment and backend when invoking torch.compile + inside torch.export region or inside HOP. + """ + from torch._dynamo.backends.debugging import ( + make_eager_backend_with_torch_function_modes, + ) + from torch.fx.experimental.proxy_tensor import ( + _temp_remove_pre_dispatch_torch_function_mode, + ) + + with ( + _set_compilation_env(), + torch._dynamo.utils.disable_cache_limit(), + _temp_remove_pre_dispatch_torch_function_mode() as pre_dispatch_mode, + _temp_remove_metadata_torch_function_mode() as metadata_mode, + ): + modes = [ + mode for mode in (pre_dispatch_mode, metadata_mode) if mode is not None + ] + if modes: + yield make_eager_backend_with_torch_function_modes(modes) + else: + yield "eager" + + +@contextmanager +def _set_compilation_env(): + _old_is_tracing = torch.fx._symbolic_trace._is_fx_tracing_flag + _old_allow_empty_graphs = torch._dynamo.config.allow_empty_graphs + _old_capture_scalar_outputs = torch._dynamo.config.capture_scalar_outputs + # The issue is tracked in https://github.com/pytorch/pytorch/issues/144360: when dynamo finds + # the top-level frame produces no graph, the default behavior is to fallback to eager. + # Then when it encounters an inner function, it will try to trace that function again, which is unnecessary. + # For while_loop, during inspecting the inner call, we trace into the python dispathcer + # logic, which is not tracable as of today. So the proper fix can be either 1. allow dispatch + # logic to be dynamo tracable or 2. fixing https://github.com/pytorch/pytorch/issues/144360. + # but it exposes some bugs in existing tests so we have to have a temporary flag to control + # the behavior, which allows dynamo to store an empty graph for a frame without falling back to eager + try: + # We need to turn off the is_fx_tracing_flag. Remove this flag check from dyanmo + # once we are confident fx tracing works with dynamo. + torch.fx._symbolic_trace._is_fx_tracing_flag = False + # pyrefly: ignore [bad-assignment] + torch._dynamo.config.allow_empty_graphs = True + torch._dynamo.config.capture_scalar_outputs = True + yield + finally: + torch.fx._symbolic_trace._is_fx_tracing_flag = _old_is_tracing + torch._dynamo.config.allow_empty_graphs = _old_allow_empty_graphs + torch._dynamo.config.capture_scalar_outputs = _old_capture_scalar_outputs + + +# The invariant here is that we always trace the branch with fake tensor +def _maybe_fake_tracing(fn, inputs: list[Any], pre_dispatch): + fake_mode_det = detect_fake_mode(inputs) + fake_mode: AbstractContextManager = nullcontext() + tracing_mode = "fake" + if fake_mode_det is not None: + fake_mode = fake_mode_det + tracing_mode = "real" + + # Note: we need to turn off proxy tensor mode to avoid tracing infra + # code that happens in make_fx e.g. we now call as_strided when wrapping tensor + # as fake tensor. + with fake_mode, disable_proxy_modes_tracing(): + gm = make_fx( + fn, + tracing_mode=tracing_mode, + pre_dispatch=pre_dispatch, + _error_on_data_dependent_ops=False, + )(*inputs) + if not isinstance(fake_mode, nullcontext) and fake_mode.shape_env is not None: # type: ignore[attr-defined] + insert_deferred_runtime_asserts( + gm, + fake_mode.shape_env, # type: ignore[attr-defined] + "hoo_maybe_fake_tracing", + export=True, # type: ignore[attr-defined] + ) + return gm + + +def potential_input_alias_or_mutation(gm, inputs, pre_dispatch=False): + try: + gm = _maybe_fake_tracing(gm, inputs, pre_dispatch) + except UnsupportedAliasMutationException: + # this can happen when nested cond_op is + # functionalized + return True + except Exception as e: + raise e + + example_inputs = [ + ph.meta.get("val", None) for ph in gm.graph.find_nodes(op="placeholder") + ] + ( + inp_inp_alias_map, + inp_out_alias_map, + out_out_alias_map, + inp_mutation, + ) = check_input_alias_and_mutation(gm, example_inputs) + return (inp_inp_alias_map, inp_out_alias_map, out_out_alias_map), inp_mutation + + +def analyze_potential_input_alias_or_mutation(name, aliases, input_mutations): + if any(len(a) > 0 for a in aliases): + # TODO: Investigate here further which node is exactly aliasing + raise RuntimeError( + f"{name} where aliases appear. " + + f"In particular, these inputs \ + {set(el for el_map in aliases if len(el_map.keys()) > 0 for el in el_map)} " # noqa: C401 + + "get aliased. Please ensure that this doesn't happen." + ) + if len(input_mutations): + # TODO: Investigate here further which node is exactly mutating the inputs + raise RuntimeError( + f"{name} where the inputs are mutated. " + + f"In particular, these nodes are mutating the inputs \ + {set(el for el in input_mutations)}." # noqa: C401 + + "Please ensure that this doesn't happen." + ) + + +def _has_potential_branch_input_mutation(gm, inputs, pre_dispatch=False): + ( + (_, _, _), + inp_mutation, + ) = potential_input_alias_or_mutation(gm, inputs, pre_dispatch) + + return len(inp_mutation) > 0 + + +def has_potential_input_alias_or_mutation(gm, inputs, pre_dispatch=False): + ( + ( + inp_inp_alias_map, + inp_out_alias_map, + out_out_alias_map, + ), + inp_mutation, + ) = potential_input_alias_or_mutation(gm, inputs, pre_dispatch) + return ( + any( + ( + len(inp_inp_alias_map) > 0, + len(inp_out_alias_map) > 0, + len(out_out_alias_map) > 0, + ) + ), + len(inp_mutation) > 0, + ) + + +def _collect_fake_inputs(inputs): + from torch._subclasses.fake_tensor import FakeTensor + + # Get the example values of the inputs. + inputs_fake: list[Union[FakeTensor, torch.Tensor, int]] = [] + for inp in inputs: + if isinstance(inp, (torch.fx.proxy.Proxy, torch.fx.node.Node)): + inp = inp.node if isinstance(inp, torch.fx.proxy.Proxy) else inp + if hasattr(inp, "meta"): + val = inp.meta["example_value"] + if isinstance(val, torch.Tensor): + if torch._C._functorch.is_batchedtensor( + val + ) or torch._C._functorch.is_functionaltensor(val): + # This case is for batched or functional tensors + # Unwrap the tensors + while torch._C._functorch.is_batchedtensor( + val + ) or torch._C._functorch.is_functionaltensor(val): + val = torch._C._functorch.get_unwrapped(val) + assert isinstance(val, FakeTensor) + inputs_fake.append(val) + else: + # This is the standard case of a TensorVariable + assert isinstance(val, FakeTensor) + inputs_fake.append(val) + else: + # This case is for SymInts and other non-Tensor elements + assert not isinstance(val, torch.Tensor) + inputs_fake.append(val) + else: + # This case is for ints + assert isinstance(inp, int) + inputs_fake.append(inp) + + return inputs_fake + + +def _check_alias_and_mutation(graph_module, inputs_fake, name, pre_dispatch): + aliases, inp_mutation = has_potential_input_alias_or_mutation( + graph_module, inputs_fake, pre_dispatch=pre_dispatch + ) + if aliases: + raise RuntimeError(f"{name} might be aliasing the input or the output!") # noqa: F541 + if inp_mutation: + raise RuntimeError(f"{name} might be modifying the input!") # noqa: F541 + + +def unique_graph_id(proxy_mode, prefix): + """Returns a unique name and id for a graph to be added to a proxy_mode tracer""" + # There are probably better ways - I know that create_arg has some self incrementing name + # magic to it, but since we explicitly have to get the name for register_module, + # I was not sure how to do that. This kinda simulates it. + return unique_graph_name_with_root(proxy_mode.tracer.root, prefix) + + +def unique_graph_name_with_root( + root: torch.fx.GraphModule, prefix: str +) -> tuple[int, str]: + next_name = None + i = 0 + # pyrefly: ignore [bad-assignment] + while not next_name: + candidate = f"{prefix}_{i}" + if hasattr(root, candidate): + i += 1 + else: + next_name = candidate + return i, next_name + + +def _from_fun(t): + from torch._functorch.aot_autograd import from_fun + + if isinstance(t, torch.Tensor): + if t.dtype != torch.bool: + return torch.empty_strided( + t.size(), + t.stride(), + dtype=t.dtype, + requires_grad=t.requires_grad, + device=t.device, + ) + else: + # clone of a functional tensor produces a functional tensor + # but we want to avoid it so we clone a non-functional version + maybe_unfunc_t = t + if isinstance(t, FunctionalTensor): + torch._sync(t) + maybe_unfunc_t = from_fun(t) + elif torch._is_functional_tensor(t): + # need to handle both types of functionalization here: + # these are the tensors that came from the user, + # which could be either FunctionalTensorWrapper or FunctionalTensor + torch._sync(t) + maybe_unfunc_t = torch._from_functional_tensor(t) + return maybe_unfunc_t.clone() + return t + + +def clone_outputs_aliasing_inputs(args): + input_storage = { + StorageWeakRef(arg._typed_storage()) + for arg in args + if isinstance(arg, torch.Tensor) + } + + def maybe_clone(t): + if ( + isinstance(t, torch.Tensor) + and StorageWeakRef(t._typed_storage()) in input_storage + ): + return t.clone() + return t + + return maybe_clone + + +def prepare_fw_with_masks(fn): + def fw_with_masks(*args): + fw_out = fn(*args) + return fw_out, [ + bool(isinstance(ret, torch.Tensor) and ret.requires_grad) for ret in fw_out + ] + + return fw_with_masks + + +def prepare_fw_with_masks_all_requires_grad(fn): + def fw_with_masks(*args): + fw_out = fn(*args) + # Note [force all outputs to be require grad] + # Instead of using the original fn, we set the output of original + # fn to all require grad. This is consistent with the behavior + # of autograd.Function, where if any one of the inputs requires grad + # all output will be require grad. This also makes the downstream + # require_gradness reasoning much easier. + if pytree.tree_any_only(torch.Tensor, lambda t: t.requires_grad, args): + fw_out = pytree.tree_map_only( + torch.Tensor, + lambda x: x.requires_grad_(True) if x.dtype.is_floating_point else x, + fw_out, + ) + + def _query_requires_grad(t: torch.Tensor) -> bool: + if torch._is_functional_tensor(t): + t = torch._from_functional_tensor(t) + return t.requires_grad + + return fw_out, pytree.tree_map_only(torch.Tensor, _query_requires_grad, fw_out) + + return fw_with_masks + + +# This function replaces None gradients with all-zero gradients. +# `None` gradients are problematic for CUDA graphs. Those gradients are +# replaced with an all-zero tensor for better optimization +def unmask_none_gradients(grads, operands): + allowed_types = (torch.Tensor, int, torch.SymInt) + assert all(isinstance(o, allowed_types) for o in operands), ( + f"operands can only be of {allowed_types} but got {[type(o) for o in operands]}" + ) + + unmasked_grads = [] + for g, o in zip(grads, operands): + if g is not None: + unmasked_grads.append(g) + else: + # In case the operand is an int or a torch.SymInt, return None + # This can happen for lifted_arguments. E.g., the shapes of a dynamic tensor are lifted and passed + # as additional arguments + unmasked_grads.append( + torch.zeros_like(o) if isinstance(o, torch.Tensor) else None + ) + + return unmasked_grads + + +def _maybe_fake_prop_ignore_unbacked(fn, args): + with ExitStack() as ctx_stack: + if (fake_mode := detect_fake_mode(args)) is not None: + ctx_stack.enter_context(fake_mode) + if fake_mode.shape_env is not None: + ctx_stack.enter_context( + fake_mode.shape_env.ignore_fresh_unbacked_symbols() + ) + return fn(*args) + + +def redirect_to_mode(hop: OperatorBase, mode): + """Utility for redispatching HOP to underlying mode + + Args: + hop: The HOP to redispatch + mode: The mode to redispatch to + + Returns: + A decorated function that implements the HOP for the given mode + """ + + @hop.py_impl(mode) + def impl(mode, *args, **kwargs): + return mode.__torch_dispatch__(hop, [], args, kwargs) + + return impl + + +# TODO: The parameter use_output_and_grad_bw is required because some operations +# that utilize this function, such as the while_loop, may require (grad, fwd_outputs) +def create_fw_bw_graph(fn, use_output_and_grad_bw, fw_inputs, fw_outputs): + from torch._functorch.aot_autograd import AOTConfig, create_joint + + # Note:[HOP create fw_bw graph] We create "clean" environments for make_fx by suspending all dispatch keys + # between Autograd and Python key. Currently, we only suspend functionalization but more can be + # added when required. Will encounter two problems if we don't suspend functionalization: + # + # 1. make_fx fails to capture operations on input: the inputs are wrapped as _to_functional_tensor_wrapper, + # but they will be unwrapped before entering ProxyTorchDispatchMode as part of the dispatching. + # However, it's the outside wrapper that tracer creates proxies for. This casuses tracer fail to + # fetch the proxy for the inputs and fail to capture any operations on them. + # + # 2. make_fx fails to capture output: the outputs after ProxyTorchDispatchMode are further + # wrapped as FunctionalTensorWrapper in Functionalize key after return. However, the tracer + # only associates the inner tensor with proxy in ProxyTorchDispatchMode. Therefore, + # when creating the output node, it fails to associate the wrapped tensor with its proxy. + # Instead, it will create _tensor_constant as output. + + dummy_aot_config = AOTConfig( + fw_compiler=None, # type: ignore[arg-type] + bw_compiler=None, # type: ignore[arg-type] + partition_fn=None, # type: ignore[arg-type] + decompositions={}, + num_params_buffers=0, + aot_id=0, + keep_inference_input_mutations=False, + ) + + example_grad = [_from_fun(out) for out in fw_outputs] + num_grads = len(example_grad) + fw_graph = _maybe_reenter_make_fx(fn)(*fw_inputs) + + def joint_fn(*joint_operands_grads): + if use_output_and_grad_bw: + grads = joint_operands_grads[0] + inputs = joint_operands_grads[1][-1:] + else: + grads = joint_operands_grads[:num_grads] + inputs = joint_operands_grads[num_grads:] + + joint = create_joint(prepare_fw_with_masks(fn), aot_config=dummy_aot_config) + _, grads = joint( + list(inputs), + [grad for grad in grads if grad is not None and grad.requires_grad], + ) + + # Unmask None gradients to all-zero gradients + unmasked_grads = unmask_none_gradients(grads, inputs) + + # In order to keep map functional for backward graph, + # we clone outputs that are aliasing inputs + maybe_clone = clone_outputs_aliasing_inputs(joint_operands_grads) + + return pytree.tree_map(maybe_clone, unmasked_grads) + + if use_output_and_grad_bw: + example_xs_out = list(fw_inputs) + list(fw_outputs) + joint_graph = _maybe_reenter_make_fx(joint_fn)( + (list(example_grad), list(example_xs_out)) + ) + else: + example_xs_out = list(fw_inputs) + joint_graph = _maybe_reenter_make_fx(joint_fn)( + *(list(example_grad) + list(example_xs_out)) + ) + + return fw_graph, joint_graph + + +def _unstack_pytree(xs): + flat_xs, inspec = pytree.tree_flatten(xs) + if not all(isinstance(xs, torch.Tensor) for xs in flat_xs): + raise RuntimeError(f"Leaves of xs must be Tensor {flat_xs}") + + if not all(xs.shape[0] == flat_xs[0].shape[0] for xs in flat_xs): + raise RuntimeError( + f"Leaves of xs must have same leading dimension size {[xs.shape for xs in flat_xs]}" + ) + + a = zip(*flat_xs) + + pytrees = [pytree.tree_unflatten(tuple, inspec) for tuple in a] + return pytrees + + +def _stack_pytree(pytrees): + flat_out = [] + out_spec = None + for pt in pytrees: + flat_pt, out_spec = pytree.tree_flatten(pt) + flat_out.append(flat_pt) + assert out_spec is not None + b = zip(*flat_out) + stacked_out = [] + for leaves in b: + if all(isinstance(leaf, torch.Tensor) for leaf in leaves): + stacked_out.append(torch.stack(leaves)) + elif all(leaf is None for leaf in leaves): + # Backward graph can return None output when forward inputs doesn't require grad. + # When we eagerly execute backward graph, we need to call _stack_pytree on its output, + # therefore we need to deal with None output. + stacked_out.append(None) # type: ignore[arg-type] + else: + raise RuntimeError(f"Cannot stack {leaves}.") + return pytree.tree_unflatten(stacked_out, out_spec) + + +# We cannot call save_for_backward for symints. This helper function +# can be used to save symints as direct attributes of ctx in autograd.Function. +# +# For example, if args = (x, y, s0, z, s1), +# save_tensors_and_symints_for_backward will partition the args into two lists, and a bookkeeping list pos: +# partitioned_args[0] = (x, y, z) +# partitioned_args[1] = (s0, s1) +# pos = (0, 0, 1, 0, 1) +# pos list keeps track of which partition the args +# is partitioned into in order to recover it in saved_tensors_and_symints. +# +# In saved_tensors_and_symints, we can recover the original args by: +# iterating over the pos list and pop one item from the front of partitioned_args[pos[i]]. +# We use t_idx and s_idx to keep track of the next index of the item we are going to pop for the two lists. +def save_tensors_and_symints_for_backward(ctx, args): + assert all( + isinstance(arg, (torch.Tensor, torch.SymInt, int, type(None))) for arg in args + ), args + partitioned_args: list[Any] = [[], []] + pos = [] + for arg in args: + idx = 0 if isinstance(arg, torch.Tensor) else 1 + partitioned_args[idx].append(arg) + pos.append(idx) + + assert not hasattr(ctx, "sym_int_args"), "ctx already has sym_int_args attribute." + assert not hasattr(ctx, "pos"), "ctx already has pos attribute." + ctx.save_for_backward(*partitioned_args[0]) + ctx.sym_int_args = partitioned_args[1] + ctx.pos = pos + + +def saved_tensors_and_symints(ctx): + args = [] + t_idx = 0 + s_idx = 0 + saved_tensors = ctx.saved_tensors + for p in ctx.pos: + if p == 0: + args.append(saved_tensors[t_idx]) + t_idx += 1 + else: + args.append(ctx.sym_int_args[s_idx]) + s_idx += 1 + assert t_idx + s_idx == len(ctx.pos) + return tuple(args) + + +def split_into_chunks(iterable: Sequence[Any], chunk_sizes: list[int]) -> list[Any]: + assert sum(chunk_sizes) == len(iterable), ( + "the sum of all chunks needs to match the length of the iterable." + ) + elements = [] + idx = 0 + for size in chunk_sizes: + elements.append(iterable[idx : idx + size]) + idx += size + return elements + + +def _clone_aliasing_output(inputs: Sequence[Any], outputs: Sequence[Any]): + # For tensors whose grad is None, create zero tensors as gradients + # This invariant is useful for cudagraph. + + # Elimitate input-output, output-output aliasing + seen_input_storages = { + StorageWeakRef(t._typed_storage()) + for t in inputs + if isinstance(t, torch.Tensor) + } + seen_output_storages = set() + final_outputs = [] + for out in outputs: + if isinstance(out, torch.Tensor): + out_storage = StorageWeakRef(out._typed_storage()) + if ( + out_storage in seen_input_storages + or out_storage in seen_output_storages + ): + out = out.clone() + seen_output_storages.add(StorageWeakRef(out._typed_storage())) + final_outputs.append(out) + return final_outputs + + +def create_bw_fn( + fn: Callable, args: tuple[Any, ...], return_fw_outputs: bool = False +) -> Callable: + """ + For a fn that accepts flat inputs and returns flat outputs: + fw_out = fn(*args), + this function returns: + grad_args = bw_fn(*args_and_grad_output) + with the following invariants: + 1. args + fw_out has an 1-1 correspondence to args_and_grad_output + 2. grad_args has an 1-1 corresponsence to args + 3. for tensor arg whose requires_grad is False, its corresponding grad in + grad_args will be a zero tensor with the same shape. + """ + + from torch._functorch.aot_autograd import AOTConfig, create_joint + + # pyrefly: ignore [missing-module-attribute] + from torch._higher_order_ops.utils import prepare_fw_with_masks_all_requires_grad + + dummy_aot_config = AOTConfig( + fw_compiler=None, # type: ignore[arg-type] + bw_compiler=None, # type: ignore[arg-type] + partition_fn=None, # type: ignore[arg-type] + decompositions={}, + num_params_buffers=0, + aot_id=0, + keep_inference_input_mutations=False, + ) + n_primals = len(args) + + bw_fn = create_joint( + prepare_fw_with_masks_all_requires_grad(fn), aot_config=dummy_aot_config + ) + + def flat_fn(*args_and_grad_outs): + primals = args_and_grad_outs[:n_primals] + tangents = args_and_grad_outs[n_primals:] + fw_outs, grad_args = bw_fn(primals, tangents) + assert len(args) == len(grad_args) + + # For tensors whose grad is None, create zero tensors as gradients + # This invariant is useful for cudagraph. + grad_args = [ + torch.zeros_like(arg) + if isinstance(arg, torch.Tensor) and grad is None + else grad + for grad, arg in zip(grad_args, primals) + ] + + final_grads = _clone_aliasing_output(args_and_grad_outs, grad_args) + if return_fw_outputs: + return *fw_outs, *final_grads + return final_grads + + return flat_fn + + +def get_dummy_aot_autograd_config(): + from torch._functorch.aot_autograd import AOTConfig + + return AOTConfig( + fw_compiler=None, # type: ignore[arg-type] + bw_compiler=None, # type: ignore[arg-type] + partition_fn=None, # type: ignore[arg-type] + decompositions={}, + num_params_buffers=0, + aot_id=0, + keep_inference_input_mutations=False, + ) + + +# Slices off the first element of a given dimension +def first_slice_copy(t: torch.Tensor, dim: int = 0) -> torch.Tensor: + return torch.select_copy(t, dim, 0) + + +# Returns a mask whether a list element is a tensor or not +def get_tensor_mask(tensor_list: Iterable[Any]) -> list[bool]: + return [bool(isinstance(v, torch.Tensor)) for v in tensor_list] + + +def mask_list( + mask: list[bool], inp: list[Any], other: Optional[list[Any]] = None +) -> list[Any]: + # Masks elements on an `inp` list. + # If other is None, then the elements of the `inp` list where the mask is False are removed + # If other is not None, then the elements of the `inp` list where the mask is False are + # replaced with the elements of the `other` list + assert len(mask) == len(inp), ( + "The length of the mask needs to be identical to the length of the input" + ) + if other is not None: + assert len(inp) == len(other), ( + "If an input and an other list is provided, they need to have the same length" + ) + return [i if m else o for m, i, o in zip(mask, inp, other)] + else: + return [i for m, i in zip(mask, inp) if m] + + +def first_slice_copy_with_grad(li: Iterable[Any]) -> list[Any]: + # First_slice_copy does not keep the original requires_grad flag, + # but we need it for materialize_as_graph + # in order to compute the correct gradients + # The reason why first_slice_copy doesn't keep requires_grad flag is + # because it's called in torch.autograd.Function.backward/forward. + slc = [first_slice_copy(x).requires_grad_(x.requires_grad) for x in li] + return slc + + +# Reports the difference between meta of two tensors in a string +def diff_tensor_meta( + meta1: TensorMetadata, meta2: TensorMetadata, check_grad=True +) -> list[str]: + from torch.fx.experimental.symbolic_shapes import GuardOnDataDependentSymNode + + pair_diffs = [] + for meta_name in TensorMetadata._fields: + if not check_grad and meta_name == "requires_grad": + continue + val1 = getattr(meta1, meta_name) + val2 = getattr(meta2, meta_name) + try: + if val1 != val2: + pair_diffs.append(f"'{meta_name}: {val1} vs {val2}'") + except GuardOnDataDependentSymNode: + pair_diffs.append(f"'{meta_name}: {val1} vs {val2}'") + continue + return pair_diffs + + +# Note [lifted arg types in hop] +# For dynamoed hops, we automatically lift the free symbols in tensors as arguments. +# This has implications for the types of lifted args for different dispatch keys: +# 1. functionalization, FakeTensorMode, ProxyTorchDispatchMode, Autograd need to support torch.Symint +# lifted args because it's on the path of torch.compile(dynamic=True). +# 2. functionalization, FakeTensorMode, ProxyTorchDispatchMode, Autograd, CompositeExplicitAutograd need +# to support int arguments. In the eager run case, we re-trace the subgraph in AutogradKey, so inner +# hops may receive int inputs from the shape of outer tensor inputs. +# However, CompositeExplicitAutograd won't receive SymInt inputs because it only accepts real tensor inputs. +def validate_subgraph_args_types(lifted_args: Union[tuple[Any, ...], list[Any]]): + allowed_types = (torch.Tensor, int, torch.SymInt) + assert all( + isinstance(arg, (torch.Tensor, int, torch.SymInt)) for arg in lifted_args + ), ( + f"{lifted_args} can only be of {allowed_types} but got {tuple(type(arg) for arg in lifted_args)}" + ) + + +# TODO: Return a more detailed information as to which node +# causes a mutation or an alias. This may requires a per operator tensor version checking +def check_input_alias_and_mutation( + gm: torch.fx.GraphModule, + fake_args: list[FakeTensor], +) -> tuple[dict[int, int], dict[int, int], dict[int, int], list[int]]: + ( + inp_inp_alias_map, + inp_out_alias_map, + out_out_alias_map, + mutated_inputs, + ) = check_input_alias_and_mutation_return_outputs(gm)[:-1] + # pyrefly: ignore [bad-return] + return inp_inp_alias_map, inp_out_alias_map, out_out_alias_map, mutated_inputs + + +def _tensor_storage(t) -> StorageWeakRef: + return StorageWeakRef(t._typed_storage()) + + +def check_input_alias_and_mutation_return_outputs( + gm: torch.fx.GraphModule, +) -> tuple[ + dict[int, int], + dict[int, int], + dict[int, int], + list[int], + Union[tuple[Any, ...], list[Any]], +]: + def _get_example_value(n): + if not isinstance(n, torch.fx.Node): + return n + else: + return n.meta["val"] if "val" in n.meta else n.meta["example_value"] + + fake_args = [ + _get_example_value(n) + for n in gm.graph.find_nodes(op="placeholder") + if isinstance(n, torch.fx.Node) and "val" in n.meta + ] + outputs = [ + _get_example_value(n) + for n in pytree.tree_flatten(gm.graph.find_nodes(op="output")[0].args[0])[0] + ] + + # We need to analyze the original fake_args to detect + # inp-inp alias. + inp_storage_map = { + _tensor_storage(inp): i + for i, inp in enumerate(fake_args) + if isinstance(inp, torch.Tensor) + } + out_storage_map = { + _tensor_storage(out): i + for i, out in enumerate(outputs) + if isinstance(out, torch.Tensor) + } + inp_inp_alias_map = { + i: inp_storage_map[_tensor_storage(inp)] + for i, inp in enumerate(fake_args) + if isinstance(inp, torch.Tensor) and inp_storage_map[_tensor_storage(inp)] != i + } + out_out_alias_map = { + i: out_storage_map[_tensor_storage(out)] + for i, out in enumerate(outputs) + if isinstance(out, torch.Tensor) and out_storage_map[_tensor_storage(out)] != i + } + inp_out_alias_map = { + i: out_storage_map[_tensor_storage(inp)] + for i, inp in enumerate(fake_args) + if isinstance(inp, torch.Tensor) and _tensor_storage(inp) in out_storage_map + } + mutated_inputs = [] + for node in gm.graph.nodes: + if node.op == "call_function" and isinstance( + node.target, torch._ops.OpOverload + ): + for arg_node, arg_schema in zip(node.args, node.target._schema.arguments): + if arg_schema.is_write: + arg_val = _get_example_value(arg_node) + assert isinstance(arg_val, torch.Tensor) + if _tensor_storage(arg_val) in inp_storage_map: + mutated_inputs.append(inp_storage_map[_tensor_storage(arg_val)]) + + return ( + inp_inp_alias_map, + inp_out_alias_map, + out_out_alias_map, + mutated_inputs, + outputs, + ) + + +registered_hop_fake_fns: dict[torch._ops.OpOverload, Callable] = {} + + +F = TypeVar("F", bound=Callable) + + +@overload +def register_fake(hop, fn: None = None) -> Callable[[F], F]: ... + + +@overload +def register_fake(hop, fn: F) -> F: ... + + +def register_fake(hop, fn=None): + """ + Register a fake function for a HOP. This is conceptually equivalent of the + register_fake utility for the custom ops. The registered function is called + inside the fake_tensor _dispatch_impl. + """ + assert hop not in registered_hop_fake_fns + + def register(func: F) -> F: + from torch._subclasses.fake_tensor import FakeTensorMode + + redirect_to_mode(hop, FakeTensorMode) + + registered_hop_fake_fns[hop] = func + return func + + if fn is None: + return register + return register(fn) + + +class FunctionalizeCtxWrapper: + """ + This is a dummy wrapper to facilitate fake tensor caching. + + For AOT Dispatcher metadata collection pass, HOPs go from functionalization + key to fake tensor key. The functionalization key wraps the subgraphs in a + function, which changes from call to call even though the subgraph might + still be same. + + To enable fake tensor caching, we just wrap the ctx and subgraph in this + class and then use the subgraph as the hash. + """ + + # Prevents PYTORCH_TEST_WITH_DYNAMO=1 test failures + @torch._disable_dynamo + def __init__(self, ctx, subgraph): + self.ctx = ctx + self.subgraph = subgraph + + def __hash__(self): + return id(self.subgraph) + + def __repr__(self): + return f"FunctionalizeCtxWrapper on subgraph {self.subgraph})" + + def __call__(self, *args, **kwargs): + if isinstance(self.subgraph, torch.fx.GraphModule): + # Running graph with interpreter is needed for propagating the stack_trace + with fx_traceback.preserve_node_meta(): + return self.ctx.functionalize(torch.fx.Interpreter(self.subgraph).run)( + *args, **kwargs + ) + return self.ctx.functionalize(self.subgraph)(*args, **kwargs) + + +# A wrapper over HigherOrderOperator that also carries its schema +class HopInstance: + def __init__(self, op: HigherOrderOperator, schema: HopSchema): + assert isinstance(op, HigherOrderOperator), op + self._op = op + # Using "_" to be consistent with how we access _schema of OpOverload + self._schema = schema + + def __call__(self, *args, **kwargs): + return self._op(*args, **kwargs) + + @staticmethod + def create(hop: HigherOrderOperator, *args, **kwargs): + return HopInstance(hop, hop.gen_schema(*args, **kwargs)) + + +# This call_op can be used to call a HopInstance with +# flat args and kwargs. We need to make use of the hop's schema's tree_spec +# to unflatten the args and kwargs before calling the hop. +def call_op(op: Union[OpOverload, HopInstance], args, kwargs): + if isinstance(op, OpOverload): + return op(*args, **kwargs) + + assert isinstance(op, HopInstance), op + schema = op._schema + bound_args = list(args) + bound_kwargs = {} + for arg in schema.arguments[len(bound_args) :]: + assert arg.name in kwargs, (arg.name, kwargs) + val = kwargs[arg.name] + if not arg.kwarg_only: + bound_args.append(val) + else: + bound_kwargs[arg.name] = val + + if schema.tree_spec is not None: + assert len(bound_args) == len(schema.arguments) and len(bound_kwargs) == 0 + args, kwargs = pytree.tree_unflatten(bound_args, schema.tree_spec) + return op(*args, **kwargs) + else: + assert len(bound_args) + len(bound_kwargs) == len(schema.arguments) + return op(*bound_args, **bound_kwargs) + + +def materialize_as_graph( + fn: Callable, + args: tuple[Any, ...], + include_key_set: Optional[torch._C.DispatchKeySet] = None, + exclude_key_set: Optional[torch._C.DispatchKeySet] = None, + force_enable_grad=False, +) -> torch.fx.GraphModule: + if include_key_set is None: + include_key_set = torch._C._dispatch_tls_local_include_set() + if exclude_key_set is None: + exclude_key_set = torch._C._dispatch_tls_local_exclude_set() + + @torch._dynamo.disable(recursive=True, reason=None) + def _materialize_as_graph_inner(): + with suspend_functionalization(), disable_functional_mode(): + with disable_proxy_modes_tracing(): + unfunc_t = [_from_fun(arg) for arg in args] + + with contextlib.ExitStack() as stack: + stack.enter_context( + torch.utils._python_dispatch._disable_current_modes() + ) + stack.enter_context( + torch._C._ForceDispatchKeyGuard(include_key_set, exclude_key_set), + ) + if force_enable_grad: + stack.enter_context(torch.enable_grad()) + # fake_mode is needed because parent tracer's fake_mode might + # be None but the associated inputs have fake mode or there + # is a global tracing context with fake mode. We nneed to + # make sure the fake mode when tracing subgraph is consistent. + if fake_mode := detect_fake_mode(unfunc_t): + stack.enter_context(fake_mode) + return _maybe_reenter_make_fx(fn)(*unfunc_t) + + gm = _materialize_as_graph_inner() + assert gm is not None + return gm + + +def materialize_callable_in_args(op: HopInstance, args, kwargs): + schema = op._schema + hop = op._op + flat_args, flat_spec = pytree.tree_flatten((args, kwargs)) + + def wrapped_fn(*flat_args): + return call_op(op, args, kwargs) + + # We need to trace the higher order op in order to materilaize the callable inputs that + # are a callable (e.g. after functionalization key) + gm = reenter_make_fx(wrapped_fn)(*flat_args) + hop_node = gm.graph.find_nodes(op="call_function", target=hop)[0] + arg_proxies = pytree.tree_leaves((hop_node.args, hop_node.kwargs)) + assert isinstance(schema, torch._C.FunctionSchema) and len(arg_proxies) == len( + schema.arguments + ) + + # call_op preserves ordering of proxies via schema + materialized_args = [] + for i, proxy in enumerate(arg_proxies): + if ( + isinstance(proxy, torch.fx.Node) + and proxy.op == "get_attr" + and isinstance(getattr(gm, proxy.target), torch.fx.GraphModule) # type: ignore[arg-type] + ): + assert callable(flat_args[i]), (schema, args, kwargs) + materialized_args.append(getattr(gm, proxy.target)) # type: ignore[arg-type] + else: + materialized_args.append(flat_args[i]) + + return pytree.tree_unflatten(materialized_args, flat_spec) + + +def has_user_subclass(args, allowed_subclasses): + """Check if any tensor arguments are user subclasses. + + This is used to determine if tensor subclasses should get a chance to run + their own implementation first before falling back to the default implementation. + + Args: + args: Arguments to check (will be flattened with pytree) + allowed_subclasses: Tuple of allowed subclass types + + Returns: + True if user tensor subclasses are found, False otherwise + """ + flat_args, _ = pytree.tree_flatten(args) + + val = any( + isinstance(a, torch.Tensor) + and type(a) is not torch.Tensor + and not isinstance(a, allowed_subclasses) + for a in flat_args + ) + return val + + +def _has_gen_schema(op: HigherOrderOperator): + # There is an InvokeQuant argument we cannot gen_schema. + if op is torch.ops.higher_order.invoke_quant_packed: + return False + method = "gen_schema" + return hasattr(type(op), method) and getattr(type(op), method) is not getattr( + HigherOrderOperator, method + ) + + +def filter_with_masks(data: list[Optional[torch.Tensor]], masks: list[bool]): + assert len(data) == len(masks) + return [item for item, keep in zip(data, masks) if keep] + + +def fill_none_with_masks(data: list[Optional[torch.Tensor]], masks: list[bool]): + data_iter = iter(data) + return [next(data_iter) if kept else None for kept in masks] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/while_loop.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/while_loop.py new file mode 100644 index 0000000000000000000000000000000000000000..148f4c516bbd262e62913640fdb8398d351b1814 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/while_loop.py @@ -0,0 +1,928 @@ +# mypy: allow-untyped-defs +import contextlib +import functools +from collections.abc import Callable +from typing import Any, Union + +import torch +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._higher_order_ops.utils import ( + _maybe_run_with_interpreter, + _set_compilation_env, + autograd_not_implemented, + check_input_alias_and_mutation_return_outputs, + check_meta_consistency, + fill_none_with_masks, + filter_with_masks, + materialize_as_graph, + reenter_make_fx, + validate_subgraph_args_types, +) +from torch._ops import HigherOrderOperator +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ( + _temp_remove_metadata_torch_function_mode, + disable_proxy_modes_tracing, + ProxyTorchDispatchMode, + track_tensor_tree, +) + + +class WhileLoopOp(HigherOrderOperator): + def __init__(self) -> None: + super().__init__("while_loop") + + def __call__( + self, + cond_fn: Callable, + body_fn: Callable, + carried_inputs: tuple[Union[torch.Tensor, int, float, bool]], + additional_inputs: tuple[Union[torch.Tensor, torch.SymInt, int], ...], + /, + ): + if not isinstance(carried_inputs, (tuple, list)): + raise RuntimeError( + f"carried_inputs must be a tuple or list, got {type(carried_inputs)}" + ) + if not isinstance(additional_inputs, (tuple, list)): + raise RuntimeError( + f"additional_inputs must be a tuple or list, got {type(additional_inputs)}" + ) + + validate_subgraph_args_types(carried_inputs) + validate_subgraph_args_types(additional_inputs) + return super().__call__(cond_fn, body_fn, carried_inputs, additional_inputs) + + # pyrefly: ignore [bad-override] + def gen_schema(self, cond_fn, body_fn, carried_inputs, additional_inputs): + from torch._higher_order_ops.schema import HopSchemaGenerator + from torch._higher_order_ops.utils import materialize_as_graph + + all_inputs = carried_inputs + additional_inputs + + cond_gm: torch.fx.GraphModule = ( + cond_fn + if isinstance(cond_fn, torch.fx.GraphModule) + else materialize_as_graph(cond_fn, all_inputs) + ) + body_gm: torch.fx.GraphModule = ( + body_fn + if isinstance(body_fn, torch.fx.GraphModule) + else materialize_as_graph(body_fn, all_inputs) + ) + + def _find_example_value(n, real_inp): + if "val" in n.meta: + return n.meta["val"] + elif "example_value" in n.meta: + return n.meta["example_value"] + else: + assert not isinstance(real_inp, torch.Tensor) + return real_inp + + ( + _, + _, + _, + body_mutated_inputs, + body_outputs, + ) = check_input_alias_and_mutation_return_outputs(body_gm) + + ( + _, + _, + _, + cond_mutated_inputs, + _, + ) = check_input_alias_and_mutation_return_outputs(cond_gm) + + mutated_inputs = set(body_mutated_inputs) | set(cond_mutated_inputs) + + schema_gen = HopSchemaGenerator(self) + schema_gen.add_arg("cond_fn", cond_gm) + schema_gen.add_arg("body_fn", body_gm) + + for idx, arg in enumerate(carried_inputs): + schema_gen.add_arg( + f"carried_input{idx}", arg, is_mutated=idx in mutated_inputs + ) + + for idx, arg in enumerate(additional_inputs): + additional_idx = len(carried_inputs) + idx + schema_gen.add_arg( + f"additional_input{idx}", + arg, + is_mutated=additional_idx in mutated_inputs, + ) + + for out in body_outputs: + schema_gen.add_output(out) + + schema_gen.add_schema_tree_spec( + cond_fn, body_fn, carried_inputs, additional_inputs + ) + return schema_gen.gen_schema() + + +while_loop_op = WhileLoopOp() + + +def while_loop(cond_fn, body_fn, carried_inputs): + r""" + Run body_fn(*carried_inputs) while cond_fn(*carried_inputs) returns a True scalar tensor. Returns the output of body_fn or + initial carried_inputs. + + .. warning:: + `torch.while_loop` is a prototype feature in PyTorch. It has limited support for input and output types and + doesn't support training currently. Please look forward to a more stable implementation in a future version of PyTorch. + Read more about feature classification at: https://pytorch.org/blog/pytorch-feature-classification-changes/#prototype + + `while_loop` is a structured control flow operator. It preserves the loop semantic across the torch.compile and torch.export. + + `while_loop` is equivalent to the following: + + def while_loop(cond_fn, body_fn, carried_inputs): + val = carried_inputs + while cond_fn(*val): + val = body_fn(*val) + return val + + Args: + cond_fn (Callable): A callable function that returns a boolean Scalar tensor or a python boolean. + + body_fn (Callable): A callable function that takes the same inputs as `cond_fn` and returns a tuple of tensors or ints + + carried_inputs (Tuple of possibly nested dict/list/tuple of tensors or ints): A tuple of inputs to cond_fn and body_fn. + It's also the initial value of states that are carried across iterations. Note that when pass an integer as carry, + the corresponding return of while_loop will be another int with unknown values because we don't know how many + iterations while_loop will run. + + Example 1: + + def cond_fn(iter, x): + return iter.sum() < 10 + + def body_fn(iter, x): + return iter + 1, x.sin() + + while_loop(cond_fn, body_fn, (torch.zeros(1), torch.randn(3, 4))) + + Example 2: + + def cond_fn(int_iter, x): + return 2 * int_iter < x.shape[0] + + def body_fn(int_iter, x): + return int_iter + 1, x + int_iter + + while_loop(cond,_fn, body_fn, (0, torch.randn(3, 4))) + + Restrictions: + + - body_fn must return tensors or int with the same metadata (e.g.shape, dtype) as inputs. + + - body_fn and cond_fn must not in-place mutate the carried_inputs. A clone before the mutation is required. + + - body_fn and cond_fn must not mutate python variables (e.g. list/dict) created outside of the body_fn. + + - body_fn and cond_fn's output cannot alias any of the inputs. A clone is required. + + .. warning:: + Temporal Limitations: + + - 'while_loop' only supports **inference** right now. Autograd will be supported in the future. + + """ + from torch._dynamo.backends.debugging import ( + make_eager_backend_with_torch_function_mode, + ) + + # Currently, additional_inputs is not a user-facing input. It will be automatically set in dynamo. + # parameters and buffers accessed in cond_fn or body_fn or tensor closures will become additional_inputs. + additional_inputs: tuple = () + + # The reason we flatten the output before calling into dynamo is that + # we want to create a consistent input ordering for cond_fn and body_fn. + # and we also want to the input ordering matches the output ordering. + # Also see NOTE: [why we cannot use "automatic" for while_loop] + # Construct flat cond_fn and flat_body_fn, which takes flattened inputs + flat_inputs, in_spec = pytree.tree_flatten((carried_inputs, additional_inputs)) + + def flat_cond_fn(*flat_args): + carried, additional = pytree.tree_unflatten(flat_args, in_spec) + return cond_fn(*carried, *additional) + + def flat_body_fn(*flat_args): + carried, additional = pytree.tree_unflatten(flat_args, in_spec) + return body_fn(*carried, *additional) + + if torch.compiler.is_dynamo_compiling(): + return while_loop_op(flat_cond_fn, flat_body_fn, tuple(flat_inputs), tuple()) + + def _validate_input(cond_fn, body_fn, carried_inputs): + from torch._higher_order_ops.utils import validate_subgraph_args_types + + if not callable(cond_fn) or not callable(body_fn): + raise RuntimeError("Expect cond_fn and body_fn to be callable.") + + validate_subgraph_args_types(flat_inputs) + + if not pytree.tree_all( + lambda t: isinstance(t, (torch.Tensor, torch.SymInt, int)), carried_inputs + ): + raise RuntimeError( + "Expect carried_inputs to be a tuple of possibly nested dict/list/tuple that only" + f"consists of tensor or int leaves, but got {carried_inputs}." + ) + + _validate_input(cond_fn, body_fn, carried_inputs) + + # Dynamo is expecting a callable with "__code__" attribute. + # We cannot directly pass cond_op to it. So we wrap it in a dummy function. + def _while_loop_op_wrapper(*args, **kwargs): + return while_loop_op(*args, **kwargs) + + with _set_compilation_env(), torch._dynamo.utils.disable_cache_limit(): + with _temp_remove_metadata_torch_function_mode() as metadata_mode: + with _temp_remove_metadata_torch_function_mode() as metadata_mode: + if metadata_mode: + backend: Union[str, Callable[..., Any]] = ( + make_eager_backend_with_torch_function_mode(metadata_mode) + ) + else: + backend = "eager" + return torch.compile( + _while_loop_op_wrapper, backend=backend, fullgraph=True + )(flat_cond_fn, flat_body_fn, tuple(flat_inputs), tuple()) + + +@while_loop_op.py_impl(DispatchKey.CompositeExplicitAutograd) +def while_loop_dense( + cond_fn, body_fn, carried_inputs, additional_inputs, stack_output=False +): + carried_vals = carried_inputs + + def _validate_cond_output(pred): + if ( + isinstance(pred, torch.Tensor) + and pred.size() == torch.Size([]) + and pred.dtype == torch.bool + ) or isinstance(pred, bool): + return + else: + raise RuntimeError( + f"cond_fn must return a boolean scalar tensor or a boolean but got {pred}" + ) + + if not isinstance(carried_inputs, (tuple, list)): + raise RuntimeError( + f"carried_inputs must be a tuple or list but got {type(carried_inputs)}" + ) + + # Check condition and set up flag + should_loop = cond_fn(*carried_vals, *additional_inputs) + _validate_cond_output(should_loop) + + if not should_loop: + if stack_output: + return tuple( + val.unsqueeze(0).clone() if isinstance(val, torch.Tensor) else val + for val in carried_vals + ) + else: + return tuple( + val.clone() if isinstance(val, torch.Tensor) else val + for val in carried_vals + ) + + outputs: list[list[torch.Tensor]] = [[] for _ in carried_vals] + + while should_loop: + out = body_fn(*carried_vals, *additional_inputs) + if stack_output: + for i, o in enumerate(out): + outputs[i].append(o) + + assert isinstance(out, tuple), ( + f"body_fn should return a tuple but got {type(out)}" + ) + assert len(out) == len(carried_inputs), ( + "body_fn should return the same number of elements as carried_inputs" + ) + carried_vals = out + + should_loop = cond_fn(*carried_vals, *additional_inputs) + + if stack_output: + outs: list[torch.Tensor] = [] + for out in outputs: + outs.append(torch.stack(out, dim=0)) + return tuple(outs) + + return carried_vals + + +@while_loop_op.py_autograd_impl +def while_loop_autograd(cond_fn, body_fn, operands, additional_inputs): + return WhileLoopAutogradOp.apply( + cond_fn, + body_fn, + len(operands), + len(additional_inputs), + *operands, + *additional_inputs, + ) + + +def _find_or_create_fake_mode() -> FakeTensorMode: + from torch.fx.experimental.symbolic_shapes import ShapeEnv + + fake_mode = torch._guards.detect_fake_mode() + if fake_mode is None: + fake_mode = FakeTensorMode(shape_env=ShapeEnv()) + + return fake_mode + + +def _create_unbacked_symint( + fake_mode: FakeTensorMode, ignore_fresh_unbacked_symbols: bool +) -> torch.SymInt: + assert fake_mode is not None and fake_mode.shape_env is not None, ( + "Must provide a fake_mode with shape_env." + ) + ctx = ( + contextlib.nullcontext() + if not ignore_fresh_unbacked_symbols + else fake_mode.shape_env.ignore_fresh_unbacked_symbols() + ) + with ctx: + return fake_mode.shape_env.create_unbacked_symint() + + +@while_loop_op.py_impl(ProxyTorchDispatchMode) +def while_loop_tracing( + mode, + cond_fn, + body_fn, + carried_inputs, + additional_inputs, + stack_output=False, +): + op = while_loop_stack_output_op if stack_output else while_loop_op + + def _trace_while_loop( + proxy_mode, op, cond_fn, body_fn, carried_inputs, additional_inputs + ): + # NOTE [unspecialize int carry with unbacked symints] + # When we support int carry, we'll also need to support int output of body_fn because. + # previous iteration's output is next iteration's input and they must match. + # For carries, when we start tracing while_loop, they can be + # - constants e.g. (0, [1, 3]) + # - backed symints (x.shape[0], [x.shape[1] + x.stride[1], x.shape[2]]) + # - unbacked symints e.g. (u0, [u0 + u1, u2]) + # We choose the most conservative design: in all cases, we create new unbacked symints to trace the + # subgraph. It's possible to do some analysis on initial carry and the output of first + # iteration to determine a better range for the output unbacked symbol e.g. when input is an unbacked + # symint >= 0 before the while_loop but in general this is difficult because we don't know + # the number of iterations. Users would have to re-constrain the unbacked symint in subgraph if needed. + # + # For output of fake cond_fn, it could be constant bool or SymBool (e.g. return x.shape[0] < 4, + # where x.shape[0] can be either static of dynamic). In the case of constant bool, we should do a + # specialization (NYI). + + # For output of fake body_fn, it could be all three types though from user's point of view, + # they're all integers e.g. + + # init_carry = (0, s0, u1, t) + # def body_fn(u0, s0, u1, t): + # ... + # return (t.shape[0], t.shape[1], t.shape[2], y + 1) + # + # It may seem that a constant output isn't possible: users shouldn't write a while_loop + # that always return 0. But it could be that a shape is not set as dynamic properly (e.g. + # automatic dynamic hasn't been triggered). + # + # For this reason, we treat int, symint outputs in the same way: + # - they can match against any of int, symint carry + # - we unspecialize them with new unbacked symints in fake while_loop + # Similarly, we could do some analysis to refine the output ranges but it's easier to start with + # fresh unbacked symints. One surprising case can be: an input unbacked symint is constrained by + # users to be >= 0 (either before while_loop or inside body_fn) and it increments by 1 in each + # iteration. Ideally, we should know that the final output is >= 0 but we didn't constrain the + # unbacked symint output of subgraph as of today because this requires a smart range analysis. + fake_mode: FakeTensorMode = _find_or_create_fake_mode() + + def _unspecialize_carried_inputs(x): + if isinstance(x, (int, torch.SymInt)): + return _create_unbacked_symint( + fake_mode, ignore_fresh_unbacked_symbols=True + ) + # Note: [unspecialize constant tensor carry] + # We need to disable constant specialization for tensor inputs that become loop carries. + # Here's the problem: when a user creates a constant tensor e.g. torch.tensor(0), PyTorch calls aten.lift_fresh_copy + # to create a safe copy (avoiding aliasing issues), which creates a FakeTensor with constant=True. + # But when this FakeTensor becomes a loop carry, we have a problem: + # - Operations like .item() will read the constant value and bake it into the traced code + # - This is incorrect because carry variables change between loop iterations + # - The traced code would use the wrong constant value for all iterations + # Solution: We clone the constant tensors and mark the cloned tensor as non-constant so they won't + # be specialized to fixed values during tracing body_fn or cond_fn. + elif isinstance(x, torch.Tensor): + x = x.clone() + if hasattr(x, "constant") and x.constant is not None: + # pyrefly: ignore [missing-attribute] + x.constant = None + return x + + with disable_proxy_modes_tracing(): + unspecialized_carried_inputs = pytree.tree_map_only( + (int, torch.SymInt, torch.Tensor), + # For temporarily created unbacked symints, we don't need to bind them to any proxy + lambda x: _unspecialize_carried_inputs(x), + carried_inputs, + ) + + def produce_graph(fn): + cloned_carried_inputs = pytree.tree_map_only( + torch.Tensor, lambda x: x.clone(), unspecialized_carried_inputs + ) + return reenter_make_fx(fn)(*cloned_carried_inputs, *additional_inputs) + + cond_graph = produce_graph(cond_fn) + body_graph = produce_graph(body_fn) + + next_name = None + i = 0 + # pyrefly: ignore [bad-assignment] + while not next_name: + candidate = f"while_loop_cond_graph_{i}" + if hasattr(proxy_mode.tracer.root, candidate): + i += 1 + else: + next_name = candidate + cond_graph_name = next_name + body_graph_name = f"while_loop_body_graph_{i}" + assert not hasattr(proxy_mode.tracer.root, body_graph_name) + + proxy_mode.tracer.root.register_module(cond_graph_name, cond_graph) + proxy_mode.tracer.root.register_module(body_graph_name, body_graph) + + args = (cond_graph, body_graph, carried_inputs, additional_inputs) + + proxy_args = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, args) + + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", op, proxy_args, {}, name=op._name + ) + + out = op( + cond_graph, body_graph, unspecialized_carried_inputs, additional_inputs + ) + return track_tensor_tree( + out, out_proxy, constant=None, tracer=proxy_mode.tracer + ) + + return _trace_while_loop( + mode, + op, + cond_fn, + body_fn, + carried_inputs, + additional_inputs, + ) + + +@while_loop_op.py_impl(FakeTensorMode) +def while_loop_fake_tensor_mode( + mode, cond_fn, body_fn, carried_inputs, additional_inputs, stack_output=False +): + with mode: + # NOTE: [Handling unback symints in subgraph of while_loop] + # The idea is that the scope of unbacked symints are limited to the subgraph. + # + # We're implementing the fake tensor mode of while_loop operator. + # and we run body_fn once to get an fake output. + # Let's first consider the case that unbacked symints are tensor shapes: + # + # Case 1: + # if the unbacked symints is local to the subgraph e.g. + # def body_fn(it, x): + # nz = x.nonzero() + # return it+1. nz.sum() + # we can just ignore the newly created unbacked symints because it has + # no effect on the output of while_loop and it's tracked when we tracing. + # the subgraph. + # + # Case 2: + # if the unbacked symints are shape of output of while_loop e.g. + # def body_fn(it, x): + # nz = x.nonzero() + # return it+1, nz + # This will fail the shape check because in each iteration, the carried_input's shape + # must match the output shape as nz.shape contains newly allocated unbacked symint, this + # won't match the carried_input's shape. + # + # Case 3: + # if the unbacked symints are shape of carried_inputs e.g. + # nz = a.nonzero() + # body_fn(it, nz): + # return it+1. nz.sin() + 1, + # There's no new unbacked symints allocated in subgraph, so we're safe. + with mode.shape_env.ignore_fresh_unbacked_symbols(): + # body_fn return output with the same pytree and tensor meta data as carried_inputs + # so we could just return the output after one iteration. + body_outs = body_fn(*carried_inputs, *additional_inputs) + check_meta_consistency( + carried_inputs, + body_outs, + "carried_inputs", + "body_output", + include_contiguity=False, + ) + + if stack_output: + n_iter = _create_unbacked_symint(mode, ignore_fresh_unbacked_symbols=False) + assert all(isinstance(x, torch.Tensor) for x in carried_inputs) + fake_outputs = tuple( + out.clone() + .unsqueeze(0) + .repeat((n_iter,) + tuple(1 for _ in range(out.dim()))) + for out in body_outs + ) + return pytree.tree_map_only( + (int, torch.SymInt), + # For while_loop's unbacked symint output, we want them to be bound + # to the proxy of while_loop's output. + lambda _: _create_unbacked_symint( + mode, ignore_fresh_unbacked_symbols=False + ), + fake_outputs, + ) + + # See NOTE [unspecialize int carry with unbacked symints] + return pytree.tree_map_only( + (int, torch.SymInt), + # For while_loop's unbacked symint output, we want them to be bound + # to the proxy of while_loop's output. + lambda _: _create_unbacked_symint( + mode, ignore_fresh_unbacked_symbols=False + ), + body_outs, + ) + + +@while_loop_op.py_functionalize_impl +def while_loop_func( + ctx, cond_fn, body_fn, carried_inputs, additional_inputs, stack_output=False +): + from torch._higher_order_ops.utils import _check_alias_and_mutation + + op = while_loop_stack_output_op if stack_output else while_loop_op + + unwrapped_carried_inputs = ctx.unwrap_tensors(carried_inputs) + unwrapped_additional_inputs = ctx.unwrap_tensors(additional_inputs) + unwrapped_inputs = unwrapped_carried_inputs + unwrapped_additional_inputs + with ctx.redispatch_to_next(): + functional_cond_fn = ctx.functionalize(_maybe_run_with_interpreter(cond_fn)) + functional_body_fn = ctx.functionalize(_maybe_run_with_interpreter(body_fn)) + pre_dispatch = hasattr(ctx, "mode") and ctx.mode.pre_dispatch + for fn, fn_name in [ + (cond_fn, "cond_fn"), + (body_fn, "body_fn"), + ]: + _check_alias_and_mutation(fn, unwrapped_inputs, fn_name, pre_dispatch) + ret = op( + functional_cond_fn, + functional_body_fn, + unwrapped_carried_inputs, + unwrapped_additional_inputs, + ) + return ctx.wrap_tensors(ret) + + +class WhileLoopStackOutputOp(HigherOrderOperator): + """ + while_loop_stack_output is a variant of while_loop that returns a stack of outputs. + Its semantic can be illurated using python code as: + def while_loop_stack_output(cond_fn, body_fn, carried_inputs, additional_inputs): + outs = [] + while cond_fn(*carried_inputs, *additional_inputs): + out = body_fn(*carried_inputs, *additional_inputs) + outs.append(out) + return torch.stack(outs) + + It's useful for supporting autograd of while_loop. + """ + + def __init__(self) -> None: + super().__init__("while_loop_stack_output") + + def __call__( + self, + cond_fn: Callable, + body_fn: Callable, + carried_inputs: tuple[Union[torch.Tensor, int, float, bool]], + additional_inputs: tuple[Union[torch.Tensor, torch.SymInt, int], ...], + /, + ): + if not isinstance(carried_inputs, (tuple, list)): + raise RuntimeError( + f"carried_inputs must be a tuple or list, got {type(carried_inputs)}" + ) + if not isinstance(additional_inputs, (tuple, list)): + raise RuntimeError( + f"additional_inputs must be a tuple or list, got {type(additional_inputs)}" + ) + + validate_subgraph_args_types(carried_inputs) + validate_subgraph_args_types(additional_inputs) + return super().__call__(cond_fn, body_fn, carried_inputs, additional_inputs) + + +# Note [while_loop autograd] +# Consider wthe following while_loop that can be visualized as: +# additional_inputs +# ┌─────┬─────┼─────┬─────┐ +# | | | | | +# ↓ ↓ ↓ ↓ ↓ +# x ──→ y0 ─→ y1 ─→ y2 ─→ y3 ─→ y4 +# +# The bacwkard can be visualized as follows: +# +# g_additional_inputs +# ┌──────┬──────┼──────┬──────┐ +# | | | | | +# | | | | | +# gx <── gy0 <─ gy1 <─ gy2 <─ gy3 <─ gy4 +# +# We can compute gx using chain rule: +# +# gx = gy0 * bw(y0, x), +# +# where gy0 denotes the gradient of loss with respect to y0, and bw(y0, x) denotes the gradient of y0 with +# respect to x. Note that bw can be computed from forward body_fn easily using torch.autograd.grad. +# We could substitute the unknowns gy0, gy1, ..., with chain rule until gy4: +# +# gx = gy1 * bw(y1, y0) * bw(y0, x) +# = gy2 * bw(y2, y1) * bw(y1, y0) * bw(y0, x) +# = ... +# = gy4 * bw(y4, y3) * bw(y3, y2) * bw(y2, y1) * bw(y1, y0) * bw(y0, x) +# +# since gy4 is the graient of the final output, which is given as the backward input, we've got a formula +# to compute gx. A abbr for the formula is: gy4 * bw43210x +# +# In a similar way, we can compute g_additional_inputs using chain rule: +# +# g_additional_inputs = gy0 * bw(y0, addi) + gy1 * bw(y1, addi) + gy2 * bw(y2, addi) + ... + gy4 * bw(y4, addi) +# +# Notice that gy0 = gy4 * bw43210, gy1 = gy4 * bw4321 etc, we now also get a formula for g_additional_inputs. +# +# Implementation: +# The idea of implementation is to construct a while_loop to calculate both gx and g_additional_inputs. +# Specifically, we can implement the backward of while_loop with as follows: +# +# def cond_fn(idx, grad_carries, grad_additional_inputs, fw_additional_inputs, fw_inps): +# return idx < fw_inps.size(0) +# +# def body_fn(idx, grad_carries, grad_additional_inputs, fw_additional_inputs, fw_inps): +# reversed_idx = fw_inps.size(0) - 1 - idx +# next_grad_carry, next_grad_additional_inputs = bw(fw_inps[reversed_idx], fw_additional_inputs, grad_carries) +# return idx + 1, next_grad_carry, next_grad_additional_inputs + grad_additional_inputs +# +# idx = 0 +# init_grad_carries = grads +# init_grad_additional_inputs = torch.zeros_like(g_additional_inputs) +# fw_inps = torch.cat([ctx.fw_carried_inputs, fw_outputs[:-1]]) +# while_loop(cond_fn, body_fn, (idx, init_grad_carries, init_grad_additional_inputs,), (fw_additional_inputs, fw_inps)) + + +class WhileLoopAutogradOp(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward( + ctx, + cond_fn, + body_fn, + num_carried_inputs, + num_additional_inputs, + *carries_and_inputs, + ): + from torch._higher_order_ops.scan import split_into_chunks + + carries, additional_inputs = split_into_chunks( + carries_and_inputs, [num_carried_inputs, num_additional_inputs] + ) + with torch._C._AutoDispatchBelowAutograd(): + fw_outputs = while_loop_stack_output_op( + cond_fn, body_fn, carries, additional_inputs + ) + + assert not hasattr(ctx, "fw_cond_fn") + assert not hasattr(ctx, "fw_body_fn") + assert not hasattr(ctx, "carries") + assert not hasattr(ctx, "additional_inputs") + assert not hasattr(ctx, "fw_outputs") + ctx.fw_cond_fn = cond_fn + ctx.fw_body_fn = body_fn + ctx.carries = carries + ctx.additional_inputs = additional_inputs + ctx.fw_outputs = fw_outputs + loop_count = None + # pyrefly: ignore [bad-assignment] + for out in fw_outputs: + if isinstance(out, torch.Tensor): + if loop_count is not None: + assert out.size(0) == loop_count + else: + loop_count = out.size(0) + assert loop_count is not None + + # Remove the loop_count from pending_fresh_unbacked_symbols + # because it's not part of forward output and it's impossible + # to bind it to a proxy in forward graph anyways. + if ( + isinstance(loop_count, torch.SymInt) + and (shape_env := loop_count.node.shape_env) + and loop_count in shape_env.pending_fresh_unbacked_symbols + ): + shape_env.pending_fresh_unbacked_symbols.remove(loop_count) + + # Even when body function is not executed, we clone and unsqueeze the input + # to avoid the aliasing, therefore loop_count is always >= 1 + torch._check(loop_count >= 1) + # We snapshot the dispatch keys in forward for materializing the + # the bw_graph in backward. + ctx._fw_include_key_set = torch._C._dispatch_tls_local_include_set() + ctx._fw_exclude_key_set = torch._C._dispatch_tls_local_exclude_set() + assert len(fw_outputs) > 0, "fw_outputs shouldn't be empty" + # Only the last of the output fw_outputs need to be returned + return tuple(ckp[-1] for ckp in fw_outputs) + + @staticmethod + def backward(ctx, *grads): + from torch._higher_order_ops.cond import create_bw_fn + from torch._higher_order_ops.scan import split_into_chunks + + # set up single step bw fn + bw_body_fn = create_bw_fn(ctx.fw_body_fn, ctx.carries + ctx.additional_inputs) + # Note [Handle inputs that're not differentiable] + # When a forward input is non-differentiable e.g. a symint or an integer tensor, their gradients + # will be None. However, we don't want to return None in the subgraph because this complicates the + # inductor codegen, where we need to do a non-uniform treatment for None and tensors. + # So we set up masks and filter the None gradients so that only tensors are returned from each step. + carries_tensor_masks = [ + bool(isinstance(t, torch.Tensor) and t.dtype.is_floating_point) + for t in ctx.carries + ] + additional_inputs_tensor_masks = [ + bool(isinstance(t, torch.Tensor) and t.dtype.is_floating_point) + for t in ctx.additional_inputs + ] + + init_idx = torch.zeros((), dtype=torch.int64) + init_grad_carries = filter_with_masks(grads, carries_tensor_masks) # type: ignore[arg-type] + init_grad_additional_inputs = tuple( + torch.zeros_like(t) + for need_keep, t in zip( + additional_inputs_tensor_masks, ctx.additional_inputs + ) + if need_keep + ) + # We need to the forward inputs to each iteration to compute the backward + # which is the concatenation of first iteraiton input i.e. ctx.carries and all iterations's + # output except the last iteration. + fw_carries = [ + torch.cat([carry.unsqueeze(0), carries[:-1]]) + for carry, carries in zip(ctx.carries, ctx.fw_outputs) + ] + for fw_carry, carry in zip(fw_carries, ctx.carries): + fw_carry.requires_grad_(carry.requires_grad) + + _, spec = pytree.tree_flatten( + ( + init_idx, + init_grad_carries, + init_grad_additional_inputs, + ctx.fw_outputs, + ctx.additional_inputs, + ) + ) + + def cond_fn(*flat_args): + ( + idx, + grad_carries, + grad_additional_inputs, + fw_carries, + additional_inputs, + ) = pytree.tree_unflatten(flat_args, spec) + assert isinstance(fw_carries[0], torch.Tensor), fw_carries[0] + # excluding the last iteration's output + return idx < fw_carries[0].size(0) + + def body_fn(*flat_args): + ( + idx, + grad_carries, + grad_additional_inputs, + fw_carries, + additional_inputs, + ) = pytree.tree_unflatten(flat_args, spec) + reversed_idx = fw_carries[0].size(0) - idx - 1 + selected_fw_carries = [ + ckp.select(0, reversed_idx.item()) for ckp in fw_carries + ] + cur_grad_carries, cur_grad_additional_inputs = split_into_chunks( + bw_body_fn(*selected_fw_carries, *additional_inputs, *grad_carries), + [len(ctx.carries), len(ctx.additional_inputs)], + ) + assert all(isinstance(t, torch.Tensor) for t in cur_grad_carries) + cur_grad_carries_tensors = filter_with_masks( + cur_grad_carries, carries_tensor_masks + ) + cur_grad_additional_inputs_tensors = filter_with_masks( + cur_grad_additional_inputs, additional_inputs_tensor_masks + ) + return ( + idx + 1, + *cur_grad_carries_tensors, + *( + cur_grad + grad + for cur_grad, grad in zip( + cur_grad_additional_inputs_tensors, grad_additional_inputs + ) + ), + ) + + args_single_step_bw = ( + init_idx, + *init_grad_carries, + *init_grad_additional_inputs, + *fw_carries, + *ctx.additional_inputs, + ) + + cond_gm = materialize_as_graph( + cond_fn, + args_single_step_bw, + ctx._fw_include_key_set, + ctx._fw_exclude_key_set, + force_enable_grad=True, + ) + + body_gm = materialize_as_graph( + body_fn, + args_single_step_bw, + ctx._fw_include_key_set, + ctx._fw_exclude_key_set, + force_enable_grad=True, + ) + + _, final_grad_carries, final_grad_additional_inputs = split_into_chunks( + while_loop_op( + cond_gm, + body_gm, + # pyrefly: ignore [bad-argument-type] + ( + init_idx, + *init_grad_carries, + *init_grad_additional_inputs, + ), + (*fw_carries, *ctx.additional_inputs), + ), + [1, len(init_grad_carries), len(init_grad_additional_inputs)], + ) + return ( + None, + None, + None, + None, + *fill_none_with_masks(final_grad_carries, carries_tensor_masks), + *fill_none_with_masks( + final_grad_additional_inputs, additional_inputs_tensor_masks + ), + ) + + +while_loop_stack_output_op = WhileLoopStackOutputOp() + +while_loop_stack_output_op.py_impl(DispatchKey.CompositeExplicitAutograd)( + functools.partial(while_loop_dense, stack_output=True) +) + +while_loop_stack_output_op.py_impl(ProxyTorchDispatchMode)( + functools.partial(while_loop_tracing, stack_output=True) +) + +while_loop_stack_output_op.py_impl(FakeTensorMode)( + functools.partial(while_loop_fake_tensor_mode, stack_output=True) +) + +while_loop_stack_output_op.py_functionalize_impl( + functools.partial(while_loop_func, stack_output=True) +) + +while_loop_stack_output_op.py_autograd_impl( + autograd_not_implemented(while_loop_stack_output_op, deferred_error=True) +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/wrap.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/wrap.py new file mode 100644 index 0000000000000000000000000000000000000000..3a7f287425ae1462f338ba6e92f20898d7de050f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_higher_order_ops/wrap.py @@ -0,0 +1,375 @@ +# mypy: allow-untyped-defs +import inspect +import itertools +import logging +from typing import Any, Optional + +import torch +import torch.utils._pytree as pytree +from torch._C import DispatchKey +from torch._higher_order_ops.utils import redirect_to_mode, reenter_make_fx +from torch._logging import warning_once +from torch._ops import HigherOrderOperator +from torch.fx import GraphModule +from torch.fx.experimental.proxy_tensor import ProxyTorchDispatchMode, track_tensor_tree +from torch.types import _dtype +from torch.utils._debug_mode import DebugMode +from torch.utils.checkpoint import _CachedTorchDispatchMode, _CachingTorchDispatchMode + + +log = logging.getLogger(__name__) + +uid = itertools.count(1) + + +# Used for testing the HigherOrderOperator mechanism +class Wrap(HigherOrderOperator): + def __init__(self) -> None: + super().__init__("wrap") + + def __call__(self, func, *args, **kwargs): + # Dynamo already traces the body of HigherOrderOp beforehand when it + # so no need to trace into it. + import torch._dynamo # noqa: F401 + from torch._dynamo import disable + + @disable + def wrapper(): + result = func(*args, **kwargs) + return result + + return wrapper() + + +wrap = Wrap() + + +class InductorCompiledCode(HigherOrderOperator): + """ + Defines a HOP for wrapping inductor compiled functions as a callable. + When used with torch.compile via "wrap_inductor_compiled_regions", + this HOP will automatically be wrapped and redirect various torch dispatch modes. + """ + + def __init__(self) -> None: + super().__init__("inductor_compiled_code") + + def __call__(self, func, *args, **kwargs): + return super().__call__(func, *args, **kwargs) + + +inductor_compiled_code = InductorCompiledCode() +inductor_compiled_code.fallthrough(DispatchKey.AutogradCPU) +inductor_compiled_code.fallthrough(DispatchKey.AutogradCUDA) + + +@inductor_compiled_code.py_impl(DispatchKey.CompositeExplicitAutograd) +def inductor_compiled_code_impl(func, inputs): + return func(inputs) + + +redirect_to_mode(inductor_compiled_code, DebugMode) +redirect_to_mode(inductor_compiled_code, _CachingTorchDispatchMode) +redirect_to_mode(inductor_compiled_code, _CachedTorchDispatchMode) + + +class WrapWithSetGradEnabled(HigherOrderOperator): + def __init__(self) -> None: + super().__init__("wrap_with_set_grad_enabled") + + def __call__(self, enable_grad, wrapped_func, *args, **kwargs): + # Dynamo already traces the body of HigherOrderOp beforehand when it + # so no need to trace into it. + import torch._dynamo # noqa: F401 + from torch._dynamo import disable + + @disable + def wrapper(): + prev = torch.is_grad_enabled() + torch.set_grad_enabled(enable_grad) + res = wrapped_func(*args, **kwargs) + torch.set_grad_enabled(prev) + return res + + return wrapper() + + +wrap_with_set_grad_enabled = WrapWithSetGradEnabled() + + +class WrapWithAutocast(HigherOrderOperator): + def __init__(self): + super().__init__("wrap_with_autocast") + + def __call__( + self, + device_type: str, + dtype: Optional[_dtype], + enabled: bool, + cache_enabled: Optional[bool], + wrapped_func, + *args, + **kwargs, + ): + # Dynamo already traces the body of HigherOrderOp beforehand when it + # so no need to trace into it. + import torch._dynamo # noqa: F401 + from torch._dynamo import disable + + @disable + def wrapper(): + with torch.autocast(device_type, dtype, enabled, cache_enabled): + return wrapped_func(*args, **kwargs) + + return wrapper() + + +wrap_with_autocast = WrapWithAutocast() + + +# This HOP allows you to bypass dynamo tracing of the wrapper function while +# still tracing the inner function. +# Takes two callables: The first, `wrapper_fn`, accepts `inner_fn` and returns a +# callable with the same signature. The second is the `inner_fn` itself. Any +# extra *args and **kwargs are forwarded to `wrapper_fn(inner_fn)` when it is +# executed. +class DynamoBypassingWrapper(HigherOrderOperator): + def __init__(self): + super().__init__("dynamo_bypassing_wrapper") + + def __call__( + self, + wrapper_fn_or_key, + inner_fn, + *args, + **kwargs, + ): + # Dynamo already traces the body of HigherOrderOp beforehand when it + # so no need to trace into it. + import torch._dynamo # noqa: F401 + from torch._dynamo import disable + + is_compiling = isinstance(wrapper_fn_or_key, str) + if is_compiling: + assert isinstance(inner_fn, torch.fx.GraphModule) + wrapper_fn = inner_fn.meta[wrapper_fn_or_key] + else: + wrapper_fn = wrapper_fn_or_key + + @disable + def wrapper(): + return wrapper_fn(inner_fn)(*args, **kwargs) + + return wrapper() + + +dynamo_bypassing_wrapper = DynamoBypassingWrapper() + + +class WrapActivationCheckpoint(HigherOrderOperator): + """ + This operator is used to wrap torch.utils.checkpoint. This avoids + TorchDynamo to look into saved tensor hooks and directly passes the control + to AOT Autograd, which is ok with tracing saved tensor hooks. As a result of + AOT tracing torch.utils.checkpoint code, we have a backward graph with + recomputed forward nodes. + + However, we might deprecate this operator soon. The difficulty arises in the + functionalization of rng ops. Today, there are two different + functionalization of rng ops - one at AOT autograd and other at Inductor. + And they are difficult to map to each other. The rng states also complicate + pattern matching in Inductor. Due to the ease of implementation, we are + currently inclined towards functionalization at Inductor level, which means + that duplication/recomputation is done as a compiler pass in the + partitioners. See TagActivationCheckpoint for more information. + """ + + def __init__(self) -> None: + super().__init__("wrap_activation_checkpoint", cacheable=False) + + def __call__(self, function, *args, **kwargs): + # use_reentrant is set to False because this op is going to be traced. + # And we ensure that AOT Autograd traces through the non reentrant + # version of checkpointing. + import torch.fx.traceback as fx_traceback + from torch.fx import Interpreter + + kwargs["use_reentrant"] = False + kwargs["preserve_rng_state"] = False + # Using interpreter allows preservation of metadata through torch.compile stack. + with fx_traceback.preserve_node_meta(): + from torch.utils.checkpoint import checkpoint + + return checkpoint(Interpreter(function).run, *args, **kwargs) + + +wrap_activation_checkpoint = WrapActivationCheckpoint() + + +class TagActivationCheckpoint(HigherOrderOperator): + """ + This operator is supposed to be used only with torch.compile stack. This + accepts a Fx graph module which needs to be checkpointed. This operator adds + "recomputable" tag to the nodes of the Fx graph that should be recomputed. + + The goal is to: + 1. Avoid using Dynamo to trace through saved tensor hooks. + 2. For selective checkpointing case, let AOTAutograd trace through + saved tensor hooks but has special logic with TorchDispatchMode to override + the usual saved_tensor_hooks fn logic in order to tag the nodes. + 3. Rely on the partitioners to actually duplicate the nodes. + This sits well in the torch.compile stack, because by the time graph + reaches partitioner, inductor has already run its functionalization of rng + ops (by setting fixed seed for each random op, see `replace_random_passes`). + Therefore, the duplication of nodes, by design, respects the rng states in + the forward and recomputed forward in backward. + """ + + def __init__(self) -> None: + super().__init__("tag_activation_checkpoint", cacheable=False) + + @staticmethod + def divide_kwargs(kwargs): + """ + checkpoint fn can have mixed kwargs between checkpointed fn and + checkpoint fn itself. For example + >> def gn(x, y, z=None): + >> a = torch.matmul(x, y) + >> if z is not None: + >> return torch.matmul(a, z) + >> return a + >> def fn(x, y, z): + >> return torch.cos(checkpoint(gn, x, y, use_reentrant=False, z=z)) + In the above case, z belongs to checkpointed function gn, but + use_reentrant belongs to the checkpoint function. This function splits + the kwargs into checkpoint_kwargs and gmod_kwargs (or + checkpointed_fn_kwargs). + We do sorting to ensure same graph from run to run for better + debuggability. It is not required for correctness. + """ + from torch.utils.checkpoint import checkpoint + + ckpt_signature = inspect.signature(checkpoint) + checkpoint_keys = set() + for name in ckpt_signature.parameters: + if name in ("function", "args", "kwargs"): + continue + checkpoint_keys.add(name) + + # `preserve_rng_state` is not a regular kwarg + checkpoint_keys.add("preserve_rng_state") + + checkpoint_kwargs = { + name: kwargs[name] for name in kwargs if name in checkpoint_keys + } + gmod_kwargs = { + name: kwargs[name] for name in kwargs if name not in checkpoint_keys + } + return checkpoint_kwargs, gmod_kwargs + + @staticmethod + def tag_nodes(gmod, is_sac): + from torch.utils.checkpoint import CheckpointPolicy + + unique_graph_id = next(uid) + for node in gmod.graph.nodes: + if node.op in ("call_function", "call_method", "call_module"): + node.meta["ac_graph_id"] = unique_graph_id + if is_sac: + # For selective checkpointing, we will populate this tag later in _CachingTorchDispatchMode. + node.meta["recompute"] = None + else: + # Under vanilla activation checkpointing, all nodes should be recomputed. + node.meta["recompute"] = CheckpointPolicy.PREFER_RECOMPUTE + return gmod + + def __call__(self, gmod, *args, **kwargs): + dispatch_key_set = torch._ops._compute_keyset( + args, kwargs, self.non_fallthrough_keys + ) + dispatch_key = dispatch_key_set.highestPriorityTypeId() + if dispatch_key == torch._C.DispatchKey.PreDispatch: + return super().__call__(gmod, *args, **kwargs) + + return tag_activation_checkpoint_impl(gmod, *args, **kwargs) + + +tag_activation_checkpoint = TagActivationCheckpoint() + + +def tag_activation_checkpoint_impl(gmod, *args, **kwargs): + import torch.fx.traceback as fx_traceback + from torch.fx import Interpreter + + if "_checkpoint_context_fn" in gmod.meta: + warning_once( + log, + """ +Detected that context_fn is passed to torch.utils.checkpoint under torch.compile. +Please make sure the checkpointed region does not contain in-place ops (e.g. torch.relu_). +""", + ) + # use_reentrant is set to False because this op is going to be traced. + # And we ensure that AOT Autograd traces through the non reentrant + # version of checkpointing. + kwargs["use_reentrant"] = False + # preserve_rng_state is set to False because we want to prevent AOTAutograd from tracing through + # `torch.random.fork_rng` op (which is not supported yet under CUDA). + # This doesn't mean that we don't preserve RNG state. Instead, we will always preserve RNG state + # regardless of this flag (by doing RNG functionalization via `replace_random_passes` in Inductor + # instead of in AOTAutograd). + kwargs["preserve_rng_state"] = False + kwargs["context_fn"] = gmod.meta["_checkpoint_context_fn"] + # We first tag all nodes as "recompute" in this graph, and then we undo the "recompute" tag + # for specific nodes in _CachingTorchDispatchMode in torch/utils/checkpoint.py. + gmod = TagActivationCheckpoint.tag_nodes(gmod, is_sac=True) + # Using interpreter allows preservation of metadata through torch.compile stack. + with fx_traceback.preserve_node_meta(): + from torch.utils.checkpoint import checkpoint + + return checkpoint(Interpreter(gmod).run, *args, **kwargs) + else: + gmod = TagActivationCheckpoint.tag_nodes(gmod, is_sac=False) + # Using interpreter allows preservation of metadata through torch.compile stack. + # TODO: We want to use the same `checkpoint(Interpreter(gmod).run, *args, **kwargs)` here + # as the `context_fn != None` case, but that depends on in-place op support in TorchDispatchMode + torch.compile. + # (for details on in-place op issue, run `test_compile_selective_checkpoint_inplace_op` unit test) + with fx_traceback.preserve_node_meta(): + return Interpreter(gmod).run(*args) + + +@tag_activation_checkpoint.py_impl(ProxyTorchDispatchMode) +def proxy_mode_key( + proxy_mode: ProxyTorchDispatchMode, + gmod: GraphModule, + *args: Any, + **kwargs: Any, +) -> tuple[torch.Tensor]: + import torch.fx.traceback as fx_traceback + from torch.fx import Interpreter + + assert proxy_mode.pre_dispatch, ( + "post-dispatch mode should have inlined in the Autograd key" + ) + example_out = tag_activation_checkpoint(gmod, *args, **kwargs) + proxy_args = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, args) # type: ignore[union-attr] + proxy_kwargs = pytree.tree_map(proxy_mode.tracer.unwrap_proxy, kwargs) # type: ignore[union-attr] + qualname = proxy_mode.tracer.get_fresh_qualname("wrap_body") # type: ignore[union-attr] + + # TODO (tmanlaibaatar) don't we need flat_apply here?? + # Dynamo already traced the gmod body without kwargs + flat_args, _ = pytree.tree_flatten(args) + with fx_traceback.preserve_node_meta(): + gmod_aten = reenter_make_fx(Interpreter(gmod).run)(*flat_args) + gmod_aten.meta["_checkpoint_context_fn"] = gmod.meta["_checkpoint_context_fn"] + proxy_mode.tracer.root.register_module(qualname, gmod_aten) # type: ignore[union-attr] + proxy_gmod = proxy_mode.tracer.unwrap_proxy(gmod_aten) # type: ignore[union-attr, call-overload] + out_proxy = proxy_mode.tracer.create_proxy( + "call_function", + tag_activation_checkpoint, + (proxy_gmod, *proxy_args), + proxy_kwargs, + ) + return track_tensor_tree( + example_out, out_proxy, constant=None, tracer=proxy_mode.tracer + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8d90efa40e58841a11a25569ca6722b791894999 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/__init__.py @@ -0,0 +1,55 @@ +# mypy: allow-untyped-defs + +import torch._C._lazy +from torch.utils._pytree import tree_flatten, tree_unflatten + +from .closure import add_step_closure, run_step_closures + + +def mark_step(device: str = "", wait=False): + """Triggers a mark step, which amounts to + - collecting a group of 'live' lazy tensors to index into the compilation cache + (lowering/compiling their IR graphs if not cached) + - kicking off execution of the compiled function + - (optionally, wait=True) waiting for cpu-side execution to complete (does not sync the accelerator) + """ + # TODO(whc) expand this to include backend hooks and align with XLA backend needs + torch._C._lazy._mark_step(device, [], wait=wait) + + run_step_closures() + + +def wait_device_ops(devices=None): + """Waits for all the async operations on the given devices to complete. + Args: + devices (string..., optional): The devices whose async ops need to be waited + for. If empty, all the local devices will be waited for. + """ + if devices is None: + devices = [] + torch._C._lazy._wait_device_ops(devices=devices) + + +def sync_multi(tensors, devices): + """ + Sync the list of lazy tensors so there IR get lowered for the activate backend + and the compiled computation graph get cached. + """ + torch._C._lazy._sync_multi(tensors, devices) + + +def get_tensor_id(tensor): + """Return a unique id of the lazy tensor maintained by LTC""" + return torch._C._lazy._get_tensor_id(tensor) + + +def to_cpu(tensors, devices=None): + devices = devices or ["lazy"] + + flattened, spec = tree_flatten(tensors) + sync_multi(flattened, devices) + return tree_unflatten([t.to("cpu") for t in flattened], spec) + + +def save(tensors, *args, **kwargs): + torch.save(to_cpu(tensors), *args, **kwargs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/closure.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/closure.py new file mode 100644 index 0000000000000000000000000000000000000000..bbe1a43bb12e26c1a51910f124b3b4d9e60958ad --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/closure.py @@ -0,0 +1,137 @@ +# mypy: allow-untyped-defs +import os +import threading +from queue import Empty as EmptyQueue, Queue + +from torch._lazy.device_context import get_device_context + + +class ClosureHandler: + def __init__(self) -> None: + pass + + def run(self, closure): + """Run closure function + + Args: + closure: callable function to run + """ + closure() + + def __call__(self, closures): + for closure in closures: + self.run(closure) + + +class AsyncClosureHandler(ClosureHandler): + """Handler for Asynchronous Step Closures + Args: + max_queue_size: The maximum length of the closure queue after which + the training loop will block until closures are evaluated. + By default, a reasonable limit of a maximum of 100 on the queue. + This value can be set using the `XLA_MAX_ASYNC_QUEUE` environment + variable. + """ + + def __init__(self, max_queue_size=100): + super().__init__() + self._closure_queue: Queue = Queue( + int(os.environ.get("LTC_MAX_ASYNC_QUEUE", max_queue_size)) + ) + self._closure_exception: Queue = Queue() + self._closure_lock = threading.Lock() + self._closure_event_loop_finished = threading.Event() + self._closure_event_loop = None + + def start_event_loop(self): + """Start closure event loop if not started""" + if self._closure_event_loop is None: + + def event_loop(): + # Run loop until closure event is set and closure queue is empty + while True: + try: + closure = self._closure_queue.get(block=True, timeout=3) + closure() + self._closure_queue.task_done() + except EmptyQueue: + with self._closure_lock: + if self._closure_queue.empty(): + self._closure_event_loop_finished.set() + return + except Exception as e: + self._closure_exception.put(e) + return + + self._closure_event_loop = threading.Thread( + target=event_loop + ) # pyrefly: ignore [bad-assignment] + self._closure_event_loop.start() # pyrefly: ignore [missing-attribute] + + def run(self, closure): + with self._closure_lock: + self._closure_queue.put(closure, block=True) + if ( + self._closure_event_loop is None + or not self._closure_event_loop.is_alive() + ): + try: + e = self._closure_exception.get(block=False) + raise RuntimeError( + "Cannot run asynchronous closure due to previously raised exception" + ) from e + except EmptyQueue: + self._closure_event_loop = None + self.start_event_loop() + + +def add_step_closure(closure, args=(), run_async=False): + """Adds a closure to the list of the ones to be run at the end of the step. + Many times during model training there is the need to print/report (print to + console, post to tensorboard, etc...) information which require the content of + intermediary tensors to be inspected. + Inspecting different tensors content in different points of the model code + requires many executions and typically causes performance issues. + Adding a step closure will ensure that it will be run after the barrier, when + all the live tensors will be already materialized to device data. + Live tensors which will include the ones captured by the closure arguments. + So using `add_step_closure()` will ensure a single execution will be + performed, even when multiple closures are queued, requiring multiple tensors + to be inspected. + Step closures will be run sequentially in the order they have been queued. + Note that even though using this API the execution will be optimized, it is + advised to throttle the printing/reporting events once every N steps. + Args: + closure (callable): The function to be called. + args (tuple): The arguments to be passed to the closure. + run_async: If True, run the closure asynchronously. + """ + devctx = get_device_context() + closures_type = "async_step_closures" if run_async else "step_closures" + step_closures = getattr(devctx, closures_type, None) + if step_closures is None: + step_closures = [] + setattr(devctx, closures_type, step_closures) + step_closures.append(lambda a=args: closure(*a)) + + +def run_step_closures(): + devctx = get_device_context() + async_step_closures = getattr(devctx, "async_step_closures", None) + if async_step_closures is not None: + devctx.async_step_closures = [] # type: ignore[attr-defined] + async_closure_handler = getattr(devctx, "async_closure_handler", None) + if async_closure_handler is None: + async_closure_handler = AsyncClosureHandler() + devctx.async_closure_handler = async_closure_handler # type: ignore[attr-defined] + async_closure_handler(async_step_closures) + + step_closures = getattr(devctx, "step_closures", None) + if step_closures is not None: + devctx.step_closures = [] # type: ignore[attr-defined] + closure_handler = getattr(devctx, "closure_handler", None) + if closure_handler is None: + closure_handler = ClosureHandler() + devctx.closure_handler = closure_handler # type: ignore[attr-defined] + closure_handler(step_closures) + return devctx diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/computation.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/computation.py new file mode 100644 index 0000000000000000000000000000000000000000..17a61e36cb9f2a46461d14caa3c1a3ff6e8c9094 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/computation.py @@ -0,0 +1,27 @@ +# mypy: allow-untyped-defs +import torch._C._lazy +import torch._C._lazy_ts_backend + + +def get_tensors_ts_device_data_node(tensors): + """Return tensor ids and eager tensors for DeviceData nodes in the + IR for the passed in lazy tensors. + + TODO: This API is currently ts backend specific. We are working on + generalizing it to all backends including XLA. + """ + return torch._C._lazy_ts_backend._get_tensors_ts_device_data_node(tensors) + + +def get_graph_hash(tensors): + """Return the graph hash for the passed in lazy tensors""" + return torch._C._lazy._get_graph_hash(tensors) + + +def run_cached_graph(hash_str, graph_inputs): + """Running the cached computation graph with the given inputs + + TODO: This API is currently ts backend specific. We are working on + generalizing it to all backends including XLA. + """ + return torch._C._lazy_ts_backend._run_cached_graph(hash_str, graph_inputs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/config.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/config.py new file mode 100644 index 0000000000000000000000000000000000000000..46839094d89a04568dd602f4cdb532450f9fa130 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/config.py @@ -0,0 +1,16 @@ +import torch._C._lazy + + +def get_force_fallback() -> str: + """Get the config used to force LTC fallback""" + return torch._C._lazy._get_force_fallback() + + +def set_force_fallback(configval: str) -> None: + """Set the config used to force LTC fallback""" + torch._C._lazy._set_force_fallback(configval) + + +def set_reuse_ir(val: bool) -> None: + """Set the config to reuse IR nodes for faster tracing""" + torch._C._lazy._set_reuse_ir(val) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/debug.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/debug.py new file mode 100644 index 0000000000000000000000000000000000000000..84534fb232509f0c9bbe722820bd1ae649d53e07 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/debug.py @@ -0,0 +1,22 @@ +# mypy: allow-untyped-defs +import torch._C._lazy + + +def render_ir_graph(tensors): + """Return a text dump of the LTC IR graph in dot format for the tensors. + The text can be processed by tools like dot to be rendered in pdf,png etc.""" + return torch._C._lazy._get_tensors_dot(tensors) + + +def dump_ir(tensors, ir_format): + """Return a dump of the tensors in the specified format. + Valid format are + - text: for LTC IR + - backend: for the activate backend IR + """ + if ir_format == "text": + return torch._C._lazy._get_tensors_text(tensors) + elif ir_format == "backend": + return torch._C._lazy._get_tensors_backend(tensors) + else: + raise RuntimeError(f"Unrecognized IR format: {ir_format}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/device_context.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/device_context.py new file mode 100644 index 0000000000000000000000000000000000000000..49f33cf7f7c6d64095c714fa4d39cee4a2aa508a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/device_context.py @@ -0,0 +1,25 @@ +import threading +from typing import Any, Optional + +import torch._C._lazy + + +class DeviceContext: + _CONTEXTS: dict[str, Any] = {} + _CONTEXTS_LOCK = threading.Lock() + + def __init__(self, device: str) -> None: + self.device = device + + +def get_device_context(device: Optional[str] = None) -> DeviceContext: + if device is None: + device = torch._C._lazy._get_default_device_type() + else: + device = str(device) + with DeviceContext._CONTEXTS_LOCK: + devctx = DeviceContext._CONTEXTS.get(device, None) + if devctx is None: + devctx = DeviceContext(device) + DeviceContext._CONTEXTS[device] = devctx + return devctx diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/extract_compiled_graph.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/extract_compiled_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..78455ebc964bf4ee5059228ea68e53df00fbf744 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/extract_compiled_graph.py @@ -0,0 +1,226 @@ +# mypy: allow-untyped-defs +import copy +import dataclasses +import itertools +import os +from collections.abc import Callable +from typing import Any + +import torch +import torch._lazy as lazy +import torch._lazy.metrics as metrics +from torch import fx +from torch._lazy import computation, debug as lazy_debug +from torch._lazy.tensor_factory_functions import tensor_factory_functions + + +debug = os.environ.get("debug_extract_compiled_graph") is not None + + +@dataclasses.dataclass +class GraphInputMatcher: + """ + The GraphInputMatcher class setup the graph inputs for future calls after lazy tracing. + Specifically, those graph inputs corresponding to method parameters should be replaced with the + arguments for the current call. + + tensor_id_to_arg_idx maps the tensor id to the parameter index. + graph_input_tensor_ids, graph_input_ivalues list the tensor_id and ivalue for each of the + TS/XLA graph inputs. + """ + + tensor_id_to_arg_idx: dict[int, int] + graph_input_tensor_ids: list[int] + # there are 2 categories of graph_input_tensors. + # Category 1: those whose id are not found in tensor_id_to_arg_idx. These are + # most likely const tensors and we can get its content from graph_input_tensors + # Category 2: those whose id are found in tensor_id_to_arg_idx. We should get + # the tensor from method arguments + graph_input_ivalues: list[Any] + + # get the real graph input tensors + def __call__(self, args): + real_input = [] + for tensor_id, traced_ivalue in zip( + self.graph_input_tensor_ids, self.graph_input_ivalues + ): + arg_idx = self.tensor_id_to_arg_idx.get(tensor_id, None) + if arg_idx is None: + inp = traced_ivalue + else: + inp = args[arg_idx] + real_input.append(inp) + return real_input + + +class ReturnValueHandler: + r""" + When ltc_sync_multi is called on multi tensors, the compiled graph + will contain output only for unique tensors - if a tensor appears multiple + times in the input to _ltc_sync_multi, only the first occurrence matters. + + However from python level, we still expect multi tensors returned with duplication + even if the TS graph dedup the output. e.g. for method: + + def forward(self, a): + return a, a + + the TS graph captured by LTC will return a single tensor, but Python method expects 2. + + This class dedup the lazy tensors first to get the index that will be used + to duplicate the eager tensors later. + """ + + def __init__(self, lazy_out_list): + self.index: list[list[int]] = [] + self.total_count = len(lazy_out_list) + + tensor_id_to_idx: dict[int, int] = {} + for dup_idx, lazy_tensor in enumerate(lazy_out_list): + uniq_idx = tensor_id_to_idx.get(id(lazy_tensor), None) + if uniq_idx is not None: + self.index[uniq_idx].append(dup_idx) + else: + uniq_idx = len(self.index) + self.index.append([dup_idx]) + tensor_id_to_idx[id(lazy_tensor)] = uniq_idx + + def duplicate_eager_tensors(self, eager_tensor_list): + duplicated_list = [None] * self.total_count + assert len(eager_tensor_list) == len(self.index) + + for uniq_idx, eager_tensor in enumerate(eager_tensor_list): + for dup_idx in self.index[uniq_idx]: + duplicated_list[dup_idx] = eager_tensor + return duplicated_list + + +def force_lazy_device(model: fx.GraphModule): + """ + Factory methods in a Fx graph may create tensors for a specific eager devices. + If we take no actions, those eager tensors will be mixed with lazy tensors and + cause crash. This method overwrite those eager device to lazy device. + """ + + def tolazydevice(dev): + if isinstance(dev, torch.device): + return torch.device("lazy", index=dev.index) + return dev + + def hasDeviceArg(args, kwargs): + return any( + isinstance(arg, torch.device) + for arg in itertools.chain(args, kwargs.values()) + ) + + for nd in model.graph.nodes: + nd.args = tuple(tolazydevice(arg) for arg in nd.args) + nd.kwargs = {k: tolazydevice(v) for k, v in nd.kwargs.items()} + + # For torchbench like yolov3, hf_Bart, dynamo generates Fx graph that return + # eager tensors on the default device + # (check https://gist.github.com/shunting314/eabdf6c769c59bc384469717b8f9bb7f for yolove, + # and https://gist.github.com/shunting314/8d5e2d9348a3258959d3954186c48814 for hf_Bart). + # To force those tensors on the lazy device, we can not simply override + # the device argument since there is no explicit device argument. + # What we are doing here is, for the list of covered tensor factory methods + # we add a lazy device argument explicitly. + # + # TODO: This solution is no ideal since we may miss some factory methods. In future + # when we support lazy mode, this method can be replaced by that. + if nd.target in tensor_factory_functions and not hasDeviceArg( + nd.args, nd.kwargs + ): + kwargs = dict(nd.kwargs) # nd.kwargs is immutable. make a mutable copy. + kwargs["device"] = torch.device("lazy") + nd.kwargs = kwargs + + model.recompile() + + +def get_fallback_ops(): + fallback_ops = [] + for opname in metrics.counter_names(): + if "aten::" not in opname: + continue + val = int(metrics.counter_value(opname)) + if val > 0: + fallback_ops.append(f"{opname}={val}") + + return fallback_ops + + +def extract_compiled_graph(model: fx.GraphModule, example_inputs) -> Callable: + """ + Optimize an eager model with LTC and returns a wrapper to execute the + compiled graph directly without retracing. It depends on other mechanisms + like TorchDynamo guards to guarantee the returned wrapper is only called + when it's safe. + """ + lazy_args = [arg.to(device="lazy") for arg in example_inputs] + args_tensor_ids = [lazy.get_tensor_id(lazy_arg) for lazy_arg in lazy_args] + tensor_id_to_arg_idx = {tensor_id: i for i, tensor_id in enumerate(args_tensor_ids)} + lazy_model = copy.deepcopy(model).to(device=torch.device("lazy")) + force_lazy_device(lazy_model) + + # This line executes lazy tracing and enable us extracting compiled graph later + metrics.reset() + lazy_out = lazy_model(*lazy_args) + fallback_ops = get_fallback_ops() + metrics.reset() + + if len(fallback_ops) > 0: + raise RuntimeError( + f"Fail to extract the compiled graph because of fallback: {','.join(fallback_ops)}" + ) + + if not isinstance(lazy_out, (tuple, list)): + lazy_out = (lazy_out,) + + args_and_out = tuple(lazy_args) + tuple(lazy_out) + return_value_handler = ReturnValueHandler(args_and_out) + if debug: + print("Fx code:\n", model.code) + print("LTC IR:", lazy_debug.dump_ir(args_and_out, "text")) + + # TODO: this part is TS backend specific for now and will be generalized to + # support XLA + ( + graph_input_tensor_ids, + graph_input_ivalues, + ) = computation.get_tensors_ts_device_data_node(args_and_out) + assert len(graph_input_tensor_ids) == len(graph_input_ivalues) + graph_input_matcher = GraphInputMatcher( + tensor_id_to_arg_idx, graph_input_tensor_ids, graph_input_ivalues + ) + + graph_hash = computation.get_graph_hash(args_and_out) + + if debug: + print("graph_hash", graph_hash) + print(f"args_tensor_ids {args_tensor_ids}") + print("tensor ids from device data:", graph_input_tensor_ids) + + # sync the list of output tensors so the computation graph for these + # tensors will be cached. Those computation graphs can be retrieved + # by graph hash later. + lazy.sync_multi(args_and_out, []) + + def optimized_mod(*args): + if len(args_and_out) == 0: + return () + graph_input = graph_input_matcher(args) + res = return_value_handler.duplicate_eager_tensors( + computation.run_cached_graph(graph_hash, graph_input) + ) + + assert len(res) == len(args_and_out) + for i, arg in enumerate(args): + # only copy those tensors that get inplace updated + if arg is not res[i]: + arg.copy_(res[i]) + + # skip the args + return res[len(args) :] + + return optimized_mod diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/ir_cache.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/ir_cache.py new file mode 100644 index 0000000000000000000000000000000000000000..a6e654566f29bce166eb52e721b694f3b1f7862b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/ir_cache.py @@ -0,0 +1,14 @@ +# mypy: allow-untyped-defs +import torch._C._lazy + + +def dump(dot_file_name: str): + """Dump TrieCache in the dot format""" + return torch._C._lazy._dump_ir_cache(dot_file_name) + + +def reset(): + """Clear TrieCache. This is needed in testing to avoid + node reusing between different tests. + """ + return torch._C._lazy._clear_ir_cache() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/metrics.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..3f676ec1f8ae022636a6021eeeb97bfb6032432f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/metrics.py @@ -0,0 +1,22 @@ +# mypy: allow-untyped-defs +import torch._C._lazy + + +def reset(): + """Resets all metric counters.""" + torch._C._lazy._reset_metrics() + + +def counter_names(): + """Retrieves all the currently active counter names.""" + return torch._C._lazy._counter_names() + + +def counter_value(name: str): + """Return the value of the counter with the specified name""" + return torch._C._lazy._counter_value(name) + + +def metrics_report(): + """Return the combined (lazy core and backend) metric report""" + return torch._C._lazy._metrics_report() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/tensor_factory_functions.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/tensor_factory_functions.py new file mode 100644 index 0000000000000000000000000000000000000000..3b8ddc8b11c7e036ba6beac440d04eb1835b26d4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/tensor_factory_functions.py @@ -0,0 +1,49 @@ +import torch + + +""" +tensor_factory_functions defines the list of torch functions that create tensors. +The list is grabbed by searching thru native_functions.yaml by the following +regular expression: + + cat native_functions.yaml | grep 'func:' | grep -v "Tensor.*->" | grep "[-]>.*Tensor" + +It's possible that new tensor factory functions are added making this list stale. +Use at your own risk or regenerate the list. +""" +tensor_factory_functions = ( + torch._cudnn_init_dropout_state, + torch.arange, + torch.bartlett_window, + torch.blackman_window, + torch._empty_affine_quantized, + torch.empty_strided, + torch.eye, + torch.full, + torch.from_file, + torch.hann_window, + torch.hamming_window, + torch.kaiser_window, + torch.linspace, + torch.logspace, + torch.ones, + torch.scalar_tensor, + torch.rand, + torch.randint, + torch.randn, + torch.randperm, + torch.range, + torch._efficientzerotensor, + torch.zeros, + torch.tril_indices, + torch.triu_indices, + # Note: the following functions match the regular expression search above but + # they are not available in the torch module. Comment out. + # torch._sparse_coo_tensor_with_dims, + # torch.fft_fftfreq, + # torch.fft_rfftfreq, +) + ( + # torch.tensor is special since it's not in native_functions.yaml + # add it separately + torch.tensor, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/ts_backend.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/ts_backend.py new file mode 100644 index 0000000000000000000000000000000000000000..5c6ce13746e913db8e27081b8b0dcf8f4e0d4c88 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_lazy/ts_backend.py @@ -0,0 +1,7 @@ +# mypy: allow-untyped-defs +import torch._C._lazy_ts_backend + + +def init(): + """Initializes the lazy Torchscript backend""" + torch._C._lazy_ts_backend._init() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d0fdebb23bde98f0a04b1f4b902c06984a443ec8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/__init__.py @@ -0,0 +1,20 @@ +# Top level logging module for torch logging +# Design doc: https://docs.google.com/document/d/1ZRfTWKa8eaPq1AxaiHrq4ASTPouzzlPiuquSBEJYwS8/edit# +# Simple setup for onboarding (see above doc for more detail): +# 1. register any top-level log qualified name for your module in torch._logging._registrations (see there for examples) +# 2. register any artifacts ( below) in torch._logging._registrations +# a. call getArtifactLogger(__name__, ) at your logging site instead of the standard logger to log your artifact +import torch._logging._registrations + +from ._internal import ( + _init_logs, + DEFAULT_LOGGING, + dtrace_structured, + get_structured_logging_overhead, + getArtifactLogger, + hide_warnings, + LazyString, + set_logs, + trace_structured, + warning_once, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/_internal.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/_internal.py new file mode 100644 index 0000000000000000000000000000000000000000..23dc6f46576b5c23bb920cefbc94e4ae90a1ef22 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/_internal.py @@ -0,0 +1,1438 @@ +# mypy: allow-untyped-defs +import contextlib +import functools +import hashlib +import importlib.util +import itertools +import json +import logging +import os +import os.path +import pathlib +import re +import sys +import tempfile +import time +import warnings +from collections import defaultdict +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any, Generic, Optional, Union +from typing_extensions import ParamSpec +from weakref import WeakSet + +import torch._logging.structured +from torch._guards import CompileId +from torch._utils_internal import log_trace_structured_event +from torch.utils._traceback import CapturedTraceback + + +_P = ParamSpec("_P") + +log = logging.getLogger(__name__) + +# This is a synthetic logger which doesn't correspond to an actual logger, +# but handles all of our "tracing" logging, which is structured and doesn't go +# to stderr but always goes to a dedicated log file. We don't put these +# loggers in the classic module hierarchy, because we don't want a suppression +# of logs to also cause a trace to get suppressed (traces typically are not +# collected, unless we are in prod, in which case they always are collected.) +# +# TODO: Maybe we should allow for some sub-hierarchy so you can control which +# traces you want to collect, for performance reasons. +# +# See https://docs.google.com/document/d/1CX_hJ0PNy9f3R1y8TJrfkSeLkvGjjjLU84BSXgS2AZ8/edit +trace_log = logging.getLogger("torch.__trace") + +DEFAULT_LOG_LEVEL = logging.WARNING +LOG_ENV_VAR = "TORCH_LOGS" +LOG_OUT_ENV_VAR = "TORCH_LOGS_OUT" +LOG_FORMAT_ENV_VAR = "TORCH_LOGS_FORMAT" +LOG_TRACE_ID_FILTER = "TORCH_LOGS_TRACE_ID_FILTER" +TRACE_ENV_VAR = "TORCH_TRACE" +DTRACE_ENV_VAR = "TORCH_DTRACE" + +LOG_TRACE_HANDLER: Optional["LazyTraceHandler"] = None + +GET_DTRACE_STRUCTURED = False + + +@dataclass +class LogRegistry: + # shorthand name to log qualified name + # Note: this only contains loggers registered + # from register_log + # e.g. "dynamo" -> "torch._dynamo" + log_alias_to_log_qnames: dict[str, list[str]] = field(default_factory=dict) + + # artifact logger qualified names, + # this is populated lazily, as calls to getArtifactLogger + # currently formatted as .__ + # e.g. "torch._dynamo.convert_frame.__guards" + artifact_log_qnames: set[str] = field(default_factory=set) + + # child logs of registered logs if specified via open + # registration by the user (ie placing "torch._dynamo.output_graph" in the env var) + # these need to be tracked so their levels can be reset properly + # e.g. "torch._dynamo.output_graph" + child_log_qnames: set[str] = field(default_factory=set) + + # artifact names, populated by register_artifact + # e.g. "guards" + artifact_names: set[str] = field(default_factory=set) + + # Artifacts that should be visible by default in the error message + visible_artifacts: set[str] = field(default_factory=set) + + # A short description of each artifact + artifact_descriptions: dict[str, str] = field(default_factory=dict) + + # artifacts which are not displayed unless explicitly named in the + # settings. Ex. output_code is NOT displayed even if the inductor + # log level is set to DEBUG. It must be explicitly named in the settings + off_by_default_artifact_names: set[str] = field(default_factory=set) + + # logging format string for artifacts + artifact_log_formatters: dict[str, logging.Formatter] = field(default_factory=dict) + + def is_artifact(self, name): + return name in self.artifact_names + + def is_log(self, alias): + return alias in self.log_alias_to_log_qnames + + # register a log with an alias + def register_log(self, alias, log_qnames: Union[str, list[str]]) -> None: + if isinstance(log_qnames, str): + log_qnames = [log_qnames] + self.log_alias_to_log_qnames[alias] = log_qnames + + # register an artifact name + def register_artifact_name( + self, name, description, visible, off_by_default, log_format + ) -> None: + self.artifact_names.add(name) + if visible: + self.visible_artifacts.add(name) + self.artifact_descriptions[name] = description + + # if off by default, don't enable it + # when log_name's log_level is set to DEBUG + if off_by_default: + self.off_by_default_artifact_names.add(name) + + if log_format is not None: + self.artifact_log_formatters[name] = logging.Formatter(log_format) + + # register the qualified name of an artifact log + # this is needed to know which logs need to be reset + # whenever the log_state is changed + def register_artifact_log(self, artifact_log_qname) -> None: + self.artifact_log_qnames.add(artifact_log_qname) + + def register_child_log(self, log_qname) -> None: + self.child_log_qnames.add(log_qname) + + # flattens all the qnames together (TODO: consider memoizing?) + def get_log_qnames(self) -> set[str]: + return set(itertools.chain.from_iterable(self.log_alias_to_log_qnames.values())) + + def get_artifact_log_qnames(self): + return set(self.artifact_log_qnames) + + def get_child_log_qnames(self): + return set(self.child_log_qnames) + + def is_off_by_default(self, artifact_qname): + return artifact_qname in self.off_by_default_artifact_names + + +@dataclass +class LogState: + # qualified log names -> currently set log level + log_qname_to_level: dict[str, str] = field(default_factory=dict) + + # the set of currently enabled artifacts + artifact_names: set[str] = field(default_factory=set) + + def enable_artifact(self, artifact_name) -> None: + self.artifact_names.add(artifact_name) + + def is_artifact_enabled(self, name): + return name in self.artifact_names + + def enable_log(self, log_qnames, log_level) -> None: + if isinstance(log_qnames, str): + log_qnames = [log_qnames] + for log_qname in log_qnames: + self.log_qname_to_level[log_qname] = log_level + + def get_log_level_pairs(self): + """Returns all qualified module names for which the user requested + explicit logging settings. + + .. warning: + + This function used to return all loggers, regardless of whether + or not the user specified them or not; it now only returns logs + which were explicitly mentioned by the user (and torch, which + always is implicitly requested when we initialize our logging + subsystem.) + """ + return self.log_qname_to_level.items() + + def clear(self) -> None: + self.log_qname_to_level.clear() + self.artifact_names.clear() + + +log_registry = LogRegistry() +log_state = LogState() + +# sample usage: torch._logging.set_logs(**torch._logging.DEFAULT_LOGGING) +DEFAULT_LOGGING = { + "dynamo": logging.INFO, + "aot": logging.INFO, + "inductor": logging.INFO, + "fsdp": logging.INFO, + "ddp_graphs": True, + "graph_breaks": True, + "guards": True, + "recompiles": True, + "dynamic": logging.INFO, +} + + +def set_logs( + *, + all: Optional[int] = None, + dynamo: Optional[int] = None, + aot: Optional[int] = None, + autograd: Optional[int] = None, + dynamic: Optional[int] = None, + inductor: Optional[int] = None, + distributed: Optional[int] = None, + c10d: Optional[int] = None, + ddp: Optional[int] = None, + fsdp: Optional[int] = None, + dtensor: Optional[int] = None, + onnx: Optional[int] = None, + bytecode: bool = False, + aot_graphs: bool = False, + aot_joint_graph: bool = False, + ddp_graphs: bool = False, + graph: bool = False, + graph_code: bool = False, + graph_code_verbose: bool = False, + graph_breaks: bool = False, + graph_sizes: bool = False, + guards: bool = False, + recompiles: bool = False, + recompiles_verbose: bool = False, + trace_source: bool = False, + trace_call: bool = False, + trace_bytecode: bool = False, + output_code: bool = False, + kernel_code: bool = False, + schedule: bool = False, + perf_hints: bool = False, + pre_grad_graphs: bool = False, + post_grad_graphs: bool = False, + ir_pre_fusion: bool = False, + ir_post_fusion: bool = False, + onnx_diagnostics: bool = False, + fusion: bool = False, + overlap: bool = False, + export: Optional[int] = None, + modules: Optional[dict[str, Union[int, bool]]] = None, + cudagraphs: bool = False, + sym_node: bool = False, + compiled_autograd: bool = False, + compiled_autograd_verbose: bool = False, + cudagraph_static_inputs: bool = False, + benchmarking: bool = False, + autotuning: bool = False, + graph_region_expansion: bool = False, + inductor_metrics: bool = False, + hierarchical_compile: bool = False, + compute_dependencies: bool = False, +) -> None: + """ + Sets the log level for individual components and toggles individual log + artifact types. + + .. warning:: This feature is a prototype and may have compatibility + breaking changes in the future. + + .. note:: The ``TORCH_LOGS`` environment variable has complete precedence + over this function, so if it was set, this function does nothing. + + A component is a set of related features in PyTorch. All of the log + messages emitted from a given component have their own log levels. If the + log level of a particular message has priority greater than or equal to its + component's log level setting, it is emitted. Otherwise, it is suppressed. + This allows you to, for instance, silence large groups of log messages that + are not relevant to you and increase verbosity of logs for components that + are relevant. The expected log level values, ordered from highest to lowest + priority, are: + + * ``logging.CRITICAL`` + * ``logging.ERROR`` + * ``logging.WARNING`` + * ``logging.INFO`` + * ``logging.DEBUG`` + * ``logging.NOTSET`` + + See documentation for the Python ``logging`` module for more information on + log levels: ``_ + + An artifact is a particular type of log message. Each artifact is assigned + to a parent component. A component can emit many different kinds of + artifacts. In general, an artifact is emitted if either its corresponding + setting in the argument list below is turned on or if its parent component + is set to a log level less than or equal to the log level of the artifact. + + Keyword args: + all (:class:`Optional[int]`): + The default log level for all components. Default: ``logging.WARN`` + + dynamo (:class:`Optional[int]`): + The log level for the TorchDynamo component. Default: ``logging.WARN`` + + aot (:class:`Optional[int]`): + The log level for the AOTAutograd component. Default: ``logging.WARN`` + + autograd (:class:`Optional[int]`): + The log level for autograd. Default: ``logging.WARN`` + + inductor (:class:`Optional[int]`): + The log level for the TorchInductor component. Default: ``logging.WARN`` + + dynamic (:class:`Optional[int]`): + The log level for dynamic shapes. Default: ``logging.WARN`` + + distributed (:class:`Optional[int]`): + Whether to log c10d communication operations and other debug info from PyTorch Distributed components. + Default: ``logging.WARN`` + + c10d (:class:`Optional[int]`): + Whether to log c10d communication operations related debug info in PyTorch Distributed components. + Default: ``logging.WARN`` + + ddp (:class:`Optional[int]`): + Whether to log debug info related to ``DistributedDataParallel``(DDP) from PyTorch Distributed components. + Default: ``logging.WARN`` + + fsdp (:class:`Optional[int]`): + Whether to log debug info related to ``FullyShardedDataParallel``(FSDP) in PyTorch Distributed components. + Default: ``logging.WARN`` + + dtensor (:class:`Optional[int]`): + Whether to log debug info related to ``DTensor``(DTensor) in PyTorch Distributed components. + Default: ``logging.WARN`` + + onnx (:class:`Optional[int]`): + The log level for the ONNX exporter component. Default: ``logging.WARN`` + + bytecode (:class:`bool`): + Whether to emit the original and generated bytecode from TorchDynamo. + Default: ``False`` + + aot_graphs (:class:`bool`): + Whether to emit the graphs generated by AOTAutograd. Default: ``False`` + + aot_joint_graph (:class:`bool`): + Whether to emit the joint forward-backward graph generated by AOTAutograd. Default: ``False`` + + ddp_graphs (:class:`bool`): + Whether to emit graphs generated by DDPOptimizer. Default: ``False`` + + graph (:class:`bool`): + Whether to emit the graph captured by TorchDynamo in tabular format. + Default: ``False`` + + graph_code (:class:`bool`): + Whether to emit the python source of the graph captured by TorchDynamo. + Default: ``False`` + + graph_code_verbose (:class:`bool`): + Whether to emit verbose/intermediate FX pass logs for graph code. Default: ``False`` + + graph_breaks (:class:`bool`): + Whether to emit the graph breaks encountered by TorchDynamo. + Default: ``False`` + + graph_sizes (:class:`bool`): + Whether to emit tensor sizes of the graph captured by TorchDynamo. + Default: ``False`` + + guards (:class:`bool`): + Whether to emit the guards generated by TorchDynamo for each compiled + function. Default: ``False`` + + recompiles (:class:`bool`): + Whether to emit a guard failure reason and message every time + TorchDynamo recompiles a function. Default: ``False`` + + recompiles_verbose (:class:`bool`): + Whether to emit all guard failure reasons when TorchDynamo recompiles + a function, even those that are not actually run. Default: ``False`` + + trace_source (:class:`bool`): + Whether to emit when TorchDynamo begins tracing a new line. Default: ``False`` + + trace_call (:class:`bool`): + Whether to emit detailed line location when TorchDynamo creates an FX node + corresponding to function call. Python 3.11+ only. Default: ``False`` + + trace_bytecode (:class:`bool`): + Whether to emit bytecode instructions and traced stack state as TorchDynamo + traces bytecode. Default: ``False`` + + output_code (:class:`bool`): + Whether to emit the TorchInductor output code on a per-graph basis. Default: ``False`` + + kernel_code (:class:`bool`): + Whether to emit the TorchInductor output code on a per-kernel bases. Default: ``False`` + + schedule (:class:`bool`): + Whether to emit the TorchInductor schedule. Default: ``False`` + + perf_hints (:class:`bool`): + Whether to emit the TorchInductor perf hints. Default: ``False`` + + pre_grad_graphs (:class:`bool`): + Whether to emit the graphs before inductor grad passes. Default: ``False`` + + post_grad_graphs (:class:`bool`): + Whether to emit the graphs generated by after post grad passes. Default: ``False`` + + ir_pre_fusion (:class:`bool`): + Whether to emit the graphs before inductor fusion passes. Default: ``False`` + + ir_post_fusion (:class:`bool`): + Whether to emit the graphs after inductor fusion passes. Default: ``False`` + + onnx_diagnostics (:class:`bool`): + Whether to emit the ONNX exporter diagnostics in logging. Default: ``False`` + + fusion (:class:`bool`): + Whether to emit detailed Inductor fusion decisions. Default: ``False`` + + overlap (:class:`bool`): + Whether to emit detailed Inductor compute/comm overlap decisions. Default: ``False`` + + sym_node (:class:`bool`): + Whether to emit debug info for various SymNode opterations. Default: ``False`` + + export (:class:`Optional[int]`): + The log level for export. Default: ``logging.WARN`` + + benchmarking (:class:`bool`): + Whether to emit detailed Inductor benchmarking information. Default: ``False`` + + modules (dict): + This argument provides an alternate way to specify the above log + component and artifact settings, in the format of a keyword args + dictionary given as a single argument. There are two cases + where this is useful (1) if a new log component or artifact has + been registered but a keyword argument for it has not been added + to this function and (2) if the log level for an unregistered module + needs to be set. This can be done by providing the fully-qualified module + name as the key, with the log level as the value. Default: ``None`` + + cudagraph_static_inputs (:class:`bool`): + Whether to emit debug info for cudagraph static input detection. Default: ``False`` + + autotuning (:class:`bool`): + Autotuning choice logs, such as kernel source, perf, and tuning parameters. Default: ``False`` + + graph_region_expansion (:class:`bool`): + Whether to emit the detailed steps of the duplicate graph region tracker expansion algorithm. Default: ``False`` + + inductor_metrics (:class:`bool`): + Whether to estimate the runtimes of the nodes in a graph and log them to the metrics table. Default: ``False`` + + hierarchical_compile (:class:`bool`): + Whether to emit debug info for hierarchical compilation. Default: ``False`` + + Example:: + + >>> # xdoctest: +SKIP + >>> import logging + + # The following changes the "dynamo" component to emit DEBUG-level + # logs, and to emit "graph_code" artifacts. + + >>> torch._logging.set_logs(dynamo=logging.DEBUG, graph_code=True) + + # The following enables the logs for a different module + + >>> torch._logging.set_logs(modules={"unregistered.module.name": logging.DEBUG}) + """ + # ignore if env var is set + if LOG_ENV_VAR in os.environ: + log.warning( + "Using TORCH_LOGS environment variable for log settings, ignoring call to set_logs" + ) + return + + log_state.clear() + + modules = modules or {} + + def _set_logs(**kwargs) -> None: + for alias, val in itertools.chain(kwargs.items(), modules.items()): # type: ignore[union-attr] + if val is None: + continue + + if log_registry.is_artifact(alias): + if not isinstance(val, bool): + raise ValueError( + f"Expected bool to enable artifact {alias}, received {val}" + ) + + if val: + log_state.enable_artifact(alias) + elif log_registry.is_log(alias) or alias in log_registry.child_log_qnames: + if val not in logging._levelToName: + raise ValueError( + f"Unrecognized log level for log {alias}: {val}, valid level values " + f"are: {','.join([str(k) for k in logging._levelToName])}" + ) + + log_state.enable_log( + log_registry.log_alias_to_log_qnames.get(alias, alias), val + ) + elif _is_valid_module(alias): + if not _has_registered_parent(alias): + log_registry.register_log(alias, alias) + else: + log_registry.register_child_log(alias) + log_state.enable_log( + log_registry.log_alias_to_log_qnames.get(alias, alias), val + ) + else: + raise ValueError( + f"Unrecognized log or artifact name passed to set_logs: {alias}" + ) + + _init_logs() + + _set_logs( + torch=all, + dynamo=dynamo, + aot=aot, + autograd=autograd, + inductor=inductor, + dynamic=dynamic, + bytecode=bytecode, + aot_graphs=aot_graphs, + aot_joint_graph=aot_joint_graph, + ddp_graphs=ddp_graphs, + distributed=distributed, + c10d=c10d, + ddp=ddp, + fsdp=fsdp, + dtensor=dtensor, + graph=graph, + graph_code=graph_code, + graph_code_verbose=graph_code_verbose, + graph_breaks=graph_breaks, + graph_sizes=graph_sizes, + guards=guards, + recompiles=recompiles, + recompiles_verbose=recompiles_verbose, + trace_source=trace_source, + trace_call=trace_call, + trace_bytecode=trace_bytecode, + output_code=output_code, + kernel_code=kernel_code, + schedule=schedule, + perf_hints=perf_hints, + pre_grad_graphs=pre_grad_graphs, + post_grad_graphs=post_grad_graphs, + ir_pre_fusion=ir_pre_fusion, + ir_post_fusion=ir_post_fusion, + onnx=onnx, + onnx_diagnostics=onnx_diagnostics, + fusion=fusion, + overlap=overlap, + sym_node=sym_node, + export=export, + cudagraphs=cudagraphs, + compiled_autograd=compiled_autograd, + compiled_autograd_verbose=compiled_autograd_verbose, + cudagraph_static_inputs=cudagraph_static_inputs, + benchmarking=benchmarking, + autotuning=autotuning, + graph_region_expansion=graph_region_expansion, + inductor_metrics=inductor_metrics, + hierarchical_compile=hierarchical_compile, + compute_dependencies=compute_dependencies, + ) + + +def get_loggers() -> list[logging.Logger]: + """ + Returns: a list of all registered loggers + """ + return [logging.getLogger(qname) for qname in log_registry.get_log_qnames()] + + +def register_log(setting_name, log_name) -> None: + """ + Enables a log to be controlled by the env var and user API with the setting_name + Args: + setting_name: the shorthand name used in the env var and user API + log_name: the log name that the setting_name is associated with + """ + log_registry.register_log(setting_name, log_name) + + +def register_artifact( + setting_name, description, visible=False, off_by_default=False, log_format=None +) -> None: + """ + Enables an artifact to be controlled by the env var and user API with name + Args: + setting_name: the shorthand name used in the env var and user API + description: A description of what this outputs + visible: Whether it gets suggested to users by default + off_by_default: whether this artifact should be logged when the ancestor loggers + are enabled at level DEBUG + """ + log_registry.register_artifact_name( + setting_name, description, visible, off_by_default, log_format + ) + + +def getArtifactLogger(module_qname, artifact_name) -> logging.Logger: + if artifact_name not in log_registry.artifact_names: + raise ValueError( + f"Artifact name: {repr(artifact_name)} not registered," + f"please call register_artifact({repr(artifact_name)}) in torch._logging.registrations." + ) + qname = module_qname + f".__{artifact_name}" + log = logging.getLogger(qname) + log.artifact_name = artifact_name # type: ignore[attr-defined] + log_registry.register_artifact_log(qname) + configure_artifact_log(log) + return log + + +INCR_VERBOSITY_CHAR = "+" +DECR_VERBOSITY_CHAR = "-" +VERBOSITY_REGEX = ( + "(" + + "|".join([re.escape(INCR_VERBOSITY_CHAR), re.escape(DECR_VERBOSITY_CHAR)]) + + "?)" +) + + +def configure_artifact_log(log) -> None: + # If the artifact is off by default, then it should only be logged when explicitly + # enabled; set propagate to False so that this artifact is not propagated + # to its ancestor logger + if log_registry.is_off_by_default(log.artifact_name): + log.propagate = False + + # enable artifact logging when explicitly enabled + if log_state.is_artifact_enabled(log.artifact_name): + log.setLevel(logging.DEBUG) + log.propagate = True + + +# match a comma separated list of loggable names (whitespace allowed after commas) +def _gen_settings_regex(): + return re.compile(r"((\+|-)?[\w\.]+,\s*)*(\+|-)?[\w\.]+?") + + +def _validate_settings(settings): + return re.fullmatch(_gen_settings_regex(), settings) is not None + + +def help_message(verbose=False): + def pad_to(s, length=30): + assert len(s) <= length + return s + " " * (length - len(s)) + + if verbose: + printed_artifacts = log_registry.artifact_names + else: + printed_artifacts = log_registry.visible_artifacts + if verbose: + heading = "All registered names" + else: + heading = "Visible registered names (use TORCH_LOGS='+help' for full list)" + lines = ( + ["all"] + + sorted(log_registry.log_alias_to_log_qnames.keys()) + + sorted( + [ + f"{pad_to(name)}\t{log_registry.artifact_descriptions[name]}" + for name in printed_artifacts + ] + ) + ) + setting_info = " " + "\n ".join(lines) + examples = """ +Examples: + TORCH_LOGS="+dynamo,aot" will set the log level of TorchDynamo to + logging.DEBUG and AOT to logging.INFO + + TORCH_LOGS="-dynamo,+inductor" will set the log level of TorchDynamo to + logging.ERROR and TorchInductor to logging.DEBUG + + TORCH_LOGS="aot_graphs" will enable the aot_graphs artifact + + TORCH_LOGS="+dynamo,schedule" will enable set the log level of TorchDynamo + to logging.DEBUG and enable the schedule artifact + + TORCH_LOGS="+some.random.module,schedule" will set the log level of + some.random.module to logging.DEBUG and enable the schedule artifact + + TORCH_LOGS_FORMAT="%(levelname)s: %(message)s" or any provided format + string will set the output format + Valid keys are "levelname", "message", "pathname", "levelno", "lineno", + "filename" and "name". + + TORCH_LOGS_OUT=/tmp/output.txt will output the logs to /tmp/output.txt as + well. This is useful when the output is long. +""" + msg = f""" +TORCH_LOGS Info +{examples} + +{heading} +{setting_info} +""" + return msg + + +def _invalid_settings_err_msg(settings, verbose=False): + valid_settings = ( + ["all"] + + list(log_registry.log_alias_to_log_qnames.keys()) + + list(log_registry.artifact_names) + ) + valid_settings = ", ".join(sorted(valid_settings)) + msg = f""" +Invalid log settings: {settings}, must be a comma separated list of fully +qualified module names, registered log names or registered artifact names. +For more info on various settings, try TORCH_LOGS="help" +Valid settings: +{valid_settings} +""" + return msg + + +def process_env_var_string_for_windows(env_var_str: str) -> str: + """ + When we setup logging config as guide: https://docs.pytorch.org/docs/stable/logging.html + Such as: + TORCH_LOGS="+schedule,+inductor,+output_code" + + On Linux, it shows as: + declare -x SSH_TTY="/dev/pts/0" + declare -x TERM="xterm" + declare -x TORCH_LOGS="+schedule,+inductor,+output_code" + declare -x USER="xu" + + On Windows, it shows as: + TORCHINDUCTOR_WINDOWS_TESTS=1 + TORCH_LOGS="+schedule,+inductor,+output_code" + UCRTVersion=10.0.22000.0 + + For Linux, it shows quotes by default, And Windows is not shows quotes. + Besides that, Windows would auto assemble quotes when env var processing. + On Linux, we will get variable: "+schedule,+inductor,+output_code" + On Windows, we will get variable: '"+schedule,+inductor,+output_code"' + + So, we need remove the outer quotes for Windows. + """ + _IS_WINDOWS = sys.platform == "win32" + + def remove_outer_quotes(s: str) -> str: + if len(s) >= 2 and ( + (s[0] == '"' and s[-1] == '"') or (s[0] == "'" and s[-1] == "'") + ): + return s[1:-1] + return s + + if _IS_WINDOWS: + env_var_str = remove_outer_quotes(env_var_str) + + return env_var_str + + +@functools.lru_cache +def _parse_log_settings(settings): + settings = process_env_var_string_for_windows(settings) + + if settings == "": + return {} + + if settings == "help": + raise ValueError(help_message(verbose=False)) + elif settings == "+help": + raise ValueError(help_message(verbose=True)) + if not _validate_settings(settings): + raise ValueError(_invalid_settings_err_msg(settings)) + + settings = re.sub(r"\s+", "", settings) + log_names = settings.split(",") + + def get_name_level_pair(name): + clean_name = name.replace(INCR_VERBOSITY_CHAR, "") + clean_name = clean_name.replace(DECR_VERBOSITY_CHAR, "") + + if name[0] == INCR_VERBOSITY_CHAR: + level = logging.DEBUG + elif name[0] == DECR_VERBOSITY_CHAR: + level = logging.ERROR + else: + level = logging.INFO + + return clean_name, level + + log_state = LogState() + + for name in log_names: + name, level = get_name_level_pair(name) + + if name == "all": + name = "torch" + + if log_registry.is_log(name): + assert level is not None + log_qnames = log_registry.log_alias_to_log_qnames[name] + log_state.enable_log(log_qnames, level) + elif log_registry.is_artifact(name): + log_state.enable_artifact(name) + elif _is_valid_module(name): + if not _has_registered_parent(name): + log_registry.register_log(name, name) + else: + log_registry.register_child_log(name) + log_state.enable_log(name, level) + else: + raise ValueError(_invalid_settings_err_msg(settings)) + + return log_state + + +def _is_valid_module(qname): + spec = importlib.util.find_spec(qname) + return spec is not None + + +def _update_log_state_from_env() -> None: + global log_state + log_setting = os.environ.get(LOG_ENV_VAR, None) + if log_setting is not None: + log_state = _parse_log_settings(log_setting) + + +def _has_registered_parent(log_qname) -> bool: + cur_log = logging.getLogger(log_qname) + + registered_log_qnames = log_registry.get_log_qnames() + + while cur_log.parent: + if cur_log.name in registered_log_qnames: + return True + cur_log = cur_log.parent + + return False + + +def make_module_path_relative(abs_path): + """ + Given an absolute filepath corresponding to a Python module which was + loaded via normal import mechanisms using sys.path, convert it into + a relative path relative to one of the Python search paths. + """ + + abs_path = pathlib.Path(abs_path).resolve() + + for path in sys.path: + try: + rel_path = abs_path.relative_to(path) + except ValueError: + continue + else: + return str(rel_path) + + return str(abs_path) + + +# apply custom formats to artifacts when necessary +class TorchLogsFormatter(logging.Formatter): + def __init__( + self, *, trace: bool = False, trace_id_filter: Optional[set[str]] = None + ) -> None: + super().__init__() + self._is_trace = trace + self._trace_id_filter = trace_id_filter + + def format(self, record): + artifact_name = getattr(logging.getLogger(record.name), "artifact_name", None) + if artifact_name is not None: + artifact_formatter = log_registry.artifact_log_formatters.get( + artifact_name, None + ) + if artifact_formatter is not None: + return artifact_formatter.format(record) + + record.message = record.getMessage() + record.asctime = self.formatTime(record, "%m%d %H:%M:%S") + + # exception handling - copied from logging.Formatter.format + s = record.message + if record.exc_info: + from torch._dynamo import config + + should_format_exc = config.verbose or artifact_name != "graph_breaks" + # Cache the traceback text to avoid converting it multiple times + # (it's constant anyway) + if should_format_exc: + if not record.exc_text: + record.exc_text = self.formatException(record.exc_info) + if record.exc_text: + if s[-1:] != "\n": + s = s + "\n" + s = s + record.exc_text + if record.stack_info: + if s[-1:] != "\n": + s = s + "\n" + s = s + self.formatStack(record.stack_info) + + record.rankprefix = "" + if not self._is_trace and dist.is_available() and dist.is_initialized(): + record.rankprefix = f"[rank{dist.get_rank()}]:" + + record.traceid = "" + if ( + not self._is_trace + and (trace_id := torch._guards.CompileContext.current_trace_id()) + is not None + ): + record.traceid = f" [{trace_id}]" + + glog_level_to_abbr = { + "DEBUG": "V", # V is for VERBOSE in glog + "INFO": "I", + "WARNING": "W", + "ERROR": "E", + "CRITICAL": "C", + } + + shortlevel = glog_level_to_abbr.get(record.levelname, record.levelname) + + record.artifactprefix = "" + if artifact_name is not None: + record.artifactprefix = f" [__{artifact_name}]" + + filepath = make_module_path_relative(record.pathname) + + if ( + self._trace_id_filter + and record.traceid.strip() not in self._trace_id_filter + ): + return "" + + prefix = ( + f"{record.rankprefix}{shortlevel}{record.asctime}.{int(record.msecs * 1000):06d} {record.process} " + f"{filepath}:" + f"{record.lineno}]{record.traceid}{record.artifactprefix}" + ) + if self._is_trace: + assert s == "" + try: + r = f"{prefix} {json.dumps(record.metadata)}" + except TypeError: + log.warning("failing metadata: %r", record.metadata) + raise + if record.payload is not None: + r += "".join(f"\n\t{l}" for l in record.payload.split("\n")) + return r + else: + lines = s.split("\n") + return "\n".join(f"{prefix} {l}" for l in lines) + + +def _default_formatter(): + fmt = os.environ.get(LOG_FORMAT_ENV_VAR, None) + trace_id_filter = { + item.strip() + for item in os.environ.get(LOG_TRACE_ID_FILTER, "").split(",") + if item.strip() + } + if fmt is None: + return TorchLogsFormatter(trace_id_filter=trace_id_filter) + else: + if fmt in ("short", "basic"): + fmt = logging.BASIC_FORMAT + return logging.Formatter(fmt) + + +DEFAULT_FORMATTER = _default_formatter() + + +def _setup_handlers(create_handler_fn, log) -> None: + debug_handler = _track_handler(create_handler_fn()) + debug_handler.setFormatter(DEFAULT_FORMATTER) + debug_handler.setLevel(logging.DEBUG) + log.addHandler(debug_handler) + + +handlers = WeakSet() # type: ignore[var-annotated] + + +# mark handlers that we've created +# so we don't modify user handlers +def _track_handler(handler): + handlers.add(handler) + return handler + + +def _is_torch_handler(handler): + return handler in handlers + + +# clears all torch handlers on specified loggers +def _clear_handlers(log) -> None: + to_remove = [handler for handler in log.handlers if _is_torch_handler(handler)] + for handler in to_remove: + log.removeHandler(handler) + + +def _reset_logs() -> None: + # reset all registered logs + for log_qname in log_registry.get_log_qnames(): + log = logging.getLogger(log_qname) + log.setLevel(logging.WARNING) + log.propagate = False + _clear_handlers(log) + + # reset all artifact and child logs + for artifact_log_qname in itertools.chain( + log_registry.get_artifact_log_qnames(), log_registry.get_child_log_qnames() + ): + log = logging.getLogger(artifact_log_qname) + log.setLevel(logging.NOTSET) + log.propagate = True + + trace_log.propagate = False + _clear_handlers(trace_log) + + +def _get_log_state(): + return log_state + + +def _set_log_state(state) -> None: + global log_state + log_state = state + + +def _init_logs(log_file_name=None) -> None: + global GET_DTRACE_STRUCTURED + + _reset_logs() + _update_log_state_from_env() + + out = os.environ.get(LOG_OUT_ENV_VAR, None) + if out is not None: + log_file_name = out + + # First, reset all known (registered) loggers to NOTSET, so that they + # respect their parent log level + for log_qname in log_registry.get_log_qnames(): + # But not the top level torch level: this defaults to WARNING so + # that our log messages don't leak to the lower levels + if log_qname == "torch": + continue + log = logging.getLogger(log_qname) + log.setLevel(logging.NOTSET) + + # Now, for all loggers which the user requested to have non-standard + # logging behavior, modify their log levels + for log_qname, level in log_state.get_log_level_pairs(): + log = logging.getLogger(log_qname) + log.setLevel(level) + + # Finally, setup handlers for all registered loggers + for log_qname in log_registry.get_log_qnames(): + log = logging.getLogger(log_qname) + _setup_handlers( + logging.StreamHandler, + log, + ) + + if log_file_name is not None: + _setup_handlers( + lambda: logging.FileHandler(log_file_name), + log, + ) + + # configure artifact loggers, note: this must happen last + # since the levels of ancestor loggers are taken into account + for artifact_log_qname in log_registry.get_artifact_log_qnames(): + log = logging.getLogger(artifact_log_qname) + configure_artifact_log(log) + + # Setup handler for the special trace_log, with different default + # configuration + trace_dir_name = os.environ.get(TRACE_ENV_VAR, None) + + if dtrace_dir_name := os.environ.get(DTRACE_ENV_VAR, None): + GET_DTRACE_STRUCTURED = True + trace_dir_name = dtrace_dir_name + + # This handler may remove itself if trace_dir_name is None and we are not + # actually in an FB environment. This allows us to defer actually + # initializing it until we actually need to log anything. This is + # important because JK initializes a C++ singleton, which will pork our + # process if we subsequently fork. + global LOG_TRACE_HANDLER + if LOG_TRACE_HANDLER is None: + LOG_TRACE_HANDLER = LazyTraceHandler(trace_dir_name) + # This log is ALWAYS at debug level. We will additionally test if there + # are any handlers before deciding to actually call logging on this. Do + # not manually call + trace_log.setLevel(logging.DEBUG) + trace_log_handler = _track_handler(LOG_TRACE_HANDLER) + trace_log_handler.setFormatter(TorchLogsFormatter(trace=True)) + trace_log.addHandler(trace_log_handler) + + +class LazyTraceHandler(logging.StreamHandler): + """Like FileHandler, but the file is allocated lazily only upon the first log message""" + + def __init__(self, root_dir: Optional[str]) -> None: + # This is implemented in the same way that delay is implemented on + # FileHandler + self.root_dir = root_dir + logging.Handler.__init__(self) + self.stream = None + self._builtin_open = open + + # cloned from FileHandler in cpython + def close(self) -> None: + self.acquire() + try: + try: + if self.stream: + try: + self.flush() + finally: + stream = self.stream + self.stream = None + if hasattr(stream, "close"): + stream.close() + finally: + # Issue #19523: call unconditionally to + # prevent a handler leak when delay is set + # Also see Issue #42378: we also rely on + # self._closed being set to True there + logging.StreamHandler.close(self) + finally: + self.release() + + def emit(self, record) -> None: + if self.stream is None: + if self.root_dir is None: + TRACE_LOG_DIR = "/logs" + + import torch.version as torch_version + + if ( + hasattr(torch_version, "git_version") + and os.getenv("MAST_HPC_JOB_NAME") is None + ): + log.info( + "LazyTraceHandler: disabled because not fbcode or conda on mast" + ) + elif not torch._utils_internal.justknobs_check("pytorch/trace:enable"): + log.info( + "LazyTraceHandler: disabled because justknobs_check('pytorch/trace:enable') returned False" + ) + elif not os.path.exists(TRACE_LOG_DIR): + log.info( + "LazyTraceHandler: disabled because %s does not exist", + TRACE_LOG_DIR, + ) + elif not os.access(TRACE_LOG_DIR, os.W_OK): + log.info( + "LazyTraceHandler: disabled because %s is not writeable", + TRACE_LOG_DIR, + ) + else: + self.root_dir = TRACE_LOG_DIR + + if self.root_dir is not None: + os.makedirs(self.root_dir, exist_ok=True) + ranksuffix = "" + if dist.is_available() and dist.is_initialized(): + ranksuffix = f"rank_{dist.get_rank()}_" + self.stream = tempfile.NamedTemporaryFile( # noqa: SIM115 + mode="w+", + suffix=".log", + prefix=f"dedicated_log_torch_trace_{ranksuffix}", + dir=self.root_dir, + delete=False, + ) + log.info("LazyTraceHandler: logging to %s", self.stream.name) + else: + # We go poof, remove and no-op + trace_log.removeHandler(self) + return + if self.stream: + super().emit(record) + + +@functools.cache +def warning_once(logger_obj, *args, **kwargs) -> None: + """ + This function is similar to `logger.warning()`, but will emit the warning with the same message only once + Note: The cache is for the function arguments, so 2 different callers using the same arguments will hit the cache. + The assumption here is that all warning messages are unique across the code. If they aren't then need to switch to + another type of cache that includes the caller frame information in the hashing function. + """ + logger_obj.warning(*args, **kwargs) + + +def safe_grad_filter(message, category, filename, lineno, file=None, line=None) -> bool: + return "The .grad attribute of a Tensor" not in str(message) + + +def user_warning_filter( + message, category, filename, lineno, file=None, line=None +) -> bool: + return category is not UserWarning + + +@contextlib.contextmanager +def hide_warnings(filter_fn=lambda *args, **kwargs: True): + """ + A context manager that temporarily suppresses warnings, + using public API: https://docs.python.org/3/library/warnings.html#warnings.showwarning. + + Useful to hide warnings without mutating warnings module state, see: + https://github.com/pytorch/pytorch/issues/128427#issuecomment-2161496162. + + NOTE: Warnings issued under this context will still be cached in the __warningregistry__ + and count towards the once/default rule. So you should NEVER use this on a user-land function. + + Filter must implement the showwarning API: + def filter_fn(message, category, filename, lineno, file=None, line=None) -> bool: + return True # show this warning entry + """ + prior = warnings.showwarning + + def _showwarning(*args, **kwargs): + if filter_fn(*args, **kwargs): + prior(*args, **kwargs) + + try: + warnings.showwarning = _showwarning + yield + finally: + warnings.showwarning = prior + + +class LazyString(Generic[_P]): + def __init__( + self, func: Callable[_P, str], *args: _P.args, **kwargs: _P.kwargs + ) -> None: + self.func = func + self.args = args + self.kwargs = kwargs + + def __str__(self) -> str: + return self.func(*self.args, **self.kwargs) + + +# Logs the time it takes to do structured logging by frame/compile id +# key is always {frame_id}_{frame_compile_id} +structured_logging_overhead: dict[str, float] = defaultdict(float) + + +def add_structured_logging_overhead(time_spent: float) -> None: + global structured_logging_overhead + key = None + if (trace_id := torch._guards.CompileContext.current_trace_id()) is not None: + frame_id = trace_id.compile_id.frame_id + frame_compile_id = trace_id.compile_id.frame_compile_id + # Why not trace_id.attempt, like structured logging? + # We aggregate across all attempts because + # a compilation metric is logged per successful attempt + key = f"{frame_id}_{frame_compile_id}" + # TODO: deal with structured logging that occurs outside of specific compile ids + # It's hard to figure out where we would log that if we want it in compilation metrics + # itself. + if key is not None: + key = str(key) + structured_logging_overhead[key] += time_spent + + +def get_structured_logging_overhead() -> Optional[float]: + key = None + if (trace_id := torch._guards.CompileContext.current_trace_id()) is not None: + frame_id = trace_id.compile_id.frame_id + frame_compile_id = trace_id.compile_id.frame_compile_id + key = f"{frame_id}_{frame_compile_id}" + if key is not None: + return structured_logging_overhead.get(key) + else: + return None + + +def trace_structured_artifact( + name: str, # this will go in metadata + encoding: str, + payload_fn: Callable[[], Optional[Union[str, object]]] = lambda: None, + compile_id: Optional[CompileId] = None, +) -> None: + trace_structured( + "artifact", + metadata_fn=lambda: { + "name": name, + "encoding": encoding, + }, + payload_fn=payload_fn, + compile_id=compile_id, + ) + + +def trace_structured( + name: str, + # NB: metadata expected to be dict so adding more info is forward compatible + # Tuple[str, int] is a special case for string interning + metadata_fn: Callable[[], Union[dict[str, Any], tuple[str, int]]] = dict, + *, + payload_fn: Callable[[], Optional[Union[str, object]]] = lambda: None, + suppress_context: bool = False, + expect_trace_id: bool = True, # Whether or not we expect to have a current trace id + record_logging_overhead: bool = True, # Whether or not to record the time spent on structured logging + compile_id: Optional[CompileId] = None, # Optional if unavailable in the trace +) -> None: + """ + metadata is an arbitrary JSON compatible struct, but it's expected to not be + too long (e.g., less than 1MB) + + payload is an arbitrary string, which can be arbitrarily long (but expected to have + newlines so no lines are too long) + """ + assert name not in [ + "rank", + "compiled_autograd_id", + "frame_id", + "frame_compile_id", + "attempt", + "severity", + "timestamp", + "pathname", + "thread", + ] + assert callable(metadata_fn), ( + f"metadata_fn should be callable, but got {type(metadata_fn)}" + ) + assert callable(payload_fn), ( + f"payload_fn should be callable, but got {type(payload_fn)}" + ) + # trace_log never propagates and is ALWAYS DEBUG, so also check that there + # are handlers instead of checking the log level + if trace_log.handlers: + start_time = time.time_ns() + record: dict[str, object] = {} + record[name] = metadata_fn() + if not suppress_context: + # TODO: Actually, the rank probably should just be emitted once at + # the top, and not repeatedly spammed in all the logs, since it + # never changes and we assume no interleaving + if dist.is_available() and dist.is_initialized(): + record["rank"] = dist.get_rank() + + trace_id = torch._guards.CompileContext.current_trace_id() + if expect_trace_id and trace_id is None and compile_id is None: + # Record the stack of the log call to better diagnose why we + # don't have a frame id for it + record["stack"] = torch._logging.structured.from_traceback( + CapturedTraceback.extract(skip=1).summary() + ) + else: + cid = trace_id.compile_id if trace_id else compile_id + if cid is not None: + if cid.compiled_autograd_id is not None: + record["compiled_autograd_id"] = cid.compiled_autograd_id + if cid.frame_id is not None: + record["frame_id"] = cid.frame_id + if cid.frame_compile_id is not None: + record["frame_compile_id"] = cid.frame_compile_id + if trace_id: + record["attempt"] = trace_id.attempt + + payload = payload_fn() + if payload is not None: + if not isinstance(payload, str): + if isinstance(payload, list): + # special case to look better + payload = "[\n" + ",\n".join(json.dumps(i) for i in payload) + "\n]" + else: + + def json_default(obj): + # Sets aren't json serializable + if isinstance(obj, set): + return list(obj) + raise TypeError( + f"Object of type {type(obj)} is not JSON serializable" + ) + + # force newlines so we are unlikely to overflow line limit + payload = json.dumps(payload, default=json_default, indent=0) + h = hashlib.md5(usedforsecurity=False) + h.update(payload.encode("utf-8")) + record["has_payload"] = h.hexdigest() + trace_log.debug( + "", extra={"metadata": record, "payload": payload}, stacklevel=2 + ) + log_trace_structured_event(name, record) + + if record_logging_overhead: + # Convert to seconds from nanoseconds, add it to the frame compile total + structured_logging_overhead_s = (time.time_ns() - start_time) / 1e9 + add_structured_logging_overhead(structured_logging_overhead_s) + + +def dtrace_structured( + name: str, + # NB: metadata expected to be dict so adding more info is forward compatible + # Tuple[str, int] is a special case for string interning + metadata_fn: Callable[[], Union[dict[str, Any], tuple[str, int]]] = dict, + *, + payload_fn: Callable[[], Optional[Union[str, object]]] = lambda: None, + suppress_context: bool = False, + expect_trace_id: bool = False, # Whether or not we expect to have a current trace id + record_logging_overhead: bool = True, # Whether or not to record the time spent on structured logging +) -> None: + """ + For logging more detailed information used for debugging. This may result in + the program becoming slow. + """ + if GET_DTRACE_STRUCTURED: + trace_structured( + name, + metadata_fn, + payload_fn=payload_fn, + suppress_context=suppress_context, + expect_trace_id=expect_trace_id, + record_logging_overhead=record_logging_overhead, + ) + + +import torch._guards +import torch._utils_internal +import torch.distributed as dist diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/_registrations.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/_registrations.py new file mode 100644 index 0000000000000000000000000000000000000000..f0077f0f9bb7d975d6ff7cd8e2efc4899509a39d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/_registrations.py @@ -0,0 +1,259 @@ +# flake8: noqa: B950 +from ._internal import register_artifact, register_log + + +DYNAMIC = [ + "torch.fx.experimental.symbolic_shapes", + "torch.fx.experimental.sym_node", + "torch.fx.experimental.recording", +] +DISTRIBUTED = [ + "torch.distributed", + "torch._dynamo.backends.distributed", + "torch.nn.parallel.distributed", +] + +register_log( + "async_compile", + [ + "torch._inductor.async_compile", + "torch._inductor.compile_worker.tracked_process_pool", + ], +) +register_log( + "cache", ("torch._inductor.remote_cache", "torch._inductor.fb.remote_cache") +) +register_log("dynamo", ["torch._dynamo", *DYNAMIC]) +register_log("fake_tensor", ["torch._subclasses.fake_tensor"]) +register_log("aot", ["torch._functorch.aot_autograd", "torch._functorch._aot_autograd"]) +register_log("autograd", "torch.autograd") +register_log("inductor", ["torch._inductor", "torch._inductor.cudagraph_trees"]) + +register_artifact( + "cudagraphs", + "Logs information from wrapping inductor generated code with cudagraphs.", +) + +register_log("dynamic", DYNAMIC) +register_log("torch", "torch") +register_log("distributed", DISTRIBUTED) +register_log( + "c10d", ["torch.distributed.distributed_c10d", "torch.distributed.rendezvous"] +) +register_log( + "ddp", ["torch.nn.parallel.distributed", "torch._dynamo.backends.distributed"] +) +register_log("pp", ["torch.distributed.pipelining"]) +register_log("fsdp", ["torch.distributed.fsdp", "torch.distributed._composable.fsdp"]) +register_log("dtensor", ["torch.distributed._tensor", "torch.distributed.tensor"]) +register_log("onnx", "torch.onnx") +register_log( + "export", + [ + "torch._dynamo", + "torch.export", + "torch.export.dynamic_shapes", + *DYNAMIC, + "torch._export.converter", + "torch._export.non_strict_utils", + "torch._export.serde.serialize", + "torch.fx.experimental.proxy_tensor", + ], +) + +register_artifact( + "guards", + "This prints the guards for every compiled Dynamo frame. It does not tell you where the guards come from.", + visible=True, +) +register_artifact("verbose_guards", "", off_by_default=True) +register_artifact( + "bytecode", + "Prints the original and modified bytecode from Dynamo. Mostly useful if you're debugging our bytecode generation in Dynamo.", + off_by_default=True, +) +register_artifact( + "graph", + "Prints the dynamo traced graph (prior to AOTDispatch) in a table. If you prefer python code use `graph_code` instead. ", +) +register_artifact("graph_code", "Like `graph`, but gives you the Python code instead.") +register_artifact( + "graph_code_verbose", + "Verbose FX pass logs, e.g. from tensorify_python_scalars and runtime_assert.", +) +register_artifact( + "graph_sizes", "Prints the sizes of all FX nodes in the dynamo graph." +) +register_artifact( + "trace_source", + "As we execute bytecode, prints the file name / line number we are processing and the actual source code. Useful with `bytecode`", +) +register_artifact( + "trace_call", + "Like trace_source, but it will give you the per-expression blow-by-blow if your Python is recent enough.", +) +register_artifact( + "trace_bytecode", + "As we trace bytecode, prints the instruction and the current stack.", +) +register_artifact( + "aot_graphs", + "Prints the FX forward and backward graph generated by AOTDispatch, after partitioning. Useful to understand what's being given to Inductor", + visible=True, +) +register_artifact( + "aot_joint_graph", + "Print FX joint graph from AOTAutograd, prior to partitioning. Useful for debugging partitioning", +) +register_artifact( + "aot_graphs_effects", + "Prints the FX forward and backward graph generated by AOTDispatch, useful for debugging effects processing.", + visible=True, +) +register_artifact( + "pre_grad_graphs", + "Prints the FX graph before inductor pre grad passes. Useful to understand what's being given to Inductor before grad passes", +) +register_artifact( + "post_grad_graphs", + "Prints the FX graph generated by post grad passes. Useful to understand what's being given to Inductor after post grad passes", +) +register_artifact( + "ir_pre_fusion", + "Prints the IR before inductor fusion passes.", + off_by_default=True, +) +register_artifact( + "ir_post_fusion", + "Prints the IR after inductor fusion passes.", + off_by_default=True, +) +register_artifact( + "compiled_autograd", + "Prints various logs in compiled_autograd, including but not limited to the graphs. Useful for debugging compiled_autograd.", + visible=True, +) +register_artifact( + "compiled_autograd_verbose", + "Will affect performance. Prints compiled_autograd logs with C++ info e.g. autograd node -> fx node mapping", + off_by_default=True, +) +register_artifact( + "ddp_graphs", + "Only relevant for compiling DDP. DDP splits into multiple graphs to trigger comms early. This will print each individual graph here.", +) +register_artifact( + "recompiles", + "Prints the reason why we recompiled a graph. Very, very useful.", + visible=True, +) +register_artifact( + "recompiles_verbose", + "Prints all guard checks that fail during a recompilation. " + "At runtime, Dynamo will stop at the first failed check for each failing guard. " + "So not all logged failing checks are actually ran by Dynamo.", + visible=True, + off_by_default=True, +) +register_artifact( + "graph_breaks", + "Prints whenever Dynamo decides that it needs to graph break (i.e. create a new graph). Useful for debugging why torch.compile has poor performance", + visible=True, +) +register_artifact( + "not_implemented", + "Prints log messages whenever we return NotImplemented in a multi-dispatch, letting you trace through each object we attempted to dispatch to", +) +register_artifact( + "output_code", + "Prints the code that Inductor generates (either Triton or C++)", + off_by_default=True, + visible=True, +) +register_artifact( + "kernel_code", + "Prints the code that Inductor generates (on a per-kernel basis)", + off_by_default=True, + visible=True, +) +register_artifact( + "schedule", + "Inductor scheduler information. Useful if working on Inductor fusion algo", + off_by_default=True, +) +register_artifact("perf_hints", "", off_by_default=True) +register_artifact("onnx_diagnostics", "", off_by_default=True) +register_artifact("compute_dependencies", "", off_by_default=True) +register_artifact( + "fusion", + "Detailed Inductor fusion decisions. More detailed than 'schedule'", + off_by_default=True, +) +register_artifact( + "loop_ordering", + "Logs related to loop ordering", + off_by_default=True, +) +register_artifact( + "loop_tiling", + "Logs related to loop ordering", + off_by_default=True, +) + +register_artifact( + "overlap", + "Detailed Inductor compute/comm overlap decisions", + off_by_default=True, +) +register_artifact( + "sym_node", + "Logs extra info for various SymNode operations", + off_by_default=True, +) +register_artifact( + "trace_shape_events", + "Logs traces for every ShapeEnv operation that we record for replay", + off_by_default=True, +) +register_artifact( + "cudagraph_static_inputs", + "Logs static inputs handling in dynamo, AOT, and cudagraphs", + off_by_default=True, +) +register_artifact( + "benchmarking", + "Detailed Inductor benchmarking information.", + off_by_default=True, +) +register_artifact( + "node_runtime_estimation", + "Node runtime estimation for compile-time optimization decisions.", + off_by_default=True, +) +register_artifact( + "autotuning", + "Autotuning choice logs, such as kernel source, perf, and tuning parameters.", + off_by_default=True, +) +register_artifact( + "graph_region_expansion", + "Logs detailed steps of the duplicate graph region tracker expansion algorithm", + off_by_default=True, +) + +register_artifact( + "inductor_metrics", + "Logs Inductor metrics, such as num_bytes, nodes_num_elem, node_runtimes", + off_by_default=True, +) +register_artifact( + "hierarchical_compile", + "Logs debug info for hierarchical compilation", + off_by_default=True, +) +register_artifact( + "annotation", + "Logs detailed steps of the creating annotation on graph nodes", + off_by_default=True, +) +register_artifact("custom_format_test_artifact", "Testing only", log_format="") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/scribe.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/scribe.py new file mode 100644 index 0000000000000000000000000000000000000000..2feb814d4a2c7ede43c590e96cfa723addd676b6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/scribe.py @@ -0,0 +1,63 @@ +from collections.abc import Callable +from typing import TypeAlias, Union + + +try: + from fbscribelogger import ( # type: ignore[import-untyped, import-not-found] + make_scribe_logger, + ) +except ImportError: + TAtom: TypeAlias = Union[int, float, bool, str] + TField: TypeAlias = Union[TAtom, list[TAtom]] + TLazyField: TypeAlias = Union[TField, Callable[[], TField]] + + def make_scribe_logger(name: str, thrift_src: str) -> Callable[..., None]: + def inner(**kwargs: TLazyField) -> None: + pass + + return inner + + +open_source_signpost = make_scribe_logger( + "TorchOpenSourceSignpost", + """ +struct TorchOpenSourceSignpostLogEntry { + + # The commit SHA that triggered the workflow, e.g., 02a6b1d30f338206a71d0b75bfa09d85fac0028a. Derived from GITHUB_SHA. + 4: optional string commit_sha; + + # Commit date (not author date) of the commit in commit_sha as timestamp, e.g., 1724208105. Increasing if merge bot is used, though not monotonic; duplicates occur when stack is landed. + 5: optional i64 commit_date; + + # The fully-formed ref of the branch or tag that triggered the workflow run, e.g., refs/pull/133891/merge or refs/heads/main. Derived from GITHUB_REF. + 6: optional string github_ref; + + # Indicates if branch protections or rulesets are configured for the ref that triggered the workflow run. Derived from GITHUB_REF_PROTECTED. + 7: optional bool github_ref_protected; + + # A unique number for each attempt of a particular workflow run in a repository, e.g., 1. Derived from GITHUB_RUN_ATTEMPT. + 8: optional string github_run_attempt; + + # A unique number for each workflow run within a repository, e.g., 19471190684. Derived from GITHUB_RUN_ID. + 9: optional string github_run_id; + + # A unique number for each run of a particular workflow in a repository, e.g., 238742. Derived from GITHUB_RUN_NUMBER. + 10: optional string github_run_number_str; + + # The name of the current job. Derived from JOB_NAME, e.g., linux-jammy-py3.8-gcc11 / test (default, 3, 4, linux.2xlarge). + 11: optional string job_name; + + # The GitHub user who triggered the job. Derived from GITHUB_TRIGGERING_ACTOR. + 12: optional string github_triggering_actor; + 13: optional string name; # Event name + 14: optional string parameters; # Parameters (JSON data) + 16: optional string subsystem; # Subsystem the event is associated with + + # The unit timestamp in second for the Scuba Time Column override + 17: optional i64 time; + + # The weight of the record according to current sampling rate + 18: optional i64 weight; +} +""", # noqa: B950 +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/structured.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/structured.py new file mode 100644 index 0000000000000000000000000000000000000000..e6dd36a6c69683370df9b9779f476ce880be6d72 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_logging/structured.py @@ -0,0 +1,109 @@ +""" +Utilities for converting data types into structured JSON for dumping. +""" + +import inspect +import os +import traceback +from collections.abc import Sequence +from typing import Any, Optional + +import torch._logging._internal + + +INTERN_TABLE: dict[str, int] = {} + + +DUMPED_FILES: set[str] = set() + + +def intern_string(s: Optional[str]) -> int: + if s is None: + return -1 + + r = INTERN_TABLE.get(s) + if r is None: + r = len(INTERN_TABLE) + INTERN_TABLE[s] = r + torch._logging._internal.trace_structured( + "str", lambda: (s, r), suppress_context=True + ) + return r + + +def dump_file(filename: str) -> None: + if "eval_with_key" not in filename: + return + if filename in DUMPED_FILES: + return + DUMPED_FILES.add(filename) + from torch.fx.graph_module import _loader + + torch._logging._internal.trace_structured( + "dump_file", + metadata_fn=lambda: { + "name": filename, + }, + payload_fn=lambda: _loader.get_source(filename), + ) + + +def from_traceback(tb: Sequence[traceback.FrameSummary]) -> list[dict[str, Any]]: + # dict naming convention here coincides with + # python/combined_traceback.cpp + r = [ + { + "line": frame.lineno, + "name": frame.name, + "filename": intern_string(frame.filename), + "loc": frame.line, + } + for frame in tb + ] + return r + + +def get_user_stack(num_frames: int) -> list[dict[str, Any]]: + from torch._guards import TracingContext + from torch.utils._traceback import CapturedTraceback + + user_tb = TracingContext.extract_stack() + if user_tb: + return from_traceback(user_tb[-1 * num_frames :]) + + tb = CapturedTraceback.extract().summary() + + # Filter out frames that are within the torch/ codebase + torch_filepath = os.path.dirname(inspect.getfile(torch)) + os.path.sep + for i, frame in enumerate(reversed(tb)): + if torch_filepath not in frame.filename: + # Only display `num_frames` frames in the traceback + filtered_tb = tb[len(tb) - i - num_frames : len(tb) - i] + return from_traceback(filtered_tb) + + return from_traceback(tb[-1 * num_frames :]) + + +def get_framework_stack( + num_frames: int = 25, cpp: bool = False +) -> list[dict[str, Any]]: + """ + Returns the traceback for the user stack and the framework stack + """ + from torch.fx.experimental.symbolic_shapes import uninteresting_files + from torch.utils._traceback import CapturedTraceback + + tb = CapturedTraceback.extract(cpp=cpp).summary() + tb = [ + frame + for frame in tb + if ( + ( + frame.filename.endswith(".py") + and frame.filename not in uninteresting_files() + ) + or ("at::" in frame.name or "torch::" in frame.name) + ) + ] + + return from_traceback(tb[-1 * num_frames :]) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ff60965d6067e1a8281b665f361adaab221cd4f7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/__init__.py @@ -0,0 +1,34 @@ +# mypy: ignore-errors + +from . import fft, linalg, random +from ._dtypes import * # noqa: F403 +from ._funcs import * # noqa: F403 +from ._getlimits import finfo, iinfo +from ._ndarray import ( + array, + asarray, + ascontiguousarray, + can_cast, + from_dlpack, + ndarray, + newaxis, + result_type, +) +from ._ufuncs import * # noqa: F403 +from ._util import AxisError, UFuncTypeError + + +from math import pi, e # usort: skip + + +all = all # noqa: PLW0127 +alltrue = all + +any = any # noqa: PLW0127 +sometrue = any + +inf = float("inf") +nan = float("nan") + +False_ = False +True_ = True diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_binary_ufuncs_impl.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_binary_ufuncs_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..68183fa7d6e965dc59cbb3fd0418d6455d693074 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_binary_ufuncs_impl.py @@ -0,0 +1,85 @@ +# mypy: ignore-errors + +"""Export torch work functions for binary ufuncs, rename/tweak to match numpy. +This listing is further exported to public symbols in the `torch._numpy/_ufuncs.py` module. +""" + +import torch +from torch import ( # noqa: F401 + add, + arctan2, + bitwise_and, + bitwise_left_shift as left_shift, + bitwise_or, + bitwise_right_shift as right_shift, + bitwise_xor, + copysign, + divide, + eq as equal, + float_power, + floor_divide, + fmax, + fmin, + fmod, + gcd, + greater, + greater_equal, + heaviside, + hypot, + lcm, + ldexp, + less, + less_equal, + logaddexp, + logaddexp2, + logical_and, + logical_or, + logical_xor, + maximum, + minimum, + multiply, + nextafter, + not_equal, + pow as power, + remainder, + remainder as mod, + subtract, + true_divide, +) + +from . import _dtypes_impl, _util + + +# work around torch limitations w.r.t. numpy +def matmul(x, y): + # work around: + # - RuntimeError: expected scalar type Int but found Double + # - RuntimeError: "addmm_impl_cpu_" not implemented for 'Bool' + # - RuntimeError: "addmm_impl_cpu_" not implemented for 'Half' + dtype = _dtypes_impl.result_type_impl(x, y) + is_bool = dtype == torch.bool + is_half = (x.dtype == torch.float16 or y.dtype == torch.float16) and ( + x.is_cpu or y.is_cpu + ) + + work_dtype = dtype + if is_bool: + work_dtype = torch.uint8 + if is_half: + work_dtype = torch.float32 + + x = _util.cast_if_needed(x, work_dtype) + y = _util.cast_if_needed(y, work_dtype) + + result = torch.matmul(x, y) + + if work_dtype != dtype: + result = result.to(dtype) + + return result + + +# a stub implementation of divmod, should be improved after +# https://github.com/pytorch/pytorch/issues/90820 is fixed in pytorch +def divmod(x, y): + return x // y, x % y diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_casting_dicts.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_casting_dicts.py new file mode 100644 index 0000000000000000000000000000000000000000..3c859b855dd5d806b4e9e08bb61ada58f4c14c90 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_casting_dicts.py @@ -0,0 +1,1368 @@ +# mypy: ignore-errors + +import torch + + +# These two dicts are autogenerated with autogen/gen_dtypes.py, +# using numpy version 1.24.3. + +_can_cast_dict = { + "no": { + torch.float16: { + torch.float16: True, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.float32: { + torch.float16: False, + torch.float32: True, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.float64: { + torch.float16: False, + torch.float32: False, + torch.float64: True, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.complex64: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: True, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.complex128: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.uint8: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: True, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.uint16: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: True, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.uint32: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: True, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.uint64: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: True, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.int8: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: True, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.int16: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: True, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.int32: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: True, + torch.int64: False, + torch.bool: False, + }, + torch.int64: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: True, + torch.bool: False, + }, + torch.bool: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: True, + }, + }, + "equiv": { + torch.float16: { + torch.float16: True, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.float32: { + torch.float16: False, + torch.float32: True, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.float64: { + torch.float16: False, + torch.float32: False, + torch.float64: True, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.complex64: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: True, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.complex128: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.uint8: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: True, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.uint16: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: True, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.uint32: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: True, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.uint64: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: True, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.int8: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: True, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.int16: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: True, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.int32: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: True, + torch.int64: False, + torch.bool: False, + }, + torch.int64: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: True, + torch.bool: False, + }, + torch.bool: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: False, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: True, + }, + }, + "safe": { + torch.float16: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.float32: { + torch.float16: False, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.float64: { + torch.float16: False, + torch.float32: False, + torch.float64: True, + torch.complex64: False, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.complex64: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.complex128: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: False, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.uint8: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: False, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: False, + }, + torch.uint16: { + torch.float16: False, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: False, + torch.int16: False, + torch.int32: True, + torch.int64: True, + torch.bool: False, + }, + torch.uint32: { + torch.float16: False, + torch.float32: False, + torch.float64: True, + torch.complex64: False, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: True, + torch.uint64: True, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: True, + torch.bool: False, + }, + torch.uint64: { + torch.float16: False, + torch.float32: False, + torch.float64: True, + torch.complex64: False, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: True, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.int8: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: False, + }, + torch.int16: { + torch.float16: False, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: False, + }, + torch.int32: { + torch.float16: False, + torch.float32: False, + torch.float64: True, + torch.complex64: False, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: True, + torch.int64: True, + torch.bool: False, + }, + torch.int64: { + torch.float16: False, + torch.float32: False, + torch.float64: True, + torch.complex64: False, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: True, + torch.bool: False, + }, + torch.bool: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + }, + "same_kind": { + torch.float16: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.float32: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.float64: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.complex64: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.complex128: { + torch.float16: False, + torch.float32: False, + torch.float64: False, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: False, + torch.int16: False, + torch.int32: False, + torch.int64: False, + torch.bool: False, + }, + torch.uint8: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: False, + }, + torch.uint16: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: False, + }, + torch.uint32: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: False, + }, + torch.uint64: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: False, + }, + torch.int8: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: False, + }, + torch.int16: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: False, + }, + torch.int32: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: False, + }, + torch.int64: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: False, + torch.uint16: False, + torch.uint32: False, + torch.uint64: False, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: False, + }, + torch.bool: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + }, + "unsafe": { + torch.float16: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + torch.float32: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + torch.float64: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + torch.complex64: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + torch.complex128: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + torch.uint8: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + torch.uint16: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + torch.uint32: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + torch.uint64: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + torch.int8: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + torch.int16: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + torch.int32: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + torch.int64: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + torch.bool: { + torch.float16: True, + torch.float32: True, + torch.float64: True, + torch.complex64: True, + torch.complex128: True, + torch.uint8: True, + torch.uint16: True, + torch.uint32: True, + torch.uint64: True, + torch.int8: True, + torch.int16: True, + torch.int32: True, + torch.int64: True, + torch.bool: True, + }, + }, +} + + +_result_type_dict = { + torch.float16: { + torch.float16: torch.float16, + torch.float32: torch.float32, + torch.float64: torch.float64, + torch.complex64: torch.complex64, + torch.complex128: torch.complex128, + torch.uint8: torch.float16, + torch.uint16: torch.float32, + torch.uint32: torch.float64, + torch.uint64: torch.float64, + torch.int8: torch.float16, + torch.int16: torch.float32, + torch.int32: torch.float64, + torch.int64: torch.float64, + torch.bool: torch.float16, + }, + torch.float32: { + torch.float16: torch.float32, + torch.float32: torch.float32, + torch.float64: torch.float64, + torch.complex64: torch.complex64, + torch.complex128: torch.complex128, + torch.uint8: torch.float32, + torch.uint16: torch.float32, + torch.uint32: torch.float64, + torch.uint64: torch.float64, + torch.int8: torch.float32, + torch.int16: torch.float32, + torch.int32: torch.float64, + torch.int64: torch.float64, + torch.bool: torch.float32, + }, + torch.float64: { + torch.float16: torch.float64, + torch.float32: torch.float64, + torch.float64: torch.float64, + torch.complex64: torch.complex128, + torch.complex128: torch.complex128, + torch.uint8: torch.float64, + torch.uint16: torch.float64, + torch.uint32: torch.float64, + torch.uint64: torch.float64, + torch.int8: torch.float64, + torch.int16: torch.float64, + torch.int32: torch.float64, + torch.int64: torch.float64, + torch.bool: torch.float64, + }, + torch.complex64: { + torch.float16: torch.complex64, + torch.float32: torch.complex64, + torch.float64: torch.complex128, + torch.complex64: torch.complex64, + torch.complex128: torch.complex128, + torch.uint8: torch.complex64, + torch.uint16: torch.complex64, + torch.uint32: torch.complex128, + torch.uint64: torch.complex128, + torch.int8: torch.complex64, + torch.int16: torch.complex64, + torch.int32: torch.complex128, + torch.int64: torch.complex128, + torch.bool: torch.complex64, + }, + torch.complex128: { + torch.float16: torch.complex128, + torch.float32: torch.complex128, + torch.float64: torch.complex128, + torch.complex64: torch.complex128, + torch.complex128: torch.complex128, + torch.uint8: torch.complex128, + torch.uint16: torch.complex128, + torch.uint32: torch.complex128, + torch.uint64: torch.complex128, + torch.int8: torch.complex128, + torch.int16: torch.complex128, + torch.int32: torch.complex128, + torch.int64: torch.complex128, + torch.bool: torch.complex128, + }, + torch.uint8: { + torch.float16: torch.float16, + torch.float32: torch.float32, + torch.float64: torch.float64, + torch.complex64: torch.complex64, + torch.complex128: torch.complex128, + torch.uint8: torch.uint8, + torch.uint16: torch.uint16, + torch.uint32: torch.uint32, + torch.uint64: torch.uint64, + torch.int8: torch.int16, + torch.int16: torch.int16, + torch.int32: torch.int32, + torch.int64: torch.int64, + torch.bool: torch.uint8, + }, + torch.uint16: { + torch.float16: torch.float32, + torch.float32: torch.float32, + torch.float64: torch.float64, + torch.complex64: torch.complex64, + torch.complex128: torch.complex128, + torch.uint8: torch.uint16, + torch.uint16: torch.uint16, + torch.uint32: torch.uint32, + torch.uint64: torch.uint64, + torch.int8: torch.int32, + torch.int16: torch.int32, + torch.int32: torch.int32, + torch.int64: torch.int64, + torch.bool: torch.uint16, + }, + torch.uint32: { + torch.float16: torch.float64, + torch.float32: torch.float64, + torch.float64: torch.float64, + torch.complex64: torch.complex128, + torch.complex128: torch.complex128, + torch.uint8: torch.uint32, + torch.uint16: torch.uint32, + torch.uint32: torch.uint32, + torch.uint64: torch.uint64, + torch.int8: torch.int64, + torch.int16: torch.int64, + torch.int32: torch.int64, + torch.int64: torch.int64, + torch.bool: torch.uint32, + }, + torch.uint64: { + torch.float16: torch.float64, + torch.float32: torch.float64, + torch.float64: torch.float64, + torch.complex64: torch.complex128, + torch.complex128: torch.complex128, + torch.uint8: torch.uint64, + torch.uint16: torch.uint64, + torch.uint32: torch.uint64, + torch.uint64: torch.uint64, + torch.int8: torch.float64, + torch.int16: torch.float64, + torch.int32: torch.float64, + torch.int64: torch.float64, + torch.bool: torch.uint64, + }, + torch.int8: { + torch.float16: torch.float16, + torch.float32: torch.float32, + torch.float64: torch.float64, + torch.complex64: torch.complex64, + torch.complex128: torch.complex128, + torch.uint8: torch.int16, + torch.uint16: torch.int32, + torch.uint32: torch.int64, + torch.uint64: torch.float64, + torch.int8: torch.int8, + torch.int16: torch.int16, + torch.int32: torch.int32, + torch.int64: torch.int64, + torch.bool: torch.int8, + }, + torch.int16: { + torch.float16: torch.float32, + torch.float32: torch.float32, + torch.float64: torch.float64, + torch.complex64: torch.complex64, + torch.complex128: torch.complex128, + torch.uint8: torch.int16, + torch.uint16: torch.int32, + torch.uint32: torch.int64, + torch.uint64: torch.float64, + torch.int8: torch.int16, + torch.int16: torch.int16, + torch.int32: torch.int32, + torch.int64: torch.int64, + torch.bool: torch.int16, + }, + torch.int32: { + torch.float16: torch.float64, + torch.float32: torch.float64, + torch.float64: torch.float64, + torch.complex64: torch.complex128, + torch.complex128: torch.complex128, + torch.uint8: torch.int32, + torch.uint16: torch.int32, + torch.uint32: torch.int64, + torch.uint64: torch.float64, + torch.int8: torch.int32, + torch.int16: torch.int32, + torch.int32: torch.int32, + torch.int64: torch.int64, + torch.bool: torch.int32, + }, + torch.int64: { + torch.float16: torch.float64, + torch.float32: torch.float64, + torch.float64: torch.float64, + torch.complex64: torch.complex128, + torch.complex128: torch.complex128, + torch.uint8: torch.int64, + torch.uint16: torch.int64, + torch.uint32: torch.int64, + torch.uint64: torch.float64, + torch.int8: torch.int64, + torch.int16: torch.int64, + torch.int32: torch.int64, + torch.int64: torch.int64, + torch.bool: torch.int64, + }, + torch.bool: { + torch.float16: torch.float16, + torch.float32: torch.float32, + torch.float64: torch.float64, + torch.complex64: torch.complex64, + torch.complex128: torch.complex128, + torch.uint8: torch.uint8, + torch.uint16: torch.uint16, + torch.uint32: torch.uint32, + torch.uint64: torch.uint64, + torch.int8: torch.int8, + torch.int16: torch.int16, + torch.int32: torch.int32, + torch.int64: torch.int64, + torch.bool: torch.bool, + }, +} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_dtypes.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_dtypes.py new file mode 100644 index 0000000000000000000000000000000000000000..134f7617b758add06dfcc94cdbd26f757791a00f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_dtypes.py @@ -0,0 +1,452 @@ +# mypy: ignore-errors + +"""Define analogs of numpy dtypes supported by pytorch. +Define the scalar types and supported dtypes and numpy <--> torch dtype mappings. +""" + +import builtins + +import torch + +from . import _dtypes_impl + + +# ### Scalar types ### + + +class generic: + name = "generic" + + def __new__(cls, value): + # NumPy scalars are modelled as 0-D arrays + # so a call to np.float32(4) produces a 0-D array. + + from ._ndarray import asarray, ndarray + + if isinstance(value, str) and value in ["inf", "nan"]: + value = {"inf": torch.inf, "nan": torch.nan}[value] + + if isinstance(value, ndarray): + return value.astype(cls) + else: + return asarray(value, dtype=cls) + + +################## +# abstract types # +################## + + +class number(generic): + name = "number" + + +class integer(number): + name = "integer" + + +class inexact(number): + name = "inexact" + + +class signedinteger(integer): + name = "signedinteger" + + +class unsignedinteger(integer): + name = "unsignedinteger" + + +class floating(inexact): + name = "floating" + + +class complexfloating(inexact): + name = "complexfloating" + + +_abstract_dtypes = [ + "generic", + "number", + "integer", + "signedinteger", + "unsignedinteger", + "inexact", + "floating", + "complexfloating", +] + +# ##### concrete types + +# signed integers + + +class int8(signedinteger): + name = "int8" + typecode = "b" + torch_dtype = torch.int8 + + +class int16(signedinteger): + name = "int16" + typecode = "h" + torch_dtype = torch.int16 + + +class int32(signedinteger): + name = "int32" + typecode = "i" + torch_dtype = torch.int32 + + +class int64(signedinteger): + name = "int64" + typecode = "l" + torch_dtype = torch.int64 + + +# unsigned integers + + +class uint8(unsignedinteger): + name = "uint8" + typecode = "B" + torch_dtype = torch.uint8 + + +class uint16(unsignedinteger): + name = "uint16" + typecode = "H" + torch_dtype = torch.uint16 + + +class uint32(signedinteger): + name = "uint32" + typecode = "I" + torch_dtype = torch.uint32 + + +class uint64(signedinteger): + name = "uint64" + typecode = "L" + torch_dtype = torch.uint64 + + +# floating point + + +class float16(floating): + name = "float16" + typecode = "e" + torch_dtype = torch.float16 + + +class float32(floating): + name = "float32" + typecode = "f" + torch_dtype = torch.float32 + + +class float64(floating): + name = "float64" + typecode = "d" + torch_dtype = torch.float64 + + +class complex64(complexfloating): + name = "complex64" + typecode = "F" + torch_dtype = torch.complex64 + + +class complex128(complexfloating): + name = "complex128" + typecode = "D" + torch_dtype = torch.complex128 + + +class bool_(generic): + name = "bool_" + typecode = "?" + torch_dtype = torch.bool + + +# name aliases +_name_aliases = { + "intp": int64, + "int_": int64, + "intc": int32, + "byte": int8, + "short": int16, + "longlong": int64, # XXX: is this correct? + "ulonglong": uint64, + "ubyte": uint8, + "half": float16, + "single": float32, + "double": float64, + "float_": float64, + "csingle": complex64, + "singlecomplex": complex64, + "cdouble": complex128, + "cfloat": complex128, + "complex_": complex128, +} +# We register float_ = float32 and so on +for name, obj in _name_aliases.items(): + vars()[name] = obj + + +# Replicate this NumPy-defined way of grouping scalar types, +# cf tests/core/test_scalar_methods.py +sctypes = { + "int": [int8, int16, int32, int64], + "uint": [uint8, uint16, uint32, uint64], + "float": [float16, float32, float64], + "complex": [complex64, complex128], + "others": [bool_], +} + + +# Support mappings/functions + +_names = {st.name: st for cat in sctypes for st in sctypes[cat]} +_typecodes = {st.typecode: st for cat in sctypes for st in sctypes[cat]} +_torch_dtypes = {st.torch_dtype: st for cat in sctypes for st in sctypes[cat]} + + +_aliases = { + "u1": uint8, + "i1": int8, + "i2": int16, + "i4": int32, + "i8": int64, + "b": int8, # XXX: srsly? + "f2": float16, + "f4": float32, + "f8": float64, + "c8": complex64, + "c16": complex128, + # numpy-specific trailing underscore + "bool_": bool_, +} + + +_python_types = { + int: int64, + float: float64, + complex: complex128, + builtins.bool: bool_, + # also allow stringified names of python types + int.__name__: int64, + float.__name__: float64, + complex.__name__: complex128, + builtins.bool.__name__: bool_, +} + + +def sctype_from_string(s): + """Normalize a string value: a type 'name' or a typecode or a width alias.""" + if s in _names: + return _names[s] + if s in _name_aliases: + return _name_aliases[s] + if s in _typecodes: + return _typecodes[s] + if s in _aliases: + return _aliases[s] + if s in _python_types: + return _python_types[s] + raise TypeError(f"data type {s!r} not understood") + + +def sctype_from_torch_dtype(torch_dtype): + return _torch_dtypes[torch_dtype] + + +# ### DTypes. ### + + +def dtype(arg): + if arg is None: + arg = _dtypes_impl.default_dtypes().float_dtype + return DType(arg) + + +class DType: + def __init__(self, arg): + # a pytorch object? + if isinstance(arg, torch.dtype): + sctype = _torch_dtypes[arg] + elif isinstance(arg, torch.Tensor): + sctype = _torch_dtypes[arg.dtype] + # a scalar type? + elif issubclass_(arg, generic): + sctype = arg + # a dtype already? + elif isinstance(arg, DType): + sctype = arg._scalar_type + # a has a right attribute? + elif hasattr(arg, "dtype"): + sctype = arg.dtype._scalar_type + else: + sctype = sctype_from_string(arg) + self._scalar_type = sctype + + @property + def name(self): + return self._scalar_type.name + + @property + def type(self): + return self._scalar_type + + @property + def kind(self): + # https://numpy.org/doc/stable/reference/generated/numpy.dtype.kind.html + return _torch_dtypes[self.torch_dtype].name[0] + + @property + def typecode(self): + return self._scalar_type.typecode + + def __eq__(self, other): + if isinstance(other, DType): + return self._scalar_type == other._scalar_type + try: + other_instance = DType(other) + except TypeError: + return False + return self._scalar_type == other_instance._scalar_type + + @property + def torch_dtype(self): + return self._scalar_type.torch_dtype + + def __hash__(self): + return hash(self._scalar_type.name) + + def __repr__(self): + return f'dtype("{self.name}")' + + __str__ = __repr__ + + @property + def itemsize(self): + elem = self.type(1) + return elem.tensor.element_size() + + def __getstate__(self): + return self._scalar_type + + def __setstate__(self, value): + self._scalar_type = value + + +typecodes = { + "All": "efdFDBbhil?", + "AllFloat": "efdFD", + "AllInteger": "Bbhil", + "Integer": "bhil", + "UnsignedInteger": "B", + "Float": "efd", + "Complex": "FD", +} + + +# ### Defaults and dtype discovery + + +def set_default_dtype(fp_dtype="numpy", int_dtype="numpy"): + """Set the (global) defaults for fp, complex, and int dtypes. + + The complex dtype is inferred from the float (fp) dtype. It has + a width at least twice the width of the float dtype, + i.e., it's complex128 for float64 and complex64 for float32. + + Parameters + ---------- + fp_dtype + Allowed values are "numpy", "pytorch" or dtype_like things which + can be converted into a DType instance. + Default is "numpy" (i.e. float64). + int_dtype + Allowed values are "numpy", "pytorch" or dtype_like things which + can be converted into a DType instance. + Default is "numpy" (i.e. int64). + + Returns + ------- + The old default dtype state: a namedtuple with attributes ``float_dtype``, + ``complex_dtypes`` and ``int_dtype``. These attributes store *pytorch* + dtypes. + + Notes + ------------ + This functions has a side effect: it sets the global state with the provided dtypes. + + The complex dtype has bit width of at least twice the width of the float + dtype, i.e. it's complex128 for float64 and complex64 for float32. + + """ + if fp_dtype not in ["numpy", "pytorch"]: + fp_dtype = dtype(fp_dtype).torch_dtype + if int_dtype not in ["numpy", "pytorch"]: + int_dtype = dtype(int_dtype).torch_dtype + + if fp_dtype == "numpy": + float_dtype = torch.float64 + elif fp_dtype == "pytorch": + float_dtype = torch.float32 + else: + float_dtype = fp_dtype + + complex_dtype = { + torch.float64: torch.complex128, + torch.float32: torch.complex64, + torch.float16: torch.complex64, + }[float_dtype] + + if int_dtype in ["numpy", "pytorch"]: + int_dtype = torch.int64 + + new_defaults = _dtypes_impl.DefaultDTypes( + float_dtype=float_dtype, complex_dtype=complex_dtype, int_dtype=int_dtype + ) + + # set the new global state and return the old state + old_defaults = _dtypes_impl.default_dtypes + _dtypes_impl._default_dtypes = new_defaults + return old_defaults + + +def issubclass_(arg, klass): + try: + return issubclass(arg, klass) + except TypeError: + return False + + +def issubdtype(arg1, arg2): + # cf https://github.com/numpy/numpy/blob/v1.24.0/numpy/core/numerictypes.py#L356-L420 + + # We also accept strings even if NumPy doesn't as dtypes are serialized as their + # string representation in dynamo's graph + def str_to_abstract(t): + if isinstance(t, str) and t in _abstract_dtypes: + return globals()[t] + return t + + arg1 = str_to_abstract(arg1) + arg2 = str_to_abstract(arg2) + + if not issubclass_(arg1, generic): + arg1 = dtype(arg1).type + if not issubclass_(arg2, generic): + arg2 = dtype(arg2).type + return issubclass(arg1, arg2) + + +__all__ = ["dtype", "DType", "typecodes", "issubdtype", "set_default_dtype", "sctypes"] +__all__ += list(_names.keys()) # noqa: PLE0605 +__all__ += list(_name_aliases.keys()) # noqa: PLE0605 +__all__ += _abstract_dtypes # noqa: PLE0605 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_dtypes_impl.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_dtypes_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..feed9c460050139a49bb205066e095b673af0101 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_dtypes_impl.py @@ -0,0 +1,218 @@ +# mypy: ignore-errors + +"""Dtypes/scalar type implementations with torch dtypes. + +Here `dtype` is always a torch.dtype, this module knows nothing about +scalar types, wrapper dtypes or anything like that. PyTorch only. +""" + +from collections import namedtuple + +import torch + + +# defaults : mimic NumPy, allow user control +DefaultDTypes = namedtuple( + "DefaultDTypes", ["float_dtype", "complex_dtype", "int_dtype"] +) + +# a global state +# We set it the first time we call default_dtypes() to avoid importing +# torch._dynamo.config and create a circular reference +_default_dtypes = None + + +def default_dtypes(): + global _default_dtypes + if _default_dtypes is None: + import torch._dynamo.config as config + + _default_dtypes = DefaultDTypes( + float_dtype=getattr(torch, config.numpy_default_float), + complex_dtype=getattr(torch, config.numpy_default_complex), + int_dtype=getattr(torch, config.numpy_default_int), + ) + assert isinstance(_default_dtypes.float_dtype, torch.dtype) + assert isinstance(_default_dtypes.complex_dtype, torch.dtype) + assert isinstance(_default_dtypes.int_dtype, torch.dtype) + return _default_dtypes + + +def get_default_dtype_for(dtype): + """Default scalar type given sctype category.""" + if dtype == torch.bool: + return dtype + if dtype.is_complex: + return default_dtypes().complex_dtype + if dtype.is_floating_point: + return default_dtypes().float_dtype + # else, it must be (some) integer + return default_dtypes().int_dtype + + +from . import _casting_dicts as _cd + + +def can_cast_impl(from_torch_dtype, to_torch_dtype, casting): + return _cd._can_cast_dict[casting][from_torch_dtype][to_torch_dtype] + + +def result_type_impl(*tensors): + # NB: torch dtypes here + dtyp = tensors[0].dtype + if len(tensors) == 1: + return dtyp + + for curr in tensors[1:]: + dtyp = _cd._result_type_dict[dtyp][curr.dtype] + + return dtyp + + +def python_type_for_torch(dtyp): + """Get a python scalar type a torch dtype""" + if dtyp.is_floating_point: + typ = float + elif dtyp.is_complex: + typ = complex + elif dtyp == torch.bool: + typ = bool + else: + typ = int + return typ + + +# ### NEP 50 helpers ### + +_SCALAR_TYPES = (int, bool, float, complex) + +_SCALAR_AND_SYMBOLIC_TYPES = ( + *_SCALAR_TYPES, + torch.SymInt, + torch.SymFloat, + torch.SymBool, +) + +_NEP50_FUNCS_TENSOR_ONLY = ( + "minimum", + "maximum", + "logaddexp", + "logaddexp2", + "lcm", + "gcd", + "hypot", + "heaviside", + "fmod", + "fmin", + "fmax", + "copysign", + "arctan2", +) + + +def is_scalar(x): + return isinstance(x, _SCALAR_TYPES) + + +def is_scalar_or_symbolic(x): + return isinstance(x, _SCALAR_AND_SYMBOLIC_TYPES) + + +def _dtype_for_scalar(py_type): + return { + bool: torch.bool, + torch.SymBool: torch.bool, + int: torch.int64, + torch.SymInt: torch.int64, + float: torch.float64, + torch.SymFloat: torch.float64, + complex: torch.complex128, + }[py_type] + + +def _dtype_for_scalar_or_tensor(x): + return x.dtype if isinstance(x, torch.Tensor) else _dtype_for_scalar(type(x)) + + +def is_float_or_fp_tensor(x): + return _dtype_for_scalar_or_tensor(x).is_floating_point + + +def is_complex_or_complex_tensor(x): + return _dtype_for_scalar_or_tensor(x).is_complex + + +def _category(dtype): + return { + torch.bool: 0, + torch.SymBool: 0, + # int + torch.uint8: 1, + torch.int8: 1, + torch.int16: 1, + torch.int32: 1, + torch.int64: 1, + torch.SymInt: 1, + # float + torch.float16: 2, + torch.float32: 2, + torch.float64: 2, + torch.SymFloat: 2, + # complex + torch.complex64: 3, + torch.complex128: 3, + }[dtype] + + +def nep50_to_tensors(x1, x2, handle_weaks, function_name): + """If either of inputs is a python scalar, type-promote with NEP 50.""" + + def to_tensor(scalar, dtype=None): + if dtype is None: + dtype = _dtype_for_scalar(type(scalar)) + dtype = get_default_dtype_for(dtype) + return torch.as_tensor(scalar, dtype=dtype) + + x1_is_weak = not isinstance(x1, torch.Tensor) + x2_is_weak = not isinstance(x2, torch.Tensor) + if not handle_weaks or (x1_is_weak and x2_is_weak): + x1 = to_tensor(x1) if x1_is_weak else x1 + x2 = to_tensor(x2) if x2_is_weak else x2 + return x1, x2 + + # scalar tensor: NEP 50 + assert x1_is_weak != x2_is_weak + + weak, not_weak = (x1, x2) if x1_is_weak else (x2, x1) + + # find the dtype for the weak's type + weak_dtype = _dtype_for_scalar(type(weak)) + + cat_weak = _category(weak_dtype) + cat_not_weak = _category(not_weak.dtype) + + dt = not_weak.dtype if cat_weak <= cat_not_weak else None + + # special-case complex + float32 + if weak_dtype.is_complex and not_weak.dtype == torch.float32: + dt = torch.complex64 + + # detect overflows: in PyTorch, uint8(-1) wraps around to 255, + # while NEP50 mandates an exception. + # + # Note that we only check if each element of the binop overflows, + # not the result. Consider, e.g. `uint8(100) + 200`. Operands are OK + # in uint8, but the result overflows and wrap around 255. + # Numpy emits a RuntimeWarning, PyTorch does not, and we do not either. + if cat_weak == 1 and cat_not_weak == 1: + # integers + iinfo = torch.iinfo(not_weak.dtype) + if not (iinfo.min <= weak <= iinfo.max): + raise OverflowError( + f"Python integer {weak} out of bounds for {not_weak.dtype}" + ) + if weak_dtype != dt or function_name in _NEP50_FUNCS_TENSOR_ONLY: + # finally, can make `weak` into a 0D tensor, if both parameters are required to be tensor. + weak = to_tensor(weak, dt) + + return (weak, not_weak) if x1_is_weak else (not_weak, weak) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_funcs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_funcs.py new file mode 100644 index 0000000000000000000000000000000000000000..c0341c4588ed5359ff68a90c8c53e667eedeb12d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_funcs.py @@ -0,0 +1,76 @@ +# mypy: ignore-errors + +import inspect +import itertools + +from . import _funcs_impl, _reductions_impl +from ._normalizations import normalizer + + +# _funcs_impl.py contains functions which mimic NumPy's eponymous equivalents, +# and consume/return PyTorch tensors/dtypes. +# They are also type annotated. +# Pull these functions from _funcs_impl and decorate them with @normalizer, which +# - Converts any input `np.ndarray`, `torch._numpy.ndarray`, list of lists, Python scalars, etc into a `torch.Tensor`. +# - Maps NumPy dtypes to PyTorch dtypes +# - If the input to the `axis` kwarg is an ndarray, it maps it into a tuple +# - Implements the semantics for the `out=` arg +# - Wraps back the outputs into `torch._numpy.ndarrays` + + +def _public_functions(mod): + def is_public_function(f): + return inspect.isfunction(f) and not f.__name__.startswith("_") + + return inspect.getmembers(mod, is_public_function) + + +# We fill in __all__ in the loop below +__all__ = [] + +# decorate implementer functions with argument normalizers and export to the top namespace +for name, func in itertools.chain( + _public_functions(_funcs_impl), _public_functions(_reductions_impl) +): + if name in ["percentile", "quantile", "median"]: + decorated = normalizer(func, promote_scalar_result=True) + elif name == "einsum": + # normalized manually + decorated = func + else: + decorated = normalizer(func) + + decorated.__qualname__ = name + decorated.__name__ = name + vars()[name] = decorated + __all__.append(name) + + +""" +Vendored objects from numpy.lib.index_tricks +""" + + +class IndexExpression: + """ + Written by Konrad Hinsen + last revision: 1999-7-23 + + Cosmetic changes by T. Oliphant 2001 + """ + + def __init__(self, maketuple): + self.maketuple = maketuple + + def __getitem__(self, item): + if self.maketuple and not isinstance(item, tuple): + return (item,) + else: + return item + + +index_exp = IndexExpression(maketuple=True) +s_ = IndexExpression(maketuple=False) + + +__all__ += ["index_exp", "s_"] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_funcs_impl.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_funcs_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..3417a401acb052ea66b2b0bee069e40a40034c52 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_funcs_impl.py @@ -0,0 +1,2061 @@ +# mypy: ignore-errors + +"""A thin pytorch / numpy compat layer. + +Things imported from here have numpy-compatible signatures but operate on +pytorch tensors. +""" + +# Contents of this module ends up in the main namespace via _funcs.py +# where type annotations are used in conjunction with the @normalizer decorator. +from __future__ import annotations + +import builtins +import itertools +import operator +from typing import Optional, TYPE_CHECKING + +import torch + +from . import _dtypes_impl, _util + + +if TYPE_CHECKING: + from collections.abc import Sequence + + from ._normalizations import ( + ArrayLike, + ArrayLikeOrScalar, + CastingModes, + DTypeLike, + NDArray, + NotImplementedType, + OutArray, + ) + + +def copy( + a: ArrayLike, order: NotImplementedType = "K", subok: NotImplementedType = False +): + return a.clone() + + +def copyto( + dst: NDArray, + src: ArrayLike, + casting: Optional[CastingModes] = "same_kind", + where: NotImplementedType = None, +): + (src,) = _util.typecast_tensors((src,), dst.dtype, casting=casting) + dst.copy_(src) + + +def atleast_1d(*arys: ArrayLike): + res = torch.atleast_1d(*arys) + if isinstance(res, tuple): + return list(res) + else: + return res + + +def atleast_2d(*arys: ArrayLike): + res = torch.atleast_2d(*arys) + if isinstance(res, tuple): + return list(res) + else: + return res + + +def atleast_3d(*arys: ArrayLike): + res = torch.atleast_3d(*arys) + if isinstance(res, tuple): + return list(res) + else: + return res + + +def _concat_check(tup, dtype, out): + if tup == (): + raise ValueError("need at least one array to concatenate") + + """Check inputs in concatenate et al.""" + if out is not None and dtype is not None: + # mimic numpy + raise TypeError( + "concatenate() only takes `out` or `dtype` as an " + "argument, but both were provided." + ) + + +def _concat_cast_helper(tensors, out=None, dtype=None, casting="same_kind"): + """Figure out dtypes, cast if necessary.""" + + if out is not None or dtype is not None: + # figure out the type of the inputs and outputs + out_dtype = out.dtype.torch_dtype if dtype is None else dtype + else: + out_dtype = _dtypes_impl.result_type_impl(*tensors) + + # cast input arrays if necessary; do not broadcast them against `out` + tensors = _util.typecast_tensors(tensors, out_dtype, casting) + + return tensors + + +def _concatenate( + tensors, axis=0, out=None, dtype=None, casting: Optional[CastingModes] = "same_kind" +): + # pure torch implementation, used below and in cov/corrcoef below + tensors, axis = _util.axis_none_flatten(*tensors, axis=axis) + tensors = _concat_cast_helper(tensors, out, dtype, casting) + return torch.cat(tensors, axis) + + +def concatenate( + ar_tuple: Sequence[ArrayLike], + axis=0, + out: Optional[OutArray] = None, + dtype: Optional[DTypeLike] = None, + casting: Optional[CastingModes] = "same_kind", +): + _concat_check(ar_tuple, dtype, out=out) + result = _concatenate(ar_tuple, axis=axis, out=out, dtype=dtype, casting=casting) + return result + + +def vstack( + tup: Sequence[ArrayLike], + *, + dtype: Optional[DTypeLike] = None, + casting: Optional[CastingModes] = "same_kind", +): + _concat_check(tup, dtype, out=None) + tensors = _concat_cast_helper(tup, dtype=dtype, casting=casting) + return torch.vstack(tensors) + + +row_stack = vstack + + +def hstack( + tup: Sequence[ArrayLike], + *, + dtype: Optional[DTypeLike] = None, + casting: Optional[CastingModes] = "same_kind", +): + _concat_check(tup, dtype, out=None) + tensors = _concat_cast_helper(tup, dtype=dtype, casting=casting) + return torch.hstack(tensors) + + +def dstack( + tup: Sequence[ArrayLike], + *, + dtype: Optional[DTypeLike] = None, + casting: Optional[CastingModes] = "same_kind", +): + # XXX: in numpy 1.24 dstack does not have dtype and casting keywords + # but {h,v}stack do. Hence add them here for consistency. + _concat_check(tup, dtype, out=None) + tensors = _concat_cast_helper(tup, dtype=dtype, casting=casting) + return torch.dstack(tensors) + + +def column_stack( + tup: Sequence[ArrayLike], + *, + dtype: Optional[DTypeLike] = None, + casting: Optional[CastingModes] = "same_kind", +): + # XXX: in numpy 1.24 column_stack does not have dtype and casting keywords + # but row_stack does. (because row_stack is an alias for vstack, really). + # Hence add these keywords here for consistency. + _concat_check(tup, dtype, out=None) + tensors = _concat_cast_helper(tup, dtype=dtype, casting=casting) + return torch.column_stack(tensors) + + +def stack( + arrays: Sequence[ArrayLike], + axis=0, + out: Optional[OutArray] = None, + *, + dtype: Optional[DTypeLike] = None, + casting: Optional[CastingModes] = "same_kind", +): + _concat_check(arrays, dtype, out=out) + + tensors = _concat_cast_helper(arrays, dtype=dtype, casting=casting) + result_ndim = tensors[0].ndim + 1 + axis = _util.normalize_axis_index(axis, result_ndim) + return torch.stack(tensors, axis=axis) + + +def append(arr: ArrayLike, values: ArrayLike, axis=None): + if axis is None: + if arr.ndim != 1: + arr = arr.flatten() + values = values.flatten() + axis = arr.ndim - 1 + return _concatenate((arr, values), axis=axis) + + +# ### split ### + + +def _split_helper(tensor, indices_or_sections, axis, strict=False): + if isinstance(indices_or_sections, int): + return _split_helper_int(tensor, indices_or_sections, axis, strict) + elif isinstance(indices_or_sections, (list, tuple)): + # NB: drop split=..., it only applies to split_helper_int + return _split_helper_list(tensor, list(indices_or_sections), axis) + else: + raise TypeError("split_helper: ", type(indices_or_sections)) + + +def _split_helper_int(tensor, indices_or_sections, axis, strict=False): + if not isinstance(indices_or_sections, int): + raise NotImplementedError("split: indices_or_sections") + + axis = _util.normalize_axis_index(axis, tensor.ndim) + + # numpy: l%n chunks of size (l//n + 1), the rest are sized l//n + l, n = tensor.shape[axis], indices_or_sections + + if n <= 0: + raise ValueError + + if l % n == 0: + num, sz = n, l // n + lst = [sz] * num + else: + if strict: + raise ValueError("array split does not result in an equal division") + + num, sz = l % n, l // n + 1 + lst = [sz] * num + + lst += [sz - 1] * (n - num) + + return torch.split(tensor, lst, axis) + + +def _split_helper_list(tensor, indices_or_sections, axis): + if not isinstance(indices_or_sections, list): + raise NotImplementedError("split: indices_or_sections: list") + # numpy expects indices, while torch expects lengths of sections + # also, numpy appends zero-size arrays for indices above the shape[axis] + lst = [x for x in indices_or_sections if x <= tensor.shape[axis]] + num_extra = len(indices_or_sections) - len(lst) + + lst.append(tensor.shape[axis]) + lst = [ + lst[0], + ] + [a - b for a, b in zip(lst[1:], lst[:-1])] + lst += [0] * num_extra + + return torch.split(tensor, lst, axis) + + +def array_split(ary: ArrayLike, indices_or_sections, axis=0): + return _split_helper(ary, indices_or_sections, axis) + + +def split(ary: ArrayLike, indices_or_sections, axis=0): + return _split_helper(ary, indices_or_sections, axis, strict=True) + + +def hsplit(ary: ArrayLike, indices_or_sections): + if ary.ndim == 0: + raise ValueError("hsplit only works on arrays of 1 or more dimensions") + axis = 1 if ary.ndim > 1 else 0 + return _split_helper(ary, indices_or_sections, axis, strict=True) + + +def vsplit(ary: ArrayLike, indices_or_sections): + if ary.ndim < 2: + raise ValueError("vsplit only works on arrays of 2 or more dimensions") + return _split_helper(ary, indices_or_sections, 0, strict=True) + + +def dsplit(ary: ArrayLike, indices_or_sections): + if ary.ndim < 3: + raise ValueError("dsplit only works on arrays of 3 or more dimensions") + return _split_helper(ary, indices_or_sections, 2, strict=True) + + +def kron(a: ArrayLike, b: ArrayLike): + return torch.kron(a, b) + + +def vander(x: ArrayLike, N=None, increasing=False): + return torch.vander(x, N, increasing) + + +# ### linspace, geomspace, logspace and arange ### + + +def linspace( + start: ArrayLike, + stop: ArrayLike, + num=50, + endpoint=True, + retstep=False, + dtype: Optional[DTypeLike] = None, + axis=0, +): + if axis != 0 or retstep or not endpoint: + raise NotImplementedError + if dtype is None: + dtype = _dtypes_impl.default_dtypes().float_dtype + # XXX: raises TypeError if start or stop are not scalars + return torch.linspace(start, stop, num, dtype=dtype) + + +def geomspace( + start: ArrayLike, + stop: ArrayLike, + num=50, + endpoint=True, + dtype: Optional[DTypeLike] = None, + axis=0, +): + if axis != 0 or not endpoint: + raise NotImplementedError + base = torch.pow(stop / start, 1.0 / (num - 1)) + logbase = torch.log(base) + return torch.logspace( + torch.log(start) / logbase, + torch.log(stop) / logbase, + num, + base=base, + ) + + +def logspace( + start, + stop, + num=50, + endpoint=True, + base=10.0, + dtype: Optional[DTypeLike] = None, + axis=0, +): + if axis != 0 or not endpoint: + raise NotImplementedError + return torch.logspace(start, stop, num, base=base, dtype=dtype) + + +def arange( + start: Optional[ArrayLikeOrScalar] = None, + stop: Optional[ArrayLikeOrScalar] = None, + step: Optional[ArrayLikeOrScalar] = 1, + dtype: Optional[DTypeLike] = None, + *, + like: NotImplementedType = None, +): + if step == 0: + raise ZeroDivisionError + if stop is None and start is None: + raise TypeError + if stop is None: + # XXX: this breaks if start is passed as a kwarg: + # arange(start=4) should raise (no stop) but doesn't + start, stop = 0, start + if start is None: + start = 0 + + # the dtype of the result + if dtype is None: + dtype = ( + _dtypes_impl.default_dtypes().float_dtype + if any(_dtypes_impl.is_float_or_fp_tensor(x) for x in (start, stop, step)) + else _dtypes_impl.default_dtypes().int_dtype + ) + work_dtype = torch.float64 if dtype.is_complex else dtype + + # RuntimeError: "lt_cpu" not implemented for 'ComplexFloat'. Fall back to eager. + if any(_dtypes_impl.is_complex_or_complex_tensor(x) for x in (start, stop, step)): + raise NotImplementedError + + if (step > 0 and start > stop) or (step < 0 and start < stop): + # empty range + return torch.empty(0, dtype=dtype) + + result = torch.arange(start, stop, step, dtype=work_dtype) + result = _util.cast_if_needed(result, dtype) + return result + + +# ### zeros/ones/empty/full ### + + +def empty( + shape, + dtype: Optional[DTypeLike] = None, + order: NotImplementedType = "C", + *, + like: NotImplementedType = None, +): + if dtype is None: + dtype = _dtypes_impl.default_dtypes().float_dtype + return torch.empty(shape, dtype=dtype) + + +# NB: *_like functions deliberately deviate from numpy: it has subok=True +# as the default; we set subok=False and raise on anything else. + + +def empty_like( + prototype: ArrayLike, + dtype: Optional[DTypeLike] = None, + order: NotImplementedType = "K", + subok: NotImplementedType = False, + shape=None, +): + result = torch.empty_like(prototype, dtype=dtype) + if shape is not None: + result = result.reshape(shape) + return result + + +def full( + shape, + fill_value: ArrayLike, + dtype: Optional[DTypeLike] = None, + order: NotImplementedType = "C", + *, + like: NotImplementedType = None, +): + if isinstance(shape, int): + shape = (shape,) + if dtype is None: + dtype = fill_value.dtype + if not isinstance(shape, (tuple, list)): + shape = (shape,) + return torch.full(shape, fill_value, dtype=dtype) + + +def full_like( + a: ArrayLike, + fill_value, + dtype: Optional[DTypeLike] = None, + order: NotImplementedType = "K", + subok: NotImplementedType = False, + shape=None, +): + # XXX: fill_value broadcasts + result = torch.full_like(a, fill_value, dtype=dtype) + if shape is not None: + result = result.reshape(shape) + return result + + +def ones( + shape, + dtype: Optional[DTypeLike] = None, + order: NotImplementedType = "C", + *, + like: NotImplementedType = None, +): + if dtype is None: + dtype = _dtypes_impl.default_dtypes().float_dtype + return torch.ones(shape, dtype=dtype) + + +def ones_like( + a: ArrayLike, + dtype: Optional[DTypeLike] = None, + order: NotImplementedType = "K", + subok: NotImplementedType = False, + shape=None, +): + result = torch.ones_like(a, dtype=dtype) + if shape is not None: + result = result.reshape(shape) + return result + + +def zeros( + shape, + dtype: Optional[DTypeLike] = None, + order: NotImplementedType = "C", + *, + like: NotImplementedType = None, +): + if dtype is None: + dtype = _dtypes_impl.default_dtypes().float_dtype + return torch.zeros(shape, dtype=dtype) + + +def zeros_like( + a: ArrayLike, + dtype: Optional[DTypeLike] = None, + order: NotImplementedType = "K", + subok: NotImplementedType = False, + shape=None, +): + result = torch.zeros_like(a, dtype=dtype) + if shape is not None: + result = result.reshape(shape) + return result + + +# ### cov & corrcoef ### + + +def _xy_helper_corrcoef(x_tensor, y_tensor=None, rowvar=True): + """Prepare inputs for cov and corrcoef.""" + + # https://github.com/numpy/numpy/blob/v1.24.0/numpy/lib/function_base.py#L2636 + if y_tensor is not None: + # make sure x and y are at least 2D + ndim_extra = 2 - x_tensor.ndim + if ndim_extra > 0: + x_tensor = x_tensor.view((1,) * ndim_extra + x_tensor.shape) + if not rowvar and x_tensor.shape[0] != 1: + x_tensor = x_tensor.mT + x_tensor = x_tensor.clone() + + ndim_extra = 2 - y_tensor.ndim + if ndim_extra > 0: + y_tensor = y_tensor.view((1,) * ndim_extra + y_tensor.shape) + if not rowvar and y_tensor.shape[0] != 1: + y_tensor = y_tensor.mT + y_tensor = y_tensor.clone() + + x_tensor = _concatenate((x_tensor, y_tensor), axis=0) + + return x_tensor + + +def corrcoef( + x: ArrayLike, + y: Optional[ArrayLike] = None, + rowvar=True, + bias=None, + ddof=None, + *, + dtype: Optional[DTypeLike] = None, +): + if bias is not None or ddof is not None: + # deprecated in NumPy + raise NotImplementedError + xy_tensor = _xy_helper_corrcoef(x, y, rowvar) + + is_half = (xy_tensor.dtype == torch.float16) and xy_tensor.is_cpu + if is_half: + # work around torch's "addmm_impl_cpu_" not implemented for 'Half'" + dtype = torch.float32 + + xy_tensor = _util.cast_if_needed(xy_tensor, dtype) + result = torch.corrcoef(xy_tensor) + + if is_half: + result = result.to(torch.float16) + + return result + + +def cov( + m: ArrayLike, + y: Optional[ArrayLike] = None, + rowvar=True, + bias=False, + ddof=None, + fweights: Optional[ArrayLike] = None, + aweights: Optional[ArrayLike] = None, + *, + dtype: Optional[DTypeLike] = None, +): + m = _xy_helper_corrcoef(m, y, rowvar) + + if ddof is None: + ddof = 1 if bias == 0 else 0 + + is_half = (m.dtype == torch.float16) and m.is_cpu + if is_half: + # work around torch's "addmm_impl_cpu_" not implemented for 'Half'" + dtype = torch.float32 + + m = _util.cast_if_needed(m, dtype) + result = torch.cov(m, correction=ddof, aweights=aweights, fweights=fweights) + + if is_half: + result = result.to(torch.float16) + + return result + + +def _conv_corr_impl(a, v, mode): + dt = _dtypes_impl.result_type_impl(a, v) + a = _util.cast_if_needed(a, dt) + v = _util.cast_if_needed(v, dt) + + padding = v.shape[0] - 1 if mode == "full" else mode + + if padding == "same" and v.shape[0] % 2 == 0: + # UserWarning: Using padding='same' with even kernel lengths and odd + # dilation may require a zero-padded copy of the input be created + # (Triggered internally at pytorch/aten/src/ATen/native/Convolution.cpp:1010.) + raise NotImplementedError("mode='same' and even-length weights") + + # NumPy only accepts 1D arrays; PyTorch requires 2D inputs and 3D weights + aa = a[None, :] + vv = v[None, None, :] + + result = torch.nn.functional.conv1d(aa, vv, padding=padding) + + # torch returns a 2D result, numpy returns a 1D array + return result[0, :] + + +def convolve(a: ArrayLike, v: ArrayLike, mode="full"): + # NumPy: if v is longer than a, the arrays are swapped before computation + if a.shape[0] < v.shape[0]: + a, v = v, a + + # flip the weights since numpy does and torch does not + v = torch.flip(v, (0,)) + + return _conv_corr_impl(a, v, mode) + + +def correlate(a: ArrayLike, v: ArrayLike, mode="valid"): + v = torch.conj_physical(v) + return _conv_corr_impl(a, v, mode) + + +# ### logic & element selection ### + + +def bincount(x: ArrayLike, /, weights: Optional[ArrayLike] = None, minlength=0): + if x.numel() == 0: + # edge case allowed by numpy + x = x.new_empty(0, dtype=int) + + int_dtype = _dtypes_impl.default_dtypes().int_dtype + (x,) = _util.typecast_tensors((x,), int_dtype, casting="safe") + + return torch.bincount(x, weights, minlength) + + +def where( + condition: ArrayLike, + x: Optional[ArrayLikeOrScalar] = None, + y: Optional[ArrayLikeOrScalar] = None, + /, +): + if (x is None) != (y is None): + raise ValueError("either both or neither of x and y should be given") + + if condition.dtype != torch.bool: + condition = condition.to(torch.bool) + + if x is None and y is None: + result = torch.where(condition) + else: + result = torch.where(condition, x, y) + return result + + +# ###### module-level queries of object properties + + +def ndim(a: ArrayLike): + return a.ndim + + +def shape(a: ArrayLike): + return tuple(a.shape) + + +def size(a: ArrayLike, axis=None): + if axis is None: + return a.numel() + else: + return a.shape[axis] + + +# ###### shape manipulations and indexing + + +def expand_dims(a: ArrayLike, axis): + shape = _util.expand_shape(a.shape, axis) + return a.view(shape) # never copies + + +def flip(m: ArrayLike, axis=None): + # XXX: semantic difference: np.flip returns a view, torch.flip copies + if axis is None: + axis = tuple(range(m.ndim)) + else: + axis = _util.normalize_axis_tuple(axis, m.ndim) + return torch.flip(m, axis) + + +def flipud(m: ArrayLike): + return torch.flipud(m) + + +def fliplr(m: ArrayLike): + return torch.fliplr(m) + + +def rot90(m: ArrayLike, k=1, axes=(0, 1)): + axes = _util.normalize_axis_tuple(axes, m.ndim) + return torch.rot90(m, k, axes) + + +# ### broadcasting and indices ### + + +def broadcast_to(array: ArrayLike, shape, subok: NotImplementedType = False): + return torch.broadcast_to(array, size=shape) + + +# This is a function from tuples to tuples, so we just reuse it. However, +# dynamo expects its __module__ to be torch._numpy +def broadcast_shapes(*args): + return torch.broadcast_shapes(*args) + + +def broadcast_arrays(*args: ArrayLike, subok: NotImplementedType = False): + return torch.broadcast_tensors(*args) + + +def meshgrid(*xi: ArrayLike, copy=True, sparse=False, indexing="xy"): + ndim = len(xi) + + if indexing not in ["xy", "ij"]: + raise ValueError("Valid values for `indexing` are 'xy' and 'ij'.") + + s0 = (1,) * ndim + output = [x.reshape(s0[:i] + (-1,) + s0[i + 1 :]) for i, x in enumerate(xi)] + + if indexing == "xy" and ndim > 1: + # switch first and second axis + output[0] = output[0].reshape((1, -1) + s0[2:]) + output[1] = output[1].reshape((-1, 1) + s0[2:]) + + if not sparse: + # Return the full N-D matrix (not only the 1-D vector) + output = torch.broadcast_tensors(*output) + + if copy: + output = [x.clone() for x in output] + + return list(output) # match numpy, return a list + + +def indices(dimensions, dtype: Optional[DTypeLike] = int, sparse=False): + # https://github.com/numpy/numpy/blob/v1.24.0/numpy/core/numeric.py#L1691-L1791 + dimensions = tuple(dimensions) + N = len(dimensions) + shape = (1,) * N + if sparse: + res = () + else: + res = torch.empty((N,) + dimensions, dtype=dtype) + for i, dim in enumerate(dimensions): + idx = torch.arange(dim, dtype=dtype).reshape( + shape[:i] + (dim,) + shape[i + 1 :] + ) + if sparse: + res = res + (idx,) + else: + res[i] = idx + return res + + +# ### tri*-something ### + + +def tril(m: ArrayLike, k=0): + return torch.tril(m, k) + + +def triu(m: ArrayLike, k=0): + return torch.triu(m, k) + + +def tril_indices(n, k=0, m=None): + if m is None: + m = n + return torch.tril_indices(n, m, offset=k) + + +def triu_indices(n, k=0, m=None): + if m is None: + m = n + return torch.triu_indices(n, m, offset=k) + + +def tril_indices_from(arr: ArrayLike, k=0): + if arr.ndim != 2: + raise ValueError("input array must be 2-d") + # Return a tensor rather than a tuple to avoid a graphbreak + return torch.tril_indices(arr.shape[0], arr.shape[1], offset=k) + + +def triu_indices_from(arr: ArrayLike, k=0): + if arr.ndim != 2: + raise ValueError("input array must be 2-d") + # Return a tensor rather than a tuple to avoid a graphbreak + return torch.triu_indices(arr.shape[0], arr.shape[1], offset=k) + + +def tri( + N, + M=None, + k=0, + dtype: Optional[DTypeLike] = None, + *, + like: NotImplementedType = None, +): + if M is None: + M = N + tensor = torch.ones((N, M), dtype=dtype) + return torch.tril(tensor, diagonal=k) + + +# ### equality, equivalence, allclose ### + + +def isclose(a: ArrayLike, b: ArrayLike, rtol=1.0e-5, atol=1.0e-8, equal_nan=False): + dtype = _dtypes_impl.result_type_impl(a, b) + a = _util.cast_if_needed(a, dtype) + b = _util.cast_if_needed(b, dtype) + return torch.isclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan) + + +def allclose(a: ArrayLike, b: ArrayLike, rtol=1e-05, atol=1e-08, equal_nan=False): + dtype = _dtypes_impl.result_type_impl(a, b) + a = _util.cast_if_needed(a, dtype) + b = _util.cast_if_needed(b, dtype) + return torch.allclose(a, b, rtol=rtol, atol=atol, equal_nan=equal_nan) + + +def _tensor_equal(a1, a2, equal_nan=False): + # Implementation of array_equal/array_equiv. + if a1.shape != a2.shape: + return False + cond = a1 == a2 + if equal_nan: + cond = cond | (torch.isnan(a1) & torch.isnan(a2)) + return cond.all().item() + + +def array_equal(a1: ArrayLike, a2: ArrayLike, equal_nan=False): + return _tensor_equal(a1, a2, equal_nan=equal_nan) + + +def array_equiv(a1: ArrayLike, a2: ArrayLike): + # *almost* the same as array_equal: _equiv tries to broadcast, _equal does not + try: + a1_t, a2_t = torch.broadcast_tensors(a1, a2) + except RuntimeError: + # failed to broadcast => not equivalent + return False + return _tensor_equal(a1_t, a2_t) + + +def nan_to_num( + x: ArrayLike, copy: NotImplementedType = True, nan=0.0, posinf=None, neginf=None +): + # work around RuntimeError: "nan_to_num" not implemented for 'ComplexDouble' + if x.is_complex(): + re = torch.nan_to_num(x.real, nan=nan, posinf=posinf, neginf=neginf) + im = torch.nan_to_num(x.imag, nan=nan, posinf=posinf, neginf=neginf) + return re + 1j * im + else: + return torch.nan_to_num(x, nan=nan, posinf=posinf, neginf=neginf) + + +# ### put/take_along_axis ### + + +def take( + a: ArrayLike, + indices: ArrayLike, + axis=None, + out: Optional[OutArray] = None, + mode: NotImplementedType = "raise", +): + (a,), axis = _util.axis_none_flatten(a, axis=axis) + axis = _util.normalize_axis_index(axis, a.ndim) + idx = (slice(None),) * axis + (indices, ...) + result = a[idx] + return result + + +def take_along_axis(arr: ArrayLike, indices: ArrayLike, axis): + (arr,), axis = _util.axis_none_flatten(arr, axis=axis) + axis = _util.normalize_axis_index(axis, arr.ndim) + return torch.take_along_dim(arr, indices, axis) + + +def put( + a: NDArray, + indices: ArrayLike, + values: ArrayLike, + mode: NotImplementedType = "raise", +): + v = values.type(a.dtype) + # If indices is larger than v, expand v to at least the size of indices. Any + # unnecessary trailing elements are then trimmed. + if indices.numel() > v.numel(): + ratio = (indices.numel() + v.numel() - 1) // v.numel() + v = v.unsqueeze(0).expand((ratio,) + v.shape) + # Trim unnecessary elements, regardless if v was expanded or not. Note + # np.put() trims v to match indices by default too. + if indices.numel() < v.numel(): + v = v.flatten() + v = v[: indices.numel()] + a.put_(indices, v) + return None + + +def put_along_axis(arr: ArrayLike, indices: ArrayLike, values: ArrayLike, axis): + (arr,), axis = _util.axis_none_flatten(arr, axis=axis) + axis = _util.normalize_axis_index(axis, arr.ndim) + + indices, values = torch.broadcast_tensors(indices, values) + values = _util.cast_if_needed(values, arr.dtype) + result = torch.scatter(arr, axis, indices, values) + arr.copy_(result.reshape(arr.shape)) + return None + + +def choose( + a: ArrayLike, + choices: Sequence[ArrayLike], + out: Optional[OutArray] = None, + mode: NotImplementedType = "raise", +): + # First, broadcast elements of `choices` + choices = torch.stack(torch.broadcast_tensors(*choices)) + + # Use an analog of `gather(choices, 0, a)` which broadcasts `choices` vs `a`: + # (taken from https://github.com/pytorch/pytorch/issues/9407#issuecomment-1427907939) + idx_list = [ + torch.arange(dim).view((1,) * i + (dim,) + (1,) * (choices.ndim - i - 1)) + for i, dim in enumerate(choices.shape) + ] + + idx_list[0] = a + return choices[tuple(idx_list)].squeeze(0) + + +# ### unique et al. ### + + +def unique( + ar: ArrayLike, + return_index: NotImplementedType = False, + return_inverse=False, + return_counts=False, + axis=None, + *, + equal_nan: NotImplementedType = True, +): + (ar,), axis = _util.axis_none_flatten(ar, axis=axis) + axis = _util.normalize_axis_index(axis, ar.ndim) + + result = torch.unique( + ar, return_inverse=return_inverse, return_counts=return_counts, dim=axis + ) + + return result + + +def nonzero(a: ArrayLike): + return torch.nonzero(a, as_tuple=True) + + +def argwhere(a: ArrayLike): + return torch.argwhere(a) + + +def flatnonzero(a: ArrayLike): + return torch.flatten(a).nonzero(as_tuple=True)[0] + + +def clip( + a: ArrayLike, + min: Optional[ArrayLike] = None, + max: Optional[ArrayLike] = None, + out: Optional[OutArray] = None, +): + return torch.clamp(a, min, max) + + +def repeat(a: ArrayLike, repeats: ArrayLikeOrScalar, axis=None): + return torch.repeat_interleave(a, repeats, axis) + + +def tile(A: ArrayLike, reps): + if isinstance(reps, int): + reps = (reps,) + return torch.tile(A, reps) + + +def resize(a: ArrayLike, new_shape=None): + # implementation vendored from + # https://github.com/numpy/numpy/blob/v1.24.0/numpy/core/fromnumeric.py#L1420-L1497 + if new_shape is None: + return a + + if isinstance(new_shape, int): + new_shape = (new_shape,) + + a = a.flatten() + + new_size = 1 + for dim_length in new_shape: + new_size *= dim_length + if dim_length < 0: + raise ValueError("all elements of `new_shape` must be non-negative") + + if a.numel() == 0 or new_size == 0: + # First case must zero fill. The second would have repeats == 0. + return torch.zeros(new_shape, dtype=a.dtype) + + repeats = -(-new_size // a.numel()) # ceil division + a = concatenate((a,) * repeats)[:new_size] + + return reshape(a, new_shape) + + +# ### diag et al. ### + + +def diagonal(a: ArrayLike, offset=0, axis1=0, axis2=1): + axis1 = _util.normalize_axis_index(axis1, a.ndim) + axis2 = _util.normalize_axis_index(axis2, a.ndim) + return torch.diagonal(a, offset, axis1, axis2) + + +def trace( + a: ArrayLike, + offset=0, + axis1=0, + axis2=1, + dtype: Optional[DTypeLike] = None, + out: Optional[OutArray] = None, +): + result = torch.diagonal(a, offset, dim1=axis1, dim2=axis2).sum(-1, dtype=dtype) + return result + + +def eye( + N, + M=None, + k=0, + dtype: Optional[DTypeLike] = None, + order: NotImplementedType = "C", + *, + like: NotImplementedType = None, +): + if dtype is None: + dtype = _dtypes_impl.default_dtypes().float_dtype + if M is None: + M = N + z = torch.zeros(N, M, dtype=dtype) + z.diagonal(k).fill_(1) + return z + + +def identity(n, dtype: Optional[DTypeLike] = None, *, like: NotImplementedType = None): + return torch.eye(n, dtype=dtype) + + +def diag(v: ArrayLike, k=0): + return torch.diag(v, k) + + +def diagflat(v: ArrayLike, k=0): + return torch.diagflat(v, k) + + +def diag_indices(n, ndim=2): + idx = torch.arange(n) + return (idx,) * ndim + + +def diag_indices_from(arr: ArrayLike): + if not arr.ndim >= 2: + raise ValueError("input array must be at least 2-d") + # For more than d=2, the strided formula is only valid for arrays with + # all dimensions equal, so we check first. + s = arr.shape + if s[1:] != s[:-1]: + raise ValueError("All dimensions of input must be of equal length") + return diag_indices(s[0], arr.ndim) + + +def fill_diagonal(a: ArrayLike, val: ArrayLike, wrap=False): + if a.ndim < 2: + raise ValueError("array must be at least 2-d") + if val.numel() == 0 and not wrap: + a.fill_diagonal_(val) + return a + + if val.ndim == 0: + val = val.unsqueeze(0) + + # torch.Tensor.fill_diagonal_ only accepts scalars + # If the size of val is too large, then val is trimmed + if a.ndim == 2: + tall = a.shape[0] > a.shape[1] + # wrap does nothing for wide matrices... + if not wrap or not tall: + # Never wraps + diag = a.diagonal() + diag.copy_(val[: diag.numel()]) + else: + # wraps and tall... leaving one empty line between diagonals?! + max_, min_ = a.shape + idx = torch.arange(max_ - max_ // (min_ + 1)) + mod = idx % min_ + div = idx // min_ + a[(div * (min_ + 1) + mod, mod)] = val[: idx.numel()] + else: + idx = diag_indices_from(a) + # a.shape = (n, n, ..., n) + a[idx] = val[: a.shape[0]] + + return a + + +def vdot(a: ArrayLike, b: ArrayLike, /): + # 1. torch only accepts 1D arrays, numpy flattens + # 2. torch requires matching dtype, while numpy casts (?) + t_a, t_b = torch.atleast_1d(a, b) + if t_a.ndim > 1: + t_a = t_a.flatten() + if t_b.ndim > 1: + t_b = t_b.flatten() + + dtype = _dtypes_impl.result_type_impl(t_a, t_b) + is_half = dtype == torch.float16 and (t_a.is_cpu or t_b.is_cpu) + is_bool = dtype == torch.bool + + # work around torch's "dot" not implemented for 'Half', 'Bool' + if is_half: + dtype = torch.float32 + elif is_bool: + dtype = torch.uint8 + + t_a = _util.cast_if_needed(t_a, dtype) + t_b = _util.cast_if_needed(t_b, dtype) + + result = torch.vdot(t_a, t_b) + + if is_half: + result = result.to(torch.float16) + elif is_bool: + result = result.to(torch.bool) + + return result + + +def tensordot(a: ArrayLike, b: ArrayLike, axes=2): + if isinstance(axes, (list, tuple)): + axes = [[ax] if isinstance(ax, int) else ax for ax in axes] + + target_dtype = _dtypes_impl.result_type_impl(a, b) + a = _util.cast_if_needed(a, target_dtype) + b = _util.cast_if_needed(b, target_dtype) + + return torch.tensordot(a, b, dims=axes) + + +def dot(a: ArrayLike, b: ArrayLike, out: Optional[OutArray] = None): + dtype = _dtypes_impl.result_type_impl(a, b) + is_bool = dtype == torch.bool + if is_bool: + dtype = torch.uint8 + + a = _util.cast_if_needed(a, dtype) + b = _util.cast_if_needed(b, dtype) + + if a.ndim == 0 or b.ndim == 0: + result = a * b + else: + result = torch.matmul(a, b) + + if is_bool: + result = result.to(torch.bool) + + return result + + +def inner(a: ArrayLike, b: ArrayLike, /): + dtype = _dtypes_impl.result_type_impl(a, b) + is_half = dtype == torch.float16 and (a.is_cpu or b.is_cpu) + is_bool = dtype == torch.bool + + if is_half: + # work around torch's "addmm_impl_cpu_" not implemented for 'Half'" + dtype = torch.float32 + elif is_bool: + dtype = torch.uint8 + + a = _util.cast_if_needed(a, dtype) + b = _util.cast_if_needed(b, dtype) + + result = torch.inner(a, b) + + if is_half: + result = result.to(torch.float16) + elif is_bool: + result = result.to(torch.bool) + return result + + +def outer(a: ArrayLike, b: ArrayLike, out: Optional[OutArray] = None): + return torch.outer(a, b) + + +def cross(a: ArrayLike, b: ArrayLike, axisa=-1, axisb=-1, axisc=-1, axis=None): + # implementation vendored from + # https://github.com/numpy/numpy/blob/v1.24.0/numpy/core/numeric.py#L1486-L1685 + if axis is not None: + axisa, axisb, axisc = (axis,) * 3 + + # Check axisa and axisb are within bounds + axisa = _util.normalize_axis_index(axisa, a.ndim) + axisb = _util.normalize_axis_index(axisb, b.ndim) + + # Move working axis to the end of the shape + a = torch.moveaxis(a, axisa, -1) + b = torch.moveaxis(b, axisb, -1) + msg = "incompatible dimensions for cross product\n(dimension must be 2 or 3)" + if a.shape[-1] not in (2, 3) or b.shape[-1] not in (2, 3): + raise ValueError(msg) + + # Create the output array + shape = broadcast_shapes(a[..., 0].shape, b[..., 0].shape) + if a.shape[-1] == 3 or b.shape[-1] == 3: + shape += (3,) + # Check axisc is within bounds + axisc = _util.normalize_axis_index(axisc, len(shape)) + dtype = _dtypes_impl.result_type_impl(a, b) + cp = torch.empty(shape, dtype=dtype) + + # recast arrays as dtype + a = _util.cast_if_needed(a, dtype) + b = _util.cast_if_needed(b, dtype) + + # create local aliases for readability + a0 = a[..., 0] + a1 = a[..., 1] + if a.shape[-1] == 3: + a2 = a[..., 2] + b0 = b[..., 0] + b1 = b[..., 1] + if b.shape[-1] == 3: + b2 = b[..., 2] + if cp.ndim != 0 and cp.shape[-1] == 3: + cp0 = cp[..., 0] + cp1 = cp[..., 1] + cp2 = cp[..., 2] + + if a.shape[-1] == 2: + if b.shape[-1] == 2: + # a0 * b1 - a1 * b0 + cp[...] = a0 * b1 - a1 * b0 + return cp + else: + assert b.shape[-1] == 3 + # cp0 = a1 * b2 - 0 (a2 = 0) + # cp1 = 0 - a0 * b2 (a2 = 0) + # cp2 = a0 * b1 - a1 * b0 + cp0[...] = a1 * b2 + cp1[...] = -a0 * b2 + cp2[...] = a0 * b1 - a1 * b0 + else: + assert a.shape[-1] == 3 + if b.shape[-1] == 3: + cp0[...] = a1 * b2 - a2 * b1 + cp1[...] = a2 * b0 - a0 * b2 + cp2[...] = a0 * b1 - a1 * b0 + else: + assert b.shape[-1] == 2 + cp0[...] = -a2 * b1 + cp1[...] = a2 * b0 + cp2[...] = a0 * b1 - a1 * b0 + + return torch.moveaxis(cp, -1, axisc) + + +def einsum(*operands, out=None, dtype=None, order="K", casting="safe", optimize=False): + # Have to manually normalize *operands and **kwargs, following the NumPy signature + # We have a local import to avoid polluting the global space, as it will be then + # exported in funcs.py + from ._ndarray import ndarray + from ._normalizations import ( + maybe_copy_to, + normalize_array_like, + normalize_casting, + normalize_dtype, + wrap_tensors, + ) + + dtype = normalize_dtype(dtype) + casting = normalize_casting(casting) + if out is not None and not isinstance(out, ndarray): + raise TypeError("'out' must be an array") + if order != "K": + raise NotImplementedError("'order' parameter is not supported.") + + # parse arrays and normalize them + sublist_format = not isinstance(operands[0], str) + if sublist_format: + # op, str, op, str ... [sublistout] format: normalize every other argument + + # - if sublistout is not given, the length of operands is even, and we pick + # odd-numbered elements, which are arrays. + # - if sublistout is given, the length of operands is odd, we peel off + # the last one, and pick odd-numbered elements, which are arrays. + # Without [:-1], we would have picked sublistout, too. + array_operands = operands[:-1][::2] + else: + # ("ij->", arrays) format + subscripts, array_operands = operands[0], operands[1:] + + tensors = [normalize_array_like(op) for op in array_operands] + target_dtype = _dtypes_impl.result_type_impl(*tensors) if dtype is None else dtype + + # work around 'bmm' not implemented for 'Half' etc + is_half = target_dtype == torch.float16 and all(t.is_cpu for t in tensors) + if is_half: + target_dtype = torch.float32 + + is_short_int = target_dtype in [torch.uint8, torch.int8, torch.int16, torch.int32] + if is_short_int: + target_dtype = torch.int64 + + tensors = _util.typecast_tensors(tensors, target_dtype, casting) + + from torch.backends import opt_einsum + + try: + # set the global state to handle the optimize=... argument, restore on exit + if opt_einsum.is_available(): + old_strategy = torch.backends.opt_einsum.strategy + old_enabled = torch.backends.opt_einsum.enabled + + # torch.einsum calls opt_einsum.contract_path, which runs into + # https://github.com/dgasmith/opt_einsum/issues/219 + # for strategy={True, False} + if optimize is True: + optimize = "auto" + elif optimize is False: + torch.backends.opt_einsum.enabled = False + + torch.backends.opt_einsum.strategy = optimize + + if sublist_format: + # recombine operands + sublists = operands[1::2] + has_sublistout = len(operands) % 2 == 1 + if has_sublistout: + sublistout = operands[-1] + operands = list(itertools.chain.from_iterable(zip(tensors, sublists))) + if has_sublistout: + operands.append(sublistout) + + result = torch.einsum(*operands) + else: + result = torch.einsum(subscripts, *tensors) + + finally: + if opt_einsum.is_available(): + torch.backends.opt_einsum.strategy = old_strategy + torch.backends.opt_einsum.enabled = old_enabled + + result = maybe_copy_to(out, result) + return wrap_tensors(result) + + +# ### sort and partition ### + + +def _sort_helper(tensor, axis, kind, order): + if tensor.dtype.is_complex: + raise NotImplementedError(f"sorting {tensor.dtype} is not supported") + (tensor,), axis = _util.axis_none_flatten(tensor, axis=axis) + axis = _util.normalize_axis_index(axis, tensor.ndim) + + stable = kind == "stable" + + return tensor, axis, stable + + +def sort(a: ArrayLike, axis=-1, kind=None, order: NotImplementedType = None): + # `order` keyword arg is only relevant for structured dtypes; so not supported here. + a, axis, stable = _sort_helper(a, axis, kind, order) + result = torch.sort(a, dim=axis, stable=stable) + return result.values + + +def argsort(a: ArrayLike, axis=-1, kind=None, order: NotImplementedType = None): + a, axis, stable = _sort_helper(a, axis, kind, order) + return torch.argsort(a, dim=axis, stable=stable) + + +def searchsorted( + a: ArrayLike, v: ArrayLike, side="left", sorter: Optional[ArrayLike] = None +): + if a.dtype.is_complex: + raise NotImplementedError(f"searchsorted with dtype={a.dtype}") + + return torch.searchsorted(a, v, side=side, sorter=sorter) + + +# ### swap/move/roll axis ### + + +def moveaxis(a: ArrayLike, source, destination): + source = _util.normalize_axis_tuple(source, a.ndim, "source") + destination = _util.normalize_axis_tuple(destination, a.ndim, "destination") + return torch.moveaxis(a, source, destination) + + +def swapaxes(a: ArrayLike, axis1, axis2): + axis1 = _util.normalize_axis_index(axis1, a.ndim) + axis2 = _util.normalize_axis_index(axis2, a.ndim) + return torch.swapaxes(a, axis1, axis2) + + +def rollaxis(a: ArrayLike, axis, start=0): + # Straight vendor from: + # https://github.com/numpy/numpy/blob/v1.24.0/numpy/core/numeric.py#L1259 + # + # Also note this function in NumPy is mostly retained for backwards compat + # (https://stackoverflow.com/questions/29891583/reason-why-numpy-rollaxis-is-so-confusing) + # so let's not touch it unless hard pressed. + n = a.ndim + axis = _util.normalize_axis_index(axis, n) + if start < 0: + start += n + msg = "'%s' arg requires %d <= %s < %d, but %d was passed in" + if not (0 <= start < n + 1): + raise _util.AxisError(msg % ("start", -n, "start", n + 1, start)) + if axis < start: + # it's been removed + start -= 1 + if axis == start: + # numpy returns a view, here we try returning the tensor itself + # return tensor[...] + return a + axes = list(range(n)) + axes.remove(axis) + axes.insert(start, axis) + return a.view(axes) + + +def roll(a: ArrayLike, shift, axis=None): + if axis is not None: + axis = _util.normalize_axis_tuple(axis, a.ndim, allow_duplicate=True) + if not isinstance(shift, tuple): + shift = (shift,) * len(axis) + return torch.roll(a, shift, axis) + + +# ### shape manipulations ### + + +def squeeze(a: ArrayLike, axis=None): + if axis == (): + result = a + elif axis is None: + result = a.squeeze() + else: + if isinstance(axis, tuple): + result = a + for ax in axis: + result = a.squeeze(ax) + else: + result = a.squeeze(axis) + return result + + +def reshape(a: ArrayLike, newshape, order: NotImplementedType = "C"): + # if sh = (1, 2, 3), numpy allows both .reshape(sh) and .reshape(*sh) + newshape = newshape[0] if len(newshape) == 1 else newshape + return a.reshape(newshape) + + +# NB: cannot use torch.reshape(a, newshape) above, because of +# (Pdb) torch.reshape(torch.as_tensor([1]), 1) +# *** TypeError: reshape(): argument 'shape' (position 2) must be tuple of SymInts, not int + + +def transpose(a: ArrayLike, axes=None): + # numpy allows both .transpose(sh) and .transpose(*sh) + # also older code uses axes being a list + if axes in [(), None, (None,)]: + axes = tuple(reversed(range(a.ndim))) + elif len(axes) == 1: + axes = axes[0] + return a.permute(axes) + + +def ravel(a: ArrayLike, order: NotImplementedType = "C"): + return torch.flatten(a) + + +def diff( + a: ArrayLike, + n=1, + axis=-1, + prepend: Optional[ArrayLike] = None, + append: Optional[ArrayLike] = None, +): + axis = _util.normalize_axis_index(axis, a.ndim) + + if n < 0: + raise ValueError(f"order must be non-negative but got {n}") + + if n == 0: + # match numpy and return the input immediately + return a + + if prepend is not None: + shape = list(a.shape) + shape[axis] = prepend.shape[axis] if prepend.ndim > 0 else 1 + prepend = torch.broadcast_to(prepend, shape) + + if append is not None: + shape = list(a.shape) + shape[axis] = append.shape[axis] if append.ndim > 0 else 1 + append = torch.broadcast_to(append, shape) + + return torch.diff(a, n, axis=axis, prepend=prepend, append=append) + + +# ### math functions ### + + +def angle(z: ArrayLike, deg=False): + result = torch.angle(z) + if deg: + result = result * (180 / torch.pi) + return result + + +def sinc(x: ArrayLike): + return torch.sinc(x) + + +# NB: have to normalize *varargs manually +def gradient(f: ArrayLike, *varargs, axis=None, edge_order=1): + N = f.ndim # number of dimensions + + varargs = _util.ndarrays_to_tensors(varargs) + + if axis is None: + axes = tuple(range(N)) + else: + axes = _util.normalize_axis_tuple(axis, N) + + len_axes = len(axes) + n = len(varargs) + if n == 0: + # no spacing argument - use 1 in all axes + dx = [1.0] * len_axes + elif n == 1 and (_dtypes_impl.is_scalar(varargs[0]) or varargs[0].ndim == 0): + # single scalar or 0D tensor for all axes (np.ndim(varargs[0]) == 0) + dx = varargs * len_axes + elif n == len_axes: + # scalar or 1d array for each axis + dx = list(varargs) + for i, distances in enumerate(dx): + distances = torch.as_tensor(distances) + if distances.ndim == 0: + continue + elif distances.ndim != 1: + raise ValueError("distances must be either scalars or 1d") + if len(distances) != f.shape[axes[i]]: + raise ValueError( + "when 1d, distances must match " + "the length of the corresponding dimension" + ) + if not (distances.dtype.is_floating_point or distances.dtype.is_complex): + distances = distances.double() + + diffx = torch.diff(distances) + # if distances are constant reduce to the scalar case + # since it brings a consistent speedup + if (diffx == diffx[0]).all(): + diffx = diffx[0] + dx[i] = diffx + else: + raise TypeError("invalid number of arguments") + + if edge_order > 2: + raise ValueError("'edge_order' greater than 2 not supported") + + # use central differences on interior and one-sided differences on the + # endpoints. This preserves second order-accuracy over the full domain. + + outvals = [] + + # create slice objects --- initially all are [:, :, ..., :] + slice1 = [slice(None)] * N + slice2 = [slice(None)] * N + slice3 = [slice(None)] * N + slice4 = [slice(None)] * N + + otype = f.dtype + if _dtypes_impl.python_type_for_torch(otype) in (int, bool): + # Convert to floating point. + # First check if f is a numpy integer type; if so, convert f to float64 + # to avoid modular arithmetic when computing the changes in f. + f = f.double() + otype = torch.float64 + + for axis, ax_dx in zip(axes, dx): + if f.shape[axis] < edge_order + 1: + raise ValueError( + "Shape of array too small to calculate a numerical gradient, " + "at least (edge_order + 1) elements are required." + ) + # result allocation + out = torch.empty_like(f, dtype=otype) + + # spacing for the current axis (NB: np.ndim(ax_dx) == 0) + uniform_spacing = _dtypes_impl.is_scalar(ax_dx) or ax_dx.ndim == 0 + + # Numerical differentiation: 2nd order interior + slice1[axis] = slice(1, -1) + slice2[axis] = slice(None, -2) + slice3[axis] = slice(1, -1) + slice4[axis] = slice(2, None) + + if uniform_spacing: + out[tuple(slice1)] = (f[tuple(slice4)] - f[tuple(slice2)]) / (2.0 * ax_dx) + else: + dx1 = ax_dx[0:-1] + dx2 = ax_dx[1:] + a = -(dx2) / (dx1 * (dx1 + dx2)) + b = (dx2 - dx1) / (dx1 * dx2) + c = dx1 / (dx2 * (dx1 + dx2)) + # fix the shape for broadcasting + shape = [1] * N + shape[axis] = -1 + a = a.reshape(shape) + b = b.reshape(shape) + c = c.reshape(shape) + # 1D equivalent -- out[1:-1] = a * f[:-2] + b * f[1:-1] + c * f[2:] + out[tuple(slice1)] = ( + a * f[tuple(slice2)] + b * f[tuple(slice3)] + c * f[tuple(slice4)] + ) + + # Numerical differentiation: 1st order edges + if edge_order == 1: + slice1[axis] = 0 + slice2[axis] = 1 + slice3[axis] = 0 + dx_0 = ax_dx if uniform_spacing else ax_dx[0] + # 1D equivalent -- out[0] = (f[1] - f[0]) / (x[1] - x[0]) + out[tuple(slice1)] = (f[tuple(slice2)] - f[tuple(slice3)]) / dx_0 + + slice1[axis] = -1 + slice2[axis] = -1 + slice3[axis] = -2 + dx_n = ax_dx if uniform_spacing else ax_dx[-1] + # 1D equivalent -- out[-1] = (f[-1] - f[-2]) / (x[-1] - x[-2]) + out[tuple(slice1)] = (f[tuple(slice2)] - f[tuple(slice3)]) / dx_n + + # Numerical differentiation: 2nd order edges + else: + slice1[axis] = 0 + slice2[axis] = 0 + slice3[axis] = 1 + slice4[axis] = 2 + if uniform_spacing: + a = -1.5 / ax_dx + b = 2.0 / ax_dx + c = -0.5 / ax_dx + else: + dx1 = ax_dx[0] + dx2 = ax_dx[1] + a = -(2.0 * dx1 + dx2) / (dx1 * (dx1 + dx2)) + b = (dx1 + dx2) / (dx1 * dx2) + c = -dx1 / (dx2 * (dx1 + dx2)) + # 1D equivalent -- out[0] = a * f[0] + b * f[1] + c * f[2] + out[tuple(slice1)] = ( + a * f[tuple(slice2)] + b * f[tuple(slice3)] + c * f[tuple(slice4)] + ) + + slice1[axis] = -1 + slice2[axis] = -3 + slice3[axis] = -2 + slice4[axis] = -1 + if uniform_spacing: + a = 0.5 / ax_dx + b = -2.0 / ax_dx + c = 1.5 / ax_dx + else: + dx1 = ax_dx[-2] + dx2 = ax_dx[-1] + a = (dx2) / (dx1 * (dx1 + dx2)) + b = -(dx2 + dx1) / (dx1 * dx2) + c = (2.0 * dx2 + dx1) / (dx2 * (dx1 + dx2)) + # 1D equivalent -- out[-1] = a * f[-3] + b * f[-2] + c * f[-1] + out[tuple(slice1)] = ( + a * f[tuple(slice2)] + b * f[tuple(slice3)] + c * f[tuple(slice4)] + ) + + outvals.append(out) + + # reset the slice object in this dimension to ":" + slice1[axis] = slice(None) + slice2[axis] = slice(None) + slice3[axis] = slice(None) + slice4[axis] = slice(None) + + if len_axes == 1: + return outvals[0] + else: + return outvals + + +# ### Type/shape etc queries ### + + +def round(a: ArrayLike, decimals=0, out: Optional[OutArray] = None): + if a.is_floating_point(): + result = torch.round(a, decimals=decimals) + elif a.is_complex(): + # RuntimeError: "round_cpu" not implemented for 'ComplexFloat' + result = torch.complex( + torch.round(a.real, decimals=decimals), + torch.round(a.imag, decimals=decimals), + ) + else: + # RuntimeError: "round_cpu" not implemented for 'int' + result = a + return result + + +around = round +round_ = round + + +def real_if_close(a: ArrayLike, tol=100): + if not torch.is_complex(a): + return a + if tol > 1: + # Undocumented in numpy: if tol < 1, it's an absolute tolerance! + # Otherwise, tol > 1 is relative tolerance, in units of the dtype epsilon + # https://github.com/numpy/numpy/blob/v1.24.0/numpy/lib/type_check.py#L577 + tol = tol * torch.finfo(a.dtype).eps + + mask = torch.abs(a.imag) < tol + return a.real if mask.all() else a + + +def real(a: ArrayLike): + return torch.real(a) + + +def imag(a: ArrayLike): + if a.is_complex(): + return a.imag + return torch.zeros_like(a) + + +def iscomplex(x: ArrayLike): + if torch.is_complex(x): + return x.imag != 0 + return torch.zeros_like(x, dtype=torch.bool) + + +def isreal(x: ArrayLike): + if torch.is_complex(x): + return x.imag == 0 + return torch.ones_like(x, dtype=torch.bool) + + +def iscomplexobj(x: ArrayLike): + return torch.is_complex(x) + + +def isrealobj(x: ArrayLike): + return not torch.is_complex(x) + + +def isneginf(x: ArrayLike, out: Optional[OutArray] = None): + return torch.isneginf(x) + + +def isposinf(x: ArrayLike, out: Optional[OutArray] = None): + return torch.isposinf(x) + + +def i0(x: ArrayLike): + return torch.special.i0(x) + + +def isscalar(a): + # We need to use normalize_array_like, but we don't want to export it in funcs.py + from ._normalizations import normalize_array_like + + try: + t = normalize_array_like(a) + return t.numel() == 1 + except Exception: + return False + + +# ### Filter windows ### + + +def hamming(M): + dtype = _dtypes_impl.default_dtypes().float_dtype + return torch.hamming_window(M, periodic=False, dtype=dtype) + + +def hanning(M): + dtype = _dtypes_impl.default_dtypes().float_dtype + return torch.hann_window(M, periodic=False, dtype=dtype) + + +def kaiser(M, beta): + dtype = _dtypes_impl.default_dtypes().float_dtype + return torch.kaiser_window(M, beta=beta, periodic=False, dtype=dtype) + + +def blackman(M): + dtype = _dtypes_impl.default_dtypes().float_dtype + return torch.blackman_window(M, periodic=False, dtype=dtype) + + +def bartlett(M): + dtype = _dtypes_impl.default_dtypes().float_dtype + return torch.bartlett_window(M, periodic=False, dtype=dtype) + + +# ### Dtype routines ### + +# vendored from https://github.com/numpy/numpy/blob/v1.24.0/numpy/lib/type_check.py#L666 + + +array_type = [ + [torch.float16, torch.float32, torch.float64], + [None, torch.complex64, torch.complex128], +] +array_precision = { + torch.float16: 0, + torch.float32: 1, + torch.float64: 2, + torch.complex64: 1, + torch.complex128: 2, +} + + +def common_type(*tensors: ArrayLike): + is_complex = False + precision = 0 + for a in tensors: + t = a.dtype + if iscomplexobj(a): + is_complex = True + if not (t.is_floating_point or t.is_complex): + p = 2 # array_precision[_nx.double] + else: + p = array_precision.get(t) + if p is None: + raise TypeError("can't get common type for non-numeric array") + precision = builtins.max(precision, p) + if is_complex: + return array_type[1][precision] + else: + return array_type[0][precision] + + +# ### histograms ### + + +def histogram( + a: ArrayLike, + bins: ArrayLike = 10, + range=None, + normed=None, + weights: Optional[ArrayLike] = None, + density=None, +): + if normed is not None: + raise ValueError("normed argument is deprecated, use density= instead") + + if weights is not None and weights.dtype.is_complex: + raise NotImplementedError("complex weights histogram.") + + is_a_int = not (a.dtype.is_floating_point or a.dtype.is_complex) + is_w_int = weights is None or not weights.dtype.is_floating_point + if is_a_int: + a = a.double() + + if weights is not None: + weights = _util.cast_if_needed(weights, a.dtype) + + if isinstance(bins, torch.Tensor): + if bins.ndim == 0: + # bins was a single int + bins = operator.index(bins) + else: + bins = _util.cast_if_needed(bins, a.dtype) + + if range is None: + h, b = torch.histogram(a, bins, weight=weights, density=bool(density)) + else: + h, b = torch.histogram( + a, bins, range=range, weight=weights, density=bool(density) + ) + + if not density and is_w_int: + h = h.long() + if is_a_int: + b = b.long() + + return h, b + + +def histogram2d( + x, + y, + bins=10, + range: Optional[ArrayLike] = None, + normed=None, + weights: Optional[ArrayLike] = None, + density=None, +): + # vendored from https://github.com/numpy/numpy/blob/v1.24.0/numpy/lib/twodim_base.py#L655-L821 + if len(x) != len(y): + raise ValueError("x and y must have the same length.") + + try: + N = len(bins) + except TypeError: + N = 1 + + if N != 1 and N != 2: + bins = [bins, bins] + + h, e = histogramdd((x, y), bins, range, normed, weights, density) + + return h, e[0], e[1] + + +def histogramdd( + sample, + bins=10, + range: Optional[ArrayLike] = None, + normed=None, + weights: Optional[ArrayLike] = None, + density=None, +): + # have to normalize manually because `sample` interpretation differs + # for a list of lists and a 2D array + if normed is not None: + raise ValueError("normed argument is deprecated, use density= instead") + + from ._normalizations import normalize_array_like, normalize_seq_array_like + + if isinstance(sample, (list, tuple)): + sample = normalize_array_like(sample).T + else: + sample = normalize_array_like(sample) + + sample = torch.atleast_2d(sample) + + if not (sample.dtype.is_floating_point or sample.dtype.is_complex): + sample = sample.double() + + # bins is either an int, or a sequence of ints or a sequence of arrays + bins_is_array = not ( + isinstance(bins, int) or builtins.all(isinstance(b, int) for b in bins) + ) + if bins_is_array: + bins = normalize_seq_array_like(bins) + bins_dtypes = [b.dtype for b in bins] + bins = [_util.cast_if_needed(b, sample.dtype) for b in bins] + + if range is not None: + range = range.flatten().tolist() + + if weights is not None: + # range=... is required : interleave min and max values per dimension + mm = sample.aminmax(dim=0) + range = torch.cat(mm).reshape(2, -1).T.flatten() + range = tuple(range.tolist()) + weights = _util.cast_if_needed(weights, sample.dtype) + w_kwd = {"weight": weights} + else: + w_kwd = {} + + h, b = torch.histogramdd(sample, bins, range, density=bool(density), **w_kwd) + + if bins_is_array: + b = [_util.cast_if_needed(bb, dtyp) for bb, dtyp in zip(b, bins_dtypes)] + + return h, b + + +# ### odds and ends + + +def min_scalar_type(a: ArrayLike, /): + # https://github.com/numpy/numpy/blob/maintenance/1.24.x/numpy/core/src/multiarray/convert_datatype.c#L1288 + + from ._dtypes import DType + + if a.numel() > 1: + # numpy docs: "For non-scalar array a, returns the vector's dtype unmodified." + return DType(a.dtype) + + if a.dtype == torch.bool: + dtype = torch.bool + + elif a.dtype.is_complex: + fi = torch.finfo(torch.float32) + fits_in_single = a.dtype == torch.complex64 or ( + fi.min <= a.real <= fi.max and fi.min <= a.imag <= fi.max + ) + dtype = torch.complex64 if fits_in_single else torch.complex128 + + elif a.dtype.is_floating_point: + for dt in [torch.float16, torch.float32, torch.float64]: + fi = torch.finfo(dt) + if fi.min <= a <= fi.max: + dtype = dt + break + else: + # must be integer + for dt in [torch.uint8, torch.int8, torch.int16, torch.int32, torch.int64]: + # Prefer unsigned int where possible, as numpy does. + ii = torch.iinfo(dt) + if ii.min <= a <= ii.max: + dtype = dt + break + + return DType(dtype) + + +def pad(array: ArrayLike, pad_width: ArrayLike, mode="constant", **kwargs): + if mode != "constant": + raise NotImplementedError + value = kwargs.get("constant_values", 0) + # `value` must be a python scalar for torch.nn.functional.pad + typ = _dtypes_impl.python_type_for_torch(array.dtype) + value = typ(value) + + pad_width = torch.broadcast_to(pad_width, (array.ndim, 2)) + pad_width = torch.flip(pad_width, (0,)).flatten() + + return torch.nn.functional.pad(array, tuple(pad_width), value=value) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_getlimits.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_getlimits.py new file mode 100644 index 0000000000000000000000000000000000000000..b0c46094e8782f8309ba6cfa15cb59f93cf667a0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_getlimits.py @@ -0,0 +1,15 @@ +# mypy: ignore-errors + +import torch + +from . import _dtypes + + +def finfo(dtyp): + torch_dtype = _dtypes.dtype(dtyp).torch_dtype + return torch.finfo(torch_dtype) + + +def iinfo(dtyp): + torch_dtype = _dtypes.dtype(dtyp).torch_dtype + return torch.iinfo(torch_dtype) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_ndarray.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_ndarray.py new file mode 100644 index 0000000000000000000000000000000000000000..e3f3836754017af5f018e3b81688ba6022f25c9b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_ndarray.py @@ -0,0 +1,720 @@ +# mypy: ignore-errors + +from __future__ import annotations + +import builtins +import math +import operator +from collections.abc import Sequence + +import torch + +from . import _dtypes, _dtypes_impl, _funcs, _ufuncs, _util +from ._normalizations import ( + ArrayLike, + normalize_array_like, + normalizer, + NotImplementedType, +) + + +newaxis = None + +FLAGS = [ + "C_CONTIGUOUS", + "F_CONTIGUOUS", + "OWNDATA", + "WRITEABLE", + "ALIGNED", + "WRITEBACKIFCOPY", + "FNC", + "FORC", + "BEHAVED", + "CARRAY", + "FARRAY", +] + +SHORTHAND_TO_FLAGS = { + "C": "C_CONTIGUOUS", + "F": "F_CONTIGUOUS", + "O": "OWNDATA", + "W": "WRITEABLE", + "A": "ALIGNED", + "X": "WRITEBACKIFCOPY", + "B": "BEHAVED", + "CA": "CARRAY", + "FA": "FARRAY", +} + + +class Flags: + def __init__(self, flag_to_value: dict): + assert all(k in FLAGS for k in flag_to_value) # sanity check + self._flag_to_value = flag_to_value + + def __getattr__(self, attr: str): + if attr.islower() and attr.upper() in FLAGS: + return self[attr.upper()] + else: + raise AttributeError(f"No flag attribute '{attr}'") + + def __getitem__(self, key): + if key in SHORTHAND_TO_FLAGS: + key = SHORTHAND_TO_FLAGS[key] + if key in FLAGS: + try: + return self._flag_to_value[key] + except KeyError as e: + raise NotImplementedError(f"{key=}") from e + else: + raise KeyError(f"No flag key '{key}'") + + def __setattr__(self, attr, value): + if attr.islower() and attr.upper() in FLAGS: + self[attr.upper()] = value + else: + super().__setattr__(attr, value) + + def __setitem__(self, key, value): + if key in FLAGS or key in SHORTHAND_TO_FLAGS: + raise NotImplementedError("Modifying flags is not implemented") + else: + raise KeyError(f"No flag key '{key}'") + + +def create_method(fn, name=None): + name = name or fn.__name__ + + def f(*args, **kwargs): + return fn(*args, **kwargs) + + f.__name__ = name + f.__qualname__ = f"ndarray.{name}" + return f + + +# Map ndarray.name_method -> np.name_func +# If name_func == None, it means that name_method == name_func +methods = { + "clip": None, + "nonzero": None, + "repeat": None, + "round": None, + "squeeze": None, + "swapaxes": None, + "ravel": None, + # linalg + "diagonal": None, + "dot": None, + "trace": None, + # sorting + "argsort": None, + "searchsorted": None, + # reductions + "argmax": None, + "argmin": None, + "any": None, + "all": None, + "max": None, + "min": None, + "ptp": None, + "sum": None, + "prod": None, + "mean": None, + "var": None, + "std": None, + # scans + "cumsum": None, + "cumprod": None, + # advanced indexing + "take": None, + "choose": None, +} + +dunder = { + "abs": "absolute", + "invert": None, + "pos": "positive", + "neg": "negative", + "gt": "greater", + "lt": "less", + "ge": "greater_equal", + "le": "less_equal", +} + +# dunder methods with right-looking and in-place variants +ri_dunder = { + "add": None, + "sub": "subtract", + "mul": "multiply", + "truediv": "divide", + "floordiv": "floor_divide", + "pow": "power", + "mod": "remainder", + "and": "bitwise_and", + "or": "bitwise_or", + "xor": "bitwise_xor", + "lshift": "left_shift", + "rshift": "right_shift", + "matmul": None, +} + + +def _upcast_int_indices(index): + if isinstance(index, torch.Tensor): + if index.dtype in (torch.int8, torch.int16, torch.int32, torch.uint8): + return index.to(torch.int64) + elif isinstance(index, tuple): + return tuple(_upcast_int_indices(i) for i in index) + return index + + +def _has_advanced_indexing(index): + """Check if there's any advanced indexing""" + return any( + isinstance(idx, (Sequence, bool)) + or (isinstance(idx, torch.Tensor) and (idx.dtype == torch.bool or idx.ndim > 0)) + for idx in index + ) + + +def _numpy_compatible_indexing(index): + """Convert scalar indices to lists when advanced indexing is present for NumPy compatibility.""" + if not isinstance(index, tuple): + index = (index,) + + # Check if there's any advanced indexing (sequences, booleans, or tensors) + has_advanced = _has_advanced_indexing(index) + + if not has_advanced: + return index + + # Convert integer scalar indices to single-element lists when advanced indexing is present + # Note: Do NOT convert boolean scalars (True/False) as they have special meaning in NumPy + converted = [] + for idx in index: + if isinstance(idx, int) and not isinstance(idx, bool): + # Integer scalars should be converted to lists + converted.append([idx]) + elif ( + isinstance(idx, torch.Tensor) + and idx.ndim == 0 + and not torch.is_floating_point(idx) + and idx.dtype != torch.bool + ): + # Zero-dimensional tensors holding integers should be treated the same as integer scalars + converted.append([idx]) + else: + # Everything else (booleans, lists, slices, etc.) stays as is + converted.append(idx) + + return tuple(converted) + + +def _get_bool_depth(s): + """Returns the depth of a boolean sequence/tensor""" + if isinstance(s, bool): + return True, 0 + if isinstance(s, torch.Tensor) and s.dtype == torch.bool: + return True, s.ndim + if not (isinstance(s, Sequence) and s and s[0] != s): + return False, 0 + is_bool, depth = _get_bool_depth(s[0]) + return is_bool, depth + 1 + + +def _numpy_empty_ellipsis_patch(index, tensor_ndim): + """ + Patch for NumPy-compatible ellipsis behavior when ellipsis doesn't match any dimensions. + + In NumPy, when an ellipsis (...) doesn't actually match any dimensions of the input array, + it still acts as a separator between advanced indices. PyTorch doesn't have this behavior. + + This function detects when we have: + 1. Advanced indexing on both sides of an ellipsis + 2. The ellipsis doesn't actually match any dimensions + """ + if not isinstance(index, tuple): + index = (index,) + + # Find ellipsis position + ellipsis_pos = None + for i, idx in enumerate(index): + if idx is Ellipsis: + ellipsis_pos = i + break + + # If no ellipsis, no patch needed + if ellipsis_pos is None: + return index, lambda x: x, lambda x: x + + # Count non-ellipsis dimensions consumed by the index + consumed_dims = 0 + for idx in index: + is_bool, depth = _get_bool_depth(idx) + if is_bool: + consumed_dims += depth + elif idx is Ellipsis or idx is None: + continue + else: + consumed_dims += 1 + + # Calculate how many dimensions the ellipsis should match + ellipsis_dims = tensor_ndim - consumed_dims + + # Check if ellipsis doesn't match any dimensions + if ellipsis_dims == 0: + # Check if we have advanced indexing on both sides of ellipsis + left_advanced = _has_advanced_indexing(index[:ellipsis_pos]) + right_advanced = _has_advanced_indexing(index[ellipsis_pos + 1 :]) + + if left_advanced and right_advanced: + # This is the case where NumPy and PyTorch differ + # We need to ensure the advanced indices are treated as separated + new_index = index[:ellipsis_pos] + (None,) + index[ellipsis_pos + 1 :] + end_ndims = 1 + sum( + 1 for idx in index[ellipsis_pos + 1 :] if isinstance(idx, slice) + ) + + def squeeze_fn(x): + return x.squeeze(-end_ndims) + + def unsqueeze_fn(x): + if isinstance(x, torch.Tensor) and x.ndim >= end_ndims: + return x.unsqueeze(-end_ndims) + return x + + return new_index, squeeze_fn, unsqueeze_fn + + return index, lambda x: x, lambda x: x + + +# Used to indicate that a parameter is unspecified (as opposed to explicitly +# `None`) +class _Unspecified: + pass + + +_Unspecified.unspecified = _Unspecified() + +############################################################### +# ndarray class # +############################################################### + + +class ndarray: + def __init__(self, t=None): + if t is None: + self.tensor = torch.Tensor() + elif isinstance(t, torch.Tensor): + self.tensor = t + else: + raise ValueError( + "ndarray constructor is not recommended; prefer" + "either array(...) or zeros/empty(...)" + ) + + # Register NumPy functions as methods + for method, name in methods.items(): + fn = getattr(_funcs, name or method) + vars()[method] = create_method(fn, method) + + # Regular methods but coming from ufuncs + conj = create_method(_ufuncs.conjugate, "conj") + conjugate = create_method(_ufuncs.conjugate) + + for method, name in dunder.items(): + fn = getattr(_ufuncs, name or method) + method = f"__{method}__" + vars()[method] = create_method(fn, method) + + for method, name in ri_dunder.items(): + fn = getattr(_ufuncs, name or method) + plain = f"__{method}__" + vars()[plain] = create_method(fn, plain) + rvar = f"__r{method}__" + vars()[rvar] = create_method(lambda self, other, fn=fn: fn(other, self), rvar) + ivar = f"__i{method}__" + vars()[ivar] = create_method( + lambda self, other, fn=fn: fn(self, other, out=self), ivar + ) + + # There's no __idivmod__ + __divmod__ = create_method(_ufuncs.divmod, "__divmod__") + __rdivmod__ = create_method( + lambda self, other: _ufuncs.divmod(other, self), "__rdivmod__" + ) + + # prevent loop variables leaking into the ndarray class namespace + del ivar, rvar, name, plain, fn, method + + @property + def shape(self): + return tuple(self.tensor.shape) + + @property + def size(self): + return self.tensor.numel() + + @property + def ndim(self): + return self.tensor.ndim + + @property + def dtype(self): + return _dtypes.dtype(self.tensor.dtype) + + @property + def strides(self): + elsize = self.tensor.element_size() + return tuple(stride * elsize for stride in self.tensor.stride()) + + @property + def itemsize(self): + return self.tensor.element_size() + + @property + def flags(self): + # Note contiguous in torch is assumed C-style + return Flags( + { + "C_CONTIGUOUS": self.tensor.is_contiguous(), + "F_CONTIGUOUS": self.T.tensor.is_contiguous(), + "OWNDATA": self.tensor._base is None, + "WRITEABLE": True, # pytorch does not have readonly tensors + } + ) + + @property + def data(self): + return self.tensor.data_ptr() + + @property + def nbytes(self): + return self.tensor.storage().nbytes() + + @property + def T(self): + return self.transpose() + + @property + def real(self): + return _funcs.real(self) + + @real.setter + def real(self, value): + self.tensor.real = asarray(value).tensor + + @property + def imag(self): + return _funcs.imag(self) + + @imag.setter + def imag(self, value): + self.tensor.imag = asarray(value).tensor + + # ctors + def astype(self, dtype, order="K", casting="unsafe", subok=True, copy=True): + if order != "K": + raise NotImplementedError(f"astype(..., order={order} is not implemented.") + if casting != "unsafe": + raise NotImplementedError( + f"astype(..., casting={casting} is not implemented." + ) + if not subok: + raise NotImplementedError(f"astype(..., subok={subok} is not implemented.") + if not copy: + raise NotImplementedError(f"astype(..., copy={copy} is not implemented.") + torch_dtype = _dtypes.dtype(dtype).torch_dtype + t = self.tensor.to(torch_dtype) + return ndarray(t) + + @normalizer + def copy(self: ArrayLike, order: NotImplementedType = "C"): + return self.clone() + + @normalizer + def flatten(self: ArrayLike, order: NotImplementedType = "C"): + return torch.flatten(self) + + def resize(self, *new_shape, refcheck=False): + # NB: differs from np.resize: fills with zeros instead of making repeated copies of input. + if refcheck: + raise NotImplementedError( + f"resize(..., refcheck={refcheck} is not implemented." + ) + if new_shape in [(), (None,)]: + return + + # support both x.resize((2, 2)) and x.resize(2, 2) + if len(new_shape) == 1: + new_shape = new_shape[0] + if isinstance(new_shape, int): + new_shape = (new_shape,) + + if builtins.any(x < 0 for x in new_shape): + raise ValueError("all elements of `new_shape` must be non-negative") + + new_numel, old_numel = math.prod(new_shape), self.tensor.numel() + + self.tensor.resize_(new_shape) + + if new_numel >= old_numel: + # zero-fill new elements + assert self.tensor.is_contiguous() + b = self.tensor.flatten() # does not copy + b[old_numel:].zero_() + + def view(self, dtype=_Unspecified.unspecified, type=_Unspecified.unspecified): + if dtype is _Unspecified.unspecified: + dtype = self.dtype + if type is not _Unspecified.unspecified: + raise NotImplementedError(f"view(..., type={type} is not implemented.") + torch_dtype = _dtypes.dtype(dtype).torch_dtype + tview = self.tensor.view(torch_dtype) + return ndarray(tview) + + @normalizer + def fill(self, value: ArrayLike): + # Both Pytorch and NumPy accept 0D arrays/tensors and scalars, and + # error out on D > 0 arrays + self.tensor.fill_(value) + + def tolist(self): + return self.tensor.tolist() + + def __iter__(self): + return (ndarray(x) for x in self.tensor.__iter__()) + + def __str__(self): + return ( + str(self.tensor) + .replace("tensor", "torch.ndarray") + .replace("dtype=torch.", "dtype=") + ) + + __repr__ = create_method(__str__) + + def __eq__(self, other): + try: + return _ufuncs.equal(self, other) + except (RuntimeError, TypeError): + # Failed to convert other to array: definitely not equal. + falsy = torch.full(self.shape, fill_value=False, dtype=bool) + return asarray(falsy) + + def __ne__(self, other): + return ~(self == other) + + def __index__(self): + try: + return operator.index(self.tensor.item()) + except Exception as exc: + raise TypeError( + "only integer scalar arrays can be converted to a scalar index" + ) from exc + + def __bool__(self): + return bool(self.tensor) + + def __int__(self): + return int(self.tensor) + + def __float__(self): + return float(self.tensor) + + def __complex__(self): + return complex(self.tensor) + + def is_integer(self): + try: + v = self.tensor.item() + result = int(v) == v + except Exception: + result = False + return result + + def __len__(self): + return self.tensor.shape[0] + + def __contains__(self, x): + return self.tensor.__contains__(x) + + def transpose(self, *axes): + # np.transpose(arr, axis=None) but arr.transpose(*axes) + return _funcs.transpose(self, axes) + + def reshape(self, *shape, order="C"): + # arr.reshape(shape) and arr.reshape(*shape) + return _funcs.reshape(self, shape, order=order) + + def sort(self, axis=-1, kind=None, order=None): + # ndarray.sort works in-place + _funcs.copyto(self, _funcs.sort(self, axis, kind, order)) + + def item(self, *args): + # Mimic NumPy's implementation with three special cases (no arguments, + # a flat index and a multi-index): + # https://github.com/numpy/numpy/blob/main/numpy/_core/src/multiarray/methods.c#L702 + if args == (): + return self.tensor.item() + elif len(args) == 1: + # int argument + return self.ravel()[args[0]] + else: + return self.__getitem__(args) + + def __getitem__(self, index): + tensor = self.tensor + + def neg_step(i, s): + if not (isinstance(s, slice) and s.step is not None and s.step < 0): + return s + + nonlocal tensor + tensor = torch.flip(tensor, (i,)) + + # Account for the fact that a slice includes the start but not the end + assert isinstance(s.start, int) or s.start is None + assert isinstance(s.stop, int) or s.stop is None + start = s.stop + 1 if s.stop else None + stop = s.start + 1 if s.start else None + + return slice(start, stop, -s.step) + + if isinstance(index, Sequence): + index = type(index)(neg_step(i, s) for i, s in enumerate(index)) + else: + index = neg_step(0, index) + index = _util.ndarrays_to_tensors(index) + index = _upcast_int_indices(index) + # Apply NumPy-compatible indexing conversion + index = _numpy_compatible_indexing(index) + # Apply NumPy-compatible empty ellipsis behavior + index, maybe_squeeze, _ = _numpy_empty_ellipsis_patch(index, tensor.ndim) + return maybe_squeeze(ndarray(tensor.__getitem__(index))) + + def __setitem__(self, index, value): + index = _util.ndarrays_to_tensors(index) + index = _upcast_int_indices(index) + # Apply NumPy-compatible indexing conversion + index = _numpy_compatible_indexing(index) + # Apply NumPy-compatible empty ellipsis behavior + index, _, maybe_unsqueeze = _numpy_empty_ellipsis_patch(index, self.tensor.ndim) + + if not _dtypes_impl.is_scalar(value): + value = normalize_array_like(value) + value = _util.cast_if_needed(value, self.tensor.dtype) + + return self.tensor.__setitem__(index, maybe_unsqueeze(value)) + + take = _funcs.take + put = _funcs.put + + def __dlpack__(self, *, stream=None): + return self.tensor.__dlpack__(stream=stream) + + def __dlpack_device__(self): + return self.tensor.__dlpack_device__() + + +def _tolist(obj): + """Recursively convert tensors into lists.""" + a1 = [] + for elem in obj: + if isinstance(elem, (list, tuple)): + elem = _tolist(elem) + if isinstance(elem, ndarray): + a1.append(elem.tensor.tolist()) + else: + a1.append(elem) + return a1 + + +# This is the ideally the only place which talks to ndarray directly. +# The rest goes through asarray (preferred) or array. + + +def array(obj, dtype=None, *, copy=True, order="K", subok=False, ndmin=0, like=None): + if subok is not False: + raise NotImplementedError("'subok' parameter is not supported.") + if like is not None: + raise NotImplementedError("'like' parameter is not supported.") + if order != "K": + raise NotImplementedError + + # a happy path + if ( + isinstance(obj, ndarray) + and copy is False + and dtype is None + and ndmin <= obj.ndim + ): + return obj + + if isinstance(obj, (list, tuple)): + # FIXME and they have the same dtype, device, etc + if obj and all(isinstance(x, torch.Tensor) for x in obj): + # list of arrays: *under torch.Dynamo* these are FakeTensors + obj = torch.stack(obj) + else: + # XXX: remove tolist + # lists of ndarrays: [1, [2, 3], ndarray(4)] convert to lists of lists + obj = _tolist(obj) + + # is obj an ndarray already? + if isinstance(obj, ndarray): + obj = obj.tensor + + # is a specific dtype requested? + torch_dtype = None + if dtype is not None: + torch_dtype = _dtypes.dtype(dtype).torch_dtype + + tensor = _util._coerce_to_tensor(obj, torch_dtype, copy, ndmin) + return ndarray(tensor) + + +def asarray(a, dtype=None, order="K", *, like=None): + return array(a, dtype=dtype, order=order, like=like, copy=False, ndmin=0) + + +def ascontiguousarray(a, dtype=None, *, like=None): + arr = asarray(a, dtype=dtype, like=like) + if not arr.tensor.is_contiguous(): + arr.tensor = arr.tensor.contiguous() + return arr + + +def from_dlpack(x, /): + t = torch.from_dlpack(x) + return ndarray(t) + + +def _extract_dtype(entry): + try: + dty = _dtypes.dtype(entry) + except Exception: + dty = asarray(entry).dtype + return dty + + +def can_cast(from_, to, casting="safe"): + from_ = _extract_dtype(from_) + to_ = _extract_dtype(to) + + return _dtypes_impl.can_cast_impl(from_.torch_dtype, to_.torch_dtype, casting) + + +def result_type(*arrays_and_dtypes): + tensors = [] + for entry in arrays_and_dtypes: + try: + t = asarray(entry).tensor + except (RuntimeError, ValueError, TypeError): + dty = _dtypes.dtype(entry) + t = torch.empty(1, dtype=dty.torch_dtype) + tensors.append(t) + + torch_dtype = _dtypes_impl.result_type_impl(*tensors) + return _dtypes.dtype(torch_dtype) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_normalizations.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_normalizations.py new file mode 100644 index 0000000000000000000000000000000000000000..82cdb2b0b11b3325e5e42dc6d4bdf14f74cefedd --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_normalizations.py @@ -0,0 +1,261 @@ +# mypy: ignore-errors + +""" "Normalize" arguments: convert array_likes to tensors, dtypes to torch dtypes and so on.""" + +from __future__ import annotations + +import functools +import inspect +import operator +import typing + +import torch + +from . import _dtypes, _dtypes_impl, _util + + +ArrayLike = typing.TypeVar("ArrayLike") +Scalar = typing.Union[int, float, complex, bool] +ArrayLikeOrScalar = typing.Union[ArrayLike, Scalar] + +DTypeLike = typing.TypeVar("DTypeLike") +AxisLike = typing.TypeVar("AxisLike") +NDArray = typing.TypeVar("NDArray") +CastingModes = typing.TypeVar("CastingModes") +KeepDims = typing.TypeVar("KeepDims") + +# OutArray is to annotate the out= array argument. +# +# This one is special is several respects: +# First, It needs to be an NDArray, and we need to preserve the `result is out` +# semantics. Therefore, we cannot just extract the Tensor from the out array. +# So we never pass the out array to implementer functions and handle it in the +# `normalizer` below. +# Second, the out= argument can be either keyword or positional argument, and +# as a positional arg, it can be anywhere in the signature. +# To handle all this, we define a special `OutArray` annotation and dispatch on it. +# +OutArray = typing.TypeVar("OutArray") + +try: + from typing import NotImplementedType +except ImportError: + NotImplementedType = typing.TypeVar("NotImplementedType") + + +def normalize_array_like(x, parm=None): # codespell:ignore + from ._ndarray import asarray + + return asarray(x).tensor + + +def normalize_array_like_or_scalar(x, parm=None): # codespell:ignore + if _dtypes_impl.is_scalar_or_symbolic(x): + return x + return normalize_array_like(x, parm) # codespell:ignore + + +def normalize_optional_array_like_or_scalar(x, parm=None): # codespell:ignore + if x is None: + return None + return normalize_array_like_or_scalar(x, parm) # codespell:ignore + + +def normalize_optional_array_like(x, parm=None): # codespell:ignore + # This explicit normalizer is needed because otherwise normalize_array_like + # does not run for a parameter annotated as Optional[ArrayLike] + return None if x is None else normalize_array_like(x, parm) # codespell:ignore + + +def normalize_seq_array_like(x, parm=None): # codespell:ignore + return tuple(normalize_array_like(value) for value in x) + + +def normalize_dtype(dtype, parm=None): # codespell:ignore + # cf _decorators.dtype_to_torch + torch_dtype = None + if dtype is not None: + dtype = _dtypes.dtype(dtype) + torch_dtype = dtype.torch_dtype + return torch_dtype + + +def normalize_not_implemented(arg, parm): # codespell:ignore + if arg != parm.default: # codespell:ignore + raise NotImplementedError( + f"'{parm.name}' parameter is not supported." # codespell:ignore + ) + + +def normalize_axis_like(arg, parm=None): # codespell:ignore + from ._ndarray import ndarray + + if isinstance(arg, ndarray): + arg = operator.index(arg) + return arg + + +def normalize_ndarray(arg, parm=None): # codespell:ignore + # check the arg is an ndarray, extract its tensor attribute + if arg is None: + return arg + + from ._ndarray import ndarray + + if not isinstance(arg, ndarray): + raise TypeError(f"'{parm.name}' must be an array") # codespell:ignore + return arg.tensor + + +def normalize_outarray(arg, parm=None): # codespell:ignore + # almost normalize_ndarray, only return the array, not its tensor + if arg is None: + return arg + from ._ndarray import ndarray + + # Dynamo can pass torch tensors as out arguments, + # wrap it in an ndarray before processing + if isinstance(arg, torch.Tensor): + arg = ndarray(arg) + + if not isinstance(arg, ndarray): + raise TypeError(f"'{parm.name}' must be an array") # codespell:ignore + return arg + + +def normalize_casting(arg, parm=None): # codespell:ignore + if arg not in ["no", "equiv", "safe", "same_kind", "unsafe"]: + raise ValueError( + f"casting must be one of 'no', 'equiv', 'safe', 'same_kind', or 'unsafe' (got '{arg}')" + ) + return arg + + +normalizers = { + "ArrayLike": normalize_array_like, + "ArrayLikeOrScalar": normalize_array_like_or_scalar, + "Optional[ArrayLike]": normalize_optional_array_like, + "Sequence[ArrayLike]": normalize_seq_array_like, + "Optional[ArrayLikeOrScalar]": normalize_optional_array_like_or_scalar, + "Optional[NDArray]": normalize_ndarray, + "Optional[OutArray]": normalize_outarray, + "NDArray": normalize_ndarray, + "Optional[DTypeLike]": normalize_dtype, + "AxisLike": normalize_axis_like, + "NotImplementedType": normalize_not_implemented, + "Optional[CastingModes]": normalize_casting, +} + + +def maybe_normalize(arg, parm): # codespell:ignore + """Normalize arg if a normalizer is registered.""" + normalizer = normalizers.get(parm.annotation, None) # codespell:ignore + return normalizer(arg, parm) if normalizer else arg # codespell:ignore + + +# ### Return value helpers ### + + +def maybe_copy_to(out, result, promote_scalar_result=False): + # NB: here out is either an ndarray or None + if out is None: + return result + elif isinstance(result, torch.Tensor): + if result.shape != out.shape: + can_fit = result.numel() == 1 and out.ndim == 0 + if promote_scalar_result and can_fit: + result = result.squeeze() + else: + raise ValueError( + f"Bad size of the out array: out.shape = {out.shape}" + f" while result.shape = {result.shape}." + ) + out.tensor.copy_(result) + return out + elif isinstance(result, (tuple, list)): + return type(result)( + maybe_copy_to(o, r, promote_scalar_result) for o, r in zip(out, result) + ) + else: + raise AssertionError # We should never hit this path + + +def wrap_tensors(result): + from ._ndarray import ndarray + + if isinstance(result, torch.Tensor): + return ndarray(result) + elif isinstance(result, (tuple, list)): + result = type(result)(wrap_tensors(x) for x in result) + return result + + +def array_or_scalar(values, py_type=float, return_scalar=False): + if return_scalar: + return py_type(values.item()) + else: + from ._ndarray import ndarray + + return ndarray(values) + + +# ### The main decorator to normalize arguments / postprocess the output ### + + +def normalizer(_func=None, *, promote_scalar_result=False): + def normalizer_inner(func): + @functools.wraps(func) + def wrapped(*args, **kwds): + sig = inspect.signature(func) + params = sig.parameters + first_param = next(iter(params.values())) + + # NumPy's API does not have positional args before variadic positional args + if first_param.kind == inspect.Parameter.VAR_POSITIONAL: + args = [maybe_normalize(arg, first_param) for arg in args] + else: + # NB: extra unknown arguments: pass through, will raise in func(*args) below + args = ( + tuple( + maybe_normalize(arg, parm) # codespell:ignore + for arg, parm in zip(args, params.values()) # codespell:ignore + ) + + args[len(params.values()) :] + ) + + kwds = { + name: maybe_normalize(arg, params[name]) if name in params else arg + for name, arg in kwds.items() + } + + result = func(*args, **kwds) + + # keepdims + bound_args = None + if "keepdims" in params and params["keepdims"].annotation == "KeepDims": + # keepdims can be in any position so we need sig.bind + bound_args = sig.bind(*args, **kwds).arguments + if bound_args.get("keepdims", False): + # In this case the first arg is the initial tensor and + # the second arg is (optionally) the axis + tensor = args[0] + axis = bound_args.get("axis") + result = _util.apply_keepdims(result, axis, tensor.ndim) + + # out + if "out" in params: + # out can be in any position so we need sig.bind + if bound_args is None: + bound_args = sig.bind(*args, **kwds).arguments + out = bound_args.get("out") + result = maybe_copy_to(out, result, promote_scalar_result) + result = wrap_tensors(result) + + return result + + return wrapped + + if _func is None: + return normalizer_inner + else: + return normalizer_inner(_func) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_reductions_impl.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_reductions_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..a4ebc094a7284575fad083db01e871227f28ea7e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_reductions_impl.py @@ -0,0 +1,460 @@ +# mypy: ignore-errors + +"""Implementation of reduction operations, to be wrapped into arrays, dtypes etc +in the 'public' layer. + +Anything here only deals with torch objects, e.g. "dtype" is a torch.dtype instance etc +""" + +from __future__ import annotations + +import functools +from typing import Optional, TYPE_CHECKING + +import torch + +from . import _dtypes_impl, _util + + +if TYPE_CHECKING: + from ._normalizations import ( + ArrayLike, + AxisLike, + DTypeLike, + KeepDims, + NotImplementedType, + OutArray, + ) + + +def _deco_axis_expand(func): + """ + Generically handle axis arguments in reductions. + axis is *always* the 2nd arg in the function so no need to have a look at its signature + """ + + @functools.wraps(func) + def wrapped(a, axis=None, *args, **kwds): + if axis is not None: + axis = _util.normalize_axis_tuple(axis, a.ndim) + + if axis == (): + # So we insert a length-one axis and run the reduction along it. + # We cannot return a.clone() as this would sidestep the checks inside the function + newshape = _util.expand_shape(a.shape, axis=0) + a = a.reshape(newshape) + axis = (0,) + + return func(a, axis, *args, **kwds) + + return wrapped + + +def _atleast_float(dtype, other_dtype): + """Return a dtype that is real or complex floating-point. + + For inputs that are boolean or integer dtypes, this returns the default + float dtype; inputs that are complex get converted to the default complex + dtype; real floating-point dtypes (`float*`) get passed through unchanged + """ + if dtype is None: + dtype = other_dtype + if not (dtype.is_floating_point or dtype.is_complex): + return _dtypes_impl.default_dtypes().float_dtype + return dtype + + +@_deco_axis_expand +def count_nonzero(a: ArrayLike, axis: AxisLike = None, *, keepdims: KeepDims = False): + return a.count_nonzero(axis) + + +@_deco_axis_expand +def argmax( + a: ArrayLike, + axis: AxisLike = None, + out: Optional[OutArray] = None, + *, + keepdims: KeepDims = False, +): + if a.is_complex(): + raise NotImplementedError(f"argmax with dtype={a.dtype}.") + + axis = _util.allow_only_single_axis(axis) + + if a.dtype == torch.bool: + # RuntimeError: "argmax_cpu" not implemented for 'Bool' + a = a.to(torch.uint8) + + return torch.argmax(a, axis) + + +@_deco_axis_expand +def argmin( + a: ArrayLike, + axis: AxisLike = None, + out: Optional[OutArray] = None, + *, + keepdims: KeepDims = False, +): + if a.is_complex(): + raise NotImplementedError(f"argmin with dtype={a.dtype}.") + + axis = _util.allow_only_single_axis(axis) + + if a.dtype == torch.bool: + # RuntimeError: "argmin_cpu" not implemented for 'Bool' + a = a.to(torch.uint8) + + return torch.argmin(a, axis) + + +@_deco_axis_expand +def any( + a: ArrayLike, + axis: AxisLike = None, + out: Optional[OutArray] = None, + keepdims: KeepDims = False, + *, + where: NotImplementedType = None, +): + axis = _util.allow_only_single_axis(axis) + axis_kw = {} if axis is None else {"dim": axis} + return torch.any(a, **axis_kw) + + +@_deco_axis_expand +def all( + a: ArrayLike, + axis: AxisLike = None, + out: Optional[OutArray] = None, + keepdims: KeepDims = False, + *, + where: NotImplementedType = None, +): + axis = _util.allow_only_single_axis(axis) + axis_kw = {} if axis is None else {"dim": axis} + return torch.all(a, **axis_kw) + + +@_deco_axis_expand +def amax( + a: ArrayLike, + axis: AxisLike = None, + out: Optional[OutArray] = None, + keepdims: KeepDims = False, + initial: NotImplementedType = None, + where: NotImplementedType = None, +): + if a.is_complex(): + raise NotImplementedError(f"amax with dtype={a.dtype}") + + return a.amax(axis) + + +max = amax + + +@_deco_axis_expand +def amin( + a: ArrayLike, + axis: AxisLike = None, + out: Optional[OutArray] = None, + keepdims: KeepDims = False, + initial: NotImplementedType = None, + where: NotImplementedType = None, +): + if a.is_complex(): + raise NotImplementedError(f"amin with dtype={a.dtype}") + + return a.amin(axis) + + +min = amin + + +@_deco_axis_expand +def ptp( + a: ArrayLike, + axis: AxisLike = None, + out: Optional[OutArray] = None, + keepdims: KeepDims = False, +): + return a.amax(axis) - a.amin(axis) + + +@_deco_axis_expand +def sum( + a: ArrayLike, + axis: AxisLike = None, + dtype: Optional[DTypeLike] = None, + out: Optional[OutArray] = None, + keepdims: KeepDims = False, + initial: NotImplementedType = None, + where: NotImplementedType = None, +): + assert dtype is None or isinstance(dtype, torch.dtype) + + if dtype == torch.bool: + dtype = _dtypes_impl.default_dtypes().int_dtype + + axis_kw = {} if axis is None else {"dim": axis} + return a.sum(dtype=dtype, **axis_kw) + + +@_deco_axis_expand +def prod( + a: ArrayLike, + axis: AxisLike = None, + dtype: Optional[DTypeLike] = None, + out: Optional[OutArray] = None, + keepdims: KeepDims = False, + initial: NotImplementedType = None, + where: NotImplementedType = None, +): + axis = _util.allow_only_single_axis(axis) + + if dtype == torch.bool: + dtype = _dtypes_impl.default_dtypes().int_dtype + + axis_kw = {} if axis is None else {"dim": axis} + return a.prod(dtype=dtype, **axis_kw) + + +product = prod + + +@_deco_axis_expand +def mean( + a: ArrayLike, + axis: AxisLike = None, + dtype: Optional[DTypeLike] = None, + out: Optional[OutArray] = None, + keepdims: KeepDims = False, + *, + where: NotImplementedType = None, +): + dtype = _atleast_float(dtype, a.dtype) + + axis_kw = {} if axis is None else {"dim": axis} + result = a.mean(dtype=dtype, **axis_kw) + + return result + + +@_deco_axis_expand +def std( + a: ArrayLike, + axis: AxisLike = None, + dtype: Optional[DTypeLike] = None, + out: Optional[OutArray] = None, + ddof=0, + keepdims: KeepDims = False, + *, + where: NotImplementedType = None, +): + in_dtype = dtype + dtype = _atleast_float(dtype, a.dtype) + tensor = _util.cast_if_needed(a, dtype) + result = tensor.std(dim=axis, correction=ddof) + return _util.cast_if_needed(result, in_dtype) + + +@_deco_axis_expand +def var( + a: ArrayLike, + axis: AxisLike = None, + dtype: Optional[DTypeLike] = None, + out: Optional[OutArray] = None, + ddof=0, + keepdims: KeepDims = False, + *, + where: NotImplementedType = None, +): + in_dtype = dtype + dtype = _atleast_float(dtype, a.dtype) + tensor = _util.cast_if_needed(a, dtype) + result = tensor.var(dim=axis, correction=ddof) + return _util.cast_if_needed(result, in_dtype) + + +# cumsum / cumprod are almost reductions: +# 1. no keepdims +# 2. axis=None flattens + + +def cumsum( + a: ArrayLike, + axis: AxisLike = None, + dtype: Optional[DTypeLike] = None, + out: Optional[OutArray] = None, +): + if dtype == torch.bool: + dtype = _dtypes_impl.default_dtypes().int_dtype + if dtype is None: + dtype = a.dtype + + (a,), axis = _util.axis_none_flatten(a, axis=axis) + axis = _util.normalize_axis_index(axis, a.ndim) + + return a.cumsum(axis=axis, dtype=dtype) + + +def cumprod( + a: ArrayLike, + axis: AxisLike = None, + dtype: Optional[DTypeLike] = None, + out: Optional[OutArray] = None, +): + if dtype == torch.bool: + dtype = _dtypes_impl.default_dtypes().int_dtype + if dtype is None: + dtype = a.dtype + + (a,), axis = _util.axis_none_flatten(a, axis=axis) + axis = _util.normalize_axis_index(axis, a.ndim) + + return a.cumprod(axis=axis, dtype=dtype) + + +cumproduct = cumprod + + +def average( + a: ArrayLike, + axis=None, + weights: ArrayLike = None, + returned=False, + *, + keepdims=False, +): + if weights is None: + result = mean(a, axis=axis) + wsum = torch.as_tensor(a.numel() / result.numel(), dtype=result.dtype) + else: + if not a.dtype.is_floating_point: + a = a.double() + + # axis & weights + if a.shape != weights.shape: + if axis is None: + raise TypeError( + "Axis must be specified when shapes of a and weights differ." + ) + if weights.ndim != 1: + raise TypeError( + "1D weights expected when shapes of a and weights differ." + ) + if weights.shape[0] != a.shape[axis]: + raise ValueError( + "Length of weights not compatible with specified axis." + ) + + # setup weight to broadcast along axis + weights = torch.broadcast_to(weights, (a.ndim - 1) * (1,) + weights.shape) + weights = weights.swapaxes(-1, axis) + + # do the work + result_dtype = _dtypes_impl.result_type_impl(a, weights) + numerator = sum(a * weights, axis, dtype=result_dtype) + wsum = sum(weights, axis, dtype=result_dtype) + result = numerator / wsum + + # We process keepdims manually because the decorator does not deal with variadic returns + if keepdims: + result = _util.apply_keepdims(result, axis, a.ndim) + + if returned: + if wsum.shape != result.shape: + wsum = torch.broadcast_to(wsum, result.shape).clone() + return result, wsum + else: + return result + + +# Not using deco_axis_expand as it assumes that axis is the second arg +def quantile( + a: ArrayLike, + q: ArrayLike, + axis: AxisLike = None, + out: Optional[OutArray] = None, + overwrite_input=False, + method="linear", + keepdims: KeepDims = False, + *, + interpolation: NotImplementedType = None, +): + if overwrite_input: + # raise NotImplementedError("overwrite_input in quantile not implemented.") + # NumPy documents that `overwrite_input` MAY modify inputs: + # https://numpy.org/doc/stable/reference/generated/numpy.percentile.html#numpy-percentile + # Here we choose to work out-of-place because why not. + pass + + if not a.dtype.is_floating_point: + dtype = _dtypes_impl.default_dtypes().float_dtype + a = a.to(dtype) + + # edge case: torch.quantile only supports float32 and float64 + if a.dtype == torch.float16: + a = a.to(torch.float32) + + if axis is None: + a = a.flatten() + q = q.flatten() + axis = (0,) + else: + axis = _util.normalize_axis_tuple(axis, a.ndim) + + # FIXME(Mario) Doesn't np.quantile accept a tuple? + # torch.quantile does accept a number. If we don't want to implement the tuple behaviour + # (it's deffo low prio) change `normalize_axis_tuple` into a normalize_axis index above. + axis = _util.allow_only_single_axis(axis) + + q = _util.cast_if_needed(q, a.dtype) + + return torch.quantile(a, q, axis=axis, interpolation=method) + + +def percentile( + a: ArrayLike, + q: ArrayLike, + axis: AxisLike = None, + out: Optional[OutArray] = None, + overwrite_input=False, + method="linear", + keepdims: KeepDims = False, + *, + interpolation: NotImplementedType = None, +): + # np.percentile(float_tensor, 30) : q.dtype is int64 => q / 100.0 is float32 + if _dtypes_impl.python_type_for_torch(q.dtype) is int: + q = q.to(_dtypes_impl.default_dtypes().float_dtype) + qq = q / 100.0 + + return quantile( + a, + qq, + axis=axis, + overwrite_input=overwrite_input, + method=method, + keepdims=keepdims, + interpolation=interpolation, + ) + + +def median( + a: ArrayLike, + axis=None, + out: Optional[OutArray] = None, + overwrite_input=False, + keepdims: KeepDims = False, +): + return quantile( + a, + torch.as_tensor(0.5), + axis=axis, + overwrite_input=overwrite_input, + out=out, + keepdims=keepdims, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_ufuncs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_ufuncs.py new file mode 100644 index 0000000000000000000000000000000000000000..0543aad2f7a6ba551d8ce7483e1dc8e0474ac57e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_ufuncs.py @@ -0,0 +1,334 @@ +# mypy: ignore-errors + +from __future__ import annotations + +from typing import Optional + +import torch + +from . import _binary_ufuncs_impl, _dtypes_impl, _unary_ufuncs_impl, _util +from ._normalizations import ( + ArrayLike, + ArrayLikeOrScalar, + CastingModes, + DTypeLike, + normalizer, + NotImplementedType, + OutArray, +) + + +def _ufunc_postprocess(result, out, casting): + if out is not None: + result = _util.typecast_tensor(result, out.dtype.torch_dtype, casting) + result = torch.broadcast_to(result, out.shape) + return result + + +# ############# Binary ufuncs ###################### + +_binary = [ + name + for name in dir(_binary_ufuncs_impl) + if not name.startswith("_") and name not in ["torch", "matmul", "divmod", "ldexp"] +] + + +NEP50_FUNCS = ( + "add", + "subtract", + "multiply", + "floor_divide", + "true_divide", + "divide", + "remainder", + "bitwise_and", + "bitwise_or", + "bitwise_xor", + "bitwise_left_shift", + "bitwise_right_shift", + "hypot", + "arctan2", + "logaddexp", + "logaddexp2", + "heaviside", + "copysign", + "fmax", + "minimum", + "fmin", + "maximum", + "fmod", + "gcd", + "lcm", + "pow", +) + + +def deco_binary_ufunc(torch_func): + """Common infra for binary ufuncs. + + Normalize arguments, sort out type casting, broadcasting and delegate to + the pytorch functions for the actual work. + """ + + @normalizer + def wrapped( + x1: ArrayLikeOrScalar, + x2: ArrayLikeOrScalar, + /, + out: Optional[OutArray] = None, + *, + where: NotImplementedType = True, + casting: Optional[CastingModes] = "same_kind", + order: NotImplementedType = "K", + dtype: Optional[DTypeLike] = None, + subok: NotImplementedType = False, + signature: NotImplementedType = None, + extobj: NotImplementedType = None, + ): + if dtype is not None: + + def cast(x, dtype): + if isinstance(x, torch.Tensor): + return _util.typecast_tensor(x, dtype, casting) + else: + return torch.as_tensor(x, dtype=dtype) + + x1 = cast(x1, dtype) + x2 = cast(x2, dtype) + elif isinstance(x1, torch.Tensor) and isinstance(x2, torch.Tensor): + dtype = _dtypes_impl.result_type_impl(x1, x2) + x1, x2 = _util.typecast_tensors((x1, x2), dtype, casting) + else: + x1, x2 = _dtypes_impl.nep50_to_tensors( + x1, x2, torch_func.__name__ in NEP50_FUNCS, torch_func.__name__ + ) + + result = torch_func(x1, x2) + + return _ufunc_postprocess(result, out, casting) + + wrapped.__qualname__ = torch_func.__name__ + wrapped.__name__ = torch_func.__name__ + + return wrapped + + +# matmul's signature is _slightly_ different from other ufuncs: +# - no where=... +# - additional axis=..., axes=... +# - no NEP50 scalars in or out +@normalizer +def matmul( + x1: ArrayLike, + x2: ArrayLike, + /, + out: Optional[OutArray] = None, + *, + casting: Optional[CastingModes] = "same_kind", + order: NotImplementedType = "K", + dtype: Optional[DTypeLike] = None, + subok: NotImplementedType = False, + signature: NotImplementedType = None, + extobj: NotImplementedType = None, + axes: NotImplementedType = None, + axis: NotImplementedType = None, +): + if dtype is None: + dtype = _dtypes_impl.result_type_impl(x1, x2) + x1, x2 = _util.typecast_tensors((x1, x2), dtype, casting) + + result = _binary_ufuncs_impl.matmul(x1, x2) + + result = _ufunc_postprocess(result, out, casting) + return result + + +# ldexp casting is special : the dtype of the result == dtype of the 1st arg +@normalizer +def ldexp( + x1: ArrayLikeOrScalar, + x2: ArrayLikeOrScalar, + /, + out: Optional[OutArray] = None, + *, + where: NotImplementedType = True, + casting: Optional[CastingModes] = "same_kind", + order: NotImplementedType = "K", + dtype: Optional[DTypeLike] = None, + subok: NotImplementedType = False, + signature: NotImplementedType = None, + extobj: NotImplementedType = None, +): + if dtype is not None: + if isinstance(x1, torch.Tensor): + x1 = _util.typecast_tensor(x1, dtype, casting) + else: + x1 = torch.as_tensor(x1, dtype=dtype) + else: + if not isinstance(x1, torch.Tensor): + x1 = torch.as_tensor(x1) + x1 = _util.cast_int_to_float(x1) + + x2 = torch.as_tensor(x2) + # the second arg must be integer + if _dtypes_impl._category(x2.dtype) != 1: + raise ValueError("ldexp 2nd arg must be integer") + + result = _binary_ufuncs_impl.ldexp(x1, x2) + + if x1.dtype == torch.float16: + # torch.ldexp(f16, int) -> f32, undo it + result = result.to(torch.float16) + + return _ufunc_postprocess(result, out, casting) + + +# nin=2, nout=2 +@normalizer +def divmod( + x1: ArrayLike, + x2: ArrayLike, + out1: Optional[OutArray] = None, + out2: Optional[OutArray] = None, + /, + out: tuple[Optional[OutArray], Optional[OutArray]] = (None, None), + *, + where: NotImplementedType = True, + casting: Optional[CastingModes] = "same_kind", + order: NotImplementedType = "K", + dtype: Optional[DTypeLike] = None, + subok: NotImplementedType = False, + signature: NotImplementedType = None, + extobj: NotImplementedType = None, +): + # make sure we either have no out arrays at all, or there is either + # out1, out2, or out=tuple, but not both + num_outs = sum(x is not None for x in [out1, out2]) + if num_outs == 1: + raise ValueError("both out1 and out2 need to be provided") + elif num_outs == 2: + o1, o2 = out + if o1 is not None or o2 is not None: + raise TypeError( + "cannot specify 'out' as both a positional and keyword argument" + ) + else: + out1, out2 = out + + if dtype is None: + dtype = _dtypes_impl.result_type_impl(x1, x2) + x1, x2 = _util.typecast_tensors((x1, x2), dtype, casting) + + quot, rem = _binary_ufuncs_impl.divmod(x1, x2) + + quot = _ufunc_postprocess(quot, out1, casting) + rem = _ufunc_postprocess(rem, out2, casting) + return quot, rem + + +# +# Attach ufuncs to this module, for a further export to the public namespace in __init__.py +# +for name in _binary: + ufunc = getattr(_binary_ufuncs_impl, name) + vars()[name] = deco_binary_ufunc(ufunc) + + +def modf(x, /, *args, **kwds): + quot, rem = divmod(x, 1, *args, **kwds) + return rem, quot + + +_binary = _binary + ["divmod", "modf", "matmul", "ldexp"] + + +# ############# Unary ufuncs ###################### + + +_unary = [ + name + for name in dir(_unary_ufuncs_impl) + if not name.startswith("_") and name != "torch" +] + + +# these are ufunc(int) -> float +_fp_unary = [ + "arccos", + "arccosh", + "arcsin", + "arcsinh", + "arctan", + "arctanh", + "cbrt", + "cos", + "cosh", + "deg2rad", + "degrees", + "exp", + "exp2", + "expm1", + "log", + "log10", + "log1p", + "log2", + "rad2deg", + "radians", + "reciprocal", + "sin", + "sinh", + "sqrt", + "square", + "tan", + "tanh", + "trunc", +] + + +def deco_unary_ufunc(torch_func): + """Common infra for unary ufuncs. + + Normalize arguments, sort out type casting, broadcasting and delegate to + the pytorch functions for the actual work. + """ + + @normalizer + def wrapped( + x: ArrayLike, + /, + out: Optional[OutArray] = None, + *, + where=True, + casting: Optional[CastingModes] = "same_kind", + order="K", + dtype: Optional[DTypeLike] = None, + subok: NotImplementedType = False, + signature=None, + extobj=None, + ): + if dtype is not None: + x = _util.typecast_tensor(x, dtype, casting) + + if torch_func.__name__ in _fp_unary: + x = _util.cast_int_to_float(x) + + result = torch_func(x) + result = _ufunc_postprocess(result, out, casting) + return result + + wrapped.__qualname__ = torch_func.__name__ + wrapped.__name__ = torch_func.__name__ + + return wrapped + + +# +# Attach ufuncs to this module, for a further export to the public namespace in __init__.py +# +for name in _unary: + ufunc = getattr(_unary_ufuncs_impl, name) + vars()[name] = deco_unary_ufunc(ufunc) + + +__all__ = _binary + _unary # noqa: PLE0605 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_unary_ufuncs_impl.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_unary_ufuncs_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..9b7fe08ab4b8cf6b2e892b8956fff00d93c08d11 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_unary_ufuncs_impl.py @@ -0,0 +1,72 @@ +# mypy: ignore-errors + +"""Export torch work functions for unary ufuncs, rename/tweak to match numpy. +This listing is further exported to public symbols in the `_numpy/_ufuncs.py` module. +""" + +import torch +from torch import ( # noqa: F401 + absolute as fabs, + arccos, + arccosh, + arcsin, + arcsinh, + arctan, + arctanh, + bitwise_not, + bitwise_not as invert, + ceil, + conj_physical as conjugate, + cos, + cosh, + deg2rad, + deg2rad as radians, + exp, + exp2, + expm1, + floor, + isfinite, + isinf, + isnan, + log, + log10, + log1p, + log2, + logical_not, + negative, + rad2deg, + rad2deg as degrees, + reciprocal, + round as fix, + round as rint, + sign, + signbit, + sin, + sinh, + sqrt, + square, + tan, + tanh, + trunc, +) + + +# special cases: torch does not export these names +def cbrt(x): + return torch.pow(x, 1 / 3) + + +def positive(x): + return +x + + +def absolute(x): + # work around torch.absolute not impl for bools + if x.dtype == torch.bool: + return x + return torch.absolute(x) + + +# TODO set __name__ and __qualname__ +abs = absolute +conj = conjugate diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_util.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_util.py new file mode 100644 index 0000000000000000000000000000000000000000..636c95cf8167d1ea9f40ccbeb0debf885794ab56 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/_util.py @@ -0,0 +1,266 @@ +# mypy: ignore-errors + +"""Assorted utilities, which do not need anything other then torch and stdlib.""" + +import operator + +import torch + +from . import _dtypes_impl + + +# https://github.com/numpy/numpy/blob/v1.23.0/numpy/distutils/misc_util.py#L497-L504 +def is_sequence(seq): + if isinstance(seq, str): + return False + try: + len(seq) + except Exception: + return False + return True + + +class AxisError(ValueError, IndexError): + pass + + +class UFuncTypeError(TypeError, RuntimeError): + pass + + +def cast_if_needed(tensor, dtype): + # NB: no casting if dtype=None + if dtype is not None and tensor.dtype != dtype: + tensor = tensor.to(dtype) + return tensor + + +def cast_int_to_float(x): + # cast integers and bools to the default float dtype + if _dtypes_impl._category(x.dtype) < 2: + x = x.to(_dtypes_impl.default_dtypes().float_dtype) + return x + + +# a replica of the version in ./numpy/numpy/core/src/multiarray/common.h +def normalize_axis_index(ax, ndim, argname=None): + if not (-ndim <= ax < ndim): + raise AxisError(f"axis {ax} is out of bounds for array of dimension {ndim}") + if ax < 0: + ax += ndim + return ax + + +# from https://github.com/numpy/numpy/blob/main/numpy/core/numeric.py#L1378 +def normalize_axis_tuple(axis, ndim, argname=None, allow_duplicate=False): + """ + Normalizes an axis argument into a tuple of non-negative integer axes. + + This handles shorthands such as ``1`` and converts them to ``(1,)``, + as well as performing the handling of negative indices covered by + `normalize_axis_index`. + + By default, this forbids axes from being specified multiple times. + Used internally by multi-axis-checking logic. + + Parameters + ---------- + axis : int, iterable of int + The un-normalized index or indices of the axis. + ndim : int + The number of dimensions of the array that `axis` should be normalized + against. + argname : str, optional + A prefix to put before the error message, typically the name of the + argument. + allow_duplicate : bool, optional + If False, the default, disallow an axis from being specified twice. + + Returns + ------- + normalized_axes : tuple of int + The normalized axis index, such that `0 <= normalized_axis < ndim` + """ + # Optimization to speed-up the most common cases. + if type(axis) not in (tuple, list): + try: + axis = [operator.index(axis)] + except TypeError: + pass + # Going via an iterator directly is slower than via list comprehension. + axis = tuple(normalize_axis_index(ax, ndim, argname) for ax in axis) + if not allow_duplicate and len(set(map(int, axis))) != len(axis): + if argname: + raise ValueError(f"repeated axis in `{argname}` argument") + else: + raise ValueError("repeated axis") + return axis + + +def allow_only_single_axis(axis): + if axis is None: + return axis + if len(axis) != 1: + raise NotImplementedError("does not handle tuple axis") + return axis[0] + + +def expand_shape(arr_shape, axis): + # taken from numpy 1.23.x, expand_dims function + if type(axis) not in (list, tuple): + axis = (axis,) + out_ndim = len(axis) + len(arr_shape) + axis = normalize_axis_tuple(axis, out_ndim) + shape_it = iter(arr_shape) + shape = [1 if ax in axis else next(shape_it) for ax in range(out_ndim)] + return shape + + +def apply_keepdims(tensor, axis, ndim): + if axis is None: + # tensor was a scalar + shape = (1,) * ndim + tensor = tensor.expand(shape).contiguous() + else: + shape = expand_shape(tensor.shape, axis) + tensor = tensor.reshape(shape) + return tensor + + +def axis_none_flatten(*tensors, axis=None): + """Flatten the arrays if axis is None.""" + if axis is None: + tensors = tuple(ar.flatten() for ar in tensors) + return tensors, 0 + else: + return tensors, axis + + +def typecast_tensor(t, target_dtype, casting): + """Dtype-cast tensor to target_dtype. + + Parameters + ---------- + t : torch.Tensor + The tensor to cast + target_dtype : torch dtype object + The array dtype to cast all tensors to + casting : str + The casting mode, see `np.can_cast` + + Returns + ------- + `torch.Tensor` of the `target_dtype` dtype + + Raises + ------ + ValueError + if the argument cannot be cast according to the `casting` rule + + """ + can_cast = _dtypes_impl.can_cast_impl + + if not can_cast(t.dtype, target_dtype, casting=casting): + raise TypeError( + f"Cannot cast array data from {t.dtype} to" + f" {target_dtype} according to the rule '{casting}'" + ) + return cast_if_needed(t, target_dtype) + + +def typecast_tensors(tensors, target_dtype, casting): + return tuple(typecast_tensor(t, target_dtype, casting) for t in tensors) + + +def _try_convert_to_tensor(obj): + try: + tensor = torch.as_tensor(obj) + except Exception as e: + mesg = f"failed to convert {obj} to ndarray. \nInternal error is: {str(e)}." + raise NotImplementedError(mesg) # noqa: B904 + return tensor + + +def _coerce_to_tensor(obj, dtype=None, copy=False, ndmin=0): + """The core logic of the array(...) function. + + Parameters + ---------- + obj : tensor_like + The thing to coerce + dtype : torch.dtype object or None + Coerce to this torch dtype + copy : bool + Copy or not + ndmin : int + The results as least this many dimensions + is_weak : bool + Whether obj is a weakly typed python scalar. + + Returns + ------- + tensor : torch.Tensor + a tensor object with requested dtype, ndim and copy semantics. + + Notes + ----- + This is almost a "tensor_like" coercive function. Does not handle wrapper + ndarrays (those should be handled in the ndarray-aware layer prior to + invoking this function). + """ + if isinstance(obj, torch.Tensor): + tensor = obj + else: + # tensor.dtype is the pytorch default, typically float32. If obj's elements + # are not exactly representable in float32, we've lost precision: + # >>> torch.as_tensor(1e12).item() - 1e12 + # -4096.0 + default_dtype = torch.get_default_dtype() + torch.set_default_dtype(_dtypes_impl.get_default_dtype_for(torch.float32)) + try: + tensor = _try_convert_to_tensor(obj) + finally: + torch.set_default_dtype(default_dtype) + + # type cast if requested + tensor = cast_if_needed(tensor, dtype) + + # adjust ndim if needed + ndim_extra = ndmin - tensor.ndim + if ndim_extra > 0: + tensor = tensor.view((1,) * ndim_extra + tensor.shape) + + # special handling for np._CopyMode + try: + copy = bool(copy) + except ValueError: + # TODO handle _CopyMode.IF_NEEDED correctly + copy = False + # copy if requested + if copy: + tensor = tensor.clone() + + return tensor + + +def ndarrays_to_tensors(*inputs): + """Convert all ndarrays from `inputs` to tensors. (other things are intact)""" + from ._ndarray import ndarray + + if len(inputs) == 0: + return ValueError() + elif len(inputs) == 1: + input_ = inputs[0] + if isinstance(input_, ndarray): + return input_.tensor + elif isinstance(input_, tuple): + result = [] + for sub_input in input_: + sub_result = ndarrays_to_tensors(sub_input) + result.append(sub_result) + return tuple(result) + else: + return input_ + else: + assert isinstance(inputs, tuple) # sanity check + return ndarrays_to_tensors(inputs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/fft.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/fft.py new file mode 100644 index 0000000000000000000000000000000000000000..b7d2f8365dbbd96c30c047d743dda806f12cb211 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/fft.py @@ -0,0 +1,130 @@ +# mypy: ignore-errors + +from __future__ import annotations + +import functools + +import torch + +from . import _dtypes_impl, _util +from ._normalizations import ArrayLike, normalizer + + +def upcast(func): + """NumPy fft casts inputs to 64 bit and *returns 64-bit results*.""" + + @functools.wraps(func) + def wrapped(tensor, *args, **kwds): + target_dtype = ( + _dtypes_impl.default_dtypes().complex_dtype + if tensor.is_complex() + else _dtypes_impl.default_dtypes().float_dtype + ) + tensor = _util.cast_if_needed(tensor, target_dtype) + return func(tensor, *args, **kwds) + + return wrapped + + +@normalizer +@upcast +def fft(a: ArrayLike, n=None, axis=-1, norm=None): + return torch.fft.fft(a, n, dim=axis, norm=norm) + + +@normalizer +@upcast +def ifft(a: ArrayLike, n=None, axis=-1, norm=None): + return torch.fft.ifft(a, n, dim=axis, norm=norm) + + +@normalizer +@upcast +def rfft(a: ArrayLike, n=None, axis=-1, norm=None): + return torch.fft.rfft(a, n, dim=axis, norm=norm) + + +@normalizer +@upcast +def irfft(a: ArrayLike, n=None, axis=-1, norm=None): + return torch.fft.irfft(a, n, dim=axis, norm=norm) + + +@normalizer +@upcast +def fftn(a: ArrayLike, s=None, axes=None, norm=None): + return torch.fft.fftn(a, s, dim=axes, norm=norm) + + +@normalizer +@upcast +def ifftn(a: ArrayLike, s=None, axes=None, norm=None): + return torch.fft.ifftn(a, s, dim=axes, norm=norm) + + +@normalizer +@upcast +def rfftn(a: ArrayLike, s=None, axes=None, norm=None): + return torch.fft.rfftn(a, s, dim=axes, norm=norm) + + +@normalizer +@upcast +def irfftn(a: ArrayLike, s=None, axes=None, norm=None): + return torch.fft.irfftn(a, s, dim=axes, norm=norm) + + +@normalizer +@upcast +def fft2(a: ArrayLike, s=None, axes=(-2, -1), norm=None): + return torch.fft.fft2(a, s, dim=axes, norm=norm) + + +@normalizer +@upcast +def ifft2(a: ArrayLike, s=None, axes=(-2, -1), norm=None): + return torch.fft.ifft2(a, s, dim=axes, norm=norm) + + +@normalizer +@upcast +def rfft2(a: ArrayLike, s=None, axes=(-2, -1), norm=None): + return torch.fft.rfft2(a, s, dim=axes, norm=norm) + + +@normalizer +@upcast +def irfft2(a: ArrayLike, s=None, axes=(-2, -1), norm=None): + return torch.fft.irfft2(a, s, dim=axes, norm=norm) + + +@normalizer +@upcast +def hfft(a: ArrayLike, n=None, axis=-1, norm=None): + return torch.fft.hfft(a, n, dim=axis, norm=norm) + + +@normalizer +@upcast +def ihfft(a: ArrayLike, n=None, axis=-1, norm=None): + return torch.fft.ihfft(a, n, dim=axis, norm=norm) + + +@normalizer +def fftfreq(n, d=1.0): + return torch.fft.fftfreq(n, d) + + +@normalizer +def rfftfreq(n, d=1.0): + return torch.fft.rfftfreq(n, d) + + +@normalizer +def fftshift(x: ArrayLike, axes=None): + return torch.fft.fftshift(x, axes) + + +@normalizer +def ifftshift(x: ArrayLike, axes=None): + return torch.fft.ifftshift(x, axes) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/linalg.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/linalg.py new file mode 100644 index 0000000000000000000000000000000000000000..4ea3b46f23e6a5144c14ce49f8349d4b2cf3fcf5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/linalg.py @@ -0,0 +1,243 @@ +# mypy: ignore-errors + +from __future__ import annotations + +import functools +import math +from typing import TYPE_CHECKING + +import torch + +from . import _dtypes_impl, _util +from ._normalizations import ArrayLike, KeepDims, normalizer + + +if TYPE_CHECKING: + from collections.abc import Sequence + + +class LinAlgError(Exception): + pass + + +def _atleast_float_1(a): + if not (a.dtype.is_floating_point or a.dtype.is_complex): + a = a.to(_dtypes_impl.default_dtypes().float_dtype) + return a + + +def _atleast_float_2(a, b): + dtyp = _dtypes_impl.result_type_impl(a, b) + if not (dtyp.is_floating_point or dtyp.is_complex): + dtyp = _dtypes_impl.default_dtypes().float_dtype + + a = _util.cast_if_needed(a, dtyp) + b = _util.cast_if_needed(b, dtyp) + return a, b + + +def linalg_errors(func): + @functools.wraps(func) + def wrapped(*args, **kwds): + try: + return func(*args, **kwds) + except torch._C._LinAlgError as e: + raise LinAlgError(*e.args) # noqa: B904 + + return wrapped + + +# ### Matrix and vector products ### + + +@normalizer +@linalg_errors +def matrix_power(a: ArrayLike, n): + a = _atleast_float_1(a) + return torch.linalg.matrix_power(a, n) + + +@normalizer +@linalg_errors +def multi_dot(inputs: Sequence[ArrayLike], *, out=None): + return torch.linalg.multi_dot(inputs) + + +# ### Solving equations and inverting matrices ### + + +@normalizer +@linalg_errors +def solve(a: ArrayLike, b: ArrayLike): + a, b = _atleast_float_2(a, b) + return torch.linalg.solve(a, b) + + +@normalizer +@linalg_errors +def lstsq(a: ArrayLike, b: ArrayLike, rcond=None): + a, b = _atleast_float_2(a, b) + # NumPy is using gelsd: https://github.com/numpy/numpy/blob/v1.24.0/numpy/linalg/umath_linalg.cpp#L3991 + # on CUDA, only `gels` is available though, so use it instead + driver = "gels" if a.is_cuda or b.is_cuda else "gelsd" + return torch.linalg.lstsq(a, b, rcond=rcond, driver=driver) + + +@normalizer +@linalg_errors +def inv(a: ArrayLike): + a = _atleast_float_1(a) + result = torch.linalg.inv(a) + return result + + +@normalizer +@linalg_errors +def pinv(a: ArrayLike, rcond=1e-15, hermitian=False): + a = _atleast_float_1(a) + return torch.linalg.pinv(a, rtol=rcond, hermitian=hermitian) + + +@normalizer +@linalg_errors +def tensorsolve(a: ArrayLike, b: ArrayLike, axes=None): + a, b = _atleast_float_2(a, b) + return torch.linalg.tensorsolve(a, b, dims=axes) + + +@normalizer +@linalg_errors +def tensorinv(a: ArrayLike, ind=2): + a = _atleast_float_1(a) + return torch.linalg.tensorinv(a, ind=ind) + + +# ### Norms and other numbers ### + + +@normalizer +@linalg_errors +def det(a: ArrayLike): + a = _atleast_float_1(a) + return torch.linalg.det(a) + + +@normalizer +@linalg_errors +def slogdet(a: ArrayLike): + a = _atleast_float_1(a) + return torch.linalg.slogdet(a) + + +@normalizer +@linalg_errors +def cond(x: ArrayLike, p=None): + x = _atleast_float_1(x) + + # check if empty + # cf: https://github.com/numpy/numpy/blob/v1.24.0/numpy/linalg/linalg.py#L1744 + if x.numel() == 0 and math.prod(x.shape[-2:]) == 0: + raise LinAlgError("cond is not defined on empty arrays") + + result = torch.linalg.cond(x, p=p) + + # Convert nans to infs (numpy does it in a data-dependent way, depending on + # whether the input array has nans or not) + # XXX: NumPy does this: https://github.com/numpy/numpy/blob/v1.24.0/numpy/linalg/linalg.py#L1744 + return torch.where(torch.isnan(result), float("inf"), result) + + +@normalizer +@linalg_errors +def matrix_rank(a: ArrayLike, tol=None, hermitian=False): + a = _atleast_float_1(a) + + if a.ndim < 2: + return int((a != 0).any()) + + if tol is None: + # follow https://github.com/numpy/numpy/blob/v1.24.0/numpy/linalg/linalg.py#L1885 + atol = 0 + rtol = max(a.shape[-2:]) * torch.finfo(a.dtype).eps + else: + atol, rtol = tol, 0 + return torch.linalg.matrix_rank(a, atol=atol, rtol=rtol, hermitian=hermitian) + + +@normalizer +@linalg_errors +def norm(x: ArrayLike, ord=None, axis=None, keepdims: KeepDims = False): + x = _atleast_float_1(x) + return torch.linalg.norm(x, ord=ord, dim=axis) + + +# ### Decompositions ### + + +@normalizer +@linalg_errors +def cholesky(a: ArrayLike): + a = _atleast_float_1(a) + return torch.linalg.cholesky(a) + + +@normalizer +@linalg_errors +def qr(a: ArrayLike, mode="reduced"): + a = _atleast_float_1(a) + result = torch.linalg.qr(a, mode=mode) + if mode == "r": + # match NumPy + result = result.R + return result + + +@normalizer +@linalg_errors +def svd(a: ArrayLike, full_matrices=True, compute_uv=True, hermitian=False): + a = _atleast_float_1(a) + if not compute_uv: + return torch.linalg.svdvals(a) + + # NB: ignore the hermitian= argument (no pytorch equivalent) + result = torch.linalg.svd(a, full_matrices=full_matrices) + return result + + +# ### Eigenvalues and eigenvectors ### + + +@normalizer +@linalg_errors +def eig(a: ArrayLike): + a = _atleast_float_1(a) + w, vt = torch.linalg.eig(a) + + if not a.is_complex() and w.is_complex() and (w.imag == 0).all(): + w = w.real + vt = vt.real + return w, vt + + +@normalizer +@linalg_errors +def eigh(a: ArrayLike, UPLO="L"): + a = _atleast_float_1(a) + return torch.linalg.eigh(a, UPLO=UPLO) + + +@normalizer +@linalg_errors +def eigvals(a: ArrayLike): + a = _atleast_float_1(a) + result = torch.linalg.eigvals(a) + if not a.is_complex() and result.is_complex() and (result.imag == 0).all(): + result = result.real + return result + + +@normalizer +@linalg_errors +def eigvalsh(a: ArrayLike, UPLO="L"): + a = _atleast_float_1(a) + return torch.linalg.eigvalsh(a, UPLO=UPLO) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/random.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/random.py new file mode 100644 index 0000000000000000000000000000000000000000..a3d4a1c73241f852c10917e95ebb38968bc92acc --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_numpy/random.py @@ -0,0 +1,192 @@ +# mypy: ignore-errors + +"""Wrapper to mimic (parts of) np.random API surface. + +NumPy has strict guarantees on reproducibility etc; here we don't give any. + +Q: default dtype is float64 in numpy + +""" + +from __future__ import annotations + +import functools +from math import sqrt +from typing import Optional + +import torch + +from . import _dtypes_impl, _util +from ._normalizations import array_or_scalar, ArrayLike, normalizer + + +__all__ = [ + "seed", + "random_sample", + "sample", + "random", + "rand", + "randn", + "normal", + "choice", + "randint", + "shuffle", + "uniform", +] + + +def use_numpy_random(): + # local import to avoid ref cycles + import torch._dynamo.config as config + + return config.use_numpy_random_stream + + +def deco_stream(func): + @functools.wraps(func) + def inner(*args, **kwds): + if not use_numpy_random(): + return func(*args, **kwds) + else: + import numpy + + from ._ndarray import ndarray + + f = getattr(numpy.random, func.__name__) + + # numpy funcs accept numpy ndarrays, unwrap + args = tuple( + arg.tensor.numpy() if isinstance(arg, ndarray) else arg for arg in args + ) + kwds = { + key: val.tensor.numpy() if isinstance(val, ndarray) else val + for key, val in kwds.items() + } + + value = f(*args, **kwds) + + # `value` can be either numpy.ndarray or python scalar (or None) + if isinstance(value, numpy.ndarray): + value = ndarray(torch.as_tensor(value)) + + return value + + return inner + + +@deco_stream +def seed(seed=None): + if seed is not None: + torch.random.manual_seed(seed) + + +@deco_stream +def random_sample(size=None): + if size is None: + size = () + dtype = _dtypes_impl.default_dtypes().float_dtype + values = torch.empty(size, dtype=dtype).uniform_() + return array_or_scalar(values, return_scalar=size == ()) + + +def rand(*size): + if size == (): + size = None + return random_sample(size) + + +sample = random_sample +random = random_sample + + +@deco_stream +def uniform(low=0.0, high=1.0, size=None): + if size is None: + size = () + dtype = _dtypes_impl.default_dtypes().float_dtype + values = torch.empty(size, dtype=dtype).uniform_(low, high) + return array_or_scalar(values, return_scalar=size == ()) + + +@deco_stream +def randn(*size): + dtype = _dtypes_impl.default_dtypes().float_dtype + values = torch.randn(size, dtype=dtype) + return array_or_scalar(values, return_scalar=size == ()) + + +@deco_stream +def normal(loc=0.0, scale=1.0, size=None): + if size is None: + size = () + dtype = _dtypes_impl.default_dtypes().float_dtype + values = torch.empty(size, dtype=dtype).normal_(loc, scale) + return array_or_scalar(values, return_scalar=size == ()) + + +@deco_stream +def shuffle(x): + # no @normalizer because we do not cast e.g. lists to tensors + from ._ndarray import ndarray + + if isinstance(x, torch.Tensor): + tensor = x + elif isinstance(x, ndarray): + tensor = x.tensor + else: + raise NotImplementedError("We do not random.shuffle lists in-place") + + perm = torch.randperm(tensor.shape[0]) + xp = tensor[perm] + tensor.copy_(xp) + + +@deco_stream +def randint(low, high=None, size=None): + if size is None: + size = () + if not isinstance(size, (tuple, list)): + size = (size,) + if high is None: + low, high = 0, low + values = torch.randint(low, high, size=size) + return array_or_scalar(values, int, return_scalar=size == ()) + + +@deco_stream +@normalizer +def choice(a: ArrayLike, size=None, replace=True, p: Optional[ArrayLike] = None): + # https://stackoverflow.com/questions/59461811/random-choice-with-pytorch + if a.numel() == 1: + a = torch.arange(a) + + # TODO: check a.dtype is integer -- cf np.random.choice(3.4) which raises + + # number of draws + if size is None: + num_el = 1 + elif _util.is_sequence(size): + num_el = 1 + for el in size: + num_el *= el + else: + num_el = size + + # prepare the probabilities + if p is None: + p = torch.ones_like(a) / a.shape[0] + + # cf https://github.com/numpy/numpy/blob/main/numpy/random/mtrand.pyx#L973 + atol = sqrt(torch.finfo(p.dtype).eps) + if abs(p.sum() - 1.0) > atol: + raise ValueError("probabilities do not sum to 1.") + + # actually sample + indices = torch.multinomial(p, num_el, replacement=replace) + + if _util.is_sequence(size): + indices = indices.reshape(size) + + samples = a[indices] + + return samples diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1fbc5fc3ebca56a3e9a6cabb3e663a1813429618 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/__init__.py @@ -0,0 +1,3001 @@ +# mypy: allow-untyped-defs +import operator +from collections.abc import Callable, Sequence +from enum import Enum +from functools import partial, reduce +from typing import Optional, Union + +import torch +import torch._prims_common as utils +import torch.library +from torch import sym_float, Tensor +from torch._C import _get_default_device +from torch._higher_order_ops.effects import new_token_tensor +from torch._library.utils import is_functional_schema +from torch._prims.debug_prims import register_debug_prims +from torch._prims.rng_prims import register_rng_prims +from torch._prims_common import ( + Dim, + DimsSequenceType, + DimsType, + IntLike, + Number, + NumberType, + RETURN_TYPE, + ShapeType, + StrideType, + TensorLike, + TensorLikeType, + type_to_dtype, +) +from torch._prims_common.wrappers import backwards_not_supported +from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode +from torch.overrides import handle_torch_function, has_torch_function +from torch.utils._pytree import tree_flatten, tree_map, tree_unflatten + + +prim = torch.library.Library("prims", "DEF") +prim_impl = torch.library.Library("prims", "IMPL", "CompositeExplicitAutograd") +prim_backend_select_impl = torch.library.Library("prims", "IMPL", "BackendSelect") +prim_autograd_impl = torch.library.Library("prims", "IMPL", "Autograd") +prim_meta_impl = torch.library.Library("prims", "IMPL", "Meta") + +# Experimental module containing prototype "primitive" operations. + +__all__ = [ + # + # Common datastructures and helpers + # + "RETURN_TYPE", + # + # Elementwise unary prims + # + "abs", + "acos", + "acosh", + "asin", + "asinh", + "atan", + "atanh", + "cos", + "cosh", + "bessel_i0", + "bessel_i0e", + "bessel_i1", + "bessel_i1e", + "bessel_j0", + "bessel_j1", + "bitwise_not", + "cbrt", + "ceil", + "conj_physical", + "digamma", + "erf", + "erf_inv", + "erfc", + "erfcx", + "exp", + "expm1", + "exp2", + "fill", + "floor", + "imag", + "isfinite", + "lgamma", + "log", + "log1p", + "log2", + "log10", + "ndtri", + "neg", + "real", + "reciprocal", + "round", + "sign", + "signbit", + "sin", + "sinh", + "spherical_bessel_j0", + "sqrt", + "tan", + "tanh", + "trunc", + # + # Elementwise binary prims + # + "add", + "atan2", + "bitwise_and", + "bitwise_or", + "bitwise_xor", + # 'complex', # needs custom meta + "div", + "eq", + "fmax", + "fmin", + "fmod", + "frexp", + "gcd", + "ge", + "gt", + "hypot", + "igamma", + "igammac", + "le", + "lt", + "maximum", + "minimum", + "mul", + "ne", + "nextafter", + "pow", + "remainder", + "rsqrt", + "shift_left", + "shift_right_arithmetic", + "shift_right_logical", # not implemented + "sub", + "zeta", + # + # View prims + # + "as_strided", + "broadcast_in_dim", + "collapse_view", + "conj", + "expand_dims", + "slice", + "split_dim", + "squeeze", + "transpose", + "view_of", + "view_element_type", + # + # Functionalized view mutations + # + "as_strided_scatter", + # + # Shape prims + # + "collapse", + "cat", + "reshape", + "rev", + # + # Conditional prims + # + "where", + # + # Data conversion and movement prims + # + "clone", + "convert_element_type", + "device_put", + "item", + "maximum_value", + "minimum_value", + "copy_strided", + # + # Inplace prims + # + "copy_to", + "resize", + # "_set", # Commented out, see note below + # + # Reduction prims + # + "amax", + "amin", + "prod", + "sum", + "xor_sum", + "var", + # + # Tensor Creation Prims + # + "empty_strided", + "empty_permuted", + "scalar_tensor", + "iota", + # + # Linear algebra (linalg) Prims + # + "svd", + # + # Randomness Prims + # + "normal", + "_uniform_helper", + # + # FFT prims + # + "fft_r2c", + "fft_c2c", + "fft_c2r", + # + # prims for making/sinking tokens + # + "_make_token", + "_sink_tokens", +] + + +def TensorMeta( + tensorlike: NumberType | torch.Tensor | None = None, + *, + shape: ShapeType | None = None, + strides: StrideType | None = None, + dtype: torch.dtype | None = None, + device: torch.device | str | None = None, +): + if isinstance(tensorlike, Number): + assert not shape and (shape is None or isinstance(shape, Sequence)) + assert not strides and (strides is None or isinstance(strides, Sequence)) + inferred_shape: tuple[int, ...] = () + inferred_strides: tuple[int, ...] = () + inferred_dtype = type_to_dtype(type(tensorlike)) + inferred_device = torch.device("cpu") + # TODO: This looks wrong, a number that is wrapped into a tensor + # needs to behave differently than a scalar tensor for type + # promotion purposes + elif tensorlike is not None: + assert isinstance(tensorlike, torch.Tensor) + inferred_shape = tuple(tensorlike.shape) + inferred_strides = tuple(tensorlike.stride()) + inferred_dtype = tensorlike.dtype + inferred_device = tensorlike.device + else: + # If no tensorlike "example" is given then all metadata + # must be provided explicitly + assert shape is not None + assert strides is not None + assert dtype is not None + assert device is not None + + shape = inferred_shape if shape is None else tuple(shape) # type: ignore[possibly-undefined] + strides = inferred_strides if strides is None else tuple(strides) # type: ignore[possibly-undefined] + dtype = inferred_dtype if dtype is None else dtype # type: ignore[possibly-undefined] + device = inferred_device if device is None else device # type: ignore[possibly-undefined] + + if isinstance(device, str): + device = torch.device(device) + + return torch.empty_strided(shape, strides, dtype=dtype, device=device) + + +def _make_prim( + *, + schema: str, + return_type: RETURN_TYPE | tuple[RETURN_TYPE, ...], + meta: Callable, + impl_aten: Callable, + doc: str, + tags: Sequence[torch.Tag] | None = None, + use_old_custom_ops_api: bool = False, + register_conj_neg_fallthrough: bool = False, +): + """ + Creates a primitive operation. + + """ + + def _prim_impl(*args, **kwargs): + # always run the meta function because aten implementation will + # typically accept more inputs (e.g., it will do promotion and + # broadcasting) which we want to reject + meta(*args, **kwargs) + return impl_aten(*args, **kwargs) + + # Right now prims don't support autograd (we can and should add an + # argument that provides an implementation for backward here.) Because we + # don't have derivative formulas, we must setup a custom autograd function + # that raises an error if backwards is invoked + def _autograd_impl(*args, **kwargs): + return backwards_not_supported(_prim)(*args, **kwargs) + + def _backend_select_impl(*args, **kwargs): + if kwargs.get("device") and kwargs["device"].type == "meta": + return meta(*args, **kwargs) + if any(isinstance(x, torch.device) and x.type == "meta" for x in args): + return meta(*args, **kwargs) + else: + return _prim_impl(*args, **kwargs) + + name = schema.split("(", maxsplit=1)[0] + schema = schema[len(name) :] + + # register non-functional ops with old custom ops API + cpp_schema = torch._C.parse_schema(name + schema) + if use_old_custom_ops_api or not is_functional_schema(cpp_schema): + prim.define(name + schema, tags=torch.Tag.pt2_compliant_tag) + prim_impl.impl(name, _prim_impl) + prim_autograd_impl.impl(name, _autograd_impl) + prim_meta_impl.impl(name, meta) + else: + mutates_args = [ + arg.name + for arg in cpp_schema.arguments + if arg.alias_info is not None and arg.alias_info.is_write + ] + prim_def = torch.library.custom_op( + "prims::" + name, + _prim_impl, + mutates_args=tuple(mutates_args), + schema=schema, + ) + prim_def.register_fake(meta) + + # all view ops get conj/neg fallthroughs + if return_type == RETURN_TYPE.VIEW or register_conj_neg_fallthrough: + prim_def._lib.impl(name, torch.library.fallthrough_kernel, "Conjugate") + prim_def._lib.impl(name, torch.library.fallthrough_kernel, "Negative") + + _prim_packet = getattr(torch._ops.ops.prims, name) + _prim = _prim_packet.default + if tags: + _prim._tags = tags + elif aten_packet := getattr(torch.ops.aten, name, None): + overload_tags = [ + getattr(aten_packet, overload).tags for overload in aten_packet.overloads() + ] + tags_intersection = set(overload_tags[0]) + tags_intersection.intersection_update(*overload_tags[1:]) + + # dont inadvertently add to prim ops + tags_intersection.discard(torch.Tag.core) + # causes errors with python ref executor tests, none of the + # data dependent pytorch ops actually decompose to prims + tags_intersection.discard(torch.Tag.data_dependent_output) + + # iter over first tags for determinism + _prim._tags = tuple(t for t in overload_tags[0] if t in tags_intersection) + + from torch._subclasses.fake_tensor import contains_tensor_types + + if ( + not any(contains_tensor_types(a.type) for a in _prim._schema.arguments) + or str( + _prim + # See https://github.com/pytorch/pytorch/issues/103532 + ) + == "prims.device_put.default" + ): + prim_backend_select_impl.impl(name, _backend_select_impl) + + for p in (_prim_packet, _prim): + p.__doc__ = doc + p.return_type = return_type # type: ignore[attr-defined] + + p.schema = schema + p.prim_impl = _prim_impl + p.prim_meta_impl = meta + p.impl_aten = impl_aten + + return _prim + + +class ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND(Enum): + DEFAULT = (0,) + INT_TO_FLOAT = (2,) + ALWAYS_BOOL = (3,) + COMPLEX_TO_FLOAT = (4,) + + +# TODO: implement dtype validation here, too, or on the corresponding refs +def _prim_elementwise_meta( + *args, + type_promotion: ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND, + args_with_fixed_dtypes: tuple[TensorLikeType, ...] | None = None, +) -> FakeTensor: + """ + Meta function for elementwise operations that produce outputs in the same dtype + as their inputs. + + Stride logic is currently incorrect. + """ + + assert len(args) > 0 + + utils.check_same_dtype(*args) + + args_ = list(args) + if args_with_fixed_dtypes is not None: + args_ = list(args_with_fixed_dtypes) + args_ + + utils.check_same_device(*args_, allow_cpu_scalar_tensors=True) + utils.check_same_shape(*args_, allow_cpu_scalar_tensors=True) + + l2p_perm, _ = utils.compute_elementwise_output_logical_to_physical_perm(*args_) + shape = utils.extract_shape(*args_, allow_cpu_scalar_tensors=True) + + # Acquires the dtype + dtype = None + scalar_type = None + for arg in args: + if isinstance(arg, TensorLike): + if not utils.is_cpu_scalar_tensor(arg): + dtype = arg.dtype + break + else: + dtype = arg.dtype + elif isinstance(arg, Number): + scalar_type = type(arg) + + if dtype is None and scalar_type is not None: + dtype = utils.type_to_dtype(scalar_type) + + # Acquires the device (if it exists) or number + device = None + number = None + # pyrefly: ignore [bad-assignment] + for arg in args_: + if isinstance(arg, TensorLike): + if utils.is_cpu_scalar_tensor(arg): + if device is None: + device = arg.device + # keep going, in case there is a cuda tensor later + else: + device = arg.device + break + + elif isinstance(arg, Number): + if number is None: + number = arg + + # NOTE: type promotion behavior here is mostly hidden from tests because + # references will typically handle the type promotion properly even if this doesn't + # (but getting it wrong will cause too many casts to be inserted in traces!) + if device is not None: + assert dtype is not None + if type_promotion == ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.ALWAYS_BOOL: + dtype = torch.bool + elif type_promotion == ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.INT_TO_FLOAT: + if utils.is_integer_dtype(dtype) or utils.is_boolean_dtype(dtype): + dtype = torch.get_default_dtype() + elif type_promotion == ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.COMPLEX_TO_FLOAT: + if utils.is_complex_dtype(dtype): + dtype = utils.corresponding_real_dtype(dtype) + + assert shape is not None + return torch.empty_permuted(shape, l2p_perm, device=device, dtype=dtype) # type: ignore[return-value] + + # Number case + # TODO: fix number type promotion (bool, complex->float) + + # For now for symint/float, just implementing the common / simple cases of (int,float,symint,symfloat) + seen_float = False + if isinstance(number, (torch.SymInt, torch.SymFloat)): + for a in args: + assert isinstance(a, (int, float, torch.SymInt, torch.SymFloat)), "NYI" + seen_float = seen_float or isinstance(a, (float, torch.SymFloat)) + if seen_float: + number = sym_float(number) + + return TensorMeta(number) # type: ignore[arg-type] + + +def _complex_only_elementwise_meta(*args, **kwargs): + torch._check( + utils.is_complex_dtype(args[0].dtype), lambda: "Only complex dtype is supported" + ) + return _prim_elementwise_meta(*args, **kwargs) + + +def _make_elementwise_unary_prim( + name: str, *, type_promotion: ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND, **kwargs +): + """ + Creates an elementwise unary prim. + """ + + return _make_prim( + schema=f"{name}(Tensor self) -> Tensor", + meta=partial(_prim_elementwise_meta, type_promotion=type_promotion), + return_type=RETURN_TYPE.NEW, + **kwargs, + ) + + +def _make_elementwise_binary_prim( + name: str, *, type_promotion: ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND, **kwargs +): + """ + Creates an elementwise binary prim. + """ + + return _make_prim( + schema=f"{name}(Tensor self, Tensor other) -> Tensor", + meta=partial(_prim_elementwise_meta, type_promotion=type_promotion), + return_type=RETURN_TYPE.NEW, + **kwargs, + ) + + +def _not_impl(*args, **kwargs): + raise NotImplementedError + + +# +# Elementwise unary operations +# + + +abs = _make_elementwise_unary_prim( + "abs", + impl_aten=torch.abs, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.COMPLEX_TO_FLOAT, +) + +acos = _make_elementwise_unary_prim( + "acos", + impl_aten=torch.acos, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +acosh = _make_elementwise_unary_prim( + "acosh", + impl_aten=torch.acosh, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +asin = _make_elementwise_unary_prim( + "asin", + impl_aten=torch.asin, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +asinh = _make_elementwise_unary_prim( + "asinh", + impl_aten=torch.asinh, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +atan = _make_elementwise_unary_prim( + "atan", + impl_aten=torch.atan, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +atanh = _make_elementwise_unary_prim( + "atanh", + impl_aten=torch.atanh, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +cos = _make_elementwise_unary_prim( + "cos", + impl_aten=torch.cos, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +cosh = _make_elementwise_unary_prim( + "cosh", + impl_aten=torch.cosh, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +bessel_j0 = _make_elementwise_unary_prim( + "bessel_j0", + impl_aten=torch.special.bessel_j0, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +bessel_j1 = _make_elementwise_unary_prim( + "bessel_j1", + impl_aten=torch.special.bessel_j1, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +bessel_i0 = _make_elementwise_unary_prim( + "bessel_i0", + impl_aten=torch.i0, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +bessel_i0e = _make_elementwise_unary_prim( + "bessel_i0e", + impl_aten=torch.special.i0e, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +bessel_i1 = _make_elementwise_unary_prim( + "bessel_i1", + impl_aten=torch.special.i1, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +bessel_i1e = _make_elementwise_unary_prim( + "bessel_i1e", + impl_aten=torch.special.i1e, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +bitwise_not = _make_elementwise_unary_prim( + "bitwise_not", + impl_aten=torch.bitwise_not, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + + +def _cbrt_aten(a: torch.Tensor) -> Tensor: + torch._check( + not a.is_complex(), + lambda: "cbrt: Complex inputs not supported. Consider calling torch.pow(a, 1.0/3.0)", + ) + # Returns the real cubic root of the number. + # Note that if a < 0, pow(a, (1. / 3.)) returns th complex number + # exp(1/3 * log(a)) = exp(1/3 * (log(abs(a)) + pi*i)) = cbrt(abs(a)) * e^{pi/3*i} + # which is a complex number. + # For more info see the section Note in + # https://en.cppreference.com/w/cpp/numeric/math/cbrt + return torch.copysign(torch.pow(a.abs(), 1 / 3), a) + + +cbrt = _make_elementwise_unary_prim( + "cbrt", + impl_aten=_cbrt_aten, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +ceil = _make_elementwise_unary_prim( + "ceil", + impl_aten=torch.ceil, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + + +def _conj_physical_meta(input: TensorLikeType) -> TensorLikeType: + if not input.dtype.is_complex: + raise RuntimeError("prims.conj_physical is only defined for complex dtypes") + + strides = utils.compute_elementwise_output_strides(input) + return TensorMeta(input, strides=strides) + + +conj_physical = _make_prim( + schema="conj_physical(Tensor self) -> Tensor", + meta=_conj_physical_meta, + impl_aten=torch._conj_physical, + doc="Returns the physical conjugation of a complex tensor", + return_type=RETURN_TYPE.NEW, +) + + +def _clone_meta( + input: TensorLikeType, *, memory_format: torch.memory_format = torch.preserve_format +) -> TensorLikeType: + if memory_format != torch.preserve_format: + return torch.empty( + input.shape, + dtype=input.dtype, + layout=input.layout, + device=input.device, + memory_format=memory_format, + ) + else: + # Match eager behavior by preserving strides for non_overlapping_and_dense tensors + # If not, eager clone creates contiguous strides + computed_stride = None + if utils.is_non_overlapping_and_dense(input): + computed_stride = input.stride() + else: + computed_stride = utils.compute_elementwise_output_strides(input) + + return torch.empty_strided( + input.shape, + computed_stride, + dtype=input.dtype, + layout=input.layout, + device=input.device, + ) + + +clone = _make_prim( + schema="clone(Tensor self, *, MemoryFormat? memory_format=None) -> Tensor", + meta=_clone_meta, + impl_aten=torch.clone, + doc="Returns the copy of a tensor", + return_type=RETURN_TYPE.NEW, + register_conj_neg_fallthrough=True, +) + +digamma = _make_elementwise_unary_prim( + "digamma", + impl_aten=torch.digamma, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +erf = _make_elementwise_unary_prim( + "erf", + impl_aten=torch.erf, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +erf_inv = _make_elementwise_unary_prim( + "erf_inv", + impl_aten=torch.special.erfinv, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +erfc = _make_elementwise_unary_prim( + "erfc", + impl_aten=torch.special.erfc, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +erfcx = _make_elementwise_unary_prim( + "erfcx", + impl_aten=torch.special.erfcx, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +exp = _make_elementwise_unary_prim( + "exp", + impl_aten=torch.exp, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +expm1 = _make_elementwise_unary_prim( + "expm1", + impl_aten=torch.special.expm1, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +exp2 = _make_elementwise_unary_prim( + "exp2", + impl_aten=torch.special.exp2, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + + +def _fill_meta(a: TensorLikeType, value: NumberType) -> TensorLikeType: + return _prim_elementwise_meta( + a, type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT + ) + + +# NOTE: fill uses _make_prim directly because it has a value parameter +fill = _make_prim( + schema="fill(Tensor self, Scalar value) -> Tensor", + return_type=RETURN_TYPE.NEW, + meta=_fill_meta, + impl_aten=torch.fill, + doc="", +) + +floor = _make_elementwise_unary_prim( + "floor", + impl_aten=torch.floor, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +imag = _make_prim( + schema="imag(Tensor(a) self) -> Tensor(a)", + meta=partial( + _complex_only_elementwise_meta, + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.COMPLEX_TO_FLOAT, + ), + return_type=RETURN_TYPE.VIEW, + impl_aten=torch.imag, + doc="", +) + +isfinite = _make_elementwise_unary_prim( + "isfinite", + impl_aten=torch.isfinite, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.ALWAYS_BOOL, +) + +lgamma = _make_elementwise_unary_prim( + "lgamma", + impl_aten=torch.lgamma, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +log = _make_elementwise_unary_prim( + "log", + impl_aten=torch.log, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +log1p = _make_elementwise_unary_prim( + "log1p", + impl_aten=torch.log1p, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +log2 = _make_elementwise_unary_prim( + "log2", + impl_aten=torch.log2, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +log10 = _make_elementwise_unary_prim( + "log10", + impl_aten=torch.log10, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +real = _make_prim( + schema="real(Tensor(a) self) -> Tensor(a)", + meta=partial( + _complex_only_elementwise_meta, + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.COMPLEX_TO_FLOAT, + ), + return_type=RETURN_TYPE.VIEW, + impl_aten=torch.real, + doc="", +) + +reciprocal = _make_elementwise_unary_prim( + "reciprocal", + impl_aten=torch.reciprocal, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +ndtri = _make_elementwise_unary_prim( + "ndtri", + impl_aten=torch.special.ndtri, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +neg = _make_elementwise_unary_prim( + "neg", + impl_aten=torch.neg, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +round = _make_elementwise_unary_prim( + "round", + impl_aten=torch.round, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +rsqrt = _make_elementwise_unary_prim( + "rsqrt", + impl_aten=torch.rsqrt, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +sign = _make_elementwise_unary_prim( + "sign", + impl_aten=torch.sign, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +signbit = _make_elementwise_unary_prim( + "signbit", + impl_aten=torch.signbit, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +sin = _make_elementwise_unary_prim( + "sin", + impl_aten=torch.sin, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +sinh = _make_elementwise_unary_prim( + "sinh", + impl_aten=torch.sinh, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +spherical_bessel_j0 = _make_elementwise_unary_prim( + "spherical_bessel_j0", + impl_aten=torch.special.spherical_bessel_j0, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +sqrt = _make_elementwise_unary_prim( + "sqrt", + impl_aten=torch.sqrt, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +tan = _make_elementwise_unary_prim( + "tan", + impl_aten=torch.tan, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +tanh = _make_elementwise_unary_prim( + "tanh", + impl_aten=torch.tanh, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +trunc = _make_elementwise_unary_prim( + "trunc", + impl_aten=torch.trunc, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +# +# Elementwise binary operations +# + +add = _make_elementwise_binary_prim( + name="add", + impl_aten=torch.add, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +atan2 = _make_elementwise_binary_prim( + name="atan2", + impl_aten=torch.atan2, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +bitwise_and = _make_elementwise_binary_prim( + "bitwise_and", + impl_aten=torch.bitwise_and, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +bitwise_or = _make_elementwise_binary_prim( + "bitwise_or", + impl_aten=torch.bitwise_or, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +bitwise_xor = _make_elementwise_binary_prim( + "bitwise_xor", + impl_aten=torch.bitwise_xor, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +# TODO: complex needs a special meta to account for its float -> complex behavior +# complex = _make_elementwise_binary_prim( +# impl_aten=torch.complex, +# doc="", +# ) + + +# div prim performs truncation division on integer inputs +# and true division for floating and complex inputs +def _div_aten(a, b): + is_integral = isinstance(a, (bool, int, torch.SymInt)) or ( + isinstance(a, torch.Tensor) and utils.is_integer_dtype(a.dtype) + ) + + if is_integral: + # pyrefly: ignore [bad-argument-type] + return torch.div(a, b, rounding_mode="trunc") + else: + # pyrefly: ignore [bad-argument-type] + return torch.true_divide(a, b) + + +div = _make_elementwise_binary_prim( + "div", + impl_aten=_div_aten, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +eq = _make_elementwise_binary_prim( + "eq", + impl_aten=torch.eq, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.ALWAYS_BOOL, +) + +fmax = _make_elementwise_binary_prim( + "fmax", + impl_aten=torch.fmax, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +fmin = _make_elementwise_binary_prim( + "fmin", + impl_aten=torch.fmin, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +fmod = _make_elementwise_binary_prim( + "fmod", + impl_aten=torch.fmod, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + + +gcd = _make_elementwise_binary_prim( + "gcd", + impl_aten=torch.gcd, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + + +ge = _make_elementwise_binary_prim( + "ge", + impl_aten=torch.ge, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.ALWAYS_BOOL, +) + +gt = _make_elementwise_binary_prim( + "gt", + impl_aten=torch.gt, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.ALWAYS_BOOL, +) + +hypot = _make_elementwise_binary_prim( + "hypot", + impl_aten=torch.hypot, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +igamma = _make_elementwise_binary_prim( + "igamma", + impl_aten=torch.special.gammainc, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +igammac = _make_elementwise_binary_prim( + "igammac", + impl_aten=torch.special.gammaincc, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +le = _make_elementwise_binary_prim( + "le", + impl_aten=torch.le, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.ALWAYS_BOOL, +) + +lt = _make_elementwise_binary_prim( + "lt", + impl_aten=torch.lt, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.ALWAYS_BOOL, +) + + +# Note: the following impls are because torch.maximum and torch.minimum do not support scalar inputs +def _maximum_aten( + a: TensorLikeType | NumberType, b: TensorLikeType | NumberType +) -> TensorLikeType: + if isinstance(a, TensorLike) and isinstance(b, Number): + b = scalar_tensor(b, dtype=a.dtype, device=a.device) + elif isinstance(b, TensorLike) and isinstance(a, Number): + a = scalar_tensor(a, dtype=b.dtype, device=b.device) + + return torch.maximum(a, b) # type: ignore[arg-type] + + +maximum = _make_elementwise_binary_prim( + "maximum", + impl_aten=_maximum_aten, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + + +def _minimum_aten( + a: TensorLikeType | NumberType, b: TensorLikeType | NumberType +) -> TensorLikeType: + if isinstance(a, TensorLike) and isinstance(b, Number): + b = scalar_tensor(b, dtype=a.dtype, device=a.device) + elif isinstance(b, TensorLike) and isinstance(a, Number): + a = scalar_tensor(a, dtype=b.dtype, device=b.device) + + return torch.minimum(a, b) # type: ignore[arg-type] + + +minimum = _make_elementwise_binary_prim( + "minimum", + impl_aten=_minimum_aten, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +mul = _make_elementwise_binary_prim( + "mul", + impl_aten=torch.mul, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +ne = _make_elementwise_binary_prim( + "ne", + impl_aten=torch.ne, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.ALWAYS_BOOL, +) + +nextafter = _make_elementwise_binary_prim( + "nextafter", + impl_aten=torch.nextafter, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +pow = _make_elementwise_binary_prim( + "pow", + impl_aten=torch.pow, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +remainder = _make_elementwise_binary_prim( + "remainder", + impl_aten=torch.remainder, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + + +shift_left = _make_elementwise_binary_prim( + "shift_left", + impl_aten=torch.bitwise_left_shift, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +shift_right_arithmetic = _make_elementwise_binary_prim( + "shift_right_arithmetic", + impl_aten=torch.bitwise_right_shift, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +shift_right_logical = _not_impl + +sub = _make_elementwise_binary_prim( + "sub", + impl_aten=torch.sub, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + +zeta = _make_elementwise_binary_prim( + "zeta", + impl_aten=torch.special.zeta, + doc="", + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, +) + + +# +# View operations +def _as_strided_meta( + a: TensorLikeType, size: ShapeType, stride: StrideType, storage_offset: int +) -> TensorLikeType: + assert len(size) == len(stride) + assert storage_offset >= 0 + utils.validate_strides(stride) + utils.validate_shape(size) + + if reduce(operator.mul, size) == 0: + # NOTE: This special case is to avoid having to acquire the storage below + # as_strided to shapes with no elements are trivially valid, so it's OK + pass + elif isinstance(a, torch.Tensor): + utils.check_in_bounds_for_storage( + a._typed_storage(), size, stride, storage_offset + ) + + return torch.as_strided(a, size, stride, storage_offset) + + +def _as_strided_aten( + a: Tensor, size: ShapeType, stride: StrideType, storage_offset: int +) -> Tensor: + return torch.as_strided(a, size, stride, storage_offset) + + +_as_strided_doc = """ + Creates a view of the tensor with the given shape (size), strides (stride) and + storage offset (storage_offset). +""" + +as_strided = _make_prim( + schema="as_strided(Tensor(a!) a, SymInt[] size, SymInt[] stride, SymInt storage_offset) -> Tensor(a!)", + meta=_as_strided_meta, + impl_aten=_as_strided_aten, + return_type=RETURN_TYPE.VIEW, + doc=_as_strided_doc, +) + + +def _broadcast_in_dim_meta( + a: TensorLikeType, shape: ShapeType, broadcast_dimensions: Sequence[int] +): + from torch.fx.experimental.symbolic_shapes import ( + guard_or_false, + guard_or_true, + sym_or, + ) + + # Type checks + assert isinstance(a, TensorLike) + assert isinstance(shape, Sequence) + assert isinstance(broadcast_dimensions, Sequence) + + # every dimension must be accounted for + assert a.ndim == len(broadcast_dimensions) + + # broadcast shape must have weakly more dimensions + assert len(shape) >= a.ndim + + # broadcast_dimensions must be an ascending sequence + # (no relative reordering of dims) of integers and + # each dimension must be within the new shape + def _greater_than_reduce(acc, x): + assert isinstance(x, Dim) + assert x > acc + assert x < len(shape) + + return x + + reduce(_greater_than_reduce, broadcast_dimensions, -1) + + # shape must be broadcastable to + for idx, new_idx in enumerate(broadcast_dimensions): + torch._check( + sym_or(a.shape[idx] == 1, shape[new_idx] == a.shape[idx]), + lambda: f"{a.shape[idx]} must be broadcastable to {shape[new_idx]}", + ) + + new_strides = [] + original_idx = 0 + for idx in range(len(shape)): + if idx in broadcast_dimensions: + # Assigns a stride of zero to dimensions + # which were actually broadcast + if guard_or_false(a.shape[original_idx] == 1): + if guard_or_false(a.shape[original_idx] == shape[idx]): + new_strides.append(a.stride()[original_idx]) + else: + new_strides.append(0) + else: + torch._check( + a.shape[original_idx] == shape[idx], + lambda: f"non-broadcasting semantics require {a.shape[original_idx]} == {shape[idx]}", + ) + new_strides.append(a.stride()[original_idx]) + original_idx = original_idx + 1 + else: + if guard_or_true(shape[idx] != 1): + # consistent with previous use of guard_size_oblivious + new_strides.append(0) + elif original_idx == a.ndim: + new_strides.append(1) + else: + new_strides.append(a.stride()[original_idx] * a.size()[original_idx]) + + return a.as_strided(shape, new_strides, a.storage_offset()) + + +def _broadcast_in_dim_aten(a, shape, broadcast_dimensions): + s = list(shape) + for broadcast_dimension in broadcast_dimensions: + s[broadcast_dimension] = -1 + + v = a + for idx, x in enumerate(s): + if x != -1: + v = v.unsqueeze(idx) + + return v.expand(shape) + + +_broadcast_in_dim_doc = """ + Creates a view of a with the specified shape. + + Allows adding dimensions of any length and broadcasting + dimensions of length one in a to any length. + + The location of the broadcast dimensions must be specified + using the broadcast_dimensions argument. Changing the + relative order of dimensions is not supported. + """ + +broadcast_in_dim = _make_prim( + schema="broadcast_in_dim(Tensor(a) a, SymInt[] shape, int[] broadcast_dimensions) -> Tensor(a)", + meta=_broadcast_in_dim_meta, + impl_aten=_broadcast_in_dim_aten, + return_type=RETURN_TYPE.VIEW, + doc=_broadcast_in_dim_doc, +) + + +def _validate_collapse_args(a: Tensor, start: int, end: int) -> None: + # Special-case for zero dimensional tensors + ndim = max(1, a.dim()) + utils.validate_idx(ndim, start) + utils.validate_idx(ndim, end) + + # Verifies end is strictly greater than start + # (Collapse requires a non-empty interval) + torch._check_value( + end >= start, + lambda: f"Attempting to collapse but end, {end}, is less than start, {start}!", + ) + + +def _collapsed_shape(shape: ShapeType, start: int, end: int) -> tuple[int, ...]: + """ + Returns the shape of a with dims in [start, end) merged into a single dimension. + """ + # Special-case for zero dimensional tensors + shape = (1,) if len(shape) == 0 else tuple(shape) + + dim_length = 1 + for s in shape[start : end + 1]: + dim_length = dim_length * s + + return shape[0:start] + (dim_length,) + shape[end + 1 :] + + +# If the collapse is invalid or cannot be determined (because of unbacked data) +# then `must_be_valid` determines the behavior: +# None: return None, None. +# str: Do a torch._check() to ensure the collapse is valid and if it isn't +# then fail with the provided string. +def _collapse_view_helper( + a: TensorLikeType, start: int, end: int, must_be_valid: str | None +) -> tuple[ShapeType | None, StrideType | None]: + assert isinstance(a, TensorLike) + + from torch.fx.experimental.symbolic_shapes import ( + guard_or_false, + guard_or_true, + sym_and, + sym_or, + ) + + _validate_collapse_args(a, start, end) + + # Special-case for zero dimensional tensors + if a.ndim == 0: + shape = (1,) + strides = (1,) + else: + shape = a.shape # type: ignore[assignment] + strides = a.stride() # type: ignore[assignment] + + if a.ndim == 0 or (end == start): + return shape, strides + + valid_op = True + if guard_or_false(a.numel() != 0): + for idx in range(end - 1, start - 1, -1): + valid_op = sym_and( + valid_op, + sym_or( + shape[idx] == 1, + shape[idx + 1] == 1, + strides[idx] == strides[idx + 1] * shape[idx + 1], + ), + ) # type: ignore[assignment] + + # early exit if we already know its invalid. + if guard_or_false(valid_op is False): + break + + # for unbacked this become a runtime assertion. + valid_op = sym_or(valid_op, a.numel() == 0) + + if must_be_valid: + torch._check(valid_op, lambda: must_be_valid) + else: + if not guard_or_false(valid_op): + return None, None + + # compute stride + stride = strides[end] + for idx in range(end - 1, start - 1, -1): + if shape[idx] != 1: + # TODO with unbacked we should really exclude when shape[idx] == 1 + # something like + # min(stride[end], torch.ite(shape[x]!=1,stride[idx], inf), ...) + stride = min(stride, strides[idx]) + + # compute length + length = shape[end] + if guard_or_true(length != 0): + for idx in range(end - 1, start - 1, -1): + if guard_or_false(shape[idx] == 0): + length = 0 + stride = 0 + break + length = length * shape[idx] + else: + stride = 0 + + new_shape = shape[:start] + (length,) + shape[end + 1 :] + new_strides = strides[:start] + (stride,) + strides[end + 1 :] + + # NOTE: when the input has no elements it's restrided as if it were contiguous + # except for unbacked. + if guard_or_false(a.numel() == 0): + new_strides = utils.make_contiguous_strides_for(new_shape) + + return new_shape, new_strides + + +def _collapse_view_meta(a: TensorLikeType, start: int, end: int) -> TensorLikeType: + new_shape, new_strides = _collapse_view_helper( + a, start, end, "Attempting to view a collapsed tensor, but no such view exists!" + ) + assert new_strides is not None + assert new_shape is not None + return a.as_strided(new_shape, new_strides, a.storage_offset()) + + +def _collapse_view_aten(a: Tensor, start: int, end: int) -> Tensor: + new_shape = _collapsed_shape(a.shape, start, end) + return a.view(new_shape) + + +_collapse_view_doc = """ + Creates a view of a with the dimensions between + start (inclusive) and end (exclusive) merged into a + single dimension. + + If it's not possible to take such a view then an error + is thrown. See collapse instead. + + The dimensions can be merged if and only if + they are all "nested" with each other. That is, they all + have the property that + + stride[i] = stride[i+1] * shape[i+1] + + for all i in [start, end - 1). + """ + +collapse_view = _make_prim( + schema="collapse_view(Tensor(a) a, int start, int end) -> Tensor(a)", + meta=_collapse_view_meta, + impl_aten=_collapse_view_aten, + return_type=RETURN_TYPE.VIEW, + doc=_collapse_view_doc, +) + + +def _conj_meta(a: TensorLikeType) -> TensorLikeType: + if not a.dtype.is_complex: + raise RuntimeError("Expected complex dtype in prims.conj") + out = a.as_strided(a.shape, a.stride(), a.storage_offset()) + torch._C._set_conj(out, not a.is_conj()) + return out + + +_conj_doc = """ +Returns a conjugated view of the original tensor +""" + +conj = _make_prim( + schema="conj(Tensor(a) a) -> Tensor(a)", + meta=_conj_meta, + impl_aten=torch.conj, + return_type=RETURN_TYPE.VIEW, + doc=_conj_doc, +) + + +def expand_dims( + a: TensorLikeType, dimensions: DimsSequenceType, ndim=None +) -> TensorLikeType: + """ + Creates a view of a with a.ndim + len(dimensions) dimensions, with new + dimensions of length one at the dimensions specified by dimensions. + """ + if ndim is not None: + # TODO: this is only here to support the unsqueeze ref + dims = sorted(utils.canonicalize_dims(ndim, dimensions)) # type: ignore[arg-type] + else: + dims = sorted(utils.canonicalize_dims(a.ndim, dimensions)) # type: ignore[arg-type] + if len(set(dims)) != len(dims): + msg = f"Received duplicate dimensions to expand in {str(dimensions)}" + raise ValueError(msg) + + new_shape = list(a.shape) + for idx in dims: + new_shape.insert(idx, 1) + + broadcast_dimensions = [ + idx for idx in range(len(new_shape)) if idx not in dimensions + ] + return broadcast_in_dim(a, new_shape, broadcast_dimensions) + + +def _split_dim_meta(a: TensorLikeType, dim: int, outer_length: int) -> TensorLikeType: + assert isinstance(a, TensorLike) + utils.validate_idx(a.ndim, dim) + utils.validate_dim_length(outer_length) + + # Verifies the dim can be split with the specified lhs_length + inner_length = a.shape[dim] // outer_length + + if (a.shape[dim] % outer_length) != 0: + msg = ( + f"Attempting to split dimension of length {a.shape[dim]}, " + f"but outer length of {outer_length} divides it with a remainder!" + ) + raise ValueError(msg) + + new_shape: list[int] = [] + new_strides: list[int] = [] + for idx in range(a.ndim): + if idx == dim: + new_shape.extend((outer_length, inner_length)) + new_strides.extend((a.stride()[idx] * inner_length, a.stride()[idx])) + else: + new_shape.append(a.shape[idx]) + new_strides.append(a.stride()[idx]) + + return a.as_strided(new_shape, new_strides, a.storage_offset()) + + +def _split_dim_aten(a: Tensor, dim: int, outer_length: int) -> Tensor: + inner_length = a.shape[dim] // outer_length + new_shape = a.shape[0:dim] + (outer_length, inner_length) + a.shape[dim + 1 :] + + return a.view(new_shape) + + +_split_dim_doc = """ + Creates a view of a with the given dimension (of length l) split + into two dimensions, with the outer of the two having + length outer_length and the inner of the two having computed + length inner_length such outer_length * inner_length = l. + """ + +# TODO: consider renaming split_dim_view +split_dim = _make_prim( + schema="split_dim(Tensor(a) a, int dim, SymInt outer_length) -> Tensor(a)", + meta=_split_dim_meta, + impl_aten=_split_dim_aten, + return_type=RETURN_TYPE.VIEW, + doc=_split_dim_doc, +) + + +# Note: allows dimensions to be specified redundantly +def _squeeze_meta(a: TensorLikeType, dimensions: Sequence) -> TensorLikeType: + assert isinstance(a, TensorLike) + + for idx in dimensions: + utils.validate_idx(a.ndim, idx) + assert a.shape[idx] == 1 + + new_shape = [] + new_strides = [] + for idx in range(len(a.shape)): + if idx in dimensions: + continue + + new_shape.append(a.shape[idx]) + new_strides.append(a.stride()[idx]) + + return a.as_strided(new_shape, new_strides, a.storage_offset()) + + +_squeeze_doc = """ + Creates a view of the tensor with the specified dimensions removed. + + The removed dimensions must each have length one. + """ + +squeeze = _make_prim( + schema="squeeze(Tensor(a) a, int[] dimensions) -> Tensor(a)", + meta=_squeeze_meta, + impl_aten=torch.squeeze, + return_type=RETURN_TYPE.VIEW, + doc=_squeeze_doc, +) + + +def _transpose_meta(a: TensorLikeType, permutation: DimsSequenceType) -> TensorLikeType: + if a.ndim != len(permutation): + msg = f"Attempting to permute a tensor of rank {a.ndim}, but received a permutation of length {len(permutation)}!" + raise ValueError(msg) + + if not utils.is_valid_permutation(a.ndim, permutation): + msg = f"Received an invalid permutation, {permutation}!" + raise ValueError(msg) + + new_shape = [0] * a.ndim + new_strides = [0] * a.ndim + for idx, dim in enumerate(permutation): + new_shape[idx] = a.shape[dim] + new_strides[idx] = a.stride()[dim] + + return a.as_strided(tuple(new_shape), tuple(new_strides), a.storage_offset()) + + +def _transpose_aten(a: Tensor, permutation: DimsSequenceType) -> Tensor: + return torch.permute(a, permutation) + + +_transpose_doc = """ + Creates a view of the tensor with its dimensions permuted. + + The length of the permutation must be the rank of the tensor, + and each element of the permutation specifies the new order + for the corresponding dimension. + """ + +transpose = _make_prim( + schema="transpose(Tensor(a) a, int[] permutation) -> Tensor(a)", + meta=_transpose_meta, + impl_aten=_transpose_aten, + return_type=RETURN_TYPE.VIEW, + doc=_transpose_doc, +) + + +def _view_of_meta(a: TensorLikeType) -> TensorLikeType: + return a.as_strided(a.shape, a.stride(), a.storage_offset()) + + +def _view_of_aten(a: Tensor) -> Tensor: + return a.view(a.shape) + + +_view_of_doc = """ + Creates a view of the tensor. + """ + +view_of = _make_prim( + schema="view_of(Tensor(a) a) -> Tensor(a)", + meta=_view_of_meta, + impl_aten=_view_of_aten, + return_type=RETURN_TYPE.VIEW, + doc=_view_of_doc, +) + + +def _view_element_type_meta(a: TensorLikeType, dtype: torch.dtype) -> TensorLikeType: + return a.view(dtype) + + +def _view_element_type_aten(a: Tensor, dtype: torch.dtype) -> Tensor: + return a.view(dtype) + + +_view_element_type_doc = """ + Creates a view of the tensor with a different dtype. + """ + +view_element_type = _make_prim( + schema="view_of_dtype(Tensor(a) a, ScalarType dtype) -> Tensor(a)", + meta=_view_element_type_meta, + impl_aten=_view_element_type_aten, + return_type=RETURN_TYPE.VIEW, + doc=_view_element_type_doc, +) + +# +# Functionalized view mutations +# + + +def _as_strided_scatter_meta( + input: TensorLikeType, + src: TensorLikeType, + size: ShapeType, + stride: StrideType, + storage_offset: int, +) -> TensorLikeType: + utils.validate_shape(size) + utils.validate_strides(stride) + + required_size = utils.compute_required_storage_length(size, stride, storage_offset) + torch._check( + input.numel() >= required_size, + lambda: ( + f"as_strided_scatter: sizes {size}, strides {stride}, storage offset {storage_offset} " + f" and itemsize {input.element_size()} requiring a storage size of " + f"{required_size * input.element_size()} are out of bounds " + f"for storage of size {input.numel() * input.element_size()}" + ), + ) + torch._check( + utils.is_same_shape(src.shape, size), + lambda: f"expected src to have a size equal to the slice of self. src size = {src.shape}, slice size = {size}", + ) + + return utils.clone_preserve_strides(input) + + +_as_strided_scatter_doc = """ + Creates a new tensor equivalent to ``out = input.clone()`` after mutation by + ``out.as_strided(size, stride, storage_offset).copy_(src)``. +""" + +as_strided_scatter = _make_prim( + schema="as_strided_scatter(Tensor self, Tensor src, SymInt[] size, SymInt[] stride, SymInt storage_offset) -> Tensor", + meta=_as_strided_scatter_meta, + impl_aten=torch.as_strided_scatter, + return_type=RETURN_TYPE.NEW, + doc=_as_strided_scatter_doc, +) + + +# +# Shape operations +# + + +def _collapse_meta(a: Tensor, start: int, end: int) -> Tensor: + # Special-case for zero dimensional tensors + _validate_collapse_args(a, start, end) + new_shape = _collapsed_shape(a.shape, start, end) + return a.new_empty(new_shape) + + +def _collapse_aten(a: Tensor, start: int, end: int) -> Tensor: + new_shape = _collapsed_shape(a.shape, start, end) + out = a.new_empty(new_shape) + with torch.no_grad(): + out.view_as(a).copy_(a) + return out + + +_collapse_doc = """ +Collapse a span of neighboring dimensions into one. + +See collapse_view for the corresponding view operation. +""" +collapse = _make_prim( + schema="collapse(Tensor a, int start, int end) -> Tensor", + meta=_collapse_meta, + impl_aten=_collapse_aten, + return_type=RETURN_TYPE.NEW, + doc=_collapse_doc, +) + + +# TODO: review stride logic +# NB: unlike torch.cat, this is more strict about empty tensors and dim is +# never negative +def _cat_meta(tensors: Sequence[TensorLikeType], dim: int) -> TensorLikeType: + # Verifies same shape (except in the concat dimension) + assert dim >= 0 + shape = tensors[0].shape + sym_sum_args = [] + for tensor_idx, tensor in enumerate(tensors): + assert len(shape) == len(tensor.shape) + for idx, (common_length, length) in enumerate(zip(shape, tensor.shape)): + if idx == dim: + sym_sum_args.append(length) + else: + torch._check( + length == common_length, + lambda: f"Sizes of tensors must match except in dimension {dim}. " + f"Expected {common_length} in dimension {idx} but got {length} for tensor number " + f"{tensor_idx} in the list", + ) + + new_shape = list(tensors[0].shape).copy() + new_shape[dim] = torch.sym_sum(sym_sum_args) + return TensorMeta( + tensors[0], + shape=new_shape, + strides=utils.make_contiguous_strides_for(new_shape), + ) + + +def _cat_aten(tensors: tuple[Tensor, ...] | list[Tensor], dim: int) -> Tensor: + return torch.cat(tensors, dim) + + +_cat_doc = """ + Concatenates tensors along the specified dimension. + + The tensors' shapes must have the same rank and same length for other dimensions. + """ + +cat = _make_prim( + schema="cat(Tensor[] tensors, int dim) -> Tensor", + meta=_cat_meta, + impl_aten=_cat_aten, + return_type=RETURN_TYPE.NEW, + doc=_cat_doc, +) + + +def _reshape_meta(a: TensorLikeType, shape: ShapeType): + assert isinstance(a, TensorLike) + utils.validate_shape(shape) + + # Validates the tensor and the requested shape have the + # same number of elements + numel = reduce(operator.mul, shape) + if numel != a.numel(): + msg = f"Attempting to reshape a tensor with {a.numel()} elements to a shape with {numel} elements!" + raise ValueError(msg) + + return TensorMeta(a, shape=shape, strides=utils.make_contiguous_strides_for(shape)) + + +def _reshape_aten(a: Tensor, shape: ShapeType) -> Tensor: + return a.reshape(shape).clone(memory_format=torch.contiguous_format) + + +_reshape_doc = """ + Creates a contiguous tensor with the specified shape + containing a copy of the data in a. + """ +reshape = _make_prim( + schema="reshape(Tensor a, SymInt[] shape) -> Tensor", + meta=_reshape_meta, + impl_aten=_reshape_aten, + return_type=RETURN_TYPE.NEW, + doc=_reshape_doc, +) + + +def _rev_meta(a: TensorLikeType, dims: DimsSequenceType) -> TensorLikeType: + utils.validate_dimension_indices(a.ndim, dims) + return torch.empty_like(a, memory_format=torch.preserve_format) + + +_rev_doc = """ + Reverses the order of elements along the given dimensions. + """ + +rev = _make_prim( + schema="rev(Tensor a, int[] dims) -> Tensor", + meta=_rev_meta, + impl_aten=torch.flip, + return_type=RETURN_TYPE.NEW, + doc=_rev_doc, +) + +# +# Conditional prims +# + + +def _where_meta( + pred: TensorLikeType, a: TensorLikeType, b: TensorLikeType +) -> TensorLikeType: + return _prim_elementwise_meta( + a, + b, + type_promotion=ELEMENTWISE_PRIM_TYPE_PROMOTION_KIND.DEFAULT, + args_with_fixed_dtypes=(pred,), + ) + + +_where_doc = """ + Selects elements from a and b according to pred. + + Where pred is true the result contains the element from a, and + where pred is false the result contains the element from b. + """ + +where = _make_prim( + schema="where(Tensor pred, Tensor a, Tensor b) -> Tensor", + meta=_where_meta, + impl_aten=torch.where, + return_type=RETURN_TYPE.NEW, + doc=_where_doc, +) + + +# +# Type conversions +# +def _convert_element_type_meta(a: TensorLikeType, dtype: torch.dtype) -> TensorLikeType: + # Type checks + assert isinstance(a, TensorLike) + assert isinstance(dtype, torch.dtype) + + # dtype conversion preserves dense strides + if torch._prims_common.is_non_overlapping_and_dense(a): + strides = a.stride() + else: + strides = utils.compute_elementwise_output_strides(a) + + return TensorMeta(a, strides=strides, dtype=dtype) + + +def _convert_element_type_aten(a: Tensor, dtype: torch.dtype) -> Tensor: + # Propagates requires grad when possible + if not utils.is_grad_dtype(dtype): + requires_grad = False + else: + # TODO: update meta objects so this can be acquired directly + try: + requires_grad = a.requires_grad + except Exception: + requires_grad = False + + result = torch.empty_like( + a, device=a.device, dtype=dtype, requires_grad=requires_grad + ) + with torch.no_grad(): + return copy_to(result, a) + + +_convert_element_type_doc = """ + Creates a copy of a tensor with the given dtype. + """ + +convert_element_type = _make_prim( + schema="convert_element_type(Tensor a, ScalarType dtype) -> Tensor", + meta=_convert_element_type_meta, + impl_aten=_convert_element_type_aten, + return_type=RETURN_TYPE.NEW, + doc=_convert_element_type_doc, + tags=(torch.Tag.pointwise,), +) + + +def _device_put_meta( + a: TensorLikeType, device: str | torch.device, non_blocking=False +) -> TensorLikeType: + assert isinstance(a, TensorLike) + assert isinstance(device, (str, torch.device)) + assert isinstance(non_blocking, bool) + + return TensorMeta(a, device=utils.canonicalize_device(device)) + + +def _device_put_aten( + a: Tensor, device: str | torch.device, non_blocking=False +) -> Tensor: + return a.to(device, non_blocking=non_blocking) + + +_device_put_doc = """ + Creates a copy of a tensor on the given device. + """ + +device_put = _make_prim( + schema="device_put(Tensor a, Device device, bool non_blocking=False) -> Tensor", + meta=_device_put_meta, + impl_aten=_device_put_aten, + return_type=RETURN_TYPE.NEW, + doc=_device_put_doc, +) + + +# NOTE: need to model meta scalars +# See https://github.com/pytorch/pytorch/issues/78070 +def _item_meta(a: TensorLikeType) -> FakeTensor: + number_type = utils.dtype_to_type(a.dtype) + return TensorMeta(number_type(-1)) + + +_item_doc = """ + Converts a tensor with one element to a Python number. +""" + + +# We can't call into python dispatcher for item again +# because the current prim decomp calls into python dispatcher +# again. https://github.com/pytorch/pytorch/issues/136050 +def _item_aten_no_python_dispatcher(*args, **kwargs): + with torch._dispatch.python.no_python_dispatcher(): + return torch.Tensor.item(*args, **kwargs) + + +# TODO: create a new return type for scalars? +# FIXME: currently returns integers for boolean tensors +# https://github.com/pytorch/pytorch/issues/78071 +item = _make_prim( + schema="item(Tensor a) -> Scalar", + meta=_item_meta, + impl_aten=_item_aten_no_python_dispatcher, + return_type=RETURN_TYPE.NEW, + doc=_item_doc, +) + + +# NOTE: need to model meta scalars +# See https://github.com/pytorch/pytorch/issues/78070 +def _maximum_value_meta(dtype: torch.dtype) -> FakeTensor: + number_type = utils.dtype_to_type(dtype) + return TensorMeta(number_type(-1)) + + +def _maximum_value_aten(dtype: torch.dtype): + if dtype == torch.bool: + return True + elif dtype.is_complex or dtype.is_floating_point: + return torch.finfo(dtype).max + else: + return torch.iinfo(dtype).max + + +_maximum_value_doc = """ + Return the maximum finite value for a dtype. +""" + +# TODO: create a new return type for scalars? +# FIXME: currently returns integers for boolean tensors +# https://github.com/pytorch/pytorch/issues/78071 +maximum_value = _make_prim( + schema="maximum_value(ScalarType dtype) -> Scalar", + meta=_maximum_value_meta, + impl_aten=_maximum_value_aten, + return_type=RETURN_TYPE.NEW, + doc=_maximum_value_doc, +) + + +# NOTE: need to model meta scalars +# See https://github.com/pytorch/pytorch/issues/78070 +def _minimum_value_meta(dtype: torch.dtype) -> FakeTensor: + number_type = utils.dtype_to_type(dtype) + return TensorMeta(number_type(-1)) + + +def _minimum_value_aten(dtype: torch.dtype): + if dtype == torch.bool: + return False + elif dtype.is_complex or dtype.is_floating_point: + return torch.finfo(dtype).min + else: + return torch.iinfo(dtype).min + + +_minimum_value_doc = """ + Return the minimum finite value for a dtype. +""" + +# TODO: create a new return type for scalars? +# FIXME: currently returns integers for boolean tensors +# https://github.com/pytorch/pytorch/issues/78071 +minimum_value = _make_prim( + schema="minimum_value(ScalarType dtype) -> Scalar", + meta=_minimum_value_meta, + impl_aten=_minimum_value_aten, + return_type=RETURN_TYPE.NEW, + doc=_minimum_value_doc, +) + +# +# Inplace operators +# + + +def _copy_to_meta(a: TensorLikeType, b: TensorLikeType): + assert isinstance(a, TensorLike) + assert isinstance(b, TensorLike) + + # Validates the cast is safe + # TODO: move this as an option on the reference + # a_typ = utils.dtype_to_type(a.dtype) + # b_typ = utils.dtype_to_type(b.dtype) + # if a_typ is not utils.get_higher_type(a_typ, b_typ): + # raise RuntimeError(str(b.dtype), " can't be cast safely to ", str(a.dtype), "!") + + # Validates the tensors have the same number of elements + if a.numel() != b.numel(): + msg = f"Attempting to copy {b.numel()} elements to a tensor with {a.numel()} elements!" + raise RuntimeError(msg) + + return a + + +def _copy_to_aten(a: Tensor, b: Tensor) -> Tensor: + return a.copy_(b) + + +_copy_to_doc = """ + Copies the data in b to a and returns the modified a. + """ + +# TODO: Remove safe casting and implement on reference instead +copy_to = _make_prim( + schema="copy_to(Tensor(a!) a, Tensor b) -> Tensor(a!)", + meta=_copy_to_meta, + impl_aten=_copy_to_aten, + return_type=RETURN_TYPE.INPLACE, + doc=_copy_to_doc, + register_conj_neg_fallthrough=True, +) + + +def _copy_strided_meta(a: TensorLikeType, stride: ShapeType): + assert isinstance(a, TensorLike) + return torch.empty_strided( + a.shape, + stride, + dtype=a.dtype, + layout=a.layout, + device=a.device, + requires_grad=a.requires_grad, + ) + + +def _copy_strided_aten(a: Tensor, stride: ShapeType) -> Tensor: + out = torch.empty_strided( + a.size(), + stride=stride, + dtype=a.dtype, + layout=a.layout, + device=a.device, + requires_grad=a.requires_grad, + ) + out.copy_(a) + return out + + +_copy_strided_doc = """ + Copies the data in a to a new tensor, the new tensor has same shape with a size, but has different stride. + """ + + +copy_strided = _make_prim( + schema="copy_strided(Tensor a, SymInt[] stride) -> Tensor", + meta=_copy_strided_meta, + impl_aten=_copy_strided_aten, + return_type=RETURN_TYPE.NEW, + doc=_copy_strided_doc, +) + + +def _resize_meta(a: TensorLikeType, shape: ShapeType): + return a.resize_(shape) + + +def _resize_aten(a: Tensor, shape: ShapeType) -> Tensor: + return a.resize_(shape) + + +_resize_doc = """ + Gives a tensor with no elements a new shape, returning the modified tensor. + + The tensor's strides are contiguous and its values are uninitialized. + """ + +# TODO: review support arbitrary resizes +resize = _make_prim( + schema="resize(Tensor(a!) a, SymInt[] shape) -> Tensor(a!)", + meta=_resize_meta, + impl_aten=_resize_aten, + return_type=RETURN_TYPE.INPLACE, + doc=_resize_doc, +) + + +def _reduction_meta(inp, dims, *, output_dtype=None): + """ + Meta function for single output reduction operations + Stride logic is incorrect + """ + assert isinstance(inp, TensorLike) + if output_dtype is None: + output_dtype = inp.dtype + output_shape = utils.compute_reduction_output_shape(inp.shape, dims) + return TensorMeta( + shape=output_shape, + strides=utils.make_contiguous_strides_for(output_shape), + dtype=output_dtype, + device=inp.device, + ) + + +def _var_reduction_meta(inp, dims, correction): + if utils.is_complex_dtype(inp.dtype): + output_dtype = utils.corresponding_real_dtype(inp.dtype) + else: + output_dtype = inp.dtype + return _reduction_meta(inp, dims, output_dtype=output_dtype) + + +_sum_doc = """ + Computes the sum of elements in the input tensor over the list of dimensions + specified in the dim argument + """ +_xor_sum_doc = """ + Computes the xor sum of elements in the input tensor over the list of dimensions + specified in the dim argument + """ +_prod_doc = """ + Computes the product of elements in the input tensor over the list of dimensions + specified in the dim argument + """ +_amax_doc = """ + Computes the maximum value of elements in the input tensor over the list of dimensions + specified in the dim argument + """ +_amin_doc = """ + Computes the minimum value of elements in the input tensor over the list of dimensions + specified in the dim argument + """ +_var_doc = """ + Computes the biased variance of x over the list of dimensions specified in the dim argument + """ + + +def _make_reduction_prim(name: str, impl_aten, doc): + """Creates a reduction prim.""" + return _make_prim( + schema=f"{name}(Tensor inp, int[]? dims, *, ScalarType? output_dtype=None) -> Tensor", + meta=_reduction_meta, + impl_aten=impl_aten, + return_type=RETURN_TYPE.NEW, + doc=doc, + ) + + +def _make_var_reduction_prim(name: str, impl_aten, doc): + """Creates a reduction prim.""" + return _make_prim( + schema=f"{name}(Tensor inp, int[]? dims, float? correction=1, *, ScalarType? output_dtype=None) -> Tensor", + meta=_var_reduction_meta, + impl_aten=impl_aten, + return_type=RETURN_TYPE.NEW, + doc=doc, + ) + + +sum = _make_reduction_prim( + name="sum", + impl_aten=torch.sum, + doc=_sum_doc, +) + + +def _xor_sum_aten( + inp: TensorLikeType, + dims: DimsSequenceType | None, + *, + dtype: torch.dtype | None = None, +) -> Tensor: + raise NotImplementedError("xor_sum only implemented with inductor") + + +xor_sum = _make_reduction_prim( + name="xor_sum", + impl_aten=_xor_sum_aten, + doc=_xor_sum_doc, +) + + +def _prod_aten( + inp: TensorLikeType, + dims: DimsSequenceType | None, + *, + dtype: torch.dtype | None = None, +) -> Tensor: + if dims is not None: + if len(dims) == 0: + return inp.clone() + for d in sorted(dims, reverse=True): + assert d >= 0 + inp = torch.prod(inp, d, dtype=dtype) + return inp + else: + return torch.prod(inp, dims, dtype=dtype) + + +prod = _make_reduction_prim( + name="prod", + impl_aten=_prod_aten, + doc=_prod_doc, +) + + +# torch.var, but correction is not kwarg-only +def torch_var(input, dim=None, correction=1, **kwargs): + return torch.var(input, dim=dim, correction=correction, **kwargs) + + +var = _make_var_reduction_prim( + name="var", + impl_aten=torch_var, + doc=_var_doc, +) + +amax = _make_reduction_prim( + name="amax", + impl_aten=torch.amax, + doc=_amax_doc, +) + +amin = _make_reduction_prim( + name="amin", + impl_aten=torch.amin, + doc=_amin_doc, +) + + +_iota_doc = """ + Constructs a 1-D tensor t where ``t[i] == start + i * step``. +""" + + +# TODO: layout, pin_memory, memory_format +# TODO: model requires_grad on TensorMeta +def _iota_meta( + length: int, + *, + start: int, + step: int, + dtype: torch.dtype, + device: torch.device, + requires_grad: bool, +) -> TensorLikeType: + torch._check( + utils.is_integer_dtype(dtype), + lambda: "prims.iota only supports integer dtypes", + ) + torch._check(step != 0, lambda: "step must be nonzero") + return torch.empty( + length, + dtype=dtype, + device=device, + requires_grad=requires_grad, + ) + + +def _iota_aten( + length: int, + *, + start: int, + step: int, + dtype: torch.dtype, + device: torch.device, + requires_grad: bool, +) -> TensorLikeType: + end = start + length * step + return torch.arange( + start, end, step, dtype=dtype, device=device, requires_grad=requires_grad + ) + + +iota = _make_prim( + schema="iota(SymInt length, *, SymInt start, SymInt step, ScalarType dtype, Device device, bool requires_grad) -> Tensor", # noqa: B950 + return_type=RETURN_TYPE.NEW, + meta=_iota_meta, + impl_aten=_iota_aten, + doc=_iota_doc, +) + + +# TODO: layout, pin_memory, memory_format +# TODO: model requires_grad on TensorMeta +def _empty_meta( + shape: ShapeType, *, dtype: torch.dtype, device: torch.device, requires_grad: bool +) -> TensorLikeType: + strides = utils.make_contiguous_strides_for(shape) + return TensorMeta(shape=shape, strides=strides, dtype=dtype, device=device) + + +def _empty_aten( + shape: ShapeType, *, dtype: torch.dtype, device: torch.device, requires_grad: bool +) -> Tensor: + return torch.empty(shape, dtype=dtype, device=device, requires_grad=requires_grad) + + +_empty_doc = """ + Creates a tensor with uninitialized values and the specified shape, dtype, and device. +""" + +empty = _make_prim( + schema="empty(SymInt[] shape, *, ScalarType dtype, Device device, bool requires_grad) -> Tensor", + meta=_empty_meta, + impl_aten=_empty_aten, + return_type=RETURN_TYPE.NEW, + doc=_empty_doc, +) + + +def _empty_strided_meta( + shape: ShapeType, + strides: StrideType, + *, + dtype: torch.dtype, + device: torch.device, + requires_grad: bool, +) -> TensorLikeType: + return TensorMeta(shape=shape, strides=strides, dtype=dtype, device=device) + + +_empty_strided_doc = """ + Creates a tensor with uninitialized values. +""" + +# TODO: add layout, pin_memory +empty_strided = _make_prim( + schema="empty_strided(SymInt[] shape, SymInt[] strides, *, ScalarType dtype, Device device, bool requires_grad) -> Tensor", + return_type=RETURN_TYPE.NEW, + meta=_empty_strided_meta, + impl_aten=torch.empty_strided, + doc=_empty_strided_doc, +) + + +def _empty_permuted_meta( + shape: ShapeType, + physical_layout: DimsSequenceType, + *, + dtype: torch.dtype, + device: torch.device, + requires_grad: bool, +) -> TensorLikeType: + p_strides = utils.make_contiguous_strides_for([shape[l] for l in physical_layout]) + dim = len(shape) + torch._check( + len(physical_layout) == dim, + lambda: ( + "Number of dimensions in the tensor input does not match the " + f"length of the physical layout; i.e. len(size) = {dim} " + f"is not equal to len(physical_layout) = {len(physical_layout)}" + ), + ) + strides = [0] * len(shape) + seen_dims = set() + for p, l in enumerate(physical_layout): + torch._check( + 0 <= l < dim, + lambda: ( + f"Dimension out of range (expected to be between 0 and {dim - 1}, but got " + f"{l} at index {p}). NB: negative dims " + "not currently supported; file an issue if you want it." + ), + ) + torch._check(l not in seen_dims, lambda: "Duplicate dim not allowed") + strides[l] = p_strides[p] + seen_dims.add(l) + return TensorMeta( + shape=shape, + strides=strides, + dtype=dtype, + device=device, + ) + + +_empty_permuted_doc = """ + Creates a tensor with uninitialized values according to some physical layout, + that is guaranteed to be non-overlapping and dense. +""" + +# TODO: add layout, pin_memory +empty_permuted = _make_prim( + schema="empty_permuted(SymInt[] shape, int[] physical_layout, *, ScalarType dtype, Device device, bool requires_grad) -> Tensor", # noqa: B950 + return_type=RETURN_TYPE.NEW, + meta=_empty_permuted_meta, + impl_aten=torch.empty_permuted, + doc=_empty_permuted_doc, +) + + +def _full_meta( + shape: ShapeType, + fill_value: NumberType, + *, + dtype: torch.dtype, + device: torch.device, + requires_grad: bool, +) -> TensorLikeType: + strides = utils.make_contiguous_strides_for(shape) + return TensorMeta(shape=shape, strides=strides, dtype=dtype, device=device) + + +def _full_aten( + shape: ShapeType, + fill_value: NumberType, + *, + dtype: torch.dtype, + device: torch.device, + requires_grad: bool, +) -> Tensor: + # Note that Mypy thinks torch.full can't accept a complex fill_value + return torch.full( + shape, + fill_value, + dtype=dtype, + device=device, + requires_grad=requires_grad, # type: ignore[arg-type] + ) + + +_full_doc = """ + Creates a tensor filled with the given fill value, and with the specified shape, dtype, and device. +""" + +# TODO: add layout +full = _make_prim( + schema="full(SymInt[] shape, Scalar fill_value, *, ScalarType dtype, Device device, bool requires_grad) -> Tensor", + meta=_full_meta, + impl_aten=_full_aten, + return_type=RETURN_TYPE.NEW, + doc=_full_doc, +) + + +def _full_like_meta( + a: TensorLikeType, + fill_value: NumberType, + *, + dtype: torch.dtype, + device: torch.device, + requires_grad: bool, +) -> TensorLikeType: + strides = utils.compute_elementwise_output_strides(a) + if a.numel() == 0: + strides = a.stride() + + return TensorMeta(a, strides=strides, dtype=dtype, device=device) + + +def _full_like_aten( + a: Tensor, + fill_value: NumberType, + *, + dtype: torch.dtype, + device: torch.device, + requires_grad: bool, +) -> Tensor: + # Note that Mypy thinks torch.full can't accept a complex fill_value + return torch.full_like( + a, + fill_value, + dtype=dtype, + device=device, + requires_grad=requires_grad, # type: ignore[arg-type] + ) + + +_full_like_doc = """ + Creates a tensor filled with the given fill value, and the same shape, dtype, and device as the + given tensor by default. The dtype and device settings can be overridden + by specifying them explicitly. +""" + +full_like = _make_prim( + schema="full_like(Tensor a, Scalar fill_value, *, ScalarType dtype, Device device, bool requires_grad) -> Tensor", + meta=_full_like_meta, + impl_aten=_full_like_aten, + return_type=RETURN_TYPE.NEW, + doc=_full_like_doc, +) + + +def _scalar_tensor_meta( + scalar: NumberType, + *, + dtype: torch.dtype, + device: torch.device, +) -> TensorLikeType: + shape: ShapeType = [] + strides = utils.make_contiguous_strides_for(shape) + return TensorMeta(scalar, shape=shape, strides=strides, dtype=dtype, device=device) + + +def _scalar_tensor_aten( + scalar: NumberType, + *, + dtype: torch.dtype, + device: torch.device, +) -> Tensor: + if isinstance(scalar, complex) and ( + dtype is None or not utils.is_complex_dtype(dtype) + ): + raise TypeError("Complex scalar requires complex tensor dtype.") + # Note that Mypy thinks torch.scalar can't accept a complex scalar + return torch.scalar_tensor(scalar, dtype=dtype, device=device) # type: ignore[arg-type] + + +_scalar_tensor_doc = """ + Wraps a Number into a Tensor with the specified dtype and device. +""" + +# TODO: add layout and pin_memory support +scalar_tensor = _make_prim( + schema="scalar_tensor(Scalar s, *, ScalarType? dtype=None, Device? device=None) -> Tensor", + meta=_scalar_tensor_meta, + impl_aten=_scalar_tensor_aten, + return_type=RETURN_TYPE.NEW, + doc=_scalar_tensor_doc, +) + + +# +# Linear algebra (linalg) prims +# + + +def _svd_meta( + A: TensorLikeType, *, full_matrices: bool +) -> tuple[TensorLikeType, TensorLikeType, TensorLikeType]: + utils.check_is_matrix(A, "linalg.svd") + utils.check_fp_or_complex(A.dtype, "linalg.svd", allow_low_precision_dtypes=False) + + A_shape = A.shape + batch = A_shape[:-2] + m, n = A_shape[-2:] + k = min(m, n) + + shape_U = batch + (m, m if full_matrices else k) + strides_U = utils.make_contiguous_strides_for(shape_U, row_major=False) + U = TensorMeta(shape=shape_U, strides=strides_U, dtype=A.dtype, device=A.device) + + shape_S = batch + (k,) + strides_S = utils.make_contiguous_strides_for(shape_S) + S = TensorMeta( + shape=shape_S, + strides=strides_S, + dtype=utils.corresponding_real_dtype(A.dtype) if A.is_complex() else A.dtype, + device=A.device, + ) + + shape_Vh = batch + (n if full_matrices else k, n) + # The CPU backend returns V, but the cuSolver backend returns V^H + # TODO The MAGMA backend returns V, so this is wrong if used with the MAGMA backend + is_cuda = A.device.type == "cuda" + strides_Vh = utils.make_contiguous_strides_for(shape_Vh, row_major=is_cuda) + Vh = TensorMeta(shape=shape_Vh, strides=strides_Vh, dtype=A.dtype, device=A.device) + # Also makes sure this is CUDA or HIP: + # https://pytorch.org/docs/stable/notes/hip.html#checking-for-hip + if A.numel() != 0 and Vh.is_complex() and torch.cuda.is_available(): + Vh = Vh.conj() + return U, S, Vh + + +def _svd_aten( + A: TensorLikeType, *, full_matrices: bool +) -> tuple[Tensor, Tensor, Tensor]: + return torch.linalg.svd(A, full_matrices=full_matrices) + + +_svd_doc = """ + Returns the SVD of a matrix or batch of matrices. + + The `full_matrices` flag controls whether the full or reduced SVD decomposition is returned. +""" + +svd = _make_prim( + schema="svd(Tensor A, *, bool full_matrices) -> (Tensor U, Tensor S, Tensor Vh)", + meta=_svd_meta, + impl_aten=_svd_aten, + return_type=(RETURN_TYPE.NEW, RETURN_TYPE.NEW, RETURN_TYPE.NEW), + doc=_svd_doc, +) + + +# +# Randomness Prims +# + + +def _normal_meta( + shape: ShapeType, + *, + mean: float | complex, + std: float, + dtype: torch.dtype, + device: torch.device, + requires_grad: bool, + generator: torch.Generator | None = None, +) -> TensorLikeType: + torch._check( + std >= 0.0, + lambda: f"expected non-negative standard deviation, but got std={std}", + ) + + torch._check( + utils.is_float_dtype(dtype) or utils.is_complex_dtype(dtype), + lambda: f"expected a floating-point or complex dtype, but got dtype={dtype}", + ) + + strides = utils.make_contiguous_strides_for(shape) + return TensorMeta(shape=shape, strides=strides, dtype=dtype, device=device) + + +def _normal_aten( + shape: ShapeType, + *, + mean: float | complex, + std: float, + dtype: torch.dtype, + device: torch.device, + requires_grad: bool, + generator: torch.Generator | None = None, +) -> Tensor: + a = torch.empty(shape, dtype=dtype, device=device, requires_grad=requires_grad) + with torch.no_grad(): + # NOTE: normal_ is incorrectly annotated to expect mean to be a float + a.normal_(mean, std, generator=generator) # type: ignore[arg-type] + return a + + +_normal_doc = """ + Constructs a tensor filled with values drawn from a normal distribution with the specified mean + and standard deviation. + + Only supports floating-point types. +""" + +normal = _make_prim( + schema=( + "normal(SymInt[] shape, *, Scalar mean, Scalar std, ScalarType dtype, Device device, bool requires_grad, Generator? generator=None) -> Tensor" # noqa: B950 + ), + return_type=RETURN_TYPE.NEW, + meta=_normal_meta, + impl_aten=_normal_aten, + doc=_normal_doc, +) + + +def _uniform_meta( + shape: ShapeType, + *, + low: float, + high: float, + dtype: torch.dtype, + device: torch.device, + generator: torch.Generator | None = None, +) -> TensorLikeType: + strides = utils.make_contiguous_strides_for(shape) + return TensorMeta(shape=shape, strides=strides, dtype=dtype, device=device) + + +def _uniform_aten( + shape: ShapeType, + *, + low: float, + high: float, + dtype: torch.dtype, + device: torch.device, + generator: torch.Generator | None = None, +) -> Tensor: + a = torch.empty(shape, dtype=dtype, device=device) + a.uniform_(low, high, generator=generator) + return a + + +_uniform_doc = """ + Constructs a tensor filled with values drawn uniformly from low to high. +""" + +# TODO: we should more seriously review randomness modeling and prims +_uniform_helper = _make_prim( + schema=( + "uniform(SymInt[] shape, *, Scalar low, Scalar high, ScalarType dtype, Device device, Generator? generator=None) -> Tensor" + ), + return_type=RETURN_TYPE.NEW, + meta=_uniform_meta, + impl_aten=_uniform_aten, + doc=_uniform_doc, +) + +# +# FFT prims +# + + +def _fft_r2c_meta( + input: TensorLike, + *, + dim: DimsSequenceType, + onesided: bool, +) -> TensorLikeType: + dim = utils.canonicalize_dims(input.ndim, dim) + utils.validate_no_repeating_dims(dim) + + shape = list(input.shape) + if onesided: + last_dim = dim[-1] + shape[last_dim] = shape[last_dim] // 2 + 1 + + dtype = utils.corresponding_complex_dtype(input.dtype) + strides = utils.make_contiguous_strides_for(shape) + return TensorMeta(shape=shape, strides=strides, dtype=dtype, device=input.device) + + +def _fft_r2c_aten( + input: TensorLike, + *, + dim: DimsSequenceType, + onesided: bool, +) -> TensorLikeType: + normalization = 0 # No normalization + return torch._fft_r2c(input, dim, normalization, onesided) + + +_fft_r2c_doc = """ + Performs a real to complex Fast Fourier Transform +""" + + +fft_r2c = _make_prim( + schema="fft_r2c(Tensor self, *, int[] dim, bool onesided) -> Tensor", + meta=_fft_r2c_meta, + impl_aten=_fft_r2c_aten, + return_type=RETURN_TYPE.NEW, + doc=_fft_r2c_doc, +) + + +def _fft_c2c_meta( + input: TensorLike, + *, + dim: DimsSequenceType, + forward: bool, +) -> TensorLikeType: + dim = utils.canonicalize_dims(input.ndim, dim) + utils.validate_no_repeating_dims(dim) + + shape = input.shape + strides = utils.make_contiguous_strides_for(shape) + return TensorMeta( + shape=shape, strides=strides, dtype=input.dtype, device=input.device + ) + + +def _fft_c2c_aten( + input: TensorLike, + *, + dim: DimsSequenceType, + forward: bool, +) -> TensorLikeType: + normalization = 0 # No normalization + return torch._fft_c2c(input, dim, normalization, forward) + + +_fft_c2c_doc = """ + Performs either a Fast Fourier Transform, or its inverse +""" + + +fft_c2c = _make_prim( + schema="fft_c2c(Tensor self, *, int[] dim, bool forward) -> Tensor", + meta=_fft_c2c_meta, + impl_aten=_fft_c2c_aten, + return_type=RETURN_TYPE.NEW, + doc=_fft_c2c_doc, +) + + +def _fft_c2r_meta( + input: TensorLike, + *, + dim: DimsSequenceType, + last_dim_size: int, +) -> TensorLikeType: + dim = utils.canonicalize_dims(input.ndim, dim) + utils.validate_no_repeating_dims(dim) + + shape = list(input.shape) + shape[dim[-1]] = last_dim_size + dtype = utils.corresponding_real_dtype(input.dtype) + strides = utils.make_contiguous_strides_for(shape) + return TensorMeta(shape=shape, strides=strides, dtype=dtype, device=input.device) + + +def _fft_c2r_aten( + input: TensorLike, + *, + dim: DimsSequenceType, + last_dim_size: int, +) -> TensorLikeType: + normalization = 0 # No normalization + return torch._fft_c2r(input, dim, normalization, last_dim_size) + + +_fft_c2r_doc = """ + Performs a complex to real Inverse Fast Fourier Transform +""" + + +fft_c2r = _make_prim( + schema="fft_c2r(Tensor self, *, int[] dim, SymInt last_dim_size) -> Tensor", + meta=_fft_c2r_meta, + impl_aten=_fft_c2r_aten, + return_type=RETURN_TYPE.NEW, + doc=_fft_c2r_doc, +) + + +def _frexp_meta(self: TensorLikeType) -> tuple[TensorLikeType, TensorLikeType]: + torch._check( + self.dtype.is_floating_point, + lambda: "torch.frexp() only supports floating-point dtypes", + ) + return torch.empty_like(self), torch.empty_like(self, dtype=torch.int32) + + +frexp = _make_prim( + schema="frexp(Tensor self) -> (Tensor mantissa, Tensor exponent)", + meta=_frexp_meta, + return_type=(RETURN_TYPE.NEW, RETURN_TYPE.NEW), + impl_aten=torch.frexp, + doc="", +) + + +def _make_token_aten() -> TensorLikeType: + return new_token_tensor() + + +_make_token = _make_prim( + schema="_make_token() -> Tensor", + meta=_make_token_aten, + return_type=RETURN_TYPE.NEW, + impl_aten=_make_token_aten, + doc="Creates a token used for keeping track of side effects.", +) + + +def _sink_tokens_aten(tokens) -> None: + pass + + +_sink_tokens = _make_prim( + schema="_sink_tokens(Tensor[] tokens) -> ()", + meta=_sink_tokens_aten, + return_type=RETURN_TYPE.NONE, + impl_aten=_sink_tokens_aten, + doc="Sink all of the tokens which were previously used for keeping track of side effects.", +) + +torch.fx.node.has_side_effect(_sink_tokens) + + +register_rng_prims() +register_debug_prims() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/context.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/context.py new file mode 100644 index 0000000000000000000000000000000000000000..e41e3e904d85740fb6068977bd290505c0bf2d79 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/context.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import functools +from contextlib import nullcontext +from typing import Any, TYPE_CHECKING, TypeVar +from typing_extensions import ParamSpec + + +if TYPE_CHECKING: + from collections.abc import Callable, Sequence + +import torch +import torch._decomp +import torch._prims +import torch._refs +import torch._refs.nn +import torch._refs.nn.functional +import torch._refs.special +import torch.overrides +from torch._prims_common import torch_function_passthrough + + +_P = ParamSpec("_P") +_R = TypeVar("_R") + + +@functools.cache +def torch_to_refs_map() -> dict[Any, Any]: + """ + Mapping of torch API functions to torch._refs functions. + E.g. torch_to_refs_map()[torch.add] == torch._refs.add + """ + modules = [ + (torch, torch._refs), + (torch.nn, torch._refs.nn), + (torch.nn.functional, torch._refs.nn.functional), + (torch.special, torch._refs.special), + (torch.fft, torch._refs.fft), + (torch.linalg, torch._refs.linalg), + ] + r: dict[Any, Any] = { + torch.Tensor.__invert__: torch._refs.bitwise_not, + torch.Tensor.__xor__: torch._refs.bitwise_xor, + torch.Tensor.__and__: torch._refs.bitwise_and, + torch.Tensor.__or__: torch._refs.bitwise_or, + torch.Tensor.__eq__: torch._refs.eq, + torch.Tensor.__rsub__: torch._refs.rsub, + torch.Tensor.__rtruediv__: torch._refs.rtruediv, + torch.Tensor.__floordiv__: torch._refs.floor_divide, + torch.Tensor.__rfloordiv__: torch._refs.rfloordiv, + torch.Tensor.__pow__: torch._refs.pow, + torch.Tensor.__rpow__: torch._refs.rpow, + torch.Tensor.new_empty: torch._refs.new_empty, + torch.Tensor.new_full: torch._refs.new_full, + torch.Tensor.new_zeros: torch._refs.new_zeros, + torch.Tensor.new_ones: torch._refs.new_ones, + torch.Tensor.fill_: torch._refs.fill_, + torch.Tensor.zero_: torch._refs.zero_, + torch.Tensor.to: torch._refs.to, + torch.Tensor.sum_to_size: torch._refs.sum_to_size, + # TODO: Should these methods be mapped some other way? + torch.Tensor.copy_: torch._prims.copy_to, + torch.Tensor.resize: torch._prims.resize, + } + for mod_torch, mod_refs in modules: + for s in mod_refs.__all__: # type: ignore[attr-defined] + r[mod_torch.__dict__.get(s)] = mod_refs.__dict__.get(s) + + # Support remapping torch.Tensor.foo to _refs.foo + for s in dir(torch.Tensor): + if s in torch._refs.__all__: + r[getattr(torch.Tensor, s)] = torch._refs.__dict__.get(s) + + # Support conversions + for s in torch._refs._conversions.__all__: + tensor_attr = getattr(torch.Tensor, s, None) or getattr(torch, s) + r[tensor_attr] = torch._refs._conversions.__dict__.get(s) + + return r + + +@functools.cache +def all_prims() -> set[Any]: + """ + Set of all prim functions, e.g., torch._prims.add in all_prims() + """ + return {torch._prims.__dict__.get(s) for s in torch._prims.__all__} + + +class TorchRefsMode(torch.overrides.TorchFunctionMode): + """ + Switches the interpretation of torch.* functions and Tensor methods to + use PrimTorch refs in torch._refs. (Direct calls to _refs are unaffected.) + + >>> # xdoctest: +SKIP + >>> with TorchRefsMode(): + ... torch.add(x, y) # calls torch._refs.add(x, y) + + By default, this context manager will fall back on the torch.* if the + ref does not exist; set strict=True to error if this occurs. + If the ref exists we still would like to fall back on the torch.* sometimes, + this behavior can be customized by passing a function to should_fallback_fn. + """ + + def __init__( + self, + strict: bool = False, + should_fallback_fn: Callable[..., bool] = lambda *_: False, + prims_mode_cls: type = nullcontext, + ) -> None: + self.strict = strict + self.should_fallback_fn = should_fallback_fn + self.prims_mode_cls = prims_mode_cls + + def __torch_function__( + self, + orig_func: Callable[_P, _R], + types: Sequence[type], + args: Sequence[Any] = (), + kwargs: dict[str, Any] | None = None, + ) -> Any: + if kwargs is None: + kwargs = {} + # For primitive operations, run them as is without interception + # Unless we are in prims_mode, in which case we want to use nvprims + if orig_func in torch_function_passthrough or orig_func in all_prims(): + with self.prims_mode_cls(): + # pyrefly: ignore [invalid-param-spec] + return orig_func(*args, **kwargs) + mapping = torch_to_refs_map() + func = mapping.get(orig_func, None) + + # For torch.ops.aten.*, use registered decompositions from torch._decomp + # torch._decomp.decomposition_table provides a mapping from + # torch.ops.aten.* to torch._refs or torch._decomp.decompositions + # implementations. + # There're other ways to implement this functionality, + # see https://github.com/pytorch/pytorch/pull/82657#discussion_r939776417 + if func is None and isinstance(orig_func, torch._ops.OpOverload): + func = torch._decomp.decomposition_table.get(orig_func, None) + elif func is None and isinstance(orig_func, torch._ops.OpOverloadPacket): + default = getattr(orig_func, "default", None) + if default is None and orig_func._dir: + default = getattr(orig_func, orig_func._dir[0], None) + if default is not None: + func = torch._decomp.decomposition_table.get(default, None) + + if func is not None: + # If the ref exists query whether we should use it or not + if self.should_fallback_fn(self, orig_func, func, args, kwargs): + # pyrefly: ignore [invalid-param-spec] + return orig_func(*args, **kwargs) + # torch calls inside func should be interpreted as refs calls + with self: + return func(*args, **kwargs) + if self.strict: + raise RuntimeError( + f"no _refs support for {torch.overrides.resolve_name(orig_func)}" + ) + # pyrefly: ignore [invalid-param-spec] + return orig_func(*args, **kwargs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/debug_prims.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/debug_prims.py new file mode 100644 index 0000000000000000000000000000000000000000..83c9e754fb2bafd9ae73e2b8883eb444a6de976f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/debug_prims.py @@ -0,0 +1,59 @@ +import contextlib +from collections.abc import Generator, Sequence + +import torch +from torch.utils._content_store import ContentStoreReader + + +LOAD_TENSOR_READER: ContentStoreReader | None = None + + +@contextlib.contextmanager +def load_tensor_reader(loc: str) -> Generator[None, None, None]: + global LOAD_TENSOR_READER + assert LOAD_TENSOR_READER is None + # load_tensor is an "op", and we will play merry hell on + # Inductor's memory planning if we return a tensor that + # aliases another tensor that we previously returned from + # an operator. So unlike standard ContentStoreReader use, + # we disable the cache so that you always get fresh storages + # (no aliasing for you!) + LOAD_TENSOR_READER = ContentStoreReader(loc, cache=False) + try: + yield + finally: + LOAD_TENSOR_READER = None + + +def register_debug_prims() -> None: + torch.library.define( + "debugprims::load_tensor", + "(str name, int[] size, int[] stride, *, ScalarType dtype, Device device) -> Tensor", + ) + + @torch.library.impl("debugprims::load_tensor", "BackendSelect") + def load_tensor_factory( + name: str, + size: Sequence[int], + stride: Sequence[int], + dtype: torch.dtype, + device: torch.device, + ) -> torch.Tensor: + if LOAD_TENSOR_READER is None: + from torch._dynamo.testing import rand_strided + + return rand_strided(size, stride, dtype, device) + else: + from torch._dynamo.utils import clone_input + + # device argument here takes care of coercion + r = LOAD_TENSOR_READER.read_tensor(name, device=device) + assert list(r.size()) == size, f"{r.size()} != {size}" + assert list(r.stride()) == stride, f"{r.stride()} != {stride}" + assert r.device == device, f"{r.device} != {device}" + + # Unlike the other properties, we will do coercions for dtype + # mismatch + if r.dtype != dtype: + r = clone_input(r, dtype=dtype) # type: ignore[no-untyped-call] + return r diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/executor.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/executor.py new file mode 100644 index 0000000000000000000000000000000000000000..be3245b8403593cacaf7e7ad8124dd20e5e2be5f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/executor.py @@ -0,0 +1,68 @@ +from collections.abc import Callable +from typing import Any, TypeVar +from typing_extensions import ParamSpec, TypeVarTuple, Unpack + +from torch._prims.context import TorchRefsMode +from torch.fx import GraphModule +from torch.fx.experimental.proxy_tensor import make_fx, wrapper_and_args_for_make_fx + + +T = TypeVar("T") +P = ParamSpec("P") +Ts = TypeVarTuple("Ts") + + +def execute( + gm: GraphModule, + *args: Unpack[Ts], + executor: str = "aten", + executor_parameters: dict | None = None, +) -> Any: + """ + Prototype ATen executor. + + Just executes the context's graph. + """ + + if executor == "aten": + return gm.forward(*args) + + msg = f"Received unexpected value for 'executor': {executor}. Allowed values are: aten." + raise ValueError(msg) + + +def make_traced(fn: Callable[P, T]) -> Callable[P, T]: + """ + Returns a function that, when called, will + trace its torch operations to prims and then + execute those prims on the requested trace executor + (possibly lowering them to that trace executor first). + + Only supports the torch operations defined in _torch_to_reference_map + in context.py and operations with positional args. All args must + be tensors. + In the near future all these restrictions will be lifted. + + Example usage: + + def foo(a, b): + return torch.add(a, b) + + traced_foo = make_traced(foo) + + a = torch.randn((1, 2, 3, 4, 5), device='cuda') + b = torch.randn((1, 2, 3, 4, 5), device='cuda') + result = traced_foo(a, b, executor='aten') + """ + + def _traced(*args: P.args, **kwargs: P.kwargs) -> T: + executor = str(kwargs.pop("executor", "aten")) + + # TODO: caching + wrapped, all_args = wrapper_and_args_for_make_fx(fn, args, kwargs) + + with TorchRefsMode(): + gm = make_fx(wrapped)(all_args) + return execute(gm, all_args, executor=executor) + + return _traced # type: ignore[return-value] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/rng_prims.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/rng_prims.py new file mode 100644 index 0000000000000000000000000000000000000000..2dc334f3a69a4ddcba976eeebd849072b2750692 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims/rng_prims.py @@ -0,0 +1,390 @@ +# mypy: allow-untyped-defs +from typing import cast + +import torch +import torch.utils._pytree as pytree +from torch import _prims +from torch._C import DispatchKey +from torch._higher_order_ops.utils import autograd_not_implemented +from torch._ops import HigherOrderOperator +from torch._prims_common import CUDARngStateHelper, make_contiguous_strides_for +from torch._subclasses.fake_tensor import FakeTensorMode +from torch.fx.experimental.proxy_tensor import ( + disable_proxy_modes_tracing, + ProxyTorchDispatchMode, + track_tensor_tree, +) +from torch.types import _device, _dtype + + +def throw_on_non_cuda(device): + raise RuntimeError( + f"You are trying to functionalize a {device.type} RNG operator but {device.type} does not " + f"use Philox/counter-based RNG. Therefore, functionalizing a {device.type} RNG operator is " + "not supported. We are discussing the possibility of a Philox-based RNG implementation for CPU." + ) + + +def register_rng_prim(name, schema, impl_aten, impl_meta, doc, tags=None): + rngprim_def = torch.library.custom_op( + "rngprims::" + name, impl_aten, mutates_args=(), schema=schema + ) + # pyrefly: ignore [missing-attribute] + rngprim_def.register_fake(impl_meta) + + prim_packet = getattr(torch._ops.ops.rngprims, name) + prim = prim_packet.default + if tags: + prim._tags = tags + + for p in (prim_packet, prim): + p.__doc__ = doc + p.return_type = torch._prims_common.RETURN_TYPE.NEW # type: ignore[attr-defined] + + p.schema = name + schema + p.impl_aten = impl_aten + p.prim_meta_impl = impl_meta + + +# Philox rand offsets could be shared in future with other philox ops, so +# keeping these functions in global scope. +def philox_rand_offset_meta( + shape: torch.Size, +): + return _prims.TensorLike(torch.tensor(0, dtype=torch.int64)) + + +def philox_rand_offset( + shape: torch.Size, +): + # For impl, look at the function calc_execution_policy in the file + # aten/src/ATen/native/cuda/DistributionTemplates.h. The impl was copied at + # commit hash 72aa0667bd16707d50eb8fa337092a1f5d11dfb6 + numel_scalar = 1 + for dim_size in shape: + numel_scalar *= dim_size + numel = torch.scalar_tensor(numel_scalar, dtype=torch.int64) + + block_size = 256 + unroll = 4 + curand4_engine_calls = 4 + device_property = torch.cuda.get_device_properties(torch.cuda.current_device()) + blocks_per_sm = device_property.max_threads_per_multi_processor // block_size + num = cast(int, numel) + grid_size = (num + block_size - 1) // block_size + grid_size = min(grid_size, device_property.multi_processor_count * blocks_per_sm) + return ((num - 1) // (block_size * grid_size * unroll) + 1) * curand4_engine_calls + + +def register_philox_rand(): + name = "philox_rand" + schema = "(SymInt[] size, Tensor seed, Tensor offset, int[]? stride, Device? device=None, ScalarType? dtype=None) -> (Tensor, Tensor)" # noqa: B950 + + def _philox_rand_meta( + shape: torch.Size, + seed: torch.Tensor, + offset: torch.Tensor, + stride: tuple[int, ...] | None, + device: _device, + dtype: _dtype, + ): + # stride arg will be useful for distributed usecase. Currently, its unused. + assert stride is None + stride = make_contiguous_strides_for(shape) + random_values = _prims.TensorMeta( + shape=shape, strides=stride, dtype=dtype, device=device + ) + offset = philox_rand_offset_meta(shape) + return (random_values, offset) + + def _philox_rand( + shape: torch.Size, + seed: torch.Tensor, + offset: torch.Tensor, + stride: tuple[int, ...] | None, + device: _device, + dtype: _dtype, + ): + # stride arg will be useful for distributed usecase. Currently, its unused. + assert stride is None + if device.type == "cpu": + devices = [] + else: + devices = [device] + + if device.type != "cuda": + raise throw_on_non_cuda(device) + + with torch.random.fork_rng(devices): + CUDARngStateHelper.set_torch_state_tensor(seed, offset) + random_values = torch.rand(shape, device=device, dtype=dtype) + + return random_values, philox_rand_offset(shape) + + register_rng_prim( + name=name, + schema=schema, + impl_aten=_philox_rand, + impl_meta=_philox_rand_meta, + doc="Philox based stateless rand operator", + tags=(torch.Tag.nondeterministic_seeded,), + ) + + +def get_device(args, kwargs): + if kwargs.get("device"): + device = kwargs.get("device") + if isinstance(device, str): + device = torch.device(device) + return device.type + + devices = {arg.device.type for arg in args if isinstance(arg, torch.Tensor)} + if any(dev == "cuda" for dev in devices): + return "cuda" + elif any(dev == "xpu" for dev in devices): + return "xpu" + elif any(dev == "hpu" for dev in devices): + return "hpu" + elif any(dev == "cpu" for dev in devices): + return "cpu" + return None + + +def register_run_and_save_rng_state_op(): + class RunAndSaveRngState(HigherOrderOperator): + def __init__(self): + super().__init__("run_and_save_rng_state") + + def __call__(self, op, *args, **kwargs): + return super().__call__(op, *args, **kwargs) + + run_and_save_rng_state = RunAndSaveRngState() + + run_and_save_rng_state.py_impl(DispatchKey.Autograd)( + autograd_not_implemented(run_and_save_rng_state, deferred_error=True) + ) + + @run_and_save_rng_state.py_impl(DispatchKey.CUDA) + def impl_cuda(op, *args, **kwargs): + return torch.cuda.get_rng_state(), op(*args, **kwargs) + + @run_and_save_rng_state.py_impl(DispatchKey.CPU) + def impl_cpu(op, *args, **kwargs): + return torch.get_rng_state(), op(*args, **kwargs) + + @run_and_save_rng_state.py_impl(DispatchKey.HPU) + def impl_hpu(op, *args, **kwargs): + if hasattr(torch, "hpu"): + return torch.hpu.get_rng_state(), op(*args, **kwargs) + raise RuntimeError("functionalize a hpu RNG operator is not supported.") + + @run_and_save_rng_state.py_impl(DispatchKey.XPU) + def impl_xpu(op, *args, **kwargs): + return torch.xpu.get_rng_state(), op(*args, **kwargs) + + @run_and_save_rng_state.py_impl(DispatchKey.BackendSelect) + def impl_backend_select(op, *args, **kwargs): + impl_map = { + "cuda": impl_cuda, + "cpu": impl_cpu, + "hpu": impl_hpu, + "xpu": impl_xpu, + } + device = get_device(args, kwargs) + assert device in impl_map, f"Backend not supported for {device}" + impl = impl_map[device] + return impl(op, *args, **kwargs) + + @run_and_save_rng_state.py_impl(FakeTensorMode) + def impl_fake_tensor_mode(mode, op, *args, **kwargs): + # Check device to call the right impl + with mode: + return impl_backend_select(op, *args, **kwargs) + + @run_and_save_rng_state.py_impl(ProxyTorchDispatchMode) + def impl_proxy_dispatch_mode(mode, op, *args, **kwargs): + out = impl_backend_select(op, *args, **kwargs) + proxy_args = pytree.tree_map(mode.tracer.unwrap_proxy, (op, *args)) + proxy_kwargs = pytree.tree_map(mode.tracer.unwrap_proxy, kwargs) + out_proxy = mode.tracer.create_proxy( + "call_function", run_and_save_rng_state, proxy_args, proxy_kwargs + ) + return track_tensor_tree(out, out_proxy, constant=None, tracer=mode.tracer) + + return run_and_save_rng_state + + +def register_run_with_rng_state_op(): + class RunWithRngState(HigherOrderOperator): + def __init__(self): + super().__init__("run_with_rng_state") + + def __call__(self, rng_state, op, *args, **kwargs): + return super().__call__(rng_state, op, *args, **kwargs) + + run_with_rng_state = RunWithRngState() + + run_with_rng_state.py_impl(DispatchKey.Autograd)( + autograd_not_implemented(run_with_rng_state, deferred_error=True) + ) + + @run_with_rng_state.py_impl(DispatchKey.CUDA) + def impl_cuda(rng_state, op, *args, **kwargs): + current_state = torch.cuda.get_rng_state() + torch.cuda.set_rng_state(rng_state.cpu()) + out = op(*args, **kwargs) + torch.cuda.set_rng_state(current_state) + return out + + @run_with_rng_state.py_impl(DispatchKey.CPU) + def impl_cpu(rng_state, op, *args, **kwargs): + current_state = torch.get_rng_state() + torch.set_rng_state(rng_state) + out = op(*args, **kwargs) + torch.set_rng_state(current_state) + return out + + @run_with_rng_state.py_impl(DispatchKey.HPU) + def impl_hpu(rng_state, op, *args, **kwargs): + if hasattr(torch, "hpu"): + current_state = torch.hpu.get_rng_state() + torch.hpu.set_rng_state(rng_state) + out = op(*args, **kwargs) + torch.hpu.set_rng_state(current_state) + return out + raise RuntimeError("functionalize a hpu RNG operator is not supported.") + + @run_with_rng_state.py_impl(DispatchKey.XPU) + def impl_xpu(rng_state, op, *args, **kwargs): + current_state = torch.xpu.get_rng_state() + torch.xpu.set_rng_state(rng_state) + out = op(*args, **kwargs) + torch.xpu.set_rng_state(current_state) + return out + + @run_with_rng_state.py_impl(ProxyTorchDispatchMode) + def impl_proxy_dispatch_mode(mode, rng_state, op, *args, **kwargs): + # TODO: you don't need to do this, the dispatch here already disabled + # it + with disable_proxy_modes_tracing(): + out = run_with_rng_state(rng_state, op, *args, **kwargs) + proxy_args = pytree.tree_map(mode.tracer.unwrap_proxy, (rng_state, op, *args)) + proxy_kwargs = pytree.tree_map(mode.tracer.unwrap_proxy, kwargs) + out_proxy = mode.tracer.create_proxy( + "call_function", run_with_rng_state, proxy_args, proxy_kwargs + ) + return track_tensor_tree(out, out_proxy, constant=None, tracer=mode.tracer) + + @run_with_rng_state.py_impl(DispatchKey.BackendSelect) + def impl_backend_select(rng_state, op, *args, **kwargs): + impl_map = { + "cuda": impl_cuda, + "cpu": impl_cpu, + "hpu": impl_hpu, + "xpu": impl_xpu, + } + device = get_device(args, kwargs) + assert device in impl_map, f"Backend not supported for {device}" + impl = impl_map[device] + return impl(rng_state, op, *args, **kwargs) + + @run_with_rng_state.py_impl(FakeTensorMode) + def impl_fake_tensor_mode(mode, rng_state, op, *args, **kwargs): + # Skip setting the set_rng_state as it does not work well with fake tensors. + # And it does not matter for the fake tensor mode. + with mode: + return op(*args, **kwargs) + + @run_with_rng_state.py_functionalize_impl + def impl_functional(ctx, rng_state, op, *args, **kwargs): + unwrapped_rng_state = ctx.unwrap_tensors(rng_state) + unwrapped_args = ctx.unwrap_tensors(args) + unwrapped_kwargs = ctx.unwrap_tensors(kwargs) + + with ctx.redispatch_to_next(): + out = run_with_rng_state( + unwrapped_rng_state, op, *unwrapped_args, **unwrapped_kwargs + ) + return ctx.wrap_tensors(out) + + return run_with_rng_state + + +run_and_save_rng_state = register_run_and_save_rng_state_op() +run_with_rng_state = register_run_with_rng_state_op() + + +def register_graphsafe_run_with_rng_state_op(): + class GraphSafeRunWithRngState(HigherOrderOperator): + def __init__(self): + super().__init__("graphsafe_run_with_rng_state") + + def __call__(self, op, *args, rng_state=None, **kwargs): + return super().__call__(op, *args, rng_state=rng_state, **kwargs) + + graphsafe_run_with_rng_state = GraphSafeRunWithRngState() + + graphsafe_run_with_rng_state.py_impl(DispatchKey.Autograd)( + autograd_not_implemented(graphsafe_run_with_rng_state, deferred_error=True) + ) + + @graphsafe_run_with_rng_state.py_impl(DispatchKey.CUDA) + def impl_cuda(op, *args, rng_state=None, **kwargs): + # pyrefly: ignore [missing-attribute] + device_idx = rng_state.device.index + generator = torch.cuda.default_generators[device_idx] + current_state = generator.graphsafe_get_state() + # pyrefly: ignore [bad-argument-type] + generator.graphsafe_set_state(rng_state) + out = op(*args, **kwargs) + generator.graphsafe_set_state(current_state) + return out + + @graphsafe_run_with_rng_state.py_impl(DispatchKey.BackendSelect) + def impl_backend_select(op, *args, rng_state=None, **kwargs): + device = get_device(args, kwargs) + assert device == "cuda", ( + f"GraphSafe RNG operations only supported for CUDA, got {device}" + ) + return impl_cuda(op, *args, rng_state=rng_state, **kwargs) + + @graphsafe_run_with_rng_state.py_impl(FakeTensorMode) + def impl_fake_tensor_mode(mode, op, *args, rng_state=None, **kwargs): + with mode: + return op(*args, **kwargs) + + @graphsafe_run_with_rng_state.py_impl(ProxyTorchDispatchMode) + def impl_proxy_dispatch_mode(mode, op, *args, rng_state=None, **kwargs): + with disable_proxy_modes_tracing(): + out = graphsafe_run_with_rng_state(op, *args, rng_state=rng_state, **kwargs) + proxy_args = pytree.tree_map(mode.tracer.unwrap_proxy, (op, *args)) + proxy_kwargs = pytree.tree_map( + mode.tracer.unwrap_proxy, {"rng_state": rng_state, **kwargs} + ) + out_proxy = mode.tracer.create_proxy( + "call_function", graphsafe_run_with_rng_state, proxy_args, proxy_kwargs + ) + return track_tensor_tree(out, out_proxy, constant=None, tracer=mode.tracer) + + @graphsafe_run_with_rng_state.py_functionalize_impl + def impl_functional(ctx, op, *args, rng_state=None, **kwargs): + unwrapped_rng_state = ( + ctx.unwrap_tensors(rng_state) if rng_state is not None else None + ) + unwrapped_args = ctx.unwrap_tensors(args) + unwrapped_kwargs = ctx.unwrap_tensors(kwargs) + + with ctx.redispatch_to_next(): + out = graphsafe_run_with_rng_state( + op, *unwrapped_args, rng_state=unwrapped_rng_state, **unwrapped_kwargs + ) + return ctx.wrap_tensors(out) + + return graphsafe_run_with_rng_state + + +graphsafe_run_with_rng_state = register_graphsafe_run_with_rng_state_op() + + +def register_rng_prims(): + register_philox_rand() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims_common/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims_common/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..019e9c59f24235ba3545ff9eade7814b28aa8177 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims_common/__init__.py @@ -0,0 +1,2178 @@ +# mypy: allow-untyped-defs +from __future__ import annotations + +import operator +import typing +import warnings +from collections.abc import Callable, Sequence +from contextlib import AbstractContextManager, nullcontext +from enum import Enum +from functools import reduce +from typing import ( + Any, + cast, + NamedTuple, + Optional, + overload, + TYPE_CHECKING, + TypeAlias, + TypeGuard, + TypeVar, + Union, +) +from typing_extensions import deprecated + +import torch +from torch import sym_float, sym_int, sym_max + + +if TYPE_CHECKING: + # Import the following modules during type checking to enable code intelligence features, + # such as auto-completion in tools like pylance, even when these modules are not explicitly + # imported in user code. + + import sympy + + class _WorksWithInt(typing.Protocol): + def __add__(self, other: Any) -> typing.Self: ... + + def __radd__(self, other: Any) -> typing.Self: ... + + def __mul__(self, other: Any) -> typing.Self: ... + + def __rmul__(self, other: Any) -> typing.Self: ... + + _IntLikeT = TypeVar("_IntLikeT", bound=_WorksWithInt) + + +ShapeType: TypeAlias = Union[torch.Size, list[int], tuple[int, ...]] +StrideType: TypeAlias = Union[list[int], tuple[int, ...]] +DimsType: TypeAlias = Union[int, list[int], tuple[int, ...]] +DimsSequenceType: TypeAlias = Union[list[int], tuple[int, ...]] +# TODO: Type[torch.SymInt], Type[torch.SymFloat] +NumberTypeType: TypeAlias = Union[type[bool], type[int], type[float], type[complex]] +# TODO: This needs a lot more type annotations +# NumberType = Union[bool, int, float, complex, torch.SymInt, torch.SymFloat] +NumberType: TypeAlias = Union[bool, int, float, complex] +RealNumberType: TypeAlias = Union[bool, int, float] + +Number = (bool, int, float, complex, torch.SymInt, torch.SymFloat, torch.SymBool) +# I don't call it Integral because numbers.Integral includes bool, but IntLike +# does not +Dim = int +IntLike = (int, torch.SymInt) +FloatLike = (float, torch.SymFloat) +BoolLike = (bool, torch.SymBool) +IntWithoutSymInt = int +FloatWithoutSymFloat = float +DeviceLikeType: TypeAlias = Union[str, torch.device, int] +Tensor = torch.Tensor + + +torch_function_passthrough = { + torch.device, + torch.sym_not, + torch.sym_float, + torch.sym_int, + torch.sym_max, + torch.sym_min, + torch._sym_sqrt, # type: ignore[attr-defined] + torch.sym_ite, + torch.Tensor.dim, + torch.Tensor.ndim.__get__, # type: ignore[attr-defined] + torch.Tensor.numel, + torch.Tensor.size, + torch.Tensor.storage_offset, + torch.Tensor.stride, + torch.Tensor.dtype.__get__, # type: ignore[attr-defined] + torch.Tensor.is_sparse.__get__, # type: ignore[attr-defined] + torch.Tensor.shape.__get__, # type: ignore[attr-defined] + torch.Tensor.device.__get__, # type: ignore[attr-defined] + torch.Tensor.requires_grad.__get__, # type: ignore[attr-defined] + torch.Tensor.layout.__get__, # type: ignore[attr-defined] + torch.Tensor.is_contiguous, + # For TorchRefsMode only + torch.Tensor.__format__, + torch.Tensor.__repr__, + torch.Tensor.requires_grad.__get__, # type: ignore[attr-defined] + torch.Tensor.__getitem__, +} + + +TensorLikeType = torch.Tensor +TensorLike = torch.Tensor +TensorSequenceType: TypeAlias = Union[list[TensorLikeType], tuple[TensorLikeType, ...]] +TensorOrNumberLikeType: TypeAlias = Union[TensorLikeType, NumberType] + +CustomOutParamAnnotation = "__custom_out_param__" + + +def same_shape(a: ShapeType, b: ShapeType, *, allow_rhs_unbacked=False) -> bool: + from torch.fx.experimental.symbolic_shapes import guard_or_true + + if len(a) != len(b): + return False + + for x, y in zip(a, b): + if allow_rhs_unbacked: + if isinstance(y, torch.SymInt): + continue + + # if we do not know, then they are not the same. + if guard_or_true(x != y): + return False + + return True + + +def _maybe_get_pytype(t): + if t is torch.SymFloat: + return float + elif t is torch.SymInt: + return int + elif t is torch.SymBool: + return bool + else: + return t + + +# TODO: look at using torch.testing.assert_close instead with an option +# to just compare metadata +def compare_tensor_meta( + a: TensorLikeType, + b: TensorLikeType, + check_sizes=True, + check_strides=False, + *, + allow_rhs_unbacked=False, + check_conj=True, +): + """ + Checks that two tensor likes have the same shape, + dtype and device. + + In the future this will validate additional metadata, like + strides. + """ + from torch._subclasses.fake_tensor import MetadataMismatchError + + assert isinstance(a, TensorLike) + assert isinstance(b, TensorLike) + + if check_sizes and not same_shape( + a.shape, b.shape, allow_rhs_unbacked=allow_rhs_unbacked + ): + msg = f"Shapes {a.shape} and {b.shape} are not equal!" + raise MetadataMismatchError(msg) + + if a.dtype != b.dtype: + msg = f"Dtypes {a.dtype} and {b.dtype} are not equal!" + raise MetadataMismatchError(msg) + + if a.device != b.device: + # Handles special cuda:0 vs cuda case + # TODO: we should review why this happens and see about fixing it + if (str(a.device) == "cuda:0" or str(a.device) == "cuda") and ( + str(b.device) == "cuda:0" or str(b.device) == "cuda" + ): + pass + else: + msg = f"Devices {a.device} and {b.device} are not equal!" + raise MetadataMismatchError(msg) + + # Stride checking is currently disabled, see https://github.com/pytorch/pytorch/issues/78050 + if check_strides: + same_strides, idx = check_significant_strides( + a, b, allow_rhs_unbacked=allow_rhs_unbacked + ) + if not same_strides: + msg = f"Stride mismatch! Strides are {a.stride()} and {b.stride()} (mismatched at {idx})!" + raise MetadataMismatchError(msg) + + if a.storage_offset() != b.storage_offset(): + msg = f"Storage offset mismatch! Storage offsets are {a.storage_offset()} and {b.storage_offset()}!" + raise MetadataMismatchError(msg) + + if check_conj: + if a.is_conj() != b.is_conj(): + raise MetadataMismatchError( + f"Conj mismatch! is_conj is set to {a.is_conj()} and {b.is_conj()}" + ) + + if a.is_neg() != b.is_neg(): + raise MetadataMismatchError( + f"Neg mismatch! is_neg is set to {a.is_neg()} and {b.is_neg()}" + ) + + +def _check_strides_helper( + a: TensorLikeType, + b: TensorLikeType, + *, + only_cuda=True, + significant_only=True, + allow_rhs_unbacked=False, +) -> tuple[bool, Optional[int]]: + # NOTE: only on CUDA because CPU elementwise strides are incorrect in PyTorch + # See https://github.com/pytorch/pytorch/issues/77553 + # Only compares strides that are "meaningful" -- strides for dimensions with length > 1 + # and for tensors with more than one element + if ( + not only_cuda or a.device.type == "cuda" or b.device.type == "cuda" + ) and a.numel() > 0: + for idx in range(a.ndim): + check = not significant_only or a.shape[idx] > 1 + # TODO: Check the symbols are consistent with each other + if isinstance(b.stride()[idx], torch.SymInt): + continue + if a.stride()[idx] != b.stride()[idx] and check: + return False, idx + + return True, None + + +def check_significant_strides( + a: TensorLikeType, b: TensorLikeType, *, only_cuda=True, allow_rhs_unbacked=False +) -> tuple[bool, Optional[int]]: + return _check_strides_helper( + a, + b, + only_cuda=only_cuda, + significant_only=True, + allow_rhs_unbacked=allow_rhs_unbacked, + ) + + +def check_all_strides( + a: TensorLikeType, b: TensorLikeType, *, only_cuda=True +) -> tuple[bool, Optional[int]]: + return _check_strides_helper(a, b, only_cuda=only_cuda, significant_only=False) + + +def check_contiguous_sizes_strides(sizes, strides, false_if_dde=False): + """ + Performs an equality check between actual stride & expected stride (based on composed sizes), + handling contiguous stride representations: + e.g. torch.empty(u0, u1, u2).contiguous().stride() -> (Max(1, u1) * Max(1, u2), Max(1, u2), 1) + and we'd like to treat this equal to (u1 * u2, u2, 1) for comparison purposes. + """ + + from torch.fx.experimental.symbolic_shapes import ( + guard_or_false, + guard_or_true, + is_nested_int, + ) + + def eval_eager(x): + return bool(x) + + maybe_guard_or_false = guard_or_false if false_if_dde else eval_eager + maybe_guard_or_true = guard_or_true if false_if_dde else eval_eager + + expected_stride = 1 + expected_stride_max = 1 + + for x, y in reversed(tuple(zip(sizes, strides))): + # Skips checking strides when a dimension has length 1. + if maybe_guard_or_false(x == 1): + continue + + if maybe_guard_or_true(y != expected_stride) and maybe_guard_or_true( + y != expected_stride_max + ): + return False + + # We symbolically check both paths to maximize the cases where this function + # returns true. This is because make_contiguous_strides_for adds the max + # symbolically, and in some other situations the max might not be there. + # And we want to ensure we return true in both cases. + expected_stride_max *= x if is_nested_int(x) else sym_max(x, 1) # type:ignore[assignment] + + expected_stride *= x + + return True + + +# This function is equivalent to compute_contiguous() from TensorImpl.cpp +def is_contiguous(a: TensorLikeType, false_if_dde=False) -> bool: + """ + Tests whether a tensor is contiguous or not. + + Tensors are contiguous when they have no elements, + one element, or when they have "nested" strides. + """ + from torch.fx.experimental.symbolic_shapes import ( + guard_or_false, + guard_size_oblivious, + ) + + def eval_eager(x): + return bool(x) + + maybe_guard_or_false = guard_or_false if false_if_dde else eval_eager + + if maybe_guard_or_false(a.numel() < 2): + return True + + return check_contiguous_sizes_strides( + a.shape, a.stride(), false_if_dde=false_if_dde + ) + + +# This function is equivalent to compute_channels_last_contiguous_2d() in TensorImpl.cpp +def is_channels_last_contiguous_2d(a: Tensor, false_if_dde=False) -> bool: + # NHWC or not channels last 2D contiguous + if a.ndim != 4: + return False + + from torch.fx.experimental.symbolic_shapes import guard_or_false, guard_or_true + + def eval_eager(x): + return bool(x) + + maybe_guard_or_false = guard_or_false if false_if_dde else eval_eager + maybe_guard_or_true = guard_or_true if false_if_dde else eval_eager + + expected_stride = 1 + for idx in (1, 3, 2, 0): + length = a.shape[idx] + if maybe_guard_or_false(length == 1): + continue + + stride = a.stride()[idx] + if maybe_guard_or_true(stride != expected_stride): + return False + + expected_stride *= length + + return True + + +def is_channels_last_contiguous_3d(a: Tensor, false_if_dde=False) -> bool: + # NDHWC or not channels last 3D contiguous + if a.ndim != 5: + return False + + from torch.fx.experimental.symbolic_shapes import guard_or_false, guard_or_true + + def eval_eager(x): + return bool(x) + + maybe_guard_or_false = guard_or_false if false_if_dde else eval_eager + maybe_guard_or_true = guard_or_true if false_if_dde else eval_eager + + expected_stride = 1 + for idx in (1, 4, 3, 2, 0): + length = a.shape[idx] + if maybe_guard_or_false(length == 1): + continue + + stride = a.stride()[idx] + if maybe_guard_or_true(stride != expected_stride): + return False + + expected_stride *= length + + return True + + +_memory_formats = { + torch.contiguous_format, + torch.preserve_format, + torch.channels_last, + torch.channels_last_3d, +} + + +def validate_memory_format(memory_format: torch.memory_format): + torch._check( + memory_format in _memory_formats, + lambda: f"Received unknown memory format {memory_format}!", + ) + + +def is_contiguous_for_memory_format( # type: ignore[return] + a: Tensor, + *, + memory_format: torch.memory_format, + false_if_dde=False, + # pyrefly: ignore [bad-return] +) -> bool: + validate_memory_format(memory_format) + + if memory_format == torch.contiguous_format: + return is_contiguous(a, false_if_dde) + if memory_format == torch.channels_last: + return is_channels_last_contiguous_2d(a, false_if_dde) + if memory_format == torch.channels_last_3d: + return is_channels_last_contiguous_3d(a, false_if_dde) + + torch._check( + False, + lambda: f"is_contiguous received unsupported memory format {memory_format}", + ) + + +def is_contiguous_or_false(a: TensorLikeType) -> bool: + return is_contiguous(a, false_if_dde=True) + + +# similar to is_channels_last_contiguous_2d but return false on data dependency. +def is_channels_last_contiguous_or_false_2d(a: Tensor) -> bool: + return is_channels_last_contiguous_2d(a, false_if_dde=True) + + +# similar to is_channels_last_contiguous_3d but return false on data dependency. +def is_channels_last_contiguous_or_false_3d(a: Tensor) -> bool: + return is_channels_last_contiguous_3d(a, false_if_dde=True) + + +# similar to is_contiguous_for_memory_format but return false on data dependency. +def is_contiguous_for_memory_format_or_false( # type: ignore[return] + a: Tensor, *, memory_format: torch.memory_format +) -> bool: + return is_contiguous_for_memory_format( + a, memory_format=memory_format, false_if_dde=True + ) + + +# NOTE: that tensors with no elements and channels last is ??? +def is_channels_last_contiguous(a: Tensor) -> bool: + """ + True when a tensor is channels-last contiguous. + + This requires that: + + - the tensor is conceptually either 4 (NHWC) or 5 (NDHWC) dimensions + - if we name the tensor's dimensions NCHW or NCDHW, then the strides are such that the + stride of the 'C' dimension (Cs) is 1 and the strides corresponding to + each dimension (Xs) can be ordered Cs <= Ws <= Hs <= (Ds) <= Ns and are + "nested" -- so Ws = Cs * Cl, where Cl is the length of the 'C' dimension, + for example. + """ + return is_channels_last_contiguous_2d(a) or is_channels_last_contiguous_3d(a) + + +# similar to is_channels_last_contiguous but return false on data dependency. +def is_channels_last_contiguous_or_false(a: Tensor) -> bool: + return is_channels_last_contiguous_or_false_2d( + a + ) or is_channels_last_contiguous_or_false_3d(a) + + +def _is_non_overlapping_and_dense_or_false(sizes, strides) -> bool: + """ + Helper function for is_non_overlapping_and_dense. + For unbacked sizes & strides, returns True only if symbolically non-overlapping & dense, + and False otherwise. + + e.g. sizes: [u0, u1], strides: [u2, u3] + this may be non-overlapping & dense at runtime, for values {u0: 4, u1: 4, u2: 4, u3: 1}, + but isn't true for all values. + """ + from torch.fx.experimental.symbolic_shapes import guard_or_false, guard_or_true + from torch.utils._sympy.functions import Max + + # Short-circuits for 0/1-element tensors + if guard_or_false(prod(sizes) < 2): # type: ignore[operator] + return True + + # Short-circuits for tensors of rank one, which are + # non-overlapping and "dense" if their stride is one + if len(sizes) == 1: + return guard_or_false(strides[0] == 1) + + # Checks that there exists a permutation of the strides s.t. the tensor would be contiguous + # Sorts (length, stride) pairs by stride + # + # This sort is done in a size-oblivious way, which helps if we do a + # comparison like 2048*u0 > u0; we just want this to return True + # (and not worry about what if u0 is zero). + class K(NamedTuple): + size: int + stride: int + + def __lt__(self, other): + # for backed symbols, this is practically a < operation + # for unbacked, we return True if < is statically known, + # then try to answer this symbolically, with stride ordering semantics + # (e.g. u0 < u0 is False, u0 < u1 is False with no axioms, u0 < 2 * u0 is True) + return ( + guard_or_false( + self.stride < other.stride + ) # checks statically known inequality + or ( + ( + guard_or_false(self.stride == 0) + or guard_or_false(other.stride % self.stride == 0) + ) + and guard_or_true(self.stride != other.stride) + ) # checks symbolic inequality (e.g. u0 < 2048 * u0) + ) + + lengths_and_strides = sorted(map(K, sizes, strides)) + + # verify actual strides match the expected (composed sizes) + sizes = [x.size for x in lengths_and_strides][::-1] + strides = [x.stride for x in lengths_and_strides][::-1] + return check_contiguous_sizes_strides(sizes, strides, false_if_dde=True) + + +def is_non_overlapping_and_dense(a: Tensor) -> bool: + """ + True when a tensor is non-overlapping and dense. + + A tensor is non-overlapping and dense when there exists a permutation of + its dimensions that is contiguous. + """ + from torch.fx.experimental.symbolic_shapes import guard_or_false + + if a.is_sparse: + return False + + return _is_non_overlapping_and_dense_or_false(a.shape, a.stride()) + + +# NOTE: Based on the implementation in TensorIterator.cpp, but note that +# the note [Computing output strides] is incorrect, because it +# says that strides will be preserved even if they are not +# "non overlapping and dense", but this is incorrect. The +# output of elementwise operations are always given +# non overlapping and dense strides. +# This is also INCORRECT because it does not model TensorIterator's +# short-circuit, which can cause different strides. +def compute_elementwise_output_logical_to_physical_perm( + *tensors, _skip_checks=False, ambiguity_check=False +) -> tuple[list[int], bool]: + from torch.fx.experimental.symbolic_shapes import guard_or_false + + if not _skip_checks and len(tensors) == 0: + msg = "Can't compute elementwise output strides for zero tensors!" + raise ValueError(msg) + + if not _skip_checks: + check_same_shape(*tensors, allow_cpu_scalar_tensors=True) + + # Filters the tensors to actual tensors + if not _skip_checks: + tensors = tuple( + a + for a in tensors + if isinstance(a, TensorLike) and not is_cpu_scalar_tensor(a) + ) + + # Short-circuits for CPU scalar case + if len(tensors) == 0: + return [], False + + # Short-circuits for shapes with zero or one dimensions + # TODO: are these necessary? + ndim = tensors[0].ndim + if ndim == 0: + return [], False + if ndim == 1: + return [0], False + + # Short-circuits if contiguous or channels last, following the fake fast path. + # This reduces the number of guards we end up making + is_contiguous = True + is_channels_last = True + for t in tensors: + is_contiguous = is_contiguous and is_contiguous_for_memory_format_or_false( + t, memory_format=torch.contiguous_format + ) + is_channels_last = ( + is_channels_last + and is_contiguous_for_memory_format_or_false( + t, memory_format=torch.channels_last + ) + ) + + if is_contiguous and not is_channels_last: + return list(range(ndim)), False + + if is_channels_last and not is_contiguous: + return [0, *list(range(2, ndim)), 1], False + + shape = tensors[0].shape + + def should_swap(idx_a, idx_b): + def ge(a, b): + """ + Returns true if a is symbolically greater than or equal to b, assuming a >= 0, b >= 0. + """ + if guard_or_false(b == 0): + return True + elif guard_or_false(a == 0): + return False + return guard_or_false(a >= b) or guard_or_false(a % b == 0) + + for tensor in tensors: + stride_a = tensor.stride()[idx_a] + stride_b = tensor.stride()[idx_b] + + if guard_or_false(stride_a == 0) or guard_or_false(stride_b == 0): + continue + + if guard_or_false(stride_a == stride_b): + if ge(shape[idx_b], shape[idx_a]): + continue + return 1 + + if ge(stride_b, stride_a): + return -1 + + if ge(stride_a, stride_b): + return 1 + + # Note: this case is hit if all strides are zero, + # or all strides are equal and all dimensions have the same length + return 0 + + # The "sort" order for the permutation is back-to-front, but + # the natural order for permutations is front-to-back. Do the + # sorting back-to-front and then reverse it on output. + # + # also, note this returns the logical to physical shape permutation + perm = list(reversed(range(ndim))) + + # insertion sort with support for ambiguous comparisons + for i in range(1, ndim): + dim1 = i + for dim0 in reversed(range(i)): + comparison = should_swap(perm[dim0], perm[dim1]) + if comparison > 0: + perm[dim0], perm[dim1] = perm[dim1], perm[dim0] + dim1 = dim0 + elif comparison < 0: + break + + # verify we've imposed ordering if ambiguity_check=True + raise_ambiguous = False + if ambiguity_check: + for i, j in zip(range(ndim - 1), range(1, ndim)): + order = should_swap(perm[i], perm[j]) + if order != -1: + raise_ambiguous = True + break + + return list(reversed(perm)), raise_ambiguous + + +def compute_elementwise_output_strides(*tensors) -> tuple[int, ...]: + """ + Computes the output strides for elementwise operations. + """ + if len(tensors) == 0: + msg = "Can't compute elementwise output strides for zero tensors!" + raise ValueError(msg) + + check_same_shape(*tensors, allow_cpu_scalar_tensors=True) + + # Filters the tensors to actual tensors + tensors = tuple( + a for a in tensors if isinstance(a, TensorLike) and not is_cpu_scalar_tensor(a) + ) + + # Short-circuits for CPU scalar case + if len(tensors) == 0: + return () + + ndim = tensors[0].ndim + shape = tensors[0].shape + + if ndim == 0: + return () + if ndim == 1: + return (1,) + + logical_to_physical_perm, _ = compute_elementwise_output_logical_to_physical_perm( + *tensors, _skip_checks=True + ) + permuted_shape = apply_perm(shape, logical_to_physical_perm) # to physical + + new_strides = make_contiguous_strides_for(permuted_shape) + permuted_strides = apply_perm( + new_strides, invert_perm(logical_to_physical_perm) + ) # to logical + + return tuple(permuted_strides) + + +# Identity permutation is [0, 1, 2] +def apply_perm(inp, perm): + ndim = len(inp) + permuted_inp = [-1] * ndim + for idx, x in enumerate(perm): + permuted_inp[idx] = inp[x] + return permuted_inp + + +def invert_perm(perm): + ndim = len(perm) + new_perm = [-1] * ndim + for idx, x in enumerate(perm): + new_perm[x] = idx + return new_perm + + +# +# Common helper functions +# + + +def validate_dim_length(length: int): + """ + Validates that an object represents a valid + dimension length. + """ + + if isinstance(length, (int, torch.SymInt)): + torch._check(length >= 0) + else: + # sometimes called with sympy expression by inductor + assert length >= 0 + + +def validate_shape(shape: ShapeType): + """ + Validates that a sequence represents a valid shape. + """ + + assert isinstance(shape, Sequence), type(shape) + for l in shape: + validate_dim_length(l) + + +def validate_strides(strides: StrideType): + """ + Verifies the object specifies valid strides. + """ + + assert isinstance(strides, Sequence) + for stride in strides: + assert stride >= 0 + + +def validate_idx(rank: int, idx: int): + """ + Validates that idx is a valid index for the given shape. + Assumes the index is already canonicalized. + """ + + assert isinstance(idx, Dim) + assert isinstance(rank, Dim) + + assert idx >= 0 and idx < rank or idx == 0 + + +def validate_dimension_indices(rank: int, indices: DimsSequenceType): + for idx in indices: + validate_idx(rank, idx) + + +def validate_exclusive_idx(rank: int, ex_idx: int): + """ + Validates that ex_idx is a valid exclusive index + for the given shape. + """ + + assert isinstance(ex_idx, Dim) + assert isinstance(rank, Dim) + assert ex_idx > 0 and ex_idx <= rank + + +# "Wraps" a dim (up to one time) for the given rank, allowing dims to be +# specified using negative indices. If `wrap_scalar` is true then scalar +# tensors of rank 0 will allow dimensions in the range [-1, 0]. Otherwise, +# idx should be in the range [-rank, rank-1]. +def canonicalize_dim(rank: int, idx: int, wrap_scalar: bool = True) -> int: + if rank < 0: + msg = f"Rank cannot be negative but got {rank}" + raise IndexError(msg) + + if rank == 0: + if not wrap_scalar: + msg = f"Dimension specified as {idx} but tensor has no dimensions" + raise IndexError(msg) + rank = 1 + + if idx >= 0 and idx < rank: + return idx + + if idx < 0: + _idx = idx + rank + else: + _idx = idx + + if _idx < 0 or _idx >= rank: + # Same error message as in aten/src/ATen/WrapDimUtils.h:49 + msg = f"Dimension out of range (expected to be in range of [{-rank}, {rank - 1}], but got {idx})" + raise IndexError(msg) + + return _idx + + +# Takes a dimension or sequence of dimensions and "wraps" them, +# mapping negative offsets to positive ones +@overload +def canonicalize_dims( + rank: int, + indices: Sequence[int], + wrap_scalar: bool = True, + # pyrefly: ignore [bad-return] +) -> tuple[int, ...]: + pass + + +@overload +# pyrefly: ignore [bad-return] +def canonicalize_dims(rank: int, indices: int, wrap_scalar: bool = True) -> int: + pass + + +def canonicalize_dims(rank, indices, wrap_scalar=True): + if isinstance(indices, Dim): + return canonicalize_dim(rank, indices, wrap_scalar) + + return tuple(canonicalize_dim(rank, x, wrap_scalar) for x in indices) + + +def is_valid_permutation(rank: int, perm: DimsSequenceType) -> bool: + """ + Validates that perm is a permutation of length rank. + """ + + return isinstance(perm, Sequence) and sorted(perm) == list(range(rank)) + + +def is_same_shape(a: Sequence, b: Sequence) -> bool: + """ + Compares two shapes a and b, returning True if they are the same + (their ranks and corresponding lengths match) and False otherwise. + """ + + return tuple(a) == tuple(b) + + +def is_cpu_scalar_tensor(a: object) -> TypeGuard[TensorLike]: + return isinstance(a, TensorLike) and a.ndim == 0 and a.device.type == "cpu" + + +def check_same_device(*args, allow_cpu_scalar_tensors): + """ + Checks that all Tensors in args have the same device. + + Raises a RuntimeError when: + - args contains an object whose type is not Tensor or Number + - two Tensor objects in args have different devices, unless one is a CPU scalar tensor and allow_cpu_scalar_tensors is True + """ + # Short-circuits if all (one or fewer) arguments are trivially on the same device + if len(args) <= 1: + return + + # Note: cannot initialize device to the first arg's device (it may not have one) + device = None + # pyrefly: ignore [bad-assignment] + for arg in args: + if isinstance(arg, Number): + continue + elif isinstance(arg, TensorLike): + if allow_cpu_scalar_tensors and is_cpu_scalar_tensor(arg): + continue + + if device is None: + device = arg.device + + if device != arg.device: + msg = ( + "Tensor on device " + + str(arg.device) + + " is not on the expected device " + + str(device) + + "!" + ) + raise RuntimeError(msg) + else: + msg = ( + "Unexpected type when checking for same device, " + str(type(arg)) + "!" + ) + raise RuntimeError(msg) + + +def canonicalize_device(device: DeviceLikeType) -> torch.device: + if isinstance(device, torch.device): + return device + + assert isinstance(device, str) + return torch.device(device) + + +# Asserts if any of the following are true: +# - a non-scalar or non-Tensor is given +# - the shape of any tensors is distinct +def check_same_shape(*args, allow_cpu_scalar_tensors: bool): + """ + Checks that all Tensors in args have the same shape. + + Raises a RuntimeError when: + - args contains an object whose type is not Tensor or Number + - two Tensor objects in args have different devices + """ + shape = None + + # pyrefly: ignore [bad-assignment] + for arg in args: + if isinstance(arg, Number): + continue + elif isinstance(arg, TensorLike): + if allow_cpu_scalar_tensors and is_cpu_scalar_tensor(arg): + continue + + if shape is None: + shape = arg.shape + + if not is_same_shape(shape, arg.shape): + msg = f"Shape {arg.shape} is not the expected shape {shape}!" + raise RuntimeError(msg) + else: + msg = ( + "Unexpected type when checking for same shape, " + str(type(arg)) + "!" + ) + raise RuntimeError(msg) + + +# Acquires a common shape, if it exists, from one or more tensor arguments, +# filtering number arguments +def extract_shape(*args, allow_cpu_scalar_tensors: bool) -> Optional[ShapeType]: + shape = None + scalar_shape = None + + # pyrefly: ignore [bad-assignment] + for arg in args: + if isinstance(arg, Number): + continue + elif isinstance(arg, TensorLike): + if allow_cpu_scalar_tensors and is_cpu_scalar_tensor(arg): + scalar_shape = arg.shape + continue + + if shape is None: + shape = arg.shape + + if not is_same_shape(shape, arg.shape): + return None + else: + return None + + return shape if shape is not None else scalar_shape + + +# Extracts dimensions that might be passed either as a list/tuple or as varargs. +# A typical case is Tensor.permute . +def extract_dims_from_varargs( + dims: Union[DimsSequenceType, tuple[DimsSequenceType, ...]], +) -> DimsSequenceType: + if dims and isinstance(dims[0], Sequence): + assert len(dims) == 1 + dims = cast(tuple[DimsSequenceType], dims) + return dims[0] + else: + return cast(DimsSequenceType, dims) + + +def extract_shape_from_varargs( + shape: Union[ShapeType, tuple[ShapeType]], + validate=True, +) -> tuple[int, ...]: + """ + Returns a shape from varargs. + + In PyTorch, operations that accept shapes often accept them as varargs, like + foo(*shape). However a user can pass the shape as a sequence of integers, + like this: + + foo(1, 2, 3) + + or as a sequence of integers + + foo((1, 2, 3)) + + In the first case shape will be a tuple of integers, and in the second case it's a tuple + containing a tuple of integers. This validates those inputs and canonicalizes them + to a tuple of integers. + """ + + # Handles tuple unwrapping + if len(shape) == 1 and isinstance(shape[0], Sequence): + # pyrefly: ignore [bad-assignment] + shape = shape[0] + + if validate: + validate_shape(shape) # type: ignore[arg-type] + return shape # type: ignore[return-value] + + +def infer_size_shapes(a: ShapeType, b: ShapeType) -> tuple[int, ...]: + ndim = max(len(a), len(b)) + expandedSizes = [0] * ndim + + for i in range(ndim - 1, -1, -1): + offset = ndim - 1 - i + dimA = len(a) - 1 - offset + dimB = len(b) - 1 - offset + sizeA = a[dimA] if dimA >= 0 else 1 + sizeB = b[dimB] if dimB >= 0 else 1 + + torch._check( + (sizeA == sizeB) or (sizeA == 1) or (sizeB == 1), + lambda: ( + f"The size of tensor a ({sizeA}) must match the size of " + f"tensor b ({sizeB}) at non-jagged dimension {i}" + ), + ) + + # 1s map to the other size (even 0) + expandedSizes[i] = sizeB if sizeA == 1 else sizeA + + return tuple(expandedSizes) + + +def infer_size(shape: ShapeType, numel: int) -> tuple[int, ...]: + """ + Infers the size of a dim with size -1, if it exists. + Also checks that new shape is compatible with the number of elements. + """ + from torch.fx.experimental.symbolic_shapes import guard_or_false + + dim = None + newsize = 1 + for i, d in enumerate(shape): + if guard_or_false(d == -1): + torch._check(dim is None, lambda: "only one dimension can be inferred") + dim = i + else: + torch._check( + d >= 0, + lambda: ( + f"invalid shape dimension {d}. If this was symbolic, it was assumed to not be -1." + "If this was meant to be inferred, please explicitly pass in -1." + ), + ) + newsize *= d + if dim is None: + torch._check( + numel == newsize, + lambda: f"shape '{list(shape)}' is invalid for input of size {numel}", + ) + else: + torch._check( + newsize != 0, + lambda: ( + f"cannot reshape tensor of 0 elements into shape {list(shape)} because the " + f"unspecified dimension size -1 can be any value and is ambiguous" + if guard_or_false(numel == 0) + else f"shape '{list(shape)}' is invalid for input of size {numel}" + ), + ) + torch._check( + numel % newsize == 0, + lambda: f"shape '{list(shape)}' is invalid for input of size {numel}", + ) + # Convert to list to produce a compatible error message with core + # PyTorch, which prints sequences in square brackets. + shape = list(shape) + shape[dim] = numel // newsize + torch._check(shape[dim] >= 0) + return tuple(shape) + + +_integer_dtypes = ( + torch.uint8, + torch.uint16, + torch.uint32, + torch.uint64, + torch.int8, + torch.int16, + torch.int32, + torch.int64, +) +_low_precision_dtypes = (torch.float16, torch.bfloat16, torch.complex32) +_complex_dtypes = (torch.complex32, torch.complex64, torch.complex128) + + +def is_boolean_dtype(dtype: torch.dtype) -> bool: + assert isinstance(dtype, torch.dtype) + return dtype is torch.bool + + +def is_integer_dtype(dtype: torch.dtype) -> bool: + assert isinstance(dtype, torch.dtype) + return dtype in _integer_dtypes + + +def is_low_precision_dtype(dtype: torch.dtype) -> bool: + assert isinstance(dtype, torch.dtype) + return dtype in _low_precision_dtypes + + +def is_float_dtype(dtype: torch.dtype) -> bool: + assert isinstance(dtype, torch.dtype) + return dtype.is_floating_point + + +def is_complex_dtype(dtype: torch.dtype) -> bool: + assert isinstance(dtype, torch.dtype) + return dtype in _complex_dtypes + + +def is_grad_dtype(dtype: torch.dtype) -> bool: + """ + Checks if the dtype can require a gradient. + """ + return dtype.is_floating_point or is_complex_dtype(dtype) + + +_complex_to_real_dtype_map = { + torch.complex128: torch.float64, + torch.complex64: torch.float32, + torch.complex32: torch.float16, +} + +_real_to_complex_dtype_map = { + torch.float16: torch.complex32, + torch.bfloat16: torch.complex64, + torch.float32: torch.complex64, + torch.float64: torch.complex128, +} + + +def corresponding_real_dtype(dtype: torch.dtype) -> torch.dtype: + return _complex_to_real_dtype_map[dtype] + + +def corresponding_complex_dtype(dtype: torch.dtype) -> torch.dtype: + return _real_to_complex_dtype_map[dtype] + + +def dtype_to_type(dtype: torch.dtype) -> type: + """ + Computes the corresponding Python type (AKA "type kind") for the + given dtype. + """ + assert isinstance(dtype, torch.dtype) + + if dtype is torch.bool: + return bool + if dtype in _integer_dtypes: + return int + if dtype.is_floating_point: + return float + if dtype in _complex_dtypes: + return complex + + raise ValueError("Invalid dtype!") + + +def dtype_to_type_ctor(dtype: torch.dtype) -> Callable[[NumberType], NumberType]: + """ + Computes the corresponding Python type constructor for the + given dtype. + """ + assert isinstance(dtype, torch.dtype) + + if dtype is torch.bool: + return lambda x: bool(x) + if dtype in _integer_dtypes: + return sym_int + if dtype.is_floating_point: + return sym_float + if dtype in _complex_dtypes: + # TODO: type error here is real, replace with sym_complex + return lambda x: complex(x) # type: ignore[arg-type] + + raise ValueError("Invalid dtype!") + + +def type_to_dtype(typ: type) -> torch.dtype: + """ + Computes the corresponding dtype for a Number type. + """ + + assert isinstance(typ, type) + + if typ in (bool, torch.SymBool): + return torch.bool + if typ in (int, torch.SymInt): + return torch.long + if typ in (float, torch.SymFloat): + return torch.get_default_dtype() + # TODO: sym_complex_float? + if typ is complex: + return corresponding_complex_dtype(torch.get_default_dtype()) + + raise ValueError(f"Invalid type {typ}!") + + +def get_dtype(x: Union[torch.Tensor, NumberType]): + if isinstance(x, torch.Tensor): + return x.dtype + else: + return type_to_dtype(type(x)) + + +_ordered_types = (bool, int, float, complex) + + +def check_fp_or_complex( + dtype: torch.dtype, fn_name: str, allow_low_precision_dtypes: bool = True +): + """ + Checks whether the input is floating point or complex. + If allow_low_precision_dtypes is True, it allows having float16, bfloat16, and complex32 + """ + torch._check( + is_float_dtype(dtype) or is_complex_dtype(dtype), + lambda: f"{fn_name}: Expected a floating point or complex tensor as input. Got {dtype}", + ) + torch._check( + allow_low_precision_dtypes or not is_low_precision_dtype(dtype), + lambda: f"{fn_name}: Half precision dtypes not supported. Got {dtype}", + ) + + +def check_is_matrix(A: TensorLikeType, f_name: str, arg_name: str = "A"): + torch._check( + len(A.shape) >= 2, + lambda: f"{f_name}: The input tensor {arg_name} must have at least 2 dimensions.", + ) + + +def get_higher_type(a: type, b: type) -> type: + """ + Returns the higher of the two given Number types. + + The types are ordered bool -> int -> float -> complex. + """ + a, b = _maybe_get_pytype(a), _maybe_get_pytype(b) + # Type checking + if a not in _ordered_types or b not in _ordered_types: + raise RuntimeError(f"Expected builtin numeric types, found {a}, {b}") + + if a is b: + return a + + for typ in _ordered_types: + if a is typ: + return b + if b is typ: + return a + + raise ValueError("Unknown Python scalar type!") + + +# Returns the higher of two torch datatypes a and b or, if the two +# are not ordered relative to each other, the next +# higher datatype +def get_higher_dtype( + a: Optional[Union[torch.dtype, TensorLikeType, NumberType]], + b: Optional[Union[torch.dtype, TensorLikeType, NumberType]], +) -> Optional[torch.dtype]: + """ + Computes the "lowest" datatype that is weakly + "higher" than both a and b. + """ + + # Type checking + assert a is None or isinstance(a, (torch.dtype, TensorLike, Number)) + assert b is None or isinstance(b, (torch.dtype, TensorLike, Number)) + + def _extract_dtype( + x: Optional[Union[torch.dtype, TensorLikeType, NumberType]], + ) -> Optional[torch.dtype]: + if x is None: + return None + if isinstance(x, torch.dtype): + return x + if isinstance(x, TensorLike): + return x.dtype + if isinstance(x, Number): + return type_to_dtype(type(x)) + + raise RuntimeError("Unexpected type given to _extract_dtype!") + + # pyrefly: ignore [bad-argument-type] + a, b = _extract_dtype(a), _extract_dtype(b) + + if a is b: + return a + + if a is None: + return b + + if b is None: + return a + + ordered_datatypes = ( + (torch.bool,), + (torch.uint8, torch.int8), + (torch.int16,), + (torch.int32,), + (torch.int64,), + (torch.float16, torch.bfloat16), + (torch.float32,), + (torch.float64,), + (torch.complex32,), + (torch.complex64,), + (torch.complex128,), + ) + + for idx, dtypes in enumerate(ordered_datatypes): + if a in dtypes and b in dtypes: + return ordered_datatypes[idx + 1][0] + if a in dtypes: + return b + if b in dtypes: + return a + + raise RuntimeError("Unexpected termination!") + + +def check_pin_memory(pin_memory: bool): + torch._check_not_implemented( + not pin_memory, lambda: "PrimTorch does not support pinned memory" + ) + + +def check_layout(layout: torch.layout): + torch._check_not_implemented( + layout == torch.strided, lambda: f"PrimTorch doesn't support layout={layout}" + ) + + +# TODO: maybe unify with can_cast_to? +def is_weakly_lesser_type(a: type, b: type) -> bool: + """ + Compares two types, a and b, returning True if a is weakly "less" than b. + + The comparison is determined by the following type ordering: bool, int, float, complex. + """ + + a, b = _maybe_get_pytype(a), _maybe_get_pytype(b) + + if a not in _ordered_types or b not in _ordered_types: + raise RuntimeError(f"Expected builtin numeric types, found {a}, {b}") + + for typ in _ordered_types: + if a == typ: + return True + if b == typ: + return False + + raise RuntimeError("Unexpected termination!") + + +def can_safe_cast_to(*, cast_to: torch.dtype, cast_from: torch.dtype) -> bool: + for fn in (is_complex_dtype, is_float_dtype, is_integer_dtype, is_boolean_dtype): + if fn(cast_to): + return True + if fn(cast_from): + return False + + raise ValueError(f"Received unknown dtypes {cast_to}, {cast_from}!") + + +def check_same_dtype(*args): + """ + Checks that all Tensors in args have the same device and that all Numbers have the + same corresponding Python type. + + Raises a RuntimeError when: + - args contains an object whose type is not Tensor or Number + - two Tensors objects in args have different dtypes + - two Number objects in args have different types + - there are Tensors and Numbers in args, and one of those Tensors corresponding + Python types is different from the type of one of those Numbers + """ + full_dtype = None + scalar_type = None + + # pyrefly: ignore [bad-assignment] + for arg in args: + if isinstance(arg, Number): + # Scalar type checking is disabled (and may be removed in the future) + continue + # if scalar_type is None: + # scalar_type = type(arg) + + # if scalar_type is not type(arg): + # msg = ( + # "Scalar of type " + # + str(type(arg)) + # + " is not the expected type of " + # + str(scalar_type) + # + "!" + # ) + # raise RuntimeError(msg) + elif isinstance(arg, TensorLike): + if full_dtype is None: + full_dtype = arg.dtype + if scalar_type is None: + scalar_type = dtype_to_type(arg.dtype) + + if full_dtype is not arg.dtype: + msg = ( + "Tensor with dtype " + + str(arg.dtype) + + " is not the expected dtype of " + + str(full_dtype) + + "!" + ) + raise RuntimeError(msg) + + arg_type = dtype_to_type(arg.dtype) + if arg_type is not scalar_type: + msg = ( + "Tensor with corresponding Python type " + + str(arg_type) + + " is not the expected type of " + + str(scalar_type) + + "!" + ) + raise RuntimeError(msg) + else: + msg = ( + "Unexpected type when checking for same dtype, " + str(type(arg)) + "!" + ) + raise RuntimeError(msg) + + +# Maps datatypes to their computation types for elementwise operations +_computation_dtype_map = { + torch.bfloat16: torch.float32, + torch.float16: torch.float32, + torch.complex32: torch.complex64, +} + + +def get_computation_dtype(dtype: torch.dtype) -> torch.dtype: + return _computation_dtype_map.get(dtype, dtype) + + +_cpu_acc_type_map = { + torch.bfloat16: torch.float64, + torch.float16: torch.float64, + torch.float32: torch.float64, + torch.complex32: torch.complex128, + torch.complex64: torch.complex128, +} + + +def get_acc_type(dtype: torch.dtype, device: torch.device) -> torch.dtype: + # Equivalent to at::toAccumulateType, prefer computation_dtype where possible + if device.type == "cpu": + return _cpu_acc_type_map.get(dtype, dtype) + else: + return get_computation_dtype(dtype) + + +class ELEMENTWISE_TYPE_PROMOTION_KIND(Enum): + DEFAULT = (0,) + NO_OPMATH = (1,) + INT_TO_FLOAT = (2,) + ALWAYS_BOOL = (3,) + COMPLEX_TO_FLOAT = (4,) + BOOL_TO_LONG = (5,) + + +class REDUCTION_OUTPUT_TYPE_KIND(Enum): + SAME = (0,) + COMPLEX_TO_FLOAT = (1,) # for complex types outputs corresponding real type + KEEP_PROMOTED_TYPE = (2,) # keep output in opmath type, needed for mean + ALWAYS_BOOL = (3,) + + +# Describes the return type of the primitive: +# +# - NEW, a new tensor is created +# - VIEW, a view of an input tensor is returned +# - INPLACE, one or more input tensors is modified +# +# these descriptors are mututally exclusive and exhaustive. +class RETURN_TYPE(Enum): + NEW = (0,) + VIEW = (1,) + INPLACE = (2,) + NONE = (3,) + + +# TODO: when NumberType contains the sym types, can simplify this +def number_type( + x: Union[NumberType, torch.SymInt, torch.SymFloat, torch.SymBool], +) -> type: + if isinstance(x, torch.SymInt): + return int + elif isinstance(x, torch.SymFloat): + return float + elif isinstance(x, torch.SymBool): + return bool + else: + return type(x) + + +def expr_type(x: sympy.Basic) -> type: + import sympy + + if x.kind is sympy.core.kind.BooleanKind: + return bool + elif x.is_integer: # type: ignore[attr-defined] + return int + else: + # NB: Not strictly correct, but we don't support SymPy complex or bool. + return float + + +# TODO: document type promotion kinds +def elementwise_dtypes( + *_args, + type_promotion_kind: ELEMENTWISE_TYPE_PROMOTION_KIND, +) -> tuple[torch.dtype, torch.dtype]: + """ + Computes the computation and result dtypes for elementwise type promotion + on the given arguments and with the given elementwise type promotion kind. + + Note that not all inputs to an elementwise operation necessarily participate in type promotion. + For example, the "alpha" parameter of torch.add does not participate in type promotion, + although it may be cast to the Python type corresponding to the computation dtype that + the type promotion algorithm determines. + + Default elementwise type promotion, which all other type promotion kinds tweak (see below), + first decides which of four ordered types to use: + + bool -> integer -> floating point -> complex + + The selected type is the "lowest" type in the above list such that all number arguments + have a weakly "lower" type and all tensor arguments have a weakly lower corresponding + type for their dtype. + + Once the type is determined, the particular result dtype is found. The dtypes are + partially ordered as follows: + + bool -> uint8, int8 -> int16 -> int32 -> int64 -> + float16, bfloat16 -> float32 -> float64 -> complex32 -> complex64 -> complex128 + + The result dtype is selected by: + - if no tensor's dtype has the same corresponding type as the one selected, + then the result dtype is the (default) dtype corresponding to the selected type + (for example, 1.5 + an integer tensor has a result dtype of the default floating point dtype) + - if the result type is complex then the dtype is: + - the default complex dtype if there are no floating point or complex tensors + - if there are floating point or complex tensors with one or more dimensions, then + the complex dtype corresponding to the highest corresponding complex dtype among those tensors + (for example, double + cfloat -> cdouble) + - if there are only floating point or complex tensors with zero dimensions, then + the complex dtype corresponding to the highest corresponding complex dtype among those tensors + - if the first two cases do not apply, the result dtype is the highest dtype among + all tensors with one or more dimensions of the output type, and if there are no such + tensors then it's the highest dtype among all tensors with zero dimensions of the output type + (for example, long + half -> half, even if the half tensor has zero dimensions) + + The "corresponding complex dtypes" are: + float16 -> complex32 + bfloat16 -> complex64 + float32 -> complex64 + float64 -> complex128 + complex32 -> complex32 + complex64 -> complex64 + complex128 -> complex128 + + The DEFAULT type promotion kind computes per above, and then uses the result dtype to pick a computation + dtype by mapping low precision floating point and complex dtypes as follows: + + float16 -> float32 + bfloat16 -> float32 + complex32 -> complex64 + + This is referred to as "op math", and the NO_OPMATH type promotion kind disables this mapping, making the + computation dtype the same as the result dtype when it's selected. NO_OPMATH is appropriate for kernels + which perform no mathematical operations on their tensors (see below for examples). + + The INT_TO_FLOAT type promotion kind maps boolean and integer result dtypes to the default floating point dtype, + and computation dtypes to the appropriate op math dtype. + + The COMPLEX_TO_FLOAT type promotion kind maps complex result dtypes to the corresponding float dtype, following this + mapping: + + complex32 -> float16 + complex64 -> float32 + complex128 -> float64 + + Note that COMPLEX_TO_FLOAT derives the computation dtype as the DEFAULT setting does. + + The BOOL_TO_LONG type promotion kind maps boolean computation and result dtypes to long. + + The ALWAYS_BOOL type promotion kind always sets the result dtype to bool. + + Example operators for each type promotion option: + DEFAULT : add + NO_OPMATH : where, nextafter, cat + INT_TO_FLOAT : sin + COMPLEX_TO_FLOAT : abs + BOOL_TO_LONG : pow + ALWAYS_BOOL : eq + + """ + + args = tuple(x for x in _args if x is not None) + + highest_type: type = bool + + # Import sympy locally, as importing it eagerly at a module level is too slow + # See https://dev-discuss.pytorch.org/t/delving-into-what-happens-when-you-import-torch/1589 + import sympy + + for x in args: + if not isinstance(x, (Number, TensorLike, sympy.Basic)): + msg = f"Unexpected type {str(type(x))} when computing elementwise type promotion!" + raise ValueError(msg) + + if isinstance(x, Number): + highest_type = get_higher_type(highest_type, number_type(x)) + elif isinstance(x, sympy.Basic): + highest_type = get_higher_type(highest_type, expr_type(x)) + else: + # x is a TensorLike + highest_type = get_higher_type(highest_type, dtype_to_type(x.dtype)) + + result_dtype = None + + def _find_highest_dtype_filtered( + args, filter, *, float_as_complex=False + ) -> Optional[torch.dtype]: + zero_dim_tensor_dtype = None + one_plus_dim_tensor_dtype = None + for x in args: + if isinstance(x, TensorLike) and filter(x.dtype): + _dtype = x.dtype + if float_as_complex and is_float_dtype(_dtype): + _dtype = corresponding_complex_dtype(_dtype) + if x.ndim == 0: + zero_dim_tensor_dtype = get_higher_dtype( + zero_dim_tensor_dtype, _dtype + ) + else: + # x.ndim > 0 + one_plus_dim_tensor_dtype = get_higher_dtype( + one_plus_dim_tensor_dtype, _dtype + ) + + # Prefers dtype of tensors with one or more dimensions + if one_plus_dim_tensor_dtype is not None: + # pyrefly: ignore [bad-return] + return one_plus_dim_tensor_dtype + + # pyrefly: ignore [bad-return] + return zero_dim_tensor_dtype + + if highest_type is float: + result_dtype = _find_highest_dtype_filtered(args, is_float_dtype) + result_dtype = ( + torch.get_default_dtype() if result_dtype is None else result_dtype + ) + elif highest_type is complex: + result_dtype = _find_highest_dtype_filtered( + args, + lambda x: is_float_dtype(x) or is_complex_dtype(x), + float_as_complex=True, + ) + if result_dtype is None: + result_dtype = corresponding_complex_dtype(torch.get_default_dtype()) + elif highest_type is int: + result_dtype = _find_highest_dtype_filtered(args, is_integer_dtype) + result_dtype = torch.long if result_dtype is None else result_dtype + else: + # highest_type is bool + result_dtype = torch.bool + + if type_promotion_kind is ELEMENTWISE_TYPE_PROMOTION_KIND.DEFAULT: + return get_computation_dtype(result_dtype), result_dtype + elif type_promotion_kind is ELEMENTWISE_TYPE_PROMOTION_KIND.NO_OPMATH: + return result_dtype, result_dtype + elif type_promotion_kind is ELEMENTWISE_TYPE_PROMOTION_KIND.INT_TO_FLOAT: + if is_integer_dtype(result_dtype) or is_boolean_dtype(result_dtype): + result_dtype = torch.get_default_dtype() + return get_computation_dtype(result_dtype), result_dtype + elif type_promotion_kind is ELEMENTWISE_TYPE_PROMOTION_KIND.COMPLEX_TO_FLOAT: + # NOTE: computation can still occur in a complex dtype + computation_dtype = get_computation_dtype(result_dtype) + if is_complex_dtype(result_dtype): + result_dtype = corresponding_real_dtype(result_dtype) + return computation_dtype, result_dtype + elif type_promotion_kind is ELEMENTWISE_TYPE_PROMOTION_KIND.BOOL_TO_LONG: + if is_boolean_dtype(result_dtype): + return torch.long, torch.long + return get_computation_dtype(result_dtype), result_dtype + elif type_promotion_kind is ELEMENTWISE_TYPE_PROMOTION_KIND.ALWAYS_BOOL: + return get_computation_dtype(result_dtype), torch.bool + else: + raise ValueError(f"Unknown type promotion kind {str(type_promotion_kind)}") + + +def reduction_dtypes( + arg, + output_dtype_kind: REDUCTION_OUTPUT_TYPE_KIND, + dtype: Optional[torch.dtype] = None, +) -> tuple[torch.dtype, Optional[torch.dtype]]: + # even though some reductions, like amin or amax, don't strictly require type promotion, + # all the math ops (including comparisons) are still defined only for a computation type, + # so promotion will still happen. We are doing it explicitly here + inp_dtype = dtype if dtype is not None else arg.dtype + computation_dtype = get_computation_dtype(inp_dtype) + if ( + output_dtype_kind == REDUCTION_OUTPUT_TYPE_KIND.SAME + or output_dtype_kind == REDUCTION_OUTPUT_TYPE_KIND.COMPLEX_TO_FLOAT + ): + result_dtype = dtype if dtype else arg.dtype + if ( + output_dtype_kind == REDUCTION_OUTPUT_TYPE_KIND.COMPLEX_TO_FLOAT + and is_complex_dtype(result_dtype) + ): + result_dtype = corresponding_real_dtype(result_dtype) + elif output_dtype_kind == REDUCTION_OUTPUT_TYPE_KIND.KEEP_PROMOTED_TYPE: + result_dtype = None + else: # ALWAYS_BOOL + result_dtype = torch.bool + return computation_dtype, result_dtype + + +# This function's logic is borrowed from the following functions defined in C++: +# batched_matrix_contiguous_strides and contiguous_strides +def make_contiguous_strides_for( + shape: ShapeType, row_major: bool = True +) -> tuple[Union[_IntLikeT, int], ...]: + """ + Returns the strides of a contiguous tensor if row_major + If row_major=True, it returns the strides of a contiguous batch of Fortran-contiguous matrices + This is often used when calling external libraries like BLAS/LAPACK/cuSolver... + """ + # contiguous_strides from c10/util/strides.h + validate_shape(shape) + if not shape: + return () + + from torch.fx.experimental.symbolic_shapes import is_nested_int + + multiplier: Union[_IntLikeT, int] = 1 + strides = [] + for l in reversed(shape): + strides.append(multiplier) + multiplier *= l if is_nested_int(l) else sym_max(l, 1) # type:ignore[assignment] + + result = tuple(reversed(strides)) + + # batched_matrix_contiguous_strides from aten/src/ATen/native/LinearAlgebraUtils.h + if row_major: + return result + else: + if len(shape) < 2: + return result + return result[:-2] + (1, max(shape[-2], 1)) + + +def make_channels_last_1d_strides_for( + shape: Sequence[_IntLikeT], +) -> tuple[Union[_IntLikeT, int], ...]: + torch._check( + len(shape) == 3, + lambda: "Only tensors of rank 3 can use the channels_last_1d memory format", + ) + + multiplier: Union[_IntLikeT, int] = 1 + strides: list[Union[_IntLikeT, int]] = [0] * 3 + for idx in (1, -1, 0): + # NOTE: intentionally divergence from make_contiguous_strides_for + # This is consistent with eager + strides[idx] = multiplier + multiplier *= shape[idx] + + return tuple(strides) + + +def make_channels_last_2d_strides_for( + shape: Sequence[_IntLikeT], +) -> tuple[Union[_IntLikeT, int], ...]: + # TODO: maybe inform the user of channels_last_3d if rank of the tensor is 5? + torch._check( + len(shape) == 4, + lambda: "Only tensors of rank 4 can use the channels_last memory format", + ) + + multiplier: Union[_IntLikeT, int] = 1 + strides: list[Union[_IntLikeT, int]] = [0] * 4 + for idx in (1, -1, -2, 0): + # NOTE: intentionally divergence from make_contiguous_strides_for + # This is consistent with eager + strides[idx] = multiplier + multiplier *= shape[idx] + + return tuple(strides) + + +def make_channels_last_3d_strides_for( + shape: Sequence[_IntLikeT], +) -> tuple[Union[_IntLikeT, int], ...]: + torch._check( + len(shape) == 5, + lambda: "Only tensors of rank 5 can use the channels_last_3d memory format", + ) + + multiplier: Union[_IntLikeT, int] = 1 + strides: list[Union[_IntLikeT, int]] = [0] * 5 + for idx in (1, -1, -2, -3, 0): + # NOTE: intentionally divergence from make_contiguous_strides_for + # This is consistent with eager + strides[idx] = multiplier + multiplier *= shape[idx] + + return tuple(strides) + + +def make_channels_last_strides_for( + shape: Sequence[_IntLikeT], +) -> tuple[Union[_IntLikeT, int], ...]: + ndim = len(shape) if isinstance(shape, Sequence) else 1 + if ndim == 3: + return make_channels_last_1d_strides_for(shape) + elif ndim == 4: + return make_channels_last_2d_strides_for(shape) + elif ndim == 5: + return make_channels_last_3d_strides_for(shape) + else: + raise RuntimeError( + f"no channels last format strides exist in {ndim} dimensions" + ) + + +def compute_reduction_output_shape( + shape: ShapeType, dimensions: Sequence +) -> tuple[int, ...]: + for idx in dimensions: + validate_idx(len(shape), idx) + + new_shape = [] + for idx in range(len(shape)): + if idx in dimensions: + continue + + new_shape.append(shape[idx]) + + return tuple(new_shape) + + +def validate_no_repeating_dims(dims: Sequence): + if len(dims) != len(set(dims)): + raise RuntimeError("duplicate value in the list of dims") + + +def reduction_dims(shape: ShapeType, dims: Optional[Sequence]) -> tuple[int, ...]: + if dims is None: + return tuple(range(len(shape))) + dims = tuple(canonicalize_dim(len(shape), idx) for idx in dims) + validate_no_repeating_dims(dims) + return dims + + +def set_correction( + unbiased: Optional[bool] = None, + correction: Optional[NumberType] = None, +) -> float: + if correction is not None and unbiased is not None: + raise RuntimeError("cannot specify both correction and unbiased arguments") + elif correction is None and unbiased is None: + correction = 1.0 + elif correction is None and unbiased is not None: + correction = 0.0 if unbiased is False else 1.0 + # NB: we don't actually support symint here, but it's harmless to accept + if not isinstance(correction, (IntLike, FloatLike)): + raise ValueError("correction argument should be integer or float") + return sym_float(correction) + + +def compute_required_storage_length( + shape: ShapeType, strides: StrideType, storage_offset: int +) -> int: + """Computes the minimum storage size to hold the given tensor geometry. + + Example + ======= + + This is the size of a newly allocated tensor's storage, in units of elements + + >>> t = torch.empty((10, 20)) + >>> compute_required_storage_length(t.shape, t.stride(), t.storage_offset()) + 200 + + >>> # xdoctest: +SKIP(failing) + >>> t2 = torch.empty_strided((1, 2, 3), (5, 7, 11)) + >>> size = compute_required_storage_length( + ... t2.shape, t2.stride(), t2.storage_offset() + ... ) + >>> size == t.storage().size() + True + + A valid tensor may have a larger storage size, but never smaller + + >>> slice = torch.empty(100)[20:40] + >>> slice.storage().size() + 100 + + >>> compute_required_storage_length( + ... slice.shape, slice.stride(), slice.storage_offset() + ... ) + 40 + + """ + from torch.fx.experimental.symbolic_shapes import guard_or_false + + # Short-circuits if the shape has no elements + # Note: we are unsafely assuming tensor is not empty here, without + # runtime assertions. + if guard_or_false(reduce(operator.mul, shape, 1) == 0): + return 0 + + max_offset = sum((x - 1) * y for x, y in zip(shape, strides)) + # +1 to account for the first element which offsets are taken from + return 1 + storage_offset + max_offset + + +def check_in_bounds_for_storage( + a: torch.TypedStorage, shape: ShapeType, strides: StrideType, storage_offset: int +): + """ + Determines if the given shape, strides, and offset are valid for the given storage. + """ + + required_length = compute_required_storage_length(shape, strides, storage_offset) + if a.size() < required_length: + msg = ( + f"Can't view a storage of size {a.size()} with an offset of {storage_offset}, " + f"shape of {str(shape)}, and strides of {str(strides)}, " + f"which requires a storage of size {required_length}" + ) + raise ValueError(msg) + + +# NOTE: This function should ideally be removed, but some Meta internal models +# packaged with `torch.package` are using it, so it will have to be removed +# at some point in the future when those models no longer use this function. +@deprecated( + "`torch._prims_common.check` is deprecated and will be removed in the future. " + "Please use `torch._check*` functions instead.", + category=FutureWarning, +) +def check( + b: bool, s: Callable[[], str], exc_type: type[Exception] = RuntimeError +) -> None: + """ + Helper function for raising an error_type (default: RuntimeError) if a boolean condition fails. + Error message is a callable producing a string (to avoid wasting time + string formatting in non-error case, and also to make it easier for torchdynamo + to trace.) + + .. note:: This function is planned for removal in the future. Please use + `torch._check*` functions instead. + """ + torch._check_with(exc_type, b, s) + + +# This combines is_channels_last_strides_2d and is_channels_last_strides_3d in +# c10/core/MemoryFormat.h into one function +# May return False when input sizes are data-dependent and the property is not +# determined. +def are_strides_like_channels_last_or_false( + shape: Sequence[int], strides: Sequence[int] +) -> bool: + from torch.fx.experimental.symbolic_shapes import ( + guard_or_true, + statically_known_true, + ) + + ndim = len(shape) + + if ndim == 4: + # Check for channels_last_2d + dim_order = [1, 3, 2, 0] + elif ndim == 5: + # Check for channels_last_3d + dim_order = [1, 4, 3, 2, 0] + else: + return False + + if guard_or_true(strides[1] == 0): + return False + + min = 0 + for d in dim_order: + if guard_or_true(shape[d] == 0): + return False + if guard_or_true(strides[d] < min): + return False + if d == 0 and min == strides[1]: + return False + min = strides[d] + # Assume stride is not 1, the consequence is min could be larger than needed, + # which would result in returning False for this function but not vice versa, + # so it's ok. + if guard_or_true(strides[d] > 1): + min *= shape[d] + return True + + +def suggest_memory_format(x: TensorLikeType) -> torch.memory_format: + if x.layout != torch.strided: + return torch.contiguous_format + + if are_strides_like_channels_last_or_false(x.shape, x.stride()): + return torch.channels_last if x.ndim == 4 else torch.channels_last_3d + + return torch.contiguous_format + + +def prod(xs: Sequence[NumberType]) -> NumberType: + """Product of elements in input sequence. Returns 1 for empty sequence""" + return reduce(operator.mul, xs, 1) + + +def is_expandable_to(shape: ShapeType, desired: ShapeType) -> bool: + """Checks if a shape can be expanded to another shape. + This is equivalent to checking if the two shapes are broadcastable. + """ + # This is a Python implementation of + # aten/src/ATen/ExpandUtils.h:is_expandable_to + if len(shape) > len(desired): + return False + for i in range(len(shape)): + if shape[-i - 1] != desired[-i - 1] and shape[-i - 1] != 1: + return False + return True + + +def mask_tensor(mask: TensorLikeType, t: TensorLikeType): + """ + Similar to torch.where(mask, t, 0) but if t is boolean, + result is also boolean and not promoted to int. + """ + # torch.where(mask, t, False) is equivalent + # but feels hacky and might break in the future + if t.dtype is torch.bool: + return mask.logical_and(t) + else: + return torch.where(mask, t, 0) + + +def get_aten_op(fn: Callable, name: str): + """ + Given the __module__ of reference and its name, it returns + (our best guess of) the ATen name of the associated operation + + Note: In ATen, the __name__ of a function within a module often + starts by the module name. E.g. linalg_eigh, or special_zeta + """ + module = fn.__module__ + prefix = "torch._refs" + assert module.startswith(prefix) + module = module[len(prefix) :] + # We want to go from .special / .nn.functional + # to special and special_ / nn_functional_ + if module: + module = module[1:] + module = module.replace(".", "_") + module = module + "_" + return getattr(torch._ops.ops.aten, f"{module}{name}") + + +def dtype_or_default(dtype: Optional[torch.dtype]) -> torch.dtype: + return dtype if dtype is not None else torch.get_default_dtype() + + +def device_or_default(device: Optional[DeviceLikeType]) -> DeviceLikeType: + return device if device is not None else torch.device("cpu") + + +def layout_or_default(layout: Optional[torch.layout]) -> torch.layout: + return layout if layout is not None else torch.strided + + +def clone_preserve_strides(x): + needed_size = compute_required_storage_length( + x.size(), x.stride(), x.storage_offset() + ) + # Our eager implementations for *_scatter ops are all primitives w.r.t autograd, + # so these as_strided() calls are not seen by autograd. + # We need to mimic this behavior in our ref/prim implementations. + # TODO: a better way to handle this would be with a new op, "_unsafe_as_strided" + # We should revisit this when we add a compositional as_strided op, + # and also as part of https://github.com/pytorch/pytorch/issues/90507 + try: + old = torch._C._dispatch_tls_is_dispatch_key_excluded( + torch._C.DispatchKey.ADInplaceOrView + ) + torch._C._dispatch_tls_set_dispatch_key_excluded( + torch._C.DispatchKey.ADInplaceOrView, True + ) + buffer = torch.as_strided(x, (needed_size,), (1,), 0).clone() + return torch.as_strided(buffer, x.size(), x.stride(), x.storage_offset()) + finally: + torch._C._dispatch_tls_set_dispatch_key_excluded( + torch._C.DispatchKey.ADInplaceOrView, old + ) + + +def alert_not_deterministic(caller: str): + if torch.are_deterministic_algorithms_enabled(): + if torch.is_deterministic_algorithms_warn_only_enabled(): + warnings.warn( + f"{caller} does not have a deterministic implementation, but you set " + f"'torch.use_deterministic_algorithms(True, warn_only=True)'. " + f"You can file an issue at https://github.com/pytorch/pytorch/issues " + f"to help us prioritize adding deterministic support for this operation.", + stacklevel=2, + ) + else: + torch._check( + False, + lambda: ( + f"{caller} does not have a deterministic implementation, but you set " + f"'torch.use_deterministic_algorithms(True)'. You can turn off " + f"determinism just for this operation, or you can use the " + f"'warn_only=True' option, if that's acceptable for your application. " + f"You can also file an issue at https://github.com/pytorch/pytorch/issues " + f"to help us prioritize adding deterministic support for this operation." + ), + ) + + +class CUDARngStateHelper: + @staticmethod + def get_torch_state_as_tuple( + fake_mode: AbstractContextManager[Any] = nullcontext(), + ): + if not torch.cuda.is_available(): + raise RuntimeError("CUDA not available") + + with fake_mode: + seed = torch.tensor(torch.cuda.initial_seed()) + offset = torch.tensor(torch.cuda._get_rng_state_offset()) + return seed, offset + + @staticmethod + def set_torch_state_tensor(seed, offset): + # Rng state is [64-bit seed, 64-bit offset] + seed_portion = seed.reshape([1]).view(torch.uint8) + offset_portion = offset.reshape([1]).view(torch.uint8) + new_state = torch.cat([seed_portion, offset_portion]) + torch.cuda.set_rng_state(new_state) + + @staticmethod + def set_new_offset(relative_offset): + torch.cuda._set_rng_state_offset(relative_offset.item()) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims_common/wrappers.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims_common/wrappers.py new file mode 100644 index 0000000000000000000000000000000000000000..e369481c1044b3c5e25ffa46edabd555aa73b798 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_prims_common/wrappers.py @@ -0,0 +1,497 @@ +# mypy: allow-untyped-defs +import inspect +import types +import warnings +from collections.abc import Callable, Sequence +from functools import wraps +from types import GenericAlias +from typing import NamedTuple, Optional, overload, TypeVar, Union +from typing_extensions import ParamSpec + +import torch +import torch._prims_common as utils +from torch._prims_common import ( + CustomOutParamAnnotation, + ELEMENTWISE_TYPE_PROMOTION_KIND, + Number, + NumberType, + ShapeType, + TensorLike, + TensorLikeType, +) +from torch.utils import _pytree as pytree +from torch.utils._pytree import tree_flatten, tree_unflatten + + +_T = TypeVar("_T") +_P = ParamSpec("_P") + + +@overload +# pyrefly: ignore [bad-return] +def _maybe_convert_to_dtype(a: TensorLikeType, dtype: torch.dtype) -> TensorLikeType: + pass + + +@overload +# pyrefly: ignore [bad-return] +def _maybe_convert_to_dtype(a: NumberType, dtype: torch.dtype) -> NumberType: + pass + + +@overload +# pyrefly: ignore [bad-return] +def _maybe_convert_to_dtype(a: Sequence, dtype: torch.dtype) -> Sequence: + pass + + +@overload +def _maybe_convert_to_dtype(a: None, dtype: torch.dtype) -> None: + pass + + +# TODO: implement ref.cast with an option to enforce safe casting +def _maybe_convert_to_dtype(a, dtype): + if isinstance(a, TensorLike): + if a.dtype != dtype: + return a.to(dtype) + return a + if isinstance(a, Number): + return utils.dtype_to_type_ctor(dtype)(a) # type: ignore[arg-type] + if isinstance(a, Sequence): + return tuple(_maybe_convert_to_dtype(x, dtype) for x in a) + # Passthrough None because some functions wrapped with type promotion + # wrapper might have optional args + if a is None: + return None + + raise ValueError( + f"Received unsupported type {type(a)}. Expected TensorLike, Number, or Sequence." + ) + + +def _maybe_convert_to_type(a: NumberType, typ: type) -> NumberType: + if not isinstance(a, Number): + msg = f"Found unknown type {type(a)} when trying to convert scalars!" + raise ValueError(msg) + if not utils.is_weakly_lesser_type(type(a), typ): + msg = f"Scalar {a} of type {type(a)} cannot be safely cast to type {typ}!" + raise ValueError(msg) + + return typ(a) + + +def _annotation_has_type(*, typ, annotation): + if hasattr(annotation, "__args__"): + for a in annotation.__args__: + if _annotation_has_type(typ=typ, annotation=a): + return True + return False + + return typ is annotation + + +class elementwise_type_promotion_wrapper: + """ + Adds elementwise type promotion to a Python reference implementation. + + Takes two kwargs, type_promoting_args and type_promotion_kind. + + type_promoting_args must be a string Sequence specifying the argument names of all + arguments that participate in type promotion (and should be type promoted). If the + arg specifies a Sequence-type then every element of the Sequence will participate in + type promotion. + + type_promotion_kind must be one of the kinds specified by ELEMENTWISE_TYPE_PROMOTION_KIND. + See its documentation for details. + + The return_dtype will be coerced to the wrapped function's dtype arg if it is available and + not None. + + Other type promotion behavior, like validating the Python type of scalar arguments, must + be handled separately. + """ + + def __init__( + self, + *, + type_promotion_kind: ELEMENTWISE_TYPE_PROMOTION_KIND, + type_promoting_args: Optional[Sequence[str]] = None, + ): + self.type_promoting_arg_names = type_promoting_args + self.type_promotion_kind = type_promotion_kind + + def __call__(self, fn: Callable) -> Callable: + sig = inspect.signature(fn) + + # TorchDynamo tracing of inspect causes fake tensor dynamo_wrapped tests to fail + # PYTORCH_TEST_WITH_DYNAMO=1 python test/test_fake_tensor.py FakeTensorTest.test_basic + @torch._disable_dynamo + @wraps(fn) + def _fn(*args, **kwargs): + bound = sig.bind(*args, **kwargs) + type_promoting_args = tuple( + bound.arguments[x] + for x in self.type_promoting_arg_names # type: ignore[union-attr] + if x in bound.arguments + ) + + flattened_type_promoting_args = pytree.arg_tree_leaves(*type_promoting_args) + compute_dtype, result_dtype = utils.elementwise_dtypes( + *flattened_type_promoting_args, + type_promotion_kind=self.type_promotion_kind, + ) + + promoted_args = { + x: _maybe_convert_to_dtype(bound.arguments[x], compute_dtype) + for x in self.type_promoting_arg_names # type: ignore[union-attr] + if x in bound.arguments + } + bound.arguments.update(promoted_args) + + result = fn(**bound.arguments) + + # Override the return_dtype if a dtype arg is present and not None + if "dtype" in bound.arguments: + maybe_dtype = bound.arguments["dtype"] + if maybe_dtype: # dtype cannot be None + result_dtype = maybe_dtype + + if isinstance(result, TensorLike): + return _maybe_convert_to_dtype(result, result_dtype) + if isinstance(result, Sequence): + return tuple(_maybe_convert_to_dtype(x, result_dtype) for x in result) + raise AssertionError(f"Unhandled result type: {type(result)}") + + _fn.__signature__ = sig # type: ignore[attr-defined] + return _fn + + +# Returns True if resize is necessary +def _resize_output_check(out: TensorLikeType, shape: ShapeType): + # If the shapes are correct there's nothing to do + if utils.same_shape(out.shape, shape): + return False + if out.numel() != 0: + msg = ( + f"An output with one or more elements was resized since it had shape {str(out.shape)} " + "which does not match the required output shape {str(shape)}. " + "This behavior is deprecated, and in a future PyTorch release outputs will not " + "be resized unless they have zero elements. " + "You can explicitly reuse an out tensor t by resizing it, inplace, to zero elements with t.resize_(0)." + ) + warnings.warn(msg, stacklevel=2) + return True + + +# TODO: handle tuples of tensors +def _maybe_resize_out( + out: TensorLikeType, + shape: ShapeType, + memory_format: Optional[torch.memory_format] = None, +): + if _resize_output_check(out, shape): + return out.resize_(shape, memory_format=memory_format) + else: + return out + + +def is_cpu_scalar(x: TensorLikeType) -> bool: + return x.dim() == 0 and x.device.type == "cpu" + + +def check_copy_devices(*, copy_from: TensorLikeType, copy_to: TensorLikeType) -> None: + if copy_from.device != copy_to.device: + msg = ( + f"Attempting to copy from device {copy_from.device} " + f"to device {copy_to.device}, but cross-device copies are not allowed!" + ) + raise RuntimeError(msg) + + +def _safe_copy_out( + *, copy_from: TensorLikeType, copy_to: TensorLikeType, exact_dtype: bool = False +): + # Checks same device + if not is_cpu_scalar(copy_from): + check_copy_devices(copy_from=copy_from, copy_to=copy_to) + + # Checks safe cast + if exact_dtype: + torch._check( + copy_from.dtype == copy_to.dtype, + lambda: f"Expected out tensor to have dtype {copy_from.dtype} " + f"but got {copy_to.dtype} instead", + ) + else: + torch._check( + utils.can_safe_cast_to(cast_from=copy_from.dtype, cast_to=copy_to.dtype), + lambda: f"Attempting to cast from {copy_from.dtype} to out tensor with dtype {copy_to.dtype}, " + "but this can't be cast because it is not safe!", + ) + + return copy_to.copy_(copy_from) + + +def out_wrapper( + *out_names: str, + exact_dtype: bool = False, + pass_is_out: bool = False, + preserve_memory_format: bool = False, +) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: + # The wrapped function needs to convert the output parameters to ensure + # compatibility between the Python API (which always uses "out" as the + # parameter name and may be a tuple) and the Aten API (which may have + # multiple output parameters and use different parameter names such as + # "grad_input", "indices" or "values".) + + default_out_names = ("out",) + if len(out_names) == 0: + # Use default in out name + out_names = default_out_names + + is_tensor = len(out_names) == 1 + + def maybe_compute_memory_format(t): + return utils.suggest_memory_format(t) if preserve_memory_format else None + + def _out_wrapper(fn: Callable[_P, _T]) -> Callable[_P, _T]: + """ + Adds the out parameter to a Python reference. + """ + out_type = ( + TensorLikeType + if is_tensor + else GenericAlias( + tuple, tuple(TensorLikeType for _ in range(len(out_names))) + ) + ) + # For backward compatibility - should be able to remove once PEP585 + # conversion is complete. + bc_out_type = ( + TensorLikeType + if is_tensor + else types.GenericAlias( + tuple, tuple(TensorLikeType for _ in range(len(out_names))) + ) + ) + return_type = ( + TensorLikeType + if is_tensor + else NamedTuple( + f"return_types_{fn.__name__}", + # pyrefly: ignore [bad-argument-count] + [(o, TensorLikeType) for o in out_names], + ) + ) + + sig = inspect.signature(fn) + factory_kwargs = ("device", "dtype") + is_factory_fn = all(p in sig.parameters for p in factory_kwargs) + + @wraps(fn) + def _fn(*args: _P.args, **kwargs: _P.kwargs): + out = kwargs.pop("out", None) + if is_factory_fn and out is not None: + for k in factory_kwargs: + out_attr = getattr(out, k) + if k not in kwargs: + kwargs[k] = out_attr + + def maybe_check_copy_devices(out): + # pyrefly: ignore [unsupported-operation] + if isinstance(out, TensorLike) and isinstance(args[0], TensorLike): + check_copy_devices(copy_from=args[0], copy_to=out) + + if isinstance(out, (tuple, list)): + for o in out: + maybe_check_copy_devices(o) + else: + maybe_check_copy_devices(out) + + if pass_is_out: + result = fn(*args, is_out=(out is not None), **kwargs) # type: ignore[arg-type] + else: + result = fn(*args, **kwargs) + if result is NotImplemented: + return NotImplemented + assert ( + (isinstance(result, TensorLike) and is_tensor) + or ( + isinstance(result, tuple) # type: ignore[arg-type] + and len(result) == len(out_names) # type: ignore[arg-type] + ) + or ( + fn.__name__ == "unbind" and isinstance(result, (list, tuple)) # type: ignore[arg-type] + ) + ) + # unbind_copy is a special case: see https://github.com/pytorch/pytorch/issues/130829 + if out is not None: + # Naively you might expect this assert to be true, but + # it's not: + # + # assert type(out) is type(result) + # + # The reason is that functions under this wrapper can + # get registered to the Meta dispatch key, and that + # means they can be executed in a context where tensor + # subclasses are disabled (with no_dispatch), which is a + # handy way for an is-a tensor subclass (e.g., + # FakeTensor) to have the normal meta backend create a + # meta tensor, to be wrapped once it gets returned. + # In this situation, you will get a FakeTensor as + # the output tensor, but not the result--which will + # be a normal meta tensor, but this is perfectly + # harmless. + if is_tensor and fn.__name__ != "unbind": + assert isinstance(out, TensorLike) + # These two operations are done in-place + _maybe_resize_out( + out, + result.shape, # type: ignore[union-attr] + maybe_compute_memory_format(result), + ) + _safe_copy_out( + copy_from=result, # type: ignore[arg-type] + copy_to=out, + exact_dtype=exact_dtype, + ) + else: + if fn.__name__ != "unbind": + assert isinstance(out, tuple) # type: ignore[arg-type] + else: + assert isinstance(out, (list, tuple)) # type: ignore[arg-type] + torch._check_type( + len(out) == len(result), # type: ignore[arg-type] + lambda: f"expected tuple of {len(result)} elements but got {len(out)}", # type: ignore[arg-type] + ) + for r, o in zip(result, out): # type: ignore[arg-type] + # These two operations are done in-place + _maybe_resize_out(o, r.shape, maybe_compute_memory_format(r)) + _safe_copy_out(copy_from=r, copy_to=o, exact_dtype=exact_dtype) # type: ignore[arg-type] + else: + out = result + # mypy does not see through the definition of out_type given that it's in a different scope + return out if is_tensor else return_type(*out) # type: ignore[operator] + + out_param = inspect.Parameter( + "out", + kind=inspect.Parameter.KEYWORD_ONLY, + default=None, + annotation=out_type, + ) + # Mark that the function now returns a tuple + assert isinstance( + sig.return_annotation, (str, TypeVar) + ) or sig.return_annotation in ( + sig.empty, + out_type, + bc_out_type, + ) + params = *sig.parameters.values(), out_param + + # If there's a Parameter.VAR_KEYWORD parameter (like **kwds), it must appear + # after the out= parameter, which is Parameter.KEYWORD_ONLY. Sorting by + # Parameter.kind guarantees that all the parameters are in legal order. + params = sorted(params, key=lambda p: p.kind) + + _fn.__signature__ = inspect.Signature( # type: ignore[attr-defined] + parameters=params, + return_annotation=return_type, # type: ignore[arg-type] + ) + + _fn.__annotations__ = dict(getattr(fn, "__annotations__", {})) + _fn.__annotations__["out"] = out_type + _fn.__annotations__["return"] = return_type + + # In the special case of having a single tensor out parameter with a + # name other than out, add a special annotation to name the parameter + if is_tensor and out_names != default_out_names: + _fn.__annotations__[CustomOutParamAnnotation] = out_names[0] + + # Add an indicator attribute that can be used in special cases + # where having a function wrapped by `out_wrapper` is not desirable e.g. + # jit + _fn._torch_decompositions_out_wrapper = ( # type: ignore[attr-defined] + f"This function is wrapped by {out_wrapper.__module__}.out_wrapper" + ) + + return _fn + + return _out_wrapper + + +def _maybe_remove_out_wrapper(fn: Callable): + return inspect.unwrap( + fn, + stop=lambda f: not hasattr(f, "_torch_decompositions_out_wrapper"), + ) + + +def backwards_not_supported(prim): + def redispatch_prim(args, kwargs): + with torch._C._AutoDispatchBelowAutograd(): + return prim(*args, **kwargs) + + class BackwardsNotSupported(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(ctx, args_spec, *flat_args): + args, kwargs = tree_unflatten(flat_args, args_spec) # type: ignore[arg-type] + return redispatch_prim(args, kwargs) + + @staticmethod + def backward(ctx, *args): + raise RuntimeError("backwards not supported on prim") + + @wraps(prim) + def _autograd_impl(*args, **kwargs): + flat_args, args_spec = tree_flatten((args, kwargs)) + if torch.is_grad_enabled() and any( + a.requires_grad for a in flat_args if isinstance(a, torch.Tensor) + ): + # TODO: There is a subtle bug here: prims like copy_to + # return their input argument after mutating it; and custom + # autograd function will incorrectly turn the result into + # a view which will fail test_python_ref_executor tests. + # At the moment, we sidestep this by observing that the + # unit tests don't ever try to run the executor with + # autograd, so we don't exercise the buggy case, but if + # you ever want to feed autograd through this, be aware + # of it! We need a way of properly implementing autograd + # for mutating operations in Python to do this. + return BackwardsNotSupported.apply(args_spec, *flat_args) + else: + return redispatch_prim(args, kwargs) + + return _autograd_impl + + +# TODO: when tracing this will add torch tensors and not TensorMeta objects +# to the trace -- we should fix this by adding a tracing context and NumberMeta classes +# TODO: this wrapper is currently untested +def elementwise_unary_scalar_wrapper( + fn: Callable[_P, _T], +) -> Callable[_P, Union[_T, NumberType]]: + """ + Allows unary operators that accept tensors to work with Python numbers. + """ + sig = inspect.signature(fn) + + @wraps(fn) + def _fn(*args, **kwargs): + if len(args) > 0 and isinstance(args[0], Number): + dtype = utils.type_to_dtype(type(args[0])) + args_ = list(args) + args_[0] = torch.tensor(args[0], dtype=dtype) + # pyrefly: ignore [invalid-param-spec] + result = fn(*args_, **kwargs) + assert isinstance(result, torch.Tensor) + return result.item() + + # pyrefly: ignore [invalid-param-spec] + return fn(*args, **kwargs) + + _fn.__signature__ = sig # type: ignore[attr-defined] + # pyrefly: ignore [bad-return] + return _fn diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_vendor/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/_vendor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/accelerator/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/accelerator/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b0dfbe400bfbcb1d0eed42b2ab16fdfa1f42ada6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/accelerator/__init__.py @@ -0,0 +1,299 @@ +r""" +This package introduces support for the current :ref:`accelerator` in python. +""" + +from functools import cache +from typing import Any +from typing_extensions import deprecated + +import torch + +from ._utils import _device_t, _get_device_index +from .memory import ( + empty_cache, + get_memory_info, + max_memory_allocated, + max_memory_reserved, + memory_allocated, + memory_reserved, + memory_stats, + reset_accumulated_memory_stats, + reset_peak_memory_stats, +) + + +__all__ = [ + "current_accelerator", + "current_device_idx", # deprecated + "current_device_index", + "get_device_capability", + "current_stream", + "device_count", + "device_index", + "empty_cache", + "get_memory_info", + "is_available", + "max_memory_allocated", + "max_memory_reserved", + "memory_allocated", + "memory_reserved", + "memory_stats", + "reset_accumulated_memory_stats", + "reset_peak_memory_stats", + "set_device_idx", # deprecated + "set_device_index", + "set_stream", + "synchronize", +] + + +def device_count() -> int: + r"""Return the number of current :ref:`accelerator` available. + + Returns: + int: the number of the current :ref:`accelerator` available. + If there is no available accelerators, return 0. + + .. note:: This API delegates to the device-specific version of `device_count`. + On CUDA, this API will NOT poison fork if NVML discovery succeeds. + Otherwise, it will. For more details, see :ref:`multiprocessing-poison-fork-note`. + """ + acc = current_accelerator() + if acc is None: + return 0 + + mod = torch.get_device_module(acc) + return mod.device_count() + + +def is_available() -> bool: + r"""Check if the current accelerator is available at runtime: it was build, all the + required drivers are available and at least one device is visible. + See :ref:`accelerator` for details. + + Returns: + bool: A boolean indicating if there is an available :ref:`accelerator`. + + .. note:: This API delegates to the device-specific version of `is_available`. + On CUDA, when the environment variable ``PYTORCH_NVML_BASED_CUDA_CHECK=1`` is set, + this function will NOT poison fork. Otherwise, it will. For more details, see + :ref:`multiprocessing-poison-fork-note`. + + Example:: + + >>> assert torch.accelerator.is_available() "No available accelerators detected." + """ + # Why not just check "device_count() > 0" like other is_available call? + # Because device like CUDA have a python implementation of is_available that is + # non-poisoning and some features like Dataloader rely on it. + # So we are careful to delegate to the Python version of the accelerator here + acc = current_accelerator() + if acc is None: + return False + + mod = torch.get_device_module(acc) + return mod.is_available() + + +def current_accelerator(check_available: bool = False) -> torch.device | None: + r"""Return the device of the accelerator available at compilation time. + If no accelerator were available at compilation time, returns None. + See :ref:`accelerator` for details. + + Args: + check_available (bool, optional): if True, will also do a runtime check to see + if the device :func:`torch.accelerator.is_available` on top of the compile-time + check. + Default: ``False`` + + Returns: + torch.device: return the current accelerator as :class:`torch.device`. + + .. note:: The index of the returned :class:`torch.device` will be ``None``, please use + :func:`torch.accelerator.current_device_index` to know the current index being used. + This API does NOT poison fork. For more details, see :ref:`multiprocessing-poison-fork-note`. + + Example:: + + >>> # xdoctest: + >>> # If an accelerator is available, sent the model to it + >>> model = torch.nn.Linear(2, 2) + >>> if (current_device := current_accelerator(check_available=True)) is not None: + >>> model.to(current_device) + """ + if (acc := torch._C._accelerator_getAccelerator()) is not None: + if (not check_available) or (check_available and is_available()): + return acc + return None + + +def current_device_index() -> int: + r"""Return the index of a currently selected device for the current :ref:`accelerator`. + + Returns: + int: the index of a currently selected device. + """ + return torch._C._accelerator_getDeviceIndex() + + +current_device_idx = deprecated( + "Use `current_device_index` instead.", + category=FutureWarning, +)(current_device_index) + +current_device_idx.__doc__ = r""" + (Deprecated) Return the index of a currently selected device for the current :ref:`accelerator`. + + Returns: + int: the index of a currently selected device. + + .. warning:: + + :func:`torch.accelerator.current_device_idx` is deprecated in favor of :func:`torch.accelerator.current_device_index` + and will be removed in a future PyTorch release. + """ + + +@cache +def get_device_capability(device: _device_t = None, /) -> dict[str, Any]: + r"""Return the capability of the currently selected device. + + Args: + device (:class:`torch.device`, str, int, optional): The device to query capabilities for + :ref:`accelerator` device type. If not given, + use :func:`torch.accelerator.current_device_index` by default. + + Returns: + dict[str, Any]: A dictionary containing device capability information. The dictionary includes: + - ``supported_dtypes`` (set(torch.dtype)): Set of PyTorch data types supported by the device + + Examples: + >>> # xdoctest: +SKIP("requires cuda") + >>> # Query capabilities for current device + >>> capabilities = torch.accelerator.get_device_capability("cuda:0") + >>> print("Supported dtypes:", capabilities["supported_dtypes"]) + """ + device_index = _get_device_index(device, optional=True) + return torch._C._accelerator_getDeviceCapability(device_index) + + +def set_device_index(device: _device_t, /) -> None: + r"""Set the current device index to a given device. + + Args: + device (:class:`torch.device`, str, int): a given device that must match the current + :ref:`accelerator` device type. + + .. note:: This function is a no-op if this device index is negative. + """ + device_index = _get_device_index(device, optional=False) + torch._C._accelerator_setDeviceIndex(device_index) + + +set_device_idx = deprecated( + "Use `set_device_index` instead.", + category=FutureWarning, +)(set_device_index) + +set_device_idx.__doc__ = r""" + (Deprecated) Set the current device index to a given device. + + Args: + device (:class:`torch.device`, str, int): a given device that must match the current + :ref:`accelerator` device type. + + .. warning:: + + :func:`torch.accelerator.set_device_idx` is deprecated in favor of :func:`torch.accelerator.set_device_index` + and will be removed in a future PyTorch release. + """ + + +def current_stream(device: _device_t = None, /) -> torch.Stream: + r"""Return the currently selected stream for a given device. + + Args: + device (:class:`torch.device`, str, int, optional): a given device that must match the current + :ref:`accelerator` device type. If not given, + use :func:`torch.accelerator.current_device_index` by default. + + Returns: + torch.Stream: the currently selected stream for a given device. + """ + device_index = _get_device_index(device, optional=True) + return torch._C._accelerator_getStream(device_index) + + +def set_stream(stream: torch.Stream) -> None: + r"""Set the current stream to a given stream. + + Args: + stream (torch.Stream): a given stream that must match the current :ref:`accelerator` device type. + + .. note:: This function will set the current device index to the device index of the given stream. + """ + torch._C._accelerator_setStream(stream) + + +def synchronize(device: _device_t = None, /) -> None: + r"""Wait for all kernels in all streams on the given device to complete. + + Args: + device (:class:`torch.device`, str, int, optional): device for which to synchronize. It must match + the current :ref:`accelerator` device type. If not given, + use :func:`torch.accelerator.current_device_index` by default. + + .. note:: This function is a no-op if the current :ref:`accelerator` is not initialized. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> assert torch.accelerator.is_available() "No available accelerators detected." + >>> start_event = torch.Event(enable_timing=True) + >>> end_event = torch.Event(enable_timing=True) + >>> start_event.record() + >>> tensor = torch.randn(100, device=torch.accelerator.current_accelerator()) + >>> sum = torch.sum(tensor) + >>> end_event.record() + >>> torch.accelerator.synchronize() + >>> elapsed_time_ms = start_event.elapsed_time(end_event) + """ + device_index = _get_device_index(device, optional=True) + torch._C._accelerator_synchronizeDevice(device_index) + + +class device_index: + r"""Context manager to set the current device index for the current :ref:`accelerator`. + Temporarily changes the current device index to the specified value for the duration + of the context, and automatically restores the previous device index when exiting + the context. + + Args: + device (Optional[int]): a given device index to temporarily set. If None, + no device index switching occurs. + + Examples: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> # Set device 0 as the current device temporarily + >>> with torch.accelerator.device_index(0): + ... # Code here runs with device 0 as the current device + ... pass + >>> # Original device is now restored + >>> # No-op when None is passed + >>> with torch.accelerator.device_index(None): + ... # No device switching occurs + ... pass + """ + + def __init__(self, device: int | None, /) -> None: + self.idx = device + self.prev_idx = -1 + + def __enter__(self) -> None: + if self.idx is not None: + self.prev_idx = torch._C._accelerator_exchangeDevice(self.idx) + + def __exit__(self, *exc_info: object) -> None: + if self.idx is not None: + torch._C._accelerator_maybeExchangeDevice(self.prev_idx) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/accelerator/_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/accelerator/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ef62e298de1c820380f09391e0d8dba257a90d8b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/accelerator/_utils.py @@ -0,0 +1,26 @@ +import torch +from torch.types import Device as _device_t + + +def _get_device_index(device: _device_t, optional: bool = False) -> int: + if isinstance(device, int): + return device + if isinstance(device, str): + device = torch.device(device) + device_index: int | None = None + if isinstance(device, torch.device): + acc = torch.accelerator.current_accelerator() + if acc is None: + raise RuntimeError("Accelerator expected") + if acc.type != device.type: + raise ValueError( + f"{device.type} doesn't match the current accelerator {acc}." + ) + device_index = device.index + if device_index is None: + if not optional: + raise ValueError( + f"Expected a torch.device with a specified index or an integer, but got:{device}" + ) + return torch.accelerator.current_device_index() + return device_index diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/accelerator/memory.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/accelerator/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..513e497f3883c0af572d365ba89424e8b7a2abd7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/accelerator/memory.py @@ -0,0 +1,236 @@ +from collections import OrderedDict +from typing import Any + +import torch + +from ._utils import _device_t, _get_device_index + + +__all__ = [ + "empty_cache", + "get_memory_info", + "max_memory_allocated", + "max_memory_reserved", + "memory_allocated", + "memory_reserved", + "memory_stats", + "reset_accumulated_memory_stats", + "reset_peak_memory_stats", +] + + +def empty_cache() -> None: + r"""Release all unoccupied cached memory currently held by the caching + allocator so that those can be used in other application. + + .. note:: This function is a no-op if the memory allocator for the current + :ref:`accelerator ` has not been initialized. + """ + if not torch._C._accelerator_isAllocatorInitialized(): + return + torch._C._accelerator_emptyCache() + + +def memory_stats(device_index: _device_t = None, /) -> OrderedDict[str, Any]: + r"""Return a dictionary of accelerator device memory allocator statistics for a given device index. + + The return value of this function is a dictionary of statistics, each of + which is a non-negative integer. + + Core statistics: + + - ``"allocated.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of allocation requests received by the memory allocator. + - ``"allocated_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of allocated memory. + - ``"segment.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of reserved segments from device memory allocation. + - ``"reserved_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of reserved memory. + - ``"active.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of active memory blocks. + - ``"active_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of active memory. + - ``"inactive_split.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of inactive, non-releasable memory blocks. + - ``"inactive_split_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of inactive, non-releasable memory. + + For these core statistics, values are broken down as follows. + + Pool type: + + - ``all``: combined statistics across all memory pools. + - ``large_pool``: statistics for the large allocation pool + (as of June 2025, for size >= 1MB allocations). + - ``small_pool``: statistics for the small allocation pool + (as of June 2025, for size < 1MB allocations). + + Metric type: + + - ``current``: current value of this metric. + - ``peak``: maximum value of this metric. + - ``allocated``: historical total increase in this metric. + - ``freed``: historical total decrease in this metric. + + In addition to the core statistics, we also provide some simple event + counters: + + - ``"num_alloc_retries"``: number of failed device memory allocation calls that + result in a cache flush and retry. + - ``"num_ooms"``: number of out-of-memory errors thrown. + - ``"num_sync_all_streams"``: number of ``synchronize_and_free_events`` calls. + - ``"num_device_alloc"``: number of device memory allocation calls. + - ``"num_device_free"``: number of device memory free calls. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + Returns: + OrderedDict[str, Any]: an ordered dictionary mapping statistic names to their values. + """ + if not torch._C._accelerator_isAllocatorInitialized(): + return OrderedDict() + device_index = _get_device_index(device_index, optional=True) + stats = torch._C._accelerator_getDeviceStats(device_index) + flat_stats = [] + + def flatten(prefix: str, value: Any) -> None: + if isinstance(value, dict): + for k, v in value.items(): + nested_prefix = f"{prefix}.{k}" if prefix else k + flatten(nested_prefix, v) + else: + flat_stats.append((prefix, value)) + + flatten("", stats) + flat_stats.sort() + # pyrefly: ignore [no-matching-overload] + return OrderedDict(flat_stats) + + +def memory_allocated(device_index: _device_t = None, /) -> int: + r"""Return the current :ref:`accelerator` device memory occupied by tensors + in bytes for a given device index. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + Returns: + int: the current memory occupied by live tensors (in bytes) within the current process. + """ + return memory_stats(device_index).get("allocated_bytes.all.current", 0) + + +def max_memory_allocated(device_index: _device_t = None, /) -> int: + r"""Return the current :ref:`accelerator` maximum device memory occupied by tensors + in bytes for a given device index. + + By default, this returns the peak allocated memory since the beginning of + this program. :func:`~torch.accelerator.reset_peak_memory_stats` can be used to + reset the starting point in tracking this metric. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + Returns: + int: the peak memory occupied by live tensors (in bytes) within the current process. + """ + return memory_stats(device_index).get("allocated_bytes.all.peak", 0) + + +def memory_reserved(device_index: _device_t = None, /) -> int: + r"""Return the current :ref:`accelerator` device memory managed by the caching allocator + in bytes for a given device index. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + Returns: + int: the current memory reserved by PyTorch (in bytes) within the current process. + """ + return memory_stats(device_index).get("reserved_bytes.all.current", 0) + + +def max_memory_reserved(device_index: _device_t = None, /) -> int: + r"""Return the current :ref:`accelerator` maximum device memory managed by the caching allocator + in bytes for a given device index. + + By default, this returns the peak cached memory since the beginning of this + program. :func:`~torch.accelerator.reset_peak_memory_stats` can be used to reset + the starting point in tracking this metric. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + Returns: + int: the peak memory reserved by PyTorch (in bytes) within the current process. + """ + return memory_stats(device_index).get("reserved_bytes.all.peak", 0) + + +def reset_accumulated_memory_stats(device_index: _device_t = None, /) -> None: + r"""Reset the "accumulated" (historical) stats tracked by the current :ref:`accelerator` + memory allocator for a given device index. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + .. note:: This function is a no-op if the memory allocator for the current + :ref:`accelerator ` has not been initialized. + """ + device_index = _get_device_index(device_index, optional=True) + return torch._C._accelerator_resetAccumulatedStats(device_index) + + +def reset_peak_memory_stats(device_index: _device_t = None, /) -> None: + r"""Reset the "peak" stats tracked by the current :ref:`accelerator` + memory allocator for a given device index. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + .. note:: This function is a no-op if the memory allocator for the current + :ref:`accelerator ` has not been initialized. + """ + device_index = _get_device_index(device_index, optional=True) + return torch._C._accelerator_resetPeakStats(device_index) + + +def get_memory_info(device_index: _device_t = None, /) -> tuple[int, int]: + r"""Return the current device memory information for a given device index. + + Args: + device_index (:class:`torch.device`, str, int, optional): the index of the device to target. + If not given, use :func:`torch.accelerator.current_device_index` by default. + If a :class:`torch.device` or str is provided, its type must match the current + :ref:`accelerator` device type. + + Returns: + tuple[int, int]: a tuple of two integers (free_memory, total_memory) in bytes. + The first value is the free memory on the device (available across all processes and applications), + The second value is the device's total hardware memory capacity. + """ + device_index = _get_device_index(device_index, optional=True) + return torch._C._accelerator_getMemoryInfo(device_index) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/amp/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/amp/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..263908eff68bab7c7d9a123bd77a5f5c048b8c5a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/amp/__init__.py @@ -0,0 +1,9 @@ +from .autocast_mode import ( + _enter_autocast, + _exit_autocast, + autocast, + custom_bwd, + custom_fwd, + is_autocast_available, +) +from .grad_scaler import GradScaler diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/amp/autocast_mode.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/amp/autocast_mode.py new file mode 100644 index 0000000000000000000000000000000000000000..cd9039ce9f0e503d67d29f10a13d33b5af1b3ead --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/amp/autocast_mode.py @@ -0,0 +1,525 @@ +# mypy: allow-untyped-defs +import collections +import functools +import warnings +from typing import Any, Optional + +import torch +from torch.types import _dtype + + +try: + import numpy as np + + HAS_NUMPY = True +except ModuleNotFoundError: + HAS_NUMPY = False + np = None # type: ignore[assignment] + +__all__ = [ + "autocast_decorator", + "autocast", + "is_autocast_available", + "custom_fwd", + "custom_bwd", +] + + +def is_autocast_available(device_type: str) -> bool: + r""" + Return a bool indicating if autocast is available on :attr:`device_type`. + + Args: + device_type(str): Device type to use. Possible values are: 'cuda', 'cpu', 'mtia', 'maia', 'xpu', and so on. + The type is the same as the `type` attribute of a :class:`torch.device`. + Thus, you may obtain the device type of a tensor using `Tensor.device.type`. + """ + return torch._C._is_autocast_available(device_type) + + +def autocast_decorator(autocast_instance, func): + @functools.wraps(func) + def decorate_autocast(*args, **kwargs): + with autocast_instance: + return func(*args, **kwargs) + + decorate_autocast.__script_unsupported = ( # type: ignore[attr-defined] + "@autocast() decorator is not supported in script mode" + ) + return decorate_autocast + + +class autocast: + r""" + Instances of :class:`autocast` serve as context managers or decorators that + allow regions of your script to run in mixed precision. + + In these regions, ops run in an op-specific dtype chosen by autocast + to improve performance while maintaining accuracy. + See the :ref:`Autocast Op Reference` for details. + + When entering an autocast-enabled region, Tensors may be any type. + You should not call ``half()`` or ``bfloat16()`` on your model(s) or inputs when using autocasting. + + :class:`autocast` should wrap only the forward pass(es) of your network, including the loss + computation(s). Backward passes under autocast are not recommended. + Backward ops run in the same type that autocast used for corresponding forward ops. + + Example for CUDA Devices:: + + # Creates model and optimizer in default precision + model = Net().cuda() + optimizer = optim.SGD(model.parameters(), ...) + + for input, target in data: + optimizer.zero_grad() + + # Enables autocasting for the forward pass (model + loss) + with torch.autocast(device_type="cuda"): + output = model(input) + loss = loss_fn(output, target) + + # Exits the context manager before backward() + loss.backward() + optimizer.step() + + See the :ref:`Automatic Mixed Precision examples` for usage (along with gradient scaling) + in more complex scenarios (e.g., gradient penalty, multiple models/losses, custom autograd functions). + + :class:`autocast` can also be used as a decorator, e.g., on the ``forward`` method of your model:: + + class AutocastModel(nn.Module): + ... + + @torch.autocast(device_type="cuda") + def forward(self, input): ... + + Floating-point Tensors produced in an autocast-enabled region may be ``float16``. + After returning to an autocast-disabled region, using them with floating-point + Tensors of different dtypes may cause type mismatch errors. If so, cast the Tensor(s) + produced in the autocast region back to ``float32`` (or other dtype if desired). + If a Tensor from the autocast region is already ``float32``, the cast is a no-op, + and incurs no additional overhead. + CUDA Example:: + + # Creates some tensors in default dtype (here assumed to be float32) + a_float32 = torch.rand((8, 8), device="cuda") + b_float32 = torch.rand((8, 8), device="cuda") + c_float32 = torch.rand((8, 8), device="cuda") + d_float32 = torch.rand((8, 8), device="cuda") + + with torch.autocast(device_type="cuda"): + # torch.mm is on autocast's list of ops that should run in float16. + # Inputs are float32, but the op runs in float16 and produces float16 output. + # No manual casts are required. + e_float16 = torch.mm(a_float32, b_float32) + # Also handles mixed input types + f_float16 = torch.mm(d_float32, e_float16) + + # After exiting autocast, calls f_float16.float() to use with d_float32 + g_float32 = torch.mm(d_float32, f_float16.float()) + + CPU Training Example:: + + # Creates model and optimizer in default precision + model = Net() + optimizer = optim.SGD(model.parameters(), ...) + + for epoch in epochs: + for input, target in data: + optimizer.zero_grad() + + # Runs the forward pass with autocasting. + with torch.autocast(device_type="cpu", dtype=torch.bfloat16): + output = model(input) + loss = loss_fn(output, target) + + loss.backward() + optimizer.step() + + + CPU Inference Example:: + + # Creates model in default precision + model = Net().eval() + + with torch.autocast(device_type="cpu", dtype=torch.bfloat16): + for input in data: + # Runs the forward pass with autocasting. + output = model(input) + + CPU Inference Example with Jit Trace:: + + class TestModel(nn.Module): + def __init__(self, input_size, num_classes): + super().__init__() + self.fc1 = nn.Linear(input_size, num_classes) + + def forward(self, x): + return self.fc1(x) + + + input_size = 2 + num_classes = 2 + model = TestModel(input_size, num_classes).eval() + + # For now, we suggest to disable the Jit Autocast Pass, + # As the issue: https://github.com/pytorch/pytorch/issues/75956 + torch._C._jit_set_autocast_mode(False) + + with torch.cpu.amp.autocast(cache_enabled=False): + model = torch.jit.trace(model, torch.randn(1, input_size)) + model = torch.jit.freeze(model) + # Models Run + for _ in range(3): + model(torch.randn(1, input_size)) + + Type mismatch errors *in* an autocast-enabled region are a bug; if this is what you observe, + please file an issue. + + ``autocast(enabled=False)`` subregions can be nested in autocast-enabled regions. + Locally disabling autocast can be useful, for example, if you want to force a subregion + to run in a particular ``dtype``. Disabling autocast gives you explicit control over + the execution type. In the subregion, inputs from the surrounding region + should be cast to ``dtype`` before use:: + + # Creates some tensors in default dtype (here assumed to be float32) + a_float32 = torch.rand((8, 8), device="cuda") + b_float32 = torch.rand((8, 8), device="cuda") + c_float32 = torch.rand((8, 8), device="cuda") + d_float32 = torch.rand((8, 8), device="cuda") + + with torch.autocast(device_type="cuda"): + e_float16 = torch.mm(a_float32, b_float32) + with torch.autocast(device_type="cuda", enabled=False): + # Calls e_float16.float() to ensure float32 execution + # (necessary because e_float16 was created in an autocasted region) + f_float32 = torch.mm(c_float32, e_float16.float()) + + # No manual casts are required when re-entering the autocast-enabled region. + # torch.mm again runs in float16 and produces float16 output, regardless of input types. + g_float16 = torch.mm(d_float32, f_float32) + + The autocast state is thread-local. If you want it enabled in a new thread, the context manager or decorator + must be invoked in that thread. This affects :class:`torch.nn.DataParallel` and + :class:`torch.nn.parallel.DistributedDataParallel` when used with more than one GPU per process + (see :ref:`Working with Multiple GPUs`). + + Args: + device_type(str, required): Device type to use. Possible values are: 'cuda', 'cpu', 'mtia', 'maia', 'xpu', and 'hpu'. + The type is the same as the `type` attribute of a :class:`torch.device`. + Thus, you may obtain the device type of a tensor using `Tensor.device.type`. + enabled(bool, optional): Whether autocasting should be enabled in the region. + Default: ``True`` + dtype(torch_dtype, optional): Data type for ops run in autocast. It uses the default value + (``torch.float16`` for CUDA and ``torch.bfloat16`` for CPU), given by + :func:`~torch.get_autocast_dtype`, if :attr:`dtype` is ``None``. + Default: ``None`` + cache_enabled(bool, optional): Whether the weight cache inside autocast should be enabled. + Default: ``True`` + """ + + def __init__( + self, + device_type: str, + dtype: Optional[_dtype] = None, + enabled: bool = True, + cache_enabled: Optional[bool] = None, + ): + if not isinstance(device_type, str): + raise ValueError( + f"Expected `device_type` of type `str`, got: `{type(device_type)}`" + ) + self.fast_dtype = ( + torch.get_autocast_dtype(device_type) if dtype is None else dtype + ) + if torch._jit_internal.is_scripting(): + self._enabled = enabled + self.device = device_type + assert self.fast_dtype is not None + return + self.device = device_type + if not is_autocast_available(self.device): + raise RuntimeError( + f"User specified an unsupported autocast device_type '{self.device}'" + ) + + device_supported_dtypes = [torch.bfloat16, torch.float16] + + self.custom_backend_name = torch._C._get_privateuse1_backend_name() + if self.device == self.custom_backend_name: + necessary_funcs = [ + "get_amp_supported_dtype", + ] + message = f"Tried to use AMP with the `{self.custom_backend_name}` backend, but the backend has not " + message += "registered a module or the module miss some necessary funcs. The backend should register " + message += "a module by `torch._register_device_module`, and the module must have these funcs: \n" + message += "`get_amp_supported_dtype() -> List[torch.dtype]`. \n" + + assert hasattr(torch, self.custom_backend_name), message + self.custom_device_mod = getattr(torch, self.custom_backend_name) + for func in necessary_funcs: + assert hasattr(self.custom_device_mod, func), ( + message + f"But the func `{func}` is missing. \n" + ) + device_supported_dtypes = self.custom_device_mod.get_amp_supported_dtype() + + self._cache_enabled = ( + torch.is_autocast_cache_enabled() + if cache_enabled is None + else cache_enabled + ) + + device_name = ( + self.device + if self.device == self.custom_backend_name + else self.device.upper() + ) + if enabled: + # Special case for CUDA AMP and bfloat16 support + if self.device == "cuda": + if torch.cuda.amp.common.amp_definitely_not_available(): + warnings.warn( + "CUDA is not available or torch_xla is imported. Disabling autocast.", + stacklevel=2, + ) + enabled = False + elif ( + self.fast_dtype == torch.bfloat16 + and not torch.cuda.is_bf16_supported() + ): + raise RuntimeError( + "Current CUDA Device does not support bfloat16. Please switch dtype to float16." + ) + elif self.fast_dtype not in device_supported_dtypes: + error_message = ( + f"In {device_name} autocast, but the target dtype is not supported. Disabling autocast.\n" + f"{device_name} Autocast only supports dtypes of " + + ", ".join(map(str, device_supported_dtypes)) + + " currently." + ) + warnings.warn(error_message, stacklevel=2) + enabled = False + # Special case for MPS bfloat16 support on macOS < 14 + if ( + self.device == "mps" + and self.fast_dtype == torch.bfloat16 + and not torch.backends.mps.is_macos_or_newer(14, 0) + ): + error_message = ( + "In MPS autocast, but the target dtype torch.bfloat16 is not supported " + "on macOS versions below 14. Disabling autocast." + ) + warnings.warn(error_message, stacklevel=2) + enabled = False + self._enabled = enabled + + def __enter__(self): + if torch._jit_internal.is_scripting(): + assert self.fast_dtype is not None + return self + + self.prev_cache_enabled = torch.is_autocast_cache_enabled() + self.prev = torch.is_autocast_enabled(self.device) + self.prev_fastdtype = torch.get_autocast_dtype(self.device) + torch.set_autocast_enabled(self.device, self._enabled) + torch.set_autocast_dtype(self.device, self.fast_dtype) # type: ignore[arg-type] + torch.autocast_increment_nesting() + torch.set_autocast_cache_enabled(self._cache_enabled) + + # only dispatch to PreDispatchTorchFunctionMode to avoid exposing this + # API to other functional modes. We only expose to PreDispatchTorchFunctionMode + # for preserving autocast in torch.export.export. + if torch._C._is_torch_function_mode_enabled(): + stacks = torch.overrides._get_current_function_mode_stack() + for mode in stacks: + if isinstance( + mode, + torch.fx.experimental.proxy_tensor.PreDispatchTorchFunctionMode, + ): + args = ( + self.device, + self.fast_dtype, + self._enabled, + self._cache_enabled, + ) + mode.__torch_function__(torch.amp._enter_autocast, (), args) + return self + + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any): # type: ignore[override] + if torch._jit_internal.is_scripting(): + return + + # Drop the cache when we exit to a nesting level that's outside any instance of autocast. + if torch.autocast_decrement_nesting() == 0: + torch.clear_autocast_cache() + torch.set_autocast_enabled(self.device, self.prev) + torch.set_autocast_dtype(self.device, self.prev_fastdtype) + torch.set_autocast_cache_enabled(self.prev_cache_enabled) + + # only dispatch to PreDispatchTorchFunctionMode to avoid exposing this + # API to other functional modes. We only expose to PreDispatchTorchFunctionMode + # for preserving autocast in torch.export.export. + if torch._C._is_torch_function_mode_enabled(): + stacks = torch.overrides._get_current_function_mode_stack() + for mode in stacks: + if isinstance( + mode, + torch.fx.experimental.proxy_tensor.PreDispatchTorchFunctionMode, + ): + mode.__torch_function__(torch.amp._exit_autocast, (), ()) + # This is very important because the above line actually doesn't + # run exit code so it end up swallowing exceptions. + return False + return False + + def __call__(self, func): + if torch._jit_internal.is_scripting(): + return func + return autocast_decorator(self, func) + + +# These functions aren't meant for public usage. +# They are what we trace into a graph during pre_dispatch tracing +# when we encounter an autocast context manager. +def _enter_autocast(*vals): + # For pre-dispatch tracing, if a TorchFunction mode is active, we'll want to trace this into a graph. + if torch._C._is_torch_function_mode_enabled(): + return torch.overrides.handle_torch_function( + torch.amp._enter_autocast, [], *vals + ) + mode = torch.amp.autocast(*vals) + mode.__enter__() + return mode + + +def _exit_autocast(mode): + if torch._C._is_torch_function_mode_enabled(): + return torch.overrides.handle_torch_function(torch.amp._exit_autocast, [], mode) + mode.__exit__(None, None, None) + + +# Casts Tensors and containers of Tensors. Special-cases passthroughs for strings and np.ndarrays, which +# may be falsely detected as "Iterables." +def _cast(value, device_type: str, dtype: _dtype): + if isinstance(value, torch.Tensor): + is_eligible = ( + value.is_floating_point() + and value.device.type == device_type + and (value.dtype is not torch.float64) + ) + return value.to(dtype) if is_eligible else value + elif isinstance(value, (str, bytes)): + return value + elif HAS_NUMPY and isinstance( + value, + # pyrefly: ignore [missing-attribute] + np.ndarray, + ): + return value + elif isinstance(value, collections.abc.Mapping): + return { + _cast(k, device_type, dtype): _cast(v, device_type, dtype) + for k, v in value.items() + } + elif isinstance(value, collections.abc.Iterable): + iterable = (_cast(v, device_type, dtype) for v in value) + if isinstance(value, (list, tuple)): + return type(value)(iterable) + else: + return iterable + else: + return value + + +def custom_fwd( + fwd=None, + *, + device_type: str, + cast_inputs: Optional[_dtype] = None, +): + """ + Create a helper decorator for ``forward`` methods of custom autograd functions. + + Autograd functions are subclasses of :class:`torch.autograd.Function`. + See the :ref:`example page` for more detail. + + Args: + device_type(str): Device type to use. 'cuda', 'cpu', 'mtia', 'maia', 'xpu' and so on. + The type is the same as the `type` attribute of a :class:`torch.device`. + Thus, you may obtain the device type of a tensor using `Tensor.device.type`. + cast_inputs (:class:`torch.dtype` or None, optional, default=None): If not ``None``, + when ``forward`` runs in an autocast-enabled region, casts incoming + floating-point Tensors to the target dtype (non-floating-point Tensors are not affected), + then executes ``forward`` with autocast disabled. + If ``None``, ``forward``'s internal ops execute with the current autocast state. + + .. note:: + If the decorated ``forward`` is called outside an autocast-enabled region, + :func:`custom_fwd` is a no-op and ``cast_inputs`` has no effect. + """ + if not isinstance(device_type, str): + raise ValueError( + f"Expected `device_type` of type `str`, got: `{type(device_type)}`" + ) + if fwd is None: + return functools.partial( + custom_fwd, device_type=device_type, cast_inputs=cast_inputs + ) + + @functools.wraps(fwd) + def decorate_fwd(*args, **kwargs): + args[0]._dtype = torch.get_autocast_dtype(device_type) + if cast_inputs is None: + args[0]._fwd_used_autocast = torch.is_autocast_enabled(device_type) + return fwd(*args, **kwargs) # pyrefly: ignore [not-callable] + else: + autocast_context = torch.is_autocast_enabled(device_type) + args[0]._fwd_used_autocast = False + if autocast_context: + with autocast(device_type=device_type, enabled=False): + return fwd( # pyrefly: ignore # not-callable + *_cast(args, device_type, cast_inputs), + **_cast(kwargs, device_type, cast_inputs), + ) + else: + return fwd(*args, **kwargs) # pyrefly: ignore [not-callable] + + return decorate_fwd + + +# Autograd ensures incoming gradients are the same type as forward outputs. Allowing a separate +# cast_inputs argument on custom_bwd is unnecessary and could cause errors if it doesn't match +# cast_inputs supplied to custom_fwd. +def custom_bwd(bwd=None, *, device_type: str): + """Create a helper decorator for backward methods of custom autograd functions. + + Autograd functions are subclasses of :class:`torch.autograd.Function`. + Ensures that ``backward`` executes with the same autocast state as ``forward``. + See the :ref:`example page` for more detail. + + Args: + device_type(str): Device type to use. 'cuda', 'cpu', 'mtia', 'maia', 'xpu' and so on. + The type is the same as the `type` attribute of a :class:`torch.device`. + Thus, you may obtain the device type of a tensor using `Tensor.device.type`. + """ + + if not isinstance(device_type, str): + raise ValueError( + f"Expected `device_type` of type `str`, got: `{type(device_type)}`" + ) + if bwd is None: + return functools.partial(custom_bwd, device_type=device_type) + + @functools.wraps(bwd) + def decorate_bwd(*args, **kwargs): + with autocast( + device_type=device_type, + enabled=args[0]._fwd_used_autocast, + dtype=args[0]._dtype, + ): + return bwd(*args, **kwargs) # pyrefly: ignore [not-callable] + + return decorate_bwd diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/amp/grad_scaler.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/amp/grad_scaler.py new file mode 100644 index 0000000000000000000000000000000000000000..506c2cf901600e6d78cbb6bd5831bc7a28c09837 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/amp/grad_scaler.py @@ -0,0 +1,693 @@ +# mypy: allow-untyped-defs +from __future__ import annotations + +import inspect +import warnings +from collections import abc, defaultdict +from enum import Enum +from typing import Any, cast, Optional, overload, TYPE_CHECKING, Union + +import torch + + +if TYPE_CHECKING: + from collections.abc import Iterable + + +__all__ = ["OptState", "GradScaler"] + + +class _MultiDeviceReplicator: + """Lazily serves copies of a tensor to requested devices. + + Copies are cached per-device. + """ + + def __init__(self, master_tensor: torch.Tensor) -> None: + self.master = master_tensor + self._per_device_tensors: dict[torch.device, torch.Tensor] = {} + + def get(self, device: torch.device) -> torch.Tensor: + retval = self._per_device_tensors.get(device, None) + if retval is None: + retval = self.master.to(device=device, non_blocking=True, copy=True) + self._per_device_tensors[device] = retval + return retval + + +# Defines default_factory for GradScaler's _per_optimizer_states defaultdict, +# as well as associated "enum" values. Prefers defining these at top level because +# - Lambdas can't be pickled, so we don't want to supply a lambda as the factory. +# - Defining READY, UNSCALED, STEPPED and _refresh_per_optimizer_state within GradScaler +# causes a circular reference, which we'd rather avoid. +class OptState(Enum): + READY = 0 + UNSCALED = 1 + STEPPED = 2 + + +def _refresh_per_optimizer_state() -> dict[str, Any]: + return {"stage": OptState.READY, "found_inf_per_device": {}} + + +class GradScaler: + """An instance ``scaler`` of :class:`GradScaler`. + + Helps perform the steps of gradient scaling + conveniently. + + * ``scaler.scale(loss)`` multiplies a given loss by ``scaler``'s current scale factor. + * ``scaler.step(optimizer)`` safely unscales gradients and calls ``optimizer.step()``. + * ``scaler.update()`` updates ``scaler``'s scale factor. + + Example:: + + # Creates a GradScaler once at the beginning of training. + scaler = GradScaler() + + for epoch in epochs: + for input, target in data: + optimizer.zero_grad() + output = model(input) + loss = loss_fn(output, target) + + # Scales loss. Calls backward() on scaled loss to create scaled gradients. + scaler.scale(loss).backward() + + # scaler.step() first unscales gradients of the optimizer's params. + # If gradients don't contain infs/NaNs, optimizer.step() is then called, + # otherwise, optimizer.step() is skipped. + scaler.step(optimizer) + + # Updates the scale for next iteration. + scaler.update() + + See the :ref:`Automatic Mixed Precision examples` for usage + (along with autocasting) in more complex cases like gradient clipping, gradient accumulation, gradient penalty, + and multiple losses/optimizers. + + ``scaler`` dynamically estimates the scale factor each iteration. To minimize gradient underflow, + a large scale factor should be used. However, ``float16`` values can "overflow" (become inf or NaN) if + the scale factor is too large. Therefore, the optimal scale factor is the largest factor that can be used + without incurring inf or NaN gradient values. + ``scaler`` approximates the optimal scale factor over time by checking the gradients for infs and NaNs during every + ``scaler.step(optimizer)`` (or optional separate ``scaler.unscale_(optimizer)``, see :meth:`unscale_`). + + * If infs/NaNs are found, ``scaler.step(optimizer)`` skips the underlying ``optimizer.step()`` (so the params + themselves remain uncorrupted) and ``update()`` multiplies the scale by ``backoff_factor``. + + * If no infs/NaNs are found, ``scaler.step(optimizer)`` runs the underlying ``optimizer.step()`` as usual. + If ``growth_interval`` unskipped iterations occur consecutively, ``update()`` multiplies the scale by + ``growth_factor``. + + The scale factor often causes infs/NaNs to appear in gradients for the first few iterations as its + value calibrates. ``scaler.step`` will skip the underlying ``optimizer.step()`` for these + iterations. After that, step skipping should occur rarely (once every few hundred or thousand iterations). + + Args: + device (str, optional, default="cuda"): Device type to use. Possible values are: 'cuda' and 'cpu'. + The type is the same as the `type` attribute of a :class:`torch.device`. + Thus, you may obtain the device type of a tensor using `Tensor.device.type`. + init_scale (float, optional, default=2.**16): Initial scale factor. + growth_factor (float, optional, default=2.0): Factor by which the scale is multiplied during + :meth:`update` if no inf/NaN gradients occur for ``growth_interval`` consecutive iterations. + backoff_factor (float, optional, default=0.5): Factor by which the scale is multiplied during + :meth:`update` if inf/NaN gradients occur in an iteration. + growth_interval (int, optional, default=2000): Number of consecutive iterations without inf/NaN gradients + that must occur for the scale to be multiplied by ``growth_factor``. + enabled (bool, optional): If ``False``, disables gradient scaling. :meth:`step` simply + invokes the underlying ``optimizer.step()``, and other methods become no-ops. + Default: ``True`` + """ + + def __init__( + self, + device: str = "cuda", + init_scale: float = 2.0**16, + growth_factor: float = 2.0, + backoff_factor: float = 0.5, + growth_interval: int = 2000, + enabled: bool = True, + ) -> None: + self._device = device + self._enabled = enabled + if self._device == "cuda": + if enabled and torch.cuda.amp.common.amp_definitely_not_available(): + warnings.warn( + "torch.cuda.amp.GradScaler is enabled, but CUDA is not available. Disabling.", + stacklevel=2, + ) + self._enabled = False + + if self._enabled: + assert growth_factor > 1.0, "The growth factor must be > 1.0." + assert backoff_factor < 1.0, "The backoff factor must be < 1.0." + + self._init_scale = init_scale + # self._scale will be lazily initialized during the first call to scale() + self._scale: Optional[torch.Tensor] = None + self._growth_factor = growth_factor + self._backoff_factor = backoff_factor + self._growth_interval = growth_interval + self._init_growth_tracker = 0 + # self._growth_tracker will be lazily initialized during the first call to scale() + self._growth_tracker: Optional[torch.Tensor] = None + self._per_optimizer_states: dict[int, dict[str, Any]] = defaultdict( + _refresh_per_optimizer_state + ) + + def _check_scale_growth_tracker( + self, funcname: str + ) -> tuple[torch.Tensor, torch.Tensor]: + fix = "This may indicate your script did not use scaler.scale(loss or outputs) earlier in the iteration." + assert self._scale is not None, ( + f"Attempted {funcname} but _scale is None. " + fix + ) + assert self._growth_tracker is not None, ( + f"Attempted {funcname} but _growth_tracker is None. " + fix + ) + return (self._scale, self._growth_tracker) + + def _lazy_init_scale_growth_tracker(self, dev: torch.device) -> None: + assert self._growth_tracker is None, "_growth_tracker initialized before _scale" + self._scale = torch.full((), self._init_scale, dtype=torch.float32, device=dev) + self._growth_tracker = torch.full( + (), self._init_growth_tracker, dtype=torch.int32, device=dev + ) + + @overload + def scale(self, outputs: torch.Tensor) -> torch.Tensor: ... + + @overload + def scale(self, outputs: list[torch.Tensor]) -> list[torch.Tensor]: ... + + @overload + def scale(self, outputs: tuple[torch.Tensor, ...]) -> tuple[torch.Tensor, ...]: ... + + @overload + def scale(self, outputs: Iterable[torch.Tensor]) -> Iterable[torch.Tensor]: ... + + def scale( + self, + outputs: Union[torch.Tensor, Iterable[torch.Tensor]], + ) -> Union[torch.Tensor, Iterable[torch.Tensor]]: + """ + Multiplies ('scales') a tensor or list of tensors by the scale factor. + + Returns scaled outputs. If this instance of :class:`GradScaler` is not enabled, outputs are returned + unmodified. + + Args: + outputs (Tensor or iterable of Tensors): Outputs to scale. + """ + if not self._enabled: + return outputs + + # Short-circuit for the common case. + if isinstance(outputs, torch.Tensor): + if self._scale is None: + self._lazy_init_scale_growth_tracker(outputs.device) + assert self._scale is not None + return outputs * self._scale.to(device=outputs.device, non_blocking=True) + + # Invoke the more complex machinery only if we're treating multiple outputs. + stash: list[ + _MultiDeviceReplicator + ] = [] # holds a reference that can be overwritten by apply_scale + + def apply_scale(val: Union[torch.Tensor, Iterable[torch.Tensor]]): + if isinstance(val, torch.Tensor): + if len(stash) == 0: + if self._scale is None: + self._lazy_init_scale_growth_tracker(val.device) + assert self._scale is not None + stash.append(_MultiDeviceReplicator(self._scale)) + return val * stash[0].get(val.device) + if isinstance(val, abc.Iterable): + iterable = map(apply_scale, val) + if isinstance(val, (list, tuple)): + return type(val)(iterable) + return iterable + raise ValueError("outputs must be a Tensor or an iterable of Tensors") + + return apply_scale(outputs) + + def _unscale_grads_( + self, + optimizer: torch.optim.Optimizer, + inv_scale: torch.Tensor, + found_inf: torch.Tensor, + allow_fp16: bool, + ) -> dict[torch.device, torch.Tensor]: + per_device_inv_scale = _MultiDeviceReplicator(inv_scale) + per_device_found_inf = _MultiDeviceReplicator(found_inf) + + # To set up _amp_foreach_non_finite_check_and_unscale_, split grads by device and dtype. + # There could be hundreds of grads, so we'd like to iterate through them just once. + # However, we don't know their devices or dtypes in advance. + + # https://stackoverflow.com/questions/5029934/defaultdict-of-defaultdict + # Google says mypy struggles with defaultdicts type annotations. + per_device_and_dtype_grads: dict[ + torch.device, dict[torch.dtype, list[torch.Tensor]] + ] = defaultdict(lambda: defaultdict(list)) + with torch.no_grad(): + for group in optimizer.param_groups: + for param in group["params"]: + assert isinstance(param, torch.Tensor) + if param.grad is None: + continue + if (not allow_fp16) and param.grad.dtype == torch.float16: + raise ValueError("Attempting to unscale FP16 gradients.") + if param.grad.is_sparse: + # is_coalesced() == False means the sparse grad has values with duplicate indices. + # coalesce() deduplicates indices and adds all values that have the same index. + # For scaled fp16 values, there's a good chance coalescing will cause overflow, + # so we should check the coalesced _values(). + if param.grad.dtype is torch.float16: + param.grad = param.grad.coalesce() + to_unscale = param.grad._values() + else: + to_unscale = param.grad + + # TODO: is there a way to split by device and dtype without appending in the inner loop? + per_device_and_dtype_grads[to_unscale.device][ + to_unscale.dtype + ].append(to_unscale) + + for device, per_dtype_grads in per_device_and_dtype_grads.items(): + for grads in per_dtype_grads.values(): + torch._amp_foreach_non_finite_check_and_unscale_( + grads, + per_device_found_inf.get(device), + per_device_inv_scale.get(device), + ) + + return per_device_found_inf._per_device_tensors + + def unscale_(self, optimizer: torch.optim.Optimizer) -> None: + """ + Divides ("unscales") the optimizer's gradient tensors by the scale factor. + + :meth:`unscale_` is optional, serving cases where you need to + :ref:`modify or inspect gradients` + between the backward pass(es) and :meth:`step`. + If :meth:`unscale_` is not called explicitly, gradients will be unscaled automatically during :meth:`step`. + + Simple example, using :meth:`unscale_` to enable clipping of unscaled gradients:: + + ... + scaler.scale(loss).backward() + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm) + scaler.step(optimizer) + scaler.update() + + Args: + optimizer (torch.optim.Optimizer): Optimizer that owns the gradients to be unscaled. + + .. note:: + :meth:`unscale_` does not incur a CPU-GPU sync. + + .. warning:: + :meth:`unscale_` should only be called once per optimizer per :meth:`step` call, + and only after all gradients for that optimizer's assigned parameters have been accumulated. + Calling :meth:`unscale_` twice for a given optimizer between each :meth:`step` triggers a RuntimeError. + + .. warning:: + :meth:`unscale_` may unscale sparse gradients out of place, replacing the ``.grad`` attribute. + """ + if not self._enabled: + return + + self._check_scale_growth_tracker("unscale_") + + optimizer_state = self._per_optimizer_states[id(optimizer)] + + if optimizer_state["stage"] is OptState.UNSCALED: + raise RuntimeError( + "unscale_() has already been called on this optimizer since the last update()." + ) + elif optimizer_state["stage"] is OptState.STEPPED: + raise RuntimeError("unscale_() is being called after step().") + + # FP32 division can be imprecise for certain compile options, so we carry out the reciprocal in FP64. + assert self._scale is not None + inv_scale = ( + self._scale.double().reciprocal().float() + if self._scale.device != torch.device("mps:0") + else self._scale.reciprocal() + ) + found_inf = torch.full((), 0.0, dtype=torch.float32, device=self._scale.device) + + optimizer_state["found_inf_per_device"] = self._unscale_grads_( + optimizer, inv_scale, found_inf, False + ) + optimizer_state["stage"] = OptState.UNSCALED + + def _maybe_opt_step( + self, + optimizer: torch.optim.Optimizer, + optimizer_state: dict[str, Any], + *args: Any, + **kwargs: Any, + ) -> Optional[float]: + retval: Optional[float] = None + if not sum(v.item() for v in optimizer_state["found_inf_per_device"].values()): + retval = optimizer.step(*args, **kwargs) + return retval + + def step( + self, optimizer: torch.optim.Optimizer, *args: Any, **kwargs: Any + ) -> Optional[float]: + """Invoke ``unscale_(optimizer)`` followed by parameter update, if gradients are not infs/NaN. + + :meth:`step` carries out the following two operations: + + 1. Internally invokes ``unscale_(optimizer)`` (unless :meth:`unscale_` was explicitly called for ``optimizer`` + earlier in the iteration). As part of the :meth:`unscale_`, gradients are checked for infs/NaNs. + 2. If no inf/NaN gradients are found, invokes ``optimizer.step()`` using the unscaled + gradients. Otherwise, ``optimizer.step()`` is skipped to avoid corrupting the params. + + ``*args`` and ``**kwargs`` are forwarded to ``optimizer.step()``. + + Returns the return value of ``optimizer.step(*args, **kwargs)``. + + Args: + optimizer (torch.optim.Optimizer): Optimizer that applies the gradients. + args: Any arguments. + kwargs: Any keyword arguments. + + .. warning:: + Closure use is not currently supported. + """ + if not self._enabled: + return optimizer.step(*args, **kwargs) + + if "closure" in kwargs: + raise RuntimeError( + "Closure use is not currently supported if GradScaler is enabled." + ) + + self._check_scale_growth_tracker("step") + + optimizer_state = self._per_optimizer_states[id(optimizer)] + + if optimizer_state["stage"] is OptState.STEPPED: + raise RuntimeError( + "step() has already been called since the last update()." + ) + + retval: Optional[float] = None + + if getattr(optimizer, "_step_supports_amp_scaling", False): + # This optimizer has customized scale-handling logic, so we can call optimizer.step() directly. + # The contract with custom optimizers is that their step() should accept an additional, + # optional grad_scaler kwarg. We append self to the kwargs so the custom optimizer has full information: + # it can query its own state, invoke unscale_ on itself, etc + # The contract above is being deprecated to avoid introducing `grad_scaler: GradScaler` argument + # to `Optimizer.step`. The new behavior is going to add two Tensor attributes of `grad_scale` + # and `found_inf` to the passed optimizer so that the optimizer can utilize those + # to skip the parameter updates or unscale gradients before updating parameters in + # the fused kernel, e.g. `FusedAdamMathFunctor`. + # In this behavior, `GradScaler._check_inf_per_device` is called if `OptState.READY`, + # while the method is expected to be called by users side, i.e. their optimizers. + kwargs_ = kwargs + has_grad_scaler_kwarg = ( + "grad_scaler" in inspect.signature(optimizer.step).parameters + ) + if has_grad_scaler_kwarg: + warnings.warn( + "GradScaler is going to stop passing itself as a keyword argument to the passed " + "optimizer. In the near future GradScaler registers `grad_scale: Tensor` and " + "`found_inf: Tensor` to the passed optimizer and let the optimizer use them directly.", + FutureWarning, + stacklevel=2, + ) + kwargs_.update({"grad_scaler": self}) + else: + if optimizer_state["stage"] is OptState.READY: + self._check_inf_per_device(optimizer) + scaler = self._get_scale_async() + assert scaler is not None + found_inf = cast( + torch.Tensor, + sum( + [ # noqa: C419 + t.to(scaler.device, non_blocking=True) + for t in optimizer_state["found_inf_per_device"].values() + ] + ), + ) + # Take the product of the scales, if the user has already set `optimizer.grad_scale`. + optimizer.grad_scale = ( # type: ignore[attr-defined] + getattr(optimizer, "grad_scale", None) + if optimizer_state["stage"] == OptState.UNSCALED + else scaler * getattr(optimizer, "grad_scale", 1) + ) + optimizer.found_inf = found_inf # type: ignore[attr-defined] + retval = optimizer.step(*args, **kwargs_) + optimizer_state["stage"] = OptState.STEPPED + if not has_grad_scaler_kwarg: + del optimizer.grad_scale # type: ignore[attr-defined] + del optimizer.found_inf # type: ignore[attr-defined] + return retval + + if optimizer_state["stage"] is OptState.READY: + self.unscale_(optimizer) + + assert len(optimizer_state["found_inf_per_device"]) > 0, ( + "No inf checks were recorded for this optimizer." + ) + + retval = self._maybe_opt_step(optimizer, optimizer_state, *args, **kwargs) + + optimizer_state["stage"] = OptState.STEPPED + + return retval + + def update(self, new_scale: Optional[Union[float, torch.Tensor]] = None) -> None: + """Update the scale factor. + + If any optimizer steps were skipped the scale is multiplied by ``backoff_factor`` + to reduce it. If ``growth_interval`` unskipped iterations occurred consecutively, + the scale is multiplied by ``growth_factor`` to increase it. + + Passing ``new_scale`` sets the new scale value manually. (``new_scale`` is not + used directly, it's used to fill GradScaler's internal scale tensor. So if + ``new_scale`` was a tensor, later in-place changes to that tensor will not further + affect the scale GradScaler uses internally.) + + Args: + new_scale (float or :class:`torch.Tensor`, optional, default=None): New scale factor. + + .. warning:: + :meth:`update` should only be called at the end of the iteration, after ``scaler.step(optimizer)`` has + been invoked for all optimizers used this iteration. + + .. warning:: + For performance reasons, we do not check the scale factor value to avoid synchronizations, + so the scale factor is not guaranteed to be above 1. If the scale falls below 1 and/or + you are seeing NaNs in your gradients or loss, something is likely wrong. For example, + bf16-pretrained models are often incompatible with AMP/fp16 due to differing dynamic ranges. + """ + if not self._enabled: + return + + _scale, _growth_tracker = self._check_scale_growth_tracker("update") + + if new_scale is not None: + assert self._scale is not None + # Accept a new user-defined scale. + if isinstance(new_scale, float): + self._scale.fill_(new_scale) + else: + reason = ( + "new_scale should be a float or a 1-element torch.cuda.FloatTensor or " + "torch.FloatTensor with requires_grad=False." + ) + assert new_scale.device.type == self._device, reason + assert new_scale.numel() == 1, reason + assert new_scale.requires_grad is False, reason + self._scale.copy_(new_scale) + else: + # Consume shared inf/nan data collected from optimizers to update the scale. + # If all found_inf tensors are on the same device as self._scale, this operation is asynchronous. + found_infs = [ + found_inf.to(device=_scale.device, non_blocking=True) + for state in self._per_optimizer_states.values() + for found_inf in state["found_inf_per_device"].values() + ] + + assert len(found_infs) > 0, "No inf checks were recorded prior to update." + + found_inf_combined = found_infs[0] + if len(found_infs) > 1: + for i in range(1, len(found_infs)): + found_inf_combined += found_infs[i] + + torch._amp_update_scale_( + _scale, + _growth_tracker, + found_inf_combined, + self._growth_factor, + self._backoff_factor, + self._growth_interval, + ) + + # To prepare for next iteration, clear the data collected from optimizers this iteration. + self._per_optimizer_states = defaultdict(_refresh_per_optimizer_state) + + def _get_scale_async(self) -> Optional[torch.Tensor]: + return self._scale + + def get_scale(self) -> float: + """Return a Python float containing the current scale, or 1.0 if scaling is disabled. + + .. warning:: + :meth:`get_scale` incurs a CPU-GPU sync. + """ + if self._enabled: + return ( + self._init_scale + if (scale := self._get_scale_async()) is None + else cast(float, scale.item()) + ) + return 1.0 + + def get_growth_factor(self) -> float: + r"""Return a Python float containing the scale growth factor.""" + return self._growth_factor + + def set_growth_factor(self, new_factor: float) -> None: + r"""Set a new scale growth factor. + + Args: + new_scale (float): Value to use as the new scale growth factor. + """ + self._growth_factor = new_factor + + def get_backoff_factor(self) -> float: + r"""Return a Python float containing the scale backoff factor.""" + return self._backoff_factor + + def set_backoff_factor(self, new_factor: float) -> None: + r"""Set a new scale backoff factor. + + Args: + new_scale (float): Value to use as the new scale backoff factor. + """ + self._backoff_factor = new_factor + + def get_growth_interval(self) -> int: + r"""Return a Python int containing the growth interval.""" + return self._growth_interval + + def set_growth_interval(self, new_interval: int) -> None: + r"""Set a new growth interval. + + Args: + new_interval (int): Value to use as the new growth interval. + """ + self._growth_interval = new_interval + + def _get_growth_tracker(self) -> int: + if self._enabled: + return ( + self._init_growth_tracker + if self._growth_tracker is None + else cast(int, self._growth_tracker.item()) + ) + return 0 + + def is_enabled(self) -> bool: + r"""Return a bool indicating whether this instance is enabled.""" + return self._enabled + + def state_dict(self) -> dict[str, Any]: + r"""Return the state of the scaler as a :class:`dict`. + + It contains five entries: + + * ``"scale"`` - a Python float containing the current scale + * ``"growth_factor"`` - a Python float containing the current growth factor + * ``"backoff_factor"`` - a Python float containing the current backoff factor + * ``"growth_interval"`` - a Python int containing the current growth interval + * ``"_growth_tracker"`` - a Python int containing the number of recent consecutive unskipped steps. + + If this instance is not enabled, returns an empty dict. + + .. note:: + If you wish to checkpoint the scaler's state after a particular iteration, :meth:`state_dict` + should be called after :meth:`update`. + """ + if self._enabled: + return { + "scale": self.get_scale(), + "growth_factor": self._growth_factor, + "backoff_factor": self._backoff_factor, + "growth_interval": self._growth_interval, + "_growth_tracker": self._get_growth_tracker(), + } + return {} + + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + r"""Load the scaler state. + + If this instance is disabled, :meth:`load_state_dict` is a no-op. + + Args: + state_dict(dict): scaler state. Should be an object returned from a call to :meth:`state_dict`. + """ + if not self._enabled: + return + + if len(state_dict) == 0: + raise RuntimeError( + "The source state dict is empty, possibly because it was saved " + "from a disabled instance of GradScaler." + ) + + self._init_scale = cast(float, state_dict["scale"]) + if self._scale is not None: + self._scale.fill_(state_dict["scale"]) + self._growth_factor = cast(float, state_dict["growth_factor"]) + self._backoff_factor = cast(float, state_dict["backoff_factor"]) + self._growth_interval = cast(int, state_dict["growth_interval"]) + self._init_growth_tracker = cast(int, state_dict["_growth_tracker"]) + if self._growth_tracker is not None: + self._growth_tracker.fill_(state_dict["_growth_tracker"]) + + def __getstate__(self) -> dict[str, Any]: + state = self.__dict__.copy() + if self._enabled: + assert len(self._per_optimizer_states) == 0, ( + "A GradScaler instance may only be pickled at the beginning " + "of an iteration, or at the end after scaler.update()." + ) + # Pickling _scale and _growth_tracker Tensors directly triggers + # "warnings.warn("pickle support for Storage will be removed in 1.5..." + # so instead, we set the unpickled instance up to reinitialize them lazily. + state["_init_scale"] = self.get_scale() + state["_init_growth_tracker"] = self._get_growth_tracker() + state["_scale"] = None + state["_growth_tracker"] = None + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__.update(state) + + def _check_inf_per_device(self, optimizer: torch.optim.Optimizer) -> dict[str, Any]: + _scale, _ = self._check_scale_growth_tracker("_check_inf_per_device") + + dummy_inv_scale = torch.full((), 1.0, dtype=torch.float32, device=_scale.device) + found_inf = torch.full((), 0.0, dtype=torch.float32, device=_scale.device) + + self._per_optimizer_states[id(optimizer)]["found_inf_per_device"] = ( + self._unscale_grads_(optimizer, dummy_inv_scale, found_inf, True) + ) + + return self._per_optimizer_states[id(optimizer)]["found_inf_per_device"] + + def _found_inf_per_device(self, optimizer: torch.optim.Optimizer) -> dict[str, Any]: + return self._per_optimizer_states[id(optimizer)]["found_inf_per_device"] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/ao/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/ao/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ac866b5073deb7909c5a18ed38dcd0d81ca473a8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/ao/__init__.py @@ -0,0 +1,31 @@ +# torch.ao is a package with a lot of interdependencies. +# We will use lazy import to avoid cyclic dependencies here. + +from typing import TYPE_CHECKING as _TYPE_CHECKING + + +if _TYPE_CHECKING: + from types import ModuleType + + from torch.ao import ( # noqa: TC004 + nn as nn, + ns as ns, + pruning as pruning, + quantization as quantization, + ) + + +__all__ = [ + "nn", + "ns", + "pruning", + "quantization", +] + + +def __getattr__(name: str) -> "ModuleType": + if name in __all__: + import importlib + + return importlib.import_module("." + name, __name__) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cfab4fa5e2d5fc7363475cb217bea6dfa0b49042 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/__init__.py @@ -0,0 +1,618 @@ +# mypy: allow-untyped-defs +""" +``torch.autograd`` provides classes and functions implementing automatic differentiation of arbitrary scalar valued functions. + +It requires minimal changes to the existing code - you only need to declare :class:`Tensor` s +for which gradients should be computed with the ``requires_grad=True`` keyword. +As of now, we only support autograd for floating point :class:`Tensor` types ( +half, float, double and bfloat16) and complex :class:`Tensor` types (cfloat, cdouble). +""" + +import warnings +from collections.abc import Sequence +from typing import cast, Optional, Union + +import torch +from torch import _vmap_internals +from torch.overrides import handle_torch_function, has_torch_function, is_tensor_like +from torch.types import _size, _TensorOrTensors, _TensorOrTensorsOrGradEdge + +from . import forward_ad, functional, graph +from .anomaly_mode import detect_anomaly, set_detect_anomaly +from .function import Function, NestedIOFunction +from .grad_mode import ( + _force_original_view_tracking, + _unsafe_preserve_version_counter, + enable_grad, + inference_mode, + no_grad, + set_grad_enabled, + set_multithreading_enabled, +) +from .gradcheck import gradcheck, gradgradcheck +from .graph import _engine_run_backward +from .variable import Variable + + +__all__ = [ + "Variable", + "Function", + "backward", + "grad_mode", + "NestedIOFunction", + "detect_anomaly", + "enable_grad", + "grad", + "gradcheck", + "gradgradcheck", + "inference_mode", + "no_grad", + "set_detect_anomaly", + "set_grad_enabled", + "set_multithreading_enabled", + "variable", +] + +_OptionalTensor = Optional[torch.Tensor] +_ShapeorNestedShape = Union[_size, Sequence[_size], torch.Tensor] + + +def _calculate_shape( + output: Union[torch.Tensor, graph.GradientEdge], + grad: torch.Tensor, + is_grads_batched: bool, +) -> tuple[_ShapeorNestedShape, _ShapeorNestedShape]: + # is_same_size ensures that both tensors are either nested or non nested + # circular import + from torch.nested._internal.nested_tensor import NestedTensor + + if isinstance(output, graph.GradientEdge): + # We have already checked that we are not a C++ NestedTensor + if is_grads_batched: + raise RuntimeError("Batched grads are not supported with GradientEdge") + out_metadata = output.node._input_metadata[output.output_nr] + return torch.Size(out_metadata.shape), grad.shape + + if output.is_nested and not isinstance(output, NestedTensor): + if is_grads_batched: + raise RuntimeError("Batched grads are not supported with Nested Tensor.") + out_shape = output._nested_tensor_size() + grad_shape = grad._nested_tensor_size() + + return out_shape, grad_shape + + reg_out_shape = output.shape + reg_grad_shape = grad.shape if not is_grads_batched else grad.shape[1:] + return reg_out_shape, reg_grad_shape + + +def _make_grads( + outputs: Union[Sequence[torch.Tensor], Sequence[graph.GradientEdge]], + grads: Sequence[_OptionalTensor], + is_grads_batched: bool, +) -> tuple[_OptionalTensor, ...]: + new_grads: list[_OptionalTensor] = [] + + for out, grad in zip(outputs, grads): + out = cast(Union[torch.Tensor, graph.GradientEdge], out) + out_size = None + out_device = None + + if isinstance(out, graph.GradientEdge): + out_metadata = out.node._input_metadata[out.output_nr] + out_size = torch.Size(out_metadata.shape) + out_dtype = out_metadata.dtype + out_device = out_metadata.device + out_is_nested = out_metadata.is_nested_tensor + if out_metadata.is_cpp_nested_tensor: + raise RuntimeError( + "C++ NestedTensor are not supported with GradientEdge" + ) + out_is_cpp_nested = False + else: + # circular import + from torch.nested._internal.nested_tensor import NestedTensor + + if not isinstance(out, torch.Tensor): + raise AssertionError("Expected output to be a torch.Tensor") + out_dtype = out.dtype + out_is_nested = out.is_nested + out_is_cpp_nested = out_is_nested and not isinstance(out, NestedTensor) + if not out_is_cpp_nested: + out_size = out.shape + + if isinstance(grad, torch.Tensor): + from torch.fx.experimental.symbolic_shapes import expect_true, sym_eq + + first_grad = grad if not is_grads_batched else grad[0] + + # TODO: We can remove this conditional once we uniformly use + # singleton int to represent jagged dimension, so that size() call + # on nested tensor works. + if out_is_cpp_nested: + if not isinstance(out, torch.Tensor): + raise AssertionError("Expected output to be a torch.Tensor.") + shape_matches = torch.is_same_size(out, first_grad) + else: + # We need to do a regular size check, without going through + # the operator, to be able to handle unbacked symints + # (expect_true ensures we can deal with unbacked) + if out_size is None: + raise AssertionError("Expected out_size to be set.") + shape_matches = expect_true(sym_eq(out_size, first_grad.size())) + + if not shape_matches: + out = cast(Union[torch.Tensor, graph.GradientEdge], out) # type: ignore[redundant-cast] + out_shape, grad_shape = _calculate_shape( + out, first_grad, is_grads_batched + ) + if is_grads_batched: + raise RuntimeError( + "If `is_grads_batched=True`, we interpret the first " + "dimension of each grad_output as the batch dimension. " + "The sizes of the remaining dimensions are expected to match " + "the shape of corresponding output, but a mismatch " + "was detected: grad_output[" + + str(grads.index(grad)) + + "] has a shape of " + + str(grad_shape) + + " and output[" + + str(outputs.index(out)) + + "] has a shape of " + + str(out_shape) + + ". " + "If you only want some tensors in `grad_output` to be considered " + "batched, consider using vmap." + ) + else: + raise RuntimeError( + "Mismatch in shape: grad_output[" + + str(grads.index(grad)) + + "] has a shape of " + + str(grad_shape) + + " and output[" + + str(outputs.index(out)) + + "] has a shape of " + + str(out_shape) + + "." + ) + if out_dtype.is_complex != grad.dtype.is_complex: + raise RuntimeError( + "For complex Tensors, both grad_output and output" + " are required to have the same dtype." + " Mismatch in dtype: grad_output[" + + str(grads.index(grad)) + + "] has a dtype of " + + str(grad.dtype) + + " and output[" + + str(outputs.index(out)) + + "] has a dtype of " + + str(out_dtype) + + "." + ) + new_grads.append(grad) + elif grad is None: + if isinstance(out, graph.GradientEdge) or out.requires_grad: # type: ignore[attr-defined] + if isinstance(out, graph.GradientEdge): + if out_size is None: + raise AssertionError("Expected out_size to be set.") + out_numel_is_1 = all(o == 1 for o in out_size) + else: + if not isinstance(out, torch.Tensor): + raise AssertionError("Expected output to be a torch.Tensor") + out_numel_is_1 = out.numel() == 1 + if not out_numel_is_1: + raise RuntimeError( + "grad can be implicitly created only for scalar outputs" + ) + if not out_dtype.is_floating_point: + msg = ( + "grad can be implicitly created only for real scalar outputs" + f" but got {out_dtype}" + ) + raise RuntimeError(msg) + if isinstance(out, graph.GradientEdge): + if out_size is None: + raise AssertionError("Expected out_size to be set.") + if out_device is None: + raise AssertionError("Expected out_device to be set.") + new_grads.append( + torch.ones( + out_size, + dtype=out_dtype, + device=out_device, + ) + ) + else: + if not isinstance(out, torch.Tensor): + raise AssertionError("Expected output to be a torch.Tensor") + new_grads.append( + torch.ones_like(out, memory_format=torch.preserve_format) + ) + else: + new_grads.append(None) + else: + raise TypeError( + "gradients can be either Tensors or None, but got " + + type(grad).__name__ + ) + return tuple(new_grads) + + +def _tensor_or_tensors_to_tuple( + tensors: Optional[_TensorOrTensors], length: int +) -> tuple[_OptionalTensor, ...]: + if tensors is None: + return (None,) * length + if isinstance(tensors, torch.Tensor): + return (tensors,) + return tuple(tensors) + + +def backward( + tensors: _TensorOrTensorsOrGradEdge, + grad_tensors: Optional[_TensorOrTensors] = None, + retain_graph: Optional[bool] = None, + create_graph: bool = False, + grad_variables: Optional[_TensorOrTensors] = None, + inputs: Optional[_TensorOrTensorsOrGradEdge] = None, +) -> None: + r"""Compute the sum of gradients of given tensors with respect to graph leaves. + + The graph is differentiated using the chain rule. If any of ``tensors`` + are non-scalar (i.e. their data has more than one element) and require + gradient, then the Jacobian-vector product would be computed, in this + case the function additionally requires specifying ``grad_tensors``. + It should be a sequence of matching length, that contains the "vector" + in the Jacobian-vector product, usually the gradient of the differentiated + function w.r.t. corresponding tensors (``None`` is an acceptable value for + all tensors that don't need gradient tensors). + + This function accumulates gradients in the leaves - you might need to zero + ``.grad`` attributes or set them to ``None`` before calling it. + See :ref:`Default gradient layouts` + for details on the memory layout of accumulated gradients. + + .. note:: + Using this method with ``create_graph=True`` will create a reference cycle + between the parameter and its gradient which can cause a memory leak. + We recommend using ``autograd.grad`` when creating the graph to avoid this. + If you have to use this function, make sure to reset the ``.grad`` fields of your + parameters to ``None`` after use to break the cycle and avoid the leak. + + .. note:: + + If you run any forward ops, create ``grad_tensors``, and/or call ``backward`` + in a user-specified CUDA stream context, see + :ref:`Stream semantics of backward passes`. + + .. note:: + + When ``inputs`` are provided and a given input is not a leaf, + the current implementation will call its grad_fn (even though it is not strictly needed to get this gradients). + It is an implementation detail on which the user should not rely. + See https://github.com/pytorch/pytorch/pull/60521#issuecomment-867061780 for more details. + + Args: + tensors (Sequence[Tensor] or Tensor or Sequence[GradientEdge] or GradientEdge): Tensors of which + the derivative will be computed. + grad_tensors (Sequence[Tensor or None] or Tensor, optional): The "vector" in + the Jacobian-vector product, usually gradients w.r.t. each element of + corresponding tensors. None values can be specified for scalar Tensors or + ones that don't require grad. If a None value would be acceptable for all + grad_tensors, then this argument is optional. + retain_graph (bool, optional): If ``False``, the graph used to compute the grad + will be freed. Note that in nearly all cases setting this option to ``True`` + is not needed and often can be worked around in a much more efficient + way. Defaults to the value of ``create_graph``. + create_graph (bool, optional): If ``True``, graph of the derivative will + be constructed, allowing to compute higher order derivative products. + Defaults to ``False``. + inputs (Sequence[Tensor] or Tensor or Sequence[GradientEdge], optional): Inputs w.r.t. which the gradient + be will accumulated into ``.grad``. All other Tensors will be ignored. If + not provided, the gradient is accumulated into all the leaf Tensors that + were used to compute the :attr:`tensors`. + """ + if torch._C._are_functorch_transforms_active(): + raise RuntimeError( + "backward() called inside a functorch transform. This is not " + "supported, please use functorch.grad or functorch.vjp instead " + "or call backward() outside of functorch transforms." + ) + + if grad_variables is not None: + warnings.warn( + "`grad_variables` is deprecated. Use `grad_tensors` instead.", + FutureWarning, + stacklevel=2, + ) + if grad_tensors is None: + grad_tensors = grad_variables + else: + raise RuntimeError( + "`grad_tensors` and `grad_variables` (deprecated) " + "arguments both passed to `backward()`. Please only " + "use `grad_tensors`." + ) + + inputs_tuple: tuple[Union[torch.Tensor, graph.GradientEdge], ...] + if inputs is None: + inputs_tuple = () + elif isinstance(inputs, (torch.Tensor, graph.GradientEdge)): + inputs_tuple = (inputs,) + else: + inputs_tuple = tuple(inputs) + if len(inputs_tuple) == 0: + raise RuntimeError("`inputs` argument to `backward()` cannot be empty.") + + if is_tensor_like(tensors) or isinstance(tensors, graph.GradientEdge): + tensors = cast( + Union[tuple[torch.Tensor], tuple[graph.GradientEdge]], (tensors,) + ) + else: + # pyrefly: ignore [bad-argument-type] + tensors = tuple(tensors) + + grad_tensors_ = _tensor_or_tensors_to_tuple(grad_tensors, len(tensors)) + grad_tensors_ = _make_grads(tensors, grad_tensors_, is_grads_batched=False) + if retain_graph is None: + retain_graph = create_graph + + # The reason we repeat the same comment below is that + # some Python versions print out the first line of a multi-line function + # calls in the traceback and some print out the last line + _engine_run_backward( + tensors, + grad_tensors_, + retain_graph, + create_graph, + inputs_tuple, + allow_unreachable=True, + accumulate_grad=True, + ) + + +def grad( + outputs: _TensorOrTensorsOrGradEdge, + inputs: _TensorOrTensorsOrGradEdge, + grad_outputs: Optional[_TensorOrTensors] = None, + retain_graph: Optional[bool] = None, + create_graph: bool = False, + only_inputs: bool = True, + allow_unused: Optional[bool] = None, + is_grads_batched: bool = False, + materialize_grads: bool = False, +) -> tuple[torch.Tensor, ...]: + r"""Compute and return the sum of gradients of outputs with respect to the inputs. + + ``grad_outputs`` should be a sequence of length matching ``output`` + containing the "vector" in vector-Jacobian product, usually the pre-computed + gradients w.r.t. each of the outputs. If an output doesn't require_grad, + then the gradient can be ``None``). + + .. note:: + + If you run any forward ops, create ``grad_outputs``, and/or call ``grad`` + in a user-specified CUDA stream context, see + :ref:`Stream semantics of backward passes`. + + .. note:: + + ``only_inputs`` argument is deprecated and is ignored now (defaults to ``True``). + To accumulate gradient for other parts of the graph, please use + ``torch.autograd.backward``. + + Args: + outputs (sequence of Tensor or GradientEdge): outputs of the differentiated function. + inputs (sequence of Tensor or GradientEdge): Inputs w.r.t. which the gradient will be + returned (and not accumulated into ``.grad``). + grad_outputs (sequence of Tensor): The "vector" in the vector-Jacobian product. + Usually gradients w.r.t. each output. None values can be specified for scalar + Tensors or ones that don't require grad. If a None value would be acceptable + for all grad_tensors, then this argument is optional. Default: None. + retain_graph (bool, optional): If ``False``, the graph used to compute the grad + will be freed. Note that in nearly all cases setting this option to ``True`` + is not needed and often can be worked around in a much more efficient + way. Defaults to the value of ``create_graph``. + create_graph (bool, optional): If ``True``, graph of the derivative will + be constructed, allowing to compute higher order derivative products. + Default: ``False``. + allow_unused (Optional[bool], optional): If ``False``, specifying inputs + that were not used when computing outputs (and therefore their grad is + always zero) is an error. Defaults to the value of ``materialize_grads``. + is_grads_batched (bool, optional): If ``True``, the first dimension of each + tensor in ``grad_outputs`` will be interpreted as the batch dimension. + Instead of computing a single vector-Jacobian product, we compute a + batch of vector-Jacobian products for each "vector" in the batch. + We use the vmap prototype feature as the backend to vectorize calls + to the autograd engine so that this computation can be performed in a + single call. This should lead to performance improvements when compared + to manually looping and performing backward multiple times. Note that + due to this feature being experimental, there may be performance + cliffs. Please use ``torch._C._debug_only_display_vmap_fallback_warnings(True)`` + to show any performance warnings and file an issue on github if warnings exist + for your use case. Defaults to ``False``. + materialize_grads (bool, optional): If ``True``, set the gradient for unused inputs + to zero instead of None. This is useful when computing higher-order derivatives. + If ``materialize_grads`` is ``True`` and ``allow_unused`` is ``False``, an error + will be raised. Defaults to ``False``. + + """ + if materialize_grads and allow_unused is False: + raise ValueError( + "Expected allow_unused to be True or not passed when materialize_grads=True, " + "but got: allow_unused=False." + ) + if allow_unused is None: + allow_unused = materialize_grads + if is_tensor_like(outputs) or isinstance(outputs, graph.GradientEdge): + outputs = cast( + Union[Sequence[torch.Tensor], Sequence[graph.GradientEdge]], (outputs,) + ) + else: + # pyrefly: ignore [bad-argument-type] + outputs = tuple(outputs) + if is_tensor_like(inputs) or isinstance(inputs, graph.GradientEdge): + inputs = cast(_TensorOrTensorsOrGradEdge, (inputs,)) + else: + # pyrefly: ignore [bad-argument-type] + inputs = tuple(inputs) + t_outputs = tuple(i for i in outputs if is_tensor_like(i)) + t_inputs = tuple(i for i in inputs if is_tensor_like(i)) + overridable_args = t_outputs + t_inputs + if has_torch_function(overridable_args): + return handle_torch_function( + grad, + overridable_args, + outputs, + inputs, + grad_outputs=grad_outputs, + retain_graph=retain_graph, + create_graph=create_graph, + only_inputs=only_inputs, + allow_unused=allow_unused, + is_grads_batched=is_grads_batched, + materialize_grads=materialize_grads, + ) + + if not only_inputs: + warnings.warn( + "only_inputs argument is deprecated and is ignored now " + "(defaults to True). To accumulate gradient for other " + "parts of the graph, please use torch.autograd.backward.", + FutureWarning, + stacklevel=2, + ) + + grad_outputs_ = _tensor_or_tensors_to_tuple(grad_outputs, len(outputs)) + grad_outputs_ = _make_grads( + outputs, grad_outputs_, is_grads_batched=is_grads_batched + ) + + if retain_graph is None: + retain_graph = create_graph + + # The reason we repeat the same comment several times below is because + # some Python versions print out the first line of multi-line function + # calls in the traceback and some print out the last line + if is_grads_batched: + + def vjp(gO): + return _engine_run_backward( + outputs, + gO, + retain_graph, + create_graph, + inputs, + allow_unused, + accumulate_grad=False, + ) + + result = _vmap_internals._vmap(vjp, 0, 0, allow_none_pass_through=True)( + grad_outputs_ + ) + else: + result = _engine_run_backward( + outputs, + grad_outputs_, + retain_graph, + create_graph, + inputs, + allow_unused, + accumulate_grad=False, + ) + if materialize_grads: + if any( + result[i] is None and not is_tensor_like(inputs[i]) + for i in range(len(inputs)) + ): + raise RuntimeError( + "materialize_grads cannot be used when the given input is a GradientEdge" + ) + result = tuple( + output + if output is not None + else torch.zeros_like(input, requires_grad=create_graph) + for (output, input) in zip(result, inputs) + ) + return result + + +# This function applies in case of gradient checkpointing for memory +# optimization. Currently, gradient checkpointing is supported only if the +# execution engine is invoked through torch.autograd.backward() and its +# inputs argument is not passed. It is not supported for torch.autograd.grad(). +# This is because if inputs are specified, the gradient won't be calculated for +# anything else e.g. model parameters like weights, bias etc. +# +# This function returns whether the checkpointing is valid i.e. torch.autograd.backward +# or not i.e. torch.autograd.grad. The implementation works by maintaining a thread +# local variable in torch/csrc/autograd/engine.cpp which looks at the NodeTask +# in the stack and before a NodeTask is executed in evaluate_function, it +# checks for whether reentrant backwards is imperative or not. +# See https://github.com/pytorch/pytorch/pull/4594 for more discussion/context +def _is_checkpoint_valid(): + return Variable._execution_engine.is_checkpoint_valid() + + +def variable(*args, **kwargs): # noqa: D103 + raise RuntimeError( + "torch.autograd.variable(...) is deprecated, use torch.tensor(...) instead" + ) + + +# Monkey patching variable.Variable to fix FX codegen. FX generates a call by roughly doing +# f"{fn.__module__}.{fn.__name__}(...). This yields torch.autograd.variable.Variable(...) in the +# output of an FX graph. Unfortunately the module name torch.autograd.variable is shadowed by the +# deprecated function - variable(...). +variable.Variable = Variable # type: ignore[attr-defined] + +if not torch._C._autograd_init(): + raise RuntimeError("autograd initialization failed") + +# Import all native method/classes +from torch._C._autograd import ( + _add_metadata_json, + _disable_profiler, + _disable_profiler_legacy, + _enable_profiler, + _enable_profiler_legacy, + _enable_record_function, + _get_sequence_nr, + _kineto_step, + _KinetoEvent, + _pop_saved_tensors_default_hooks, + _prepare_profiler, + _profiler_enabled, + _ProfilerResult, + _push_saved_tensors_default_hooks, + _record_function_with_args_enter, + _record_function_with_args_exit, + _set_empty_test_observer, + _supported_activities, + _toggle_collection_dynamic, + DeviceType, + kineto_available, + ProfilerEvent, + SavedTensor, +) +from torch._C._profiler import ProfilerActivity, ProfilerConfig, ProfilerState + +from . import profiler + + +def _register_py_tensor_class_for_device(device, cls): + if not isinstance(cls, type): + raise RuntimeError("cls isn't a typeinfo object") + torch._C._register_py_class_for_device(device, cls) + + +is_multithreading_enabled = torch._C._is_multithreading_enabled +torch._C._add_docstr( + is_multithreading_enabled, "Returns True if multithreading is currently enabled." +) + +is_view_replay_enabled = torch._C._is_view_replay_enabled +torch._C._add_docstr( + is_view_replay_enabled, "Returns True if view-replay is currently enabled." +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/anomaly_mode.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/anomaly_mode.py new file mode 100644 index 0000000000000000000000000000000000000000..0277f1b75541f669272c9e3334a7a7783e9b773b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/anomaly_mode.py @@ -0,0 +1,123 @@ +# mypy: allow-untyped-defs +r"""Autograd anomaly mode.""" + +import warnings + +import torch + + +__all__ = ["detect_anomaly", "set_detect_anomaly"] + + +class detect_anomaly: + r"""Context-manager that enable anomaly detection for the autograd engine. + + This does two things: + + - Running the forward pass with detection enabled will allow the backward + pass to print the traceback of the forward operation that created the failing + backward function. + - If ``check_nan`` is ``True``, any backward computation that generate "nan" + value will raise an error. Default ``True``. + + .. warning:: + This mode should be enabled only for debugging as the different tests + will slow down your program execution. + + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_ANOMALY) + >>> import torch + >>> from torch import autograd + >>> class MyFunc(autograd.Function): + ... @staticmethod + ... def forward(ctx, inp): + ... return inp.clone() + ... + ... @staticmethod + ... def backward(ctx, gO): + ... # Error during the backward pass + ... raise RuntimeError("Some error in backward") + ... return gO.clone() + >>> def run_fn(a): + ... out = MyFunc.apply(a) + ... return out.sum() + >>> inp = torch.rand(10, 10, requires_grad=True) + >>> out = run_fn(inp) + >>> out.backward() + Traceback (most recent call last): + File "", line 1, in + File "/your/pytorch/install/torch/_tensor.py", line 93, in backward + torch.autograd.backward(self, gradient, retain_graph, create_graph) + File "/your/pytorch/install/torch/autograd/__init__.py", line 90, in backward + allow_unreachable=True) # allow_unreachable flag + File "/your/pytorch/install/torch/autograd/function.py", line 76, in apply + return self._forward_cls.backward(self, *args) + File "", line 8, in backward + RuntimeError: Some error in backward + >>> with autograd.detect_anomaly(): + ... inp = torch.rand(10, 10, requires_grad=True) + ... out = run_fn(inp) + ... out.backward() + Traceback of forward call that caused the error: + File "tmp.py", line 53, in + out = run_fn(inp) + File "tmp.py", line 44, in run_fn + out = MyFunc.apply(a) + Traceback (most recent call last): + File "", line 4, in + File "/your/pytorch/install/torch/_tensor.py", line 93, in backward + torch.autograd.backward(self, gradient, retain_graph, create_graph) + File "/your/pytorch/install/torch/autograd/__init__.py", line 90, in backward + allow_unreachable=True) # allow_unreachable flag + File "/your/pytorch/install/torch/autograd/function.py", line 76, in apply + return self._forward_cls.backward(self, *args) + File "", line 8, in backward + RuntimeError: Some error in backward + + """ + + def __init__(self, check_nan=True) -> None: # noqa: D107 + self.prev = torch.is_anomaly_enabled() + self.check_nan = check_nan + self.prev_check_nan = torch.is_anomaly_check_nan_enabled() + warnings.warn( + "Anomaly Detection has been enabled. " + "This mode will increase the runtime " + "and should only be enabled for debugging.", + stacklevel=2, + ) + + def __enter__(self) -> None: # noqa: D105 + torch.set_anomaly_enabled(True, self.check_nan) + + def __exit__(self, *args: object) -> None: # noqa: D105 + torch.set_anomaly_enabled(self.prev, self.prev_check_nan) + + +class set_detect_anomaly: + r"""Context-manager that sets the anomaly detection for the autograd engine on or off. + + ``set_detect_anomaly`` will enable or disable the autograd anomaly detection + based on its argument :attr:`mode`. + It can be used as a context-manager or as a function. + + See ``detect_anomaly`` above for details of the anomaly detection behaviour. + + Args: + mode (bool): Flag whether to enable anomaly detection (``True``), + or disable (``False``). + check_nan (bool): Flag whether to raise an error when the backward + generate "nan" + + """ + + def __init__(self, mode: bool, check_nan: bool = True) -> None: # noqa: D107 + self.prev = torch.is_anomaly_enabled() + self.prev_check_nan = torch.is_anomaly_check_nan_enabled() + torch.set_anomaly_enabled(mode, check_nan) + + def __enter__(self) -> None: # noqa: D105 + pass + + def __exit__(self, *args: object) -> None: # noqa: D105 + torch.set_anomaly_enabled(self.prev, self.prev_check_nan) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/forward_ad.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/forward_ad.py new file mode 100644 index 0000000000000000000000000000000000000000..8fcb64beba3b4741b1562a682f369a1cc7009761 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/forward_ad.py @@ -0,0 +1,229 @@ +# mypy: allow-untyped-defs +import os +from typing import Any, NamedTuple, Optional + +import torch + +from .grad_mode import _DecoratorContextManager + + +__all__ = [ + "UnpackedDualTensor", + "enter_dual_level", + "exit_dual_level", + "make_dual", + "unpack_dual", + "dual_level", +] + +# Global variable used to make the python API simpler to use +_current_level = -1 + + +def enter_dual_level(): + r"""Enter a new forward grad level. + + This level can be used to make and unpack dual Tensors to compute + forward gradients. + + This function also updates the current level that is used by default + by the other functions in this API. + """ + global _current_level + new_level = torch._C._enter_dual_level() + if new_level != _current_level + 1: + raise RuntimeError( + "Entering a new forward AD level but the current level " + "is not valid. Make sure you did not modified it directly." + ) + _current_level = new_level + return new_level + + +def exit_dual_level(*, level=None): + r"""Exit a forward grad level. + + This function deletes all the gradients associated with this + level. Only deleting the latest entered level is allowed. + + This function also updates the current level that is used by default + by the other functions in this API. + """ + global _current_level + if level is None: + level = _current_level + if level != _current_level: + raise RuntimeError( + "Trying to exit a forward AD level that was not the last one " + "that was created. This is not supported." + ) + torch._C._exit_dual_level(level=level) + _current_level = level - 1 + + +def _maybe_load_decompositions(): + if os.environ.get("PYTORCH_JIT", "1") == "1" and __debug__: + from torch._decomp import decompositions_for_jvp # noqa: F401 + + +def make_dual(tensor, tangent, *, level=None): + r"""Associate a tensor value with its tangent to create a "dual tensor" for forward AD gradient computation. + + The result is a new tensor aliased to :attr:`tensor` with :attr:`tangent` embedded + as an attribute as-is if it has the same storage layout or copied otherwise. + The tangent attribute can be recovered with :func:`unpack_dual`. + + This function is backward differentiable. + + Given a function `f` whose jacobian is `J`, it allows one to compute the Jacobian-vector product (`jvp`) + between `J` and a given vector `v` as follows. + + Example:: + + >>> # xdoctest: +SKIP("Undefined variables") + >>> with dual_level(): + ... inp = make_dual(x, v) + ... out = f(inp) + ... y, jvp = unpack_dual(out) + + Please see the `forward-mode AD tutorial `__ + for detailed steps on how to use this API. + + """ + # See NOTE: [forward-mode AD decompositions mechanism] + # + # Import from torch._decomp import decompositions_for_jvp to register + # decompositions for jvp to the jit registry + # + # FIXME: We specify that __debug__ must be True because + # if python is run with -OO or -O flags (i.e., __debug__ is False), we encounter the + # following error: + # + # Return value was annotated as having type Tuple[NoneType, NoneType] but is actually of + # type Tuple[Tensor, Tensor]: + # File ".../torch/_decomp/__init__.py", line 1585 + # else: + # buffer = z + # return min - torch.log1p(z), buffer + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ <--- HERE + _maybe_load_decompositions() + + if level is None: + level = _current_level + + if level < 0: + raise RuntimeError( + "Trying to create a dual Tensor for forward AD but no level " + "exists, make sure to enter_dual_level() first." + ) + if not (tensor.is_floating_point() or tensor.is_complex()): + raise ValueError( + f"Expected primal to be floating point or complex, but got: {tensor.dtype}" + ) + if not (tangent.is_floating_point() or tangent.is_complex()): + raise ValueError( + f"Expected tangent to be floating point or complex, but got: {tangent.dtype}" + ) + + return torch._VF._make_dual(tensor, tangent, level=level) + + +class UnpackedDualTensor(NamedTuple): + r"""Namedtuple returned by :func:`unpack_dual` containing the primal and tangent components of the dual tensor. + + See :func:`unpack_dual` for more details. + """ + + primal: torch.Tensor + tangent: Optional[torch.Tensor] + + +def unpack_dual(tensor, *, level=None): + r"""Unpack a "dual tensor" to get both its Tensor value and its forward AD gradient. + + The result is a namedtuple ``(primal, tangent)`` where ``primal`` is a view of + :attr:`tensor`'s primal and ``tangent`` is :attr:`tensor`'s tangent as-is. + Neither of these tensors can be dual tensor of level :attr:`level`. + + This function is backward differentiable. + + Example:: + + >>> # xdoctest: +SKIP("Undefined variables") + >>> with dual_level(): + ... inp = make_dual(x, x_t) + ... out = f(inp) + ... y, jvp = unpack_dual(out) + ... jvp = unpack_dual(out).tangent + + Please see the `forward-mode AD tutorial `__ + for detailed steps on how to use this API. + """ + if level is None: + level = _current_level + + if level < 0: + return UnpackedDualTensor(tensor, None) + + primal, dual = torch._VF._unpack_dual(tensor, level=level) + + return UnpackedDualTensor(primal, dual) + + +class dual_level(_DecoratorContextManager): + r"""Context-manager for forward AD, where all forward AD computation must occur within the ``dual_level`` context. + + .. Note:: + + The ``dual_level`` context appropriately enters and exit the dual level to + controls the current forward AD level, which is used by default by the other + functions in this API. + + We currently don't plan to support nested ``dual_level`` contexts, however, so + only a single forward AD level is supported. To compute higher-order + forward grads, one can use :func:`torch.func.jvp`. + + Example:: + + >>> # xdoctest: +SKIP("Undefined variables") + >>> x = torch.tensor([1]) + >>> x_t = torch.tensor([1]) + >>> with dual_level(): + ... inp = make_dual(x, x_t) + ... # Do computations with inp + ... out = your_fn(inp) + ... _, grad = unpack_dual(out) + >>> grad is None + False + >>> # After exiting the level, the grad is deleted + >>> _, grad_after = unpack_dual(out) + >>> grad is None + True + + Please see the `forward-mode AD tutorial `__ + for detailed steps on how to use this API. + """ + + def __enter__(self): + return enter_dual_level() + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + exit_dual_level() + + +# Private helper functions +_is_fwd_grad_enabled = torch._C._is_fwd_grad_enabled + + +# Private helper function to enable or disable fwd grad. +# If you're a user and want to use this, please file an issue to discuss the use case. +class _set_fwd_grad_enabled(_DecoratorContextManager): + def __init__(self, mode: bool) -> None: + self.prev = _is_fwd_grad_enabled() + torch._C._set_fwd_grad_enabled(mode) + + def __enter__(self) -> None: + pass + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_fwd_grad_enabled(self.prev) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/function.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/function.py new file mode 100644 index 0000000000000000000000000000000000000000..70c6cad4e99678ac86ddd6d2cafca182ca8047f4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/function.py @@ -0,0 +1,856 @@ +# mypy: allow-untyped-defs +import functools +import inspect +import itertools +import warnings +from collections import OrderedDict +from collections.abc import Callable +from typing import Any, Concatenate, Optional, TypeVar +from typing_extensions import deprecated, ParamSpec + +import torch +import torch._C as _C +import torch._functorch as _functorch +import torch.utils.hooks as hooks +from torch._C import _functions +from torch._functorch.autograd_function import custom_function_call + + +__all__ = [ + "FunctionCtx", + "BackwardCFunction", + "FunctionMeta", + "Function", + "once_differentiable", + "InplaceFunction", + "NestedIOFunction", +] + +# Unique id provider for each class inheriting from Function +# This is incremented in FunctionMeta during class definition +AUTOGRAD_FUNCTION_COUNTER = itertools.count() + +_T = TypeVar("_T") +_R = TypeVar("_R") +_P = ParamSpec("_P") + + +# Formerly known as: _ContextMethodMixin +class FunctionCtx: + def save_for_backward(self, *tensors: torch.Tensor): + r"""Save given tensors for a future call to :func:`~Function.backward`. + + ``save_for_backward`` should be called at most once, in either the + :func:`setup_context` or :func:`forward` methods, and only with tensors. + + All tensors intended to be used in the backward pass should be saved + with ``save_for_backward`` (as opposed to directly on ``ctx``) to prevent + incorrect gradients and memory leaks, and enable the application of saved + tensor hooks. See :class:`torch.autograd.graph.saved_tensors_hooks`. + See :ref:`extending-autograd` for more details. + + Note that if intermediary tensors, tensors that are neither inputs + nor outputs of :func:`forward`, are saved for backward, your custom Function + may not support double backward. + Custom Functions that do not support double backward should decorate their + :func:`backward` method with ``@once_differentiable`` so that performing + double backward raises an error. If you'd like to support double backward, + you can either recompute intermediaries based on the inputs during backward + or return the intermediaries as the outputs of the custom Function. See the + `double backward tutorial `_ + for more details. + + In :func:`backward`, saved tensors can be accessed through the :attr:`saved_tensors` + attribute. Before returning them to the user, a check is made to ensure + they weren't used in any in-place operation that modified their content. + + Arguments can also be ``None``. This is a no-op. + + See :ref:`extending-autograd` for more details on how to use this method. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> class Func(Function): + >>> @staticmethod + >>> def forward(ctx, x: torch.Tensor, y: torch.Tensor, z: int): + >>> w = x * z + >>> out = x * y + y * z + w * y + >>> ctx.save_for_backward(x, y, w, out) + >>> ctx.z = z # z is not a tensor + >>> return out + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, grad_out): + >>> x, y, w, out = ctx.saved_tensors + >>> z = ctx.z + >>> gx = grad_out * (y + y * z) + >>> gy = grad_out * (x + z + w) + >>> gz = None + >>> return gx, gy, gz + >>> + >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double) + >>> b = torch.tensor(2., requires_grad=True, dtype=torch.double) + >>> c = 4 + >>> d = Func.apply(a, b, c) + + """ + self.to_save = tensors + + def save_for_forward(self, *tensors: torch.Tensor): + r"""Save given tensors for a future call to :func:`~Function.jvp`. + + ``save_for_forward`` should be called at most once, in either the + :func:`setup_context` or :func:`forward` methods, and all arguments + should be tensors. + + In :func:`jvp`, saved objects can be accessed through the :attr:`saved_tensors` + attribute. + + Arguments can also be ``None``. This is a no-op. + + See :ref:`extending-autograd` for more details on how to use this method. + + Example:: + + >>> # xdoctest: +SKIP + >>> class Func(torch.autograd.Function): + >>> @staticmethod + >>> def forward(ctx, x: torch.Tensor, y: torch.Tensor, z: int): + >>> ctx.save_for_backward(x, y) + >>> ctx.save_for_forward(x, y) + >>> ctx.z = z + >>> return x * y * z + >>> + >>> @staticmethod + >>> def jvp(ctx, x_t, y_t, _): + >>> x, y = ctx.saved_tensors + >>> z = ctx.z + >>> return z * (y * x_t + x * y_t) + >>> + >>> @staticmethod + >>> def vjp(ctx, grad_out): + >>> x, y = ctx.saved_tensors + >>> z = ctx.z + >>> return z * grad_out * y, z * grad_out * x, None + >>> + >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double) + >>> t = torch.tensor(1., dtype=torch.double) + >>> b = torch.tensor(2., requires_grad=True, dtype=torch.double) + >>> c = 4 + >>> + >>> with fwAD.dual_level(): + >>> a_dual = fwAD.make_dual(a, t) + >>> d = Func.apply(a_dual, b, c) + + """ + for tensor in tensors: + if not (isinstance(tensor, torch.Tensor) or tensor is None): + raise AssertionError( + "save_for_forward expects all arguments to be tensors; you should " + "save non-tensors as attributes on ctx." + ) + + self.saved_for_forward = tensors + + def mark_dirty(self, *args: torch.Tensor): + r"""Mark given tensors as modified in an in-place operation. + + This should be called at most once, in either the :func:`setup_context` + or :func:`forward` methods, and all arguments should be inputs. + + Every tensor that's been modified in-place in a call to :func:`forward` + should be given to this function, to ensure correctness of our checks. + It doesn't matter whether the function is called before or after + modification. + + Examples:: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> class Inplace(Function): + >>> @staticmethod + >>> def forward(ctx, x): + >>> x_npy = x.numpy() # x_npy shares storage with x + >>> x_npy += 1 + >>> ctx.mark_dirty(x) + >>> return x + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, grad_output): + >>> return grad_output + >>> + >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double).clone() + >>> b = a * a + >>> Inplace.apply(a) # This would lead to wrong gradients! + >>> # but the engine would not know unless we mark_dirty + >>> # xdoctest: +SKIP + >>> b.backward() # RuntimeError: one of the variables needed for gradient + >>> # computation has been modified by an inplace operation + + """ + self.dirty_tensors = args + + @deprecated( + "`mark_shared_storage` is deprecated. " + "Tensors with shared storages are automatically tracked. " + "Note that calls to `set_()` are not tracked", + category=FutureWarning, + ) + def mark_shared_storage(self, *pairs): + pass + + def mark_non_differentiable(self, *args: torch.Tensor): + r"""Mark outputs as non-differentiable. + + This should be called at most once, in either the :func:`setup_context` + or :func:`forward` methods, and all arguments should be tensor outputs. + + This will mark outputs as not requiring gradients, increasing the + efficiency of backward computation. You still need to accept a gradient + for each output in :meth:`~Function.backward`, but it's always going to + be a zero tensor with the same shape as the shape of a corresponding + output. + + This is used e.g. for indices returned from a sort. See example:: + >>> class Func(Function): + >>> @staticmethod + >>> def forward(ctx, x): + >>> sorted, idx = x.sort() + >>> ctx.mark_non_differentiable(idx) + >>> ctx.save_for_backward(x, idx) + >>> return sorted, idx + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, g1, g2): # still need to accept g2 + >>> x, idx = ctx.saved_tensors + >>> grad_input = torch.zeros_like(x) + >>> grad_input.index_add_(0, idx, g1) + >>> return grad_input + + """ + self.non_differentiable = args + + def set_materialize_grads(self, value: bool): + r"""Set whether to materialize grad tensors. Default is ``True``. + + This should be called only from either the :func:`setup_context` or + :func:`forward` methods. + + If ``True``, undefined grad tensors will be expanded to tensors full of zeros + prior to calling the :func:`backward` and :func:`jvp` methods. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> class SimpleFunc(Function): + >>> @staticmethod + >>> def forward(ctx, x): + >>> return x.clone(), x.clone() + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, g1, g2): + >>> return g1 + g2 # No check for None necessary + >>> + >>> # We modify SimpleFunc to handle non-materialized grad outputs + >>> class Func(Function): + >>> @staticmethod + >>> def forward(ctx, x): + >>> ctx.set_materialize_grads(False) + >>> ctx.save_for_backward(x) + >>> return x.clone(), x.clone() + >>> + >>> @staticmethod + >>> @once_differentiable + >>> def backward(ctx, g1, g2): + >>> x, = ctx.saved_tensors + >>> grad_input = torch.zeros_like(x) + >>> if g1 is not None: # We must check for None now + >>> grad_input += g1 + >>> if g2 is not None: + >>> grad_input += g2 + >>> return grad_input + >>> + >>> a = torch.tensor(1., requires_grad=True) + >>> b, _ = Func.apply(a) # induces g2 to be undefined + + """ + self.materialize_grads = value + + +# DO NOT USE: This is only defined to be able to load old serialized models +_ContextMethodMixin = FunctionCtx + + +class _HookMixin: + @staticmethod + def _register_hook(backward_hooks, hook): + if backward_hooks is None: + backward_hooks = OrderedDict() + handle = hooks.RemovableHandle(backward_hooks) + backward_hooks[handle.id] = hook + return backward_hooks, handle + + +class BackwardCFunction(_C._FunctionBase, FunctionCtx, _HookMixin): + r""" + This class is used for internal autograd work. Do not use. + """ + + def apply(self, *args): + r""" + Apply method used when executing this Node during the backward + """ + # _forward_cls is defined by derived class + # The user should define either backward or vjp but never both. + backward_fn = self._forward_cls.backward # type: ignore[attr-defined] + vjp_fn = self._forward_cls.vjp # type: ignore[attr-defined] + if backward_fn is not Function.backward and vjp_fn is not Function.vjp: + raise RuntimeError( + "Implementing both 'backward' and 'vjp' for a custom " + "Function is not allowed. You should only implement one " + "of them." + ) + user_fn = vjp_fn if vjp_fn is not Function.vjp else backward_fn + return user_fn(self, *args) + + def apply_jvp(self, *args): + r""" + Apply method used when executing forward mode AD during the forward + """ + # _forward_cls is defined by derived class + return self._forward_cls.jvp(self, *args) # type: ignore[attr-defined] + + def _compiled_autograd_key(self): + return self._forward_cls._compiled_autograd_key(self) # type: ignore[attr-defined] + + +class FunctionMeta(type): + """Function metaclass. + + This metaclass sets up the following properties: + _backward_cls: The Function class corresponding to the differentiated + version of this function (which is generated on the fly by this + metaclass). + """ + + def __init__(cls, name, bases, attrs): + backward_fn = type( + name + "Backward", (BackwardCFunction,), {"_forward_cls": cls} + ) + backward_fn._autograd_function_id = next(AUTOGRAD_FUNCTION_COUNTER) # type: ignore[attr-defined] + cls._backward_cls = backward_fn + + super().__init__(name, bases, attrs) + + +class _SingleLevelFunction( + _C._FunctionBase, FunctionCtx, _HookMixin, metaclass=FunctionMeta +): + @staticmethod + def forward(*args: Any, **kwargs: Any) -> Any: + r"""Define the forward of the custom autograd Function. + + This function is to be overridden by all subclasses. + There are two ways to define forward: + + Usage 1 (Combined forward and ctx):: + + @staticmethod + def forward(ctx: Any, *args: Any, **kwargs: Any) -> Any: + pass + + - It must accept a context ctx as the first argument, followed by any + number of arguments (tensors or other types). + - See :ref:`combining-forward-context` for more details + + Usage 2 (Separate forward and ctx):: + + @staticmethod + def forward(*args: Any, **kwargs: Any) -> Any: + pass + + + @staticmethod + def setup_context(ctx: Any, inputs: Tuple[Any, ...], output: Any) -> None: + pass + + - The forward no longer accepts a ctx argument. + - Instead, you must also override the :meth:`torch.autograd.Function.setup_context` + staticmethod to handle setting up the ``ctx`` object. + ``output`` is the output of the forward, ``inputs`` are a Tuple of inputs + to the forward. + - See :ref:`extending-autograd` for more details + + The context can be used to store arbitrary data that can be then + retrieved during the backward pass. Tensors should not be stored + directly on `ctx` (though this is not currently enforced for + backward compatibility). Instead, tensors should be saved either with + :func:`ctx.save_for_backward` if they are intended to be used in + ``backward`` (equivalently, ``vjp``) or :func:`ctx.save_for_forward` + if they are intended to be used for in ``jvp``. + """ + raise NotImplementedError( + "You must implement the forward function for custom autograd.Function." + ) + + @staticmethod + def setup_context(ctx: Any, inputs: tuple[Any, ...], output: Any) -> Any: + r"""There are two ways to define the forward pass of an autograd.Function. + + Either: + + 1. Override forward with the signature ``forward(ctx, *args, **kwargs)``. + ``setup_context`` is not overridden. Setting up the ctx for backward + happens inside the ``forward``. + 2. Override forward with the signature ``forward(*args, **kwargs)`` and + override ``setup_context``. Setting up the ctx for backward happens + inside ``setup_context`` (as opposed to inside the ``forward``) + + See :meth:`torch.autograd.Function.forward` and :ref:`extending-autograd` for more details. + """ + raise NotImplementedError("setup_context is not implemented.") + + @staticmethod + def backward(ctx: Any, *grad_outputs: Any) -> Any: + r"""Define a formula for differentiating the operation with backward mode automatic differentiation. + + This function is to be overridden by all subclasses. + (Defining this function is equivalent to defining the ``vjp`` function.) + + It must accept a context :attr:`ctx` as the first argument, followed by + as many outputs as the :func:`forward` returned (None will be passed in + for non tensor outputs of the forward function), + and it should return as many tensors, as there were inputs to + :func:`forward`. Each argument is the gradient w.r.t the given output, + and each returned value should be the gradient w.r.t. the + corresponding input. If an input is not a Tensor or is a Tensor not + requiring grads, you can just pass None as a gradient for that input. + + The context can be used to retrieve tensors saved during the forward + pass. It also has an attribute :attr:`ctx.needs_input_grad` as a tuple + of booleans representing whether each input needs gradient. E.g., + :func:`backward` will have ``ctx.needs_input_grad[0] = True`` if the + first input to :func:`forward` needs gradient computed w.r.t. the + output. + """ + raise NotImplementedError( + "You must implement either the backward or vjp method for " + "your custom autograd.Function to use it with backward " + "mode AD." + ) + + # vjp and backward are alias of each other + vjp = backward + + @staticmethod + def jvp(ctx: Any, *grad_inputs: Any) -> Any: + r"""Define a formula for differentiating the operation with forward mode automatic differentiation. + + This function is to be overridden by all subclasses. + It must accept a context :attr:`ctx` as the first argument, followed by + as many inputs as the :func:`forward` got (None will be passed in + for non tensor inputs of the forward function), + and it should return as many tensors as there were outputs to + :func:`forward`. Each argument is the gradient w.r.t the given input, + and each returned value should be the gradient w.r.t. the + corresponding output. If an output is not a Tensor or the function is not + differentiable with respect to that output, you can just pass None as a + gradient for that input. + + You can use the :attr:`ctx` object to pass any value from the forward to this + functions. + """ + raise NotImplementedError( + "You must implement the jvp function for custom " + "autograd.Function to use it with forward mode AD." + ) + + +class Function(_SingleLevelFunction): + r"""Base class to create custom `autograd.Function`. + + To create a custom `autograd.Function`, subclass this class and implement + the :meth:`forward` and :meth:`backward` static methods. Then, to use your custom + op in the forward pass, call the class method ``apply``. Do not call + :meth:`forward` directly. + + To ensure correctness and best performance, make sure you are calling the + correct methods on ``ctx`` and validating your backward function using + :func:`torch.autograd.gradcheck`. + + See :ref:`extending-autograd` for more details on how to use this class. + + Examples:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> class Exp(Function): + >>> @staticmethod + >>> def forward(ctx, i): + >>> result = i.exp() + >>> ctx.save_for_backward(result) + >>> return result + >>> + >>> @staticmethod + >>> def backward(ctx, grad_output): + >>> result, = ctx.saved_tensors + >>> return grad_output * result + >>> + >>> # Use it by calling the apply method: + >>> # xdoctest: +SKIP + >>> output = Exp.apply(input) + """ + + def __init__(self, *args, **kwargs): + warnings.warn( + f"{self.__class__} should not be instantiated. Methods on autograd functions " + "are all static, so you should invoke them on the class itself. " + "Instantiating an autograd function will raise an " + "error in a future version of PyTorch.", + DeprecationWarning, + stacklevel=2, + ) + + def __call__(self, *args, **kwargs): + raise RuntimeError( + "Legacy autograd function with non-static forward method is deprecated. " + "Please use new-style autograd function with static forward method. " + "(Example: https://pytorch.org/docs/stable/autograd.html#torch.autograd.Function)" + ) + + """ + Bool that specifies if PyTorch should attempt to autogenerate + :func:`torch.vmap` support for this autograd.Function. You may set this to + True only if this autograd.Function's forward, backward, and jvp (if they + exist) are written using PyTorch operations; otherwise, please override + :meth:`torch.autograd.Function.vmap` to add support for :func:`torch.vmap`. + + Please see :ref:`func-autograd-function` for more details. + """ + generate_vmap_rule = False + + @staticmethod + def vmap(info, in_dims, *args): + r"""Define the behavior for this autograd.Function underneath :func:`torch.vmap`. + + For a :func:`torch.autograd.Function` to support + :func:`torch.vmap`, you must either override this static method, or set + ``generate_vmap_rule`` to ``True`` (you may not do both). + + If you choose to override this staticmethod: it must accept + + - an ``info`` object as the first argument. ``info.batch_size`` + specifies the size of the dimension being vmapped over, + while ``info.randomness`` is the randomness option passed to + :func:`torch.vmap`. + - an ``in_dims`` tuple as the second argument. + For each arg in ``args``, ``in_dims`` has a corresponding + ``Optional[int]``. It is ``None`` if the arg is not a Tensor or if + the arg is not being vmapped over, otherwise, it is an integer + specifying what dimension of the Tensor is being vmapped over. + - ``*args``, which is the same as the args to :meth:`~Function.forward`. + + The return of the vmap staticmethod is a tuple of ``(output, out_dims)``. + Similar to ``in_dims``, ``out_dims`` should be of the same structure as + ``output`` and contain one ``out_dim`` per output that specifies if the + output has the vmapped dimension and what index it is in. + + Please see :ref:`func-autograd-function` for more details. + """ + raise NotImplementedError( + "To use autograd.Function with vmap, you must either override the " + "vmap staticmethod or set generate_vmap_rule=True." + ) + + @classmethod + def apply(cls, *args, **kwargs): + def bind_default_args(func, *args, **kwargs): + signature = inspect.signature(func) + bound_args = signature.bind(*args, **kwargs) + bound_args.apply_defaults() + + return bound_args.args + + is_setup_ctx_defined = _is_setup_context_defined(cls.setup_context) + if is_setup_ctx_defined: + args = bind_default_args(cls.forward, *args, **kwargs) + + if not torch._C._are_functorch_transforms_active(): + # See NOTE: [functorch vjp and autograd interaction] + args = _functorch.utils.unwrap_dead_wrappers(args) + return super().apply(*args, **kwargs) # type: ignore[misc] + + if not is_setup_ctx_defined: + raise RuntimeError( + "In order to use an autograd.Function with functorch transforms " + "(vmap, grad, jvp, jacrev, ...), it must override the setup_context " + "staticmethod. For more details, please see " + "https://pytorch.org/docs/main/notes/extending.func.html" + ) + + return custom_function_call(cls, *args, **kwargs) + + @staticmethod + def _compiled_autograd_key(ctx): + return (ctx._autograd_function_id,) + + +def _is_setup_context_defined(fn): + return fn != _SingleLevelFunction.setup_context + + +def once_differentiable( + fn: Callable[Concatenate[_T, _P], _R], +) -> Callable[Concatenate[_T, _P], _R]: + @functools.wraps(fn) + def wrapper(ctx: _T, *args: _P.args, **kwargs: _P.kwargs) -> _R: + with torch.no_grad(): + outputs = fn(ctx, *args, **kwargs) + + if not torch.is_grad_enabled(): + return outputs + + # If any of the inputs have requires_grad=True, we force the outputs + # to have requires_grad=True but point to a grad_fn which throws an + # error message during (double) back-propagation. + # XXX: this is only an approximation of requires_grad - there's no way + # to figure out if fn didn't use ctx.saved_tensors and as a result + # some Tensors might require grad, even if no args do. + # Unfortunately, this leads to unexpected error messages ("no nodes + # require computing gradients"), but I don't have a better idea. + # These functions would raise an error in backward anyway. + requires_grad = any( + isinstance(arg, torch.Tensor) and arg.requires_grad for arg in args + ) + if not requires_grad: + return outputs + + if not isinstance(outputs, tuple): + outputs_ = (outputs,) + else: + outputs_ = outputs + + err_fn = _functions.DelayedError( + b"trying to differentiate twice a function that was marked " + b"with @once_differentiable", + len(outputs_), + ) + + # Create aliases of each output that has requires_grad=True. We need + # at least one of the inputs to err_fn to require grad so that the + # output will have a grad_fn. + def fake_requires_grad(var): + if var is not None: + var = var.detach() + var.requires_grad = True + return var + + return err_fn(*[fake_requires_grad(v) for v in outputs_]) # type: ignore[return-value] + + return wrapper + + +class InplaceFunction(Function): + r""" + This class is here only for backward compatibility reasons. + Use :class:`Function` instead of this for any new use case. + """ + + def __init__(self, inplace=False): + super().__init__() + self.inplace = inplace + + +def _nested_map(condition, fn, condition_msg=None): + def _map(obj): + if condition(obj): + return fn(obj) + elif obj is None: + return None + elif isinstance(obj, (list, tuple)): + mapped = (_map(x) for x in obj) + if hasattr(obj, "_fields"): + # obj is namedtuple + return type(obj)(*mapped) + return type(obj)(mapped) + elif isinstance(obj, dict): + return {x: _map(obj[x]) for x in obj} + else: + raise ValueError( + "Auto nesting doesn't know how to process " + "an input object of type " + + torch.typename(obj) + + ( + ". Accepted types: " + condition_msg + ", or lists/tuples of them" + if condition_msg + else "" + ) + ) + + return _map + + +def _jit_unwrap_structured(obj): + if hasattr(obj, "_jit_unwrap"): + return obj._jit_unwrap() + return obj + + +def _iter_filter(condition, allow_unknown=False, condition_msg=None, conversion=None): + def _iter(obj): + if conversion is not None: + obj = conversion(obj) + if condition(obj): + yield obj + elif obj is None: + return + elif isinstance(obj, (list, tuple)): + for o in obj: + yield from _iter(o) + elif isinstance(obj, dict): + # We only accept primitive key types, so we needn't inspect them + for o in obj.values(): + yield from _iter(o) + elif allow_unknown: + yield obj + else: + raise ValueError( + "Auto nesting doesn't know how to process " + "an input object of type " + + torch.typename(obj) + + ( + ". Accepted types: " + condition_msg + ", or lists/tuples of them" + if condition_msg + else "" + ) + ) + + return _iter + + +def _unflatten(input, proto): + # unflatten a list or tuple input into a nested list/tuple structure + # specified by proto + def unflatten_helper(input, proto): + res: list[Optional[torch.Tensor]] = [] + if hasattr(proto, "_jit_wrap"): + return proto._jit_wrap(input) + if not isinstance(proto, (list, tuple)): + return input[0], input[1:] + for e in proto: + if e is None: + res.append(e) + else: + res_e, input = unflatten_helper(input, e) + res.append(res_e) + return type(proto)(res), input + + return unflatten_helper(input, proto)[0] + + +_iter_jit_values = _iter_filter( + lambda o: o is None or isinstance(o, torch._C.Value), + condition_msg="jit's Values or None", +) +_iter_tensors = _iter_filter( + lambda x: isinstance(x, torch.Tensor), + condition_msg="Tensors", + conversion=_jit_unwrap_structured, +) +_iter_tensors_permissive = _iter_filter( + lambda x: isinstance(x, torch.Tensor), + allow_unknown=True, + condition_msg="Tensors (permissive)", +) +_iter_None_tensors = _iter_filter( + lambda o: o is None or isinstance(o, torch.Tensor), condition_msg="Tensors or None" +) +_map_tensor_data = _nested_map( + lambda x: isinstance(x, torch.Tensor), lambda o: o.data, condition_msg="Tensors" +) + + +class NestedIOFunction(Function): + r""" + This class is here only for backward compatibility reasons. + Use :class:`Function` instead of this for any new use case. + """ + + # The 'type: ignore' statements are needed here because these functions are declared as '@staticmethod' in the + # superclass (Function) but are instance methods here, which mypy reports as incompatible. + + def _do_forward(self, *input): + self._nested_input = input + flat_input = tuple(_iter_tensors(input)) + flat_output = super()._do_forward(*flat_input) # type: ignore[misc] + nested_tensors = _unflatten(flat_output, self._nested_output) + return nested_tensors + + def _do_backward(self, gradients, retain_variables): + self.retain_variables = retain_variables + result = super()._do_backward(gradients, retain_variables) # type: ignore[misc] + if not retain_variables: + del self._nested_output + del self._to_save_nested + return result + + def backward(self, *gradients: Any) -> Any: # type: ignore[override] + r""" + Shared backward utility. + """ + nested_gradients = _unflatten(gradients, self._nested_output) + result = self.backward_extended(*nested_gradients) # type: ignore[func-returns-value] + return tuple(_iter_None_tensors(result)) + + __call__ = _do_forward + + def forward(self, *args: Any) -> Any: # type: ignore[override] + r""" + Shared forward utility. + """ + nested_tensors = _map_tensor_data(self._nested_input) + result = self.forward_extended(*nested_tensors) # type: ignore[func-returns-value] + del self._nested_input + self._nested_output = result + return tuple(_iter_tensors(result)) + + def save_for_backward(self, *args: Any) -> None: + r""" + See :meth:`Function.save_for_backward`. + """ + self.to_save = tuple(_iter_tensors(args)) + self._to_save_nested = args + + @property + def saved_tensors(self): # type: ignore[override] + r""" + See :meth:`Function.saved_tensors`. + """ + flat_tensors = super().saved_tensors # type: ignore[misc] + return _unflatten(flat_tensors, self._to_save_nested) + + def mark_dirty(self, *args: Any, **kwargs: Any) -> None: + r""" + See :meth:`Function.mark_dirty`. + """ + self.dirty_tensors = tuple(_iter_tensors((args, kwargs))) + + def mark_non_differentiable(self, *args: Any, **kwargs: Any) -> None: + r""" + See :meth:`Function.mark_non_differentiable`. + """ + self.non_differentiable = tuple(_iter_tensors((args, kwargs))) + + def forward_extended(self, *input: Any) -> None: + r""" + User defined forward. + """ + raise NotImplementedError + + def backward_extended(self, *grad_output: Any) -> None: + r""" + User defined backward. + """ + raise NotImplementedError diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/functional.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/functional.py new file mode 100644 index 0000000000000000000000000000000000000000..e8bce9ed7c54f0a333ea8043c3fab60d46b7b94c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/functional.py @@ -0,0 +1,1210 @@ +# mypy: allow-untyped-defs + +import torch +from torch._vmap_internals import _vmap + +from . import forward_ad as fwAD + + +__all__ = ["vjp", "jvp", "jacobian", "hessian", "hvp", "vhp"] + +# Utility functions + + +def _as_tuple_nocheck(x): + if isinstance(x, tuple): + return x + elif isinstance(x, list): + return tuple(x) + else: + return (x,) + + +def _as_tuple(inp, arg_name=None, fn_name=None): + # Ensures that inp is a tuple of Tensors + # Returns whether or not the original inp was a tuple and the tupled version of the input + if arg_name is None and fn_name is None: + return _as_tuple_nocheck(inp) + + is_inp_tuple = True + if not isinstance(inp, tuple): + inp = (inp,) + is_inp_tuple = False + + for i, el in enumerate(inp): + if not isinstance(el, torch.Tensor): + if is_inp_tuple: + raise TypeError( + f"The {arg_name} given to {fn_name} must be either a Tensor or a tuple of Tensors but the" + f" value at index {i} has type {type(el)}." + ) + else: + raise TypeError( + f"The {arg_name} given to {fn_name} must be either a Tensor or a tuple of Tensors but the" + f" given {arg_name} has type {type(el)}." + ) + + return is_inp_tuple, inp + + +def _tuple_postprocess(res, to_unpack): + # Unpacks a potentially nested tuple of Tensors + # to_unpack should be a single boolean or a tuple of two booleans. + # It is used to: + # - invert _as_tuple when res should match the inp given to _as_tuple + # - optionally remove nesting of two tuples created by multiple calls to _as_tuple + if isinstance(to_unpack, tuple): + if len(to_unpack) != 2: + raise AssertionError("Expected to_unpack tuple to have exactly 2 elements") + if not to_unpack[1]: + res = tuple(el[0] for el in res) + if not to_unpack[0]: + res = res[0] + else: + if not to_unpack: + res = res[0] + return res + + +def _grad_preprocess(inputs, create_graph, need_graph): + # Preprocess the inputs to make sure they require gradient + # inputs is a tuple of Tensors to preprocess + # create_graph specifies if the user wants gradients to flow back to the Tensors in inputs + # need_graph specifies if we internally want gradients to flow back to the Tensors in res + # Note that we *always* create a new Tensor object to be able to see the difference between + # inputs given as arguments and the same Tensors automatically captured by the user function. + # Check this issue for more details on how that can happen: https://github.com/pytorch/pytorch/issues/32576 + res = [] + for inp in inputs: + if create_graph and inp.requires_grad: + # Create at least a new Tensor object in a differentiable way + if not inp.is_sparse: + # Use .view_as() to get a shallow copy + res.append(inp.view_as(inp)) + else: + # We cannot use view for sparse Tensors so we clone + res.append(inp.clone()) + else: + res.append(inp.detach().requires_grad_(need_graph)) + return tuple(res) + + +def _grad_postprocess(inputs, create_graph): + # Postprocess the generated Tensors to avoid returning Tensors with history when the user did not + # request it. + if isinstance(inputs[0], torch.Tensor): + if not create_graph: + return tuple(inp.detach() for inp in inputs) + else: + return inputs + else: + return tuple(_grad_postprocess(inp, create_graph) for inp in inputs) + + +def _validate_v(v, other, is_other_tuple): + # This assumes that other is the correct shape, and v should match + # Both are assumed to be tuples of Tensors + if len(other) != len(v): + if is_other_tuple: + raise RuntimeError( + f"v is a tuple of invalid length: should be {len(other)} but got {len(v)}." + ) + else: + raise RuntimeError("The given v should contain a single Tensor.") + + for idx, (el_v, el_other) in enumerate(zip(v, other)): + if el_v.size() != el_other.size(): + prepend = "" + if is_other_tuple: + prepend = f"Entry {idx} in " + raise RuntimeError( + f"{prepend}v has invalid size: should be {el_other.size()} but got {el_v.size()}." + ) + + +def _check_requires_grad(inputs, input_type, strict): + # Used to make all the necessary checks to raise nice errors in strict mode. + if not strict: + return + + if input_type not in ["outputs", "grad_inputs", "jacobian", "hessian"]: + raise RuntimeError("Invalid input_type to _check_requires_grad") + for i, inp in enumerate(inputs): + if inp is None: + # This can only be reached for grad_inputs. + raise RuntimeError( + f"The output of the user-provided function is independent of input {i}." + " This is not allowed in strict mode." + ) + if not inp.requires_grad: + if input_type == "hessian": + raise RuntimeError( + f"The hessian of the user-provided function with respect to input {i}" + " is independent of the input. This is not allowed in strict mode." + " You should ensure that your function is thrice differentiable and that" + " the hessian depends on the inputs." + ) + elif input_type == "jacobian": + raise RuntimeError( + "While computing the hessian, found that the jacobian of the user-provided" + f" function with respect to input {i} is independent of the input. This is not" + " allowed in strict mode. You should ensure that your function is twice" + " differentiable and that the jacobian depends on the inputs (this would be" + " violated by a linear function for example)." + ) + elif input_type == "grad_inputs": + raise RuntimeError( + f"The gradient with respect to input {i} is independent of the inputs of the" + " user-provided function. This is not allowed in strict mode." + ) + else: + raise RuntimeError( + f"Output {i} of the user-provided function does not require gradients." + " The outputs must be computed in a differentiable manner from the input" + " when running in strict mode." + ) + + +def _autograd_grad( + outputs, + inputs, + grad_outputs=None, + create_graph=False, + retain_graph=None, + is_grads_batched=False, +): + # Version of autograd.grad that accepts `None` in outputs and do not compute gradients for them. + # This has the extra constraint that inputs has to be a tuple + if not isinstance(outputs, tuple): + raise AssertionError("Expected outputs to be a tuple") + if grad_outputs is None: + grad_outputs = (None,) * len(outputs) + if not isinstance(grad_outputs, tuple): + raise AssertionError("Expected grad_outputs to be a tuple") + if len(outputs) != len(grad_outputs): + raise AssertionError( + f"Expected outputs and grad_outputs to have the same length, " + f"but got {len(outputs)} and {len(grad_outputs)}" + ) + + new_outputs: tuple[torch.Tensor, ...] = () + new_grad_outputs: tuple[torch.Tensor, ...] = () + for out, grad_out in zip(outputs, grad_outputs): + if out is not None and out.requires_grad: + new_outputs += (out,) + new_grad_outputs += (grad_out,) + + if len(new_outputs) == 0: + # No differentiable output, we don't need to call the autograd engine + return (None,) * len(inputs) + else: + return torch.autograd.grad( + new_outputs, + inputs, + new_grad_outputs, + allow_unused=True, + create_graph=create_graph, + retain_graph=retain_graph, + is_grads_batched=is_grads_batched, + ) + + +def _fill_in_zeros(grads, refs, strict, create_graph, stage): + # Used to detect None in the grads and depending on the flags, either replace them + # with Tensors full of 0s of the appropriate size based on the refs or raise an error. + # strict and create graph allow us to detect when it is appropriate to raise an error + # stage gives us information of which backward call we consider to give good error message + if stage not in ["back", "back_trick", "double_back", "double_back_trick"]: + raise RuntimeError(f"Invalid stage argument '{stage}' to _fill_in_zeros") + + res: tuple[torch.Tensor, ...] = () + for i, grads_i in enumerate(grads): + if grads_i is None: + if strict: + if stage == "back": + raise RuntimeError( + "The output of the user-provided function is independent of " + f"input {i}. This is not allowed in strict mode." + ) + elif stage == "back_trick": + raise RuntimeError( + f"The gradient with respect to the input is independent of entry {i}" + " in the grad_outputs when using the double backward trick to compute" + " forward mode gradients. This is not allowed in strict mode." + ) + elif stage == "double_back": + raise RuntimeError( + "The jacobian of the user-provided function is independent of " + f"input {i}. This is not allowed in strict mode." + ) + else: + raise RuntimeError( + "The hessian of the user-provided function is independent of " + f"entry {i} in the grad_jacobian. This is not allowed in strict " + "mode as it prevents from using the double backward trick to " + "replace forward mode AD." + ) + + grads_i = torch.zeros_like(refs[i]) + else: + if strict and create_graph and not grads_i.requires_grad: + if "double" not in stage: + raise RuntimeError( + "The jacobian of the user-provided function is independent of " + f"input {i}. This is not allowed in strict mode when create_graph=True." + ) + else: + raise RuntimeError( + "The hessian of the user-provided function is independent of " + f"input {i}. This is not allowed in strict mode when create_graph=True." + ) + + res += (grads_i,) + + return res + + +# Public API + + +def vjp(func, inputs, v=None, create_graph=False, strict=False): + r"""Compute the dot product between a vector ``v`` and the Jacobian of the given function at the point given by the inputs. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a tuple of Tensors or a Tensor. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + v (tuple of Tensors or Tensor): The vector for which the vector + Jacobian product is computed. Must be the same size as the output + of ``func``. This argument is optional when the output of ``func`` + contains a single element and (if it is not provided) will be set + as a Tensor containing a single ``1``. + create_graph (bool, optional): If ``True``, both the output and result + will be computed in a differentiable way. Note that when ``strict`` + is ``False``, the result can not require gradients or be + disconnected from the inputs. Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + vjp for said inputs, which is the expected mathematical value. + Defaults to ``False``. + + Returns: + output (tuple): tuple with: + func_output (tuple of Tensors or Tensor): output of ``func(inputs)`` + + vjp (tuple of Tensors or Tensor): result of the dot product with + the same shape as the inputs. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def exp_reducer(x): + ... return x.exp().sum(dim=1) + >>> inputs = torch.rand(4, 4) + >>> v = torch.ones(4) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> vjp(exp_reducer, inputs, v) + (tensor([5.7817, 7.2458, 5.7830, 6.7782]), + tensor([[1.4458, 1.3962, 1.3042, 1.6354], + [2.1288, 1.0652, 1.5483, 2.5035], + [2.2046, 1.1292, 1.1432, 1.3059], + [1.3225, 1.6652, 1.7753, 2.0152]])) + + >>> vjp(exp_reducer, inputs, v, create_graph=True) + (tensor([5.7817, 7.2458, 5.7830, 6.7782], grad_fn=), + tensor([[1.4458, 1.3962, 1.3042, 1.6354], + [2.1288, 1.0652, 1.5483, 2.5035], + [2.2046, 1.1292, 1.1432, 1.3059], + [1.3225, 1.6652, 1.7753, 2.0152]], grad_fn=)) + + >>> def adder(x, y): + ... return 2 * x + 3 * y + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> v = torch.ones(2) + >>> vjp(adder, inputs, v) + (tensor([2.4225, 2.3340]), + (tensor([2., 2.]), tensor([3., 3.]))) + """ + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "vjp") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "vjp" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + + if v is not None: + _, v = _as_tuple(v, "v", "vjp") + v = _grad_preprocess(v, create_graph=create_graph, need_graph=False) + _validate_v(v, outputs, is_outputs_tuple) + else: + if len(outputs) != 1 or outputs[0].nelement() != 1: + raise RuntimeError( + "The vector v can only be None if the " + "user-provided function returns " + "a single Tensor with a single element." + ) + + enable_grad = True if create_graph else torch.is_grad_enabled() + with torch.set_grad_enabled(enable_grad): + grad_res = _autograd_grad(outputs, inputs, v, create_graph=create_graph) + vjp = _fill_in_zeros(grad_res, inputs, strict, create_graph, "back") + + # Cleanup objects and return them to the user + outputs = _grad_postprocess(outputs, create_graph) + vjp = _grad_postprocess(vjp, create_graph) + + return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess( + vjp, is_inputs_tuple + ) + + +def jvp(func, inputs, v=None, create_graph=False, strict=False): + r"""Compute the dot product between the Jacobian of the given function at the point given by the inputs and a vector ``v``. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a tuple of Tensors or a Tensor. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + v (tuple of Tensors or Tensor): The vector for which the Jacobian + vector product is computed. Must be the same size as the input of + ``func``. This argument is optional when the input to ``func`` + contains a single element and (if it is not provided) will be set + as a Tensor containing a single ``1``. + create_graph (bool, optional): If ``True``, both the output and result + will be computed in a differentiable way. Note that when ``strict`` + is ``False``, the result can not require gradients or be + disconnected from the inputs. Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + jvp for said inputs, which is the expected mathematical value. + Defaults to ``False``. + + Returns: + output (tuple): tuple with: + func_output (tuple of Tensors or Tensor): output of ``func(inputs)`` + + jvp (tuple of Tensors or Tensor): result of the dot product with + the same shape as the output. + + Note: + ``autograd.functional.jvp`` computes the jvp by using the backward of + the backward (sometimes called the double backwards trick). This is not + the most performant way of computing the jvp. Please consider using + :func:`torch.func.jvp` or the + :ref:`low-level forward-mode AD API ` instead. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def exp_reducer(x): + ... return x.exp().sum(dim=1) + >>> inputs = torch.rand(4, 4) + >>> v = torch.ones(4, 4) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> jvp(exp_reducer, inputs, v) + (tensor([6.3090, 4.6742, 7.9114, 8.2106]), + tensor([6.3090, 4.6742, 7.9114, 8.2106])) + + >>> jvp(exp_reducer, inputs, v, create_graph=True) + (tensor([6.3090, 4.6742, 7.9114, 8.2106], grad_fn=), + tensor([6.3090, 4.6742, 7.9114, 8.2106], grad_fn=)) + + >>> def adder(x, y): + ... return 2 * x + 3 * y + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> v = (torch.ones(2), torch.ones(2)) + >>> jvp(adder, inputs, v) + (tensor([2.2399, 2.5005]), + tensor([5., 5.])) + + """ + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jvp") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + if v is not None: + _, v = _as_tuple(v, "v", "jvp") + v = _grad_preprocess(v, create_graph=create_graph, need_graph=False) + _validate_v(v, inputs, is_inputs_tuple) + else: + if len(inputs) != 1 or inputs[0].nelement() != 1: + raise RuntimeError( + "The vector v can only be None if the input to " + "the user-provided function is a single Tensor " + "with a single element." + ) + + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "jvp" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + # The backward is linear so the value of grad_outputs is not important as + # it won't appear in the double backward graph. We only need to ensure that + # it does not contain inf or nan. + grad_outputs = tuple( + torch.zeros_like(out, requires_grad=True) for out in outputs + ) + + grad_inputs = _autograd_grad(outputs, inputs, grad_outputs, create_graph=True) + _check_requires_grad(grad_inputs, "grad_inputs", strict=strict) + + if create_graph: + with torch.enable_grad(): + grad_res = _autograd_grad( + grad_inputs, grad_outputs, v, create_graph=create_graph + ) + jvp = _fill_in_zeros(grad_res, outputs, strict, create_graph, "back_trick") + else: + grad_res = _autograd_grad( + grad_inputs, grad_outputs, v, create_graph=create_graph + ) + jvp = _fill_in_zeros(grad_res, outputs, strict, create_graph, "back_trick") + + # Cleanup objects and return them to the user + outputs = _grad_postprocess(outputs, create_graph) + jvp = _grad_postprocess(jvp, create_graph) + + return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess( + jvp, is_outputs_tuple + ) + + +def _construct_standard_basis_for( + tensors: tuple[torch.Tensor, ...], tensor_numels: tuple[int, ...] +) -> tuple[torch.Tensor, ...]: + # This function: + # - constructs a N=sum(tensor_numels) standard basis. i.e. an NxN identity matrix. + # - Splits the identity matrix into chunks with each chunk size determined by `tensor_numels`. + # - Each chunk corresponds to one tensor. The chunk has the same dtype and + # device as the tensor + # + # For example, with tensor_numels = [1, 2, 1], this function returns: + # ( tensor([[1], tensor([[0, 0], tensor([[0], + # [0], [1, 0], [0], + # [0], [0, 1], [0], + # [0]]) , [0, 0]]) , [1]]) ) + # + # Precondition: tensor_numels == tuple(tensor.numel() for tensor in tensors) + # Precondition: tensors always has at least one element. + # + # See NOTE: [Computing jacobian with vmap and grad for multiple tensors] + # for context behind this function. All the pre-conditions are guarded for + # in torch.autograd.functional.jacobian. + if len(tensors) != len(tensor_numels): + raise AssertionError( + f"Expected tensors and tensor_numels to have the same length, " + f"but got {len(tensors)} and {len(tensor_numels)}" + ) + if len(tensors) == 0: + raise AssertionError("Expected at least one tensor") + total_numel = sum(tensor_numels) + chunks = tuple( + tensor.new_zeros(total_numel, tensor_numel) + for tensor, tensor_numel in zip(tensors, tensor_numels) + ) + diag_start_idx = 0 + for chunk, numel in zip(chunks, tensor_numels): + chunk.diagonal(diag_start_idx).fill_(1) + diag_start_idx -= numel + return chunks + + +def _jacfwd(func, inputs, strict=False, vectorize=False): + if strict: + raise RuntimeError( + "torch.autograd.functional.jacobian: `strict=True` " + 'and `strategy="forward-mode"` are not supported together (yet). ' + "Please either set `strict=False` or " + '`strategy="reverse-mode"`.' + ) + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jacobian") + output_info = [] + + if vectorize: + # See NOTE: [Computing jacobian with vmap and grad for multiple outputs] + input_numels = tuple(input.numel() for input in inputs) + + # Step 1: Prepare tangents + tangents = _construct_standard_basis_for(inputs, input_numels) + + # Step 2: Compute vmap over computation with dual tensors + def jvp(tangents): + with fwAD.dual_level(): + dual_inputs = tuple( + fwAD.make_dual(input, tangent.view_as(input)) + for input, tangent in zip(inputs, tangents) + ) + _is_outputs_tuple, dual_outputs = _as_tuple( + func(*dual_inputs), "outputs" + ) + output_info.append(_is_outputs_tuple) + jv = [] + primal_outs = [] + for dual_out in dual_outputs: + primal, tangent = fwAD.unpack_dual(dual_out) + primal_outs.append(primal) + if tangent is not None: + jv.append(tangent) + else: + jv.append(torch.zeros_like(primal)) + output_info.append(primal_outs) + return tuple(jv) + + outputs_before_split = _vmap(jvp)(tangents) + is_outputs_tuple, outputs = output_info + # Step 3: for each of the output tangents, split along dim 0 + jacobian_input_output = [] + for jac_output_i, output_i in zip(outputs_before_split, outputs): + jacobian_output_i_output = [] + for jac, input_j in zip(jac_output_i.split(input_numels, dim=0), inputs): + # We need to transpose the Jacobian because in forward AD, the + # batch dimension represents that of the inputs + jacobian_input_i_output_j = jac.permute(*range(1, jac.ndim), 0).reshape( + (*output_i.shape, *input_j.shape) + ) # noqa: C409 + + jacobian_output_i_output.append(jacobian_input_i_output_j) + jacobian_input_output.append(jacobian_output_i_output) + + # Omit [Step 4] because everything is already transposed w/ forward AD + return _tuple_postprocess( + jacobian_input_output, (is_outputs_tuple, is_inputs_tuple) + ) + else: + raise NotImplementedError( + "Computing Jacobian using forward-AD or forward-over-reverse Hessian is" + "only implemented for `vectorize=True`." + ) + + +def jacobian( + func, + inputs, + create_graph=False, + strict=False, + vectorize=False, + strategy="reverse-mode", +): + r"""Compute the Jacobian of a given function. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a tuple of Tensors or a Tensor. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + create_graph (bool, optional): If ``True``, the Jacobian will be + computed in a differentiable manner. Note that when ``strict`` is + ``False``, the result can not require gradients or be disconnected + from the inputs. Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + jacobian for said inputs, which is the expected mathematical value. + Defaults to ``False``. + vectorize (bool, optional): This feature is experimental. + Please consider using :func:`torch.func.jacrev` or + :func:`torch.func.jacfwd` instead if you are looking for something + less experimental and more performant. + When computing the jacobian, usually we invoke + ``autograd.grad`` once per row of the jacobian. If this flag is + ``True``, we perform only a single ``autograd.grad`` call with + ``batched_grad=True`` which uses the vmap prototype feature. + Though this should lead to performance improvements in many cases, + because this feature is still experimental, there may be performance + cliffs. See :func:`torch.autograd.grad`'s ``batched_grad`` parameter for + more information. + strategy (str, optional): Set to ``"forward-mode"`` or ``"reverse-mode"`` to + determine whether the Jacobian will be computed with forward or reverse + mode AD. Currently, ``"forward-mode"`` requires ``vectorized=True``. + Defaults to ``"reverse-mode"``. If ``func`` has more outputs than + inputs, ``"forward-mode"`` tends to be more performant. Otherwise, + prefer to use ``"reverse-mode"``. + + Returns: + Jacobian (Tensor or nested tuple of Tensors): if there is a single + input and output, this will be a single Tensor containing the + Jacobian for the linearized inputs and output. If one of the two is + a tuple, then the Jacobian will be a tuple of Tensors. If both of + them are tuples, then the Jacobian will be a tuple of tuple of + Tensors where ``Jacobian[i][j]`` will contain the Jacobian of the + ``i``\th output and ``j``\th input and will have as size the + concatenation of the sizes of the corresponding output and the + corresponding input and will have same dtype and device as the + corresponding input. If strategy is ``forward-mode``, the dtype will be + that of the output; otherwise, the input. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def exp_reducer(x): + ... return x.exp().sum(dim=1) + >>> inputs = torch.rand(2, 2) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> jacobian(exp_reducer, inputs) + tensor([[[1.4917, 2.4352], + [0.0000, 0.0000]], + [[0.0000, 0.0000], + [2.4369, 2.3799]]]) + + >>> jacobian(exp_reducer, inputs, create_graph=True) + tensor([[[1.4917, 2.4352], + [0.0000, 0.0000]], + [[0.0000, 0.0000], + [2.4369, 2.3799]]], grad_fn=) + + >>> def exp_adder(x, y): + ... return 2 * x.exp() + 3 * y + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> jacobian(exp_adder, inputs) + (tensor([[2.8052, 0.0000], + [0.0000, 3.3963]]), + tensor([[3., 0.], + [0., 3.]])) + + >>> def linear_model(x): + ... W = torch.tensor([[2.0, -1.0], [0.0, 1.0]]) + ... b = torch.tensor([1.0, 0.5]) + ... return x @ W.T + b + + >>> x = torch.randn(4, 2, requires_grad=True) + >>> jac = jacobian(linear_model, x, vectorize=True) + >>> jac.shape + torch.Size([4, 2, 4, 2]) + """ + if strategy not in ("forward-mode", "reverse-mode"): + raise AssertionError( + 'Expected strategy to be either "forward-mode" or "reverse-mode". Hint: If your ' + 'function has more outputs than inputs, "forward-mode" tends to be more performant. ' + 'Otherwise, prefer to use "reverse-mode".' + ) + if strategy == "forward-mode": + if create_graph: + raise NotImplementedError( + "torch.autograd.functional.jacobian: `create_graph=True` " + 'and `strategy="forward-mode"` are not supported together (yet). ' + "Please either set `create_graph=False` or " + '`strategy="reverse-mode"`.' + ) + return _jacfwd(func, inputs, strict, vectorize) + + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "jacobian") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "jacobian" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + + if vectorize: + if strict: + raise RuntimeError( + "torch.autograd.functional.jacobian: `strict=True` " + "and `vectorized=True` are not supported together. " + "Please either set `strict=False` or " + "`vectorize=False`." + ) + # NOTE: [Computing jacobian with vmap and grad for multiple outputs] + # + # Let's consider f(x) = (x**2, x.sum()) and let x = torch.randn(3). + # It turns out we can compute the jacobian of this function with a single + # call to autograd.grad by using vmap over the correct grad_outputs. + # + # Firstly, one way to compute the jacobian is to stack x**2 and x.sum() + # into a 4D vector. E.g., use g(x) = torch.stack([x**2, x.sum()]) + # + # To get the first row of the jacobian, we call + # >>> autograd.grad(g(x), x, grad_outputs=torch.tensor([1, 0, 0, 0])) + # To get the 2nd row of the jacobian, we call + # >>> autograd.grad(g(x), x, grad_outputs=torch.tensor([0, 1, 0, 0])) + # and so on. + # + # Using vmap, we can vectorize all 4 of these computations into one by + # passing the standard basis for R^4 as the grad_output. + # vmap(partial(autograd.grad, g(x), x))(torch.eye(4)). + # + # Now, how do we compute the jacobian *without stacking the output*? + # We can just split the standard basis across the outputs. So to + # compute the jacobian of f(x), we'd use + # >>> autograd.grad(f(x), x, grad_outputs=_construct_standard_basis_for(...)) + # The grad_outputs looks like the following: + # ( torch.tensor([[1, 0, 0], + # [0, 1, 0], + # [0, 0, 1], + # [0, 0, 0]]), + # torch.tensor([[0], + # [0], + # [0], + # [1]]) ) + # + # But we're not done yet! + # >>> vmap(partial(autograd.grad(f(x), x, grad_outputs=...))) + # returns a Tensor of shape [4, 3]. We have to remember to split the + # jacobian of shape [4, 3] into two: + # - one of shape [3, 3] for the first output + # - one of shape [ 3] for the second output + + # Step 1: Construct grad_outputs by splitting the standard basis + output_numels = tuple(output.numel() for output in outputs) + grad_outputs = _construct_standard_basis_for(outputs, output_numels) + flat_outputs = tuple(output.reshape(-1) for output in outputs) + + # Step 2: Call vmap + autograd.grad + def vjp(grad_output): + vj = list( + _autograd_grad( + flat_outputs, + inputs, + grad_output, + create_graph=create_graph, + is_grads_batched=True, + ) + ) + for el_idx, vj_el in enumerate(vj): + if vj_el is not None: + continue + vj[el_idx] = torch.zeros_like(inputs[el_idx]).expand( + (sum(output_numels),) + inputs[el_idx].shape + ) + return tuple(vj) + + jacobians_of_flat_output = vjp(grad_outputs) + + # Step 3: The returned jacobian is one big tensor per input. In this step, + # we split each Tensor by output. + jacobian_input_output = [] + for jac_input_i, input_i in zip(jacobians_of_flat_output, inputs): + jacobian_input_i_output = [] + for jac, output_j in zip( + jac_input_i.split(output_numels, dim=0), outputs + ): + jacobian_input_i_output_j = jac.view(output_j.shape + input_i.shape) + jacobian_input_i_output.append(jacobian_input_i_output_j) + jacobian_input_output.append(jacobian_input_i_output) + + # Step 4: Right now, `jacobian` is a List[List[Tensor]]. + # The outer List corresponds to the number of inputs, + # the inner List corresponds to the number of outputs. + # We need to exchange the order of these and convert to tuples + # before returning. + jacobian_output_input = tuple(zip(*jacobian_input_output)) + + jacobian_output_input = _grad_postprocess( + jacobian_output_input, create_graph + ) + return _tuple_postprocess( + jacobian_output_input, (is_outputs_tuple, is_inputs_tuple) + ) + + jacobian: tuple[torch.Tensor, ...] = () + + for i, out in enumerate(outputs): + # mypy complains that expression and variable have different types due to the empty list + jac_i: tuple[list[torch.Tensor]] = tuple([] for _ in range(len(inputs))) # type: ignore[assignment] + for j in range(out.nelement()): + vj = _autograd_grad( + (out.reshape(-1)[j],), + inputs, + retain_graph=True, + create_graph=create_graph, + ) + + for el_idx, (jac_i_el, vj_el, inp_el) in enumerate( + zip(jac_i, vj, inputs) + ): + if vj_el is not None: + if strict and create_graph and not vj_el.requires_grad: + msg = ( + "The jacobian of the user-provided function is " + f"independent of input {i}. This is not allowed in " + "strict mode when create_graph=True." + ) + raise RuntimeError(msg) + jac_i_el.append(vj_el) + else: + if strict: + msg = ( + f"Output {i} of the user-provided function is " + f"independent of input {el_idx}. This is not allowed in " + "strict mode." + ) + raise RuntimeError(msg) + jac_i_el.append(torch.zeros_like(inp_el)) + + jacobian += ( + tuple( + torch.stack(jac_i_el, dim=0).view( + out.size() + inputs[el_idx].size() # type: ignore[operator] + ) + for (el_idx, jac_i_el) in enumerate(jac_i) + ), + ) + + jacobian = _grad_postprocess(jacobian, create_graph) + + return _tuple_postprocess(jacobian, (is_outputs_tuple, is_inputs_tuple)) + + +def hessian( + func, + inputs, + create_graph=False, + strict=False, + vectorize=False, + outer_jacobian_strategy="reverse-mode", +): + r"""Compute the Hessian of a given scalar function. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor with a single element. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + create_graph (bool, optional): If ``True``, the Hessian will be computed in + a differentiable manner. Note that when ``strict`` is ``False``, the result can not + require gradients or be disconnected from the inputs. + Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we detect that there exists an input + such that all the outputs are independent of it. If ``False``, we return a Tensor of zeros as the + hessian for said inputs, which is the expected mathematical value. + Defaults to ``False``. + vectorize (bool, optional): This feature is experimental. + Please consider using :func:`torch.func.hessian` + instead if you are looking for something less experimental and more performant. + When computing the hessian, usually we invoke + ``autograd.grad`` once per row of the hessian. If this flag is + ``True``, we use the vmap prototype feature as the backend to + vectorize calls to ``autograd.grad`` so we only invoke it once + instead of once per row. This should lead to performance + improvements in many use cases, however, due to this feature + being incomplete, there may be performance cliffs. Please + use `torch._C._debug_only_display_vmap_fallback_warnings(True)` + to show any performance warnings and file us issues if + warnings exist for your use case. Defaults to ``False``. + outer_jacobian_strategy (str, optional): The Hessian is computed by + computing the Jacobian of a Jacobian. The inner Jacobian is always + computed in reverse-mode AD. Setting strategy to ``"forward-mode"`` + or ``"reverse-mode"`` determines whether the outer Jacobian will be + computed with forward or reverse mode AD. Currently, computing the outer + Jacobian in ``"forward-mode"`` requires ``vectorized=True``. Defaults + to ``"reverse-mode"``. + + Returns: + Hessian (Tensor or a tuple of tuple of Tensors): if there is a single input, + this will be a single Tensor containing the Hessian for the input. + If it is a tuple, then the Hessian will be a tuple of tuples where + ``Hessian[i][j]`` will contain the Hessian of the ``i``\th input + and ``j``\th input with size the sum of the size of the ``i``\th input plus + the size of the ``j``\th input. ``Hessian[i][j]`` will have the same + dtype and device as the corresponding ``i``\th input. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def pow_reducer(x): + ... return x.pow(3).sum() + >>> inputs = torch.rand(2, 2) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> hessian(pow_reducer, inputs) + tensor([[[[5.2265, 0.0000], + [0.0000, 0.0000]], + [[0.0000, 4.8221], + [0.0000, 0.0000]]], + [[[0.0000, 0.0000], + [1.9456, 0.0000]], + [[0.0000, 0.0000], + [0.0000, 3.2550]]]]) + + >>> hessian(pow_reducer, inputs, create_graph=True) + tensor([[[[5.2265, 0.0000], + [0.0000, 0.0000]], + [[0.0000, 4.8221], + [0.0000, 0.0000]]], + [[[0.0000, 0.0000], + [1.9456, 0.0000]], + [[0.0000, 0.0000], + [0.0000, 3.2550]]]], grad_fn=) + + + >>> def pow_adder_reducer(x, y): + ... return (2 * x.pow(2) + 3 * y.pow(2)).sum() + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> hessian(pow_adder_reducer, inputs) + ((tensor([[4., 0.], + [0., 4.]]), + tensor([[0., 0.], + [0., 0.]])), + (tensor([[0., 0.], + [0., 0.]]), + tensor([[6., 0.], + [0., 6.]]))) + """ + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "hessian") + if outer_jacobian_strategy not in ( + "forward-mode", + "reverse-mode", + ): + raise AssertionError( + 'Expected strategy to be either "forward-mode" or "reverse-mode".' + ) + + def ensure_single_output_function(*inp): + out = func(*inp) + is_out_tuple, t_out = _as_tuple( + out, "outputs of the user-provided function", "hessian" + ) + _check_requires_grad(t_out, "outputs", strict=strict) + + if is_out_tuple or not isinstance(out, torch.Tensor): + raise RuntimeError( + "The function given to hessian should return a single Tensor" + ) + + if out.nelement() != 1: + raise RuntimeError( + "The Tensor returned by the function given to hessian should contain a single element" + ) + + return out.squeeze() + + def jac_func(*inp): + if outer_jacobian_strategy == "forward-mode": + # _grad_preprocess requires create_graph=True and input to require_grad + # or else the input will be detached + inp = tuple(t.requires_grad_(True) for t in inp) + jac = jacobian(ensure_single_output_function, inp, create_graph=True) + _check_requires_grad(jac, "jacobian", strict=strict) + return jac + + res = jacobian( + jac_func, + inputs, + create_graph=create_graph, + strict=strict, + vectorize=vectorize, + strategy=outer_jacobian_strategy, + ) + return _tuple_postprocess(res, (is_inputs_tuple, is_inputs_tuple)) + + +def vhp(func, inputs, v=None, create_graph=False, strict=False): + r"""Compute the dot product between vector ``v`` and Hessian of a given scalar function at a specified point. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor with a single element. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + v (tuple of Tensors or Tensor): The vector for which the vector Hessian + product is computed. Must be the same size as the input of + ``func``. This argument is optional when ``func``'s input contains + a single element and (if it is not provided) will be set as a + Tensor containing a single ``1``. + create_graph (bool, optional): If ``True``, both the output and result + will be computed in a differentiable way. Note that when ``strict`` + is ``False``, the result can not require gradients or be + disconnected from the inputs. + Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + vhp for said inputs, which is the expected mathematical value. + Defaults to ``False``. + + Returns: + output (tuple): tuple with: + func_output (tuple of Tensors or Tensor): output of ``func(inputs)`` + + vhp (tuple of Tensors or Tensor): result of the dot product with the + same shape as the inputs. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def pow_reducer(x): + ... return x.pow(3).sum() + >>> inputs = torch.rand(2, 2) + >>> v = torch.ones(2, 2) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> vhp(pow_reducer, inputs, v) + (tensor(0.5591), + tensor([[1.0689, 1.2431], + [3.0989, 4.4456]])) + >>> vhp(pow_reducer, inputs, v, create_graph=True) + (tensor(0.5591, grad_fn=), + tensor([[1.0689, 1.2431], + [3.0989, 4.4456]], grad_fn=)) + >>> def pow_adder_reducer(x, y): + ... return (2 * x.pow(2) + 3 * y.pow(2)).sum() + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> v = (torch.zeros(2), torch.ones(2)) + >>> vhp(pow_adder_reducer, inputs, v) + (tensor(4.8053), + (tensor([0., 0.]), + tensor([6., 6.]))) + """ + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "vhp") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + if v is not None: + _, v = _as_tuple(v, "v", "vhp") + v = _grad_preprocess(v, create_graph=create_graph, need_graph=False) + _validate_v(v, inputs, is_inputs_tuple) + else: + if len(inputs) != 1 or inputs[0].nelement() != 1: + raise RuntimeError( + "The vector v can only be None if the input to the user-provided function " + "is a single Tensor with a single element." + ) + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "vhp" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + + if is_outputs_tuple or not isinstance(outputs[0], torch.Tensor): + raise RuntimeError( + "The function given to vhp should return a single Tensor" + ) + + if outputs[0].nelement() != 1: + raise RuntimeError( + "The Tensor returned by the function given to vhp should contain a single element" + ) + + jac = _autograd_grad(outputs, inputs, create_graph=True) + _check_requires_grad(jac, "jacobian", strict=strict) + + enable_grad = True if create_graph else torch.is_grad_enabled() + with torch.set_grad_enabled(enable_grad): + grad_res = _autograd_grad(jac, inputs, v, create_graph=create_graph) + vhp = _fill_in_zeros(grad_res, inputs, strict, create_graph, "double_back") + + outputs = _grad_postprocess(outputs, create_graph) + vhp = _grad_postprocess(vhp, create_graph) + + return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess( + vhp, is_inputs_tuple + ) + + +def hvp(func, inputs, v=None, create_graph=False, strict=False): + r"""Compute the dot product between the scalar function's Hessian and a vector ``v`` at a specified point. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor with a single element. + inputs (tuple of Tensors or Tensor): inputs to the function ``func``. + v (tuple of Tensors or Tensor): The vector for which the Hessian vector + product is computed. Must be the same size as the input of + ``func``. This argument is optional when ``func``'s input contains + a single element and (if it is not provided) will be set as a + Tensor containing a single ``1``. + create_graph (bool, optional): If ``True``, both the output and result will be + computed in a differentiable way. Note that when ``strict`` is + ``False``, the result can not require gradients or be disconnected + from the inputs. Defaults to ``False``. + strict (bool, optional): If ``True``, an error will be raised when we + detect that there exists an input such that all the outputs are + independent of it. If ``False``, we return a Tensor of zeros as the + hvp for said inputs, which is the expected mathematical value. + Defaults to ``False``. + Returns: + output (tuple): tuple with: + func_output (tuple of Tensors or Tensor): output of ``func(inputs)`` + + hvp (tuple of Tensors or Tensor): result of the dot product with + the same shape as the inputs. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def pow_reducer(x): + ... return x.pow(3).sum() + >>> inputs = torch.rand(2, 2) + >>> v = torch.ones(2, 2) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> hvp(pow_reducer, inputs, v) + (tensor(0.1448), + tensor([[2.0239, 1.6456], + [2.4988, 1.4310]])) + + >>> hvp(pow_reducer, inputs, v, create_graph=True) + (tensor(0.1448, grad_fn=), + tensor([[2.0239, 1.6456], + [2.4988, 1.4310]], grad_fn=)) + + + >>> def pow_adder_reducer(x, y): + ... return (2 * x.pow(2) + 3 * y.pow(2)).sum() + >>> inputs = (torch.rand(2), torch.rand(2)) + >>> v = (torch.zeros(2), torch.ones(2)) + >>> hvp(pow_adder_reducer, inputs, v) + (tensor(2.3030), + (tensor([0., 0.]), + tensor([6., 6.]))) + + Note: + + This function is significantly slower than `vhp` due to backward mode AD constraints. + If your functions is twice continuously differentiable, then hvp = vhp.t(). So if you + know that your function satisfies this condition, you should use vhp instead that is + much faster with the current implementation. + + """ + with torch.enable_grad(): + is_inputs_tuple, inputs = _as_tuple(inputs, "inputs", "hvp") + inputs = _grad_preprocess(inputs, create_graph=create_graph, need_graph=True) + + if v is not None: + _, v = _as_tuple(v, "v", "hvp") + v = _grad_preprocess(v, create_graph=create_graph, need_graph=False) + _validate_v(v, inputs, is_inputs_tuple) + else: + if len(inputs) != 1 or inputs[0].nelement() != 1: + raise RuntimeError( + "The vector v can only be None if the input to the user-provided function " + "is a single Tensor with a single element." + ) + outputs = func(*inputs) + is_outputs_tuple, outputs = _as_tuple( + outputs, "outputs of the user-provided function", "hvp" + ) + _check_requires_grad(outputs, "outputs", strict=strict) + + if is_outputs_tuple or not isinstance(outputs[0], torch.Tensor): + raise RuntimeError( + "The function given to hvp should return a single Tensor" + ) + + if outputs[0].nelement() != 1: + raise RuntimeError( + "The Tensor returned by the function given to hvp should contain a single element" + ) + + jac = _autograd_grad(outputs, inputs, create_graph=True) + _check_requires_grad(jac, "jacobian", strict=strict) + + grad_jac = tuple(torch.zeros_like(inp, requires_grad=True) for inp in inputs) + + double_back = _autograd_grad(jac, inputs, grad_jac, create_graph=True) + _check_requires_grad(jac, "hessian", strict=strict) + + enable_grad = True if create_graph else torch.is_grad_enabled() + with torch.set_grad_enabled(enable_grad): + grad_res = _autograd_grad(double_back, grad_jac, v, create_graph=create_graph) + hvp = _fill_in_zeros( + grad_res, inputs, strict, create_graph, "double_back_trick" + ) + + outputs = _grad_postprocess(outputs, create_graph) + hvp = _grad_postprocess(hvp, create_graph) + + return _tuple_postprocess(outputs, is_outputs_tuple), _tuple_postprocess( + hvp, is_inputs_tuple + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/grad_mode.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/grad_mode.py new file mode 100644 index 0000000000000000000000000000000000000000..bc882ec08cb76d2864e673e496f01af155ccc2a4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/grad_mode.py @@ -0,0 +1,420 @@ +# mypy: allow-untyped-defs +from typing import Any, Union + +import torch +from torch.utils._contextlib import ( + _DecoratorContextManager, + _NoParamDecoratorContextManager, + F, +) + + +__all__ = [ + "no_grad", + "enable_grad", + "set_grad_enabled", + "inference_mode", + "set_multithreading_enabled", +] + + +class no_grad(_NoParamDecoratorContextManager): + r"""Context-manager that disables gradient calculation. + + Disabling gradient calculation is useful for inference, when you are sure + that you will not call :meth:`Tensor.backward()`. It will reduce memory + consumption for computations that would otherwise have `requires_grad=True`. + + In this mode, the result of every computation will have + `requires_grad=False`, even when the inputs have `requires_grad=True`. + There is an exception! All factory functions, or functions that create + a new Tensor and take a requires_grad kwarg, will NOT be affected by + this mode. + + This context manager is thread local; it will not affect computation + in other threads. + + Also functions as a decorator. + + .. note:: + No-grad is one of several mechanisms that can enable or + disable gradients locally see :ref:`locally-disable-grad-doc` for + more information on how they compare. + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + If you want to disable forward AD for a computation, you can unpack + your dual tensors. + + Example:: + >>> # xdoctest: +SKIP + >>> x = torch.tensor([1.], requires_grad=True) + >>> with torch.no_grad(): + ... y = x * 2 + >>> y.requires_grad + False + >>> @torch.no_grad() + ... def doubler(x): + ... return x * 2 + >>> z = doubler(x) + >>> z.requires_grad + False + >>> @torch.no_grad() + ... def tripler(x): + ... return x * 3 + >>> z = tripler(x) + >>> z.requires_grad + False + >>> # factory function exception + >>> with torch.no_grad(): + ... a = torch.nn.Parameter(torch.rand(10)) + >>> a.requires_grad + True + """ + + def __init__(self) -> None: + if not torch._jit_internal.is_scripting(): + super().__init__() + self.prev = False + + def __enter__(self) -> None: + self.prev = torch.is_grad_enabled() + torch.set_grad_enabled(False) + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch.set_grad_enabled(self.prev) + + +class enable_grad(_NoParamDecoratorContextManager): + r"""Context-manager that enables gradient calculation. + + Enables gradient calculation, if it has been disabled via :class:`~no_grad` + or :class:`~set_grad_enabled`. + + This context manager is thread local; it will not affect computation + in other threads. + + Also functions as a decorator. + + .. note:: + enable_grad is one of several mechanisms that can enable or + disable gradients locally see :ref:`locally-disable-grad-doc` for + more information on how they compare. + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + + Example:: + >>> # xdoctest: +SKIP + >>> x = torch.tensor([1.], requires_grad=True) + >>> with torch.no_grad(): + ... with torch.enable_grad(): + ... y = x * 2 + >>> y.requires_grad + True + >>> y.backward() + >>> x.grad + tensor([2.]) + >>> @torch.enable_grad() + ... def doubler(x): + ... return x * 2 + >>> with torch.no_grad(): + ... z = doubler(x) + >>> z.requires_grad + True + >>> @torch.enable_grad() + ... def tripler(x): + ... return x * 3 + >>> with torch.no_grad(): + ... z = tripler(x) + >>> z.requires_grad + True + + """ + + def __enter__(self) -> None: + self.prev = torch.is_grad_enabled() + torch._C._set_grad_enabled(True) + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_grad_enabled(self.prev) + + +class set_grad_enabled(_DecoratorContextManager): + r"""Context-manager that sets gradient calculation on or off. + + ``set_grad_enabled`` will enable or disable grads based on its argument :attr:`mode`. + It can be used as a context-manager or as a function. + + This context manager is thread local; it will not affect computation + in other threads. + + Args: + mode (bool): Flag whether to enable grad (``True``), or disable + (``False``). This can be used to conditionally enable + gradients. + + .. note:: + set_grad_enabled is one of several mechanisms that can enable or + disable gradients locally see :ref:`locally-disable-grad-doc` for + more information on how they compare. + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + + Example:: + >>> # xdoctest: +SKIP + >>> x = torch.tensor([1.], requires_grad=True) + >>> is_train = False + >>> with torch.set_grad_enabled(is_train): + ... y = x * 2 + >>> y.requires_grad + False + >>> _ = torch.set_grad_enabled(True) + >>> y = x * 2 + >>> y.requires_grad + True + >>> _ = torch.set_grad_enabled(False) + >>> y = x * 2 + >>> y.requires_grad + False + + """ + + def __init__(self, mode: bool) -> None: + self.prev = torch.is_grad_enabled() + self.mode = mode + torch._C._set_grad_enabled(mode) + + def __call__(self, orig_func: F) -> F: + torch._C._set_grad_enabled(self.prev) + return super().__call__(orig_func) + + def __enter__(self) -> None: + torch._C._set_grad_enabled(self.mode) + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_grad_enabled(self.prev) + + def __str__(self) -> str: + return f"{torch.typename(self)}(mode={self.mode})" + + def __repr__(self) -> str: + return str(self) + + def clone(self) -> "set_grad_enabled": + r""" + Create a copy of this class + """ + return self.__class__(self.mode) + + +class inference_mode(_DecoratorContextManager): + r"""Context manager that enables or disables inference mode. + + InferenceMode is analogous to :class:`~no_grad` and should be used + when you are certain your operations will not interact with autograd + (e.g., during data loading or model evaluation). Compared to + :class:`~no_grad`, it removes additional overhead by disabling view + tracking and version counter bumps. It is also more restrictive, in + that tensors created in this mode cannot be used in computations + recorded by autograd. + + This context manager is thread-local; it does not affect computation + in other threads. + + Also functions as a decorator. + + .. note:: + Inference mode is one of several mechanisms that can locally enable + or disable gradients. See :ref:`locally-disable-grad-doc` for a + comparison. If avoiding the use of tensors created in inference mode + in autograd-tracked regions is difficult, consider benchmarking your + code with and without inference mode to weigh the performance benefits + against the trade-offs. You can always use :class:`~no_grad` instead. + + .. note:: + Unlike some other mechanisms that locally enable or disable grad, + entering inference_mode also disables :ref:`forward-mode AD `. + + .. warning:: + `inference_mode` does NOT automatically set the model to evaluation mode. + For proper inference behavior (e.g., disabling dropout, using running statistics + in batch normalization), you must explicitly set your model to evaluation mode using + `model.eval()` in addition to using this context manager. + + Args: + mode (bool or function): Either a boolean flag to enable or disable + inference mode, or a Python function to decorate with inference + mode enabled. + + Example:: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> import torch + >>> x = torch.ones(1, 2, 3, requires_grad=True) + >>> with torch.inference_mode(): + ... y = x * x + >>> y.requires_grad + False + >>> # xdoctest: +SKIP("want string isn't quite right") + >>> y._version + Traceback (most recent call last): + File "", line 1, in + RuntimeError: Inference tensors do not track version counter. + >>> @torch.inference_mode() + ... def func(x): + ... return x * x + >>> out = func(x) + >>> out.requires_grad + False + >>> @torch.inference_mode() + ... def doubler(x): + ... return x * 2 + >>> out = doubler(x) + >>> out.requires_grad + False + + """ + + def __init__(self, mode: bool = True) -> None: + if not torch._jit_internal.is_scripting(): + super().__init__() + self.mode = mode + + def __new__(cls, mode=True): + if isinstance(mode, bool): + return super().__new__(cls) + return cls()(mode) + + def __enter__(self) -> None: + self._inference_mode_context = torch._C._InferenceMode(self.mode) + self._inference_mode_context.__enter__() + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + self._inference_mode_context.__exit__(exc_type, exc_value, traceback) + + def clone(self) -> "inference_mode": + r""" + Create a copy of this class + """ + return self.__class__(self.mode) + + +def _enter_inference_mode(mode): + mode_context = torch._C._InferenceMode(mode) + mode_context.__enter__() + return mode_context + + +def _exit_inference_mode(mode): + mode.__exit__(None, None, None) + + +class set_multithreading_enabled(_DecoratorContextManager): + r"""Context-manager that sets multithreaded backwards on or off. + + ``set_multithreading_enabled`` will enable or disable multithreaded backwards based on its argument :attr:`mode`. + It can be used as a context-manager or as a function. + + This context manager is thread local; it will not affect computation + in other threads. + + Args: + mode (bool): Flag whether to enable multithreaded backwards (``True``), or disable + (``False``). + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + + """ + + def __init__(self, mode: bool) -> None: + self.prev = torch._C._is_multithreading_enabled() + torch._C._set_multithreading_enabled(mode) + self.mode = mode + + def __enter__(self) -> None: + pass + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_multithreading_enabled(self.prev) + + def clone(self) -> "set_multithreading_enabled": + r""" + Create a copy of this class + """ + return self.__class__(self.mode) + + +class _force_original_view_tracking(_DecoratorContextManager): + r"""Context-manager that sets whether or not to always enable view-replay in autograd. + + ``set_view_replay_enabled`` will enable or disable view-replay based on its argument :attr:`mode`. + It can be used as a context-manager or as a function. + + This context manager is thread local; it will not affect computation + in other threads. + + When a tensor view is mutated, the autograd engine needs to decide whether or not + to regenerate the "updated view" by either replaying the chain of views from the updated base, + or with a single call to as_strided. + + If set_view_replay_enabled is set to True, then autograd will always use view replay. + Otherwise, it will fall back to its existing logic. + + Args: + mode (bool): Flag whether to enable view-replay (``True``), or disable + (``False``). + + """ + + def __init__(self, mode: bool) -> None: + self.prev = torch._C._is_view_replay_enabled() + torch._C._set_view_replay_enabled(mode) + self.mode = mode + + def __enter__(self) -> None: + pass + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + torch._C._set_view_replay_enabled(self.prev) + + def clone(self): + return self.__class__(self.mode) + + +class _unsafe_preserve_version_counter(_DecoratorContextManager): + r"""DO NOT USE THIS UNLESS YOU KNOW EXACTLY WHAT YOU'RE DOING. + + This context manager can lead to arbitrary silent-correctness issues in any other part of your code + (even the ones not touched directly by the context manager)! + + Ordinarily, autograd will track mutations to tensors by incrementing it's `._version` attribute. + This is generally important for correctness, as for example, mutating a tensor that autograd has saved + for the backwards pass can result in incorrect gradients, and autograd uses the version counter to detect + and error out in this situation. + + However, there are rare instances where it might be useful to hide mutations from autograd. For example: + if a tensor is very large, and you'd like to free its memory by storing it elsewhere, and re-populate + the tensor right before it is needed by autograd. + + Args: + tensor (torch.Tensor): the tensor in question, that you would like to preserve the version counter of. + + .. note:: + This API does not apply to :ref:`forward-mode AD `. + + """ + + def __init__(self, tensors: Union[torch.Tensor, tuple[torch.Tensor, ...]]) -> None: + self.tensors = (tensors,) if isinstance(tensors, torch.Tensor) else tensors + if not isinstance(self.tensors, tuple): + raise AssertionError("Expected tensors to be a tuple") + self.prev_versions = tuple(t._version for t in self.tensors) + + def __enter__(self) -> None: + pass + + # pyrefly: ignore [bad-override] + def __exit__(self, *args) -> None: + torch._C._autograd._unsafe_set_version_counter(self.tensors, self.prev_versions) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/gradcheck.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/gradcheck.py new file mode 100644 index 0000000000000000000000000000000000000000..053be3450d6d226431f3b70b582f627658e319f2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/gradcheck.py @@ -0,0 +1,2306 @@ +# mypy: allow-untyped-defs +import collections +import functools +import warnings +from collections.abc import Callable, Iterable +from itertools import product +from typing import Optional, Union +from typing_extensions import deprecated + +import torch +import torch.testing + +# pyrefly: ignore [deprecated] +from torch._vmap_internals import _vmap, vmap +from torch.overrides import is_tensor_like +from torch.types import _TensorOrTensors + + +# Note: `get_*_jacobian` functions are added here even though we didn't intend to make them public +# since they have been exposed from before we added `__all__` and we already maintain BC for them +# We should eventually deprecate them and remove them from `__all__` +__all__ = [ + "gradcheck", + "gradgradcheck", + "GradcheckError", + "get_numerical_jacobian", + "get_analytical_jacobian", + "get_numerical_jacobian_wrt_specific_input", +] + + +class GradcheckError(RuntimeError): + r"""Error raised by :func:`gradcheck` and :func:`gradgradcheck`.""" + + +def _is_sparse_compressed_tensor(obj: torch.Tensor): + return obj.layout in { + torch.sparse_csr, + torch.sparse_csc, + torch.sparse_bsr, + torch.sparse_bsc, + } + + +def _is_sparse_any_tensor(obj: torch.Tensor): + return _is_sparse_compressed_tensor(obj) or obj.layout is torch.sparse_coo + + +def _is_float_or_complex_tensor(obj): + return is_tensor_like(obj) and (obj.is_floating_point() or obj.is_complex()) + + +def _allocate_jacobians_with_inputs( + input_tensors: tuple, numel_output +) -> tuple[torch.Tensor, ...]: + # Makes zero-filled tensors from inputs. If `numel_output` is not None, for + # each tensor in `input_tensors`, returns a new zero-filled tensor with height + # of `t.numel` and width of `numel_output`. Otherwise, for each tensor, returns + # a 1-d tensor with size `(t.numel,)`. Each new tensor will be strided and have + # the same dtype and device as those of the corresponding input. + out: list[torch.Tensor] = [ + t.new_zeros((t.numel(), numel_output), layout=torch.strided) + for t in input_tensors + if _is_float_or_complex_tensor(t) and t.requires_grad + ] + return tuple(out) + + +def _allocate_jacobians_with_outputs( + output_tensors: tuple, numel_input, dtype=None, device=None +) -> tuple[torch.Tensor, ...]: + # Makes zero-filled tensors from outputs. If `dim` is not None, for each tensor + # in `output_tensors`, returns a new zero-filled tensor with height of `dim` and + # width of `t.numel`. Otherwise, for each tensor, returns a 1-d tensor with size + # (t.numel,). + options = {"dtype": dtype, "device": device, "layout": torch.strided} + out: list[torch.Tensor] = [ + t.new_zeros((numel_input, t.numel()), **options) + for t in output_tensors + if _is_float_or_complex_tensor(t) + ] + return tuple(out) + + +def _iter_tensors( + x: Union[torch.Tensor, Iterable[torch.Tensor]], only_requiring_grad: bool = False +) -> Iterable[torch.Tensor]: + if is_tensor_like(x): + # mypy doesn't narrow type of `x` to torch.Tensor + if x.requires_grad or not only_requiring_grad: # type: ignore[union-attr] + yield x # type: ignore[misc] + elif isinstance(x, collections.abc.Iterable) and not isinstance(x, str): + for elem in x: + yield from _iter_tensors(elem, only_requiring_grad) + + +def _densify(x): + # return a copy of sparse x with all unspecified elements + # "replaced" with zero-valued elements + if isinstance(x, (list, tuple)): + return type(x)(map(_densify, x)) + elif not is_tensor_like(x) or x.layout in {torch.strided, torch._mkldnn}: # type: ignore[attr-defined] # no attr _mkldnn + return x + elif x.layout is torch.sparse_coo: + device = x.device + indices_dtype = x._indices().dtype + tmp = torch.ones(x.shape[: x.sparse_dim()], dtype=torch.int8, device=device) + indices = tmp.nonzero().t().to(dtype=indices_dtype) + values = torch.zeros( + (tmp.numel(), *x.shape[x.sparse_dim() :]), dtype=x.dtype, device=device + ) + x_coalesced = x.detach().coalesce() + if x_coalesced.numel() > 0: + stride = tmp.stride() + flat_indices = ( + x_coalesced.indices() + .mul( + torch.tensor(stride, dtype=indices_dtype, device=device).unsqueeze( + 1 + ) + ) + .sum(0) + ) + values[flat_indices] = x_coalesced.values() + return ( + torch.sparse_coo_tensor(indices, values, x.shape) + ._coalesced_(True) + .requires_grad_(x.requires_grad) + ) + elif _is_sparse_compressed_tensor(x): + blocksize = ( + x.values().shape[1:3] + if x.layout in {torch.sparse_bsr, torch.sparse_bsc} + else None + ) + compressed_indices = ( + x.crow_indices() + if x.layout in {torch.sparse_csr, torch.sparse_bsr} + else x.ccol_indices() + ) + # We'll use intermediate sparse COO for simplicity + r = _densify(x.detach().to_sparse(layout=torch.sparse_coo)).to_sparse( + layout=x.layout, blocksize=blocksize + ) + # Check that all elements are specified also after `to_sparse` op: + dense_numel = r.values().numel() // max(1, r.values().shape[0]) + batch_numel = compressed_indices.numel() // compressed_indices.shape[-1] + sparse_numel = r.numel() // max(1, dense_numel * batch_numel) + if sparse_numel != r._nnz(): + raise AssertionError( + f"{x.layout} densify failed: expected nnz={sparse_numel} but got {r._nnz()}" + ) + return r.requires_grad_(x.requires_grad) + elif _is_sparse_any_tensor(x): + raise NotImplementedError(x.layout) + return x + + +def _iter_tensor(x_tensor): + # (Only used for slow gradcheck) Returns a generator that yields the following + # elements at each iteration: + # 1) a tensor: the same tensor is returned across all iterations. The tensor + # is not the same as the original x_tensor as given as input - it is + # prepared so that it can be modified in-place. Depending on whether the + # input tensor is strided, sparse, or dense, the returned tensor may or may + # not share storage with x_tensor. + # 2) a tuple of indices that can be used with advanced indexing (yielded in + # dictionary order) + # 3) flattened index that will be used to index into the Jacobian tensor + # + # For a tensor t with size (2, 2), _iter_tensor yields: + # `x, (0, 0), 0`, `x, (0, 1), 1`, `x, (1, 0), 2`, `x, (1, 1), 3` + # + # where x is the t.data of the original tensor. Perturbing the entry of x + # at index (1, 1) yields the 3rd column of the overall Jacobian matrix. + if _is_sparse_any_tensor(x_tensor): + + def get_stride(size): + dim = len(size) + tmp = 1 + stride = [0] * dim + for i in reversed(range(dim)): + stride[i] = tmp + tmp *= size[i] + return stride + + x_nnz = x_tensor._nnz() + x_size = list(x_tensor.size()) + if x_tensor.layout is torch.sparse_coo: + x_indices = x_tensor._indices().t() + x_values = x_tensor._values() + elif x_tensor.layout is torch.sparse_csr: + x_indices = torch._convert_indices_from_csr_to_coo( + x_tensor.crow_indices(), x_tensor.col_indices() + ).t() + x_values = x_tensor.values() + elif x_tensor.layout is torch.sparse_csc: + x_indices = torch._convert_indices_from_csr_to_coo( + x_tensor.ccol_indices(), x_tensor.row_indices(), transpose=True + ).t() + x_values = x_tensor.values() + elif x_tensor.layout is torch.sparse_bsr: + x_block_values = x_tensor.values() + x_blocksize = x_block_values.size()[1:3] + x_indices = ( + torch._convert_indices_from_csr_to_coo( + x_tensor.crow_indices(), x_tensor.col_indices() + ) + .repeat_interleave(x_blocksize[0] * x_blocksize[1], 1) + .mul_(torch.tensor(x_blocksize, device=x_tensor.device).reshape(2, 1)) + .add_( + torch.stack( + torch.where(torch.ones(x_blocksize, device=x_tensor.device)) + ).repeat(1, x_nnz) + ) + .t() + ) + x_values = x_block_values.flatten(0, 2) + x_nnz = x_values.size(0) + elif x_tensor.layout is torch.sparse_bsc: + x_block_values = x_tensor.values() + x_blocksize = x_block_values.size()[1:3] + x_indices = ( + torch._convert_indices_from_csr_to_coo( + x_tensor.ccol_indices(), x_tensor.row_indices(), transpose=True + ) + .repeat_interleave(x_blocksize[0] * x_blocksize[1], 1) + .mul_(torch.tensor(x_blocksize, device=x_tensor.device).reshape(2, 1)) + .add_( + torch.stack( + torch.where(torch.ones(x_blocksize, device=x_tensor.device)) + ).repeat(1, x_nnz) + ) + .t() + ) + x_values = x_block_values.flatten(0, 2) + x_nnz = x_values.size(0) + else: + raise NotImplementedError(f"_iter_tensor for {x_tensor.layout} input") + x_stride = get_stride(x_size) + # Use .data here to get around the version check + x_values = x_values.data + for i in range(x_nnz): + x_value = x_values[i] + for x_idx in product(*[range(m) for m in x_values.size()[1:]]): + indices = x_indices[i].tolist() + list(x_idx) + d_idx = sum(indices[k] * x_stride[k] for k in range(len(x_size))) + yield x_value, x_idx, d_idx + elif x_tensor.layout == torch._mkldnn: # type: ignore[attr-defined] + for d_idx, x_idx in enumerate(product(*[range(m) for m in x_tensor.size()])): + # this is really inefficient, but without indexing implemented, there's + # not really a better way than converting back and forth + x_tensor_dense = x_tensor.to_dense() + yield x_tensor_dense, x_idx, d_idx + else: + # Use .data here to get around the version check + x_tensor = x_tensor.data + for d_idx, x_idx in enumerate(product(*[range(m) for m in x_tensor.size()])): + yield x_tensor, x_idx, d_idx + + +def _get_numerical_jacobian( + fn, inputs, outputs=None, target=None, eps=1e-3, is_forward_ad=False +) -> list[tuple[torch.Tensor, ...]]: + """Compute the numerical Jacobian of `fn(inputs)` with respect to `target`. + + If not specified, targets are the input. Returns M * N Jacobians where N is the + number of tensors in target that require grad and M is the number of non-integral + outputs. + + Args: + fn: the function to compute the jacobian for + inputs: inputs to `fn` + outputs: provide precomputed outputs to avoid one extra invocation of fn + target: the Tensors wrt whom Jacobians are calculated (default=`inputs`) + eps: the magnitude of the perturbation during finite differencing + (default=`1e-3`) + is_forward_ad: if this numerical jacobian is computed to be checked wrt + forward AD gradients (this is used for error checking only) + + Returns: + A list of M N-tuples of tensors + + Note that `target` may not even be part of `input` to `fn`, so please be + **very careful** in this to not clone `target`. + """ + jacobians: list[tuple[torch.Tensor, ...]] = [] + if outputs is None: + outputs = _as_tuple(fn(*_as_tuple(inputs))) + if not is_forward_ad and any(o.is_complex() for o in outputs): + raise ValueError( + "Expected output to be non-complex. get_numerical_jacobian no " + "longer supports functions that return complex outputs." + ) + if target is None: + target = inputs + inp_indices = [ + i for i, a in enumerate(target) if is_tensor_like(a) and a.requires_grad + ] + for inp, inp_idx in zip(_iter_tensors(target, True), inp_indices): + jacobians += [ + get_numerical_jacobian_wrt_specific_input( + fn, + inp_idx, + inputs, + outputs, + eps, + input=inp, + is_forward_ad=is_forward_ad, + ) + ] + return jacobians + + +@deprecated( + "`get_numerical_jacobian` was part of PyTorch's private API and not " + "meant to be exposed. We are deprecating it and it will be removed " + "in a future version of PyTorch. If you have a specific use for " + "this or feature request for this to be a stable API, please file " + "us an issue at https://github.com/pytorch/pytorch/issues/new", + category=FutureWarning, +) +def get_numerical_jacobian(fn, inputs, target=None, eps=1e-3, grad_out=1.0): + """Compute the numerical Jacobian for a given fn and its inputs. + + This is a Deprecated API. + + Args: + fn: the function to compute the Jacobian for (must take inputs as a tuple) + inputs: input to `fn` + target: the Tensors wrt whom Jacobians are calculated (default=`input`) + eps: the magnitude of the perturbation during finite differencing + (default=`1e-3`) + grad_out: defaults to 1.0. + + Returns: + A list of Jacobians of `fn` (restricted to its first output) with respect to + each input or target, if provided. + + Note that `target` may not even be part of `input` to `fn`, so please be + **very careful** in this to not clone `target`. + """ + if ( + grad_out != 1.0 + ): # grad_out param is only kept for backward compatibility reasons + raise ValueError( + "Expected grad_out to be 1.0. get_numerical_jacobian no longer " + "supports values of grad_out != 1.0." + ) + + def fn_pack_inps(*inps): + return fn(inps) + + jacobians = _get_numerical_jacobian(fn_pack_inps, inputs, None, target, eps) + + return tuple(jacobian_for_each_output[0] for jacobian_for_each_output in jacobians) + + +def _compute_numerical_gradient(fn, entry, v, norm_v, nbhd_checks_fn): + # Computes numerical directional derivative as finite difference + # of function `fn` at input `entry`, perturbed by vector `v`. + if _is_sparse_compressed_tensor(entry): + # sparse compressed tensors don't implement sub/add/copy_ + # yet. However, in non-masked semantics context entry and v + # have the same sparse indices ... + if entry.layout != v.layout: + raise AssertionError( + f"Expected entry and v to have the same layout, but got {entry.layout} and {v.layout}" + ) + if entry._nnz() != v._nnz(): + raise AssertionError( + f"Expected entry and v to have the same nnz, but got {entry._nnz()} and {v._nnz()} " + f"with entry shape {entry.shape}" + ) + # ... the finite differencing can be performed on values only: + entry = entry.values() + v = v.values() + # we'll detach to avoid backward computations that sparse + # tensors have limited support for. + entry = entry.detach() + + orig = entry.clone() + entry.copy_(orig - v) + outa = fn() + entry.copy_(orig + v) + outb = fn() + entry.copy_(orig) + + def compute(a, b): + nbhd_checks_fn(a, b) + ret = (b - a) / (2 * norm_v) # use central difference approx + return ret.detach().reshape(-1) + + return tuple(compute(a, b) for (a, b) in zip(outa, outb)) + + +def _compute_numerical_jvps_wrt_specific_input( + jvp_fn, delta, input_is_complex, is_forward_ad=False +) -> list[torch.Tensor]: + # Computing the jacobian only works for real delta + # For details on the algorithm used here, refer: + # Section 3.5.3 https://arxiv.org/pdf/1701.00392.pdf + # s = fn(z) where z = x for real valued input + # and z = x + yj for complex valued input + jvps: list[torch.Tensor] = [] + ds_dx_tup = jvp_fn(delta[0] if isinstance(delta, tuple) else delta) + + if input_is_complex: # C -> R + ds_dy_tup = ( + jvp_fn(delta[1] * 1j) if isinstance(delta, tuple) else jvp_fn(delta * 1j) + ) + for ds_dx, ds_dy in zip(ds_dx_tup, ds_dy_tup): + if ds_dx.is_complex(): + raise AssertionError("Expected ds_dx to be real-valued, not complex") + # conjugate wirtinger derivative + conj_w_d = ds_dx + ds_dy * 1j + jvps.append(conj_w_d) + else: + for ds_dx in ds_dx_tup: # R -> R or (R -> C for the forward AD case) + if not is_forward_ad and ds_dx.is_complex(): + raise AssertionError("Expected ds_dx to be real-valued, not complex.") + jvps.append(ds_dx) + return jvps + + +def _combine_jacobian_cols( + jacobians_cols: dict[int, list[torch.Tensor]], outputs, input, numel +) -> tuple[torch.Tensor, ...]: + # jacobian_cols maps column_idx -> output_idx -> single column of jacobian Tensor + # we return a list that maps output_idx -> full jacobian Tensor + jacobians = _allocate_jacobians_with_outputs( + outputs, numel, dtype=input.dtype if input.dtype.is_complex else None + ) + for i, jacobian in enumerate(jacobians): + for k, v in jacobians_cols.items(): + jacobian[k] = v[i] + return jacobians + + +def _prepare_input( + input: torch.Tensor, maybe_perturbed_input: Optional[torch.Tensor], fast_mode=False +) -> torch.Tensor: + # Prepares the inputs to be passed into the function while including the new + # modified input. + if input.layout == torch._mkldnn: # type: ignore[attr-defined] # no attr _mkldnn + # Convert back to mkldnn + if maybe_perturbed_input is not None: + return maybe_perturbed_input.to_mkldnn() + else: + return input + elif _is_sparse_any_tensor(input): + if fast_mode and maybe_perturbed_input is not None: + # entry is already a "cloned" version of the original tensor + # thus changes to entry are not reflected in the input + return maybe_perturbed_input + else: + return input + else: + # We cannot use entry (input.data) if we want gradgrad to work because + # fn (in the gradgrad case) needs to compute grad wrt input + return input + + +def _check_outputs_same_dtype_and_shape(output1, output2, eps, idx=None) -> None: + # Check that the returned outputs don't have different dtype or shape when you + # perturb the input + on_index = f"on index {idx} " if idx is not None else "" + if output1.shape != output2.shape: + raise AssertionError( + f"Expected `func` to return outputs with the same shape" + f" when inputs are perturbed {on_index}by {eps}, but got:" + f" shapes {output1.shape} and {output2.shape}." + ) + if output1.dtype != output2.dtype: + raise AssertionError( + f"Expected `func` to return outputs with the same dtype" + f" when inputs are perturbed {on_index}by {eps}, but got:" + f" dtypes {output1.dtype} and {output2.dtype}." + ) + + +def get_numerical_jacobian_wrt_specific_input( + fn, input_idx, inputs, outputs, eps, input=None, is_forward_ad=False +) -> tuple[torch.Tensor, ...]: + # Computes the numerical jacobians wrt to a single input. Returns N jacobian + # tensors, where N is the number of outputs. We use a dictionary for + # jacobian_cols because indices aren't necessarily consecutive for sparse inputs + # When we perturb only a single element of the input tensor at a time, the jvp + # is equivalent to a single col of the Jacobian matrix of fn. + jacobian_cols: dict[int, list[torch.Tensor]] = {} + input = inputs[input_idx] if input is None else input + if not input.requires_grad: + raise AssertionError("Expected input to have requires_grad=True") + for x, idx, d_idx in _iter_tensor(input): + wrapped_fn = _with_prepare_inputs(fn, inputs, input_idx, x) + input_to_perturb = x[idx] + nbhd_checks_fn = functools.partial( + _check_outputs_same_dtype_and_shape, idx=idx, eps=eps + ) + jvp_fn = _get_numerical_jvp_fn( + wrapped_fn, input_to_perturb, eps, nbhd_checks_fn + ) + jacobian_cols[d_idx] = _compute_numerical_jvps_wrt_specific_input( + jvp_fn, eps, x.is_complex(), is_forward_ad + ) + return _combine_jacobian_cols(jacobian_cols, outputs, input, input.numel()) + + +def _get_analytical_jacobian_forward_ad( + fn, inputs, outputs, *, check_grad_dtypes=False, all_u=None +) -> tuple[tuple[torch.Tensor, ...], ...]: + """Compute the analytical Jacobian using forward mode AD of `fn(inputs)` using forward mode AD with respect to `target`. + + Return N * M Jacobians where N is the number of tensors in target that require grad and + M is the number of non-integral outputs. + Contrary to other functions here, this function requires "inputs" to actually be used by the function. + The computed value is expected to be wrong if the function captures the inputs by side effect instead of + using the passed ones (many torch.nn tests do this). + + Args: + fn: the function to compute the jacobian for + inputs: inputs to `fn` + outputs: provide precomputed outputs to avoid one extra invocation of fn + check_grad_dtypes: if True, will check that the gradient dtype are valid + all_u (optional): if provided, the Jacobian will be right multiplied with this vector + + Returns: + A tuple of M N-tuples of tensors + """ + # To avoid early import issues + fwAD = torch.autograd.forward_ad + + tensor_inputs = tuple(i for i in inputs if is_tensor_like(i) and i.requires_grad) + + if any(i.is_complex() for i in tensor_inputs): + raise ValueError( + "Expected inputs to be non-complex for _get_analytical_jacobian_forward_ad." + ) + + if all_u: + jacobians = tuple( + _allocate_jacobians_with_outputs(outputs, 1) for i in tensor_inputs + ) + else: + jacobians = tuple( + _allocate_jacobians_with_outputs(outputs, i.numel()) for i in tensor_inputs + ) + + with fwAD.dual_level(): + fw_grads = [] + dual_inputs = [] + for inp in inputs: + if is_tensor_like(inp) and inp.requires_grad: + if inp.layout == torch._mkldnn: # type: ignore[attr-defined] + raise ValueError( + "MKLDNN inputs are not support for forward AD gradcheck." + ) + + inp = fwAD.make_dual(inp.detach(), torch.zeros_like(inp)) + # If inp is a differentiable view, the dual might not be the tangent given to + # make_dual, so read it explicitly from the dual tensor + fw_grads.append(fwAD.unpack_dual(inp)[1]) + dual_inputs.append(inp) + + if all_u: + # Do the full reduction in one pass + # To be consistent with numerical evaluation, we actually compute one reduction per input + for i, (fw_grad, u) in enumerate(zip(fw_grads, all_u)): + fw_grad.copy_(u.view_as(fw_grad)) + raw_outputs = _as_tuple(fn(*dual_inputs)) + dual_outputs = filter(_is_float_or_complex_tensor, raw_outputs) + for index_o, d_o in enumerate(dual_outputs): + val, res = fwAD.unpack_dual(d_o) + if ( + check_grad_dtypes + and res is not None + and val.is_complex() != res.is_complex() + ): + raise GradcheckError("Forward AD gradient has dtype mismatch.") + + # Remove extra dimension of size 1 corresponding to the reduced input + jacobians[i][index_o].squeeze_(0) + if res is None: + jacobians[i][index_o].zero_() + else: + jacobians[i][index_o].copy_(res.reshape(-1)) + fw_grad.zero_() + else: + # Reconstruct the full Jacobian column by column + for i, fw_grad in enumerate(fw_grads): + for lin_idx, grad_idx in enumerate( + product(*[range(m) for m in fw_grad.size()]) + ): + fw_grad[grad_idx] = 1.0 + raw_outputs = _as_tuple(fn(*dual_inputs)) + dual_outputs = filter(_is_float_or_complex_tensor, raw_outputs) + for index_o, d_o in enumerate(dual_outputs): + val, res = fwAD.unpack_dual(d_o) + if ( + check_grad_dtypes + and res is not None + and val.is_complex() != res.is_complex() + ): + raise GradcheckError( + "Forward AD gradient has dtype mismatch." + ) + + if res is None: + jacobians[i][index_o][lin_idx].zero_() + else: + jacobians[i][index_o][lin_idx].copy_(res.reshape(-1)) + fw_grad[grad_idx] = 0.0 + + return jacobians + + +def _get_input_to_perturb(input): + # Prepare the input so that it can be modified in-place and do certain + # operations that require the tensor to have strides. If fast_mode=False, + # _iter_tensor would handle the below cases: + if input.layout == torch._mkldnn: # type: ignore[attr-defined] # no attr _mkldnn + # Convert to dense so we can perform operations that require strided tensors + input_to_perturb = input.to_dense() + elif _is_sparse_any_tensor(input): + # Clone because input may require grad, and copy_ calls resize_, + # which is not allowed for .data + input_to_perturb = input.clone() + else: + input_to_perturb = input.data + return input_to_perturb + + +def _with_prepare_inputs(fn, inputs, input_idx, input_to_perturb, fast_mode=False): + # Wraps `fn` so that its inputs are already supplied + def wrapped_fn(): + inp = tuple( + _prepare_input(a, input_to_perturb if i == input_idx else None, fast_mode) + if is_tensor_like(a) + else a + for i, a in enumerate(_as_tuple(inputs)) + ) + return tuple(a.clone() for a in _as_tuple(fn(*inp))) + + return wrapped_fn + + +def _get_numerical_jvp_fn(wrapped_fn, input_to_perturb, eps, nbhd_checks_fn): + # Wraps jvp_fn so that certain arguments are already supplied + def jvp_fn(delta): + return _compute_numerical_gradient( + wrapped_fn, input_to_perturb, delta, eps, nbhd_checks_fn + ) + + return jvp_fn + + +def _reshape_tensor_or_tuple(u, shape): + # We don't need to reshape when input corresponding to u is sparse + if isinstance(u, tuple): + if not _is_sparse_any_tensor(u[0]): + return (u[0].reshape(shape), u[1].reshape(shape)) + else: + if not _is_sparse_any_tensor(u): + return u.reshape(shape) + return u + + +def _mul_tensor_or_tuple(u, k): + if isinstance(u, tuple): + return (k * u[0], k * u[1]) + else: + return k * u + + +def _get_numerical_jvp_wrt_specific_input( + fn, input_idx, inputs, u, eps, is_forward_ad=False +) -> list[torch.Tensor]: + input = inputs[input_idx] + input_to_perturb = _get_input_to_perturb(input) + wrapped_fn = _with_prepare_inputs(fn, inputs, input_idx, input_to_perturb, True) + nbhd_checks_fn = functools.partial(_check_outputs_same_dtype_and_shape, eps=eps) + jvp_fn = _get_numerical_jvp_fn(wrapped_fn, input_to_perturb, eps, nbhd_checks_fn) + u = _reshape_tensor_or_tuple(u, input_to_perturb.shape) + u = _mul_tensor_or_tuple(u, eps) + return _compute_numerical_jvps_wrt_specific_input( + jvp_fn, u, input.is_complex(), is_forward_ad + ) + + +def _get_numerical_vJu( + fn, inputs, inp_indices, func_out, all_u, all_v, eps, is_forward_ad +): + # Note that all_v can also be None, in that case, this function only computes Ju. + reduced_jacobians: list[list[torch.Tensor]] = [] + for inp_idx, u in zip(inp_indices, all_u): + all_Ju = _get_numerical_jvp_wrt_specific_input( + fn, inp_idx, inputs, u, eps, is_forward_ad + ) + # Filter out the Ju for non floating point outputs + filtered_Ju = [] + func_out = _as_tuple(func_out) + if len(all_Ju) != len(func_out): + raise AssertionError( + f"Expected all_Ju and func_out to have the same length, " + f"but got {len(all_Ju)} and {len(func_out)}" + ) + for Ju, output in zip(all_Ju, func_out): + if _is_float_or_complex_tensor(output): + filtered_Ju.append(Ju) + else: + # TODO: handle the other Ju + pass + if all_v is not None: + jacobian_scalars: list[torch.Tensor] = [] + for v, Ju in zip(all_v, filtered_Ju): + jacobian_scalars.append(_dot_with_type_promotion(v, Ju)) + reduced_jacobians.append(jacobian_scalars) + else: + reduced_jacobians.append(filtered_Ju) + return reduced_jacobians + + +def _check_jacobians_equal(j1, j2, atol): + # Check whether the max difference between two Jacobian tensors are within some + # tolerance `atol`. + for j1_x, j2_x in zip(j1, j2): + if j1_x.numel() != 0 and (j1_x - j2_x).abs().max() > atol: + return False + return True + + +def _stack_and_check_tensors( + list_of_list_of_tensors, inputs, numel_outputs +) -> tuple[tuple[torch.Tensor, ...], bool, bool]: + # For the ith tensor in the inner list checks whether it has the same size and + # dtype as the ith differentiable input. + out_jacobians = _allocate_jacobians_with_inputs(inputs, numel_outputs) + diff_input_list = list(_iter_tensors(inputs, True)) + correct_grad_sizes = True + correct_grad_types = True + for i, tensor_list in enumerate(list_of_list_of_tensors): + inp = diff_input_list[i] + out_jacobian = out_jacobians[i] + for j, tensor in enumerate(tensor_list): + if tensor is not None and tensor.size() != inp.size(): + correct_grad_sizes = False + elif tensor is not None and tensor.dtype != inp.dtype: + correct_grad_types = False + if tensor is None: + out_jacobian[:, j].zero_() + else: + dense = tensor.to_dense() if tensor.layout != torch.strided else tensor + if out_jacobian[:, j].numel() != dense.numel(): + raise AssertionError( + f"Expected out_jacobian column to have {dense.numel()} elements, " + f"but got {out_jacobian[:, j].numel()}" + ) + out_jacobian[:, j] = dense.reshape(-1) + return out_jacobians, correct_grad_sizes, correct_grad_types + + +FAILED_NONDET_MSG = """\n +NOTE: If your op relies on non-deterministic operations i.e., it is listed here: +https://pytorch.org/docs/stable/generated/torch.use_deterministic_algorithms.html +this failure might be expected. + +If you are adding a new operator, please file an issue and then use one of the +workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck. +If the test +- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck + with `nondet_tol=` as a keyword argument. +- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test + to have `gradcheck_nondet_tol=`. +- is a Module test (e.g., in common_nn.py), then modify the corresponding + module_test entry to have `gradcheck_nondet_tol=` +""" + + +def _check_analytical_jacobian_attributes( + inputs, output, nondet_tol, check_grad_dtypes, fast_mode=False, v=None +) -> tuple[torch.Tensor, ...]: + # This is used by both fast and slow mode: + # - For slow mode, vjps[i][j] is the jth row of the Jacobian wrt the ith + # input. + # - For fast mode, vjps[i][0] is a linear combination of the rows + # of the Jacobian wrt the ith input + diff_input_list = list(_iter_tensors(inputs, True)) + + def vjp_fn(grad_output): + return torch.autograd.grad( + output, diff_input_list, grad_output, retain_graph=True, allow_unused=True + ) + + # Compute everything twice to check for nondeterminism (which we call reentrancy) + if fast_mode: + vjps1 = _get_analytical_vjps_wrt_specific_output(vjp_fn, output.clone(), v) + vjps2 = _get_analytical_vjps_wrt_specific_output(vjp_fn, output.clone(), v) + else: + vjps1 = _compute_analytical_jacobian_rows(vjp_fn, output.clone()) + vjps2 = _compute_analytical_jacobian_rows(vjp_fn, output.clone()) + + output_numel = output.numel() if not fast_mode else 1 + jacobians1, types_ok, sizes_ok = _stack_and_check_tensors( + vjps1, inputs, output_numel + ) + jacobians2, _, _ = _stack_and_check_tensors(vjps2, inputs, output_numel) + reentrant = _check_jacobians_equal(jacobians1, jacobians2, nondet_tol) + + if not types_ok and check_grad_dtypes: + raise GradcheckError("Gradient has dtype mismatch") + if not sizes_ok: + raise GradcheckError("Analytical gradient has incorrect size") + if not reentrant: + raise GradcheckError( + "Backward is not reentrant, i.e., running backward with " + "same input and grad_output multiple times gives different values, " + "although analytical gradient matches numerical gradient." + f"The tolerance for nondeterminism was {nondet_tol}." + FAILED_NONDET_MSG + ) + return jacobians1 + + +def _get_analytical_vJu_backward_mode( + inputs, outputs, nondet_tol, check_grad_dtypes, all_v, all_u +): + reduced_jacobians: list[list[torch.Tensor]] = [] + for output, v in zip(outputs, all_v): + all_vJ = _check_analytical_jacobian_attributes( + inputs, output, nondet_tol, check_grad_dtypes, fast_mode=True, v=v + ) + jacobian_scalars: list[torch.Tensor] = [] + for vJ, u in zip(all_vJ, all_u): + # Why do we need squeeze here? vJ is a 2-d tensor so that we can reuse + # the error checking logic from slow mode + vJ = vJ.T.squeeze(0) + if vJ.is_complex(): # C -> R + tv = torch.view_as_real(vJ.resolve_conj()) + tr = tv.select(-1, 0) + ti = tv.select(-1, 1) + jacobian_scalars.append(tr.dot(u[0]) + 1j * ti.dot(u[1])) + else: # R -> R + jacobian_scalars.append(vJ.dot(u)) + reduced_jacobians.append(jacobian_scalars) + return reduced_jacobians + + +@deprecated( + "`get_analytical_jacobian` was part of PyTorch's private API and not " + "meant to be exposed. We are deprecating it and it will be removed " + "in a future version of PyTorch. If you have a specific use for " + "this or feature request for this to be a stable API, please file " + "us an issue at https://github.com/pytorch/pytorch/issues/new", + category=FutureWarning, +) +def get_analytical_jacobian(inputs, output, nondet_tol=0.0, grad_out=1.0): + # Replicates the behavior of the old get_analytical_jacobian before the refactor + # This shares much of its code with _check_analytical_jacobian_attributes + if ( + grad_out != 1.0 + ): # grad_out param is only kept for backward compatibility reasons + raise ValueError( + "Expected grad_out to be 1.0. get_analytical_jacobian no longer " + "supports values of grad_out != 1.0." + ) + if output.is_complex(): + raise ValueError( + "Expected output to be non-complex. get_analytical_jacobian no " + "longer supports functions that return complex outputs." + ) + diff_input_list = list(_iter_tensors(inputs, True)) + + def vjp_fn(grad_output): + return torch.autograd.grad( + output, diff_input_list, grad_output, retain_graph=True, allow_unused=True + ) + + # Compute everything twice to check for nondeterminism (which we call reentrancy) + vjps1 = _compute_analytical_jacobian_rows(vjp_fn, output.clone()) + vjps2 = _compute_analytical_jacobian_rows(vjp_fn, output.clone()) + + output_numel = output.numel() + jacobians1, types_ok, sizes_ok = _stack_and_check_tensors( + vjps1, inputs, output_numel + ) + jacobians2, _, _ = _stack_and_check_tensors(vjps2, inputs, output_numel) + reentrant = _check_jacobians_equal(jacobians1, jacobians2, nondet_tol) + + return jacobians1, reentrant, sizes_ok, types_ok + + +def _get_analytical_jacobian(inputs, outputs, input_idx, output_idx): + # Computes the analytical Jacobian in slow mode for a single input-output pair. + # Forgoes performing checks on dtype, shape, and reentrancy. + jacobians = _check_analytical_jacobian_attributes( + inputs, outputs[output_idx], nondet_tol=float("inf"), check_grad_dtypes=False + ) + return jacobians[input_idx] + + +def _compute_analytical_jacobian_rows( + vjp_fn, sample_output +) -> list[list[Optional[torch.Tensor]]]: + # Computes Jacobian row-by-row by projecting `vjp_fn` = v^T J on standard basis + # vectors: vjp_fn(e) = e^T J is a corresponding row of the Jacobian. + # NB: this function does not assume vjp_fn(v) to return tensors with the same + # number of elements for different v. This is checked when we later combine the + # rows into a single tensor. + grad_out_base = torch.zeros_like( + sample_output, memory_format=torch.legacy_contiguous_format + ) + flat_grad_out = grad_out_base.view(-1) + # jacobians_rows[i][j] is the Jacobian jth row for the ith input + jacobians_rows: list[list[Optional[torch.Tensor]]] = [] + for j in range(flat_grad_out.numel()): + flat_grad_out.zero_() + flat_grad_out[j] = 1.0 # projection for jth row of Jacobian + grad_inputs = vjp_fn(grad_out_base) + for i, d_x in enumerate(grad_inputs): + if j == 0: + jacobians_rows.append([]) + jacobians_rows[i] += [ + d_x.clone() if isinstance(d_x, torch.Tensor) else None + ] + return jacobians_rows + + +def _get_analytical_vjps_wrt_specific_output( + vjp_fn, sample_output, v +) -> list[list[Optional[torch.Tensor]]]: + grad_inputs = vjp_fn(v.reshape(sample_output.shape)) + vjps: list[list[Optional[torch.Tensor]]] = [ + [vjp.clone() if isinstance(vjp, torch.Tensor) else None] for vjp in grad_inputs + ] + return vjps + + +def _check_inputs(tupled_inputs) -> bool: + # Make sure that gradients are saved for at least one input + any_input_requiring_grad = False + for idx, inp in enumerate(tupled_inputs): + if is_tensor_like(inp) and inp.requires_grad: + if not (inp.dtype == torch.float64 or inp.dtype == torch.complex128): + warnings.warn( + f"Input #{idx} requires gradient and " + "is not a double precision floating point or complex. " + "This check will likely fail if all the inputs are " + "not of double precision floating point or complex. ", + stacklevel=2, + ) + if inp.is_sparse: + content = inp._values() + elif _is_sparse_compressed_tensor(inp): + content = inp.values() + else: + content = inp + # TODO: To cover more problematic cases, replace stride = 0 check with + # "any overlap in memory" once we have a proper function to check it. + if content.layout is not torch._mkldnn: # type: ignore[attr-defined] + if not all( + st > 0 or sz <= 1 + for st, sz in zip(content.stride(), content.size()) + ): + raise RuntimeError( + f"The {idx}th input has a dimension with stride 0. gradcheck only " + "supports inputs that are non-overlapping to be able to " + "compute the numerical gradients correctly. You should call " + ".contiguous on the input before passing it to gradcheck." + ) + any_input_requiring_grad = True + + if not any_input_requiring_grad: + raise ValueError( + "gradcheck expects at least one input tensor to require gradient, " + "but none of the them have requires_grad=True." + ) + return True + + +def _check_outputs(outputs) -> None: + if any(_is_sparse_any_tensor(t) for t in outputs if isinstance(t, torch.Tensor)): + # it is easier to call to_dense() on the sparse output than + # to modify analytical jacobian + raise ValueError( + "Sparse output is not supported at gradcheck yet. " + "Please call to_dense(masked_grad=...) on the output of fn for gradcheck." + ) + if any(t.layout == torch._mkldnn for t in outputs if isinstance(t, torch.Tensor)): # type: ignore[attr-defined] + raise ValueError( + "MKLDNN output is not supported at gradcheck yet. " + "Please call to_dense(masked_grad=...) on the output of fn for gradcheck." + ) + + +def _check_no_differentiable_outputs( + func, inputs, func_out, eps, *, is_forward_ad +) -> bool: + # When there are no differentiable outputs, numerical gradient for a function is + # expected to be zero. + jacobians_all_inputs_outputs = _get_numerical_jacobian( + func, inputs, func_out, eps=eps, is_forward_ad=is_forward_ad + ) + for jacobians_all_outputs_and_fixed_input in jacobians_all_inputs_outputs: + for jacobian in jacobians_all_outputs_and_fixed_input: + if torch.ne(jacobian, 0).sum() > 0: + raise GradcheckError( + "Numerical gradient for function expected to be zero" + ) + return True + + +def _check_no_differentiable_outputs_fast( + func, func_out, all_inputs, inputs_indices, all_u, eps, nondet_tol +): + for inp_idx, u in zip(inputs_indices, all_u): + jvps = _get_numerical_jvp_wrt_specific_input(func, inp_idx, all_inputs, u, eps) + for jvp in jvps: + if jvp.numel() == 0: + continue + if (jvp - torch.zeros_like(jvp)).abs().max() > nondet_tol: + raise GradcheckError( + "Numerical gradient for function expected to be zero" + ) + return True + + +FAILED_BATCHED_GRAD_MSG = """ +gradcheck or gradgradcheck failed while testing batched gradient computation. +This could have been invoked in a number of ways (via a test that calls +gradcheck/gradgradcheck directly or via an autogenerated test). + +If you are adding a new operator, please file an issue and then use one of the +workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck. +If the test +- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck + with `check_batched_grad=False` as a keyword argument. +- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test + to have `check_batched_grad=False` and/or `check_batched_gradgrad=False`. + +If you're modifying an existing operator that supports batched grad computation, +or wish to make a new operator work with batched grad computation, please read +the following. + +To compute batched grads (e.g., jacobians, hessians), we vmap over the backward +computation. The most common failure case is if there is a 'vmap-incompatible +operation' in the backward pass. Please see +NOTE: [How to write vmap-compatible backward formulas] +in the codebase for an explanation of how to fix this. +""".strip() + +FAILED_BATCHED_GRAD_MSG_FWD_AD = """ +gradcheck failed while testing batched gradient computation with forward-mode AD. +This test is enabled automatically when both `check_batched_grad=True` +and `check_forward_ad=True`, but can be disabled in the following ways +dependong on how the test was invoked (via a test that calls gradcheck +directly or via an autogenerated test). + +If you are adding a new operator, please file an issue and then use one of the +workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck. +If the test +- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck + with `check_batched_forward_grad=False` as a keyword argument. +- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test + to have `check_batched_forward_grad=False` +""" + + +def _get_failed_batched_grad_test_msg( + output_idx, input_idx, res, exp, is_forward_ad=False +): + return f""" +For output {output_idx} and input {input_idx}: + +{FAILED_BATCHED_GRAD_MSG_FWD_AD if is_forward_ad else FAILED_BATCHED_GRAD_MSG} + +Got: +{res} + +Expected: +{exp} +""".strip() + + +def _test_batched_grad_forward_ad(func, inputs) -> bool: + fwAD = torch.autograd.forward_ad # To avoid early import issues (do we need this?) + if not isinstance(inputs, tuple): + raise AssertionError("Expected inputs to be a tuple") + + for input_idx, current_input in enumerate(inputs): + if not (is_tensor_like(current_input) and current_input.requires_grad): + continue + + def jvp(tangent: torch.Tensor): + with fwAD.dual_level(): + dual = fwAD.make_dual(current_input.detach(), tangent) + inputs_with_dual = tuple( + dual + if idx == input_idx + else (inp.detach() if is_tensor_like(inp) else inp) + for idx, inp in enumerate(inputs) + ) + dual_outputs = _as_tuple(func(*inputs_with_dual)) + ret = [] + for dual_output in dual_outputs: + if dual_output is None: + continue + primal_out, tangent_out = fwAD.unpack_dual(dual_output) + if tangent_out is not None: + ret.append(tangent_out) + else: + ret.append( + torch.zeros( + [], dtype=primal_out.dtype, device=primal_out.device + ).expand(primal_out.shape) + ) + return tuple(ret) + + if not _is_float_or_complex_tensor(current_input): + continue + + tangents = [torch.randn_like(current_input) for _ in range(2)] + expected = [jvp(t) for t in tangents] + expected = [torch.stack(shards) for shards in zip(*expected)] + + try: + result = _vmap(jvp)(torch.stack(tangents)) + except RuntimeError as ex: + # Rethrow to provide a better error message + raise GradcheckError( + f"While computing batched gradients, got: {ex}\n\n{FAILED_BATCHED_GRAD_MSG_FWD_AD}" + ) from ex + + for input_idx, (res, exp) in enumerate(zip(result, expected)): + if torch.allclose(res, exp): + continue + raise GradcheckError( + _get_failed_batched_grad_test_msg( + input_idx, input_idx, res, exp, is_forward_ad=True + ) + ) + return True + + +def _test_batched_grad(input, output, output_idx) -> bool: + # NB: _test_batched_grad compares two autograd.grad invocations with a single + # vmap(autograd.grad) invocation. It's not exactly a "gradcheck" in the + # sense that we're not comparing an analytical jacobian with a numeric one, + # but it is morally similar (we could have computed a full analytic jac + # via vmap, but that is potentially slow) + diff_input_list = list(_iter_tensors(input, True)) + grad = functools.partial( + torch.autograd.grad, + output, + diff_input_list, + retain_graph=True, + allow_unused=True, + ) + + def vjp(v): + results = grad(v) + results = tuple( + grad + if grad is not None + else torch.zeros([], dtype=inp.dtype, device=inp.device).expand(inp.shape) + for grad, inp in zip(results, diff_input_list) + ) + return results + + grad_outputs = [torch.randn_like(output) for _ in range(2)] + + expected = [vjp(gO) for gO in grad_outputs] + expected = [torch.stack(shards) for shards in zip(*expected)] + + # Squash warnings since these are expected to happen in most cases + # NB: this doesn't work for CUDA tests: https://github.com/pytorch/pytorch/issues/50209 + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="There is a performance drop") + warnings.filterwarnings("ignore", message="Please use `torch.vmap`") + try: + result = vmap(vjp)(torch.stack(grad_outputs)) + except RuntimeError as ex: + # It's OK that we're not raising the error at the correct callsite. + # That's because the callsite is always going to inside the Python + # autograd.grad instead of the C++ traceback of what line in the + # backward formula + raise GradcheckError( + f"While computing batched gradients, got: {ex}\n\n{FAILED_BATCHED_GRAD_MSG}" + ) from ex + + for input_idx, (res, exp) in enumerate(zip(result, expected)): + if torch.allclose(res, exp): + continue + raise GradcheckError( + _get_failed_batched_grad_test_msg(output_idx, input_idx, res, exp) + ) + return True + + +def _test_backward_mul_by_grad_output(outputs, inputs, masked) -> bool: + # Tests that backward is multiplied by grad_output + diff_input_list: list[torch.Tensor] = list(_iter_tensors(inputs, True)) + if not diff_input_list: + raise GradcheckError("no Tensors requiring grad found in input") + grads_input = torch.autograd.grad( + outputs, + diff_input_list, + [ + torch.zeros_like(o, memory_format=torch.legacy_contiguous_format) + for o in outputs + ], + allow_unused=True, + ) + for gi, di in zip(grads_input, diff_input_list): + if gi is None: + continue + if isinstance(gi, torch.Tensor) and gi.layout != torch.strided: + if gi.layout != di.layout: + raise GradcheckError( + "grad is incorrect layout (" + + str(gi.layout) + + " is not " + + str(di.layout) + + ")" + ) + if _is_sparse_any_tensor(gi): + sparse_kind = str(gi.layout).replace("torch.", "").replace("_coo", "") + if gi.sparse_dim() != di.sparse_dim(): + raise GradcheckError( + f"grad is {sparse_kind} tensor, but has incorrect sparse_dim" + f" {gi.sparse_dim()}, expected {di.sparse_dim()}" + ) + if gi.dense_dim() != di.dense_dim(): + raise GradcheckError( + f"grad is {sparse_kind} tensor, but has incorrect dense_dim" + f" {gi.dense_dim()}, expected {di.dense_dim()}" + ) + gi = gi.to_dense() + di = di.to_dense() + if masked: + if not torch.allclose(gi, torch.zeros_like(gi)): + raise GradcheckError("backward not multiplied by grad_output") + elif not gi.eq(0).all(): + raise GradcheckError("backward not multiplied by grad_output") + if gi.dtype != di.dtype: + raise GradcheckError("grad is incorrect type") + if gi.device != di.device: + raise GradcheckError("grad is incorrect device") + if gi.size() != di.size(): + raise GradcheckError("grad is incorrect size") + return True + + +def _test_undefined_forward_mode(func, outputs, inputs): + fwAD = torch.autograd.forward_ad + + _inp_tensors_idx, inp_tensors = _get_inp_tensors(inputs) + _all_v, all_u, _all_u_dense = _make_vectors( + inp_tensors, outputs, use_forward_ad=True + ) + + with fwAD.dual_level(): + fw_grads = [] + dual_inputs = [] + tensor_indices = set() + for i, inp in enumerate(inputs): + if is_tensor_like(inp) and inp.requires_grad: + if inp.layout == torch._mkldnn: # type: ignore[attr-defined] + raise ValueError( + "MKLDNN inputs are not support for forward AD gradcheck." + ) + + inp = fwAD.make_dual(inp.detach(), torch.zeros_like(inp)) + # If inp is a differentiable view, the dual might not be the tangent given to + # make_dual, so read it explicitly from the dual tensor + fw_grads.append(fwAD.unpack_dual(inp)[1]) + tensor_indices.add(i) + dual_inputs.append(inp) + + for fw_grad, u in zip(fw_grads, all_u): + fw_grad.copy_(u.view_as(fw_grad)) + + for idx, inp in enumerate(inputs): + if idx not in tensor_indices: + continue + dual_inp_obj = dual_inputs[idx] + + # case 1 (Materialized Zero Tensor Tangent) + dual_inputs[idx] = fwAD.make_dual(inp.detach(), torch.zeros_like(inp)) + raw_outputs = _as_tuple(func(*dual_inputs)) + dual_outputs1 = filter(_is_float_or_complex_tensor, raw_outputs) + + # case 2 (Efficient Zero Tensor Tangent since we don't make a dual object and pass a regular tensor) + dual_inputs[idx] = inp.detach() + raw_outputs = _as_tuple(func(*dual_inputs)) + dual_outputs2 = filter(_is_float_or_complex_tensor, raw_outputs) + + # reset + dual_inputs[idx] = dual_inp_obj + + for index_o, (d_o1, d_o2) in enumerate(zip(dual_outputs1, dual_outputs2)): + _val1, res1 = fwAD.unpack_dual(d_o1) + _val2, res2 = fwAD.unpack_dual(d_o2) + + if not (res1 is None or res2 is None): + if not torch.allclose(res1, res2): + raise GradcheckError( + "Mismatch in tangent values for output with index: ", + index_o, + " when input: ", + inp, + " has an undefined tangent value. ", + " Got: ", + res1, + " but expected: ", + res2, + ) + return True + + +def _test_undefined_backward_mode(func, outputs, inputs) -> bool: + diff_input_list: list[torch.Tensor] = list(_iter_tensors(inputs, True)) + if not diff_input_list: + raise GradcheckError("no Tensors requiring grad found in input") + + def warn_bc_breaking(): + warnings.warn( + "Backwards compatibility: New undefined gradient support checking " + "feature is enabled by default, but it may break existing callers " + "of this function. If this is true for you, you can call this " + 'function with "check_undefined_grad=False" to disable the feature', + stacklevel=2, + ) + + def check_undefined_grad_support(output_to_check): + grads_output = [ + torch.zeros_like(o, memory_format=torch.legacy_contiguous_format) + for o in output_to_check + ] + try: + grads_input = torch.autograd.grad( + output_to_check, diff_input_list, grads_output, allow_unused=True + ) + except RuntimeError as e: + warn_bc_breaking() + raise GradcheckError( + "Expected backward function to handle undefined output grads. " + 'Please look at "Notes about undefined output gradients" in ' + '"tools/autograd/derivatives.yaml"' + ) from e + + for gi in grads_input: + if (gi is not None) and (not gi.eq(0).all()): + warn_bc_breaking() + raise GradcheckError( + "Expected all input grads to be undefined or zero when all output grads are undefined " + 'or zero. Please look at "Notes about undefined output gradients" in ' + '"tools/autograd/derivatives.yaml"' + ) + return True + + # All backward functions must work properly if all output grads are undefined + outputs_to_check = [ + [ + torch._C._functions.UndefinedGrad()(o) + for o in _differentiable_outputs(func(*inputs)) + # This check filters out Tensor-likes that aren't instances of Tensor. + if isinstance(o, torch.Tensor) + ] + ] + + # If there are multiple output grads, we should be able to undef one at a time without error + if len(outputs_to_check[0]) > 1: + for undef_grad_idx in range(len(outputs)): + output_to_check = _differentiable_outputs(func(*inputs)) + outputs_to_check.append( + [ + torch._C._functions.UndefinedGrad()(o) + if idx == undef_grad_idx + else o + for idx, o in enumerate(output_to_check) + ] + ) + + return all(check_undefined_grad_support(output) for output in outputs_to_check) + + +def _as_tuple(x): + if isinstance(x, tuple): + return x + elif isinstance(x, list): + return tuple(x) + else: + return (x,) + + +def _differentiable_outputs(x): + return tuple(o for o in _as_tuple(x) if o.requires_grad) + + +def _get_notallclose_msg( + analytical, + numerical, + output_idx, + input_idx, + complex_indices, + test_imag=False, + is_forward_ad=False, +) -> str: + out_is_complex = ( + (not is_forward_ad) and complex_indices and output_idx in complex_indices + ) + inp_is_complex = is_forward_ad and complex_indices and input_idx in complex_indices + part = "imaginary" if test_imag else "real" + element = "inputs" if is_forward_ad else "outputs" + prefix = ( + "" + if not (out_is_complex or inp_is_complex) + else f"While considering the {part} part of complex {element} only, " + ) + mode = "computed with forward mode " if is_forward_ad else "" + return ( + prefix + + f"Jacobian {mode}mismatch for output {output_idx:d} with respect to input {input_idx:d},\n" + f"numerical:{numerical}\nanalytical:{analytical}\n" + ) + + +def _transpose(matrix_of_tensors): + # returns list of tuples + return list(zip(*matrix_of_tensors)) + + +def _real_and_imag_output(fn): + # returns new functions real(fn), and imag(fn) where real(fn) and imag(fn) behave the same as + # the original fn, except torch.real or torch.imag are applied to the complex outputs + def apply_to_c_outs(fn, fn_to_apply): + def wrapped_fn(*inputs): + outs = _as_tuple(fn(*inputs)) + return tuple(fn_to_apply(o) if o.is_complex() else o for o in outs) + + return wrapped_fn + + return apply_to_c_outs(fn, torch.real), apply_to_c_outs(fn, torch.imag) + + +def _real_and_imag_input(fn, complex_inp_indices, tupled_inputs): + # returns new functions that take real inputs instead of complex inputs as + # (x, y) -> fn(x + y * 1j). And it computes: inp -> fn(inp + y * 1j) and inp -> fn(x + inp * 1j). + # In each case, the other part is considered constant. + # We do not use 0 for the constant here to make sure we always call the user function with a valid input. + def apply_to_c_inps(fn, fn_to_apply): + def wrapped_fn(*inputs): + new_inputs = list(inputs) + for should_be_complex in complex_inp_indices: + new_inputs[should_be_complex] = fn_to_apply( + new_inputs[should_be_complex], tupled_inputs[should_be_complex] + ) + return _as_tuple(fn(*new_inputs)) + + return wrapped_fn + + real_fn = apply_to_c_inps(fn, lambda inp, orig: inp + orig.imag * 1j) + imag_fn = apply_to_c_inps(fn, lambda inp, orig: orig.real + inp * 1j) + return real_fn, imag_fn + + +def _gradcheck_real_imag( + gradcheck_fn, + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + check_forward_ad, + check_backward_ad, + nondet_tol, + check_undefined_grad, +): + complex_out_indices = [i for i, o in enumerate(outputs) if o.is_complex()] + has_any_complex_output = any(o.is_complex() for o in _as_tuple(func_out)) + if check_backward_ad: + if has_any_complex_output: + real_fn, imag_fn = _real_and_imag_output(func) + + imag_func_out = imag_fn(*tupled_inputs) + imag_outputs = _differentiable_outputs(imag_func_out) + gradcheck_fn( + imag_fn, + imag_func_out, + tupled_inputs, + imag_outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + complex_indices=complex_out_indices, + test_imag=True, + ) + + real_func_out = real_fn(*tupled_inputs) + real_outputs = _differentiable_outputs(real_func_out) + gradcheck_fn( + real_fn, + real_func_out, + tupled_inputs, + real_outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + complex_indices=complex_out_indices, + ) + else: + gradcheck_fn( + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + ) + + if check_forward_ad: + complex_inp_indices = [ + i + for i, inp in enumerate(tupled_inputs) + if is_tensor_like(inp) and inp.is_complex() + ] + if complex_inp_indices: + real_fn, imag_fn = _real_and_imag_input( + func, complex_inp_indices, tupled_inputs + ) + + imag_inputs = [ + inp.imag if is_tensor_like(inp) and inp.is_complex() else inp + for inp in tupled_inputs + ] + imag_func_out = imag_fn(*imag_inputs) + diff_imag_func_out = _differentiable_outputs(imag_func_out) + gradcheck_fn( + imag_fn, + imag_func_out, + imag_inputs, + diff_imag_func_out, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + complex_indices=complex_inp_indices, + test_imag=True, + use_forward_ad=True, + ) + + real_inputs = [ + inp.real if is_tensor_like(inp) and inp.is_complex() else inp + for inp in tupled_inputs + ] + real_func_out = real_fn(*real_inputs) + diff_real_func_out = _differentiable_outputs(real_func_out) + gradcheck_fn( + real_fn, + real_func_out, + real_inputs, + diff_real_func_out, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + complex_indices=complex_inp_indices, + use_forward_ad=True, + ) + if check_undefined_grad: + _test_undefined_forward_mode(imag_fn, imag_func_out, imag_inputs) + _test_undefined_forward_mode(real_fn, real_func_out, real_inputs) + else: + gradcheck_fn( + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + use_forward_ad=True, + ) + if check_undefined_grad: + _test_undefined_forward_mode(func, outputs, tupled_inputs) + + +def _slow_gradcheck( + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + *, + use_forward_ad=False, + complex_indices=None, + test_imag=False, + masked=False, +): + func_out = _as_tuple(func_out) + if not outputs: + return _check_no_differentiable_outputs( + func, tupled_inputs, func_out, eps=eps, is_forward_ad=use_forward_ad + ) + tupled_inputs_numerical = tupled_inputs if masked else _densify(tupled_inputs) + + numerical = _transpose( + _get_numerical_jacobian( + func, + tupled_inputs_numerical, + func_out, + eps=eps, + is_forward_ad=use_forward_ad, + ) + ) + # Note: [numerical vs analytical output length] + # The numerical path returns jacobian quantity for all outputs, even if requires_grad of that + # output is False. This behavior is necessary for _check_no_differentiable_outputs to work. + numerical = [nj for o, nj in zip(func_out, numerical) if o.requires_grad] + if use_forward_ad: + analytical_forward = _get_analytical_jacobian_forward_ad( + func, tupled_inputs, func_out, check_grad_dtypes=check_grad_dtypes + ) + + for i, n_per_out in enumerate(numerical): + for j, n in enumerate(n_per_out): + a = analytical_forward[j][i] + if not _allclose_with_type_promotion(a, n.to(a.device), rtol, atol): + raise GradcheckError( + _get_notallclose_msg( + a, n, i, j, complex_indices, test_imag, is_forward_ad=True + ) + ) + else: + for i, o in enumerate(outputs): + analytical = _check_analytical_jacobian_attributes( + tupled_inputs, o, nondet_tol, check_grad_dtypes + ) + + for j, (a, n) in enumerate(zip(analytical, numerical[i])): + if not _allclose_with_type_promotion(a, n.to(a.device), rtol, atol): + raise GradcheckError( + _get_notallclose_msg(a, n, i, j, complex_indices, test_imag) + ) + + return True + + +def _dot_with_type_promotion(u, v): + if u.dim() != 1 or v.dim() != 1: + raise AssertionError( + f"Expected u and v to be 1D tensors, but got dims {u.dim()} and {v.dim()}" + ) + return (u * v).sum() + + +def _allclose_with_type_promotion(a, b, rtol, atol): + promoted_type = torch.promote_types(a.dtype, b.dtype) + a = a.to(dtype=promoted_type) + b = b.to(dtype=promoted_type) + return torch.allclose(a, b, rtol, atol) + + +def _to_real_dtype(dtype): + if dtype == torch.complex128: + return torch.float64 + elif dtype == torch.complex64: + return torch.float32 + else: + return dtype + + +def _vec_from_tensor(x, generator, downcast_complex=False): + # Create a random vector with the same number of elements as x and the same + # dtype/device. If x is complex and downcast_complex is False, we create a + # complex tensor with only real component. + if x.layout == torch.sparse_coo: + # For sparse, create a random sparse vec with random values in the same + # indices. Make sure size is set so that it isn't inferred to be smaller. + x_values = x._values() + dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype + values = ( + torch.rand(x_values.numel(), generator=generator) + .to(dtype=dtype, device=x.device) + .view(x_values.shape) + ) + values /= values.norm() + vec = torch.sparse_coo_tensor(x._indices(), values, x.size(), device=x.device) + elif _is_sparse_compressed_tensor(x): + if x.layout in {torch.sparse_csr, torch.sparse_bsr}: + compressed_indices, plain_indices = x.crow_indices(), x.col_indices() + else: + compressed_indices, plain_indices = x.ccol_indices(), x.row_indices() + x_values = x.values() + dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype + values = ( + torch.rand(x_values.numel(), generator=generator) + .to(dtype=dtype, device=x.device) + .view(x_values.shape) + ) + values /= values.norm() + vec = torch.sparse_compressed_tensor( + compressed_indices, + plain_indices, + values, + x.size(), + layout=x.layout, + device=x.device, + ) + else: + dtype = _to_real_dtype(x.dtype) if downcast_complex else x.dtype + vec = torch.rand(x.numel(), generator=generator).to( + dtype=dtype, device=x.device + ) + vec /= vec.norm() + return vec + + +def _get_inp_tensors(tupled_inputs): + inp_idx_tup = [ + (i, t) + for i, t in enumerate(tupled_inputs) + if is_tensor_like(t) and t.requires_grad + ] + return [tup[0] for tup in inp_idx_tup], [tup[1] for tup in inp_idx_tup] + + +def _adjusted_atol(atol, u, v): + # In slow gradcheck, we compare A and B element-wise, i.e., for some a, b we + # allow: |a - b| < atol + rtol * b. But since we now compare q1 = v^T A u and + # q2 = v^T B u, we must allow |q1 - q2| < v^T E u + rtol * v^T B u, where E is + # the correctly sized matrix in which each entry is atol. + # + # We see that atol needs to be scaled by v^T M u (where M is an all-ones M x N + # matrix): v^T M u = \sum_{i} \sum_{j} u_i * v_j = (\sum_{i} u_i)(\sum_{i} v_i) + # TODO: properly handle case when u is tuple instead of only taking first element + u = u[0] if isinstance(u, tuple) else u + sum_u = u.sum() + sum_v = 1.0 if v is None else v.sum() + return atol * float(sum_u) * float(sum_v) + + +FAST_FAIL_SLOW_OK_MSG = """ +Fast gradcheck failed but element-wise differences are small. This means that the +test might've passed in slow_mode! + +If you are adding a new operator, please file an issue and then use one of the +workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck: + +If the test +- manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck + with `fast_mode=False` as a keyword argument. +- is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test + to have `gradcheck_fast_mode=False` +- is a Module test (e.g., in common_nn.py), then modify the corresponding + module_test entry to have `gradcheck_fast_mode=False` +""".strip() + + +def _run_slow_mode_and_get_error( + func, tupled_inputs, outputs, input_idx, output_idx, rtol, atol, eps, is_forward_ad +): + # Compute jacobians in slow mode for better error message + slow_numerical = _get_numerical_jacobian( + func, tupled_inputs, outputs, eps=eps, is_forward_ad=is_forward_ad + )[input_idx][output_idx] + if is_forward_ad: + + def new_fn(inp): + new_inputs = list(tupled_inputs) + new_inputs[input_idx] = inp + return _as_tuple(func(*new_inputs))[output_idx] + + slow_analytical = _get_analytical_jacobian_forward_ad( + new_fn, (tupled_inputs[input_idx],), (outputs[output_idx],) + )[0][0] + else: + slow_analytical = _get_analytical_jacobian( + tupled_inputs, outputs, input_idx, output_idx + ) + + # Assume jacobians are non-empty and have the same shape + slow_max_diff = (slow_numerical - slow_analytical).abs().max() + + slow_allclose = torch.allclose(slow_analytical, slow_numerical, rtol, atol) + msg = ( + "\nThe above quantities relating the numerical and analytical jacobians are computed \n" + "in fast mode. See: https://github.com/pytorch/pytorch/issues/53876 for more background \n" + "about fast mode. Below, we recompute numerical and analytical jacobians in slow mode:\n\n" + f"Numerical:\n {slow_numerical}\n" + f"Analytical:\n{slow_analytical}\n\n" + f"The max per-element difference (slow mode) is: {slow_max_diff}.\n" + ) + if slow_allclose: + # Slow gradcheck would've passed! + msg += FAST_FAIL_SLOW_OK_MSG + return msg + + +def _to_flat_dense_if_sparse(tensor): + if _is_sparse_any_tensor(tensor): + return tensor.to_dense().reshape(-1) + else: + return tensor + + +def _make_vectors(inp_tensors, outputs, *, use_forward_ad): + # Use our own generator to avoid messing with the user's RNG state + g_cpu = torch.Generator() + + def _vec_from_tensor_cpu(*args): + # Default allocate all tensors on CPU, so they are on the same device as the generator + # even if the user specified a default device + with torch.device("cpu"): + return _vec_from_tensor(*args) + + all_u = [] + all_u_dense = [] + for inp in inp_tensors: + ur = _vec_from_tensor_cpu(inp, g_cpu, True) + ur_dense = _to_flat_dense_if_sparse(ur) + if inp.is_complex(): + ui = _vec_from_tensor_cpu(inp, g_cpu, True) + all_u.append((ur, ui)) + ui_dense = _to_flat_dense_if_sparse(ui) + all_u_dense.append((ur_dense, ui_dense)) + else: + all_u.append(ur) + all_u_dense.append(ur_dense) + all_v = ( + None + if use_forward_ad + else [_vec_from_tensor_cpu(out, g_cpu) for out in outputs] + ) + return all_v, all_u, all_u_dense + + +def _check_analytical_numerical_equal( + all_analytical, + all_numerical, + complex_indices, + tupled_inputs, + outputs, + func, + all_v, + all_u, + rtol, + atol, + eps, + test_imag, + *, + is_forward_ad=False, +): + for i, all_numerical_for_input_i in enumerate(all_numerical): + for j, n in enumerate(all_numerical_for_input_i): + # Forward AD generates the transpose of what this function expects + if is_forward_ad: + a = all_analytical[i][j] + else: + a = all_analytical[j][i] + n = n.to(device=a.device) + updated_atol = _adjusted_atol(atol, all_u[i], all_v[j] if all_v else None) + if not _allclose_with_type_promotion(a, n.to(a.device), rtol, updated_atol): + jacobians_str = _run_slow_mode_and_get_error( + func, tupled_inputs, outputs, i, j, rtol, atol, eps, is_forward_ad + ) + raise GradcheckError( + _get_notallclose_msg( + a, n, j, i, complex_indices, test_imag, is_forward_ad + ) + + jacobians_str + ) + + +def _fast_gradcheck( + func, + func_out, + inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + nondet_tol, + *, + use_forward_ad=False, + complex_indices=None, + test_imag=False, + masked=False, +): + # See https://github.com/pytorch/pytorch/issues/53876 for details + inp_tensors_idx, inp_tensors = _get_inp_tensors(inputs) + # Backward mode computes v^T * J (VJP) + # Since we computed J * u (JVP) through finite difference method, we perform an equality check + # between VJP * u, v * JVP + # ---- + # Forward mode computes J * u (JVP) + # Since we already compute JVP through finite difference method, + # we don't need v for correctness check here as asserted below + all_v, all_u, all_u_dense = _make_vectors( + inp_tensors, outputs, use_forward_ad=use_forward_ad + ) + + inputs_numerical, all_u_numerical, all_v_numerical = ( + (inputs, all_u, all_v) if masked else _densify((inputs, all_u, all_v)) + ) + + numerical_vJu = _get_numerical_vJu( + func, + inputs_numerical, + inp_tensors_idx, + func_out, + all_u_numerical, + all_v_numerical, + eps, + is_forward_ad=use_forward_ad, + ) + # TODO: replicate https://github.com/pytorch/pytorch/pull/77743 for fast gradcheck as well + if use_forward_ad: + if all_v is not None: + raise AssertionError("Expected all_v to be None.") + analytical_vJu = _get_analytical_jacobian_forward_ad( + func, + inputs, + _as_tuple(func_out), + all_u=all_u, + check_grad_dtypes=check_grad_dtypes, + ) + else: + if not outputs: + _check_no_differentiable_outputs_fast( + func, func_out, inputs, inp_tensors_idx, all_u, eps, nondet_tol + ) + + analytical_vJu = _get_analytical_vJu_backward_mode( + inputs, outputs, nondet_tol, check_grad_dtypes, all_v, all_u_dense + ) + + _check_analytical_numerical_equal( + analytical_vJu, + numerical_vJu, + complex_indices, + inputs, + outputs, + func, + all_v, + all_u, + rtol, + atol, + eps, + test_imag, + is_forward_ad=use_forward_ad, + ) + + return True + + +# Note [VarArg of Tensors] +# ~~~~~~~~~~~~~~~~~~~~~~~~ +# 'func' accepts a vararg of tensors, which isn't expressible in the type system at the moment. +# If https://mypy.readthedocs.io/en/latest/additional_features.html?highlight=callable#extended-callable-types is accepted, +# the '...' first argument of Callable can be replaced with VarArg(Tensor). +# For now, we permit any input. +def gradcheck( + func: Callable[..., Union[_TensorOrTensors]], # See Note [VarArg of Tensors] + inputs: _TensorOrTensors, + *, + eps: float = 1e-6, + atol: float = 1e-5, + rtol: float = 1e-3, + raise_exception: bool = True, + nondet_tol: float = 0.0, + check_undefined_grad: bool = True, + check_grad_dtypes: bool = False, + check_batched_grad: bool = False, + check_batched_forward_grad: bool = False, + check_forward_ad: bool = False, + check_backward_ad: bool = True, + fast_mode: bool = False, + masked: Optional[bool] = None, +) -> bool: # noqa: D400,D205 + r"""Check gradients computed via small finite differences against analytical + gradients wrt tensors in :attr:`inputs` that are of floating point or complex type + and with ``requires_grad=True``. + + The check between numerical and analytical gradients uses :func:`~torch.allclose`. + + For most of the complex functions we consider for optimization purposes, no notion of + Jacobian exists. Instead, gradcheck verifies if the numerical and analytical values of + the Wirtinger and Conjugate Wirtinger derivatives are consistent. Because the gradient + computation is done under the assumption that the overall function has a real-valued + output, we treat functions with complex output in a special way. For these functions, + gradcheck is applied to two real-valued functions corresponding to taking the real + components of the complex outputs for the first, and taking the imaginary components + of the complex outputs for the second. For more details, check out + :ref:`complex_autograd-doc`. + + .. note:: + The default values are designed for :attr:`input` of double precision. + This check will likely fail if :attr:`input` is of less precision, e.g., + ``FloatTensor``. + + .. note:: + Gradcheck may fail when evaluated on non-differentiable points + because the numerically computed gradients via finite differencing may differ + those computed analytically (not necessarily because either is incorrect). + For more context, see :ref:`non-differentiable-func-grad`. + + .. warning:: + If any checked tensor in :attr:`input` has overlapping memory, i.e., + different indices pointing to the same memory address (e.g., from + :func:`torch.Tensor.expand`), this check will likely fail because the numerical + gradients computed by point perturbation at such indices will change + values at all other indices that share the same memory address. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor or a tuple of Tensors + inputs (tuple of Tensor or Tensor): inputs to the function + eps (float, optional): perturbation for finite differences + atol (float, optional): absolute tolerance + rtol (float, optional): relative tolerance + raise_exception (bool, optional): indicating whether to raise an exception if + the check fails. The exception gives more information about the + exact nature of the failure. This is helpful when debugging gradchecks. + nondet_tol (float, optional): tolerance for non-determinism. When running + identical inputs through the differentiation, the results must either match + exactly (default, 0.0) or be within this tolerance. + check_undefined_grad (bool, optional): if ``True``, check if undefined output grads + are supported and treated as zeros, for ``Tensor`` outputs. + check_batched_grad (bool, optional): if ``True``, check if we can compute + batched gradients using prototype vmap support. Defaults to False. + check_batched_forward_grad (bool, optional): if ``True``, checks if we can compute + batched forward gradients using forward ad and prototype vmap support. Defaults to ``False``. + check_forward_ad (bool, optional): if ``True``, check that the gradients computed with forward + mode AD match the numerical ones. Defaults to ``False``. + check_backward_ad (bool, optional): if ``False``, do not perform any checks that rely on + backward mode AD to be implemented. Defaults to ``True``. + fast_mode (bool, optional): Fast mode for gradcheck and gradgradcheck is currently only + implemented for R to R functions. If none of the inputs and outputs are complex + a faster implementation of gradcheck that no longer computes the entire jacobian + is run; otherwise, we fall back to the slow implementation. + masked (bool, optional): if ``True``, the gradients of unspecified elements of + sparse tensors are ignored. Defaults to ``False``. + Returns: + ``True`` if all differences satisfy allclose condition + + """ + if not (check_forward_ad or check_backward_ad): + raise AssertionError( + "Expected at least one of check_forward_ad or check_backward_ad to be True" + ) + if check_batched_grad and not check_backward_ad: + raise AssertionError( + "Setting check_batched_grad=True requires check_backward_ad to be True" + ) + if check_batched_forward_grad and not check_forward_ad: + raise AssertionError( + "Setting check_batched_forward_grad=True requires check_forward_ad to be True" + ) + args = locals().copy() + args.pop("raise_exception") + if not raise_exception: + try: + return _gradcheck_helper(**args) + except GradcheckError: + return False + else: + return _gradcheck_helper(**args) + + +def _gradcheck_helper( + func, + inputs, + eps, + atol, + rtol, + nondet_tol, + check_undefined_grad, + check_grad_dtypes, + check_batched_grad, + check_batched_forward_grad, + check_forward_ad, + check_backward_ad, + fast_mode, + masked, +): + tupled_inputs = _as_tuple(inputs) + _check_inputs(tupled_inputs) + + func_out = func(*tupled_inputs) + outputs = _differentiable_outputs(func_out) + _check_outputs(outputs) + + gradcheck_fn = functools.partial( + _fast_gradcheck if fast_mode else _slow_gradcheck, masked=masked + ) + _gradcheck_real_imag( + gradcheck_fn, + func, + func_out, + tupled_inputs, + outputs, + eps, + rtol, + atol, + check_grad_dtypes, + check_forward_ad=check_forward_ad, + check_backward_ad=check_backward_ad, + nondet_tol=nondet_tol, + check_undefined_grad=check_undefined_grad, + ) + + if check_batched_forward_grad: + _test_batched_grad_forward_ad(func, tupled_inputs) + + # Short circuit because remaining tests rely on backward AD to be implemented + if not check_backward_ad: + return True + + for i, o in enumerate(outputs): + if check_batched_grad: + _test_batched_grad(tupled_inputs, o, i) + + _test_backward_mul_by_grad_output(outputs, tupled_inputs, masked) + + if check_undefined_grad and check_backward_ad: + _test_undefined_backward_mode(func, outputs, tupled_inputs) + return True + + +def gradgradcheck( + func: Callable[..., _TensorOrTensors], # See Note [VarArg of Tensors] + inputs: _TensorOrTensors, + grad_outputs: Optional[_TensorOrTensors] = None, + *, + eps: float = 1e-6, + atol: float = 1e-5, + rtol: float = 1e-3, + gen_non_contig_grad_outputs: bool = False, + raise_exception: bool = True, + nondet_tol: float = 0.0, + check_undefined_grad: bool = True, + check_grad_dtypes: bool = False, + check_batched_grad: bool = False, + check_fwd_over_rev: bool = False, + check_rev_over_rev: bool = True, + fast_mode: bool = False, + masked: bool = False, +) -> bool: # noqa: D400,D205 + r"""Check gradients of gradients computed via small finite differences + against analytical gradients wrt tensors in :attr:`inputs` and + :attr:`grad_outputs` that are of floating point or complex type and with + ``requires_grad=True``. + + This function checks that backpropagating through the gradients computed + to the given :attr:`grad_outputs` are correct. + + The check between numerical and analytical gradients uses :func:`~torch.allclose`. + + .. note:: + The default values are designed for :attr:`input` and + :attr:`grad_outputs` of double precision. This check will likely fail if + they are of less precision, e.g., ``FloatTensor``. + + .. warning:: + If any checked tensor in :attr:`input` and :attr:`grad_outputs` has + overlapping memory, i.e., different indices pointing to the same memory + address (e.g., from :func:`torch.Tensor.expand`), this check will likely fail + because the numerical gradients computed by point perturbation at such + indices will change values at all other indices that share the same + memory address. + + Args: + func (function): a Python function that takes Tensor inputs and returns + a Tensor or a tuple of Tensors + inputs (tuple of Tensor or Tensor): inputs to the function + grad_outputs (tuple of Tensor or Tensor, optional): The gradients with + respect to the function's outputs. + eps (float, optional): perturbation for finite differences + atol (float, optional): absolute tolerance + rtol (float, optional): relative tolerance + gen_non_contig_grad_outputs (bool, optional): if :attr:`grad_outputs` is + ``None`` and :attr:`gen_non_contig_grad_outputs` is ``True``, the + randomly generated gradient outputs are made to be noncontiguous + raise_exception (bool, optional): indicating whether to raise an exception if + the check fails. The exception gives more information about the + exact nature of the failure. This is helpful when debugging gradchecks. + nondet_tol (float, optional): tolerance for non-determinism. When running + identical inputs through the differentiation, the results must either match + exactly (default, 0.0) or be within this tolerance. Note that a small amount + of nondeterminism in the gradient will lead to larger inaccuracies in + the second derivative. + check_undefined_grad (bool, optional): if True, check if undefined output grads + are supported and treated as zeros + check_batched_grad (bool, optional): if True, check if we can compute + batched gradients using prototype vmap support. Defaults to False. + fast_mode (bool, optional): if True, run a faster implementation of gradgradcheck that + no longer computes the entire jacobian. + masked (bool, optional): if True, the gradients of unspecified elements of + sparse tensors are ignored (default, False). + Returns: + True if all differences satisfy allclose condition + """ + if not (check_fwd_over_rev or check_rev_over_rev): + raise AssertionError( + "Expected at least one of check_fwd_over_rev or check_rev_over_rev to be True" + ) + if check_undefined_grad and not check_rev_over_rev: + raise AssertionError( + "Setting check_undefined_grad=True requires check_rev_over_rev to be True" + ) + if check_batched_grad and not check_rev_over_rev: + raise AssertionError( + "Setting check_batched_grad=True requires check_rev_over_rev to be True" + ) + # TODO: do we want to test this too? + # assert not (check_batched_forward_grad and not check_fwd_over_rev), ( + # "Setting check_batched_forward_grad=True requires check_fwd_over_rev to be True") + tupled_inputs = _as_tuple(inputs) + + if grad_outputs is None: + # If grad_outputs is not specified, create random Tensors of the same shape, type, and device as the outputs + + outputs = _differentiable_outputs(func(*tupled_inputs)) + tupled_grad_outputs = tuple( + torch.testing.make_tensor( + x.shape, + dtype=x.dtype + if x.is_floating_point() or x.is_complex() + else torch.double, + device=x.device, + low=-1, + high=1, + requires_grad=True, + noncontiguous=gen_non_contig_grad_outputs, + ) + for x in outputs + ) + else: + tupled_grad_outputs = _as_tuple(grad_outputs) + + num_outputs = len(tupled_grad_outputs) + + # NB: We need to save the requires_grad information about the inputs here because gradcheck detaches inputs + # before running forward mode AD + diff_input_args_indices = { + i for i, x in enumerate(tupled_inputs) if is_tensor_like(x) and x.requires_grad + } + diff_grad_output_indices = { + i for i, x in enumerate(tupled_grad_outputs) if x.requires_grad + } + + def new_func(*args): + # Restore the requires_grad information + input_args = tuple( + x.requires_grad_() if i in diff_input_args_indices else x + for i, x in enumerate(args[:-num_outputs]) + ) + outputs = _differentiable_outputs(func(*input_args)) + grad_outputs = tuple( + x.requires_grad_() if i in diff_grad_output_indices else x + for i, x in enumerate(args[-num_outputs:]) + ) + diff_input_args = tuple( + x for i, x in enumerate(input_args) if i in diff_input_args_indices + ) + grad_inputs = torch.autograd.grad( + outputs, diff_input_args, grad_outputs, create_graph=True, allow_unused=True + ) + grad_inputs = tuple(g for g in grad_inputs if g is not None) + return grad_inputs + + return gradcheck( + new_func, + tupled_inputs + tupled_grad_outputs, + eps=eps, + atol=atol, + rtol=rtol, + raise_exception=raise_exception, + nondet_tol=nondet_tol, + check_undefined_grad=check_undefined_grad, + check_grad_dtypes=check_grad_dtypes, + check_batched_grad=check_batched_grad, + fast_mode=fast_mode, + check_forward_ad=check_fwd_over_rev, + check_backward_ad=check_rev_over_rev, + masked=masked, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/graph.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/graph.py new file mode 100644 index 0000000000000000000000000000000000000000..f7c7150aa7e9d5bd88ea7dfc7cc86a565c9f9cb5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/graph.py @@ -0,0 +1,870 @@ +import abc +import contextlib +import functools +import logging +import threading +from collections import defaultdict, deque +from collections.abc import ( + Callable, + Generator, + Iterable, + Iterator, + MutableMapping, + Sequence, +) +from typing import ( + Any, + cast, + Literal, + NamedTuple, + Optional, + TYPE_CHECKING, + TypeAlias, + Union, +) +from weakref import WeakKeyDictionary, WeakValueDictionary + +import torch +from torch.autograd.variable import Variable +from torch.utils._python_dispatch import TorchDispatchMode +from torch.utils.hooks import RemovableHandle + + +if TYPE_CHECKING: + from torch._ops import OpOverload + + +__all__ = [ + "saved_tensors_hooks", + "save_on_cpu", + "disable_saved_tensors_hooks", + "register_multi_grad_hook", + "allow_mutation_on_saved_tensors", + "Node", + "GradientEdge", + "get_gradient_edge", + "increment_version", + "set_warn_on_accumulate_grad_stream_mismatch", +] + + +log = logging.getLogger(__name__) + + +class Node(abc.ABC): + @abc.abstractmethod + def name(self) -> str: + r"""Return the name. + + Example:: + + >>> import torch + >>> a = torch.tensor([0., 0., 0.], requires_grad=True) + >>> b = a.clone() + >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) + >>> print(b.grad_fn.name()) + CloneBackward0 + """ + raise NotImplementedError + + @property + @abc.abstractmethod + def next_functions(self) -> tuple[tuple[Optional["Node"], int], ...]: + raise NotImplementedError + + @abc.abstractmethod + def metadata(self) -> dict: + r"""Return the metadata.""" + raise NotImplementedError + + @property + @abc.abstractmethod + def _input_metadata(self) -> list[Any]: + raise NotImplementedError + + @abc.abstractmethod + def _register_hook_dict(self, tensor: torch.Tensor) -> None: + raise NotImplementedError + + @abc.abstractmethod + def register_hook(self, fn: Callable[..., Any]) -> RemovableHandle: + r"""Register a backward hook. + + The hook will be called every time a gradient with respect to the + Node is computed. The hook should have the following signature:: + + hook(grad_inputs: Tuple[Tensor], grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None + + + The hook should not modify its argument, but it can optionally return + a new gradient which will be used in place of :attr:`grad_inputs`. + + This function returns a handle with a method ``handle.remove()`` + that removes the hook from the module. + + .. note:: + See :ref:`backward-hooks-execution` for more information on how when this hook + is executed, and how its execution is ordered relative to other hooks. + + .. note:: + In the rare case where the hook is registered while the Node has already + begun execution, there is no longer any guarantee on :attr:`grad_outputs` + content (it might be as usual or empty depending on other factors). The + hook can still optionally return a new gradient to be used in place of + :attr:`grad_inputs` independent of :attr:`grad_outputs`. + + Example:: + + >>> import torch + >>> a = torch.tensor([0., 0., 0.], requires_grad=True) + >>> b = a.clone() + >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) + >>> handle = b.grad_fn.register_hook(lambda gI, gO: (gO[0] * 2,)) + >>> b.sum().backward(retain_graph=True) + >>> print(a.grad) + tensor([2., 2., 2.]) + >>> handle.remove() # Removes the hook + >>> a.grad = None + >>> b.sum().backward(retain_graph=True) + >>> print(a.grad) + tensor([1., 1., 1.]) + """ + raise NotImplementedError + + @abc.abstractmethod + def register_prehook(self, fn: Callable[..., Any]) -> RemovableHandle: + r"""Register a backward pre-hook. + + The hook will be called every time a gradient with respect to the + Node is computed. The hook should have the following signature:: + + hook(grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None + + The hook should not modify its argument, but it can optionally return + a new gradient which will be used in place of :attr:`grad_outputs`. + + This function returns a handle with a method ``handle.remove()`` + that removes the hook from the module. + + .. note:: + See :ref:`backward-hooks-execution` for more information on how when this hook + is executed, and how its execution is ordered relative to other hooks. + + Example:: + + >>> a = torch.tensor([0., 0., 0.], requires_grad=True) + >>> b = a.clone() + >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) + >>> handle = b.grad_fn.register_prehook(lambda gI: (gI[0] * 2,)) + >>> b.sum().backward(retain_graph=True) + >>> print(a.grad) + tensor([2., 2., 2.]) + >>> handle.remove() + >>> a.grad = None + >>> b.sum().backward(retain_graph=True) + >>> print(a.grad) + tensor([1., 1., 1.]) + """ + raise NotImplementedError + + @classmethod + def __subclasshook__(cls, subclass: type) -> bool: + if cls is Node and ( + ( + subclass is not None + and subclass is getattr(torch._C._functions, subclass.__name__, None) + ) + or issubclass(subclass, torch.autograd.function.BackwardCFunction) + ): + return True + return NotImplemented + + +def _get_grad_fn_or_grad_acc(t: Union[torch.Tensor, "GradientEdge"]) -> Node: + if isinstance(t, GradientEdge): + return t.node + if t.requires_grad and t.grad_fn is None: + with torch.enable_grad(): + node = t.view_as(t).grad_fn.next_functions[0][0] # type: ignore[union-attr] + else: + node = t.grad_fn + if node is None: + raise AssertionError("Expected gradient function to be set") + return node + + +class GradientEdge(NamedTuple): + """Object representing a given gradient edge within the autograd graph. + + To get the gradient edge where a given Tensor gradient will be computed, + you can do ``edge = autograd.graph.get_gradient_edge(tensor)``. + """ + + node: Node + output_nr: int + # This token can be used to ensure the graph stays alive when it cannot be + # done via the node field + ownership_token: Optional[Node] = None + + +def get_gradient_edge(tensor: torch.Tensor) -> GradientEdge: + """Get the gradient edge for computing the gradient of the given Tensor. + + In particular, it is equivalent to call + ``g = autograd.grad(loss, input)`` and ``g = autograd.grad(loss, get_gradient_edge(input))``. + """ + if not tensor.requires_grad: + raise RuntimeError( + "It is not possible to get the gradient edge for a Tensor " + "that does not require gradients", + ) + grad_fn = _get_grad_fn_or_grad_acc(tensor) + + # Python-based Node are owned by the C++ side meaning the python grad_fn + # object we hold here does NOT keep the C++ graph alive. + # Create an ownership token by creating a new C++ node that own the graph + # we care about here. + token = None + if isinstance(grad_fn, torch._C._FunctionBase): + with torch.enable_grad(): + token = tensor.view_as(tensor).grad_fn + + # Note that output_nr default to 0 which is the right value + # for the AccumulateGrad node. + # pyrefly: ignore [bad-argument-type] + return GradientEdge(grad_fn, tensor.output_nr, ownership_token=token) + + +def increment_version(tensor: Union[torch.Tensor, Iterable[torch.Tensor]]) -> None: + """Update autograd metadata tracking whether the given Tensor was modified in place. + + This is to enable more accurate error checking within the autograd engine. + It is already done automatically by PyTorch functions and within custom Function + when mark_dirty() is called appropriately so you only need to call this explicitly + if you are doing inplace operation on the Tensor data in a way that Pytorch doesn't + know about. For example a custom kernel that reads the Tensor data_ptr and modifies + the memory inplace based on this pointer. Can accept either a tensor, or a list of tensors. + + Note that incrementing the version counter multiple times for a single inplace operation + is not problematic. + + Note that if you pass in tensor constructed under torch.inference_mode(), + we will not bump its version counter (because your tensor does not have one). + """ + if isinstance(tensor, torch.Tensor): + tensor = (tensor,) + torch._C._increment_version(tensor) + + +class saved_tensors_hooks: + """Context-manager that sets a pair of pack / unpack hooks for saved tensors. + + Use this context-manager to define how intermediary results of an operation + should be packed before saving, and unpacked on retrieval. + + In that context, the ``pack_hook`` function will be called every time an + operation saves a tensor for backward (this includes intermediary results + saved using + :func:`~torch.autograd.function._ContextMethodMixin.save_for_backward` but + also those recorded by a PyTorch-defined operation). The output of + ``pack_hook`` is then stored in the computation graph instead of the + original tensor. + + The ``unpack_hook`` is called when the saved tensor needs to be accessed, + namely when executing :func:`torch.Tensor.backward()` or + :func:`torch.autograd.grad()`. It takes as argument the *packed* object + returned by ``pack_hook`` and should return a tensor which has the same + content as the original tensor (passed as input to the corresponding + ``pack_hook``). + + The hooks should have the following signatures: + + pack_hook(tensor: Tensor) -> Any + + unpack_hook(Any) -> Tensor + + where the return value of ``pack_hook`` is a valid input to ``unpack_hook``. + + In general, you want ``unpack_hook(pack_hook(t))`` to be equal to ``t`` in terms + of value, size, dtype and device. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> def pack_hook(x): + ... print("Packing", x) + ... return x.detach() + >>> + >>> def unpack_hook(x): + ... print("Unpacking", x) + ... return x + >>> + >>> a = torch.ones(5, requires_grad=True) + >>> b = torch.ones(5, requires_grad=True) * 2 + >>> with torch.autograd.graph.saved_tensors_hooks(pack_hook, unpack_hook): + ... y = a * b + Packing tensor([1., 1., 1., 1., 1.], requires_grad=True) + Packing tensor([2., 2., 2., 2., 2.], grad_fn=) + >>> y.sum().backward() + Unpacking tensor([1., 1., 1., 1., 1.], requires_grad=True) + Unpacking tensor([2., 2., 2., 2., 2.], grad_fn=) + + .. warning :: + Performing an inplace operation on the input to either hooks may lead + to undefined behavior. + + .. warning :: + Only one pair of hooks is allowed at a time. When recursively nesting this + context-manager, only the inner-most pair of hooks will be applied. + + .. warning :: + To avoid reference cycle, the return value of ``pack_hook`` cannot hold a + reference to the input tensor. For example, use `lambda x: x.detach()` + instead of `lambda x: x` as the pack hook. + """ + + def __init__( + self, + pack_hook: Callable[[torch.Tensor], Any], + unpack_hook: Callable[[Any], torch.Tensor], + ) -> None: + self.pack_hook = pack_hook + self.unpack_hook = unpack_hook + + def __enter__(self) -> None: + torch._C._autograd._push_saved_tensors_default_hooks( + self.pack_hook, self.unpack_hook + ) + + def __exit__(self, *args: object) -> None: + torch._C._autograd._pop_saved_tensors_default_hooks() + + +class save_on_cpu(saved_tensors_hooks): + """Context manager under which tensors saved by the forward pass will be stored on cpu, then retrieved for backward. + + When performing operations within this context manager, intermediary + results saved in the graph during the forward pass will be moved to CPU, + then copied back to the original device when needed for the backward pass. + If the graph was already on CPU, no tensor copy is performed. + + Use this context-manager to trade compute for GPU memory usage (e.g. + when your model doesn't fit in GPU memory during training). + + Args: + pin_memory (bool): If ``True`` tensors will be saved to CPU pinned memory + during packing and copied to GPU asynchronously during unpacking. + Defaults to ``False``. + Also see :ref:`cuda-memory-pinning`. + + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) + >>> a = torch.randn(5, requires_grad=True, device="cuda") + >>> b = torch.randn(5, requires_grad=True, device="cuda") + >>> c = torch.randn(5, requires_grad=True, device="cuda") + >>> + >>> def f(a, b, c): + ... prod_1 = a * b # a and b are saved on GPU + ... with torch.autograd.graph.save_on_cpu(): + ... prod_2 = prod_1 * c # prod_1 and c are saved on CPU + ... y = prod_2 * a # prod_2 and a are saved on GPU + ... return y + >>> + >>> y = f(a, b, c) + >>> del a, b, c # for illustration only + >>> # the content of a, b, and prod_2 are still alive on GPU + >>> # the content of prod_1 and c only live on CPU + >>> y.sum().backward() # all CPU tensors are moved back to GPU, for backward + >>> # all intermediary tensors are released (deleted) after the call to backward + """ + + def __init__(self, pin_memory: bool = False, device_type: str = "cuda") -> None: + device_module = getattr(torch, device_type, torch.cuda) + + def pack_to_cpu(tensor: torch.Tensor) -> tuple[torch.device, torch.Tensor]: + if not pin_memory: + return (tensor.device, tensor.cpu()) + packed = torch.empty( + tensor.size(), + dtype=tensor.dtype, + layout=tensor.layout, + pin_memory=(device_module.is_available() and not tensor.is_sparse), + ) + packed.copy_(tensor) + return (tensor.device, packed) + + def unpack_from_cpu(packed: tuple[torch.device, torch.Tensor]) -> torch.Tensor: + device, tensor = packed + return tensor.to(device, non_blocking=pin_memory) + + super().__init__(pack_to_cpu, unpack_from_cpu) + + +@contextlib.contextmanager +def disable_saved_tensors_hooks(error_message: str) -> Generator[None, None, None]: + """Context-manager that disables the saved tensors default hooks feature. + + Useful for if you are creating a feature that does not work with saved + tensors default hooks. + + Args: + error_message (str): When saved tensors default hooks are used when they + have been are disabled, a RuntimeError with this + error message gets raised. + + Example:: + + >>> # xdoctest: +SKIP(failing) + >>> message = "saved tensors default hooks are disabled" + >>> with torch.autograd.graph.disable_saved_tensors_hooks(message): + ... # Raises RuntimeError: saved tensors default hooks are disabled + ... with torch.autograd.graph.save_on_cpu(): + ... pass + """ + maybe_prev_message = None + try: + maybe_prev_message = ( + torch._C._autograd._saved_tensors_hooks_get_disabled_error_message() + ) + torch._C._autograd._saved_tensors_hooks_disable(error_message) + yield + finally: + # See NOTE: [disabled_error_message invariant] + if maybe_prev_message is None: + torch._C._autograd._saved_tensors_hooks_enable() + else: + torch._C._autograd._saved_tensors_hooks_disable(maybe_prev_message) + + +def set_warn_on_accumulate_grad_stream_mismatch(enabled: bool) -> None: + """Whether to warn when the AccumulateGrad node's stream does not match the stream + of the node that produced the incoming gradient. + """ + return torch._C._set_warn_on_accumulate_grad_stream_mismatch(enabled) + + +class _MultiHandle(RemovableHandle): + handles: tuple[RemovableHandle, ...] + + def __init__(self, handles: tuple[RemovableHandle, ...]) -> None: + self.handles = handles + + def remove(self) -> None: + for handle in self.handles: + handle.remove() + + def __getstate__(self) -> tuple[RemovableHandle, ...]: + return self.handles + + def __setstate__(self, state: tuple[RemovableHandle, ...]) -> None: + self.handles = state + + +def register_multi_grad_hook( + tensors: Sequence[torch.Tensor], + fn: Union[ + Callable[[Sequence[Optional[torch.Tensor]]], None], + Callable[[torch.Tensor], None], + ], + *, + mode: Literal["all", "any"] = "all", +) -> RemovableHandle: + r"""Register a multi-grad backward hook. + + There are two supported modes: ``"all"`` and ``"any"``. + + Under the ``"all"`` mode, the hook will be called after gradients with respect to every tensor in + :attr:`tensors` have been computed. If a tensor is in :attr:`tensors` but + is not part of the graph, or if a tensor is not needed to compute the gradients + for any ``inputs`` specified for the current ``.backward()`` or ``.grad()`` call, + this tensor will be ignored and the hook will not wait for its gradient to be + computed. + + After every non-ignored tensor's gradient has been computed, :attr:`fn` will be + called with those gradients. ``None`` will be passed for tensors that did not + have their gradients computed. + + Under the ``"any"`` mode, the hook will be called after the first gradient + with respect to a tensor in :attr:`tensors` has been computed. The hook + will be called with that gradient as its argument. + + The hook should not modify its arguments. + + This function returns a handle with a method ``handle.remove()`` that removes the hook. + + .. note:: + See :ref:`backward-hooks-execution` for more information on how when this hook + is executed, and how its execution is ordered relative to other hooks. + + Example:: + + >>> import torch + >>> + >>> a = torch.rand(2, 3, requires_grad=True) + >>> b = torch.rand(2, 3, requires_grad=True) + >>> c = a * b + >>> d = a * b + >>> + >>> def fn(grads): + ... print([g is not None for g in grads]) + ... + >>> torch.autograd.graph.register_multi_grad_hook((a, b, c, d), fn) + >>> + >>> c.sum().backward(retain_graph=True) + [True, True, True, False] + >>> c.sum().backward(inputs=(a,), retain_graph=True) + [True, False, True, False] + >>> + """ + supported_modes = ("all", "any") + lock = threading.Lock() + + if mode not in supported_modes: + raise ValueError(f"Expects mode to be one of {supported_modes} but got {mode}") + + if mode == "all": + count: dict[int, int] = {} + nb_calls = None + buffer: dict[int, list[Optional[torch.Tensor]]] = {} + + grad_fns = list(map(_get_grad_fn_or_grad_acc, tensors)) + len_tensors = len(tensors) + + def get_inner_hook(idx: int) -> Callable[[torch.Tensor], None]: + def inner_hook(grad: torch.Tensor) -> None: + nonlocal count, nb_calls, buffer, fn + id = torch._C._current_graph_task_id() + if id == -1: + raise AssertionError( + "expected this hook to be called inside a backward call" + ) + count[id] = count.get(id, 0) + # pyrefly: ignore [unsupported-operation] + buffer[id] = buffer.get(id, [None] * len_tensors) + + with lock: + curr_count, count[id] = count[id], count[id] + 1 + + if curr_count == 0: + # On the first call, compute the actual nb_calls and buffer + nb_calls = sum( + map(torch._C._will_engine_execute_node, grad_fns) + ) + + buffer[id][idx] = grad + + if nb_calls is None: + raise AssertionError("Expected nb_calls to be set") + if curr_count == nb_calls - 1: + fn = cast(Callable[[Sequence[Optional[torch.Tensor]]], None], fn) + fn(buffer[id]) + del count[id] + del buffer[id] + + return inner_hook + + handles = tuple( + t.register_hook(get_inner_hook(i)) for i, t in enumerate(tensors) + ) + elif mode == "any": + fn = cast(Callable[[torch.Tensor], None], fn) + ran_hook: dict[int, bool] = defaultdict(bool) + + @functools.wraps(fn) + def wrapped_fn(grad: torch.Tensor) -> None: + nonlocal ran_hook + id = torch._C._current_graph_task_id() + if id == -1: + raise AssertionError( + "expected this hook to be called inside a backward call" + ) + with lock: + prev, ran_hook[id] = ran_hook[id], True + if prev: + return + fn(grad) + + handles = tuple( + tensor.register_hook(wrapped_fn) + for tensor in tensors + if tensor.requires_grad + ) + + return _MultiHandle(handles) # type: ignore[possibly-undefined] + + +# NOTE [Allow mutation on tensors saved for backward] +# +# 1. Tensor gets saved for backward +# - remember the python object id and the version of the tensor +# - remember aliasing information (data_ptr of base + version) +# - save the original so we control its lifetime +# 2. Any time a tensor gets in-placed +# - for each tensor aliased to it: +# - check using its object id and version to see if it has been saved +# - if it has been saved, clone it +# - delete the reference to the original +# 3. during backward +# - if the clone exists, the tensor must've been modified in-place +_allow_mutation_on_saved_tensors_enabled: bool = False + + +_TID: TypeAlias = tuple[int, int, int] +_SID: TypeAlias = tuple[int, int] + + +def _get_tid(tensor: torch.Tensor) -> _TID: + # FIXME: This is almost definitely a bug. + if isinstance( + tensor, + ( + torch._subclasses.fake_tensor.FakeTensor, + torch._subclasses.functional_tensor.FunctionalTensor, + ), + ): + data_ptr = 0 + else: + data_ptr = tensor.data_ptr() + return (id(tensor), data_ptr, tensor._version) + + +def _get_sid(tensor: torch.Tensor) -> _SID: + # FIXME: This is almost definitely a bug. + if isinstance( + tensor, + ( + torch._subclasses.fake_tensor.FakeTensor, + torch._subclasses.functional_tensor.FunctionalTensor, + ), + ): + data_ptr = 0 + else: + data_ptr = tensor.data_ptr() + return (data_ptr, tensor._version) + + +class _Handle: + pass + + +class _swap_with_cloned(saved_tensors_hooks): + def __init__(self, ctx: "_AllowMutationOnSavedContext") -> None: + def pack_hook(tensor: torch.Tensor) -> _Handle: + tid = _get_tid(tensor) + sid = _get_sid(tensor) + # Tensors saved for backward have an entry in _tid_to_weakhandle + handle: Optional[_Handle] = None + + # Save aliasing information + ctx.sid_to_tid[sid].add(tid) + + # NB: The same tensor (of the same version) can be saved multiple times + if tid not in ctx.tid_to_weakhandle: + handle = _Handle() + ctx.tid_to_weakhandle[tid] = handle + ctx.original[handle] = tensor + else: + # Store an additional strong reference to the handle + handle = ctx.tid_to_weakhandle[tid] + return handle + + def unpack_hook(handle: _Handle) -> torch.Tensor: + error_msg = ( + "Trying to backward outside of the 'allow_mutation_on_saved_tensors' context" + "in which the graph was originally recorded." + ) + if not _allow_mutation_on_saved_tensors_enabled: + raise AssertionError(error_msg) + if handle in ctx.cloned: + res = ctx.cloned[handle] + else: + if handle not in ctx.original: + raise AssertionError(error_msg) + res = ctx.original[handle] + return res + + super().__init__(pack_hook, unpack_hook) + + +class _CloneArgBeforeMutateMode(TorchDispatchMode): + def __init__(self, ctx: "_AllowMutationOnSavedContext") -> None: + self.ctx = ctx + + def __torch_dispatch__( + self, + func: "OpOverload", + types: Iterable[type], + args: tuple[Any, ...] = (), + kwargs: Optional[dict[Any, Any]] = None, + ) -> Any: + kwargs = kwargs or {} + + def maybe_clone(t: torch.Tensor) -> None: + tid = _get_tid(t) + sid = _get_sid(t) + ctx = self.ctx + if sid in ctx.sid_to_tid: + for tid in ctx.sid_to_tid[sid]: + if tid not in ctx.tid_to_weakhandle: + # We know that if tid is in sid_to_tid, then it must also be in + # tid_to_weakhandle. However, it is possible for the tensor to be + # saved at one point, but cleared by backward before it is modified + # in-place. Consider the following example: + # + # >>> a = torch.randn(2, 3, requires_grad=True).clone() + # >>> out = (a**2).sum() + # >>> out.backward() + # >>> a.sin_() + continue + handle = ctx.tid_to_weakhandle[tid] + if handle in ctx.cloned: + # The same exact tensor has been cloned already + continue + ctx.cloned[handle] = ctx.original[handle].clone() + del ctx.original[handle] + + for idx, arg in enumerate(func._schema.arguments): + if arg.alias_info is not None and arg.alias_info.is_write: + if arg.is_out: + maybe_clone(kwargs["out"]) + elif isinstance(args[idx], list): + # Foreach case. (Possible optimization: if most of the + # tensors need to be cloned, use a for each clone?) + for t in args[idx]: + maybe_clone(t) + else: + maybe_clone(args[idx]) + + return func(*args, **kwargs) + + +class _AllowMutationOnSavedContext: + def __init__(self) -> None: + self.cloned: MutableMapping[_Handle, torch.Tensor] = WeakKeyDictionary() + self.original: MutableMapping[_Handle, torch.Tensor] = WeakKeyDictionary() + self.tid_to_weakhandle: MutableMapping[_TID, _Handle] = WeakValueDictionary() + self.sid_to_tid: dict[_SID, set[_TID]] = defaultdict(set) + + def clear(self) -> None: + self.cloned.clear() + self.original.clear() + self.tid_to_weakhandle.clear() + self.sid_to_tid.clear() + + +@contextlib.contextmanager +def allow_mutation_on_saved_tensors() -> Generator[ + _AllowMutationOnSavedContext, None, None +]: + """Context manager under which mutating tensors saved for backward is allowed. + + Under this context manager, tensors saved for backward are cloned on mutation, + so the original version can still be used during backward. Normally, mutating a tensor + saved for backward will result in an error raised when it's used during backward. + + To ensure the correct behavior, both the forward and backward should be run under + the same context manager. + + Returns: + An _AllowMutationOnSavedContext object storing the state managed by this + context manager. This object can be useful for debugging purposes. The state + managed by the context manager is automatically cleared upon exiting. + + Example:: + + >>> import torch + >>> with torch.autograd.graph.allow_mutation_on_saved_tensors(): + ... # forward + ... a = torch.ones(2, 3, requires_grad=True) + ... b = a.clone() + ... out = (b**2).sum() + ... b.sin_() + ... # backward + ... out.sum().backward() + ... + tensor([[0.8415, 0.8415, 0.8415], + [0.8415, 0.8415, 0.8415]], grad_fn=) + """ + global _allow_mutation_on_saved_tensors_enabled + + ctx = _AllowMutationOnSavedContext() + + with _swap_with_cloned(ctx), _CloneArgBeforeMutateMode(ctx): + try: + if _allow_mutation_on_saved_tensors_enabled: + raise RuntimeError( + "allow_mutation_on_saved_tensors contexts cannot be nested" + ) + _allow_mutation_on_saved_tensors_enabled = True + yield ctx + finally: + ctx.clear() + _allow_mutation_on_saved_tensors_enabled = False + + +def _register_logging_hooks_on_whole_graph( + t_outputs: Sequence[Union[torch.Tensor, GradientEdge]], +) -> Callable[[], None]: + grad_fns = list(map(_get_grad_fn_or_grad_acc, t_outputs)) + + def iter_graph(roots: list[Node]) -> Iterator[Node]: + if not roots: + return + seen: set[Node] = set() + q: deque[Node] = deque() + for node in roots: + if node is not None: + seen.add(node) + q.append(node) + + while q: + node = q.popleft() + for fn, _ in node.next_functions: + if fn in seen or fn is None: + continue + seen.add(fn) + q.append(fn) + + yield node + + def fmt(t: Optional[torch.Tensor]) -> str: + # Avoid circular import + from torch.utils._dtype_abbrs import dtype_abbrs + + if t is None: + return "None" + return f"{dtype_abbrs[t.dtype]}[{', '.join(map(str, t.shape))}]" + + def prehook(grad_outputs: Sequence[Optional[torch.Tensor]]) -> None: + node = torch._C._current_autograd_node() + grad_outputs_str = f"[{','.join(fmt(t) for t in grad_outputs)}]" + log_str = f"Executing: {node} with grad_outputs: {grad_outputs_str}" + log.debug(log_str) + + handles = [node.register_prehook(prehook) for node in iter_graph(grad_fns)] + + def unregister_hooks() -> None: + for handle in handles: + handle.remove() + + return unregister_hooks + + +def _engine_run_backward( + t_outputs: Sequence[Union[torch.Tensor, GradientEdge]], + *args: Any, + **kwargs: Any, +) -> tuple[torch.Tensor, ...]: + attach_logging_hooks = log.getEffectiveLevel() <= logging.DEBUG + if attach_logging_hooks: + unregister_hooks = _register_logging_hooks_on_whole_graph(t_outputs) + try: + return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass + t_outputs, *args, **kwargs + ) # Calls into the C++ engine to run the backward pass + finally: + if attach_logging_hooks: + unregister_hooks() # type: ignore[possibly-undefined] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/profiler.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..fa43af270117109158e7e6c79f2eb19322c7ab7e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/profiler.py @@ -0,0 +1,1242 @@ +# mypy: allow-untyped-defs +import uuid +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass +from time import perf_counter_ns +from typing import Any, Optional +from warnings import warn + +import torch +import torch.cuda +from torch._C import _get_privateuse1_backend_name +from torch._C._profiler import _ExperimentalConfig +from torch.autograd import ( + _disable_profiler, + _enable_profiler, + _kineto_step, + _prepare_profiler, + _ProfilerResult, + _supported_activities, + _toggle_collection_dynamic, + DeviceType, + kineto_available, + ProfilerActivity, + ProfilerConfig, + ProfilerState, +) +from torch.autograd.profiler_util import ( + _filter_name, + _filter_stack_entry, + _rewrite_name, + EventList, + FunctionEvent, + MEMORY_EVENT_NAME, + MemRecordsAcc, + OUT_OF_MEMORY_EVENT_NAME, +) +from torch.futures import Future + + +__all__ = [ + "profile", + "record_function", + "emit_itt", + "emit_nvtx", + "load_nvprof", + "EnforceUnique", + "parse_nvprof_trace", + "KinetoStepTracker", + "EventList", + "FunctionEvent", + "MemRecordsAcc", +] + +try: + # Available in Python >= 3.2 + from contextlib import ContextDecorator as _ContextDecorator +except ImportError: + import functools + + class _ContextDecorator: # type: ignore[no-redef] + def __enter__(self): + raise NotImplementedError + + def __exit__(self, exc_type, exc_val, exc_tb): + raise NotImplementedError + + def __call__(self, func): + @functools.wraps(func) + def wrapped(*args, **kwargs): + with self: + return func(*args, **kwargs) + + return wrapped + + +# global python state - whether profiler is currently enabled +# useful for fast python checks to reduce latency +_is_profiler_enabled: bool = False + + +def _set_is_profiler_enabled(enable: bool): + global _is_profiler_enabled + _is_profiler_enabled = enable + + +def _run_on_profiler_start(): + _set_is_profiler_enabled(True) + + +def _run_on_profiler_stop(): + _set_is_profiler_enabled(False) + + +@dataclass +class _ProfilerStats: + "Profiler timing and stats used by developers to catch issues/regressions" + + profiling_window_duration_sec: float = 0 + number_of_events: int = 0 + profiler_prepare_call_duration_us: int = 0 + profiler_enable_call_duration_us: int = 0 + profiler_disable_call_duration_us: int = 0 + parse_kineto_call_duration_us: int = 0 + function_events_build_tree_call_duration_us: int = 0 + + +class profile: + """Context manager that manages autograd profiler state and holds a summary of results. + + .. note:: + This is the backend, most people should use :mod:`torch.profiler` instead. + + Under the hood it just records events of functions being executed in C++ and + exposes those events to Python. You can wrap any code into it and it will + only report runtime of PyTorch functions. + Note: profiler is thread local and is automatically propagated into the async tasks + + Args: + enabled (bool, optional): Setting this to False makes this context manager a no-op. + + use_cuda (bool, optional): Enables timing of CUDA events as well + using the cudaEvent API. (will be deprecated) + + use_device (str, optional): Enables timing of device events. + Adds approximately 4us of overhead to each tensor operation when use cuda. + The valid devices options are 'cuda', 'xpu', 'mtia' and 'privateuseone'. + + record_shapes (bool, optional): If shapes recording is set, information + about input dimensions will be collected. This allows one to see which + dimensions have been used under the hood and further group by them + using prof.key_averages(group_by_input_shape=True). Please note that + shape recording might skew your profiling data. It is recommended to + use separate runs with and without shape recording to validate the timing. + Most likely the skew will be negligible for bottom most events (in a case + of nested function calls). But for higher level functions the total + self cpu time might be artificially increased because of the shape + collection. + + with_flops (bool, optional): If with_flops is set, the profiler will estimate + the FLOPs (floating point operations) value using the operator's input shape. + This allows one to estimate the hardware performance. Currently, + this option only works for the matrix multiplication and 2D convolution operators. + + profile_memory (bool, optional): track tensor memory allocation/deallocation. + + with_stack (bool, optional): record source information (file and line number) for the ops. + + with_modules (bool): record module hierarchy (including function names) + corresponding to the callstack of the op. e.g. If module A's forward call's + module B's forward which contains an aten::add op, + then aten::add's module hierarchy is A.B + Note that this support exist, at the moment, only for TorchScript models + and not eager mode models. + + use_kineto (bool, optional): experimental, enable profiling with Kineto profiler. + + use_cpu (bool, optional): profile CPU events; setting to ``False`` requires + ``use_kineto=True`` and can be used to lower the overhead for GPU-only profiling. + + experimental_config (_ExperimentalConfig) : A set of experimental options + used by profiler libraries like Kineto. Note, backward compatibility is not guaranteed. + + acc_events (bool): Enable the accumulation of FunctionEvents across multiple profiling cycles + + + .. warning:: + Enabling memory profiling or source attribution incurs additional profiler + overhead + + .. warning:: + This context managers should not be called recursively, i.e. no nested + instances are allowed + + .. warning:: + Due to some CUDA multiprocessing limitations (see :ref:`multiprocessing-cuda-note`), + one cannot use the profiler with ``use_device = 'cuda'`` to benchmark + DataLoaders with ``num_workers > 0``. If you wish to benchmark data loading, + please use ``use_device = None`` or ``num_workers = 0``. + + Example: + >>> # xdoctest: +SKIP + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER) + >>> x = torch.randn((1, 1), requires_grad=True) + >>> with torch.autograd.profiler.profile() as prof: + >>> for _ in range(100): # any normal python code, really! + >>> y = x ** 2 + >>> y.backward() + >>> # NOTE: some columns were removed for brevity + >>> print(prof.key_averages().table(sort_by="self_cpu_time_total")) + ----------------------------------- --------------- --------------- --------------- + Name Self CPU total CPU time avg Number of Calls + ----------------------------------- --------------- --------------- --------------- + mul 32.048ms 32.048ms 200 + pow 27.041ms 27.041ms 200 + PowBackward0 9.727ms 55.483ms 100 + torch::autograd::AccumulateGrad 9.148ms 9.148ms 100 + torch::autograd::GraphRoot 691.816us 691.816us 100 + ----------------------------------- --------------- --------------- --------------- + + """ + + def __init__( + self, + enabled=True, + *, + use_cuda=False, # Deprecated + use_device=None, + record_shapes=False, + with_flops=False, + profile_memory=False, + with_stack=False, + with_modules=False, + use_kineto=False, + use_cpu=True, + experimental_config=None, + acc_events=False, + custom_trace_id_callback=None, + ): + self.enabled: bool = enabled + if not self.enabled: + return + self.use_cuda = use_cuda + if self.use_cuda: + warn( + "The attribute `use_cuda` will be deprecated soon, " + "please use ``use_device = 'cuda'`` instead.", + FutureWarning, + stacklevel=2, + ) + self.use_device: Optional[str] = "cuda" + else: + self.use_device = use_device + # TODO Consider changing _function_events into data structure with size cap + self._function_events: Optional[EventList] = None + self._old_function_events: Optional[EventList] = None + # Function event processing is done lazily + self._needs_processing = False + self.entered = False + self.record_shapes = record_shapes + self.with_flops = with_flops + self.record_shapes |= self.with_flops + self.profile_memory = profile_memory + self.with_stack = with_stack + self.with_modules = with_modules + self.use_cpu = use_cpu + self.acc_events = acc_events + if experimental_config is None: + experimental_config = _ExperimentalConfig() + self.experimental_config = experimental_config + self.kineto_results: Optional[_ProfilerResult] = None + self.profiling_start_time_ns = 0 + self.profiling_end_time_ns = 0 + self._stats = _ProfilerStats() + self.custom_trace_id_callback = custom_trace_id_callback + self.trace_id = "" + if not self.use_cpu: + if not use_kineto: + raise AssertionError( + "Device-only events supported only with Kineto (use_kineto=True)" + ) + + if self.use_device is not None: + VALID_DEVICE_OPTIONS = ["cuda", "xpu", "mtia", "hpu"] + if _get_privateuse1_backend_name() != "privateuseone": + VALID_DEVICE_OPTIONS.append(_get_privateuse1_backend_name()) + if self.use_device not in VALID_DEVICE_OPTIONS: + warn( + f"The {self.use_device} is not a valid device option.", stacklevel=2 + ) + self.use_device = None + + if self.use_device == "cuda" and not torch.cuda.is_available(): + warn("CUDA is not available, disabling CUDA profiling", stacklevel=2) + self.use_cuda = False + self.use_device = None + + if self.use_device == "xpu" and not torch.xpu.is_available(): + warn("XPU is not available, disabling XPU profiling", stacklevel=2) + self.use_device = None + + if self.use_device == "hpu" and not ( + hasattr(torch, "hpu") and torch.hpu.is_available() + ): + warn("HPU is not available, disabling HPU profiling", stacklevel=2) + self.use_device = None + + self.kineto_activities = set() + if self.use_cpu: + self.kineto_activities.add(ProfilerActivity.CPU) + + self.profiler_kind = ProfilerState.KINETO + if self.use_device == "cuda": + if not use_kineto or ProfilerActivity.CUDA not in _supported_activities(): + if not self.use_cpu: + raise AssertionError("Legacy CUDA profiling requires use_cpu=True") + self.profiler_kind = ProfilerState.KINETO_GPU_FALLBACK + else: + self.kineto_activities.add(ProfilerActivity.CUDA) + elif self.use_device == "xpu": + if not (use_kineto and ProfilerActivity.XPU in _supported_activities()): + raise AssertionError( + "Legacy XPU profiling is not supported. Requires use_kineto=True on XPU devices." + ) + self.kineto_activities.add(ProfilerActivity.XPU) + elif self.use_device == "mtia": + if not (use_kineto and ProfilerActivity.MTIA in _supported_activities()): + raise AssertionError( + "Legacy MTIA profiling is not supported. Requires use_kineto=True on MTIA devices." + ) + self.kineto_activities.add(ProfilerActivity.MTIA) + elif self.use_device == "hpu": + if not (use_kineto and ProfilerActivity.HPU in _supported_activities()): + raise AssertionError( + "Legacy HPU profiling is not supported. Requires use_kineto=True on HPU devices." + ) + self.kineto_activities.add(ProfilerActivity.HPU) + elif self.use_device is not None and self.use_device != "privateuseone": + if ( + not use_kineto + or ProfilerActivity.PrivateUse1 not in _supported_activities() + ): + if not self.use_cpu: + raise AssertionError( + "Legacy custombackend profiling requires use_cpu=True" + ) + self.profiler_kind = ProfilerState.KINETO_PRIVATEUSE1_FALLBACK + else: + self.kineto_activities.add(ProfilerActivity.PrivateUse1) + + if len(self.kineto_activities) == 0: + raise AssertionError("No activities specified for the profiler") + + def default_trace_id(self): + # Generate a UUID + uuid_raw = uuid.uuid4() + + return f"{uuid_raw.int:032X}" + + def create_trace_id(self): + if self.custom_trace_id_callback: + return self.custom_trace_id_callback() + return self.default_trace_id() + + def config(self, create_trace_id=False): + # only need to generate new trace id upon prepare trace not start trace + if create_trace_id: + trace_id = self.create_trace_id() + self.trace_id = trace_id + return ProfilerConfig( + self.profiler_kind, + self.record_shapes, + self.profile_memory, + self.with_stack, + self.with_flops, + self.with_modules, + self.experimental_config, + self.trace_id, + ) + + def __enter__(self): + if not self.enabled: + return + if self.entered: + raise RuntimeError("Profiler context manager is not reentrant") + self._prepare_trace() + self._start_trace() + return self + + def _prepare_trace(self): + self.entered = True + t0 = perf_counter_ns() + _prepare_profiler(self.config(create_trace_id=True), self.kineto_activities) + t1 = perf_counter_ns() + self._stats.profiler_prepare_call_duration_us = int((t1 - t0) / 1000) + + def _start_trace(self): + self.entered = True + _run_on_profiler_start() + t0 = perf_counter_ns() + _enable_profiler(self.config(create_trace_id=False), self.kineto_activities) + t1 = perf_counter_ns() + self._stats.profiler_enable_call_duration_us = int((t1 - t0) / 1000) + self.profiling_start_time_ns = t1 + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.enabled: + return + if self.use_device and hasattr(torch, self.use_device): + device_module = getattr(torch, self.use_device) + if hasattr(device_module, "synchronize"): + device_module.synchronize() + + if self._function_events and self.acc_events: + self._old_function_events = self._function_events + self._function_events = None + self._needs_processing = True + + t0 = perf_counter_ns() + + self.kineto_results = _disable_profiler() + t1 = perf_counter_ns() + self._stats.profiler_disable_call_duration_us = int((t1 - t0) / 1000) + self.profiling_end_time_ns = t0 + + _run_on_profiler_stop() + + self._stats.profiling_window_duration_sec = ( + (self.profiling_end_time_ns - self.profiling_start_time_ns) * 1.0 / 1e9 + ) + + # If we plan to accumulate events we should post process the function events + # right away to retain the state across multiple start/stop calls + if self.acc_events: + self._ensure_function_events() + return False + + def __repr__(self): + if self._needs_processing: + self._ensure_function_events() + if self._function_events is None: + return "" + return repr(self._function_events) + + def __str__(self): + if self._needs_processing: + self._ensure_function_events() + if self._function_events is None: + return "" + return str(self._function_events) + + def _ensure_function_events(self): + """Process function events lazily if required""" + if self._function_events is not None: + return + self._needs_processing = False + + t0 = perf_counter_ns() + parsed_results = [] + if self.kineto_results: + parsed_results = self._parse_kineto_results(self.kineto_results) + t1 = perf_counter_ns() + self._stats.parse_kineto_call_duration_us = int((t1 - t0) / 1000) + + self._function_events = EventList( + parsed_results, + use_device=self.use_device, + profile_memory=self.profile_memory, + with_flops=self.with_flops, + ) + t0 = perf_counter_ns() + self._function_events._build_tree() + t1 = perf_counter_ns() + self._stats.function_events_build_tree_call_duration_us = int((t1 - t0) / 1000) + self._stats.number_of_events = len(self._function_events) + + if self._old_function_events and self.acc_events: + for evt in self._old_function_events: + self._function_events.append(evt) + self._old_function_events = None + + if self._function_events is None: + raise RuntimeError("Profiler didn't finish running") + + @property + def function_events(self): + if self._function_events is None or self._needs_processing: + self._ensure_function_events() + return self._function_events + + def table( + self, + sort_by=None, + row_limit=100, + max_src_column_width=75, + max_name_column_width=55, + max_shapes_column_width=80, + header=None, + top_level_events_only=False, + ): + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + return self._function_events.table( + sort_by=sort_by, + row_limit=row_limit, + max_src_column_width=max_src_column_width, + max_name_column_width=max_name_column_width, + max_shapes_column_width=max_shapes_column_width, + header=header, + top_level_events_only=top_level_events_only, + ) + + table.__doc__ = EventList.table.__doc__ + + def export_chrome_trace(self, path): + """ + Exports the collected trace in Chrome JSON format. If kineto is enabled, only + last cycle in schedule is exported. + """ + if kineto_available(): + self.kineto_results.save(path) # type: ignore[union-attr] + else: + self._ensure_function_events() + return self._function_events.export_chrome_trace(path) # type: ignore[union-attr] + + export_chrome_trace.__doc__ = EventList.export_chrome_trace.__doc__ + + def export_stacks(self, path: str, metric: str = "self_cpu_time_total"): + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + if not self.with_stack: + raise AssertionError("export_stacks() requires with_stack=True") + return self._function_events.export_stacks(path, metric) + + def toggle_collection_dynamic( + self, enabled: bool, activities: Iterable[ProfilerActivity] + ): + """ + Toggles the collection of activities for the current profiler instance. + """ + return _toggle_collection_dynamic(enabled, set(activities)) + + def key_averages( + self, + group_by_input_shape=False, + group_by_stack_n=0, + group_by_overload_name=False, + ): + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + return self._function_events.key_averages( + group_by_input_shape, group_by_stack_n, group_by_overload_name + ) + + key_averages.__doc__ = EventList.key_averages.__doc__ + + def total_average(self): + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + return self._function_events.total_average() + + total_average.__doc__ = EventList.total_average.__doc__ + + @property + def self_cpu_time_total(self): + """Returns total time spent on CPU. + + The total time is a sum of all self times across all the events. + """ + self._ensure_function_events() + if self._function_events is None: + raise AssertionError("Expected profiling results") + return self._function_events.self_cpu_time_total + + def _parse_kineto_results(self, result: _ProfilerResult): + # result.events() has most of the events - PyTorch op-level and device-level events + + trace_start_ns = result.trace_start_ns() + mem_records = [ + [evt, False] for evt in result.events() if evt.name() == MEMORY_EVENT_NAME + ] + oom_records = [ + evt for evt in result.events() if evt.name() == OUT_OF_MEMORY_EVENT_NAME + ] + mem_records_acc = MemRecordsAcc(mem_records) + + def _cpu_memory_usage(mem_record): + return ( + mem_record.nbytes() + if mem_record.device_type() + in [DeviceType.CPU, DeviceType.MKLDNN, DeviceType.IDEEP] + else 0 + ) + + def _device_memory_usage(mem_record): + return ( + mem_record.nbytes() + if mem_record.device_type() + in [ + DeviceType.CUDA, + DeviceType.PrivateUse1, + DeviceType.HIP, + DeviceType.XPU, + ] + else 0 + ) + + # Create and return FunctionEvent list, which contains all function events + # Here 2 function events are created: + # all_function_events contains all events associated with each kineto event from result + all_function_events = [] + # frontend_function_events contains the events in aten or torch frontend level, + # whose correlation id is 0 + frontend_function_events = [] + device_corr_map: dict[int, list[FunctionEvent]] = {} + max_evt_id = 0 + for kineto_event in result.events(): + if ( + _filter_name(kineto_event.name()) + or getattr(kineto_event, "is_hidden_event", lambda: False)() + ): + continue + rel_start_ns = kineto_event.start_ns() - trace_start_ns + rel_end_ns = kineto_event.end_ns() - trace_start_ns + abs_end_ns = kineto_event.end_ns() + + cpu_memory_usage = 0 + device_memory_usage = 0 + if kineto_event.device_type() == DeviceType.CPU: + # find the corresponding memory allocation events + for mem_record in mem_records_acc.in_interval( + kineto_event.start_ns() / 1000, abs_end_ns / 1000 + ): + cpu_memory_usage += _cpu_memory_usage(mem_record[0]) + device_memory_usage += _device_memory_usage(mem_record[0]) + mem_record[1] = True + + is_async = kineto_event.is_async() or ( + kineto_event.start_thread_id() != kineto_event.end_thread_id() + ) + + fe = FunctionEvent( + id=kineto_event.correlation_id(), + name=_rewrite_name(name=kineto_event.name(), with_wildcard=True), + overload_name=kineto_event.overload_name(), + trace_name=_rewrite_name(name=kineto_event.name(), with_wildcard=False), + thread=kineto_event.start_thread_id(), + start_us=rel_start_ns / 1000, + end_us=rel_end_ns / 1000, + fwd_thread=kineto_event.fwd_thread_id(), + input_shapes=kineto_event.shapes(), + concrete_inputs=kineto_event.concrete_inputs(), + kwinputs=kineto_event.kwinputs(), + stack=[ + entry + for entry in kineto_event.stack() + if _filter_stack_entry(entry) + ], + scope=kineto_event.scope(), + use_device=self.use_device, + cpu_memory_usage=cpu_memory_usage, + device_memory_usage=device_memory_usage, + is_async=is_async, + sequence_nr=kineto_event.sequence_nr(), + device_type=kineto_event.device_type(), + device_index=kineto_event.device_index(), + device_resource_id=kineto_event.device_resource_id(), + flops=kineto_event.flops(), + is_user_annotation=kineto_event.is_user_annotation(), + metadata_json=kineto_event.metadata_json(), + ) + max_evt_id = max(max_evt_id, fe.id) + if fe.device_type == DeviceType.CPU and not fe.is_async: + if self.use_device == _get_privateuse1_backend_name(): + privateuse1_time = kineto_event.privateuse1_elapsed_us() + if privateuse1_time > 0: + fe.append_kernel(fe.name, fe.device_index, privateuse1_time) + fe.is_legacy = True + elif self.use_device == "cuda": + # Check if we have CUDA time as a fallback + cuda_time = kineto_event.cuda_elapsed_us() + if cuda_time > 0: + fe.append_kernel(fe.name, fe.device_index, cuda_time) + fe.is_legacy = True + all_function_events.append(fe) + corr_id = kineto_event.linked_correlation_id() + if corr_id > 0: + if corr_id not in device_corr_map: + device_corr_map[corr_id] = [] + device_corr_map[corr_id].append(fe) + elif corr_id == 0: + frontend_function_events.append(fe) + else: + raise RuntimeError( + f"Got negative correlation id {corr_id} in profiler post processing" + ) + + # associate device kernels and device runtime (CPU) with CPU events + for fe in frontend_function_events: + if ( + fe.device_type == DeviceType.CPU + and not fe.is_async + and fe.id in device_corr_map + ): + for f_evt in device_corr_map[fe.id]: + if ( + f_evt.device_type == DeviceType.CUDA + or f_evt.device_type == DeviceType.PrivateUse1 + ): + fe.append_kernel( + f_evt.name, + f_evt.device_index, + f_evt.time_range.end - f_evt.time_range.start, + ) + elif f_evt.device_type == DeviceType.CPU: + # make sure that 'thread' of a CPU Kineto (e.g. Device Runtime) event is associated + # with the 'thread' of the corresponding linked PyTorch event to properly track + # parents and children + f_evt.thread = fe.thread + + def createFunctionEventForMemoryEvents(evt): + rel_start_ns = evt.start_ns() - trace_start_ns + fe = FunctionEvent( + id=max_evt_id, + name=evt.name(), + overload_name="", + trace_name=None, # not outputting in the trace + thread=evt.start_thread_id(), + start_us=rel_start_ns / 1000, + end_us=rel_start_ns / 1000, # no duration + fwd_thread=evt.start_thread_id(), + input_shapes=[], + stack=[], + scope=0, # RecordScope::FUNCTION + use_device=self.use_device, + cpu_memory_usage=_cpu_memory_usage(evt), + device_memory_usage=_device_memory_usage(evt), + is_async=False, + sequence_nr=-1, + device_type=DeviceType.CPU, + device_index=0, + ) + return fe + + # output top-level memory events + for mem_record in mem_records: + if not mem_record[1]: + max_evt_id += 1 + fe = createFunctionEventForMemoryEvents(mem_record[0]) + all_function_events.append(fe) + + for oom_record in oom_records: + max_evt_id += 1 + fe = createFunctionEventForMemoryEvents(oom_record) + all_function_events.append(fe) + + all_function_events.sort( + key=lambda evt: [evt.time_range.start, -evt.time_range.end] + ) + return all_function_events + + +# pyrefly: ignore [invalid-inheritance] +class record_function(_ContextDecorator): + """Context manager/function decorator that adds a label to a code block/function when running autograd profiler. + Label will only appear if CPU activity tracing is enabled. + + It is useful when tracing the code profile. + + Args: + name (str): Label assigned to the block of code. + node_id (int): ID of node, for distributed profiling. Unset in + non-distributed cases. + + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER) + >>> x = torch.randn((1, 1), requires_grad=True) + >>> with torch.autograd.profiler.profile() as prof: + ... y = x**2 + ... with torch.autograd.profiler.record_function( + ... "label-z" + ... ): # label the block + ... z = y**3 + ... y.backward() + >>> # xdoctest: +IGNORE_WANT + >>> # NOTE: some columns were removed for brevity + >>> print(prof.key_averages().table(sort_by="self_cpu_time_total")) + ----------------------------------- --------------- --------------- --------------- + Name Self CPU total % CPU time avg Number of Calls + ----------------------------------- --------------- --------------- --------------- + pow 60.77% 47.470us 3 + mul 21.73% 25.465us 2 + PowBackward0 12.03% 121.891us 1 + torch::autograd::AccumulateGrad 2.70% 6.324us 1 + label-z 2.13% 12.421us 1 + torch::autograd::GraphRoot 0.64% 1.503us 1 + ----------------------------------- --------------- --------------- --------------- + Self CPU time total: 234.344us + CUDA time total: 0.000us + + """ + + def __init__(self, name: str, args: Optional[str] = None): + self.name: str = name + self.args: Optional[str] = args + # Whether or not we should run record function's end callbacks when exiting. + self.run_callbacks_on_exit: bool = True + # TODO: TorchScript ignores standard type annotation here + # self.record: Optional["torch.classes.profiler._RecordFunction"] = None + self.record = torch.jit.annotate( + # pyrefly: ignore [not-a-type] + Optional["torch.classes.profiler._RecordFunction"], + None, + ) + + def __enter__(self): + self.record = torch.ops.profiler._record_function_enter_new( + self.name, self.args + ) + return self + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any): + if not self.run_callbacks_on_exit: + return + + # Local variable is needed by TorchScript to refine Optional[T] to T + record = self.record + if record is None: + raise AssertionError("Expected record to be set") + + # TODO: Too slow with __torch_function__ handling enabled + # See https://github.com/pytorch/pytorch/issues/76410 + if not torch.jit.is_scripting(): + with torch._C.DisableTorchFunctionSubclass(): + torch.ops.profiler._record_function_exit._RecordFunction(record) + else: + torch.ops.profiler._record_function_exit(record) + + def _call_end_callbacks_on_future(self, fut: Future[Any]) -> Future[Any]: + """Use for profiling async calls that return a future. + + Calling this function will extend recording beyond this scope, until the future is + satisfied. It is useful for profiling the end to end time of asynchronous calls. + This function should only be called once to attach the callback onto the future, and + will throw if called multiple times. + + Args: + fut: (torch._C.Future): future for which to schedule + callback for. + + Returns: + A future that completes with the value of the passed in future when + the profiling callbacks have ran. + + """ + # Throw if we have already attached a callback onto the future. + if not self.run_callbacks_on_exit: + raise RuntimeError("_call_end_callbacks_on_future can only be called once.") + + # We are scheduling to run this RecordFunction's end callbacks when the + # passed in future completes, so don't run end callbacks on exit. + self.run_callbacks_on_exit = False + + # Local variable is needed by TorchScript to refine Optional[T] to T + record = self.record + if record is None: + raise AssertionError("Expected record to be set") + + # TODO: Too slow with __torch_function__ handling enabled + # See https://github.com/pytorch/pytorch/issues/76410 + if not torch.jit.is_scripting(): + with torch._C.DisableTorchFunctionSubclass(): + profiled_future = ( + torch.ops.profiler._call_end_callbacks_on_jit_fut._RecordFunction( + record, fut + ) + ) + else: + profiled_future = torch.ops.profiler._call_end_callbacks_on_jit_fut( + record, fut + ) + return profiled_future + + +class emit_itt: + """Context manager that makes every autograd operation emit an ITT range. + + It is useful when running the program under Intel(R) VTune Profiler:: + + vtune <--vtune-flags> + + The Instrumentation and Tracing Technology (ITT) API enables your application to generate and + control the collection of trace data during its execution across different Intel tools. + This context manager is to annotate Intel(R) VTune Profiling trace. With help of this context manager, + you will be able to see labeled ranges in Intel(R) VTune Profiler GUI. + + .. warning: + This context manager should not be called recursively, i.e. at most one + instance should be enabled at any given time. + + Args: + enabled (bool, optional): Setting ``enabled=False`` makes this context manager a no-op. + Default: ``True``. + record_shapes (bool, optional): If ``record_shapes=True``, the itt range wrapping + each autograd op will append information about the sizes of Tensor arguments received + by that op, in the following format: + ``[[arg0.size(0), arg0.size(1), ...], [arg1.size(0), arg1.size(1), ...], ...]`` + Non-tensor arguments will be represented by ``[]``. + Arguments will be listed in the order they are received by the backend op. + Please note that this order may not match the order in which those arguments were passed + on the Python side. Also note that shape recording may increase the overhead of itt range creation. + Default: ``False`` + + Example: + >>> # xdoctest: +SKIP("Undefined variables") + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER) + >>> with torch.autograd.profiler.emit_itt(): + ... model(x) + + """ + + def __init__(self, enabled=True, record_shapes=False): + self.enabled = enabled + self.entered = False + self.record_shapes = record_shapes + + def __enter__(self): + if not self.enabled: + return + if self.entered: + raise RuntimeError("ITT annotation context manager is not reentrant") + self.entered = True + _run_on_profiler_start() + _enable_profiler( + ProfilerConfig( + ProfilerState.ITT, + self.record_shapes, + False, + False, + False, + False, + _ExperimentalConfig(), + ), + set(), + ) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.enabled: + return + _disable_profiler() + _run_on_profiler_stop() + return False + + +class emit_nvtx: + """Context manager that makes every autograd operation emit an NVTX range. + + It is useful when running the program under nvprof:: + + nvprof --profile-from-start off -o trace_name.prof -- + + Unfortunately, there's no way to force nvprof to flush the data it collected + to disk, so for CUDA profiling one has to use this context manager to annotate + nvprof traces and wait for the process to exit before inspecting them. + Then, either NVIDIA Visual Profiler (nvvp) can be used to visualize the timeline, or + :func:`torch.autograd.profiler.load_nvprof` can load the results for inspection + e.g. in Python REPL. + + .. warning: + This context manager should not be called recursively, i.e. at most one + instance should be enabled at any given time. + + Args: + enabled (bool, optional): Setting ``enabled=False`` makes this context manager a no-op. + Default: ``True``. + record_shapes (bool, optional): If ``record_shapes=True``, the nvtx range wrapping + each autograd op will append information about the sizes of Tensor arguments received + by that op, in the following format: + ``[[arg0.size(0), arg0.size(1), ...], [arg1.size(0), arg1.size(1), ...], ...]`` + Non-tensor arguments will be represented by ``[]``. + Arguments will be listed in the order they are received by the backend op. + Please note that this order may not match the order in which those arguments were passed + on the Python side. Also note that shape recording may increase the overhead of nvtx range creation. + Default: ``False`` + + Example: + >>> # xdoctest: +SKIP("undefined variables") + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD_PROFILER) + >>> with torch.cuda.profiler.profile(): + ... model(x) # Warmup CUDA memory allocator and profiler + ... with torch.autograd.profiler.emit_nvtx(): + ... model(x) + + **Forward-backward correlation** + + When viewing a profile created using :class:`emit_nvtx` in the Nvidia Visual Profiler, + correlating each backward-pass op with the corresponding forward-pass op can be difficult. + To ease this task, :class:`emit_nvtx` appends sequence number information to the ranges it + generates. + + During the forward pass, each function range is decorated with ``seq=``. ``seq`` is a running + counter, incremented each time a new backward Function object is created and stashed for backward. + Thus, the ``seq=`` annotation associated with each forward function range tells you that + if a backward Function object is created by this forward function, + the backward object will receive sequence number N. + During the backward pass, the top-level range wrapping each C++ backward Function's + ``apply()`` call is decorated with ``stashed seq=``. ``M`` is the sequence number that + the backward object was created with. By comparing ``stashed seq`` numbers in backward with ``seq`` + numbers in forward, you can track down which forward op created each backward Function. + + Any functions executed during the backward pass are also decorated with ``seq=``. During + default backward (with ``create_graph=False``) this information is irrelevant, and in fact, + ``N`` may simply be 0 for all such functions. Only the top-level ranges associated with + backward Function objects' ``apply()`` methods are useful, as a way to correlate these Function + objects with the earlier forward pass. + + **Double-backward** + + If, on the other hand, a backward pass with ``create_graph=True`` is underway (in other words, + if you are setting up for a double-backward), each function's execution during backward + is given a nonzero, useful ``seq=``. Those functions may themselves create Function objects + to be executed later during double-backward, just as the original functions in the forward pass did. + The relationship between backward and double-backward is conceptually the same as the relationship + between forward and backward: The functions still emit current-sequence-number-tagged ranges, + the Function objects they create still stash those sequence numbers, and during the eventual + double-backward, the Function objects' ``apply()`` ranges are still tagged with ``stashed seq`` + numbers, which can be compared to `seq` numbers from the backward pass. + + .. warning: + The sequence number is thread-local, and some forward functions don't create an associated + backward Function object (instead delegating that to sub-functions further down the call chain). + For these reasons, the correspondence of stashed sequence numbers in + backward Function ``apply()`` ranges with `seq` numbers in forward-pass ranges is + not guaranteed to be 1 to 1. The sequence numbers alone may not be enough to fully + disambiguate which forward function created which + backward Function object. You may need to make a judgment based on analytic knowledge of what + the expected correspondence should be. + """ + + def __init__(self, enabled=True, record_shapes=False): + self.enabled = enabled + self.entered = False + self.record_shapes = record_shapes + + def __enter__(self): + if not self.enabled: + return + if self.entered: + raise RuntimeError("NVTX annotation context manager is not reentrant") + self.entered = True + torch.cuda.synchronize() + _run_on_profiler_start() + _enable_profiler( + ProfilerConfig( + ProfilerState.NVTX, + self.record_shapes, + False, + False, + False, + False, + _ExperimentalConfig(), + ), + set(), + ) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.enabled: + return + torch.cuda.synchronize() + _disable_profiler() + _run_on_profiler_stop() + return False + + +def load_nvprof(path): + """Open an nvprof trace file and parses autograd annotations. + + Args: + path (str): path to nvprof trace + """ + return EventList(parse_nvprof_trace(path)) + + +class EnforceUnique: + """Raises an error if a key is seen more than once.""" + + def __init__(self): + self.seen = set() + + def see(self, *key): + r""" + Observe a key and raise an error if it is seen multiple times. + """ + if key in self.seen: + raise RuntimeError("duplicate key: " + str(key)) + self.seen.add(key) + + +def parse_nvprof_trace(path): + import sqlite3 + + conn = sqlite3.connect(path) + conn.row_factory = sqlite3.Row + + # Parse strings table + strings = {} + for r in conn.execute("SELECT _id_ as id, value FROM StringTable"): + strings[r["id"]] = torch._C._demangle(r["value"]) + + # First, find all functions and create FunctionEvents for them + marker_query = """ + SELECT + start.id AS marker_id, start.name, start.timestamp AS start_time, end.timestamp AS end_time + FROM + CUPTI_ACTIVITY_KIND_MARKER AS start INNER JOIN CUPTI_ACTIVITY_KIND_MARKER AS end + ON start.id = end.id + WHERE + start.name != 0 AND end.name = 0 + """ + functions = [] + functions_map = {} + unique = EnforceUnique() + for row in conn.execute(marker_query): + unique.see(row["marker_id"]) + evt = FunctionEvent( + id=row["marker_id"], + node_id=0, # missing a node_id when calling FunctionEvent. This is just to ensure + # that pytorch doesn't crash when creating a FunctionEvent() object + name=strings[row["name"]], + start_us=row["start_time"], + end_us=row["end_time"], + thread=0, + ) # TODO: find in sqlite database + functions.append(evt) + functions_map[evt.id] = evt + + # Now, correlate all kernels with FunctionEvents + kernel_query = """ + SELECT + start.id AS marker_id, start.name, start.timestamp, end.timestamp, + runtime._id_ AS runtime_id, runtime.cbid, runtime.start AS runtime_start, runtime.end AS runtime_end, + kernel.start AS kernel_start, kernel.end AS kernel_end, kernel.name AS kernel_name + FROM + CUPTI_ACTIVITY_KIND_MARKER AS start + INNER JOIN CUPTI_ACTIVITY_KIND_MARKER AS end + ON start.id = end.id + INNER JOIN CUPTI_ACTIVITY_KIND_RUNTIME as runtime + ON (start.timestamp < runtime.start AND runtime.end < end.timestamp) + INNER JOIN CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL AS kernel + ON kernel.correlationId = runtime.correlationId + """ + unique = EnforceUnique() + for row in conn.execute(kernel_query): + unique.see(row["marker_id"], row["runtime_id"]) + # 211 is cudaKernelLaunch for cuda >= 9.2 + if row["cbid"] != 211: + raise AssertionError(f"Expected cbid to be 211, but got {row['cbid']}") + evt = functions_map[row["marker_id"]] + evt.append_kernel( + row["kernel_name"], 0, row["kernel_end"] - row["kernel_start"] + ) + + functions.sort(key=lambda evt: evt.time_range.start) + return functions + + +class KinetoStepTracker: + """Provides an abstraction for incrementing the step count globally. + + Previously, we only had one place to mark that a step() has occurred + in the program via pytorch profiler step(). We will now add step hooks + in the Optimizer class https://github.com/pytorch/pytorch/issues/88446 + + - This could mean programs that already call profiler.step() every + iteration can end up double incrementing step count. + - If a model uses multiple optimizers we can also have double or more + counting of the step. + + We fix this by adding a layer of abstraction before calling step() + to the kineto library. The idea is to maintain steps per requester in a dict: + + .. code-block:: + + { + "ProfilerStep": 100, # triggered by profiler step() call + "Optimizer1Step": 100, # Optimizer 1 or 2 are just examples, could be SGD, Adam etc + "Optimizer2Step": 100, + } + + To figure out the global step count just take the max of dict values (100). + + If one of the count increments the max will go up. + + .. code-block:: + + { + "ProfilerStep": 100, + "Optimizer1Step": 101, # Optimizer1 got incremented first say + "Optimizer2Step": 100, + } + + Then global step count is 101 + We only call the kineto step() function when global count increments. + + NOTE: Please do not use the KinetoStepTracker in modules beside the Optimizer + for now. The result could be incorrect increments of the step count. + """ + + _current_step = 0 + _step_dict: dict[str, int] = defaultdict(int) + + @classmethod + def init_step_count(cls, requester: str): + r""" + Initialize for a given requester. + """ + cls._step_dict[requester] = cls._current_step + + @classmethod + def erase_step_count(cls, requester: str) -> bool: + r""" + Remove a given requester. + """ + return cls._step_dict.pop(requester, None) is not None + + @classmethod + def increment_step(cls, requester: str) -> int: + """Increments the step count for the requester. + + Additionally if the max over all step counts has incremented then + trigger the _kineto_step() returns global step count + """ + if requester not in cls._step_dict: + cls.init_step_count(requester) + cls._step_dict[requester] += 1 + + new_step = max(cls._step_dict.values()) + if new_step > cls._current_step: + delta = new_step - cls._current_step + if delta > 1: + warn( + "Profiler step count has increased more than 1 - " + f"current_step = {cls._current_step} step dict = {cls._step_dict}", + stacklevel=2, + ) + for _ in range(delta): + _kineto_step() + cls._current_step = new_step + return cls._current_step + + @classmethod + def current_step(cls) -> int: + r""" + Get the latest step for any requester + """ + return cls._current_step diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/profiler_legacy.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/profiler_legacy.py new file mode 100644 index 0000000000000000000000000000000000000000..5dd26c0881370486ffbaf7896fcffb8e334ddd6b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/profiler_legacy.py @@ -0,0 +1,327 @@ +# mypy: allow-untyped-defs +import itertools +import warnings +from typing_extensions import deprecated + +import torch +import torch.cuda +from torch.autograd import ( + _disable_profiler_legacy, + _enable_profiler_legacy, + DeviceType, + ProfilerConfig, + ProfilerState, +) +from torch.autograd.profiler_util import ( + _filter_name, + _filter_stack_entry, + _rewrite_name, + EventList, + FunctionEvent, + MEMORY_EVENT_NAME, +) + + +__all__ = ["profile"] + + +@deprecated( + "`torch.autograd.profiler_legacy.profile` is deprecated and will be removed in a future release. " + "Please use `torch.profiler` instead.", + category=None, # TODO: change to `FutureWarning` +) +class profile: + """DEPRECATED: use torch.profiler instead.""" + + def __init__( + self, + enabled=True, + *, + use_cuda=False, + record_shapes=False, + with_flops=False, + profile_memory=False, + with_stack=False, + with_modules=False, + ): + self.enabled: bool = enabled + if not self.enabled: + return + self.use_cuda = use_cuda + self.function_events = None + self.entered = False + self.record_shapes = record_shapes + self.with_flops = with_flops + self.record_shapes |= self.with_flops + self.profile_memory = profile_memory + self.with_stack = with_stack + self.with_modules = with_modules + + if self.use_cuda and not torch.cuda.is_available(): + warnings.warn( + "CUDA is not available, disabling CUDA profiling", + stacklevel=2, + ) + self.use_cuda = False + + if self.use_cuda: + self.profiler_kind = ProfilerState.CUDA + else: + self.profiler_kind = ProfilerState.CPU + + def config(self): + return ProfilerConfig( + self.profiler_kind, + self.record_shapes, + self.profile_memory, + self.with_stack, + self.with_flops, + self.with_modules, + # avoid exposing _ExperimentalConfig this in legacy public API + torch._C._profiler._ExperimentalConfig(), + ) + + def __enter__(self): + if not self.enabled: + return + if self.entered: + raise RuntimeError("Profiler context manager is not reentrant") + self.entered = True + self._start_trace() + return self + + def _start_trace(self): + _enable_profiler_legacy(self.config()) + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.enabled: + return + if self.use_cuda: + torch.cuda.synchronize() + + records = _disable_profiler_legacy() + parsed_results = _parse_legacy_records(records) + # pyrefly: ignore [bad-assignment] + self.function_events = EventList( + parsed_results, + use_device="cuda" if self.use_cuda else None, + profile_memory=self.profile_memory, + with_flops=self.with_flops, + ) + # pyrefly: ignore [missing-attribute] + self.function_events._build_tree() + return False + + def __repr__(self): + if self.function_events is None: + return "" + return repr(self.function_events) + + def __str__(self): + if self.function_events is None: + return "" + return str(self.function_events) + + def _check_finish(self): + if self.function_events is None: + raise RuntimeError("Profiler didn't finish running") + + def table( + self, + sort_by=None, + row_limit=100, + max_src_column_width=75, + max_name_column_width=55, + max_shapes_column_width=80, + header=None, + top_level_events_only=False, + ): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.table( + sort_by=sort_by, + row_limit=row_limit, + max_src_column_width=max_src_column_width, + max_name_column_width=max_name_column_width, + max_shapes_column_width=max_shapes_column_width, + header=header, + top_level_events_only=top_level_events_only, + ) + + table.__doc__ = EventList.table.__doc__ + + def export_chrome_trace(self, path): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.export_chrome_trace(path) + + export_chrome_trace.__doc__ = EventList.export_chrome_trace.__doc__ + + def export_stacks(self, path: str, metric: str = "self_cpu_time_total"): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + if not self.with_stack: + raise AssertionError("export_stacks() requires with_stack=True") + return self.function_events.export_stacks(path, metric) + + def key_averages(self, group_by_input_shape=False, group_by_stack_n=0): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.key_averages(group_by_input_shape, group_by_stack_n) + + key_averages.__doc__ = EventList.key_averages.__doc__ + + def total_average(self): + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.total_average() + + total_average.__doc__ = EventList.total_average.__doc__ + + @property + def self_cpu_time_total(self): + """Return CPU time as the sum of self times across all events.""" + self._check_finish() + if self.function_events is None: + raise AssertionError("Expected profiling results") + return self.function_events.self_cpu_time_total + + +def _parse_legacy_records(thread_records): + def _get_record_key(record): + """Return a tuple for correlating start and end records in `_parse_legacy_records`.""" + return (record.handle(), record.node_id()) + + start_record = None + functions = [] + + # '__start_profile' is not guaranteed to be first, so we must find it here + for record in itertools.chain.from_iterable(thread_records): + name = record.name() + if start_record is None and name == "__start_profile": + start_record = record + + if start_record is None or start_record.is_remote(): + raise AssertionError("Expected a valid local start_record") + + for thread_record_list in thread_records: + # accumulated memory allocations per handle + cpu_memory_allocs = {} + cuda_memory_allocs = {} + # ranges per handle + range_starts = {} + + filtered_handles = set() + prev_record = None + for record in thread_record_list: + record_key = _get_record_key(record) + if _filter_name(record.name()) or record_key in filtered_handles: + filtered_handles.add(record_key) + continue + + if record.kind() == "push": + # workaround to reduce double logging from operator + # wrappers and redispatch + if prev_record is not None: + duplicate = ( + prev_record.name() == record.name() + and prev_record.kind() == record.kind() + and prev_record.node_id() == record.node_id() + ) + if duplicate: + filtered_handles.add(record_key) + continue + + range_starts[record_key] = record + cpu_memory_allocs[record_key] = 0 + cuda_memory_allocs[record_key] = 0 + elif record.kind() == "pop": + if record_key not in range_starts: + raise AssertionError( + f"Expected record with key {record_key} to exist in range_starts. " + "This means that the pop event did not have a corresponding push." + ) + + start = range_starts[record_key] + + cpu_memory_usage = cpu_memory_allocs[record_key] + cuda_memory_usage = cuda_memory_allocs[record_key] + is_async = start.is_async() or (start.thread_id() != record.thread_id()) + is_remote_event = record.is_remote() + start_flops = start.flops() + + fe = FunctionEvent( + id=record.handle(), + node_id=record.node_id(), + name=_rewrite_name(name=start.name(), with_wildcard=True), + trace_name=_rewrite_name(name=start.name(), with_wildcard=False), + thread=start.thread_id(), + start_us=start_record.cpu_elapsed_us(start), + end_us=start_record.cpu_elapsed_us(record), + fwd_thread=start.fwd_thread_id(), + input_shapes=start.shapes(), + stack=[ + entry for entry in start.stack() if _filter_stack_entry(entry) + ], + scope=start.scope(), + use_device="cuda" if start.has_cuda() else None, + cpu_memory_usage=cpu_memory_usage, + device_memory_usage=cuda_memory_usage, + is_async=is_async, + is_remote=is_remote_event, + sequence_nr=start.sequence_nr(), + device_type=DeviceType.CPU, + is_legacy=True, + flops=start_flops, + ) + # note: async events have only cpu total time + if not is_async and start.has_cuda(): + duration = start.cuda_elapsed_us(record) + if duration > 0: + fe.append_kernel(start.name(), start.device(), duration) + functions.append(fe) + del range_starts[record_key] + del cpu_memory_allocs[record_key] + del cuda_memory_allocs[record_key] + elif record.kind() == "memory_alloc": + num_open_handles_cpu = len(cpu_memory_allocs) + num_open_handles_cuda = len(cuda_memory_allocs) + if num_open_handles_cpu != num_open_handles_cuda: + raise AssertionError( + f"Expected CPU and CUDA memory allocation handles to match, " + f"but got {num_open_handles_cpu} CPU and {num_open_handles_cuda} CUDA" + ) + for handle in cpu_memory_allocs: + cpu_memory_allocs[handle] += record.cpu_memory_usage() + for handle in cuda_memory_allocs: + cuda_memory_allocs[handle] += record.cuda_memory_usage() + if num_open_handles_cpu == 0: + # output event as a top-level memory event + fe = FunctionEvent( + id=0, + name=MEMORY_EVENT_NAME, + trace_name=None, + thread=0, + start_us=0, + end_us=0, + stack=[], + cpu_memory_usage=record.cpu_memory_usage(), + device_memory_usage=record.cuda_memory_usage(), + is_legacy=True, + ) + functions.append(fe) + prev_record = record + + # Sort functions by start time then by end time ascending. + # This ensures that--in the case of nested events which + # have the same start time (which may happen due to the + # granularity of the given clock tick)--we always show + # the outermost nested call first. This adds stability + # in how FunctionEvents appear + functions.sort(key=lambda evt: [evt.time_range.start, -evt.time_range.end]) + return functions diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/profiler_util.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/profiler_util.py new file mode 100644 index 0000000000000000000000000000000000000000..1db1e15d9ef243e4f261932b74ba6bbf48d2ea0f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/profiler_util.py @@ -0,0 +1,1445 @@ +# mypy: allow-untyped-defs +import bisect +import itertools +import math +from collections import defaultdict, namedtuple +from operator import attrgetter +from typing import Any, Optional +from typing_extensions import deprecated + +import torch +from torch.autograd import DeviceType + + +__all__ = [ + "EventList", + "FormattedTimesMixin", + "Interval", + "Kernel", + "FunctionEvent", + "FunctionEventAvg", + "StringTable", + "MemRecordsAcc", +] + + +class EventList(list): + """A list of profiling events with helper methods for analysis and visualization. + + EventList extends the standard Python list to provide specialized methods for + working with profiling events (FunctionEvent or FunctionEventAvg objects). + It includes utilities for aggregating statistics, formatting output tables, + and exporting profiling data. + + This class is typically returned by profiler methods and should not be + instantiated directly by users. + + Args: + *args: Standard list arguments. + use_device (str, optional): Device type for profiling ("cuda", "xpu", etc.). + profile_memory (bool, optional): Whether memory profiling was enabled. Default: False. + with_flops (bool, optional): Whether to include FLOP counts. Default: False. + + Attributes: + _use_device (str): Device type being profiled. + _profile_memory (bool): Whether memory profiling is enabled. + _with_flops (bool): Whether FLOP counting is enabled. + _tree_built (bool): Whether the event tree structure has been built. + + Key Methods: + table(...): Format events as a table string for display. + export_chrome_trace(path): Export to Chrome tracing format. + export_stacks(path, metric): Export stack traces with metrics. + key_averages(...): Compute averaged statistics grouped by operation name. + total_average(): Compute aggregate totals across all events (sums, not averages). + + Properties: + self_cpu_time_total: Sum of self CPU time across all events. + + Example:: + + import torch + from torch.profiler import profile, ProfilerActivity + + with profile(activities=[ProfilerActivity.CPU]) as prof: + x = torch.randn(100, 100) + y = torch.matmul(x, x) + + # EventList is returned by prof.events() + events = prof.events() + + # Display as formatted table + print( + events.table( + sort_by="cpu_time_total", row_limit=20, top_level_events_only=False + ) + ) + + # Export to Chrome tracing format + events.export_chrome_trace("trace.json") + + # Get averaged statistics + avg_events = events.key_averages() + print(avg_events.table()) + + # Export stack traces + events.export_stacks("stacks.txt", "self_cpu_time_total") + + See Also: + - :class:`FunctionEvent`: Individual profiling event + - :class:`FunctionEventAvg`: Averaged profiling statistics + - :meth:`table`: Format events as a readable table + - :meth:`key_averages`: Aggregate events by operation name + """ + + def __init__(self, *args, **kwargs): + use_device = kwargs.pop("use_device", None) + profile_memory = kwargs.pop("profile_memory", False) + with_flops = kwargs.pop("with_flops", False) + # pyrefly: ignore [not-iterable] + super().__init__(*args, **kwargs) + self._use_device = use_device + self._profile_memory = profile_memory + self._tree_built = False + self._with_flops = with_flops + + def _build_tree(self): + self._populate_cpu_children() + self._remove_dup_nodes() + self._set_backward_stacktraces() + self._tree_built = True + + def __str__(self): + return self.table() + + def _remove_dup_nodes(self): + while True: + to_delete = set() + + for idx in range(len(self)): + if ( + self[idx].cpu_parent is not None + and self[idx].cpu_parent.name == self[idx].name + and len(self[idx].cpu_parent.cpu_children) == 1 + ): + self[idx].cpu_parent.cpu_children = self[idx].cpu_children + self[idx].cpu_parent.kernels = self[idx].kernels # lift kernels up + for ch in self[idx].cpu_children: + ch.cpu_parent = self[idx].cpu_parent + to_delete.add(idx) + if len(to_delete) == 0: + break + + new_evts = [ev for ind, ev in enumerate(self) if ind not in to_delete] + + self.clear() + + self.extend(new_evts) + + def _populate_cpu_children(self): + """Populate child events into each underlying FunctionEvent object. + + One event is a child of another if [s1, e1) is inside [s2, e2). Where + s1 and e1 would be start and end of the child event's interval. And + s2 and e2 start and end of the parent event's interval + + Example: In event list [[0, 10], [1, 3], [3, 4]] would have make [0, 10] + be a parent of two other intervals. + + If for any reason two intervals intersect only partially, this function + will not record a parent child relationship between then. + """ + # Some events can be async (i.e. start and end on different threads), + # since it's generally undefined how to attribute children ranges to + # async ranges, we do not use them when calculating nested ranges and stats + sync_events = [ + evt + for evt in self + if not evt.is_async and evt.device_type == DeviceType.CPU + ] + events = sorted( + sync_events, + key=attrgetter("thread"), + ) + # Group by both thread and node_id, so that events that happen to have + # the same thread_id but are from different nodes aren't incorrectly + # grouped together. + threads = itertools.groupby( + events, key=lambda event: (event.thread, event.node_id) + ) + + # For each thread we keep a stack of current nested parents. + # We maintain the invariant that each interval is a subset of all other + # intervals lower in the stack. + # + # First we sort the intervals by their start time. Then we iterate over them. + # Every time we see a new interval we remove several parents from + # the top until we restore the invariant. Then parent child relationship + # if recorded if the stack is not empty. + # Finally we add new interval to the list + # + # Algorithm has O(N * log(N)) complexity where N is number of + # intervals + for _thread_id, thread_events in threads: + thread_events_ = sorted( + thread_events, + key=lambda event: [event.time_range.start, -event.time_range.end], + ) + current_events: list[FunctionEvent] = [] + for event in thread_events_: + while len(current_events) > 0: + parent = current_events[-1] + if ( + event.time_range.start >= parent.time_range.end + or event.time_range.end > parent.time_range.end + ): + # this can't be a parent + current_events.pop() + else: + parent.append_cpu_child(event) + if event.cpu_parent is not None: + raise AssertionError( + f"There is already a CPU parent event for {event.key}" + ) + event.set_cpu_parent(parent) + break + + current_events.append(event) + + def _set_backward_stacktraces(self): + def bw_parent(evt): + if evt is None: + return None + elif evt.scope == 1: # BACKWARD_FUNCTION + return evt + else: + return bw_parent(evt.cpu_parent) + + fwd_stacks = {} + for evt in self: + if bw_parent(evt) is None and evt.stack is not None: + t = (evt.sequence_nr, evt.thread) + if t not in fwd_stacks: + fwd_stacks[t] = evt.stack + + for evt in self: + p = bw_parent(evt) + if p is not None: + if p.fwd_thread is None: + raise AssertionError( + "Expected fwd_thread to be set for backward parent" + ) + t = (p.sequence_nr, p.fwd_thread) + evt.stack = fwd_stacks.get(t, []) + + @property + def self_cpu_time_total(self): + return sum(event.self_cpu_time_total for event in self) + + def table( + self, + sort_by=None, + row_limit=100, + max_src_column_width=75, + max_name_column_width=55, + max_shapes_column_width=80, + header=None, + top_level_events_only=False, + time_unit=None, + ): + """Print an EventList as a nicely formatted table. + + Args: + sort_by (str, optional): Attribute used to sort entries. By default + they are printed in the same order as they were registered. + Valid keys include: ``cpu_time``, ``cuda_time``, ``xpu_time``, + ``cpu_time_total``, ``cuda_time_total``, ``xpu_time_total``, + ``cpu_memory_usage``, ``cuda_memory_usage``, ``xpu_memory_usage``, + ``self_cpu_memory_usage``, ``self_cuda_memory_usage``, + ``self_xpu_memory_usage``, ``count``. + top_level_events_only(bool, optional): Boolean flag to determine the + selection of events to display. If true, the profiler will only + display events at top level like top-level invocation of python + `lstm`, python `add` or other functions, nested events like low-level + cpu/cuda/xpu ops events are omitted for profiler result readability. + time_unit(str, optional): A time unit to be used for all values in the + table. Valid options are: ``s``, ``ms`` and ``us``. + + Returns: + A string containing the table. + """ + return _build_table( + self, + sort_by=sort_by, + row_limit=row_limit, + max_src_column_width=max_src_column_width, + max_name_column_width=max_name_column_width, + max_shapes_column_width=max_shapes_column_width, + header=header, + profile_memory=self._profile_memory, + with_flops=self._with_flops, + top_level_events_only=top_level_events_only, + time_unit=time_unit, + ) + + def export_chrome_trace(self, path): + """Export an EventList as a Chrome tracing tools file. + + The checkpoint can be later loaded and inspected under ``chrome://tracing`` URL. + + Args: + path (str): Path where the trace will be written. + """ + import os + + device_name = "cuda" if not self._use_device else self._use_device + with open(path, "w") as f: + next_id = 0 + # Use file IO over using json.dump since JSON dumping is very slow and + # this technique is proven to give a 4x speedup. + f.write("[") + for evt in self: + if evt.trace_name is None: + continue + f.write( + '{{"name": "{}", ' + '"ph": "X", ' + '"ts": {}, ' + '"dur": {}, ' + '"tid": {}, ' + '"pid": "CPU functions", ' + '"args": {{}}}}, '.format( + evt.trace_name, + evt.time_range.start, + evt.time_range.elapsed_us(), + evt.thread + if not evt.is_remote + else f'" node_id:{evt.node_id}, thread_id:{evt.thread} "', + ) + ) + for _ in evt.kernels: + # 's' and 'f' draw Flow arrows from + # the CPU launch to the GPU kernel + f.write( + f'{{"name": "{evt.trace_name}", ' + '"ph": "s", ' + f'"ts": {evt.time_range.start}, ' + f'"tid": {evt.thread}, ' + '"pid": "CPU functions", ' + f'"id": {next_id}, ' + f'"cat": "cpu_to_{device_name}", ' + '"args": {}}, ' + ) + # Note: use torch.profiler to get device kernel trace + next_id += 1 + if len(self) > 0: + # remove trailing whitespace and comma + f.seek(f.tell() - 2, os.SEEK_SET) + f.truncate() + f.write("]") + + def supported_export_stacks_metrics(self): + return [ + "self_cpu_time_total", + "self_cuda_time_total", + "self_xpu_time_total", + "self_privateuse1_time_total", + ] + + def export_stacks(self, path: str, metric: str): + if metric not in self.supported_export_stacks_metrics(): + raise ValueError( + "metric should be one of: " + + str(self.supported_export_stacks_metrics()) + ) + translate_table = str.maketrans(" ;\t\n", "____") + with open(path, "w") as f: + for evt in self: + if evt.stack and len(evt.stack) > 0: + metric_value = getattr( + evt, + metric.replace("cuda", "device") + .replace("xpu", "device") + .replace("privateuse1", "device"), + ) + if int(metric_value) > 0: + stack_str = "" + for entry in reversed(evt.stack): + stack_str += entry.translate(translate_table) + stack_str += ";" + stack_str = stack_str[:-1] + " " + str(int(metric_value)) + f.write(stack_str + "\n") + + def key_averages( + self, + group_by_input_shapes=False, + group_by_stack_n=0, + group_by_overload_name=False, + ): + """Averages all function events over their keys. + + Args: + group_by_input_shapes: group entries by + (event name, input shapes) rather than just event name. + This is useful to see which input shapes contribute to the runtime + the most and may help with size-specific optimizations or + choosing the best candidates for quantization (aka fitting a roof line) + + group_by_stack_n: group by top n stack trace entries + + group_by_overload_name: Differentiate operators by their overload name e.g. aten::add.Tensor + and aten::add.out will be aggregated separately + + Returns: + An EventList containing FunctionEventAvg objects. + """ + if not self._tree_built: + raise AssertionError( + "Expected tree to be built before calling key_averages" + ) + stats: dict[tuple[str, ...], FunctionEventAvg] = defaultdict(FunctionEventAvg) + + def get_key( + event, group_by_input_shapes, group_by_stack_n, group_by_overload_name + ) -> tuple[str, ...]: + key = [ + str(event.key), + str(event.node_id), + str(event.device_type), + str(event.is_legacy), + str(event.is_user_annotation), + ] + if group_by_overload_name: + key.append(evt.overload_name) + if group_by_input_shapes: + key.append(str(event.input_shapes)) + if group_by_stack_n > 0: + key += event.stack[:group_by_stack_n] + return tuple(key) + + for evt in self: + stats[ + get_key( + evt, group_by_input_shapes, group_by_stack_n, group_by_overload_name + ) + ].add(evt) + + avg_list = EventList( + stats.values(), + use_device=self._use_device, + profile_memory=self._profile_memory, + with_flops=self._with_flops, + ) + for evt in avg_list: + evt.stack = evt.stack[:group_by_stack_n] + if not group_by_input_shapes: + evt.input_shapes = "" + if not group_by_overload_name: + evt.overload_name = "" + return avg_list + + def total_average(self): + """Compute aggregate statistics across all events. + + Accumulates statistics from all events into a single FunctionEventAvg object. + This is primarily useful for computing total metrics (total CPU time, total + memory usage, etc.) across the entire profiling session, regardless of + operation type. + + Note: + This sums up times and counts across ALL different operations, so the + "average" metrics (like cpu_time) represent the average time per operation + call across the entire session, mixing all operation types together. + For per-operation averages, use :meth:`key_averages` instead. + + Returns: + FunctionEventAvg: A single aggregate object with key="Total" containing + accumulated statistics. + + """ + total_stat = FunctionEventAvg() + for evt in self: + total_stat += evt + total_stat.key = None + total_stat.key = "Total" + return total_stat + + +def _format_time(time_us): + """Define how to format time in FunctionEvent.""" + US_IN_SECOND = 1000.0 * 1000.0 + US_IN_MS = 1000.0 + if time_us >= US_IN_SECOND: + return f"{time_us / US_IN_SECOND:.3f}s" + if time_us >= US_IN_MS: + return f"{time_us / US_IN_MS:.3f}ms" + return f"{time_us:.3f}us" + + +def _format_time_share(time_us, total_time_us): + """Define how to format time in FunctionEvent.""" + if total_time_us == 0: + if time_us != 0: + raise AssertionError(f"Expected time_us == 0 but got {time_us}") + return "NaN" + return f"{time_us * 100.0 / total_time_us:.2f}%" + + +def _format_memory(nbytes): + """Return a formatted memory size string.""" + KB = 1024 + MB = 1024 * KB + GB = 1024 * MB + if abs(nbytes) >= GB: + return f"{nbytes * 1.0 / GB:.2f} GB" + elif abs(nbytes) >= MB: + return f"{nbytes * 1.0 / MB:.2f} MB" + elif abs(nbytes) >= KB: + return f"{nbytes * 1.0 / KB:.2f} KB" + else: + return str(nbytes) + " B" + + +def _attr_formatter(name): + return property(lambda self: _format_time(getattr(self, name))) + + +class FormattedTimesMixin: + """Helpers for FunctionEvent and FunctionEventAvg. + + The subclass should define `*_time_total` and `count` attributes. + """ + + cpu_time_str = _attr_formatter("cpu_time") + device_time_str = _attr_formatter("device_time") + cpu_time_total_str = _attr_formatter("cpu_time_total") + device_time_total_str = _attr_formatter("device_time_total") + self_cpu_time_total_str = _attr_formatter("self_cpu_time_total") + self_device_time_total_str = _attr_formatter("self_device_time_total") + + @property + def cpu_time(self): + return 0.0 if self.count == 0 else 1.0 * self.cpu_time_total / self.count # type: ignore[attr-defined] + + @property + def device_time(self): + return 0.0 if self.count == 0 else 1.0 * self.device_time_total / self.count # type: ignore[attr-defined] + + @property + @deprecated( + "`cuda_time` is deprecated, please use `device_time` instead.", + category=FutureWarning, + ) + def cuda_time(self): # To be deprecated + return self.device_time + + +class Interval: + def __init__(self, start, end): + self.start = start + self.end = end + + def elapsed_us(self): + r""" + Returns the length of the interval + """ + return self.end - self.start + + +Kernel = namedtuple("Kernel", ["name", "device", "duration"]) + + +class FunctionEvent(FormattedTimesMixin): + """Profiling information about a single function. + + FunctionEvent records the execution of a single operation during profiling. + These events are obtained from the profiler/kineto and contain detailed + timing and memory usage information. + + .. note:: + FunctionEvent objects are typically created by the profiler/kineto and should not + be instantiated directly by users. Access them through the profiler's output. + + Attributes: + id (int): Unique identifier for this event. + node_id (int): Node identifier for distributed profiling (-1 if not applicable). + name (str): Name of the profiled function/operator. + overload_name (str): Overload name for the operator (requires _ExperimentalConfig(capture_overload_names=True) set). + trace_name (str): Same as name, just changes ProfilerStep* to ProfilerStep# + time_range (Interval): Time interval containing start and end timestamps in microseconds. + thread (int): Thread ID where the operation started. + fwd_thread (int): Thread ID of the corresponding forward operation. + kernels (List[Kernel]): List of device kernels launched by this operation. + count (int): Number of times this event was called (usually 1). + cpu_children (List[FunctionEvent]): Direct CPU child operations. + cpu_parent (FunctionEvent): Direct CPU parent operation. + input_shapes (Tuple[int, ...]): Shapes of input tensors (requires record_shapes=true). + concrete_inputs (List[Any]): Concrete input values (requires record_shapes=true). + kwinputs (Dict[str, Any]): Keyword arguments (requires record_shapes=true). + stack (List[str]): Python stack trace where the operation was called (requires with_stack=true). + scope (int): at::RecordScope identifier (0=forward, 1=backward, etc.). + use_device (str): Device type being profiled ("cuda", "xpu", etc.). + cpu_memory_usage (int): CPU memory allocated in bytes. + device_memory_usage (int): Device memory allocated in bytes. + is_async (bool): Whether this is an asynchronous operation. + is_remote (bool): Whether this operation occurred on a remote node. + sequence_nr (int): Sequence number for autograd operations. + device_type (DeviceType): Type of device (CPU, CUDA, XPU, PrivateUse1, etc.). + device_index (int): Index of the device (e.g., GPU 0, 1, 2). + device_resource_id (int): Resource ID on the device (ie. stream ID). + is_legacy (bool): Whether this is from the legacy profiler. + flops (int): Estimated floating point operations. + is_user_annotation (bool): Whether this is a user-annotated region. + metadata_json (str): Additional metadata in JSON format. + + Properties: + cpu_time_total (float): Total CPU time in microseconds. + device_time_total (float): Total device (CUDA/XPU/etc) time in microseconds. + self_cpu_time_total (float): CPU time excluding child operations. + self_device_time_total (float): Device time excluding child operations. + self_cpu_memory_usage (int): CPU memory usage excluding child operations. + self_device_memory_usage (int): Device memory usage excluding child operations. + cpu_time (float): Average CPU time per call. + device_time (float): Average device time per call. + key (str): Key used for grouping events (usually same as name). + + See Also: + - :class:`torch.profiler.profile`: Context manager for profiling + - :class:`EventList`: List container for FunctionEvent objects with helper methods + - :class:`FunctionEventAvg`: Averaged statistics over multiple FunctionEvent objects + """ + + def __init__( + self, + id, + name, + thread, + start_us, + end_us, + overload_name=None, + fwd_thread=None, + input_shapes=None, + stack=None, + scope=0, + use_device=None, + cpu_memory_usage=0, + device_memory_usage=0, + is_async=False, + is_remote=False, + sequence_nr=-1, + node_id=-1, + device_type=DeviceType.CPU, + device_index=0, + device_resource_id=None, + is_legacy=False, + flops=None, + trace_name=None, + concrete_inputs=None, + kwinputs=None, + is_user_annotation=False, + metadata_json=None, + ): + self.id: int = id + self.node_id: int = node_id + self.name: str = name + # pyrefly: ignore [bad-assignment] + self.overload_name: str = overload_name + # pyrefly: ignore [bad-assignment] + self.trace_name: str = trace_name + self.time_range: Interval = Interval(start_us, end_us) + self.thread: int = thread + self.fwd_thread: Optional[int] = fwd_thread + self.kernels: list[Kernel] = [] + self.count: int = 1 + self.cpu_children: list[FunctionEvent] = [] + self.cpu_parent: Optional[FunctionEvent] = None + # pyrefly: ignore [bad-assignment] + self.input_shapes: tuple[int, ...] = input_shapes + # pyrefly: ignore [bad-assignment] + self.concrete_inputs: list[Any] = concrete_inputs + # pyrefly: ignore [bad-assignment] + self.kwinputs: dict[str, Any] = kwinputs + # pyrefly: ignore [bad-assignment] + self.stack: list = stack + self.scope: int = scope + self.use_device: Optional[str] = use_device + self.cpu_memory_usage: int = cpu_memory_usage + self.device_memory_usage: int = device_memory_usage + self.is_async: bool = is_async + self.is_remote: bool = is_remote + self.sequence_nr: int = sequence_nr + self.device_type: DeviceType = device_type + self.device_index: int = device_index + self.device_resource_id: int = ( + thread if device_resource_id is None else device_resource_id + ) + self.is_legacy: bool = is_legacy + self.flops: Optional[int] = flops + self.is_user_annotation: Optional[bool] = is_user_annotation + self.self_cpu_percent = -1 + self.total_cpu_percent = -1 + self.total_device_percent = -1 + self.metadata_json = metadata_json + + def append_kernel(self, name, device, duration): + if self.device_type != DeviceType.CPU: + raise AssertionError("Expected device_type to be CPU") + self.kernels.append(Kernel(name, device, duration)) + + def append_cpu_child(self, child): + """Append a CPU child of type FunctionEvent. + + One is supposed to append only direct children to the event to have + correct self cpu time being reported. + """ + if self.device_type != DeviceType.CPU: + raise AssertionError("Expected device_type to be CPU") + if not isinstance(child, FunctionEvent): + raise AssertionError("Expected child to be a FunctionEvent") + if child.device_type != DeviceType.CPU: + raise AssertionError("Expected child device_type to be CPU") + self.cpu_children.append(child) + + def set_cpu_parent(self, parent): + """Set the immediate CPU parent of type FunctionEvent. + + One profiling FunctionEvent should have only one CPU parent such that + the child's range interval is completely inside the parent's. We use + this connection to determine the event is from top-level op or not. + """ + if self.device_type != DeviceType.CPU: + raise AssertionError("Expected device_type to be CPU") + if not isinstance(parent, FunctionEvent): + raise AssertionError("Expected parent to be a FunctionEvent") + if parent.device_type != DeviceType.CPU: + raise AssertionError("Expected parent device_type to be CPU") + self.cpu_parent = parent + + # Note: async events don't have children, are not used when computing 'self' + # metrics of other events, have only total cpu time + @property + def self_cpu_memory_usage(self): + if self.is_async or self.device_type != DeviceType.CPU: + return 0 + return self.cpu_memory_usage - sum( + child.cpu_memory_usage for child in self.cpu_children + ) + + @property + def self_device_memory_usage(self): + if self.is_async or self.device_type != DeviceType.CPU: + return 0 + return self.device_memory_usage - sum( + child.device_memory_usage for child in self.cpu_children + ) + + @property + @deprecated( + "`self_cuda_memory_usage` is deprecated. Use `self_device_memory_usage` instead.", + category=FutureWarning, + ) + def self_cuda_memory_usage(self): # To be deprecated + return self.self_device_memory_usage + + @property + def cpu_time_total(self): + if self.device_type == DeviceType.CPU: + return self.time_range.elapsed_us() + else: + return 0 + + @property + def self_cpu_time_total(self): + if self.is_async or self.device_type != DeviceType.CPU: + return 0 + return self.cpu_time_total - sum( + child.cpu_time_total for child in self.cpu_children + ) + + @property + def device_time_total(self): + if self.is_async or not self.use_device: + return 0 + if self.device_type == DeviceType.CPU: + if not self.is_legacy: + # account for the kernels in the children ops + return sum(kinfo.duration for kinfo in self.kernels) + sum( + ch.device_time_total for ch in self.cpu_children + ) + else: + # each legacy cpu events has a single (fake) kernel + return sum(kinfo.duration for kinfo in self.kernels) + else: + if self.device_type not in [ + DeviceType.CUDA, + DeviceType.PrivateUse1, + DeviceType.MTIA, + DeviceType.HPU, + ]: + raise AssertionError( + f"Expected device_type to be CUDA, PrivateUse1, MTIA, or HPU, but got {self.device_type}" + ) + return self.time_range.elapsed_us() + + @property + @deprecated( + "`cuda_time_total` is deprecated. Use `device_time_total` instead.", + category=FutureWarning, + ) + def cuda_time_total(self): # To be deprecated + return self.device_time_total + + @property + def self_device_time_total(self): + if self.is_async or not self.use_device: + return 0 + if self.device_type == DeviceType.CPU: + return self.device_time_total - sum( + child.device_time_total for child in self.cpu_children + ) + else: + if self.device_type not in [ + DeviceType.CUDA, + DeviceType.PrivateUse1, + DeviceType.MTIA, + DeviceType.HPU, + ]: + raise AssertionError( + f"Expected device_type to be CUDA, PrivateUse1, MTIA, or HPU, but got {self.device_type}" + ) + return self.device_time_total + + @property + @deprecated( + "`self_cuda_time_total` is deprecated. Use `self_device_time_total` instead.", + category=FutureWarning, + ) + def self_cuda_time_total(self): # To be deprecated + return self.self_device_time_total + + @property + def key(self): + return self.name + + def __repr__(self): + device_name = self.use_device + device_time = self.device_time_str + device_memory_usage = self.device_memory_usage + return ( + f"" + ) + + +class FunctionEventAvg(FormattedTimesMixin): + """Averaged profiling statistics over multiple FunctionEvent objects. + + FunctionEventAvg aggregates statistics from multiple FunctionEvent objects + with the same key (typically same operation name). This is useful for getting + average performance metrics across multiple invocations of the same operation. + + This class is typically created by calling :meth:`EventList.key_averages()` on + a profiler's event list. + + Attributes: + key (str): Grouping key for the events (typically operation name). + count (int): Total number of events aggregated. + node_id (int): Node identifier for distributed profiling (-1 if not applicable). + is_async (bool): Whether the operations are asynchronous. + is_remote (bool): Whether the operations occurred on a remote node. + use_device (str): Device type being profiled ("cuda", "xpu", etc.). + cpu_time_total (int): Accumulated total CPU time in microseconds. + device_time_total (int): Accumulated total device time in microseconds. + self_cpu_time_total (int): Accumulated self CPU time (excluding children) in microseconds. + self_device_time_total (int): Accumulated self device time (excluding children) in microseconds. + input_shapes (List[List[int]]): Input tensor shapes (requires record_shapes=true). + overload_name (str): Operator overload name (requires _ExperimentalConfig(capture_overload_names=True) set). + stack (List[str]): Python stack trace where the operation was called (requires with_stack=true). + scope (int): at::RecordScope identifier (0=forward, 1=backward, etc.). + cpu_memory_usage (int): Accumulated CPU memory usage in bytes. + device_memory_usage (int): Accumulated device memory usage in bytes. + self_cpu_memory_usage (int): Accumulated self CPU memory usage in bytes. + self_device_memory_usage (int): Accumulated self device memory usage in bytes. + cpu_children (List[FunctionEvent]): CPU child events. + cpu_parent (FunctionEvent): CPU parent event. + device_type (DeviceType): Type of device (CPU, CUDA, XPU, PrivateUse1, etc.). + is_legacy (bool): Whether from legacy profiler. + flops (int): Total floating point operations. + is_user_annotation (bool): Whether this is a user-annotated region. + + Properties: + cpu_time (float): Average CPU time per invocation. + device_time (float): Average device time per invocation. + + See Also: + - :class:`EventList.key_averages`: Method that creates FunctionEventAvg objects + - :class:`FunctionEvent`: Individual profiling event + - :class:`EventList`: Container for profiling events + """ + + def __init__(self) -> None: + self.key: Optional[str] = None + self.count: int = 0 + self.node_id: int = 0 + self.is_async: bool = False + self.is_remote: bool = False + self.use_device: Optional[str] = None + self.cpu_time_total: int = 0 + self.device_time_total: int = 0 + self.self_cpu_time_total: int = 0 + self.self_device_time_total: int = 0 + self.input_shapes: Optional[list[list[int]]] = None + self.overload_name: Optional[str] = None + self.stack: Optional[list] = None + self.scope: Optional[int] = None + self.cpu_memory_usage: int = 0 + self.device_memory_usage: int = 0 + self.self_cpu_memory_usage: int = 0 + self.self_device_memory_usage: int = 0 + self.cpu_children: Optional[list[FunctionEvent]] = None + self.cpu_parent: Optional[FunctionEvent] = None + self.device_type: DeviceType = DeviceType.CPU + self.is_legacy: bool = False + self.flops: int = 0 + + def add(self, other): + if self.key is None: + # First function being recorded as part of FunctionEventAvg, propagate + # fields. + self.key = other.key + self.node_id = other.node_id + self.is_async = other.is_async + self.is_remote = other.is_remote + self.cpu_parent = other.cpu_parent + self.cpu_children = other.cpu_children + + self.overload_name = other.overload_name + self.input_shapes = other.input_shapes + self.stack = other.stack + self.scope = other.scope + self.device_type = other.device_type + self.is_legacy = other.is_legacy + self.use_device = other.use_device + self.is_user_annotation = other.is_user_annotation + + if not isinstance(other, (FunctionEvent, FunctionEventAvg)): + raise AssertionError( + "Expected other to be a FunctionEvent or FunctionEventAvg" + ) + if other.key != self.key: + raise AssertionError( + f"Expected keys to match, but got {other.key} vs {self.key}" + ) + + self.cpu_time_total += other.cpu_time_total + self.device_time_total += other.device_time_total + self.self_cpu_time_total += other.self_cpu_time_total + self.self_device_time_total += other.self_device_time_total + self.cpu_memory_usage += other.cpu_memory_usage + self.device_memory_usage += other.device_memory_usage + self.self_cpu_memory_usage += other.self_cpu_memory_usage + self.self_device_memory_usage += other.self_device_memory_usage + self.count += other.count + if self.flops is None: + # pyrefly: ignore [bad-assignment] + self.flops = other.flops + elif other.flops is not None: + self.flops += other.flops + return self + + def __iadd__(self, other): + return self.add(other) + + def __repr__(self): + device_name = "cuda" if not self.use_device else self.use_device + self_device_time = self.self_device_time_total_str + device_time = self.device_time_str + device_memory = self.device_memory_usage + return ( + f"" + ) + + +class StringTable(defaultdict): + def __missing__(self, key): + # manage cases like 't' (demangled to 'unsigned short') separately, + # for now simply check the length to avoid unexpected results for + # the short sequences + self[key] = torch._C._demangle(key) if len(key) > 1 else key + return self[key] + + +class MemRecordsAcc: + """Acceleration structure for accessing mem_records in interval.""" + + def __init__(self, mem_records): + self._mem_records = mem_records + self._start_nses: list[int] = [] + self._indices: list[int] = [] + if len(mem_records) > 0: + tmp = sorted([(r[0].start_ns(), i) for i, r in enumerate(mem_records)]) + self._start_nses, self._indices = zip(*tmp) # type: ignore[assignment] + + def in_interval(self, start_us, end_us): + r""" + Return all records in the given interval + To maintain backward compatibility, convert us to ns in function + """ + start_idx = bisect.bisect_left(self._start_nses, start_us * 1000) + end_idx = bisect.bisect_right(self._start_nses, end_us * 1000) + for i in range(start_idx, end_idx): + yield self._mem_records[self._indices[i]] + + +def _filter_stack_entry(entry): + filtered_entries = [ + ("autograd/__init__", "_make_grads"), + ("autograd/__init__", "backward"), + ("torch/tensor", "backward"), + ("_internal/common_utils", "prof_callable"), + ("_internal/common_utils", "prof_func_call"), + ("_internal/common_utils", "prof_meth_call"), + ] + return all(not (f[0] in entry and f[1] in entry) for f in filtered_entries) + + +MEMORY_EVENT_NAME = "[memory]" +OUT_OF_MEMORY_EVENT_NAME = "[OutOfMemory]" + + +def _filter_name(name): + # ignoring the following utility ops + filtered_out_names = [ + MEMORY_EVENT_NAME, # used only for the top-level memory events + OUT_OF_MEMORY_EVENT_NAME, + "profiler::_record_function_enter", + "profiler::_record_function_enter_new", + "profiler::_record_function_exit", + "aten::is_leaf", + "aten::output_nr", + "aten::_version", + ] + return name in filtered_out_names + + +# Demangles and optionally rewrites the provided event name, +# with_wildcard - whether to replace certain numbered event names +# with a wildcard name to aggregate them together in the profiler table +# output +def _rewrite_name(name, with_wildcard=False): + string_table = StringTable() + name = string_table[name] + if with_wildcard: + if name.startswith("ProfilerStep#"): + name = "ProfilerStep*" + return name + + +def _build_table( + events, + sort_by=None, + header=None, + row_limit=100, + max_src_column_width=75, + max_name_column_width=55, + max_shapes_column_width=80, + with_flops=False, + profile_memory=False, + top_level_events_only=False, + time_unit=None, +): + """Print a summary of events (which can be a list of FunctionEvent or FunctionEventAvg).""" + if len(events) == 0: + return "" + + has_device_time = any(event.self_device_time_total > 0 for event in events) + has_device_mem = any(event.self_device_memory_usage > 0 for event in events) + use_device = events[0].use_device + # Running on PrivateUse1 device with profiler but not enable + # ProfilerActivity.PrivateUse1 can also catch privateuse1 memory usage. + # Here only need to check has_privateuse1_time if not use_device. + if not use_device and has_device_time: + raise RuntimeError("use_device is None, but there is device performance data.") + + has_input_shapes = any( + (event.input_shapes is not None and len(event.input_shapes) > 0) + for event in events + ) + + has_overload_names = any( + (event.overload_name is not None and len(event.overload_name) > 0) + for event in events + ) + + if sort_by is not None: + events = EventList( + sorted( + events, + key=lambda evt: getattr( + evt, + sort_by.replace("cuda", "device") + .replace("xpu", "device") + .replace("privateuse1", "device"), + ), + reverse=True, + ), + use_device=use_device, + profile_memory=profile_memory, + with_flops=with_flops, + ) + + name_column_width = max(len(evt.key) for evt in events) + 4 + if max_name_column_width is not None: + name_column_width = min(name_column_width, max_name_column_width) + + shapes_column_width = max(len(str(evt.input_shapes)) for evt in events) + 4 + if max_shapes_column_width is not None: + shapes_column_width = min(shapes_column_width, max_shapes_column_width) + + DEFAULT_COLUMN_WIDTH = 12 + flops_column_width = DEFAULT_COLUMN_WIDTH + + src_column_width = None + stacks = [ + evt.stack for evt in events if evt.stack is not None and len(evt.stack) > 0 + ] + has_stack = len(stacks) > 0 + if has_stack: + src_column_width = ( + max(max(len(entry) for entry in stack) for stack in stacks) + 4 + ) + if max_src_column_width is not None: + src_column_width = min(src_column_width, max_src_column_width) + + headers = ["Name"] + if has_overload_names: + headers.append("Overload Name") + headers += [ + "Self CPU %", + "Self CPU", + "CPU total %", + "CPU total", + "CPU time avg", + ] + + device_name = use_device.upper() if use_device is not None else "None" + if has_device_time: + headers.extend( + [ + f"Self {device_name}", + f"Self {device_name} %", + f"{device_name} total", + f"{device_name} time avg", + ] + ) + if profile_memory: + headers.extend( + [ + "CPU Mem", + "Self CPU Mem", + ] + ) + if use_device and has_device_mem: + headers.extend( + [ + f"{device_name} Mem", + f"Self {device_name} Mem", + ] + ) + headers.append("# of Calls") + # Only append Node ID if any event has a valid (>= 0) Node ID + append_node_id = any(evt.node_id != -1 for evt in events) + if append_node_id: + headers.append("Node ID") + + # Have to use a list because nonlocal is Py3 only... + SPACING_SIZE = 2 + row_format_lst = [""] + header_sep_lst = [""] + line_length_lst = [-SPACING_SIZE] + + def add_column(padding, text_dir=">"): + row_format_lst[0] += ( + "{: " + text_dir + str(padding) + "}" + (" " * SPACING_SIZE) + ) + header_sep_lst[0] += "-" * padding + (" " * SPACING_SIZE) + line_length_lst[0] += padding + SPACING_SIZE + + def auto_scale_flops(flops): + flop_headers = [ + "FLOPs", + "KFLOPs", + "MFLOPs", + "GFLOPs", + "TFLOPs", + "PFLOPs", + ] + if flops <= 0: + raise AssertionError(f"Expected flops to be positive, but got {flops}") + # pyrefly: ignore [no-matching-overload] + log_flops = max(0, min(math.log10(flops) / 3, float(len(flop_headers) - 1))) + if not (log_flops >= 0 and log_flops < len(flop_headers)): + raise AssertionError( + f"Expected log_flops to be in range [0, {len(flop_headers)}), but got {log_flops}" + ) + return (pow(10, (math.floor(log_flops) * -3.0)), flop_headers[int(log_flops)]) + + add_column(name_column_width) + if has_overload_names: + add_column(name_column_width) + for _ in headers[1 + has_overload_names :]: + add_column(DEFAULT_COLUMN_WIDTH) + + if has_input_shapes: + headers.append("Input Shapes") + add_column(shapes_column_width) + + if has_stack: + headers.append("Source Location") + add_column(src_column_width, text_dir="<") + + if with_flops: + # Auto-scaling of flops header + raw_flops = [evt.flops for evt in events if evt.flops > 0] + if len(raw_flops) != 0: + (flops_scale, flops_header) = auto_scale_flops(min(raw_flops)) + headers.append(f"Total {flops_header}") + add_column(flops_column_width) + else: + with_flops = False # can't find any valid flops + + row_format = row_format_lst[0] + header_sep = header_sep_lst[0] + line_length = line_length_lst[0] + add_column = None # type: ignore[assignment] + + # Have to use a list because nonlocal is Py3 only... + result = [] + + def append(s): + result.append(s) + result.append("\n") # Yes, newline after the end as well + + sum_self_cpu_time_total = 0 + sum_self_device_time_total = 0 + for evt in events: + sum_self_cpu_time_total += evt.self_cpu_time_total + if evt.device_type == DeviceType.CPU and evt.is_legacy: + # in legacy profiler, kernel info is stored in cpu events + sum_self_device_time_total += evt.self_device_time_total + elif ( + evt.device_type + in [ + DeviceType.CUDA, + DeviceType.PrivateUse1, + DeviceType.MTIA, + ] + and not evt.is_user_annotation + ): + # in kineto profiler, there're events with the correct device type (e.g. CUDA) + sum_self_device_time_total += evt.self_device_time_total + + # Actual printing + if header is not None: + append("=" * line_length) + append(header) + if top_level_events_only: + append("=" * line_length) + append("This report only display top-level ops statistics") + append(header_sep) + append(row_format.format(*headers)) + + append(header_sep) + + def trim_path(path, src_column_width): + if len(path) > src_column_width: + offset = len(path) - src_column_width + path = path[offset:] + if len(path) > 3: + path = "..." + path[3:] + return path + + def override_time_unit(time_us, default_str, time_unit): + US_IN_SECOND = 1000.0 * 1000.0 + US_IN_MS = 1000.0 + if time_unit == "s": + return f"{time_us / US_IN_SECOND:.3f}s" + elif time_unit == "ms": + return f"{time_us / US_IN_MS:.3f}ms" + elif time_unit == "us": + return f"{time_us:.3f}us" + else: + return default_str + + event_limit = 0 + for evt in events: + if event_limit == row_limit: + break + if top_level_events_only and evt.cpu_parent is not None: + continue + else: + event_limit += 1 + name = evt.key + if max_name_column_width is not None and len(name) >= max_name_column_width - 3: + name = name[: (max_name_column_width - 3)] + "..." + + evt.self_cpu_percent = _format_time_share( + evt.self_cpu_time_total, sum_self_cpu_time_total + ) + evt.total_cpu_percent = ( + _format_time_share(evt.cpu_time_total, sum_self_cpu_time_total) + if not evt.is_async + else 0 + ) + + row_values = [name] + if has_overload_names: + overload_name = evt.overload_name + if ( + max_name_column_width is not None + and len(overload_name) >= max_name_column_width - 3 + ): + overload_name = overload_name[: (max_name_column_width - 3)] + "..." + row_values += [overload_name] + row_values += [ + # Self CPU total %, 0 for async events. + evt.self_cpu_percent, + override_time_unit( + evt.self_cpu_time_total, evt.self_cpu_time_total_str, time_unit + ), # Self CPU total + # CPU total %, 0 for async events. + evt.total_cpu_percent, + override_time_unit( + evt.cpu_time_total, evt.cpu_time_total_str, time_unit + ), # CPU total + override_time_unit( + evt.cpu_time, evt.cpu_time_str, time_unit + ), # CPU time avg + ] + if has_device_time: + evt.total_device_percent = _format_time_share( + evt.self_device_time_total, sum_self_device_time_total + ) + row_values.extend( + [ + override_time_unit( + evt.self_device_time_total, + evt.self_device_time_total_str, + time_unit, + ), + # device time total % + evt.total_device_percent, + override_time_unit( + evt.device_time_total, evt.device_time_total_str, time_unit + ), + override_time_unit( + evt.device_time, evt.device_time_str, time_unit + ), # device time avg + ] + ) + if profile_memory: + row_values.extend( + [ + # CPU Mem Total + _format_memory(evt.cpu_memory_usage), + # Self CPU Mem Total + _format_memory(evt.self_cpu_memory_usage), + ] + ) + if use_device and has_device_mem: + row_values.extend( + [ + # Device Mem Total + _format_memory(evt.device_memory_usage), + # Self Device Mem Total + _format_memory(evt.self_device_memory_usage), + ] + ) + row_values.append( + evt.count, # Number of calls + ) + + if append_node_id: + row_values.append(evt.node_id) + if has_input_shapes: + row_values.append(str(evt.input_shapes)[:shapes_column_width]) + if with_flops: + if evt.flops <= 0: + row_values.append("--") + else: + row_values.append(f"{evt.flops * flops_scale:8.3f}") # type: ignore[possibly-undefined] + if has_stack: + src_field = "" + if len(evt.stack) > 0: + src_field = trim_path(evt.stack[0], src_column_width) + row_values.append(src_field) + append(row_format.format(*row_values)) + + if has_stack: + empty_headers = [""] * (len(headers) - 1) + for entry in evt.stack[1:]: + append( + row_format.format( + *(empty_headers + [trim_path(entry, src_column_width)]) + ) + ) + empty_headers.append("") + append(row_format.format(*empty_headers)) + + append(header_sep) + append( + f"Self CPU time total: {override_time_unit(sum_self_cpu_time_total, _format_time(sum_self_cpu_time_total), time_unit)}" + ) + if has_device_time: + append( + f"Self {use_device.upper() if use_device is not None else 'None'} " + f"time total: {override_time_unit(sum_self_device_time_total, _format_time(sum_self_device_time_total), time_unit)}" + ) + return "".join(result) + + +# Collect all events with stack traces and format them canonically +def _canonicalize_profiler_events(events): + """ + Extract and format all events with stack traces in a canonical way + for deterministic testing. + """ + events_with_traces = [] + + for event in events: + # Extract relevant fields + event_name = event.get("name", "") + node_name = event["args"].get("node_name", "") + stack_trace = event["args"].get("stack_trace", "") + + # Get the last non-empty line of the stack trace + lines = [s.strip() for s in stack_trace.split("\n") if s.strip()] + stack_trace = lines[-1] if lines else "" + + events_with_traces.append( + { + "event_name": event_name[:20], + "node_name": node_name, + "stack_trace": stack_trace, + "start_time": event.get("ts", 0), + } + ) + + # Sort by node_name for deterministic ordering + events_with_traces.sort(key=lambda x: x["start_time"]) + + # Format as a string + lines: list[str] = [] + for evt in events_with_traces: + lines.append( + f"event={evt['event_name']} node={evt['node_name']} stack_trace={evt['stack_trace']}" + ) + + return "\n".join(lines) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/variable.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/variable.py new file mode 100644 index 0000000000000000000000000000000000000000..84b504a9c82c7ab855df9ba58d934fa92d936253 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/autograd/variable.py @@ -0,0 +1,15 @@ +# mypy: allow-untyped-defs +import torch +from torch._C import _ImperativeEngine as ImperativeEngine + + +__all__ = ["VariableMeta", "Variable"] + + +class VariableMeta(type): + def __instancecheck__(cls, other): + return isinstance(other, torch.Tensor) + + +class Variable(torch._C._LegacyVariableBase, metaclass=VariableMeta): # type: ignore[misc] + _execution_engine = ImperativeEngine() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/backends/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f54a3fd6820c75f885aa50e6beacd3b2a5a12871 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/backends/__init__.py @@ -0,0 +1,139 @@ +# mypy: allow-untyped-defs +import sys +import types +from contextlib import contextmanager + +import torch + + +# The idea for this parameter is that we forbid bare assignment +# to torch.backends..enabled and friends when running our +# test suite, where it's very easy to forget to undo the change +# later. +__allow_nonbracketed_mutation_flag = True + + +def disable_global_flags(): + global __allow_nonbracketed_mutation_flag + __allow_nonbracketed_mutation_flag = False + + +def flags_frozen(): + return not __allow_nonbracketed_mutation_flag + + +@contextmanager +def __allow_nonbracketed_mutation(): + global __allow_nonbracketed_mutation_flag + old = __allow_nonbracketed_mutation_flag + __allow_nonbracketed_mutation_flag = True + try: + yield + finally: + __allow_nonbracketed_mutation_flag = old + + +class ContextProp: + def __init__(self, getter, setter): + self.getter = getter + self.setter = setter + + def __get__(self, obj, objtype): + return self.getter() + + def __set__(self, obj, val): + if not flags_frozen(): + self.setter(val) + else: + raise RuntimeError( + f"not allowed to set {obj.__name__} flags " + "after disable_global_flags; please use flags() context manager instead" + ) + + +class PropModule(types.ModuleType): + def __init__(self, m, name): + super().__init__(name) + self.m = m + + def __getattr__(self, attr): + return self.m.__getattribute__(attr) + + +class _FP32Precision: + def __init__(self, backend, op): + self.backend = backend + self.op = op + + def __setattr__(self, name, value): + if name == "fp32_precision": + torch._C._set_fp32_precision_setter(self.backend, self.op, value) + elif name in ("backend", "op"): + super().__setattr__(name, value) + else: + raise AttributeError("Unknown attribute " + name) + + def __getattr__(self, name): + if name == "fp32_precision": + return torch._C._get_fp32_precision_getter(self.backend, self.op) + else: + raise AttributeError("Unknown attribute " + name) + + +def set_flags(_fp32_precision="none"): + orig_flags = (torch._C._get_fp32_precision_getter("generic", "all"),) + if _fp32_precision is not None: + torch._C._set_fp32_precision_setter("generic", "all", _fp32_precision) + return orig_flags + + +@contextmanager +def flags(fp32_precision="none"): + with __allow_nonbracketed_mutation(): + orig_flags = set_flags(fp32_precision) + try: + yield + finally: + with __allow_nonbracketed_mutation(): + set_flags(*orig_flags) + + +def _get_fp32_precision_getter(backend, op): + def inner(): + return torch._C._get_fp32_precision_getter(backend, op) + + return inner + + +def _set_fp32_precision_setter(backend, op): + def inner(precision): + return torch._C._set_fp32_precision_setter(backend, op, precision) + + return inner + + +class GenericModule(PropModule): + fp32_precision = ContextProp( + _get_fp32_precision_getter("generic", "all"), + _set_fp32_precision_setter("generic", "all"), + ) + + +sys.modules[__name__] = GenericModule(sys.modules[__name__], __name__) + +from torch.backends import ( + cpu as cpu, + cuda as cuda, + cudnn as cudnn, + cusparselt as cusparselt, + kleidiai as kleidiai, + mha as mha, + miopen as miopen, + mkl as mkl, + mkldnn as mkldnn, + mps as mps, + nnpack as nnpack, + openmp as openmp, + opt_einsum as opt_einsum, + quantized as quantized, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/bin/torch_shm_manager b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/bin/torch_shm_manager new file mode 100644 index 0000000000000000000000000000000000000000..7870cec348094e49fe3341f835b4fda177de810b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/bin/torch_shm_manager differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/contrib/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/contrib/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/contrib/_tensorboard_vis.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/contrib/_tensorboard_vis.py new file mode 100644 index 0000000000000000000000000000000000000000..fd54513d76e390f80d0079e88e399658832d0eb9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/contrib/_tensorboard_vis.py @@ -0,0 +1,152 @@ +# mypy: allow-untyped-defs +import time +from collections import defaultdict +from functools import partial + +import torch + + +# Unfortunately it doesn't seem as if there was any way to get TensorBoard to do +# anything without having TF installed, and so this file has a hard dependency on it +# as well. It really is a debugging tool, so it doesn't matter. +try: + from tensorflow.core.framework import graph_pb2 + from tensorflow.core.util import event_pb2 + from tensorflow.python.summary.writer.writer import FileWriter +except ImportError: + raise ImportError( + "TensorBoard visualization of GraphExecutors requires having " + "TensorFlow installed" + ) from None + + +def dump_tensorboard_summary(graph_executor, logdir): + with FileWriter(logdir) as w: + pb_graph = visualize(graph_executor) + evt = event_pb2.Event( + wall_time=time.time(), graph_def=pb_graph.SerializeToString() + ) + w.add_event(evt) + + +def visualize(graph, name_prefix="", pb_graph=None, executors_it=None): + """Visualizes an independent graph, or a graph executor.""" + value_map = {} + pb_graph = pb_graph or graph_pb2.GraphDef() + + if isinstance(graph, torch._C.GraphExecutorState): + visualize_graph_executor( + graph, name_prefix, pb_graph, partial(visualize, pb_graph=pb_graph) + ) + return pb_graph + + # Set up an input node + pb_graph.node.add(op="input", name=name_prefix + "input") + for i, value in enumerate(graph.param_node().outputs()): + value_map[value.unique()] = name_prefix + "input:" + str(i) + + visualize_rec(graph, value_map, name_prefix, pb_graph, executors_it) + + # Gather all outputs + return_node = pb_graph.node.add(op="output", name=name_prefix + "output") + for value in graph.return_node().inputs(): + return_node.input.append(value_map[value.unique()]) + + return pb_graph + + +def visualize_graph_executor(state, name_prefix, pb_graph, inline_graph): + """Append the state of a given GraphExecutor to the graph protobuf. + + Args: + state (GraphExecutor or GraphExecutorState): GraphExecutor to display. + name_prefix (str): Name prefix of the containing subgraph. + pb_graph (GraphDef): graph to append to. + inline_graph (Callable): a function that handles setting up a value_map, + so that some graphs in here can be inlined. This is necessary, because + this will simply be `visualize` for the top-level GraphExecutor, + or `inline_graph` for all nested ones. + + The signature should look like (Graph, name_prefix) -> (). + It will be called exactly once. + + The strategy is to embed all different configurations as independent subgraphs, + while inlining the original graph as the one that actually produces the values. + """ + if state.autograd_fallback_graph is not None: + visualize( + graph=state.autograd_fallback_graph, + name_prefix=name_prefix + "autograd_fallback/", + pb_graph=pb_graph, + executors_it=iter(state.autograd_fallback.executors()), + ) + + for i, (arg_spec, plan) in enumerate(state.execution_plans.items()): + subgraph_name = name_prefix + f"plan{i}/" + + # Create a disconnected node that will keep information regarding the input + # types of this trace. This is unfortunately a bit too verbose to be included + # in the subgraph name. + input_kinds = pb_graph.node.add(op="INPUT_KIND", name=subgraph_name) + input_kinds.attr["inputs"].s = repr(arg_spec).encode("ascii") + + visualize(plan.graph, subgraph_name, pb_graph, iter(plan.code.executors())) + + # Show gradient as an independent subgraph of this plan + if plan.grad_executor is not None: + grad_subgraph_name = subgraph_name + "grad/" + visualize(plan.grad_executor, grad_subgraph_name, pb_graph) + + return inline_graph(state.graph, name_prefix + "original/") + + +def visualize_rec(graph, value_map, name_prefix, pb_graph, executors_it=None): + """Recursive part of visualize (basically skips setting up the input and output nodes).""" + + def inline_graph(subgraph, name, node): + rec_value_map = { + inp.unique(): value_map[val.unique()] + for inp, val in zip(subgraph.inputs(), node.inputs()) + } + visualize_rec( + graph=subgraph, value_map=rec_value_map, name_prefix=name, pb_graph=pb_graph + ) + for out, val in zip(subgraph.outputs(), node.outputs()): + value_map[val.unique()] = rec_value_map[out.unique()] + + op_id_counter: defaultdict[str, int] = defaultdict(int) + + def name_for(node): + kind = node.kind()[node.kind().index("::") + 2 :] + op_id_counter[kind] += 1 + return kind, name_prefix + kind + "_" + str(op_id_counter[kind]) + + def add_fusion_group(node): + op, name = name_for(node) + inline_graph(node.g("Subgraph"), name + "/", node) + + def add_graph_executor(node): + op, name = name_for(node) + if executors_it is None: + add_node(node) + else: + ge = next(executors_it) + visualize_graph_executor( + ge, name + "/", pb_graph, partial(inline_graph, node=node) + ) + + def add_node(node): + if node.kind() == "prim::FusionGroup": + return add_fusion_group(node) + elif node.kind() == "prim::GraphExecutor": + return add_graph_executor(node) + op, name = name_for(node) + pb_node = pb_graph.node.add(op=op, name=name) + for value in node.inputs(): + pb_node.input.append(value_map[value.unique()]) + # TODO: handle attrs + for i, value in enumerate(node.outputs()): + value_map[value.unique()] = name + ":" + str(i) + + for node in graph.nodes(): + add_node(node) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..23d297b6d95e03064187b191c8849920e65a4ad7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/__init__.py @@ -0,0 +1,1956 @@ +# mypy: allow-untyped-defs +r""" +This package adds support for CUDA tensor types. + +It implements the same function as CPU tensors, but they utilize +GPUs for computation. + +It is lazily initialized, so you can always import it, and use +:func:`is_available()` to determine if your system supports CUDA. + +:ref:`cuda-semantics` has more details about working with CUDA. +""" + +import importlib +import os +import sys +import threading +import traceback +import warnings +from collections.abc import Callable +from functools import lru_cache +from typing import Any, cast, NewType, Optional, TYPE_CHECKING, Union + +import torch +import torch._C +from torch import device as _device +from torch._utils import _dummy_type, _LazySeedTracker, classproperty + +from . import _device_limits, gds +from ._utils import _get_device_index +from .graphs import ( + CUDAGraph, + graph, + graph_pool_handle, + is_current_stream_capturing, + make_graphed_callables, +) +from .green_contexts import GreenContext +from .streams import Event, ExternalStream, Stream + + +if TYPE_CHECKING: + from torch.types import Device + +try: + from torch._C import _cudart # type: ignore[attr-defined] +except ImportError: + _cudart = None + +_initialized = False +_tls = threading.local() +_initialization_lock = threading.Lock() +_queued_calls: list[ + tuple[Callable[[], None], list[str]] +] = [] # don't invoke these until initialization occurs +_is_in_bad_fork = getattr(torch._C, "_cuda_isInBadFork", lambda: False) + +_HAS_PYNVML = False +_PYNVML_ERR = None +try: + from torch import version as _version + + try: + if not _version.hip: + import pynvml # type: ignore[import] + else: + import ctypes + from pathlib import Path + + # In ROCm (at least up through 6.3.2) there're 2 copies of libamd_smi.so: + # - One at lib/libamd_smi.so + # - One at share/amd_smi/amdsmi/libamd_smi.so + # + # The amdsmi python module hardcodes loading the second one in share- + # https://github.com/ROCm/amdsmi/blob/1d305dc9708e87080f64f668402887794cd46584/py-interface/amdsmi_wrapper.py#L174 + # + # See also https://github.com/ROCm/amdsmi/issues/72. + # + # This creates an ODR violation if the copy of libamd_smi.so from lib + # is also loaded (via `ld` linking, `LD_LIBRARY_PATH` or `rpath`). + # + # In order to avoid the violation we hook CDLL and try using the + # already loaded version of amdsmi, or any version in the processes + # rpath/LD_LIBRARY_PATH first, so that we only load a single copy + # of the .so. + class _amdsmi_cdll_hook: + def __init__(self) -> None: + self.original_CDLL = ctypes.CDLL # type: ignore[misc,assignment] + paths = ["libamd_smi.so"] + if rocm_home := os.getenv("ROCM_HOME", os.getenv("ROCM_PATH")): + paths = [os.path.join(rocm_home, "lib/libamd_smi.so")] + paths + self.paths: list[str] = paths + + def hooked_CDLL( + self, name: Union[str, Path, None], *args: Any, **kwargs: Any + ) -> ctypes.CDLL: + if name and Path(name).name == "libamd_smi.so": + for path in self.paths: + try: + return self.original_CDLL(path, *args, **kwargs) + except OSError: + pass + return self.original_CDLL(name, *args, **kwargs) # type: ignore[arg-type] + + def __enter__(self) -> None: + ctypes.CDLL = self.hooked_CDLL # type: ignore[misc,assignment] + + def __exit__(self, type: Any, value: Any, traceback: Any) -> None: + ctypes.CDLL = self.original_CDLL # type: ignore[misc] + + with _amdsmi_cdll_hook(): + import amdsmi # type: ignore[import] + + _HAS_PYNVML = True + except ModuleNotFoundError: + pass + finally: + del _version +except ImportError as err: + _PYNVML_ERR = err # sometimes a lib is installed but the import fails for some other reason, so we log the error for later + +_lazy_seed_tracker = _LazySeedTracker() + +# Define dummy _CudaDeviceProperties type if PyTorch was compiled without CUDA +if hasattr(torch._C, "_CudaDeviceProperties"): + _CudaDeviceProperties = torch._C._CudaDeviceProperties +else: + _CudaDeviceProperties = _dummy_type("_CudaDeviceProperties") # type: ignore[assignment, misc] + +if hasattr(torch._C, "_cuda_exchangeDevice"): + _exchange_device = torch._C._cuda_exchangeDevice +else: + + def _exchange_device(device: int) -> int: + if device < 0: + return -1 + raise RuntimeError("PyTorch was compiled without CUDA support") + + +if hasattr(torch._C, "_cuda_maybeExchangeDevice"): + _maybe_exchange_device = torch._C._cuda_maybeExchangeDevice +else: + + def _maybe_exchange_device(device: int) -> int: + if device < 0: + return -1 + raise RuntimeError("PyTorch was compiled without CUDA support") + + +has_half: bool = True +has_magma: bool = torch._C._has_magma + +default_generators: tuple[torch._C.Generator] = () # type: ignore[assignment] + + +def _is_compiled() -> bool: + r"""Return true if compile with CUDA support.""" + return hasattr(torch._C, "_cuda_getDeviceCount") + + +def _nvml_based_avail() -> bool: + return os.getenv("PYTORCH_NVML_BASED_CUDA_CHECK") == "1" + + +def is_available() -> bool: + r""" + Return a bool indicating if CUDA is currently available. + + .. note:: This function will NOT poison fork if the environment variable + ``PYTORCH_NVML_BASED_CUDA_CHECK=1`` is set. For more details, see + :ref:`multiprocessing-poison-fork-note`. + """ + if not _is_compiled(): + return False + if _nvml_based_avail(): + # The user has set an env variable to request this availability check that attempts to avoid fork poisoning by + # using NVML at the cost of a weaker CUDA availability assessment. Note that if NVML discovery/initialization + # fails, this assessment falls back to the default CUDA Runtime API assessment (`cudaGetDeviceCount`) + return device_count() > 0 + else: + # The default availability inspection never throws and returns 0 if the driver is missing or can't + # be initialized. This uses the CUDA Runtime API `cudaGetDeviceCount` which in turn initializes the CUDA Driver + # API via `cuInit` + return torch._C._cuda_getDeviceCount() > 0 + + +def is_bf16_supported(including_emulation: bool = True): + r"""Return a bool indicating if the current CUDA/ROCm device supports dtype bfloat16.""" + # Check for ROCm, if true return true, no ROCM_VERSION check required, + # since it is supported on AMD GPU archs. + if torch.version.hip: + return True + + # If CUDA is not available, than it does not support bf16 either + if not is_available(): + return False + + device = torch.cuda.current_device() + + # Check for CUDA version and device compute capability. + # This is a fast way to check for it. + cuda_version = torch.version.cuda + if cuda_version is not None and torch.cuda.get_device_properties(device).major >= 8: + return True + + if not including_emulation: + return False + + # Finally try to create a bfloat16 device. + return _check_bf16_tensor_supported(device) + + +@lru_cache(maxsize=16) +def _check_bf16_tensor_supported(device: "Device"): + try: + torch.tensor([1.0], dtype=torch.bfloat16, device=device) + return True + except Exception: + return False + + +def is_tf32_supported() -> bool: + r"""Return a bool indicating if the current CUDA/ROCm device supports dtype tf32.""" + if torch.version.hip: + prop_name = torch.cuda.get_device_properties().gcnArchName + archs = ("gfx94", "gfx95") + for arch in archs: + if arch in prop_name: + return True + return False + + # Otherwise, tf32 is supported on CUDA platforms that natively (i.e. no emulation) + # support bfloat16. + return is_bf16_supported(including_emulation=False) + + +def _sleep(cycles): + torch._C._cuda_sleep(cycles) + + +def _busy_wait_for_flag(): + torch._C._cuda_busy_wait_for_flag() + + +def _clear_flag(): + torch._C._cuda_clear_flag() + + +def _extract_arch_version(arch_string: str) -> int: + """Extracts the architecture string from a CUDA version""" + base = arch_string.split("_", maxsplit=2)[1] + base = base.removesuffix("a").removesuffix("f") + return int(base) + + +def _check_capability(): + incompatible_gpu_warn = """ + Found GPU%d %s which is of cuda capability %d.%d. + Minimum and Maximum cuda capability supported by this version of PyTorch is + (%d.%d) - (%d.%d) + """ + matched_cuda_warn = """ + Please install PyTorch with a following CUDA + configurations: {} following instructions at + https://pytorch.org/get-started/locally/ + """ + + # Binary CUDA_ARCHES SUPPORTED by PyTorch + CUDA_ARCHES_SUPPORTED = { + "12.6": {"min": 50, "max": 90}, + "12.8": {"min": 70, "max": 120}, + "13.0": {"min": 75, "max": 120}, + } + + if ( + torch.version.cuda is not None and torch.cuda.get_arch_list() + ): # on ROCm we don't want this check + for d in range(device_count()): + capability = get_device_capability(d) + major = capability[0] + minor = capability[1] + name = get_device_name(d) + current_arch = major * 10 + minor + min_arch = min( + (_extract_arch_version(arch) for arch in torch.cuda.get_arch_list()), + default=50, + ) + max_arch = max( + (_extract_arch_version(arch) for arch in torch.cuda.get_arch_list()), + default=50, + ) + if current_arch < min_arch or current_arch > max_arch: + warnings.warn( + incompatible_gpu_warn + % ( + d, + name, + major, + minor, + min_arch // 10, + min_arch % 10, + max_arch // 10, + max_arch % 10, + ), + stacklevel=2, + ) + matched_arches = "" + for arch, arch_info in CUDA_ARCHES_SUPPORTED.items(): + if ( + current_arch >= arch_info["min"] + and current_arch <= arch_info["max"] + ): + matched_arches += f" {arch}" + if matched_arches != "": + warnings.warn( + matched_cuda_warn.format(matched_arches), stacklevel=2 + ) + + +def _check_cubins(): + incompatible_device_warn = """ +{} with CUDA capability sm_{} is not compatible with the current PyTorch installation. +The current PyTorch install supports CUDA capabilities {}. +If you want to use the {} GPU with PyTorch, please check the instructions at https://pytorch.org/get-started/locally/ +""" + if torch.version.cuda is None: # on ROCm we don't want this check + return + arch_list = get_arch_list() + if len(arch_list) == 0: + return + supported_sm = [_extract_arch_version(arch) for arch in arch_list if "sm_" in arch] + for idx in range(device_count()): + cap_major, cap_minor = get_device_capability(idx) + # NVIDIA GPU compute architectures are backward compatible within major version + supported = any(sm // 10 == cap_major for sm in supported_sm) + if not supported: + device_name = get_device_name(idx) + capability = cap_major * 10 + cap_minor + warnings.warn( + incompatible_device_warn.format( + device_name, capability, " ".join(arch_list), device_name + ), + stacklevel=2, + ) + + +def is_initialized(): + r"""Return whether PyTorch's CUDA state has been initialized.""" + return _initialized and not _is_in_bad_fork() + + +def _lazy_call(callable, **kwargs): + with _initialization_lock: + if is_initialized(): + callable() + else: + # TODO(torch_deploy): this accesses linecache, which attempts to read the + # file system to get traceback info. Patch linecache or do something + # else here if this ends up being important. + global _lazy_seed_tracker + if kwargs.get("seed_all", False): + _lazy_seed_tracker.queue_seed_all(callable, traceback.format_stack()) + elif kwargs.get("seed", False): + _lazy_seed_tracker.queue_seed(callable, traceback.format_stack()) + else: + # Don't store the actual traceback to avoid memory cycle + _queued_calls.append((callable, traceback.format_stack())) + + +_lazy_call(_check_capability) +_lazy_call(_check_cubins) + + +class DeferredCudaCallError(Exception): + pass + + +AcceleratorError = torch._C.AcceleratorError +OutOfMemoryError = torch._C.OutOfMemoryError + + +def init(): + r"""Initialize PyTorch's CUDA state. + + You may need to call this explicitly if you are interacting with + PyTorch via its C API, as Python bindings for CUDA functionality + will not be available until this initialization takes place. + Ordinary users should not need this, as all of PyTorch's CUDA methods + automatically initialize CUDA state on-demand. + + Does nothing if the CUDA state is already initialized. + """ + _lazy_init() + + +def _lazy_init(): + global _initialized, _queued_calls + if is_initialized() or hasattr(_tls, "is_initializing"): + return + with _initialization_lock: + # We be double-checked locking, boys! This is OK because + # the above test was GIL protected anyway. The inner test + # is for when a thread blocked on some other thread which was + # doing the initialization; when they get the lock, they will + # find there is nothing left to do. + if is_initialized(): + return + # It is important to prevent other threads from entering _lazy_init + # immediately, while we are still guaranteed to have the GIL, because some + # of the C calls we make below will release the GIL + if _is_in_bad_fork(): + raise RuntimeError( + "Cannot re-initialize CUDA in forked subprocess. To use CUDA with " + "multiprocessing, you must use the 'spawn' start method" + ) + if not hasattr(torch._C, "_cuda_getDeviceCount"): + raise AssertionError("Torch not compiled with CUDA enabled") + if _cudart is None: + raise AssertionError( + "libcudart functions unavailable. It looks like you have a broken build?" + ) + # This function throws if there's a driver initialization error, no GPUs + # are found or any other error occurs + torch._C._cuda_init() + # Some of the queued calls may reentrantly call _lazy_init(); + # we need to just return without initializing in that case. + # However, we must not let any *other* threads in! + _tls.is_initializing = True + + _queued_calls.extend(calls for calls in _lazy_seed_tracker.get_calls() if calls) + + try: + for queued_call, orig_traceback in _queued_calls: + try: + queued_call() + except Exception as e: + msg = ( + f"CUDA call failed lazily at initialization with error: {str(e)}\n\n" + f"CUDA call was originally invoked at:\n\n{''.join(orig_traceback)}" + ) + raise DeferredCudaCallError(msg) from e + finally: + delattr(_tls, "is_initializing") + _initialized = True + + +def cudart(): + r"""Retrieves the CUDA runtime API module. + + + This function initializes the CUDA runtime environment if it is not already + initialized and returns the CUDA runtime API module (_cudart). The CUDA + runtime API module provides access to various CUDA runtime functions. + + Args: + ``None`` + + Returns: + module: The CUDA runtime API module (_cudart). + + Raises: + RuntimeError: If CUDA cannot be re-initialized in a forked subprocess. + AssertionError: If PyTorch is not compiled with CUDA support or if libcudart functions are unavailable. + + Example of CUDA operations with profiling: + >>> import torch + >>> from torch.cuda import cudart, check_error + >>> import os + >>> + >>> os.environ["CUDA_PROFILE"] = "1" + >>> + >>> def perform_cuda_operations_with_streams(): + >>> stream = torch.cuda.Stream() + >>> with torch.cuda.stream(stream): + >>> x = torch.randn(100, 100, device='cuda') + >>> y = torch.randn(100, 100, device='cuda') + >>> z = torch.mul(x, y) + >>> return z + >>> + >>> torch.cuda.synchronize() + >>> print("====== Start nsys profiling ======") + >>> check_error(cudart().cudaProfilerStart()) + >>> with torch.autograd.profiler.emit_nvtx(): + >>> result = perform_cuda_operations_with_streams() + >>> print("CUDA operations completed.") + >>> check_error(torch.cuda.cudart().cudaProfilerStop()) + >>> print("====== End nsys profiling ======") + + To run this example and save the profiling information, execute: + >>> $ nvprof --profile-from-start off --csv --print-summary -o trace_name.prof -f -- python cudart_test.py + + This command profiles the CUDA operations in the provided script and saves + the profiling information to a file named `trace_name.prof`. + The `--profile-from-start off` option ensures that profiling starts only + after the `cudaProfilerStart` call in the script. + The `--csv` and `--print-summary` options format the profiling output as a + CSV file and print a summary, respectively. + The `-o` option specifies the output file name, and the `-f` option forces the + overwrite of the output file if it already exists. + """ + _lazy_init() + return _cudart + + +class cudaStatus: + SUCCESS: int = 0 + ERROR_NOT_READY: int = 34 + + +class CudaError(RuntimeError): + def __init__(self, code: int) -> None: + # pyrefly: ignore [missing-attribute] + msg = _cudart.cudaGetErrorString(_cudart.cudaError(code)) + super().__init__(f"{msg} ({code})") + + +def check_error(res: int) -> None: + r"""Raise an error if the result of a CUDA runtime API call is not success.""" + # pyrefly: ignore [missing-attribute] + if res != _cudart.cudaError.success: + raise CudaError(res) + + +class _DeviceGuard: + def __init__(self, index: int): + self.idx = index + self.prev_idx = -1 + + def __enter__(self): + self.prev_idx = torch.cuda._exchange_device(self.idx) + + def __exit__(self, type: Any, value: Any, traceback: Any): + self.idx = torch.cuda._maybe_exchange_device(self.prev_idx) + return False + + +class device: + r"""Context-manager that changes the selected device. + + Args: + device (torch.device or int): device index to select. It's a no-op if + this argument is a negative integer or ``None``. + """ + + def __init__(self, device: Any): + self.idx = _get_device_index(device, optional=True) + self.prev_idx = -1 + + def __enter__(self): + self.prev_idx = torch.cuda._exchange_device(self.idx) + + def __exit__(self, type: Any, value: Any, traceback: Any): + self.idx = torch.cuda._maybe_exchange_device(self.prev_idx) + return False + + +class device_of(device): + r"""Context-manager that changes the current device to that of given object. + + You can use both tensors and storages as arguments. If a given object is + not allocated on a GPU, this is a no-op. + + Args: + obj (Tensor or Storage): object allocated on the selected device. + """ + + def __init__(self, obj): + idx = obj.get_device() if obj.is_cuda else -1 + super().__init__(idx) + + +def set_device(device: "Device") -> None: + r"""Set the current device. + + Usage of this function is discouraged in favor of :any:`device`. In most + cases it's better to use ``CUDA_VISIBLE_DEVICES`` environmental variable. + + Args: + device (torch.device or int): selected device. This function is a no-op + if this argument is negative. + """ + device = _get_device_index(device) + if device >= 0: + torch._C._cuda_setDevice(device) + + +def get_device_name(device: "Device" = None) -> str: + r"""Get the name of a device. + + Args: + device (torch.device or int or str, optional): device for which to return the + name. This function is a no-op if this argument is a negative + integer. It uses the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + Returns: + str: the name of the device + """ + return get_device_properties(device).name + + +def get_device_capability(device: "Device" = None) -> tuple[int, int]: + r"""Get the cuda capability of a device. + + Args: + device (torch.device or int or str, optional): device for which to return the + device capability. This function is a no-op if this argument is + a negative integer. It uses the current device, given by + :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` + (default). + + Returns: + tuple(int, int): the major and minor cuda capability of the device + """ + prop = get_device_properties(device) + return prop.major, prop.minor + + +# pyrefly: ignore [not-a-type] +def get_device_properties(device: "Device" = None) -> _CudaDeviceProperties: + r"""Get the properties of a device. + + Args: + device (torch.device or int or str, optional): device for which to return the + properties of the device. It uses the current device, given by + :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` + (default). + + Returns: + _CudaDeviceProperties: the properties of the device + """ + _lazy_init() # will define _get_device_properties + device = _get_device_index(device, optional=True) + if device < 0 or device >= device_count(): + raise AssertionError("Invalid device id") + return _get_device_properties(device) # type: ignore[name-defined] + + +def can_device_access_peer(device: "Device", peer_device: "Device") -> bool: + r"""Check if peer access between two devices is possible.""" + _lazy_init() + device = _get_device_index(device, optional=True) + peer_device = _get_device_index(peer_device) + if device < 0 or device >= device_count(): + raise AssertionError("Invalid device id") + if peer_device < 0 or peer_device >= device_count(): + raise AssertionError("Invalid peer device id") + return torch._C._cuda_canDeviceAccessPeer(device, peer_device) + + +class StreamContext: + r"""Context-manager that selects a given stream. + + All CUDA kernels queued within its context will be enqueued on a selected + stream. + + Args: + Stream (Stream): selected stream. This manager is a no-op if it's + ``None``. + .. note:: Streams are per-device. + """ + + cur_stream: Optional["torch.cuda.Stream"] + + def __init__(self, stream: Optional["torch.cuda.Stream"]): + self.stream = stream + self.idx = _get_device_index(None, True) + if not torch.jit.is_scripting(): + if self.idx is None: + # pyrefly: ignore [bad-assignment] + self.idx = -1 + + self.src_prev_stream = ( + None if not torch.jit.is_scripting() else torch.cuda.default_stream(None) + ) + self.dst_prev_stream = ( + None if not torch.jit.is_scripting() else torch.cuda.default_stream(None) + ) + + def __enter__(self): + # Local cur_stream variable for type refinement + cur_stream = self.stream + # Return if stream is None or CUDA device not available + if cur_stream is None or self.idx == -1: + return + self.src_prev_stream = torch.cuda.current_stream(None) + + # If the stream is not on the current device, then + # set the current stream on the device + if self.src_prev_stream.device != cur_stream.device: + with device(cur_stream.device): + self.dst_prev_stream = torch.cuda.current_stream(cur_stream.device) + torch.cuda.set_stream(cur_stream) + + def __exit__(self, type: Any, value: Any, traceback: Any): + # Local cur_stream variable for type refinement + cur_stream = self.stream + # If stream is None or no CUDA device available, return + if cur_stream is None or self.idx == -1: + return + + # Reset the stream on the original device + # and destination device + if self.src_prev_stream.device != cur_stream.device: # type: ignore[union-attr] + torch.cuda.set_stream(self.dst_prev_stream) # type: ignore[arg-type] + torch.cuda.set_stream(self.src_prev_stream) # type: ignore[arg-type] + + +def stream(stream: Optional["torch.cuda.Stream"]) -> StreamContext: + r"""Wrap around the Context-manager StreamContext that selects a given stream. + + Arguments: + stream (Stream): selected stream. This manager is a no-op if it's + ``None``. + .. note:: + In eager mode stream is of type Stream class while in JIT it is + an object of the custom class ``torch.classes.cuda.Stream``. + """ + return StreamContext(stream) + + +def _set_stream_by_id(stream_id, device_index, device_type): + r"""set stream specified by the stream id, device index and + device type + + Args: stream_id (int): stream id in stream pool + device_index (int): device index in topo + device_type (int): enum device type + """ + torch._C._cuda_setStream( + stream_id=stream_id, + device_index=device_index, + device_type=device_type, + ) + + +def set_stream(stream: Stream): + r"""Set the current stream.This is a wrapper API to set the stream. + Usage of this function is discouraged in favor of the ``stream`` + context manager. + + Args: + stream (Stream): selected stream. This function is a no-op + if this argument is ``None``. + """ + if stream is None: + return + _set_stream_by_id( + stream_id=stream.stream_id, + device_index=stream.device_index, + device_type=stream.device_type, + ) + + +def _parse_visible_devices() -> Union[list[int], list[str]]: + r"""Parse CUDA_VISIBLE_DEVICES environment variable.""" + var = os.getenv("CUDA_VISIBLE_DEVICES") + + if torch.version.hip: + hip_devices = os.getenv("HIP_VISIBLE_DEVICES") + rocr_devices = os.getenv("ROCR_VISIBLE_DEVICES") + + # You must take care if both HIP and ROCR env vars are set as they have + # different meanings. Both env vars accept either a list of ints or a + # list of UUIDs. The ROCR env var is processed first which then reduces + # the number of GPUs that HIP can select from. + if rocr_devices is not None: + rocr_count = len(rocr_devices.split(",")) + if hip_devices is not None: + # sanity check if both env vars are set + if len(hip_devices.split(",")) > rocr_count: + raise RuntimeError( + "HIP_VISIBLE_DEVICES contains more devices than ROCR_VISIBLE_DEVICES" + ) + # HIP_VISIBLE_DEVICES is preferred over ROCR_VISIBLE_DEVICES + var = hip_devices + else: + return list(range(rocr_count)) + elif hip_devices is not None: + var = hip_devices + + if var is None: + return list(range(64)) + + def _strtoul(s: str) -> int: + """Return -1 or positive integer sequence string starts with.""" + if not s: + return -1 + for idx, c in enumerate(s): + if not (c.isdigit() or (idx == 0 and c in "+-")): + break + if idx + 1 == len(s): + idx += 1 + return int(s[:idx]) if idx > 0 else -1 + + def parse_list_with_prefix(lst: str, prefix: str) -> list[str]: + rcs: list[str] = [] + for elem in lst.split(","): + # Repeated id results in empty set + if elem in rcs: + return cast(list[str], []) + # Anything other but prefix is ignored + if not elem.startswith(prefix): + break + rcs.append(elem) + return rcs + + if var.startswith("GPU-"): + return parse_list_with_prefix(var, "GPU-") + if var.startswith("MIG-"): + return parse_list_with_prefix(var, "MIG-") + # CUDA_VISIBLE_DEVICES uses something like strtoul + # which makes `1gpu2,2ampere` is equivalent to `1,2` + rc: list[int] = [] + for elem in var.split(","): + x = _strtoul(elem.strip()) + # Repeated ordinal results in empty set + if x in rc: + return cast(list[int], []) + # Negative value aborts the sequence + if x < 0: + break + rc.append(x) + return rc + + +def _raw_device_count_amdsmi() -> int: + if not _HAS_PYNVML: # If amdsmi is not available + return -1 + try: + amdsmi.amdsmi_init() + except amdsmi.AmdSmiException as e: + warnings.warn( + f"Can't initialize amdsmi - Error code: {e.err_code}", stacklevel=2 + ) + return -1 + socket_handles = amdsmi.amdsmi_get_processor_handles() + return len(socket_handles) + + +def _raw_device_count_nvml() -> int: + r"""Return number of devices as reported by NVML or negative value if NVML discovery/initialization failed.""" + from ctypes import byref, c_int, CDLL + + nvml_h = CDLL("libnvidia-ml.so.1") + rc = nvml_h.nvmlInit() + if rc != 0: + warnings.warn("Can't initialize NVML", stacklevel=2) + return -1 + dev_count = c_int(-1) + rc = nvml_h.nvmlDeviceGetCount_v2(byref(dev_count)) + if rc != 0: + warnings.warn("Can't get nvml device count", stacklevel=2) + return -1 + del nvml_h + return dev_count.value + + +def _raw_device_uuid_amdsmi() -> Optional[list[str]]: + from ctypes import byref, c_int, c_void_p, CDLL, create_string_buffer + + if not _HAS_PYNVML: # If amdsmi is not available + return None + try: + amdsmi.amdsmi_init() + except amdsmi.AmdSmiException: + warnings.warn("Can't initialize amdsmi", stacklevel=2) + return None + try: + socket_handles = amdsmi.amdsmi_get_processor_handles() + dev_count = len(socket_handles) + except amdsmi.AmdSmiException: + warnings.warn("Can't get amdsmi device count", stacklevel=2) + return None + uuids: list[str] = [] + for idx in range(dev_count): + try: + handler = amdsmi.amdsmi_get_processor_handles()[idx] + except amdsmi.AmdSmiException: + warnings.warn("Cannot get amd device handler", stacklevel=2) + return None + try: + uuid = amdsmi.amdsmi_get_gpu_asic_info(handler)["asic_serial"][ + 2: + ] # Removes 0x prefix from serial + except amdsmi.AmdSmiException: + warnings.warn("Cannot get uuid for amd device", stacklevel=2) + return None + uuids.append( + str(uuid).lower() + ) # Lower-case to match expected HIP_VISIBLE_DEVICES uuid input + return uuids + + +def _raw_device_uuid_nvml() -> Optional[list[str]]: + r"""Return list of device UUID as reported by NVML or None if NVM discovery/initialization failed.""" + from ctypes import byref, c_int, c_void_p, CDLL, create_string_buffer + + nvml_h = CDLL("libnvidia-ml.so.1") + rc = nvml_h.nvmlInit() + if rc != 0: + warnings.warn("Can't initialize NVML", stacklevel=2) + return None + dev_count = c_int(-1) + rc = nvml_h.nvmlDeviceGetCount_v2(byref(dev_count)) + if rc != 0: + warnings.warn("Can't get nvml device count", stacklevel=2) + return None + uuids: list[str] = [] + for idx in range(dev_count.value): + dev_id = c_void_p() + rc = nvml_h.nvmlDeviceGetHandleByIndex_v2(idx, byref(dev_id)) + if rc != 0: + warnings.warn("Can't get device handle", stacklevel=2) + return None + buf_len = 96 + buf = create_string_buffer(buf_len) + rc = nvml_h.nvmlDeviceGetUUID(dev_id, buf, buf_len) + if rc != 0: + warnings.warn("Can't get device UUID", stacklevel=2) + return None + uuids.append(buf.raw.decode("ascii").strip("\0")) + del nvml_h + return uuids + + +def _transform_uuid_to_ordinals(candidates: list[str], uuids: list[str]) -> list[int]: + r"""Given the set of partial uuids and list of known uuids builds a set of ordinals excluding ambiguous partials IDs.""" + + def uuid_to_ordinal(candidate: str, uuids: list[str]) -> int: + best_match = -1 + for idx, uuid in enumerate(uuids): + if not uuid.startswith(candidate): + continue + # Ambiguous candidate + if best_match != -1: + return -1 + best_match = idx + return best_match + + rc: list[int] = [] + for candidate in candidates: + if torch.version.hip: + candidate = candidate.replace( + "GPU-", "", 1 + ) # Remove GPU-prefix to match amdsmi asic serial + idx = uuid_to_ordinal(candidate, uuids) + # First invalid ordinal stops parsing + if idx < 0: + break + # Duplicates result in empty set + if idx in rc: + return cast(list[int], []) + rc.append(idx) + return rc + + +def _device_count_amdsmi() -> int: + visible_devices = _parse_visible_devices() + if not visible_devices: + return 0 + try: + if type(visible_devices[0]) is str: + uuids = _raw_device_uuid_amdsmi() + if uuids is None: + return -1 + # Create string version of visible devices to avoid mypy warnings + visible_device_str = cast(list[str], visible_devices) + visible_devices = _transform_uuid_to_ordinals(visible_device_str, uuids) + else: + raw_cnt = _raw_device_count_amdsmi() + if raw_cnt <= 0: + return raw_cnt + # Trim the list up to a maximum available device + # pyrefly: ignore [bad-argument-type] + for idx, val in enumerate(visible_devices): + # pyrefly: ignore [redundant-cast] + if cast(int, val) >= raw_cnt: + return idx + except OSError: + return -1 + except AttributeError: + return -1 + return len(visible_devices) + + +def _device_count_nvml() -> int: + r"""Return number of devices as reported by NVML taking CUDA_VISIBLE_DEVICES into account. + + Negative value is returned if NVML discovery or initialization has failed. + """ + visible_devices = _parse_visible_devices() + if not visible_devices: + return 0 + try: + if type(visible_devices[0]) is str: + # Skip MIG parsing + if visible_devices[0].startswith("MIG-"): + return -1 + uuids = _raw_device_uuid_nvml() + if uuids is None: + return -1 + visible_devices = _transform_uuid_to_ordinals( + cast(list[str], visible_devices), uuids + ) + else: + raw_cnt = _raw_device_count_nvml() + if raw_cnt <= 0: + return raw_cnt + # Trim the list up to a maximum available device + # pyrefly: ignore [bad-argument-type] + for idx, val in enumerate(visible_devices): + # pyrefly: ignore [redundant-cast] + if cast(int, val) >= raw_cnt: + return idx + except OSError: + return -1 + except AttributeError: + return -1 + return len(visible_devices) + + +def _get_nvml_device_index(device: "Device") -> int: + r"""Return the NVML index of the device, taking CUDA_VISIBLE_DEVICES into account.""" + idx = _get_device_index(device, optional=True) + visible_devices = _parse_visible_devices() + if type(visible_devices[0]) is str: + uuids = _raw_device_uuid_nvml() + if uuids is None: + raise RuntimeError("Can't get device UUIDs") + visible_devices = _transform_uuid_to_ordinals( + cast(list[str], visible_devices), uuids + ) + visible_devices = cast(list[int], visible_devices) + if idx < 0 or idx >= len(visible_devices): + raise RuntimeError( + f"device {idx} is not visible (CUDA_VISIBLE_DEVICES={visible_devices})" + ) + return visible_devices[idx] + + +_cached_device_count: Optional[int] = None + + +def device_count() -> int: + r""" + Return the number of GPUs available. + + .. note:: This API will NOT poison fork if NVML discovery succeeds. + See :ref:`multiprocessing-poison-fork-note` for more details. + """ + global _cached_device_count + if not _is_compiled(): + return 0 + if _cached_device_count is not None: + return _cached_device_count + # bypass _device_count_nvml() if rocm (not supported) + nvml_count = _device_count_amdsmi() if torch.version.hip else _device_count_nvml() + r = torch._C._cuda_getDeviceCount() if nvml_count < 0 else nvml_count + # NB: Do not cache the device count prior to CUDA initialization, because + # the number of devices can change due to changes to CUDA_VISIBLE_DEVICES + # setting prior to CUDA initialization. + if _initialized: + _cached_device_count = r + return r + + +def get_arch_list() -> list[str]: + r"""Return list CUDA architectures this library was compiled for.""" + if not is_available(): + return [] + arch_flags = torch._C._cuda_getArchFlags() + if arch_flags is None: + return [] + return arch_flags.split() + + +def get_gencode_flags() -> str: + r"""Return NVCC gencode flags this library was compiled with.""" + arch_list = get_arch_list() + if len(arch_list) == 0: + return "" + arch_list_ = [arch.split("_") for arch in arch_list] + return " ".join( + [ + f"-gencode compute=compute_{arch},code={kind}_{arch}" + for (kind, arch) in arch_list_ + ] + ) + + +def current_device() -> int: + r"""Return the index of a currently selected device.""" + _lazy_init() + return torch._C._cuda_getDevice() + + +def synchronize(device: "Device" = None) -> None: + r"""Wait for all kernels in all streams on a CUDA device to complete. + + Args: + device (torch.device or int, optional): device for which to synchronize. + It uses the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + """ + _lazy_init() + with torch.cuda.device(device): + return torch._C._cuda_synchronize() + + +def ipc_collect(): + r"""Force collects GPU memory after it has been released by CUDA IPC. + + .. note:: + Checks if any sent CUDA tensors could be cleaned from the memory. Force + closes shared memory file used for reference counting if there is no + active counters. Useful when the producer process stopped actively sending + tensors and want to release unused memory. + """ + _lazy_init() + return torch._C._cuda_ipc_collect() + + +def current_stream(device: "Device" = None) -> Stream: + r"""Return the currently selected :class:`Stream` for a given device. + + Args: + device (torch.device or int, optional): selected device. Returns + the currently selected :class:`Stream` for the current device, given + by :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` + (default). + """ + _lazy_init() + streamdata = torch._C._cuda_getCurrentStream( + _get_device_index(device, optional=True) + ) + return Stream( + stream_id=streamdata[0], device_index=streamdata[1], device_type=streamdata[2] + ) + + +def default_stream(device: "Device" = None) -> Stream: + r"""Return the default :class:`Stream` for a given device. + + Args: + device (torch.device or int, optional): selected device. Returns + the default :class:`Stream` for the current device, given by + :func:`~torch.cuda.current_device`, if :attr:`device` is ``None`` + (default). + """ + _lazy_init() + streamdata = torch._C._cuda_getDefaultStream( + _get_device_index(device, optional=True) + ) + return Stream( + stream_id=streamdata[0], device_index=streamdata[1], device_type=streamdata[2] + ) + + +def get_stream_from_external(data_ptr: int, device: "Device" = None) -> Stream: + r"""Return a :class:`Stream` from an externally allocated CUDA stream. + + This function is used to wrap streams allocated in other libraries in order + to facilitate data exchange and multi-library interactions. + + .. note:: This function doesn't manage the stream life-cycle, it is the user + responsibility to keep the referenced stream alive while this returned + stream is being used. + + Args: + data_ptr(int): Integer representation of the `cudaStream_t` value that + is allocated externally. + device(torch.device or int, optional): the device where the stream + was originally allocated. If device is specified incorrectly, + subsequent launches using this stream may fail. + """ + _lazy_init() + streamdata = torch._C._cuda_getStreamFromExternal( + data_ptr, _get_device_index(device, optional=True) + ) + return Stream( + stream_id=streamdata[0], device_index=streamdata[1], device_type=streamdata[2] + ) + + +def current_blas_handle(): + r"""Return cublasHandle_t pointer to current cuBLAS handle""" + _lazy_init() + return torch._C._cuda_getCurrentBlasHandle() + + +def set_sync_debug_mode(debug_mode: Union[int, str]) -> None: + r"""Set the debug mode for cuda synchronizing operations. + + Args: + debug_mode(str or int): if "default" or 0, don't error or warn on synchronizing operations, + if "warn" or 1, warn on synchronizing operations, if "error" or 2, error out synchronizing operations. + + Warning: + This is an experimental feature, and not all synchronizing operations will trigger warning or error. In + particular, operations in torch.distributed and torch.sparse namespaces are not covered yet. + """ + _lazy_init() + if isinstance(debug_mode, str): + if debug_mode == "default": + debug_mode = 0 + elif debug_mode == "warn": + debug_mode = 1 + elif debug_mode == "error": + debug_mode = 2 + else: + raise RuntimeError( + "invalid value of debug_mode, expected one of `default`, `warn`, `error`" + ) + + torch._C._cuda_set_sync_debug_mode(debug_mode) + + +def get_sync_debug_mode() -> int: + r"""Return current value of debug mode for cuda synchronizing operations.""" + _lazy_init() + return torch._C._cuda_get_sync_debug_mode() + + +def _get_pynvml_handler(device: "Device" = None): + if not _HAS_PYNVML: + raise ModuleNotFoundError( + "nvidia-ml-py does not seem to be installed or it can't be imported." + # pyrefly: ignore [invalid-inheritance] + ) from _PYNVML_ERR + # pyrefly: ignore [import-error,missing-module-attribute] + from pynvml import NVMLError_DriverNotLoaded + + try: + pynvml.nvmlInit() + except NVMLError_DriverNotLoaded as e: + raise RuntimeError("cuda driver can't be loaded, is cuda enabled?") from e + + device = _get_nvml_device_index(device) + handle = pynvml.nvmlDeviceGetHandleByIndex(device) + return handle + + +def _get_amdsmi_handler(device: "Device" = None): + if not _HAS_PYNVML: + raise ModuleNotFoundError( + "amdsmi does not seem to be installed or it can't be imported." + # pyrefly: ignore [invalid-inheritance] + ) from _PYNVML_ERR + try: + amdsmi.amdsmi_init() + except amdsmi.AmdSmiException as e: + raise RuntimeError( + "amdsmi driver can't be loaded, requires >=ROCm5.6 installation" + ) from e + device = _get_amdsmi_device_index(device) + handle = amdsmi.amdsmi_get_processor_handles()[device] + return handle + + +def _get_amdsmi_device_index(device: "Device") -> int: + r"""Return the amdsmi index of the device, taking visible_devices into account.""" + idx = _get_device_index(device, optional=True) + visible_devices = _parse_visible_devices() + if type(visible_devices[0]) is str: + uuids = _raw_device_uuid_amdsmi() + if uuids is None: + raise RuntimeError("Can't get device UUIDs") + visible_devices_str = cast( + list[str], visible_devices + ) # Create str variable for mypy + visible_devices = _transform_uuid_to_ordinals(visible_devices_str, uuids) + idx_map = dict(enumerate(cast(list[int], visible_devices))) + if idx not in idx_map: + raise RuntimeError( + f"device {idx} is not visible (HIP_VISIBLE_DEVICES={visible_devices})" + ) + return idx_map[idx] + + +def _get_amdsmi_device_memory_used(device: "Device" = None) -> int: + handle = _get_amdsmi_handler(device) + # amdsmi_get_gpu_vram_usage returns mem usage in megabytes + mem_mega_bytes = amdsmi.amdsmi_get_gpu_vram_usage(handle)["vram_used"] + mem_bytes = mem_mega_bytes * 1024 * 1024 + return mem_bytes + + +def _get_amdsmi_memory_usage(device: "Device" = None) -> int: + handle = _get_amdsmi_handler(device) + return amdsmi.amdsmi_get_gpu_activity(handle)["umc_activity"] + + +def _get_amdsmi_utilization(device: "Device" = None) -> int: + handle = _get_amdsmi_handler(device) + return amdsmi.amdsmi_get_gpu_activity(handle)["gfx_activity"] + + +def _get_amdsmi_temperature(device: "Device" = None) -> int: + handle = _get_amdsmi_handler(device) + return amdsmi.amdsmi_get_temp_metric( + handle, + amdsmi.AmdSmiTemperatureType.JUNCTION, + amdsmi.AmdSmiTemperatureMetric.CURRENT, + ) + + +def _get_amdsmi_power_draw(device: "Device" = None) -> int: + handle = _get_amdsmi_handler(device) + socket_power = amdsmi.amdsmi_get_power_info(handle)["average_socket_power"] + if socket_power != "N/A": + return socket_power + else: + socket_power = amdsmi.amdsmi_get_power_info(handle)["current_socket_power"] + if socket_power != "N/A": + return socket_power + else: + return 0 + + +def _get_amdsmi_clock_rate(device: "Device" = None) -> int: + handle = _get_amdsmi_handler(device) + clock_info = amdsmi.amdsmi_get_clock_info(handle, amdsmi.AmdSmiClkType.GFX) + if "cur_clk" in clock_info: # ROCm 6.2 deprecation + clock_rate = clock_info["cur_clk"] + else: + clock_rate = clock_info["clk"] + if clock_rate != "N/A": + return clock_rate + else: + return 0 + + +def device_memory_used(device: "Device" = None) -> int: + r"""Return used global (device) memory in bytes as given by `nvidia-smi` or `amd-smi`. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + """ + if not torch.version.hip: + handle = _get_pynvml_handler() + device = _get_nvml_device_index(device) + handle = pynvml.nvmlDeviceGetHandleByIndex(device) + return pynvml.nvmlDeviceGetMemoryInfo(handle).used + else: + return _get_amdsmi_device_memory_used(device) + + +def memory_usage(device: "Device" = None) -> int: + r"""Return the percent of time over the past sample period during which global (device) + memory was being read or written as given by `nvidia-smi`. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + Warning: Each sample period may be between 1 second and 1/6 second, + depending on the product being queried. + """ + if not torch.version.hip: + handle = _get_pynvml_handler() + device = _get_nvml_device_index(device) + handle = pynvml.nvmlDeviceGetHandleByIndex(device) + return pynvml.nvmlDeviceGetUtilizationRates(handle).memory + else: + return _get_amdsmi_memory_usage(device) + + +def utilization(device: "Device" = None) -> int: + r"""Return the percent of time over the past sample period during which one or + more kernels was executing on the GPU as given by `nvidia-smi`. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + Warning: Each sample period may be between 1 second and 1/6 second, + depending on the product being queried. + """ + if not torch.version.hip: + handle = _get_pynvml_handler(device) + device = _get_nvml_device_index(device) + handle = pynvml.nvmlDeviceGetHandleByIndex(device) + return pynvml.nvmlDeviceGetUtilizationRates(handle).gpu + else: + return _get_amdsmi_utilization(device) + + +def temperature(device: "Device" = None) -> int: + r"""Return the average temperature of the GPU sensor in Degrees C (Centigrades). + + The average temperature is computed based on past sample period as given by `nvidia-smi`. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + Warning: Each sample period may be between 1 second and 1/6 second, + depending on the product being queried. + """ + if not torch.version.hip: + handle = _get_pynvml_handler(device) + # 0 refers to the temperature sensor for the GPU die. + return pynvml.nvmlDeviceGetTemperature(handle, 0) + else: + return _get_amdsmi_temperature(device) + + +def power_draw(device: "Device" = None) -> int: + r"""Return the average power draw of the GPU sensor in mW (MilliWatts) + over the past sample period as given by `nvidia-smi` for Fermi or newer fully supported devices. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + Warning: Each sample period may be between 1 second and 1/6 second, + depending on the product being queried. + """ + if not torch.version.hip: + handle = _get_pynvml_handler(device) + return pynvml.nvmlDeviceGetPowerUsage(handle) + else: + return _get_amdsmi_power_draw(device) + + +def clock_rate(device: "Device" = None) -> int: + r"""Return the clock speed of the GPU SM in MHz (megahertz) over the past sample period as given by `nvidia-smi`. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + Warning: Each sample period may be between 1 second and 1/6 second, + depending on the product being queried. + """ + if not torch.version.hip: + handle = _get_pynvml_handler(device) + return pynvml.nvmlDeviceGetClockInfo(handle, 1) + else: + return _get_amdsmi_clock_rate(device) + + +def _get_device(device: Union[int, str, torch.device]) -> torch.device: + r"""Return the torch.device type object from the passed in device. + + Args: + device (torch.device or int): selected device. + """ + if isinstance(device, str): + device = torch.device(device) + elif isinstance(device, int): + device = torch.device("cuda", device) + return device + + +def _get_generator(device: torch.device) -> torch._C.Generator: + r"""Return the CUDA Generator object for the given device. + + Args: + device (torch.device): selected device. + """ + idx = device.index + if idx is None: + idx = current_device() + return torch.cuda.default_generators[idx] + + +def _set_rng_state_offset( + offset: int, device: Union[int, str, torch.device] = "cuda" +) -> None: + r"""Set the random number generator state offset of the specified GPU. + + Args: + offset (int): The desired offset + device (torch.device or int, optional): The device to set the RNG state. + Default: ``'cuda'`` (i.e., ``torch.device('cuda')``, the current CUDA device). + """ + final_device = _get_device(device) + + def cb(): + default_generator = _get_generator(final_device) + default_generator.set_offset(offset) + + _lazy_call(cb) + + +def _get_rng_state_offset(device: Union[int, str, torch.device] = "cuda") -> int: + r"""Return the random number generator state offset of the specified GPU. + + Args: + device (torch.device or int, optional): The device to return the RNG state offset of. + Default: ``'cuda'`` (i.e., ``torch.device('cuda')``, the current CUDA device). + + .. warning:: + This function eagerly initializes CUDA. + """ + _lazy_init() + final_device = _get_device(device) + default_generator = _get_generator(final_device) + return default_generator.get_offset() + + +# pyrefly: ignore [deprecated] +from .memory import * # noqa: F403 +from .random import * # noqa: F403 + + +################################################################################ +# Define Storage and Tensor classes +################################################################################ + + +@staticmethod # type: ignore[misc] +def _lazy_new(cls, *args, **kwargs): + _lazy_init() + # We may need to call lazy init again if we are a forked child + # del _CudaBase.__new__ + return super(_CudaBase, cls).__new__(cls, *args, **kwargs) + + +class _CudaBase: + is_cuda = True + is_sparse = False + + def type(self, *args, **kwargs): + # We could use a Protocol here to tell mypy that self has `get_device` method + # but it is only available in the typing module on Python >= 3.8 + # or on typing_extensions module on Python >= 3.6 + with device(self.get_device()): # type: ignore[attr-defined] + return super().type(*args, **kwargs) # type: ignore[misc] + + __new__ = _lazy_new + + +from torch.storage import _LegacyStorage, _warn_typed_storage_removal + + +class _CudaLegacyStorage(_LegacyStorage): + @classmethod + def from_buffer(cls, *args, **kwargs): + _warn_typed_storage_removal() + raise RuntimeError("from_buffer: Not available for CUDA storage") + + @classmethod + def _new_with_weak_ptr(cls, *args, **kwargs): + raise RuntimeError("_new_with_weak_ptr: Not available for CUDA storage") + + @classmethod + def _new_shared_filename(cls, manager, obj, size, *, device=None, dtype=None): + raise RuntimeError("_new_shared_filename: Not available for CUDA storage") + + +class ByteStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.uint8 + + +class DoubleStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.double + + +class FloatStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.float + + +class HalfStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.half + + +class LongStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.long + + +class IntStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.int + + +class ShortStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.short + + +class CharStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.int8 + + +class BoolStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.bool + + +class BFloat16Storage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.bfloat16 + + +class ComplexDoubleStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.cdouble + + +class ComplexFloatStorage(_CudaLegacyStorage): + @classproperty + def dtype(self): + _warn_typed_storage_removal() + return self._dtype + + @classproperty + def _dtype(self): + return torch.cfloat + + +del _LegacyStorage +del _CudaLegacyStorage + +torch._storage_classes.add(DoubleStorage) +torch._storage_classes.add(FloatStorage) +torch._storage_classes.add(LongStorage) +torch._storage_classes.add(IntStorage) +torch._storage_classes.add(ShortStorage) +torch._storage_classes.add(CharStorage) +torch._storage_classes.add(ByteStorage) +torch._storage_classes.add(HalfStorage) +torch._storage_classes.add(BoolStorage) +torch._storage_classes.add(BFloat16Storage) +torch._storage_classes.add(ComplexDoubleStorage) +torch._storage_classes.add(ComplexFloatStorage) + + +class _WrappedTritonKernel: + """Just a simple wrapper to store some metadata for testing purposes.""" + + def __init__(self, kernel): + self.kernel = kernel + self.kernel_invoked = False + + def __call__(self, *args, **kwargs): + res = self.kernel(*args, **kwargs) + self.kernel_invoked = True + return res + + +def _register_triton_kernels(): + @_WrappedTritonKernel + def kernel_impl(*args, **kwargs): + from torch.sparse._triton_ops import bsr_dense_mm + + # pyrefly: ignore [not-callable] + return bsr_dense_mm(*args, skip_checks=True, **kwargs) + + @_WrappedTritonKernel + def addmm_kernel_impl(*args, **kwargs): + from torch.sparse._triton_ops import bsr_dense_addmm + + return bsr_dense_addmm(*args, skip_checks=True, **kwargs) + + has_triton = importlib.util.find_spec("triton") is not None + if has_triton: + torch._TritonLibrary.registerOp( + "_triton_bsr_dense_mm_out", + "_triton_bsr_dense_mm_out(Tensor bsr, Tensor dense, *, Tensor(a!) out) -> Tensor(a!)", + kernel_impl, + "SparseCsrCUDA", + ) + + torch._TritonLibrary.registerOp( + "_triton_bsr_dense_addmm_out", + ( + "_triton_bsr_dense_addmm_out(Tensor input, Tensor bsr, Tensor dense," + " *, Scalar beta, Scalar alpha, Tensor(a!) out) -> Tensor(a!)" + ), + addmm_kernel_impl, + "SparseCsrCUDA", + ) + + +_lazy_call(_register_triton_kernels) + + +def _compile_kernel( + kernel_source: str, + kernel_name: str, + compute_capability: Optional[str] = None, + cuda_include_dirs: Optional[list] = None, + nvcc_options: Optional[list] = None, +): + """ + Compiles a CUDA kernel using NVRTC and returns a callable function. + + This function is a wrapper for NVRTC that enables runtime compilation of CUDA kernels. + Note that this returns a raw CUDA kernel that operates on raw memory pointers. + To use this kernel as a proper PyTorch operator, you should wrap it following the guide at: + pytorch.org/tutorials/advanced/python_custom_ops.html + + Args: + kernel_source (str): The CUDA kernel source code as a string + kernel_name (str): The name of the kernel function to compile + compute_capability (str, optional): The compute capability to target (e.g., "86"). + If None, will detect from current device. + cuda_include_dirs (list, optional): List of directories containing CUDA headers + nvcc_options (list, optional): Additional options to pass to NVRTC + + Returns: + callable: A Python function that can be called with PyTorch tensor arguments to execute the kernel + + Example: + >>> # xdoctest: +SKIP + >>> kernel_code = ''' + extern "C" + __global__ void add_tensors(const float* a, const float* b, float* c, int n) { + int i = threadIdx.x + blockIdx.x * blockDim.x; + if (i < n) + c[i] = a[i] + b[i]; + } + ''' + >>> add_kernel = torch.cuda.compile_kernel(kernel_code, "add_tensors") + >>> a = torch.randn(1024, device="cuda") + >>> b = torch.randn(1024, device="cuda") + >>> c = torch.empty_like(a) + >>> add_kernel(grid=(4, 1, 1), block=(256, 1, 1), args=[a, b, c, a.numel()]) + """ + from torch.cuda._utils import _cuda_load_module, _nvrtc_compile + + # Compile the kernel to PTX + ptx, mangled_name = _nvrtc_compile( + kernel_source, + kernel_name, + compute_capability, + cuda_include_dirs, + nvcc_options, + ) + + # Load the module and get the kernel + result = _cuda_load_module(ptx, [mangled_name]) + + if isinstance(result, dict): + return result[mangled_name] + else: + # This branch shouldn't be executed if kernel_names is provided, + # but MyPy needs this to understand type narrowing + return getattr(result, mangled_name) + + +from . import amp, jiterator, nvtx, profiler, sparse, tunable + + +_POOL_HANDLE = NewType("_POOL_HANDLE", tuple[int, int]) + + +__all__ = [ + # Typed storage and tensors + "BFloat16Storage", + "BFloat16Tensor", + "BoolStorage", + "BoolTensor", + "ByteStorage", + "ByteTensor", + "CharStorage", + "CharTensor", + "ComplexDoubleStorage", + "ComplexFloatStorage", + "DoubleStorage", + "DoubleTensor", + "FloatStorage", + "FloatTensor", + "HalfStorage", + "HalfTensor", + "IntStorage", + "IntTensor", + "LongStorage", + "LongTensor", + "ShortStorage", + "ShortTensor", + "CUDAGraph", + "CudaError", + "DeferredCudaCallError", + "Event", + "ExternalStream", + "Stream", + "StreamContext", + "GreenContext", + "amp", + "caching_allocator_alloc", + "caching_allocator_delete", + "caching_allocator_enable", + "can_device_access_peer", + "check_error", + "cudaStatus", + "cudart", + "current_blas_handle", + "current_device", + "current_stream", + "default_generators", + "default_stream", + "device", + "device_count", + "device_memory_used", + "device_of", + "empty_cache", + "get_allocator_backend", + "CUDAPluggableAllocator", + "change_current_allocator", + "get_arch_list", + "get_device_capability", + "get_device_name", + "get_device_properties", + "get_gencode_flags", + "get_per_process_memory_fraction", + "get_rng_state", + "get_rng_state_all", + "get_stream_from_external", + "get_sync_debug_mode", + "graph", + "graph_pool_handle", + "graphs", + "has_half", + "has_magma", + "host_memory_stats", + "host_memory_stats_as_nested_dict", + "init", + "initial_seed", + "ipc_collect", + "is_available", + "is_bf16_supported", + "is_current_stream_capturing", + "is_initialized", + "is_tf32_supported", + "jiterator", + "list_gpu_processes", + "make_graphed_callables", + "manual_seed", + "manual_seed_all", + "max_memory_allocated", + "max_memory_cached", + "max_memory_reserved", + "mem_get_info", + "memory", + "memory_allocated", + "memory_cached", + "memory_reserved", + "memory_snapshot", + "memory_stats", + "memory_stats_as_nested_dict", + "memory_summary", + "memory_usage", + "MemPool", + "use_mem_pool", + "temperature", + "power_draw", + "clock_rate", + "nccl", + "nvtx", + "profiler", + "random", + "reset_accumulated_host_memory_stats", + "reset_accumulated_memory_stats", + "reset_max_memory_allocated", + "reset_max_memory_cached", + "reset_peak_host_memory_stats", + "reset_peak_memory_stats", + "seed", + "seed_all", + "set_device", + "set_per_process_memory_fraction", + "set_rng_state", + "set_rng_state_all", + "set_stream", + "set_sync_debug_mode", + "sparse", + "stream", + "streams", + "synchronize", + "tunable", + "utilization", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_device_limits.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_device_limits.py new file mode 100644 index 0000000000000000000000000000000000000000..60aeedc8053abfeb3d6772c6e1408eebe7affbf7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_device_limits.py @@ -0,0 +1,140 @@ +import torch +from torch._C import dtype + + +__all__ = ["GPULimits"] + + +class GPULimits: + r"""Utility class that provides the theoretical limits of Nvidia GPU devices. The + limits don't take into account thermal throttling (assume that the GPU run at its + peak rated frequency). This is because user hardware configuration may influence + power behavior. + """ + + def __init__(self, target_device: torch.device): + # The device properties object is obtained by calling 'cudaGetDeviceProperties' CUDA + # runtime function. We need the total memory bus width and the memory clock rate to + # calculate the memory bandwidth. + self.device_properties = torch.cuda.get_device_properties(target_device) + + # The compute capability is needed to determine the number of FLOPs per cycle per SM + self.compute_capability = int( + f"{self.device_properties.major}{self.device_properties.minor}" + ) + + # FLOPs per cycle information derived from Table 2 in: + # https://resources.nvidia.com/en-us-hopper-architecture/nvidia-h100-tensor-c + + # Returns the number of FMA instructions retired per cycle per SM for a given + # data type, when tensor cores are NOT used + def get_fma_per_cycle_per_sm_cuda_cores(self, data_type: dtype) -> int: + hardcoded_device_values = { + # Ampere Architecture + "fp16_80": 256, + "fp32_80": 64, + "fp64_80": 32, + # Hopper Architecture + "fp16_90": 64, + "fp32_90": 128, + "fp64_90": 64, + # Blackwell Architecture + "fp16_100": 256, + "fp32_100": 128, + "fp64_100": 64, + } + dict_key = "" + if data_type is torch.float16: + dict_key = f"fp16_{self.compute_capability}" + elif data_type is torch.float32: + dict_key = f"fp32_{self.compute_capability}" + elif data_type is torch.float64: + dict_key = f"fp64_{self.compute_capability}" + else: + dict_key = "unknown" + + if dict_key not in hardcoded_device_values: + raise RuntimeError( + f"No data for sm_{self.compute_capability} and {data_type}." + ) + + return hardcoded_device_values[dict_key] + + # Returns the number of FMA instructions retired per cycle per SM for a given + # data type, when tensor cores ARE used + def get_fma_per_cycle_per_sm_tensor_cores(self, data_type: dtype) -> int: + hardcoded_device_values = { + # Ampere Architecture + "int8_80": 2048, + "fp16_80": 1024, + "fp32_80": 512, + "fp64_80": 64, + # Hopper Architecture + "int8_90": 4096, + "fp8_90": 4096, + "fp16_90": 2048, + "fp32_90": 1024, + "fp64_90": 128, + # Blackwell Architecture + "int8_100": 8192, + "fp8_100": 8192, + "fp16_100": 4096, + "fp32_100": 2048, + } + dict_key = "" + if data_type is torch.float16: + dict_key = f"fp16_{self.compute_capability}" + elif data_type is torch.bfloat16: + # FP16 and BF16 are equivalent in terms of FLOPs per cycle per SM + dict_key = f"fp16_{self.compute_capability}" + elif data_type is torch.float32: + dict_key = f"fp32_{self.compute_capability}" + elif data_type is torch.int8: + dict_key = f"int8_{self.compute_capability}" + elif data_type is torch.float64: + dict_key = f"fp64_{self.compute_capability}" + else: + dict_key = "unknown" + + if dict_key not in hardcoded_device_values: + raise RuntimeError( + f"No data for sm_{self.compute_capability} and {data_type}." + ) + + return hardcoded_device_values[dict_key] + + def get_tflops_per_second( + self, data_type: dtype, use_tensor_cores: bool = True + ) -> float: + num_sms = self.device_properties.multi_processor_count + clock_rate = self.device_properties.clock_rate # KHz + + fma_per_cycle = 0 + if use_tensor_cores: + fma_per_cycle = self.get_fma_per_cycle_per_sm_tensor_cores(data_type) + else: + fma_per_cycle = self.get_fma_per_cycle_per_sm_cuda_cores(data_type) + + # 1 FMA counts as 2 floating point operations + # Clock rate is in KHz + tflops_per_second = num_sms * fma_per_cycle * 2 * clock_rate / 1e9 + return tflops_per_second + + def get_memory_bandwidth_Bps(self) -> int: + # DRAM devices are Double-Data which means they provide an output at both fronts of + # a clock beat + bus_bytes_per_cycle = int(2 * self.device_properties.memory_bus_width / 8) + mem_clock_rate_Hz = self.device_properties.memory_clock_rate * 1000 + bytes_per_second = bus_bytes_per_cycle * mem_clock_rate_Hz * 2 + return bytes_per_second + + def get_shared_memory_bandwidth_Bps(self) -> int: + # Each warp can LD or ST 32 x 4 bytes per cycle. To calculate the + # device's throughput we need to multiply with frequency and number of SMs. + num_sms = self.device_properties.multi_processor_count + bytes_per_cycle_per_sm = 128 + bytes_per_cycle_per_device = num_sms * bytes_per_cycle_per_sm + bytes_per_second = ( + bytes_per_cycle_per_device * self.device_properties.clock_rate * 1000 + ) + return bytes_per_second diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_gpu_trace.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_gpu_trace.py new file mode 100644 index 0000000000000000000000000000000000000000..d3b8f7e4626f9bec478a7c74292f78894d62ee6f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_gpu_trace.py @@ -0,0 +1,73 @@ +from collections.abc import Callable + +from torch._utils import CallbackRegistry + + +EventCreationCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA event creation" +) +EventDeletionCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA event deletion" +) +EventRecordCallbacks: "CallbackRegistry[int, int]" = CallbackRegistry( + "CUDA event record" +) +EventWaitCallbacks: "CallbackRegistry[int, int]" = CallbackRegistry("CUDA event wait") +MemoryAllocationCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA memory allocation" +) +MemoryDeallocationCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA memory deallocation" +) +StreamCreationCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA stream creation" +) +DeviceSynchronizationCallbacks: "CallbackRegistry[[]]" = CallbackRegistry( + "CUDA device synchronization" +) +StreamSynchronizationCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA stream synchronization" +) +EventSynchronizationCallbacks: "CallbackRegistry[int]" = CallbackRegistry( + "CUDA event synchronization" +) + + +def register_callback_for_event_creation(cb: Callable[[int], None]) -> None: + EventCreationCallbacks.add_callback(cb) + + +def register_callback_for_event_deletion(cb: Callable[[int], None]) -> None: + EventDeletionCallbacks.add_callback(cb) + + +def register_callback_for_event_record(cb: Callable[[int, int], None]) -> None: + EventRecordCallbacks.add_callback(cb) + + +def register_callback_for_event_wait(cb: Callable[[int, int], None]) -> None: + EventWaitCallbacks.add_callback(cb) + + +def register_callback_for_memory_allocation(cb: Callable[[int], None]) -> None: + MemoryAllocationCallbacks.add_callback(cb) + + +def register_callback_for_memory_deallocation(cb: Callable[[int], None]) -> None: + MemoryDeallocationCallbacks.add_callback(cb) + + +def register_callback_for_stream_creation(cb: Callable[[int], None]) -> None: + StreamCreationCallbacks.add_callback(cb) + + +def register_callback_for_device_synchronization(cb: Callable[[], None]) -> None: + DeviceSynchronizationCallbacks.add_callback(cb) + + +def register_callback_for_stream_synchronization(cb: Callable[[int], None]) -> None: + StreamSynchronizationCallbacks.add_callback(cb) + + +def register_callback_for_event_synchronization(cb: Callable[[int], None]) -> None: + EventSynchronizationCallbacks.add_callback(cb) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_memory_viz.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_memory_viz.py new file mode 100644 index 0000000000000000000000000000000000000000..56da01b202d62cc4968a252482dbb43df984915d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_memory_viz.py @@ -0,0 +1,795 @@ +# mypy: allow-untyped-defs +import base64 +import io +import json +import operator +import os +import pickle +import subprocess +import sys +import warnings +from functools import lru_cache +from itertools import groupby +from typing import Any + + +cache = lru_cache(None) + +__all__ = ["format_flamegraph", "segments", "memory", "compare"] + + +def _frame_fmt(f, full_filename=False): + i = f["line"] + fname = f["filename"] + if not full_filename: + fname = fname.split("/")[-1] + func = f["name"] + return f"{fname}:{i}:{func}" + + +@cache +def _frame_filter(name, filename): + omit_functions = [ + "unwind::unwind", + "CapturedTraceback::gather", + "gather_with_cpp", + "_start", + "__libc_start_main", + "PyEval_", + "PyObject_", + "PyFunction_", + ] + omit_filenames = [ + "core/boxing", + "/Register", + "/Redispatch", + "pythonrun.c", + "Modules/main.c", + "Objects/call.c", + "Objects/methodobject.c", + "pycore_ceval.h", + "ceval.c", + "cpython/abstract.h", + ] + for of in omit_functions: + if of in name: + return False + for of in omit_filenames: + if of in filename: + return False + return True + + +def _frames_fmt(frames, full_filename=False, reverse=False): + if reverse: + frames = reversed(frames) + return [ + _frame_fmt(f, full_filename) + for f in frames + if _frame_filter(f["name"], f["filename"]) + ] + + +def _block_extra_legacy(b): + if "history" in b: + frames = b["history"][0].get("frames", []) + real_size = b["history"][0]["real_size"] + else: + real_size = b.get("requested_size", b["size"]) + frames = [] + return frames, real_size + + +def _block_extra(b): + if "frames" not in b: + # old snapshot format made it more complicated to get frames/allocated size + return _block_extra_legacy(b) + return b["frames"], b["requested_size"] + + +def format_flamegraph(flamegraph_lines, flamegraph_script=None): + if flamegraph_script is None: + cache_dir = os.path.expanduser("~/.cache/") + os.makedirs(cache_dir, exist_ok=True) + flamegraph_script = f"{cache_dir}/flamegraph.pl" + if not os.path.exists(flamegraph_script): + import tempfile + import urllib.request + + print(f"Downloading flamegraph.pl to: {flamegraph_script}") + with tempfile.NamedTemporaryFile(mode="wb", suffix=".pl") as f: + urllib.request.urlretrieve( + "https://raw.githubusercontent.com/brendangregg/FlameGraph/master/flamegraph.pl", + f.name, + ) + try: + os.chmod(f.name, 0o755) + os.rename(f.name, flamegraph_script) + except OSError: # noqa: B001,E722 + # Ok to skip, the file will be removed by tempfile + pass + args = [flamegraph_script, "--countname", "bytes"] + with subprocess.Popen( + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, encoding="utf-8" + ) as p: + assert p.stdin is not None + assert p.stdout is not None + p.stdin.write(flamegraph_lines) + p.stdin.close() + result = p.stdout.read() + p.stdout.close() + p.wait() + assert p.wait() == 0 + return result + + +def _write_blocks(f, prefix, blocks): + def frames_fragment(frames): + if not frames: + return "" + return ";".join(_frames_fmt(frames, reverse=True)) + + for b in blocks: + if "history" not in b: + frames, accounted_for_size = _block_extra(b) + f.write( + f"{prefix};{b['state']};{frames_fragment(frames)} {accounted_for_size}\n" + ) + else: + accounted_for_size = 0 + for h in b["history"]: + sz = h["real_size"] + accounted_for_size += sz + if "frames" in h: + frames = h["frames"] + f.write(f"{prefix};{b['state']};{frames_fragment(frames)} {sz}\n") + else: + f.write(f"{prefix};{b['state']}; {sz}\n") + gaps = b["size"] - accounted_for_size + if gaps: + f.write(f"{prefix};{b['state']}; {gaps}\n") + + +def segments(snapshot, format_flamegraph=format_flamegraph): + f = io.StringIO() + for seg in snapshot["segments"]: + prefix = f"stream_{seg['stream']};seg_{seg['address']}" + _write_blocks(f, prefix, seg["blocks"]) + return format_flamegraph(f.getvalue()) + + +def memory(snapshot, format_flamegraph=format_flamegraph): + f = io.StringIO() + for seg in snapshot["segments"]: + prefix = f"stream_{seg['stream']}" + _write_blocks(f, prefix, seg["blocks"]) + return format_flamegraph(f.getvalue()) + + +def compare(before, after, format_flamegraph=format_flamegraph): + def _seg_key(seg): + return (seg["address"], seg["total_size"]) + + def _seg_info(seg): + return f"stream_{seg['stream']};seg_{seg['address']}" + + f = io.StringIO() + + before_segs = {_seg_key(seg) for seg in before} + after_segs = {_seg_key(seg) for seg in after} + + print(f"only_before = {[a for a, _ in (before_segs - after_segs)]}") + print(f"only_after = {[a for a, _ in (after_segs - before_segs)]}") + + for seg in before: + if _seg_key(seg) not in after_segs: + _write_blocks(f, f"only_before;{_seg_info(seg)}", seg["blocks"]) + + for seg in after: + if _seg_key(seg) not in before_segs: + _write_blocks(f, f"only_after;{_seg_info(seg)}", seg["blocks"]) + + return format_flamegraph(f.getvalue()) + + +def _format_size(num): + # https://stackoverflow.com/questions/1094841/get-human-readable-version-of-file-size + for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: + if abs(num) < 1024.0: + return f"{num:3.1f}{unit}B" + num /= 1024.0 + return f"{num:.1f}YiB" + + +class Bytes: + def __init__(self, value): + self.value = value + + def __add__(self, rhs): + return Bytes(self.value + rhs) + + def __repr__(self): + return _format_size(self.value) + + +def calc_active(seg): + return sum(b["size"] for b in seg["blocks"] if b["state"] == "active_allocated") + + +def _report_free(free_external, free_internal): + total = free_external + free_internal + suffix = "" + if total != 0: + pct = (free_internal / total) * 100 + suffix = f" ({pct:.1f}% internal)" + return f"{Bytes(total)}{suffix}" + + +PAGE_SIZE = 1024 * 1024 * 20 +legend = f"""\ + +Legend: + [a ] - a segment in the allocator + ^-- a page {Bytes(PAGE_SIZE)} of memory in the segment + a-z: pages filled with a single block's content + ' ': page is completely free + *: page if completely full with multiple blocks + 0-9: page is partially full with tensors of multiple blocks (9 == 90% full) + (X% internal) - of the free memory, X% is free because we rounded the size of the allocation. +""" + + +def segsum(data): + r"""Visually reports how the allocator has filled its segments. + + This printout can help debug fragmentation issues since free fragments + will appear as gaps in this printout. The amount of free space is reported + for each segment. + We distinguish between internal free memory which occurs because the + allocator rounds the allocation size, and external free memory, which are + the gaps between allocations in a segment. + Args: + data: snapshot dictionary created from _snapshot() + """ + out = io.StringIO() + out.write(f"Summary of segments >= {Bytes(PAGE_SIZE)} in size\n") + total_reserved = 0 + total_allocated = 0 + free_external = 0 + free_internal = 0 + for seg in sorted( + data["segments"], key=lambda x: (x["total_size"], calc_active(x)) + ): + total_reserved += seg["total_size"] + + seg_free_external = 0 + seg_free_internal = 0 + seg_allocated = 0 + all_ranges = [] + boffset = 0 + for b in seg["blocks"]: + active = b["state"] == "active_allocated" + if active: + _, allocated_size = _block_extra(b) + all_ranges.append((boffset, allocated_size, True)) + seg_allocated += allocated_size + seg_free_internal += b["size"] - allocated_size + else: + seg_free_external += b["size"] + + boffset += b["size"] + + total_allocated += seg_allocated + free_external += seg_free_external + free_internal += seg_free_internal + + nseg = (seg["total_size"] - 1) // PAGE_SIZE + 1 + occupied = [" " for _ in range(nseg)] + frac = [0.0 for _ in range(nseg)] + active_size = 0 + for i, (start_, size, active) in enumerate(all_ranges): + active_size += size + finish_ = start_ + size + start = start_ // PAGE_SIZE + finish = (finish_ - 1) // PAGE_SIZE + 1 + m = chr(ord("a" if active else "A") + (i % 26)) + for j in range(start, finish): + s = max(start_, j * PAGE_SIZE) + e = min(finish_, (j + 1) * PAGE_SIZE) + frac[j] += (e - s) / PAGE_SIZE + if occupied[j] != " ": + occupied[j] = "0123456789*"[int(frac[j] * 10)] + else: + occupied[j] = m + stream = "" if seg["stream"] == 0 else f", stream_{seg['stream']}" + body = "".join(occupied) + assert ( + seg_free_external + seg_free_internal + seg_allocated == seg["total_size"] + ) + stream = f" stream_{seg['stream']}" if seg["stream"] != 0 else "" + if seg["total_size"] >= PAGE_SIZE: + out.write( + f"[{body}] {Bytes(seg['total_size'])} allocated, " + f"{_report_free(seg_free_external, seg_free_internal)} free{stream}\n" + ) + out.write(f"segments: {len(data['segments'])}\n") + out.write(f"total_reserved: {Bytes(total_reserved)}\n") + out.write(f"total_allocated: {Bytes(total_allocated)}\n") + out.write(f"total_free: {_report_free(free_external, free_internal)}\n") + out.write(legend) + assert free_internal + free_external + total_allocated == total_reserved + return out.getvalue() + + +def trace(data): + out = io.StringIO() + + def format(entries): + segment_intervals: list = [] + segment_addr_to_name = {} + allocation_addr_to_name = {} + + free_names: list = [] + next_name = 0 + + def _name(): + nonlocal next_name + if free_names: + return free_names.pop() + r, m = next_name // 26, next_name % 26 + next_name += 1 + return f"{chr(ord('a') + m)}{'' if r == 0 else r}" + + def find_segment(addr): + for name, saddr, size in segment_intervals: + if addr >= saddr and addr < saddr + size: + return name, saddr + for i, seg in enumerate(data["segments"]): + saddr = seg["address"] + size = seg["allocated_size"] + if addr >= saddr and addr < saddr + size: + return f"seg_{i}", saddr + return None, None + + count = 0 + out.write(f"{len(entries)} entries\n") + + total_reserved = 0 + for seg in data["segments"]: + total_reserved += seg["total_size"] + + for count, e in enumerate(entries): + if e["action"] == "alloc": + addr, size = e["addr"], e["size"] + n = _name() + seg_name, seg_addr = find_segment(addr) + if seg_name is None: + seg_name = "MEM" + offset = addr + else: + offset = addr - seg_addr + out.write(f"{n} = {seg_name}[{offset}:{Bytes(size)}]\n") + allocation_addr_to_name[addr] = (n, size, count) + count += size + elif e["action"] == "free_requested": + addr, size = e["addr"], e["size"] + name, _, _ = allocation_addr_to_name.get(addr, (addr, None, None)) + out.write(f"del {name} # {Bytes(size)}\n") + elif e["action"] == "free_completed": + addr, size = e["addr"], e["size"] + count -= size + name, _, _ = allocation_addr_to_name.get(addr, (addr, None, None)) + out.write(f"# free completed for {name} {Bytes(size)}\n") + if name in allocation_addr_to_name: + free_names.append(name) + del allocation_addr_to_name[name] + elif e["action"] == "segment_alloc": + addr, size = e["addr"], e["size"] + name = _name() + out.write(f"{name} = cudaMalloc({addr}, {Bytes(size)})\n") + segment_intervals.append((name, addr, size)) + segment_addr_to_name[addr] = name + elif e["action"] == "segment_free": + addr, size = e["addr"], e["size"] + name = segment_addr_to_name.get(addr, addr) + out.write(f"cudaFree({name}) # {Bytes(size)}\n") + if name in segment_addr_to_name: + free_names.append(name) + del segment_addr_to_name[name] + elif e["action"] == "oom": + size = e["size"] + free = e["device_free"] + out.write( + f"raise OutOfMemoryError # {Bytes(size)} requested, {Bytes(free)} free in CUDA\n" + ) + else: + out.write(f"{e}\n") + out.write(f"TOTAL MEM: {Bytes(count)}") + + for i, d in enumerate(data["device_traces"]): + if d: + out.write(f"Device {i} ----------------\n") + format(d) + return out.getvalue() + + +_memory_viz_template = r""" + + + + + + + +""" + + +def _format_viz(data, viz_kind, device): + if device is not None: + warnings.warn( + "device argument is deprecated, plots now contain all device", + FutureWarning, + stacklevel=3, + ) + buffer = pickle.dumps(data) + buffer += b"\x00" * (3 - len(buffer) % 3) + # Encode the buffer with base64 + encoded_buffer = base64.b64encode(buffer).decode("utf-8") + + json_format = json.dumps([{"name": "snapshot.pickle", "base64": encoded_buffer}]) + return _memory_viz_template.replace("$VIZ_KIND", repr(viz_kind)).replace( + "$SNAPSHOT", json_format + ) + + +def filter_alloc_free_pairs(data): + for dev_id in range(len(data["device_traces"])): + # set of indexes of trace events for alloc-free pairs + filterSet = set() + # map from addr to index of alloc event + allocMap = {} + # set of addrs from free_requested events + freeRequested = set() + for idx, event in enumerate(data["device_traces"][dev_id]): + if event["action"] == "alloc": + allocMap[event["addr"]] = idx + elif event["action"] == "free_requested": + freeRequested.add(event["addr"]) + if allocMap.get(event["addr"]) is not None: + filterSet.add(idx) + filterSet.add(allocMap[event["addr"]]) + allocMap.pop(event["addr"]) + elif event["action"] == "free_completed": + if event["addr"] in freeRequested: + freeRequested.remove(event["addr"]) + filterSet.add(idx) + else: + print(f"free_completed without free_requested: {event}") + + # Remove events whose index is in filterSet + if filterSet: + # Create a new list excluding events with indices in filterSet + data["device_traces"][dev_id] = [ + event + for idx, event in enumerate(data["device_traces"][dev_id]) + if idx not in filterSet + ] + + return data + + +def trace_plot(data, device=None, plot_segments=False, filter_freed=False): + """Generate a visualization over time of the memory usage recorded by the trace as an html file. + + Args: + data: Memory snapshot as generated from torch.cuda.memory._snapshot() + device (torch.device, optional): Generate the trace for this device, needed if multiple devices have allocations. + plot_segments (bool, optional): Plots memory returned from cudaMalloc, rather than individual allocations. + Defaults to False. + filter_freed (bool, optional): Filter out alloc-free paired events to only plot allocations that are not freed yet. + Defaults to False to plot all trace events. + + Returns: + str: HTML of visualization + """ + if filter_freed: + data = filter_alloc_free_pairs(data) + + return _format_viz( + data, + "Active Memory Timeline" + if not plot_segments + else "Active Cached Memory Timeline", + device, + ) + + +def _profile_to_snapshot(profile): + import torch + from torch._C._profiler import _EventType + from torch.profiler._memory_profiler import Action, TensorKey + + memory_profile = profile._memory_profile() + + allocation_stacks = {} + for event in memory_profile._op_tree.sorted_nodes: + if event.tag == _EventType.Allocation: + parent = event.parent + python_parents = [] + while parent: + if parent.tag in (_EventType.PyCall, _EventType.PyCCall): + python_parents.append(parent) + parent = parent.parent + key = TensorKey.from_allocation(event.extra_fields) + + # Corner case: If allocation doesn't have an ID (can't prove it was used as a Tensor) + # key will be None. I should add some way to identify these, I just haven't yet. + if key and event.extra_fields.alloc_size > 0: + allocation_stacks[key] = python_parents + + device_count = torch.cuda.device_count() + snapshot: dict[str, list[Any]] = { + "device_traces": [[] for _ in range(device_count + 1)], + "segments": [ + { + "device": device, + "address": None, + "total_size": 0, + "stream": 0, + "blocks": [], + } + for device in range(device_count + 1) + ], + } + + def to_device(device): + if device.type == "cuda": + return device.index + else: + return device_count + + def allocate(size, tensor_key, version, during_trace=True): + device = to_device(tensor_key.device) + addr = tensor_key.storage.ptr + + seg = snapshot["segments"][device] # type: ignore[index] + if seg["address"] is None or seg["address"] > addr: + seg["address"] = addr + seg["total_size"] = max( + seg["total_size"], addr + size + ) # record max addr for now, we will make it the size later + category = memory_profile._categories.get(tensor_key, version) + category = category.name.lower() if category is not None else "unknown" + stack = allocation_stacks.get(tensor_key, ()) + stack = [{"filename": "none", "line": 0, "name": p.name} for p in stack] + r = { + "action": "alloc", + "addr": addr, + "size": size, + "stream": 0, + "frames": stack, + "category": category, + } + if during_trace: + snapshot["device_traces"][device].append(r) + return r + + def free(alloc, device): + for e in ("free_requested", "free_completed"): + snapshot["device_traces"][device].append( + { + "action": e, + "addr": alloc["addr"], + "size": alloc["size"], + "stream": 0, + "frames": alloc["frames"], + } + ) + + kv_to_elem = {} + + # create the device trace + for _time, action, (tensor_key, version), size in memory_profile.timeline: + if not isinstance(tensor_key, TensorKey): + continue + if action == Action.CREATE: + kv_to_elem[(tensor_key, version)] = allocate(size, tensor_key, version) + elif action == Action.DESTROY: + free(kv_to_elem.pop((tensor_key, version)), to_device(tensor_key.device)) + elif action == Action.INCREMENT_VERSION: + free(kv_to_elem.pop((tensor_key, version)), to_device(tensor_key.device)) + kv_to_elem[(tensor_key, version + 1)] = allocate( + size, tensor_key, version + 1 + ) + elif action == Action.PREEXISTING: + kv_to_elem[(tensor_key, version)] = allocate( + size, tensor_key, version, during_trace=False + ) + + # create the final snapshot state + blocks_at_end = [ + (to_device(tensor_key.device), event["addr"], event["size"], event["frames"]) + for (tensor_key, version), event in kv_to_elem.items() + ] + for device, blocks in groupby(sorted(blocks_at_end), key=operator.itemgetter(0)): + seg = snapshot["segments"][device] # type: ignore[index] + last_addr = seg["address"] + for _, addr, size, frames in blocks: + if last_addr < addr: + seg["blocks"].append({"size": addr - last_addr, "state": "inactive"}) + seg["blocks"].append( + { + "size": size, + "state": "active_allocated", + "requested_size": size, + "frames": frames, + } + ) + last_addr = addr + size + if last_addr < seg["total_size"]: + seg["blocks"].append( + {"size": seg["total_size"] - last_addr, "state": "inactive"} + ) + + snapshot["segments"] = [seg for seg in snapshot["segments"] if seg["blocks"]] # type: ignore[attr-defined] + for seg in snapshot["segments"]: # type: ignore[attr-defined, name-defined, no-redef] + seg["total_size"] -= seg["address"] + if not seg["blocks"]: + seg["blocks"].append({"size": seg["total_size"], "state": "inactive"}) + + return snapshot + + +def profile_plot(profile, device=None): + """Generate a visualization over time of the memory usage recorded by kineto memory profiling as an html file. + + Args: + profile: profile as generated by `torch.profiler.profile(profile_memory=True)` + device (torch.device, optional): Generate the trace for this device, needed if multiple devices have allocations. + + Returns: + str: HTML of visualization + """ + snapshot = _profile_to_snapshot(profile) + return _format_viz(snapshot, "Active Memory Timeline", device) + + +def segment_plot(data: Any, device=None): + return _format_viz(data, "Allocator State History", device) + + +if __name__ == "__main__": + import os.path + + thedir = os.path.realpath(os.path.dirname(__file__)) + if thedir in sys.path: + # otherwise we find cuda/random.py as random... + sys.path.remove(thedir) + import argparse + + fn_name = "torch.cuda.memory._snapshot()" + pickled = f"pickled memory statistics from {fn_name}" + parser = argparse.ArgumentParser( + description=f"Visualize memory dumps produced by {fn_name}" + ) + + subparsers = parser.add_subparsers(dest="action") + + def _output(p): + p.add_argument( + "-o", + "--output", + default="output.svg", + help="flamegraph svg (default: output.svg)", + ) + + description = "Prints overall allocation statistics and a visualization of how the allocators segments are currently filled." + stats_a = subparsers.add_parser("stats", description=description) + stats_a.add_argument("input", help=pickled) + + description = "Prints buffer of the most recent allocation events embedded in the snapshot in a Pythonic style." + trace_a = subparsers.add_parser("trace", description=description) + trace_a.add_argument("input", help=pickled) + + description = "Generate a flamegraph that visualizes what memory is stored in each allocator segment (aka block)" + segments_a = subparsers.add_parser("segments", description=description) + segments_a.add_argument("input", help=pickled) + _output(segments_a) + + description = ( + "Generate a flamegraph the program locations contributing to CUDA memory usage." + ) + memory_a = subparsers.add_parser("memory", description=description) + memory_a.add_argument("input", help=pickled) + _output(memory_a) + + description = ( + "Generate a flamegraph that shows segments (aka blocks) that have been added " + "or removed between two different memorys snapshots." + ) + compare_a = subparsers.add_parser("compare", description=description) + compare_a.add_argument("before", help=pickled) + compare_a.add_argument("after", help=pickled) + _output(compare_a) + + plots = ( + ( + "trace_plot", + "Generate a visualization over time of the memory usage recorded by the trace as an html file.", + ), + ( + "segment_plot", + "Visualize how allocations are packed into allocator segments at each point in a trace as an html file.", + ), + ) + for cmd, description in plots: + trace_plot_a = subparsers.add_parser(cmd, description=description) + trace_plot_a.add_argument("input", help=pickled) + help = "visualize trace from this device (default: chooses the only device with trace info or errors)" + trace_plot_a.add_argument("-d", "--device", type=int, default=None, help=help) + help = "path to save the visualization(default: output.html)" + trace_plot_a.add_argument("-o", "--output", default="output.html", help=help) + if cmd == "trace_plot": + help = "visualize change to segments rather than individual allocations" + trace_plot_a.add_argument( + "-s", "--segments", action="store_true", help=help + ) + + help = ( + "filter out allocation-free pairs to only visualize the allocations that are not freed yet;" + "useful to reduce the number of events for large traces for debugging OOM" + ) + trace_plot_a.add_argument( + "-f", "--filter_freed", action="store_true", help=help + ) + + args = parser.parse_args() + + def _read(name): + if name == "-": + data = pickle.load(sys.stdin.buffer) + else: + with open(name, "rb") as f: + data = pickle.load(f) + if isinstance(data, list): # segments only... + data = {"segments": data, "traces": []} + return data + + def _write(name, data): + with open(name, "w") as f: + f.write(data) + + if args.action == "segments": + data = _read(args.input) + _write(args.output, segments(data)) + elif args.action == "memory": + data = _read(args.input) + _write(args.output, memory(data)) + elif args.action == "stats": + data = _read(args.input) + print(segsum(data)) + elif args.action == "trace": + data = _read(args.input) + print(trace(data)) + elif args.action == "compare": + before = _read(args.before) + after = _read(args.after) + _write(args.output, compare(before, after)) + elif args.action == "trace_plot": + data = _read(args.input) + _write( + args.output, + trace_plot( + data, + device=args.device, + plot_segments=args.segments, + filter_freed=args.filter_freed, + ), + ) + elif args.action == "segment_plot": + data = _read(args.input) + _write(args.output, segment_plot(data, device=args.device)) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_pin_memory_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_pin_memory_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d3c01f3293f7e35cc5381aee7f6bb2b25957924c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_pin_memory_utils.py @@ -0,0 +1,24 @@ +import torch + + +def pin_memory(data_ptr: int, size: int) -> None: + cudart = torch.cuda.cudart() + succ = int( + cudart.cudaHostRegister( + data_ptr, + size, + 1, # lines up with 'cudaHostRegisterPortable' + ) + ) + + if succ != 0: + raise RuntimeError( + f"Registering memory failed with cudaError: {succ}." + " It's possible that this is an asynchronous error raised from a previous cuda operation." + " Consider launching with CUDA_LAUNCH_BLOCKING=1 to debug." + ) + + +def unpin_memory(data_ptr: int) -> None: + succ = int(torch.cuda.cudart().cudaHostUnregister(data_ptr)) + assert succ == 0, f"Unpinning shared memory failed with error-code: {succ}" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_sanitizer.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_sanitizer.py new file mode 100644 index 0000000000000000000000000000000000000000..8f215a730923bf144405f5827efdf4197b1ce09f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_sanitizer.py @@ -0,0 +1,662 @@ +# mypy: allow-untyped-defs +r""" +This module introduces CUDA Sanitizer, a tool for detecting synchronization errors between kernels ran on different streams. + +It stores information on accesses to tensors to determine if they are synchronized +or not. When enabled in a python program and a possible data race is detected, a +detailed warning will be printed and the program will exit. + +It can be enabled either by importing this module and calling +:func:`enable_cuda_sanitizer()` or by exporting the ``TORCH_CUDA_SANITIZER`` +environment variable. +""" + +import enum +import functools +import inspect +import io +import logging +import re +import sys +import textwrap +import traceback +from collections.abc import Iterator +from dataclasses import dataclass, field +from typing import Any, Optional, TypeVar + +import torch +import torch.cuda._gpu_trace as gpu_trace +from torch.utils import _pytree as pytree +from torch.utils._python_dispatch import TorchDispatchMode + + +DEFAULT_STREAM_ID = 0 + +TK = TypeVar("TK") +TVa = TypeVar("TVa") +TVb = TypeVar("TVb") + +DataPtr = int +StreamId = int +EventId = int +SeqNum = int + +logger = logging.getLogger(__name__) + +# Note that this is only factories that take Tensor as input as they are +# the ones we care about. +FACTORY_FUNCTION_REGEX = re.compile("(new_.*|.*_like)") + + +class AccessType(enum.Enum): + READ = enum.auto() + WRITE = enum.auto() + + def __str__(self): + return "reading from" if self is AccessType.READ else "writing to" + + +@dataclass +class Access: + r"""Stores information about a single access to a tensor by a kernel. + + Args: + type: either AccessType.READ or AccessType.Write. + seq_num: the sequential number of the kernel performing the access. + stream: the stream id of the stream executing the kernel. + operator: the schema of the launched kernel, which lists the + arguments and return type. + aliases: the arguments in the schema this access corresponds to. + is_output: Whether the tensor was an output of the kernel. + stack_trace: the stack summary object captured during access. + """ + + type: AccessType + seq_num: SeqNum + stream: StreamId + operator: str + aliases: list[str] + is_output: bool + stack_trace: traceback.StackSummary + + +class SynchronizationError(Exception): + """Base class for errors detected by CUDA Sanitizer.""" + + +class UnsynchronizedAccessError(SynchronizationError): + """Stores information about two unsynchronized accesses to one data pointer.""" + + def __init__( + self, + data_ptr: DataPtr, + allocation_stack_trace: Optional[traceback.StackSummary], + current_access: Access, + previous_access: Access, + ): + self.data_ptr = data_ptr + self.allocation_stack_trace = allocation_stack_trace + self.current_access = current_access + self.previous_access = previous_access + + def __str__(self): + def format_access(access: Access): + message.write(f"{access.operator}\n{access.type}") + if access.aliases: + message.write(" argument(s) " + ", ".join(access.aliases)) + if access.is_output: + message.write(", and to") + if access.is_output: + message.write(" the output") + message.write( + f"\nWith stack trace:\n{''.join(access.stack_trace.format())}\n" + ) + + with io.StringIO() as message: + message.write( + textwrap.dedent( + f"""\ + ============================ + CSAN detected a possible data race on tensor with data pointer {self.data_ptr} + Access by stream {self.current_access.stream} during kernel: + """ + ) + ) + format_access(self.current_access) + + message.write( + f"Previous access by stream {self.previous_access.stream} during kernel:\n" + ) + format_access(self.previous_access) + + if self.allocation_stack_trace: + message.write( + "Tensor was allocated with stack trace:\n" + f"{''.join(self.allocation_stack_trace.format())}" + ) + else: + message.write("Trace for tensor allocation not found.") + return message.getvalue() + + +class CUDASanitizerErrors(Exception): + """Wrapper class for errors reported by CUDA Sanitizer.""" + + def __init__(self, errors: list[SynchronizationError]): + self.errors = errors + + def __str__(self): + return f"detected {len(self.errors)} errors" + + +@dataclass +class TensorInfo: + r"""Stores information about a single tensor and recent accesses to it. + + Args: + allocation_stack_trace: the stack summary object captured during tensor + allocation. Can be ``None`` if the allocation wasn't caught by CSAN. + reads: list of read accesses to the tensor that were performed since + the last write. + write: the last write access to the tensor. + """ + + allocation_stack_trace: Optional[traceback.StackSummary] + reads: list[Access] = field(default_factory=list) + write: Optional[Access] = None + + +class _TensorsAccessed: + def __init__(self) -> None: + self.accesses: dict[DataPtr, TensorInfo] = {} + + def ensure_tensor_exists(self, data_ptr: DataPtr) -> None: + if data_ptr not in self.accesses: + logger.info( + "Found tensor with pointer: %s, but no matching tensor " + "allocation in the trace. Backfilling the trace now. " + "Perhaps the sanitizer was enabled after some torch operations?", + data_ptr, + ) + self.create_tensor(data_ptr, None) + + def ensure_tensor_does_not_exist(self, data_ptr: DataPtr) -> None: + if data_ptr in self.accesses: + logger.info( + "Found duplicate tensor allocation in the trace for tensor with " + "pointer: %s. Assuming the trace for tensor deallocation " + "wasn't caught and backfilling it now. " + "Perhaps the sanitizer was enabled after some torch operations?", + data_ptr, + ) + self.delete_tensor(data_ptr) + + def create_tensor( + self, data_ptr: DataPtr, stack_trace: Optional[traceback.StackSummary] + ) -> None: + self.accesses[data_ptr] = TensorInfo(stack_trace) + + def delete_tensor(self, data_ptr: DataPtr) -> None: + del self.accesses[data_ptr] + + def were_there_reads_since_last_write(self, data_ptr: DataPtr) -> bool: + return bool(self.accesses[data_ptr].reads) + + def get_allocation_stack_trace( + self, data_ptr: DataPtr + ) -> Optional[traceback.StackSummary]: + return self.accesses[data_ptr].allocation_stack_trace + + def get_write(self, data_ptr: DataPtr) -> Optional[Access]: + return self.accesses[data_ptr].write + + def get_reads(self, data_ptr: DataPtr) -> list[Access]: + return self.accesses[data_ptr].reads + + def add_read(self, data_ptr: DataPtr, access: Access) -> None: + self.accesses[data_ptr].reads.append(access) + + def set_write(self, data_ptr: DataPtr, access: Access) -> None: + self.accesses[data_ptr].write = access + self.accesses[data_ptr].reads = [] + + +class StreamSynchronizations: + def __init__(self) -> None: + self.current_sync_states: dict[StreamId, dict[StreamId, SeqNum]] = {} + self.recorded_sync_states: dict[EventId, dict[StreamId, SeqNum]] = {} + self.host_sync_state: dict[StreamId, SeqNum] = {} + self.create_stream(DEFAULT_STREAM_ID) + + def _ensure_stream_exists(self, stream: StreamId) -> None: + if stream not in self.current_sync_states: + logger.info( + "Found Stream with id: %s, but no matching stream " + "creation in the trace. Backfilling the trace now. " + "Perhaps the sanitizer was enabled after some torch operations?", + stream, + ) + self.create_stream(stream) + + def _ensure_event_exists(self, event: EventId) -> None: + if event not in self.recorded_sync_states: + logger.info( + "Found Event with id: %s, but no matching event " + "creation in the trace. Backfilling the trace now. " + "Perhaps the sanitizer was enabled after some torch operations?", + event, + ) + self.create_event(event) + + def _ensure_event_does_not_exist(self, event: EventId) -> None: + if event in self.recorded_sync_states: + logger.info( + "Found duplicate event creation in the trace for event with " + "id: %s. Assuming the trace for event deletion wasn't caught " + "and backfilling it now. " + "Perhaps the sanitizer was enabled after some torch operations?", + event, + ) + self.delete_event(event) + + def create_stream(self, stream: StreamId) -> None: + if stream in self.current_sync_states: + logger.info( + "Found duplicate Stream creation in the trace for Stream with " + "id: %s. PyTorch Streams are only created once, so this " + "trace entry is ignored.", + stream, + ) + else: + self.host_sync_state[stream] = 0 + self.current_sync_states[stream] = self.host_sync_state.copy() + + def create_event(self, event: EventId) -> None: + self._ensure_event_does_not_exist(event) + self.recorded_sync_states[event] = {} + + def delete_event(self, event: EventId) -> None: + self._ensure_event_exists(event) + del self.recorded_sync_states[event] + + def update_seq_num(self, stream: StreamId, seq_num: SeqNum) -> None: + self._ensure_stream_exists(stream) + self.current_sync_states[stream][stream] = seq_num + + def record_state(self, event: EventId, stream: StreamId) -> None: + self._ensure_event_exists(event) + self._ensure_stream_exists(stream) + self.recorded_sync_states[event] = self.current_sync_states[stream].copy() + + def _state_wait_for_other( + self, state: dict[StreamId, SeqNum], other: dict[StreamId, SeqNum] + ) -> None: + for stream, seq_num in other.items(): + state[stream] = max(state.get(stream, -1), seq_num) + + def stream_wait_for_event(self, stream: StreamId, event: EventId) -> None: + self._ensure_stream_exists(stream) + self._ensure_event_exists(event) + self._state_wait_for_other( + self.current_sync_states[stream], self.recorded_sync_states[event] + ) + + def all_streams_wait_for_event(self, event: EventId) -> None: + self._ensure_event_exists(event) + for stream in self.current_sync_states: + self.stream_wait_for_event(stream, event) + + self._state_wait_for_other( + self.host_sync_state, self.recorded_sync_states[event] + ) + + def all_streams_wait_for_stream(self, stream: StreamId) -> None: + self._ensure_stream_exists(stream) + for state in self.current_sync_states.values(): + self._state_wait_for_other(state, self.current_sync_states[stream]) + + self._state_wait_for_other( + self.host_sync_state, self.current_sync_states[stream] + ) + + def sync_all_streams(self) -> None: + for stream, state in self.current_sync_states.items(): + self.host_sync_state[stream] = state[stream] + + for state in self.current_sync_states.values(): + self._state_wait_for_other(state, self.host_sync_state) + + def is_ordered_after( + self, current_stream: StreamId, seq_num: SeqNum, other_stream: StreamId + ) -> bool: + self._ensure_stream_exists(current_stream) + self._ensure_stream_exists(other_stream) + return seq_num <= self.current_sync_states[current_stream].get(other_stream, -1) + + +class EventHandler: + """Analyzes CSAN trace for synchronization errors. + + Stores information on each stream's synchronizations with other streams as well + as tensor accesses to determine whether a given kernel launch might cause a + data race. + """ + + def __init__(self) -> None: + self.tensors_accessed = _TensorsAccessed() + self.syncs = StreamSynchronizations() + self.seq_num: SeqNum = 0 + + def _handle_kernel_launch( + self, + stream: StreamId, + read_only: set[DataPtr], + read_write: set[DataPtr], + outputs: set[DataPtr], + operator: str, + tensor_aliases: dict[int, list[str]], + ) -> list[SynchronizationError]: + def check_conflict( + data_ptr: DataPtr, current_access: Access, previous_access: Optional[Access] + ) -> None: + if previous_access is None: + return + if not self.syncs.is_ordered_after( + current_access.stream, previous_access.seq_num, previous_access.stream + ): + error_list.append( + UnsynchronizedAccessError( + data_ptr, + self.tensors_accessed.get_allocation_stack_trace(data_ptr), + current_access, + previous_access, + ) + ) + + error_list: list[SynchronizationError] = [] + self.seq_num += 1 + self.syncs.update_seq_num(stream, self.seq_num) + stack_trace = traceback.StackSummary.extract( + traceback.walk_stack(inspect.currentframe()), lookup_lines=False + ) + # The stack trace generated in this way is in the inverse order, so it must be + # reversed. + stack_trace.reverse() + + for data_ptr in read_only: + self.tensors_accessed.ensure_tensor_exists(data_ptr) + current_access = Access( + AccessType.READ, + self.seq_num, + stream, + operator, + tensor_aliases[data_ptr], + data_ptr in outputs, + stack_trace, + ) + check_conflict( + data_ptr, current_access, self.tensors_accessed.get_write(data_ptr) + ) + self.tensors_accessed.add_read(data_ptr, current_access) + + for data_ptr in read_write: + self.tensors_accessed.ensure_tensor_exists(data_ptr) + current_access = Access( + AccessType.WRITE, + self.seq_num, + stream, + operator, + tensor_aliases[data_ptr], + data_ptr in outputs, + stack_trace, + ) + if self.tensors_accessed.were_there_reads_since_last_write(data_ptr): + for previous_access in self.tensors_accessed.get_reads(data_ptr): + check_conflict(data_ptr, current_access, previous_access) + else: + check_conflict( + data_ptr, current_access, self.tensors_accessed.get_write(data_ptr) + ) + self.tensors_accessed.set_write(data_ptr, current_access) + + return error_list + + def _handle_event_creation(self, event: EventId) -> None: + self.syncs.create_event(event) + + def _handle_event_deletion(self, event: EventId) -> None: + self.syncs.delete_event(event) + + def _handle_event_record(self, event: EventId, stream: StreamId) -> None: + self.syncs.record_state(event, stream) + + def _handle_event_wait(self, event: EventId, stream: StreamId) -> None: + self.syncs.stream_wait_for_event(stream, event) + + def _handle_memory_allocation(self, data_ptr: DataPtr) -> None: + self.tensors_accessed.ensure_tensor_does_not_exist(data_ptr) + stack_trace = traceback.StackSummary.extract( + traceback.walk_stack(inspect.currentframe()), lookup_lines=False + ) + # The stack trace generated in this way is in the inverse order, so it must be + # reversed. + stack_trace.reverse() + self.tensors_accessed.create_tensor( + data_ptr, + stack_trace, + ) + + def _handle_memory_deallocation(self, data_ptr: DataPtr) -> None: + self.tensors_accessed.ensure_tensor_exists(data_ptr) + self.tensors_accessed.delete_tensor(data_ptr) + + def _handle_stream_creation(self, stream: StreamId) -> None: + self.syncs.create_stream(stream) + + def _handle_device_synchronization(self) -> None: + self.syncs.sync_all_streams() + + def _handle_stream_synchronization(self, stream: StreamId) -> None: + self.syncs.all_streams_wait_for_stream(stream) + + def _handle_event_synchronization(self, event: EventId) -> None: + self.syncs.all_streams_wait_for_event(event) + + +def zip_by_key(a: dict[TK, TVa], b: dict[TK, TVb]) -> Iterator[tuple[TK, TVa, TVb]]: + for arg, value in a.items(): + if arg in b: + yield arg, value, b[arg] + + +def zip_arguments( + schema: torch.FunctionSchema, args: tuple[Any, ...], kwargs: dict[str, Any] +) -> Iterator[tuple[torch.Argument, Any]]: + schema_args = schema.arguments[: len(args)] + schema_kwargs = {arg.name: arg for arg in schema.arguments[len(args) :]} + + yield from zip(schema_args, args) + + for _, argument, value in zip_by_key(schema_kwargs, kwargs): + yield (argument, value) + + +class ArgumentHandler: + def __init__(self) -> None: + self.dataptrs_read: set[DataPtr] = set() + self.dataptrs_written: set[DataPtr] = set() + self.tensor_aliases: dict[DataPtr, list[str]] = {} + self.outputs: set[DataPtr] = set() + + def _handle_argument( + self, + value: Any, + is_write: bool, + metadata_only: bool, + name: Optional[str] = None, + is_output: bool = False, + ) -> None: + if isinstance(value, torch.Tensor) and value.is_cuda: + data_ptr = value.data_ptr() + if is_write: + self.dataptrs_written.add(data_ptr) + elif not metadata_only: + self.dataptrs_read.add(data_ptr) + + self.tensor_aliases.setdefault(data_ptr, []) + if name is not None: + self.tensor_aliases[data_ptr].append(name) + if is_output: + self.outputs.add(data_ptr) + + def parse_inputs( + self, + schema: torch.FunctionSchema, + args: tuple[Any, ...], + kwargs: dict[str, Any], + *, + is_factory: bool, + ) -> None: + for argument, value in zip_arguments(schema, args, kwargs): + is_write = argument.alias_info is not None and argument.alias_info.is_write + # A change is metadata only if it is a view or a factory function that + # reads only metadata + metadata_only = is_factory or ( + argument.alias_info is not None and not argument.alias_info.is_write + ) + pytree.tree_map_( + functools.partial( + self._handle_argument, + is_write=is_write, + name=argument.name, + metadata_only=metadata_only, + ), + value, + ) + + def parse_outputs( + self, schema: torch.FunctionSchema, outputs: Any, *, is_factory: bool + ) -> None: + for res, value in zip(schema.returns, (outputs,)): + metadata_only = is_factory or ( + res.alias_info is not None and not res.alias_info.is_write + ) + pytree.tree_map_( + functools.partial( + self._handle_argument, + is_write=not metadata_only, + is_output=True, + metadata_only=metadata_only, + ), + value, + ) + + +class CUDASanitizerDispatchMode(TorchDispatchMode): + def __init__(self) -> None: + self.event_handler = EventHandler() + torch._C._activate_gpu_trace() + gpu_trace.register_callback_for_event_creation( + self.event_handler._handle_event_creation + ) + gpu_trace.register_callback_for_event_deletion( + self.event_handler._handle_event_deletion + ) + gpu_trace.register_callback_for_event_record( + self.event_handler._handle_event_record + ) + gpu_trace.register_callback_for_event_wait( + self.event_handler._handle_event_wait + ) + gpu_trace.register_callback_for_memory_allocation( + self.event_handler._handle_memory_allocation + ) + gpu_trace.register_callback_for_memory_deallocation( + self.event_handler._handle_memory_deallocation + ) + gpu_trace.register_callback_for_stream_creation( + self.event_handler._handle_stream_creation + ) + gpu_trace.register_callback_for_device_synchronization( + self.event_handler._handle_device_synchronization + ) + gpu_trace.register_callback_for_stream_synchronization( + self.event_handler._handle_stream_synchronization + ) + gpu_trace.register_callback_for_event_synchronization( + self.event_handler._handle_event_synchronization + ) + + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + if kwargs is None: + kwargs = {} + + is_factory = bool(FACTORY_FUNCTION_REGEX.match(func._schema.name)) + + argument_handler = ArgumentHandler() + argument_handler.parse_inputs(func._schema, args, kwargs, is_factory=is_factory) + + outputs = func(*args, **kwargs) + + argument_handler.parse_outputs(func._schema, outputs, is_factory=is_factory) + errors = self.event_handler._handle_kernel_launch( + torch.cuda.current_stream().cuda_stream, + argument_handler.dataptrs_read - argument_handler.dataptrs_written, + argument_handler.dataptrs_written, + argument_handler.outputs, + func._schema, + argument_handler.tensor_aliases, + ) + if errors: + for error in errors: + print(error, file=sys.stderr) + raise CUDASanitizerErrors(errors) + + return outputs + + +class CUDASanitizer: + """Manages the lifetime of a CUDASanitizer dispatch mode object. + + The CUDASanitizer class wraps the entering/exiting functions of the dispatch mode + context manager in the enable function/destructor, respectively. This is to + explicitly set the lifetime of the dispatch mode object to that of the application. + This approach was deemed more elegant than using the atexit module. + """ + + def __init__(self) -> None: + self.dispatch = CUDASanitizerDispatchMode() + self.enabled = False + + def enable(self): + self.dispatch.__enter__() + self.enabled = True + + def disable(self): + self.dispatch.__exit__(None, None, None) + self.enabled = False + + def __del__(self): + # Since this object lifetime is linked to the `torch.cuda._sanitizer` python + # module, it often gets deleted as part of the overall `torch` module cleanup + # At that time, depending on CPython version, the torch.* module might be in + # different states of being already cleaned up. + # Similarly other imports might already have been cleaned up so `sys` might + # be already gone as well. + # Skip exiting the mode if it outlived the runtime. + if (sys is not None) and (not sys.is_finalizing()) and self.enabled: + self.disable() + + +def enable_cuda_sanitizer(): + """Enable CUDA Sanitizer. + + The sanitizer will begin to analyze low-level CUDA calls invoked by torch functions + for synchronization errors. All data races found will be printed to the standard + error output along with stack traces of suspected causes. For best results, the + sanitizer should be enabled at the very beginning of the program. + """ + cuda_sanitizer.enable() + + +cuda_sanitizer = CUDASanitizer() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..24b6aeeef0d8c5632282786494c19d3664763d14 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/_utils.py @@ -0,0 +1,529 @@ +import ctypes +import sys +from typing import Any, Optional, Union + +import torch + +# The _get_device_index has been moved to torch.utils._get_device_index +from torch._utils import _get_device_index as _torch_get_device_index + + +def _get_hip_runtime_library() -> ctypes.CDLL: + if sys.platform == "win32": + lib = ctypes.CDLL(f"amdhip64_{torch.version.hip[0]}.dll") + else: # Unix-based systems + lib = ctypes.CDLL("libamdhip64.so") + lib.cuGetErrorString = lib.hipGetErrorString # type: ignore[attr-defined] + lib.cuModuleLoadData = lib.hipModuleLoadData # type: ignore[attr-defined] + lib.cuModuleGetFunction = lib.hipModuleGetFunction # type: ignore[attr-defined] + lib.cuLaunchKernel = lib.hipModuleLaunchKernel # type: ignore[attr-defined] + lib.cuFuncSetAttribute = lib.hipFuncSetAttribute # type: ignore[attr-defined] + return lib + + +def _get_cuda_library() -> ctypes.CDLL: + if sys.platform == "win32": + return ctypes.CDLL("nvcuda.dll") + else: # Unix-based systems + return ctypes.CDLL("libcuda.so.1") + + +# Load GPU driver runtime +def _get_gpu_runtime_library() -> ctypes.CDLL: + if torch.version.hip: + return _get_hip_runtime_library() + else: + return _get_cuda_library() + + +# Helper: check CUDA errors +def _check_cuda(result: int) -> None: + if result == 0: + return + err_str = ctypes.c_char_p() + libcuda = _get_gpu_runtime_library() # Get reference to CUDA library + libcuda.cuGetErrorString(result, ctypes.byref(err_str)) + error_message = ( + err_str.value.decode() if err_str.value is not None else "Unknown CUDA error" + ) + raise RuntimeError(f"CUDA error: {error_message}") + + +def _get_hiprtc_library() -> ctypes.CDLL: + if sys.platform == "win32": + version_str = "".join(["0", torch.version.hip[0], "0", torch.version.hip[2]]) + lib = ctypes.CDLL(f"hiprtc{version_str}.dll") + else: + lib = ctypes.CDLL("libhiprtc.so") + + # Provide aliases for HIP RTC functions to match NVRTC API + lib.nvrtcGetErrorString = lib.hiprtcGetErrorString # type: ignore[attr-defined] + lib.nvrtcCreateProgram = lib.hiprtcCreateProgram # type: ignore[attr-defined] + lib.nvrtcDestroyProgram = lib.hiprtcDestroyProgram # type: ignore[attr-defined] + lib.nvrtcCompileProgram = lib.hiprtcCompileProgram # type: ignore[attr-defined] + lib.nvrtcGetPTXSize = lib.hiprtcGetCodeSize # type: ignore[attr-defined] + lib.nvrtcGetPTX = lib.hiprtcGetCode # type: ignore[attr-defined] + lib.nvrtcGetProgramLogSize = lib.hiprtcGetProgramLogSize # type: ignore[attr-defined] + lib.nvrtcGetProgramLog = lib.hiprtcGetProgramLog # type: ignore[attr-defined] + lib.nvrtcAddNameExpression = lib.hiprtcAddNameExpression # type: ignore[attr-defined] + lib.nvrtcGetLoweredName = lib.hiprtcGetLoweredName # type: ignore[attr-defined] + return lib + + +def _get_nvrtc_library() -> ctypes.CDLL: + major_version = int(torch.version.cuda.split(".")[0]) # type: ignore[union-attr] + if sys.platform == "win32": + nvrtc_libs = [ + f"nvrtc64_{major_version}0_0.dll", + ] + else: + nvrtc_libs = [ + f"libnvrtc.so.{major_version}", + "libnvrtc.so", # Fallback to unversioned + ] + for lib_name in nvrtc_libs: + try: + return ctypes.CDLL(lib_name) + except OSError: + continue + raise OSError("Could not find any NVRTC library") + + +def _get_gpu_rtc_library() -> ctypes.CDLL: + # Since PyTorch already loads the GPU RTC library, we can use the system library + # which should be compatible with PyTorch's version + if torch.version.hip: + return _get_hiprtc_library() + else: + return _get_nvrtc_library() + + +def _get_gpu_rtc_compatible_flags() -> list[str]: + """ + Get HIPCC/NVCC flags that are compatible with NVRTC compilation. + + Returns: + List of HIPCC/NVCC flags that can be safely used with NVRTC. + """ + from torch.utils.cpp_extension import COMMON_HIPCC_FLAGS, COMMON_NVCC_FLAGS + + nvrtc_unsupported_flags = { + "--expt-relaxed-constexpr", + } + + # Filter out unsupported flags + compatible_flags = [ + flag for flag in COMMON_NVCC_FLAGS if flag not in nvrtc_unsupported_flags + ] + + if torch.version.hip: + compatible_flags.extend(COMMON_HIPCC_FLAGS) + + return compatible_flags + + +def _nvrtc_compile( + kernel_source: str, + kernel_name: str, + compute_capability: Optional[str] = None, + cuda_include_dirs: Optional[list] = None, + nvcc_options: Optional[list] = None, + auto_pch: bool = False, +) -> tuple[bytes, str]: + """ + Compiles a CUDA kernel using NVRTC and returns the PTX code. + + Args: + kernel_source (str): The CUDA kernel source code as a string + kernel_name (str): The name of the kernel function to compile + compute_capability (str, None): The compute capability to target (e.g., "86"). + If None, will detect from current device. + cuda_include_dirs (list, None): List of directories containing CUDA headers + nvcc_options (list, None): Additional options to pass to NVRTC + auto_pch (bool): Enable automatic precompiled headers (CUDA 12.8+) + + Returns: + Tuple[bytes, str]: The compiled PTX code and mangled kernel name + """ + # Ensure CUDA is initialized + import torch.cuda + + # Load NVRTC library + libnvrtc = _get_gpu_rtc_library() + + # NVRTC constants + NVRTC_SUCCESS = 0 + + # Helper: check NVRTC errors + def check_nvrtc(result: int) -> None: + if result != NVRTC_SUCCESS: + err_str = ctypes.c_char_p() + libnvrtc.nvrtcGetErrorString(result, ctypes.byref(err_str)) + error_message = ( + err_str.value.decode() + if err_str.value is not None + else "Unknown CUDA error" + ) + raise RuntimeError(f"CUDA error: {error_message}") + + # Convert source to bytes + source_bytes = kernel_source.encode("utf-8") + + # Get compute capability if not provided + if compute_capability is None: + props = torch.cuda.get_device_properties(torch.cuda.current_device()) + if torch.version.hip: + compute_capability = f"{props.gcnArchName}" + else: + compute_capability = f"{props.major}{props.minor}" + + # Prepare compilation options + options = [] + if torch.version.hip: + options.append(f"--offload-arch={compute_capability}".encode()) + else: + options.append(f"--gpu-architecture=sm_{compute_capability}".encode()) + + # Auto-detect and add CUDA include paths + from torch.utils.cpp_extension import include_paths + + cuda_include_paths = include_paths("cuda") + for cuda_path in cuda_include_paths: + options.append(f"-I{cuda_path}".encode()) + + # Add custom include directories + if cuda_include_dirs: + for directory in cuda_include_dirs: + options.append(f"-I{directory}".encode()) + + # Enable automatic precompiled headers (CUDA 12.8+) + if auto_pch: + assert str(torch.version.cuda) >= "12.8", "PCH requires CUDA 12.8+" + if nvcc_options is None: + nvcc_options = [] + nvcc_options.append("--pch") + + # Add custom NVCC options + if nvcc_options: + for option in nvcc_options: + options.append(option.encode("utf-8")) + + nvrtc_compatible_flags = _get_gpu_rtc_compatible_flags() + options.extend([flag.encode("utf-8") for flag in nvrtc_compatible_flags]) + + # Convert options to C array + num_options = len(options) + options_array = (ctypes.c_char_p * num_options)(*options) + + # Create program + prog = ctypes.c_void_p() + check_nvrtc( + libnvrtc.nvrtcCreateProgram( + ctypes.byref(prog), + source_bytes, + f"{kernel_name}.cu".encode(), + 0, + None, + None, + ) + ) + + # Add kernel name, which can be a template expression + c_kernel_name = kernel_name.encode("utf-8") + check_nvrtc(libnvrtc.nvrtcAddNameExpression(prog, c_kernel_name)) + + # Compile program + res = libnvrtc.nvrtcCompileProgram(prog, num_options, options_array) + + # Handle compilation errors + if res != NVRTC_SUCCESS: + # Get log + log_size = ctypes.c_size_t() + libnvrtc.nvrtcGetProgramLogSize(prog, ctypes.byref(log_size)) + log = ctypes.create_string_buffer(log_size.value) + libnvrtc.nvrtcGetProgramLog(prog, log) + raise RuntimeError(f"Kernel compilation failed:\n{log.value.decode()}") + + # Get PTX + ptx_size = ctypes.c_size_t() + check_nvrtc(libnvrtc.nvrtcGetPTXSize(prog, ctypes.byref(ptx_size))) + ptx = ctypes.create_string_buffer(ptx_size.value) + check_nvrtc(libnvrtc.nvrtcGetPTX(prog, ptx)) + + # Get mangled name + c_mangled_name = ctypes.c_char_p() + check_nvrtc( + libnvrtc.nvrtcGetLoweredName(prog, c_kernel_name, ctypes.byref(c_mangled_name)) + ) + if c_mangled_name.value is not None: + mangled_name = c_mangled_name.value.decode() # make a copy + else: + mangled_name = "" + + libnvrtc.nvrtcDestroyProgram(ctypes.byref(prog)) + + # For HIP, hipRTC generates raw CO binaries instead of PTX, + # and for some reason, ".value" causes the string to be truncated, + # likely due to the presence of '\0' in the string. So we use .raw instead. + ptx_bytes = ptx.raw if torch.version.hip else ptx.value + return ptx_bytes, mangled_name + + +class _CudaModule: + def __init__(self, module: ctypes.c_void_p) -> None: + self._module = module + self._kernels: dict[str, _CudaKernel] = {} + + def __getattr__(self, name: str) -> "_CudaKernel": + if name in self._kernels: + return self._kernels[name] + + # Import the CUDA library inside the method + # pyrefly: ignore [missing-module-attribute] + from torch.cuda._utils import _get_gpu_runtime_library + + libcuda = _get_gpu_runtime_library() + + func = ctypes.c_void_p() + try: + _check_cuda( + libcuda.cuModuleGetFunction( + ctypes.byref(func), self._module, name.encode("utf-8") + ) + ) + kernel = _CudaKernel(func, self._module) + self._kernels[name] = kernel + return kernel + + except RuntimeError as err: + raise AttributeError(f"No kernel named '{name}' in this module") from err + + +class _CudaKernel: + """ + Represents a compiled CUDA kernel that can be called with PyTorch tensors. + """ + + def __init__(self, func: ctypes.c_void_p, module: ctypes.c_void_p) -> None: + self.func = func + self.module = module + self._max_shared_mem_bytes = 0 + + def __call__( + self, + grid: tuple[int, int, int] = (1, 1, 1), + block: tuple[int, int, int] = (1, 1, 1), + args: Optional[list] = None, + shared_mem: int = 0, + stream: Optional[Any] = None, + ) -> None: + """ + Call the compiled CUDA kernel + + Args: + grid (tuple): Grid dimensions (grid_x, grid_y, grid_z) + block (tuple): Block dimensions (block_x, block_y, block_z) + args (list): List of arguments to pass to the kernel. + PyTorch tensor arguments will be automatically converted to pointers. + shared_mem (int): Shared memory size in bytes + stream (torch.cuda.Stream): CUDA stream to use. If None, uses current stream. + """ + import torch + + libcuda = torch.cuda._utils._get_gpu_runtime_library() + + if not args: + args = [] + + # Process arguments and convert tensors to pointers + processed_args: list[ctypes.c_void_p] = [] + c_args = [] + + for arg in args: + if isinstance(arg, torch.Tensor): + if not arg.is_cuda and not (arg.is_cpu and arg.is_pinned()): + raise ValueError( + "All tensor arguments must be CUDA tensors or pinned CPU tensors" + ) + # Get pointer to tensor data + ptr = ctypes.c_void_p(arg.data_ptr()) + processed_args.append(ptr) + c_args.append(ctypes.byref(ptr)) + elif isinstance(arg, int): + # Convert integers to C int + c_int = ctypes.c_int(arg) + # Store the C int for reference keeping, not in processed_args + c_args.append(ctypes.byref(c_int)) + elif isinstance(arg, float): + # Python floats are doubles - use double by default + c_double = ctypes.c_double(arg) + # Store the C double for reference keeping, not in processed_args + c_args.append(ctypes.byref(c_double)) + else: + raise TypeError(f"Unsupported argument type: {type(arg)}") + + # Convert to array of void pointers + c_args_array = (ctypes.c_void_p * len(c_args))() + for i, arg in enumerate(c_args): + c_args_array[i] = ctypes.cast(arg, ctypes.c_void_p) + + # Get the stream + if stream is None: + # Defer import to avoid circular imports + import torch.cuda + + stream = torch.cuda.current_stream() + + # Check if kernel requires large shared memory but hasn't been configured + if shared_mem >= 48 * 1024 and ( + self._max_shared_mem_bytes == 0 or shared_mem > self._max_shared_mem_bytes + ): + configured_msg = ( + "not configured" + if self._max_shared_mem_bytes == 0 + else f"only {self._max_shared_mem_bytes} bytes configured" + ) + raise RuntimeError( + f"Kernel requires {shared_mem} bytes of shared memory (>= 48KB), " + f"but {configured_msg}. " + "Call kernel.set_shared_memory_config(shared_mem) after compilation " + "and before launching the kernel." + ) + + _check_cuda( + libcuda.cuLaunchKernel( + self.func, + grid[0], + grid[1], + grid[2], + block[0], + block[1], + block[2], + shared_mem, + stream._as_parameter_, + c_args_array, + None, + ) + ) + + def set_shared_memory_config(self, shared_mem_bytes: int) -> None: + if shared_mem_bytes < 48 * 1024: + # No configuration needed for <= 48KB, just update the value + self._max_shared_mem_bytes = shared_mem_bytes + return + + libcuda = _get_gpu_runtime_library() + + # Get device properties to validate against limits + device_props = torch.cuda.get_device_properties() + # HIP doesn't have shared_memory_per_block_optin in device properties, so we hard-code it here + if torch.version.hip: + # navi, CDNA1-CDNA3 allows a max of 64KB shared memory + # CDNA4 allows a max of 160KB shared memory + max_shared_mem = ( + 65536 if device_props.gcnArchName != "gfx950" else 160 * 1024 + ) + else: + max_shared_mem = getattr( + device_props, "shared_memory_per_block_optin", 49152 + ) + + if shared_mem_bytes > max_shared_mem: + raise RuntimeError( + f"Requested shared memory ({shared_mem_bytes} bytes) exceeds " + f"device limit ({max_shared_mem} bytes). " + "Consider reducing block size or shared memory usage." + ) + + # Set the function attribute once + # https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html + cudaFuncAttributeMaxDynamicSharedMemorySize = 8 + _check_cuda( + libcuda.cuFuncSetAttribute( + self.func, + cudaFuncAttributeMaxDynamicSharedMemorySize, + shared_mem_bytes, + ) + ) + + self._max_shared_mem_bytes = shared_mem_bytes + + +def _cuda_load_module( + ptx: Union[str, bytes], kernel_names: Optional[list[str]] = None +) -> Union[_CudaModule, dict[str, "_CudaKernel"]]: + """ + Loads a CUDA module from PTX code and returns a module object that can access kernels. + + Args: + ptx (bytes or str): The PTX code to load + kernel_names (list, optional): List of kernel names to extract from the module. + If None, will return a module object with __getattr__. + + Returns: + object: If kernel_names is None, returns a module object with __getattr__ to access kernels. + If kernel_names is provided, returns a dict mapping kernel names to _CudaKernel objects. + """ + # Ensure CUDA is initialized + import torch.cuda + + # Load CUDA driver library + libcuda = _get_gpu_runtime_library() + + # Convert PTX to bytes if it's a string + if isinstance(ptx, str): + ptx = ptx.encode("utf-8") + + # Load PTX module + module = ctypes.c_void_p() + # Get the current stream without directly importing torch.cuda at module level + stream = torch.cuda.current_stream() + with stream: + _check_cuda(libcuda.cuModuleLoadData(ctypes.byref(module), ptx)) + + if not kernel_names: + return _CudaModule(module) + + # Return specific kernels + kernels = {} + for name in kernel_names: + func = ctypes.c_void_p() + _check_cuda( + libcuda.cuModuleGetFunction( + ctypes.byref(func), module, name.encode("utf-8") + ) + ) + kernels[name] = _CudaKernel(func, module) + return kernels + + +def _get_device_index( + device: Any, optional: bool = False, allow_cpu: bool = False +) -> int: + r"""Get the device index from :attr:`device`, which can be a torch.device object, a Python integer, or ``None``. + + If :attr:`device` is a torch.device object, returns the device index if it + is a CUDA device. Note that for a CUDA device without a specified index, + i.e., ``torch.device('cuda')``, this will return the current default CUDA + device if :attr:`optional` is ``True``. If :attr:`allow_cpu` is ``True``, + CPU devices will be accepted and ``-1`` will be returned in this case. + + If :attr:`device` is a Python integer, it is returned as is. + + If :attr:`device` is ``None``, this will return the current default CUDA + device if :attr:`optional` is ``True``. + """ + if isinstance(device, int): + return device + if isinstance(device, str): + device = torch.device(device) + if isinstance(device, torch.device): + if allow_cpu: + if device.type not in ["cuda", "cpu"]: + raise ValueError(f"Expected a cuda or cpu device, but got: {device}") + elif device.type != "cuda": + raise ValueError(f"Expected a cuda device, but got: {device}") + if not torch.jit.is_scripting(): + if isinstance(device, torch.cuda.device): + return device.idx + return _torch_get_device_index(device, optional, allow_cpu) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/comm.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/comm.py new file mode 100644 index 0000000000000000000000000000000000000000..2915de5d090fd18c82540beedb9971a0b7b6cc3e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/comm.py @@ -0,0 +1,19 @@ +# The functions here have been moved to torch.nn.parallel.comm +from torch.nn.parallel.comm import ( + broadcast, + broadcast_coalesced, + gather, + reduce_add, + reduce_add_coalesced, + scatter, +) + + +__all__ = [ + "broadcast", + "broadcast_coalesced", + "reduce_add", + "reduce_add_coalesced", + "scatter", + "gather", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/gds.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/gds.py new file mode 100644 index 0000000000000000000000000000000000000000..5a7dfa388caa0420900408edf70afde7798b30a2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/gds.py @@ -0,0 +1,167 @@ +import os +import sys +from collections.abc import Callable +from typing import Optional + +import torch +from torch.types import Storage + + +__all__: list[str] = [ + "gds_register_buffer", + "gds_deregister_buffer", + "GdsFile", +] + + +def _dummy_fn(name: str) -> Callable: + def fn(*args, **kwargs): # type: ignore[no-untyped-def] + raise RuntimeError(f"torch._C.{name} is not supported on this platform") + + return fn + + +if not hasattr(torch._C, "_gds_register_buffer"): + assert not hasattr(torch._C, "_gds_deregister_buffer") + assert not hasattr(torch._C, "_gds_register_handle") + assert not hasattr(torch._C, "_gds_deregister_handle") + assert not hasattr(torch._C, "_gds_load_storage") + assert not hasattr(torch._C, "_gds_save_storage") + # Define functions + torch._C.__dict__["_gds_register_buffer"] = _dummy_fn("_gds_register_buffer") + torch._C.__dict__["_gds_deregister_buffer"] = _dummy_fn("_gds_deregister_buffer") + torch._C.__dict__["_gds_register_handle"] = _dummy_fn("_gds_register_handle") + torch._C.__dict__["_gds_deregister_handle"] = _dummy_fn("_gds_deregister_handle") + torch._C.__dict__["_gds_load_storage"] = _dummy_fn("_gds_load_storage") + torch._C.__dict__["_gds_save_storage"] = _dummy_fn("_gds_save_storage") + + +def gds_register_buffer(s: Storage) -> None: + """Registers a storage on a CUDA device as a cufile buffer. + + Example:: + + >>> # xdoctest: +SKIP("gds filesystem requirements") + >>> src = torch.randn(1024, device="cuda") + >>> s = src.untyped_storage() + >>> gds_register_buffer(s) + + Args: + s (Storage): Buffer to register. + """ + torch._C._gds_register_buffer(s) + + +def gds_deregister_buffer(s: Storage) -> None: + """Deregisters a previously registered storage on a CUDA device as a cufile buffer. + + Example:: + + >>> # xdoctest: +SKIP("gds filesystem requirements") + >>> src = torch.randn(1024, device="cuda") + >>> s = src.untyped_storage() + >>> gds_register_buffer(s) + >>> gds_deregister_buffer(s) + + Args: + s (Storage): Buffer to register. + """ + torch._C._gds_deregister_buffer(s) + + +class GdsFile: + r"""Wrapper around cuFile. + + cuFile is a file-like interface to the GPUDirect Storage (GDS) API. + + See the `cufile docs `_ + for more details. + + Args: + filename (str): Name of the file to open. + flags (int): Flags to pass to ``os.open`` when opening the file. ``os.O_DIRECT`` will + be added automatically. + + Example:: + + >>> # xdoctest: +SKIP("gds filesystem requirements") + >>> src1 = torch.randn(1024, device="cuda") + >>> src2 = torch.randn(2, 1024, device="cuda") + >>> file = torch.cuda.gds.GdsFile(f, os.O_CREAT | os.O_RDWR) + >>> file.save_storage(src1.untyped_storage(), offset=0) + >>> file.save_storage(src2.untyped_storage(), offset=src1.nbytes) + >>> dest1 = torch.empty(1024, device="cuda") + >>> dest2 = torch.empty(2, 1024, device="cuda") + >>> file.load_storage(dest1.untyped_storage(), offset=0) + >>> file.load_storage(dest2.untyped_storage(), offset=src1.nbytes) + >>> torch.equal(src1, dest1) + True + >>> torch.equal(src2, dest2) + True + + """ + + def __init__(self, filename: str, flags: int): + if sys.platform == "win32": + raise RuntimeError("GdsFile is not supported on this platform.") + self.filename = filename + self.flags = flags + self.fd = os.open(filename, flags | os.O_DIRECT) # type: ignore[attr-defined] + self.handle: Optional[int] = None + self.register_handle() + + def __del__(self) -> None: + if self.handle is not None: + self.deregister_handle() + os.close(self.fd) + + def register_handle(self) -> None: + """Registers file descriptor to cuFile Driver. + + This is a wrapper around ``cuFileHandleRegister``. + """ + assert self.handle is None, ( + "Cannot register a handle that is already registered." + ) + self.handle = torch._C._gds_register_handle(self.fd) + + def deregister_handle(self) -> None: + """Deregisters file descriptor from cuFile Driver. + + This is a wrapper around ``cuFileHandleDeregister``. + """ + assert self.handle is not None, ( + "Cannot deregister a handle that is not registered." + ) + torch._C._gds_deregister_handle(self.handle) + self.handle = None + + def load_storage(self, storage: Storage, offset: int = 0) -> None: + """Loads data from the file into the storage. + + This is a wrapper around ``cuFileRead``. ``storage.nbytes()`` of data + will be loaded from the file at ``offset`` into the storage. + + Args: + storage (Storage): Storage to load data into. + offset (int, optional): Offset into the file to start loading from. (Default: 0) + """ + assert self.handle is not None, ( + "Cannot load data from a file that is not registered." + ) + torch._C._gds_load_storage(self.handle, storage, offset) + + def save_storage(self, storage: Storage, offset: int = 0) -> None: + """Saves data from the storage into the file. + + This is a wrapper around ``cuFileWrite``. All bytes of the storage + will be written to the file at ``offset``. + + Args: + storage (Storage): Storage to save data from. + offset (int, optional): Offset into the file to start saving to. (Default: 0) + """ + assert self.handle is not None, ( + "Cannot save data to a file that is not registered." + ) + torch._C._gds_save_storage(self.handle, storage, offset) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/graphs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/graphs.py new file mode 100644 index 0000000000000000000000000000000000000000..b4d2cb43f8bc3d964ea620606127b5f7a661de4c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/graphs.py @@ -0,0 +1,613 @@ +# pylint: disable=useless-parent-delegation +from __future__ import annotations + +import gc +import typing +from collections.abc import Callable +from typing import Optional, overload, TYPE_CHECKING, TypeAlias, Union +from typing_extensions import ParamSpec, Self, TypeVar + +import torch +from torch import Tensor + + +if TYPE_CHECKING: + # importing _POOL_HANDLE at runtime toplevel causes an import cycle + from torch.cuda import _POOL_HANDLE + +from .._utils import _dummy_type + + +__all__ = [ + "is_current_stream_capturing", + "graph_pool_handle", + "CUDAGraph", + "graph", + "make_graphed_callables", +] + + +_R = TypeVar("_R") +_P = ParamSpec("_P") + + +if not hasattr(torch._C, "_CudaStreamBase"): + # Define dummy base classes + torch._C.__dict__["_CUDAGraph"] = _dummy_type("_CUDAGraph") + torch._C.__dict__["_graph_pool_handle"] = _dummy_type("_graph_pool_handle") + torch._C.__dict__["_cuda_isCurrentStreamCapturing"] = _dummy_type( + "_cuda_isCurrentStreamCapturing" + ) + +from torch._C import ( # noqa: F401 + _cuda_isCurrentStreamCapturing, + _CUDAGraph, + _graph_pool_handle, +) + + +def is_current_stream_capturing() -> bool: + r"""Return True if CUDA graph capture is underway on the current CUDA stream, False otherwise. + + If a CUDA context does not exist on the current device, returns False without initializing the context. + """ + return _cuda_isCurrentStreamCapturing() + + +# Python shim helps Sphinx process docstrings more reliably. +def graph_pool_handle() -> _POOL_HANDLE: + r"""Return an opaque token representing the id of a graph memory pool. + + See :ref:`Graph memory management`. + + .. warning:: + This API is in beta and may change in future releases. + """ + return torch.cuda._POOL_HANDLE(_graph_pool_handle()) + + +# Python shim helps Sphinx process docstrings more reliably. +class CUDAGraph(torch._C._CUDAGraph): + r"""Wrapper around a CUDA graph. + + Arguments: + keep_graph (bool, optional): If ``keep_graph=False``, the + cudaGraphExec_t will be instantiated on GPU at the end of + ``capture_end`` and the underlying cudaGraph_t will be + destroyed. Users who want to query or otherwise modify the + underlying cudaGraph_t before instantiation can set + ``keep_graph=True`` and access it via ``raw_cuda_graph`` after + ``capture_end``. Note that the cudaGraphExec_t will not be + instantiated at the end of ``capture_end`` in this + case. Instead, it will be instantiated via an explicit called + to ``instantiate`` or automatically on the first call to + ``replay`` if ``instantiate`` was not already called. Calling + ``instantiate`` manually before ``replay`` is recommended to + prevent increased latency on the first call to ``replay``. It + is allowed to modify the raw cudaGraph_t after first calling + ``instantiate``, but the user must call ``instantiate`` again + manually to make sure the instantiated graph has these + changes. Pytorch has no means of tracking these changes. + + .. warning:: + This API is in beta and may change in future releases. + + """ + + def __new__(cls, keep_graph: bool = False) -> Self: + return super().__new__(cls, keep_graph) + + def capture_begin( + self, pool: Optional[_POOL_HANDLE] = None, capture_error_mode: str = "global" + ) -> None: + r"""Begin capturing CUDA work on the current stream. + + Typically, you shouldn't call ``capture_begin`` yourself. + Use :class:`~torch.cuda.graph` or :func:`~torch.cuda.make_graphed_callables`, + which call ``capture_begin`` internally. + + Arguments: + pool (optional): Token (returned by :func:`~torch.cuda.graph_pool_handle` or + :meth:`other_Graph_instance.pool()`) that hints this graph may share memory + with the indicated pool. See :ref:`Graph memory management`. + capture_error_mode (str, optional): specifies the cudaStreamCaptureMode for the graph capture stream. + Can be "global", "thread_local" or "relaxed". During cuda graph capture, some actions, such as cudaMalloc, + may be unsafe. "global" will error on actions in other threads, "thread_local" will only error for + actions in the current thread, and "relaxed" will not error on these actions. Do NOT change this setting + unless you're familiar with `cudaStreamCaptureMode `_ + """ # noqa: B950 + super().capture_begin(pool=pool, capture_error_mode=capture_error_mode) + + def capture_end(self) -> None: + r"""End CUDA graph capture on the current stream. + + After ``capture_end``, ``replay`` may be called on this instance. + + Typically, you shouldn't call ``capture_end`` yourself. + Use :class:`~torch.cuda.graph` or :func:`~torch.cuda.make_graphed_callables`, + which call ``capture_end`` internally. + """ + super().capture_end() + + def instantiate(self) -> None: + r"""Instantiate the CUDA graph. Will be called by + ``capture_end`` if ``keep_graph=False``, or by ``replay`` if + ``keep_graph=True`` and ``instantiate`` has not already been + explicitly called. Does not destroy the cudaGraph_t returned + by ``raw_cuda_graph``. + """ + super().instantiate() + + def replay(self) -> None: + r"""Replay the CUDA work captured by this graph.""" + super().replay() + + def reset(self) -> None: + r"""Delete the graph currently held by this instance.""" + super().reset() + + def pool(self) -> _POOL_HANDLE: + r"""Return an opaque token representing the id of this graph's memory pool. + + This id can optionally be passed to another graph's ``capture_begin``, + which hints the other graph may share the same memory pool. + """ + return super().pool() + + def enable_debug_mode(self) -> None: + r"""Enable debugging mode for CUDAGraph.debug_dump.""" + return super().enable_debug_mode() + + def debug_dump(self, debug_path: str) -> None: + r""" + Arguments: + debug_path (required): Path to dump the graph to. + + Calls a debugging function to dump the graph if the debugging is + enabled via CUDAGraph.enable_debug_mode() + """ + return super().debug_dump(debug_path) + + def raw_cuda_graph(self) -> int: + r"""Returns the underlying cudaGraph_t. ``keep_graph`` must be True. + + See the following for APIs for how to manipulate this object: `Graph Managmement `_ and `cuda-python Graph Management bindings `_ + """ # noqa: B950 + return super().raw_cuda_graph() + + def raw_cuda_graph_exec(self) -> int: + r"""Returns the underlying cudaGraphExec_t. ``instantiate`` must have been called if ``keep_graph`` is True, or ``capture_end`` must have been called if ``keep_graph`` is False. If you call ``instantiate()`` after ``raw_cuda_graph_exec()``, the previously returned cudaGraphExec_t will be destroyed. It is your responsibility not to use this object after destruction. + + See the following for APIs for how to manipulate this object: `Graph Execution `_ and `cuda-python Graph Execution bindings `_ + """ # noqa: B950 + return super().raw_cuda_graph_exec() + + +class graph: + r"""Context-manager that captures CUDA work into a :class:`torch.cuda.CUDAGraph` object for later replay. + + See :ref:`CUDA Graphs ` for a general introduction, + detailed use, and constraints. + + Arguments: + cuda_graph (torch.cuda.CUDAGraph): Graph object used for capture. + pool (optional): Opaque token (returned by a call to :func:`~torch.cuda.graph_pool_handle()` or + :meth:`other_Graph_instance.pool()`) hinting this graph's capture + may share memory from the specified pool. See :ref:`Graph memory management`. + stream (torch.cuda.Stream, optional): If supplied, will be set as the current stream in the context. + If not supplied, ``graph`` sets its own internal side stream as the current stream in the context. + capture_error_mode (str, optional): specifies the cudaStreamCaptureMode for the graph capture stream. + Can be "global", "thread_local" or "relaxed". During cuda graph capture, some actions, such as cudaMalloc, + may be unsafe. "global" will error on actions in other threads, "thread_local" will only error for + actions in the current thread, and "relaxed" will not error on actions. Do NOT change this setting + unless you're familiar with `cudaStreamCaptureMode `_ + + .. note:: + For effective memory sharing, if you pass a ``pool`` used by a previous capture and the previous capture + used an explicit ``stream`` argument, you should pass the same ``stream`` argument to this capture. + + .. warning:: + This API is in beta and may change in future releases. + + .. _cudaStreamCaptureMode: + https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__STREAM.html#group__CUDART__STREAM_1g9d0535d93a214cbf126835257b16ba85 + """ # noqa: B950 + + default_capture_stream: Optional[torch.cuda.Stream] = None + + def __init__( + self, + cuda_graph: CUDAGraph, + pool: Optional[_POOL_HANDLE] = None, + stream: Optional[torch.cuda.Stream] = None, + capture_error_mode: str = "global", + ): + # Lazy-init of default_capture_stream helps avoid circular-import errors. + # Not thread safe, but graphs already have the general (explicitly documented) + # restriction that only one capture may be underway at a time in the process. + if self.__class__.default_capture_stream is None: + self.__class__.default_capture_stream = torch.cuda.Stream() + + self.pool: Union[tuple[()], tuple[_POOL_HANDLE]] = ( + () if pool is None else (pool,) + ) + self.capture_stream = ( + stream if stream is not None else self.__class__.default_capture_stream + ) + assert self.capture_stream is not None + self.stream_ctx = torch.cuda.stream(self.capture_stream) + self.cuda_graph = cuda_graph + self.capture_error_mode = capture_error_mode + + def __enter__(self) -> None: + # Free as much memory as we can for the graph + torch.cuda.synchronize() + + if torch.compiler.config.force_cudagraph_gc: + # Originally we unconditionally garbage collected here. On one hand + # that's nice because we have a chance to collect more memory, but + # on the other hand it is REALLY expensive, especially for doing + # multiple cudagraph captures in a row. In theory it will only help + # when a dead python cycle is holding onto CUDA memory. + gc.collect() + + torch.cuda.empty_cache() + + # Stackoverflow seems comfortable with this pattern + # https://stackoverflow.com/questions/26635684/calling-enter-and-exit-manually#39172487 + self.stream_ctx.__enter__() + + self.cuda_graph.capture_begin( + # type: ignore[misc] + *self.pool, + # pyrefly: ignore [bad-keyword-argument] + capture_error_mode=self.capture_error_mode, + ) + + def __exit__(self, *args: object) -> None: + self.cuda_graph.capture_end() + self.stream_ctx.__exit__(*args) + # returning None should propagate exceptions from either capture_end or stream_ctx.__exit__() + + +_ModuleOrCallable: TypeAlias = Union["torch.nn.Module", Callable[..., object]] + + +@overload +def make_graphed_callables( + callables: _ModuleOrCallable, + sample_args: tuple[Tensor, ...], + num_warmup_iters: int = 3, + allow_unused_input: bool = False, + pool: Optional[_POOL_HANDLE] = None, +) -> _ModuleOrCallable: ... + + +@overload +def make_graphed_callables( + callables: tuple[_ModuleOrCallable, ...], + sample_args: tuple[tuple[Tensor, ...], ...], + num_warmup_iters: int = 3, + allow_unused_input: bool = False, + pool: Optional[_POOL_HANDLE] = None, +) -> tuple[_ModuleOrCallable, ...]: ... + + +def make_graphed_callables( + callables: Union[_ModuleOrCallable, tuple[_ModuleOrCallable, ...]], + sample_args: Union[tuple[Tensor, ...], tuple[tuple[Tensor, ...], ...]], + num_warmup_iters: int = 3, + allow_unused_input: bool = False, + pool: Optional[_POOL_HANDLE] = None, +) -> Union[_ModuleOrCallable, tuple[_ModuleOrCallable, ...]]: + r"""Accept callables (functions or :class:`nn.Module`\ s) and returns graphed versions. + + Each graphed callable's forward pass runs its source callable's + forward CUDA work as a CUDA graph inside a single autograd node. + + The graphed callable's forward pass also appends + a backward node to the autograd graph. During backward, this node runs the + callable's backward work as a CUDA graph. + + Therefore, each graphed callable should be a drop-in replacement for its source callable + in an autograd-enabled training loop. + + See :ref:`Partial-network capture` for detailed use and constraints. + + If you pass a tuple of several callables, their captures will use the same memory pool. + See :ref:`Graph memory management` for when this is appropriate. + + Arguments: + callables (torch.nn.Module or Python function, or tuple of these): Callable or callables to graph. + See :ref:`Graph memory management` for when passing a tuple of callables + is appropriate. If you pass a tuple of callables, their order in the tuple must be the same order + they'll run in the live workload. + sample_args (tuple of Tensors, or tuple of tuples of Tensors): Samples args for each callable. + If a single callable was passed, ``sample_args`` must be a single tuple of argument Tensors. + If a tuple of callables was passed, ``sample_args`` must be tuple of tuples of argument Tensors. + num_warmup_iters (int): The number of warmup iterations. Currently, ``DataDistributedParallel`` needs + 11 iterations for warm up. Default: ``3``. + allow_unused_input (bool): If False, specifying inputs that were not used when computing outputs + (and therefore their grad is always zero) is an error. Defaults to False. + pool (optional): Token (returned by :func:`~torch.cuda.graph_pool_handle` or + :meth:`other_Graph_instance.pool()`) that hints this graph may share memory + with the indicated pool. See :ref:`Graph memory management`. + .. note:: + The ``requires_grad`` state of each Tensor in ``sample_args`` must match the state + that's expected for the corresponding real input in the training loop. + + .. warning:: + This API is in beta and may change in future releases. + + .. warning:: + ``sample_args`` for each callable must contain only Tensors. Other types are not allowed. + + .. warning:: + Returned callables do not support higher order differentiation (e.g., double backward). + + .. warning:: + In any :class:`~torch.nn.Module` passed to :func:`~make_graphed_callables`, only parameters + may be trainable. Buffers must have ``requires_grad=False``. + + .. warning:: + After you pass a :class:`torch.nn.Module` through :func:`~make_graphed_callables`, + you may not add or remove any of that Module's parameters or buffers. + + .. warning:: + :class:`torch.nn.Module`\s passed to :func:`~torch.cuda.make_graphed_callables` must not have module hooks + registered on them at the time they are passed. However, registering hooks on modules *after* passing them + through :func:`~torch.cuda.make_graphed_callables` is allowed. + + .. warning:: + When running a graphed callable, you must pass its arguments in the same order and format + they appeared in that callable's ``sample_args``. + + .. warning:: + The automatic mixed precision is supported in :func:`~torch.cuda.make_graphed_callables` only with disabled + caching. The context manager `torch.cuda.amp.autocast()` must have `cache_enabled=False`. + """ + if torch.is_autocast_enabled() and torch.is_autocast_cache_enabled(): + raise RuntimeError( + "make_graphed_callables does not support the autocast caching. Please set `cache_enabled=False`." + ) + + just_one_callable = False + + _sample_args: tuple[tuple[Tensor, ...], ...] + if not isinstance(callables, tuple): + just_one_callable = True + callables = (callables,) + _sample_args = (typing.cast(tuple[Tensor, ...], sample_args),) + else: + _sample_args = typing.cast(tuple[tuple[Tensor, ...], ...], sample_args) + + flatten_sample_args = [] + + for c, args in zip(callables, _sample_args): + if isinstance(c, torch.nn.Module): + assert ( + len(c._backward_hooks) == 0 + and len(c._forward_hooks) == 0 + and len(c._forward_pre_hooks) == 0 + ), ( + "Modules must not have hooks registered at the time they are passed. However, registering hooks " + + "on modules after passing them through make_graphed_callables is allowed." + ) + assert all(b.requires_grad is False for b in c.buffers()), ( + "In any :class:`~torch.nn.Module` passed to " + + ":func:`~make_graphed_callables`, only parameters may be trainable. All buffers must have " + + "``requires_grad=False``." + ) + flatten_arg = torch.utils._pytree.arg_tree_leaves(*args) + flatten_sample_args.append(tuple(flatten_arg)) + assert all(isinstance(arg, torch.Tensor) for arg in flatten_arg), ( + "In the beta API, sample_args " + + "for each callable must contain only Tensors. Other types are not allowed." + ) + + # If a callable is an nn.Module, its graph's full input surface is the args the user explicitly + # passes to forward (ie, its sample_args) AND the module's parameter attributes. + per_callable_len_user_args = [len(args) for args in flatten_sample_args] + per_callable_module_params = [ + tuple(c.parameters()) if isinstance(c, torch.nn.Module) else () + for c in callables + ] + per_callable_static_input_surfaces = [ + flatten_sample_args[i] + per_callable_module_params[i] + for i in range(len(callables)) + ] + + fwd_graphs = [torch.cuda.CUDAGraph() for _ in range(len(callables))] + bwd_graphs = [torch.cuda.CUDAGraph() for _ in range(len(callables))] + + mempool = graph_pool_handle() if pool is None else pool + + # Warmup + # Hopefully prevents cudnn benchmarking and other lazy-initialization cuda work + # from ending up in any captures. + torch.cuda.synchronize() + with torch.cuda.stream(torch.cuda.Stream()): + for func, args, static_input_surface in zip( + callables, _sample_args, per_callable_static_input_surfaces + ): + grad_inputs, outputs, outputs_grad = None, None, None + for _ in range(num_warmup_iters): + outputs = torch.utils._pytree.tree_leaves(func(*args)) + outputs_grad = tuple(o for o in outputs if o.requires_grad) + if len(outputs_grad) > 0: + grad_inputs = torch.autograd.grad( + outputs=outputs_grad, + inputs=tuple( + i for i in static_input_surface if i.requires_grad + ), + grad_outputs=tuple( + torch.empty_like(o) for o in outputs if o.requires_grad + ), + only_inputs=True, + allow_unused=allow_unused_input, + ) + for v in [outputs, outputs_grad, grad_inputs]: + del v + + torch.cuda.synchronize() + + # All captures here share a mempool. To avoid replays corrupting each other's memory, + # the safest approach is to capture all passes in the same order they'll run: + # fwd 1, fwd 2, ... fwd N, then bwd N, bwd N-1, ... bwd 1. + + # Capture forward graphs + per_callable_static_outputs = [] + per_callable_output_unflatten_spec = [] + for func, args, fwd_graph in zip(callables, _sample_args, fwd_graphs): + with torch.cuda.graph(fwd_graph, pool=mempool): + func_outputs = func(*args) + + flatten_outputs, spec = torch.utils._pytree.tree_flatten(func_outputs) + per_callable_static_outputs.append(tuple(flatten_outputs)) + per_callable_output_unflatten_spec.append(spec) + + # Capture backward graphs in reverse order + per_callable_static_grad_outputs = [] + per_callable_static_grad_inputs = [] + for static_input_surface, static_outputs, bwd_graph in zip( + reversed(per_callable_static_input_surfaces), + reversed(per_callable_static_outputs), + reversed(bwd_graphs), + ): + # For now, assumes all static_outputs require grad + # assert all(o.requires_grad for o in static_outputs), "Outputs of graphed callables must require grad." + static_grad_outputs = tuple( + torch.empty_like(o) if o.requires_grad else None for o in static_outputs + ) + + outputs_grad = tuple(o for o in static_outputs if o.requires_grad) + grad_inputs = None + if len(outputs_grad) > 0: + with torch.cuda.graph(bwd_graph, pool=mempool): + grad_inputs = torch.autograd.grad( + outputs=outputs_grad, + inputs=tuple(i for i in static_input_surface if i.requires_grad), + grad_outputs=tuple(o for o in static_grad_outputs if o is not None), + only_inputs=True, + allow_unused=allow_unused_input, + ) + + # Constructs a tuple suitable for returning from Graphed.backward: + # Pads out the actually-needed grads with Nones in gradient slots for inputs that don't require grad. + # I couldn't think of a slick one-liner for this pattern. + static_grad_inputs = [] + grad_idx = 0 + for arg in static_input_surface: + if arg.requires_grad and grad_inputs is not None: + static_grad_inputs.append(grad_inputs[grad_idx]) + grad_idx += 1 + else: + static_grad_inputs.append(None) # type: ignore[arg-type] + static_grad_inputs = tuple(static_grad_inputs) # type: ignore[assignment] + + per_callable_static_grad_outputs.append(static_grad_outputs) + per_callable_static_grad_inputs.append(static_grad_inputs) + + # Reverses the most recent two lists + per_callable_static_grad_outputs.reverse() + per_callable_static_grad_inputs.reverse() + # Now for every per_callable list, per_callable_*[i] holds the stuff for the ith callable. + + def make_graphed_autograd_function( + fwd_graph: CUDAGraph, + bwd_graph: CUDAGraph, + module_params: tuple[torch.nn.Parameter, ...], + len_user_args: int, + output_unflatten_spec: torch.utils._pytree.TreeSpec, + static_input_surface: tuple[Tensor, ...], + static_outputs: tuple[Tensor, ...], + static_grad_outputs: tuple[Optional[Tensor], ...], + static_grad_inputs: tuple[Tensor, ...], + ) -> Callable[..., object]: + class Graphed(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(ctx: object, *inputs: Tensor) -> tuple[Tensor, ...]: + # At this stage, only the user args may (potentially) be new tensors. + for i in range(len_user_args): + if static_input_surface[i].data_ptr() != inputs[i].data_ptr(): + static_input_surface[i].copy_(inputs[i]) + fwd_graph.replay() + assert isinstance(static_outputs, tuple) + return tuple(o.detach() for o in static_outputs) + + @staticmethod + @torch.autograd.function.once_differentiable + # pyrefly: ignore [bad-override] + def backward(ctx: object, *grads: Tensor) -> tuple[Tensor, ...]: + assert len(grads) == len(static_grad_outputs) + for g, grad in zip(static_grad_outputs, grads): + if g is not None: + # don't copy if autograd gods have been kind and the + # incoming grad is already in the right place + if g.data_ptr() != grad.data_ptr(): + g.copy_(grad) + bwd_graph.replay() + + # Input args that didn't require grad expect a None gradient. + assert isinstance(static_grad_inputs, tuple) + return tuple( + # pyrefly: ignore [bad-argument-type] + b.detach() if b is not None else b + for b in static_grad_inputs + ) + + def functionalized(*user_args: object) -> object: + # Runs the autograd function with inputs == all inputs to the graph that might require grad + # (explicit user args + module parameters) + # Assumes module params didn't change since capture. + flatten_user_args = torch.utils._pytree.arg_tree_leaves(*user_args) + out = Graphed.apply(*(tuple(flatten_user_args) + module_params)) + return torch.utils._pytree.tree_unflatten(out, output_unflatten_spec) + + return functionalized + + # Put together the final graphed callables + ret: list[_ModuleOrCallable] = [] + for i, func in enumerate(callables): + graphed = make_graphed_autograd_function( + fwd_graphs[i], + bwd_graphs[i], + per_callable_module_params[i], + per_callable_len_user_args[i], + per_callable_output_unflatten_spec[i], + per_callable_static_input_surfaces[i], + per_callable_static_outputs[i], + per_callable_static_grad_outputs[i], + per_callable_static_grad_inputs[i], + ) + + if isinstance(func, torch.nn.Module): + + def make_graphed_forward( + func: torch.nn.Module, + graph_training_state: bool, + graphed: Callable[_P, _R], + orig_fwd: Callable[_P, _R], + ) -> Callable[_P, _R]: + def new_fwd(*user_args: _P.args, **user_kwargs: _P.kwargs) -> _R: + # If the module's training-or-eval state matches what we graphed, + # run the graph, otherwise run the original forward method + if func.training == graph_training_state: + return graphed(*user_args, **user_kwargs) + else: + return orig_fwd(*user_args, **user_kwargs) + + return new_fwd + + func.forward = make_graphed_forward( + func, func.training, graphed, func.forward + ) + ret.append(func) + else: + ret.append(graphed) + + if just_one_callable: + return ret[0] + + return tuple(ret) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/green_contexts.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/green_contexts.py new file mode 100644 index 0000000000000000000000000000000000000000..2b8f7f176499818c3cdc842683b825771031eea5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/green_contexts.py @@ -0,0 +1,43 @@ +import torch + + +_GreenContext = object +SUPPORTED = False + +if hasattr(torch._C, "_CUDAGreenContext"): + _GreenContext = torch._C._CUDAGreenContext # type: ignore[misc] + SUPPORTED = True + + +# Python shim helps Sphinx process docstrings more reliably. +# pyrefly: ignore [invalid-inheritance] +class GreenContext(_GreenContext): + r"""Wrapper around a CUDA green context. + + .. warning:: + This API is in beta and may change in future releases. + """ + + @staticmethod + def create(num_sms: int, device_id: int = 0) -> _GreenContext: + r"""Create a CUDA green context. + + Arguments: + num_sms (int): The number of SMs to use in the green context. + device_id (int, optional): The device index of green context. + """ + if not SUPPORTED: + raise RuntimeError("PyTorch was not built with Green Context support!") + return _GreenContext.create(num_sms, device_id) # type: ignore[attr-defined] + + # Note that these functions are bypassed by we define them here + # for Sphinx documentation purposes + def set_context(self) -> None: # pylint: disable=useless-parent-delegation + r"""Make the green context the current context.""" + return super().set_context() # type: ignore[misc] + + def pop_context(self) -> None: # pylint: disable=useless-parent-delegation + r"""Assuming the green context is the current context, pop it from the + context stack and restore the previous context. + """ + return super().pop_context() # type: ignore[misc] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/jiterator.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/jiterator.py new file mode 100644 index 0000000000000000000000000000000000000000..6eaa54b5b795c78ed97bb25e335d3bf978410474 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/jiterator.py @@ -0,0 +1,189 @@ +# mypy: allow-untyped-defs +import re +from collections.abc import Callable + +import torch +from torch import Tensor + + +__all__: list[str] = [] + + +class _CodeParser: + def __init__(self, code_string: str): + optional_ws = r"\s*" + required_ws = r"\s+" + template_params = r"(?P\<.+\>)" + return_type = r"(?P\w+)" + function_name = r"(?P\w+)" + function_params = r"(?P\(.+\))" + function_body = r"(?P\{.+\})" + + pattern = ( + optional_ws + + "template" + + optional_ws + + template_params + + optional_ws + + return_type + + required_ws + + function_name + + optional_ws + + function_params + + optional_ws + + function_body + + optional_ws + ) + + result = re.match( + pattern, code_string, re.DOTALL + ) # DOTALL for matching multiline + + if result is None: + raise Exception( # noqa: TRY002 + f"Couldn't parse code, please check correctness:\n {code_string}" + ) + + self.template_params = result["template_params"] + self.return_type = result["return_type"] + self.function_name = result["function_name"] + self.function_params = result["function_params"] + self.function_body = result["function_body"] + + +class _JittedFunction: + def __init__( + self, code_string: str, return_by_ref: bool, num_outputs: int, **kwargs + ): + self.code_string = code_string + + assert return_by_ref or num_outputs == 1, ( + "Return by value only works for single output. " + ) + self.return_by_ref = return_by_ref + self.num_outputs = num_outputs + + parsed_code = _CodeParser(code_string) + self.kernel_name = parsed_code.function_name + + self.kwargs_dict = kwargs + self.is_cuda_available = torch.cuda.is_available() + + def __call__(self, *tensors: Tensor, **kwargs): + # Jiterator follow torch.cuda's lazy initialization behavior + # Defer checking cuda's availability at the function invocation time + assert self.is_cuda_available, ( + "Jiterator is only supported on CUDA and ROCm GPUs, none are available." + ) + + assert len(tensors) <= 8, "jiterator only supports up to 8 tensor inputs." + + expanded_kwargs = self.kwargs_dict.copy() + for key, value in kwargs.items(): + if key in self.kwargs_dict: + expanded_kwargs[key] = value + else: + raise KeyError(f"{key} is not declared in function definition") + + return torch._C._cuda_jiterator_compile_and_launch_kernel( + self.code_string, + self.kernel_name, + self.return_by_ref, + self.num_outputs, + tensors, + expanded_kwargs, + ) + + +def _create_jit_fn(code_string: str, **kwargs) -> Callable: + """ + Create a jiterator-generated cuda kernel for an elementwise op. + + The code string has to be a valid CUDA function that describes the computation for a single element. The code + string has to follow the c++ template pattern, as shown in the example below. This function will be inlined + into elementwise kernel template, and compiled on the fly. Compiled kernel will be cached in memory, as well as + local temp dir. + + Jiterator-generated kernels accepts noncontiguous tensors, and supports broadcasting and type promotion. + + Args: + code_string (str): CUDA code string to be compiled by jiterator. The entry functor must return by value. + kwargs (Dict, optional): Keyword arguments for generated function + + Example:: + + code_string = "template T my_kernel(T x, T y, T alpha) { return -x + alpha * y; }" + jitted_fn = create_jit_fn(code_string, alpha=1.0) + a = torch.rand(3, device="cuda") + b = torch.rand(3, device="cuda") + # invoke jitted function like a regular python function + result = jitted_fn(a, b, alpha=3.14) + + code_string also allows multiple function definitions, and the last function will be treated as the entry function. + + Example:: + + code_string = ( + "template T util_fn(T x, T y) { return ::sin(x) + ::cos(y); }" + ) + code_string += "template T my_kernel(T x, T y, T val) { return ::min(val, util_fn(x, y)); }" + jitted_fn = create_jit_fn(code_string, val=0.0) + a = torch.rand(3, device="cuda") + b = torch.rand(3, device="cuda") + # invoke jitted function like a regular python function + result = jitted_fn(a, b) # using default val=0.0 + + Jiterator can be used together with python registration to override an operator's cuda kernel. + Following example is overriding gelu's cuda kernel with relu. + + Example:: + + code_string = "template T my_gelu(T a) { return a > 0 ? a : 0; }" + my_gelu = create_jit_fn(code_string) + my_lib = torch.library.Library("aten", "IMPL") + my_lib.impl("aten::gelu", my_gelu, "CUDA") + # torch.nn.GELU and torch.nn.function.gelu are now overridden + a = torch.rand(3, device="cuda") + torch.allclose(torch.nn.functional.gelu(a), torch.nn.functional.relu(a)) + + .. warning:: + This API is in beta and may change in future releases. + + .. warning:: + This API only supports up to 8 inputs and 1 output + + .. warning:: + All input tensors must live in CUDA device + """ + return _JittedFunction(code_string, return_by_ref=False, num_outputs=1, **kwargs) + + +def _create_multi_output_jit_fn( + code_string: str, num_outputs: int, **kwargs +) -> Callable: + """ + Create a jiterator-generated cuda kernel for an elementwise op that supports returning one or more outputs. + + Args: + code_string (str): CUDA code string to be compiled by jiterator. The entry functor must return value by reference. + num_outputs(int): number of outputs return by the kernel + kwargs (Dict, optional): Keyword arguments for generated function + + Example:: + + code_string = "template void my_kernel(T x, T y, T alpha, T& out) { out = -x + alpha * y; }" + jitted_fn = create_jit_fn(code_string, alpha=1.0) + a = torch.rand(3, device="cuda") + b = torch.rand(3, device="cuda") + # invoke jitted function like a regular python function + result = jitted_fn(a, b, alpha=3.14) + + .. warning:: + This API is in beta and may change in future releases. + + .. warning:: + This API only supports up to 8 inputs and 8 outputs + """ + return _JittedFunction( + code_string, return_by_ref=True, num_outputs=num_outputs, **kwargs + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/memory.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..de5fd7baa8cd20a311530323edfcb7acfcf6e22b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/memory.py @@ -0,0 +1,1463 @@ +# mypy: allow-untyped-defs +r"""This package adds support for device memory management implemented in CUDA.""" + +import collections +import contextlib +import ctypes +import os +import pickle +import re +import sys +import warnings +from inspect import signature +from typing import Any, cast, Literal, Optional, TYPE_CHECKING, TypedDict +from typing_extensions import deprecated, NotRequired + +import torch +from torch import _C +from torch._utils import _dummy_type + +from . import ( + _get_amdsmi_device_index, + _get_device_index, + _get_nvml_device_index, + _lazy_init, + is_initialized, +) +from ._memory_viz import memory as _memory, segments as _segments + + +if TYPE_CHECKING: + from torch.types import Device + + +# Type definitions for memory profiler +class _Frame(TypedDict): + """Frame information from memory profiler snapshots.""" + + filename: str + line: int + name: str + # Fields added by FX augmentation (optional) + fx_node_op: NotRequired[str] + fx_node_name: NotRequired[str] + fx_node_target: NotRequired[str] + fx_original_trace: NotRequired[str] + + +class _Block(TypedDict): + """Memory block information.""" + + size: int + requested_size: int + address: int + state: str + frames: list[_Frame] + + +class _Segment(TypedDict): + """Memory segment information.""" + + address: int + total_size: int + stream: int + segment_type: str + allocated_size: int + active_size: int + blocks: list[_Block] + + +class _TraceEntry(TypedDict): + """Memory trace entry information.""" + + action: str + addr: NotRequired[int] + frames: list[_Frame] + size: int + stream: int + device_free: NotRequired[int] + + +class _Snapshot(TypedDict): + """Memory snapshot structure.""" + + segments: list[_Segment] + device_traces: NotRequired[list[list[_TraceEntry]]] + + +__all__ = [ + "caching_allocator_alloc", + "caching_allocator_delete", + "caching_allocator_enable", + "get_per_process_memory_fraction", + "set_per_process_memory_fraction", + "empty_cache", + "memory_stats", + "memory_stats_as_nested_dict", + "reset_accumulated_memory_stats", + "reset_peak_memory_stats", + "reset_max_memory_allocated", + "reset_max_memory_cached", + "host_memory_stats", + "host_memory_stats_as_nested_dict", + "reset_accumulated_host_memory_stats", + "reset_peak_host_memory_stats", + "memory_allocated", + "max_memory_allocated", + "memory_reserved", + "max_memory_reserved", + "memory_cached", + "max_memory_cached", + "memory_snapshot", + "memory_summary", + "list_gpu_processes", + "mem_get_info", + "get_allocator_backend", + "CUDAPluggableAllocator", + "change_current_allocator", + "MemPool", + "use_mem_pool", +] + + +if not hasattr(torch._C, "_cuda_CUDAAllocator"): + # Define dummy base classes + torch._C.__dict__["_cuda_CUDAAllocator"] = _dummy_type("_cuda_CUDAAllocator") + + +if not hasattr(torch._C, "_MemPool"): + # Define dummy base classes + torch._C.__dict__["_MemPool"] = _dummy_type("_MemPool") + torch._C.__dict__["_cuda_beginAllocateToPool"] = _dummy_type( + "_cuda_beginAllocateToPool" + ) + torch._C.__dict__["_cuda_beginAllocateCurrentThreadToPool"] = _dummy_type( + "_cuda_beginAllocateCurrentThreadToPool" + ) + torch._C.__dict__["_cuda_endAllocateToPool"] = _dummy_type( + "_cuda_endAllocateToPool" + ) + torch._C.__dict__["_cuda_releasePool"] = _dummy_type("_cuda_releasePool") + +from torch._C import ( # noqa: F401 + _cuda_beginAllocateCurrentThreadToPool, + _cuda_beginAllocateToPool, + _cuda_CUDAAllocator, + _cuda_endAllocateToPool, + _cuda_releasePool, + _MemPool, +) + + +def _host_allocator(): + _lazy_init() + return torch._C._cuda_cudaHostAllocator() + + +@contextlib.contextmanager +def _free_mutex(): + torch._C._cuda_lock_mutex() + try: + yield + finally: + torch._C._cuda_unlock_mutex() + + +def caching_allocator_alloc(size, device: "Device" = None, stream=None): + r"""Perform a memory allocation using the CUDA memory allocator. + + Memory is allocated for a given device and a stream, this + function is intended to be used for interoperability with other + frameworks. Allocated memory is released through + :func:`~torch.cuda.caching_allocator_delete`. + + Args: + size (int): number of bytes to be allocated. + device (torch.device or int, optional): selected device. If it is + ``None`` the default CUDA device is used. + stream (torch.cuda.Stream or int, optional): selected stream. If is ``None`` then + the default stream for the selected device is used. + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + if device is None: + device = torch.cuda.current_device() + device = _get_device_index(device) + if stream is None: + stream = torch.cuda.current_stream(device) + if isinstance(stream, torch.cuda.streams.Stream): + stream = stream.cuda_stream + if not isinstance(stream, int): + raise TypeError( + "Invalid type for stream argument, must be " + "`torch.cuda.Stream` or `int` representing a pointer " + "to a existing stream" + ) + with torch.cuda.device(device): + return torch._C._cuda_cudaCachingAllocator_raw_alloc(size, stream) + + +def caching_allocator_delete(mem_ptr): + r"""Delete memory allocated using the CUDA memory allocator. + + Memory allocated with :func:`~torch.cuda.caching_allocator_alloc`. + is freed here. The associated device and stream are tracked inside + the allocator. + + Args: + mem_ptr (int): memory address to be freed by the allocator. + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + torch._C._cuda_cudaCachingAllocator_raw_delete(mem_ptr) + + +def caching_allocator_enable(value: bool = True) -> None: + r"""Enable or disable the CUDA memory allocator. On by default.""" + if is_initialized(): + torch._C._cuda_cudaCachingAllocator_enable(value) + + +def set_per_process_memory_fraction(fraction, device: "Device" = None) -> None: + r"""Set memory fraction for a process. + + The fraction is used to limit an caching allocator to allocated memory on a CUDA device. + The allowed value equals the total visible memory multiplied fraction. + If trying to allocate more than the allowed value in a process, will raise an out of + memory error in allocator. + + Args: + fraction(float): Range: 0~1. Allowed memory equals total_memory * fraction. + device (torch.device or int, optional): selected device. If it is + ``None`` the default CUDA device is used. + .. note:: + In general, the total available free memory is less than the total capacity. + """ + _lazy_init() + if device is None: + device = torch.cuda.current_device() + device = _get_device_index(device) + if not isinstance(fraction, float): + raise TypeError("Invalid type for fraction argument, must be `float`") + if fraction < 0 or fraction > 1: + raise ValueError(f"Invalid fraction value: {fraction}. Allowed range: 0~1") + + torch._C._cuda_setMemoryFraction(fraction, device) + + +def get_per_process_memory_fraction(device: "Device" = None) -> float: + r"""Get memory fraction for a process. + + Args: + device (torch.device or int, optional): selected device. If it is + ``None`` the default CUDA device is used. + Returns: + memory fraction, in range 0~1. Allowed memory equals total_memory * fraction. + """ + _lazy_init() + if device is None: + device = torch.cuda.current_device() + device = _get_device_index(device) + return torch._C._cuda_getMemoryFraction(device) + + +def empty_cache() -> None: + r"""Release all unoccupied cached memory currently held by the caching + allocator so that those can be used in other GPU application and visible in + `nvidia-smi`. + + .. note:: + :func:`~torch.cuda.empty_cache` doesn't increase the amount of GPU + memory available for PyTorch. However, it may help reduce fragmentation + of GPU memory in certain cases. See :ref:`cuda-memory-management` for + more details about GPU memory management. + """ + if is_initialized(): + torch._C._cuda_emptyCache() + + +def memory_stats(device: "Device" = None) -> dict[str, Any]: + r"""Return a dictionary of CUDA memory allocator statistics for a given device. + + The return value of this function is a dictionary of statistics, each of + which is a non-negative integer. + + Core statistics: + + - ``"allocated.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of allocation requests received by the memory allocator. + - ``"allocated_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of allocated memory. + - ``"segment.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of reserved segments from ``cudaMalloc()``. + - ``"reserved_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of reserved memory. + - ``"active.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of active memory blocks. + - ``"active_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of active memory. + - ``"inactive_split.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + number of inactive, non-releasable memory blocks. + - ``"inactive_split_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + amount of inactive, non-releasable memory. + + For these core statistics, values are broken down as follows. + + Pool type: + + - ``all``: combined statistics across all memory pools. + - ``large_pool``: statistics for the large allocation pool + (as of June 2025, for size >= 1MB allocations). + - ``small_pool``: statistics for the small allocation pool + (as of June 2025, for size < 1MB allocations). + + Metric type: + + - ``current``: current value of this metric. + - ``peak``: maximum value of this metric. + - ``allocated``: historical total increase in this metric. + - ``freed``: historical total decrease in this metric. + + In addition to the core statistics, we also provide some simple event + counters: + + - ``"num_alloc_retries"``: number of failed ``cudaMalloc`` calls that + result in a cache flush and retry. + - ``"num_ooms"``: number of out-of-memory errors thrown. + - ``"num_sync_all_streams"``: number of ``synchronize_and_free_events`` calls. + - ``"num_device_alloc"``: number of CUDA allocation calls. This includes both + cuMemMap and cudaMalloc. + - ``"num_device_free"``: number of CUDA free calls. This includes both cuMemUnmap + and cudaFree. + + The caching allocator can be configured via ENV to not split blocks larger than a + defined size (see Memory Management section of the Cuda Semantics documentation). + This helps avoid memory fragmentation but may have a performance + penalty. Additional outputs to assist with tuning and evaluating impact: + + - ``"max_split_size"``: blocks above this size will not be split. + - ``"oversize_allocations.{current,peak,allocated,freed}"``: + number of over-size allocation requests received by the memory allocator. + - ``"oversize_segments.{current,peak,allocated,freed}"``: + number of over-size reserved segments from ``cudaMalloc()``. + + The caching allocator can be configured via ENV to round memory allocations in order + to reduce fragmentation. Sometimes the overhead from rounding can be higher than + the fragmentation it helps reduce. The following stat can be used to check if + rounding adds too much overhead: + + - ``"requested_bytes.{all,large_pool,small_pool}.{current,peak,allocated,freed}"``: + memory requested by client code, compare this with allocated_bytes to check if + allocation rounding adds too much overhead. + + Args: + device (torch.device or int, optional): selected device. Returns + statistics for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + + .. note:: + With :ref:`backend:cudaMallocAsync`, some stats are not + meaningful, and are always reported as zero. + """ + result = [] + + def _recurse_add_to_result(prefix, obj): + if isinstance(obj, dict): + if len(prefix) > 0: + prefix += "." + for k, v in obj.items(): + _recurse_add_to_result(prefix + k, v) + else: + result.append((prefix, obj)) + + stats = memory_stats_as_nested_dict(device=device) + _recurse_add_to_result("", stats) + result.sort() + + return collections.OrderedDict(result) + + +def memory_stats_as_nested_dict(device: "Device" = None) -> dict[str, Any]: + r"""Return the result of :func:`~torch.cuda.memory_stats` as a nested dictionary.""" + if not is_initialized(): + return {} + device = _get_device_index(device, optional=True) + return torch._C._cuda_memoryStats(device) + + +def reset_accumulated_memory_stats(device: "Device" = None) -> None: + r"""Reset the "accumulated" (historical) stats tracked by the CUDA memory allocator. + + See :func:`~torch.cuda.memory_stats` for details. Accumulated stats correspond to + the `"allocated"` and `"freed"` keys in each individual stat dict, as well as + `"num_alloc_retries"` and `"num_ooms"`. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + device = _get_device_index(device, optional=True) + return torch._C._cuda_resetAccumulatedMemoryStats(device) + + +def reset_peak_memory_stats(device: "Device" = None) -> None: + r"""Reset the "peak" stats tracked by the CUDA memory allocator. + + See :func:`~torch.cuda.memory_stats` for details. Peak stats correspond to the + `"peak"` key in each individual stat dict. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + device = _get_device_index(device, optional=True) + return torch._C._cuda_resetPeakMemoryStats(device) + + +def host_memory_stats() -> dict[str, Any]: + r"""Return a dictionary of CUDA memory allocator statistics for a given device. + + The return value of this function is a dictionary of statistics, each of + which is a non-negative integer. + + Core statistics: + + - ``"allocated.{current,peak,allocated,freed}"``: + number of allocation requests received by the memory allocator. + - ``"allocated_bytes.{current,peak,allocated,freed}"``: + amount of allocated memory. + - ``"segment.{current,peak,allocated,freed}"``: + number of reserved segments from ``cudaMalloc()``. + - ``"reserved_bytes.{current,peak,allocated,freed}"``: + amount of reserved memory. + + For these core statistics, values are broken down as follows. + + Metric type: + + - ``current``: current value of this metric. + - ``peak``: maximum value of this metric. + - ``allocated``: historical total increase in this metric. + - ``freed``: historical total decrease in this metric. + + In addition to the core statistics, we also provide some simple event + counters: + + - ``"num_host_alloc"``: number of CUDA allocation calls. This includes both + cudaHostAlloc and cudaHostRegister. + - ``"num_host_free"``: number of CUDA free calls. This includes both cudaHostFree + and cudaHostUnregister. + + Finally, we also provide some simple timing counters: + + - ``"host_alloc_time.{total,max,min,count,avg}"``: + timing of allocation requests going through CUDA calls. + - ``"host_free_time.{total,max,min,count,avg}"``: + timing of free requests going through CUDA calls. + + For these timing statistics, values are broken down as follows. + + Metric type: + + - ``total``: total time spent. + - ``max``: maximum value per call. + - ``min``: minimum value per call. + - ``count``: number of times it was called. + - ``avg``: average time per call. + """ + result = [] + + def _recurse_add_to_result(prefix, obj): + if isinstance(obj, dict): + if len(prefix) > 0: + prefix += "." + for k, v in obj.items(): + _recurse_add_to_result(prefix + k, v) + else: + result.append((prefix, obj)) + + stats = host_memory_stats_as_nested_dict() + _recurse_add_to_result("", stats) + result.sort() + + return collections.OrderedDict(result) + + +def host_memory_stats_as_nested_dict() -> dict[str, Any]: + r"""Return the result of :func:`~torch.cuda.host_memory_stats` as a nested dictionary.""" + if not is_initialized(): + return {} + return torch._C._cuda_hostMemoryStats() + + +def reset_accumulated_host_memory_stats() -> None: + r"""Reset the "accumulated" (historical) stats tracked by the host memory allocator. + + See :func:`~torch.cuda.host_memory_stats` for details. Accumulated stats correspond to + the `"allocated"` and `"freed"` keys in each individual stat dict. + """ + return torch._C._cuda_resetAccumulatedHostMemoryStats() + + +def reset_peak_host_memory_stats() -> None: + r"""Reset the "peak" stats tracked by the host memory allocator. + + See :func:`~torch.cuda.host_memory_stats` for details. Peak stats correspond to the + `"peak"` key in each individual stat dict. + """ + return torch._C._cuda_resetPeakHostMemoryStats() + + +def reset_max_memory_allocated(device: "Device" = None) -> None: + r"""Reset the starting point in tracking maximum GPU memory occupied by tensors for a given device. + + See :func:`~torch.cuda.max_memory_allocated` for details. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. warning:: + This function now calls :func:`~torch.cuda.reset_peak_memory_stats`, which resets + /all/ peak memory stats. + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + warnings.warn( + "torch.cuda.reset_max_memory_allocated now calls torch.cuda.reset_peak_memory_stats, " + "which resets /all/ peak memory stats.", + FutureWarning, + stacklevel=2, + ) + return reset_peak_memory_stats(device=device) + + +def reset_max_memory_cached(device: "Device" = None) -> None: + r"""Reset the starting point in tracking maximum GPU memory managed by the caching allocator for a given device. + + See :func:`~torch.cuda.max_memory_cached` for details. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. warning:: + This function now calls :func:`~torch.cuda.reset_peak_memory_stats`, which resets + /all/ peak memory stats. + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + warnings.warn( + "torch.cuda.reset_max_memory_cached now calls torch.cuda.reset_peak_memory_stats, " + "which resets /all/ peak memory stats.", + FutureWarning, + stacklevel=2, + ) + return reset_peak_memory_stats(device=device) + + +def memory_allocated(device: "Device" = None) -> int: + r"""Return the current GPU memory occupied by tensors in bytes for a given device. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + This is likely less than the amount shown in `nvidia-smi` since some + unused memory can be held by the caching allocator and some context + needs to be created on GPU. See :ref:`cuda-memory-management` for more + details about GPU memory management. + """ + return memory_stats(device=device).get("allocated_bytes.all.current", 0) + + +def max_memory_allocated(device: "Device" = None) -> int: + r"""Return the maximum GPU memory occupied by tensors in bytes for a given device. + + By default, this returns the peak allocated memory since the beginning of + this program. :func:`~torch.cuda.reset_peak_memory_stats` can be used to + reset the starting point in tracking this metric. For example, these two + functions can measure the peak allocated memory usage of each iteration in a + training loop. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + return memory_stats(device=device).get("allocated_bytes.all.peak", 0) + + +def memory_reserved(device: "Device" = None) -> int: + r"""Return the current GPU memory managed by the caching allocator in bytes for a given device. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + return memory_stats(device=device).get("reserved_bytes.all.current", 0) + + +def max_memory_reserved(device: "Device" = None) -> int: + r"""Return the maximum GPU memory managed by the caching allocator in bytes for a given device. + + By default, this returns the peak cached memory since the beginning of this + program. :func:`~torch.cuda.reset_peak_memory_stats` can be used to reset + the starting point in tracking this metric. For example, these two functions + can measure the peak cached memory amount of each iteration in a training + loop. + + Args: + device (torch.device or int, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + return memory_stats(device=device).get("reserved_bytes.all.peak", 0) + + +@deprecated( + "`torch.cuda.memory_cached` has been renamed to `torch.cuda.memory_reserved`", + category=FutureWarning, +) +def memory_cached(device: "Device" = None) -> int: + r"""Deprecated; see :func:`~torch.cuda.memory_reserved`.""" + return memory_reserved(device=device) + + +@deprecated( + "`torch.cuda.max_memory_cached` has been renamed to `torch.cuda.max_memory_reserved`", + category=FutureWarning, +) +def max_memory_cached(device: "Device" = None) -> int: + r"""Deprecated; see :func:`~torch.cuda.max_memory_reserved`.""" + return max_memory_reserved(device=device) + + +def memory_snapshot(mempool_id=None): + r"""Return a snapshot of the CUDA memory allocator state across all devices. + + Interpreting the output of this function requires familiarity with the + memory allocator internals. + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + return torch._C._cuda_memorySnapshot(mempool_id)["segments"] + + +def memory_summary(device: "Device" = None, abbreviated: bool = False) -> str: + r"""Return a human-readable printout of the current memory allocator statistics for a given device. + + This can be useful to display periodically during training, or when + handling out-of-memory exceptions. + + Args: + device (torch.device or int, optional): selected device. Returns + printout for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + abbreviated (bool, optional): whether to return an abbreviated summary + (default: False). + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + device = _get_device_index(device, optional=True) + stats = memory_stats(device=device) + + def _format_size(sz, pref_sz): + prefixes = ["B ", "KiB", "MiB", "GiB", "TiB", "PiB"] + prefix = prefixes[0] + for new_prefix in prefixes[1:]: + if pref_sz < 768 * 1024: + break + prefix = new_prefix + sz //= 1024 + pref_sz /= 1024 + return f"{sz:6d} {prefix}" + + def _format_count(cnt, pref_cnt): + prefixes = [" ", "K", "M"] + prefix = prefixes[0] + for new_prefix in prefixes[1:]: + if pref_cnt < 750 * 1000: + break + prefix = new_prefix + cnt //= 1000 + pref_cnt /= 1000 + return f"{cnt:7d} {prefix} " + + metrics_to_display = [ + ("allocated_bytes", "Allocated memory", _format_size), + ("active_bytes", "Active memory", _format_size), + ("requested_bytes", "Requested memory", _format_size), + ("reserved_bytes", "GPU reserved memory", _format_size), + ("inactive_split_bytes", "Non-releasable memory", _format_size), + ("allocation", "Allocations", _format_count), + ("active", "Active allocs", _format_count), + ("segment", "GPU reserved segments", _format_count), + ("inactive_split", "Non-releasable allocs", _format_count), + ] + + lines = [] + lines.append("=" * 75) + lines.append(" {_:16} PyTorch CUDA memory summary, device ID {device:<17d} ") + lines.append("-" * 75) + lines.append( + " {_:9} CUDA OOMs: {num_ooms:<12d} | {_:6} cudaMalloc retries: {num_alloc_retries:<8d} " + ) + lines.append("=" * 75) + lines.append( + " Metric | Cur Usage | Peak Usage | Tot Alloc | Tot Freed " + ) + + for metric_key, metric_name, formatter in metrics_to_display: + lines.append("-" * 75) + submetrics = [("all", metric_name)] + if not abbreviated: + submetrics.append(("large_pool", " from large pool")) + submetrics.append(("small_pool", " from small pool")) + + current_prefval, peak_prefval, allocated_prefval, freed_prefval = ( + None, + None, + None, + None, + ) + + for submetric_key, submetric_name in submetrics: + prefix = metric_key + "." + submetric_key + "." + + current = stats[prefix + "current"] + peak = stats[prefix + "peak"] + allocated = stats[prefix + "allocated"] + freed = stats[prefix + "freed"] + + if current_prefval is None: + current_prefval = current + peak_prefval = peak + allocated_prefval = allocated + freed_prefval = freed + + lines.append( + f" {submetric_name:<21} | {formatter(current, current_prefval)} | {formatter(peak, peak_prefval)} | " + f"{formatter(allocated, allocated_prefval)} | {formatter(freed, freed_prefval)} ", + ) + + metrics_to_display = [ + ("oversize_allocations", "Oversize allocations", _format_count), + ("oversize_segments", "Oversize GPU segments", _format_count), + ] + + for metric_key, metric_name, formatter in metrics_to_display: + lines.append("-" * 75) + + prefix = metric_key + "." + + current = stats[prefix + "current"] + peak = stats[prefix + "peak"] + allocated = stats[prefix + "allocated"] + freed = stats[prefix + "freed"] + + lines.append( + f" {metric_name:<21} | {formatter(current, current)} | {formatter(peak, peak)} | " + f"{formatter(allocated, allocated)} | {formatter(freed, freed)} ", + ) + + lines.append("=" * 75) + + fmt_dict = {"_": "", "device": device} + for k, v in stats.items(): + fmt_dict[k.replace(".", "-")] = v + return "|" + "|\n|".join(lines).format(**fmt_dict) + "|\n" + + +def list_gpu_processes(device: "Device" = None) -> str: + r"""Return a human-readable printout of the running processes and their GPU memory use for a given device. + + This can be useful to display periodically during training, or when + handling out-of-memory exceptions. + + Args: + device (torch.device or int, optional): selected device. Returns + printout for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + """ + if not torch.version.hip: + try: + import pynvml # type: ignore[import] + except ModuleNotFoundError: + return "pynvml module not found, please install nvidia-ml-py" + # pyrefly: ignore [import-error,missing-module-attribute] + from pynvml import NVMLError_DriverNotLoaded + + try: + pynvml.nvmlInit() + except NVMLError_DriverNotLoaded: + return "cuda driver can't be loaded, is cuda enabled?" + + device = _get_nvml_device_index(device) + handle = pynvml.nvmlDeviceGetHandleByIndex(device) + procs = pynvml.nvmlDeviceGetComputeRunningProcesses(handle) + else: + try: + import amdsmi # type: ignore[import] + except ModuleNotFoundError: + return "amdsmi module not found, please install amdsmi" + try: + amdsmi.amdsmi_init() # type: ignore[attr-defined] + except amdsmi.AmdSmiException: # type: ignore[attr-defined] + return "amdsmi driver can't be loaded, is ROCm installed?" + + device = _get_amdsmi_device_index(device) + + try: + handle = amdsmi.amdsmi_get_processor_handles()[device] # type: ignore[attr-defined] + procs = amdsmi.amdsmi_get_gpu_process_list(handle) # type: ignore[attr-defined] + except amdsmi.AmdSmiException: # type: ignore[attr-defined] + return "amdsmi cannot list processes from other users" + + lines = [] + lines.append(f"GPU:{device}") + if len(procs) == 0: + lines.append("no processes are running") + for p in procs: + if not torch.version.hip: + mem = p.usedGpuMemory / (1024 * 1024) + pid = p.pid + else: + try: + proc_info = amdsmi.amdsmi_get_gpu_process_info(handle, p) # type: ignore[possibly-undefined] + except AttributeError: + # https://github.com/ROCm/amdsmi/commit/c551c3caedbd903ba828e7fdffa5b56d475a15e7 + # is a BC-breaking change that removes amdsmi_get_gpu_process_info API from amdsmi + proc_info = p + mem = proc_info["memory_usage"]["vram_mem"] / (1024 * 1024) + pid = proc_info["pid"] + lines.append(f"process {pid:>10d} uses {mem:>12.3f} MB GPU memory") + return "\n".join(lines) + + +def mem_get_info(device: "Device" = None) -> tuple[int, int]: + r"""Return the global free and total GPU memory for a given device using cudaMemGetInfo. + + Args: + device (torch.device or int or str, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default) or if the device index is not specified. + + .. note:: + See :ref:`cuda-memory-management` for more + details about GPU memory management. + """ + if device is None: + device = torch.cuda.current_device() + # optional=True allows `device = torch.device('cuda')` for which device.index is None + device = _get_device_index(device, optional=True) + return torch.cuda.cudart().cudaMemGetInfo(device) + + +def _record_memory_history_legacy( + enabled: bool, + record_context=True, + trace_alloc_max_entries=1, + trace_alloc_record_context=False, + device: "Device" = None, + record_context_cpp=False, + clear_history=False, + compile_context=False, + global_record_annotations=False, +): + _C._cuda_record_memory_history_legacy( # type: ignore[call-arg] + enabled, + record_context, + # pyrefly: ignore [bad-argument-type] + trace_alloc_max_entries, + trace_alloc_record_context, + record_context_cpp, + clear_history, + compile_context, + global_record_annotations, + ) + + +def _record_memory_history( + enabled: Optional[Literal["state", "all"]] = "all", *args, **kwargs +) -> None: + """Enable recording of stack traces associated with memory + allocations, so you can tell what allocated any piece of memory in + :func:`torch.cuda.memory._snapshot()`. + + In addition to keeping stack traces with each current allocation and free, + this will also enable recording of a history of all alloc/free events. + + Use :func:`torch.cuda.memory._snapshot()` to retrieve this information, + and the tools in `_memory_viz.py` to visualize snapshots. + + Buffer behavior + --------------- + + This will store up to `max_entries` instances of `TraceEntry` when enabled. + Python trace collection defaults to `sys.maxsize`, meaning long-running + or indefinitely running jobs should set a reasonable limit to avoid excessive + memory use. Expect each entry to be several KB. + + Longer running workflows or those with smaller `max_entries` values will only + store the last accumulated `max_entries` entries, meaning new entries overwrite + older entries. + + C++ implementation for reference to ring buffer implementation: + + .. code-block:: cpp + + if (record_history) { + if (alloc_trace->size() < alloc_trace_max_entries_) { + alloc_trace->emplace_back(te); + } else { + (*alloc_trace)[alloc_trace_next++] = te; + if (alloc_trace_next == alloc_trace_max_entries_) { + alloc_trace_next = 0; + } + } + } + + Latency impact + -------------- + + The Python trace collection is fast (2us per trace), so you may consider + enabling this on production jobs if you anticipate ever having to debug + memory issues. + + C++ trace collection is also fast (~50ns/frame), which for many typical programs + works out to ~2us per trace, but can vary depending on stack depth. + + Args: + enabled (Literal[None, "state", "all"], optional): + `None`, disable recording memory history. + `"state"`, keep information for currently allocated memory. + `"all"`, additionally keep a history of all alloc/free calls. + Defaults to "all". + context (Literal[None, "state", "alloc", "all"], optional): + `None`, Do not record any tracebacks. + `"state"`, Record tracebacks for currently allocated memory. + `"alloc"`, additionally keep tracebacks for alloc calls. + `"all"`, additionally keep tracebacks for free calls. + Defaults to "all". + stacks (Literal["python", "all"], optional): + `"python"`, include Python, TorchScript, and inductor frames in tracebacks + `"all"`, additionally include C++ frames + Defaults to "all". + max_entries (int, optional): Keep a maximum of `max_entries` + alloc/free events in the recorded history recorded. + """ + if isinstance(enabled, bool): + return _record_memory_history_legacy(enabled, *args, **kwargs) + else: + return _record_memory_history_impl(enabled, *args, **kwargs) + + +def _record_memory_history_impl( + enabled: Optional[str] = "all", + context: Optional[str] = "all", + stacks: str = "all", + max_entries: int = sys.maxsize, + device: "Device" = None, + clear_history: bool = False, + compile_context: bool = False, + global_record_annotations: bool = False, +): + _C._cuda_record_memory_history( # type: ignore[call-arg] + enabled, + context, + stacks, + max_entries, + clear_history, + compile_context, + global_record_annotations, + ) + + +_record_memory_history.__signature__ = signature(_record_memory_history_impl) # type: ignore[attr-defined] + + +def _augment_frames(frames: list[_Frame]) -> int: + """ + Augment a list of frames with FX debug information. + + Args: + frames: List of frame dictionaries to augment + + Returns: + The count of frames that were augmented. + """ + from torch.fx.graph_module import FX_GRAPH_MODULE_FILE_PREFIX + + # Regex pattern to match FX generated files + _FX_GENERATED_PATTERN = re.compile( + rf"{re.escape(FX_GRAPH_MODULE_FILE_PREFIX)}.*\.py$" + ) + + count = 0 + if not frames: + return count + + for frame in frames: + if "filename" in frame and "line" in frame: + filename = frame["filename"] + lineno = frame["line"] + + # Check if this looks like an FX generated file + if not _FX_GENERATED_PATTERN.search(os.path.basename(filename)): + continue + + # Look up metadata from the global registry + from torch.fx.traceback import _FX_METADATA_REGISTRY + + metadata = _FX_METADATA_REGISTRY.get(filename) + if metadata is None: + continue + + lineno_map = metadata.get("lineno_map", {}) + node_metadata = metadata.get("node_metadata", {}) + prologue_start = metadata.get("prologue_start", 0) + + # Get the node index for this line + node_idx = lineno_map.get(lineno - prologue_start) + + if node_idx is not None and node_idx in node_metadata: + node_info = node_metadata[node_idx] + original_trace = node_info.get("stack_trace") + node_op = node_info.get("op") + node_name = node_info.get("name") + node_target = node_info.get("target") + + # Always add node metadata + frame["fx_node_op"] = node_op + frame["fx_node_name"] = node_name + frame["fx_node_target"] = str(node_target) + + # Add original trace if available + if original_trace: + frame["fx_original_trace"] = original_trace + + count += 1 + + return count + + +def _augment_memory_snapshot_stack_traces( + snapshot: str | _Snapshot, +) -> _Snapshot: + """ + Augment a memory snapshot with original source stack traces from FX metadata. + + IMPORTANT: This function reads from a global in-memory registry (_FX_METADATA_REGISTRY) + that is populated during graph module compilation. It must be called in the same + Python process where the FX graphs were compiled. It cannot be used to augment + snapshots loaded from disk in a different process. + + Args: + snapshot: Either a memory snapshot dict or path to a snapshot pickle file + + Returns: + The augmented snapshot dictionary with fx_node_op, fx_node_name, + fx_original_trace, and fx_node_info fields added to frames + """ + + snapshot_dict: _Snapshot + if isinstance(snapshot, str): + # Load the memory snapshot + with open(snapshot, "rb") as f: + snapshot_dict = cast(_Snapshot, pickle.load(f)) + else: + snapshot_dict = snapshot + + # Process stack traces in the snapshot + augmented_count = 0 + + # Process blocks in segments (for regular allocations) + if "segments" in snapshot_dict: + for segment in snapshot_dict["segments"]: + if "blocks" in segment: + for block in segment["blocks"]: + if "frames" in block: + augmented_count += _augment_frames(block["frames"]) + + # Process device traces (for memory history) + if "device_traces" in snapshot_dict: + for trace_list in snapshot_dict["device_traces"]: + for trace_entry in trace_list: + if isinstance(trace_entry, dict) and "frames" in trace_entry: + augmented_count += _augment_frames(trace_entry["frames"]) + + return snapshot_dict + + +def _snapshot(device: "Device" = None, augment_with_fx_traces=False): + """Save a snapshot of CUDA memory state at the time it was called. + + The state is represented as a dictionary with the following structure. + + .. code-block:: python + + class Snapshot(TypedDict): + segments: List[Segment] + device_traces: List[List[TraceEntry]] + + + class Segment(TypedDict): + # Segments are memory returned from a cudaMalloc call. + # The size of reserved memory is the sum of all Segments. + # Segments are cached and reused for future allocations. + # If the reuse is smaller than the segment, the segment + # is split into more then one Block. + # empty_cache() frees Segments that are entirely inactive. + address: int + total_size: int # cudaMalloc'd size of segment + stream: int + segment_type: Literal["small", "large"] # 'large' (>1MB) + allocated_size: int # size of memory in use + active_size: int # size of memory in use or in active_awaiting_free state + blocks: List[Block] + + + class Block(TypedDict): + # A piece of memory returned from the allocator, or + # current cached but inactive. + size: int + requested_size: int # size requested during malloc, may be smaller than + # size due to rounding + address: int + state: Literal[ + "active_allocated", # used by a tensor + "active_awaiting_free", # waiting for another stream to finish using + # this, then it will become free + "inactive", + ] # free for reuse + frames: List[Frame] # stack trace from where the allocation occurred + + + class Frame(TypedDict): + filename: str + line: int + name: str + # Optional FX debug fields (present when augment_with_fx_traces=True + # and the frame corresponds to FX-generated code) + fx_node_op: str # FX node operation type (e.g., 'call_function', 'output') + fx_node_name: str # FX node name (e.g., 'linear', 'relu_1') + fx_original_trace: str # Original model source code stack trace + + + class TraceEntry(TypedDict): + # When `torch.cuda.memory._record_memory_history()` is enabled, + # the snapshot will contain TraceEntry objects that record each + # action the allocator took. + action: Literal[ + "alloc" # memory allocated + "free_requested", # the allocated received a call to free memory + "free_completed", # the memory that was requested to be freed is now + # able to be used in future allocation calls + "segment_alloc", # the caching allocator ask cudaMalloc for more memory + # and added it as a segment in its cache + "segment_free", # the caching allocator called cudaFree to return memory + # to cuda possibly trying free up memory to + # allocate more segments or because empty_caches was called + "oom", # the allocator threw an OOM exception. 'size' is + # the requested number of bytes that did not succeed + "snapshot", # the allocator generated a memory snapshot + # useful to coorelate a previously taken + # snapshot with this trace + ] + addr: int # not present for OOM + frames: List[Frame] + size: int + stream: int + device_free: int # only present for OOM, the amount of + # memory cuda still reports to be free + + Args: + device: Device to capture snapshot for. If None, captures for current device. + augment_with_fx_traces: If True, augment stack trace frames with FX debug information + that maps generated FX code back to original model source code. + This adds fx_node_op, fx_node_name, fx_original_trace, and + fx_node_info fields to Frame objects. Default: False. + + Returns: + The Snapshot dictionary object + """ + s = _C._cuda_memorySnapshot(None) + if augment_with_fx_traces: + s = _augment_memory_snapshot_stack_traces(s) # type: ignore[assignment, arg-type] + return s + + +def _dump_snapshot(filename="dump_snapshot.pickle", augment_with_fx_traces=False): + """ + Save a pickled version of the `torch.memory._snapshot()` dictionary to a file. + + This file can be opened by the interactive snapshot viewer at pytorch.org/memory_viz + + Snapshot file sizes scale with `max_entries` and stack trace depth per entry, + with several KB per entry. These can easily be in the GB range for longer running + workflows with large `max_entries`. + + Args: + filename (str, optional): Name of the file to create. Defaults to "dump_snapshot.pickle". + augment_with_fx_traces (bool, optional): If True, augment the snapshot with FX debug information + before dumping. This maps generated FX code stack traces + back to original model source code. Defaults to False. + verbose (bool, optional): If True and augment_with_fx_traces is True, print verbose debug output + during augmentation. Defaults to False. + """ + s = _snapshot(augment_with_fx_traces=augment_with_fx_traces) + + with open(filename, "wb") as f: + pickle.dump(s, f) + + +def _set_memory_metadata(metadata: str): + """ + Set custom metadata that will be attached to all subsequent CUDA memory allocations. + + This metadata will be recorded in the memory snapshot for all allocations made + after this call until the metadata is cleared or changed. + + Args: + metadata (str): Custom metadata string to attach to allocations. + Pass an empty string to clear the metadata. + """ + # pyrefly: ignore [missing-attribute] + torch._C._cuda_setMemoryMetadata(metadata) + + +def _get_memory_metadata() -> str: + """ + Get the current custom metadata that is being attached to CUDA memory allocations. + + Returns: + str: The current metadata string, or empty string if no metadata is set. + """ + # pyrefly: ignore [missing-attribute] + return torch._C._cuda_getMemoryMetadata() + + +def _save_segment_usage(filename="output.svg", snapshot=None): + if snapshot is None: + snapshot = _snapshot() + with open(filename, "w") as f: + f.write(_segments(snapshot)) + + +def _save_memory_usage(filename="output.svg", snapshot=None): + if snapshot is None: + snapshot = _snapshot() + with open(filename, "w") as f: + f.write(_memory(snapshot)) + + +@deprecated( + "torch.cuda._set_allocator_settings is deprecated. Use torch._C._accelerator_setAllocatorSettings instead.", + category=FutureWarning, +) +def _set_allocator_settings(env: str): + # pyrefly: ignore [missing-attribute] + return torch._C._accelerator_setAllocatorSettings(env) + + +def get_allocator_backend() -> str: + r"""Return a string describing the active allocator backend as set by + ``PYTORCH_ALLOC_CONF``. Currently available backends are + ``native`` (PyTorch's native caching allocator) and `cudaMallocAsync`` + (CUDA's built-in asynchronous allocator). + + .. note:: + See :ref:`cuda-memory-management` for details on choosing the allocator backend. + """ + return torch._C._cuda_getAllocatorBackend() + + +class _CUDAAllocator: + r"""Wrapper over internal CUDA memory allocators.""" + + def __init__(self, allocator: torch._C._cuda_CUDAAllocator): + self._allocator = allocator + + def allocator(self): + return self._allocator + + +class CUDAPluggableAllocator(_CUDAAllocator): + r"""CUDA memory allocator loaded from a so file.""" + + def __init__(self, path_to_so_file: str, alloc_fn_name: str, free_fn_name: str): + r"""Memory allocators are compiled in .so files and loaded dynamically using ctypes. + + To change the active allocator use the :func:`torch.memory.cuda.change_current_allocator` function. + + Args: + path_to_so_file(str): Path in the filesystem to the `.so` file containing + the allocator functions + alloc_fn_name(str): Name of the function to perform the memory allocation + in the so file. The signature must be: + void* alloc_fn_name(ssize_t size, int device, cudaStream_t stream); + free_fn_name(str): Name of the function to perform the memory release + in the so file. The signature must be: + void free_fn_name(void* ptr, size_t size, cudaStream_t stream); + + .. warning:: + This is currently supported only in unix OSs + + .. note:: + See :ref:`cuda-memory-management` for details on creating and using a custom allocator + """ + allocator = ctypes.CDLL(path_to_so_file) + alloc_fn = ctypes.cast(getattr(allocator, alloc_fn_name), ctypes.c_void_p).value + free_fn = ctypes.cast(getattr(allocator, free_fn_name), ctypes.c_void_p).value + assert alloc_fn is not None + assert free_fn is not None + self._allocator = torch._C._cuda_customAllocator(alloc_fn, free_fn) + + +def change_current_allocator(allocator: _CUDAAllocator) -> None: + r"""Change the currently used memory allocator to be the one provided. + + If the current allocator has already been used/initialized, this function will error. + + + Args: + allocator (torch.cuda.memory._CUDAAllocator): allocator to be set as the active one. + .. note:: + See :ref:`cuda-memory-management` for details on creating and using a custom allocator + """ + torch._C._cuda_changeCurrentAllocator(allocator.allocator()) + + +def _get_current_allocator() -> _CUDAAllocator: + r"""Return the allocator being currently used. + + .. note:: + See :ref:`cuda-memory-management` for details on creating and using a custom allocator + """ + return _CUDAAllocator(torch._C._cuda_getAllocator()) + + +class MemPool(_MemPool): + r"""MemPool represents a pool of memory in a caching allocator. Currently, + it's just the ID of the pool object maintained in the CUDACachingAllocator. + + Args: + allocator(torch._C._cuda_CUDAAllocator, optional): a + torch._C._cuda_CUDAAllocator object that can be used to + define how memory gets allocated in the pool. If :attr:`allocator` + is ``None`` (default), memory allocation follows the default/ + current configuration of the CUDACachingAllocator. + use_on_oom(bool): a bool that indicates if this pool can be used + as a last resort if a memory allocation outside of the pool fails due + to Out Of Memory. This is False by default. + no_split(bool): a bool that indicates if this pool should not split a segment. + This is False by default. + """ + + def __init__( + self, + allocator: Optional[_cuda_CUDAAllocator] = None, + use_on_oom: bool = False, + no_split: bool = False, + ): + super().__init__(allocator, True, use_on_oom, no_split) + + @property + def id(self) -> tuple[int, int]: + r"""Returns the ID of this pool as a tuple of two ints.""" + return super().id + + @property + def allocator(self) -> Optional[_cuda_CUDAAllocator]: + r"""Returns the allocator this MemPool routes allocations to.""" + return super().allocator + + def use_count(self) -> int: # pylint: disable=useless-parent-delegation + r"""Returns the reference count of this pool.""" + return super().use_count() + + def snapshot(self): + r"""Return a snapshot of the CUDA memory allocator pool state across all + devices. + + Interpreting the output of this function requires familiarity with the + memory allocator internals. + + .. note:: + See :ref:`cuda-memory-management` for more details about GPU memory + management. + """ + snapshot = torch.cuda.memory_snapshot(self.id) + return snapshot + + +@contextlib.contextmanager +def use_mem_pool(pool: MemPool, device: "Device" = None): + r"""A context manager that routes allocations to a given pool. + + Args: + pool(torch.cuda.MemPool): a MemPool object to be made active so that + allocations route to this pool. + device (torch.device or int, optional): selected device. Uses MemPool on + the current device, given by :func:`~torch.cuda.current_device`, + if :attr:`device` is ``None`` (default). + + .. note:: + This context manager makes only current thread's allocations route to + the given pool. If a new thread is spawned inside the context manager + (e.g. by calling backward) the allocations in that thread will not + route to the given pool. + """ + device_index = ( + torch.cuda.current_device() if device is None else _get_device_index(device) + ) + _cuda_beginAllocateCurrentThreadToPool(device_index, pool.id) + try: + yield + finally: + _cuda_endAllocateToPool(device_index, pool.id) + _cuda_releasePool(device_index, pool.id) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/nccl.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/nccl.py new file mode 100644 index 0000000000000000000000000000000000000000..bef781c19a0e92cc8658c1503312ec4580756d1f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/nccl.py @@ -0,0 +1,152 @@ +# mypy: allow-untyped-defs +import collections +import warnings +from collections.abc import Sequence +from typing import Optional, Union + +import torch.cuda + + +__all__ = ["all_reduce", "reduce", "broadcast", "all_gather", "reduce_scatter"] + +SUM = 0 # ncclRedOp_t + + +def is_available(tensors): + if not hasattr(torch._C, "_nccl_all_reduce"): + warnings.warn("PyTorch is not compiled with NCCL support", stacklevel=2) + return False + + devices = set() + for tensor in tensors: + if tensor.is_sparse: + return False + if not tensor.is_contiguous(): + return False + if not tensor.is_cuda: + return False + device = tensor.get_device() + if device in devices: + return False + devices.add(device) + + return True + + +def version(): + """ + Returns the version of the NCCL. + + + This function returns a tuple containing the major, minor, and patch version numbers of the NCCL. + The suffix is also included in the tuple if a version suffix exists. + Returns: + tuple: The version information of the NCCL. + """ + ver = torch._C._nccl_version() + major = ver >> 32 + minor = (ver >> 16) & 65535 + patch = ver & 65535 + suffix = torch._C._nccl_version_suffix().decode("utf-8") + if suffix == "": + return (major, minor, patch) + else: + return (major, minor, patch, suffix) + + +def unique_id(): + return torch._C._nccl_unique_id() + + +def init_rank(num_ranks, uid, rank): + return torch._C._nccl_init_rank(num_ranks, uid, rank) + + +def _check_sequence_type(inputs: Union[torch.Tensor, Sequence[torch.Tensor]]) -> None: + if not isinstance(inputs, collections.abc.Container) or isinstance( + inputs, torch.Tensor + ): + raise TypeError("Inputs should be a collection of tensors") + + +def all_reduce(inputs, outputs=None, op=SUM, streams=None, comms=None): + _check_sequence_type(inputs) + if outputs is None: + outputs = inputs + _check_sequence_type(outputs) + torch._C._nccl_all_reduce(inputs, outputs, op, streams, comms) + + +# `output` used to be `outputs`, taking in a list of tensors. So we have two +# arguments for BC reasons. +def reduce( + inputs: Sequence[torch.Tensor], + output: Optional[Union[torch.Tensor, Sequence[torch.Tensor]]] = None, + root: int = 0, + op: int = SUM, + streams: Optional[Sequence[torch.cuda.Stream]] = None, + comms=None, + *, + outputs: Optional[Sequence[torch.Tensor]] = None, +) -> None: + _check_sequence_type(inputs) + _output: torch.Tensor + if outputs is not None: + if output is not None: + raise ValueError( + "'output' and 'outputs' can not be both specified. 'outputs' is deprecated in " + "favor of 'output', taking in a single output tensor. The signature of reduce is: " + "reduce(inputs, output=None, root=0, op=SUM, streams=None, comms=None)." + ) + else: + warnings.warn( + "`nccl.reduce` with an output tensor list is deprecated. " + "Please specify a single output tensor with argument 'output' instead instead.", + FutureWarning, + stacklevel=2, + ) + _output = outputs[root] + elif not isinstance(output, torch.Tensor) and isinstance( + output, collections.abc.Sequence + ): + # User called old API with positional arguments of list of output tensors. + warnings.warn( + "nccl.reduce with an output tensor list is deprecated. " + "Please specify a single output tensor.", + FutureWarning, + stacklevel=2, + ) + _output = output[root] + else: + _output = inputs[root] if output is None else output + torch._C._nccl_reduce(inputs, _output, root, op, streams, comms) + + +def broadcast( + inputs: Sequence[torch.Tensor], root: int = 0, streams=None, comms=None +) -> None: + _check_sequence_type(inputs) + torch._C._nccl_broadcast(inputs, root, streams, comms) + + +def all_gather( + inputs: Sequence[torch.Tensor], + outputs: Sequence[torch.Tensor], + streams=None, + comms=None, +) -> None: + _check_sequence_type(inputs) + _check_sequence_type(outputs) + torch._C._nccl_all_gather(inputs, outputs, streams, comms) + + +def reduce_scatter( + inputs: Sequence[torch.Tensor], + outputs: Sequence[torch.Tensor], + op: int = SUM, + streams=None, + comms=None, +) -> None: + _check_sequence_type(inputs) + _check_sequence_type(outputs) + torch._C._nccl_reduce_scatter(inputs, outputs, op, streams, comms) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/nvtx.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/nvtx.py new file mode 100644 index 0000000000000000000000000000000000000000..f77f9a0c9fd73576a362be526e8b1396a1901233 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/nvtx.py @@ -0,0 +1,129 @@ +# mypy: allow-untyped-defs +r"""This package adds support for NVIDIA Tools Extension (NVTX) used in profiling.""" + +from contextlib import contextmanager + + +try: + from torch._C import _nvtx +except ImportError: + + class _NVTXStub: + @staticmethod + def _fail(*args, **kwargs): + raise RuntimeError( + "NVTX functions not installed. Are you sure you have a CUDA build?" + ) + + rangePushA = _fail + rangePop = _fail + markA = _fail + + _nvtx = _NVTXStub() # type: ignore[assignment] + +__all__ = ["range_push", "range_pop", "range_start", "range_end", "mark", "range"] + + +def range_push(msg): + """ + Push a range onto a stack of nested range span. Returns zero-based depth of the range that is started. + + Args: + msg (str): ASCII message to associate with range + """ + return _nvtx.rangePushA(msg) + + +def range_pop(): + """Pop a range off of a stack of nested range spans. Returns the zero-based depth of the range that is ended.""" + return _nvtx.rangePop() + + +def range_start(msg) -> int: + """ + Mark the start of a range with string message. It returns an unique handle + for this range to pass to the corresponding call to rangeEnd(). + + A key difference between this and range_push/range_pop is that the + range_start/range_end version supports range across threads (start on one + thread and end on another thread). + + Returns: A range handle (uint64_t) that can be passed to range_end(). + + Args: + msg (str): ASCII message to associate with the range. + """ + # pyrefly: ignore [missing-attribute] + return _nvtx.rangeStartA(msg) + + +def range_end(range_id) -> None: + """ + Mark the end of a range for a given range_id. + + Args: + range_id (int): an unique handle for the start range. + """ + # pyrefly: ignore [missing-attribute] + _nvtx.rangeEnd(range_id) + + +def _device_range_start(msg: str, stream: int = 0) -> object: + """ + Marks the start of a range with string message. + It returns an opaque heap-allocated handle for this range + to pass to the corresponding call to device_range_end(). + + A key difference between this and range_start is that the + range_start marks the range right away, while _device_range_start + marks the start of the range as soon as all the tasks on the + CUDA stream are completed. + + Returns: An opaque heap-allocated handle that should be passed to _device_range_end(). + + Args: + msg (str): ASCII message to associate with the range. + stream (int): CUDA stream id. + """ + # pyrefly: ignore [missing-attribute] + return _nvtx.deviceRangeStart(msg, stream) + + +def _device_range_end(range_handle: object, stream: int = 0) -> None: + """ + Mark the end of a range for a given range_handle as soon as all the tasks + on the CUDA stream are completed. + + Args: + range_handle: an unique handle for the start range. + stream (int): CUDA stream id. + """ + # pyrefly: ignore [missing-attribute] + _nvtx.deviceRangeEnd(range_handle, stream) + + +def mark(msg): + """ + Describe an instantaneous event that occurred at some point. + + Args: + msg (str): ASCII message to associate with the event. + """ + return _nvtx.markA(msg) + + +@contextmanager +def range(msg, *args, **kwargs): + """ + Context manager / decorator that pushes an NVTX range at the beginning + of its scope, and pops it at the end. If extra arguments are given, + they are passed as arguments to msg.format(). + + Args: + msg (str): message to associate with the range + """ + range_push(msg.format(*args, **kwargs)) + try: + yield + finally: + range_pop() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/profiler.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..ae0674f4a4a1d9d3f20a759301606c8236b09828 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/profiler.py @@ -0,0 +1,86 @@ +# mypy: allow-untyped-defs +import contextlib +import tempfile + +import torch + +from . import check_error, cudart + + +__all__ = ["init", "start", "stop", "profile"] + +DEFAULT_FLAGS = [ + "gpustarttimestamp", + "gpuendtimestamp", + "gridsize3d", + "threadblocksize", + "streamid", + "enableonstart 0", + "conckerneltrace", +] + + +def init(output_file, flags=None, output_mode="key_value"): + rt = cudart() + if not hasattr(rt, "cudaOutputMode"): + raise AssertionError("HIP does not support profiler initialization!") + if ( + hasattr(torch.version, "cuda") + and torch.version.cuda is not None + and int(torch.version.cuda.split(".")[0]) >= 12 + ): + # Check https://github.com/pytorch/pytorch/pull/91118 + # cudaProfilerInitialize is no longer needed after CUDA 12 + raise AssertionError("CUDA12+ does not need profiler initialization!") + flags = DEFAULT_FLAGS if flags is None else flags + if output_mode == "key_value": + output_mode_enum = rt.cudaOutputMode.KeyValuePair + elif output_mode == "csv": + output_mode_enum = rt.cudaOutputMode.CSV + else: + raise RuntimeError( + "supported CUDA profiler output modes are: key_value and csv" + ) + with tempfile.NamedTemporaryFile(delete=True) as f: + f.write(b"\n".join(f.encode("ascii") for f in flags)) + f.flush() + check_error(rt.cudaProfilerInitialize(f.name, output_file, output_mode_enum)) + + +def start(): + r"""Starts cuda profiler data collection. + + .. warning:: + Raises CudaError in case of it is unable to start the profiler. + """ + check_error(cudart().cudaProfilerStart()) + + +def stop(): + r"""Stops cuda profiler data collection. + + .. warning:: + Raises CudaError in case of it is unable to stop the profiler. + """ + check_error(cudart().cudaProfilerStop()) + + +@contextlib.contextmanager +def profile(): + """ + Enable profiling. + + Context Manager to enabling profile collection by the active profiling tool from CUDA backend. + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> import torch + >>> model = torch.nn.Linear(20, 30).cuda() + >>> inputs = torch.randn(128, 20).cuda() + >>> with torch.cuda.profiler.profile() as prof: + ... model(inputs) + """ + try: + start() + yield + finally: + stop() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/random.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/random.py new file mode 100644 index 0000000000000000000000000000000000000000..fe7c53c3d7ebefc1e3b37864dc86a24f923f337b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/random.py @@ -0,0 +1,184 @@ +# mypy: allow-untyped-defs +from collections.abc import Iterable +from typing import Union + +import torch +from torch import Tensor + +from . import _lazy_call, _lazy_init, current_device, device_count, is_initialized + + +__all__ = [ + "get_rng_state", + "get_rng_state_all", + "set_rng_state", + "set_rng_state_all", + "manual_seed", + "manual_seed_all", + "seed", + "seed_all", + "initial_seed", +] + + +def get_rng_state(device: Union[int, str, torch.device] = "cuda") -> Tensor: + r"""Return the random number generator state of the specified GPU as a ByteTensor. + + Args: + device (torch.device or int, optional): The device to return the RNG state of. + Default: ``'cuda'`` (i.e., ``torch.device('cuda')``, the current CUDA device). + + .. warning:: + This function eagerly initializes CUDA. + """ + _lazy_init() + if isinstance(device, str): + device = torch.device(device) + elif isinstance(device, int): + device = torch.device("cuda", device) + idx = device.index + if idx is None: + idx = current_device() + default_generator = torch.cuda.default_generators[idx] + return default_generator.get_state() + + +def get_rng_state_all() -> list[Tensor]: + r"""Return a list of ByteTensor representing the random number states of all devices.""" + results = [get_rng_state(i) for i in range(device_count())] + return results + + +def set_rng_state( + new_state: Tensor, device: Union[int, str, torch.device] = "cuda" +) -> None: + r"""Set the random number generator state of the specified GPU. + + Args: + new_state (torch.ByteTensor): The desired state + device (torch.device or int, optional): The device to set the RNG state. + Default: ``'cuda'`` (i.e., ``torch.device('cuda')``, the current CUDA device). + """ + if not is_initialized(): + with torch._C._DisableFuncTorch(): + # Clone the state because the callback will be triggered + # later when CUDA is lazy initialized. + new_state = new_state.clone(memory_format=torch.contiguous_format) + if isinstance(device, str): + device = torch.device(device) + elif isinstance(device, int): + device = torch.device("cuda", device) + + def cb(): + idx = device.index + if idx is None: + idx = current_device() + default_generator = torch.cuda.default_generators[idx] + default_generator.set_state(new_state) + + _lazy_call(cb) + + +def set_rng_state_all(new_states: Iterable[Tensor]) -> None: + r"""Set the random number generator state of all devices. + + Args: + new_states (Iterable of torch.ByteTensor): The desired state for each device. + """ + for i, state in enumerate(new_states): + set_rng_state(state, i) + + +def manual_seed(seed: int) -> None: + r"""Set the seed for generating random numbers for the current GPU. + + It's safe to call this function if CUDA is not available; in that + case, it is silently ignored. + + Args: + seed (int): The desired seed. + + .. warning:: + If you are working with a multi-GPU model, this function is insufficient + to get determinism. To seed all GPUs, use :func:`manual_seed_all`. + """ + seed = int(seed) + + def cb(): + idx = current_device() + default_generator = torch.cuda.default_generators[idx] + default_generator.manual_seed(seed) + + _lazy_call(cb, seed=True) + + +def manual_seed_all(seed: int) -> None: + r"""Set the seed for generating random numbers on all GPUs. + + It's safe to call this function if CUDA is not available; in that + case, it is silently ignored. + + Args: + seed (int): The desired seed. + """ + seed = int(seed) + + def cb(): + for i in range(device_count()): + default_generator = torch.cuda.default_generators[i] + default_generator.manual_seed(seed) + + _lazy_call(cb, seed_all=True) + + +def seed() -> None: + r"""Set the seed for generating random numbers to a random number for the current GPU. + + It's safe to call this function if CUDA is not available; in that + case, it is silently ignored. + + .. warning:: + If you are working with a multi-GPU model, this function will only initialize + the seed on one GPU. To initialize all GPUs, use :func:`seed_all`. + """ + + def cb(): + idx = current_device() + default_generator = torch.cuda.default_generators[idx] + default_generator.seed() + + _lazy_call(cb) + + +def seed_all() -> None: + r"""Set the seed for generating random numbers to a random number on all GPUs. + + It's safe to call this function if CUDA is not available; in that + case, it is silently ignored. + """ + + def cb(): + random_seed = 0 + seeded = False + for i in range(device_count()): + default_generator = torch.cuda.default_generators[i] + if not seeded: + default_generator.seed() + random_seed = default_generator.initial_seed() + seeded = True + else: + default_generator.manual_seed(random_seed) + + _lazy_call(cb) + + +def initial_seed() -> int: + r"""Return the current random seed of the current GPU. + + .. warning:: + This function eagerly initializes CUDA. + """ + _lazy_init() + idx = current_device() + default_generator = torch.cuda.default_generators[idx] + return default_generator.initial_seed() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/sparse.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/sparse.py new file mode 100644 index 0000000000000000000000000000000000000000..f37a34118d2d8f73437dee54337a666df1b99a09 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/sparse.py @@ -0,0 +1 @@ +# The Tensor classes are added to this module by python_tensor.cpp diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/streams.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/streams.py new file mode 100644 index 0000000000000000000000000000000000000000..3670937cf9f75136201f2de68243ca905c367053 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/streams.py @@ -0,0 +1,262 @@ +# mypy: allow-untyped-defs +# pylint: disable=useless-parent-delegation +import ctypes + +import torch +from torch._utils import _dummy_type + + +if not hasattr(torch._C, "_CudaStreamBase"): + # Define dummy base classes + torch._C.__dict__["_CudaStreamBase"] = _dummy_type("_CudaStreamBase") + torch._C.__dict__["_CudaEventBase"] = _dummy_type("_CudaEventBase") + + +class Stream(torch._C._CudaStreamBase): + r"""Wrapper around a CUDA stream. + + A CUDA stream is a linear sequence of execution that belongs to a specific + device, independent from other streams. It supports with statement as a + context manager to ensure the operators within the with block are running + on the corresponding stream. See :ref:`cuda-semantics` for details. + + Args: + device(torch.device or int, optional): a device on which to allocate + the stream. If :attr:`device` is ``None`` (default) or a negative + integer, this will use the current device. + priority(int, optional): priority of the stream, which can be positive, 0, or negative. + A lower number indicates a higher priority. By default, the priority is set to 0. + If the value falls outside of the allowed priority range, it will automatically be + mapped to the nearest valid priority (lowest for large positive numbers or + highest for large negative numbers). + + """ + + def __new__(cls, device=None, priority=0, **kwargs): + # Check CUDA availability + if not torch.backends.cuda.is_built(): + raise RuntimeError("torch.cuda.Stream requires CUDA support") + # setting device manager is expensive, so we avoid it unless necessary + if device is None or ("stream_id" in kwargs and "device_index" in kwargs): + return super().__new__(cls, priority=priority, **kwargs) + else: + with torch.cuda.device(device): + return super().__new__(cls, priority=priority, **kwargs) + + def wait_event(self, event) -> None: + r"""Make all future work submitted to the stream wait for an event. + + Args: + event (torch.cuda.Event): an event to wait for. + + .. note:: This is a wrapper around ``cudaStreamWaitEvent()``: see + `CUDA Stream documentation`_ for more info. + + This function returns without waiting for :attr:`event`: only future + operations are affected. + + .. _CUDA Stream documentation: + https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__STREAM.html + """ + event.wait(self) + + def wait_stream(self, stream) -> None: + r"""Synchronize with another stream. + + All future work submitted to this stream will wait until all kernels + submitted to a given stream at the time of call complete. + + Args: + stream (Stream): a stream to synchronize. + + .. note:: This function returns without waiting for currently enqueued + kernels in :attr:`stream`: only future operations are affected. + """ + self.wait_event(stream.record_event()) + + def record_event(self, event=None): + r"""Record an event. + + Args: + event (torch.cuda.Event, optional): event to record. If not given, a new one + will be allocated. + + Returns: + Recorded event. + """ + if event is None: + event = Event() + event.record(self) + return event + + def query(self) -> bool: + r"""Check if all the work submitted has been completed. + + Returns: + A boolean indicating if all kernels in this stream are completed. + """ + return super().query() + + def synchronize(self) -> None: + r"""Wait for all the kernels in this stream to complete. + + .. note:: This is a wrapper around ``cudaStreamSynchronize()``: see + `CUDA Stream documentation`_ for more info. + """ + super().synchronize() + + @property + def _as_parameter_(self): + return ctypes.c_void_p(self.cuda_stream) + + def __eq__(self, o) -> bool: + if isinstance(o, Stream): + return super().__eq__(o) + return False + + def __hash__(self): + return hash((self.cuda_stream, self.device)) + + def __repr__(self): + return f"" + + def __cuda_stream__(self): + """Implements the CUDA Stream Protocol: + https://nvidia.github.io/cuda-python/cuda-core/latest/interoperability.html#cuda-stream-protocol + + Returns: + tuple: A 2-tuple of (version, handle) where version is the protocol version + and handle is the address of cudaStream_t (CUDA) or hipStream_t (ROCm) as a Python int. + """ + return (0, self.cuda_stream) + + +class ExternalStream(Stream): + r"""Wrapper around an externally allocated CUDA stream. + + This class is used to wrap streams allocated in other libraries in order + to facilitate data exchange and multi-library interactions. + + .. note:: This class doesn't manage the stream life-cycle, it is the user + responsibility to keep the referenced stream alive while this class is + being used. + + Args: + stream_ptr(int): Integer representation of the `cudaStream_t` value. + allocated externally. + device(torch.device or int, optional): the device where the stream + was originally allocated. If device is specified incorrectly, + subsequent launches using this stream may fail. + """ + + def __new__(cls, stream_ptr, device=None, **kwargs): + with torch.cuda.device(device): + return super().__new__(cls, stream_ptr=stream_ptr, **kwargs) + + +class Event(torch._C._CudaEventBase): + r"""Wrapper around a CUDA event. + + CUDA events are synchronization markers that can be used to monitor the + device's progress, to accurately measure timing, and to synchronize CUDA + streams. + + The underlying CUDA events are lazily initialized when the event is first + recorded or exported to another process. After creation, only streams on the + same device may record the event. However, streams on any device can wait on + the event. + + Args: + enable_timing (bool, optional): indicates if the event should measure time + (default: ``False``) + blocking (bool, optional): if ``True``, :meth:`wait` will be blocking (default: ``False``) + interprocess (bool): if ``True``, the event can be shared between processes + (default: ``False``) + external (bool, optional): indicates whether this event should create event record and event wait nodes, or create an internal cross-stream dependency, when captured in a cuda graph. See `cross-stream dependencies `_, `cudaEventRecordExternal `_, and `cudaEventWaitExternal `_ for more information about internal vs. external events. (default: ``False``) + + .. _CUDA Event Documentation: + https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__EVENT.html + """ # noqa: B950 + + def __new__( + cls, enable_timing=False, blocking=False, interprocess=False, external=False + ): + return super().__new__( + cls, + enable_timing=enable_timing, + blocking=blocking, + interprocess=interprocess, + external=external, + ) + + @classmethod + def from_ipc_handle(cls, device, handle): + r"""Reconstruct an event from an IPC handle on the given device.""" + return super().from_ipc_handle(device, handle) + + def record(self, stream=None): + r"""Record the event in a given stream. + + Uses ``torch.cuda.current_stream()`` if no stream is specified. The + stream's device must match the event's device. + """ + if stream is None: + stream = torch.cuda.current_stream() + super().record(stream) + + def wait(self, stream=None) -> None: + r"""Make all future work submitted to the given stream wait for this event. + + Use ``torch.cuda.current_stream()`` if no stream is specified. + + .. note:: This is a wrapper around ``cudaStreamWaitEvent()``: see + `CUDA Event documentation`_ for more info. + """ + if stream is None: + stream = torch.cuda.current_stream() + super().wait(stream) + + def query(self): + r"""Check if all work currently captured by event has completed. + + Returns: + A boolean indicating if all work currently captured by event has + completed. + """ + return super().query() + + def elapsed_time(self, end_event): + r"""Return the time elapsed. + + Time reported in milliseconds after the event was recorded and + before the end_event was recorded. + """ + return super().elapsed_time(end_event) + + def synchronize(self) -> None: + r"""Wait for the event to complete. + + Waits until the completion of all work currently captured in this event. + This prevents the CPU thread from proceeding until the event completes. + + .. note:: This is a wrapper around ``cudaEventSynchronize()``: see + `CUDA Event documentation`_ for more info. + """ + super().synchronize() + + def ipc_handle(self): + r"""Return an IPC handle of this event. + + If not recorded yet, the event will use the current device. + """ + return super().ipc_handle() + + @property + def _as_parameter_(self): + return ctypes.c_void_p(self.cuda_event) + + def __repr__(self) -> str: + if self.cuda_event: + return f"" + else: + return "" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/tunable.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/tunable.py new file mode 100644 index 0000000000000000000000000000000000000000..4a5ee73cbddd39c05f6fe269c1e280b3f5624a64 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/cuda/tunable.py @@ -0,0 +1,802 @@ +r""" +This module exposes a TunableOp interface. + +Some operations, such as GEMMs, could be implemented using more than one library +or more than one technique. For example, a GEMM could be implemented for CUDA or +ROCm using either the blas or blasLt libraries. Further, ROCm's rocblas and +hipblaslt libraries allow the user to query for all possible algorithms and then +choose one. How does one know which implementation is the fastest and should be +chosen? That's what TunableOp provides. + +Enabling TunableOp and Tuning Separately +======================================== + +The TunableOp feature is enabled separately from enabling the tuning phase +itself. Enabling TunableOp means that PyTorch will replace any standard +operators with their Tunable implementations. Any call to a TunableOp first +checks whether it has already been tuned for the given operator inputs. If so, +it will immediately call the tuned operation; no further tuning will take place +even when the tuning setting is enabled. Instead if no tuning result is found, +and tuning is enabled, the TunableOp will benchmark every registered +implementation of that operator for the given set of inputs and select the +fastest. + +File Input and Output +===================== + +The first time any TunableOp is invoked, the internal database of tuned +operations will be prepared by attempting to read the results from the given +file. The default filename is 'tunableop_results.csv'. To support tuning when +multiple GPUs are used across multiple processes, the GPU device ordinal is +automatically inserted into the filename to avoid multiple processes overwriting +the same file. + +If tuning is enabled and new tunings are discovered during the course of your +workload, it will also write out to this same filename with all tunings, both +the ones it read in at startup as well as the new ones found at runtime. This +can be used, for example, to build up a tunings file across many workloads by +reusing the same file. The output file is automatically created when the +application terminates. This behavior can be controlled by the C++ and Python +APIs but not the environment variables. + +Assuming you specified a filename, you'll end up with a CSV file with contents +like so:: + + Validator,PT_VERSION,2.2.0 + Validator,ROCM_VERSION,6.0.0.0-12969-1544e39 + Validator,HIPBLASLT_VERSION,0.6.0-a9c5cc7 + Validator,ROCBLAS_VERSION,4.0.0-72e57364-dirty + GemmTunableOp_float_NT,nt_25088_4096_64,Gemm_Hipblaslt_1219,1.262 + GemmTunableOp_float_NT,nt_4096_4096_64,Gemm_Rocblas_1216,0.033 + +Note the "Validator" lines. If you change a library version, or ROCm version, or +PyTorch version, TunableOp will detect this and reject the tunings file because +the prior tunings are likely affected by other software changes. + +The remaining lines are the tuned solutions for each TunableOp encountered +during your execution. Each line consists of 4 comma-separated fields: operator +name, operator parameters, solution name, and average execution time. The +execution time is an optional field. The CSV file can be edited, but with +caution. For example, the solution name (field 3) can be changed to "Default" +and it will fall back to the original PyTorch untuned implementation. Or, in the +case of ROCm's hipBLAS or hipBLASLt libraries, if you know the specific solution +index you can override the solution that TunableOp selected by replacing the +value. The operator name and parameters (fields 1 and 2) are internally named +and should not be modified. In the case of GemmTunableOp, field 1 indicates the +datatype and whether the inputs are transposed (T) or not (N) and field 2 +indicates the M, N, K input shapes. + +There is an option to enable verbose output but it is only recommended for +debugging purposes. This will produce a lot of diagnostic messages but may be +useful to see if TunableOp is being used at all. Otherwise, TunableOp is +completely silent, besides file output, unless there is a warning or error +during its use. The verbose option is only available by setting the environment +variable PYTORCH_TUNABLEOP_VEROBSE=1. + +A Note on Tuning Behavior, Warmup, and Cache Effects +==================================================== + +Tuning an operator consists of iterating through the list or registered +implementations and profiling each one. The profile is established by running a +single implementation in a loop multiple times and taking the average execution +time. There is also an optional warmup phase prior to tuning that can help with +reaching stable power states by the hardware. During tuning of a workload the +various hardware caches will more likely produce hits than when not tuning. +There are options for flushing the instruction cache and rotate the input tensors +which might help produce a more faithful profile of the tuned operator as if the +operator were run within a larger workload instead of in a tight, repetitive loop. + +By default, each possible solution for a given operator will be run for either +100 iterations or as many iterations that can be run within 30ms, whichever is +smaller, and its average execution will be calculated. The fastest solution +among all that were successfully profiled will be chosen. A profile might fail +if the given solution doesn't achieve the same accuracy as the default +implementation or if the solution returns an error code. + +Current Tunable Operators +========================= + +TunableGemm for ROCm +-------------------- + +Currently only a TunableGemm for ROCm is implemented. Note that CUDA builds of +PyTorch will function correctly when using TunableOp but the only solution +available to CUDA builds is the 'Default' implementation i.e. the original +cuBLAS default, now called through TunableOp. Any call to at::cuda::blas::gemm() +or ::bgemm() will be routed through TunableOp when enabled. Calling gemm() for a +given set of input arguments (transa, transb, m, n, k) will attempt to use the +fastest available implementation across both rocblas and hipblaslt. + +Offline Tuning +============== + +Motivation +---------- +There are several use cases for offline tuning. + +One use case involves a workload with a high-memory utilization, where regular tuning might lead to running out of memory. + +Another use case is for compute-intensive workloads. In such cases, it is more resource-efficient to collect +the GEMMs for the workload once and then tune repeatedly with different tuning parameters or libraries. + +Workflow +-------- +There are basically two steps: +1) Set the environment variables to collect the untuned GEMM and this will generate ``tunableop_untuned0.csv``: + +.. code-block:: bash + + export PYTORCH_TUNABLEOP_ENABLED=1 + export PYTORCH_TUNABLEOP_TUNING=0 + export PYTORCH_TUNABLEOP_RECORD_UNTUNED=1 + ... + +2) Run a Python script that reads the ``tunableop_untuned0.csv`` and generates the ``tunableop_results0.csv``, like this: + +.. code-block:: python + + import torch.cuda.tunable as tunable + import os + + os.putenv("PYTORCH_TUNABLEOP_ENABLED", "1") + os.putenv("PYTORCH_TUNABLEOP_TUNING", "1") + os.putenv("PYTORCH_TUNABLEOP_RECORD_UNTUNED", "0") + tunable.tune_gemm_in_file("tunableop_untuned0.csv") + + +It is also possible to take multiple untuned files and distribute the GEMMs for tuning to multiple GPUs +within a single node. In the first step, the GEMMs are first gathered and duplicate GEMMs are eliminated. +Next, the GEMMs are distributed to different GPUs for tuning. After all GEMMs are tuned, the results from +all the GPUs are then gathered into a single file whose base filename has ``_full0`` appended to it +(for example ``tunableop_results_full0.csv``). Finally, this new file, containing the gathered results, will be +duplicated N times, once for each GPU as convenience to the user will run the workload with the tuned +configuration on N GPUs. + +.. code-block:: python + + if __name__ == "__main__": + num_gpus = 8 # number of GPUs that will be used during the tuning process + tunable.mgpu_tune_gemm_in_file("tunableop_untuned?.csv", num_gpus) + +Note that the usage of the ``mgpu_tune_gemm_in_file`` API is different from its single GPU counterpart +(``tune_gemm_in_file``). The body of the Python script that calls the API must be wrapped in ``main()`` as shown +due to the use of concurrent futures module. The argument to ``mgpu_tune_gemm_in_file`` must contain a wild card +expression (``?`` or ``*``) to generate the list of untuned files containing the GEMMs to be processed. The ``num_gpus`` +must between 1 and the total number of GPUs available. + +Tuning Context +============== + +The behavior of TunableOp is currently manipulated through environment +variables, the C++ interface of at::cuda::tunable::getTuningContext(), or the +torch.cuda.tunable python interfaces. The environment variables take precedence +over any setting you manipulate using the C++ or Python APIs. + +Environment Variable Interface +------------------------------ +Environment variables are cached the first time they are read. You cannot use the +environment variable interface programmatically since the settings become fixed. +Use the C++ or Python APIs instead. + +""" + +import concurrent.futures +import glob +import multiprocessing as mp +import os +import shutil +import warnings +from typing import Optional + +import torch + + +__all__ = [ + "enable", + "is_enabled", + "tuning_enable", + "tuning_is_enabled", + "record_untuned_enable", + "record_untuned_is_enabled", + "set_max_tuning_duration", + "get_max_tuning_duration", + "set_max_tuning_iterations", + "get_max_tuning_iterations", + "set_filename", + "get_filename", + "get_results", + "get_validators", + "read_file", + "tune_gemm_in_file", + "mgpu_tune_gemm_in_file", + "set_rotating_buffer_size", + "get_rotating_buffer_size", + "set_numerical_check_tolerances", +] + + +def enable(val: bool = True) -> None: + r"""This is the big on/off switch for all TunableOp implementations.""" + torch._C._cuda_tunableop_enable(val) # type: ignore[attr-defined] + + +def is_enabled() -> bool: + r"""Returns whether the TunableOp feature is enabled.""" + return torch._C._cuda_tunableop_is_enabled() # type: ignore[attr-defined] + + +def tuning_enable(val: bool = True) -> None: + r"""Enable tuning of TunableOp implementations. + + When enabled, if a tuned entry isn't found, run the tuning step and record + the entry. + """ + torch._C._cuda_tunableop_tuning_enable(val) # type: ignore[attr-defined] + + +def tuning_is_enabled() -> bool: + r"""Returns whether TunableOp implementations can be tuned.""" + return torch._C._cuda_tunableop_tuning_is_enabled() # type: ignore[attr-defined] + + +def record_untuned_enable(val: bool = True) -> None: + r"""Enable recording untuned of TunableOp perations for offline tuning. + + When enabled, if a tuned entry isn't found, write it to the untuned file. + """ + torch._C._cuda_record_untuned_enable(val) # type: ignore[attr-defined] + + +def record_untuned_is_enabled() -> bool: + r"""Returns whether TunableOp operations are recorded for offline tuning.""" + return torch._C._cuda_record_untuned_is_enabled() # type: ignore[attr-defined] + + +def set_max_tuning_duration(duration: int) -> None: + r"""Set max time in milliseconds to spend tuning a given solution. + + If both max tuning duration and iterations are set, the smaller of the two + will be honored. At minimum 1 tuning iteration will always be run. + """ + torch._C._cuda_tunableop_set_max_tuning_duration(duration) # type: ignore[attr-defined] + + +def get_max_tuning_duration() -> int: + r"""Get max time to spend tuning a given solution.""" + return torch._C._cuda_tunableop_get_max_tuning_duration() # type: ignore[attr-defined] + + +def set_max_tuning_iterations(iterations: int) -> None: + r"""Set max number of iterations to spend tuning a given solution. + + If both max tuning duration and iterations are set, the smaller of the two + will be honored. At minimum 1 tuning iteration will always be run. + """ + torch._C._cuda_tunableop_set_max_tuning_iterations(iterations) # type: ignore[attr-defined] + + +def get_max_tuning_iterations() -> int: + r"""Get max iterations to spend tuning a given solution.""" + return torch._C._cuda_tunableop_get_max_tuning_iterations() # type: ignore[attr-defined] + + +def set_filename(filename: str, insert_device_ordinal: bool = False) -> None: + r"""Set the filename to use for input/output of tuning results. + + If :attr:`insert_device_ordinal` is ``True`` then the current device ordinal + will be added to the given filename automatically. This can be used in a + 1-process-per-gpu scenario to ensure all processes write to a separate file. + """ + torch._C._cuda_tunableop_set_filename(filename, insert_device_ordinal) # type: ignore[attr-defined] + + +def get_filename() -> str: + r"""Get the results filename.""" + return torch._C._cuda_tunableop_get_filename() # type: ignore[attr-defined] + + +def get_results() -> tuple[str, str, str, float]: + r"""Return all TunableOp results.""" + return torch._C._cuda_tunableop_get_results() # type: ignore[attr-defined] + + +def get_validators() -> tuple[str, str]: + r"""Return the TunableOp validators.""" + return torch._C._cuda_tunableop_get_validators() # type: ignore[attr-defined] + + +def read_file(filename: Optional[str] = None) -> bool: + r"""Read results from a TunableOp CSV file. + + If :attr:`filename` is not given, ``get_filename()`` is called. + """ + if filename is None: + filename = get_filename() + return torch._C._cuda_tunableop_read_file(filename) # type: ignore[attr-defined] + + +def set_rotating_buffer_size(buffer_size: int) -> None: + r"""Set rotating buffer size to this value in MB, if the buffer size is greater than zero. + + If less than zero, query L2 cache size. If equal to zero, means deactivate rotating buffer. + """ + return torch._C._cuda_tunableop_set_rotating_buffer_size(buffer_size) # type: ignore[attr-defined] + + +def get_rotating_buffer_size() -> int: + r"""Get the rotating buffer size in kilobytes.""" + return torch._C._cuda_tunableop_get_rotating_buffer_size() # type: ignore[attr-defined] + + +def set_numerical_check_tolerances( + enable: bool, atol: float = 1e-5, rtol: float = 1e-5 +) -> None: + r"""Set the atol and rtol values in numeric check""" + return torch._C._cuda_tunableop_set_numerical_check_tolerances(enable, atol, rtol) # type: ignore[attr-defined] + + +def tune_gemm_in_file(filename: str) -> None: + r"""tune GEMM in file.""" + + assert is_enabled() + assert tuning_is_enabled() + + deviceid = torch.cuda.current_device() + + with open(filename) as file: + for line in file: + if line.startswith(("Gemm", "ScaledGemm")): + _process_single_offline_gemm(line, deviceid) + + +def _gather_unique_untuned_gemm_from_files(filename_pattern: str) -> set[str]: + r"""Process multiple untuned results file and return a set with duplicates removed.""" + unique_gemm_entries = set() # set will avoid duplicates + + for file_path in glob.glob(filename_pattern): + with open(file_path) as file: + for line in file: + if line.startswith(("Gemm", "ScaledGemm")): + unique_gemm_entries.add(line) + + return unique_gemm_entries + + +def _gather_tunableop_results() -> None: + r"""Gather results from multiple tunableop results file and create a single file.""" + gemm_lines = set() + validator_lines = [] + + # Need to allow for the possibility that results filename was + # set with the Python API instead of with environment variable. + # Also possible that results filename was not set at all. + # There are several test cases to check, but ultimately we + # need a glob-able expression + results_filename = get_filename() # Note empty string could be returned here + + if ( + results_filename is not None and results_filename != "" + ): # Case were the Python API was used to set the filename + dot_pos = results_filename.find(".") + if dot_pos != -1 and dot_pos > 0: + # Replace the character just to the left of the dot + filename_pattern = ( + results_filename[: dot_pos - 1] + "?" + results_filename[dot_pos:] + ) + else: + filename_pattern = "" # Needed to make linter happy + else: # Case where the environment variable was used to set the filename. + results_filename_env = os.getenv("PYTORCH_TUNABLEOP_FILENAME") + if results_filename_env is None or results_filename_env == "": + filename_pattern = "tunableop_results?.csv" + elif "%d" in results_filename_env: + filename_pattern = results_filename_env.replace("%d", "?") + else: + filename_pattern = results_filename_env.replace(".", "?.") + + assert "?" in filename_pattern + + FirstFile = False + matching_files = glob.glob(filename_pattern) + num_matching_files = len(matching_files) + for file_path in matching_files: + with open(file_path) as file: + for line in file: + if line.startswith("Validator"): + if not (FirstFile): + # Only read Validator from first file + validator_lines.append(line) + else: + gemm_lines.add(line) + + FirstFile = True + + output_file = filename_pattern.replace("?", "_full0") + + with open(output_file, "w") as out_file: + for line in validator_lines: + out_file.write(line) + for line in gemm_lines: + out_file.write(line) + + # Create num_matching_copies of the results file + for i in range(1, num_matching_files): + duplicate_file = output_file.replace("0", str(i)) + shutil.copy(output_file, duplicate_file) + + +def _create_matrices( + m: int, + n: int, + k: int, + lda: int, + ldb: int, + ldc: int, + transA: bool, + transB: bool, + dtypeA: torch.dtype, + deviceid: str, + dtypeB: Optional[torch.dtype] = None, + randn: bool = True, + subMatrix: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + r"""Helper function for _process_single_offline_gemm. + Creates matrices that are then consumed by one of the Torch GEMM APIs. + """ + # Fill parameters set for use with ScaledGEMM + fillA = 0.25 + fillB = 0.75 + + if dtypeB is None: + dtypeB = dtypeA + + if subMatrix: + # User reference for understanding leading dimension: + # https://github.com/Reference-LAPACK/lapack/blob/master/BLAS/SRC/dgemm.f + # TO DO: According to lines 108 - 133, there is no lower bound on rowsA, + # but there is a restriction on rowsB. Using this formula for now as it + # seems to work for all UTs. + rowsA = rowsB = max(ldc, k) + + if randn: + matA = torch.randn(rowsA, lda, dtype=dtypeA, device=deviceid) + matB = torch.randn(rowsB, ldb, dtype=dtypeA, device=deviceid) + else: + matA = torch.full((rowsA, lda), fillA, dtype=dtypeB, device=deviceid) + matB = torch.full((rowsB, ldb), fillB, dtype=dtypeB, device=deviceid) + + subA = matA[:k, :m].t() if transA else matA[:m, :k] + subB = matB[:n, :k].t() if transB else matB[:k, :n] + return subA, subB + else: + if randn: + matA = ( + torch.rand(k, m, dtype=dtypeA, device=deviceid).t() + if transA + else torch.rand(m, k, dtype=dtypeA, device=deviceid) + ) + matB = ( + torch.rand(n, k, dtype=dtypeB, device=deviceid).t() + if transB + else torch.rand(k, n, dtype=dtypeB, device=deviceid) + ) + else: + matA = ( + torch.full((k, m), fillA, dtype=dtypeA, device=deviceid).t() + if transA + else torch.full((m, k), fillA, dtype=dtypeA, device=deviceid) + ) + matB = ( + torch.full((n, k), fillB, dtype=dtypeB, device=deviceid).t() + if transB + else torch.full((k, n), fillB, dtype=dtypeB, device=deviceid) + ) + return matA, matB + + +def _create_batch_matrices( + m: int, + n: int, + k: int, + b: int, + lda: int, + ldb: int, + ldc: int, + transA: bool, + transB: bool, + dtype: torch.dtype, + deviceid: str, + subMatrix: bool = False, +) -> tuple[torch.Tensor, torch.Tensor]: + r"""Helper function for _process_single_offline_gemm. + Creates batch matrices that are then consumed by one of the Torch GEMM APIs. + Similar to _create_matrices but for 3D batch matrices. + """ + if subMatrix: + # User reference for understanding leading dimension: + # https://github.com/Reference-LAPACK/lapack/blob/master/BLAS/SRC/dgemm.f + # TO DO: According to lines 108 - 133, there is no lower bound on rowsA, + # but there is a restriction on rowsB. Using this formula for now as it + # seems to work for all UTs. + rowsA = rowsB = max(ldc, k) + + matA = torch.randn(b, rowsA, lda, dtype=dtype, device=deviceid) + matB = torch.randn(b, rowsB, ldb, dtype=dtype, device=deviceid) + + subA = matA[:b, :k, :m].transpose(1, 2) if transA else matA[:b, :m, :k] + subB = matB[:b, :n, :k].transpose(1, 2) if transB else matB[:b, :k, :n] + return subA, subB + else: + matA = ( + torch.rand(b, k, m, dtype=dtype, device=deviceid) + if transA + else torch.rand(b, m, k, dtype=dtype, device=deviceid) + ) + matB = ( + torch.rand(b, n, k, dtype=dtype, device=deviceid) + if transB + else torch.rand(b, k, n, dtype=dtype, device=deviceid) + ) + matA = matA.transpose(1, 2) if transA else matA + matB = matB.transpose(1, 2) if transB else matB + return matA, matB + + +def _process_single_offline_gemm(untuned_gemm_line: str, gpu_id: int) -> None: + r"""Process a single untuned GEMM.""" + + deviceid = "cuda:" + str(gpu_id) + + dtype_dict = { + "float": torch.float32, + "tf32": torch.float32, + "double": torch.float64, + "BFloat16": torch.bfloat16, + "Half": torch.half, + "c10::complex": torch.complex128, + "c10::complex": torch.complex64, + "Float8_e4m3fn": torch.float8_e4m3fn, + "Float8_e5m2": torch.float8_e5m2, + "Float8_e4m3fnuz": torch.float8_e4m3fnuz, + "Float8_e5m2fnuz": torch.float8_e5m2fnuz, + } + + untuned_gemm = untuned_gemm_line.strip().split(",")[:] + + underscore_count = untuned_gemm[0].count("_") + + # Initialize dtype to make linter happy + dtype = None + dtypeA = None + dtypeB = None + dtypeC = None + + # Extract BLAS parameters + if underscore_count == 2: + [op_sig, data_type, layout] = untuned_gemm[0].split("_") + transB = layout[0] == "T" + transA = layout[1] == "T" + dtype = dtype_dict.get(data_type) + if data_type == "tf32": + torch.backends.cuda.matmul.allow_tf32 = True + else: + torch.backends.cuda.matmul.allow_tf32 = False + + else: # ScaledGEMM + count = untuned_gemm[0].count("_") + assert count in [6, 7] + untuned_gemm_temp = untuned_gemm[0].split("_") + # dtypeC = might not be FP8 type, keep track + # of the number of underscores + op_sig = untuned_gemm_temp[0] + data_typeA = untuned_gemm_temp[1] + "_" + untuned_gemm_temp[2] + data_typeB = untuned_gemm_temp[3] + "_" + untuned_gemm_temp[4] + if count == 7: + data_typeC = untuned_gemm_temp[5] + "_" + untuned_gemm_temp[6] + else: + data_typeC = untuned_gemm_temp[5] + transB = untuned_gemm_temp[count][0] == "T" + transA = untuned_gemm_temp[count][1] == "T" + dtypeA = dtype_dict.get(data_typeA) + dtypeB = dtype_dict.get(data_typeB) + dtypeC = dtype_dict.get(data_typeC) + + untuned_gemm_temp = untuned_gemm[1].split("_") + [n, m, k] = [int(g) for g in untuned_gemm_temp[1:4]] + if op_sig == "GemmStridedBatchedTunableOp": + assert untuned_gemm_temp[6] == "ld" + [ldb, lda, ldc] = [int(g) for g in untuned_gemm_temp[7:10]] + else: + assert untuned_gemm_temp[4] == "ld" + [ldb, lda, ldc] = [int(g) for g in untuned_gemm_temp[5:8]] + + # Detect subMatrix case + if all(item in [n, m, k] for item in [lda, ldb, ldc]): + subMatrix = False + else: + subMatrix = True + + if op_sig == "GemmTunableOp": + # Warnings for unsupported cases: + if m == 1 or n == 1 or k == 1: + if (not transA) and (not transB): + pass # case is supported + elif transA and n == 1: + pass # case is supported + else: + warnings.warn( + "Offline tuning is not supported for this GEMM. Use online tuning instead. " + + f"Skipped tuning for: {untuned_gemm[1]}", + stacklevel=2, + ) + return + + # Resolve linter issue + if dtype is None or not isinstance(dtype, torch.dtype): + raise TypeError(f"dtype must be a torch.dtype, but got {dtype}") + + matA, matB = _create_matrices( + m, n, k, lda, ldb, ldc, transA, transB, dtype, deviceid, subMatrix=subMatrix + ) + torch.mm(matA, matB) + + elif op_sig == "GemmStridedBatchedTunableOp": + # Warnings for unsupported cases: + if m == 1 or n == 1 or k == 1: + warnings.warn( + "Offline tuning is not support for this GEMM. Use online tuning instead. " + + f"Skipped tuning for: {untuned_gemm[1]}", + stacklevel=2, + ) + return + + [b] = [int(g) for g in untuned_gemm_temp[5:6]] + + # Resolve linter issue + if dtype is None or not isinstance(dtype, torch.dtype): + raise TypeError(f"dtype must be a torch.dtype, but got {dtype}") + + matA, matB = _create_batch_matrices( + m, + n, + k, + b, + lda, + ldb, + ldc, + transA, + transB, + dtype, + deviceid, + subMatrix=subMatrix, + ) + torch.bmm(matA, matB) + elif op_sig == "ScaledGemmTunableOp": + # Only combination supported by PyTorch + assert transB is True + assert transA is False + + # Resolve linter issue + if dtypeA is None or not isinstance(dtypeA, torch.dtype): + raise TypeError(f"dtype must be a torch.dtype, but got {dtypeA}") + + matA, matB = _create_matrices( + m, + n, + k, + lda, + ldb, + ldc, + transA, + transB, + dtypeA, + deviceid, + dtypeB=dtypeB, + randn=False, + subMatrix=subMatrix, + ) + + assert untuned_gemm_temp[8] == "rw" + if untuned_gemm_temp[9] == "1": + rowwise = True + else: + rowwise = False + if rowwise: + scaleA = ( + torch.ones((1, m), device=deviceid) + if transA + else torch.ones((m, 1), device=deviceid) + ) + scaleB = ( + torch.ones((1, n), device=deviceid) + if transB + else torch.ones((n, 1), device=deviceid) + ) + else: + scaleA = torch.tensor(0.8, device=deviceid) + scaleB = torch.tensor(0.9, device=deviceid) + + assert untuned_gemm_temp[10] == "bias" + if untuned_gemm_temp[11] == "None": # no bias vector + torch._scaled_mm( + matA, matB, scale_a=scaleA, scale_b=scaleB, out_dtype=dtypeC + ) + else: # bias vector present + fillbias = 0.10 + bias_dtype = dtype_dict.get(untuned_gemm_temp[11]) + bias = ( + torch.full((n,), fillbias, dtype=bias_dtype, device=deviceid) + if transB + else torch.full((m,), fillbias, dtype=bias_dtype, device=deviceid) + ) + torch._scaled_mm( + matA, matB, scale_a=scaleA, scale_b=scaleB, out_dtype=dtypeC, bias=bias + ) + + elif op_sig == "GemmAndBiasTunableOp": + # y = x*A^T + b + assert transA != transB + + # Resolve linter issue + if dtype is None or not isinstance(dtype, torch.dtype): + raise TypeError(f"dtype must be a torch.dtype, but got {dtype}") + + bias = torch.rand(n, dtype=dtype, device=deviceid) + + X, matA = _create_matrices( + m, n, k, lda, ldb, ldc, transA, transB, dtype, deviceid, subMatrix=subMatrix + ) + matA = matA.t() + torch.nn.functional.linear(X, matA, bias) + else: + warnings.warn(f"error: unknown op {op_sig}", stacklevel=2) + + +def _check_tuning_assertions() -> None: + r"""Helper function for multi-GPU tuning case. Need to check that TunableOp feature + is enabled and that tuning is enabled. + """ + + if is_enabled() is False: + warnings.warn("TunableOp was disabled. Trying to enable now.", stacklevel=2) + enable(True) + assert is_enabled() is True + assert tuning_is_enabled() is True + assert record_untuned_is_enabled() is False + + +def mgpu_tune_gemm_in_file(filename_pattern: str, num_gpus: int) -> None: + r"""Process one or more files and distribute work over one or more GPUs.""" + unique_gemm_entries = _gather_unique_untuned_gemm_from_files(filename_pattern) + + total_gpus = torch.cuda.device_count() + + assert 1 <= num_gpus <= total_gpus + + mp_context = mp.get_context("spawn") + + futures = [] # empty list to hold futures + + # GEMM are assigned to GPUs in a round robin manner + h = 0 + with concurrent.futures.ProcessPoolExecutor( + max_workers=num_gpus, + mp_context=mp_context, + initializer=_check_tuning_assertions, + ) as executor: + # The workers are a separate process. TunableOp will be + # enabled in the child processes if PYTORCH_TUNABLEOP_ENABLED=1 + # In the initializer, we also try to enable TunableOP if th + # environment variable was NOT set. + + for line in unique_gemm_entries: + future = executor.submit(_process_single_offline_gemm, line, h) + futures.append(future) + h = (h + 1) % num_gpus + + for future in concurrent.futures.as_completed(futures): + future.result() + + torch.cuda.synchronize() + + _gather_tunableop_results() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6567bb5078ac53ce2be2bd04546bff5ac8d9e40c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/__init__.py @@ -0,0 +1,174 @@ +r""" +The ``distributions`` package contains parameterizable probability distributions +and sampling functions. This allows the construction of stochastic computation +graphs and stochastic gradient estimators for optimization. This package +generally follows the design of the `TensorFlow Distributions`_ package. + +.. _`TensorFlow Distributions`: + https://arxiv.org/abs/1711.10604 + +It is not possible to directly backpropagate through random samples. However, +there are two main methods for creating surrogate functions that can be +backpropagated through. These are the score function estimator/likelihood ratio +estimator/REINFORCE and the pathwise derivative estimator. REINFORCE is commonly +seen as the basis for policy gradient methods in reinforcement learning, and the +pathwise derivative estimator is commonly seen in the reparameterization trick +in variational autoencoders. Whilst the score function only requires the value +of samples :math:`f(x)`, the pathwise derivative requires the derivative +:math:`f'(x)`. The next sections discuss these two in a reinforcement learning +example. For more details see +`Gradient Estimation Using Stochastic Computation Graphs`_ . + +.. _`Gradient Estimation Using Stochastic Computation Graphs`: + https://arxiv.org/abs/1506.05254 + +Score function +^^^^^^^^^^^^^^ + +When the probability density function is differentiable with respect to its +parameters, we only need :meth:`~torch.distributions.Distribution.sample` and +:meth:`~torch.distributions.Distribution.log_prob` to implement REINFORCE: + +.. math:: + + \Delta\theta = \alpha r \frac{\partial\log p(a|\pi^\theta(s))}{\partial\theta} + +where :math:`\theta` are the parameters, :math:`\alpha` is the learning rate, +:math:`r` is the reward and :math:`p(a|\pi^\theta(s))` is the probability of +taking action :math:`a` in state :math:`s` given policy :math:`\pi^\theta`. + +In practice we would sample an action from the output of a network, apply this +action in an environment, and then use ``log_prob`` to construct an equivalent +loss function. Note that we use a negative because optimizers use gradient +descent, whilst the rule above assumes gradient ascent. With a categorical +policy, the code for implementing REINFORCE would be as follows:: + + probs = policy_network(state) + # Note that this is equivalent to what used to be called multinomial + m = Categorical(probs) + action = m.sample() + next_state, reward = env.step(action) + loss = -m.log_prob(action) * reward + loss.backward() + +Pathwise derivative +^^^^^^^^^^^^^^^^^^^ + +The other way to implement these stochastic/policy gradients would be to use the +reparameterization trick from the +:meth:`~torch.distributions.Distribution.rsample` method, where the +parameterized random variable can be constructed via a parameterized +deterministic function of a parameter-free random variable. The reparameterized +sample therefore becomes differentiable. The code for implementing the pathwise +derivative would be as follows:: + + params = policy_network(state) + m = Normal(*params) + # Any distribution with .has_rsample == True could work based on the application + action = m.rsample() + next_state, reward = env.step(action) # Assuming that reward is differentiable + loss = -reward + loss.backward() +""" + +from . import transforms +from .bernoulli import Bernoulli +from .beta import Beta +from .binomial import Binomial +from .categorical import Categorical +from .cauchy import Cauchy +from .chi2 import Chi2 +from .constraint_registry import biject_to, transform_to +from .continuous_bernoulli import ContinuousBernoulli +from .dirichlet import Dirichlet +from .distribution import Distribution +from .exp_family import ExponentialFamily +from .exponential import Exponential +from .fishersnedecor import FisherSnedecor +from .gamma import Gamma +from .generalized_pareto import GeneralizedPareto +from .geometric import Geometric +from .gumbel import Gumbel +from .half_cauchy import HalfCauchy +from .half_normal import HalfNormal +from .independent import Independent +from .inverse_gamma import InverseGamma +from .kl import _add_kl_info, kl_divergence, register_kl +from .kumaraswamy import Kumaraswamy +from .laplace import Laplace +from .lkj_cholesky import LKJCholesky +from .log_normal import LogNormal +from .logistic_normal import LogisticNormal +from .lowrank_multivariate_normal import LowRankMultivariateNormal +from .mixture_same_family import MixtureSameFamily +from .multinomial import Multinomial +from .multivariate_normal import MultivariateNormal +from .negative_binomial import NegativeBinomial +from .normal import Normal +from .one_hot_categorical import OneHotCategorical, OneHotCategoricalStraightThrough +from .pareto import Pareto +from .poisson import Poisson +from .relaxed_bernoulli import RelaxedBernoulli +from .relaxed_categorical import RelaxedOneHotCategorical +from .studentT import StudentT +from .transformed_distribution import TransformedDistribution +from .transforms import * # noqa: F403 +from .uniform import Uniform +from .von_mises import VonMises +from .weibull import Weibull +from .wishart import Wishart + + +_add_kl_info() +del _add_kl_info + +__all__ = [ + "Bernoulli", + "Beta", + "Binomial", + "Categorical", + "Cauchy", + "Chi2", + "ContinuousBernoulli", + "Dirichlet", + "Distribution", + "Exponential", + "ExponentialFamily", + "FisherSnedecor", + "Gamma", + "GeneralizedPareto", + "Geometric", + "Gumbel", + "HalfCauchy", + "HalfNormal", + "Independent", + "InverseGamma", + "Kumaraswamy", + "LKJCholesky", + "Laplace", + "LogNormal", + "LogisticNormal", + "LowRankMultivariateNormal", + "MixtureSameFamily", + "Multinomial", + "MultivariateNormal", + "NegativeBinomial", + "Normal", + "OneHotCategorical", + "OneHotCategoricalStraightThrough", + "Pareto", + "RelaxedBernoulli", + "RelaxedOneHotCategorical", + "StudentT", + "Poisson", + "Uniform", + "VonMises", + "Weibull", + "Wishart", + "TransformedDistribution", + "biject_to", + "kl_divergence", + "register_kl", + "transform_to", +] +__all__.extend(transforms.__all__) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/bernoulli.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/bernoulli.py new file mode 100644 index 0000000000000000000000000000000000000000..99193da689e8d985d750451258add334b7f6158f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/bernoulli.py @@ -0,0 +1,145 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import nan, Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import ( + broadcast_all, + lazy_property, + logits_to_probs, + probs_to_logits, +) +from torch.nn.functional import binary_cross_entropy_with_logits +from torch.types import _Number, Number + + +__all__ = ["Bernoulli"] + + +class Bernoulli(ExponentialFamily): + r""" + Creates a Bernoulli distribution parameterized by :attr:`probs` + or :attr:`logits` (but not both). + + Samples are binary (0 or 1). They take the value `1` with probability `p` + and `0` with probability `1 - p`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Bernoulli(torch.tensor([0.3])) + >>> m.sample() # 30% chance 1; 70% chance 0 + tensor([ 0.]) + + Args: + probs (Number, Tensor): the probability of sampling `1` + logits (Number, Tensor): the log-odds of sampling `1` + validate_args (bool, optional): whether to validate arguments, None by default + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.unit_interval, "logits": constraints.real} + support = constraints.boolean + has_enumerate_support = True + _mean_carrier_measure = 0 + + def __init__( + self, + probs: Optional[Union[Tensor, Number]] = None, + logits: Optional[Union[Tensor, Number]] = None, + validate_args: Optional[bool] = None, + ) -> None: + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + is_scalar = isinstance(probs, _Number) + # pyrefly: ignore [read-only] + (self.probs,) = broadcast_all(probs) + else: + assert logits is not None # helps mypy + is_scalar = isinstance(logits, _Number) + # pyrefly: ignore [read-only] + (self.logits,) = broadcast_all(logits) + self._param = self.probs if probs is not None else self.logits + if is_scalar: + batch_shape = torch.Size() + else: + batch_shape = self._param.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Bernoulli, _instance) + batch_shape = torch.Size(batch_shape) + if "probs" in self.__dict__: + new.probs = self.probs.expand(batch_shape) + new._param = new.probs + if "logits" in self.__dict__: + new.logits = self.logits.expand(batch_shape) + new._param = new.logits + super(Bernoulli, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._param.new(*args, **kwargs) + + @property + def mean(self) -> Tensor: + return self.probs + + @property + def mode(self) -> Tensor: + mode = (self.probs >= 0.5).to(self.probs) + mode[self.probs == 0.5] = nan + return mode + + @property + def variance(self) -> Tensor: + return self.probs * (1 - self.probs) + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs, is_binary=True) + + @lazy_property + def probs(self) -> Tensor: + return logits_to_probs(self.logits, is_binary=True) + + @property + def param_shape(self) -> torch.Size: + return self._param.size() + + def sample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + with torch.no_grad(): + return torch.bernoulli(self.probs.expand(shape)) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + logits, value = broadcast_all(self.logits, value) + return -binary_cross_entropy_with_logits(logits, value, reduction="none") + + def entropy(self): + return binary_cross_entropy_with_logits( + self.logits, self.probs, reduction="none" + ) + + def enumerate_support(self, expand=True): + values = torch.arange(2, dtype=self._param.dtype, device=self._param.device) + values = values.view((-1,) + (1,) * len(self._batch_shape)) + if expand: + values = values.expand((-1,) + self._batch_shape) + return values + + @property + def _natural_params(self) -> tuple[Tensor]: + return (torch.logit(self.probs),) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x): + return torch.log1p(torch.exp(x)) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/beta.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/beta.py new file mode 100644 index 0000000000000000000000000000000000000000..1131dacf7d09ee8c40a95282c204f484df1f54bc --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/beta.py @@ -0,0 +1,119 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.dirichlet import Dirichlet +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Beta"] + + +class Beta(ExponentialFamily): + r""" + Beta distribution parameterized by :attr:`concentration1` and :attr:`concentration0`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Beta(torch.tensor([0.5]), torch.tensor([0.5])) + >>> m.sample() # Beta distributed with concentration concentration1 and concentration0 + tensor([ 0.1046]) + + Args: + concentration1 (float or Tensor): 1st concentration parameter of the distribution + (often referred to as alpha) + concentration0 (float or Tensor): 2nd concentration parameter of the distribution + (often referred to as beta) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "concentration1": constraints.positive, + "concentration0": constraints.positive, + } + support = constraints.unit_interval + has_rsample = True + + def __init__( + self, + concentration1: Union[Tensor, float], + concentration0: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + if isinstance(concentration1, _Number) and isinstance(concentration0, _Number): + concentration1_concentration0 = torch.tensor( + [float(concentration1), float(concentration0)] + ) + else: + concentration1, concentration0 = broadcast_all( + concentration1, concentration0 + ) + concentration1_concentration0 = torch.stack( + [concentration1, concentration0], -1 + ) + self._dirichlet = Dirichlet( + concentration1_concentration0, validate_args=validate_args + ) + super().__init__(self._dirichlet._batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Beta, _instance) + batch_shape = torch.Size(batch_shape) + new._dirichlet = self._dirichlet.expand(batch_shape) + super(Beta, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + @property + def mean(self) -> Tensor: + return self.concentration1 / (self.concentration1 + self.concentration0) + + @property + def mode(self) -> Tensor: + return self._dirichlet.mode[..., 0] + + @property + def variance(self) -> Tensor: + total = self.concentration1 + self.concentration0 + return self.concentration1 * self.concentration0 / (total.pow(2) * (total + 1)) + + def rsample(self, sample_shape: _size = ()) -> Tensor: + return self._dirichlet.rsample(sample_shape).select(-1, 0) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + heads_tails = torch.stack([value, 1.0 - value], -1) + return self._dirichlet.log_prob(heads_tails) + + def entropy(self): + return self._dirichlet.entropy() + + @property + def concentration1(self) -> Tensor: + result = self._dirichlet.concentration[..., 0] + if isinstance(result, _Number): + return torch.tensor([result]) + else: + return result + + @property + def concentration0(self) -> Tensor: + result = self._dirichlet.concentration[..., 1] + if isinstance(result, _Number): + return torch.tensor([result]) + else: + return result + + @property + def _natural_params(self) -> tuple[Tensor, Tensor]: + return (self.concentration1, self.concentration0) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x, y): + return torch.lgamma(x) + torch.lgamma(y) - torch.lgamma(x + y) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/binomial.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/binomial.py new file mode 100644 index 0000000000000000000000000000000000000000..87936c9febf8cf0c5e105cce256b556f0fbdd3f8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/binomial.py @@ -0,0 +1,182 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import ( + broadcast_all, + lazy_property, + logits_to_probs, + probs_to_logits, +) + + +__all__ = ["Binomial"] + + +def _clamp_by_zero(x): + # works like clamp(x, min=0) but has grad at 0 is 0.5 + return (x.clamp(min=0) + x - x.clamp(max=0)) / 2 + + +class Binomial(Distribution): + r""" + Creates a Binomial distribution parameterized by :attr:`total_count` and + either :attr:`probs` or :attr:`logits` (but not both). :attr:`total_count` must be + broadcastable with :attr:`probs`/:attr:`logits`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Binomial(100, torch.tensor([0 , .2, .8, 1])) + >>> x = m.sample() + tensor([ 0., 22., 71., 100.]) + + >>> m = Binomial(torch.tensor([[5.], [10.]]), torch.tensor([0.5, 0.8])) + >>> x = m.sample() + tensor([[ 4., 5.], + [ 7., 6.]]) + + Args: + total_count (int or Tensor): number of Bernoulli trials + probs (Tensor): Event probabilities + logits (Tensor): Event log-odds + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "total_count": constraints.nonnegative_integer, + "probs": constraints.unit_interval, + "logits": constraints.real, + } + has_enumerate_support = True + + def __init__( + self, + total_count: Union[Tensor, int] = 1, + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + ( + self.total_count, + # pyrefly: ignore [read-only] + self.probs, + ) = broadcast_all(total_count, probs) + self.total_count = self.total_count.type_as(self.probs) + else: + assert logits is not None # helps mypy + ( + self.total_count, + # pyrefly: ignore [read-only] + self.logits, + ) = broadcast_all(total_count, logits) + self.total_count = self.total_count.type_as(self.logits) + + self._param = self.probs if probs is not None else self.logits + batch_shape = self._param.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Binomial, _instance) + batch_shape = torch.Size(batch_shape) + new.total_count = self.total_count.expand(batch_shape) + if "probs" in self.__dict__: + new.probs = self.probs.expand(batch_shape) + new._param = new.probs + if "logits" in self.__dict__: + new.logits = self.logits.expand(batch_shape) + new._param = new.logits + super(Binomial, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._param.new(*args, **kwargs) + + @constraints.dependent_property(is_discrete=True, event_dim=0) + # pyrefly: ignore [bad-override] + def support(self): + return constraints.integer_interval(0, self.total_count) + + @property + def mean(self) -> Tensor: + return self.total_count * self.probs + + @property + def mode(self) -> Tensor: + return ((self.total_count + 1) * self.probs).floor().clamp(max=self.total_count) + + @property + def variance(self) -> Tensor: + return self.total_count * self.probs * (1 - self.probs) + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs, is_binary=True) + + @lazy_property + def probs(self) -> Tensor: + return logits_to_probs(self.logits, is_binary=True) + + @property + def param_shape(self) -> torch.Size: + return self._param.size() + + def sample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + with torch.no_grad(): + return torch.binomial( + self.total_count.expand(shape), self.probs.expand(shape) + ) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + log_factorial_n = torch.lgamma(self.total_count + 1) + log_factorial_k = torch.lgamma(value + 1) + log_factorial_nmk = torch.lgamma(self.total_count - value + 1) + # k * log(p) + (n - k) * log(1 - p) = k * (log(p) - log(1 - p)) + n * log(1 - p) + # (case logit < 0) = k * logit - n * log1p(e^logit) + # (case logit > 0) = k * logit - n * (log(p) - log(1 - p)) + n * log(p) + # = k * logit - n * logit - n * log1p(e^-logit) + # (merge two cases) = k * logit - n * max(logit, 0) - n * log1p(e^-|logit|) + normalize_term = ( + self.total_count * _clamp_by_zero(self.logits) + + self.total_count * torch.log1p(torch.exp(-torch.abs(self.logits))) + - log_factorial_n + ) + return ( + value * self.logits - log_factorial_k - log_factorial_nmk - normalize_term + ) + + def entropy(self): + total_count = int(self.total_count.max()) + if not self.total_count.min() == total_count: + raise NotImplementedError( + "Inhomogeneous total count not supported by `entropy`." + ) + + log_prob = self.log_prob(self.enumerate_support(False)) + return -(torch.exp(log_prob) * log_prob).sum(0) + + def enumerate_support(self, expand=True): + total_count = int(self.total_count.max()) + if not self.total_count.min() == total_count: + raise NotImplementedError( + "Inhomogeneous total count not supported by `enumerate_support`." + ) + values = torch.arange( + 1 + total_count, dtype=self._param.dtype, device=self._param.device + ) + values = values.view((-1,) + (1,) * len(self._batch_shape)) + if expand: + values = values.expand((-1,) + self._batch_shape) + return values diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/categorical.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/categorical.py new file mode 100644 index 0000000000000000000000000000000000000000..6a798b6c28f21813e3a1692238144e17c743a828 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/categorical.py @@ -0,0 +1,170 @@ +# mypy: allow-untyped-defs +from typing import Optional + +import torch +from torch import nan, Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import lazy_property, logits_to_probs, probs_to_logits + + +__all__ = ["Categorical"] + + +class Categorical(Distribution): + r""" + Creates a categorical distribution parameterized by either :attr:`probs` or + :attr:`logits` (but not both). + + .. note:: + It is equivalent to the distribution that :func:`torch.multinomial` + samples from. + + Samples are integers from :math:`\{0, \ldots, K-1\}` where `K` is ``probs.size(-1)``. + + If `probs` is 1-dimensional with length-`K`, each element is the relative probability + of sampling the class at that index. + + If `probs` is N-dimensional, the first N-1 dimensions are treated as a batch of + relative probability vectors. + + .. note:: The `probs` argument must be non-negative, finite and have a non-zero sum, + and it will be normalized to sum to 1 along the last dimension. :attr:`probs` + will return this normalized value. + The `logits` argument will be interpreted as unnormalized log probabilities + and can therefore be any real number. It will likewise be normalized so that + the resulting probabilities sum to 1 along the last dimension. :attr:`logits` + will return this normalized value. + + See also: :func:`torch.multinomial` + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Categorical(torch.tensor([ 0.25, 0.25, 0.25, 0.25 ])) + >>> m.sample() # equal probability of 0, 1, 2, 3 + tensor(3) + + Args: + probs (Tensor): event probabilities + logits (Tensor): event log probabilities (unnormalized) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.simplex, "logits": constraints.real_vector} + has_enumerate_support = True + + def __init__( + self, + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + if probs.dim() < 1: + raise ValueError("`probs` parameter must be at least one-dimensional.") + # pyrefly: ignore [read-only] + self.probs = probs / probs.sum(-1, keepdim=True) + else: + assert logits is not None # helps mypy + if logits.dim() < 1: + raise ValueError("`logits` parameter must be at least one-dimensional.") + # Normalize + # pyrefly: ignore [read-only] + self.logits = logits - logits.logsumexp(dim=-1, keepdim=True) + self._param = self.probs if probs is not None else self.logits + self._num_events = self._param.size()[-1] + batch_shape = ( + self._param.size()[:-1] if self._param.ndimension() > 1 else torch.Size() + ) + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Categorical, _instance) + batch_shape = torch.Size(batch_shape) + param_shape = batch_shape + torch.Size((self._num_events,)) + if "probs" in self.__dict__: + new.probs = self.probs.expand(param_shape) + new._param = new.probs + if "logits" in self.__dict__: + new.logits = self.logits.expand(param_shape) + new._param = new.logits + new._num_events = self._num_events + super(Categorical, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._param.new(*args, **kwargs) + + @constraints.dependent_property(is_discrete=True, event_dim=0) + # pyrefly: ignore [bad-override] + def support(self): + return constraints.integer_interval(0, self._num_events - 1) + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs) + + @lazy_property + def probs(self) -> Tensor: + return logits_to_probs(self.logits) + + @property + def param_shape(self) -> torch.Size: + return self._param.size() + + @property + def mean(self) -> Tensor: + return torch.full( + self._extended_shape(), + nan, + dtype=self.probs.dtype, + device=self.probs.device, + ) + + @property + def mode(self) -> Tensor: + return self.probs.argmax(dim=-1) + + @property + def variance(self) -> Tensor: + return torch.full( + self._extended_shape(), + nan, + dtype=self.probs.dtype, + device=self.probs.device, + ) + + def sample(self, sample_shape=torch.Size()): + if not isinstance(sample_shape, torch.Size): + sample_shape = torch.Size(sample_shape) + probs_2d = self.probs.reshape(-1, self._num_events) + samples_2d = torch.multinomial(probs_2d, sample_shape.numel(), True).T + return samples_2d.reshape(self._extended_shape(sample_shape)) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + value = value.long().unsqueeze(-1) + value, log_pmf = torch.broadcast_tensors(value, self.logits) + value = value[..., :1] + return log_pmf.gather(-1, value).squeeze(-1) + + def entropy(self): + min_real = torch.finfo(self.logits.dtype).min + logits = torch.clamp(self.logits, min=min_real) + p_log_p = logits * self.probs + return -p_log_p.sum(-1) + + def enumerate_support(self, expand=True): + num_events = self._num_events + values = torch.arange(num_events, dtype=torch.long, device=self._param.device) + values = values.view((-1,) + (1,) * len(self._batch_shape)) + if expand: + values = values.expand((-1,) + self._batch_shape) + return values diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/cauchy.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/cauchy.py new file mode 100644 index 0000000000000000000000000000000000000000..5a7f60e03c2f67c307a79e20c6e86dd1e977e925 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/cauchy.py @@ -0,0 +1,100 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import inf, nan, Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Cauchy"] + + +class Cauchy(Distribution): + r""" + Samples from a Cauchy (Lorentz) distribution. The distribution of the ratio of + independent normally distributed random variables with means `0` follows a + Cauchy distribution. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Cauchy(torch.tensor([0.0]), torch.tensor([1.0])) + >>> m.sample() # sample from a Cauchy distribution with loc=0 and scale=1 + tensor([ 2.3214]) + + Args: + loc (float or Tensor): mode or median of the distribution. + scale (float or Tensor): half width at half maximum. + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"loc": constraints.real, "scale": constraints.positive} + support = constraints.real + has_rsample = True + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.loc, self.scale = broadcast_all(loc, scale) + if isinstance(loc, _Number) and isinstance(scale, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.loc.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Cauchy, _instance) + batch_shape = torch.Size(batch_shape) + new.loc = self.loc.expand(batch_shape) + new.scale = self.scale.expand(batch_shape) + super(Cauchy, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + @property + def mean(self) -> Tensor: + return torch.full( + self._extended_shape(), nan, dtype=self.loc.dtype, device=self.loc.device + ) + + @property + def mode(self) -> Tensor: + return self.loc + + @property + def variance(self) -> Tensor: + return torch.full( + self._extended_shape(), inf, dtype=self.loc.dtype, device=self.loc.device + ) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + eps = self.loc.new(shape).cauchy_() + return self.loc + eps * self.scale + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + return ( + -math.log(math.pi) + - self.scale.log() + - (((value - self.loc) / self.scale) ** 2).log1p() + ) + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return torch.atan((value - self.loc) / self.scale) / math.pi + 0.5 + + def icdf(self, value): + return torch.tan(math.pi * (value - 0.5)) * self.scale + self.loc + + def entropy(self): + return math.log(4 * math.pi) + self.scale.log() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/chi2.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/chi2.py new file mode 100644 index 0000000000000000000000000000000000000000..fa23115fc0353c68c20a65ad694f43422cb95083 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/chi2.py @@ -0,0 +1,43 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.gamma import Gamma + + +__all__ = ["Chi2"] + + +class Chi2(Gamma): + r""" + Creates a Chi-squared distribution parameterized by shape parameter :attr:`df`. + This is exactly equivalent to ``Gamma(alpha=0.5*df, beta=0.5)`` + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Chi2(torch.tensor([1.0])) + >>> m.sample() # Chi2 distributed with shape df=1 + tensor([ 0.1046]) + + Args: + df (float or Tensor): shape parameter of the distribution + """ + + arg_constraints = {"df": constraints.positive} + + def __init__( + self, + df: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + super().__init__(0.5 * df, 0.5, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Chi2, _instance) + return super().expand(batch_shape, new) + + @property + def df(self) -> Tensor: + return self.concentration * 2 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/constraint_registry.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/constraint_registry.py new file mode 100644 index 0000000000000000000000000000000000000000..8907e5b467abf400f806e70197f70f526b93b5f7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/constraint_registry.py @@ -0,0 +1,291 @@ +# mypy: allow-untyped-defs +r""" +PyTorch provides two global :class:`ConstraintRegistry` objects that link +:class:`~torch.distributions.constraints.Constraint` objects to +:class:`~torch.distributions.transforms.Transform` objects. These objects both +input constraints and return transforms, but they have different guarantees on +bijectivity. + +1. ``biject_to(constraint)`` looks up a bijective + :class:`~torch.distributions.transforms.Transform` from ``constraints.real`` + to the given ``constraint``. The returned transform is guaranteed to have + ``.bijective = True`` and should implement ``.log_abs_det_jacobian()``. +2. ``transform_to(constraint)`` looks up a not-necessarily bijective + :class:`~torch.distributions.transforms.Transform` from ``constraints.real`` + to the given ``constraint``. The returned transform is not guaranteed to + implement ``.log_abs_det_jacobian()``. + +The ``transform_to()`` registry is useful for performing unconstrained +optimization on constrained parameters of probability distributions, which are +indicated by each distribution's ``.arg_constraints`` dict. These transforms often +overparameterize a space in order to avoid rotation; they are thus more +suitable for coordinate-wise optimization algorithms like Adam:: + + loc = torch.zeros(100, requires_grad=True) + unconstrained = torch.zeros(100, requires_grad=True) + scale = transform_to(Normal.arg_constraints["scale"])(unconstrained) + loss = -Normal(loc, scale).log_prob(data).sum() + +The ``biject_to()`` registry is useful for Hamiltonian Monte Carlo, where +samples from a probability distribution with constrained ``.support`` are +propagated in an unconstrained space, and algorithms are typically rotation +invariant.:: + + dist = Exponential(rate) + unconstrained = torch.zeros(100, requires_grad=True) + sample = biject_to(dist.support)(unconstrained) + potential_energy = -dist.log_prob(sample).sum() + +.. note:: + + An example where ``transform_to`` and ``biject_to`` differ is + ``constraints.simplex``: ``transform_to(constraints.simplex)`` returns a + :class:`~torch.distributions.transforms.SoftmaxTransform` that simply + exponentiates and normalizes its inputs; this is a cheap and mostly + coordinate-wise operation appropriate for algorithms like SVI. In + contrast, ``biject_to(constraints.simplex)`` returns a + :class:`~torch.distributions.transforms.StickBreakingTransform` that + bijects its input down to a one-fewer-dimensional space; this a more + expensive less numerically stable transform but is needed for algorithms + like HMC. + +The ``biject_to`` and ``transform_to`` objects can be extended by user-defined +constraints and transforms using their ``.register()`` method either as a +function on singleton constraints:: + + transform_to.register(my_constraint, my_transform) + +or as a decorator on parameterized constraints:: + + @transform_to.register(MyConstraintClass) + def my_factory(constraint): + assert isinstance(constraint, MyConstraintClass) + return MyTransform(constraint.param1, constraint.param2) + +You can create your own registry by creating a new :class:`ConstraintRegistry` +object. +""" + +from torch.distributions import constraints, transforms +from torch.types import _Number + + +__all__ = [ + "ConstraintRegistry", + "biject_to", + "transform_to", +] + + +class ConstraintRegistry: + """ + Registry to link constraints to transforms. + """ + + def __init__(self): + self._registry = {} + super().__init__() + + def register(self, constraint, factory=None): + """ + Registers a :class:`~torch.distributions.constraints.Constraint` + subclass in this registry. Usage:: + + @my_registry.register(MyConstraintClass) + def construct_transform(constraint): + assert isinstance(constraint, MyConstraint) + return MyTransform(constraint.arg_constraints) + + Args: + constraint (subclass of :class:`~torch.distributions.constraints.Constraint`): + A subclass of :class:`~torch.distributions.constraints.Constraint`, or + a singleton object of the desired class. + factory (Callable): A callable that inputs a constraint object and returns + a :class:`~torch.distributions.transforms.Transform` object. + """ + # Support use as decorator. + if factory is None: + return lambda factory: self.register(constraint, factory) + + # Support calling on singleton instances. + if isinstance(constraint, constraints.Constraint): + constraint = type(constraint) + + if not isinstance(constraint, type) or not issubclass( + constraint, constraints.Constraint + ): + raise TypeError( + f"Expected constraint to be either a Constraint subclass or instance, but got {constraint}" + ) + + self._registry[constraint] = factory + return factory + + def __call__(self, constraint): + """ + Looks up a transform to constrained space, given a constraint object. + Usage:: + + constraint = Normal.arg_constraints["scale"] + scale = transform_to(constraint)(torch.zeros(1)) # constrained + u = transform_to(constraint).inv(scale) # unconstrained + + Args: + constraint (:class:`~torch.distributions.constraints.Constraint`): + A constraint object. + + Returns: + A :class:`~torch.distributions.transforms.Transform` object. + + Raises: + `NotImplementedError` if no transform has been registered. + """ + # Look up by Constraint subclass. + try: + factory = self._registry[type(constraint)] + except KeyError: + raise NotImplementedError( + f"Cannot transform {type(constraint).__name__} constraints" + ) from None + return factory(constraint) + + +biject_to = ConstraintRegistry() +transform_to = ConstraintRegistry() + + +################################################################################ +# Registration Table +################################################################################ + + +@biject_to.register(constraints.real) +@transform_to.register(constraints.real) +def _transform_to_real(constraint): + return transforms.identity_transform + + +@biject_to.register(constraints.independent) +def _biject_to_independent(constraint): + base_transform = biject_to(constraint.base_constraint) + return transforms.IndependentTransform( + base_transform, constraint.reinterpreted_batch_ndims + ) + + +@transform_to.register(constraints.independent) +def _transform_to_independent(constraint): + base_transform = transform_to(constraint.base_constraint) + return transforms.IndependentTransform( + base_transform, constraint.reinterpreted_batch_ndims + ) + + +@biject_to.register(constraints.positive) +@biject_to.register(constraints.nonnegative) +@transform_to.register(constraints.positive) +@transform_to.register(constraints.nonnegative) +def _transform_to_positive(constraint): + return transforms.ExpTransform() + + +@biject_to.register(constraints.greater_than) +@biject_to.register(constraints.greater_than_eq) +@transform_to.register(constraints.greater_than) +@transform_to.register(constraints.greater_than_eq) +def _transform_to_greater_than(constraint): + return transforms.ComposeTransform( + [ + transforms.ExpTransform(), + transforms.AffineTransform(constraint.lower_bound, 1), + ] + ) + + +@biject_to.register(constraints.less_than) +@transform_to.register(constraints.less_than) +def _transform_to_less_than(constraint): + return transforms.ComposeTransform( + [ + transforms.ExpTransform(), + transforms.AffineTransform(constraint.upper_bound, -1), + ] + ) + + +@biject_to.register(constraints.interval) +@biject_to.register(constraints.half_open_interval) +@transform_to.register(constraints.interval) +@transform_to.register(constraints.half_open_interval) +def _transform_to_interval(constraint): + # Handle the special case of the unit interval. + lower_is_0 = ( + isinstance(constraint.lower_bound, _Number) and constraint.lower_bound == 0 + ) + upper_is_1 = ( + isinstance(constraint.upper_bound, _Number) and constraint.upper_bound == 1 + ) + if lower_is_0 and upper_is_1: + return transforms.SigmoidTransform() + + loc = constraint.lower_bound + scale = constraint.upper_bound - constraint.lower_bound + return transforms.ComposeTransform( + [transforms.SigmoidTransform(), transforms.AffineTransform(loc, scale)] + ) + + +@biject_to.register(constraints.simplex) +def _biject_to_simplex(constraint): + return transforms.StickBreakingTransform() + + +@transform_to.register(constraints.simplex) +def _transform_to_simplex(constraint): + return transforms.SoftmaxTransform() + + +# TODO define a bijection for LowerCholeskyTransform +@transform_to.register(constraints.lower_cholesky) +def _transform_to_lower_cholesky(constraint): + return transforms.LowerCholeskyTransform() + + +@transform_to.register(constraints.positive_definite) +@transform_to.register(constraints.positive_semidefinite) +def _transform_to_positive_definite(constraint): + return transforms.PositiveDefiniteTransform() + + +@biject_to.register(constraints.corr_cholesky) +@transform_to.register(constraints.corr_cholesky) +def _transform_to_corr_cholesky(constraint): + return transforms.CorrCholeskyTransform() + + +@biject_to.register(constraints.cat) +def _biject_to_cat(constraint): + return transforms.CatTransform( + [biject_to(c) for c in constraint.cseq], constraint.dim, constraint.lengths + ) + + +@transform_to.register(constraints.cat) +def _transform_to_cat(constraint): + return transforms.CatTransform( + [transform_to(c) for c in constraint.cseq], constraint.dim, constraint.lengths + ) + + +@biject_to.register(constraints.stack) +def _biject_to_stack(constraint): + return transforms.StackTransform( + [biject_to(c) for c in constraint.cseq], constraint.dim + ) + + +@transform_to.register(constraints.stack) +def _transform_to_stack(constraint): + return transforms.StackTransform( + [transform_to(c) for c in constraint.cseq], constraint.dim + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/constraints.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/constraints.py new file mode 100644 index 0000000000000000000000000000000000000000..9181a87abe4df29cc7c65ddf221c52f9aee147c5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/constraints.py @@ -0,0 +1,738 @@ +# mypy: allow-untyped-defs + +from collections.abc import Callable +from typing import Any, Optional + + +r""" +The following constraints are implemented: + +- ``constraints.boolean`` +- ``constraints.cat`` +- ``constraints.corr_cholesky`` +- ``constraints.dependent`` +- ``constraints.greater_than(lower_bound)`` +- ``constraints.greater_than_eq(lower_bound)`` +- ``constraints.independent(constraint, reinterpreted_batch_ndims)`` +- ``constraints.integer_interval(lower_bound, upper_bound)`` +- ``constraints.interval(lower_bound, upper_bound)`` +- ``constraints.less_than(upper_bound)`` +- ``constraints.lower_cholesky`` +- ``constraints.lower_triangular`` +- ``constraints.MixtureSameFamilyConstraint(base_constraint)`` +- ``constraints.multinomial`` +- ``constraints.nonnegative`` +- ``constraints.nonnegative_integer`` +- ``constraints.one_hot`` +- ``constraints.positive_integer`` +- ``constraints.positive`` +- ``constraints.positive_semidefinite`` +- ``constraints.positive_definite`` +- ``constraints.real_vector`` +- ``constraints.real`` +- ``constraints.simplex`` +- ``constraints.symmetric`` +- ``constraints.stack`` +- ``constraints.square`` +- ``constraints.symmetric`` +- ``constraints.unit_interval`` +""" + +import torch + + +__all__ = [ + "Constraint", + "boolean", + "cat", + "corr_cholesky", + "dependent", + "dependent_property", + "greater_than", + "greater_than_eq", + "independent", + "integer_interval", + "interval", + "half_open_interval", + "is_dependent", + "less_than", + "lower_cholesky", + "lower_triangular", + "MixtureSameFamilyConstraint", + "multinomial", + "nonnegative", + "nonnegative_integer", + "one_hot", + "positive", + "positive_semidefinite", + "positive_definite", + "positive_integer", + "real", + "real_vector", + "simplex", + "square", + "stack", + "symmetric", + "unit_interval", +] + + +class Constraint: + """ + Abstract base class for constraints. + + A constraint object represents a region over which a variable is valid, + e.g. within which a variable can be optimized. + + Attributes: + is_discrete (bool): Whether constrained space is discrete. + Defaults to False. + event_dim (int): Number of rightmost dimensions that together define + an event. The :meth:`check` method will remove this many dimensions + when computing validity. + """ + + is_discrete = False # Default to continuous. + event_dim = 0 # Default to univariate. + + def check(self, value): + """ + Returns a byte tensor of ``sample_shape + batch_shape`` indicating + whether each event in value satisfies this constraint. + """ + raise NotImplementedError + + def __repr__(self): + return self.__class__.__name__[1:] + "()" + + +class _Dependent(Constraint): + """ + Placeholder for variables whose support depends on other variables. + These variables obey no simple coordinate-wise constraints. + + Args: + is_discrete (bool): Optional value of ``.is_discrete`` in case this + can be computed statically. If not provided, access to the + ``.is_discrete`` attribute will raise a NotImplementedError. + event_dim (int): Optional value of ``.event_dim`` in case this + can be computed statically. If not provided, access to the + ``.event_dim`` attribute will raise a NotImplementedError. + """ + + def __init__(self, *, is_discrete=NotImplemented, event_dim=NotImplemented): + self._is_discrete = is_discrete + self._event_dim = event_dim + super().__init__() + + @property + def is_discrete(self) -> bool: # type: ignore[override] + if self._is_discrete is NotImplemented: + raise NotImplementedError(".is_discrete cannot be determined statically") + return self._is_discrete + + @property + def event_dim(self) -> int: # type: ignore[override] + if self._event_dim is NotImplemented: + raise NotImplementedError(".event_dim cannot be determined statically") + return self._event_dim + + def __call__(self, *, is_discrete=NotImplemented, event_dim=NotImplemented): + """ + Support for syntax to customize static attributes:: + + constraints.dependent(is_discrete=True, event_dim=1) + """ + if is_discrete is NotImplemented: + is_discrete = self._is_discrete + if event_dim is NotImplemented: + event_dim = self._event_dim + return _Dependent(is_discrete=is_discrete, event_dim=event_dim) + + def check(self, x): + raise ValueError("Cannot determine validity of dependent constraint") + + +def is_dependent(constraint): + """ + Checks if ``constraint`` is a ``_Dependent`` object. + + Args: + constraint : A ``Constraint`` object. + + Returns: + ``bool``: True if ``constraint`` can be refined to the type ``_Dependent``, False otherwise. + + Examples: + >>> import torch + >>> from torch.distributions import Bernoulli + >>> from torch.distributions.constraints import is_dependent + + >>> dist = Bernoulli(probs=torch.tensor([0.6], requires_grad=True)) + >>> constraint1 = dist.arg_constraints["probs"] + >>> constraint2 = dist.arg_constraints["logits"] + + >>> for constraint in [constraint1, constraint2]: + >>> if is_dependent(constraint): + >>> continue + """ + return isinstance(constraint, _Dependent) + + +class _DependentProperty(property, _Dependent): + """ + Decorator that extends @property to act like a `Dependent` constraint when + called on a class and act like a property when called on an object. + + Example:: + + class Uniform(Distribution): + def __init__(self, low, high): + self.low = low + self.high = high + + @constraints.dependent_property(is_discrete=False, event_dim=0) + def support(self): + return constraints.interval(self.low, self.high) + + Args: + fn (Callable): The function to be decorated. + is_discrete (bool): Optional value of ``.is_discrete`` in case this + can be computed statically. If not provided, access to the + ``.is_discrete`` attribute will raise a NotImplementedError. + event_dim (int): Optional value of ``.event_dim`` in case this + can be computed statically. If not provided, access to the + ``.event_dim`` attribute will raise a NotImplementedError. + """ + + def __init__( + self, + fn: Optional[Callable[..., Any]] = None, + *, + is_discrete: Optional[bool] = NotImplemented, + event_dim: Optional[int] = NotImplemented, + ) -> None: + super().__init__(fn) + self._is_discrete = is_discrete + self._event_dim = event_dim + + def __call__(self, fn: Callable[..., Any]) -> "_DependentProperty": # type: ignore[override] + """ + Support for syntax to customize static attributes:: + + @constraints.dependent_property(is_discrete=True, event_dim=1) + def support(self): ... + """ + return _DependentProperty( + fn, is_discrete=self._is_discrete, event_dim=self._event_dim + ) + + +class _IndependentConstraint(Constraint): + """ + Wraps a constraint by aggregating over ``reinterpreted_batch_ndims``-many + dims in :meth:`check`, so that an event is valid only if all its + independent entries are valid. + """ + + def __init__(self, base_constraint, reinterpreted_batch_ndims): + assert isinstance(base_constraint, Constraint) + assert isinstance(reinterpreted_batch_ndims, int) + assert reinterpreted_batch_ndims >= 0 + self.base_constraint = base_constraint + self.reinterpreted_batch_ndims = reinterpreted_batch_ndims + super().__init__() + + @property + def is_discrete(self) -> bool: # type: ignore[override] + return self.base_constraint.is_discrete + + @property + def event_dim(self) -> int: # type: ignore[override] + return self.base_constraint.event_dim + self.reinterpreted_batch_ndims + + def check(self, value): + result = self.base_constraint.check(value) + if result.dim() < self.reinterpreted_batch_ndims: + expected = self.base_constraint.event_dim + self.reinterpreted_batch_ndims + raise ValueError( + f"Expected value.dim() >= {expected} but got {value.dim()}" + ) + result = result.reshape( + result.shape[: result.dim() - self.reinterpreted_batch_ndims] + (-1,) + ) + result = result.all(-1) + return result + + def __repr__(self): + return f"{self.__class__.__name__[1:]}({repr(self.base_constraint)}, {self.reinterpreted_batch_ndims})" + + +class MixtureSameFamilyConstraint(Constraint): + """ + Constraint for the :class:`~torch.distribution.MixtureSameFamily` + distribution that adds back the rightmost batch dimension before + performing the validity check with the component distribution + constraint. + + Args: + base_constraint: The ``Constraint`` object of + the component distribution of + the :class:`~torch.distribution.MixtureSameFamily` distribution. + """ + + def __init__(self, base_constraint): + assert isinstance(base_constraint, Constraint) + self.base_constraint = base_constraint + super().__init__() + + @property + def is_discrete(self) -> bool: # type: ignore[override] + return self.base_constraint.is_discrete + + @property + def event_dim(self) -> int: # type: ignore[override] + return self.base_constraint.event_dim + + def check(self, value): + """ + Check validity of ``value`` as a possible outcome of sampling + the :class:`~torch.distribution.MixtureSameFamily` distribution. + """ + unsqueezed_value = value.unsqueeze(-1 - self.event_dim) + result = self.base_constraint.check(unsqueezed_value) + if value.dim() < self.event_dim: + raise ValueError( + f"Expected value.dim() >= {self.event_dim} but got {value.dim()}" + ) + num_dim_to_keep = value.dim() - self.event_dim + result = result.reshape(result.shape[:num_dim_to_keep] + (-1,)) + result = result.all(-1) + return result + + def __repr__(self): + return f"{self.__class__.__name__}({repr(self.base_constraint)})" + + +class _Boolean(Constraint): + """ + Constrain to the two values `{0, 1}`. + """ + + is_discrete = True + + def check(self, value): + return (value == 0) | (value == 1) + + +class _OneHot(Constraint): + """ + Constrain to one-hot vectors. + """ + + is_discrete = True + event_dim = 1 + + def check(self, value): + is_boolean = (value == 0) | (value == 1) + is_normalized = value.sum(-1).eq(1) + return is_boolean.all(-1) & is_normalized + + +class _IntegerInterval(Constraint): + """ + Constrain to an integer interval `[lower_bound, upper_bound]`. + """ + + is_discrete = True + + def __init__(self, lower_bound, upper_bound): + self.lower_bound = lower_bound + self.upper_bound = upper_bound + super().__init__() + + def check(self, value): + return ( + (value % 1 == 0) & (self.lower_bound <= value) & (value <= self.upper_bound) + ) + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += ( + f"(lower_bound={self.lower_bound}, upper_bound={self.upper_bound})" + ) + return fmt_string + + +class _IntegerLessThan(Constraint): + """ + Constrain to an integer interval `(-inf, upper_bound]`. + """ + + is_discrete = True + + def __init__(self, upper_bound): + self.upper_bound = upper_bound + super().__init__() + + def check(self, value): + return (value % 1 == 0) & (value <= self.upper_bound) + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += f"(upper_bound={self.upper_bound})" + return fmt_string + + +class _IntegerGreaterThan(Constraint): + """ + Constrain to an integer interval `[lower_bound, inf)`. + """ + + is_discrete = True + + def __init__(self, lower_bound): + self.lower_bound = lower_bound + super().__init__() + + def check(self, value): + return (value % 1 == 0) & (value >= self.lower_bound) + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += f"(lower_bound={self.lower_bound})" + return fmt_string + + +class _Real(Constraint): + """ + Trivially constrain to the extended real line `[-inf, inf]`. + """ + + def check(self, value): + return value == value # False for NANs. + + +class _GreaterThan(Constraint): + """ + Constrain to a real half line `(lower_bound, inf]`. + """ + + def __init__(self, lower_bound): + self.lower_bound = lower_bound + super().__init__() + + def check(self, value): + return self.lower_bound < value + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += f"(lower_bound={self.lower_bound})" + return fmt_string + + +class _GreaterThanEq(Constraint): + """ + Constrain to a real half line `[lower_bound, inf)`. + """ + + def __init__(self, lower_bound): + self.lower_bound = lower_bound + super().__init__() + + def check(self, value): + return self.lower_bound <= value + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += f"(lower_bound={self.lower_bound})" + return fmt_string + + +class _LessThan(Constraint): + """ + Constrain to a real half line `[-inf, upper_bound)`. + """ + + def __init__(self, upper_bound): + self.upper_bound = upper_bound + super().__init__() + + def check(self, value): + return value < self.upper_bound + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += f"(upper_bound={self.upper_bound})" + return fmt_string + + +class _Interval(Constraint): + """ + Constrain to a real interval `[lower_bound, upper_bound]`. + """ + + def __init__(self, lower_bound, upper_bound): + self.lower_bound = lower_bound + self.upper_bound = upper_bound + super().__init__() + + def check(self, value): + return (self.lower_bound <= value) & (value <= self.upper_bound) + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += ( + f"(lower_bound={self.lower_bound}, upper_bound={self.upper_bound})" + ) + return fmt_string + + +class _HalfOpenInterval(Constraint): + """ + Constrain to a real interval `[lower_bound, upper_bound)`. + """ + + def __init__(self, lower_bound, upper_bound): + self.lower_bound = lower_bound + self.upper_bound = upper_bound + super().__init__() + + def check(self, value): + return (self.lower_bound <= value) & (value < self.upper_bound) + + def __repr__(self): + fmt_string = self.__class__.__name__[1:] + fmt_string += ( + f"(lower_bound={self.lower_bound}, upper_bound={self.upper_bound})" + ) + return fmt_string + + +class _Simplex(Constraint): + """ + Constrain to the unit simplex in the innermost (rightmost) dimension. + Specifically: `x >= 0` and `x.sum(-1) == 1`. + """ + + event_dim = 1 + + def check(self, value): + return torch.all(value >= 0, dim=-1) & ((value.sum(-1) - 1).abs() < 1e-6) + + +class _Multinomial(Constraint): + """ + Constrain to nonnegative integer values summing to at most an upper bound. + + Note due to limitations of the Multinomial distribution, this currently + checks the weaker condition ``value.sum(-1) <= upper_bound``. In the future + this may be strengthened to ``value.sum(-1) == upper_bound``. + """ + + is_discrete = True + event_dim = 1 + + def __init__(self, upper_bound): + self.upper_bound = upper_bound + + def check(self, x): + return (x >= 0).all(dim=-1) & (x.sum(dim=-1) <= self.upper_bound) + + +class _LowerTriangular(Constraint): + """ + Constrain to lower-triangular square matrices. + """ + + event_dim = 2 + + def check(self, value): + value_tril = value.tril() + return (value_tril == value).view(value.shape[:-2] + (-1,)).min(-1)[0] + + +class _LowerCholesky(Constraint): + """ + Constrain to lower-triangular square matrices with positive diagonals. + """ + + event_dim = 2 + + def check(self, value): + value_tril = value.tril() + lower_triangular = ( + (value_tril == value).view(value.shape[:-2] + (-1,)).min(-1)[0] + ) + + positive_diagonal = (value.diagonal(dim1=-2, dim2=-1) > 0).min(-1)[0] + return lower_triangular & positive_diagonal + + +class _CorrCholesky(Constraint): + """ + Constrain to lower-triangular square matrices with positive diagonals and each + row vector being of unit length. + """ + + event_dim = 2 + + def check(self, value): + tol = ( + torch.finfo(value.dtype).eps * value.size(-1) * 10 + ) # 10 is an adjustable fudge factor + row_norm = torch.linalg.norm(value.detach(), dim=-1) + unit_row_norm = (row_norm - 1.0).abs().le(tol).all(dim=-1) + return _LowerCholesky().check(value) & unit_row_norm + + +class _Square(Constraint): + """ + Constrain to square matrices. + """ + + event_dim = 2 + + def check(self, value): + return torch.full( + size=value.shape[:-2], + fill_value=(value.shape[-2] == value.shape[-1]), + dtype=torch.bool, + device=value.device, + ) + + +class _Symmetric(_Square): + """ + Constrain to Symmetric square matrices. + """ + + def check(self, value): + square_check = super().check(value) + if not square_check.all(): + return square_check + return torch.isclose(value, value.mT, atol=1e-6).all(-2).all(-1) + + +class _PositiveSemidefinite(_Symmetric): + """ + Constrain to positive-semidefinite matrices. + """ + + def check(self, value): + sym_check = super().check(value) + if not sym_check.all(): + return sym_check + return torch.linalg.eigvalsh(value).ge(0).all(-1) + + +class _PositiveDefinite(_Symmetric): + """ + Constrain to positive-definite matrices. + """ + + def check(self, value): + sym_check = super().check(value) + if not sym_check.all(): + return sym_check + return torch.linalg.cholesky_ex(value).info.eq(0) + + +class _Cat(Constraint): + """ + Constraint functor that applies a sequence of constraints + `cseq` at the submatrices at dimension `dim`, + each of size `lengths[dim]`, in a way compatible with :func:`torch.cat`. + """ + + def __init__(self, cseq, dim=0, lengths=None): + assert all(isinstance(c, Constraint) for c in cseq) + self.cseq = list(cseq) + if lengths is None: + lengths = [1] * len(self.cseq) + self.lengths = list(lengths) + assert len(self.lengths) == len(self.cseq) + self.dim = dim + super().__init__() + + @property + def is_discrete(self) -> bool: # type: ignore[override] + return any(c.is_discrete for c in self.cseq) + + @property + def event_dim(self) -> int: # type: ignore[override] + return max(c.event_dim for c in self.cseq) + + def check(self, value): + assert -value.dim() <= self.dim < value.dim() + checks = [] + start = 0 + for constr, length in zip(self.cseq, self.lengths): + v = value.narrow(self.dim, start, length) + checks.append(constr.check(v)) + start = start + length # avoid += for jit compat + return torch.cat(checks, self.dim) + + +class _Stack(Constraint): + """ + Constraint functor that applies a sequence of constraints + `cseq` at the submatrices at dimension `dim`, + in a way compatible with :func:`torch.stack`. + """ + + def __init__(self, cseq, dim=0): + assert all(isinstance(c, Constraint) for c in cseq) + self.cseq = list(cseq) + self.dim = dim + super().__init__() + + @property + def is_discrete(self) -> bool: # type: ignore[override] + return any(c.is_discrete for c in self.cseq) + + @property + def event_dim(self) -> int: # type: ignore[override] + dim = max(c.event_dim for c in self.cseq) + if self.dim + dim < 0: + dim += 1 + return dim + + def check(self, value): + assert -value.dim() <= self.dim < value.dim() + vs = [value.select(self.dim, i) for i in range(value.size(self.dim))] + return torch.stack( + [constr.check(v) for v, constr in zip(vs, self.cseq)], self.dim + ) + + +# Public interface. +dependent = _Dependent() +dependent_property = _DependentProperty +independent = _IndependentConstraint +boolean = _Boolean() +one_hot = _OneHot() +nonnegative_integer = _IntegerGreaterThan(0) +positive_integer = _IntegerGreaterThan(1) +integer_interval = _IntegerInterval +real = _Real() +real_vector = independent(real, 1) +positive = _GreaterThan(0.0) +nonnegative = _GreaterThanEq(0.0) +greater_than = _GreaterThan +greater_than_eq = _GreaterThanEq +less_than = _LessThan +multinomial = _Multinomial +unit_interval = _Interval(0.0, 1.0) +interval = _Interval +half_open_interval = _HalfOpenInterval +simplex = _Simplex() +lower_triangular = _LowerTriangular() +lower_cholesky = _LowerCholesky() +corr_cholesky = _CorrCholesky() +square = _Square() +symmetric = _Symmetric() +positive_semidefinite = _PositiveSemidefinite() +positive_definite = _PositiveDefinite() +cat = _Cat +stack = _Stack diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/continuous_bernoulli.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/continuous_bernoulli.py new file mode 100644 index 0000000000000000000000000000000000000000..94220cb8d6d42c1bfaebddf89d3a0a984ce0fbdb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/continuous_bernoulli.py @@ -0,0 +1,250 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import ( + broadcast_all, + clamp_probs, + lazy_property, + logits_to_probs, + probs_to_logits, +) +from torch.nn.functional import binary_cross_entropy_with_logits +from torch.types import _Number, _size, Number + + +__all__ = ["ContinuousBernoulli"] + + +class ContinuousBernoulli(ExponentialFamily): + r""" + Creates a continuous Bernoulli distribution parameterized by :attr:`probs` + or :attr:`logits` (but not both). + + The distribution is supported in [0, 1] and parameterized by 'probs' (in + (0,1)) or 'logits' (real-valued). Note that, unlike the Bernoulli, 'probs' + does not correspond to a probability and 'logits' does not correspond to + log-odds, but the same names are used due to the similarity with the + Bernoulli. See [1] for more details. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = ContinuousBernoulli(torch.tensor([0.3])) + >>> m.sample() + tensor([ 0.2538]) + + Args: + probs (Number, Tensor): (0,1) valued parameters + logits (Number, Tensor): real valued parameters whose sigmoid matches 'probs' + + [1] The continuous Bernoulli: fixing a pervasive error in variational + autoencoders, Loaiza-Ganem G and Cunningham JP, NeurIPS 2019. + https://arxiv.org/abs/1907.06845 + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.unit_interval, "logits": constraints.real} + support = constraints.unit_interval + _mean_carrier_measure = 0 + has_rsample = True + + def __init__( + self, + probs: Optional[Union[Tensor, Number]] = None, + logits: Optional[Union[Tensor, Number]] = None, + lims: tuple[float, float] = (0.499, 0.501), + validate_args: Optional[bool] = None, + ) -> None: + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + is_scalar = isinstance(probs, _Number) + # pyrefly: ignore [read-only] + (self.probs,) = broadcast_all(probs) + # validate 'probs' here if necessary as it is later clamped for numerical stability + # close to 0 and 1, later on; otherwise the clamped 'probs' would always pass + if validate_args is not None: + if not self.arg_constraints["probs"].check(self.probs).all(): + raise ValueError("The parameter probs has invalid values") + # pyrefly: ignore [read-only] + self.probs = clamp_probs(self.probs) + else: + assert logits is not None # helps mypy + is_scalar = isinstance(logits, _Number) + # pyrefly: ignore [read-only] + (self.logits,) = broadcast_all(logits) + self._param = self.probs if probs is not None else self.logits + if is_scalar: + batch_shape = torch.Size() + else: + batch_shape = self._param.size() + self._lims = lims + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(ContinuousBernoulli, _instance) + new._lims = self._lims + batch_shape = torch.Size(batch_shape) + if "probs" in self.__dict__: + new.probs = self.probs.expand(batch_shape) + new._param = new.probs + if "logits" in self.__dict__: + new.logits = self.logits.expand(batch_shape) + new._param = new.logits + super(ContinuousBernoulli, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._param.new(*args, **kwargs) + + def _outside_unstable_region(self): + return torch.max( + torch.le(self.probs, self._lims[0]), torch.gt(self.probs, self._lims[1]) + ) + + def _cut_probs(self): + return torch.where( + self._outside_unstable_region(), + self.probs, + self._lims[0] * torch.ones_like(self.probs), + ) + + def _cont_bern_log_norm(self): + """computes the log normalizing constant as a function of the 'probs' parameter""" + cut_probs = self._cut_probs() + cut_probs_below_half = torch.where( + torch.le(cut_probs, 0.5), cut_probs, torch.zeros_like(cut_probs) + ) + cut_probs_above_half = torch.where( + torch.ge(cut_probs, 0.5), cut_probs, torch.ones_like(cut_probs) + ) + log_norm = torch.log( + torch.abs(torch.log1p(-cut_probs) - torch.log(cut_probs)) + ) - torch.where( + torch.le(cut_probs, 0.5), + torch.log1p(-2.0 * cut_probs_below_half), + torch.log(2.0 * cut_probs_above_half - 1.0), + ) + x = torch.pow(self.probs - 0.5, 2) + taylor = math.log(2.0) + (4.0 / 3.0 + 104.0 / 45.0 * x) * x + return torch.where(self._outside_unstable_region(), log_norm, taylor) + + @property + def mean(self) -> Tensor: + cut_probs = self._cut_probs() + mus = cut_probs / (2.0 * cut_probs - 1.0) + 1.0 / ( + torch.log1p(-cut_probs) - torch.log(cut_probs) + ) + x = self.probs - 0.5 + taylor = 0.5 + (1.0 / 3.0 + 16.0 / 45.0 * torch.pow(x, 2)) * x + return torch.where(self._outside_unstable_region(), mus, taylor) + + @property + def stddev(self) -> Tensor: + return torch.sqrt(self.variance) + + @property + def variance(self) -> Tensor: + cut_probs = self._cut_probs() + vars = cut_probs * (cut_probs - 1.0) / torch.pow( + 1.0 - 2.0 * cut_probs, 2 + ) + 1.0 / torch.pow(torch.log1p(-cut_probs) - torch.log(cut_probs), 2) + x = torch.pow(self.probs - 0.5, 2) + taylor = 1.0 / 12.0 - (1.0 / 15.0 - 128.0 / 945.0 * x) * x + return torch.where(self._outside_unstable_region(), vars, taylor) + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs, is_binary=True) + + @lazy_property + def probs(self) -> Tensor: + return clamp_probs(logits_to_probs(self.logits, is_binary=True)) + + @property + def param_shape(self) -> torch.Size: + return self._param.size() + + def sample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + u = torch.rand(shape, dtype=self.probs.dtype, device=self.probs.device) + with torch.no_grad(): + return self.icdf(u) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + u = torch.rand(shape, dtype=self.probs.dtype, device=self.probs.device) + return self.icdf(u) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + logits, value = broadcast_all(self.logits, value) + return ( + -binary_cross_entropy_with_logits(logits, value, reduction="none") + + self._cont_bern_log_norm() + ) + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + cut_probs = self._cut_probs() + cdfs = ( + torch.pow(cut_probs, value) * torch.pow(1.0 - cut_probs, 1.0 - value) + + cut_probs + - 1.0 + ) / (2.0 * cut_probs - 1.0) + unbounded_cdfs = torch.where(self._outside_unstable_region(), cdfs, value) + return torch.where( + torch.le(value, 0.0), + torch.zeros_like(value), + torch.where(torch.ge(value, 1.0), torch.ones_like(value), unbounded_cdfs), + ) + + def icdf(self, value): + cut_probs = self._cut_probs() + return torch.where( + self._outside_unstable_region(), + ( + torch.log1p(-cut_probs + value * (2.0 * cut_probs - 1.0)) + - torch.log1p(-cut_probs) + ) + / (torch.log(cut_probs) - torch.log1p(-cut_probs)), + value, + ) + + def entropy(self): + log_probs0 = torch.log1p(-self.probs) + log_probs1 = torch.log(self.probs) + return ( + self.mean * (log_probs0 - log_probs1) + - self._cont_bern_log_norm() + - log_probs0 + ) + + @property + def _natural_params(self) -> tuple[Tensor]: + return (self.logits,) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x): + """computes the log normalizing constant as a function of the natural parameter""" + out_unst_reg = torch.max( + torch.le(x, self._lims[0] - 0.5), torch.gt(x, self._lims[1] - 0.5) + ) + cut_nat_params = torch.where( + out_unst_reg, x, (self._lims[0] - 0.5) * torch.ones_like(x) + ) + log_norm = torch.log( + torch.abs(torch.special.expm1(cut_nat_params)) + ) - torch.log(torch.abs(cut_nat_params)) + taylor = 0.5 * x + torch.pow(x, 2) / 24.0 - torch.pow(x, 4) / 2880.0 + return torch.where(out_unst_reg, log_norm, taylor) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/dirichlet.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/dirichlet.py new file mode 100644 index 0000000000000000000000000000000000000000..cdcbe8f6e996427fea8e873b2dc32a845bdc1434 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/dirichlet.py @@ -0,0 +1,138 @@ +# mypy: allow-untyped-defs +from typing import Optional + +import torch +from torch import Tensor +from torch.autograd import Function +from torch.autograd.function import once_differentiable +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.types import _size + + +__all__ = ["Dirichlet"] + + +# This helper is exposed for testing. +def _Dirichlet_backward(x, concentration, grad_output): + total = concentration.sum(-1, True).expand_as(concentration) + grad = torch._dirichlet_grad(x, concentration, total) + return grad * (grad_output - (x * grad_output).sum(-1, True)) + + +class _Dirichlet(Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(ctx, concentration): + x = torch._sample_dirichlet(concentration) + ctx.save_for_backward(x, concentration) + return x + + @staticmethod + @once_differentiable + # pyrefly: ignore [bad-override] + def backward(ctx, grad_output): + x, concentration = ctx.saved_tensors + return _Dirichlet_backward(x, concentration, grad_output) + + +class Dirichlet(ExponentialFamily): + r""" + Creates a Dirichlet distribution parameterized by concentration :attr:`concentration`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Dirichlet(torch.tensor([0.5, 0.5])) + >>> m.sample() # Dirichlet distributed with concentration [0.5, 0.5] + tensor([ 0.1046, 0.8954]) + + Args: + concentration (Tensor): concentration parameter of the distribution + (often referred to as alpha) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "concentration": constraints.independent(constraints.positive, 1) + } + support = constraints.simplex + has_rsample = True + + def __init__( + self, + concentration: Tensor, + validate_args: Optional[bool] = None, + ) -> None: + if concentration.dim() < 1: + raise ValueError( + "`concentration` parameter must be at least one-dimensional." + ) + self.concentration = concentration + batch_shape, event_shape = concentration.shape[:-1], concentration.shape[-1:] + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Dirichlet, _instance) + batch_shape = torch.Size(batch_shape) + new.concentration = self.concentration.expand(batch_shape + self.event_shape) + super(Dirichlet, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + def rsample(self, sample_shape: _size = ()) -> Tensor: + shape = self._extended_shape(sample_shape) + concentration = self.concentration.expand(shape) + return _Dirichlet.apply(concentration) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + return ( + torch.xlogy(self.concentration - 1.0, value).sum(-1) + + torch.lgamma(self.concentration.sum(-1)) + - torch.lgamma(self.concentration).sum(-1) + ) + + @property + def mean(self) -> Tensor: + return self.concentration / self.concentration.sum(-1, True) + + @property + def mode(self) -> Tensor: + concentrationm1 = (self.concentration - 1).clamp(min=0.0) + mode = concentrationm1 / concentrationm1.sum(-1, True) + mask = (self.concentration < 1).all(dim=-1) + mode[mask] = torch.nn.functional.one_hot( + mode[mask].argmax(dim=-1), concentrationm1.shape[-1] + ).to(mode) + return mode + + @property + def variance(self) -> Tensor: + con0 = self.concentration.sum(-1, True) + return ( + self.concentration + * (con0 - self.concentration) + / (con0.pow(2) * (con0 + 1)) + ) + + def entropy(self): + k = self.concentration.size(-1) + a0 = self.concentration.sum(-1) + return ( + torch.lgamma(self.concentration).sum(-1) + - torch.lgamma(a0) + - (k - a0) * torch.digamma(a0) + - ((self.concentration - 1.0) * torch.digamma(self.concentration)).sum(-1) + ) + + @property + def _natural_params(self) -> tuple[Tensor]: + return (self.concentration,) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x): + return x.lgamma().sum(-1) - torch.lgamma(x.sum(-1)) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/distribution.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/distribution.py new file mode 100644 index 0000000000000000000000000000000000000000..dcdb2762cfffec3877ae0805af33f594f160e55e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/distribution.py @@ -0,0 +1,348 @@ +# mypy: allow-untyped-defs +import warnings +from typing import Optional +from typing_extensions import deprecated + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.utils import lazy_property +from torch.types import _size + + +__all__ = ["Distribution"] + + +class Distribution: + r""" + Distribution is the abstract base class for probability distributions. + + Args: + batch_shape (torch.Size): The shape over which parameters are batched. + event_shape (torch.Size): The shape of a single sample (without batching). + validate_args (bool, optional): Whether to validate arguments. Default: None. + """ + + has_rsample = False + has_enumerate_support = False + _validate_args = __debug__ + + @staticmethod + def set_default_validate_args(value: bool) -> None: + """ + Sets whether validation is enabled or disabled. + + The default behavior mimics Python's ``assert`` statement: validation + is on by default, but is disabled if Python is run in optimized mode + (via ``python -O``). Validation may be expensive, so you may want to + disable it once a model is working. + + Args: + value (bool): Whether to enable validation. + """ + if value not in [True, False]: + raise ValueError + Distribution._validate_args = value + + def __init__( + self, + batch_shape: torch.Size = torch.Size(), + event_shape: torch.Size = torch.Size(), + validate_args: Optional[bool] = None, + ) -> None: + self._batch_shape = batch_shape + self._event_shape = event_shape + if validate_args is not None: + self._validate_args = validate_args + if self._validate_args: + try: + arg_constraints = self.arg_constraints + except NotImplementedError: + arg_constraints = {} + warnings.warn( + f"{self.__class__} does not define `arg_constraints`. " + + "Please set `arg_constraints = {}` or initialize the distribution " + + "with `validate_args=False` to turn off validation.", + stacklevel=2, + ) + for param, constraint in arg_constraints.items(): + if constraints.is_dependent(constraint): + continue # skip constraints that cannot be checked + if param not in self.__dict__ and isinstance( + getattr(type(self), param), lazy_property + ): + continue # skip checking lazily-constructed args + value = getattr(self, param) + valid = constraint.check(value) + if not torch._is_all_true(valid): + raise ValueError( + f"Expected parameter {param} " + f"({type(value).__name__} of shape {tuple(value.shape)}) " + f"of distribution {repr(self)} " + f"to satisfy the constraint {repr(constraint)}, " + f"but found invalid values:\n{value}" + ) + super().__init__() + + def expand(self, batch_shape: _size, _instance=None): + """ + Returns a new distribution instance (or populates an existing instance + provided by a derived class) with batch dimensions expanded to + `batch_shape`. This method calls :class:`~torch.Tensor.expand` on + the distribution's parameters. As such, this does not allocate new + memory for the expanded distribution instance. Additionally, + this does not repeat any args checking or parameter broadcasting in + `__init__.py`, when an instance is first created. + + Args: + batch_shape (torch.Size): the desired expanded size. + _instance: new instance provided by subclasses that + need to override `.expand`. + + Returns: + New distribution instance with batch dimensions expanded to + `batch_size`. + """ + raise NotImplementedError + + @property + def batch_shape(self) -> torch.Size: + """ + Returns the shape over which parameters are batched. + """ + return self._batch_shape + + @property + def event_shape(self) -> torch.Size: + """ + Returns the shape of a single sample (without batching). + """ + return self._event_shape + + @property + def arg_constraints(self) -> dict[str, constraints.Constraint]: + """ + Returns a dictionary from argument names to + :class:`~torch.distributions.constraints.Constraint` objects that + should be satisfied by each argument of this distribution. Args that + are not tensors need not appear in this dict. + """ + raise NotImplementedError + + @property + def support(self) -> Optional[constraints.Constraint]: + """ + Returns a :class:`~torch.distributions.constraints.Constraint` object + representing this distribution's support. + """ + raise NotImplementedError + + @property + def mean(self) -> Tensor: + """ + Returns the mean of the distribution. + """ + raise NotImplementedError + + @property + def mode(self) -> Tensor: + """ + Returns the mode of the distribution. + """ + raise NotImplementedError(f"{self.__class__} does not implement mode") + + @property + def variance(self) -> Tensor: + """ + Returns the variance of the distribution. + """ + raise NotImplementedError + + @property + def stddev(self) -> Tensor: + """ + Returns the standard deviation of the distribution. + """ + return self.variance.sqrt() + + def sample(self, sample_shape: _size = torch.Size()) -> Tensor: + """ + Generates a sample_shape shaped sample or sample_shape shaped batch of + samples if the distribution parameters are batched. + """ + with torch.no_grad(): + return self.rsample(sample_shape) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + """ + Generates a sample_shape shaped reparameterized sample or sample_shape + shaped batch of reparameterized samples if the distribution parameters + are batched. + """ + raise NotImplementedError + + @deprecated( + "`sample_n(n)` will be deprecated. Use `sample((n,))` instead.", + category=FutureWarning, + ) + def sample_n(self, n: int) -> Tensor: + """ + Generates n samples or n batches of samples if the distribution + parameters are batched. + """ + return self.sample(torch.Size((n,))) + + def log_prob(self, value: Tensor) -> Tensor: + """ + Returns the log of the probability density/mass function evaluated at + `value`. + + Args: + value (Tensor): + """ + raise NotImplementedError + + def cdf(self, value: Tensor) -> Tensor: + """ + Returns the cumulative density/mass function evaluated at + `value`. + + Args: + value (Tensor): + """ + raise NotImplementedError + + def icdf(self, value: Tensor) -> Tensor: + """ + Returns the inverse cumulative density/mass function evaluated at + `value`. + + Args: + value (Tensor): + """ + raise NotImplementedError + + def enumerate_support(self, expand: bool = True) -> Tensor: + """ + Returns tensor containing all values supported by a discrete + distribution. The result will enumerate over dimension 0, so the shape + of the result will be `(cardinality,) + batch_shape + event_shape` + (where `event_shape = ()` for univariate distributions). + + Note that this enumerates over all batched tensors in lock-step + `[[0, 0], [1, 1], ...]`. With `expand=False`, enumeration happens + along dim 0, but with the remaining batch dimensions being + singleton dimensions, `[[0], [1], ..`. + + To iterate over the full Cartesian product use + `itertools.product(m.enumerate_support())`. + + Args: + expand (bool): whether to expand the support over the + batch dims to match the distribution's `batch_shape`. + + Returns: + Tensor iterating over dimension 0. + """ + raise NotImplementedError + + def entropy(self) -> Tensor: + """ + Returns entropy of distribution, batched over batch_shape. + + Returns: + Tensor of shape batch_shape. + """ + raise NotImplementedError + + def perplexity(self) -> Tensor: + """ + Returns perplexity of distribution, batched over batch_shape. + + Returns: + Tensor of shape batch_shape. + """ + return torch.exp(self.entropy()) + + def _extended_shape(self, sample_shape: _size = torch.Size()) -> torch.Size: + """ + Returns the size of the sample returned by the distribution, given + a `sample_shape`. Note, that the batch and event shapes of a distribution + instance are fixed at the time of construction. If this is empty, the + returned shape is upcast to (1,). + + Args: + sample_shape (torch.Size): the size of the sample to be drawn. + """ + if not isinstance(sample_shape, torch.Size): + sample_shape = torch.Size(sample_shape) + return torch.Size(sample_shape + self._batch_shape + self._event_shape) + + def _validate_sample(self, value: Tensor) -> None: + """ + Argument validation for distribution methods such as `log_prob`, + `cdf` and `icdf`. The rightmost dimensions of a value to be + scored via these methods must agree with the distribution's batch + and event shapes. + + Args: + value (Tensor): the tensor whose log probability is to be + computed by the `log_prob` method. + Raises + ValueError: when the rightmost dimensions of `value` do not match the + distribution's batch and event shapes. + """ + if not isinstance(value, torch.Tensor): + raise ValueError("The value argument to log_prob must be a Tensor") + + event_dim_start = len(value.size()) - len(self._event_shape) + if value.size()[event_dim_start:] != self._event_shape: + raise ValueError( + f"The right-most size of value must match event_shape: {value.size()} vs {self._event_shape}." + ) + + actual_shape = value.size() + expected_shape = self._batch_shape + self._event_shape + for i, j in zip(reversed(actual_shape), reversed(expected_shape)): + if i != 1 and j != 1 and i != j: + raise ValueError( + f"Value is not broadcastable with batch_shape+event_shape: {actual_shape} vs {expected_shape}." + ) + try: + support = self.support + except NotImplementedError: + warnings.warn( + f"{self.__class__} does not define `support` to enable " + + "sample validation. Please initialize the distribution with " + + "`validate_args=False` to turn off validation.", + stacklevel=2, + ) + return + assert support is not None + valid = support.check(value) + if not torch._is_all_true(valid): + raise ValueError( + "Expected value argument " + f"({type(value).__name__} of shape {tuple(value.shape)}) " + f"to be within the support ({repr(support)}) " + f"of the distribution {repr(self)}, " + f"but found invalid values:\n{value}" + ) + + def _get_checked_instance(self, cls, _instance=None): + if _instance is None and type(self).__init__ != cls.__init__: + raise NotImplementedError( + f"Subclass {self.__class__.__name__} of {cls.__name__} that defines a custom __init__ method " + "must also define a custom .expand() method." + ) + return self.__new__(type(self)) if _instance is None else _instance + + def __repr__(self) -> str: + param_names = [k for k, _ in self.arg_constraints.items() if k in self.__dict__] + args_string = ", ".join( + [ + f"{p}: {self.__dict__[p] if self.__dict__[p].numel() == 1 else self.__dict__[p].size()}" + for p in param_names + ] + ) + return self.__class__.__name__ + "(" + args_string + ")" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/exp_family.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/exp_family.py new file mode 100644 index 0000000000000000000000000000000000000000..ab8d340bd79310bab477a63f48f7d26c06f61919 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/exp_family.py @@ -0,0 +1,67 @@ +# mypy: allow-untyped-defs +from typing import Union + +import torch +from torch import Tensor +from torch.distributions.distribution import Distribution + + +__all__ = ["ExponentialFamily"] + + +class ExponentialFamily(Distribution): + r""" + ExponentialFamily is the abstract base class for probability distributions belonging to an + exponential family, whose probability mass/density function has the form is defined below + + .. math:: + + p_{F}(x; \theta) = \exp(\langle t(x), \theta\rangle - F(\theta) + k(x)) + + where :math:`\theta` denotes the natural parameters, :math:`t(x)` denotes the sufficient statistic, + :math:`F(\theta)` is the log normalizer function for a given family and :math:`k(x)` is the carrier + measure. + + Note: + This class is an intermediary between the `Distribution` class and distributions which belong + to an exponential family mainly to check the correctness of the `.entropy()` and analytic KL + divergence methods. We use this class to compute the entropy and KL divergence using the AD + framework and Bregman divergences (courtesy of: Frank Nielsen and Richard Nock, Entropies and + Cross-entropies of Exponential Families). + """ + + @property + def _natural_params(self) -> tuple[Tensor, ...]: + """ + Abstract method for natural parameters. Returns a tuple of Tensors based + on the distribution + """ + raise NotImplementedError + + def _log_normalizer(self, *natural_params): + """ + Abstract method for log normalizer function. Returns a log normalizer based on + the distribution and input + """ + raise NotImplementedError + + @property + def _mean_carrier_measure(self) -> float: + """ + Abstract method for expected carrier measure, which is required for computing + entropy. + """ + raise NotImplementedError + + def entropy(self): + """ + Method to compute the entropy using Bregman divergence of the log normalizer. + """ + result: Union[Tensor, float] = -self._mean_carrier_measure + nparams = [p.detach().requires_grad_() for p in self._natural_params] + lg_normal = self._log_normalizer(*nparams) + gradients = torch.autograd.grad(lg_normal.sum(), nparams, create_graph=True) + result += lg_normal + for np, g in zip(nparams, gradients): + result -= (np * g).reshape(self._batch_shape + (-1,)).sum(-1) + return result diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/exponential.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/exponential.py new file mode 100644 index 0000000000000000000000000000000000000000..b9596b7c0488f7d7a5cb72cfd5ed0ad0c749dbd3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/exponential.py @@ -0,0 +1,95 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Exponential"] + + +class Exponential(ExponentialFamily): + r""" + Creates a Exponential distribution parameterized by :attr:`rate`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Exponential(torch.tensor([1.0])) + >>> m.sample() # Exponential distributed with rate=1 + tensor([ 0.1046]) + + Args: + rate (float or Tensor): rate = 1 / scale of the distribution + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"rate": constraints.positive} + support = constraints.nonnegative + has_rsample = True + _mean_carrier_measure = 0 + + @property + def mean(self) -> Tensor: + return self.rate.reciprocal() + + @property + def mode(self) -> Tensor: + return torch.zeros_like(self.rate) + + @property + def stddev(self) -> Tensor: + return self.rate.reciprocal() + + @property + def variance(self) -> Tensor: + return self.rate.pow(-2) + + def __init__( + self, + rate: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + (self.rate,) = broadcast_all(rate) + batch_shape = torch.Size() if isinstance(rate, _Number) else self.rate.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Exponential, _instance) + batch_shape = torch.Size(batch_shape) + new.rate = self.rate.expand(batch_shape) + super(Exponential, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + return self.rate.new(shape).exponential_() / self.rate + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + return self.rate.log() - self.rate * value + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return 1 - torch.exp(-self.rate * value) + + def icdf(self, value): + return -torch.log1p(-value) / self.rate + + def entropy(self): + return 1.0 - torch.log(self.rate) + + @property + def _natural_params(self) -> tuple[Tensor]: + return (-self.rate,) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x): + return -torch.log(-x) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/fishersnedecor.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/fishersnedecor.py new file mode 100644 index 0000000000000000000000000000000000000000..8ebe2900f9b0a56f31b513172312d2e11dceb7b6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/fishersnedecor.py @@ -0,0 +1,108 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import nan, Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.gamma import Gamma +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["FisherSnedecor"] + + +class FisherSnedecor(Distribution): + r""" + Creates a Fisher-Snedecor distribution parameterized by :attr:`df1` and :attr:`df2`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = FisherSnedecor(torch.tensor([1.0]), torch.tensor([2.0])) + >>> m.sample() # Fisher-Snedecor-distributed with df1=1 and df2=2 + tensor([ 0.2453]) + + Args: + df1 (float or Tensor): degrees of freedom parameter 1 + df2 (float or Tensor): degrees of freedom parameter 2 + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"df1": constraints.positive, "df2": constraints.positive} + support = constraints.positive + has_rsample = True + + def __init__( + self, + df1: Union[Tensor, float], + df2: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.df1, self.df2 = broadcast_all(df1, df2) + self._gamma1 = Gamma(self.df1 * 0.5, self.df1) + self._gamma2 = Gamma(self.df2 * 0.5, self.df2) + + if isinstance(df1, _Number) and isinstance(df2, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.df1.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(FisherSnedecor, _instance) + batch_shape = torch.Size(batch_shape) + new.df1 = self.df1.expand(batch_shape) + new.df2 = self.df2.expand(batch_shape) + new._gamma1 = self._gamma1.expand(batch_shape) + new._gamma2 = self._gamma2.expand(batch_shape) + super(FisherSnedecor, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + @property + def mean(self) -> Tensor: + df2 = self.df2.clone(memory_format=torch.contiguous_format) + df2[df2 <= 2] = nan + return df2 / (df2 - 2) + + @property + def mode(self) -> Tensor: + mode = (self.df1 - 2) / self.df1 * self.df2 / (self.df2 + 2) + mode[self.df1 <= 2] = nan + return mode + + @property + def variance(self) -> Tensor: + df2 = self.df2.clone(memory_format=torch.contiguous_format) + df2[df2 <= 4] = nan + return ( + 2 + * df2.pow(2) + * (self.df1 + df2 - 2) + / (self.df1 * (df2 - 2).pow(2) * (df2 - 4)) + ) + + def rsample(self, sample_shape: _size = torch.Size(())) -> Tensor: + shape = self._extended_shape(sample_shape) + # X1 ~ Gamma(df1 / 2, 1 / df1), X2 ~ Gamma(df2 / 2, 1 / df2) + # Y = df2 * df1 * X1 / (df1 * df2 * X2) = X1 / X2 ~ F(df1, df2) + X1 = self._gamma1.rsample(sample_shape).view(shape) + X2 = self._gamma2.rsample(sample_shape).view(shape) + tiny = torch.finfo(X2.dtype).tiny + X2.clamp_(min=tiny) + Y = X1 / X2 + Y.clamp_(min=tiny) + return Y + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + ct1 = self.df1 * 0.5 + ct2 = self.df2 * 0.5 + ct3 = self.df1 / self.df2 + t1 = (ct1 + ct2).lgamma() - ct1.lgamma() - ct2.lgamma() + t2 = ct1 * ct3.log() + (ct1 - 1) * torch.log(value) + t3 = (ct1 + ct2) * torch.log1p(ct3 * value) + return t1 + t2 - t3 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/gamma.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/gamma.py new file mode 100644 index 0000000000000000000000000000000000000000..9044c2a7a0e03ba4ecd31d10f15d5199bb11007b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/gamma.py @@ -0,0 +1,120 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Gamma"] + + +def _standard_gamma(concentration): + return torch._standard_gamma(concentration) + + +class Gamma(ExponentialFamily): + r""" + Creates a Gamma distribution parameterized by shape :attr:`concentration` and :attr:`rate`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Gamma(torch.tensor([1.0]), torch.tensor([1.0])) + >>> m.sample() # Gamma distributed with concentration=1 and rate=1 + tensor([ 0.1046]) + + Args: + concentration (float or Tensor): shape parameter of the distribution + (often referred to as alpha) + rate (float or Tensor): rate parameter of the distribution + (often referred to as beta), rate = 1 / scale + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "concentration": constraints.positive, + "rate": constraints.positive, + } + support = constraints.nonnegative + has_rsample = True + _mean_carrier_measure = 0 + + @property + def mean(self) -> Tensor: + return self.concentration / self.rate + + @property + def mode(self) -> Tensor: + return ((self.concentration - 1) / self.rate).clamp(min=0) + + @property + def variance(self) -> Tensor: + return self.concentration / self.rate.pow(2) + + def __init__( + self, + concentration: Union[Tensor, float], + rate: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.concentration, self.rate = broadcast_all(concentration, rate) + if isinstance(concentration, _Number) and isinstance(rate, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.concentration.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Gamma, _instance) + batch_shape = torch.Size(batch_shape) + new.concentration = self.concentration.expand(batch_shape) + new.rate = self.rate.expand(batch_shape) + super(Gamma, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + value = _standard_gamma(self.concentration.expand(shape)) / self.rate.expand( + shape + ) + value.detach().clamp_( + min=torch.finfo(value.dtype).tiny + ) # do not record in autograd graph + return value + + def log_prob(self, value): + value = torch.as_tensor(value, dtype=self.rate.dtype, device=self.rate.device) + if self._validate_args: + self._validate_sample(value) + return ( + torch.xlogy(self.concentration, self.rate) + + torch.xlogy(self.concentration - 1, value) + - self.rate * value + - torch.lgamma(self.concentration) + ) + + def entropy(self): + return ( + self.concentration + - torch.log(self.rate) + + torch.lgamma(self.concentration) + + (1.0 - self.concentration) * torch.digamma(self.concentration) + ) + + @property + def _natural_params(self) -> tuple[Tensor, Tensor]: + return (self.concentration - 1, -self.rate) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x, y): + return torch.lgamma(x + 1) + (x + 1) * torch.log(-y.reciprocal()) + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return torch.special.gammainc(self.concentration, self.rate * value) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/generalized_pareto.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/generalized_pareto.py new file mode 100644 index 0000000000000000000000000000000000000000..e04f38b87c707252999ab95fdc4414176e9177d3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/generalized_pareto.py @@ -0,0 +1,153 @@ +# mypy: allow-untyped-defs +import math +from numbers import Number, Real + +import torch +from torch import inf, nan +from torch.distributions import constraints, Distribution +from torch.distributions.utils import broadcast_all + + +__all__ = ["GeneralizedPareto"] + + +class GeneralizedPareto(Distribution): + r""" + Creates a Generalized Pareto distribution parameterized by :attr:`loc`, :attr:`scale`, and :attr:`concentration`. + + The Generalized Pareto distribution is a family of continuous probability distributions on the real line. + Special cases include Exponential (when :attr:`loc` = 0, :attr:`concentration` = 0), Pareto (when :attr:`concentration` > 0, + :attr:`loc` = :attr:`scale` / :attr:`concentration`), and Uniform (when :attr:`concentration` = -1). + + This distribution is often used to model the tails of other distributions. This implementation is based on the + implementation in TensorFlow Probability. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = GeneralizedPareto(torch.tensor([0.1]), torch.tensor([2.0]), torch.tensor([0.4])) + >>> m.sample() # sample from a Generalized Pareto distribution with loc=0.1, scale=2.0, and concentration=0.4 + tensor([ 1.5623]) + + Args: + loc (float or Tensor): Location parameter of the distribution + scale (float or Tensor): Scale parameter of the distribution + concentration (float or Tensor): Concentration parameter of the distribution + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "loc": constraints.real, + "scale": constraints.positive, + "concentration": constraints.real, + } + has_rsample = True + + def __init__(self, loc, scale, concentration, validate_args=None): + self.loc, self.scale, self.concentration = broadcast_all( + loc, scale, concentration + ) + if ( + isinstance(loc, Number) + and isinstance(scale, Number) + and isinstance(concentration, Number) + ): + batch_shape = torch.Size() + else: + batch_shape = self.loc.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(GeneralizedPareto, _instance) + batch_shape = torch.Size(batch_shape) + new.loc = self.loc.expand(batch_shape) + new.scale = self.scale.expand(batch_shape) + new.concentration = self.concentration.expand(batch_shape) + super(GeneralizedPareto, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def rsample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + u = torch.rand(shape, dtype=self.loc.dtype, device=self.loc.device) + return self.icdf(u) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + z = self._z(value) + eq_zero = torch.isclose(self.concentration, torch.tensor(0.0)) + safe_conc = torch.where( + eq_zero, torch.ones_like(self.concentration), self.concentration + ) + y = 1 / safe_conc + torch.ones_like(z) + where_nonzero = torch.where(y == 0, y, y * torch.log1p(safe_conc * z)) + log_scale = ( + math.log(self.scale) if isinstance(self.scale, Real) else self.scale.log() + ) + return -log_scale - torch.where(eq_zero, z, where_nonzero) + + def log_survival_function(self, value): + if self._validate_args: + self._validate_sample(value) + z = self._z(value) + eq_zero = torch.isclose(self.concentration, torch.tensor(0.0)) + safe_conc = torch.where( + eq_zero, torch.ones_like(self.concentration), self.concentration + ) + where_nonzero = -torch.log1p(safe_conc * z) / safe_conc + return torch.where(eq_zero, -z, where_nonzero) + + def log_cdf(self, value): + return torch.log1p(-torch.exp(self.log_survival_function(value))) + + def cdf(self, value): + return torch.exp(self.log_cdf(value)) + + def icdf(self, value): + loc = self.loc + scale = self.scale + concentration = self.concentration + eq_zero = torch.isclose(concentration, torch.zeros_like(concentration)) + safe_conc = torch.where(eq_zero, torch.ones_like(concentration), concentration) + logu = torch.log1p(-value) + where_nonzero = loc + scale / safe_conc * torch.expm1(-safe_conc * logu) + where_zero = loc - scale * logu + return torch.where(eq_zero, where_zero, where_nonzero) + + def _z(self, x): + return (x - self.loc) / self.scale + + @property + def mean(self): + concentration = self.concentration + valid = concentration < 1 + safe_conc = torch.where(valid, concentration, 0.5) + result = self.loc + self.scale / (1 - safe_conc) + return torch.where(valid, result, nan) + + @property + def variance(self): + concentration = self.concentration + valid = concentration < 0.5 + safe_conc = torch.where(valid, concentration, 0.25) + # pyrefly: ignore [unsupported-operation] + result = self.scale**2 / ((1 - safe_conc) ** 2 * (1 - 2 * safe_conc)) + return torch.where(valid, result, nan) + + def entropy(self): + ans = torch.log(self.scale) + self.concentration + 1 + return torch.broadcast_to(ans, self._batch_shape) + + @property + def mode(self): + return self.loc + + @constraints.dependent_property(is_discrete=False, event_dim=0) + # pyrefly: ignore [bad-override] + def support(self): + lower = self.loc + upper = torch.where( + self.concentration < 0, lower - self.scale / self.concentration, inf + ) + return constraints.interval(lower, upper) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/geometric.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/geometric.py new file mode 100644 index 0000000000000000000000000000000000000000..fe0aca5f2b1f3ddf30fb642b6f3531d22f8ad7c0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/geometric.py @@ -0,0 +1,143 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import ( + broadcast_all, + lazy_property, + logits_to_probs, + probs_to_logits, +) +from torch.nn.functional import binary_cross_entropy_with_logits +from torch.types import _Number, Number + + +__all__ = ["Geometric"] + + +class Geometric(Distribution): + r""" + Creates a Geometric distribution parameterized by :attr:`probs`, + where :attr:`probs` is the probability of success of Bernoulli trials. + + .. math:: + + P(X=k) = (1-p)^{k} p, k = 0, 1, ... + + .. note:: + :func:`torch.distributions.geometric.Geometric` :math:`(k+1)`-th trial is the first success + hence draws samples in :math:`\{0, 1, \ldots\}`, whereas + :func:`torch.Tensor.geometric_` `k`-th trial is the first success hence draws samples in :math:`\{1, 2, \ldots\}`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Geometric(torch.tensor([0.3])) + >>> m.sample() # underlying Bernoulli has 30% chance 1; 70% chance 0 + tensor([ 2.]) + + Args: + probs (Number, Tensor): the probability of sampling `1`. Must be in range (0, 1] + logits (Number, Tensor): the log-odds of sampling `1`. + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.unit_interval, "logits": constraints.real} + support = constraints.nonnegative_integer + + def __init__( + self, + probs: Optional[Union[Tensor, Number]] = None, + logits: Optional[Union[Tensor, Number]] = None, + validate_args: Optional[bool] = None, + ) -> None: + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + # pyrefly: ignore [read-only] + (self.probs,) = broadcast_all(probs) + else: + assert logits is not None # helps mypy + # pyrefly: ignore [read-only] + (self.logits,) = broadcast_all(logits) + probs_or_logits = probs if probs is not None else logits + if isinstance(probs_or_logits, _Number): + batch_shape = torch.Size() + else: + assert probs_or_logits is not None # helps mypy + batch_shape = probs_or_logits.size() + super().__init__(batch_shape, validate_args=validate_args) + if self._validate_args and probs is not None: + # Add an extra check beyond unit_interval + value = self.probs + valid = value > 0 + if not valid.all(): + invalid_value = value.data[~valid] + raise ValueError( + "Expected parameter probs " + f"({type(value).__name__} of shape {tuple(value.shape)}) " + f"of distribution {repr(self)} " + f"to be positive but found invalid values:\n{invalid_value}" + ) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Geometric, _instance) + batch_shape = torch.Size(batch_shape) + if "probs" in self.__dict__: + new.probs = self.probs.expand(batch_shape) + if "logits" in self.__dict__: + new.logits = self.logits.expand(batch_shape) + super(Geometric, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + @property + def mean(self) -> Tensor: + return 1.0 / self.probs - 1.0 + + @property + def mode(self) -> Tensor: + return torch.zeros_like(self.probs) + + @property + def variance(self) -> Tensor: + return (1.0 / self.probs - 1.0) / self.probs + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs, is_binary=True) + + @lazy_property + def probs(self) -> Tensor: + return logits_to_probs(self.logits, is_binary=True) + + def sample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + tiny = torch.finfo(self.probs.dtype).tiny + with torch.no_grad(): + if torch._C._get_tracing_state(): + # [JIT WORKAROUND] lack of support for .uniform_() + u = torch.rand(shape, dtype=self.probs.dtype, device=self.probs.device) + u = u.clamp(min=tiny) + else: + u = self.probs.new(shape).uniform_(tiny, 1) + return (u.log() / (-self.probs).log1p()).floor() + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + value, probs = broadcast_all(value, self.probs) + probs = probs.clone(memory_format=torch.contiguous_format) + probs[(probs == 1) & (value == 0)] = 0 + return value * (-probs).log1p() + self.probs.log() + + def entropy(self): + return ( + binary_cross_entropy_with_logits(self.logits, self.probs, reduction="none") + / self.probs + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/gumbel.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/gumbel.py new file mode 100644 index 0000000000000000000000000000000000000000..c1c45221c92efe1cca8435668b2c5d5a97a34ae7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/gumbel.py @@ -0,0 +1,92 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import AffineTransform, ExpTransform +from torch.distributions.uniform import Uniform +from torch.distributions.utils import broadcast_all, euler_constant +from torch.types import _Number + + +__all__ = ["Gumbel"] + + +class Gumbel(TransformedDistribution): + r""" + Samples from a Gumbel Distribution. + + Examples:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Gumbel(torch.tensor([1.0]), torch.tensor([2.0])) + >>> m.sample() # sample from Gumbel distribution with loc=1, scale=2 + tensor([ 1.0124]) + + Args: + loc (float or Tensor): Location parameter of the distribution + scale (float or Tensor): Scale parameter of the distribution + """ + + arg_constraints = {"loc": constraints.real, "scale": constraints.positive} + # pyrefly: ignore [bad-override] + support = constraints.real + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.loc, self.scale = broadcast_all(loc, scale) + finfo = torch.finfo(self.loc.dtype) + if isinstance(loc, _Number) and isinstance(scale, _Number): + base_dist = Uniform(finfo.tiny, 1 - finfo.eps, validate_args=validate_args) + else: + base_dist = Uniform( + torch.full_like(self.loc, finfo.tiny), + torch.full_like(self.loc, 1 - finfo.eps), + validate_args=validate_args, + ) + transforms = [ + ExpTransform().inv, + AffineTransform(loc=0, scale=-torch.ones_like(self.scale)), + ExpTransform().inv, + AffineTransform(loc=loc, scale=-self.scale), + ] + super().__init__(base_dist, transforms, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Gumbel, _instance) + new.loc = self.loc.expand(batch_shape) + new.scale = self.scale.expand(batch_shape) + return super().expand(batch_shape, _instance=new) + + # Explicitly defining the log probability function for Gumbel due to precision issues + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + y = (self.loc - value) / self.scale + return (y - y.exp()) - self.scale.log() + + @property + def mean(self) -> Tensor: + return self.loc + self.scale * euler_constant + + @property + def mode(self) -> Tensor: + return self.loc + + @property + def stddev(self) -> Tensor: + return (math.pi / math.sqrt(6)) * self.scale + + @property + def variance(self) -> Tensor: + return self.stddev.pow(2) + + def entropy(self): + return self.scale.log() + (1 + euler_constant) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/half_cauchy.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/half_cauchy.py new file mode 100644 index 0000000000000000000000000000000000000000..cc6518f286b525cd1fa05ef7f9a9f5744213d4c8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/half_cauchy.py @@ -0,0 +1,93 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import inf, Tensor +from torch.distributions import constraints +from torch.distributions.cauchy import Cauchy +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import AbsTransform + + +__all__ = ["HalfCauchy"] + + +class HalfCauchy(TransformedDistribution): + r""" + Creates a half-Cauchy distribution parameterized by `scale` where:: + + X ~ Cauchy(0, scale) + Y = |X| ~ HalfCauchy(scale) + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = HalfCauchy(torch.tensor([1.0])) + >>> m.sample() # half-cauchy distributed with scale=1 + tensor([ 2.3214]) + + Args: + scale (float or Tensor): scale of the full Cauchy distribution + """ + + arg_constraints = {"scale": constraints.positive} + # pyrefly: ignore [bad-override] + support = constraints.nonnegative + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: Cauchy + + def __init__( + self, + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + base_dist = Cauchy(0, scale, validate_args=False) + super().__init__(base_dist, AbsTransform(), validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(HalfCauchy, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def scale(self) -> Tensor: + return self.base_dist.scale + + @property + def mean(self) -> Tensor: + return torch.full( + self._extended_shape(), + math.inf, + dtype=self.scale.dtype, + device=self.scale.device, + ) + + @property + def mode(self) -> Tensor: + return torch.zeros_like(self.scale) + + @property + def variance(self) -> Tensor: + return self.base_dist.variance + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + value = torch.as_tensor( + value, dtype=self.base_dist.scale.dtype, device=self.base_dist.scale.device + ) + log_prob = self.base_dist.log_prob(value) + math.log(2) + log_prob = torch.where(value >= 0, log_prob, -inf) + return log_prob + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return 2 * self.base_dist.cdf(value) - 1 + + def icdf(self, prob): + return self.base_dist.icdf((prob + 1) / 2) + + def entropy(self): + return self.base_dist.entropy() - math.log(2) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/half_normal.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/half_normal.py new file mode 100644 index 0000000000000000000000000000000000000000..148e69ac9ce100a39fd9da1cce0d784cea1e431c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/half_normal.py @@ -0,0 +1,85 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import inf, Tensor +from torch.distributions import constraints +from torch.distributions.normal import Normal +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import AbsTransform + + +__all__ = ["HalfNormal"] + + +class HalfNormal(TransformedDistribution): + r""" + Creates a half-normal distribution parameterized by `scale` where:: + + X ~ Normal(0, scale) + Y = |X| ~ HalfNormal(scale) + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = HalfNormal(torch.tensor([1.0])) + >>> m.sample() # half-normal distributed with scale=1 + tensor([ 0.1046]) + + Args: + scale (float or Tensor): scale of the full Normal distribution + """ + + arg_constraints = {"scale": constraints.positive} + # pyrefly: ignore [bad-override] + support = constraints.nonnegative + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: Normal + + def __init__( + self, + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + base_dist = Normal(0, scale, validate_args=False) + super().__init__(base_dist, AbsTransform(), validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(HalfNormal, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def scale(self) -> Tensor: + return self.base_dist.scale + + @property + def mean(self) -> Tensor: + return self.scale * math.sqrt(2 / math.pi) + + @property + def mode(self) -> Tensor: + return torch.zeros_like(self.scale) + + @property + def variance(self) -> Tensor: + return self.scale.pow(2) * (1 - 2 / math.pi) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + log_prob = self.base_dist.log_prob(value) + math.log(2) + log_prob = torch.where(value >= 0, log_prob, -inf) + return log_prob + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return 2 * self.base_dist.cdf(value) - 1 + + def icdf(self, prob): + return self.base_dist.icdf((prob + 1) / 2) + + def entropy(self): + return self.base_dist.entropy() - math.log(2) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/independent.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/independent.py new file mode 100644 index 0000000000000000000000000000000000000000..31443d772e1bd94131b594954ad909348993bac2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/independent.py @@ -0,0 +1,138 @@ +# mypy: allow-untyped-defs +from typing import Generic, Optional, TypeVar + +import torch +from torch import Size, Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import _sum_rightmost +from torch.types import _size + + +__all__ = ["Independent"] + + +D = TypeVar("D", bound=Distribution) + + +class Independent(Distribution, Generic[D]): + r""" + Reinterprets some of the batch dims of a distribution as event dims. + + This is mainly useful for changing the shape of the result of + :meth:`log_prob`. For example to create a diagonal Normal distribution with + the same shape as a Multivariate Normal distribution (so they are + interchangeable), you can:: + + >>> from torch.distributions.multivariate_normal import MultivariateNormal + >>> from torch.distributions.normal import Normal + >>> loc = torch.zeros(3) + >>> scale = torch.ones(3) + >>> mvn = MultivariateNormal(loc, scale_tril=torch.diag(scale)) + >>> [mvn.batch_shape, mvn.event_shape] + [torch.Size([]), torch.Size([3])] + >>> normal = Normal(loc, scale) + >>> [normal.batch_shape, normal.event_shape] + [torch.Size([3]), torch.Size([])] + >>> diagn = Independent(normal, 1) + >>> [diagn.batch_shape, diagn.event_shape] + [torch.Size([]), torch.Size([3])] + + Args: + base_distribution (torch.distributions.distribution.Distribution): a + base distribution + reinterpreted_batch_ndims (int): the number of batch dims to + reinterpret as event dims + """ + + arg_constraints: dict[str, constraints.Constraint] = {} + base_dist: D + + def __init__( + self, + base_distribution: D, + reinterpreted_batch_ndims: int, + validate_args: Optional[bool] = None, + ) -> None: + if reinterpreted_batch_ndims > len(base_distribution.batch_shape): + raise ValueError( + "Expected reinterpreted_batch_ndims <= len(base_distribution.batch_shape), " + f"actual {reinterpreted_batch_ndims} vs {len(base_distribution.batch_shape)}" + ) + shape: Size = base_distribution.batch_shape + base_distribution.event_shape + event_dim: int = reinterpreted_batch_ndims + len(base_distribution.event_shape) + batch_shape = shape[: len(shape) - event_dim] + event_shape = shape[len(shape) - event_dim :] + self.base_dist = base_distribution + self.reinterpreted_batch_ndims = reinterpreted_batch_ndims + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Independent, _instance) + batch_shape = torch.Size(batch_shape) + new.base_dist = self.base_dist.expand( + batch_shape + self.event_shape[: self.reinterpreted_batch_ndims] + ) + new.reinterpreted_batch_ndims = self.reinterpreted_batch_ndims + super(Independent, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + @property + def has_rsample(self) -> bool: # type: ignore[override] + return self.base_dist.has_rsample + + @property + def has_enumerate_support(self) -> bool: # type: ignore[override] + if self.reinterpreted_batch_ndims > 0: + return False + return self.base_dist.has_enumerate_support + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def support(self): + result = self.base_dist.support + if self.reinterpreted_batch_ndims: + result = constraints.independent(result, self.reinterpreted_batch_ndims) + return result + + @property + def mean(self) -> Tensor: + return self.base_dist.mean + + @property + def mode(self) -> Tensor: + return self.base_dist.mode + + @property + def variance(self) -> Tensor: + return self.base_dist.variance + + def sample(self, sample_shape=torch.Size()) -> Tensor: + return self.base_dist.sample(sample_shape) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + return self.base_dist.rsample(sample_shape) + + def log_prob(self, value): + log_prob = self.base_dist.log_prob(value) + return _sum_rightmost(log_prob, self.reinterpreted_batch_ndims) + + def entropy(self): + entropy = self.base_dist.entropy() + return _sum_rightmost(entropy, self.reinterpreted_batch_ndims) + + def enumerate_support(self, expand=True): + if self.reinterpreted_batch_ndims > 0: + raise NotImplementedError( + "Enumeration over cartesian product is not implemented" + ) + return self.base_dist.enumerate_support(expand=expand) + + def __repr__(self): + return ( + self.__class__.__name__ + + f"({self.base_dist}, {self.reinterpreted_batch_ndims})" + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/inverse_gamma.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/inverse_gamma.py new file mode 100644 index 0000000000000000000000000000000000000000..e87169c98de674b4b30d28bcb337f87ec689ccec --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/inverse_gamma.py @@ -0,0 +1,93 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.gamma import Gamma +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import PowerTransform + + +__all__ = ["InverseGamma"] + + +class InverseGamma(TransformedDistribution): + r""" + Creates an inverse gamma distribution parameterized by :attr:`concentration` and :attr:`rate` + where:: + + X ~ Gamma(concentration, rate) + Y = 1 / X ~ InverseGamma(concentration, rate) + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterinistic") + >>> m = InverseGamma(torch.tensor([2.0]), torch.tensor([3.0])) + >>> m.sample() + tensor([ 1.2953]) + + Args: + concentration (float or Tensor): shape parameter of the distribution + (often referred to as alpha) + rate (float or Tensor): rate = 1 / scale of the distribution + (often referred to as beta) + """ + + arg_constraints = { + "concentration": constraints.positive, + "rate": constraints.positive, + } + # pyrefly: ignore [bad-override] + support = constraints.positive + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: Gamma + + def __init__( + self, + concentration: Union[Tensor, float], + rate: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + base_dist = Gamma(concentration, rate, validate_args=validate_args) + neg_one = -base_dist.rate.new_ones(()) + super().__init__( + base_dist, PowerTransform(neg_one), validate_args=validate_args + ) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(InverseGamma, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def concentration(self) -> Tensor: + return self.base_dist.concentration + + @property + def rate(self) -> Tensor: + return self.base_dist.rate + + @property + def mean(self) -> Tensor: + result = self.rate / (self.concentration - 1) + return torch.where(self.concentration > 1, result, torch.inf) + + @property + def mode(self) -> Tensor: + return self.rate / (self.concentration + 1) + + @property + def variance(self) -> Tensor: + result = self.rate.square() / ( + (self.concentration - 1).square() * (self.concentration - 2) + ) + return torch.where(self.concentration > 2, result, torch.inf) + + def entropy(self): + return ( + self.concentration + + self.rate.log() + + self.concentration.lgamma() + - (1 + self.concentration) * self.concentration.digamma() + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/kl.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/kl.py new file mode 100644 index 0000000000000000000000000000000000000000..85932828d21af2d419849ba5bc507fef5cfd303c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/kl.py @@ -0,0 +1,973 @@ +# mypy: allow-untyped-defs +import math +import warnings +from collections.abc import Callable +from functools import total_ordering + +import torch +from torch import inf, Tensor + +from .bernoulli import Bernoulli +from .beta import Beta +from .binomial import Binomial +from .categorical import Categorical +from .cauchy import Cauchy +from .continuous_bernoulli import ContinuousBernoulli +from .dirichlet import Dirichlet +from .distribution import Distribution +from .exp_family import ExponentialFamily +from .exponential import Exponential +from .gamma import Gamma +from .geometric import Geometric +from .gumbel import Gumbel +from .half_normal import HalfNormal +from .independent import Independent +from .laplace import Laplace +from .lowrank_multivariate_normal import ( + _batch_lowrank_logdet, + _batch_lowrank_mahalanobis, + LowRankMultivariateNormal, +) +from .multivariate_normal import _batch_mahalanobis, MultivariateNormal +from .normal import Normal +from .one_hot_categorical import OneHotCategorical +from .pareto import Pareto +from .poisson import Poisson +from .transformed_distribution import TransformedDistribution +from .uniform import Uniform +from .utils import _sum_rightmost, euler_constant as _euler_gamma + + +_KL_REGISTRY: dict[ + tuple[type, type], Callable +] = {} # Source of truth mapping a few general (type, type) pairs to functions. +_KL_MEMOIZE: dict[ + tuple[type, type], Callable +] = {} # Memoized version mapping many specific (type, type) pairs to functions. + +__all__ = ["register_kl", "kl_divergence"] + + +def register_kl(type_p, type_q): + """ + Decorator to register a pairwise function with :meth:`kl_divergence`. + Usage:: + + @register_kl(Normal, Normal) + def kl_normal_normal(p, q): + # insert implementation here + + Lookup returns the most specific (type,type) match ordered by subclass. If + the match is ambiguous, a `RuntimeWarning` is raised. For example to + resolve the ambiguous situation:: + + @register_kl(BaseP, DerivedQ) + def kl_version1(p, q): ... + @register_kl(DerivedP, BaseQ) + def kl_version2(p, q): ... + + you should register a third most-specific implementation, e.g.:: + + register_kl(DerivedP, DerivedQ)(kl_version1) # Break the tie. + + Args: + type_p (type): A subclass of :class:`~torch.distributions.Distribution`. + type_q (type): A subclass of :class:`~torch.distributions.Distribution`. + """ + if not isinstance(type_p, type) and issubclass(type_p, Distribution): + raise TypeError( + f"Expected type_p to be a Distribution subclass but got {type_p}" + ) + if not isinstance(type_q, type) and issubclass(type_q, Distribution): + raise TypeError( + f"Expected type_q to be a Distribution subclass but got {type_q}" + ) + + def decorator(fun): + _KL_REGISTRY[type_p, type_q] = fun + _KL_MEMOIZE.clear() # reset since lookup order may have changed + return fun + + return decorator + + +@total_ordering +class _Match: + __slots__ = ["types"] + + def __init__(self, *types): + self.types = types + + def __eq__(self, other): + return self.types == other.types + + def __le__(self, other): + for x, y in zip(self.types, other.types): + if not issubclass(x, y): + return False + if x is not y: + break + return True + + +def _dispatch_kl(type_p, type_q): + """ + Find the most specific approximate match, assuming single inheritance. + """ + matches = [ + (super_p, super_q) + for super_p, super_q in _KL_REGISTRY + if issubclass(type_p, super_p) and issubclass(type_q, super_q) + ] + if not matches: + return NotImplemented + # Check that the left- and right- lexicographic orders agree. + # mypy isn't smart enough to know that _Match implements __lt__ + # see: https://github.com/python/typing/issues/760#issuecomment-710670503 + left_p, left_q = min(_Match(*m) for m in matches).types # type: ignore[type-var] + right_q, right_p = min(_Match(*reversed(m)) for m in matches).types # type: ignore[type-var] + left_fun = _KL_REGISTRY[left_p, left_q] + right_fun = _KL_REGISTRY[right_p, right_q] + if left_fun is not right_fun: + warnings.warn( + f"Ambiguous kl_divergence({type_p.__name__}, {type_q.__name__}). " + f"Please register_kl({left_p.__name__}, {right_q.__name__})", + RuntimeWarning, + stacklevel=2, + ) + return left_fun + + +def _infinite_like(tensor): + """ + Helper function for obtaining infinite KL Divergence throughout + """ + return torch.full_like(tensor, inf) + + +def _x_log_x(tensor): + """ + Utility function for calculating x log x + """ + return torch.special.xlogy(tensor, tensor) # produces correct result for x=0 + + +def _batch_trace_XXT(bmat): + """ + Utility function for calculating the trace of XX^{T} with X having arbitrary trailing batch dimensions + """ + n = bmat.size(-1) + m = bmat.size(-2) + flat_trace = bmat.reshape(-1, m * n).pow(2).sum(-1) + return flat_trace.reshape(bmat.shape[:-2]) + + +def kl_divergence(p: Distribution, q: Distribution) -> Tensor: + r""" + Compute Kullback-Leibler divergence :math:`KL(p \| q)` between two distributions. + + .. math:: + + KL(p \| q) = \int p(x) \log\frac {p(x)} {q(x)} \,dx + + Args: + p (Distribution): A :class:`~torch.distributions.Distribution` object. + q (Distribution): A :class:`~torch.distributions.Distribution` object. + + Returns: + Tensor: A batch of KL divergences of shape `batch_shape`. + + Raises: + NotImplementedError: If the distribution types have not been registered via + :meth:`register_kl`. + """ + try: + fun = _KL_MEMOIZE[type(p), type(q)] + except KeyError: + fun = _dispatch_kl(type(p), type(q)) + _KL_MEMOIZE[type(p), type(q)] = fun + if fun is NotImplemented: + raise NotImplementedError( + f"No KL(p || q) is implemented for p type {p.__class__.__name__} and q type {q.__class__.__name__}" + ) + return fun(p, q) + + +################################################################################ +# KL Divergence Implementations +################################################################################ + +# Same distributions + + +@register_kl(Bernoulli, Bernoulli) +def _kl_bernoulli_bernoulli(p, q): + t1 = p.probs * ( + torch.nn.functional.softplus(-q.logits) + - torch.nn.functional.softplus(-p.logits) + ) + t1[q.probs == 0] = inf + t1[p.probs == 0] = 0 + t2 = (1 - p.probs) * ( + torch.nn.functional.softplus(q.logits) - torch.nn.functional.softplus(p.logits) + ) + t2[q.probs == 1] = inf + t2[p.probs == 1] = 0 + return t1 + t2 + + +@register_kl(Beta, Beta) +def _kl_beta_beta(p, q): + sum_params_p = p.concentration1 + p.concentration0 + sum_params_q = q.concentration1 + q.concentration0 + t1 = q.concentration1.lgamma() + q.concentration0.lgamma() + (sum_params_p).lgamma() + t2 = p.concentration1.lgamma() + p.concentration0.lgamma() + (sum_params_q).lgamma() + t3 = (p.concentration1 - q.concentration1) * torch.digamma(p.concentration1) + t4 = (p.concentration0 - q.concentration0) * torch.digamma(p.concentration0) + t5 = (sum_params_q - sum_params_p) * torch.digamma(sum_params_p) + return t1 - t2 + t3 + t4 + t5 + + +@register_kl(Binomial, Binomial) +def _kl_binomial_binomial(p, q): + # from https://math.stackexchange.com/questions/2214993/ + # kullback-leibler-divergence-for-binomial-distributions-p-and-q + if (p.total_count < q.total_count).any(): + raise NotImplementedError( + "KL between Binomials where q.total_count > p.total_count is not implemented" + ) + kl = p.total_count * ( + p.probs * (p.logits - q.logits) + (-p.probs).log1p() - (-q.probs).log1p() + ) + inf_idxs = p.total_count > q.total_count + kl[inf_idxs] = _infinite_like(kl[inf_idxs]) + return kl + + +@register_kl(Categorical, Categorical) +def _kl_categorical_categorical(p, q): + t = p.probs * (p.logits - q.logits) + t[(q.probs == 0).expand_as(t)] = inf + t[(p.probs == 0).expand_as(t)] = 0 + return t.sum(-1) + + +@register_kl(ContinuousBernoulli, ContinuousBernoulli) +def _kl_continuous_bernoulli_continuous_bernoulli(p, q): + t1 = p.mean * (p.logits - q.logits) + t2 = p._cont_bern_log_norm() + torch.log1p(-p.probs) + t3 = -q._cont_bern_log_norm() - torch.log1p(-q.probs) + return t1 + t2 + t3 + + +@register_kl(Dirichlet, Dirichlet) +def _kl_dirichlet_dirichlet(p, q): + # From http://bariskurt.com/kullback-leibler-divergence-between-two-dirichlet-and-beta-distributions/ + sum_p_concentration = p.concentration.sum(-1) + sum_q_concentration = q.concentration.sum(-1) + t1 = sum_p_concentration.lgamma() - sum_q_concentration.lgamma() + t2 = (p.concentration.lgamma() - q.concentration.lgamma()).sum(-1) + t3 = p.concentration - q.concentration + t4 = p.concentration.digamma() - sum_p_concentration.digamma().unsqueeze(-1) + return t1 - t2 + (t3 * t4).sum(-1) + + +@register_kl(Exponential, Exponential) +def _kl_exponential_exponential(p, q): + rate_ratio = q.rate / p.rate + t1 = -rate_ratio.log() + return t1 + rate_ratio - 1 + + +@register_kl(ExponentialFamily, ExponentialFamily) +def _kl_expfamily_expfamily(p, q): + if type(p) is not type(q): + raise NotImplementedError( + "The cross KL-divergence between different exponential families cannot \ + be computed using Bregman divergences" + ) + p_nparams = [np.detach().requires_grad_() for np in p._natural_params] + q_nparams = q._natural_params + lg_normal = p._log_normalizer(*p_nparams) + gradients = torch.autograd.grad(lg_normal.sum(), p_nparams, create_graph=True) + result = q._log_normalizer(*q_nparams) - lg_normal + for pnp, qnp, g in zip(p_nparams, q_nparams, gradients): + term = (qnp - pnp) * g + result -= _sum_rightmost(term, len(q.event_shape)) + return result + + +@register_kl(Gamma, Gamma) +def _kl_gamma_gamma(p, q): + t1 = q.concentration * (p.rate / q.rate).log() + t2 = torch.lgamma(q.concentration) - torch.lgamma(p.concentration) + t3 = (p.concentration - q.concentration) * torch.digamma(p.concentration) + t4 = (q.rate - p.rate) * (p.concentration / p.rate) + return t1 + t2 + t3 + t4 + + +@register_kl(Gumbel, Gumbel) +def _kl_gumbel_gumbel(p, q): + ct1 = p.scale / q.scale + ct2 = q.loc / q.scale + ct3 = p.loc / q.scale + t1 = -ct1.log() - ct2 + ct3 + t2 = ct1 * _euler_gamma + t3 = torch.exp(ct2 + (1 + ct1).lgamma() - ct3) + return t1 + t2 + t3 - (1 + _euler_gamma) + + +@register_kl(Geometric, Geometric) +def _kl_geometric_geometric(p, q): + return -p.entropy() - torch.log1p(-q.probs) / p.probs - q.logits + + +@register_kl(HalfNormal, HalfNormal) +def _kl_halfnormal_halfnormal(p, q): + return _kl_normal_normal(p.base_dist, q.base_dist) + + +@register_kl(Laplace, Laplace) +def _kl_laplace_laplace(p, q): + # From http://www.mast.queensu.ca/~communications/Papers/gil-msc11.pdf + scale_ratio = p.scale / q.scale + loc_abs_diff = (p.loc - q.loc).abs() + t1 = -scale_ratio.log() + t2 = loc_abs_diff / q.scale + t3 = scale_ratio * torch.exp(-loc_abs_diff / p.scale) + return t1 + t2 + t3 - 1 + + +@register_kl(LowRankMultivariateNormal, LowRankMultivariateNormal) +def _kl_lowrankmultivariatenormal_lowrankmultivariatenormal(p, q): + if p.event_shape != q.event_shape: + raise ValueError( + "KL-divergence between two Low Rank Multivariate Normals with\ + different event shapes cannot be computed" + ) + + term1 = _batch_lowrank_logdet( + q._unbroadcasted_cov_factor, q._unbroadcasted_cov_diag, q._capacitance_tril + ) - _batch_lowrank_logdet( + p._unbroadcasted_cov_factor, p._unbroadcasted_cov_diag, p._capacitance_tril + ) + term3 = _batch_lowrank_mahalanobis( + q._unbroadcasted_cov_factor, + q._unbroadcasted_cov_diag, + q.loc - p.loc, + q._capacitance_tril, + ) + # Expands term2 according to + # inv(qcov) @ pcov = [inv(qD) - inv(qD) @ qW @ inv(qC) @ qW.T @ inv(qD)] @ (pW @ pW.T + pD) + # = [inv(qD) - A.T @ A] @ (pD + pW @ pW.T) + qWt_qDinv = q._unbroadcasted_cov_factor.mT / q._unbroadcasted_cov_diag.unsqueeze(-2) + A = torch.linalg.solve_triangular(q._capacitance_tril, qWt_qDinv, upper=False) + term21 = (p._unbroadcasted_cov_diag / q._unbroadcasted_cov_diag).sum(-1) + term22 = _batch_trace_XXT( + p._unbroadcasted_cov_factor * q._unbroadcasted_cov_diag.rsqrt().unsqueeze(-1) + ) + term23 = _batch_trace_XXT(A * p._unbroadcasted_cov_diag.sqrt().unsqueeze(-2)) + term24 = _batch_trace_XXT(A.matmul(p._unbroadcasted_cov_factor)) + term2 = term21 + term22 - term23 - term24 + return 0.5 * (term1 + term2 + term3 - p.event_shape[0]) + + +@register_kl(MultivariateNormal, LowRankMultivariateNormal) +def _kl_multivariatenormal_lowrankmultivariatenormal(p, q): + if p.event_shape != q.event_shape: + raise ValueError( + "KL-divergence between two (Low Rank) Multivariate Normals with\ + different event shapes cannot be computed" + ) + + term1 = _batch_lowrank_logdet( + q._unbroadcasted_cov_factor, q._unbroadcasted_cov_diag, q._capacitance_tril + ) - 2 * p._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1).log().sum(-1) + term3 = _batch_lowrank_mahalanobis( + q._unbroadcasted_cov_factor, + q._unbroadcasted_cov_diag, + q.loc - p.loc, + q._capacitance_tril, + ) + # Expands term2 according to + # inv(qcov) @ pcov = [inv(qD) - inv(qD) @ qW @ inv(qC) @ qW.T @ inv(qD)] @ p_tril @ p_tril.T + # = [inv(qD) - A.T @ A] @ p_tril @ p_tril.T + qWt_qDinv = q._unbroadcasted_cov_factor.mT / q._unbroadcasted_cov_diag.unsqueeze(-2) + A = torch.linalg.solve_triangular(q._capacitance_tril, qWt_qDinv, upper=False) + term21 = _batch_trace_XXT( + p._unbroadcasted_scale_tril * q._unbroadcasted_cov_diag.rsqrt().unsqueeze(-1) + ) + term22 = _batch_trace_XXT(A.matmul(p._unbroadcasted_scale_tril)) + term2 = term21 - term22 + return 0.5 * (term1 + term2 + term3 - p.event_shape[0]) + + +@register_kl(LowRankMultivariateNormal, MultivariateNormal) +def _kl_lowrankmultivariatenormal_multivariatenormal(p, q): + if p.event_shape != q.event_shape: + raise ValueError( + "KL-divergence between two (Low Rank) Multivariate Normals with\ + different event shapes cannot be computed" + ) + + term1 = 2 * q._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1).log().sum( + -1 + ) - _batch_lowrank_logdet( + p._unbroadcasted_cov_factor, p._unbroadcasted_cov_diag, p._capacitance_tril + ) + term3 = _batch_mahalanobis(q._unbroadcasted_scale_tril, (q.loc - p.loc)) + # Expands term2 according to + # inv(qcov) @ pcov = inv(q_tril @ q_tril.T) @ (pW @ pW.T + pD) + combined_batch_shape = torch._C._infer_size( + q._unbroadcasted_scale_tril.shape[:-2], p._unbroadcasted_cov_factor.shape[:-2] + ) + n = p.event_shape[0] + q_scale_tril = q._unbroadcasted_scale_tril.expand(combined_batch_shape + (n, n)) + p_cov_factor = p._unbroadcasted_cov_factor.expand( + combined_batch_shape + (n, p.cov_factor.size(-1)) + ) + p_cov_diag = torch.diag_embed(p._unbroadcasted_cov_diag.sqrt()).expand( + combined_batch_shape + (n, n) + ) + term21 = _batch_trace_XXT( + torch.linalg.solve_triangular(q_scale_tril, p_cov_factor, upper=False) + ) + term22 = _batch_trace_XXT( + torch.linalg.solve_triangular(q_scale_tril, p_cov_diag, upper=False) + ) + term2 = term21 + term22 + return 0.5 * (term1 + term2 + term3 - p.event_shape[0]) + + +@register_kl(MultivariateNormal, MultivariateNormal) +def _kl_multivariatenormal_multivariatenormal(p, q): + # From https://en.wikipedia.org/wiki/Multivariate_normal_distribution#Kullback%E2%80%93Leibler_divergence + if p.event_shape != q.event_shape: + raise ValueError( + "KL-divergence between two Multivariate Normals with\ + different event shapes cannot be computed" + ) + + half_term1 = q._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1).log().sum( + -1 + ) - p._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1).log().sum(-1) + combined_batch_shape = torch._C._infer_size( + q._unbroadcasted_scale_tril.shape[:-2], p._unbroadcasted_scale_tril.shape[:-2] + ) + n = p.event_shape[0] + q_scale_tril = q._unbroadcasted_scale_tril.expand(combined_batch_shape + (n, n)) + p_scale_tril = p._unbroadcasted_scale_tril.expand(combined_batch_shape + (n, n)) + term2 = _batch_trace_XXT( + torch.linalg.solve_triangular(q_scale_tril, p_scale_tril, upper=False) + ) + term3 = _batch_mahalanobis(q._unbroadcasted_scale_tril, (q.loc - p.loc)) + return half_term1 + 0.5 * (term2 + term3 - n) + + +@register_kl(Normal, Normal) +def _kl_normal_normal(p, q): + var_ratio = (p.scale / q.scale).pow(2) + t1 = ((p.loc - q.loc) / q.scale).pow(2) + return 0.5 * (var_ratio + t1 - 1 - var_ratio.log()) + + +@register_kl(OneHotCategorical, OneHotCategorical) +def _kl_onehotcategorical_onehotcategorical(p, q): + return _kl_categorical_categorical(p._categorical, q._categorical) + + +@register_kl(Pareto, Pareto) +def _kl_pareto_pareto(p, q): + # From http://www.mast.queensu.ca/~communications/Papers/gil-msc11.pdf + scale_ratio = p.scale / q.scale + alpha_ratio = q.alpha / p.alpha + t1 = q.alpha * scale_ratio.log() + t2 = -alpha_ratio.log() + result = t1 + t2 + alpha_ratio - 1 + result[p.support.lower_bound < q.support.lower_bound] = inf + return result + + +@register_kl(Poisson, Poisson) +def _kl_poisson_poisson(p, q): + return p.rate * (p.rate.log() - q.rate.log()) - (p.rate - q.rate) + + +@register_kl(TransformedDistribution, TransformedDistribution) +def _kl_transformed_transformed(p, q): + if p.transforms != q.transforms: + raise NotImplementedError + if p.event_shape != q.event_shape: + raise NotImplementedError + return kl_divergence(p.base_dist, q.base_dist) + + +@register_kl(Uniform, Uniform) +def _kl_uniform_uniform(p, q): + result = ((q.high - q.low) / (p.high - p.low)).log() + result[(q.low > p.low) | (q.high < p.high)] = inf + return result + + +# Different distributions +@register_kl(Bernoulli, Poisson) +def _kl_bernoulli_poisson(p, q): + return -p.entropy() - (p.probs * q.rate.log() - q.rate) + + +@register_kl(Beta, ContinuousBernoulli) +def _kl_beta_continuous_bernoulli(p, q): + return ( + -p.entropy() + - p.mean * q.logits + - torch.log1p(-q.probs) + - q._cont_bern_log_norm() + ) + + +@register_kl(Beta, Pareto) +def _kl_beta_infinity(p, q): + return _infinite_like(p.concentration1) + + +@register_kl(Beta, Exponential) +def _kl_beta_exponential(p, q): + return ( + -p.entropy() + - q.rate.log() + + q.rate * (p.concentration1 / (p.concentration1 + p.concentration0)) + ) + + +@register_kl(Beta, Gamma) +def _kl_beta_gamma(p, q): + t1 = -p.entropy() + t2 = q.concentration.lgamma() - q.concentration * q.rate.log() + t3 = (q.concentration - 1) * ( + p.concentration1.digamma() - (p.concentration1 + p.concentration0).digamma() + ) + t4 = q.rate * p.concentration1 / (p.concentration1 + p.concentration0) + return t1 + t2 - t3 + t4 + + +# TODO: Add Beta-Laplace KL Divergence + + +@register_kl(Beta, Normal) +def _kl_beta_normal(p, q): + E_beta = p.concentration1 / (p.concentration1 + p.concentration0) + var_normal = q.scale.pow(2) + t1 = -p.entropy() + t2 = 0.5 * (var_normal * 2 * math.pi).log() + t3 = ( + E_beta * (1 - E_beta) / (p.concentration1 + p.concentration0 + 1) + + E_beta.pow(2) + ) * 0.5 + t4 = q.loc * E_beta + t5 = q.loc.pow(2) * 0.5 + return t1 + t2 + (t3 - t4 + t5) / var_normal + + +@register_kl(Beta, Uniform) +def _kl_beta_uniform(p, q): + result = -p.entropy() + (q.high - q.low).log() + result[(q.low > p.support.lower_bound) | (q.high < p.support.upper_bound)] = inf + return result + + +# Note that the KL between a ContinuousBernoulli and Beta has no closed form + + +@register_kl(ContinuousBernoulli, Pareto) +def _kl_continuous_bernoulli_infinity(p, q): + return _infinite_like(p.probs) + + +@register_kl(ContinuousBernoulli, Exponential) +def _kl_continuous_bernoulli_exponential(p, q): + return -p.entropy() - torch.log(q.rate) + q.rate * p.mean + + +# Note that the KL between a ContinuousBernoulli and Gamma has no closed form +# TODO: Add ContinuousBernoulli-Laplace KL Divergence + + +@register_kl(ContinuousBernoulli, Normal) +def _kl_continuous_bernoulli_normal(p, q): + t1 = -p.entropy() + t2 = 0.5 * (math.log(2.0 * math.pi) + torch.square(q.loc / q.scale)) + torch.log( + q.scale + ) + t3 = (p.variance + torch.square(p.mean) - 2.0 * q.loc * p.mean) / ( + 2.0 * torch.square(q.scale) + ) + return t1 + t2 + t3 + + +@register_kl(ContinuousBernoulli, Uniform) +def _kl_continuous_bernoulli_uniform(p, q): + result = -p.entropy() + (q.high - q.low).log() + return torch.where( + torch.max( + torch.ge(q.low, p.support.lower_bound), + torch.le(q.high, p.support.upper_bound), + ), + torch.ones_like(result) * inf, + result, + ) + + +@register_kl(Exponential, Beta) +@register_kl(Exponential, ContinuousBernoulli) +@register_kl(Exponential, Pareto) +@register_kl(Exponential, Uniform) +def _kl_exponential_infinity(p, q): + return _infinite_like(p.rate) + + +@register_kl(Exponential, Gamma) +def _kl_exponential_gamma(p, q): + ratio = q.rate / p.rate + t1 = -q.concentration * torch.log(ratio) + return ( + t1 + + ratio + + q.concentration.lgamma() + + q.concentration * _euler_gamma + - (1 + _euler_gamma) + ) + + +@register_kl(Exponential, Gumbel) +def _kl_exponential_gumbel(p, q): + scale_rate_prod = p.rate * q.scale + loc_scale_ratio = q.loc / q.scale + t1 = scale_rate_prod.log() - 1 + t2 = torch.exp(loc_scale_ratio) * scale_rate_prod / (scale_rate_prod + 1) + t3 = scale_rate_prod.reciprocal() + return t1 - loc_scale_ratio + t2 + t3 + + +# TODO: Add Exponential-Laplace KL Divergence + + +@register_kl(Exponential, Normal) +def _kl_exponential_normal(p, q): + var_normal = q.scale.pow(2) + rate_sqr = p.rate.pow(2) + t1 = 0.5 * torch.log(rate_sqr * var_normal * 2 * math.pi) + t2 = rate_sqr.reciprocal() + t3 = q.loc / p.rate + t4 = q.loc.pow(2) * 0.5 + return t1 - 1 + (t2 - t3 + t4) / var_normal + + +@register_kl(Gamma, Beta) +@register_kl(Gamma, ContinuousBernoulli) +@register_kl(Gamma, Pareto) +@register_kl(Gamma, Uniform) +def _kl_gamma_infinity(p, q): + return _infinite_like(p.concentration) + + +@register_kl(Gamma, Exponential) +def _kl_gamma_exponential(p, q): + return -p.entropy() - q.rate.log() + q.rate * p.concentration / p.rate + + +@register_kl(Gamma, Gumbel) +def _kl_gamma_gumbel(p, q): + beta_scale_prod = p.rate * q.scale + loc_scale_ratio = q.loc / q.scale + t1 = ( + (p.concentration - 1) * p.concentration.digamma() + - p.concentration.lgamma() + - p.concentration + ) + t2 = beta_scale_prod.log() + p.concentration / beta_scale_prod + t3 = ( + torch.exp(loc_scale_ratio) + * (1 + beta_scale_prod.reciprocal()).pow(-p.concentration) + - loc_scale_ratio + ) + return t1 + t2 + t3 + + +# TODO: Add Gamma-Laplace KL Divergence + + +@register_kl(Gamma, Normal) +def _kl_gamma_normal(p, q): + var_normal = q.scale.pow(2) + beta_sqr = p.rate.pow(2) + t1 = ( + 0.5 * torch.log(beta_sqr * var_normal * 2 * math.pi) + - p.concentration + - p.concentration.lgamma() + ) + t2 = 0.5 * (p.concentration.pow(2) + p.concentration) / beta_sqr + t3 = q.loc * p.concentration / p.rate + t4 = 0.5 * q.loc.pow(2) + return ( + t1 + + (p.concentration - 1) * p.concentration.digamma() + + (t2 - t3 + t4) / var_normal + ) + + +@register_kl(Gumbel, Beta) +@register_kl(Gumbel, ContinuousBernoulli) +@register_kl(Gumbel, Exponential) +@register_kl(Gumbel, Gamma) +@register_kl(Gumbel, Pareto) +@register_kl(Gumbel, Uniform) +def _kl_gumbel_infinity(p, q): + return _infinite_like(p.loc) + + +# TODO: Add Gumbel-Laplace KL Divergence + + +@register_kl(Gumbel, Normal) +def _kl_gumbel_normal(p, q): + param_ratio = p.scale / q.scale + t1 = (param_ratio / math.sqrt(2 * math.pi)).log() + t2 = (math.pi * param_ratio * 0.5).pow(2) / 3 + t3 = ((p.loc + p.scale * _euler_gamma - q.loc) / q.scale).pow(2) * 0.5 + return -t1 + t2 + t3 - (_euler_gamma + 1) + + +@register_kl(Laplace, Beta) +@register_kl(Laplace, ContinuousBernoulli) +@register_kl(Laplace, Exponential) +@register_kl(Laplace, Gamma) +@register_kl(Laplace, Pareto) +@register_kl(Laplace, Uniform) +def _kl_laplace_infinity(p, q): + return _infinite_like(p.loc) + + +@register_kl(Laplace, Normal) +def _kl_laplace_normal(p, q): + var_normal = q.scale.pow(2) + scale_sqr_var_ratio = p.scale.pow(2) / var_normal + t1 = 0.5 * torch.log(2 * scale_sqr_var_ratio / math.pi) + t2 = 0.5 * p.loc.pow(2) + t3 = p.loc * q.loc + t4 = 0.5 * q.loc.pow(2) + return -t1 + scale_sqr_var_ratio + (t2 - t3 + t4) / var_normal - 1 + + +@register_kl(Normal, Beta) +@register_kl(Normal, ContinuousBernoulli) +@register_kl(Normal, Exponential) +@register_kl(Normal, Gamma) +@register_kl(Normal, Pareto) +@register_kl(Normal, Uniform) +def _kl_normal_infinity(p, q): + return _infinite_like(p.loc) + + +@register_kl(Normal, Gumbel) +def _kl_normal_gumbel(p, q): + mean_scale_ratio = p.loc / q.scale + var_scale_sqr_ratio = (p.scale / q.scale).pow(2) + loc_scale_ratio = q.loc / q.scale + t1 = var_scale_sqr_ratio.log() * 0.5 + t2 = mean_scale_ratio - loc_scale_ratio + t3 = torch.exp(-mean_scale_ratio + 0.5 * var_scale_sqr_ratio + loc_scale_ratio) + return -t1 + t2 + t3 - (0.5 * (1 + math.log(2 * math.pi))) + + +@register_kl(Normal, Laplace) +def _kl_normal_laplace(p, q): + loc_diff = p.loc - q.loc + scale_ratio = p.scale / q.scale + loc_diff_scale_ratio = loc_diff / p.scale + t1 = torch.log(scale_ratio) + t2 = ( + math.sqrt(2 / math.pi) * p.scale * torch.exp(-0.5 * loc_diff_scale_ratio.pow(2)) + ) + t3 = loc_diff * torch.erf(math.sqrt(0.5) * loc_diff_scale_ratio) + return -t1 + (t2 + t3) / q.scale - (0.5 * (1 + math.log(0.5 * math.pi))) + + +@register_kl(Pareto, Beta) +@register_kl(Pareto, ContinuousBernoulli) +@register_kl(Pareto, Uniform) +def _kl_pareto_infinity(p, q): + return _infinite_like(p.scale) + + +@register_kl(Pareto, Exponential) +def _kl_pareto_exponential(p, q): + scale_rate_prod = p.scale * q.rate + t1 = (p.alpha / scale_rate_prod).log() + t2 = p.alpha.reciprocal() + t3 = p.alpha * scale_rate_prod / (p.alpha - 1) + result = t1 - t2 + t3 - 1 + result[p.alpha <= 1] = inf + return result + + +@register_kl(Pareto, Gamma) +def _kl_pareto_gamma(p, q): + common_term = p.scale.log() + p.alpha.reciprocal() + t1 = p.alpha.log() - common_term + t2 = q.concentration.lgamma() - q.concentration * q.rate.log() + t3 = (1 - q.concentration) * common_term + t4 = q.rate * p.alpha * p.scale / (p.alpha - 1) + result = t1 + t2 + t3 + t4 - 1 + result[p.alpha <= 1] = inf + return result + + +# TODO: Add Pareto-Laplace KL Divergence + + +@register_kl(Pareto, Normal) +def _kl_pareto_normal(p, q): + var_normal = 2 * q.scale.pow(2) + common_term = p.scale / (p.alpha - 1) + t1 = (math.sqrt(2 * math.pi) * q.scale * p.alpha / p.scale).log() + t2 = p.alpha.reciprocal() + t3 = p.alpha * common_term.pow(2) / (p.alpha - 2) + t4 = (p.alpha * common_term - q.loc).pow(2) + result = t1 - t2 + (t3 + t4) / var_normal - 1 + result[p.alpha <= 2] = inf + return result + + +@register_kl(Poisson, Bernoulli) +@register_kl(Poisson, Binomial) +def _kl_poisson_infinity(p, q): + return _infinite_like(p.rate) + + +@register_kl(Uniform, Beta) +def _kl_uniform_beta(p, q): + common_term = p.high - p.low + t1 = torch.log(common_term) + t2 = ( + (q.concentration1 - 1) + * (_x_log_x(p.high) - _x_log_x(p.low) - common_term) + / common_term + ) + t3 = ( + (q.concentration0 - 1) + * (_x_log_x(1 - p.high) - _x_log_x(1 - p.low) + common_term) + / common_term + ) + t4 = ( + q.concentration1.lgamma() + + q.concentration0.lgamma() + - (q.concentration1 + q.concentration0).lgamma() + ) + result = t3 + t4 - t1 - t2 + result[(p.high > q.support.upper_bound) | (p.low < q.support.lower_bound)] = inf + return result + + +@register_kl(Uniform, ContinuousBernoulli) +def _kl_uniform_continuous_bernoulli(p, q): + result = ( + -p.entropy() + - p.mean * q.logits + - torch.log1p(-q.probs) + - q._cont_bern_log_norm() + ) + return torch.where( + torch.max( + torch.ge(p.high, q.support.upper_bound), + torch.le(p.low, q.support.lower_bound), + ), + torch.ones_like(result) * inf, + result, + ) + + +@register_kl(Uniform, Exponential) +def _kl_uniform_exponetial(p, q): + result = q.rate * (p.high + p.low) / 2 - ((p.high - p.low) * q.rate).log() + result[p.low < q.support.lower_bound] = inf + return result + + +@register_kl(Uniform, Gamma) +def _kl_uniform_gamma(p, q): + common_term = p.high - p.low + t1 = common_term.log() + t2 = q.concentration.lgamma() - q.concentration * q.rate.log() + t3 = ( + (1 - q.concentration) + * (_x_log_x(p.high) - _x_log_x(p.low) - common_term) + / common_term + ) + t4 = q.rate * (p.high + p.low) / 2 + result = -t1 + t2 + t3 + t4 + result[p.low < q.support.lower_bound] = inf + return result + + +@register_kl(Uniform, Gumbel) +def _kl_uniform_gumbel(p, q): + common_term = q.scale / (p.high - p.low) + high_loc_diff = (p.high - q.loc) / q.scale + low_loc_diff = (p.low - q.loc) / q.scale + t1 = common_term.log() + 0.5 * (high_loc_diff + low_loc_diff) + t2 = common_term * (torch.exp(-high_loc_diff) - torch.exp(-low_loc_diff)) + return t1 - t2 + + +# TODO: Uniform-Laplace KL Divergence + + +@register_kl(Uniform, Normal) +def _kl_uniform_normal(p, q): + common_term = p.high - p.low + t1 = (math.sqrt(math.pi * 2) * q.scale / common_term).log() + t2 = (common_term).pow(2) / 12 + t3 = ((p.high + p.low - 2 * q.loc) / 2).pow(2) + return t1 + 0.5 * (t2 + t3) / q.scale.pow(2) + + +@register_kl(Uniform, Pareto) +def _kl_uniform_pareto(p, q): + support_uniform = p.high - p.low + t1 = (q.alpha * q.scale.pow(q.alpha) * (support_uniform)).log() + t2 = (_x_log_x(p.high) - _x_log_x(p.low) - support_uniform) / support_uniform + result = t2 * (q.alpha + 1) - t1 + result[p.low < q.support.lower_bound] = inf + return result + + +@register_kl(Independent, Independent) +def _kl_independent_independent(p, q): + if p.reinterpreted_batch_ndims != q.reinterpreted_batch_ndims: + raise NotImplementedError + result = kl_divergence(p.base_dist, q.base_dist) + return _sum_rightmost(result, p.reinterpreted_batch_ndims) + + +@register_kl(Cauchy, Cauchy) +def _kl_cauchy_cauchy(p, q): + # From https://arxiv.org/abs/1905.10965 + t1 = ((p.scale + q.scale).pow(2) + (p.loc - q.loc).pow(2)).log() + t2 = (4 * p.scale * q.scale).log() + return t1 - t2 + + +def _add_kl_info(): + """Appends a list of implemented KL functions to the doc for kl_divergence.""" + rows = [ + "KL divergence is currently implemented for the following distribution pairs:" + ] + for p, q in sorted( + _KL_REGISTRY, key=lambda p_q: (p_q[0].__name__, p_q[1].__name__) + ): + rows.append( + f"* :class:`~torch.distributions.{p.__name__}` and :class:`~torch.distributions.{q.__name__}`" + ) + kl_info = "\n\t".join(rows) + if kl_divergence.__doc__: + kl_divergence.__doc__ += kl_info diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/kumaraswamy.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/kumaraswamy.py new file mode 100644 index 0000000000000000000000000000000000000000..2c70b66e07a889a496f7bfd5f2946e49405ee661 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/kumaraswamy.py @@ -0,0 +1,108 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import nan, Tensor +from torch.distributions import constraints +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import AffineTransform, PowerTransform +from torch.distributions.uniform import Uniform +from torch.distributions.utils import broadcast_all, euler_constant + + +__all__ = ["Kumaraswamy"] + + +def _moments(a, b, n): + """ + Computes nth moment of Kumaraswamy using using torch.lgamma + """ + arg1 = 1 + n / a + log_value = torch.lgamma(arg1) + torch.lgamma(b) - torch.lgamma(arg1 + b) + return b * torch.exp(log_value) + + +class Kumaraswamy(TransformedDistribution): + r""" + Samples from a Kumaraswamy distribution. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Kumaraswamy(torch.tensor([1.0]), torch.tensor([1.0])) + >>> m.sample() # sample from a Kumaraswamy distribution with concentration alpha=1 and beta=1 + tensor([ 0.1729]) + + Args: + concentration1 (float or Tensor): 1st concentration parameter of the distribution + (often referred to as alpha) + concentration0 (float or Tensor): 2nd concentration parameter of the distribution + (often referred to as beta) + """ + + arg_constraints = { + "concentration1": constraints.positive, + "concentration0": constraints.positive, + } + # pyrefly: ignore [bad-override] + support = constraints.unit_interval + has_rsample = True + + def __init__( + self, + concentration1: Union[Tensor, float], + concentration0: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.concentration1, self.concentration0 = broadcast_all( + concentration1, concentration0 + ) + base_dist = Uniform( + torch.full_like(self.concentration0, 0), + torch.full_like(self.concentration0, 1), + validate_args=validate_args, + ) + transforms = [ + PowerTransform(exponent=self.concentration0.reciprocal()), + AffineTransform(loc=1.0, scale=-1.0), + PowerTransform(exponent=self.concentration1.reciprocal()), + ] + # pyrefly: ignore [bad-argument-type] + super().__init__(base_dist, transforms, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Kumaraswamy, _instance) + new.concentration1 = self.concentration1.expand(batch_shape) + new.concentration0 = self.concentration0.expand(batch_shape) + return super().expand(batch_shape, _instance=new) + + @property + def mean(self) -> Tensor: + return _moments(self.concentration1, self.concentration0, 1) + + @property + def mode(self) -> Tensor: + # Evaluate in log-space for numerical stability. + log_mode = ( + self.concentration0.reciprocal() * (-self.concentration0).log1p() + - (-self.concentration0 * self.concentration1).log1p() + ) + log_mode[(self.concentration0 < 1) | (self.concentration1 < 1)] = nan + return log_mode.exp() + + @property + def variance(self) -> Tensor: + return _moments(self.concentration1, self.concentration0, 2) - torch.pow( + self.mean, 2 + ) + + def entropy(self): + t1 = 1 - self.concentration1.reciprocal() + t0 = 1 - self.concentration0.reciprocal() + H0 = torch.digamma(self.concentration0 + 1) + euler_constant + return ( + t0 + + t1 * H0 + - torch.log(self.concentration1) + - torch.log(self.concentration0) + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/laplace.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/laplace.py new file mode 100644 index 0000000000000000000000000000000000000000..5d0573bda6f73a90d712677dd48a32692be6774e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/laplace.py @@ -0,0 +1,105 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Laplace"] + + +class Laplace(Distribution): + r""" + Creates a Laplace distribution parameterized by :attr:`loc` and :attr:`scale`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Laplace(torch.tensor([0.0]), torch.tensor([1.0])) + >>> m.sample() # Laplace distributed with loc=0, scale=1 + tensor([ 0.1046]) + + Args: + loc (float or Tensor): mean of the distribution + scale (float or Tensor): scale of the distribution + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"loc": constraints.real, "scale": constraints.positive} + support = constraints.real + has_rsample = True + + @property + def mean(self) -> Tensor: + return self.loc + + @property + def mode(self) -> Tensor: + return self.loc + + @property + def variance(self) -> Tensor: + return 2 * self.scale.pow(2) + + @property + def stddev(self) -> Tensor: + return (2**0.5) * self.scale + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.loc, self.scale = broadcast_all(loc, scale) + if isinstance(loc, _Number) and isinstance(scale, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.loc.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Laplace, _instance) + batch_shape = torch.Size(batch_shape) + new.loc = self.loc.expand(batch_shape) + new.scale = self.scale.expand(batch_shape) + super(Laplace, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + finfo = torch.finfo(self.loc.dtype) + if torch._C._get_tracing_state(): + # [JIT WORKAROUND] lack of support for .uniform_() + u = torch.rand(shape, dtype=self.loc.dtype, device=self.loc.device) * 2 - 1 + return self.loc - self.scale * u.sign() * torch.log1p( + -u.abs().clamp(min=finfo.tiny) + ) + u = self.loc.new(shape).uniform_(finfo.eps - 1, 1) + # TODO: If we ever implement tensor.nextafter, below is what we want ideally. + # u = self.loc.new(shape).uniform_(self.loc.nextafter(-.5, 0), .5) + return self.loc - self.scale * u.sign() * torch.log1p(-u.abs()) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + return -torch.log(2 * self.scale) - torch.abs(value - self.loc) / self.scale + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return 0.5 - 0.5 * (value - self.loc).sign() * torch.expm1( + -(value - self.loc).abs() / self.scale + ) + + def icdf(self, value): + term = value - 0.5 + return self.loc - self.scale * (term).sign() * torch.log1p(-2 * term.abs()) + + def entropy(self): + return 1 + torch.log(2 * self.scale) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/lkj_cholesky.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/lkj_cholesky.py new file mode 100644 index 0000000000000000000000000000000000000000..631abdd7dd5aa1741930f504b7c0d5c8cfb0b1d1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/lkj_cholesky.py @@ -0,0 +1,153 @@ +# mypy: allow-untyped-defs +""" +This closely follows the implementation in NumPyro (https://github.com/pyro-ppl/numpyro). + +Original copyright notice: + +# Copyright: Contributors to the Pyro project. +# SPDX-License-Identifier: Apache-2.0 +""" + +import math +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import Beta, constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import broadcast_all + + +__all__ = ["LKJCholesky"] + + +class LKJCholesky(Distribution): + r""" + LKJ distribution for lower Cholesky factor of correlation matrices. + The distribution is controlled by ``concentration`` parameter :math:`\eta` + to make the probability of the correlation matrix :math:`M` generated from + a Cholesky factor proportional to :math:`\det(M)^{\eta - 1}`. Because of that, + when ``concentration == 1``, we have a uniform distribution over Cholesky + factors of correlation matrices:: + + L ~ LKJCholesky(dim, concentration) + X = L @ L' ~ LKJCorr(dim, concentration) + + Note that this distribution samples the + Cholesky factor of correlation matrices and not the correlation matrices + themselves and thereby differs slightly from the derivations in [1] for + the `LKJCorr` distribution. For sampling, this uses the Onion method from + [1] Section 3. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> l = LKJCholesky(3, 0.5) + >>> l.sample() # l @ l.T is a sample of a correlation 3x3 matrix + tensor([[ 1.0000, 0.0000, 0.0000], + [ 0.3516, 0.9361, 0.0000], + [-0.1899, 0.4748, 0.8593]]) + + Args: + dimension (dim): dimension of the matrices + concentration (float or Tensor): concentration/shape parameter of the + distribution (often referred to as eta) + + **References** + + [1] `Generating random correlation matrices based on vines and extended onion method` (2009), + Daniel Lewandowski, Dorota Kurowicka, Harry Joe. + Journal of Multivariate Analysis. 100. 10.1016/j.jmva.2009.04.008 + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"concentration": constraints.positive} + support = constraints.corr_cholesky + + def __init__( + self, + dim: int, + concentration: Union[Tensor, float] = 1.0, + validate_args: Optional[bool] = None, + ) -> None: + if dim < 2: + raise ValueError( + f"Expected dim to be an integer greater than or equal to 2. Found dim={dim}." + ) + self.dim = dim + (self.concentration,) = broadcast_all(concentration) + batch_shape = self.concentration.size() + event_shape = torch.Size((dim, dim)) + # This is used to draw vectorized samples from the beta distribution in Sec. 3.2 of [1]. + marginal_conc = self.concentration + 0.5 * (self.dim - 2) + offset = torch.arange( + self.dim - 1, + dtype=self.concentration.dtype, + device=self.concentration.device, + ) + offset = torch.cat([offset.new_zeros((1,)), offset]) + beta_conc1 = offset + 0.5 + beta_conc0 = marginal_conc.unsqueeze(-1) - 0.5 * offset + self._beta = Beta(beta_conc1, beta_conc0) + super().__init__(batch_shape, event_shape, validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(LKJCholesky, _instance) + batch_shape = torch.Size(batch_shape) + new.dim = self.dim + new.concentration = self.concentration.expand(batch_shape) + new._beta = self._beta.expand(batch_shape + (self.dim,)) + super(LKJCholesky, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + def sample(self, sample_shape=torch.Size()): + # This uses the Onion method, but there are a few differences from [1] Sec. 3.2: + # - This vectorizes the for loop and also works for heterogeneous eta. + # - Same algorithm generalizes to n=1. + # - The procedure is simplified since we are sampling the cholesky factor of + # the correlation matrix instead of the correlation matrix itself. As such, + # we only need to generate `w`. + y = self._beta.sample(sample_shape).unsqueeze(-1) + u_normal = torch.randn( + self._extended_shape(sample_shape), dtype=y.dtype, device=y.device + ).tril(-1) + u_hypersphere = u_normal / u_normal.norm(dim=-1, keepdim=True) + # Replace NaNs in first row + u_hypersphere[..., 0, :].fill_(0.0) + w = torch.sqrt(y) * u_hypersphere + # Fill diagonal elements; clamp for numerical stability + eps = torch.finfo(w.dtype).tiny + diag_elems = torch.clamp(1 - torch.sum(w**2, dim=-1), min=eps).sqrt() + w += torch.diag_embed(diag_elems) + return w + + def log_prob(self, value): + # See: https://mc-stan.org/docs/2_25/functions-reference/cholesky-lkj-correlation-distribution.html + # The probability of a correlation matrix is proportional to + # determinant ** (concentration - 1) = prod(L_ii ^ 2(concentration - 1)) + # Additionally, the Jacobian of the transformation from Cholesky factor to + # correlation matrix is: + # prod(L_ii ^ (D - i)) + # So the probability of a Cholesky factor is proportional to + # prod(L_ii ^ (2 * concentration - 2 + D - i)) = prod(L_ii ^ order_i) + # with order_i = 2 * concentration - 2 + D - i + if self._validate_args: + self._validate_sample(value) + diag_elems = value.diagonal(dim1=-1, dim2=-2)[..., 1:] + order = torch.arange(2, self.dim + 1, device=self.concentration.device) + order = 2 * (self.concentration - 1).unsqueeze(-1) + self.dim - order + unnormalized_log_pdf = torch.sum(order * diag_elems.log(), dim=-1) + # Compute normalization constant (page 1999 of [1]) + dm1 = self.dim - 1 + alpha = self.concentration + 0.5 * dm1 + denominator = torch.lgamma(alpha) * dm1 + numerator = torch.mvlgamma(alpha - 0.5, dm1) + # pi_constant in [1] is D * (D - 1) / 4 * log(pi) + # pi_constant in multigammaln is (D - 1) * (D - 2) / 4 * log(pi) + # hence, we need to add a pi_constant = (D - 1) * log(pi) / 2 + pi_constant = 0.5 * dm1 * math.log(math.pi) + normalize_term = pi_constant + numerator - denominator + return unnormalized_log_pdf - normalize_term diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/log_normal.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/log_normal.py new file mode 100644 index 0000000000000000000000000000000000000000..f6ad414ce8275591203251c9b3809a8b803f7b89 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/log_normal.py @@ -0,0 +1,76 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.normal import Normal +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import ExpTransform + + +__all__ = ["LogNormal"] + + +class LogNormal(TransformedDistribution): + r""" + Creates a log-normal distribution parameterized by + :attr:`loc` and :attr:`scale` where:: + + X ~ Normal(loc, scale) + Y = exp(X) ~ LogNormal(loc, scale) + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = LogNormal(torch.tensor([0.0]), torch.tensor([1.0])) + >>> m.sample() # log-normal distributed with mean=0 and stddev=1 + tensor([ 0.1046]) + + Args: + loc (float or Tensor): mean of log of distribution + scale (float or Tensor): standard deviation of log of the distribution + """ + + arg_constraints = {"loc": constraints.real, "scale": constraints.positive} + # pyrefly: ignore [bad-override] + support = constraints.positive + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: Normal + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + base_dist = Normal(loc, scale, validate_args=validate_args) + super().__init__(base_dist, ExpTransform(), validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(LogNormal, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def loc(self) -> Tensor: + return self.base_dist.loc + + @property + def scale(self) -> Tensor: + return self.base_dist.scale + + @property + def mean(self) -> Tensor: + return (self.loc + self.scale.pow(2) / 2).exp() + + @property + def mode(self) -> Tensor: + return (self.loc - self.scale.square()).exp() + + @property + def variance(self) -> Tensor: + scale_sq = self.scale.pow(2) + return scale_sq.expm1() * (2 * self.loc + scale_sq).exp() + + def entropy(self): + return self.base_dist.entropy() + self.loc diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/logistic_normal.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/logistic_normal.py new file mode 100644 index 0000000000000000000000000000000000000000..ef23b8c8a7afe4ae4306259de7989e0c03a11e68 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/logistic_normal.py @@ -0,0 +1,68 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +from torch import Tensor +from torch.distributions import constraints, Independent +from torch.distributions.normal import Normal +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import StickBreakingTransform + + +__all__ = ["LogisticNormal"] + + +class LogisticNormal(TransformedDistribution): + r""" + Creates a logistic-normal distribution parameterized by :attr:`loc` and :attr:`scale` + that define the base `Normal` distribution transformed with the + `StickBreakingTransform` such that:: + + X ~ LogisticNormal(loc, scale) + Y = log(X / (1 - X.cumsum(-1)))[..., :-1] ~ Normal(loc, scale) + + Args: + loc (float or Tensor): mean of the base distribution + scale (float or Tensor): standard deviation of the base distribution + + Example:: + + >>> # logistic-normal distributed with mean=(0, 0, 0) and stddev=(1, 1, 1) + >>> # of the base Normal distribution + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = LogisticNormal(torch.tensor([0.0] * 3), torch.tensor([1.0] * 3)) + >>> m.sample() + tensor([ 0.7653, 0.0341, 0.0579, 0.1427]) + + """ + + arg_constraints = {"loc": constraints.real, "scale": constraints.positive} + # pyrefly: ignore [bad-override] + support = constraints.simplex + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: Independent[Normal] + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + base_dist = Normal(loc, scale, validate_args=validate_args) + if not base_dist.batch_shape: + base_dist = base_dist.expand([1]) + super().__init__( + base_dist, StickBreakingTransform(), validate_args=validate_args + ) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(LogisticNormal, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def loc(self) -> Tensor: + return self.base_dist.base_dist.loc + + @property + def scale(self) -> Tensor: + return self.base_dist.base_dist.scale diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/lowrank_multivariate_normal.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/lowrank_multivariate_normal.py new file mode 100644 index 0000000000000000000000000000000000000000..682a01a1098b0ff2a9d2efa751054683650b877a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/lowrank_multivariate_normal.py @@ -0,0 +1,252 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.multivariate_normal import _batch_mahalanobis, _batch_mv +from torch.distributions.utils import _standard_normal, lazy_property +from torch.types import _size + + +__all__ = ["LowRankMultivariateNormal"] + + +def _batch_capacitance_tril(W, D): + r""" + Computes Cholesky of :math:`I + W.T @ inv(D) @ W` for a batch of matrices :math:`W` + and a batch of vectors :math:`D`. + """ + m = W.size(-1) + Wt_Dinv = W.mT / D.unsqueeze(-2) + K = torch.matmul(Wt_Dinv, W).contiguous() + K.view(-1, m * m)[:, :: m + 1] += 1 # add identity matrix to K + return torch.linalg.cholesky(K) + + +def _batch_lowrank_logdet(W, D, capacitance_tril): + r""" + Uses "matrix determinant lemma":: + log|W @ W.T + D| = log|C| + log|D|, + where :math:`C` is the capacitance matrix :math:`I + W.T @ inv(D) @ W`, to compute + the log determinant. + """ + return 2 * capacitance_tril.diagonal(dim1=-2, dim2=-1).log().sum(-1) + D.log().sum( + -1 + ) + + +def _batch_lowrank_mahalanobis(W, D, x, capacitance_tril): + r""" + Uses "Woodbury matrix identity":: + inv(W @ W.T + D) = inv(D) - inv(D) @ W @ inv(C) @ W.T @ inv(D), + where :math:`C` is the capacitance matrix :math:`I + W.T @ inv(D) @ W`, to compute the squared + Mahalanobis distance :math:`x.T @ inv(W @ W.T + D) @ x`. + """ + Wt_Dinv = W.mT / D.unsqueeze(-2) + Wt_Dinv_x = _batch_mv(Wt_Dinv, x) + mahalanobis_term1 = (x.pow(2) / D).sum(-1) + mahalanobis_term2 = _batch_mahalanobis(capacitance_tril, Wt_Dinv_x) + return mahalanobis_term1 - mahalanobis_term2 + + +class LowRankMultivariateNormal(Distribution): + r""" + Creates a multivariate normal distribution with covariance matrix having a low-rank form + parameterized by :attr:`cov_factor` and :attr:`cov_diag`:: + + covariance_matrix = cov_factor @ cov_factor.T + cov_diag + + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_LAPACK) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = LowRankMultivariateNormal( + ... torch.zeros(2), torch.tensor([[1.0], [0.0]]), torch.ones(2) + ... ) + >>> m.sample() # normally distributed with mean=`[0,0]`, cov_factor=`[[1],[0]]`, cov_diag=`[1,1]` + tensor([-0.2102, -0.5429]) + + Args: + loc (Tensor): mean of the distribution with shape `batch_shape + event_shape` + cov_factor (Tensor): factor part of low-rank form of covariance matrix with shape + `batch_shape + event_shape + (rank,)` + cov_diag (Tensor): diagonal part of low-rank form of covariance matrix with shape + `batch_shape + event_shape` + + Note: + The computation for determinant and inverse of covariance matrix is avoided when + `cov_factor.shape[1] << cov_factor.shape[0]` thanks to `Woodbury matrix identity + `_ and + `matrix determinant lemma `_. + Thanks to these formulas, we just need to compute the determinant and inverse of + the small size "capacitance" matrix:: + + capacitance = I + cov_factor.T @ inv(cov_diag) @ cov_factor + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "loc": constraints.real_vector, + "cov_factor": constraints.independent(constraints.real, 2), + "cov_diag": constraints.independent(constraints.positive, 1), + } + support = constraints.real_vector + has_rsample = True + + def __init__( + self, + loc: Tensor, + cov_factor: Tensor, + cov_diag: Tensor, + validate_args: Optional[bool] = None, + ) -> None: + if loc.dim() < 1: + raise ValueError("loc must be at least one-dimensional.") + event_shape = loc.shape[-1:] + if cov_factor.dim() < 2: + raise ValueError( + "cov_factor must be at least two-dimensional, " + "with optional leading batch dimensions" + ) + if cov_factor.shape[-2:-1] != event_shape: + raise ValueError( + f"cov_factor must be a batch of matrices with shape {event_shape[0]} x m" + ) + if cov_diag.shape[-1:] != event_shape: + raise ValueError( + f"cov_diag must be a batch of vectors with shape {event_shape}" + ) + + loc_ = loc.unsqueeze(-1) + cov_diag_ = cov_diag.unsqueeze(-1) + try: + loc_, self.cov_factor, cov_diag_ = torch.broadcast_tensors( + loc_, cov_factor, cov_diag_ + ) + except RuntimeError as e: + raise ValueError( + f"Incompatible batch shapes: loc {loc.shape}, cov_factor {cov_factor.shape}, cov_diag {cov_diag.shape}" + ) from e + self.loc = loc_[..., 0] + self.cov_diag = cov_diag_[..., 0] + batch_shape = self.loc.shape[:-1] + + self._unbroadcasted_cov_factor = cov_factor + self._unbroadcasted_cov_diag = cov_diag + self._capacitance_tril = _batch_capacitance_tril(cov_factor, cov_diag) + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(LowRankMultivariateNormal, _instance) + batch_shape = torch.Size(batch_shape) + loc_shape = batch_shape + self.event_shape + new.loc = self.loc.expand(loc_shape) + new.cov_diag = self.cov_diag.expand(loc_shape) + new.cov_factor = self.cov_factor.expand(loc_shape + self.cov_factor.shape[-1:]) + new._unbroadcasted_cov_factor = self._unbroadcasted_cov_factor + new._unbroadcasted_cov_diag = self._unbroadcasted_cov_diag + new._capacitance_tril = self._capacitance_tril + super(LowRankMultivariateNormal, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + @property + def mean(self) -> Tensor: + return self.loc + + @property + def mode(self) -> Tensor: + return self.loc + + @lazy_property + def variance(self) -> Tensor: # type: ignore[override] + return ( + self._unbroadcasted_cov_factor.pow(2).sum(-1) + self._unbroadcasted_cov_diag + ).expand(self._batch_shape + self._event_shape) + + @lazy_property + def scale_tril(self) -> Tensor: + # The following identity is used to increase the numerically computation stability + # for Cholesky decomposition (see http://www.gaussianprocess.org/gpml/, Section 3.4.3): + # W @ W.T + D = D1/2 @ (I + D-1/2 @ W @ W.T @ D-1/2) @ D1/2 + # The matrix "I + D-1/2 @ W @ W.T @ D-1/2" has eigenvalues bounded from below by 1, + # hence it is well-conditioned and safe to take Cholesky decomposition. + n = self._event_shape[0] + cov_diag_sqrt_unsqueeze = self._unbroadcasted_cov_diag.sqrt().unsqueeze(-1) + Dinvsqrt_W = self._unbroadcasted_cov_factor / cov_diag_sqrt_unsqueeze + K = torch.matmul(Dinvsqrt_W, Dinvsqrt_W.mT).contiguous() + K.view(-1, n * n)[:, :: n + 1] += 1 # add identity matrix to K + scale_tril = cov_diag_sqrt_unsqueeze * torch.linalg.cholesky(K) + return scale_tril.expand( + self._batch_shape + self._event_shape + self._event_shape + ) + + @lazy_property + def covariance_matrix(self) -> Tensor: + covariance_matrix = torch.matmul( + self._unbroadcasted_cov_factor, self._unbroadcasted_cov_factor.mT + ) + torch.diag_embed(self._unbroadcasted_cov_diag) + return covariance_matrix.expand( + self._batch_shape + self._event_shape + self._event_shape + ) + + @lazy_property + def precision_matrix(self) -> Tensor: + # We use "Woodbury matrix identity" to take advantage of low rank form:: + # inv(W @ W.T + D) = inv(D) - inv(D) @ W @ inv(C) @ W.T @ inv(D) + # where :math:`C` is the capacitance matrix. + Wt_Dinv = ( + self._unbroadcasted_cov_factor.mT + / self._unbroadcasted_cov_diag.unsqueeze(-2) + ) + A = torch.linalg.solve_triangular(self._capacitance_tril, Wt_Dinv, upper=False) + precision_matrix = ( + torch.diag_embed(self._unbroadcasted_cov_diag.reciprocal()) - A.mT @ A + ) + return precision_matrix.expand( + self._batch_shape + self._event_shape + self._event_shape + ) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + W_shape = shape[:-1] + self.cov_factor.shape[-1:] + eps_W = _standard_normal(W_shape, dtype=self.loc.dtype, device=self.loc.device) + eps_D = _standard_normal(shape, dtype=self.loc.dtype, device=self.loc.device) + return ( + self.loc + + _batch_mv(self._unbroadcasted_cov_factor, eps_W) + + self._unbroadcasted_cov_diag.sqrt() * eps_D + ) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + diff = value - self.loc + M = _batch_lowrank_mahalanobis( + self._unbroadcasted_cov_factor, + self._unbroadcasted_cov_diag, + diff, + self._capacitance_tril, + ) + log_det = _batch_lowrank_logdet( + self._unbroadcasted_cov_factor, + self._unbroadcasted_cov_diag, + self._capacitance_tril, + ) + return -0.5 * (self._event_shape[0] * math.log(2 * math.pi) + log_det + M) + + def entropy(self): + log_det = _batch_lowrank_logdet( + self._unbroadcasted_cov_factor, + self._unbroadcasted_cov_diag, + self._capacitance_tril, + ) + H = 0.5 * (self._event_shape[0] * (1.0 + math.log(2 * math.pi)) + log_det) + if len(self._batch_shape) == 0: + return H + else: + return H.expand(self._batch_shape) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/mixture_same_family.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/mixture_same_family.py new file mode 100644 index 0000000000000000000000000000000000000000..f2066688339fe17650c1de01d2861eadfcd12333 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/mixture_same_family.py @@ -0,0 +1,221 @@ +# mypy: allow-untyped-defs +from typing import Optional + +import torch +from torch import Tensor +from torch.distributions import Categorical, constraints +from torch.distributions.constraints import MixtureSameFamilyConstraint +from torch.distributions.distribution import Distribution + + +__all__ = ["MixtureSameFamily"] + + +class MixtureSameFamily(Distribution): + r""" + The `MixtureSameFamily` distribution implements a (batch of) mixture + distribution where all component are from different parameterizations of + the same distribution type. It is parameterized by a `Categorical` + "selecting distribution" (over `k` component) and a component + distribution, i.e., a `Distribution` with a rightmost batch shape + (equal to `[k]`) which indexes each (batch of) component. + + Examples:: + + >>> # xdoctest: +SKIP("undefined vars") + >>> # Construct Gaussian Mixture Model in 1D consisting of 5 equally + >>> # weighted normal distributions + >>> mix = D.Categorical(torch.ones(5,)) + >>> comp = D.Normal(torch.randn(5,), torch.rand(5,)) + >>> gmm = MixtureSameFamily(mix, comp) + + >>> # Construct Gaussian Mixture Model in 2D consisting of 5 equally + >>> # weighted bivariate normal distributions + >>> mix = D.Categorical(torch.ones(5,)) + >>> comp = D.Independent(D.Normal( + ... torch.randn(5,2), torch.rand(5,2)), 1) + >>> gmm = MixtureSameFamily(mix, comp) + + >>> # Construct a batch of 3 Gaussian Mixture Models in 2D each + >>> # consisting of 5 random weighted bivariate normal distributions + >>> mix = D.Categorical(torch.rand(3,5)) + >>> comp = D.Independent(D.Normal( + ... torch.randn(3,5,2), torch.rand(3,5,2)), 1) + >>> gmm = MixtureSameFamily(mix, comp) + + Args: + mixture_distribution: `torch.distributions.Categorical`-like + instance. Manages the probability of selecting component. + The number of categories must match the rightmost batch + dimension of the `component_distribution`. Must have either + scalar `batch_shape` or `batch_shape` matching + `component_distribution.batch_shape[:-1]` + component_distribution: `torch.distributions.Distribution`-like + instance. Right-most batch dimension indexes component. + """ + + arg_constraints: dict[str, constraints.Constraint] = {} + has_rsample = False + + def __init__( + self, + mixture_distribution: Categorical, + component_distribution: Distribution, + validate_args: Optional[bool] = None, + ) -> None: + self._mixture_distribution = mixture_distribution + self._component_distribution = component_distribution + + if not isinstance(self._mixture_distribution, Categorical): + raise ValueError( + " The Mixture distribution needs to be an " + " instance of torch.distributions.Categorical" + ) + + if not isinstance(self._component_distribution, Distribution): + raise ValueError( + "The Component distribution need to be an " + "instance of torch.distributions.Distribution" + ) + + # Check that batch size matches + mdbs = self._mixture_distribution.batch_shape + cdbs = self._component_distribution.batch_shape[:-1] + for size1, size2 in zip(reversed(mdbs), reversed(cdbs)): + if size1 != 1 and size2 != 1 and size1 != size2: + raise ValueError( + f"`mixture_distribution.batch_shape` ({mdbs}) is not " + "compatible with `component_distribution." + f"batch_shape`({cdbs})" + ) + + # Check that the number of mixture component matches + km = self._mixture_distribution.logits.shape[-1] + kc = self._component_distribution.batch_shape[-1] + if km is not None and kc is not None and km != kc: + raise ValueError( + f"`mixture_distribution component` ({km}) does not" + " equal `component_distribution.batch_shape[-1]`" + f" ({kc})" + ) + self._num_component = km + + event_shape = self._component_distribution.event_shape + self._event_ndims = len(event_shape) + super().__init__( + batch_shape=cdbs, event_shape=event_shape, validate_args=validate_args + ) + + def expand(self, batch_shape, _instance=None): + batch_shape = torch.Size(batch_shape) + batch_shape_comp = batch_shape + (self._num_component,) + new = self._get_checked_instance(MixtureSameFamily, _instance) + new._component_distribution = self._component_distribution.expand( + batch_shape_comp + ) + new._mixture_distribution = self._mixture_distribution.expand(batch_shape) + new._num_component = self._num_component + new._event_ndims = self._event_ndims + event_shape = new._component_distribution.event_shape + super(MixtureSameFamily, new).__init__( + batch_shape=batch_shape, event_shape=event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def support(self): + return MixtureSameFamilyConstraint(self._component_distribution.support) + + @property + def mixture_distribution(self) -> Categorical: + return self._mixture_distribution + + @property + def component_distribution(self) -> Distribution: + return self._component_distribution + + @property + def mean(self) -> Tensor: + probs = self._pad_mixture_dimensions(self.mixture_distribution.probs) + return torch.sum( + probs * self.component_distribution.mean, dim=-1 - self._event_ndims + ) # [B, E] + + @property + def variance(self) -> Tensor: + # Law of total variance: Var(Y) = E[Var(Y|X)] + Var(E[Y|X]) + probs = self._pad_mixture_dimensions(self.mixture_distribution.probs) + mean_cond_var = torch.sum( + probs * self.component_distribution.variance, dim=-1 - self._event_ndims + ) + var_cond_mean = torch.sum( + probs * (self.component_distribution.mean - self._pad(self.mean)).pow(2.0), + dim=-1 - self._event_ndims, + ) + return mean_cond_var + var_cond_mean + + def cdf(self, x): + x = self._pad(x) + cdf_x = self.component_distribution.cdf(x) + mix_prob = self.mixture_distribution.probs + + return torch.sum(cdf_x * mix_prob, dim=-1) + + def log_prob(self, x): + if self._validate_args: + self._validate_sample(x) + x = self._pad(x) + log_prob_x = self.component_distribution.log_prob(x) # [S, B, k] + log_mix_prob = torch.log_softmax( + self.mixture_distribution.logits, dim=-1 + ) # [B, k] + return torch.logsumexp(log_prob_x + log_mix_prob, dim=-1) # [S, B] + + def sample(self, sample_shape=torch.Size()): + with torch.no_grad(): + sample_len = len(sample_shape) + batch_len = len(self.batch_shape) + gather_dim = sample_len + batch_len + es = self.event_shape + + # mixture samples [n, B] + mix_sample = self.mixture_distribution.sample(sample_shape) + mix_shape = mix_sample.shape + + # component samples [n, B, k, E] + comp_samples = self.component_distribution.sample(sample_shape) + + # Gather along the k dimension + mix_sample_r = mix_sample.reshape( + mix_shape + torch.Size([1] * (len(es) + 1)) + ) + mix_sample_r = mix_sample_r.repeat( + torch.Size([1] * len(mix_shape)) + torch.Size([1]) + es + ) + + samples = torch.gather(comp_samples, gather_dim, mix_sample_r) + return samples.squeeze(gather_dim) + + def _pad(self, x): + return x.unsqueeze(-1 - self._event_ndims) + + def _pad_mixture_dimensions(self, x): + dist_batch_ndims = len(self.batch_shape) + cat_batch_ndims = len(self.mixture_distribution.batch_shape) + pad_ndims = 0 if cat_batch_ndims == 1 else dist_batch_ndims - cat_batch_ndims + xs = x.shape + x = x.reshape( + xs[:-1] + + torch.Size(pad_ndims * [1]) + + xs[-1:] + + torch.Size(self._event_ndims * [1]) + ) + return x + + def __repr__(self): + args_string = ( + f"\n {self.mixture_distribution},\n {self.component_distribution}" + ) + return "MixtureSameFamily" + "(" + args_string + ")" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/multinomial.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/multinomial.py new file mode 100644 index 0000000000000000000000000000000000000000..e6e957d63106cd419e343d4a1263b9e3dc320617 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/multinomial.py @@ -0,0 +1,148 @@ +# mypy: allow-untyped-defs +from typing import Optional + +import torch +from torch import inf, Tensor +from torch.distributions import Categorical, constraints +from torch.distributions.binomial import Binomial +from torch.distributions.distribution import Distribution +from torch.distributions.utils import broadcast_all + + +__all__ = ["Multinomial"] + + +class Multinomial(Distribution): + r""" + Creates a Multinomial distribution parameterized by :attr:`total_count` and + either :attr:`probs` or :attr:`logits` (but not both). The innermost dimension of + :attr:`probs` indexes over categories. All other dimensions index over batches. + + Note that :attr:`total_count` need not be specified if only :meth:`log_prob` is + called (see example below) + + .. note:: The `probs` argument must be non-negative, finite and have a non-zero sum, + and it will be normalized to sum to 1 along the last dimension. :attr:`probs` + will return this normalized value. + The `logits` argument will be interpreted as unnormalized log probabilities + and can therefore be any real number. It will likewise be normalized so that + the resulting probabilities sum to 1 along the last dimension. :attr:`logits` + will return this normalized value. + + - :meth:`sample` requires a single shared `total_count` for all + parameters and samples. + - :meth:`log_prob` allows different `total_count` for each parameter and + sample. + + Example:: + + >>> # xdoctest: +SKIP("FIXME: found invalid values") + >>> m = Multinomial(100, torch.tensor([ 1., 1., 1., 1.])) + >>> x = m.sample() # equal probability of 0, 1, 2, 3 + tensor([ 21., 24., 30., 25.]) + + >>> Multinomial(probs=torch.tensor([1., 1., 1., 1.])).log_prob(x) + tensor([-4.1338]) + + Args: + total_count (int): number of trials + probs (Tensor): event probabilities + logits (Tensor): event log probabilities (unnormalized) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.simplex, "logits": constraints.real_vector} + total_count: int + + @property + def mean(self) -> Tensor: + return self.probs * self.total_count + + @property + def variance(self) -> Tensor: + return self.total_count * self.probs * (1 - self.probs) + + def __init__( + self, + total_count: int = 1, + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + if not isinstance(total_count, int): + raise NotImplementedError("inhomogeneous total_count is not supported") + self.total_count = total_count + self._categorical = Categorical(probs=probs, logits=logits) + self._binomial = Binomial(total_count=total_count, probs=self.probs) + batch_shape = self._categorical.batch_shape + event_shape = self._categorical.param_shape[-1:] + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Multinomial, _instance) + batch_shape = torch.Size(batch_shape) + new.total_count = self.total_count + new._categorical = self._categorical.expand(batch_shape) + super(Multinomial, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._categorical._new(*args, **kwargs) + + @constraints.dependent_property(is_discrete=True, event_dim=1) + # pyrefly: ignore [bad-override] + def support(self): + return constraints.multinomial(self.total_count) + + @property + def logits(self) -> Tensor: + return self._categorical.logits + + @property + def probs(self) -> Tensor: + return self._categorical.probs + + @property + def param_shape(self) -> torch.Size: + return self._categorical.param_shape + + def sample(self, sample_shape=torch.Size()): + sample_shape = torch.Size(sample_shape) + samples = self._categorical.sample( + torch.Size((self.total_count,)) + sample_shape + ) + # samples.shape is (total_count, sample_shape, batch_shape), need to change it to + # (sample_shape, batch_shape, total_count) + shifted_idx = list(range(samples.dim())) + shifted_idx.append(shifted_idx.pop(0)) + samples = samples.permute(*shifted_idx) + counts = samples.new(self._extended_shape(sample_shape)).zero_() + counts.scatter_add_(-1, samples, torch.ones_like(samples)) + return counts.type_as(self.probs) + + def entropy(self): + n = torch.tensor(self.total_count) + + cat_entropy = self._categorical.entropy() + term1 = n * cat_entropy - torch.lgamma(n + 1) + + support = self._binomial.enumerate_support(expand=False)[1:] + binomial_probs = torch.exp(self._binomial.log_prob(support)) + weights = torch.lgamma(support + 1) + term2 = (binomial_probs * weights).sum([0, -1]) + + return term1 + term2 + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + logits, value = broadcast_all(self.logits, value) + logits = logits.clone(memory_format=torch.contiguous_format) + log_factorial_n = torch.lgamma(value.sum(-1) + 1) + log_factorial_xs = torch.lgamma(value + 1).sum(-1) + logits[(value == 0) & (logits == -inf)] = 0 + log_powers = (logits * value).sum(-1) + return log_factorial_n - log_factorial_xs + log_powers diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/multivariate_normal.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/multivariate_normal.py new file mode 100644 index 0000000000000000000000000000000000000000..9b006e400ebd1566c5af0c7fc6a9ee6cfef84d92 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/multivariate_normal.py @@ -0,0 +1,273 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import _standard_normal, lazy_property +from torch.types import _size + + +__all__ = ["MultivariateNormal"] + + +def _batch_mv(bmat, bvec): + r""" + Performs a batched matrix-vector product, with compatible but different batch shapes. + + This function takes as input `bmat`, containing :math:`n \times n` matrices, and + `bvec`, containing length :math:`n` vectors. + + Both `bmat` and `bvec` may have any number of leading dimensions, which correspond + to a batch shape. They are not necessarily assumed to have the same batch shape, + just ones which can be broadcasted. + """ + return torch.matmul(bmat, bvec.unsqueeze(-1)).squeeze(-1) + + +def _batch_mahalanobis(bL, bx): + r""" + Computes the squared Mahalanobis distance :math:`\mathbf{x}^\top\mathbf{M}^{-1}\mathbf{x}` + for a factored :math:`\mathbf{M} = \mathbf{L}\mathbf{L}^\top`. + + Accepts batches for both bL and bx. They are not necessarily assumed to have the same batch + shape, but `bL` one should be able to broadcasted to `bx` one. + """ + n = bx.size(-1) + bx_batch_shape = bx.shape[:-1] + + # Assume that bL.shape = (i, 1, n, n), bx.shape = (..., i, j, n), + # we are going to make bx have shape (..., 1, j, i, 1, n) to apply batched tri.solve + bx_batch_dims = len(bx_batch_shape) + bL_batch_dims = bL.dim() - 2 + outer_batch_dims = bx_batch_dims - bL_batch_dims + old_batch_dims = outer_batch_dims + bL_batch_dims + new_batch_dims = outer_batch_dims + 2 * bL_batch_dims + # Reshape bx with the shape (..., 1, i, j, 1, n) + bx_new_shape = bx.shape[:outer_batch_dims] + for sL, sx in zip(bL.shape[:-2], bx.shape[outer_batch_dims:-1]): + bx_new_shape += (sx // sL, sL) + bx_new_shape += (n,) + bx = bx.reshape(bx_new_shape) + # Permute bx to make it have shape (..., 1, j, i, 1, n) + permute_dims = ( + list(range(outer_batch_dims)) + + list(range(outer_batch_dims, new_batch_dims, 2)) + + list(range(outer_batch_dims + 1, new_batch_dims, 2)) + + [new_batch_dims] + ) + bx = bx.permute(permute_dims) + + flat_L = bL.reshape(-1, n, n) # shape = b x n x n + flat_x = bx.reshape(-1, flat_L.size(0), n) # shape = c x b x n + flat_x_swap = flat_x.permute(1, 2, 0) # shape = b x n x c + M_swap = ( + torch.linalg.solve_triangular(flat_L, flat_x_swap, upper=False).pow(2).sum(-2) + ) # shape = b x c + M = M_swap.t() # shape = c x b + + # Now we revert the above reshape and permute operators. + permuted_M = M.reshape(bx.shape[:-1]) # shape = (..., 1, j, i, 1) + permute_inv_dims = list(range(outer_batch_dims)) + for i in range(bL_batch_dims): + permute_inv_dims += [outer_batch_dims + i, old_batch_dims + i] + reshaped_M = permuted_M.permute(permute_inv_dims) # shape = (..., 1, i, j, 1) + return reshaped_M.reshape(bx_batch_shape) + + +def _precision_to_scale_tril(P): + # Ref: https://nbviewer.jupyter.org/gist/fehiepsi/5ef8e09e61604f10607380467eb82006#Precision-to-scale_tril + Lf = torch.linalg.cholesky(torch.flip(P, (-2, -1))) + L_inv = torch.transpose(torch.flip(Lf, (-2, -1)), -2, -1) + Id = torch.eye(P.shape[-1], dtype=P.dtype, device=P.device) + L = torch.linalg.solve_triangular(L_inv, Id, upper=False) + return L + + +class MultivariateNormal(Distribution): + r""" + Creates a multivariate normal (also called Gaussian) distribution + parameterized by a mean vector and a covariance matrix. + + The multivariate normal distribution can be parameterized either + in terms of a positive definite covariance matrix :math:`\mathbf{\Sigma}` + or a positive definite precision matrix :math:`\mathbf{\Sigma}^{-1}` + or a lower-triangular matrix :math:`\mathbf{L}` with positive-valued + diagonal entries, such that + :math:`\mathbf{\Sigma} = \mathbf{L}\mathbf{L}^\top`. This triangular matrix + can be obtained via e.g. Cholesky decomposition of the covariance. + + Example: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_LAPACK) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = MultivariateNormal(torch.zeros(2), torch.eye(2)) + >>> m.sample() # normally distributed with mean=`[0,0]` and covariance_matrix=`I` + tensor([-0.2102, -0.5429]) + + Args: + loc (Tensor): mean of the distribution + covariance_matrix (Tensor): positive-definite covariance matrix + precision_matrix (Tensor): positive-definite precision matrix + scale_tril (Tensor): lower-triangular factor of covariance, with positive-valued diagonal + + Note: + Only one of :attr:`covariance_matrix` or :attr:`precision_matrix` or + :attr:`scale_tril` can be specified. + + Using :attr:`scale_tril` will be more efficient: all computations internally + are based on :attr:`scale_tril`. If :attr:`covariance_matrix` or + :attr:`precision_matrix` is passed instead, it is only used to compute + the corresponding lower triangular matrices using a Cholesky decomposition. + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "loc": constraints.real_vector, + "covariance_matrix": constraints.positive_definite, + "precision_matrix": constraints.positive_definite, + "scale_tril": constraints.lower_cholesky, + } + support = constraints.real_vector + has_rsample = True + + def __init__( + self, + loc: Tensor, + covariance_matrix: Optional[Tensor] = None, + precision_matrix: Optional[Tensor] = None, + scale_tril: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + if loc.dim() < 1: + raise ValueError("loc must be at least one-dimensional.") + if (covariance_matrix is not None) + (scale_tril is not None) + ( + precision_matrix is not None + ) != 1: + raise ValueError( + "Exactly one of covariance_matrix or precision_matrix or scale_tril may be specified." + ) + + if scale_tril is not None: + if scale_tril.dim() < 2: + raise ValueError( + "scale_tril matrix must be at least two-dimensional, " + "with optional leading batch dimensions" + ) + batch_shape = torch.broadcast_shapes(scale_tril.shape[:-2], loc.shape[:-1]) + # pyrefly: ignore [read-only] + self.scale_tril = scale_tril.expand(batch_shape + (-1, -1)) + elif covariance_matrix is not None: + if covariance_matrix.dim() < 2: + raise ValueError( + "covariance_matrix must be at least two-dimensional, " + "with optional leading batch dimensions" + ) + batch_shape = torch.broadcast_shapes( + covariance_matrix.shape[:-2], loc.shape[:-1] + ) + # pyrefly: ignore [read-only] + self.covariance_matrix = covariance_matrix.expand(batch_shape + (-1, -1)) + else: + assert precision_matrix is not None # helps mypy + if precision_matrix.dim() < 2: + raise ValueError( + "precision_matrix must be at least two-dimensional, " + "with optional leading batch dimensions" + ) + batch_shape = torch.broadcast_shapes( + precision_matrix.shape[:-2], loc.shape[:-1] + ) + # pyrefly: ignore [read-only] + self.precision_matrix = precision_matrix.expand(batch_shape + (-1, -1)) + self.loc = loc.expand(batch_shape + (-1,)) + + event_shape = self.loc.shape[-1:] + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + if scale_tril is not None: + self._unbroadcasted_scale_tril = scale_tril + elif covariance_matrix is not None: + self._unbroadcasted_scale_tril = torch.linalg.cholesky(covariance_matrix) + else: # precision_matrix is not None + self._unbroadcasted_scale_tril = _precision_to_scale_tril(precision_matrix) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(MultivariateNormal, _instance) + batch_shape = torch.Size(batch_shape) + loc_shape = batch_shape + self.event_shape + cov_shape = batch_shape + self.event_shape + self.event_shape + new.loc = self.loc.expand(loc_shape) + new._unbroadcasted_scale_tril = self._unbroadcasted_scale_tril + if "covariance_matrix" in self.__dict__: + new.covariance_matrix = self.covariance_matrix.expand(cov_shape) + if "scale_tril" in self.__dict__: + new.scale_tril = self.scale_tril.expand(cov_shape) + if "precision_matrix" in self.__dict__: + new.precision_matrix = self.precision_matrix.expand(cov_shape) + super(MultivariateNormal, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + @lazy_property + def scale_tril(self) -> Tensor: + return self._unbroadcasted_scale_tril.expand( + self._batch_shape + self._event_shape + self._event_shape + ) + + @lazy_property + def covariance_matrix(self) -> Tensor: + return torch.matmul( + self._unbroadcasted_scale_tril, self._unbroadcasted_scale_tril.mT + ).expand(self._batch_shape + self._event_shape + self._event_shape) + + @lazy_property + def precision_matrix(self) -> Tensor: + return torch.cholesky_inverse(self._unbroadcasted_scale_tril).expand( + self._batch_shape + self._event_shape + self._event_shape + ) + + @property + def mean(self) -> Tensor: + return self.loc + + @property + def mode(self) -> Tensor: + return self.loc + + @property + def variance(self) -> Tensor: + return ( + self._unbroadcasted_scale_tril.pow(2) + .sum(-1) + .expand(self._batch_shape + self._event_shape) + ) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + eps = _standard_normal(shape, dtype=self.loc.dtype, device=self.loc.device) + return self.loc + _batch_mv(self._unbroadcasted_scale_tril, eps) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + diff = value - self.loc + M = _batch_mahalanobis(self._unbroadcasted_scale_tril, diff) + half_log_det = ( + self._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1).log().sum(-1) + ) + return -0.5 * (self._event_shape[0] * math.log(2 * math.pi) + M) - half_log_det + + def entropy(self): + half_log_det = ( + self._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1).log().sum(-1) + ) + H = 0.5 * self._event_shape[0] * (1.0 + math.log(2 * math.pi)) + half_log_det + if len(self._batch_shape) == 0: + return H + else: + return H.expand(self._batch_shape) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/negative_binomial.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/negative_binomial.py new file mode 100644 index 0000000000000000000000000000000000000000..a8f4356212bd0409701fc0060d23c517ccae06cc --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/negative_binomial.py @@ -0,0 +1,150 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +import torch.nn.functional as F +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.gamma import Gamma +from torch.distributions.utils import ( + broadcast_all, + lazy_property, + logits_to_probs, + probs_to_logits, +) + + +__all__ = ["NegativeBinomial"] + + +class NegativeBinomial(Distribution): + r""" + Creates a Negative Binomial distribution, i.e. distribution + of the number of successful independent and identical Bernoulli trials + before :attr:`total_count` failures are achieved. The probability + of success of each Bernoulli trial is :attr:`probs`. + + Args: + total_count (float or Tensor): non-negative number of negative Bernoulli + trials to stop, although the distribution is still valid for real + valued count + probs (Tensor): Event probabilities of success in the half open interval [0, 1) + logits (Tensor): Event log-odds for probabilities of success + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "total_count": constraints.greater_than_eq(0), + "probs": constraints.half_open_interval(0.0, 1.0), + "logits": constraints.real, + } + support = constraints.nonnegative_integer + + def __init__( + self, + total_count: Union[Tensor, float], + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + ( + self.total_count, + # pyrefly: ignore [read-only] + self.probs, + ) = broadcast_all(total_count, probs) + self.total_count = self.total_count.type_as(self.probs) + else: + assert logits is not None # helps mypy + ( + self.total_count, + # pyrefly: ignore [read-only] + self.logits, + ) = broadcast_all(total_count, logits) + self.total_count = self.total_count.type_as(self.logits) + + self._param = self.probs if probs is not None else self.logits + batch_shape = self._param.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(NegativeBinomial, _instance) + batch_shape = torch.Size(batch_shape) + new.total_count = self.total_count.expand(batch_shape) + if "probs" in self.__dict__: + new.probs = self.probs.expand(batch_shape) + new._param = new.probs + if "logits" in self.__dict__: + new.logits = self.logits.expand(batch_shape) + new._param = new.logits + super(NegativeBinomial, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._param.new(*args, **kwargs) + + @property + def mean(self) -> Tensor: + return self.total_count * torch.exp(self.logits) + + @property + def mode(self) -> Tensor: + return ((self.total_count - 1) * self.logits.exp()).floor().clamp(min=0.0) + + @property + def variance(self) -> Tensor: + return self.mean / torch.sigmoid(-self.logits) + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs, is_binary=True) + + @lazy_property + def probs(self) -> Tensor: + return logits_to_probs(self.logits, is_binary=True) + + @property + def param_shape(self) -> torch.Size: + return self._param.size() + + @lazy_property + def _gamma(self) -> Gamma: + # Note we avoid validating because self.total_count can be zero. + return Gamma( + concentration=self.total_count, + rate=torch.exp(-self.logits), + validate_args=False, + ) + + def sample(self, sample_shape=torch.Size()): + with torch.no_grad(): + rate = self._gamma.sample(sample_shape=sample_shape) + return torch.poisson(rate) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + + log_unnormalized_prob = self.total_count * F.logsigmoid( + -self.logits + ) + value * F.logsigmoid(self.logits) + + log_normalization = ( + -torch.lgamma(self.total_count + value) + + torch.lgamma(1.0 + value) + + torch.lgamma(self.total_count) + ) + # The case self.total_count == 0 and value == 0 has probability 1 but + # lgamma(0) is infinite. Handle this case separately using a function + # that does not modify tensors in place to allow Jit compilation. + log_normalization = log_normalization.masked_fill( + self.total_count + value == 0.0, 0.0 + ) + + return log_unnormalized_prob - log_normalization diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/normal.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/normal.py new file mode 100644 index 0000000000000000000000000000000000000000..5a7b40aa1c10294dc4b55ea118b49493b9e644cb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/normal.py @@ -0,0 +1,124 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import _standard_normal, broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Normal"] + + +class Normal(ExponentialFamily): + r""" + Creates a normal (also called Gaussian) distribution parameterized by + :attr:`loc` and :attr:`scale`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Normal(torch.tensor([0.0]), torch.tensor([1.0])) + >>> m.sample() # normally distributed with loc=0 and scale=1 + tensor([ 0.1046]) + + Args: + loc (float or Tensor): mean of the distribution (often referred to as mu) + scale (float or Tensor): standard deviation of the distribution + (often referred to as sigma) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"loc": constraints.real, "scale": constraints.positive} + support = constraints.real + has_rsample = True + _mean_carrier_measure = 0 + + @property + def mean(self) -> Tensor: + return self.loc + + @property + def mode(self) -> Tensor: + return self.loc + + @property + def stddev(self) -> Tensor: + return self.scale + + @property + def variance(self) -> Tensor: + return self.stddev.pow(2) + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.loc, self.scale = broadcast_all(loc, scale) + if isinstance(loc, _Number) and isinstance(scale, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.loc.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Normal, _instance) + batch_shape = torch.Size(batch_shape) + new.loc = self.loc.expand(batch_shape) + new.scale = self.scale.expand(batch_shape) + super(Normal, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def sample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + with torch.no_grad(): + return torch.normal(self.loc.expand(shape), self.scale.expand(shape)) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + eps = _standard_normal(shape, dtype=self.loc.dtype, device=self.loc.device) + return self.loc + eps * self.scale + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + # compute the variance + # pyrefly: ignore [unsupported-operation] + var = self.scale**2 + log_scale = ( + math.log(self.scale) + if isinstance(self.scale, _Number) + else self.scale.log() + ) + return ( + -((value - self.loc) ** 2) / (2 * var) + - log_scale + - math.log(math.sqrt(2 * math.pi)) + ) + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + return 0.5 * ( + 1 + torch.erf((value - self.loc) * self.scale.reciprocal() / math.sqrt(2)) + ) + + def icdf(self, value): + return self.loc + self.scale * torch.erfinv(2 * value - 1) * math.sqrt(2) + + def entropy(self): + return 0.5 + 0.5 * math.log(2 * math.pi) + torch.log(self.scale) + + @property + def _natural_params(self) -> tuple[Tensor, Tensor]: + return (self.loc / self.scale.pow(2), -0.5 * self.scale.pow(2).reciprocal()) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x, y): + return -0.25 * x.pow(2) / y + 0.5 * torch.log(-math.pi / y) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/one_hot_categorical.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/one_hot_categorical.py new file mode 100644 index 0000000000000000000000000000000000000000..c0bfb59293a9ccee8023786e7739e445deac2d7d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/one_hot_categorical.py @@ -0,0 +1,143 @@ +# mypy: allow-untyped-defs +from typing import Optional + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.categorical import Categorical +from torch.distributions.distribution import Distribution +from torch.types import _size + + +__all__ = ["OneHotCategorical", "OneHotCategoricalStraightThrough"] + + +class OneHotCategorical(Distribution): + r""" + Creates a one-hot categorical distribution parameterized by :attr:`probs` or + :attr:`logits`. + + Samples are one-hot coded vectors of size ``probs.size(-1)``. + + .. note:: The `probs` argument must be non-negative, finite and have a non-zero sum, + and it will be normalized to sum to 1 along the last dimension. :attr:`probs` + will return this normalized value. + The `logits` argument will be interpreted as unnormalized log probabilities + and can therefore be any real number. It will likewise be normalized so that + the resulting probabilities sum to 1 along the last dimension. :attr:`logits` + will return this normalized value. + + See also: :func:`torch.distributions.Categorical` for specifications of + :attr:`probs` and :attr:`logits`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = OneHotCategorical(torch.tensor([ 0.25, 0.25, 0.25, 0.25 ])) + >>> m.sample() # equal probability of 0, 1, 2, 3 + tensor([ 0., 0., 0., 1.]) + + Args: + probs (Tensor): event probabilities + logits (Tensor): event log probabilities (unnormalized) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.simplex, "logits": constraints.real_vector} + support = constraints.one_hot + has_enumerate_support = True + + def __init__( + self, + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + self._categorical = Categorical(probs, logits) + batch_shape = self._categorical.batch_shape + event_shape = self._categorical.param_shape[-1:] + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(OneHotCategorical, _instance) + batch_shape = torch.Size(batch_shape) + new._categorical = self._categorical.expand(batch_shape) + super(OneHotCategorical, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._categorical._new(*args, **kwargs) + + @property + def _param(self) -> Tensor: + return self._categorical._param + + @property + def probs(self) -> Tensor: + return self._categorical.probs + + @property + def logits(self) -> Tensor: + return self._categorical.logits + + @property + def mean(self) -> Tensor: + return self._categorical.probs + + @property + def mode(self) -> Tensor: + probs = self._categorical.probs + mode = probs.argmax(dim=-1) + return torch.nn.functional.one_hot(mode, num_classes=probs.shape[-1]).to(probs) + + @property + def variance(self) -> Tensor: + return self._categorical.probs * (1 - self._categorical.probs) + + @property + def param_shape(self) -> torch.Size: + return self._categorical.param_shape + + def sample(self, sample_shape=torch.Size()): + sample_shape = torch.Size(sample_shape) + probs = self._categorical.probs + num_events = self._categorical._num_events + indices = self._categorical.sample(sample_shape) + return torch.nn.functional.one_hot(indices, num_events).to(probs) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + indices = value.max(-1)[1] + return self._categorical.log_prob(indices) + + def entropy(self): + return self._categorical.entropy() + + def enumerate_support(self, expand=True): + n = self.event_shape[0] + values = torch.eye(n, dtype=self._param.dtype, device=self._param.device) + values = values.view((n,) + (1,) * len(self.batch_shape) + (n,)) + if expand: + values = values.expand((n,) + self.batch_shape + (n,)) + return values + + +class OneHotCategoricalStraightThrough(OneHotCategorical): + r""" + Creates a reparameterizable :class:`OneHotCategorical` distribution based on the straight- + through gradient estimator from [1]. + + [1] Estimating or Propagating Gradients Through Stochastic Neurons for Conditional Computation + (Bengio et al., 2013) + """ + + has_rsample = True + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + samples = self.sample(sample_shape) + probs = self._categorical.probs # cached via @lazy_property + return samples + (probs - probs.detach()) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/pareto.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/pareto.py new file mode 100644 index 0000000000000000000000000000000000000000..66e16b11353b3527cd7146bd4302d13066530b47 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/pareto.py @@ -0,0 +1,74 @@ +from typing import Optional, Union + +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exponential import Exponential +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import AffineTransform, ExpTransform +from torch.distributions.utils import broadcast_all +from torch.types import _size + + +__all__ = ["Pareto"] + + +class Pareto(TransformedDistribution): + r""" + Samples from a Pareto Type 1 distribution. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Pareto(torch.tensor([1.0]), torch.tensor([1.0])) + >>> m.sample() # sample from a Pareto distribution with scale=1 and alpha=1 + tensor([ 1.5623]) + + Args: + scale (float or Tensor): Scale parameter of the distribution + alpha (float or Tensor): Shape parameter of the distribution + """ + + arg_constraints = {"alpha": constraints.positive, "scale": constraints.positive} + + def __init__( + self, + scale: Union[Tensor, float], + alpha: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.scale, self.alpha = broadcast_all(scale, alpha) + base_dist = Exponential(self.alpha, validate_args=validate_args) + transforms = [ExpTransform(), AffineTransform(loc=0, scale=self.scale)] + # pyrefly: ignore [bad-argument-type] + super().__init__(base_dist, transforms, validate_args=validate_args) + + def expand( + self, batch_shape: _size, _instance: Optional["Pareto"] = None + ) -> "Pareto": + new = self._get_checked_instance(Pareto, _instance) + new.scale = self.scale.expand(batch_shape) + new.alpha = self.alpha.expand(batch_shape) + return super().expand(batch_shape, _instance=new) + + @property + def mean(self) -> Tensor: + # mean is inf for alpha <= 1 + a = self.alpha.clamp(min=1) + return a * self.scale / (a - 1) + + @property + def mode(self) -> Tensor: + return self.scale + + @property + def variance(self) -> Tensor: + # var is inf for alpha <= 2 + a = self.alpha.clamp(min=2) + return self.scale.pow(2) * a / ((a - 1).pow(2) * (a - 2)) + + @constraints.dependent_property(is_discrete=False, event_dim=0) + def support(self) -> constraints.Constraint: + return constraints.greater_than_eq(self.scale) + + def entropy(self) -> Tensor: + return (self.scale / self.alpha).log() + (1 + self.alpha.reciprocal()) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/poisson.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/poisson.py new file mode 100644 index 0000000000000000000000000000000000000000..452c6eeb6c91eb59da0ab66668a4ecc22f2fe954 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/poisson.py @@ -0,0 +1,88 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.utils import broadcast_all +from torch.types import _Number, Number + + +__all__ = ["Poisson"] + + +class Poisson(ExponentialFamily): + r""" + Creates a Poisson distribution parameterized by :attr:`rate`, the rate parameter. + + Samples are nonnegative integers, with a pmf given by + + .. math:: + \mathrm{rate}^k \frac{e^{-\mathrm{rate}}}{k!} + + Example:: + + >>> # xdoctest: +SKIP("poisson_cpu not implemented for 'Long'") + >>> m = Poisson(torch.tensor([4])) + >>> m.sample() + tensor([ 3.]) + + Args: + rate (Number, Tensor): the rate parameter + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"rate": constraints.nonnegative} + support = constraints.nonnegative_integer + + @property + def mean(self) -> Tensor: + return self.rate + + @property + def mode(self) -> Tensor: + return self.rate.floor() + + @property + def variance(self) -> Tensor: + return self.rate + + def __init__( + self, + rate: Union[Tensor, Number], + validate_args: Optional[bool] = None, + ) -> None: + (self.rate,) = broadcast_all(rate) + if isinstance(rate, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.rate.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Poisson, _instance) + batch_shape = torch.Size(batch_shape) + new.rate = self.rate.expand(batch_shape) + super(Poisson, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def sample(self, sample_shape=torch.Size()): + shape = self._extended_shape(sample_shape) + with torch.no_grad(): + return torch.poisson(self.rate.expand(shape)) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + rate, value = broadcast_all(self.rate, value) + return value.xlogy(rate) - rate - (value + 1).lgamma() + + @property + def _natural_params(self) -> tuple[Tensor]: + return (torch.log(self.rate),) + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x): + return torch.exp(x) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/relaxed_bernoulli.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/relaxed_bernoulli.py new file mode 100644 index 0000000000000000000000000000000000000000..51f59245c4c8461611fcbb68592e71e9e9536d1f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/relaxed_bernoulli.py @@ -0,0 +1,174 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import SigmoidTransform +from torch.distributions.utils import ( + broadcast_all, + clamp_probs, + lazy_property, + logits_to_probs, + probs_to_logits, +) +from torch.types import _Number, _size, Number + + +__all__ = ["LogitRelaxedBernoulli", "RelaxedBernoulli"] + + +class LogitRelaxedBernoulli(Distribution): + r""" + Creates a LogitRelaxedBernoulli distribution parameterized by :attr:`probs` + or :attr:`logits` (but not both), which is the logit of a RelaxedBernoulli + distribution. + + Samples are logits of values in (0, 1). See [1] for more details. + + Args: + temperature (Tensor): relaxation temperature + probs (Number, Tensor): the probability of sampling `1` + logits (Number, Tensor): the log-odds of sampling `1` + + [1] The Concrete Distribution: A Continuous Relaxation of Discrete Random + Variables (Maddison et al., 2017) + + [2] Categorical Reparametrization with Gumbel-Softmax + (Jang et al., 2017) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.unit_interval, "logits": constraints.real} + support = constraints.real + + def __init__( + self, + temperature: Tensor, + probs: Optional[Union[Tensor, Number]] = None, + logits: Optional[Union[Tensor, Number]] = None, + validate_args: Optional[bool] = None, + ) -> None: + self.temperature = temperature + if (probs is None) == (logits is None): + raise ValueError( + "Either `probs` or `logits` must be specified, but not both." + ) + if probs is not None: + is_scalar = isinstance(probs, _Number) + # pyrefly: ignore [read-only] + (self.probs,) = broadcast_all(probs) + else: + assert logits is not None # helps mypy + is_scalar = isinstance(logits, _Number) + # pyrefly: ignore [read-only] + (self.logits,) = broadcast_all(logits) + self._param = self.probs if probs is not None else self.logits + if is_scalar: + batch_shape = torch.Size() + else: + batch_shape = self._param.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(LogitRelaxedBernoulli, _instance) + batch_shape = torch.Size(batch_shape) + new.temperature = self.temperature + if "probs" in self.__dict__: + new.probs = self.probs.expand(batch_shape) + new._param = new.probs + if "logits" in self.__dict__: + new.logits = self.logits.expand(batch_shape) + new._param = new.logits + super(LogitRelaxedBernoulli, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._param.new(*args, **kwargs) + + @lazy_property + def logits(self) -> Tensor: + return probs_to_logits(self.probs, is_binary=True) + + @lazy_property + def probs(self) -> Tensor: + return logits_to_probs(self.logits, is_binary=True) + + @property + def param_shape(self) -> torch.Size: + return self._param.size() + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + probs = clamp_probs(self.probs.expand(shape)) + uniforms = clamp_probs( + torch.rand(shape, dtype=probs.dtype, device=probs.device) + ) + return ( + uniforms.log() - (-uniforms).log1p() + probs.log() - (-probs).log1p() + ) / self.temperature + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + logits, value = broadcast_all(self.logits, value) + diff = logits - value.mul(self.temperature) + return self.temperature.log() + diff - 2 * diff.exp().log1p() + + +class RelaxedBernoulli(TransformedDistribution): + r""" + Creates a RelaxedBernoulli distribution, parametrized by + :attr:`temperature`, and either :attr:`probs` or :attr:`logits` + (but not both). This is a relaxed version of the `Bernoulli` distribution, + so the values are in (0, 1), and has reparametrizable samples. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = RelaxedBernoulli(torch.tensor([2.2]), + ... torch.tensor([0.1, 0.2, 0.3, 0.99])) + >>> m.sample() + tensor([ 0.2951, 0.3442, 0.8918, 0.9021]) + + Args: + temperature (Tensor): relaxation temperature + probs (Number, Tensor): the probability of sampling `1` + logits (Number, Tensor): the log-odds of sampling `1` + """ + + arg_constraints = {"probs": constraints.unit_interval, "logits": constraints.real} + # pyrefly: ignore [bad-override] + support = constraints.unit_interval + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: LogitRelaxedBernoulli + + def __init__( + self, + temperature: Tensor, + probs: Optional[Union[Tensor, Number]] = None, + logits: Optional[Union[Tensor, Number]] = None, + validate_args: Optional[bool] = None, + ) -> None: + base_dist = LogitRelaxedBernoulli(temperature, probs, logits) + super().__init__(base_dist, SigmoidTransform(), validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(RelaxedBernoulli, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def temperature(self) -> Tensor: + return self.base_dist.temperature + + @property + def logits(self) -> Tensor: + return self.base_dist.logits + + @property + def probs(self) -> Tensor: + return self.base_dist.probs diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/relaxed_categorical.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/relaxed_categorical.py new file mode 100644 index 0000000000000000000000000000000000000000..00052ad909889d6d45b8cff2651f7d563336b925 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/relaxed_categorical.py @@ -0,0 +1,163 @@ +# mypy: allow-untyped-defs +from typing import Optional + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.categorical import Categorical +from torch.distributions.distribution import Distribution +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import ExpTransform +from torch.distributions.utils import broadcast_all, clamp_probs +from torch.types import _size + + +__all__ = ["ExpRelaxedCategorical", "RelaxedOneHotCategorical"] + + +class ExpRelaxedCategorical(Distribution): + r""" + Creates a ExpRelaxedCategorical parameterized by + :attr:`temperature`, and either :attr:`probs` or :attr:`logits` (but not both). + Returns the log of a point in the simplex. Based on the interface to + :class:`OneHotCategorical`. + + Implementation based on [1]. + + See also: :func:`torch.distributions.OneHotCategorical` + + Args: + temperature (Tensor): relaxation temperature + probs (Tensor): event probabilities + logits (Tensor): unnormalized log probability for each event + + [1] The Concrete Distribution: A Continuous Relaxation of Discrete Random Variables + (Maddison et al., 2017) + + [2] Categorical Reparametrization with Gumbel-Softmax + (Jang et al., 2017) + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"probs": constraints.simplex, "logits": constraints.real_vector} + support = ( + constraints.real_vector + ) # The true support is actually a submanifold of this. + has_rsample = True + + def __init__( + self, + temperature: Tensor, + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + self._categorical = Categorical(probs, logits) + self.temperature = temperature + batch_shape = self._categorical.batch_shape + event_shape = self._categorical.param_shape[-1:] + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(ExpRelaxedCategorical, _instance) + batch_shape = torch.Size(batch_shape) + new.temperature = self.temperature + new._categorical = self._categorical.expand(batch_shape) + super(ExpRelaxedCategorical, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + def _new(self, *args, **kwargs): + return self._categorical._new(*args, **kwargs) + + @property + def param_shape(self) -> torch.Size: + return self._categorical.param_shape + + @property + def logits(self) -> Tensor: + return self._categorical.logits + + @property + def probs(self) -> Tensor: + return self._categorical.probs + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + uniforms = clamp_probs( + torch.rand(shape, dtype=self.logits.dtype, device=self.logits.device) + ) + gumbels = -((-(uniforms.log())).log()) + scores = (self.logits + gumbels) / self.temperature + return scores - scores.logsumexp(dim=-1, keepdim=True) + + def log_prob(self, value): + K = self._categorical._num_events + if self._validate_args: + self._validate_sample(value) + logits, value = broadcast_all(self.logits, value) + log_scale = torch.full_like( + self.temperature, float(K) + ).lgamma() - self.temperature.log().mul(-(K - 1)) + score = logits - value.mul(self.temperature) + score = (score - score.logsumexp(dim=-1, keepdim=True)).sum(-1) + return score + log_scale + + +class RelaxedOneHotCategorical(TransformedDistribution): + r""" + Creates a RelaxedOneHotCategorical distribution parametrized by + :attr:`temperature`, and either :attr:`probs` or :attr:`logits`. + This is a relaxed version of the :class:`OneHotCategorical` distribution, so + its samples are on simplex, and are reparametrizable. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = RelaxedOneHotCategorical(torch.tensor([2.2]), + ... torch.tensor([0.1, 0.2, 0.3, 0.4])) + >>> m.sample() + tensor([ 0.1294, 0.2324, 0.3859, 0.2523]) + + Args: + temperature (Tensor): relaxation temperature + probs (Tensor): event probabilities + logits (Tensor): unnormalized log probability for each event + """ + + arg_constraints = {"probs": constraints.simplex, "logits": constraints.real_vector} + # pyrefly: ignore [bad-override] + support = constraints.simplex + has_rsample = True + # pyrefly: ignore [bad-override] + base_dist: ExpRelaxedCategorical + + def __init__( + self, + temperature: Tensor, + probs: Optional[Tensor] = None, + logits: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + base_dist = ExpRelaxedCategorical( + temperature, probs, logits, validate_args=validate_args + ) + super().__init__(base_dist, ExpTransform(), validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(RelaxedOneHotCategorical, _instance) + return super().expand(batch_shape, _instance=new) + + @property + def temperature(self) -> Tensor: + return self.base_dist.temperature + + @property + def logits(self) -> Tensor: + return self.base_dist.logits + + @property + def probs(self) -> Tensor: + return self.base_dist.probs diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/studentT.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/studentT.py new file mode 100644 index 0000000000000000000000000000000000000000..1249255409a9cdfb5dcd6b70a3b7572f60e7343f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/studentT.py @@ -0,0 +1,128 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional, Union + +import torch +from torch import inf, nan, Tensor +from torch.distributions import Chi2, constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import _standard_normal, broadcast_all +from torch.types import _size + + +__all__ = ["StudentT"] + + +class StudentT(Distribution): + r""" + Creates a Student's t-distribution parameterized by degree of + freedom :attr:`df`, mean :attr:`loc` and scale :attr:`scale`. + + Example:: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = StudentT(torch.tensor([2.0])) + >>> m.sample() # Student's t-distributed with degrees of freedom=2 + tensor([ 0.1046]) + + Args: + df (float or Tensor): degrees of freedom + loc (float or Tensor): mean of the distribution + scale (float or Tensor): scale of the distribution + """ + + # pyrefly: ignore [bad-override] + arg_constraints = { + "df": constraints.positive, + "loc": constraints.real, + "scale": constraints.positive, + } + support = constraints.real + has_rsample = True + + @property + def mean(self) -> Tensor: + m = self.loc.clone(memory_format=torch.contiguous_format) + m[self.df <= 1] = nan + return m + + @property + def mode(self) -> Tensor: + return self.loc + + @property + def variance(self) -> Tensor: + m = self.df.clone(memory_format=torch.contiguous_format) + m[self.df > 2] = ( + self.scale[self.df > 2].pow(2) + * self.df[self.df > 2] + / (self.df[self.df > 2] - 2) + ) + m[(self.df <= 2) & (self.df > 1)] = inf + m[self.df <= 1] = nan + return m + + def __init__( + self, + df: Union[Tensor, float], + loc: Union[Tensor, float] = 0.0, + scale: Union[Tensor, float] = 1.0, + validate_args: Optional[bool] = None, + ) -> None: + self.df, self.loc, self.scale = broadcast_all(df, loc, scale) + self._chi2 = Chi2(self.df) + batch_shape = self.df.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(StudentT, _instance) + batch_shape = torch.Size(batch_shape) + new.df = self.df.expand(batch_shape) + new.loc = self.loc.expand(batch_shape) + new.scale = self.scale.expand(batch_shape) + new._chi2 = self._chi2.expand(batch_shape) + super(StudentT, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + # NOTE: This does not agree with scipy implementation as much as other distributions. + # (see https://github.com/fritzo/notebooks/blob/master/debug-student-t.ipynb). Using DoubleTensor + # parameters seems to help. + + # X ~ Normal(0, 1) + # Z ~ Chi2(df) + # Y = X / sqrt(Z / df) ~ StudentT(df) + shape = self._extended_shape(sample_shape) + X = _standard_normal(shape, dtype=self.df.dtype, device=self.df.device) + Z = self._chi2.rsample(sample_shape) + Y = X * torch.rsqrt(Z / self.df) + return self.loc + self.scale * Y + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + y = (value - self.loc) / self.scale + Z = ( + self.scale.log() + + 0.5 * self.df.log() + + 0.5 * math.log(math.pi) + + torch.lgamma(0.5 * self.df) + - torch.lgamma(0.5 * (self.df + 1.0)) + ) + return -0.5 * (self.df + 1.0) * torch.log1p(y**2.0 / self.df) - Z + + def entropy(self): + lbeta = ( + torch.lgamma(0.5 * self.df) + + math.lgamma(0.5) + - torch.lgamma(0.5 * (self.df + 1)) + ) + return ( + self.scale.log() + + 0.5 + * (self.df + 1) + * (torch.digamma(0.5 * (self.df + 1)) - torch.digamma(0.5 * self.df)) + + 0.5 * self.df.log() + + lbeta + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/transformed_distribution.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/transformed_distribution.py new file mode 100644 index 0000000000000000000000000000000000000000..4fd375b12fedb267897ae1538b3423c59b530570 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/transformed_distribution.py @@ -0,0 +1,224 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.independent import Independent +from torch.distributions.transforms import ComposeTransform, Transform +from torch.distributions.utils import _sum_rightmost +from torch.types import _size + + +__all__ = ["TransformedDistribution"] + + +class TransformedDistribution(Distribution): + r""" + Extension of the Distribution class, which applies a sequence of Transforms + to a base distribution. Let f be the composition of transforms applied:: + + X ~ BaseDistribution + Y = f(X) ~ TransformedDistribution(BaseDistribution, f) + log p(Y) = log p(X) + log |det (dX/dY)| + + Note that the ``.event_shape`` of a :class:`TransformedDistribution` is the + maximum shape of its base distribution and its transforms, since transforms + can introduce correlations among events. + + An example for the usage of :class:`TransformedDistribution` would be:: + + # Building a Logistic Distribution + # X ~ Uniform(0, 1) + # f = a + b * logit(X) + # Y ~ f(X) ~ Logistic(a, b) + base_distribution = Uniform(0, 1) + transforms = [SigmoidTransform().inv, AffineTransform(loc=a, scale=b)] + logistic = TransformedDistribution(base_distribution, transforms) + + For more examples, please look at the implementations of + :class:`~torch.distributions.gumbel.Gumbel`, + :class:`~torch.distributions.half_cauchy.HalfCauchy`, + :class:`~torch.distributions.half_normal.HalfNormal`, + :class:`~torch.distributions.log_normal.LogNormal`, + :class:`~torch.distributions.pareto.Pareto`, + :class:`~torch.distributions.weibull.Weibull`, + :class:`~torch.distributions.relaxed_bernoulli.RelaxedBernoulli` and + :class:`~torch.distributions.relaxed_categorical.RelaxedOneHotCategorical` + """ + + arg_constraints: dict[str, constraints.Constraint] = {} + + def __init__( + self, + base_distribution: Distribution, + transforms: Union[Transform, list[Transform]], + validate_args: Optional[bool] = None, + ) -> None: + if isinstance(transforms, Transform): + self.transforms = [ + transforms, + ] + elif isinstance(transforms, list): + if not all(isinstance(t, Transform) for t in transforms): + raise ValueError( + "transforms must be a Transform or a list of Transforms" + ) + self.transforms = transforms + else: + raise ValueError( + f"transforms must be a Transform or list, but was {transforms}" + ) + + # Reshape base_distribution according to transforms. + base_shape = base_distribution.batch_shape + base_distribution.event_shape + base_event_dim = len(base_distribution.event_shape) + transform = ComposeTransform(self.transforms) + if len(base_shape) < transform.domain.event_dim: + raise ValueError( + f"base_distribution needs to have shape with size at least {transform.domain.event_dim}, but got {base_shape}." + ) + forward_shape = transform.forward_shape(base_shape) + expanded_base_shape = transform.inverse_shape(forward_shape) + if base_shape != expanded_base_shape: + base_batch_shape = expanded_base_shape[ + : len(expanded_base_shape) - base_event_dim + ] + base_distribution = base_distribution.expand(base_batch_shape) + reinterpreted_batch_ndims = transform.domain.event_dim - base_event_dim + if reinterpreted_batch_ndims > 0: + base_distribution = Independent( + base_distribution, reinterpreted_batch_ndims + ) + self.base_dist = base_distribution + + # Compute shapes. + transform_change_in_event_dim = ( + transform.codomain.event_dim - transform.domain.event_dim + ) + event_dim = max( + transform.codomain.event_dim, # the transform is coupled + base_event_dim + transform_change_in_event_dim, # the base dist is coupled + ) + assert len(forward_shape) >= event_dim + cut = len(forward_shape) - event_dim + batch_shape = forward_shape[:cut] + event_shape = forward_shape[cut:] + super().__init__(batch_shape, event_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(TransformedDistribution, _instance) + batch_shape = torch.Size(batch_shape) + shape = batch_shape + self.event_shape + for t in reversed(self.transforms): + shape = t.inverse_shape(shape) + base_batch_shape = shape[: len(shape) - len(self.base_dist.event_shape)] + new.base_dist = self.base_dist.expand(base_batch_shape) + new.transforms = self.transforms + super(TransformedDistribution, new).__init__( + batch_shape, self.event_shape, validate_args=False + ) + new._validate_args = self._validate_args + return new + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def support(self): + if not self.transforms: + return self.base_dist.support + support = self.transforms[-1].codomain + if len(self.event_shape) > support.event_dim: + support = constraints.independent( + support, len(self.event_shape) - support.event_dim + ) + return support + + @property + def has_rsample(self) -> bool: # type: ignore[override] + return self.base_dist.has_rsample + + def sample(self, sample_shape=torch.Size()): + """ + Generates a sample_shape shaped sample or sample_shape shaped batch of + samples if the distribution parameters are batched. Samples first from + base distribution and applies `transform()` for every transform in the + list. + """ + with torch.no_grad(): + x = self.base_dist.sample(sample_shape) + for transform in self.transforms: + x = transform(x) + return x + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + """ + Generates a sample_shape shaped reparameterized sample or sample_shape + shaped batch of reparameterized samples if the distribution parameters + are batched. Samples first from base distribution and applies + `transform()` for every transform in the list. + """ + x = self.base_dist.rsample(sample_shape) + for transform in self.transforms: + x = transform(x) + return x + + def log_prob(self, value): + """ + Scores the sample by inverting the transform(s) and computing the score + using the score of the base distribution and the log abs det jacobian. + """ + if self._validate_args: + self._validate_sample(value) + event_dim = len(self.event_shape) + log_prob: Union[Tensor, float] = 0.0 + y = value + for transform in reversed(self.transforms): + x = transform.inv(y) + event_dim += transform.domain.event_dim - transform.codomain.event_dim + log_prob = log_prob - _sum_rightmost( + transform.log_abs_det_jacobian(x, y), + event_dim - transform.domain.event_dim, + ) + y = x + + log_prob = log_prob + _sum_rightmost( + self.base_dist.log_prob(y), event_dim - len(self.base_dist.event_shape) + ) + return log_prob + + def _monotonize_cdf(self, value): + """ + This conditionally flips ``value -> 1-value`` to ensure :meth:`cdf` is + monotone increasing. + """ + sign = 1 + for transform in self.transforms: + sign = sign * transform.sign + if isinstance(sign, int) and sign == 1: + return value + return sign * (value - 0.5) + 0.5 + + def cdf(self, value): + """ + Computes the cumulative distribution function by inverting the + transform(s) and computing the score of the base distribution. + """ + for transform in self.transforms[::-1]: + value = transform.inv(value) + if self._validate_args: + self.base_dist._validate_sample(value) + value = self.base_dist.cdf(value) + value = self._monotonize_cdf(value) + return value + + def icdf(self, value): + """ + Computes the inverse cumulative distribution function using + transform(s) and computing the score of the base distribution. + """ + value = self._monotonize_cdf(value) + value = self.base_dist.icdf(value) + for transform in self.transforms: + value = transform(value) + return value diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/transforms.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..26644775aff1156318a2f07b0c8e79382d50ea8d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/transforms.py @@ -0,0 +1,1302 @@ +# mypy: allow-untyped-defs +import functools +import math +import operator +import weakref +from collections.abc import Sequence +from typing import Optional, Union + +import torch +import torch.nn.functional as F +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import ( + _sum_rightmost, + broadcast_all, + lazy_property, + tril_matrix_to_vec, + vec_to_tril_matrix, +) +from torch.nn.functional import pad, softplus +from torch.types import _Number + + +__all__ = [ + "AbsTransform", + "AffineTransform", + "CatTransform", + "ComposeTransform", + "CorrCholeskyTransform", + "CumulativeDistributionTransform", + "ExpTransform", + "IndependentTransform", + "LowerCholeskyTransform", + "PositiveDefiniteTransform", + "PowerTransform", + "ReshapeTransform", + "SigmoidTransform", + "SoftplusTransform", + "TanhTransform", + "SoftmaxTransform", + "StackTransform", + "StickBreakingTransform", + "Transform", + "identity_transform", +] + + +class Transform: + """ + Abstract class for invertable transformations with computable log + det jacobians. They are primarily used in + :class:`torch.distributions.TransformedDistribution`. + + Caching is useful for transforms whose inverses are either expensive or + numerically unstable. Note that care must be taken with memoized values + since the autograd graph may be reversed. For example while the following + works with or without caching:: + + y = t(x) + t.log_abs_det_jacobian(x, y).backward() # x will receive gradients. + + However the following will error when caching due to dependency reversal:: + + y = t(x) + z = t.inv(y) + grad(z.sum(), [y]) # error because z is x + + Derived classes should implement one or both of :meth:`_call` or + :meth:`_inverse`. Derived classes that set `bijective=True` should also + implement :meth:`log_abs_det_jacobian`. + + Args: + cache_size (int): Size of cache. If zero, no caching is done. If one, + the latest single value is cached. Only 0 and 1 are supported. + + Attributes: + domain (:class:`~torch.distributions.constraints.Constraint`): + The constraint representing valid inputs to this transform. + codomain (:class:`~torch.distributions.constraints.Constraint`): + The constraint representing valid outputs to this transform + which are inputs to the inverse transform. + bijective (bool): Whether this transform is bijective. A transform + ``t`` is bijective iff ``t.inv(t(x)) == x`` and + ``t(t.inv(y)) == y`` for every ``x`` in the domain and ``y`` in + the codomain. Transforms that are not bijective should at least + maintain the weaker pseudoinverse properties + ``t(t.inv(t(x)) == t(x)`` and ``t.inv(t(t.inv(y))) == t.inv(y)``. + sign (int or Tensor): For bijective univariate transforms, this + should be +1 or -1 depending on whether transform is monotone + increasing or decreasing. + """ + + bijective = False + domain: constraints.Constraint + codomain: constraints.Constraint + + def __init__(self, cache_size: int = 0) -> None: + self._cache_size = cache_size + self._inv: Optional[weakref.ReferenceType[Transform]] = None + if cache_size == 0: + pass # default behavior + elif cache_size == 1: + self._cached_x_y = None, None + else: + raise ValueError("cache_size must be 0 or 1") + super().__init__() + + def __getstate__(self): + state = self.__dict__.copy() + state["_inv"] = None + return state + + @property + def event_dim(self) -> int: + if self.domain.event_dim == self.codomain.event_dim: + return self.domain.event_dim + raise ValueError("Please use either .domain.event_dim or .codomain.event_dim") + + @property + def inv(self) -> "Transform": + """ + Returns the inverse :class:`Transform` of this transform. + This should satisfy ``t.inv.inv is t``. + """ + inv = None + if self._inv is not None: + inv = self._inv() + if inv is None: + inv = _InverseTransform(self) + self._inv = weakref.ref(inv) + return inv + + @property + def sign(self) -> int: + """ + Returns the sign of the determinant of the Jacobian, if applicable. + In general this only makes sense for bijective transforms. + """ + raise NotImplementedError + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + if type(self).__init__ is Transform.__init__: + return type(self)(cache_size=cache_size) + raise NotImplementedError(f"{type(self)}.with_cache is not implemented") + + def __eq__(self, other): + return self is other + + def __ne__(self, other): + # Necessary for Python2 + return not self.__eq__(other) + + def __call__(self, x): + """ + Computes the transform `x => y`. + """ + if self._cache_size == 0: + return self._call(x) + x_old, y_old = self._cached_x_y + if x is x_old: + return y_old + y = self._call(x) + self._cached_x_y = x, y + return y + + def _inv_call(self, y): + """ + Inverts the transform `y => x`. + """ + if self._cache_size == 0: + return self._inverse(y) + x_old, y_old = self._cached_x_y + if y is y_old: + return x_old + x = self._inverse(y) + self._cached_x_y = x, y + return x + + def _call(self, x): + """ + Abstract method to compute forward transformation. + """ + raise NotImplementedError + + def _inverse(self, y): + """ + Abstract method to compute inverse transformation. + """ + raise NotImplementedError + + def log_abs_det_jacobian(self, x, y): + """ + Computes the log det jacobian `log |dy/dx|` given input and output. + """ + raise NotImplementedError + + def __repr__(self): + return self.__class__.__name__ + "()" + + def forward_shape(self, shape): + """ + Infers the shape of the forward computation, given the input shape. + Defaults to preserving shape. + """ + return shape + + def inverse_shape(self, shape): + """ + Infers the shapes of the inverse computation, given the output shape. + Defaults to preserving shape. + """ + return shape + + +class _InverseTransform(Transform): + """ + Inverts a single :class:`Transform`. + This class is private; please instead use the ``Transform.inv`` property. + """ + + def __init__(self, transform: Transform) -> None: + super().__init__(cache_size=transform._cache_size) + self._inv: Transform = transform # type: ignore[assignment] + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def domain(self): + assert self._inv is not None + return self._inv.codomain + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def codomain(self): + assert self._inv is not None + return self._inv.domain + + @property + def bijective(self) -> bool: # type: ignore[override] + assert self._inv is not None + return self._inv.bijective + + @property + def sign(self) -> int: + assert self._inv is not None + return self._inv.sign + + @property + def inv(self) -> Transform: + return self._inv + + def with_cache(self, cache_size=1): + assert self._inv is not None + return self.inv.with_cache(cache_size).inv + + def __eq__(self, other): + if not isinstance(other, _InverseTransform): + return False + assert self._inv is not None + return self._inv == other._inv + + def __repr__(self): + return f"{self.__class__.__name__}({repr(self._inv)})" + + def __call__(self, x): + assert self._inv is not None + return self._inv._inv_call(x) + + def log_abs_det_jacobian(self, x, y): + assert self._inv is not None + return -self._inv.log_abs_det_jacobian(y, x) + + def forward_shape(self, shape): + return self._inv.inverse_shape(shape) + + def inverse_shape(self, shape): + return self._inv.forward_shape(shape) + + +class ComposeTransform(Transform): + """ + Composes multiple transforms in a chain. + The transforms being composed are responsible for caching. + + Args: + parts (list of :class:`Transform`): A list of transforms to compose. + cache_size (int): Size of cache. If zero, no caching is done. If one, + the latest single value is cached. Only 0 and 1 are supported. + """ + + def __init__(self, parts: list[Transform], cache_size: int = 0) -> None: + if cache_size: + parts = [part.with_cache(cache_size) for part in parts] + super().__init__(cache_size=cache_size) + self.parts = parts + + def __eq__(self, other): + if not isinstance(other, ComposeTransform): + return False + return self.parts == other.parts + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def domain(self): + if not self.parts: + return constraints.real + domain = self.parts[0].domain + # Adjust event_dim to be maximum among all parts. + event_dim = self.parts[-1].codomain.event_dim + for part in reversed(self.parts): + event_dim += part.domain.event_dim - part.codomain.event_dim + event_dim = max(event_dim, part.domain.event_dim) + assert event_dim >= domain.event_dim + if event_dim > domain.event_dim: + domain = constraints.independent(domain, event_dim - domain.event_dim) + return domain + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def codomain(self): + if not self.parts: + return constraints.real + codomain = self.parts[-1].codomain + # Adjust event_dim to be maximum among all parts. + event_dim = self.parts[0].domain.event_dim + for part in self.parts: + event_dim += part.codomain.event_dim - part.domain.event_dim + event_dim = max(event_dim, part.codomain.event_dim) + assert event_dim >= codomain.event_dim + if event_dim > codomain.event_dim: + codomain = constraints.independent(codomain, event_dim - codomain.event_dim) + return codomain + + @lazy_property + def bijective(self) -> bool: # type: ignore[override] + return all(p.bijective for p in self.parts) + + @lazy_property + def sign(self) -> int: # type: ignore[override] + sign = 1 + for p in self.parts: + sign = sign * p.sign + return sign + + @property + def inv(self) -> Transform: + inv = None + if self._inv is not None: + inv = self._inv() + if inv is None: + inv = ComposeTransform([p.inv for p in reversed(self.parts)]) + self._inv = weakref.ref(inv) + inv._inv = weakref.ref(self) + return inv + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return ComposeTransform(self.parts, cache_size=cache_size) + + def __call__(self, x): + for part in self.parts: + x = part(x) + return x + + def log_abs_det_jacobian(self, x, y): + if not self.parts: + return torch.zeros_like(x) + + # Compute intermediates. This will be free if parts[:-1] are all cached. + xs = [x] + for part in self.parts[:-1]: + xs.append(part(xs[-1])) + xs.append(y) + + terms = [] + event_dim = self.domain.event_dim + for part, x, y in zip(self.parts, xs[:-1], xs[1:]): + terms.append( + _sum_rightmost( + part.log_abs_det_jacobian(x, y), event_dim - part.domain.event_dim + ) + ) + event_dim += part.codomain.event_dim - part.domain.event_dim + return functools.reduce(operator.add, terms) + + def forward_shape(self, shape): + for part in self.parts: + shape = part.forward_shape(shape) + return shape + + def inverse_shape(self, shape): + for part in reversed(self.parts): + shape = part.inverse_shape(shape) + return shape + + def __repr__(self): + fmt_string = self.__class__.__name__ + "(\n " + fmt_string += ",\n ".join([p.__repr__() for p in self.parts]) + fmt_string += "\n)" + return fmt_string + + +identity_transform = ComposeTransform([]) + + +class IndependentTransform(Transform): + """ + Wrapper around another transform to treat + ``reinterpreted_batch_ndims``-many extra of the right most dimensions as + dependent. This has no effect on the forward or backward transforms, but + does sum out ``reinterpreted_batch_ndims``-many of the rightmost dimensions + in :meth:`log_abs_det_jacobian`. + + Args: + base_transform (:class:`Transform`): A base transform. + reinterpreted_batch_ndims (int): The number of extra rightmost + dimensions to treat as dependent. + """ + + def __init__( + self, + base_transform: Transform, + reinterpreted_batch_ndims: int, + cache_size: int = 0, + ) -> None: + super().__init__(cache_size=cache_size) + self.base_transform = base_transform.with_cache(cache_size) + self.reinterpreted_batch_ndims = reinterpreted_batch_ndims + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return IndependentTransform( + self.base_transform, self.reinterpreted_batch_ndims, cache_size=cache_size + ) + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def domain(self): + return constraints.independent( + self.base_transform.domain, self.reinterpreted_batch_ndims + ) + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def codomain(self): + return constraints.independent( + self.base_transform.codomain, self.reinterpreted_batch_ndims + ) + + @property + def bijective(self) -> bool: # type: ignore[override] + return self.base_transform.bijective + + @property + def sign(self) -> int: + return self.base_transform.sign + + def _call(self, x): + if x.dim() < self.domain.event_dim: + raise ValueError("Too few dimensions on input") + return self.base_transform(x) + + def _inverse(self, y): + if y.dim() < self.codomain.event_dim: + raise ValueError("Too few dimensions on input") + return self.base_transform.inv(y) + + def log_abs_det_jacobian(self, x, y): + result = self.base_transform.log_abs_det_jacobian(x, y) + result = _sum_rightmost(result, self.reinterpreted_batch_ndims) + return result + + def __repr__(self): + return f"{self.__class__.__name__}({repr(self.base_transform)}, {self.reinterpreted_batch_ndims})" + + def forward_shape(self, shape): + return self.base_transform.forward_shape(shape) + + def inverse_shape(self, shape): + return self.base_transform.inverse_shape(shape) + + +class ReshapeTransform(Transform): + """ + Unit Jacobian transform to reshape the rightmost part of a tensor. + + Note that ``in_shape`` and ``out_shape`` must have the same number of + elements, just as for :meth:`torch.Tensor.reshape`. + + Arguments: + in_shape (torch.Size): The input event shape. + out_shape (torch.Size): The output event shape. + cache_size (int): Size of cache. If zero, no caching is done. If one, + the latest single value is cached. Only 0 and 1 are supported. (Default 0.) + """ + + bijective = True + + def __init__( + self, + in_shape: torch.Size, + out_shape: torch.Size, + cache_size: int = 0, + ) -> None: + self.in_shape = torch.Size(in_shape) + self.out_shape = torch.Size(out_shape) + if self.in_shape.numel() != self.out_shape.numel(): + raise ValueError("in_shape, out_shape have different numbers of elements") + super().__init__(cache_size=cache_size) + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def domain(self): + return constraints.independent(constraints.real, len(self.in_shape)) + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def codomain(self): + return constraints.independent(constraints.real, len(self.out_shape)) + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return ReshapeTransform(self.in_shape, self.out_shape, cache_size=cache_size) + + def _call(self, x): + batch_shape = x.shape[: x.dim() - len(self.in_shape)] + return x.reshape(batch_shape + self.out_shape) + + def _inverse(self, y): + batch_shape = y.shape[: y.dim() - len(self.out_shape)] + return y.reshape(batch_shape + self.in_shape) + + def log_abs_det_jacobian(self, x, y): + batch_shape = x.shape[: x.dim() - len(self.in_shape)] + return x.new_zeros(batch_shape) + + def forward_shape(self, shape): + if len(shape) < len(self.in_shape): + raise ValueError("Too few dimensions on input") + cut = len(shape) - len(self.in_shape) + if shape[cut:] != self.in_shape: + raise ValueError( + f"Shape mismatch: expected {shape[cut:]} but got {self.in_shape}" + ) + return shape[:cut] + self.out_shape + + def inverse_shape(self, shape): + if len(shape) < len(self.out_shape): + raise ValueError("Too few dimensions on input") + cut = len(shape) - len(self.out_shape) + if shape[cut:] != self.out_shape: + raise ValueError( + f"Shape mismatch: expected {shape[cut:]} but got {self.out_shape}" + ) + return shape[:cut] + self.in_shape + + +class ExpTransform(Transform): + r""" + Transform via the mapping :math:`y = \exp(x)`. + """ + + domain = constraints.real + codomain = constraints.positive + bijective = True + sign = +1 + + def __eq__(self, other): + return isinstance(other, ExpTransform) + + def _call(self, x): + return x.exp() + + def _inverse(self, y): + return y.log() + + def log_abs_det_jacobian(self, x, y): + return x + + +class PowerTransform(Transform): + r""" + Transform via the mapping :math:`y = x^{\text{exponent}}`. + """ + + domain = constraints.positive + codomain = constraints.positive + bijective = True + + def __init__(self, exponent: Tensor, cache_size: int = 0) -> None: + super().__init__(cache_size=cache_size) + (self.exponent,) = broadcast_all(exponent) + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return PowerTransform(self.exponent, cache_size=cache_size) + + @lazy_property + def sign(self) -> int: # type: ignore[override] + return self.exponent.sign() # type: ignore[return-value] + + def __eq__(self, other): + if not isinstance(other, PowerTransform): + return False + return self.exponent.eq(other.exponent).all().item() + + def _call(self, x): + return x.pow(self.exponent) + + def _inverse(self, y): + return y.pow(1 / self.exponent) + + def log_abs_det_jacobian(self, x, y): + return (self.exponent * y / x).abs().log() + + def forward_shape(self, shape): + return torch.broadcast_shapes(shape, getattr(self.exponent, "shape", ())) + + def inverse_shape(self, shape): + return torch.broadcast_shapes(shape, getattr(self.exponent, "shape", ())) + + +def _clipped_sigmoid(x): + finfo = torch.finfo(x.dtype) + return torch.clamp(torch.sigmoid(x), min=finfo.tiny, max=1.0 - finfo.eps) + + +class SigmoidTransform(Transform): + r""" + Transform via the mapping :math:`y = \frac{1}{1 + \exp(-x)}` and :math:`x = \text{logit}(y)`. + """ + + domain = constraints.real + codomain = constraints.unit_interval + bijective = True + sign = +1 + + def __eq__(self, other): + return isinstance(other, SigmoidTransform) + + def _call(self, x): + return _clipped_sigmoid(x) + + def _inverse(self, y): + finfo = torch.finfo(y.dtype) + y = y.clamp(min=finfo.tiny, max=1.0 - finfo.eps) + return y.log() - (-y).log1p() + + def log_abs_det_jacobian(self, x, y): + return -F.softplus(-x) - F.softplus(x) + + +class SoftplusTransform(Transform): + r""" + Transform via the mapping :math:`\text{Softplus}(x) = \log(1 + \exp(x))`. + The implementation reverts to the linear function when :math:`x > 20`. + """ + + domain = constraints.real + codomain = constraints.positive + bijective = True + sign = +1 + + def __eq__(self, other): + return isinstance(other, SoftplusTransform) + + def _call(self, x): + return softplus(x) + + def _inverse(self, y): + return (-y).expm1().neg().log() + y + + def log_abs_det_jacobian(self, x, y): + return -softplus(-x) + + +class TanhTransform(Transform): + r""" + Transform via the mapping :math:`y = \tanh(x)`. + + It is equivalent to + + .. code-block:: python + + ComposeTransform( + [ + AffineTransform(0.0, 2.0), + SigmoidTransform(), + AffineTransform(-1.0, 2.0), + ] + ) + + However this might not be numerically stable, thus it is recommended to use `TanhTransform` + instead. + + Note that one should use `cache_size=1` when it comes to `NaN/Inf` values. + + """ + + domain = constraints.real + codomain = constraints.interval(-1.0, 1.0) + bijective = True + sign = +1 + + def __eq__(self, other): + return isinstance(other, TanhTransform) + + def _call(self, x): + return x.tanh() + + def _inverse(self, y): + # We do not clamp to the boundary here as it may degrade the performance of certain algorithms. + # one should use `cache_size=1` instead + return torch.atanh(y) + + def log_abs_det_jacobian(self, x, y): + # We use a formula that is more numerically stable, see details in the following link + # https://github.com/tensorflow/probability/blob/master/tensorflow_probability/python/bijectors/tanh.py#L69-L80 + return 2.0 * (math.log(2.0) - x - softplus(-2.0 * x)) + + +class AbsTransform(Transform): + r"""Transform via the mapping :math:`y = |x|`.""" + + domain = constraints.real + codomain = constraints.positive + + def __eq__(self, other): + return isinstance(other, AbsTransform) + + def _call(self, x): + return x.abs() + + def _inverse(self, y): + return y + + +class AffineTransform(Transform): + r""" + Transform via the pointwise affine mapping :math:`y = \text{loc} + \text{scale} \times x`. + + Args: + loc (Tensor or float): Location parameter. + scale (Tensor or float): Scale parameter. + event_dim (int): Optional size of `event_shape`. This should be zero + for univariate random variables, 1 for distributions over vectors, + 2 for distributions over matrices, etc. + """ + + bijective = True + + def __init__( + self, + loc: Union[Tensor, float], + scale: Union[Tensor, float], + event_dim: int = 0, + cache_size: int = 0, + ) -> None: + super().__init__(cache_size=cache_size) + self.loc = loc + self.scale = scale + self._event_dim = event_dim + + @property + def event_dim(self) -> int: + return self._event_dim + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def domain(self): + if self.event_dim == 0: + return constraints.real + return constraints.independent(constraints.real, self.event_dim) + + @constraints.dependent_property(is_discrete=False) + # pyrefly: ignore [bad-override] + def codomain(self): + if self.event_dim == 0: + return constraints.real + return constraints.independent(constraints.real, self.event_dim) + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return AffineTransform( + self.loc, self.scale, self.event_dim, cache_size=cache_size + ) + + def __eq__(self, other): + if not isinstance(other, AffineTransform): + return False + + if isinstance(self.loc, _Number) and isinstance(other.loc, _Number): + if self.loc != other.loc: + return False + else: + if not (self.loc == other.loc).all().item(): # type: ignore[union-attr] + return False + + if isinstance(self.scale, _Number) and isinstance(other.scale, _Number): + if self.scale != other.scale: + return False + else: + if not (self.scale == other.scale).all().item(): # type: ignore[union-attr] + return False + + return True + + @property + def sign(self) -> Union[Tensor, int]: # type: ignore[override] + if isinstance(self.scale, _Number): + return 1 if float(self.scale) > 0 else -1 if float(self.scale) < 0 else 0 + return self.scale.sign() + + def _call(self, x): + return self.loc + self.scale * x + + def _inverse(self, y): + return (y - self.loc) / self.scale + + def log_abs_det_jacobian(self, x, y): + shape = x.shape + scale = self.scale + if isinstance(scale, _Number): + result = torch.full_like(x, math.log(abs(scale))) + else: + result = torch.abs(scale).log() + if self.event_dim: + result_size = result.size()[: -self.event_dim] + (-1,) + result = result.view(result_size).sum(-1) + shape = shape[: -self.event_dim] + return result.expand(shape) + + def forward_shape(self, shape): + return torch.broadcast_shapes( + shape, getattr(self.loc, "shape", ()), getattr(self.scale, "shape", ()) + ) + + def inverse_shape(self, shape): + return torch.broadcast_shapes( + shape, getattr(self.loc, "shape", ()), getattr(self.scale, "shape", ()) + ) + + +class CorrCholeskyTransform(Transform): + r""" + Transforms an unconstrained real vector :math:`x` with length :math:`D*(D-1)/2` into the + Cholesky factor of a D-dimension correlation matrix. This Cholesky factor is a lower + triangular matrix with positive diagonals and unit Euclidean norm for each row. + The transform is processed as follows: + + 1. First we convert x into a lower triangular matrix in row order. + 2. For each row :math:`X_i` of the lower triangular part, we apply a *signed* version of + class :class:`StickBreakingTransform` to transform :math:`X_i` into a + unit Euclidean length vector using the following steps: + - Scales into the interval :math:`(-1, 1)` domain: :math:`r_i = \tanh(X_i)`. + - Transforms into an unsigned domain: :math:`z_i = r_i^2`. + - Applies :math:`s_i = StickBreakingTransform(z_i)`. + - Transforms back into signed domain: :math:`y_i = sign(r_i) * \sqrt{s_i}`. + """ + + domain = constraints.real_vector + codomain = constraints.corr_cholesky + bijective = True + + def _call(self, x): + x = torch.tanh(x) + eps = torch.finfo(x.dtype).eps + x = x.clamp(min=-1 + eps, max=1 - eps) + r = vec_to_tril_matrix(x, diag=-1) + # apply stick-breaking on the squared values + # Note that y = sign(r) * sqrt(z * z1m_cumprod) + # = (sign(r) * sqrt(z)) * sqrt(z1m_cumprod) = r * sqrt(z1m_cumprod) + # pyrefly: ignore [unsupported-operation] + z = r**2 + z1m_cumprod_sqrt = (1 - z).sqrt().cumprod(-1) + # Diagonal elements must be 1. + r = r + torch.eye(r.shape[-1], dtype=r.dtype, device=r.device) + y = r * pad(z1m_cumprod_sqrt[..., :-1], [1, 0], value=1) + return y + + def _inverse(self, y): + # inverse stick-breaking + # See: https://mc-stan.org/docs/2_18/reference-manual/cholesky-factors-of-correlation-matrices-1.html + y_cumsum = 1 - torch.cumsum(y * y, dim=-1) + y_cumsum_shifted = pad(y_cumsum[..., :-1], [1, 0], value=1) + y_vec = tril_matrix_to_vec(y, diag=-1) + y_cumsum_vec = tril_matrix_to_vec(y_cumsum_shifted, diag=-1) + t = y_vec / (y_cumsum_vec).sqrt() + # inverse of tanh + x = (t.log1p() - t.neg().log1p()) / 2 + return x + + def log_abs_det_jacobian(self, x, y, intermediates=None): + # Because domain and codomain are two spaces with different dimensions, determinant of + # Jacobian is not well-defined. We return `log_abs_det_jacobian` of `x` and the + # flattened lower triangular part of `y`. + + # See: https://mc-stan.org/docs/2_18/reference-manual/cholesky-factors-of-correlation-matrices-1.html + y1m_cumsum = 1 - (y * y).cumsum(dim=-1) + # by taking diagonal=-2, we don't need to shift z_cumprod to the right + # also works for 2 x 2 matrix + y1m_cumsum_tril = tril_matrix_to_vec(y1m_cumsum, diag=-2) + stick_breaking_logdet = 0.5 * (y1m_cumsum_tril).log().sum(-1) + tanh_logdet = -2 * (x + softplus(-2 * x) - math.log(2.0)).sum(dim=-1) + return stick_breaking_logdet + tanh_logdet + + def forward_shape(self, shape): + # Reshape from (..., N) to (..., D, D). + if len(shape) < 1: + raise ValueError("Too few dimensions on input") + N = shape[-1] + D = round((0.25 + 2 * N) ** 0.5 + 0.5) + if D * (D - 1) // 2 != N: + raise ValueError("Input is not a flattened lower-diagonal number") + return shape[:-1] + (D, D) + + def inverse_shape(self, shape): + # Reshape from (..., D, D) to (..., N). + if len(shape) < 2: + raise ValueError("Too few dimensions on input") + if shape[-2] != shape[-1]: + raise ValueError("Input is not square") + D = shape[-1] + N = D * (D - 1) // 2 + return shape[:-2] + (N,) + + +class SoftmaxTransform(Transform): + r""" + Transform from unconstrained space to the simplex via :math:`y = \exp(x)` then + normalizing. + + This is not bijective and cannot be used for HMC. However this acts mostly + coordinate-wise (except for the final normalization), and thus is + appropriate for coordinate-wise optimization algorithms. + """ + + domain = constraints.real_vector + codomain = constraints.simplex + + def __eq__(self, other): + return isinstance(other, SoftmaxTransform) + + def _call(self, x): + logprobs = x + probs = (logprobs - logprobs.max(-1, True)[0]).exp() + return probs / probs.sum(-1, True) + + def _inverse(self, y): + probs = y + return probs.log() + + def forward_shape(self, shape): + if len(shape) < 1: + raise ValueError("Too few dimensions on input") + return shape + + def inverse_shape(self, shape): + if len(shape) < 1: + raise ValueError("Too few dimensions on input") + return shape + + +class StickBreakingTransform(Transform): + """ + Transform from unconstrained space to the simplex of one additional + dimension via a stick-breaking process. + + This transform arises as an iterated sigmoid transform in a stick-breaking + construction of the `Dirichlet` distribution: the first logit is + transformed via sigmoid to the first probability and the probability of + everything else, and then the process recurses. + + This is bijective and appropriate for use in HMC; however it mixes + coordinates together and is less appropriate for optimization. + """ + + domain = constraints.real_vector + codomain = constraints.simplex + bijective = True + + def __eq__(self, other): + return isinstance(other, StickBreakingTransform) + + def _call(self, x): + offset = x.shape[-1] + 1 - x.new_ones(x.shape[-1]).cumsum(-1) + z = _clipped_sigmoid(x - offset.log()) + z_cumprod = (1 - z).cumprod(-1) + y = pad(z, [0, 1], value=1) * pad(z_cumprod, [1, 0], value=1) + return y + + def _inverse(self, y): + y_crop = y[..., :-1] + offset = y.shape[-1] - y.new_ones(y_crop.shape[-1]).cumsum(-1) + sf = 1 - y_crop.cumsum(-1) + # we clamp to make sure that sf is positive which sometimes does not + # happen when y[-1] ~ 0 or y[:-1].sum() ~ 1 + sf = torch.clamp(sf, min=torch.finfo(y.dtype).tiny) + x = y_crop.log() - sf.log() + offset.log() + return x + + def log_abs_det_jacobian(self, x, y): + offset = x.shape[-1] + 1 - x.new_ones(x.shape[-1]).cumsum(-1) + x = x - offset.log() + # use the identity 1 - sigmoid(x) = exp(-x) * sigmoid(x) + detJ = (-x + F.logsigmoid(x) + y[..., :-1].log()).sum(-1) + return detJ + + def forward_shape(self, shape): + if len(shape) < 1: + raise ValueError("Too few dimensions on input") + return shape[:-1] + (shape[-1] + 1,) + + def inverse_shape(self, shape): + if len(shape) < 1: + raise ValueError("Too few dimensions on input") + return shape[:-1] + (shape[-1] - 1,) + + +class LowerCholeskyTransform(Transform): + """ + Transform from unconstrained matrices to lower-triangular matrices with + nonnegative diagonal entries. + + This is useful for parameterizing positive definite matrices in terms of + their Cholesky factorization. + """ + + domain = constraints.independent(constraints.real, 2) + codomain = constraints.lower_cholesky + + def __eq__(self, other): + return isinstance(other, LowerCholeskyTransform) + + def _call(self, x): + return x.tril(-1) + x.diagonal(dim1=-2, dim2=-1).exp().diag_embed() + + def _inverse(self, y): + return y.tril(-1) + y.diagonal(dim1=-2, dim2=-1).log().diag_embed() + + +class PositiveDefiniteTransform(Transform): + """ + Transform from unconstrained matrices to positive-definite matrices. + """ + + domain = constraints.independent(constraints.real, 2) + codomain = constraints.positive_definite + + def __eq__(self, other): + return isinstance(other, PositiveDefiniteTransform) + + def _call(self, x): + x = LowerCholeskyTransform()(x) + return x @ x.mT + + def _inverse(self, y): + y = torch.linalg.cholesky(y) + return LowerCholeskyTransform().inv(y) + + +class CatTransform(Transform): + """ + Transform functor that applies a sequence of transforms `tseq` + component-wise to each submatrix at `dim`, of length `lengths[dim]`, + in a way compatible with :func:`torch.cat`. + + Example:: + + x0 = torch.cat([torch.range(1, 10), torch.range(1, 10)], dim=0) + x = torch.cat([x0, x0], dim=0) + t0 = CatTransform([ExpTransform(), identity_transform], dim=0, lengths=[10, 10]) + t = CatTransform([t0, t0], dim=0, lengths=[20, 20]) + y = t(x) + """ + + transforms: list[Transform] + + def __init__( + self, + tseq: Sequence[Transform], + dim: int = 0, + lengths: Optional[Sequence[int]] = None, + cache_size: int = 0, + ) -> None: + assert all(isinstance(t, Transform) for t in tseq) + if cache_size: + tseq = [t.with_cache(cache_size) for t in tseq] + super().__init__(cache_size=cache_size) + self.transforms = list(tseq) + if lengths is None: + lengths = [1] * len(self.transforms) + self.lengths = list(lengths) + assert len(self.lengths) == len(self.transforms) + self.dim = dim + + @lazy_property + def event_dim(self) -> int: # type: ignore[override] + return max(t.event_dim for t in self.transforms) + + @lazy_property + def length(self) -> int: + return sum(self.lengths) + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return CatTransform(self.transforms, self.dim, self.lengths, cache_size) + + def _call(self, x): + assert -x.dim() <= self.dim < x.dim() + assert x.size(self.dim) == self.length + yslices = [] + start = 0 + for trans, length in zip(self.transforms, self.lengths): + xslice = x.narrow(self.dim, start, length) + yslices.append(trans(xslice)) + start = start + length # avoid += for jit compat + return torch.cat(yslices, dim=self.dim) + + def _inverse(self, y): + assert -y.dim() <= self.dim < y.dim() + assert y.size(self.dim) == self.length + xslices = [] + start = 0 + for trans, length in zip(self.transforms, self.lengths): + yslice = y.narrow(self.dim, start, length) + xslices.append(trans.inv(yslice)) + start = start + length # avoid += for jit compat + return torch.cat(xslices, dim=self.dim) + + def log_abs_det_jacobian(self, x, y): + assert -x.dim() <= self.dim < x.dim() + assert x.size(self.dim) == self.length + assert -y.dim() <= self.dim < y.dim() + assert y.size(self.dim) == self.length + logdetjacs = [] + start = 0 + for trans, length in zip(self.transforms, self.lengths): + xslice = x.narrow(self.dim, start, length) + yslice = y.narrow(self.dim, start, length) + logdetjac = trans.log_abs_det_jacobian(xslice, yslice) + if trans.event_dim < self.event_dim: + logdetjac = _sum_rightmost(logdetjac, self.event_dim - trans.event_dim) + logdetjacs.append(logdetjac) + start = start + length # avoid += for jit compat + # Decide whether to concatenate or sum. + dim = self.dim + if dim >= 0: + dim = dim - x.dim() + dim = dim + self.event_dim + if dim < 0: + return torch.cat(logdetjacs, dim=dim) + else: + return sum(logdetjacs) + + @property + def bijective(self) -> bool: # type: ignore[override] + return all(t.bijective for t in self.transforms) + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def domain(self): + return constraints.cat( + [t.domain for t in self.transforms], self.dim, self.lengths + ) + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def codomain(self): + return constraints.cat( + [t.codomain for t in self.transforms], self.dim, self.lengths + ) + + +class StackTransform(Transform): + """ + Transform functor that applies a sequence of transforms `tseq` + component-wise to each submatrix at `dim` + in a way compatible with :func:`torch.stack`. + + Example:: + + x = torch.stack([torch.range(1, 10), torch.range(1, 10)], dim=1) + t = StackTransform([ExpTransform(), identity_transform], dim=1) + y = t(x) + """ + + transforms: list[Transform] + + def __init__( + self, tseq: Sequence[Transform], dim: int = 0, cache_size: int = 0 + ) -> None: + assert all(isinstance(t, Transform) for t in tseq) + if cache_size: + tseq = [t.with_cache(cache_size) for t in tseq] + super().__init__(cache_size=cache_size) + self.transforms = list(tseq) + self.dim = dim + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return StackTransform(self.transforms, self.dim, cache_size) + + def _slice(self, z): + return [z.select(self.dim, i) for i in range(z.size(self.dim))] + + def _call(self, x): + assert -x.dim() <= self.dim < x.dim() + assert x.size(self.dim) == len(self.transforms) + yslices = [] + for xslice, trans in zip(self._slice(x), self.transforms): + yslices.append(trans(xslice)) + return torch.stack(yslices, dim=self.dim) + + def _inverse(self, y): + assert -y.dim() <= self.dim < y.dim() + assert y.size(self.dim) == len(self.transforms) + xslices = [] + for yslice, trans in zip(self._slice(y), self.transforms): + xslices.append(trans.inv(yslice)) + return torch.stack(xslices, dim=self.dim) + + def log_abs_det_jacobian(self, x, y): + assert -x.dim() <= self.dim < x.dim() + assert x.size(self.dim) == len(self.transforms) + assert -y.dim() <= self.dim < y.dim() + assert y.size(self.dim) == len(self.transforms) + logdetjacs = [] + yslices = self._slice(y) + xslices = self._slice(x) + for xslice, yslice, trans in zip(xslices, yslices, self.transforms): + logdetjacs.append(trans.log_abs_det_jacobian(xslice, yslice)) + return torch.stack(logdetjacs, dim=self.dim) + + @property + def bijective(self) -> bool: # type: ignore[override] + return all(t.bijective for t in self.transforms) + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def domain(self): + return constraints.stack([t.domain for t in self.transforms], self.dim) + + @constraints.dependent_property + # pyrefly: ignore [bad-override] + def codomain(self): + return constraints.stack([t.codomain for t in self.transforms], self.dim) + + +class CumulativeDistributionTransform(Transform): + """ + Transform via the cumulative distribution function of a probability distribution. + + Args: + distribution (Distribution): Distribution whose cumulative distribution function to use for + the transformation. + + Example:: + + # Construct a Gaussian copula from a multivariate normal. + base_dist = MultivariateNormal( + loc=torch.zeros(2), + scale_tril=LKJCholesky(2).sample(), + ) + transform = CumulativeDistributionTransform(Normal(0, 1)) + copula = TransformedDistribution(base_dist, [transform]) + """ + + bijective = True + codomain = constraints.unit_interval + sign = +1 + + def __init__(self, distribution: Distribution, cache_size: int = 0) -> None: + super().__init__(cache_size=cache_size) + self.distribution = distribution + + @property + def domain(self) -> Optional[constraints.Constraint]: # type: ignore[override] + return self.distribution.support + + def _call(self, x): + return self.distribution.cdf(x) + + def _inverse(self, y): + return self.distribution.icdf(y) + + def log_abs_det_jacobian(self, x, y): + return self.distribution.log_prob(x) + + def with_cache(self, cache_size=1): + if self._cache_size == cache_size: + return self + return CumulativeDistributionTransform(self.distribution, cache_size=cache_size) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/uniform.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/uniform.py new file mode 100644 index 0000000000000000000000000000000000000000..bd9c68ca15fb8bd11f540f79062f3c419adb8910 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/uniform.py @@ -0,0 +1,109 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import nan, Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import broadcast_all +from torch.types import _Number, _size + + +__all__ = ["Uniform"] + + +class Uniform(Distribution): + r""" + Generates uniformly distributed random samples from the half-open interval + ``[low, high)``. + + Example:: + + >>> m = Uniform(torch.tensor([0.0]), torch.tensor([5.0])) + >>> m.sample() # uniformly distributed in the range [0.0, 5.0) + >>> # xdoctest: +SKIP + tensor([ 2.3418]) + + Args: + low (float or Tensor): lower range (inclusive). + high (float or Tensor): upper range (exclusive). + """ + + has_rsample = True + + @property + def arg_constraints(self): + # TODO allow (loc,scale) parameterization to allow independent constraints. + return { + "low": constraints.less_than(self.high), + "high": constraints.greater_than(self.low), + } + + @property + def mean(self) -> Tensor: + return (self.high + self.low) / 2 + + @property + def mode(self) -> Tensor: + return nan * self.high + + @property + def stddev(self) -> Tensor: + return (self.high - self.low) / 12**0.5 + + @property + def variance(self) -> Tensor: + return (self.high - self.low).pow(2) / 12 + + def __init__( + self, + low: Union[Tensor, float], + high: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.low, self.high = broadcast_all(low, high) + + if isinstance(low, _Number) and isinstance(high, _Number): + batch_shape = torch.Size() + else: + batch_shape = self.low.size() + super().__init__(batch_shape, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Uniform, _instance) + batch_shape = torch.Size(batch_shape) + new.low = self.low.expand(batch_shape) + new.high = self.high.expand(batch_shape) + super(Uniform, new).__init__(batch_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + @constraints.dependent_property(is_discrete=False, event_dim=0) + # pyrefly: ignore [bad-override] + def support(self): + return constraints.interval(self.low, self.high) + + def rsample(self, sample_shape: _size = torch.Size()) -> Tensor: + shape = self._extended_shape(sample_shape) + rand = torch.rand(shape, dtype=self.low.dtype, device=self.low.device) + return self.low + rand * (self.high - self.low) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + lb = self.low.le(value).type_as(self.low) + ub = self.high.gt(value).type_as(self.low) + return torch.log(lb.mul(ub)) - torch.log(self.high - self.low) + + def cdf(self, value): + if self._validate_args: + self._validate_sample(value) + result = (value - self.low) / (self.high - self.low) + return result.clamp(min=0, max=1) + + def icdf(self, value): + result = value * (self.high - self.low) + self.low + return result + + def entropy(self): + return torch.log(self.high - self.low) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..a5afc5395ee7815d629ce0fc5382fb05cc379a40 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/utils.py @@ -0,0 +1,221 @@ +from collections.abc import Callable, Sequence +from functools import update_wrapper +from typing import Any, Final, Generic, Optional, overload, TypeVar, Union + +import torch +import torch.nn.functional as F +from torch import SymInt, Tensor +from torch.overrides import is_tensor_like +from torch.types import _dtype, _Number, Device, Number + + +euler_constant: Final[float] = 0.57721566490153286060 # Euler Mascheroni Constant + +__all__ = [ + "broadcast_all", + "logits_to_probs", + "clamp_probs", + "probs_to_logits", + "lazy_property", + "tril_matrix_to_vec", + "vec_to_tril_matrix", +] + + +# FIXME: Use (*values: *Ts) -> tuple[Tensor for T in Ts] if Mapping-Type is ever added. +# See https://github.com/python/typing/issues/1216#issuecomment-2126153831 +def broadcast_all(*values: Union[Tensor, Number]) -> tuple[Tensor, ...]: + r""" + Given a list of values (possibly containing numbers), returns a list where each + value is broadcasted based on the following rules: + - `torch.*Tensor` instances are broadcasted as per :ref:`_broadcasting-semantics`. + - Number instances (scalars) are upcast to tensors having + the same size and type as the first tensor passed to `values`. If all the + values are scalars, then they are upcasted to scalar Tensors. + + Args: + values (list of `Number`, `torch.*Tensor` or objects implementing __torch_function__) + + Raises: + ValueError: if any of the values is not a `Number` instance, + a `torch.*Tensor` instance, or an instance implementing __torch_function__ + """ + if not all(is_tensor_like(v) or isinstance(v, _Number) for v in values): + raise ValueError( + "Input arguments must all be instances of Number, " + "torch.Tensor or objects implementing __torch_function__." + ) + if not all(is_tensor_like(v) for v in values): + options: dict[str, Any] = dict(dtype=torch.get_default_dtype()) + for value in values: + if isinstance(value, torch.Tensor): + options = dict(dtype=value.dtype, device=value.device) + break + new_values = [ + v if is_tensor_like(v) else torch.tensor(v, **options) for v in values + ] + return torch.broadcast_tensors(*new_values) + return torch.broadcast_tensors(*values) + + +def _standard_normal( + shape: Sequence[Union[int, SymInt]], + dtype: Optional[_dtype], + device: Optional[Device], +) -> Tensor: + if torch._C._get_tracing_state(): + # [JIT WORKAROUND] lack of support for .normal_() + return torch.normal( + torch.zeros(shape, dtype=dtype, device=device), + torch.ones(shape, dtype=dtype, device=device), + ) + return torch.empty(shape, dtype=dtype, device=device).normal_() + + +def _sum_rightmost(value: Tensor, dim: int) -> Tensor: + r""" + Sum out ``dim`` many rightmost dimensions of a given tensor. + + Args: + value (Tensor): A tensor of ``.dim()`` at least ``dim``. + dim (int): The number of rightmost dims to sum out. + """ + if dim == 0: + return value + required_shape = value.shape[:-dim] + (-1,) + return value.reshape(required_shape).sum(-1) + + +def logits_to_probs(logits: Tensor, is_binary: bool = False) -> Tensor: + r""" + Converts a tensor of logits into probabilities. Note that for the + binary case, each value denotes log odds, whereas for the + multi-dimensional case, the values along the last dimension denote + the log probabilities (possibly unnormalized) of the events. + """ + if is_binary: + return torch.sigmoid(logits) + return F.softmax(logits, dim=-1) + + +def clamp_probs(probs: Tensor) -> Tensor: + """Clamps the probabilities to be in the open interval `(0, 1)`. + + The probabilities would be clamped between `eps` and `1 - eps`, + and `eps` would be the smallest representable positive number for the input data type. + + Args: + probs (Tensor): A tensor of probabilities. + + Returns: + Tensor: The clamped probabilities. + + Examples: + >>> probs = torch.tensor([0.0, 0.5, 1.0]) + >>> clamp_probs(probs) + tensor([1.1921e-07, 5.0000e-01, 1.0000e+00]) + + >>> probs = torch.tensor([0.0, 0.5, 1.0], dtype=torch.float64) + >>> clamp_probs(probs) + tensor([2.2204e-16, 5.0000e-01, 1.0000e+00], dtype=torch.float64) + + """ + eps = torch.finfo(probs.dtype).eps + return probs.clamp(min=eps, max=1 - eps) + + +def probs_to_logits(probs: Tensor, is_binary: bool = False) -> Tensor: + r""" + Converts a tensor of probabilities into logits. For the binary case, + this denotes the probability of occurrence of the event indexed by `1`. + For the multi-dimensional case, the values along the last dimension + denote the probabilities of occurrence of each of the events. + """ + ps_clamped = clamp_probs(probs) + if is_binary: + return torch.log(ps_clamped) - torch.log1p(-ps_clamped) + return torch.log(ps_clamped) + + +T = TypeVar("T", contravariant=True) +R = TypeVar("R", covariant=True) + + +class lazy_property(Generic[T, R]): + r""" + Used as a decorator for lazy loading of class attributes. This uses a + non-data descriptor that calls the wrapped method to compute the property on + first call; thereafter replacing the wrapped method into an instance + attribute. + """ + + def __init__(self, wrapped: Callable[[T], R]) -> None: + self.wrapped: Callable[[T], R] = wrapped + update_wrapper(self, wrapped) # type:ignore[arg-type] + + @overload + def __get__( + self, instance: None, obj_type: Any = None + ) -> "_lazy_property_and_property[T, R]": ... + + @overload + def __get__(self, instance: T, obj_type: Any = None) -> R: ... + + def __get__( + self, instance: Union[T, None], obj_type: Any = None + ) -> "R | _lazy_property_and_property[T, R]": + if instance is None: + return _lazy_property_and_property(self.wrapped) + with torch.enable_grad(): + value = self.wrapped(instance) + setattr(instance, self.wrapped.__name__, value) + return value + + +class _lazy_property_and_property(lazy_property[T, R], property): + """We want lazy properties to look like multiple things. + + * property when Sphinx autodoc looks + * lazy_property when Distribution validate_args looks + """ + + def __init__(self, wrapped: Callable[[T], R]) -> None: + property.__init__(self, wrapped) + + +def tril_matrix_to_vec(mat: Tensor, diag: int = 0) -> Tensor: + r""" + Convert a `D x D` matrix or a batch of matrices into a (batched) vector + which comprises of lower triangular elements from the matrix in row order. + """ + n = mat.shape[-1] + if not torch._C._get_tracing_state() and (diag < -n or diag >= n): + raise ValueError(f"diag ({diag}) provided is outside [{-n}, {n - 1}].") + arange = torch.arange(n, device=mat.device) + tril_mask = arange < arange.view(-1, 1) + (diag + 1) + vec = mat[..., tril_mask] + return vec + + +def vec_to_tril_matrix(vec: Tensor, diag: int = 0) -> Tensor: + r""" + Convert a vector or a batch of vectors into a batched `D x D` + lower triangular matrix containing elements from the vector in row order. + """ + # +ve root of D**2 + (1+2*diag)*D - |diag| * (diag+1) - 2*vec.shape[-1] = 0 + n = ( + -(1 + 2 * diag) + + ((1 + 2 * diag) ** 2 + 8 * vec.shape[-1] + 4 * abs(diag) * (diag + 1)) ** 0.5 + ) / 2 + eps = torch.finfo(vec.dtype).eps + if not torch._C._get_tracing_state() and (round(n) - n > eps): + raise ValueError( + f"The size of last dimension is {vec.shape[-1]} which cannot be expressed as " + + "the lower triangular part of a square D x D matrix." + ) + n = round(n.item()) if isinstance(n, torch.Tensor) else round(n) + mat = vec.new_zeros(vec.shape[:-1] + torch.Size((n, n))) + arange = torch.arange(n, device=vec.device) + tril_mask = arange < arange.view(-1, 1) + (diag + 1) + mat[..., tril_mask] = vec + return mat diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/von_mises.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/von_mises.py new file mode 100644 index 0000000000000000000000000000000000000000..47588ffedd0b57f14b0fef6aca0346f9042d7bc8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/von_mises.py @@ -0,0 +1,223 @@ +# mypy: allow-untyped-defs +import math +from typing import Optional + +import torch +import torch.jit +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.distribution import Distribution +from torch.distributions.utils import broadcast_all, lazy_property + + +__all__ = ["VonMises"] + + +def _eval_poly(y, coef): + coef = list(coef) + result = coef.pop() + while coef: + result = coef.pop() + y * result + return result + + +_I0_COEF_SMALL = [ + 1.0, + 3.5156229, + 3.0899424, + 1.2067492, + 0.2659732, + 0.360768e-1, + 0.45813e-2, +] +_I0_COEF_LARGE = [ + 0.39894228, + 0.1328592e-1, + 0.225319e-2, + -0.157565e-2, + 0.916281e-2, + -0.2057706e-1, + 0.2635537e-1, + -0.1647633e-1, + 0.392377e-2, +] +_I1_COEF_SMALL = [ + 0.5, + 0.87890594, + 0.51498869, + 0.15084934, + 0.2658733e-1, + 0.301532e-2, + 0.32411e-3, +] +_I1_COEF_LARGE = [ + 0.39894228, + -0.3988024e-1, + -0.362018e-2, + 0.163801e-2, + -0.1031555e-1, + 0.2282967e-1, + -0.2895312e-1, + 0.1787654e-1, + -0.420059e-2, +] + +_COEF_SMALL = [_I0_COEF_SMALL, _I1_COEF_SMALL] +_COEF_LARGE = [_I0_COEF_LARGE, _I1_COEF_LARGE] + + +def _log_modified_bessel_fn(x, order=0): + """ + Returns ``log(I_order(x))`` for ``x > 0``, + where `order` is either 0 or 1. + """ + assert order == 0 or order == 1 + + # compute small solution + y = x / 3.75 + y = y * y + small = _eval_poly(y, _COEF_SMALL[order]) + if order == 1: + small = x.abs() * small + small = small.log() + + # compute large solution + y = 3.75 / x + large = x - 0.5 * x.log() + _eval_poly(y, _COEF_LARGE[order]).log() + + result = torch.where(x < 3.75, small, large) + return result + + +@torch.jit.script_if_tracing +def _rejection_sample(loc, concentration, proposal_r, x): + done = torch.zeros(x.shape, dtype=torch.bool, device=loc.device) + # pyrefly: ignore [bad-assignment] + while not done.all(): + u = torch.rand((3,) + x.shape, dtype=loc.dtype, device=loc.device) + u1, u2, u3 = u.unbind() + z = torch.cos(math.pi * u1) + f = (1 + proposal_r * z) / (proposal_r + z) + c = concentration * (proposal_r - f) + accept = ((c * (2 - c) - u2) > 0) | ((c / u2).log() + 1 - c >= 0) + if accept.any(): + # pyrefly: ignore [no-matching-overload] + x = torch.where(accept, (u3 - 0.5).sign() * f.acos(), x) + done = done | accept + return (x + math.pi + loc) % (2 * math.pi) - math.pi + + +class VonMises(Distribution): + """ + A circular von Mises distribution. + + This implementation uses polar coordinates. The ``loc`` and ``value`` args + can be any real number (to facilitate unconstrained optimization), but are + interpreted as angles modulo 2 pi. + + Example:: + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = VonMises(torch.tensor([1.0]), torch.tensor([1.0])) + >>> m.sample() # von Mises distributed with loc=1 and concentration=1 + tensor([1.9777]) + + :param torch.Tensor loc: an angle in radians. + :param torch.Tensor concentration: concentration parameter + """ + + # pyrefly: ignore [bad-override] + arg_constraints = {"loc": constraints.real, "concentration": constraints.positive} + support = constraints.real + has_rsample = False + + def __init__( + self, + loc: Tensor, + concentration: Tensor, + validate_args: Optional[bool] = None, + ) -> None: + self.loc, self.concentration = broadcast_all(loc, concentration) + batch_shape = self.loc.shape + event_shape = torch.Size() + super().__init__(batch_shape, event_shape, validate_args) + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + log_prob = self.concentration * torch.cos(value - self.loc) + log_prob = ( + log_prob + - math.log(2 * math.pi) + - _log_modified_bessel_fn(self.concentration, order=0) + ) + return log_prob + + @lazy_property + def _loc(self) -> Tensor: + return self.loc.to(torch.double) + + @lazy_property + def _concentration(self) -> Tensor: + return self.concentration.to(torch.double) + + @lazy_property + def _proposal_r(self) -> Tensor: + kappa = self._concentration + # pyrefly: ignore [unsupported-operation] + tau = 1 + (1 + 4 * kappa**2).sqrt() + rho = (tau - (2 * tau).sqrt()) / (2 * kappa) + # pyrefly: ignore [unsupported-operation] + _proposal_r = (1 + rho**2) / (2 * rho) + # second order Taylor expansion around 0 for small kappa + _proposal_r_taylor = 1 / kappa + kappa + return torch.where(kappa < 1e-5, _proposal_r_taylor, _proposal_r) + + @torch.no_grad() + def sample(self, sample_shape=torch.Size()): + """ + The sampling algorithm for the von Mises distribution is based on the + following paper: D.J. Best and N.I. Fisher, "Efficient simulation of the + von Mises distribution." Applied Statistics (1979): 152-157. + + Sampling is always done in double precision internally to avoid a hang + in _rejection_sample() for small values of the concentration, which + starts to happen for single precision around 1e-4 (see issue #88443). + """ + shape = self._extended_shape(sample_shape) + x = torch.empty(shape, dtype=self._loc.dtype, device=self.loc.device) + return _rejection_sample( + self._loc, self._concentration, self._proposal_r, x + ).to(self.loc.dtype) + + def expand(self, batch_shape, _instance=None): + try: + return super().expand(batch_shape) + except NotImplementedError: + validate_args = self.__dict__.get("_validate_args") + loc = self.loc.expand(batch_shape) + concentration = self.concentration.expand(batch_shape) + return type(self)(loc, concentration, validate_args=validate_args) + + @property + def mean(self) -> Tensor: + """ + The provided mean is the circular one. + """ + return self.loc + + @property + def mode(self) -> Tensor: + return self.loc + + @lazy_property + def variance(self) -> Tensor: # type: ignore[override] + """ + The provided variance is the circular one. + """ + return ( + 1 + - ( + _log_modified_bessel_fn(self.concentration, order=1) + - _log_modified_bessel_fn(self.concentration, order=0) + ).exp() + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/weibull.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/weibull.py new file mode 100644 index 0000000000000000000000000000000000000000..dfc440296ad6cead7942526da0ab58ed8e42b249 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/weibull.py @@ -0,0 +1,97 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +from torch import Tensor +from torch.distributions import constraints +from torch.distributions.exponential import Exponential +from torch.distributions.gumbel import euler_constant +from torch.distributions.transformed_distribution import TransformedDistribution +from torch.distributions.transforms import AffineTransform, PowerTransform +from torch.distributions.utils import broadcast_all + + +__all__ = ["Weibull"] + + +class Weibull(TransformedDistribution): + r""" + Samples from a two-parameter Weibull distribution. + + Example: + + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> m = Weibull(torch.tensor([1.0]), torch.tensor([1.0])) + >>> m.sample() # sample from a Weibull distribution with scale=1, concentration=1 + tensor([ 0.4784]) + + Args: + scale (float or Tensor): Scale parameter of distribution (lambda). + concentration (float or Tensor): Concentration parameter of distribution (k/shape). + validate_args (bool, optional): Whether to validate arguments. Default: None. + """ + + arg_constraints = { + "scale": constraints.positive, + "concentration": constraints.positive, + } + # pyrefly: ignore [bad-override] + support = constraints.positive + + def __init__( + self, + scale: Union[Tensor, float], + concentration: Union[Tensor, float], + validate_args: Optional[bool] = None, + ) -> None: + self.scale, self.concentration = broadcast_all(scale, concentration) + self.concentration_reciprocal = self.concentration.reciprocal() + base_dist = Exponential( + torch.ones_like(self.scale), validate_args=validate_args + ) + transforms = [ + PowerTransform(exponent=self.concentration_reciprocal), + AffineTransform(loc=0, scale=self.scale), + ] + # pyrefly: ignore [bad-argument-type] + super().__init__(base_dist, transforms, validate_args=validate_args) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Weibull, _instance) + new.scale = self.scale.expand(batch_shape) + new.concentration = self.concentration.expand(batch_shape) + new.concentration_reciprocal = new.concentration.reciprocal() + base_dist = self.base_dist.expand(batch_shape) + transforms = [ + PowerTransform(exponent=new.concentration_reciprocal), + AffineTransform(loc=0, scale=new.scale), + ] + super(Weibull, new).__init__(base_dist, transforms, validate_args=False) + new._validate_args = self._validate_args + return new + + @property + def mean(self) -> Tensor: + return self.scale * torch.exp(torch.lgamma(1 + self.concentration_reciprocal)) + + @property + def mode(self) -> Tensor: + return ( + self.scale + * ((self.concentration - 1) / self.concentration) + ** self.concentration.reciprocal() + ) + + @property + def variance(self) -> Tensor: + return self.scale.pow(2) * ( + torch.exp(torch.lgamma(1 + 2 * self.concentration_reciprocal)) + - torch.exp(2 * torch.lgamma(1 + self.concentration_reciprocal)) + ) + + def entropy(self): + return ( + euler_constant * (1 - self.concentration_reciprocal) + + torch.log(self.scale * self.concentration_reciprocal) + + 1 + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/wishart.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/wishart.py new file mode 100644 index 0000000000000000000000000000000000000000..2d230d48533b1b586f1f2c141e2bd5b36a4c4fbc --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/distributions/wishart.py @@ -0,0 +1,347 @@ +# mypy: allow-untyped-defs +import math +import warnings +from typing import Optional, Union + +import torch +from torch import nan, Tensor +from torch.distributions import constraints +from torch.distributions.exp_family import ExponentialFamily +from torch.distributions.multivariate_normal import _precision_to_scale_tril +from torch.distributions.utils import lazy_property +from torch.types import _Number, _size, Number + + +__all__ = ["Wishart"] + +_log_2 = math.log(2) + + +def _mvdigamma(x: Tensor, p: int) -> Tensor: + assert x.gt((p - 1) / 2).all(), "Wrong domain for multivariate digamma function." + return torch.digamma( + x.unsqueeze(-1) + - torch.arange(p, dtype=x.dtype, device=x.device).div(2).expand(x.shape + (-1,)) + ).sum(-1) + + +def _clamp_above_eps(x: Tensor) -> Tensor: + # We assume positive input for this function + return x.clamp(min=torch.finfo(x.dtype).eps) + + +class Wishart(ExponentialFamily): + r""" + Creates a Wishart distribution parameterized by a symmetric positive definite matrix :math:`\Sigma`, + or its Cholesky decomposition :math:`\mathbf{\Sigma} = \mathbf{L}\mathbf{L}^\top` + + Example: + >>> # xdoctest: +SKIP("FIXME: scale_tril must be at least two-dimensional") + >>> m = Wishart(torch.Tensor([2]), covariance_matrix=torch.eye(2)) + >>> m.sample() # Wishart distributed with mean=`df * I` and + >>> # variance(x_ij)=`df` for i != j and variance(x_ij)=`2 * df` for i == j + + Args: + df (float or Tensor): real-valued parameter larger than the (dimension of Square matrix) - 1 + covariance_matrix (Tensor): positive-definite covariance matrix + precision_matrix (Tensor): positive-definite precision matrix + scale_tril (Tensor): lower-triangular factor of covariance, with positive-valued diagonal + Note: + Only one of :attr:`covariance_matrix` or :attr:`precision_matrix` or + :attr:`scale_tril` can be specified. + Using :attr:`scale_tril` will be more efficient: all computations internally + are based on :attr:`scale_tril`. If :attr:`covariance_matrix` or + :attr:`precision_matrix` is passed instead, it is only used to compute + the corresponding lower triangular matrices using a Cholesky decomposition. + 'torch.distributions.LKJCholesky' is a restricted Wishart distribution.[1] + + **References** + + [1] Wang, Z., Wu, Y. and Chu, H., 2018. `On equivalence of the LKJ distribution and the restricted Wishart distribution`. + [2] Sawyer, S., 2007. `Wishart Distributions and Inverse-Wishart Sampling`. + [3] Anderson, T. W., 2003. `An Introduction to Multivariate Statistical Analysis (3rd ed.)`. + [4] Odell, P. L. & Feiveson, A. H., 1966. `A Numerical Procedure to Generate a SampleCovariance Matrix`. JASA, 61(313):199-203. + [5] Ku, Y.-C. & Bloomfield, P., 2010. `Generating Random Wishart Matrices with Fractional Degrees of Freedom in OX`. + """ + + support = constraints.positive_definite + has_rsample = True + _mean_carrier_measure = 0 + + @property + def arg_constraints(self): + return { + "covariance_matrix": constraints.positive_definite, + "precision_matrix": constraints.positive_definite, + "scale_tril": constraints.lower_cholesky, + "df": constraints.greater_than(self.event_shape[-1] - 1), + } + + def __init__( + self, + df: Union[Tensor, Number], + covariance_matrix: Optional[Tensor] = None, + precision_matrix: Optional[Tensor] = None, + scale_tril: Optional[Tensor] = None, + validate_args: Optional[bool] = None, + ) -> None: + assert (covariance_matrix is not None) + (scale_tril is not None) + ( + precision_matrix is not None + ) == 1, ( + "Exactly one of covariance_matrix or precision_matrix or scale_tril may be specified." + ) + + param = next( + p + for p in (covariance_matrix, precision_matrix, scale_tril) + if p is not None + ) + + if param.dim() < 2: + raise ValueError( + "scale_tril must be at least two-dimensional, with optional leading batch dimensions" + ) + + if isinstance(df, _Number): + batch_shape = torch.Size(param.shape[:-2]) + self.df = torch.tensor(df, dtype=param.dtype, device=param.device) + else: + batch_shape = torch.broadcast_shapes(param.shape[:-2], df.shape) + self.df = df.expand(batch_shape) + event_shape = param.shape[-2:] + + if self.df.le(event_shape[-1] - 1).any(): + raise ValueError( + f"Value of df={df} expected to be greater than ndim - 1 = {event_shape[-1] - 1}." + ) + + if scale_tril is not None: + # pyrefly: ignore [read-only] + self.scale_tril = param.expand(batch_shape + (-1, -1)) + elif covariance_matrix is not None: + # pyrefly: ignore [read-only] + self.covariance_matrix = param.expand(batch_shape + (-1, -1)) + elif precision_matrix is not None: + # pyrefly: ignore [read-only] + self.precision_matrix = param.expand(batch_shape + (-1, -1)) + + if self.df.lt(event_shape[-1]).any(): + warnings.warn( + "Low df values detected. Singular samples are highly likely to occur for ndim - 1 < df < ndim.", + stacklevel=2, + ) + + super().__init__(batch_shape, event_shape, validate_args=validate_args) + self._batch_dims = [-(x + 1) for x in range(len(self._batch_shape))] + + if scale_tril is not None: + self._unbroadcasted_scale_tril = scale_tril + elif covariance_matrix is not None: + self._unbroadcasted_scale_tril = torch.linalg.cholesky(covariance_matrix) + else: # precision_matrix is not None + self._unbroadcasted_scale_tril = _precision_to_scale_tril(precision_matrix) + + # Chi2 distribution is needed for Bartlett decomposition sampling + self._dist_chi2 = torch.distributions.chi2.Chi2( + df=( + self.df.unsqueeze(-1) + - torch.arange( + self._event_shape[-1], + dtype=self._unbroadcasted_scale_tril.dtype, + device=self._unbroadcasted_scale_tril.device, + ).expand(batch_shape + (-1,)) + ) + ) + + def expand(self, batch_shape, _instance=None): + new = self._get_checked_instance(Wishart, _instance) + batch_shape = torch.Size(batch_shape) + cov_shape = batch_shape + self.event_shape + new._unbroadcasted_scale_tril = self._unbroadcasted_scale_tril.expand(cov_shape) + new.df = self.df.expand(batch_shape) + + new._batch_dims = [-(x + 1) for x in range(len(batch_shape))] + + if "covariance_matrix" in self.__dict__: + new.covariance_matrix = self.covariance_matrix.expand(cov_shape) + if "scale_tril" in self.__dict__: + new.scale_tril = self.scale_tril.expand(cov_shape) + if "precision_matrix" in self.__dict__: + new.precision_matrix = self.precision_matrix.expand(cov_shape) + + # Chi2 distribution is needed for Bartlett decomposition sampling + new._dist_chi2 = torch.distributions.chi2.Chi2( + df=( + new.df.unsqueeze(-1) + - torch.arange( + self.event_shape[-1], + dtype=new._unbroadcasted_scale_tril.dtype, + device=new._unbroadcasted_scale_tril.device, + ).expand(batch_shape + (-1,)) + ) + ) + + super(Wishart, new).__init__(batch_shape, self.event_shape, validate_args=False) + new._validate_args = self._validate_args + return new + + @lazy_property + def scale_tril(self) -> Tensor: + return self._unbroadcasted_scale_tril.expand( + self._batch_shape + self._event_shape + ) + + @lazy_property + def covariance_matrix(self) -> Tensor: + return ( + self._unbroadcasted_scale_tril + @ self._unbroadcasted_scale_tril.transpose(-2, -1) + ).expand(self._batch_shape + self._event_shape) + + @lazy_property + def precision_matrix(self) -> Tensor: + identity = torch.eye( + self._event_shape[-1], + device=self._unbroadcasted_scale_tril.device, + dtype=self._unbroadcasted_scale_tril.dtype, + ) + return torch.cholesky_solve(identity, self._unbroadcasted_scale_tril).expand( + self._batch_shape + self._event_shape + ) + + @property + def mean(self) -> Tensor: + return self.df.view(self._batch_shape + (1, 1)) * self.covariance_matrix + + @property + def mode(self) -> Tensor: + factor = self.df - self.covariance_matrix.shape[-1] - 1 + factor[factor <= 0] = nan + return factor.view(self._batch_shape + (1, 1)) * self.covariance_matrix + + @property + def variance(self) -> Tensor: + V = self.covariance_matrix # has shape (batch_shape x event_shape) + diag_V = V.diagonal(dim1=-2, dim2=-1) + return self.df.view(self._batch_shape + (1, 1)) * ( + V.pow(2) + torch.einsum("...i,...j->...ij", diag_V, diag_V) + ) + + def _bartlett_sampling(self, sample_shape=torch.Size()): + p = self._event_shape[-1] # has singleton shape + + # Implemented Sampling using Bartlett decomposition + noise = _clamp_above_eps( + self._dist_chi2.rsample(sample_shape).sqrt() + ).diag_embed(dim1=-2, dim2=-1) + + i, j = torch.tril_indices(p, p, offset=-1) + noise[..., i, j] = torch.randn( + torch.Size(sample_shape) + self._batch_shape + (int(p * (p - 1) / 2),), + dtype=noise.dtype, + device=noise.device, + ) + chol = self._unbroadcasted_scale_tril @ noise + return chol @ chol.transpose(-2, -1) + + def rsample( + self, sample_shape: _size = torch.Size(), max_try_correction=None + ) -> Tensor: + r""" + .. warning:: + In some cases, sampling algorithm based on Bartlett decomposition may return singular matrix samples. + Several tries to correct singular samples are performed by default, but it may end up returning + singular matrix samples. Singular samples may return `-inf` values in `.log_prob()`. + In those cases, the user should validate the samples and either fix the value of `df` + or adjust `max_try_correction` value for argument in `.rsample` accordingly. + """ + + if max_try_correction is None: + max_try_correction = 3 if torch._C._get_tracing_state() else 10 + + sample_shape = torch.Size(sample_shape) + sample = self._bartlett_sampling(sample_shape) + + # Below part is to improve numerical stability temporally and should be removed in the future + is_singular = self.support.check(sample) + if self._batch_shape: + is_singular = is_singular.amax(self._batch_dims) + + if torch._C._get_tracing_state(): + # Less optimized version for JIT + for _ in range(max_try_correction): + sample_new = self._bartlett_sampling(sample_shape) + sample = torch.where(is_singular, sample_new, sample) + + is_singular = ~self.support.check(sample) + if self._batch_shape: + is_singular = is_singular.amax(self._batch_dims) + + else: + # More optimized version with data-dependent control flow. + if is_singular.any(): + warnings.warn("Singular sample detected.", stacklevel=2) + + for _ in range(max_try_correction): + sample_new = self._bartlett_sampling(is_singular[is_singular].shape) + sample[is_singular] = sample_new + + is_singular_new = ~self.support.check(sample_new) + if self._batch_shape: + is_singular_new = is_singular_new.amax(self._batch_dims) + is_singular[is_singular.clone()] = is_singular_new + + if not is_singular.any(): + break + + return sample + + def log_prob(self, value): + if self._validate_args: + self._validate_sample(value) + nu = self.df # has shape (batch_shape) + p = self._event_shape[-1] # has singleton shape + return ( + -nu + * ( + p * _log_2 / 2 + + self._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1) + .log() + .sum(-1) + ) + - torch.mvlgamma(nu / 2, p=p) + + (nu - p - 1) / 2 * torch.linalg.slogdet(value).logabsdet + - torch.cholesky_solve(value, self._unbroadcasted_scale_tril) + .diagonal(dim1=-2, dim2=-1) + .sum(dim=-1) + / 2 + ) + + def entropy(self): + nu = self.df # has shape (batch_shape) + p = self._event_shape[-1] # has singleton shape + return ( + (p + 1) + * ( + p * _log_2 / 2 + + self._unbroadcasted_scale_tril.diagonal(dim1=-2, dim2=-1) + .log() + .sum(-1) + ) + + torch.mvlgamma(nu / 2, p=p) + - (nu - p - 1) / 2 * _mvdigamma(nu / 2, p=p) + + nu * p / 2 + ) + + @property + def _natural_params(self) -> tuple[Tensor, Tensor]: + nu = self.df # has shape (batch_shape) + p = self._event_shape[-1] # has singleton shape + return -self.precision_matrix / 2, (nu - p - 1) / 2 + + # pyrefly: ignore [bad-override] + def _log_normalizer(self, x, y): + p = self._event_shape[-1] + return (y + (p + 1) / 2) * ( + -torch.linalg.slogdet(-2 * x).logabsdet + _log_2 * p + ) + torch.mvlgamma(y + (p + 1) / 2, p=p) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/fft/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/fft/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b48cd28bb17df5194e27af9b8c53c0d53a856e03 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/fft/__init__.py @@ -0,0 +1,1442 @@ +import torch +from torch._C import _add_docstr, _fft # type: ignore[attr-defined] +from torch._torch_docs import common_args, factory_common_args + + +__all__ = [ + "fft", + "ifft", + "fft2", + "ifft2", + "fftn", + "ifftn", + "rfft", + "irfft", + "rfft2", + "irfft2", + "rfftn", + "irfftn", + "hfft", + "ihfft", + "fftfreq", + "rfftfreq", + "fftshift", + "ifftshift", + "Tensor", +] + +Tensor = torch.Tensor + +# Note: This not only adds the doc strings for the spectral ops, but +# connects the torch.fft Python namespace to the torch._C._fft builtins. + +fft = _add_docstr( + _fft.fft_fft, + r""" +fft(input, n=None, dim=-1, norm=None, *, out=None) -> Tensor + +Computes the one dimensional discrete Fourier transform of :attr:`input`. + +Note: + The Fourier domain representation of any real signal satisfies the + Hermitian property: `X[i] = conj(X[-i])`. This function always returns both + the positive and negative frequency terms even though, for real inputs, the + negative frequencies are redundant. :func:`~torch.fft.rfft` returns the + more compact one-sided representation where only the positive frequencies + are returned. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimension. + +Args: + input (Tensor): the input tensor + n (int, optional): Signal length. If given, the input will either be zero-padded + or trimmed to this length before computing the FFT. + dim (int, optional): The dimension along which to take the one dimensional FFT. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.fft`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the FFT orthonormal) + + Calling the backward transform (:func:`~torch.fft.ifft`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ifft` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + >>> t = torch.arange(4) + >>> t + tensor([0, 1, 2, 3]) + >>> torch.fft.fft(t) + tensor([ 6.+0.j, -2.+2.j, -2.+0.j, -2.-2.j]) + + >>> t = torch.tensor([0.+1.j, 2.+3.j, 4.+5.j, 6.+7.j]) + >>> torch.fft.fft(t) + tensor([12.+16.j, -8.+0.j, -4.-4.j, 0.-8.j]) +""".format(**common_args), +) + +ifft = _add_docstr( + _fft.fft_ifft, + r""" +ifft(input, n=None, dim=-1, norm=None, *, out=None) -> Tensor + +Computes the one dimensional inverse discrete Fourier transform of :attr:`input`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimension. + +Args: + input (Tensor): the input tensor + n (int, optional): Signal length. If given, the input will either be zero-padded + or trimmed to this length before computing the IFFT. + dim (int, optional): The dimension along which to take the one dimensional IFFT. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.ifft`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the IFFT orthonormal) + + Calling the forward transform (:func:`~torch.fft.fft`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ifft` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> t = torch.tensor([ 6.+0.j, -2.+2.j, -2.+0.j, -2.-2.j]) + >>> torch.fft.ifft(t) + tensor([0.+0.j, 1.+0.j, 2.+0.j, 3.+0.j]) +""".format(**common_args), +) + +fft2 = _add_docstr( + _fft.fft_fft2, + r""" +fft2(input, s=None, dim=(-2, -1), norm=None, *, out=None) -> Tensor + +Computes the 2 dimensional discrete Fourier transform of :attr:`input`. +Equivalent to :func:`~torch.fft.fftn` but FFTs only the last two dimensions by default. + +Note: + The Fourier domain representation of any real signal satisfies the + Hermitian property: ``X[i, j] = conj(X[-i, -j])``. This + function always returns all positive and negative frequency terms even + though, for real inputs, half of these values are redundant. + :func:`~torch.fft.rfft2` returns the more compact one-sided representation + where only the positive frequencies of the last dimension are returned. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: last two dimensions. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.fft2`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the FFT orthonormal) + + Where ``n = prod(s)`` is the logical FFT size. + Calling the backward transform (:func:`~torch.fft.ifft2`) with the same + normalization mode will apply an overall normalization of ``1/n`` + between the two transforms. This is required to make + :func:`~torch.fft.ifft2` the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + >>> x = torch.rand(10, 10, dtype=torch.complex64) + >>> fft2 = torch.fft.fft2(x) + + The discrete Fourier transform is separable, so :func:`~torch.fft.fft2` + here is equivalent to two one-dimensional :func:`~torch.fft.fft` calls: + + >>> two_ffts = torch.fft.fft(torch.fft.fft(x, dim=0), dim=1) + >>> torch.testing.assert_close(fft2, two_ffts, check_stride=False) + +""".format(**common_args), +) + +ifft2 = _add_docstr( + _fft.fft_ifft2, + r""" +ifft2(input, s=None, dim=(-2, -1), norm=None, *, out=None) -> Tensor + +Computes the 2 dimensional inverse discrete Fourier transform of :attr:`input`. +Equivalent to :func:`~torch.fft.ifftn` but IFFTs only the last two dimensions by default. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the IFFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: last two dimensions. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.ifft2`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the IFFT orthonormal) + + Where ``n = prod(s)`` is the logical IFFT size. + Calling the forward transform (:func:`~torch.fft.fft2`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ifft2` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> x = torch.rand(10, 10, dtype=torch.complex64) + >>> ifft2 = torch.fft.ifft2(x) + + The discrete Fourier transform is separable, so :func:`~torch.fft.ifft2` + here is equivalent to two one-dimensional :func:`~torch.fft.ifft` calls: + + >>> two_iffts = torch.fft.ifft(torch.fft.ifft(x, dim=0), dim=1) + >>> torch.testing.assert_close(ifft2, two_iffts, check_stride=False) + +""".format(**common_args), +) + +fftn = _add_docstr( + _fft.fft_fftn, + r""" +fftn(input, s=None, dim=None, norm=None, *, out=None) -> Tensor + +Computes the N dimensional discrete Fourier transform of :attr:`input`. + +Note: + The Fourier domain representation of any real signal satisfies the + Hermitian property: ``X[i_1, ..., i_n] = conj(X[-i_1, ..., -i_n])``. This + function always returns all positive and negative frequency terms even + though, for real inputs, half of these values are redundant. + :func:`~torch.fft.rfftn` returns the more compact one-sided representation + where only the positive frequencies of the last dimension are returned. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: all dimensions, or the last ``len(s)`` dimensions if :attr:`s` is given. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.fftn`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the FFT orthonormal) + + Where ``n = prod(s)`` is the logical FFT size. + Calling the backward transform (:func:`~torch.fft.ifftn`) with the same + normalization mode will apply an overall normalization of ``1/n`` + between the two transforms. This is required to make + :func:`~torch.fft.ifftn` the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + >>> x = torch.rand(10, 10, dtype=torch.complex64) + >>> fftn = torch.fft.fftn(x) + + The discrete Fourier transform is separable, so :func:`~torch.fft.fftn` + here is equivalent to two one-dimensional :func:`~torch.fft.fft` calls: + + >>> two_ffts = torch.fft.fft(torch.fft.fft(x, dim=0), dim=1) + >>> torch.testing.assert_close(fftn, two_ffts, check_stride=False) + +""".format(**common_args), +) + +ifftn = _add_docstr( + _fft.fft_ifftn, + r""" +ifftn(input, s=None, dim=None, norm=None, *, out=None) -> Tensor + +Computes the N dimensional inverse discrete Fourier transform of :attr:`input`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the IFFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: all dimensions, or the last ``len(s)`` dimensions if :attr:`s` is given. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.ifftn`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the IFFT orthonormal) + + Where ``n = prod(s)`` is the logical IFFT size. + Calling the forward transform (:func:`~torch.fft.fftn`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ifftn` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> x = torch.rand(10, 10, dtype=torch.complex64) + >>> ifftn = torch.fft.ifftn(x) + + The discrete Fourier transform is separable, so :func:`~torch.fft.ifftn` + here is equivalent to two one-dimensional :func:`~torch.fft.ifft` calls: + + >>> two_iffts = torch.fft.ifft(torch.fft.ifft(x, dim=0), dim=1) + >>> torch.testing.assert_close(ifftn, two_iffts, check_stride=False) + +""".format(**common_args), +) + +rfft = _add_docstr( + _fft.fft_rfft, + r""" +rfft(input, n=None, dim=-1, norm=None, *, out=None) -> Tensor + +Computes the one dimensional Fourier transform of real-valued :attr:`input`. + +The FFT of a real signal is Hermitian-symmetric, ``X[i] = conj(X[-i])`` so +the output contains only the positive frequencies below the Nyquist frequency. +To compute the full output, use :func:`~torch.fft.fft` + +Note: + Supports torch.half on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimension. + +Args: + input (Tensor): the real input tensor + n (int, optional): Signal length. If given, the input will either be zero-padded + or trimmed to this length before computing the real FFT. + dim (int, optional): The dimension along which to take the one dimensional real FFT. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.rfft`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the FFT orthonormal) + + Calling the backward transform (:func:`~torch.fft.irfft`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.irfft` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + >>> t = torch.arange(4) + >>> t + tensor([0, 1, 2, 3]) + >>> torch.fft.rfft(t) + tensor([ 6.+0.j, -2.+2.j, -2.+0.j]) + + Compare against the full output from :func:`~torch.fft.fft`: + + >>> torch.fft.fft(t) + tensor([ 6.+0.j, -2.+2.j, -2.+0.j, -2.-2.j]) + + Notice that the symmetric element ``T[-1] == T[1].conj()`` is omitted. + At the Nyquist frequency ``T[-2] == T[2]`` is it's own symmetric pair, + and therefore must always be real-valued. +""".format(**common_args), +) + +irfft = _add_docstr( + _fft.fft_irfft, + r""" +irfft(input, n=None, dim=-1, norm=None, *, out=None) -> Tensor + +Computes the inverse of :func:`~torch.fft.rfft`. + +:attr:`input` is interpreted as a one-sided Hermitian signal in the Fourier +domain, as produced by :func:`~torch.fft.rfft`. By the Hermitian property, the +output will be real-valued. + +Note: + Some input frequencies must be real-valued to satisfy the Hermitian + property. In these cases the imaginary component will be ignored. + For example, any imaginary component in the zero-frequency term cannot + be represented in a real output and so will always be ignored. + +Note: + The correct interpretation of the Hermitian input depends on the length of + the original data, as given by :attr:`n`. This is because each input shape + could correspond to either an odd or even length signal. By default, the + signal is assumed to be even length and odd signals will not round-trip + properly. So, it is recommended to always pass the signal length :attr:`n`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimension. + With default arguments, size of the transformed dimension should be (2^n + 1) as argument + `n` defaults to even output size = 2 * (transformed_dim_size - 1) + +Args: + input (Tensor): the input tensor representing a half-Hermitian signal + n (int, optional): Output signal length. This determines the length of the + output signal. If given, the input will either be zero-padded or trimmed to this + length before computing the real IFFT. + Defaults to even output: ``n=2*(input.size(dim) - 1)``. + dim (int, optional): The dimension along which to take the one dimensional real IFFT. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.irfft`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the real IFFT orthonormal) + + Calling the forward transform (:func:`~torch.fft.rfft`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.irfft` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> t = torch.linspace(0, 1, 5) + >>> t + tensor([0.0000, 0.2500, 0.5000, 0.7500, 1.0000]) + >>> T = torch.fft.rfft(t) + >>> T + tensor([ 2.5000+0.0000j, -0.6250+0.8602j, -0.6250+0.2031j]) + + Without specifying the output length to :func:`~torch.fft.irfft`, the output + will not round-trip properly because the input is odd-length: + + >>> torch.fft.irfft(T) + tensor([0.1562, 0.3511, 0.7812, 1.2114]) + + So, it is recommended to always pass the signal length :attr:`n`: + + >>> roundtrip = torch.fft.irfft(T, t.numel()) + >>> torch.testing.assert_close(roundtrip, t, check_stride=False) + +""".format(**common_args), +) + +rfft2 = _add_docstr( + _fft.fft_rfft2, + r""" +rfft2(input, s=None, dim=(-2, -1), norm=None, *, out=None) -> Tensor + +Computes the 2-dimensional discrete Fourier transform of real :attr:`input`. +Equivalent to :func:`~torch.fft.rfftn` but FFTs only the last two dimensions by default. + +The FFT of a real signal is Hermitian-symmetric, ``X[i, j] = conj(X[-i, -j])``, +so the full :func:`~torch.fft.fft2` output contains redundant information. +:func:`~torch.fft.rfft2` instead omits the negative frequencies in the last +dimension. + +Note: + Supports torch.half on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the real FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: last two dimensions. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.rfft2`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the real FFT orthonormal) + + Where ``n = prod(s)`` is the logical FFT size. + Calling the backward transform (:func:`~torch.fft.irfft2`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.irfft2` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + >>> t = torch.rand(10, 10) + >>> rfft2 = torch.fft.rfft2(t) + >>> rfft2.size() + torch.Size([10, 6]) + + Compared against the full output from :func:`~torch.fft.fft2`, we have all + elements up to the Nyquist frequency. + + >>> fft2 = torch.fft.fft2(t) + >>> torch.testing.assert_close(fft2[..., :6], rfft2, check_stride=False) + + The discrete Fourier transform is separable, so :func:`~torch.fft.rfft2` + here is equivalent to a combination of :func:`~torch.fft.fft` and + :func:`~torch.fft.rfft`: + + >>> two_ffts = torch.fft.fft(torch.fft.rfft(t, dim=1), dim=0) + >>> torch.testing.assert_close(rfft2, two_ffts, check_stride=False) + +""".format(**common_args), +) + +irfft2 = _add_docstr( + _fft.fft_irfft2, + r""" +irfft2(input, s=None, dim=(-2, -1), norm=None, *, out=None) -> Tensor + +Computes the inverse of :func:`~torch.fft.rfft2`. +Equivalent to :func:`~torch.fft.irfftn` but IFFTs only the last two dimensions by default. + +:attr:`input` is interpreted as a one-sided Hermitian signal in the Fourier +domain, as produced by :func:`~torch.fft.rfft2`. By the Hermitian property, the +output will be real-valued. + +Note: + Some input frequencies must be real-valued to satisfy the Hermitian + property. In these cases the imaginary component will be ignored. + For example, any imaginary component in the zero-frequency term cannot + be represented in a real output and so will always be ignored. + +Note: + The correct interpretation of the Hermitian input depends on the length of + the original data, as given by :attr:`s`. This is because each input shape + could correspond to either an odd or even length signal. By default, the + signal is assumed to be even length and odd signals will not round-trip + properly. So, it is recommended to always pass the signal shape :attr:`s`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + With default arguments, the size of last dimension should be (2^n + 1) as argument + `s` defaults to even output size = 2 * (last_dim_size - 1) + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the real FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Defaults to even output in the last dimension: + ``s[-1] = 2*(input.size(dim[-1]) - 1)``. + dim (Tuple[int], optional): Dimensions to be transformed. + The last dimension must be the half-Hermitian compressed dimension. + Default: last two dimensions. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.irfft2`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the real IFFT orthonormal) + + Where ``n = prod(s)`` is the logical IFFT size. + Calling the forward transform (:func:`~torch.fft.rfft2`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.irfft2` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> t = torch.rand(10, 9) + >>> T = torch.fft.rfft2(t) + + Without specifying the output length to :func:`~torch.fft.irfft2`, the output + will not round-trip properly because the input is odd-length in the last + dimension: + + >>> torch.fft.irfft2(T).size() + torch.Size([10, 8]) + + So, it is recommended to always pass the signal shape :attr:`s`. + + >>> roundtrip = torch.fft.irfft2(T, t.size()) + >>> roundtrip.size() + torch.Size([10, 9]) + >>> torch.testing.assert_close(roundtrip, t, check_stride=False) + +""".format(**common_args), +) + +rfftn = _add_docstr( + _fft.fft_rfftn, + r""" +rfftn(input, s=None, dim=None, norm=None, *, out=None) -> Tensor + +Computes the N-dimensional discrete Fourier transform of real :attr:`input`. + +The FFT of a real signal is Hermitian-symmetric, +``X[i_1, ..., i_n] = conj(X[-i_1, ..., -i_n])`` so the full +:func:`~torch.fft.fftn` output contains redundant information. +:func:`~torch.fft.rfftn` instead omits the negative frequencies in the +last dimension. + +Note: + Supports torch.half on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the real FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: all dimensions, or the last ``len(s)`` dimensions if :attr:`s` is given. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.rfftn`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the real FFT orthonormal) + + Where ``n = prod(s)`` is the logical FFT size. + Calling the backward transform (:func:`~torch.fft.irfftn`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.irfftn` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + >>> t = torch.rand(10, 10) + >>> rfftn = torch.fft.rfftn(t) + >>> rfftn.size() + torch.Size([10, 6]) + + Compared against the full output from :func:`~torch.fft.fftn`, we have all + elements up to the Nyquist frequency. + + >>> fftn = torch.fft.fftn(t) + >>> torch.testing.assert_close(fftn[..., :6], rfftn, check_stride=False) + + The discrete Fourier transform is separable, so :func:`~torch.fft.rfftn` + here is equivalent to a combination of :func:`~torch.fft.fft` and + :func:`~torch.fft.rfft`: + + >>> two_ffts = torch.fft.fft(torch.fft.rfft(t, dim=1), dim=0) + >>> torch.testing.assert_close(rfftn, two_ffts, check_stride=False) + +""".format(**common_args), +) + +irfftn = _add_docstr( + _fft.fft_irfftn, + r""" +irfftn(input, s=None, dim=None, norm=None, *, out=None) -> Tensor + +Computes the inverse of :func:`~torch.fft.rfftn`. + +:attr:`input` is interpreted as a one-sided Hermitian signal in the Fourier +domain, as produced by :func:`~torch.fft.rfftn`. By the Hermitian property, the +output will be real-valued. + +Note: + Some input frequencies must be real-valued to satisfy the Hermitian + property. In these cases the imaginary component will be ignored. + For example, any imaginary component in the zero-frequency term cannot + be represented in a real output and so will always be ignored. + +Note: + The correct interpretation of the Hermitian input depends on the length of + the original data, as given by :attr:`s`. This is because each input shape + could correspond to either an odd or even length signal. By default, the + signal is assumed to be even length and odd signals will not round-trip + properly. So, it is recommended to always pass the signal shape :attr:`s`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + With default arguments, the size of last dimension should be (2^n + 1) as argument + `s` defaults to even output size = 2 * (last_dim_size - 1) + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the real FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Defaults to even output in the last dimension: + ``s[-1] = 2*(input.size(dim[-1]) - 1)``. + dim (Tuple[int], optional): Dimensions to be transformed. + The last dimension must be the half-Hermitian compressed dimension. + Default: all dimensions, or the last ``len(s)`` dimensions if :attr:`s` is given. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.irfftn`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the real IFFT orthonormal) + + Where ``n = prod(s)`` is the logical IFFT size. + Calling the forward transform (:func:`~torch.fft.rfftn`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.irfftn` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> t = torch.rand(10, 9) + >>> T = torch.fft.rfftn(t) + + Without specifying the output length to :func:`~torch.fft.irfft`, the output + will not round-trip properly because the input is odd-length in the last + dimension: + + >>> torch.fft.irfftn(T).size() + torch.Size([10, 8]) + + So, it is recommended to always pass the signal shape :attr:`s`. + + >>> roundtrip = torch.fft.irfftn(T, t.size()) + >>> roundtrip.size() + torch.Size([10, 9]) + >>> torch.testing.assert_close(roundtrip, t, check_stride=False) + +""".format(**common_args), +) + +hfft = _add_docstr( + _fft.fft_hfft, + r""" +hfft(input, n=None, dim=-1, norm=None, *, out=None) -> Tensor + +Computes the one dimensional discrete Fourier transform of a Hermitian +symmetric :attr:`input` signal. + +Note: + + :func:`~torch.fft.hfft`/:func:`~torch.fft.ihfft` are analogous to + :func:`~torch.fft.rfft`/:func:`~torch.fft.irfft`. The real FFT expects + a real signal in the time-domain and gives a Hermitian symmetry in the + frequency-domain. The Hermitian FFT is the opposite; Hermitian symmetric in + the time-domain and real-valued in the frequency-domain. For this reason, + special care needs to be taken with the length argument :attr:`n`, in the + same way as with :func:`~torch.fft.irfft`. + +Note: + Because the signal is Hermitian in the time-domain, the result will be + real in the frequency domain. Note that some input frequencies must be + real-valued to satisfy the Hermitian property. In these cases the imaginary + component will be ignored. For example, any imaginary component in + ``input[0]`` would result in one or more complex frequency terms which + cannot be represented in a real output and so will always be ignored. + +Note: + The correct interpretation of the Hermitian input depends on the length of + the original data, as given by :attr:`n`. This is because each input shape + could correspond to either an odd or even length signal. By default, the + signal is assumed to be even length and odd signals will not round-trip + properly. So, it is recommended to always pass the signal length :attr:`n`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimension. + With default arguments, size of the transformed dimension should be (2^n + 1) as argument + `n` defaults to even output size = 2 * (transformed_dim_size - 1) + +Args: + input (Tensor): the input tensor representing a half-Hermitian signal + n (int, optional): Output signal length. This determines the length of the + real output. If given, the input will either be zero-padded or trimmed to this + length before computing the Hermitian FFT. + Defaults to even output: ``n=2*(input.size(dim) - 1)``. + dim (int, optional): The dimension along which to take the one dimensional Hermitian FFT. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.hfft`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the Hermitian FFT orthonormal) + + Calling the backward transform (:func:`~torch.fft.ihfft`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ihfft` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + Taking a real-valued frequency signal and bringing it into the time domain + gives Hermitian symmetric output: + + >>> t = torch.linspace(0, 1, 5) + >>> t + tensor([0.0000, 0.2500, 0.5000, 0.7500, 1.0000]) + >>> T = torch.fft.ifft(t) + >>> T + tensor([ 0.5000-0.0000j, -0.1250-0.1720j, -0.1250-0.0406j, -0.1250+0.0406j, + -0.1250+0.1720j]) + + Note that ``T[1] == T[-1].conj()`` and ``T[2] == T[-2].conj()`` is + redundant. We can thus compute the forward transform without considering + negative frequencies: + + >>> torch.fft.hfft(T[:3], n=5) + tensor([0.0000, 0.2500, 0.5000, 0.7500, 1.0000]) + + Like with :func:`~torch.fft.irfft`, the output length must be given in order + to recover an even length output: + + >>> torch.fft.hfft(T[:3]) + tensor([0.1250, 0.2809, 0.6250, 0.9691]) +""".format(**common_args), +) + +ihfft = _add_docstr( + _fft.fft_ihfft, + r""" +ihfft(input, n=None, dim=-1, norm=None, *, out=None) -> Tensor + +Computes the inverse of :func:`~torch.fft.hfft`. + +:attr:`input` must be a real-valued signal, interpreted in the Fourier domain. +The IFFT of a real signal is Hermitian-symmetric, ``X[i] = conj(X[-i])``. +:func:`~torch.fft.ihfft` represents this in the one-sided form where only the +positive frequencies below the Nyquist frequency are included. To compute the +full output, use :func:`~torch.fft.ifft`. + +Note: + Supports torch.half on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimension. + +Args: + input (Tensor): the real input tensor + n (int, optional): Signal length. If given, the input will either be zero-padded + or trimmed to this length before computing the Hermitian IFFT. + dim (int, optional): The dimension along which to take the one dimensional Hermitian IFFT. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.ihfft`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the IFFT orthonormal) + + Calling the forward transform (:func:`~torch.fft.hfft`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ihfft` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> t = torch.arange(5) + >>> t + tensor([0, 1, 2, 3, 4]) + >>> torch.fft.ihfft(t) + tensor([ 2.0000-0.0000j, -0.5000-0.6882j, -0.5000-0.1625j]) + + Compare against the full output from :func:`~torch.fft.ifft`: + + >>> torch.fft.ifft(t) + tensor([ 2.0000-0.0000j, -0.5000-0.6882j, -0.5000-0.1625j, -0.5000+0.1625j, + -0.5000+0.6882j]) +""".format(**common_args), +) + +hfft2 = _add_docstr( + _fft.fft_hfft2, + r""" +hfft2(input, s=None, dim=(-2, -1), norm=None, *, out=None) -> Tensor + +Computes the 2-dimensional discrete Fourier transform of a Hermitian symmetric +:attr:`input` signal. Equivalent to :func:`~torch.fft.hfftn` but only +transforms the last two dimensions by default. + +:attr:`input` is interpreted as a one-sided Hermitian signal in the time +domain. By the Hermitian property, the Fourier transform will be real-valued. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + With default arguments, the size of last dimension should be (2^n + 1) as argument + `s` defaults to even output size = 2 * (last_dim_size - 1) + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the Hermitian FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Defaults to even output in the last dimension: + ``s[-1] = 2*(input.size(dim[-1]) - 1)``. + dim (Tuple[int], optional): Dimensions to be transformed. + The last dimension must be the half-Hermitian compressed dimension. + Default: last two dimensions. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.hfft2`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the Hermitian FFT orthonormal) + + Where ``n = prod(s)`` is the logical FFT size. + Calling the backward transform (:func:`~torch.fft.ihfft2`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ihfft2` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + Starting from a real frequency-space signal, we can generate a + Hermitian-symmetric time-domain signal: + >>> T = torch.rand(10, 9) + >>> t = torch.fft.ihfft2(T) + + Without specifying the output length to :func:`~torch.fft.hfftn`, the + output will not round-trip properly because the input is odd-length in the + last dimension: + + >>> torch.fft.hfft2(t).size() + torch.Size([10, 10]) + + So, it is recommended to always pass the signal shape :attr:`s`. + + >>> roundtrip = torch.fft.hfft2(t, T.size()) + >>> roundtrip.size() + torch.Size([10, 9]) + >>> torch.allclose(roundtrip, T) + True + +""".format(**common_args), +) + +ihfft2 = _add_docstr( + _fft.fft_ihfft2, + r""" +ihfft2(input, s=None, dim=(-2, -1), norm=None, *, out=None) -> Tensor + +Computes the 2-dimensional inverse discrete Fourier transform of real +:attr:`input`. Equivalent to :func:`~torch.fft.ihfftn` but transforms only the +two last dimensions by default. + +Note: + Supports torch.half on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the Hermitian IFFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: last two dimensions. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.ihfft2`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the Hermitian IFFT orthonormal) + + Where ``n = prod(s)`` is the logical IFFT size. + Calling the forward transform (:func:`~torch.fft.hfft2`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ihfft2` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> T = torch.rand(10, 10) + >>> t = torch.fft.ihfft2(t) + >>> t.size() + torch.Size([10, 6]) + + Compared against the full output from :func:`~torch.fft.ifft2`, the + Hermitian time-space signal takes up only half the space. + + >>> fftn = torch.fft.ifft2(t) + >>> torch.allclose(fftn[..., :6], rfftn) + True + + The discrete Fourier transform is separable, so :func:`~torch.fft.ihfft2` + here is equivalent to a combination of :func:`~torch.fft.ifft` and + :func:`~torch.fft.ihfft`: + + >>> two_ffts = torch.fft.ifft(torch.fft.ihfft(t, dim=1), dim=0) + >>> torch.allclose(t, two_ffts) + True + +""".format(**common_args), +) + +hfftn = _add_docstr( + _fft.fft_hfftn, + r""" +hfftn(input, s=None, dim=None, norm=None, *, out=None) -> Tensor + +Computes the n-dimensional discrete Fourier transform of a Hermitian symmetric +:attr:`input` signal. + +:attr:`input` is interpreted as a one-sided Hermitian signal in the time +domain. By the Hermitian property, the Fourier transform will be real-valued. + +Note: + :func:`~torch.fft.hfftn`/:func:`~torch.fft.ihfftn` are analogous to + :func:`~torch.fft.rfftn`/:func:`~torch.fft.irfftn`. The real FFT expects + a real signal in the time-domain and gives Hermitian symmetry in the + frequency-domain. The Hermitian FFT is the opposite; Hermitian symmetric in + the time-domain and real-valued in the frequency-domain. For this reason, + special care needs to be taken with the shape argument :attr:`s`, in the + same way as with :func:`~torch.fft.irfftn`. + +Note: + Some input frequencies must be real-valued to satisfy the Hermitian + property. In these cases the imaginary component will be ignored. + For example, any imaginary component in the zero-frequency term cannot + be represented in a real output and so will always be ignored. + +Note: + The correct interpretation of the Hermitian input depends on the length of + the original data, as given by :attr:`s`. This is because each input shape + could correspond to either an odd or even length signal. By default, the + signal is assumed to be even length and odd signals will not round-trip + properly. It is recommended to always pass the signal shape :attr:`s`. + +Note: + Supports torch.half and torch.chalf on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + With default arguments, the size of last dimension should be (2^n + 1) as argument + `s` defaults to even output size = 2 * (last_dim_size - 1) + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the real FFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Defaults to even output in the last dimension: + ``s[-1] = 2*(input.size(dim[-1]) - 1)``. + dim (Tuple[int], optional): Dimensions to be transformed. + The last dimension must be the half-Hermitian compressed dimension. + Default: all dimensions, or the last ``len(s)`` dimensions if :attr:`s` is given. + norm (str, optional): Normalization mode. For the forward transform + (:func:`~torch.fft.hfftn`), these correspond to: + + * ``"forward"`` - normalize by ``1/n`` + * ``"backward"`` - no normalization + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the Hermitian FFT orthonormal) + + Where ``n = prod(s)`` is the logical FFT size. + Calling the backward transform (:func:`~torch.fft.ihfftn`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ihfftn` + the exact inverse. + + Default is ``"backward"`` (no normalization). + +Keyword args: + {out} + +Example: + + Starting from a real frequency-space signal, we can generate a + Hermitian-symmetric time-domain signal: + >>> T = torch.rand(10, 9) + >>> t = torch.fft.ihfftn(T) + + Without specifying the output length to :func:`~torch.fft.hfftn`, the + output will not round-trip properly because the input is odd-length in the + last dimension: + + >>> torch.fft.hfftn(t).size() + torch.Size([10, 10]) + + So, it is recommended to always pass the signal shape :attr:`s`. + + >>> roundtrip = torch.fft.hfftn(t, T.size()) + >>> roundtrip.size() + torch.Size([10, 9]) + >>> torch.allclose(roundtrip, T) + True + +""".format(**common_args), +) + +ihfftn = _add_docstr( + _fft.fft_ihfftn, + r""" +ihfftn(input, s=None, dim=None, norm=None, *, out=None) -> Tensor + +Computes the N-dimensional inverse discrete Fourier transform of real :attr:`input`. + +:attr:`input` must be a real-valued signal, interpreted in the Fourier domain. +The n-dimensional IFFT of a real signal is Hermitian-symmetric, +``X[i, j, ...] = conj(X[-i, -j, ...])``. :func:`~torch.fft.ihfftn` represents +this in the one-sided form where only the positive frequencies below the +Nyquist frequency are included in the last signal dimension. To compute the +full output, use :func:`~torch.fft.ifftn`. + +Note: + Supports torch.half on CUDA with GPU Architecture SM53 or greater. + However it only supports powers of 2 signal length in every transformed dimensions. + +Args: + input (Tensor): the input tensor + s (Tuple[int], optional): Signal size in the transformed dimensions. + If given, each dimension ``dim[i]`` will either be zero-padded or + trimmed to the length ``s[i]`` before computing the Hermitian IFFT. + If a length ``-1`` is specified, no padding is done in that dimension. + Default: ``s = [input.size(d) for d in dim]`` + dim (Tuple[int], optional): Dimensions to be transformed. + Default: all dimensions, or the last ``len(s)`` dimensions if :attr:`s` is given. + norm (str, optional): Normalization mode. For the backward transform + (:func:`~torch.fft.ihfftn`), these correspond to: + + * ``"forward"`` - no normalization + * ``"backward"`` - normalize by ``1/n`` + * ``"ortho"`` - normalize by ``1/sqrt(n)`` (making the Hermitian IFFT orthonormal) + + Where ``n = prod(s)`` is the logical IFFT size. + Calling the forward transform (:func:`~torch.fft.hfftn`) with the same + normalization mode will apply an overall normalization of ``1/n`` between + the two transforms. This is required to make :func:`~torch.fft.ihfftn` + the exact inverse. + + Default is ``"backward"`` (normalize by ``1/n``). + +Keyword args: + {out} + +Example: + + >>> T = torch.rand(10, 10) + >>> ihfftn = torch.fft.ihfftn(T) + >>> ihfftn.size() + torch.Size([10, 6]) + + Compared against the full output from :func:`~torch.fft.ifftn`, we have all + elements up to the Nyquist frequency. + + >>> ifftn = torch.fft.ifftn(t) + >>> torch.allclose(ifftn[..., :6], ihfftn) + True + + The discrete Fourier transform is separable, so :func:`~torch.fft.ihfftn` + here is equivalent to a combination of :func:`~torch.fft.ihfft` and + :func:`~torch.fft.ifft`: + + >>> two_iffts = torch.fft.ifft(torch.fft.ihfft(t, dim=1), dim=0) + >>> torch.allclose(ihfftn, two_iffts) + True + +""".format(**common_args), +) + +fftfreq = _add_docstr( + _fft.fft_fftfreq, + r""" +fftfreq(n, d=1.0, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + +Computes the discrete Fourier Transform sample frequencies for a signal of size :attr:`n`. + +Note: + By convention, :func:`~torch.fft.fft` returns positive frequency terms + first, followed by the negative frequencies in reverse order, so that + ``f[-i]`` for all :math:`0 < i \leq n/2`` in Python gives the negative + frequency terms. For an FFT of length :attr:`n` and with inputs spaced in + length unit :attr:`d`, the frequencies are:: + + f = [0, 1, ..., (n - 1) // 2, -(n // 2), ..., -1] / (d * n) + +Note: + For even lengths, the Nyquist frequency at ``f[n/2]`` can be thought of as + either negative or positive. :func:`~torch.fft.fftfreq` follows NumPy's + convention of taking it to be negative. + +Args: + n (int): the FFT length + d (float, optional): The sampling length scale. + The spacing between individual samples of the FFT input. + The default assumes unit spacing, dividing that result by the actual + spacing gives the result in physical frequency units. + +Keyword Args: + {out} + {dtype} + {layout} + {device} + {requires_grad} + +Example: + + >>> torch.fft.fftfreq(5) + tensor([ 0.0000, 0.2000, 0.4000, -0.4000, -0.2000]) + + For even input, we can see the Nyquist frequency at ``f[2]`` is given as + negative: + + >>> torch.fft.fftfreq(4) + tensor([ 0.0000, 0.2500, -0.5000, -0.2500]) + +""".format(**factory_common_args), +) + +rfftfreq = _add_docstr( + _fft.fft_rfftfreq, + r""" +rfftfreq(n, d=1.0, *, out=None, dtype=None, layout=torch.strided, device=None, requires_grad=False) -> Tensor + +Computes the sample frequencies for :func:`~torch.fft.rfft` with a signal of size :attr:`n`. + +Note: + :func:`~torch.fft.rfft` returns Hermitian one-sided output, so only the + positive frequency terms are returned. For a real FFT of length :attr:`n` + and with inputs spaced in length unit :attr:`d`, the frequencies are:: + + f = torch.arange((n + 1) // 2) / (d * n) + +Note: + For even lengths, the Nyquist frequency at ``f[n/2]`` can be thought of as + either negative or positive. Unlike :func:`~torch.fft.fftfreq`, + :func:`~torch.fft.rfftfreq` always returns it as positive. + +Args: + n (int): the real FFT length + d (float, optional): The sampling length scale. + The spacing between individual samples of the FFT input. + The default assumes unit spacing, dividing that result by the actual + spacing gives the result in physical frequency units. + +Keyword Args: + {out} + {dtype} + {layout} + {device} + {requires_grad} + +Example: + + >>> torch.fft.rfftfreq(5) + tensor([0.0000, 0.2000, 0.4000]) + + >>> torch.fft.rfftfreq(4) + tensor([0.0000, 0.2500, 0.5000]) + + Compared to the output from :func:`~torch.fft.fftfreq`, we see that the + Nyquist frequency at ``f[2]`` has changed sign: + >>> torch.fft.fftfreq(4) + tensor([ 0.0000, 0.2500, -0.5000, -0.2500]) + +""".format(**factory_common_args), +) + +fftshift = _add_docstr( + _fft.fft_fftshift, + r""" +fftshift(input, dim=None) -> Tensor + +Reorders n-dimensional FFT data, as provided by :func:`~torch.fft.fftn`, to have +negative frequency terms first. + +This performs a periodic shift of n-dimensional data such that the origin +``(0, ..., 0)`` is moved to the center of the tensor. Specifically, to +``input.shape[dim] // 2`` in each selected dimension. + +Note: + By convention, the FFT returns positive frequency terms first, followed by + the negative frequencies in reverse order, so that ``f[-i]`` for all + :math:`0 < i \leq n/2` in Python gives the negative frequency terms. + :func:`~torch.fft.fftshift` rearranges all frequencies into ascending order + from negative to positive with the zero-frequency term in the center. + +Note: + For even lengths, the Nyquist frequency at ``f[n/2]`` can be thought of as + either negative or positive. :func:`~torch.fft.fftshift` always puts the + Nyquist term at the 0-index. This is the same convention used by + :func:`~torch.fft.fftfreq`. + +Args: + input (Tensor): the tensor in FFT order + dim (int, Tuple[int], optional): The dimensions to rearrange. + Only dimensions specified here will be rearranged, any other dimensions + will be left in their original order. + Default: All dimensions of :attr:`input`. + +Example: + + >>> f = torch.fft.fftfreq(4) + >>> f + tensor([ 0.0000, 0.2500, -0.5000, -0.2500]) + + >>> torch.fft.fftshift(f) + tensor([-0.5000, -0.2500, 0.0000, 0.2500]) + + Also notice that the Nyquist frequency term at ``f[2]`` was moved to the + beginning of the tensor. + + This also works for multi-dimensional transforms: + + >>> x = torch.fft.fftfreq(5, d=1/5) + 0.1 * torch.fft.fftfreq(5, d=1/5).unsqueeze(1) + >>> x + tensor([[ 0.0000, 1.0000, 2.0000, -2.0000, -1.0000], + [ 0.1000, 1.1000, 2.1000, -1.9000, -0.9000], + [ 0.2000, 1.2000, 2.2000, -1.8000, -0.8000], + [-0.2000, 0.8000, 1.8000, -2.2000, -1.2000], + [-0.1000, 0.9000, 1.9000, -2.1000, -1.1000]]) + + >>> torch.fft.fftshift(x) + tensor([[-2.2000, -1.2000, -0.2000, 0.8000, 1.8000], + [-2.1000, -1.1000, -0.1000, 0.9000, 1.9000], + [-2.0000, -1.0000, 0.0000, 1.0000, 2.0000], + [-1.9000, -0.9000, 0.1000, 1.1000, 2.1000], + [-1.8000, -0.8000, 0.2000, 1.2000, 2.2000]]) + + :func:`~torch.fft.fftshift` can also be useful for spatial data. If our + data is defined on a centered grid (``[-(N//2), (N-1)//2]``) then we can + use the standard FFT defined on an uncentered grid (``[0, N)``) by first + applying an :func:`~torch.fft.ifftshift`. + + >>> x_centered = torch.arange(-5, 5) + >>> x_uncentered = torch.fft.ifftshift(x_centered) + >>> fft_uncentered = torch.fft.fft(x_uncentered) + + Similarly, we can convert the frequency domain components to centered + convention by applying :func:`~torch.fft.fftshift`. + + >>> fft_centered = torch.fft.fftshift(fft_uncentered) + + The inverse transform, from centered Fourier space back to centered spatial + data, can be performed by applying the inverse shifts in reverse order: + + >>> x_centered_2 = torch.fft.fftshift(torch.fft.ifft(torch.fft.ifftshift(fft_centered))) + >>> torch.testing.assert_close(x_centered.to(torch.complex64), x_centered_2, check_stride=False) + + +""", +) + +ifftshift = _add_docstr( + _fft.fft_ifftshift, + r""" +ifftshift(input, dim=None) -> Tensor + +Inverse of :func:`~torch.fft.fftshift`. + +Args: + input (Tensor): the tensor in FFT order + dim (int, Tuple[int], optional): The dimensions to rearrange. + Only dimensions specified here will be rearranged, any other dimensions + will be left in their original order. + Default: All dimensions of :attr:`input`. + +Example: + + >>> f = torch.fft.fftfreq(5) + >>> f + tensor([ 0.0000, 0.2000, 0.4000, -0.4000, -0.2000]) + + A round-trip through :func:`~torch.fft.fftshift` and + :func:`~torch.fft.ifftshift` gives the same result: + + >>> shifted = torch.fft.fftshift(f) + >>> torch.fft.ifftshift(shifted) + tensor([ 0.0000, 0.2000, 0.4000, -0.4000, -0.2000]) + +""", +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/func/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/func/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..35743fcf429ae779d5608314bcd89fc0af333d38 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/func/__init__.py @@ -0,0 +1,31 @@ +from torch._functorch.apis import grad, grad_and_value, vmap +from torch._functorch.batch_norm_replacement import replace_all_batch_norm_modules_ +from torch._functorch.eager_transforms import ( + debug_unwrap, + functionalize, + hessian, + jacfwd, + jacrev, + jvp, + linearize, + vjp, +) +from torch._functorch.functional_call import functional_call, stack_module_state + + +__all__ = [ + "grad", + "grad_and_value", + "vmap", + "replace_all_batch_norm_modules_", + "functionalize", + "hessian", + "jacfwd", + "jacrev", + "jvp", + "linearize", + "vjp", + "functional_call", + "stack_module_state", + "debug_unwrap", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/futures/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/futures/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d265a56914290014f94f7c92e4e0edd4bd2323a9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/futures/__init__.py @@ -0,0 +1,336 @@ +# mypy: allow-untyped-defs +# pylint: disable=useless-parent-delegation +from __future__ import annotations + +from typing import cast, Generic, Optional, TYPE_CHECKING, TypeVar, Union + +import torch + + +if TYPE_CHECKING: + from collections.abc import Callable + + +__all__ = ["Future", "collect_all", "wait_all"] + + +T = TypeVar("T") +S = TypeVar("S") + + +class Future(torch._C.Future, Generic[T]): + r""" + Wrapper around a ``torch._C.Future`` which encapsulates an asynchronous + execution of a callable, e.g. :meth:`~torch.distributed.rpc.rpc_async`. It + also exposes a set of APIs to add callback functions and set results. + + .. warning:: GPU support is a beta feature, subject to changes. + """ + + def __init__( + self, *, devices: Optional[list[Union[int, str, torch.device]]] = None + ): + r""" + Create an empty unset ``Future``. If the future is intended to hold + values containing CUDA tensors, (a superset of) their CUDA devices must + be specified at construction. (This is only supported if + ``torch.cuda.is_available()`` returns ``True``). This is needed to + ensure proper CUDA stream synchronization. The child futures, returned + by the ``then`` method, will inherit these devices. + + Args: + devices(``List[Union[int, str, torch.device]]``, optional): the set + of devices on which tensors contained in this future's value are + allowed to reside and on which callbacks are allowed to operate. + """ + if devices is None: + devices = [] + super().__init__([torch.device(d) for d in devices]) + + def done(self) -> bool: + r""" + Return ``True`` if this ``Future`` is done. A ``Future`` is done if it + has a result or an exception. + + If the value contains tensors that reside on GPUs, ``Future.done()`` + will return ``True`` even if the asynchronous kernels that are + populating those tensors haven't yet completed running on the device, + because at such stage the result is already usable, provided one + performs the appropriate synchronizations (see :meth:`wait`). + """ + return super().done() + + def wait(self) -> T: + r""" + Block until the value of this ``Future`` is ready. + + If the value contains tensors that reside on GPUs, then an additional + synchronization is performed with the kernels (executing on the device) + which may be asynchronously populating those tensors. Such sync is + non-blocking, which means that ``wait()`` will insert the necessary + instructions in the current streams to ensure that further operations + enqueued on those streams will be properly scheduled after the async + kernels but, once that is done, ``wait()`` will return, even if those + kernels are still running. No further synchronization is required when + accessing and using the values, as long as one doesn't change streams. + + Returns: + The value held by this ``Future``. If the function (callback or RPC) + creating the value has thrown an error, this ``wait`` method will + also throw an error. + """ + return super().wait() + + def value(self) -> T: + r""" + Obtain the value of an already-completed future. + + This method should only be called after a call to :meth:`wait` has + completed, or inside a callback function passed to :meth:`then`. In + other cases this ``Future`` may not yet hold a value and calling + ``value()`` could fail. + + If the value contains tensors that reside on GPUs, then this method will + *not* perform any additional synchronization. This should be done + beforehand, separately, through a call to :meth:`wait` (except within + callbacks, for which it's already being taken care of by :meth:`then`). + + Returns: + The value held by this ``Future``. If the function (callback or RPC) + creating the value has thrown an error, this ``value()`` method will + also throw an error. + """ + return super().value() + + def then(self, callback: Callable[[Future[T]], S]) -> Future[S]: + r""" + Append the given callback function to this ``Future``, which will be run + when the ``Future`` is completed. Multiple callbacks can be added to + the same ``Future``, but the order in which they will be executed cannot + be guaranteed (to enforce a certain order consider chaining: + ``fut.then(cb1).then(cb2)``). The callback must take one argument, which + is the reference to this ``Future``. The callback function can use the + :meth:`value` method to get the value. Note that if this ``Future`` is + already completed, the given callback will be run immediately inline. + + If the ``Future``'s value contains tensors that reside on GPUs, the + callback might be invoked while the async kernels that are populating + those tensors haven't yet finished executing on the device. However, the + callback will be invoked with some dedicated streams set as current + (fetched from a global pool) which will be synchronized with those + kernels. Hence any operation performed by the callback on these tensors + will be scheduled on the device after the kernels complete. In other + words, as long as the callback doesn't switch streams, it can safely + manipulate the result without any additional synchronization. This is + similar to the non-blocking behavior of :meth:`wait`. + + Similarly, if the callback returns a value that contains tensors that + reside on a GPU, it can do so even if the kernels that are producing + these tensors are still running on the device, as long as the callback + didn't change streams during its execution. If one wants to change + streams, one must be careful to re-synchronize them with the original + streams, that is, those that were current when the callback was invoked. + + Args: + callback(``Callable``): a ``Callable`` that takes this ``Future`` as + the only argument. + + Returns: + A new ``Future`` object that holds the return value of the + ``callback`` and will be marked as completed when the given + ``callback`` finishes. + + .. note:: Note that if the callback function throws, either + through the original future being completed with an exception and + calling ``fut.wait()``, or through other code in the callback, the + future returned by ``then`` will be marked appropriately with the + encountered error. However, if this callback later completes + additional futures, those futures are not marked as completed with + an error and the user is responsible for handling completion/waiting + on those futures independently. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_FUTURES) + >>> def callback(fut): + ... print(f"RPC return value is {fut.wait()}.") + >>> fut = torch.futures.Future() + >>> # The inserted callback will print the return value when + >>> # receiving the response from "worker1" + >>> cb_fut = fut.then(callback) + >>> chain_cb_fut = cb_fut.then( + ... lambda x : print(f"Chained cb done. {x.wait()}") + ... ) + >>> fut.set_result(5) + RPC return value is 5. + Chained cb done. None + """ + return cast(Future[S], super().then(callback)) + + def add_done_callback(self, callback: Callable[[Future[T]], None]) -> None: + r""" + Append the given callback function to this ``Future``, which will be run + when the ``Future`` is completed. Multiple callbacks can be added to + the same ``Future``, but the order in which they will be executed cannot + be guaranteed. The callback must take one argument, which is the + reference to this ``Future``. The callback function can use the + :meth:`value` method to get the value. Note that if this ``Future`` is + already completed, the given callback will be run inline. + + We recommend that you use the :meth:`then` method as it provides a way + to synchronize after your callback has completed. ``add_done_callback`` + can be cheaper if your callback does not return anything. But both + :meth:`then` and ``add_done_callback`` use the same callback + registration API under the hood. + + With respect to GPU tensors, this method behaves in the same way as + :meth:`then`. + + Args: + callback(``Future``): a ``Callable`` that takes in one argument, + which is the reference to this ``Future``. + + .. note:: Note that if the callback function throws, either + through the original future being completed with an exception and + calling ``fut.wait()``, or through other code in the callback, + error handling must be carefully taken care of. For example, if + this callback later completes additional futures, those futures are + not marked as completed with an error and the user is responsible + for handling completion/waiting on those futures independently. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_FUTURES) + >>> def callback(fut): + ... print("This will run after the future has finished.") + ... print(fut.wait()) + >>> fut = torch.futures.Future() + >>> fut.add_done_callback(callback) + >>> fut.set_result(5) + This will run after the future has finished. + 5 + """ + super().add_done_callback(callback) + + def set_result(self, result: T) -> None: + r""" + Set the result for this ``Future``, which will mark this ``Future`` as + completed and trigger all attached callbacks. Note that a ``Future`` + cannot be marked completed twice. + + If the result contains tensors that reside on GPUs, this method can be + called even if the asynchronous kernels that are populating those + tensors haven't yet completed running on the device, provided that the + streams on which those kernels were enqueued are set as the current ones + when this method is called. Put simply, it's safe to call this method + immediately after launching those kernels, without any additional + synchronization, as long as one doesn't change streams in between. This + method will record events on all the relevant current streams and will + use them to ensure proper scheduling for all the consumers of this + ``Future``. + + Args: + result (object): the result object of this ``Future``. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_FUTURES) + >>> import threading + >>> import time + >>> def slow_set_future(fut, value): + ... time.sleep(0.5) + ... fut.set_result(value) + >>> fut = torch.futures.Future() + >>> t = threading.Thread( + ... target=slow_set_future, + ... args=(fut, torch.ones(2) * 3) + ... ) + >>> t.start() + >>> print(fut.wait()) + tensor([3., 3.]) + >>> t.join() + """ + super().set_result(result) + + def set_exception(self, result: T) -> None: + r""" + Set an exception for this ``Future``, which will mark this ``Future`` as + completed with an error and trigger all attached callbacks. Note that + when calling wait()/value() on this ``Future``, the exception set here + will be raised inline. + + Args: + result (BaseException): the exception for this ``Future``. + + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_FUTURES) + >>> fut = torch.futures.Future() + >>> fut.set_exception(ValueError("foo")) + >>> fut.wait() + Traceback (most recent call last): + ... + ValueError: foo + """ + assert isinstance(result, Exception), ( + f"{result} is of type {type(result)}, not an Exception." + ) + + def raise_error(fut_result): + raise fut_result + + super()._set_unwrap_func(raise_error) + self.set_result(result) # type: ignore[arg-type] + + +def collect_all(futures: list[Future]) -> Future[list[Future]]: + r""" + Collects the provided :class:`~torch.futures.Future` objects into a single + combined :class:`~torch.futures.Future` that is completed when all of the + sub-futures are completed. + + Args: + futures (list): a list of :class:`~torch.futures.Future` objects. + + Returns: + Returns a :class:`~torch.futures.Future` object to a list of the passed + in Futures. + + Example:: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_FUTURES) + >>> fut0 = torch.futures.Future() + >>> fut1 = torch.futures.Future() + >>> fut = torch.futures.collect_all([fut0, fut1]) + >>> fut0.set_result(0) + >>> fut1.set_result(1) + >>> fut_list = fut.wait() + >>> print(f"fut0 result = {fut_list[0].wait()}") + fut0 result = 0 + >>> print(f"fut1 result = {fut_list[1].wait()}") + fut1 result = 1 + """ + return cast( + Future[list[Future]], + torch._C._collect_all(cast(list[torch._C.Future], futures)), + ) + + +def wait_all(futures: list[Future]) -> list: + r""" + Waits for all provided futures to be complete, and returns + the list of completed values. If any of the futures encounters an error, + the method will exit early and report the error not waiting for other + futures to complete. + + Args: + futures (list): a list of :class:`~torch.futures.Future` object. + + Returns: + A list of the completed :class:`~torch.futures.Future` results. This + method will throw an error if ``wait`` on any + :class:`~torch.futures.Future` throws. + """ + return [ + fut.wait() + for fut in torch._C._collect_all(cast(list[torch._C.Future], futures)).wait() + ] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/advisor-annotate.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/advisor-annotate.h new file mode 100644 index 0000000000000000000000000000000000000000..3a99056bb382f1871433470f3b7809741bcc03d5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/advisor-annotate.h @@ -0,0 +1,525 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/* + * Copyright (C) 2005-2019 Intel Corporation + * SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause + */ + +/* This file defines macros and inline functions used by + * the Intel(R) Advisor XE "Dependencies Modeling" and + * "Suitability Modeling" analysis, and are in described + * in the "Annotations" section of the help. + * + * Expansion Options + * + * There are several options you can used to control how advisor-annotate.h + * is included. To use these, define the option prior to including + * advisor-annotate.h. e.g. + * #define ANNOTATE_DECLARE + * #include "advisor-annotate.h" + * + * Controlling inclusion of windows.h + * + * windows.h is included for declarations for LoadLibrary, GetProcSymbol, + * but this can have interactions with user code, such as conflicting + * definitions of types. There are two general approaches to work around + * this if this triggers problems building your application: + * + * 1. Reduce the amount declared by windows.h by using the following: + * #define NOMINMAX + * #define WIN32_LEAN_AND_MEAN + * prior to including advisor-annotate.h in your code. + * The first avoids problems with STL min/max in particular + * This is sufficient in some cases, and may be the easiest. + * + * 2. Use a declaration/definition approach, where all uses of advisor-annotate.h + * other than one, generate a set of declarations, and windows.h is only + * needed in a single implementation module. In this model, all includes + * of advisor-annotate.h except one specify ANNOTATE_DECLARE, which causes + * advisor-annotate.h to declare an external routine, and not include + * windows.h. A final include of advisor-annotate.h than specifies + * ANNOTATE_DEFINE, to actually define the global routine to resolve + * the external reference. This one include is the only one that winds up + * using windows.h. If necessary, this can be placed in a file by itself. + * + * An example using this mechanism: + * + * ... + * // Some header(s) used in places in your system where you want + * // to be able to use annotations + * #define ANNOTATE_DECLARE + * #include "advisor-annotate.h" + * ... + * // annotation uses + * ANNOTATE_SITE_BEGIN(MySite1) + * ... + * ANNOTATE_SITE_END(MySite1) + * ... + * + * ... + * // A single implementation file (.cpp/.cxx) causes windows.h + * // to be included, and the support routine to be defined as a + * // global routine called from the various annotation uses. + * #define ANNOTATE_DEFINE + * #include "advisor-annotate.h" + * ... + * + * Null expansion of annotations + * + * Some people may find it useful to have no expansion for annotations, + * if you have a project that you want to build without any annotation + * effects at all. (e.g. if you have a project where you want to have + * some annotations in a shared source pool, but only particular + * developers are actually building with the annotations enabled.) + * Defining ANNOTATE_EXPAND_NULL avoids declaring comdat routines, + * and avoids any textual expansion for annotation macros. + */ + +#ifndef _ADVISOR_ANNOTATE_H_ +#define _ADVISOR_ANNOTATE_H_ + +/* Version of the annotations. + * The presence of this macro serves to idetify the annotation definition + * file and the form of annotations. + */ +#define INTEL_ADVISOR_ANNOTATION_VERSION 1.0 + +#ifdef ANNOTATE_EXPAND_NULL + +#define ANNOTATE_SITE_BEGIN(_SITE) +#define ANNOTATE_SITE_END(...) +#define ANNOTATE_TASK_BEGIN(_TASK) +#define ANNOTATE_TASK_END(...) +#define ANNOTATE_ITERATION_TASK(_TASK) +#define ANNOTATE_LOCK_ACQUIRE(_ADDR) +#define ANNOTATE_LOCK_RELEASE(_ADDR) +#define ANNOTATE_RECORD_ALLOCATION(_ADDR, _SIZE) +#define ANNOTATE_RECORD_DEALLOCATION(_ADDR) +#define ANNOTATE_INDUCTION_USES(_ADDR, _SIZE) +#define ANNOTATE_REDUCTION_USES(_ADDR, _SIZE) +#define ANNOTATE_OBSERVE_USES(_ADDR, _SIZE) +#define ANNOTATE_CLEAR_USES(_ADDR) +#define ANNOTATE_DISABLE_OBSERVATION_PUSH +#define ANNOTATE_DISABLE_OBSERVATION_POP +#define ANNOTATE_DISABLE_COLLECTION_PUSH +#define ANNOTATE_DISABLE_COLLECTION_POP +#define ANNOTATE_AGGREGATE_TASK(_COUNT) + +#else /* ANNOTATE_EXPAND_NULL */ + +#if defined(WIN32) || defined(_WIN32) + +#define ANNOTATEAPI __cdecl + +#ifndef ANNOTATE_DECLARE +#include + +typedef HMODULE lib_t; + +#define __itt_get_proc(lib, name) GetProcAddress(lib, name) +#define __itt_load_lib(name) LoadLibraryA(name) +#define __itt_unload_lib(handle) FreeLibrary(handle) +#define __itt_system_error() (int)GetLastError() +#endif /* ANNOTATE_DECLARE */ + +#else /* defined(WIN32) || defined(_WIN32) */ + +#if defined _M_IX86 || __i386__ +# define ANNOTATEAPI __attribute__ ((cdecl)) +#else +# define ANNOTATEAPI /* actual only on x86 platform */ +#endif + + +#ifndef ANNOTATE_DECLARE +#include +#include +#include + +typedef void* lib_t; + +#define __itt_get_proc(lib, name) dlsym(lib, name) +#define __itt_load_lib(name) dlopen(name, RTLD_LAZY) +#define __itt_unload_lib(handle) dlclose(handle) +#define __itt_system_error() errno +#endif /* ANNOTATE_DECLARE */ + +#endif /* defined(WIN32) || defined(_WIN32) */ + +#include + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#ifndef _ITTNOTIFY_H_ + +/* Handles for sites and tasks. + */ +typedef void* __itt_model_site; /* handle for lexical site */ +typedef void* __itt_model_site_instance; /* handle for dynamic instance */ +typedef void* __itt_model_task; /* handle for lexical site */ +typedef void* __itt_model_task_instance; /* handle for dynamic instance */ + +typedef enum { + __itt_model_disable_observation, + __itt_model_disable_collection +} __itt_model_disable; + +#endif /* _ITTNOTIFY_H_ */ + +/*** Use the routines in libittnotify.dll. ***/ + +/* Basic approach: + * For the case of calling the dll, there is an __annotate_routine function + * declared as a comdat in each compilation unit with annotations present. + * That routine in turn has an internal static structure that is initialized + * once to contain the address of functions occuring in libittnotify.dll. + * Each time an annotation macro is invoked, that causes a call to the + * __annotate_routine function to get addresses of the routines, followed + * by calling the specific routine, provided the address is non-null. + */ + +/* This set of macros generates calls that are part of application images, + * which call the __itt_model_xxx routines in the dynamically loaded + * libittnotify.dll. + */ +#ifndef _ITTNOTIFY_H_ +#define ITT_NOTIFY_DECL(_text) _text +#else +#define ITT_NOTIFY_DECL(_text) +#endif + +/* For C++, a static initialization is used */ +#if defined(__cplusplus) && defined(WIN32) +#define _ANNOTATE_ROUTINES_ADDR __annotate_routines_s +#else +#define _ANNOTATE_ROUTINES_ADDR __annotate_routines_init( __annotate_routines() ) +#endif /* __cplusplus */ + + +#define _ANNOTATE_DECLARE_0(_BASENAME) \ +typedef void (ANNOTATEAPI * __annotate_##_BASENAME##_t)(); \ +static __inline void ANNOTATEAPI __annotate_##_BASENAME##_t_nop() { }; \ +ITT_NOTIFY_DECL( extern void ANNOTATEAPI __itt_model_##_BASENAME(); ) + +#define _ANNOTATE_DECLARE_0_INT(_BASENAME) \ +typedef int (ANNOTATEAPI * __annotate_##_BASENAME##_t)(); \ +static __inline int ANNOTATEAPI __annotate_##_BASENAME##_t_nop() { return 0; }; \ +ITT_NOTIFY_DECL( extern void ANNOTATEAPI __itt_model_##_BASENAME(); ) + +#define _ANNOTATE_CALL_0(_BASENAME) { _ANNOTATE_ROUTINES_ADDR->_BASENAME(); } + +#define _ANNOTATE_DECLARE_1(_BASENAME, _P1TYPE) \ +typedef void (ANNOTATEAPI * __annotate_##_BASENAME##_t)(_P1TYPE p1); \ +static __inline void ANNOTATEAPI __annotate_##_BASENAME##_t_nop(_P1TYPE p1) { (void)p1; }; \ +ITT_NOTIFY_DECL( extern void ANNOTATEAPI __itt_model_##_BASENAME(_P1TYPE p1); ) + +#define _ANNOTATE_CALL_1(_BASENAME, _P1) { _ANNOTATE_ROUTINES_ADDR->_BASENAME(_P1); } + +#define _ANNOTATE_DECLARE_2(_BASENAME, _P1TYPE, _P2TYPE) \ +typedef void (ANNOTATEAPI * __annotate_##_BASENAME##_t)(_P1TYPE p1, _P2TYPE p2); \ +static __inline void ANNOTATEAPI __annotate_##_BASENAME##_t_nop(_P1TYPE p1, _P2TYPE p2) { (void)p1; (void)p2; }; \ +ITT_NOTIFY_DECL( extern void ANNOTATEAPI __itt_model_##_BASENAME(_P1TYPE p1, _P2TYPE p2); ) + +#define _ANNOTATE_CALL_2(_BASENAME, _P1, _P2) { _ANNOTATE_ROUTINES_ADDR->_BASENAME((_P1), (_P2)); } + +/*** Declare routines appropriately based on usage style ***/ + +/* Depending on above, this will either expand to comdats that are + * used directly, or comdats that call routines in libittnotify.dll + */ +_ANNOTATE_DECLARE_1(site_beginA, const char *) +_ANNOTATE_DECLARE_0(site_end_2) +_ANNOTATE_DECLARE_1(task_beginA, const char *) +_ANNOTATE_DECLARE_0(task_end_2) +_ANNOTATE_DECLARE_1(iteration_taskA, const char *) +_ANNOTATE_DECLARE_1(lock_acquire_2, void *) +_ANNOTATE_DECLARE_1(lock_release_2, void *) +_ANNOTATE_DECLARE_2(record_allocation, void *, size_t) +_ANNOTATE_DECLARE_1(record_deallocation, void *) +_ANNOTATE_DECLARE_2(induction_uses, void *, size_t) +_ANNOTATE_DECLARE_2(reduction_uses, void *, size_t) +_ANNOTATE_DECLARE_2(observe_uses, void *, size_t) +_ANNOTATE_DECLARE_1(clear_uses, void *) +_ANNOTATE_DECLARE_1(disable_push, __itt_model_disable) +_ANNOTATE_DECLARE_0(disable_pop) +_ANNOTATE_DECLARE_1(aggregate_task, size_t) +_ANNOTATE_DECLARE_0_INT(is_collection_disabled) + +/* All of the symbols potentially in the library + */ +struct __annotate_routines { + volatile int initialized; + __annotate_site_beginA_t site_beginA; + __annotate_site_end_2_t site_end_2; + __annotate_task_beginA_t task_beginA; + __annotate_task_end_2_t task_end_2; + __annotate_iteration_taskA_t iteration_taskA; + __annotate_lock_acquire_2_t lock_acquire_2; + __annotate_lock_release_2_t lock_release_2; + __annotate_record_allocation_t record_allocation; + __annotate_record_deallocation_t record_deallocation; + __annotate_induction_uses_t induction_uses; + __annotate_reduction_uses_t reduction_uses; + __annotate_observe_uses_t observe_uses; + __annotate_clear_uses_t clear_uses; + __annotate_disable_push_t disable_push; + __annotate_disable_pop_t disable_pop; + __annotate_aggregate_task_t aggregate_task; + __annotate_is_collection_disabled_t is_collection_disabled; +}; + +/* This comdat-ed routine means there is a single instance of the function pointer + * structure per image + */ +static __inline struct __annotate_routines* __annotate_routines() +{ + static struct __annotate_routines __annotate_routines; + return &__annotate_routines; +} + +/* This routine is called to get the address of an initialized + * set of function pointers for the annotation routines. + */ + +#ifdef ANNOTATE_DECLARE +extern struct __annotate_routines* ANNOTATEAPI __annotate_routines_init(struct __annotate_routines* itt); +#else +#ifdef ANNOTATE_DEFINE + /* */ +#else + static __inline +#endif +struct __annotate_routines* +ANNOTATEAPI +__annotate_routines_init(struct __annotate_routines* itt) { + + if (itt->initialized) { + return itt; + } else { + + /* Initialized by first invocation + * This assumes that the code here can be executed successfully + * by multiple threads, should that ever happen. + */ + int do_disable_pop = 0; + char* lib_name = NULL; + lib_t itt_notify = 0; + + if (sizeof(void*) > 4) { + lib_name = getenv("INTEL_LIBITTNOTIFY64"); + } else { + lib_name = getenv("INTEL_LIBITTNOTIFY32"); + } + + if (lib_name) { + itt_notify = __itt_load_lib(lib_name); + } else { +#if defined(WIN32) || defined(_WIN32) + itt_notify = __itt_load_lib("libittnotify.dll"); +#elif defined(__APPLE__) + itt_notify = __itt_load_lib("libittnotify.dylib"); +#else + itt_notify = __itt_load_lib("libittnotify.so"); +#endif + } + + if (itt_notify != NULL) { + /* The static variables initialized and itt are reported as race conditions + * or inconsistent lock usage by Dependencies Modeling in some obscure cases + * involving multiple dlls. Ignoring this initialization phase gets rid of + * this problem. + */ + __annotate_disable_push_t disable_push; + __annotate_is_collection_disabled_t is_collection_disabled; + disable_push = (__annotate_disable_push_t) __itt_get_proc(itt_notify, "__itt_model_disable_push"); + is_collection_disabled = (__annotate_is_collection_disabled_t) __itt_get_proc(itt_notify, "__itt_model_is_collection_disabled"); + if (disable_push) { + if ( ! (is_collection_disabled && is_collection_disabled()) ) + { + // disable collection only if it is not disabled already (for example, started paused) + disable_push(__itt_model_disable_observation); + do_disable_pop = 1; + } + } + itt->site_beginA = (__annotate_site_beginA_t) __itt_get_proc(itt_notify, "__itt_model_site_beginA"); + itt->site_end_2 = (__annotate_site_end_2_t) __itt_get_proc(itt_notify, "__itt_model_site_end_2"); + itt->task_beginA = (__annotate_task_beginA_t) __itt_get_proc(itt_notify, "__itt_model_task_beginA"); + itt->task_end_2 = (__annotate_task_end_2_t) __itt_get_proc(itt_notify, "__itt_model_task_end_2"); + itt->iteration_taskA = (__annotate_iteration_taskA_t) __itt_get_proc(itt_notify, "__itt_model_iteration_taskA"); + itt->lock_acquire_2 = (__annotate_lock_acquire_2_t) __itt_get_proc(itt_notify, "__itt_model_lock_acquire_2"); + itt->lock_release_2 = (__annotate_lock_release_2_t) __itt_get_proc(itt_notify, "__itt_model_lock_release_2"); + itt->record_allocation = (__annotate_record_allocation_t) __itt_get_proc(itt_notify, "__itt_model_record_allocation"); + itt->record_deallocation = (__annotate_record_deallocation_t)__itt_get_proc(itt_notify, "__itt_model_record_deallocation"); + itt->induction_uses = (__annotate_induction_uses_t) __itt_get_proc(itt_notify, "__itt_model_induction_uses"); + itt->reduction_uses = (__annotate_reduction_uses_t) __itt_get_proc(itt_notify, "__itt_model_reduction_uses"); + itt->observe_uses = (__annotate_observe_uses_t) __itt_get_proc(itt_notify, "__itt_model_observe_uses"); + itt->clear_uses = (__annotate_clear_uses_t) __itt_get_proc(itt_notify, "__itt_model_clear_uses"); + itt->disable_push = disable_push; + itt->disable_pop = (__annotate_disable_pop_t) __itt_get_proc(itt_notify, "__itt_model_disable_pop"); + itt->aggregate_task = (__annotate_aggregate_task_t) __itt_get_proc(itt_notify, "__itt_model_aggregate_task"); + itt->is_collection_disabled = is_collection_disabled; + } + /* No-op routine for any that didn't get resolved */ + if (!itt->site_beginA) itt->site_beginA = __annotate_site_beginA_t_nop; + if (!itt->site_end_2) itt->site_end_2 = __annotate_site_end_2_t_nop; + if (!itt->task_beginA) itt->task_beginA = __annotate_task_beginA_t_nop; + if (!itt->task_end_2) itt->task_end_2 = __annotate_task_end_2_t_nop; + if (!itt->iteration_taskA) itt->iteration_taskA = __annotate_iteration_taskA_t_nop; + if (!itt->lock_acquire_2) itt->lock_acquire_2 = __annotate_lock_acquire_2_t_nop; + if (!itt->lock_release_2) itt->lock_release_2 = __annotate_lock_release_2_t_nop; + if (!itt->record_allocation) itt->record_allocation = __annotate_record_allocation_t_nop; + if (!itt->record_deallocation) itt->record_deallocation=__annotate_record_deallocation_t_nop; + if (!itt->induction_uses) itt->induction_uses = __annotate_induction_uses_t_nop; + if (!itt->reduction_uses) itt->reduction_uses = __annotate_reduction_uses_t_nop; + if (!itt->observe_uses) itt->observe_uses = __annotate_observe_uses_t_nop; + if (!itt->clear_uses) itt->clear_uses = __annotate_clear_uses_t_nop; + if (!itt->disable_push) itt->disable_push = __annotate_disable_push_t_nop; + if (!itt->disable_pop) itt->disable_pop = __annotate_disable_pop_t_nop; + if (!itt->aggregate_task) itt->aggregate_task = __annotate_aggregate_task_t_nop; + if (!itt->is_collection_disabled) itt->is_collection_disabled = __annotate_is_collection_disabled_t_nop; + + itt->initialized = 1; + + if (do_disable_pop) { + itt->disable_pop(); + } + } + return itt; +} +#endif /* ANNOTATE_DECLARE */ + +/* For C++ only, use a class to force initialization */ + +#if defined(__cplusplus) && defined(WIN32) +/* Force one-shot initialization so individual calls don't need it */ +static struct __annotate_routines* __annotate_routines_s = __annotate_routines_init( __annotate_routines() ); +#endif + +/* For C++, allow the Annotate::SiteBegin(x) form. For Windows CLR, this is the default + * expansion for the macros (with no-inline) to get the best call stacks in the tools. */ +#if defined(__cplusplus) +/* Ensure this code is managed and non-inlinable */ +#if defined(WIN32) && defined(__CLR_VER) +#pragma managed(push, on) +#define ANNOTATE_CLR_NOINLINE __declspec(noinline) +#else +#define ANNOTATE_CLR_NOINLINE +#endif +class Annotate { +public: + static ANNOTATE_CLR_NOINLINE void SiteBegin(const char* site) { _ANNOTATE_ROUTINES_ADDR->site_beginA(site); } + static ANNOTATE_CLR_NOINLINE void SiteEnd() { _ANNOTATE_ROUTINES_ADDR->site_end_2(); } + static ANNOTATE_CLR_NOINLINE void TaskBegin(const char* task) { _ANNOTATE_ROUTINES_ADDR->task_beginA(task); } + static ANNOTATE_CLR_NOINLINE void TaskEnd() { _ANNOTATE_ROUTINES_ADDR->task_end_2(); } + static ANNOTATE_CLR_NOINLINE void IterationTask(const char* task) { _ANNOTATE_ROUTINES_ADDR->iteration_taskA(task); } + static ANNOTATE_CLR_NOINLINE void LockAcquire(void* lockId) { _ANNOTATE_ROUTINES_ADDR->lock_acquire_2(lockId); } + static ANNOTATE_CLR_NOINLINE void LockRelease(void* lockId) { _ANNOTATE_ROUTINES_ADDR->lock_release_2(lockId); } + static ANNOTATE_CLR_NOINLINE void RecordAllocation(void *p, size_t s) { _ANNOTATE_ROUTINES_ADDR->record_allocation(p, s); } + static ANNOTATE_CLR_NOINLINE void RecordDeallocation(void *p) { _ANNOTATE_ROUTINES_ADDR->record_deallocation(p); } + static ANNOTATE_CLR_NOINLINE void InductionUses(void *p, size_t s) { _ANNOTATE_ROUTINES_ADDR->induction_uses(p, s); } + static ANNOTATE_CLR_NOINLINE void ReductionUses(void *p, size_t s) { _ANNOTATE_ROUTINES_ADDR->reduction_uses(p, s); } + static ANNOTATE_CLR_NOINLINE void ObserveUses(void *p, size_t s) { _ANNOTATE_ROUTINES_ADDR->observe_uses(p, s); } + static ANNOTATE_CLR_NOINLINE void ClearUses(void *p) { _ANNOTATE_ROUTINES_ADDR->clear_uses(p); } + static ANNOTATE_CLR_NOINLINE void DisablePush(__itt_model_disable d) { _ANNOTATE_ROUTINES_ADDR->disable_push(d); } + static ANNOTATE_CLR_NOINLINE void DisablePop() { _ANNOTATE_ROUTINES_ADDR->disable_pop(); } + static ANNOTATE_CLR_NOINLINE void AggregateTask(size_t c) { _ANNOTATE_ROUTINES_ADDR->aggregate_task(c); } +}; +#if defined(WIN32) && defined(__CLR_VER) +#pragma managed(pop) +#endif +#undef ANNOTATE_CLR_NOINLINE +#endif + +#if defined(__cplusplus) && defined(WIN32) && defined(__CLR_VER) + +#define ANNOTATE_SITE_BEGIN(_SITE) Annotate::SiteBegin(#_SITE) +#define ANNOTATE_SITE_END(...) Annotate::SiteEnd() +#define ANNOTATE_TASK_BEGIN(_TASK) Annotate::TaskBegin(#_TASK) +#define ANNOTATE_TASK_END(...) Annotate::TaskEnd() +#define ANNOTATE_ITERATION_TASK(_TASK) Annotate::IterationTask(#_TASK) +#define ANNOTATE_LOCK_ACQUIRE(_ADDR) Annotate::LockAcquire(_ADDR) +#define ANNOTATE_LOCK_RELEASE(_ADDR) Annotate::LockRelease(_ADDR) +#define ANNOTATE_RECORD_ALLOCATION(_ADDR, _SIZE) Annotate::RecordAllocation((_ADDR), (_SIZE)) +#define ANNOTATE_RECORD_DEALLOCATION(_ADDR) Annotate::RecordDeallocation(_ADDR) +#define ANNOTATE_INDUCTION_USES(_ADDR, _SIZE) Annotate::InductionUses((_ADDR), (_SIZE)) +#define ANNOTATE_REDUCTION_USES(_ADDR, _SIZE) Annotate::ReductionUses((_ADDR), (_SIZE)) +#define ANNOTATE_OBSERVE_USES(_ADDR, _SIZE) Annotate::ObserveUses((_ADDR), (_SIZE)) +#define ANNOTATE_CLEAR_USES(_ADDR) Annotate::ClearUses(_ADDR) +#define ANNOTATE_DISABLE_OBSERVATION_PUSH Annotate::DisablePush(itt_model_disable_observation) +#define ANNOTATE_DISABLE_OBSERVATION_POP Annotate::DisablePop() +#define ANNOTATE_DISABLE_COLLECTION_PUSH Annotate::DisablePush(__itt_model_disable_collection) +#define ANNOTATE_DISABLE_COLLECTION_POP Annotate::DisablePop() +#define ANNOTATE_AGGREGATE_TASK(_COUNT) Annotate::AggregateTask(_COUNT) + +#else + +/* Mark the start of a site (region) to be analyzed by the tool */ +#define ANNOTATE_SITE_BEGIN(_SITE) _ANNOTATE_CALL_1(site_beginA, #_SITE) + +/* Mark the end of a site (region) to be analyzed by the tool and + * indicate a WaitForAll task synchronization */ +#define ANNOTATE_SITE_END(...) _ANNOTATE_CALL_0(site_end_2) + +/* Mark the beginning of a region of code that constitutes a task */ +#define ANNOTATE_TASK_BEGIN(_TASK) _ANNOTATE_CALL_1(task_beginA, #_TASK) + +/* Mark the end of a region of code that constitutes a task */ +#define ANNOTATE_TASK_END(...) _ANNOTATE_CALL_0(task_end_2) + +/* Mark the break between one task and the next task (a "split" description model + * rather than a "begin/end" description model. */ +#define ANNOTATE_ITERATION_TASK(_TASK) _ANNOTATE_CALL_1(iteration_taskA, #_TASK) + +/* Acquire a lock identified by lockId */ +#define ANNOTATE_LOCK_ACQUIRE(_ADDR) _ANNOTATE_CALL_1(lock_acquire_2, (_ADDR)) + +/* Release a lock identified by lockId */ +#define ANNOTATE_LOCK_RELEASE(_ADDR) _ANNOTATE_CALL_1(lock_release_2, (_ADDR)) + +/* Record user allocation of memory */ +#define ANNOTATE_RECORD_ALLOCATION(_ADDR, _SIZE) _ANNOTATE_CALL_2(record_allocation, (_ADDR), (_SIZE)) + +/* Record user deallocation of memory */ +#define ANNOTATE_RECORD_DEALLOCATION(_ADDR) _ANNOTATE_CALL_1(record_deallocation, (_ADDR)) + +/* Denote storage as an inductive value */ +#define ANNOTATE_INDUCTION_USES(_ADDR, _SIZE) _ANNOTATE_CALL_2(induction_uses, (_ADDR), (_SIZE)) + +/* Denote storage as a reduction */ +#define ANNOTATE_REDUCTION_USES(_ADDR, _SIZE) _ANNOTATE_CALL_2(reduction_uses, (_ADDR), (_SIZE)) + +/* Record all observations of uses */ +#define ANNOTATE_OBSERVE_USES(_ADDR, _SIZE) _ANNOTATE_CALL_2(observe_uses, (_ADDR), (_SIZE)) + +/* Clear handling of values */ +#define ANNOTATE_CLEAR_USES(_ADDR) _ANNOTATE_CALL_1(clear_uses, (_ADDR)) + +/* Push disable of observations */ +#define ANNOTATE_DISABLE_OBSERVATION_PUSH _ANNOTATE_CALL_1(disable_push, __itt_model_disable_observation) + +/* Pop disable of observations */ +#define ANNOTATE_DISABLE_OBSERVATION_POP _ANNOTATE_CALL_0(disable_pop) + +/* Push disable of collection */ +#define ANNOTATE_DISABLE_COLLECTION_PUSH _ANNOTATE_CALL_1(disable_push, __itt_model_disable_collection) + +/* Pop disable of collection */ +#define ANNOTATE_DISABLE_COLLECTION_POP _ANNOTATE_CALL_0(disable_pop) + +/* Task aggregation */ +#define ANNOTATE_AGGREGATE_TASK(_COUNT) _ANNOTATE_CALL_1(aggregate_task, (_COUNT)) + +#endif + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* ANNOTATE_EXPAND_NULL */ + +#endif /* _ADVISOR_ANNOTATE_H_ */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/clog.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/clog.h new file mode 100644 index 0000000000000000000000000000000000000000..e12d3ecc0efbfbcfd577d5a6cef63b65587e5014 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/clog.h @@ -0,0 +1,128 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#include +#include +#include + +#define CLOG_NONE 0 +#define CLOG_FATAL 1 +#define CLOG_ERROR 2 +#define CLOG_WARNING 3 +#define CLOG_INFO 4 +#define CLOG_DEBUG 5 + +#ifndef CLOG_VISIBILITY +#if defined(__ELF__) +#define CLOG_VISIBILITY __attribute__((__visibility__("internal"))) +#elif defined(__MACH__) +#define CLOG_VISIBILITY __attribute__((__visibility__("hidden"))) +#else +#define CLOG_VISIBILITY +#endif +#endif + +#ifndef CLOG_ARGUMENTS_FORMAT +#if defined(__GNUC__) +#define CLOG_ARGUMENTS_FORMAT __attribute__((__format__(__printf__, 1, 2))) +#else +#define CLOG_ARGUMENTS_FORMAT +#endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +CLOG_VISIBILITY void clog_vlog_debug( + const char* module, + const char* format, + va_list args); +CLOG_VISIBILITY void clog_vlog_info( + const char* module, + const char* format, + va_list args); +CLOG_VISIBILITY void clog_vlog_warning( + const char* module, + const char* format, + va_list args); +CLOG_VISIBILITY void clog_vlog_error( + const char* module, + const char* format, + va_list args); +CLOG_VISIBILITY void clog_vlog_fatal( + const char* module, + const char* format, + va_list args); + +#define CLOG_DEFINE_LOG_DEBUG(log_debug_function_name, module, level) \ + CLOG_ARGUMENTS_FORMAT \ + inline static void log_debug_function_name(const char* format, ...) { \ + if (level >= CLOG_DEBUG) { \ + va_list args; \ + va_start(args, format); \ + clog_vlog_debug(module, format, args); \ + va_end(args); \ + } \ + } + +#define CLOG_DEFINE_LOG_INFO(log_info_function_name, module, level) \ + CLOG_ARGUMENTS_FORMAT \ + inline static void log_info_function_name(const char* format, ...) { \ + if (level >= CLOG_INFO) { \ + va_list args; \ + va_start(args, format); \ + clog_vlog_info(module, format, args); \ + va_end(args); \ + } \ + } + +#define CLOG_DEFINE_LOG_WARNING(log_warning_function_name, module, level) \ + CLOG_ARGUMENTS_FORMAT \ + inline static void log_warning_function_name(const char* format, ...) { \ + if (level >= CLOG_WARNING) { \ + va_list args; \ + va_start(args, format); \ + clog_vlog_warning(module, format, args); \ + va_end(args); \ + } \ + } + +#define CLOG_DEFINE_LOG_ERROR(log_error_function_name, module, level) \ + CLOG_ARGUMENTS_FORMAT \ + inline static void log_error_function_name(const char* format, ...) { \ + if (level >= CLOG_ERROR) { \ + va_list args; \ + va_start(args, format); \ + clog_vlog_error(module, format, args); \ + va_end(args); \ + } \ + } + +#define CLOG_DEFINE_LOG_FATAL(log_fatal_function_name, module, level) \ + CLOG_ARGUMENTS_FORMAT \ + inline static void log_fatal_function_name(const char* format, ...) { \ + if (level >= CLOG_FATAL) { \ + va_list args; \ + va_start(args, format); \ + clog_vlog_fatal(module, format, args); \ + va_end(args); \ + } \ + abort(); \ + } + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/cpuinfo.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/cpuinfo.h new file mode 100644 index 0000000000000000000000000000000000000000..3b2429f42848c6d971003d3f24a0a00eda806ade --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/cpuinfo.h @@ -0,0 +1,2366 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#pragma once +#ifndef CPUINFO_H +#define CPUINFO_H + +#ifndef __cplusplus +#include +#endif + +#ifdef __APPLE__ +#include +#endif + +#include + +/* Identify architecture and define corresponding macro */ + +#if defined(__i386__) || defined(__i486__) || defined(__i586__) || defined(__i686__) || defined(_M_IX86) +#define CPUINFO_ARCH_X86 1 +#endif + +#if defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) +#define CPUINFO_ARCH_X86_64 1 +#endif + +#if defined(__arm__) || defined(_M_ARM) +#define CPUINFO_ARCH_ARM 1 +#endif + +#if defined(__aarch64__) || defined(_M_ARM64) +#define CPUINFO_ARCH_ARM64 1 +#endif + +#if defined(__PPC64__) || defined(__powerpc64__) || defined(_ARCH_PPC64) +#define CPUINFO_ARCH_PPC64 1 +#endif + +#if defined(__asmjs__) +#define CPUINFO_ARCH_ASMJS 1 +#endif + +#if defined(__wasm__) +#if defined(__wasm_simd128__) +#define CPUINFO_ARCH_WASMSIMD 1 +#else +#define CPUINFO_ARCH_WASM 1 +#endif +#endif + +#if defined(__riscv) +#if (__riscv_xlen == 32) +#define CPUINFO_ARCH_RISCV32 1 +#elif (__riscv_xlen == 64) +#define CPUINFO_ARCH_RISCV64 1 +#endif +#endif + +/* Define other architecture-specific macros as 0 */ + +#ifndef CPUINFO_ARCH_X86 +#define CPUINFO_ARCH_X86 0 +#endif + +#ifndef CPUINFO_ARCH_X86_64 +#define CPUINFO_ARCH_X86_64 0 +#endif + +#ifndef CPUINFO_ARCH_ARM +#define CPUINFO_ARCH_ARM 0 +#endif + +#ifndef CPUINFO_ARCH_ARM64 +#define CPUINFO_ARCH_ARM64 0 +#endif + +#ifndef CPUINFO_ARCH_PPC64 +#define CPUINFO_ARCH_PPC64 0 +#endif + +#ifndef CPUINFO_ARCH_ASMJS +#define CPUINFO_ARCH_ASMJS 0 +#endif + +#ifndef CPUINFO_ARCH_WASM +#define CPUINFO_ARCH_WASM 0 +#endif + +#ifndef CPUINFO_ARCH_WASMSIMD +#define CPUINFO_ARCH_WASMSIMD 0 +#endif + +#ifndef CPUINFO_ARCH_RISCV32 +#define CPUINFO_ARCH_RISCV32 0 +#endif + +#ifndef CPUINFO_ARCH_RISCV64 +#define CPUINFO_ARCH_RISCV64 0 +#endif + +#if CPUINFO_ARCH_X86 && defined(_MSC_VER) +#define CPUINFO_ABI __cdecl +#elif CPUINFO_ARCH_X86 && defined(__GNUC__) +#define CPUINFO_ABI __attribute__((__cdecl__)) +#else +#define CPUINFO_ABI +#endif + +#define CPUINFO_CACHE_UNIFIED 0x00000001 +#define CPUINFO_CACHE_INCLUSIVE 0x00000002 +#define CPUINFO_CACHE_COMPLEX_INDEXING 0x00000004 + +struct cpuinfo_cache { + /** Cache size in bytes */ + uint32_t size; + /** Number of ways of associativity */ + uint32_t associativity; + /** Number of sets */ + uint32_t sets; + /** Number of partitions */ + uint32_t partitions; + /** Line size in bytes */ + uint32_t line_size; + /** + * Binary characteristics of the cache (unified cache, inclusive cache, + * cache with complex indexing). + * + * @see CPUINFO_CACHE_UNIFIED, CPUINFO_CACHE_INCLUSIVE, + * CPUINFO_CACHE_COMPLEX_INDEXING + */ + uint32_t flags; + /** Index of the first logical processor that shares this cache */ + uint32_t processor_start; + /** Number of logical processors that share this cache */ + uint32_t processor_count; +}; + +struct cpuinfo_trace_cache { + uint32_t uops; + uint32_t associativity; +}; + +#define CPUINFO_PAGE_SIZE_4KB 0x1000 +#define CPUINFO_PAGE_SIZE_1MB 0x100000 +#define CPUINFO_PAGE_SIZE_2MB 0x200000 +#define CPUINFO_PAGE_SIZE_4MB 0x400000 +#define CPUINFO_PAGE_SIZE_16MB 0x1000000 +#define CPUINFO_PAGE_SIZE_1GB 0x40000000 + +struct cpuinfo_tlb { + uint32_t entries; + uint32_t associativity; + uint64_t pages; +}; + +/** Vendor of processor core design */ +enum cpuinfo_vendor { + /** Processor vendor is not known to the library, or the library failed + to get vendor information from the OS. */ + cpuinfo_vendor_unknown = 0, + + /* Active vendors of modern CPUs */ + + /** + * Intel Corporation. Vendor of x86, x86-64, IA64, and ARM processor + * microarchitectures. + * + * Sold its ARM design subsidiary in 2006. The last ARM processor design + * was released in 2004. + */ + cpuinfo_vendor_intel = 1, + /** Advanced Micro Devices, Inc. Vendor of x86 and x86-64 processor + microarchitectures. */ + cpuinfo_vendor_amd = 2, + /** ARM Holdings plc. Vendor of ARM and ARM64 processor + microarchitectures. */ + cpuinfo_vendor_arm = 3, + /** Qualcomm Incorporated. Vendor of ARM and ARM64 processor + microarchitectures. */ + cpuinfo_vendor_qualcomm = 4, + /** Apple Inc. Vendor of ARM and ARM64 processor microarchitectures. */ + cpuinfo_vendor_apple = 5, + /** Samsung Electronics Co., Ltd. Vendir if ARM64 processor + microarchitectures. */ + cpuinfo_vendor_samsung = 6, + /** Nvidia Corporation. Vendor of ARM64-compatible processor + microarchitectures. */ + cpuinfo_vendor_nvidia = 7, + /** MIPS Technologies, Inc. Vendor of MIPS processor microarchitectures. + */ + cpuinfo_vendor_mips = 8, + /** International Business Machines Corporation. Vendor of PowerPC + processor microarchitectures. */ + cpuinfo_vendor_ibm = 9, + /** Ingenic Semiconductor. Vendor of MIPS processor microarchitectures. + */ + cpuinfo_vendor_ingenic = 10, + /** + * VIA Technologies, Inc. Vendor of x86 and x86-64 processor + * microarchitectures. + * + * Processors are designed by Centaur Technology, a subsidiary of VIA + * Technologies. + */ + cpuinfo_vendor_via = 11, + /** Cavium, Inc. Vendor of ARM64 processor microarchitectures. */ + cpuinfo_vendor_cavium = 12, + /** Broadcom, Inc. Vendor of ARM processor microarchitectures. */ + cpuinfo_vendor_broadcom = 13, + /** Applied Micro Circuits Corporation (APM). Vendor of ARM64 processor + microarchitectures. */ + cpuinfo_vendor_apm = 14, + /** + * Huawei Technologies Co., Ltd. Vendor of ARM64 processor + * microarchitectures. + * + * Processors are designed by HiSilicon, a subsidiary of Huawei. + */ + cpuinfo_vendor_huawei = 15, + /** + * Hygon (Chengdu Haiguang Integrated Circuit Design Co., Ltd), Vendor + * of x86-64 processor microarchitectures. + * + * Processors are variants of AMD cores. + */ + cpuinfo_vendor_hygon = 16, + /** SiFive, Inc. Vendor of RISC-V processor microarchitectures. */ + cpuinfo_vendor_sifive = 17, + + /* Active vendors of embedded CPUs */ + + /** Texas Instruments Inc. Vendor of ARM processor microarchitectures. + */ + cpuinfo_vendor_texas_instruments = 30, + /** Marvell Technology Group Ltd. Vendor of ARM processor + * microarchitectures. + */ + cpuinfo_vendor_marvell = 31, + /** RDC Semiconductor Co., Ltd. Vendor of x86 processor + microarchitectures. */ + cpuinfo_vendor_rdc = 32, + /** DM&P Electronics Inc. Vendor of x86 processor microarchitectures. */ + cpuinfo_vendor_dmp = 33, + /** Motorola, Inc. Vendor of PowerPC and ARM processor + microarchitectures. */ + cpuinfo_vendor_motorola = 34, + + /* Defunct CPU vendors */ + + /** + * Transmeta Corporation. Vendor of x86 processor microarchitectures. + * + * Now defunct. The last processor design was released in 2004. + * Transmeta processors implemented VLIW ISA and used binary translation + * to execute x86 code. + */ + cpuinfo_vendor_transmeta = 50, + /** + * Cyrix Corporation. Vendor of x86 processor microarchitectures. + * + * Now defunct. The last processor design was released in 1996. + */ + cpuinfo_vendor_cyrix = 51, + /** + * Rise Technology. Vendor of x86 processor microarchitectures. + * + * Now defunct. The last processor design was released in 1999. + */ + cpuinfo_vendor_rise = 52, + /** + * National Semiconductor. Vendor of x86 processor microarchitectures. + * + * Sold its x86 design subsidiary in 1999. The last processor design was + * released in 1998. + */ + cpuinfo_vendor_nsc = 53, + /** + * Silicon Integrated Systems. Vendor of x86 processor + * microarchitectures. + * + * Sold its x86 design subsidiary in 2001. The last processor design was + * released in 2001. + */ + cpuinfo_vendor_sis = 54, + /** + * NexGen. Vendor of x86 processor microarchitectures. + * + * Now defunct. The last processor design was released in 1994. + * NexGen designed the first x86 microarchitecture which decomposed x86 + * instructions into simple microoperations. + */ + cpuinfo_vendor_nexgen = 55, + /** + * United Microelectronics Corporation. Vendor of x86 processor + * microarchitectures. + * + * Ceased x86 in the early 1990s. The last processor design was released + * in 1991. Designed U5C and U5D processors. Both are 486 level. + */ + cpuinfo_vendor_umc = 56, + /** + * Digital Equipment Corporation. Vendor of ARM processor + * microarchitecture. + * + * Sold its ARM designs in 1997. The last processor design was released + * in 1997. + */ + cpuinfo_vendor_dec = 57, +}; + +/** + * Processor microarchitecture + * + * Processors with different microarchitectures often have different instruction + * performance characteristics, and may have dramatically different pipeline + * organization. + */ +enum cpuinfo_uarch { + /** Microarchitecture is unknown, or the library failed to get + information about the microarchitecture from OS */ + cpuinfo_uarch_unknown = 0, + + /** Pentium and Pentium MMX microarchitecture. */ + cpuinfo_uarch_p5 = 0x00100100, + /** Intel Quark microarchitecture. */ + cpuinfo_uarch_quark = 0x00100101, + + /** Pentium Pro, Pentium II, and Pentium III. */ + cpuinfo_uarch_p6 = 0x00100200, + /** Pentium M. */ + cpuinfo_uarch_dothan = 0x00100201, + /** Intel Core microarchitecture. */ + cpuinfo_uarch_yonah = 0x00100202, + /** Intel Core 2 microarchitecture on 65 nm process. */ + cpuinfo_uarch_conroe = 0x00100203, + /** Intel Core 2 microarchitecture on 45 nm process. */ + cpuinfo_uarch_penryn = 0x00100204, + /** Intel Nehalem and Westmere microarchitectures (Core i3/i5/i7 1st + gen). */ + cpuinfo_uarch_nehalem = 0x00100205, + /** Intel Sandy Bridge microarchitecture (Core i3/i5/i7 2nd gen). */ + cpuinfo_uarch_sandy_bridge = 0x00100206, + /** Intel Ivy Bridge microarchitecture (Core i3/i5/i7 3rd gen). */ + cpuinfo_uarch_ivy_bridge = 0x00100207, + /** Intel Haswell microarchitecture (Core i3/i5/i7 4th gen). */ + cpuinfo_uarch_haswell = 0x00100208, + /** Intel Broadwell microarchitecture. */ + cpuinfo_uarch_broadwell = 0x00100209, + /** Intel Sky Lake microarchitecture (14 nm, including + Kaby/Coffee/Whiskey/Amber/Comet/Cascade/Cooper Lake). */ + cpuinfo_uarch_sky_lake = 0x0010020A, + /** DEPRECATED (Intel Kaby Lake microarchitecture). */ + cpuinfo_uarch_kaby_lake = 0x0010020A, + /** Intel Palm Cove microarchitecture (10 nm, Cannon Lake). */ + cpuinfo_uarch_palm_cove = 0x0010020B, + /** Intel Sunny Cove microarchitecture (10 nm, Ice Lake). */ + cpuinfo_uarch_sunny_cove = 0x0010020C, + /** Intel Willow Cove microarchitecture (10 nm, Tiger Lake). */ + cpuinfo_uarch_willow_cove = 0x0010020D, + + /** Pentium 4 with Willamette, Northwood, or Foster cores. */ + cpuinfo_uarch_willamette = 0x00100300, + /** Pentium 4 with Prescott and later cores. */ + cpuinfo_uarch_prescott = 0x00100301, + + /** Intel Atom on 45 nm process. */ + cpuinfo_uarch_bonnell = 0x00100400, + /** Intel Atom on 32 nm process. */ + cpuinfo_uarch_saltwell = 0x00100401, + /** Intel Silvermont microarchitecture (22 nm out-of-order Atom). */ + cpuinfo_uarch_silvermont = 0x00100402, + /** Intel Airmont microarchitecture (14 nm out-of-order Atom). */ + cpuinfo_uarch_airmont = 0x00100403, + /** Intel Goldmont microarchitecture (Denverton, Apollo Lake). */ + cpuinfo_uarch_goldmont = 0x00100404, + /** Intel Goldmont Plus microarchitecture (Gemini Lake). */ + cpuinfo_uarch_goldmont_plus = 0x00100405, + /** Intel Airmont microarchitecture (10 nm out-of-order Atom). */ + cpuinfo_uarch_tremont = 0x00100406, + /** Intel Gracemont microarchitecture (AlderLake N). */ + cpuinfo_uarch_gracemont = 0x00100407, + /** Intel Crestmont microarchitecture (Sierra Forest). */ + cpuinfo_uarch_crestmont = 0x00100408, + /** Intel Darkmont microarchitecture (e-core used in Clearwater Forest). */ + cpuinfo_uarch_darkmont = 0x00100409, + + /** Intel Knights Ferry HPC boards. */ + cpuinfo_uarch_knights_ferry = 0x00100500, + /** Intel Knights Corner HPC boards (aka Xeon Phi). */ + cpuinfo_uarch_knights_corner = 0x00100501, + /** Intel Knights Landing microarchitecture (second-gen MIC). */ + cpuinfo_uarch_knights_landing = 0x00100502, + /** Intel Knights Hill microarchitecture (third-gen MIC). */ + cpuinfo_uarch_knights_hill = 0x00100503, + /** Intel Knights Mill Xeon Phi. */ + cpuinfo_uarch_knights_mill = 0x00100504, + + /** Intel/Marvell XScale series. */ + cpuinfo_uarch_xscale = 0x00100600, + + /** AMD K5. */ + cpuinfo_uarch_k5 = 0x00200100, + /** AMD K6 and alike. */ + cpuinfo_uarch_k6 = 0x00200101, + /** AMD Athlon and Duron. */ + cpuinfo_uarch_k7 = 0x00200102, + /** AMD Athlon 64, Opteron 64. */ + cpuinfo_uarch_k8 = 0x00200103, + /** AMD Family 10h (Barcelona, Istambul, Magny-Cours). */ + cpuinfo_uarch_k10 = 0x00200104, + /** + * AMD Bulldozer microarchitecture + * Zambezi FX-series CPUs, Zurich, Valencia and Interlagos Opteron CPUs. + */ + cpuinfo_uarch_bulldozer = 0x00200105, + /** + * AMD Piledriver microarchitecture + * Vishera FX-series CPUs, Trinity and Richland APUs, Delhi, Seoul, Abu + * Dhabi Opteron CPUs. + */ + cpuinfo_uarch_piledriver = 0x00200106, + /** AMD Steamroller microarchitecture (Kaveri APUs). */ + cpuinfo_uarch_steamroller = 0x00200107, + /** AMD Excavator microarchitecture (Carizzo APUs). */ + cpuinfo_uarch_excavator = 0x00200108, + /** AMD Zen microarchitecture (12/14 nm Ryzen and EPYC CPUs). */ + cpuinfo_uarch_zen = 0x00200109, + /** AMD Zen 2 microarchitecture (7 nm Ryzen and EPYC CPUs). */ + cpuinfo_uarch_zen2 = 0x0020010A, + /** AMD Zen 3 microarchitecture. */ + cpuinfo_uarch_zen3 = 0x0020010B, + /** AMD Zen 4 microarchitecture. */ + cpuinfo_uarch_zen4 = 0x0020010C, + /** AMD Zen 5 microarchitecture. */ + cpuinfo_uarch_zen5 = 0x0020010D, + + /** NSC Geode and AMD Geode GX and LX. */ + cpuinfo_uarch_geode = 0x00200200, + /** AMD Bobcat mobile microarchitecture. */ + cpuinfo_uarch_bobcat = 0x00200201, + /** AMD Jaguar mobile microarchitecture. */ + cpuinfo_uarch_jaguar = 0x00200202, + /** AMD Puma mobile microarchitecture. */ + cpuinfo_uarch_puma = 0x00200203, + + /** ARM7 series. */ + cpuinfo_uarch_arm7 = 0x00300100, + /** ARM9 series. */ + cpuinfo_uarch_arm9 = 0x00300101, + /** ARM 1136, ARM 1156, ARM 1176, or ARM 11MPCore. */ + cpuinfo_uarch_arm11 = 0x00300102, + + /** ARM Cortex-A5. */ + cpuinfo_uarch_cortex_a5 = 0x00300205, + /** ARM Cortex-A7. */ + cpuinfo_uarch_cortex_a7 = 0x00300207, + /** ARM Cortex-A8. */ + cpuinfo_uarch_cortex_a8 = 0x00300208, + /** ARM Cortex-A9. */ + cpuinfo_uarch_cortex_a9 = 0x00300209, + /** ARM Cortex-A12. */ + cpuinfo_uarch_cortex_a12 = 0x00300212, + /** ARM Cortex-A15. */ + cpuinfo_uarch_cortex_a15 = 0x00300215, + /** ARM Cortex-A17. */ + cpuinfo_uarch_cortex_a17 = 0x00300217, + + /** ARM Cortex-A32. */ + cpuinfo_uarch_cortex_a32 = 0x00300332, + /** ARM Cortex-A35. */ + cpuinfo_uarch_cortex_a35 = 0x00300335, + /** ARM Cortex-A53. */ + cpuinfo_uarch_cortex_a53 = 0x00300353, + /** ARM Cortex-A55 revision 0 (restricted dual-issue capabilities + compared to revision 1+). */ + cpuinfo_uarch_cortex_a55r0 = 0x00300354, + /** ARM Cortex-A55. */ + cpuinfo_uarch_cortex_a55 = 0x00300355, + /** ARM Cortex-A57. */ + cpuinfo_uarch_cortex_a57 = 0x00300357, + /** ARM Cortex-A65. */ + cpuinfo_uarch_cortex_a65 = 0x00300365, + /** ARM Cortex-A72. */ + cpuinfo_uarch_cortex_a72 = 0x00300372, + /** ARM Cortex-A73. */ + cpuinfo_uarch_cortex_a73 = 0x00300373, + /** ARM Cortex-A75. */ + cpuinfo_uarch_cortex_a75 = 0x00300375, + /** ARM Cortex-A76. */ + cpuinfo_uarch_cortex_a76 = 0x00300376, + /** ARM Cortex-A77. */ + cpuinfo_uarch_cortex_a77 = 0x00300377, + /** ARM Cortex-A78. */ + cpuinfo_uarch_cortex_a78 = 0x00300378, + + /** ARM Neoverse N1. */ + cpuinfo_uarch_neoverse_n1 = 0x00300400, + /** ARM Neoverse E1. */ + cpuinfo_uarch_neoverse_e1 = 0x00300401, + /** ARM Neoverse V1. */ + cpuinfo_uarch_neoverse_v1 = 0x00300402, + /** ARM Neoverse N2. */ + cpuinfo_uarch_neoverse_n2 = 0x00300403, + /** ARM Neoverse V2. */ + cpuinfo_uarch_neoverse_v2 = 0x00300404, + + /** ARM Cortex-X1. */ + cpuinfo_uarch_cortex_x1 = 0x00300501, + /** ARM Cortex-X2. */ + cpuinfo_uarch_cortex_x2 = 0x00300502, + /** ARM Cortex-X3. */ + cpuinfo_uarch_cortex_x3 = 0x00300503, + /** ARM Cortex-X4. */ + cpuinfo_uarch_cortex_x4 = 0x00300504, + /** ARM Cortex-X925. */ + cpuinfo_uarch_cortex_x925 = 0x00300505, + + /** ARM Cortex-A510. */ + cpuinfo_uarch_cortex_a510 = 0x00300551, + /** ARM Cortex-A520. */ + cpuinfo_uarch_cortex_a520 = 0x00300552, + /** ARM Cortex-A710. */ + cpuinfo_uarch_cortex_a710 = 0x00300571, + /** ARM Cortex-A715. */ + cpuinfo_uarch_cortex_a715 = 0x00300572, + /** ARM Cortex-A720. */ + cpuinfo_uarch_cortex_a720 = 0x00300573, + /** ARM Cortex-A725. */ + cpuinfo_uarch_cortex_a725 = 0x00300574, + + /** Qualcomm Scorpion. */ + cpuinfo_uarch_scorpion = 0x00400100, + /** Qualcomm Krait. */ + cpuinfo_uarch_krait = 0x00400101, + /** Qualcomm Kryo. */ + cpuinfo_uarch_kryo = 0x00400102, + /** Qualcomm Falkor. */ + cpuinfo_uarch_falkor = 0x00400103, + /** Qualcomm Saphira. */ + cpuinfo_uarch_saphira = 0x00400104, + /** Qualcomm Oryon. */ + cpuinfo_uarch_oryon = 0x00400105, + + /** Nvidia Denver. */ + cpuinfo_uarch_denver = 0x00500100, + /** Nvidia Denver 2. */ + cpuinfo_uarch_denver2 = 0x00500101, + /** Nvidia Carmel. */ + cpuinfo_uarch_carmel = 0x00500102, + + /** Samsung Exynos M1 (Exynos 8890 big cores). */ + cpuinfo_uarch_exynos_m1 = 0x00600100, + /** Samsung Exynos M2 (Exynos 8895 big cores). */ + cpuinfo_uarch_exynos_m2 = 0x00600101, + /** Samsung Exynos M3 (Exynos 9810 big cores). */ + cpuinfo_uarch_exynos_m3 = 0x00600102, + /** Samsung Exynos M4 (Exynos 9820 big cores). */ + cpuinfo_uarch_exynos_m4 = 0x00600103, + /** Samsung Exynos M5 (Exynos 9830 big cores). */ + cpuinfo_uarch_exynos_m5 = 0x00600104, + + /* Deprecated synonym for Cortex-A76 */ + cpuinfo_uarch_cortex_a76ae = 0x00300376, + /* Deprecated names for Exynos. */ + cpuinfo_uarch_mongoose_m1 = 0x00600100, + cpuinfo_uarch_mongoose_m2 = 0x00600101, + cpuinfo_uarch_meerkat_m3 = 0x00600102, + cpuinfo_uarch_meerkat_m4 = 0x00600103, + + /** Apple A6 and A6X processors. */ + cpuinfo_uarch_swift = 0x00700100, + /** Apple A7 processor. */ + cpuinfo_uarch_cyclone = 0x00700101, + /** Apple A8 and A8X processor. */ + cpuinfo_uarch_typhoon = 0x00700102, + /** Apple A9 and A9X processor. */ + cpuinfo_uarch_twister = 0x00700103, + /** Apple A10 and A10X processor. */ + cpuinfo_uarch_hurricane = 0x00700104, + /** Apple A11 processor (big cores). */ + cpuinfo_uarch_monsoon = 0x00700105, + /** Apple A11 processor (little cores). */ + cpuinfo_uarch_mistral = 0x00700106, + /** Apple A12 processor (big cores). */ + cpuinfo_uarch_vortex = 0x00700107, + /** Apple A12 processor (little cores). */ + cpuinfo_uarch_tempest = 0x00700108, + /** Apple A13 processor (big cores). */ + cpuinfo_uarch_lightning = 0x00700109, + /** Apple A13 processor (little cores). */ + cpuinfo_uarch_thunder = 0x0070010A, + /** Apple A14 / M1 processor (big cores). */ + cpuinfo_uarch_firestorm = 0x0070010B, + /** Apple A14 / M1 processor (little cores). */ + cpuinfo_uarch_icestorm = 0x0070010C, + /** Apple A15 / M2 processor (big cores). */ + cpuinfo_uarch_avalanche = 0x0070010D, + /** Apple A15 / M2 processor (little cores). */ + cpuinfo_uarch_blizzard = 0x0070010E, + /** Apple A16 processor (big cores). */ + cpuinfo_uarch_everest = 0x00700200, + /** Apple A16 processor (little cores). */ + cpuinfo_uarch_sawtooth = 0x00700201, + /** Apple A17 processor (big cores). */ + cpuinfo_uarch_coll_everest = 0x00700202, + /** Apple A17 processor (little cores). */ + cpuinfo_uarch_coll_sawtooth = 0x00700203, + /** Apple A18 processor (big cores). */ + cpuinfo_uarch_tupai_everest = 0x00700204, + /** Apple A18 processor (little cores). */ + cpuinfo_uarch_tupai_sawtooth = 0x00700205, + /** Apple A18 pro processor (big cores). */ + cpuinfo_uarch_tahiti_everest = 0x00700206, + /** Apple A18 pro processor (little cores). */ + cpuinfo_uarch_tahiti_sawtooth = 0x00700207, + + /** Cavium ThunderX. */ + cpuinfo_uarch_thunderx = 0x00800100, + /** Cavium ThunderX2 (originally Broadcom Vulkan). */ + cpuinfo_uarch_thunderx2 = 0x00800200, + + /** Marvell PJ4. */ + cpuinfo_uarch_pj4 = 0x00900100, + + /** Broadcom Brahma B15. */ + cpuinfo_uarch_brahma_b15 = 0x00A00100, + /** Broadcom Brahma B53. */ + cpuinfo_uarch_brahma_b53 = 0x00A00101, + + /** Applied Micro X-Gene. */ + cpuinfo_uarch_xgene = 0x00B00100, + + /* Hygon Dhyana (a modification of AMD Zen for Chinese market). */ + cpuinfo_uarch_dhyana = 0x01000100, + + /** HiSilicon TaiShan v110 (Huawei Kunpeng 920 series processors). */ + cpuinfo_uarch_taishan_v110 = 0x00C00100, +}; + +struct cpuinfo_processor { + /** SMT (hyperthread) ID within a core */ + uint32_t smt_id; + /** Core containing this logical processor */ + const struct cpuinfo_core* core; + /** Cluster of cores containing this logical processor */ + const struct cpuinfo_cluster* cluster; + /** Physical package containing this logical processor */ + const struct cpuinfo_package* package; +#if defined(__linux__) + /** + * Linux-specific ID for the logical processor: + * - Linux kernel exposes information about this logical processor in + * /sys/devices/system/cpu/cpu/ + * - Bit in the cpu_set_t identifies this logical processor + */ + int linux_id; +#endif +#if defined(_WIN32) || defined(__CYGWIN__) + /** Windows-specific ID for the group containing the logical processor. + */ + uint16_t windows_group_id; + /** + * Windows-specific ID of the logical processor within its group: + * - Bit in the KAFFINITY mask identifies this + * logical processor within its group. + */ + uint16_t windows_processor_id; +#endif +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + /** APIC ID (unique x86-specific ID of the logical processor) */ + uint32_t apic_id; +#endif + struct { + /** Level 1 instruction cache */ + const struct cpuinfo_cache* l1i; + /** Level 1 data cache */ + const struct cpuinfo_cache* l1d; + /** Level 2 unified or data cache */ + const struct cpuinfo_cache* l2; + /** Level 3 unified or data cache */ + const struct cpuinfo_cache* l3; + /** Level 4 unified or data cache */ + const struct cpuinfo_cache* l4; + } cache; +}; + +struct cpuinfo_core { + /** Index of the first logical processor on this core. */ + uint32_t processor_start; + /** Number of logical processors on this core */ + uint32_t processor_count; + /** Core ID within a package */ + uint32_t core_id; + /** Cluster containing this core */ + const struct cpuinfo_cluster* cluster; + /** Physical package containing this core. */ + const struct cpuinfo_package* package; + /** Vendor of the CPU microarchitecture for this core */ + enum cpuinfo_vendor vendor; + /** CPU microarchitecture for this core */ + enum cpuinfo_uarch uarch; +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + /** Value of CPUID leaf 1 EAX register for this core */ + uint32_t cpuid; +#elif CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + /** Value of Main ID Register (MIDR) for this core */ + uint32_t midr; +#endif + /** Clock rate (non-Turbo) of the core, in Hz */ + uint64_t frequency; +}; + +struct cpuinfo_cluster { + /** Index of the first logical processor in the cluster */ + uint32_t processor_start; + /** Number of logical processors in the cluster */ + uint32_t processor_count; + /** Index of the first core in the cluster */ + uint32_t core_start; + /** Number of cores on the cluster */ + uint32_t core_count; + /** Cluster ID within a package */ + uint32_t cluster_id; + /** Physical package containing the cluster */ + const struct cpuinfo_package* package; + /** CPU microarchitecture vendor of the cores in the cluster */ + enum cpuinfo_vendor vendor; + /** CPU microarchitecture of the cores in the cluster */ + enum cpuinfo_uarch uarch; +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + /** Value of CPUID leaf 1 EAX register of the cores in the cluster */ + uint32_t cpuid; +#elif CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + /** Value of Main ID Register (MIDR) of the cores in the cluster */ + uint32_t midr; +#endif + /** Clock rate (non-Turbo) of the cores in the cluster, in Hz */ + uint64_t frequency; +}; + +#define CPUINFO_PACKAGE_NAME_MAX 64 + +struct cpuinfo_package { + /** SoC or processor chip model name */ + char name[CPUINFO_PACKAGE_NAME_MAX]; + /** Index of the first logical processor on this physical package */ + uint32_t processor_start; + /** Number of logical processors on this physical package */ + uint32_t processor_count; + /** Index of the first core on this physical package */ + uint32_t core_start; + /** Number of cores on this physical package */ + uint32_t core_count; + /** Index of the first cluster of cores on this physical package */ + uint32_t cluster_start; + /** Number of clusters of cores on this physical package */ + uint32_t cluster_count; +}; + +struct cpuinfo_uarch_info { + /** Type of CPU microarchitecture */ + enum cpuinfo_uarch uarch; +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + /** Value of CPUID leaf 1 EAX register for the microarchitecture */ + uint32_t cpuid; +#elif CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + /** Value of Main ID Register (MIDR) for the microarchitecture */ + uint32_t midr; +#endif + /** Number of logical processors with the microarchitecture */ + uint32_t processor_count; + /** Number of cores with the microarchitecture */ + uint32_t core_count; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +bool CPUINFO_ABI cpuinfo_initialize(void); + +void CPUINFO_ABI cpuinfo_deinitialize(void); + +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 +/* This structure is not a part of stable API. Use cpuinfo_has_x86_* functions + * instead. */ +struct cpuinfo_x86_isa { +#if CPUINFO_ARCH_X86 + bool rdtsc; +#endif + bool rdtscp; + bool rdpid; + bool sysenter; +#if CPUINFO_ARCH_X86 + bool syscall; +#endif + bool msr; + bool clzero; + bool clflush; + bool clflushopt; + bool mwait; + bool mwaitx; +#if CPUINFO_ARCH_X86 + bool emmx; +#endif + bool fxsave; + bool xsave; +#if CPUINFO_ARCH_X86 + bool fpu; + bool mmx; + bool mmx_plus; +#endif + bool three_d_now; + bool three_d_now_plus; +#if CPUINFO_ARCH_X86 + bool three_d_now_geode; +#endif + bool prefetch; + bool prefetchw; + bool prefetchwt1; +#if CPUINFO_ARCH_X86 + bool daz; + bool sse; + bool sse2; +#endif + bool sse3; + bool ssse3; + bool sse4_1; + bool sse4_2; + bool sse4a; + bool misaligned_sse; + bool avx; + bool avxvnni; + bool fma3; + bool fma4; + bool xop; + bool f16c; + bool avx2; + bool avx512f; + bool avx512pf; + bool avx512er; + bool avx512cd; + bool avx512dq; + bool avx512bw; + bool avx512vl; + bool avx512ifma; + bool avx512vbmi; + bool avx512vbmi2; + bool avx512bitalg; + bool avx512vpopcntdq; + bool avx512vnni; + bool avx512bf16; + bool avx512fp16; + bool avx512vp2intersect; + bool avx512_4vnniw; + bool avx512_4fmaps; + bool avx10_1; + bool avx10_2; + bool amx_bf16; + bool amx_tile; + bool amx_int8; + bool amx_fp16; + bool avx_vnni_int8; + bool avx_vnni_int16; + bool avx_ne_convert; + bool hle; + bool rtm; + bool xtest; + bool mpx; +#if CPUINFO_ARCH_X86 + bool cmov; + bool cmpxchg8b; +#endif + bool cmpxchg16b; + bool clwb; + bool movbe; +#if CPUINFO_ARCH_X86_64 + bool lahf_sahf; +#endif + bool fs_gs_base; + bool lzcnt; + bool popcnt; + bool tbm; + bool bmi; + bool bmi2; + bool adx; + bool aes; + bool vaes; + bool pclmulqdq; + bool vpclmulqdq; + bool gfni; + bool rdrand; + bool rdseed; + bool sha; + bool rng; + bool ace; + bool ace2; + bool phe; + bool pmm; + bool lwp; +}; + +extern struct cpuinfo_x86_isa cpuinfo_isa; +#endif + +static inline bool cpuinfo_has_x86_rdtsc(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.rdtsc; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_rdtscp(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rdtscp; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_rdpid(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rdpid; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_clzero(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.clzero; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_mwait(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.mwait; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_mwaitx(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.mwaitx; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_fxsave(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.fxsave; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_xsave(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.xsave; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_fpu(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.fpu; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_mmx(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.mmx; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_mmx_plus(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.mmx_plus; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_3dnow(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.three_d_now; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_3dnow_plus(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.three_d_now_plus; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_3dnow_geode(void) { +#if CPUINFO_ARCH_X86_64 + return false; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return false; +#else + return cpuinfo_isa.three_d_now_geode; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_prefetch(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.prefetch; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_prefetchw(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.prefetchw; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_prefetchwt1(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.prefetchwt1; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_daz(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.daz; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sse(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.sse; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sse2(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.sse2; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sse3(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.sse3; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_ssse3(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.ssse3; +#endif +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sse4_1(void) { +#if CPUINFO_ARCH_X86_64 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.sse4_1; +#endif +#elif CPUINFO_ARCH_X86 + return cpuinfo_isa.sse4_1; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sse4_2(void) { +#if CPUINFO_ARCH_X86_64 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.sse4_2; +#endif +#elif CPUINFO_ARCH_X86 + return cpuinfo_isa.sse4_2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sse4a(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.sse4a; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_misaligned_sse(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.misaligned_sse; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avxvnni(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avxvnni; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_fma3(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.fma3; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_fma4(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.fma4; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_xop(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.xop; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_f16c(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.f16c; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx2(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512f(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512f; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512pf(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512pf; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512er(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512er; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512cd(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512cd; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512dq(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512dq; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512bw(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512bw; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512vl(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vl; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512ifma(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512ifma; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512vbmi(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vbmi; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512vbmi2(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vbmi2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512bitalg(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512bitalg; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512vpopcntdq(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vpopcntdq; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512vnni(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vnni; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512bf16(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512bf16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512fp16(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512fp16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512vp2intersect(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512vp2intersect; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512_4vnniw(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512_4vnniw; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx512_4fmaps(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx512_4fmaps; +#else + return false; +#endif +} + +/* [NOTE] Intel Advanced Matrix Extensions (AMX) detection + * + * I. AMX is a new extensions to the x86 ISA to work on matrices, consists of + * 1) 2-dimentional registers (tiles), hold sub-matrices from larger matrices in memory + * 2) Accelerator called Tile Matrix Multiply (TMUL), contains instructions operating on tiles + * + * II. Platforms that supports AMX: + * +-----------------+-----+----------+----------+----------+----------+ + * | Platforms | Gen | amx-bf16 | amx-tile | amx-int8 | amx-fp16 | + * +-----------------+-----+----------+----------+----------+----------+ + * | Sapphire Rapids | 4th | YES | YES | YES | NO | + * +-----------------+-----+----------+----------+----------+----------+ + * | Emerald Rapids | 5th | YES | YES | YES | NO | + * +-----------------+-----+----------+----------+----------+----------+ + * | Granite Rapids | 6th | YES | YES | YES | YES | + * +-----------------+-----+----------+----------+----------+----------+ + * + * Reference: https://www.intel.com/content/www/us/en/products/docs + * /accelerator-engines/advanced-matrix-extensions/overview.html + */ +static inline bool cpuinfo_has_x86_amx_bf16(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.amx_bf16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_amx_tile(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.amx_tile; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_amx_int8(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.amx_int8; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_amx_fp16(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.amx_fp16; +#else + return false; +#endif +} + +/* + * Intel AVX Vector Neural Network Instructions (VNNI) INT8 + * Supported Platfroms: Sierra Forest, Arrow Lake, Lunar Lake + */ +static inline bool cpuinfo_has_x86_avx_vnni_int8(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx_vnni_int8; +#else + return false; +#endif +} + +/* + * Intel AVX Vector Neural Network Instructions (VNNI) INT16 + * Supported Platfroms: Arrow Lake, Lunar Lake + */ +static inline bool cpuinfo_has_x86_avx_vnni_int16(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx_vnni_int16; +#else + return false; +#endif +} + +/* + * A new set of instructions, which can convert low precision floating point + * like BF16/FP16 to high precision floating point FP32, as well as convert FP32 + * elements to BF16. This instruction allows the platform to have improved AI + * capabilities and better compatibility. + * + * Supported Platforms: Sierra Forest, Arrow Lake, Lunar Lake + */ +static inline bool cpuinfo_has_x86_avx_ne_convert(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx_ne_convert; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx10_1(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx10_1; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_avx10_2(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.avx10_2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_hle(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.hle; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_rtm(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rtm; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_xtest(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.xtest; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_mpx(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.mpx; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_cmov(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 + return cpuinfo_isa.cmov; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_cmpxchg8b(void) { +#if CPUINFO_ARCH_X86_64 + return true; +#elif CPUINFO_ARCH_X86 + return cpuinfo_isa.cmpxchg8b; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_cmpxchg16b(void) { +#if CPUINFO_ARCH_X86_64 + return cpuinfo_isa.cmpxchg16b; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_clwb(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.clwb; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_movbe(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.movbe; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_lahf_sahf(void) { +#if CPUINFO_ARCH_X86 + return true; +#elif CPUINFO_ARCH_X86_64 + return cpuinfo_isa.lahf_sahf; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_lzcnt(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.lzcnt; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_popcnt(void) { +#if CPUINFO_ARCH_X86_64 +#if defined(__ANDROID__) + return true; +#else + return cpuinfo_isa.popcnt; +#endif +#elif CPUINFO_ARCH_X86 + return cpuinfo_isa.popcnt; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_tbm(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.tbm; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_bmi(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.bmi; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_bmi2(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.bmi2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_adx(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.adx; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_aes(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.aes; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_vaes(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.vaes; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_pclmulqdq(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.pclmulqdq; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_vpclmulqdq(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.vpclmulqdq; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_gfni(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.gfni; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_rdrand(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rdrand; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_rdseed(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.rdseed; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_x86_sha(void) { +#if CPUINFO_ARCH_X86 || CPUINFO_ARCH_X86_64 + return cpuinfo_isa.sha; +#else + return false; +#endif +} + +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 +/* This structure is not a part of stable API. Use cpuinfo_has_arm_* functions + * instead. */ +struct cpuinfo_arm_isa { +#if CPUINFO_ARCH_ARM + bool thumb; + bool thumb2; + bool thumbee; + bool jazelle; + bool armv5e; + bool armv6; + bool armv6k; + bool armv7; + bool armv7mp; + bool armv8; + bool idiv; + + bool vfpv2; + bool vfpv3; + bool d32; + bool fp16; + bool fma; + + bool wmmx; + bool wmmx2; + bool neon; +#endif +#if CPUINFO_ARCH_ARM64 + bool atomics; + bool bf16; + bool sve; + bool sve2; + bool i8mm; + bool sme; + bool sme2; + bool sme2p1; + bool sme_i16i32; + bool sme_bi32i32; + bool sme_b16b16; + bool sme_f16f16; + uint32_t svelen; + uint32_t smelen; +#endif + bool rdm; + bool fp16arith; + bool dot; + bool jscvt; + bool fcma; + bool fhm; + + bool aes; + bool sha1; + bool sha2; + bool pmull; + bool crc32; +}; + +extern struct cpuinfo_arm_isa cpuinfo_isa; +#endif + +static inline bool cpuinfo_has_arm_thumb(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.thumb; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_thumb2(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.thumb2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_v5e(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv5e; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_v6(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv6; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_v6k(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv6k; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_v7(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv7; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_v7mp(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.armv7mp; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_v8(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.armv8; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_idiv(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.idiv; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv2(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv3(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv3_d32(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.d32; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv3_fp16(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.fp16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv3_fp16_d32(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.fp16 && cpuinfo_isa.d32; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv4(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.fma; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_vfpv4_d32(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.vfpv3 && cpuinfo_isa.fma && cpuinfo_isa.d32; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_fp16_arith(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.fp16arith; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_bf16(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.bf16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_wmmx(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.wmmx; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_wmmx2(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.wmmx2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.neon; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_fp16(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.neon && cpuinfo_isa.fp16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_fma(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.neon && cpuinfo_isa.fma; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_v8(void) { +#if CPUINFO_ARCH_ARM64 + return true; +#elif CPUINFO_ARCH_ARM + return cpuinfo_isa.neon && cpuinfo_isa.armv8; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_atomics(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.atomics; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_rdm(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.rdm; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_fp16_arith(void) { +#if CPUINFO_ARCH_ARM + return cpuinfo_isa.neon && cpuinfo_isa.fp16arith; +#elif CPUINFO_ARCH_ARM64 + return cpuinfo_isa.fp16arith; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_fhm(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.fhm; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_dot(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.dot; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_neon_bf16(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.bf16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_jscvt(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.jscvt; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_fcma(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.fcma; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_i8mm(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.i8mm; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_aes(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.aes; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sha1(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sha1; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sha2(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sha2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_pmull(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.pmull; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_crc32(void) { +#if CPUINFO_ARCH_ARM || CPUINFO_ARCH_ARM64 + return cpuinfo_isa.crc32; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sve(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sve; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sve_bf16(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sve && cpuinfo_isa.bf16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sve2(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sve2; +#else + return false; +#endif +} + +// Function to get the max SVE vector length on ARM CPU's which support SVE. +static inline uint32_t cpuinfo_get_max_arm_sve_length(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.svelen * 8; // bytes * 8 = bit length(vector length) +#else + return 0; +#endif +} + +// Function to get the max SME vector length on ARM CPU's which support SME. +static inline uint32_t cpuinfo_get_max_arm_sme_length(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.smelen * 8; // bytes * 8 = bit length(vector length) +#else + return 0; +#endif +} + +static inline bool cpuinfo_has_arm_sme(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sme2(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme2; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sme2p1(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme2p1; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sme_i16i32(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme_i16i32; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sme_bi32i32(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme_bi32i32; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sme_b16b16(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme_b16b16; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_arm_sme_f16f16(void) { +#if CPUINFO_ARCH_ARM64 + return cpuinfo_isa.sme_f16f16; +#else + return false; +#endif +} + +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 +/* This structure is not a part of stable API. Use cpuinfo_has_riscv_* functions + * instead. */ +struct cpuinfo_riscv_isa { + /** + * Keep fields in line with the canonical order as defined by + * Section 27.11 Subset Naming Convention. + */ + /* RV32I/64I/128I Base ISA. */ + bool i; +#if CPUINFO_ARCH_RISCV32 + /* RV32E Base ISA. */ + bool e; +#endif + /* Integer Multiply/Divide Extension. */ + bool m; + /* Atomic Extension. */ + bool a; + /* Single-Precision Floating-Point Extension. */ + bool f; + /* Double-Precision Floating-Point Extension. */ + bool d; + /* Compressed Extension. */ + bool c; + /* Vector Extension. */ + bool v; +}; + +extern struct cpuinfo_riscv_isa cpuinfo_isa; +#endif + +static inline bool cpuinfo_has_riscv_i(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.i; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_e(void) { +#if CPUINFO_ARCH_RISCV32 + return cpuinfo_isa.e; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_m(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.m; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_a(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.a; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_f(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.f; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_d(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.d; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_g(void) { + // The 'G' extension is simply shorthand for 'IMAFD'. + return cpuinfo_has_riscv_i() && cpuinfo_has_riscv_m() && cpuinfo_has_riscv_a() && cpuinfo_has_riscv_f() && + cpuinfo_has_riscv_d(); +} + +static inline bool cpuinfo_has_riscv_c(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.c; +#else + return false; +#endif +} + +static inline bool cpuinfo_has_riscv_v(void) { +#if CPUINFO_ARCH_RISCV32 || CPUINFO_ARCH_RISCV64 + return cpuinfo_isa.v; +#else + return false; +#endif +} + +const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_processors(void); +const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_cores(void); +const struct cpuinfo_cluster* CPUINFO_ABI cpuinfo_get_clusters(void); +const struct cpuinfo_package* CPUINFO_ABI cpuinfo_get_packages(void); +const struct cpuinfo_uarch_info* CPUINFO_ABI cpuinfo_get_uarchs(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1i_caches(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1d_caches(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l2_caches(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l3_caches(void); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l4_caches(void); + +const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_processor(uint32_t index); +const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_core(uint32_t index); +const struct cpuinfo_cluster* CPUINFO_ABI cpuinfo_get_cluster(uint32_t index); +const struct cpuinfo_package* CPUINFO_ABI cpuinfo_get_package(uint32_t index); +const struct cpuinfo_uarch_info* CPUINFO_ABI cpuinfo_get_uarch(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1i_cache(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l1d_cache(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l2_cache(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l3_cache(uint32_t index); +const struct cpuinfo_cache* CPUINFO_ABI cpuinfo_get_l4_cache(uint32_t index); + +uint32_t CPUINFO_ABI cpuinfo_get_processors_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_cores_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_clusters_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_packages_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_uarchs_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l1i_caches_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l1d_caches_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l2_caches_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l3_caches_count(void); +uint32_t CPUINFO_ABI cpuinfo_get_l4_caches_count(void); + +/** + * Returns upper bound on cache size. + */ +uint32_t CPUINFO_ABI cpuinfo_get_max_cache_size(void); + +/** + * Identify the logical processor that executes the current thread. + * + * There is no guarantee that the thread will stay on the same logical processor + * for any time. Callers should treat the result as only a hint, and be prepared + * to handle NULL return value. + */ +const struct cpuinfo_processor* CPUINFO_ABI cpuinfo_get_current_processor(void); + +/** + * Identify the core that executes the current thread. + * + * There is no guarantee that the thread will stay on the same core for any + * time. Callers should treat the result as only a hint, and be prepared to + * handle NULL return value. + */ +const struct cpuinfo_core* CPUINFO_ABI cpuinfo_get_current_core(void); + +/** + * Identify the microarchitecture index of the core that executes the current + * thread. If the system does not support such identification, the function + * returns 0. + * + * There is no guarantee that the thread will stay on the same type of core for + * any time. Callers should treat the result as only a hint. + */ +uint32_t CPUINFO_ABI cpuinfo_get_current_uarch_index(void); + +/** + * Identify the microarchitecture index of the core that executes the current + * thread. If the system does not support such identification, the function + * returns the user-specified default value. + * + * There is no guarantee that the thread will stay on the same type of core for + * any time. Callers should treat the result as only a hint. + */ +uint32_t CPUINFO_ABI cpuinfo_get_current_uarch_index_with_default(uint32_t default_uarch_index); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* CPUINFO_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl.h new file mode 100644 index 0000000000000000000000000000000000000000..2f23ccc356d39007d0e1526a1915c9f263b353ca --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_H +#define DNNL_H + +#include "oneapi/dnnl/dnnl.h" + +#endif /* DNNL_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl.hpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl.hpp new file mode 100644 index 0000000000000000000000000000000000000000..cfeaa7ae8104ef4ee187593e8293d4fb332a1d7f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl.hpp @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_HPP +#define DNNL_HPP + +#include "oneapi/dnnl/dnnl.hpp" + +#endif /* DNNL_HPP */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_config.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_config.h new file mode 100644 index 0000000000000000000000000000000000000000..9ed9094b2e8de55cbe29f97203b0a038017bd4e9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_config.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_CONFIG_H +#define DNNL_CONFIG_H + +#include "oneapi/dnnl/dnnl_config.h" + +#endif /* DNNL_CONFIG_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_debug.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_debug.h new file mode 100644 index 0000000000000000000000000000000000000000..7542b65d3a82845d45550ac22c352532b0e65161 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_debug.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_DEBUG_H +#define DNNL_DEBUG_H + +#include "oneapi/dnnl/dnnl_debug.h" + +#endif /* DNNL_DEBUG_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_ocl.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_ocl.h new file mode 100644 index 0000000000000000000000000000000000000000..f7c5cdc04199555228ba220445acc0a6f427c77c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_ocl.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_OCL_H +#define DNNL_OCL_H + +#include "oneapi/dnnl/dnnl_ocl.h" + +#endif /* DNNL_OCL_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_ocl.hpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_ocl.hpp new file mode 100644 index 0000000000000000000000000000000000000000..55e02e842d8b1dde9b932d7bc0b22b39a21df014 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_ocl.hpp @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_OCL_HPP +#define DNNL_OCL_HPP + +#include "oneapi/dnnl/dnnl_ocl.hpp" + +#endif /* DNNL_OCL_HPP */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_sycl.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_sycl.h new file mode 100644 index 0000000000000000000000000000000000000000..3017d8ce576b0fe44ba275810cf7021f4df649a7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_sycl.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_SYCL_H +#define DNNL_SYCL_H + +#include "oneapi/dnnl/dnnl_sycl.h" + +#endif /* DNNL_SYCL_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_sycl.hpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_sycl.hpp new file mode 100644 index 0000000000000000000000000000000000000000..6001dea399240f83e06f38956fb5aee7355264d6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_sycl.hpp @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_SYCL_HPP +#define DNNL_SYCL_HPP + +#include "oneapi/dnnl/dnnl_sycl.hpp" + +#endif /* DNNL_SYCL_HPP */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_sycl_types.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_sycl_types.h new file mode 100644 index 0000000000000000000000000000000000000000..ede1cdcb3bec776331b64fb27724c72ead100d19 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_sycl_types.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_SYCL_TYPES_H +#define DNNL_SYCL_TYPES_H + +#include "oneapi/dnnl/dnnl_sycl_types.h" + +#endif /* DNNL_SYCL_TYPES_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_threadpool.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_threadpool.h new file mode 100644 index 0000000000000000000000000000000000000000..56427210e54c1516fa2453519965945042a5798c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_threadpool.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_THREADPOOL_H +#define DNNL_THREADPOOL_H + +#include "oneapi/dnnl/dnnl_threadpool.h" + +#endif /* DNNL_THREADPOOL_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_threadpool.hpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_threadpool.hpp new file mode 100644 index 0000000000000000000000000000000000000000..436afe2c58752f61287a888ef0b2af1eff290bdd --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_threadpool.hpp @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_THREADPOOL_HPP +#define DNNL_THREADPOOL_HPP + +#include "oneapi/dnnl/dnnl_threadpool.hpp" + +#endif /* DNNL_THREADPOOL_HPP */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_threadpool_iface.hpp b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_threadpool_iface.hpp new file mode 100644 index 0000000000000000000000000000000000000000..beead174e90480e97efc996ba20e0c4330e061fa --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_threadpool_iface.hpp @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_THREADPOOL_IFACE_HPP +#define DNNL_THREADPOOL_IFACE_HPP + +#include "oneapi/dnnl/dnnl_threadpool_iface.hpp" + +#endif /* DNNL_THREADPOOL_IFACE_HPP */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_types.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_types.h new file mode 100644 index 0000000000000000000000000000000000000000..35e54d4a3ae678a1674bbadb42c72c5352238743 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_types.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_TYPES_H +#define DNNL_TYPES_H + +#include "oneapi/dnnl/dnnl_types.h" + +#endif /* DNNL_TYPES_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_version.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_version.h new file mode 100644 index 0000000000000000000000000000000000000000..6a58705775014639fed51001a5069bbf7b2b7efe --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/dnnl_version.h @@ -0,0 +1,27 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/******************************************************************************* +* Copyright 2020 Intel Corporation +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*******************************************************************************/ + +#ifndef DNNL_VERSION_H +#define DNNL_VERSION_H + +#include "oneapi/dnnl/dnnl_version.h" + +#endif /* DNNL_VERSION_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/experiments-config.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/experiments-config.h new file mode 100644 index 0000000000000000000000000000000000000000..1196b00620d85b155ee587b626057d2735d4b4cf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/experiments-config.h @@ -0,0 +1,30 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +// Copyright 2023 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct xnn_experiment_config { + int dummy; // C requires that a struct or union has at least one member +}; + +struct xnn_experiment_config* xnn_get_experiment_config(); + +void xnn_experiment_enable_adaptive_avx_optimization(); + + +#ifdef __cplusplus +} // extern "C" +#endif + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/fp16.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/fp16.h new file mode 100644 index 0000000000000000000000000000000000000000..f5dc61d18ded7ef41d8e9698de12c2fe21094b81 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/fp16.h @@ -0,0 +1,16 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#pragma once +#ifndef FP16_H +#define FP16_H + +#include + +#if defined(PSIMD_H) +#include +#endif + +#endif /* FP16_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/fxdiv.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/fxdiv.h new file mode 100644 index 0000000000000000000000000000000000000000..abdd7c70f51f5cbdb1880901060e3df90f91e682 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/fxdiv.h @@ -0,0 +1,430 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#pragma once +#ifndef FXDIV_H +#define FXDIV_H + +#if defined(__cplusplus) && (__cplusplus >= 201103L) + #include + #include + #include +#elif !defined(__OPENCL_VERSION__) + #include + #include + #include +#endif + +#if defined(_MSC_VER) + #include + #if defined(_M_IX86) || defined(_M_X64) + #include + #endif +#endif + +#ifndef FXDIV_USE_INLINE_ASSEMBLY + #define FXDIV_USE_INLINE_ASSEMBLY 0 +#endif + +static inline uint64_t fxdiv_mulext_uint32_t(uint32_t a, uint32_t b) { +#if defined(_MSC_VER) && defined(_M_IX86) + return (uint64_t) __emulu((unsigned int) a, (unsigned int) b); +#else + return (uint64_t) a * (uint64_t) b; +#endif +} + +static inline uint32_t fxdiv_mulhi_uint32_t(uint32_t a, uint32_t b) { +#if defined(__OPENCL_VERSION__) + return mul_hi(a, b); +#elif defined(__CUDA_ARCH__) + return (uint32_t) __umulhi((unsigned int) a, (unsigned int) b); +#elif defined(_MSC_VER) && defined(_M_IX86) + return (uint32_t) (__emulu((unsigned int) a, (unsigned int) b) >> 32); +#elif defined(_MSC_VER) && defined(_M_ARM) + return (uint32_t) _MulUnsignedHigh((unsigned long) a, (unsigned long) b); +#else + return (uint32_t) (((uint64_t) a * (uint64_t) b) >> 32); +#endif +} + +static inline uint64_t fxdiv_mulhi_uint64_t(uint64_t a, uint64_t b) { +#if defined(__OPENCL_VERSION__) + return mul_hi(a, b); +#elif defined(__CUDA_ARCH__) + return (uint64_t) __umul64hi((unsigned long long) a, (unsigned long long) b); +#elif defined(_MSC_VER) && defined(_M_X64) + return (uint64_t) __umulh((unsigned __int64) a, (unsigned __int64) b); +#elif defined(__GNUC__) && defined(__SIZEOF_INT128__) + return (uint64_t) (((((unsigned __int128) a) * ((unsigned __int128) b))) >> 64); +#else + const uint32_t a_lo = (uint32_t) a; + const uint32_t a_hi = (uint32_t) (a >> 32); + const uint32_t b_lo = (uint32_t) b; + const uint32_t b_hi = (uint32_t) (b >> 32); + + const uint64_t t = fxdiv_mulext_uint32_t(a_hi, b_lo) + + (uint64_t) fxdiv_mulhi_uint32_t(a_lo, b_lo); + return fxdiv_mulext_uint32_t(a_hi, b_hi) + (t >> 32) + + ((fxdiv_mulext_uint32_t(a_lo, b_hi) + (uint64_t) (uint32_t) t) >> 32); +#endif +} + +static inline size_t fxdiv_mulhi_size_t(size_t a, size_t b) { +#if SIZE_MAX == UINT32_MAX + return (size_t) fxdiv_mulhi_uint32_t((uint32_t) a, (uint32_t) b); +#elif SIZE_MAX == UINT64_MAX + return (size_t) fxdiv_mulhi_uint64_t((uint64_t) a, (uint64_t) b); +#else + #error Unsupported platform +#endif +} + +struct fxdiv_divisor_uint32_t { + uint32_t value; + uint32_t m; + uint8_t s1; + uint8_t s2; +}; + +struct fxdiv_result_uint32_t { + uint32_t quotient; + uint32_t remainder; +}; + +struct fxdiv_divisor_uint64_t { + uint64_t value; + uint64_t m; + uint8_t s1; + uint8_t s2; +}; + +struct fxdiv_result_uint64_t { + uint64_t quotient; + uint64_t remainder; +}; + +struct fxdiv_divisor_size_t { + size_t value; + size_t m; + uint8_t s1; + uint8_t s2; +}; + +struct fxdiv_result_size_t { + size_t quotient; + size_t remainder; +}; + +static inline struct fxdiv_divisor_uint32_t fxdiv_init_uint32_t(uint32_t d) { + struct fxdiv_divisor_uint32_t result = { d }; + if (d == 1) { + result.m = UINT32_C(1); + result.s1 = 0; + result.s2 = 0; + } else { + #if defined(__OPENCL_VERSION__) + const uint32_t l_minus_1 = 31 - clz(d - 1); + #elif defined(__CUDA_ARCH__) + const uint32_t l_minus_1 = 31 - __clz((int) (d - 1)); + #elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64)) + unsigned long l_minus_1; + _BitScanReverse(&l_minus_1, (unsigned long) (d - 1)); + #elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) && FXDIV_USE_INLINE_ASSEMBLY + uint32_t l_minus_1; + __asm__("BSRL %[d_minus_1], %[l_minus_1]" + : [l_minus_1] "=r" (l_minus_1) + : [d_minus_1] "r" (d - 1) + : "cc"); + #elif defined(__GNUC__) + const uint32_t l_minus_1 = 31 - __builtin_clz(d - 1); + #else + /* Based on Algorithm 2 from Hacker's delight */ + + uint32_t l_minus_1 = 0; + uint32_t x = d - 1; + uint32_t y = x >> 16; + if (y != 0) { + l_minus_1 += 16; + x = y; + } + y = x >> 8; + if (y != 0) { + l_minus_1 += 8; + x = y; + } + y = x >> 4; + if (y != 0) { + l_minus_1 += 4; + x = y; + } + y = x >> 2; + if (y != 0) { + l_minus_1 += 2; + x = y; + } + if ((x & 2) != 0) { + l_minus_1 += 1; + } + #endif + uint32_t u_hi = (UINT32_C(2) << (uint32_t) l_minus_1) - d; + + /* Division of 64-bit number u_hi:UINT32_C(0) by 32-bit number d, 32-bit quotient output q */ + #if defined(__GNUC__) && defined(__i386__) && FXDIV_USE_INLINE_ASSEMBLY + uint32_t q; + __asm__("DIVL %[d]" + : "=a" (q), "+d" (u_hi) + : [d] "r" (d), "a" (0) + : "cc"); + #elif (defined(_MSC_VER) && _MSC_VER >= 1920) && !defined(__clang__) && !defined(__INTEL_COMPILER) && (defined(_M_IX86) || defined(_M_X64)) + unsigned int remainder; + const uint32_t q = (uint32_t) _udiv64((unsigned __int64) ((uint64_t) u_hi << 32), (unsigned int) d, &remainder); + #else + const uint32_t q = ((uint64_t) u_hi << 32) / d; + #endif + + result.m = q + UINT32_C(1); + result.s1 = 1; + result.s2 = (uint8_t) l_minus_1; + } + return result; +} + +static inline struct fxdiv_divisor_uint64_t fxdiv_init_uint64_t(uint64_t d) { + struct fxdiv_divisor_uint64_t result = { d }; + if (d == 1) { + result.m = UINT64_C(1); + result.s1 = 0; + result.s2 = 0; + } else { + #if defined(__OPENCL_VERSION__) + const uint32_t nlz_d = clz(d); + const uint32_t l_minus_1 = 63 - clz(d - 1); + #elif defined(__CUDA_ARCH__) + const uint32_t nlz_d = __clzll((long long) d); + const uint32_t l_minus_1 = 63 - __clzll((long long) (d - 1)); + #elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_ARM64)) + unsigned long l_minus_1; + _BitScanReverse64(&l_minus_1, (unsigned __int64) (d - 1)); + unsigned long bsr_d; + _BitScanReverse64(&bsr_d, (unsigned __int64) d); + const uint32_t nlz_d = bsr_d ^ 0x3F; + #elif defined(_MSC_VER) && (defined(_M_IX86) || defined(_M_ARM)) + const uint64_t d_minus_1 = d - 1; + const uint8_t d_is_power_of_2 = (d & d_minus_1) == 0; + unsigned long l_minus_1; + if ((uint32_t) (d_minus_1 >> 32) == 0) { + _BitScanReverse(&l_minus_1, (unsigned long) d_minus_1); + } else { + _BitScanReverse(&l_minus_1, (unsigned long) (uint32_t) (d_minus_1 >> 32)); + l_minus_1 += 32; + } + const uint32_t nlz_d = ((uint8_t) l_minus_1 ^ UINT8_C(0x3F)) - d_is_power_of_2; + #elif defined(__GNUC__) && defined(__x86_64__) && FXDIV_USE_INLINE_ASSEMBLY + uint64_t l_minus_1; + __asm__("BSRQ %[d_minus_1], %[l_minus_1]" + : [l_minus_1] "=r" (l_minus_1) + : [d_minus_1] "r" (d - 1) + : "cc"); + #elif defined(__GNUC__) + const uint32_t l_minus_1 = 63 - __builtin_clzll(d - 1); + const uint32_t nlz_d = __builtin_clzll(d); + #else + /* Based on Algorithm 2 from Hacker's delight */ + const uint64_t d_minus_1 = d - 1; + const uint32_t d_is_power_of_2 = (d & d_minus_1) == 0; + uint32_t l_minus_1 = 0; + uint32_t x = (uint32_t) d_minus_1; + uint32_t y = d_minus_1 >> 32; + if (y != 0) { + l_minus_1 += 32; + x = y; + } + y = x >> 16; + if (y != 0) { + l_minus_1 += 16; + x = y; + } + y = x >> 8; + if (y != 0) { + l_minus_1 += 8; + x = y; + } + y = x >> 4; + if (y != 0) { + l_minus_1 += 4; + x = y; + } + y = x >> 2; + if (y != 0) { + l_minus_1 += 2; + x = y; + } + if ((x & 2) != 0) { + l_minus_1 += 1; + } + const uint32_t nlz_d = (l_minus_1 ^ UINT32_C(0x3F)) - d_is_power_of_2; + #endif + uint64_t u_hi = (UINT64_C(2) << (uint32_t) l_minus_1) - d; + + /* Division of 128-bit number u_hi:UINT64_C(0) by 64-bit number d, 64-bit quotient output q */ + #if defined(__GNUC__) && defined(__x86_64__) && FXDIV_USE_INLINE_ASSEMBLY + uint64_t q; + __asm__("DIVQ %[d]" + : "=a" (q), "+d" (u_hi) + : [d] "r" (d), "a" (UINT64_C(0)) + : "cc"); + #elif 0 && defined(__GNUC__) && defined(__SIZEOF_INT128__) + /* GCC, Clang, and Intel Compiler fail to inline optimized implementation and call into support library for 128-bit division */ + const uint64_t q = (uint64_t) (((unsigned __int128) u_hi << 64) / ((unsigned __int128) d)); + #elif (defined(_MSC_VER) && _MSC_VER >= 1920) && !defined(__clang__) && !defined(__INTEL_COMPILER) && defined(_M_X64) + unsigned __int64 remainder; + const uint64_t q = (uint64_t) _udiv128((unsigned __int64) u_hi, 0, (unsigned __int64) d, &remainder); + #else + /* Implementation based on code from Hacker's delight */ + + /* Normalize divisor and shift divident left */ + d <<= nlz_d; + u_hi <<= nlz_d; + /* Break divisor up into two 32-bit digits */ + const uint64_t d_hi = (uint32_t) (d >> 32); + const uint32_t d_lo = (uint32_t) d; + + /* Compute the first quotient digit, q1 */ + uint64_t q1 = u_hi / d_hi; + uint64_t r1 = u_hi - q1 * d_hi; + + while ((q1 >> 32) != 0 || fxdiv_mulext_uint32_t((uint32_t) q1, d_lo) > (r1 << 32)) { + q1 -= 1; + r1 += d_hi; + if ((r1 >> 32) != 0) { + break; + } + } + + /* Multiply and subtract. */ + u_hi = (u_hi << 32) - q1 * d; + + /* Compute the second quotient digit, q0 */ + uint64_t q0 = u_hi / d_hi; + uint64_t r0 = u_hi - q0 * d_hi; + + while ((q0 >> 32) != 0 || fxdiv_mulext_uint32_t((uint32_t) q0, d_lo) > (r0 << 32)) { + q0 -= 1; + r0 += d_hi; + if ((r0 >> 32) != 0) { + break; + } + } + const uint64_t q = (q1 << 32) | (uint32_t) q0; + #endif + result.m = q + UINT64_C(1); + result.s1 = 1; + result.s2 = (uint8_t) l_minus_1; + } + return result; +} + +static inline struct fxdiv_divisor_size_t fxdiv_init_size_t(size_t d) { +#if SIZE_MAX == UINT32_MAX + const struct fxdiv_divisor_uint32_t uint_result = fxdiv_init_uint32_t((uint32_t) d); +#elif SIZE_MAX == UINT64_MAX + const struct fxdiv_divisor_uint64_t uint_result = fxdiv_init_uint64_t((uint64_t) d); +#else + #error Unsupported platform +#endif + struct fxdiv_divisor_size_t size_result = { + (size_t) uint_result.value, + (size_t) uint_result.m, + uint_result.s1, + uint_result.s2 + }; + return size_result; +} + +static inline uint32_t fxdiv_quotient_uint32_t(uint32_t n, const struct fxdiv_divisor_uint32_t divisor) { + const uint32_t t = fxdiv_mulhi_uint32_t(n, divisor.m); + return (t + ((n - t) >> divisor.s1)) >> divisor.s2; +} + +static inline uint64_t fxdiv_quotient_uint64_t(uint64_t n, const struct fxdiv_divisor_uint64_t divisor) { + const uint64_t t = fxdiv_mulhi_uint64_t(n, divisor.m); + return (t + ((n - t) >> divisor.s1)) >> divisor.s2; +} + +static inline size_t fxdiv_quotient_size_t(size_t n, const struct fxdiv_divisor_size_t divisor) { +#if SIZE_MAX == UINT32_MAX + const struct fxdiv_divisor_uint32_t uint32_divisor = { + (uint32_t) divisor.value, + (uint32_t) divisor.m, + divisor.s1, + divisor.s2 + }; + return fxdiv_quotient_uint32_t((uint32_t) n, uint32_divisor); +#elif SIZE_MAX == UINT64_MAX + const struct fxdiv_divisor_uint64_t uint64_divisor = { + (uint64_t) divisor.value, + (uint64_t) divisor.m, + divisor.s1, + divisor.s2 + }; + return fxdiv_quotient_uint64_t((uint64_t) n, uint64_divisor); +#else + #error Unsupported platform +#endif +} + +static inline uint32_t fxdiv_remainder_uint32_t(uint32_t n, const struct fxdiv_divisor_uint32_t divisor) { + const uint32_t quotient = fxdiv_quotient_uint32_t(n, divisor); + return n - quotient * divisor.value; +} + +static inline uint64_t fxdiv_remainder_uint64_t(uint64_t n, const struct fxdiv_divisor_uint64_t divisor) { + const uint64_t quotient = fxdiv_quotient_uint64_t(n, divisor); + return n - quotient * divisor.value; +} + +static inline size_t fxdiv_remainder_size_t(size_t n, const struct fxdiv_divisor_size_t divisor) { + const size_t quotient = fxdiv_quotient_size_t(n, divisor); + return n - quotient * divisor.value; +} + +static inline uint32_t fxdiv_round_down_uint32_t(uint32_t n, const struct fxdiv_divisor_uint32_t granularity) { + const uint32_t quotient = fxdiv_quotient_uint32_t(n, granularity); + return quotient * granularity.value; +} + +static inline uint64_t fxdiv_round_down_uint64_t(uint64_t n, const struct fxdiv_divisor_uint64_t granularity) { + const uint64_t quotient = fxdiv_quotient_uint64_t(n, granularity); + return quotient * granularity.value; +} + +static inline size_t fxdiv_round_down_size_t(size_t n, const struct fxdiv_divisor_size_t granularity) { + const size_t quotient = fxdiv_quotient_size_t(n, granularity); + return quotient * granularity.value; +} + +static inline struct fxdiv_result_uint32_t fxdiv_divide_uint32_t(uint32_t n, const struct fxdiv_divisor_uint32_t divisor) { + const uint32_t quotient = fxdiv_quotient_uint32_t(n, divisor); + const uint32_t remainder = n - quotient * divisor.value; + struct fxdiv_result_uint32_t result = { quotient, remainder }; + return result; +} + +static inline struct fxdiv_result_uint64_t fxdiv_divide_uint64_t(uint64_t n, const struct fxdiv_divisor_uint64_t divisor) { + const uint64_t quotient = fxdiv_quotient_uint64_t(n, divisor); + const uint64_t remainder = n - quotient * divisor.value; + struct fxdiv_result_uint64_t result = { quotient, remainder }; + return result; +} + +static inline struct fxdiv_result_size_t fxdiv_divide_size_t(size_t n, const struct fxdiv_divisor_size_t divisor) { + const size_t quotient = fxdiv_quotient_size_t(n, divisor); + const size_t remainder = n - quotient * divisor.value; + struct fxdiv_result_size_t result = { quotient, remainder }; + return result; +} + +#endif /* FXDIV_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/ittnotify-zca.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/ittnotify-zca.h new file mode 100644 index 0000000000000000000000000000000000000000..b67e229749262849bf6a7947afc4a1470c730c65 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/ittnotify-zca.h @@ -0,0 +1,86 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/* + Copyright (C) 2005-2019 Intel Corporation + + SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause +*/ + +/** + * Zero Cost Annotations (ZCA) + * + * Intel Compiler supports two intrinsics that could be used for code annotations + * without incurring significant run-time costs when the tools are not in use. + * Each annotation is more than a mere mark in the instruction stream. + * It can accept an expression argument like a call to a routine does. + * There are two forms of the intrinsic, with the following signatures: + * + * extern "C" void __notify_intrinsic( const char *annotation, const volatile void *tag); + * extern "C" void __notify_zc_intrinsic(const char *annotation, const volatile void *tag); + * + * The string annotation must be a compile-time constant. It specifies the type of the annotation. + * The pointer tag is computed at run time. It specifies the data associated with the annotation. + * Each intrinsic implies a compiler fence: the compiler must not move any memory + * operation across it. The reason for this restriction is that annotation might denote an + * event that must be precisely placed with respect to memory operations. + * + * The difference between the two intrinsics is that __notify_intrinsic must leave a + * probe-ready instruction sequence in the instruction stream where the instrinsic + * occurs. The __notify_zc_intrinsic does not leave such a sequence, and hence is closer to "zero cost". + **/ + +#pragma once +#include "ittnotify.h" + +#ifndef INTEL_NO_ITTNOTIFY_API +#if (defined(__INTEL_COMPILER) || defined(__INTEL_LLVM_COMPILER)) && (ITT_PLATFORM == ITT_PLATFORM_WIN || ITT_PLATFORM == ITT_PLATFORM_POSIX) +#define ITT_ENABLE_LOW_OVERHEAD_ANNOTATIONS +#else +#error Zero cost (low overhead) annotations are not supported on this platform +#endif +#endif + +/** + * Zero cost annotations for memory allocation and deallocation + **/ +#ifdef ITT_ENABLE_LOW_OVERHEAD_ANNOTATIONS +#pragma pack(push, 1) +typedef struct ___itt_zca_allocation_info { + size_t size; /*!< Size of allocated memory */ + void** ptr; /*!< Pointer to allocated memory pointer */ + int initialized; /*!< Is allocated memory initialized */ +} __itt_zca_allocation_info; +#pragma pack(pop) + +#define __itt_zca_mem_allocate_begin() __notify_intrinsic((char*)"mem_allocate_begin", 0) +#define __itt_zca_mem_allocate_end(ptr, size, init) { __itt_zca_allocation_info __itt_zca_alloc_info = { size, ptr, init }; __notify_intrinsic((char*)"mem_allocate_end", (void*)&__itt_zca_alloc_info); } +#define __itt_zca_mem_free_begin(ptr) __notify_intrinsic((char*)"mem_free_begin", (void*)ptr) +#define __itt_zca_mem_free_end() __notify_intrinsic((char*)"mem_free_end", 0) +#else +#define __itt_zca_mem_allocate_begin() +#define __itt_zca_mem_allocate_end(ptr, size, init) +#define __itt_zca_mem_free_begin(ptr) +#define __itt_zca_mem_free_end() +#endif + +/** + * Zero cost annotations for threading + **/ +#ifdef ITT_ENABLE_LOW_OVERHEAD_ANNOTATIONS +#define __itt_zca_suppress_push(id) __notify_zc_intrinsic((char*)"__itt_suppress_push", (void*)id); +#define __itt_zca_suppress_pop(id) __notify_zc_intrinsic((char*)"__itt_suppress_pop", (void*)id); +#define __itt_zca_sync_create(id) __notify_zc_intrinsic((char*)"__itt_sync_create", (void*)id) +#define __itt_zca_sync_acquired(id) __notify_zc_intrinsic((char*)"__itt_sync_acquired", (void*)id) +#define __itt_zca_sync_releasing(id) __notify_zc_intrinsic((char*)"__itt_sync_releasing", (void*)id) +#define __itt_zca_sync_destroy(id) __notify_zc_intrinsic((char*)"__itt_sync_destroy", (void*)id) +#else +#define __itt_zca_suppress_push(id) +#define __itt_zca_suppress_pop(id) +#define __itt_zca_sync_create(id) +#define __itt_zca_sync_acquired(id) +#define __itt_zca_sync_releasing(id) +#define __itt_zca_sync_destroy(id) +#endif + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/ittnotify.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/ittnotify.h new file mode 100644 index 0000000000000000000000000000000000000000..879a5b42b1598269ecacc08411fd7f58efd993b7 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/ittnotify.h @@ -0,0 +1,4670 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/* + Copyright (C) 2005-2019 Intel Corporation + + SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause +*/ +#ifndef _ITTNOTIFY_H_ +#define _ITTNOTIFY_H_ + +/** +@file +@brief Public User API functions and types +@mainpage + +The Instrumentation and Tracing Technology API (ITT API) is used to +annotate a user's program with additional information +that can be used by correctness and performance tools. The user inserts +calls in their program. Those calls generate information that is collected +at runtime, and used by Intel(R) Threading Tools. + +@section API Concepts +The following general concepts are used throughout the API. + +@subsection Unicode Support +Many API functions take character string arguments. On Windows, there +are two versions of each such function. The function name is suffixed +by W if Unicode support is enabled, and by A otherwise. Any API function +that takes a character string argument adheres to this convention. + +@subsection Conditional Compilation +Many users prefer having an option to modify ITT API code when linking it +inside their runtimes. ITT API header file provides a mechanism to replace +ITT API function names inside your code with empty strings. To do this, +define the macros INTEL_NO_ITTNOTIFY_API during compilation and remove the +static library from the linker script. + +@subsection Domains +[see domains] +Domains provide a way to separate notification for different modules or +libraries in a program. Domains are specified by dotted character strings, +e.g. TBB.Internal.Control. + +A mechanism (to be specified) is provided to enable and disable +domains. By default, all domains are enabled. +@subsection Named Entities and Instances +Named entities (frames, regions, tasks, and markers) communicate +information about the program to the analysis tools. A named entity often +refers to a section of program code, or to some set of logical concepts +that the programmer wants to group together. + +Named entities relate to the programmer's static view of the program. When +the program actually executes, many instances of a given named entity +may be created. + +The API annotations denote instances of named entities. The actual +named entities are displayed using the analysis tools. In other words, +the named entities come into existence when instances are created. + +Instances of named entities may have instance identifiers (IDs). Some +API calls use instance identifiers to create relationships between +different instances of named entities. Other API calls associate data +with instances of named entities. + +Some named entities must always have instance IDs. In particular, regions +and frames always have IDs. Task and markers need IDs only if the ID is +needed in another API call (such as adding a relation or metadata). + +The lifetime of instance IDs is distinct from the lifetime of +instances. This allows various relationships to be specified separate +from the actual execution of instances. This flexibility comes at the +expense of extra API calls. + +The same ID may not be reused for different instances, unless a previous +[ref] __itt_id_destroy call for that ID has been issued. +*/ + +/** @cond exclude_from_documentation */ +#ifndef ITT_OS_WIN +# define ITT_OS_WIN 1 +#endif /* ITT_OS_WIN */ + +#ifndef ITT_OS_LINUX +# define ITT_OS_LINUX 2 +#endif /* ITT_OS_LINUX */ + +#ifndef ITT_OS_MAC +# define ITT_OS_MAC 3 +#endif /* ITT_OS_MAC */ + +#ifndef ITT_OS_FREEBSD +# define ITT_OS_FREEBSD 4 +#endif /* ITT_OS_FREEBSD */ + +#ifndef ITT_OS_OPENBSD +# define ITT_OS_OPENBSD 5 +#endif /* ITT_OS_OPENBSD */ + +#ifndef ITT_OS +# if defined WIN32 || defined _WIN32 +# define ITT_OS ITT_OS_WIN +# elif defined( __APPLE__ ) && defined( __MACH__ ) +# define ITT_OS ITT_OS_MAC +# elif defined( __FreeBSD__ ) +# define ITT_OS ITT_OS_FREEBSD +# elif defined( __OpenBSD__) +# define ITT_OS ITT_OS_OPENBSD +# else +# define ITT_OS ITT_OS_LINUX +# endif +#endif /* ITT_OS */ + +#ifndef ITT_PLATFORM_WIN +# define ITT_PLATFORM_WIN 1 +#endif /* ITT_PLATFORM_WIN */ + +#ifndef ITT_PLATFORM_POSIX +# define ITT_PLATFORM_POSIX 2 +#endif /* ITT_PLATFORM_POSIX */ + +#ifndef ITT_PLATFORM_MAC +# define ITT_PLATFORM_MAC 3 +#endif /* ITT_PLATFORM_MAC */ + +#ifndef ITT_PLATFORM_FREEBSD +# define ITT_PLATFORM_FREEBSD 4 +#endif /* ITT_PLATFORM_FREEBSD */ + +#ifndef ITT_PLATFORM_OPENBSD +# define ITT_PLATFORM_OPENBSD 5 +#endif /* ITT_PLATFORM_OPENBSD */ + +#ifndef ITT_PLATFORM +# if ITT_OS==ITT_OS_WIN +# define ITT_PLATFORM ITT_PLATFORM_WIN +# elif ITT_OS==ITT_OS_MAC +# define ITT_PLATFORM ITT_PLATFORM_MAC +# elif ITT_OS==ITT_OS_FREEBSD +# define ITT_PLATFORM ITT_PLATFORM_FREEBSD +# elif ITT_OS==ITT_OS_OPENBSD +# define ITT_PLATFORM ITT_PLATFORM_OPENBSD +# else +# define ITT_PLATFORM ITT_PLATFORM_POSIX +# endif +#endif /* ITT_PLATFORM */ + +#if defined(_UNICODE) && !defined(UNICODE) +#define UNICODE +#endif + +#include +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#include +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#include +#if defined(UNICODE) || defined(_UNICODE) +#include +#endif /* UNICODE || _UNICODE */ +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#ifndef ITTAPI_CDECL +# if ITT_PLATFORM==ITT_PLATFORM_WIN +# define ITTAPI_CDECL __cdecl +# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# if defined _M_IX86 || defined __i386__ +# define ITTAPI_CDECL __attribute__ ((cdecl)) +# else /* _M_IX86 || __i386__ */ +# define ITTAPI_CDECL /* actual only on x86 platform */ +# endif /* _M_IX86 || __i386__ */ +# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* ITTAPI_CDECL */ + +#ifndef STDCALL +# if ITT_PLATFORM==ITT_PLATFORM_WIN +# define STDCALL __stdcall +# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# if defined _M_IX86 || defined __i386__ +# define STDCALL __attribute__ ((stdcall)) +# else /* _M_IX86 || __i386__ */ +# define STDCALL /* supported only on x86 platform */ +# endif /* _M_IX86 || __i386__ */ +# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* STDCALL */ + +#define ITTAPI ITTAPI_CDECL +#define LIBITTAPI ITTAPI_CDECL + +/* TODO: Temporary for compatibility! */ +#define ITTAPI_CALL ITTAPI_CDECL +#define LIBITTAPI_CALL ITTAPI_CDECL + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +/* use __forceinline (VC++ specific) */ +#if defined(__MINGW32__) && !defined(__cplusplus) +#define ITT_INLINE static __inline__ __attribute__((__always_inline__,__gnu_inline__)) +#else +#define ITT_INLINE static __forceinline +#endif /* __MINGW32__ */ + +#define ITT_INLINE_ATTRIBUTE /* nothing */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +/* + * Generally, functions are not inlined unless optimization is specified. + * For functions declared inline, this attribute inlines the function even + * if no optimization level was specified. + */ +#ifdef __STRICT_ANSI__ +#define ITT_INLINE static +#define ITT_INLINE_ATTRIBUTE __attribute__((unused)) +#else /* __STRICT_ANSI__ */ +#define ITT_INLINE static inline +#define ITT_INLINE_ATTRIBUTE __attribute__((always_inline, unused)) +#endif /* __STRICT_ANSI__ */ +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +/** @endcond */ + +#ifdef INTEL_ITTNOTIFY_ENABLE_LEGACY +# if ITT_PLATFORM==ITT_PLATFORM_WIN +# pragma message("WARNING!!! Deprecated API is used. Please undefine INTEL_ITTNOTIFY_ENABLE_LEGACY macro") +# else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# warning "Deprecated API is used. Please undefine INTEL_ITTNOTIFY_ENABLE_LEGACY macro" +# endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +# include "legacy/ittnotify.h" +#endif /* INTEL_ITTNOTIFY_ENABLE_LEGACY */ + +/** @cond exclude_from_documentation */ +/* Helper macro for joining tokens */ +#define ITT_JOIN_AUX(p,n) p##n +#define ITT_JOIN(p,n) ITT_JOIN_AUX(p,n) + +#ifdef ITT_MAJOR +#undef ITT_MAJOR +#endif +#ifdef ITT_MINOR +#undef ITT_MINOR +#endif +#define ITT_MAJOR 3 +#define ITT_MINOR 0 + +/* Standard versioning of a token with major and minor version numbers */ +#define ITT_VERSIONIZE(x) \ + ITT_JOIN(x, \ + ITT_JOIN(_, \ + ITT_JOIN(ITT_MAJOR, \ + ITT_JOIN(_, ITT_MINOR)))) + +#ifndef INTEL_ITTNOTIFY_PREFIX +# define INTEL_ITTNOTIFY_PREFIX __itt_ +#endif /* INTEL_ITTNOTIFY_PREFIX */ +#ifndef INTEL_ITTNOTIFY_POSTFIX +# define INTEL_ITTNOTIFY_POSTFIX _ptr_ +#endif /* INTEL_ITTNOTIFY_POSTFIX */ + +#define ITTNOTIFY_NAME_AUX(n) ITT_JOIN(INTEL_ITTNOTIFY_PREFIX,n) +#define ITTNOTIFY_NAME(n) ITT_VERSIONIZE(ITTNOTIFY_NAME_AUX(ITT_JOIN(n,INTEL_ITTNOTIFY_POSTFIX))) + +#define ITTNOTIFY_VOID(n) (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n) +#define ITTNOTIFY_DATA(n) (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n) + +#define ITTNOTIFY_VOID_D0(n,d) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d) +#define ITTNOTIFY_VOID_D1(n,d,x) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x) +#define ITTNOTIFY_VOID_D2(n,d,x,y) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y) +#define ITTNOTIFY_VOID_D3(n,d,x,y,z) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z) +#define ITTNOTIFY_VOID_D4(n,d,x,y,z,a) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) +#define ITTNOTIFY_VOID_D5(n,d,x,y,z,a,b) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) +#define ITTNOTIFY_VOID_D6(n,d,x,y,z,a,b,c) (d == NULL) ? (void)0 : (!(d)->flags) ? (void)0 : (!ITTNOTIFY_NAME(n)) ? (void)0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) +#define ITTNOTIFY_DATA_D0(n,d) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d) +#define ITTNOTIFY_DATA_D1(n,d,x) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x) +#define ITTNOTIFY_DATA_D2(n,d,x,y) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y) +#define ITTNOTIFY_DATA_D3(n,d,x,y,z) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z) +#define ITTNOTIFY_DATA_D4(n,d,x,y,z,a) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a) +#define ITTNOTIFY_DATA_D5(n,d,x,y,z,a,b) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b) +#define ITTNOTIFY_DATA_D6(n,d,x,y,z,a,b,c) (d == NULL) ? 0 : (!(d)->flags) ? 0 : (!ITTNOTIFY_NAME(n)) ? 0 : ITTNOTIFY_NAME(n)(d,x,y,z,a,b,c) + +#ifdef ITT_STUB +#undef ITT_STUB +#endif +#ifdef ITT_STUBV +#undef ITT_STUBV +#endif +#define ITT_STUBV(api,type,name,args) \ + typedef type (api* ITT_JOIN(ITTNOTIFY_NAME(name),_t)) args; \ + extern ITT_JOIN(ITTNOTIFY_NAME(name),_t) ITTNOTIFY_NAME(name); +#define ITT_STUB ITT_STUBV +/** @endcond */ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** @cond exclude_from_gpa_documentation */ +/** + * @defgroup public Public API + * @{ + * @} + */ + +/** + * @defgroup control Collection Control + * @ingroup public + * General behavior: application continues to run, but no profiling information is being collected + * + * Pausing occurs not only for the current thread but for all process as well as spawned processes + * - Intel(R) Parallel Inspector and Intel(R) Inspector XE: + * - Does not analyze or report errors that involve memory access. + * - Other errors are reported as usual. Pausing data collection in + * Intel(R) Parallel Inspector and Intel(R) Inspector XE + * only pauses tracing and analyzing memory access. + * It does not pause tracing or analyzing threading APIs. + * . + * - Intel(R) VTune(TM) Profiler: + * - Does continue to record when new threads are started. + * . + * - Other effects: + * - Possible reduction of runtime overhead. + * . + * @{ + */ +/** @brief Pause collection */ +void ITTAPI __itt_pause(void); +/** @brief Resume collection */ +void ITTAPI __itt_resume(void); +/** @brief Detach collection */ +void ITTAPI __itt_detach(void); + +/** + * @enum __itt_collection_scope + * @brief Enumerator for collection scopes + */ +typedef enum { + __itt_collection_scope_host = 1 << 0, + __itt_collection_scope_offload = 1 << 1, + __itt_collection_scope_all = 0x7FFFFFFF +} __itt_collection_scope; + +/** @brief Pause scoped collection */ +void ITTAPI __itt_pause_scoped(__itt_collection_scope); +/** @brief Resume scoped collection */ +void ITTAPI __itt_resume_scoped(__itt_collection_scope); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, pause, (void)) +ITT_STUBV(ITTAPI, void, pause_scoped, (__itt_collection_scope)) +ITT_STUBV(ITTAPI, void, resume, (void)) +ITT_STUBV(ITTAPI, void, resume_scoped, (__itt_collection_scope)) +ITT_STUBV(ITTAPI, void, detach, (void)) +#define __itt_pause ITTNOTIFY_VOID(pause) +#define __itt_pause_ptr ITTNOTIFY_NAME(pause) +#define __itt_pause_scoped ITTNOTIFY_VOID(pause_scoped) +#define __itt_pause_scoped_ptr ITTNOTIFY_NAME(pause_scoped) +#define __itt_resume ITTNOTIFY_VOID(resume) +#define __itt_resume_ptr ITTNOTIFY_NAME(resume) +#define __itt_resume_scoped ITTNOTIFY_VOID(resume_scoped) +#define __itt_resume_scoped_ptr ITTNOTIFY_NAME(resume_scoped) +#define __itt_detach ITTNOTIFY_VOID(detach) +#define __itt_detach_ptr ITTNOTIFY_NAME(detach) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_pause() +#define __itt_pause_ptr 0 +#define __itt_pause_scoped(scope) +#define __itt_pause_scoped_ptr 0 +#define __itt_resume() +#define __itt_resume_ptr 0 +#define __itt_resume_scoped(scope) +#define __itt_resume_scoped_ptr 0 +#define __itt_detach() +#define __itt_detach_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_pause_ptr 0 +#define __itt_pause_scoped_ptr 0 +#define __itt_resume_ptr 0 +#define __itt_resume_scoped_ptr 0 +#define __itt_detach_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} control group */ +/** @endcond */ + +/** + * @defgroup Intel Processor Trace control + * API from this group provides control over collection and analysis of Intel Processor Trace (Intel PT) data + * Information about Intel Processor Trace technology can be found here (Volume 3 chapter 35): + * https://software.intel.com/sites/default/files/managed/39/c5/325462-sdm-vol-1-2abcd-3abcd.pdf + * Use this API to mark particular code regions for loading detailed performance statistics. + * This mode makes your analysis faster and more accurate. + * @{ +*/ +typedef unsigned char __itt_pt_region; + +/** + * @brief function saves a region name marked with Intel PT API and returns a region id. + * Only 7 names can be registered. Attempts to register more names will be ignored and a region id with auto names will be returned. + * For automatic naming of regions pass NULL as function parameter +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_pt_region ITTAPI __itt_pt_region_createA(const char *name); +__itt_pt_region ITTAPI __itt_pt_region_createW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_pt_region_create __itt_pt_region_createW +#else /* UNICODE */ +# define __itt_pt_region_create __itt_pt_region_createA +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_pt_region ITTAPI __itt_pt_region_create(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_pt_region, pt_region_createA, (const char *name)) +ITT_STUB(ITTAPI, __itt_pt_region, pt_region_createW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_pt_region, pt_region_create, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_pt_region_createA ITTNOTIFY_DATA(pt_region_createA) +#define __itt_pt_region_createA_ptr ITTNOTIFY_NAME(pt_region_createA) +#define __itt_pt_region_createW ITTNOTIFY_DATA(pt_region_createW) +#define __itt_pt_region_createW_ptr ITTNOTIFY_NAME(pt_region_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_pt_region_create ITTNOTIFY_DATA(pt_region_create) +#define __itt_pt_region_create_ptr ITTNOTIFY_NAME(pt_region_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_pt_region_createA(name) (__itt_pt_region)0 +#define __itt_pt_region_createA_ptr 0 +#define __itt_pt_region_createW(name) (__itt_pt_region)0 +#define __itt_pt_region_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_pt_region_create(name) (__itt_pt_region)0 +#define __itt_pt_region_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_pt_region_createA_ptr 0 +#define __itt_pt_region_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_pt_region_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief function contains a special code pattern identified on the post-processing stage and + * marks the beginning of a code region targeted for Intel PT analysis + * @param[in] region - region id, 0 <= region < 8 +*/ +void __itt_mark_pt_region_begin(__itt_pt_region region); +/** + * @brief function contains a special code pattern identified on the post-processing stage and + * marks the end of a code region targeted for Intel PT analysis + * @param[in] region - region id, 0 <= region < 8 +*/ +void __itt_mark_pt_region_end(__itt_pt_region region); +/** @} Intel PT control group*/ + +/** + * @defgroup threads Threads + * @ingroup public + * Give names to threads + * @{ + */ +/** + * @brief Sets thread name of calling thread + * @param[in] name - name of thread + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_thread_set_nameA(const char *name); +void ITTAPI __itt_thread_set_nameW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_thread_set_name __itt_thread_set_nameW +# define __itt_thread_set_name_ptr __itt_thread_set_nameW_ptr +#else /* UNICODE */ +# define __itt_thread_set_name __itt_thread_set_nameA +# define __itt_thread_set_name_ptr __itt_thread_set_nameA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_thread_set_name(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, thread_set_nameA, (const char *name)) +ITT_STUBV(ITTAPI, void, thread_set_nameW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, thread_set_name, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_thread_set_nameA ITTNOTIFY_VOID(thread_set_nameA) +#define __itt_thread_set_nameA_ptr ITTNOTIFY_NAME(thread_set_nameA) +#define __itt_thread_set_nameW ITTNOTIFY_VOID(thread_set_nameW) +#define __itt_thread_set_nameW_ptr ITTNOTIFY_NAME(thread_set_nameW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_thread_set_name ITTNOTIFY_VOID(thread_set_name) +#define __itt_thread_set_name_ptr ITTNOTIFY_NAME(thread_set_name) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_thread_set_nameA(name) +#define __itt_thread_set_nameA_ptr 0 +#define __itt_thread_set_nameW(name) +#define __itt_thread_set_nameW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_thread_set_name(name) +#define __itt_thread_set_name_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_thread_set_nameA_ptr 0 +#define __itt_thread_set_nameW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_thread_set_name_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_gpa_documentation */ + +/** + * @brief Mark current thread as ignored from this point on, for the duration of its existence. + */ +void ITTAPI __itt_thread_ignore(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, thread_ignore, (void)) +#define __itt_thread_ignore ITTNOTIFY_VOID(thread_ignore) +#define __itt_thread_ignore_ptr ITTNOTIFY_NAME(thread_ignore) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_thread_ignore() +#define __itt_thread_ignore_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_thread_ignore_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} threads group */ + +/** + * @defgroup suppress Error suppression + * @ingroup public + * General behavior: application continues to run, but errors are suppressed + * + * @{ + */ + +/*****************************************************************//** + * @name group of functions used for error suppression in correctness tools + *********************************************************************/ +/** @{ */ +/** + * @hideinitializer + * @brief possible value for suppression mask + */ +#define __itt_suppress_all_errors 0x7fffffff + +/** + * @hideinitializer + * @brief possible value for suppression mask (suppresses errors from threading analysis) + */ +#define __itt_suppress_threading_errors 0x000000ff + +/** + * @hideinitializer + * @brief possible value for suppression mask (suppresses errors from memory analysis) + */ +#define __itt_suppress_memory_errors 0x0000ff00 + +/** + * @brief Start suppressing errors identified in mask on this thread + */ +void ITTAPI __itt_suppress_push(unsigned int mask); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, suppress_push, (unsigned int mask)) +#define __itt_suppress_push ITTNOTIFY_VOID(suppress_push) +#define __itt_suppress_push_ptr ITTNOTIFY_NAME(suppress_push) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_suppress_push(mask) +#define __itt_suppress_push_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_suppress_push_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Undo the effects of the matching call to __itt_suppress_push + */ +void ITTAPI __itt_suppress_pop(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, suppress_pop, (void)) +#define __itt_suppress_pop ITTNOTIFY_VOID(suppress_pop) +#define __itt_suppress_pop_ptr ITTNOTIFY_NAME(suppress_pop) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_suppress_pop() +#define __itt_suppress_pop_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_suppress_pop_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @enum __itt_suppress_mode + * @brief Enumerator for the suppressing modes + */ +typedef enum __itt_suppress_mode { + __itt_unsuppress_range, + __itt_suppress_range +} __itt_suppress_mode_t; + +/** + * @enum __itt_collection_state + * @brief Enumerator for collection state. + */ +typedef enum { + __itt_collection_uninitialized = 0, /* uninitialized */ + __itt_collection_init_fail = 1, /* failed to init */ + __itt_collection_collector_absent = 2, /* non work state collector is absent */ + __itt_collection_collector_exists = 3, /* work state collector exists */ + __itt_collection_init_successful = 4 /* success to init */ +} __itt_collection_state; + +/** + * @brief Mark a range of memory for error suppression or unsuppression for error types included in mask + */ +void ITTAPI __itt_suppress_mark_range(__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, suppress_mark_range, (__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size)) +#define __itt_suppress_mark_range ITTNOTIFY_VOID(suppress_mark_range) +#define __itt_suppress_mark_range_ptr ITTNOTIFY_NAME(suppress_mark_range) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_suppress_mark_range(mask) +#define __itt_suppress_mark_range_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_suppress_mark_range_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Undo the effect of a matching call to __itt_suppress_mark_range. If not matching + * call is found, nothing is changed. + */ +void ITTAPI __itt_suppress_clear_range(__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, suppress_clear_range, (__itt_suppress_mode_t mode, unsigned int mask, void * address, size_t size)) +#define __itt_suppress_clear_range ITTNOTIFY_VOID(suppress_clear_range) +#define __itt_suppress_clear_range_ptr ITTNOTIFY_NAME(suppress_clear_range) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_suppress_clear_range(mask) +#define __itt_suppress_clear_range_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_suppress_clear_range_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} */ +/** @} suppress group */ + +/** + * @defgroup sync Synchronization + * @ingroup public + * Indicate user-written synchronization code + * @{ + */ +/** + * @hideinitializer + * @brief possible value of attribute argument for sync object type + */ +#define __itt_attr_barrier 1 + +/** + * @hideinitializer + * @brief possible value of attribute argument for sync object type + */ +#define __itt_attr_mutex 2 + +/** +@brief Name a synchronization object +@param[in] addr Handle for the synchronization object. You should +use a real address to uniquely identify the synchronization object. +@param[in] objtype null-terminated object type string. If NULL is +passed, the name will be "User Synchronization". +@param[in] objname null-terminated object name string. If NULL, +no name will be assigned to the object. +@param[in] attribute one of [#__itt_attr_barrier, #__itt_attr_mutex] + */ + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_sync_createA(void *addr, const char *objtype, const char *objname, int attribute); +void ITTAPI __itt_sync_createW(void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_sync_create __itt_sync_createW +# define __itt_sync_create_ptr __itt_sync_createW_ptr +#else /* UNICODE */ +# define __itt_sync_create __itt_sync_createA +# define __itt_sync_create_ptr __itt_sync_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_sync_create (void *addr, const char *objtype, const char *objname, int attribute); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, sync_createA, (void *addr, const char *objtype, const char *objname, int attribute)) +ITT_STUBV(ITTAPI, void, sync_createW, (void *addr, const wchar_t *objtype, const wchar_t *objname, int attribute)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, sync_create, (void *addr, const char* objtype, const char* objname, int attribute)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_createA ITTNOTIFY_VOID(sync_createA) +#define __itt_sync_createA_ptr ITTNOTIFY_NAME(sync_createA) +#define __itt_sync_createW ITTNOTIFY_VOID(sync_createW) +#define __itt_sync_createW_ptr ITTNOTIFY_NAME(sync_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_create ITTNOTIFY_VOID(sync_create) +#define __itt_sync_create_ptr ITTNOTIFY_NAME(sync_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_createA(addr, objtype, objname, attribute) +#define __itt_sync_createA_ptr 0 +#define __itt_sync_createW(addr, objtype, objname, attribute) +#define __itt_sync_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_create(addr, objtype, objname, attribute) +#define __itt_sync_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_createA_ptr 0 +#define __itt_sync_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** +@brief Rename a synchronization object + +You can use the rename call to assign or reassign a name to a given +synchronization object. +@param[in] addr handle for the synchronization object. +@param[in] name null-terminated object name string. +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_sync_renameA(void *addr, const char *name); +void ITTAPI __itt_sync_renameW(void *addr, const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_sync_rename __itt_sync_renameW +# define __itt_sync_rename_ptr __itt_sync_renameW_ptr +#else /* UNICODE */ +# define __itt_sync_rename __itt_sync_renameA +# define __itt_sync_rename_ptr __itt_sync_renameA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_sync_rename(void *addr, const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, sync_renameA, (void *addr, const char *name)) +ITT_STUBV(ITTAPI, void, sync_renameW, (void *addr, const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, sync_rename, (void *addr, const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_renameA ITTNOTIFY_VOID(sync_renameA) +#define __itt_sync_renameA_ptr ITTNOTIFY_NAME(sync_renameA) +#define __itt_sync_renameW ITTNOTIFY_VOID(sync_renameW) +#define __itt_sync_renameW_ptr ITTNOTIFY_NAME(sync_renameW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_rename ITTNOTIFY_VOID(sync_rename) +#define __itt_sync_rename_ptr ITTNOTIFY_NAME(sync_rename) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_renameA(addr, name) +#define __itt_sync_renameA_ptr 0 +#define __itt_sync_renameW(addr, name) +#define __itt_sync_renameW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_rename(addr, name) +#define __itt_sync_rename_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_sync_renameA_ptr 0 +#define __itt_sync_renameW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_sync_rename_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + @brief Destroy a synchronization object. + @param addr Handle for the synchronization object. + */ +void ITTAPI __itt_sync_destroy(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_destroy, (void *addr)) +#define __itt_sync_destroy ITTNOTIFY_VOID(sync_destroy) +#define __itt_sync_destroy_ptr ITTNOTIFY_NAME(sync_destroy) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_destroy(addr) +#define __itt_sync_destroy_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_destroy_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/*****************************************************************//** + * @name group of functions is used for performance measurement tools + *********************************************************************/ +/** @{ */ +/** + * @brief Enter spin loop on user-defined sync object + */ +void ITTAPI __itt_sync_prepare(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_prepare, (void *addr)) +#define __itt_sync_prepare ITTNOTIFY_VOID(sync_prepare) +#define __itt_sync_prepare_ptr ITTNOTIFY_NAME(sync_prepare) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_prepare(addr) +#define __itt_sync_prepare_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_prepare_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Quit spin loop without acquiring spin object + */ +void ITTAPI __itt_sync_cancel(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_cancel, (void *addr)) +#define __itt_sync_cancel ITTNOTIFY_VOID(sync_cancel) +#define __itt_sync_cancel_ptr ITTNOTIFY_NAME(sync_cancel) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_cancel(addr) +#define __itt_sync_cancel_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_cancel_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Successful spin loop completion (sync object acquired) + */ +void ITTAPI __itt_sync_acquired(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_acquired, (void *addr)) +#define __itt_sync_acquired ITTNOTIFY_VOID(sync_acquired) +#define __itt_sync_acquired_ptr ITTNOTIFY_NAME(sync_acquired) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_acquired(addr) +#define __itt_sync_acquired_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_acquired_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Start sync object releasing code. Is called before the lock release call. + */ +void ITTAPI __itt_sync_releasing(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, sync_releasing, (void *addr)) +#define __itt_sync_releasing ITTNOTIFY_VOID(sync_releasing) +#define __itt_sync_releasing_ptr ITTNOTIFY_NAME(sync_releasing) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_sync_releasing(addr) +#define __itt_sync_releasing_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_sync_releasing_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} */ + +/** @} sync group */ + +/**************************************************************//** + * @name group of functions is used for correctness checking tools + ******************************************************************/ +/** @{ */ +/** + * @ingroup legacy + * @deprecated Legacy API + * @brief Fast synchronization which does no require spinning. + * - This special function is to be used by TBB and OpenMP libraries only when they know + * there is no spin but they need to suppress TC warnings about shared variable modifications. + * - It only has corresponding pointers in static library and does not have corresponding function + * in dynamic library. + * @see void __itt_sync_prepare(void* addr); + */ +void ITTAPI __itt_fsync_prepare(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, fsync_prepare, (void *addr)) +#define __itt_fsync_prepare ITTNOTIFY_VOID(fsync_prepare) +#define __itt_fsync_prepare_ptr ITTNOTIFY_NAME(fsync_prepare) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_fsync_prepare(addr) +#define __itt_fsync_prepare_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_fsync_prepare_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup legacy + * @deprecated Legacy API + * @brief Fast synchronization which does no require spinning. + * - This special function is to be used by TBB and OpenMP libraries only when they know + * there is no spin but they need to suppress TC warnings about shared variable modifications. + * - It only has corresponding pointers in static library and does not have corresponding function + * in dynamic library. + * @see void __itt_sync_cancel(void *addr); + */ +void ITTAPI __itt_fsync_cancel(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, fsync_cancel, (void *addr)) +#define __itt_fsync_cancel ITTNOTIFY_VOID(fsync_cancel) +#define __itt_fsync_cancel_ptr ITTNOTIFY_NAME(fsync_cancel) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_fsync_cancel(addr) +#define __itt_fsync_cancel_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_fsync_cancel_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup legacy + * @deprecated Legacy API + * @brief Fast synchronization which does no require spinning. + * - This special function is to be used by TBB and OpenMP libraries only when they know + * there is no spin but they need to suppress TC warnings about shared variable modifications. + * - It only has corresponding pointers in static library and does not have corresponding function + * in dynamic library. + * @see void __itt_sync_acquired(void *addr); + */ +void ITTAPI __itt_fsync_acquired(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, fsync_acquired, (void *addr)) +#define __itt_fsync_acquired ITTNOTIFY_VOID(fsync_acquired) +#define __itt_fsync_acquired_ptr ITTNOTIFY_NAME(fsync_acquired) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_fsync_acquired(addr) +#define __itt_fsync_acquired_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_fsync_acquired_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup legacy + * @deprecated Legacy API + * @brief Fast synchronization which does no require spinning. + * - This special function is to be used by TBB and OpenMP libraries only when they know + * there is no spin but they need to suppress TC warnings about shared variable modifications. + * - It only has corresponding pointers in static library and does not have corresponding function + * in dynamic library. + * @see void __itt_sync_releasing(void* addr); + */ +void ITTAPI __itt_fsync_releasing(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, fsync_releasing, (void *addr)) +#define __itt_fsync_releasing ITTNOTIFY_VOID(fsync_releasing) +#define __itt_fsync_releasing_ptr ITTNOTIFY_NAME(fsync_releasing) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_fsync_releasing(addr) +#define __itt_fsync_releasing_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_fsync_releasing_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} */ + +/** + * @defgroup model Modeling by Intel(R) Parallel Advisor + * @ingroup public + * This is the subset of itt used for modeling by Intel(R) Parallel Advisor. + * This API is called ONLY using annotate.h, by "Annotation" macros + * the user places in their sources during the parallelism modeling steps. + * + * site_begin/end and task_begin/end take the address of handle variables, + * which are writeable by the API. Handles must be 0 initialized prior + * to the first call to begin, or may cause a run-time failure. + * The handles are initialized in a multi-thread safe way by the API if + * the handle is 0. The commonly expected idiom is one static handle to + * identify a site or task. If a site or task of the same name has already + * been started during this collection, the same handle MAY be returned, + * but is not required to be - it is unspecified if data merging is done + * based on name. These routines also take an instance variable. Like + * the lexical instance, these must be 0 initialized. Unlike the lexical + * instance, this is used to track a single dynamic instance. + * + * API used by the Intel(R) Parallel Advisor to describe potential concurrency + * and related activities. User-added source annotations expand to calls + * to these procedures to enable modeling of a hypothetical concurrent + * execution serially. + * @{ + */ +#if !defined(_ADVISOR_ANNOTATE_H_) || defined(ANNOTATE_EXPAND_NULL) + +typedef void* __itt_model_site; /*!< @brief handle for lexical site */ +typedef void* __itt_model_site_instance; /*!< @brief handle for dynamic instance */ +typedef void* __itt_model_task; /*!< @brief handle for lexical site */ +typedef void* __itt_model_task_instance; /*!< @brief handle for dynamic instance */ + +/** + * @enum __itt_model_disable + * @brief Enumerator for the disable methods + */ +typedef enum { + __itt_model_disable_observation, + __itt_model_disable_collection +} __itt_model_disable; + +#endif /* !_ADVISOR_ANNOTATE_H_ || ANNOTATE_EXPAND_NULL */ + +/** + * @brief ANNOTATE_SITE_BEGIN/ANNOTATE_SITE_END support. + * + * site_begin/end model a potential concurrency site. + * site instances may be recursively nested with themselves. + * site_end exits the most recently started but unended site for the current + * thread. The handle passed to end may be used to validate structure. + * Instances of a site encountered on different threads concurrently + * are considered completely distinct. If the site name for two different + * lexical sites match, it is unspecified whether they are treated as the + * same or different for data presentation. + */ +void ITTAPI __itt_model_site_begin(__itt_model_site *site, __itt_model_site_instance *instance, const char *name); +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_model_site_beginW(const wchar_t *name); +#endif +void ITTAPI __itt_model_site_beginA(const char *name); +void ITTAPI __itt_model_site_beginAL(const char *name, size_t siteNameLen); +void ITTAPI __itt_model_site_end (__itt_model_site *site, __itt_model_site_instance *instance); +void ITTAPI __itt_model_site_end_2(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_site_begin, (__itt_model_site *site, __itt_model_site_instance *instance, const char *name)) +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, model_site_beginW, (const wchar_t *name)) +#endif +ITT_STUBV(ITTAPI, void, model_site_beginA, (const char *name)) +ITT_STUBV(ITTAPI, void, model_site_beginAL, (const char *name, size_t siteNameLen)) +ITT_STUBV(ITTAPI, void, model_site_end, (__itt_model_site *site, __itt_model_site_instance *instance)) +ITT_STUBV(ITTAPI, void, model_site_end_2, (void)) +#define __itt_model_site_begin ITTNOTIFY_VOID(model_site_begin) +#define __itt_model_site_begin_ptr ITTNOTIFY_NAME(model_site_begin) +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_site_beginW ITTNOTIFY_VOID(model_site_beginW) +#define __itt_model_site_beginW_ptr ITTNOTIFY_NAME(model_site_beginW) +#endif +#define __itt_model_site_beginA ITTNOTIFY_VOID(model_site_beginA) +#define __itt_model_site_beginA_ptr ITTNOTIFY_NAME(model_site_beginA) +#define __itt_model_site_beginAL ITTNOTIFY_VOID(model_site_beginAL) +#define __itt_model_site_beginAL_ptr ITTNOTIFY_NAME(model_site_beginAL) +#define __itt_model_site_end ITTNOTIFY_VOID(model_site_end) +#define __itt_model_site_end_ptr ITTNOTIFY_NAME(model_site_end) +#define __itt_model_site_end_2 ITTNOTIFY_VOID(model_site_end_2) +#define __itt_model_site_end_2_ptr ITTNOTIFY_NAME(model_site_end_2) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_site_begin(site, instance, name) +#define __itt_model_site_begin_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_site_beginW(name) +#define __itt_model_site_beginW_ptr 0 +#endif +#define __itt_model_site_beginA(name) +#define __itt_model_site_beginA_ptr 0 +#define __itt_model_site_beginAL(name, siteNameLen) +#define __itt_model_site_beginAL_ptr 0 +#define __itt_model_site_end(site, instance) +#define __itt_model_site_end_ptr 0 +#define __itt_model_site_end_2() +#define __itt_model_site_end_2_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_site_begin_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_site_beginW_ptr 0 +#endif +#define __itt_model_site_beginA_ptr 0 +#define __itt_model_site_beginAL_ptr 0 +#define __itt_model_site_end_ptr 0 +#define __itt_model_site_end_2_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_TASK_BEGIN/ANNOTATE_TASK_END support + * + * task_begin/end model a potential task, which is contained within the most + * closely enclosing dynamic site. task_end exits the most recently started + * but unended task. The handle passed to end may be used to validate + * structure. It is unspecified if bad dynamic nesting is detected. If it + * is, it should be encoded in the resulting data collection. The collector + * should not fail due to construct nesting issues, nor attempt to directly + * indicate the problem. + */ +void ITTAPI __itt_model_task_begin(__itt_model_task *task, __itt_model_task_instance *instance, const char *name); +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_model_task_beginW(const wchar_t *name); +void ITTAPI __itt_model_iteration_taskW(const wchar_t *name); +#endif +void ITTAPI __itt_model_task_beginA(const char *name); +void ITTAPI __itt_model_task_beginAL(const char *name, size_t taskNameLen); +void ITTAPI __itt_model_iteration_taskA(const char *name); +void ITTAPI __itt_model_iteration_taskAL(const char *name, size_t taskNameLen); +void ITTAPI __itt_model_task_end (__itt_model_task *task, __itt_model_task_instance *instance); +void ITTAPI __itt_model_task_end_2(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_task_begin, (__itt_model_task *task, __itt_model_task_instance *instance, const char *name)) +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, model_task_beginW, (const wchar_t *name)) +ITT_STUBV(ITTAPI, void, model_iteration_taskW, (const wchar_t *name)) +#endif +ITT_STUBV(ITTAPI, void, model_task_beginA, (const char *name)) +ITT_STUBV(ITTAPI, void, model_task_beginAL, (const char *name, size_t taskNameLen)) +ITT_STUBV(ITTAPI, void, model_iteration_taskA, (const char *name)) +ITT_STUBV(ITTAPI, void, model_iteration_taskAL, (const char *name, size_t taskNameLen)) +ITT_STUBV(ITTAPI, void, model_task_end, (__itt_model_task *task, __itt_model_task_instance *instance)) +ITT_STUBV(ITTAPI, void, model_task_end_2, (void)) +#define __itt_model_task_begin ITTNOTIFY_VOID(model_task_begin) +#define __itt_model_task_begin_ptr ITTNOTIFY_NAME(model_task_begin) +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_task_beginW ITTNOTIFY_VOID(model_task_beginW) +#define __itt_model_task_beginW_ptr ITTNOTIFY_NAME(model_task_beginW) +#define __itt_model_iteration_taskW ITTNOTIFY_VOID(model_iteration_taskW) +#define __itt_model_iteration_taskW_ptr ITTNOTIFY_NAME(model_iteration_taskW) +#endif +#define __itt_model_task_beginA ITTNOTIFY_VOID(model_task_beginA) +#define __itt_model_task_beginA_ptr ITTNOTIFY_NAME(model_task_beginA) +#define __itt_model_task_beginAL ITTNOTIFY_VOID(model_task_beginAL) +#define __itt_model_task_beginAL_ptr ITTNOTIFY_NAME(model_task_beginAL) +#define __itt_model_iteration_taskA ITTNOTIFY_VOID(model_iteration_taskA) +#define __itt_model_iteration_taskA_ptr ITTNOTIFY_NAME(model_iteration_taskA) +#define __itt_model_iteration_taskAL ITTNOTIFY_VOID(model_iteration_taskAL) +#define __itt_model_iteration_taskAL_ptr ITTNOTIFY_NAME(model_iteration_taskAL) +#define __itt_model_task_end ITTNOTIFY_VOID(model_task_end) +#define __itt_model_task_end_ptr ITTNOTIFY_NAME(model_task_end) +#define __itt_model_task_end_2 ITTNOTIFY_VOID(model_task_end_2) +#define __itt_model_task_end_2_ptr ITTNOTIFY_NAME(model_task_end_2) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_task_begin(task, instance, name) +#define __itt_model_task_begin_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_task_beginW(name) +#define __itt_model_task_beginW_ptr 0 +#endif +#define __itt_model_task_beginA(name) +#define __itt_model_task_beginA_ptr 0 +#define __itt_model_task_beginAL(name, siteNameLen) +#define __itt_model_task_beginAL_ptr 0 +#define __itt_model_iteration_taskA(name) +#define __itt_model_iteration_taskA_ptr 0 +#define __itt_model_iteration_taskAL(name, siteNameLen) +#define __itt_model_iteration_taskAL_ptr 0 +#define __itt_model_task_end(task, instance) +#define __itt_model_task_end_ptr 0 +#define __itt_model_task_end_2() +#define __itt_model_task_end_2_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_task_begin_ptr 0 +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_model_task_beginW_ptr 0 +#endif +#define __itt_model_task_beginA_ptr 0 +#define __itt_model_task_beginAL_ptr 0 +#define __itt_model_iteration_taskA_ptr 0 +#define __itt_model_iteration_taskAL_ptr 0 +#define __itt_model_task_end_ptr 0 +#define __itt_model_task_end_2_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_LOCK_ACQUIRE/ANNOTATE_LOCK_RELEASE support + * + * lock_acquire/release model a potential lock for both lockset and + * performance modeling. Each unique address is modeled as a separate + * lock, with invalid addresses being valid lock IDs. Specifically: + * no storage is accessed by the API at the specified address - it is only + * used for lock identification. Lock acquires may be self-nested and are + * unlocked by a corresponding number of releases. + * (These closely correspond to __itt_sync_acquired/__itt_sync_releasing, + * but may not have identical semantics.) + */ +void ITTAPI __itt_model_lock_acquire(void *lock); +void ITTAPI __itt_model_lock_acquire_2(void *lock); +void ITTAPI __itt_model_lock_release(void *lock); +void ITTAPI __itt_model_lock_release_2(void *lock); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_lock_acquire, (void *lock)) +ITT_STUBV(ITTAPI, void, model_lock_acquire_2, (void *lock)) +ITT_STUBV(ITTAPI, void, model_lock_release, (void *lock)) +ITT_STUBV(ITTAPI, void, model_lock_release_2, (void *lock)) +#define __itt_model_lock_acquire ITTNOTIFY_VOID(model_lock_acquire) +#define __itt_model_lock_acquire_ptr ITTNOTIFY_NAME(model_lock_acquire) +#define __itt_model_lock_acquire_2 ITTNOTIFY_VOID(model_lock_acquire_2) +#define __itt_model_lock_acquire_2_ptr ITTNOTIFY_NAME(model_lock_acquire_2) +#define __itt_model_lock_release ITTNOTIFY_VOID(model_lock_release) +#define __itt_model_lock_release_ptr ITTNOTIFY_NAME(model_lock_release) +#define __itt_model_lock_release_2 ITTNOTIFY_VOID(model_lock_release_2) +#define __itt_model_lock_release_2_ptr ITTNOTIFY_NAME(model_lock_release_2) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_lock_acquire(lock) +#define __itt_model_lock_acquire_ptr 0 +#define __itt_model_lock_acquire_2(lock) +#define __itt_model_lock_acquire_2_ptr 0 +#define __itt_model_lock_release(lock) +#define __itt_model_lock_release_ptr 0 +#define __itt_model_lock_release_2(lock) +#define __itt_model_lock_release_2_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_lock_acquire_ptr 0 +#define __itt_model_lock_acquire_2_ptr 0 +#define __itt_model_lock_release_ptr 0 +#define __itt_model_lock_release_2_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_RECORD_ALLOCATION/ANNOTATE_RECORD_DEALLOCATION support + * + * record_allocation/deallocation describe user-defined memory allocator + * behavior, which may be required for correctness modeling to understand + * when storage is not expected to be actually reused across threads. + */ +void ITTAPI __itt_model_record_allocation (void *addr, size_t size); +void ITTAPI __itt_model_record_deallocation(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_record_allocation, (void *addr, size_t size)) +ITT_STUBV(ITTAPI, void, model_record_deallocation, (void *addr)) +#define __itt_model_record_allocation ITTNOTIFY_VOID(model_record_allocation) +#define __itt_model_record_allocation_ptr ITTNOTIFY_NAME(model_record_allocation) +#define __itt_model_record_deallocation ITTNOTIFY_VOID(model_record_deallocation) +#define __itt_model_record_deallocation_ptr ITTNOTIFY_NAME(model_record_deallocation) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_record_allocation(addr, size) +#define __itt_model_record_allocation_ptr 0 +#define __itt_model_record_deallocation(addr) +#define __itt_model_record_deallocation_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_record_allocation_ptr 0 +#define __itt_model_record_deallocation_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_INDUCTION_USES support + * + * Note particular storage is inductive through the end of the current site + */ +void ITTAPI __itt_model_induction_uses(void* addr, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_induction_uses, (void *addr, size_t size)) +#define __itt_model_induction_uses ITTNOTIFY_VOID(model_induction_uses) +#define __itt_model_induction_uses_ptr ITTNOTIFY_NAME(model_induction_uses) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_induction_uses(addr, size) +#define __itt_model_induction_uses_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_induction_uses_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_REDUCTION_USES support + * + * Note particular storage is used for reduction through the end + * of the current site + */ +void ITTAPI __itt_model_reduction_uses(void* addr, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_reduction_uses, (void *addr, size_t size)) +#define __itt_model_reduction_uses ITTNOTIFY_VOID(model_reduction_uses) +#define __itt_model_reduction_uses_ptr ITTNOTIFY_NAME(model_reduction_uses) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_reduction_uses(addr, size) +#define __itt_model_reduction_uses_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_reduction_uses_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_OBSERVE_USES support + * + * Have correctness modeling record observations about uses of storage + * through the end of the current site + */ +void ITTAPI __itt_model_observe_uses(void* addr, size_t size); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_observe_uses, (void *addr, size_t size)) +#define __itt_model_observe_uses ITTNOTIFY_VOID(model_observe_uses) +#define __itt_model_observe_uses_ptr ITTNOTIFY_NAME(model_observe_uses) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_observe_uses(addr, size) +#define __itt_model_observe_uses_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_observe_uses_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_CLEAR_USES support + * + * Clear the special handling of a piece of storage related to induction, + * reduction or observe_uses + */ +void ITTAPI __itt_model_clear_uses(void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_clear_uses, (void *addr)) +#define __itt_model_clear_uses ITTNOTIFY_VOID(model_clear_uses) +#define __itt_model_clear_uses_ptr ITTNOTIFY_NAME(model_clear_uses) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_clear_uses(addr) +#define __itt_model_clear_uses_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_clear_uses_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief ANNOTATE_DISABLE_*_PUSH/ANNOTATE_DISABLE_*_POP support + * + * disable_push/disable_pop push and pop disabling based on a parameter. + * Disabling observations stops processing of memory references during + * correctness modeling, and all annotations that occur in the disabled + * region. This allows description of code that is expected to be handled + * specially during conversion to parallelism or that is not recognized + * by tools (e.g. some kinds of synchronization operations.) + * This mechanism causes all annotations in the disabled region, other + * than disable_push and disable_pop, to be ignored. (For example, this + * might validly be used to disable an entire parallel site and the contained + * tasks and locking in it for data collection purposes.) + * The disable for collection is a more expensive operation, but reduces + * collector overhead significantly. This applies to BOTH correctness data + * collection and performance data collection. For example, a site + * containing a task might only enable data collection for the first 10 + * iterations. Both performance and correctness data should reflect this, + * and the program should run as close to full speed as possible when + * collection is disabled. + */ +void ITTAPI __itt_model_disable_push(__itt_model_disable x); +void ITTAPI __itt_model_disable_pop(void); +void ITTAPI __itt_model_aggregate_task(size_t x); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, model_disable_push, (__itt_model_disable x)) +ITT_STUBV(ITTAPI, void, model_disable_pop, (void)) +ITT_STUBV(ITTAPI, void, model_aggregate_task, (size_t x)) +#define __itt_model_disable_push ITTNOTIFY_VOID(model_disable_push) +#define __itt_model_disable_push_ptr ITTNOTIFY_NAME(model_disable_push) +#define __itt_model_disable_pop ITTNOTIFY_VOID(model_disable_pop) +#define __itt_model_disable_pop_ptr ITTNOTIFY_NAME(model_disable_pop) +#define __itt_model_aggregate_task ITTNOTIFY_VOID(model_aggregate_task) +#define __itt_model_aggregate_task_ptr ITTNOTIFY_NAME(model_aggregate_task) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_model_disable_push(x) +#define __itt_model_disable_push_ptr 0 +#define __itt_model_disable_pop() +#define __itt_model_disable_pop_ptr 0 +#define __itt_model_aggregate_task(x) +#define __itt_model_aggregate_task_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_model_disable_push_ptr 0 +#define __itt_model_disable_pop_ptr 0 +#define __itt_model_aggregate_task_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} model group */ + +/** + * @defgroup heap Heap + * @ingroup public + * Heap group + * @{ + */ + +typedef void* __itt_heap_function; + +/** + * @brief Create an identification for heap function + * @return non-zero identifier or NULL + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_heap_function ITTAPI __itt_heap_function_createA(const char* name, const char* domain); +__itt_heap_function ITTAPI __itt_heap_function_createW(const wchar_t* name, const wchar_t* domain); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_heap_function_create __itt_heap_function_createW +# define __itt_heap_function_create_ptr __itt_heap_function_createW_ptr +#else +# define __itt_heap_function_create __itt_heap_function_createA +# define __itt_heap_function_create_ptr __itt_heap_function_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_heap_function ITTAPI __itt_heap_function_create(const char* name, const char* domain); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_heap_function, heap_function_createA, (const char* name, const char* domain)) +ITT_STUB(ITTAPI, __itt_heap_function, heap_function_createW, (const wchar_t* name, const wchar_t* domain)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_heap_function, heap_function_create, (const char* name, const char* domain)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_heap_function_createA ITTNOTIFY_DATA(heap_function_createA) +#define __itt_heap_function_createA_ptr ITTNOTIFY_NAME(heap_function_createA) +#define __itt_heap_function_createW ITTNOTIFY_DATA(heap_function_createW) +#define __itt_heap_function_createW_ptr ITTNOTIFY_NAME(heap_function_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_heap_function_create ITTNOTIFY_DATA(heap_function_create) +#define __itt_heap_function_create_ptr ITTNOTIFY_NAME(heap_function_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_heap_function_createA(name, domain) (__itt_heap_function)0 +#define __itt_heap_function_createA_ptr 0 +#define __itt_heap_function_createW(name, domain) (__itt_heap_function)0 +#define __itt_heap_function_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_heap_function_create(name, domain) (__itt_heap_function)0 +#define __itt_heap_function_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_heap_function_createA_ptr 0 +#define __itt_heap_function_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_heap_function_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an allocation begin occurrence. + */ +void ITTAPI __itt_heap_allocate_begin(__itt_heap_function h, size_t size, int initialized); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_allocate_begin, (__itt_heap_function h, size_t size, int initialized)) +#define __itt_heap_allocate_begin ITTNOTIFY_VOID(heap_allocate_begin) +#define __itt_heap_allocate_begin_ptr ITTNOTIFY_NAME(heap_allocate_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_allocate_begin(h, size, initialized) +#define __itt_heap_allocate_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_allocate_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an allocation end occurrence. + */ +void ITTAPI __itt_heap_allocate_end(__itt_heap_function h, void** addr, size_t size, int initialized); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_allocate_end, (__itt_heap_function h, void** addr, size_t size, int initialized)) +#define __itt_heap_allocate_end ITTNOTIFY_VOID(heap_allocate_end) +#define __itt_heap_allocate_end_ptr ITTNOTIFY_NAME(heap_allocate_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_allocate_end(h, addr, size, initialized) +#define __itt_heap_allocate_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_allocate_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record a free begin occurrence. + */ +void ITTAPI __itt_heap_free_begin(__itt_heap_function h, void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_free_begin, (__itt_heap_function h, void* addr)) +#define __itt_heap_free_begin ITTNOTIFY_VOID(heap_free_begin) +#define __itt_heap_free_begin_ptr ITTNOTIFY_NAME(heap_free_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_free_begin(h, addr) +#define __itt_heap_free_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_free_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record a free end occurrence. + */ +void ITTAPI __itt_heap_free_end(__itt_heap_function h, void* addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_free_end, (__itt_heap_function h, void* addr)) +#define __itt_heap_free_end ITTNOTIFY_VOID(heap_free_end) +#define __itt_heap_free_end_ptr ITTNOTIFY_NAME(heap_free_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_free_end(h, addr) +#define __itt_heap_free_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_free_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record a reallocation begin occurrence. + */ +void ITTAPI __itt_heap_reallocate_begin(__itt_heap_function h, void* addr, size_t new_size, int initialized); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_reallocate_begin, (__itt_heap_function h, void* addr, size_t new_size, int initialized)) +#define __itt_heap_reallocate_begin ITTNOTIFY_VOID(heap_reallocate_begin) +#define __itt_heap_reallocate_begin_ptr ITTNOTIFY_NAME(heap_reallocate_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_reallocate_begin(h, addr, new_size, initialized) +#define __itt_heap_reallocate_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_reallocate_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record a reallocation end occurrence. + */ +void ITTAPI __itt_heap_reallocate_end(__itt_heap_function h, void* addr, void** new_addr, size_t new_size, int initialized); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_reallocate_end, (__itt_heap_function h, void* addr, void** new_addr, size_t new_size, int initialized)) +#define __itt_heap_reallocate_end ITTNOTIFY_VOID(heap_reallocate_end) +#define __itt_heap_reallocate_end_ptr ITTNOTIFY_NAME(heap_reallocate_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_reallocate_end(h, addr, new_addr, new_size, initialized) +#define __itt_heap_reallocate_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_reallocate_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief internal access begin */ +void ITTAPI __itt_heap_internal_access_begin(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_internal_access_begin, (void)) +#define __itt_heap_internal_access_begin ITTNOTIFY_VOID(heap_internal_access_begin) +#define __itt_heap_internal_access_begin_ptr ITTNOTIFY_NAME(heap_internal_access_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_internal_access_begin() +#define __itt_heap_internal_access_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_internal_access_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief internal access end */ +void ITTAPI __itt_heap_internal_access_end(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_internal_access_end, (void)) +#define __itt_heap_internal_access_end ITTNOTIFY_VOID(heap_internal_access_end) +#define __itt_heap_internal_access_end_ptr ITTNOTIFY_NAME(heap_internal_access_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_internal_access_end() +#define __itt_heap_internal_access_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_internal_access_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief record memory growth begin */ +void ITTAPI __itt_heap_record_memory_growth_begin(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_record_memory_growth_begin, (void)) +#define __itt_heap_record_memory_growth_begin ITTNOTIFY_VOID(heap_record_memory_growth_begin) +#define __itt_heap_record_memory_growth_begin_ptr ITTNOTIFY_NAME(heap_record_memory_growth_begin) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_record_memory_growth_begin() +#define __itt_heap_record_memory_growth_begin_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_record_memory_growth_begin_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief record memory growth end */ +void ITTAPI __itt_heap_record_memory_growth_end(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_record_memory_growth_end, (void)) +#define __itt_heap_record_memory_growth_end ITTNOTIFY_VOID(heap_record_memory_growth_end) +#define __itt_heap_record_memory_growth_end_ptr ITTNOTIFY_NAME(heap_record_memory_growth_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_record_memory_growth_end() +#define __itt_heap_record_memory_growth_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_record_memory_growth_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Specify the type of heap detection/reporting to modify. + */ +/** + * @hideinitializer + * @brief Report on memory leaks. + */ +#define __itt_heap_leaks 0x00000001 + +/** + * @hideinitializer + * @brief Report on memory growth. + */ +#define __itt_heap_growth 0x00000002 + + +/** @brief heap reset detection */ +void ITTAPI __itt_heap_reset_detection(unsigned int reset_mask); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_reset_detection, (unsigned int reset_mask)) +#define __itt_heap_reset_detection ITTNOTIFY_VOID(heap_reset_detection) +#define __itt_heap_reset_detection_ptr ITTNOTIFY_NAME(heap_reset_detection) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_reset_detection() +#define __itt_heap_reset_detection_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_reset_detection_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @brief report */ +void ITTAPI __itt_heap_record(unsigned int record_mask); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, heap_record, (unsigned int record_mask)) +#define __itt_heap_record ITTNOTIFY_VOID(heap_record) +#define __itt_heap_record_ptr ITTNOTIFY_NAME(heap_record) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_heap_record() +#define __itt_heap_record_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_heap_record_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @} heap group */ +/** @endcond */ +/* ========================================================================== */ + +/** + * @defgroup domains Domains + * @ingroup public + * Domains group + * @{ + */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_domain +{ + volatile int flags; /*!< Zero if disabled, non-zero if enabled. The meaning of different non-zero values is reserved to the runtime */ + const char* nameA; /*!< Copy of original name in ASCII. */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* nameW; /*!< Copy of original name in UNICODE. */ +#else /* UNICODE || _UNICODE */ + void* nameW; +#endif /* UNICODE || _UNICODE */ + int extra1; /*!< Reserved to the runtime */ + void* extra2; /*!< Reserved to the runtime */ + struct ___itt_domain* next; +} __itt_domain; + +#pragma pack(pop) +/** @endcond */ + +/** + * @ingroup domains + * @brief Create a domain. + * Create domain using some domain name: the URI naming style is recommended. + * Because the set of domains is expected to be static over the application's + * execution time, there is no mechanism to destroy a domain. + * Any domain can be accessed by any thread in the process, regardless of + * which thread created the domain. This call is thread-safe. + * @param[in] name name of domain + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_domain* ITTAPI __itt_domain_createA(const char *name); +__itt_domain* ITTAPI __itt_domain_createW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_domain_create __itt_domain_createW +# define __itt_domain_create_ptr __itt_domain_createW_ptr +#else /* UNICODE */ +# define __itt_domain_create __itt_domain_createA +# define __itt_domain_create_ptr __itt_domain_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_domain* ITTAPI __itt_domain_create(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_domain*, domain_createA, (const char *name)) +ITT_STUB(ITTAPI, __itt_domain*, domain_createW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_domain*, domain_create, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_domain_createA ITTNOTIFY_DATA(domain_createA) +#define __itt_domain_createA_ptr ITTNOTIFY_NAME(domain_createA) +#define __itt_domain_createW ITTNOTIFY_DATA(domain_createW) +#define __itt_domain_createW_ptr ITTNOTIFY_NAME(domain_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_domain_create ITTNOTIFY_DATA(domain_create) +#define __itt_domain_create_ptr ITTNOTIFY_NAME(domain_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_domain_createA(name) (__itt_domain*)0 +#define __itt_domain_createA_ptr 0 +#define __itt_domain_createW(name) (__itt_domain*)0 +#define __itt_domain_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_domain_create(name) (__itt_domain*)0 +#define __itt_domain_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_domain_createA_ptr 0 +#define __itt_domain_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_domain_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} domains group */ + +/** + * @defgroup ids IDs + * @ingroup public + * IDs group + * @{ + */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_id +{ + unsigned long long d1, d2, d3; +} __itt_id; + +#pragma pack(pop) +/** @endcond */ + +static const __itt_id __itt_null = { 0, 0, 0 }; + +/** + * @ingroup ids + * @brief A convenience function is provided to create an ID without domain control. + * @brief This is a convenience function to initialize an __itt_id structure. This function + * does not affect the collector runtime in any way. After you make the ID with this + * function, you still must create it with the __itt_id_create function before using the ID + * to identify a named entity. + * @param[in] addr The address of object; high QWORD of the ID value. + * @param[in] extra The extra data to unique identify object; low QWORD of the ID value. + */ + +ITT_INLINE __itt_id ITTAPI __itt_id_make(void* addr, unsigned long long extra) ITT_INLINE_ATTRIBUTE; +ITT_INLINE __itt_id ITTAPI __itt_id_make(void* addr, unsigned long long extra) +{ + __itt_id id = __itt_null; + id.d1 = (unsigned long long)((uintptr_t)addr); + id.d2 = (unsigned long long)extra; + id.d3 = (unsigned long long)0; /* Reserved. Must be zero */ + return id; +} + +/** + * @ingroup ids + * @brief Create an instance of identifier. + * This establishes the beginning of the lifetime of an instance of + * the given ID in the trace. Once this lifetime starts, the ID + * can be used to tag named entity instances in calls such as + * __itt_task_begin, and to specify relationships among + * identified named entity instances, using the \ref relations APIs. + * Instance IDs are not domain specific! + * @param[in] domain The domain controlling the execution of this call. + * @param[in] id The ID to create. + */ +void ITTAPI __itt_id_create(const __itt_domain *domain, __itt_id id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, id_create, (const __itt_domain *domain, __itt_id id)) +#define __itt_id_create(d,x) ITTNOTIFY_VOID_D1(id_create,d,x) +#define __itt_id_create_ptr ITTNOTIFY_NAME(id_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_id_create(domain,id) +#define __itt_id_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_id_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup ids + * @brief Destroy an instance of identifier. + * This ends the lifetime of the current instance of the given ID value in the trace. + * Any relationships that are established after this lifetime ends are invalid. + * This call must be performed before the given ID value can be reused for a different + * named entity instance. + * @param[in] domain The domain controlling the execution of this call. + * @param[in] id The ID to destroy. + */ +void ITTAPI __itt_id_destroy(const __itt_domain *domain, __itt_id id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, id_destroy, (const __itt_domain *domain, __itt_id id)) +#define __itt_id_destroy(d,x) ITTNOTIFY_VOID_D1(id_destroy,d,x) +#define __itt_id_destroy_ptr ITTNOTIFY_NAME(id_destroy) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_id_destroy(domain,id) +#define __itt_id_destroy_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_id_destroy_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} ids group */ + +/** + * @defgroup handless String Handles + * @ingroup public + * String Handles group + * @{ + */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_string_handle +{ + const char* strA; /*!< Copy of original string in ASCII. */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* strW; /*!< Copy of original string in UNICODE. */ +#else /* UNICODE || _UNICODE */ + void* strW; +#endif /* UNICODE || _UNICODE */ + int extra1; /*!< Reserved. Must be zero */ + void* extra2; /*!< Reserved. Must be zero */ + struct ___itt_string_handle* next; +} __itt_string_handle; + +#pragma pack(pop) +/** @endcond */ + +/** + * @ingroup handles + * @brief Create a string handle. + * Create and return handle value that can be associated with a string. + * Consecutive calls to __itt_string_handle_create with the same name + * return the same value. Because the set of string handles is expected to remain + * static during the application's execution time, there is no mechanism to destroy a string handle. + * Any string handle can be accessed by any thread in the process, regardless of which thread created + * the string handle. This call is thread-safe. + * @param[in] name The input string + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_string_handle* ITTAPI __itt_string_handle_createA(const char *name); +__itt_string_handle* ITTAPI __itt_string_handle_createW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_string_handle_create __itt_string_handle_createW +# define __itt_string_handle_create_ptr __itt_string_handle_createW_ptr +#else /* UNICODE */ +# define __itt_string_handle_create __itt_string_handle_createA +# define __itt_string_handle_create_ptr __itt_string_handle_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_string_handle* ITTAPI __itt_string_handle_create(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createA, (const char *name)) +ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_createW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_string_handle*, string_handle_create, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_string_handle_createA ITTNOTIFY_DATA(string_handle_createA) +#define __itt_string_handle_createA_ptr ITTNOTIFY_NAME(string_handle_createA) +#define __itt_string_handle_createW ITTNOTIFY_DATA(string_handle_createW) +#define __itt_string_handle_createW_ptr ITTNOTIFY_NAME(string_handle_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_string_handle_create ITTNOTIFY_DATA(string_handle_create) +#define __itt_string_handle_create_ptr ITTNOTIFY_NAME(string_handle_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_string_handle_createA(name) (__itt_string_handle*)0 +#define __itt_string_handle_createA_ptr 0 +#define __itt_string_handle_createW(name) (__itt_string_handle*)0 +#define __itt_string_handle_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_string_handle_create(name) (__itt_string_handle*)0 +#define __itt_string_handle_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_string_handle_createA_ptr 0 +#define __itt_string_handle_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_string_handle_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} handles group */ + +/** @cond exclude_from_documentation */ +typedef unsigned long long __itt_timestamp; +/** @endcond */ + +#define __itt_timestamp_none ((__itt_timestamp)-1LL) + +/** @cond exclude_from_gpa_documentation */ + +/** + * @ingroup timestamps + * @brief Return timestamp corresponding to the current moment. + * This returns the timestamp in the format that is the most relevant for the current + * host or platform (RDTSC, QPC, and others). You can use the "<" operator to + * compare __itt_timestamp values. + */ +__itt_timestamp ITTAPI __itt_get_timestamp(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_timestamp, get_timestamp, (void)) +#define __itt_get_timestamp ITTNOTIFY_DATA(get_timestamp) +#define __itt_get_timestamp_ptr ITTNOTIFY_NAME(get_timestamp) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_get_timestamp() +#define __itt_get_timestamp_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_get_timestamp_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} timestamps */ +/** @endcond */ + +/** @cond exclude_from_gpa_documentation */ + +/** + * @defgroup regions Regions + * @ingroup public + * Regions group + * @{ + */ +/** + * @ingroup regions + * @brief Begin of region instance. + * Successive calls to __itt_region_begin with the same ID are ignored + * until a call to __itt_region_end with the same ID + * @param[in] domain The domain for this region instance + * @param[in] id The instance ID for this region instance. Must not be __itt_null + * @param[in] parentid The instance ID for the parent of this region instance, or __itt_null + * @param[in] name The name of this region + */ +void ITTAPI __itt_region_begin(const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name); + +/** + * @ingroup regions + * @brief End of region instance. + * The first call to __itt_region_end with a given ID ends the + * region. Successive calls with the same ID are ignored, as are + * calls that do not have a matching __itt_region_begin call. + * @param[in] domain The domain for this region instance + * @param[in] id The instance ID for this region instance + */ +void ITTAPI __itt_region_end(const __itt_domain *domain, __itt_id id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, region_begin, (const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, region_end, (const __itt_domain *domain, __itt_id id)) +#define __itt_region_begin(d,x,y,z) ITTNOTIFY_VOID_D3(region_begin,d,x,y,z) +#define __itt_region_begin_ptr ITTNOTIFY_NAME(region_begin) +#define __itt_region_end(d,x) ITTNOTIFY_VOID_D1(region_end,d,x) +#define __itt_region_end_ptr ITTNOTIFY_NAME(region_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_region_begin(d,x,y,z) +#define __itt_region_begin_ptr 0 +#define __itt_region_end(d,x) +#define __itt_region_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_region_begin_ptr 0 +#define __itt_region_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} regions group */ + +/** + * @defgroup frames Frames + * @ingroup public + * Frames are similar to regions, but are intended to be easier to use and to implement. + * In particular: + * - Frames always represent periods of elapsed time + * - By default, frames have no nesting relationships + * @{ + */ + +/** + * @ingroup frames + * @brief Begin a frame instance. + * Successive calls to __itt_frame_begin with the + * same ID are ignored until a call to __itt_frame_end with the same ID. + * @param[in] domain The domain for this frame instance + * @param[in] id The instance ID for this frame instance or NULL + */ +void ITTAPI __itt_frame_begin_v3(const __itt_domain *domain, __itt_id *id); + +/** + * @ingroup frames + * @brief End a frame instance. + * The first call to __itt_frame_end with a given ID + * ends the frame. Successive calls with the same ID are ignored, as are + * calls that do not have a matching __itt_frame_begin call. + * @param[in] domain The domain for this frame instance + * @param[in] id The instance ID for this frame instance or NULL for current + */ +void ITTAPI __itt_frame_end_v3(const __itt_domain *domain, __itt_id *id); + +/** + * @ingroup frames + * @brief Submits a frame instance. + * Successive calls to __itt_frame_begin or __itt_frame_submit with the + * same ID are ignored until a call to __itt_frame_end or __itt_frame_submit + * with the same ID. + * Passing special __itt_timestamp_none value as "end" argument means + * take the current timestamp as the end timestamp. + * @param[in] domain The domain for this frame instance + * @param[in] id The instance ID for this frame instance or NULL + * @param[in] begin Timestamp of the beginning of the frame + * @param[in] end Timestamp of the end of the frame + */ +void ITTAPI __itt_frame_submit_v3(const __itt_domain *domain, __itt_id *id, + __itt_timestamp begin, __itt_timestamp end); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, frame_begin_v3, (const __itt_domain *domain, __itt_id *id)) +ITT_STUBV(ITTAPI, void, frame_end_v3, (const __itt_domain *domain, __itt_id *id)) +ITT_STUBV(ITTAPI, void, frame_submit_v3, (const __itt_domain *domain, __itt_id *id, __itt_timestamp begin, __itt_timestamp end)) +#define __itt_frame_begin_v3(d,x) ITTNOTIFY_VOID_D1(frame_begin_v3,d,x) +#define __itt_frame_begin_v3_ptr ITTNOTIFY_NAME(frame_begin_v3) +#define __itt_frame_end_v3(d,x) ITTNOTIFY_VOID_D1(frame_end_v3,d,x) +#define __itt_frame_end_v3_ptr ITTNOTIFY_NAME(frame_end_v3) +#define __itt_frame_submit_v3(d,x,b,e) ITTNOTIFY_VOID_D3(frame_submit_v3,d,x,b,e) +#define __itt_frame_submit_v3_ptr ITTNOTIFY_NAME(frame_submit_v3) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_frame_begin_v3(domain,id) +#define __itt_frame_begin_v3_ptr 0 +#define __itt_frame_end_v3(domain,id) +#define __itt_frame_end_v3_ptr 0 +#define __itt_frame_submit_v3(domain,id,begin,end) +#define __itt_frame_submit_v3_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_frame_begin_v3_ptr 0 +#define __itt_frame_end_v3_ptr 0 +#define __itt_frame_submit_v3_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} frames group */ +/** @endcond */ + +/** + * @defgroup taskgroup Task Group + * @ingroup public + * Task Group + * @{ + */ +/** + * @ingroup task_groups + * @brief Denotes a task_group instance. + * Successive calls to __itt_task_group with the same ID are ignored. + * @param[in] domain The domain for this task_group instance + * @param[in] id The instance ID for this task_group instance. Must not be __itt_null. + * @param[in] parentid The instance ID for the parent of this task_group instance, or __itt_null. + * @param[in] name The name of this task_group + */ +void ITTAPI __itt_task_group(const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, task_group, (const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name)) +#define __itt_task_group(d,x,y,z) ITTNOTIFY_VOID_D3(task_group,d,x,y,z) +#define __itt_task_group_ptr ITTNOTIFY_NAME(task_group) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_task_group(d,x,y,z) +#define __itt_task_group_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_task_group_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} taskgroup group */ + +/** + * @defgroup tasks Tasks + * @ingroup public + * A task instance represents a piece of work performed by a particular + * thread for a period of time. A call to __itt_task_begin creates a + * task instance. This becomes the current instance for that task on that + * thread. A following call to __itt_task_end on the same thread ends the + * instance. There may be multiple simultaneous instances of tasks with the + * same name on different threads. If an ID is specified, the task instance + * receives that ID. Nested tasks are allowed. + * + * Note: The task is defined by the bracketing of __itt_task_begin and + * __itt_task_end on the same thread. If some scheduling mechanism causes + * task switching (the thread executes a different user task) or task + * switching (the user task switches to a different thread) then this breaks + * the notion of current instance. Additional API calls are required to + * deal with that possibility. + * @{ + */ + +/** + * @ingroup tasks + * @brief Begin a task instance. + * @param[in] domain The domain for this task + * @param[in] taskid The instance ID for this task instance, or __itt_null + * @param[in] parentid The parent instance to which this task instance belongs, or __itt_null + * @param[in] name The name of this task + */ +void ITTAPI __itt_task_begin(const __itt_domain *domain, __itt_id taskid, __itt_id parentid, __itt_string_handle *name); + +/** + * @ingroup tasks + * @brief Begin a task instance. + * @param[in] domain The domain for this task + * @param[in] taskid The identifier for this task instance (may be 0) + * @param[in] parentid The parent of this task (may be 0) + * @param[in] fn The pointer to the function you are tracing + */ +void ITTAPI __itt_task_begin_fn(const __itt_domain *domain, __itt_id taskid, __itt_id parentid, void* fn); + +/** + * @ingroup tasks + * @brief End the current task instance. + * @param[in] domain The domain for this task + */ +void ITTAPI __itt_task_end(const __itt_domain *domain); + +/** + * @ingroup tasks + * @brief Begin an overlapped task instance. + * @param[in] domain The domain for this task. + * @param[in] taskid The identifier for this task instance, *cannot* be __itt_null. + * @param[in] parentid The parent of this task, or __itt_null. + * @param[in] name The name of this task. + */ +void ITTAPI __itt_task_begin_overlapped(const __itt_domain* domain, __itt_id taskid, __itt_id parentid, __itt_string_handle* name); + +/** + * @ingroup tasks + * @brief End an overlapped task instance. + * @param[in] domain The domain for this task + * @param[in] taskid Explicit ID of finished task + */ +void ITTAPI __itt_task_end_overlapped(const __itt_domain *domain, __itt_id taskid); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, task_begin, (const __itt_domain *domain, __itt_id id, __itt_id parentid, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, task_begin_fn, (const __itt_domain *domain, __itt_id id, __itt_id parentid, void* fn)) +ITT_STUBV(ITTAPI, void, task_end, (const __itt_domain *domain)) +ITT_STUBV(ITTAPI, void, task_begin_overlapped, (const __itt_domain *domain, __itt_id taskid, __itt_id parentid, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, task_end_overlapped, (const __itt_domain *domain, __itt_id taskid)) +#define __itt_task_begin(d,x,y,z) ITTNOTIFY_VOID_D3(task_begin,d,x,y,z) +#define __itt_task_begin_ptr ITTNOTIFY_NAME(task_begin) +#define __itt_task_begin_fn(d,x,y,z) ITTNOTIFY_VOID_D3(task_begin_fn,d,x,y,z) +#define __itt_task_begin_fn_ptr ITTNOTIFY_NAME(task_begin_fn) +#define __itt_task_end(d) ITTNOTIFY_VOID_D0(task_end,d) +#define __itt_task_end_ptr ITTNOTIFY_NAME(task_end) +#define __itt_task_begin_overlapped(d,x,y,z) ITTNOTIFY_VOID_D3(task_begin_overlapped,d,x,y,z) +#define __itt_task_begin_overlapped_ptr ITTNOTIFY_NAME(task_begin_overlapped) +#define __itt_task_end_overlapped(d,x) ITTNOTIFY_VOID_D1(task_end_overlapped,d,x) +#define __itt_task_end_overlapped_ptr ITTNOTIFY_NAME(task_end_overlapped) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_task_begin(domain,id,parentid,name) +#define __itt_task_begin_ptr 0 +#define __itt_task_begin_fn(domain,id,parentid,fn) +#define __itt_task_begin_fn_ptr 0 +#define __itt_task_end(domain) +#define __itt_task_end_ptr 0 +#define __itt_task_begin_overlapped(domain,taskid,parentid,name) +#define __itt_task_begin_overlapped_ptr 0 +#define __itt_task_end_overlapped(domain,taskid) +#define __itt_task_end_overlapped_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_task_begin_ptr 0 +#define __itt_task_begin_fn_ptr 0 +#define __itt_task_end_ptr 0 +#define __itt_task_begin_overlapped_ptr 0 +#define __itt_task_end_overlapped_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} tasks group */ + + +/** + * @defgroup markers Markers + * Markers represent a single discreet event in time. Markers have a scope, + * described by an enumerated type __itt_scope. Markers are created by + * the API call __itt_marker. A marker instance can be given an ID for use in + * adding metadata. + * @{ + */ + +/** + * @brief Describes the scope of an event object in the trace. + */ +typedef enum +{ + __itt_scope_unknown = 0, + __itt_scope_global, + __itt_scope_track_group, + __itt_scope_track, + __itt_scope_task, + __itt_scope_marker +} __itt_scope; + +/** @cond exclude_from_documentation */ +#define __itt_marker_scope_unknown __itt_scope_unknown +#define __itt_marker_scope_global __itt_scope_global +#define __itt_marker_scope_process __itt_scope_track_group +#define __itt_marker_scope_thread __itt_scope_track +#define __itt_marker_scope_task __itt_scope_task +/** @endcond */ + +/** + * @ingroup markers + * @brief Create a marker instance + * @param[in] domain The domain for this marker + * @param[in] id The instance ID for this marker or __itt_null + * @param[in] name The name for this marker + * @param[in] scope The scope for this marker + */ +void ITTAPI __itt_marker(const __itt_domain *domain, __itt_id id, __itt_string_handle *name, __itt_scope scope); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, marker, (const __itt_domain *domain, __itt_id id, __itt_string_handle *name, __itt_scope scope)) +#define __itt_marker(d,x,y,z) ITTNOTIFY_VOID_D3(marker,d,x,y,z) +#define __itt_marker_ptr ITTNOTIFY_NAME(marker) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_marker(domain,id,name,scope) +#define __itt_marker_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_marker_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} markers group */ + +/** + * @defgroup metadata Metadata + * The metadata API is used to attach extra information to named + * entities. Metadata can be attached to an identified named entity by ID, + * or to the current entity (which is always a task). + * + * Conceptually metadata has a type (what kind of metadata), a key (the + * name of the metadata), and a value (the actual data). The encoding of + * the value depends on the type of the metadata. + * + * The type of metadata is specified by an enumerated type __itt_metdata_type. + * @{ + */ + +/** + * @ingroup parameters + * @brief describes the type of metadata + */ +typedef enum { + __itt_metadata_unknown = 0, + __itt_metadata_u64, /**< Unsigned 64-bit integer */ + __itt_metadata_s64, /**< Signed 64-bit integer */ + __itt_metadata_u32, /**< Unsigned 32-bit integer */ + __itt_metadata_s32, /**< Signed 32-bit integer */ + __itt_metadata_u16, /**< Unsigned 16-bit integer */ + __itt_metadata_s16, /**< Signed 16-bit integer */ + __itt_metadata_float, /**< Signed 32-bit floating-point */ + __itt_metadata_double /**< SIgned 64-bit floating-point */ +} __itt_metadata_type; + +/** + * @ingroup parameters + * @brief Add metadata to an instance of a named entity. + * @param[in] domain The domain controlling the call + * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task + * @param[in] key The name of the metadata + * @param[in] type The type of the metadata + * @param[in] count The number of elements of the given type. If count == 0, no metadata will be added. + * @param[in] data The metadata itself +*/ +void ITTAPI __itt_metadata_add(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, metadata_add, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data)) +#define __itt_metadata_add(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(metadata_add,d,x,y,z,a,b) +#define __itt_metadata_add_ptr ITTNOTIFY_NAME(metadata_add) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_metadata_add(d,x,y,z,a,b) +#define __itt_metadata_add_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_metadata_add_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup parameters + * @brief Add string metadata to an instance of a named entity. + * @param[in] domain The domain controlling the call + * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task + * @param[in] key The name of the metadata + * @param[in] data The metadata itself + * @param[in] length The number of characters in the string, or -1 if the length is unknown but the string is null-terminated +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_metadata_str_addA(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length); +void ITTAPI __itt_metadata_str_addW(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const wchar_t *data, size_t length); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_metadata_str_add __itt_metadata_str_addW +# define __itt_metadata_str_add_ptr __itt_metadata_str_addW_ptr +#else /* UNICODE */ +# define __itt_metadata_str_add __itt_metadata_str_addA +# define __itt_metadata_str_add_ptr __itt_metadata_str_addA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_metadata_str_add(const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length); +#endif + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, metadata_str_addA, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length)) +ITT_STUBV(ITTAPI, void, metadata_str_addW, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const wchar_t *data, size_t length)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, metadata_str_add, (const __itt_domain *domain, __itt_id id, __itt_string_handle *key, const char *data, size_t length)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_addA(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_addA,d,x,y,z,a) +#define __itt_metadata_str_addA_ptr ITTNOTIFY_NAME(metadata_str_addA) +#define __itt_metadata_str_addW(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_addW,d,x,y,z,a) +#define __itt_metadata_str_addW_ptr ITTNOTIFY_NAME(metadata_str_addW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add,d,x,y,z,a) +#define __itt_metadata_str_add_ptr ITTNOTIFY_NAME(metadata_str_add) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_addA(d,x,y,z,a) +#define __itt_metadata_str_addA_ptr 0 +#define __itt_metadata_str_addW(d,x,y,z,a) +#define __itt_metadata_str_addW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add(d,x,y,z,a) +#define __itt_metadata_str_add_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_addA_ptr 0 +#define __itt_metadata_str_addW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup parameters + * @brief Add metadata to an instance of a named entity. + * @param[in] domain The domain controlling the call + * @param[in] scope The scope of the instance to which the metadata is to be added + + * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task + + * @param[in] key The name of the metadata + * @param[in] type The type of the metadata + * @param[in] count The number of elements of the given type. If count == 0, no metadata will be added. + * @param[in] data The metadata itself +*/ +void ITTAPI __itt_metadata_add_with_scope(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, metadata_add_with_scope, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, __itt_metadata_type type, size_t count, void *data)) +#define __itt_metadata_add_with_scope(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(metadata_add_with_scope,d,x,y,z,a,b) +#define __itt_metadata_add_with_scope_ptr ITTNOTIFY_NAME(metadata_add_with_scope) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_metadata_add_with_scope(d,x,y,z,a,b) +#define __itt_metadata_add_with_scope_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_metadata_add_with_scope_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup parameters + * @brief Add string metadata to an instance of a named entity. + * @param[in] domain The domain controlling the call + * @param[in] scope The scope of the instance to which the metadata is to be added + + * @param[in] id The identifier of the instance to which the metadata is to be added, or __itt_null to add to the current task + + * @param[in] key The name of the metadata + * @param[in] data The metadata itself + * @param[in] length The number of characters in the string, or -1 if the length is unknown but the string is null-terminated +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_metadata_str_add_with_scopeA(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length); +void ITTAPI __itt_metadata_str_add_with_scopeW(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const wchar_t *data, size_t length); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_metadata_str_add_with_scope __itt_metadata_str_add_with_scopeW +# define __itt_metadata_str_add_with_scope_ptr __itt_metadata_str_add_with_scopeW_ptr +#else /* UNICODE */ +# define __itt_metadata_str_add_with_scope __itt_metadata_str_add_with_scopeA +# define __itt_metadata_str_add_with_scope_ptr __itt_metadata_str_add_with_scopeA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_metadata_str_add_with_scope(const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length); +#endif + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUBV(ITTAPI, void, metadata_str_add_with_scopeA, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length)) +ITT_STUBV(ITTAPI, void, metadata_str_add_with_scopeW, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const wchar_t *data, size_t length)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUBV(ITTAPI, void, metadata_str_add_with_scope, (const __itt_domain *domain, __itt_scope scope, __itt_string_handle *key, const char *data, size_t length)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_add_with_scopeA(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add_with_scopeA,d,x,y,z,a) +#define __itt_metadata_str_add_with_scopeA_ptr ITTNOTIFY_NAME(metadata_str_add_with_scopeA) +#define __itt_metadata_str_add_with_scopeW(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add_with_scopeW,d,x,y,z,a) +#define __itt_metadata_str_add_with_scopeW_ptr ITTNOTIFY_NAME(metadata_str_add_with_scopeW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add_with_scope(d,x,y,z,a) ITTNOTIFY_VOID_D4(metadata_str_add_with_scope,d,x,y,z,a) +#define __itt_metadata_str_add_with_scope_ptr ITTNOTIFY_NAME(metadata_str_add_with_scope) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_add_with_scopeA(d,x,y,z,a) +#define __itt_metadata_str_add_with_scopeA_ptr 0 +#define __itt_metadata_str_add_with_scopeW(d,x,y,z,a) +#define __itt_metadata_str_add_with_scopeW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add_with_scope(d,x,y,z,a) +#define __itt_metadata_str_add_with_scope_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_metadata_str_add_with_scopeA_ptr 0 +#define __itt_metadata_str_add_with_scopeW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_metadata_str_add_with_scope_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @} metadata group */ + +/** + * @defgroup relations Relations + * Instances of named entities can be explicitly associated with other + * instances using instance IDs and the relationship API calls. + * + * @{ + */ + +/** + * @ingroup relations + * @brief The kind of relation between two instances is specified by the enumerated type __itt_relation. + * Relations between instances can be added with an API call. The relation + * API uses instance IDs. Relations can be added before or after the actual + * instances are created and persist independently of the instances. This + * is the motivation for having different lifetimes for instance IDs and + * the actual instances. + */ +typedef enum +{ + __itt_relation_is_unknown = 0, + __itt_relation_is_dependent_on, /**< "A is dependent on B" means that A cannot start until B completes */ + __itt_relation_is_sibling_of, /**< "A is sibling of B" means that A and B were created as a group */ + __itt_relation_is_parent_of, /**< "A is parent of B" means that A created B */ + __itt_relation_is_continuation_of, /**< "A is continuation of B" means that A assumes the dependencies of B */ + __itt_relation_is_child_of, /**< "A is child of B" means that A was created by B (inverse of is_parent_of) */ + __itt_relation_is_continued_by, /**< "A is continued by B" means that B assumes the dependencies of A (inverse of is_continuation_of) */ + __itt_relation_is_predecessor_to /**< "A is predecessor to B" means that B cannot start until A completes (inverse of is_dependent_on) */ +} __itt_relation; + +/** + * @ingroup relations + * @brief Add a relation to the current task instance. + * The current task instance is the head of the relation. + * @param[in] domain The domain controlling this call + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add_to_current(const __itt_domain *domain, __itt_relation relation, __itt_id tail); + +/** + * @ingroup relations + * @brief Add a relation between two instance identifiers. + * @param[in] domain The domain controlling this call + * @param[in] head The ID for the head of the relation + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add(const __itt_domain *domain, __itt_id head, __itt_relation relation, __itt_id tail); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, relation_add_to_current, (const __itt_domain *domain, __itt_relation relation, __itt_id tail)) +ITT_STUBV(ITTAPI, void, relation_add, (const __itt_domain *domain, __itt_id head, __itt_relation relation, __itt_id tail)) +#define __itt_relation_add_to_current(d,x,y) ITTNOTIFY_VOID_D2(relation_add_to_current,d,x,y) +#define __itt_relation_add_to_current_ptr ITTNOTIFY_NAME(relation_add_to_current) +#define __itt_relation_add(d,x,y,z) ITTNOTIFY_VOID_D3(relation_add,d,x,y,z) +#define __itt_relation_add_ptr ITTNOTIFY_NAME(relation_add) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_relation_add_to_current(d,x,y) +#define __itt_relation_add_to_current_ptr 0 +#define __itt_relation_add(d,x,y,z) +#define __itt_relation_add_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_relation_add_to_current_ptr 0 +#define __itt_relation_add_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} relations group */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_clock_info +{ + unsigned long long clock_freq; /*!< Clock domain frequency */ + unsigned long long clock_base; /*!< Clock domain base timestamp */ +} __itt_clock_info; + +#pragma pack(pop) +/** @endcond */ + +/** @cond exclude_from_documentation */ +typedef void (ITTAPI *__itt_get_clock_info_fn)(__itt_clock_info* clock_info, void* data); +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_clock_domain +{ + __itt_clock_info info; /*!< Most recent clock domain info */ + __itt_get_clock_info_fn fn; /*!< Callback function pointer */ + void* fn_data; /*!< Input argument for the callback function */ + int extra1; /*!< Reserved. Must be zero */ + void* extra2; /*!< Reserved. Must be zero */ + struct ___itt_clock_domain* next; +} __itt_clock_domain; + +#pragma pack(pop) +/** @endcond */ + +/** + * @ingroup clockdomains + * @brief Create a clock domain. + * Certain applications require the capability to trace their application using + * a clock domain different than the CPU, for instance the instrumentation of events + * that occur on a GPU. + * Because the set of domains is expected to be static over the application's execution time, + * there is no mechanism to destroy a domain. + * Any domain can be accessed by any thread in the process, regardless of which thread created + * the domain. This call is thread-safe. + * @param[in] fn A pointer to a callback function which retrieves alternative CPU timestamps + * @param[in] fn_data Argument for a callback function; may be NULL + */ +__itt_clock_domain* ITTAPI __itt_clock_domain_create(__itt_get_clock_info_fn fn, void* fn_data); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_clock_domain*, clock_domain_create, (__itt_get_clock_info_fn fn, void* fn_data)) +#define __itt_clock_domain_create ITTNOTIFY_DATA(clock_domain_create) +#define __itt_clock_domain_create_ptr ITTNOTIFY_NAME(clock_domain_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_clock_domain_create(fn,fn_data) (__itt_clock_domain*)0 +#define __itt_clock_domain_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_clock_domain_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup clockdomains + * @brief Recalculate clock domains frequencies and clock base timestamps. + */ +void ITTAPI __itt_clock_domain_reset(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, clock_domain_reset, (void)) +#define __itt_clock_domain_reset ITTNOTIFY_VOID(clock_domain_reset) +#define __itt_clock_domain_reset_ptr ITTNOTIFY_NAME(clock_domain_reset) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_clock_domain_reset() +#define __itt_clock_domain_reset_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_clock_domain_reset_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup clockdomain + * @brief Create an instance of identifier. This establishes the beginning of the lifetime of + * an instance of the given ID in the trace. Once this lifetime starts, the ID can be used to + * tag named entity instances in calls such as __itt_task_begin, and to specify relationships among + * identified named entity instances, using the \ref relations APIs. + * @param[in] domain The domain controlling the execution of this call. + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] id The ID to create. + */ +void ITTAPI __itt_id_create_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id); + +/** + * @ingroup clockdomain + * @brief Destroy an instance of identifier. This ends the lifetime of the current instance of the + * given ID value in the trace. Any relationships that are established after this lifetime ends are + * invalid. This call must be performed before the given ID value can be reused for a different + * named entity instance. + * @param[in] domain The domain controlling the execution of this call. + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] id The ID to destroy. + */ +void ITTAPI __itt_id_destroy_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, id_create_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id)) +ITT_STUBV(ITTAPI, void, id_destroy_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id)) +#define __itt_id_create_ex(d,x,y,z) ITTNOTIFY_VOID_D3(id_create_ex,d,x,y,z) +#define __itt_id_create_ex_ptr ITTNOTIFY_NAME(id_create_ex) +#define __itt_id_destroy_ex(d,x,y,z) ITTNOTIFY_VOID_D3(id_destroy_ex,d,x,y,z) +#define __itt_id_destroy_ex_ptr ITTNOTIFY_NAME(id_destroy_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_id_create_ex(domain,clock_domain,timestamp,id) +#define __itt_id_create_ex_ptr 0 +#define __itt_id_destroy_ex(domain,clock_domain,timestamp,id) +#define __itt_id_destroy_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_id_create_ex_ptr 0 +#define __itt_id_destroy_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup clockdomain + * @brief Begin a task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] taskid The instance ID for this task instance, or __itt_null + * @param[in] parentid The parent instance to which this task instance belongs, or __itt_null + * @param[in] name The name of this task + */ +void ITTAPI __itt_task_begin_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, __itt_string_handle* name); + +/** + * @ingroup clockdomain + * @brief Begin a task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] taskid The identifier for this task instance, or __itt_null + * @param[in] parentid The parent of this task, or __itt_null + * @param[in] fn The pointer to the function you are tracing + */ +void ITTAPI __itt_task_begin_fn_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, void* fn); + +/** + * @ingroup clockdomain + * @brief End the current task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + */ +void ITTAPI __itt_task_end_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, task_begin_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, task_begin_fn_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_id parentid, void* fn)) +ITT_STUBV(ITTAPI, void, task_end_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp)) +#define __itt_task_begin_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(task_begin_ex,d,x,y,z,a,b) +#define __itt_task_begin_ex_ptr ITTNOTIFY_NAME(task_begin_ex) +#define __itt_task_begin_fn_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(task_begin_fn_ex,d,x,y,z,a,b) +#define __itt_task_begin_fn_ex_ptr ITTNOTIFY_NAME(task_begin_fn_ex) +#define __itt_task_end_ex(d,x,y) ITTNOTIFY_VOID_D2(task_end_ex,d,x,y) +#define __itt_task_end_ex_ptr ITTNOTIFY_NAME(task_end_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_task_begin_ex(domain,clock_domain,timestamp,id,parentid,name) +#define __itt_task_begin_ex_ptr 0 +#define __itt_task_begin_fn_ex(domain,clock_domain,timestamp,id,parentid,fn) +#define __itt_task_begin_fn_ex_ptr 0 +#define __itt_task_end_ex(domain,clock_domain,timestamp) +#define __itt_task_end_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_task_begin_ex_ptr 0 +#define __itt_task_begin_fn_ex_ptr 0 +#define __itt_task_end_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @defgroup counters Counters + * @ingroup public + * Counters are user-defined objects with a monotonically increasing + * value. Counter values are 64-bit unsigned integers. + * Counters have names that can be displayed in + * the tools. + * @{ + */ + +/** + * @brief opaque structure for counter identification + */ +/** @cond exclude_from_documentation */ + +typedef struct ___itt_counter* __itt_counter; + +/** + * @brief Create an unsigned 64 bits integer counter with given name/domain + * + * After __itt_counter_create() is called, __itt_counter_inc(id), __itt_counter_inc_delta(id, delta), + * __itt_counter_set_value(id, value_ptr) or __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr) + * can be used to change the value of the counter, where value_ptr is a pointer to an unsigned 64 bits integer + * + * The call is equal to __itt_counter_create_typed(name, domain, __itt_metadata_u64) + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_counter ITTAPI __itt_counter_createA(const char *name, const char *domain); +__itt_counter ITTAPI __itt_counter_createW(const wchar_t *name, const wchar_t *domain); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_counter_create __itt_counter_createW +# define __itt_counter_create_ptr __itt_counter_createW_ptr +#else /* UNICODE */ +# define __itt_counter_create __itt_counter_createA +# define __itt_counter_create_ptr __itt_counter_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_counter ITTAPI __itt_counter_create(const char *name, const char *domain); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_counter, counter_createA, (const char *name, const char *domain)) +ITT_STUB(ITTAPI, __itt_counter, counter_createW, (const wchar_t *name, const wchar_t *domain)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_counter, counter_create, (const char *name, const char *domain)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA ITTNOTIFY_DATA(counter_createA) +#define __itt_counter_createA_ptr ITTNOTIFY_NAME(counter_createA) +#define __itt_counter_createW ITTNOTIFY_DATA(counter_createW) +#define __itt_counter_createW_ptr ITTNOTIFY_NAME(counter_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create ITTNOTIFY_DATA(counter_create) +#define __itt_counter_create_ptr ITTNOTIFY_NAME(counter_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA(name, domain) +#define __itt_counter_createA_ptr 0 +#define __itt_counter_createW(name, domain) +#define __itt_counter_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create(name, domain) +#define __itt_counter_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA_ptr 0 +#define __itt_counter_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Increment the unsigned 64 bits integer counter value + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_inc(__itt_counter id); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_inc, (__itt_counter id)) +#define __itt_counter_inc ITTNOTIFY_VOID(counter_inc) +#define __itt_counter_inc_ptr ITTNOTIFY_NAME(counter_inc) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_inc(id) +#define __itt_counter_inc_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_inc_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** + * @brief Increment the unsigned 64 bits integer counter value with x + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_inc_delta(__itt_counter id, unsigned long long value); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_inc_delta, (__itt_counter id, unsigned long long value)) +#define __itt_counter_inc_delta ITTNOTIFY_VOID(counter_inc_delta) +#define __itt_counter_inc_delta_ptr ITTNOTIFY_NAME(counter_inc_delta) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_inc_delta(id, value) +#define __itt_counter_inc_delta_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_inc_delta_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Decrement the unsigned 64 bits integer counter value + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_dec(__itt_counter id); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_dec, (__itt_counter id)) +#define __itt_counter_dec ITTNOTIFY_VOID(counter_dec) +#define __itt_counter_dec_ptr ITTNOTIFY_NAME(counter_dec) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_dec(id) +#define __itt_counter_dec_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_dec_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** + * @brief Decrement the unsigned 64 bits integer counter value with x + * + * Calling this function to non-unsigned 64 bits integer counters has no effect + */ +void ITTAPI __itt_counter_dec_delta(__itt_counter id, unsigned long long value); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_dec_delta, (__itt_counter id, unsigned long long value)) +#define __itt_counter_dec_delta ITTNOTIFY_VOID(counter_dec_delta) +#define __itt_counter_dec_delta_ptr ITTNOTIFY_NAME(counter_dec_delta) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_dec_delta(id, value) +#define __itt_counter_dec_delta_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_dec_delta_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup counters + * @brief Increment a counter by one. + * The first call with a given name creates a counter by that name and sets its + * value to zero. Successive calls increment the counter value. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + */ +void ITTAPI __itt_counter_inc_v3(const __itt_domain *domain, __itt_string_handle *name); + +/** + * @ingroup counters + * @brief Increment a counter by the value specified in delta. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + * @param[in] delta The amount by which to increment the counter + */ +void ITTAPI __itt_counter_inc_delta_v3(const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_inc_v3, (const __itt_domain *domain, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, counter_inc_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta)) +#define __itt_counter_inc_v3(d,x) ITTNOTIFY_VOID_D1(counter_inc_v3,d,x) +#define __itt_counter_inc_v3_ptr ITTNOTIFY_NAME(counter_inc_v3) +#define __itt_counter_inc_delta_v3(d,x,y) ITTNOTIFY_VOID_D2(counter_inc_delta_v3,d,x,y) +#define __itt_counter_inc_delta_v3_ptr ITTNOTIFY_NAME(counter_inc_delta_v3) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_inc_v3(domain,name) +#define __itt_counter_inc_v3_ptr 0 +#define __itt_counter_inc_delta_v3(domain,name,delta) +#define __itt_counter_inc_delta_v3_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_inc_v3_ptr 0 +#define __itt_counter_inc_delta_v3_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + + +/** + * @ingroup counters + * @brief Decrement a counter by one. + * The first call with a given name creates a counter by that name and sets its + * value to zero. Successive calls decrement the counter value. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + */ +void ITTAPI __itt_counter_dec_v3(const __itt_domain *domain, __itt_string_handle *name); + +/** + * @ingroup counters + * @brief Decrement a counter by the value specified in delta. + * @param[in] domain The domain controlling the call. Counter names are not domain specific. + * The domain argument is used only to enable or disable the API calls. + * @param[in] name The name of the counter + * @param[in] delta The amount by which to decrement the counter + */ +void ITTAPI __itt_counter_dec_delta_v3(const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_dec_v3, (const __itt_domain *domain, __itt_string_handle *name)) +ITT_STUBV(ITTAPI, void, counter_dec_delta_v3, (const __itt_domain *domain, __itt_string_handle *name, unsigned long long delta)) +#define __itt_counter_dec_v3(d,x) ITTNOTIFY_VOID_D1(counter_dec_v3,d,x) +#define __itt_counter_dec_v3_ptr ITTNOTIFY_NAME(counter_dec_v3) +#define __itt_counter_dec_delta_v3(d,x,y) ITTNOTIFY_VOID_D2(counter_dec_delta_v3,d,x,y) +#define __itt_counter_dec_delta_v3_ptr ITTNOTIFY_NAME(counter_dec_delta_v3) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_dec_v3(domain,name) +#define __itt_counter_dec_v3_ptr 0 +#define __itt_counter_dec_delta_v3(domain,name,delta) +#define __itt_counter_dec_delta_v3_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_dec_v3_ptr 0 +#define __itt_counter_dec_delta_v3_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @} counters group */ + + +/** + * @brief Set the counter value + */ +void ITTAPI __itt_counter_set_value(__itt_counter id, void *value_ptr); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_set_value, (__itt_counter id, void *value_ptr)) +#define __itt_counter_set_value ITTNOTIFY_VOID(counter_set_value) +#define __itt_counter_set_value_ptr ITTNOTIFY_NAME(counter_set_value) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_set_value(id, value_ptr) +#define __itt_counter_set_value_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_set_value_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Set the counter value + */ +void ITTAPI __itt_counter_set_value_ex(__itt_counter id, __itt_clock_domain *clock_domain, unsigned long long timestamp, void *value_ptr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_set_value_ex, (__itt_counter id, __itt_clock_domain *clock_domain, unsigned long long timestamp, void *value_ptr)) +#define __itt_counter_set_value_ex ITTNOTIFY_VOID(counter_set_value_ex) +#define __itt_counter_set_value_ex_ptr ITTNOTIFY_NAME(counter_set_value_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr) +#define __itt_counter_set_value_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_set_value_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Create a typed counter with given name/domain + * + * After __itt_counter_create_typed() is called, __itt_counter_inc(id), __itt_counter_inc_delta(id, delta), + * __itt_counter_set_value(id, value_ptr) or __itt_counter_set_value_ex(id, clock_domain, timestamp, value_ptr) + * can be used to change the value of the counter + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_counter ITTAPI __itt_counter_create_typedA(const char *name, const char *domain, __itt_metadata_type type); +__itt_counter ITTAPI __itt_counter_create_typedW(const wchar_t *name, const wchar_t *domain, __itt_metadata_type type); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_counter_create_typed __itt_counter_create_typedW +# define __itt_counter_create_typed_ptr __itt_counter_create_typedW_ptr +#else /* UNICODE */ +# define __itt_counter_create_typed __itt_counter_create_typedA +# define __itt_counter_create_typed_ptr __itt_counter_create_typedA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_counter ITTAPI __itt_counter_create_typed(const char *name, const char *domain, __itt_metadata_type type); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_counter, counter_create_typedA, (const char *name, const char *domain, __itt_metadata_type type)) +ITT_STUB(ITTAPI, __itt_counter, counter_create_typedW, (const wchar_t *name, const wchar_t *domain, __itt_metadata_type type)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_counter, counter_create_typed, (const char *name, const char *domain, __itt_metadata_type type)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_create_typedA ITTNOTIFY_DATA(counter_create_typedA) +#define __itt_counter_create_typedA_ptr ITTNOTIFY_NAME(counter_create_typedA) +#define __itt_counter_create_typedW ITTNOTIFY_DATA(counter_create_typedW) +#define __itt_counter_create_typedW_ptr ITTNOTIFY_NAME(counter_create_typedW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_typed ITTNOTIFY_DATA(counter_create_typed) +#define __itt_counter_create_typed_ptr ITTNOTIFY_NAME(counter_create_typed) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_create_typedA(name, domain, type) +#define __itt_counter_create_typedA_ptr 0 +#define __itt_counter_create_typedW(name, domain, type) +#define __itt_counter_create_typedW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_typed(name, domain, type) +#define __itt_counter_create_typed_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_create_typedA_ptr 0 +#define __itt_counter_create_typedW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_typed_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Destroy the counter identified by the pointer previously returned by __itt_counter_create() or + * __itt_counter_create_typed() + */ +void ITTAPI __itt_counter_destroy(__itt_counter id); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_destroy, (__itt_counter id)) +#define __itt_counter_destroy ITTNOTIFY_VOID(counter_destroy) +#define __itt_counter_destroy_ptr ITTNOTIFY_NAME(counter_destroy) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_destroy(id) +#define __itt_counter_destroy_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_destroy_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} counters group */ + +/** + * @ingroup markers + * @brief Create a marker instance. + * @param[in] domain The domain for this marker + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] id The instance ID for this marker, or __itt_null + * @param[in] name The name for this marker + * @param[in] scope The scope for this marker + */ +void ITTAPI __itt_marker_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, marker_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id id, __itt_string_handle *name, __itt_scope scope)) +#define __itt_marker_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(marker_ex,d,x,y,z,a,b) +#define __itt_marker_ex_ptr ITTNOTIFY_NAME(marker_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_marker_ex(domain,clock_domain,timestamp,id,name,scope) +#define __itt_marker_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_marker_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @ingroup clockdomain + * @brief Add a relation to the current task instance. + * The current task instance is the head of the relation. + * @param[in] domain The domain controlling this call + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add_to_current_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail); + +/** + * @ingroup clockdomain + * @brief Add a relation between two instance identifiers. + * @param[in] domain The domain controlling this call + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] head The ID for the head of the relation + * @param[in] relation The kind of relation + * @param[in] tail The ID for the tail of the relation + */ +void ITTAPI __itt_relation_add_ex(const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, relation_add_to_current_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_relation relation, __itt_id tail)) +ITT_STUBV(ITTAPI, void, relation_add_ex, (const __itt_domain *domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id head, __itt_relation relation, __itt_id tail)) +#define __itt_relation_add_to_current_ex(d,x,y,z,a) ITTNOTIFY_VOID_D4(relation_add_to_current_ex,d,x,y,z,a) +#define __itt_relation_add_to_current_ex_ptr ITTNOTIFY_NAME(relation_add_to_current_ex) +#define __itt_relation_add_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(relation_add_ex,d,x,y,z,a,b) +#define __itt_relation_add_ex_ptr ITTNOTIFY_NAME(relation_add_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_relation_add_to_current_ex(domain,clock_domain,timestame,relation,tail) +#define __itt_relation_add_to_current_ex_ptr 0 +#define __itt_relation_add_ex(domain,clock_domain,timestamp,head,relation,tail) +#define __itt_relation_add_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_relation_add_to_current_ex_ptr 0 +#define __itt_relation_add_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_documentation */ +typedef enum ___itt_track_group_type +{ + __itt_track_group_type_normal = 0 +} __itt_track_group_type; +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_track_group +{ + __itt_string_handle* name; /*!< Name of the track group */ + struct ___itt_track* track; /*!< List of child tracks */ + __itt_track_group_type tgtype; /*!< Type of the track group */ + int extra1; /*!< Reserved. Must be zero */ + void* extra2; /*!< Reserved. Must be zero */ + struct ___itt_track_group* next; +} __itt_track_group; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Placeholder for custom track types. Currently, "normal" custom track + * is the only available track type. + */ +typedef enum ___itt_track_type +{ + __itt_track_type_normal = 0 +#ifdef INTEL_ITTNOTIFY_API_PRIVATE + , __itt_track_type_queue +#endif /* INTEL_ITTNOTIFY_API_PRIVATE */ +} __itt_track_type; + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_track +{ + __itt_string_handle* name; /*!< Name of the track group */ + __itt_track_group* group; /*!< Parent group to a track */ + __itt_track_type ttype; /*!< Type of the track */ + int extra1; /*!< Reserved. Must be zero */ + void* extra2; /*!< Reserved. Must be zero */ + struct ___itt_track* next; +} __itt_track; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Create logical track group. + */ +__itt_track_group* ITTAPI __itt_track_group_create(__itt_string_handle* name, __itt_track_group_type track_group_type); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_track_group*, track_group_create, (__itt_string_handle* name, __itt_track_group_type track_group_type)) +#define __itt_track_group_create ITTNOTIFY_DATA(track_group_create) +#define __itt_track_group_create_ptr ITTNOTIFY_NAME(track_group_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_track_group_create(name) (__itt_track_group*)0 +#define __itt_track_group_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_track_group_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Create logical track. + */ +__itt_track* ITTAPI __itt_track_create(__itt_track_group* track_group, __itt_string_handle* name, __itt_track_type track_type); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_track*, track_create, (__itt_track_group* track_group,__itt_string_handle* name, __itt_track_type track_type)) +#define __itt_track_create ITTNOTIFY_DATA(track_create) +#define __itt_track_create_ptr ITTNOTIFY_NAME(track_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_track_create(track_group,name,track_type) (__itt_track*)0 +#define __itt_track_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_track_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Set the logical track. + */ +void ITTAPI __itt_set_track(__itt_track* track); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, set_track, (__itt_track *track)) +#define __itt_set_track ITTNOTIFY_VOID(set_track) +#define __itt_set_track_ptr ITTNOTIFY_NAME(set_track) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_set_track(track) +#define __itt_set_track_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_set_track_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/* ========================================================================== */ +/** @cond exclude_from_gpa_documentation */ +/** + * @defgroup events Events + * @ingroup public + * Events group + * @{ + */ +/** @brief user event type */ +typedef int __itt_event; + +/** + * @brief Create an event notification + * @note name or namelen being null/name and namelen not matching, user event feature not enabled + * @return non-zero event identifier upon success and __itt_err otherwise + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_event LIBITTAPI __itt_event_createA(const char *name, int namelen); +__itt_event LIBITTAPI __itt_event_createW(const wchar_t *name, int namelen); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_event_create __itt_event_createW +# define __itt_event_create_ptr __itt_event_createW_ptr +#else +# define __itt_event_create __itt_event_createA +# define __itt_event_create_ptr __itt_event_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_event LIBITTAPI __itt_event_create(const char *name, int namelen); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(LIBITTAPI, __itt_event, event_createA, (const char *name, int namelen)) +ITT_STUB(LIBITTAPI, __itt_event, event_createW, (const wchar_t *name, int namelen)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(LIBITTAPI, __itt_event, event_create, (const char *name, int namelen)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_event_createA ITTNOTIFY_DATA(event_createA) +#define __itt_event_createA_ptr ITTNOTIFY_NAME(event_createA) +#define __itt_event_createW ITTNOTIFY_DATA(event_createW) +#define __itt_event_createW_ptr ITTNOTIFY_NAME(event_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_event_create ITTNOTIFY_DATA(event_create) +#define __itt_event_create_ptr ITTNOTIFY_NAME(event_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_event_createA(name, namelen) (__itt_event)0 +#define __itt_event_createA_ptr 0 +#define __itt_event_createW(name, namelen) (__itt_event)0 +#define __itt_event_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_event_create(name, namelen) (__itt_event)0 +#define __itt_event_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_event_createA_ptr 0 +#define __itt_event_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_event_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an event occurrence. + * @return __itt_err upon failure (invalid event id/user event feature not enabled) + */ +int LIBITTAPI __itt_event_start(__itt_event event); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(LIBITTAPI, int, event_start, (__itt_event event)) +#define __itt_event_start ITTNOTIFY_DATA(event_start) +#define __itt_event_start_ptr ITTNOTIFY_NAME(event_start) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_event_start(event) (int)0 +#define __itt_event_start_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_event_start_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Record an event end occurrence. + * @note It is optional if events do not have durations. + * @return __itt_err upon failure (invalid event id/user event feature not enabled) + */ +int LIBITTAPI __itt_event_end(__itt_event event); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(LIBITTAPI, int, event_end, (__itt_event event)) +#define __itt_event_end ITTNOTIFY_DATA(event_end) +#define __itt_event_end_ptr ITTNOTIFY_NAME(event_end) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_event_end(event) (int)0 +#define __itt_event_end_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_event_end_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} events group */ + + +/** + * @defgroup arrays Arrays Visualizer + * @ingroup public + * Visualize arrays + * @{ + */ + +/** + * @enum __itt_av_data_type + * @brief Defines types of arrays data (for C/C++ intrinsic types) + */ +typedef enum +{ + __itt_e_first = 0, + __itt_e_char = 0, /* 1-byte integer */ + __itt_e_uchar, /* 1-byte unsigned integer */ + __itt_e_int16, /* 2-byte integer */ + __itt_e_uint16, /* 2-byte unsigned integer */ + __itt_e_int32, /* 4-byte integer */ + __itt_e_uint32, /* 4-byte unsigned integer */ + __itt_e_int64, /* 8-byte integer */ + __itt_e_uint64, /* 8-byte unsigned integer */ + __itt_e_float, /* 4-byte floating */ + __itt_e_double, /* 8-byte floating */ + __itt_e_last = __itt_e_double +} __itt_av_data_type; + +/** + * @brief Save an array data to a file. + * Output format is defined by the file extension. The csv and bmp formats are supported (bmp - for 2-dimensional array only). + * @param[in] data - pointer to the array data + * @param[in] rank - the rank of the array + * @param[in] dimensions - pointer to an array of integers, which specifies the array dimensions. + * The size of dimensions must be equal to the rank + * @param[in] type - the type of the array, specified as one of the __itt_av_data_type values (for intrinsic types) + * @param[in] filePath - the file path; the output format is defined by the file extension + * @param[in] columnOrder - defines how the array is stored in the linear memory. + * It should be 1 for column-major order (e.g. in FORTRAN) or 0 - for row-major order (e.g. in C). + */ + +#if ITT_PLATFORM==ITT_PLATFORM_WIN +int ITTAPI __itt_av_saveA(void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder); +int ITTAPI __itt_av_saveW(void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_av_save __itt_av_saveW +# define __itt_av_save_ptr __itt_av_saveW_ptr +#else /* UNICODE */ +# define __itt_av_save __itt_av_saveA +# define __itt_av_save_ptr __itt_av_saveA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +int ITTAPI __itt_av_save(void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, int, av_saveA, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder)) +ITT_STUB(ITTAPI, int, av_saveW, (void *data, int rank, const int *dimensions, int type, const wchar_t *filePath, int columnOrder)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, int, av_save, (void *data, int rank, const int *dimensions, int type, const char *filePath, int columnOrder)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_av_saveA ITTNOTIFY_DATA(av_saveA) +#define __itt_av_saveA_ptr ITTNOTIFY_NAME(av_saveA) +#define __itt_av_saveW ITTNOTIFY_DATA(av_saveW) +#define __itt_av_saveW_ptr ITTNOTIFY_NAME(av_saveW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_av_save ITTNOTIFY_DATA(av_save) +#define __itt_av_save_ptr ITTNOTIFY_NAME(av_save) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_av_saveA(name) +#define __itt_av_saveA_ptr 0 +#define __itt_av_saveW(name) +#define __itt_av_saveW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_av_save(name) +#define __itt_av_save_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_av_saveA_ptr 0 +#define __itt_av_saveW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_av_save_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +void ITTAPI __itt_enable_attach(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, enable_attach, (void)) +#define __itt_enable_attach ITTNOTIFY_VOID(enable_attach) +#define __itt_enable_attach_ptr ITTNOTIFY_NAME(enable_attach) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_enable_attach() +#define __itt_enable_attach_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_enable_attach_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_gpa_documentation */ + +/** @} arrays group */ + +/** @endcond */ + +/** + * @brief Module load notification + * This API is used to report necessary information in case of bypassing default system loader. + * Notification should be done immidiatelly after this module is loaded to process memory. + * @param[in] start_addr - module start address + * @param[in] end_addr - module end address + * @param[in] path - file system full path to the module + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +void ITTAPI __itt_module_loadA(void *start_addr, void *end_addr, const char *path); +void ITTAPI __itt_module_loadW(void *start_addr, void *end_addr, const wchar_t *path); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_module_load __itt_module_loadW +# define __itt_module_load_ptr __itt_module_loadW_ptr +#else /* UNICODE */ +# define __itt_module_load __itt_module_loadA +# define __itt_module_load_ptr __itt_module_loadA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +void ITTAPI __itt_module_load(void *start_addr, void *end_addr, const char *path); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, void, module_loadA, (void *start_addr, void *end_addr, const char *path)) +ITT_STUB(ITTAPI, void, module_loadW, (void *start_addr, void *end_addr, const wchar_t *path)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, void, module_load, (void *start_addr, void *end_addr, const char *path)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_module_loadA ITTNOTIFY_VOID(module_loadA) +#define __itt_module_loadA_ptr ITTNOTIFY_NAME(module_loadA) +#define __itt_module_loadW ITTNOTIFY_VOID(module_loadW) +#define __itt_module_loadW_ptr ITTNOTIFY_NAME(module_loadW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_module_load ITTNOTIFY_VOID(module_load) +#define __itt_module_load_ptr ITTNOTIFY_NAME(module_load) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_module_loadA(start_addr, end_addr, path) +#define __itt_module_loadA_ptr 0 +#define __itt_module_loadW(start_addr, end_addr, path) +#define __itt_module_loadW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_module_load(start_addr, end_addr, path) +#define __itt_module_load_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_module_loadA_ptr 0 +#define __itt_module_loadW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_module_load_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Report module unload + * This API is used to report necessary information in case of bypassing default system loader. + * Notification should be done just before the module is unloaded from process memory. + * @param[in] addr - base address of loaded module + */ +void ITTAPI __itt_module_unload(void *addr); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, module_unload, (void *addr)) +#define __itt_module_unload ITTNOTIFY_VOID(module_unload) +#define __itt_module_unload_ptr ITTNOTIFY_NAME(module_unload) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_module_unload(addr) +#define __itt_module_unload_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_module_unload_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_documentation */ +typedef enum +{ + __itt_module_type_unknown = 0, + __itt_module_type_elf, + __itt_module_type_coff +} __itt_module_type; +/** @endcond */ + +/** @cond exclude_from_documentation */ +typedef enum +{ + itt_section_type_unknown, + itt_section_type_bss, /* notifies that the section contains uninitialized data. These are the relevant section types and the modules that contain them: + * ELF module: SHT_NOBITS section type + * COFF module: IMAGE_SCN_CNT_UNINITIALIZED_DATA section type + */ + itt_section_type_data, /* notifies that section contains initialized data. These are the relevant section types and the modules that contain them: + * ELF module: SHT_PROGBITS section type + * COFF module: IMAGE_SCN_CNT_INITIALIZED_DATA section type + */ + itt_section_type_text /* notifies that the section contains executable code. These are the relevant section types and the modules that contain them: + * ELF module: SHT_PROGBITS section type + * COFF module: IMAGE_SCN_CNT_CODE section type + */ +} __itt_section_type; +/** @endcond */ + +/** + * @hideinitializer + * @brief bit-mask, detects a section attribute that indicates whether a section can be executed as code: + * These are the relevant section attributes and the modules that contain them: + * ELF module: PF_X section attribute + * COFF module: IMAGE_SCN_MEM_EXECUTE attribute + */ +#define __itt_section_exec 0x20000000 + +/** + * @hideinitializer + * @brief bit-mask, detects a section attribute that indicates whether a section can be read. + * These are the relevant section attributes and the modules that contain them: + * ELF module: PF_R attribute + * COFF module: IMAGE_SCN_MEM_READ attribute + */ +#define __itt_section_read 0x40000000 + +/** + * @hideinitializer + * @brief bit-mask, detects a section attribute that indicates whether a section can be written to. + * These are the relevant section attributes and the modules that contain them: + * ELF module: PF_W attribute + * COFF module: IMAGE_SCN_MEM_WRITE attribute + */ +#define __itt_section_write 0x80000000 + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_section_info +{ + const char* name; /*!< Section name in UTF8 */ + __itt_section_type type; /*!< Section content and semantics description */ + size_t flags; /*!< Section bit flags that describe attributes using bit mask + * Zero if disabled, non-zero if enabled + */ + void* start_addr; /*!< Section load(relocated) start address */ + size_t size; /*!< Section file offset */ + size_t file_offset; /*!< Section size */ +} __itt_section_info; + +#pragma pack(pop) +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_module_object +{ + unsigned int version; /*!< API version*/ + __itt_id module_id; /*!< Unique identifier. This is unchanged for sections that belong to the same module */ + __itt_module_type module_type; /*!< Binary module format */ + const char* module_name; /*!< Unique module name or path to module in UTF8 + * Contains module name when module_bufer and module_size exist + * Contains module path when module_bufer and module_size absent + * module_name remains the same for the certain module_id + */ + void* module_buffer; /*!< Module buffer content */ + size_t module_size; /*!< Module buffer size */ + /*!< If module_buffer and module_size exist, the binary module is dumped onto the system. + * If module_buffer and module_size do not exist, + * the binary module exists on the system already. + * The module_name parameter contains the path to the module. + */ + __itt_section_info* section_array; /*!< Reference to section information */ + size_t section_number; +} __itt_module_object; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Load module content and its loaded(relocated) sections. + * This API is useful to save a module, or specify its location on the system and report information about loaded sections. + * The target module is saved on the system if module buffer content and size are available. + * If module buffer content and size are unavailable, the module name contains the path to the existing binary module. + * @param[in] module_obj - provides module and section information, along with unique module identifiers (name,module ID) + * which bind the binary module to particular sections. + */ +void ITTAPI __itt_module_load_with_sections(__itt_module_object* module_obj); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, module_load_with_sections, (__itt_module_object* module_obj)) +#define __itt_module_load_with_sections ITTNOTIFY_VOID(module_load_with_sections) +#define __itt_module_load_with_sections_ptr ITTNOTIFY_NAME(module_load_with_sections) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_module_load_with_sections(module_obj) +#define __itt_module_load_with_sections_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_module_load_with_sections_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Unload a module and its loaded(relocated) sections. + * This API notifies that the module and its sections were unloaded. + * @param[in] module_obj - provides module and sections information, along with unique module identifiers (name,module ID) + * which bind the binary module to particular sections. + */ +void ITTAPI __itt_module_unload_with_sections(__itt_module_object* module_obj); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, module_unload_with_sections, (__itt_module_object* module_obj)) +#define __itt_module_unload_with_sections ITTNOTIFY_VOID(module_unload_with_sections) +#define __itt_module_unload_with_sections_ptr ITTNOTIFY_NAME(module_unload_with_sections) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_module_unload_with_sections(module_obj) +#define __itt_module_unload_with_sections_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_module_unload_with_sections_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_histogram +{ + const __itt_domain* domain; /*!< Domain of the histogram*/ + const char* nameA; /*!< Name of the histogram */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* nameW; +#else /* UNICODE || _UNICODE */ + void* nameW; +#endif /* UNICODE || _UNICODE */ + __itt_metadata_type x_type; /*!< Type of the histogram X axis */ + __itt_metadata_type y_type; /*!< Type of the histogram Y axis */ + int extra1; /*!< Reserved to the runtime */ + void* extra2; /*!< Reserved to the runtime */ + struct ___itt_histogram* next; +} __itt_histogram; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Create a typed histogram instance with given name/domain. + * @param[in] domain The domain controlling the call. + * @param[in] name The name of the histogram. + * @param[in] x_type The type of the X axis in histogram (may be 0 to calculate batch statistics). + * @param[in] y_type The type of the Y axis in histogram. +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_histogram* ITTAPI __itt_histogram_createA(const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type); +__itt_histogram* ITTAPI __itt_histogram_createW(const __itt_domain* domain, const wchar_t* name, __itt_metadata_type x_type, __itt_metadata_type y_type); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_histogram_create __itt_histogram_createW +# define __itt_histogram_create_ptr __itt_histogram_createW_ptr +#else /* UNICODE */ +# define __itt_histogram_create __itt_histogram_createA +# define __itt_histogram_create_ptr __itt_histogram_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_histogram* ITTAPI __itt_histogram_create(const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_histogram*, histogram_createA, (const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type)) +ITT_STUB(ITTAPI, __itt_histogram*, histogram_createW, (const __itt_domain* domain, const wchar_t* name, __itt_metadata_type x_type, __itt_metadata_type y_type)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_histogram*, histogram_create, (const __itt_domain* domain, const char* name, __itt_metadata_type x_type, __itt_metadata_type y_type)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_histogram_createA ITTNOTIFY_DATA(histogram_createA) +#define __itt_histogram_createA_ptr ITTNOTIFY_NAME(histogram_createA) +#define __itt_histogram_createW ITTNOTIFY_DATA(histogram_createW) +#define __itt_histogram_createW_ptr ITTNOTIFY_NAME(histogram_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_histogram_create ITTNOTIFY_DATA(histogram_create) +#define __itt_histogram_create_ptr ITTNOTIFY_NAME(histogram_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_histogram_createA(domain, name, x_type, y_type) (__itt_histogram*)0 +#define __itt_histogram_createA_ptr 0 +#define __itt_histogram_createW(domain, name, x_type, y_type) (__itt_histogram*)0 +#define __itt_histogram_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_histogram_create(domain, name, x_type, y_type) (__itt_histogram*)0 +#define __itt_histogram_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_histogram_createA_ptr 0 +#define __itt_histogram_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_histogram_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Submit statistics for a histogram instance. + * @param[in] hist Pointer to the histogram instance to which the histogram statistic is to be dumped. + * @param[in] length The number of elements in dumped axis data array. + * @param[in] x_data The X axis dumped data itself (may be NULL to calculate batch statistics). + * @param[in] y_data The Y axis dumped data itself. +*/ +void ITTAPI __itt_histogram_submit(__itt_histogram* hist, size_t length, void* x_data, void* y_data); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, histogram_submit, (__itt_histogram* hist, size_t length, void* x_data, void* y_data)) +#define __itt_histogram_submit ITTNOTIFY_VOID(histogram_submit) +#define __itt_histogram_submit_ptr ITTNOTIFY_NAME(histogram_submit) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_histogram_submit(hist, length, x_data, y_data) +#define __itt_histogram_submit_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_histogram_submit_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ + +/** +* @brief function allows to obtain the current collection state at the moment +* @return collection state as a enum __itt_collection_state +*/ +__itt_collection_state __itt_get_collection_state(void); + +/** +* @brief function releases resources allocated by ITT API static part +* this API should be called from the library destructor +* @return void +*/ +void __itt_release_resources(void); +/** @endcond */ + +/** + * @brief Create a typed counter with given domain pointer, string name and counter type +*/ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_counter ITTAPI __itt_counter_createA_v3(const __itt_domain* domain, const char* name, __itt_metadata_type type); +__itt_counter ITTAPI __itt_counter_createW_v3(const __itt_domain* domain, const wchar_t* name, __itt_metadata_type type); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_counter_create_v3 __itt_counter_createW_v3 +# define __itt_counter_create_v3_ptr __itt_counter_createW_v3_ptr +#else /* UNICODE */ +# define __itt_counter_create_v3 __itt_counter_createA_v3 +# define __itt_counter_create_v3_ptr __itt_counter_createA_v3_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_counter ITTAPI __itt_counter_create_v3(const __itt_domain* domain, const char* name, __itt_metadata_type type); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_counter, counter_createA_v3, (const __itt_domain* domain, const char* name, __itt_metadata_type type)) +ITT_STUB(ITTAPI, __itt_counter, counter_createW_v3, (const __itt_domain* domain, const wchar_t* name, __itt_metadata_type type)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_counter, counter_create_v3, (const __itt_domain* domain, const char* name, __itt_metadata_type type)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA_v3 ITTNOTIFY_DATA(counter_createA_v3) +#define __itt_counter_createA_v3_ptr ITTNOTIFY_NAME(counter_createA_v3) +#define __itt_counter_createW_v3 ITTNOTIFY_DATA(counter_createW_v3) +#define __itt_counter_createW_v3_ptr ITTNOTIFY_NAME(counter_createW_v3) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_v3 ITTNOTIFY_DATA(counter_create_v3) +#define __itt_counter_create_v3_ptr ITTNOTIFY_NAME(counter_create_v3) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA_v3(domain, name, type) (__itt_counter)0 +#define __itt_counter_createA_v3_ptr 0 +#define __itt_counter_createW_v3(domain, name, type) (__itt_counter)0 +#define __itt_counter_create_typedW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_v3(domain, name, type) (__itt_counter)0 +#define __itt_counter_create_v3_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_counter_createA_v3_ptr 0 +#define __itt_counter_createW_v3_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_counter_create_v3_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Set the counter value api + */ +void ITTAPI __itt_counter_set_value_v3(__itt_counter counter, void *value_ptr); + +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, counter_set_value_v3, (__itt_counter counter, void *value_ptr)) +#define __itt_counter_set_value_v3 ITTNOTIFY_VOID(counter_set_value_v3) +#define __itt_counter_set_value_v3_ptr ITTNOTIFY_NAME(counter_set_value_v3) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_counter_set_value_v3(counter, value_ptr) +#define __itt_counter_set_value_v3_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_counter_set_value_v3_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief describes the type of context metadata +*/ +typedef enum { + __itt_context_unknown = 0, /*!< Undefined type */ + __itt_context_nameA, /*!< ASCII string char* type */ + __itt_context_nameW, /*!< Unicode string wchar_t* type */ + __itt_context_deviceA, /*!< ASCII string char* type */ + __itt_context_deviceW, /*!< Unicode string wchar_t* type */ + __itt_context_unitsA, /*!< ASCII string char* type */ + __itt_context_unitsW, /*!< Unicode string wchar_t* type */ + __itt_context_pci_addrA, /*!< ASCII string char* type */ + __itt_context_pci_addrW, /*!< Unicode string wchar_t* type */ + __itt_context_tid, /*!< Unsigned 64-bit integer type */ + __itt_context_max_val, /*!< Unsigned 64-bit integer type */ + __itt_context_bandwidth_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_latency_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_occupancy_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_on_thread_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_is_abs_val_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_cpu_instructions_flag, /*!< Unsigned 64-bit integer type */ + __itt_context_cpu_cycles_flag /*!< Unsigned 64-bit integer type */ +} __itt_context_type; + +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_context_name __itt_context_nameW +# define __itt_context_device __itt_context_deviceW +# define __itt_context_units __itt_context_unitsW +# define __itt_context_pci_addr __itt_context_pci_addrW +#else /* UNICODE || _UNICODE */ +# define __itt_context_name __itt_context_nameA +# define __itt_context_device __itt_context_deviceA +# define __itt_context_units __itt_context_unitsA +# define __itt_context_pci_addr __itt_context_pci_addrA +#endif /* UNICODE || _UNICODE */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_context_metadata +{ + __itt_context_type type; /*!< Type of the context metadata value */ + void* value; /*!< Pointer to context metadata value itself */ +} __itt_context_metadata; + +#pragma pack(pop) +/** @endcond */ + +/** @cond exclude_from_documentation */ +#pragma pack(push, 8) + +typedef struct ___itt_counter_metadata +{ + __itt_counter counter; /*!< Associated context metadata counter */ + __itt_context_type type; /*!< Type of the context metadata value */ + const char* str_valueA; /*!< String context metadata value */ +#if defined(UNICODE) || defined(_UNICODE) + const wchar_t* str_valueW; +#else /* UNICODE || _UNICODE */ + void* str_valueW; +#endif /* UNICODE || _UNICODE */ + unsigned long long value; /*!< Numeric context metadata value */ + int extra1; /*!< Reserved to the runtime */ + void* extra2; /*!< Reserved to the runtime */ + struct ___itt_counter_metadata* next; +} __itt_counter_metadata; + +#pragma pack(pop) +/** @endcond */ + +/** + * @brief Bind context metadata to counter instance + * @param[in] counter Pointer to the counter instance to which the context metadata is to be associated. + * @param[in] length The number of elements in context metadata array. + * @param[in] metadata The context metadata itself. +*/ +void ITTAPI __itt_bind_context_metadata_to_counter(__itt_counter counter, size_t length, __itt_context_metadata* metadata); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, bind_context_metadata_to_counter, (__itt_counter counter, size_t length, __itt_context_metadata* metadata)) +#define __itt_bind_context_metadata_to_counter ITTNOTIFY_VOID(bind_context_metadata_to_counter) +#define __itt_bind_context_metadata_to_counter_ptr ITTNOTIFY_NAME(bind_context_metadata_to_counter) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_bind_context_metadata_to_counter(counter, length, metadata) +#define __itt_bind_context_metadata_to_counter_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_bind_context_metadata_to_counter_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* _ITTNOTIFY_H_ */ + +#ifdef INTEL_ITTNOTIFY_API_PRIVATE + +#ifndef _ITTNOTIFY_PRIVATE_ +#define _ITTNOTIFY_PRIVATE_ + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** + * @ingroup clockdomain + * @brief Begin an overlapped task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] taskid The identifier for this task instance, *cannot* be __itt_null. + * @param[in] parentid The parent of this task, or __itt_null. + * @param[in] name The name of this task. + */ +void ITTAPI __itt_task_begin_overlapped_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, __itt_string_handle* name); + +/** + * @ingroup clockdomain + * @brief End an overlapped task instance. + * @param[in] domain The domain for this task + * @param[in] clock_domain The clock domain controlling the execution of this call. + * @param[in] timestamp The user defined timestamp. + * @param[in] taskid Explicit ID of finished task + */ +void ITTAPI __itt_task_end_overlapped_ex(const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, task_begin_overlapped_ex, (const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid, __itt_id parentid, __itt_string_handle* name)) +ITT_STUBV(ITTAPI, void, task_end_overlapped_ex, (const __itt_domain* domain, __itt_clock_domain* clock_domain, unsigned long long timestamp, __itt_id taskid)) +#define __itt_task_begin_overlapped_ex(d,x,y,z,a,b) ITTNOTIFY_VOID_D5(task_begin_overlapped_ex,d,x,y,z,a,b) +#define __itt_task_begin_overlapped_ex_ptr ITTNOTIFY_NAME(task_begin_overlapped_ex) +#define __itt_task_end_overlapped_ex(d,x,y,z) ITTNOTIFY_VOID_D3(task_end_overlapped_ex,d,x,y,z) +#define __itt_task_end_overlapped_ex_ptr ITTNOTIFY_NAME(task_end_overlapped_ex) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_task_begin_overlapped_ex(domain,clock_domain,timestamp,taskid,parentid,name) +#define __itt_task_begin_overlapped_ex_ptr 0 +#define __itt_task_end_overlapped_ex(domain,clock_domain,timestamp,taskid) +#define __itt_task_end_overlapped_ex_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_task_begin_overlapped_ex_ptr 0 +#define __itt_task_end_overlapped_ptr 0 +#define __itt_task_end_overlapped_ex_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @defgroup makrs_internal Marks + * @ingroup internal + * Marks group + * @warning Internal API: + * - It is not shipped to outside of Intel + * - It is delivered to internal Intel teams using e-mail or SVN access only + * @{ + */ +/** @brief user mark type */ +typedef int __itt_mark_type; + +/** + * @brief Creates a user mark type with the specified name using char or Unicode string. + * @param[in] name - name of mark to create + * @return Returns a handle to the mark type + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +__itt_mark_type ITTAPI __itt_mark_createA(const char *name); +__itt_mark_type ITTAPI __itt_mark_createW(const wchar_t *name); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_mark_create __itt_mark_createW +# define __itt_mark_create_ptr __itt_mark_createW_ptr +#else /* UNICODE */ +# define __itt_mark_create __itt_mark_createA +# define __itt_mark_create_ptr __itt_mark_createA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +__itt_mark_type ITTAPI __itt_mark_create(const char *name); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, __itt_mark_type, mark_createA, (const char *name)) +ITT_STUB(ITTAPI, __itt_mark_type, mark_createW, (const wchar_t *name)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, __itt_mark_type, mark_create, (const char *name)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_createA ITTNOTIFY_DATA(mark_createA) +#define __itt_mark_createA_ptr ITTNOTIFY_NAME(mark_createA) +#define __itt_mark_createW ITTNOTIFY_DATA(mark_createW) +#define __itt_mark_createW_ptr ITTNOTIFY_NAME(mark_createW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_create ITTNOTIFY_DATA(mark_create) +#define __itt_mark_create_ptr ITTNOTIFY_NAME(mark_create) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_createA(name) (__itt_mark_type)0 +#define __itt_mark_createA_ptr 0 +#define __itt_mark_createW(name) (__itt_mark_type)0 +#define __itt_mark_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_create(name) (__itt_mark_type)0 +#define __itt_mark_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_createA_ptr 0 +#define __itt_mark_createW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_create_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Creates a "discrete" user mark type of the specified type and an optional parameter using char or Unicode string. + * + * - The mark of "discrete" type is placed to collection results in case of success. It appears in overtime view(s) as a special tick sign. + * - The call is "synchronous" - function returns after mark is actually added to results. + * - This function is useful, for example, to mark different phases of application + * (beginning of the next mark automatically meand end of current region). + * - Can be used together with "continuous" marks (see below) at the same collection session + * @param[in] mt - mark, created by __itt_mark_create(const char* name) function + * @param[in] parameter - string parameter of mark + * @return Returns zero value in case of success, non-zero value otherwise. + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +int ITTAPI __itt_markA(__itt_mark_type mt, const char *parameter); +int ITTAPI __itt_markW(__itt_mark_type mt, const wchar_t *parameter); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_mark __itt_markW +# define __itt_mark_ptr __itt_markW_ptr +#else /* UNICODE */ +# define __itt_mark __itt_markA +# define __itt_mark_ptr __itt_markA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +int ITTAPI __itt_mark(__itt_mark_type mt, const char *parameter); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, int, markA, (__itt_mark_type mt, const char *parameter)) +ITT_STUB(ITTAPI, int, markW, (__itt_mark_type mt, const wchar_t *parameter)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, int, mark, (__itt_mark_type mt, const char *parameter)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_markA ITTNOTIFY_DATA(markA) +#define __itt_markA_ptr ITTNOTIFY_NAME(markA) +#define __itt_markW ITTNOTIFY_DATA(markW) +#define __itt_markW_ptr ITTNOTIFY_NAME(markW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark ITTNOTIFY_DATA(mark) +#define __itt_mark_ptr ITTNOTIFY_NAME(mark) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_markA(mt, parameter) (int)0 +#define __itt_markA_ptr 0 +#define __itt_markW(mt, parameter) (int)0 +#define __itt_markW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark(mt, parameter) (int)0 +#define __itt_mark_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_markA_ptr 0 +#define __itt_markW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Use this if necessary to create a "discrete" user event type (mark) for process + * rather then for one thread + * @see int __itt_mark(__itt_mark_type mt, const char* parameter); + */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +int ITTAPI __itt_mark_globalA(__itt_mark_type mt, const char *parameter); +int ITTAPI __itt_mark_globalW(__itt_mark_type mt, const wchar_t *parameter); +#if defined(UNICODE) || defined(_UNICODE) +# define __itt_mark_global __itt_mark_globalW +# define __itt_mark_global_ptr __itt_mark_globalW_ptr +#else /* UNICODE */ +# define __itt_mark_global __itt_mark_globalA +# define __itt_mark_global_ptr __itt_mark_globalA_ptr +#endif /* UNICODE */ +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +int ITTAPI __itt_mark_global(__itt_mark_type mt, const char *parameter); +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#if ITT_PLATFORM==ITT_PLATFORM_WIN +ITT_STUB(ITTAPI, int, mark_globalA, (__itt_mark_type mt, const char *parameter)) +ITT_STUB(ITTAPI, int, mark_globalW, (__itt_mark_type mt, const wchar_t *parameter)) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +ITT_STUB(ITTAPI, int, mark_global, (__itt_mark_type mt, const char *parameter)) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_globalA ITTNOTIFY_DATA(mark_globalA) +#define __itt_mark_globalA_ptr ITTNOTIFY_NAME(mark_globalA) +#define __itt_mark_globalW ITTNOTIFY_DATA(mark_globalW) +#define __itt_mark_globalW_ptr ITTNOTIFY_NAME(mark_globalW) +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_global ITTNOTIFY_DATA(mark_global) +#define __itt_mark_global_ptr ITTNOTIFY_NAME(mark_global) +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#else /* INTEL_NO_ITTNOTIFY_API */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_globalA(mt, parameter) (int)0 +#define __itt_mark_globalA_ptr 0 +#define __itt_mark_globalW(mt, parameter) (int)0 +#define __itt_mark_globalW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_global(mt, parameter) (int)0 +#define __itt_mark_global_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#if ITT_PLATFORM==ITT_PLATFORM_WIN +#define __itt_mark_globalA_ptr 0 +#define __itt_mark_globalW_ptr 0 +#else /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#define __itt_mark_global_ptr 0 +#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */ +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Creates an "end" point for "continuous" mark with specified name. + * + * - Returns zero value in case of success, non-zero value otherwise. + * Also returns non-zero value when preceding "begin" point for the + * mark with the same name failed to be created or not created. + * - The mark of "continuous" type is placed to collection results in + * case of success. It appears in overtime view(s) as a special tick + * sign (different from "discrete" mark) together with line from + * corresponding "begin" mark to "end" mark. + * @note Continuous marks can overlap and be nested inside each other. + * Discrete mark can be nested inside marked region + * @param[in] mt - mark, created by __itt_mark_create(const char* name) function + * @return Returns zero value in case of success, non-zero value otherwise. + */ +int ITTAPI __itt_mark_off(__itt_mark_type mt); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, int, mark_off, (__itt_mark_type mt)) +#define __itt_mark_off ITTNOTIFY_DATA(mark_off) +#define __itt_mark_off_ptr ITTNOTIFY_NAME(mark_off) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_mark_off(mt) (int)0 +#define __itt_mark_off_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_mark_off_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Use this if necessary to create an "end" point for mark of process + * @see int __itt_mark_off(__itt_mark_type mt); + */ +int ITTAPI __itt_mark_global_off(__itt_mark_type mt); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, int, mark_global_off, (__itt_mark_type mt)) +#define __itt_mark_global_off ITTNOTIFY_DATA(mark_global_off) +#define __itt_mark_global_off_ptr ITTNOTIFY_NAME(mark_global_off) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_mark_global_off(mt) (int)0 +#define __itt_mark_global_off_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_mark_global_off_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ +/** @} marks group */ + +/** + * @defgroup counters_internal Counters + * @ingroup internal + * Counters group + * @{ + */ + + +/** + * @defgroup stitch Stack Stitching + * @ingroup internal + * Stack Stitching group + * @{ + */ +/** + * @brief opaque structure for counter identification + */ +typedef struct ___itt_caller *__itt_caller; + +/** + * @brief Create the stitch point e.g. a point in call stack where other stacks should be stitched to. + * The function returns a unique identifier which is used to match the cut points with corresponding stitch points. + */ +__itt_caller ITTAPI __itt_stack_caller_create(void); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUB(ITTAPI, __itt_caller, stack_caller_create, (void)) +#define __itt_stack_caller_create ITTNOTIFY_DATA(stack_caller_create) +#define __itt_stack_caller_create_ptr ITTNOTIFY_NAME(stack_caller_create) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_stack_caller_create() (__itt_caller)0 +#define __itt_stack_caller_create_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_stack_caller_create_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Destroy the information about stitch point identified by the pointer previously returned by __itt_stack_caller_create() + */ +void ITTAPI __itt_stack_caller_destroy(__itt_caller id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, stack_caller_destroy, (__itt_caller id)) +#define __itt_stack_caller_destroy ITTNOTIFY_VOID(stack_caller_destroy) +#define __itt_stack_caller_destroy_ptr ITTNOTIFY_NAME(stack_caller_destroy) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_stack_caller_destroy(id) +#define __itt_stack_caller_destroy_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_stack_caller_destroy_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief Sets the cut point. Stack from each event which occurs after this call will be cut + * at the same stack level the function was called and stitched to the corresponding stitch point. + */ +void ITTAPI __itt_stack_callee_enter(__itt_caller id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, stack_callee_enter, (__itt_caller id)) +#define __itt_stack_callee_enter ITTNOTIFY_VOID(stack_callee_enter) +#define __itt_stack_callee_enter_ptr ITTNOTIFY_NAME(stack_callee_enter) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_stack_callee_enter(id) +#define __itt_stack_callee_enter_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_stack_callee_enter_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** + * @brief This function eliminates the cut point which was set by latest __itt_stack_callee_enter(). + */ +void ITTAPI __itt_stack_callee_leave(__itt_caller id); + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +ITT_STUBV(ITTAPI, void, stack_callee_leave, (__itt_caller id)) +#define __itt_stack_callee_leave ITTNOTIFY_VOID(stack_callee_leave) +#define __itt_stack_callee_leave_ptr ITTNOTIFY_NAME(stack_callee_leave) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_stack_callee_leave(id) +#define __itt_stack_callee_leave_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_stack_callee_leave_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +/** @} stitch group */ + +/* ***************************************************************************************************************************** */ + +#include + +/** @cond exclude_from_documentation */ +typedef enum __itt_error_code +{ + __itt_error_success = 0, /*!< no error */ + __itt_error_no_module = 1, /*!< module can't be loaded */ + /* %1$s -- library name; win: %2$d -- system error code; unx: %2$s -- system error message. */ + __itt_error_no_symbol = 2, /*!< symbol not found */ + /* %1$s -- library name, %2$s -- symbol name. */ + __itt_error_unknown_group = 3, /*!< unknown group specified */ + /* %1$s -- env var name, %2$s -- group name. */ + __itt_error_cant_read_env = 4, /*!< GetEnvironmentVariable() failed */ + /* %1$s -- env var name, %2$d -- system error. */ + __itt_error_env_too_long = 5, /*!< variable value too long */ + /* %1$s -- env var name, %2$d -- actual length of the var, %3$d -- max allowed length. */ + __itt_error_system = 6 /*!< pthread_mutexattr_init or pthread_mutex_init failed */ + /* %1$s -- function name, %2$d -- errno. */ +} __itt_error_code; + +typedef void (__itt_error_handler_t)(__itt_error_code code, va_list); +__itt_error_handler_t* __itt_set_error_handler(__itt_error_handler_t*); + +const char* ITTAPI __itt_api_version(void); +/** @endcond */ + +/** @cond exclude_from_documentation */ +#ifndef INTEL_NO_MACRO_BODY +#ifndef INTEL_NO_ITTNOTIFY_API +#define __itt_error_handler ITT_JOIN(INTEL_ITTNOTIFY_PREFIX, error_handler) +void __itt_error_handler(__itt_error_code code, va_list args); +extern const int ITTNOTIFY_NAME(err); +#define __itt_err ITTNOTIFY_NAME(err) +ITT_STUB(ITTAPI, const char*, api_version, (void)) +#define __itt_api_version ITTNOTIFY_DATA(api_version) +#define __itt_api_version_ptr ITTNOTIFY_NAME(api_version) +#else /* INTEL_NO_ITTNOTIFY_API */ +#define __itt_api_version() (const char*)0 +#define __itt_api_version_ptr 0 +#endif /* INTEL_NO_ITTNOTIFY_API */ +#else /* INTEL_NO_MACRO_BODY */ +#define __itt_api_version_ptr 0 +#endif /* INTEL_NO_MACRO_BODY */ +/** @endcond */ + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* _ITTNOTIFY_PRIVATE_ */ + +#endif /* INTEL_ITTNOTIFY_API_PRIVATE */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/jitprofiling.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/jitprofiling.h new file mode 100644 index 0000000000000000000000000000000000000000..a2c68e032008c6a43dff11ae2ce2027b9bdbb4fb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/jitprofiling.h @@ -0,0 +1,647 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/* + Copyright (C) 2005-2019 Intel Corporation + + SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause +*/ + +#ifndef __JITPROFILING_H__ +#define __JITPROFILING_H__ + +/** + * @brief JIT Profiling APIs + * + * The JIT Profiling API is used to report information about just-in-time + * generated code that can be used by performance tools. The user inserts + * calls in the code generator to report information before JIT-compiled + * code goes to execution. This information is collected at runtime and used + * by tools like Intel(R) VTune(TM) Profiler to display performance metrics + * associated with JIT-compiled code. + * + * These APIs can be used to\n + * - **Profile trace-based and method-based JIT-compiled + * code**. Some examples of environments that you can profile with these APIs: + * dynamic JIT compilation of JavaScript code traces, JIT execution in OpenCL(TM) + * software technology, Java/.NET managed execution environments, and custom + * ISV JIT engines. + * @code + * #include + * + * if (iJIT_IsProfilingActive != iJIT_SAMPLING_ON) { + * return; + * } + * + * iJIT_Method_Load jmethod = {0}; + * jmethod.method_id = iJIT_GetNewMethodID(); + * jmethod.method_name = "method_name"; + * jmethod.class_file_name = "class_name"; + * jmethod.source_file_name = "source_file_name"; + * jmethod.method_load_address = code_addr; + * jmethod.method_size = code_size; + * + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&jmethod); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_SHUTDOWN, NULL); + * @endcode + * + * * Expected behavior: + * * If any iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED event overwrites an + * already reported method, then such a method becomes invalid and its + * memory region is treated as unloaded. VTune Profiler displays the metrics + * collected by the method until it is overwritten. + * * If supplied line number information contains multiple source lines for + * the same assembly instruction (code location), then VTune Profiler picks up + * the first line number. + * * Dynamically generated code can be associated with a module name. + * Use the iJIT_Method_Load_V2 structure.\n + * Clarification of some cases: + * * If you register a function with the same method ID multiple times, + * specifying different module names, then the VTune Profiler picks up + * the module name registered first. If you want to distinguish the same + * function between different JIT engines, supply different method IDs for + * each function. Other symbolic information (for example, source file) + * can be identical. + * + * - **Analyze split functions** (multiple joint or disjoint code regions + * belonging to the same function) **including re-JIT** + * with potential overlapping of code regions in time, which is common in + * resource-limited environments. + * @code + * #include + * + * unsigned int method_id = iJIT_GetNewMethodID(); + * + * iJIT_Method_Load a = {0}; + * a.method_id = method_id; + * a.method_load_address = 0x100; + * a.method_size = 0x20; + * + * iJIT_Method_Load b = {0}; + * b.method_id = method_id; + * b.method_load_address = 0x200; + * b.method_size = 0x30; + * + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&a); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&b); + * @endcode + * + * * Expected behaviour: + * * If a iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED event overwrites an + * already reported method, then such a method becomes invalid and + * its memory region is treated as unloaded. + * * All code regions reported with the same method ID are considered as + * belonging to the same method. Symbolic information (method name, + * source file name) will be taken from the first notification, and all + * subsequent notifications with the same method ID will be processed + * only for line number table information. So, the VTune Profiler will map + * samples to a source line using the line number table from the current + * notification while taking the source file name from the very first one.\n + * Clarification of some cases:\n + * * If you register a second code region with a different source file + * name and the same method ID, then this information will be saved and + * will not be considered as an extension of the first code region, but + * VTune Profiler will use the source file of the first code region and map + * performance metrics incorrectly. + * * If you register a second code region with the same source file as + * for the first region and the same method ID, then the source file will be + * discarded but VTune Profiler will map metrics to the source file correctly. + * * If you register a second code region with a null source file and + * the same method ID, then provided line number info will be associated + * with the source file of the first code region. + * + * - **Explore inline functions** including multi-level hierarchy of + * nested inline methods which shows how performance metrics are distributed through them. + * @code + * #include + * + * // method_id parent_id + * // [-- c --] 3000 2000 + * // [---- d -----] 2001 1000 + * // [---- b ----] 2000 1000 + * // [------------ a ----------------] 1000 n/a + * + * iJIT_Method_Load a = {0}; + * a.method_id = 1000; + * + * iJIT_Method_Inline_Load b = {0}; + * b.method_id = 2000; + * b.parent_method_id = 1000; + * + * iJIT_Method_Inline_Load c = {0}; + * c.method_id = 3000; + * c.parent_method_id = 2000; + * + * iJIT_Method_Inline_Load d = {0}; + * d.method_id = 2001; + * d.parent_method_id = 1000; + * + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&a); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, (void*)&b); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, (void*)&c); + * iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, (void*)&d); + * @endcode + * + * * Requirements: + * * Each inline (iJIT_Method_Inline_Load) method should be associated + * with two method IDs: one for itself; one for its immediate parent. + * * Address regions of inline methods of the same parent method cannot + * overlap each other. + * * Execution of the parent method must not be started until it and all + * its inline methods are reported. + * * Expected behaviour: + * * In case of nested inline methods an order of + * iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED events is not important. + * * If any event overwrites either inline method or top parent method, + * then the parent, including inline methods, becomes invalid and its memory + * region is treated as unloaded. + * + * **Life time of allocated data**\n + * The client sends an event notification to the agent with event-specific + * data, which is a structure. The pointers in the structure refer to memory + * allocated by the client, which responsible for releasing it. The pointers are + * used by the iJIT_NotifyEvent method to copy client's data in a trace file, + * and they are not used after the iJIT_NotifyEvent method returns. + */ + +/** + * @defgroup jitapi JIT Profiling + * @ingroup internal + * @{ + */ + +/** + * @brief Enumerator for the types of notifications + */ +typedef enum iJIT_jvm_event +{ + iJVM_EVENT_TYPE_SHUTDOWN = 2, /**<\brief Send this to shutdown the agent. + * Use NULL for event data. */ + + iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED = 13, /**<\brief Send when dynamic code is + * JIT compiled and loaded into + * memory by the JIT engine, but + * before the code is executed. + * Use iJIT_Method_Load as event + * data. */ +/** @cond exclude_from_documentation */ + iJVM_EVENT_TYPE_METHOD_UNLOAD_START, /**<\brief Send when compiled dynamic + * code is being unloaded from memory. + * Use iJIT_Method_Load as event data.*/ +/** @endcond */ + + iJVM_EVENT_TYPE_METHOD_UPDATE, /**<\brief Send to provide new content for + * a previously reported dynamic code. + * The previous content will be invalidated + * starting from the time of the notification. + * Use iJIT_Method_Load as event data but + * required fields are following: + * - method_id identify the code to update. + * - method_load_address specify start address + * within identified code range + * where update should be started. + * - method_size specify length of updated code + * range. */ + + + iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED, /**<\brief Send when an inline dynamic + * code is JIT compiled and loaded + * into memory by the JIT engine, + * but before the parent code region + * starts executing. + * Use iJIT_Method_Inline_Load as event data.*/ + +/** @cond exclude_from_documentation */ + iJVM_EVENT_TYPE_METHOD_UPDATE_V2, +/** @endcond */ + + iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V2 = 21, /**<\brief Send when a dynamic code is + * JIT compiled and loaded into + * memory by the JIT engine, but + * before the code is executed. + * Use iJIT_Method_Load_V2 as event data. */ + + iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V3 /**<\brief Send when a dynamic code is + * JIT compiled and loaded into + * memory by the JIT engine, but + * before the code is executed. + * Use iJIT_Method_Load_V3 as event data. */ +} iJIT_JVM_EVENT; + +/** + * @brief Enumerator for the agent's mode + */ +typedef enum _iJIT_IsProfilingActiveFlags +{ + iJIT_NOTHING_RUNNING = 0x0000, /**<\brief The agent is not running; + * iJIT_NotifyEvent calls will + * not be processed. */ + iJIT_SAMPLING_ON = 0x0001, /**<\brief The agent is running and + * ready to process notifications. */ +} iJIT_IsProfilingActiveFlags; + +/** + * @brief Description of a single entry in the line number information of a code region. + * @details A table of line number entries gives information about how the reported code region + * is mapped to source file. + * Intel(R) VTune(TM) Profiler uses line number information to attribute + * the samples (virtual address) to a line number. \n + * It is acceptable to report different code addresses for the same source line: + * @code + * Offset LineNumber + * 1 2 + * 12 4 + * 15 2 + * 18 1 + * 21 30 + * + * VTune Profiler constructs the following table using the client data + * + * Code subrange Line number + * 0-1 2 + * 1-12 4 + * 12-15 2 + * 15-18 1 + * 18-21 30 + * @endcode + */ +typedef struct _LineNumberInfo +{ + unsigned int Offset; /**<\brief Offset from the begining of the code region. */ + unsigned int LineNumber; /**<\brief Matching source line number offset (from beginning of source file). */ + +} *pLineNumberInfo, LineNumberInfo; + +/** + * @brief Enumerator for the code architecture. + */ +typedef enum _iJIT_CodeArchitecture +{ + iJIT_CA_NATIVE = 0, /**<\brief Native to the process architecture that is calling it. */ + + iJIT_CA_32, /**<\brief 32-bit machine code. */ + + iJIT_CA_64 /**<\brief 64-bit machine code. */ + +} iJIT_CodeArchitecture; + +#pragma pack(push, 8) + +/** + * @brief Description of a JIT-compiled method + * @details When you use the iJIT_Method_Load structure to describe + * the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED + * as an event type to report it. + */ +typedef struct _iJIT_Method_Load +{ + unsigned int method_id; /**<\brief Unique method ID. Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or else manage ID uniqueness + * and correct range by yourself.\n + * You must use the same method ID for all code + * regions of the same method, otherwise different + * method IDs specify different methods. */ + + char* method_name; /**<\brief The name of the method. It can be optionally + * prefixed with its class name and appended with + * its complete signature. Can't be NULL. */ + + void* method_load_address; /**<\brief The start virtual address of the method code + * region. If NULL, data provided with + * event are not accepted. */ + + unsigned int method_size; /**<\brief The code size of the method in memory. + * If 0, then data provided with the event are not + * accepted. */ + + unsigned int line_number_size; /**<\brief The number of entries in the line number + * table.0 if none. */ + + pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info + * array. Can be NULL if + * line_number_size is 0. See + * LineNumberInfo Structure for a + * description of a single entry in + * the line number info array */ + + unsigned int class_id; /**<\brief This field is obsolete. */ + + char* class_file_name; /**<\brief Class name. Can be NULL.*/ + + char* source_file_name; /**<\brief Source file name. Can be NULL.*/ + +} *piJIT_Method_Load, iJIT_Method_Load; + +/** + * @brief Description of a JIT-compiled method + * @details When you use the iJIT_Method_Load_V2 structure to describe + * the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V2 + * as an event type to report it. + */ +typedef struct _iJIT_Method_Load_V2 +{ + unsigned int method_id; /**<\brief Unique method ID. Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or else manage ID uniqueness + * and correct range by yourself.\n + * You must use the same method ID for all code + * regions of the same method, otherwise different + * method IDs specify different methods. */ + + char* method_name; /**<\brief The name of the method. It can be optionally + * prefixed with its class name and appended with + * its complete signature. Can't be NULL. */ + + void* method_load_address; /**<\brief The start virtual address of the method code + * region. If NULL, then data provided with the + * event are not accepted. */ + + unsigned int method_size; /**<\brief The code size of the method in memory. + * If 0, then data provided with the event are not + * accepted. */ + + unsigned int line_number_size; /**<\brief The number of entries in the line number + * table. 0 if none. */ + + pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info + * array. Can be NULL if + * line_number_size is 0. See + * LineNumberInfo Structure for a + * description of a single entry in + * the line number info array. */ + + char* class_file_name; /**<\brief Class name. Can be NULL. */ + + char* source_file_name; /**<\brief Source file name. Can be NULL. */ + + char* module_name; /**<\brief Module name. Can be NULL. + The module name can be useful for distinguishing among + different JIT engines. VTune Profiler will display + reported methods grouped by specific module. */ + +} *piJIT_Method_Load_V2, iJIT_Method_Load_V2; + +/** + * @brief Description of a JIT-compiled method + * @details The iJIT_Method_Load_V3 structure is the same as iJIT_Method_Load_V2 + * with a newly introduced 'arch' field that specifies architecture of the code region. + * When you use the iJIT_Method_Load_V3 structure to describe + * the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED_V3 + * as an event type to report it. + */ +typedef struct _iJIT_Method_Load_V3 +{ + unsigned int method_id; /**<\brief Unique method ID. Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or manage ID uniqueness + * and correct range by yourself.\n + * You must use the same method ID for all code + * regions of the same method, otherwise they are + * treated as regions of different methods. */ + + char* method_name; /**<\brief The name of the method. It can be optionally + * prefixed with its class name and appended with + * its complete signature. Cannot be NULL. */ + + void* method_load_address; /**<\brief The start virtual address of the method code + * region. If NULL, then data provided with the + * event are not accepted. */ + + unsigned int method_size; /**<\brief The code size of the method in memory. + * If 0, then data provided with the event are not + * accepted. */ + + unsigned int line_number_size; /**<\brief The number of entries in the line number + * table. 0 if none. */ + + pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info + * array. Can be NULL if + * line_number_size is 0. See + * LineNumberInfo Structure for a + * description of a single entry in + * the line number info array. */ + + char* class_file_name; /**<\brief Class name. Can be NULL. */ + + char* source_file_name; /**<\brief Source file name. Can be NULL. */ + + char* module_name; /**<\brief Module name. Can be NULL. + * The module name can be useful for distinguishing among + * different JIT engines. VTune Profiler will display + * reported methods grouped by specific module. */ + + iJIT_CodeArchitecture module_arch; /**<\brief Architecture of the method's code region. + * By default, it is the same as the process + * architecture that is calling it. + * For example, you can use it if your 32-bit JIT + * engine generates 64-bit code. + * + * If JIT engine reports both 32-bit and 64-bit types + * of methods then VTune Profiler splits the methods + * with the same module name but with different + * architectures in two different modules. VTune Profiler + * modifies the original name provided with a 64-bit method + * version by ending it with '(64)' */ + +} *piJIT_Method_Load_V3, iJIT_Method_Load_V3; + +/** + * @brief Description of an inline JIT-compiled method + * @details When you use the_iJIT_Method_Inline_Load structure to describe + * the JIT compiled method, use iJVM_EVENT_TYPE_METHOD_INLINE_LOAD_FINISHED + * as an event type to report it. + */ +typedef struct _iJIT_Method_Inline_Load +{ + unsigned int method_id; /**<\brief Unique method ID. Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or else manage ID uniqueness + * and correct range by yourself. */ + + unsigned int parent_method_id; /**<\brief Unique immediate parent's method ID. + * Cannot be 0. + * You must either use the API function + * iJIT_GetNewMethodID to get a valid and unique + * method ID, or else manage ID uniqueness + * and correct range by yourself. */ + + char* method_name; /**<\brief The name of the method. It can be optionally + * prefixed with its class name and appended with + * its complete signature. Can't be NULL. */ + + void* method_load_address; /** <\brief The virtual address on which the method + * is inlined. If NULL, then data provided with + * the event are not accepted. */ + + unsigned int method_size; /**<\brief The code size of the method in memory. + * If 0, then data provided with the event are not + * accepted. */ + + unsigned int line_number_size; /**<\brief The number of entries in the line number + * table. 0 if none. */ + + pLineNumberInfo line_number_table; /**<\brief Pointer to the line numbers info + * array. Can be NULL if + * line_number_size is 0. See + * LineNumberInfo Structure for a + * description of a single entry in + * the line number info array */ + + char* class_file_name; /**<\brief Class name. Can be NULL.*/ + + char* source_file_name; /**<\brief Source file name. Can be NULL.*/ + +} *piJIT_Method_Inline_Load, iJIT_Method_Inline_Load; + +/** @cond exclude_from_documentation */ +/** + * @brief Description of a segment type + * @details Use the segment type to specify a type of data supplied + * with the iJVM_EVENT_TYPE_METHOD_UPDATE_V2 event to be applied to + * a certain code trace. + */ +typedef enum _iJIT_SegmentType +{ + iJIT_CT_UNKNOWN = 0, + + iJIT_CT_CODE, /**<\brief Executable code. */ + + iJIT_CT_DATA, /**<\brief Data (not executable code). + * VTune Profiler uses the format string + * (see iJIT_Method_Update) to represent + * this data in the VTune Profiler GUI */ + + iJIT_CT_KEEP, /**<\brief Use the previous markup for the trace. + * Can be used for the following + * iJVM_EVENT_TYPE_METHOD_UPDATE_V2 events, + * if the type of the previously reported segment + * type is the same. */ + iJIT_CT_EOF +} iJIT_SegmentType; + +/** + * @brief Description of a dynamic update of the content within JIT-compiled method + * @details The JIT engine may generate the methods that are updated at runtime + * partially by mixed (data + executable code) content. When you use the iJIT_Method_Update + * structure to describe the update of the content within a JIT-compiled method, + * use iJVM_EVENT_TYPE_METHOD_UPDATE_V2 as an event type to report it. + * + * On the first Update event, VTune Profiler copies the original code range reported by + * the iJVM_EVENT_TYPE_METHOD_LOAD event, then modifies it with the supplied bytes and + * adds the modified range to the original method. For next update events, VTune Profiler + * does the same but it uses the latest modified version of a code region for update. + * Eventually, VTune Profiler GUI displays multiple code ranges for the method reported by + * the iJVM_EVENT_TYPE_METHOD_LOAD event. + * Notes: + * - Multiple update events with different types for the same trace are allowed + * but they must be reported for the same code ranges. + * Example, + * @code + * [-- data---] Allowed + * [-- code --] Allowed + * [code] Ignored + * [-- data---] Allowed + * [-- code --] Allowed + * [------------ trace ---------] + * @endcode + * - The types of previously reported events can be changed but they must be reported + * for the same code ranges. + * Example, + * @code + * [-- data---] Allowed + * [-- code --] Allowed + * [-- data---] Allowed + * [-- code --] Allowed + * [------------ trace ---------] + * @endcode + */ + +typedef struct _iJIT_Method_Update +{ + void* load_address; /**<\brief Start address of the update within a method */ + + unsigned int size; /**<\brief The update size */ + + iJIT_SegmentType type; /**<\brief Type of the update */ + + const char* data_format; /**<\brief C string that contains a format string + * that follows the same specifications as format in printf. + * The format string is used for iJIT_CT_CODE only + * and cannot be NULL. + * Format can be changed on the fly. */ +} *piJIT_Method_Update, iJIT_Method_Update; + +/** @endcond */ + +#pragma pack(pop) + +/** @cond exclude_from_documentation */ +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#ifndef JITAPI_CDECL +# if defined WIN32 || defined _WIN32 +# define JITAPI_CDECL __cdecl +# else /* defined WIN32 || defined _WIN32 */ +# if defined _M_IX86 || defined __i386__ +# define JITAPI_CDECL __attribute__ ((cdecl)) +# else /* _M_IX86 || __i386__ */ +# define JITAPI_CDECL /* actual only on x86_64 platform */ +# endif /* _M_IX86 || __i386__ */ +# endif /* defined WIN32 || defined _WIN32 */ +#endif /* JITAPI_CDECL */ + +#define JITAPI JITAPI_CDECL +/** @endcond */ + +/** + * @brief Generates a new unique method ID. + * + * You must use this API to obtain unique and valid method IDs for methods or + * traces reported to the agent if you don't have your own mechanism to generate + * unique method IDs. + * + * @return a new unique method ID. When out of unique method IDs, this API + * returns 0, which is not an accepted value. + */ +unsigned int JITAPI iJIT_GetNewMethodID(void); + +/** + * @brief Returns the current mode of the agent. + * + * @return iJIT_SAMPLING_ON, indicating that agent is running, or + * iJIT_NOTHING_RUNNING if no agent is running. + */ +iJIT_IsProfilingActiveFlags JITAPI iJIT_IsProfilingActive(void); + +/** + * @brief Reports infomation about JIT-compiled code to the agent. + * + * The reported information is used to attribute samples obtained from any + * Intel(R) VTune(TM) Profiler collector. This API needs to be called + * after JIT compilation and before the first entry into the JIT-compiled + * code. + * + * @param[in] event_type - type of the data sent to the agent + * @param[in] EventSpecificData - pointer to event-specific data + * + * @returns 1 on success, otherwise 0. + */ +int JITAPI iJIT_NotifyEvent(iJIT_JVM_EVENT event_type, void *EventSpecificData); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ +/** @endcond */ + +/** @} jitapi group */ + +#endif /* __JITPROFILING_H__ */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/libittnotify.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/libittnotify.h new file mode 100644 index 0000000000000000000000000000000000000000..cb5190ff7d3c2bfe1685dafaddc7cd744e971fa8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/libittnotify.h @@ -0,0 +1,24 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +/* + Copyright (C) 2005-2019 Intel Corporation + + SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause +*/ + +#ifndef _LIBITTNOTIFY_H_ +#define _LIBITTNOTIFY_H_ + +#ifndef __ITT_INTERNAL_INCLUDE +# if defined WIN32 || defined _WIN32 +# pragma message("WARNING!!! Include file libittnotify.h is deprecated and should not be included anymore") +# else /* WIN32 */ +# warning "Include file libittnotify.h is deprecated and should not be included anymore" +# endif /* WIN32 */ +#endif /* __ITT_INTERNAL_INCLUDE */ +#include "legacy/ittnotify.h" + +#endif /* _LIBITTNOTIFY_H_ */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/libshm.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/libshm.h new file mode 100644 index 0000000000000000000000000000000000000000..58d368dcf7aea4472e91efb60983e58e70c61654 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/libshm.h @@ -0,0 +1,51 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#pragma once + +#include + +#ifdef __cplusplus + +void libshm_init(const char* manager_exec_path); + +// Superclass to run a constructor before at::RefcountedMapAllocator +class THManagedMapAllocatorInit { + protected: + THManagedMapAllocatorInit(const char* manager_handle, const char* filename); + std::string manager_handle_; +}; + +// Like a at::RefcountedMapAllocator, but it also makes use of an external +// shared memory manager process to ensure that shared memory regions actually +// get freed in the end (even if processes lose the memory). +class THManagedMapAllocator : private THManagedMapAllocatorInit, + public at::RefcountedMapAllocator { + public: + THManagedMapAllocator( + const char* manager_handle, + const char* filename, + int flags, + size_t size); + + void close() override; + + ~THManagedMapAllocator() override { + close(); + } + + static at::DataPtr makeDataPtr( + const char* manager_handle, + const char* filename, + int flags, + size_t size); + static THManagedMapAllocator* fromDataPtr(const at::DataPtr& /*dptr*/); + + const char* manager_handle() const { + return manager_handle_.c_str(); + } +}; + +#endif + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/nnpack.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/nnpack.h new file mode 100644 index 0000000000000000000000000000000000000000..62e785e8c9e9f4a28c50f4f5260bc89dbb2d5c83 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/nnpack.h @@ -0,0 +1,664 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#pragma once + +#include +#include +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Status code for any NNPACK function call. + */ +enum nnp_status { + /** The call succeeded, and all output arguments now contain valid data. */ + nnp_status_success = 0, + /** NNPACK function was called with batch_size == 0. */ + nnp_status_invalid_batch_size = 2, + /** NNPACK function was called with channels == 0. */ + nnp_status_invalid_channels = 3, + /** NNPACK function was called with input_channels == 0. */ + nnp_status_invalid_input_channels = 4, + /** NNPACK function was called with output_channels == 0. */ + nnp_status_invalid_output_channels = 5, + /** NNPACK function was called with input_size.height == 0 or input_size.width == 0 */ + nnp_status_invalid_input_size = 10, + /** NNPACK function was called with input_stride.height == 0 or input_stride.width == 0 */ + nnp_status_invalid_input_stride = 11, + /** NNPACK function was called with input_padding not less than respective kernel (or pooling) size, i.e.: + * + * - input_padding.left >= kernel_size.width (>= pooling_size.width) + * - input_padding.right >= kernel_size.width (>= pooling_size.width) + * - input_padding.top >= kernel_size.height (>= pooling_size.height) + * - input_padding.bottom >= kernel_size.height (>= pooling_size.height) + */ + nnp_status_invalid_input_padding = 12, + /** NNPACK function was called with kernel_size.height == 0 or kernel_size.width == 0 */ + nnp_status_invalid_kernel_size = 13, + /** NNPACK function was called with pooling_size.height == 0 or pooling_size.width == 0 */ + nnp_status_invalid_pooling_size = 14, + /** NNPACK function was called with pooling_stride.height == 0 or pooling_stride.width == 0 */ + nnp_status_invalid_pooling_stride = 15, + /** NNPACK function was called with convolution algorithm not in nnp_convolution_algorithm enumeration */ + nnp_status_invalid_algorithm = 16, + /** NNPACK function was called with convolution transform strategy not in nnp_convolution_transform_strategy enum */ + nnp_status_invalid_transform_strategy = 17, + /** NNPACK function was called with output_subsampling.height == 0 or output_subsampling.width == 0 */ + nnp_status_invalid_output_subsampling = 13, + /** NNPACK function was called with activation not in nnp_activation enum */ + nnp_status_invalid_activation = 14, + /** NNPACK function was called with invalid activation parameters */ + nnp_status_invalid_activation_parameters = 15, + + /** NNPACK does not support the particular input size for the function */ + nnp_status_unsupported_input_size = 20, + /** NNPACK does not support the particular input stride for the function */ + nnp_status_unsupported_input_stride = 21, + /** NNPACK does not support the particular input padding for the function */ + nnp_status_unsupported_input_padding = 22, + /** NNPACK does not support the particular kernel size for the function */ + nnp_status_unsupported_kernel_size = 23, + /** NNPACK does not support the particular pooling size for the function */ + nnp_status_unsupported_pooling_size = 24, + /** NNPACK does not support the particular pooling stride for the function */ + nnp_status_unsupported_pooling_stride = 25, + /** NNPACK does not support the particular convolution algorithm for the function */ + nnp_status_unsupported_algorithm = 26, + /** NNPACK does not support the particular convolution transform strategy for the algorithm */ + nnp_status_unsupported_transform_strategy = 27, + /** NNPACK does not support the particular activation function for the function */ + nnp_status_unsupported_activation = 28, + /** NNPACK does not support the particular activation function parameters for the function */ + nnp_status_unsupported_activation_parameters = 29, + + /** NNPACK function was called before the library was initialized */ + nnp_status_uninitialized = 50, + /** NNPACK does not implement this function for the host CPU */ + nnp_status_unsupported_hardware = 51, + /** NNPACK failed to allocate memory for temporary buffers */ + nnp_status_out_of_memory = 52, + /** Scratch space buffer is too small */ + nnp_status_insufficient_buffer = 53, + /** Scratch space buffer is not properly aligned */ + nnp_status_misaligned_buffer = 54 +}; + +/** + * @brief Activation applied applied after a convolutional or fully-connected layer. + */ +enum nnp_activation { + /** Identity activation f(x) := x, i.e. no transformation */ + nnp_activation_identity = 0, + /** ReLU activation f(x) := max(0, x) */ + nnp_activation_relu = 1, +}; + +/** + * @brief Algorithm for computing convolutional layers. + */ +enum nnp_convolution_algorithm { + /** Let NNPACK choose the algorithm depending on layer parameters */ + nnp_convolution_algorithm_auto = 0, + /** Tiled convolution based on 2D Fourier transform with 8x8 blocks. Supports kernels up to 8x8. */ + nnp_convolution_algorithm_ft8x8 = 1, + /** Tiled convolution based on 2D Fourier transform with 16x16 blocks. Supports kernels up to 16x16. */ + nnp_convolution_algorithm_ft16x16 = 2, + /** Tiled convolution based on 2D Winograd transform F(3x3, 6x6) with 8x8 blocks. Supports only 3x3 kernels. */ + nnp_convolution_algorithm_wt8x8 = 3, + /** Direct convolution via implicit GEMM. */ + nnp_convolution_algorithm_implicit_gemm = 4, + /** Direct convolution implementation. */ + nnp_convolution_algorithm_direct = 5, + /** + * Tiled convolution based on 2D Winograd transform F(3x3, 6x6) with 8x8 blocks in FP16. + * Supports only 3x3 kernels. Implemented only for new ARM processors (with NEON-HP), + * on non-supported processors falls back to nnp_convolution_algorithm_wt8x8. + */ + nnp_convolution_algorithm_wt8x8_fp16 = 6, +}; + +enum nnp_convolution_transform_strategy { + nnp_convolution_transform_strategy_compute = 1, + nnp_convolution_transform_strategy_precompute = 2, + nnp_convolution_transform_strategy_reuse = 3 +}; + +/* For backward compatibility */ +#define nnp_convolution_transform_strategy_block_based nnp_convolution_transform_strategy_compute +#define nnp_convolution_transform_strategy_tuple_based nnp_convolution_transform_strategy_compute + +/** + * @brief Size of images, kernels, and pooling filters in NNPACK. + */ +struct nnp_size { + /** Width (horizontal size) of an image, kernel, or pooling filter. */ + size_t width; + /** Height (vertical size) of an image, kernel, or pooling filter. */ + size_t height; +}; + +/** + * @brief Padding of images in NNPACK. + */ +struct nnp_padding { + /** Padding above the image data */ + size_t top; + /** Padding on the right of image data */ + size_t right; + /** Padding below the image data */ + size_t bottom; + /** Padding on the left of image data */ + size_t left; +}; + +/** + * @brief Profiling information about time spent in different phases of a function call. + */ +struct nnp_profile { + /** Time spent inside the function call, in seconds. */ + double total; + /** Time spend on transformation of the input or input gradient tensor, in seconds. */ + double input_transform; + /** Time spend on transformation of the kernel or kernel gradient tensor, in seconds. */ + double kernel_transform; + /** Time spend on transformation of the output or output gradient tensor, in seconds. */ + double output_transform; + /** Time spend on multiplication-accumulation of transformed coefficients, in seconds. */ + double block_multiplication; +}; + +enum nnp_status nnp_initialize(void); + +enum nnp_status nnp_deinitialize(void); + +/** + * @brief Computes output of a 2D convolutional layer from input and kernel tensors. + * @details This function targets training of convolutional neural networks and performs forward propagation. + * It is optimized for moderate minibatch sizes (64-128) and can be inefficient on a small minibatch. + * For minibatch size 1, use nnp_convolution_inference for optimal performance. + * @param algorithm The type of algorithm to use for convolution. Possible values are: + * + * - nnp_convolution_algorithm_auto -- let the function choose the algorithm. + * - nnp_convolution_algorithm_ft8x8 -- tiled convolution based on 2D Fourier transform with 8x8 blocks. + * Supports kernels up to 8x8. + * - nnp_convolution_algorithm_ft16x16 -- tiled convolution based on 2D Fourier transform with 16x16 blocks. + * Supports kernels up to 16x16. + * - nnp_convolution_algorithm_wt8x8 -- tiled convolution based on 2D Winograd transform F(3x3, 6x6). + * Supports only 3x3 kernels. + * + * @param batch_size The number of images on the input and output of the convolutional layer. + * @param input_channels The number of channels (AKA features, dimensions) in the input images. + * @param output_channels The number of channels (AKA features, dimensions) in the output images. + * @param input_size Size of input images, excluding implicit zero-padding. + * @param input_padding Implicit zero-padding of input images. + * @param kernel_size Kernel size. + * @param[in] input A 4D tensor input[batch_size][input_channels][input_size.height][input_size.width]. + * @param[in] kernel A 4D tensor kernel[output_channels][input_channels][kernel_size.height][kernel_size.width]. + * @param[in] bias A 1D array bias[output_channels]. + * @param[out] output A 4D tensor output[batch_size][output_channels][output_size.height][output_size.width] where + * output_size.height = (input_padding.top + input_size.height + input_padding.bottom) - + * (kernel_size.height - 1) + * output_size.width = (input_padding.left + input_size.width + input_padding.right) - + * (kernel_size.width - 1) + * @param threadpool A thread pool for parallelization of the computation. + * If threadpool is NULL, the computation would run on the caller thread without parallelization. + * @param[out] profile An optional pointer to profiling structure. + * If provided, the structure would record time spent in different phases of the computation. + */ + +enum nnp_status nnp_convolution_output( + enum nnp_convolution_algorithm algorithm, + size_t batch_size, + size_t input_channels, + size_t output_channels, + struct nnp_size input_size, + struct nnp_padding input_padding, + struct nnp_size kernel_size, + const float* input, + const float* kernel, + const float* bias, + float* output, + void* workspace_buffer, + size_t* workspace_size, + enum nnp_activation activation, + const void* activation_parameters, + pthreadpool_t threadpool, + struct nnp_profile* profile); + +/** + * @brief Computes gradient of input of a 2D convolutional layer from gradient of output and kernel tensors. + * @details This function targets training of convolutional neural networks and performs backward propagation. + * It is optimized for moderate minibatch sizes (64-128) and can be inefficient on a small minibatch. + * @param algorithm The type of algorithm to use for convolution. Possible values are: + * + * - nnp_convolution_algorithm_auto -- let the function choose the algorithm. + * - nnp_convolution_algorithm_ft8x8 -- tiled convolution based on 2D Fourier transform with 8x8 blocks. + * Supports kernels up to 8x8. + * - nnp_convolution_algorithm_ft16x16 -- tiled convolution based on 2D Fourier transform with 16x16 blocks. + * Supports kernels up to 16x16. + * - nnp_convolution_algorithm_wt8x8 -- tiled convolution based on 2D Winograd transform F(3x3, 6x6). + * Supports only 3x3 kernels. + * + * @param batch_size The number of images (and their gradients) on the input and output of the convolutional layer. + * @param input_channels The number of channels (AKA features, dimensions) in the input images (and gradients). + * @param output_channels The number of channels (AKA features, dimensions) in the output images (and gradients). + * @param input_size Size of input images and their gradients, excluding implicit zero-padding. + * @param input_padding Implicit zero-padding of input images. + * @param kernel_size Kernel size. + * @param[in] grad_output A 4D tensor grad_output[batch_size][output_channels][output_size.height][output_size.width] + * where + * output_size.height = (input_padding.top + input_size.height + input_padding.bottom) - + * (kernel_size.height - 1) + * output_size.width = (input_padding.left + input_size.width + input_padding.right) - + * (kernel_size.width - 1) + * @param[in] kernel A 4D tensor kernel[output_channels][input_channels][kernel_size.height][kernel_size.width]. + * @param[out] grad_input A 4D tensor grad_input[batch_size][input_channels][input_size.height][input_size.width]. + * @param threadpool A thread pool for parallelization of the computation. + * If threadpool is NULL, the computation would run on the caller thread without parallelization. + * @param[out] profile An optional pointer to profiling structure. + * If provided, the structure would record time spent in different phases of the computation. + */ +enum nnp_status nnp_convolution_input_gradient( + enum nnp_convolution_algorithm algorithm, + size_t batch_size, + size_t input_channels, + size_t output_channels, + struct nnp_size input_size, + struct nnp_padding input_padding, + struct nnp_size kernel_size, + const float* grad_output, + const float* kernel, + float* grad_input, + void* workspace_buffer, + size_t* workspace_size, + enum nnp_activation activation, + const void* activation_parameters, + pthreadpool_t threadpool, + struct nnp_profile* profile); + +/** + * @brief Computes gradient of kernel of a 2D convolutional layer from gradient of output and input tensors. + * @details This function targets training of convolutional neural networks and performs backward propagation. + * It is optimized for moderate minibatch sizes (64-128) and can be inefficient on a small minibatch. + * @param algorithm The type of algorithm to use for convolution. Possible values are: + * + * - nnp_convolution_algorithm_auto -- let the function choose the algorithm. + * - nnp_convolution_algorithm_ft8x8 -- tiled convolution based on 2D Fourier transform with 8x8 blocks. + * Supports kernels up to 8x8. + * - nnp_convolution_algorithm_ft16x16 -- tiled convolution based on 2D Fourier transform with 16x16 blocks. + * Supports kernels up to 16x16. + * + * @param batch_size The number of images (and their gradients) on the input and output of the convolutional layer. + * @param input_channels The number of channels (AKA features, dimensions) in the input images. + * @param output_channels The number of channels (AKA features, dimensions) in the output images (and gradients). + * @param input_size Size of input images and their gradients, excluding implicit zero-padding. + * @param input_padding Implicit zero-padding of input images. + * @param kernel_size Kernel size. + * @param[in] input A 4D tensor input[batch_size][input_channels][input_size.height][input_size.width]. + * @param[in] grad_output A 4D tensor grad_output[batch_size][output_channels][output_size.height][output_size.width] + * where + * output_size.height = (input_padding.top + input_size.height + input_padding.bottom) - + * (kernel_size.height - 1) + * output_size.width = (input_padding.left + input_size.width + input_padding.right) - + * (kernel_size.width - 1) + * @param[out] grad_kernel A 4D tensor + * grad_kernel[output_channels][input_channels][kernel_size.height][kernel_size.width]. + * @param threadpool A thread pool for parallelization of the computation. + * If threadpool is NULL, the computation would run on the caller thread without parallelization. + * @param[out] profile An optional pointer to profiling structure. + * If provided, the structure would record time spent in different phases of the computation. + */ +enum nnp_status nnp_convolution_kernel_gradient( + enum nnp_convolution_algorithm algorithm, + size_t batch_size, + size_t input_channels, + size_t output_channels, + struct nnp_size input_size, + struct nnp_padding input_padding, + struct nnp_size kernel_size, + const float* input, + const float* grad_output, + float* grad_kernel, + void* workspace_buffer, + size_t* workspace_size, + enum nnp_activation activation, + const void* activation_parameters, + pthreadpool_t threadpool, + struct nnp_profile* profile); + +/** + * @brief Computes output of a 2D convolutional layer for a single input image and a kernel tensor. + * @details This function targets prediction with convolutional neural networks and performs forward propagation. + * @param algorithm The type of algorithm to use for convolution. Possible values are: + * + * - nnp_convolution_algorithm_auto -- let the function choose the algorithm. + * - nnp_convolution_algorithm_ft8x8 -- tiled convolution based on 2D Fourier transform with 8x8 blocks. + * Supports kernels up to 8x8. + * - nnp_convolution_algorithm_ft16x16 -- tiled convolution based on 2D Fourier transform with 16x16 blocks. + * Supports kernels up to 16x16. + * - nnp_convolution_algorithm_wt8x8 -- tiled convolution based on 2D Winograd transform F(3x3, 6x6). + * Supports only 3x3 kernels. + * + * @param transform_strategy A strategy that guides computation of kernel transforms coefficients. + * Possible values are: + * + * - nnp_convolution_transform_strategy_block_based -- do multiplication-accumulations on blocks of transformed + * coefficients. + * - nnp_convolution_transform_strategy_tuple_based -- do multiplication-accumulations on tuples of transformed + * coefficients. + * + * @param input_channels The number of channels (AKA features, dimensions) in the input image. + * @param output_channels The number of channels (AKA features, dimensions) in the output image. + * @param input_size Size of input image, excluding implicit zero-padding. + * @param input_padding Implicit zero-padding of input image. + * @param kernel_size Kernel size. + * @param output_subsampling Subsample region for output, also known as convolution stride. + * @param[in] input A 3D tensor input[input_channels][input_size.height][input_size.width]. + * @param[in] kernel A 4D tensor kernel[output_channels][input_channels][kernel_size.height][kernel_size.width]. + * @param[in] bias A 1D array bias[output_channels]. + * @param[out] output A 3D tensor output[output_channels][output_size.height][output_size.width] where + * output_size.height = (input_padding.top + input_size.height + input_padding.bottom) - + * (kernel_size.height - 1) + * output_size.width = (input_padding.left + input_size.width + input_padding.right) - + * (kernel_size.width - 1) + * @param[in] workspace_buffer Buffer for scratch memory used during computation. Buffer must be aligned on 64 bytes. + * If workspace_buffer is NULL and workspace_size is non-NULL, NNPACK would store the size + * of required workspace memory at the workspace_size location, and exit without + * computations. + * If workspace_buffer is NULL and workspace_size is NULL, NNPACK would allocate memory + * before and deallocate after this computation, potentially at significant runtime cost. + * @param[in,out] workspace_size Pointer to the size of workspace buffer. + * If workspace_buffer is NULL, NNPACK will write the size of required scratch memory to + * the location specified by this pointer. + * If workspace_buffer is non-NULL, NNPACK expects workspace_size to specify the size of + * the buffer, in bytes. + * If workspace_size is NULL, workspace_buffer must be NULL as well. In this case NNPACK + * would allocate memory before and deallocate after this computation, potentially at + * significant runtime cost. + * @param threadpool A thread pool for parallelization of the computation. + * If threadpool is NULL, the computation would run on the caller thread without parallelization. + * @param[out] profile An optional pointer to profiling structure. + * If provided, the structure would record time spent in different phases of the computation. + */ +enum nnp_status nnp_convolution_inference( + enum nnp_convolution_algorithm algorithm, + enum nnp_convolution_transform_strategy transform_strategy, + size_t input_channels, + size_t output_channels, + struct nnp_size input_size, + struct nnp_padding input_padding, + struct nnp_size kernel_size, + struct nnp_size output_subsampling, + const float* input, + const float* kernel, + const float* bias, + float* output, + void* workspace_buffer, + size_t* workspace_size, + enum nnp_activation activation, + const void* activation_parameters, + pthreadpool_t threadpool, + struct nnp_profile* profile); + +/** + * @brief Computes output of a fully connected layer from input and kernel matrices. + * @details This function targets training of convolutional neural networks and performs forward propagation. + * It is optimized for moderate minibatch sizes (64-128) and can be inefficient on a small minibatch. + * For minibatch size 1, use nnp_fully_connected_inference for optimal performance. + * @param batch_size The number of vectors on the input and output of the fully connected layer. + * @param input_channels The number of channels (AKA features, dimensions) in the input matrix. + * @param output_channels The number of channels (AKA features, dimensions) in the output matrix. + * @param[in] input A 2D matrix input[batch_size][input_channels]. + * @param[in] kernel A 2D matrix kernel[output_channels][input_channels]. + * @param[out] output A 2D matrix output[batch_size][output_channels]. + * @param threadpool A thread pool for parallelization of the computation. + * If threadpool is NULL, the computation would run on the caller thread without parallelization. + */ +enum nnp_status nnp_fully_connected_output( + size_t batch_size, + size_t input_channels, + size_t output_channels, + const float input[], + const float kernel[], + float output[], + pthreadpool_t threadpool, + struct nnp_profile* profile); + +/** + * @brief Computes output of a fully connected layer for a single input vector and a kernel matrix. + * @details This function targets prediction with convolutional neural networks and performs forward propagation. + * @param input_channels The number of channels (AKA features, dimensions) in the input vector. + * @param output_channels The number of channels (AKA features, dimensions) in the output vector. + * @param[in] input A 1D array input[input_channels] of FP32 elements. + * @param[in] kernel A 2D matrix kernel[output_channels][input_channels] of FP32 elements. + * @param[out] output A 1D array output[output_channels] of FP32 elements. + * @param threadpool A thread pool for parallelization of the computation. + * If threadpool is NULL, the computation would run on the caller thread without parallelization. + */ +enum nnp_status nnp_fully_connected_inference( + size_t input_channels, + size_t output_channels, + const float* input, + const float* kernel, + float* output, + pthreadpool_t threadpool); + +/** + * @brief Computes output of a fully connected layer for a single input vector and a kernel matrix. + * @details This function targets prediction with convolutional neural networks and performs forward propagation. + * @param input_channels The number of channels (AKA features, dimensions) in the input vector. + * @param output_channels The number of channels (AKA features, dimensions) in the output vector. + * @param[in] input A 1D array input[input_channels] of FP32 elements. + * @param[in] kernel A 2D matrix kernel[output_channels][input_channels] of FP16 (ARM alternative format) elements. + * @param[out] output A 1D array output[output_channels] of FP32 elements. + * @param threadpool A thread pool for parallelization of the computation. + * If threadpool is NULL, the computation would run on the caller thread without parallelization. + */ +enum nnp_status nnp_fully_connected_inference_f16f32( + size_t input_channels, + size_t output_channels, + const float* input, + const void* kernel, + float* output, + pthreadpool_t threadpool); + +/** + * @brief Computes output of a max-pooling layer for an input tensor. + * @details This function targets both prediction and training of convolutional neural networks and performs forward + * propagation. Is is optimized for both large and small minibatch sizes. + * @param batch_size The number of images on the input and output of the max-pooling layer. + * @param channels The number of channels (AKA features, dimensions) in both input and output images. + * @param input_size Size of input images, excluding implicit zero-padding. + * @param input_padding Implicit padding of input images. The padding pixels are ignored by the pooling filter, but + * affect the output size. + * @param pooling_size Size of the pooling filter. Only 2x2 filter are currently supported. + * @param pooling_stride Stride of the pooling filter. Only 2x2 strides are currently supported. + * @param[in] input A 4D tensor input[batch_size][channels][input_size.height][input_size.width]. + * @param[out] output A 4D tensor output[batch_size][channels][output_size.height][output_size.width] where + * output_size.height = ceil( + * (input_padding.top + input_size.height + input_padding.bottom - pooling_size.height) / + * pooling_stride.height) + 1 + * output_size.width = ceil( + * (input_padding.left + input_size.width + input_padding.right - pooling_size.width) / + * pooling_stride.width) + 1 + * @param threadpool A thread pool for parallelization of the computation. + * If threadpool is NULL, the computation would run on the caller thread without parallelization. + */ +enum nnp_status nnp_max_pooling_output( + size_t batch_size, + size_t channels, + struct nnp_size input_size, + struct nnp_padding input_padding, + struct nnp_size pooling_size, + struct nnp_size pooling_stride, + const float input[], + float output[], + pthreadpool_t threadpool); + +/** + * @brief Computes output of a softmax layer for an input matrix. + * @details This function targets both prediction and training of convolutional neural networks and performs forward + * propagation. Is is optimized for both large and small minibatch sizes. + * @param batch_size The number of vectors on the input and output of the softmax layer. + * @param channels The number of channels (AKA features, dimensions) in both input and output vectors. + * @param[in] input A 2D matrix input[batch_size][channels]. + * @param[out] output A 2D matrix output[batch_size][channels]. + * @param threadpool A thread pool for parallelization of the computation. + * If threadpool is NULL, the computation would run on the caller thread without parallelization. + */ +enum nnp_status nnp_softmax_output( + size_t batch_size, + size_t channels, + const float input[], + float output[], + pthreadpool_t threadpool); + +/** + * @brief Computes output of a rectified linear unit (ReLU) layer for an input matrix. + * @details This function targets both prediction and training of convolutional neural networks and performs forward + * propagation. Is is optimized for both large and small minibatch sizes. + * @param batch_size The number of vectors on the input and output of the ReLU layer. + * @param channels The number of channels (AKA features, dimensions) in both input and output matrices. + * @param[in] input A 2D matrix input[batch_size][channels]. + * @param[out] output A 2D matrix output[batch_size][channels]. + * @param threadpool A thread pool for parallelization of the computation. + * If threadpool is NULL, the computation would run on the caller thread without parallelization. + */ +enum nnp_status nnp_relu_output( + size_t batch_size, + size_t channels, + const float input[], + float output[], + float negative_slope, + pthreadpool_t threadpool); + +/** + * @brief Computes gradient of input of a rectified linear unit (ReLU) layer from gradient of output and input matrices. + * @details This function targets training of convolutional neural networks and performs backward propagation. + * Is is optimized for both large and small minibatch sizes. + * @param batch_size The number of vectors on the input and output of the ReLU layer. + * @param channels The number of channels (AKA features, dimensions) in both input and output matrices. + * @param[in] input A 2D matrix input[batch_size][channels]. + * @param[out] output A 2D matrix output[batch_size][channels]. + * @param threadpool A thread pool for parallelization of the computation. + * If threadpool is NULL, the computation would run on the caller thread without parallelization. + */ +enum nnp_status nnp_relu_input_gradient( + size_t batch_size, + size_t channels, + const float grad_output[], + const float input[], + float grad_input[], + float negative_slope, + pthreadpool_t threadpool); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#ifdef __cplusplus +// Backward compatible implementations for nnp_convolution_*, if we are in C++ +// mode. +inline enum nnp_status nnp_convolution_output( + enum nnp_convolution_algorithm algorithm, + size_t batch_size, + size_t input_channels, + size_t output_channels, + struct nnp_size input_size, + struct nnp_padding input_padding, + struct nnp_size kernel_size, + const float input[], + const float kernel[], + const float bias[], + float output[], + pthreadpool_t threadpool, + struct nnp_profile* profile) +{ + return nnp_convolution_output( + algorithm, + batch_size, input_channels, output_channels, + input_size, input_padding, kernel_size, + input, kernel, bias, output, + NULL, NULL, + nnp_activation_identity, NULL, threadpool, profile); +} + +inline enum nnp_status nnp_convolution_input_gradient( + enum nnp_convolution_algorithm algorithm, + size_t batch_size, + size_t input_channels, + size_t output_channels, + struct nnp_size input_size, + struct nnp_padding input_padding, + struct nnp_size kernel_size, + const float grad_output[], + const float kernel[], + float grad_input[], + pthreadpool_t threadpool, + struct nnp_profile* profile) +{ + return nnp_convolution_input_gradient( + algorithm, + batch_size, input_channels, output_channels, + input_size, input_padding, kernel_size, + grad_output, kernel, grad_input, + NULL, NULL, + nnp_activation_identity, NULL, threadpool, profile); +} + +inline enum nnp_status nnp_convolution_kernel_gradient( + enum nnp_convolution_algorithm algorithm, + size_t batch_size, + size_t input_channels, + size_t output_channels, + struct nnp_size input_size, + struct nnp_padding input_padding, + struct nnp_size kernel_size, + const float input[], + const float grad_output[], + float grad_kernel[], + pthreadpool_t threadpool, + struct nnp_profile* profile) +{ + return nnp_convolution_kernel_gradient( + algorithm, + batch_size, input_channels, output_channels, + input_size, input_padding, kernel_size, + input, grad_output, grad_kernel, + NULL, NULL, + nnp_activation_identity, NULL, threadpool, profile); +} + +inline enum nnp_status nnp_convolution_inference( + enum nnp_convolution_algorithm algorithm, + enum nnp_convolution_transform_strategy transform_strategy, + size_t input_channels, + size_t output_channels, + struct nnp_size input_size, + struct nnp_padding input_padding, + struct nnp_size kernel_size, + struct nnp_size output_subsampling, + const float input[], + const float kernel[], + const float bias[], + float output[], + pthreadpool_t threadpool, + struct nnp_profile* profile) { + return nnp_convolution_inference( + algorithm, transform_strategy, + input_channels, output_channels, + input_size, input_padding, kernel_size, output_subsampling, + input, kernel, bias, output, NULL, NULL, + nnp_activation_identity, NULL, + threadpool, profile); +} + +#endif // __cplusplus + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/psimd.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/psimd.h new file mode 100644 index 0000000000000000000000000000000000000000..8c8a96bf67bb294d6e0f4763bf42a54fbccc7541 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/psimd.h @@ -0,0 +1,1389 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#pragma once +#ifndef PSIMD_H +#define PSIMD_H + +#if defined(__CUDA_ARCH__) + /* CUDA compiler */ + #define PSIMD_INTRINSIC __forceinline__ __device__ +#elif defined(__OPENCL_VERSION__) + /* OpenCL compiler */ + #define PSIMD_INTRINSIC inline static +#elif defined(__INTEL_COMPILER) + /* Intel compiler, even on Windows */ + #define PSIMD_INTRINSIC inline static __attribute__((__always_inline__)) +#elif defined(__GNUC__) + /* GCC-compatible compiler (gcc/clang/icc) */ + #define PSIMD_INTRINSIC inline static __attribute__((__always_inline__)) +#elif defined(_MSC_VER) + /* MSVC-compatible compiler (cl/icl/clang-cl) */ + #define PSIMD_INTRINSIC __forceinline static +#elif defined(__cplusplus) + /* Generic C++ compiler */ + #define PSIMD_INTRINSIC inline static +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) + /* Generic C99 compiler */ + #define PSIMD_INTRINSIC inline static +#else + /* Generic C compiler */ + #define PSIMD_INTRINSIC static +#endif + +#if defined(__GNUC__) || defined(__clang__) + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + #include + #endif + + #if defined(__SSE2__) + #include + #endif + + #if defined(__SSE3__) + #include + #endif + + #if defined(__SSSE3__) + #include + #endif + + #if defined(__SSE4_1__) + #include + #endif + + #if defined(__SSE4_2__) + #include + #endif + + #if defined(__AVX__) + #include + #endif +#elif defined(_MSC_VER) + #include +#endif + +#if defined(__cplusplus) + #define PSIMD_CXX_SYNTAX +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) + #define PSIMD_C11_SYNTAX +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) + #define PSIMD_C99_SYNTAX +#else + #define PSIMD_C89_SYNTAX +#endif + +#if defined(__cplusplus) && (__cplusplus >= 201103L) + #include + #include +#elif !defined(__OPENCL_VERSION__) + #include + #include +#endif + +#if defined(__GNUC__) || defined(__clang__) + #define PSIMD_HAVE_F64 0 + #define PSIMD_HAVE_F32 1 + #define PSIMD_HAVE_U8 1 + #define PSIMD_HAVE_S8 1 + #define PSIMD_HAVE_U16 1 + #define PSIMD_HAVE_S16 1 + #define PSIMD_HAVE_U32 1 + #define PSIMD_HAVE_S32 1 + #define PSIMD_HAVE_U64 0 + #define PSIMD_HAVE_S64 0 + + typedef int8_t psimd_s8 __attribute__((vector_size(16), aligned(1))); + typedef uint8_t psimd_u8 __attribute__((vector_size(16), aligned(1))); + typedef int16_t psimd_s16 __attribute__((vector_size(16), aligned(2))); + typedef uint16_t psimd_u16 __attribute__((vector_size(16), aligned(2))); + typedef int32_t psimd_s32 __attribute__((vector_size(16), aligned(4))); + typedef uint32_t psimd_u32 __attribute__((vector_size(16), aligned(4))); + typedef float psimd_f32 __attribute__((vector_size(16), aligned(4))); + + typedef struct { + psimd_s8 lo; + psimd_s8 hi; + } psimd_s8x2; + + typedef struct { + psimd_u8 lo; + psimd_u8 hi; + } psimd_u8x2; + + typedef struct { + psimd_s16 lo; + psimd_s16 hi; + } psimd_s16x2; + + typedef struct { + psimd_u16 lo; + psimd_u16 hi; + } psimd_u16x2; + + typedef struct { + psimd_s32 lo; + psimd_s32 hi; + } psimd_s32x2; + + typedef struct { + psimd_u32 lo; + psimd_u32 hi; + } psimd_u32x2; + + typedef struct { + psimd_f32 lo; + psimd_f32 hi; + } psimd_f32x2; + + /* Bit casts */ + PSIMD_INTRINSIC psimd_u32x2 psimd_cast_s32x2_u32x2(psimd_s32x2 v) { + return (psimd_u32x2) { .lo = (psimd_u32) v.lo, .hi = (psimd_u32) v.hi }; + } + + PSIMD_INTRINSIC psimd_f32x2 psimd_cast_s32x2_f32x2(psimd_s32x2 v) { + return (psimd_f32x2) { .lo = (psimd_f32) v.lo, .hi = (psimd_f32) v.hi }; + } + + PSIMD_INTRINSIC psimd_s32x2 psimd_cast_u32x2_s32x2(psimd_u32x2 v) { + return (psimd_s32x2) { .lo = (psimd_s32) v.lo, .hi = (psimd_s32) v.hi }; + } + + PSIMD_INTRINSIC psimd_f32x2 psimd_cast_u32x2_f32x2(psimd_u32x2 v) { + return (psimd_f32x2) { .lo = (psimd_f32) v.lo, .hi = (psimd_f32) v.hi }; + } + + PSIMD_INTRINSIC psimd_s32x2 psimd_cast_f32x2_s32x2(psimd_f32x2 v) { + return (psimd_s32x2) { .lo = (psimd_s32) v.lo, .hi = (psimd_s32) v.hi }; + } + + PSIMD_INTRINSIC psimd_u32x2 psimd_cast_f32x2_u32x2(psimd_f32x2 v) { + return (psimd_u32x2) { .lo = (psimd_u32) v.lo, .hi = (psimd_u32) v.hi }; + } + + /* Swap */ + PSIMD_INTRINSIC void psimd_swap_s8(psimd_s8 a[1], psimd_s8 b[1]) { + const psimd_s8 new_a = *b; + const psimd_s8 new_b = *a; + *a = new_a; + *b = new_b; + } + + PSIMD_INTRINSIC void psimd_swap_u8(psimd_u8 a[1], psimd_u8 b[1]) { + const psimd_u8 new_a = *b; + const psimd_u8 new_b = *a; + *a = new_a; + *b = new_b; + } + + PSIMD_INTRINSIC void psimd_swap_s16(psimd_s16 a[1], psimd_s16 b[1]) { + const psimd_s16 new_a = *b; + const psimd_s16 new_b = *a; + *a = new_a; + *b = new_b; + } + + PSIMD_INTRINSIC void psimd_swap_u16(psimd_u16 a[1], psimd_u16 b[1]) { + const psimd_u16 new_a = *b; + const psimd_u16 new_b = *a; + *a = new_a; + *b = new_b; + } + + PSIMD_INTRINSIC void psimd_swap_s32(psimd_s32 a[1], psimd_s32 b[1]) { + const psimd_s32 new_a = *b; + const psimd_s32 new_b = *a; + *a = new_a; + *b = new_b; + } + + PSIMD_INTRINSIC void psimd_swap_u32(psimd_u32 a[1], psimd_u32 b[1]) { + const psimd_u32 new_a = *b; + const psimd_u32 new_b = *a; + *a = new_a; + *b = new_b; + } + + PSIMD_INTRINSIC void psimd_swap_f32(psimd_f32 a[1], psimd_f32 b[1]) { + const psimd_f32 new_a = *b; + const psimd_f32 new_b = *a; + *a = new_a; + *b = new_b; + } + + /* Zero-initialization */ + PSIMD_INTRINSIC psimd_s8 psimd_zero_s8(void) { + return (psimd_s8) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_u8 psimd_zero_u8(void) { + return (psimd_u8) { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_s16 psimd_zero_s16(void) { + return (psimd_s16) { 0, 0, 0, 0, 0, 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_u16 psimd_zero_u16(void) { + return (psimd_u16) { 0, 0, 0, 0, 0, 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_s32 psimd_zero_s32(void) { + return (psimd_s32) { 0, 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_u32 psimd_zero_u32(void) { + return (psimd_u32) { 0, 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_zero_f32(void) { + return (psimd_f32) { 0.0f, 0.0f, 0.0f, 0.0f }; + } + + /* Initialization to the same constant */ + PSIMD_INTRINSIC psimd_s8 psimd_splat_s8(int8_t c) { + return (psimd_s8) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c }; + } + + PSIMD_INTRINSIC psimd_u8 psimd_splat_u8(uint8_t c) { + return (psimd_u8) { c, c, c, c, c, c, c, c, c, c, c, c, c, c, c, c }; + } + + PSIMD_INTRINSIC psimd_s16 psimd_splat_s16(int16_t c) { + return (psimd_s16) { c, c, c, c, c, c, c, c }; + } + + PSIMD_INTRINSIC psimd_u16 psimd_splat_u16(uint16_t c) { + return (psimd_u16) { c, c, c, c, c, c, c, c }; + } + + PSIMD_INTRINSIC psimd_s32 psimd_splat_s32(int32_t c) { + return (psimd_s32) { c, c, c, c }; + } + + PSIMD_INTRINSIC psimd_u32 psimd_splat_u32(uint32_t c) { + return (psimd_u32) { c, c, c, c }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat_f32(float c) { + return (psimd_f32) { c, c, c, c }; + } + + /* Load vector */ + PSIMD_INTRINSIC psimd_s8 psimd_load_s8(const void* address) { + return *((const psimd_s8*) address); + } + + PSIMD_INTRINSIC psimd_u8 psimd_load_u8(const void* address) { + return *((const psimd_u8*) address); + } + + PSIMD_INTRINSIC psimd_s16 psimd_load_s16(const void* address) { + return *((const psimd_s16*) address); + } + + PSIMD_INTRINSIC psimd_u16 psimd_load_u16(const void* address) { + return *((const psimd_u16*) address); + } + + PSIMD_INTRINSIC psimd_s32 psimd_load_s32(const void* address) { + return *((const psimd_s32*) address); + } + + PSIMD_INTRINSIC psimd_u32 psimd_load_u32(const void* address) { + return *((const psimd_u32*) address); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load_f32(const void* address) { + return *((const psimd_f32*) address); + } + + PSIMD_INTRINSIC psimd_s8 psimd_load_splat_s8(const void* address) { + return psimd_splat_s8(*((const int8_t*) address)); + } + + PSIMD_INTRINSIC psimd_u8 psimd_load_splat_u8(const void* address) { + return psimd_splat_u8(*((const uint8_t*) address)); + } + + PSIMD_INTRINSIC psimd_s16 psimd_load_splat_s16(const void* address) { + return psimd_splat_s16(*((const int16_t*) address)); + } + + PSIMD_INTRINSIC psimd_u16 psimd_load_splat_u16(const void* address) { + return psimd_splat_u16(*((const uint16_t*) address)); + } + + PSIMD_INTRINSIC psimd_s32 psimd_load_splat_s32(const void* address) { + return psimd_splat_s32(*((const int32_t*) address)); + } + + PSIMD_INTRINSIC psimd_u32 psimd_load_splat_u32(const void* address) { + return psimd_splat_u32(*((const uint32_t*) address)); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load_splat_f32(const void* address) { + return psimd_splat_f32(*((const float*) address)); + } + + PSIMD_INTRINSIC psimd_s32 psimd_load1_s32(const void* address) { + return (psimd_s32) { *((const int32_t*) address), 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_u32 psimd_load1_u32(const void* address) { + return (psimd_u32) { *((const uint32_t*) address), 0, 0, 0 }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load1_f32(const void* address) { + return (psimd_f32) { *((const float*) address), 0.0f, 0.0f, 0.0f }; + } + + PSIMD_INTRINSIC psimd_s32 psimd_load2_s32(const void* address) { + const int32_t* address_s32 = (const int32_t*) address; + return (psimd_s32) { address_s32[0], address_s32[1], 0, 0 }; + } + + PSIMD_INTRINSIC psimd_u32 psimd_load2_u32(const void* address) { + const uint32_t* address_u32 = (const uint32_t*) address; + return (psimd_u32) { address_u32[0], address_u32[1], 0, 0 }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load2_f32(const void* address) { + const float* address_f32 = (const float*) address; + return (psimd_f32) { address_f32[0], address_f32[1], 0.0f, 0.0f }; + } + + PSIMD_INTRINSIC psimd_s32 psimd_load3_s32(const void* address) { + const int32_t* address_s32 = (const int32_t*) address; + return (psimd_s32) { address_s32[0], address_s32[1], address_s32[2], 0 }; + } + + PSIMD_INTRINSIC psimd_u32 psimd_load3_u32(const void* address) { + const uint32_t* address_u32 = (const uint32_t*) address; + return (psimd_u32) { address_u32[0], address_u32[1], address_u32[2], 0 }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load3_f32(const void* address) { + const float* address_f32 = (const float*) address; + return (psimd_f32) { address_f32[0], address_f32[1], address_f32[2], 0.0f }; + } + + PSIMD_INTRINSIC psimd_s32 psimd_load4_s32(const void* address) { + return psimd_load_s32(address); + } + + PSIMD_INTRINSIC psimd_u32 psimd_load4_u32(const void* address) { + return psimd_load_u32(address); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load4_f32(const void* address) { + return psimd_load_f32(address); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load_stride2_f32(const void* address) { + const psimd_f32 v0x1x = psimd_load_f32(address); + const psimd_f32 vx2x3 = psimd_load_f32((const float*) address + 3); + #if defined(__clang__) + return __builtin_shufflevector(v0x1x, vx2x3, 0, 2, 5, 7); + #else + return __builtin_shuffle(v0x1x, vx2x3, (psimd_s32) { 0, 2, 5, 7 }); + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_load1_stride2_f32(const void* address) { + return psimd_load_f32(address); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load2_stride2_f32(const void* address) { + const float* address_f32 = (const float*) address; + return (psimd_f32) { address_f32[0], address_f32[2], 0.0f, 0.0f }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load3_stride2_f32(const void* address) { + const psimd_f32 v0x1x = psimd_load_f32(address); + const psimd_f32 v2zzz = psimd_load1_f32((const float*) address + 2); + #if defined(__clang__) + return __builtin_shufflevector(v0x1x, v2zzz, 0, 2, 4, 6); + #else + return __builtin_shuffle(v0x1x, v2zzz, (psimd_s32) { 0, 2, 4, 6 }); + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_load4_stride2_f32(const void* address) { + return psimd_load_stride2_f32(address); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load_stride_f32(const void* address, size_t stride) { + const float* address0_f32 = (const float*) address; + const float* address1_f32 = address0_f32 + stride; + const float* address2_f32 = address1_f32 + stride; + const float* address3_f32 = address2_f32 + stride; + return (psimd_f32) { *address0_f32, *address1_f32, *address2_f32, *address3_f32 }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load1_stride_f32(const void* address, size_t stride) { + return psimd_load1_f32(address); + } + + PSIMD_INTRINSIC psimd_f32 psimd_load2_stride_f32(const void* address, size_t stride) { + const float* address_f32 = (const float*) address; + return (psimd_f32) { address_f32[0], address_f32[stride], 0.0f, 0.0f }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load3_stride_f32(const void* address, size_t stride) { + const float* address0_f32 = (const float*) address; + const float* address1_f32 = address0_f32 + stride; + const float* address2_f32 = address1_f32 + stride; + return (psimd_f32) { *address0_f32, *address1_f32, *address2_f32, 0.0f }; + } + + PSIMD_INTRINSIC psimd_f32 psimd_load4_stride_f32(const void* address, size_t stride) { + return psimd_load_stride_f32(address, stride); + } + + /* Store vector */ + PSIMD_INTRINSIC void psimd_store_s8(void* address, psimd_s8 value) { + *((psimd_s8*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store_u8(void* address, psimd_u8 value) { + *((psimd_u8*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store_s16(void* address, psimd_s16 value) { + *((psimd_s16*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store_u16(void* address, psimd_u16 value) { + *((psimd_u16*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store_s32(void* address, psimd_s32 value) { + *((psimd_s32*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store_u32(void* address, psimd_u32 value) { + *((psimd_u32*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store_f32(void* address, psimd_f32 value) { + *((psimd_f32*) address) = value; + } + + PSIMD_INTRINSIC void psimd_store1_s32(void* address, psimd_s32 value) { + *((int32_t*) address) = value[0]; + } + + PSIMD_INTRINSIC void psimd_store1_u32(void* address, psimd_u32 value) { + *((uint32_t*) address) = value[0]; + } + + PSIMD_INTRINSIC void psimd_store1_f32(void* address, psimd_f32 value) { + *((float*) address) = value[0]; + } + + PSIMD_INTRINSIC void psimd_store2_s32(void* address, psimd_s32 value) { + int32_t* address_s32 = (int32_t*) address; + address_s32[0] = value[0]; + address_s32[1] = value[1]; + } + + PSIMD_INTRINSIC void psimd_store2_u32(void* address, psimd_u32 value) { + uint32_t* address_u32 = (uint32_t*) address; + address_u32[0] = value[0]; + address_u32[1] = value[1]; + } + + PSIMD_INTRINSIC void psimd_store2_f32(void* address, psimd_f32 value) { + float* address_f32 = (float*) address; + address_f32[0] = value[0]; + address_f32[1] = value[1]; + } + + PSIMD_INTRINSIC void psimd_store3_s32(void* address, psimd_s32 value) { + int32_t* address_s32 = (int32_t*) address; + address_s32[0] = value[0]; + address_s32[1] = value[1]; + address_s32[2] = value[2]; + } + + PSIMD_INTRINSIC void psimd_store3_u32(void* address, psimd_u32 value) { + uint32_t* address_u32 = (uint32_t*) address; + address_u32[0] = value[0]; + address_u32[1] = value[1]; + address_u32[2] = value[2]; + } + + PSIMD_INTRINSIC void psimd_store3_f32(void* address, psimd_f32 value) { + float* address_f32 = (float*) address; + address_f32[0] = value[0]; + address_f32[1] = value[1]; + address_f32[2] = value[2]; + } + + PSIMD_INTRINSIC void psimd_store4_s32(void* address, psimd_s32 value) { + psimd_store_s32(address, value); + } + + PSIMD_INTRINSIC void psimd_store4_u32(void* address, psimd_u32 value) { + psimd_store_u32(address, value); + } + + PSIMD_INTRINSIC void psimd_store4_f32(void* address, psimd_f32 value) { + psimd_store_f32(address, value); + } + + PSIMD_INTRINSIC void psimd_store_stride_f32(void* address, size_t stride, psimd_f32 value) { + float* address0_f32 = (float*) address; + float* address1_f32 = address0_f32 + stride; + float* address2_f32 = address1_f32 + stride; + float* address3_f32 = address2_f32 + stride; + *address0_f32 = value[0]; + *address1_f32 = value[1]; + *address2_f32 = value[2]; + *address3_f32 = value[3]; + } + + PSIMD_INTRINSIC void psimd_store1_stride_f32(void* address, size_t stride, psimd_f32 value) { + psimd_store1_f32(address, value); + } + + PSIMD_INTRINSIC void psimd_store2_stride_f32(void* address, size_t stride, psimd_f32 value) { + float* address_f32 = (float*) address; + address_f32[0] = value[0]; + address_f32[stride] = value[1]; + } + + PSIMD_INTRINSIC void psimd_store3_stride_f32(void* address, size_t stride, psimd_f32 value) { + float* address0_f32 = (float*) address; + float* address1_f32 = address0_f32 + stride; + float* address2_f32 = address1_f32 + stride; + *address0_f32 = value[0]; + *address1_f32 = value[1]; + *address2_f32 = value[2]; + } + + /* Vector addition */ + PSIMD_INTRINSIC psimd_s8 psimd_add_s8(psimd_s8 a, psimd_s8 b) { + return a + b; + } + + PSIMD_INTRINSIC psimd_u8 psimd_add_u8(psimd_u8 a, psimd_u8 b) { + return a + b; + } + + PSIMD_INTRINSIC psimd_s16 psimd_add_s16(psimd_s16 a, psimd_s16 b) { + return a + b; + } + + PSIMD_INTRINSIC psimd_u16 psimd_add_u16(psimd_u16 a, psimd_u16 b) { + return a + b; + } + + PSIMD_INTRINSIC psimd_s32 psimd_add_s32(psimd_s32 a, psimd_s32 b) { + return a + b; + } + + PSIMD_INTRINSIC psimd_u32 psimd_add_u32(psimd_u32 a, psimd_u32 b) { + return a + b; + } + + PSIMD_INTRINSIC psimd_f32 psimd_add_f32(psimd_f32 a, psimd_f32 b) { + #if defined(__ARM_ARCH_7A__) && defined(__ARM_NEON__) && !defined(__FAST_MATH__) + return (psimd_f32) vaddq_f32((float32x4_t) a, (float32x4_t) b); + #else + return a + b; + #endif + } + + /* Vector subtraction */ + PSIMD_INTRINSIC psimd_s8 psimd_sub_s8(psimd_s8 a, psimd_s8 b) { + return a - b; + } + + PSIMD_INTRINSIC psimd_u8 psimd_sub_u8(psimd_u8 a, psimd_u8 b) { + return a - b; + } + + PSIMD_INTRINSIC psimd_s16 psimd_sub_s16(psimd_s16 a, psimd_s16 b) { + return a - b; + } + + PSIMD_INTRINSIC psimd_u16 psimd_sub_u16(psimd_u16 a, psimd_u16 b) { + return a - b; + } + + PSIMD_INTRINSIC psimd_s32 psimd_sub_s32(psimd_s32 a, psimd_s32 b) { + return a - b; + } + + PSIMD_INTRINSIC psimd_u32 psimd_sub_u32(psimd_u32 a, psimd_u32 b) { + return a - b; + } + + PSIMD_INTRINSIC psimd_f32 psimd_sub_f32(psimd_f32 a, psimd_f32 b) { + #if defined(__ARM_ARCH_7A__) && defined(__ARM_NEON__) && !defined(__FAST_MATH__) + return (psimd_f32) vsubq_f32((float32x4_t) a, (float32x4_t) b); + #else + return a - b; + #endif + } + + /* Vector multiplication */ + PSIMD_INTRINSIC psimd_s8 psimd_mul_s8(psimd_s8 a, psimd_s8 b) { + return a * b; + } + + PSIMD_INTRINSIC psimd_u8 psimd_mul_u8(psimd_u8 a, psimd_u8 b) { + return a * b; + } + + PSIMD_INTRINSIC psimd_s16 psimd_mul_s16(psimd_s16 a, psimd_s16 b) { + return a * b; + } + + PSIMD_INTRINSIC psimd_u16 psimd_mul_u16(psimd_u16 a, psimd_u16 b) { + return a * b; + } + + PSIMD_INTRINSIC psimd_s32 psimd_mul_s32(psimd_s32 a, psimd_s32 b) { + return a * b; + } + + PSIMD_INTRINSIC psimd_u32 psimd_mul_u32(psimd_u32 a, psimd_u32 b) { + return a * b; + } + + PSIMD_INTRINSIC psimd_f32 psimd_mul_f32(psimd_f32 a, psimd_f32 b) { + #if defined(__ARM_ARCH_7A__) && defined(__ARM_NEON__) && !defined(__FAST_MATH__) + return (psimd_f32) vmulq_f32((float32x4_t) a, (float32x4_t) b); + #else + return a * b; + #endif + } + + /* Quasi-Fused Multiply-Add */ + PSIMD_INTRINSIC psimd_f32 psimd_qfma_f32(psimd_f32 a, psimd_f32 b, psimd_f32 c) { + #if defined(__aarch64__) || defined(__ARM_NEON__) && defined(__ARM_FEATURE_FMA) + return (psimd_f32) vfmaq_f32((float32x4_t) a, (float32x4_t) b, (float32x4_t) c); + #elif (defined(__x86_64__) || defined(__i386__) || defined(__i686__)) && defined(__FMA__) + return (psimd_f32) _mm_fmadd_ps((__m128) b, (__m128) c, (__m128) a); + #elif (defined(__x86_64__) || defined(__i386__) || defined(__i686__)) && defined(__FMA4__) + return (psimd_f32) _mm_macc_ps((__m128) b, (__m128) c, (__m128) a); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) && PSIMD_ENABLE_WASM_QFMA + return (psimd_f32) __builtin_wasm_qfma_f32x4(a, b, c); + #else + return a + b * c; + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_div_f32(psimd_f32 a, psimd_f32 b) { + return a / b; + } + + /* Vector and */ + PSIMD_INTRINSIC psimd_f32 psimd_andmask_f32(psimd_s32 mask, psimd_f32 v) { + return (psimd_f32) (mask & (psimd_s32) v); + } + + /* Vector and-not */ + PSIMD_INTRINSIC psimd_f32 psimd_andnotmask_f32(psimd_s32 mask, psimd_f32 v) { + return (psimd_f32) (~mask & (psimd_s32) v); + } + + /* Vector blend */ + PSIMD_INTRINSIC psimd_s8 psimd_blend_s8(psimd_s8 mask, psimd_s8 a, psimd_s8 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s8) vbslq_s8((uint8x16_t) mask, (int8x16_t) a, (int8x16_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_s8) __builtin_wasm_bitselect(a, b, mask); + #else + return (mask & a) | (~mask & b); + #endif + } + + PSIMD_INTRINSIC psimd_u8 psimd_blend_u8(psimd_s8 mask, psimd_u8 a, psimd_u8 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u8) vbslq_u8((uint8x16_t) mask, (uint8x16_t) a, (uint8x16_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_u8) __builtin_wasm_bitselect(a, b, mask); + #else + return (psimd_u8) ((mask & (psimd_s8) a) | (~mask & (psimd_s8) b)); + #endif + } + + PSIMD_INTRINSIC psimd_s16 psimd_blend_s16(psimd_s16 mask, psimd_s16 a, psimd_s16 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s16) vbslq_s16((uint16x8_t) mask, (int16x8_t) a, (int16x8_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_s16) __builtin_wasm_bitselect(a, b, mask); + #else + return (mask & a) | (~mask & b); + #endif + } + + PSIMD_INTRINSIC psimd_u16 psimd_blend_u16(psimd_s16 mask, psimd_u16 a, psimd_u16 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u16) vbslq_u16((uint16x8_t) mask, (uint16x8_t) a, (uint16x8_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_u16) __builtin_wasm_bitselect(a, b, mask); + #else + return (psimd_u16) ((mask & (psimd_s16) a) | (~mask & (psimd_s16) b)); + #endif + } + + PSIMD_INTRINSIC psimd_s32 psimd_blend_s32(psimd_s32 mask, psimd_s32 a, psimd_s32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s32) vbslq_s32((uint32x4_t) mask, (int32x4_t) a, (int32x4_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_s32) __builtin_wasm_bitselect(a, b, mask); + #else + return (mask & a) | (~mask & b); + #endif + } + + PSIMD_INTRINSIC psimd_u32 psimd_blend_u32(psimd_s32 mask, psimd_u32 a, psimd_u32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u32) vbslq_u32((uint32x4_t) mask, (uint32x4_t) a, (uint32x4_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_u32) __builtin_wasm_bitselect(a, b, mask); + #else + return (psimd_u32) ((mask & (psimd_s32) a) | (~mask & (psimd_s32) b)); + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_blend_f32(psimd_s32 mask, psimd_f32 a, psimd_f32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_f32) vbslq_f32((uint32x4_t) mask, (float32x4_t) a, (float32x4_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return (psimd_f32) __builtin_wasm_bitselect(a, b, mask); + #else + return (psimd_f32) ((mask & (psimd_s32) a) | (~mask & (psimd_s32) b)); + #endif + } + + /* Vector blend on sign */ + PSIMD_INTRINSIC psimd_s8 psimd_signblend_s8(psimd_s8 x, psimd_s8 a, psimd_s8 b) { + return psimd_blend_s8(x >> psimd_splat_s8(7), a, b); + } + + PSIMD_INTRINSIC psimd_u8 psimd_signblend_u8(psimd_s8 x, psimd_u8 a, psimd_u8 b) { + return psimd_blend_u8((x >> psimd_splat_s8(7)), a, b); + } + + PSIMD_INTRINSIC psimd_s16 psimd_signblend_s16(psimd_s16 x, psimd_s16 a, psimd_s16 b) { + return psimd_blend_s16(x >> psimd_splat_s16(15), a, b); + } + + PSIMD_INTRINSIC psimd_u16 psimd_signblend_u16(psimd_s16 x, psimd_u16 a, psimd_u16 b) { + return psimd_blend_u16((x >> psimd_splat_s16(15)), a, b); + } + + PSIMD_INTRINSIC psimd_s32 psimd_signblend_s32(psimd_s32 x, psimd_s32 a, psimd_s32 b) { + return psimd_blend_s32(x >> psimd_splat_s32(31), a, b); + } + + PSIMD_INTRINSIC psimd_u32 psimd_signblend_u32(psimd_s32 x, psimd_u32 a, psimd_u32 b) { + return psimd_blend_u32((x >> psimd_splat_s32(31)), a, b); + } + + PSIMD_INTRINSIC psimd_f32 psimd_signblend_f32(psimd_f32 x, psimd_f32 a, psimd_f32 b) { + const psimd_s32 mask = (psimd_s32) x >> psimd_splat_s32(31); + return psimd_blend_f32(mask, a, b); + } + + /* Vector absolute value */ + PSIMD_INTRINSIC psimd_f32 psimd_abs_f32(psimd_f32 v) { + const psimd_s32 mask = (psimd_s32) psimd_splat_f32(-0.0f); + return (psimd_f32) ((psimd_s32) v & ~mask); + } + + /* Vector negation */ + PSIMD_INTRINSIC psimd_f32 psimd_neg_f32(psimd_f32 v) { + const psimd_s32 mask = (psimd_s32) psimd_splat_f32(-0.0f); + return (psimd_f32) ((psimd_s32) v ^ mask); + } + + /* Vector maximum */ + PSIMD_INTRINSIC psimd_s8 psimd_max_s8(psimd_s8 a, psimd_s8 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s8) vmaxq_s8((int8x16_t) a, (int8x16_t) b); + #else + return psimd_blend_s8(a > b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_u8 psimd_max_u8(psimd_u8 a, psimd_u8 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u8) vmaxq_u8((uint8x16_t) a, (uint8x16_t) b); + #else + return psimd_blend_u8(a > b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_s16 psimd_max_s16(psimd_s16 a, psimd_s16 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s16) vmaxq_s16((int16x8_t) a, (int16x8_t) b); + #else + return psimd_blend_s16(a > b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_u16 psimd_max_u16(psimd_u16 a, psimd_u16 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u16) vmaxq_u16((uint16x8_t) a, (uint16x8_t) b); + #else + return psimd_blend_u16(a > b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_s32 psimd_max_s32(psimd_s32 a, psimd_s32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s32) vmaxq_s32((int32x4_t) a, (int32x4_t) b); + #else + return psimd_blend_s32(a > b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_u32 psimd_max_u32(psimd_u32 a, psimd_u32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u32) vmaxq_u32((uint32x4_t) a, (uint32x4_t) b); + #else + return psimd_blend_u32(a > b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_max_f32(psimd_f32 a, psimd_f32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_f32) vmaxq_f32((float32x4_t) a, (float32x4_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return __builtin_wasm_max_f32x4(a, b); + #else + return psimd_blend_f32(a > b, a, b); + #endif + } + + /* Vector minimum */ + PSIMD_INTRINSIC psimd_s8 psimd_min_s8(psimd_s8 a, psimd_s8 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s8) vminq_s8((int8x16_t) a, (int8x16_t) b); + #else + return psimd_blend_s8(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_u8 psimd_min_u8(psimd_u8 a, psimd_u8 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u8) vminq_u8((uint8x16_t) a, (uint8x16_t) b); + #else + return psimd_blend_u8(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_s16 psimd_min_s16(psimd_s16 a, psimd_s16 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s16) vminq_s16((int16x8_t) a, (int16x8_t) b); + #else + return psimd_blend_s16(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_u16 psimd_min_u16(psimd_u16 a, psimd_u16 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u16) vminq_u16((uint16x8_t) a, (uint16x8_t) b); + #else + return psimd_blend_u16(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_s32 psimd_min_s32(psimd_s32 a, psimd_s32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_s32) vminq_s32((int32x4_t) a, (int32x4_t) b); + #else + return psimd_blend_s32(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_u32 psimd_min_u32(psimd_u32 a, psimd_u32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_u32) vminq_u32((uint32x4_t) a, (uint32x4_t) b); + #else + return psimd_blend_u32(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_min_f32(psimd_f32 a, psimd_f32 b) { + #if defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_f32) vminq_f32((float32x4_t) a, (float32x4_t) b); + #elif defined(__wasm__) && defined(__wasm_simd128__) && defined(__clang__) + return __builtin_wasm_min_f32x4(a, b); + #else + return psimd_blend_f32(a < b, a, b); + #endif + } + + PSIMD_INTRINSIC psimd_f32 psimd_cvt_s32_f32(psimd_s32 v) { + #if defined(__clang__) + return __builtin_convertvector(v, psimd_f32); + #elif defined(__ARM_NEON__) || defined(__ARM_NEON) + return (psimd_f32) vcvtq_f32_s32((int32x4_t) v); + #elif defined(__SSE2__) + return (psimd_f32) _mm_cvtepi32_ps((__m128i) v); + #else + return (psimd_f32) { (float) v[0], (float) v[1], (float) v[2], (float) v[3] }; + #endif + } + + /* Broadcast vector element */ + #if defined(__clang__) + PSIMD_INTRINSIC psimd_f32 psimd_splat0_f32(psimd_f32 v) { + return __builtin_shufflevector(v, v, 0, 0, 0, 0); + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat1_f32(psimd_f32 v) { + return __builtin_shufflevector(v, v, 1, 1, 1, 1); + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat2_f32(psimd_f32 v) { + return __builtin_shufflevector(v, v, 2, 2, 2, 2); + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat3_f32(psimd_f32 v) { + return __builtin_shufflevector(v, v, 3, 3, 3, 3); + } + #else + PSIMD_INTRINSIC psimd_f32 psimd_splat0_f32(psimd_f32 v) { + return __builtin_shuffle(v, (psimd_s32) { 0, 0, 0, 0 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat1_f32(psimd_f32 v) { + return __builtin_shuffle(v, (psimd_s32) { 1, 1, 1, 1 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat2_f32(psimd_f32 v) { + return __builtin_shuffle(v, (psimd_s32) { 2, 2, 2, 2 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_splat3_f32(psimd_f32 v) { + return __builtin_shuffle(v, (psimd_s32) { 3, 3, 3, 3 }); + } + #endif + + /* Reversal of vector elements */ + #if defined(__clang__) + PSIMD_INTRINSIC psimd_s8 psimd_reverse_s8(psimd_s8 v) { + return __builtin_shufflevector(v, v, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + } + + PSIMD_INTRINSIC psimd_u8 psimd_reverse_u8(psimd_u8 v) { + return __builtin_shufflevector(v, v, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0); + } + + PSIMD_INTRINSIC psimd_s16 psimd_reverse_s16(psimd_s16 v) { + return __builtin_shufflevector(v, v, 7, 6, 5, 4, 3, 2, 1, 0); + } + + PSIMD_INTRINSIC psimd_u16 psimd_reverse_u16(psimd_u16 v) { + return __builtin_shufflevector(v, v, 7, 6, 5, 4, 3, 2, 1, 0); + } + + PSIMD_INTRINSIC psimd_s32 psimd_reverse_s32(psimd_s32 v) { + return __builtin_shufflevector(v, v, 3, 2, 1, 0); + } + + PSIMD_INTRINSIC psimd_u32 psimd_reverse_u32(psimd_u32 v) { + return __builtin_shufflevector(v, v, 3, 2, 1, 0); + } + + PSIMD_INTRINSIC psimd_f32 psimd_reverse_f32(psimd_f32 v) { + return __builtin_shufflevector(v, v, 3, 2, 1, 0); + } + #else + PSIMD_INTRINSIC psimd_s8 psimd_reverse_s8(psimd_s8 v) { + return __builtin_shuffle(v, (psimd_s8) { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 }); + } + + PSIMD_INTRINSIC psimd_u8 psimd_reverse_u8(psimd_u8 v) { + return __builtin_shuffle(v, (psimd_s8) { 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 }); + } + + PSIMD_INTRINSIC psimd_s16 psimd_reverse_s16(psimd_s16 v) { + return __builtin_shuffle(v, (psimd_s16) { 7, 6, 5, 4, 3, 2, 1, 0 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_reverse_u16(psimd_u16 v) { + return __builtin_shuffle(v, (psimd_s16) { 7, 6, 5, 4, 3, 2, 1, 0 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_reverse_s32(psimd_s32 v) { + return __builtin_shuffle(v, (psimd_s32) { 3, 2, 1, 0 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_reverse_u32(psimd_u32 v) { + return __builtin_shuffle(v, (psimd_s32) { 3, 2, 1, 0 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_reverse_f32(psimd_f32 v) { + return __builtin_shuffle(v, (psimd_s32) { 3, 2, 1, 0 }); + } + #endif + + /* Interleaving of vector elements */ + #if defined(__clang__) + PSIMD_INTRINSIC psimd_s16 psimd_interleave_lo_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shufflevector(a, b, 0, 8+0, 1, 8+1, 2, 8+2, 3, 8+3); + } + + PSIMD_INTRINSIC psimd_s16 psimd_interleave_hi_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shufflevector(a, b, 4, 8+4, 5, 8+5, 6, 8+6, 7, 8+7); + } + + PSIMD_INTRINSIC psimd_u16 psimd_interleave_lo_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shufflevector(a, b, 0, 8+0, 1, 8+1, 2, 8+2, 3, 8+3); + } + + PSIMD_INTRINSIC psimd_u16 psimd_interleave_hi_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shufflevector(a, b, 4, 8+4, 5, 8+5, 6, 8+6, 7, 8+7); + } + + PSIMD_INTRINSIC psimd_s32 psimd_interleave_lo_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shufflevector(a, b, 0, 4+0, 1, 4+1); + } + + PSIMD_INTRINSIC psimd_s32 psimd_interleave_hi_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shufflevector(a, b, 2, 4+2, 3, 4+3); + } + + PSIMD_INTRINSIC psimd_u32 psimd_interleave_lo_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shufflevector(a, b, 0, 4+0, 1, 4+1); + } + + PSIMD_INTRINSIC psimd_u32 psimd_interleave_hi_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shufflevector(a, b, 2, 4+2, 3, 4+3); + } + + PSIMD_INTRINSIC psimd_f32 psimd_interleave_lo_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shufflevector(a, b, 0, 4+0, 1, 4+1); + } + + PSIMD_INTRINSIC psimd_f32 psimd_interleave_hi_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shufflevector(a, b, 2, 4+2, 3, 4+3); + } + #else + PSIMD_INTRINSIC psimd_s16 psimd_interleave_lo_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 0, 8+0, 1, 8+1, 2, 8+2, 3, 8+3 }); + } + + PSIMD_INTRINSIC psimd_s16 psimd_interleave_hi_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 4, 8+4, 5, 8+5, 6, 8+6, 7, 8+7 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_interleave_lo_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 0, 8+0, 1, 8+1, 2, 8+2, 3, 8+3 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_interleave_hi_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 4, 8+4, 5, 8+5, 6, 8+6, 7, 8+7 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_interleave_lo_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 4+0, 1, 4+1 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_interleave_hi_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 2, 4+2, 3, 4+3 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_interleave_lo_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 4+0, 1, 4+1 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_interleave_hi_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 2, 4+2, 3, 4+3 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_interleave_lo_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 4+0, 1, 4+1 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_interleave_hi_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 2, 4+2, 3, 4+3 }); + } + #endif + + /* Concatenation of low/high vector elements */ + #if defined(__clang__) + PSIMD_INTRINSIC psimd_s16 psimd_concat_lo_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shufflevector(a, b, 0, 1, 2, 3, 8+0, 8+1, 8+2, 8+3); + } + + PSIMD_INTRINSIC psimd_s16 psimd_concat_hi_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shufflevector(a, b, 4, 5, 6, 7, 8+4, 8+5, 8+6, 8+7); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_lo_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shufflevector(a, b, 0, 1, 2, 3, 8+0, 8+1, 8+2, 8+3); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_hi_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shufflevector(a, b, 4, 5, 6, 7, 8+4, 8+5, 8+6, 8+7); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_lo_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shufflevector(a, b, 0, 1, 4+0, 4+1); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_hi_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shufflevector(a, b, 2, 3, 4+2, 4+3); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_lo_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shufflevector(a, b, 0, 1, 4+0, 4+1); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_hi_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shufflevector(a, b, 2, 3, 4+2, 4+3); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_lo_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shufflevector(a, b, 0, 1, 4+0, 4+1); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_hi_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shufflevector(a, b, 2, 3, 4+2, 4+3); + } + #else + PSIMD_INTRINSIC psimd_s16 psimd_concat_lo_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 0, 1, 2, 3, 8+0, 8+1, 8+2, 8+3 }); + } + + PSIMD_INTRINSIC psimd_s16 psimd_concat_hi_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 4, 5, 6, 7, 8+4, 8+5, 8+6, 8+7 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_lo_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 0, 1, 2, 3, 8+0, 8+1, 8+2, 8+3 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_hi_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 4, 5, 6, 7, 8+4, 8+5, 8+6, 8+7 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_lo_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 1, 4+0, 4+1 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_hi_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 2, 3, 4+2, 4+3 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_lo_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 1, 4+0, 4+1 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_hi_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 2, 3, 4+2, 4+3 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_lo_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 1, 4+0, 4+1 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_hi_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 2, 3, 4+2, 4+3 }); + } + #endif + + /* Concatenation of even/odd vector elements */ + #if defined(__clang__) + PSIMD_INTRINSIC psimd_s8 psimd_concat_even_s8(psimd_s8 a, psimd_s8 b) { + return __builtin_shufflevector(a, b, + 0, 2, 4, 6, 8, 10, 12, 14, 16+0, 16+2, 16+4, 16+6, 16+8, 16+10, 16+12, 16+14); + } + + PSIMD_INTRINSIC psimd_s8 psimd_concat_odd_s8(psimd_s8 a, psimd_s8 b) { + return __builtin_shufflevector(a, b, + 1, 3, 5, 7, 9, 11, 13, 15, 16+1, 16+3, 16+5, 16+7, 16+9, 16+11, 16+13, 16+15); + } + + PSIMD_INTRINSIC psimd_u8 psimd_concat_even_u8(psimd_u8 a, psimd_u8 b) { + return __builtin_shufflevector(a, b, + 0, 2, 4, 6, 8, 10, 12, 14, 16+0, 16+2, 16+4, 16+6, 16+8, 16+10, 16+12, 16+14); + } + + PSIMD_INTRINSIC psimd_u8 psimd_concat_odd_u8(psimd_u8 a, psimd_u8 b) { + return __builtin_shufflevector(a, b, + 1, 3, 5, 7, 9, 11, 13, 15, 16+1, 16+3, 16+5, 16+7, 16+9, 16+11, 16+13, 16+15); + } + + PSIMD_INTRINSIC psimd_s16 psimd_concat_even_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shufflevector(a, b, 0, 2, 4, 6, 8+0, 8+2, 8+4, 8+6); + } + + PSIMD_INTRINSIC psimd_s16 psimd_concat_odd_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shufflevector(a, b, 1, 3, 5, 7, 8+1, 8+3, 8+5, 8+7); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_even_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shufflevector(a, b, 0, 2, 4, 6, 8+0, 8+2, 8+4, 8+6); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_odd_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shufflevector(a, b, 1, 3, 5, 7, 8+1, 8+3, 8+5, 8+7); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_even_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shufflevector(a, b, 0, 2, 4+0, 4+2); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_odd_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shufflevector(a, b, 1, 3, 4+1, 4+3); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_even_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shufflevector(a, b, 0, 2, 4+0, 4+2); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_odd_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shufflevector(a, b, 1, 3, 4+1, 4+3); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_even_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shufflevector(a, b, 0, 2, 4+0, 4+2); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_odd_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shufflevector(a, b, 1, 3, 4+1, 4+3); + } + #else + PSIMD_INTRINSIC psimd_s8 psimd_concat_even_s8(psimd_s8 a, psimd_s8 b) { + return __builtin_shuffle(a, b, + (psimd_s8) { 0, 2, 4, 6, 8, 10, 12, 14, 16+0, 16+2, 16+4, 16+6, 16+8, 16+10, 16+12, 16+14 }); + } + + PSIMD_INTRINSIC psimd_s8 psimd_concat_odd_s8(psimd_s8 a, psimd_s8 b) { + return __builtin_shuffle(a, b, + (psimd_s8) { 1, 3, 5, 7, 9, 11, 13, 15, 16+1, 16+3, 16+5, 16+7, 16+9, 16+11, 16+13, 16+15 }); + } + + PSIMD_INTRINSIC psimd_u8 psimd_concat_even_u8(psimd_u8 a, psimd_u8 b) { + return __builtin_shuffle(a, b, + (psimd_s8) { 0, 2, 4, 6, 8, 10, 12, 14, 16+0, 16+2, 16+4, 16+6, 16+8, 16+10, 16+12, 16+14 }); + } + + PSIMD_INTRINSIC psimd_u8 psimd_concat_odd_u8(psimd_u8 a, psimd_u8 b) { + return __builtin_shuffle(a, b, + (psimd_s8) { 1, 3, 5, 7, 9, 11, 13, 15, 16+1, 16+3, 16+5, 16+7, 16+9, 16+11, 16+13, 16+15 }); + } + + PSIMD_INTRINSIC psimd_s16 psimd_concat_even_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 0, 2, 4, 6, 8+0, 8+2, 8+4, 8+6 }); + } + + PSIMD_INTRINSIC psimd_s16 psimd_concat_odd_s16(psimd_s16 a, psimd_s16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 1, 3, 5, 7, 8+1, 8+3, 8+5, 8+7 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_even_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 0, 2, 4, 6, 8+0, 8+2, 8+4, 8+6 }); + } + + PSIMD_INTRINSIC psimd_u16 psimd_concat_odd_u16(psimd_u16 a, psimd_u16 b) { + return __builtin_shuffle(a, b, (psimd_s16) { 1, 3, 5, 7, 8+1, 8+3, 8+5, 8+7 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_even_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 2, 4+0, 4+2 }); + } + + PSIMD_INTRINSIC psimd_s32 psimd_concat_odd_s32(psimd_s32 a, psimd_s32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 1, 3, 4+1, 4+3 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_even_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 2, 4+0, 4+2 }); + } + + PSIMD_INTRINSIC psimd_u32 psimd_concat_odd_u32(psimd_u32 a, psimd_u32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 1, 3, 4+1, 4+3 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_even_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 0, 2, 4+0, 4+2 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_concat_odd_f32(psimd_f32 a, psimd_f32 b) { + return __builtin_shuffle(a, b, (psimd_s32) { 1, 3, 4+1, 4+3 }); + } + #endif + + /* Vector reduce */ + #if defined(__clang__) + PSIMD_INTRINSIC psimd_f32 psimd_allreduce_sum_f32(psimd_f32 v) { + const psimd_f32 temp = v + __builtin_shufflevector(v, v, 2, 3, 0, 1); + return temp + __builtin_shufflevector(temp, temp, 1, 0, 3, 2); + } + + PSIMD_INTRINSIC psimd_f32 psimd_allreduce_max_f32(psimd_f32 v) { + const psimd_f32 temp = psimd_max_f32(v, __builtin_shufflevector(v, v, 2, 3, 0, 1)); + return psimd_max_f32(temp, __builtin_shufflevector(temp, temp, 1, 0, 3, 2)); + } + + PSIMD_INTRINSIC psimd_f32 psimd_allreduce_min_f32(psimd_f32 v) { + const psimd_f32 temp = psimd_min_f32(v, __builtin_shufflevector(v, v, 2, 3, 0, 1)); + return psimd_min_f32(temp, __builtin_shufflevector(temp, temp, 1, 0, 3, 2)); + } + + PSIMD_INTRINSIC float psimd_reduce_sum_f32(psimd_f32 v) { + const psimd_f32 temp = v + __builtin_shufflevector(v, v, 2, 3, -1, -1); + const psimd_f32 result = temp + __builtin_shufflevector(temp, temp, 1, -1, -1, -1); + return result[0]; + } + + PSIMD_INTRINSIC float psimd_reduce_max_f32(psimd_f32 v) { + const psimd_f32 temp = psimd_max_f32(v, __builtin_shufflevector(v, v, 2, 3, -1, -1)); + const psimd_f32 result = psimd_max_f32(temp, __builtin_shufflevector(temp, temp, 1, -1, -1, -1)); + return result[0]; + } + + PSIMD_INTRINSIC float psimd_reduce_min_f32(psimd_f32 v) { + const psimd_f32 temp = psimd_min_f32(v, __builtin_shufflevector(v, v, 2, 3, -1, -1)); + const psimd_f32 result = psimd_min_f32(temp, __builtin_shufflevector(temp, temp, 1, -1, -1, -1)); + return result[0]; + } + #else + PSIMD_INTRINSIC psimd_f32 psimd_allreduce_sum_f32(psimd_f32 v) { + const psimd_f32 temp = v + __builtin_shuffle(v, (psimd_s32) { 2, 3, 0, 1 }); + return temp + __builtin_shuffle(temp, (psimd_s32) { 1, 0, 3, 2 }); + } + + PSIMD_INTRINSIC psimd_f32 psimd_allreduce_max_f32(psimd_f32 v) { + const psimd_f32 temp = psimd_max_f32(v, __builtin_shuffle(v, (psimd_s32) { 2, 3, 0, 1 })); + return psimd_max_f32(temp, __builtin_shuffle(temp, (psimd_s32) { 1, 0, 3, 2 })); + } + + PSIMD_INTRINSIC psimd_f32 psimd_allreduce_min_f32(psimd_f32 v) { + const psimd_f32 temp = psimd_min_f32(v, __builtin_shuffle(v, (psimd_s32) { 2, 3, 0, 1 })); + return psimd_min_f32(temp, __builtin_shuffle(temp, (psimd_s32) { 1, 0, 3, 2 })); + } + + PSIMD_INTRINSIC float psimd_reduce_sum_f32(psimd_f32 v) { + const psimd_f32 result = psimd_allreduce_sum_f32(v); + return result[0]; + } + + PSIMD_INTRINSIC float psimd_reduce_max_f32(psimd_f32 v) { + const psimd_f32 result = psimd_allreduce_max_f32(v); + return result[0]; + } + + PSIMD_INTRINSIC float psimd_reduce_min_f32(psimd_f32 v) { + const psimd_f32 result = psimd_allreduce_min_f32(v); + return result[0]; + } + #endif +#endif + +#endif /* PSIMD_H */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/pthreadpool.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/pthreadpool.h new file mode 100644 index 0000000000000000000000000000000000000000..5cab9f53220467ab7f0c68a49b9c24badb0217c6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/pthreadpool.h @@ -0,0 +1,2560 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#ifndef PTHREADPOOL_H_ +#define PTHREADPOOL_H_ + +#include +#include + +typedef struct pthreadpool* pthreadpool_t; + +typedef void (*pthreadpool_task_1d_t)(void*, size_t); +typedef void (*pthreadpool_task_1d_with_thread_t)(void*, size_t, size_t); +typedef void (*pthreadpool_task_1d_tile_1d_t)(void*, size_t, size_t); +typedef void (*pthreadpool_task_2d_t)(void*, size_t, size_t); +typedef void (*pthreadpool_task_2d_with_thread_t)(void*, size_t, size_t, size_t); +typedef void (*pthreadpool_task_2d_tile_1d_t)(void*, size_t, size_t, size_t); +typedef void (*pthreadpool_task_2d_tile_2d_t)(void*, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_t)(void*, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_tile_1d_t)(void*, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_tile_1d_with_thread_t)(void*, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_4d_t)(void*, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_4d_tile_1d_t)(void*, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_4d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_5d_t)(void*, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_5d_tile_1d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_5d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_6d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_6d_tile_1d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_6d_tile_2d_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t, size_t, size_t); + +typedef void (*pthreadpool_task_1d_with_id_t)(void*, uint32_t, size_t); +typedef void (*pthreadpool_task_2d_tile_1d_with_id_t)(void*, uint32_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_2d_tile_2d_with_id_t)(void*, uint32_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_tile_1d_with_id_t)(void*, uint32_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_tile_2d_with_id_t)(void*, uint32_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_4d_tile_2d_with_id_t)(void*, uint32_t, size_t, size_t, size_t, size_t, size_t, size_t); + +typedef void (*pthreadpool_task_2d_tile_1d_with_id_with_thread_t)(void*, uint32_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_task_3d_tile_1d_with_id_with_thread_t)(void*, uint32_t, size_t, size_t, size_t, size_t, size_t); + + +/** + * Disable support for denormalized numbers to the maximum extent possible for + * the duration of the computation. + * + * Handling denormalized floating-point numbers is often implemented in + * microcode, and incurs significant performance degradation. This hint + * instructs the thread pool to disable support for denormalized numbers before + * running the computation by manipulating architecture-specific control + * registers, and restore the initial value of control registers after the + * computation is complete. The thread pool temporary disables denormalized + * numbers on all threads involved in the computation (i.e. the caller threads, + * and potentially worker threads). + * + * Disabling denormalized numbers may have a small negative effect on results' + * accuracy. As various architectures differ in capabilities to control + * processing of denormalized numbers, using this flag may also hurt results' + * reproducibility across different instruction set architectures. + */ +#define PTHREADPOOL_FLAG_DISABLE_DENORMALS 0x00000001 + +/** + * Yield worker threads to the system scheduler after the operation is finished. + * + * Force workers to use kernel wait (instead of active spin-wait by default) for + * new commands after this command is processed. This flag affects only the + * immediate next operation on this thread pool. To make the thread pool always + * use kernel wait, pass this flag to all parallelization functions. + */ +#define PTHREADPOOL_FLAG_YIELD_WORKERS 0x00000002 + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Create a thread pool with the specified number of threads. + * + * @param threads_count the number of threads in the thread pool. + * A value of 0 has special interpretation: it creates a thread pool with as + * many threads as there are logical processors in the system. + * + * @returns A pointer to an opaque thread pool object if the call is + * successful, or NULL pointer if the call failed. + */ +pthreadpool_t pthreadpool_create(size_t threads_count); + +/** + * Query the number of threads in a thread pool. + * + * @param threadpool the thread pool to query. + * + * @returns The number of threads in the thread pool. + */ +size_t pthreadpool_get_threads_count(pthreadpool_t threadpool); + +/** + * Process items on a 1D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range; i++) + * function(context, i); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each item. + * @param context the first argument passed to the specified function. + * @param range the number of items on the 1D grid to process. The + * specified function will be called once for each item. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_1d( + pthreadpool_t threadpool, + pthreadpool_task_1d_t function, + void* context, + size_t range, + uint32_t flags); + +/** + * Process items on a 1D grid passing along the current thread id. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range; i++) + * function(context, thread_index, i); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each item. + * @param context the first argument passed to the specified function. + * @param range the number of items on the 1D grid to process. The + * specified function will be called once for each item. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_1d_with_thread( + pthreadpool_t threadpool, + pthreadpool_task_1d_with_thread_t function, + void* context, + size_t range, + uint32_t flags); + +/** + * Process items on a 1D grid using a microarchitecture-aware task function. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range; i++) + * function(context, uarch_index, i); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If + * threadpool is NULL, all items are processed serially on the calling + * thread. + * @param function the function to call for each item. + * @param context the first argument passed to the specified + * function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range the number of items on the 1D grid to process. + * The specified function will be called once for each item. + * @param flags a bitwise combination of zero or more optional + * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or + * PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_1d_with_uarch( + pthreadpool_t threadpool, + pthreadpool_task_1d_with_id_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range, + uint32_t flags); + +/** + * Process items on a 1D grid with specified maximum tile size. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range; i += tile) + * function(context, i, min(range - i, tile)); + * + * When the call returns, all items have been processed and the thread pool is + * ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, + * the calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range the number of items on the 1D grid to process. + * @param tile the maximum number of items on the 1D grid to process in + * one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_1d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_1d_tile_1d_t function, + void* context, + size_t range, + size_t tile, + uint32_t flags); + +/** + * Process items on a 2D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * function(context, i, j); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each item. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d( + pthreadpool_t threadpool, + pthreadpool_task_2d_t function, + void* context, + size_t range_i, + size_t range_j, + uint32_t flags); + +/** + * Process items on a 2D grid passing along the current thread id. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * function(context, thread_index, i, j); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each item. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d_with_thread( + pthreadpool_t threadpool, + pthreadpool_task_2d_with_thread_t function, + void* context, + size_t range_i, + size_t range_j, + uint32_t flags); + +/** + * Process items on a 2D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * function(context, i, j, min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param tile_j the maximum number of items along the second dimension of + * the 2D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_2d_tile_1d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t tile_j, + uint32_t flags); + +/** + * Process items on a 2D grid with the specified maximum tile size along the + * last grid dimension using a microarchitecture-aware task function. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * function(context, uarch_index, i, j, min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param tile_j the maximum number of items along the second dimension of + * the 2D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d_tile_1d_with_uarch( + pthreadpool_t threadpool, + pthreadpool_task_2d_tile_1d_with_id_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t tile_j, + uint32_t flags); + +/** + * Process items on a 2D grid with the specified maximum tile size along the + * last grid dimension using a microarchitecture-aware task function and passing + * along the current thread id. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * function(context, uarch_index, thread_index, i, j, min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param tile_j the maximum number of items along the second dimension of + * the 2D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d_tile_1d_with_uarch_with_thread( + pthreadpool_t threadpool, + pthreadpool_task_2d_tile_1d_with_id_with_thread_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t tile_j, + uint32_t flags); + +/** + * Process items on a 2D grid with the specified maximum tile size along each + * grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i += tile_i) + * for (size_t j = 0; j < range_j; j += tile_j) + * function(context, i, j, + * min(range_i - i, tile_i), min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param tile_j the maximum number of items along the first dimension of + * the 2D grid to process in one function call. + * @param tile_j the maximum number of items along the second dimension of + * the 2D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d_tile_2d( + pthreadpool_t threadpool, + pthreadpool_task_2d_tile_2d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t tile_i, + size_t tile_j, + uint32_t flags); + +/** + * Process items on a 2D grid with the specified maximum tile size along each + * grid dimension using a microarchitecture-aware task function. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i += tile_i) + * for (size_t j = 0; j < range_j; j += tile_j) + * function(context, uarch_index, i, j, + * min(range_i - i, tile_i), min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If + * threadpool is NULL, all items are processed serially on the calling + * thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified + * function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, + * cpuinfo initialization failed, or index returned + * by cpuinfo_get_current_uarch_index() exceeds + * the max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected + * by the specified function. If the index returned + * by cpuinfo_get_current_uarch_index() exceeds this + * value, default_uarch_index will be used instead. + * default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first + * dimension of the 2D grid. + * @param range_j the number of items to process along the second + * dimension of the 2D grid. + * @param tile_j the maximum number of items along the first + * dimension of the 2D grid to process in one function call. + * @param tile_j the maximum number of items along the second + * dimension of the 2D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional + * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or + * PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_2d_tile_2d_with_uarch( + pthreadpool_t threadpool, + pthreadpool_task_2d_tile_2d_with_id_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t tile_i, + size_t tile_j, + uint32_t flags); + +/** + * Process items on a 3D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * function(context, i, j, k); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d( + pthreadpool_t threadpool, + pthreadpool_task_3d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + uint32_t flags); + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * function(context, i, j, k, min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param tile_k the maximum number of items along the third dimension of + * the 3D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_1d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_k, + uint32_t flags); + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last grid dimension and passing along the current thread id. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * function(context, thread_index, i, j, k, min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param tile_k the maximum number of items along the third dimension of + * the 3D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d_tile_1d_with_thread( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_1d_with_thread_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_k, + uint32_t flags); + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last grid dimension using a microarchitecture-aware task function. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * function(context, uarch_index, i, j, k, min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If + * threadpool is NULL, all items are processed serially on the calling + * thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified + * function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first + * dimension of the 3D grid. + * @param range_j the number of items to process along the second + * dimension of the 3D grid. + * @param range_k the number of items to process along the third + * dimension of the 3D grid. + * @param tile_k the maximum number of items along the third + * dimension of the 3D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional + * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or + * PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d_tile_1d_with_uarch( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_1d_with_id_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_k, + uint32_t flags); + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last grid dimension using a microarchitecture-aware task function and passing + * along the current thread id. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * function(context, uarch_index, thread_index, i, j, k, min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If + * threadpool is NULL, all items are processed serially on the calling + * thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified + * function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first + * dimension of the 3D grid. + * @param range_j the number of items to process along the second + * dimension of the 3D grid. + * @param range_k the number of items to process along the third + * dimension of the 3D grid. + * @param tile_k the maximum number of items along the third + * dimension of the 3D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional + * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or + * PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d_tile_1d_with_uarch_with_thread( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_1d_with_id_with_thread_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_k, + uint32_t flags); + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * for (size_t k = 0; k < range_k; k += tile_k) + * function(context, i, j, k, + * min(range_j - j, tile_j), min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param tile_j the maximum number of items along the second dimension of + * the 3D grid to process in one function call. + * @param tile_k the maximum number of items along the third dimension of + * the 3D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d_tile_2d( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_2d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_j, + size_t tile_k, + uint32_t flags); + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last two grid dimensions using a microarchitecture-aware task function. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * for (size_t k = 0; k < range_k; k += tile_k) + * function(context, uarch_index, i, j, k, + * min(range_j - j, tile_j), min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If + * threadpool is NULL, all items are processed serially on the calling + * thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified + * function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first + * dimension of the 3D grid. + * @param range_j the number of items to process along the second + * dimension of the 3D grid. + * @param range_k the number of items to process along the third + * dimension of the 3D grid. + * @param tile_j the maximum number of items along the second + * dimension of the 3D grid to process in one function call. + * @param tile_k the maximum number of items along the third + * dimension of the 3D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional + * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or + * PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_3d_tile_2d_with_uarch( + pthreadpool_t threadpool, + pthreadpool_task_3d_tile_2d_with_id_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_j, + size_t tile_k, + uint32_t flags); + +/** + * Process items on a 4D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * function(context, i, j, k, l); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_4d( + pthreadpool_t threadpool, + pthreadpool_task_4d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + uint32_t flags); + +/** + * Process items on a 4D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l += tile_l) + * function(context, i, j, k, l, min(range_l - l, tile_l)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param tile_l the maximum number of items along the fourth dimension of + * the 4D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_4d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_4d_tile_1d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_l, + uint32_t flags); + +/** + * Process items on a 4D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * for (size_t l = 0; l < range_l; l += tile_l) + * function(context, i, j, k, l, + * min(range_k - k, tile_k), min(range_l - l, tile_l)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param tile_k the maximum number of items along the third dimension of + * the 4D grid to process in one function call. + * @param tile_l the maximum number of items along the fourth dimension of + * the 4D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_4d_tile_2d( + pthreadpool_t threadpool, + pthreadpool_task_4d_tile_2d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_k, + size_t tile_l, + uint32_t flags); + +/** + * Process items on a 4D grid with the specified maximum tile size along the + * last two grid dimensions using a microarchitecture-aware task function. + * + * The function implements a parallel version of the following snippet: + * + * uint32_t uarch_index = cpuinfo_initialize() ? + * cpuinfo_get_current_uarch_index() : default_uarch_index; + * if (uarch_index > max_uarch_index) uarch_index = default_uarch_index; + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * for (size_t l = 0; l < range_l; l += tile_l) + * function(context, uarch_index, i, j, k, l, + * min(range_k - k, tile_k), min(range_l - l, tile_l)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If + * threadpool is NULL, all items are processed serially on the calling + * thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified + * function. + * @param default_uarch_index the microarchitecture index to use when + * pthreadpool is configured without cpuinfo, cpuinfo initialization failed, + * or index returned by cpuinfo_get_current_uarch_index() exceeds the + * max_uarch_index value. + * @param max_uarch_index the maximum microarchitecture index expected by + * the specified function. If the index returned by + * cpuinfo_get_current_uarch_index() exceeds this value, default_uarch_index + * will be used instead. default_uarch_index can exceed max_uarch_index. + * @param range_i the number of items to process along the first + * dimension of the 4D grid. + * @param range_j the number of items to process along the second + * dimension of the 4D grid. + * @param range_k the number of items to process along the third + * dimension of the 4D grid. + * @param range_l the number of items to process along the fourth + * dimension of the 4D grid. + * @param tile_k the maximum number of items along the third + * dimension of the 4D grid to process in one function call. + * @param tile_l the maximum number of items along the fourth + * dimension of the 4D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional + * flags (PTHREADPOOL_FLAG_DISABLE_DENORMALS or + * PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_4d_tile_2d_with_uarch( + pthreadpool_t threadpool, + pthreadpool_task_4d_tile_2d_with_id_t function, + void* context, + uint32_t default_uarch_index, + uint32_t max_uarch_index, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_k, + size_t tile_l, + uint32_t flags); + +/** + * Process items on a 5D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m++) + * function(context, i, j, k, l, m); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_5d( + pthreadpool_t threadpool, + pthreadpool_task_5d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + uint32_t flags); + +/** + * Process items on a 5D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m += tile_m) + * function(context, i, j, k, l, m, min(range_m - m, tile_m)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param tile_m the maximum number of items along the fifth dimension of + * the 5D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_5d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_5d_tile_1d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t tile_m, + uint32_t flags); + +/** + * Process items on a 5D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l += tile_l) + * for (size_t m = 0; m < range_m; m += tile_m) + * function(context, i, j, k, l, m, + * min(range_l - l, tile_l), min(range_m - m, tile_m)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param tile_l the maximum number of items along the fourth dimension of + * the 5D grid to process in one function call. + * @param tile_m the maximum number of items along the fifth dimension of + * the 5D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_5d_tile_2d( + pthreadpool_t threadpool, + pthreadpool_task_5d_tile_2d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t tile_l, + size_t tile_m, + uint32_t flags); + +/** + * Process items on a 6D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m++) + * for (size_t n = 0; n < range_n; n++) + * function(context, i, j, k, l, m, n); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 6D grid. + * @param range_j the number of items to process along the second dimension + * of the 6D grid. + * @param range_k the number of items to process along the third dimension + * of the 6D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 6D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 6D grid. + * @param range_n the number of items to process along the sixth dimension + * of the 6D grid. + * @param tile_n the maximum number of items along the sixth dimension of + * the 6D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_6d( + pthreadpool_t threadpool, + pthreadpool_task_6d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t range_n, + uint32_t flags); + +/** + * Process items on a 6D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m++) + * for (size_t n = 0; n < range_n; n += tile_n) + * function(context, i, j, k, l, m, n, min(range_n - n, tile_n)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 6D grid. + * @param range_j the number of items to process along the second dimension + * of the 6D grid. + * @param range_k the number of items to process along the third dimension + * of the 6D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 6D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 6D grid. + * @param range_n the number of items to process along the sixth dimension + * of the 6D grid. + * @param tile_n the maximum number of items along the sixth dimension of + * the 6D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_6d_tile_1d( + pthreadpool_t threadpool, + pthreadpool_task_6d_tile_1d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t range_n, + size_t tile_n, + uint32_t flags); + +/** + * Process items on a 6D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m += tile_m) + * for (size_t n = 0; n < range_n; n += tile_n) + * function(context, i, j, k, l, m, n, + * min(range_m - m, tile_m), min(range_n - n, tile_n)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param function the function to call for each tile. + * @param context the first argument passed to the specified function. + * @param range_i the number of items to process along the first dimension + * of the 6D grid. + * @param range_j the number of items to process along the second dimension + * of the 6D grid. + * @param range_k the number of items to process along the third dimension + * of the 6D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 6D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 6D grid. + * @param range_n the number of items to process along the sixth dimension + * of the 6D grid. + * @param tile_m the maximum number of items along the fifth dimension of + * the 6D grid to process in one function call. + * @param tile_n the maximum number of items along the sixth dimension of + * the 6D grid to process in one function call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +void pthreadpool_parallelize_6d_tile_2d( + pthreadpool_t threadpool, + pthreadpool_task_6d_tile_2d_t function, + void* context, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t range_n, + size_t tile_m, + size_t tile_n, + uint32_t flags); + +/** + * Terminates threads in the thread pool and releases associated resources. + * + * @warning Accessing the thread pool after a call to this function constitutes + * undefined behaviour and may cause data corruption. + * + * @param[in,out] threadpool The thread pool to destroy. + */ +void pthreadpool_destroy(pthreadpool_t threadpool); + +#ifndef PTHREADPOOL_NO_DEPRECATED_API + +/* Legacy API for compatibility with pre-existing users (e.g. NNPACK) */ +#if defined(__GNUC__) + #define PTHREADPOOL_DEPRECATED __attribute__((__deprecated__)) +#else + #define PTHREADPOOL_DEPRECATED +#endif + +typedef void (*pthreadpool_function_1d_t)(void*, size_t); +typedef void (*pthreadpool_function_1d_tiled_t)(void*, size_t, size_t); +typedef void (*pthreadpool_function_2d_t)(void*, size_t, size_t); +typedef void (*pthreadpool_function_2d_tiled_t)(void*, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_function_3d_tiled_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t); +typedef void (*pthreadpool_function_4d_tiled_t)(void*, size_t, size_t, size_t, size_t, size_t, size_t, size_t, size_t); + +void pthreadpool_compute_1d( + pthreadpool_t threadpool, + pthreadpool_function_1d_t function, + void* argument, + size_t range) PTHREADPOOL_DEPRECATED; + +void pthreadpool_compute_1d_tiled( + pthreadpool_t threadpool, + pthreadpool_function_1d_tiled_t function, + void* argument, + size_t range, + size_t tile) PTHREADPOOL_DEPRECATED; + +void pthreadpool_compute_2d( + pthreadpool_t threadpool, + pthreadpool_function_2d_t function, + void* argument, + size_t range_i, + size_t range_j) PTHREADPOOL_DEPRECATED; + +void pthreadpool_compute_2d_tiled( + pthreadpool_t threadpool, + pthreadpool_function_2d_tiled_t function, + void* argument, + size_t range_i, + size_t range_j, + size_t tile_i, + size_t tile_j) PTHREADPOOL_DEPRECATED; + +void pthreadpool_compute_3d_tiled( + pthreadpool_t threadpool, + pthreadpool_function_3d_tiled_t function, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_i, + size_t tile_j, + size_t tile_k) PTHREADPOOL_DEPRECATED; + +void pthreadpool_compute_4d_tiled( + pthreadpool_t threadpool, + pthreadpool_function_4d_tiled_t function, + void* argument, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_i, + size_t tile_j, + size_t tile_k, + size_t tile_l) PTHREADPOOL_DEPRECATED; + +#endif /* PTHREADPOOL_NO_DEPRECATED_API */ + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#ifdef __cplusplus + +namespace libpthreadpool { +namespace detail { +namespace { + +template +void call_wrapper_1d(void* arg, size_t i) { + (*static_cast(arg))(i); +} + +template +void call_wrapper_1d_tile_1d(void* arg, size_t range_i, size_t tile_i) { + (*static_cast(arg))(range_i, tile_i); +} + +template +void call_wrapper_2d(void* functor, size_t i, size_t j) { + (*static_cast(functor))(i, j); +} + +template +void call_wrapper_2d_tile_1d(void* functor, + size_t i, size_t range_j, size_t tile_j) +{ + (*static_cast(functor))(i, range_j, tile_j); +} + +template +void call_wrapper_2d_tile_2d(void* functor, + size_t range_i, size_t range_j, + size_t tile_i, size_t tile_j) +{ + (*static_cast(functor))(range_i, range_j, tile_i, tile_j); +} + +template +void call_wrapper_3d(void* functor, size_t i, size_t j, size_t k) { + (*static_cast(functor))(i, j, k); +} + +template +void call_wrapper_3d_tile_1d(void* functor, + size_t i, size_t j, size_t range_k, + size_t tile_k) +{ + (*static_cast(functor))(i, j, range_k, tile_k); +} + +template +void call_wrapper_3d_tile_2d(void* functor, + size_t i, size_t range_j, size_t range_k, + size_t tile_j, size_t tile_k) +{ + (*static_cast(functor))(i, range_j, range_k, tile_j, tile_k); +} + +template +void call_wrapper_4d(void* functor, size_t i, size_t j, size_t k, size_t l) { + (*static_cast(functor))(i, j, k, l); +} + +template +void call_wrapper_4d_tile_1d(void* functor, + size_t i, size_t j, size_t k, size_t range_l, + size_t tile_l) +{ + (*static_cast(functor))(i, j, k, range_l, tile_l); +} + +template +void call_wrapper_4d_tile_2d(void* functor, + size_t i, size_t j, size_t range_k, size_t range_l, + size_t tile_k, size_t tile_l) +{ + (*static_cast(functor))(i, j, range_k, range_l, tile_k, tile_l); +} + +template +void call_wrapper_5d(void* functor, size_t i, size_t j, size_t k, size_t l, size_t m) { + (*static_cast(functor))(i, j, k, l, m); +} + +template +void call_wrapper_5d_tile_1d(void* functor, + size_t i, size_t j, size_t k, size_t l, size_t range_m, + size_t tile_m) +{ + (*static_cast(functor))(i, j, k, l, range_m, tile_m); +} + +template +void call_wrapper_5d_tile_2d(void* functor, + size_t i, size_t j, size_t k, size_t range_l, size_t range_m, + size_t tile_l, size_t tile_m) +{ + (*static_cast(functor))(i, j, k, range_l, range_m, tile_l, tile_m); +} + +template +void call_wrapper_6d(void* functor, size_t i, size_t j, size_t k, size_t l, size_t m, size_t n) { + (*static_cast(functor))(i, j, k, l, m, n); +} + +template +void call_wrapper_6d_tile_1d(void* functor, + size_t i, size_t j, size_t k, size_t l, size_t m, size_t range_n, + size_t tile_n) +{ + (*static_cast(functor))(i, j, k, l, m, range_n, tile_n); +} + +template +void call_wrapper_6d_tile_2d(void* functor, + size_t i, size_t j, size_t k, size_t l, size_t range_m, size_t range_n, + size_t tile_m, size_t tile_n) +{ + (*static_cast(functor))(i, j, k, l, range_m, range_n, tile_m, tile_n); +} + +} /* namespace */ +} /* namespace detail */ +} /* namespace libpthreadpool */ + +/** + * Process items on a 1D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range; i++) + * functor(i); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each item. + * @param range the number of items on the 1D grid to process. The + * specified functor will be called once for each item. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range, + uint32_t flags = 0) +{ + pthreadpool_parallelize_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_1d, + const_cast(static_cast(&functor)), + range, + flags); +} + +/** + * Process items on a 1D grid with specified maximum tile size. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range; i += tile) + * functor(i, min(range - i, tile)); + * + * When the call returns, all items have been processed and the thread pool is + * ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, + * the calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range the number of items on the 1D grid to process. + * @param tile the maximum number of items on the 1D grid to process in + * one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_1d_tile_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range, + size_t tile, + uint32_t flags = 0) +{ + pthreadpool_parallelize_1d_tile_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_1d_tile_1d, + const_cast(static_cast(&functor)), + range, + tile, + flags); +} + +/** + * Process items on a 2D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * functor(i, j); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each item. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_2d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + uint32_t flags = 0) +{ + pthreadpool_parallelize_2d( + threadpool, + &libpthreadpool::detail::call_wrapper_2d, + const_cast(static_cast(&functor)), + range_i, + range_j, + flags); +} + +/** + * Process items on a 2D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * functor(i, j, min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param tile_j the maximum number of items along the second dimension of + * the 2D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_2d_tile_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t tile_j, + uint32_t flags = 0) +{ + pthreadpool_parallelize_2d_tile_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_2d_tile_1d, + const_cast(static_cast(&functor)), + range_i, + range_j, + tile_j, + flags); +} + +/** + * Process items on a 2D grid with the specified maximum tile size along each + * grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i += tile_i) + * for (size_t j = 0; j < range_j; j += tile_j) + * functor(i, j, + * min(range_i - i, tile_i), min(range_j - j, tile_j)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 2D grid. + * @param range_j the number of items to process along the second dimension + * of the 2D grid. + * @param tile_j the maximum number of items along the first dimension of + * the 2D grid to process in one functor call. + * @param tile_j the maximum number of items along the second dimension of + * the 2D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_2d_tile_2d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t tile_i, + size_t tile_j, + uint32_t flags = 0) +{ + pthreadpool_parallelize_2d_tile_2d( + threadpool, + &libpthreadpool::detail::call_wrapper_2d_tile_2d, + const_cast(static_cast(&functor)), + range_i, + range_j, + tile_i, + tile_j, + flags); +} + +/** + * Process items on a 3D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * functor(i, j, k); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_3d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + uint32_t flags = 0) +{ + pthreadpool_parallelize_3d( + threadpool, + &libpthreadpool::detail::call_wrapper_3d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + flags); +} + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * functor(i, j, k, min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param tile_k the maximum number of items along the third dimension of + * the 3D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_3d_tile_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_k, + uint32_t flags = 0) +{ + pthreadpool_parallelize_3d_tile_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_3d_tile_1d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + tile_k, + flags); +} + +/** + * Process items on a 3D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j += tile_j) + * for (size_t k = 0; k < range_k; k += tile_k) + * functor(i, j, k, + * min(range_j - j, tile_j), min(range_k - k, tile_k)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 3D grid. + * @param range_j the number of items to process along the second dimension + * of the 3D grid. + * @param range_k the number of items to process along the third dimension + * of the 3D grid. + * @param tile_j the maximum number of items along the second dimension of + * the 3D grid to process in one functor call. + * @param tile_k the maximum number of items along the third dimension of + * the 3D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_3d_tile_2d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t tile_j, + size_t tile_k, + uint32_t flags = 0) +{ + pthreadpool_parallelize_3d_tile_2d( + threadpool, + &libpthreadpool::detail::call_wrapper_3d_tile_2d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + tile_j, + tile_k, + flags); +} + +/** + * Process items on a 4D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * functor(i, j, k, l); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_4d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + uint32_t flags = 0) +{ + pthreadpool_parallelize_4d( + threadpool, + &libpthreadpool::detail::call_wrapper_4d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + flags); +} + +/** + * Process items on a 4D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l += tile_l) + * functor(i, j, k, l, min(range_l - l, tile_l)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param tile_l the maximum number of items along the fourth dimension of + * the 4D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_4d_tile_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_l, + uint32_t flags = 0) +{ + pthreadpool_parallelize_4d_tile_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_4d_tile_1d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + tile_l, + flags); +} + +/** + * Process items on a 4D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k += tile_k) + * for (size_t l = 0; l < range_l; l += tile_l) + * functor(i, j, k, l, + * min(range_k - k, tile_k), min(range_l - l, tile_l)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 4D grid. + * @param range_j the number of items to process along the second dimension + * of the 4D grid. + * @param range_k the number of items to process along the third dimension + * of the 4D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 4D grid. + * @param tile_k the maximum number of items along the third dimension of + * the 4D grid to process in one functor call. + * @param tile_l the maximum number of items along the fourth dimension of + * the 4D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_4d_tile_2d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t tile_k, + size_t tile_l, + uint32_t flags = 0) +{ + pthreadpool_parallelize_4d_tile_2d( + threadpool, + &libpthreadpool::detail::call_wrapper_4d_tile_2d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + tile_k, + tile_l, + flags); +} + +/** + * Process items on a 5D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m++) + * functor(i, j, k, l, m); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_5d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + uint32_t flags = 0) +{ + pthreadpool_parallelize_5d( + threadpool, + &libpthreadpool::detail::call_wrapper_5d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + range_m, + flags); +} + +/** + * Process items on a 5D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m += tile_m) + * functor(i, j, k, l, m, min(range_m - m, tile_m)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param tile_m the maximum number of items along the fifth dimension of + * the 5D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_5d_tile_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t tile_m, + uint32_t flags = 0) +{ + pthreadpool_parallelize_5d_tile_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_5d_tile_1d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + range_m, + tile_m, + flags); +} + +/** + * Process items on a 5D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l += tile_l) + * for (size_t m = 0; m < range_m; m += tile_m) + * functor(i, j, k, l, m, + * min(range_l - l, tile_l), min(range_m - m, tile_m)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 5D grid. + * @param range_j the number of items to process along the second dimension + * of the 5D grid. + * @param range_k the number of items to process along the third dimension + * of the 5D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 5D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 5D grid. + * @param tile_l the maximum number of items along the fourth dimension of + * the 5D grid to process in one functor call. + * @param tile_m the maximum number of items along the fifth dimension of + * the 5D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_5d_tile_2d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t tile_l, + size_t tile_m, + uint32_t flags = 0) +{ + pthreadpool_parallelize_5d_tile_2d( + threadpool, + &libpthreadpool::detail::call_wrapper_5d_tile_2d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + range_m, + tile_l, + tile_m, + flags); +} + +/** + * Process items on a 6D grid. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m++) + * for (size_t n = 0; n < range_n; n++) + * functor(i, j, k, l, m, n); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 6D grid. + * @param range_j the number of items to process along the second dimension + * of the 6D grid. + * @param range_k the number of items to process along the third dimension + * of the 6D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 6D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 6D grid. + * @param range_n the number of items to process along the sixth dimension + * of the 6D grid. + * @param tile_n the maximum number of items along the sixth dimension of + * the 6D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_6d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t range_n, + uint32_t flags = 0) +{ + pthreadpool_parallelize_6d( + threadpool, + &libpthreadpool::detail::call_wrapper_6d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + range_m, + range_n, + flags); +} + +/** + * Process items on a 6D grid with the specified maximum tile size along the + * last grid dimension. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m++) + * for (size_t n = 0; n < range_n; n += tile_n) + * functor(i, j, k, l, m, n, min(range_n - n, tile_n)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 6D grid. + * @param range_j the number of items to process along the second dimension + * of the 6D grid. + * @param range_k the number of items to process along the third dimension + * of the 6D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 6D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 6D grid. + * @param range_n the number of items to process along the sixth dimension + * of the 6D grid. + * @param tile_n the maximum number of items along the sixth dimension of + * the 6D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_6d_tile_1d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t range_n, + size_t tile_n, + uint32_t flags = 0) +{ + pthreadpool_parallelize_6d_tile_1d( + threadpool, + &libpthreadpool::detail::call_wrapper_6d_tile_1d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + range_m, + range_n, + tile_n, + flags); +} + +/** + * Process items on a 6D grid with the specified maximum tile size along the + * last two grid dimensions. + * + * The function implements a parallel version of the following snippet: + * + * for (size_t i = 0; i < range_i; i++) + * for (size_t j = 0; j < range_j; j++) + * for (size_t k = 0; k < range_k; k++) + * for (size_t l = 0; l < range_l; l++) + * for (size_t m = 0; m < range_m; m += tile_m) + * for (size_t n = 0; n < range_n; n += tile_n) + * functor(i, j, k, l, m, n, + * min(range_m - m, tile_m), min(range_n - n, tile_n)); + * + * When the function returns, all items have been processed and the thread pool + * is ready for a new task. + * + * @note If multiple threads call this function with the same thread pool, the + * calls are serialized. + * + * @param threadpool the thread pool to use for parallelisation. If threadpool + * is NULL, all items are processed serially on the calling thread. + * @param functor the functor to call for each tile. + * @param range_i the number of items to process along the first dimension + * of the 6D grid. + * @param range_j the number of items to process along the second dimension + * of the 6D grid. + * @param range_k the number of items to process along the third dimension + * of the 6D grid. + * @param range_l the number of items to process along the fourth dimension + * of the 6D grid. + * @param range_m the number of items to process along the fifth dimension + * of the 6D grid. + * @param range_n the number of items to process along the sixth dimension + * of the 6D grid. + * @param tile_m the maximum number of items along the fifth dimension of + * the 6D grid to process in one functor call. + * @param tile_n the maximum number of items along the sixth dimension of + * the 6D grid to process in one functor call. + * @param flags a bitwise combination of zero or more optional flags + * (PTHREADPOOL_FLAG_DISABLE_DENORMALS or PTHREADPOOL_FLAG_YIELD_WORKERS) + */ +template +inline void pthreadpool_parallelize_6d_tile_2d( + pthreadpool_t threadpool, + const T& functor, + size_t range_i, + size_t range_j, + size_t range_k, + size_t range_l, + size_t range_m, + size_t range_n, + size_t tile_m, + size_t tile_n, + uint32_t flags = 0) +{ + pthreadpool_parallelize_6d_tile_2d( + threadpool, + &libpthreadpool::detail::call_wrapper_6d_tile_2d, + const_cast(static_cast(&functor)), + range_i, + range_j, + range_k, + range_l, + range_m, + range_n, + tile_m, + tile_n, + flags); +} + +#endif /* __cplusplus */ + +#endif /* PTHREADPOOL_H_ */ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/qnnpack_func.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/qnnpack_func.h new file mode 100644 index 0000000000000000000000000000000000000000..667a57fa62d16b5b9ff5bce8655ef553ff97cb0b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/qnnpack_func.h @@ -0,0 +1,171 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +#pragma once + +#include +#include + +namespace qnnpack { +class PrePackConvWeights final { + public: + PrePackConvWeights( + const pytorch_qnnp_operator_t convolution, + const uint8_t* kernel_zero_points, + const uint8_t* kernel, + const int32_t* bias); + + void* getPackedWeights() const + { + return packed_weights_; + } + + int64_t getOutputChannels() const + { + return output_channels_; + } + + ~PrePackConvWeights() + { + if (packed_weights_ != nullptr) { + free(packed_weights_); + } + } + + PrePackConvWeights() = delete; + PrePackConvWeights(const PrePackConvWeights&) = delete; + PrePackConvWeights& operator=(const PrePackConvWeights&) = delete; + + private: + void* packed_weights_ = nullptr; + int64_t output_channels_; +}; + +class PackBMatrix final { + public: + PackBMatrix( + size_t input_channels, + size_t output_channels, + const uint8_t* kernel_zero_points, + const float* requantization_scale, + const uint8_t* kernel, + const int32_t* bias); + + // This constructor is to be used for dynamic mode + // quantization. In dynamic mode, we dont yet support + // per channel quantization, and paying the cost of + // memory allocation for per channel zero point and + // requant scale will hurt performance. + PackBMatrix( + size_t input_channels, + size_t output_channels, + const uint8_t kernel_zero_point, + const float requantization_scale, + const uint8_t* kernel, + const int32_t* bias); + + void* getPackedWeights() const + { + return packed_weights_; + } + + void unpackWeights( + const uint8_t* kernel_zero_points, + int8_t* kernel + ) const; + + size_t getInputChannels() const + { + return input_channels_; + } + + size_t getOutputChannels() const + { + return output_channels_; + } + + ~PackBMatrix() + { + if (packed_weights_ != nullptr) { + free(packed_weights_); + } + } + + PackBMatrix() = delete; + PackBMatrix(const PackBMatrix&) = delete; + PackBMatrix& operator=(const PackBMatrix&) = delete; + + private: + void* packed_weights_ = nullptr; + size_t input_channels_; + size_t output_channels_; +}; + +enum pytorch_qnnp_status qnnpackLinear( + const size_t batch_size, + const size_t input_channels, + const size_t output_channels, + const uint8_t input_zero_point, + const uint8_t* kernel_zero_points, + const float* requantization_scales, + const uint8_t output_zero_point, + const uint8_t output_min, + const uint8_t output_max, + const uint8_t* input, + const size_t input_stride, + void* packed_weights, + uint8_t* output, + const size_t output_stride, + pthreadpool_t threadpool); + +enum pytorch_qnnp_status qnnpackConv( + const pytorch_qnnp_operator_t convolution, + void* packed_weights, + const size_t batch_size, + const size_t input_depth, + const size_t input_height, + const size_t input_width, + const uint8_t input_zero_point, + const uint8_t* input, + const uint8_t* kernel_zero_points, + const float* requantization_scales, + const uint8_t output_zero_point, + const uint8_t output_min, + const uint8_t output_max, + uint8_t* output, + pthreadpool_t threadpool); + +enum pytorch_qnnp_status qnnpackDeConv( + const pytorch_qnnp_operator_t deconvolution, + void* packed_weights, + const size_t batch_size, + const size_t input_height, + const size_t input_width, + const uint8_t input_zero_point, + const uint8_t* input, + const uint8_t* kernel_zero_points, + const float* requantization_scales, + const uint8_t output_zero_point, + const uint8_t output_min, + const uint8_t output_max, + uint8_t* output, + pthreadpool_t threadpool); + +enum pytorch_qnnp_status qnnpackLinearDynamic( + const size_t batch_size, + const size_t input_channels, + const size_t output_channels, + const uint8_t input_zero_point, + const uint8_t* kernel_zero_points, + const float* dequantization_scales, + const uint8_t* input, + const size_t input_stride, + void* packed_weights, + const float* bias, + float* output, + const size_t output_stride, + pthreadpool_t threadpool); + +} // namespace qnnpack + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/sleef.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/sleef.h new file mode 100644 index 0000000000000000000000000000000000000000..d07a4f1cae6150546e64502e65d2875d633b5729 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/sleef.h @@ -0,0 +1,4221 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +// Copyright Naoki Shibata and contributors 2010 - 2024. +// Distributed under the Boost Software License, Version 1.0. +// (See accompanying file LICENSE.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#ifndef __SLEEF_H__ +#define __SLEEF_H__ + +#define SLEEF_VERSION_MAJOR 3 +#define SLEEF_VERSION_MINOR 8 +#define SLEEF_VERSION_PATCHLEVEL 0 + +#include +#include + +#if defined (__GNUC__) || defined (__clang__) || defined(__INTEL_COMPILER) +#define SLEEF_CONST __attribute__((const)) +#define SLEEF_INLINE __attribute__((always_inline)) +#elif defined(_MSC_VER) +#define SLEEF_CONST +#define SLEEF_INLINE __forceinline +#endif + +#if defined(__AVX2__) || defined(__aarch64__) || defined(__arm__) || defined(__powerpc64__) || defined(__zarch__) +#ifndef FP_FAST_FMA +#define FP_FAST_FMA +#endif +#ifndef FP_FAST_FMAF +#define FP_FAST_FMAF +#endif +#endif + +#if defined(_MSC_VER) && !defined(__STDC__) +#define __STDC__ 1 +#endif + +#if (defined(__MINGW32__) || defined(__MINGW64__) || defined(__CYGWIN__) || defined(_MSC_VER)) && !defined(SLEEF_STATIC_LIBS) +#ifdef SLEEF_IMPORT_IS_EXPORT +#define SLEEF_IMPORT __declspec(dllexport) +#else // #ifdef SLEEF_IMPORT_IS_EXPORT +#define SLEEF_IMPORT __declspec(dllimport) +#if (defined(_MSC_VER)) +#pragma comment(lib,"sleef.lib") +#endif // #if (defined(_MSC_VER)) +#endif // #ifdef SLEEF_IMPORT_IS_EXPORT +#else // #if (defined(__MINGW32__) || defined(__MINGW64__) || defined(__CYGWIN__) || defined(_MSC_VER)) && !defined(SLEEF_STATIC_LIBS) +#define SLEEF_IMPORT +#endif // #if (defined(__MINGW32__) || defined(__MINGW64__) || defined(__CYGWIN__) || defined(_MSC_VER)) && !defined(SLEEF_STATIC_LIBS) + +#if (defined(__GNUC__) || defined(__CLANG__)) && (defined(__i386__) || defined(__x86_64__)) +#include +#endif + +#if (defined(_MSC_VER)) +#include +#endif + +#if defined(__ARM_NEON__) || defined(__ARM_NEON) +#include +#endif + +#if defined(__ARM_FEATURE_SVE) +#include +#endif + +#if defined(__VSX__) && defined(__PPC64__) && defined(__LITTLE_ENDIAN__) +#include +typedef __vector double SLEEF_VECTOR_DOUBLE; +typedef __vector float SLEEF_VECTOR_FLOAT; +typedef __vector int SLEEF_VECTOR_INT; +typedef __vector unsigned int SLEEF_VECTOR_UINT; +typedef __vector long long SLEEF_VECTOR_LONGLONG; +typedef __vector unsigned long long SLEEF_VECTOR_ULONGLONG; +#endif + +#if defined(__VX__) && defined(__VEC__) +#ifndef SLEEF_VECINTRIN_H_INCLUDED +#include +#define SLEEF_VECINTRIN_H_INCLUDED +#endif +typedef __vector double SLEEF_VECTOR_DOUBLE; +typedef __vector float SLEEF_VECTOR_FLOAT; +typedef __vector int SLEEF_VECTOR_INT; +typedef __vector unsigned int SLEEF_VECTOR_UINT; +typedef __vector long long SLEEF_VECTOR_LONGLONG; +typedef __vector unsigned long long SLEEF_VECTOR_ULONGLONG; +#endif + +// + +#if defined(SLEEF_ENABLE_OMP_SIMD) && (defined(__GNUC__) || defined(__CLANG__)) && !defined(__INTEL_COMPILER) +#if defined(__aarch64__) +//#define SLEEF_PRAGMA_OMP_SIMD_DP _Pragma ("omp declare simd simdlen(2) notinbranch") +//#define SLEEF_PRAGMA_OMP_SIMD_SP _Pragma ("omp declare simd simdlen(4) notinbranch") +//#elif defined(__x86_64__) && defined(__AVX512F__) +//#define SLEEF_PRAGMA_OMP_SIMD_DP _Pragma ("omp declare simd simdlen(8) notinbranch") +//#define SLEEF_PRAGMA_OMP_SIMD_SP _Pragma ("omp declare simd simdlen(16) notinbranch") +#elif defined(__x86_64__) && defined(__AVX__) +#define SLEEF_PRAGMA_OMP_SIMD_DP _Pragma ("omp declare simd simdlen(4) notinbranch") +#define SLEEF_PRAGMA_OMP_SIMD_SP _Pragma ("omp declare simd simdlen(8) notinbranch") +#elif defined(__x86_64__) && defined(__SSE2__) +#define SLEEF_PRAGMA_OMP_SIMD_DP _Pragma ("omp declare simd simdlen(2) notinbranch") +#define SLEEF_PRAGMA_OMP_SIMD_SP _Pragma ("omp declare simd simdlen(4) notinbranch") +#endif +#endif + +#ifndef SLEEF_PRAGMA_OMP_SIMD_DP +#define SLEEF_PRAGMA_OMP_SIMD_DP +#define SLEEF_PRAGMA_OMP_SIMD_SP +#endif + +// + +#ifndef SLEEF_FP_ILOGB0 +#define SLEEF_FP_ILOGB0 ((int)0x80000000) +#endif + +#ifndef SLEEF_FP_ILOGBNAN +#define SLEEF_FP_ILOGBNAN ((int)2147483647) +#endif + +// + +SLEEF_IMPORT void *Sleef_malloc(size_t z); +SLEEF_IMPORT void Sleef_free(void *ptr); +SLEEF_IMPORT uint64_t Sleef_currentTimeMicros(); + +#if defined(__i386__) || defined(__x86_64__) || defined(_MSC_VER) +SLEEF_IMPORT void Sleef_x86CpuID(int32_t out[4], uint32_t eax, uint32_t ecx); +#endif + +// + +#if defined(__riscv_v) +#include +typedef vfloat64m2_t Sleef_vfloat64m1_t_2; +typedef vfloat32m2_t Sleef_vfloat32m1_t_2; +typedef vfloat64m4_t Sleef_vfloat64m2_t_2; +typedef vfloat32m4_t Sleef_vfloat32m2_t_2; +#define Sleef_vfloat64m1_t_2_DEFINED +#define Sleef_vfloat32m1_t_2_DEFINED +#define Sleef_vfloat64m2_t_2_DEFINED +#define Sleef_vfloat32m2_t_2_DEFINED +#endif + +#ifndef Sleef_double2_DEFINED +#define Sleef_double2_DEFINED +typedef struct { + double x, y; +} Sleef_double2; +#endif + +#ifndef Sleef_float2_DEFINED +#define Sleef_float2_DEFINED +typedef struct { + float x, y; +} Sleef_float2; +#endif + +#ifndef Sleef_longdouble2_DEFINED +#define Sleef_longdouble2_DEFINED +typedef struct { + long double x, y; +} Sleef_longdouble2; +#endif + +#if (defined(__SIZEOF_FLOAT128__) && __SIZEOF_FLOAT128__ == 16) || (defined(__linux__) && defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) || (defined(__PPC64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 8) +#define SLEEF_FLOAT128_IS_IEEEQP +#endif + +#if !defined(SLEEF_FLOAT128_IS_IEEEQP) && defined(__SIZEOF_LONG_DOUBLE__) && __SIZEOF_LONG_DOUBLE__ == 16 && (defined(__aarch64__) || defined(__zarch__)) +#define SLEEF_LONGDOUBLE_IS_IEEEQP +#endif + +#if !defined(Sleef_quad_DEFINED) +#define Sleef_quad_DEFINED +typedef struct { uint64_t x, y; } Sleef_uint64_2t; +#if defined(SLEEF_FLOAT128_IS_IEEEQP) || defined(ENABLEFLOAT128) +typedef __float128 Sleef_quad; +#define SLEEF_QUAD_C(x) (x ## Q) +#elif defined(SLEEF_LONGDOUBLE_IS_IEEEQP) +typedef long double Sleef_quad; +#define SLEEF_QUAD_C(x) (x ## L) +#else +typedef Sleef_uint64_2t Sleef_quad; +#endif +#endif +#if !defined(Sleef_quad2_DEFINED) +#define Sleef_quad2_DEFINED +typedef union { + struct { + Sleef_quad x, y; + }; + Sleef_quad s[2]; +} Sleef_quad2; +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif + +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sin_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cos_u35(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double2 Sleef_sincos_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tan_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_asin_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_acos_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atan_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atan2_u35(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cbrt_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sin_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cos_u10(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double2 Sleef_sincos_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tan_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_asin_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_acos_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atan_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atan2_u10(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cbrt_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_pow_u10(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sinh_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cosh_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tanh_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sinh_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cosh_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tanh_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_asinh_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_acosh_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atanh_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp2_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp10_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp2_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp10_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_expm1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log10_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log2_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log2_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log1p_u10(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double2 Sleef_sincospi_u05(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double2 Sleef_sincospi_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sinpi_u05(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cospi_u05(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_ldexp(double, int); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST int Sleef_ilogb(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fma(double, double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sqrt(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sqrt_u05(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sqrt_u35(double); + +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_hypot_u05(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_hypot_u35(double, double); + +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fabs(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_copysign(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fmax(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fmin(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fdim(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_trunc(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_floor(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_ceil(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_round(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_rint(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_nextafter(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_frfrexp(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST int Sleef_expfrexp(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fmod(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_remainder(double, double); +SLEEF_IMPORT SLEEF_CONST Sleef_double2 Sleef_modf(double); + +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_lgamma_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tgamma_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_erf_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_erfc_u15(double); + +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cosf_u35(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float2 Sleef_sincosf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_asinf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_acosf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atanf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f_u35(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_logf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cosf_u10(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float2 Sleef_sincosf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fastsinf_u3500(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fastcosf_u3500(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_asinf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_acosf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atanf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f_u10(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_logf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_expf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_powf_u10(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fastpowf_u3500(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_coshf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_coshf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_asinhf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_acoshf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atanhf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_expm1f_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log10f_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log2f_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log2f_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log1pf_u10(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float2 Sleef_sincospif_u05(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float2 Sleef_sincospif_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinpif_u05(float d); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cospif_u05(float d); +SLEEF_IMPORT SLEEF_CONST float Sleef_ldexpf(float, int); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST int Sleef_ilogbf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fmaf(float, float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf_u05(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf_u35(float); + +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf_u05(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf_u35(float, float); + +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fabsf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_copysignf(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fmaxf(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fminf(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fdimf(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_truncf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_floorf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_ceilf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_roundf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_rintf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_nextafterf(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_frfrexpf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST int Sleef_expfrexpf(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fmodf(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_remainderf(float, float); +SLEEF_IMPORT SLEEF_CONST Sleef_float2 Sleef_modff(float); + +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_lgammaf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tgammaf_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_erff_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_erfcf_u15(float); + +SLEEF_IMPORT SLEEF_CONST Sleef_longdouble2 Sleef_sincospil_u05(long double); +SLEEF_IMPORT SLEEF_CONST Sleef_longdouble2 Sleef_sincospil_u35(long double); + +#if defined(Sleef_quad2_DEFINED) +SLEEF_IMPORT SLEEF_CONST Sleef_quad2 Sleef_sincospiq_u05(Sleef_quad); +SLEEF_IMPORT SLEEF_CONST Sleef_quad2 Sleef_sincospiq_u35(Sleef_quad); +#endif +#ifdef __SSE2__ + +#ifndef Sleef___m128d_2_DEFINED +typedef struct { + __m128d x, y; +} Sleef___m128d_2; +#define Sleef___m128d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u35(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u10(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_powd2_u10(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastsind2_u3500(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastcosd2_u3500(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastpowd2_u3500(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asinhd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acoshd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atanhd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expm1d2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log10d2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log1pd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u05(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinpid2_u05(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cospid2_u05(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ldexpd2(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmad2(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u05(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u35(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u05(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u35(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fabsd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_copysignd2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmaxd2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmind2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fdimd2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_truncd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_floord2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ceild2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_roundd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_rintd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_nextafterd2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_frfrexpd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmodd2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_remainderd2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_modfd2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_lgammad2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tgammad2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfd2_u10(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfcd2_u15(__m128d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd2(int); + +#ifndef Sleef___m128_2_DEFINED +typedef struct { + __m128 x, y; +} Sleef___m128_2; +#define Sleef___m128_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u35(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u10(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_powf4_u10(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastsinf4_u3500(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastcosf4_u3500(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastpowf4_u3500(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinhf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acoshf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanhf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expm1f4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log10f4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log1pf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u05(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinpif4_u05(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cospif4_u05(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaf4(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u05(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u35(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u05(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u35(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fabsf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_copysignf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaxf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fminf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fdimf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_truncf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_floorf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_ceilf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_roundf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_rintf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_nextafterf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_frfrexpf4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmodf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_remainderf4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_modff4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_lgammaf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tgammaf4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erff4_u10(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erfcf4_u15(__m128); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf4(int); +#endif +#ifdef __SSE2__ + +#ifndef Sleef___m128d_2_DEFINED +typedef struct { + __m128d x, y; +} Sleef___m128d_2; +#define Sleef___m128d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sind2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cosd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincosd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tand2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_asind2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_acosd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atand2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u35sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atan2d2_u35sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_logd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cbrtd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sind2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cosd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincosd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tand2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_asind2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_acosd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atand2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u10sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atan2d2_u10sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_logd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cbrtd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_expd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_powd2_u10sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_powd2_u10sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sinhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_coshd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tanhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sinhd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_coshd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tanhd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastsind2_u3500sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fastsind2_u3500sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastcosd2_u3500sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fastcosd2_u3500sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastpowd2_u3500sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fastpowd2_u3500sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asinhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_asinhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acoshd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_acoshd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atanhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atanhd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp2d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp2d2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp10d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp10d2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expm1d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_expm1d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log10d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log10d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log2d2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log2d2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log1pd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log1pd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincospid2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincospid2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinpid2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sinpid2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cospid2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cospid2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ldexpd2_sse2(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_ldexpd2_sse2(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_cinz_ilogbd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmad2_sse2(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmad2_sse2(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sqrtd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sqrtd2_u05sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sqrtd2_u35sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u05sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_hypotd2_u05sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u35sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_hypotd2_u35sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fabsd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fabsd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_copysignd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_copysignd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmaxd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmaxd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmind2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmind2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fdimd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fdimd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_truncd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_truncd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_floord2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_floord2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ceild2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_ceild2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_roundd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_roundd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_rintd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_rintd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_nextafterd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_nextafterd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_frfrexpd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_frfrexpd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_cinz_expfrexpd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmodd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmodd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_remainderd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_remainderd2_sse2(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_modfd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_modfd2_sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_lgammad2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_lgammad2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tgammad2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tgammad2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_erfd2_u10sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfcd2_u15sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_erfcd2_u15sse2(__m128d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd2_sse2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd2_sse2(int); + +#ifndef Sleef___m128_2_DEFINED +typedef struct { + __m128 x, y; +} Sleef___m128_2; +#define Sleef___m128_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cosf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincosf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_asinf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_acosf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atanf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u35sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atan2f4_u35sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_logf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cbrtf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cosf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincosf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_asinf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_acosf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atanf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u10sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atan2f4_u10sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_logf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cbrtf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_expf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_powf4_u10sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_powf4_u10sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_coshf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinhf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_coshf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanhf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastsinf4_u3500sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fastsinf4_u3500sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastcosf4_u3500sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fastcosf4_u3500sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastpowf4_u3500sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fastpowf4_u3500sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_asinhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acoshf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_acoshf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atanhf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp2f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp2f4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp10f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp10f4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expm1f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_expm1f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log10f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log10f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log2f4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log2f4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log1pf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log1pf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincospif4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincospif4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinpif4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinpif4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cospif4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cospif4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaf4_sse2(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fmaf4_sse2(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sqrtf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sqrtf4_u05sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sqrtf4_u35sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u05sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_hypotf4_u05sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u35sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_hypotf4_u35sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fabsf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fabsf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_copysignf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_copysignf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaxf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fmaxf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fminf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fminf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fdimf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fdimf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_truncf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_truncf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_floorf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_floorf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_ceilf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_ceilf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_roundf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_roundf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_rintf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_rintf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_nextafterf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_nextafterf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_frfrexpf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_frfrexpf4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmodf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fmodf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_remainderf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_remainderf4_sse2(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_modff4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_modff4_sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_lgammaf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_lgammaf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tgammaf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tgammaf4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erff4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_erff4_u10sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erfcf4_u15sse2(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_erfcf4_u15sse2(__m128); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf4_sse2(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_cinz_getIntf4_sse2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf4_sse2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_cinz_getPtrf4_sse2(int); +#endif +#ifdef __SSE2__ + +#ifndef Sleef___m128d_2_DEFINED +typedef struct { + __m128d x, y; +} Sleef___m128d_2; +#define Sleef___m128d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sind2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cosd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincosd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tand2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_asind2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_acosd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atand2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u35sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atan2d2_u35sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_logd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cbrtd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sind2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cosd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincosd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tand2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_asind2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_acosd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atand2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u10sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atan2d2_u10sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_logd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cbrtd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_expd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_powd2_u10sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_powd2_u10sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sinhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_coshd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tanhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sinhd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_coshd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tanhd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastsind2_u3500sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fastsind2_u3500sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastcosd2_u3500sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fastcosd2_u3500sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastpowd2_u3500sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fastpowd2_u3500sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asinhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_asinhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acoshd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_acoshd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atanhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_atanhd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp2d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp2d2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp10d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_exp10d2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expm1d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_expm1d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log10d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log10d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log2d2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log2d2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log1pd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_log1pd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincospid2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_sincospid2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinpid2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sinpid2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cospid2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_cospid2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ldexpd2_sse4(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_ldexpd2_sse4(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_cinz_ilogbd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmad2_sse4(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmad2_sse4(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sqrtd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sqrtd2_u05sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_sqrtd2_u35sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u05sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_hypotd2_u05sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u35sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_hypotd2_u35sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fabsd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fabsd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_copysignd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_copysignd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmaxd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmaxd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmind2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmind2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fdimd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fdimd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_truncd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_truncd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_floord2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_floord2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ceild2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_ceild2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_roundd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_roundd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_rintd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_rintd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_nextafterd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_nextafterd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_frfrexpd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_frfrexpd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_cinz_expfrexpd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmodd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_fmodd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_remainderd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_remainderd2_sse4(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_modfd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_cinz_modfd2_sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_lgammad2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_lgammad2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tgammad2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_tgammad2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_erfd2_u10sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfcd2_u15sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cinz_erfcd2_u15sse4(__m128d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd2_sse4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd2_sse4(int); + +#ifndef Sleef___m128_2_DEFINED +typedef struct { + __m128 x, y; +} Sleef___m128_2; +#define Sleef___m128_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cosf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincosf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_asinf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_acosf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atanf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u35sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atan2f4_u35sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_logf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cbrtf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cosf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincosf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_asinf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_acosf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atanf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u10sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atan2f4_u10sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_logf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cbrtf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_expf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_powf4_u10sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_powf4_u10sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_coshf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinhf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_coshf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tanhf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastsinf4_u3500sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fastsinf4_u3500sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastcosf4_u3500sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fastcosf4_u3500sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastpowf4_u3500sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fastpowf4_u3500sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_asinhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acoshf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_acoshf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_atanhf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp2f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp2f4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp10f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_exp10f4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expm1f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_expm1f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log10f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log10f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log2f4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log2f4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log1pf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_log1pf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincospif4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_sincospif4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinpif4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sinpif4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cospif4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_cospif4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaf4_sse4(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fmaf4_sse4(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sqrtf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sqrtf4_u05sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_sqrtf4_u35sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u05sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_hypotf4_u05sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u35sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_hypotf4_u35sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fabsf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fabsf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_copysignf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_copysignf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaxf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fmaxf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fminf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fminf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fdimf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fdimf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_truncf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_truncf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_floorf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_floorf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_ceilf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_ceilf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_roundf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_roundf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_rintf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_rintf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_nextafterf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_nextafterf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_frfrexpf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_frfrexpf4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmodf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_fmodf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_remainderf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_remainderf4_sse4(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_modff4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_cinz_modff4_sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_lgammaf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_lgammaf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tgammaf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_tgammaf4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erff4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_erff4_u10sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erfcf4_u15sse4(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cinz_erfcf4_u15sse4(__m128); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf4_sse4(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_cinz_getIntf4_sse4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf4_sse4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_cinz_getPtrf4_sse4(int); +#endif +#ifdef __AVX__ + +#ifndef Sleef___m256d_2_DEFINED +typedef struct { + __m256d x, y; +} Sleef___m256d_2; +#define Sleef___m256d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u35(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u10(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_powd4_u10(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastsind4_u3500(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastcosd4_u3500(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastpowd4_u3500(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asinhd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acoshd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atanhd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expm1d4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log10d4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log1pd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u05(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinpid4_u05(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cospid4_u05(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ldexpd4(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmad4(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u05(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u35(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u05(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u35(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fabsd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_copysignd4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmaxd4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmind4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fdimd4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_truncd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_floord4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ceild4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_roundd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_rintd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_nextafterd4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_frfrexpd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmodd4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_remainderd4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_modfd4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_lgammad4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tgammad4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfd4_u10(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfcd4_u15(__m256d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd4(int); + +#ifndef Sleef___m256_2_DEFINED +typedef struct { + __m256 x, y; +} Sleef___m256_2; +#define Sleef___m256_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u35(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u10(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_powf8_u10(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastsinf8_u3500(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastcosf8_u3500(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastpowf8_u3500(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinhf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acoshf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanhf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expm1f8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log10f8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log1pf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u05(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinpif8_u05(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cospif8_u05(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaf8(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u05(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u35(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u05(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u35(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fabsf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_copysignf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaxf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fminf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fdimf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_truncf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_floorf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_ceilf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_roundf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_rintf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_nextafterf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_frfrexpf8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmodf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_remainderf8(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_modff8(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_lgammaf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tgammaf8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erff8_u10(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erfcf8_u15(__m256); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf8(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf8(int); +#endif +#ifdef __AVX__ + +#ifndef Sleef___m256d_2_DEFINED +typedef struct { + __m256d x, y; +} Sleef___m256d_2; +#define Sleef___m256d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sind4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_cosd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_cinz_sincosd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_tand4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_asind4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_acosd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_atand4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u35avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_atan2d4_u35avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_logd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_cbrtd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sind4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_cosd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_cinz_sincosd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_tand4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_asind4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_acosd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_atand4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u10avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_atan2d4_u10avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_logd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_cbrtd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_expd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_powd4_u10avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_powd4_u10avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sinhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_coshd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_tanhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sinhd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_coshd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_tanhd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastsind4_u3500avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fastsind4_u3500avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastcosd4_u3500avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fastcosd4_u3500avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastpowd4_u3500avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fastpowd4_u3500avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asinhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_asinhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acoshd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_acoshd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atanhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_atanhd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_exp2d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_exp2d4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_exp10d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_exp10d4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expm1d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_expm1d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log10d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_log10d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_log2d4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_log2d4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log1pd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_log1pd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_cinz_sincospid4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_cinz_sincospid4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinpid4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sinpid4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cospid4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_cospid4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ldexpd4_avx(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_ldexpd4_avx(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_cinz_ilogbd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmad4_avx(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fmad4_avx(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sqrtd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sqrtd4_u05avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_sqrtd4_u35avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u05avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_hypotd4_u05avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u35avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_hypotd4_u35avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fabsd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fabsd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_copysignd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_copysignd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmaxd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fmaxd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmind4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fmind4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fdimd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fdimd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_truncd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_truncd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_floord4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_floord4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ceild4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_ceild4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_roundd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_roundd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_rintd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_rintd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_nextafterd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_nextafterd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_frfrexpd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_frfrexpd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_cinz_expfrexpd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmodd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_fmodd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_remainderd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_remainderd4_avx(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_modfd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_cinz_modfd4_avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_lgammad4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_lgammad4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tgammad4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_tgammad4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_erfd4_u10avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfcd4_u15avx(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cinz_erfcd4_u15avx(__m256d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd4_avx(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd4_avx(int); + +#ifndef Sleef___m256_2_DEFINED +typedef struct { + __m256 x, y; +} Sleef___m256_2; +#define Sleef___m256_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sinf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_cosf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_cinz_sincosf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_tanf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_asinf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_acosf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_atanf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u35avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_atan2f8_u35avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_logf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_cbrtf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sinf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_cosf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_cinz_sincosf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_tanf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_asinf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_acosf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_atanf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u10avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_atan2f8_u10avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_logf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_cbrtf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_expf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_powf8_u10avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_powf8_u10avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sinhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_coshf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_tanhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sinhf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_coshf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_tanhf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastsinf8_u3500avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fastsinf8_u3500avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastcosf8_u3500avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fastcosf8_u3500avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastpowf8_u3500avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fastpowf8_u3500avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_asinhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acoshf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_acoshf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_atanhf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_exp2f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_exp2f8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_exp10f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_exp10f8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expm1f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_expm1f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log10f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_log10f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_log2f8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_log2f8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log1pf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_log1pf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_cinz_sincospif8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_cinz_sincospif8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinpif8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sinpif8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cospif8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_cospif8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaf8_avx(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fmaf8_avx(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sqrtf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sqrtf8_u05avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_sqrtf8_u35avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u05avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_hypotf8_u05avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u35avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_hypotf8_u35avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fabsf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fabsf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_copysignf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_copysignf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaxf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fmaxf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fminf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fminf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fdimf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fdimf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_truncf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_truncf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_floorf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_floorf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_ceilf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_ceilf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_roundf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_roundf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_rintf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_rintf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_nextafterf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_nextafterf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_frfrexpf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_frfrexpf8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmodf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_fmodf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_remainderf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_remainderf8_avx(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_modff8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_cinz_modff8_avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_lgammaf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_lgammaf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tgammaf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_tgammaf8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erff8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_erff8_u10avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erfcf8_u15avx(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cinz_erfcf8_u15avx(__m256); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf8_avx(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_cinz_getIntf8_avx(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf8_avx(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_cinz_getPtrf8_avx(int); +#endif +#ifdef __AVX__ + +#ifndef Sleef___m256d_2_DEFINED +typedef struct { + __m256d x, y; +} Sleef___m256d_2; +#define Sleef___m256d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sind4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cosd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincosd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tand4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_asind4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_acosd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atand4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u35fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atan2d4_u35fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_logd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cbrtd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sind4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cosd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincosd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tand4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_asind4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_acosd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atand4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u10fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atan2d4_u10fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_logd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cbrtd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_expd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_powd4_u10fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_powd4_u10fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sinhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_coshd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tanhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sinhd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_coshd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tanhd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastsind4_u3500fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fastsind4_u3500fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastcosd4_u3500fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fastcosd4_u3500fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastpowd4_u3500fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fastpowd4_u3500fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asinhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_asinhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acoshd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_acoshd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atanhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atanhd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp2d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp2d4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp10d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp10d4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expm1d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_expm1d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log10d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log10d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log2d4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log2d4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log1pd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log1pd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincospid4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincospid4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinpid4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sinpid4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cospid4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cospid4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ldexpd4_fma4(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_ldexpd4_fma4(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_finz_ilogbd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmad4_fma4(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmad4_fma4(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sqrtd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sqrtd4_u05fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sqrtd4_u35fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u05fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_hypotd4_u05fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u35fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_hypotd4_u35fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fabsd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fabsd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_copysignd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_copysignd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmaxd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmaxd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmind4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmind4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fdimd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fdimd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_truncd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_truncd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_floord4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_floord4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ceild4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_ceild4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_roundd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_roundd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_rintd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_rintd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_nextafterd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_nextafterd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_frfrexpd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_frfrexpd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_finz_expfrexpd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmodd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmodd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_remainderd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_remainderd4_fma4(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_modfd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_modfd4_fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_lgammad4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_lgammad4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tgammad4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tgammad4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_erfd4_u10fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfcd4_u15fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_erfcd4_u15fma4(__m256d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd4_fma4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd4_fma4(int); + +#ifndef Sleef___m256_2_DEFINED +typedef struct { + __m256 x, y; +} Sleef___m256_2; +#define Sleef___m256_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cosf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincosf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_asinf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_acosf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atanf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u35fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atan2f8_u35fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_logf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cbrtf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cosf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincosf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_asinf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_acosf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atanf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u10fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atan2f8_u10fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_logf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cbrtf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_expf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_powf8_u10fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_powf8_u10fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_coshf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinhf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_coshf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanhf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastsinf8_u3500fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fastsinf8_u3500fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastcosf8_u3500fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fastcosf8_u3500fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastpowf8_u3500fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fastpowf8_u3500fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_asinhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acoshf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_acoshf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atanhf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp2f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp2f8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp10f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp10f8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expm1f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_expm1f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log10f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log10f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log2f8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log2f8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log1pf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log1pf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincospif8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincospif8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinpif8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinpif8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cospif8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cospif8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaf8_fma4(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fmaf8_fma4(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sqrtf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sqrtf8_u05fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sqrtf8_u35fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u05fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_hypotf8_u05fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u35fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_hypotf8_u35fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fabsf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fabsf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_copysignf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_copysignf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaxf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fmaxf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fminf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fminf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fdimf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fdimf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_truncf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_truncf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_floorf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_floorf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_ceilf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_ceilf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_roundf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_roundf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_rintf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_rintf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_nextafterf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_nextafterf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_frfrexpf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_frfrexpf8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmodf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fmodf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_remainderf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_remainderf8_fma4(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_modff8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_modff8_fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_lgammaf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_lgammaf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tgammaf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tgammaf8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erff8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_erff8_u10fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erfcf8_u15fma4(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_erfcf8_u15fma4(__m256); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf8_fma4(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_finz_getIntf8_fma4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf8_fma4(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_finz_getPtrf8_fma4(int); +#endif +#ifdef __AVX__ + +#ifndef Sleef___m256d_2_DEFINED +typedef struct { + __m256d x, y; +} Sleef___m256d_2; +#define Sleef___m256d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sind4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cosd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincosd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tand4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_asind4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_acosd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atand4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u35avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atan2d4_u35avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_logd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cbrtd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sind4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sind4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cosd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cosd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincosd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincosd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tand4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tand4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asind4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_asind4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acosd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_acosd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atand4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atand4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atan2d4_u10avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atan2d4_u10avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_logd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_logd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cbrtd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cbrtd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_expd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_powd4_u10avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_powd4_u10avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sinhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_coshd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tanhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinhd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sinhd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_coshd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_coshd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tanhd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tanhd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastsind4_u3500avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fastsind4_u3500avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastcosd4_u3500avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fastcosd4_u3500avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fastpowd4_u3500avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fastpowd4_u3500avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_asinhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_asinhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_acoshd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_acoshd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_atanhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_atanhd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp2d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp2d4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp2d4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp10d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_exp10d4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_exp10d4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_expm1d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_expm1d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log10d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log10d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log2d4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log2d4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log2d4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_log1pd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_log1pd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincospid4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_sincospid4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_sincospid4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sinpid4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sinpid4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_cospid4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_cospid4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ldexpd4_avx2(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_ldexpd4_avx2(__m256d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_finz_ilogbd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmad4_avx2(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmad4_avx2(__m256d, __m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sqrtd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sqrtd4_u05avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_sqrtd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_sqrtd4_u35avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u05avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_hypotd4_u05avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_hypotd4_u35avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_hypotd4_u35avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fabsd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fabsd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_copysignd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_copysignd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmaxd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmaxd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmind4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmind4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fdimd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fdimd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_truncd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_truncd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_floord4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_floord4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_ceild4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_ceild4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_roundd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_roundd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_rintd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_rintd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_nextafterd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_nextafterd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_frfrexpd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_frfrexpd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_finz_expfrexpd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_fmodd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_fmodd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_remainderd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_remainderd4_avx2(__m256d, __m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_modfd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST Sleef___m256d_2 Sleef_finz_modfd4_avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_lgammad4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_lgammad4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_tgammad4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_tgammad4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_erfd4_u10avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_erfcd4_u15avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST __m256d Sleef_finz_erfcd4_u15avx2(__m256d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd4_avx2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd4_avx2(int); + +#ifndef Sleef___m256_2_DEFINED +typedef struct { + __m256 x, y; +} Sleef___m256_2; +#define Sleef___m256_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cosf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincosf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_asinf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_acosf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atanf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u35avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atan2f8_u35avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_logf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cbrtf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cosf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cosf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincosf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincosf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_asinf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acosf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_acosf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atanf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atan2f8_u10avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atan2f8_u10avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_logf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_logf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cbrtf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cbrtf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_expf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_powf8_u10avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_powf8_u10avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_coshf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinhf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinhf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_coshf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_coshf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tanhf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tanhf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastsinf8_u3500avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fastsinf8_u3500avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastcosf8_u3500avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fastcosf8_u3500avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fastpowf8_u3500avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fastpowf8_u3500avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_asinhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_asinhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_acoshf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_acoshf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_atanhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_atanhf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp2f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp2f8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp2f8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp10f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_exp10f8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_exp10f8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_expm1f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_expm1f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log10f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log10f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log2f8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log2f8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log2f8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_log1pf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_log1pf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincospif8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_sincospif8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_sincospif8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sinpif8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sinpif8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_cospif8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_cospif8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaf8_avx2(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fmaf8_avx2(__m256, __m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sqrtf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sqrtf8_u05avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_sqrtf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_sqrtf8_u35avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u05avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_hypotf8_u05avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_hypotf8_u35avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_hypotf8_u35avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fabsf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fabsf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_copysignf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_copysignf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmaxf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fmaxf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fminf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fminf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fdimf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fdimf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_truncf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_truncf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_floorf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_floorf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_ceilf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_ceilf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_roundf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_roundf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_rintf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_rintf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_nextafterf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_nextafterf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_frfrexpf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_frfrexpf8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_fmodf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_fmodf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_remainderf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_remainderf8_avx2(__m256, __m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_modff8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST Sleef___m256_2 Sleef_finz_modff8_avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_lgammaf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_lgammaf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_tgammaf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_tgammaf8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erff8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_erff8_u10avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_erfcf8_u15avx2(__m256); +SLEEF_IMPORT SLEEF_CONST __m256 Sleef_finz_erfcf8_u15avx2(__m256); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf8_avx2(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_finz_getIntf8_avx2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf8_avx2(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_finz_getPtrf8_avx2(int); +#endif +#ifdef __SSE2__ + +#ifndef Sleef___m128d_2_DEFINED +typedef struct { + __m128d x, y; +} Sleef___m128d_2; +#define Sleef___m128d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sind2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_cosd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_finz_sincosd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_tand2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_asind2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_acosd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_atand2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u35avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_atan2d2_u35avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_logd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_cbrtd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sind2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sind2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cosd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_cosd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincosd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_finz_sincosd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tand2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_tand2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asind2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_asind2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acosd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_acosd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atand2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_atand2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atan2d2_u10avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_atan2d2_u10avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_logd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_logd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cbrtd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_cbrtd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_expd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_powd2_u10avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_powd2_u10avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sinhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_coshd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_tanhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinhd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sinhd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_coshd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_coshd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tanhd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_tanhd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastsind2_u3500avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fastsind2_u3500avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastcosd2_u3500avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fastcosd2_u3500avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fastpowd2_u3500avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fastpowd2_u3500avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_asinhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_asinhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_acoshd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_acoshd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_atanhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_atanhd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_exp2d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp2d2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_exp2d2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_exp10d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_exp10d2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_exp10d2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_expm1d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_expm1d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log10d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_log10d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_log2d2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log2d2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_log2d2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_log1pd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_log1pd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_finz_sincospid2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_sincospid2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_finz_sincospid2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sinpid2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sinpid2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_cospid2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_cospid2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ldexpd2_avx2128(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_ldexpd2_avx2128(__m128d, __m128i); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_ilogbd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_finz_ilogbd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmad2_avx2128(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fmad2_avx2128(__m128d, __m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sqrtd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sqrtd2_u05avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_sqrtd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_sqrtd2_u35avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u05avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_hypotd2_u05avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_hypotd2_u35avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_hypotd2_u35avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fabsd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fabsd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_copysignd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_copysignd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmaxd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fmaxd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmind2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fmind2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fdimd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fdimd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_truncd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_truncd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_floord2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_floord2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_ceild2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_ceild2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_roundd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_roundd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_rintd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_rintd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_nextafterd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_nextafterd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_frfrexpd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_frfrexpd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_expfrexpd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128i Sleef_finz_expfrexpd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_fmodd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_fmodd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_remainderd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_remainderd2_avx2128(__m128d, __m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_modfd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST Sleef___m128d_2 Sleef_finz_modfd2_avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_lgammad2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_lgammad2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_tgammad2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_tgammad2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_erfd2_u10avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_erfcd2_u15avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST __m128d Sleef_finz_erfcd2_u15avx2128(__m128d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd2_avx2128(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd2_avx2128(int); + +#ifndef Sleef___m128_2_DEFINED +typedef struct { + __m128 x, y; +} Sleef___m128_2; +#define Sleef___m128_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sinf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_cosf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_finz_sincosf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_tanf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_asinf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_acosf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_atanf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u35avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_atan2f4_u35avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_logf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_cbrtf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sinf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cosf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_cosf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincosf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_finz_sincosf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_tanf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_asinf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acosf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_acosf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_atanf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atan2f4_u10avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_atan2f4_u10avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_logf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_logf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cbrtf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_cbrtf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_expf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_powf4_u10avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_powf4_u10avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sinhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_coshf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_tanhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinhf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sinhf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_coshf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_coshf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tanhf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_tanhf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastsinf4_u3500avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fastsinf4_u3500avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastcosf4_u3500avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fastcosf4_u3500avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fastpowf4_u3500avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fastpowf4_u3500avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_asinhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_asinhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_acoshf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_acoshf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_atanhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_atanhf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_exp2f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp2f4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_exp2f4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_exp10f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_exp10f4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_exp10f4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_expm1f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_expm1f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log10f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_log10f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_log2f4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log2f4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_log2f4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_log1pf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_log1pf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_finz_sincospif4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_sincospif4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_finz_sincospif4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sinpif4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sinpif4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_cospif4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_cospif4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaf4_avx2128(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fmaf4_avx2128(__m128, __m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sqrtf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sqrtf4_u05avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_sqrtf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_sqrtf4_u35avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u05avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_hypotf4_u05avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_hypotf4_u35avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_hypotf4_u35avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fabsf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fabsf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_copysignf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_copysignf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmaxf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fmaxf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fminf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fminf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fdimf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fdimf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_truncf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_truncf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_floorf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_floorf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_ceilf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_ceilf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_roundf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_roundf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_rintf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_rintf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_nextafterf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_nextafterf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_frfrexpf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_frfrexpf4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_fmodf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_fmodf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_remainderf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_remainderf4_avx2128(__m128, __m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_modff4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST Sleef___m128_2 Sleef_finz_modff4_avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_lgammaf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_lgammaf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_tgammaf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_tgammaf4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erff4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_erff4_u10avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_erfcf4_u15avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST __m128 Sleef_finz_erfcf4_u15avx2128(__m128); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf4_avx2128(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_finz_getIntf4_avx2128(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf4_avx2128(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_finz_getPtrf4_avx2128(int); +#endif +#ifdef __AVX512F__ + +#ifndef Sleef___m512d_2_DEFINED +typedef struct { + __m512d x, y; +} Sleef___m512d_2; +#define Sleef___m512d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sind8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cosd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincosd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tand8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asind8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acosd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atand8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atan2d8_u35(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_logd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cbrtd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sind8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cosd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincosd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tand8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asind8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acosd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atand8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atan2d8_u10(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_logd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cbrtd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_expd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_powd8_u10(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinhd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_coshd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tanhd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinhd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_coshd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tanhd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastsind8_u3500(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastcosd8_u3500(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastpowd8_u3500(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asinhd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acoshd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atanhd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp2d8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp2d8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp10d8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp10d8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_expm1d8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log10d8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log2d8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log2d8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log1pd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincospid8_u05(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincospid8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinpid8_u05(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cospid8_u05(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_ldexpd8(__m512d, __m256i); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_ilogbd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmad8(__m512d, __m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_u05(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_u35(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_hypotd8_u05(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_hypotd8_u35(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fabsd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_copysignd8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmaxd8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmind8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fdimd8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_truncd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_floord8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_ceild8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_roundd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_rintd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_nextafterd8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_frfrexpd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_expfrexpd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmodd8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_remainderd8(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_modfd8(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_lgammad8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tgammad8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_erfd8_u10(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_erfcd8_u15(__m512d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd8(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd8(int); + +#ifndef Sleef___m512_2_DEFINED +typedef struct { + __m512 x, y; +} Sleef___m512_2; +#define Sleef___m512_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cosf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincosf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acosf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atan2f16_u35(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_logf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cbrtf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cosf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincosf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acosf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atan2f16_u10(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_logf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cbrtf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_expf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_powf16_u10(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinhf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_coshf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanhf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinhf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_coshf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanhf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastsinf16_u3500(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastcosf16_u3500(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastpowf16_u3500(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinhf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acoshf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanhf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp2f16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp2f16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp10f16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp10f16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_expm1f16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log10f16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log2f16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log2f16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log1pf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincospif16_u05(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincospif16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinpif16_u05(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cospif16_u05(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmaf16(__m512, __m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_u05(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_u35(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_hypotf16_u05(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_hypotf16_u35(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fabsf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_copysignf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmaxf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fminf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fdimf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_truncf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_floorf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_ceilf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_roundf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_rintf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_nextafterf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_frfrexpf16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmodf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_remainderf16(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_modff16(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_lgammaf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tgammaf16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_erff16_u10(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_erfcf16_u15(__m512); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf16(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf16(int); +#endif +#ifdef __AVX512F__ + +#ifndef Sleef___m512d_2_DEFINED +typedef struct { + __m512d x, y; +} Sleef___m512d_2; +#define Sleef___m512d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sind8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sind8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cosd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_cosd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincosd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_finz_sincosd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tand8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_tand8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asind8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_asind8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acosd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_acosd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atand8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_atand8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atan2d8_u35avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_atan2d8_u35avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_logd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_logd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cbrtd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_cbrtd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sind8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sind8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cosd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_cosd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincosd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_finz_sincosd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tand8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_tand8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asind8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_asind8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acosd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_acosd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atand8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_atand8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atan2d8_u10avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_atan2d8_u10avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_logd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_logd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cbrtd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_cbrtd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_expd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_expd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_powd8_u10avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_powd8_u10avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sinhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_coshd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_coshd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tanhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_tanhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinhd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sinhd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_coshd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_coshd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tanhd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_tanhd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastsind8_u3500avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fastsind8_u3500avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastcosd8_u3500avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fastcosd8_u3500avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastpowd8_u3500avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fastpowd8_u3500avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asinhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_asinhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acoshd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_acoshd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atanhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_atanhd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp2d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_exp2d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp2d8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_exp2d8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp10d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_exp10d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp10d8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_exp10d8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_expm1d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_expm1d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log10d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_log10d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log2d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_log2d8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log2d8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_log2d8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log1pd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_log1pd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincospid8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_finz_sincospid8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincospid8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_finz_sincospid8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinpid8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sinpid8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cospid8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_cospid8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_ldexpd8_avx512f(__m512d, __m256i); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_ldexpd8_avx512f(__m512d, __m256i); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_ilogbd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_finz_ilogbd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmad8_avx512f(__m512d, __m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fmad8_avx512f(__m512d, __m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sqrtd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sqrtd8_u05avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_sqrtd8_u35avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_hypotd8_u05avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_hypotd8_u05avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_hypotd8_u35avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_hypotd8_u35avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fabsd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fabsd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_copysignd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_copysignd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmaxd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fmaxd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmind8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fmind8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fdimd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fdimd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_truncd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_truncd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_floord8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_floord8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_ceild8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_ceild8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_roundd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_roundd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_rintd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_rintd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_nextafterd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_nextafterd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_frfrexpd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_frfrexpd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_expfrexpd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_finz_expfrexpd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmodd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_fmodd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_remainderd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_remainderd8_avx512f(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_modfd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_finz_modfd8_avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_lgammad8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_lgammad8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tgammad8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_tgammad8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_erfd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_erfd8_u10avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_erfcd8_u15avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_finz_erfcd8_u15avx512f(__m512d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd8_avx512f(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd8_avx512f(int); + +#ifndef Sleef___m512_2_DEFINED +typedef struct { + __m512 x, y; +} Sleef___m512_2; +#define Sleef___m512_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sinf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cosf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_cosf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincosf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_finz_sincosf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_tanf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_asinf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acosf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_acosf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_atanf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atan2f16_u35avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_atan2f16_u35avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_logf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_logf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cbrtf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_cbrtf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sinf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cosf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_cosf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincosf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_finz_sincosf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_tanf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_asinf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acosf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_acosf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_atanf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atan2f16_u10avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_atan2f16_u10avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_logf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_logf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cbrtf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_cbrtf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_expf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_expf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_powf16_u10avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_powf16_u10avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sinhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_coshf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_coshf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_tanhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinhf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sinhf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_coshf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_coshf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanhf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_tanhf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastsinf16_u3500avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fastsinf16_u3500avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastcosf16_u3500avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fastcosf16_u3500avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastpowf16_u3500avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fastpowf16_u3500avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_asinhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acoshf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_acoshf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_atanhf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp2f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_exp2f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp2f16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_exp2f16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp10f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_exp10f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp10f16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_exp10f16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_expm1f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_expm1f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log10f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_log10f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log2f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_log2f16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log2f16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_log2f16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log1pf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_log1pf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincospif16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_finz_sincospif16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincospif16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_finz_sincospif16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinpif16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sinpif16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cospif16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_cospif16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmaf16_avx512f(__m512, __m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fmaf16_avx512f(__m512, __m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sqrtf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sqrtf16_u05avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_sqrtf16_u35avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_hypotf16_u05avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_hypotf16_u05avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_hypotf16_u35avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_hypotf16_u35avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fabsf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fabsf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_copysignf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_copysignf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmaxf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fmaxf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fminf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fminf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fdimf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fdimf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_truncf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_truncf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_floorf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_floorf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_ceilf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_ceilf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_roundf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_roundf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_rintf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_rintf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_nextafterf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_nextafterf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_frfrexpf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_frfrexpf16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmodf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_fmodf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_remainderf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_remainderf16_avx512f(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_modff16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_finz_modff16_avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_lgammaf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_lgammaf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tgammaf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_tgammaf16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_erff16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_erff16_u10avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_erfcf16_u15avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_finz_erfcf16_u15avx512f(__m512); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf16_avx512f(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_finz_getIntf16_avx512f(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf16_avx512f(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_finz_getPtrf16_avx512f(int); +#endif +#ifdef __AVX512F__ + +#ifndef Sleef___m512d_2_DEFINED +typedef struct { + __m512d x, y; +} Sleef___m512d_2; +#define Sleef___m512d_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sind8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sind8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cosd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_cosd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincosd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_cinz_sincosd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tand8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_tand8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asind8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_asind8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acosd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_acosd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atand8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_atand8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atan2d8_u35avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_atan2d8_u35avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_logd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_logd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cbrtd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_cbrtd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sind8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sind8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cosd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_cosd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincosd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_cinz_sincosd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tand8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_tand8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asind8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_asind8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acosd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_acosd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atand8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_atand8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atan2d8_u10avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_atan2d8_u10avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_logd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_logd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cbrtd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_cbrtd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_expd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_expd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_powd8_u10avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_powd8_u10avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sinhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_coshd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_coshd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tanhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_tanhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinhd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sinhd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_coshd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_coshd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tanhd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_tanhd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastsind8_u3500avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fastsind8_u3500avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastcosd8_u3500avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fastcosd8_u3500avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fastpowd8_u3500avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fastpowd8_u3500avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_asinhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_asinhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_acoshd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_acoshd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_atanhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_atanhd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp2d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_exp2d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp2d8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_exp2d8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp10d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_exp10d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_exp10d8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_exp10d8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_expm1d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_expm1d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log10d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_log10d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log2d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_log2d8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log2d8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_log2d8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_log1pd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_log1pd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincospid8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_cinz_sincospid8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_sincospid8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_cinz_sincospid8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sinpid8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sinpid8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cospid8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_cospid8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_ldexpd8_avx512fnofma(__m512d, __m256i); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_ldexpd8_avx512fnofma(__m512d, __m256i); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_ilogbd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_cinz_ilogbd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmad8_avx512fnofma(__m512d, __m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fmad8_avx512fnofma(__m512d, __m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sqrtd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sqrtd8_u05avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_sqrtd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_sqrtd8_u35avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_hypotd8_u05avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_hypotd8_u05avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_hypotd8_u35avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_hypotd8_u35avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fabsd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fabsd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_copysignd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_copysignd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmaxd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fmaxd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmind8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fmind8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fdimd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fdimd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_truncd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_truncd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_floord8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_floord8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_ceild8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_ceild8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_roundd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_roundd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_rintd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_rintd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_nextafterd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_nextafterd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_frfrexpd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_frfrexpd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_expfrexpd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m256i Sleef_cinz_expfrexpd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_fmodd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_fmodd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_remainderd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_remainderd8_avx512fnofma(__m512d, __m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_modfd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST Sleef___m512d_2 Sleef_cinz_modfd8_avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_lgammad8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_lgammad8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_tgammad8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_tgammad8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_erfd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_erfd8_u10avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_erfcd8_u15avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST __m512d Sleef_cinz_erfcd8_u15avx512fnofma(__m512d); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd8_avx512fnofma(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd8_avx512fnofma(int); + +#ifndef Sleef___m512_2_DEFINED +typedef struct { + __m512 x, y; +} Sleef___m512_2; +#define Sleef___m512_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sinf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cosf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_cosf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincosf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_cinz_sincosf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_tanf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_asinf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acosf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_acosf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_atanf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atan2f16_u35avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_atan2f16_u35avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_logf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_logf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cbrtf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_cbrtf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sinf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cosf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_cosf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincosf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_cinz_sincosf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_tanf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_asinf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acosf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_acosf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_atanf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atan2f16_u10avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_atan2f16_u10avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_logf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_logf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cbrtf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_cbrtf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_expf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_expf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_powf16_u10avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_powf16_u10avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sinhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_coshf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_coshf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_tanhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinhf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sinhf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_coshf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_coshf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tanhf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_tanhf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastsinf16_u3500avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fastsinf16_u3500avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastcosf16_u3500avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fastcosf16_u3500avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fastpowf16_u3500avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fastpowf16_u3500avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_asinhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_asinhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_acoshf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_acoshf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_atanhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_atanhf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp2f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_exp2f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp2f16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_exp2f16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp10f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_exp10f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_exp10f16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_exp10f16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_expm1f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_expm1f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log10f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_log10f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log2f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_log2f16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log2f16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_log2f16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_log1pf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_log1pf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincospif16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_cinz_sincospif16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_sincospif16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_cinz_sincospif16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sinpif16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sinpif16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cospif16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_cospif16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmaf16_avx512fnofma(__m512, __m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fmaf16_avx512fnofma(__m512, __m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sqrtf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sqrtf16_u05avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_sqrtf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_sqrtf16_u35avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_hypotf16_u05avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_hypotf16_u05avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_hypotf16_u35avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_hypotf16_u35avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fabsf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fabsf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_copysignf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_copysignf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmaxf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fmaxf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fminf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fminf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fdimf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fdimf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_truncf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_truncf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_floorf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_floorf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_ceilf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_ceilf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_roundf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_roundf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_rintf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_rintf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_nextafterf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_nextafterf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_frfrexpf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_frfrexpf16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_fmodf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_fmodf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_remainderf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_remainderf16_avx512fnofma(__m512, __m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_modff16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST Sleef___m512_2 Sleef_cinz_modff16_avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_lgammaf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_lgammaf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_tgammaf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_tgammaf16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_erff16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_erff16_u10avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_erfcf16_u15avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST __m512 Sleef_cinz_erfcf16_u15avx512fnofma(__m512); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf16_avx512fnofma(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_cinz_getIntf16_avx512fnofma(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf16_avx512fnofma(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_cinz_getPtrf16_avx512fnofma(int); +#endif +#ifdef __STDC__ + +#ifndef Sleef_double_2_DEFINED +typedef Sleef_double2 Sleef_double_2; +#define Sleef_double_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST double Sleef_sind1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sind1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cosd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_cosd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincosd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_cinz_sincosd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tand1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_tand1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_asind1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_asind1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_acosd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_acosd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atand1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_atand1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atan2d1_u35purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_atan2d1_u35purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_logd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_logd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cbrtd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_cbrtd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sind1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sind1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cosd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_cosd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincosd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_cinz_sincosd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tand1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_tand1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_asind1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_asind1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_acosd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_acosd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atand1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_atand1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atan2d1_u10purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_atan2d1_u10purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_logd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_logd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cbrtd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_cbrtd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_expd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_expd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_powd1_u10purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_powd1_u10purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sinhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sinhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_coshd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_coshd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tanhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_tanhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sinhd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sinhd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_coshd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_coshd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tanhd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_tanhd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fastsind1_u3500purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fastsind1_u3500purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fastcosd1_u3500purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fastcosd1_u3500purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fastpowd1_u3500purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fastpowd1_u3500purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_asinhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_asinhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_acoshd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_acoshd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atanhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_atanhd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp2d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_exp2d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp2d1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_exp2d1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp10d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_exp10d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp10d1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_exp10d1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_expm1d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_expm1d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log10d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_log10d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log2d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_log2d1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log2d1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_log2d1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log1pd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_log1pd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincospid1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_cinz_sincospid1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincospid1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_cinz_sincospid1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sinpid1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sinpid1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cospid1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_cospid1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_ldexpd1_purec(double, int32_t); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_ldexpd1_purec(double, int32_t); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_ilogbd1_purec(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_cinz_ilogbd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmad1_purec(double, double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fmad1_purec(double, double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sqrtd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sqrtd1_u05purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_sqrtd1_u35purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_hypotd1_u05purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_hypotd1_u05purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_hypotd1_u35purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_hypotd1_u35purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fabsd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fabsd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_copysignd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_copysignd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmaxd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fmaxd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmind1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fmind1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fdimd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fdimd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_truncd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_truncd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_floord1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_floord1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_ceild1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_ceild1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_roundd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_roundd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_rintd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_rintd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_nextafterd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_nextafterd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_frfrexpd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_frfrexpd1_purec(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_expfrexpd1_purec(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_cinz_expfrexpd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmodd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_fmodd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_remainderd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_remainderd1_purec(double, double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_modfd1_purec(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_cinz_modfd1_purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_lgammad1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_lgammad1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tgammad1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_tgammad1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_erfd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_erfd1_u10purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_erfcd1_u15purec(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cinz_erfcd1_u15purec(double); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd1_purec(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd1_purec(int); + +#ifndef Sleef_float_2_DEFINED +typedef Sleef_float2 Sleef_float_2; +#define Sleef_float_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST float Sleef_sinf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sinf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cosf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_cosf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincosf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_cinz_sincosf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_tanf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_asinf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_asinf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_acosf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_acosf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atanf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_atanf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f1_u35purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_atan2f1_u35purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_logf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_logf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_cbrtf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sinf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cosf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_cosf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincosf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_cinz_sincosf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_tanf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_asinf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_asinf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_acosf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_acosf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atanf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_atanf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f1_u10purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_atan2f1_u10purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_logf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_logf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_cbrtf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_expf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_expf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_powf1_u10purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_powf1_u10purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sinhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_coshf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_coshf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_tanhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sinhf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_coshf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_coshf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_tanhf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fastsinf1_u3500purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fastsinf1_u3500purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fastcosf1_u3500purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fastcosf1_u3500purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fastpowf1_u3500purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fastpowf1_u3500purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_asinhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_asinhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_acoshf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_acoshf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atanhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_atanhf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_exp2f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_exp2f1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_exp10f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_exp10f1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_expm1f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_expm1f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log10f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_log10f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log2f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_log2f1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log2f1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_log2f1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log1pf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_log1pf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincospif1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_cinz_sincospif1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincospif1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_cinz_sincospif1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinpif1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sinpif1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cospif1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_cospif1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fmaf1_purec(float, float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fmaf1_purec(float, float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sqrtf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sqrtf1_u05purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_sqrtf1_u35purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf1_u05purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_hypotf1_u05purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf1_u35purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_hypotf1_u35purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fabsf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fabsf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_copysignf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_copysignf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fmaxf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fmaxf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fminf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fminf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fdimf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fdimf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_truncf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_truncf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_floorf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_floorf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_ceilf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_ceilf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_roundf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_roundf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_rintf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_rintf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_nextafterf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_nextafterf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_frfrexpf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_frfrexpf1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fmodf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_fmodf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_remainderf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_remainderf1_purec(float, float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_modff1_purec(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_cinz_modff1_purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_lgammaf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_lgammaf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tgammaf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_tgammaf1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_erff1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_erff1_u10purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_erfcf1_u15purec(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cinz_erfcf1_u15purec(float); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf1_purec(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_cinz_getIntf1_purec(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf1_purec(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_cinz_getPtrf1_purec(int); +#endif +#ifdef __STDC__ + +#ifndef Sleef_double_2_DEFINED +typedef Sleef_double2 Sleef_double_2; +#define Sleef_double_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST double Sleef_sind1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sind1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cosd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_cosd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincosd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_finz_sincosd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tand1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_tand1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_asind1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_asind1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_acosd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_acosd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atand1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_atand1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atan2d1_u35purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_atan2d1_u35purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_logd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_logd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cbrtd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_cbrtd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sind1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sind1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cosd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_cosd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincosd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_finz_sincosd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tand1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_tand1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_asind1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_asind1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_acosd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_acosd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atand1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_atand1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atan2d1_u10purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_atan2d1_u10purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_logd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_logd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cbrtd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_cbrtd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_expd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_expd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_powd1_u10purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_powd1_u10purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sinhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sinhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_coshd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_coshd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tanhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_tanhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sinhd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sinhd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_coshd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_coshd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tanhd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_tanhd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fastsind1_u3500purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fastsind1_u3500purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fastcosd1_u3500purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fastcosd1_u3500purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fastpowd1_u3500purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fastpowd1_u3500purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_asinhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_asinhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_acoshd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_acoshd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_atanhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_atanhd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp2d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_exp2d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp2d1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_exp2d1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp10d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_exp10d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_exp10d1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_exp10d1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_expm1d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_expm1d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log10d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_log10d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log2d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_log2d1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log2d1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_log2d1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_log1pd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_log1pd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincospid1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_finz_sincospid1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincospid1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_finz_sincospid1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sinpid1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sinpid1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_cospid1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_cospid1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_ldexpd1_purecfma(double, int32_t); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_ldexpd1_purecfma(double, int32_t); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_ilogbd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_finz_ilogbd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmad1_purecfma(double, double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fmad1_purecfma(double, double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sqrtd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sqrtd1_u05purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_sqrtd1_u35purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_hypotd1_u05purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_hypotd1_u05purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_hypotd1_u35purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_hypotd1_u35purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fabsd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fabsd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_copysignd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_copysignd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmaxd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fmaxd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmind1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fmind1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fdimd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fdimd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_truncd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_truncd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_floord1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_floord1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_ceild1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_ceild1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_roundd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_roundd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_rintd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_rintd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_nextafterd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_nextafterd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_frfrexpd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_frfrexpd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_expfrexpd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_finz_expfrexpd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmodd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_fmodd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_remainderd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_remainderd1_purecfma(double, double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_modfd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_finz_modfd1_purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_lgammad1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_lgammad1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_tgammad1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_tgammad1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_erfd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_erfd1_u10purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_erfcd1_u15purecfma(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_finz_erfcd1_u15purecfma(double); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd1_purecfma(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd1_purecfma(int); + +#ifndef Sleef_float_2_DEFINED +typedef Sleef_float2 Sleef_float_2; +#define Sleef_float_2_DEFINED +#endif + +SLEEF_IMPORT SLEEF_CONST float Sleef_sinf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sinf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cosf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_cosf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincosf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_finz_sincosf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_tanf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_asinf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_asinf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_acosf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_acosf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atanf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_atanf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f1_u35purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_atan2f1_u35purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_logf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_logf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_cbrtf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sinf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cosf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_cosf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincosf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_finz_sincosf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_tanf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_asinf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_asinf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_acosf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_acosf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atanf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_atanf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f1_u10purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_atan2f1_u10purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_logf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_logf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_cbrtf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_expf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_expf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_powf1_u10purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_powf1_u10purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sinhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_coshf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_coshf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_tanhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sinhf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_coshf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_coshf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_tanhf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fastsinf1_u3500purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fastsinf1_u3500purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fastcosf1_u3500purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fastcosf1_u3500purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fastpowf1_u3500purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fastpowf1_u3500purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_asinhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_asinhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_acoshf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_acoshf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_atanhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_atanhf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_exp2f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_exp2f1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_exp10f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_exp10f1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_expm1f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_expm1f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log10f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_log10f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log2f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_log2f1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log2f1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_log2f1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_log1pf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_log1pf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincospif1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_finz_sincospif1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincospif1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_finz_sincospif1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sinpif1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sinpif1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_cospif1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_cospif1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fmaf1_purecfma(float, float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fmaf1_purecfma(float, float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sqrtf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sqrtf1_u05purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_sqrtf1_u35purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf1_u05purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_hypotf1_u05purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf1_u35purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_hypotf1_u35purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fabsf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fabsf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_copysignf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_copysignf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fmaxf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fmaxf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fminf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fminf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fdimf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fdimf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_truncf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_truncf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_floorf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_floorf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_ceilf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_ceilf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_roundf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_roundf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_rintf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_rintf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_nextafterf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_nextafterf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_frfrexpf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_frfrexpf1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_fmodf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_fmodf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_remainderf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_remainderf1_purecfma(float, float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_modff1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_finz_modff1_purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_lgammaf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_lgammaf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_tgammaf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_tgammaf1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_erff1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_erff1_u10purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_erfcf1_u15purecfma(float); +SLEEF_IMPORT SLEEF_CONST float Sleef_finz_erfcf1_u15purecfma(float); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf1_purecfma(int); +SLEEF_IMPORT SLEEF_CONST int Sleef_finz_getIntf1_purecfma(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf1_purecfma(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_finz_getPtrf1_purecfma(int); +#endif +#ifdef __STDC__ + +#ifndef Sleef_double_2_DEFINED +typedef Sleef_double2 Sleef_double_2; +#define Sleef_double_2_DEFINED +#endif + +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sind1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cosd1_u35(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincosd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tand1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_asind1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_acosd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atand1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atan2d1_u35(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_logd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cbrtd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sind1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cosd1_u10(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincosd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tand1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_asind1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_acosd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atand1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atan2d1_u10(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_logd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cbrtd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_expd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_powd1_u10(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sinhd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_coshd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tanhd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sinhd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_coshd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tanhd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fastsind1_u3500(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fastcosd1_u3500(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fastpowd1_u3500(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_asinhd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_acoshd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_atanhd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp2d1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp2d1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp10d1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_exp10d1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_expm1d1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log10d1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log2d1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log2d1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_log1pd1_u10(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincospid1_u05(double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_sincospid1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sinpid1_u05(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_cospid1_u05(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_ldexpd1(double, int32_t); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_ilogbd1(double); +SLEEF_IMPORT SLEEF_CONST double Sleef_fmad1(double, double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_u05(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_sqrtd1_u35(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_hypotd1_u05(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_hypotd1_u35(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fabsd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_copysignd1(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fmaxd1(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fmind1(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fdimd1(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_truncd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_floord1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_ceild1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_roundd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_rintd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_nextafterd1(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_frfrexpd1(double); +SLEEF_IMPORT SLEEF_CONST int32_t Sleef_expfrexpd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_fmodd1(double, double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_remainderd1(double, double); +SLEEF_IMPORT SLEEF_CONST Sleef_double_2 Sleef_modfd1(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_lgammad1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_tgammad1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_erfd1_u10(double); +SLEEF_PRAGMA_OMP_SIMD_DP SLEEF_IMPORT SLEEF_CONST double Sleef_erfcd1_u15(double); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntd1(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrd1(int); + +#ifndef Sleef_float_2_DEFINED +typedef Sleef_float2 Sleef_float_2; +#define Sleef_float_2_DEFINED +#endif + +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cosf1_u35(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincosf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_asinf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_acosf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atanf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f1_u35(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_logf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cosf1_u10(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincosf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_asinf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_acosf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atanf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atan2f1_u10(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_logf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cbrtf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_expf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_powf1_u10(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_coshf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinhf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_coshf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tanhf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fastsinf1_u3500(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fastcosf1_u3500(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fastpowf1_u3500(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_asinhf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_acoshf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_atanhf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp2f1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_exp10f1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_expm1f1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log10f1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log2f1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log2f1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_log1pf1_u10(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincospif1_u05(float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_sincospif1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sinpif1_u05(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_cospif1_u05(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fmaf1(float, float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_u05(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_sqrtf1_u35(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf1_u05(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_hypotf1_u35(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fabsf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_copysignf1(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fmaxf1(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fminf1(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fdimf1(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_truncf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_floorf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_ceilf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_roundf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_rintf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_nextafterf1(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_frfrexpf1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_fmodf1(float, float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_remainderf1(float, float); +SLEEF_IMPORT SLEEF_CONST Sleef_float_2 Sleef_modff1(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_lgammaf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_tgammaf1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_erff1_u10(float); +SLEEF_PRAGMA_OMP_SIMD_SP SLEEF_IMPORT SLEEF_CONST float Sleef_erfcf1_u15(float); +SLEEF_IMPORT SLEEF_CONST int Sleef_getIntf1(int); +SLEEF_IMPORT SLEEF_CONST void *Sleef_getPtrf1(int); +#endif + +// + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // #ifndef __SLEEF_H__ + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/xnnpack.h b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/xnnpack.h new file mode 100644 index 0000000000000000000000000000000000000000..1429f94d1fae5de4878e320aeea73911dc1aaf90 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/include/xnnpack.h @@ -0,0 +1,4860 @@ +#if !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) +// Copyright (c) Facebook, Inc. and its affiliates. +// All rights reserved. +// +// Copyright 2019 Google LLC +// +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include + +#include "pthreadpool.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/// The number of bytes XNNPACK may read beyond array bounds. +/// The caller must allocate at least this many extra bytes after the tensor data passed to XNNPACK. +/// +/// Note: XNNPACK reads, but never writes beyond array bounds. +#if XNN_ARCH_HEXAGON +#define XNN_EXTRA_BYTES 128 +#else +#define XNN_EXTRA_BYTES 16 +#endif // XNN_ARCH_HEXAGON + +/// Maximum number of dimensions in tensor shape. +#define XNN_MAX_TENSOR_DIMS 6 + +/// A value ID that cannot be valid. +#define XNN_INVALID_VALUE_ID UINT32_MAX + +/// Allow sparse inference in a Runtime. +/// +/// Note: this flag is a hint to XNNPACK that it should consider sparse inference, but does not guarantee it. +#define XNN_FLAG_HINT_SPARSE_INFERENCE 0x00000001 + +/// Allow IEEE FP16 inference in a Runtime. +/// +/// Note: this flag hints XNNPACK to consider IEEE FP16 inference, but does not guarantee it. +#define XNN_FLAG_HINT_FP16_INFERENCE 0x00000002 + +/// Force IEEE FP16 inference in a Runtime, and fail if FP16 inference is not possible. +/// +/// Note: this flag guarantees that XNNPACK will use IEEE FP16 inference, or fail to create the Runtime object. +/// Warning: on x86 systems FP16 computations will be emulated at a substantial performance cost. +#define XNN_FLAG_FORCE_FP16_INFERENCE 0x00000004 + +/// Enable timing of each operator's runtime. +#define XNN_FLAG_BASIC_PROFILING 0x00000008 + +/// Enable the just-in-time compiler. +#define XNN_FLAG_JIT 0x00000010 + +/// The convolution operator represents a depthwise convolution, and use HWGo layout for filters. +#define XNN_FLAG_DEPTHWISE_CONVOLUTION 0x00000001 + +/// Assume transposed weights in a fully connected operator. +#define XNN_FLAG_TRANSPOSE_WEIGHTS 0x00000001 + +/// The operator assumes NHWC layout for the input, regardless of the output layout. +#define XNN_FLAG_INPUT_NHWC 0x00000002 + +/// Match "SAME" padding in TensorFlow. Exact padding values are computed dynamically depending on input size. +#define XNN_FLAG_TENSORFLOW_SAME_PADDING 0x00000004 + +/// Assume transposed weights in a batch matrix multiply operator. +#define XNN_FLAG_TRANSPOSE_B XNN_FLAG_TRANSPOSE_WEIGHTS + +/// Assume transposed input in a batch matrix multiply operator. +#define XNN_FLAG_TRANSPOSE_A 0x00000002 + +/// Implicitly flatten and reshape input of a Fully Connected operator into a 2D tensor. +#define XNN_FLAG_TENSORFLOW_RESHAPE_2D 0x00000004 + +/// Match behaviour of TensorFlow 1.x. +#define XNN_FLAG_TENSORFLOW_LEGACY_MODE 0x00000004 + +/// Static weights of the FP16 operator are in FP32 format. +#define XNN_FLAG_FP32_STATIC_WEIGHTS 0x00000008 + +/// Static biases of the FP16 operator are in FP32 format. +#define XNN_FLAG_FP32_STATIC_BIASES 0x00000080 + +/// Align corners of input and output images in resize operations. +#define XNN_FLAG_ALIGN_CORNERS 0x00000008 + +/// Yield worker threads of the thread pool to the system scheduler after the inference. +#define XNN_FLAG_YIELD_WORKERS 0x00000010 + +/// Use transient indirection buffer to reduce memory footprint +#define XNN_FLAG_TRANSIENT_INDIRECTION_BUFFER 0x00000020 + +/// Retain reduced dimensions with length 1. +#define XNN_FLAG_KEEP_DIMS 0x00000040 + +// Next unused flag value: 0x00000100. + +/// The number of entries in an array of xnn_quantization_params that XNNPACK may read beyond array bounds. +/// The caller must allocate at least this many extra xnn_quantization_params before passing the array to XNNPACK. +/// +/// Note: XNNPACK reads, but never writes beyond array bounds. +#define XNN_EXTRA_QUANTIZATION_PARAMS 15 + +/// The minimum blocksize for blockwise quantized operators. +#define XNN_MIN_BLOCKSIZE 32 + +#ifdef __GNUC__ +#define XNN_DEPRECATED __attribute__((deprecated)) +#else +#define XNN_DEPRECATED +#endif + +struct xnn_quantization_params { + int32_t zero_point; + float scale; +}; + +/// Status code for any XNNPACK function call. +enum xnn_status { + /// The call succeeded, and all output arguments now contain valid data. + xnn_status_success = 0, + xnn_status_uninitialized = 1, + xnn_status_invalid_parameter = 2, + xnn_status_invalid_state = 3, + xnn_status_unsupported_parameter = 4, + xnn_status_unsupported_hardware = 5, + xnn_status_out_of_memory = 6, + xnn_status_reallocation_required = 7, + xnn_status_deprecated = 8, +}; + +struct xnn_allocator { + /// User-specified pointer that will be passed as-is to all functions in this structure. + void* context; + /// Pointer to a function to be called for general memory allocation. + /// + /// @param context - The user-specified pointer from xnn_allocator structure. + /// @param size - The size of the memory block to allocate, in bytes. + /// + /// @returns Pointer to the allocated memory block of at least @ref size bytes. + /// If allocation fails, the function must return NULL. + void* (*allocate)(void* context, size_t size); + /// Pointer to a function to be called for general memory re-allocation, i.e. to increase or shrink a previously + /// allocated memory block. The content of the old memory block is copied to the new memory block. + /// + /// @param context - The user-specified pointer from xnn_allocator structure. + /// @param pointer - Pointer to a memory block allocated by @ref allocate or @ref reallocate functions. Can be NULL. + /// If the pointer is NULL, the @ref reallocate call is equivalent to an @ref allocate call. + /// @param size - The new size of the memory block to allocate, in bytes. + /// + /// @returns Pointer to the newly allocated memory block of at least @ref size bytes with the content of the previous + /// memory block. + /// If allocation fails, the function must return NULL, but must not release the previous memory block. + void* (*reallocate)(void* context, void* pointer, size_t size); + /// Pointer to a function to be called for general memory de-allocation. + /// + /// @param context - The user-specified pointer from xnn_allocator structure. + /// @param pointer - Pointer to a memory block allocated by @ref allocate or @ref reallocate functions. Can be NULL. + /// If the pointer is NULL, the @ref deallocate call is a no-op. + void (*deallocate)(void* context, void* pointer); + /// Pointer to a function to be called for aligned memory allocation. + /// + /// @param context - The user-specified pointer from xnn_allocator structure. + /// @param alignment - The alignment of the memory block to allocate, in bytes. Alignment is always a power-of-2. + /// @param size - The size of the memory block to allocate, in bytes. + /// + /// @returns Pointer to the allocated memory block of at least @ref size bytes. + /// If allocation fails, the function must return NULL. + void* (*aligned_allocate)(void* context, size_t alignment, size_t size); + /// Pointer to a function to be called for aligned memory deallocation. + /// + /// @param context - The user-specified pointer from xnn_allocator structure. + /// @param pointer - Pointer to a memory block allocated by @ref aligned_allocate function. Can be NULL. + /// If the pointer is NULL, the @ref aligned_deallocate call is a no-op. + void (*aligned_deallocate)(void* context, void* pointer); +}; + +/// Initialize XNNPACK library. +/// +/// XNNPACK must be successfully initialized before use. During initialization, XNNPACK populates internal structures +/// depending on the host processor. Initialization can be time-consuming. +/// +/// @param[in] allocator - structure with function pointers to be use for memory allocation and de-allocation. +/// If this argument is NULL, system-provided memory management functions (e.g. malloc/free) +/// will be used. +/// +/// @retval xnn_status_success - XNNPACK is successfully initialized and ready to use. +/// @retval xnn_status_out_of_memory - initialization failed due to out-of-memory condition. +/// @retval xnn_status_unsupported_hardware - initialization failed because the host processor does not satisfy the +/// minimum hardware requirements for XNNPACK. E.g. this may happen on x86 +/// processors without SSE2 extension, or on 32-bit ARM processors without +/// the NEON SIMD extension. +enum xnn_status xnn_initialize(const struct xnn_allocator* allocator); + +/// Deinitialize XNNPACK library. +/// +/// To avoid memory and resource leaks, users must call xnn_deinitialize once for each successful xnn_initialize call. +/// +/// @retval xnn_status_success - deinitialization call succeeded. +enum xnn_status xnn_deinitialize(void); + +/// Get the microkernel implementation build identifier's data. +/// +/// That identifier will be unique for the current set of microkernels implementations. +/// +/// @returns A pointer to the current identifier's data. +const void* xnn_experimental_get_build_identifier_data(); + +/// Get the microkernel implementation build identifier's data size. +/// +/// @returns The size in bytes of the identifier's data. +size_t xnn_experimental_get_build_identifier_size(); + +/// Check whether the given data matches this version's identifier. +/// +/// @returns The size in bytes of the identifier's data. +bool xnn_experimental_check_build_identifier(const void* data, size_t size); + +/// Subgraph is an abstract representation of a neural network model. +/// Subgraph objects are used to define Values (tensors) and Nodes (operators) comprising the model. +typedef struct xnn_subgraph* xnn_subgraph_t; + +/// Create a empty Subgraph object. +/// +/// @param external_value_ids - number of Value IDs to reserve for communication with external graph representation. +/// The Subgraph object would avoid creating internal Value IDs in the +/// [0, reserved_value_ids-1] range. +/// @param flags - binary features of the subgraph. No supported flags are currently defined. +/// @param subgraph_out - pointer to the variable that will be initialized with a handle to the Subgraph object upon +/// successful return. +enum xnn_status xnn_create_subgraph( + uint32_t external_value_ids, + uint32_t flags, + xnn_subgraph_t* subgraph_out); + +/// Destroy a Subgraph object, as well as Values, and Nodes associated with the subgraph. +/// +/// @param subgraph - the Subgraph object to destroy. +enum xnn_status xnn_delete_subgraph( + xnn_subgraph_t subgraph); + +#define XNN_VALUE_FLAG_EXTERNAL_INPUT 0x00000001 +#define XNN_VALUE_FLAG_EXTERNAL_OUTPUT 0x00000002 +#define XNN_VALUE_FLAG_PERSISTENT 0x00000004 + +#define XNN_INVALID_VALUE_ID UINT32_MAX + +/// Type of elements in a Value object. +enum xnn_datatype { + /// Invalid data type. Valid Values never have this datatype. + xnn_datatype_invalid = 0, + /// IEEE754 single-precision floating-point. + xnn_datatype_fp32 = 1, + /// IEEE754 half-precision floating-point. + xnn_datatype_fp16 = 2, + /// Quantized 8-bit signed integer with shared per-Value quantization + /// parameters. + xnn_datatype_qint8 = 3, + /// Quantized 8-bit unsigned integer with shared per-Value quantization + /// parameters. + xnn_datatype_quint8 = 4, + /// Quantized 32-bit signed integer with shared per-Value quantization + /// parameters. + xnn_datatype_qint32 = 5, + /// Quantized 8-bit signed integer with shared per-channel quantization + /// parameters. + xnn_datatype_qcint8 = 6, + /// Quantized 32-bit signed integer with shared per-channel quantization + /// parameters. + xnn_datatype_qcint32 = 7, + /// Quantized 4-bit signed integer with shared per-channel quantization + /// parameters. + xnn_datatype_qcint4 = 8, + /// Dynamically quantized 8-bit signed integer with per-batch quantization + /// parameters. + xnn_datatype_qdint8 = 9, + /// Dynamically quantized 8-bit signed integers packed with their per-row + /// quantization parameters. + xnn_datatype_qpint8 = 10, + /// 32-bit signed integers. + xnn_datatype_int32 = 11, + /// Quantized 4-bit signed integer with shared per-channel-block quantization + /// parameters. + xnn_datatype_qbint4 = 12, + /// IEEE754 single-precision packed floating-point. + xnn_datatype_pfp32 = 13, + /// BFloat16, i.e. the upper 16 bits of a float32. + xnn_datatype_bf16 = 14, + /// Dynamically quantized 8-bit unsigned integer with per-batch quantization + /// parameters. + xnn_datatype_qduint8 = 15, +}; + +/// Define a tensor-type Value and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Value. +/// @param datatype - type of the tensor elements. +/// @param num_dims - number of dimensions in the shape. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +/// @param data - pointer to static data used for tensor initialization. If the tensor is not statically initialized, +/// this pointer must be is NULL. If non-NULL, the life-time of the static data must exceed the life-time +/// of the Subgraph object, and of any Runtime objects created from the Subgraph. +/// @param external_id - external ID for the Value. The ID must be within the range of reversed Value IDs specified on +/// the Subgraph creation. If the external ID is XNN_INVALID_VALUE_ID, an internal ID will be +/// created for the Value. +/// @param flags - binary features of the Value. Supported values are any combination of XNN_VALUE_FLAG_EXTERNAL_INPUT +/// and XNN_VALUE_FLAG_EXTERNAL_OUTPUT. +/// @param id_out - pointer to the variable that will be initialized with the Value ID upon successful return. If a +/// valid @a external_id was provided, the variable will be initialized with the @a external_id value. +enum xnn_status xnn_define_tensor_value( + xnn_subgraph_t subgraph, + enum xnn_datatype datatype, + size_t num_dims, + const size_t* dims, + const void* data, + uint32_t external_id, + uint32_t flags, + uint32_t* id_out); + +/// Define a quantized tensor-type Value and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Value. +/// @param datatype - type of the tensor elements. +/// @param zero_point - offset from zero to subtract from the quantized elements in the Value. +/// @param scale - multiplication factor to convert quantized elements to real representation. +/// @param num_dims - number of dimensions in the shape. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +/// @param data - pointer to static data used for tensor initialization. If the tensor is not statically initialized, +/// this pointer must be is NULL. If non-NULL, the life-time of the static data must exceed the life-time +/// of the Subgraph object, and of any Runtime objects created from the Subgraph. +/// @param external_id - external ID for the Value. The ID must be within the range of reversed Value IDs specified on +/// the Subgraph creation. If the external ID is XNN_INVALID_VALUE_ID, an internal ID will be +/// created for the Value. +/// @param flags - binary features of the Value. Supported values are any combination of XNN_VALUE_FLAG_EXTERNAL_INPUT +/// and XNN_VALUE_FLAG_EXTERNAL_OUTPUT. +/// @param id_out - pointer to the variable that will be initialized with the Value ID upon successful return. If a +/// valid @a external_id was provided, the variable will be initialized with the @a external_id value. +enum xnn_status xnn_define_quantized_tensor_value( + xnn_subgraph_t subgraph, + enum xnn_datatype datatype, + int32_t zero_point, + float scale, + size_t num_dims, + const size_t* dims, + const void* data, + uint32_t external_id, + uint32_t flags, + uint32_t* id_out); + +enum xnn_status xnn_define_channelwise_quantized_tensor_value( + xnn_subgraph_t subgraph, + enum xnn_datatype datatype, + const float* scale, + size_t num_dims, + size_t channel_dim, + const size_t* dims, + const void* data, + uint32_t external_id, + uint32_t flags, + uint32_t* id_out); + +/// Validate the dimensions, channel_dim, zero point, datatype, and scale of a quantized tensor-type. +/// +/// @param datatype - type of the tensor elements. +/// @param zero_point - offset from zero to subtract from the quantized elements in the Value. +/// @param scale - multiplication factor to convert quantized elements to real representation. +/// @param num_dims - number of dimensions in the shape. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +enum xnn_status xnn_validate_quantized_tensor( + enum xnn_datatype datatype, + int32_t zero_point, + float scale, + size_t num_dims, + const size_t* dims); + +/// Validate the dimensions, channel_dim, zero point, datatype, and scales of a channelwise quantized tensor-type. +/// +/// @param datatype - type of the tensor elements. +/// @param zero_point - offset from zero to subtract from the quantized elements in the Value. +/// @param scale - per-channel multiplication factors to convert quantized elements to real representation. +/// @param num_dims - number of dimensions in the shape. +/// @param channel_dim - index of the channel dimension in the tensor with per-channel quantization parameters. +/// Typically this is the first dimension (dimension #0) of the filter tensors in the Convolution, +/// Deconvolution, and Fully Connected operators and the last dimension of the filter tensors in +/// the Depthwise Convolution operators. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +enum xnn_status xnn_validate_channelwise_quantized_tensor( + enum xnn_datatype datatype, + int32_t zero_point, + const float* scale, + size_t num_dims, + size_t channel_dim, + const size_t* dims); + +/// Define a channelwise quantized tensor-type Value and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Value. +/// @param datatype - type of the tensor elements. +/// @param zero_point - offset from zero to subtract from the quantized elements in the Value. +/// @param scale - per-channel multiplication factors to convert quantized elements to real representation. +/// @param num_dims - number of dimensions in the shape. +/// @param channel_dim - index of the channel dimension in the tensor with per-channel quantization parameters. +/// Typically this is the first dimension (dimension #0) of the filter tensors in the Convolution, +/// Deconvolution, and Fully Connected operators and the last dimension of the filter tensors in +/// the Depthwise Convolution operators. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +/// @param data - pointer to static data used for tensor initialization. If the tensor is not statically initialized, +/// this pointer must be is NULL. If non-NULL, the life-time of the static data must exceed the life-time +/// of the Subgraph object, and of any Runtime objects created from the Subgraph. +/// @param external_id - external ID for the Value. The ID must be within the range of reversed Value IDs specified on +/// the Subgraph creation. If the external ID is XNN_INVALID_VALUE_ID, an internal ID will be +/// created for the Value. +/// @param flags - binary features of the Value. Supported values are any combination of XNN_VALUE_FLAG_EXTERNAL_INPUT +/// and XNN_VALUE_FLAG_EXTERNAL_OUTPUT. +/// @param id_out - pointer to the variable that will be initialized with the Value ID upon successful return. If a +/// valid @a external_id was provided, the variable will be initialized with the @a external_id value. +enum xnn_status xnn_define_channelwise_quantized_tensor_value_v2( + xnn_subgraph_t subgraph, + enum xnn_datatype datatype, + int32_t zero_point, + const float* scale, + size_t num_dims, + size_t channel_dim, + const size_t* dims, + const void* data, + uint32_t external_id, + uint32_t flags, + uint32_t* id_out); + +/// Define a blockwise quantized tensor-type Value and add it to a Subgraph. +/// @param block_size - size of a block in the tensor with blockwise quantization parameters. Block is defined as +/// number of input channel element per output channel. +/// For Fully connected operators with 2d filters of size [output_channels, input_channels], +/// expecting number of scale values to be = output_channels * (input_channels / block_size). +enum xnn_status xnn_define_blockwise_quantized_tensor_value( + xnn_subgraph_t subgraph, + enum xnn_datatype datatype, + int32_t zero_point, + const uint16_t* scale, + size_t num_dims, + size_t channel_dim, + size_t block_size, + const size_t* dims, + const void* data, + uint32_t external_id, + uint32_t flags, + uint32_t* id_out); + +/// Define a dynamically quantized tensor-type Value and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Value. +/// @param datatype - type of the tensor elements. +/// @param num_dims - number of dimensions in the shape. +/// @param num_non_batch_dims - number of non-batch dimensions in the shape. The leading (num_dims - num_non_batch_dims) +/// dimensions will be flattened and treated as batch size. A set of quantization parameters +/// will be calculated for each batch element. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +/// @param external_id - external ID for the Value. The ID must be within the range of reversed Value IDs specified on +/// the Subgraph creation. If the external ID is XNN_INVALID_VALUE_ID, an internal ID will be +/// created for the Value. +/// @param flags - binary features of the Value. No supported flags are currently defined. +/// @param id_out - pointer to the variable that will be initialized with the Value ID upon successful return. If a +/// valid @a external_id was provided, the variable will be initialized with the @a external_id value. +enum xnn_status xnn_define_dynamically_quantized_tensor_value( + xnn_subgraph_t subgraph, + enum xnn_datatype datatype, + size_t num_dims, + size_t num_nonbatch_dims, + const size_t* dims, + uint32_t external_id, + uint32_t flags, + uint32_t* id_out); + +/// Type of unary operation +enum xnn_unary_operator { + xnn_unary_invalid = -1, + xnn_unary_convert, + xnn_unary_clamp, + xnn_unary_abs, + xnn_unary_bankers_rounding, + xnn_unary_ceiling, + xnn_unary_elu, + xnn_unary_exp, + xnn_unary_floor, + xnn_unary_gelu, + xnn_unary_hardswish, + xnn_unary_leaky_relu, + xnn_unary_log, + xnn_unary_negate, + xnn_unary_sigmoid, + xnn_unary_square, + xnn_unary_square_root, + xnn_unary_reciprocal_square_root, + xnn_unary_tanh, + // The following operators are experimental and may be removed. + xnn_unary_cube_root, + xnn_unary_cosine, + xnn_unary_sine, + xnn_unary_count_leading_zeros, + xnn_unary_bitwise_not, + xnn_unary_popcount, + xnn_unary_sign, +}; + +/// Parameters for xnn_define_unary +union xnn_unary_params { + struct { + /// lower bound for clipping output values. + float min; + /// upper bound for clipping output values. + float max; + } clamp; + struct { + /// scale factor for negative input elements. + float alpha; + } elu; + struct { + /// scale factor for negative input elements. + float negative_slope; + } leaky_relu; +}; + +/// Define a unary operator Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param operator - type of unary operator to define. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param params - parameters to be interpreted by the specific operator type. +/// @param flags - binary features of the Node. No supported flags are currently defined. +enum xnn_status xnn_define_unary( + xnn_subgraph_t subgraph, + enum xnn_unary_operator type, + const union xnn_unary_params* params, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Convert Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Convert Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_convert( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Convolution Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_padding_top - implicit zero-padding above 2D input data. Must be 0 if XNN_FLAG_TENSORFLOW_SAME_PADDING +/// flag is specified. +/// @param input_padding_right - implicit zero-padding to the right of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_bottom - implicit zero-padding below 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_left - implicit zero-padding to the left of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param kernel_height - kernel (filter) height. +/// @param kernel_width - kernel (filter) width. +/// @param subsampling_height - height of subsampling region for convolution output (convolution height stride). +/// @param subsampling_width - width of subsampling region for convolution output (convolution width stride). +/// @param dilation_height - dilation of kernel elements along the height dimension. +/// @param dilation_width - dilation of kernel elements along the width dimension. +/// @param groups - number of convolution groups. +/// @param group_input_channels - number of input channels per group. +/// @param group_output_channels - number of output channels per group. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, groups * group_input_channels] dimensions +/// @param filter_id - Value ID for the filter tensor. The filter tensor must ge a 4D tensor defined in the @a subgraph +/// with [groups * group_output_channels, kernel_height, kernel_width, group_input_channels] +/// dimensions. +/// @param bias_id - Value ID for the bias tensor, or XNN_INVALID_VALUE_ID for a 2D Convolution Node without a bias. If +/// present, the bias tensor must be a 1D tensor defined in the @a subgraph with [groups * +/// group_output_channels] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, OH, OW, groups * group_output_channels] dimensions. +/// @param flags - binary features of the 2D Convolution Node. The only currently supported values is +/// XNN_FLAG_TENSORFLOW_SAME_PADDING. +enum xnn_status xnn_define_convolution_2d( + xnn_subgraph_t subgraph, + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + float output_min, + float output_max, + uint32_t input_id, + uint32_t filter_id, + uint32_t bias_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Deconvolution (Transposed Convolution) Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param padding_top - implicit padding above 2D output data. +/// @param padding_right - implicit padding to the right of 2D output data. +/// @param padding_bottom - implicit padding below 2D output data. +/// @param padding_left - implicit padding to the left of 2D output data. +/// @param adjustment_height - additional elements in the bottom of the 2D output data. +/// @param adjustment_width - additional elements to the right of the 2D output data. +/// @param kernel_height - kernel (filter) height. +/// @param kernel_width - kernel (filter) width. +/// @param upsampling_height - height of upsampling region for deconvolution input (deconvolution height stride). +/// @param upsampling_width - width of upsampling region for deconvolution input (deconvolution width stride). +/// @param dilation_height - dilation of kernel elements along the height dimension. +/// @param dilation_width - dilation of kernel elements along the width dimension. +/// @param groups - number of convolution groups. +/// @param group_input_channels - number of input channels per group. +/// @param group_output_channels - number of output channels per group. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, groups * group_input_channels] dimensions +/// @param filter_id - Value ID for the filter tensor. The filter tensor must ge a 4D tensor defined in the @a subgraph +/// with [groups * group_output_channels, kernel_height, kernel_width, group_input_channels] +/// dimensions. +/// @param bias_id - Value ID for the bias tensor, or XNN_INVALID_VALUE_ID for a 2D Convolution Node without a bias. If +/// present, the bias tensor must be a 1D tensor defined in the @a subgraph with +/// [groups * group_output_channels] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, OH, OW, groups * group_output_channels] dimensions. +/// @param flags - binary features of the 2D Deconvolution Node. No supported flags are currently defined. +enum xnn_status xnn_define_deconvolution_2d( + xnn_subgraph_t subgraph, + uint32_t padding_top, + uint32_t padding_right, + uint32_t padding_bottom, + uint32_t padding_left, + uint32_t adjustment_height, + uint32_t adjustment_width, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t upsampling_height, + uint32_t upsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + float output_min, + float output_max, + uint32_t input_id, + uint32_t filter_id, + uint32_t bias_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Depthwise Convolution Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_padding_top - implicit zero-padding above 2D input data. Must be 0 if XNN_FLAG_TENSORFLOW_SAME_PADDING +/// flag is specified. +/// @param input_padding_right - implicit zero-padding to the right of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_bottom - implicit zero-padding below 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_left - implicit zero-padding to the left of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param kernel_height - kernel (filter) height. +/// @param kernel_width - kernel (filter) width. +/// @param subsampling_height - height of subsampling region for convolution output (convolution height stride). +/// @param subsampling_width - width of subsampling region for convolution output (convolution width stride). +/// @param dilation_height - dilation of kernel elements along the height dimension. +/// @param dilation_width - dilation of kernel elements along the width dimension. +/// @param depth_multiplier - ratio of output channels to input channels. +/// @param input_channels - number of input channels. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, input_channels] dimensions +/// @param filter_id - Value ID for the filter tensor. The filter tensor must ge a 4D tensor defined in the @a subgraph +/// with [1, kernel_height, kernel_width, input_channels * depth_multiplier] dimensions. +/// @param bias_id - Value ID for the bias tensor, or XNN_INVALID_VALUE_ID for a 2D Depthwise Convolution Node without +/// a bias. If present, the bias tensor must be a 1D tensor defined in the @a subgraph with +/// [input_channels * depth_multiplier] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, OH, OW, input_channels * depth_multiplier] dimensions. +/// @param flags - binary features of the 2D Depthwise Convolution Node. The only currently supported values is +/// XNN_FLAG_TENSORFLOW_SAME_PADDING. +enum xnn_status xnn_define_depthwise_convolution_2d( + xnn_subgraph_t subgraph, + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t depth_multiplier, + size_t input_channels, + float output_min, + float output_max, + uint32_t input_id, + uint32_t filter_id, + uint32_t bias_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Depth To Space Node 2D and add it to a Subgraph. +/// +/// The Depth To Space 2D Node rearranges data from depth into blocks of spatial data (a reverse transform to +/// Space To Depth). For a given input pixel, an output square of pixels with side @a block_size is formed from values +/// in the corresponding number of its channels. The output depth is therefore @a block_size x @a block_size times +/// smaller than that of the input. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param block_size - the size of the spatial block. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, OC * block_size * block_size] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH * block_size, IW * block_size, OC] dimensions. +/// @param flags - binary features of the input_channels Node. No supported flags are currently defined. +enum xnn_status xnn_define_depth_to_space_2d( + xnn_subgraph_t subgraph, + uint32_t block_size, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +enum xnn_status xnn_define_depth_to_space( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t block_size, + uint32_t flags); + +/// Define a 1D Global Average Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a dense tensor with 2 or more dimensions +/// defined in the @a subgraph. Averaging is performed across the second-innermost dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a dense tensor with 2 or more +/// dimensions defined in the @a subgraph. +/// @param flags - binary features of the 1D Global Average Pooling Node. The only currently supported value is +/// XNN_FLAG_KEEP_DIMS. +XNN_DEPRECATED enum xnn_status xnn_define_global_average_pooling_1d( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Global Average Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a dense tensor with 3 or more dimensions +/// defined in the @a subgraph. Averaging is performed across the second- and third-innermost +/// dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a dense tensor with 3 or more +/// dimensions defined in the @a subgraph. +/// @param flags - binary features of the 2D Global Average Pooling Node. The only currently supported value is +/// XNN_FLAG_KEEP_DIMS. +XNN_DEPRECATED enum xnn_status xnn_define_global_average_pooling_2d( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 1D Global Sum Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a dense tensor with 2 or more dimensions +/// defined in the @a subgraph. Averaging is performed across the second-innermost dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a dense tensor with 2 or more +/// dimensions defined in the @a subgraph. +/// @param flags - binary features of the 1D Global Sum Pooling Node. The only currently supported value is +/// XNN_FLAG_KEEP_DIMS. +XNN_DEPRECATED enum xnn_status xnn_define_global_sum_pooling_1d( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Global Sum Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a dense tensor with 3 or more dimensions +/// defined in the @a subgraph. Averaging is performed across the second- and third-innermost +/// dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a dense tensor with 3 or more +/// dimensions defined in the @a subgraph. +/// @param flags - binary features of the 2D Global Sum Pooling Node. The only currently supported value is +/// XNN_FLAG_KEEP_DIMS. +XNN_DEPRECATED enum xnn_status xnn_define_global_sum_pooling_2d( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Average Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_padding_top - implicit zero-padding above 2D input data. Must be 0 if XNN_FLAG_TENSORFLOW_SAME_PADDING +/// flag is specified. +/// @param input_padding_right - implicit zero-padding to the right of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_bottom - implicit zero-padding below 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_left - implicit zero-padding to the left of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param pooling_height - pooling (kernel) height. +/// @param pooling_width - pooling (kernel) width. +/// @param stride_height - displacing of the pooling window in the vertical dimension of the input pixels corresponding +/// to vertically adjacent output pixels. +/// @param stride_width - displacing of the pooling window in the horizontal dimension of the input pixels corresponding +/// to horizontally adjacent output pixels. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, channels] dimensions +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, OH, OW, channels] dimensions. +/// @param flags - binary features of the 2D Average Pooling Node. The only currently supported values is +/// XNN_FLAG_TENSORFLOW_SAME_PADDING. +enum xnn_status xnn_define_average_pooling_2d( + xnn_subgraph_t subgraph, + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Fully Connected Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be an N-dimensional tensor defined in the +/// @a subgraph. If XNN_FLAG_TENSORFLOW_RESHAPE_2D is not specified, the input tensor must be at least +/// 1D and its last dimension must match the last dimension of the filter tensor. In particular, if +/// input is a 2D tensor, it must have [batch_size, input_channels] dimensions. +/// If XNN_FLAG_TENSORFLOW_RESHAPE_2D is specified, the number of elements in the input tensor must be +/// divisible by the input_channels. The tensor will be first flattened into a 1D tensor of +/// [num_input_elements] dimensions, then reshaped into a 2D tensor of +/// [num_input_elements / input_channels, input_channels] dimensions where num_input_elements is the +/// total number of elements in the input tensor. +/// @param filter_id - Value ID for the filter tensor. The filter tensor must a 2D tensor defined in the @a subgraph. +/// If the XNN_FLAG_TRANSPOSE_WEIGHTS flag is not specified, the filter tensor must have +/// [output_channels, input_channels] dimensions. If the XNN_FLAG_TRANSPOSE_WEIGHTS flag is +/// specified, the filter tensor must have [input_channels, output_channels] dimensions. +/// @param bias_id - Value ID for the bias tensor, or XNN_INVALID_VALUE_ID for a Fully Connected Node without a bias. +/// If present, the bias tensor must be a 1D tensor defined in the @a subgraph with [output_channels] +/// dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph. +/// If XNN_FLAG_TENSORFLOW_RESHAPE_2D is not specified, the output tensor must have the same +/// dimensionality as the input tensor, all its dimensions but the last one must match the +/// corresponding dimensions of the input tensor, and the last dimensions of the output tensor must +/// match the first dimension of the filter tensor. In particular, if input is a 2D tensor, output +/// must be a 2D tensor of [batch_size, output_channels] dimensions. +/// If XNN_FLAG_TENSORFLOW_RESHAPE_2D is specified, output must be a 2D tensor of +/// [num_input_elements / input_channels, output_channels] dimensions where num_input_elements is the +/// total number of elements in the input tensor. +/// @param flags - binary features of the Fully Connected Node. The only currently supported values are +/// XNN_FLAG_TENSORFLOW_RESHAPE_2D and XNN_FLAG_TRANSPOSE_WEIGHTS. +enum xnn_status xnn_define_fully_connected( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t filter_id, + uint32_t bias_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Sparse Fully Connected Node and add it to a Subgraph. +/// +/// This operator is experimental, and will be removed in the future. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be an N-dimensional tensor defined in the +/// @a subgraph. If XNN_FLAG_TENSORFLOW_RESHAPE_2D is not specified, the input tensor must be at least +/// 1D and its last dimension must match the last dimension of the filter tensor. In particular, if +/// input is a 2D tensor, it must have [batch_size, input_channels] dimensions. +/// If XNN_FLAG_TENSORFLOW_RESHAPE_2D is specified, the number of elements in the input tensor must be +/// divisible by the input_channels. The tensor will be first flattened into a 1D tensor of +/// [num_input_elements] dimensions, then reshaped into a 2D tensor of +/// [num_input_elements / input_channels, input_channels] dimensions where num_input_elements is the +/// total number of elements in the input tensor. +/// @param filter_id - Value ID for the filter tensor. The filter tensor must a 2D tensor defined in the @a subgraph. +/// If the XNN_FLAG_TRANSPOSE_WEIGHTS flag is not specified, the filter tensor must have +/// [output_channels, input_channels] dimensions. If the XNN_FLAG_TRANSPOSE_WEIGHTS flag is +/// specified, the filter tensor must have [input_channels, output_channels] dimensions. +/// @param bias_id - Value ID for the bias tensor, or XNN_INVALID_VALUE_ID for a Fully Connected Node without a bias. +/// If present, the bias tensor must be a 1D tensor defined in the @a subgraph with [output_channels] +/// dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph. +/// If XNN_FLAG_TENSORFLOW_RESHAPE_2D is not specified, the output tensor must have the same +/// dimensionality as the input tensor, all its dimensions but the last one must match the +/// corresponding dimensions of the input tensor, and the last dimensions of the output tensor must +/// match the first dimension of the filter tensor. In particular, if input is a 2D tensor, output +/// must be a 2D tensor of [batch_size, output_channels] dimensions. +/// If XNN_FLAG_TENSORFLOW_RESHAPE_2D is specified, output must be a 2D tensor of +/// [num_input_elements / input_channels, output_channels] dimensions where num_input_elements is the +/// total number of elements in the input tensor. +/// @param flags - binary features of the Fully Connected Node. The only currently supported values are +/// XNN_FLAG_TENSORFLOW_RESHAPE_2D and XNN_FLAG_TRANSPOSE_WEIGHTS. +enum xnn_status xnn_define_fully_connected_sparse( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t filter_id, + uint32_t bias_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Max Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_padding_top - implicit zero-padding above 2D input data. Must be 0 if XNN_FLAG_TENSORFLOW_SAME_PADDING +/// flag is specified. +/// @param input_padding_right - implicit zero-padding to the right of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_bottom - implicit zero-padding below 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param input_padding_left - implicit zero-padding to the left of 2D input data. Must be 0 if +/// XNN_FLAG_TENSORFLOW_SAME_PADDING flag is specified. +/// @param pooling_height - pooling (kernel) height. +/// @param pooling_width - pooling (kernel) width. +/// @param stride_height - displacing of the pooling window in the vertical dimension of the input pixels corresponding +/// to vertically adjacent output pixels. +/// @param stride_width - displacing of the pooling window in the horizontal dimension of the input pixels corresponding +/// to horizontally adjacent output pixels. +/// @param dilation_height - dilation of pooling elements along the height dimension. +/// @param dilation_width - dilation of pooling elements along the width dimension. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, channels] dimensions +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, OH, OW, channels] dimensions. +/// @param flags - binary features of the 2D Max Pooling Node. The only currently supported values is +/// XNN_FLAG_TENSORFLOW_SAME_PADDING. +enum xnn_status xnn_define_max_pooling_2d( + xnn_subgraph_t subgraph, + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D ArgMax Pooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_padding_top - implicit zero-padding above 2D input data. +/// @param input_padding_right - implicit zero-padding to the right of 2D input data. +/// @param input_padding_bottom - implicit zero-padding below 2D input data. +/// @param input_padding_left - implicit zero-padding to the left of 2D input data. +/// @param pooling_height - pooling (kernel) height. Vertical stride between pooling regions match this value. +/// @param pooling_width - pooling (kernel) width. Horizontal stride between pooling regions match this value. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, channels] dimensions +/// @param output_value_id - Value ID for the output tensor with the maximum values in the pools. The output tensor must +/// be a 4D tensor defined in the @a subgraph with [N, OH, OW, channels] dimensions. +/// @param output_index_id - Value ID for the output tensor with the indexes of the maximum values in the pools. The +/// output tensor must be a 4D tensor defined in the @a subgraph with [N, OH, OW, channels] +/// dimensions. +/// @param flags - binary features of the 2D ArgMax Pooling Node. No supported flags are currently defined. +enum xnn_status xnn_define_argmax_pooling_2d( + xnn_subgraph_t subgraph, + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t input_id, + uint32_t output_value_id, + uint32_t output_index_id, + uint32_t flags); + +/// Define a 2D UnPooling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param padding_top - implicit padding above 2D output data. +/// @param padding_right - implicit padding to the right of 2D output data. +/// @param padding_bottom - implicit padding below 2D output data. +/// @param padding_left - implicit padding to the left of 2D output data. +/// @param pooling_height - height of the pooling window. +/// @param pooling_width - width of the pooling window. +/// @param input_value_id - Value ID for the input tensor with the max-pooling values to invert. The input value tensor +/// must be a 4D tensor defined in the @a subgraph with [N, IH, IW, channels] dimensions. +/// @param input_index_id - Value ID for the input tensor with the indices of the per-pool maximum values produced by +/// a 2D UnPooling Node. The input tensor must be a 4D tensor defined in the @a subgraph with +/// [N, IH, IW, channels] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, OH, OW, channels] dimensions. +/// @param flags - binary features of the 2D UnPooling Node. No supported flags are currently defined. +enum xnn_status xnn_define_unpooling_2d( + xnn_subgraph_t subgraph, + uint32_t padding_top, + uint32_t padding_right, + uint32_t padding_bottom, + uint32_t padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t input_value_id, + uint32_t input_index_id, + uint32_t output_id, + uint32_t flags); + +enum xnn_binary_operator { + xnn_binary_invalid = -1, + xnn_binary_add, + xnn_binary_subtract, + xnn_binary_multiply, + xnn_binary_divide, + xnn_binary_maximum, + xnn_binary_minimum, + xnn_binary_copysign, + xnn_binary_squared_difference, + xnn_binary_prelu, + // The following operators are experimental and may be removed. + xnn_binary_modulus, + xnn_binary_atan2, + xnn_binary_pow, + xnn_binary_bitwise_and, + xnn_binary_bitwise_or, + xnn_binary_bitwise_xor, + xnn_binary_shift_left, + xnn_binary_shift_right_logical, + xnn_binary_shift_right_arithmetic, +}; + +struct xnn_binary_params { + /// lower bound for clipping output values. + double output_min; + /// upper bound for clipping output values. + double output_max; +}; + +/// Define a 2-Input binary operator Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param type - Type of operator to apply to the two inputs. +/// @param params - Optional parameters for the operator. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Node. No supported flags are currently defined. +enum xnn_status xnn_define_binary( + xnn_subgraph_t subgraph, + enum xnn_binary_operator type, + const struct xnn_binary_params* params, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2-Input Add Node and add it to a Subgraph. +/// +/// The 2-Input Add Node computes elementwise addition of two tensor inputs with numpy broadcasting rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Add Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_add2( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2-Input Multiply Node and add it to a Subgraph. +/// +/// The 2-Input Multiply Node computes elementwise multiplication of two tensor inputs with numpy broadcasting rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Multiply Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_multiply2( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +// Cap operations applied to logits (Q * K) of attention operator. +enum xnn_attention_logits_cap_type { + // No capping. + xnn_attention_logits_cap_type_none = 0, + // Cap the absolute values of logits by tanh: tanh(logits / cap) * cap + xnn_attention_logits_cap_type_tanh +}; + +// Params when the cap type is xnn_attention_logits_cap_type_tanh. +struct xnn_attention_logits_cap_tanh_params { + float cap; +}; + +/// Define a Scaled Dot-Product Attention Node and add it to a Subgraph. +/// +/// This operator is experimental. +/// +/// The Scaled Dot-Product Attention Node computes a multi-head or multi-query scaled dot attention on the query, key, +/// and value tensors. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param cap_type - type of cap to be applied to the logits. +/// @param cap_params - parameters for the cap. Must be a pointer to xnn_attention_logits_cap_tanh_params if cap_type +/// is xnn_attention_logits_cap_type_tanh. +/// @param query_id - Value ID for the query tensor. The query tensor must be a 3+-dimensional tensor defined in the +/// @a subgraph with the dimensions as [*, H, T, C], where H/T/C are the heads/tokens/channels, and * +/// is the 0 or more dimensions treated as batch size. +/// @param key_id - Value ID for the key tensor. The key tensor must be a 2+--dimensional tensor defined in the +/// @a subgraph. It can have the same number of dimensions as the query, with the dimensions as +/// [*, H, U, C] (multi-head), or have 1 less dimension than the query, with the dimensions as +/// as [*, U, C] (multi-query, number of heads omitted implies single head), where H/U/C are the +/// heads/key_value_tokens/channels, and * is the 0 or more dimensions treated as batch size. These +/// batch size dimensions must be the same as query. +/// @param value_id - Value ID for the value tensor. The value tensor must be a 2+--dimensional tensor defined in the +/// @a subgraph. It can have the same number of dimensions as the query, with the dimensions as +/// [*, H, U, D] (multi-head), or have 1 less dimension than the query, with the dimensions as +/// as [*, U, D] (multi-query, number of heads omitted implies single head), where H/U/D are the +/// heads/key_value_tokens/value_channels, and * is the 0 or more dimensions treated as batch size. +/// These batch size dimensions must be the same as query and key. +/// @param scale_id - Value ID for the scale tensor. The scale tensor must be a 1D tensor defined in the @a subgraph +/// with [C] dimensions. The query tensor is multiplied with this scale tensor before the dot product +/// with the key tensor. +/// @param mask_id - Value ID for the mask tensor. The mask tensor must be a 2D tensor defined in the @a subgraph with +/// [T, U] dimensions. The mask tensor is added to the logits (query dot value). +/// @param output_id - Value ID for the output tensor. The output tensor must be a 3+-dimensional tensor defined in the +/// @a subgraph with the dimensions as [*, H, T, D], where H/T/D are the heads/tokens/value_channels, +/// and * is the 0 or more dimensions treated as batch size. These batch size dimensions must be the +/// same as query, key, and value. +/// @param flags - binary features of the Scaled Dot Product Attention Node. No supported flags are currently defined. +enum xnn_status xnn_define_scaled_dot_product_attention( + xnn_subgraph_t subgraph, + enum xnn_attention_logits_cap_type cap_type, + const void* cap_params, + uint32_t query_id, + uint32_t key_id, + uint32_t value_id, + uint32_t scale_id, + uint32_t mask_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Subtract Node and add it to a Subgraph. +/// +/// The Subtract Node computes elementwise subtraction of two tensor inputs with numpy broadcasting rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Subtract Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_subtract( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Divide Node and add it to a Subgraph. +/// +/// The Divide Node computes elementwise division of two tensor inputs with numpy broadcasting rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Divide Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_divide( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2-Input Maximum Node and add it to a Subgraph. +/// +/// The 2-Input Maximum Node computes elementwise maximum of two tensor inputs with numpy broadcasting rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Maximum Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_maximum2( + xnn_subgraph_t subgraph, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2-Input Minimum Node and add it to a Subgraph. +/// +/// The 2-Input Minimum Node computes elementwise minimum of two tensor inputs with numpy broadcasting rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Minimum Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_minimum2( + xnn_subgraph_t subgraph, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Squared Difference Node and add it to a Subgraph. +/// +/// The Squared Difference Node computes elementwise squared difference of two tensor inputs with numpy broadcasting +/// rules. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the second +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an M-dimensional tensor defined in +/// the @a subgraph with each dimension either equal to the corresponding dimension of the first +/// input, or equal to 1. In the latter case, the elements of the input tensor are broadcasted along +/// that dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be a max(N,M)-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the maximum between the corresponding dimension +/// of the two inputs. +/// @param flags - binary features of the Squared Difference Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_squared_difference( + xnn_subgraph_t subgraph, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Constant Pad Node with static padding specification and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param pre_paddings - number of padding elements to insert before input elements for every dimension. This array +/// must have as many elements as the number of dimensions in the input tensor. +/// @param post_paddings - number of padding elements to insert after input elements for every dimension. This array +/// must have as many elements as the number of dimensions in the input tensor. +/// @param padding_value - constant value used to initialize padding elements. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor with padding. +/// @param flags - binary features of the Constant Pad Node. No supported flags are currently defined. +enum xnn_status xnn_define_static_constant_pad( + xnn_subgraph_t subgraph, + const size_t* pre_paddings, + const size_t* post_paddings, + float padding_value, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Expand Dims Node with and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param num_new_axes - number of new axes of size 1 to be inserted. +/// @param new_axes - The axis positions of the new axes in the expanded dimensions. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor with padding. +/// @param flags - binary features of the Constant Pad Node. No supported flags are currently defined. +enum xnn_status xnn_define_static_expand_dims( + xnn_subgraph_t subgraph, + size_t num_new_axes, + const size_t* new_axes, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Mean Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param num_reduction_axes - number of axes along which mean is computed. +/// @param reduction_axes - axes along which mean is computed. +/// @param input_id - Value ID for the input tensor. The input tensor must be a dense tensor with at least +/// @a num_reduction_axes dimensions defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be a dense tensor defined in the +/// @a subgraph with @a num_reduction_axes fewer dimensions than the input tensor (if +/// XNN_FLAG_KEEP_DIMS is not specified), or has same dimension rank but the dimension at +/// @a reduction_axes reduced to 1 (if XNN_FLAG_KEEP_DIMS is specified). +/// @param flags - binary features of the Mean Node. The only currently supported value is XNN_FLAG_KEEP_DIMS +XNN_DEPRECATED enum xnn_status xnn_define_static_mean( + xnn_subgraph_t subgraph, + size_t num_reduction_axes, + const size_t* reduction_axes, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +enum xnn_reduce_operator { + xnn_reduce_invalid = -1, + xnn_reduce_sum, + xnn_reduce_mean, +}; + +/// Define a Reduce Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param num_reduction_axes - number of axes along which reduce is computed. +/// @param reduction_axes - axes along which reduce is computed. +/// @param input_id - Value ID for the input tensor. The input tensor must be a dense tensor with at least +/// @a num_reduction_axes dimensions defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be a dense tensor defined in the +/// @a subgraph with @a num_reduction_axes fewer dimensions than the input tensor (if +/// XNN_FLAG_KEEP_DIMS is not specified), or has same dimension rank but the dimension at +/// @a reduction_axes reduced to 1 (if XNN_FLAG_KEEP_DIMS is specified). +/// @param flags - binary features of the Reduce Node. The only currently supported value is XNN_FLAG_KEEP_DIMS +enum xnn_status xnn_define_static_reduce( + xnn_subgraph_t subgraph, + enum xnn_reduce_operator reduce_operator_type, + size_t num_reduction_axes, + const size_t* reduction_axes, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Reduce Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param num_reduction_axes - number of axes along which reduce is computed. +/// @param reduction_axes - axes along which reduce is computed. Negative values +/// are interpreted as offsets from @a +/// num_reduction_axes. +/// @param input_id - Value ID for the input tensor. The input tensor must be a +/// dense tensor with at least @a num_reduction_axes +/// dimensions defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be +/// a dense tensor defined in the @a subgraph with @a +/// num_reduction_axes fewer dimensions than the input tensor +/// (if XNN_FLAG_KEEP_DIMS is not specified), or has same +/// dimension rank but the dimension at +/// @a reduction_axes reduced to 1 (if XNN_FLAG_KEEP_DIMS is +/// specified). +/// @param flags - binary features of the Reduce Node. The only currently +/// supported value is XNN_FLAG_KEEP_DIMS +enum xnn_status xnn_define_static_reduce_v2( // + xnn_subgraph_t subgraph, // + enum xnn_reduce_operator reduce_operator_type, // + size_t num_reduction_axes, // + const int64_t* reduction_axes, // + uint32_t input_id, // + uint32_t output_id, // + uint32_t flags); + +/// Define a 2-Input Concatenate Node and add it to a Subgraph. +/// +/// The 2-Input Concatenate Node concatenates two tensors along a specified axis. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param axis - the axis to concatenate the two input tensors along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// second input. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// first input. +/// @param output_id - Value ID for the output tensor. The output tensor must be a N-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the dimension of both inputs, except the axis +/// dimension, where it is the sum of the corresponding dimensions of both inputs. +/// @param flags - binary features of the Concatenate Node. No supported flags are currently defined. +enum xnn_status xnn_define_concatenate2( + xnn_subgraph_t subgraph, + int32_t axis, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 3-Input Concatenate Node and add it to a Subgraph. +/// +/// The 3-Input Concatenate Node concatenates three tensors along a specified axis. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param axis - the axis to concatenate the two input tensors along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input3_id - Value ID for the third input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param output_id - Value ID for the output tensor. The output tensor must be a N-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the dimension of all inputs, except the axis +/// dimension, where it is the sum of the corresponding dimensions of all inputs. +/// @param flags - binary features of the Concatenate Node. No supported flags are currently defined. +enum xnn_status xnn_define_concatenate3( + xnn_subgraph_t subgraph, + int32_t axis, + uint32_t input1_id, + uint32_t input2_id, + uint32_t input3_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 4-Input Concatenate Node and add it to a Subgraph. +/// +/// The 4-Input Concatenate Node concatenates four tensors along a specified axis. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param axis - the axis to concatenate the two input tensors along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input3_id - Value ID for the third input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input4_id - Value ID for the fourth input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param output_id - Value ID for the output tensor. The output tensor must be a N-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the dimension of all inputs, except the axis +/// dimension, where it is the sum of the corresponding dimensions of all inputs. +/// @param flags - binary features of the Concatenate Node. No supported flags are currently defined. +enum xnn_status xnn_define_concatenate4( + xnn_subgraph_t subgraph, + int32_t axis, + uint32_t input1_id, + uint32_t input2_id, + uint32_t input3_id, + uint32_t input4_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 5-Input Concatenate Node and add it to a Subgraph. +/// +/// The 5-Input Concatenate Node concatenates four tensors along a specified axis. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param axis - the axis to concatenate the two input tensors along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input3_id - Value ID for the third input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input4_id - Value ID for the fourth input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param input5_id - Value ID for the fourth input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph with each dimension, except the axis, equal to the corresponding dimension of the +/// other inputs. +/// @param output_id - Value ID for the output tensor. The output tensor must be a N-dimensional tensor defined +/// in the @a subgraph with each dimension equal to the dimension of all inputs, except the axis +/// dimension, where it is the sum of the corresponding dimensions of all inputs. +enum xnn_status xnn_define_concatenate5( + xnn_subgraph_t subgraph, + int32_t axis, + uint32_t input1_id, + uint32_t input2_id, + uint32_t input3_id, + uint32_t input4_id, + uint32_t input5_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Copy Sign Node and add it to a Subgraph. +/// +/// The Copy Sign Node copies the sign of the second input to the first input. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be defined in the @a subgraph. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. +/// @param flags - binary features of the Copy Sign Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_copysign( + xnn_subgraph_t subgraph, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Copy Node and add it to a Subgraph. +/// +/// The Copy Node copies an input tensor to an output tensor. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the first input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Copy Node. No supported flags are currently defined. +enum xnn_status xnn_define_copy( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2-Output Split Node and add it to a Subgraph. +/// +/// The 2-Output Split Node splits an input tensor into two output tensors along a specified axis evenly. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param split_dim - the dimension to split the input tensor along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input_id - Value ID for the input tensor. The input tensor must be an N-dimensional tensor defined in the @a +/// subgraph. +/// @param output1_id - Value ID for the first output tensor. The output tensor must be an N-dimensional tensor defined +/// in the @a subgraph with each dimension, except the axis, equal to the corresponding dimension +/// of the second output. The split_dim dimension is half of the input's split_dim. +/// @param output2_id - Value ID for the second output tensor. The output tensor must be an N-dimensional tensor +/// defined in the @a subgraph with each dimension, except the axis, equal to the corresponding +/// dimension of the first output. The split_dim dimension is half of the input's split_dim. +/// @param flags - binary features of the Split Node. No supported flags are currently defined. +enum xnn_status xnn_define_even_split2( + xnn_subgraph_t subgraph, + int32_t split_dim, + uint32_t input_id, + uint32_t output1_id, + uint32_t output2_id, + uint32_t flags); + +/// Define a 3-Output Split Node and add it to a Subgraph. +/// +/// The 3-Output Split Node splits an input tensor into three output tensors along a specified axis evenly. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param split_dim - the dimension to split the input tensor along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input_id - Value ID for the input tensor. The input tensor must be an N-dimensional tensor defined in the @a +/// subgraph. +/// @param output1_id - Value ID for the first output tensor. The output tensor must be an N-dimensional tensor defined +/// in the @a subgraph with each dimension, except the axis, equal to the corresponding dimension +/// of the second and third output. The split_dim dimension is one third of the input's split_dim. +/// @param output2_id - Value ID for the second output tensor. The output tensor must be an N-dimensional tensor +/// defined in the @a subgraph with each dimension, except the axis, equal to the corresponding +/// dimension of the first and third output. The split_dim dimension is one third of the input's +/// split_dim. +/// @param output3_id - Value ID for the third output tensor. The output tensor must be an N-dimensional tensor +/// defined in the @a subgraph with each dimension, except the axis, equal to the corresponding +/// dimension of the second and third output. The split_dim dimension is one third of the input's +/// split_dim. +/// @param flags - binary features of the Split Node. No supported flags are currently defined. +enum xnn_status xnn_define_even_split3( + xnn_subgraph_t subgraph, + int32_t split_dim, + uint32_t input_id, + uint32_t output1_id, + uint32_t output2_id, + uint32_t output3_id, + uint32_t flags); + +/// Define a 4-Output Split Node and add it to a Subgraph. +/// +/// The 4-Output Split Node splits an input tensor into four output tensors along a specified axis evenly. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param split_dim - the dimension to split the input tensor along. If this is less than zero, the number of +/// dimensions is added to it. +/// @param input_id - Value ID for the input tensor. The input tensor must be an N-dimensional tensor defined in the @a +/// subgraph. +/// @param output1_id - Value ID for the first output tensor. The output tensor must be an N-dimensional tensor defined +/// in the @a subgraph with each dimension, except the axis, equal to the corresponding dimension +/// of the other output tensors. The split_dim dimension is one fourth of the input's split_dim. +/// @param output2_id - Value ID for the second output tensor. The output tensor must be an N-dimensional tensor +/// defined in the @a subgraph with each dimension, except the axis, equal to the corresponding +/// dimension of the other output tensors. The split_dim dimension is one fourth of the input's +/// split_dim. +/// @param output3_id - Value ID for the third output tensor. The output tensor must be an N-dimensional tensor +/// defined in the @a subgraph with each dimension, except the axis, equal to the corresponding +/// dimension of the other output tensors. The split_dim dimension is one fourth of the input's +/// split_dim. +/// @param output4_id - Value ID for the fourth output tensor. The output tensor must be an N-dimensional tensor +/// defined in the @a subgraph with each dimension, except the axis, equal to the corresponding +/// dimension of the other output tensors. The split_dim dimension is one fourth of the input's +/// split_dim. +/// @param flags - binary features of the Split Node. No supported flags are currently defined. +enum xnn_status xnn_define_even_split4( + xnn_subgraph_t subgraph, + int32_t split_dim, + uint32_t input_id, + uint32_t output1_id, + uint32_t output2_id, + uint32_t output3_id, + uint32_t output4_id, + uint32_t flags); + +/// Define a Reshape Node with static shape specification and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param num_dims - number of shape dimensions in the output tensor. +/// @param new_shape - shape dimensions of the output tensor. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor with padding. +/// @param flags - binary features of the Reshape Node. No supported flags are currently defined. +enum xnn_status xnn_define_static_reshape( + xnn_subgraph_t subgraph, + size_t num_dims, + const size_t* new_shape, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a 2D Resize Bilinear Node with static output height & width specification and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param new_height - height dimension of the output tensor. +/// @param new_width - width dimension of the output tensor. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, H, W, C] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, new_height, new_width, C] dimensions. +/// @param flags - binary features of the 2D Resize Bilinear Node. The only currently supported values are +/// XNN_FLAG_TENSORFLOW_LEGACY_MODE and XNN_FLAG_ALIGN_CORNERS, which are mutually exclusive. +enum xnn_status xnn_define_static_resize_bilinear_2d( + xnn_subgraph_t subgraph, + size_t new_height, + size_t new_width, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a PReLU (Parametric ReLU) Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, H, W, channels] dimensions. +/// @param slope_id - Value ID for the slope tensor. The slope tensor must be a 1D tensor defined in the @a subgraph with +/// either [1] or [channels] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, H, W, channels] dimensions. +/// @param flags - binary features of the PReLU Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_prelu( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t slope_id, + uint32_t output_id, + uint32_t flags); + +/// Define a RoPE (Rotary Positional Embeddings) Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param max_tokens - deprecated. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [batch, tokens, heads, channels] dimensions. +/// @param weights_id - Value ID for the weights tensor. The weights tensor must be a 2D tensor defined in the +/// @a subgraph with [max_tokens, channels] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [batch, tokens, heads, channels] dimensions. +/// @param flags - binary features of the RoPE Node. No supported flags are currently defined. +enum xnn_status xnn_define_rope( + xnn_subgraph_t subgraph, + size_t max_sequence_size, + uint32_t input_id, + uint32_t weights_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Abs Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Abs Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_abs( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Bankers' Rounding Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Bankers' Rounding Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_bankers_rounding( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Batch Matrix Multiply Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input1_id - Value ID for the first input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph. It must be at least 3D. The first N-2 dimensions must match the second input +/// tensor. The last 2 dimensions are [M, K]. If XNN_FLAG_TRANSPOSE_B is not specified, the last +/// dimension must match the second last dimension of the second input tensor. If +/// XNN_FLAG_TRANSPOSE_B is specified, the last dimension must match the last dimension of the +/// second input tensor. +/// @param input2_id - Value ID for the second input tensor. The input tensor must be an N-dimensional tensor defined +/// in the @a subgraph. It must be at least 3D. The first N-2 dimensions must match the first input +/// tensor. If XNN_FLAG_TRANSPOSE_B is not specified, the last 2 dimensions are [K, N], and the +/// second last dimension must match the last dimension of the first input tensor. If +/// XNN_FLAG_TRANSPOSE_B is specified, the last 2 dimensions are [N, K], and the last dimension must +/// match the last dimension of the first input tensor. +/// @param output_id - Value ID for the output tensor. The output tensor must be an N-dimensional tensor defined in the +/// @a subgraph. It must be at least 3D. The first N-2 dimensions must match the first and second +/// input tensors . The last 2 dimensions must be [M, N]. +/// @param flags - binary features of the Batch Matrix Multiply Node. The only currently supported value is +/// XNN_FLAG_TRANSPOSE_B. +enum xnn_status xnn_define_batch_matrix_multiply( + xnn_subgraph_t subgraph, + uint32_t input1_id, + uint32_t input2_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Ceiling Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Ceiling Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_ceiling( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Clamp Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param output_min - lower bound for clipping output values. +/// @param output_max - upper bound for clipping output values. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Clamp Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_clamp( + xnn_subgraph_t subgraph, + float output_min, + float output_max, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define an ELU (Exponential Linear Unit) Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param alpha - scale factor for negative output elements. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the ELU Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_elu( + xnn_subgraph_t subgraph, + float alpha, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Exp Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Exp Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_exp( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Floor Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Floor Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_floor( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define an GELU (Gaussian Error Linear Unit) Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the GELU Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_gelu( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a HardSwish Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the HardSwish Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_hardswish( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Leaky ReLU Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param negative_slope - scale factor for negative input elements. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Leaky ReLU Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_leaky_relu( + xnn_subgraph_t subgraph, + float negative_slope, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Log Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Log Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_log( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Negate Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Negate Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_negate( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Sigmoid Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Sigmoid Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_sigmoid( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a SoftMax Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph, and have at +/// least one dimension. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the SoftMax Node. No supported flags are currently defined. +enum xnn_status xnn_define_softmax( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Space To Depth 2D Node and add it to a Subgraph. +/// +/// The Space To Depth 2D Node rearranges blocks of spatial data into blocks (a reverse transform to Depth To Space 2D). +/// For a given input pixel, an output square of pixels with side @a block_size is formed from values in the +/// corresponding number of its channels. The output depth is therefore @a block_size x @a block_size times greater +/// than that of the input. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param block_size - the size of the spatial block. +/// @param input_id - Value ID for the input tensor. The input tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH * block_size, IW * block_size, OC] dimensions. +/// @param output_id - Value ID for the output tensor. The output tensor must be a 4D tensor defined in the @a subgraph +/// with [N, IH, IW, OC * block_size * block_size] dimensions. +/// @param flags - binary features of the input_channels Node. No supported flags are currently defined. +enum xnn_status xnn_define_space_to_depth_2d( + xnn_subgraph_t subgraph, + uint32_t block_size, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Square Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Square Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_square( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Square Root Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Square Root Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_square_root( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Reciprocal Square Root Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be +/// defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be +/// defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Square Root Node. No supported flags +/// are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_reciprocal_square_root( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +enum xnn_status xnn_define_static_slice( + xnn_subgraph_t subgraph, + size_t num_dims, + const size_t* offsets, + const size_t* sizes, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); +/// Define a Static Slice Node add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param num_dims - number of shape dimensions in the input and output tensor. +/// @param offsets - offsets in each dimension of the input tensor. This array must have @a num_dims elements. Can be +/// negative meaning that the offset is relative to the end of the dimension. +/// @param sizes - size of each dimension in output tensor. This array must have @a num_dims elements. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// dimensions must match @a sizes. +/// @param flags - binary features of the Static Slice Node. No supported flags are currently defined. +enum xnn_status xnn_define_static_slice_v2( // + xnn_subgraph_t subgraph, // + size_t num_dims, // + const int64_t* offsets, // + const size_t* sizes, // + uint32_t input_id, // + uint32_t output_id, // + uint32_t flags); + +/// Define a Static Transpose Node and add it to a Subgraph. +/// +/// The Static Transpose Node applies a generalized transpose to the input tensor using the permuation in perm. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be an N-dimensional tensor defined in +/// the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be an N-dimensional tensor defined +/// in the @a subgraph with each dimension equal to its corresponding permuted input dimension. +/// @param num_dims - the number of permutation dimensions. This must be equal to the number of input dimensions. +/// @param perm - The permutation of the axis of the input tensor. The perm array must must contain 0 to N-1 in the +/// permuted order. +/// @param flags - binary features of the Static Transpose Node. No supported flags are currently defined. +enum xnn_status xnn_define_static_transpose( + xnn_subgraph_t subgraph, + size_t num_dims, + const size_t* perm, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Define a Tanh Node and add it to a Subgraph. +/// +/// @param subgraph - a Subgraph object that will own the created Node. +/// @param input_id - Value ID for the input tensor. The input tensor must be defined in the @a subgraph. +/// @param output_id - Value ID for the output tensor. The output tensor must be defined in the @a subgraph, and its +/// shape must match the shape of the input tensor. +/// @param flags - binary features of the Tanh Node. No supported flags are currently defined. +XNN_DEPRECATED enum xnn_status xnn_define_tanh( + xnn_subgraph_t subgraph, + uint32_t input_id, + uint32_t output_id, + uint32_t flags); + +/// Code cache is a cache for JIT generated code. +typedef struct xnn_code_cache* xnn_code_cache_t; + +/// Weights cache can be finalized in these ways: +enum xnn_weights_cache_finalization_kind { + /// Weights cache is finalized, no insert operations into the weights cache is allowed, even if the "inserted" + /// weights already exist in thee cache. Weights cache memory will also be trimmed to page boundary and set to + /// read-only (to prevent writes). + xnn_weights_cache_finalization_kind_hard, + /// Weights cache will be finalized with some extra space at the end, this allows for "inserting" into the cache only + /// if the weights are already in the cache, and errors on inserting uncached weights. There is memory overhead. + xnn_weights_cache_finalization_kind_soft, +}; + +/// A combination of multiple factors to uniquely locate the weights cache. +struct xnn_weights_cache_look_up_key { + /// The unique seed for each ukernel. It is guaranteed that each ukernel provides + /// a consistent and identical seed. + uint32_t seed; + /// Pointer to the original kernel. + const void* kernel; + /// Pointer to the original bias, could be NULL. + const void* bias; +}; + +/// A group of function pointers to manage weights cache. All functions may be +/// called on multi threads. +struct xnn_weights_cache_provider { + /// User-specified pointer that will be passed as-is to all functions in this + /// structure. + void* context; + + /// Looks up the tuple of {cache_key, kernel, bias} in the cache. If it is found, + /// returns the offset to the found entry for reuse. Otherwise, returns SIZE_MAX. + /// @param context - The user-specified pointer from xnn_weights_cache_provider structure. + /// @param cache_key - The key used to locate the weights cache entry. + size_t (*look_up)(void* context, const struct xnn_weights_cache_look_up_key* cache_key); + + /// Ensures that cache has enough space for `n` bytes. Returns the address to + /// store weight cache. Returns NULL if fails to reserve space. + /// @param context - The user-specified pointer from xnn_weights_cache_provider structure. + /// @param n - size to be reserved. + void* (*reserve_space)(void* context, size_t n); + + /// Looks up packed weights at `ptr` in the cache. If it is found, reuse it. + /// Otherwise, it is added to the cache. Returns the offset to the cache. + /// @param context - The user-specified pointer from xnn_weights_cache_provider structure. + /// @param cache_key - The key used to locate the weights cache entry. + /// @param ptr - pointer pointing to the packed weight. + /// @param size - size of the packed weight. + size_t (*look_up_or_insert)(void* context, const struct xnn_weights_cache_look_up_key* cache_key, void* ptr, size_t size); + + /// Returns whether the cache is finalized. + /// @param context - The user-specified pointer from xnn_weights_cache_provider structure. + bool (*is_finalized)(void* context); + + /// Returns the absolute pointer corresponding to `offset`, where the offset is returned from + /// `look_up` or `get_or_insert`. This function must be called after finalize. + /// @param context - The user-specified pointer from xnn_weights_cache_provider structure. + /// @param offset - offset to the start of internal buffer + void* (*offset_to_addr)(void* context, size_t offset); + + /// Destroy a weights cache object, as well as memory used for the cache. + /// @param context - The user-specified pointer from xnn_weights_cache_provider structure. + enum xnn_status (*delete_cache)(void* context); +}; + +/// Weights cache is a cache for packed weights. It can be reused between runtimes. +typedef struct xnn_weights_cache_provider* xnn_weights_cache_t; + +/// Create a weights cache object specifying the initial size of weights cache (in bytes). +/// +/// @param[in] size - initial capacity of the weights cache (in bytes), i.e. it can hold size bytes without growing. +/// @param weights_cache_out - pointer to the variable that will be initialized to a handle to the weights cache provider +/// upon successful return. Once created, the weights cache provider can be shared between +/// different Runtime objects. +enum xnn_status xnn_create_weights_cache_with_size(size_t size, xnn_weights_cache_t* weights_cache_out); + +enum xnn_status xnn_create_weights_cache(xnn_weights_cache_t* weights_cache_out); + +/// Finalizes the weights cache. The kind of finalization is specified by `finalization_kind`. +/// @param weights_cache - the weights cache object to finalize. +/// @param finalization_kind - the kind of finalization. +enum xnn_status xnn_finalize_weights_cache( + xnn_weights_cache_t weights_cache, + enum xnn_weights_cache_finalization_kind finalization_kind); + +// Wrapper function of the function pointers in `xnn_weights_cache_t`. +bool xnn_weights_cache_is_finalized(xnn_weights_cache_t cache); + +/// Destroy a weights cache object, as well as memory used for the cache. +/// @param weights_cache - the weights cache object to destroy. +enum xnn_status xnn_delete_weights_cache(xnn_weights_cache_t weights_cache); + +typedef struct xnn_workspace* xnn_workspace_t; + +/// Create a workspace object. +/// @param workspace_out - pointer to the variable that will be initialized to a handle to the workspace object upon +/// successful return. Once created, the workspace can be shared between different Runtime +/// objects. +enum xnn_status xnn_create_workspace(xnn_workspace_t* workspace_out); +/// Destroy a workspace object, as well as memory used by the workspace. Object destruction can be deferred until all +/// Runtime objects created with this workspace are destroyed. +/// @param workspace - the workspace object to destroy. +enum xnn_status xnn_release_workspace(xnn_workspace_t workspace); + +/// Runtime is a combination of an execution plan for subgraph Nodes and a memory manager for subgraph Values. +typedef struct xnn_runtime* xnn_runtime_t; + +enum xnn_profile_info { + /// Returns a size_t containing the number of operators. + xnn_profile_info_num_operators, + /// Returns a char[] containing the null character separated names of all operators. + xnn_profile_info_operator_name, + /// Returns a uint64_t[] with the runtimes of all operators in the same order as xnn_profile_info_operator_name. + xnn_profile_info_operator_timing, +}; + +/// Return profile information for all operators. +/// +/// @param runtime - a Runtime object created with @ref xnn_create_runtime, @ref xnn_create_runtime_v2 or +/// @ref xnn_create_runtime_v3. +/// @param param_name - type of profile information required. +/// @param param_value_size - the size in bytes of memory pointed to by param_value. If this is not sufficient then +/// param_value_size_ret will be set to the required size and xnn_status_out_of_memory will be +/// returned. +/// @param param_value - a pointer to memory location where appropriate values for a given param_value will be written. +/// @param param_value_size_ret - returns number of bytes required to write the result if param_value_size is not +/// sufficient. +enum xnn_status xnn_get_runtime_profiling_info(xnn_runtime_t runtime, + enum xnn_profile_info param_name, + size_t param_value_size, + void* param_value, + size_t* param_value_size_ret); + +/// Create a Runtime object from a subgraph. +/// +/// @param subgraph - a Subgraph object with all Values and Nodes that would be handled by the runtime. No Values or +/// Nodes can be added to the runtime once it is constructed. +/// @param weights_cache - a cache for packed weights. The runtime will look up and reuse packed weights in this cache, +/// this will reduce memory allocated for packed weights. +/// @param workspace - a workspace to hold internal tensors. The runtime will allocate space used for internal tensors +/// and track them using workspace. Workspace can be shared and reused across different runtimes. If +/// workspace is NULL, there will be no sharing: each runtime has its own workspace. +/// @param threadpool - the thread pool to be used for parallelisation of computations in the runtime. If the thread +/// pool is NULL, the computation would run on the caller thread without parallelization. +/// @param flags - binary features of the runtime. The only currently supported values are +/// XNN_FLAG_HINT_SPARSE_INFERENCE, XNN_FLAG_HINT_FP16_INFERENCE, XNN_FLAG_FORCE_FP16_INFERENCE, +/// XNN_FLAG_YIELD_WORKERS, and XNN_FLAG_TRANSIENT_INDIRECTION_BUFFER. If XNN_FLAG_YIELD_WORKERS is +/// specified, worker threads would be yielded to the system scheduler after processing the last operator +/// in the Runtime. If XNN_FLAG_TRANSIENT_INDIRECTION_BUFFER is specified, convolution operators will +/// initialize indirection buffers on each inference run using temporary memory in the workspace, instead +/// of initializing persistent indirection buffers once. +/// @param runtime_out - pointer to the variable that will be initialized with a handle to the Runtime object upon +/// successful return. Once constructed, the Runtime object is independent of the Subgraph object +/// used to create it. +enum xnn_status xnn_create_runtime_v4( + xnn_subgraph_t subgraph, + xnn_weights_cache_t weights_cache, + xnn_workspace_t workspace, + pthreadpool_t threadpool, + uint32_t flags, + xnn_runtime_t* runtime_out); + +enum xnn_status xnn_create_runtime_v3( + xnn_subgraph_t subgraph, + xnn_weights_cache_t weights_cache, + pthreadpool_t threadpool, + uint32_t flags, + xnn_runtime_t* runtime_out); + +enum xnn_status xnn_create_runtime_v2( + xnn_subgraph_t subgraph, + pthreadpool_t threadpool, + uint32_t flags, + xnn_runtime_t* runtime_out); + +enum xnn_status xnn_create_runtime( + xnn_subgraph_t subgraph, + xnn_runtime_t* runtime_out); + +struct xnn_external_value { + uint32_t id; + void* data; +}; + +/// Reshape an external value. +/// +/// @param external_id - external ID for the Value. The ID must be within the range of reversed Value IDs specified on +/// the Subgraph creation. If the external ID is XNN_INVALID_VALUE_ID, an internal ID will be +/// created for the Value. +/// @param num_dims - number of dimensions in the shape. +/// @param dims - pointer to an array of @a num_dims shape dimensions. If num_dims is 0, this pointer can be NULL. +/// XNNPACK does not keep any pointers to this array after the function returns. +enum xnn_status xnn_reshape_external_value( + xnn_runtime_t runtime, + uint32_t external_id, + size_t num_dims, + const size_t* dims); + +/// Get the external value shape. +/// +/// @param external_id - external ID for the Value. The ID must be within the range of reversed Value IDs specified on +/// the Subgraph creation. The external ID can not be XNN_INVALID_VALUE_ID. +/// @param num_dims - A valid pointer into which the number of dimensions in the shape will be written. It can not be larger than XNN_MAX_TENSOR_DIMS. +/// @param dims - pointer to an array of @a num_dims shape dimensions. This pointer can't be NULL. It must be large enough to hold +/// at least @a num_dims elements. XNNPACK does not keep any pointers to this array after the function returns. +enum xnn_status xnn_get_external_value_shape( + xnn_runtime_t runtime, + uint32_t external_id, + size_t* num_dims, + size_t* dims); + +/// Reshape the XNNPACK runtime. +/// +/// Propagates the shapes of input tensors through the graph to determine the shapes of intermediate and output tensors. +/// Memory is allocated if required. Output tensor shapes are returned by xnn_get_external_value_shape. +/// +/// @param runtime - a Runtime object created with @ref xnn_create_runtime or @ref xnn_create_runtime_v2. +enum xnn_status xnn_reshape_runtime( + xnn_runtime_t runtime); + +/// Deprecated. Use xnn_reshape_runtime and xnn_setup_runtime_v2. +/// +/// Setup data pointers for external inputs and outputs in a Runtime object and +/// allocate memory. +/// +/// @param runtime - a Runtime object created with @ref xnn_create_runtime or @ref xnn_create_runtime_v2. +/// @param num_external_values - the number of external inputs and outputs specified in this call. This number must +/// match the number of external inputs and outputs in the runtime, i.e. all external +/// inputs and outputs in the runtime must be specified in one call. +/// @param external_values - array with location information for all external inputs and outputs in the runtime. +enum xnn_status xnn_setup_runtime( + xnn_runtime_t runtime, + size_t num_external_values, + const struct xnn_external_value* external_values); + +/// Setup data pointers for external inputs and outputs in a Runtime object. +/// Should be called after xnn_reshape_runtime. +/// +/// @param runtime - a Runtime object created with @ref xnn_create_runtime or @ref xnn_create_runtime_v2. +/// @param num_external_values - the number of external inputs and outputs specified in this call. This number must +/// match the number of external inputs and outputs in the runtime, i.e. all external +/// inputs and outputs in the runtime must be specified in one call. +/// @param external_values - array with location information for all external inputs and outputs in the runtime. +enum xnn_status xnn_setup_runtime_v2( + xnn_runtime_t runtime, + size_t num_external_values, + const struct xnn_external_value* external_values); + +/// Execute forward pass for all operators in the runtime. +/// +/// @param runtime - the Runtime object with the execution plan to invoke. +enum xnn_status xnn_invoke_runtime( + xnn_runtime_t runtime); + +/// Destroy a Runtime object, as well as operators and memory associated with it. +/// +/// @param runtime - the Runtime object to destroy. +enum xnn_status xnn_delete_runtime( + xnn_runtime_t runtime); + +typedef struct xnn_operator* xnn_operator_t; + +enum xnn_status xnn_run_operator( + xnn_operator_t op, + pthreadpool_t threadpool); + +enum xnn_status xnn_delete_operator( + xnn_operator_t op); + +/// Operator API: +/// - create operator will create and populate a xnn_operator_t +/// - reshape operator will update fields in xnn_operator_t with shape/dimensions and parallelization information +/// - setup operator will update pointers to input and outputs +/// Each supported operator must have a create, reshape, and setup function. (Optionally a run function.) +/// Operators listed below are in alphabetical order by operator name; within each operator, we sort alphabetically by +/// data layout and type. We also group create, reshape, setup (and optionally run) functions of each operator together. + +enum xnn_status xnn_create_binary_elementwise_nd( + enum xnn_binary_operator type, + enum xnn_datatype datatype, + const struct xnn_quantization_params* input1_quantization, + const struct xnn_quantization_params* input2_quantization, + const struct xnn_quantization_params* output_quantization, + uint32_t flags, + xnn_operator_t* binary_op_out); + +enum xnn_status xnn_reshape_binary_elementwise_nd( + xnn_operator_t binary_op, + size_t num_input1_dims, + const size_t* input1_shape, + size_t num_input2_dims, + const size_t* input2_shape, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_binary_elementwise_nd( + xnn_operator_t binary_op, + const void* input1, + const void* input2, + void* output); + +enum xnn_status xnn_run_binary_elementwise_nd( + enum xnn_binary_operator type, + enum xnn_datatype datatype, + const struct xnn_quantization_params* input1_quantization, + const struct xnn_quantization_params* input2_quantization, + const struct xnn_quantization_params* output_quantization, + uint32_t flags, + size_t num_input1_dims, + const size_t* input1_shape, + size_t num_input2_dims, + const size_t* input2_shape, + const void* input1, + const void* input2, + void* output, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_unary_elementwise_nc( + enum xnn_unary_operator op_type, + enum xnn_datatype input_datatype, + enum xnn_datatype output_datatype, + const union xnn_unary_params* params, + const struct xnn_quantization_params* input_quantization, + const struct xnn_quantization_params* output_quantization, + uint32_t flags, + xnn_operator_t* op_out); + +enum xnn_status xnn_reshape_unary_elementwise_nc( + xnn_operator_t op, + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_unary_elementwise_nc( + xnn_operator_t op, + const void* input, + void* output); + +enum xnn_status xnn_run_unary_elementwise_nc( + // create parameters + enum xnn_unary_operator op_type, + enum xnn_datatype input_datatype, + enum xnn_datatype output_datatype, + const union xnn_unary_params* params, + const struct xnn_quantization_params* input_quantization, + const struct xnn_quantization_params* output_quantization, + uint32_t flags, + // reshape parameters + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool, + // setup parameters + const void* input, + void* output); + +enum xnn_status xnn_create_argmax_pooling2d_nhwc_f32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t flags, + xnn_operator_t* argmax_pooling_op_out); + +enum xnn_status xnn_reshape_argmax_pooling2d_nhwc_f32( + xnn_operator_t argmax_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_argmax_pooling2d_nhwc_f32( + xnn_operator_t argmax_pooling_op, + void* workspace, + const float* input, + float* output, + uint32_t* index); + +enum xnn_status xnn_create_average_pooling2d_nhwc_f16( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + float output_min, + float output_max, + uint32_t flags, + xnn_operator_t* average_pooling_op_out); + +enum xnn_status xnn_reshape_average_pooling2d_nhwc_f16( + xnn_operator_t average_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_average_pooling2d_nhwc_f16( + xnn_operator_t average_pooling_op, + void* workspace, + const void* input, + void* output); + +enum xnn_status xnn_create_average_pooling2d_nhwc_f32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + float output_min, + float output_max, + uint32_t flags, + xnn_operator_t* average_pooling_op_out); + +enum xnn_status xnn_reshape_average_pooling2d_nhwc_f32( + xnn_operator_t average_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_average_pooling2d_nhwc_f32( + xnn_operator_t average_pooling_op, + void* workspace, + const float* input, + float* output); + +enum xnn_status xnn_create_average_pooling2d_nhwc_qu8( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + uint8_t input_zero_point, + float input_scale, + uint8_t output_zero_point, + float output_scale, + uint8_t output_min, + uint8_t output_max, + uint32_t flags, + xnn_operator_t* average_pooling_op_out); + +enum xnn_status xnn_reshape_average_pooling2d_nhwc_qu8( + xnn_operator_t average_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_average_pooling2d_nhwc_qu8( + xnn_operator_t average_pooling_op, + void* workspace, + const uint8_t* input, + uint8_t* output); + +enum xnn_status xnn_create_batch_matrix_multiply_nc_f16( + uint32_t flags, + xnn_operator_t* batch_matrix_multiply_op); + +enum xnn_status xnn_reshape_batch_matrix_multiply_nc_f16( + xnn_operator_t batch_matrix_multiply_op, size_t num_batch_dims, + const size_t* batch_dims_a, const size_t* batch_dims_b, size_t m, size_t k, + size_t n, size_t* workspace_size, size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_batch_matrix_multiply_nc_f16( + xnn_operator_t batch_matrix_multiply_op, void* workspace, + const void* input_a, const void* input_b, void* output); + +enum xnn_status xnn_create_batch_matrix_multiply_nc_f32( + uint32_t flags, xnn_operator_t* batch_matrix_multiply_op); + +enum xnn_status xnn_create_batch_matrix_multiply_nc_f32_const_weights( + size_t batch_size_b, size_t k, size_t n, const float* data_b, + uint32_t flags, xnn_operator_t* batch_matrix_multiply_op); + +enum xnn_status xnn_reshape_batch_matrix_multiply_nc_f32( + xnn_operator_t batch_matrix_multiply_op, size_t num_batch_dims, + const size_t* batch_dims_a, const size_t* batch_dims_b, size_t m, size_t k, + size_t n, size_t* workspace_size, size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_batch_matrix_multiply_nc_f32( + xnn_operator_t batch_matrix_multiply_op, void* workspace, + const float* input_a, const float* input_b, float* output); + +enum xnn_status xnn_create_batch_matrix_multiply_nc_qd8_f32_qc8w( + size_t batch_size_b, size_t k, size_t n, const int8_t* data_b, + const float* scale_b, uint32_t flags, + xnn_operator_t* batch_matrix_multiply_op); + +enum xnn_status xnn_reshape_batch_matrix_multiply_nc_qd8_f32_qc8w( + xnn_operator_t batch_matrix_multiply_op, size_t num_batch_dims, + const size_t* batch_dims_a, const size_t* batch_dims_b, size_t m, size_t k, + size_t n, pthreadpool_t threadpool); + +enum xnn_status xnn_setup_batch_matrix_multiply_nc_qd8_f32_qc8w( + xnn_operator_t batch_matrix_multiply_op, const int8_t* input_a, + const struct xnn_quantization_params* quantization_params, + float* output); + +enum xnn_status xnn_create_channel_shuffle_nc_x8( + size_t groups, + size_t group_channels, + size_t input_stride, + size_t output_stride, + uint32_t flags, + xnn_operator_t* channel_shuffle_op_out); + +enum xnn_status xnn_reshape_channel_shuffle_nc_x8( + xnn_operator_t channel_shuffle_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_channel_shuffle_nc_x8( + xnn_operator_t channel_shuffle_op, + const void* input, + void* output); + +enum xnn_status xnn_create_channel_shuffle_nc_x32( + size_t groups, + size_t group_channels, + size_t input_stride, + size_t output_stride, + uint32_t flags, + xnn_operator_t* channel_shuffle_op_out); + +enum xnn_status xnn_reshape_channel_shuffle_nc_x32( + xnn_operator_t channel_shuffle_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_channel_shuffle_nc_x32( + xnn_operator_t channel_shuffle_op, + const void* input, + void* output); + +enum xnn_status xnn_create_constant_pad_nd_x8( + const void* padding_value, + uint32_t flags, + xnn_operator_t* constant_pad_op_out); + +enum xnn_status xnn_reshape_constant_pad_nd_x8( + xnn_operator_t constant_pad_op, + size_t num_dims, + const size_t* input_shape, + const size_t* pre_padding, + const size_t* post_padding, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_constant_pad_nd_x8( + xnn_operator_t constant_pad_op, + const void* input, + void* output); + +enum xnn_status xnn_run_constant_pad_nd_x8( + uint32_t flags, + size_t num_dims, + const size_t* input_shape, + const size_t* pre_paddings, + const size_t* post_paddings, + const void* input, + void* output, + const void* padding_value, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_constant_pad_nd_x16( + const void* padding_value, + uint32_t flags, + xnn_operator_t* constant_pad_op_out); + +enum xnn_status xnn_reshape_constant_pad_nd_x16( + xnn_operator_t constant_pad_op, + size_t num_dims, + const size_t* input_shape, + const size_t* pre_padding, + const size_t* post_padding, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_constant_pad_nd_x16( + xnn_operator_t constant_pad_op, + const void* input, + void* output); + +enum xnn_status xnn_run_constant_pad_nd_x16( + uint32_t flags, + size_t num_dims, + const size_t* input_shape, + const size_t* pre_paddings, + const size_t* post_paddings, + const void* input, + void* output, + const void* padding_value, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_constant_pad_nd_x32( + const void* padding_value, + uint32_t flags, + xnn_operator_t* constant_pad_op_out); + +enum xnn_status xnn_reshape_constant_pad_nd_x32( + xnn_operator_t constant_pad_op, + size_t num_dims, + const size_t* input_shape, + const size_t* pre_padding, + const size_t* post_padding, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_constant_pad_nd_x32( + xnn_operator_t constant_pad_op, + const void* input, + void* output); + +enum xnn_status xnn_run_constant_pad_nd_x32( + uint32_t flags, + size_t num_dims, + const size_t* input_shape, + const size_t* pre_paddings, + const size_t* post_paddings, + const void* input, + void* output, + const void* padding_value, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_convert_nc_f16_qd8( + uint32_t flags, + xnn_operator_t* convert_op_out); + +enum xnn_status xnn_reshape_convert_nc_f16_qd8( + xnn_operator_t convert_op, + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool); + +// quantization_params must be padded with at least XNN_EXTRA_QUANTIZATION_PARAMS entries. +enum xnn_status xnn_setup_convert_nc_f16_qd8( + xnn_operator_t convert_op, + const void* input, + int8_t* output, + struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_create_convert_nc_f32_qd8( + uint32_t flags, + xnn_operator_t* convert_op_out); + +enum xnn_status xnn_reshape_convert_nc_f32_qd8( + xnn_operator_t convert_op, + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool); + +// quantization_params must be padded with at least XNN_EXTRA_QUANTIZATION_PARAMS entries. +enum xnn_status xnn_setup_convert_nc_f32_qd8( + xnn_operator_t convert_op, + const float* input, + int8_t* output, + struct xnn_quantization_params* quantization_params); + +XNN_DEPRECATED enum xnn_status xnn_run_convert_nc_f32_f16( + size_t channels, + size_t input_stride, + size_t output_stride, + size_t batch_size, + const float* input, + void* output, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_convolution2d_nchw_f16( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nchw_f16( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nchw_f16( + xnn_operator_t convolution_op, + const void* input, + void* output); + +enum xnn_status xnn_create_convolution2d_nchw_f32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + const float* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nchw_f32( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nchw_f32( + xnn_operator_t convolution_op, + const float* input, + float* output); + +enum xnn_status xnn_create_convolution2d_nhwc_f16( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nhwc_f16( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nhwc_f16( + xnn_operator_t convolution_op, + void* workspace, + const void* input, + void* output); + +enum xnn_status xnn_create_convolution2d_nhwc_f32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + const float* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_create_convolution2d_nhwc_f32_f16( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +// Forward declare. +struct xnn_post_operation; + +/// Deprecated +enum xnn_status xnn_create_fused_convolution2d_nhwc_f32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + const float* kernel, + const float* bias, + size_t num_post_operations, + struct xnn_post_operation* post_operations, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nhwc_f32( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nhwc_f32( + xnn_operator_t convolution_op, + void* workspace, + const float* input, + float* output); + +enum xnn_status xnn_create_convolution2d_nhwc_qd8_f16_qc8w( + uint32_t input_padding_top, uint32_t input_padding_right, + uint32_t input_padding_bottom, uint32_t input_padding_left, + uint32_t kernel_height, uint32_t kernel_width, uint32_t subsampling_height, + uint32_t subsampling_width, uint32_t dilation_height, + uint32_t dilation_width, uint32_t groups, size_t group_input_channels, + size_t group_output_channels, size_t input_channel_stride, + size_t output_channel_stride, const float* kernel_scale, + const int8_t* kernel, const float* bias, float output_min, float output_max, + uint32_t flags, xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_create_convolution2d_nhwc_qd8_f32_qc8w( + uint32_t input_padding_top, uint32_t input_padding_right, + uint32_t input_padding_bottom, uint32_t input_padding_left, + uint32_t kernel_height, uint32_t kernel_width, uint32_t subsampling_height, + uint32_t subsampling_width, uint32_t dilation_height, + uint32_t dilation_width, uint32_t groups, size_t group_input_channels, + size_t group_output_channels, size_t input_channel_stride, + size_t output_channel_stride, const float* kernel_scale, + const int8_t* kernel, const float* bias, float output_min, float output_max, + uint32_t flags, xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_create_convolution2d_nhwc_qs8( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + int8_t input_zero_point, + float input_scale, + float kernel_scale, + const int8_t* kernel, + const int32_t* bias, + int8_t output_zero_point, + float output_scale, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nhwc_qd8_f16_qc8w( + xnn_operator_t convolution_op, size_t batch_size, size_t input_height, + size_t input_width, size_t* workspace_size, size_t* workspace_alignment, + size_t* output_height_out, size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_reshape_convolution2d_nhwc_qd8_f32_qc8w( + xnn_operator_t convolution_op, size_t batch_size, size_t input_height, + size_t input_width, size_t* workspace_size, size_t* workspace_alignment, + size_t* output_height_out, size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_reshape_convolution2d_nhwc_qs8( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nhwc_qd8_f16_qc8w( + xnn_operator_t convolution_op, void* workspace, const int8_t* input, + void* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_setup_convolution2d_nhwc_qd8_f32_qc8w( + xnn_operator_t convolution_op, void* workspace, const int8_t* input, + float* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_setup_convolution2d_nhwc_qs8( + xnn_operator_t convolution_op, + void* workspace, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_convolution2d_nhwc_qs8_qc8w( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + int8_t input_zero_point, + float input_scale, + const float* kernel_scale, + const int8_t* kernel, + const int32_t* bias, + int8_t output_zero_point, + float output_scale, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nhwc_qs8_qc8w( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nhwc_qs8_qc8w( + xnn_operator_t convolution_op, + void* workspace, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_convolution2d_nhwc_qu8( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t subsampling_height, + uint32_t subsampling_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_channel_stride, + size_t output_channel_stride, + uint8_t input_zero_point, + float input_scale, + uint8_t kernel_zero_point, + float kernel_scale, + const uint8_t* kernel, + const int32_t* bias, + uint8_t output_zero_point, + float output_scale, + uint8_t output_min, + uint8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* convolution_op_out); + +enum xnn_status xnn_reshape_convolution2d_nhwc_qu8( + xnn_operator_t convolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* workspace_size, + size_t* workspace_alignment, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_convolution2d_nhwc_qu8( + xnn_operator_t convolution_op, + void* workspace, + const uint8_t* input, + uint8_t* output); + +enum xnn_status xnn_create_copy_nc_x8( + uint32_t flags, + xnn_operator_t* copy_op_out); + +enum xnn_status xnn_reshape_copy_nc_x8( + xnn_operator_t copy_op, + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_copy_nc_x8( + xnn_operator_t copy_op, + const void* input, + void* output); + +enum xnn_status xnn_create_copy_nc_x16( + uint32_t flags, + xnn_operator_t* copy_op_out); + +enum xnn_status xnn_reshape_copy_nc_x16( + xnn_operator_t copy_op, + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_copy_nc_x16( + xnn_operator_t copy_op, + const void* input, + void* output); + +enum xnn_status xnn_create_copy_nc_x32( + uint32_t flags, + xnn_operator_t* copy_op_out); + +enum xnn_status xnn_reshape_copy_nc_x32( + xnn_operator_t copy_op, + size_t batch_size, + size_t channels, + size_t input_stride, + size_t output_stride, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_copy_nc_x32( + xnn_operator_t copy_op, + const void* input, + void* output); + +enum xnn_status xnn_run_copy_nc_x32( + size_t channels, + size_t input_stride, + size_t output_stride, + size_t batch_size, + const uint32_t* input, + uint32_t* output, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_deconvolution2d_nhwc_f16( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_reshape_deconvolution2d_nhwc_f16( + xnn_operator_t deconvolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + uint32_t adjustment_height, + uint32_t adjustment_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_deconvolution2d_nhwc_f16( + xnn_operator_t deconvolution_op, + const void* input, + void* output); + +enum xnn_status xnn_create_deconvolution2d_nhwc_f32( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + const float* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_create_deconvolution2d_nhwc_f32_f16( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_reshape_deconvolution2d_nhwc_f32( + xnn_operator_t deconvolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + uint32_t adjustment_height, + uint32_t adjustment_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_deconvolution2d_nhwc_f32( + xnn_operator_t deconvolution_op, + const float* input, + float* output); + +enum xnn_status xnn_create_deconvolution2d_nhwc_qd8_f32_qc8w( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + const float* kernel_scale, + const int8_t* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_reshape_deconvolution2d_nhwc_qd8_f32_qc8w( + xnn_operator_t deconvolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + uint32_t adjustment_height, + uint32_t adjustment_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_deconvolution2d_nhwc_qd8_f32_qc8w( + xnn_operator_t deconvolution_op, + const int8_t* input, + float* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_create_deconvolution2d_nhwc_qs8( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + int8_t input_zero_point, + float input_scale, + float kernel_scale, + const int8_t* kernel, + const int32_t* bias, + int8_t output_zero_point, + float output_scale, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_reshape_deconvolution2d_nhwc_qs8( + xnn_operator_t deconvolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + uint32_t adjustment_height, + uint32_t adjustment_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_deconvolution2d_nhwc_qs8( + xnn_operator_t deconvolution_op, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_deconvolution2d_nhwc_qs8_qc8w( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + int8_t input_zero_point, + float input_scale, + const float* kernel_scale, + const int8_t* kernel, + const int32_t* bias, + int8_t output_zero_point, + float output_scale, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_reshape_deconvolution2d_nhwc_qs8_qc8w( + xnn_operator_t deconvolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + uint32_t adjustment_height, + uint32_t adjustment_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_deconvolution2d_nhwc_qs8_qc8w( + xnn_operator_t deconvolution_op, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_deconvolution2d_nhwc_qu8( + uint32_t output_padding_top, + uint32_t output_padding_right, + uint32_t output_padding_bottom, + uint32_t output_padding_left, + uint32_t kernel_height, + uint32_t kernel_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint32_t groups, + size_t group_input_channels, + size_t group_output_channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + uint8_t input_zero_point, + float input_scale, + uint8_t kernel_zero_point, + float kernel_scale, + const uint8_t* kernel, + const int32_t* bias, + uint8_t output_zero_point, + float output_scale, + uint8_t output_min, + uint8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* deconvolution_op_out); + +enum xnn_status xnn_reshape_deconvolution2d_nhwc_qu8( + xnn_operator_t deconvolution_op, + size_t batch_size, + size_t input_height, + size_t input_width, + uint32_t adjustment_height, + uint32_t adjustment_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_deconvolution2d_nhwc_qu8( + xnn_operator_t deconvolution_op, + const uint8_t* input, + uint8_t* output); + +enum xnn_status xnn_create_depth_to_space_nchw2nhwc_x16( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* depth_to_space_op_out); + +enum xnn_status xnn_reshape_depth_to_space_nchw2nhwc_x16( + xnn_operator_t depth_to_space_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_depth_to_space_nchw2nhwc_x16( + xnn_operator_t depth_to_space_op, + const void* input, + void* output); + +enum xnn_status xnn_create_depth_to_space_nchw2nhwc_x32( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* depth_to_space_op_out); + +enum xnn_status xnn_reshape_depth_to_space_nchw2nhwc_x32( + xnn_operator_t depth_to_space_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_depth_to_space_nchw2nhwc_x32( + xnn_operator_t depth_to_space_op, + const void* input, + void* output); + +enum xnn_status xnn_create_depth_to_space_nhwc_x8( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* depth_to_space_op_out); + +enum xnn_status xnn_reshape_depth_to_space_nhwc_x8( + xnn_operator_t depth_to_space_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_depth_to_space_nhwc_x8( + xnn_operator_t depth_to_space_op, + const void* input, + void* output); + +enum xnn_status xnn_create_depth_to_space_nhwc_x16( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* depth_to_space_op_out); + +enum xnn_status xnn_reshape_depth_to_space_nhwc_x16( + xnn_operator_t depth_to_space_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_depth_to_space_nhwc_x16( + xnn_operator_t depth_to_space_op, + const void* input, + void* output); + +enum xnn_status xnn_create_depth_to_space_nhwc_x32( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* depth_to_space_op_out); + +enum xnn_status xnn_reshape_depth_to_space_nhwc_x32( + xnn_operator_t depth_to_space_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_depth_to_space_nhwc_x32( + xnn_operator_t depth_to_space_op, + const void* input, + void* output); + +enum xnn_status xnn_create_dynamic_fully_connected_nc_f16( + float output_min, + float output_max, + uint32_t flags, + xnn_operator_t* dynamic_fully_connected_op_out); + +enum xnn_status xnn_reshape_dynamic_fully_connected_nc_f16( + xnn_operator_t dynamic_fully_connected_op, + size_t batch_size, + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_dynamic_fully_connected_nc_f16( + xnn_operator_t dynamic_fully_connected_op, + void* workspace, + const void* input, + const void* kernel, + const void* bias, + void* output); + +enum xnn_status xnn_create_dynamic_fully_connected_nc_f32( + float output_min, + float output_max, + uint32_t flags, + xnn_operator_t* dynamic_fully_connected_op_out); + +enum xnn_status xnn_reshape_dynamic_fully_connected_nc_f32( + xnn_operator_t dynamic_fully_connected_op, + size_t batch_size, + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_dynamic_fully_connected_nc_f32( + xnn_operator_t dynamic_fully_connected_op, + void* workspace, + const float* input, + const float* kernel, + const float* bias, + float* output); + +enum xnn_status xnn_create_fully_connected_nc_f16( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_f16( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_f16( + xnn_operator_t fully_connected_op, + const void* input, + void* output); + +enum xnn_status xnn_create_fully_connected_nc_f32_f16( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + const void* kernel, + const void* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_create_fully_connected_nc_f32( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + const float* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_f32_f16( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_reshape_fully_connected_nc_f32( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_f32_f16( + xnn_operator_t fully_connected_op, + const float* input, + float* output); + +enum xnn_status xnn_setup_fully_connected_nc_f32( + xnn_operator_t fully_connected_op, + const float* input, + float* output); + +enum xnn_status xnn_create_fully_connected_nc_f32_qc4w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + uint8_t kernel_zero_point, + const float* kernel_scale, + const uint8_t* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_f32_qc4w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_f32_qc4w( + xnn_operator_t fully_connected_op, + const float* input, + float* output); + +enum xnn_status xnn_create_fully_connected_nc_f32_qc8w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + const float* kernel_scale, + const int8_t* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_f32_qc8w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_f32_qc8w( + xnn_operator_t fully_connected_op, + const float* input, + float* output); + +enum xnn_status xnn_create_fully_connected_nc_qd8_f16_qc4w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + uint8_t kernel_zero_point, + const float* kernel_scale, + const void* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_setup_fully_connected_nc_qd8_f16_qc4w( + xnn_operator_t fully_connected_op, + const int8_t* input, + void* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_reshape_fully_connected_nc_qd8_f16_qc4w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_fully_connected_nc_qd8_f16_qb4w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + size_t block_size, + uint8_t kernel_zero_point, + const uint16_t* kernel_scale, + const void* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_qd8_f16_qb4w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_qd8_f16_qb4w( + xnn_operator_t fully_connected_op, + const int8_t* input, + void* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_create_fully_connected_nc_qd8_f32_qc4w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + uint8_t kernel_zero_point, + const float* kernel_scale, + const void* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_setup_fully_connected_nc_qd8_f32_qc4w( + xnn_operator_t fully_connected_op, + const int8_t* input, + float* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_reshape_fully_connected_nc_qd8_f32_qc4w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_fully_connected_nc_qd8_f32_qb4w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + size_t block_size, + uint8_t kernel_zero_point, + const uint16_t* kernel_scale, + const void* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_qd8_f32_qb4w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_qd8_f32_qb4w( + xnn_operator_t fully_connected_op, + const int8_t* input, + float* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_create_fully_connected_nc_qd8_f16_qc8w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + const float* kernel_scale, + const int8_t* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_setup_fully_connected_nc_qd8_f16_qc8w( + xnn_operator_t fully_connected_op, + const int8_t* input, + void* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_reshape_fully_connected_nc_qd8_f16_qc8w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_fully_connected_nc_qd8_f32_qc8w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + const float* kernel_scale, + const int8_t* kernel, + const float* bias, + float output_min, + float output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_setup_fully_connected_nc_qd8_f32_qc8w( + xnn_operator_t fully_connected_op, + const int8_t* input, + float* output, + const struct xnn_quantization_params* quantization_params); + +enum xnn_status xnn_reshape_fully_connected_nc_qd8_f32_qc8w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_fully_connected_nc_qs8( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + int8_t input_zero_point, + float input_scale, + float kernel_scale, + const int8_t* kernel, + const int32_t* bias, + int8_t output_zero_point, + float output_scale, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_qs8( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_qs8( + xnn_operator_t fully_connected_op, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_fully_connected_nc_qs8_qc8w( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + int8_t input_zero_point, + float input_scale, + const float* kernel_scale, + const int8_t* kernel, + const int32_t* bias, + int8_t output_zero_point, + float output_scale, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_qs8_qc8w( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_qs8_qc8w( + xnn_operator_t fully_connected_op, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_fully_connected_nc_qu8( + size_t input_channels, + size_t output_channels, + size_t input_stride, + size_t output_stride, + uint8_t input_zero_point, + float input_scale, + uint8_t kernel_zero_point, + float kernel_scale, + const uint8_t* kernel, + const int32_t* bias, + uint8_t output_zero_point, + float output_scale, + uint8_t output_min, + uint8_t output_max, + uint32_t flags, + xnn_code_cache_t code_cache, + xnn_weights_cache_t weights_cache, + xnn_operator_t* fully_connected_op_out); + +enum xnn_status xnn_reshape_fully_connected_nc_qu8( + xnn_operator_t fully_connected_op, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_fully_connected_nc_qu8( + xnn_operator_t fully_connected_op, + const uint8_t* input, + uint8_t* output); + + +enum xnn_status xnn_create_max_pooling2d_nhwc_f16( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + float output_min, + float output_max, + uint32_t flags, + xnn_operator_t* max_pooling_op_out); + +enum xnn_status xnn_reshape_max_pooling2d_nhwc_f16( + xnn_operator_t max_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_max_pooling2d_nhwc_f16( + xnn_operator_t max_pooling_op, + const void* input, + void* output); + +enum xnn_status xnn_create_max_pooling2d_nhwc_f32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + float output_min, + float output_max, + uint32_t flags, + xnn_operator_t* max_pooling_op_out); + +enum xnn_status xnn_reshape_max_pooling2d_nhwc_f32( + xnn_operator_t max_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_max_pooling2d_nhwc_f32( + xnn_operator_t max_pooling_op, + const float* input, + float* output); + +enum xnn_status xnn_create_max_pooling2d_nhwc_s8( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + int8_t output_min, + int8_t output_max, + uint32_t flags, + xnn_operator_t* max_pooling_op_out); + +enum xnn_status xnn_reshape_max_pooling2d_nhwc_s8( + xnn_operator_t max_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_max_pooling2d_nhwc_s8( + xnn_operator_t max_pooling_op, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_max_pooling2d_nhwc_u8( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + uint32_t stride_height, + uint32_t stride_width, + uint32_t dilation_height, + uint32_t dilation_width, + uint8_t output_min, + uint8_t output_max, + uint32_t flags, + xnn_operator_t* max_pooling_op_out); + +enum xnn_status xnn_reshape_max_pooling2d_nhwc_u8( + xnn_operator_t max_pooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_max_pooling2d_nhwc_u8( + xnn_operator_t max_pooling_op, + const uint8_t* input, + uint8_t* output); + +enum xnn_status xnn_create_reduce_nd( + enum xnn_reduce_operator reduce_operator_type, + enum xnn_datatype datatype, + const struct xnn_quantization_params* input_quantization, + const struct xnn_quantization_params* output_quantization, + uint32_t flags, + xnn_operator_t* reduce_op_out); + +enum xnn_status xnn_reshape_reduce_nd( // + xnn_operator_t reduce_op, // + size_t num_reduction_axes, // + const int64_t* reduction_axes, // + size_t num_input_dims, // + const size_t* input_shape, // + size_t* workspace_size, // + size_t* workspace_alignment, // + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_reduce_nd( + xnn_operator_t reduce_op, + void* workspace, + const void* input, + void* output); + +enum xnn_status xnn_create_resize_bilinear2d_nchw_f32( + size_t output_height, + size_t output_width, + uint32_t flags, + xnn_operator_t* resize_op_out); + +enum xnn_status xnn_reshape_resize_bilinear2d_nchw_f32( + xnn_operator_t resize_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_resize_bilinear2d_nchw_f32( + xnn_operator_t resize_op, + const float* input, + float* output); + +enum xnn_status xnn_create_resize_bilinear2d_nchw_f16( + size_t output_height, + size_t output_width, + uint32_t flags, + xnn_operator_t* resize_op_out); + +enum xnn_status xnn_reshape_resize_bilinear2d_nchw_f16( + xnn_operator_t resize_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_resize_bilinear2d_nchw_f16( + xnn_operator_t resize_op, + const void* input, + void* output); + +enum xnn_status xnn_create_resize_bilinear2d_nhwc_f16( + size_t output_height, + size_t output_width, + uint32_t flags, + xnn_operator_t* resize_op_out); + +enum xnn_status xnn_reshape_resize_bilinear2d_nhwc_f16( + xnn_operator_t resize_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_resize_bilinear2d_nhwc_f16( + xnn_operator_t resize_op, + void* workspace, + const void* input, + void* output); + +enum xnn_status xnn_create_resize_bilinear2d_nhwc_f32( + size_t output_height, + size_t output_width, + uint32_t flags, + xnn_operator_t* resize_op_out); + +enum xnn_status xnn_reshape_resize_bilinear2d_nhwc_f32( + xnn_operator_t resize_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_resize_bilinear2d_nhwc_f32( + xnn_operator_t resize_op, + void* workspace, + const float* input, + float* output); + +enum xnn_status xnn_create_resize_bilinear2d_nhwc_s8( + size_t output_height, + size_t output_width, + uint32_t flags, + xnn_operator_t* resize_op_out); + +enum xnn_status xnn_reshape_resize_bilinear2d_nhwc_s8( + xnn_operator_t resize_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_resize_bilinear2d_nhwc_s8( + xnn_operator_t resize_op, + void* workspace, + const int8_t* input, + int8_t* output); + +enum xnn_status xnn_create_resize_bilinear2d_nhwc_u8( + size_t output_height, + size_t output_width, + uint32_t flags, + xnn_operator_t* resize_op_out); + +enum xnn_status xnn_reshape_resize_bilinear2d_nhwc_u8( + xnn_operator_t resize_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_resize_bilinear2d_nhwc_u8( + xnn_operator_t resize_op, + void* workspace, + const uint8_t* input, + uint8_t* output); + +enum xnn_status xnn_create_rope_nthc_f16( + uint32_t flags, + xnn_operator_t* rope_op_out); + +enum xnn_status xnn_reshape_rope_nthc_f16( + xnn_operator_t rope_op, + size_t batch_size, + size_t tokens, + size_t heads, + size_t channels, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_rope_nthc_f16( + xnn_operator_t rope_op, + const void* input, + const void* weights, + void* output); + +enum xnn_status xnn_create_rope_nthc_f32( + uint32_t flags, + xnn_operator_t* rope_op_out); + +enum xnn_status xnn_reshape_rope_nthc_f32( + xnn_operator_t rope_op, + size_t batch_size, + size_t tokens, + size_t heads, + size_t channels, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_rope_nthc_f32( + xnn_operator_t rope_op, + const float* input, + const float* weights, + float* output); + +// N: batch size +// H: number of heads +// T: tokens (sequence length) +// C: channels (head dimension) +enum xnn_status xnn_create_scaled_dot_product_attention_nhtc_f16( + enum xnn_attention_logits_cap_type cap_type, + const void* cap_params, + uint32_t flags, + xnn_operator_t* attention_op_out); + +enum xnn_status xnn_reshape_scaled_dot_product_attention_nhtc_f16( + xnn_operator_t attention_op, + size_t batch_size, + size_t query_heads, + // Number of tokens in query. + size_t query_tokens, + size_t key_value_heads, + // Number of tokens in key/value. For self-attention, this is same as tokens. + size_t key_value_tokens, + size_t query_key_channels, + size_t value_channels, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +// Query is of dimension [batch_size, query_heads, query_tokens, channels]. +// Key and value are of dimension [batch_size, key_value_heads, key_value_tokens, channels]. +// Scale is of dimension [channels]. +// Mask is of dimension [query_tokens, key_value_tokens]. +enum xnn_status xnn_setup_scaled_dot_product_attention_nhtc_f16( + xnn_operator_t attention_op, + void* workspace, + const void* query, + const void* key, + const void* value, + const void* scale, + const void* mask, + void* output); + +// N: batch size +// H: number of heads +// T: tokens (sequence length) +// C: channels (head dimension) +enum xnn_status xnn_create_scaled_dot_product_attention_nhtc_f32( + enum xnn_attention_logits_cap_type cap_type, + const void* cap_params, + uint32_t flags, + xnn_operator_t* attention_op_out); + +enum xnn_status xnn_reshape_scaled_dot_product_attention_nhtc_f32( + xnn_operator_t attention_op, + size_t batch_size, + size_t query_heads, + // Number of tokens in query. + size_t query_tokens, + size_t key_value_heads, + // Number of tokens in key/value. For self-attention, this is same as tokens. + size_t key_value_tokens, + size_t query_key_channels, + size_t value_channels, + size_t* workspace_size, + size_t* workspace_alignment, + pthreadpool_t threadpool); + +// Query is of dimension [batch_size, query_heads, query_tokens, query_key_channels]. +// Key and value are of dimension [batch_size, key_value_heads, key_value_tokens, query_key_channels]. +// Scale is of dimension [query_key_channels]. +// Mask is of dimension [query_tokens, key_value_tokens]. +// Output is of dimension [batch_size, query_heads, query_tokens, value_channels]. +enum xnn_status xnn_setup_scaled_dot_product_attention_nhtc_f32( + xnn_operator_t attention_op, + void* workspace, + const float* query, + const float* key, + const float* value, + const float* scale, + const float* mask, + float* output); + + +enum xnn_status xnn_create_slice_nd_x16( + uint32_t flags, + xnn_operator_t* slice_op_out); + +enum xnn_status xnn_reshape_slice_nd_x16( + xnn_operator_t slice_op, + size_t num_dims, + const size_t* input_shape, + const size_t* offsets, + const size_t* sizes, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_slice_nd_x16( + xnn_operator_t slice_op, + const void* input, + void* output); + +enum xnn_status xnn_create_slice_nd_x32( + uint32_t flags, + xnn_operator_t* slice_op_out); + +enum xnn_status xnn_reshape_slice_nd_x32( + xnn_operator_t slice_op, + size_t num_dims, + const size_t* input_shape, + const size_t* offsets, + const size_t* sizes, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_slice_nd_x32( + xnn_operator_t slice_op, + const void* input, + void* output); + +enum xnn_status xnn_run_slice_nd_x32( + size_t num_dims, + const size_t* input_shape, + const size_t* offsets, + const size_t* sizes, + const void* input, + void* output, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_softmax_nc_f16( + uint32_t flags, + xnn_operator_t* softmax_op_out); + +enum xnn_status xnn_reshape_softmax_nc_f16( + xnn_operator_t softmax_op, + size_t channels, + size_t input_stride, + size_t output_stride, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_softmax_nc_f16( + xnn_operator_t softmax_op, + const void* input, + void* output); + +enum xnn_status xnn_create_softmax_nc_f32( + uint32_t flags, + xnn_operator_t* softmax_op_out); + +enum xnn_status xnn_reshape_softmax_nc_f32( + xnn_operator_t softmax_op, + size_t channels, + size_t input_stride, + size_t output_stride, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_softmax_nc_f32( + xnn_operator_t softmax_op, + const float* input, + float* output); + +enum xnn_status xnn_create_softmax_nc_qu8( + float input_scale, + uint8_t output_zero_point, + float output_scale, + uint32_t flags, + xnn_operator_t* softmax_op_out); + +enum xnn_status xnn_reshape_softmax_nc_qu8( + xnn_operator_t softmax_op, + size_t channels, + size_t input_stride, + size_t output_stride, + size_t batch_size, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_softmax_nc_qu8( + xnn_operator_t softmax_op, + const uint8_t* input, + uint8_t* output); + +enum xnn_status xnn_create_space_to_depth_nhwc_x16( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* space_to_depth_op_out); + +enum xnn_status xnn_reshape_space_to_depth_nhwc_x16( + xnn_operator_t space_to_depth_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_space_to_depth_nhwc_x16( + xnn_operator_t space_to_depth_op, + const void* input, + void* output); + +enum xnn_status xnn_create_space_to_depth_nhwc_x32( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* space_to_depth_op_out); + +enum xnn_status xnn_reshape_space_to_depth_nhwc_x32( + xnn_operator_t space_to_depth_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_space_to_depth_nhwc_x32( + xnn_operator_t space_to_depth_op, + const void* input, + void* output); + +enum xnn_status xnn_create_transpose_nd_x8( + uint32_t flags, + xnn_operator_t* transpose_op_out); + +enum xnn_status xnn_reshape_transpose_nd_x8( + xnn_operator_t transpose_op, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_transpose_nd_x8( + xnn_operator_t transpose_op, + const void* input, + void* output); + +enum xnn_status xnn_run_transpose_nd_x8( + const void* input, + void* output, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_transpose_nd_x16( + uint32_t flags, + xnn_operator_t* transpose_op_out); + +enum xnn_status xnn_reshape_transpose_nd_x16( + xnn_operator_t transpose_op, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_transpose_nd_x16( + xnn_operator_t transpose_op, + const void* input, + void* output); + +enum xnn_status xnn_run_transpose_nd_x16( + const void* input, + void* output, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_transpose_nd_x32( + uint32_t flags, + xnn_operator_t* transpose_op_out); + +enum xnn_status xnn_reshape_transpose_nd_x32( + xnn_operator_t transpose_op, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_transpose_nd_x32( + xnn_operator_t transpose_op, + const void* input, + void* output); + +enum xnn_status xnn_run_transpose_nd_x32( + const void* input, + void* output, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_transpose_nd_x64( + uint32_t flags, + xnn_operator_t* transpose_op_out); + +enum xnn_status xnn_reshape_transpose_nd_x64( + xnn_operator_t transpose_op, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_transpose_nd_x64( + xnn_operator_t transpose_op, + const void* input, + void* output); + +enum xnn_status xnn_run_transpose_nd_x64( + const void* input, + void* output, + size_t num_dims, + const size_t* input_shape, + const size_t* output_perm, + uint32_t flags, + pthreadpool_t threadpool); + +enum xnn_status xnn_create_unpooling2d_nhwc_x32( + uint32_t input_padding_top, + uint32_t input_padding_right, + uint32_t input_padding_bottom, + uint32_t input_padding_left, + uint32_t pooling_height, + uint32_t pooling_width, + size_t channels, + size_t input_pixel_stride, + size_t output_pixel_stride, + uint32_t flags, + xnn_operator_t* unpooling_op_out); + +enum xnn_status xnn_reshape_unpooling2d_nhwc_x32( + xnn_operator_t unpooling_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t* output_height_out, + size_t* output_width_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_unpooling2d_nhwc_x32( + xnn_operator_t unpooling_op, + const void* input, + const uint32_t* index, + void* output); + +enum xnn_status xnn_create_slice_nd_x8( + uint32_t flags, + xnn_operator_t* slice_op_out); + +enum xnn_status xnn_reshape_slice_nd_x8( + xnn_operator_t slice_op, + size_t num_dims, + const size_t* input_shape, + const size_t* offsets, + const size_t* sizes, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_slice_nd_x8( + xnn_operator_t slice_op, + const void* input, + void* output); + +enum xnn_status xnn_create_space_to_depth_nhwc_x8( + uint32_t block_size, + uint32_t flags, + xnn_operator_t* space_to_depth_op_out); + +enum xnn_status xnn_reshape_space_to_depth_nhwc_x8( + xnn_operator_t space_to_depth_op, + size_t batch_size, + size_t input_height, + size_t input_width, + size_t input_channels, + size_t* output_height_out, + size_t* output_width_out, + size_t* output_channels_out, + pthreadpool_t threadpool); + +enum xnn_status xnn_setup_space_to_depth_nhwc_x8( + xnn_operator_t space_to_depth_op, + const void* input, + void* output); + +#ifdef __cplusplus +} // extern "C" +#endif + +#else +#error "This file should not be included when either TORCH_STABLE_ONLY or TORCH_TARGET_VERSION is defined." +#endif // !defined(TORCH_STABLE_ONLY) && !defined(TORCH_TARGET_VERSION) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c277d2e3ab71a72500a179b5e3c374c66a48f06c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/__init__.py @@ -0,0 +1,295 @@ +# mypy: allow-untyped-defs +import warnings +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +import torch._C + +# These are imported so users can access them from the `torch.jit` module +from torch._jit_internal import ( + _Await, + _drop, + _IgnoreContextManager, + _isinstance, + _overload, + _overload_method, + export, + Final, + Future, + ignore, + is_scripting, + unused, +) +from torch.jit._async import fork, wait +from torch.jit._await import _awaitable, _awaitable_nowait, _awaitable_wait +from torch.jit._decomposition_utils import _register_decomposition +from torch.jit._freeze import freeze, optimize_for_inference, run_frozen_optimizations +from torch.jit._fuser import ( + fuser, + last_executed_optimized_graph, + optimized_execution, + set_fusion_strategy, +) +from torch.jit._ir_utils import _InsertPoint +from torch.jit._script import ( + _ScriptProfile, + _unwrap_optional, + Attribute, + CompilationUnit, + interface, + RecursiveScriptClass, + RecursiveScriptModule, + script, + script_method, + ScriptFunction, + ScriptModule, + ScriptWarning, +) +from torch.jit._serialization import ( + jit_module_from_flatbuffer, + load, + save, + save_jit_module_to_flatbuffer, +) +from torch.jit._trace import ( + _flatten, + _get_trace_graph, + _script_if_tracing, + _unique_state_dict, + is_tracing, + ONNXTracedModule, + TopLevelTracedModule, + trace, + trace_module, + TracedModule, + TracerWarning, + TracingCheckError, +) +from torch.utils import set_module + + +__all__ = [ + "Attribute", + "CompilationUnit", + "Error", + "Future", + "ScriptFunction", + "ScriptModule", + "annotate", + "enable_onednn_fusion", + "export", + "export_opnames", + "fork", + "freeze", + "interface", + "ignore", + "isinstance", + "load", + "onednn_fusion_enabled", + "optimize_for_inference", + "save", + "script", + "script_if_tracing", + "set_fusion_strategy", + "strict_fusion", + "trace", + "trace_module", + "unused", + "wait", +] + +# For backwards compatibility +_fork = fork +_wait = wait +_set_fusion_strategy = set_fusion_strategy + + +def export_opnames(m): + r""" + Generate new bytecode for a Script module. + + Returns what the op list would be for a Script Module based off the current code base. + + If you have a LiteScriptModule and want to get the currently present + list of ops call _export_operator_list instead. + """ + return torch._C._export_opnames(m._c) + + +# torch.jit.Error +Error = torch._C.JITException +set_module(Error, "torch.jit") +# This is not perfect but works in common cases +Error.__name__ = "Error" +Error.__qualname__ = "Error" + + +# for use in python if using annotate +def annotate(the_type, the_value): + """Use to give type of `the_value` in TorchScript compiler. + + This method is a pass-through function that returns `the_value`, used to hint TorchScript + compiler the type of `the_value`. It is a no-op when running outside of TorchScript. + + Though TorchScript can infer correct type for most Python expressions, there are some cases where + type inference can be wrong, including: + + - Empty containers like `[]` and `{}`, which TorchScript assumes to be container of `Tensor` + - Optional types like `Optional[T]` but assigned a valid value of type `T`, TorchScript would assume + it is type `T` rather than `Optional[T]` + + Note that `annotate()` does not help in `__init__` method of `torch.nn.Module` subclasses because it + is executed in eager mode. To annotate types of `torch.nn.Module` attributes, + use :meth:`~torch.jit.Attribute` instead. + + Example: + + .. testcode:: + + import torch + from typing import Dict + + @torch.jit.script + def fn(): + # Telling TorchScript that this empty dictionary is a (str -> int) dictionary + # instead of default dictionary type of (str -> Tensor). + d = torch.jit.annotate(Dict[str, int], {}) + + # Without `torch.jit.annotate` above, following statement would fail because of + # type mismatch. + d["name"] = 20 + + .. testcleanup:: + + del fn + + Args: + the_type: Python type that should be passed to TorchScript compiler as type hint for `the_value` + the_value: Value or expression to hint type for. + + Returns: + `the_value` is passed back as return value. + """ + return the_value + + +def script_if_tracing(fn): + """ + Compiles ``fn`` when it is first called during tracing. + + ``torch.jit.script`` has a non-negligible start up time when it is first called due to + lazy-initializations of many compiler builtins. Therefore you should not use + it in library code. However, you may want to have parts of your library work + in tracing even if they use control flow. In these cases, you should use + ``@torch.jit.script_if_tracing`` to substitute for + ``torch.jit.script``. + + Args: + fn: A function to compile. + + Returns: + If called during tracing, a :class:`ScriptFunction` created by `torch.jit.script` is returned. + Otherwise, the original function `fn` is returned. + """ + return _script_if_tracing(fn) + + +# for torch.jit.isinstance +def isinstance(obj, target_type): + """ + Provide container type refinement in TorchScript. + + It can refine parameterized containers of the List, Dict, Tuple, and Optional types. E.g. ``List[str]``, + ``Dict[str, List[torch.Tensor]]``, ``Optional[Tuple[int,str,int]]``. It can also + refine basic types such as bools and ints that are available in TorchScript. + + Args: + obj: object to refine the type of + target_type: type to try to refine obj to + Returns: + ``bool``: True if obj was successfully refined to the type of target_type, + False otherwise with no new type refinement + + + Example (using ``torch.jit.isinstance`` for type refinement): + .. testcode:: + + import torch + from typing import Any, Dict, List + + class MyModule(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + + def forward(self, input: Any): # note the Any type + if torch.jit.isinstance(input, List[torch.Tensor]): + for t in input: + y = t.clamp(0, 0.5) + elif torch.jit.isinstance(input, Dict[str, str]): + for val in input.values(): + print(val) + + m = torch.jit.script(MyModule()) + x = [torch.rand(3,3), torch.rand(4,3)] + m(x) + y = {"key1":"val1","key2":"val2"} + m(y) + """ + return _isinstance(obj, target_type) + + +class strict_fusion: + """ + Give errors if not all nodes have been fused in inference, or symbolically differentiated in training. + + Example: + Forcing fusion of additions. + + .. code-block:: python + + @torch.jit.script + def foo(x): + with torch.jit.strict_fusion(): + return x + x + x + + """ + + def __init__(self) -> None: + if not torch._jit_internal.is_scripting(): + warnings.warn("Only works in script mode", stacklevel=2) + + def __enter__(self): + pass + + def __exit__(self, type: Any, value: Any, tb: Any) -> None: + pass + + +# Context manager for globally hiding source ranges when printing graphs. +# Note that these functions are exposed to Python as static members of the +# Graph class, so mypy checks need to be skipped. +@contextmanager +def _hide_source_ranges() -> Iterator[None]: + old_enable_source_ranges = torch._C.Graph.global_print_source_ranges # type: ignore[attr-defined] + try: + torch._C.Graph.set_global_print_source_ranges(False) # type: ignore[attr-defined] + yield + finally: + torch._C.Graph.set_global_print_source_ranges(old_enable_source_ranges) # type: ignore[attr-defined] + + +def enable_onednn_fusion(enabled: bool) -> None: + """Enable or disables onednn JIT fusion based on the parameter `enabled`.""" + torch._C._jit_set_llga_enabled(enabled) + + +def onednn_fusion_enabled(): + """Return whether onednn JIT fusion is enabled.""" + return torch._C._jit_llga_enabled() + + +del Any + +if not torch._C._jit_init(): + raise RuntimeError("JIT initialization failed") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_async.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_async.py new file mode 100644 index 0000000000000000000000000000000000000000..533064b6cd76f39e3934b5ca378023068c0762cb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_async.py @@ -0,0 +1,115 @@ +# mypy: allow-untyped-defs +"""Async API. + +This module contains the API for parallelism in TorchScript, notably: + * torch.jit.fork + * torch.jit.wait + +This is not intended to be imported directly; please use the exposed +functionalities in `torch.jit`. +""" + +import torch +from torch._jit_internal import Future +from torch.jit._builtins import _register_builtin +from torch.utils import set_module + + +set_module(Future, "torch.jit") + + +def fork(func, *args, **kwargs): + r""" + Create an asynchronous task executing `func` and a reference to the value of the result of this execution. + + `fork` will return immediately, so the return value of `func` may not have been computed yet. To force completion + of the task and access the return value invoke `torch.jit.wait` on the Future. `fork` invoked + with a `func` which returns `T` is typed as `torch.jit.Future[T]`. `fork` calls can be arbitrarily + nested, and may be invoked with positional and keyword arguments. + Asynchronous execution will only occur when run in TorchScript. If run in pure python, + `fork` will not execute in parallel. `fork` will also not execute in parallel when invoked + while tracing, however the `fork` and `wait` calls will be captured in the exported IR Graph. + + .. warning:: + `fork` tasks will execute non-deterministically. We recommend only spawning + parallel fork tasks for pure functions that do not modify their inputs, + module attributes, or global state. + + Args: + func (callable or torch.nn.Module): A Python function or `torch.nn.Module` + that will be invoked. If executed in TorchScript, it will execute asynchronously, + otherwise it will not. Traced invocations of fork will be captured in the IR. + ``*args``, ``**kwargs``: arguments to invoke `func` with. + Returns: + `torch.jit.Future[T]`: a reference to the execution of `func`. The value `T` + can only be accessed by forcing completion of `func` through `torch.jit.wait`. + + Example (fork a free function): + + .. code-block:: python + + import torch + from torch import Tensor + + + def foo(a: Tensor, b: int) -> Tensor: + return a + b + + + def bar(a): + fut: torch.jit.Future[Tensor] = torch.jit.fork(foo, a, b=2) + return torch.jit.wait(fut) + + + script_bar = torch.jit.script(bar) + input = torch.tensor(2) + # only the scripted version executes asynchronously + assert script_bar(input) == bar(input) + # trace is not run asynchronously, but fork is captured in IR + graph = torch.jit.trace(bar, (input,)).graph + assert "fork" in str(graph) + + Example (fork a module method): + + .. code-block:: python + + import torch + from torch import Tensor + + + class AddMod(torch.nn.Module): + def forward(self, a: Tensor, b: int): + return a + b + + + class Mod(torch.nn.Module): + def __init__(self) -> None: + super(self).__init__() + self.mod = AddMod() + + def forward(self, input): + fut = torch.jit.fork(self.mod, a, b=2) + return torch.jit.wait(fut) + + + input = torch.tensor(2) + mod = Mod() + assert mod(input) == torch.jit.script(mod).forward(input) + """ + return torch._C.fork(func, *args, **kwargs) + + +def wait(future): + r""" + Force completion of a `torch.jit.Future[T]` asynchronous task, returning the result of the task. + + See :func:`~fork` for docs and examples. + Args: + future (torch.jit.Future[T]): an asynchronous task reference, created through `torch.jit.fork` + Returns: + `T`: the return value of the completed task + """ + return torch._C.wait(future) + + +_register_builtin(wait, "aten::wait") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_await.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_await.py new file mode 100644 index 0000000000000000000000000000000000000000..b00da7b3384b7a263aeab613b3e613c4bd3c3667 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_await.py @@ -0,0 +1,27 @@ +# mypy: allow-untyped-defs +import torch +from torch._jit_internal import _Await +from torch.jit._builtins import _register_builtin +from torch.utils import set_module + + +set_module(_Await, "torch.jit") + + +def _awaitable(func, *args, **kwargs): + r"""Create Await object that will call specified functioni with specified args, when it is requested for the result.""" + return torch._C._awaitable(func, *args, **kwargs) + + +def _awaitable_wait(aw): + r"""Request await the result of execution, if Await is not completed yet, the func will be called immediately.""" + return torch._C._awaitable_wait(aw) + + +def _awaitable_nowait(o): + r"""Create completed Await with specified result.""" + return torch._C._awaitable_nowait(o) + + +_register_builtin(_awaitable_wait, "prim::awaitable_wait") +_register_builtin(_awaitable_nowait, "prim::awaitable_nowait") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_builtins.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_builtins.py new file mode 100644 index 0000000000000000000000000000000000000000..f61fadf375bf1658dbd8a4b736baa4187d00b7a0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_builtins.py @@ -0,0 +1,204 @@ +# mypy: allow-untyped-defs +import cmath +import math +import warnings +from collections import OrderedDict +from typing import Optional + +import torch +import torch.backends.cudnn as cudnn +from torch.nn.modules.utils import ( + _list_with_default, + _pair, + _quadruple, + _single, + _triple, +) + + +_builtin_table: Optional[dict[int, str]] = None + +_modules_containing_builtins = ( + torch, + torch._C._nn, + torch._C._fft, # type: ignore[attr-defined] + torch._C._linalg, # type: ignore[attr-defined] + torch._C._nested, # type: ignore[attr-defined] + torch._C._sparse, # type: ignore[attr-defined] + torch._C._special, # type: ignore[attr-defined] +) + +_builtin_ops = [ + # Pairs of (function, op_name) + (_pair, "aten::_pair"), + (_quadruple, "aten::_quadruple"), + (_single, "aten::_single"), + (_triple, "aten::_triple"), + (_list_with_default, "aten::list_with_default"), + (OrderedDict, "aten::dict"), + (dict, "aten::dict"), + (cudnn.is_acceptable, "aten::cudnn_is_acceptable"), + (math.ceil, "aten::ceil"), + (math.copysign, "aten::copysign"), + (math.erf, "aten::erf"), + (math.erfc, "aten::erfc"), + (math.exp, "aten::exp"), + (math.expm1, "aten::expm1"), + (math.fabs, "aten::fabs"), + (math.floor, "aten::floor"), + (math.gamma, "aten::gamma"), + (math.lgamma, "aten::lgamma"), + (math.log, "aten::log"), + (math.log10, "aten::log10"), + (math.log1p, "aten::log1p"), + (math.pow, "aten::pow"), + (math.sqrt, "aten::sqrt"), + (math.isnan, "aten::isnan"), + (math.asinh, "aten::asinh"), + (math.atanh, "aten::atanh"), + (math.cosh, "aten::cosh"), + (math.sinh, "aten::sinh"), + (math.tanh, "aten::tanh"), + (math.acos, "aten::acos"), + (math.asin, "aten::asin"), + (math.atan, "aten::atan"), + (math.atan2, "aten::atan2"), + (math.cos, "aten::cos"), + (math.sin, "aten::sin"), + (math.tan, "aten::tan"), + (math.asinh, "aten::asinh"), + (math.atanh, "aten::atanh"), + (math.acosh, "aten::acosh"), + (math.fmod, "aten::fmod"), + (math.modf, "aten::modf"), + (math.factorial, "aten::factorial"), + (math.frexp, "aten::frexp"), + (math.isinf, "aten::isinf"), + (math.degrees, "aten::degrees"), + (math.radians, "aten::radians"), + (cmath.isnan, "aten::isnan"), + (cmath.isfinite, "aten::isfinite"), + (cmath.isinf, "aten::isinf"), + (cmath.phase, "aten::angle"), + (cmath.rect, "aten::polar"), + (cmath.log, "aten::log"), + (cmath.log10, "aten::log10"), + (cmath.sqrt, "aten::sqrt"), + (cmath.exp, "aten::exp"), + (cmath.sin, "aten::sin"), + (cmath.tan, "aten::tan"), + (cmath.cos, "aten::cos"), + (cmath.asin, "aten::asin"), + (cmath.acos, "aten::acos"), + (cmath.atan, "aten::atan"), + (cmath.sinh, "aten::sinh"), + (cmath.cosh, "aten::cosh"), + (cmath.tanh, "aten::tanh"), + (cmath.asinh, "aten::asinh"), + (cmath.acosh, "aten::acosh"), + (cmath.atanh, "aten::atanh"), + (math.ldexp, "aten::ldexp"), + (torch._assert, "aten::_assert"), + (torch.autograd.grad, "aten::grad"), + (torch.autograd.backward, "aten::backward"), + (torch._C._infer_size, "aten::_infer_size"), + ( + torch.nn.functional._no_grad_embedding_renorm_, # type: ignore[attr-defined] + "aten::_no_grad_embedding_renorm_", + ), + (torch.nn.functional.assert_int_or_pair, "aten::_assert_int_or_pair"), + (torch.nn.init._no_grad_fill_, "aten::_no_grad_fill_"), + (torch.nn.init._no_grad_normal_, "aten::_no_grad_normal_"), + (torch.nn.init._no_grad_uniform_, "aten::_no_grad_uniform_"), + (torch.nn.init._no_grad_zero_, "aten::_no_grad_zero_"), + (torch._C._get_tracing_state, "aten::_get_tracing_state"), + (torch._C._get_cpu_capability, "aten::_get_cpu_capability"), + (warnings.warn, "aten::warn"), + (torch._VF.stft, "aten::stft"), # type: ignore[attr-defined] + (torch._VF.istft, "aten::istft"), # type: ignore[attr-defined] + (torch._VF.cdist, "aten::cdist"), # type: ignore[attr-defined] + (torch._VF.norm, "aten::norm"), # type: ignore[attr-defined] + (torch._VF.unique_dim, "aten::unique_dim"), + (torch._VF.unique_consecutive, "aten::unique_consecutive"), # type: ignore[attr-defined] + (torch._VF.nuclear_norm, "aten::nuclear_norm"), + (torch._VF.frobenius_norm, "aten::frobenius_norm"), + (torch._VF.tensordot, "aten::tensordot"), # type: ignore[attr-defined] +] + +# ops in torch.functional are bound to torch +# in these cases, we want to resolve the function to their python implementation +# instead looking up a builtin "aten::" schema + + +def _gen_torch_functional_registered_ops(): + # eventually ops should encompass all of torch/functional.py, (torch.functional.__all__) + # but we are currently only able to compile some of the functions. additionally, + # some functions directly map to their aten:: implementations. + # TODO: add support for more ops + ops = [ + "stft", + "istft", + "lu", + "cdist", + "norm", + "unique", + "unique_consecutive", + "tensordot", + ] + return {getattr(torch.functional, name) for name in ops} + + +_functional_registered_ops = _gen_torch_functional_registered_ops() + + +def _is_special_functional_bound_op(fn): + return fn in _functional_registered_ops + + +# lazily built to ensure the correct initialization order +def _get_builtin_table(): + global _builtin_table + if _builtin_table is not None: + return _builtin_table + _builtin_table = {} + + def register_all(mod) -> None: + for name in dir(mod): + v = getattr(mod, name) + if ( + callable(v) + and not _is_special_functional_bound_op(v) + and v is not torch.no_grad + and v is not torch.autocast + ): + # Fixup inconsistency in segment_reduce + if name == "_segment_reduce": + name = name[1:] + _builtin_ops.append((v, "aten::" + name)) + + for mod in _modules_containing_builtins: + register_all(mod) + + _builtin_ops.append((math.gcd, "aten::gcd")) + _builtin_ops.append((math.isfinite, "aten::isfinite")) + _builtin_ops.append((math.remainder, "aten::mathremainder")) # type: ignore[attr-defined] + + import torch.distributed.autograd as dist_autograd + + if dist_autograd.is_available(): + _builtin_ops.append((dist_autograd.get_gradients, "aten::get_gradients")) + _builtin_ops.append((dist_autograd.backward, "aten::dist_backward")) + + # populate the _builtin_table from _builtin_ops + for builtin, aten_op in _builtin_ops: + _builtin_table[id(builtin)] = aten_op + + return _builtin_table + + +def _register_builtin(fn, op) -> None: + _get_builtin_table()[id(fn)] = op + + +def _find_builtin(fn): + return _get_builtin_table().get(id(fn)) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_check.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_check.py new file mode 100644 index 0000000000000000000000000000000000000000..36440769f063ffe391cfe745d2d625276f4eac81 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_check.py @@ -0,0 +1,261 @@ +# mypy: allow-untyped-defs +import ast +import inspect +import sys +import textwrap +import warnings + +import torch + + +class AttributeTypeIsSupportedChecker(ast.NodeVisitor): + """Check the ``__init__`` method of a given ``nn.Module``. + + It ensures that all instance-level attributes can be properly initialized. + + Specifically, we do type inference based on attribute values...even + if the attribute in question has already been typed using + Python3-style annotations or ``torch.jit.annotate``. This means that + setting an instance-level attribute to ``[]`` (for ``List``), + ``{}`` for ``Dict``), or ``None`` (for ``Optional``) isn't enough + information for us to properly initialize that attribute. + + An object of this class can walk a given ``nn.Module``'s AST and + determine if it meets our requirements or not. + + Known limitations + 1. We can only check the AST nodes for certain constructs; we can't + ``eval`` arbitrary expressions. This means that function calls, + class instantiations, and complex expressions that resolve to one of + the "empty" values specified above will NOT be flagged as + problematic. + 2. We match on string literals, so if the user decides to use a + non-standard import (e.g. `from typing import List as foo`), we + won't catch it. + + Example: + .. code-block:: python + + class M(torch.nn.Module): + def fn(self): + return [] + + def __init__(self) -> None: + super().__init__() + self.x: List[int] = [] + + def forward(self, x: List[int]): + self.x = x + return 1 + + The above code will pass the ``AttributeTypeIsSupportedChecker`` + check since we have a function call in ``__init__``. However, + it will still fail later with the ``RuntimeError`` "Tried to set + nonexistent attribute: x. Did you forget to initialize it in + __init__()?". + + Args: + nn_module - The instance of ``torch.nn.Module`` whose + ``__init__`` method we wish to check + """ + + def check(self, nn_module: torch.nn.Module) -> None: + source_lines = inspect.getsource(nn_module.__class__.__init__) + + # Ignore comments no matter the indentation + def is_useless_comment(line): + line = line.strip() + return line.startswith("#") and not line.startswith("# type:") + + source_lines = "\n".join( + [l for l in source_lines.split("\n") if not is_useless_comment(l)] + ) + + # This AST only contains the `__init__` method of the nn.Module + init_ast = ast.parse(textwrap.dedent(source_lines)) + + # Get items annotated in the class body + if sys.version_info >= (3, 14): + import annotationlib + + self.class_level_annotations = list( + annotationlib.get_annotations( + nn_module, format=annotationlib.Format.FORWARDREF + ).keys() + ) + else: + self.class_level_annotations = list(nn_module.__annotations__.keys()) + + # Flag for later + self.visiting_class_level_ann = False + + self.visit(init_ast) + + def _is_empty_container(self, node: ast.AST, ann_type: str) -> bool: + if ann_type == "List": + # Assigning `[]` to a `List` type gives you a Node where + # value=List(elts=[], ctx=Load()) + if not isinstance(node, ast.List): + return False + if node.elts: + return False + elif ann_type == "Dict": + # Assigning `{}` to a `Dict` type gives you a Node where + # value=Dict(keys=[], values=[]) + if not isinstance(node, ast.Dict): + return False + if node.keys: + return False + elif ann_type == "Optional": + # Assigning `None` to an `Optional` type gives you a + # Node where value=Constant(value=None, kind=None) + if not isinstance(node, ast.Constant): + return False + if node.value: # type: ignore[attr-defined] + return False + + return True + + def visit_Assign(self, node) -> None: + """Store assignment state when assigning to a Call Node. + + If we're visiting a Call Node (the right-hand side of an + assignment statement), we won't be able to check the variable + that we're assigning to (the left-hand side of an assignment). + Because of this, we need to store this state in visitAssign. + (Luckily, we only have to do this if we're assigning to a Call + Node, i.e. ``torch.jit.annotate``. If we're using normal Python + annotations, we'll be visiting an AnnAssign Node, which has its + target built in.) + """ + try: + if ( + isinstance(node.value, ast.Call) + and node.targets[0].attr in self.class_level_annotations + ): + self.visiting_class_level_ann = True + except AttributeError: + return + self.generic_visit(node) + self.visiting_class_level_ann = False + + def visit_AnnAssign(self, node) -> None: + """Visit an AnnAssign node in an ``nn.Module``'s ``__init__`` method. + + It checks if it conforms to our attribute annotation rules.""" + # If we have a local variable + try: + if node.target.value.id != "self": + return + except AttributeError: + return + + # If we have an attribute that's already been annotated at the + # class level + if node.target.attr in self.class_level_annotations: + return + + # TODO @ansley: add `Union` once landed + + # NB: Even though `Tuple` is a "container", we don't want to + # check for it here. `Tuple` functions as an type with an + # "infinite" number of subtypes, in the sense that you can have + # `Tuple[())]`, `Tuple[T1]`, `Tuple[T2]`, `Tuple[T1, T2]`, + # `Tuple[T2, T1]` and so on, and none of these subtypes can be + # used in place of the other. Therefore, assigning an empty + # tuple in `__init__` CORRECTLY means that that variable + # cannot be reassigned later to a non-empty tuple. Same + # deal with `NamedTuple` + + containers = {"List", "list", "Dict", "dict", "Optional"} + + # If we're not evaluating one of the specified problem types + try: + if node.annotation.value.id not in containers: + return + except AttributeError: + # To evaluate a base type (`str`, `int`, etc.), we would + # have needed to get the name through `node.annotation.id` + # instead of `node.annotation.value.id`. Seems that we're + # not evaluating one of our "containers" + return + + # Check if the assigned variable is empty + ann_type = node.annotation.value.id + if not self._is_empty_container(node.value, ann_type): + return + + warnings.warn( + "The TorchScript type system doesn't support " + "instance-level annotations on empty non-base " + "types in `__init__`. Instead, either 1) use a " + "type annotation in the class body, or 2) wrap " + "the type in `torch.jit.Attribute`.", + stacklevel=2, + ) + + def visit_Call(self, node) -> None: + """Determine if a Call node is 'torch.jit.annotate' in __init__. + + Visit a Call node in an ``nn.Module``'s ``__init__`` + method and determine if it's ``torch.jit.annotate``. If so, + see if it conforms to our attribute annotation rules. + """ + # If we have an attribute that's already been annotated at the + # class level + if self.visiting_class_level_ann: + return + + # If this isn't a call to `torch.jit.annotate` + try: + if ( + node.func.value.value.id != "torch" + or node.func.value.attr != "jit" + or node.func.attr != "annotate" + ): + self.generic_visit(node) + elif ( + node.func.value.value.id != "jit" or node.func.value.attr != "annotate" + ): + self.generic_visit(node) + except AttributeError: + # Looks like we didn't even have the right node structure + # to check for `torch.jit.annotate` in the first place + self.generic_visit(node) + + # Invariant: we have a `torch.jit.annotate` or a + # `torch.annotate` call + + # A Call Node for `torch.jit.annotate` should have an `args` + # list of length 2 where args[0] represents the annotation and + # args[1] represents the actual value + if len(node.args) != 2: + return + + if not isinstance(node.args[0], ast.Subscript): + return + + # See notes in `visit_AnnAssign` r.e. containers + + containers = {"List", "Dict", "Optional"} + + try: + ann_type = node.args[0].value.id # type: ignore[attr-defined] + except AttributeError: + return + + if ann_type not in containers: + return + + # Check if the assigned variable is empty + if not self._is_empty_container(node.args[1], ann_type): + return + + warnings.warn( + "The TorchScript type system doesn't support " + "instance-level annotations on empty non-base " + "types in `__init__`. Instead, either 1) use a " + "type annotation in the class body, or 2) wrap " + "the type in `torch.jit.Attribute`.", + stacklevel=2, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_dataclass_impls.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_dataclass_impls.py new file mode 100644 index 0000000000000000000000000000000000000000..67da5e20206327f23ea3c92ae48a0452c344fead --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_dataclass_impls.py @@ -0,0 +1,190 @@ +# mypy: allow-untyped-defs +# Functions for synthesizing magic methods for JIT-compiled dataclasses +import ast +import dataclasses +import inspect +import os +from collections.abc import Callable +from functools import partial + +from torch._jit_internal import FAKE_FILENAME_PREFIX, is_optional +from torch._sources import ParsedDef, SourceContext + + +def _get_fake_filename(cls, method_name): + return os.path.join(FAKE_FILENAME_PREFIX, cls.__name__, method_name) + + +def compose_fn(cls, name: str, body_lines: list[str], signature: str) -> ParsedDef: + body = "\n".join(f" {b}" for b in body_lines) + decl = f"def {name}{signature}:\n{body}" + + # Parse the function declaration + try: + py_ast = ast.parse(decl) + except SyntaxError as e: + # This should only happen if there's some unforeseeable change + # in the dataclasses module that makes our synthesized code fail + raise RuntimeError( + f"TorchScript failed to synthesize dataclass method '{name}' for class '{cls.__name__}'. " + "Please file a bug report at " + ) from e + fake_filename = _get_fake_filename(cls, name) + # Parse the function + return ParsedDef( + py_ast, + ctx=SourceContext( + source=decl, filename=fake_filename, file_lineno=0, leading_whitespace_len=0 + ), + source=decl, + filename=fake_filename, + file_lineno=0, + ) + + +def synthesize__init__(cls) -> ParsedDef: + # Supporting default factories in the way that people expect would sort of require us to + # allow compiling lambda functions, which is not currently supported. + if any( + field.default_factory is not dataclasses.MISSING + for field in dataclasses.fields(cls) + ): + raise NotImplementedError( + "Default factory initializers are not supported in TorchScript dataclasses" + ) + + # Simply read off the generated __init__ signature from CPython's implementation. It'll be + # almost correct except for InitVar annotations, which we need to handle specially. + signature = inspect.signature(cls.__init__) + + # Handle InitVars if needed (only works on Python 3.8+, when a `type` attribute was added to InitVar); + # see CPython commit here https://github.com/python/cpython/commit/01ee12ba35a333e8a6a25c4153c4a21838e9585c + init_vars: list[str] = [] + params = [] + for name, param in signature.parameters.items(): + ann = param.annotation + + if isinstance(ann, dataclasses.InitVar): + # The TorchScript interpreter can't handle InitVar annotations, so we unwrap the underlying type here + init_vars.append(name) + params.append(param.replace(annotation=ann.type)) # type: ignore[attr-defined] + else: + params.append(param) + + signature = signature.replace(parameters=params) + + body = [ + # Assign all attributes to self + f"self.{field.name} = {field.name}" + for field in dataclasses.fields(cls) + if field.init and field.name not in init_vars + ] + # Call user's impl of __post_init__ if it exists + if hasattr(cls, "__post_init__"): + body.append("self.__post_init__(" + ", ".join(init_vars) + ")") + + return compose_fn(cls, "__init__", body or ["pass"], signature=str(signature)) + + +# This is a placeholder at the moment since the TorchScript interpreter doesn't call __repr__ +def synthesize__repr__(cls) -> ParsedDef: + return compose_fn( + cls, + "__repr__", + [ + f"return '{cls.__name__}(" + + ", ".join( + [ + f"{field.name}=self.{field.name}" + for field in dataclasses.fields(cls) + if field.repr + ] + ) + + ")'" + ], + signature="(self) -> str", + ) + + +def synthesize__hash__(cls) -> ParsedDef: + return compose_fn( + cls, + "__hash__", + [ + # This is just a placeholder to prevent compilation from failing; this won't even get called at + # all right now because the TorchScript interpreter doesn't call custom __hash__ implementations + "raise NotImplementedError('__hash__ is not supported for dataclasses in TorchScript')" + ], + signature="(self) -> int", + ) + + +# Implementation for __eq__ and __ne__ +def synthesize_equality(cls, name: str, converse: str) -> ParsedDef: + return synthesize_comparison( + cls, + name, + allow_eq=True, + raise_on_none=False, + inner=[f"if val1 {converse} val2: return False"], + ) + + +def synthesize_inequality(cls, name: str, op: str, allow_eq: bool) -> ParsedDef: + return synthesize_comparison( + cls, + name, + allow_eq, + raise_on_none=True, + inner=[ + f"if val1 {op} val2: return True", + f"elif val2 {op} val1: return False", + ], + ) + + +def synthesize_comparison( + cls, name: str, allow_eq: bool, raise_on_none: bool, inner: list[str] +) -> ParsedDef: + body = [] + for field in dataclasses.fields(cls): + if not field.compare: + continue + + body.extend( + [ + f"val1 = self.{field.name}", + f"val2 = other.{field.name}", + ] + ) + body.extend( + inner + if not is_optional(field.type) + else [ + # Type refinement for optional fields; we need this to avoid type errors from the interpreter + "if val1 is not None and val2 is not None:", + *[" " + line for line in inner], + "elif (val1 is None) != (val2 is None):", + f" raise TypeError('Cannot compare {cls.__name__} with None')" + if raise_on_none + else " return False", + ] + ) + + body.append(f"return {allow_eq}") + return compose_fn( + cls, name, body, signature=f"(self, other: {cls.__name__}) -> bool" + ) + + +DATACLASS_MAGIC_METHODS: dict[str, Callable] = { + "__init__": synthesize__init__, + "__repr__": synthesize__repr__, + "__hash__": synthesize__hash__, + "__eq__": partial(synthesize_equality, name="__eq__", converse="!="), + "__ne__": partial(synthesize_equality, name="__ne__", converse="=="), + "__lt__": partial(synthesize_inequality, name="__lt__", op="<", allow_eq=False), + "__le__": partial(synthesize_inequality, name="__le__", op="<", allow_eq=True), + "__gt__": partial(synthesize_inequality, name="__gt__", op=">", allow_eq=False), + "__ge__": partial(synthesize_inequality, name="__ge__", op=">", allow_eq=True), +} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_decomposition_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_decomposition_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..48f24f0d85d6cce8c19684c3b53951fc891b41f0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_decomposition_utils.py @@ -0,0 +1,12 @@ +# mypy: allow-untyped-defs +import torch +from torch._ops import OpOverload, OpOverloadPacket + + +def _register_decomposition(op: OpOverload, graph: torch._C.Graph) -> None: + assert not isinstance(op, OpOverloadPacket), ( + f"Must pass specific op overload, not overload packet, found {op}" + ) + assert isinstance(op, OpOverload) + + torch._C._jit_register_decomposition_for_schema(op._schema, graph) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_decompositions.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_decompositions.py new file mode 100644 index 0000000000000000000000000000000000000000..bb628f82a8ef02eddf8f6e187eced58d9353e2b1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_decompositions.py @@ -0,0 +1,141 @@ +# mypy: allow-untyped-defs +import torch +from torch import Tensor + + +aten = torch.ops.aten +import inspect +import warnings +from collections.abc import Callable +from typing import Optional, TypeVar +from typing_extensions import ParamSpec + +from torch.types import Number + + +decomposition_table: dict[str, torch.jit.ScriptFunction] = {} +function_name_set: set[str] = set() + +_T = TypeVar("_T") +_P = ParamSpec("_P") + + +def check_decomposition_has_type_annotations(f) -> None: + inspect_empty = inspect._empty # type: ignore[attr-defined] + sig = inspect.signature(f) + for param in sig.parameters.values(): + assert param.annotation != inspect_empty, ( + f"No signature on param {param.name} for function {f.name}" + ) + + assert sig.return_annotation != inspect_empty, ( + f"No return annotation for function {f.name}" + ) + + +def signatures_match(decomposition_sig, torch_op_sig): + decomp_params = decomposition_sig.parameters + op_params = torch_op_sig.parameters + + if len(decomp_params) != len(op_params): + return False + + for decomp_param, op_param in zip(decomp_params.values(), op_params.values()): + # can't check full equality yet because not all fields are correctly deduced + # in the torch_op_sig - like default value + # can't check 'kind' bc + # kwarg-only values with defaults not yet supported in TS + inspect_empty = inspect._empty # type: ignore[attr-defined] + for field in ["name", "annotation"]: + if field == "name" and decomp_param.name == "self": + warnings.warn( + "PyTorch uses 'input' instead of 'self' on public api", stacklevel=2 + ) + + if getattr(decomp_param, field) != getattr(op_param, field): + return False + + decomp_default = decomp_param.default + op_default = op_param.default + # default value not always correctly inferred as being present on torch schema, + # but if specified on both they should be equal + if decomp_default != inspect_empty and op_default != inspect_empty: + if decomp_default != op_default: + return False + + return decomposition_sig.return_annotation == torch_op_sig.return_annotation + + +def register_decomposition( + aten_op: torch._ops.OpOverload, + registry: Optional[dict[str, torch.jit.ScriptFunction]] = None, +) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: + def decomposition_decorator(f: Callable[_P, _T]) -> Callable[_P, _T]: + nonlocal registry + if registry is None: + registry = decomposition_table + + assert isinstance(aten_op, torch._ops.OpOverload) + + # Need unique name for jit function serialization + assert f.__name__ not in function_name_set, ( + f"Duplicated function name {f.__name__}" + ) + function_name_set.add(f.__name__) + + scripted_func = torch.jit.script(f) + torch._C._jit_pass_inline(scripted_func.graph) + + for _ in range(2): + torch._C._jit_pass_peephole(scripted_func.graph) + torch._C._jit_pass_constant_propagation(scripted_func.graph) + + registry[str(aten_op._schema)] = scripted_func + return f + + return decomposition_decorator + + +# TODO: replace torch.sigmoid -> aten.sigmoid + + +@register_decomposition(aten.var.correction) +def var_decomposition( + input: Tensor, + dim: Optional[list[int]] = None, + correction: Optional[Number] = None, + keepdim: bool = False, +) -> Tensor: + if dim is None: + dim_i: list[int] = [] + dim = dim_i + + if isinstance(dim, (tuple, list)) and len(dim) == 0: + n = input.numel() + else: + n = 1 + for dim_i in dim: # type: ignore[assignment] + n *= input.shape[dim_i] # type: ignore[call-overload] + + mean = aten.mean(input, dim, True) + sub = input - mean + sq = sub * sub + sum = aten.sum(sq, dim, keepdim) + + if correction is None: + denom = float(n - 1) + else: + if isinstance(correction, int): + denom = float(n - correction) + elif isinstance(correction, float): + denom = float(n) - correction + else: + raise RuntimeError("correction must be int or float") + + # pyrefly: ignore [no-matching-overload] + return sum / max(0, denom) + + +@register_decomposition(aten.var.default) +def var(input: Tensor, unbiased: bool = True) -> Tensor: + return var_decomposition(input, correction=(1 if unbiased else 0)) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_freeze.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_freeze.py new file mode 100644 index 0000000000000000000000000000000000000000..7fd8ac1a772bb9a705115ec51a058a2f0b57672d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_freeze.py @@ -0,0 +1,234 @@ +# mypy: allow-untyped-defs +"""Freezing. + +This is not intended to be imported directly; please use the exposed +functionalities in `torch.jit`. +""" + +from typing import Optional + +import torch +from torch.jit._script import RecursiveScriptModule, ScriptModule + + +def freeze( + mod, preserved_attrs: Optional[list[str]] = None, optimize_numerics: bool = True +): + r"""Freeze ScriptModule, inline submodules, and attributes as constants. + + Freezing a :class:`ScriptModule` will clone it and attempt to inline the cloned + module's submodules, parameters, and attributes as constants in the TorchScript IR Graph. + By default, `forward` will be preserved, as well as attributes & methods specified in + `preserved_attrs`. Additionally, any attribute that is modified within a preserved + method will be preserved. + + Freezing currently only accepts ScriptModules that are in eval mode. + + Freezing applies generic optimization that will speed up your model regardless of machine. + To further optimize using server-specific settings, run `optimize_for_inference` after + freezing. + + Args: + mod (:class:`ScriptModule`): a module to be frozen + preserved_attrs (Optional[List[str]]): a list of attributes to preserve in addition to the forward method. + Attributes modified in preserved methods will also be preserved. + optimize_numerics (bool): If ``True``, a set of optimization passes will be run that does not strictly + preserve numerics. Full details of optimization can be found at `torch.jit.run_frozen_optimizations`. + + Returns: + Frozen :class:`ScriptModule`. + + Example (Freezing a simple module with a Parameter): + + .. testcode:: + import torch + class MyModule(torch.nn.Module): + def __init__(self, N, M): + super().__init__() + self.weight = torch.nn.Parameter(torch.rand(N, M)) + self.linear = torch.nn.Linear(N, M) + + def forward(self, input): + output = self.weight.mm(input) + output = self.linear(output) + return output + + scripted_module = torch.jit.script(MyModule(2, 3).eval()) + frozen_module = torch.jit.freeze(scripted_module) + # parameters have been removed and inlined into the Graph as constants + assert len(list(frozen_module.named_parameters())) == 0 + # See the compiled graph as Python code + print(frozen_module.code) + + Example (Freezing a module with preserved attributes) + + .. testcode:: + import torch + class MyModule2(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.modified_tensor = torch.tensor(10.) + self.version = 1 + + def forward(self, input): + self.modified_tensor += 1 + return input + self.modified_tensor + + scripted_module = torch.jit.script(MyModule2().eval()) + frozen_module = torch.jit.freeze(scripted_module, preserved_attrs=["version"]) + # we've manually preserved `version`, so it still exists on the frozen module and can be modified + assert frozen_module.version == 1 + frozen_module.version = 2 + # `modified_tensor` is detected as being mutated in the forward, so freezing preserves + # it to retain model semantics + assert frozen_module(torch.tensor(1)) == torch.tensor(12) + # now that we've run it once, the next result will be incremented by one + assert frozen_module(torch.tensor(1)) == torch.tensor(13) + + Note: + Freezing submodule attributes is also supported: + frozen_module = torch.jit.freeze(scripted_module, preserved_attrs=["submodule.version"]) + + Note: + If you're not sure why an attribute is not being inlined as a constant, you can run + `dump_alias_db` on frozen_module.forward.graph to see if freezing has detected the + attribute is being modified. + + Note: + Because freezing makes weights constants and removes module hierarchy, `to` and other + nn.Module methods to manipulate device or dtype no longer work. As a workaround, + You can remap devices by specifying `map_location` in `torch.jit.load`, however + device-specific logic may have been baked into the model. + """ + if not isinstance(mod, ScriptModule): + raise RuntimeError( + "Freezing expects a ScriptModule as input. " + "Please use torch.jit.script or torch.jit.trace to script your 'nn.Module'." + ) + + if mod.training: + raise RuntimeError( + "Freezing is currently only implemented for modules in eval mode. " + "Please call .eval() on your module before freezing." + ) + + preserved_attrs = preserved_attrs if preserved_attrs is not None else [] + + out = RecursiveScriptModule(torch._C._freeze_module(mod._c, preserved_attrs)) + RecursiveScriptModule._finalize_scriptmodule(out) + + preserved_methods = [x for x in preserved_attrs if mod._c._has_method(x)] + run_frozen_optimizations(out, optimize_numerics, preserved_methods) + + return out + + +def run_frozen_optimizations( + mod, optimize_numerics: bool = True, preserved_methods: Optional[list[str]] = None +) -> None: + r""" + Run a series of optimizations looking for patterns that occur in frozen graphs. + + The current set of optimizations includes: + - Dropout Removal + - Pretranspose Linear Layers + - Concat Linear Layers with same input Tensor + - Conv -> Batchnorm folding + - Conv -> Add/Sub folding + - Conv -> Mul/Div folding + + Args: + mod (:class:`ScriptModule`): a frozen module to be optimized + + optimize_numerics (bool): If ``True``, a set of optimization passes will be run that does not strictly + preserve numerics. These optimizations preserve default rtol and atol of `torch.testing.assert_close` + when applied on a single transformation, however in a module where many transformations are applied + the rtol or atol may no longer fall within the default `assert_close` tolerance. Conv -> Batchnorm folding, + Conv-Add/Sub, and Conv -> Mul/Div folding all may alter numerics. + + Returns: + None + + Note: + In rare occasions, this can result in slower execution. + + Example (Freezing a module with Conv->Batchnorm) + .. code-block:: python + import torch + + in_channels, out_channels = 3, 32 + conv = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=2, bias=True + ) + bn = torch.nn.BatchNorm2d(out_channels, eps=0.001) + mod = torch.nn.Sequential(conv, bn) + # set optimize to False here, by default freezing runs run_frozen_optimizations + frozen_mod = torch.jit.freeze(torch.jit.script(mod.eval()), optimize=False) + # inspect frozen mod + assert "batch_norm" in str(frozen_mod.graph) + torch.jit.run_frozen_optimizations(frozen_mod) + assert "batch_norm" not in str(frozen_mod.graph) + + """ + if mod._c._has_method("forward"): + torch._C._jit_pass_optimize_frozen_graph(mod.graph, optimize_numerics) + + if preserved_methods is None: + preserved_methods = [] + + for method in preserved_methods: + torch._C._jit_pass_optimize_frozen_graph( + mod.__getattr__(method).graph, optimize_numerics + ) + + +def optimize_for_inference( + mod: ScriptModule, other_methods: Optional[list[str]] = None +) -> ScriptModule: + """ + Perform a set of optimization passes to optimize a model for the purposes of inference. + + If the model is not already frozen, optimize_for_inference + will invoke `torch.jit.freeze` automatically. + + In addition to generic optimizations that should speed up your model regardless + of environment, prepare for inference will also bake in build specific settings + such as the presence of CUDNN or MKLDNN, and may in the future make transformations + which speed things up on one machine but slow things down on another. Accordingly, + serialization is not implemented following invoking `optimize_for_inference` and + is not guaranteed. + + This is still in prototype, and may have the potential to slow down your model. + Primary use cases that have been targeted so far have been vision models on cpu + and gpu to a lesser extent. + + Example (optimizing a module with Conv->Batchnorm):: + + import torch + + in_channels, out_channels = 3, 32 + conv = torch.nn.Conv2d( + in_channels, out_channels, kernel_size=3, stride=2, bias=True + ) + bn = torch.nn.BatchNorm2d(out_channels, eps=0.001) + mod = torch.nn.Sequential(conv, bn) + frozen_mod = torch.jit.optimize_for_inference(torch.jit.script(mod.eval())) + assert "batch_norm" not in str(frozen_mod.graph) + # if built with MKLDNN, convolution will be run with MKLDNN weights + assert "MKLDNN" in frozen_mod.graph + """ + if not isinstance(mod, ScriptModule): + raise RuntimeError( + "optimize_for_inference expects a ScriptModule as input. " + "Please use torch.jit.script or torch.jit.trace to script your 'nn.Module'." + ) + + if other_methods is None: + other_methods = [] + + if hasattr(mod, "training"): + mod = freeze(mod.eval(), preserved_attrs=other_methods) + + torch._C._jit_pass_optimize_for_inference(mod._c, other_methods) + + return mod diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_fuser.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_fuser.py new file mode 100644 index 0000000000000000000000000000000000000000..a6a2f1cce67e8a7af0f52e39b3a6fd3433b0f6be --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_fuser.py @@ -0,0 +1,160 @@ +# mypy: allow-untyped-defs +import contextlib + +import torch + + +@contextlib.contextmanager +def optimized_execution(should_optimize): + """Context manager that controls whether the JIT's executor will run optimizations before executing a function.""" + stored_flag = torch._C._get_graph_executor_optimize() + torch._C._set_graph_executor_optimize(should_optimize) + try: + yield + finally: + torch._C._set_graph_executor_optimize(stored_flag) + + +@contextlib.contextmanager +def fuser(name): + """Context manager that facilitates switching between backend fusers. + + Valid names: + * ``fuser0`` - enables only legacy fuser + * ``fuser1`` - enables only NNC + * ``fuser2`` - enables only nvFuser + * ``fuser3`` - enables oneDNN Graph + """ + old_cpu_fuse = torch._C._jit_can_fuse_on_cpu() + old_gpu_fuse = torch._C._jit_can_fuse_on_gpu() + old_texpr_fuser_state = torch._C._jit_texpr_fuser_enabled() + old_nvfuser_state = torch._C._jit_nvfuser_enabled() + old_llga_state = torch._C._jit_llga_enabled() + if name == "fuser0": # legacy fuser + torch._C._jit_override_can_fuse_on_cpu(True) + torch._C._jit_override_can_fuse_on_gpu(True) + torch._C._jit_set_texpr_fuser_enabled(False) + torch._C._jit_set_nvfuser_enabled(False) + torch._C._jit_set_llga_enabled(False) + elif name == "fuser1": # NNC + old_profiling_executor = torch._C._jit_set_profiling_executor(True) + old_profiling_mode = torch._C._get_graph_executor_optimize(True) + torch._C._jit_override_can_fuse_on_cpu(True) + torch._C._jit_override_can_fuse_on_gpu(True) + torch._C._jit_set_texpr_fuser_enabled(True) + torch._C._jit_set_nvfuser_enabled(False) + torch._C._jit_set_llga_enabled(False) + elif name == "fuser2": # nvFuser + torch._C._jit_override_can_fuse_on_cpu(False) + torch._C._jit_override_can_fuse_on_gpu(False) + torch._C._jit_set_texpr_fuser_enabled(False) + torch._C._jit_set_nvfuser_enabled(True) + torch._C._jit_set_llga_enabled(False) + elif name == "fuser3": # oneDNN Graph + old_profiling_executor = torch._C._jit_set_profiling_executor(True) + old_profiling_mode = torch._C._get_graph_executor_optimize(True) + torch._C._jit_override_can_fuse_on_cpu(True) + torch._C._jit_override_can_fuse_on_gpu(False) + torch._C._jit_set_texpr_fuser_enabled(True) + torch._C._jit_set_nvfuser_enabled(False) + torch._C._jit_set_llga_enabled(True) + elif name == "none": # Turn Pytorch fuser off + torch._C._jit_override_can_fuse_on_cpu(False) + torch._C._jit_override_can_fuse_on_gpu(False) + torch._C._jit_set_texpr_fuser_enabled(False) + torch._C._jit_set_nvfuser_enabled(False) + torch._C._jit_set_llga_enabled(False) + else: + raise Exception(f"unrecognized fuser option (name: {name})") # noqa: TRY002 + try: + yield + finally: + if name in ["fuser1", "fuser3"]: # NNC or oneDNN Graph + torch._C._jit_set_profiling_executor(old_profiling_executor) # type: ignore[possibly-undefined] + torch._C._get_graph_executor_optimize(old_profiling_mode) # type: ignore[possibly-undefined] + # recover the previous values + torch._C._jit_override_can_fuse_on_cpu(old_cpu_fuse) + torch._C._jit_override_can_fuse_on_gpu(old_gpu_fuse) + torch._C._jit_set_texpr_fuser_enabled(old_texpr_fuser_state) + torch._C._jit_set_nvfuser_enabled(old_nvfuser_state) + torch._C._jit_set_llga_enabled(old_llga_state) + + +last_executed_optimized_graph = torch._C._last_executed_optimized_graph + + +def _get_differentiable_graph_node(node, diff_node) -> None: + if node.kind() == "prim::DifferentiableGraph": + diff_node.append(node) + else: + for block in node.blocks(): + for n in block.nodes(): + _get_differentiable_graph_node(n, diff_node) + + +def _graph_for(self, *args, **kwargs): + return _script_method_graph_for(self, self, *args, **kwargs) + + +def _script_method_graph_for(self, parent, *args, **kwargs): + try: + dbs = parent.get_debug_state() + eps = list(dbs.execution_plans.values()) + assert len(eps) == 1 + graph = eps[0].graph.copy() + + # graph_executor_states for differentiable node + fw_states = eps[0].code.differentiable_op_executor_states() + diff_nodes: list[torch._C.Node] = [] + for n in graph.nodes(): + _get_differentiable_graph_node(n, diff_nodes) + + assert len(fw_states) == len(diff_nodes) + # swap each differentiable graph with optimized graph in their execution plan + for n, state in zip(diff_nodes, fw_states): + fw_execution_plans = list(state.execution_plans.values()) + # we can only update the subgraph when there's a unique execution + # plan. Avoid assert here so we would skip the ones that can't be + # updated while try the best effort to update other nodes. + if len(fw_execution_plans) == 1: + n.g_("Subgraph", fw_execution_plans[0].graph) + + return graph + except Exception: + # fallback approach, we just ran the graph and return the recorded optimized + # graph + self(*args, **kwargs) + return last_executed_optimized_graph() + + +def set_fusion_strategy(strategy: list[tuple[str, int]]): + """Set the type and number of specializations that can occur during fusion. + + Usage: provide a list of pairs (type, depth) where type is one of "STATIC" or "DYNAMIC" + and depth is an integer. + + Behavior - static vs dynamic: + In STATIC fusion, fused ops are compiled to have fixed input shapes. The shape is determined + based on some initial profiling runs. + In DYNAMIC fusion, fused ops are compiled to have variable input shapes, so that multiple + shapes are possible. + + In both cases, we also recompile on new striding behavior, device, or dtype. + + Behavior - fallback functions & depth: + When an input doesn't match the format required by the specialized compiled op, it will run + a fallback function. Fallback functions are recursively be compiled and specialized based + on the observed tensor shapes. Since compilation can be slow, the "depth" parameter is provided to + limit the number of specializations that can be compiled, before giving up on recompiling and + falling back to a completely un-fused, un-specialized implementation. + + The list of (type, depth) pairs controls the type of specializations and the number of + specializations. For example: [("STATIC", 2), ("DYNAMIC", 2)] indicates that the first + two specializations will use static fusions, the following two specializations will use + dynamic fusion, and any inputs that satisfy none of the 4 options will run an + unfused implementation. + + NB: in the future, if more as more fusion backends are added there may be more granular + apis for specific fusers. + """ + return torch._C._jit_set_fusion_strategy(strategy) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_ir_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_ir_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7a775717de07f65693df1dc4e4c33327e6f9cb3b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_ir_utils.py @@ -0,0 +1,33 @@ +from types import TracebackType +from typing import Optional, Union + +import torch + + +class _InsertPoint: + def __init__( + self, + insert_point_graph: torch._C.Graph, + insert_point: Union[torch._C.Node, torch._C.Block], + ) -> None: + self.insert_point = insert_point + self.g = insert_point_graph + self.guard = None + + def __enter__(self) -> None: + self.prev_insert_point = self.g.insertPoint() + self.g.setInsertPoint(self.insert_point) + + def __exit__( + self, + exc_type: Optional[type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + self.g.setInsertPoint(self.prev_insert_point) + + +def insert_point_guard( + self: torch._C.Graph, insert_point: Union[torch._C.Node, torch._C.Block] +) -> _InsertPoint: + return _InsertPoint(self, insert_point) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_logging.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_logging.py new file mode 100644 index 0000000000000000000000000000000000000000..d0687023386ee7745af4e2cb0731f76528125477 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_logging.py @@ -0,0 +1,11 @@ +import torch + + +add_stat_value = torch.ops.prim.AddStatValue + +set_logger = torch._C._logging_set_logger +LockingLogger = torch._C.LockingLogger +AggregationType = torch._C.AggregationType +NoopLogger = torch._C.NoopLogger + +time_point = torch.ops.prim.TimePoint diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_monkeytype_config.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_monkeytype_config.py new file mode 100644 index 0000000000000000000000000000000000000000..e5ddc1e443a29a8087fcaa4089fe289142bcb095 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_monkeytype_config.py @@ -0,0 +1,196 @@ +# mypy: allow-untyped-defs +import inspect +import sys +import typing +from collections import defaultdict +from collections.abc import Iterable +from pathlib import Path +from types import CodeType +from typing import Optional + +import torch + + +_IS_MONKEYTYPE_INSTALLED = True +try: + import monkeytype # type: ignore[import] + + # pyrefly: ignore [import-error] + from monkeytype import trace as monkeytype_trace + from monkeytype.config import _startswith, LIB_PATHS # type: ignore[import] + from monkeytype.db.base import ( # type: ignore[import] + CallTraceStore, + CallTraceStoreLogger, + CallTraceThunk, + ) + from monkeytype.tracing import CallTrace, CodeFilter # type: ignore[import] +except ImportError: + _IS_MONKEYTYPE_INSTALLED = False + + +# Checks whether a class is defined in `torch.*` modules +def is_torch_native_class(cls): + if not hasattr(cls, "__module__"): + return False + + parent_modules = cls.__module__.split(".") + if not parent_modules: + return False + + root_module = sys.modules.get(parent_modules[0]) + return root_module is torch + + +def get_type(type): + """Convert the given type to a torchScript acceptable format.""" + if isinstance(type, str): + return type + elif inspect.getmodule(type) == typing: + # If the type is a type imported from typing + # like Tuple, List, Dict then replace `typing.` + # with a null string. This needs to be done since + # typing.List is not accepted by TorchScript. + type_to_string = str(type) + return type_to_string.replace(type.__module__ + ".", "") + elif is_torch_native_class(type): + # If the type is a subtype of torch module, then TorchScript expects a fully qualified name + # for the type which is obtained by combining the module name and type name. + return type.__module__ + "." + type.__name__ + else: + # For all other types use the name for the type. + return type.__name__ + + +def get_optional_of_element_type(types): + """Extract element type, return as `Optional[element type]` from consolidated types. + + Helper function to extracts the type of the element to be annotated to Optional + from the list of consolidated types and returns `Optional[element type]`. + TODO: To remove this check once Union support lands. + """ + elem_type = types[1] if type(None) is types[0] else types[0] + elem_type = get_type(elem_type) + + # Optional type is internally converted to Union[type, NoneType], which + # is not supported yet in TorchScript. Hence, representing the optional type as string. + return "Optional[" + elem_type + "]" + + +def get_qualified_name(func): + return func.__qualname__ + + +if _IS_MONKEYTYPE_INSTALLED: + + class JitTypeTraceStoreLogger(CallTraceStoreLogger): + """A JitTypeCallTraceLogger that stores logged traces in a CallTraceStore.""" + + def log(self, trace: CallTrace) -> None: + # pyrefly: ignore [missing-attribute] + self.traces.append(trace) + + class JitTypeTraceStore(CallTraceStore): + def __init__(self) -> None: + super().__init__() + # A dictionary keeping all collected CallTrace + # key is fully qualified name of called function + # value is list of all CallTrace + self.trace_records: dict[str, list] = defaultdict(list) + + def add(self, traces: Iterable[CallTrace]) -> None: + for t in traces: + qualified_name = get_qualified_name(t.func) + self.trace_records[qualified_name].append(t) + + def filter( + self, + qualified_name: str, + qualname_prefix: Optional[str] = None, + limit: int = 2000, + ) -> list[CallTraceThunk]: + return self.trace_records[qualified_name] + + def analyze(self, qualified_name: str) -> dict: + # Analyze the types for the given module + # and create a dictionary of all the types + # for arguments. + records = self.trace_records[qualified_name] + all_args = defaultdict(set) + for record in records: + for arg, arg_type in record.arg_types.items(): + all_args[arg].add(arg_type) + return all_args + + def consolidate_types(self, qualified_name: str) -> dict: + all_args = self.analyze(qualified_name) + # If there are more types for an argument, + # then consolidate the type to `Any` and replace the entry + # by type `Any`. + for arg, types in all_args.items(): + types = list(types) + type_length = len(types) + if type_length == 2 and type(None) in types: + # TODO: To remove this check once Union support in TorchScript lands. + all_args[arg] = get_optional_of_element_type(types) + elif type_length > 1: + all_args[arg] = "Any" + elif type_length == 1: + all_args[arg] = get_type(types[0]) + return all_args + + def get_args_types(self, qualified_name: str) -> dict: + return self.consolidate_types(qualified_name) + + class JitTypeTraceConfig(monkeytype.config.Config): + def __init__(self, s: JitTypeTraceStore) -> None: + super().__init__() + self.s = s + + def trace_logger(self) -> JitTypeTraceStoreLogger: + """Return a JitCallTraceStoreLogger that logs to the configured trace store.""" + # pyrefly: ignore [bad-argument-count] + return JitTypeTraceStoreLogger(self.trace_store()) + + def trace_store(self) -> CallTraceStore: + return self.s + + def code_filter(self) -> Optional[CodeFilter]: + return jit_code_filter + +else: + # When MonkeyType is not installed, we provide dummy class definitions + # for the below classes. + class JitTypeTraceStoreLogger: # type: ignore[no-redef] + def __init__(self) -> None: + pass + + class JitTypeTraceStore: # type: ignore[no-redef] + def __init__(self) -> None: + self.trace_records = None + + class JitTypeTraceConfig: # type: ignore[no-redef] + def __init__(self) -> None: + pass + + monkeytype_trace = None # type: ignore[assignment] # noqa: F811 + + +def jit_code_filter(code: CodeType) -> bool: + """Codefilter for Torchscript to trace forward calls. + + The custom CodeFilter is required while scripting a FX Traced forward calls. + FX Traced forward calls have `code.co_filename` start with '<' which is used + to exclude tracing of stdlib and site-packages in the default code filter. + Since we need all forward calls to be traced, this custom code filter + checks for code.co_name to be 'forward' and enables tracing for all such calls. + The code filter is similar to default code filter for monkeytype and + excludes tracing of stdlib and site-packages. + """ + # Filter code without a source file and exclude this check for 'forward' calls. + if code.co_name != "forward" and ( + not code.co_filename or code.co_filename[0] == "<" + ): + return False + + filename = Path(code.co_filename).resolve() + return not any(_startswith(filename, lib_path) for lib_path in LIB_PATHS) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_pickle.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_pickle.py new file mode 100644 index 0000000000000000000000000000000000000000..9d6b64a52a9838babc6aff508489a43d645ac0d0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_pickle.py @@ -0,0 +1,40 @@ +# These functions are referenced from the pickle archives produced by +# ScriptModule.save() + + +# These (`build_*`) functions used to be used by `pickler.cpp` to specify +# the type of the list for certain special types, but now all lists get +# a type attached and restored via `restore_type_tag` below. The legacy +# functions should stick around for backwards-compatibility. + +from typing import Union + + +def build_intlist(data: list[int]) -> list[int]: + return data + + +def build_tensorlist(data: list[object]) -> list[object]: + return data + + +def build_doublelist(data: list[float]) -> list[float]: + return data + + +def build_boollist(data: list[bool]) -> list[bool]: + return data + + +def build_tensor_from_id(data: Union[int, object]) -> Union[int, None]: + if isinstance(data, int): + # just the id, can't really do anything + return data + return None + + +def restore_type_tag(value: object, type_str: str) -> object: + # The type_ptr is used by the jit unpickler to restore the full static type + # to container types like list when they are re-loaded, but this doesn't + # matter for Python, so just return the plain value + return value diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_recursive.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_recursive.py new file mode 100644 index 0000000000000000000000000000000000000000..ec4bbd125119d5d40a1ab482b3469e50c9133c3f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_recursive.py @@ -0,0 +1,1062 @@ +# mypy: allow-untyped-defs +import collections +import functools +import inspect +import textwrap +import types +import warnings + +import torch +import torch._jit_internal as _jit_internal +from torch._sources import fake_range +from torch.jit._builtins import _find_builtin +from torch.jit._check import AttributeTypeIsSupportedChecker +from torch.jit._state import _add_script_class, _get_script_class, _python_cu +from torch.jit.frontend import ( + get_class_properties, + get_default_args, + get_jit_class_def, + get_jit_def, +) +from torch.nn import Module + + +ScriptMethodStub = collections.namedtuple( + "ScriptMethodStub", ("resolution_callback", "def_", "original_method") +) +PropertyStub = collections.namedtuple("PropertyStub", ("resolution_callback", "def_")) + + +# TODO: there should be a more principled way of doing this. +ignored_attributes = [ + "_version", + "_parameters", + "_buffers", + "_non_persistent_buffers_set", + "_backward_hooks", + "_backward_pre_hooks", + "_forward_hooks", + "_forward_hooks_with_kwargs", + "_forward_pre_hooks", + "_forward_pre_hooks_with_kwargs", + "_forward_hooks_always_called", + "_state_dict_hooks", + "_state_dict_pre_hooks", + "_load_state_dict_pre_hooks", + "_load_state_dict_post_hooks", + "_modules", + "_initializing", + "dump_patches", +] + + +def _compile_and_register_class(obj, rcb, qualified_name): + script_class = _get_script_class(obj) + + if not script_class: + ast = get_jit_class_def(obj, obj.__name__) + defaults = torch.jit.frontend.get_default_args_for_class(obj) + script_class = torch._C._jit_script_class_compile( + qualified_name, ast, defaults, rcb + ) + _add_script_class(obj, script_class) + + return script_class + + +def make_stub(func, name): + rcb = _jit_internal.createResolutionCallbackFromClosure(func) + ast = get_jit_def(func, name, self_name="RecursiveScriptModule") + return ScriptMethodStub(rcb, ast, func) + + +def make_stub_from_method(nn_module, method_name): + func = getattr(nn_module, method_name) + if isinstance(func, ScriptMethodStub): + return func + # Make sure the name present in the resulting AST will match the name + # requested here. The only time they don't match is if you do something + # like: + # def _forward(self): + # pass + # forward = _forward + # In this case, the actual function object will have the name `_forward`, + # even though we requested a stub for `forward`. + return make_stub(func, method_name) + + +def make_stubs_from_exported_methods(mod): + stubs = [] + for name in dir(mod): + item = getattr(mod, name, None) + if ( + _jit_internal.get_torchscript_modifier(item) + is _jit_internal.FunctionModifiers.EXPORT + ): + stubs.append(make_stub_from_method(mod, name)) + + return stubs + + +def jit_ignored_properties(module): + user_annotated_ignored_attributes = getattr( + module, "__jit_ignored_attributes__", [] + ) + + def get_properties_names(module): + return {k for k, v in vars(module).items() if isinstance(v, property)} + + properties = get_properties_names(type(module)) + user_annoted_ignored_properties = set() + + for ignored_attr in user_annotated_ignored_attributes: + if ignored_attr in properties: + user_annoted_ignored_properties.add(ignored_attr) + return user_annoted_ignored_properties + + +# base types that can be constants +# in addition, tuples and lists of these base types are also considered constants +# If you edit this list, then you also need to edit the handlers in +# ConstantValue in jit/script/init.cpp +_constant_types = ( + bool, + float, + int, + str, + type(None), + torch.device, + torch.layout, + torch.dtype, + torch.qscheme, +) + + +def _get_valid_constant(attr, v, owner_type): + if isinstance(v, _constant_types): + return v + elif isinstance(v, (tuple, list)): + return tuple(_get_valid_constant(attr, x, owner_type) for x in v) + constants = ", ".join(torch.typename(typ) for typ in _constant_types) + raise TypeError( + textwrap.dedent( + f""" + '{torch.typename(type(v))}' object in attribute '{owner_type}.{attr}' is not a valid constant. + Valid constants are: + 1. a nn.ModuleList + 2. a value of type {{{constants}}} + 3. a list or tuple of (2) + """ + ) + ) + + +class SourceContext(torch._C._jit_tree_views.SourceRangeFactory): + pass + + +def get_annotations(obj): + # In Python-3.10+ it is recommended to use inspect.get_annotations + # See https://docs.python.org/3.10/howto/annotations.html + # But also, in 3.10 annotations from base class are not inherited + # by unannotated derived one, so they must be manually extracted + annotations = inspect.get_annotations(obj) + if annotations: + return annotations + + def get_cls_annotations(cls): + cls_annotations = inspect.get_annotations(cls) + if cls_annotations: + return cls_annotations + for base in cls.__bases__: + cls_annotations = get_cls_annotations(base) + if cls_annotations: + return cls_annotations + return {} + + cls = obj if isinstance(obj, type) else type(obj) + return get_cls_annotations(cls) + + +def infer_concrete_type_builder(nn_module, share_types=True): + """ + Build a ConcreteModuleTypeBuilder from an nn.Module. + + This ConcreteModuleType doesn't have a JIT type associated with it yet, it + must be filled in by the caller. + """ + concrete_type_builder = torch._C.ConcreteModuleTypeBuilder(type(nn_module)) + if isinstance(nn_module, (torch.nn.ModuleDict)): + concrete_type_builder.set_module_dict() + if isinstance(nn_module, (torch.nn.ModuleList, torch.nn.Sequential)): + concrete_type_builder.set_module_list() + if isinstance(nn_module, (torch.nn.ParameterList)): + concrete_type_builder.set_parameter_list() + if isinstance(nn_module, (torch.nn.ParameterDict)): + concrete_type_builder.set_parameter_dict() + + class_annotations = get_annotations(nn_module) + if isinstance(nn_module, (torch.ao.quantization.QuantWrapper)): + class_annotations = {} + + # Get user-annotated ignored attributes. + user_annotated_ignored_attributes = getattr( + nn_module, "__jit_ignored_attributes__", [] + ) + concrete_type_builder.add_ignored_attributes(user_annotated_ignored_attributes) + ignored_properties = jit_ignored_properties(nn_module) + + # try to infer the type from type annotation or from the object itself + def infer_type(name, item): + # The forward function from Module is special; never use this annotations; we + # need to infer type directly using JIT. I originally wanted to write + # this test as isinstance(class_annotations[name], Callable) but + # isinstance on typing things doesn't seem to work: isinstance(list, Callable) + # is also true! + inferred = False + try: + if ( + name in class_annotations + and class_annotations[name] + != torch.nn.Module.__annotations__["forward"] + ): + ann_to_type = torch.jit.annotations.ann_to_type( + class_annotations[name], fake_range() + ) + attr_type = torch._C.InferredType(ann_to_type) + elif isinstance(item, torch.jit.Attribute): + ann_to_type = torch.jit.annotations.ann_to_type(item.type, fake_range()) + attr_type = torch._C.InferredType(ann_to_type) + else: + attr_type = torch._C._jit_try_infer_type(item) + inferred = True + except RuntimeError as re: + raise RuntimeError(f"Error inferring type for {name}: {item}: {re}") from re + + return attr_type, inferred + + added_names = set() + + for name, item in nn_module._parameters.items(): + if name in user_annotated_ignored_attributes: + continue + + assert item is None or isinstance(item, torch.Tensor) + attr_type, _ = infer_type(name, item) + # We currently have the invariant in various places in our code + # that parameters must be Tensors. However, the nn.Module API also + # allows NoneType parameters. These parameters are not returned as + # part of `parameters()` and its variants, but are available + # through direct attribute access. + concrete_type_builder.add_attribute(name, attr_type.type(), True, False) + added_names.add(name) + + for name, item in nn_module._buffers.items(): + if name in user_annotated_ignored_attributes: + continue + + assert item is None or isinstance(item, torch.Tensor) + attr_type, _ = infer_type(name, item) + concrete_type_builder.add_attribute(name, attr_type.type(), False, True) + added_names.add(name) + + for name, item in nn_module._modules.items(): + if name in user_annotated_ignored_attributes: + continue + + attr_type, _ = infer_type(name, item) + if item is None: + # Modules can be None. We don't have direct support for optional + # Modules, so the register it as an NoneType attribute instead. + concrete_type_builder.add_attribute(name, attr_type.type(), False, False) + continue + if attr_type.success(): + assert attr_type.type().is_interface_type() + # if the type can be inferred, it should be a module interface type + sub_concrete_type = torch._C.ConcreteModuleType.from_jit_type( + attr_type.type() + ) + else: + # otherwise we get the concrete module type for item and add it to concrete_type + sub_concrete_type = get_module_concrete_type(item, share_types) + concrete_type_builder.add_module(name, sub_concrete_type) + + added_names.add(name) + + # populate constants_set + constants_set = set(getattr(nn_module, "__constants__", ())) + + # Constants annotated via `Final[T]` rather than being added to `__constants__` + for name, ann in class_annotations.items(): + if torch._jit_internal.is_final(ann): + constants_set.add(name) + + for name in constants_set: + if name in added_names: + # TODO: We should really error in this case, but its bc-breaking so + # we need to warn for at least one release + if name in nn_module._modules: + hint = "submodule" + elif name in nn_module._buffers: + hint = "buffer" + elif name in nn_module._parameters: + hint = "parameter" + else: + raise AssertionError( + "added_names must be submodule, parameter, or buffer" + ) + + warnings.warn( + f"'{name}' was found in ScriptModule constants, " + f" but it is a non-constant {hint}. Consider removing it.", + stacklevel=2, + ) + continue + if not hasattr(nn_module, name): + # TODO: We should really error in this case, but its bc-breaking so + # we need to warn for at least one release + warnings.warn( + f"'{name}' was found in ScriptModule constants, " + "but was not actually set in __init__. " + "Consider removing it.", + stacklevel=2, + ) + continue + value = getattr(nn_module, name) + concrete_type_builder.add_constant( + name, _get_valid_constant(name, value, type(nn_module).__name__) + ) + added_names.add(name) + + # populate overloads + overloads = getattr(nn_module, "__overloads__", {}) + # update with any annotated overloads + overloads.update( + get_overload_name_mapping( + get_overload_annotations(nn_module, ignored_properties) + ) + ) + for name, overloaded_names in overloads.items(): + concrete_type_builder.add_overload(name, overloaded_names) + + for name, value in nn_module.__dict__.items(): + if name in ignored_attributes or name.startswith("__"): + # Python objects have lots of random attributes attached to them; + # PyTorch adds a few more. Prevent these from getting compiled. + continue + + if name in user_annotated_ignored_attributes: + continue + + if name in added_names: + # Don't re-add anything we already added + continue + + isoverloadpacket = isinstance(value, torch._ops.OpOverloadPacket) + if isoverloadpacket: + value = value.op + # Handle Python function attributes + if inspect.isfunction(value): + try: + scripted_fn = torch.jit.script(value) + concrete_type_builder.add_function_attribute( + name, torch._C._jit_try_infer_type(scripted_fn).type(), value + ) + except Exception as e: + # If we fail to script the function, it isn't a hard error. + # Instead, we will add it to the list of attributes we failed + # to convert, with the compilation error. + hint = ( + "(This function exists as an attribute on the Python module, " + "but we failed to compile it to a TorchScript function. " + f"\nThe error stack is reproduced here:\n{e})" + ) + concrete_type_builder.add_failed_attribute(name, hint) + + continue + + # Handle calls to builtin functions (either bespoke builtins from torch.jit._builtins or + # a call to an aten function like torch.add) + builtin_symbol_name = _find_builtin(value) + if builtin_symbol_name: + concrete_type_builder.add_builtin_function(name, builtin_symbol_name) + continue + + # Handle Script function attributes + if isinstance(value, torch.jit.ScriptFunction): + concrete_type_builder.add_function_attribute( + name, torch._C._jit_try_infer_type(value).type(), value + ) + continue + + # If we got here, this is a regular "data" attribute, add it to the concrete type + attr_type, inferred = infer_type(name, value) + if attr_type.success(): + concrete_type_builder.add_attribute(name, attr_type.type(), False, False) + else: + # TODO: could add more detail here. For example, what the user should do + # when the pytype is `list` or `NoneType` + inferred_msg = ( + "Its type was inferred; try adding a type annotation for the attribute." + if inferred + else "" + ) + additional_info = f"{attr_type.reason()}. {inferred_msg}" + hint = ( + "(This attribute exists on the Python module, " + f"but we failed to convert Python type: '{torch.typename(type(value))}' " + f"to a TorchScript type. {additional_info})" + ) + concrete_type_builder.add_failed_attribute(name, hint) + + # add hooks to concrete type + for hook in nn_module._forward_hooks.values(): + concrete_type_builder.add_forward_hook(hook) + for pre_hook in nn_module._forward_pre_hooks.values(): + concrete_type_builder.add_forward_pre_hook(pre_hook) + + return concrete_type_builder + + +class ConcreteTypeStore: + type_store: dict[type[Module], list[torch._C.ConcreteModuleType]] + methods_compiled: set[torch._C.ConcreteModuleType] + + def __init__(self) -> None: + # Python module type => List[ConcreteModuleType)] + self.type_store = {} + # ConcreteTypes that have had their methods already compiled + self.methods_compiled = set() + + def get_or_create_concrete_type(self, nn_module): + """Infer a ConcreteType from this `nn.Module` instance. Underlying JIT types are reused if possible.""" + concrete_type_builder = infer_concrete_type_builder(nn_module) + + nn_module_type = type(nn_module) + if nn_module_type not in self.type_store: + self.type_store[nn_module_type] = [] + + # Search the type store for an already-available JIT type + known_types = self.type_store[nn_module_type] + for known_type in known_types: + if known_type.equals(concrete_type_builder): + return known_type + + # We didn't find anything; generate a new JIT type from this concrete type + concrete_type = concrete_type_builder.build() + self.type_store[nn_module_type].append(concrete_type) + return concrete_type + + +concrete_type_store = ConcreteTypeStore() + + +def create_methods_and_properties_from_stubs( + concrete_type, method_stubs, property_stubs +) -> None: + method_defs = [m.def_ for m in method_stubs] + method_rcbs = [m.resolution_callback for m in method_stubs] + method_defaults = [get_default_args(m.original_method) for m in method_stubs] + + property_defs = [p.def_ for p in property_stubs] + property_rcbs = [p.resolution_callback for p in property_stubs] + + concrete_type._create_methods_and_properties( + property_defs, property_rcbs, method_defs, method_rcbs, method_defaults + ) + + +def create_hooks_from_stubs(concrete_type, hook_stubs, pre_hook_stubs) -> None: + hook_defs = [h.def_ for h in hook_stubs] + hook_rcbs = [h.resolution_callback for h in hook_stubs] + + pre_hook_defs = [h.def_ for h in pre_hook_stubs] + pre_hook_rcbs = [h.resolution_callback for h in pre_hook_stubs] + + concrete_type._create_hooks(hook_defs, hook_rcbs, pre_hook_defs, pre_hook_rcbs) + + +def get_module_concrete_type(nn_module, share_types=True): + """ + Get a concrete type for nn_modules. + + If share_types is True, the concrete type is fetched from concrete_type_store. + If it is False, a new concrete type is created without first searching concrete_type_store. + + Args: + nn_module: The original Python nn.Module that we are creating a ScriptModule for. + share_types = Whether to share underlying JIT types between modules (if possible). + + Returns: + A concrete type for nn_module. + """ + assert isinstance(nn_module, Module) + if isinstance(nn_module, torch.jit.ScriptModule) and hasattr( + nn_module, "_concrete_type" + ): + return nn_module._concrete_type + + if share_types: + # Look into the store of cached JIT types + concrete_type = concrete_type_store.get_or_create_concrete_type(nn_module) + else: + # Get a concrete type directly, without trying to reuse an existing JIT + # type from the type store. + concrete_type_builder = infer_concrete_type_builder(nn_module, share_types) + concrete_type_builder.set_poisoned() + concrete_type = concrete_type_builder.build() + + return concrete_type + + +def create_script_class(obj): + """ + Create and return a RecursiveScriptClass instance from a Python object. + + Arguments: + obj: A Python object. + """ + qualified_class_name = _jit_internal._qualified_name(type(obj)) + rcb = _jit_internal.createResolutionCallbackForClassMethods(type(obj)) + # Script the type of obj if it hasn't already been scripted. + _compile_and_register_class(type(obj), rcb, qualified_class_name) + class_ty = _python_cu.get_class(qualified_class_name) + # Create an empty torch._C.ScriptObject with the scripted type. + cpp_object = torch._C._create_object_with_type(class_ty) + # Copy all of the attributes over to the torch._C.ScriptObject. + for name, value in obj.__dict__.items(): + cpp_object.setattr(name, value) + + # Wrap the torch._C.ScriptObject in a RecursiveScriptClass instance. + return wrap_cpp_class(cpp_object) + + +def create_script_module(nn_module, stubs_fn, share_types=True, is_tracing=False): + """ + Create a new ScriptModule from an nn.Module. + + Args: + nn_module: The original Python nn.Module that we are creating a ScriptModule for. + stubs_fn: Lambda that takes an nn.Module and generates a list of ScriptMethodStubs to compile. + share_types: Whether to share underlying JIT types between modules (if possible). + NOTE: Only set to False this when we cannot guarantee type sharing will work + correctly. This only happens today for traced modules, where the same + module can produce different traced methods depending on the inputs. + is_tracing: Whether this function is called during tracing or scripting. If tracing, + we don't need to do AttributeTypeIsSupportedChecker because all the unsupported + attributes will be baked as constant in the tracing graph. In addition, + this check significantly slows down the traced modules when the module size is big. + """ + assert not isinstance(nn_module, torch.jit.RecursiveScriptModule) + check_module_initialized(nn_module) + concrete_type = get_module_concrete_type(nn_module, share_types) + if not is_tracing: + AttributeTypeIsSupportedChecker().check(nn_module) + return create_script_module_impl(nn_module, concrete_type, stubs_fn) + + +def create_script_module_impl(nn_module, concrete_type, stubs_fn): + """ + Convert an nn.Module to a RecursiveScriptModule. + + Args: + nn_module: The original Python nn.Module that we are creating a ScriptModule for. + concrete_type: The fully initialized ConcreteType of the module. + stubs_fn: Lambda that takes an nn.Module and generates a list of ScriptMethodStubs to compile. + """ + cpp_module = torch._C._create_module_with_type(concrete_type.jit_type) + method_stubs = stubs_fn(nn_module) + property_stubs = get_property_stubs(nn_module) + hook_stubs, pre_hook_stubs = get_hook_stubs(nn_module) + ignored_properties = jit_ignored_properties(nn_module) + + def init_fn(script_module) -> None: + # Initialize the ScriptModule: + # 1. Copy the attributes/parameters/buffers from the original `nn_module` to the new ScriptModule. + for name in concrete_type.get_attributes(): + orig_value = getattr(nn_module, name) + orig_value = ( + orig_value.value + if isinstance(orig_value, torch.jit.Attribute) + else orig_value + ) + cpp_module.setattr(name, orig_value) + + # 2. Copy the submodules from the original `nn_module` to the new ScriptModule, + # recursively scripting them. + for name, sub_concrete_type in concrete_type.get_modules(): + orig_value = getattr(nn_module, name) + assert isinstance(orig_value, Module), ( + f"Expected Module but got {type(orig_value)}" + ) + module_type = sub_concrete_type.jit_type + if isinstance(module_type, torch._C.InterfaceType): + # use the interface inference rule to compile the module + scripted = interface_script(module_type, orig_value) + elif isinstance(orig_value, torch.jit.ScriptModule): + scripted = orig_value + else: + # always reuse the provided stubs_fn to infer the methods to compile + scripted = create_script_module_impl( + orig_value, sub_concrete_type, stubs_fn + ) + + cpp_module.setattr(name, scripted) + script_module._modules[name] = scripted + + # 3. Copy @ignored/@unused methods and attrs from the original `nn_module` to the new ScriptModule. + # This ensures we can access these Python methods on the ScriptModule. + for name in dir(nn_module): + if name in ignored_properties: + continue + item = getattr(nn_module, name, None) + if inspect.ismethod(item) and _jit_internal.is_ignored_fn(item): + unbound_function = getattr(nn_module, name).__func__ + bound_method = unbound_function.__get__(script_module) + setattr(script_module, name, bound_method) + elif concrete_type.is_ignored_attribute(name): + setattr(script_module, name, item) + + # For convenience, attach the concrete type to the new ScriptModule + script_module._concrete_type = concrete_type + + # Actually create the ScriptModule, initializing it with the function we just defined + script_module = torch.jit.RecursiveScriptModule._construct(cpp_module, init_fn) + + # Compile methods if necessary + if concrete_type not in concrete_type_store.methods_compiled: + create_methods_and_properties_from_stubs( + concrete_type, method_stubs, property_stubs + ) + # Create hooks after methods to ensure no name collisions between hooks and methods. + # If done before, hooks can overshadow methods that aren't exported. + create_hooks_from_stubs(concrete_type, hook_stubs, pre_hook_stubs) + torch._C._run_emit_module_hook(cpp_module) + concrete_type_store.methods_compiled.add(concrete_type) + + # Copy the forward hooks and pre-hooks to the new ScriptModule + # to allow the hooks to be run from eager as ScriptFunctions + for idx, fn in enumerate(script_module._c._get_forward_pre_hooks()): + script_module._forward_pre_hooks[idx] = fn + for idx, fn in enumerate(script_module._c._get_forward_hooks()): + script_module._forward_hooks[idx] = fn + + # Special handling so methods like __len__ work in script methods on classes derived from containers + if ( + isinstance( + nn_module, (torch.nn.ModuleList, torch.nn.Sequential, torch.nn.ModuleDict) + ) + and "__len__" not in cpp_module._method_names() + ): + script_module.define(f"def __len__(self):\n return {len(nn_module)}\n") + if ( + isinstance(nn_module, torch.nn.ModuleDict) + and "__contains__" not in cpp_module._method_names() + ): + if len(nn_module.keys()): + keys = repr(list(nn_module.keys())) + script_module.define( + f"def __contains__(self, key: str):\n return key in {keys}\n" + ) + else: + script_module.define("def __contains__(self, key: str):\n return False\n") + + # Make the compiled methods available to the Python ScriptModule class. + for method_stub in method_stubs: + if method_stub.original_method is None: + # define()'d methods don't have an Python original_method, so we + # don't need to do any Python re-wrapping stuff + continue + + name = method_stub.original_method.__name__ + if name != method_stub.def_.name().name: + # TODO: Why skip this? Because @torch.jit._overload_method will + # mangle the name of the function. + continue + script_method = cpp_module._get_method(name) + + # Wrap the original to propagate docstrings and such. + # TODO: we don't currently do this functions that are recursively + # compiled, we should. + wrapped_script_method = functools.wraps(method_stub.original_method)( + script_method + ) + + # Add the methods to the script_module directly. This ensures they will + # be found first when `name` is looked up (as opposed to the stubs or + # nn.Module.forward) + script_module.__dict__[name] = wrapped_script_method + + # Make module properties available on the Python ScriptModule class. + for property_stub in property_stubs: + property_name = property_stub.def_.name().name + fget = cpp_module._get_method(property_stub.def_.getter_name().name) + # Setter is optional, so it may not exist. + setter_name = property_stub.def_.setter_name() + fset = cpp_module._get_method(setter_name.name) if setter_name else None + script_module.__dict__[property_name] = property(property_name, fget, fset) # type: ignore[arg-type] + + # copy over python methods to script module if they aren't defined on the script module + # this is currently an internal api used only on module containers + for name in dir(nn_module): + if name in ignored_properties: + continue + item = getattr(nn_module, name, None) + if ( + _jit_internal.get_torchscript_modifier(item) + is _jit_internal.FunctionModifiers.COPY_TO_SCRIPT_WRAPPER + ): + add_python_attr_to_scripted_model(script_module, nn_module, name) + + return script_module + + +# We define shims of certain attributes on the RecursiveScriptModule to support +# magic methods. To check if a script model defines an attribute we need +# to also check that the attribute is not the shim +def script_model_defines_attr(script_model, attr): + script_attr = getattr(script_model, attr, None) + if script_attr is None: + return False + default_attr = getattr(torch.jit.RecursiveScriptModule, attr, None) + if default_attr is None: + return False + return script_attr != default_attr + + +def add_python_attr_to_scripted_model(script_model, orig, attr) -> None: + if hasattr(orig, attr) and script_model_defines_attr(script_model, attr): + setattr(script_model, attr, getattr(orig, attr)) + + +def get_overload_annotations(mod, jit_ignored_properties): + # original function => [(mangled overload name, overload function)] + overloads = {} + + for name in dir(type(mod)): + if name in jit_ignored_properties: + continue + item = getattr(mod, name, None) + if not callable(item): + continue + + # builtin functions like repr() in python 2 do not have __module__ defined + if hasattr(item, "__module__") and item.__module__ is not None: + method_overloads = _jit_internal._get_overloaded_methods( + item, mod.__class__ + ) + if method_overloads is None: + continue + + # pyrefly: ignore [missing-attribute] + if item.__func__ in method_overloads: + raise RuntimeError( + _jit_internal.get_overload_no_implementation_error_message( + "method", item.__func__ + ) + ) + + names = [name + "__" + str(i) for i in range(len(method_overloads))] + overloads[item] = list(zip(names, method_overloads)) + + return overloads + + +def get_overload_name_mapping(overload_info): + # Same format as __overloads__ + # original function => [overload names] + overload_name_mappings: dict[str, list[str]] = {} + for orig_fn, overloads in overload_info.items(): + original_name = orig_fn.__name__ + if original_name not in overload_name_mappings: + overload_name_mappings[original_name] = [] + + for overload_name, _ in overloads: + overload_name_mappings[original_name].append(overload_name) + return overload_name_mappings + + +def _check_no_signature(func) -> None: + signature = torch.jit.annotations.get_signature( + func, None, fake_range(), inspect.ismethod(func) + ) + if signature is None: + qual_name = _jit_internal._qualified_name(func) + raise RuntimeError( + f"Must explicitly add type annotations to overloaded functions: {qual_name}" + ) + + +def make_stubs_for_overloads(overload_info): + overload_stubs = [] + for orig_fn, overloads in overload_info.items(): + orig_ast = get_jit_def( + orig_fn, orig_fn.__name__, self_name="RecursiveScriptModule" + ) + for overload_name, overload_fn in overloads: + _check_no_signature(overload_fn) + over_ast = get_jit_def( + overload_fn, overload_fn.__name__, self_name="RecursiveScriptModule" + ) + new_ast = torch._C._replace_overloaded_method_decl( + over_ast.decl(), orig_ast, overload_name + ) + _rcb = _jit_internal.createResolutionCallbackFromClosure(orig_fn) + overload_stubs.append(ScriptMethodStub(_rcb, new_ast, overload_fn)) + return overload_stubs + + +def check_module_initialized(mod) -> None: + assert isinstance(mod, torch.nn.Module) + if not hasattr(mod, "_parameters"): + raise RuntimeError( + f"'{torch.typename(type(mod))}' has not been initialized, did you forget to call 'super()'?" + ) + + # This is to avoid importing torch.distributed.nn + if not hasattr(mod, "remote_parameters"): + for name, param in mod._parameters.items(): + if param is not None and torch.nn.parameter.is_lazy(param): + raise RuntimeError( + f"'{torch.typename(type(mod))}' has uninitialized parameters {name}. Did you forget to run a forward pass?" + ) + for name, buf in mod._buffers.items(): + if buf is not None and torch.nn.parameter.is_lazy(buf): + raise RuntimeError( + f"'{torch.typename(type(mod))}' has uninitialized buffers {name}. Did you forget to run a forward pass?" + ) + + +def infer_methods_to_compile(nn_module): + """Implement the default rules for which methods should act as starting points for compilation. + + (TODO add a link when the rules are published). + """ + check_module_initialized(nn_module) + ignored_properties = jit_ignored_properties(nn_module) + + methods: list[str] = [] + if hasattr(nn_module, "forward") and not _jit_internal.is_ignored_fn( + nn_module.forward + ): + forward_func = getattr(nn_module.forward, "__func__", None) + module_forward = getattr(torch.nn.Module, "forward", None) + if forward_func != module_forward: + methods = ["forward"] + + exported = [] + for name in dir(nn_module): + if name in ignored_properties: + continue + item = getattr(nn_module, name, None) + if ( + _jit_internal.get_torchscript_modifier(item) + is _jit_internal.FunctionModifiers.EXPORT + ): + exported.append(name) + + methods = methods + exported + + overload_name_mappings = dict(getattr(nn_module, "__overloads__", {})) + overload_info = get_overload_annotations(nn_module, ignored_properties) + overload_name_mappings.update(get_overload_name_mapping(overload_info)) + overload_stubs = make_stubs_for_overloads(overload_info) + + nn_module.__overloads__ = overload_name_mappings + + # we shouldn't directly compile overloaded methods, just its overloads + def ignore_overloaded(method_name): + return method_name not in overload_name_mappings + + filtered_methods = filter(ignore_overloaded, methods) + + # Unique the methods. We don't want to use a set to store the methods because it + # introduces non-determinism to compile order. + uniquer: set[str] = set() + uniqued_methods = [] + for name in filtered_methods: + if name in uniquer: + continue + uniqued_methods.append(name) + uniquer.add(name) + + stubs = [make_stub_from_method(nn_module, method) for method in uniqued_methods] + return overload_stubs + stubs + + +def get_hook_stubs(nn_module): + """Return forward hook and pre_hook ScriptModuleStubs.""" + check_module_initialized(nn_module) + hook_map: dict = {} + + hook_stubs = [] + for hook in nn_module._forward_hooks.values(): + if hook.__name__ in hook_map: + if id(hook) != id(hook_map[hook.__name__]): + raise RuntimeError( + f"Hook '{hook.__name__}' on {type(nn_module).__name__} " + "has at least two different python definitions." + " Please use unique names for all hooks." + ) + else: + hook_map[hook.__name__] = hook + hook_stubs.append(make_stub(hook, hook.__name__)) + + pre_hook_stubs = [] + for pre_hook in nn_module._forward_pre_hooks.values(): + if pre_hook.__name__ in hook_map: + if id(pre_hook) != id(hook_map[pre_hook.__name__]): + raise RuntimeError( + f"Pre-hook '{pre_hook.__name__}' on {type(nn_module).__name__} " + "has at least two different python definitions." + " Please use unique names for all hooks." + ) + else: + hook_map[pre_hook.__name__] = pre_hook + pre_hook_stubs.append(make_stub(pre_hook, pre_hook.__name__)) + + return hook_stubs, pre_hook_stubs + + +def get_property_stubs(nn_module): + """Create property stubs for the properties of the module by creating method stubs for the getter and setter.""" + module_ty = type(nn_module) + properties_asts = get_class_properties(module_ty, self_name="RecursiveScriptModule") + rcbs = {} + + for name in dir(module_ty): + item = getattr(module_ty, name, None) + if isinstance(item, property): + if not item.fget: + raise RuntimeError( + f"Property {name} of {nn_module.__name__} must have a getter" + ) + + rcbs[name] = _jit_internal.createResolutionCallbackFromClosure(item.fget) + + stubs = [PropertyStub(rcbs[ast.name().name], ast) for ast in properties_asts] + return stubs + + +def interface_script(mod_interface, nn_module): + """ + Make a ScriptModule from an nn.Module, using the interface methods rule for determining which methods to compile. + + Args: + mod_interface: the interface type that the module have + nn_module: The original Python nn.Module that we are creating a ScriptModule for. + """ + if isinstance(nn_module, torch.jit.ScriptModule): + return nn_module + + check_module_initialized(nn_module) + + def infer_interface_methods_to_compile(nn_module): + """Rule to infer the methods from the interface type. + + It is used to know which methods need to act as starting points for compilation. + """ + stubs = [ + make_stub_from_method(nn_module, method) + for method in mod_interface.getMethodNames() + ] + return stubs + + return create_script_module(nn_module, infer_interface_methods_to_compile) + + +def try_compile_fn(fn, loc): + if _jit_internal.is_ignored_fn(fn): + # Don't do anything for @ignore'd functions + return None + + if isinstance(fn, torch.nn.Module): + # Since modules are callable pybind recognizes them as functions, but + # don't do anything for them + return None + + if not inspect.isfunction(fn) and not inspect.ismethod(fn): + raise RuntimeError( + f"`{fn}` is not a function. Recursive scripting only supports " + "Python functions or methods currently.\n" + f"Consider manually annotating `{fn}` with @torch.jit.script." + ) + + # The object returned by __prepare_scriptable__ might have a different closure. + # Resolve it here to get the right resolution callback. + fn = fn.__prepare_scriptable__() if hasattr(fn, "__prepare_scriptable__") else fn # type: ignore[operator] + + # We don't have the actual scope where the function was defined, but we can + # extract the necessary info from the closed over variables on the function + # object + rcb = _jit_internal.createResolutionCallbackFromClosure(fn) + return torch.jit.script(fn, _rcb=rcb) + + +def wrap_cpp_class(cpp_class): + """Wrap this torch._C.Object in a Python RecursiveScriptClass.""" + return torch.jit.RecursiveScriptClass(cpp_class) + + +def wrap_cpp_module(cpp_module): + """Wrap this torch._C.ScriptModule in a Python ScriptModule, recursively for all submodules.""" + + def init_fn(script_module) -> None: + for name, cpp_module in torch._C.ModuleDict(script_module._c).items(): + setattr(script_module, name, wrap_cpp_module(cpp_module)) + script_module._concrete_type = torch._C.ConcreteModuleType.from_jit_type( + script_module._c._type() + ) + + for idx, fn in enumerate(script_module._c._get_forward_pre_hooks()): + script_module._forward_pre_hooks[idx] = fn + for idx, fn in enumerate(script_module._c._get_forward_hooks()): + script_module._forward_hooks[idx] = fn + + return torch.jit.RecursiveScriptModule._construct(cpp_module, init_fn) + + +def compile_unbound_method(concrete_type, fn): + if _jit_internal.is_ignored_fn(fn): + return None + stub = make_stub(fn, fn.__name__) + with torch._jit_internal._disable_emit_hooks(): + # We don't want to call the hooks here since the graph that is calling + # this function is not yet complete + create_methods_and_properties_from_stubs(concrete_type, (stub,), ()) + return stub + + +def lazy_bind(concrete_type, unbound_method): + """ + Return a function that lazily binds `unbound_method` to a provided Module IValue, then invokes the method. + + We do this so that any Python shenanigans that + will poison type sharing are impossible at compile time. + """ + + def lazy_binding_method(cpp_module, *args): + def init_fn(script_module) -> None: + orig_class = concrete_type.py_class + + # Copy @ignored/@unused methods from the original module to the new one. + # This ensures they are available during execution. + for name in dir(orig_class): + item = getattr(orig_class, name, None) + if _jit_internal.is_ignored_fn(item): + setattr(script_module, name, item) + + # Copy constants over so they are available during execution. + for name, value in concrete_type.get_constants().items(): + setattr(script_module, name, value) + + script_module = torch.jit.RecursiveScriptModule._construct(cpp_module, init_fn) + method = types.MethodType(unbound_method, script_module) + return method(*args) + + # make the lazy binding method "look like" the original method + lazy_binding_method.original_fn = unbound_method # type: ignore[attr-defined] + lazy_binding_method.__name__ = unbound_method.__name__ + torch._jit_internal.copy_torchscript_modifier(unbound_method, lazy_binding_method) + + return lazy_binding_method diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_script.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_script.py new file mode 100644 index 0000000000000000000000000000000000000000..8bf8248b7dfffaeb83a331d09735c6ed1d810341 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_script.py @@ -0,0 +1,1787 @@ +"""TorchScript. + +This module contains functionality to support the JIT's scripting frontend, notably: + - torch.jit.script + +This is not intended to be imported directly; please use the exposed +functionalities in `torch.jit`. +""" + +import collections +import copy +import enum +import functools +import inspect +import pickle +import sys +import warnings +from collections.abc import Callable +from typing import Any, Union +from typing_extensions import deprecated + +import torch +import torch._jit_internal as _jit_internal +from torch._classes import classes +from torch._jit_internal import _get_model_id, _qualified_name +from torch._utils_internal import log_torchscript_usage +from torch.jit._builtins import _register_builtin +from torch.jit._fuser import _graph_for, _script_method_graph_for +from torch.jit._monkeytype_config import ( + JitTypeTraceConfig, + JitTypeTraceStore, + monkeytype_trace, +) +from torch.jit._recursive import ( + _compile_and_register_class, + infer_methods_to_compile, + ScriptMethodStub, + wrap_cpp_module, +) +from torch.jit._state import ( + _enabled, + _set_jit_function_cache, + _set_jit_overload_cache, + _try_get_jit_cached_function, + _try_get_jit_cached_overloads, +) +from torch.jit.frontend import get_default_args, get_jit_class_def, get_jit_def +from torch.nn import Module +from torch.overrides import ( + has_torch_function, + has_torch_function_unary, + has_torch_function_variadic, +) +from torch.package import PackageExporter, PackageImporter +from torch.utils import set_module + +from ._serialization import validate_map_location + + +type_trace_db = JitTypeTraceStore() # DB to hold all call traces from MonkeyType + +torch._C.ScriptMethod.graph_for = _script_method_graph_for # type: ignore[attr-defined] +torch._C.ScriptFunction.graph_for = _graph_for # type: ignore[attr-defined] +ScriptFunction = torch._C.ScriptFunction +ScriptFunction.__doc__ = """ +Functionally equivalent to a :class:`ScriptModule`, but represents a single +function and does not have any attributes or Parameters. +""" +ScriptFunction.__name__ = "ScriptFunction" +ScriptFunction.__qualname__ = "torch.jit.ScriptFunction" +set_module(ScriptFunction, "torch.jit") + + +# Throws an error if a jit function is pickled. +# Helps to avoid Python crashes for Python versions 3.9.5 + when protocol 0 or 1 is given as an argument. +def _reduce(cls): + raise pickle.PickleError("ScriptFunction cannot be pickled") + + +ScriptFunction.__reduce__ = _reduce # type: ignore[assignment] + + +if _enabled: + Attribute = collections.namedtuple("Attribute", ["value", "type"]) +else: + + def Attribute(value, type): # type: ignore[no-redef] + return value + + +Attribute.__doc__ = """ + This method is a pass-through function that returns `value`, mostly + used to indicate to the TorchScript compiler that the left-hand side + expression is a class instance attribute with type of `type`. Note that + `torch.jit.Attribute` should only be used in `__init__` method of `jit.ScriptModule` + subclasses. + + Though TorchScript can infer correct type for most Python expressions, there are some cases where + type inference can be wrong, including: + + - Empty containers like `[]` and `{}`, which TorchScript assumes to be container of `Tensor` + - Optional types like `Optional[T]` but assigned a valid value of type `T`, TorchScript would assume + it is type `T` rather than `Optional[T]` + + In eager mode, it is simply a pass-through function that returns `value` + without other implications. + + Example: + + .. testcode:: + + import torch + from typing import Dict + + class AttributeModule(torch.jit.ScriptModule): + def __init__(self) -> None: + super().__init__() + self.foo = torch.jit.Attribute(0.1, float) + + # we should be able to use self.foo as a float here + assert 0.0 < self.foo + + self.names_ages = torch.jit.Attribute({}, Dict[str, int]) + self.names_ages["someone"] = 20 + assert isinstance(self.names_ages["someone"], int) + + m = AttributeModule() + # m will contain two attributes + # 1. foo of type float + # 2. names_ages of type Dict[str, int] + + .. testcleanup:: + + del AttributeModule + del m + + Note: it's now preferred to instead use type annotations instead of `torch.jit.Attribute`: + + .. testcode:: + + import torch + from typing import Dict + + class AttributeModule(torch.nn.Module): + names: Dict[str, int] + + def __init__(self) -> None: + super().__init__() + self.names = {} + + m = AttributeModule() + + .. testcleanup:: + + del AttributeModule + del m + + Args: + value: An initial value to be assigned to attribute. + type: A Python type + + Returns: + Returns `value` +""" + + +def _get_type_trace_db(): + # This is a private API. Use of this for external purposes is discouraged. + return type_trace_db + + +# Gets a function from the name of a method on a type +def _get_function_from_type(cls, name): + return getattr(cls, name, None) + + +# ScriptClasses must be new-style classes because we construct them using their +# __new__ method. +def _is_new_style_class(cls): + if hasattr(cls, "__class__"): + return "__dict__" in dir(cls) or hasattr(cls, "__slots__") + + +# These OrderedDictWrapper classes replace the actual OrderedDicts in +# module with versions that get/set properties inside of Module. +# This allows us to reuse most of nn.Module while still storing the +# data in C++. +# Each OrderedDict needs to support: +# x not in view +# x in view +# view[name] = ... +# view.values() +# del view[name] +# view.items() +# view.keys() +# len(view) + + +class OrderedDictWrapper: + def __init__(self, _c): + self._c = _c + + def keys(self): + return [k for k, v in self.items()] + + def values(self): + return [v for k, v in self.items()] + + def __len__(self): + return len(self.values()) + + def __delitem__(self, k): + raise RuntimeError("cannot delete methods or parameters of a script module") + + def items(self): + return self._c.items() + + def __setitem__(self, k, v): + if k not in self: + raise RuntimeError( + f"Can't add a new parameter after ScriptModule construction. Tried to add '{k}" + ) + self._c.setattr(k, v) + + def __contains__(self, k): + return self._c.contains(k) + + def __getitem__(self, k): + if k not in self: + raise KeyError(k) + return self._c.getattr(k) + + +class OrderedModuleDict(OrderedDictWrapper): + def __init__(self, module, python_dict): + super().__init__(torch._C.ModuleDict(module)) + # contains _both_ script modules and non-script python-only modules + + # because script modules are subclassed in python and the + # C++ Module class will not hold references to them, + # to ensure that you always get the same python value here + # we store it in the python dict as well + self._python_modules = python_dict + + def items(self): + r = self._python_modules.items() + return r + + def __contains__(self, k): + return k in self._python_modules + + def __setitem__(self, k, v): + # Cases where sub-module can be re-assigned after ScriptModule construction + # 1. If the attr is an module interface type, it's guaranteed that the module is + # not inlined in the graph, so it's safe to swap a new ScriptModule in. + # 2. if the new value if a ScriptModule with the same JIT type, IR won't change + # and it's legit to swap a new module in. + # In these two cases we allow swapping a new scripted module and update the + # corresponding python module dict to keep sync. + # Note: the value to be swapped in has to be ScriptModule instead of nn.Module, + # otherwise it's illegal and we throw error. + if isinstance(v, ScriptModule): + self._c.setattr(k, v) + self._python_modules[k] = v + else: + raise RuntimeError( + "Cannot re-assign modules in a ScriptModule with non-scripted " + f"module, tried to replace existing module '{k}': {v}" + ) + + def __getitem__(self, k): + return self._python_modules[k] + + +# For each user-defined class that subclasses ScriptModule, this meta-class: +# (1) finds all the methods annotated with @script_method in a ScriptModule and +# removes them from the class attributes +# (2) puts a wrapper around the class's __init__ method to recursively compile +# all of the script_methods with the module after the original __init__ has +# run. This has to occur after the user-defined __init__ so that submodules and +# parameters are initialized _before_ the script compiler resolve references to +# `self.param` or `self.module`. +class ScriptMeta(type): + def __init__(cls, name, bases, attrs): # noqa: B902 + # Aggregate all the ScriptMethods and constants from superclasses + cls._methods: dict[str, Any] = {} + cls._constants_set = set(getattr(cls, "__constants__", ())) + for base in reversed(bases): + for k, v in getattr(base, "_methods", {}).items(): + cls._methods[k] = v + base_constants: set = getattr(base, "_constants_set", set()) + cls._constants_set = cls._constants_set.union(base_constants) + + # find all the script methods of the current class + for k, v in sorted(attrs.items()): + if isinstance(v, ScriptMethodStub): + delattr(cls, k) + cls._methods[v.original_method.__name__] = v + + if getattr(cls, "_disable_script_meta", False): + # We leave built-in ScriptModule types alone, since this metaclass + # is only for compiling user classes that inherit from + # ScriptModule. + super().__init__(name, bases, attrs) + return + + original_init = getattr(cls, "__init__", lambda self: None) + + @functools.wraps(original_init) + def init_then_script(self, *args, **kwargs): + num_methods = len(cls._methods) + original_init(self, *args, **kwargs) + added_methods_in_init = len(cls._methods) > num_methods + + if type(self) is cls: + + def make_stubs(module): + cls = type(module) + if hasattr(cls, "_methods"): + return [v for k, v in sorted(cls._methods.items())] + else: + return infer_methods_to_compile(module) + + self.__dict__["_actual_script_module"] = ( + torch.jit._recursive.create_script_module( + self, make_stubs, share_types=not added_methods_in_init + ) + ) + + # Delete the Python attributes that now shadow the ScriptModule + # ones, so that __getattr__ and __setattr__ will properly find + # the scripted versions. + concrete_type = self._actual_script_module._concrete_type + for name in concrete_type.get_attributes(): + delattr(self, name) + for name, _ in concrete_type.get_modules(): + delattr(self, name) + for name in ("_parameters", "_buffers", "_modules"): + delattr(self, name) + + cls.__init__ = init_then_script # type: ignore[misc] + super().__init__(name, bases, attrs) + + +class _CachedForward: + def __get__(self, obj, cls): + return self.__getattr__("forward") # type: ignore[attr-defined] + + +class ScriptWarning(Warning): + pass + + +def script_method(fn): + if sys.version_info >= (3, 14): + warnings.warn( + "`torch.jit.script_method` is not supported in Python 3.14+ and may break. " + "Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + else: + warnings.warn( + "`torch.jit.script_method` is deprecated. Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + if not _enabled: + return fn + # NOTE: we need to traverse two frames here because the meta-class frame + # for ScriptModule will be present, as opposed to invoking @script on a + # a function or invoking define() on a CompilationUnit. + # The stack will look like: + # + # 0. createResolutionCallback() + # 1. script_method() + # 2. ScriptModule metaclass frame + # 3. Surrounding scope + # + # createResolutionCallback internally adds 1 to get us to the scope of this + # function (the calling function). Adding 2 gets us to the proper surrounding scope. + _rcb = _jit_internal.createResolutionCallbackFromFrame(frames_up=2) + ast = get_jit_def(fn, fn.__name__, self_name="ScriptModule") + return ScriptMethodStub(_rcb, ast, fn) + + +class ConstMap: + def __init__(self, const_mapping): + self.const_mapping = const_mapping + + def __getattr__(self, attr): + return self.const_mapping[attr] + + +def unpackage_script_module( + importer: PackageImporter, script_module_id: str +) -> torch.nn.Module: + """ + Call by ``torch.package.PackageImporter``'s Pickler's ``persistent_load`` function. + + Performs work of loading and returning a ScriptModule from a ``torch.package`` archive. + """ + if not isinstance(importer.zip_reader, torch._C.PyTorchFileReader): + raise RuntimeError( + "Loading ScriptObjects from a PackageImporter created from a " + "directory is not supported. Use a package archive file instead." + ) + cu = torch._C.CompilationUnit() + cpp_module = torch._C._import_ir_module_from_package( + cu, + importer.zip_reader, + importer.storage_context, + validate_map_location(importer.last_map_location), + script_module_id, + ) + return wrap_cpp_module(cpp_module) + + +if _enabled: + _magic_methods = [ + "__iter__", + "__len__", + "__neg__", + "__mul__", + "__contains__", + "__add__", + "__sub__", + "__pow__", + "__truediv__", + "__mod__", + "__ne__", + "__eq__", + "__lt__", + "__gt__", + "__le__", + "__ge__", + "__and__", + "__or__", + "__xor__", + "__getitem__", + "__setitem__", + "__call__", + "__int__", + "__float__", + "__bool__", + "__str__", + "__enter__", + "__exit__", + ] + + class RecursiveScriptClass: + """Wrapper for a TorchScript class instance for use in Python. + + An analogue of RecursiveScriptModule for regular objects that are not modules. + This class is a wrapper around a torch._C.ScriptObject that represents an instance + of a TorchScript class and allows it to be used in Python. + + Attributes: + _c [torch._C.ScriptObject]: The C++ object to which attribute lookups and method + calls are forwarded. + _props [Dict[str, property]]: A dictionary of properties fetched from self._c and + exposed on this wrppaer. + """ + + def __init__(self, cpp_class): + super().__init__() + self.__dict__["_initializing"] = True + self._c = cpp_class + + # Add wrapped object's properties to this class instance. + self._props = { + prop.name: property(prop.getter, prop.setter) + for prop in self._c._properties() + } + + self.__dict__["_initializing"] = False + + def __getattr__(self, attr): + if self.__dict__.get("_initializing"): + return super().__getattr__(attr) # type: ignore[misc] + + if attr in self._props: + return self._props[attr].fget() # type: ignore[call-arg, misc] + + return getattr(self._c, attr) + + def __setattr__(self, attr, value): + if self.__dict__.get("_initializing"): + return super().__setattr__(attr, value) + + if attr in self._props: + return self._props[attr].fset(value) # type: ignore[call-arg, misc] + + setattr(self._c, attr, value) + + # Delegate calls to magic methods like __len__ to the C++ module backing the + # RecursiveScriptClass. + def forward_magic_method(self, method_name, *args, **kwargs): + if not self._c._has_method(method_name): + raise TypeError + + self_method = self.__getattr__(method_name) + return self_method(*args, **kwargs) + + def __getstate__(self): + raise pickle.PickleError("ScriptClasses cannot be pickled") + + def __iadd__(self, other): + if self._c._has_method("__iadd__"): + return self.forward_magic_method("__iadd__", other) + else: + return self.forward_magic_method("__add__", other) + + for method_name in _magic_methods: + + def method_template(self, *args, **kwargs): + return self.forward_magic_method(method_name, *args, **kwargs) + + setattr(RecursiveScriptClass, method_name, method_template) + + # this is a Python 'non-data descriptor' that causes the first access + # to ScriptModule's forward to look up the forward method and stash + # it in the objects dict. Due to the standard rules for attribute lookup, + # subsequent lookups will just directly return the previously looked up method. + # This is necessary because nn.Module defines forward as a method. If we + # did nothing, __getattr__ would not be called. Instead we'd get nn.Module.forward + # which always throws an exception. + + class ScriptModule(Module, metaclass=ScriptMeta): + r"""Wrapper for C++ torch::jit::Module with methods, attributes, and parameters. + + A wrapper around C++ ``torch::jit::Module``. ``ScriptModule``\s + contain methods, attributes, parameters, and + constants. These can be accessed the same way as on a normal ``nn.Module``. + """ + + __jit_unused_properties__ = [ + "code", + "code_with_constants", + "graph", + "inlined_graph", + "original_name", + ] + + def __init__(self) -> None: + super().__init__() + + forward: Callable[..., Any] = _CachedForward() # type: ignore[assignment] + + def __getattr__(self, attr): + if "_actual_script_module" not in self.__dict__: + return super().__getattr__(attr) + return getattr(self._actual_script_module, attr) + + def __setattr__(self, attr, value): + if "_actual_script_module" not in self.__dict__: + # Unwrap torch.jit.Attribute into a regular setattr + record + # the provided type in __annotations__. + # + # This ensures that if we use the attr again in `__init__`, it + # will look like the actual value, not an instance of Attribute. + # pyrefly: ignore [invalid-argument] + if isinstance(value, Attribute): + # NB: Ensure that we set __annotations__ on the specific + # class in question, and not on a superclass (which would + # be wrong wrong wrong!). + # See also https://github.com/pytorch/pytorch/issues/39463 + if "__annotations__" not in self.__class__.__dict__: + self.__class__.__annotations__ = {} + self.__annotations__[attr] = value.type + value = value.value + return super().__setattr__(attr, value) + + setattr(self._actual_script_module, attr, value) + + def define(self, src): + if "_actual_script_module" in self.__dict__: + # If we have completed initialization, just defer to the + # backing RecursiveScriptModule to eagerly compile the provided + # source. + return self._actual_script_module.define(src) + + # Otherwise, we are still in the object's __init__. + # In that case, add `src` as a stub to be compiled. + # + # We use frames_up=1 to get to the proper surrounding scope. The stack + # will look like: + # 0. createResolutionCallback + # 1. define() + # 2. surrounding scope. + # + # createResolutionCallback internally adds 1 to get us to our frame, then + # we add 1 to get to the proper surrounding scope. + rcb = _jit_internal.createResolutionCallbackFromFrame(frames_up=1) + ast = torch._C._parse_source_def(src) + self._methods[ast.name().name] = ScriptMethodStub(rcb, ast, None) + + def _replicate_for_data_parallel(self): + return self._actual_script_module._replicate_for_data_parallel() + + def __reduce_package__(self, exporter: PackageExporter): + """Save a ScriptModule inside of a ``torch.package`` archive. + + Called by ``torch.package.PackageExporter``'s Pickler's ``persistent_id`` when + saving TorchScript objects. Performs act of saving a ScriptModule inside of + a ``torch.package`` archive. + + Returns method to load the ScriptModule from a ``torch.package.PackageImporter``'s + Pickler's ``persistent_load`` function. + """ + script_module_id = exporter.get_unique_id() + exporter.script_module_serializer.serialize(self._c, int(script_module_id)) + return (unpackage_script_module, (script_module_id,)) + + class RecursiveScriptModule(ScriptModule): + # XXX: RecursiveScriptModule inherits from ScriptModule for the sole + # reason that it retains the existing isinstance(ScriptModule) + # behavior. + r"""Retain the existing isinstance(ScriptModule) behavior. + + The core data structure in TorchScript is the ``ScriptModule``. It is an + analogue of torch's ``nn.Module`` and represents an entire model as a tree of + submodules. Like normal modules, each individual module in a ``ScriptModule`` can + have submodules, parameters, and methods. In ``nn.Module``\s methods are implemented + as Python functions, but in ``ScriptModule``\s methods are implemented as + TorchScript functions, a statically-typed subset of Python that contains all + of PyTorch's built-in Tensor operations. This difference allows your + ``ScriptModule``\s code to run without the need for a Python interpreter. + + ``ScriptModule``\s should not be created manually, instead use + either :func:`tracing ` or :func:`scripting `. + Tracing and scripting can be applied incrementally and :ref:`composed as necessary `. + + * Tracing records the tensor operations as executed with a set of example inputs and uses these + operations to construct a computation graph. You can use the full dynamic behavior of Python with tracing, + but values other than Tensors and control flow aren't captured in the graph. + + * Scripting inspects the Python code of the model + and compiles it to TorchScript. Scripting allows the use of many `types`_ of values and supports dynamic control flow. + Many, but not all features of Python are supported by the compiler, so changes to the source code may be necessary. + """ + + _disable_script_meta = True + + def __init__(self, cpp_module): + self.__dict__["_initializing"] = True + self._c = cpp_module + super().__init__() + # Delete the 'training' attribute set up by `Module.__init__`. It + # will get set on the underlying cpp module, so we delete it here + # to avoid this version shadowing the cpp module version. + delattr(self, "training") + + @staticmethod + def _construct(cpp_module, init_fn): + """ + Construct a RecursiveScriptModule that's ready for use. + + PyTorch code should use this to construct a RecursiveScriptModule instead + of instead of calling `__init__` directly, as it makes sure the + object is properly finalized (and in the future, we may take + control of how the RecursiveScriptModule instance is created). + + Args: + cpp_module: The C++ Module that will hold the actual state of + this RecursiveScriptModule instance. + init_fn: Lambda that initializes the RecursiveScriptModule passed to it. + """ + script_module = RecursiveScriptModule(cpp_module) + init_fn(script_module) + + # Finalize the ScriptModule: replace the nn.Module state with our + # custom implementations and flip the _initializing bit. + # pyrefly: ignore [missing-attribute] + RecursiveScriptModule._finalize_scriptmodule(script_module) + return script_module + + @staticmethod + def _finalize_scriptmodule(script_module): + script_module._parameters = OrderedDictWrapper( + torch._C.ParameterDict(script_module._c) + ) + script_module._buffers = OrderedDictWrapper( + torch._C.BufferDict(script_module._c) + ) + script_module._modules = OrderedModuleDict( + script_module._c, script_module._modules + ) + script_module._initializing = False + + def _reconstruct(self, cpp_module): + """ + Re-construct an instance of RecursiveScriptModule using an instance of a C++ module. + + Args: + cpp_module: The C++ module that this RecursiveScriptModule will be rebuilt around. + """ + self.__init__(cpp_module) # type: ignore[misc] + + # Copy the concrete type from the C++ module to this ScriptModule. + self._concrete_type = torch._C.ConcreteModuleType.from_jit_type( + self._c._type() + ) + + # Copy submodules from the C++ module to this ScriptModule. + modules = {} + for name, cpp_module in torch._C.ModuleDict(self._c).items(): + modules[name] = wrap_cpp_module(cpp_module) + self._modules = OrderedModuleDict(self._c, modules) # type: ignore[assignment] + + # Copy parameters and buffers. + self._parameters = OrderedDictWrapper(torch._C.ParameterDict(self._c)) # type: ignore[assignment] + self._buffers = OrderedDictWrapper(torch._C.BufferDict(self._c)) # type: ignore[assignment] + + # Get rid of the functions from the old C++ module. + self.__dict__ = { + k: v + for k, v in self.__dict__.items() + if not isinstance(v, torch._C.ScriptMethod) + } + self.__dict__["_initializing"] = False + + @property + def graph(self): + r"""Return a string representation of the internal graph for the ``forward`` method.""" + return self._c._get_method("forward").graph + + @property + def inlined_graph(self): + r""" + Return a string representation of the internal graph for the ``forward`` method. + + This graph will be preprocessed to inline all function and method calls. + """ + return self.forward.inlined_graph # type: ignore[attr-defined] + + @property + def code(self): + r""" + Return a pretty-printed representation (as valid Python syntax) of the internal graph for the ``forward`` method. + + """ + return self.forward.code # type: ignore[attr-defined] + + @property + def code_with_constants(self): + r"""Return a tuple. + + Returns a tuple of: + + [0] a pretty-printed representation (as valid Python syntax) of + the internal graph for the ``forward`` method. See `code`. + [1] a ConstMap following the CONSTANT.cN format of the output in [0]. + The indices in the [0] output are keys to the underlying constant's values. + + """ + r = self.forward.code_with_constants # type: ignore[attr-defined] + return (r[0], ConstMap(r[1])) + + def save(self, f, **kwargs): + r"""Save with a file-like object. + + save(f, _extra_files={}) + + See :func:`torch.jit.save ` which accepts a file-like object. + This function, torch.save(), converts the object to a string, treating it as a path. + DO NOT confuse these two functions when it comes to the 'f' parameter functionality. + """ + return self._c.save(str(f), **kwargs) + + @deprecated( + "Lite Interpreter is deprecated. Please consider switching to ExecuTorch. \ + https://docs.pytorch.org/executorch/stable/getting-started.html" + ) + def _save_for_lite_interpreter(self, *args, **kwargs): + r"""Add (or update) the bytecode session to the script model. + + _save_for_lite_interpreter(f) + + The updated model is used + in lite interpreter for mobile applications. + + Args: + f: a string containing a file name. + _extra_files: Map from filename to contents which will be stored as part of 'f'. + + """ + warnings.warn( + "Lite Interpreter is deprecated. Please consider switching to ExecuTorch. \ + https://docs.pytorch.org/executorch/stable/getting-started.html", + DeprecationWarning, + stacklevel=2, + ) + return self._c._save_for_mobile(*args, **kwargs) + + @deprecated( + "Lite Interpreter is deprecated. Please consider switching to ExecuTorch. \ + https://docs.pytorch.org/executorch/stable/getting-started.html" + ) + def _save_to_buffer_for_lite_interpreter(self, *args, **kwargs): + warnings.warn( + "Lite Interpreter is deprecated. Please consider switching to ExecuTorch. \ + https://docs.pytorch.org/executorch/stable/getting-started.html", + DeprecationWarning, + stacklevel=2, + ) + return self._c._save_to_buffer_for_mobile(*args, **kwargs) + + def save_to_buffer(self, *args, **kwargs): + return self._c.save_to_buffer(*args, **kwargs) + + def get_debug_state(self, *args, **kwargs): + return self._c.get_debug_state() + + def extra_repr(self): + return f"original_name={self.original_name}" + + def graph_for(self, *args, **kwargs): + return self.forward.graph_for(self, *args, **kwargs) # type: ignore[attr-defined] + + @property + def original_name(self): + if type(self) is str(self._c._type().name()): + return "" + return str(self._c._type().name()) + + def define(self, src): + # We use frames_up=1 to get to the proper surrounding scope. The stack + # will look like: + # 0. createResolutionCallback + # 1. define() + # 2. surrounding scope. + # + # createResolutionCallback internally adds 1 to get us to our frame, then + # we add 1 to get to the proper surrounding scope. + rcb = _jit_internal.createResolutionCallbackFromFrame(frames_up=1) + self._c._define(self._concrete_type, src, rcb) + + def __getattr__(self, attr): + if "_initializing" not in self.__dict__: + raise RuntimeError( + "ScriptModule has not been initialized, did you forget to call super's init?" + ) + + if self._initializing: + return super().__getattr__(attr) + + # _modules check is before hasattr since modules are included as attributes in _c, + # but we want to get the python wrapper from _modules instead of the raw _c object. + if attr in self._modules: + return self._modules[attr] + elif self._c.hasattr(attr): + return self._c.getattr(attr) + elif self._c._has_method(attr): + script_method = self._c._get_method(attr) + # cache method so future calls do not go through __getattr__ + # to improve invocation performance + self.__dict__[attr] = script_method + return script_method + + return super().__getattr__(attr) + + def __setattr__(self, attr, value): + if self._initializing: + return super().__setattr__(attr, value) + + if attr in self._modules: + self._modules[attr] = value + elif self._c.hasattr(attr): + self._c.setattr(attr, value) + elif ( + hasattr(self, "_concrete_type") + and attr in self._concrete_type.get_constants() + ): + # TODO: we don't have _concrete_type set after load(), and in general we lose constant information. + # We should encode constants as class type attributes (or something) so it persists across save/load. + raise AttributeError( + f"Cannot mutate TorchScript constant value: '{attr}'. Value: '{value}'" + ) + else: + # We allow setting Python attributes on the ScriptModule, for + # when people want to stash some convenience info on it. + # TODO: it's possible that the following is confusing: + # s = torch.jit.script(...) + # s.python_attr = ... + # s.save() <--- this doesn't have `python_attr` + # It's fairly trivial to save enough info to warn in this case. + return super().__setattr__(attr, value) + + def __copy__(self): + return torch.jit._recursive.wrap_cpp_module(copy.copy(self._c)) + + def __deepcopy__(self, memo): + return torch.jit._recursive.wrap_cpp_module(copy.deepcopy(self._c, memo)) + + # Python magic methods do method lookups on an object's class type, instead of looking up + # the method defines on the class instance. In order to continue to expose the magic methods + # of builtin-containers (ModuleList, Sequential, ModuleDict) to Python, we + # define magic methods here as a shim to the correct attribute. + def forward_magic_method(self, method_name, *args, **kwargs): + self_method = getattr(self, method_name) + if getattr(self_method, "__func__", None) == getattr( + RecursiveScriptModule, method_name + ): + raise NotImplementedError + return self_method(*args, **kwargs) + + def __iter__(self): + return self.forward_magic_method("__iter__") + + def __getitem__(self, idx): + return self.forward_magic_method("__getitem__", idx) + + def __len__(self): + return self.forward_magic_method("__len__") + + def __contains__(self, key): + return self.forward_magic_method("__contains__", key) + + # dir is defined by the base nn.Module, so instead of throwing if + # it is not overridden, we call into the nn.Module __dir__ method + def __dir__(self): + self_method = self.__dir__ + if ( + self_method.__func__ # type: ignore[attr-defined] + is _get_function_from_type(RecursiveScriptModule, "__dir__") + ): + return super().__dir__() + return self_method() + + # to resolve bool(value), Python looks if __bool__ is defined then __iter__ + # is defined then returns true for classes. Since __iter__() on this + # class throws if it isn't overridden, we define __bool__ to preserve default behavior + def __bool__(self): + self_method = self.__bool__ + if ( + self_method.__func__ # type: ignore[attr-defined] + is _get_function_from_type(RecursiveScriptModule, "__bool__") + ): + return True + return self_method() + + def _replicate_for_data_parallel(self): + # we have to initialize ScriptModule properly so that + # it works with pybind11 + def init_fn(script_module): + # Don't do anything here, we'll initialize the ScriptModule below + return + + # pyrefly: ignore [missing-attribute] + return RecursiveScriptModule._construct( + self._c._replicate_for_data_parallel(), init_fn + ) + + # Need to copy all RecursiveScriptModule methods to ScriptModule. + # + # This is because `super().foo()` does not use + # `__getattr__` to look up `foo`. So we need to make each method available on + # the ScriptModule manually. + # pyrefly: ignore [missing-attribute] + for name, item in RecursiveScriptModule.__dict__.items(): + if not callable(item) and not isinstance(item, property): + continue + if name.startswith("__") or hasattr(ScriptModule, name): + continue + # We can copy over the implementation wholesale because besides the + # `super()` thing above, ScriptModule behaves exactly like + # RecursiveScriptModule + setattr(ScriptModule, name, item) + + def _get_methods(cls): + import inspect + + # In Python 3 unbound methods are functions, but in Python 2 they are methods + return inspect.getmembers( + cls, predicate=lambda x: inspect.isfunction(x) or inspect.ismethod(x) + ) + + _compiled_methods_allowlist = { + "forward", + "register_buffer", + "register_parameter", + "register_module", + "add_module", + "_apply", + "apply", + "cuda", + "cpu", + "to", + "type", + "float", + "double", + "half", + "state_dict", + "_save_to_state_dict", + "load_state_dict", + "_load_from_state_dict", + "_named_members", + "parameters", + "named_parameters", + "buffers", + "named_buffers", + "children", + "named_children", + "modules", + "named_modules", + "zero_grad", + "share_memory", + "_get_name", + "extra_repr", + "_slow_forward", + "_tracing_name", + "eval", + "train", + "get_extra_state", + "set_extra_state", + } + + def _make_fail(name): + def fail(self, *args, **kwargs): + raise RuntimeError(name + " is not supported on ScriptModules") + + return fail + + for name, method in _get_methods(torch.nn.Module): + if name.startswith("__") or name.endswith("_call_impl"): + continue + if ( + # pyrefly: ignore [missing-attribute] + name not in RecursiveScriptModule.__dict__ + and name not in _compiled_methods_allowlist + ): + setattr(RecursiveScriptModule, method.__name__, _make_fail(name)) + + +else: + # TODO MAKE SURE THAT DISABLING WORKS + class RecursiveScriptClass: # type: ignore[no-redef] + pass + + class ScriptModule(torch.nn.Module): # type: ignore[no-redef] + def __init__(self, arg=None): + super().__init__() + + class RecursiveScriptModule(ScriptModule): # type: ignore[no-redef] + def __init__(self, arg=None): + super().__init__() + + +def call_prepare_scriptable_func_impl(obj, memo): + if not isinstance(obj, torch.nn.Module): + return obj + + obj_id = id(obj) + + # If obj_id is in memo, obj has already been prepared or is being + # prepared in another call up the stack. + if obj_id in memo: + return memo[id(obj)] + + obj = ( + # pyrefly: ignore [not-callable] + obj.__prepare_scriptable__() if hasattr(obj, "__prepare_scriptable__") else obj + ) # type: ignore[operator] + # Record obj in memo to avoid infinite recursion in the case of cycles in the module + # hierarchy when recursing below. + memo[obj_id] = obj + + new_obj_dict = {} + + for name, sub_module in obj.__dict__.items(): + if name == "_modules": + for k, v in sub_module.items(): + sub_module[k] = call_prepare_scriptable_func_impl(v, memo) + new_obj_dict[name] = sub_module + elif isinstance(sub_module, torch.nn.Module) and not isinstance( + sub_module, ScriptModule + ): + new_obj_dict[name] = call_prepare_scriptable_func_impl(sub_module, memo) + else: + new_obj_dict[name] = sub_module + + for v in new_obj_dict.values(): + obj.__dict__[name] = v + + return obj + + +def call_prepare_scriptable_func(obj): + memo: dict[int, torch.nn.Module] = {} + return call_prepare_scriptable_func_impl(obj, memo) + + +def create_script_dict(obj): + """ + Create a ``torch._C.ScriptDict`` instance with the data from ``obj``. + + Args: + obj (dict): The Python dictionary that is used to initialize the ``ScriptDict`` + returned by this function. + + Returns: + An instance of ``torch._C.ScriptDict`` that has the same data as ``obj`` + and can be passed between Python and TorchScript with reference semantics and + zero copy overhead. + """ + return torch._C.ScriptDict(obj) # type: ignore[attr-defined] + + +def create_script_list(obj, type_hint=None): + """ + Create a ``torch._C.ScriptList`` instance with the data from ``obj``. + + Args: + obj (dict): The Python list that is used to initialize the ``ScriptList`` + returned by this function. + Returns: + An instance of ``torch._C.ScriptList`` that has the same data as ``obj`` + and can be passed between Python and TorchScript with reference semantics and + zero copy overhead. + """ + return torch._C.ScriptList(obj) # type: ignore[attr-defined] + + +_TOPLEVEL: bool = True + + +def _script_impl( + obj, + optimize=None, + _frames_up=0, + _rcb=None, + example_inputs: Union[list[tuple], dict[Callable, list[tuple]], None] = None, +): + global type_trace_db + + if optimize is not None: + warnings.warn( + "`optimize` is deprecated and has no effect. " + "Use `with torch.jit.optimized_execution()` instead", + FutureWarning, + stacklevel=3, + ) + + # No-op for modules, functions, class instances that are already scripted + if isinstance(obj, RecursiveScriptClass): + return obj + if isinstance(obj, ScriptModule): + return obj + if isinstance(obj, ScriptFunction): + return obj + + if example_inputs: + # If MonkeyType is installed, enable profile directed type annotation + # Check if example_inputs are defined and generate call traces + # for the method by running eager mode version of the method with + # the provide example inputs. This logs all the traces in type_trace_db + type_trace_db = JitTypeTraceStore() + if monkeytype_trace: + # pyrefly: ignore [bad-argument-count] + monkeytype_config = JitTypeTraceConfig(type_trace_db) + with monkeytype_trace(monkeytype_config): + if isinstance(example_inputs, dict): + # If the obj is an nn.Module or a class, then each method is + # executed with the arguments provided in the example inputs. + # example inputs here will be of type Dict(class.method, (arguments)) + # This is used to infer type annotations for those methods + # which are not called directly under the hood of monkeytype. + for module, example_input in example_inputs.items(): + for example in example_input: + module(*example) + elif isinstance(example_inputs, list): + for examples in example_inputs: + obj(*examples) + else: + raise ValueError( + "Error: Unable to infer types. Please format the inputs to type `List[Tuple]`" + " or `Dict[Callable, List[Tuple]]` to be run with MonkeyType." + ) + else: + warnings.warn( + "Warning: monkeytype is not installed. Please install https://github.com/Instagram/MonkeyType " + "to enable Profile-Directed Typing in TorchScript. Refer to " + "https://github.com/Instagram/MonkeyType/blob/master/README.rst to install MonkeyType. ", + stacklevel=2, + ) + + if isinstance(obj, torch.nn.Module): + obj = call_prepare_scriptable_func(obj) + return torch.jit._recursive.create_script_module( + obj, torch.jit._recursive.infer_methods_to_compile + ) + else: + obj = ( + obj.__prepare_scriptable__() + if hasattr(obj, "__prepare_scriptable__") + else obj + ) # type: ignore[operator] + + if isinstance(obj, dict): + return create_script_dict(obj) + if isinstance(obj, list): + return create_script_list(obj) + + if inspect.isclass(obj): + qualified_name = _qualified_name(obj) + # If this type is a `nn.Module` subclass, they probably meant to pass + # an instance instead of a Module + if issubclass(obj, torch.nn.Module): + raise RuntimeError( + f"Type '{obj}' cannot be compiled since it inherits from nn.Module, pass an instance instead" + ) + + # Enums are automatically usable in TorchScript, explicitly scripting + # is not necessary, but not harmful either. + if issubclass(obj, enum.Enum): + return obj + + if not _is_new_style_class(obj): + raise RuntimeError( + "TorchScript classes must be new-style classes. " + "Please inherit from 'object'." + ) + if len(obj.mro()) > 2: + raise RuntimeError( + "TorchScript classes does not support inheritance yet. " + "Please directly inherit from 'object'." + ) + if _rcb is None: + _rcb = _jit_internal.createResolutionCallbackFromFrame(_frames_up + 1) + _compile_and_register_class(obj, _rcb, qualified_name) + return obj + elif inspect.isfunction(obj) or inspect.ismethod(obj): + qualified_name = _qualified_name(obj) + # this is a decorated fn, and we need to the underlying fn and its rcb + if hasattr(obj, "__script_if_tracing_wrapper"): + obj = obj.__original_fn # type: ignore[union-attr] + _rcb = _jit_internal.createResolutionCallbackFromClosure(obj) + + # some functions are explicitly marked as not supported in script mode + if hasattr(obj, "__script_unsupported"): + raise RuntimeError("TorchScript error: " + obj.__script_unsupported) + + _check_directly_compile_overloaded(obj) + maybe_already_compiled_fn = _try_get_jit_cached_function(obj) + if maybe_already_compiled_fn: + maybe_already_compiled_fn._torchdynamo_inline = obj # type: ignore[attr-defined] + return maybe_already_compiled_fn + ast = get_jit_def(obj, obj.__name__) + if _rcb is None: + _rcb = _jit_internal.createResolutionCallbackFromClosure(obj) + fn = torch._C._jit_script_compile( + qualified_name, ast, _rcb, get_default_args(obj) + ) + # Forward docstrings + fn.__doc__ = obj.__doc__ + fn.__name__ = "ScriptFunction" + fn.__qualname__ = "torch.jit.ScriptFunction" + # Allow torch.compile() to inline + fn._torchdynamo_inline = obj # type: ignore[attr-defined] + _set_jit_function_cache(obj, fn) + return fn + else: + return torch.jit._recursive.create_script_class(obj) + + +def script( + obj, + optimize=None, + _frames_up=0, + _rcb=None, + example_inputs: Union[list[tuple], dict[Callable, list[tuple]], None] = None, +): + r"""Script the function. + + Scripting a function or ``nn.Module`` will inspect the source code, compile + it as TorchScript code using the TorchScript compiler, and return a :class:`ScriptModule` or + :class:`ScriptFunction`. TorchScript itself is a subset of the Python language, so not all + features in Python work, but we provide enough functionality to compute on + tensors and do control-dependent operations. For a complete guide, see the + :ref:`language-reference`. + + Scripting a dictionary or list copies the data inside it into a TorchScript instance than can be + subsequently passed by reference between Python and TorchScript with zero copy overhead. + + ``torch.jit.script`` can be used as a function for modules, functions, dictionaries and lists + and as a decorator ``@torch.jit.script`` for torchscript-classes and functions. + + Args: + obj (Callable, class, or nn.Module): The ``nn.Module``, function, class type, + dictionary, or list to compile. + example_inputs (Union[List[Tuple], Dict[Callable, List[Tuple]], None]): Provide example inputs + to annotate the arguments for a function or ``nn.Module``. + + Returns: + If ``obj`` is ``nn.Module``, ``script`` returns + a :class:`ScriptModule` object. The returned :class:`ScriptModule` will + have the same set of sub-modules and parameters as the + original ``nn.Module``. If ``obj`` is a standalone function, + a :class:`ScriptFunction` will be returned. If ``obj`` is a ``dict``, then + ``script`` returns an instance of `torch._C.ScriptDict`. If ``obj`` is a ``list``, + then ``script`` returns an instance of `torch._C.ScriptList`. + + **Scripting a function** + The ``@torch.jit.script`` decorator will construct a :class:`ScriptFunction` + by compiling the body of the function. + + Example (scripting a function): + + .. testcode:: + + import torch + + @torch.jit.script + def foo(x, y): + if x.max() > y.max(): + r = x + else: + r = y + return r + + print(type(foo)) # torch.jit.ScriptFunction + + # See the compiled graph as Python code + print(foo.code) + + # Call the function using the TorchScript interpreter + foo(torch.ones(2, 2), torch.ones(2, 2)) + + .. testoutput:: + :hide: + + ... + + ****Scripting a function using example_inputs** + Example inputs can be used to annotate a function arguments. + + Example (annotating a function before scripting): + + .. testcode:: + + import torch + + def test_sum(a, b): + return a + b + + # Annotate the arguments to be int + scripted_fn = torch.jit.script(test_sum, example_inputs=[(3, 4)]) + + print(type(scripted_fn)) # torch.jit.ScriptFunction + + # See the compiled graph as Python code + print(scripted_fn.code) + + # Call the function using the TorchScript interpreter + scripted_fn(20, 100) + + .. testoutput:: + :hide: + + ... + + **Scripting an nn.Module** + Scripting an ``nn.Module`` by default will compile the ``forward`` method and recursively + compile any methods, submodules, and functions called by ``forward``. If a ``nn.Module`` only uses + features supported in TorchScript, no changes to the original module code should be necessary. ``script`` + will construct :class:`ScriptModule` that has copies of the attributes, parameters, and methods of + the original module. + + Example (scripting a simple module with a Parameter): + + .. testcode:: + + import torch + + class MyModule(torch.nn.Module): + def __init__(self, N, M): + super().__init__() + # This parameter will be copied to the new ScriptModule + self.weight = torch.nn.Parameter(torch.rand(N, M)) + + # When this submodule is used, it will be compiled + self.linear = torch.nn.Linear(N, M) + + def forward(self, input): + output = self.weight.mv(input) + + # This calls the `forward` method of the `nn.Linear` module, which will + # cause the `self.linear` submodule to be compiled to a `ScriptModule` here + output = self.linear(output) + return output + + scripted_module = torch.jit.script(MyModule(2, 3)) + + Example (scripting a module with traced submodules): + + .. testcode:: + + import torch + import torch.nn as nn + import torch.nn.functional as F + + class MyModule(nn.Module): + def __init__(self) -> None: + super().__init__() + # torch.jit.trace produces a ScriptModule's conv1 and conv2 + self.conv1 = torch.jit.trace(nn.Conv2d(1, 20, 5), torch.rand(1, 1, 16, 16)) + self.conv2 = torch.jit.trace(nn.Conv2d(20, 20, 5), torch.rand(1, 20, 16, 16)) + + def forward(self, input): + input = F.relu(self.conv1(input)) + input = F.relu(self.conv2(input)) + return input + + scripted_module = torch.jit.script(MyModule()) + + To compile a method other than ``forward`` (and recursively compile anything it calls), add + the :func:`@torch.jit.export ` decorator to the method. To opt out of compilation + use :func:`@torch.jit.ignore ` or :func:`@torch.jit.unused `. + + Example (an exported and ignored method in a module):: + + import torch + import torch.nn as nn + + + class MyModule(nn.Module): + def __init__(self) -> None: + super().__init__() + + @torch.jit.export + def some_entry_point(self, input): + return input + 10 + + @torch.jit.ignore + def python_only_fn(self, input): + # This function won't be compiled, so any + # Python APIs can be used + import pdb + + pdb.set_trace() + + def forward(self, input): + if self.training: + self.python_only_fn(input) + return input * 99 + + + scripted_module = torch.jit.script(MyModule()) + print(scripted_module.some_entry_point(torch.randn(2, 2))) + print(scripted_module(torch.randn(2, 2))) + + Example ( Annotating forward of nn.Module using example_inputs):: + + import torch + import torch.nn as nn + from typing import NamedTuple + + class MyModule(NamedTuple): + result: List[int] + + class TestNNModule(torch.nn.Module): + def forward(self, a) -> MyModule: + result = MyModule(result=a) + return result + + pdt_model = TestNNModule() + + # Runs the pdt_model in eager model with the inputs provided and annotates the arguments of forward + scripted_model = torch.jit.script(pdt_model, example_inputs={pdt_model: [([10, 20, ], ), ], }) + + # Run the scripted_model with actual inputs + print(scripted_model([20])) + """ + if sys.version_info >= (3, 14): + warnings.warn( + "`torch.jit.script` is not supported in Python 3.14+ and may break. " + "Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + else: + warnings.warn( + "`torch.jit.script` is deprecated. Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + if not _enabled: + return obj + try: + global _TOPLEVEL + prev = _TOPLEVEL + _TOPLEVEL = False + ret = _script_impl( + obj=obj, + optimize=optimize, + _frames_up=_frames_up + 1, + _rcb=_rcb, + example_inputs=example_inputs, + ) + + if prev: + log_torchscript_usage("script", model_id=_get_model_id(ret)) + + return ret + finally: + _TOPLEVEL = prev + + +# overloads are registered in _jit_internal and compiled here so that _overload +# can be used in nn/functional.py without an import cycle + + +def _check_overload_defaults(impl_defaults, overload_defaults, loc): + for name, overload_value in overload_defaults.items(): + if name not in impl_defaults or impl_defaults[name] != overload_value: + raise torch.jit.frontend.FrontendError( + loc, + "Default parameters on overloads do not affect the runtime so they " + "must equal to the default parameter on the implementation function. Found on " + f"parameter {name}", + ) + + +def _compile_function_with_overload(overload_fn, qual_name, impl_fn): + overload_decl = get_jit_def(overload_fn, overload_fn.__name__).decl() + overload_signature = torch.jit.annotations.get_signature( + overload_fn, None, None, inspect.ismethod(overload_fn) + ) + impl_ast = get_jit_def(impl_fn, impl_fn.__name__) + overload_defaults = get_default_args(overload_fn) + implementation_defaults = get_default_args(impl_fn) + _rcb = _jit_internal.createResolutionCallbackFromClosure(impl_fn) + _check_overload_defaults( + implementation_defaults, overload_defaults, overload_decl.range() + ) + fn = torch._C._jit_script_compile_overload( + qual_name, + overload_decl, + impl_ast, + _rcb, + implementation_defaults, + overload_signature, + ) + return fn + + +def _get_overloads(obj): + # check for cached compiled fns + existing_compiled_fns = _try_get_jit_cached_overloads(obj) + qual_name = _qualified_name(obj) + uncompiled_overloads = _jit_internal._get_fn_overloads(qual_name) + if uncompiled_overloads is None: + return existing_compiled_fns + + if obj in uncompiled_overloads: + raise RuntimeError( + _jit_internal.get_overload_no_implementation_error_message("function", obj) + ) + + compiled_fns = [ + _compile_function_with_overload(overload_fn, qual_name, obj) + for overload_fn in uncompiled_overloads + ] + + if existing_compiled_fns: + compiled_fns = existing_compiled_fns + compiled_fns + + # cache compilation, remove information stored to do compilation + _set_jit_overload_cache(obj, compiled_fns) + _jit_internal._clear_fn_overloads(qual_name) + return compiled_fns + + +def _check_directly_compile_overloaded(obj): + qual_name = _qualified_name(obj) + if _jit_internal._get_fn_overloads(qual_name) or _try_get_jit_cached_overloads(obj): + raise RuntimeError( + f"Function {qual_name} cannot be directly compiled because it" + " is overloaded. It must be used in a context of a function" + " where its inputs can determine which overload to call." + ) + + +def interface(obj): + r"""Decorate to annotate classes or modules of different types. + + This decorator can be used to define an interface that can be used to annotate + classes or modules of different types. This can be used for to annotate a submodule + or attribute class that could have different types that implement the same + interface, or which could be swapped at runtime; or to store a list of modules or + classes of varying types. + + It is sometimes used to implement "Callables" - functions or modules that implement + an interface but whose implementations differ and which can be swapped out. + + Example: + .. testcode:: + + import torch + from typing import List + + @torch.jit.interface + class InterfaceType: + def run(self, x: torch.Tensor) -> torch.Tensor: + pass + + # implements InterfaceType + @torch.jit.script + class Impl1: + def run(self, x: torch.Tensor) -> torch.Tensor: + return x.relu() + + class Impl2(torch.nn.Module): + def __init__(self) -> None: + super().__init__() + self.val = torch.rand(()) + + @torch.jit.export + def run(self, x: torch.Tensor) -> torch.Tensor: + return x + self.val + + def user_fn(impls: List[InterfaceType], idx: int, val: torch.Tensor) -> torch.Tensor: + return impls[idx].run(val) + + user_fn_jit = torch.jit.script(user_fn) + + impls = [Impl1(), torch.jit.script(Impl2())] + val = torch.rand(4, 4) + user_fn_jit(impls, 0, val) + user_fn_jit(impls, 1, val) + """ + if not inspect.isclass(obj): + raise RuntimeError("interface must be applied to a class") + if not _is_new_style_class(obj): + raise RuntimeError("TorchScript interfaces must inherit from 'object'") + + # Expected MRO is: + # User module + # torch.nn.modules.module.Module + # object + is_module_interface = issubclass(obj, torch.nn.Module) and len(obj.mro()) == 3 + + if not is_module_interface and len(obj.mro()) > 2: + raise RuntimeError( + "TorchScript interface does not support inheritance yet. " + "Please directly inherit from 'object' or 'nn.Module'." + ) + + qualified_name = _qualified_name(obj) + rcb = _jit_internal.createResolutionCallbackFromFrame(1) + # if this type is a `nn.Module` subclass, generate a module interface type + # instead of a class interface type; a module interface type only compiles + # the user provided methods as part of the interface + ast = get_jit_class_def(obj, obj.__name__) + mangled_classname = torch._C._jit_script_interface_compile( + qualified_name, ast, rcb, is_module_interface + ) + obj.__torch_script_interface__ = mangled_classname + return obj + + +def _recursive_compile_class(obj, loc): + _qual_name = _qualified_name(obj) + # We're starting a new compilation, so update the error call stack in + # case it fails + error_stack = torch._C.CallStack(_qual_name, loc) # noqa: F841 + rcb = _jit_internal.createResolutionCallbackForClassMethods(obj) + return _compile_and_register_class(obj, rcb, _qual_name) + + +CompilationUnit = torch._C.CompilationUnit +set_module(CompilationUnit, "torch.jit") + + +def pad(s: str, padding: int, offset: int = 0, char: str = " "): + if padding >= len(s): + padding -= len(s) + return "".join([char for _ in range(padding + offset)]) + s + + +class _ScriptProfileColumn: + def __init__(self, header: str, alignment: int = 4, offset: int = 0): + self.header = header + self.alignment = alignment + self.offset = offset + self.rows: dict[int, Any] = {} + + def add_row(self, lineno: int, value: Any): + self.rows[lineno] = value + + def materialize(self): + max_length = len(self.header) + rows: list[tuple[int, str]] = [] + for key, value in self.rows.items(): + cell = str(value) + rows.append((key, cell)) + max_length = max(len(cell), max_length) + + if self.alignment > 0: + padding = max_length + self.alignment + padding -= padding % self.alignment + else: + padding = 0 + + rows = [(key, pad(cell, padding, self.offset)) for key, cell in rows] + return pad(self.header, padding, self.offset), rows + + +class _ScriptProfileTable: + def __init__(self, cols: list[_ScriptProfileColumn], source_range: list[int]): + self.cols = cols + self.source_range = source_range + + def dump_string(self): + outputs: list[str] = [] + cells: list[tuple[str, dict[int, str]]] = [] + header_buffer = "" + for col in self.cols: + header, rows = col.materialize() + header_buffer += header + cells.append((header, dict(rows))) + + outputs.append(header_buffer) + outputs.append(pad("", len(header_buffer), 0, "=")) + for line in self.source_range: + row_buffer = "" + for header, rows in cells: + cell = rows.get(line) + if cell is None: + row_buffer += pad("", len(header)) + else: + row_buffer += cell + outputs.append(row_buffer) + return "\n".join(outputs) + + +class _ScriptProfile: + def __init__(self) -> None: + self.profile = classes.profiling._ScriptProfile() + + def enable(self): + self.profile.enable() + + def disable(self): + self.profile.disable() + + def dump_string(self) -> str: + outputs: list[str] = [] + for source_stats in self.profile._dump_stats(): + source_ref = source_stats.source() + source_lines = source_ref.text().splitlines() + dedent = min(len(line) - len(line.lstrip(" ")) for line in source_lines) + source_lines = [line[dedent:] for line in source_lines] + + start_line = source_ref.starting_lineno() + end_line = start_line + len(source_lines) + source_range = range(start_line, end_line) + lineno = _ScriptProfileColumn("Line #") + hits = _ScriptProfileColumn("Hits") + time_ns = _ScriptProfileColumn("Time (ns)") + line_contents = _ScriptProfileColumn("Line Contents", 0, 1) + stats = source_stats.line_map() + for line in source_range: + lineno.add_row(line, line) + line_contents.add_row(line, source_lines[line - start_line]) + stat = stats.get(line) + if stat is not None: + hits.add_row(line, stat.count()) + time_ns.add_row(line, stat.duration_ns()) + + table = _ScriptProfileTable( + [lineno, hits, time_ns, line_contents], list(source_range) + ) + outputs.append(table.dump_string()) + return "\n\n".join(outputs) + + def dump(self): + print(self.dump_string()) + + +def _unwrap_optional(x): + assert x is not None, "Unwrapping null optional" + return x + + +_register_builtin(_unwrap_optional, "aten::_unwrap_optional") +_register_builtin(_jit_internal.is_scripting, "aten::is_scripting") +_register_builtin(has_torch_function, "aten::has_torch_function") +_register_builtin(has_torch_function_unary, "aten::has_torch_function") +_register_builtin(has_torch_function_variadic, "aten::has_torch_function") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_script.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_script.pyi new file mode 100644 index 0000000000000000000000000000000000000000..7d3a5de62a969d74886dd7eefe211cbe5d5f6547 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_script.pyi @@ -0,0 +1,297 @@ +# mypy: allow-untyped-defs +# mypy: disable-error-code="type-arg" +from collections.abc import Callable +from typing import Any, NamedTuple, overload, TypeAlias, TypeVar +from typing_extensions import Never + +from _typeshed import Incomplete + +import torch +from torch._classes import classes as classes +from torch._jit_internal import _qualified_name as _qualified_name +from torch.jit._builtins import _register_builtin as _register_builtin +from torch.jit._fuser import ( + _graph_for as _graph_for, + _script_method_graph_for as _script_method_graph_for, +) +from torch.jit._monkeytype_config import ( + JitTypeTraceConfig as JitTypeTraceConfig, + JitTypeTraceStore as JitTypeTraceStore, + monkeytype_trace as monkeytype_trace, +) +from torch.jit._recursive import ( + _compile_and_register_class as _compile_and_register_class, + infer_methods_to_compile as infer_methods_to_compile, + ScriptMethodStub as ScriptMethodStub, + wrap_cpp_module as wrap_cpp_module, +) +from torch.jit._serialization import validate_map_location as validate_map_location +from torch.jit._state import ( + _enabled as _enabled, + _set_jit_function_cache as _set_jit_function_cache, + _set_jit_overload_cache as _set_jit_overload_cache, + _try_get_jit_cached_function as _try_get_jit_cached_function, + _try_get_jit_cached_overloads as _try_get_jit_cached_overloads, +) +from torch.jit.frontend import ( + get_default_args as get_default_args, + get_jit_class_def as get_jit_class_def, + get_jit_def as get_jit_def, +) +from torch.nn import Module as Module +from torch.overrides import ( + has_torch_function as has_torch_function, + has_torch_function_unary as has_torch_function_unary, + has_torch_function_variadic as has_torch_function_variadic, +) +from torch.package import ( + PackageExporter as PackageExporter, + PackageImporter as PackageImporter, +) +from torch.utils import set_module as set_module + +ScriptFunction = torch._C.ScriptFunction + +type_trace_db: JitTypeTraceStore + +# Defined in torch/csrc/jit/python/script_init.cpp +ResolutionCallback: TypeAlias = Callable[[str], Callable[..., Any]] +_ClassVar = TypeVar("_ClassVar", bound=type) + +def _reduce(cls) -> None: ... + +class Attribute(NamedTuple): + value: Incomplete + type: Incomplete + +def _get_type_trace_db(): ... +def _get_function_from_type(cls, name): ... +def _is_new_style_class(cls): ... + +class OrderedDictWrapper: + _c: Incomplete + def __init__(self, _c) -> None: ... + def keys(self): ... + def values(self): ... + def __len__(self) -> int: ... + def __delitem__(self, k) -> None: ... + def items(self): ... + def __setitem__(self, k, v) -> None: ... + def __contains__(self, k) -> bool: ... + def __getitem__(self, k): ... + +class OrderedModuleDict(OrderedDictWrapper): + _python_modules: Incomplete + def __init__(self, module, python_dict) -> None: ... + def items(self): ... + def __contains__(self, k) -> bool: ... + def __setitem__(self, k, v) -> None: ... + def __getitem__(self, k): ... + +class ScriptMeta(type): + def __init__(cls, name, bases, attrs) -> None: ... + +class _CachedForward: + def __get__(self, obj, cls): ... + +class ScriptWarning(Warning): ... + +def script_method(fn): ... + +class ConstMap: + const_mapping: Incomplete + def __init__(self, const_mapping) -> None: ... + def __getattr__(self, attr): ... + +def unpackage_script_module( + importer: PackageImporter, + script_module_id: str, +) -> torch.nn.Module: ... + +_magic_methods: Incomplete + +class RecursiveScriptClass: + _c: Incomplete + _props: Incomplete + def __init__(self, cpp_class) -> None: ... + def __getattr__(self, attr): ... + def __setattr__(self, attr, value) -> None: ... + def forward_magic_method(self, method_name, *args, **kwargs): ... + def __getstate__(self) -> None: ... + def __iadd__(self, other): ... + +def method_template(self, *args, **kwargs): ... + +class ScriptModule(Module, metaclass=ScriptMeta): + __jit_unused_properties__: Incomplete + def __init__(self) -> None: ... + forward: Callable[..., Any] + def __getattr__(self, attr): ... + def __setattr__(self, attr, value) -> None: ... + def define(self, src): ... + def _replicate_for_data_parallel(self): ... + def __reduce_package__(self, exporter: PackageExporter): ... + # add __jit_unused_properties__ + @property + def code(self) -> str: ... + @property + def code_with_constants(self) -> tuple[str, ConstMap]: ... + @property + def graph(self) -> torch.Graph: ... + @property + def inlined_graph(self) -> torch.Graph: ... + @property + def original_name(self) -> str: ... + +class RecursiveScriptModule(ScriptModule): + _disable_script_meta: bool + _c: Incomplete + def __init__(self, cpp_module) -> None: ... + @staticmethod + def _construct(cpp_module, init_fn): ... + @staticmethod + def _finalize_scriptmodule(script_module) -> None: ... + _concrete_type: Incomplete + _modules: Incomplete + _parameters: Incomplete + _buffers: Incomplete + __dict__: Incomplete + def _reconstruct(self, cpp_module) -> None: ... + def save(self, f, **kwargs): ... + def _save_for_lite_interpreter(self, *args, **kwargs): ... + def _save_to_buffer_for_lite_interpreter(self, *args, **kwargs): ... + def save_to_buffer(self, *args, **kwargs): ... + def get_debug_state(self, *args, **kwargs): ... + def extra_repr(self): ... + def graph_for(self, *args, **kwargs): ... + def define(self, src) -> None: ... + def __getattr__(self, attr): ... + def __setattr__(self, attr, value) -> None: ... + def __copy__(self): ... + def __deepcopy__(self, memo): ... + def forward_magic_method(self, method_name, *args, **kwargs): ... + def __iter__(self): ... + def __getitem__(self, idx): ... + def __len__(self) -> int: ... + def __contains__(self, key) -> bool: ... + def __dir__(self): ... + def __bool__(self) -> bool: ... + def _replicate_for_data_parallel(self): ... + +def _get_methods(cls): ... + +_compiled_methods_allowlist: Incomplete + +def _make_fail(name): ... +def call_prepare_scriptable_func_impl(obj, memo): ... +def call_prepare_scriptable_func(obj): ... +def create_script_dict(obj): ... +def create_script_list(obj, type_hint: Incomplete | None = ...): ... +@overload +def script( + obj: type[Module], + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> Never: ... +@overload +def script( + obj: dict, + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> torch.ScriptDict: ... +@overload +def script( + obj: list, + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> torch.ScriptList: ... +@overload +def script( # type: ignore[overload-overlap] + obj: Module, + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> RecursiveScriptModule: ... +@overload +def script( # type: ignore[overload-overlap] + obj: _ClassVar, + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> _ClassVar: ... +@overload +def script( + obj: Callable, + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> ScriptFunction: ... +@overload +def script( + obj: Any, + optimize: bool | None = None, + _frames_up: int = 0, + _rcb: ResolutionCallback | None = None, + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = None, +) -> RecursiveScriptClass: ... +@overload +def script( + obj, + optimize: Incomplete | None = ..., + _frames_up: int = ..., + _rcb: Incomplete | None = ..., + example_inputs: list[tuple] | dict[Callable, list[tuple]] | None = ..., +): ... +def _check_overload_defaults(impl_defaults, overload_defaults, loc) -> None: ... +def _compile_function_with_overload(overload_fn, qual_name, impl_fn): ... +def _get_overloads(obj): ... +def _check_directly_compile_overloaded(obj) -> None: ... +def interface(obj): ... +def _recursive_compile_class(obj, loc): ... + +CompilationUnit: Incomplete + +def pad(s: str, padding: int, offset: int = ..., char: str = ...): ... + +class _ScriptProfileColumn: + header: Incomplete + alignment: Incomplete + offset: Incomplete + rows: Incomplete + def __init__( + self, + header: str, + alignment: int = ..., + offset: int = ..., + ) -> None: ... + def add_row(self, lineno: int, value: Any): ... + def materialize(self): ... + +class _ScriptProfileTable: + cols: Incomplete + source_range: Incomplete + def __init__( + self, + cols: list[_ScriptProfileColumn], + source_range: list[int], + ) -> None: ... + def dump_string(self): ... + +class _ScriptProfile: + profile: Incomplete + def __init__(self) -> None: ... + def enable(self) -> None: ... + def disable(self) -> None: ... + def dump_string(self) -> str: ... + def dump(self) -> None: ... + +def _unwrap_optional(x): ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_serialization.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_serialization.py new file mode 100644 index 0000000000000000000000000000000000000000..41490aebd2c49d8db53a7686e015116797e7ec7d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_serialization.py @@ -0,0 +1,315 @@ +# mypy: allow-untyped-defs +"""Serialization. + +This module contains functionality for serializing TorchScript modules, notably: + * torch.jit.save + * torch.jit.load + +This is not intended to be imported directly; please use the exposed +functionalities in `torch.jit`. +""" + +import os +import sys +import warnings + +import torch +from torch._jit_internal import _get_model_id +from torch._utils_internal import log_torchscript_usage +from torch.jit._recursive import wrap_cpp_module +from torch.serialization import validate_cuda_device + + +def save(m, f, _extra_files=None) -> None: + r""" + Save an offline version of this module for use in a separate process. + + The saved module serializes all of the methods, submodules, parameters, and + attributes of this module. It can be loaded into the C++ API using + ``torch::jit::load(filename)`` or into the Python API with + :func:`torch.jit.load `. + + To be able to save a module, it must not make any calls to native Python + functions. This means that all submodules must be subclasses of + :class:`ScriptModule` as well. + + .. DANGER:: + All modules, no matter their device, are always loaded onto the CPU + during loading. This is different from :func:`torch.load`'s semantics + and may change in the future. + + Args: + m: A :class:`ScriptModule` to save. + f: A file-like object (has to implement write and flush) or a string + containing a file name. + _extra_files: Map from filename to contents which will be stored as part of `f`. + + .. note:: + torch.jit.save attempts to preserve the behavior of some operators + across versions. For example, dividing two integer tensors in + PyTorch 1.5 performed floor division, and if the module + containing that code is saved in PyTorch 1.5 and loaded in PyTorch 1.6 + its division behavior will be preserved. The same module saved in + PyTorch 1.6 will fail to load in PyTorch 1.5, however, since the + behavior of division changed in 1.6, and 1.5 does not know how to + replicate the 1.6 behavior. + + Example: + .. testcode:: + + import torch + import io + + class MyModule(torch.nn.Module): + def forward(self, x): + return x + 10 + + m = torch.jit.script(MyModule()) + + # Save to file + torch.jit.save(m, 'scriptmodule.pt') + # This line is equivalent to the previous + m.save("scriptmodule.pt") + + # Save to io.BytesIO buffer + buffer = io.BytesIO() + torch.jit.save(m, buffer) + + # Save with extra files + extra_files = {'foo.txt': b'bar'} + torch.jit.save(m, 'scriptmodule.pt', _extra_files=extra_files) + """ + if sys.version_info >= (3, 14): + warnings.warn( + "`torch.jit.save` is not supported in Python 3.14+ and may break. " + "Please switch to `torch.export`.", + DeprecationWarning, + ) + else: + warnings.warn( + "`torch.jit.save` is deprecated. Please switch to `torch.export`.", + DeprecationWarning, + ) + log_torchscript_usage("save", model_id=_get_model_id(m)) + if _extra_files is None: + _extra_files = {} + if isinstance(f, (str, os.PathLike)): + m.save(f, _extra_files=_extra_files) + else: + ret = m.save_to_buffer(_extra_files=_extra_files) + f.write(ret) + + +def load(f, map_location=None, _extra_files=None, _restore_shapes=False): + r""" + Load a :class:`ScriptModule` or :class:`ScriptFunction` previously saved with :func:`torch.jit.save `. + + All previously saved modules, no matter their device, are first loaded onto CPU, + and then are moved to the devices they were saved from. If this fails (e.g. + because the run time system doesn't have certain devices), an exception is + raised. + + Args: + f: a file-like object (has to implement read, readline, tell, and seek), + or a string containing a file name + map_location (string or torch.device): A simplified version of + ``map_location`` in `torch.jit.save` used to dynamically remap + storages to an alternative set of devices. + _extra_files (dictionary of filename to content): The extra + filenames given in the map would be loaded and their content + would be stored in the provided map. + _restore_shapes (bool): Whether or not to retrace the module on load using stored inputs + + Returns: + A :class:`ScriptModule` object. + + .. warning:: + It is possible to construct malicious pickle data which will execute arbitrary code + during func:`torch.jit.load`. Never load data that could have come from an untrusted + source, or that could have been tampered with. **Only load data you trust**. + + Example: + .. testcode:: + + import torch + import io + + torch.jit.load('scriptmodule.pt') + + # Load ScriptModule from io.BytesIO object + with open('scriptmodule.pt', 'rb') as f: + buffer = io.BytesIO(f.read()) + + # Load all tensors to the original device + torch.jit.load(buffer) + + # Load all tensors onto CPU, using a device + buffer.seek(0) + torch.jit.load(buffer, map_location=torch.device('cpu')) + + # Load all tensors onto CPU, using a string + buffer.seek(0) + torch.jit.load(buffer, map_location='cpu') + + # Load with extra files. + extra_files = {'foo.txt': ''} # values will be replaced with data + torch.jit.load('scriptmodule.pt', _extra_files=extra_files) + print(extra_files['foo.txt']) + + .. testoutput:: + :hide: + + ... + + .. testcleanup:: + + import os + os.remove("scriptmodule.pt") + """ + if sys.version_info >= (3, 14): + warnings.warn( + "`torch.jit.load` is not supported in Python 3.14+ and may break. " + "Please switch to `torch.export`.", + DeprecationWarning, + ) + else: + warnings.warn( + "`torch.jit.load` is deprecated. Please switch to `torch.export`.", + DeprecationWarning, + ) + if isinstance(f, (str, os.PathLike)): + if not os.path.exists(f): + raise ValueError(f"The provided filename {f} does not exist") + if os.path.isdir(f): + raise ValueError(f"The provided filename {f} is a directory") + + map_location = validate_map_location(map_location) + if _extra_files is None: + _extra_files = {} + + cu = torch._C.CompilationUnit() + if isinstance(f, (str, os.PathLike)): + cpp_module = torch._C.import_ir_module( + cu, + os.fspath(f), + map_location, + _extra_files, + # pyrefly: ignore [bad-argument-count] + _restore_shapes, + ) # type: ignore[call-arg] + else: + cpp_module = torch._C.import_ir_module_from_buffer( + cu, + # pyrefly: ignore [missing-attribute] + f.read(), + map_location, + _extra_files, + # pyrefly: ignore [bad-argument-count] + _restore_shapes, + ) # type: ignore[call-arg] + + # TODO: Pretty sure this approach loses ConstSequential status and such + ret = wrap_cpp_module(cpp_module) + log_torchscript_usage("load", model_id=_get_model_id(ret)) + return ret + + +def validate_map_location(map_location=None): + if isinstance(map_location, str): + map_location = torch.device(map_location) + elif not (map_location is None or isinstance(map_location, torch.device)): + raise ValueError( + "map_location should be either None, string or torch.device, " + "but got type: " + str(type(map_location)) + ) + + if str(map_location).startswith("cuda"): + validate_cuda_device(map_location) + + return map_location + + +def jit_module_from_flatbuffer(f): + if isinstance(f, (str, os.PathLike)): + f = os.fspath(f) + return wrap_cpp_module(torch._C._load_jit_module_from_file(f)) + else: + return wrap_cpp_module(torch._C._load_jit_module_from_bytes(f.read())) + + +def save_jit_module_to_flatbuffer(m, f, _extra_files=None) -> None: + r""" + Save an offline version of this module for use in a separate process. + + The saved module serializes all of the methods, submodules, parameters, and + attributes of this module. It can be loaded into the C++ API using + ``torch::jit::load_jit_module_from_file(filename)`` or into the Python API with + :func:`torch.jit.jit_module_from_flatbuffer`. + + To be able to save a module, it must not make any calls to native Python + functions. This means that all submodules must be subclasses of + :class:`ScriptModule` as well. + + .. DANGER:: + All modules, no matter their device, are always loaded onto the CPU + during loading. This is different from :func:`torch.load`'s semantics + and may change in the future. + + Args: + m: A :class:`ScriptModule` to save. + f: A string for file path + + + Example: + .. testcode:: + + import torch + import io + + class MyModule(torch.nn.Module): + def forward(self, x): + return x + 10 + + m = torch.jit.script(MyModule()) + + # Save to file + torch.jit.save_jit_module_to_flatbuffer(m, 'scriptmodule.ff') + """ + extra_files = _extra_files + if extra_files is None: + extra_files = {} + + if isinstance(f, (str, os.PathLike)): + f = os.fspath(f) + torch._C._save_jit_module(m._c, f, extra_files) + else: + s = torch._C._save_jit_module_to_bytes(m._c, extra_files) + f.write(s) + + +def get_flatbuffer_module_info(path_or_file): + r"""Get some information regarding a model file in flatbuffer format. + + Args: + path_or_file: Either str, Path or file like object (BytesIO OK). + If it's str or Path, we will read the file referenced by that + path as Bytes. + + Returns: + A dict with metadata on what that file contains, currently looks like + this: + { + 'bytecode_version': 4, # int + 'operator_version': 4, # int + 'function_names': { + '__torch__.___torch_mangle_0.Foo.forward'}, # set + 'type_names': set(), # set + 'opname_to_num_args': {'aten::linear': 3} # Dict[str, int] + } + """ + if isinstance(path_or_file, (str, os.PathLike)): + with open(path_or_file, "rb") as f: + all_bytes = f.read() + else: + all_bytes = path_or_file.read() + return torch._C._get_module_info_from_flatbuffer(all_bytes) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_shape_functions.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_shape_functions.py new file mode 100644 index 0000000000000000000000000000000000000000..1f95de46f6f2aba51c9f48ba8716d8910baec3b8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_shape_functions.py @@ -0,0 +1,1475 @@ +# mypy: allow-untyped-defs +import math +from collections.abc import Callable +from typing import Any, Optional, Union + + +number = Union[int, float] +# flake8: noqa + +### +# There are generated files that depend on this file +# To re-generate, please run from the root of the repo: +# python torchgen/shape_functions/gen_jit_shape_functions.py + +# How to test: +# After regenerating files, compile PyTorch. +# Then run: ./build/bin/test_jit --gtest_filter=TestShapeGraphLinting.Basic +# If you have enabled opinfo testing for the op, also run: +# python test/test_ops_jit.py TestJitCPU.test_variant_consistency_jit_[FAILING_OP]_cpu_float32 +# to reproduce errors from opinfo tests. + +# Example PR: https://github.com/pytorch/pytorch/pull/80860/files +#### + +import torch + + +def broadcast(a: list[int], b: list[int]): + dimsA = len(a) + dimsB = len(b) + ndim = max(dimsA, dimsB) + expandedSizes: list[int] = [] + + for i in range(ndim): + offset = ndim - 1 - i + dimA = dimsA - 1 - offset + dimB = dimsB - 1 - offset + sizeA = a[dimA] if (dimA >= 0) else 1 + sizeB = b[dimB] if (dimB >= 0) else 1 + + if sizeA != sizeB and sizeA != 1 and sizeB != 1: + # TODO: only assertion error is bound in C++ compilation right now + raise AssertionError( + f"The size of tensor a {sizeA} must match the size of tensor b ({sizeB}) at non-singleton dimension {i}" + ) + + expandedSizes.append(sizeB if sizeA == 1 else sizeA) + + return expandedSizes + + +def broadcast_three(a: list[int], b: list[int], c: list[int]): + return broadcast(broadcast(a, b), c) + + +def broadcast_one_three(a: list[int], b: Any, c: list[int]): + return broadcast(a, c) + + +def adaptive_avg_pool2d(self: list[int], out: list[int]): + assert len(out) == 2 + assert len(self) == 3 or len(self) == 4 + for i in range(1, len(self)): + assert self[i] != 0 + + shape: list[int] = [] + for i in range(0, len(self) - 2): + shape.append(self[i]) + for elem in out: + shape.append(elem) + return shape + + +def _copy(self: list[int]): + out: list[int] = [] + for elem in self: + out.append(elem) + return out + + +def unary(self: list[int]): + return _copy(self) + + +def broadcast_inplace(a: list[int], b: list[int]): + dimsA = len(a) + dimsB = len(b) + if dimsB > dimsA: + raise AssertionError( + f"The dims of tensor b ({dimsB}) must be less than or equal to the dims of tensor a ({dimsA}) " + ) + for dimA in range(dimsA): + dimB = dimsB - dimsA + dimA + sizeA = a[dimA] + sizeB = b[dimB] if (dimB >= 0) else 1 + if sizeA != sizeB and sizeB != 1: + # TODO: only assertion error is bound in C++ compilation right now + raise AssertionError( + "The size of tensor a {} must match the size of tensor b (" + "{}) at non-singleton dimension {}".format(sizeA, sizeB, dimA) + ) + return _copy(a) + + +def expand(self: list[int], sizes: list[int]): + assert len(sizes) >= len(self) + ndim = len(sizes) + tensor_dim = len(self) + if ndim == 0: + return _copy(sizes) + out: list[int] = [] + for i in range(ndim): + offset = ndim - 1 - i + dim = tensor_dim - 1 - offset + size = self[dim] if dim >= 0 else 1 + targetSize = sizes[i] + if targetSize == -1: + assert dim >= 0 + targetSize = size + if size != targetSize: + assert size == 1 + size = targetSize + out.append(size) + return out + + +def expand_one_unused(self: list[int], sizes: list[int], inp0: Any): + return expand(self, sizes) + + +def infer_size_impl(shape: list[int], numel: int) -> list[int]: + newsize = 1 + infer_dim: Optional[int] = None + for dim in range(len(shape)): + if shape[dim] == -1: + if infer_dim is not None: + raise AssertionError("only one dimension can be inferred") + infer_dim = dim + elif shape[dim] >= 0: + newsize *= shape[dim] + else: + raise AssertionError("invalid shape dimensions") + if not ( + numel == newsize + or (infer_dim is not None and newsize > 0 and numel % newsize == 0) + ): + raise AssertionError("invalid shape") + out = _copy(shape) + if infer_dim is not None: + out[infer_dim] = numel // newsize + return out + + +def numel(sizes: list[int]): + numel = 1 + for elem in sizes: + numel *= elem + return numel + + +def view(self: list[int], sizes: list[int]): + return infer_size_impl(sizes, numel(self)) + + +def view_one_unused(self: list[int], sizes: list[int], *, implicit: bool = False): + return view(self, sizes) + + +def sum_mean_dim( + self: list[int], opt_dims: Optional[list[int]], keep_dim: bool, dt: Any +): + out: list[int] = [] + if opt_dims is None or len(opt_dims) == 0: + dims: list[int] = list(range(len(self))) + else: + dims = opt_dims + + for idx in range(len(self)): + is_mean_dim: bool = False + for reduce_dim in dims: + if idx == maybe_wrap_dim(reduce_dim, len(self)): + is_mean_dim = True + if is_mean_dim: + if keep_dim: + out.append(1) + else: + out.append(self[idx]) + return out + + +def max_dim(self: list[int], dim: int, keep_dim: bool): + out = sum_mean_dim(self, [dim], keep_dim, None) + return out, out + + +# note: python already rounds down towards negative infinity on integer division, special arithmetic not needed +def div_rtn(x: int, y: int): + return x // y + + +def pooling_output_shape_pad_lr( + inputSize: int, + kernelSize: int, + pad_l: int, + pad_r: int, + stride: int, + dilation: int, + ceil_mode: bool, +): + outputSize = ( + div_rtn( + inputSize + + pad_l + + pad_r + - dilation * (kernelSize - 1) + - 1 + + (stride - 1 if ceil_mode else 0), + stride, + ) + + 1 + ) + if ceil_mode: + if (outputSize - 1) * stride >= inputSize + pad_l: + outputSize = outputSize - 1 + return outputSize + + +def pooling_output_shape( + inputSize: int, + kernelSize: int, + pad_l: int, + stride: int, + dilation: int, + ceil_mode: bool, +): + assert stride != 0, "stride should not be zeero" + return pooling_output_shape_pad_lr( + inputSize, kernelSize, pad_l, pad_l, stride, dilation, ceil_mode + ) + + +def pool2d_shape_check( + input: list[int], + kH: int, + kW: int, + dH: int, + dW: int, + padH: int, + padW: int, + dilationH: int, + dilationW: int, + nInputPlane: int, + inputHeight: int, + inputWidth: int, + outputHeight: int, + outputWidth: int, +): + ndim = len(input) + + assert kW > 0 and kH > 0 + assert dW > 0 and dH > 0 + assert dilationH > 0 and dilationW > 0 + + valid_dims = input[1] != 0 and input[2] != 0 + assert ( + ndim == 3 + and input[0] != 0 + and valid_dims + or (ndim == 4 and valid_dims and input[3] != 0) + ) + + assert kW // 2 >= padW and kH // 2 >= padH + assert outputWidth >= 1 and outputHeight >= 1 + + +def max_pool2d( + input: list[int], + kernel_size: list[int], + stride: list[int], + padding: list[int], + dilation: list[int], + ceil_mode: bool, +): + assert len(kernel_size) == 1 or len(kernel_size) == 2, ( + "max_pool2d: kernel_size must either be a single int, or a tuple of two ints" + ) + kH = kernel_size[0] + kW = kH if len(kernel_size) == 1 else kernel_size[1] + + assert len(stride) == 0 or len(stride) == 1 or len(stride) == 2, ( + "max_pool2d: stride must either be omitted, a single int, or a tuple of two ints" + ) + dH = kH if len(stride) == 0 else stride[0] + if len(stride) == 0: + dW = kW + elif len(stride) == 1: + dW = dH + else: + dW = stride[1] + + assert len(padding) == 1 or len(padding) == 2, ( + "max_pool2d: padding must either be a single int, or a tuple of two ints" + ) + padH = padding[0] + padW = padH if len(padding) == 1 else padding[1] + + assert len(dilation) == 1 or len(dilation) == 2, ( + "max_pool2d: dilation must be either a single int, or a tuple of two ints" + ) + dilationH = dilation[0] + dilationW = dilationH if len(dilation) == 1 else dilation[1] + + assert len(input) == 3 or len(input) == 4 + + nbatch = input[-4] if len(input) == 4 else 1 + nInputPlane = input[-3] + inputHeight = input[-2] + inputWidth = input[-1] + + outputHeight = pooling_output_shape(inputHeight, kH, padH, dH, dilationH, ceil_mode) + outputWidth = pooling_output_shape(inputWidth, kW, padW, dW, dilationW, ceil_mode) + + pool2d_shape_check( + input, + kH, + kW, + dH, + dW, + padH, + padW, + dilationH, + dilationW, + nInputPlane, + inputHeight, + inputWidth, + outputHeight, + outputWidth, + ) + + if len(input) == 3: + return [nInputPlane, outputHeight, outputWidth] + else: + return [nbatch, nInputPlane, outputHeight, outputWidth] + + +def max_pool2d_with_indices( + input: list[int], + kernel_size: list[int], + stride: list[int], + padding: list[int], + dilation: list[int], + ceil_mode: bool, +): + out = max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode) + return (out, out) + + +def upsample_nearest2d( + input: list[int], + output_size: Optional[list[int]], + scale_factors: Optional[list[float]], +): + out: list[int] = [] + out.append(input[0]) + out.append(input[1]) + + if scale_factors is None and output_size is None: + assert 0, "Either output_size or scale_factors must be presented" + + if output_size is not None: + assert scale_factors is None, ( + "Must specify exactly one of output_size and scale_factors" + ) + assert len(output_size) == 2 + out.append(output_size[0]) + out.append(output_size[1]) + + if scale_factors is not None: + assert output_size is None, ( + "Must specify exactly one of output_size and scale_factors" + ) + assert len(scale_factors) == 2 + out.append(int(input[2] * scale_factors[0])) + out.append(int(input[3] * scale_factors[1])) + + return out + + +def mm(self: list[int], mat2: list[int]): + assert len(self) == 2, "self must be a matrix" + assert len(mat2) == 2, "mat2 must be a matrix" + + assert self[1] == mat2[0] + return [self[0], mat2[1]] + + +def dot(self: list[int], tensor: list[int]): + assert len(self) == 1 and len(tensor) == 1 + assert self[0] == tensor[0] + out: list[int] = [] + return out + + +def mv(self: list[int], vec: list[int]): + assert len(self) == 2 and len(vec) == 1 + assert self[1] == vec[0] + # TODO: return self + return [self[0]] + + +def unsqueeze(li: list[int], dim: int): + dim = maybe_wrap_dim(dim, len(li) + 1) + out = _copy(li) + out.insert(dim, 1) + return out + + +def squeeze_nodim(li: list[int]): + out: list[int] = [] + for i in range(len(li)): + if li[i] != 1: + out.append(li[i]) + return out + + +def squeeze(li: list[int], dim: int): + out: list[int] = [] + wrapped_dim = maybe_wrap_dim(dim, len(li)) + for i in range(len(li)): + if i == wrapped_dim: + if li[i] != 1: + out.append(li[i]) + else: + out.append(li[i]) + return out + + +def squeeze_dims(li: list[int], dims: list[int]): + if len(dims) == 0: + return li + wrapped_dims = _copy(dims) + for i in range(len(dims)): + wrapped_dims[i] = maybe_wrap_dim(wrapped_dims[i], len(li)) + result: list[int] = [] + for i in range(len(li)): + if li[i] == 1: + if i not in wrapped_dims: + result.append(li[i]) + else: + result.append(li[i]) + return result + + +def index_select(self: list[int], dim: int, index: list[int]): + dim = maybe_wrap_dim(dim, len(self)) + numel = multiply_integers(index) + assert len(index) <= 1 + assert dim == 0 or dim < len(self) + result_size: list[int] = [] + for i in range(len(self)): + if dim == i: + result_size.append(numel) + else: + result_size.append(self[i]) + return result_size + + +def embedding( + weight: list[int], + indices: list[int], + padding_idx: int = -1, + scale_grad_by_freq: bool = False, + sparse: bool = False, +): + assert len(weight) == 2 + if len(indices) == 1: + return index_select(weight, 0, indices) + size = _copy(indices) + size.append(weight[1]) + return size + + +def max_int(): + return 9223372036854775807 + + +def slice( + self: list[int], dim: int, start: Optional[int], end: Optional[int], step: int +): + ndim = len(self) + assert ndim != 0 + dim = maybe_wrap_dim(dim, ndim) + start_val = start if start is not None else 0 + end_val = end if end is not None else max_int() + assert step > 0 + if start_val == max_int(): + start_val = 0 + if start_val < 0: + start_val += self[dim] + if end_val < 0: + end_val += self[dim] + if start_val < 0: + start_val = 0 + elif start_val > self[dim]: + start_val = self[dim] + if end_val < start_val: + end_val = start_val + elif end_val >= self[dim]: + end_val = self[dim] + slice_len = end_val - start_val + out = _copy(self) + out[dim] = (slice_len + step - 1) // step + return out + + +def check_cat_no_zero_dim(tensors: list[list[int]]): + for tensor in tensors: + assert len(tensor) > 0 + + +def legacy_cat_wrap_dim(dim: int, tensor_sizes: list[list[int]]): + out_dim: Optional[int] = None + for size in tensor_sizes: + if not (len(size) == 1 and size[0] == 0): + if out_dim is None: + out_dim = maybe_wrap_dim(dim, len(size)) + if out_dim is None: + out_dim = dim + return out_dim + + +def should_skip(tensor: list[int]): + return numel(tensor) == 0 and len(tensor) == 1 + + +def check_cat_shape_except_dim( + first: list[int], second: list[int], dimension: int, index: int +): + first_dims = len(first) + second_dims = len(second) + assert first_dims == second_dims, "Tensors must have same number of dimensions" + for dim in range(0, first_dims): + if dim != dimension: + assert first[dim] == second[dim], ( + "Sizes of tensors must match except in dimension" + ) + + +def cat(tensors: list[list[int]], dim: int): + check_cat_no_zero_dim(tensors) + dim = legacy_cat_wrap_dim(dim, tensors) + assert len(tensors) > 0 + not_skipped_tensor: Optional[list[int]] = None + for tensor in tensors: + if not should_skip(tensor): + not_skipped_tensor = tensor + if not_skipped_tensor is None: + return [0] + + cat_dim_size = 0 + + for i in range(len(tensors)): + tensor = tensors[i] + if not should_skip(tensor): + check_cat_shape_except_dim(not_skipped_tensor, tensor, dim, i) + cat_dim_size = cat_dim_size + tensor[dim] + + result_size = _copy(not_skipped_tensor) + result_size[dim] = cat_dim_size + return result_size + + +def stack(tensors: list[list[int]], dim: int): + unsqueezed_tensors: list[list[int]] = [] + for tensor in tensors: + unsqueezed = unsqueeze(tensor, dim) + unsqueezed_tensors.append(unsqueezed) + return cat(unsqueezed_tensors, dim) + + +def select(self: list[int], dim: int, index: int): + ndim = len(self) + assert ndim != 0 + dim = maybe_wrap_dim(dim, ndim) + size = self[dim] + assert not (index < -size or index >= size) + if index < 0: + index += size + out: list[int] = [] + for i in range(ndim): + if i != dim: + out.append(self[i]) + return out + + +def matmul(tensor1: list[int], tensor2: list[int]): + dim_tensor1 = len(tensor1) + dim_tensor2 = len(tensor2) + if dim_tensor1 == 1 and dim_tensor2 == 1: + return dot(tensor1, tensor2) + elif dim_tensor1 == 2 and dim_tensor2 == 1: + return mv(tensor1, tensor2) + elif dim_tensor1 == 1 and dim_tensor2 == 2: + return squeeze(mm(unsqueeze(tensor1, 0), tensor2), 0) + elif dim_tensor1 == 2 and dim_tensor2 == 2: + return mm(tensor1, tensor2) + elif dim_tensor1 >= 1 and dim_tensor2 >= 1: + # We are multiplying b1 x n x m1 by x2 x m2 x p (where b1 can be a list); + # we track m1 vs m2 separately even though they must match for nicer error messages + n = tensor1[-2] if dim_tensor1 > 1 else 1 + batch_tensor1: list[int] = [] + # TODO: handling of slice + for i in range(dim_tensor1 - 2): + batch_tensor1.append(tensor1[i]) + p = tensor2[-1] + batch_tensor2: list[int] = [] + # TODO: handling of slice + for i in range(dim_tensor2 - 2): + batch_tensor2.append(tensor2[i]) + + # expand the batch portion (i.e. cut off matrix dimensions and expand rest) + expand_batch_portion = broadcast(batch_tensor1, batch_tensor2) + + # todo: copy ? + output_shape = expand_batch_portion + if dim_tensor1 > 1: + output_shape.append(n) + + if dim_tensor2 > 1: + output_shape.append(p) + + return output_shape + else: + assert False, "both arguments to matmul need to be at least 1D" + + +def t(self: list[int]): + assert len(self) <= 2 + self_len = len(self) + if self_len == 0: + out: list[int] = [] + return out + elif self_len == 1: + return [self[0]] + else: + return [self[1], self[0]] + + +def transpose(self: list[int], dim0: int, dim1: int): + ndims = len(self) + dim0 = maybe_wrap_dim(dim0, ndims) + dim1 = maybe_wrap_dim(dim1, ndims) + if dim0 == dim1: + return _copy(self) + out: list[int] = [] + for i in range(ndims): + if i == dim0: + out.append(self[dim1]) + elif i == dim1: + out.append(self[dim0]) + else: + out.append(self[i]) + return out + + +def linear(input: list[int], weight: list[int], bias: Optional[list[int]]): + out = matmul(input, t(weight)) + if bias is not None: + assert broadcast(bias, out) == out + return out + + +def addmm(self: list[int], mat1: list[int], mat2: list[int], beta: Any, alpha: Any): + return broadcast(self, mm(mat1, mat2)) + + +def check_non_negative(array: list[int]) -> bool: + # TODO: look into rewriting with early return and getting loop unrolling to fire + non_negative = False + for val in array: + if val < 0: + non_negative = True + return non_negative + + +def check_shape_forward( + input: list[int], + weight_sizes: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + groups: int, +): + k = len(input) + weight_dim = len(weight_sizes) + + # TODO: assertions could be expanded with the error messages + assert not check_non_negative(padding) + assert not check_non_negative(stride) + + assert weight_dim == k + assert weight_sizes[0] >= groups + assert (weight_sizes[0] % groups) == 0 + # only handling not transposed + assert input[1] == weight_sizes[1] * groups + assert bias is None or (len(bias) == 1 and bias[0] == weight_sizes[0]) + + for i in range(2, k): + assert (input[i] + 2 * padding[i - 2]) >= ( + dilation[i - 2] * (weight_sizes[i] - 1) + 1 + ) + + # this is not handling transposed convolution yet + + +def conv_output_size( + input_size: list[int], + weight_size: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + groups: int, +): + check_shape_forward( + input_size, weight_size, bias, stride, padding, dilation, groups + ) + + has_dilation = len(dilation) > 0 + dim = len(input_size) + output_size: list[int] = [] + input_batch_size_dim = 0 + weight_output_channels_dim = 0 + output_size.append(input_size[input_batch_size_dim]) + output_size.append(weight_size[weight_output_channels_dim]) + + for d in range(2, dim): + dilation_ = dilation[d - 2] if has_dilation else 1 + kernel = dilation_ * (weight_size[d] - 1) + 1 + output_size.append( + (input_size[d] + (2 * padding[d - 2]) - kernel) // stride[d - 2] + 1 + ) + return output_size + + +def conv1d( + input: list[int], + weight: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + groups: int, +): + assert len(weight) == 3 + assert len(input) == 3 + return conv_output_size(input, weight, bias, stride, padding, dilation, groups) + + +def conv2d( + input: list[int], + weight: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + groups: int, +): + assert len(weight) == 4 + assert len(input) == 4 + return conv_output_size(input, weight, bias, stride, padding, dilation, groups) + + +def conv_backwards( + grad_output: list[int], + input: list[int], + weight: list[int], + biases: Optional[list[int]], +): + # Bias gradient is always generated regardess of if biases is supplied + return _copy(input), _copy(weight), [grad_output[1]] + + +def conv_transpose2d_input( + input: list[int], + weight: list[int], + bias: Optional[list[int]] = None, + stride: Optional[list[int]] = None, + padding: Optional[list[int]] = None, + output_padding: Optional[list[int]] = None, + groups: int = 1, + dilation: Optional[list[int]] = None, +) -> list[int]: + if stride is None: + stride = [1, 1] + if padding is None: + padding = [0, 0] + if output_padding is None: + output_padding = [0, 0] + if dilation is None: + dilation = [1, 1] + has_dilation = len(dilation) > 0 + dim = len(input) + output_size: list[int] = [] + input_batch_size_dim = 0 + weight_output_channels_dim = 1 + output_size.append(input[input_batch_size_dim]) + output_size.append(weight[weight_output_channels_dim] * groups) + + for d in range(2, dim): + dilation_ = dilation[d - 2] if has_dilation else 1 + kernel = dilation_ * (weight[d] - 1) + output_size.append( + (input[d] - 1) * stride[d - 2] + - 2 * padding[d - 2] + + kernel + + output_padding[d - 2] + + 1 + ) + return output_size + + +def conv_forwards( + input: list[int], + weight: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + transposed: bool, + output_padding: list[int], + groups: int, +) -> list[int]: + has_dilation = len(dilation) > 0 + has_output_padding = len(output_padding) > 0 + dim = len(input) + output_size: list[int] = [] + input_batch_size_dim = 0 + weight_output_channels_dim = 1 if transposed else 0 + output_size.append(input[input_batch_size_dim]) + if transposed: + output_size.append(weight[weight_output_channels_dim] * groups) + else: + output_size.append(weight[weight_output_channels_dim]) + + for d in range(2, dim): + dilation_ = dilation[d - 2] if has_dilation else 1 + output_padding_ = output_padding[d - 2] if has_output_padding else 0 + if transposed: + kernel = dilation_ * (weight[d] - 1) + output_size.append( + (input[d] - 1) * stride[d - 2] + - 2 * padding[d - 2] + + kernel + + output_padding_ + + 1 + ) + else: + kernel = dilation_ * (weight[d] - 1) + 1 + output_size.append( + (input[d] + (2 * padding[d - 2]) - kernel) // stride[d - 2] + 1 + ) + return output_size + + +def _conv_forwards( + input: list[int], + weight: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + transposed: bool, + output_padding: list[int], + groups: int, + benchmark: bool, + deterministic: bool, + cudnn_enabled: bool, + allow_tf32: bool, +) -> list[int]: + return conv_forwards( + input, + weight, + bias, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + ) + + +def batch_norm( + input: list[int], + weight: Optional[list[int]], + bias: Optional[list[int]], + running_mean: Optional[list[int]], + running_var: Optional[list[int]], + training: bool, + momentum: float, + eps: float, + cudnn_enabled: bool, +): + out: list[int] = [] + for elem in input: + out.append(elem) + return out + + +def conv3d( + input: list[int], + weight: list[int], + bias: Optional[list[int]], + stride: list[int], + padding: list[int], + dilation: list[int], + groups: int, +): + assert len(weight) == 5 + assert len(input) == 5 + return conv_output_size(input, weight, bias, stride, padding, dilation, groups) + + +def maybe_wrap_dim(dim: int, dim_post_expr: int, wrap_scalar: bool = True): + if dim_post_expr <= 0: + assert wrap_scalar + dim_post_expr = 1 + min = -dim_post_expr + max = dim_post_expr - 1 + assert not (dim < min or dim > max) + if dim < 0: + dim += dim_post_expr + return dim + + +def zero_dim_tensor(input: Any): + out: list[int] = [] + return out + + +def multiply_integers(li: list[int]): + out = 1 + for elem in li: + out = out * elem + return out + + +def arange_end(end: number, inp0: Any, inp1: Any, inp2: Any, inp3: Any): + assert end >= 0 + return [int(math.ceil(end))] + + +def arange_start( + start: number, end: number, inp0: Any, inp1: Any, inp2: Any, inp3: Any +): + assert end >= 0 + assert end >= start + return [int(math.ceil(end - start))] + + +def arange_start_step( + start: number, end: number, step: number, inp0: Any, inp1: Any, inp2: Any, inp3: Any +): + assert step != 0 + if step < 0: + assert start >= end + else: + assert end >= start + return [int(math.ceil((end - start) / step))] + + +def permute(input: list[int], dims: list[int]): + assert len(input) == len(dims) + ndim = len(dims) + seen_dims: list[int] = [] + newSizes: list[int] = [] + for i in range(ndim): + dim = maybe_wrap_dim(dims[i], ndim) + seen_dims.append(dim) + newSizes.append(input[dim]) + for i in range(1, ndim): + for j in range(i): + assert seen_dims[i] != seen_dims[j] + return newSizes + + +def movedim(self: list[int], source: list[int], destination: list[int]) -> list[int]: + self_dim = len(self) + if self_dim <= 1: + return self + normalized_src: list[int] = [] + normalized_dst: list[int] = [] + for i in range(len(source)): + normalized_src.append(maybe_wrap_dim(source[i], self_dim)) + normalized_dst.append(maybe_wrap_dim(destination[i], self_dim)) + order = [-1 for i in range(self_dim)] + src_dims = [i for i in range(self_dim)] + dst_dims = [i for i in range(self_dim)] + + for i in range(len(source)): + order[normalized_dst[i]] = normalized_src[i] + src_dims[normalized_src[i]] = -1 + dst_dims[normalized_dst[i]] = -1 + + source_dims: list[int] = [] + destination_dims: list[int] = [] + for ele in src_dims: + if ele != -1: + source_dims.append(ele) + for ele in dst_dims: + if ele != -1: + destination_dims.append(ele) + + rest_dim = self_dim - len(source) + for i in range(rest_dim): + order[destination_dims[i]] = source_dims[i] + return permute(self, order) + + +def flatten(input: list[int], start_dim: int, end_dim: int): + start_dim = maybe_wrap_dim(start_dim, len(input)) + end_dim = maybe_wrap_dim(end_dim, len(input)) + assert start_dim <= end_dim + if len(input) == 0: + return [1] + if start_dim == end_dim: + # TODO: return self + out: list[int] = [] + for elem in input: + out.append(elem) + return out + slice_numel = 1 + for i in range(start_dim, end_dim + 1): + slice_numel *= input[i] + # TODO: use slicing when slice optimization has landed + # slice_numel = multiply_integers(input[start_dim:end_dim - start_dim + 1]) + shape: list[int] = [] + for i in range(start_dim): + shape.append(input[i]) + shape.append(slice_numel) + for i in range(end_dim + 1, len(input)): + shape.append(input[i]) + return shape + + +def nonzero_lower_bound(input: list[int]): + return [0, len(input)] + + +def nonzero_upper_bound(input: list[int]): + return [numel(input), len(input)] + + +def _reduce_along_dim(self: list[int], dim: int, keepdim: bool): + dim = maybe_wrap_dim(dim, len(self)) + out: list[int] = [] + for i, self_dim in enumerate(self): + if i == dim: + if keepdim: + out.append(1) + else: + out.append(self_dim) + return out + + +def argmax( + self: list[int], dim: Optional[int] = None, keepdim: bool = False +) -> list[int]: + if dim is None: + return [] + return _reduce_along_dim(self, dim, keepdim) + + +def bmm(self: list[int], mat2: list[int]) -> list[int]: + assert len(self) == 3, "bmm only supports 3D tensors" + assert len(mat2) == 3, "bmm only supports 3D tensors" + assert self[0] == mat2[0], "mismatching batch dimension" + assert self[2] == mat2[1], "mismatching contracting dimension" + return [self[0], self[1], mat2[2]] + + +def _shape_as_tensor(self: list[int]) -> list[int]: + return [len(self)] + + +def topk(self: list[int], k: int, dim: int = -1) -> tuple[list[int], list[int]]: + if len(self) == 0: + result: list[int] = [] + else: + assert k <= self[dim], ( + f"k ({k}) is too big for dimension {dim} of size {self[dim]}" + ) + result = _copy(self) + result[dim] = k + return result, result + + +def nll_loss_forward( + self: list[int], target: list[int], weight: Optional[list[int]], reduction: int +) -> tuple[list[int], list[int]]: + # This is taken shamelessly from the meta function in LossNLL.cpp + self_dim = len(self) + target_dim = len(target) + assert 0 < self_dim <= 2 + assert target_dim <= 1 + no_batch_dim = self_dim == 1 and target_dim == 0 + assert no_batch_dim or (self[0] == target[0]) + n_classes = self[-1] + scalar_shape: list[int] = [] + assert weight is None or (len(weight) == 1 and weight[0] == n_classes) + if reduction == 0 and self_dim == 2: + reduction_shape = [self[0]] + else: + reduction_shape = scalar_shape + return reduction_shape, scalar_shape + + +def native_layer_norm( + input: list[int], normalized_shape: list[int] +) -> tuple[list[int], list[int], list[int]]: + reduction_shape: list[int] = [] + num_unreduced_dimensions = len(input) - len(normalized_shape) + assert num_unreduced_dimensions >= 0 + for i in range(num_unreduced_dimensions): + reduction_shape.append(input[i]) + for i in range(num_unreduced_dimensions, len(input)): + reduction_shape.append(1) + return _copy(input), reduction_shape, reduction_shape + + +def native_batch_norm( + input: list[int], + weight: Optional[list[int]], + bias: Optional[list[int]], + running_mean: Optional[list[int]], + running_var: Optional[list[int]], + training: bool, +) -> tuple[list[int], list[int], list[int]]: + if training: + _size = [input[1]] + else: + _size = [0] + return _copy(input), _size, _size + + +def _batch_norm_with_update( + input: list[int], + weight: Optional[list[int]], + bias: Optional[list[int]], + running_mean: Optional[list[int]], + running_var: Optional[list[int]], +) -> tuple[list[int], list[int], list[int], list[int]]: + _size = [input[1]] + return _copy(input), _size, _size, [0] + + +def cross_entropy_loss( + self: list[int], + target: list[int], + weight: Optional[list[int]] = None, + reduction: int = 1, + ignore_index: int = -100, + label_smoothing: float = 0.0, +) -> list[int]: + result_shape = nll_loss_forward(self, target, weight, reduction)[0] + return result_shape + + +""" +Currently deferring the enabling of this, as part of the propoasal to suspend +adding ops. +There are currently cases in the test case where this is being called +in the SSA opinfo tests with with unexpected values (eg list of two ints, see the first +opinfo test). The behavior of index is significantly dependent on the inputs. + +This could be an error with how we are matching up shape functions, or that this +function needs to just implement everything. + +def index_Tensor(self: List[int], indices: List[Optional[List[int]]]) -> List[int]: + assert len(indices) <= len(self), "More indices than dimensions to index" + broadcasted_shape: List[int] = [] + for index_tensor_shape in indices: + if index_tensor_shape is not None: + broadcasted_shape = broadcast(broadcasted_shape, index_tensor_shape) + return broadcasted_shape +""" + +ScriptFn = torch._C.ScriptFunction +shape_compute_graph_mapping: dict[str, ScriptFn] = {} +bounded_compute_graph_mapping: dict[str, tuple[ScriptFn, ScriptFn]] = {} +script_func_map: dict[Callable, ScriptFn] = {} + + +def process_func(func: Callable): + if func not in script_func_map: + scripted_func = torch.jit.script(func) + + torch._C._jit_pass_inline(scripted_func.graph) + + for _ in range(2): + torch._C._jit_pass_peephole(scripted_func.graph) + torch._C._jit_pass_constant_propagation(scripted_func.graph) + + script_func_map[func] = scripted_func + return script_func_map[func] + + +def add_shape_compute_mapping(operator_schema: str, func: Callable): + global shape_compute_graph_mapping + + shape_compute_graph_mapping[operator_schema] = process_func(func) + + +def add_bounded_compute_mapping( + operator_schema: str, lower_bound_func: Callable, upper_bound_func: Callable +): + # Adds a shape compute function for both upper and lower bounds + fns = (process_func(lower_bound_func), process_func(upper_bound_func)) + bounded_compute_graph_mapping[operator_schema] = fns + + +add_shape_compute_mapping( + "aten::contiguous(Tensor(a) self, *, MemoryFormat memory_format=contiguous_format) -> Tensor(a)", + unary, +) +add_shape_compute_mapping( + "aten::rsub.Tensor(Tensor self, Scalar other, Scalar alpha=1) -> Tensor", unary +) +add_shape_compute_mapping( + "aten::dropout(Tensor input, float p, bool train) -> Tensor", unary +) +add_shape_compute_mapping( + "aten::adaptive_avg_pool2d(Tensor self, int[2] output_size) -> Tensor", + adaptive_avg_pool2d, +) +add_shape_compute_mapping( + "prim::NumToTensor.Scalar(Scalar a) -> Tensor", zero_dim_tensor +) +add_shape_compute_mapping("prim::NumToTensor.bool(bool a) -> Tensor", zero_dim_tensor) +add_shape_compute_mapping( + "aten::zeros(int[] size, *, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None) -> (Tensor)", + unary, +) +add_shape_compute_mapping( + "aten::to.dtype(Tensor(a) self, int dtype, bool non_blocking=False, bool copy=False, int? memory_format=None) -> (Tensor(a))", + unary, +) +add_shape_compute_mapping( + "aten::arange(Scalar end, *, int? dtype=None, int? layout=None, Device? device=None, bool? pin_memory=None) -> (Tensor)", + arange_end, +) +add_shape_compute_mapping( + "aten::arange.start(Scalar start, Scalar end, *, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None) -> Tensor", + arange_start, +) +add_shape_compute_mapping( + "aten::arange.start_step(Scalar start, Scalar end, Scalar step, *, ScalarType? dtype=None, Layout? layout=None, Device? device=None, bool? pin_memory=None) -> Tensor", + arange_start_step, +) +add_shape_compute_mapping("aten::squeeze(Tensor(a) self) -> Tensor(a)", squeeze_nodim) +add_shape_compute_mapping( + "aten::squeeze.dim(Tensor(a) self, int dim) -> Tensor(a)", squeeze +) +add_shape_compute_mapping( + "aten::squeeze.dims(Tensor(a) self, int[] dim) -> Tensor(a)", squeeze_dims +) +add_shape_compute_mapping( + "aten::unsqueeze(Tensor(a) self, int dim) -> Tensor(a)", unsqueeze +) +add_shape_compute_mapping( + "aten::slice.Tensor(Tensor(a) self, int dim=0, int? start=None, int? end=None, int step=1) -> Tensor(a)", + slice, +) +add_shape_compute_mapping( + "aten::select.int(Tensor(a) self, int dim, int index) -> Tensor(a)", select +) +add_shape_compute_mapping( + "aten::index_select(Tensor self, int dim, Tensor index) -> Tensor", index_select +) +add_shape_compute_mapping( + "aten::layer_norm(Tensor input, int[] normalized_shape, Tensor? weight=None, Tensor? bias=None, " + "float eps=1e-05, bool cudnn_enable=True) -> Tensor", + unary, +) +add_shape_compute_mapping( + "aten::softmax.int(Tensor self, int dim, ScalarType? dtype=None) -> Tensor", unary +) +add_shape_compute_mapping( + "aten::_no_grad_embedding_renorm_(Tensor weight, Tensor input, float max_norm, float norm_type) -> Tensor", + unary, +) +add_shape_compute_mapping( + "aten::embedding_renorm_(Tensor(a!) self, Tensor indices, float max_norm, float norm_type) -> Tensor(a!)", + unary, +) +add_shape_compute_mapping( + "aten::embedding(Tensor weight, Tensor indices, int padding_idx=-1, bool scale_grad_by_freq=False, bool sparse=False) -> Tensor", + embedding, +) +add_shape_compute_mapping("aten::mm(Tensor self, Tensor mat2) -> Tensor", mm) +add_shape_compute_mapping("aten::dot(Tensor self, Tensor tensor) -> Tensor", dot) +add_shape_compute_mapping("aten::mv(Tensor self, Tensor vec) -> Tensor", mv) +add_shape_compute_mapping("aten::matmul(Tensor self, Tensor other) -> Tensor", matmul) +add_shape_compute_mapping( + "aten::linear(Tensor input, Tensor weight, Tensor? bias=None) -> Tensor", linear +) +add_shape_compute_mapping( + "aten::max_pool2d(Tensor self, int[2] kernel_size, int[2] stride=[], int[2] padding=0, int[2] dilation=1, bool ceil_mode=False) -> Tensor", + max_pool2d, +) +add_shape_compute_mapping( + "aten::max_pool2d_with_indices(Tensor self, int[2] kernel_size, int[2] stride=[], int[2] padding=0, int[2] dilation=1, bool ceil_mode=False) -> (Tensor, Tensor)", + max_pool2d_with_indices, +) +add_shape_compute_mapping("aten::t(Tensor(a) self) -> Tensor(a)", t) +add_shape_compute_mapping( + "aten::transpose.int(Tensor(a) self, int dim0, int dim1) -> Tensor(a)", transpose +) +add_shape_compute_mapping( + "aten::conv1d(Tensor input, Tensor weight, Tensor? bias=None, int[1] stride=1, int[1] padding=0, int[1] dilation=1, int groups=1) -> Tensor", + conv1d, +) +add_shape_compute_mapping( + "aten::conv2d(Tensor input, Tensor weight, Tensor? bias=None, int[2] stride=1, int[2] padding=0, int[2] dilation=1, int groups=1) -> Tensor", + conv2d, +) +add_shape_compute_mapping( + "aten::batch_norm(Tensor input, Tensor? weight, Tensor? bias, Tensor? running_mean, Tensor? running_var, bool training, float momentum, float eps, bool cudnn_enabled) -> Tensor", + batch_norm, +) +add_shape_compute_mapping( + "aten::conv3d(Tensor input, Tensor weight, Tensor? bias=None, int[3] stride=1, int[3] padding=0, int[3] dilation=1, int groups=1) -> Tensor", + conv3d, +) +add_shape_compute_mapping( + "aten::convolution_backward(Tensor grad_output, Tensor input, Tensor weight, int[]? bias_sizes, int[] stride, int[] padding, int[] dilation, bool transposed, int[] output_padding, int groups, bool[3] output_mask) -> (Tensor, Tensor, Tensor)", + conv_backwards, +) +add_shape_compute_mapping( + "aten::convolution(Tensor input, Tensor weight, Tensor? bias, int[] stride, int[] padding, int[] dilation, bool transposed, int[] output_padding, int groups) -> Tensor", + conv_forwards, +) +add_shape_compute_mapping( + "aten::_convolution(Tensor input, Tensor weight, Tensor? bias, int[] stride, int[] padding, int[] dilation, bool transposed, int[] output_padding, int groups, bool benchmark, bool deterministic, bool cudnn_enabled, bool allow_tf32) -> Tensor", + _conv_forwards, +) +add_shape_compute_mapping( + "aten::conv_transpose2d.input(Tensor input, Tensor weight, Tensor? bias=None, int[2] stride=1, int[2] padding=0, int[2] output_padding=0, int groups=1, int[2] dilation=1) -> Tensor", + conv_transpose2d_input, +) +add_shape_compute_mapping( + "aten::flatten.using_ints(Tensor(a) self, int start_dim=0, int end_dim=-1) -> Tensor(a)", + flatten, +) +add_shape_compute_mapping("aten::cat(Tensor[] tensors, int dim=0) -> Tensor", cat) +add_shape_compute_mapping("aten::stack(Tensor[] tensors, int dim=0) -> Tensor", stack) +add_shape_compute_mapping( + "aten::permute(Tensor(a) self, int[] dims) -> Tensor(a)", permute +) +add_shape_compute_mapping( + "aten::movedim.intlist(Tensor(a) self, int[] source, int[] destination) -> Tensor(a)", + movedim, +) +add_shape_compute_mapping("aten::view(Tensor(a) self, int[] size) -> Tensor(a)", view) +add_shape_compute_mapping( + "aten::expand_as(Tensor(a) self, Tensor other) -> Tensor(a)", expand +) +add_shape_compute_mapping( + "aten::expand(Tensor(a) self, int[] size, *, bool implicit=False) -> Tensor(a)", + expand_one_unused, +) +add_shape_compute_mapping( + "aten::mean.dim(Tensor self, int[1]? dim, bool keepdim=False, *, ScalarType? dtype=None) -> Tensor", + sum_mean_dim, +) +add_shape_compute_mapping( + "aten::sum.dim_IntList(Tensor self, int[1]? dim, bool keepdim=False, *, ScalarType? dtype=None) -> Tensor", + sum_mean_dim, +) +add_shape_compute_mapping( + "aten::max.dim(Tensor self, int dim, bool keepdim=False) -> (Tensor values, Tensor indices)", + max_dim, +) +add_shape_compute_mapping( + "aten::mean(Tensor self, *, ScalarType? dtype=None) -> Tensor", zero_dim_tensor +) +add_shape_compute_mapping( + "aten::sum(Tensor self, *, ScalarType? dtype=None) -> Tensor", zero_dim_tensor +) +add_shape_compute_mapping( + "aten::addmm(Tensor self, Tensor mat1, Tensor mat2, *, Scalar beta=1, Scalar alpha=1) -> Tensor", + addmm, +) +add_shape_compute_mapping( + "aten::upsample_nearest2d.vec(Tensor input, int[]? output_size, float[]? scale_factors) -> (Tensor)", + upsample_nearest2d, +) +add_shape_compute_mapping( + "aten::quantize_per_tensor(Tensor self, float scale, int zero_point, ScalarType dtype) -> Tensor", + unary, +) +add_shape_compute_mapping( + "aten::quantize_per_tensor.tensor_qparams(Tensor self, Tensor scale, Tensor zero_point, ScalarType dtype) -> Tensor", + unary, +) +add_shape_compute_mapping("aten::dequantize(Tensor self) -> Tensor", unary) +add_shape_compute_mapping( + "quantized::add(Tensor qa, Tensor qb, float scale, int zero_point) -> Tensor qc", + broadcast, +) +add_shape_compute_mapping( + "aten::argmax(Tensor self, int? dim=None, bool keepdim=False) -> Tensor", argmax +) +add_shape_compute_mapping("aten::bmm(Tensor self, Tensor mat2) -> Tensor", bmm) +add_shape_compute_mapping( + "aten::_shape_as_tensor(Tensor self) -> Tensor", _shape_as_tensor +) +add_shape_compute_mapping( + "aten::topk(Tensor self, int k, int dim=-1, bool largest=True, bool sorted=True) -> (Tensor values, Tensor indices)", + topk, +) +add_shape_compute_mapping( + "aten::nll_loss_forward(Tensor self, Tensor target, Tensor? weight, int reduction, int ignore_index) -> (Tensor output, Tensor total_weight)", + nll_loss_forward, +) +add_shape_compute_mapping( + "aten::native_layer_norm(Tensor input, int[] normalized_shape, Tensor? weight, Tensor? bias, float eps) -> (Tensor, Tensor, Tensor)", + native_layer_norm, +) +add_shape_compute_mapping( + "aten::native_batch_norm(Tensor input, Tensor? weight, Tensor? bias, Tensor? running_mean, Tensor? running_var, bool training, float momentum, float eps) -> (Tensor, Tensor, Tensor)", + native_batch_norm, +) +add_shape_compute_mapping( + "aten::_native_batch_norm_legit(Tensor input, Tensor? weight, Tensor? bias, Tensor running_mean, Tensor running_var, bool training, float momentum, float eps) -> (Tensor, Tensor, Tensor)", + native_batch_norm, +) +add_shape_compute_mapping( + "aten::_native_batch_norm_legit.no_stats(Tensor input, Tensor? weight, Tensor? bias, Tensor running_mean, Tensor running_var, bool training, float momentum, float eps) -> (Tensor, Tensor, Tensor)", + native_batch_norm, +) +add_shape_compute_mapping( + "_batch_norm_with_update(Tensor input, Tensor? weight, Tensor? bias, Tensor(a!) running_mean, Tensor(b!) running_var, float momentum, float eps) -> (Tensor, Tensor, Tensor, Tensor)", + _batch_norm_with_update, +) + +add_shape_compute_mapping( + "aten::cross_entropy_loss(Tensor self, Tensor target, Tensor? weight=None, int reduction=Mean, SymInt ignore_index=-100, float label_smoothing=0.0) -> Tensor", + cross_entropy_loss, +) +# add_shape_compute_mapping("aten::index.Tensor(Tensor self, Tensor?[] indices) -> Tensor", index_Tensor) + +# TODO: migrate over all of symbolic_shape_registry_util.cpp +# These are duplicated here so that the functions will be serialized +add_shape_compute_mapping( + "aten::lerp.Tensor(Tensor self, Tensor end, Tensor weight) -> Tensor", + broadcast_three, +) +add_shape_compute_mapping( + "aten::where.ScalarSelf(Tensor condition, Scalar self, Tensor other) -> Tensor", + broadcast_one_three, +) +add_shape_compute_mapping( + "aten::add_.Tensor(Tensor(a!) self, Tensor other, *, Scalar alpha=1) -> Tensor(a!)", + broadcast_inplace, +) + +# quantized_conv_prepack TODO + +# Shape Compute Fn with upper and lower bounds +add_bounded_compute_mapping( + "aten::nonzero(Tensor self) -> (Tensor)", nonzero_lower_bound, nonzero_upper_bound +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_state.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_state.py new file mode 100644 index 0000000000000000000000000000000000000000..2ebbee553ab11fa35f1cb779decf18dfc20b56ae --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_state.py @@ -0,0 +1,128 @@ +# mypy: allow-untyped-defs +"""JIT-related state. + +This module stores various pieces of Python-global state relating to the JIT. + +This is not intended to be imported directly; please the exposed +functionalities in `torch.jit`. +""" + +import os +import weakref +from typing import Any + +import torch + + +class EnabledProxy: + """Stores whether the JIT is enabled or not. + + This is just a wrapper for a bool, so that we get reference semantics + """ + + def __init__(self) -> None: + self.enabled = self.parse_env( + "PYTORCH_JIT", True, "> Using PyTorch JIT", "> PyTorch JIT DISABLED" + ) + + def parse_env(self, name, default, true_message, false_message): + value = os.environ.get(name) + if value is None: + return default + if value.lower() in {"1", "true", "yes"}: + return True + elif value.lower() in {"0", "false", "no"}: + return False + if value == "1v": + print(true_message) + return True + elif value == "0v": + print(false_message) + return False + raise ValueError(f"Unknown setting of {name}. Try using 0 or 1.") + + def __bool__(self) -> bool: + return self.enabled + + +_enabled = EnabledProxy() + + +def disable() -> None: + _enabled.enabled = False + + +def enable() -> None: + _enabled.enabled = True + + +# The Python CompilationUnit. All functions and modules defined in Python will +# live in here. It's defined in Python because doing in cpp creates static +# destruction order issues. +_python_cu = torch._C.CompilationUnit() + + +# python class => ScriptClass mapping +_script_classes: dict[type[Any], type[Any]] = {} +_name_to_pyclass: dict[str, type[Any]] = {} + + +def _add_script_class(python_class, script_class) -> None: + _script_classes[python_class] = script_class + _name_to_pyclass[script_class.qualified_name()] = python_class + + +def _get_script_class(python_class): + override = getattr(python_class, "_jit_override_qualname", None) + if override is not None: + python_class = _get_python_class(override) + return _script_classes.get(python_class) + + +def _get_python_class(qualified_name): + return _name_to_pyclass.get(qualified_name) + + +def _clear_class_state() -> None: + _script_classes.clear() + _name_to_pyclass.clear() + + +# Caching: we currently cache compilation of free functions and overloaded functions. +# To cache free functions we hold a weak ref to the function object and +# map to the compiled fn's qualified name. +# To cache overloaded functions we hold a weak ref to the function obj and +# map to all of its overloaded compiled fns. +# In the future we could consider caching more types of objects so that +# aliasing is preserved across separate compilations of the same object. + +_jit_caching_layer: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary() +_jit_function_overload_caching: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary() + + +def _try_get_jit_cached_overloads(key): + qual_names = _jit_function_overload_caching.get(key, None) + if qual_names: + return [_python_cu.find_function(qual_name) for qual_name in qual_names] + else: + return None + + +def _set_jit_overload_cache(key, compiled_fns) -> None: + _jit_function_overload_caching[key] = [fn.qualified_name for fn in compiled_fns] + + +def _try_get_jit_cached_function(key): + if getattr(key, "__disable_jit_function_caching__", False) is True: + return None + qual_name = _jit_caching_layer.get(key, None) + if qual_name: + return _python_cu.find_function(qual_name) + else: + return None + + +def _set_jit_function_cache(key, value) -> None: + # only free functions currently supported + assert isinstance(value, torch.jit.ScriptFunction) + _jit_caching_layer[key] = value.qualified_name diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_trace.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_trace.py new file mode 100644 index 0000000000000000000000000000000000000000..005385f3fd9694ef2a6f841fce6abf7c10218765 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/_trace.py @@ -0,0 +1,1435 @@ +# mypy: allow-untyped-defs +"""Tracing. + +This module contains functionality to support the JIT's tracing frontend, notably: + * torch.jit.trace + * torch.jit.trace_module + +This is not intended to be imported directly; please use the exposed +functionalities in `torch.jit`. +""" + +import contextlib +import copy +import functools +import inspect +import os +import re +import sys +import warnings +from collections.abc import Callable +from enum import Enum +from typing import Any, Optional, TypeVar +from typing_extensions import ParamSpec + +import torch +from torch._jit_internal import ( + _get_model_id, + _qualified_name, + get_callable_argument_names, + is_scripting, +) +from torch.autograd import function +from torch.jit._script import _CachedForward, script, ScriptModule +from torch.jit._state import _enabled, _python_cu +from torch.nn import Module +from torch.testing._comparison import default_tolerances + + +_flatten = torch._C._jit_flatten +_unflatten = torch._C._jit_unflatten + +R = TypeVar("R", covariant=True) # return type (always covariant) +P = ParamSpec("P") + + +def _create_interpreter_name_lookup_fn(frames_up=1): + def _get_interpreter_name_for_var(var): + frame = inspect.currentframe() + if not frame: + raise RuntimeError("failed to inspect frame") + + i = 0 + while i < frames_up + 1: + frame = frame.f_back + if not frame: + raise RuntimeError("failed to get frame") + i += 1 + + f_locals = frame.f_locals + + for k, v in f_locals.items(): + if isinstance(v, torch.Tensor) and var is v: + return k if k != "self" else "" + return "" + + return _get_interpreter_name_for_var + + +def _unique_state_dict(module, keep_vars=False): + # since Parameter.detach() always creates a new torch.Tensor instance, + # id(v) doesn't work with it. So we always get the Parameter or Buffer + # as values, and deduplicate the params using Parameters and Buffers + state_dict = module.state_dict(keep_vars=True) + filtered_dict = type(state_dict)() + seen_ids: set[int] = set() + for k, v in state_dict.items(): + if id(v) in seen_ids: + continue + seen_ids.add(id(v)) + if keep_vars: + filtered_dict[k] = v + else: + filtered_dict[k] = v.detach() + return filtered_dict + + +class ONNXTracedModule(torch.nn.Module): + def __init__( + self, + inner, + strict=True, + force_outplace=False, + return_inputs=False, + return_inputs_states=False, + ): + super().__init__() + # inner may be a Module, or it may be an arbitrary callable + # If it's a Module, we get its parameters automatically, which lets + # us avoid a special casing functions versus modules. + self.inner = inner + self.strict = strict + self._force_outplace = force_outplace + self._return_inputs = return_inputs + self._return_inputs_states = return_inputs_states + + def forward(self, *args: torch.Tensor): + in_vars, in_desc = _flatten(args) + # NOTE: use full state, because we need it for BatchNorm export + # This differs from the compiler path, which doesn't support it at the moment. + module_state = list(_unique_state_dict(self, keep_vars=True).values()) + + ret_inputs = [] + inputs_states = [] + outs = [] + + def wrapper(*args): + in_args: list[torch.Tensor] = [] + for i in range(len(in_vars)): + if not isinstance(args[i], torch.Tensor): + raise RuntimeError("Expected Tensor argument") + in_args.append(args[i]) + + trace_inputs = _unflatten(in_args, in_desc) + + if self._return_inputs: + ret_inputs.append( + tuple(x.clone(memory_format=torch.preserve_format) for x in args) + ) + if self._return_inputs_states: + inputs_states.append(_unflatten(in_args, in_desc)) + outs.append(self.inner(*trace_inputs)) + if self._return_inputs_states: + inputs_states[0] = (inputs_states[0], trace_inputs) + out_vars, _ = _flatten(outs) + if len(out_vars) == 1: + return out_vars[0] + else: + return tuple(out_vars) + + graph, _out = torch._C._create_graph_by_tracing( + wrapper, + in_vars + module_state, + _create_interpreter_name_lookup_fn(), + self.strict, + self._force_outplace, + ) + + if self._return_inputs: + return graph, outs[0], ret_inputs[0] + if self._return_inputs_states: + return graph, outs[0], inputs_states[0] + else: + return graph, outs[0] + + +def _clone_inputs(args): + def clone_input(a): + if a is None: + return None + elif isinstance(a, torch.Tensor): + # TODO: figure out one liner to .clone() and set requires_grad + v = ( + a.detach() + .clone(memory_format=None if a.is_mkldnn else torch.preserve_format) + .requires_grad_(a.requires_grad) + ) + if a.grad is not None: + v.grad = clone_input(v.grad) + return v + else: + return a.clone(memory_format=torch.preserve_format) + + # pyrefly: ignore [missing-attribute] + return function._nested_map( + lambda x: isinstance(x, torch.Tensor), clone_input, condition_msg="tensors" + )(args) + + +# This is purely for developer debugging. We are not going to advertise it. +_JIT_TIME = os.environ.get("PYTORCH_JIT_TIME", False) # CUDA-only timing +_JIT_DISABLE = os.environ.get("PYTORCH_JIT_DISABLE", False) +_JIT_STATS = os.environ.get("PYTORCH_JIT_STATS", False) + + +@contextlib.contextmanager +def _time(trace_name, name, time=True): + if (not _JIT_TIME and not time) or not torch.cuda.is_available(): + yield + return + stream = torch.cuda.current_stream() + start = torch.cuda.Event(enable_timing=True) + end = torch.cuda.Event(enable_timing=True) + stream.record_event(start) + try: + yield + finally: + stream.record_event(end) + end.synchronize() + print(f"{trace_name} {name} time: {start.elapsed_time(end)} ms") + + +def verify(model, args, loss_fn=torch.sum, devices=None): + """ + Verify that a JIT compiled model has the same behavior as its uncompiled version along with its backwards pass. + + If your model returns multiple outputs, + you must also specify a `loss_fn` to produce a loss for which + the backwards will be computed. + + This function has side-effects (e.g., it executes your model / saves and loads + parameters), so don't expect the model to come out exactly the same as what + you passed in. + + Args: + model (compiled torch.nn.Module or function): the module/function to be + verified. The module/function definition MUST have been decorated with + `@torch.jit.compile`. + args (tuple or Tensor): the positional arguments to pass to the + compiled function/module to be verified. A non-tuple is assumed to + be a single positional argument to be passed to the model. + loss_fn (function, optional): the loss function to be applied to + the output of the model, before backwards is invoked. By default, + we assume that a model returns a single result, and we :func:`torch.sum` + before calling backwards; if this is inappropriate, you can pass your + own loss function. Note that if a model returns a tuple of results, + these are passed as separate positional arguments to `loss_fn`. + devices (iterable of device IDs, optional): the GPU devices which the + compiled module will be run on. This determines the RNG state we + must save when running both compiled and uncompiled versions of the model. + """ + # TODO: In principle, we track device information in our trace, so it + # should be possible to check if our execution actually obeyed the 'devices' + # the user provided. + + # TODO: Consider adding a utility function to torch.jit to test + # for this case + if not isinstance(model, torch._C.CompiledFunction): # type: ignore[attr-defined] + raise TypeError( + "Cannot verify an uncompiled module. Add @torch.jit.compile to compile it" + ) + is_module = isinstance(model, Module) + + if not isinstance(args, tuple): + args = (args,) + + if is_module: + saved_state = copy.deepcopy(model.state_dict()) + + def run_fwd_bwd(args, force_trace=False, assert_compiled=False): + params = list(model.parameters()) if is_module else [] + in_vars, _ = _flatten((args, params)) + # We use a special API to reset the trace and compile it from scratch. + compiled_fn = model + if force_trace: + compiled_fn.clear_cache() + if assert_compiled: + hits = compiled_fn.hits + out = model(*args) + if assert_compiled and compiled_fn.hits == hits: # type: ignore[possibly-undefined] + raise RuntimeError("failed to use the compiled function") + if not isinstance(out, tuple): + out = (out,) + if loss_fn == torch.sum and len(out) != 1: + raise ValueError( + f"Model returns {len(out)} outputs, but default loss function " + "(torch.sum) can only handle a single output" + ) + out_vars, _ = _flatten(out) + saved_outs = [ + v.detach().clone(memory_format=torch.preserve_format) for v in out_vars + ] + loss = loss_fn(*out) + grads = torch.autograd.grad([loss], in_vars) + # TODO: I'm not sure if the clone here is necessary but it is safer + saved_grads = [ + v.detach().clone(memory_format=torch.preserve_format) for v in grads + ] + return (saved_outs, saved_grads) + + with torch.random.fork_rng(devices, _caller="torch.jit.verify"): + uncompiled_outs, uncompiled_grads = run_fwd_bwd(args, force_trace=True) + assert model.has_trace_for(*args) + + if is_module: + model.load_state_dict(saved_state) # type: ignore[possibly-undefined] + compiled_outs, compiled_grads = run_fwd_bwd(args, assert_compiled=True) + + _verify_equal(uncompiled_outs, compiled_outs) + _verify_equal(uncompiled_grads, compiled_grads) + + +def _verify_equal(xs, ys): + for x, y in zip(xs, ys): + if x.sub(y).abs().max() > 1e-6: + raise RuntimeError("JIT and real computation mismatch") + + +def indent(s): + return "\n".join(["\t" + line for line in s.splitlines()]) + + +class TracingCheckError(Exception): + def __init__(self, graph_diff_error, tensor_compare_error, extra_msg=None): + self.message = "Tracing failed sanity checks!\n" + if extra_msg is not None: + self.message += extra_msg + "\n" + if graph_diff_error is not None: + self.message += "ERROR: Graphs differed across invocations!\n" + self.message += indent(graph_diff_error) + "\n" + if tensor_compare_error is not None: + self.message += ( + "ERROR: Tensor-valued Constant nodes differed in value " + "across invocations. This often indicates that the tracer has" + " encountered untraceable code.\n" + ) + self.message += indent(tensor_compare_error) + "\n" + super().__init__(self.message) + + +# Check the traced module against a set of user-provided validation inputs +@torch.no_grad() +def _check_trace( + check_inputs, + func, + traced_func, + check_tolerance, + strict, + force_outplace, + is_trace_module, + _module_class, + example_inputs_is_kwarg=False, +): + # Note: tracing is independent of optimizations, which consume the trace + for inputs in check_inputs: + if isinstance(inputs, torch.Tensor): + inputs = (inputs,) + + if is_trace_module: + copied_dict = {} + # pyrefly: ignore [missing-attribute] + for name, data in inputs.items(): + copied_dict[name] = _clone_inputs(data) + check_mod = torch.jit.trace_module( + getattr(func, "__self__", func), + copied_dict, + check_trace=False, + strict=strict, + _force_outplace=force_outplace, + _module_class=_module_class, + _compilation_unit=torch._C.CompilationUnit(), + example_inputs_is_kwarg=example_inputs_is_kwarg, + _store_inputs=False, + ) + check_mod_func = check_mod._c._get_method(traced_func.name) + inputs = inputs[traced_func.name] + if ( + isinstance(inputs, (torch.Tensor)) + or isinstance(inputs, dict) + and not example_inputs_is_kwarg + ): + inputs = (inputs,) + else: + if example_inputs_is_kwarg: + check_mod = torch.jit.trace( + func, + check_trace=False, + strict=strict, + _force_outplace=force_outplace, + _module_class=_module_class, + example_kwarg_inputs=_clone_inputs(inputs), + _store_inputs=False, + ) + else: + check_mod = torch.jit.trace( + func, + _clone_inputs(inputs), + check_trace=False, + strict=strict, + _force_outplace=force_outplace, + _module_class=_module_class, + _store_inputs=False, + ) + check_mod_func = check_mod + + def graph_diagnostic_info(): + mod_canonicalized = torch._C._jit_pass_canonicalize(traced_func.graph) + torch._C._jit_pass_inline(mod_canonicalized) + torch._C._jit_pass_erase_shape_information(mod_canonicalized) + mod_str = str(mod_canonicalized) + mod_str = re.sub(r"___torch_mangle_[0-9]+\.", "", mod_str) + check_canonicalized = torch._C._jit_pass_canonicalize(check_mod_func.graph) + torch._C._jit_pass_inline(check_canonicalized) + torch._C._jit_pass_erase_shape_information(check_canonicalized) + check_str = str(check_canonicalized) + check_str = re.sub(r"___torch_mangle_[0-9]+\.", "", check_str) + + graph_diff_errors = None + if mod_str != check_str: + import difflib + + graph_diff = difflib.ndiff( + mod_str.splitlines(True), check_str.splitlines(True) + ) + graph_diff_errors = "Graph diff:\n" + indent("".join(graph_diff)) + "\n" + + for n_mod, n_check in zip( + mod_canonicalized.nodes(), check_canonicalized.nodes() + ): + if str(n_mod) != str(n_check): + graph_diff_errors += "First diverging operator:\n" + node_diff = difflib.ndiff( + str(n_mod).splitlines(True), str(n_check).splitlines(True) + ) + source_printout = ( + "Node diff:\n" + indent("".join(node_diff)) + "\n" + ) + mod_stack = n_mod.sourceRange() + if mod_stack: + source_printout += ( + "Trace source location:\n" + indent(mod_stack) + "\n" + ) + check_stack = n_check.sourceRange() + if check_stack: + source_printout += ( + "Check source location:\n" + indent(check_stack) + "\n" + ) + graph_diff_errors += source_printout + + break # For now, only print out the first pair of nodes that diverges + + tensor_compare_errors = None + # Check Tensor-valued constant nodes + for n_mod, n_check in zip( + mod_canonicalized.nodes(), check_canonicalized.nodes() + ): + if n_mod.kind() != n_check.kind(): + break # Graphs have already diverged + + if n_mod.kind() == "prim::Constant" and not ( + n_mod.mustBeNone() or n_check.mustBeNone() + ): + if not n_mod.hasAttribute("value"): + continue + if n_mod.kindOf("value") != "t" or n_check.kindOf("value") != "t": + continue + + mod_tensor_val = n_mod.t("value") + check_tensor_val = n_check.t("value") + + try: + torch.testing.assert_close( + mod_tensor_val, check_tensor_val, equal_nan=True + ) + except (RuntimeError, AssertionError) as e: + if tensor_compare_errors is None: + tensor_compare_errors = "" + tensor_compare_errors += "Node:\n" + indent(str(n_mod)) + "\n" + compare_stack = n_mod.sourceRange() + if compare_stack: + tensor_compare_errors += ( + "Source Location:\n" + indent(compare_stack) + "\n" + ) + tensor_compare_errors += "Comparison exception: " + indent( + str(e) + ) + + break # For now, only print the first diverging pair + + return graph_diff_errors, tensor_compare_errors + + def wrap_retval(x): + return x if isinstance(x, tuple) else (x,) + + def run_mod_and_filter_tensor_outputs(mod, inputs, running_what): + try: + if isinstance(inputs, dict) and example_inputs_is_kwarg: + outs = wrap_retval(mod(**inputs)) + else: + outs = wrap_retval(mod(*_clone_inputs(inputs))) + outs = [out for out in outs if isinstance(out, torch.Tensor)] + return outs + except Exception as e: + graph_diff_errors, tensor_compare_errors = graph_diagnostic_info() + msg = f"encountered an exception while running the {running_what} with test inputs.\nException:\n{indent(str(e))}" + raise TracingCheckError( + graph_diff_errors, + tensor_compare_errors, + extra_msg=msg, + ) from e + + has_warned = [False] + + def maybe_warn_nondeterministic(): + if has_warned[0]: + return + has_warned[0] = True + nondeterm_ops = [ + op for op in traced_func.graph.nodes() if op.isNondeterministic() + ] + if len(nondeterm_ops) > 0: + nondeterministic_ops_warning = "Trace had nondeterministic nodes. " + nondeterministic_ops_warning += ( + "Did you forget call .eval() on your model? Nodes:\n" + ) + nondeterministic_ops_warning += "\n".join( + [indent(str(op)) for op in nondeterm_ops][:20] + ) + nondeterministic_ops_warning += ( + "\nThis may cause errors in trace checking. To disable trace checking," + " pass check_trace=False to torch.jit.trace()" + ) + warnings.warn( + nondeterministic_ops_warning, category=TracerWarning, stacklevel=5 + ) + + def compare_outputs(original, reference, match_what): + all_ok = True + for i, (orig, ref) in enumerate(zip(original, reference)): + try: + if orig.is_quantized: + orig = orig.dequantize() + if ref.is_quantized: + ref = ref.dequantize() + if orig.is_mkldnn: + orig = orig.to_dense() + if ref.is_mkldnn: + ref = ref.to_dense() + if ref.is_complex() or orig.is_complex(): + torch.testing.assert_close( + orig.to(torch.cdouble), + ref.to(torch.cdouble), + rtol=check_tolerance, + atol=default_tolerances(orig, ref)[1], + equal_nan=True, + ) + else: + if orig.is_mps or ref.is_mps: + torch.testing.assert_close( + orig.float(), + ref.float(), + rtol=check_tolerance, + atol=default_tolerances(orig, ref)[1], + equal_nan=True, + ) + elif getattr(orig, "is_nested", None) or getattr( + ref, "is_nested", None + ): + assert getattr(orig, "is_nested", None) == getattr( + ref, "is_nested", None + ) + for t_orig, t_ref in zip(orig.unbind(), ref.unbind()): + torch.testing.assert_close( + t_orig.double(), + t_ref.double(), + rtol=check_tolerance, + atol=default_tolerances(t_orig, t_ref)[1], + equal_nan=True, + ) + else: + torch.testing.assert_close( + orig.double(), + ref.double(), + rtol=check_tolerance, + atol=default_tolerances(orig, ref)[1], + equal_nan=True, + ) + except AssertionError as e: + maybe_warn_nondeterministic() + warnings.warn( + "Output nr " + + str(i + 1) + + ". of the traced function does not match " + "the corresponding output of the " + + match_what + + ". Detailed error:\n" + + str(e), + category=TracerWarning, + stacklevel=4, + ) + all_ok = False + + return all_ok + + traced_outs = run_mod_and_filter_tensor_outputs(traced_func, inputs, "trace") + fn_outs = run_mod_and_filter_tensor_outputs(func, inputs, "Python function") + if compare_outputs(traced_outs, fn_outs, "Python function"): + check_outs = run_mod_and_filter_tensor_outputs( + check_mod_func, inputs, "repeated trace" + ) + compare_outputs(traced_outs, check_outs, "repeated trace") + + diag_info = graph_diagnostic_info() + if any(info is not None for info in diag_info): + raise TracingCheckError(*diag_info) + + +class TracerWarning(Warning): + @staticmethod + def ignore_lib_warnings(): + # We ignore warnings from all submodules excluding the JIT, because we need them e.g. for _check_trace + warnings.filterwarnings( + "ignore", category=TracerWarning, module="torch.(?!jit)" + ) + warnings.filterwarnings("ignore", "torch::jit::fuser::cuda") + + +# We ignore the tracer warnings coming form inside the library, because all our shape +# checks in nn will trigger them. +TracerWarning.ignore_lib_warnings() +torch._C._tracer_warn_use_python() + + +def make_tuple(example_inputs): + if isinstance(example_inputs, (torch.Tensor, dict)): + return (example_inputs,) + # done primarily so that weird iterables fail here and not pybind11 code + if not isinstance(example_inputs, tuple): + return tuple(example_inputs) + return example_inputs + + +def make_module(mod, _module_class, _compilation_unit): + if isinstance(mod, ScriptModule): + return mod + elif torch._jit_internal.module_has_exports(mod): + infer_methods_stubs_fn = torch.jit._recursive.make_stubs_from_exported_methods + return torch.jit._recursive.create_script_module( + mod, infer_methods_stubs_fn, share_types=False, is_tracing=True + ) + else: + if _module_class is None: + _module_class = TopLevelTracedModule + return _module_class(mod, _compilation_unit=_compilation_unit) + + +def wrap_check_inputs(check_inputs): + if check_inputs is None: + return None + + return [{"forward": c} for c in check_inputs] + + +def analyze_ts_result_with_export_result(export, trace): + import torch.utils._pytree as pytree + + flat_export = pytree.tree_leaves(export) + flat_trace = pytree.tree_leaves(trace) + + for orig, loaded in zip(flat_export, flat_trace): + if orig.layout != loaded.layout: + return False + # mkldnn is not supported for torch.allclose + if orig.layout == torch._mkldnn: # type: ignore[attr-defined] + return True + if type(orig) is not type(loaded): + return False + + if isinstance(orig, torch._subclasses.FakeTensor): + # Skip for FakeTensor. + return True + elif isinstance(orig, torch.Tensor): + if orig.dtype != loaded.dtype: + return False + if not torch.allclose(orig, loaded): + return False + else: + if orig != loaded: + return False + return True + + +def _trace_impl( + func, + example_inputs=None, + optimize=None, + check_trace=True, + check_inputs=None, + check_tolerance=1e-5, + strict=True, + _force_outplace=False, + _module_class=None, + _compilation_unit=_python_cu, + example_kwarg_inputs=None, + _store_inputs=True, +): + if isinstance(func, torch.jit.ScriptModule): + # it is hard to trace it because the forward method on ScriptModule is already defined, so it + # would result in an error. + warnings.warn( + "The input to trace is already a ScriptModule, tracing it is a no-op. Returning the object as is.", + stacklevel=2, + ) + return func + + if isinstance(func, torch.nn.Module): + if example_inputs is None: + if isinstance(example_kwarg_inputs, dict): + example_inputs = example_kwarg_inputs + else: + raise RuntimeError("example_kwarg_inputs should be a dict") + return trace_module( + func, + {"forward": example_inputs}, + None, + check_trace, + wrap_check_inputs(check_inputs), + check_tolerance, + strict, + _force_outplace, + _module_class, + example_inputs_is_kwarg=isinstance(example_kwarg_inputs, dict), + _store_inputs=_store_inputs, + ) + if ( + hasattr(func, "__self__") + and isinstance(func.__self__, torch.nn.Module) + and func.__name__ == "forward" + ): + if example_inputs is None: + if isinstance(example_kwarg_inputs, dict): + example_inputs = example_kwarg_inputs + else: + raise RuntimeError("example_kwarg_inputs should be a dict") + return trace_module( + func.__self__, + {"forward": example_inputs}, + None, + check_trace, + wrap_check_inputs(check_inputs), + check_tolerance, + strict, + _force_outplace, + _module_class, + example_inputs_is_kwarg=isinstance(example_kwarg_inputs, dict), + _store_inputs=_store_inputs, + ) + + # Special case for common case of passing a single Tensor + if ( + isinstance(example_inputs, (torch.Tensor, dict)) + and example_kwarg_inputs is None + ): + example_inputs = (example_inputs,) + # done primarily so that weird iterables fail here and not pybind11 code + elif example_kwarg_inputs is None and not isinstance(example_inputs, tuple): + # pyrefly: ignore [bad-argument-type] + example_inputs = tuple(example_inputs) + + var_lookup_fn = _create_interpreter_name_lookup_fn(0) + + if hasattr(func, "__self__") and isinstance(func.__self__, torch.nn.Module): + raise AttributeError( + "trace doesn't support compiling individual module's functions.\n" + "Please use trace_module" + ) + + name = _qualified_name(func) + if isinstance(example_kwarg_inputs, dict): + example_inputs = example_kwarg_inputs + traced = torch._C._create_function_from_trace_with_dict( + name, + func, + example_kwarg_inputs, + var_lookup_fn, + strict, + _force_outplace, + get_callable_argument_names(func), + ) + else: + traced = torch._C._create_function_from_trace( + name, + func, + # pyrefly: ignore [bad-argument-type] + example_inputs, + var_lookup_fn, + strict, + _force_outplace, + get_callable_argument_names(func), + ) + + # Check the trace against new traces created from user-specified inputs + if check_trace: + if check_inputs is not None: + _check_trace( + check_inputs, + func, + traced, + check_tolerance, + strict, + _force_outplace, + False, + _module_class, + example_inputs_is_kwarg=isinstance(example_kwarg_inputs, dict), + ) + else: + _check_trace( + [example_inputs], + func, + traced, + check_tolerance, + strict, + _force_outplace, + False, + _module_class, + example_inputs_is_kwarg=isinstance(example_kwarg_inputs, dict), + ) + + # Allow torch.compile() to inline + traced._torchdynamo_inline = func # type: ignore[attr-defined] + return traced + + +class _ExportType(str, Enum): + DIRECT_EXPORT = "DIRECT_EXPORT" + TRACE_AND_EXPORT = "TRACE_AND_EXPORT" + SOURCE_TO_SOURCE = "SOURCE_TO_SOURCE" + + def __str__(self) -> str: + return self.value + + +class _ExportOutcome(str, Enum): + SUCCESS = "SUCCESS" + FAILED_TO_EXPORT = "FAILED_TO_EXPORT" + FAILED_TO_RUN = "FAILED_TO_RUN" + ACCURACY_ERROR = "ACCURACY_ERROR" + + def __str__(self) -> str: + return self.value + + +def trace( + func, + example_inputs=None, + optimize=None, + check_trace=True, + check_inputs=None, + check_tolerance=1e-5, + strict=True, + _force_outplace=False, + _module_class=None, + _compilation_unit=_python_cu, + example_kwarg_inputs=None, + _store_inputs=True, +): + r""" + Trace a function and return an executable or :class:`ScriptFunction` that will be optimized using just-in-time compilation. + + Tracing is ideal for code that operates only on + ``Tensor``\\s and lists, dictionaries, and + tuples of ``Tensor``\\s. + + Using `torch.jit.trace` and `torch.jit.trace_module`, you can turn an + existing module or Python function into a TorchScript + :class:`ScriptFunction` or :class:`ScriptModule`. You must provide example + inputs, and we run the function, recording the operations performed on all + the tensors. + + * The resulting recording of a standalone function produces `ScriptFunction`. + * The resulting recording of `nn.Module.forward` or `nn.Module` produces + `ScriptModule`. + + This module also contains any parameters that the original + module had as well. + + Warning: + Tracing only correctly records functions and modules which are not data + dependent (e.g., do not have conditionals on data in tensors) and do not have + any untracked external dependencies (e.g., perform input/output or + access global variables). Tracing only records operations done when the given + function is run on the given tensors. Therefore, the returned + `ScriptModule` will always run the same traced graph on any input. This + has some important implications when your module is expected to run + different sets of operations, depending on the input and/or the module + state. For example, + + * Tracing will not record any control-flow like if-statements or loops. + When this control-flow is constant across your module, this is fine + and it often inlines the control-flow decisions. But sometimes the + control-flow is actually part of the model itself. For instance, a + recurrent network is a loop over the (possibly dynamic) length of an + input sequence. + * In the returned :class:`ScriptModule`, operations that have different + behaviors in ``training`` and ``eval`` modes will always behave as if + it is in the mode it was in during tracing, no matter which mode the + `ScriptModule` is in. + + In cases like these, tracing would not be appropriate and + :func:`scripting ` is a better choice. If you trace + such models, you may silently get incorrect results on subsequent + invocations of the model. The tracer will try to emit warnings when + doing something that may cause an incorrect trace to be produced. + + Args: + func (callable or torch.nn.Module): A Python function or `torch.nn.Module` + that will be run with `example_inputs`. `func` arguments and return + values must be tensors or (possibly nested) tuples that contain + tensors. When a module is passed `torch.jit.trace`, only the + ``forward`` method is run and traced (see :func:`torch.jit.trace + ` for details). + + Keyword arguments: + example_inputs (tuple or torch.Tensor or None, optional): A tuple of example + inputs that will be passed to the function while tracing. + Default: ``None``. Either this argument or ``example_kwarg_inputs`` + should be specified. The resulting trace can be run with inputs of + different types and shapes assuming the traced operations support those + types and shapes. `example_inputs` may also be a single Tensor in which + case it is automatically wrapped in a tuple. When the value is None, + ``example_kwarg_inputs`` should be specified. + + check_trace (``bool``, optional): Check if the same inputs run through + traced code produce the same outputs. Default: ``True``. You might want + to disable this if, for example, your network contains non- + deterministic ops or if you are sure that the network is correct despite + a checker failure. + + check_inputs (list of tuples, optional): A list of tuples of input + arguments that should be used to check the trace against what is + expected. Each tuple is equivalent to a set of input arguments that + would be specified in ``example_inputs``. For best results, pass in + a set of checking inputs representative of the space of shapes and + types of inputs you expect the network to see. If not specified, + the original ``example_inputs`` are used for checking + check_tolerance (float, optional): Floating-point comparison tolerance + to use in the checker procedure. This can be used to relax the + checker strictness in the event that results diverge numerically + for a known reason, such as operator fusion. + strict (``bool``, optional): run the tracer in a strict mode or not + (default: ``True``). Only turn this off when you want the tracer to + record your mutable container types (currently ``list``/``dict``) + and you are sure that the container you are using in your + problem is a ``constant`` structure and does not get used as + control flow (if, for) conditions. + example_kwarg_inputs (dict, optional): This parameter is a pack of keyword + arguments of example inputs that will be passed to the function while + tracing. Default: ``None``. Either this argument or ``example_inputs`` + should be specified. The dict will be unpacking by the arguments name + of the traced function. If the keys of the dict don't not match with + the traced function's arguments name, a runtime exception will be raised. + + Returns: + If `func` is `nn.Module` or ``forward`` of `nn.Module`, `trace` returns + a :class:`ScriptModule` object with a single ``forward`` method + containing the traced code. The returned `ScriptModule` will + have the same set of sub-modules and parameters as the original + ``nn.Module``. If ``func`` is a standalone function, ``trace`` + returns `ScriptFunction`. + + Example (tracing a function): + + .. testcode:: + + import torch + + def foo(x, y): + return 2 * x + y + + # Run `foo` with the provided inputs and record the tensor operations + traced_foo = torch.jit.trace(foo, (torch.rand(3), torch.rand(3))) + + # `traced_foo` can now be run with the TorchScript interpreter or saved + # and loaded in a Python-free environment + + Example (tracing an existing module):: + + import torch + import torch.nn as nn + + + class Net(nn.Module): + def __init__(self) -> None: + super().__init__() + self.conv = nn.Conv2d(1, 1, 3) + + def forward(self, x): + return self.conv(x) + + + n = Net() + example_weight = torch.rand(1, 1, 3, 3) + example_forward_input = torch.rand(1, 1, 3, 3) + + # Trace a specific method and construct `ScriptModule` with + # a single `forward` method + module = torch.jit.trace(n.forward, example_forward_input) + + # Trace a module (implicitly traces `forward`) and construct a + # `ScriptModule` with a single `forward` method + module = torch.jit.trace(n, example_forward_input) + + """ + if sys.version_info >= (3, 14): + warnings.warn( + "`torch.jit.trace` is not supported in Python 3.14+ and may break. " + "Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + else: + warnings.warn( + "`torch.jit.trace` is deprecated. Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + if not _enabled: + return func + if optimize is not None: + warnings.warn( + "`optimize` is deprecated and has no effect. " + "Use `with torch.jit.optimized_execution()` instead", + FutureWarning, + stacklevel=2, + ) + + from torch._utils_internal import log_torchscript_usage + + traced_func = _trace_impl( + func, + example_inputs, + optimize, + check_trace, + check_inputs, + check_tolerance, + strict, + _force_outplace, + _module_class, + _compilation_unit, + example_kwarg_inputs, + _store_inputs, + ) + log_torchscript_usage("trace", model_id=_get_model_id(traced_func)) + return traced_func + + +_trace_module_map: Optional[dict[Any, Any]] = None + + +def trace_module( + mod, + inputs, + optimize=None, + check_trace=True, + check_inputs=None, + check_tolerance=1e-5, + strict=True, + _force_outplace=False, + _module_class=None, + _compilation_unit=_python_cu, + example_inputs_is_kwarg=False, + _store_inputs=True, +): + """ + Trace a module and return an executable :class:`ScriptModule` that will be optimized using just-in-time compilation. + + When a module is passed to :func:`torch.jit.trace `, only + the ``forward`` method is run and traced. With ``trace_module``, you can specify a dictionary of + method names to example inputs to trace (see the ``inputs``) argument below. + + See :func:`torch.jit.trace ` for more information on tracing. + + Args: + mod (torch.nn.Module): A ``torch.nn.Module`` containing methods whose names are + specified in ``inputs``. The given methods will be compiled + as a part of a single `ScriptModule`. + inputs (dict): A dict containing sample inputs indexed by method names in ``mod``. + The inputs will be passed to methods whose names correspond to inputs' + keys while tracing. + ``{ 'forward' : example_forward_input, 'method2': example_method2_input}`` + Keyword arguments: + check_trace (``bool``, optional): Check if the same inputs run through + traced code produce the same outputs. Default: ``True``. You might want + to disable this if, for example, your network contains non- + deterministic ops or if you are sure that the network is correct despite + a checker failure. + + check_inputs (list of dicts, optional): A list of dicts of input arguments that should be used + to check the trace against what is expected. Each tuple + is equivalent to a set of input arguments that would + be specified in ``inputs``. For best results, pass in a + set of checking inputs representative of the space of + shapes and types of inputs you expect the network to see. + If not specified, the original ``inputs`` are used for checking + check_tolerance (float, optional): Floating-point comparison tolerance to use in the checker procedure. + This can be used to relax the checker strictness in the event that + results diverge numerically for a known reason, such as operator fusion. + example_inputs_is_kwarg (``bool``, optional): This parameter indicate whether the example inputs is a pack + pack of keyword arguments. Default: ``False``. + + Returns: + A :class:`ScriptModule` object with a single ``forward`` method containing the traced code. + When ``func`` is a ``torch.nn.Module``, the returned :class:`ScriptModule` will have the same set of + sub-modules and parameters as ``func``. + + Example (tracing a module with multiple methods):: + + import torch + import torch.nn as nn + + + class Net(nn.Module): + def __init__(self) -> None: + super().__init__() + self.conv = nn.Conv2d(1, 1, 3) + + def forward(self, x): + return self.conv(x) + + def weighted_kernel_sum(self, weight): + return weight * self.conv.weight + + + n = Net() + example_weight = torch.rand(1, 1, 3, 3) + example_forward_input = torch.rand(1, 1, 3, 3) + + # Trace a specific method and construct `ScriptModule` with + # a single `forward` method + module = torch.jit.trace(n.forward, example_forward_input) + + # Trace a module (implicitly traces `forward`) and construct a + # `ScriptModule` with a single `forward` method + module = torch.jit.trace(n, example_forward_input) + + # Trace specific methods on a module (specified in `inputs`), constructs + # a `ScriptModule` with `forward` and `weighted_kernel_sum` methods + inputs = { + "forward": example_forward_input, + "weighted_kernel_sum": example_weight, + } + module = torch.jit.trace_module(n, inputs) + + """ + if sys.version_info >= (3, 14): + warnings.warn( + "`torch.jit.trace_method` is not supported in Python 3.14+ and may break. " + "Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + else: + warnings.warn( + "`torch.jit.trace_method` is deprecated. Please switch to `torch.compile` or `torch.export`.", + DeprecationWarning, + ) + if not _enabled: + return mod + if optimize is not None: + warnings.warn( + "`optimize` is deprecated and has no effect. " + "Use `with torch.jit.optimized_execution()` instead", + FutureWarning, + stacklevel=2, + ) + + var_lookup_fn = _create_interpreter_name_lookup_fn(0) + + if not isinstance(mod, torch.nn.Module): + raise AttributeError("expected torch.nn.Module as the first argument") + + if not isinstance(inputs, dict): + raise AttributeError("expected a dictionary of (method_name, input) pairs") + + old_module_map = torch.jit._trace._trace_module_map + try: + trace_module_map: dict[Any, Any] = {} + + def register_submods(mod, prefix): + for name, child in mod.named_children(): + submod_qualname = prefix + "." + name + trace_module_map[child] = submod_qualname + register_submods(child, submod_qualname) + + trace_module_map["__module"] = mod + torch.jit._trace._trace_module_map = trace_module_map + register_submods(mod, "__module") + + module = make_module(mod, _module_class, _compilation_unit) + + for method_name, example_inputs in inputs.items(): + if method_name == "forward": + # "forward" is a special case because we need to trace + # `Module.__call__`, which sets up some extra tracing, but uses + # argument names of the real `Module.forward` method. + func = mod + forward_method = getattr(mod, method_name) + argument_names = get_callable_argument_names(forward_method) + else: + func = getattr(mod, method_name) + argument_names = get_callable_argument_names(func) + + if isinstance(example_inputs, dict) and example_inputs_is_kwarg: + # Raise exception when the user provided key names are not aligned with forward() method's arguments' name/ + for key in example_inputs: + if key not in argument_names: + valid_arguments = "[" + ",".join(argument_names) + "]" + raise NameError( + f"""'{key}' is not in forward() method's arguments, + valid arguments name are {valid_arguments}""" + ) + module._c._create_method_from_trace_with_dict( + method_name, + func, + example_inputs, + var_lookup_fn, + strict, + _force_outplace, + argument_names, + _store_inputs, + ) + else: + example_inputs = make_tuple(example_inputs) + module._c._create_method_from_trace( + method_name, + func, + example_inputs, + var_lookup_fn, + strict, + _force_outplace, + argument_names, + _store_inputs, + ) + + check_trace_method = module._c._get_method(method_name) + + # Check the trace against new traces created from user-specified inputs + if check_trace: + if check_inputs is not None: + _check_trace( + check_inputs, + func, + check_trace_method, + check_tolerance, + strict, + _force_outplace, + True, + _module_class, + example_inputs_is_kwarg=example_inputs_is_kwarg, + ) + else: + _check_trace( + [inputs], + func, + check_trace_method, + check_tolerance, + strict, + _force_outplace, + True, + _module_class, + example_inputs_is_kwarg=example_inputs_is_kwarg, + ) + finally: + torch.jit._trace._trace_module_map = old_module_map + + return module + + +def is_tracing(): + """Return a boolean value. + + Returns ``True`` in tracing (if a function is called during the + tracing of code with ``torch.jit.trace``) and ``False`` otherwise. + """ + if is_scripting(): + return False + return torch._C._is_tracing() + + +class TracedModule(ScriptModule): + _disable_script_meta = True + + def __init__(self, orig, id_set=None, _compilation_unit=None): + # XXX: orig can be a nn.Module or a function! + super().__init__() + assert isinstance(orig, torch.nn.Module) + + # Copy a subset of `orig` to a temporary nn.Module. + # This is a way to customize what will actually get compiled by create_script_module + id_set = set() + + # This allows us to preserve the original module's qualified name by defining a new + # type with the attribute _jit_override_qualname. In torch._jit_internal._qualified_name + # we have a special case that will look up this attribute to override whatever qualname + # we would get from the python type system + class QualnameWrapper(torch.nn.Module): + pass + + QualnameWrapper._jit_override_qualname = torch._jit_internal._qualified_name( # type: ignore[attr-defined] + type(orig) + ) + + tmp_module = QualnameWrapper() + + def check_unique(param): + if param in id_set: + raise ValueError( + "TracedModules don't support parameter sharing between modules" + ) + id_set.add(param) + + tmp_module.training = orig.training + + for name, param in orig._parameters.items(): + if param is not None: + tmp_module._parameters[name] = param + check_unique(param) + for name, buf in orig._buffers.items(): + if buf is not None: + tmp_module._buffers[name] = buf + check_unique(buf) + for name, val in orig.__dict__.items(): + if ( + torch._C._jit_is_script_object(val) + and name not in orig._parameters + and name not in orig._buffers + ): + setattr(tmp_module, name, val) + + if orig._backward_hooks: + raise ValueError( + "Modules that have backward hooks assigned can't be compiled: " + + str(orig) + ) + + for name, submodule in orig._modules.items(): + if submodule is None: + continue + tmp_module._modules[name] = make_module( + submodule, TracedModule, _compilation_unit=None + ) + + script_module = torch.jit._recursive.create_script_module( + tmp_module, lambda module: (), share_types=False, is_tracing=True + ) + + self.__dict__["_name"] = type(orig).__name__ + self.__dict__["_actual_script_module"] = script_module + for name in ("_parameters", "_buffers", "_modules", "training"): + delattr(self, name) + + def forward(self, *args, **kwargs): + raise RuntimeError("Trace submodules cannot be called.") + + def __getattr__(self, attr): + if "_actual_script_module" not in self.__dict__: + return super().__getattr__(attr) + return getattr(self._actual_script_module, attr) + + def __setattr__(self, attr, value): + if "_actual_script_module" not in self.__dict__: + return super().__setattr__(attr, value) + setattr(self._actual_script_module, attr, value) + + def _get_name(self): + return self._name + + def extra_repr(self): + return f"original_name={self._name}" + + +class TopLevelTracedModule(TracedModule): + forward: Callable[..., Any] = _CachedForward() # type: ignore[assignment] + + def _reconstruct(self, cpp_module): + """ + Re-construct an instance of TopLevelTracedModule using an instance of a C++ module. + + Args: + cpp_module: The C++ module that this TopLevelTracedModule will be rebuilt around. + """ + self.__dict__["_actual_script_module"]._reconstruct(cpp_module) + + +def _script_if_tracing(fn: Callable[P, R]) -> Callable[P, R]: + @functools.wraps(fn) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: + if not is_tracing(): + # Not tracing, don't do anything + return fn(*args, **kwargs) + + compiled_fn: Callable[P, R] = script(wrapper.__original_fn) # type: ignore[attr-defined] + return compiled_fn(*args, **kwargs) + + wrapper.__original_fn = fn # type: ignore[attr-defined] + wrapper.__script_if_tracing_wrapper = True # type: ignore[attr-defined] + + return wrapper + + +def _get_trace_graph( + f, + args=(), + kwargs=None, + strict=True, + _force_outplace=False, + return_inputs=False, + _return_inputs_states=False, +): + """Return a tuple on tracing a function or model. + + .. warning:: + This function is internal-only and should only be used by the ONNX + exporter. If you are trying to get a graph through tracing, please go + through the public API instead:: + + trace = torch.jit.trace(nn.LSTMCell(), (input, hidden)) + trace_graph = trace.graph + + Trace a function or model, returning a tuple consisting of the both the + *trace* of an execution, as well as the original return value. If return_inputs, + also returns the trace inputs as part of the tuple + + Tracing is guaranteed not to change the semantics of the function/module + that is traced. + + Args: + f (torch.nn.Module or function): the function or module + to be traced. + args (tuple or Tensor): the positional arguments to pass to the + function/module to be traced. A non-tuple is assumed to + be a single positional argument to be passed to the model. + kwargs (dict): the keyword arguments to pass to the function/module + to be traced. + + Example (trace a cell): + + .. testcode:: + + trace = torch.jit.trace(nn.LSTMCell(), (input, hidden)) + """ + if kwargs is None: + kwargs = {} + if not isinstance(args, tuple): + args = (args,) + outs = ONNXTracedModule( + f, strict, _force_outplace, return_inputs, _return_inputs_states + )(*args, **kwargs) + return outs diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/annotations.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/annotations.py new file mode 100644 index 0000000000000000000000000000000000000000..cf1be7bac8f9d7d26709c35c040c96afef5446f2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/annotations.py @@ -0,0 +1,555 @@ +# mypy: allow-untyped-defs +import ast +import builtins +import dis +import enum +import inspect +import re +import typing +import warnings +from textwrap import dedent + +import torch +from torch._C import ( + _GeneratorType, + AnyType, + AwaitType, + BoolType, + ComplexType, + DeviceObjType, + DictType, + EnumType, + FloatType, + FutureType, + InterfaceType, + IntType, + ListType, + NoneType, + NumberType, + OptionalType, + StreamObjType, + StringType, + TensorType, + TupleType, + UnionType, +) +from torch._jit_internal import ( # type: ignore[attr-defined] + _Await, + _qualified_name, + Any, + BroadcastingList1, + BroadcastingList2, + BroadcastingList3, + Dict, + Future, + is_await, + is_dict, + is_future, + is_ignored_fn, + is_list, + is_optional, + is_tuple, + is_union, + List, + Optional, + Tuple, + Union, +) +from torch._sources import get_source_lines_and_file + +from ._state import _get_script_class + + +if torch.distributed.rpc.is_available(): + from torch._C import RRefType + from torch._jit_internal import is_rref, RRef + +from torch._ops import OpOverloadPacket + + +class Module: + def __init__(self, name, members) -> None: + self.name = name + self.members = members + + def __getattr__(self, name): + try: + return self.members[name] + except KeyError: + raise RuntimeError( + f"Module {self.name} has no member called {name}" + ) from None + + +class EvalEnv: + env = { + "torch": Module("torch", {"Tensor": torch.Tensor}), + "Tensor": torch.Tensor, + "typing": Module("typing", {"Tuple": Tuple}), + "Tuple": Tuple, + "List": List, + "Dict": Dict, + "Optional": Optional, + "Union": Union, + "Future": Future, + "Await": _Await, + } + + def __init__(self, rcb) -> None: + self.rcb = rcb + if torch.distributed.rpc.is_available(): + # pyrefly: ignore [unsupported-operation] + self.env["RRef"] = RRef + + def __getitem__(self, name): + if name in self.env: + return self.env[name] + if self.rcb is not None: + return self.rcb(name) + return getattr(builtins, name, None) + + +def get_signature(fn, rcb, loc, is_method): + if isinstance(fn, OpOverloadPacket): + signature = try_real_annotations(fn.op, loc) + else: + signature = try_real_annotations(fn, loc) + if signature is not None and is_method: + # If this is a method, then the signature will include a type for + # `self`, but type comments do not contain a `self`. So strip it + # away here so everything is consistent (`inspect.ismethod` does + # not work here since `fn` is unbound at this point) + param_types, return_type = signature + param_types = param_types[1:] + signature = (param_types, return_type) + + if signature is None: + type_line, source = None, None + try: + source = dedent("".join(get_source_lines_and_file(fn)[0])) + type_line = get_type_line(source) + except TypeError: + pass + # This might happen both because we failed to get the source of fn, or + # because it didn't have any annotations. + if type_line is not None: + signature = parse_type_line(type_line, rcb, loc) + + return signature + + +def is_function_or_method(the_callable): + # A stricter version of `inspect.isroutine` that does not pass for built-in + # functions + return inspect.isfunction(the_callable) or inspect.ismethod(the_callable) + + +def is_vararg(the_callable): + if not is_function_or_method(the_callable) and callable(the_callable): # noqa: B004 + # If `the_callable` is a class, de-sugar the call so we can still get + # the signature + the_callable = the_callable.__call__ + + if is_function_or_method(the_callable): + return inspect.getfullargspec(the_callable).varargs is not None + else: + return False + + +def get_param_names(fn, n_args): + if isinstance(fn, OpOverloadPacket): + fn = fn.op + + if ( + not is_function_or_method(fn) + and callable(fn) + and is_function_or_method(fn.__call__) + ): # noqa: B004 + # De-sugar calls to classes + fn = fn.__call__ + + if is_function_or_method(fn): + if is_ignored_fn(fn): + fn = inspect.unwrap(fn) + return inspect.getfullargspec(fn).args + else: + # The `fn` was not a method or function (maybe a class with a __call__ + # method, so use a default param name list) + return [str(i) for i in range(n_args)] + + +def check_fn(fn, loc) -> None: + # Make sure the function definition is not a class instantiation + try: + source = dedent("".join(get_source_lines_and_file(fn)[0])) + except (OSError, TypeError): + return + if source is None: + return + + py_ast = ast.parse(source) + if len(py_ast.body) == 1 and isinstance(py_ast.body[0], ast.ClassDef): + raise torch.jit.frontend.FrontendError( + loc, + f"Cannot instantiate class '{py_ast.body[0].name}' in a script function", + ) + if len(py_ast.body) != 1 or not isinstance(py_ast.body[0], ast.FunctionDef): + raise torch.jit.frontend.FrontendError( + loc, "Expected a single top-level function" + ) + + +def _eval_no_call(stmt, glob, loc): + """Evaluate statement as long as it does not contain any method/function calls.""" + bytecode = compile(stmt, "", mode="eval") + for insn in dis.get_instructions(bytecode): + if "CALL" in insn.opname: + raise RuntimeError( + f"Type annotation should not contain calls, but '{stmt}' does" + ) + return eval(bytecode, glob, loc) # type: ignore[arg-type] # noqa: P204 + + +def parse_type_line(type_line, rcb, loc): + """Parse a type annotation specified as a comment. + + Example inputs: + # type: (Tensor, torch.Tensor) -> Tuple[Tensor] + # type: (Tensor, Tuple[Tensor, Tensor]) -> Tensor + """ + arg_ann_str, ret_ann_str = split_type_line(type_line) + + try: + arg_ann = _eval_no_call(arg_ann_str, {}, EvalEnv(rcb)) + except (NameError, SyntaxError) as e: + raise RuntimeError( + "Failed to parse the argument list of a type annotation" + ) from e + + if not isinstance(arg_ann, tuple): + arg_ann = (arg_ann,) + + try: + ret_ann = _eval_no_call(ret_ann_str, {}, EvalEnv(rcb)) + except (NameError, SyntaxError) as e: + raise RuntimeError( + "Failed to parse the return type of a type annotation" + ) from e + + arg_types = [ann_to_type(ann, loc) for ann in arg_ann] + return arg_types, ann_to_type(ret_ann, loc) + + +def get_type_line(source): + """Try to find the line containing a comment with the type annotation.""" + type_comment = "# type:" + + lines = source.split("\n") + lines = list(enumerate(lines)) + type_lines = list(filter(lambda line: type_comment in line[1], lines)) + # `type: ignore` comments may be needed in JIT'ed functions for mypy, due + # to the hack in torch/_VF.py. + + # An ignore type comment can be of following format: + # 1) type: ignore + # 2) type: ignore[rule-code] + # This ignore statement must be at the end of the line + + # adding an extra backslash before the space, to avoid triggering + # one of the checks in .github/workflows/lint.yml + type_pattern = re.compile("# type:\\ ignore(\\[[a-zA-Z-]+\\])?$") + type_lines = list(filter(lambda line: not type_pattern.search(line[1]), type_lines)) + + if len(type_lines) == 0: + # Catch common typo patterns like extra spaces, typo in 'ignore', etc. + wrong_type_pattern = re.compile("#[\t ]*type[\t ]*(?!: ignore(\\[.*\\])?$):") + wrong_type_lines = list( + filter(lambda line: wrong_type_pattern.search(line[1]), lines) + ) + if len(wrong_type_lines) > 0: + raise RuntimeError( + "The annotation prefix in line " + + str(wrong_type_lines[0][0]) + + " is probably invalid.\nIt must be '# type:'" + + "\nSee PEP 484 (https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code)" # noqa: B950 + + "\nfor examples" + ) + return None + elif len(type_lines) == 1: + # Only 1 type line, quit now + return type_lines[0][1].strip() + + # Parse split up argument types according to PEP 484 + # https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code + return_line = None + parameter_type_lines = [] + for line_num, line in type_lines: + if "# type: (...) -> " in line: + return_line = (line_num, line) + break + elif type_comment in line: + parameter_type_lines.append(line) + if return_line is None: + raise RuntimeError( + "Return type line '# type: (...) -> ...' not found on multiline " + "type annotation\nfor type lines:\n" + + "\n".join([line[1] for line in type_lines]) + + "\n(See PEP 484 https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code)" + ) + + def get_parameter_type(line): + item_type = line[line.find(type_comment) + len(type_comment) :] + return item_type.strip() + + types = map(get_parameter_type, parameter_type_lines) + parameter_types = ", ".join(types) + + return return_line[1].replace("...", parameter_types) + + +def split_type_line(type_line): + """Split the comment with the type annotation into parts for argument and return types. + + For example, for an input of: + # type: (Tensor, torch.Tensor) -> Tuple[Tensor, Tensor] + + This function will return: + ("(Tensor, torch.Tensor)", "Tuple[Tensor, Tensor]") + + """ + start_offset = len("# type:") + try: + arrow_pos = type_line.index("->") + except ValueError: + raise RuntimeError( + "Syntax error in type annotation (couldn't find `->`)" + ) from None + return type_line[start_offset:arrow_pos].strip(), type_line[arrow_pos + 2 :].strip() + + +def try_real_annotations(fn, loc): + """Try to use the Py3.5+ annotation syntax to get the type.""" + try: + # Note: anything annotated as `Optional[T]` will automatically + # be returned as `Union[T, None]` per + # https://github.com/python/cpython/blob/main/Lib/typing.py#L732 + sig = inspect.signature(fn) + except ValueError: + return None + + all_annots = [sig.return_annotation] + [ + p.annotation for p in sig.parameters.values() + ] + if all(ann is sig.empty for ann in all_annots): + return None + + arg_types = [ann_to_type(p.annotation, loc) for p in sig.parameters.values()] + return_type = ann_to_type(sig.return_annotation, loc) + return arg_types, return_type + + +# Finds common type for enum values belonging to an Enum class. If not all +# values have the same type, AnyType is returned. +def get_enum_value_type(e: type[enum.Enum], loc): + enum_values: List[enum.Enum] = list(e) + if not enum_values: + raise ValueError(f"No enum values defined for: '{e.__class__}'") + + types = {type(v.value) for v in enum_values} + ir_types = [try_ann_to_type(t, loc) for t in types] + + # If Enum values are of different types, an exception will be raised here. + # Even though Python supports this case, we chose to not implement it to + # avoid overcomplicate logic here for a rare use case. Please report a + # feature request if you find it necessary. + res = torch._C.unify_type_list(ir_types) + if not res: + return AnyType.get() + return res + + +def is_tensor(ann) -> bool: + if issubclass(ann, torch.Tensor): + return True + + if issubclass( + ann, + ( + torch.LongTensor, + torch.DoubleTensor, + torch.FloatTensor, + torch.IntTensor, + torch.ShortTensor, + torch.HalfTensor, + torch.CharTensor, + torch.ByteTensor, + torch.BoolTensor, + ), + ): + warnings.warn( + "TorchScript will treat type annotations of Tensor " + "dtype-specific subtypes as if they are normal Tensors. " + "dtype constraints are not enforced in compilation either.", + stacklevel=2, + ) + return True + + return False + + +def _fake_rcb(inp) -> None: + return None + + +def try_ann_to_type(ann, loc, rcb=None): + ann_args = typing.get_args(ann) # always returns a tuple! + + if ann is inspect.Signature.empty: + return TensorType.getInferred() + if ann is None: + return NoneType.get() + if inspect.isclass(ann) and is_tensor(ann): + return TensorType.get() + if is_tuple(ann): + # Special case for the empty Tuple type annotation `Tuple[()]` + if len(ann_args) == 1 and ann_args[0] == (): + return TupleType([]) + return TupleType([try_ann_to_type(a, loc) for a in ann_args]) + if is_list(ann): + elem_type = try_ann_to_type(ann_args[0], loc) + if elem_type: + return ListType(elem_type) + if is_dict(ann): + key = try_ann_to_type(ann_args[0], loc) + value = try_ann_to_type(ann_args[1], loc) + # Raise error if key or value is None + if key is None: + raise ValueError( + f"Unknown type annotation: '{ann_args[0]}' at {loc.highlight()}" + ) + if value is None: + raise ValueError( + f"Unknown type annotation: '{ann_args[1]}' at {loc.highlight()}" + ) + return DictType(key, value) + if is_optional(ann): + if issubclass(ann_args[1], type(None)): + contained = ann_args[0] + else: + contained = ann_args[1] + valid_type = try_ann_to_type(contained, loc) + msg = "Unsupported annotation {} could not be resolved because {} could not be resolved. At\n{}" + assert valid_type, msg.format(repr(ann), repr(contained), repr(loc)) + return OptionalType(valid_type) + if is_union(ann): + # TODO: this is hack to recognize NumberType + if set(ann_args) == {int, float, complex}: + return NumberType.get() + inner: List = [] + # We need these extra checks because both `None` and invalid + # values will return `None` + # TODO: Determine if the other cases need to be fixed as well + for a in typing.get_args(ann): + if a is None: + inner.append(NoneType.get()) + maybe_type = try_ann_to_type(a, loc) + msg = "Unsupported annotation {} could not be resolved because {} could not be resolved. At\n{}" + assert maybe_type, msg.format(repr(ann), repr(maybe_type), repr(loc)) + inner.append(maybe_type) + return UnionType(inner) # type: ignore[arg-type] + if torch.distributed.rpc.is_available() and is_rref(ann): + return RRefType(try_ann_to_type(ann_args[0], loc)) + if is_future(ann): + return FutureType(try_ann_to_type(ann_args[0], loc)) + if is_await(ann): + elementType = try_ann_to_type(ann_args[0], loc) if ann_args else AnyType.get() + return AwaitType(elementType) + if ann is float: + return FloatType.get() + if ann is complex: + return ComplexType.get() + if ann is int or ann is torch.SymInt: + return IntType.get() + if ann is str: + return StringType.get() + if ann is bool: + return BoolType.get() + if ann is Any: + return AnyType.get() + if ann is type(None): + return NoneType.get() + if inspect.isclass(ann) and hasattr(ann, "__torch_script_interface__"): + return InterfaceType(ann.__torch_script_interface__) + if ann is torch.device: + return DeviceObjType.get() + if ann is torch.Generator: + return _GeneratorType.get() + if ann is torch.Stream: + return StreamObjType.get() + if ann is torch.dtype: + return IntType.get() # dtype not yet bound in as its own type + if ann is torch.qscheme: + return IntType.get() # qscheme not yet bound in as its own type + + if inspect.isclass(ann) and issubclass(ann, enum.Enum): + if _get_script_class(ann) is None: + scripted_class = torch.jit._script._recursive_compile_class(ann, loc) + name = scripted_class.qualified_name() + else: + name = _qualified_name(ann) + return EnumType(name, get_enum_value_type(ann, loc), list(ann)) + if inspect.isclass(ann): + maybe_script_class = _get_script_class(ann) + if maybe_script_class is not None: + return maybe_script_class + if torch._jit_internal.can_compile_class(ann): + return torch.jit._script._recursive_compile_class(ann, loc) + + # Maybe resolve a NamedTuple to a Tuple Type + if rcb is None: + rcb = _fake_rcb + return torch._C._resolve_type_from_object(ann, loc, rcb) + + +def ann_to_type(ann, loc, rcb=None): + the_type = try_ann_to_type(ann, loc, rcb) + if the_type is not None: + return the_type + raise ValueError(f"Unknown type annotation: '{ann}' at {loc.highlight()}") + + +__all__ = [ + "Any", + "List", + "BroadcastingList1", + "BroadcastingList2", + "BroadcastingList3", + "Tuple", + "is_tuple", + "is_list", + "Dict", + "is_dict", + "is_optional", + "is_union", + "TensorType", + "TupleType", + "FloatType", + "ComplexType", + "IntType", + "ListType", + "StringType", + "DictType", + "AnyType", + "Module", + # TODO: Consider not exporting these during wildcard import (reserve + # that for the types; for idiomatic typing code.) + "get_signature", + "check_fn", + "get_param_names", + "parse_type_line", + "get_type_line", + "split_type_line", + "try_real_annotations", + "try_ann_to_type", + "ann_to_type", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/frontend.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/frontend.py new file mode 100644 index 0000000000000000000000000000000000000000..9f686a5a626f3e96e20179c313b026f60a9bb845 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/frontend.py @@ -0,0 +1,1284 @@ +# mypy: allow-untyped-defs +import ast +import copy +import dataclasses +import inspect +import re +import string +from collections import namedtuple +from textwrap import dedent + +import torch +import torch.jit.annotations +from torch import _jit_internal +from torch._C._jit_tree_views import ( + Apply, + Assert, + Assign, + Attribute, + AugAssign, + BinOp, + Break, + ClassDef, + Const, + Continue, + Decl, + Def, + Delete, + DictComp, + DictLiteral, + Dots, + EmptyTypeAnnotation, + ExprStmt, + FalseLiteral, + For, + Ident, + If, + ListComp, + ListLiteral, + NoneLiteral, + Param, + Pass, + Property, + Raise, + Return, + Select, + SliceExpr, + Starred, + Stmt, + StringLiteral, + Subscript, + TernaryIf, + TrueLiteral, + TupleLiteral, + UnaryOp, + Var, + While, + With, + WithItem, +) +from torch._jit_internal import ( # noqa: F401 + _is_drop_fn, + FunctionModifiers, + is_static_fn, + should_drop, +) +from torch._sources import ( + get_source_lines_and_file, + make_source_context, + parse_def, + ParsedDef as _ParsedDef, +) +from torch.jit._dataclass_impls import DATACLASS_MAGIC_METHODS +from torch.jit._monkeytype_config import get_qualified_name, monkeytype_trace + + +# Borrowed from cPython implementation +# https://github.com/python/cpython/blob/561612d8456cfab5672c9b445521113b847bd6b3/Lib/textwrap.py#L411# + +_reserved_prefix = "__jit" +_reserved_names = {"print"} +_identifier_chars = set(string.ascii_lowercase + string.ascii_uppercase + string.digits) + + +def is_reserved_name(name): + return name.startswith(_reserved_prefix) or name in _reserved_names + + +pretty_node_names = { + ast.FunctionDef: "function definitions", + ast.For: "for loops", + ast.Delete: "del statements", + ast.ClassDef: "class definitions", + ast.With: "with statements", + ast.Raise: "raise statements", + ast.Assert: "assertions", + ast.Import: "import statements", + ast.ImportFrom: "import statements", + ast.Global: "global variables", + ast.Break: "break statements", + ast.Continue: "continue statements", +} + +node_start_tokens = { + ast.FunctionDef: "def", + ast.For: "for", + ast.Delete: "del", + ast.ClassDef: "class", + ast.With: "with", + ast.Raise: "raise", + ast.Assert: "assert", + ast.Import: "import", + ast.ImportFrom: "from", + ast.Global: "global", + ast.Break: "break", + ast.Continue: "continue", +} + +# pyrefly: ignore [no-matching-overload] +pretty_node_names.update( + { + ast.AsyncFunctionDef: "async function definitions", + ast.AsyncFor: "async for loops", + ast.AsyncWith: "async with statements", + ast.Try: "try blocks", + ast.Nonlocal: "nonlocal variables", + } +) + +# pyrefly: ignore [no-matching-overload] +node_start_tokens.update( + { + ast.AsyncFunctionDef: "async def", + ast.AsyncFor: "async for", + ast.AsyncWith: "async with", + ast.Try: "try", + ast.Nonlocal: "nonlocal", + } +) + +# pyrefly: ignore [no-matching-overload] +pretty_node_names.update( + { + ast.AnnAssign: "annotated assignments", + } +) +# NB: no specific token for AnnAssign + + +class FrontendError(Exception): + def __init__(self, source_range, msg) -> None: + self.source_range = source_range + self.msg = msg + + # This has to be instantiated here so the ErrorReport is accurate to the + # call stack when the FrontendError was raised + self.error_report = torch._C.ErrorReport(self.source_range) + + def __str__(self) -> str: + return self.msg + self.error_report.what().lstrip() + + +class NotSupportedError(FrontendError): + pass + + +class UnsupportedNodeError(NotSupportedError): + def __init__(self, ctx, offending_node, reason="") -> None: + # If we don't have a specific token, we default to length of 1 + node_type = type(offending_node) + range_len = len(node_start_tokens.get(node_type, " ")) + source_range = ctx.make_range( + offending_node.lineno, + offending_node.col_offset, + offending_node.col_offset + range_len, + ) + feature_name = pretty_node_names.get(node_type, node_type.__name__) + msg = f"{feature_name} {reason + ' ' if reason else ''}aren't supported" + super().__init__(source_range, msg) + + +class FrontendTypeError(FrontendError): + pass + + +def build_withitems(ctx, items): + items = [build_withitem(ctx, i) for i in items] + return list(items) + + +def build_stmts(ctx, stmts): + stmts = [build_stmt(ctx, s) for s in stmts] + return list(filter(None, stmts)) + + +def get_class_properties(cls, self_name): + """ + Get a list of Property objects representing the properties of a class. + + Args: + cls: The class to get properties of. + self_name: The name of the class that the properties should belong to. + Returns: + A list of Property objects corresponding to the properties of cls. Property + here refers to the subclass of TreeView. + """ + props = inspect.getmembers(cls, predicate=lambda m: isinstance(m, property)) + # Any property that should not compiled must be in this list on the Module. + unused_properties = getattr(cls, "__jit_unused_properties__", []) + + # Create Property TreeView objects from inspected property objects. + properties = [] + for prop in props: + if prop[0] not in unused_properties and not should_drop(prop[1].fget): + getter = get_jit_def( + prop[1].fget, f"__{prop[0]}_getter", self_name=self_name + ) + setter = ( + get_jit_def(prop[1].fset, f"__{prop[0]}_setter", self_name=self_name) + if prop[1].fset + else None + ) + properties.append( + Property(getter.range(), Ident(getter.range(), prop[0]), getter, setter) + ) + + return properties + + +def get_class_assigns(ctx, cls_ast): + assigns = [] + + def maybe_build_assign(builder, entry) -> None: + nonlocal assigns + try: + assigns.append(builder(ctx, entry)) + except NotSupportedError: + pass + + for entry in cls_ast.body: + if isinstance(entry, ast.Assign): + maybe_build_assign(StmtBuilder.build_Assign, entry) + elif isinstance(entry, ast.AnnAssign): + maybe_build_assign(StmtBuilder.build_AnnAssign, entry) + return assigns + + +def get_jit_class_def(cls, self_name): + """Get definitions for each method within the current class independently. + + Args: + cls: The class to get definition of. + self_name: The name of the class that the properties should belong to. + + Returns: + torch._C._jit_tree_views.ClassDef: A representation of the class, + the methods in the class and their definition as a tree. + """ + # TODO: proper overriding analysis when implementing class inheritance + methods = inspect.getmembers( + cls, + predicate=lambda m: (inspect.ismethod(m) or inspect.isfunction(m)) + and not is_static_fn(cls, m.__name__) + and m.__name__ in cls.__dict__ + and not _is_drop_fn(m), + ) + + def is_classmethod(fn): + return inspect.ismethod(fn) and getattr(fn, "__self__", None) == cls + + # Get and parse the source code for this class + sourcelines, file_lineno, filename = get_source_lines_and_file( + cls, torch._C.ErrorReport.call_stack() + ) + source = "".join(sourcelines) + + dedent_src = dedent(source) + py_ast = ast.parse(dedent_src) + + class_ast = py_ast.body[0] + assert isinstance(class_ast, ast.ClassDef) + + # Special case for dataclasses. In general we need access to the source code for + # an object in order to JIT compile it. But the dataclasses module dynamically synthesizes + # magic methods for classes, and we can't get the source code for these methods. As a + # workaround, we synthesize TorchScript-friendly implementations ourselves. + if dataclasses.is_dataclass(cls): + # Detect whether the user manually implemented any of the magic methods. If they did, + # we don't want to synthesize/override them. + overrides = { + method.name + for method in class_ast.body + if isinstance(method, ast.FunctionDef) + and method.name in DATACLASS_MAGIC_METHODS + } + for i, (name, _) in enumerate(methods): + # Is this a magic method we can synthesize? + synthesizer_fn = DATACLASS_MAGIC_METHODS.get(name) + if synthesizer_fn and name not in overrides: + parsed_def = synthesizer_fn(cls) + methods[i] = name, parsed_def + func = getattr(cls, name) + _jit_internal.loader.cache(func, parsed_def.source) + + method_defs = [ + get_jit_def(obj, name, self_name=self_name, is_classmethod=is_classmethod(obj)) + for (name, obj) in methods + ] + properties = get_class_properties(cls, self_name) + + leading_whitespace_len = len(source.split("\n", 1)[0]) - len( + dedent_src.split("\n", 1)[0] + ) + ctx = make_source_context( + source, filename, file_lineno, leading_whitespace_len, False + ) + assigns = get_class_assigns(ctx, class_ast) + + return build_class_def(ctx, class_ast, method_defs, properties, self_name, assigns) + + +def get_jit_def(fn, def_name, self_name=None, is_classmethod=False): + """ + Build a JIT AST (TreeView) from the given function. + + Args: + fn: A function object to compile or a pre-parsed ParsedDef object + def_name: The name to give to the resulting AST object. This is not + always the same as `fn.__name__`, for example: + def _forward(self): + ... + forward = _forward + In this case, the `__name__` attribute of the function object is "_forward", + but we want the result AST to have the name "forward". + self_name: If this function is a method, what the type name of `self` is. + """ + parsed_def = parse_def(fn) if not isinstance(fn, _ParsedDef) else fn + type_line = torch.jit.annotations.get_type_line(parsed_def.source) + fn_def = parsed_def.ast.body[0] + + if is_classmethod: + arg_name = fn_def.args.args[0].arg # type:ignore[union-attr] + # Insert a statement that assigns the first argument to the class + assign_stmt = ast.parse(f"{arg_name} = {self_name}").body[0] + fn_def.body.insert(0, assign_stmt) # type:ignore[union-attr] + + # Swap out the function signature and body if it is unused + if should_drop(fn): + unused_fn_def = ast.parse( + 'def unused_fn(self: Any):\n\traise RuntimeError("Cannot call @unused methods")' + ) + if len(unused_fn_def.body) != 1 or not isinstance( + unused_fn_def.body[0], ast.FunctionDef + ): + raise RuntimeError( + f"Expected a single top-level function: {parsed_def.filename}:{parsed_def.file_lineno}" + ) + unused_def = unused_fn_def.body[0] + fn_def.body = unused_def.body # type:ignore[union-attr] + # kwarg/vararg not supported by `build_def` + fn_def.args.kwarg = fn_def.args.vararg = None # type:ignore[union-attr] + for arg in fn_def.args.args + fn_def.args.kwonlyargs: # type:ignore[union-attr] + # Replace potentially unsupported type annotations by "Any" + arg.annotation = unused_def.args.args[0].annotation + if _is_drop_fn(fn): + # Dropping potentially unsupported return type annotation for jit._drop + fn_def.returns = None # type:ignore[union-attr] + fn_def.type_comment = None # type:ignore[union-attr] + + # If MonkeyType is installed, get all the consolidated type traces + # for the arguments from type_trace_db + type_trace_db = torch.jit._script._get_type_trace_db() + pdt_arg_types = None + if monkeytype_trace and not isinstance(fn, _ParsedDef): # type: ignore[truthy-function] + qualname = get_qualified_name(fn) + pdt_arg_types = type_trace_db.get_args_types(qualname) + + return build_def( + parsed_def.ctx, + fn_def, + type_line, + def_name, + self_name=self_name, + pdt_arg_types=pdt_arg_types, + ) + + +# TODO: more robust handling of recognizing ignore context manager +def is_torch_jit_ignore_context_manager(stmt) -> bool: + # checks if the statement is torch.jit.ignore context manager + if isinstance(stmt.items[0].context_expr, ast.Call): + # extract torch part + function = stmt.items[0].context_expr.func + if isinstance(function, ast.Attribute): + attr_name = function.attr + attr_value = function.value + if attr_name == "_IgnoreContextManager" and isinstance( + attr_value, ast.Attribute + ): + # there should be at most two nested attributes (e.g torch.jit._IgnoreContextManager) + if attr_value.attr == "jit" and isinstance(attr_value.value, ast.Name): + if attr_value.value.id == "torch": + return True + return False + + +class Builder: + def __call__(self, ctx, node): + method = getattr(self, "build_" + node.__class__.__name__, None) + if method is None: + raise UnsupportedNodeError(ctx, node) + return method(ctx, node) + + +def build_class_def(ctx, py_def, methods, properties, self_name, assigns): + r = ctx.make_range( + py_def.lineno, py_def.col_offset, py_def.col_offset + len("class") + ) + return ClassDef( + Ident(r, self_name), [Stmt(method) for method in methods], properties, assigns + ) + + +def build_def(ctx, py_def, type_line, def_name, self_name=None, pdt_arg_types=None): + body = py_def.body + r = ctx.make_range(py_def.lineno, py_def.col_offset, py_def.col_offset + len("def")) + + param_list = build_param_list(ctx, py_def.args, self_name, pdt_arg_types) + return_type = None + if getattr(py_def, "returns", None) is not None: + return_type = build_expr(ctx, py_def.returns) + + decl = Decl(r, param_list, return_type) + is_method = self_name is not None + if type_line is not None: + type_comment_decl = torch._C.parse_type_comment(type_line) + decl = torch._C.merge_type_from_type_comment( + decl, # type: ignore[arg-type] + type_comment_decl, + is_method, # type: ignore[assignment] + ) + + return Def(Ident(r, def_name), decl, build_stmts(ctx, body)) + + +_vararg_kwarg_err = ( + "Compiled functions can't take variable number of arguments " + "or use keyword-only arguments with defaults" +) + + +def build_param_list(ctx, py_args, self_name, pdt_arg_types=None): + if py_args.kwarg is not None: + expr = py_args.kwarg + ctx_range = ctx.make_range( + expr.lineno, expr.col_offset - 1, expr.col_offset + len(expr.arg) + ) + raise NotSupportedError(ctx_range, _vararg_kwarg_err) + if py_args.vararg is not None: + expr = py_args.vararg + ctx_range = ctx.make_range( + expr.lineno, expr.col_offset - 1, expr.col_offset + len(expr.arg) + ) + raise NotSupportedError(ctx_range, _vararg_kwarg_err) + if len(py_args.kw_defaults) > 0: + # kw_defaults is a list of the values for the kwargs (which default to None), + # so they don't actually have line numbers. + for arg in py_args.kw_defaults: + if arg is not None: + ctx_range = build_expr(ctx, arg).range() + raise NotSupportedError(ctx_range, _vararg_kwarg_err) + + # List of Tuple of args and type as inferred by profile directed typing + arg_and_types = [ + ( + arg, + pdt_arg_types[arg.arg] + if pdt_arg_types and bool(pdt_arg_types[arg.arg]) + else None, + ) + for arg in py_args.args + ] + arg_and_types_kwonlyargs = [ + ( + arg, + pdt_arg_types[arg.arg] + if pdt_arg_types and bool(pdt_arg_types[arg.arg]) + else None, + ) + for arg in py_args.kwonlyargs + ] + + result = [ + build_param(ctx, arg, self_name, kwarg_only=False, pdt_arg_type=arg_type) + for arg, arg_type in arg_and_types + ] + result += [ + build_param(ctx, arg, self_name, kwarg_only=True, pdt_arg_type=arg_type) + for arg, arg_type in arg_and_types_kwonlyargs + ] + return result + + +def build_param(ctx, py_arg, self_name, kwarg_only, pdt_arg_type=None): + # NB: In Python3 py_arg is a pair of (str arg, expr? annotation) + name = py_arg.arg + r = ctx.make_range(py_arg.lineno, py_arg.col_offset, py_arg.col_offset + len(name)) + if getattr(py_arg, "annotation", None) is not None: + annotation_expr = build_expr(ctx, py_arg.annotation) + elif pdt_arg_type: + annotation_expr = Var(Ident(r, pdt_arg_type)) + elif self_name is not None and name == "self": + annotation_expr = Var(Ident(r, self_name)) + else: + annotation_expr = EmptyTypeAnnotation(r) + return Param(annotation_expr, Ident(r, name), kwarg_only) + + +def build_ignore_context_manager(ctx, stmt): + InputType = namedtuple("InputType", ["name", "ann"]) + OutputType = namedtuple("OutputType", ["name", "ann"]) + + def process_ins_outs(args): + # parse the context manager to figure out inputs and outputs + # with their annotated types + # TODO: add input, output validator + inputs = [] + outputs = [] + for arg in args: + var_name = arg.arg + var_ann = arg.value.value + var_decl_type, var_ann = var_ann.split(":") + if var_decl_type == "inp": + inputs.append(InputType(var_name, var_ann)) + if var_decl_type == "out": + outputs.append(OutputType(var_name, var_ann)) + return inputs, outputs + + def create_unique_name_ext(ctx, stmt) -> str: + # extension will be based on the full path filename plus + # the line number of original context manager + fn = re.sub(r"[^a-zA-Z0-9_]", "_", ctx.filename) + return f"{fn}_{stmt.lineno}" + + def build_return_ann_stmt(outputs): + return_type_ann = "" + return_statement_str = "return " + if len(outputs) == 0: + return_type_ann += " -> None" + if len(outputs) == 1: + return_type_ann = " -> " + outputs[0].ann + return_statement_str += outputs[0].name + if len(outputs) > 1: + return_type_ann = " -> tuple" + return_type_ann += "[" + ", ".join([var.ann for var in outputs]) + "]" + return_statement_str += ", ".join([var.name for var in outputs]) + return return_type_ann, return_statement_str + + def build_args(args): + return ", ".join([arg.name for arg in args]) + + inputs, outputs = process_ins_outs(stmt.items[0].context_expr.keywords) + + # build the replacement function str with given inputs and outputs + ignore_function_name = "func_ignore_" + create_unique_name_ext(ctx, stmt) + ignore_function_str = "\ndef " + ignore_function_name + ignore_function_str += ( + "(" + ", ".join([var.name + " :" + var.ann for var in inputs]) + ")" + ) + + return_ann, return_stmt = build_return_ann_stmt(outputs) + ignore_function_str += return_ann + ": pass" + + # first create the functionDef object from just declaration + ignore_function = ast.parse(ignore_function_str).body[0] + + # dump the body of context manager to dummy function + ignore_function.body = stmt.body # type: ignore[attr-defined] + + # insert return statement to the function + return_stmt = ast.parse(return_stmt).body[0] + ignore_function.body.append(return_stmt) # type: ignore[attr-defined] + + ignore_func_str = f"""\ +# Backward compat: These used to be imported into the outer global scope so some +# code may still expect them. +from typing import List, Dict, Tuple + +@torch.jit.ignore +{ast.unparse(ignore_function)} +""" + g = copy.copy(globals()) + exec(ignore_func_str, g) # noqa: P204 + # registers the custom function in the global context + globals()[ignore_function_name] = g[ignore_function_name] + + # build the statements as: + # , , ... = torch.jit.frontend.(, ) + assign_str_lhs = build_args(outputs) + # this function will be registered in torch.jit.frontend module by default + assign_str_rhs = ( + f"torch.jit.frontend.{ignore_function_name}(" + build_args(inputs) + ")" + ) + + if len(outputs) > 0: + assign_str = assign_str_lhs + " = " + assign_str_rhs + else: + assign_str = assign_str_rhs + assign_ast = ast.parse(assign_str).body[0] + return assign_ast + + +def get_default_args(fn): + """ + Get a dictionary of default arguments for a function. + + Args: + fn: Callable - The function to inspect for default arguments. + Returns: + (Dict[str, Any]): mapping argument names to their default values if + :attr:`fn` is not None, else empty dictionary. + """ + if fn is None: + return {} + + signature = inspect.signature(fn) + + return { + k: v.default + for k, v in signature.parameters.items() + if v.default is not inspect.Parameter.empty + } + + +def get_default_args_for_class(cls): + """ + Get default arguments for all methods in a class (except for static methods). + + Args: + cls: type - The class type to inspect for default arguments. + Returns: + A Dict[str, Dict[str, Any]] which maps each method name to a Dict[str, Any] + that maps each argument name to its default value. + """ + # Get methods (except static methods because those are compiled separately as + # if they were independent script functions). + methods = inspect.getmembers( + cls, + predicate=lambda m: (inspect.ismethod(m) or inspect.isfunction(m)) + and not is_static_fn(cls, m.__name__) + and m.__name__ in cls.__dict__, + ) + + # Get method defaults. Property defaults do not need to be considered + # because setters cannot be invoked without a value. + defaults = { + method_name: get_default_args(method_impl) + for method_name, method_impl in methods + } + + return defaults + + +class WithItemBuilder(Builder): + @staticmethod + def build_withitem(ctx, item): + lineno = item.context_expr.lineno + start = item.context_expr.col_offset + end = start + len(pretty_node_names[ast.With]) + op_vars = item.optional_vars + r = ctx.make_range(lineno, start, end) + + return WithItem( + r, + build_expr(ctx, item.context_expr), + build_expr(ctx, op_vars) if op_vars else None, + ) + + +class StmtBuilder(Builder): + augassign_map = { + ast.Add: "+", + ast.Sub: "-", + ast.Mult: "*", + ast.Div: "/", + ast.Mod: "%", + ast.BitOr: "|", + ast.BitAnd: "&", + ast.BitXor: "^", + ast.LShift: "<<", + ast.RShift: ">>", + ast.Pow: "**", + } + + @staticmethod + def build_Expr(ctx, stmt): + value = stmt.value + if value.__class__.__name__ == "Str": + # If a statement is a string literal expression, + # then it is a docstring. Just ignore it. + return None + else: + return ExprStmt(build_expr(ctx, value)) + + @staticmethod + def build_Assign(ctx, stmt): + rhs = build_expr(ctx, stmt.value) + lhs = [build_expr(ctx, x) for x in stmt.targets] + return Assign(lhs, rhs) + + @staticmethod + def build_AnnAssign(ctx, stmt): + if stmt.value is None: + raise UnsupportedNodeError(ctx, stmt, reason="without assigned value") + + # Disallow type annotations on instance attributes outside of __init__ + if ( + type(stmt.target) is ast.Attribute + and stmt.target.value.id == "self" # type: ignore[attr-defined] + and ctx.funcname != "__init__" + ): + start = stmt.col_offset + end = start + len(f"self.{stmt.target.attr}") + if hasattr(stmt.annotation, "id"): + end += len(f": {stmt.annotation.id}") + sr = ctx.make_range(stmt.lineno, start, end) + raise ValueError( + "Type annotations on instance attributes must be declared in " + f"__init__, not '{ctx.funcname}': {sr}" + ) + + rhs = build_expr(ctx, stmt.value) + lhs = build_expr(ctx, stmt.target) + the_type = build_expr(ctx, stmt.annotation) + return Assign([lhs], rhs, the_type) + + @staticmethod + def build_Delete(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("del")) + + return Delete(r, [build_expr(ctx, target) for target in stmt.targets]) + + @staticmethod + def build_Return(ctx, stmt): + r = ctx.make_range( + stmt.lineno, stmt.col_offset, stmt.col_offset + len("return") + ) + return Return(r, None if stmt.value is None else build_expr(ctx, stmt.value)) + + @staticmethod + def build_Raise(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("raise")) + expr = build_expr(ctx, stmt.exc) + return Raise(r, expr) + + @staticmethod + def build_Assert(ctx, stmt): + r = ctx.make_range( + stmt.lineno, stmt.col_offset, stmt.col_offset + len("assert") + ) + test = build_expr(ctx, stmt.test) + msg = build_expr(ctx, stmt.msg) if stmt.msg is not None else None + return Assert(r, test, msg) + + @staticmethod + def build_AugAssign(ctx, stmt): + lhs = build_expr(ctx, stmt.target) + rhs = build_expr(ctx, stmt.value) + op = type(stmt.op) + if op in StmtBuilder.augassign_map: + op_token = StmtBuilder.augassign_map[op] + else: + raise NotSupportedError( + find_before(ctx, rhs.range().start, "=", offsets=(-1, 0)), + "unsupported kind of augmented assignment: " + op.__name__, + ) + return AugAssign(lhs, op_token, rhs) + + @staticmethod + def build_While(ctx, stmt): + if stmt.orelse: + # TODO: try to recover the location of else:? Python doesn't give us useful + # annotations in this case + raise NotSupportedError( + None, "else branches of while loops aren't supported" + ) + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("while")) + return While(r, build_expr(ctx, stmt.test), build_stmts(ctx, stmt.body)) + + @staticmethod + def build_For(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("for")) + if stmt.orelse: + raise NotSupportedError(r, "else branches of for loops aren't supported") + + return For( + r, + [build_expr(ctx, stmt.target)], + [build_expr(ctx, stmt.iter)], + build_stmts(ctx, stmt.body), + ) + + @staticmethod + def build_If(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("if")) + return If( + r, + build_expr(ctx, stmt.test), + build_stmts(ctx, stmt.body), + build_stmts(ctx, stmt.orelse), + ) + + @staticmethod + def build_Print(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("print")) + if stmt.dest: + raise NotSupportedError( + r, "print statements with non-default destinations aren't supported" + ) + args = [build_expr(ctx, val) for val in stmt.values] + return ExprStmt(Apply(Var(Ident(r, "print")), args, [])) + + @staticmethod + def build_Pass(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("pass")) + return Pass(r) + + @staticmethod + def build_Break(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("break")) + return Break(r) + + @staticmethod + def build_Continue(ctx, stmt): + r = ctx.make_range( + stmt.lineno, stmt.col_offset, stmt.col_offset + len("continue") + ) + return Continue(r) + + @staticmethod + def build_With(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset + len("with")) + # Handle ignore context manager + if is_torch_jit_ignore_context_manager(stmt): + assign_ast = build_ignore_context_manager(ctx, stmt) + return build_stmt(ctx, assign_ast) + return With(r, build_withitems(ctx, stmt.items), build_stmts(ctx, stmt.body)) + + +class ExprBuilder(Builder): + binop_map = { + ast.Add: "+", + ast.Sub: "-", + ast.Mult: "*", + ast.Div: "/", + ast.Pow: "**", + ast.Mod: "%", + ast.FloorDiv: "//", + ast.BitAnd: "&", + ast.BitXor: "^", + ast.BitOr: "|", + ast.LShift: "<<", + ast.RShift: ">>", + } + + # pyrefly: ignore [unsupported-operation] + binop_map[ast.MatMult] = "@" + + unop_map = { + ast.Not: "not", + ast.USub: "-", + ast.Invert: "~", + } + + boolop_map = { + ast.And: "and", + ast.Or: "or", + } + + cmpop_map = { + ast.Eq: "==", + ast.NotEq: "!=", + ast.LtE: "<=", + ast.Lt: "<", + ast.GtE: ">=", + ast.Gt: ">", + ast.Is: "is", + ast.IsNot: "is not", + ast.In: "in", + ast.NotIn: "not in", + } + + @staticmethod + def build_Attribute(ctx, expr): + base = build_expr(ctx, expr.value) + # expr.attr is just a string, so it's not annotated in any way, so we have + # to build the range manually + source = ctx.source.encode("utf-8") + + def get_char(index): + return chr(source[index]) + + start_pos = base.range().end + 1 + while get_char(start_pos) in string.whitespace: # Skip whitespace + start_pos += 1 + end_pos = start_pos + len(expr.attr) + name_range = ctx.make_raw_range(start_pos, end_pos) + return Select(base, Ident(name_range, expr.attr)) + + @staticmethod + def build_Call(ctx, expr): + func = build_expr(ctx, expr.func) + args = [build_expr(ctx, py_arg) for py_arg in expr.args] + if hasattr(expr, "starargs") and expr.starargs: + stararg_expr = build_expr(ctx, expr.starargs) + args += [Starred(stararg_expr.range(), stararg_expr)] + kwargs = [] + for kw in expr.keywords: + kw_expr = build_expr(ctx, kw.value) + # XXX: we could do a better job at figuring out the range for the name here + if not kw.arg: + raise NotSupportedError( + kw_expr.range(), "keyword-arg expansion is not supported" + ) + kwargs.append(Attribute(Ident(kw_expr.range(), kw.arg), kw_expr)) + return Apply(func, args, kwargs) + + @staticmethod + def build_Ellipsis(ctx, expr): + r = ctx.make_range( + expr.lineno, expr.col_offset, expr.col_offset + 3 + ) # len("...") == 3 + return Dots(r) + + @staticmethod + def build_Name(ctx, expr): + r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + len(expr.id)) + if expr.id.startswith(_reserved_prefix): + raise NotSupportedError( + r, + "names of variables used in JIT-ed functions " + "can't start with " + _reserved_prefix, + ) + if expr.id == "True": + return TrueLiteral(r) + elif expr.id == "False": + return FalseLiteral(r) + elif expr.id == "None": + return NoneLiteral(r) + elif expr.id == "Ellipsis": + return Dots(r) + return Var(Ident(r, expr.id)) + + @staticmethod + def build_NameConstant(ctx, expr): + r = ctx.make_range( + expr.lineno, expr.col_offset, expr.col_offset + len(str(expr.value)) + ) + if expr.value is True: + return TrueLiteral(r) + elif expr.value is False: + return FalseLiteral(r) + elif expr.value is None: + return NoneLiteral(r) + elif expr.value == Ellipsis: + return Dots(r) + else: + raise ValueError("Name constant value unsupported: " + str(expr.value)) + + @staticmethod + def build_BinOp(ctx, expr): + lhs = build_expr(ctx, expr.left) + rhs = build_expr(ctx, expr.right) + op = type(expr.op) + + if op == ast.Div and not ctx.uses_true_division: + err_range = ctx.make_raw_range(lhs.range().end, rhs.range().start) + raise FrontendError( + err_range, + "Division of ints in TorchScript uses Python 3 true " + "division semantics. Please put `from __future__ " + "import division` at the top of your file", + ) + op_token = ExprBuilder.binop_map.get(op) + if op_token is None: + err_range = ctx.make_raw_range(lhs.range().end, rhs.range().start) + raise NotSupportedError( + err_range, "unsupported binary operator: " + op.__name__ + ) + return BinOp(op_token, lhs, rhs) + + @staticmethod + def build_UnaryOp(ctx, expr): + sub_expr = build_expr(ctx, expr.operand) + op = type(expr.op) + op_token = ExprBuilder.unop_map.get(op) + if op_token is None: + raise NotSupportedError( + expr.range(), "unsupported unary operator: " + op.__name__ + ) + r = ctx.make_range( + expr.lineno, expr.col_offset, expr.col_offset + len(op_token) + ) + return UnaryOp(r, op_token, sub_expr) + + @staticmethod + def build_BoolOp(ctx, expr): + if len(expr.values) < 2: + raise AssertionError( + "expected at least 2 values in BoolOp, but got " + str(len(expr.values)) + ) + sub_exprs = [build_expr(ctx, sub_expr) for sub_expr in expr.values] + op = type(expr.op) + op_token = ExprBuilder.boolop_map.get(op) + if op_token is None: + err_range = ctx.make_raw_range( + sub_exprs[0].range().end, sub_exprs[1].range().start + ) + raise NotSupportedError( + err_range, "unsupported boolean operator: " + op.__name__ + ) + lhs = sub_exprs[0] + for rhs in sub_exprs[1:]: + lhs = BinOp(op_token, lhs, rhs) + return lhs + + @staticmethod + def build_IfExp(ctx, expr): + return TernaryIf( + build_expr(ctx, expr.test), + build_expr(ctx, expr.body), + build_expr(ctx, expr.orelse), + ) + + @staticmethod + def build_Compare(ctx, expr): + operands = [build_expr(ctx, e) for e in [expr.left] + list(expr.comparators)] + result = None + for lhs, op_, rhs in zip(operands, expr.ops, operands[1:]): + op = type(op_) + op_token = ExprBuilder.cmpop_map.get(op) + r = ctx.make_raw_range(lhs.range().end, rhs.range().start) + if op_token is None: + raise NotSupportedError( + r, "unsupported comparison operator: " + op.__name__ + ) + + if op == ast.NotIn: + # NB: `not in` is just `not( in )`, so we don't introduce new tree view + # but just make it a nested call in our tree view structure + in_expr = BinOp("in", lhs, rhs) + cmp_expr = UnaryOp(r, "not", in_expr) + else: + cmp_expr = BinOp(op_token, lhs, rhs) # type: ignore[assignment] + + if result is None: + result = cmp_expr + else: + result = BinOp("and", result, cmp_expr) # type: ignore[assignment] + return result + + @staticmethod + def build_Subscript(ctx, expr): + def build_SliceExpr(ctx, base, slice_expr): + lower = ( + build_expr(ctx, slice_expr.lower) + if slice_expr.lower is not None + else None + ) + upper = ( + build_expr(ctx, slice_expr.upper) + if slice_expr.upper is not None + else None + ) + step = ( + build_expr(ctx, slice_expr.step) + if slice_expr.step is not None + else None + ) + return SliceExpr(base.range(), lower, upper, step) + + def build_Index(ctx, base, index_expr): + if isinstance(index_expr.value, ast.Tuple): + raise NotSupportedError( + base.range(), + "slicing multiple dimensions with tuples not supported yet", + ) + return build_expr(ctx, index_expr.value) + + def build_ExtSlice(ctx, base, extslice): + sub_exprs = [] + for expr in extslice.dims: + sub_type = type(expr) + if sub_type is ast.Index: + sub_exprs.append(build_Index(ctx, base, expr)) + elif sub_type is ast.Slice: + sub_exprs.append(build_SliceExpr(ctx, base, expr)) + elif sub_type is ast.Constant and expr.value is Ellipsis: + sub_exprs.append(Dots(base.range())) + else: + raise NotSupportedError( + base.range(), + f"slicing multiple dimensions with {sub_type} not supported", + ) + return sub_exprs + + base = build_expr(ctx, expr.value) + sub_type = type(expr.slice) + if sub_type is ast.Index: + if isinstance(expr.slice.value, ast.Tuple): + # N-dimensional indexing using Tuple: x[(i, j, k)] is equivalent to x[i, j, k] + # XXX: Indexing using a list is **different**! It triggers advanced indexing. + indices = [ + build_expr(ctx, index_expr) for index_expr in expr.slice.value.elts + ] + if not indices: + # `col_offset` is an int, but `end_col_offset` is + # `Optional[int]`. The magic number is here to make + # sure we can parse `()` on any machine + r = ctx.make_range( + expr.lineno, + expr.slice.value.col_offset, + expr.slice.value.col_offset + 2, + ) + tup = TupleLiteral(r, []) + indices.append(tup) + return Subscript(base, indices) + else: + return Subscript(base, [build_expr(ctx, expr.slice.value)]) + elif sub_type is ast.Slice: + return Subscript(base, [build_SliceExpr(ctx, base, expr.slice)]) + elif sub_type is ast.ExtSlice: + return Subscript(base, build_ExtSlice(ctx, base, expr.slice)) + else: # In Python3.9 array indices are not wrapped in ast.Index + if sub_type is ast.Tuple: + # N-dimensional indexing using Tuple: x[(i, j, k)] is equivalent to x[i, j, k] + indices = [] + for index_expr in expr.slice.elts: + if isinstance(index_expr, ast.Slice): + indices.append(build_SliceExpr(ctx, base, index_expr)) + else: + indices.append(build_expr(ctx, index_expr)) + # Special-case logic for `typing.Tuple[()]` + if not indices: + # See note above r.e. magic number + r = ctx.make_range( + expr.lineno, expr.slice.col_offset, expr.slice.col_offset + 2 + ) + tup = TupleLiteral(r, []) + indices.append(tup) + return Subscript(base, indices) + return Subscript(base, [build_expr(ctx, expr.slice)]) + + @staticmethod + def build_List(ctx, expr): + return ListLiteral( + ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1), + [build_expr(ctx, e) for e in expr.elts], + ) + + @staticmethod + def build_Tuple(ctx, expr): + return TupleLiteral( + ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1), + [build_expr(ctx, e) for e in expr.elts], + ) + + @staticmethod + def build_Dict(ctx, expr): + range = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1) + if expr.keys and not expr.keys[0]: + raise NotSupportedError( + range, "Dict expansion (e.g. `{**dict}`) is not supported" + ) + return DictLiteral( + range, + [build_expr(ctx, e) for e in expr.keys], + [build_expr(ctx, e) for e in expr.values], + ) + + @staticmethod + def build_Num(ctx, expr): + value = str(expr.value) + r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + len(value)) + return Const(r, value) + + @staticmethod + def build_Constant(ctx, expr): + value = expr.value + if value is None or isinstance(value, bool): + # NB: this check has to happen before the int check because bool is + # a subclass of int + return ExprBuilder.build_NameConstant(ctx, expr) + if isinstance(value, (int, float, complex)): + return ExprBuilder.build_Num(ctx, expr) + elif isinstance(value, str): + return ExprBuilder.build_Str(ctx, expr) + elif isinstance(value, type(Ellipsis)): + return ExprBuilder.build_Ellipsis(ctx, expr) + else: + error_range = ctx.make_range( + expr.lineno, expr.col_offset, expr.col_offset + len(str(value)) + ) + raise FrontendError(error_range, "Unknown Constant expression type") + + @staticmethod + def build_Str(ctx, expr): + value = str(expr.value) + r = ctx.make_range( + expr.lineno, expr.col_offset, expr.col_offset + len(value) + 1 + ) + return StringLiteral(r, value) + + @staticmethod + def build_JoinedStr(ctx, expr): + s = "" + args = [] + for value in expr.values: + r = ctx.make_range(value.lineno, value.col_offset, value.col_offset + 1) + if isinstance(value, ast.FormattedValue): + if value.conversion != -1: + raise NotSupportedError(r, "Don't support conversion in JoinedStr") + if value.format_spec is not None: + raise NotSupportedError(r, "Don't support formatting in JoinedStr") + s += "{}" + args.append(build_expr(ctx, value.value)) + elif isinstance(value, ast.Constant): + # pyrefly: ignore [unsupported-operation] + s += value.value + else: + raise NotSupportedError(r, "Unsupported value in JoinedStr") + + r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1) + return Apply(Select(StringLiteral(r, s), Ident(r, "format")), args, []) + + @staticmethod + def build_ListComp(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset) + if len(stmt.generators) != 1: + raise NotSupportedError(r, "Only a single generator is currently supported") + + if len(stmt.generators[0].ifs) != 0: + raise NotSupportedError(r, "Comprehension ifs are not supported yet") + + elt_expr = build_expr(ctx, stmt.elt) + target_expr = build_expr(ctx, stmt.generators[0].target) + iter_expr = build_expr(ctx, stmt.generators[0].iter) + + return ListComp(r, elt_expr, target_expr, iter_expr) + + @staticmethod + def build_GeneratorExp(ctx, stmt): + # Convert Generator expression to ListComp + return ExprBuilder.build_ListComp(ctx, stmt) + + @staticmethod + def build_DictComp(ctx, stmt): + r = ctx.make_range(stmt.lineno, stmt.col_offset, stmt.col_offset) + if len(stmt.generators) != 1: + raise NotSupportedError(r, "Only a single generator is currently supported") + + if len(stmt.generators[0].ifs) != 0: + raise NotSupportedError(r, "Comprehension ifs are not supported yet") + + key_expr = build_expr(ctx, stmt.key) + value_expr = build_expr(ctx, stmt.value) + target_expr = build_expr(ctx, stmt.generators[0].target) + iter_expr = build_expr(ctx, stmt.generators[0].iter) + + return DictComp(r, key_expr, value_expr, target_expr, iter_expr) + + @staticmethod + def build_Starred(ctx, expr): + r = ctx.make_range(expr.lineno, expr.col_offset, expr.col_offset + 1) + return Starred(r, build_expr(ctx, expr.value)) + + +build_expr = ExprBuilder() +build_stmt = StmtBuilder() +build_withitem = WithItemBuilder() + + +def find_before(ctx, pos, substr, offsets=(0, 0)): + new_pos = ctx.source[:pos].rindex(substr) + return ctx.make_raw_range(new_pos + offsets[0], new_pos + len(substr) + offsets[1]) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/generate_bytecode.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/generate_bytecode.py new file mode 100644 index 0000000000000000000000000000000000000000..ebdceaddca92ad8ac40dfdebb50f9eaae596d42c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/generate_bytecode.py @@ -0,0 +1,33 @@ +# mypy: allow-untyped-defs + +from torch._C import _compile_graph_to_code_table, _generate_upgraders_graph + + +def format_bytecode(table): + # given a nested tuple, convert it to nested list + def listify(content): + if not isinstance(content, tuple): + return content + return [listify(i) for i in content] + + formatted_table = {} + for entry in table: + identifier = entry[0] + content = entry[1] + content = listify(content) + formatted_table[identifier] = content + return formatted_table + + +def generate_upgraders_bytecode() -> list: + yaml_content = [] + upgraders_graph_map = _generate_upgraders_graph() + for upgrader_name, upgrader_graph in upgraders_graph_map.items(): + bytecode_table = _compile_graph_to_code_table(upgrader_name, upgrader_graph) + entry = {upgrader_name: format_bytecode(bytecode_table)} + yaml_content.append(entry) + return yaml_content + + +if __name__ == "__main__": + raise RuntimeError("This file is not meant to be run directly") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/quantized.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/quantized.py new file mode 100644 index 0000000000000000000000000000000000000000..a2500c1f1b9fed481ab713953746294dc52668af --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/quantized.py @@ -0,0 +1,100 @@ +# mypy: allow-untyped-defs +import torch + + +class QuantizedLinear(torch.jit.ScriptModule): + def __init__(self, other): + raise RuntimeError( + "torch.jit.QuantizedLinear is no longer supported. Please use " + "torch.ao.nn.quantized.dynamic.Linear instead." + ) + + +# FP16 weights +class QuantizedLinearFP16(torch.jit.ScriptModule): + def __init__(self, other): + super().__init__() + raise RuntimeError( + "torch.jit.QuantizedLinearFP16 is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.Linear instead." + ) + + +# Quantized RNN cell implementations +class QuantizedRNNCellBase(torch.jit.ScriptModule): + def __init__(self, other): + raise RuntimeError( + "torch.jit.QuantizedRNNCellBase is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.RNNCell instead." + ) + + +class QuantizedRNNCell(QuantizedRNNCellBase): + def __init__(self, other): + raise RuntimeError( + "torch.jit.QuantizedRNNCell is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.RNNCell instead." + ) + + +class QuantizedLSTMCell(QuantizedRNNCellBase): + def __init__(self, other): + super().__init__(other) + raise RuntimeError( + "torch.jit.QuantizedLSTMCell is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.LSTMCell instead." + ) + + +class QuantizedGRUCell(QuantizedRNNCellBase): + def __init__(self, other): + super().__init__(other) + raise RuntimeError( + "torch.jit.QuantizedGRUCell is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.GRUCell instead." + ) + + +class QuantizedRNNBase(torch.jit.ScriptModule): + def __init__(self, other, dtype=torch.int8): + raise RuntimeError( + "torch.jit.QuantizedRNNBase is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic instead." + ) + + +class QuantizedLSTM(QuantizedRNNBase): + def __init__(self, other, dtype): + raise RuntimeError( + "torch.jit.QuantizedLSTM is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.LSTM instead." + ) + + +class QuantizedGRU(QuantizedRNNBase): + def __init__(self, *args, **kwargs): + raise RuntimeError( + "torch.jit.QuantizedGRU is no longer supported. " + "Please use the torch.ao.nn.quantized.dynamic.GRU instead." + ) + + +def quantize_rnn_cell_modules(module): + raise RuntimeError( + "quantize_rnn_cell_modules function is no longer supported. " + "Please use torch.ao.quantization.quantize_dynamic API instead." + ) + + +def quantize_linear_modules(module, dtype=torch.int8): + raise RuntimeError( + "quantize_linear_modules function is no longer supported. " + "Please use torch.ao.quantization.quantize_dynamic API instead." + ) + + +def quantize_rnn_modules(module, dtype=torch.int8): + raise RuntimeError( + "quantize_rnn_modules function is no longer supported. " + "Please use torch.ao.quantization.quantize_dynamic API instead." + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/supported_ops.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/supported_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..8a258280ea352d993b5a56b739c28d70b0273e8a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/supported_ops.py @@ -0,0 +1,346 @@ +# mypy: allow-untyped-defs +import inspect +import textwrap + +import torch.jit +from torch.jit._builtins import _find_builtin + + +# this file is for generating documentation using sphinx autodoc +# > help(torch.jit.supported_ops) will also give a nice listed of the +# supported ops programmatically + + +def _hidden(name): + return name.startswith("_") and not name.startswith("__") + + +def _emit_type(type): + return str(type) + + +def _emit_arg(indent, i, arg): + v = f"{arg.name} : {_emit_type(arg.type)}" + default = arg.default_value + if default is not None: + v = f"{v}={str(default)}" + if i > 0: + v = f"\n{' ' * indent}{v}" + return v + + +def _emit_args(indent, arguments): + return ",".join(_emit_arg(indent, i, arg) for i, arg in enumerate(arguments)) + + +def _emit_ret(ret): + return _emit_type(ret.type) + + +def _emit_rets(returns): + if len(returns) == 1: + return _emit_ret(returns[0]) + return f"Tuple[{', '.join(_emit_ret(r) for r in returns)}]" + + +def _emit_schema(mod, name, schema, arg_start=0, padding=4): + if mod is None: + qualified_name = name + else: + qualified_name = f"{mod}.{name}" + schema_str = ( + f"{qualified_name}" + f"({_emit_args(len(qualified_name) + 1 + padding, schema.arguments[arg_start:])}) " + f"-> {_emit_rets(schema.returns)}" + ) + return schema_str + + +def _get_tensor_ops(): + def is_tensor_method(schema) -> bool: + if len(schema.arguments) == 0: + return False + self = schema.arguments[0] + if self.name != "self": + return False + if not self.type.isSubtypeOf(torch._C.TensorType.get()): + return False + return True + + methods = [] + # discover methods + for elem in dir(torch.Tensor): + if not _hidden(elem): + schemas = torch._C._jit_get_schemas_for_operator("aten::" + elem) + for schema in schemas: + if is_tensor_method(schema): + methods.append(_emit_schema("Tensor", elem, schema, arg_start=1)) + + return "Supported Tensor Methods", methods + + +def _get_nn_functional_ops(): + functions = [] + + # Iterate over torch.nn.functional + mod = torch.nn.functional + name = mod.__name__ + for elem in dir(torch.nn.functional): + attr = getattr(mod, elem) + if not inspect.isfunction(attr) or _hidden(elem[0]): + # Ignore non-functions and internal methods + continue + + attr_module = inspect.getmodule(attr) + if not attr_module: + raise RuntimeError(f"Module for {attr} not found") + + if "torch.nn.functional" not in attr_module.__name__: + # Ignore functions from outside torch.nn.functional + continue + + try: + # compile fn, get schema + scripted = torch.jit.script(attr) + scripted_schema = scripted.schema + functions.append(_emit_schema(name, elem, scripted_schema)) + except: # noqa: B001,E722 + # Skip interpolate / boolean dispatched things + pass + + # Iterate over modules that we know contain a lot of builtins + for mod in torch.jit._builtins._modules_containing_builtins: + name = mod.__name__ + for elem in dir(mod): + builtin = _find_builtin(getattr(mod, elem)) + if builtin is not None: + schemas = torch._C._jit_get_schemas_for_operator(builtin) + for schema in schemas: + # remove _tan but not __and__ + if not _hidden(elem): + functions.append(_emit_schema(name, elem, schema)) + return "Supported PyTorch Functions", functions + + +def _get_builtins_helper(): + builtins = [] + for fn, _builtin_name in torch.jit._builtins._builtin_ops: + mod = inspect.getmodule(fn) + + if not hasattr(fn, "__name__"): + # typing classes + continue + if not mod: + continue + if _hidden(fn.__name__) or _hidden(fn.__qualname__) or _hidden(mod.__name__): + # skip internal-only methods + continue + + if "torch._C" in mod.__name__: + continue + + builtins.append((fn, _builtin_name)) + + return builtins + + +def _is_math_fn(fn): + mod = inspect.getmodule(fn) + if not mod: + raise RuntimeError(f"Module for {fn} not found") + + return mod.__name__ == "math" + + +def _get_torchscript_builtins(): + functions = [] + builtins = filter(lambda fn: not _is_math_fn(fn[0]), _get_builtins_helper()) + builtins_list = list(builtins) + # Iterate over the specially added builtins + for fn, _builtin_name in builtins_list: + mod = inspect.getmodule(fn) + if not mod: + raise RuntimeError(f"Module for {fn} not found") + builtin = _find_builtin(fn) + if builtin is not None: + schemas = torch._C._jit_get_schemas_for_operator(builtin) + for schema in schemas: + functions.append(_emit_schema(mod.__name__, fn.__name__, schema)) + + return "TorchScript Builtin Functions", functions + + +def _get_math_builtins(): + functions = [] + builtins = filter(lambda fn: _is_math_fn(fn[0]), _get_builtins_helper()) + builtins_list = list(builtins) + # Iterate over the specially added builtins + for fn, _builtin_name in builtins_list: + mod = inspect.getmodule(fn) + if not mod: + raise RuntimeError(f"Module for {fn} not found") + builtin = _find_builtin(fn) + if builtin is not None: + schemas = torch._C._jit_get_schemas_for_operator(builtin) + for schema in schemas: + schema_str = _emit_schema(mod.__name__, fn.__name__, schema) + if "Tensor" in schema_str: + # Skip Tensor ops that have the same name as math functions + # (they will show up in the tensor methods section) + continue + functions.append(schema) + + return "``math`` Module", functions + + +def _get_global_builtins(): + # Taken from the 'globals' map in torch/csrc/jit/frontend/ir_emitter.cpp + supported_builtins = [ + "print", + "tuple", + "float", + "complex", + "int", + "bool", + "str", + "getattr", + "hasattr", + "isinstance", + "len", + "hex", + "oct", + "round", + "hash", + "min", + "max", + "abs", + "all", + "divmod", + "list", + "ord", + "chr", + "bin", + "range", + "zip", + "enumerate", + "sorted", + ] + + op_renames = { + "bool": "aten::Bool", + "int": "aten::Int", + "float": "aten::Float", + "complex": "aten::Complex", + "abs": "prim::abs", + "max": "prim::max", + "min": "prim::min", + "range": "fake::does_not_exist", + } + + schemaless_op_explanations = { + "print": "Print any value", + "tuple": "Lists cannot be converted to tuples with this method since their size is not statically known", + "getattr": "Attribute name must be a literal string", + "hasattr": "Attribute name must be a literal string", + "isinstance": "Result is static", + "zip": "Arguments must be iterable.", + "enumerate": "Arguments must be iterable.", + "range": "Can only be used as an iterator in a for loop", + } + + magic_methods = [ + ("complex", "__complex__"), + ("float", "__float__"), + ("int", "__int__"), + ("bool", "__bool__"), + ("str", "__str__"), + ("len", "__len__"), + ("hex", "__hex__"), + ("oct", "__oct__"), + ] + + magic_methods_rows = [] + for fn, magic_method in magic_methods: + # pyrefly: ignore [bad-argument-type] + magic_methods_rows.append(f'"{fn}", "``{magic_method}``"') + + schematized_ops = [] + schemaless_ops = [] + + for fn in supported_builtins: + op_name = f"aten::{fn}" + if fn in op_renames: + op_name = op_renames[fn] + schemas = torch._C._jit_get_schemas_for_operator(op_name) + for s in schemas: + schematized_ops.append(_emit_schema(None, fn, s, padding=0)) + if len(schemas) > 0: + schematized_ops.append("") + else: + table_row = ( + f'":external+python:py:obj:`{fn}`", "{schemaless_op_explanations[fn]}"' + ) + # pyrefly: ignore [bad-argument-type] + schemaless_ops.append(table_row) + + schematized_ops_str = "\n".join(schematized_ops) + schemaless_ops_str = "\n".join(schemaless_ops) + magic_methods_rows_str = "\n".join(magic_methods_rows) + schematized_ops_str = textwrap.indent(schematized_ops_str, "\t") + schemaless_ops_str = textwrap.indent(schemaless_ops_str, "\t") + magic_methods_rows_str = textwrap.indent(magic_methods_rows_str, "\t") + section = f""" +The functions in the following table are supported but do not have a static schema + +.. csv-table:: + :header: "Function", "Note" + +{schemaless_ops_str} + +The following functions will use the corresponding magic method on TorchScript classes + +.. csv-table:: + :header: "Function", "Magic Method" + +{magic_methods_rows_str} + +These built-in functions use the schema + +.. rst-class:: codeblock-height-limiter + +:: + +{schematized_ops_str} + """ + + return "Python Built-in Functions", section + + +def _list_supported_ops(): + def emit_block(decls): + return "\n.. rst-class:: codeblock-height-limiter\n\n::\n\n{}\n".format( + "".join(f" {d}\n\n" for d in decls) + ) + + body = "" + op_gathering_fns = ( + _get_tensor_ops, + _get_nn_functional_ops, + _get_torchscript_builtins, + _get_global_builtins, + _get_math_builtins, + ) + for fn in op_gathering_fns: + header, items = fn() + link_target = header.replace("`", "").replace("-", "").lower().replace(" ", "-") + if isinstance(items, str): + section = f"{header}\n{'~' * len(header)}\n{items}\n" + else: + section = f"{header}\n{'~' * len(header)}\n{emit_block(items)}" + section = f".. _{link_target}:" + "\n\n" + section + body += section + + return body + + +__doc__ = _list_supported_ops() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/unsupported_tensor_ops.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/unsupported_tensor_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..12bca0fce337d291a6e1c75861f0b3f020cc7a6c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/jit/unsupported_tensor_ops.py @@ -0,0 +1,78 @@ +# mypy: allow-untyped-defs +from textwrap import dedent +from typing import Any + +import torch.jit + + +def execWrapper(code, glob, loc) -> None: + exec(code, glob, loc) + + +def _gen_unsupported_methods_properties(): + tensor_attrs = set(filter(lambda x: x[0] != "_", dir(torch.Tensor))) + tensor = torch.tensor([2]) + funcs_template = dedent( + """ + def func(x): + return x.{op}() + """ + ) + + deprecated_apis = { + "volatile", + "resize", + "reinforce", + "new", + "name", + "map2_", + "has_names", + "grad_fn", + "resize_as", + } + tensor_attrs = tensor_attrs - deprecated_apis + + properties = [] + methods = [] + sorted_tensor_attrs = sorted(tensor_attrs, key=lambda x: x.lower()) + for attr in sorted_tensor_attrs: + funcs_str = funcs_template.format(op=attr) + scope: dict[str, Any] = {} + execWrapper(funcs_str, globals(), scope) + try: + torch.jit.CompilationUnit(funcs_str) + except Exception as e: + if "nonexistent attribute" not in repr(e): + continue + attr_repr = repr(getattr(tensor, attr)) + if "bound method" in attr_repr or "built-in method" in attr_repr: + methods.append(attr) + else: + properties.append(attr) + + mapped_methods = ("\t* :meth:`~torch.Tensor." + x + r"`" for x in methods) + mapped_properties = ("\t* :attr:`~torch.Tensor." + x + r"`" for x in properties) + return "\n".join(mapped_methods), "\n".join(mapped_properties) + + +def _list_unsupported_tensor_ops(): + header = """\n\n +Unsupported Tensor Methods +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + """ + methods, properties = _gen_unsupported_methods_properties() + return ( + header + + "\n" + + methods + + """ + +Unsupported Tensor Properties +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + """ + + "\n" + + properties + ) + + +__doc__ = _list_unsupported_tensor_ops() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/lib/libcaffe2_nvrtc.so b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/lib/libcaffe2_nvrtc.so new file mode 100644 index 0000000000000000000000000000000000000000..afecb8fff1db3b651b1b5b395189e8e55f33a359 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/lib/libcaffe2_nvrtc.so differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/lib/libshm.so b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/lib/libshm.so new file mode 100644 index 0000000000000000000000000000000000000000..76db8cceb79de9b8de35e921827db1657e65224d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/lib/libshm.so differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/lib/libtorch_global_deps.so b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/lib/libtorch_global_deps.so new file mode 100644 index 0000000000000000000000000000000000000000..a9e9a09e1e6a14f7f234ebe638cc40f68d03c69f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/lib/libtorch_global_deps.so differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/masked/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/masked/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d00ba1e8d5aff6a18490ac7b16a629ac36e3dcb5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/masked/__init__.py @@ -0,0 +1,57 @@ +from torch.masked._ops import ( + _canonical_dim, + _combine_input_and_mask, + _generate_docstring, + _input_mask, + _output_mask, + _reduction_identity, + _where, + amax, + amin, + argmax, + argmin, + cumprod, + cumsum, + log_softmax, + logaddexp, + logsumexp, + mean, + median, + norm, + normalize, + prod, + softmax, + softmin, + std, + sum, + var, +) +from torch.masked.maskedtensor.core import is_masked_tensor, MaskedTensor +from torch.masked.maskedtensor.creation import as_masked_tensor, masked_tensor + + +__all__ = [ + "amax", + "amin", + "argmax", + "argmin", + "as_masked_tensor", + "cumprod", + "cumsum", + "is_masked_tensor", + "log_softmax", + "logaddexp", + "logsumexp", + "masked_tensor", + "MaskedTensor", + "mean", + "median", + "norm", + "normalize", + "prod", + "softmax", + "softmin", + "std", + "sum", + "var", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/masked/_docs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/masked/_docs.py new file mode 100644 index 0000000000000000000000000000000000000000..fa130bbefbc5caa7373459ef2fc3dc5292239948 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/masked/_docs.py @@ -0,0 +1,1177 @@ +# This file is generated, do not modify it! +# +# To update this file, run the update masked docs script as follows: +# +# python tools/update_masked_docs.py +# +# The script must be called from an environment where the development +# version of torch package can be imported and is functional. +# + +amax_docstring = """amax(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns maximum of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +The identity value of maximum operation, which is used to start the +reduction, depends on input dtype. For instance, for float32, uint8, +and int32 dtypes, the identity values are ``-inf``, ``0``, and ``-2147483648``, respectively. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in maximum computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of maximum operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.amax(input, 1, mask=mask) + tensor([ -1, -9223372036854775808]) +""" + +amin_docstring = """amin(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns minimum of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +The identity value of minimum operation, which is used to start the +reduction, depends on input dtype. For instance, for float32, uint8, +and int32 dtypes, the identity values are ``inf``, ``255``, and ``2147483647``, respectively. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in minimum computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of minimum operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.amin(input, 1, mask=mask) + tensor([ -3, 9223372036854775807]) +""" + +argmax_docstring = """argmax(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor +Returns argmax of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. +The identity value of argmax operation, which is used to start the +reduction, depends on input dtype. For instance, for float32, uint8, +and int32 dtypes, the identity values are ``-inf``, ``0``, and ``-2147483648``, respectively. +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in argmax computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of argmax operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which argmax is computed. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.argmax(input, 1, mask=mask) + tensor([2, 0]) +""" + +argmin_docstring = """argmin(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor +Returns argmin of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. +The identity value of argmin operation, which is used to start the +reduction, depends on input dtype. For instance, for float32, uint8, +and int32 dtypes, the identity values are ``inf``, ``255``, and ``2147483647``, respectively. +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in argmin computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of argmin operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which argmin is computed. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.argmin(input, 1, mask=mask) + tensor([0, 0]) +""" + +cumprod_docstring = """cumprod(input, dim, *, dtype=None, mask=None) -> Tensor + +Returns cumulative_prod of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +Let ``x`` be a sequence of unmasked elements of one-dimensional slice +of the :attr:`input` tensor. Cumsum of i-th element in ``x`` is +defined as ``prod(x[:i])``. + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +cumulative_prod computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the cumulative_prod output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which cumulative_prod is computed. + +Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.cumprod(input, 1, mask=mask) + tensor([[-3., -3., 3.], + [ 1., 1., 1.]]) +""" + +cumsum_docstring = """cumsum(input, dim, *, dtype=None, mask=None) -> Tensor + +Returns cumulative_sum of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +Let ``x`` be a sequence of unmasked elements of one-dimensional slice +of the :attr:`input` tensor. Cumsum of i-th element in ``x`` is +defined as ``sum(x[:i])``. + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +cumulative_sum computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the cumulative_sum output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which cumulative_sum is computed. + +Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.cumsum(input, 1, mask=mask) + tensor([[-3., -3., -4.], + [ 0., 0., 0.]]) +""" + +log_softmax_docstring = """log_softmax(input, dim, *, dtype=None, mask=None) -> Tensor + +Returns log_softmax of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +Let ``x`` be a sequence of unmasked elements of one-dimensional slice +of the :attr:`input` tensor. LogSoftmax of i-th element in ``x`` is +defined as ``log(exp(x[i])/sum(exp(x)))``. + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +log_softmax computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the log_softmax output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which log_softmax is computed. + +Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.log_softmax(input, 1, mask=mask) + tensor([[-2.1269, -inf, -0.1269], + [ nan, nan, nan]]) +""" + +logsumexp_docstring = """logsumexp(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns logsumexp of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +The identity value of logsumexp operation, which is used to start the reduction, is ``-2147483648``. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in logsumexp computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of logsumexp operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.logsumexp(input, 1, mask=mask) + tensor([ 0, -9223372036854775808]) +""" + +mean_docstring = """mean(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns mean of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +By definition, the identity value of a mean operation is the mean +value of the tensor. If all elements of the input tensor along given +dimension(s) :attr:`dim` are masked-out, the identity value of the +mean is undefined. Due to this ambiguity, the elements of output +tensor with strided layout, that correspond to fully masked-out +elements, have ``nan`` values. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in mean computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of mean operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.mean(input, 1, mask=mask) + tensor([-2., nan]) +""" + +median_docstring = """median(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor +Returns median of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. +By definition, the identity value of a median operation is the median +value of the tensor. If all elements of the input tensor along given +dimension(s) :attr:`dim` are masked-out, the identity value of the +median is undefined. Due to this ambiguity, the elements of output +tensor with strided layout, that correspond to fully masked-out +elements, have ``nan`` values. +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in median computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of median operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which median is computed. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.median(input, 1, mask=mask) + tensor([-3., nan]) +""" + +norm_docstring = """norm(input, ord, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns norm of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +The identity value of norm operation, which is used to start the +reduction, is ``0.0``, except for ``ord=-inf`` it is +``inf``. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in norm computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of norm operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + ord (int, float, optional): the order of vector norm. Default: 2. + See :func:`torch.linalg.vector_norm` for a list of supported norms. + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.norm(input, 2.0, 1, mask=mask) + tensor([3.1623, 0.0000]) +""" + +normalize_docstring = """normalize(input, ord, dim, *, eps=1e-12, dtype=None, mask=None) -> Tensor + +Returns normalize of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +Let ``x`` be a sequence of unmasked elements of one-dimensional slice +of the :attr:`input` tensor. Normalize of i-th element in ``x`` is +defined as ``x[i]/max(norm(x, p), eps)``. + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +normalize computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the normalize output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + ord (int, float): the order of vector norm. Default: 2. + See :func:`torch.linalg.vector_norm` for a list of supported norms. + dim (int): the dimension along which normalize is computed. + +Keyword args: + eps (float, optional): small value to avoid division by zero. Default: 1e-12. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.normalize(input, 2.0, 1, mask=mask) + tensor([[-0.9487, 0.0000, -0.3162], + [ 0.0000, 0.0000, 0.0000]]) +""" + +prod_docstring = """prod(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns product of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +The identity value of product operation, which is used to start the reduction, is ``1``. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in product computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of product operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.prod(input, 1, mask=mask) + tensor([3, 1]) +""" + +softmax_docstring = """softmax(input, dim, *, dtype=None, mask=None) -> Tensor + +Returns softmax of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +Let ``x`` be a sequence of unmasked elements of one-dimensional slice +of the :attr:`input` tensor. Softmax of i-th element in ``x`` is +defined as ``exp(x[i])/sum(exp(x))``. + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +softmax computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the softmax output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which softmax is computed. + +Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.softmax(input, 1, mask=mask) + tensor([[0.1192, 0.0000, 0.8808], + [ nan, nan, nan]]) +""" + +softmin_docstring = """softmin(input, dim, *, dtype=None, mask=None) -> Tensor + +Returns softmin of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +Let ``x`` be a sequence of unmasked elements of one-dimensional slice +of the :attr:`input` tensor. Softmin of i-th element in ``x`` is +defined as ``exp(-x[i])/sum(exp(-x))``. + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +softmin computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the softmin output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int): the dimension along which softmin is computed. + +Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3., -2., -1.], [ 0., 1., 2.]]) + >>> input + tensor([[-3., -2., -1.], + [ 0., 1., 2.]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.softmin(input, 1, mask=mask) + tensor([[0.8808, 0.0000, 0.1192], + [ nan, nan, nan]]) +""" + +std_docstring = """std(input, dim, unbiased, *, keepdim=False, dtype=None, mask=None) -> Tensor +Returns standard_deviation of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. +The identity value of sample standard deviation operation is undefined. The +elements of output tensor with strided layout, that correspond to +fully masked-out elements, have ``nan`` values. +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in standard_deviation computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of standard_deviation operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + unbiased (bool): when True, use Bessel's correction, otherwise, compute + the uncorrected sample variance. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.std(input, 1, False, mask=mask) + tensor([1., nan]) +""" + +sum_docstring = """sum(input, dim, *, keepdim=False, dtype=None, mask=None) -> Tensor + +Returns sum of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. + +The identity value of sum operation, which is used to start the reduction, is ``0``. + +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in sum computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of sum operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.sum(input, 1, mask=mask) + tensor([-4, 0]) +""" + +var_docstring = """var(input, dim, unbiased, *, keepdim=False, dtype=None, mask=None) -> Tensor +Returns variance of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`. +The identity value of sample variance operation is undefined. The +elements of output tensor with strided layout, that correspond to +fully masked-out elements, have ``nan`` values. +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in variance computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of variance operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``. + unbiased (bool): when True, use Bessel's correction, otherwise, compute + the uncorrected sample variance. + +Keyword args: + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: False. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: None. + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. +Example:: + + >>> input = tensor([[-3, -2, -1], [ 0, 1, 2]]) + >>> input + tensor([[-3, -2, -1], + [ 0, 1, 2]]) + >>> mask = tensor([[ True, False, True], [False, False, False]]) + >>> mask + tensor([[ True, False, True], + [False, False, False]]) + >>> torch.masked._ops.var(input, 1, False, mask=mask) + tensor([1., nan]) +""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/masked/_ops.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/masked/_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..dd3ff69fd6af87f29430257899c2ad1d64fd8cb6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/masked/_ops.py @@ -0,0 +1,1821 @@ +# mypy: allow-untyped-defs +import warnings +from collections.abc import Callable +from typing import Any, Optional, TYPE_CHECKING, TypeAlias, TypeVar +from typing_extensions import ParamSpec + +import torch +from torch import sym_float, Tensor +from torch._prims_common import corresponding_real_dtype +from torch.masked import _docs +from torch.masked.maskedtensor.core import is_masked_tensor, MaskedTensor +from torch.masked.maskedtensor.creation import as_masked_tensor + + +if TYPE_CHECKING: + from torch._prims_common import DimsType + from torch.types import _dtype as DType + + DimOrDims: TypeAlias = DimsType | None +else: + # The JIT doesn't understand Union, nor torch.dtype here + DType = int + DimOrDims = Optional[tuple[int, ...]] + + +__all__: list[str] = [] + +_T = TypeVar("_T") +_P = ParamSpec("_P") + +# All masked reduction/normalization operations have the same +# signatures. Here we introduce docstring templates that are applied +# to docstrings of reduction/normalization functions via +# _apply_docstring_templates decorator. + + +def _apply_docstring_templates(func: Callable[_P, _T]) -> Callable[_P, _T]: + """Decorator that applies docstring templates to function docstring + and returns the function instance. + """ + + doc_string = getattr(_docs, f"{func.__name__}_docstring", None) + if doc_string is None: + warnings.warn( + f"No documentation string available for {func.__name__}." + " PyTorch team should run `python tools/update_masked_docs.py`" + " to generate the missing docstrings.", + stacklevel=2, + ) + else: + func.__doc__ = doc_string + + # Expose function as public symbol + __all__.append(func.__name__) + + return func + + +def _generate_docstring(func): + """A utility function called from tools/update_masked_docs.py + script to update the module torch.masked._docs.py + """ + docstring_templates = dict( + reduction_signature="""\ +{function_name}(input, {operation_args}, *, {operation_kwargs}) -> Tensor""", + reduction_descr="""\ +Returns {operation name} of all the elements in the :attr:`input` +tensor along the given dimension(s) :attr:`dim` while the :attr:`input` +elements are masked out according to the boolean tensor +:attr:`mask`.""", + reduction_args="""\ +If :attr:`keepdim` is ``True``, the output tensor is of the same size +as :attr:`input` except in the dimension(s) :attr:`dim` where it is of +size 1. Otherwise, :attr:`dim` is squeezed (see +:func:`torch.squeeze`), resulting in the output tensor having 1 (or +``len(dim)``) fewer dimension(s). + +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True +then the corresponding element in :attr:`input` tensor will be +included in {operation name} computation, otherwise the element is +ignored. + +When all elements of :attr:`input` along the given dimension +:attr:`dim` are ignored (fully masked-out), the corresponding element +of the output tensor will have undefined value: it may or may not +correspond to the identity value of {operation name} operation; the +choice may correspond to the value that leads to the most efficient +storage of :attr:`output` tensor. + +The mask of the output tensor can be computed as +``torch.any(torch.broadcast_to(mask, input.shape), dim, keepdim=keepdim, +dtype=torch.bool)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + {args_declarations} + +Keyword args: + {kwargs_declarations}""", + reduction_example="""\ +Example:: + + >>> input = {example_input} + >>> input + {indent_example_input} + >>> mask = {example_mask} + >>> mask + {indent_example_mask} + >>> {full_function_name}(input, {example_args}, mask=mask) + {indent_example_output} +""", + reduction_identity="""\ +The identity value of {operation name} operation, which is used to start the reduction, is ``{identity_int32}``.""", + reduction_identity_dtype="""\ +The identity value of {operation name} operation, which is used to start the +reduction, depends on input dtype. For instance, for float32, uint8, +and int32 dtypes, the identity values are ``{identity_float32}``, ``{identity_uint8}``, and ``{identity_int32}``, respectively.""", + normalization_signature="""\ +{function_name}(input, {operation_args}, *, {operation_kwargs}) -> Tensor""", + normalization_descr="""\ +Returns {operation name} of all the slices in the :attr:`input` tensor +along :attr:`dim` while the :attr:`input` elements are masked out +according to the boolean tensor :attr:`mask`. + +{definition}""", + normalization_args="""\ +The boolean tensor :attr:`mask` defines the "validity" of +:attr:`input` tensor elements: if :attr:`mask` element is True then +the corresponding element in :attr:`input` tensor will be included in +{operation name} computation, otherwise the element is ignored. + +The values of masked-out elements of the output tensor have undefined +value: it may or may not be set to zero or nan; the choice may correspond to +the value that leads to the most efficient storage of :attr:`output` +tensor. + +The mask of the {operation name} output tensor can be computed as +``torch.broadcast_to(mask, input.shape)``. + +The shapes of the :attr:`mask` tensor and the :attr:`input` tensor +don't need to match, but they must be :ref:`broadcastable +` and the dimensionality of the :attr:`mask` +tensor must not be greater than of the :attr:`input` tensor. + +Args: + input (Tensor): the input tensor + {args_declarations} + +Keyword args: + {kwargs_declarations}""", + normalization_example="""\ +Example:: + + >>> input = {example_input} + >>> input + {indent_example_input} + >>> mask = {example_mask} + >>> mask + {indent_example_mask} + >>> {full_function_name}(input, {example_args}, mask=mask) + {indent_example_output} +""", + ) + + args_and_kwargs = { + # argument name sufficies separated by double underscore will + # be removed in the final documentation string. + "sum": (("dim",), ("keepdim=False", "dtype=None", "mask=None")), + "prod": (("dim",), ("keepdim=False", "dtype=None", "mask=None")), + "cumsum": (("dim__as_int",), ("dtype=None", "mask=None")), + "cumprod": (("dim__as_int",), ("dtype=None", "mask=None")), + "amin": (("dim",), ("keepdim=False", "dtype=None", "mask=None")), + "amax": (("dim",), ("keepdim=False", "dtype=None", "mask=None")), + "argmin": (("dim__as_int",), ("keepdim=False", "dtype=None", "mask=None")), + "argmax": (("dim__as_int",), ("keepdim=False", "dtype=None", "mask=None")), + "mean": (("dim",), ("keepdim=False", "dtype=None", "mask=None")), + "median": (("dim__as_int",), ("keepdim=False", "dtype=None", "mask=None")), + "norm": ( + ( + "ord", + "dim", + ), + ("keepdim=False", "dtype=None", "mask=None"), + ), + "var": (("dim", "unbiased"), ("keepdim=False", "dtype=None", "mask=None")), + "std": (("dim", "unbiased"), ("keepdim=False", "dtype=None", "mask=None")), + "logsumexp": (("dim",), ("keepdim=False", "dtype=None", "mask=None")), + "softmax": (("dim__as_int",), ("dtype=None", "mask=None")), + "log_softmax": (("dim__as_int",), ("dtype=None", "mask=None")), + "softmin": (("dim__as_int",), ("dtype=None", "mask=None")), + "normalize": ( + ( + "ord__required", + "dim__as_int", + ), + ("eps=1e-12", "dtype=None", "mask=None"), + ), + } + + argument_declarations = { + "dim": """\ + dim (int or tuple of ints, optional): the dimension or dimensions to reduce. + Default: None that is equivalent to ``tuple(range(input.ndim))``.""", + "dim__as_int": """\ + dim (int): the dimension along which {operation name} is computed.""", + "ord": """\ + ord (int, float, optional): the order of vector norm. Default: 2. + See :func:`torch.linalg.vector_norm` for a list of supported norms.""", + "ord__required": """\ + ord (int, float): the order of vector norm. Default: 2. + See :func:`torch.linalg.vector_norm` for a list of supported norms.""", + "unbiased": """\ + unbiased (bool): when True, use Bessel's correction, otherwise, compute + the uncorrected sample variance.""", + "eps": """\ + eps (float, optional): small value to avoid division by zero. Default: {default}.""", + "keepdim": """\ + keepdim (bool, optional): whether the output tensor has + :attr:`dim` retained or not. Default: {default}.""", + "dtype": """\ + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. Default: {default}.""", + "mask": """\ + mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of input tensor + elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``.""", + } + + definitions = { + "softmax": """\ + Let ``x`` be a sequence of unmasked elements of one-dimensional slice + of the :attr:`input` tensor. Softmax of i-th element in ``x`` is + defined as ``exp(x[i])/sum(exp(x))``.""", + "log_softmax": """\ + Let ``x`` be a sequence of unmasked elements of one-dimensional slice + of the :attr:`input` tensor. LogSoftmax of i-th element in ``x`` is + defined as ``log(exp(x[i])/sum(exp(x)))``.""", + "softmin": """\ + Let ``x`` be a sequence of unmasked elements of one-dimensional slice + of the :attr:`input` tensor. Softmin of i-th element in ``x`` is + defined as ``exp(-x[i])/sum(exp(-x))``.""", + "normalize": """\ + Let ``x`` be a sequence of unmasked elements of one-dimensional slice + of the :attr:`input` tensor. Normalize of i-th element in ``x`` is + defined as ``x[i]/max(norm(x, p), eps)``.""", + "cumsum": """\ + Let ``x`` be a sequence of unmasked elements of one-dimensional slice + of the :attr:`input` tensor. Cumsum of i-th element in ``x`` is + defined as ``sum(x[:i])``.""", + "cumprod": """\ + Let ``x`` be a sequence of unmasked elements of one-dimensional slice + of the :attr:`input` tensor. Cumsum of i-th element in ``x`` is + defined as ``prod(x[:i])``.""", + } + + reduction_names = { + "sum": "sum", + "prod": "product", + "amax": "maximum", + "amin": "minimum", + "argmax": "argmax", + "argmin": "argmin", + "mean": "mean", + "median": "median", + "norm": "norm", + "var": "variance", + "std": "standard_deviation", + "logsumexp": "logsumexp", + } + + normalization_names = { + "softmax": "softmax", + "log_softmax": "log_softmax", + "softmin": "softmin", + "normalize": "normalize", + "cumsum": "cumulative_sum", + "cumprod": "cumulative_prod", + } + + operation_names = {} + operation_names.update(reduction_names) + operation_names.update(normalization_names) + + # Default example data: + example_dim = 1 + example_input = torch.tensor([[-3, -2, -1], [0, 1, 2]]) + example_mask = torch.tensor([[True, False, True], [False, False, False]]) + example_args: tuple[Any, ...] + if func.__name__ in {"norm", "normalize"}: + example_args = (2.0, example_dim) + example_input = example_input.to(dtype=torch.float32) + elif func.__name__ in {"var", "std"}: + example_args = (example_dim, False) + elif func.__name__ == "median": + example_args = (example_dim,) + example_input = example_input.to(dtype=torch.float32) + else: + example_args = (example_dim,) + + operation_args: tuple[str, ...] + operation_kwargs: tuple[str, ...] + operation_args, operation_kwargs = args_and_kwargs[func.__name__] + arg_declarations = [ + "\n ".join( + argument_declarations.get(a, f"{a.split('__', 1)[0]}: TBD.").splitlines() + ) + for a in operation_args + ] + kwarg_declarations = [ + "\n ".join( + argument_declarations.get( + a.split("=", 1)[0], f"{a.split('__', 1)[0]}: TBD." + ) + .format(default=a.split("=", 1)[1]) + .splitlines() + ) + for a in operation_kwargs + ] + + if func.__name__ in reduction_names: + op_kind = "reduction" + doc_sections = ["signature", "descr", "identity", "args", "example"] + elif func.__name__ in normalization_names: + op_kind = "normalization" + doc_sections = ["signature", "descr", "args", "example"] + example_input = example_input.to(dtype=torch.float32) + else: + assert 0 # add function name to operation names dictionaries + example_output = func(example_input, *example_args, mask=example_mask) + + template_data = { + "function_name": func.__name__, + "full_function_name": func.__module__ + "." + func.__name__, + "operation name": operation_names[func.__name__], + "operation_args": ", ".join(a.split("__", 1)[0] for a in operation_args), + "operation_kwargs": ", ".join(a.split("__", 1)[0] for a in operation_kwargs), + # one-line representation of a tensor: + "example_input": " ".join(str(example_input).split()), + "example_args": ", ".join(map(str, example_args)), + "example_mask": " ".join(str(example_mask).split()), + # multi-line representation of a tensor with indent + "indent_example_input": ("\n ").join(str(example_input).splitlines()), + "indent_example_mask": ("\n ").join(str(example_mask).splitlines()), + "indent_example_output": ("\n ").join(str(example_output).splitlines()), + } + + if func.__name__ in reduction_names: + template_data.update( + identity_uint8=_reduction_identity( + func.__name__, torch.tensor(0, dtype=torch.uint8) + ), + identity_int32=_reduction_identity( + func.__name__, torch.tensor(0, dtype=torch.int32) + ), + identity_float32=_reduction_identity( + func.__name__, torch.tensor(0, dtype=torch.float32) + ), + ) + if func.__name__ == "norm": + template_data.update( + identity_ord_ninf=_reduction_identity( + func.__name__, torch.tensor(0, dtype=torch.float32), float("-inf") + ) + ) + elif func.__name__ in normalization_names: + template_data.update(definition=definitions[func.__name__]) + else: + assert 0 # add function name to operation names dictionaries + template_data.update( + args_declarations=("\n ".join(arg_declarations)).format_map(template_data) + ) + template_data.update( + kwargs_declarations=("\n ".join(kwarg_declarations)).format_map( + template_data + ) + ) + + # Apply function name info to docstring templates: + templates = { + k: v.format_map(template_data) + for k, v in docstring_templates.items() + if k.startswith(op_kind) + } + templates.update( + (k, v.format_map(template_data) if isinstance(v, str) else v) + for k, v in template_data.items() + ) + + # Apply docstring templates to function doctring: + if func.__doc__ is None: + doc_template = "\n\n".join([f"{{{op_kind}_{sec}}}" for sec in doc_sections]) + else: + doc_template = func.__doc__ + return doc_template.format_map(templates) + + +def _reduction_identity(op_name: str, input: Tensor, *args): + """Return identity value as scalar tensor of a reduction operation on + given input, or None, if the identity value cannot be uniquely + defined for the given input. + + The identity value of the operation is defined as the initial + value to reduction operation that has a property ``op(op_identity, + value) == value`` for any value in the domain of the operation. + Or put it another way, including or excluding the identity value in + a list of operands will not change the reduction result. + + See https://github.com/pytorch/rfcs/pull/27 for more information. + + """ + dtype: DType = input.dtype + device = input.device + op_name = op_name.rsplit(".", 1)[-1] # lstrip module name when present + if op_name in {"sum", "cumsum"}: + return torch.tensor(0, dtype=dtype, device=device) + elif op_name in {"prod", "cumprod"}: + return torch.tensor(1, dtype=dtype, device=device) + elif op_name in {"amax", "argmax", "logaddexp"}: + if torch.is_floating_point(input): + return torch.tensor(-torch.inf, dtype=dtype, device=device) + elif torch.is_signed(input) or dtype == torch.uint8: + return torch.tensor(torch.iinfo(dtype).min, dtype=dtype, device=device) + elif op_name == "logsumexp": + if torch.is_floating_point(input): + return torch.tensor(-torch.inf, dtype=dtype, device=device) + elif torch.is_complex(input): + return torch.tensor(-torch.inf + 0j, dtype=dtype, device=device) + elif torch.is_signed(input) or dtype == torch.uint8: + return torch.tensor(torch.iinfo(dtype).min, dtype=dtype, device=device) + elif op_name in {"amin", "argmin"}: + if torch.is_floating_point(input): + return torch.tensor(torch.inf, dtype=dtype, device=device) + elif torch.is_signed(input) or dtype == torch.uint8: + return torch.tensor(torch.iinfo(dtype).max, dtype=dtype, device=device) + elif op_name == "mean": + # Strictly speaking, the identity value of the mean operation + # is the mean of the input. Since the mean value depends on + # the dim argument and it may be a non-scalar tensor, we + # consider the identity value of the mean operation ambiguous. + # Moreover, the mean value of empty input is undefined. + return None + elif op_name == "norm": + ord = args[0] if args else 2 + if ord == float("-inf"): + assert torch.is_floating_point(input), input.dtype + return torch.tensor(torch.inf, dtype=dtype, device=device) + return torch.tensor(0, dtype=dtype, device=device) + elif op_name == "median": + # We use NaN for now because the implementation is currently using torch.nanmedian + # and NaN is the identity for that function since it gets ignored + dtype = input.dtype if torch.is_floating_point(input) else torch.float + return torch.tensor(torch.nan, dtype=dtype, device=device) + elif op_name in {"var", "std"}: + return None + raise NotImplementedError(f"identity of {op_name} on {dtype} input") + + +def _canonical_dim(dim: DimOrDims, ndim: int) -> tuple[int, ...]: + """Return dim argument as a tuple of sorted dim values.""" + dims: list[int] = [] + if dim == (): + # Currently, `dim=()` in reductions operations means "reduce + # over all dimensions" while in future, it will read "no + # reduce". See https://github.com/pytorch/pytorch/issues/29137 + # When gh-29137 is resolved, this if-block must be deleted. + dim = None + if dim is None: + return tuple(range(ndim)) + ndim = max(ndim, 1) + dim_ = (dim,) if isinstance(dim, (int, torch.SymInt)) else dim + for d in dim_: + if d in dims: + raise RuntimeError(f"dim={d} appears multiple times in the list of dims") + if d >= ndim or d < -ndim: + raise IndexError( + f"Dimension out of range (expected to be in range of [{-ndim}, {ndim - 1}], but got {d})" + ) + # pyrefly: ignore [bad-argument-type] + dims.append(d % ndim) + return tuple(sorted(dims)) + + +def _sparse_coo_flatten_indices(indices: Tensor, shape: tuple): + # Flatted N-D indices to 1-D indices + flat_indices = indices.new_zeros(indices.size(1)) + for d, sz in enumerate(shape): + flat_indices.mul_(sz) + flat_indices.add_(indices[d]) + return flat_indices + + +def _any(input: Tensor, dim: tuple, keepdim: bool): + # Support torch.any with tuple dim argument. + # Workaround of https://github.com/pytorch/pytorch/issues/56586 + r = input + for d in reversed(dim): + r = r.any(dim=d, keepdim=keepdim) + return r + + +def _sparse_coo_where(mask: Tensor, input: Tensor, fill_value: Tensor) -> Tensor: + """Sparse variant of torch.where. Supports sparse COO and hybrid sparse COO tensors. + + _sparse_coo_where implements the following invariant: + + _sparse_coo_where(mask, input, fill_value).to_dense(fill_value) == + torch.where(mask.to_dense(), input.to_dense(), torch.full(input.shape, fill_value)) + + where `a == b` means `assertEqual(a, b)`, mask is boolean sparse + tensor, and `to_dense(fill_value)` is like `to_dense()` except + that the unspecified elements are mapped to `fill_value` rather + than to `0`. + + Returns a sparse COO tensor with the following features: + + - all specified elements correspond to masked-in elements that + have the values of the input tensor. If there exists a masked-in + element (as specified by mask) that is not specified in the + input, in the result tensor, the corresponding element has value + 0. In the dense part of the sparse tensor, the masked-out + elements are replaced with fill_value. + + - all unspecified elements correspond to masked-out elements. + """ + + assert input.layout == torch.sparse_coo + assert mask.layout == input.layout + assert mask.shape == input.shape + assert mask.dense_dim() == input.dense_dim() # TODO: eliminate this restriction + + input = input.coalesce() + + # For set operations on sparse tensor indices, we'll convert + # multi-dimensional indices to 1-D indices for efficiency. + input_flat_indices = _sparse_coo_flatten_indices( + input.indices(), input.shape[: input.sparse_dim()] + ) + mask_flat_indices = _sparse_coo_flatten_indices( + mask.indices(), mask.shape[: mask.sparse_dim()] + ) + + # the set of mask flat indices that define masked-in elements: + if mask.dense_dim() > 0: + mask_values = _any( + mask.values(), tuple(range(1, input.sparse_dim() + 1)), False + ) + else: + mask_values = mask.values() + maskin_flat_indices = mask_flat_indices[mask_values.nonzero()[:, 0]] + + def intersection(i1, i2): + union, counts = torch.cat([i1, i2]).unique(return_counts=True) + return union, torch.where(counts.gt(1)) + + def minus(i1, i2): + union, counts = torch.cat([i1, i2]).unique(return_counts=True) + return intersection(union[torch.where(counts.eq(1))], i1) + + def _apply(a): + obj, w = a + return obj[w] + + # the set of input flat indices of specified and masked-in elements: + maskin_input_flat_indices = _apply( + intersection(maskin_flat_indices, input_flat_indices) + ) + _, w = intersection(input_flat_indices, maskin_input_flat_indices) + + # the indices and values of masked-in elements + where_input_indices = input.indices()[(slice(None),) + w] + where_input_values = input.values()[w] + + if mask.dense_dim() > 0: + # apply mask to the dense part of the input values: + _, w1 = intersection(mask_flat_indices, maskin_input_flat_indices) + where_mask_values = mask.values()[w1] + where_input_values = torch.where( + where_mask_values, where_input_values, fill_value + ) + + # the set of flat indices of unspecified input and masked-in elements: + maskin_zero_flat_indices = _apply( + minus(maskin_flat_indices, maskin_input_flat_indices) + ) + + # the indices of masked-in zero elements + _, w = intersection(mask_flat_indices, maskin_zero_flat_indices) + where_zero_indices = mask.indices()[(slice(None),) + w] + + # construct result + n = where_zero_indices.size(1) + if n == 0: + # the input is coalesced, hence input_flat_indices are ordered + # and the result is guaranteed to be coalesced: + result = torch.sparse_coo_tensor( + where_input_indices, where_input_values, input.shape + ) + return result._coalesced_(True) + + where_indices = torch.cat([where_input_indices, where_zero_indices], dim=1) + where_values = torch.cat( + [ + where_input_values, + where_input_values.new_zeros((n,) + where_input_values.shape[1:]), + ] + ) + result = torch.sparse_coo_tensor(where_indices, where_values, input.shape) + + # appending zero elements leads to uncoalesced sparse tensor + return result.coalesce() + + +def _sparse_coo_scatter_reduction_helper( + op, + mask_input: Tensor, + dims: tuple[int, ...], + keepdim: bool, + dtype: DType | None = None, +) -> Tensor: + reduce = op.__name__ + valid_reductions = ["sum", "prod", "amax", "amin"] + if reduce not in valid_reductions: + raise ValueError( + f"op must be one of {' '.join(valid_reductions)}, but got {reduce} instead" + ) + + output_dtype = dtype + values, indices = mask_input._values(), mask_input._indices() + input_dims = mask_input.dim() + num_sparse_dims = mask_input.sparse_dim() + reduced_sparse_dims = [] + retained_sparse_dims = [] + reduced_dense_dims = [] + + # promote dtype if specified + if values.dtype != output_dtype: + values = values.to(output_dtype) + + if keepdim: + output_shape = tuple( + 1 if i in dims else si for (i, si) in enumerate(mask_input.shape) + ) + else: + output_shape = tuple( + si for (i, si) in enumerate(mask_input.shape) if i not in dims + ) + + for d in dims: + if d >= input_dims: + continue + + if d < num_sparse_dims: + reduced_sparse_dims.append(d) + else: + reduced_dense_dims.append(d + 1 - num_sparse_dims) + + # Reduce dense dimensions + if len(reduced_dense_dims) > 0: + if reduce == "sum": + new_values = values + new_values = op(new_values, dim=reduced_dense_dims, keepdim=bool(keepdim)) + else: + # FIXME: Implement reductions for dense dimensions for ops with non-zero reduction identities + return NotImplemented + else: + new_values = values.clone() + + # Reduce sparse dimensions + if len(reduced_sparse_dims) == num_sparse_dims: + if reduce in {"amax", "amin"} and new_values.size(0) == 0: + # IndexError: amax(): Expected reduction dim 0 to have non-zero size. + # sum()/prod() return the reduction identity when dim has size 0 but amax()/amin() do not + # See https://github.com/pytorch/pytorch/issues/61901 + new_values = _reduction_identity(reduce, new_values) + else: + new_values = op(new_values, dim=0) + if keepdim: + for _ in range(num_sparse_dims): + new_values = new_values.unsqueeze(0) + return new_values.to(dtype=output_dtype).to_sparse() + else: + new_indices = indices.clone() + if keepdim: + # zero out reduced sparse dimensions if keepdim = True + # ensures that the call to torch.unique folds duplicated indices together while preserving the dimension + new_indices[reduced_sparse_dims, :] = 0 + else: + # remove reduced sparse dimensions if keepdim = False + if len(reduced_sparse_dims) > 0: + retained_sparse_dims = [ + i + for i in range(num_sparse_dims) + if i not in set(reduced_sparse_dims) + ] + new_indices = new_indices.index_select( + 0, torch.tensor(retained_sparse_dims).to(mask_input.device) + ) + + # Use scatter_reduce to reduce items in the new_values tensor that correspond to the same indices in new_indices + if new_indices.numel() > 0: + # lexsort indices and get index tensor for scatter reduction + new_indices, inverse_indices = torch.unique( + new_indices, return_inverse=True, dim=1 + ) + out_shape = list(new_values.shape) + out_shape[0] = new_indices.shape[1] + for _ in range(new_values.ndim - 1): + inverse_indices = inverse_indices.unsqueeze(-1) + scatter_indices = inverse_indices.expand(new_values.shape) + # FIXME: temporary workaround for issue with bfloat16/float16 remove when acctype is implemented for scatter_reduce + if output_dtype in {torch.bfloat16, torch.float16}: + new_values = new_values.to(torch.float) + out = new_values.new_empty(out_shape) + new_values = out.scatter_reduce_( + 0, scatter_indices, new_values, reduce=reduce, include_self=False + ) + new_values = new_values.to(dtype=output_dtype) + else: + out = new_values.new_empty(out_shape) + new_values = out.scatter_reduce_( + 0, scatter_indices, new_values, reduce=reduce, include_self=False + ) + + return torch.sparse_coo_tensor( + new_indices, + new_values, + output_shape, + dtype=output_dtype, + device=mask_input.device, + ) + + +def _sparse_csr_segment_reduction_helper( + op, + mask_input: Tensor, + dims: tuple[int, ...], + keepdim: bool, + dtype: DType | None = None, +) -> Tensor: + # Currently, while sparse CSR is always 2D with no dense dimensions keepdim must be True + # FIXME: when dense dimensions are implemented for CSR tensors + assert keepdim, ( + "reduction operations on CSR tensors with keepdim=False is unsupported" + ) + reduce = op.__name__ + valid_reductions = ["sum", "prod", "mean", "amax", "amin"] + if reduce not in valid_reductions: + raise ValueError( + f"op must be one of {' '.join(valid_reductions)}, but got {reduce} instead" + ) + device = mask_input.device + output_dtype = dtype + values, crow_indices, col_indices = ( + mask_input.values(), + mask_input.crow_indices(), + mask_input.col_indices(), + ) + + # promote dtype if specified + if values.dtype != output_dtype: + values = values.to(output_dtype) + + if len(dims) == 0: + return mask_input + if len(dims) == 1: + if dims[0] == 0: + new_col_indices, scatter_indices = torch.unique( + col_indices, return_inverse=True + ) + new_nnz = new_col_indices.shape[0] + new_crow_indices = torch.tensor([0, new_nnz]) + new_values = values.new_empty(new_col_indices.shape) + new_values.scatter_reduce_( + 0, scatter_indices, values, reduce, include_self=False + ) + new_shape = [1, mask_input.size(1)] + else: + assert dims[0] == 1, ( + "Sparse CSR tensors are 2D and only support reduction along dim 0 or 1." + ) + # all intervals new_crow_indices[i] - new_crow_indices[i-1] are 1 + # except for where crow_indices[i] == crow_indices[i-1] where the interval remains as 0 + new_crow_indices = torch.cat( + ( + crow_indices.new_zeros(1), + torch.cumsum(torch.diff(crow_indices) != 0, 0), + ), + 0, + ) + new_nnz = new_crow_indices[-1] + new_col_indices = col_indices.new_zeros(new_nnz) # type: ignore[call-overload] + new_values = torch._segment_reduce(values, reduce, offsets=crow_indices) # type: ignore[attr-defined] + new_shape = [mask_input.size(0), 1] + else: + assert len(dims) == 2 + nnz = min(1, values.numel()) + if nnz == 1: + op_kwargs = {"keepdim": True, "dtype": output_dtype} + # amax and amin do not support dtype kwarg + if reduce in ["amax", "amin"]: + del op_kwargs["dtype"] + new_values = op(values, 0, **op_kwargs) + else: + new_values = torch.empty(0, dtype=output_dtype) + new_col_indices = col_indices.new_zeros(nnz) + new_crow_indices = torch.tensor([0, nnz]) + new_shape = [1, nnz] + + return torch.sparse_csr_tensor( + new_crow_indices, + new_col_indices, + new_values, + new_shape, + dtype=output_dtype, + device=device, + ) + + +def _sparse_csr_where(mask: Tensor, input: Tensor, fill_value: Tensor) -> Tensor: + """Sparse variant of torch.where. Supports sparse CSR tensors.""" + # TODO: implement sparse CSR specific where operator for efficiency + return _sparse_coo_where( + mask.to_sparse_coo(), input.to_sparse_coo(), fill_value + ).to_sparse_csr() + + +def _where(mask: Tensor, input: Tensor, fill_value: Tensor) -> Tensor: + """torch.where with sparse inputs support. + + _where implements the following invariant: + + _where(mask, input, fill_value).to_dense(fill_value) == + torch.where(mask.to_dense(), input.to_dense(), torch.full(input.shape, fill_value)) + + where `a == b` means `assertEqual(a, b)`, mask is boolean sparse + tensor, and `to_dense(fill_value)` is like `to_dense()` except + that the unspecified elements are mapped to `fill_value` rather + than to `0`. + + Returns a sparse tensor with the following features: + + - all specified elements correspond to masked-in elements that + have the values of the input tensor. If there exists a masked-in + element (as specified by mask) that is not specified in the + input, in the result tensor, the corresponding element has value + 0. In the dense part of the sparse tensor, the masked-out + elements are replaced with fill_value. + + - all unspecified elements correspond to masked-out elements. + """ + if mask.layout == torch.strided: + return torch.where(mask, input, fill_value) + elif mask.layout == torch.sparse_coo: + return _sparse_coo_where(mask, input, fill_value) + elif mask.layout == torch.sparse_csr: + return _sparse_csr_where(mask, input, fill_value) + else: + raise ValueError( + f"_where expects strided or sparse COO or sparse CSR tensor but got {mask.layout}" + ) + + +def _input_mask(input: Tensor | MaskedTensor, *args, **kwargs) -> Tensor: + """Return canonical input mask. + + A canonical input mask is defined as a boolean mask tensor that + shape and layout matches with the shape and the layout of the + input. + + The canonical input mask is computed from the :attr:`mask` tensor + content to meet the following criteria: + + 1. The shape of the canonical input mask is the same as the shape + of :attr:`input` tensor. If the mask tensor has a smaller shape + than the shape of the :attr:`input`, broadcasting rules will be + applied. Downcasting of mask is not supported. + + 2. The layout of the canonical input mask is the same as the + layout of the :attr:`input` tensor. If the mask has different + layout, it will be converted to the expected layout. In the + case of sparse COO layout, the canonical input mask will be + coalesced. + + 3. The dtype of the canonical input mask is torch.bool. If the + mask dtype is not bool then it will be converted to bool dtype + using `.to(dtype=bool)` method call. + + 4. The elements of the canonical input mask have boolean values + copied from the content of the :attr:`mask` tensor (after + possible broadcasting and dtype conversion transforms). In + general, the sparsity pattern of the sparse canonical input + mask need not to be the same as the sparsity pattern of the + sparse :attr:`input` tensor. + + """ + if input.layout not in {torch.strided, torch.sparse_coo, torch.sparse_csr}: + raise ValueError( + f"_input_mask expects strided or sparse COO or sparse CSR tensor but got {input.layout}" + ) + + mask = kwargs.get("mask") + + # default mask + if mask is None: + raise ValueError("_input_mask requires explicit mask") + + # mask shape must match with input shape + if mask.shape != input.shape: + if mask.ndim > input.ndim: + raise IndexError( + "_input_mask expected broadcastable mask (got mask dimensionality higher than of the input)" + ) + if mask.layout == torch.strided: + mask = torch.broadcast_to(mask.clone(), input.shape).to(dtype=torch.bool) + elif mask.layout == torch.sparse_coo: + mask = torch._sparse_broadcast_to(mask, input.shape) + else: + assert mask.layout == torch.sparse_csr + # Broadcasting of CSR tensors is not implemented. Working + # around by using COO layout. + mask = torch._sparse_broadcast_to( + mask.to_sparse(), input.shape + ).to_sparse_csr() + + # mask layout must match with input layout + if mask.layout != input.layout: + if input.layout == torch.strided: + mask = mask.to_dense() + elif input.layout == torch.sparse_coo: + if mask.layout == torch.strided: + mask = mask.to_sparse(input.sparse_dim()) + else: + mask = mask.to_sparse() + else: + assert input.layout == torch.sparse_csr + mask = mask.to_sparse_csr() + + # sparse mask must be coalesced + if mask.layout == torch.sparse_coo: + mask = mask.coalesce() + + # mask is a boolean tensor + mask = mask.to(dtype=torch.bool) + + return mask + + +def _output_mask(op, input: Tensor, *args, **kwargs) -> Tensor: + """Return output mask of masked operation applied to given arguments.""" + if callable(op): + is_reduction = op.__name__ in { + "sum", + "prod", + "amax", + "amin", + "argmax", + "argmin", + "mean", + "median", + "norm", + "var", + "std", + "logsumexp", + } + is_normalization = op.__name__ in { + "softmax", + "log_softmax", + "softmin", + "normalize", + "cumsum", + "cumprod", + } + if is_reduction: + if op.__name__ == "norm": + if args: + args = args[1:] # lstrip ord argument + dim = args[0] if args else kwargs.get("dim") + outmask = _input_mask(input, *args, **kwargs) + keepdim = kwargs.get("keepdim", False) + dim_ = _canonical_dim(dim, input.ndim) + return _any(outmask, dim_, bool(keepdim)) + elif is_normalization: + return _input_mask(input, *args, **kwargs) + else: + raise ValueError( + f"_output_mask expected masked operation (got callable {op.__module__}.{op.__name__})" + ) + else: + raise ValueError( + f"_output_mask expected masked operation (got {type(op).__name__} object)" + ) + + +def _combine_input_and_mask(op, input: MaskedTensor | Tensor, mask, *args) -> Tensor: + def helper(input, mask): + if mask is None: + return input + canonical_mask = _input_mask(input, mask=mask) + if callable(op): + fill_value = _reduction_identity(op.__name__, input, *args) + return _where(canonical_mask, input, fill_value) + else: + raise ValueError( + f"_combine_input_and_mask expected masked operation (got {type(op).__name__} object)" + ) + + class Combine(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(ctx, input, mask): + """Return input with masked-out elements eliminated for the given operations.""" + ctx.save_for_backward(mask) + + if mask is not None: + ctx.mark_non_differentiable(mask) + + return helper(input, mask) + + @staticmethod + # pyrefly: ignore [bad-override] + def backward(ctx, grad_output): + (mask,) = ctx.saved_tensors + grad_data = ( + grad_output.get_data() if is_masked_tensor(grad_output) else grad_output + ) + result = as_masked_tensor(grad_data, mask) + return result, None + + return ( + Combine.apply(input.get_data(), input.get_mask()) # type: ignore[union-attr] + if is_masked_tensor(input) + else helper(input, mask) + ) + + +@_apply_docstring_templates +def sum( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + # __doc__ is generated by _apply_docstring_templates decorator + if dtype is None: + # promote integer types to int64 when output dtype is not specified + if input.layout == torch.sparse_csr: + if input.dtype in { + torch.uint8, + torch.bool, + torch.int8, + torch.int16, + torch.int32, + }: + # csr.to(dtype=torch.int64) is not implemented, so + # using coo.to on input to ensure the promoted dtype + input = input.to_sparse_coo().to(dtype=torch.int64).to_sparse_csr() + else: + dtype = input.dtype + else: + dtype = input.dtype + if input.dtype in { + torch.uint8, + torch.bool, + torch.int8, + torch.int16, + torch.int32, + }: + dtype = torch.int64 + dim_ = _canonical_dim(dim, input.ndim) + mask_input = _combine_input_and_mask(sum, input, mask) + if mask_input.layout == torch.strided: + return torch.sum(mask_input, dim_, bool(keepdim), dtype=dtype) + elif mask_input.layout == torch.sparse_coo: + return _sparse_coo_scatter_reduction_helper( + torch.sum, mask_input, dim_, bool(keepdim), dtype + ) + elif mask_input.layout == torch.sparse_csr: + return torch._sparse_csr_sum( + mask_input, dim=list(dim_), keepdim=bool(keepdim), dtype=dtype + ) + else: + raise ValueError( + f"masked sum expects strided, sparse_coo or sparse_csr tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def prod( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + # __doc__ is generated by _apply_docstring_templates decorator + if dtype is None: + # promote integer types to int64 when output dtype is not specified + if input.layout == torch.sparse_csr: + if input.dtype in { + torch.uint8, + torch.bool, + torch.int8, + torch.int16, + torch.int32, + }: + # csr.to(dtype=torch.int64) is not implemented, so + # using coo.to on input to ensure the promoted dtype + input = input.to_sparse_coo().to(dtype=torch.int64).to_sparse_csr() + else: + dtype = input.dtype + else: + dtype = input.dtype + if input.dtype in { + torch.uint8, + torch.bool, + torch.int8, + torch.int16, + torch.int32, + }: + dtype = torch.int64 + dim_ = _canonical_dim(dim, input.ndim) + mask_input = _combine_input_and_mask(prod, input, mask) + if mask_input.layout == torch.strided: + # Workaround https://github.com/pytorch/pytorch/issues/56586 + result = mask_input + result = result.to(dtype=dtype) + for d in reversed(dim_): + result = result.prod(dim=d, keepdim=bool(keepdim)) + return result + elif mask_input.layout == torch.sparse_coo: + if mask is None: + # See comment in the sparse_csr branch, the same issue arises for sparse_coo tensors + raise ValueError( + "masked prod expects explicit mask for sparse_coo tensor input" + ) + return _sparse_coo_scatter_reduction_helper( + torch.prod, mask_input, dim_, bool(keepdim), dtype + ) + elif mask_input.layout == torch.sparse_csr: + if mask is None: + # mask is None corresponds to all-True mask. The + # unspecified elements in the CSR tensor correspond to + # zero values. Hence, the prod reduction result is + # automatically zero unless all elements are specified. + # A semi-optimal way to take this into account is to use: + # + # masked_prod(csr, ..., mask=None) == torch._sparse_csr_prod(csr, ...) * all(csr.nonzero(), ...) + # + # but that requires implementing `all` and `nonzero` + # support for sparse csr tensors. + raise ValueError( + "masked prod expects explicit mask for sparse_csr tensor input" + ) + return torch._sparse_csr_prod( + mask_input, dim=list(dim_), keepdim=bool(keepdim), dtype=dtype + ) + else: + raise ValueError( + f"masked prod expects strided, sparse_coo or sparse_csr tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def cumsum( + input: Tensor, + dim: int, + *, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim)[0] + mask_input = _combine_input_and_mask(sum, input, mask) + if mask_input.layout == torch.strided: + return torch.cumsum(mask_input, dim_, dtype=dtype).to(dtype=dtype) + else: + raise ValueError( + f"masked cumsum expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def cumprod( + input: Tensor, + dim: int, + *, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim)[0] + mask_input = _combine_input_and_mask(prod, input, mask) + if mask_input.layout == torch.strided: + return torch.cumprod(mask_input, dim_, dtype=dtype).to(dtype=dtype) + else: + raise ValueError( + f"masked cumprod expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def amax( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} + +{reduction_descr} + +{reduction_identity_dtype} + +{reduction_args} + +{reduction_example}""" + if dtype is None: + dtype = input.dtype + + mask_input = _combine_input_and_mask(amax, input, mask) + dim_ = _canonical_dim(dim, mask_input.ndim) + if mask_input.layout == torch.strided: + return torch.amax(mask_input, dim_, bool(keepdim)).to(dtype=dtype) + elif mask_input.layout == torch.sparse_coo: + if mask is None: + # See comment in the sparse_csr branch of prod, a similar issue arises here + # where unspecified elements along a dimension may need to be reduced with the result + raise ValueError( + "masked amax expects explicit mask for sparse_coo tensor input" + ) + return _sparse_coo_scatter_reduction_helper( + torch.amax, mask_input, dim_, bool(keepdim), dtype + ) + elif mask_input.layout == torch.sparse_csr: + if mask is None: + raise ValueError( + "masked amax expects explicit mask for sparse_csr tensor input" + ) + return _sparse_csr_segment_reduction_helper( + torch.amax, mask_input, dim_, bool(keepdim), dtype + ) + else: + raise ValueError( + f"masked amax expects strided, sparse_coo or sparse_csr tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def amin( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} + +{reduction_descr} + +{reduction_identity_dtype} + +{reduction_args} + +{reduction_example}""" + if dtype is None: + dtype = input.dtype + + mask_input = _combine_input_and_mask(amin, input, mask) + dim_ = _canonical_dim(dim, mask_input.ndim) + if mask_input.layout == torch.strided: + return torch.amin(mask_input, dim_, bool(keepdim)).to(dtype=dtype) + elif mask_input.layout == torch.sparse_coo: + if mask is None: + # See comment in the sparse_csr branch of prod, a similar issue arises here + # where unspecified elements along a dimension may need to be reduced with the result + raise ValueError( + "masked amax expects explicit mask for sparse_coo tensor input" + ) + return _sparse_coo_scatter_reduction_helper( + torch.amin, mask_input, dim_, bool(keepdim), dtype + ) + elif mask_input.layout == torch.sparse_csr: + if mask is None: + raise ValueError( + "masked amin expects explicit mask for sparse_csr tensor input" + ) + return _sparse_csr_segment_reduction_helper( + torch.amin, mask_input, dim_, bool(keepdim), dtype + ) + else: + raise ValueError( + f"masked amin expects strided, sparse_coo or sparse_csr tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def argmax( + input: Tensor | MaskedTensor, + dim: int | None = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} +{reduction_descr} +{reduction_identity_dtype} +{reduction_args} +{reduction_example}""" + if dtype is None: + dtype = input.dtype + mask_input = _combine_input_and_mask(argmax, input, mask) + if mask_input.layout == torch.strided: + return torch.argmax(mask_input, dim, bool(keepdim)).to(dtype=dtype) + else: + raise ValueError( + f"masked argmax expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def argmin( + input: Tensor | MaskedTensor, + dim: int | None = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} +{reduction_descr} +{reduction_identity_dtype} +{reduction_args} +{reduction_example}""" + if dtype is None: + dtype = input.dtype + mask_input = _combine_input_and_mask(argmin, input, mask) + if mask_input.layout == torch.strided: + return torch.argmin(mask_input, dim, bool(keepdim)).to(dtype=dtype) + else: + raise ValueError( + f"masked argmin expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def mean( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} + +{reduction_descr} + +By definition, the identity value of a mean operation is the mean +value of the tensor. If all elements of the input tensor along given +dimension(s) :attr:`dim` are masked-out, the identity value of the +mean is undefined. Due to this ambiguity, the elements of output +tensor with strided layout, that correspond to fully masked-out +elements, have ``nan`` values. + +{reduction_args} + +{reduction_example}""" + dtype_source = "Optional" + if dtype is None: + dtype = input.dtype + dtype_source = "Input" + + if not (dtype.is_floating_point or dtype.is_complex): + raise ValueError( + f"mean(): Could not infer output dtype. {dtype_source} dtype must be either " + f"a floating point or complex dtype. Got: {dtype}" + ) + if input.layout == torch.strided: + if mask is None: + # TODO: compute count analytically + # pyrefly: ignore [no-matching-overload] + count = sum( + torch.ones(input.shape, dtype=torch.int64, device=input.device), + dim, + keepdim=keepdim, + ) + # pyrefly: ignore [no-matching-overload] + total = sum(input, dim, keepdim=keepdim, dtype=dtype) + else: + inmask = _input_mask(input, mask=mask) + count = inmask.sum(dim=dim, keepdim=bool(keepdim)) + # pyrefly: ignore [no-matching-overload] + total = sum(input, dim, keepdim=keepdim, dtype=dtype, mask=inmask) + return total / count + elif input.layout == torch.sparse_csr: + mask_input = _combine_input_and_mask(mean, input, mask) + dim_ = _canonical_dim(dim, mask_input.ndim) + if mask is None: + raise ValueError( + "masked mean expects explicit mask for sparse_csr tensor input" + ) + return _sparse_csr_segment_reduction_helper( + torch.mean, mask_input, dim_, bool(keepdim), dtype + ) + else: + raise ValueError( + f"masked mean expects strided or sparse_csr tensor (got {input.layout} tensor)" + ) + + +@_apply_docstring_templates +def median( + input: Tensor | MaskedTensor, + dim: int = -1, + *, + keepdim: bool = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} +{reduction_descr} +By definition, the identity value of a median operation is the median +value of the tensor. If all elements of the input tensor along given +dimension(s) :attr:`dim` are masked-out, the identity value of the +median is undefined. Due to this ambiguity, the elements of output +tensor with strided layout, that correspond to fully masked-out +elements, have ``nan`` values. +{reduction_args} +{reduction_example}""" + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim)[0] + is_float = torch.is_floating_point(input) + if not is_float: + input = input.to(dtype=torch.float) + mask_input = _combine_input_and_mask(median, input, mask) + if mask_input.layout == torch.strided: + output = torch.nanmedian(mask_input, dim_, keepdim).values + if is_float: + return output + elif not is_float and not torch.isnan(output).any(): + return output.to(dtype=dtype) + else: + raise ValueError( + "masked median expects no fully masked out rows if dtype is not floating point" + ) + else: + raise ValueError( + f"masked median expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def logsumexp( + input: Tensor, + dim: DimOrDims = None, + *, + keepdim: bool = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim) + mask_input = _combine_input_and_mask(logsumexp, input, mask) + if mask_input.layout == torch.strided: + return torch.logsumexp(mask_input, dim_, keepdim=keepdim).to(dtype=dtype) + else: + raise ValueError( + f"masked logsumexp expects strided tensor (got {mask_input.layout} tensor)" + ) + + +# Cannot use _apply_docstring_templates as it is only set up for reductions and normalizations +def logaddexp( + input: Tensor | MaskedTensor, + other: Tensor | MaskedTensor, + *, + dtype: DType | None = None, + input_mask: Tensor | None = None, + other_mask: Tensor | None = None, +) -> Tensor: + """logaddexp(input, other, *, dtype=None, input_mask=None, other_mask=None) -> Tensor + + Returns logaddexp of all the elements in the :attr:`input` and the :attr:`other` + tensor. The :attr:`input` elements are masked out according to the boolean tensor + :attr:`input_mask` and the attr:`other` elements are masked out according to the boolean tensor + :attr:`other_mask`. + + The shapes of a mask tensor and the tensor to be masked + don't need to match, but they must be :ref:`broadcastable + ` and the dimensionality of the mask + tensor must not be greater than of the tensor to be masked. + + Args: + input (Tensor): the input tensor + other (Tensor): the second input tensor + + Keyword args: + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the output tensor is + casted to :attr:`dtype` after the operation is + performed. Default: None. + input_mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of :attr:`input` tensor elements. + Default: None that is equivalent to ``torch.ones(input.shape, dtype=torch.bool)``. + other_mask (:class:`torch.Tensor`, optional): the boolean tensor + containing the binary mask of validity of :attr:`other` tensor elements. + Default: None that is equivalent to ``torch.ones(other.shape, dtype=torch.bool)``. + + Example:: + + >>> input = torch.tensor([-100.0, -200, -300]) + >>> input + tensor([-100., -200., -300.]) + >>> other = torch.tensor([-1.0, -2, -3]) + >>> other + tensor([-1., -2., -3.]) + >>> mask = torch.tensor([True, False, True]) + >>> mask + tensor([ True, False, True]) + >>> torch.masked._ops.logaddexp(input, other, input_mask=mask, other_mask=mask) + tensor([-1., -inf, -3.])""" + if dtype is None: + dtype = input.dtype + if input.layout == torch.strided and other.layout == torch.strided: + mask_input = _combine_input_and_mask(logaddexp, input, input_mask) + mask_other = _combine_input_and_mask(logaddexp, other, other_mask) + return torch.logaddexp(mask_input, mask_other).to(dtype=dtype) + else: + raise ValueError( + f"masked logaddexp expects strided tensors (got {input.layout} tensor for input, {other.layout} for other)" + ) + + +@_apply_docstring_templates +def norm( + input: Tensor | MaskedTensor, + ord: float | None = 2.0, + dim: DimOrDims = None, + *, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} + +{reduction_descr} + +The identity value of norm operation, which is used to start the +reduction, is ``{identity_float32}``, except for ``ord=-inf`` it is +``{identity_ord_ninf}``. + +{reduction_args} + +{reduction_example}""" + if dtype is None: + dtype = input.dtype + mask_input = _combine_input_and_mask(norm, input, mask, ord) + if mask_input.layout == torch.strided: + dim_ = _canonical_dim(dim, input.ndim) + return torch.linalg.vector_norm( + mask_input, ord, dim_, bool(keepdim), dtype=dtype + ) + else: + raise ValueError( + f"masked norm expects strided tensor (got {mask_input.layout} tensor)" + ) + + +def _std_var( + input: Tensor | MaskedTensor, + dim: DimOrDims, + unbiased: bool | None, + *, + correction_opt: int | float | None, + keepdim: bool | None, + dtype: DType | None, + mask: Tensor | None, + take_sqrt: bool | None, +) -> Tensor: + assert unbiased is None or correction_opt is None, ( + "Only one of unbiased and correction may be given" + ) + correction = 1.0 + if unbiased is not None: + correction = 1.0 if unbiased else 0.0 + if correction_opt is not None: + correction = sym_float(correction_opt) + + if dtype is None: + dtype = input.dtype + if not (dtype.is_floating_point or dtype.is_complex): + dtype = torch.float32 + compute_dtype = dtype + if not (compute_dtype.is_floating_point or compute_dtype.is_complex): + compute_dtype = torch.float32 + if input.layout == torch.strided: + if mask is None: + # TODO: compute count analytically + # pyrefly: ignore [no-matching-overload] + count = sum( + torch.ones(input.shape, dtype=torch.int64, device=input.device), + dim, + keepdim=True, + ) + # pyrefly: ignore [no-matching-overload] + sample_total = sum(input, dim, keepdim=True, dtype=dtype) + else: + inmask = _input_mask(input, mask=mask) + count = inmask.sum(dim=dim, keepdim=True) + # pyrefly: ignore [no-matching-overload] + sample_total = sum(input, dim, keepdim=True, dtype=dtype, mask=inmask) + # TODO: replace torch.subtract/divide/square/maximum with + # masked subtract/divide/square/maximum when these will be + # available. + sample_mean = torch.divide(sample_total, count) + x = torch.subtract(input, sample_mean) + if mask is None: + # pyrefly: ignore [no-matching-overload] + total = sum(x * x.conj(), dim, keepdim=keepdim, dtype=compute_dtype) + else: + total = sum( + x * x.conj(), + dim, + keepdim=keepdim, + dtype=compute_dtype, + mask=inmask, # type: ignore[possibly-undefined] + ) + if not keepdim: + count = count.reshape(total.shape) + if correction != 0: + real_dtype = ( + corresponding_real_dtype(compute_dtype) + if compute_dtype.is_complex + else compute_dtype + ) + count = count.to(real_dtype) + count = torch.subtract(count, correction) + count = torch.maximum(count, count.new_zeros([])) + output = torch.divide(total, count).to(dtype=dtype) + if take_sqrt: + output = torch.sqrt(output) + return output + else: + raise ValueError( + f"masked std/var expects strided tensor (got {input.layout} tensor)" + ) + + +@_apply_docstring_templates +def var( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + unbiased: bool | None = None, + *, + correction: int | float | None = None, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} +{reduction_descr} +The identity value of sample variance operation is undefined. The +elements of output tensor with strided layout, that correspond to +fully masked-out elements, have ``nan`` values. +{reduction_args} +{reduction_example}""" + return _std_var( + input=input, + dim=dim, + unbiased=unbiased, + correction_opt=correction, + keepdim=keepdim, + dtype=dtype, + mask=mask, + take_sqrt=False, + ) + + +@_apply_docstring_templates +def std( + input: Tensor | MaskedTensor, + dim: DimOrDims = None, + unbiased: bool | None = None, + *, + correction: int | None = None, + keepdim: bool | None = False, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + """\ +{reduction_signature} +{reduction_descr} +The identity value of sample standard deviation operation is undefined. The +elements of output tensor with strided layout, that correspond to +fully masked-out elements, have ``nan`` values. +{reduction_args} +{reduction_example}""" + return _std_var( + input=input, + dim=dim, + unbiased=unbiased, + correction_opt=correction, + keepdim=keepdim, + dtype=dtype, + mask=mask, + take_sqrt=True, + ) + + +@_apply_docstring_templates +def softmax( + input: Tensor | MaskedTensor, + dim: int, + *, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim)[0] + mask_input = _combine_input_and_mask(amax, input, mask) + if mask_input.layout == torch.strided: + return torch.nn.functional.softmax(mask_input, dim_, dtype=dtype) + else: + raise ValueError( + f"masked softmax expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def log_softmax( + input: Tensor | MaskedTensor, + dim: int, + *, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim)[0] + mask_input = _combine_input_and_mask(amax, input, mask) + if mask_input.layout == torch.strided: + return torch.nn.functional.log_softmax(mask_input, dim_, dtype=dtype) + else: + raise ValueError( + f"masked log_softmax expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def softmin( + input: Tensor | MaskedTensor, + dim: int, + *, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + dim_ = _canonical_dim(dim, input.ndim)[0] + mask_input = _combine_input_and_mask(amin, input, mask) + if mask_input.layout == torch.strided: + return torch.nn.functional.softmin(mask_input, dim_, dtype=dtype) + else: + raise ValueError( + f"masked softmin expects strided tensor (got {mask_input.layout} tensor)" + ) + + +@_apply_docstring_templates +def normalize( + input: Tensor | MaskedTensor, + ord: float, + dim: int, + *, + eps: float = 1e-12, + dtype: DType | None = None, + mask: Tensor | None = None, +) -> Tensor: + if dtype is None: + dtype = input.dtype + # TODO: eliminate mask_input as unnecessary when using masked divide. + mask_input = _combine_input_and_mask(sum, input, mask) + if mask_input.layout == torch.strided: + nrm_ = norm(input, ord, dim, keepdim=True, dtype=dtype, mask=mask) + # TODO: replace torch.maximum with masked maximum when available. + denom = torch.maximum(nrm_, nrm_.new_full([], eps)) + # TODO: replace torch.divide with masked divide when available. + return torch.divide(mask_input, denom) + else: + raise ValueError( + f"masked normalize expects strided tensor (got {mask_input.layout} tensor)" + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/monitor/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/monitor/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..89af9a227ecda7e71a43cc00db2551e7ae10732e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/monitor/__init__.py @@ -0,0 +1,39 @@ +from typing import TYPE_CHECKING + +from torch._C._monitor import * # noqa: F403 +from torch._C._monitor import _WaitCounter, _WaitCounterTracker + + +if TYPE_CHECKING: + from torch.utils.tensorboard import SummaryWriter + +STAT_EVENT = "torch.monitor.Stat" + + +class TensorboardEventHandler: + """ + TensorboardEventHandler is an event handler that will write known events to + the provided SummaryWriter. + + This currently only supports ``torch.monitor.Stat`` events which are logged + as scalars. + + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_MONITOR) + >>> # xdoctest: +REQUIRES(module:tensorboard) + >>> from torch.utils.tensorboard import SummaryWriter + >>> from torch.monitor import TensorboardEventHandler, register_event_handler + >>> writer = SummaryWriter("log_dir") + >>> register_event_handler(TensorboardEventHandler(writer)) + """ + + def __init__(self, writer: "SummaryWriter") -> None: + """ + Constructs the ``TensorboardEventHandler``. + """ + self._writer = writer + + def __call__(self, event: Event) -> None: + if event.name == STAT_EVENT: + for k, v in event.data.items(): + self._writer.add_scalar(k, v, walltime=event.timestamp.timestamp()) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mps/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mps/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cdbf6b16ac443ede0abdb6f3935623886fc7aff8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mps/__init__.py @@ -0,0 +1,194 @@ +# mypy: allow-untyped-defs +r""" +This package enables an interface for accessing MPS (Metal Performance Shaders) backend in Python. +Metal is Apple's API for programming metal GPU (graphics processor unit). Using MPS means that increased +performance can be achieved, by running work on the metal GPU(s). +See https://developer.apple.com/documentation/metalperformanceshaders for more details. +""" + +from typing import Union + +import torch +from torch import Tensor + + +_is_in_bad_fork = getattr(torch._C, "_mps_is_in_bad_fork", lambda: False) +_default_mps_generator: torch._C.Generator = None # type: ignore[assignment] + + +# local helper function (not public or exported) +def _get_default_mps_generator() -> torch._C.Generator: + global _default_mps_generator + if _default_mps_generator is None: + _default_mps_generator = torch._C._mps_get_default_generator() + return _default_mps_generator + + +def device_count() -> int: + r"""Returns the number of available MPS devices.""" + return int(torch._C._has_mps and torch._C._mps_is_available()) + + +def synchronize() -> None: + r"""Waits for all kernels in all streams on a MPS device to complete.""" + return torch._C._mps_deviceSynchronize() + + +def get_rng_state(device: Union[int, str, torch.device] = "mps") -> Tensor: + r"""Returns the random number generator state as a ByteTensor. + + Args: + device (torch.device or int, optional): The device to return the RNG state of. + Default: ``'mps'`` (i.e., ``torch.device('mps')``, the current MPS device). + """ + return _get_default_mps_generator().get_state() + + +def set_rng_state( + new_state: Tensor, device: Union[int, str, torch.device] = "mps" +) -> None: + r"""Sets the random number generator state. + + Args: + new_state (torch.ByteTensor): The desired state + device (torch.device or int, optional): The device to set the RNG state. + Default: ``'mps'`` (i.e., ``torch.device('mps')``, the current MPS device). + """ + new_state_copy = new_state.clone(memory_format=torch.contiguous_format) + _get_default_mps_generator().set_state(new_state_copy) + + +def manual_seed(seed: int) -> None: + r"""Sets the seed for generating random numbers. + + Args: + seed (int): The desired seed. + """ + # the torch.mps.manual_seed() can be called from the global + # torch.manual_seed() in torch/random.py. So we need to make + # sure mps is available (otherwise we just return without + # erroring out) + if not torch._C._has_mps: + return + seed = int(seed) + _get_default_mps_generator().manual_seed(seed) + + +def seed() -> None: + r"""Sets the seed for generating random numbers to a random number.""" + _get_default_mps_generator().seed() + + +def empty_cache() -> None: + r"""Releases all unoccupied cached memory currently held by the caching + allocator so that those can be used in other GPU applications. + """ + torch._C._mps_emptyCache() + + +def set_per_process_memory_fraction(fraction) -> None: + r"""Set memory fraction for limiting process's memory allocation on MPS device. + The allowed value equals the fraction multiplied by recommended maximum device memory + (obtained from Metal API device.recommendedMaxWorkingSetSize). + If trying to allocate more than the allowed value in a process, it will raise an out of + memory error in allocator. + + Args: + fraction(float): Range: 0~2. Allowed memory equals total_memory * fraction. + + .. note:: + Passing 0 to fraction means unlimited allocations + (may cause system failure if out of memory). + Passing fraction greater than 1.0 allows limits beyond the value + returned from device.recommendedMaxWorkingSetSize. + """ + + if not isinstance(fraction, float): + raise TypeError("Invalid type for fraction argument, must be `float`") + if fraction < 0 or fraction > 2: + raise ValueError(f"Invalid fraction value: {fraction}. Allowed range: 0~2") + + torch._C._mps_setMemoryFraction(fraction) + + +def current_allocated_memory() -> int: + r"""Returns the current GPU memory occupied by tensors in bytes. + + .. note:: + The returned size does not include cached allocations in + memory pools of MPSAllocator. + """ + return torch._C._mps_currentAllocatedMemory() + + +def driver_allocated_memory() -> int: + r"""Returns total GPU memory allocated by Metal driver for the process in bytes. + + .. note:: + The returned size includes cached allocations in MPSAllocator pools + as well as allocations from MPS/MPSGraph frameworks. + """ + return torch._C._mps_driverAllocatedMemory() + + +def recommended_max_memory() -> int: + r"""Returns recommended max Working set size for GPU memory in bytes. + + .. note:: + Recommended max working set size for Metal. + returned from device.recommendedMaxWorkingSetSize. + """ + return torch._C._mps_recommendedMaxMemory() + + +def compile_shader(source: str): + r"""Compiles compute shader from source and allows one to invoke kernels + defined there from the comfort of Python runtime + Example:: + + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_MPS) + >>> lib = torch.mps.compile_shader( + ... "kernel void full(device float* out, constant float& val, uint idx [[thread_position_in_grid]]) { out[idx] = val; }" + ... ) + >>> x = torch.zeros(16, device="mps") + >>> lib.full(x, 3.14) + """ + from pathlib import Path + + from torch.utils._cpp_embed_headers import _embed_headers + + if not hasattr(torch._C, "_mps_compileShader"): + raise RuntimeError("MPS is not available") + source = _embed_headers( + [l + "\n" for l in source.split("\n")], + [Path(__file__).parent.parent / "include"], + set(), + ) + return torch._C._mps_compileShader(source) + + +def is_available() -> bool: + return device_count() > 0 + + +from . import profiler +from .event import Event + + +__all__ = [ + "compile_shader", + "device_count", + "get_rng_state", + "manual_seed", + "seed", + "set_rng_state", + "synchronize", + "empty_cache", + "set_per_process_memory_fraction", + "current_allocated_memory", + "driver_allocated_memory", + "Event", + "profiler", + "recommended_max_memory", + "is_available", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mps/event.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mps/event.py new file mode 100644 index 0000000000000000000000000000000000000000..3f597c66a41f835fb4a7c58438c788b5025b8383 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mps/event.py @@ -0,0 +1,45 @@ +import torch + + +class Event: + r"""Wrapper around an MPS event. + + MPS events are synchronization markers that can be used to monitor the + device's progress, to accurately measure timing, and to synchronize MPS streams. + + Args: + enable_timing (bool, optional): indicates if the event should measure time + (default: ``False``) + """ + + def __init__(self, enable_timing: bool = False) -> None: + self.__eventId = torch._C._mps_acquireEvent(enable_timing) + + def __del__(self) -> None: + # checks if torch._C is already destroyed + if hasattr(torch._C, "_mps_releaseEvent") and self.__eventId > 0: + torch._C._mps_releaseEvent(self.__eventId) + + def record(self) -> None: + r"""Records the event in the default stream.""" + torch._C._mps_recordEvent(self.__eventId) + + def wait(self) -> None: + r"""Makes all future work submitted to the default stream wait for this event.""" + torch._C._mps_waitForEvent(self.__eventId) + + def query(self) -> bool: + r"""Returns True if all work currently captured by event has completed.""" + return torch._C._mps_queryEvent(self.__eventId) + + def synchronize(self) -> None: + r"""Waits until the completion of all work currently captured in this event. + This prevents the CPU thread from proceeding until the event completes. + """ + torch._C._mps_synchronizeEvent(self.__eventId) + + def elapsed_time(self, end_event: "Event") -> float: + r"""Returns the time elapsed in milliseconds after the event was + recorded and before the end_event was recorded. + """ + return torch._C._mps_elapsedTimeOfEvents(self.__eventId, end_event.__eventId) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mps/profiler.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mps/profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..daaaf729ff2e699a9d93514d182116d366daa38d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mps/profiler.py @@ -0,0 +1,100 @@ +import contextlib +from collections.abc import Iterator +from typing import Literal + +import torch + + +__all__ = [ + "start", + "stop", + "profile", + "metal_capture", + "is_metal_capture_enabled", + "is_capturing_metal", +] + + +ProfilerMode = Literal["interval", "event", "interval,event"] + + +def start(mode: ProfilerMode = "interval", wait_until_completed: bool = False) -> None: + r"""Start OS Signpost tracing from MPS backend. + + The generated OS Signposts could be recorded and viewed in + XCode Instruments Logging tool. + + Args: + mode(str): OS Signpost tracing mode could be "interval", "event", + or both "interval,event". + The interval mode traces the duration of execution of the operations, + whereas event mode marks the completion of executions. + See document `Recording Performance Data`_ for more info. + wait_until_completed(bool): Waits until the MPS Stream complete + executing each encoded GPU operation. This helps generating single + dispatches on the trace's timeline. + Note that enabling this option would affect the performance negatively. + + .. _Recording Performance Data: + https://developer.apple.com/documentation/os/logging/recording_performance_data + """ + mode_normalized = mode.lower().replace(" ", "") + torch._C._mps_profilerStartTrace( # type: ignore[attr-defined] + mode_normalized, wait_until_completed + ) + + +def stop() -> None: + r"""Stops generating OS Signpost tracing from MPS backend.""" + torch._C._mps_profilerStopTrace() # type: ignore[attr-defined] + + +@contextlib.contextmanager +def profile( + mode: ProfilerMode = "interval", wait_until_completed: bool = False +) -> Iterator[None]: + r"""Context Manager to enabling generating OS Signpost tracing from MPS backend. + + Args: + mode(str): OS Signpost tracing mode could be "interval", "event", + or both "interval,event". + The interval mode traces the duration of execution of the operations, + whereas event mode marks the completion of executions. + See document `Recording Performance Data`_ for more info. + wait_until_completed(bool): Waits until the MPS Stream complete + executing each encoded GPU operation. This helps generating single + dispatches on the trace's timeline. + Note that enabling this option would affect the performance negatively. + + .. _Recording Performance Data: + https://developer.apple.com/documentation/os/logging/recording_performance_data + """ + try: + start(mode, wait_until_completed) + yield + finally: + stop() + + +def is_metal_capture_enabled() -> bool: + """Checks if `metal_capture` context manager is usable + To enable metal capture, set MTL_CAPTURE_ENABLED envvar + """ + return torch._C._mps_isCaptureEnabled() # type: ignore[attr-defined, no-any-return] + + +def is_capturing_metal() -> bool: + """Checks if metal capture is in progress""" + return torch._C._mps_isCapturing() # type: ignore[attr-defined, no-any-return] + + +@contextlib.contextmanager +def metal_capture(fname: str) -> Iterator[None]: + """Context manager that enables capturing of Metal calls into gputrace""" + try: + torch._C._mps_startCapture(fname) # type: ignore[attr-defined] + yield + # Drain all the work that were enqueued during the context call + torch.mps.synchronize() + finally: + torch._C._mps_stopCapture() # type: ignore[attr-defined] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mtia/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mtia/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bef3f0652e69478dc0b6ba82637df5bef15dac23 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mtia/__init__.py @@ -0,0 +1,433 @@ +# mypy: allow-untyped-defs +r""" +This package enables an interface for accessing MTIA backend in python +""" + +import threading +import warnings +from collections.abc import Callable +from typing import Any, Optional, Union + +import torch +from torch import device as _device, Tensor +from torch._utils import _dummy_type, _LazySeedTracker, classproperty +from torch.types import Device + +from ._utils import _get_device_index + + +_device_t = Union[_device, str, int] + +# torch.mtia.Event/Stream is alias of torch.Event/Stream +Event = torch.Event +Stream = torch.Stream +# Default generators are initialized inside _mtia_init +default_generators: tuple[torch._C.Generator, ...] = () # type: ignore[assignment] + +_initialized = False +_queued_calls: list[ + tuple[Callable[[], None], list[str]] +] = [] # don't invoke these until initialization occurs +_tls = threading.local() +_initialization_lock = threading.Lock() +_lazy_seed_tracker = _LazySeedTracker() + + +if hasattr(torch._C, "_mtia_exchangeDevice"): + _exchange_device = torch._C._mtia_exchangeDevice +else: + + def _exchange_device(device: int) -> int: + if device < 0: + return -1 + raise RuntimeError("PyTorch was compiled without MTIA support") + + +if hasattr(torch._C, "_mtia_maybeExchangeDevice"): + _maybe_exchange_device = torch._C._mtia_maybeExchangeDevice +else: + + def _maybe_exchange_device(device: int) -> int: + if device < 0: + return -1 + raise RuntimeError("PyTorch was compiled without MTIA support") + + +def init(): + _lazy_init() + + +def is_initialized(): + r"""Return whether PyTorch's MTIA state has been initialized.""" + return _initialized and not _is_in_bad_fork() + + +def _is_in_bad_fork() -> bool: + return torch._C._mtia_isInBadFork() + + +def _lazy_init() -> None: + global _initialized, _queued_calls + if is_initialized() or hasattr(_tls, "is_initializing"): + return + with _initialization_lock: + # We be double-checking locking, boys! This is OK because + # the above test was GIL protected anyway. The inner test + # is for when a thread blocked on some other thread which was + # doing the initialization; when they get the lock, they will + # find there is nothing left to do. + if is_initialized(): + return + # It is important to prevent other threads from entering _lazy_init + # immediately, while we are still guaranteed to have the GIL, because some + # of the C calls we make below will release the GIL + if _is_in_bad_fork(): + raise RuntimeError( + "Cannot re-initialize MTIA in forked subprocess. To use MTIA with " + "multiprocessing, you must use the 'spawn' start method" + ) + if not _is_compiled(): + raise AssertionError( + "Torch not compiled with MTIA enabled. " + "Ensure you have `import mtia.host_runtime.torch_mtia.dynamic_library` in your python " + "src file and include `//mtia/host_runtime/torch_mtia:torch_mtia` as " + "your target dependency!" + ) + + torch._C._mtia_init() + # Some of the queued calls may reentrantly call _lazy_init(); + # we need to just return without initializing in that case. + # However, we must not let any *other* threads in! + _tls.is_initializing = True + + _queued_calls.extend(calls for calls in _lazy_seed_tracker.get_calls() if calls) + + try: + for queued_call, orig_traceback in _queued_calls: + try: + queued_call() + except Exception as e: + msg = ( + f"MTIA call failed lazily at initialization with error: {str(e)}\n\n" + f"MTIA call was originally invoked at:\n\n{''.join(orig_traceback)}" + ) + raise DeferredMtiaCallError(msg) from e + finally: + delattr(_tls, "is_initializing") + _initialized = True + + +class DeferredMtiaCallError(Exception): + pass + + +def _is_compiled() -> bool: + r"""Return true if compiled with MTIA support.""" + return torch._C._mtia_isBuilt() + + +def is_available() -> bool: + r"""Return true if MTIA device is available""" + if not _is_compiled(): + return False + # MTIA has to init devices first to know if there is any devices available. + return device_count() > 0 + + +def synchronize(device: Optional[_device_t] = None) -> None: + r"""Waits for all jobs in all streams on a MTIA device to complete.""" + with torch.mtia.device(device): + return torch._C._mtia_deviceSynchronize() + + +def device_count() -> int: + r"""Return the number of MTIA devices available.""" + # TODO: Update _accelerator_hooks_device_count to abstract a MTIA device count API + return torch._C._mtia_getDeviceCount() + + +def current_device() -> int: + r"""Return the index of a currently selected device.""" + return torch._C._accelerator_hooks_get_current_device() + + +def current_stream(device: Optional[_device_t] = None) -> Stream: + r"""Return the currently selected :class:`Stream` for a given device. + + Args: + device (torch.device or int, optional): selected device. Returns + the currently selected :class:`Stream` for the current device, given + by :func:`~torch.mtia.current_device`, if :attr:`device` is ``None`` + (default). + """ + return torch._C._mtia_getCurrentStream(_get_device_index(device, optional=True)) + + +def default_stream(device: Optional[_device_t] = None) -> Stream: + r"""Return the default :class:`Stream` for a given device. + + Args: + device (torch.device or int, optional): selected device. Returns + the default :class:`Stream` for the current device, given by + :func:`~torch.mtia.current_device`, if :attr:`device` is ``None`` + (default). + """ + return torch._C._mtia_getDefaultStream(_get_device_index(device, optional=True)) + + +def record_memory_history( + enabled: Optional[str] = "all", stacks: str = "python", max_entries: int = 0 +) -> None: + r"""Enable/Disable the memory profiler on MTIA allocator + + Args: + enabled (all or state, optional) selected device. Returns + statistics for the current device, given by current_device(), + if device is None (default). + + stacks ("python" or "cpp", optional). Select the stack trace to record. + + max_entries (int, optional). Maximum number of entries to record. + """ + if not is_initialized(): + return + torch._C._mtia_recordMemoryHistory(enabled, stacks, max_entries) + + +def snapshot() -> dict[str, Any]: + r"""Return a dictionary of MTIA memory allocator history""" + + return torch._C._mtia_memorySnapshot() + + +def attach_out_of_memory_observer( + observer: Callable[[int, int, int, int], None], +) -> None: + r"""Attach an out-of-memory observer to MTIA memory allocator""" + torch._C._mtia_attachOutOfMemoryObserver(observer) + + +def is_bf16_supported(including_emulation: bool = True): + r"""Return a bool indicating if the current MTIA device supports dtype bfloat16.""" + return True + + +def get_device_capability(device: Optional[_device_t] = None) -> tuple[int, int]: + r"""Return capability of a given device as a tuple of (major version, minor version). + + Args: + device (torch.device or int, optional) selected device. Returns + statistics for the current device, given by current_device(), + if device is None (default). + """ + return torch._C._mtia_getDeviceCapability(_get_device_index(device, optional=True)) + + +def empty_cache() -> None: + r"""Empty the MTIA device cache.""" + return torch._C._mtia_emptyCache() + + +def set_stream(stream: Stream): + r"""Set the current stream.This is a wrapper API to set the stream. + Usage of this function is discouraged in favor of the ``stream`` + context manager. + + Args: + stream (Stream): selected stream. This function is a no-op + if this argument is ``None``. + """ + if stream is None: + return + torch._C._mtia_setCurrentStream(stream) + + +def set_device(device: _device_t) -> None: + r"""Set the current device. + + Args: + device (torch.device or int): selected device. This function is a no-op + if this argument is negative. + """ + device = _get_device_index(device) + if device >= 0: + torch._C._accelerator_hooks_set_current_device(device) + + +def get_device_properties(device: Optional[_device_t] = None) -> dict[str, Any]: + r"""Return a dictionary of MTIA device properties + + Args: + device (torch.device or int, optional) selected device. Returns + statistics for the current device, given by current_device(), + if device is None (default). + """ + return torch._C._mtia_getDeviceProperties(_get_device_index(device, optional=True)) + + +class device: + r"""Context-manager that changes the selected device. + + Args: + device (torch.device or int): device index to select. It's a no-op if + this argument is a negative integer or ``None``. + """ + + def __init__(self, device: Any): + self.idx = _get_device_index(device, optional=True) + self.prev_idx = -1 + + def __enter__(self): + self.prev_idx = torch._C._accelerator_hooks_maybe_exchange_device(self.idx) + + def __exit__(self, type: Any, value: Any, traceback: Any): + self.idx = torch._C._accelerator_hooks_maybe_exchange_device(self.prev_idx) + return False + + +class StreamContext: + r"""Context-manager that selects a given stream. + + All MTIA kernels queued within its context will be enqueued on a selected + stream. + + Args: + Stream (Stream): selected stream. This manager is a no-op if it's + ``None``. + .. note:: Streams are per-device. + """ + + cur_stream: Optional["torch.mtia.Stream"] + + def __init__(self, stream: Optional["torch.mtia.Stream"]): + self.cur_stream = None + self.stream = stream + self.idx = _get_device_index(None, True) + if not torch.jit.is_scripting(): + if self.idx is None: + self.idx = -1 # pyrefly: ignore [bad-assignment] + + self.src_prev_stream = ( + None if not torch.jit.is_scripting() else torch.mtia.default_stream(None) + ) + self.dst_prev_stream = ( + None if not torch.jit.is_scripting() else torch.mtia.default_stream(None) + ) + + def __enter__(self): + # Local cur_stream variable for type refinement + cur_stream = self.stream + # Return if stream is None or MTIA device not available + if cur_stream is None or self.idx == -1: + return + self.src_prev_stream = torch.mtia.current_stream(None) + + # If the stream is not on the current device, then + # set the current stream on the device + if self.src_prev_stream.device != cur_stream.device: + with device(cur_stream.device): + self.dst_prev_stream = torch.mtia.current_stream(cur_stream.device) + torch.mtia.set_stream(cur_stream) + + def __exit__(self, type: Any, value: Any, traceback: Any): + # Local cur_stream variable for type refinement + cur_stream = self.stream + # If stream is None or no MTIA device available, return + if cur_stream is None or self.idx == -1: + return + + # Reset the stream on the original device + # and destination device + if self.src_prev_stream.device != cur_stream.device: # type: ignore[union-attr] + torch.mtia.set_stream(self.dst_prev_stream) # type: ignore[arg-type] + torch.mtia.set_stream(self.src_prev_stream) # type: ignore[arg-type] + + +def _set_stream_by_id(stream_id, device_index, device_type): + r"""set stream specified by the stream id, device index and + device type + + Args: stream_id (int): stream id in stream pool + device_index (int): device index in topo + device_type (int): enum device type + """ + torch._C._mtia_setStream(stream_id, device_index, device_type) + + +def stream(stream: Optional["torch.mtia.Stream"]) -> StreamContext: + r"""Wrap around the Context-manager StreamContext that selects a given stream. + + Arguments: + stream (Stream): selected stream. This manager is a no-op if it's + ``None``. + .. note:: In eager mode stream is of type Stream class while in JIT it doesn't support torch.mtia.stream + """ + return StreamContext(stream) + + +def get_rng_state(device: Union[int, str, torch.device] = "mtia") -> Tensor: + r"""Returns the random number generator state as a ByteTensor. + + Args: + device (torch.device or int, optional): The device to return the RNG state of. + Default: ``'mtia'`` (i.e., ``torch.device('mtia')``, the current mtia device). + """ + warnings.warn( + "get_rng_state is not implemented in torch.mtia", + UserWarning, + stacklevel=2, + ) + return torch.zeros([1], dtype=torch.uint8, device=device) + + +def set_rng_state( + new_state: Tensor, device: Union[int, str, torch.device] = "mtia" +) -> None: + r"""Sets the random number generator state. + + Args: + new_state (torch.ByteTensor): The desired state + device (torch.device or int, optional): The device to set the RNG state. + Default: ``'mtia'`` (i.e., ``torch.device('mtia')``, the current mtia device). + """ + warnings.warn( + "set_rng_state is not implemented in torch.mtia", + UserWarning, + stacklevel=2, + ) + + +from .memory import * # noqa: F403 +from .mtia_graph import * # noqa: F403 + + +__all__ = [ + "init", + "is_available", + "is_initialized", + "synchronize", + "device_count", + "current_device", + "current_stream", + "default_stream", + "memory_stats", + "max_memory_allocated", + "memory_allocated", + "reset_peak_memory_stats", + "get_device_capability", + "get_device_properties", + "record_memory_history", + "snapshot", + "attach_out_of_memory_observer", + "empty_cache", + "set_device", + "set_stream", + "stream", + "device", + "set_rng_state", + "get_rng_state", + "is_bf16_supported", + "MTIAGraph", + "graph", + "graph_pool_handle", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mtia/_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mtia/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..090e26f321232f9687c2b348ac602dbb6699b03f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mtia/_utils.py @@ -0,0 +1,38 @@ +from typing import Any + +import torch + +# The _get_device_index has been moved to torch.utils._get_device_index +from torch._utils import _get_device_index as _torch_get_device_index + + +def _get_device_index( + device: Any, optional: bool = False, allow_cpu: bool = False +) -> int: + r"""Get the device index from :attr:`device`, which can be a torch.device object, a Python integer, or ``None``. + + If :attr:`device` is a torch.device object, returns the device index if it + is a MTIA device. Note that for a MTIA device without a specified index, + i.e., ``torch.device('mtia')``, this will return the current default MTIA + device if :attr:`optional` is ``True``. If :attr:`allow_cpu` is ``True``, + CPU devices will be accepted and ``-1`` will be returned in this case. + + If :attr:`device` is a Python integer, it is returned as is. + + If :attr:`device` is ``None``, this will return the current default MTIA + device if :attr:`optional` is ``True``. + """ + if isinstance(device, int): + return device + if isinstance(device, str): + device = torch.device(device) + if isinstance(device, torch.device): + if allow_cpu: + if device.type not in ["mtia", "cpu"]: + raise ValueError(f"Expected a mtia or cpu device, but got: {device}") + elif device.type != "mtia": + raise ValueError(f"Expected a mtia device, but got: {device}") + if not torch.jit.is_scripting(): + if isinstance(device, torch.mtia.device): + return device.idx + return _torch_get_device_index(device, optional, allow_cpu) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mtia/memory.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mtia/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..ff350f42b70201a4a1f7ab4667f07871af8913e0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mtia/memory.py @@ -0,0 +1,71 @@ +# pyre-strict + +r"""This package adds support for device memory management implemented in MTIA.""" + +from typing import Any, Optional + +import torch + +from . import _device_t, is_initialized +from ._utils import _get_device_index + + +def memory_stats(device: Optional[_device_t] = None) -> dict[str, Any]: + r"""Return a dictionary of MTIA memory allocator statistics for a given device. + + Args: + device (torch.device, str, or int, optional) selected device. Returns + statistics for the current device, given by current_device(), + if device is None (default). + """ + if not is_initialized(): + return {} + return torch._C._mtia_memoryStats(_get_device_index(device, optional=True)) + + +def max_memory_allocated(device: Optional[_device_t] = None) -> int: + r"""Return the maximum memory allocated in bytes for a given device. + + Args: + device (torch.device, str, or int, optional) selected device. Returns + statistics for the current device, given by current_device(), + if device is None (default). + """ + if not is_initialized(): + return 0 + return memory_stats(device).get("dram", 0).get("peak_bytes", 0) + + +def memory_allocated(device: Optional[_device_t] = None) -> int: + r"""Return the current MTIA memory occupied by tensors in bytes for a given device. + + Args: + device (torch.device or int or str, optional): selected device. Returns + statistic for the current device, given by :func:`~torch.mtia.current_device`, + if :attr:`device` is ``None`` (default). + """ + if not is_initialized(): + return 0 + return memory_stats(device).get("dram", 0).get("allocated_bytes", 0) + + +def reset_peak_memory_stats(device: Optional[_device_t] = None) -> None: + r"""Reset the peak memory stats for a given device. + + + Args: + device (torch.device, str, or int, optional) selected device. Returns + statistics for the current device, given by current_device(), + if device is None (default). + """ + if not is_initialized(): + return + torch._C._mtia_resetPeakMemoryStats(_get_device_index(device, optional=True)) + + +__all__ = [ + "memory_stats", + "max_memory_allocated", + "memory_allocated", + "reset_peak_memory_stats", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mtia/mtia_graph.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mtia/mtia_graph.py new file mode 100644 index 0000000000000000000000000000000000000000..a7b0a69deca0bd0b8be3b5ff2795f34e3a22f075 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/mtia/mtia_graph.py @@ -0,0 +1,105 @@ +# pylint: disable=useless-parent-delegation +from __future__ import annotations + +from typing import Optional, Union +from typing_extensions import Self + +import torch + + +_POOL_HANDLE = tuple[int, int] + + +def graph_pool_handle() -> _POOL_HANDLE: + """ + Return an opaque token representing the id of a graph memory pool. + """ + return torch._C._mtia_graphPoolHandle() + + +class MTIAGraph(torch._C._MTIAGraph): + """ + Wrapper around a MTIA graph. + """ + + def __new__(cls, keep_graph: bool = False) -> Self: + return super().__new__(cls, keep_graph) + + def capture_begin(self, pool: _POOL_HANDLE) -> None: + """ + Begin capturing a MTIA graph. + """ + super().capture_begin(pool) + + def capture_end(self) -> None: + """ + End the capture of a MTIA graph. + """ + super().capture_end() + + def instantiate(self) -> None: + """ + Instantiate the captured MTIA graph. + """ + super().instantiate() + + def replay(self) -> None: + """ + Replay the captured MTIA graph. + """ + super().replay() + + def reset(self) -> None: + """ + Destroy the captured graph and reset the states. + """ + super().reset() + + def pool(self) -> _POOL_HANDLE: + """ + Return an opaque token representing the id of this graph's memory pool + """ + return super().pool() + + +class graph: + default_capture_stream: Optional[torch.mtia.Stream] = None + + def __init__( + self, + mtia_graph: MTIAGraph, + pool: Optional[_POOL_HANDLE] = None, + stream: Optional[torch.mtia.Stream] = None, + ): + if self.__class__.default_capture_stream is None: + self.__class__.default_capture_stream = torch.mtia.current_stream() + + self.pool: Union[tuple[()], tuple[_POOL_HANDLE]] = ( + () if pool is None else (pool,) + ) + self.capture_stream = ( + stream if stream is not None else self.__class__.default_capture_stream + ) + assert self.capture_stream is not None + self.stream_ctx = torch.mtia.stream(self.capture_stream) + self.mtia_graph = mtia_graph + + def __enter__(self) -> None: + torch.mtia.synchronize() + torch.mtia.empty_cache() + + self.stream_ctx.__enter__() + + pool_arg = self.pool[0] if self.pool else (0, 0) + self.mtia_graph.capture_begin(pool_arg) + + def __exit__(self, *args: object) -> None: + self.mtia_graph.capture_end() + self.stream_ctx.__exit__(*args) + + +__all__ = [ + "MTIAGraph", + "graph", + "graph_pool_handle", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..aa1176d69c60c8ca3675e56f2f2c4e9e0ad923c0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/__init__.py @@ -0,0 +1,122 @@ +# mypy: allow-untyped-defs +"""torch.multiprocessing is a wrapper around the native :mod:`multiprocessing` module. + +It registers custom reducers, that use shared memory to provide shared +views on the same data in different processes. Once the tensor/storage is moved +to shared_memory (see :func:`~torch.Tensor.share_memory_`), it will be possible +to send it to other processes without making any copies. + +The API is 100% compatible with the original module - it's enough to change +``import multiprocessing`` to ``import torch.multiprocessing`` to have all the +tensors sent through the queues or shared via other mechanisms, moved to shared +memory. + +Because of the similarity of APIs we do not document most of this package +contents, and we recommend referring to very good docs of the original module. +""" + +import multiprocessing +import sys + +import torch + +from .reductions import init_reductions + + +__all__ = ["set_sharing_strategy", "get_sharing_strategy", "get_all_sharing_strategies"] + + +from multiprocessing import * # noqa: F403 + + +__all__ += multiprocessing.__all__ # noqa: PLE0605 type: ignore[attr-defined] + + +# This call adds a Linux specific prctl(2) wrapper function to this module. +# See https://github.com/pytorch/pytorch/pull/14391 for more information. +torch._C._multiprocessing_init() + + +"""Add helper function to spawn N processes and wait for completion of any of +them. This depends `mp.get_context` which was added in Python 3.4.""" +from .spawn import ( + ENV_VAR_PARALLEL_START, + ProcessContext, + ProcessExitedException, + ProcessRaisedException, + spawn, + SpawnContext, + start_processes, +) + + +if sys.platform == "darwin" or sys.platform == "win32": + _sharing_strategy = "file_system" + _all_sharing_strategies = {"file_system"} +else: + _sharing_strategy = "file_descriptor" + _all_sharing_strategies = {"file_descriptor", "file_system"} + + +def set_sharing_strategy(new_strategy): + """Set the strategy for sharing CPU tensors. + + Args: + new_strategy (str): Name of the selected strategy. Should be one of + the values returned by :func:`get_all_sharing_strategies()`. + """ + global _sharing_strategy + assert new_strategy in _all_sharing_strategies + _sharing_strategy = new_strategy + + +def get_sharing_strategy(): + """Return the current strategy for sharing CPU tensors.""" + return _sharing_strategy + + +def get_all_sharing_strategies(): + """Return a set of sharing strategies supported on a current system.""" + return _all_sharing_strategies + + +def _set_thread_name(name: str) -> None: + """Set the name of the current thread. + + Args: + name (str): Name of the current thread. + """ + torch._C._set_thread_name(name) + + +def _get_thread_name() -> str: + """Get the name of the current thread. + + Returns: + str: Name of the current thread. + """ + return torch._C._get_thread_name() + + +init_reductions() + +# Leak ResourceTracker at exit for Python-3.12 on MacOS +# See https://github.com/pytorch/pytorch/issues/153050 and +# https://github.com/python/cpython/issues/88887 for more details +from multiprocessing.resource_tracker import ResourceTracker as _RT + + +if ( + sys.platform == "darwin" + and sys.version_info >= (3, 12, 2) + and hasattr(_RT, "__del__") +): + import atexit + + def _leak_RT_at_exit(): + def _noop(x): + pass + + _RT.__del__ = _noop # type: ignore[attr-defined] + + atexit.register(_leak_RT_at_exit) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/_atfork.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/_atfork.py new file mode 100644 index 0000000000000000000000000000000000000000..ac4a97c10c07ae680765b0f362ef33c4bfb2308b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/_atfork.py @@ -0,0 +1,35 @@ +# mypy: allow-untyped-defs +import sys + + +__all__ = ["register_after_fork"] + +if sys.platform == "win32": + import multiprocessing.util as _util + + def _register(func): + def wrapper(arg): + func() + + _util.register_after_fork(_register, wrapper) + +else: + import os + + def _register(func): + os.register_at_fork(after_in_child=func) + + +def register_after_fork(func): + """Register a callable to be executed in the child process after a fork. + + Note: + In python < 3.7 this will only work with processes created using the + ``multiprocessing`` module. In python >= 3.7 it also works with + ``os.fork()``. + + Args: + func (function): Function taking no arguments to be called in the child after fork + + """ + _register(func) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/pool.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/pool.py new file mode 100644 index 0000000000000000000000000000000000000000..32a47efac0d6ea6f0c8700924aafa67564813f5b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/pool.py @@ -0,0 +1,52 @@ +import multiprocessing.pool +import multiprocessing.util as util + +from .queue import SimpleQueue + + +def clean_worker(*args, **kwargs): + import gc + + multiprocessing.pool.worker(*args, **kwargs) + # Regular multiprocessing workers don't fully clean up after themselves, + # so we have to explicitly trigger garbage collection to make sure that all + # destructors are called... + gc.collect() + + +class Pool(multiprocessing.pool.Pool): + """Pool implementation which uses our version of SimpleQueue. + + This lets us pass tensors in shared memory across processes instead of + serializing the underlying data. + """ + + def _setup_queues(self): + self._inqueue = SimpleQueue() + self._outqueue = SimpleQueue() + self._quick_put = self._inqueue._writer.send + self._quick_get = self._outqueue._reader.recv + + def _repopulate_pool(self): + """Increase the number of pool processes to the specified number. + + Bring the number of pool processes up to the specified number, for use after + reaping workers which have exited. + """ + for _ in range(self._processes - len(self._pool)): + # changed worker -> clean_worker + args = ( + self._inqueue, + self._outqueue, + self._initializer, + self._initargs, + self._maxtasksperchild, + ) + if hasattr(self, "_wrap_exception"): + args += (self._wrap_exception,) + w = self.Process(target=clean_worker, args=args) + self._pool.append(w) + w.name = w.name.replace("Process", "PoolWorker") + w.daemon = True + w.start() + util.debug("added worker") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/queue.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/queue.py new file mode 100644 index 0000000000000000000000000000000000000000..876bf8d0e7459b60a41b59b0a093608e515ba455 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/queue.py @@ -0,0 +1,43 @@ +# mypy: allow-untyped-defs +import io +import multiprocessing.queues +import pickle +from multiprocessing.reduction import ForkingPickler + + +class ConnectionWrapper: + """Proxy class for _multiprocessing.Connection which uses ForkingPickler for object serialization.""" + + def __init__(self, conn): + self.conn = conn + + def send(self, obj): + buf = io.BytesIO() + ForkingPickler(buf, pickle.HIGHEST_PROTOCOL).dump(obj) + self.send_bytes(buf.getvalue()) + + def recv(self): + buf = self.recv_bytes() + return pickle.loads(buf) + + def __getattr__(self, name): + if "conn" in self.__dict__: + return getattr(self.conn, name) + raise AttributeError(f"'{type(self).__name__}' object has no attribute 'conn'") + + +class Queue(multiprocessing.queues.Queue): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._reader: ConnectionWrapper = ConnectionWrapper(self._reader) + self._writer: ConnectionWrapper = ConnectionWrapper(self._writer) + self._send = self._writer.send + self._recv = self._reader.recv + + +class SimpleQueue(multiprocessing.queues.SimpleQueue): + def _make_methods(self): + if not isinstance(self._reader, ConnectionWrapper): + self._reader: ConnectionWrapper = ConnectionWrapper(self._reader) + self._writer: ConnectionWrapper = ConnectionWrapper(self._writer) + super()._make_methods() # type: ignore[misc] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/reductions.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/reductions.py new file mode 100644 index 0000000000000000000000000000000000000000..b0b6d9468bcb51f82a2e5e146fc285f8ac242fe5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/reductions.py @@ -0,0 +1,644 @@ +# mypy: allow-untyped-defs +import multiprocessing +import os +import threading +from multiprocessing import reduction +from multiprocessing.util import register_after_fork + +import torch +from torch._namedtensor_internals import check_serializing_named_tensor + + +try: + # Early load resource_sharer to prevent a partially initialized instance + # from being inherited in a forked child process. The reduce_storage method + # requires this module indirectly through DupFd(). The built-in mp.Queue + # class pickles arguments in a background thread which may overlap with the + # fork. + import multiprocessing.resource_sharer +except ImportError: + pass + + +class StorageWeakRef: + r"""A weak reference to a Storage. + + The cdata member is a Python number containing the integer representation of + the Storage pointer. + """ + + __slots__ = ["cdata", "_free_weak_ref"] + + def __init__(self, storage): + self.cdata = storage._weak_ref() + # Save a direct reference to _free_weak_ref because the `torch` module + # might be cleared during Python shutdown before this module is cleared. + self._free_weak_ref = torch.Storage._free_weak_ref # type: ignore[attr-defined] + + @classmethod + def from_weakref(cls, cdata): + instance = cls.__new__(cls) + instance.cdata = cdata + instance._free_weak_ref = torch.Storage._free_weak_ref # type: ignore[attr-defined] + return instance + + def expired(self): + return torch.Storage._expired(self.cdata) # type: ignore[attr-defined] + + def __del__(self): + self._free_weak_ref(self.cdata) + + def __hash__(self): + return self.cdata + + def __eq__(self, other): + if id(self) == id(other): + return True + return self.cdata == other.cdata + + +class SharedCache(dict): + """Dictionary from multiprocessing handles to StorageWeakRef.""" + + def __init__(self) -> None: + # free_dead_references() is called if the len exceeds the current + # limit. The limit scales with the number of remaining live objects. + self.limit = 128 + # `fork` inherits lock state, so in case we fork when the lock is held, + # we register a function to reset the lock to a new object to avoid + # possible deadlocks, following python multiprocessing library design. + self._after_fork() + register_after_fork(self, SharedCache._after_fork) + + def _after_fork(self): + self.lock = threading.Lock() + + def get(self, key): # type: ignore[override] + with self.lock: + return dict.get(self, key) + + def __setitem__(self, key, storage_ref): + with self.lock: + dict.__setitem__(self, key, storage_ref) + if len(self) > self.limit: + self.free_dead_references() + + def free_dead_references(self): + live = 0 + for key, storage_ref in list(self.items()): + if storage_ref.expired(): + del self[key] + else: + live += 1 + self.limit = max(128, live * 2) + + +# mapping from handles to StorageWeakRef objects +shared_cache = SharedCache() + + +def rebuild_event(device, handle): + return torch.cuda.Event.from_ipc_handle(device, handle) + + +def reduce_event(event): + handle = event.ipc_handle() + return (rebuild_event, (event.device, handle)) + + +def rebuild_tensor(cls, storage, metadata): + storage_offset, size, stride, requires_grad = metadata + t = torch._utils._rebuild_tensor(storage, storage_offset, size, stride) + if cls == torch.nn.parameter.Parameter: + # we have to pass requires_grad into constructor, rather than set it as an + # attribute later, because it's an important check for Integer Tensors to + # have requires_grad=False (or else they raise an error) + t = torch.nn.parameter.Parameter(t, requires_grad=requires_grad) + else: + t.requires_grad = requires_grad + return t + + +def rebuild_meta_tensor( + tensor_cls, + tensor_size, + tensor_stride, + tensor_offset, + dtype, + storage_size_bytes, + requires_grad, +): + untyped_storage = torch.UntypedStorage(storage_size_bytes, device="meta") + + typed_storage = torch.TypedStorage( + wrap_storage=untyped_storage, dtype=dtype, _internal=True + ) + + t = torch._utils._rebuild_tensor( + typed_storage, + tensor_offset, + tensor_size, + tensor_stride, + ) + + if tensor_cls == torch.nn.parameter.Parameter: + # It is crucial for integer tensors to receive + # the requires_grad=False as an argument in the constructor + t = torch.nn.parameter.Parameter(t, requires_grad=requires_grad) + else: + t.requires_grad = requires_grad + + return t + + +def rebuild_cuda_tensor( + tensor_cls, + tensor_size, + tensor_stride, + tensor_offset, + storage_cls, + dtype, + storage_device, + storage_handle, + storage_size_bytes, + storage_offset_bytes, + requires_grad, + ref_counter_handle, + ref_counter_offset, + event_handle, + event_sync_required, +): + # If storage_handle is None, storage points to nullptr. + if storage_handle is None or storage_size_bytes == 0: + storage = storage_cls(0, dtype=dtype, device=storage_device, _internal=True) + else: + storage = storage_from_cache( + storage_cls, (storage_handle, storage_offset_bytes) + ) + if storage is None: + torch.cuda._lazy_init() + storage = storage_cls._new_shared_cuda( + storage_device, + storage_handle, + storage_size_bytes, + storage_offset_bytes, + ref_counter_handle, + ref_counter_offset, + event_handle, + event_sync_required, + ) + shared_cache[(storage_handle, storage_offset_bytes)] = StorageWeakRef( + storage + ) + else: + # We already ref counting this Storage, but producer needs new ref-counters to be released. + storage_cls._release_ipc_counter( + ref_counter_handle, ref_counter_offset, device=storage_device + ) + + _storage = ( + storage + if isinstance(storage, torch.UntypedStorage) + else storage._untyped_storage + ) + + t = torch._utils._rebuild_tensor( + torch.storage.TypedStorage(wrap_storage=_storage, dtype=dtype, _internal=True), + tensor_offset, + tensor_size, + tensor_stride, + ) + + if tensor_cls == torch.nn.parameter.Parameter: + # It is crucial for integer tensors to receive + # the requires_grad=False as an argument in the constructor + t = torch.nn.parameter.Parameter(t, requires_grad=requires_grad) + else: + t.requires_grad = requires_grad + + return t + + +def reduce_tensor(tensor): + if tensor.requires_grad and not tensor.is_leaf: + raise RuntimeError( + "Cowardly refusing to serialize non-leaf tensor which requires_grad, " + "since autograd does not support crossing process boundaries. " + "If you just want to transfer the data, call detach() on the tensor " + "before serializing (e.g., putting it on the queue)." + ) + + check_serializing_named_tensor(tensor) + torch.utils.hooks.warn_if_has_hooks(tensor) + + # Note [CUDA IPC and the caching allocator] + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # When you send a CUDA tensor over IPC, you might expect that you will + # get out the same storage from the other end. However, the CUDA caching + # allocator makes it difficult to preserve this invariant. Consider + # the following situation: a tensor of size 0x100 points to offset 0x20 of + # a storage at 0xA100 of size 0x100. (For simplicity, all of these + # sizes are given in bytes). HOWEVER, with the caching allocator, this storage + # might be part of a larger cudaMalloc allocation 0xA000 of size 0x4000. + # + # When we want to send this CUDA tensor over IPC, we must send the + # *entire* cudaMalloc allocation, i.e., the 0xA000 region, not just + # the storage 0xA100 (because that is what CUDA supports). So, on the + # other end, there simply isn't any way to say, "Wait, you gave me + # a bigger region (0xA000) than the one I wanted (0xA100)". + # + # OK, so if you sent the cudaMalloc allocation, can you just wrap that up as + # one storage itself? No, because this cudaMalloc allocation might contain + # storages of mixed types: float, bytes, double... If you make the entire + # allocation a single storage of a type A, we'll hit an error when constructing + # a tensor of type B on the storage. + # + # cudaIpcMemHandle is an identifier to access the sender cudaMalloc allocation on the + # receiver side. However, cudaIpcMemHandles from each device in a given process may + # only be opened by one context per device per other process. + # If we open and close a memory handle multiples times in a process, CUDA is allowed + # to give it a different address; similarly, once we close the memory, we're not + # allowed to access it(and the storage/tensor built on top of it), even if it is + # still live in the original process. As we cannot make a cudaMalloc allocation + # to a single storage in one go, this requires us to cache the device pointer for + # each cudaIpcMemHandle on C++ side to reconstruct types of storages, while keep + # the old ones alives. + # See [https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__DEVICE.html] + # + # This is fine, because all we need to do is to save our position in the allocation, + # and reconstruct storage and tensor from it. + # 0xA000 -> -------CUDA Allocation------ + # | | + # | | + # | | + # | | + # 0xA100 -> --------storage1 begin------ + # | | + # 0xA120 -> --------tensor1 begin ------ + # | | + # | | + # | | + # | | + # | | + # 0xA160 -> --------tensor1 end--------- + # | | + # | | + # | | + # 0xA200 -> --------storage1 end-------- + # | | + # 0xE000 -> --------CUDA allocation----- + # + # To send tensor1, the following info are required from sender to receiver for + # storage reconstruction. + # 1. cudaIpcMemHandle of 0xA000(which can be mapped to a basePtr in receiver process). + # basePtr may not be exactly 0xA000 since it's a different process. + # 2. offset(0xA100) of storage1 in the CUDA allocation. + # 3. size of storage1(0x100). + # + # On receiver side: + # 1. Get the devPtr of the MemHandle to access the memory, reconstruct a storage + # of the same type using (basePtr, offset, size). + # 2. we can reconstruct the tensor on top of the reconstructed storage + # Tensor(size=0x040, offset=0x020, storage=Storage(data=basePtr+0xA100, size=0x0100)) + # + # This strategy has a few implications: + # + # 1. When we serialize a CUDA tensor for IPC, we cannot do it all in one + # go (non-compositionally), and this requires to have a global map + # memHandle -> devPtr for each process. + # + # 2. We MUST NOT let the new IPC tensor be resizable. Originally, a resize + # of the storage beyond 0x100 would merely have caused us to do a + # reallocation. You don't really want to do this, but if you did, + # all that would happen is that you would lose IPC sharing. But if + # you do this in the new world, we will happily let you write out of + # bounds of your "allocation", clobbering unrelated data in the cached + # allocator block. BAD! + # + # By the way, in old versions of PyTorch, we supported this situation + # natively using a "storage view", which permitted multiple storages to be + # views on each other. But this was the *only* use of storage views, so we + # eliminated it so that we could just use tensor views to implement the same + # thing. + # + + # TODO: Handle distinguishing between subclass and non-subclass versions of NT better + # https://github.com/pytorch/pytorch/issues/110543 + from torch.nested._internal.nested_tensor import NestedTensor + + if tensor.is_nested and not isinstance(tensor, NestedTensor): + return reduce_nested_tensor(tensor) + + if tensor.layout in { + torch.sparse_coo, + torch.sparse_csr, + torch.sparse_bsr, + torch.sparse_csc, + torch.sparse_bsc, + }: + return reduce_sparse_tensor(tensor) + + storage = tensor._typed_storage() + + if storage._untyped_storage.device.type == "cuda": + ( + device, + handle, + storage_size_bytes, + storage_offset_bytes, + ref_counter_handle, + ref_counter_offset, + event_handle, + event_sync_required, + ) = storage._share_cuda_() + tensor_offset = tensor.storage_offset() + shared_cache[handle] = StorageWeakRef(storage) + # _backward_hooks purposely omitted here, see + # Note [Don't serialize hooks] + return ( + rebuild_cuda_tensor, + ( + type(tensor), + tensor.size(), + tensor.stride(), + tensor_offset, # tensor offset in its storage + type(storage), + tensor.dtype, + device, + handle, # identifier which CUDA allocation is the storage in. + storage_size_bytes, # size(in bytes) of the storage + storage_offset_bytes, # offset(in bytes) of the storage in the CUDA allocation + tensor.requires_grad, + ref_counter_handle, + ref_counter_offset, + event_handle, + event_sync_required, + ), + ) + elif storage._untyped_storage.device.type == "meta": + return ( + rebuild_meta_tensor, + ( + type(tensor), + tensor.size(), + tensor.stride(), + tensor.storage_offset(), + tensor.dtype, + tensor.untyped_storage().size(), + tensor.requires_grad, + ), + ) + + # _backward_hooks purposely omitted here, see Note [Don't serialize hooks] + metadata = ( + tensor.storage_offset(), + tensor.size(), + tensor.stride(), + tensor.requires_grad, + ) + return (rebuild_tensor, (type(tensor), storage, metadata)) + + +def rebuild_nested_tensor( + rebuild_buffer_func, + rebuild_buffer_args, + rebuild_sizes_func, + rebuild_sizes_args, + rebuild_strides_func, + rebuild_strides_args, + rebuild_offsets_func, + rebuild_offsets_args, +): + buffer = rebuild_buffer_func(*rebuild_buffer_args) + sizes = rebuild_sizes_func(*rebuild_sizes_args) + strides = rebuild_strides_func(*rebuild_strides_args) + offsets = rebuild_offsets_func(*rebuild_offsets_args) + return torch._nested_view_from_buffer_copy(buffer, sizes, strides, offsets) + + +def reduce_nested_tensor(nt): + rebuild_buffer_func, rebuild_buffer_args = reduce_tensor(nt.values()) + rebuild_sizes_func, rebuild_sizes_args = reduce_tensor(nt._nested_tensor_size()) + rebuild_strides_func, rebuild_strides_args = reduce_tensor( + nt._nested_tensor_strides() + ) + rebuild_offsets_func, rebuild_offsets_args = reduce_tensor( + nt._nested_tensor_storage_offsets() + ) + + return ( + rebuild_nested_tensor, + ( + rebuild_buffer_func, + rebuild_buffer_args, + rebuild_sizes_func, + rebuild_sizes_args, + rebuild_strides_func, + rebuild_strides_args, + rebuild_offsets_func, + rebuild_offsets_args, + ), + ) + + +def rebuild_sparse_coo_tensor( + rebuild_indices_func, + rebuild_indices_args, + rebuild_values_func, + rebuild_values_args, + shape, + is_coalesced, +): + indices = rebuild_indices_func(*rebuild_indices_args) + values = rebuild_values_func(*rebuild_values_args) + return torch.sparse_coo_tensor(indices, values, shape, is_coalesced=is_coalesced) + + +def rebuild_sparse_compressed_tensor( + rebuild_compressed_indices_func, + rebuild_compressed_indices_args, + rebuild_plain_indices_func, + rebuild_plain_indices_args, + rebuild_values_func, + rebuild_values_args, + shape, + layout, +): + compressed_indices = rebuild_compressed_indices_func( + *rebuild_compressed_indices_args + ) + plain_indices = rebuild_plain_indices_func(*rebuild_plain_indices_args) + values = rebuild_values_func(*rebuild_values_args) + return torch.sparse_compressed_tensor( + compressed_indices, plain_indices, values, shape, layout=layout + ) + + +def reduce_sparse_tensor(sparse): + if sparse.layout is torch.sparse_coo: + rebuild_indices_func, rebuild_indices_args = reduce_tensor(sparse._indices()) + rebuild_values_func, rebuild_values_args = reduce_tensor(sparse._values()) + return ( + rebuild_sparse_coo_tensor, + ( + rebuild_indices_func, + rebuild_indices_args, + rebuild_values_func, + rebuild_values_args, + sparse.shape, + sparse.is_coalesced(), + ), + ) + else: + if sparse.layout in {torch.sparse_csr, torch.sparse_bsr}: + compressed_indices = sparse.crow_indices() + plain_indices = sparse.col_indices() + elif sparse.layout in {torch.sparse_csc, torch.sparse_bsc}: + compressed_indices = sparse.ccol_indices() + plain_indices = sparse.row_indices() + else: + raise NotImplementedError(sparse.layout) + ( + rebuild_compressed_indices_func, + rebuild_compressed_indices_args, + ) = reduce_tensor(compressed_indices) + rebuild_plain_indices_func, rebuild_plain_indices_args = reduce_tensor( + plain_indices + ) + rebuild_values_func, rebuild_values_args = reduce_tensor(sparse.values()) + return ( + rebuild_sparse_compressed_tensor, + ( + rebuild_compressed_indices_func, + rebuild_compressed_indices_args, + rebuild_plain_indices_func, + rebuild_plain_indices_args, + rebuild_values_func, + rebuild_values_args, + sparse.shape, + sparse.layout, + ), + ) + + +def fd_id(fd): + # Returns a tuple which uniquely identifies a file descriptor. In Mac OS, + # this doesn't work with shared memory handles, which is why we don't + # support the "file_descriptor" sharing method on that platform. + stat = os.fstat(fd) + return (stat.st_ino, stat.st_dev) + + +def storage_from_cache(cls, key): + storage_ref = shared_cache.get(key) + if storage_ref is None: + return None + return torch.UntypedStorage._new_with_weak_ptr(storage_ref.cdata) + + +def rebuild_storage_fd(cls, df, size): + fd = df.detach() + try: + storage = storage_from_cache(cls, fd_id(fd)) + if storage is not None: + return storage + storage = cls._new_shared_fd_cpu(fd, size) + shared_cache[fd_id(fd)] = StorageWeakRef(storage) + return storage + finally: + os.close(fd) + + +def rebuild_storage_filename(cls, manager, handle, size, dtype=None): + storage: torch.TypedStorage | torch.UntypedStorage = storage_from_cache(cls, handle) + if storage is not None: + return storage._shared_decref() + if dtype is None: + storage = torch.UntypedStorage._new_shared_filename_cpu(manager, handle, size) + else: + byte_size = size * torch._utils._element_size(dtype) + untyped_storage: torch.UntypedStorage = ( + torch.UntypedStorage._new_shared_filename_cpu(manager, handle, byte_size) + ) + storage = torch.TypedStorage( + wrap_storage=untyped_storage, dtype=dtype, _internal=True + ) + shared_cache[handle] = StorageWeakRef(storage) + return storage._shared_decref() + + +def rebuild_storage_empty(cls): + return cls() + + +def rebuild_typed_storage(storage, dtype): + return torch.storage.TypedStorage(wrap_storage=storage, dtype=dtype, _internal=True) + + +# Use for torch.storage.TypedStorage +def reduce_typed_storage(storage): + return (rebuild_typed_storage, (storage._untyped_storage, storage.dtype)) + + +def rebuild_typed_storage_child(storage, storage_type): + return storage_type(wrap_storage=storage, _internal=True) + + +# Use for child classes of torch.storage.TypedStorage, like torch.FloatStorage +def reduce_typed_storage_child(storage): + return (rebuild_typed_storage_child, (storage._untyped_storage, type(storage))) + + +def reduce_storage(storage): + from . import get_sharing_strategy + + if storage.is_cuda: + raise RuntimeError( + "Cannot pickle CUDA storage; try pickling a CUDA tensor instead" + ) + elif storage.device.type == "meta": + raise RuntimeError( + "Cannot pickle meta storage; try pickling a meta tensor instead" + ) + elif get_sharing_strategy() == "file_system": + metadata = storage._share_filename_cpu_() + cache_key = metadata[1] + rebuild = rebuild_storage_filename + if isinstance(storage, torch.TypedStorage): + metadata += (storage.dtype,) + storage._shared_incref() + elif storage.size() == 0: + # This is special cased because Empty tensors + # (with size 0) cannot be mmapped. + return (rebuild_storage_empty, (type(storage),)) + else: + fd, size = storage._share_fd_cpu_() + df = multiprocessing.reduction.DupFd(fd) + cache_key = fd_id(fd) + metadata = (df, size) + rebuild = rebuild_storage_fd # type: ignore[assignment] + + shared_cache[cache_key] = StorageWeakRef(storage) + return (rebuild, (type(storage),) + metadata) + + +def init_reductions(): + reduction.register(torch.cuda.Event, reduce_event) + + for t in torch._storage_classes: + if t.__name__ == "UntypedStorage": + reduction.register(t, reduce_storage) + else: + reduction.register(t, reduce_typed_storage_child) + + reduction.register(torch.storage.TypedStorage, reduce_typed_storage) + + for t in torch._tensor_classes: + reduction.register(t, reduce_tensor) + + # TODO: Maybe this should be in tensor_classes? :) + reduction.register(torch.Tensor, reduce_tensor) + + from torch.nn.parameter import Parameter + + reduction.register(Parameter, reduce_tensor) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/spawn.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/spawn.py new file mode 100644 index 0000000000000000000000000000000000000000..ecf8a43ad502683eb08306f5e5a95a641cee0595 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/multiprocessing/spawn.py @@ -0,0 +1,340 @@ +# mypy: allow-untyped-defs +import logging +import multiprocessing +import multiprocessing.connection +import os +import pickle +import signal +import sys +import tempfile +import time +import warnings +from concurrent.futures import as_completed, ThreadPoolExecutor + +from . import _prctl_pr_set_pdeathsig # type: ignore[attr-defined] + + +ENV_VAR_PARALLEL_START = "TORCH_MP_PARALLEL_START" + +log = logging.getLogger(__name__) + +__all__ = [ + "ProcessContext", + "ProcessException", + "ProcessExitedException", + "ProcessRaisedException", + "spawn", + "SpawnContext", + "start_processes", +] + + +class ProcessException(Exception): + __slots__ = ["error_index", "error_pid"] + + def __init__(self, msg: str, error_index: int, error_pid: int): + super().__init__(msg) + self.msg = msg + self.error_index = error_index + self.error_pid = error_pid + + def __reduce__(self): + return type(self), (self.msg, self.error_index, self.error_pid) + + +class ProcessRaisedException(ProcessException): + """Exception raised when a process failed due to an exception raised by the code.""" + + +class ProcessExitedException(ProcessException): + """Exception raised when a process failed due to signal or exited with a specific code.""" + + __slots__ = ["exit_code"] + + def __init__( + self, + msg: str, + error_index: int, + error_pid: int, + exit_code: int, + signal_name: str | None = None, + ): + super().__init__(msg, error_index, error_pid) + self.exit_code = exit_code + self.signal_name = signal_name + + def __reduce__(self): + return ( + type(self), + ( + self.msg, + self.error_index, + self.error_pid, + self.exit_code, + self.signal_name, + ), + ) + + +def _wrap(fn, i, args, error_file): + # prctl(2) is a Linux specific system call. + # On other systems the following function call has no effect. + # This is set to ensure that non-daemonic child processes can + # terminate if their parent terminates before they do. + _prctl_pr_set_pdeathsig(signal.SIGINT) + + try: + fn(i, *args) + except KeyboardInterrupt: + pass # SIGINT; Killed by parent, do nothing + except Exception: + # Propagate exception to parent process, keeping original traceback + import traceback + + with open(error_file, "wb") as fh: + pickle.dump(traceback.format_exc(), fh) + sys.exit(1) + + +class ProcessContext: + def __init__(self, processes, error_files): + self.error_files = error_files + self.processes = processes + self.sentinels = { + process.sentinel: index for index, process in enumerate(processes) + } + + def pids(self): + return [int(process.pid) for process in self.processes] + + def _join_procs_with_timeout(self, timeout: float): + """Attempt to join all processes with a shared timeout.""" + end = time.monotonic() + timeout + for process in self.processes: + # pyrefly: ignore [no-matching-overload] + time_to_wait = max(0, end - time.monotonic()) + process.join(time_to_wait) + + def join(self, timeout: float | None = None, grace_period: float | None = None): + r"""Join one or more processes within spawn context. + + Attempt to join one or more processes in this spawn context. + If one of them exited with a non-zero exit status, this function + kills the remaining processes (optionally with a grace period) + and raises an exception with the cause of the first process exiting. + + Returns ``True`` if all processes have been joined successfully, + ``False`` if there are more processes that need to be joined. + + Args: + timeout (float): Wait this long (in seconds) before giving up on waiting. + grace_period (float): When any processes fail, wait this long (in seconds) + for others to shutdown gracefully before terminating them. If they + still don't exit, wait another grace period before killing them. + """ + # Ensure this function can be called even when we're done. + if len(self.sentinels) == 0: + return True + + # Wait for any process to fail or all of them to succeed. + ready = multiprocessing.connection.wait( + self.sentinels.keys(), + timeout=timeout, + ) + + error_index = None + for sentinel in ready: + index = self.sentinels.pop(sentinel) + process = self.processes[index] + process.join() + if process.exitcode != 0: + error_index = index + break + + # Return if there was no error. + if error_index is None: + # Return whether or not all processes have been joined. + return len(self.sentinels) == 0 + # An error occurred. Clean-up all processes before returning. + # First, allow a grace period for processes to shutdown themselves. + if grace_period is not None: + self._join_procs_with_timeout(grace_period) + # Then, terminate processes that are still alive. Try SIGTERM first. + for process in self.processes: + if process.is_alive(): + log.warning("Terminating process %s via signal SIGTERM", process.pid) + process.terminate() + + # Try SIGKILL if the process isn't going down after another grace_period. + # The reason is related to python signal handling is limited + # to main thread and if that is in c/c++ land and stuck it won't + # to handle it. We have seen processes getting stuck not handling + # SIGTERM for the above reason. + self._join_procs_with_timeout(30 if grace_period is None else grace_period) + for process in self.processes: + if process.is_alive(): + log.warning( + "Unable to shutdown process %s via SIGTERM , forcefully exiting via SIGKILL", + process.pid, + ) + process.kill() + process.join() + + # The file will only be created if the process crashed. + failed_process = self.processes[error_index] + if not os.access(self.error_files[error_index], os.R_OK): + exitcode = self.processes[error_index].exitcode + if exitcode < 0: + try: + name = signal.Signals(-exitcode).name + except ValueError: + name = f"" + raise ProcessExitedException( + f"process {error_index:d} terminated with signal {name}", + error_index=error_index, + error_pid=failed_process.pid, + exit_code=exitcode, + signal_name=name, + ) + else: + raise ProcessExitedException( + f"process {error_index:d} terminated with exit code {exitcode:d}", + error_index=error_index, + error_pid=failed_process.pid, + exit_code=exitcode, + ) + + with open(self.error_files[error_index], "rb") as fh: + original_trace = pickle.load(fh) + msg = f"\n\n-- Process {error_index:d} terminated with the following error:\n" + msg += original_trace + raise ProcessRaisedException(msg, error_index, failed_process.pid) + + +class SpawnContext(ProcessContext): + def __init__(self, processes, error_files): + warnings.warn( + "SpawnContext is renamed to ProcessContext since 1.4 release.", stacklevel=2 + ) + super().__init__(processes, error_files) + + +# Note: [start_processes] +# mp.start_processes handles both start_method='spawn' and 'fork'. It's supposed to be a +# more generalized API than mp.spawn. Currently we only document mp.spawn as it's the +# CUDA compatible start_method. However, in environments like Ipython notebooks, 'fork' +# works better than 'spawn'. Every helper function we created for mp.spawn is indeed +# general enough, and backends like XLA can reuse them in Colab notebooks as well. +# Currently we only add this API first, we can consider adding it to documentation as +# needed in the future. +def start_processes( + fn, + args=(), + nprocs=1, + join=True, + daemon=False, + start_method="spawn", +): + # To speed up performance in certain cases (see https://github.com/pytorch/pytorch/issues/133010), + # this func will start processes in parallel if start_method is 'forkserver'. + # Please opt in to this perf optimization by setting env var (TORCH_MP_PARALLEL_START) to 1. + # todo: investigate why spawn does not work with threadpool and raises SIGINT + if ( + start_method == "forkserver" + and os.environ.get(ENV_VAR_PARALLEL_START, "0") == "1" + ): + log.info("Starting processes in parallel.") + start_parallel = True + else: + # Set env var TORCH_MP_PARALLEL_START to 0 to disable parallel start + start_parallel = False + + mp = multiprocessing.get_context(start_method) + error_files = [None] * nprocs + processes = [None] * nprocs + + def start_process(i): + # Each process is assigned a file to write tracebacks to. We + # use the file being non-empty to indicate an exception + # occurred (vs an expected shutdown). Note: this previously + # used a multiprocessing.Queue but that can be prone to + # deadlocks, so we went with a simpler solution for a one-shot + # message between processes. + tf = tempfile.NamedTemporaryFile( # noqa: SIM115 + prefix="pytorch-errorfile-", suffix=".pickle", delete=False + ) + tf.close() + os.unlink(tf.name) + + process = mp.Process( # pyrefly: ignore # missing-attribute + target=_wrap, + args=(fn, i, args, tf.name), + daemon=daemon, + ) + + process.start() + return i, process, tf.name + + if not start_parallel: + for i in range(nprocs): + idx, process, tf_name = start_process(i) + error_files[idx] = tf_name + processes[idx] = process + else: + with ThreadPoolExecutor(max_workers=nprocs) as executor: + futures = [executor.submit(start_process, i) for i in range(nprocs)] + for fut in as_completed(futures): + idx, process, tf_name = fut.result() + # idx and process rank needs to be the same. + error_files[idx] = tf_name + processes[idx] = process + context = ProcessContext(processes, error_files) + if not join: + return context + + # Loop on join until it returns True or raises an exception. + while not context.join(): + pass + + +def spawn(fn, args=(), nprocs=1, join=True, daemon=False, start_method="spawn"): + r"""Spawns ``nprocs`` processes that run ``fn`` with ``args``. + + If one of the processes exits with a non-zero exit status, the + remaining processes are killed and an exception is raised with the + cause of termination. In the case an exception was caught in the + child process, it is forwarded and its traceback is included in + the exception raised in the parent process. + + Args: + fn (function): Function is called as the entrypoint of the + spawned process. This function must be defined at the top + level of a module so it can be pickled and spawned. This + is a requirement imposed by multiprocessing. + + The function is called as ``fn(i, *args)``, where ``i`` is + the process index and ``args`` is the passed through tuple + of arguments. + + args (tuple): Arguments passed to ``fn``. + nprocs (int): Number of processes to spawn. + join (bool): Perform a blocking join on all processes. + daemon (bool): The spawned processes' daemon flag. If set to True, + daemonic processes will be created. + start_method (str): (deprecated) this method will always use ``spawn`` + as the start method. To use a different start method + use ``start_processes()``. + + Returns: + None if ``join`` is ``True``, + :class:`~ProcessContext` if ``join`` is ``False`` + + """ + if start_method != "spawn": + msg = ( + f"This method only supports start_method=spawn (got: {start_method}).\n" + "To use a different start_method use:\n\t\t" + " torch.multiprocessing.start_processes(...)" + ) + warnings.warn(msg, FutureWarning, stacklevel=2) + return start_processes(fn, args, nprocs, join, daemon, start_method="spawn") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nativert/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nativert/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nested/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nested/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3ace31a3523c862c7603ae84d96fb71d9af1727a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nested/__init__.py @@ -0,0 +1,518 @@ +# mypy: allow-untyped-defs +from typing import Optional, Union + +import torch +import torch.nn.functional as F +from torch import SymInt, Tensor +from torch._C import _add_docstr, _nested # type: ignore[attr-defined] +from torch.types import _device as Device, _dtype as DType + + +__all__ = [ + "to_padded_tensor", + "as_nested_tensor", + "nested_tensor", + "nested_tensor_from_jagged", + "narrow", + "masked_select", +] + +# Allowlist these for weights_only load of NJT +from ._internal.nested_tensor import _rebuild_njt, NestedTensor as _NestedTensor + + +torch.serialization.add_safe_globals([_NestedTensor, _rebuild_njt]) + + +def as_nested_tensor( + ts: Tensor | list[Tensor] | tuple[Tensor, ...], + dtype: DType | None = None, + device: Device | None = None, + layout=None, +) -> Tensor: + r""" + Constructs a nested tensor preserving autograd history from a tensor or a list / tuple of + tensors. + + If a nested tensor is passed, it will be returned directly unless the device / dtype / layout + differ. Note that converting device / dtype will result in a copy, while converting layout + is not currently supported by this function. + + If a non-nested tensor is passed, it is treated as a batch of constituents of consistent size. + A copy will be incurred if the passed device / dtype differ from those of the input OR if + the input is non-contiguous. Otherwise, the input's storage will be used directly. + + If a tensor list is provided, tensors in the list are always copied during construction of + the nested tensor. + + Args: + ts (Tensor or List[Tensor] or Tuple[Tensor]): a tensor to treat as a nested tensor OR a + list / tuple of tensors with the same ndim + + Keyword arguments: + dtype (:class:`torch.dtype`, optional): the desired type of returned nested tensor. + Default: if None, same :class:`torch.dtype` as leftmost tensor in the list. + device (:class:`torch.device`, optional): the desired device of returned nested tensor. + Default: if None, same :class:`torch.device` as leftmost tensor in the list + layout (:class:`torch.layout`, optional): the desired layout of returned nested tensor. + Only strided and jagged layouts are supported. Default: if None, the strided layout. + + Example:: + + >>> a = torch.arange(3, dtype=torch.float, requires_grad=True) + >>> b = torch.arange(5, dtype=torch.float, requires_grad=True) + >>> nt = torch.nested.as_nested_tensor([a, b]) + >>> nt.is_leaf + False + >>> fake_grad = torch.nested.nested_tensor([torch.ones_like(a), torch.zeros_like(b)]) + >>> nt.backward(fake_grad) + >>> a.grad + tensor([1., 1., 1.]) + >>> b.grad + tensor([0., 0., 0., 0., 0.]) + >>> c = torch.randn(3, 5, requires_grad=True) + >>> nt2 = torch.nested.as_nested_tensor(c) + """ + is_tensor_list = isinstance(ts, (list, tuple)) and all( + isinstance(t, Tensor) for t in ts + ) + if not isinstance(ts, Tensor) and not is_tensor_list: + raise TypeError( + "as_nested_tensor(): Expected first argument to be a tensor or a list / tuple of tensors " + ) + # convert tuple -> list if needed + if is_tensor_list and not isinstance(ts, list): + ts = list(ts) + + if isinstance(ts, Tensor) and ts.dim() < 2: + raise RuntimeError( + "as_nested_tensor(): Expected tensor argument to have dim() > 1" + ) + + if isinstance(ts, Tensor) and ts.is_nested: + if layout == ts.layout: + # return input directly or input copied to device / dtype + return ts.to(device=device, dtype=dtype) + else: + # TODO: Just use nt.to(layout=layout) when it exists. + raise RuntimeError( + "as_nested_tensor(): Converting between nested tensor layouts is not supported" + ) + + if layout is None: + layout = torch.strided + if layout == torch.strided: + if isinstance(ts, Tensor): + # contiguous() might be necessary to get flattened view. + # we could probably be more precise about when to do this as an optimization + buffer = ts.contiguous().view(-1).to(device=device, dtype=dtype) + nested_sizes = torch.tensor([t.shape for t in ts]) + return torch._nested_view_from_buffer( + buffer, + nested_sizes, + *torch._nested_compute_contiguous_strides_offsets(nested_sizes), + ) + else: + assert isinstance(ts, list) + return torch._nested_tensor_from_tensor_list(ts, dtype, None, device, None) + elif layout == torch.jagged: + if isinstance(ts, Tensor): + if device is None: + device = ts.device + + # contiguous() might be necessary to get flattened view. + # we could probably be more precise about when to do this as an optimization + values = ts.contiguous().flatten(0, 1).to(device=device, dtype=dtype) + batch_size = ts.shape[0] + seq_len = ts.shape[1] + offsets = torch.arange( + 0, batch_size * seq_len + 1, seq_len, device=device, dtype=torch.int64 + ) + + from torch.nested._internal.nested_tensor import ( + nested_view_from_values_offsets, + ) + + return nested_view_from_values_offsets( + values, offsets, min_seqlen=seq_len, max_seqlen=seq_len + ) + else: + from torch.nested._internal.nested_tensor import jagged_from_list + + assert isinstance(ts, list) + nt, _ = jagged_from_list(ts, offsets=None, device=device, dtype=dtype) + return nt + else: + raise RuntimeError( + f"Specified layout is unsupported for nested tensors: {layout}" + ) + + +# Note: This not only adds doc strings for the nested ops, but +# also connects the torch.nested Python namespace to the torch._C._nested builtins. + +to_padded_tensor = _add_docstr( + _nested.nested_to_padded_tensor, + r""" +to_padded_tensor(input, padding, output_size=None, out=None) -> Tensor + +Returns a new (non-nested) Tensor by padding the :attr:`input` nested tensor. +The leading entries will be filled with the nested data, +while the trailing entries will be padded. + +.. warning:: + + :func:`to_padded_tensor` always copies the underlying data, + since the nested and the non-nested tensors differ in memory layout. + +Args: + padding (float): The padding value for the trailing entries. + +Keyword args: + output_size (Tuple[int]): The size of the output tensor. + If given, it must be large enough to contain all nested data; + else, will infer by taking the max size of each nested sub-tensor along each dimension. + out (Tensor, optional): the output tensor. + +Example:: + + >>> nt = torch.nested.nested_tensor([torch.randn((2, 5)), torch.randn((3, 4))]) + nested_tensor([ + tensor([[ 1.6862, -1.1282, 1.1031, 0.0464, -1.3276], + [-1.9967, -1.0054, 1.8972, 0.9174, -1.4995]]), + tensor([[-1.8546, -0.7194, -0.2918, -0.1846], + [ 0.2773, 0.8793, -0.5183, -0.6447], + [ 1.8009, 1.8468, -0.9832, -1.5272]]) + ]) + >>> pt_infer = torch.nested.to_padded_tensor(nt, 0.0) + tensor([[[ 1.6862, -1.1282, 1.1031, 0.0464, -1.3276], + [-1.9967, -1.0054, 1.8972, 0.9174, -1.4995], + [ 0.0000, 0.0000, 0.0000, 0.0000, 0.0000]], + [[-1.8546, -0.7194, -0.2918, -0.1846, 0.0000], + [ 0.2773, 0.8793, -0.5183, -0.6447, 0.0000], + [ 1.8009, 1.8468, -0.9832, -1.5272, 0.0000]]]) + >>> pt_large = torch.nested.to_padded_tensor(nt, 1.0, (2, 4, 6)) + tensor([[[ 1.6862, -1.1282, 1.1031, 0.0464, -1.3276, 1.0000], + [-1.9967, -1.0054, 1.8972, 0.9174, -1.4995, 1.0000], + [ 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000], + [ 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000]], + [[-1.8546, -0.7194, -0.2918, -0.1846, 1.0000, 1.0000], + [ 0.2773, 0.8793, -0.5183, -0.6447, 1.0000, 1.0000], + [ 1.8009, 1.8468, -0.9832, -1.5272, 1.0000, 1.0000], + [ 1.0000, 1.0000, 1.0000, 1.0000, 1.0000, 1.0000]]]) + >>> pt_small = torch.nested.to_padded_tensor(nt, 2.0, (2, 2, 2)) + RuntimeError: Value in output_size is less than NestedTensor padded size. Truncation is not supported. + +""", +) + + +def nested_tensor( + tensor_list, + *, + dtype=None, + layout=None, + device=None, + requires_grad=False, + pin_memory=False, +) -> Tensor: + r""" + Constructs a nested tensor with no autograd history (also known as a "leaf tensor", see + :ref:`Autograd mechanics `) from :attr:`tensor_list` a list of tensors. + + Args: + tensor_list (List[array_like]): a list of tensors, or anything that can be passed to torch.tensor, + where each element of the list has the same dimensionality. + + Keyword arguments: + dtype (:class:`torch.dtype`, optional): the desired type of returned nested tensor. + Default: if None, same :class:`torch.dtype` as leftmost tensor in the list. + layout (:class:`torch.layout`, optional): the desired layout of returned nested tensor. + Only strided and jagged layouts are supported. Default: if None, the strided layout. + device (:class:`torch.device`, optional): the desired device of returned nested tensor. + Default: if None, same :class:`torch.device` as leftmost tensor in the list + requires_grad (bool, optional): If autograd should record operations on the + returned nested tensor. Default: ``False``. + pin_memory (bool, optional): If set, returned nested tensor would be allocated in + the pinned memory. Works only for CPU tensors. Default: ``False``. + + Example:: + + >>> a = torch.arange(3, dtype=torch.float, requires_grad=True) + >>> b = torch.arange(5, dtype=torch.float, requires_grad=True) + >>> nt = torch.nested.nested_tensor([a, b], requires_grad=True) + >>> nt.is_leaf + True + """ + if layout is None: + layout = torch.strided + if layout == torch.strided: + return _nested.nested_tensor( + tensor_list, + dtype=dtype, + device=device, + requires_grad=requires_grad, + pin_memory=pin_memory, + ) + elif layout == torch.jagged: + # Need to wrap lists of scalars as tensors + list_of_tensors = [ + t if isinstance(t, Tensor) else torch.as_tensor(t) for t in tensor_list + ] + + from torch.nested._internal.nested_tensor import jagged_from_list + + with torch.no_grad(): + nt, _ = jagged_from_list( + list_of_tensors, offsets=None, device=device, dtype=dtype + ) + + nt.requires_grad_(requires_grad) + if pin_memory: + nt = nt.pin_memory() # type: ignore[assignment] + + return nt + else: + raise RuntimeError( + f"Specified layout is unsupported for nested tensors: {layout}" + ) + + +def narrow( + tensor: Tensor, + dim: int, + start: int | Tensor, + length: int | Tensor, + layout=torch.strided, +) -> Tensor: + r""" + Constructs a nested tensor (which might be a view) from :attr:`tensor`, a strided tensor. This follows + similar semantics to torch.Tensor.narrow, where in the :attr:`dim`-th dimension the new nested tensor + shows only the elements in the interval `[start, start+length)`. As nested representations + allow for a different `start` and `length` at each 'row' of that dimension, :attr:`start` and :attr:`length` + can also be tensors of shape `tensor.shape[0]`. + + There's some differences depending on the layout you use for the nested tensor. If using strided layout, + torch.narrow will do a copy of the narrowed data into a contiguous NT with strided layout, while + jagged layout narrow() will create a non-contiguous view of your original strided tensor. This particular + representation is really useful for representing kv-caches in Transformer models, as specialized + SDPA kernels can deal with format easily, resulting in performance improvements. + + + Args: + tensor (:class:`torch.Tensor`): a strided tensor, which will be used as the underlying data + for the nested tensor if using the jagged layout or will be copied for the strided layout. + dim (int): the dimension where narrow will be applied. Only `dim=1` is supported for the + jagged layout, while strided supports all dim + start (Union[int, :class:`torch.Tensor`]): starting element for the narrow operation + length (Union[int, :class:`torch.Tensor`]): number of elements taken during the narrow op + + Keyword arguments: + layout (:class:`torch.layout`, optional): the desired layout of returned nested tensor. + Only strided and jagged layouts are supported. Default: if None, the strided layout. + + Example:: + + >>> starts = torch.tensor([0, 1, 2, 3, 4], dtype=torch.int64) + >>> lengths = torch.tensor([3, 2, 2, 1, 5], dtype=torch.int64) + >>> narrow_base = torch.randn(5, 10, 20) + >>> nt_narrowed = torch.nested.narrow(narrow_base, 1, starts, lengths, layout=torch.jagged) + >>> nt_narrowed.is_contiguous() + False + """ + if not isinstance(start, (int, SymInt, Tensor)): + raise RuntimeError("start must be an integer or a tensor") + + if not isinstance(length, (int, SymInt, Tensor)): + raise RuntimeError("length must be an integer or a tensor") + + if layout == torch.strided: + if isinstance(start, Tensor) or isinstance(length, Tensor): + raise RuntimeError( + "start and length must be integers for the strided layout NT impl" + ) + # TODO: switch to as_nested_tensor(tensor) when it is available + nt = as_nested_tensor(torch.unbind(tensor), layout=torch.strided).narrow( + dim, start, length + ) + elif layout == torch.jagged: + if dim != 1: + raise RuntimeError("jagged layout only supports dim=1") + + from torch.nested._internal.nested_tensor import jagged_from_tensor_and_lengths + + if isinstance(start, (int, SymInt)): + start = torch.tensor([start], device=tensor.device, dtype=torch.int64) + + if isinstance(length, (int, SymInt)): + length = torch.tensor([length], device=tensor.device, dtype=torch.int64) + + nt, _, _ = jagged_from_tensor_and_lengths(tensor, start, length) + else: + raise RuntimeError( + f"Specified layout is unsupported for nested narrow: {layout}" + ) + + return nt + + +def nested_tensor_from_jagged( + values: Tensor, + offsets: Tensor | None = None, + lengths: Tensor | None = None, + jagged_dim: int | None = None, + min_seqlen: int | None = None, + max_seqlen: int | None = None, +) -> Tensor: + r""" + Constructs a jagged layout nested tensor from the given jagged components. The jagged layout + consists of a required values buffer with the jagged dimension packed into a single dimension. + The offsets / lengths metadata determines how this dimension is split into batch elements + and are expected to be allocated on the same device as the values buffer. + + Expected metadata formats: + * offsets: Indices within the packed dimension splitting it into heterogeneously-sized + batch elements. Example: [0, 2, 3, 6] indicates that a packed jagged dim of size 6 + should be conceptually split into batch elements of length [2, 1, 3]. Note that both the + beginning and ending offsets are required for kernel convenience (i.e. shape batch_size + 1). + * lengths: Lengths of the individual batch elements; shape == batch_size. Example: [2, 1, 3] + indicates that a packed jagged dim of size 6 should be conceptually split into batch + elements of length [2, 1, 3]. + + Note that it can be useful to provide both offsets and lengths. This describes a nested tensor + with "holes", where the offsets indicate the start position of each batch item and the length + specifies the total number of elements (see example below). + + The returned jagged layout nested tensor will be a view of the input values tensor. + + Args: + values (:class:`torch.Tensor`): The underlying buffer in the shape of + (sum_B(*), D_1, ..., D_N). The jagged dimension is packed into a single dimension, + with the offsets / lengths metadata used to distinguish batch elements. + offsets (optional :class:`torch.Tensor`): Offsets into the jagged dimension of shape B + 1. + lengths (optional :class:`torch.Tensor`): Lengths of the batch elements of shape B. + jagged_dim (optional int): Indicates which dimension in values is the packed jagged + dimension. Must be >= 1 as the batch dimension (dim=0) cannot be ragged. + If None, this is set to dim=1 (i.e. the dimension immediately following the batch dimension). Default: None + min_seqlen (optional int): If set, uses the specified value as the cached minimum sequence + length for the returned nested tensor. This can be a useful alternative to computing + this value on-demand, possibly avoiding a GPU -> CPU sync. Default: None + max_seqlen (optional int): If set, uses the specified value as the cached maximum sequence + length for the returned nested tensor. This can be a useful alternative to computing + this value on-demand, possibly avoiding a GPU -> CPU sync. Default: None + + Example:: + + >>> values = torch.randn(12, 5) + >>> offsets = torch.tensor([0, 3, 5, 6, 10, 12]) + >>> nt = nested_tensor_from_jagged(values, offsets) + >>> # 3D shape with the middle dimension jagged + >>> nt.shape + torch.Size([5, j2, 5]) + >>> # Length of each item in the batch: + >>> offsets.diff() + tensor([3, 2, 1, 4, 2]) + + >>> values = torch.randn(6, 5) + >>> offsets = torch.tensor([0, 2, 3, 6]) + >>> lengths = torch.tensor([1, 1, 2]) + >>> # NT with holes + >>> nt = nested_tensor_from_jagged(values, offsets, lengths) + >>> a, b, c = nt.unbind() + >>> # Batch item 1 consists of indices [0, 1) + >>> torch.equal(a, values[0:1, :]) + True + >>> # Batch item 2 consists of indices [2, 3) + >>> torch.equal(b, values[2:3, :]) + True + >>> # Batch item 3 consists of indices [3, 5) + >>> torch.equal(c, values[3:5, :]) + True + """ + from torch.fx._symbolic_trace import is_fx_tracing + + if is_fx_tracing(): + raise RuntimeError( + "torch.nested.nested_tensor_from_jagged does not support tracing with fx.symbolic_trace. " + "Use fx.wrap to wrap the function that calls nested_tensor_from_jagged." + ) + + if offsets is None: + if lengths is None: + raise RuntimeError( + "nested_tensor_from_jagged(): At least one of offsets or lengths is required." + ) + else: + # TODO: Truly support offsets=None at some point? + # For now, just convert lengths -> offsets for kernel convenience + offsets = F.pad(lengths.cumsum(0), (1, 0)) + lengths = None + + if jagged_dim is None: + jagged_dim = 1 + elif jagged_dim < 1: + raise ValueError(f"Expected jagged_dim >=1, but got {jagged_dim}.") + + from torch.nested._internal.nested_tensor import ( + nested_view_from_values_offsets_lengths, + ) + + return nested_view_from_values_offsets_lengths( + values, + offsets, + lengths, + ragged_idx=jagged_dim, + min_seqlen=min_seqlen, + max_seqlen=max_seqlen, + ) + + +def masked_select(tensor: Tensor, mask: Tensor) -> Tensor: + r""" + Constructs a nested tensor given a strided tensor input and a strided mask, the resulting jagged layout nested tensor + will have values retain values where the mask is equal to True. The dimensionality of the mask is preserved and is + represented with the offsets, this is unlike :func:`masked_select` where the output is collapsed to a 1D tensor. + + Args: + tensor (:class:`torch.Tensor`): a strided tensor from which the jagged layout nested tensor is constructed from. + mask (:class:`torch.Tensor`): a strided mask tensor which is applied to the tensor input + + Example:: + + >>> tensor = torch.randn(3, 3) + >>> mask = torch.tensor([[False, False, True], [True, False, True], [False, False, True]]) + >>> nt = torch.nested.masked_select(tensor, mask) + >>> nt.shape + torch.Size([3, j4]) + >>> # Length of each item in the batch: + >>> nt.offsets().diff() + tensor([1, 2, 1]) + + >>> tensor = torch.randn(6, 5) + >>> mask = torch.tensor([False]) + >>> nt = torch.nested.masked_select(tensor, mask) + >>> nt.shape + torch.Size([6, j5]) + >>> # Length of each item in the batch: + >>> nt.offsets().diff() + tensor([0, 0, 0, 0, 0, 0]) + """ + if tensor.layout != torch.strided: + raise RuntimeError( + f"torch.nested.masked_select requires a strided tensor, given {tensor.layout}" + ) + + if mask.layout != torch.strided: + raise RuntimeError( + f"torch.nested.masked_select requires a strided mask, given: {mask.layout}" + ) + res_values = tensor.masked_select(mask) + expanded_mask = mask.expand(tensor.shape) + res_lengths = expanded_mask.sum(dim=tensor.ndim - 1).view(-1) + + from torch.nested._internal.nested_tensor import nested_view_from_values_offsets + + return nested_view_from_values_offsets( + values=res_values, + offsets=F.pad(res_lengths.cumsum(dim=0), (1, 0)), + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..36c3ae609bb0871214138888d29501c803705598 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/__init__.py @@ -0,0 +1,62 @@ +# mypy: allow-untyped-defs +from torch.nn.parameter import ( # usort: skip + Buffer as Buffer, + Parameter as Parameter, + UninitializedBuffer as UninitializedBuffer, + UninitializedParameter as UninitializedParameter, +) +from torch.nn.modules import * # usort: skip # noqa: F403 +from torch.nn import ( + attention as attention, + functional as functional, + init as init, + modules as modules, + parallel as parallel, + parameter as parameter, + utils as utils, +) +from torch.nn.parallel import DataParallel as DataParallel + + +def factory_kwargs(kwargs): + r"""Return a canonicalized dict of factory kwargs. + + Given kwargs, returns a canonicalized dict of factory kwargs that can be directly passed + to factory functions like torch.empty, or errors if unrecognized kwargs are present. + + This function makes it simple to write code like this:: + + class MyModule(nn.Module): + def __init__(self, **kwargs): + factory_kwargs = torch.nn.factory_kwargs(kwargs) + self.weight = Parameter(torch.empty(10, **factory_kwargs)) + + Why should you use this function instead of just passing `kwargs` along directly? + + 1. This function does error validation, so if there are unexpected kwargs we will + immediately report an error, instead of deferring it to the factory call + 2. This function supports a special `factory_kwargs` argument, which can be used to + explicitly specify a kwarg to be used for factory functions, in the event one of the + factory kwargs conflicts with an already existing argument in the signature (e.g. + in the signature ``def f(dtype, **kwargs)``, you can specify ``dtype`` for factory + functions, as distinct from the dtype argument, by saying + ``f(dtype1, factory_kwargs={"dtype": dtype2})``) + """ + if kwargs is None: + return {} + simple_keys = {"device", "dtype", "memory_format"} + expected_keys = simple_keys | {"factory_kwargs"} + if not kwargs.keys() <= expected_keys: + raise TypeError(f"unexpected kwargs {kwargs.keys() - expected_keys}") + + # guarantee no input kwargs is untouched + r = dict(kwargs.get("factory_kwargs", {})) + for k in simple_keys: + if k in kwargs: + if k in r: + raise TypeError( + f"{k} specified twice, in **kwargs and in factory_kwargs" + ) + r[k] = kwargs[k] + + return r diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/_reduction.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/_reduction.py new file mode 100644 index 0000000000000000000000000000000000000000..a3ca62929a3b53ec7108df75034d51ccb7a7949c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/_reduction.py @@ -0,0 +1,60 @@ +import warnings + + +# NB: Keep this file in sync with enums in aten/src/ATen/core/Reduction.h + + +def get_enum(reduction: str) -> int: + if reduction == "none": + ret = 0 + elif reduction == "mean": + ret = 1 + elif reduction == "elementwise_mean": + warnings.warn( + "reduction='elementwise_mean' is deprecated. " + "Please use reduction='mean' instead.", + stacklevel=2, + ) + ret = 1 + elif reduction == "sum": + ret = 2 + else: + ret = -1 # TODO: remove once JIT exceptions support control flow + raise ValueError(f"{reduction} is not a valid value for reduction") + return ret + + +# In order to support previous versions, accept boolean size_average and reduce +# and convert them into the new constants for now + + +# We use these functions in torch/legacy as well, in which case we'll silence the warning +def legacy_get_string( + size_average: bool | None, + reduce: bool | None, + emit_warning: bool = True, +) -> str: + warning = "size_average and reduce args will be deprecated, please use reduction='{}' instead." + + if size_average is None: + size_average = True + if reduce is None: + reduce = True + + if size_average and reduce: + ret = "mean" + elif reduce: + ret = "sum" + else: + ret = "none" + if emit_warning: + warnings.warn(warning.format(ret), stacklevel=2) + return ret + + +def legacy_get_enum( + size_average: bool | None, + reduce: bool | None, + emit_warning: bool = True, +) -> int: + return get_enum(legacy_get_string(size_average, reduce, emit_warning)) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/common_types.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/common_types.py new file mode 100644 index 0000000000000000000000000000000000000000..e1928414a396e9be5f4bc90e198b8c5eac77e23c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/common_types.py @@ -0,0 +1,46 @@ +from typing import TypeAlias as _TypeAlias, TypeVar + +from torch import Tensor + + +# ruff: noqa: PYI042,PYI047 + +# Create some useful type aliases + +# Template for arguments which can be supplied as a tuple, or which can be a scalar which PyTorch will internally +# broadcast to a tuple. +# Comes in several variants: A tuple of unknown size, and a fixed-size tuple for 1d, 2d, or 3d operations. +T = TypeVar("T") +_scalar_or_tuple_any_t: _TypeAlias = T | tuple[T, ...] +_scalar_or_tuple_1_t: _TypeAlias = T | tuple[T] +_scalar_or_tuple_2_t: _TypeAlias = T | tuple[T, T] +_scalar_or_tuple_3_t: _TypeAlias = T | tuple[T, T, T] +_scalar_or_tuple_4_t: _TypeAlias = T | tuple[T, T, T, T] +_scalar_or_tuple_5_t: _TypeAlias = T | tuple[T, T, T, T, T] +_scalar_or_tuple_6_t: _TypeAlias = T | tuple[T, T, T, T, T, T] + +# For arguments which represent size parameters (eg, kernel size, padding) +_size_any_t: _TypeAlias = _scalar_or_tuple_any_t[int] +_size_1_t: _TypeAlias = _scalar_or_tuple_1_t[int] +_size_2_t: _TypeAlias = _scalar_or_tuple_2_t[int] +_size_3_t: _TypeAlias = _scalar_or_tuple_3_t[int] +_size_4_t: _TypeAlias = _scalar_or_tuple_4_t[int] +_size_5_t: _TypeAlias = _scalar_or_tuple_5_t[int] +_size_6_t: _TypeAlias = _scalar_or_tuple_6_t[int] + +# For arguments which represent optional size parameters (eg, adaptive pool parameters) +_size_any_opt_t: _TypeAlias = _scalar_or_tuple_any_t[int | None] +_size_2_opt_t: _TypeAlias = _scalar_or_tuple_2_t[int | None] +_size_3_opt_t: _TypeAlias = _scalar_or_tuple_3_t[int | None] + +# For arguments that represent a ratio to adjust each dimension of an input with (eg, upsampling parameters) +_ratio_2_t: _TypeAlias = _scalar_or_tuple_2_t[float] +_ratio_3_t: _TypeAlias = _scalar_or_tuple_3_t[float] +_ratio_any_t: _TypeAlias = _scalar_or_tuple_any_t[float] + +_tensor_list_t: _TypeAlias = _scalar_or_tuple_any_t[Tensor] + +# For the return value of max pooling operations that may or may not return indices. +# With the proposed 'Literal' feature to Python typing, it might be possible to +# eventually eliminate this. +_maybe_indices_t: _TypeAlias = _scalar_or_tuple_2_t[Tensor] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/cpp.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/cpp.py new file mode 100644 index 0000000000000000000000000000000000000000..b4ffd188cd39a43c2c1099eab81f620629fff1d6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/cpp.py @@ -0,0 +1,90 @@ +# mypy: allow-untyped-defs +"""Functionality for Python <-> C++ frontend inter-op.""" + +from torch import nn + + +class OrderedDictWrapper: + """A wrapper around a C++ OrderedDict. + + It dynamically evaluates the OrderedDict getter on a bound C++ module, such + that new changes on the C++ side are picked up. Otherwise accessing e.g. + ``cpp_module._parameters`` just once would get a frozen copy of the parameters + at the time of access. ``torch.nn.Module`` accesses ``_parameters`` et al. via ``self.__dict__`` + so using properties does not work. + """ + + def __init__(self, cpp_module, attr) -> None: + self.cpp_module = cpp_module + self.attr = attr + + @property + def cpp_dict(self): + return getattr(self.cpp_module, self.attr) + + # Magic methods cannot be assigned dynamically and bypass ``getattr``, so we + # must manually override them. + + def items(self): + return self.cpp_dict.items() + + def keys(self): + return self.cpp_dict.keys() + + def values(self): + return self.cpp_dict.values() + + def __iter__(self): + return self.cpp_dict.__iter__() + + def __len__(self) -> int: + return self.cpp_dict.__len__() + + def __contains__(self, key) -> bool: + return self.cpp_dict.__contains__(key) + + def __getitem__(self, key): + return self.cpp_dict.__getitem__(key) + + +class ModuleWrapper(nn.Module): + """A subclass of ``torch.nn.Module`` that wraps a C++ frontend module and delegates all access.""" + + def __init__(self, cpp_module) -> None: + # Assign before the super class constructor so ``self.training`` can be + # assigned to in the super class constructor. + self.cpp_module = cpp_module + super().__init__() + self._parameters = OrderedDictWrapper(cpp_module, "_parameters") # type: ignore[assignment] + self._buffers: OrderedDictWrapper = OrderedDictWrapper(cpp_module, "_buffers") # type: ignore[assignment] + self._modules: OrderedDictWrapper = OrderedDictWrapper(cpp_module, "_modules") # type: ignore[assignment] + for attr in dir(cpp_module): + # Skip magic methods and the three attributes above. + if not attr.startswith("_"): + setattr(self, attr, getattr(self.cpp_module, attr)) + + def _apply(self, fn, recurse=True): + for param in self.parameters(): + # Tensors stored in modules are graph leaves, and we don't + # want to create copy nodes, so we have to unpack the data. + param.data = fn(param.data) + if param._grad is not None: + param._grad.data = fn(param._grad.data) + + for buf in self.buffers(): + buf.data = fn(buf.data) + + return self + + # nn.Module defines training as a boolean + @property # type: ignore[override] + # pyrefly: ignore [bad-override] + def training(self): + return self.cpp_module.training + + @training.setter + def training(self, mode) -> None: + self.cpp_module.train(mode) + + def __repr__(self) -> str: + return self.cpp_module.__repr__() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/functional.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/functional.py new file mode 100644 index 0000000000000000000000000000000000000000..0feb7a6acd28761e88af5f946b7d1f8d63b0b423 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/functional.py @@ -0,0 +1,6859 @@ +"""Functional interface.""" + +import importlib +import math +import warnings +from collections.abc import Callable +from typing import Any as _Any, Optional, TYPE_CHECKING + +import torch +from torch import _VF, sym_int as _sym_int, Tensor +from torch._C import ( + _add_docstr, + _infer_size, + _ScalingType as ScalingType, + _SwizzleType as SwizzleType, +) +from torch._jit_internal import ( + _overload, + boolean_dispatch, + BroadcastingList1, + BroadcastingList2, + BroadcastingList3, +) +from torch._torch_docs import reproducibility_notes, sparse_support_notes, tf32_notes +from torch.nn import _reduction as _Reduction, grad # noqa: F401 +from torch.nn.modules.utils import _list_with_default, _pair, _single, _triple +from torch.overrides import ( + handle_torch_function, + has_torch_function, + has_torch_function_unary, + has_torch_function_variadic, +) + + +# Set visibility of the bound enums to this module +ScalingType.__module__ = "torch.nn.functional" +SwizzleType.__module__ = "torch.nn.functional" + +if TYPE_CHECKING: + from torch.types import _dtype as DType +else: + # The JIT doesn't understand Union, nor torch.dtype here + DType = int + +try: + import numpy as np +except ModuleNotFoundError: + np = None + + +conv1d = _add_docstr( + torch.conv1d, + r""" +conv1d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1) -> Tensor + +Applies a 1D convolution over an input signal composed of several input +planes. + +{tf32_note} + +See :class:`~torch.nn.Conv1d` for details and output shape. + +Note: + {cudnn_reproducibility_note} + +Note: + This operator supports complex data types i.e. ``complex32, complex64, complex128``. +""".format(**reproducibility_notes, **tf32_notes) + + r""" + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)` + weight: filters of shape :math:`(\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kW)` + bias: optional bias of shape :math:`(\text{out\_channels})`. Default: ``None`` + stride: the stride of the convolving kernel. Can be a single number or + a one-element tuple `(sW,)`. Default: 1 + padding: implicit paddings on both sides of the input. Can be a string {'valid', 'same'}, + single number or a one-element tuple `(padW,)`. Default: 0 + ``padding='valid'`` is the same as no padding. ``padding='same'`` pads + the input so the output has the same shape as the input. However, this mode + doesn't support any stride values other than 1. + + .. warning:: + For ``padding='same'``, if the ``weight`` is even-length and + ``dilation`` is odd in any dimension, a full :func:`pad` operation + may be needed internally. Lowering performance. + dilation: the spacing between kernel elements. Can be a single number or + a one-element tuple `(dW,)`. Default: 1 + groups: split input into groups, :math:`\text{in\_channels}` should be divisible by + the number of groups. Default: 1 + +Examples:: + + >>> inputs = torch.randn(33, 16, 30) + >>> filters = torch.randn(20, 16, 5) + >>> F.conv1d(inputs, filters) +""", +) + +conv2d = _add_docstr( + torch.conv2d, + r""" +conv2d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1) -> Tensor + +Applies a 2D convolution over an input image composed of several input +planes. + +{tf32_note} + +See :class:`~torch.nn.Conv2d` for details and output shape. + +Note: + {cudnn_reproducibility_note} + +Note: + This operator supports complex data types i.e. ``complex32, complex64, complex128``. +""".format(**reproducibility_notes, **tf32_notes) + + r""" + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)` + weight: filters of shape :math:`(\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kH , kW)` + bias: optional bias tensor of shape :math:`(\text{out\_channels})`. Default: ``None`` + stride: the stride of the convolving kernel. Can be a single number or a + tuple `(sH, sW)`. Default: 1 + padding: implicit paddings on both sides of the input. Can be a string {'valid', 'same'}, + single number or a tuple `(padH, padW)`. Default: 0 + ``padding='valid'`` is the same as no padding. ``padding='same'`` pads + the input so the output has the same shape as the input. However, this mode + doesn't support any stride values other than 1. + + .. warning:: + For ``padding='same'``, if the ``weight`` is even-length and + ``dilation`` is odd in any dimension, a full :func:`pad` operation + may be needed internally. Lowering performance. + + dilation: the spacing between kernel elements. Can be a single number or + a tuple `(dH, dW)`. Default: 1 + groups: split input into groups, both :math:`\text{in\_channels}` and :math:`\text{out\_channels}` + should be divisible by the number of groups. Default: 1 + +Examples:: + + >>> # With square kernels and equal stride + >>> filters = torch.randn(8, 4, 3, 3) + >>> inputs = torch.randn(1, 4, 5, 5) + >>> F.conv2d(inputs, filters, padding=1) +""", +) # noqa: E501 + +conv3d = _add_docstr( + torch.conv3d, + r""" +conv3d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1) -> Tensor + +Applies a 3D convolution over an input image composed of several input +planes. + +{tf32_note} + +See :class:`~torch.nn.Conv3d` for details and output shape. + +Note: + {cudnn_reproducibility_note} + +Note: + This operator supports complex data types i.e. ``complex32, complex64, complex128``. +""".format(**reproducibility_notes, **tf32_notes) + + r""" + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iT , iH , iW)` + weight: filters of shape :math:`(\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kT , kH , kW)` + bias: optional bias tensor of shape :math:`(\text{out\_channels})`. Default: None + stride: the stride of the convolving kernel. Can be a single number or a + tuple `(sT, sH, sW)`. Default: 1 + padding: implicit paddings on both sides of the input. Can be a string {'valid', 'same'}, + single number or a tuple `(padT, padH, padW)`. Default: 0 + ``padding='valid'`` is the same as no padding. ``padding='same'`` pads + the input so the output has the same shape as the input. However, this mode + doesn't support any stride values other than 1. + + .. warning:: + For ``padding='same'``, if the ``weight`` is even-length and + ``dilation`` is odd in any dimension, a full :func:`pad` operation + may be needed internally. Lowering performance. + + dilation: the spacing between kernel elements. Can be a single number or + a tuple `(dT, dH, dW)`. Default: 1 + groups: split input into groups, :math:`\text{in\_channels}` should be divisible by + the number of groups. Default: 1 + +Examples:: + + >>> filters = torch.randn(33, 16, 3, 3, 3) + >>> inputs = torch.randn(20, 16, 50, 10, 20) + >>> F.conv3d(inputs, filters) +""", +) # noqa: E501 + +conv_transpose1d = _add_docstr( + torch.conv_transpose1d, + r""" +conv_transpose1d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1) -> Tensor + +Applies a 1D transposed convolution operator over an input signal +composed of several input planes, sometimes also called "deconvolution". + +{tf32_note} + +See :class:`~torch.nn.ConvTranspose1d` for details and output shape. + +Note: + {cudnn_reproducibility_note} +""".format(**reproducibility_notes, **tf32_notes) + + r""" + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)` + weight: filters of shape :math:`(\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kW)` + bias: optional bias of shape :math:`(\text{out\_channels})`. Default: None + stride: the stride of the convolving kernel. Can be a single number or a + tuple ``(sW,)``. Default: 1 + padding: ``dilation * (kernel_size - 1) - padding`` zero-padding will be added to both + sides of each dimension in the input. Can be a single number or a tuple + ``(padW,)``. Default: 0 + output_padding: additional size added to one side of each dimension in the + output shape. Can be a single number or a tuple ``(out_padW)``. Default: 0 + groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the + number of groups. Default: 1 + dilation: the spacing between kernel elements. Can be a single number or + a tuple ``(dW,)``. Default: 1 + +Examples:: + + >>> inputs = torch.randn(20, 16, 50) + >>> weights = torch.randn(16, 33, 5) + >>> F.conv_transpose1d(inputs, weights) +""", +) + +conv_transpose2d = _add_docstr( + torch.conv_transpose2d, + r""" +conv_transpose2d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1) -> Tensor + +Applies a 2D transposed convolution operator over an input image +composed of several input planes, sometimes also called "deconvolution". + +{tf32_note} + +See :class:`~torch.nn.ConvTranspose2d` for details and output shape. + +Note: + {cudnn_reproducibility_note} +""".format(**reproducibility_notes, **tf32_notes) + + r""" + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)` + weight: filters of shape :math:`(\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kH , kW)` + bias: optional bias of shape :math:`(\text{out\_channels})`. Default: None + stride: the stride of the convolving kernel. Can be a single number or a + tuple ``(sH, sW)``. Default: 1 + padding: ``dilation * (kernel_size - 1) - padding`` zero-padding will be added to both + sides of each dimension in the input. Can be a single number or a tuple + ``(padH, padW)``. Default: 0 + output_padding: additional size added to one side of each dimension in the + output shape. Can be a single number or a tuple ``(out_padH, out_padW)``. + Default: 0 + groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the + number of groups. Default: 1 + dilation: the spacing between kernel elements. Can be a single number or + a tuple ``(dH, dW)``. Default: 1 + +Examples:: + + >>> # With square kernels and equal stride + >>> inputs = torch.randn(1, 4, 5, 5) + >>> weights = torch.randn(4, 8, 3, 3) + >>> F.conv_transpose2d(inputs, weights, padding=1) +""", +) # noqa: E501 + +conv_transpose3d = _add_docstr( + torch.conv_transpose3d, + r""" +conv_transpose3d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1) -> Tensor + +Applies a 3D transposed convolution operator over an input image +composed of several input planes, sometimes also called "deconvolution" + +{tf32_note} + +See :class:`~torch.nn.ConvTranspose3d` for details and output shape. + +Note: + {cudnn_reproducibility_note} +""".format(**reproducibility_notes, **tf32_notes) + + r""" + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iT , iH , iW)` + weight: filters of shape :math:`(\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kT , kH , kW)` + bias: optional bias of shape :math:`(\text{out\_channels})`. Default: None + stride: the stride of the convolving kernel. Can be a single number or a + tuple ``(sT, sH, sW)``. Default: 1 + padding: ``dilation * (kernel_size - 1) - padding`` zero-padding will be added to both + sides of each dimension in the input. Can be a single number or a tuple + ``(padT, padH, padW)``. Default: 0 + output_padding: additional size added to one side of each dimension in the + output shape. Can be a single number or a tuple + ``(out_padT, out_padH, out_padW)``. Default: 0 + groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the + number of groups. Default: 1 + dilation: the spacing between kernel elements. Can be a single number or + a tuple `(dT, dH, dW)`. Default: 1 + +Examples:: + + >>> inputs = torch.randn(20, 16, 50, 10, 20) + >>> weights = torch.randn(16, 33, 3, 3, 3) + >>> F.conv_transpose3d(inputs, weights) +""", +) # noqa: E501 + +conv_tbc = _add_docstr( + torch.conv_tbc, + r""" +Applies a 1-dimensional sequence convolution over an input sequence. +Input and output dimensions are (Time, Batch, Channels) - hence TBC. + +Args: + input: input tensor of shape :math:`(\text{sequence length} \times batch \times \text{in\_channels})` + weight: filter of shape (:math:`\text{kernel width} \times \text{in\_channels} \times \text{out\_channels}`) + bias: bias of shape (:math:`\text{out\_channels}`) + pad: number of timesteps to pad. Default: 0 +""", +) + + +# Pooling +avg_pool1d = _add_docstr( + torch.avg_pool1d, + r""" +avg_pool1d(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True) -> Tensor + +Applies a 1D average pooling over an input signal composed of several +input planes. + +See :class:`~torch.nn.AvgPool1d` for details and output shape. + +Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)` + kernel_size: the size of the window. Can be a single number or a + tuple `(kW,)` + stride: the stride of the window. Can be a single number or a tuple + `(sW,)`. Default: :attr:`kernel_size` + padding: implicit zero paddings on both sides of the input. Can be a single + number or a tuple `(padW,)`. Should be at most half of effective kernel + size, that is :math:`((kernelSize - 1) * dilation + 1) / 2`. Default: 0 + ceil_mode: when True, will use `ceil` instead of `floor` to compute the + output shape. Default: ``False`` + count_include_pad: when True, will include the zero-padding in the + averaging calculation. Default: ``True`` + +Examples:: + + >>> # pool of square window of size=3, stride=2 + >>> input = torch.tensor([[[1, 2, 3, 4, 5, 6, 7]]], dtype=torch.float32) + >>> F.avg_pool1d(input, kernel_size=3, stride=2) + tensor([[[ 2., 4., 6.]]]) + +""", +) + + +avg_pool2d = _add_docstr( + torch._C._nn.avg_pool2d, + r""" +avg_pool2d(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True, divisor_override=None) -> Tensor + +Applies 2D average-pooling operation in :math:`kH \times kW` regions by step size +:math:`sH \times sW` steps. The number of output features is equal to the number of +input planes. + +See :class:`~torch.nn.AvgPool2d` for details and output shape. + +Args: + input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)` + kernel_size: size of the pooling region. Can be a single number, a single-element tuple or a + tuple `(kH, kW)` + stride: stride of the pooling operation. Can be a single number, a single-element tuple or a + tuple `(sH, sW)`. Default: :attr:`kernel_size` + padding: implicit zero paddings on both sides of the input. Can be a + single number, a single-element tuple or a tuple `(padH, padW)`. + Should be at most half of effective kernel size, that + is :math:`((kernelSize - 1) * dilation + 1) / 2`. Default: 0 + ceil_mode: when True, will use `ceil` instead of `floor` in the formula + to compute the output shape. Default: ``False`` + count_include_pad: when True, will include the zero-padding in the + averaging calculation. Default: ``True`` + divisor_override: if specified, it will be used as divisor, otherwise + size of the pooling region will be used. Default: None +""", +) + +avg_pool3d = _add_docstr( + torch._C._nn.avg_pool3d, + r""" +avg_pool3d(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True, divisor_override=None) -> Tensor + +Applies 3D average-pooling operation in :math:`kT \times kH \times kW` regions by step +size :math:`sT \times sH \times sW` steps. The number of output features is equal to +:math:`\lfloor\frac{\text{input planes}}{sT}\rfloor`. + +See :class:`~torch.nn.AvgPool3d` for details and output shape. + +Args: + input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iT \times iH , iW)` + kernel_size: size of the pooling region. Can be a single number or a + tuple `(kT, kH, kW)` + stride: stride of the pooling operation. Can be a single number or a + tuple `(sT, sH, sW)`. Default: :attr:`kernel_size` + padding: implicit zero paddings on both sides of the input. Can be a + single number or a tuple `(padT, padH, padW)`. Should be at most half + of effective kernel size, that is :math:`((kernelSize - 1) * dilation + 1) / 2`. + Default: 0 + ceil_mode: when True, will use `ceil` instead of `floor` in the formula + to compute the output shape + count_include_pad: when True, will include the zero-padding in the + averaging calculation + divisor_override: if specified, it will be used as divisor, otherwise + size of the pooling region will be used. Default: None +""", +) + + +def fractional_max_pool2d_with_indices( + input: Tensor, + kernel_size: BroadcastingList2[int], + output_size: Optional[BroadcastingList2[int]] = None, + output_ratio: Optional[BroadcastingList2[float]] = None, + return_indices: bool = False, + _random_samples: Optional[Tensor] = None, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + fractional_max_pool2d(input, kernel_size, output_size=None, output_ratio=None, return_indices=False, _random_samples=None) + + Applies 2D fractional max pooling over an input signal composed of several input planes. + + Fractional MaxPooling is described in detail in the paper `Fractional MaxPooling`_ by Ben Graham + + The max-pooling operation is applied in :math:`kH \times kW` regions by a stochastic + step size determined by the target output size. + The number of output features is equal to the number of input planes. + + Args: + kernel_size: the size of the window to take a max over. + Can be a single number :math:`k` (for a square kernel of :math:`k \times k`) + or a tuple `(kH, kW)` + output_size: the target output size of the image of the form :math:`oH \times oW`. + Can be a tuple `(oH, oW)` or a single number :math:`oH` for a square image :math:`oH \times oH` + output_ratio: If one wants to have an output size as a ratio of the input size, this option can be given. + This has to be a number or tuple in the range (0, 1) + return_indices: if ``True``, will return the indices along with the outputs. + Useful to pass to :func:`~torch.nn.functional.max_unpool2d`. + + Examples:: + >>> input = torch.randn(20, 16, 50, 32) + >>> # pool of square window of size=3, and target output size 13x12 + >>> F.fractional_max_pool2d(input, 3, output_size=(13, 12)) + >>> # pool of square window and target output size being half of input image size + >>> F.fractional_max_pool2d(input, 3, output_ratio=(0.5, 0.5)) + + .. _Fractional MaxPooling: + http://arxiv.org/abs/1412.6071 + """ + if has_torch_function_variadic(input, _random_samples): + return handle_torch_function( + fractional_max_pool2d_with_indices, + (input, _random_samples), + input, + kernel_size, + output_size=output_size, + output_ratio=output_ratio, + return_indices=return_indices, + _random_samples=_random_samples, + ) + if output_size is None and output_ratio is None: + raise ValueError( + "fractional_max_pool2d requires specifying either an output_size or an output_ratio" + ) + if output_size is None: + assert output_ratio is not None + if len(output_ratio) > 2: + raise ValueError( + "fractional_max_pool2d requires output_ratio to either be a single Int or tuple of Ints." + ) + _output_ratio = _pair(output_ratio) + output_size = [ + int(input.size(-2) * _output_ratio[0]), + int(input.size(-1) * _output_ratio[1]), + ] + + if _random_samples is None: + n_batch = 1 if input.dim() == 3 else input.size(0) + _random_samples = torch.rand( + n_batch, input.size(-3), 2, dtype=input.dtype, device=input.device + ) + return torch._C._nn.fractional_max_pool2d( + input, kernel_size, output_size, _random_samples + ) + + +def _fractional_max_pool2d( + input: Tensor, + kernel_size: BroadcastingList2[int], + output_size: Optional[BroadcastingList2[int]] = None, + output_ratio: Optional[BroadcastingList2[float]] = None, + return_indices: bool = False, + _random_samples: Optional[Tensor] = None, +) -> Tensor: + if has_torch_function_variadic(input, _random_samples): + return handle_torch_function( + fractional_max_pool2d, + (input, _random_samples), + input, + kernel_size, + output_size=output_size, + output_ratio=output_ratio, + return_indices=return_indices, + _random_samples=_random_samples, + ) + return fractional_max_pool2d_with_indices( + input, kernel_size, output_size, output_ratio, return_indices, _random_samples + )[0] + + +fractional_max_pool2d = boolean_dispatch( + arg_name="return_indices", + arg_index=4, + default=False, + if_true=fractional_max_pool2d_with_indices, + if_false=_fractional_max_pool2d, + module_name=__name__, + func_name="fractional_max_pool2d", +) + + +def fractional_max_pool3d_with_indices( + input: Tensor, + kernel_size: BroadcastingList3[int], + output_size: Optional[BroadcastingList3[int]] = None, + output_ratio: Optional[BroadcastingList3[float]] = None, + return_indices: bool = False, + _random_samples: Optional[Tensor] = None, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + fractional_max_pool3d(input, kernel_size, output_size=None, output_ratio=None, return_indices=False, _random_samples=None) + + Applies 3D fractional max pooling over an input signal composed of several input planes. + + Fractional MaxPooling is described in detail in the paper `Fractional MaxPooling`_ by Ben Graham + + The max-pooling operation is applied in :math:`kT \times kH \times kW` regions by a stochastic + step size determined by the target output size. + The number of output features is equal to the number of input planes. + + Args: + kernel_size: the size of the window to take a max over. + Can be a single number :math:`k` (for a square kernel of :math:`k \times k \times k`) + or a tuple `(kT, kH, kW)` + output_size: the target output size of the form :math:`oT \times oH \times oW`. + Can be a tuple `(oT, oH, oW)` or a single number :math:`oH` for a cubic output + :math:`oH \times oH \times oH` + output_ratio: If one wants to have an output size as a ratio of the input size, this option can be given. + This has to be a number or tuple in the range (0, 1) + return_indices: if ``True``, will return the indices along with the outputs. + Useful to pass to :func:`~torch.nn.functional.max_unpool3d`. + + Shape: + - Input: :math:`(N, C, T_{in}, H_{in}, W_{in})` or :math:`(C, T_{in}, H_{in}, W_{in})`. + - Output: :math:`(N, C, T_{out}, H_{out}, W_{out})` or :math:`(C, T_{out}, H_{out}, W_{out})`, where + :math:`(T_{out}, H_{out}, W_{out})=\text{output\_size}` or + :math:`(T_{out}, H_{out}, W_{out})=\text{output\_ratio} \times (T_{in}, H_{in}, W_{in})` + + Examples:: + >>> input = torch.randn(20, 16, 50, 32, 16) + >>> # pool of cubic window of size=3, and target output size 13x12x11 + >>> F.fractional_max_pool3d(input, 3, output_size=(13, 12, 11)) + >>> # pool of cubic window and target output size being half of input size + >>> F.fractional_max_pool3d(input, 3, output_ratio=(0.5, 0.5, 0.5)) + + .. _Fractional MaxPooling: + http://arxiv.org/abs/1412.6071 + """ + if has_torch_function_variadic(input, _random_samples): + return handle_torch_function( + fractional_max_pool3d_with_indices, + (input, _random_samples), + input, + kernel_size, + output_size=output_size, + output_ratio=output_ratio, + return_indices=return_indices, + _random_samples=_random_samples, + ) + if output_size is None and output_ratio is None: + raise ValueError( + "fractional_max_pool3d requires specifying either an output_size or an output_ratio" + ) + if output_size is None: + assert output_ratio is not None + _output_ratio = _triple(output_ratio) + output_size = [ + int(input.size(-3) * _output_ratio[0]), + int(input.size(-2) * _output_ratio[1]), + int(input.size(-1) * _output_ratio[2]), + ] + + if _random_samples is None: + n_batch = 1 if input.dim() == 4 else input.size(0) + _random_samples = torch.rand( + n_batch, input.size(-4), 3, dtype=input.dtype, device=input.device + ) + return torch._C._nn.fractional_max_pool3d( + input, kernel_size, output_size, _random_samples + ) + + +def _fractional_max_pool3d( + input: Tensor, + kernel_size: BroadcastingList3[int], + output_size: Optional[BroadcastingList3[int]] = None, + output_ratio: Optional[BroadcastingList3[float]] = None, + return_indices: bool = False, + _random_samples: Optional[Tensor] = None, +) -> Tensor: + if has_torch_function_variadic(input, _random_samples): + return handle_torch_function( + fractional_max_pool3d, + (input, _random_samples), + input, + kernel_size, + output_size=output_size, + output_ratio=output_ratio, + return_indices=return_indices, + _random_samples=_random_samples, + ) + return fractional_max_pool3d_with_indices( + input, kernel_size, output_size, output_ratio, return_indices, _random_samples + )[0] + + +fractional_max_pool3d = boolean_dispatch( + arg_name="return_indices", + arg_index=4, + default=False, + if_true=fractional_max_pool3d_with_indices, + if_false=_fractional_max_pool3d, + module_name=__name__, + func_name="fractional_max_pool3d", +) + + +def max_pool1d_with_indices( + input: Tensor, + kernel_size: BroadcastingList1[int], + stride: Optional[BroadcastingList1[int]] = None, + padding: BroadcastingList1[int] = 0, + dilation: BroadcastingList1[int] = 1, + ceil_mode: bool = False, + return_indices: bool = False, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + max_pool1d(input, kernel_size, stride=None, padding=0, dilation=1, ceil_mode=False, return_indices=False) + + Applies a 1D max pooling over an input signal composed of several input + planes. + + .. note:: + The order of :attr:`ceil_mode` and :attr:`return_indices` is different from + what seen in :class:`~torch.nn.MaxPool1d`, and will change in a future release. + + See :class:`~torch.nn.MaxPool1d` for details. + + Args: + input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)`, minibatch dim optional. + kernel_size: the size of the window. Can be a single number or a + tuple `(kW,)` + stride: the stride of the window. Can be a single number or a tuple + `(sW,)`. Default: :attr:`kernel_size` + padding: Implicit negative infinity padding to be added on both sides, must be >= 0 and <= kernel_size / 2. + dilation: The stride between elements within a sliding window, must be > 0. + ceil_mode: If ``True``, will use `ceil` instead of `floor` to compute the output shape. This + ensures that every element in the input tensor is covered by a sliding window. + return_indices: If ``True``, will return the argmax along with the max values. + Useful for :class:`torch.nn.functional.max_unpool1d` later + """ + if has_torch_function_unary(input): + return handle_torch_function( + max_pool1d_with_indices, + (input,), + input, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + ceil_mode=ceil_mode, + return_indices=return_indices, + ) + if stride is None: + stride = torch.jit.annotate(list[int], []) + return torch.max_pool1d_with_indices( + input, kernel_size, stride, padding, dilation, ceil_mode + ) + + +def _max_pool1d( + input: Tensor, + kernel_size: BroadcastingList1[int], + stride: Optional[BroadcastingList1[int]] = None, + padding: BroadcastingList1[int] = 0, + dilation: BroadcastingList1[int] = 1, + ceil_mode: bool = False, + return_indices: bool = False, +) -> Tensor: + if has_torch_function_unary(input): + return handle_torch_function( + max_pool1d, + (input,), + input, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + ceil_mode=ceil_mode, + return_indices=return_indices, + ) + if stride is None: + stride = torch.jit.annotate(list[int], []) + return torch.max_pool1d(input, kernel_size, stride, padding, dilation, ceil_mode) + + +max_pool1d = boolean_dispatch( + arg_name="return_indices", + arg_index=6, + default=False, + if_true=max_pool1d_with_indices, + if_false=_max_pool1d, + module_name=__name__, + func_name="max_pool1d", +) + + +def max_pool2d_with_indices( + input: Tensor, + kernel_size: BroadcastingList2[int], + stride: Optional[BroadcastingList2[int]] = None, + padding: BroadcastingList2[int] = 0, + dilation: BroadcastingList2[int] = 1, + ceil_mode: bool = False, + return_indices: bool = False, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + max_pool2d(input, kernel_size, stride=None, padding=0, dilation=1, ceil_mode=False, return_indices=False) + + Applies a 2D max pooling over an input signal composed of several input + planes. + + .. note:: + The order of :attr:`ceil_mode` and :attr:`return_indices` is different from + what seen in :class:`~torch.nn.MaxPool2d`, and will change in a future release. + + See :class:`~torch.nn.MaxPool2d` for details. + + Args: + input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)`, minibatch dim optional. + kernel_size: size of the pooling region. Can be a single number or a + tuple `(kH, kW)` + stride: stride of the pooling operation. Can be a single number or a + tuple `(sH, sW)`. Default: :attr:`kernel_size` + padding: Implicit negative infinity padding to be added on both sides, must be >= 0 and <= kernel_size / 2. + dilation: The stride between elements within a sliding window, must be > 0. + ceil_mode: If ``True``, will use `ceil` instead of `floor` to compute the output shape. This + ensures that every element in the input tensor is covered by a sliding window. + return_indices: If ``True``, will return the argmax along with the max values. + Useful for :class:`torch.nn.functional.max_unpool2d` later + """ + if has_torch_function_unary(input): + return handle_torch_function( + max_pool2d_with_indices, + (input,), + input, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + ceil_mode=ceil_mode, + return_indices=return_indices, + ) + if stride is None: + stride = torch.jit.annotate(list[int], []) + return torch._C._nn.max_pool2d_with_indices( + input, kernel_size, stride, padding, dilation, ceil_mode + ) + + +def _max_pool2d( + input: Tensor, + kernel_size: BroadcastingList2[int], + stride: Optional[BroadcastingList2[int]] = None, + padding: BroadcastingList2[int] = 0, + dilation: BroadcastingList2[int] = 1, + ceil_mode: bool = False, + return_indices: bool = False, +) -> Tensor: + if has_torch_function_unary(input): + return handle_torch_function( + max_pool2d, + (input,), + input, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + ceil_mode=ceil_mode, + return_indices=return_indices, + ) + if stride is None: + stride = torch.jit.annotate(list[int], []) + return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode) + + +max_pool2d = boolean_dispatch( + arg_name="return_indices", + arg_index=6, + default=False, + if_true=max_pool2d_with_indices, + if_false=_max_pool2d, + module_name=__name__, + func_name="max_pool2d", +) + + +def max_pool3d_with_indices( + input: Tensor, + kernel_size: BroadcastingList3[int], + stride: Optional[BroadcastingList3[int]] = None, + padding: BroadcastingList3[int] = 0, + dilation: BroadcastingList3[int] = 1, + ceil_mode: bool = False, + return_indices: bool = False, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + max_pool3d(input, kernel_size, stride=None, padding=0, dilation=1, ceil_mode=False, return_indices=False) + + Applies a 3D max pooling over an input signal composed of several input + planes. + + .. note:: + The order of :attr:`ceil_mode` and :attr:`return_indices` is different from + what seen in :class:`~torch.nn.MaxPool3d`, and will change in a future release. + + See :class:`~torch.nn.MaxPool3d` for details. + + Args: + input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iD, iH , iW)`, minibatch dim optional. + kernel_size: size of the pooling region. Can be a single number or a + tuple `(kT, kH, kW)` + stride: stride of the pooling operation. Can be a single number or a + tuple `(sT, sH, sW)`. Default: :attr:`kernel_size` + padding: Implicit negative infinity padding to be added on both sides, must be >= 0 and <= kernel_size / 2. + dilation: The stride between elements within a sliding window, must be > 0. + ceil_mode: If ``True``, will use `ceil` instead of `floor` to compute the output shape. This + ensures that every element in the input tensor is covered by a sliding window. + return_indices: If ``True``, will return the argmax along with the max values. + Useful for :class:`torch.nn.functional.max_unpool3d` later + """ + if has_torch_function_unary(input): + return handle_torch_function( + max_pool3d_with_indices, + (input,), + input, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + ceil_mode=ceil_mode, + return_indices=return_indices, + ) + if stride is None: + stride = torch.jit.annotate(list[int], []) + return torch._C._nn.max_pool3d_with_indices( + input, kernel_size, stride, padding, dilation, ceil_mode + ) + + +def _max_pool3d( + input: Tensor, + kernel_size: BroadcastingList3[int], + stride: Optional[BroadcastingList3[int]] = None, + padding: BroadcastingList3[int] = 0, + dilation: BroadcastingList3[int] = 1, + ceil_mode: bool = False, + return_indices: bool = False, +) -> Tensor: + if has_torch_function_unary(input): + return handle_torch_function( + max_pool3d, + (input,), + input, + kernel_size, + stride=stride, + padding=padding, + dilation=dilation, + ceil_mode=ceil_mode, + return_indices=return_indices, + ) + if stride is None: + stride = torch.jit.annotate(list[int], []) + return torch.max_pool3d(input, kernel_size, stride, padding, dilation, ceil_mode) + + +max_pool3d = boolean_dispatch( + arg_name="return_indices", + arg_index=6, + default=False, + if_true=max_pool3d_with_indices, + if_false=_max_pool3d, + module_name=__name__, + func_name="max_pool3d", +) + + +def _unpool_output_size( + input: Tensor, + kernel_size: list[int], + stride: list[int], + padding: list[int], + output_size: Optional[list[int]], +) -> list[int]: + input_size = input.size() + default_size = torch.jit.annotate(list[int], []) + for d in range(len(kernel_size)): + default_size.append( + (input_size[-len(kernel_size) + d] - 1) * stride[d] + + kernel_size[d] + - 2 * padding[d] + ) + if output_size is None: + ret = default_size + else: + if len(output_size) == len(kernel_size) + 2: + output_size = output_size[2:] + if len(output_size) != len(kernel_size): + raise ValueError( + "output_size should be a sequence containing " + f"{len(kernel_size)} or {len(kernel_size) + 2} elements, but it has a length of '{len(output_size)}'" + ) + for d in range(len(kernel_size)): + min_size = default_size[d] - stride[d] + max_size = default_size[d] + stride[d] + if not (min_size < output_size[d] < max_size): + raise ValueError( + f'invalid output_size "{output_size}" (dim {d} must be between {min_size} and {max_size})' + ) + + ret = output_size + return ret + + +def max_unpool1d( + input: Tensor, + indices: Tensor, + kernel_size: BroadcastingList1[int], + stride: Optional[BroadcastingList1[int]] = None, + padding: BroadcastingList1[int] = 0, + output_size: Optional[BroadcastingList1[int]] = None, +) -> Tensor: + r"""Compute a partial inverse of :class:`MaxPool1d`. + + See :class:`~torch.nn.MaxUnpool1d` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + max_unpool1d, + (input,), + input, + indices, + kernel_size, + stride=stride, + padding=padding, + output_size=output_size, + ) + kernel_size = _single(kernel_size) + if stride is not None: + _stride = _single(stride) + else: + _stride = kernel_size + padding = _single(padding) + output_size = _unpool_output_size(input, kernel_size, _stride, padding, output_size) + if isinstance(output_size, list): + output_size = output_size + [1] + else: + output_size = output_size + (1,) + return torch._C._nn.max_unpool2d( + input.unsqueeze(-1), indices.unsqueeze(-1), output_size + ).squeeze(-1) + + +def max_unpool2d( + input: Tensor, + indices: Tensor, + kernel_size: BroadcastingList2[int], + stride: Optional[BroadcastingList2[int]] = None, + padding: BroadcastingList2[int] = 0, + output_size: Optional[BroadcastingList2[int]] = None, +) -> Tensor: + r"""Compute a partial inverse of :class:`MaxPool2d`. + + See :class:`~torch.nn.MaxUnpool2d` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + max_unpool2d, + (input,), + input, + indices, + kernel_size, + stride=stride, + padding=padding, + output_size=output_size, + ) + kernel_size = _pair(kernel_size) + if stride is not None: + _stride = _pair(stride) + else: + _stride = kernel_size + padding = _pair(padding) + output_size = _unpool_output_size(input, kernel_size, _stride, padding, output_size) + return torch._C._nn.max_unpool2d(input, indices, output_size) + + +def max_unpool3d( + input: Tensor, + indices: Tensor, + kernel_size: BroadcastingList3[int], + stride: Optional[BroadcastingList3[int]] = None, + padding: BroadcastingList3[int] = 0, + output_size: Optional[BroadcastingList3[int]] = None, +) -> Tensor: + r"""Compute a partial inverse of :class:`MaxPool3d`. + + See :class:`~torch.nn.MaxUnpool3d` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + max_unpool3d, + (input,), + input, + indices, + kernel_size, + stride=stride, + padding=padding, + output_size=output_size, + ) + kernel_size = _triple(kernel_size) + if stride is not None: + _stride = _triple(stride) + else: + _stride = kernel_size + padding = _triple(padding) + output_size = _unpool_output_size(input, kernel_size, _stride, padding, output_size) + return torch._C._nn.max_unpool3d(input, indices, output_size, _stride, padding) + + +def lp_pool3d( + input: Tensor, + norm_type: int | float, + kernel_size: BroadcastingList3[int], + stride: Optional[BroadcastingList3[int]] = None, + ceil_mode: bool = False, +) -> Tensor: + r""" + Apply a 3D power-average pooling over an input signal composed of several input planes. + + If the sum of all inputs to the power of `p` is + zero, the gradient is set to zero as well. + + When ``ceil_mode`` is ``True``, sliding windows may go off-bounds if they start within the left + padding or the input. Sliding windows that would start in the right padded region are ignored. + + See :class:`~torch.nn.LPPool3d` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + lp_pool3d, + (input,), + input, + norm_type, + kernel_size, + stride=stride, + ceil_mode=ceil_mode, + ) + kd, kw, kh = _triple(kernel_size) + if stride is not None: + out = avg_pool3d(input.pow(norm_type), kernel_size, stride, 0, ceil_mode) + else: + out = avg_pool3d( + input.pow(norm_type), kernel_size, padding=0, ceil_mode=ceil_mode + ) + + return ( + (torch.sign(out) * relu(torch.abs(out))).mul(kd * kw * kh).pow(1.0 / norm_type) + ) + + +def lp_pool2d( + input: Tensor, + norm_type: int | float, + kernel_size: BroadcastingList2[int], + stride: Optional[BroadcastingList2[int]] = None, + ceil_mode: bool = False, +) -> Tensor: + r""" + Apply a 2D power-average pooling over an input signal composed of several input planes. + + If the sum of all inputs to the power of `p` is + zero, the gradient is set to zero as well. + + When ``ceil_mode`` is ``True``, sliding windows may go off-bounds if they start within the left + padding or the input. Sliding windows that would start in the right padded region are ignored. + + See :class:`~torch.nn.LPPool2d` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + lp_pool2d, + (input,), + input, + norm_type, + kernel_size, + stride=stride, + ceil_mode=ceil_mode, + ) + kw, kh = _pair(kernel_size) + if stride is not None: + out = avg_pool2d(input.pow(norm_type), kernel_size, stride, 0, ceil_mode) + else: + out = avg_pool2d( + input.pow(norm_type), kernel_size, padding=0, ceil_mode=ceil_mode + ) + + return (torch.sign(out) * relu(torch.abs(out))).mul(kw * kh).pow(1.0 / norm_type) + + +def lp_pool1d( + input: Tensor, + norm_type: int | float, + kernel_size: int, + stride: Optional[BroadcastingList1[int]] = None, + ceil_mode: bool = False, +) -> Tensor: + r"""Apply a 1D power-average pooling over an input signal composed of several input planes. + + If the sum of all inputs to the power of `p` is + zero, the gradient is set to zero as well. + + When ``ceil_mode`` is ``True``, sliding windows may go off-bounds if they start within the left + padding or the input. Sliding windows that would start in the right padded region are ignored. + + See :class:`~torch.nn.LPPool1d` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + lp_pool1d, + (input,), + input, + norm_type, + kernel_size, + stride=stride, + ceil_mode=ceil_mode, + ) + if stride is not None: + out = avg_pool1d(input.pow(norm_type), kernel_size, stride, 0, ceil_mode) + else: + out = avg_pool1d( + input.pow(norm_type), kernel_size, padding=0, ceil_mode=ceil_mode + ) + + return ( + (torch.sign(out) * relu(torch.abs(out))).mul(kernel_size).pow(1.0 / norm_type) + ) + + +def adaptive_max_pool1d_with_indices( + input: Tensor, + output_size: BroadcastingList1[int], + return_indices: bool = False, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + adaptive_max_pool1d(input, output_size, return_indices=False) + + Applies a 1D adaptive max pooling over an input signal composed of + several input planes. + + See :class:`~torch.nn.AdaptiveMaxPool1d` for details and output shape. + + Args: + output_size: the target output size (single integer) + return_indices: whether to return pooling indices. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + adaptive_max_pool1d_with_indices, + (input,), + input, + output_size, + return_indices=return_indices, + ) + return torch.adaptive_max_pool1d(input, output_size) + + +def _adaptive_max_pool1d( + input: Tensor, + output_size: BroadcastingList1[int], + return_indices: bool = False, +) -> Tensor: + if has_torch_function_unary(input): + return handle_torch_function( + adaptive_max_pool1d, + (input,), + input, + output_size, + return_indices=return_indices, + ) + return adaptive_max_pool1d_with_indices(input, output_size)[0] + + +adaptive_max_pool1d = boolean_dispatch( + arg_name="return_indices", + arg_index=2, + default=False, + if_true=adaptive_max_pool1d_with_indices, + if_false=_adaptive_max_pool1d, + module_name=__name__, + func_name="adaptive_max_pool1d", +) + + +def adaptive_max_pool2d_with_indices( + input: Tensor, + output_size: BroadcastingList2[int], + return_indices: bool = False, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r"""adaptive_max_pool2d(input, output_size, return_indices=False) + + Applies a 2D adaptive max pooling over an input signal composed of + several input planes. + + See :class:`~torch.nn.AdaptiveMaxPool2d` for details and output shape. + + Args: + output_size: the target output size (single integer or + double-integer tuple) + return_indices: whether to return pooling indices. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + adaptive_max_pool2d_with_indices, + (input,), + input, + output_size, + return_indices=return_indices, + ) + # pyrefly: ignore [bad-argument-type] + output_size = _list_with_default(output_size, input.size()) + return torch._C._nn.adaptive_max_pool2d(input, output_size) + + +def _adaptive_max_pool2d( + input: Tensor, + output_size: BroadcastingList2[int], + return_indices: bool = False, +) -> Tensor: + if has_torch_function_unary(input): + return handle_torch_function( + adaptive_max_pool2d, + (input,), + input, + output_size, + return_indices=return_indices, + ) + return adaptive_max_pool2d_with_indices(input, output_size)[0] + + +adaptive_max_pool2d = boolean_dispatch( + arg_name="return_indices", + arg_index=2, + default=False, + if_true=adaptive_max_pool2d_with_indices, + if_false=_adaptive_max_pool2d, + module_name=__name__, + func_name="adaptive_max_pool2d", +) + + +def adaptive_max_pool3d_with_indices( + input: Tensor, + output_size: BroadcastingList3[int], + return_indices: bool = False, +) -> tuple[Tensor, Tensor]: # noqa: D400 + r""" + adaptive_max_pool3d(input, output_size, return_indices=False) + + Applies a 3D adaptive max pooling over an input signal composed of + several input planes. + + See :class:`~torch.nn.AdaptiveMaxPool3d` for details and output shape. + + Args: + output_size: the target output size (single integer or + triple-integer tuple) + return_indices: whether to return pooling indices. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + adaptive_max_pool3d_with_indices, + (input,), + input, + output_size, + return_indices=return_indices, + ) + # pyrefly: ignore [bad-argument-type] + output_size = _list_with_default(output_size, input.size()) + return torch._C._nn.adaptive_max_pool3d(input, output_size) + + +def _adaptive_max_pool3d( + input: Tensor, + output_size: BroadcastingList3[int], + return_indices: bool = False, +) -> Tensor: + if has_torch_function_unary(input): + return handle_torch_function( + adaptive_max_pool3d, + (input,), + input, + output_size, + return_indices=return_indices, + ) + return adaptive_max_pool3d_with_indices(input, output_size)[0] + + +adaptive_max_pool3d = boolean_dispatch( + arg_name="return_indices", + arg_index=2, + default=False, + if_true=adaptive_max_pool3d_with_indices, + if_false=_adaptive_max_pool3d, + module_name=__name__, + func_name="adaptive_max_pool3d", +) + + +adaptive_avg_pool1d = _add_docstr( + torch.adaptive_avg_pool1d, + r""" +adaptive_avg_pool1d(input, output_size) -> Tensor + +Applies a 1D adaptive average pooling over an input signal composed of +several input planes. + +See :class:`~torch.nn.AdaptiveAvgPool1d` for details and output shape. + +Args: + output_size: the target output size (single integer) +""", +) + + +def adaptive_avg_pool2d(input: Tensor, output_size: BroadcastingList2[int]) -> Tensor: + r"""Apply a 2D adaptive average pooling over an input signal composed of several input planes. + + See :class:`~torch.nn.AdaptiveAvgPool2d` for details and output shape. + + Args: + output_size: the target output size (single integer or + double-integer tuple) + """ + if has_torch_function_unary(input): + return handle_torch_function(adaptive_avg_pool2d, (input,), input, output_size) + # pyrefly: ignore [bad-argument-type] + _output_size = _list_with_default(output_size, input.size()) + return torch._C._nn.adaptive_avg_pool2d(input, _output_size) + + +def adaptive_avg_pool3d(input: Tensor, output_size: BroadcastingList3[int]) -> Tensor: + r"""Apply a 3D adaptive average pooling over an input signal composed of several input planes. + + See :class:`~torch.nn.AdaptiveAvgPool3d` for details and output shape. + + Args: + output_size: the target output size (single integer or + triple-integer tuple) + """ + if has_torch_function_unary(input): + return handle_torch_function(adaptive_avg_pool3d, (input,), input, output_size) + # pyrefly: ignore [bad-argument-type] + _output_size = _list_with_default(output_size, input.size()) + return torch._C._nn.adaptive_avg_pool3d(input, _output_size) + + +# Activation functions +def dropout( + input: Tensor, + p: float = 0.5, + training: bool = True, + inplace: bool = False, +) -> Tensor: + r"""During training, randomly zeroes some elements of the input tensor with probability :attr:`p`. + + Uses samples from a Bernoulli distribution. + + See :class:`~torch.nn.Dropout` for details. + + Args: + p: probability of an element to be zeroed. Default: 0.5 + training: apply dropout if is ``True``. Default: ``True`` + inplace: If set to ``True``, will do this operation in-place. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + dropout, (input,), input, p=p, training=training, inplace=inplace + ) + if p < 0.0 or p > 1.0: + raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") + return ( + _VF.dropout_(input, p, training) if inplace else _VF.dropout(input, p, training) + ) + + +def alpha_dropout( + input: Tensor, + p: float = 0.5, + training: bool = False, + inplace: bool = False, +) -> Tensor: + r"""Apply alpha dropout to the input. + + See :class:`~torch.nn.AlphaDropout` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + alpha_dropout, (input,), input, p=p, training=training, inplace=inplace + ) + if p < 0.0 or p > 1.0: + raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") + return ( + _VF.alpha_dropout_(input, p, training) + if inplace + else _VF.alpha_dropout(input, p, training) + ) + + +def dropout1d( + input: Tensor, + p: float = 0.5, + training: bool = True, + inplace: bool = False, +) -> Tensor: + r"""Randomly zero out entire channels (a channel is a 1D feature map). + + For example, the :math:`j`-th channel of the :math:`i`-th sample in the + batched input is a 1D tensor :math:`\text{input}[i, j]` of the input tensor. + Each channel will be zeroed out independently on every forward call with + probability :attr:`p` using samples from a Bernoulli distribution. + + See :class:`~torch.nn.Dropout1d` for details. + + Args: + p: probability of a channel to be zeroed. Default: 0.5 + training: apply dropout if is ``True``. Default: ``True`` + inplace: If set to ``True``, will do this operation in-place. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + dropout1d, (input,), input, p=p, training=training, inplace=inplace + ) + if p < 0.0 or p > 1.0: + raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") + inp_dim = input.dim() + if inp_dim not in (2, 3): + raise RuntimeError( + f"dropout1d: Expected 2D or 3D input, but received a {inp_dim}D input. " + "Note that dropout1d exists to provide channel-wise dropout on inputs with 1 " + "spatial dimension, a channel dimension, and an optional batch dimension " + "(i.e. 2D or 3D inputs)." + ) + + is_batched = inp_dim == 3 + if not is_batched: + input = input.unsqueeze_(0) if inplace else input.unsqueeze(0) + + result = ( + _VF.feature_dropout_(input, p, training) + if inplace + else _VF.feature_dropout(input, p, training) + ) + + if not is_batched: + result = result.squeeze_(0) if inplace else result.squeeze(0) + + return result + + +def dropout2d( + input: Tensor, + p: float = 0.5, + training: bool = True, + inplace: bool = False, +) -> Tensor: + r"""Randomly zero out entire channels (a channel is a 2D feature map). + + For example, the :math:`j`-th channel of the :math:`i`-th sample in the + batched input is a 2D tensor :math:`\text{input}[i, j]` of the input tensor. + Each channel will be zeroed out independently on every forward call with + probability :attr:`p` using samples from a Bernoulli distribution. + + See :class:`~torch.nn.Dropout2d` for details. + + Args: + p: probability of a channel to be zeroed. Default: 0.5 + training: apply dropout if is ``True``. Default: ``True`` + inplace: If set to ``True``, will do this operation in-place. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + dropout2d, (input,), input, p=p, training=training, inplace=inplace + ) + if p < 0.0 or p > 1.0: + raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") + inp_dim = input.dim() + if inp_dim not in (3, 4): + warn_msg = ( + f"dropout2d: Received a {inp_dim}-D input to dropout2d, which is deprecated " + "and will result in an error in a future release. To retain the behavior " + "and silence this warning, please use dropout instead. Note that dropout2d " + "exists to provide channel-wise dropout on inputs with 2 spatial dimensions, " + "a channel dimension, and an optional batch dimension (i.e. 3D or 4D inputs)." + ) + warnings.warn(warn_msg, stacklevel=2) + + # TODO: Properly support no-batch-dim inputs. For now, these are NOT supported; passing + # a 3D input will perform dropout1d behavior instead. This was done historically and the + # behavior is maintained here for now. + # See https://github.com/pytorch/pytorch/issues/77081 + if inp_dim == 3: + warnings.warn( + "dropout2d: Received a 3D input to dropout2d and assuming that channel-wise " + "1D dropout behavior is desired - input is interpreted as shape (N, C, L), where C " + "is the channel dim. This behavior will change in a future release to interpret the " + "input as one without a batch dimension, i.e. shape (C, H, W). To maintain the 1D " + "channel-wise dropout behavior, please switch to using dropout1d instead.", + stacklevel=2, + ) + + result = ( + _VF.feature_dropout_(input, p, training) + if inplace + else _VF.feature_dropout(input, p, training) + ) + + return result + + +def dropout3d( + input: Tensor, + p: float = 0.5, + training: bool = True, + inplace: bool = False, +) -> Tensor: + r"""Randomly zero out entire channels (a channel is a 3D feature map). + + For example, the :math:`j`-th channel of the :math:`i`-th sample in the + batched input is a 3D tensor :math:`\text{input}[i, j]` of the input tensor. + Each channel will be zeroed out independently on every forward call with + probability :attr:`p` using samples from a Bernoulli distribution. + + See :class:`~torch.nn.Dropout3d` for details. + + Args: + p: probability of a channel to be zeroed. Default: 0.5 + training: apply dropout if is ``True``. Default: ``True`` + inplace: If set to ``True``, will do this operation in-place. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + dropout3d, (input,), input, p=p, training=training, inplace=inplace + ) + if p < 0.0 or p > 1.0: + raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") + inp_dim = input.dim() + if inp_dim not in (4, 5): + warn_msg = ( + f"dropout3d: Received a {inp_dim}-D input to dropout3d, which is deprecated " + "and will result in an error in a future release. To retain the behavior " + "and silence this warning, please use dropout instead. Note that dropout3d " + "exists to provide channel-wise dropout on inputs with 3 spatial dimensions, " + "a channel dimension, and an optional batch dimension (i.e. 4D or 5D inputs)." + ) + warnings.warn(warn_msg, stacklevel=2) + + is_batched = inp_dim == 5 + if not is_batched: + input = input.unsqueeze_(0) if inplace else input.unsqueeze(0) + + result = ( + _VF.feature_dropout_(input, p, training) + if inplace + else _VF.feature_dropout(input, p, training) + ) + + if not is_batched: + result = result.squeeze_(0) if inplace else result.squeeze(0) + return result + + +def feature_alpha_dropout( + input: Tensor, + p: float = 0.5, + training: bool = False, + inplace: bool = False, +) -> Tensor: + r"""Randomly masks out entire channels (a channel is a feature map). + + For example, the :math:`j`-th channel of the :math:`i`-th sample in the batch input + is a tensor :math:`\text{input}[i, j]` of the input tensor. Instead of + setting activations to zero, as in regular Dropout, the activations are set + to the negative saturation value of the SELU activation function. + + Each element will be masked independently on every forward call with + probability :attr:`p` using samples from a Bernoulli distribution. + The elements to be masked are randomized on every forward call, and scaled + and shifted to maintain zero mean and unit variance. + + See :class:`~torch.nn.FeatureAlphaDropout` for details. + + Args: + p: dropout probability of a channel to be zeroed. Default: 0.5 + training: apply dropout if is ``True``. Default: ``True`` + inplace: If set to ``True``, will do this operation in-place. Default: ``False`` + """ + if has_torch_function_unary(input): + return handle_torch_function( + feature_alpha_dropout, + (input,), + input, + p=p, + training=training, + inplace=inplace, + ) + if p < 0.0 or p > 1.0: + raise ValueError(f"dropout probability has to be between 0 and 1, but got {p}") + return ( + _VF.feature_alpha_dropout_(input, p, training) + if inplace + else _VF.feature_alpha_dropout(input, p, training) + ) + + +def _threshold( + input: Tensor, + threshold: float, + value: float, + inplace: bool = False, +) -> Tensor: + r"""Apply a threshold to each element of the input Tensor. + + See :class:`~torch.nn.Threshold` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + _threshold, (input,), input, threshold, value, inplace=inplace + ) + if inplace: + result = _VF.threshold_(input, threshold, value) + else: + result = _VF.threshold(input, threshold, value) + return result + + +# We define this function as _threshold because it takes an argument +# named threshold, which clobbers the recursive reference to the +# function needed for __torch_function__ support +threshold = _threshold + +threshold_ = _add_docstr( + _VF.threshold_, + r""" +threshold_(input, threshold, value) -> Tensor + +In-place version of :func:`~threshold`. +""", +) + + +def relu(input: Tensor, inplace: bool = False) -> Tensor: # noqa: D400,D402 + r"""relu(input, inplace=False) -> Tensor + + Applies the rectified linear unit function element-wise. See + :class:`~torch.nn.ReLU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(relu, (input,), input, inplace=inplace) + if inplace: + result = torch.relu_(input) + else: + result = torch.relu(input) + return result + + +relu_ = _add_docstr( + torch.relu_, + r""" +relu_(input) -> Tensor + +In-place version of :func:`~relu`. +""", +) + + +def glu(input: Tensor, dim: int = -1) -> Tensor: # noqa: D400,D402 + r""" + glu(input, dim=-1) -> Tensor + + The gated linear unit. Computes: + + .. math :: + \text{GLU}(a, b) = a \otimes \sigma(b) + + where `input` is split in half along `dim` to form `a` and `b`, :math:`\sigma` + is the sigmoid function and :math:`\otimes` is the element-wise product between matrices. + + See `Language Modeling with Gated Convolutional Networks `_. + + Args: + input (Tensor): input tensor + dim (int): dimension on which to split the input. Default: -1 + """ + if has_torch_function_unary(input): + return handle_torch_function(glu, (input,), input, dim=dim) + if input.dim() == 0: + raise RuntimeError( + "glu does not support scalars because halving size must be even" + ) + return torch._C._nn.glu(input, dim) + + +def hardtanh( + input: Tensor, + min_val: float = -1.0, + max_val: float = 1.0, + inplace: bool = False, +) -> Tensor: # noqa: D400,D402 + r""" + hardtanh(input, min_val=-1., max_val=1., inplace=False) -> Tensor + + Applies the HardTanh function element-wise. See :class:`~torch.nn.Hardtanh` for more + details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + hardtanh, (input,), input, min_val=min_val, max_val=max_val, inplace=inplace + ) + if min_val > max_val: + raise ValueError("min_val cannot be greater than max_val") + if inplace: + result = torch._C._nn.hardtanh_(input, min_val, max_val) + else: + result = torch._C._nn.hardtanh(input, min_val, max_val) + return result + + +hardtanh_ = _add_docstr( + torch._C._nn.hardtanh_, + r""" +hardtanh_(input, min_val=-1., max_val=1.) -> Tensor + +In-place version of :func:`~hardtanh`. +""", +) + + +def relu6(input: Tensor, inplace: bool = False) -> Tensor: # noqa: D400,D402 + r"""relu6(input, inplace=False) -> Tensor + + Applies the element-wise function :math:`\text{ReLU6}(x) = \min(\max(0,x), 6)`. + + See :class:`~torch.nn.ReLU6` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(relu6, (input,), input, inplace=inplace) + if inplace: + result = torch._C._nn.relu6_(input) + else: + result = torch._C._nn.relu6(input) + return result + + +def elu(input: Tensor, alpha: float = 1.0, inplace: bool = False) -> Tensor: + r"""Apply the Exponential Linear Unit (ELU) function element-wise. + + See :class:`~torch.nn.ELU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(elu, (input,), input, alpha=alpha, inplace=inplace) + if inplace: + result = torch._C._nn.elu_(input, alpha) + else: + result = torch._C._nn.elu(input, alpha) + return result + + +elu_ = _add_docstr( + torch._C._nn.elu_, + r""" +elu_(input, alpha=1.) -> Tensor + +In-place version of :func:`~elu`. +""", +) + + +def selu(input: Tensor, inplace: bool = False) -> Tensor: # noqa: D400,D402 + r"""selu(input, inplace=False) -> Tensor + + Applies element-wise, + :math:`\text{SELU}(x) = scale * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1)))`, + with :math:`\alpha=1.6732632423543772848170429916717` and + :math:`scale=1.0507009873554804934193349852946`. + + See :class:`~torch.nn.SELU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(selu, (input,), input, inplace=inplace) + if inplace: + result = torch.selu_(input) + else: + result = torch.selu(input) + return result + + +selu_ = _add_docstr( + torch.selu_, + r""" +selu_(input) -> Tensor + +In-place version of :func:`~selu`. +""", +) + + +def celu( + input: Tensor, + alpha: float = 1.0, + inplace: bool = False, +) -> Tensor: # noqa: D400,D402 + r"""celu(input, alpha=1., inplace=False) -> Tensor + + Applies element-wise, + :math:`\text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))`. + + See :class:`~torch.nn.CELU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + celu, (input,), input, alpha=alpha, inplace=inplace + ) + if inplace: + result = torch.celu_(input, alpha) + else: + result = torch.celu(input, alpha) + return result + + +celu_ = _add_docstr( + torch.celu_, + r""" +celu_(input, alpha=1.) -> Tensor + +In-place version of :func:`~celu`. +""", +) + + +def leaky_relu( + input: Tensor, + negative_slope: float = 0.01, + inplace: bool = False, +) -> Tensor: # noqa: D400,D402 + r""" + leaky_relu(input, negative_slope=0.01, inplace=False) -> Tensor + + Applies element-wise, + :math:`\text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x)` + + See :class:`~torch.nn.LeakyReLU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + leaky_relu, (input,), input, negative_slope=negative_slope, inplace=inplace + ) + if inplace: + result = torch._C._nn.leaky_relu_(input, negative_slope) + else: + result = torch._C._nn.leaky_relu(input, negative_slope) + return result + + +leaky_relu_ = _add_docstr( + torch._C._nn.leaky_relu_, + r""" +leaky_relu_(input, negative_slope=0.01) -> Tensor + +In-place version of :func:`~leaky_relu`. +""", +) + + +prelu = _add_docstr( + torch.prelu, + r"""prelu(input, weight) -> Tensor + +Applies element-wise the function +:math:`\text{PReLU}(x) = \max(0,x) + \text{weight} * \min(0,x)` where weight is a +learnable parameter. + +.. note:: + `weight` is expected to be a scalar or 1-D tensor. If `weight` is 1-D, + its size must match the number of input channels, determined by + `input.size(1)` when `input.dim() >= 2`, otherwise 1. + In the 1-D case, note that when `input` has dim > 2, `weight` can be expanded + to the shape of `input` in a way that is not possible using normal + :ref:`broadcasting semantics`. + +See :class:`~torch.nn.PReLU` for more details. +""", +) + + +def rrelu( + input: Tensor, + lower: float = 1.0 / 8, + upper: float = 1.0 / 3, + training: bool = False, + inplace: bool = False, +) -> Tensor: # noqa: D400,D402 + r"""rrelu(input, lower=1./8, upper=1./3, training=False, inplace=False) -> Tensor + + Randomized leaky ReLU. + + See :class:`~torch.nn.RReLU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + rrelu, + (input,), + input, + lower=lower, + upper=upper, + training=training, + inplace=inplace, + ) + if inplace: + result = torch.rrelu_(input, lower, upper, training) + else: + result = torch.rrelu(input, lower, upper, training) + return result + + +rrelu_ = _add_docstr( + torch.rrelu_, + r""" +rrelu_(input, lower=1./8, upper=1./3, training=False) -> Tensor + +In-place version of :func:`~rrelu`. +""", +) + +logsigmoid = _add_docstr( + torch._C._nn.log_sigmoid, + r""" +logsigmoid(input) -> Tensor + +Applies element-wise :math:`\text{LogSigmoid}(x_i) = \log \left(\frac{1}{1 + \exp(-x_i)}\right)` + +See :class:`~torch.nn.LogSigmoid` for more details. +""", +) + +gelu = _add_docstr( + torch._C._nn.gelu, + r""" +gelu(input, approximate = 'none') -> Tensor + +When the approximate argument is 'none', it applies element-wise the function +:math:`\text{GELU}(x) = x * \Phi(x)` + +where :math:`\Phi(x)` is the Cumulative Distribution Function for Gaussian Distribution. + +When the approximate argument is 'tanh', Gelu is estimated with + +.. math:: + \text{GELU}(x) = 0.5 * x * (1 + \text{Tanh}(\sqrt{2 / \pi} * (x + 0.044715 * x^3))) + +See `Gaussian Error Linear Units (GELUs) `_. +""", +) + +hardshrink = _add_docstr( + torch.hardshrink, + r""" +hardshrink(input, lambd=0.5) -> Tensor + +Applies the hard shrinkage function element-wise + +See :class:`~torch.nn.Hardshrink` for more details. +""", +) + + +def tanhshrink(input): # noqa: D400,D402 + r"""tanhshrink(input) -> Tensor + + Applies element-wise, :math:`\text{Tanhshrink}(x) = x - \text{Tanh}(x)` + + See :class:`~torch.nn.Tanhshrink` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(tanhshrink, (input,), input) + return input - input.tanh() + + +def softsign(input): # noqa: D400,D402 + r"""softsign(input) -> Tensor + + Applies element-wise, the function :math:`\text{SoftSign}(x) = \frac{x}{1 + |x|}` + + See :class:`~torch.nn.Softsign` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(softsign, (input,), input) + return input / (input.abs() + 1) + + +softplus = _add_docstr( + torch._C._nn.softplus, + r""" +softplus(input, beta=1, threshold=20) -> Tensor + +Applies element-wise, the function :math:`\text{Softplus}(x) = \frac{1}{\beta} * \log(1 + \exp(\beta * x))`. + +For numerical stability the implementation reverts to the linear function +when :math:`input \times \beta > threshold`. + +See :class:`~torch.nn.Softplus` for more details. +""", +) + + +def _get_softmax_dim(name: str, ndim: int, stacklevel: int) -> int: + warnings.warn( + f"Implicit dimension choice for {name} has been deprecated. " + "Change the call to include dim=X as an argument.", + stacklevel=stacklevel, + ) + if ndim == 0 or ndim == 1 or ndim == 3: + ret = 0 + else: + ret = 1 + return ret + + +def softmin( + input: Tensor, + dim: Optional[int] = None, + _stacklevel: int = 3, + dtype: Optional[DType] = None, +) -> Tensor: + r"""Apply a softmin function. + + Note that :math:`\text{Softmin}(x) = \text{Softmax}(-x)`. See softmax definition for mathematical formula. + + See :class:`~torch.nn.Softmin` for more details. + + Args: + input (Tensor): input + dim (int): A dimension along which softmin will be computed (so every slice + along dim will sum to 1). + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + """ + if has_torch_function_unary(input): + return handle_torch_function( + softmin, (input,), input, dim=dim, _stacklevel=_stacklevel, dtype=dtype + ) + if dim is None: + dim = _get_softmax_dim("softmin", input.dim(), _stacklevel) + if dtype is None: + ret = (-input).softmax(dim) + else: + ret = (-input).softmax(dim, dtype=dtype) + return ret + + +def softmax( + input: Tensor, + dim: Optional[int] = None, + _stacklevel: int = 3, + dtype: Optional[DType] = None, +) -> Tensor: + r"""Apply a softmax function. + + Softmax is defined as: + + :math:`\text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)}` + + It is applied to all slices along dim, and will re-scale them so that the elements + lie in the range `[0, 1]` and sum to 1. + + See :class:`~torch.nn.Softmax` for more details. + + Args: + input (Tensor): input + dim (int): A dimension along which softmax will be computed. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is casted to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + + .. note:: + This function doesn't work directly with NLLLoss, + which expects the Log to be computed between the Softmax and itself. + Use log_softmax instead (it's faster and has better numerical properties). + + """ + if has_torch_function_unary(input): + return handle_torch_function( + softmax, (input,), input, dim=dim, _stacklevel=_stacklevel, dtype=dtype + ) + if dim is None: + dim = _get_softmax_dim("softmax", input.dim(), _stacklevel) + if dtype is None: + ret = input.softmax(dim) + else: + ret = input.softmax(dim, dtype=dtype) + return ret + + +def gumbel_softmax( + logits: Tensor, + tau: float = 1, + hard: bool = False, + eps: float = 1e-10, + dim: int = -1, +) -> Tensor: + r""" + Sample from the Gumbel-Softmax distribution (`Link 1`_ `Link 2`_) and optionally discretize. + + Args: + logits: `[..., num_features]` unnormalized log probabilities + tau: non-negative scalar temperature + hard: if ``True``, the returned samples will be discretized as one-hot vectors, + but will be differentiated as if it is the soft sample in autograd + dim (int): A dimension along which softmax will be computed. Default: -1. + + Returns: + Sampled tensor of same shape as `logits` from the Gumbel-Softmax distribution. + If ``hard=True``, the returned samples will be one-hot, otherwise they will + be probability distributions that sum to 1 across `dim`. + + .. note:: + This function is here for legacy reasons, may be removed from nn.Functional in the future. + + .. note:: + The main trick for `hard` is to do `y_hard - y_soft.detach() + y_soft` + + It achieves two things: + - makes the output value exactly one-hot + (since we add then subtract y_soft value) + - makes the gradient equal to y_soft gradient + (since we strip all other gradients) + + Examples:: + >>> logits = torch.randn(20, 32) + >>> # Sample soft categorical using reparametrization trick: + >>> F.gumbel_softmax(logits, tau=1, hard=False) + >>> # Sample hard categorical using "Straight-through" trick: + >>> F.gumbel_softmax(logits, tau=1, hard=True) + + .. _Link 1: + https://arxiv.org/abs/1611.00712 + .. _Link 2: + https://arxiv.org/abs/1611.01144 + """ + if has_torch_function_unary(logits): + return handle_torch_function( + gumbel_softmax, (logits,), logits, tau=tau, hard=hard, eps=eps, dim=dim + ) + if eps != 1e-10: + warnings.warn("`eps` parameter is deprecated and has no effect.", stacklevel=2) + + gumbels = ( + -torch.empty_like(logits, memory_format=torch.legacy_contiguous_format) + .exponential_() + .log() + ) # ~Gumbel(0,1) + gumbels = (logits + gumbels) / tau # ~Gumbel(logits,tau) + y_soft = gumbels.softmax(dim) + + if hard: + # Straight through. + index = y_soft.max(dim, keepdim=True)[1] + y_hard = torch.zeros_like( + logits, memory_format=torch.legacy_contiguous_format + ).scatter_(dim, index, 1.0) + ret = y_hard - y_soft.detach() + y_soft + else: + # Reparameterization trick. + ret = y_soft + return ret + + +def log_softmax( + input: Tensor, + dim: Optional[int] = None, + _stacklevel: int = 3, + dtype: Optional[DType] = None, +) -> Tensor: + r"""Apply a softmax followed by a logarithm. + + While mathematically equivalent to log(softmax(x)), doing these two + operations separately is slower and numerically unstable. This function + uses an alternative formulation to compute the output and gradient correctly. + + See :class:`~torch.nn.LogSoftmax` for more details. + + Args: + input (Tensor): input + dim (int): A dimension along which log_softmax will be computed. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is cast to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + """ + if has_torch_function_unary(input): + return handle_torch_function( + log_softmax, (input,), input, dim=dim, _stacklevel=_stacklevel, dtype=dtype + ) + if dim is None: + dim = _get_softmax_dim("log_softmax", input.dim(), _stacklevel) + if dtype is None: + ret = input.log_softmax(dim) + else: + ret = input.log_softmax(dim, dtype=dtype) + return ret + + +softshrink = _add_docstr( + torch._C._nn.softshrink, + r""" +softshrink(input, lambd=0.5) -> Tensor + +Applies the soft shrinkage function elementwise + +See :class:`~torch.nn.Softshrink` for more details. +""", +) + + +def tanh(input): # noqa: D400,D402 + r"""tanh(input) -> Tensor + + Applies element-wise, + :math:`\text{Tanh}(x) = \tanh(x) = \frac{\exp(x) - \exp(-x)}{\exp(x) + \exp(-x)}` + + See :class:`~torch.nn.Tanh` for more details. + """ + return input.tanh() + + +def sigmoid(input): # noqa: D400,D402 + r"""sigmoid(input) -> Tensor + + Applies the element-wise function :math:`\text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)}` + + See :class:`~torch.nn.Sigmoid` for more details. + """ + return input.sigmoid() + + +def hardsigmoid(input: Tensor, inplace: bool = False) -> Tensor: + r"""Apply the Hardsigmoid function element-wise. + + .. math:: + \text{Hardsigmoid}(x) = \begin{cases} + 0 & \text{if~} x \le -3, \\ + 1 & \text{if~} x \ge +3, \\ + x / 6 + 1 / 2 & \text{otherwise} + \end{cases} + + Args: + inplace: If set to ``True``, will do this operation in-place. Default: ``False`` + + See :class:`~torch.nn.Hardsigmoid` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(hardsigmoid, (input,), input, inplace=inplace) + if inplace: + return torch._C._nn.hardsigmoid_(input) + return torch._C._nn.hardsigmoid(input) + + +linear = _add_docstr( + torch._C._nn.linear, + r""" +linear(input, weight, bias=None) -> Tensor + +Applies a linear transformation to the incoming data: :math:`y = xA^T + b`. + +This operation supports 2-D :attr:`weight` with :ref:`sparse layout` + +{sparse_beta_warning} + +This operator supports :ref:`TensorFloat32`. + +Shape: + + - Input: :math:`(*, in\_features)` where `*` means any number of + additional dimensions, including none + - Weight: :math:`(out\_features, in\_features)` or :math:`(in\_features)` + - Bias: :math:`(out\_features)` or :math:`()` + - Output: :math:`(*, out\_features)` or :math:`(*)`, based on the shape of the weight +""".format(**sparse_support_notes), +) + + +bilinear = _add_docstr( + torch.bilinear, + r""" +bilinear(input1, input2, weight, bias=None) -> Tensor + +Applies a bilinear transformation to the incoming data: +:math:`y = x_1^T A x_2 + b` + +Shape: + + - input1: :math:`(N, *, H_{in1})` where :math:`H_{in1}=\text{in1\_features}` + and :math:`*` means any number of additional dimensions. + All but the last dimension of the inputs should be the same. + - input2: :math:`(N, *, H_{in2})` where :math:`H_{in2}=\text{in2\_features}` + - weight: :math:`(\text{out\_features}, \text{in1\_features}, + \text{in2\_features})` + - bias: :math:`(\text{out\_features})` + - output: :math:`(N, *, H_{out})` where :math:`H_{out}=\text{out\_features}` + and all but the last dimension are the same shape as the input. +""", +) + + +def silu(input: Tensor, inplace: bool = False) -> Tensor: + r"""Apply the Sigmoid Linear Unit (SiLU) function, element-wise. + + The SiLU function is also known as the swish function. + + .. math:: + \text{silu}(x) = x * \sigma(x), \text{where } \sigma(x) \text{ is the logistic sigmoid.} + + .. note:: + See `Gaussian Error Linear Units (GELUs) `_ + where the SiLU (Sigmoid Linear Unit) was originally coined, and see + `Sigmoid-Weighted Linear Units for Neural Network Function Approximation + in Reinforcement Learning `_ and `Swish: + a Self-Gated Activation Function `_ + where the SiLU was experimented with later. + + See :class:`~torch.nn.SiLU` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(silu, (input,), input, inplace=inplace) + if inplace: + return torch._C._nn.silu_(input) + return torch._C._nn.silu(input) + + +def mish(input: Tensor, inplace: bool = False) -> Tensor: + r"""Apply the Mish function, element-wise. + + Mish: A Self Regularized Non-Monotonic Neural Activation Function. + + .. math:: + \text{Mish}(x) = x * \text{Tanh}(\text{Softplus}(x)) + + .. note:: + See `Mish: A Self Regularized Non-Monotonic Neural Activation Function `_ + + See :class:`~torch.nn.Mish` for more details. + """ + if has_torch_function_unary(input): + return handle_torch_function(mish, (input,), input, inplace=inplace) + if inplace: + return torch._C._nn.mish_(input) + return torch._C._nn.mish(input) + + +def hardswish(input: Tensor, inplace: bool = False) -> Tensor: + r"""Apply hardswish function, element-wise. + + Follows implementation as described in the paper: + `Searching for MobileNetV3`_. + + .. math:: + \text{Hardswish}(x) = \begin{cases} + 0 & \text{if~} x \le -3, \\ + x & \text{if~} x \ge +3, \\ + x \cdot (x + 3) /6 & \text{otherwise} + \end{cases} + + See :class:`~torch.nn.Hardswish` for more details. + + .. _`Searching for MobileNetV3`: + https://arxiv.org/abs/1905.02244 + """ + if has_torch_function_unary(input): + return handle_torch_function(hardswish, (input,), input, inplace=inplace) + if inplace: + return torch._C._nn.hardswish_(input) + return torch._C._nn.hardswish(input) + + +def _no_grad_embedding_renorm_( + weight: Tensor, + input: Tensor, + max_norm: float, + norm_type: float, + # pyrefly: ignore [bad-return] +) -> tuple[Tensor, Tensor]: + torch.embedding_renorm_(weight.detach(), input, max_norm, norm_type) + + +def embedding( + input: Tensor, + weight: Tensor, + padding_idx: Optional[int] = None, + max_norm: Optional[float] = None, + norm_type: float = 2.0, + scale_grad_by_freq: bool = False, + sparse: bool = False, +) -> Tensor: + r"""Generate a simple lookup table that looks up embeddings in a fixed dictionary and size. + + This module is often used to retrieve word embeddings using indices. + The input to the module is a list of indices, and the embedding matrix, + and the output is the corresponding word embeddings. + + See :class:`torch.nn.Embedding` for more details. + + .. note:: + Note that the analytical gradients of this function with respect to + entries in :attr:`weight` at the row specified by :attr:`padding_idx` + are expected to differ from the numerical ones. + + .. note:: + Note that `:class:`torch.nn.Embedding` differs from this function in + that it initializes the row of :attr:`weight` specified by + :attr:`padding_idx` to all zeros on construction. + + Args: + input (LongTensor): Tensor containing indices into the embedding matrix + weight (Tensor): The embedding matrix with number of rows equal to the maximum possible index + 1, + and number of columns equal to the embedding size + padding_idx (int, optional): If specified, the entries at :attr:`padding_idx` do not contribute to the gradient; + therefore, the embedding vector at :attr:`padding_idx` is not updated during training, + i.e. it remains as a fixed "pad". + max_norm (float, optional): If given, each embedding vector with norm larger than :attr:`max_norm` + is renormalized to have norm :attr:`max_norm`. + Note: this will modify :attr:`weight` in-place. + norm_type (float, optional): The p of the p-norm to compute for the :attr:`max_norm` option. Default ``2``. + scale_grad_by_freq (bool, optional): If given, this will scale gradients by the inverse of frequency of + the words in the mini-batch. Default ``False``. + sparse (bool, optional): If ``True``, gradient w.r.t. :attr:`weight` will be a sparse tensor. See Notes under + :class:`torch.nn.Embedding` for more details regarding sparse gradients. + + Shape: + - Input: LongTensor of arbitrary shape containing the indices to extract + - Weight: Embedding matrix of floating point type with shape `(V, embedding_dim)`, + where V = maximum index + 1 and embedding_dim = the embedding size + - Output: `(*, embedding_dim)`, where `*` is the input shape + + Examples:: + + >>> # a batch of 2 samples of 4 indices each + >>> input = torch.tensor([[1, 2, 4, 5], [4, 3, 2, 9]]) + >>> # an embedding matrix containing 10 tensors of size 3 + >>> embedding_matrix = torch.rand(10, 3) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> F.embedding(input, embedding_matrix) + tensor([[[ 0.8490, 0.9625, 0.6753], + [ 0.9666, 0.7761, 0.6108], + [ 0.6246, 0.9751, 0.3618], + [ 0.4161, 0.2419, 0.7383]], + + [[ 0.6246, 0.9751, 0.3618], + [ 0.0237, 0.7794, 0.0528], + [ 0.9666, 0.7761, 0.6108], + [ 0.3385, 0.8612, 0.1867]]]) + + >>> # example with padding_idx + >>> weights = torch.rand(10, 3) + >>> weights[0, :].zero_() + >>> embedding_matrix = weights + >>> input = torch.tensor([[0, 2, 0, 5]]) + >>> F.embedding(input, embedding_matrix, padding_idx=0) + tensor([[[ 0.0000, 0.0000, 0.0000], + [ 0.5609, 0.5384, 0.8720], + [ 0.0000, 0.0000, 0.0000], + [ 0.6262, 0.2438, 0.7471]]]) + """ + if has_torch_function_variadic(input, weight): + return handle_torch_function( + embedding, + (input, weight), + input, + weight, + padding_idx=padding_idx, + max_norm=max_norm, + norm_type=norm_type, + scale_grad_by_freq=scale_grad_by_freq, + sparse=sparse, + ) + if padding_idx is not None: + if padding_idx > 0: + assert padding_idx < weight.size(0), ( + "Padding_idx must be within num_embeddings" + ) + elif padding_idx < 0: + assert padding_idx >= -weight.size(0), ( + "Padding_idx must be within num_embeddings" + ) + padding_idx = weight.size(0) + padding_idx + else: + padding_idx = -1 + if max_norm is not None: + # Note [embedding_renorm contiguous] + # `embedding_renorm_` will call .contiguous() on input anyways, so we + # call it here and take advantage of the improved locality in the + # `embedding` call below too. + input = input.contiguous() + # Note [embedding_renorm set_grad_enabled] + # XXX: equivalent to + # with torch.no_grad(): + # torch.embedding_renorm_ + # remove once script supports set_grad_enabled + _no_grad_embedding_renorm_(weight, input, max_norm, norm_type) + return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse) + + +def embedding_bag( + input: Tensor, + weight: Tensor, + offsets: Optional[Tensor] = None, + max_norm: Optional[float] = None, + norm_type: float = 2, + scale_grad_by_freq: bool = False, + mode: str = "mean", + sparse: bool = False, + per_sample_weights: Optional[Tensor] = None, + include_last_offset: bool = False, + padding_idx: Optional[int] = None, +) -> Tensor: + r"""Compute sums, means or maxes of `bags` of embeddings. + + Calculation is done without instantiating the intermediate embeddings. + See :class:`torch.nn.EmbeddingBag` for more details. + + Note: + {backward_reproducibility_note} + + Args: + input (LongTensor): Tensor containing bags of indices into the embedding matrix + weight (Tensor): The embedding matrix with number of rows equal to the maximum possible index + 1, + and number of columns equal to the embedding size + offsets (LongTensor, optional): Only used when :attr:`input` is 1D. :attr:`offsets` determines + the starting index position of each bag (sequence) in :attr:`input`. + max_norm (float, optional): If given, each embedding vector with norm larger than :attr:`max_norm` + is renormalized to have norm :attr:`max_norm`. + Note: this will modify :attr:`weight` in-place. + norm_type (float, optional): The ``p`` in the ``p``-norm to compute for the :attr:`max_norm` option. + Default ``2``. + scale_grad_by_freq (bool, optional): if given, this will scale gradients by the inverse of frequency of + the words in the mini-batch. Default ``False``. + Note: this option is not supported when ``mode="max"``. + mode (str, optional): ``"sum"``, ``"mean"`` or ``"max"``. Specifies the way to reduce the bag. + Default: ``"mean"`` + sparse (bool, optional): if ``True``, gradient w.r.t. :attr:`weight` will be a sparse tensor. See Notes under + :class:`torch.nn.Embedding` for more details regarding sparse gradients. + Note: this option is not supported when ``mode="max"``. + per_sample_weights (Tensor, optional): a tensor of float / double weights, or None + to indicate all weights should be taken to be 1. If specified, :attr:`per_sample_weights` + must have exactly the same shape as input and is treated as having the same + :attr:`offsets`, if those are not None. + + include_last_offset (bool, optional): if ``True``, the size of offsets is equal to the number of bags + 1. + The last element is the size of the input, or the ending index position + of the last bag (sequence). This matches the CSR format. Ignored when + input is 2D. Default ``False``. + + padding_idx (int, optional): If specified, the entries at :attr:`padding_idx` do not contribute to the + gradient; therefore, the embedding vector at :attr:`padding_idx` is not updated + during training, i.e. it remains as a fixed "pad". Note that the embedding + vector at :attr:`padding_idx` is excluded from the reduction. + + Shape: + - :attr:`input` (LongTensor) and :attr:`offsets` (LongTensor, optional) + + - If :attr:`input` is 2D of shape `(B, N)`, it will be treated as ``B`` bags (sequences) + each of fixed length ``N``, and this will return ``B`` values aggregated in a way + depending on the :attr:`mode`. :attr:`offsets` is ignored and required to be ``None`` in this case. + + - If :attr:`input` is 1D of shape `(N)`, it will be treated as a concatenation of + multiple bags (sequences). :attr:`offsets` is required to be a 1D tensor containing + the starting index positions of each bag in :attr:`input`. Therefore, for :attr:`offsets` + of shape `(B)`, :attr:`input` will be viewed as having ``B`` bags. + Empty bags (i.e., having 0-length) will have returned vectors filled by zeros. + + - :attr:`weight` (Tensor): the learnable weights of the module of shape `(num_embeddings, embedding_dim)` + + - :attr:`per_sample_weights` (Tensor, optional). Has the same shape as :attr:`input`. + + - :attr:`output`: aggregated embedding values of shape `(B, embedding_dim)` + + Examples:: + + >>> # an Embedding module containing 10 tensors of size 3 + >>> embedding_matrix = torch.rand(10, 3) + >>> # a batch of 2 samples of 4 indices each + >>> input = torch.tensor([1, 2, 4, 5, 4, 3, 2, 9]) + >>> offsets = torch.tensor([0, 4]) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> F.embedding_bag(input, embedding_matrix, offsets) + tensor([[ 0.3397, 0.3552, 0.5545], + [ 0.5893, 0.4386, 0.5882]]) + + >>> # example with padding_idx + >>> embedding_matrix = torch.rand(10, 3) + >>> input = torch.tensor([2, 2, 2, 2, 4, 3, 2, 9]) + >>> offsets = torch.tensor([0, 4]) + >>> F.embedding_bag(input, embedding_matrix, offsets, padding_idx=2, mode='sum') + tensor([[ 0.0000, 0.0000, 0.0000], + [-0.7082, 3.2145, -2.6251]]) + """ + if has_torch_function_variadic(input, weight, offsets, per_sample_weights): + return handle_torch_function( + embedding_bag, + (input, weight, offsets, per_sample_weights), + input, + weight, + offsets=offsets, + max_norm=max_norm, + norm_type=norm_type, + scale_grad_by_freq=scale_grad_by_freq, + mode=mode, + sparse=sparse, + per_sample_weights=per_sample_weights, + include_last_offset=include_last_offset, + padding_idx=padding_idx, + ) + # Check for backward compatibility. + # Used to be embedding_bag(weight, input, ...) + # Now is embedding_bag(input, weight, ...) + if weight.dtype == torch.long and input.is_floating_point(): + warnings.warn( + "Argument order of nn.functional.embedding_bag was changed. " + "Usage `embedding_bag(weight, input, ...)` is deprecated, " + "and should now be `embedding_bag(input, weight, ...)`.", + stacklevel=2, + ) + weight, input = input, weight + + if per_sample_weights is not None and input.size() != per_sample_weights.size(): + raise ValueError( + f"embedding_bag: If per_sample_weights ({per_sample_weights.shape}) is not None, " + f"then it must have the same shape as the input ({input.shape})" + ) + + if not weight.dim() == 2: + raise ValueError( + f"weight has to be a 2D Tensor, but got Tensor of dimension {weight.dim()}" + ) + + if not torch.jit.is_scripting() and input.dim() == 2 and input.is_nested: + include_last_offset = True + # pyrefly: ignore [missing-attribute] + offsets = input.offsets() + input = input.values().reshape(-1) + if per_sample_weights is not None: + if not per_sample_weights.is_nested: + raise ValueError( + "If input is nested, then per_sample_weights must be nested if specified" + ) + per_sample_weights = per_sample_weights.values().reshape(-1) + elif input.dim() == 2: + if offsets is not None: + type_str = "" + # TODO: Remove this once script supports type() calls + if not torch.jit.is_scripting(): + type_str = str(type(offsets)) + raise ValueError( + "if input is 2D, then offsets has to be None" + ", as input is treated is a mini-batch of" + " fixed length sequences. However, found " + f"offsets of type {type_str}" + ) + offsets = torch.arange( + 0, input.numel(), input.size(1), dtype=input.dtype, device=input.device + ) + include_last_offset = False + input = input.reshape(-1) + if per_sample_weights is not None: + per_sample_weights = per_sample_weights.reshape(-1) + elif input.dim() == 1: + if offsets is None: + raise ValueError("offsets has to be a 1D Tensor but got None") + if offsets.dim() != 1: + raise ValueError("offsets has to be a 1D Tensor") + else: + raise ValueError( + f"input has to be 1D or 2D Tensor, but got Tensor of dimension {input.dim()}" + ) + if mode == "sum": + mode_enum = 0 + elif mode == "mean": + mode_enum = 1 + elif mode == "max": + mode_enum = 2 + + if scale_grad_by_freq: + raise ValueError( + "max mode does not support scaling the gradient by the frequency" + ) + + if sparse: + raise ValueError("max mode does not support sparse weights") + + else: + raise ValueError("mode has to be one of sum, mean or max") + + if max_norm is not None: + # XXX: equivalent to + # with torch.no_grad(): + # torch.nembedding_renorm_ + # remove once script supports set_grad_enabled + _no_grad_embedding_renorm_(weight, input, max_norm, norm_type) + + if per_sample_weights is not None and mode != "sum": + raise NotImplementedError( + "embedding_bag: per_sample_weights was not None. " + "per_sample_weights is only supported for mode='sum' " + f"(got mode='{mode}'). Please open a feature request on GitHub." + ) + + ret, _, _, _ = torch.embedding_bag( + weight, + input, + offsets, + scale_grad_by_freq, + mode_enum, + sparse, + per_sample_weights, + include_last_offset, + padding_idx, + ) + return ret + + +if embedding_bag.__doc__: + embedding_bag.__doc__ = embedding_bag.__doc__.format(**reproducibility_notes) + + +def _verify_batch_size(size: list[int]) -> None: + # XXX: JIT script does not support the reduce from functools, and mul op is a + # builtin, which cannot be used as a value to a func yet, so rewrite this size + # check to a simple equivalent for loop + # + # TODO: make use of reduce like below when JIT is ready with the missing features: + # from operator import mul + # from functools import reduce + # + # if reduce(mul, size[2:], size[0]) == 1 + size_prods = size[0] + for i in range(len(size) - 2): + size_prods *= size[i + 2] + if size_prods == 1: + raise ValueError( + f"Expected more than 1 value per channel when training, got input size {size}" + ) + + +def batch_norm( + input: Tensor, + running_mean: Optional[Tensor], + running_var: Optional[Tensor], + weight: Optional[Tensor] = None, + bias: Optional[Tensor] = None, + training: bool = False, + momentum: float = 0.1, + eps: float = 1e-5, +) -> Tensor: + r"""Apply Batch Normalization for each channel across a batch of data. + + See :class:`~torch.nn.BatchNorm1d`, :class:`~torch.nn.BatchNorm2d`, + :class:`~torch.nn.BatchNorm3d` for details. + """ + if has_torch_function_variadic(input, running_mean, running_var, weight, bias): + return handle_torch_function( + batch_norm, + (input, running_mean, running_var, weight, bias), + input, + running_mean, + running_var, + weight=weight, + bias=bias, + training=training, + momentum=momentum, + eps=eps, + ) + if training: + # pyrefly: ignore [bad-argument-type] + _verify_batch_size(input.size()) + + if eps <= 0.0: + raise ValueError(f"batch_norm eps must be positive, but got {eps}") + + return torch.batch_norm( + input, + weight, + bias, + running_mean, + running_var, + training, + momentum, + eps, + torch.backends.cudnn.enabled, + ) + + +def _verify_spatial_size(size: list[int]) -> None: + # Verify that there is > 1 spatial element for instance norm calculation. + size_prods = 1 + for i in range(2, len(size)): + size_prods *= size[i] + if size_prods == 1: + raise ValueError( + f"Expected more than 1 spatial element when training, got input size {size}" + ) + + +def instance_norm( + input: Tensor, + running_mean: Optional[Tensor] = None, + running_var: Optional[Tensor] = None, + weight: Optional[Tensor] = None, + bias: Optional[Tensor] = None, + use_input_stats: bool = True, + momentum: float = 0.1, + eps: float = 1e-5, +) -> Tensor: + r"""Apply Instance Normalization independently for each channel in every data sample within a batch. + + See :class:`~torch.nn.InstanceNorm1d`, :class:`~torch.nn.InstanceNorm2d`, + :class:`~torch.nn.InstanceNorm3d` for details. + """ + if has_torch_function_variadic(input, running_mean, running_var, weight, bias): + return handle_torch_function( + instance_norm, + (input, running_mean, running_var, weight, bias), + input, + running_mean=running_mean, + running_var=running_var, + weight=weight, + bias=bias, + use_input_stats=use_input_stats, + momentum=momentum, + eps=eps, + ) + if use_input_stats: + # pyrefly: ignore [bad-argument-type] + _verify_spatial_size(input.size()) + return torch.instance_norm( + input, + weight, + bias, + running_mean, + running_var, + use_input_stats, + momentum, + eps, + torch.backends.cudnn.enabled, + ) + + +def layer_norm( + input: Tensor, + normalized_shape: list[int], + weight: Optional[Tensor] = None, + bias: Optional[Tensor] = None, + eps: float = 1e-5, +) -> Tensor: + r"""Apply Layer Normalization for last certain number of dimensions. + + See :class:`~torch.nn.LayerNorm` for details. + """ + if has_torch_function_variadic(input, weight, bias): + return handle_torch_function( + layer_norm, + (input, weight, bias), + input, + normalized_shape, + weight=weight, + bias=bias, + eps=eps, + ) + return torch.layer_norm( + input, normalized_shape, weight, bias, eps, torch.backends.cudnn.enabled + ) + + +def rms_norm( + input: Tensor, + normalized_shape: list[int], + weight: Optional[Tensor] = None, + eps: Optional[float] = None, +) -> Tensor: + r"""Apply Root Mean Square Layer Normalization. + + See :class:`~torch.nn.RMSNorm` for details. + """ + if has_torch_function_variadic(input, weight): + return handle_torch_function( + rms_norm, (input, weight), input, normalized_shape, weight=weight, eps=eps + ) + return torch.rms_norm(input, normalized_shape, weight, eps) + + +def group_norm( + input: Tensor, + num_groups: int, + weight: Optional[Tensor] = None, + bias: Optional[Tensor] = None, + eps: float = 1e-5, +) -> Tensor: + r"""Apply Group Normalization for last certain number of dimensions. + + See :class:`~torch.nn.GroupNorm` for details. + """ + if has_torch_function_variadic(input, weight, bias): + return handle_torch_function( + group_norm, + ( + input, + weight, + bias, + ), + input, + num_groups, + weight=weight, + bias=bias, + eps=eps, + ) + if input.dim() < 2: + raise RuntimeError( + f"Expected at least 2 dimensions for input tensor but received {input.dim()}" + ) + _verify_batch_size( + [input.size(0) * input.size(1) // num_groups, num_groups] + + list(input.size()[2:]) + ) + return torch.group_norm( + input, num_groups, weight, bias, eps, torch.backends.cudnn.enabled + ) + + +def local_response_norm( + input: Tensor, + size: int, + alpha: float = 1e-4, + beta: float = 0.75, + k: float = 1.0, +) -> Tensor: + r"""Apply local response normalization over an input signal. + + The input signal is composed of several input planes, where channels occupy the second dimension. + Normalization is applied across channels. + + See :class:`~torch.nn.LocalResponseNorm` for details. + """ + if has_torch_function_unary(input): + return handle_torch_function( + local_response_norm, (input,), input, size, alpha=alpha, beta=beta, k=k + ) + dim = input.dim() + if dim < 3: + raise ValueError( + f"Expected 3D or higher dimensionality input (got {dim} dimensions)" + ) + + if input.numel() == 0: + return input + + div = input.mul(input) + if dim == 3: + div = div.unsqueeze(1) + # pyrefly: ignore [bad-argument-type] + div = pad(div, (0, 0, size // 2, (size - 1) // 2)) + div = avg_pool2d(div, (size, 1), stride=1).squeeze(1) + else: + sizes = input.size() + div = div.view(sizes[0], 1, sizes[1], sizes[2], -1) + # pyrefly: ignore [bad-argument-type] + div = pad(div, (0, 0, 0, 0, size // 2, (size - 1) // 2)) + div = avg_pool3d(div, (size, 1, 1), stride=1).squeeze(1) + div = div.view(sizes) + div = div.mul(alpha).add(k).pow(beta) + return input / div + + +# loss + + +def ctc_loss( + log_probs: Tensor, + targets: Tensor, + input_lengths: Tensor, + target_lengths: Tensor, + blank: int = 0, + reduction: str = "mean", + zero_infinity: bool = False, +) -> Tensor: + r"""Compute the Connectionist Temporal Classification loss. + + See :class:`~torch.nn.CTCLoss` for details. + + Note: + {cudnn_reproducibility_note} + + Note: + {backward_reproducibility_note} + + Args: + log_probs: :math:`(T, N, C)` or :math:`(T, C)` where `C = number of characters in alphabet including blank`, + `T = input length`, and `N = batch size`. + The logarithmized probabilities of the outputs + (e.g. obtained with :func:`torch.nn.functional.log_softmax`). + targets: :math:`(N, S)` or `(sum(target_lengths))`. + May be an empty tensor if all entries in `target_lengths` are zero. + In the second form, the targets are assumed to be concatenated. + input_lengths: :math:`(N)` or :math:`()`. + Lengths of the inputs (must each be :math:`\leq T`) + target_lengths: :math:`(N)` or :math:`()`. + Lengths of the targets + blank (int, optional): + Blank label. Default :math:`0`. + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the output losses will be divided by the target lengths and + then the mean over the batch is taken, ``'sum'``: the output will be + summed. Default: ``'mean'`` + zero_infinity (bool, optional): + Whether to zero infinite losses and the associated gradients. + Default: ``False`` + Infinite losses mainly occur when the inputs are too short + to be aligned to the targets. + + Example:: + + >>> log_probs = torch.randn(50, 16, 20).log_softmax(2).detach().requires_grad_() + >>> targets = torch.randint(1, 20, (16, 30), dtype=torch.long) + >>> input_lengths = torch.full((16,), 50, dtype=torch.long) + >>> target_lengths = torch.randint(10, 30, (16,), dtype=torch.long) + >>> loss = F.ctc_loss(log_probs, targets, input_lengths, target_lengths) + >>> loss.backward() + """ + if has_torch_function_variadic(log_probs, targets, input_lengths, target_lengths): + return handle_torch_function( + ctc_loss, + (log_probs, targets, input_lengths, target_lengths), + log_probs, + targets, + input_lengths, + target_lengths, + blank=blank, + reduction=reduction, + zero_infinity=zero_infinity, + ) + return torch.ctc_loss( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + _Reduction.get_enum(reduction), + zero_infinity, + ) + + +if ctc_loss.__doc__: + ctc_loss.__doc__ = ctc_loss.__doc__.format(**reproducibility_notes) + + +def nll_loss( + input: Tensor, + target: Tensor, + weight: Optional[Tensor] = None, + size_average: Optional[bool] = None, + ignore_index: int = -100, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: + r"""Compute the negative log likelihood loss. + + See :class:`~torch.nn.NLLLoss` for details. + + Args: + input: :math:`(N, C)` where `C = number of classes` or :math:`(N, C, H, W)` + in case of 2D Loss, or :math:`(N, C, d_1, d_2, ..., d_K)` where :math:`K \geq 1` + in the case of K-dimensional loss. `input` is expected to be log-probabilities. + target: :math:`(N)` where each value is :math:`0 \leq \text{targets}[i] \leq C-1`, + or :math:`(N, d_1, d_2, ..., d_K)` where :math:`K \geq 1` for + K-dimensional loss. + weight (Tensor, optional): A manual rescaling weight given to each + class. If given, has to be a Tensor of size `C` + size_average (bool, optional): Deprecated (see :attr:`reduction`). + ignore_index (int, optional): Specifies a target value that is ignored + and does not contribute to the input gradient. When :attr:`size_average` is + ``True``, the loss is averaged over non-ignored targets. Default: -100 + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of + elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` + and :attr:`reduce` are in the process of being deprecated, and in the meantime, + specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` + + Example:: + + >>> # input is of size N x C = 3 x 5 + >>> input = torch.randn(3, 5, requires_grad=True) + >>> # each element in target has to have 0 <= value < C + >>> target = torch.tensor([1, 0, 4]) + >>> output = F.nll_loss(F.log_softmax(input, dim=1), target) + >>> output.backward() + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + nll_loss, + (input, target, weight), + input, + target, + weight=weight, + size_average=size_average, + ignore_index=ignore_index, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + return torch._C._nn.nll_loss_nd( + input, + target, + weight, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + ignore_index, + ) + + +def poisson_nll_loss( + input: Tensor, + target: Tensor, + log_input: bool = True, + full: bool = False, + size_average: Optional[bool] = None, + eps: float = 1e-8, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: + r"""Compute the Poisson negative log likelihood loss. + + See :class:`~torch.nn.PoissonNLLLoss` for details. + + Args: + input: Expectation of underlying Poisson distribution. + target: Random sample :math:`target \sim \text{Poisson}(input)`. + log_input: If ``True`` the loss is computed as + :math:`\exp(\text{input}) - \text{target} * \text{input}`, if ``False`` then loss is + :math:`\text{input} - \text{target} * \log(\text{input}+\text{eps})`. Default: ``True`` + full: Whether to compute full loss, i. e. to add the Stirling + approximation term. Default: ``False`` + :math:`\text{target} * \log(\text{target}) - \text{target} + 0.5 * \log(2 * \pi * \text{target})`. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + eps (float, optional): Small value to avoid evaluation of :math:`\log(0)` when + :attr:`log_input`\ =\ ``False``. Default: 1e-8 + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of + elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` + and :attr:`reduce` are in the process of being deprecated, and in the meantime, + specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` + + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + poisson_nll_loss, + (input, target), + input, + target, + log_input=log_input, + full=full, + size_average=size_average, + eps=eps, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + if reduction != "none" and reduction != "mean" and reduction != "sum": + ret = input + raise ValueError(reduction + " is not a valid value for reduction") + + ret = torch.poisson_nll_loss( + input, target, log_input, full, eps, _Reduction.get_enum(reduction) + ) + return ret + + +def gaussian_nll_loss( + input: Tensor, + target: Tensor, + var: Tensor | float, + full: bool = False, + eps: float = 1e-6, + reduction: str = "mean", +) -> Tensor: + r"""Compute the Gaussian negative log likelihood loss. + + See :class:`~torch.nn.GaussianNLLLoss` for details. + + Args: + input: Expectation of the Gaussian distribution. + target: Sample from the Gaussian distribution. + var: Tensor of positive variance(s), one for each of the expectations + in the input (heteroscedastic), or a single one (homoscedastic), + or a positive scalar value to be used for all expectations. + full (bool, optional): Whether to include the constant term in the loss calculation. Default: ``False``. + eps (float, optional): Value added to var, for stability. Default: 1e-6. + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the output is the average of all batch member losses, + ``'sum'``: the output is the sum of all batch member losses. + Default: ``'mean'``. + """ + if has_torch_function_variadic(input, target, var): + return handle_torch_function( + gaussian_nll_loss, + (input, target, var), + input, + target, + var, + full=full, + eps=eps, + reduction=reduction, + ) + + # Entries of var must be non-negative + if isinstance(var, float): + if var < 0: + raise ValueError("var has negative entry/entries") + var = var * torch.ones_like(input) + elif torch.any(var < 0): + raise ValueError("var has negative entry/entries") + + # Check var size + # If var.size == input.size, the case is heteroscedastic and no further checks are needed. + # Otherwise: + if var.size() != input.size(): + # If var is one dimension short of input, but the sizes match otherwise, then this is a homoscedastic case. + # e.g. input.size = (10, 2, 3), var.size = (10, 2) + # -> unsqueeze var so that var.shape = (10, 2, 1) + # this is done so that broadcasting can happen in the loss calculation + if input.size()[:-1] == var.size(): + var = torch.unsqueeze(var, -1) + + # This checks if the var is broadcastable to the input and there is only one mismatched dimension. + # This is also a homoscedastic case. + # e.g. input.size = (10, 2, 3), var.size = (10, 2, 1) + # or input.size = (4, 3, 32, 32), var.size = (4, 1, 32, 32) + elif ( + input.ndim == var.ndim + and sum(y for x, y in zip(input.size(), var.size(), strict=True) if x != y) + == 1 + ): # Heteroscedastic case + pass + + # If none of the above pass, then the size of var is incorrect. + else: + raise ValueError("var is of incorrect size") + + # Check validity of reduction mode + if reduction != "none" and reduction != "mean" and reduction != "sum": + raise ValueError(reduction + " is not valid") + + # Clamp for stability + var = var.clone() + with torch.no_grad(): + var.clamp_(min=eps) + + # Calculate the loss + # pyrefly: ignore [unsupported-operation] + loss = 0.5 * (torch.log(var) + (input - target) ** 2 / var) + if full: + loss += 0.5 * math.log(2 * math.pi) + + if reduction == "mean": + return loss.mean() + elif reduction == "sum": + return loss.sum() + else: + return loss + + +def kl_div( + input: Tensor, + target: Tensor, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", + log_target: bool = False, +) -> Tensor: + r"""Compute the KL Divergence loss. + + Refer - The `Kullback-Leibler divergence Loss + `__ + + See :class:`~torch.nn.KLDivLoss` for details. + + Args: + input: Tensor of arbitrary shape in log-probabilities. + target: Tensor of the same shape as input. See :attr:`log_target` for + the target's interpretation. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'batchmean'`` | ``'sum'`` | ``'mean'``. + ``'none'``: no reduction will be applied + ``'batchmean'``: the sum of the output will be divided by the batchsize + ``'sum'``: the output will be summed + ``'mean'``: the output will be divided by the number of elements in the output + Default: ``'mean'`` + log_target (bool): A flag indicating whether ``target`` is passed in the log space. + It is recommended to pass certain distributions (like ``softmax``) + in the log space to avoid numerical issues caused by explicit ``log``. + Default: ``False`` + + .. note:: + :attr:`size_average` and :attr:`reduce` are in the process of being deprecated, + and in the meantime, specifying either of those two args will override :attr:`reduction`. + + .. warning:: + :attr:`reduction` = ``'mean'`` doesn't return the true kl divergence value, please use + :attr:`reduction` = ``'batchmean'`` which aligns with KL math definition. + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + kl_div, + (input, target), + input, + target, + size_average=size_average, + reduce=reduce, + reduction=reduction, + log_target=log_target, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + if reduction == "mean": + warnings.warn( + "reduction: 'mean' divides the total loss by both the batch size and the support size." + "'batchmean' divides only by the batch size, and aligns with the KL div math definition." + "'mean' will be changed to behave the same as 'batchmean' in the next major release.", + stacklevel=2, + ) + + # special case for batchmean + if reduction == "batchmean": + reduction_enum = _Reduction.get_enum("sum") + else: + reduction_enum = _Reduction.get_enum(reduction) + + reduced = torch.kl_div(input, target, reduction_enum, log_target=log_target) + + if reduction == "batchmean" and input.dim() != 0: + reduced = reduced / input.size()[0] + + return reduced + + +def cross_entropy( + input: Tensor, + target: Tensor, + weight: Optional[Tensor] = None, + size_average: Optional[bool] = None, + ignore_index: int = -100, + reduce: Optional[bool] = None, + reduction: str = "mean", + label_smoothing: float = 0.0, +) -> Tensor: + r"""Compute the cross entropy loss between input logits and target. + + See :class:`~torch.nn.CrossEntropyLoss` for details. + + Args: + input (Tensor) : Predicted unnormalized logits; + see Shape section below for supported shapes. + target (Tensor) : Ground truth class indices or class probabilities; + see Shape section below for supported shapes. + weight (Tensor, optional): a manual rescaling weight given to each + class. If given, has to be a Tensor of size `C` + size_average (bool, optional): Deprecated (see :attr:`reduction`). + ignore_index (int, optional): Specifies a target value that is ignored + and does not contribute to the input gradient. When :attr:`size_average` is + ``True``, the loss is averaged over non-ignored targets. Note that + :attr:`ignore_index` is only applicable when the target contains class indices. + Default: -100 + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of + elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` + and :attr:`reduce` are in the process of being deprecated, and in the meantime, + specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` + label_smoothing (float, optional): A float in [0.0, 1.0]. Specifies the amount + of smoothing when computing the loss, where 0.0 means no smoothing. The targets + become a mixture of the original ground truth and a uniform distribution as described in + `Rethinking the Inception Architecture for Computer Vision `__. Default: :math:`0.0`. + + Shape: + - Input: Shape :math:`(C)`, :math:`(N, C)` or :math:`(N, C, d_1, d_2, ..., d_K)` with :math:`K \geq 1` + in the case of `K`-dimensional loss. + - Target: If containing class indices, shape :math:`()`, :math:`(N)` or :math:`(N, d_1, d_2, ..., d_K)` with + :math:`K \geq 1` in the case of K-dimensional loss where each value should be between :math:`[0, C)`. + If containing class probabilities, same shape as the input and each value should be between :math:`[0, 1]`. + + where: + + .. math:: + \begin{aligned} + C ={} & \text{number of classes} \\ + N ={} & \text{batch size} \\ + \end{aligned} + + Examples:: + + >>> # Example of target with class indices + >>> input = torch.randn(3, 5, requires_grad=True) + >>> target = torch.randint(5, (3,), dtype=torch.int64) + >>> loss = F.cross_entropy(input, target) + >>> loss.backward() + >>> + >>> # Example of target with class probabilities + >>> input = torch.randn(3, 5, requires_grad=True) + >>> target = torch.randn(3, 5).softmax(dim=1) + >>> loss = F.cross_entropy(input, target) + >>> loss.backward() + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + cross_entropy, + (input, target, weight), + input, + target, + weight=weight, + size_average=size_average, + ignore_index=ignore_index, + reduce=reduce, + reduction=reduction, + label_smoothing=label_smoothing, + ) + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + return torch._C._nn.cross_entropy_loss( + input, + target, + weight, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + ignore_index, + label_smoothing, + ) + + +def binary_cross_entropy( + input: Tensor, + target: Tensor, + weight: Optional[Tensor] = None, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: + r"""Compute Binary Cross Entropy between the target and input probabilities. + + See :class:`~torch.nn.BCELoss` for details. + + Args: + input: Tensor of arbitrary shape as probabilities. + target: Tensor of the same shape as input with values between 0 and 1. + weight (Tensor, optional): a manual rescaling weight + if provided it's repeated to match input tensor shape + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of + elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` + and :attr:`reduce` are in the process of being deprecated, and in the meantime, + specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` + + Examples:: + + >>> input = torch.randn(3, 2, requires_grad=True) + >>> target = torch.rand(3, 2, requires_grad=False) + >>> loss = F.binary_cross_entropy(torch.sigmoid(input), target) + >>> loss.backward() + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + binary_cross_entropy, + (input, target, weight), + input, + target, + weight=weight, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + if target.size() != input.size(): + raise ValueError( + f"Using a target size ({target.size()}) that is different to the input size ({input.size()}) is deprecated. " + "Please ensure they have the same size." + ) + + if weight is not None: + new_size = _infer_size(target.size(), weight.size()) + weight = weight.expand(new_size) + + # pyrefly: ignore [bad-argument-type] + return torch._C._nn.binary_cross_entropy(input, target, weight, reduction_enum) + + +def binary_cross_entropy_with_logits( + input: Tensor, + target: Tensor, + weight: Optional[Tensor] = None, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", + pos_weight: Optional[Tensor] = None, +) -> Tensor: + r"""Compute Binary Cross Entropy between target and input logits. + + See :class:`~torch.nn.BCEWithLogitsLoss` for details. + + Args: + input: Tensor of arbitrary shape as unnormalized scores (often referred to as logits). + target: Tensor of the same shape as input with values between 0 and 1 + weight (Tensor, optional): a manual rescaling weight + if provided it's repeated to match input tensor shape + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, + ``'mean'``: the sum of the output will be divided by the number of + elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` + and :attr:`reduce` are in the process of being deprecated, and in the meantime, + specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` + pos_weight (Tensor, optional): a weight of positive examples to be broadcasted with target. + Must be a tensor with equal size along the class dimension to the number of classes. + Pay close attention to PyTorch's broadcasting semantics in order to achieve the desired + operations. For a target of size [B, C, H, W] (where B is batch size) pos_weight of + size [B, C, H, W] will apply different pos_weights to each element of the batch or + [C, H, W] the same pos_weights across the batch. To apply the same positive weight + along all spatial dimensions for a 2D multi-class target [C, H, W] use: [C, 1, 1]. + Default: ``None`` + + Examples:: + + >>> input = torch.randn(3, requires_grad=True) + >>> target = torch.empty(3).random_(2) + >>> loss = F.binary_cross_entropy_with_logits(input, target) + >>> loss.backward() + """ + if has_torch_function_variadic(input, target, weight, pos_weight): + return handle_torch_function( + binary_cross_entropy_with_logits, + (input, target, weight, pos_weight), + input, + target, + weight=weight, + size_average=size_average, + reduce=reduce, + reduction=reduction, + pos_weight=pos_weight, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + + if not (target.size() == input.size()): + raise ValueError( + f"Target size ({target.size()}) must be the same as input size ({input.size()})" + ) + + return torch.binary_cross_entropy_with_logits( + input, target, weight, pos_weight, reduction_enum + ) + + +def smooth_l1_loss( + input: Tensor, + target: Tensor, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", + beta: float = 1.0, +) -> Tensor: + r"""Compute the Smooth L1 loss. + + Function uses a squared term if the absolute + element-wise error falls below beta and an L1 term otherwise. + + See :class:`~torch.nn.SmoothL1Loss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + beta (float, optional): Specifies the threshold at which to change from the squared + term to the L1 term in the loss calculation. This value must be positive. + Default: 1.0. + + Returns: + Tensor: L1 loss (optionally weighted). + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + smooth_l1_loss, + (input, target), + input, + target, + size_average=size_average, + reduce=reduce, + reduction=reduction, + beta=beta, + ) + if target.size() != input.size(): + warnings.warn( + f"Using a target size ({target.size()}) that is different to the input size ({input.size()}). " + "This will likely lead to incorrect results due to broadcasting. " + "Please ensure they have the same size.", + stacklevel=2, + ) + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + + expanded_input, expanded_target = torch.broadcast_tensors(input, target) + + if beta == 0.0: + return torch._C._nn.l1_loss( + expanded_input, + expanded_target, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + ) + else: + return torch._C._nn.smooth_l1_loss( + expanded_input, + expanded_target, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + beta, + ) + + +def huber_loss( + input: Tensor, + target: Tensor, + reduction: str = "mean", + delta: float = 1.0, + weight: Optional[Tensor] = None, +) -> Tensor: + r"""Compute the Huber loss, with optional weighting. + + Function uses a squared term if the absolute + element-wise error falls below delta and a delta-scaled L1 term otherwise. + + When delta equals 1, this loss is equivalent to SmoothL1Loss. + In general, Huber loss differs from SmoothL1Loss by a factor of delta (AKA beta in Smooth L1). + + See :class:`~torch.nn.HuberLoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + delta (float, optional): The threshold at which to change between delta-scaled L1 and L2 loss. Default: 1.0. + weight (Tensor, optional): Weights for each sample. Default: None. + + Returns: + Tensor: Huber loss (optionally weighted). + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + huber_loss, + (input, target, weight), + input, + target, + reduction=reduction, + delta=delta, + weight=weight, + ) + + if target.size() != input.size(): + warnings.warn( + f"Using a target size ({target.size()}) that is different to the input size ({input.size()}). " + "This will likely lead to incorrect results due to broadcasting. " + "Please ensure they have the same size.", + stacklevel=2, + ) + + expanded_input, expanded_target = torch.broadcast_tensors(input, target) + + if weight is None: + # Use the optimized C++ backend for standard Huber loss + return torch._C._nn.huber_loss( + expanded_input, + expanded_target, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + delta, + ) + else: + if weight.size() != input.size(): + raise ValueError("Weights and input must have the same size.") + + # Calculate the unweighted loss first + unweighted_loss = torch._C._nn.huber_loss( + expanded_input, + expanded_target, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum("none"), + delta, + ) + + # Apply weight to the unweighted loss + weighted_loss = unweighted_loss * weight + + if reduction == "none": + return weighted_loss + elif reduction == "sum": + return torch.sum(weighted_loss) + elif reduction == "mean": + return weighted_loss.mean() + else: + raise ValueError( + f"Invalid reduction mode: {reduction}. Expected one of 'none', 'mean', 'sum'." + ) + + +def l1_loss( + input: Tensor, + target: Tensor, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", + weight: Optional[Tensor] = None, +) -> Tensor: # noqa: D400,D402 + r"""Compute the L1 loss, with optional weighting. + + Function that takes the mean element-wise absolute value difference. + + See :class:`~torch.nn.L1Loss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + weight (Tensor, optional): Weights for each sample. Default: None. + + Returns: + Tensor: L1 loss (optionally weighted). + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + l1_loss, + (input, target, weight), + input, + target, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if target.size() != input.size(): + warnings.warn( + f"Using a target size ({target.size()}) that is different to the input size ({input.size()}). " + "This will likely lead to incorrect results due to broadcasting. " + "Please ensure they have the same size.", + stacklevel=2, + ) + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + + expanded_input, expanded_target = torch.broadcast_tensors(input, target) + + if weight is not None: + if weight.size() != input.size(): + raise ValueError("Weights and input must have the same size.") + + absolute_errors = torch.abs(expanded_input - expanded_target) + weighted_absolute_errors = absolute_errors * weight + + if reduction == "none": + return weighted_absolute_errors + elif reduction == "sum": + return torch.sum(weighted_absolute_errors) + elif reduction == "mean": + return torch.sum(weighted_absolute_errors) / torch.sum(weight) + else: + raise ValueError( + f"Invalid reduction mode: {reduction}. Expected one of 'none', 'mean', 'sum'." + ) + else: + return torch._C._nn.l1_loss( + expanded_input, + expanded_target, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + ) + + +def mse_loss( + input: Tensor, + target: Tensor, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", + weight: Optional[Tensor] = None, +) -> Tensor: + r"""Compute the element-wise mean squared error, with optional weighting. + + See :class:`~torch.nn.MSELoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + weight (Tensor, optional): Weights for each sample. Default: None. + + Returns: + Tensor: Mean Squared Error loss (optionally weighted). + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + mse_loss, + (input, target, weight), + input, + target, + size_average=size_average, + reduce=reduce, + reduction=reduction, + weight=weight, + ) + + if target.size() != input.size(): + warnings.warn( + f"Using a target size ({target.size()}) that is different to the input size ({input.size()}). " + "This will likely lead to incorrect results due to broadcasting. " + "Please ensure they have the same size.", + stacklevel=2, + ) + + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + + expanded_input, expanded_target = torch.broadcast_tensors(input, target) + + if weight is not None: + if weight.size() != input.size(): + raise ValueError("Weights and input must have the same size.") + + # Perform weighted MSE loss manually + squared_errors = torch.pow(expanded_input - expanded_target, 2) + weighted_squared_errors = squared_errors * weight + + if reduction == "none": + return weighted_squared_errors + elif reduction == "sum": + return torch.sum(weighted_squared_errors) + elif reduction == "mean": + return torch.sum(weighted_squared_errors) / torch.sum(weight) + else: + raise ValueError( + f"Invalid reduction mode: {reduction}. Expected one of 'none', 'mean', 'sum'." + ) + else: + return torch._C._nn.mse_loss( + expanded_input, + expanded_target, + # pyrefly: ignore [bad-argument-type] + _Reduction.get_enum(reduction), + ) + + +def margin_ranking_loss( + input1: Tensor, + input2: Tensor, + target: Tensor, + margin: float = 0, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the margin ranking loss. + + See :class:`~torch.nn.MarginRankingLoss` for details. + + Args: + input1 (Tensor): Predicted values. + input2 (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Margin ranking loss. + """ + if has_torch_function_variadic(input1, input2, target): + return handle_torch_function( + margin_ranking_loss, + (input1, input2, target), + input1, + input2, + target, + margin=margin, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + if input1.dim() != input2.dim() or input1.dim() != target.dim(): + raise RuntimeError( + f"margin_ranking_loss : All input tensors should have same dimension but got sizes: " + f"input1: {input1.size()}, input2: {input2.size()}, target: {target.size()} " + ) + return torch.margin_ranking_loss(input1, input2, target, margin, reduction_enum) + + +def hinge_embedding_loss( + input: Tensor, + target: Tensor, + margin: float = 1.0, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the hinge embedding loss. + + See :class:`~torch.nn.HingeEmbeddingLoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + margin (float, optional): Margin for hinge loss. Has a default value of 1. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Hinge embedding loss. + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + hinge_embedding_loss, + (input, target), + input, + target, + margin=margin, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + return torch.hinge_embedding_loss(input, target, margin, reduction_enum) + + +def multilabel_margin_loss( + input: Tensor, + target: Tensor, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the multilabel margin loss. + + See :class:`~torch.nn.MultiLabelMarginLoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Mutilabel margin loss. + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + multilabel_margin_loss, + (input, target), + input, + target, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + # pyrefly: ignore [bad-argument-type] + return torch._C._nn.multilabel_margin_loss(input, target, reduction_enum) + + +def soft_margin_loss( + input: Tensor, + target: Tensor, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the soft margin loss. + + See :class:`~torch.nn.SoftMarginLoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Soft margin loss. + """ + if has_torch_function_variadic(input, target): + return handle_torch_function( + soft_margin_loss, + (input, target), + input, + target, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + # pyrefly: ignore [bad-argument-type] + return torch._C._nn.soft_margin_loss(input, target, reduction_enum) + + +def multilabel_soft_margin_loss( + input: Tensor, + target: Tensor, + weight: Optional[Tensor] = None, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the multilabel soft margin loss. + + See :class:`~torch.nn.MultiLabelSoftMarginLoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Mutilabel soft margin loss. + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + multilabel_soft_margin_loss, + (input, target, weight), + input, + target, + weight=weight, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction = _Reduction.legacy_get_string(size_average, reduce) + + loss = -(target * logsigmoid(input) + (1 - target) * logsigmoid(-input)) + + if weight is not None: + loss = loss * weight + + class_dim = input.dim() - 1 + C = input.size(class_dim) + loss = loss.sum(dim=class_dim) / C # only return N loss values + + if reduction == "none": + ret = loss + elif reduction == "mean": + ret = loss.mean() + elif reduction == "sum": + ret = loss.sum() + else: + ret = input + raise ValueError(reduction + " is not valid") + return ret + + +def cosine_embedding_loss( + input1: Tensor, + input2: Tensor, + target: Tensor, + margin: float = 0, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the cosine embedding loss. + + See :class:`~torch.nn.CosineEmbeddingLoss` for details. + + Args: + input1 (Tensor): Predicted values. + input2 (Tensor): Predicted values. + target (Tensor): Ground truth values. + margin (float, optional): Margin for cosine embedding. Has a default value of 0. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Cosine embedding loss. + """ + if has_torch_function_variadic(input1, input2, target): + return handle_torch_function( + cosine_embedding_loss, + (input1, input2, target), + input1, + input2, + target, + margin=margin, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + return torch.cosine_embedding_loss(input1, input2, target, margin, reduction_enum) + + +def multi_margin_loss( + input: Tensor, + target: Tensor, + p: int = 1, + margin: float = 1.0, + weight: Optional[Tensor] = None, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: # noqa: D400,D402 + r"""Compute the multi margin loss, with optional weighting. + + See :class:`~torch.nn.MultiMarginLoss` for details. + + Args: + input (Tensor): Predicted values. + target (Tensor): Ground truth values. + p (int, optional): Has a default value of 1. 1 and 2 are the only supported values. + margin (float, optional): Margin for multi margin loss. Has a default value of 1. + weight (Tensor, optional): Weights for each sample. Default: None. + size_average (bool, optional): Deprecated (see :attr:`reduction`). + reduce (bool, optional): Deprecated (see :attr:`reduction`). + reduction (str, optional): Specifies the reduction to apply to the output: + 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. + 'sum': the output will be summed. 'none': no reduction will be applied. + Default: 'mean'. + + Returns: + Tensor: Multi margin loss (optionally weighted). + """ + if has_torch_function_variadic(input, target, weight): + return handle_torch_function( + multi_margin_loss, + (input, target, weight), + input, + target, + p=p, + margin=margin, + weight=weight, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + if p != 1 and p != 2: + raise ValueError("only p == 1 and p == 2 supported") + if weight is not None: + if weight.dim() != 1: + raise ValueError("weight must be one-dimensional") + + return torch._C._nn.multi_margin_loss( + input, + target, + p, + margin, + weight, + # pyrefly: ignore [bad-argument-type] + reduction_enum, + ) + + +pixel_shuffle = _add_docstr( + torch.pixel_shuffle, + r""" +pixel_shuffle(input, upscale_factor) -> Tensor + +Rearranges elements in a tensor of shape :math:`(*, C \times r^2, H, W)` to a +tensor of shape :math:`(*, C, H \times r, W \times r)`, where r is the :attr:`upscale_factor`. + +See :class:`~torch.nn.PixelShuffle` for details. + +Args: + input (Tensor): the input tensor + upscale_factor (int): factor to increase spatial resolution by + +Examples:: + + >>> input = torch.randn(1, 9, 4, 4) + >>> output = torch.nn.functional.pixel_shuffle(input, 3) + >>> print(output.size()) + torch.Size([1, 1, 12, 12]) +""", +) + +pixel_unshuffle = _add_docstr( + torch.pixel_unshuffle, + r""" +pixel_unshuffle(input, downscale_factor) -> Tensor + +Reverses the :class:`~torch.nn.PixelShuffle` operation by rearranging elements in a +tensor of shape :math:`(*, C, H \times r, W \times r)` to a tensor of shape +:math:`(*, C \times r^2, H, W)`, where r is the :attr:`downscale_factor`. + +See :class:`~torch.nn.PixelUnshuffle` for details. + +Args: + input (Tensor): the input tensor + downscale_factor (int): factor to increase spatial resolution by + +Examples:: + + >>> input = torch.randn(1, 1, 12, 12) + >>> output = torch.nn.functional.pixel_unshuffle(input, 3) + >>> print(output.size()) + torch.Size([1, 9, 4, 4]) +""", +) + +channel_shuffle = _add_docstr( + torch.channel_shuffle, + r""" +channel_shuffle(input, groups) -> Tensor + +Divide the channels in a tensor of shape :math:`(*, C , H, W)` +into g groups and rearrange them as :math:`(*, C \frac g, g, H, W)`, +while keeping the original tensor shape. + +See :class:`~torch.nn.ChannelShuffle` for details. + +Args: + input (Tensor): the input tensor + groups (int): number of groups to divide channels in and rearrange. + +Examples:: + + >>> input = torch.randn(1, 4, 2, 2) + >>> print(input) + [[[[1, 2], + [3, 4]], + [[5, 6], + [7, 8]], + [[9, 10], + [11, 12]], + [[13, 14], + [15, 16]], + ]] + >>> output = torch.nn.functional.channel_shuffle(input, 2) + >>> print(output) + [[[[1, 2], + [3, 4]], + [[9, 10], + [11, 12]], + [[5, 6], + [7, 8]], + [[13, 14], + [15, 16]], + ]] +""", +) + +native_channel_shuffle = _add_docstr( + torch.native_channel_shuffle, + r""" +native_channel_shuffle(input, groups) -> Tensor + +Native kernel level implementation of the `channel_shuffle`. +This function might become private in future releases, use with caution. + +Divide the channels in a tensor of shape :math:`(*, C , H, W)` +into g groups and rearrange them as :math:`(*, C \frac g, g, H, W)`, +while keeping the original tensor shape. + +See :class:`~torch.nn.ChannelShuffle` for details. + +Args: + input (Tensor): the input tensor + groups (int): number of groups to divide channels in and rearrange. + +Examples:: + + >>> input = torch.randn(1, 4, 2, 2) + >>> print(input) + [[[[1, 2], + [3, 4]], + [[5, 6], + [7, 8]], + [[9, 10], + [11, 12]], + [[13, 14], + [15, 16]], + ]] + >>> output = torch.nn.functional.native_channel_shuffle(input, 2) + >>> print(output) + [[[[1, 2], + [3, 4]], + [[9, 10], + [11, 12]], + [[5, 6], + [7, 8]], + [[13, 14], + [15, 16]], + ]] +""", +) + + +@_overload +def upsample( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[float] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: # noqa: B950 + pass + + +@_overload +def upsample( # noqa: F811 + input: Tensor, + size: Optional[list[int]] = None, + scale_factor: Optional[float] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: # noqa: B950 + pass + + +def upsample( # noqa: F811 + input, + size=None, + scale_factor=None, + mode="nearest", + align_corners=None, +): + r"""Upsample input. + + Provided tensor is upsampled to either the given :attr:`size` or the given + :attr:`scale_factor` + + .. warning:: + This function is deprecated in favor of :func:`torch.nn.functional.interpolate`. + This is equivalent with ``nn.functional.interpolate(...)``. + + Note: + {backward_reproducibility_note} + + The algorithm used for upsampling is determined by :attr:`mode`. + + Currently temporal, spatial and volumetric upsampling are supported, i.e. + expected inputs are 3-D, 4-D or 5-D in shape. + + The input dimensions are interpreted in the form: + `mini-batch x channels x [optional depth] x [optional height] x width`. + + The modes available for upsampling are: `nearest`, `linear` (3D-only), + `bilinear`, `bicubic` (4D-only), `trilinear` (5D-only) + + Args: + input (Tensor): the input tensor + size (int or Tuple[int] or Tuple[int, int] or Tuple[int, int, int]): + output spatial size. + scale_factor (float or Tuple[float]): multiplier for spatial size. Has to match input size if it is a tuple. + mode (str): algorithm used for upsampling: + ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | ``'bicubic'`` | + ``'trilinear'``. Default: ``'nearest'`` + align_corners (bool, optional): Geometrically, we consider the pixels of the + input and output as squares rather than points. + If set to ``True``, the input and output tensors are aligned by the + center points of their corner pixels, preserving the values at the corner pixels. + If set to ``False``, the input and output tensors are aligned by the corner + points of their corner pixels, and the interpolation uses edge value padding + for out-of-boundary values, making this operation *independent* of input size + when :attr:`scale_factor` is kept the same. This only has an effect when :attr:`mode` + is ``'linear'``, ``'bilinear'``, ``'bicubic'`` or ``'trilinear'``. + Default: ``False`` + + .. note:: + With ``mode='bicubic'``, it's possible to cause overshoot, in other words it can produce + negative values or values greater than 255 for images. + Explicitly call ``result.clamp(min=0, max=255)`` if you want to reduce the overshoot + when displaying the image. + + .. warning:: + With ``align_corners = True``, the linearly interpolating modes + (`linear`, `bilinear`, and `trilinear`) don't proportionally align the + output and input pixels, and thus the output values can depend on the + input size. This was the default behavior for these modes up to version + 0.3.1. Since then, the default behavior is ``align_corners = False``. + See :class:`~torch.nn.Upsample` for concrete examples on how this + affects the outputs. + + """ + warnings.warn( + "`nn.functional.upsample` is deprecated. " + "Use `nn.functional.interpolate` instead.", + stacklevel=2, + ) + return interpolate(input, size, scale_factor, mode, align_corners) + + +if upsample.__doc__: + upsample.__doc__ = upsample.__doc__.format(**reproducibility_notes) + + +def _is_integer(x) -> bool: + r"""Type check the input number is an integer. + + Will return True for int, SymInt, Numpy integers and Tensors with integer elements. + """ + if isinstance(x, (int, torch.SymInt)): + return True + if np is not None and isinstance(x, np.integer): + return True + return isinstance(x, Tensor) and not x.is_floating_point() + + +@_overload +def interpolate( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[list[float]] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + recompute_scale_factor: Optional[bool] = None, + antialias: bool = False, + # pyrefly: ignore [bad-return] +) -> Tensor: # noqa: B950 + pass + + +@_overload +def interpolate( # noqa: F811 + input: Tensor, + size: Optional[list[int]] = None, + scale_factor: Optional[list[float]] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + recompute_scale_factor: Optional[bool] = None, + antialias: bool = False, + # pyrefly: ignore [bad-return] +) -> Tensor: # noqa: B950 + pass + + +@_overload +def interpolate( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[float] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + recompute_scale_factor: Optional[bool] = None, + antialias: bool = False, + # pyrefly: ignore [bad-return] +) -> Tensor: # noqa: B950 + pass + + +@_overload +def interpolate( # noqa: F811 + input: Tensor, + size: Optional[list[int]] = None, + scale_factor: Optional[float] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + recompute_scale_factor: Optional[bool] = None, + antialias: bool = False, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +def interpolate( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[list[float]] = None, + mode: str = "nearest", + align_corners: Optional[bool] = None, + recompute_scale_factor: Optional[bool] = None, + antialias: bool = False, +) -> Tensor: # noqa: B950 + r"""Down/up samples the input. + + Tensor interpolated to either the given :attr:`size` or the given + :attr:`scale_factor` + + The algorithm used for interpolation is determined by :attr:`mode`. + + Currently temporal, spatial and volumetric sampling are supported, i.e. + expected inputs are 3-D, 4-D or 5-D in shape. + + The input dimensions are interpreted in the form: + `mini-batch x channels x [optional depth] x [optional height] x width`. + + The modes available for resizing are: `nearest`, `linear` (3D-only), + `bilinear`, `bicubic` (4D-only), `trilinear` (5D-only), `area`, `nearest-exact` + + Args: + input (Tensor): the input tensor + size (int or Tuple[int] or Tuple[int, int] or Tuple[int, int, int]): + output spatial size. + scale_factor (float or Tuple[float]): multiplier for spatial size. If `scale_factor` is a tuple, + its length has to match the number of spatial dimensions; `input.dim() - 2`. + mode (str): algorithm used for upsampling: + ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | ``'bicubic'`` | + ``'trilinear'`` | ``'area'`` | ``'nearest-exact'``. Default: ``'nearest'`` + align_corners (bool, optional): Geometrically, we consider the pixels of the + input and output as squares rather than points. + If set to ``True``, the input and output tensors are aligned by the + center points of their corner pixels, preserving the values at the corner pixels. + If set to ``False``, the input and output tensors are aligned by the corner + points of their corner pixels, and the interpolation uses edge value padding + for out-of-boundary values, making this operation *independent* of input size + when :attr:`scale_factor` is kept the same. This only has an effect when :attr:`mode` + is ``'linear'``, ``'bilinear'``, ``'bicubic'`` or ``'trilinear'``. + Default: ``False`` + recompute_scale_factor (bool, optional): recompute the scale_factor for use in the + interpolation calculation. If `recompute_scale_factor` is ``True``, then + `scale_factor` must be passed in and `scale_factor` is used to compute the + output `size`. The computed output `size` will be used to infer new scales for + the interpolation. Note that when `scale_factor` is floating-point, it may differ + from the recomputed `scale_factor` due to rounding and precision issues. + If `recompute_scale_factor` is ``False``, then `size` or `scale_factor` will + be used directly for interpolation. Default: ``None``. + antialias (bool, optional): flag to apply anti-aliasing. Default: ``False``. Using anti-alias + option together with ``align_corners=False``, interpolation result would match Pillow + result for downsampling operation. Supported modes: ``'bilinear'``, ``'bicubic'``. + + .. note:: + With ``mode='bicubic'``, it's possible to cause overshoot. For some dtypes, it can produce + negative values or values greater than 255 for images. Explicitly call ``result.clamp(min=0,max=255)`` + if you want to reduce the overshoot when displaying the image. + For ``uint8`` inputs, it already performs saturating cast operation. So, no manual `clamp` operation is needed. + + .. note:: + Mode ``mode='nearest-exact'`` matches Scikit-Image and PIL nearest neighbours interpolation + algorithms and fixes known issues with ``mode='nearest'``. This mode is introduced to keep + backward compatibility. + Mode ``mode='nearest'`` matches buggy OpenCV's ``INTER_NEAREST`` interpolation algorithm. + + .. note:: + The gradients for the dtype ``float16`` on CUDA may be inaccurate in the upsample operation + when using modes ``['linear', 'bilinear', 'bicubic', 'trilinear', 'area']``. + For more details, please refer to the discussion in + `issue#104157 `_. + + Note: + {backward_reproducibility_note} + """ + if has_torch_function_unary(input): + return handle_torch_function( + interpolate, + (input,), + input, + size=size, + scale_factor=scale_factor, + mode=mode, + align_corners=align_corners, + recompute_scale_factor=recompute_scale_factor, + antialias=antialias, + ) + + if mode in ("nearest", "area", "nearest-exact"): + if align_corners is not None: + raise ValueError( + "align_corners option can only be set with the " + "interpolating modes: linear | bilinear | bicubic | trilinear" + ) + else: + if align_corners is None: + align_corners = False + + dim = input.dim() - 2 # Number of spatial dimensions. + + # Process size and scale_factor. Validate that exactly one is set. + # Validate its length if it is a list, or expand it if it is a scalar. + # After this block, exactly one of output_size and scale_factors will + # be non-None, and it will be a list (or tuple). + if size is not None and scale_factor is not None: + raise ValueError("only one of size or scale_factor should be defined") + elif size is not None: + assert scale_factor is None + scale_factors = None + if isinstance(size, (list, tuple)): + if len(size) != dim: + raise ValueError( + "Input and output must have the same number of spatial dimensions, but got " + f"input with spatial dimensions of {list(input.shape[2:])} and output size of {size}. " + "Please provide input tensor in (N, C, d1, d2, ...,dK) format and " + "output size in (o1, o2, ...,oK) format." + ) + if not torch.jit.is_scripting(): + if not all(_is_integer(x) for x in size): + raise TypeError( + "expected size to be one of int or Tuple[int] or Tuple[int, int] or " + f"Tuple[int, int, int], but got size with types {[type(x) for x in size]}" + ) + output_size = size + else: + output_size = [size for _ in range(dim)] + elif scale_factor is not None: + assert size is None + output_size = None + if isinstance(scale_factor, (list, tuple)): + if len(scale_factor) != dim: + raise ValueError( + "Input and scale_factor must have the same number of spatial dimensions, but " + f"got input with spatial dimensions of {list(input.shape[2:])} and " + f"scale_factor of shape {scale_factor}. " + "Please provide input tensor in (N, C, d1, d2, ...,dK) format and " + "scale_factor in (s1, s2, ...,sK) format." + ) + scale_factors = scale_factor + else: + scale_factors = [scale_factor for _ in range(dim)] + else: + raise ValueError("either size or scale_factor should be defined") + + if ( + recompute_scale_factor is not None + and recompute_scale_factor + and size is not None + ): + raise ValueError( + "recompute_scale_factor is not meaningful with an explicit size." + ) + + # "area" mode always requires an explicit size rather than scale factor. + # Reuse the recompute_scale_factor code path. + if mode == "area" and output_size is None: + recompute_scale_factor = True + + if recompute_scale_factor is not None and recompute_scale_factor: + # We compute output_size here, then un-set scale_factors. + # The C++ code will recompute it based on the (integer) output size. + assert scale_factors is not None + if not torch.jit.is_scripting() and torch._C._get_tracing_state(): + # make scale_factor a tensor in tracing so constant doesn't get baked in + output_size = [ + ( + torch.floor( + ( + # pyrefly: ignore [missing-attribute] + input.size(i + 2).float() + * torch.tensor(scale_factors[i], dtype=torch.float32) + ).float() + ) + ) + for i in range(dim) + ] + elif torch.jit.is_scripting(): + output_size = [ + math.floor(float(input.size(i + 2)) * scale_factors[i]) + for i in range(dim) + ] + else: + output_size = [ + _sym_int(input.size(i + 2) * scale_factors[i]) for i in range(dim) + ] + scale_factors = None + + if antialias and not (mode in ("bilinear", "bicubic") and input.ndim == 4): + raise ValueError( + "Anti-alias option is restricted to bilinear and bicubic modes and requires a 4-D tensor as input" + ) + + if input.dim() == 3 and mode == "nearest": + # pyrefly: ignore [bad-argument-type] + return torch._C._nn.upsample_nearest1d(input, output_size, scale_factors) + if input.dim() == 4 and mode == "nearest": + # pyrefly: ignore [bad-argument-type] + return torch._C._nn.upsample_nearest2d(input, output_size, scale_factors) + if input.dim() == 5 and mode == "nearest": + # pyrefly: ignore [bad-argument-type] + return torch._C._nn.upsample_nearest3d(input, output_size, scale_factors) + + if input.dim() == 3 and mode == "nearest-exact": + # pyrefly: ignore [bad-argument-type] + return torch._C._nn._upsample_nearest_exact1d(input, output_size, scale_factors) + if input.dim() == 4 and mode == "nearest-exact": + # pyrefly: ignore [bad-argument-type] + return torch._C._nn._upsample_nearest_exact2d(input, output_size, scale_factors) + if input.dim() == 5 and mode == "nearest-exact": + # pyrefly: ignore [bad-argument-type] + return torch._C._nn._upsample_nearest_exact3d(input, output_size, scale_factors) + + if input.dim() == 3 and mode == "area": + assert output_size is not None + # pyrefly: ignore [bad-argument-type] + return adaptive_avg_pool1d(input, output_size) + if input.dim() == 4 and mode == "area": + assert output_size is not None + return adaptive_avg_pool2d(input, output_size) + if input.dim() == 5 and mode == "area": + assert output_size is not None + return adaptive_avg_pool3d(input, output_size) + + if input.dim() == 3 and mode == "linear": + assert align_corners is not None + return torch._C._nn.upsample_linear1d( + input, + # pyrefly: ignore [bad-argument-type] + output_size, + align_corners, + scale_factors, + ) + if input.dim() == 4 and mode == "bilinear": + assert align_corners is not None + if antialias: + return torch._C._nn._upsample_bilinear2d_aa( + input, + # pyrefly: ignore [bad-argument-type] + output_size, + align_corners, + scale_factors, + ) + # Two levels are necessary to prevent TorchScript from touching + # are_deterministic_algorithms_enabled. + if not torch.jit.is_scripting(): + if not input.is_cpu and torch.are_deterministic_algorithms_enabled(): + # Use slow decomp whose backward will be in terms of index_put + # importlib is required because the import cannot be top level + # (cycle) and cannot be nested (TS doesn't support) + return importlib.import_module( + "torch._decomp.decompositions" + )._upsample_linear_vec(input, output_size, align_corners, scale_factors) + return torch._C._nn.upsample_bilinear2d( + input, + # pyrefly: ignore [bad-argument-type] + output_size, + align_corners, + scale_factors, + ) + if input.dim() == 5 and mode == "trilinear": + assert align_corners is not None + # Two levels are necessary to prevent TorchScript from touching + # are_deterministic_algorithms_enabled. + if not torch.jit.is_scripting(): + if not input.is_cpu and torch.are_deterministic_algorithms_enabled(): + # Use slow decomp whose backward will be in terms of index_put + # importlib is required because the import cannot be top level + # (cycle) and cannot be nested (TS doesn't support) + return importlib.import_module( + "torch._decomp.decompositions" + )._upsample_linear_vec(input, output_size, align_corners, scale_factors) + return torch._C._nn.upsample_trilinear3d( + input, + # pyrefly: ignore [bad-argument-type] + output_size, + align_corners, + scale_factors, + ) + if input.dim() == 4 and mode == "bicubic": + assert align_corners is not None + if antialias: + return torch._C._nn._upsample_bicubic2d_aa( + input, + # pyrefly: ignore [bad-argument-type] + output_size, + align_corners, + scale_factors, + ) + return torch._C._nn.upsample_bicubic2d( + input, + # pyrefly: ignore [bad-argument-type] + output_size, + align_corners, + scale_factors, + ) + + if input.dim() == 3 and mode == "bilinear": + raise NotImplementedError("Got 3D input, but bilinear mode needs 4D input") + if input.dim() == 3 and mode == "trilinear": + raise NotImplementedError("Got 3D input, but trilinear mode needs 5D input") + if input.dim() == 4 and mode == "linear": + raise NotImplementedError("Got 4D input, but linear mode needs 3D input") + if input.dim() == 4 and mode == "trilinear": + raise NotImplementedError("Got 4D input, but trilinear mode needs 5D input") + if input.dim() == 5 and mode == "linear": + raise NotImplementedError("Got 5D input, but linear mode needs 3D input") + if input.dim() == 5 and mode == "bilinear": + raise NotImplementedError("Got 5D input, but bilinear mode needs 4D input") + + raise NotImplementedError( + "Input Error: Only 3D, 4D and 5D input Tensors supported" + f" (got {input.dim()}D) for the modes: nearest | linear | bilinear | bicubic | trilinear | area | nearest-exact" + f" (got {mode})" + ) + + +if interpolate.__doc__: + interpolate.__doc__ = interpolate.__doc__.format(**reproducibility_notes) + + +@_overload +def upsample_nearest( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[float] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +@_overload +def upsample_nearest( # noqa: F811 + input: Tensor, + size: Optional[list[int]] = None, + scale_factor: Optional[float] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +def upsample_nearest(input, size=None, scale_factor=None): # noqa: F811 + r"""Upsamples the input, using nearest neighbours' pixel values. + + .. warning:: + This function is deprecated in favor of :func:`torch.nn.functional.interpolate`. + This is equivalent with ``nn.functional.interpolate(..., mode='nearest')``. + + Currently spatial and volumetric upsampling are supported (i.e. expected + inputs are 4 or 5 dimensional). + + Args: + input (Tensor): input + size (int or Tuple[int, int] or Tuple[int, int, int]): output spatia + size. + scale_factor (int): multiplier for spatial size. Has to be an integer. + + Note: + {backward_reproducibility_note} + """ + # DeprecationWarning is ignored by default + warnings.warn( + "`nn.functional.upsample_nearest` is deprecated. " + "Use `nn.functional.interpolate` instead.", + stacklevel=2, + ) + return interpolate(input, size, scale_factor, mode="nearest") + + +if upsample_nearest.__doc__: + upsample_nearest.__doc__ = upsample_nearest.__doc__.format(**reproducibility_notes) + + +@_overload +def upsample_bilinear( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[float] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +@_overload +def upsample_bilinear( # noqa: F811 + input: Tensor, + size: Optional[list[int]] = None, + scale_factor: Optional[float] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +@_overload +def upsample_bilinear( # noqa: F811 + input: Tensor, + size: Optional[int] = None, + scale_factor: Optional[list[float]] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +@_overload +def upsample_bilinear( # noqa: F811 + input: Tensor, + size: Optional[list[int]] = None, + scale_factor: Optional[list[float]] = None, + # pyrefly: ignore [bad-return] +) -> Tensor: + pass + + +def upsample_bilinear(input, size=None, scale_factor=None): # noqa: F811 + r"""Upsamples the input, using bilinear upsampling. + + .. warning:: + This function is deprecated in favor of :func:`torch.nn.functional.interpolate`. + This is equivalent with + ``nn.functional.interpolate(..., mode='bilinear', align_corners=True)``. + + Expected inputs are spatial (4 dimensional). Use `upsample_trilinear` for + volumetric (5 dimensional) inputs. + + Args: + input (Tensor): input + size (int or Tuple[int, int]): output spatial size. + scale_factor (int or Tuple[int, int]): multiplier for spatial size + + Note: + {backward_reproducibility_note} + """ + # DeprecationWarning is ignored by default + warnings.warn( + "`nn.functional.upsample_bilinear` is deprecated. " + "Use `nn.functional.interpolate` instead.", + stacklevel=2, + ) + return interpolate(input, size, scale_factor, mode="bilinear", align_corners=True) + + +if upsample_bilinear.__doc__: + upsample_bilinear.__doc__ = upsample_bilinear.__doc__.format( + **reproducibility_notes + ) + +GRID_SAMPLE_INTERPOLATION_MODES = { + "bilinear": 0, + "nearest": 1, + "bicubic": 2, +} + +GRID_SAMPLE_PADDING_MODES = { + "zeros": 0, + "border": 1, + "reflection": 2, +} + + +def grid_sample( + input: Tensor, + grid: Tensor, + mode: str = "bilinear", + padding_mode: str = "zeros", + align_corners: Optional[bool] = None, +) -> Tensor: + r"""Compute grid sample. + + Given an :attr:`input` and a flow-field :attr:`grid`, computes the + ``output`` using :attr:`input` values and pixel locations from :attr:`grid`. + + Currently, only spatial (4-D) and volumetric (5-D) :attr:`input` are + supported. + + In the spatial (4-D) case, for :attr:`input` with shape + :math:`(N, C, H_\text{in}, W_\text{in})` and :attr:`grid` with shape + :math:`(N, H_\text{out}, W_\text{out}, 2)`, the output will have shape + :math:`(N, C, H_\text{out}, W_\text{out})`. + + For each output location ``output[n, :, h, w]``, the size-2 vector + ``grid[n, h, w]`` specifies :attr:`input` pixel locations ``x`` and ``y``, + which are used to interpolate the output value ``output[n, :, h, w]``. + In the case of 5D inputs, ``grid[n, d, h, w]`` specifies the + ``x``, ``y``, ``z`` pixel locations for interpolating + ``output[n, :, d, h, w]``. :attr:`mode` argument specifies ``nearest`` or + ``bilinear`` interpolation method to sample the input pixels. + + :attr:`grid` specifies the sampling pixel locations normalized by the + :attr:`input` spatial dimensions. Therefore, it should have most values in + the range of ``[-1, 1]``. For example, values ``x = -1, y = -1`` is the + left-top pixel of :attr:`input`, and values ``x = 1, y = 1`` is the + right-bottom pixel of :attr:`input`. + + If :attr:`grid` has values outside the range of ``[-1, 1]``, the corresponding + outputs are handled as defined by :attr:`padding_mode`. Options are + + * ``padding_mode="zeros"``: use ``0`` for out-of-bound grid locations, + * ``padding_mode="border"``: use border values for out-of-bound grid locations, + * ``padding_mode="reflection"``: use values at locations reflected by + the border for out-of-bound grid locations. For location far away + from the border, it will keep being reflected until becoming in bound, + e.g., (normalized) pixel location ``x = -3.5`` reflects by border ``-1`` + and becomes ``x' = 1.5``, then reflects by border ``1`` and becomes + ``x'' = -0.5``. + + Note: + This function is often used in conjunction with :func:`affine_grid` + to build `Spatial Transformer Networks`_ . + + Note: + When using the CUDA backend, this operation may induce nondeterministic + behaviour in its backward pass that is not easily switched off. + Please see the notes on :doc:`/notes/randomness` for background. + + Note: + NaN values in :attr:`grid` would be interpreted as ``-1``. + + Args: + input (Tensor): input of shape :math:`(N, C, H_\text{in}, W_\text{in})` (4-D case) + or :math:`(N, C, D_\text{in}, H_\text{in}, W_\text{in})` (5-D case) + grid (Tensor): flow-field of shape :math:`(N, H_\text{out}, W_\text{out}, 2)` (4-D case) + or :math:`(N, D_\text{out}, H_\text{out}, W_\text{out}, 3)` (5-D case) + mode (str): interpolation mode to calculate output values + ``'bilinear'`` | ``'nearest'`` | ``'bicubic'``. Default: ``'bilinear'`` + Note: ``mode='bicubic'`` supports only 4-D input. + When ``mode='bilinear'`` and the input is 5-D, the interpolation mode + used internally will actually be trilinear. However, when the input is 4-D, + the interpolation mode will legitimately be bilinear. + padding_mode (str): padding mode for outside grid values + ``'zeros'`` | ``'border'`` | ``'reflection'``. Default: ``'zeros'`` + align_corners (bool, optional): Geometrically, we consider the pixels of the + input as squares rather than points. + If set to ``True``, the extrema (``-1`` and ``1``) are considered as referring + to the center points of the input's corner pixels. If set to ``False``, they + are instead considered as referring to the corner points of the input's corner + pixels, making the sampling more resolution agnostic. + This option parallels the ``align_corners`` option in + :func:`interpolate`, and so whichever option is used here + should also be used there to resize the input image before grid sampling. + Default: ``False`` + + Returns: + output (Tensor): output Tensor + + .. _`Spatial Transformer Networks`: + https://arxiv.org/abs/1506.02025 + + .. warning:: + When ``align_corners = True``, the grid positions depend on the pixel + size relative to the input image size, and so the locations sampled by + :func:`grid_sample` will differ for the same input given at different + resolutions (that is, after being upsampled or downsampled). + The default behavior up to version 1.2.0 was ``align_corners = True``. + Since then, the default behavior has been changed to ``align_corners = False``, + in order to bring it in line with the default for :func:`interpolate`. + + .. note:: + ``mode='bicubic'`` is implemented using the `cubic convolution algorithm`_ with :math:`\alpha=-0.75`. + The constant :math:`\alpha` might be different from packages to packages. + For example, `PIL`_ and `OpenCV`_ use -0.5 and -0.75 respectively. + This algorithm may "overshoot" the range of values it's interpolating. + For example, it may produce negative values or values greater than 255 when interpolating input in [0, 255]. + Clamp the results with :func:`torch.clamp` to ensure they are within the valid range. + .. _`cubic convolution algorithm`: https://en.wikipedia.org/wiki/Bicubic_interpolation + .. _`PIL`: https://github.com/python-pillow/Pillow/blob/4634eafe3c695a014267eefdce830b4a825beed7/src/libImaging/Resample.c#L51 + .. _`OpenCV`: https://github.com/opencv/opencv/blob/f345ed564a06178670750bad59526cfa4033be55/modules/imgproc/src/resize.cpp#L908 + """ + if has_torch_function_variadic(input, grid): + return handle_torch_function( + grid_sample, + (input, grid), + input, + grid, + mode=mode, + padding_mode=padding_mode, + align_corners=align_corners, + ) + if mode != "bilinear" and mode != "nearest" and mode != "bicubic": + raise ValueError( + f"nn.functional.grid_sample(): expected mode to be 'bilinear', 'nearest' or 'bicubic', but got: '{mode}'" + ) + if ( + padding_mode != "zeros" + and padding_mode != "border" + and padding_mode != "reflection" + ): + raise ValueError( + "nn.functional.grid_sample(): expected padding_mode " + "to be 'zeros', 'border', or 'reflection', " + f"but got: '{padding_mode}'" + ) + + if mode == "bilinear": + mode_enum = 0 + elif mode == "nearest": + mode_enum = 1 + else: # mode == 'bicubic' + mode_enum = 2 + + if padding_mode == "zeros": + padding_mode_enum = 0 + elif padding_mode == "border": + padding_mode_enum = 1 + else: # padding_mode == 'reflection' + padding_mode_enum = 2 + + if align_corners is None: + warnings.warn( + "Default grid_sample and affine_grid behavior has changed " + "to align_corners=False since 1.3.0. Please specify " + "align_corners=True if the old behavior is desired. " + "See the documentation of grid_sample for details.", + stacklevel=2, + ) + align_corners = False + + return torch.grid_sampler(input, grid, mode_enum, padding_mode_enum, align_corners) + + +def affine_grid( + theta: Tensor, + size: list[int], + align_corners: Optional[bool] = None, +) -> Tensor: + r"""Generate 2D or 3D flow field (sampling grid), given a batch of affine matrices :attr:`theta`. + + .. note:: + This function is often used in conjunction with :func:`grid_sample` + to build `Spatial Transformer Networks`_ . + + Args: + theta (Tensor): input batch of affine matrices with shape + (:math:`N \times 2 \times 3`) for 2D or + (:math:`N \times 3 \times 4`) for 3D + size (torch.Size): the target output image size. + (:math:`N \times C \times H \times W` for 2D or + :math:`N \times C \times D \times H \times W` for 3D) + Example: torch.Size((32, 3, 24, 24)) + align_corners (bool, optional): if ``True``, consider ``-1`` and ``1`` + to refer to the centers of the corner pixels rather than the image corners. + Refer to :func:`grid_sample` for a more complete description. + A grid generated by :func:`affine_grid` should be passed to :func:`grid_sample` + with the same setting for this option. + Default: ``False`` + + Returns: + output (Tensor): output Tensor of size (:math:`N \times H \times W \times 2`) + + .. _`Spatial Transformer Networks`: + https://arxiv.org/abs/1506.02025 + + .. warning:: + When ``align_corners = True``, the grid positions depend on the pixel + size relative to the input image size, and so the locations sampled by + :func:`grid_sample` will differ for the same input given at different + resolutions (that is, after being upsampled or downsampled). + The default behavior up to version 1.2.0 was ``align_corners = True``. + Since then, the default behavior has been changed to ``align_corners = False``, + in order to bring it in line with the default for :func:`interpolate`. + .. warning:: + When ``align_corners = True``, 2D affine transforms on 1D data and + 3D affine transforms on 2D data (that is, when one of the spatial + dimensions has unit size) are ill-defined, and not an intended use case. + This is not a problem when ``align_corners = False``. + Up to version 1.2.0, all grid points along a unit dimension were + considered arbitrarily to be at ``-1``. + From version 1.3.0, under ``align_corners = True`` all grid points + along a unit dimension are considered to be at ``0`` + (the center of the input image). + """ + if has_torch_function_unary(theta): + return handle_torch_function( + affine_grid, (theta,), theta, size, align_corners=align_corners + ) + if align_corners is None: + warnings.warn( + "Default grid_sample and affine_grid behavior has changed " + "to align_corners=False since 1.3.0. Please specify " + "align_corners=True if the old behavior is desired. " + "See the documentation of grid_sample for details.", + stacklevel=2, + ) + align_corners = False + + # enforce floating point dtype on theta + if not theta.is_floating_point(): + raise ValueError( + f"Expected theta to have floating point type, but got {theta.dtype}" + ) + # check that shapes and sizes match + if len(size) == 4: + if theta.dim() != 3 or theta.shape[-2] != 2 or theta.shape[-1] != 3: + raise ValueError( + f"Expected a batch of 2D affine matrices of shape Nx2x3 for size {size}. Got {theta.shape}." + ) + spatial_size = size[-2:] # spatial dimension sizes + elif len(size) == 5: + if theta.dim() != 3 or theta.shape[-2] != 3 or theta.shape[-1] != 4: + raise ValueError( + f"Expected a batch of 3D affine matrices of shape Nx3x4 for size {size}. Got {theta.shape}." + ) + spatial_size = size[-3:] # spatial dimension sizes + else: + raise NotImplementedError( + "affine_grid only supports 4D and 5D sizes, " + "for 2D and 3D affine transforms, respectively. " + f"Got size {size}." + ) + # check for empty span + if align_corners and min(spatial_size) == 1: + warnings.warn( + "Since version 1.3.0, affine_grid behavior has changed " + "for unit-size grids when align_corners=True. " + "This is not an intended use case of affine_grid. " + "See the documentation of affine_grid for details.", + stacklevel=2, + ) + elif min(size) <= 0: + raise ValueError(f"Expected non-zero, positive output size. Got {size}") + + return torch.affine_grid_generator(theta, size, align_corners) + + +def pad( + input: Tensor, + pad: list[int], + mode: str = "constant", + value: Optional[float] = None, +) -> Tensor: + r""" + pad(input, pad, mode="constant", value=None) -> Tensor + + Pads tensor. + + Padding size: + The padding size by which to pad some dimensions of :attr:`input` + are described starting from the last dimension and moving forward. + :math:`\left\lfloor\frac{\text{len(pad)}}{2}\right\rfloor` dimensions + of ``input`` will be padded. + For example, to pad only the last dimension of the input tensor, then + :attr:`pad` has the form + :math:`(\text{padding\_left}, \text{padding\_right})`; + to pad the last 2 dimensions of the input tensor, then use + :math:`(\text{padding\_left}, \text{padding\_right},` + :math:`\text{padding\_top}, \text{padding\_bottom})`; + to pad the last 3 dimensions, use + :math:`(\text{padding\_left}, \text{padding\_right},` + :math:`\text{padding\_top}, \text{padding\_bottom}` + :math:`\text{padding\_front}, \text{padding\_back})`. + + Padding mode: + See :class:`torch.nn.CircularPad2d`, :class:`torch.nn.ConstantPad2d`, + :class:`torch.nn.ReflectionPad2d`, and :class:`torch.nn.ReplicationPad2d` + for concrete examples on how each of the padding modes works. Constant + padding is implemented for arbitrary dimensions. Circular, replicate and + reflection padding are implemented for padding the last 3 dimensions of a + 4D or 5D input tensor, the last 2 dimensions of a 3D or 4D input tensor, + or the last dimension of a 2D or 3D input tensor. + + Note: + When using the CUDA backend, this operation may induce nondeterministic + behaviour in its backward pass that is not easily switched off. + Please see the notes on :doc:`/notes/randomness` for background. + + Args: + input (Tensor): N-dimensional tensor + pad (tuple): m-elements tuple, where + :math:`\frac{m}{2} \leq` input dimensions and :math:`m` is even. + mode: ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. + Default: ``'constant'`` + value: fill value for ``'constant'`` padding. Default: ``0`` + + Examples:: + + >>> t4d = torch.empty(3, 3, 4, 2) + >>> p1d = (1, 1) # pad last dim by 1 on each side + >>> out = F.pad(t4d, p1d, "constant", 0) # effectively zero padding + >>> print(out.size()) + torch.Size([3, 3, 4, 4]) + >>> p2d = (1, 1, 2, 2) # pad last dim by (1, 1) and 2nd to last by (2, 2) + >>> out = F.pad(t4d, p2d, "constant", 0) + >>> print(out.size()) + torch.Size([3, 3, 8, 4]) + >>> t4d = torch.empty(3, 3, 4, 2) + >>> p3d = (0, 1, 2, 1, 3, 3) # pad by (0, 1), (2, 1), and (3, 3) + >>> out = F.pad(t4d, p3d, "constant", 0) + >>> print(out.size()) + torch.Size([3, 9, 7, 3]) + """ + if has_torch_function_unary(input): + return handle_torch_function( + torch.nn.functional.pad, (input,), input, pad, mode=mode, value=value + ) + if not torch.jit.is_scripting(): + if torch.are_deterministic_algorithms_enabled() and ( + input.is_cuda or input.is_xpu + ): + if mode == "replicate": + # Use slow decomp whose backward will be in terms of index_put. + # importlib is required because the import cannot be top level + # (cycle) and cannot be nested (TS doesn't support) + return importlib.import_module( + "torch._decomp.decompositions" + )._replication_pad(input, pad) + return torch._C._nn.pad(input, pad, mode, value) + + +# TODO: Fix via https://github.com/pytorch/pytorch/issues/75798 +pad.__module__ = "torch.nn.functional" + +# distance + + +pairwise_distance = _add_docstr( + torch.pairwise_distance, + r""" +pairwise_distance(x1, x2, p=2.0, eps=1e-6, keepdim=False) -> Tensor + +See :class:`torch.nn.PairwiseDistance` for details +""", +) + + +pdist = _add_docstr( + torch.pdist, + r""" +pdist(input, p=2) -> Tensor + +Computes the p-norm distance between every pair of row vectors in the input. +This is identical to the upper triangular portion, excluding the diagonal, of +`torch.norm(input[:, None] - input, dim=2, p=p)`. This function will be faster +if the rows are contiguous. + +If input has shape :math:`N \times M` then the output will have shape +:math:`\frac{1}{2} N (N - 1)`. + +This function is equivalent to ``scipy.spatial.distance.pdist(input, +'minkowski', p=p)`` if :math:`p \in (0, \infty)`. When :math:`p = 0` it is +equivalent to ``scipy.spatial.distance.pdist(input, 'hamming') * M``. +When :math:`p = \infty`, the closest scipy function is +``scipy.spatial.distance.pdist(xn, lambda x, y: np.abs(x - y).max())``. + +Args: + input: input tensor of shape :math:`N \times M`. + p: p value for the p-norm distance to calculate between each vector pair + :math:`\in [0, \infty]`. +""", +) + + +cosine_similarity = _add_docstr( + torch.cosine_similarity, + r""" +cosine_similarity(x1, x2, dim=1, eps=1e-8) -> Tensor + +Returns cosine similarity between ``x1`` and ``x2``, computed along dim. ``x1`` and ``x2`` must be broadcastable +to a common shape. ``dim`` refers to the dimension in this common shape. Dimension ``dim`` of the output is +squeezed (see :func:`torch.squeeze`), resulting in the +output tensor having 1 fewer dimension. + +.. math :: + \text{similarity} = \dfrac{x_1 \cdot x_2}{\max(\Vert x_1 \Vert _2, \epsilon) \cdot \max(\Vert x_2 \Vert _2, \epsilon)} + +Supports :ref:`type promotion `. + +Args: + x1 (Tensor): First input. + x2 (Tensor): Second input. + dim (int, optional): Dimension along which cosine similarity is computed. Default: 1 + eps (float, optional): Small value to avoid division by zero. + Default: 1e-8 + +Example:: + + >>> input1 = torch.randn(100, 128) + >>> input2 = torch.randn(100, 128) + >>> output = F.cosine_similarity(input1, input2) + >>> print(output) +""", +) + + +one_hot = _add_docstr( + torch._C._nn.one_hot, + r""" +one_hot(tensor, num_classes=-1) -> LongTensor + +Takes LongTensor with index values of shape ``(*)`` and returns a tensor +of shape ``(*, num_classes)`` that have zeros everywhere except where the +index of last dimension matches the corresponding value of the input tensor, +in which case it will be 1. + +See also `One-hot on Wikipedia`_ . + +.. _One-hot on Wikipedia: + https://en.wikipedia.org/wiki/One-hot + +Arguments: + tensor (LongTensor): class values of any shape. + num_classes (int, optional): Total number of classes. If set to -1, the number + of classes will be inferred as one greater than the largest class + value in the input tensor. Default: -1 + +Returns: + LongTensor that has one more dimension with 1 values at the + index of last dimension indicated by the input, and 0 everywhere + else. + +Examples: + >>> F.one_hot(torch.arange(0, 5) % 3) + tensor([[1, 0, 0], + [0, 1, 0], + [0, 0, 1], + [1, 0, 0], + [0, 1, 0]]) + >>> F.one_hot(torch.arange(0, 5) % 3, num_classes=5) + tensor([[1, 0, 0, 0, 0], + [0, 1, 0, 0, 0], + [0, 0, 1, 0, 0], + [1, 0, 0, 0, 0], + [0, 1, 0, 0, 0]]) + >>> F.one_hot(torch.arange(0, 6).view(3,2) % 3) + tensor([[[1, 0, 0], + [0, 1, 0]], + [[0, 0, 1], + [1, 0, 0]], + [[0, 1, 0], + [0, 0, 1]]]) +""", +) + + +def triplet_margin_loss( + anchor: Tensor, + positive: Tensor, + negative: Tensor, + margin: float = 1.0, + p: float = 2, + eps: float = 1e-6, + swap: bool = False, + size_average: Optional[bool] = None, + reduce: Optional[bool] = None, + reduction: str = "mean", +) -> Tensor: + r"""Compute the triplet loss between given input tensors and a margin greater than 0. + + See :class:`~torch.nn.TripletMarginLoss` for details. + """ + if has_torch_function_variadic(anchor, positive, negative): + return handle_torch_function( + triplet_margin_loss, + (anchor, positive, negative), + anchor, + positive, + negative, + margin=margin, + p=p, + eps=eps, + swap=swap, + size_average=size_average, + reduce=reduce, + reduction=reduction, + ) + if size_average is not None or reduce is not None: + reduction_enum = _Reduction.legacy_get_enum(size_average, reduce) + else: + reduction_enum = _Reduction.get_enum(reduction) + if margin <= 0: + raise ValueError(f"margin must be greater than 0, got {margin}") + return torch.triplet_margin_loss( + anchor, positive, negative, margin, p, eps, swap, reduction_enum + ) + + +def triplet_margin_with_distance_loss( + anchor: Tensor, + positive: Tensor, + negative: Tensor, + *, + distance_function: Optional[Callable[[Tensor, Tensor], Tensor]] = None, + margin: float = 1.0, + swap: bool = False, + reduction: str = "mean", +) -> Tensor: + r"""Compute the triplet margin loss for input tensors using a custom distance function. + + See :class:`~torch.nn.TripletMarginWithDistanceLoss` for details. + """ + if torch.jit.is_scripting(): + raise NotImplementedError( + "F.triplet_margin_with_distance_loss does not support JIT scripting: " + "functions requiring Callables cannot be scripted." + ) + + if has_torch_function_variadic(anchor, positive, negative): + return handle_torch_function( + triplet_margin_with_distance_loss, + (anchor, positive, negative), + anchor, + positive, + negative, + distance_function=distance_function, + margin=margin, + swap=swap, + reduction=reduction, + ) + + # Check validity of reduction mode + if reduction not in ("mean", "sum", "none"): + raise ValueError(f"{reduction} is not a valid value for reduction") + + # Check validity of margin + if margin <= 0: + raise ValueError(f"margin must be greater than 0, got {margin}") + + # Check dimensions + a_dim = anchor.ndim + p_dim = positive.ndim + n_dim = negative.ndim + if not (a_dim == p_dim and p_dim == n_dim): + raise RuntimeError( + f"The anchor, positive, and negative tensors are expected to have " + f"the same number of dimensions, but got: anchor {a_dim}D, " + f"positive {p_dim}D, and negative {n_dim}D inputs" + ) + + # Calculate loss + if distance_function is None: + distance_function = torch.pairwise_distance + + dist_pos = distance_function(anchor, positive) + dist_neg = distance_function(anchor, negative) + # The distance swap is described in the paper "Learning shallow + # convolutional feature descriptors with triplet losses" by V. Balntas, E. + # Riba et al. If True, and if the positive example is closer to the + # negative example than the anchor is, swaps the positive example and the + # anchor in the loss computation. + if swap: + dist_swap = distance_function(positive, negative) + dist_neg = torch.minimum(dist_neg, dist_swap) + loss = torch.clamp_min(margin + dist_pos - dist_neg, 0) + + # Apply reduction + if reduction == "sum": + return torch.sum(loss) + elif reduction == "mean": + return torch.mean(loss) + else: # reduction == "none" + return loss + + +def normalize( + input: Tensor, + p: float = 2.0, + dim: int = 1, + eps: float = 1e-12, + out: Optional[Tensor] = None, +) -> Tensor: + r"""Perform :math:`L_p` normalization of inputs over specified dimension. + + For a tensor :attr:`input` of sizes :math:`(n_0, ..., n_{dim}, ..., n_k)`, each + :math:`n_{dim}` -element vector :math:`v` along dimension :attr:`dim` is transformed as + + .. math:: + v = \frac{v}{\max(\lVert v \rVert_p, \epsilon)}. + + With the default arguments it uses the Euclidean norm over vectors along dimension :math:`1` for normalization. + + Args: + input: input tensor of any shape + p (float): the exponent value in the norm formulation. Default: 2 + dim (int or tuple of ints): the dimension to reduce. Default: 1 + eps (float): small value to avoid division by zero. Default: 1e-12 + out (Tensor, optional): the output tensor. If :attr:`out` is used, this + operation won't be differentiable. + """ + if has_torch_function_variadic(input, out): + return handle_torch_function( + normalize, (input, out), input, p=p, dim=dim, eps=eps, out=out + ) + if out is None: + denom = input.norm(p, dim, keepdim=True).clamp_min(eps).expand_as(input) + return input / denom + else: + denom = input.norm(p, dim, keepdim=True).clamp_min_(eps).expand_as(input) + return torch.div(input, denom, out=out) + + +def assert_int_or_pair(arg: list[int], arg_name: str, message: str) -> None: + assert isinstance(arg, int) or len(arg) == 2, message.format(arg_name) + + +def unfold( + input: Tensor, + kernel_size: BroadcastingList2[int], + dilation: BroadcastingList2[int] = 1, + padding: BroadcastingList2[int] = 0, + stride: BroadcastingList2[int] = 1, +) -> Tensor: + r"""Extract sliding local blocks from a batched input tensor. + + .. warning:: + Currently, only 4-D input tensors (batched image-like tensors) are + supported. + + .. warning:: + + More than one element of the unfolded tensor may refer to a single + memory location. As a result, in-place operations (especially ones that + are vectorized) may result in incorrect behavior. If you need to write + to the tensor, please clone it first. + + + See :class:`torch.nn.Unfold` for details + """ + if has_torch_function_unary(input): + return handle_torch_function( + unfold, + (input,), + input, + kernel_size, + dilation=dilation, + padding=padding, + stride=stride, + ) + return torch._C._nn.im2col( + input, _pair(kernel_size), _pair(dilation), _pair(padding), _pair(stride) + ) + + +def fold( + input: Tensor, + output_size: BroadcastingList2[int], + kernel_size: BroadcastingList2[int], + dilation: BroadcastingList2[int] = 1, + padding: BroadcastingList2[int] = 0, + stride: BroadcastingList2[int] = 1, +) -> Tensor: + r"""Combine an array of sliding local blocks into a large containing tensor. + + .. warning:: + Currently, only unbatched (3D) or batched (4D) image-like output tensors are supported. + + See :class:`torch.nn.Fold` for details + """ + if has_torch_function_unary(input): + return handle_torch_function( + fold, + (input,), + input, + output_size, + kernel_size, + dilation=dilation, + padding=padding, + stride=stride, + ) + return torch._C._nn.col2im( + input, + _pair(output_size), + _pair(kernel_size), + _pair(dilation), + _pair(padding), + _pair(stride), + ) + + +# +# multihead attention +# + + +def _in_projection_packed( + q: Tensor, + k: Tensor, + v: Tensor, + w: Tensor, + b: Optional[Tensor] = None, +) -> list[Tensor]: + r"""Perform the in-projection step of the attention operation, using packed weights. + + Output is a triple containing projection tensors for query, key and value. + + Args: + q, k, v: query, key and value tensors to be projected. For self-attention, + these are typically the same tensor; for encoder-decoder attention, + k and v are typically the same tensor. (We take advantage of these + identities for performance if they are present.) Regardless, q, k and v + must share a common embedding dimension; otherwise their shapes may vary. + w: projection weights for q, k and v, packed into a single tensor. Weights + are packed along dimension 0, in q, k, v order. + b: optional projection biases for q, k and v, packed into a single tensor + in q, k, v order. + + Shape: + Inputs: + - q: :math:`(..., E)` where E is the embedding dimension + - k: :math:`(..., E)` where E is the embedding dimension + - v: :math:`(..., E)` where E is the embedding dimension + - w: :math:`(E * 3, E)` where E is the embedding dimension + - b: :math:`E * 3` where E is the embedding dimension + + Output: + - in output list :math:`[q', k', v']`, each output tensor will have the + same shape as the corresponding input tensor. + """ + E = q.size(-1) + if k is v: + if q is k: + # self-attention + proj = linear(q, w, b) + # reshape to 3, E and not E, 3 is deliberate for better memory coalescing and keeping same order as chunk() + proj = ( + proj.unflatten(-1, (3, E)) + .unsqueeze(0) + .transpose(0, -2) + .squeeze(-2) + .contiguous() + ) + # pyrefly: ignore [bad-return] + return proj[0], proj[1], proj[2] + else: + # encoder-decoder attention + w_q, w_kv = w.split([E, E * 2]) + if b is None: + b_q = b_kv = None + else: + b_q, b_kv = b.split([E, E * 2]) + q_proj = linear(q, w_q, b_q) + kv_proj = linear(k, w_kv, b_kv) + # reshape to 2, E and not E, 2 is deliberate for better memory coalescing and keeping same order as chunk() + kv_proj = ( + kv_proj.unflatten(-1, (2, E)) + .unsqueeze(0) + .transpose(0, -2) + .squeeze(-2) + .contiguous() + ) + # pyrefly: ignore [bad-return] + return (q_proj, kv_proj[0], kv_proj[1]) + else: + w_q, w_k, w_v = w.chunk(3) + if b is None: + b_q = b_k = b_v = None + else: + b_q, b_k, b_v = b.chunk(3) + # pyrefly: ignore [bad-return] + return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v) + + +def _in_projection( + q: Tensor, + k: Tensor, + v: Tensor, + w_q: Tensor, + w_k: Tensor, + w_v: Tensor, + b_q: Optional[Tensor] = None, + b_k: Optional[Tensor] = None, + b_v: Optional[Tensor] = None, +) -> tuple[Tensor, Tensor, Tensor]: + r"""Perform the in-projection step of the attention operation. + + This is simply a triple of linear projections, + with shape constraints on the weights which + ensure embedding dimension uniformity in the projected outputs. + Output is a triple containing projection tensors for query, key and value. + + Args: + q, k, v: query, key and value tensors to be projected. + w_q, w_k, w_v: weights for q, k and v, respectively. + b_q, b_k, b_v: optional biases for q, k and v, respectively. + + Shape: + Inputs: + - q: :math:`(Qdims..., Eq)` where Eq is the query embedding dimension and Qdims are any + number of leading dimensions. + - k: :math:`(Kdims..., Ek)` where Ek is the key embedding dimension and Kdims are any + number of leading dimensions. + - v: :math:`(Vdims..., Ev)` where Ev is the value embedding dimension and Vdims are any + number of leading dimensions. + - w_q: :math:`(Eq, Eq)` + - w_k: :math:`(Eq, Ek)` + - w_v: :math:`(Eq, Ev)` + - b_q: :math:`(Eq)` + - b_k: :math:`(Eq)` + - b_v: :math:`(Eq)` + + Output: in output triple :math:`(q', k', v')`, + - q': :math:`[Qdims..., Eq]` + - k': :math:`[Kdims..., Eq]` + - v': :math:`[Vdims..., Eq]` + + """ + Eq, Ek, Ev = q.size(-1), k.size(-1), v.size(-1) + assert w_q.shape == ( + Eq, + Eq, + ), f"expecting query weights shape of {(Eq, Eq)}, but got {w_q.shape}" + assert w_k.shape == ( + Eq, + Ek, + ), f"expecting key weights shape of {(Eq, Ek)}, but got {w_k.shape}" + assert w_v.shape == ( + Eq, + Ev, + ), f"expecting value weights shape of {(Eq, Ev)}, but got {w_v.shape}" + assert b_q is None or b_q.shape == (Eq,), ( + f"expecting query bias shape of {(Eq,)}, but got {b_q.shape}" + ) + assert b_k is None or b_k.shape == (Eq,), ( + f"expecting key bias shape of {(Eq,)}, but got {b_k.shape}" + ) + assert b_v is None or b_v.shape == (Eq,), ( + f"expecting value bias shape of {(Eq,)}, but got {b_v.shape}" + ) + return linear(q, w_q, b_q), linear(k, w_k, b_k), linear(v, w_v, b_v) + + +scaled_dot_product_attention = _add_docstr( + torch._C._nn.scaled_dot_product_attention, + r"""scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.0, + is_causal=False, scale=None, enable_gqa=False) -> Tensor: + + Computes scaled dot product attention on query, key and value tensors, using an optional attention mask if passed, + and applying dropout if a probability greater than 0.0 is specified. The optional scale argument can only be + specified as a keyword argument. + + .. code-block:: python + + # Efficient implementation equivalent to the following: + def scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.0, + is_causal=False, scale=None, enable_gqa=False) -> torch.Tensor: + L, S = query.size(-2), key.size(-2) + scale_factor = 1 / math.sqrt(query.size(-1)) if scale is None else scale + attn_bias = torch.zeros(L, S, dtype=query.dtype, device=query.device) + if is_causal: + assert attn_mask is None + temp_mask = torch.ones(L, S, dtype=torch.bool).tril(diagonal=0) + attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf")) + + if attn_mask is not None: + if attn_mask.dtype == torch.bool: + attn_bias.masked_fill_(attn_mask.logical_not(), float("-inf")) + else: + attn_bias = attn_mask + attn_bias + + if enable_gqa: + key = key.repeat_interleave(query.size(-3)//key.size(-3), -3) + value = value.repeat_interleave(query.size(-3)//value.size(-3), -3) + + attn_weight = query @ key.transpose(-2, -1) * scale_factor + attn_weight += attn_bias + attn_weight = torch.softmax(attn_weight, dim=-1) + attn_weight = torch.dropout(attn_weight, dropout_p, train=True) + return attn_weight @ value + + .. warning:: + This function is beta and subject to change. + + .. warning:: + This function always applies dropout according to the specified ``dropout_p`` argument. + To disable dropout during evaluation, be sure to pass a value of ``0.0`` when the module + that makes the function call is not in training mode. + + For example: + + .. code-block:: python + + class MyModel(nn.Module): + def __init__(self, p=0.5): + super().__init__() + self.p = p + + def forward(self, ...): + return F.scaled_dot_product_attention(..., + dropout_p=(self.p if self.training else 0.0)) + + Note: + + There are currently three supported implementations of scaled dot product attention: + + - `FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning`_ + - `Memory-Efficient Attention`_ + - A PyTorch implementation defined in C++ matching the above formulation + + The function may call optimized kernels for improved performance when using the CUDA backend. + For all other backends, the PyTorch implementation will be used. + + All implementations are enabled by default. Scaled dot product attention attempts to automatically select the + most optimal implementation based on the inputs. In order to provide more fine-grained control over what implementation + is used, the following functions are provided for enabling and disabling implementations. + The context manager is the preferred mechanism: + + - :func:`torch.nn.attention.sdpa_kernel`: A context manager used to enable or disable any of the implementations. + - :func:`torch.backends.cuda.enable_flash_sdp`: Globally enables or disables FlashAttention. + - :func:`torch.backends.cuda.enable_mem_efficient_sdp`: Globally enables or disables Memory-Efficient Attention. + - :func:`torch.backends.cuda.enable_math_sdp`: Globally enables or disables the PyTorch C++ implementation. + + Each of the fused kernels has specific input limitations. If the user requires the use of a specific fused implementation, + disable the PyTorch C++ implementation using :func:`torch.nn.attention.sdpa_kernel`. + In the event that a fused implementation is not available, a warning will be raised with the + reasons why the fused implementation cannot run. + + Due to the nature of fusing floating point operations, the output of this function may be different + depending on what backend kernel is chosen. + The c++ implementation supports torch.float64 and can be used when higher precision is required. + For math backend, all intermediates are kept in torch.float if inputs are in torch.half or torch.bfloat16. + For more information please see :doc:`/notes/numerical_accuracy` + + Grouped Query Attention (GQA) is an experimental feature. It currently works only for Flash_attention + and math kernel on CUDA tensor, and does not support Nested tensor. + Constraints for GQA: + + - number_of_heads_query % number_of_heads_key_value == 0 and, + - number_of_heads_key == number_of_heads_value + + Note: + + {cudnn_reproducibility_note} + """.format(**reproducibility_notes) + + r""" + Args: + query (Tensor): Query tensor; shape :math:`(N, ..., Hq, L, E)`. + key (Tensor): Key tensor; shape :math:`(N, ..., H, S, E)`. + value (Tensor): Value tensor; shape :math:`(N, ..., H, S, Ev)`. + attn_mask (optional Tensor): Attention mask; shape must be broadcastable to the shape of attention weights, + which is :math:`(N,..., L, S)`. Two types of masks are supported. + A boolean mask where a value of True indicates that the element *should* take part in attention. + A float mask of the same type as query, key, value that is added to the attention score. + dropout_p (float): Dropout probability; if greater than 0.0, dropout is applied + is_causal (bool): If set to true, the attention masking is a lower triangular matrix when the mask is a + square matrix. The attention masking has the form of the upper left causal bias due to the alignment + (see :class:`torch.nn.attention.bias.CausalBias`) when the mask is a non-square matrix. + An error is thrown if both attn_mask and is_causal are set. + scale (optional float, keyword-only): Scaling factor applied prior to softmax. If None, the default value is set + to :math:`\frac{1}{\sqrt{E}}`. + enable_gqa (bool): If set to True, Grouped Query Attention (GQA) is enabled, by default it is set to False. + + Returns: + output (Tensor): Attention output; shape :math:`(N, ..., Hq, L, Ev)`. + + Shape legend: + - :math:`N: \text{Batch size} ... : \text{Any number of other batch dimensions (optional)}` + - :math:`S: \text{Source sequence length}` + - :math:`L: \text{Target sequence length}` + - :math:`E: \text{Embedding dimension of the query and key}` + - :math:`Ev: \text{Embedding dimension of the value}` + - :math:`Hq: \text{Number of heads of query}` + - :math:`H: \text{Number of heads of key and value}` + + Examples: + + >>> # Optionally use the context manager to ensure one of the fused kernels is run + >>> query = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") + >>> key = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") + >>> value = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") + >>> with sdpa_kernel(backends=[SDPBackend.FLASH_ATTENTION]): + >>> F.scaled_dot_product_attention(query,key,value) + + + >>> # Sample for GQA for llama3 + >>> query = torch.rand(32, 32, 128, 64, dtype=torch.float16, device="cuda") + >>> key = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") + >>> value = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") + >>> with sdpa_kernel(backends=[SDPBackend.MATH]): + >>> F.scaled_dot_product_attention(query,key,value,enable_gqa=True) + + + .. _FlashAttention-2\: Faster Attention with Better Parallelism and Work Partitioning: + https://arxiv.org/abs/2307.08691 + .. _Memory-Efficient Attention: + https://github.com/facebookresearch/xformers + .. _Grouped-Query Attention: + https://arxiv.org/pdf/2305.13245 + """, +) + + +def _mha_shape_check( + query: Tensor, + key: Tensor, + value: Tensor, + key_padding_mask: Optional[Tensor], + attn_mask: Optional[Tensor], + num_heads: int, +): + # Verifies the expected shape for `query, `key`, `value`, `key_padding_mask` and `attn_mask` + # and returns if the input is batched or not. + # Raises an error if `query` is not 2-D (unbatched) or 3-D (batched) tensor. + + # Shape check. + if query.dim() == 3: + # Batched Inputs + is_batched = True + assert key.dim() == 3 and value.dim() == 3, ( + "For batched (3-D) `query`, expected `key` and `value` to be 3-D" + f" but found {key.dim()}-D and {value.dim()}-D tensors respectively" + ) + if key_padding_mask is not None: + assert key_padding_mask.dim() == 2, ( + "For batched (3-D) `query`, expected `key_padding_mask` to be `None` or 2-D" + f" but found {key_padding_mask.dim()}-D tensor instead" + ) + if attn_mask is not None: + assert attn_mask.dim() in (2, 3), ( + "For batched (3-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D" + f" but found {attn_mask.dim()}-D tensor instead" + ) + elif query.dim() == 2: + # Unbatched Inputs + is_batched = False + assert key.dim() == 2 and value.dim() == 2, ( + "For unbatched (2-D) `query`, expected `key` and `value` to be 2-D" + f" but found {key.dim()}-D and {value.dim()}-D tensors respectively" + ) + + if key_padding_mask is not None: + assert key_padding_mask.dim() == 1, ( + "For unbatched (2-D) `query`, expected `key_padding_mask` to be `None` or 1-D" + f" but found {key_padding_mask.dim()}-D tensor instead" + ) + + if attn_mask is not None: + assert attn_mask.dim() in (2, 3), ( + "For unbatched (2-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D" + f" but found {attn_mask.dim()}-D tensor instead" + ) + if attn_mask.dim() == 3: + expected_shape = (num_heads, query.shape[0], key.shape[0]) + assert attn_mask.shape == expected_shape, ( + f"Expected `attn_mask` shape to be {expected_shape} but got {attn_mask.shape}" + ) + else: + raise AssertionError( + f"query should be unbatched 2D or batched 3D tensor but received {query.dim()}-D query tensor" + ) + + return is_batched + + +def _canonical_mask( + mask: Optional[Tensor], + mask_name: str, + other_type: Optional[DType], + other_name: str, + target_type: DType, + check_other: bool = True, +) -> Optional[Tensor]: + if mask is not None: + _mask_dtype = mask.dtype + _mask_is_float = torch.is_floating_point(mask) + if _mask_dtype != torch.bool and not _mask_is_float: + raise AssertionError( + f"only bool and floating types of {mask_name} are supported" + ) + if check_other and other_type is not None: + if _mask_dtype != other_type: + warnings.warn( + f"Support for mismatched {mask_name} and {other_name} " + "is deprecated. Use same type for both instead.", + stacklevel=2, + ) + if not _mask_is_float: + mask = torch.zeros_like(mask, dtype=target_type).masked_fill_( + mask, float("-inf") + ) + return mask + + +def _none_or_dtype(input: Optional[Tensor]) -> Optional[DType]: + if input is None: + return None + elif isinstance(input, torch.Tensor): + return input.dtype + raise RuntimeError("input to _none_or_dtype() must be None or torch.Tensor") + + +def _check_key_padding_mask( + key_padding_mask: torch.Tensor, src_len: int, bsz: int +) -> None: + torch._check_with( + AssertionError, + key_padding_mask.shape[0] == bsz, + lambda: f"Expected key_padded_mask.shape[0] to be {bsz}, but got {key_padding_mask.shape[0]}", + ) + torch._check_with( + AssertionError, + key_padding_mask.shape[1] == src_len, + lambda: f"Expected key_padded_mask.shape[1] to be {src_len}, but got {key_padding_mask.shape[1]}", + ) + + +def multi_head_attention_forward( + query: Tensor, + key: Tensor, + value: Tensor, + embed_dim_to_check: int, + num_heads: int, + in_proj_weight: Optional[Tensor], + in_proj_bias: Optional[Tensor], + bias_k: Optional[Tensor], + bias_v: Optional[Tensor], + add_zero_attn: bool, + dropout_p: float, + out_proj_weight: Tensor, + out_proj_bias: Optional[Tensor], + training: bool = True, + key_padding_mask: Optional[Tensor] = None, + need_weights: bool = True, + attn_mask: Optional[Tensor] = None, + use_separate_proj_weight: bool = False, + q_proj_weight: Optional[Tensor] = None, + k_proj_weight: Optional[Tensor] = None, + v_proj_weight: Optional[Tensor] = None, + static_k: Optional[Tensor] = None, + static_v: Optional[Tensor] = None, + average_attn_weights: bool = True, + is_causal: bool = False, +) -> tuple[Tensor, Optional[Tensor]]: + r"""Forward method for MultiHeadAttention. + + See :class:`torch.nn.MultiheadAttention` for details. + + Args: + query, key, value: map a query and a set of key-value pairs to an output. + See "Attention Is All You Need" for more details. + embed_dim_to_check: total dimension of the model. + num_heads: parallel attention heads. + in_proj_weight, in_proj_bias: input projection weight and bias. + bias_k, bias_v: bias of the key and value sequences to be added at dim=0. + add_zero_attn: add a new batch of zeros to the key and + value sequences at dim=1. + dropout_p: probability of an element to be zeroed. + out_proj_weight, out_proj_bias: the output projection weight and bias. + training: apply dropout if is ``True``. + key_padding_mask: if provided, specified padding elements in the key will + be ignored by the attention. This is an binary mask. When the value is True, + the corresponding value on the attention layer will be filled with -inf. + need_weights: output attn_output_weights. + Default: `True` + Note: `needs_weight` defaults to `True`, but should be set to `False` + For best performance when attention weights are not needed. + *Setting needs_weights to `True` + leads to a significant performance degradation.* + attn_mask: 2D or 3D mask that prevents attention to certain positions. A 2D mask will be broadcasted for all + the batches while a 3D mask allows to specify a different mask for the entries of each batch. + is_causal: If specified, applies a causal mask as attention mask, and ignores + attn_mask for computing scaled dot product attention. + Default: ``False``. + .. warning:: + is_causal is provides a hint that the attn_mask is the + causal mask.Providing incorrect hints can result in + incorrect execution, including forward and backward + compatibility. + use_separate_proj_weight: the function accept the proj. weights for query, key, + and value in different forms. If false, in_proj_weight will be used, which is + a combination of q_proj_weight, k_proj_weight, v_proj_weight. + q_proj_weight, k_proj_weight, v_proj_weight, in_proj_bias: input projection weight and bias. + static_k, static_v: static key and value used for attention operators. + average_attn_weights: If true, indicates that the returned ``attn_weights`` should be averaged across heads. + Otherwise, ``attn_weights`` are provided separately per head. Note that this flag only has an effect + when ``need_weights=True.``. Default: True + + + Shape: + Inputs: + - query: :math:`(L, E)` or :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is + the embedding dimension. + - key: :math:`(S, E)` or :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is + the embedding dimension. + - value: :math:`(S, E)` or :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is + the embedding dimension. + - key_padding_mask: :math:`(S)` or :math:`(N, S)` where N is the batch size, S is the source sequence length. + If a FloatTensor is provided, it will be directly added to the value. + If a BoolTensor is provided, the positions with the + value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged. + - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length. + 3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length, + S is the source sequence length. attn_mask ensures that position i is allowed to attend the unmasked + positions. If a BoolTensor is provided, positions with ``True`` + are not allowed to attend while ``False`` values will be unchanged. If a FloatTensor + is provided, it will be added to the attention weight. + - static_k: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length, + N is the batch size, E is the embedding dimension. E/num_heads is the head dimension. + - static_v: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length, + N is the batch size, E is the embedding dimension. E/num_heads is the head dimension. + + Outputs: + - attn_output: :math:`(L, E)` or :math:`(L, N, E)` where L is the target sequence length, N is the batch size, + E is the embedding dimension. + - attn_output_weights: Only returned when ``need_weights=True``. If ``average_attn_weights=True``, returns + attention weights averaged across heads of shape :math:`(L, S)` when input is unbatched or + :math:`(N, L, S)`, where :math:`N` is the batch size, :math:`L` is the target sequence length, and + :math:`S` is the source sequence length. If ``average_attn_weights=False``, returns attention weights per + head of shape :math:`(num_heads, L, S)` when input is unbatched or :math:`(N, num_heads, L, S)`. + """ + tens_ops = ( + query, + key, + value, + in_proj_weight, + in_proj_bias, + bias_k, + bias_v, + out_proj_weight, + out_proj_bias, + ) + if has_torch_function(tens_ops): + return handle_torch_function( + multi_head_attention_forward, + tens_ops, + query, + key, + value, + embed_dim_to_check, + num_heads, + in_proj_weight, + in_proj_bias, + bias_k, + bias_v, + add_zero_attn, + dropout_p, + out_proj_weight, + out_proj_bias, + training=training, + key_padding_mask=key_padding_mask, + need_weights=need_weights, + attn_mask=attn_mask, + is_causal=is_causal, + use_separate_proj_weight=use_separate_proj_weight, + q_proj_weight=q_proj_weight, + k_proj_weight=k_proj_weight, + v_proj_weight=v_proj_weight, + static_k=static_k, + static_v=static_v, + average_attn_weights=average_attn_weights, + ) + + is_batched = _mha_shape_check( + query, key, value, key_padding_mask, attn_mask, num_heads + ) + + # For unbatched input, we unsqueeze at the expected batch-dim to pretend that the input + # is batched, run the computation and before returning squeeze the + # batch dimension so that the output doesn't carry this temporary batch dimension. + if not is_batched: + # unsqueeze if the input is unbatched + query = query.unsqueeze(1) + key = key.unsqueeze(1) + value = value.unsqueeze(1) + if key_padding_mask is not None: + key_padding_mask = key_padding_mask.unsqueeze(0) + + # set up shape vars + tgt_len, bsz, embed_dim = query.shape + src_len, _, _ = key.shape + + key_padding_mask = _canonical_mask( + mask=key_padding_mask, + mask_name="key_padding_mask", + other_type=_none_or_dtype(attn_mask), + other_name="attn_mask", + target_type=query.dtype, + ) + + if is_causal and attn_mask is None: + raise RuntimeError( + "Need attn_mask if specifying the is_causal hint. " + "You may use the Transformer module method " + "`generate_square_subsequent_mask` to create this mask." + ) + + if is_causal and key_padding_mask is None and not need_weights: + # when we have a kpm or need weights, we need attn_mask + # Otherwise, we use the is_causal hint go as is_causal + # indicator to SDPA. + attn_mask = None + else: + attn_mask = _canonical_mask( + mask=attn_mask, + mask_name="attn_mask", + other_type=None, + other_name="", + target_type=query.dtype, + check_other=False, + ) + + if key_padding_mask is not None: + # We have the attn_mask, and use that to merge kpm into it. + # Turn off use of is_causal hint, as the merged mask is no + # longer causal. + is_causal = False + + assert embed_dim == embed_dim_to_check, ( + f"was expecting embedding dimension of {embed_dim_to_check}, but got {embed_dim}" + ) + if isinstance(embed_dim, torch.Tensor): + # embed_dim can be a tensor when JIT tracing + head_dim = embed_dim.div(num_heads, rounding_mode="trunc") + else: + head_dim = embed_dim // num_heads + assert head_dim * num_heads == embed_dim, ( + f"embed_dim {embed_dim} not divisible by num_heads {num_heads}" + ) + if use_separate_proj_weight: + # allow MHA to have different embedding dimensions when separate projection weights are used + assert key.shape[:2] == value.shape[:2], ( + f"key's sequence and batch dims {key.shape[:2]} do not match value's {value.shape[:2]}" + ) + else: + assert key.shape == value.shape, ( + f"key shape {key.shape} does not match value shape {value.shape}" + ) + + # + # compute in-projection + # + if not use_separate_proj_weight: + assert in_proj_weight is not None, ( + "use_separate_proj_weight is False but in_proj_weight is None" + ) + q, k, v = _in_projection_packed(query, key, value, in_proj_weight, in_proj_bias) + else: + assert q_proj_weight is not None, ( + "use_separate_proj_weight is True but q_proj_weight is None" + ) + assert k_proj_weight is not None, ( + "use_separate_proj_weight is True but k_proj_weight is None" + ) + assert v_proj_weight is not None, ( + "use_separate_proj_weight is True but v_proj_weight is None" + ) + if in_proj_bias is None: + b_q = b_k = b_v = None + else: + b_q, b_k, b_v = in_proj_bias.chunk(3) + q, k, v = _in_projection( + query, + key, + value, + q_proj_weight, + k_proj_weight, + v_proj_weight, + b_q, + b_k, + b_v, + ) + + # prep attention mask + + if attn_mask is not None: + # ensure attn_mask's dim is 3 + if attn_mask.dim() == 2: + correct_2d_size = (tgt_len, src_len) + if attn_mask.shape != correct_2d_size: + raise RuntimeError( + f"The shape of the 2D attn_mask is {attn_mask.shape}, but should be {correct_2d_size}." + ) + attn_mask = attn_mask.unsqueeze(0) + elif attn_mask.dim() == 3: + correct_3d_size = (bsz * num_heads, tgt_len, src_len) + if attn_mask.shape != correct_3d_size: + raise RuntimeError( + f"The shape of the 3D attn_mask is {attn_mask.shape}, but should be {correct_3d_size}." + ) + else: + raise RuntimeError( + f"attn_mask's dimension {attn_mask.dim()} is not supported" + ) + + # add bias along batch dimension (currently second) + if bias_k is not None and bias_v is not None: + assert static_k is None, "bias cannot be added to static key." + assert static_v is None, "bias cannot be added to static value." + k = torch.cat([k, bias_k.repeat(1, bsz, 1)]) + v = torch.cat([v, bias_v.repeat(1, bsz, 1)]) + if attn_mask is not None: + # pyrefly: ignore [bad-argument-type] + attn_mask = pad(attn_mask, (0, 1)) + if key_padding_mask is not None: + # pyrefly: ignore [bad-argument-type] + key_padding_mask = pad(key_padding_mask, (0, 1)) + else: + assert bias_k is None + assert bias_v is None + + # + # reshape q, k, v for multihead attention and make them batch first + # + # pyrefly: ignore [no-matching-overload] + q = q.view(tgt_len, bsz * num_heads, head_dim).transpose(0, 1) + if static_k is None: + # pyrefly: ignore [no-matching-overload] + k = k.view(k.shape[0], bsz * num_heads, head_dim).transpose(0, 1) + else: + # TODO finish disentangling control flow so we don't do in-projections when statics are passed + assert static_k.size(0) == bsz * num_heads, ( + f"expecting static_k.size(0) of {bsz * num_heads}, but got {static_k.size(0)}" + ) + assert static_k.size(2) == head_dim, ( + f"expecting static_k.size(2) of {head_dim}, but got {static_k.size(2)}" + ) + k = static_k + if static_v is None: + # pyrefly: ignore [no-matching-overload] + v = v.view(v.shape[0], bsz * num_heads, head_dim).transpose(0, 1) + else: + # TODO finish disentangling control flow so we don't do in-projections when statics are passed + assert static_v.size(0) == bsz * num_heads, ( + f"expecting static_v.size(0) of {bsz * num_heads}, but got {static_v.size(0)}" + ) + assert static_v.size(2) == head_dim, ( + f"expecting static_v.size(2) of {head_dim}, but got {static_v.size(2)}" + ) + v = static_v + + # add zero attention along batch dimension (now first) + if add_zero_attn: + zero_attn_shape = (bsz * num_heads, 1, head_dim) + k = torch.cat( + # pyrefly: ignore [no-matching-overload] + [k, torch.zeros(zero_attn_shape, dtype=k.dtype, device=k.device)], + dim=1, + ) + v = torch.cat( + # pyrefly: ignore [no-matching-overload] + [v, torch.zeros(zero_attn_shape, dtype=v.dtype, device=v.device)], + dim=1, + ) + if attn_mask is not None: + # pyrefly: ignore [bad-argument-type] + attn_mask = pad(attn_mask, (0, 1)) + if key_padding_mask is not None: + # pyrefly: ignore [bad-argument-type] + key_padding_mask = pad(key_padding_mask, (0, 1)) + + # update source sequence length after adjustments + src_len = k.size(1) + + # merge key padding and attention masks + if key_padding_mask is not None: + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _check_key_padding_mask(key_padding_mask, src_len, bsz) + + key_padding_mask = ( + key_padding_mask.view(bsz, 1, 1, src_len) + .expand(-1, num_heads, -1, -1) + .reshape(bsz * num_heads, 1, src_len) + ) + if attn_mask is None: + attn_mask = key_padding_mask + else: + attn_mask = attn_mask + key_padding_mask + + # adjust dropout probability + if not training: + dropout_p = 0.0 + + # + # (deep breath) calculate attention and out projection + # + + if need_weights: + _B, _Nt, E = q.shape + q_scaled = q * math.sqrt(1.0 / float(E)) + + assert not (is_causal and attn_mask is None), ( + "FIXME: is_causal not implemented for need_weights" + ) + + if attn_mask is not None: + attn_output_weights = torch.baddbmm( + attn_mask, q_scaled, k.transpose(-2, -1) + ) + else: + attn_output_weights = torch.bmm(q_scaled, k.transpose(-2, -1)) + attn_output_weights = softmax(attn_output_weights, dim=-1) + if dropout_p > 0.0: + attn_output_weights = dropout(attn_output_weights, p=dropout_p) + + attn_output = torch.bmm(attn_output_weights, v) + + attn_output = ( + # pyrefly: ignore [no-matching-overload] + attn_output.transpose(0, 1).contiguous().view(tgt_len * bsz, embed_dim) + ) + attn_output = linear(attn_output, out_proj_weight, out_proj_bias) + attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1)) + + # optionally average attention weights over heads + attn_output_weights = attn_output_weights.view(bsz, num_heads, tgt_len, src_len) + if average_attn_weights: + attn_output_weights = attn_output_weights.mean(dim=1) + + if not is_batched: + # squeeze the output if input was unbatched + attn_output = attn_output.squeeze(1) + attn_output_weights = attn_output_weights.squeeze(0) + return attn_output, attn_output_weights + else: + # attn_mask can be either (L,S) or (N*num_heads, L, S) + # if attn_mask's shape is (1, L, S) we need to unsqueeze to (1, 1, L, S) + # in order to match the input for SDPA of (N, num_heads, L, S) + if attn_mask is not None: + if attn_mask.size(0) == 1 and attn_mask.dim() == 3: + attn_mask = attn_mask.unsqueeze(0) + else: + attn_mask = attn_mask.view(bsz, num_heads, -1, src_len) + + q = q.view(bsz, num_heads, tgt_len, head_dim) + # pyrefly: ignore [no-matching-overload] + k = k.view(bsz, num_heads, src_len, head_dim) + # pyrefly: ignore [no-matching-overload] + v = v.view(bsz, num_heads, src_len, head_dim) + + attn_output = scaled_dot_product_attention( + q, k, v, attn_mask, dropout_p, is_causal + ) + attn_output = ( + # pyrefly: ignore [no-matching-overload] + attn_output.permute(2, 0, 1, 3).contiguous().view(bsz * tgt_len, embed_dim) + ) + + attn_output = linear(attn_output, out_proj_weight, out_proj_bias) + attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1)) + if not is_batched: + # squeeze the output if input was unbatched + attn_output = attn_output.squeeze(1) + return attn_output, None + + +def grouped_mm( + mat_a: Tensor, + mat_b: Tensor, + *, + offs: Optional[Tensor] = None, + bias: Optional[Tensor] = None, + out_dtype: Optional[torch.dtype] = None, +) -> Tensor: + r""" + grouped_mm(mat_a, mat_b, *, offs=None, bias=None, out_dtype=None) + + Computes a grouped matrix multiply that shares weight shapes across experts but + allows jagged token counts per expert, which is common in Mixture-of-Experts + (MoE) layers. Both ``mat_a`` and ``mat_b`` must be 2D or 3D tensors that already + satisfy the physical layout restrictions of grouped GEMM kernels (e.g., row-major + ``mat_a`` and column-major ``mat_b`` for FP8 inputs). Inputs are currently + expected to be ``torch.bfloat16`` values on CUDA devices with :math:`SM \ge 80`. + + Args: + mat_a: Left operand. When 2D, its leading dimension is sliced into groups + according to ``offs``. When 3D, its first dimension enumerates the groups + directly and ``offs`` must be ``None``. + mat_b: Right operand. When both operands are 2D (e.g., MoE weight-gradient + updates), the trailing dimension of ``mat_a`` and the leading dimension of + ``mat_b`` are partitioned according to the same ``offs`` tensor. For the + common forward pass (``out = input @ weight.T``) ``mat_b`` is 3D with + shape ``(num_groups, N, K)``. + offs: Optional 1D tensor of monotonically increasing ``int32`` offsets that + delimit the jagged dimension of any 2D operand. ``offs[i]`` marks the end + of group ``i`` and ``offs[-1]`` must be strictly less than the total + length of that operand's sliced dimension; elements beyond ``offs[-1]`` + are ignored. + bias: Optional tensor that is added to the grouped outputs. Bias is not + jagged and must be broadcastable to the result shape of each group. + out_dtype: Optional dtype that controls the accumulation/output dtype. + Passing ``torch.float32`` accumulates BF16 inputs in FP32 while keeping + the grouped GEMM API non-differentiable. + + Returns: + A tensor containing the concatenated results of each per-group GEMM with + shape inferred from the operands and ``offs``. + """ + + return torch._grouped_mm(mat_a, mat_b, offs=offs, bias=bias, out_dtype=out_dtype) + + +def scaled_mm( + mat_a: Tensor, + mat_b: Tensor, + scale_a: Tensor | list[Tensor], + scale_recipe_a: ScalingType | list[ScalingType], + scale_b: Tensor | list[Tensor], + scale_recipe_b: ScalingType | list[ScalingType], + swizzle_a: SwizzleType | list[SwizzleType] | None = None, + swizzle_b: SwizzleType | list[SwizzleType] | None = None, + bias: Optional[Tensor] = None, + output_dtype: Optional[torch.dtype] = torch.bfloat16, + contraction_dim: list[int] | tuple[int, ...] = (), + use_fast_accum: bool = False, +) -> Tensor: + r""" + scaled_mm(mat_a, mat_b, scale_a, scale_recipe_a, scale_b, scale_recipe_b, swizzle_a, swizzle_b, bias, output_dtype, + contraction_dim, use_fast_accum) + + Applies a scaled matrix-multiply, mm(mat_a, mat_b) where the scaling of mat_a and mat_b are described by + scale_recipe_a and scale_recipe_b respectively. + + Args: + scale_a: Tensor containing decoding scaling factors for mat_a + scale_recipe_a: Enum describing how mat_a has been scaled + scale_b: Tensor containing decoding scaling factors for mat_b + scale_recipe_b: Enum describing how mat_b has been scaled + swizzle_a: Enum describing the swizzling pattern (if any) of scale_a + swizzle_b: Enum describing the swizzling pattern (if any) of scale_b + bias: optional bias term to be added to the output + output_dtype: dtype used for the output tensor + contraction_dim: describe which dimensions are :math:`K` in the matmul. + use_fast_accum: enable/disable tensor-core fast accumulation (Hopper-GPUs only) + """ + + def expand_single_value(v: _Any | list[_Any] | None) -> list[_Any]: + if v is None: + return [] + elif not isinstance(v, (list)): + return [ + v, + ] + else: + return v + + scale_a = expand_single_value(scale_a) + scale_recipe_a = expand_single_value(scale_recipe_a) + scale_b = expand_single_value(scale_b) + scale_recipe_b = expand_single_value(scale_recipe_b) + swizzle_a = expand_single_value(swizzle_a) + swizzle_b = expand_single_value(swizzle_b) + + # native_functions has restrictions on what can be defined + # & passed through - std::optional> for instance + # *cannot* be passed, but an empty vector (list) can. + # So, we need to convert None arguments for lists in python + # explicitly into empty lists. + def list_or_empty(l: list[_Any] | None) -> list[_Any]: + return l if l else [] + + def enum_list_as_int_list(l: _Any | list[_Any]) -> list[_Any]: + if not isinstance(l, list): + l = [ + l, + ] + return [li.value for li in l] + + out = torch._scaled_mm_v2( + mat_a, + mat_b, + scale_a, + enum_list_as_int_list(scale_recipe_a), + enum_list_as_int_list(list_or_empty(swizzle_a)), + scale_b, + enum_list_as_int_list(scale_recipe_b), + enum_list_as_int_list(list_or_empty(swizzle_b)), + bias, + output_dtype, + contraction_dim, + use_fast_accum, + ) + + return out + + +def scaled_grouped_mm( + mat_a: Tensor, + mat_b: Tensor, + scale_a: Tensor | list[Tensor], + scale_recipe_a: ScalingType | list[ScalingType], + scale_b: Tensor | list[Tensor], + scale_recipe_b: ScalingType | list[ScalingType], + swizzle_a: SwizzleType | list[SwizzleType] | None = None, + swizzle_b: SwizzleType | list[SwizzleType] | None = None, + bias: Optional[Tensor] = None, + offs: Optional[Tensor] = None, + output_dtype: Optional[torch.dtype] = torch.bfloat16, + contraction_dim: list[int] | tuple[int, ...] = (), + use_fast_accum: bool = False, +) -> Tensor: + r""" + scaled_grouped_mm(mat_a, mat_b, scale_a, scale_recipe_a, scale_b, scale_recipe_b, swizzle_a, swizzle_b, bias, offs, + output_dtype, use_fast_accum) + + Applies a grouped scaled matrix-multiply, grouped_mm(mat_a, mat_b) where the scaling of mat_a and mat_b are described by + scale_recipe_a and scale_recipe_b respectively. + + Args: + scale_a: Tensor containing decoding scaling factors for mat_a + scale_recipe_a: Enum describing how mat_a has been scaled + scale_b: Tensor containing decoding scaling factors for mat_b + scale_recipe_b: Enum describing how mat_b has been scaled + swizzle_a: Enum describing the swizzling pattern (if any) of scale_a + swizzle_b: Enum describing the swizzling pattern (if any) of scale_b + bias: optional bias term to be added to the output + offs: optional offsets into the source tensors denoting group start indices + output_dtype: dtype used for the output tensor + contraction_dim: describe which dimensions are :math:`K` in the matmul. + use_fast_accum: enable/disable tensor-core fast accumulation (Hopper-GPUs only) + """ + + def expand_single_value(v: _Any | list[_Any] | None) -> list[_Any]: + if v is None: + return [] + elif not isinstance(v, (list)): + return [ + v, + ] + else: + return v + + scale_a = expand_single_value(scale_a) + scale_recipe_a = expand_single_value(scale_recipe_a) + scale_b = expand_single_value(scale_b) + scale_recipe_b = expand_single_value(scale_recipe_b) + swizzle_a = expand_single_value(swizzle_a) + swizzle_b = expand_single_value(swizzle_b) + + # native_functions has restrictions on what can be defined + # & passed through - std::optional> for instance + # *cannot* be passed, but an empty vector (list) can. + # So, we need to convert None arguments for lists in python + # explicitly into empty lists. + def list_or_empty(l: list[_Any] | None) -> list[_Any]: + return l if l else [] + + def enum_list_as_int_list(l: _Any | list[_Any]) -> list[_Any]: + if not isinstance(l, list): + l = [ + l, + ] + return [li.value for li in l] + + out = torch._scaled_grouped_mm_v2( + mat_a, + mat_b, + scale_a, + enum_list_as_int_list(scale_recipe_a), + enum_list_as_int_list(list_or_empty(swizzle_a)), + scale_b, + enum_list_as_int_list(scale_recipe_b), + enum_list_as_int_list(list_or_empty(swizzle_b)), + offs, + bias, + output_dtype, + contraction_dim, + use_fast_accum, + ) + + return out diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/functional.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/functional.pyi new file mode 100644 index 0000000000000000000000000000000000000000..5a7e357af73cf13f38890cedd05338cb0b04b05a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/functional.pyi @@ -0,0 +1,1200 @@ +# @generated by tools/pyi/gen_pyi.py from torch/nn/functional.pyi.in +# mypy: allow-untyped-defs + +from collections.abc import Callable, Sequence +from enum import Enum +from typing import Any, Literal, overload, TypeAlias + +from torch import Tensor +from torch.types import _dtype, _int, _size + +from .common_types import ( + _ratio_any_t, + _size_1_t, + _size_2_opt_t, + _size_2_t, + _size_3_opt_t, + _size_3_t, + _size_any_t, +) + +__all__ = [ + "GRID_SAMPLE_INTERPOLATION_MODES", + "GRID_SAMPLE_PADDING_MODES", +] + +# 'TypedDict' is a new accepted type that represents a dictionary with a fixed set of allowed keys. +# It is standards-track but not in `typing` yet. We leave this hear to be uncommented once the feature +# is wide-spread. + +# from mypy_extensions import TypedDict + +# GRID_SAMPLE_INTERPOLATION_MODES = TypedDict('GRID_SAMPLE_INTERPOLATION_MODES', {'bilinear': int, 'nearest': int}) +# GRID_SAMPLE_PADDING_MODES = TypedDict('GRID_SAMPLE_PADDING_MODES', {'zeros': int, 'border': int, 'reflection': int}) + +GRID_SAMPLE_INTERPOLATION_MODES: TypeAlias = dict[str, int] +GRID_SAMPLE_PADDING_MODES: TypeAlias = dict[str, int] + +# These stubs were generated by running stubgen (`stubgen --parse-only functional.py`), followed by manual cleaning. +# +# The 'BroadcastingList{1,2,3}' types were replaced by `_size` or _output_ratio, as appropriate. +# This was necessary since the JIT uses BroadcastingList* types but static checking with mypy etc requires a `Sequence` +# type. There is no way to express the expected lengths of these lists in the current Python typing system. +# +# Functions created via `_add_docstr` in `functional.py` where merely typed as `Any` by `stubgen`, so those were +# deleted from the stub and replaced by generated declarations. See `gen_pyi` for the implementation of the code +# generation logic for those functions. In the future, it might be worth looking into using the mypy plugin system +# to encode the type semantics of `_add_docstr`, should that system ever become widespread. +def _canonical_mask( + mask: Tensor | None, + mask_name: str, + other_type: _dtype | None, + other_name: str, + target_type: _dtype, + check_other: bool = True, +) -> Tensor | None: ... + +__all__ += ["_canonical_mask"] + +def _none_or_dtype(input: Tensor | None) -> _dtype | None: ... + +__all__ += ["_none_or_dtype"] + +def adaptive_avg_pool2d(input: Tensor, output_size: _size_2_opt_t) -> Tensor: ... + +__all__ += ["adaptive_avg_pool2d"] + +def adaptive_avg_pool3d(input: Tensor, output_size: _size_3_opt_t) -> Tensor: ... + +__all__ += ["adaptive_avg_pool3d"] + +def adaptive_max_pool1d_with_indices( + input: Tensor, + output_size: _size, + return_indices: bool = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["adaptive_max_pool1d_with_indices"] + +def adaptive_max_pool2d_with_indices( + input: Tensor, + output_size: _size_2_opt_t, + return_indices: bool = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["adaptive_max_pool2d_with_indices"] + +def adaptive_max_pool3d_with_indices( + input: Tensor, + output_size: _size_3_opt_t, + return_indices: bool = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["adaptive_max_pool3d_with_indices"] + +def affine_grid( + theta: Tensor, + size: list[int], + align_corners: Any | None = ..., +) -> Tensor: ... + +__all__ += ["affine_grid"] + +def alpha_dropout( + input: Tensor, + p: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["alpha_dropout"] + +def assert_int_or_pair(arg: Any, arg_name: Any, message: Any) -> None: ... + +__all__ += ["assert_int_or_pair"] + +def batch_norm( + input: Tensor, + running_mean: Tensor | None, + running_var: Tensor | None, + weight: Tensor | None = ..., + bias: Tensor | None = ..., + training: bool = ..., + momentum: float = ..., + eps: float = ..., +) -> Tensor: ... + +__all__ += ["batch_norm"] + +def binary_cross_entropy_with_logits( + input: Tensor, + target: Tensor, + weight: Tensor | None = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., + pos_weight: Tensor | None = ..., +) -> Tensor: ... + +__all__ += ["binary_cross_entropy_with_logits"] + +def binary_cross_entropy( + input: Tensor, + target: Tensor, + weight: Tensor | None = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["binary_cross_entropy"] + +def celu(input: Tensor, alpha: float = ..., inplace: bool = ...) -> Tensor: ... + +__all__ += ["celu"] + +def cosine_embedding_loss( + input1: Tensor, + input2: Tensor, + target: Tensor, + margin: float = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["cosine_embedding_loss"] + +def cross_entropy( + input: Tensor, + target: Tensor, + weight: Tensor | None = ..., + size_average: bool | None = ..., + ignore_index: int = ..., + reduce: bool | None = ..., + reduction: str = ..., + label_smoothing: float = ..., +) -> Tensor: ... + +__all__ += ["cross_entropy"] + +def ctc_loss( + log_probs: Tensor, + targets: Tensor, + input_lengths: Tensor, + target_lengths: Tensor, + blank: int = ..., + reduction: str = ..., + zero_infinity: bool = ..., +) -> Tensor: ... + +__all__ += ["ctc_loss"] + +def dropout( + input: Tensor, + p: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["dropout"] + +def dropout1d( + input: Tensor, + p: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["dropout1d"] + +def dropout2d( + input: Tensor, + p: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["dropout2d"] + +def dropout3d( + input: Tensor, + p: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["dropout3d"] + +def elu(input: Tensor, alpha: float = ..., inplace: bool = ...) -> Tensor: ... + +__all__ += ["elu"] + +def embedding_bag( + input: Tensor, + weight: Tensor, + offsets: Tensor | None = ..., + max_norm: float | None = ..., + norm_type: float = ..., + scale_grad_by_freq: bool = ..., + mode: str = ..., + sparse: bool = ..., + per_sample_weights: Tensor | None = ..., + include_last_offset: bool = ..., + padding_idx: int | None = ..., +) -> Tensor: ... + +__all__ += ["embedding_bag"] + +def embedding( + input: Tensor, + weight: Tensor, + padding_idx: int | None = ..., + max_norm: float | None = ..., + norm_type: float = ..., + scale_grad_by_freq: bool = ..., + sparse: bool = ..., +) -> Tensor: ... + +__all__ += ["embedding"] + +def feature_alpha_dropout( + input: Tensor, + p: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["feature_alpha_dropout"] + +def fold( + input: Tensor, + output_size: _size_any_t, + kernel_size: _size_any_t, + dilation: _size_any_t = ..., + padding: _size_any_t = ..., + stride: _size_any_t = ..., +) -> Tensor: ... + +__all__ += ["fold"] + +def fractional_max_pool2d_with_indices( + input: Tensor, + kernel_size: _size, + output_size: _size | None = ..., + output_ratio: _ratio_any_t | None = ..., + return_indices: bool = ..., + _random_samples: Tensor | None = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["fractional_max_pool2d_with_indices"] + +def fractional_max_pool3d_with_indices( + input: Tensor, + kernel_size: _size, + output_size: _size | None = ..., + output_ratio: _ratio_any_t | None = ..., + return_indices: bool = ..., + _random_samples: Tensor | None = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["fractional_max_pool3d_with_indices"] + +def gaussian_nll_loss( + input: Tensor, + target: Tensor, + var: Tensor | float, + full: bool | None = ..., + eps: float | None = ..., + reduction: str | None = ..., +) -> Tensor: ... + +__all__ += ["gaussian_nll_loss"] + +def glu(input: Tensor, dim: int = ...) -> Tensor: ... + +__all__ += ["glu"] + +def grid_sample( + input: Tensor, + grid: Tensor, + mode: str = ..., + padding_mode: str = ..., + align_corners: Any | None = ..., +) -> Tensor: ... + +__all__ += ["grid_sample"] + +def group_norm( + input: Tensor, + num_groups: int, + weight: Tensor | None = ..., + bias: Tensor | None = ..., + eps: float = ..., +) -> Tensor: ... + +__all__ += ["group_norm"] + +def gumbel_softmax( + logits: Tensor, + tau: float = ..., + hard: bool = ..., + eps: float = ..., + dim: int = ..., +) -> Tensor: ... + +__all__ += ["gumbel_softmax"] + +def hardsigmoid(input: Tensor, inplace: bool = False) -> Tensor: ... + +__all__ += ["hardsigmoid"] + +def hardswish(input: Tensor, inplace: bool = False) -> Tensor: ... + +__all__ += ["hardswish"] + +def hardtanh( + input: Tensor, + min_val: float = ..., + max_val: float = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["hardtanh"] + +def hinge_embedding_loss( + input: Tensor, + target: Tensor, + margin: float = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["hinge_embedding_loss"] + +def huber_loss( + input: Tensor, + target: Tensor, + reduction: str = ..., + delta: float = ..., +) -> Tensor: ... + +__all__ += ["huber_loss"] + +def instance_norm( + input: Tensor, + running_mean: Tensor | None = ..., + running_var: Tensor | None = ..., + weight: Tensor | None = ..., + bias: Tensor | None = ..., + use_input_stats: bool = ..., + momentum: float = ..., + eps: float = ..., +) -> Tensor: ... + +__all__ += ["instance_norm"] + +def interpolate( + input: Tensor, + size: int | Sequence[int] | None = ..., + scale_factor: float | Sequence[float] | None = ..., + mode: str = ..., + align_corners: bool | None = ..., + recompute_scale_factor: bool | None = ..., + antialias: bool = ..., +) -> Tensor: ... + +__all__ += ["interpolate"] + +def kl_div( + input: Tensor, + target: Tensor, + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., + log_target: bool = ..., +) -> Tensor: ... + +__all__ += ["kl_div"] + +def l1_loss( + input: Tensor, + target: Tensor, + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["l1_loss"] + +def layer_norm( + input: Tensor, + normalized_shape: Sequence[int], + weight: Tensor | None = ..., + bias: Tensor | None = ..., + eps: float = ..., +) -> Tensor: ... + +__all__ += ["layer_norm"] + +def leaky_relu( + input: Tensor, + negative_slope: float = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["leaky_relu"] + +def local_response_norm( + input: Tensor, + size: int, + alpha: float = ..., + beta: float = ..., + k: float = ..., +) -> Tensor: ... + +__all__ += ["local_response_norm"] + +def log_softmax( + input: Tensor, + dim: int | None = ..., + _stacklevel: int = ..., + dtype: _dtype | None = ..., +) -> Tensor: ... + +__all__ += ["log_softmax"] + +def lp_pool1d( + input: Tensor, + norm_type: float, + kernel_size: _size_1_t, + stride: _size | None | int = ..., + ceil_mode: bool = ..., +) -> Tensor: ... + +__all__ += ["lp_pool1d"] + +def lp_pool2d( + input: Tensor, + norm_type: float, + kernel_size: _size_2_t, + stride: _size | None | int = ..., + ceil_mode: bool = ..., +) -> Tensor: ... + +__all__ += ["lp_pool2d"] + +def lp_pool3d( + input: Tensor, + norm_type: float, + kernel_size: _size_3_t, + stride: _size | None | int = ..., + ceil_mode: bool = ..., +) -> Tensor: ... + +__all__ += ["lp_pool3d"] + +def margin_ranking_loss( + input1: Tensor, + input2: Tensor, + target: Tensor, + margin: float = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["margin_ranking_loss"] + +def max_pool1d_with_indices( + input: Tensor, + kernel_size: _size, + stride: _size | None = ..., + padding: _size = ..., + dilation: _size = ..., + ceil_mode: bool = ..., + return_indices: bool = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["max_pool1d_with_indices"] + +def max_pool2d_with_indices( + input: Tensor, + kernel_size: _size, + stride: _size | None = ..., + padding: _size = ..., + dilation: _size = ..., + ceil_mode: bool = ..., + return_indices: bool = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["max_pool2d_with_indices"] + +def max_pool3d_with_indices( + input: Tensor, + kernel_size: _size, + stride: _size | None = ..., + padding: _size = ..., + dilation: _size = ..., + ceil_mode: bool = ..., + return_indices: bool = ..., +) -> tuple[Tensor, Tensor]: ... + +__all__ += ["max_pool3d_with_indices"] + +def max_unpool1d( + input: Tensor, + indices: Tensor, + kernel_size: _size, + stride: _size | None = ..., + padding: _size = ..., + output_size: _size | None = ..., +) -> Tensor: ... + +__all__ += ["max_unpool1d"] + +def max_unpool2d( + input: Tensor, + indices: Tensor, + kernel_size: _size, + stride: _size | None = ..., + padding: _size = ..., + output_size: _size | None = ..., +) -> Tensor: ... + +__all__ += ["max_unpool2d"] + +def max_unpool3d( + input: Tensor, + indices: Tensor, + kernel_size: _size, + stride: _size | None = ..., + padding: _size = ..., + output_size: _size | None = ..., +) -> Tensor: ... + +__all__ += ["max_unpool3d"] + +def mish(input: Tensor, inplace: bool = False) -> Tensor: ... + +__all__ += ["mish"] + +def mse_loss( + input: Tensor, + target: Tensor, + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["mse_loss"] + +def multi_head_attention_forward( + query: Tensor, + key: Tensor, + value: Tensor, + embed_dim_to_check: int, + num_heads: int, + in_proj_weight: Tensor | None, + in_proj_bias: Tensor | None, + bias_k: Tensor | None, + bias_v: Tensor | None, + add_zero_attn: bool, + dropout_p: float, + out_proj_weight: Tensor, + out_proj_bias: Tensor | None, + training: bool = True, + key_padding_mask: Tensor | None = None, + need_weights: bool = True, + attn_mask: Tensor | None = None, + use_separate_proj_weight: bool = False, + q_proj_weight: Tensor | None = None, + k_proj_weight: Tensor | None = None, + v_proj_weight: Tensor | None = None, + static_k: Tensor | None = None, + static_v: Tensor | None = None, + average_attn_weights: bool = True, + is_causal: bool = False, +) -> tuple[Tensor, Tensor | None]: ... + +__all__ += ["multi_head_attention_forward"] + +def multi_margin_loss( + input: Tensor, + target: Tensor, + p: int = ..., + margin: float = ..., + weight: Tensor | None = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["multi_margin_loss"] + +def multilabel_margin_loss( + input: Tensor, + target: Tensor, + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["multilabel_margin_loss"] + +def multilabel_soft_margin_loss( + input: Tensor, + target: Tensor, + weight: Tensor | None = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["multilabel_soft_margin_loss"] + +def nll_loss( + input: Tensor, + target: Tensor, + weight: Tensor | None = ..., + size_average: bool | None = ..., + ignore_index: int = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["nll_loss"] + +def normalize( + input: Tensor, + p: float = ..., + dim: int = ..., + eps: float = ..., + out: Tensor | None = ..., +) -> Tensor: ... + +__all__ += ["normalize"] + +def poisson_nll_loss( + input: Tensor, + target: Tensor, + log_input: bool = ..., + full: bool = ..., + size_average: bool | None = ..., + eps: float = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["poisson_nll_loss"] + +def relu(input: Tensor, inplace: bool = ...) -> Tensor: ... + +__all__ += ["relu"] + +def relu6(input: Tensor, inplace: bool = ...) -> Tensor: ... + +__all__ += ["relu6"] + +def rms_norm( + input: Tensor, + normalized_shape: Sequence[int], + weight: Tensor | None = ..., + eps: float | None = ..., +) -> Tensor: ... + +__all__ += ["rms_norm"] + +def rrelu( + input: Tensor, + lower: float = ..., + upper: float = ..., + training: bool = ..., + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["rrelu"] + +def scaled_mm( + mat_a: Tensor, + mat_b: Tensor, + scale_a: Tensor | list[Tensor], + scale_recipe_a: ScalingType | list[ScalingType], + scale_b: Tensor | list[Tensor], + scale_recipe_b: ScalingType | list[ScalingType], + swizzle_a: SwizzleType | list[SwizzleType] | None = None, + swizzle_b: SwizzleType | list[SwizzleType] | None = None, + bias: Tensor | None = None, + output_dtype: _dtype = ..., + contraction_dim: list[int] | tuple[int, ...] = (), + use_fast_accum: bool = False, +) -> Tensor: ... + +__all__ += ["scaled_mm"] + +def grouped_mm( + mat_a: Tensor, + mat_b: Tensor, + *, + offs: Tensor | None = None, + bias: Tensor | None = None, + out_dtype: _dtype | None = None, +) -> Tensor: ... + +__all__ += ["grouped_mm"] + +class SwizzleType(Enum): + NO_SWIZZLE = 0 + SWIZZLE_32_4_4 = 1 + +__all__ += ["SwizzleType"] + +class ScalingType(Enum): + TensorWise = 0 + RowWise = 1 + BlockWise1x16 = 2 + BlockWise1x32 = 3 + BlockWise1x128 = 4 + BlockWise128x128 = 5 + +__all__ += ["ScalingType"] + +def selu(input: Tensor, inplace: bool = ...) -> Tensor: ... + +__all__ += ["selu"] + +def sigmoid(input: Any) -> Tensor: ... + +__all__ += ["sigmoid"] + +def silu(input: Tensor, inplace: bool = False) -> Tensor: ... + +__all__ += ["silu"] + +def smooth_l1_loss( + input: Tensor, + target: Tensor, + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., + beta: float = ..., +) -> Tensor: ... + +__all__ += ["smooth_l1_loss"] + +def soft_margin_loss( + input: Tensor, + target: Tensor, + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["soft_margin_loss"] + +def softmax( + input: Tensor, + dim: int | None = ..., + _stacklevel: int = ..., + dtype: _dtype | None = ..., +) -> Tensor: ... + +__all__ += ["softmax"] + +def softmin( + input: Tensor, + dim: int | None = ..., + _stacklevel: int = ..., + dtype: _dtype | None = ..., +) -> Tensor: ... + +__all__ += ["softmin"] + +def softsign(input: Any): ... + +__all__ += ["softsign"] + +def tanh(input: Any): ... + +__all__ += ["tanh"] + +def tanhshrink(input: Any): ... + +__all__ += ["tanhshrink"] + +def threshold( + input: Tensor, + threshold: float, + value: float, + inplace: bool = ..., +) -> Tensor: ... + +__all__ += ["threshold"] + +def triplet_margin_loss( + anchor: Tensor, + positive: Tensor, + negative: Tensor, + margin: float = ..., + p: float = ..., + eps: float = ..., + swap: bool = ..., + size_average: bool | None = ..., + reduce: bool | None = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["triplet_margin_loss"] + +def triplet_margin_with_distance_loss( + anchor: Tensor, + positive: Tensor, + negative: Tensor, + *, + distance_function: Callable[[Tensor, Tensor], Tensor] | None = ..., + margin: float = ..., + swap: bool = ..., + reduction: str = ..., +) -> Tensor: ... + +__all__ += ["triplet_margin_with_distance_loss"] + +def unfold( + input: Tensor, + kernel_size: _size_any_t, + dilation: _size_any_t = ..., + padding: _size_any_t = ..., + stride: _size_any_t = ..., +) -> Tensor: ... + +__all__ += ["unfold"] + +def upsample_bilinear( + input: Any, + size: Any | None = ..., + scale_factor: Any | None = ..., +): ... + +__all__ += ["upsample_bilinear"] + +def upsample_nearest( + input: Any, + size: Any | None = ..., + scale_factor: Any | None = ..., +): ... + +__all__ += ["upsample_nearest"] + +def upsample( + input: Any, + size: Any | None = ..., + scale_factor: Any | None = ..., + mode: str = ..., + align_corners: Any | None = ..., +): ... + +__all__ += ["upsample"] + +from torch import ( + adaptive_avg_pool1d as adaptive_avg_pool1d, + avg_pool1d as avg_pool1d, + bilinear as bilinear, + celu_ as celu_, + channel_shuffle as channel_shuffle, + conv1d as conv1d, + conv2d as conv2d, + conv3d as conv3d, + conv_tbc as conv_tbc, + conv_transpose1d as conv_transpose1d, + conv_transpose2d as conv_transpose2d, + conv_transpose3d as conv_transpose3d, + cosine_similarity as cosine_similarity, + hardshrink as hardshrink, + native_channel_shuffle as native_channel_shuffle, + pairwise_distance as pairwise_distance, + pdist as pdist, + pixel_shuffle as pixel_shuffle, + pixel_unshuffle as pixel_unshuffle, + prelu as prelu, + relu_ as relu_, + rrelu_ as rrelu_, + selu_ as selu_, +) +from torch._C._nn import ( + avg_pool2d as avg_pool2d, + avg_pool3d as avg_pool3d, + elu_ as elu_, + gelu as gelu, + hardtanh_ as hardtanh_, + leaky_relu_ as leaky_relu_, + linear as linear, + log_sigmoid as logsigmoid, + one_hot as one_hot, + pad as pad, + scaled_dot_product_attention as scaled_dot_product_attention, + softplus as softplus, + softshrink as softshrink, +) + +@overload +def adaptive_max_pool1d( + input: Tensor, + output_size: _int | _size, + return_indices: Literal[False] = False, +) -> Tensor: ... +@overload +def adaptive_max_pool1d( + input: Tensor, + output_size: _int | _size, + return_indices: Literal[True], + /, +) -> tuple[Tensor, Tensor]: ... +@overload +def adaptive_max_pool1d( + input: Tensor, + output_size: _int | _size, + *, + return_indices: Literal[True], +) -> tuple[Tensor, Tensor]: ... +@overload +def adaptive_max_pool2d( + input: Tensor, + output_size: _int | _size, + return_indices: Literal[False] = False, +) -> Tensor: ... +@overload +def adaptive_max_pool2d( + input: Tensor, + output_size: _int | _size, + return_indices: Literal[True], + /, +) -> tuple[Tensor, Tensor]: ... +@overload +def adaptive_max_pool2d( + input: Tensor, + output_size: _int | _size, + *, + return_indices: Literal[True], +) -> tuple[Tensor, Tensor]: ... +@overload +def adaptive_max_pool3d( + input: Tensor, + output_size: _int | _size, + return_indices: Literal[False] = False, +) -> Tensor: ... +@overload +def adaptive_max_pool3d( + input: Tensor, + output_size: _int | _size, + return_indices: Literal[True], + /, +) -> tuple[Tensor, Tensor]: ... +@overload +def adaptive_max_pool3d( + input: Tensor, + output_size: _int | _size, + *, + return_indices: Literal[True], +) -> tuple[Tensor, Tensor]: ... +@overload +def fractional_max_pool2d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size | None = None, + output_ratio: _ratio_any_t | None = None, + return_indices: Literal[False] = False, + _random_samples: Tensor | None = None, +) -> Tensor: ... +@overload +def fractional_max_pool2d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size | None, + output_ratio: _ratio_any_t | None, + return_indices: Literal[True], + /, + _random_samples: Tensor | None = None, +) -> tuple[Tensor, Tensor]: ... +@overload +def fractional_max_pool2d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size | None = None, + output_ratio: _ratio_any_t | None = None, + *, + return_indices: Literal[True], + _random_samples: Tensor | None = None, +) -> tuple[Tensor, Tensor]: ... +@overload +def fractional_max_pool3d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size | None = None, + output_ratio: _ratio_any_t | None = None, + return_indices: Literal[False] = False, + _random_samples: Tensor | None = None, +) -> Tensor: ... +@overload +def fractional_max_pool3d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size | None, + output_ratio: _ratio_any_t | None, + return_indices: Literal[True], + /, + _random_samples: Tensor | None = None, +) -> tuple[Tensor, Tensor]: ... +@overload +def fractional_max_pool3d( + input: Tensor, + kernel_size: _int | _size, + output_size: _int | _size | None = None, + output_ratio: _ratio_any_t | None = None, + *, + return_indices: Literal[True], + _random_samples: Tensor | None = None, +) -> tuple[Tensor, Tensor]: ... +@overload +def max_pool1d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, + return_indices: Literal[False] = False, +) -> Tensor: ... +@overload +def max_pool1d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None, + padding: _int | _size, + dilation: _int | _size, + ceil_mode: bool, + return_indices: Literal[True], + /, +) -> tuple[Tensor, Tensor]: ... +@overload +def max_pool1d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, + *, + return_indices: Literal[True], +) -> tuple[Tensor, Tensor]: ... +@overload +def max_pool2d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, + return_indices: Literal[False] = False, +) -> Tensor: ... +@overload +def max_pool2d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None, + padding: _int | _size, + dilation: _int | _size, + ceil_mode: bool, + return_indices: Literal[True], + /, +) -> tuple[Tensor, Tensor]: ... +@overload +def max_pool2d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, + *, + return_indices: Literal[True], +) -> tuple[Tensor, Tensor]: ... +@overload +def max_pool3d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, + return_indices: Literal[False] = False, +) -> Tensor: ... +@overload +def max_pool3d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None, + padding: _int | _size, + dilation: _int | _size, + ceil_mode: bool, + return_indices: Literal[True], + /, +) -> tuple[Tensor, Tensor]: ... +@overload +def max_pool3d( + input: Tensor, + kernel_size: _int | _size, + stride: _int | _size | None = None, + padding: _int | _size = 0, + dilation: _int | _size = 1, + ceil_mode: bool = False, + *, + return_indices: Literal[True], +) -> tuple[Tensor, Tensor]: ... + +__all__ += [ + "adaptive_avg_pool1d", + "avg_pool1d", + "bilinear", + "celu_", + "channel_shuffle", + "conv_tbc", + "conv_transpose1d", + "conv_transpose2d", + "conv_transpose3d", + "conv1d", + "conv2d", + "conv3d", + "cosine_similarity", + "hardshrink", + "native_channel_shuffle", + "pairwise_distance", + "pdist", + "pixel_shuffle", + "pixel_unshuffle", + "prelu", + "relu_", + "rrelu_", + "selu_", + "avg_pool2d", + "avg_pool3d", + "elu_", + "gelu", + "hardtanh_", + "leaky_relu_", + "linear", + "logsigmoid", + "one_hot", + "pad", + "scaled_dot_product_attention", + "softplus", + "softshrink", + "max_pool1d", + "adaptive_max_pool1d", + "max_pool2d", + "fractional_max_pool2d", + "adaptive_max_pool2d", + "max_pool3d", + "fractional_max_pool3d", + "adaptive_max_pool3d", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/grad.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/grad.py new file mode 100644 index 0000000000000000000000000000000000000000..61e817dbed612e51f1ff426373fbef1d192e331e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/grad.py @@ -0,0 +1,298 @@ +# mypy: allow-untyped-defs +"""Gradient interface.""" + +import torch +from torch.nn.modules.utils import _pair, _single, _triple + + +def conv1d_input( + input_size, + weight, + grad_output, + stride=1, + padding=0, + dilation=1, + groups=1, +): + r"""Compute the gradient of conv1d with respect to the input of the convolution. + + This is same as the 1D transposed convolution operator under the hood but requires + the shape of the gradient w.r.t. input to be specified explicitly. + + Args: + input_size : Shape of the input gradient tensor + weight: weight tensor (out_channels x in_channels/groups x kW) + grad_output : output gradient tensor (minibatch x out_channels x oW) + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 + groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 + + Examples:: + + >>> input = torch.randn(1, 1, 3, requires_grad=True) + >>> weight = torch.randn(1, 1, 1, requires_grad=True) + >>> output = F.conv1d(input, weight) + >>> grad_output = torch.randn(output.shape) + >>> grad_input = torch.autograd.grad(output, input, grad_output) + >>> F.grad.conv1d_input(input.shape, weight, grad_output) + + """ + input = grad_output.new_empty(1).expand(input_size) + + return torch.ops.aten.convolution_backward( + grad_output, + input, + weight, + None, + _single(stride), + _single(padding), + _single(dilation), + False, + [0], + groups, + (True, False, False), + )[0] + + +def conv1d_weight( + input, + weight_size, + grad_output, + stride=1, + padding=0, + dilation=1, + groups=1, +): + r"""Compute the gradient of conv1d with respect to the weight of the convolution. + + Args: + input: input tensor of shape (minibatch x in_channels x iW) + weight_size : Shape of the weight gradient tensor + grad_output : output gradient tensor (minibatch x out_channels x oW) + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 + groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 + + Examples:: + + >>> input = torch.randn(1, 1, 3, requires_grad=True) + >>> weight = torch.randn(1, 1, 1, requires_grad=True) + >>> output = F.conv1d(input, weight) + >>> grad_output = torch.randn(output.shape) + >>> # xdoctest: +SKIP + >>> grad_weight = torch.autograd.grad(output, filter, grad_output) + >>> F.grad.conv1d_weight(input, weight.shape, grad_output) + + """ + weight = grad_output.new_empty(1).expand(weight_size) + + return torch.ops.aten.convolution_backward( + grad_output, + input, + weight, + None, + _single(stride), + _single(padding), + _single(dilation), + False, + [0], + groups, + (False, True, False), + )[1] + + +def conv2d_input( + input_size, + weight, + grad_output, + stride=1, + padding=0, + dilation=1, + groups=1, +): + r"""Compute the gradient of conv2d with respect to the input of the convolution. + + This is same as the 2D transposed convolution operator under the hood but requires + the shape of the gradient w.r.t. input to be specified explicitly. + + Args: + input_size : Shape of the input gradient tensor + weight: weight tensor (out_channels x in_channels/groups x kH x kW) + grad_output : output gradient tensor (minibatch x out_channels x oH x oW) + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 + groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 + + Examples:: + + >>> input = torch.randn(1, 1, 3, 3, requires_grad=True) + >>> weight = torch.randn(1, 1, 1, 2, requires_grad=True) + >>> output = F.conv2d(input, weight) + >>> grad_output = torch.randn(output.shape) + >>> grad_input = torch.autograd.grad(output, input, grad_output) + >>> F.grad.conv2d_input(input.shape, weight, grad_output) + + """ + input = grad_output.new_empty(1).expand(input_size) + + return torch.ops.aten.convolution_backward( + grad_output, + input, + weight, + None, + _pair(stride), + _pair(padding), + _pair(dilation), + False, + [0], + groups, + (True, False, False), + )[0] + + +def conv2d_weight( + input, + weight_size, + grad_output, + stride=1, + padding=0, + dilation=1, + groups=1, +): + r"""Compute the gradient of conv2d with respect to the weight of the convolution. + + Args: + input: input tensor of shape (minibatch x in_channels x iH x iW) + weight_size : Shape of the weight gradient tensor + grad_output : output gradient tensor (minibatch x out_channels x oH x oW) + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 + groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 + + Examples:: + + >>> input = torch.randn(1, 1, 3, 3, requires_grad=True) + >>> weight = torch.randn(1, 1, 1, 2, requires_grad=True) + >>> output = F.conv2d(input, weight) + >>> grad_output = torch.randn(output.shape) + >>> # xdoctest: +SKIP + >>> grad_weight = torch.autograd.grad(output, filter, grad_output) + >>> F.grad.conv2d_weight(input, weight.shape, grad_output) + + """ + weight = grad_output.new_empty(1).expand(weight_size) + + return torch.ops.aten.convolution_backward( + grad_output, + input, + weight, + None, + _pair(stride), + _pair(padding), + _pair(dilation), + False, + [0], + groups, + (False, True, False), + )[1] + + +def conv3d_input( + input_size, + weight, + grad_output, + stride=1, + padding=0, + dilation=1, + groups=1, +): + r"""Compute the gradient of conv3d with respect to the input of the convolution. + + This is same as the 3D transposed convolution operator under the hood but requires + the shape of the gradient w.r.t. input to be specified explicitly. + + Args: + input_size : Shape of the input gradient tensor + weight: weights tensor (out_channels x in_channels/groups x kT x kH x kW) + grad_output : output gradient tensor (minibatch x out_channels x oT x oH x oW) + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 + groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 + + Examples:: + + >>> input = torch.randn(2, 8, 10, 10, 20, requires_grad=True) + >>> weight = torch.randn(4, 8, 2, 3, 3, requires_grad=True) + >>> output = F.conv3d(input, weight) + >>> grad_output = torch.randn(output.shape) + >>> grad_input = torch.autograd.grad(output, input, grad_output) + >>> F.grad.conv3d_input(input.shape, weight, grad_output) + + """ + input = grad_output.new_empty(1).expand(input_size) + + return torch.ops.aten.convolution_backward( + grad_output, + input, + weight, + None, + _triple(stride), + _triple(padding), + _triple(dilation), + False, + [0], + groups, + (True, False, False), + )[0] + + +def conv3d_weight( + input, + weight_size, + grad_output, + stride=1, + padding=0, + dilation=1, + groups=1, +): + r"""Compute the gradient of conv3d with respect to the weight of the convolution. + + Args: + input: input tensor of shape (minibatch x in_channels x iT x iH x iW) + weight_size : Shape of the weight gradient tensor + grad_output : output gradient tensor (minibatch x out_channels x oT x oH x oW) + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to both sides of the input. Default: 0 + dilation (int or tuple, optional): Spacing between kernel elements. Default: 1 + groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1 + + Examples:: + + >>> input = torch.randn(2, 8, 10, 10, 20, requires_grad=True) + >>> weight = torch.randn(4, 8, 2, 3, 3, requires_grad=True) + >>> output = F.conv3d(input, weight) + >>> grad_output = torch.randn(output.shape) + >>> grad_weight = torch.autograd.grad(output, weight, grad_output) + >>> F.grad.conv3d_weight(input, weight.shape, grad_output) + + """ + weight = grad_output.new_empty(1).expand(weight_size) + + return torch.ops.aten.convolution_backward( + grad_output, + input, + weight, + None, + _triple(stride), + _triple(padding), + _triple(dilation), + False, + [0], + groups, + (False, True, False), + )[1] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/init.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/init.py new file mode 100644 index 0000000000000000000000000000000000000000..900b2d34bc08f10157e73c262dd9fb784793978e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/init.py @@ -0,0 +1,754 @@ +"""This file contains utilities for initializing neural network parameters.""" + +import math +import warnings +from collections.abc import Callable +from typing import Literal, TypeVar +from typing_extensions import ParamSpec + +import torch +from torch import Tensor + + +__all__ = [ + "calculate_gain", + "uniform_", + "normal_", + "trunc_normal_", + "constant_", + "ones_", + "zeros_", + "eye_", + "dirac_", + "xavier_uniform_", + "xavier_normal_", + "kaiming_uniform_", + "kaiming_normal_", + "orthogonal_", + "sparse_", + # Deprecated aliases (for backward compatibility) + "uniform", + "normal", + "constant", + "eye", + "dirac", + "xavier_uniform", + "xavier_normal", + "kaiming_uniform", + "kaiming_normal", + "orthogonal", + "sparse", +] + + +_R = TypeVar("_R") +_P = ParamSpec("_P") + +_NonlinearityType = Literal[ + "linear", + "conv1d", + "conv2d", + "conv3d", + "conv_transpose1d", + "conv_transpose2d", + "conv_transpose3d", + "sigmoid", + "tanh", + "relu", + "leaky_relu", + "selu", +] + +_FanMode = Literal["fan_in", "fan_out"] + + +# These no_grad_* functions are necessary as wrappers around the parts of these +# functions that use `with torch.no_grad()`. The JIT doesn't support context +# managers, so these need to be implemented as builtins. Using these wrappers +# lets us keep those builtins small and reusable. +def _no_grad_uniform_( + tensor: Tensor, a: float, b: float, generator: torch.Generator | None = None +) -> Tensor: + with torch.no_grad(): + return tensor.uniform_(a, b, generator=generator) + + +def _no_grad_normal_( + tensor: Tensor, + mean: float, + std: float, + generator: torch.Generator | None = None, +) -> Tensor: + with torch.no_grad(): + return tensor.normal_(mean, std, generator=generator) + + +def _no_grad_trunc_normal_( + tensor: Tensor, + mean: float, + std: float, + a: float, + b: float, + generator: torch.Generator | None = None, +) -> Tensor: + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x: float) -> float: + # Computes standard normal cumulative distribution function + return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0 + + if (mean < a - 2 * std) or (mean > b + 2 * std): + warnings.warn( + "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. " + "The distribution of values may be incorrect.", + stacklevel=2, + ) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + l = norm_cdf((a - mean) / std) + u = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * l - 1, 2 * u - 1, generator=generator) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.0)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def _no_grad_fill_(tensor: Tensor, val: float) -> Tensor: + with torch.no_grad(): + return tensor.fill_(val) + + +def _no_grad_zero_(tensor: Tensor) -> Tensor: + with torch.no_grad(): + return tensor.zero_() + + +def calculate_gain( + nonlinearity: _NonlinearityType, param: int | float | None = None +) -> float: + r"""Return the recommended gain value for the given nonlinearity function. + + The values are as follows: + + ================= ==================================================== + nonlinearity gain + ================= ==================================================== + Linear / Identity :math:`1` + Conv{1,2,3}D :math:`1` + Sigmoid :math:`1` + Tanh :math:`\frac{5}{3}` + ReLU :math:`\sqrt{2}` + Leaky Relu :math:`\sqrt{\frac{2}{1 + \text{negative\_slope}^2}}` + SELU :math:`\frac{3}{4}` + ================= ==================================================== + + .. warning:: + In order to implement `Self-Normalizing Neural Networks`_ , + you should use ``nonlinearity='linear'`` instead of ``nonlinearity='selu'``. + This gives the initial weights a variance of ``1 / N``, + which is necessary to induce a stable fixed point in the forward pass. + In contrast, the default gain for ``SELU`` sacrifices the normalization + effect for more stable gradient flow in rectangular layers. + + Args: + nonlinearity: the non-linear function (`nn.functional` name) + param: optional parameter for the non-linear function + + Examples: + >>> gain = nn.init.calculate_gain( + ... "leaky_relu", 0.2 + ... ) # leaky_relu with negative_slope=0.2 + + .. _Self-Normalizing Neural Networks: https://papers.nips.cc/paper/2017/hash/5d44ee6f2c3f71b73125876103c8f6c4-Abstract.html + """ + linear_fns = [ + "linear", + "conv1d", + "conv2d", + "conv3d", + "conv_transpose1d", + "conv_transpose2d", + "conv_transpose3d", + ] + if nonlinearity in linear_fns or nonlinearity == "sigmoid": + return 1 + elif nonlinearity == "tanh": + return 5.0 / 3 + elif nonlinearity == "relu": + return math.sqrt(2.0) + elif nonlinearity == "leaky_relu": + if param is None: + negative_slope = 0.01 + elif ( + not isinstance(param, bool) + and isinstance(param, int) + or isinstance(param, float) + ): + # True/False are instances of int, hence check above + negative_slope = param + else: + raise ValueError(f"negative_slope {param} not a valid number") + return math.sqrt(2.0 / (1 + negative_slope**2)) + elif nonlinearity == "selu": + return ( + 3.0 / 4 + ) # Value found empirically (https://github.com/pytorch/pytorch/pull/50664) + else: + raise ValueError(f"Unsupported nonlinearity {nonlinearity}") + + +def uniform_( + tensor: Tensor, + a: float = 0.0, + b: float = 1.0, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input Tensor with values drawn from the uniform distribution. + + :math:`\mathcal{U}(a, b)`. + + Args: + tensor: an n-dimensional `torch.Tensor` + a: the lower bound of the uniform distribution + b: the upper bound of the uniform distribution + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.uniform_(w) + """ + if torch.overrides.has_torch_function_variadic(tensor): + return torch.overrides.handle_torch_function( + uniform_, (tensor,), tensor=tensor, a=a, b=b, generator=generator + ) + return _no_grad_uniform_(tensor, a, b, generator) + + +def normal_( + tensor: Tensor, + mean: float = 0.0, + std: float = 1.0, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input Tensor with values drawn from the normal distribution. + + :math:`\mathcal{N}(\text{mean}, \text{std}^2)`. + + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.normal_(w) + """ + if torch.overrides.has_torch_function_variadic(tensor): + return torch.overrides.handle_torch_function( + normal_, (tensor,), tensor=tensor, mean=mean, std=std, generator=generator + ) + return _no_grad_normal_(tensor, mean, std, generator) + + +def trunc_normal_( + tensor: Tensor, + mean: float = 0.0, + std: float = 1.0, + a: float = -2.0, + b: float = 2.0, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input Tensor with values drawn from a truncated normal distribution. + + The values are effectively drawn from the + normal distribution :math:`\mathcal{N}(\text{mean}, \text{std}^2)` + with values outside :math:`[a, b]` redrawn until they are within + the bounds. The method used for generating the random values works + best when :math:`a \leq \text{mean} \leq b`. + + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + a: the minimum cutoff value + b: the maximum cutoff value + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.trunc_normal_(w) + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b, generator=generator) + + +def constant_(tensor: Tensor, val: float) -> Tensor: + r"""Fill the input Tensor with the value :math:`\text{val}`. + + Args: + tensor: an n-dimensional `torch.Tensor` + val: the value to fill the tensor with + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.constant_(w, 0.3) + """ + if torch.overrides.has_torch_function_variadic(tensor): + return torch.overrides.handle_torch_function( + constant_, (tensor,), tensor=tensor, val=val + ) + return _no_grad_fill_(tensor, val) + + +def ones_(tensor: Tensor) -> Tensor: + r"""Fill the input Tensor with the scalar value `1`. + + Args: + tensor: an n-dimensional `torch.Tensor` + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.ones_(w) + """ + return _no_grad_fill_(tensor, 1.0) + + +def zeros_(tensor: Tensor) -> Tensor: + r"""Fill the input Tensor with the scalar value `0`. + + Args: + tensor: an n-dimensional `torch.Tensor` + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.zeros_(w) + """ + return _no_grad_zero_(tensor) + + +def eye_(tensor: Tensor) -> Tensor: + r"""Fill the 2-dimensional input `Tensor` with the identity matrix. + + Preserves the identity of the inputs in `Linear` layers, where as + many inputs are preserved as possible. + + Args: + tensor: a 2-dimensional `torch.Tensor` + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.eye_(w) + """ + if tensor.ndimension() != 2: + raise ValueError("Only tensors with 2 dimensions are supported") + + with torch.no_grad(): + torch.eye(*tensor.shape, out=tensor, requires_grad=tensor.requires_grad) + return tensor + + +def dirac_(tensor: Tensor, groups: int = 1) -> Tensor: + r"""Fill the {3, 4, 5}-dimensional input `Tensor` with the Dirac delta function. + + Preserves the identity of the inputs in `Convolutional` + layers, where as many input channels are preserved as possible. In case + of groups>1, each group of channels preserves identity + + Args: + tensor: a {3, 4, 5}-dimensional `torch.Tensor` + groups (int, optional): number of groups in the conv layer (default: 1) + Examples: + >>> w = torch.empty(3, 16, 5, 5) + >>> nn.init.dirac_(w) + >>> w = torch.empty(3, 24, 5, 5) + >>> nn.init.dirac_(w, 3) + """ + dimensions = tensor.ndimension() + if dimensions not in [3, 4, 5]: + raise ValueError("Only tensors with 3, 4, or 5 dimensions are supported") + + sizes = tensor.size() + + if sizes[0] % groups != 0: + raise ValueError("dim 0 must be divisible by groups") + + out_chans_per_grp = sizes[0] // groups + min_dim = min(out_chans_per_grp, sizes[1]) + + with torch.no_grad(): + tensor.zero_() + + for g in range(groups): + for d in range(min_dim): + if dimensions == 3: # Temporal convolution + tensor[g * out_chans_per_grp + d, d, tensor.size(2) // 2] = 1 + elif dimensions == 4: # Spatial convolution + tensor[ + g * out_chans_per_grp + d, + d, + tensor.size(2) // 2, + tensor.size(3) // 2, + ] = 1 + else: # Volumetric convolution + tensor[ + g * out_chans_per_grp + d, + d, + tensor.size(2) // 2, + tensor.size(3) // 2, + tensor.size(4) // 2, + ] = 1 + return tensor + + +def _calculate_fan_in_and_fan_out(tensor: Tensor) -> tuple[int, int]: + dimensions = tensor.dim() + if dimensions < 2: + raise ValueError( + "Fan in and fan out can not be computed for tensor with fewer than 2 dimensions" + ) + + num_input_fmaps = tensor.size(1) + num_output_fmaps = tensor.size(0) + receptive_field_size = 1 + if tensor.dim() > 2: + # math.prod is not always available, accumulate the product manually + # we could use functools.reduce but that is not supported by TorchScript + for s in tensor.shape[2:]: + receptive_field_size *= s + fan_in = num_input_fmaps * receptive_field_size + fan_out = num_output_fmaps * receptive_field_size + + return fan_in, fan_out + + +def xavier_uniform_( + tensor: Tensor, + gain: float = 1.0, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input `Tensor` with values using a Xavier uniform distribution. + + The method is described in `Understanding the difficulty of training + deep feedforward neural networks` - Glorot, X. & Bengio, Y. (2010). + The resulting tensor will have values sampled from + :math:`\mathcal{U}(-a, a)` where + + .. math:: + a = \text{gain} \times \sqrt{\frac{6}{\text{fan\_in} + \text{fan\_out}}} + + Also known as Glorot initialization. + + Args: + tensor: an n-dimensional `torch.Tensor` + gain: an optional scaling factor + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.xavier_uniform_(w, gain=nn.init.calculate_gain("relu")) + """ + fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) + std = gain * math.sqrt(2.0 / float(fan_in + fan_out)) + a = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation + + return _no_grad_uniform_(tensor, -a, a, generator) + + +def xavier_normal_( + tensor: Tensor, + gain: float = 1.0, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input `Tensor` with values using a Xavier normal distribution. + + The method is described in `Understanding the difficulty of training deep feedforward + neural networks` - Glorot, X. & Bengio, Y. (2010). The resulting tensor + will have values sampled from :math:`\mathcal{N}(0, \text{std}^2)` where + + .. math:: + \text{std} = \text{gain} \times \sqrt{\frac{2}{\text{fan\_in} + \text{fan\_out}}} + + Also known as Glorot initialization. + + Args: + tensor: an n-dimensional `torch.Tensor` + gain: an optional scaling factor + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.xavier_normal_(w) + """ + fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) + std = gain * math.sqrt(2.0 / float(fan_in + fan_out)) + + return _no_grad_normal_(tensor, 0.0, std, generator) + + +def _calculate_correct_fan(tensor: Tensor, mode: _FanMode) -> int: + # pyrefly: ignore [bad-assignment] + mode = mode.lower() + valid_modes = ["fan_in", "fan_out"] + if mode not in valid_modes: + raise ValueError(f"Mode {mode} not supported, please use one of {valid_modes}") + + fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) + return fan_in if mode == "fan_in" else fan_out + + +def kaiming_uniform_( + tensor: Tensor, + a: float = 0, + mode: _FanMode = "fan_in", + nonlinearity: _NonlinearityType = "leaky_relu", + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input `Tensor` with values using a Kaiming uniform distribution. + + The method is described in `Delving deep into rectifiers: Surpassing + human-level performance on ImageNet classification` - He, K. et al. (2015). + The resulting tensor will have values sampled from + :math:`\mathcal{U}(-\text{bound}, \text{bound})` where + + .. math:: + \text{bound} = \text{gain} \times \sqrt{\frac{3}{\text{fan\_mode}}} + + Also known as He initialization. + + Args: + tensor: an n-dimensional `torch.Tensor` + a: the negative slope of the rectifier used after this layer (only + used with ``'leaky_relu'``) + mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'`` + preserves the magnitude of the variance of the weights in the + forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the + backwards pass. + nonlinearity: the non-linear function (`nn.functional` name), + recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default). + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.kaiming_uniform_(w, mode="fan_in", nonlinearity="relu") + + Note: + Be aware that ``fan_in`` and ``fan_out`` are calculated assuming + that the weight matrix is used in a transposed manner, + (i.e., ``x @ w.T`` in ``Linear`` layers, where ``w.shape = [fan_out, fan_in]``). + This is important for correct initialization. + If you plan to use ``x @ w``, where ``w.shape = [fan_in, fan_out]``, + pass in a transposed weight matrix, i.e. ``nn.init.kaiming_uniform_(w.T, ...)``. + """ + if torch.overrides.has_torch_function_variadic(tensor): + return torch.overrides.handle_torch_function( + kaiming_uniform_, + (tensor,), + tensor=tensor, + a=a, + mode=mode, + nonlinearity=nonlinearity, + generator=generator, + ) + + if 0 in tensor.shape: + warnings.warn("Initializing zero-element tensors is a no-op", stacklevel=2) + return tensor + fan = _calculate_correct_fan(tensor, mode) + gain = calculate_gain(nonlinearity, a) + std = gain / math.sqrt(fan) + bound = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation + with torch.no_grad(): + return tensor.uniform_(-bound, bound, generator=generator) + + +def kaiming_normal_( + tensor: Tensor, + a: float = 0, + mode: _FanMode = "fan_in", + nonlinearity: _NonlinearityType = "leaky_relu", + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input `Tensor` with values using a Kaiming normal distribution. + + The method is described in `Delving deep into rectifiers: Surpassing + human-level performance on ImageNet classification` - He, K. et al. (2015). + The resulting tensor will have values sampled from + :math:`\mathcal{N}(0, \text{std}^2)` where + + .. math:: + \text{std} = \frac{\text{gain}}{\sqrt{\text{fan\_mode}}} + + Also known as He initialization. + + Args: + tensor: an n-dimensional `torch.Tensor` + a: the negative slope of the rectifier used after this layer (only + used with ``'leaky_relu'``) + mode: either ``'fan_in'`` (default) or ``'fan_out'``. Choosing ``'fan_in'`` + preserves the magnitude of the variance of the weights in the + forward pass. Choosing ``'fan_out'`` preserves the magnitudes in the + backwards pass. + nonlinearity: the non-linear function (`nn.functional` name), + recommended to use only with ``'relu'`` or ``'leaky_relu'`` (default). + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.kaiming_normal_(w, mode="fan_out", nonlinearity="relu") + + Note: + Be aware that ``fan_in`` and ``fan_out`` are calculated assuming + that the weight matrix is used in a transposed manner, + (i.e., ``x @ w.T`` in ``Linear`` layers, where ``w.shape = [fan_out, fan_in]``). + This is important for correct initialization. + If you plan to use ``x @ w``, where ``w.shape = [fan_in, fan_out]``, + pass in a transposed weight matrix, i.e. ``nn.init.kaiming_normal_(w.T, ...)``. + """ + if 0 in tensor.shape: + warnings.warn("Initializing zero-element tensors is a no-op", stacklevel=2) + return tensor + fan = _calculate_correct_fan(tensor, mode) + gain = calculate_gain(nonlinearity, a) + std = gain / math.sqrt(fan) + with torch.no_grad(): + return tensor.normal_(0, std, generator=generator) + + +def orthogonal_( + tensor: Tensor, + gain: float = 1, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the input `Tensor` with a (semi) orthogonal matrix. + + Described in `Exact solutions to the nonlinear dynamics of learning in deep + linear neural networks` - Saxe, A. et al. (2013). The input tensor must have + at least 2 dimensions, and for tensors with more than 2 dimensions the + trailing dimensions are flattened. + + Args: + tensor: an n-dimensional `torch.Tensor`, where :math:`n \geq 2` + gain: optional scaling factor + generator: the torch Generator to sample from (default: None) + + Examples: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_LAPACK) + >>> w = torch.empty(3, 5) + >>> nn.init.orthogonal_(w) + """ + if tensor.ndimension() < 2: + raise ValueError("Only tensors with 2 or more dimensions are supported") + + if tensor.numel() == 0: + # no-op + return tensor + rows = tensor.size(0) + cols = tensor.numel() // rows + flattened = tensor.new_empty((rows, cols)).normal_(0, 1, generator=generator) + + if rows < cols: + flattened.t_() + + # Compute the qr factorization + q, r = torch.linalg.qr(flattened) + # Make Q uniform according to https://arxiv.org/pdf/math-ph/0609050.pdf + d = torch.diag(r, 0) + ph = d.sign() + q *= ph + + if rows < cols: + q.t_() + + with torch.no_grad(): + tensor.view_as(q).copy_(q) + tensor.mul_(gain) + return tensor + + +def sparse_( + tensor: Tensor, + sparsity: float, + std: float = 0.01, + generator: torch.Generator | None = None, +) -> Tensor: + r"""Fill the 2D input `Tensor` as a sparse matrix. + + The non-zero elements will be drawn from the normal distribution + :math:`\mathcal{N}(0, 0.01)`, as described in `Deep learning via + Hessian-free optimization` - Martens, J. (2010). + + Args: + tensor: an n-dimensional `torch.Tensor` + sparsity: The fraction of elements in each column to be set to zero + std: the standard deviation of the normal distribution used to generate + the non-zero values + generator: the torch Generator to sample from (default: None) + + Examples: + >>> w = torch.empty(3, 5) + >>> nn.init.sparse_(w, sparsity=0.1) + """ + if tensor.ndimension() != 2: + raise ValueError("Only tensors with 2 dimensions are supported") + + rows, cols = tensor.shape + num_zeros = math.ceil(sparsity * rows) + + with torch.no_grad(): + tensor.normal_(0, std, generator=generator) + for col_idx in range(cols): + row_indices = torch.randperm(rows) + zero_indices = row_indices[:num_zeros] + tensor[zero_indices, col_idx] = 0 + return tensor + + +# for backward compatibility +def _make_deprecate(meth: Callable[_P, _R]) -> Callable[_P, _R]: + new_name = meth.__name__ + old_name = new_name[:-1] + + def deprecated_init(*args: _P.args, **kwargs: _P.kwargs) -> _R: + warnings.warn( + f"`nn.init.{old_name}` is now deprecated in favor of `nn.init.{new_name}`.", + FutureWarning, + stacklevel=2, + ) + return meth(*args, **kwargs) + + deprecated_init.__doc__ = rf""" + {old_name}(...) + + .. warning:: + This method is now deprecated in favor of :func:`torch.nn.init.{new_name}`. + + See :func:`~torch.nn.init.{new_name}` for details.""" + deprecated_init.__name__ = old_name + return deprecated_init + + +uniform = _make_deprecate(uniform_) +normal = _make_deprecate(normal_) +constant = _make_deprecate(constant_) +eye = _make_deprecate(eye_) +dirac = _make_deprecate(dirac_) +xavier_uniform = _make_deprecate(xavier_uniform_) +xavier_normal = _make_deprecate(xavier_normal_) +kaiming_uniform = _make_deprecate(kaiming_uniform_) +kaiming_normal = _make_deprecate(kaiming_normal_) +orthogonal = _make_deprecate(orthogonal_) +sparse = _make_deprecate(sparse_) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/parameter.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/parameter.py new file mode 100644 index 0000000000000000000000000000000000000000..64e9d8c2d80f241c687e8777164542c6d0759b65 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/parameter.py @@ -0,0 +1,307 @@ +from collections import OrderedDict +from typing import Any + +import torch +from torch._C import _disabled_torch_function_impl + + +__all__ = [ + "Parameter", + "UninitializedParameter", + "is_lazy", + "Buffer", + "UninitializedBuffer", + "UninitializedTensorMixin", +] + + +# Metaclass to combine _TensorMeta and the instance check override for Parameter. +class _ParameterMeta(torch._C._TensorMeta): + # Make `isinstance(t, Parameter)` return True for custom tensor instances that have the _is_param flag. + def __instancecheck__(self, instance) -> bool: + if self is Parameter: + if isinstance(instance, torch.Tensor) and getattr( + instance, "_is_param", False + ): + return True + return super().__instancecheck__(instance) + + +class Parameter(torch.Tensor, metaclass=_ParameterMeta): + r"""A kind of Tensor that is to be considered a module parameter. + + Parameters are :class:`~torch.Tensor` subclasses, that have a + very special property when used with :class:`Module` s - when they're + assigned as Module attributes they are automatically added to the list of + its parameters, and will appear e.g. in :meth:`~Module.parameters` iterator. + Assigning a Tensor doesn't have such effect. This is because one might + want to cache some temporary state, like last hidden state of the RNN, in + the model. If there was no such class as :class:`Parameter`, these + temporaries would get registered too. + + Args: + data (Tensor): parameter tensor. + requires_grad (bool, optional): if the parameter requires gradient. Note that + the torch.no_grad() context does NOT affect the default behavior of + Parameter creation--the Parameter will still have `requires_grad=True` in + :class:`~no_grad` mode. See :ref:`locally-disable-grad-doc` for more + details. Default: `True` + """ + + def __new__(cls, data=None, requires_grad=True): + if data is None: + data = torch.empty(0) + if type(data) is torch.Tensor or type(data) is Parameter: + # For ease of BC maintenance, keep this path for standard Tensor. + # Eventually (tm), we should change the behavior for standard Tensor to match. + return torch.Tensor._make_subclass(cls, data, requires_grad) + + # Path for custom tensors: set a flag on the instance to indicate parameter-ness. + t = data.detach().requires_grad_(requires_grad) + if type(t) is not type(data): + raise RuntimeError( + f"Creating a Parameter from an instance of type {type(data).__name__} " + "requires that detach() returns an instance of the same type, but return " + f"type {type(t).__name__} was found instead. To use the type as a " + "Parameter, please correct the detach() semantics defined by " + "its __torch_dispatch__() implementation." + ) + t._is_param = True + return t + + # Note: the 3 methods below only apply to standard Tensor. Parameters of custom tensor types + # are still considered that custom tensor type and these methods will not be called for them. + def __deepcopy__(self, memo): + if id(self) in memo: + return memo[id(self)] + else: + result = type(self)( + self.data.clone(memory_format=torch.preserve_format), self.requires_grad + ) + memo[id(self)] = result + return result + + # pyrefly: ignore [bad-override] + def __repr__(self) -> str: + return "Parameter containing:\n" + super().__repr__() + + def __reduce_ex__(self, proto): + state = torch._utils._get_obj_state(self) + + # See Note [Don't serialize hooks] + hooks = OrderedDict() + if not state: + return ( + torch._utils._rebuild_parameter, + (self.data, self.requires_grad, hooks), + ) + + return ( + torch._utils._rebuild_parameter_with_state, + (self.data, self.requires_grad, hooks, state), + ) + + __torch_function__ = _disabled_torch_function_impl + + +class UninitializedTensorMixin: + _allowed_methods = [ + torch.Tensor.__hash__, + torch.Tensor.size, + torch.Tensor.copy_, + torch.Tensor.is_complex, + torch.Tensor.is_floating_point, + torch.Tensor.half, + torch.Tensor.float, + torch.Tensor.double, + torch.Tensor.char, + torch.Tensor.short, + torch.Tensor.int, + torch.Tensor.long, + torch.Tensor.cuda, + torch.Tensor.cpu, + torch.Tensor.to, + torch.Tensor.get_device, + torch._has_compatible_shallow_copy_type, + ] + + def materialize(self, shape, device=None, dtype=None) -> None: + r"""Create a Parameter or Tensor with the same properties of the uninitialized one. + + Given a shape, it materializes a parameter in the same device + and with the same `dtype` as the current one or the specified ones in the + arguments. + + Args: + shape : (tuple): the shape for the materialized tensor. + device (:class:`torch.device`): the desired device of the parameters + and buffers in this module. Optional. + dtype (:class:`torch.dtype`): the desired floating point type of + the floating point parameters and buffers in this module. Optional. + """ + if device is None: + device = self.data.device + if dtype is None: + dtype = self.data.dtype + self.data = torch.empty(shape, device=device, dtype=dtype) + # pyrefly: ignore [bad-override, missing-attribute] + self.__class__ = self.cls_to_become + + @property + def shape(self): + raise RuntimeError( + "Can't access the shape of an uninitialized parameter or buffer. " + "This error usually happens in `load_state_dict` when trying to load " + "an uninitialized parameter into an initialized one. " + "Call `forward` to initialize the parameters before accessing their attributes." + ) + + def share_memory_(self): + raise RuntimeError( + "Can't share memory on an uninitialized parameter or buffer. " + "Call `forward` to initialize the parameters before calling " + "`module.share_memory()`." + ) + + def __repr__(self) -> str: + return f"<{self.__class__.__name__}>" + + def __reduce_ex__(self, proto): + # See Note [Don't serialize hooks] + # pyrefly: ignore [missing-attribute] + return (self.__class__, (self.requires_grad,)) + + @classmethod + def __torch_function__(cls, func, types, args=(), kwargs=None): + # method-wrapper is to detect access to Tensor properties that are + # wrapped in descriptors + if func in cls._allowed_methods or func.__class__.__name__ == "method-wrapper": + if kwargs is None: + kwargs = {} + # pyrefly: ignore [missing-attribute] + return super().__torch_function__(func, types, args, kwargs) + raise ValueError( + f"Attempted to use an uninitialized parameter in {func}. " + "This error happens when you are using a `LazyModule` or " + f"explicitly manipulating `torch.nn.parameter.{cls.__name__}` " + "objects. When using LazyModules Call `forward` with a dummy batch " + "to initialize the parameters before calling torch functions" + ) + + +def is_lazy(param: Any) -> bool: + """ + Returns whether ``param`` is an ``UninitializedParameter`` or ``UninitializedBuffer``. + + Args: + param (Any): the input to check. + """ + return isinstance(param, UninitializedTensorMixin) + + +class UninitializedParameter(UninitializedTensorMixin, Parameter): + r"""A parameter that is not initialized. + + Uninitialized Parameters are a special case of :class:`torch.nn.Parameter` + where the shape of the data is still unknown. + + Unlike a :class:`torch.nn.Parameter`, uninitialized parameters + hold no data and attempting to access some properties, like their shape, + will throw a runtime error. The only operations that can be performed on a uninitialized + parameter are changing its datatype, moving it to a different device and + converting it to a regular :class:`torch.nn.Parameter`. + + The default device or dtype to use when the parameter is materialized can be set + during construction using e.g. ``device='cuda'``. + """ + + cls_to_become = Parameter + + def __new__(cls, requires_grad=True, device=None, dtype=None) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + data = torch.empty(0, **factory_kwargs) + # pyrefly: ignore [bad-return] + return torch.Tensor._make_subclass(cls, data, requires_grad) + + def __deepcopy__(self, memo): + if id(self) in memo: + return memo[id(self)] + else: + result = type(self)(self.requires_grad, self.data.device, self.data.dtype) + memo[id(self)] = result + return result + + +# Metaclass to combine _TensorMeta and the instance check override for Buffer. +class _BufferMeta(torch._C._TensorMeta): + # Make `isinstance(t, Buffer)` return True for custom tensor instances that have the _is_buffer flag. + def __instancecheck__(self, instance) -> bool: + if self is Buffer: + if isinstance(instance, torch.Tensor) and getattr( + instance, "_is_buffer", False + ): + return True + return super().__instancecheck__(instance) + + +class Buffer(torch.Tensor, metaclass=_BufferMeta): + r"""A kind of Tensor that should not be considered a model + parameter. For example, BatchNorm's ``running_mean`` is not a parameter, but is part of the module's state. + + Buffers are :class:`~torch.Tensor` subclasses, that have a + very special property when used with :class:`Module` s -- when they're + assigned as Module attributes they are automatically added to the list of + its buffers, and will appear e.g. in :meth:`~torch.nn.Module.buffers` iterator. + Assigning a Tensor doesn't have such effect. One can still assign a Tensor as explicitly by using + the :meth:`~torch.nn.Module.register_buffer` function. + + Args: + data (Tensor): buffer tensor. + persistent (bool, optional): whether the buffer is part of the module's + :attr:`state_dict`. Default: ``True`` + """ + + def __new__(cls, data=None, *, persistent=True): + if data is None: + data = torch.empty(0) + + t = data.detach().requires_grad_(data.requires_grad) + # pyrefly: ignore [missing-attribute] + t.persistent = persistent + # pyrefly: ignore [missing-attribute] + t._is_buffer = True + return t + + __torch_function__ = _disabled_torch_function_impl + + +class UninitializedBuffer(UninitializedTensorMixin, torch.Tensor): + r"""A buffer that is not initialized. + + Uninitialized Buffer is a a special case of :class:`torch.Tensor` + where the shape of the data is still unknown. + + Unlike a :class:`torch.Tensor`, uninitialized parameters + hold no data and attempting to access some properties, like their shape, + will throw a runtime error. The only operations that can be performed on a uninitialized + parameter are changing its datatype, moving it to a different device and + converting it to a regular :class:`torch.Tensor`. + + The default device or dtype to use when the buffer is materialized can be set + during construction using e.g. ``device='cuda'``. + """ + + cls_to_become = torch.Tensor + + def __new__( + cls, requires_grad=False, device=None, dtype=None, persistent=True + ) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + data = torch.empty(0, **factory_kwargs) + ret = torch.Tensor._make_subclass(cls, data, requires_grad) + # pyrefly: ignore [missing-attribute] + ret.persistent = persistent + # pyrefly: ignore [missing-attribute] + ret._is_buffer = True + # pyrefly: ignore [bad-return] + return ret diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/parameter.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/parameter.pyi new file mode 100644 index 0000000000000000000000000000000000000000..3d1cddb7e8b8bf7dd4aced5f645a8670cf266b3c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/nn/parameter.pyi @@ -0,0 +1,43 @@ +from typing_extensions import TypeIs + +from torch import device, dtype, Tensor + +class Parameter(Tensor): + def __init__(self, data: Tensor = ..., requires_grad: bool = ...) -> None: ... + +def is_lazy( + param: Tensor, +) -> TypeIs[UninitializedParameter | UninitializedBuffer]: ... + +class UninitializedParameter(Tensor): + def __init__(self, data: Tensor = ..., requires_grad: bool = ...) -> None: ... + def materialize( + self, + shape: tuple[int, ...], + device: device | None = None, + dtype: dtype | None = None, + ) -> None: ... + +class Buffer(Tensor): + persistent: bool + def __init__( + self, + data: Tensor = ..., + requires_grad: bool = ..., + persistent: bool = ..., + ) -> None: ... + +class UninitializedBuffer(Tensor): + persistent: bool + def __init__( + self, + data: Tensor = ..., + requires_grad: bool = ..., + persistent: bool = ..., + ) -> None: ... + def materialize( + self, + shape: tuple[int, ...], + device: device | None = None, + dtype: dtype | None = None, + ) -> None: ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c19827c7de1566a20355bf0145a327a6107523c8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/__init__.py @@ -0,0 +1,368 @@ +# mypy: allow-untyped-defs +from __future__ import annotations + + +__all__ = [ + # Modules + "errors", + "ops", + # Public functions + "export", + "is_in_onnx_export", + # Base error + "OnnxExporterError", + "ONNXProgram", +] + +from typing import Any, TYPE_CHECKING + +import torch +from torch._C import _onnx as _C_onnx +from torch._C._onnx import ( # Deprecated members that are excluded from __all__ + OperatorExportTypes as OperatorExportTypes, + TensorProtoDataType as TensorProtoDataType, + TrainingMode as TrainingMode, +) + +from . import errors, ops +from ._internal.exporter._onnx_program import ONNXProgram +from ._internal.torchscript_exporter import ( # Deprecated members that are excluded from __all__ + symbolic_helper, + symbolic_opset10, + symbolic_opset9, + utils, +) +from ._internal.torchscript_exporter._type_utils import ( + JitScalarType, # Deprecated members that are excluded from __all__ +) +from ._internal.torchscript_exporter.utils import ( # Deprecated members that are excluded from __all__ + register_custom_op_symbolic, + select_model_mode_for_export, # pyrefly: ignore # deprecated + unregister_custom_op_symbolic, +) +from .errors import OnnxExporterError + + +if TYPE_CHECKING: + import os + from collections.abc import Callable, Collection, Mapping, Sequence + +# Set namespace for exposed private names +ONNXProgram.__module__ = "torch.onnx" +OnnxExporterError.__module__ = "torch.onnx" + +# TODO(justinchuby): Remove these two properties +producer_name = "pytorch" +producer_version = _C_onnx.PRODUCER_VERSION + + +def export( + model: torch.nn.Module + | torch.export.ExportedProgram + | torch.jit.ScriptModule + | torch.jit.ScriptFunction, + args: tuple[Any, ...] = (), + f: str | os.PathLike | None = None, + *, + kwargs: dict[str, Any] | None = None, + verbose: bool | None = None, + input_names: Sequence[str] | None = None, + output_names: Sequence[str] | None = None, + opset_version: int | None = None, + dynamo: bool = True, + # Dynamo only options + external_data: bool = True, + dynamic_shapes: dict[str, Any] | tuple[Any, ...] | list[Any] | None = None, + custom_translation_table: dict[Callable, Callable | Sequence[Callable]] + | None = None, + report: bool = False, + optimize: bool = True, + verify: bool = False, + profile: bool = False, + dump_exported_program: bool = False, + artifacts_dir: str | os.PathLike = ".", + fallback: bool = False, + # BC options + export_params: bool = True, + keep_initializers_as_inputs: bool = False, + dynamic_axes: Mapping[str, Mapping[int, str]] + | Mapping[str, Sequence[int]] + | None = None, + # Deprecated options + training: _C_onnx.TrainingMode = _C_onnx.TrainingMode.EVAL, + operator_export_type: _C_onnx.OperatorExportTypes = _C_onnx.OperatorExportTypes.ONNX, + do_constant_folding: bool = True, + custom_opsets: Mapping[str, int] | None = None, + export_modules_as_functions: bool | Collection[type[torch.nn.Module]] = False, + autograd_inlining: bool = True, +) -> ONNXProgram | None: + r"""Exports a model into ONNX format. + + Setting ``dynamo=True`` enables the new ONNX export logic + which is based on :class:`torch.export.ExportedProgram` and a more modern + set of translation logic. This is the recommended and default way to export models + to ONNX. + + When ``dynamo=True``: + + The exporter tries the following strategies to get an ExportedProgram for conversion to ONNX. + + #. If the model is already an ExportedProgram, it will be used as-is. + #. Use :func:`torch.export.export` and set ``strict=False``. + #. Use :func:`torch.export.export` and set ``strict=True``. + + Args: + model: The model to be exported. + args: Example positional inputs. Any non-Tensor arguments will be hard-coded into the + exported model; any Tensor arguments will become inputs of the exported model, + in the order they occur in the tuple. + f: Path to the output ONNX model file. E.g. "model.onnx". This argument is kept for + backward compatibility. It is recommended to leave unspecified (None) + and use the returned :class:`torch.onnx.ONNXProgram` to serialize the model + to a file instead. + kwargs: Optional example keyword inputs. + verbose: Whether to enable verbose logging. + input_names: names to assign to the input nodes of the graph, in order. + output_names: names to assign to the output nodes of the graph, in order. + opset_version: The version of the + `default (ai.onnx) opset `_ + to target. You should set ``opset_version`` according to the supported opset versions + of the runtime backend or compiler you want to run the exported model with. + Leave as default (``None``) to use the recommended version, or refer to + the ONNX operators documentation for more information. + dynamo: Whether to export the model with ``torch.export`` ExportedProgram instead of TorchScript. + external_data: Whether to save the model weights as an external data file. + This is required for models with large weights that exceed the ONNX file size limit (2GB). + When False, the weights are saved in the ONNX file with the model architecture. + dynamic_shapes: A dictionary or a tuple of dynamic shapes for the model inputs. Refer to + :func:`torch.export.export` for more details. This is only used (and preferred) when dynamo is True. + Note that dynamic_shapes is designed to be used when the model is exported with dynamo=True, while + dynamic_axes is used when dynamo=False. + custom_translation_table: A dictionary of custom decompositions for operators in the model. + The dictionary should have the callable target in the fx Node as the key (e.g. ``torch.ops.aten.stft.default``), + and the value should be a function that builds that graph using ONNX Script. This option + is only valid when dynamo is True. + report: Whether to generate a markdown report for the export process. This option + is only valid when dynamo is True. + optimize: Whether to optimize the exported model. This option + is only valid when dynamo is True. Default is True. + verify: Whether to verify the exported model using ONNX Runtime. This option + is only valid when dynamo is True. + profile: Whether to profile the export process. This option + is only valid when dynamo is True. + dump_exported_program: Whether to dump the :class:`torch.export.ExportedProgram` to a file. + This is useful for debugging the exporter. This option is only valid when dynamo is True. + artifacts_dir: The directory to save the debugging artifacts like the report and the serialized + exported program. This option is only valid when dynamo is True. + fallback: Whether to fallback to the TorchScript exporter if the dynamo exporter fails. + This option is only valid when dynamo is True. When fallback is enabled, It is + recommended to set dynamic_axes even when dynamic_shapes is provided. + export_params: **When ``f`` is specified**: If false, parameters (weights) will not be exported. + + You can also leave it unspecified and use the returned :class:`torch.onnx.ONNXProgram` + to control how initializers are treated when serializing the model. + keep_initializers_as_inputs: **When ``f`` is specified**: If True, all the + initializers (typically corresponding to model weights) in the + exported graph will also be added as inputs to the graph. If False, + then initializers are not added as inputs to the graph, and only + the user inputs are added as inputs. + + Set this to True if you intend to supply model weights at runtime. + Set it to False if the weights are static to allow for better optimizations + (e.g. constant folding) by backends/runtimes. + + You can also leave it unspecified and use the returned :class:`torch.onnx.ONNXProgram` + to control how initializers are treated when serializing the model. + dynamic_axes: + Prefer specifying ``dynamic_shapes`` when ``dynamo=True`` and when ``fallback`` + is not enabled. + + By default the exported model will have the shapes of all input and output tensors + set to exactly match those given in ``args``. To specify axes of tensors as + dynamic (i.e. known only at run-time), set ``dynamic_axes`` to a dict with schema: + + * KEY (str): an input or output name. Each name must also be provided in ``input_names`` or + ``output_names``. + * VALUE (dict or list): If a dict, keys are axis indices and values are axis names. If a + list, each element is an axis index. + + For example:: + + class SumModule(torch.nn.Module): + def forward(self, x): + return torch.sum(x, dim=1) + + + torch.onnx.export( + SumModule(), + (torch.ones(2, 2),), + "onnx.pb", + input_names=["x"], + output_names=["sum"], + ) + + Produces:: + + input { + name: "x" + ... + shape { + dim { + dim_value: 2 # axis 0 + } + dim { + dim_value: 2 # axis 1 + ... + output { + name: "sum" + ... + shape { + dim { + dim_value: 2 # axis 0 + ... + + While:: + + torch.onnx.export( + SumModule(), + (torch.ones(2, 2),), + "onnx.pb", + input_names=["x"], + output_names=["sum"], + dynamic_axes={ + # dict value: manually named axes + "x": {0: "my_custom_axis_name"}, + # list value: automatic names + "sum": [0], + }, + ) + + Produces:: + + input { + name: "x" + ... + shape { + dim { + dim_param: "my_custom_axis_name" # axis 0 + } + dim { + dim_value: 2 # axis 1 + ... + output { + name: "sum" + ... + shape { + dim { + dim_param: "sum_dynamic_axes_1" # axis 0 + ... + + training: Deprecated option. Instead, set the training mode of the model before exporting. + operator_export_type: Deprecated option. Only ONNX is supported. + do_constant_folding: Deprecated option. + custom_opsets: Deprecated option. + export_modules_as_functions: Deprecated option. + autograd_inlining: Deprecated option. + + Returns: + :class:`torch.onnx.ONNXProgram` if dynamo is True, otherwise None. + + .. versionchanged:: 2.6 + *training* is now deprecated. Instead, set the training mode of the model before exporting. + *operator_export_type* is now deprecated. Only ONNX is supported. + *do_constant_folding* is now deprecated. It is always enabled. + *export_modules_as_functions* is now deprecated. + *autograd_inlining* is now deprecated. + .. versionchanged:: 2.7 + *optimize* is now True by default. + .. versionchanged:: 2.9 + *dynamo* is now True by default. + """ + if dynamo is True or isinstance(model, torch.export.ExportedProgram): + from torch.onnx._internal.exporter import _compat + + if isinstance(args, torch.Tensor): + args = (args,) + # Prepare legacy export parameters for potential fallback + legacy_export_kwargs = { + "training": training, + "operator_export_type": operator_export_type, + "do_constant_folding": do_constant_folding, + "custom_opsets": custom_opsets, + "export_modules_as_functions": export_modules_as_functions, + "autograd_inlining": autograd_inlining, + } + + return _compat.export_compat( + model, + args, + f, + kwargs=kwargs, + export_params=export_params, + verbose=verbose, + input_names=input_names, + output_names=output_names, + opset_version=opset_version, + custom_translation_table=custom_translation_table, + dynamic_axes=dynamic_axes, + keep_initializers_as_inputs=keep_initializers_as_inputs, + external_data=external_data, + dynamic_shapes=dynamic_shapes, + report=report, + optimize=optimize, + verify=verify, + profile=profile, + dump_exported_program=dump_exported_program, + artifacts_dir=artifacts_dir, + fallback=fallback, + legacy_export_kwargs=legacy_export_kwargs, + ) + else: + import warnings + + from ._internal.torchscript_exporter.utils import export + + warnings.warn( + "You are using the legacy TorchScript-based ONNX export. Starting in PyTorch 2.9, " + "the new torch.export-based ONNX exporter has become the default. " + "Learn more about the new export logic: https://docs.pytorch.org/docs/stable/onnx_export.html. " + "For exporting control flow: " + "https://pytorch.org/tutorials/beginner/onnx/export_control_flow_model_to_onnx_tutorial.html", + category=DeprecationWarning, + stacklevel=2, + ) + + if dynamic_shapes: + raise ValueError( + "The exporter only supports dynamic shapes " + "through parameter dynamic_axes when dynamo=False." + ) + + export( + model, + args, + f, # type: ignore[arg-type] + kwargs=kwargs, + export_params=export_params, + verbose=verbose is True, + input_names=input_names, + output_names=output_names, + opset_version=opset_version, + dynamic_axes=dynamic_axes, + keep_initializers_as_inputs=keep_initializers_as_inputs, + training=training, + operator_export_type=operator_export_type, + do_constant_folding=do_constant_folding, + custom_opsets=custom_opsets, + export_modules_as_functions=export_modules_as_functions, + autograd_inlining=autograd_inlining, + ) + return None + + +def is_in_onnx_export() -> bool: + """Returns whether it is in the middle of ONNX export.""" + from torch.onnx._internal.exporter import _flags + from torch.onnx._internal.torchscript_exporter._globals import GLOBALS + + return GLOBALS.in_onnx_export or _flags._is_onnx_exporting diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/_constants.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/_constants.py new file mode 100644 index 0000000000000000000000000000000000000000..87ff04da8cd1e6528bb3f902f56a826f17fc5f87 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/_constants.py @@ -0,0 +1,24 @@ +"""Constant values used in ONNX.""" + +ONNX_ARCHIVE_MODEL_PROTO_NAME = "__MODEL_PROTO" + +ONNX_BASE_OPSET = 9 +ONNX_MIN_OPSET = 7 +ONNX_MAX_OPSET = 23 +ONNX_TORCHSCRIPT_EXPORTER_MAX_OPSET = 20 +ONNX_DEFAULT_OPSET = 20 +ONNX_CONSTANT_FOLDING_MIN_OPSET = 9 + +PYTORCH_GITHUB_ISSUES_URL = "https://github.com/pytorch/pytorch/issues" + +INT64_MAX = 9223372036854775807 +INT32_MAX = 2147483647 +INT16_MAX = 32767 +INT8_MAX = 127 +UINT8_MAX = 255 + +INT64_MIN = -9223372036854775808 +INT32_MIN = -2147483648 +INT16_MIN = -32768 +INT8_MIN = -128 +UINT8_MIN = 0 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/_flags.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/_flags.py new file mode 100644 index 0000000000000000000000000000000000000000..bcc9f633e7584edf65273796f8baf1e70484e827 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/_flags.py @@ -0,0 +1,55 @@ +"""Internal feature flags for torch.onnx. + +NOTE: These flags are experimental only. Any flag here can be removed at any +time without notice. +""" + +import logging +import os + + +logger = logging.getLogger(__name__) + + +def _load_boolean_flag( + name: str, + *, + this_will: str, + deprecated: bool = False, + default: bool = False, +) -> bool: + """Load a boolean flag from environment variable. + + Args: + name: The name of the environment variable. + this_will: A string that describes what this flag will do. + deprecated: Whether this flag is deprecated. + default: The default value if envvar not defined. + """ + undefined = os.getenv(name) is None + state = os.getenv(name) == "1" + if state: + if deprecated: + logger.error( + "Experimental flag %s is deprecated. Please remove it from your environment.", + name, + ) + else: + logger.warning( + "Experimental flag %s is enabled. This will %s.", name, this_will + ) + if undefined: + state = default + return state + + +ENABLE_DRAFT_EXPORT: bool = _load_boolean_flag( + "TORCH_ONNX_ENABLE_DRAFT_EXPORT", + this_will="enable torch.export.draft_export as a strategy for capturing models", + default=False, +) +PREFER_DEFERRED_RUNTIME_ASSERTS_OVER_GUARDS: bool = _load_boolean_flag( + "TORCH_ONNX_PREFER_DEFERRED_RUNTIME_ASSERTS_OVER_GUARDS", + this_will="set prefer_deferred_runtime_asserts_over_guards when calling torch.export", + default=True, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/errors.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..3645e01d7a7a2c5ddd8f44a1f2a3883a9b437fd1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/errors.py @@ -0,0 +1,101 @@ +"""ONNX exporter exceptions.""" + +from __future__ import annotations + + +__all__ = [ + "OnnxExporterWarning", + "SymbolicValueError", + "UnsupportedOperatorError", +] + +import textwrap +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from torch import _C + + +class OnnxExporterWarning(UserWarning): + """Warnings in the ONNX exporter.""" + + +class OnnxExporterError(RuntimeError): + """Errors raised by the ONNX exporter. This is the base class for all exporter errors.""" + + +class UnsupportedOperatorError(OnnxExporterError): + """Raised when an operator is unsupported by the exporter.""" + + # NOTE: This is legacy and is only used by the torchscript exporter + # Clean up when the torchscript exporter is removed + def __init__(self, name: str, version: int, supported_version: int | None) -> None: + if supported_version is not None: + msg = ( + f"Exporting the operator '{name}' to ONNX opset version {version} " + "is not supported. Support for this operator was added in version " + f"{supported_version}, try exporting with this version" + ) + elif name.startswith(("aten::", "prim::", "quantized::")): + msg = ( + f"Exporting the operator '{name}' to ONNX opset version {version} " + "is not supported" + ) + else: + msg = ( + f"ONNX export failed on an operator with unrecognized namespace {name}. " + "If you are trying to export a custom operator, make sure you registered it with " + "the right domain and version." + ) + + super().__init__(msg) + + +class SymbolicValueError(OnnxExporterError): + """Errors around TorchScript values and nodes.""" + + # NOTE: This is legacy and is only used by the torchscript exporter + # Clean up when the torchscript exporter is removed + def __init__(self, msg: str, value: _C.Value) -> None: + message = ( + f"{msg} [Caused by the value '{value}' (type '{value.type()}') in the " + f"TorchScript graph. The containing node has kind '{value.node().kind()}'.] " + ) + + code_location = value.node().sourceRange() + if code_location: + message += f"\n (node defined in {code_location})" + + try: + # Add its input and output to the message. + message += "\n\n" + message += textwrap.indent( + ( + "Inputs:\n" + + ( + "\n".join( + f" #{i}: {input_} (type '{input_.type()}')" + for i, input_ in enumerate(value.node().inputs()) + ) + or " Empty" + ) + + "\n" + + "Outputs:\n" + + ( + "\n".join( + f" #{i}: {output} (type '{output.type()}')" + for i, output in enumerate(value.node().outputs()) + ) + or " Empty" + ) + ), + " ", + ) + except AttributeError: + message += ( + " Failed to obtain its input and output for debugging. " + "Please refer to the TorchScript graph for debugging information." + ) + + super().__init__(message) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/operators.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/operators.py new file mode 100644 index 0000000000000000000000000000000000000000..dd31ba8c304043d6b6a6bad92c60d8f729a034f3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/operators.py @@ -0,0 +1,47 @@ +"""This file provides a location for operators that help exporting models via onnx. + +E.g. `shape_as_tensor` and `reshape_from_tensor_shape` +are to make all dynamic sizes operations traceable. + +NOTE: at one point these functions were implemented differently. +Since then we have implemented these directly in ATen, so this +file is kept purely for backward-compatibility. +""" + +from __future__ import annotations + + +__all__: list[str] = [] + +import torch + + +"""Get the shape of a tensor as a tensor. + +Args: + x (Tensor): The input tensor. + +Returns: + Tensor: A tensor of shape [len(x.shape)] containing the size of each dimension of x. + +Example: + >>> x = torch.randn(2, 3) + >>> shape_as_tensor(x) + tensor([2, 3]) + +""" +shape_as_tensor = torch._shape_as_tensor + +"""Reshape a tensor to the given shape. + +This function is used to make dynamic size operations traceable when exporting models via ONNX. +This function is kept for backward-compatibility. It is implemented directly in ATen. + +Parameters: + x (Tensor): the tensor to be reshaped. + shape (Tensor): the target shape. + +Returns: + Tensor: the reshaped tensor. +""" +reshape_from_tensor_shape = torch._reshape_from_tensor diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_helper.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_helper.py new file mode 100644 index 0000000000000000000000000000000000000000..76b50a8eb3f77a8afa155e924d1734b601711dba --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_helper.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_helper.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_helper import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset10.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset10.py new file mode 100644 index 0000000000000000000000000000000000000000..9bda69b81ab603fdb4932fa69edefa8525b2173f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset10.py @@ -0,0 +1,11 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset10.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset10 import * # noqa: F401,F403 +from torch.onnx._internal.torchscript_exporter.symbolic_opset10 import ( # noqa: F401 + _slice, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset11.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset11.py new file mode 100644 index 0000000000000000000000000000000000000000..276ef7209bf69df1697838ac65354c73ecc29ea4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset11.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset11.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset11 import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset12.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset12.py new file mode 100644 index 0000000000000000000000000000000000000000..63e137734e8a74c8644fb273590db9396c4d9511 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset12.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset12.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset12 import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset13.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset13.py new file mode 100644 index 0000000000000000000000000000000000000000..18aff9295be8cdabfca7d6395ad562bcc7e5834d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset13.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset13.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset13 import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset14.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset14.py new file mode 100644 index 0000000000000000000000000000000000000000..367aa9eb0832afb7a1b9f0b08936ef349656a218 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset14.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset14.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset14 import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset15.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset15.py new file mode 100644 index 0000000000000000000000000000000000000000..e04e3b04521275021659a1cbbd6cae90b3440192 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset15.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset15.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset15 import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset16.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset16.py new file mode 100644 index 0000000000000000000000000000000000000000..9a248bb0f26c5469278e096ad1c7df486748e62d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset16.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset16.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset16 import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset17.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset17.py new file mode 100644 index 0000000000000000000000000000000000000000..800acd446b5dc3739d351f4796603ce23747cdd6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset17.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset17.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset17 import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset18.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset18.py new file mode 100644 index 0000000000000000000000000000000000000000..cc07a60f018d8c7c21ea01e1972252d8b849c1a6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset18.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset18.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset18 import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset19.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset19.py new file mode 100644 index 0000000000000000000000000000000000000000..4f7a54fc1dd38a62b137430dae4a5ccce93d2b73 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset19.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset19.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset19 import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset20.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset20.py new file mode 100644 index 0000000000000000000000000000000000000000..56635a781161106c02b05f2d5b82d229e8f9360f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset20.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset20.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset20 import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset7.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset7.py new file mode 100644 index 0000000000000000000000000000000000000000..c11e769677ec424a8bfc231fc1d21b8c859ef314 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset7.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset7.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset7 import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset8.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset8.py new file mode 100644 index 0000000000000000000000000000000000000000..0e4411649f3e09b2267e76f0307a833d6d15a7b0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset8.py @@ -0,0 +1,8 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset8.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset8 import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset9.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset9.py new file mode 100644 index 0000000000000000000000000000000000000000..bd0f4795340ae8d66f426d30629ab689ee4417a3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/symbolic_opset9.py @@ -0,0 +1,14 @@ +"""Backward compatibility module for torch.onnx.symbolic_opset9.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +from torch.onnx._internal.torchscript_exporter.symbolic_opset9 import * # noqa: F401,F403 +from torch.onnx._internal.torchscript_exporter.symbolic_opset9 import ( # noqa: F401 + _prepare_onnx_paddings, + _reshape_from_tensor, + _slice, + _var_mean, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/testing.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..aa168b32746f5f0ee67dda5c1035d8c975f2ac02 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/testing.py @@ -0,0 +1,8 @@ +"""Utilities to aid in testing exported ONNX models.""" + +__all__ = ["assert_onnx_program"] + +from torch.onnx._internal.exporter._testing import assert_onnx_program + + +assert_onnx_program.__module__ = "torch.onnx.testing" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ec748568f642eb46e15e8b282aded33a7546f2f8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/utils.py @@ -0,0 +1,9 @@ +"""Backward compatibility module for torch.onnx.utils.""" + +from __future__ import annotations + + +__all__: list[str] = [] + +# pyrefly: ignore [deprecated] +from torch.onnx._internal.torchscript_exporter.utils import * # noqa: F401,F403 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/verification.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/verification.py new file mode 100644 index 0000000000000000000000000000000000000000..70d901acb47a941bf03caa25f6e8dfaadebe170b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/onnx/verification.py @@ -0,0 +1,12 @@ +"""A set of tools to verify the correctness of ONNX models.""" + +__all__ = ["VerificationInfo", "verify_onnx_program"] + +from torch.onnx._internal.exporter._verification import ( + VerificationInfo, + verify_onnx_program, +) + + +VerificationInfo.__module__ = "torch.onnx.verification" +verify_onnx_program.__module__ = "torch.onnx.verification" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1060a6287a8e6e4b59aa1a46527cf0001de1ccfe --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/__init__.py @@ -0,0 +1,66 @@ +""" +:mod:`torch.optim` is a package implementing various optimization algorithms. + +Most commonly used methods are already supported, and the interface is general +enough, so that more sophisticated ones can also be easily integrated in the +future. +""" + +from torch.optim import lr_scheduler as lr_scheduler, swa_utils as swa_utils +from torch.optim._adafactor import Adafactor as Adafactor +from torch.optim._muon import Muon as Muon +from torch.optim.adadelta import Adadelta as Adadelta +from torch.optim.adagrad import Adagrad as Adagrad +from torch.optim.adam import Adam as Adam +from torch.optim.adamax import Adamax as Adamax +from torch.optim.adamw import AdamW as AdamW +from torch.optim.asgd import ASGD as ASGD +from torch.optim.lbfgs import LBFGS as LBFGS +from torch.optim.nadam import NAdam as NAdam +from torch.optim.optimizer import Optimizer as Optimizer +from torch.optim.radam import RAdam as RAdam +from torch.optim.rmsprop import RMSprop as RMSprop +from torch.optim.rprop import Rprop as Rprop +from torch.optim.sgd import SGD as SGD +from torch.optim.sparse_adam import SparseAdam as SparseAdam + + +Adafactor.__module__ = "torch.optim" +Muon.__module__ = "torch.optim" + + +del adadelta # type: ignore[name-defined] # noqa: F821 +del adagrad # type: ignore[name-defined] # noqa: F821 +del adam # type: ignore[name-defined] # noqa: F821 +del adamw # type: ignore[name-defined] # noqa: F821 +del sparse_adam # type: ignore[name-defined] # noqa: F821 +del adamax # type: ignore[name-defined] # noqa: F821 +del asgd # type: ignore[name-defined] # noqa: F821 +del sgd # type: ignore[name-defined] # noqa: F821 +del radam # type: ignore[name-defined] # noqa: F821 +del rprop # type: ignore[name-defined] # noqa: F821 +del rmsprop # type: ignore[name-defined] # noqa: F821 +del optimizer # type: ignore[name-defined] # noqa: F821 +del nadam # type: ignore[name-defined] # noqa: F821 +del lbfgs # type: ignore[name-defined] # noqa: F821 + +__all__ = [ + "Adafactor", + "Adadelta", + "Adagrad", + "Adam", + "Adamax", + "AdamW", + "ASGD", + "LBFGS", + "lr_scheduler", + "Muon", + "NAdam", + "Optimizer", + "RAdam", + "RMSprop", + "Rprop", + "SGD", + "SparseAdam", + "swa_utils", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/_adafactor.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/_adafactor.py new file mode 100644 index 0000000000000000000000000000000000000000..6aed25a36aa82f24cf6afe9df78c385723a09c31 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/_adafactor.py @@ -0,0 +1,661 @@ +# mypy: allow-untyped-decorators +# mypy: allow-untyped-defs +from typing import cast, TYPE_CHECKING + +import torch +from torch import Tensor + +from .optimizer import ( + _disable_dynamo_if_unsupported, + _get_scalar_dtype, + _maximize_doc, + _params_doc, + _to_scalar, + Optimizer, + ParamsT, + TensorListList, +) + + +__all__ = ["Adafactor", "adafactor"] + + +class Adafactor(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-2, + beta2_decay: float = -0.8, + eps: tuple[float | None, float] = (None, 1e-3), + d: float = 1.0, + weight_decay: float = 0.0, + *, + foreach: bool | None = None, + maximize: bool = False, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Learning rate should be >= 0 but is: {lr}") + if not 0.0 >= beta2_decay: + raise ValueError(f"beta2_decay should be <= 0 but is: {beta2_decay}") + if eps[0] is not None and not 0.0 <= eps[0]: + raise ValueError(f"epsilon1 should be >= 0 but is: {eps[0]}") + if not 0.0 <= eps[1]: + raise ValueError(f"epsilon2 should be >= 0 but is: {eps[1]}") + if not 1.0 <= d: + raise ValueError(f"Clipping threshold d should be >= 1 but is: {d}") + if not 0.0 <= weight_decay: + raise ValueError(f"weight_decay should be >= 0 but is: {weight_decay}") + defaults = { + "lr": lr, + "beta2_decay": beta2_decay, + "eps": eps, + "d": d, + "weight_decay": weight_decay, + "foreach": foreach, + "maximize": maximize, + } + super().__init__(params, defaults) + + def __setstate__(self, state): + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("foreach", None) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = torch.tensor(step_val, dtype=_get_scalar_dtype()) + + def _init_group( + self, + group, + params_with_grad, + grads, + row_vars, + col_vars, + variances, + state_steps, + ) -> bool: + for p in group["params"]: + if p.grad is None: + continue + if torch.is_complex(p): + raise RuntimeError("Adafactor does not support complex parameters") + if p.grad.is_sparse: + raise RuntimeError("Adafactor does not support sparse gradients") + + params_with_grad.append(p) + grads.append(p.grad) + + state = self.state[p] + + # State initialization + if len(state) == 0: + # note(crcrpar): Deliberately host `step` on CPU if both capturable and fused are off. + # This is because kernel launches are costly on CUDA and XLA. + state["step"] = torch.tensor(0.0, dtype=_get_scalar_dtype()) + + if p.grad.dim() > 1: + row_shape = list(p.grad.shape) + row_shape[-1] = 1 + # Row factor of variance, NOT the same shape as grads (will be reduced along last dim) + state["row_var"] = p.grad.new_zeros(row_shape) + + col_shape = list(p.grad.shape) + col_shape[-2] = 1 + # Col factor of variance, NOT the same shape as grads (will be reduced along penultimate dim) + state["col_var"] = p.grad.new_zeros(col_shape) + else: + state["variance"] = torch.zeros_like( + p.grad, memory_format=torch.preserve_format + ) + + row_vars.append(state.get("row_var", None)) + col_vars.append(state.get("col_var", None)) + variances.append(state.get("variance", None)) + state_steps.append(state["step"]) + return False # has_complex + + @torch.no_grad() + def step(self, closure=None): + r"""Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + row_vars: list[Tensor | None] = [] + col_vars: list[Tensor | None] = [] + variances: list[Tensor | None] = [] + state_steps: list[Tensor] = [] + eps1, eps2 = group["eps"] + + has_complex = self._init_group( + group, + params_with_grad, + grads, + row_vars, + col_vars, + variances, + state_steps, + ) + + adafactor( + params_with_grad, + grads, + row_vars, + col_vars, + variances, + state_steps, + d=group["d"], + lr=group["lr"], + beta2_decay=group["beta2_decay"], + weight_decay=group["weight_decay"], + eps1=eps1, + eps2=eps2, + foreach=group["foreach"], + maximize=group["maximize"], + grad_scale=getattr(self, "grad_scale", None), + found_inf=getattr(self, "found_inf", None), + has_complex=has_complex, + ) + + return loss + + +Adafactor.__doc__ = ( + r"""Implements Adafactor algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{(lr)}, \: \tau + \text{(}\beta_2\text{ decay)}, \: \theta_0 \text{(params)}, \: f(\theta) \text{(objective)}, \\ + &\hspace{15mm} \: \epsilon_1, \epsilon_2 \text{ (epsilons)}, \: d \text{(clipping threshold)}, \\ + &\hspace{15mm} \: \lambda \text{(weight decay)}, + \: \textit{maximize} \\ + &\textbf{initialize} : \: R_0 \leftarrow 0 \text{ (second moment row factor)}, \\ + &\hspace{23mm} \: C_0 \leftarrow 0 \text{ (second moment col factor)}, \\ + &\hspace{23mm} \: \widehat{V}_0 \leftarrow 0 \text{ (second moment for vectors)} \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + + &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\ + &\hspace{10mm}G_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}G_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\widehat{\beta}_{2_t} \leftarrow 1 - t^{\tau} \\ + &\hspace{5mm}\rho_t \leftarrow min(lr, \frac{1}{\sqrt{t}}) \\ + &\hspace{5mm}\alpha_t \leftarrow max(\epsilon_2, + \text{RMS}(\theta_{t-1}))\rho_t \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1} \\ + &\hspace{5mm}\textbf{if} \: \text{dim}(G_t) > 1: \\ + &\hspace{10mm}R_t \leftarrow \widehat{\beta}_{2_t}R_{t-1}+ + (1-\widehat{\beta}_{2_t})(G_t \odot G_t) \cdot 1_m \\ + &\hspace{10mm}C_t \leftarrow \widehat{\beta}_{2_t}C_{t-1}+ + (1-\widehat{\beta}_{2_t}) 1^\top_n \cdot (G_t \odot G_t) \\ + &\hspace{10mm}\widehat{V}_t \leftarrow + \frac{R_t \cdot C_t}{max(1^\top_n \cdot R_t, \epsilon_1)} \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}\widehat{V}_t \leftarrow \widehat{\beta}_{2_t}\widehat{V}_{t-1}+ + (1-\widehat{\beta}_{2_t}) \cdot (G_t \odot G_t) \\ + &\hspace{5mm}U_t \leftarrow + \frac{G_t}{max(\sqrt{\widehat{V}_t}, \epsilon_1)} \\ + &\hspace{5mm}\widehat{U}_t \leftarrow \frac{U_t}{max(1, \frac{\text{RMS}(U_t)}{d})} \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \alpha_t \widehat{U}_t \\ + + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `Adafactor: Adaptive Learning Rates with Sublinear Memory Cost`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): unlike other optimizers, Adafactor does not require a + learning rate, and Noam Shazeer and Mitchell Stern do not use lr at all. + Deviating from the paper, this implementation uses lr for applying weight + decay and as the maximum value for relative step size rho_t. Note that in + the paper, a constant of 0.01 is used as the maximum value for relative + step size, and so we set 0.01 as the default value. (default: 1e-2) + beta2_decay (float, optional): the decay rate of beta2. beta2 standardly refers + to the coefficient used for computing the running average of the gradient + squared. (default: -0.8) + eps (Tuple[float, float], optional): epsilon1 is the term added to the denominator + of the update calculation to improve numerical stability. This use of epsilon1 + deviates from the algorithm written in the paper! See note below for more details. + epsilon2 is the term used to avoid having too small a weight update when applying + parameter scaling. (default: (None, 1e-3)) + d (float, optional): the clipping threshold, used to avoid larger-than-desired + updates. + weight_decay (float, optional): weight decay coefficient (default: 1e-2) + foreach (bool, optional): whether foreach implementation of optimizer is used. Note + that the foreach implementation uses ~ sizeof(params) more peak memory than the + for-loop version due to the intermediates being a tensorlist vs just one tensor. + As Adafactor is commonly used when memory is prohibitive, Adafactor will default + to the slower single tensor for-loop implementation unless this flag is explicitly + True. This behavior is contrary to other optimizers, which will attempt defaulting + to foreach on CUDA for faster runtime. (default: None) + {_maximize_doc}""" + + r""" + .. Note:: + The implementation of Adafactor subtly differs from Noam Shazeer and Mitchell Stern + and implementations in some other frameworks with its use of learning rate and + :math:`\epsilon_1`. + + Regarding the learning rate hyperparameter: Noam Shazeer and Mitchell Stern do not + use lr at all, as the stated algorithm uses :math:`\rho_t` and update clipping to + affect the step size. + + This implementation allows `lr` to influence the maximum value for :math:`\rho_t`: + + .. math:: + \begin{aligned} + &\hspace{5mm}\rho_t \leftarrow min(lr, \frac{1}{\sqrt{t}}) + \end{aligned} + + This differs from Noam Shazeer and Mitchell Stern, who use a constant of 0.01 as + the maximum value of :math:`\rho_t` + + .. math:: + \begin{aligned} + &\hspace{5mm}\rho_t \leftarrow min(0.01, \frac{1}{\sqrt{t}}) + \end{aligned} + + Noam Shazeer and Mitchell Stern do not enforce an opinion on how weight decay should + be computed, and so we use the learning rate as a coefficient for decoupled weight + decay, similar to what is suggested in `Decoupled Weight Decay Regularization`_. + + Regarding the use of :math:`\epsilon_1`: The implementation attempts to replicate the + presumed intention of Noam Shazeer and Mitchell Stern to use :math:`\epsilon_1` as + a stabilizing term when the squared gradient becomes small. + + This stabilization can be written as + + .. math:: + \begin{aligned} + &\hspace{5mm}R_t \leftarrow \widehat{\beta}_{2_t}R_{t-1}+ + (1-\widehat{\beta}_{2_t})(G_t \odot G_t + 1_n \cdot 1^\top_m) \cdot 1_m \\ + &\hspace{5mm}C_t \leftarrow \widehat{\beta}_{2_t}C_{t-1}+ + (1-\widehat{\beta}_{2_t}) 1^\top_n \cdot (G_t \odot G_t + 1_n \cdot 1^\top_m) \\ + &\hspace{5mm}\widehat{V}_t \leftarrow + \frac{R_t \cdot C_t}{max(1^\top_n \cdot R_t, \epsilon_1)} \\ + &\hspace{5mm}U_t \leftarrow \frac{G_t}{max(\sqrt{\widehat{V}_t}, \epsilon_1)} \\ + \end{aligned} + + where the row and column factors of gradient squared :math:`R_t` and :math:`C_t` + are left alone, and we apply :math:`\epsilon_1` at the final calculation of + the variance estimate :math:`\widehat{V}_t` and for the update :math:`U_t`. + + This is in contrast to Noam Shazeer and Mitchell Stern and other frameworks which + apply :math:`\epsilon_1` to both row and column factors of the squared gradient, but + not in the calculations after: + + .. math:: + \begin{aligned} + &\hspace{5mm}R_t \leftarrow \widehat{\beta}_{2_t}R_{t-1}+ + (1-\widehat{\beta}_{2_t})(G_t \odot G_t + \epsilon_1 1_n \cdot 1^\top_m) \cdot 1_m \\ + &\hspace{5mm}C_t \leftarrow \widehat{\beta}_{2_t}C_{t-1}+ + (1-\widehat{\beta}_{2_t}) 1^\top_n \cdot (G_t \odot G_t + \epsilon_1 1_n \cdot 1^\top_m) \\ + &\hspace{5mm}\widehat{V}_t \leftarrow \frac{R_t \cdot C_t}{1^\top_n \cdot R_t} \\ + &\hspace{5mm}U_t \leftarrow \frac{G_t}{\sqrt{\widehat{V}_t}} \\ + \end{aligned} + + You may note that Noam Shazeer and Mitchell Stern describe using the sum of squared gradients, + while this implementation uses the mean instead. This choice is mathematically equivalent and + allows for greater numerical stability for large sums. + + .. _Adafactor\: Adaptive Learning Rates with Sublinear Memory Cost: + https://arxiv.org/pdf/1804.04235 + .. _Decoupled Weight Decay Regularization: + https://arxiv.org/abs/1711.05101 + """ +) + + +def _single_tensor_adafactor( + params: list[Tensor], + grads: list[Tensor], + # If grad is 1-dimensional (aka a vector), there is no factorization necessary + # so row_var and col_var will be None while variance will be filled. + # Contrarily, for a grad with multiple dimensions, we will factor along the last + # 2 dimensions, and so row_var and col_var will be filled and variance will be None. + row_vars: list[Tensor | None], + col_vars: list[Tensor | None], + variances: list[Tensor | None], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + d: float, + lr: Tensor | float, + beta2_decay: float, + weight_decay: float, + eps1: float | None, + eps2: float, + maximize: bool, + has_complex: bool, +) -> None: + if grad_scale is not None or found_inf is not None: + raise AssertionError("Grad scaling should occur outside of optimizer.step()") + + if torch.jit.is_scripting(): + # this assert is due to JIT being dumb and not realizing that the ops below + # have overloads to handle both float and Tensor lrs, so we just assert it's + # a float since most people using JIT are using floats + if not isinstance(lr, float): + raise AssertionError(f"Expected lr to be a float, but got {type(lr)}") + + else: + lr = _to_scalar(lr) + + for i, param in enumerate(params): + grad = grads[i] if not maximize else -grads[i] + step_t = state_steps[i] + row_var = row_vars[i] + col_var = col_vars[i] + variance = variances[i] + if eps1 is None: + eps1 = torch.finfo(param.dtype).eps + + # update step + step_t += 1 + step_float = step_t.item() + + one_minus_beta2_t = step_float**beta2_decay + rho_t = min(lr, 1 / (step_float**0.5)) + alpha = max(eps2, param.norm(2).item() / (param.numel() ** 0.5)) * rho_t + + # Perform stepweight decay + if weight_decay != 0: + param.mul_(1 - lr * weight_decay) + + if grad.dim() > 1: + if row_var is None or col_var is None: + raise AssertionError( + "row_var and col_var should be defined when grad is multidimensional" + ) + # same as (g * g).mean(dim=-1) w/o materializing an intermediate size g + row_mean = ( + torch.norm(grad, dim=-1, keepdim=True).square_().div_(grad.size(-1)) + ) + row_var.lerp_(row_mean, one_minus_beta2_t) + # same as (g * g).mean(dim=-2) w/o materializing an intermediate size g + col_mean = ( + torch.norm(grad, dim=-2, keepdim=True).square_().div_(grad.size(-2)) + ) + col_var.lerp_(col_mean, one_minus_beta2_t) + var_estimate = row_var @ col_var + var_estimate.div_(row_var.mean(dim=-2, keepdim=True).clamp_(min=eps1)) + else: + if variance is None: + raise AssertionError("variance should be defined when grad is a vector") + grad_squared = grad * grad + variance.lerp_(grad_squared, one_minus_beta2_t) + # avoid writing into variance during update + var_estimate = variance.clone() + + # square the eps1 as we sqrt after to keep eps1's magnitude + update = var_estimate.clamp_(min=eps1 * eps1).rsqrt_() + update.mul_(grad) + denom = max(1.0, update.norm(2).item() / ((update.numel() ** 0.5) * d)) + param.add_(update, alpha=-alpha / denom) + + +def _group_tensors_by_device_dtype_and_is_multidim( + tensorlists: TensorListList, +) -> dict[ + tuple[torch.device | None, torch.dtype | None, bool], + list[list[Tensor | None]], +]: + """Groups tensors by device, dtype, AND multidimensionality -- whether the tensor + has multiple dims or just one dim (is a vector). This allows the foreach impl of + Adafactor to assume that every group of params will either be factored or not.""" + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype(tensorlists) + ultra_grouped_tensors: dict[ + tuple[torch.device | None, torch.dtype | None, bool], + list[list[Tensor | None]], + ] = {} + for (device, dtype), (tensorlists, _) in grouped_tensors.items(): + matrix_key = (device, dtype, True) + vector_key = (device, dtype, False) + + # assumes grad is the second tensorlist + for j, tensor in enumerate(tensorlists[1]): + if tensor is None: + raise AssertionError("grad should not be None") + if tensor.dim() > 1: + if matrix_key not in ultra_grouped_tensors: + ultra_grouped_tensors[matrix_key] = [[] for _ in tensorlists] + for i in range(len(tensorlists)): + ultra_grouped_tensors[matrix_key][i].append(tensorlists[i][j]) + else: + if vector_key not in ultra_grouped_tensors: + ultra_grouped_tensors[vector_key] = [[] for _ in tensorlists] + for i in range(len(tensorlists)): + ultra_grouped_tensors[vector_key][i].append(tensorlists[i][j]) + return ultra_grouped_tensors + + +def _multi_tensor_adafactor( + params: list[Tensor], + grads: list[Tensor], + # If grad is 1-dimensional (aka a vector), there is no factorization necessary + # so row_var and col_var will be None while variance will be filled. + # Contrarily, for a grad with multiple dimensions, we will factor along the last + # 2 dimensions, and so row_var and col_var will be filled and variance will be None. + row_vars: list[Tensor | None], + col_vars: list[Tensor | None], + variances: list[Tensor | None], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + d: float, + lr: Tensor | float, + beta2_decay: float, + weight_decay: float, + eps1: float | None, + eps2: float, + maximize: bool, + has_complex: bool, +) -> None: + if len(params) == 0: + return + + if grad_scale is not None or found_inf is not None: + raise AssertionError("Grad scaling should occur outside of optimizer.step()") + + lr = _to_scalar(lr) + + grouped_tensors = _group_tensors_by_device_dtype_and_is_multidim( + [params, grads, row_vars, col_vars, variances, state_steps] # type: ignore[list-item] + ) + for (_, dtype, is_multidim), ( + ( + device_params_, + device_grads_, + device_row_vars_, + device_col_vars_, + device_variances_, + device_state_steps_, + ) + ) in grouped_tensors.items(): + device_params = cast(list[Tensor], device_params_) + device_grads = cast(list[Tensor], device_grads_) + device_state_steps = cast(list[Tensor], device_state_steps_) + if eps1 is None: + if dtype is None: + raise AssertionError( + "dtype is needed to compute eps1 when eps1 is unset" + ) + eps1 = torch.finfo(dtype).eps + + if TYPE_CHECKING: + assert device_state_steps[0] is not None + + if maximize: + device_grads = torch._foreach_neg(device_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and device_state_steps[0].is_cpu: + torch._foreach_add_( + device_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(device_state_steps, 1.0) + + one_minus_beta2_ts = [] + beta2_ts = [] + rho_ts = [] + for s in device_state_steps: + one_minus_beta2_ts.append(s.item() ** beta2_decay) + beta2_ts.append(1 - s.item() ** beta2_decay) + rho_ts.append(min(lr, 1 / (s.item() ** 0.5))) + + alphas = [ + max(eps2, p.norm(2).item() / (p.numel() ** 0.5)) * r + for p, r in zip(device_params, rho_ts, strict=True) + ] + + # Perform stepweight decay + if weight_decay != 0: + torch._foreach_mul_(device_params, 1 - lr * weight_decay) + + if is_multidim: + device_row_vars = cast(list[Tensor], device_row_vars_) + device_col_vars = cast(list[Tensor], device_col_vars_) + if device_row_vars[0] is None or device_col_vars[0] is None: + raise AssertionError( + "row_var and col_var should be defined when grad is multidimensional" + ) + # same as (g * g).mean(dim=-1) w/o materializing an intermediate size g + row_means = [ + torch.norm(grad, dim=-1, keepdim=True) for grad in device_grads + ] + torch._foreach_mul_(row_means, row_means) + torch._foreach_div_(row_means, [grad.size(-1) for grad in device_grads]) + torch._foreach_lerp_(device_row_vars, row_means, one_minus_beta2_ts) + del row_means + + # same as (g * g).mean(dim=-2) w/o materializing an intermediate size g + col_means = [ + torch.norm(grad, dim=-2, keepdim=True) for grad in device_grads + ] + torch._foreach_mul_(col_means, col_means) + torch._foreach_div_(col_means, [grad.size(-2) for grad in device_grads]) + torch._foreach_lerp_(device_col_vars, col_means, one_minus_beta2_ts) + del col_means + + var_estimates = [ + row_var @ col_var + for row_var, col_var in zip( + device_row_vars, device_col_vars, strict=True + ) + ] + row_var_means = [ + row_var.mean(dim=-2, keepdim=True) for row_var in device_row_vars + ] + torch._foreach_clamp_min_(row_var_means, eps1) + torch._foreach_div_(var_estimates, row_var_means) + del row_var_means + else: + device_variances = cast(list[Tensor], device_variances_) + if device_variances[0] is None: + raise AssertionError("variance should be defined when grad is a vector") + + grads_squared = torch._foreach_mul(device_grads, device_grads) + torch._foreach_lerp_(device_variances, grads_squared, one_minus_beta2_ts) + del grads_squared + + # avoid writing into variance during update + var_estimates = [v.clone() for v in device_variances] + + # square the eps1 as we sqrt after to keep eps1's magnitude + torch._foreach_clamp_min_(var_estimates, eps1 * eps1) + torch._foreach_rsqrt_(var_estimates) + torch._foreach_mul_(var_estimates, device_grads) + updates = var_estimates + + alphas = [ + -a / (max(1.0, update.norm(2).item() / ((update.numel() ** 0.5) * d))) + for a, update in zip(alphas, updates, strict=True) + ] + torch._foreach_mul_(updates, alphas) + torch._foreach_add_(device_params, updates) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_adafactor) +def adafactor( + params: list[Tensor], + grads: list[Tensor], + row_vars: list[Tensor | None], + col_vars: list[Tensor | None], + variances: list[Tensor | None], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, + has_complex: bool = False, + *, + d: float, + lr: float | Tensor, + beta2_decay: float, + weight_decay: float, + eps1: float, + eps2: float, + maximize: bool, +) -> None: + r"""Functional API that performs Adafactor algorithm computation. + + See :class:`~torch.optim.Adafactor` for details. + """ + if not torch.compiler.is_compiling() and not all( + isinstance(t, torch.Tensor) for t in state_steps + ): + raise RuntimeError( + "`state_steps` argument must contain a list of singleton tensors" + ) + + if foreach: + func = _multi_tensor_adafactor + else: + func = _single_tensor_adafactor + + func( + params, + grads, + row_vars, + col_vars, + variances, + state_steps, + d=d, + lr=lr, + beta2_decay=beta2_decay, + weight_decay=weight_decay, + eps1=eps1, + eps2=eps2, + maximize=maximize, + grad_scale=grad_scale, + found_inf=found_inf, + has_complex=has_complex, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/_functional.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/_functional.py new file mode 100644 index 0000000000000000000000000000000000000000..ba97bc997937889b93ca30b8b783f38dd895a227 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/_functional.py @@ -0,0 +1,84 @@ +# mypy: allow-untyped-defs +r"""Functional interface.""" + +import math + +from torch import Tensor + +from .adadelta import adadelta # type: ignore[attr-defined] # noqa: F401 +from .adagrad import _make_sparse, adagrad # type: ignore[attr-defined] # noqa: F401 +from .adam import adam # type: ignore[attr-defined] # noqa: F401 +from .adamax import adamax # type: ignore[attr-defined] # noqa: F401 +from .adamw import adamw # type: ignore[attr-defined] # noqa: F401 +from .asgd import asgd # type: ignore[attr-defined] # noqa: F401 +from .nadam import nadam # type: ignore[attr-defined] # noqa: F401 +from .radam import radam # type: ignore[attr-defined] # noqa: F401 +from .rmsprop import rmsprop # type: ignore[attr-defined] # noqa: F401 +from .rprop import rprop # type: ignore[attr-defined] # noqa: F401 +from .sgd import sgd # type: ignore[attr-defined] # noqa: F401 + + +# TODO: use foreach API in optim._functional to do all the computation + + +def sparse_adam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + state_steps: list[int], + *, + eps: float, + beta1: float, + beta2: float, + lr: float, + maximize: bool, +) -> None: + r"""Functional API that performs Sparse Adam algorithm computation. + + See :class:`~torch.optim.SparseAdam` for details. + """ + for i, param in enumerate(params): + grad = grads[i] + grad = grad if not maximize else -grad + grad = grad.coalesce() # the update is non-linear so indices must be unique + grad_indices = grad._indices() + grad_values = grad._values() + if grad_values.numel() == 0: + # Skip update for empty grad + continue + size = grad.size() + + exp_avg = exp_avgs[i] + exp_avg_sq = exp_avg_sqs[i] + step = state_steps[i] + + def make_sparse(values): + constructor = grad.new + if grad_indices.dim() == 0 or values.dim() == 0: + return constructor().resize_as_(grad) + return constructor(grad_indices, values, size) + + # Decay the first and second moment running average coefficient + # old <- b * old + (1 - b) * new + # <==> old += (1 - b) * (new - old) + old_exp_avg_values = exp_avg.sparse_mask(grad)._values() + exp_avg_update_values = grad_values.sub(old_exp_avg_values).mul_(1 - beta1) + exp_avg.add_(make_sparse(exp_avg_update_values)) + old_exp_avg_sq_values = exp_avg_sq.sparse_mask(grad)._values() + exp_avg_sq_update_values = ( + grad_values.pow(2).sub_(old_exp_avg_sq_values).mul_(1 - beta2) + ) + exp_avg_sq.add_(make_sparse(exp_avg_sq_update_values)) + + # Dense addition again is intended, avoiding another sparse_mask + numer = exp_avg_update_values.add_(old_exp_avg_values) + exp_avg_sq_update_values.add_(old_exp_avg_sq_values) + denom = exp_avg_sq_update_values.sqrt_().add_(eps) + del exp_avg_update_values, exp_avg_sq_update_values + + bias_correction1 = 1 - beta1**step + bias_correction2 = 1 - beta2**step + step_size = lr * math.sqrt(bias_correction2) / bias_correction1 + + param.add_(make_sparse(-step_size * numer.div_(denom))) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/_muon.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/_muon.py new file mode 100644 index 0000000000000000000000000000000000000000..e441c8b911b2f80b33857e6509aebe7ec38cfb45 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/_muon.py @@ -0,0 +1,360 @@ +# mypy: allow-untyped-defs +# mypy: disable-error-code=arg-type +"""Implementation of the Muon optimizer.""" + +import math +from collections.abc import MutableMapping + +import torch +from torch import Tensor + +from .optimizer import ( + _disable_dynamo_if_unsupported, + _params_doc, + _to_scalar, + Optimizer, + ParamsT, +) + + +__all__ = ["Muon"] + +# Constants from Keller Jordan's Muon post: https://kellerjordan.github.io/posts/muon/ +# github permlink: https://github.com/KellerJordan/Muon/blob/f90a42b28e00b8d9d2d05865fe90d9f39abcbcbd/muon.py#L16 +EPS = 1e-7 +DEFAULT_A = 3.4445 +DEFAULT_B = -4.7750 +DEFAULT_C = 2.0315 +DEFAULT_NS_STEPS = 5 + + +def _zeropower_via_newtonschulz( + grad: Tensor, ns_coefficients: tuple[float, float, float], ns_steps: int, eps: float +) -> Tensor: + """ + Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a + quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose + of minimizing steps, it turns out to be empirically effective to keep increasing the slope at + zero even beyond the point where the iteration no longer converges all the way to one everywhere + on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T + where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model + performance at all relative to UV^T, where USV^T = G is the SVD. + + Implementation reference: https://github.com/KellerJordan/Muon/blob/master/muon.py + with suggestions by @jxbz, @leloykun, and @YouJiacheng. + """ + if ns_steps >= 100: + raise ValueError( + "Number of steps must be less than 100 for computational efficiency" + ) + if len(grad.shape) != 2: + raise ValueError("Input tensor gradient must be a 2D matrix") + if len(ns_coefficients) != 3: + raise ValueError("Coefficients must be a tuple of exactly 3 values") + a, b, c = ns_coefficients + ortho_grad = grad.bfloat16() + if grad.size(0) > grad.size(1): + ortho_grad = ortho_grad.T + # Ensure spectral norm is at most 1 + ortho_grad.div_(ortho_grad.norm().clamp(min=eps)) + # Perform the NS iterations + for _ in range(ns_steps): + gram_matrix = ortho_grad @ ortho_grad.T + gram_update = torch.addmm( + gram_matrix, gram_matrix, gram_matrix, beta=b, alpha=c + ) + ortho_grad = torch.addmm(ortho_grad, gram_update, ortho_grad, beta=a) + + if grad.size(0) > grad.size(1): + ortho_grad = ortho_grad.T + return ortho_grad + + +def _adjust_lr(lr: float, adjust_lr_fn: str | None, param_shape: torch.Size) -> float: + """Default learning rate adjustment used by Muon.""" + A, B = param_shape[:2] + + if adjust_lr_fn is None or adjust_lr_fn == "original": + # pyrefly: ignore [no-matching-overload] + adjusted_ratio = math.sqrt(max(1, A / B)) + elif adjust_lr_fn == "match_rms_adamw": + adjusted_ratio = 0.2 * math.sqrt(max(A, B)) + else: + adjusted_ratio = 1.0 + return lr * adjusted_ratio + + +class Muon(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float = 1e-3, + weight_decay: float = 0.1, + momentum: float = 0.95, + nesterov: bool = True, + ns_coefficients: tuple[float, float, float] = (DEFAULT_A, DEFAULT_B, DEFAULT_C), + eps: float = EPS, + ns_steps: int = DEFAULT_NS_STEPS, + adjust_lr_fn: str | None = None, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Learning rate should be >= 0 but is: {lr}") + if not 0.0 <= momentum: + raise ValueError(f"momentum should be >= 0 but is: {momentum}") + if not 0.0 <= weight_decay: + raise ValueError(f"weight decay should be >= 0 but is: {weight_decay}") + if adjust_lr_fn is not None and adjust_lr_fn not in [ + "original", + "match_rms_adamw", + ]: + raise ValueError( + f"Adjust learning rate function {adjust_lr_fn} is not supported" + ) + + defaults = { + "lr": lr, + "weight_decay": weight_decay, + "momentum": momentum, + "nesterov": nesterov, + "ns_coefficients": ns_coefficients, + "eps": eps, + "ns_steps": ns_steps, + "adjust_lr_fn": adjust_lr_fn, + } + super().__init__(params, defaults) + + for group in self.param_groups: + for p in group["params"]: + if p.ndim != 2: + raise ValueError( + f"Muon only supports 2D parameters whereas we found a parameter with size: {p.size()}" + ) + + def _init_group( + self, + group: MutableMapping, + params_with_grad: list[Tensor], + grads: list[Tensor], + muon_momentum_bufs: list[Tensor], + ) -> bool: + for p in group["params"]: + if p.grad is None: + continue + + if torch.is_complex(p): + raise RuntimeError("Muon does not support complex parameters") + if p.grad.is_sparse: + raise RuntimeError("Muon does not support sparse gradients") + + params_with_grad.append(p) + grads.append(p.grad) + + state = self.state[p] + + if "momentum_buffer" not in state: + state["momentum_buffer"] = torch.zeros_like( + p.grad, memory_format=torch.preserve_format + ) + muon_momentum_bufs.append(state["momentum_buffer"]) + + return False # has_complex + + @torch.no_grad() + def step(self, closure=None): + """Performs a single optimization step.""" + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + lr = group["lr"] + weight_decay = group["weight_decay"] + momentum = group["momentum"] + + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + muon_momentum_bufs: list[Tensor] = [] + + has_complex = self._init_group( + group, + params_with_grad, + grads, + muon_momentum_bufs, + ) + + muon( + params_with_grad, + grads, + muon_momentum_bufs, + lr=lr, + weight_decay=weight_decay, + momentum=momentum, + nesterov=group["nesterov"], + ns_coefficients=group["ns_coefficients"], + eps=group["eps"], + ns_steps=group["ns_steps"], + adjust_lr_fn=group["adjust_lr_fn"], + has_complex=has_complex, + ) + return loss + + +Muon.__doc__ = ( + r"""Implements Muon algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)},\ \lambda \text{ (weight decay)},\ + \mu \text{ (momentum)},\ \textit{nesterov}\in\{True,False\},\\ + &\hspace{13mm}(a,b,c)\ \text{ (NS coefficients)},\ + \varepsilon \text{ (epsilon)},\ k \text{ (NS steps)},\ + \theta_0 \text{ (params)},\ f(\theta) \text{ (objective)} \\ + &\textbf{initialize} : B_0 \leftarrow 0 \text{ (momentum buffer)} \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for}\ t=1\ \textbf{to}\ \ldots\ \textbf{do} \\[0.25ex] + &\hspace{5mm} g_t \leftarrow \nabla_{\theta} f_t(\theta_{t-1}) \\[0.25ex] + &\hspace{5mm} B_t \leftarrow \mu B_{t-1} + g_t \\[0.25ex] + &\hspace{5mm} \widetilde{B}_t \leftarrow + \begin{cases} + g_t + \mu B_t, & \text{if nesterov}=True \\ + B_t, & \text{if nesterov}=False + \end{cases} \\[1.0ex] + &\hspace{5mm} O_t \leftarrow \mathrm{NS}^{(a,b,c)}_{k}\!\big(\widetilde{B}_t;\ \varepsilon\big) \\[0.5ex] + &\hspace{5mm} \theta_t \leftarrow \theta_{t-1} - \gamma\,\lambda\,\theta_{t-1} + \quad\text{(decoupled weight decay)} \\[0.25ex] + + &\hspace{5mm} \gamma \leftarrow \mathrm{AdjustLR}\!\big(\gamma;\ \mathrm{shape}\!\big(\theta_t \big) \big) \\[0.25ex] + &\hspace{5mm} \theta_t \leftarrow \theta_t - \gamma\, O_t \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\mathbf{return}\ \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt}s + \end{aligned} + + Here, :math:`\mathrm{NS}^{(a,b,c)}_{k}(\cdot;\varepsilon)` denotes :math:`k` iterations of the + Newton–Schulz orthogonalization operator parameterized by coefficients :math:`(a,b,c)` + with numerical stabilization :math:`\varepsilon`. + + The purpose for :math:`\mathrm{AdjustLR}\!\big(\gamma;\ \mathrm{shape}\!\big(\theta_t \big) \big)` + is to make the orthogonalized update have a consistent :math:`RMS` across rectangular matrices. + + Keller's original implementation scales the update by :math:`\sqrt{\max\!\left(1, \frac{A}{B}\right)}`, + where :math:`A` and :math:`B` are dimension of the matrix being optimized. + + Moonshot's implementation also focuses on matching :math:`RMS` of AdamW. The adjustment is computed as: + :math:`\gamma \leftarrow {0.2}\gamma\,\sqrt{\max\!\left({A}, {B}\right)}` + The method is adopted from `Muon is Scalable for LLM Training`_. Research + results show that with this adjustment Muon can directly reuse the learning rate + and weight decay tuned for AdamW. + + We provide two options for the learning rate adjustment: "original", which follows Keller's + implementation, and "match_rms_adamw", which refers to Moonshot's implementation. This gives users the + flexibility to choose between the two. If `adjust_lr_fn` is not specified, the default is "original". + + For further details regarding the algorithm we refer to `Muon: An optimizer for hidden layers in neural networks`_ + and `Muon is Scalable for LLM Training`_. + """ + + rf""" + Args: + {_params_doc}. Note that Muon is an optimizer for 2D parameters of neural network hidden layers. Other + parameters, such as bias, and embedding, should be optimized by a standard method such as AdamW. + lr (float, Tensor, optional): learning rate (default: 1e-3). + weight_decay (float, optional): weight decay (L2 penalty). (default: 0.1) + momentum (float, optional): momentum factor (default: 0.95) + nesterov (bool, optional): enables Nesterov momentum. Only applicable + when momentum is non-zero + ns_coefficients (tuple of three floats, optional): coefficients \(a,b,c\) for the + Newton–Schulz orthogonalization polynomial (default: ({DEFAULT_A}, {DEFAULT_B}, {DEFAULT_C})) + eps (float, optional): term added to the denominator for numerical stability. (default: {EPS}) + ns_steps (int, optional): number of Newton–Schulz iteration steps. (default: {DEFAULT_NS_STEPS}) + adjust_lr_fn (str, optional): function to adjust learning rate. One of "original" and "match_rms_adamw". + If not specified, we will default to use "original". (default: None) + + .. _Muon\: An optimizer for hidden layers in neural networks: + https://kellerjordan.github.io/posts/muon/ + .. _Muon is Scalable for LLM Training: + https://arxiv.org/pdf/2502.16982 + + """ +) + + +def _single_tensor_muon( + params: list[Tensor], + grads: list[Tensor], + muon_momentum_bufs: list[Tensor], + *, + lr: float, + weight_decay: float, + momentum: float, + nesterov: bool, + ns_coefficients: tuple[float, float, float], + ns_steps: int, + eps: float, + adjust_lr_fn: str | None, + has_complex: bool, +) -> None: + lr = _to_scalar(lr) + if has_complex: + raise ValueError("Complex parameters are not supported") + + for i, param in enumerate(params): + grad = grads[i] + if grad.ndim != 2: + raise ValueError("Param gradient must be a 2D matrix") + + buf = muon_momentum_bufs[i] + buf.lerp_(grad, 1 - momentum) + update = grad.lerp(buf, momentum) if nesterov else buf + + update = _zeropower_via_newtonschulz(update, ns_coefficients, ns_steps, eps) + + adjusted_lr = _adjust_lr(lr, adjust_lr_fn, param.shape) + + param.mul_(1 - lr * weight_decay) + param.add_(update, alpha=-adjusted_lr) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_muon) +def muon( + params: list[Tensor], + grads: list[Tensor], + muon_momentum_bufs: list[Tensor], + *, + foreach: bool | None = None, + lr: float, + weight_decay: float, + momentum: float, + nesterov: bool, + ns_coefficients: tuple[float, float, float], + ns_steps: int, + eps: float, + adjust_lr_fn: str | None, + has_complex: bool, +) -> None: + r"""Functional API that performs Muon algorithm computation. + + See :class:`~torch.optim.Muon` for details. + """ + if foreach is not None and foreach: + raise RuntimeError("Foreach is not supported for Muon yet") + + func = _single_tensor_muon + + func( + params, + grads, + muon_momentum_bufs, + lr=lr, + weight_decay=weight_decay, + momentum=momentum, + nesterov=nesterov, + ns_coefficients=ns_coefficients, + ns_steps=ns_steps, + eps=eps, + adjust_lr_fn=adjust_lr_fn, + has_complex=has_complex, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adadelta.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adadelta.py new file mode 100644 index 0000000000000000000000000000000000000000..1ee27f46f194d262f916b044658f9ada8d088b59 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adadelta.py @@ -0,0 +1,473 @@ +# mypy: allow-untyped-defs +from typing import Any, cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["Adadelta", "adadelta"] + + +class Adadelta(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1.0, + rho: float = 0.9, + eps: float = 1e-6, + weight_decay: float = 0, + foreach: bool | None = None, + *, + capturable: bool = False, + maximize: bool = False, + differentiable: bool = False, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= rho <= 1.0: + raise ValueError(f"Invalid rho value: {rho}") + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + + defaults = { + "lr": lr, + "rho": rho, + "eps": eps, + "weight_decay": weight_decay, + "maximize": maximize, + "capturable": capturable, + "foreach": foreach, + "differentiable": differentiable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + group.setdefault("capturable", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + + def _init_group( + self, + group: dict[str, Any], + params_with_grad: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + acc_deltas: list[Tensor], + state_steps: list[Tensor], + ): + has_complex = False + p: Tensor + for p in group["params"]: + if p.grad is None: + continue + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError("Adadelta does not support sparse gradients") + grads.append(p.grad) + + state = self.state[p] + + # Lazy state initialization + if len(state) == 0: + state["step"] = ( + torch.zeros((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.zeros((), dtype=_get_scalar_dtype()) + ) + + state["square_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + state["acc_delta"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + square_avgs.append(state["square_avg"]) + acc_deltas.append(state["acc_delta"]) + state_steps.append(state["step"]) + + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + square_avgs: list[Tensor] = [] + acc_deltas: list[Tensor] = [] + state_steps: list[Tensor] = [] + ( + lr, + rho, + eps, + weight_decay, + foreach, + maximize, + differentiable, + capturable, + ) = ( + group["lr"], + group["rho"], + group["eps"], + group["weight_decay"], + group["foreach"], + group["maximize"], + group["differentiable"], + group["capturable"], + ) + + has_complex = self._init_group( + group, params_with_grad, grads, square_avgs, acc_deltas, state_steps + ) + + adadelta( + params_with_grad, + grads, + square_avgs, + acc_deltas, + state_steps, + lr=lr, + rho=rho, + eps=eps, + weight_decay=weight_decay, + foreach=foreach, + maximize=maximize, + differentiable=differentiable, + capturable=capturable, + has_complex=has_complex, + ) + + return loss + + +Adadelta.__doc__ = ( + r"""Implements Adadelta algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, + \: f(\theta) \text{ (objective)}, \: \rho \text{ (decay)}, + \: \lambda \text{ (weight decay)} \\ + &\textbf{initialize} : v_0 \leftarrow 0 \: \text{ (square avg)}, + \: u_0 \leftarrow 0 \: \text{ (accumulate variables)} \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}if \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm} v_t \leftarrow v_{t-1} \rho + g^2_t (1 - \rho) \\ + &\hspace{5mm}\Delta x_t \leftarrow \frac{\sqrt{u_{t-1} + + \epsilon }}{ \sqrt{v_t + \epsilon} }g_t \hspace{21mm} \\ + &\hspace{5mm} u_t \leftarrow u_{t-1} \rho + + \Delta x^2_t (1 - \rho) \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \gamma \Delta x_t \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `ADADELTA: An Adaptive Learning Rate Method`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): coefficient that scale delta before it is applied + to the parameters (default: 1.0) + rho (float, optional): coefficient used for computing a running average + of squared gradients (default: 0.9). A higher value of `rho` will + result in a slower average, which can be helpful for preventing + oscillations in the learning process. + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-6). + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + {_foreach_doc} + {_capturable_doc} + {_maximize_doc} + {_differentiable_doc} + + .. _ADADELTA\: An Adaptive Learning Rate Method: + https://arxiv.org/abs/1212.5701 + + """ +) + + +def _single_tensor_adadelta( + params: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + acc_deltas: list[Tensor], + state_steps: list[Tensor], + *, + lr: float, + rho: float, + eps: float, + weight_decay: float, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for param, grad, square_avg, acc_delta, step in zip( + params, grads, square_avgs, acc_deltas, state_steps, strict=True + ): + step += 1 + grad = grad if not maximize else -grad + + if weight_decay != 0: + grad = grad.add(param, alpha=weight_decay) + + if torch.is_complex(param): + square_avg = torch.view_as_real(square_avg) + acc_delta = torch.view_as_real(acc_delta) + grad = torch.view_as_real(grad) + + square_avg.mul_(rho).addcmul_(grad, grad, value=1 - rho) + std = square_avg.add(eps).sqrt_() + delta = acc_delta.add(eps).sqrt_() + if differentiable: + delta = delta.clone() + delta.div_(std).mul_(grad) + acc_delta.mul_(rho).addcmul_(delta, delta, value=1 - rho) + + if torch.is_complex(param): + delta = torch.view_as_complex(delta) + param.add_(delta, alpha=-lr) + + +def _multi_tensor_adadelta( + params: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + acc_deltas: list[Tensor], + state_steps: list[Tensor], + *, + lr: float, + rho: float, + eps: float, + weight_decay: float, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + if len(params) == 0: + return + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, square_avgs, acc_deltas, state_steps] # type: ignore[list-item] + ) + for ( + device_params_, + device_grads_, + device_square_avgs_, + device_acc_deltas_, + device_state_steps_, + ), _ in grouped_tensors.values(): + device_params = cast(list[Tensor], device_params_) + device_grads = cast(list[Tensor], device_grads_) + device_square_avgs = cast(list[Tensor], device_square_avgs_) + device_acc_deltas = cast(list[Tensor], device_acc_deltas_) + device_state_steps = cast(list[Tensor], device_state_steps_) + if has_complex: + _view_as_real( + device_params, device_grads, device_square_avgs, device_acc_deltas + ) + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and device_state_steps[0].is_cpu: + torch._foreach_add_( + device_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(device_state_steps, 1) + + if maximize: + device_grads = torch._foreach_neg(device_grads) # type: ignore[assignment] + + if weight_decay != 0: + # Reuse the intermediate memory (device_grads) already allocated for maximize + if maximize: + torch._foreach_add_(device_grads, device_params, alpha=weight_decay) + else: + device_grads = torch._foreach_add( # type: ignore[assignment] + device_grads, device_params, alpha=weight_decay + ) + + torch._foreach_mul_(device_square_avgs, rho) + torch._foreach_addcmul_( + device_square_avgs, device_grads, device_grads, value=1 - rho + ) + + std = torch._foreach_add(device_square_avgs, eps) + torch._foreach_sqrt_(std) + + deltas = torch._foreach_add(device_acc_deltas, eps) + torch._foreach_sqrt_(deltas) + torch._foreach_div_(deltas, std) + torch._foreach_mul_(deltas, device_grads) + + torch._foreach_mul_(device_acc_deltas, rho) + torch._foreach_addcmul_(device_acc_deltas, deltas, deltas, value=1 - rho) + + # If LR is a tensor, the else branch will internally call item() + # which will cause silent incorrectness if we are capturing + if capturable and isinstance(lr, torch.Tensor): + torch._foreach_mul_(deltas, -lr) + torch._foreach_add_(device_params, deltas) + else: + torch._foreach_add_(device_params, deltas, alpha=-lr) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_adadelta) +def adadelta( + params: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + acc_deltas: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + capturable: bool = False, + foreach: bool | None = None, + differentiable: bool = False, + has_complex: bool = False, + *, + lr: float, + rho: float, + eps: float, + weight_decay: float, + maximize: bool, +) -> None: + r"""Functional API that performs Adadelta algorithm computation. + + See :class:`~torch.optim.Adadelta` for details. + """ + + # this check is slow during compilation, so we skip it + # if it's strictly needed we can add this check back in dynamo + if not torch.compiler.is_compiling() and not all( + isinstance(t, torch.Tensor) for t in state_steps + ): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + # We still respect when the user inputs False for foreach. + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_adadelta + else: + func = _single_tensor_adadelta + + func( + params, + grads, + square_avgs, + acc_deltas, + state_steps, + lr=lr, + rho=rho, + eps=eps, + weight_decay=weight_decay, + maximize=maximize, + differentiable=differentiable, + capturable=capturable, + has_complex=has_complex, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adagrad.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adagrad.py new file mode 100644 index 0000000000000000000000000000000000000000..a6a57fb61b8bae047ba7e19a2f8a1fcd9d55fa29 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adagrad.py @@ -0,0 +1,579 @@ +# mypy: allow-untyped-defs +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _default_to_fused_or_foreach, + _device_dtype_check_for_fused, + _differentiable_doc, + _foreach_doc, + _get_scalar_dtype, + _get_value, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["Adagrad", "adagrad"] + + +class Adagrad(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-2, + lr_decay: float = 0, + weight_decay: float = 0, + initial_accumulator_value: float = 0, + eps: float = 1e-10, + foreach: bool | None = None, + *, + maximize: bool = False, + differentiable: bool = False, + fused: bool | None = None, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= lr_decay: + raise ValueError(f"Invalid lr_decay value: {lr_decay}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + if not 0.0 <= initial_accumulator_value: + raise ValueError( + f"Invalid initial_accumulator_value value: {initial_accumulator_value}" + ) + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + + defaults = { + "lr": lr, + "lr_decay": lr_decay, + "eps": eps, + "weight_decay": weight_decay, + "initial_accumulator_value": initial_accumulator_value, + "foreach": foreach, + "maximize": maximize, + "differentiable": differentiable, + "fused": fused, + } + super().__init__(params, defaults) + + if fused: + if differentiable: + raise RuntimeError("`fused` does not support `differentiable`") + if foreach: + raise RuntimeError("`fused` and `foreach` cannot be `True` together.") + self._need_device_dtype_check_for_fused = True + + for group in self.param_groups: + for p in group["params"]: + state = self.state[p] + state["step"] = ( + torch.zeros( + (), + dtype=_get_scalar_dtype(is_fused=group["fused"]), + device=p.device, + ) + if group["fused"] + else torch.tensor(0.0, dtype=_get_scalar_dtype()) + ) + init_value = ( + complex(initial_accumulator_value, initial_accumulator_value) + if torch.is_complex(p) + else initial_accumulator_value + ) + state["sum"] = torch.full_like( + p, init_value, memory_format=torch.preserve_format + ) + + def __setstate__(self, state): + super().__setstate__(state) + # define "fused" for + # MYPY error: Name "fused" may be undefined + fused = None + for group in self.param_groups: + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + fused = group.setdefault("fused", None) + + state_values = list(self.state.values()) + step_is_tensor = (len(state_values) != 0) and torch.is_tensor( + state_values[0]["step"] + ) + if not step_is_tensor: + for s in state_values: + s["step"] = torch.tensor( + float(s["step"]), dtype=_get_scalar_dtype(is_fused=fused) + ) + + def share_memory(self) -> None: + """Calls tensor.share_memory_() on the state sum tensors.""" + for group in self.param_groups: + for p in group["params"]: + state = self.state[p] + state["sum"].share_memory_() + + def _init_group(self, group, params_with_grad, grads, state_sums, state_steps): + has_sparse_grad, has_complex = False, False + for p in group["params"]: + if p.grad is not None: + if group["fused"] and getattr( + self, + "_need_device_dtype_check_for_fused", + True, + ): + _device_dtype_check_for_fused(p, cuda_unsupported=True) + self._need_device_dtype_check_for_fused = False + has_sparse_grad |= p.grad.is_sparse + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + grads.append(p.grad) + state = self.state[p] + state_sums.append(state["sum"]) + state_steps.append(state["step"]) + + return has_sparse_grad, has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + state_sums: list[Tensor] = [] + state_steps: list[Tensor] = [] + + has_sparse_grad, has_complex = self._init_group( + group, params_with_grad, grads, state_sums, state_steps + ) + + adagrad( + params_with_grad, + grads, + state_sums, + state_steps, + lr=group["lr"], + weight_decay=group["weight_decay"], + lr_decay=group["lr_decay"], + eps=group["eps"], + has_sparse_grad=has_sparse_grad, + foreach=group["foreach"], + maximize=group["maximize"], + differentiable=group["differentiable"], + has_complex=has_complex, + fused=group["fused"], + grad_scale=getattr(self, "grad_scale", None), + found_inf=getattr(self, "found_inf", None), + ) + + return loss + + +Adagrad.__doc__ = ( + r"""Implements Adagrad algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, \: f(\theta) + \text{ (objective)}, \: \lambda \text{ (weight decay)}, \\ + &\hspace{12mm} \tau \text{ (initial accumulator value)}, \: \eta\text{ (lr decay)}\\ + &\textbf{initialize} : state\_sum_0 \leftarrow \tau \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm} \tilde{\gamma} \leftarrow \gamma / (1 +(t-1) \eta) \\ + &\hspace{5mm} \textbf{if} \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm}state\_sum_t \leftarrow state\_sum_{t-1} + g^2_t \\ + &\hspace{5mm}\theta_t \leftarrow + \theta_{t-1}- \tilde{\gamma} \frac{g_t}{\sqrt{state\_sum_t}+\epsilon} \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `Adaptive Subgradient Methods for Online Learning + and Stochastic Optimization`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-2) + lr_decay (float, optional): learning rate decay (default: 0) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + initial_accumulator_value (float, optional): initial value of the + sum of squares of gradients (default: 0) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-10) + {_foreach_doc} + {_maximize_doc} + {_differentiable_doc} + fused (bool, optional): whether the fused implementation (CPU only) is used. + Currently, `torch.float64`, `torch.float32`, `torch.float16`, and `torch.bfloat16` + are supported. (default: None). Please note that the fused implementations does not + support sparse or complex gradients. + .. _Adaptive Subgradient Methods for Online Learning and Stochastic + Optimization: http://jmlr.org/papers/v12/duchi11a.html + + """ +) + + +def adagrad( + params: list[Tensor], + grads: list[Tensor], + state_sums: list[Tensor], + state_steps: list[Tensor], + fused: bool | None = None, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting these as kwargs for now as functional API is compiled by torch/distributed/optim + has_sparse_grad: bool = False, + foreach: bool | None = None, + differentiable: bool = False, + has_complex: bool = False, + *, + lr: float, + weight_decay: float, + lr_decay: float, + eps: float, + maximize: bool, +) -> None: + r"""Functional API that performs Adagrad algorithm computation. + + See :class:`~torch.optim.Adagrad` for details. + """ + if not all(isinstance(t, torch.Tensor) for t in state_steps): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + # Respect when the user inputs False/True for foreach or fused. We only want to change + # the default when neither have been user-specified. Note that we default to foreach + # and pass False to use_fused. This is not a mistake--we want to give the fused impl + # bake-in time before making it the default, even if it is typically faster. + if fused is None and foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if fused is None: + fused = False + if foreach is None: + foreach = False + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + if fused and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with fused optimizers") + + if fused and not torch.jit.is_scripting(): + func = _fused_adagrad + elif foreach and not torch.jit.is_scripting(): + func = _multi_tensor_adagrad + else: + func = _single_tensor_adagrad + + func( + params, + grads, + state_sums, + state_steps, + lr=lr, + weight_decay=weight_decay, + lr_decay=lr_decay, + eps=eps, + has_sparse_grad=has_sparse_grad, + maximize=maximize, + differentiable=differentiable, + has_complex=has_complex, + grad_scale=grad_scale, + found_inf=found_inf, + ) + + +def _make_sparse(grad, grad_indices, values): + size = grad.size() + return torch.sparse_coo_tensor(grad_indices, values, size) + + +def _single_tensor_adagrad( + params: list[Tensor], + grads: list[Tensor], + state_sums: list[Tensor], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + lr: float, + weight_decay: float, + lr_decay: float, + eps: float, + has_sparse_grad: bool, + maximize: bool, + differentiable: bool, + has_complex: bool, +) -> None: + if grad_scale is not None or found_inf is not None: + raise AssertionError("Expected grad_scale and found_inf to be None") + + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for param, grad, state_sum, step_t in zip( + params, grads, state_sums, state_steps, strict=True + ): + # update step + step_t += 1 + step = _get_value(step_t) + grad = grad if not maximize else -grad + + if weight_decay != 0: + if grad.is_sparse: + raise RuntimeError( + "weight_decay option is not compatible with sparse gradients" + ) + grad = grad.add(param, alpha=weight_decay) + + clr = lr / (1 + (step - 1) * lr_decay) + + if grad.is_sparse: + grad = grad.coalesce() # the update is non-linear so indices must be unique + grad_indices = grad._indices() + grad_values = grad._values() + + state_sum.add_(_make_sparse(grad, grad_indices, grad_values.pow(2))) + std = state_sum.sparse_mask(grad) + std_values = std._values().sqrt_().add_(eps) + param.add_( + _make_sparse(grad, grad_indices, grad_values / std_values), alpha=-clr + ) + else: + is_complex = torch.is_complex(param) + if is_complex: + grad = torch.view_as_real(grad) + state_sum = torch.view_as_real(state_sum) + param = torch.view_as_real(param) + state_sum.addcmul_(grad, grad, value=1) + if differentiable: + std = state_sum.sqrt() + eps + else: + std = state_sum.sqrt().add_(eps) + param.addcdiv_(grad, std, value=-clr) + if is_complex: + param = torch.view_as_complex(param) + state_sum = torch.view_as_complex(state_sum) + + +def _multi_tensor_adagrad( + params: list[Tensor], + grads: list[Tensor], + state_sums: list[Tensor], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + lr: float, + weight_decay: float, + lr_decay: float, + eps: float, + has_sparse_grad: bool, + maximize: bool, + differentiable: bool, + has_complex: bool, +) -> None: + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + if grad_scale is not None or found_inf is not None: + raise AssertionError("Expected grad_scale and found_inf to be None") + + # Foreach functions will throw errors if given empty lists + if len(params) == 0: + return + + lr = _to_scalar(lr) + + grouped_tensorlists = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, state_sums, state_steps] # type: ignore[list-item] + ) + for ( + device_params_, + device_grads_, + device_state_sums_, + device_state_steps_, + ), _ in grouped_tensorlists.values(): + device_params = cast(list[Tensor], device_params_) + device_grads = cast(list[Tensor], device_grads_) + device_state_sums = cast(list[Tensor], device_state_sums_) + device_state_steps = cast(list[Tensor], device_state_steps_) + + device_has_sparse_grad = has_sparse_grad and any( + grad.is_sparse for grad in device_grads + ) + + if device_has_sparse_grad: + _single_tensor_adagrad( + device_params, + device_grads, + device_state_sums, + device_state_steps, + lr=lr, + weight_decay=weight_decay, + lr_decay=lr_decay, + eps=eps, + has_sparse_grad=True, + maximize=maximize, + differentiable=differentiable, + has_complex=has_complex, + grad_scale=grad_scale, + found_inf=found_inf, + ) + continue + + # Handle complex parameters + if has_complex: + _view_as_real(device_params, device_grads, device_state_sums) + + if maximize: + device_grads = torch._foreach_neg(device_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and device_state_steps[0].is_cpu: + torch._foreach_add_( + device_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(device_state_steps, 1) + + if weight_decay != 0: + # Reuse the intermediate memory (device_grads) already allocated for maximize + if maximize: + torch._foreach_add_(device_grads, device_params, alpha=weight_decay) + else: + device_grads = torch._foreach_add( # type: ignore[assignment] + device_grads, device_params, alpha=weight_decay + ) + + minus_clr = [ + -lr / (1 + (_get_value(step) - 1) * lr_decay) for step in device_state_steps + ] + + torch._foreach_addcmul_(device_state_sums, device_grads, device_grads, value=1) + + std = torch._foreach_sqrt(device_state_sums) + torch._foreach_add_(std, eps) + + if weight_decay != 0 or maximize: + # Again, reuse the intermediate memory (device_grads) already allocated + torch._foreach_mul_(device_grads, minus_clr) + numerator = device_grads + else: + numerator = torch._foreach_mul(device_grads, minus_clr) # type: ignore[assignment] + + torch._foreach_addcdiv_(device_params, numerator, std) + + +def _fused_adagrad( + params: list[Tensor], + grads: list[Tensor], + state_sums: list[Tensor], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + lr: float, + weight_decay: float, + lr_decay: float, + eps: float, + has_sparse_grad: bool, + maximize: bool, + differentiable: bool, + has_complex: bool, +) -> None: + if not params: + return + if has_sparse_grad or has_complex: + raise RuntimeError("`fused` does not support sparse grad or complex param") + + if differentiable: + raise RuntimeError( + "adagrad with fused=True does not support differentiable=True" + ) + + lr = _to_scalar(lr) + + grad_scale_dict = ( + {grad_scale.device: grad_scale} if grad_scale is not None else None + ) + found_inf_dict = {found_inf.device: found_inf} if found_inf is not None else None + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, state_sums, state_steps] # type: ignore[list-item] + ) + for (device, _), ( + ( + device_params_, + device_grads_, + device_state_sums_, + device_state_steps_, + ), + _, + ) in grouped_tensors.items(): + device_params = cast(list[Tensor], device_params_) + device_grads = cast(list[Tensor], device_grads_) + device_state_sums = cast(list[Tensor], device_state_sums_) + device_state_steps = cast(list[Tensor], device_state_steps_) + + device_grad_scale, device_found_inf = None, None + if grad_scale is not None and grad_scale_dict is not None: + if device not in grad_scale_dict: + grad_scale_dict[device] = grad_scale.to(device, non_blocking=True) # type: ignore[index] + device_grad_scale = grad_scale_dict[device] # type: ignore[index] + if found_inf is not None and found_inf_dict is not None: + if found_inf not in found_inf_dict: + found_inf_dict[device] = found_inf.to(device, non_blocking=True) # type: ignore[index] + device_found_inf = found_inf_dict[device] # type: ignore[index] + torch._foreach_add_(device_state_steps, 1) + torch._fused_adagrad_( + device_params, + device_grads, + device_state_sums, + device_state_steps, + lr=lr, + lr_decay=lr_decay, + weight_decay=weight_decay, + eps=eps, + maximize=maximize, + grad_scale=device_grad_scale, + found_inf=device_found_inf, + ) + if device_found_inf is not None: + torch._foreach_sub_( + device_state_steps, [device_found_inf] * len(device_state_steps) + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adam.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adam.py new file mode 100644 index 0000000000000000000000000000000000000000..64c23e7ddf391909ce0e7e3956072b7ac1c616b6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adam.py @@ -0,0 +1,990 @@ +# mypy: allow-untyped-defs +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _device_dtype_check_for_fused, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _fused_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _get_value, + _maximize_doc, + _params_doc, + _stack_if_compiling, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + DeviceDict, + DeviceDtypeDict, + Optimizer, + ParamsT, +) + + +__all__ = ["Adam", "adam"] + + +class Adam(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-3, + betas: tuple[float | Tensor, float | Tensor] = (0.9, 0.999), + eps: float = 1e-8, + weight_decay: float = 0, + amsgrad: bool = False, + *, + foreach: bool | None = None, + maximize: bool = False, + capturable: bool = False, + differentiable: bool = False, + fused: bool | None = None, + decoupled_weight_decay: bool = False, + ) -> None: + if isinstance(lr, Tensor): + if foreach and not capturable: + raise ValueError( + "lr as a Tensor is not supported for capturable=False and foreach=True" + ) + if lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= betas[0] < 1.0: + raise ValueError(f"Invalid beta parameter at index 0: {betas[0]}") + if not 0.0 <= betas[1] < 1.0: + raise ValueError(f"Invalid beta parameter at index 1: {betas[1]}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + if not ( + (isinstance(betas[0], float) and isinstance(betas[1], float)) + or (isinstance(betas[0], Tensor) and isinstance(betas[1], Tensor)) + ): + raise ValueError("betas must be either both floats or both Tensors") + if isinstance(betas[0], Tensor): + if not capturable and foreach: + raise ValueError( + "betas[0] as a Tensor is not supported for capturable=False and foreach=True" + ) + if betas[0].numel() != 1: + raise ValueError("Tensor betas[0] must be 1-element") + if isinstance(betas[1], Tensor): + if not capturable and foreach: + raise ValueError( + "betas[1] as a Tensor is not supported for capturable=False and foreach=True" + ) + if betas[1].numel() != 1: + raise ValueError("Tensor betas[1] must be 1-element") + betas = tuple(map(_to_scalar, betas)) + + defaults = { + "lr": lr, + "betas": betas, + "eps": eps, + "weight_decay": weight_decay, + "amsgrad": amsgrad, + "maximize": maximize, + "foreach": foreach, + "capturable": capturable, + "differentiable": differentiable, + "fused": fused, + "decoupled_weight_decay": decoupled_weight_decay, + } + super().__init__(params, defaults) + + if fused: + if differentiable: + raise RuntimeError("`fused` does not support `differentiable`") + self._step_supports_amp_scaling = True + # TODO(crcrpar): [low prec params & their higher prec copy] + # Support AMP with FP16/BF16 model params which would need + # higher prec copy of params to do update math in higher prec to + # alleviate the loss of information. + if foreach: + raise RuntimeError("`fused` and `foreach` cannot be `True` together.") + + def __setstate__(self, state): + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("amsgrad", False) + group.setdefault("maximize", False) + group.setdefault("foreach", None) + group.setdefault("capturable", False) + group.setdefault("differentiable", False) + group.setdefault("decoupled_weight_decay", False) + fused = group.setdefault("fused", None) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, + dtype=_get_scalar_dtype(is_fused=fused), + device=p.device, + ) + if group["capturable"] or group["fused"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + + def _init_group( + self, + group, + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + max_exp_avg_sqs, + state_steps, + ): + has_complex = False + for p in group["params"]: + if p.grad is not None: + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError( + "Adam does not support sparse gradients, please consider SparseAdam instead" + ) + grads.append(p.grad) + + state = self.state[p] + # Lazy state initialization + if len(state) == 0: + if group["fused"]: + _device_dtype_check_for_fused(p) + # note(crcrpar): [special device hosting for step] + # Deliberately host `step` on CPU if both capturable and fused are off. + # This is because kernel launches are costly on CUDA and XLA. + state["step"] = ( + torch.zeros( + (), + dtype=_get_scalar_dtype(is_fused=group["fused"]), + device=p.device, + ) + if group["capturable"] or group["fused"] + else torch.tensor(0.0, dtype=_get_scalar_dtype()) + ) + # Exponential moving average of gradient values + state["exp_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + # Exponential moving average of squared gradient values + state["exp_avg_sq"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + if group["amsgrad"]: + # Maintains max of all exp. moving avg. of sq. grad. values + state["max_exp_avg_sq"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + exp_avgs.append(state["exp_avg"]) + exp_avg_sqs.append(state["exp_avg_sq"]) + + if group["amsgrad"]: + max_exp_avg_sqs.append(state["max_exp_avg_sq"]) + if group["differentiable"] and state["step"].requires_grad: + raise RuntimeError( + "`requires_grad` is not supported for `step` in differentiable mode" + ) + + # Foreach without capturable does not support a tensor lr + if ( + group["foreach"] + and torch.is_tensor(group["lr"]) + and not group["capturable"] + ): + raise RuntimeError( + "lr as a Tensor is not supported for capturable=False and foreach=True" + ) + + state_steps.append(state["step"]) + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + exp_avgs: list[Tensor] = [] + exp_avg_sqs: list[Tensor] = [] + max_exp_avg_sqs: list[Tensor] = [] + state_steps: list[Tensor] = [] + beta1, beta2 = group["betas"] + + has_complex = self._init_group( + group, + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + max_exp_avg_sqs, + state_steps, + ) + + adam( + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + max_exp_avg_sqs, + state_steps, + amsgrad=group["amsgrad"], + has_complex=has_complex, + beta1=beta1, + beta2=beta2, + lr=group["lr"], + weight_decay=group["weight_decay"], + eps=group["eps"], + maximize=group["maximize"], + foreach=group["foreach"], + capturable=group["capturable"], + differentiable=group["differentiable"], + fused=group["fused"], + grad_scale=getattr(self, "grad_scale", None), + found_inf=getattr(self, "found_inf", None), + decoupled_weight_decay=group["decoupled_weight_decay"], + ) + + return loss + + +Adam.__doc__ = ( + r"""Implements Adam algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)}, \beta_1, \beta_2 + \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)} \\ + &\hspace{13mm} \lambda \text{ (weight decay)}, \: \textit{amsgrad}, + \:\textit{maximize}, \: \epsilon \text{ (epsilon)} \\ + &\textbf{initialize} : m_0 \leftarrow 0 \text{ ( first moment)}, + v_0\leftarrow 0 \text{ (second moment)},\: v_0^{max}\leftarrow 0 \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + + &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\ + &\hspace{10mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{if} \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ + &\hspace{5mm}v_t \leftarrow \beta_2 v_{t-1} + (1-\beta_2) g^2_t \\ + &\hspace{5mm}\widehat{m_t} \leftarrow m_t/\big(1-\beta_1^t \big) \\ + &\hspace{5mm}\textbf{if} \: amsgrad \\ + &\hspace{10mm} v_t^{max} \leftarrow \mathrm{max}(v_{t-1}^{max},v_t) \\ + &\hspace{10mm}\widehat{v_t} \leftarrow v_t^{max}/\big(1-\beta_2^t \big) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}\widehat{v_t} \leftarrow v_t/\big(1-\beta_2^t \big) \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}/ + \big(\sqrt{\widehat{v_t}} + \epsilon \big) \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR + is not yet supported for all our implementations. Please use a float + LR if you are not also specifying fused=True or capturable=True. + betas (tuple[Union[float, Tensor], Union[float, Tensor]], optional): + coefficients used for computing running averages of gradient and + its square. If a tensor is provided, must be 1-element. (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + decoupled_weight_decay (bool, optional): if True, this optimizer is + equivalent to AdamW and the algorithm will not accumulate weight + decay in the momentum nor variance. (default: False) + amsgrad (bool, optional): whether to use the AMSGrad variant of this + algorithm from the paper `On the Convergence of Adam and Beyond`_ + (default: False) + {_foreach_doc} + {_maximize_doc} + {_capturable_doc} + {_differentiable_doc} + {_fused_doc} + .. Note:: + A prototype implementation of Adam and AdamW for MPS supports `torch.float32` and `torch.float16`. + .. _Adam\: A Method for Stochastic Optimization: + https://arxiv.org/abs/1412.6980 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + + """ +) + + +def _single_tensor_adam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + max_exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + amsgrad: bool, + has_complex: bool, + beta1: float | Tensor, + beta2: float | Tensor, + lr: float | Tensor, + weight_decay: float, + eps: float, + maximize: bool, + capturable: bool, + differentiable: bool, + decoupled_weight_decay: bool, +) -> None: + if grad_scale is not None or found_inf is not None: + raise AssertionError("Expected grad_scale and found_inf to be None") + + if torch.jit.is_scripting(): + # this assert is due to JIT being dumb and not realizing that the ops below + # have overloads to handle both float and Tensor lrs, so we just assert it's + # a float since most people using JIT are using floats + if not isinstance(lr, float): + raise AssertionError(f"Expected lr to be a float, but got {type(lr)}") + if not isinstance(beta1, float): + raise AssertionError(f"Expected beta1 to be a float, but got {type(beta1)}") + if not isinstance(beta2, float): + raise AssertionError(f"Expected beta2 to be a float, but got {type(beta2)}") + else: + lr = _to_scalar(lr) + beta1 = _to_scalar(beta1) + beta2 = _to_scalar(beta2) + + # We only shuffle around the beta when it is a Tensor, otherwise, we prefer + # treating it as a scalar. + # Note: ensure type declaration is under conditional check for isinstance + # or else torchscript will get cranky about the DeviceDict type. + if isinstance(beta1, Tensor): + beta1_dict: DeviceDtypeDict | None = {(beta1.device, beta1.dtype): beta1} + else: + beta1_dict = None + + for i, param in enumerate(params): + grad = grads[i] if not maximize else -grads[i] + exp_avg = exp_avgs[i] + exp_avg_sq = exp_avg_sqs[i] + step_t = state_steps[i] + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type == step_t.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + # update step + step_t += 1 + + if weight_decay != 0: + if decoupled_weight_decay: + # Perform stepweight decay + param.mul_(1 - lr * weight_decay) + else: + # Nested if is necessary to bypass jitscript rules + if differentiable and isinstance(weight_decay, Tensor): + if weight_decay.requires_grad: + grad = grad.addcmul_(param.clone(), weight_decay) + else: + # pyrefly: ignore [bad-argument-type] + grad = grad.add(param, alpha=weight_decay) + else: + grad = grad.add(param, alpha=weight_decay) + + if torch.is_complex(param): + grad = torch.view_as_real(grad) + exp_avg = torch.view_as_real(exp_avg) + exp_avg_sq = torch.view_as_real(exp_avg_sq) + if amsgrad: + max_exp_avg_sqs[i] = torch.view_as_real(max_exp_avg_sqs[i]) + param = torch.view_as_real(param) + + device = param.device + + if beta1_dict is not None: + dtype = param.dtype # type: ignore[union-attr] + + # cast to workaround https://github.com/pytorch/pytorch/issues/140601 + key = (device, dtype) + if key not in beta1_dict: + beta1_dict[key] = beta1.to( # type: ignore[union-attr] + device=device, dtype=dtype, non_blocking=True + ) + + device_beta1: float | Tensor = beta1_dict[key] + else: + device_beta1 = beta1 + + # Decay the first and second moment running average coefficient + + exp_avg.lerp_(grad, 1 - device_beta1) + + # Nested if is necessary to bypass jitscript rules + if differentiable and isinstance(beta2, Tensor): + if beta2.requires_grad: + # Using lerp to only use 2 operations bc addcmul's value cannot be a tensor + # Showing equivalence of differentiable path and nondifferentiable path + # expavg * b2 + grad^2 * (1-b2) + # add expavg * (1-b2) - expavg * (1-b2) = 0 + # expavg * b2 + expavg * (1-b2) - expavg * (1-b2) + grad^2 * (1-b2) + # expavg - expavg * (1-b2) + grad^2 * (1-b2) + # expavg + (grad^2 - expavg) * (1-b2) + # expavg.lerp(grad^2, 1-beta2) + exp_avg_sq.lerp_(torch.square(grad), weight=1 - beta2) + else: + exp_avg_sq.mul_(beta2).addcmul_( + grad, grad, value=cast(float, 1 - beta2) + ) + else: + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) # type: ignore[arg-type] + + if capturable or differentiable: + step = step_t + + # Nested if is necessary to bypass jitscript rules + if differentiable and isinstance(beta1, Tensor): + if beta1.requires_grad: + bias_correction1 = 1 - beta1 ** step.clone() + else: + bias_correction1 = 1 - beta1**step + else: + bias_correction1 = 1 - beta1**step + + # Nested if is necessary to bypass jitscript rules + if differentiable and isinstance(beta2, Tensor): + if beta2.requires_grad: + bias_correction2 = 1 - beta2 ** step.clone() + else: + bias_correction2 = 1 - beta2**step + else: + bias_correction2 = 1 - beta2**step + + step_size = lr / bias_correction1 + step_size_neg = step_size.neg() + + bias_correction2_sqrt = bias_correction2.sqrt() + + if amsgrad: + # Maintains the maximum of all 2nd moment running avg. till now + if differentiable: + max_exp_avg_sq = max_exp_avg_sqs[i].clone() + else: + max_exp_avg_sq = max_exp_avg_sqs[i] + + max_exp_avg_sqs[i].copy_(torch.maximum(max_exp_avg_sq, exp_avg_sq)) + + # Uses the max. for normalizing running avg. of gradient + # Folds in (admittedly ugly) 1-elem step_size math here to avoid extra param-set-sized read+write + # (can't fold it into addcdiv_ below because addcdiv_ requires value is a Number, not a Tensor) + denom = ( + max_exp_avg_sqs[i].sqrt() / (bias_correction2_sqrt * step_size_neg) + ).add_(eps / step_size_neg) + else: + denom = ( + exp_avg_sq.sqrt() / (bias_correction2_sqrt * step_size_neg) + ).add_(eps / step_size_neg) + + if differentiable: + param.addcdiv_(exp_avg.clone(), denom) + else: + param.addcdiv_(exp_avg, denom) + else: + step = _get_value(step_t) + + bias_correction1 = 1 - beta1**step + bias_correction2 = 1 - beta2**step + + step_size = lr / bias_correction1 + + bias_correction2_sqrt = bias_correction2**0.5 + + if amsgrad: + # Maintains the maximum of all 2nd moment running avg. till now + torch.maximum(max_exp_avg_sqs[i], exp_avg_sq, out=max_exp_avg_sqs[i]) + + # Use the max. for normalizing running avg. of gradient + denom = (max_exp_avg_sqs[i].sqrt() / bias_correction2_sqrt).add_(eps) + else: + denom = (exp_avg_sq.sqrt() / bias_correction2_sqrt).add_(eps) + + param.addcdiv_(exp_avg, denom, value=-step_size) # type: ignore[arg-type] + + # Lastly, switch back to complex view + if amsgrad and torch.is_complex(params[i]): + max_exp_avg_sqs[i] = torch.view_as_complex(max_exp_avg_sqs[i]) + + +def _multi_tensor_adam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + max_exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + amsgrad: bool, + has_complex: bool, + beta1: float | Tensor, + beta2: float | Tensor, + lr: float | Tensor, + weight_decay: float, + eps: float, + maximize: bool, + capturable: bool, + differentiable: bool, + decoupled_weight_decay: bool, +) -> None: + if len(params) == 0: + return + + if isinstance(lr, Tensor): + if not capturable: + raise RuntimeError( + "lr as a Tensor is not supported for capturable=False and foreach=True" + ) + if lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + + if isinstance(beta1, Tensor): + if not capturable: + raise ValueError( + "beta1 as a Tensor is not supported for capturable=False and foreach=True" + ) + if beta1.numel() != 1: + raise ValueError("Tensor beta1 must be 1-element") + + if isinstance(beta2, Tensor): + if not capturable: + raise ValueError( + "beta2 as a Tensor is not supported for capturable=False and foreach=True" + ) + if beta2.numel() != 1: + raise ValueError("Tensor beta2 must be 1-element") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + if grad_scale is not None or found_inf is not None: + raise AssertionError("Expected grad_scale and found_inf to be None") + + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + lr = _to_scalar(lr) + beta1 = _to_scalar(beta1) + beta2 = _to_scalar(beta2) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps] # type: ignore[list-item] + ) + + # We only shuffle around the beta when it is a Tensor and on CUDA, otherwise, we prefer + # treating it as a scalar. + beta1_dict: DeviceDict | None = ( # type: ignore[attr-defined] + {beta1.device: beta1} + if isinstance(beta1, Tensor) and str(beta1.device) != "cpu" + else None + ) + + for ( + device_params_, + device_grads_, + device_exp_avgs_, + device_exp_avg_sqs_, + device_max_exp_avg_sqs_, + device_state_steps_, + ), _ in grouped_tensors.values(): + device_params = cast(list[Tensor], device_params_) + device_grads = cast(list[Tensor], device_grads_) + device_exp_avgs = cast(list[Tensor], device_exp_avgs_) + device_exp_avg_sqs = cast(list[Tensor], device_exp_avg_sqs_) + device_state_steps = cast(list[Tensor], device_state_steps_) + + device = device_params[0].device + if beta1_dict is not None and device not in beta1_dict: + beta1_dict[device] = beta1.to(device=device, non_blocking=True) # type: ignore[union-attr, attr-defined] + + device_beta1 = beta1_dict[device] if beta1_dict else beta1 + + # Handle complex parameters + if has_complex: + if amsgrad: + device_max_exp_avg_sqs = cast(list[Tensor], device_max_exp_avg_sqs_) + _view_as_real( + device_params, + device_grads, + device_exp_avgs, + device_exp_avg_sqs, + device_max_exp_avg_sqs, + ) + else: + _view_as_real( + device_params, device_grads, device_exp_avgs, device_exp_avg_sqs + ) + + if maximize: + device_grads = torch._foreach_neg(device_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and device_state_steps[0].is_cpu: + torch._foreach_add_( + device_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(device_state_steps, 1) + + if weight_decay != 0: + if decoupled_weight_decay: + # Perform stepweight decay + torch._foreach_mul_(device_params, 1 - lr * weight_decay) + else: + # Reuse the intermediate memory (device_grads) already allocated for maximize + if maximize: + torch._foreach_add_(device_grads, device_params, alpha=weight_decay) + else: + device_grads = torch._foreach_add( # type: ignore[assignment] + device_grads, device_params, alpha=weight_decay + ) + + # Decay the first and second moment running average coefficient + # Use device beta1 if beta1 is a tensor to ensure all + # tensors are on the same device + torch._foreach_lerp_( + device_exp_avgs, device_grads, cast(float, 1 - device_beta1) + ) + + torch._foreach_mul_(device_exp_avg_sqs, beta2) + + # Due to the strictness of the _foreach_addcmul API, we can't have a single + # tensor scalar as the scalar arg (only python number is supported there) + # as a result, separate out the value mul + # Filed https://github.com/pytorch/pytorch/issues/139795 + if isinstance(beta2, torch.Tensor): + scaled_device_grads = torch._foreach_mul(device_grads, 1 - beta2) # type: ignore[assignment] + value = 1.0 + else: + scaled_device_grads = device_grads # type: ignore[assignment] + value = 1 - beta2 + + torch._foreach_addcmul_( + device_exp_avg_sqs, scaled_device_grads, device_grads, value + ) + + # Delete the local intermediate(s) since they won't be used anymore to save on peak memory + del device_grads + del scaled_device_grads + + bias_correction1: tuple[Tensor, ...] | list[Tensor] + bias_correction2: tuple[Tensor, ...] | list[Tensor] + bias_correction2_sqrt: tuple[Tensor, ...] | list[Tensor] + + if capturable: + bias_correction1 = torch._foreach_pow(beta1, device_state_steps) # type: ignore[arg-type] + bias_correction2 = torch._foreach_pow(beta2, device_state_steps) # type: ignore[arg-type] + # foreach_sub doesn't allow a scalar as the first arg + torch._foreach_sub_(bias_correction1, 1) + torch._foreach_sub_(bias_correction2, 1) + # we do not negate bias_correction1 as it'll need to be negated later anyway + torch._foreach_neg_(bias_correction2) + + # foreach_div doesn't allow a scalar as the first arg + torch._foreach_div_(bias_correction1, lr) + torch._foreach_reciprocal_(bias_correction1) + + torch._foreach_sqrt_(bias_correction2) + + # Re-assign for clarity as we maintain minimal intermediates: we'll have + # step_size = - lr / (1 - beta1 ^ t) where t = num_steps + # bias_correction2_sqrt = sqrt(1 - beta2 ^ t) + step_size = bias_correction1 + bias_correction2_sqrt = bias_correction2 + + if amsgrad: + device_max_exp_avg_sqs = cast(list[Tensor], device_max_exp_avg_sqs_) + # Maintains the maximum of all 2nd moment running avg. till now + torch._foreach_maximum_(device_max_exp_avg_sqs, device_exp_avg_sqs) # type: ignore[assignment] + + # Set intermediate to the max. for normalizing running avg. of gradient when amsgrad + exp_avg_sq_sqrt = torch._foreach_sqrt(device_max_exp_avg_sqs) + else: + exp_avg_sq_sqrt = torch._foreach_sqrt(device_exp_avg_sqs) + + torch._foreach_div_(exp_avg_sq_sqrt, bias_correction2_sqrt) + torch._foreach_add_(exp_avg_sq_sqrt, eps) + torch._foreach_div_(exp_avg_sq_sqrt, step_size) + + # at this point, exp_avg_sq_sqrt = - (1 - beta^t) * [sqrt(exp_avg_sq / (1 - beta2^t)) + eps] / lr + torch._foreach_addcdiv_(device_params, device_exp_avgs, exp_avg_sq_sqrt) + else: + bias_correction1 = [ + 1 - beta1 ** _get_value(step) for step in device_state_steps + ] + bias_correction2 = [ + 1 - beta2 ** _get_value(step) for step in device_state_steps + ] + + step_size = _stack_if_compiling([(lr / bc) * -1 for bc in bias_correction1]) + + bias_correction2_sqrt = [bc**0.5 for bc in bias_correction2] # type: ignore[arg-type] + + if amsgrad: + device_max_exp_avg_sqs = cast(list[Tensor], device_max_exp_avg_sqs_) + # Maintains the maximum of all 2nd moment running avg. till now + torch._foreach_maximum_(device_max_exp_avg_sqs, device_exp_avg_sqs) + + # Use the max. for normalizing running avg. of gradient + exp_avg_sq_sqrt = torch._foreach_sqrt(device_max_exp_avg_sqs) + else: + exp_avg_sq_sqrt = torch._foreach_sqrt(device_exp_avg_sqs) + + torch._foreach_div_(exp_avg_sq_sqrt, bias_correction2_sqrt) + torch._foreach_add_(exp_avg_sq_sqrt, eps) + torch._foreach_addcdiv_( + device_params, + device_exp_avgs, + exp_avg_sq_sqrt, + step_size, # type: ignore[arg-type] + ) + + +def _fused_adam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + max_exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + amsgrad: bool, + has_complex: bool, # Needed for consistency. + beta1: float | Tensor, + beta2: float | Tensor, + lr: float | Tensor, + weight_decay: float, + eps: float, + maximize: bool, + capturable: bool, # Needed for consistency. + differentiable: bool, + decoupled_weight_decay: bool, +) -> None: + if not params: + return + if differentiable: + raise RuntimeError("Adam with fused=True does not support differentiable=True") + + beta1 = _to_scalar(beta1) + beta2 = _to_scalar(beta2) + + grad_scale_dict: DeviceDict = ( + {grad_scale.device: grad_scale} if grad_scale is not None else {} + ) + found_inf_dict: DeviceDict = ( + {found_inf.device: found_inf} if found_inf is not None else {} + ) + + # We only shuffle around the lr when it is a Tensor and on CUDA, otherwise, we prefer + # treating it as a scalar. + lr_dict: DeviceDict | None = ( + {lr.device: lr} if isinstance(lr, Tensor) and str(lr.device) != "cpu" else None + ) + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps] # type: ignore[list-item] + ) + for (device, _), ( + ( + device_params_, + device_grads_, + device_exp_avgs_, + device_exp_avg_sqs_, + device_max_exp_avg_sqs, + device_state_steps_, + ), + _, + ) in grouped_tensors.items(): + device_params = cast(list[Tensor], device_params_) + device_grads = cast(list[Tensor], device_grads_) + device_exp_avgs = cast(list[Tensor], device_exp_avgs_) + device_exp_avg_sqs = cast(list[Tensor], device_exp_avg_sqs_) + device_state_steps = cast(list[Tensor], device_state_steps_) + + device_grad_scale, device_found_inf = None, None + if grad_scale is not None: + device_grad_scale = grad_scale_dict.setdefault( + device, grad_scale.to(device, non_blocking=True) + ) + if found_inf is not None: + device_found_inf = found_inf_dict.setdefault( + device, found_inf.to(device, non_blocking=True) + ) + if lr_dict is not None and device not in lr_dict: + lr_dict[device] = lr.to(device=device, non_blocking=True) # type: ignore[union-attr] + lr = lr_dict[device] + torch._foreach_add_(device_state_steps, 1) + func = torch._fused_adam_ if not decoupled_weight_decay else torch._fused_adamw_ + func( + device_params, + device_grads, + device_exp_avgs, + device_exp_avg_sqs, + device_max_exp_avg_sqs, # type: ignore[arg-type] + device_state_steps, + amsgrad=amsgrad, + lr=lr, # type: ignore[arg-type] + beta1=beta1, + beta2=beta2, + weight_decay=weight_decay, + eps=eps, + maximize=maximize, + grad_scale=device_grad_scale, + found_inf=device_found_inf, + ) + if device_found_inf is not None: + torch._foreach_sub_( + device_state_steps, [device_found_inf] * len(device_state_steps) + ) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_adam) +def adam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + max_exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + capturable: bool = False, + differentiable: bool = False, + fused: bool | None = None, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, + has_complex: bool = False, + decoupled_weight_decay: bool = False, + *, + amsgrad: bool, + beta1: float | Tensor, + beta2: float | Tensor, + lr: float | Tensor, + weight_decay: float, + eps: float, + maximize: bool, +) -> None: + r"""Functional API that performs Adam algorithm computation. + + See :class:`~torch.optim.Adam` for details. + """ + # Respect when the user inputs False/True for foreach or fused. We only want to change + # the default when neither have been user-specified. Note that we default to foreach + # and pass False to use_fused. This is not a mistake--we want to give the fused impl + # bake-in time before making it the default, even if it is typically faster. + if fused is None and foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + # Do not flip on foreach for the unsupported case where lr is a Tensor and capturable=False. + if foreach and isinstance(lr, Tensor) and not capturable: + foreach = False + if fused is None: + fused = False + if foreach is None: + foreach = False + + # this check is slow during compilation, so we skip it + # if it's strictly needed we can add this check back in dynamo + if not torch.compiler.is_compiling() and not all( + isinstance(t, torch.Tensor) for t in state_steps + ): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + if fused and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with fused optimizers") + + if fused and not torch.jit.is_scripting(): + func = _fused_adam + elif foreach and not torch.jit.is_scripting(): + func = _multi_tensor_adam + else: + func = _single_tensor_adam + + func( + params, + grads, + exp_avgs, + exp_avg_sqs, + max_exp_avg_sqs, + state_steps, + amsgrad=amsgrad, + has_complex=has_complex, + beta1=beta1, + beta2=beta2, + lr=lr, + weight_decay=weight_decay, + eps=eps, + maximize=maximize, + capturable=capturable, + differentiable=differentiable, + grad_scale=grad_scale, + found_inf=found_inf, + decoupled_weight_decay=decoupled_weight_decay, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adamax.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adamax.py new file mode 100644 index 0000000000000000000000000000000000000000..320ee97d14e5a07529aa00917a30a14f9a359dd1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adamax.py @@ -0,0 +1,485 @@ +# mypy: allow-untyped-defs +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _get_value, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["Adamax", "adamax"] + + +class Adamax(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 2e-3, + betas: tuple[float, float] = (0.9, 0.999), + eps: float = 1e-8, + weight_decay: float = 0, + foreach: bool | None = None, + *, + maximize: bool = False, + differentiable: bool = False, + capturable: bool = False, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= betas[0] < 1.0: + raise ValueError(f"Invalid beta parameter at index 0: {betas[0]}") + if not 0.0 <= betas[1] < 1.0: + raise ValueError(f"Invalid beta parameter at index 1: {betas[1]}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + + defaults = { + "lr": lr, + "betas": betas, + "eps": eps, + "weight_decay": weight_decay, + "foreach": foreach, + "maximize": maximize, + "differentiable": differentiable, + "capturable": capturable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + group.setdefault("capturable", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + + def _init_group( + self, group, params_with_grad, grads, exp_avgs, exp_infs, state_steps + ): + has_complex = False + for p in group["params"]: + if p.grad is None: + continue + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError("Adamax does not support sparse gradients") + grads.append(p.grad) + + state = self.state[p] + + # State initialization + if len(state) == 0: + state["step"] = ( + torch.zeros((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.tensor(0.0, dtype=_get_scalar_dtype()) + ) + state["exp_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + state["exp_inf"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + exp_avgs.append(state["exp_avg"]) + exp_infs.append(state["exp_inf"]) + state_steps.append(state["step"]) + + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Performs a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + exp_avgs: list[Tensor] = [] + exp_infs: list[Tensor] = [] + state_steps: list[Tensor] = [] + + beta1, beta2 = group["betas"] + eps = group["eps"] + lr = group["lr"] + weight_decay = group["weight_decay"] + foreach = group["foreach"] + maximize = group["maximize"] + differentiable = group["differentiable"] + capturable = group["capturable"] + + has_complex = self._init_group( + group, params_with_grad, grads, exp_avgs, exp_infs, state_steps + ) + + adamax( + params_with_grad, + grads, + exp_avgs, + exp_infs, + state_steps, + eps=eps, + beta1=beta1, + beta2=beta2, + lr=lr, + weight_decay=weight_decay, + foreach=foreach, + maximize=maximize, + differentiable=differentiable, + capturable=capturable, + has_complex=has_complex, + ) + + return loss + + +Adamax.__doc__ = ( + r"""Implements Adamax algorithm (a variant of Adam based on infinity norm). + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)}, \beta_1, \beta_2 + \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)}, + \: \lambda \text{ (weight decay)}, \\ + &\hspace{13mm} \epsilon \text{ (epsilon)} \\ + &\textbf{initialize} : m_0 \leftarrow 0 \text{ ( first moment)}, + u_0 \leftarrow 0 \text{ ( infinity norm)} \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}if \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ + &\hspace{5mm}u_t \leftarrow \mathrm{max}(\beta_2 u_{t-1}, |g_{t}|+\epsilon) \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \frac{\gamma m_t}{(1-\beta^t_1) u_t} \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 2e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + {_foreach_doc} + {_maximize_doc} + {_differentiable_doc} + {_capturable_doc} + + .. _Adam\: A Method for Stochastic Optimization: + https://arxiv.org/abs/1412.6980 + + """ +) + + +def _single_tensor_adamax( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_infs: list[Tensor], + state_steps: list[Tensor], + *, + eps: float, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for i, param in enumerate(params): + grad = grads[i] + grad = grad if not maximize else -grad + exp_avg = exp_avgs[i] + exp_inf = exp_infs[i] + step_t = state_steps[i] + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type == step_t.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + # update step + step_t += 1 + + if weight_decay != 0: + grad = grad.add(param, alpha=weight_decay) + + if torch.is_complex(param): + param = torch.view_as_real(param) + grad = torch.view_as_real(grad) + exp_avg = torch.view_as_real(exp_avg) + exp_inf = torch.view_as_real(exp_inf) + + # Update biased first moment estimate. + exp_avg.lerp_(grad, 1 - beta1) + # Update the exponentially weighted infinity norm. + if not differentiable: + torch.maximum( + exp_inf.mul_(beta2), + grad.abs().add_(eps), + out=exp_inf, + ) + else: + norm_buf = torch.cat( + [exp_inf.mul_(beta2).unsqueeze(0), grad.abs().add_(eps).unsqueeze_(0)], + 0, + ) + exp_inf.copy_(torch.amax(norm_buf, 0, keepdim=False)) + + if capturable: + # why jump through extra hoops and negate bias_correction? check out #121238 + # once fixed, we should use bias_correction with addcdiv value=-1 for readability + neg_bias_correction = beta1**step_t - 1 + neg_bias_correction.div_(lr) + denom = exp_inf * neg_bias_correction + param.addcdiv_(exp_avg, denom) + else: + bias_correction = 1 - beta1 ** _get_value(step_t) + clr = lr / bias_correction + + param.addcdiv_(exp_avg, exp_inf, value=-clr) + + +def _multi_tensor_adamax( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_infs: list[Tensor], + state_steps: list[Tensor], + *, + eps: float, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + if len(params) == 0: + return + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, exp_avgs, exp_infs, state_steps] # type: ignore[list-item] + ) + for ( + grouped_params_, + grouped_grads_, + grouped_exp_avgs_, + grouped_exp_infs_, + grouped_state_steps_, + ), _ in grouped_tensors.values(): + grouped_params = cast(list[Tensor], grouped_params_) + grouped_grads = cast(list[Tensor], grouped_grads_) + grouped_exp_avgs = cast(list[Tensor], grouped_exp_avgs_) + grouped_exp_infs = cast(list[Tensor], grouped_exp_infs_) + grouped_state_steps = cast(list[Tensor], grouped_state_steps_) + + if has_complex: + _view_as_real( + grouped_params, grouped_grads, grouped_exp_avgs, grouped_exp_infs + ) + + if maximize: + grouped_grads = torch._foreach_neg(grouped_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and grouped_state_steps[0].is_cpu: + torch._foreach_add_( + grouped_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(grouped_state_steps, 1) + + if weight_decay != 0: + if maximize: + # Reuse the intermediate memory (grouped_grads) already allocated for maximize + torch._foreach_add_(grouped_grads, grouped_params, alpha=weight_decay) + else: + grouped_grads = torch._foreach_add( # type: ignore[assignment] + grouped_grads, grouped_params, alpha=weight_decay + ) + + # Update biased first moment estimate. + torch._foreach_lerp_(grouped_exp_avgs, grouped_grads, 1 - beta1) + + # Update the exponentially weighted infinity norm. + torch._foreach_mul_(grouped_exp_infs, beta2) + + # in this case, we need to introduce a copy of the grads + # since one has not been introduced previously + if not maximize and weight_decay == 0: + grouped_grads = torch._foreach_abs(grouped_grads) # type: ignore[assignment] + else: + torch._foreach_abs_(grouped_grads) + + torch._foreach_add_(grouped_grads, eps) + torch._foreach_maximum_(grouped_exp_infs, grouped_grads) + + bias_corrections: tuple[Tensor, ...] | list[Tensor] + if capturable: + bias_corrections = torch._foreach_pow(beta1, grouped_state_steps) + # foreach_sub doesn't allow a scalar as the first arg + torch._foreach_sub_(bias_corrections, 1) + torch._foreach_div_(bias_corrections, lr) + + denom = torch._foreach_mul(grouped_exp_infs, bias_corrections) + torch._foreach_addcdiv_(grouped_params, grouped_exp_avgs, denom) + else: + bias_corrections = [ + 1 - beta1 ** _get_value(step) for step in grouped_state_steps + ] + step_size = [(_get_value(lr) / bc) * -1 for bc in bias_corrections] + torch._foreach_addcdiv_( + grouped_params, grouped_exp_avgs, grouped_exp_infs, step_size + ) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_adamax) +def adamax( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_infs: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + maximize: bool = False, + differentiable: bool = False, + capturable: bool = False, + has_complex: bool = False, + *, + eps: float, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, +) -> None: + r"""Functional API that performs adamax algorithm computation. + + See :class:`~torch.optim.Adamax` for details. + """ + + if not torch.compiler.is_compiling() and not all( + isinstance(t, torch.Tensor) for t in state_steps + ): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_adamax + else: + func = _single_tensor_adamax + + func( + params, + grads, + exp_avgs, + exp_infs, + state_steps, + eps=eps, + beta1=beta1, + beta2=beta2, + lr=lr, + weight_decay=weight_decay, + maximize=maximize, + differentiable=differentiable, + has_complex=has_complex, + capturable=capturable, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adamw.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adamw.py new file mode 100644 index 0000000000000000000000000000000000000000..aa3b922cf90b4ed346d7acbafddab176a9f09faa --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/adamw.py @@ -0,0 +1,181 @@ +# mypy: allow-untyped-defs + +from torch import Tensor + +from .adam import Adam, adam +from .optimizer import ( + _capturable_doc, + _differentiable_doc, + _foreach_doc, + _fused_doc, + _maximize_doc, + _params_doc, + ParamsT, +) + + +__all__ = ["AdamW", "adamw"] + + +class AdamW(Adam): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-3, + betas: tuple[float | Tensor, float | Tensor] = (0.9, 0.999), + eps: float = 1e-8, + weight_decay: float = 1e-2, + amsgrad: bool = False, + *, + maximize: bool = False, + foreach: bool | None = None, + capturable: bool = False, + differentiable: bool = False, + fused: bool | None = None, + ) -> None: + super().__init__( + params, + lr, + betas, + eps, + weight_decay, + amsgrad, + foreach=foreach, + maximize=maximize, + capturable=capturable, + differentiable=differentiable, + fused=fused, + decoupled_weight_decay=True, + ) + + # Preserve decoupled_weight_decay from AdamW for backwards compatibility. The following + # guarantees that decoupled_weight_decay will always be True for loading any state into + # AdamW + def __setstate__(self, state): + super().__setstate__(state) + for group in self.param_groups: + group["decoupled_weight_decay"] = True + + +AdamW.__doc__ = ( + r"""Implements AdamW algorithm, where weight decay does not accumulate in the momentum nor variance. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{(lr)}, \: \beta_1, \beta_2 + \text{(betas)}, \: \theta_0 \text{(params)}, \: f(\theta) \text{(objective)}, + \: \epsilon \text{ (epsilon)} \\ + &\hspace{13mm} \lambda \text{(weight decay)}, \: \textit{amsgrad}, + \: \textit{maximize} \\ + &\textbf{initialize} : m_0 \leftarrow 0 \text{ (first moment)}, v_0 \leftarrow 0 + \text{ ( second moment)}, \: v_0^{max}\leftarrow 0 \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + + &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\ + &\hspace{10mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1} \\ + &\hspace{5mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ + &\hspace{5mm}v_t \leftarrow \beta_2 v_{t-1} + (1-\beta_2) g^2_t \\ + &\hspace{5mm}\widehat{m_t} \leftarrow m_t/\big(1-\beta_1^t \big) \\ + &\hspace{5mm}\textbf{if} \: amsgrad \\ + &\hspace{10mm} v_t^{max} \leftarrow \mathrm{max}(v_{t-1}^{max},v_t) \\ + &\hspace{10mm}\widehat{v_t} \leftarrow v_t^{max}/\big(1-\beta_2^t \big) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}\widehat{v_t} \leftarrow v_t/\big(1-\beta_2^t \big) \\ + &\hspace{5mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/ + \big(\sqrt{\widehat{v_t}} + \epsilon \big) \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `Decoupled Weight Decay Regularization`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR + is not yet supported for all our implementations. Please use a float + LR if you are not also specifying fused=True or capturable=True. + betas (tuple[Union[float, Tensor], Union[float, Tensor]], optional): + coefficients used for computing running averages of gradient and + its square. If a tensor is provided, must be 1-element. (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay coefficient (default: 1e-2) + amsgrad (bool, optional): whether to use the AMSGrad variant of this + algorithm from the paper `On the Convergence of Adam and Beyond`_ + (default: False) + {_maximize_doc} + {_foreach_doc} + {_capturable_doc} + {_differentiable_doc} + {_fused_doc} + .. Note:: + A prototype implementation of Adam and AdamW for MPS supports `torch.float32` and `torch.float16`. + .. _Decoupled Weight Decay Regularization: + https://arxiv.org/abs/1711.05101 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + + """ +) + + +# @_disable_dynamo_if_unsupported logic occurs in the decorator that's applied to F.adam +def adamw( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + max_exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + capturable: bool = False, + differentiable: bool = False, + fused: bool | None = None, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, + has_complex: bool = False, + *, + amsgrad: bool, + beta1: float | Tensor, + beta2: float | Tensor, + lr: float | Tensor, + weight_decay: float, + eps: float, + maximize: bool, +) -> None: + r"""Functional API that performs AdamW algorithm computation. + + See :class:`~torch.optim.AdamW` for details. + """ + adam( + params, + grads, + exp_avgs, + exp_avg_sqs, + max_exp_avg_sqs, + state_steps, + foreach=foreach, + capturable=capturable, + differentiable=differentiable, + fused=fused, + grad_scale=grad_scale, + found_inf=found_inf, + has_complex=has_complex, + amsgrad=amsgrad, + beta1=beta1, + beta2=beta2, + lr=lr, + weight_decay=weight_decay, + eps=eps, + maximize=maximize, + decoupled_weight_decay=True, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/asgd.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/asgd.py new file mode 100644 index 0000000000000000000000000000000000000000..19f2e6e25bebaef20b64a2269cdc4b9dbbee9f0b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/asgd.py @@ -0,0 +1,479 @@ +# mypy: allow-untyped-defs +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _get_value, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["ASGD", "asgd"] + + +class ASGD(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-2, + lambd: float = 1e-4, + alpha: float = 0.75, + t0: float = 1e6, + weight_decay: float = 0, + foreach: bool | None = None, + maximize: bool = False, + differentiable: bool = False, + capturable: bool = False, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + + defaults = { + "lr": lr, + "lambd": lambd, + "alpha": alpha, + "t0": t0, + "weight_decay": weight_decay, + "foreach": foreach, + "maximize": maximize, + "differentiable": differentiable, + "capturable": capturable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + group.setdefault("capturable", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0: + if not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if not torch.is_tensor(p_state["eta"]): + p_state["eta"] = torch.tensor( + p_state["eta"], dtype=_get_scalar_dtype(), device=p.device + ) + if not torch.is_tensor(p_state["mu"]): + p_state["mu"] = torch.tensor( + p_state["mu"], dtype=_get_scalar_dtype(), device=p.device + ) + + def _init_group(self, group, params_with_grad, grads, mus, axs, etas, state_steps): + has_complex = False + for p in group["params"]: + if p.grad is not None: + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError("ASGD does not support sparse gradients") + grads.append(p.grad) + + state = self.state[p] + # State initialization + if len(state) == 0: + state["step"] = torch.zeros( + (), device=p.device, dtype=_get_scalar_dtype() + ) + state["eta"] = ( + torch.as_tensor( + _to_scalar(group["lr"]), + device=p.device, + dtype=_get_scalar_dtype(), + ) + .clone() + .detach() + ) + state["mu"] = torch.ones( + (), device=p.device, dtype=_get_scalar_dtype() + ) + state["ax"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + mus.append(state["mu"]) + axs.append(state["ax"]) + etas.append(state["eta"]) + state_steps.append(state["step"]) + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + mus: list[Tensor] = [] + axs: list[Tensor] = [] + etas: list[Tensor] = [] + state_steps: list[Tensor] = [] + + has_complex = self._init_group( + group, params_with_grad, grads, mus, axs, etas, state_steps + ) + + asgd( + params_with_grad, + grads, + axs, + mus, + etas, + state_steps, + lambd=group["lambd"], + lr=group["lr"], + t0=group["t0"], + alpha=group["alpha"], + weight_decay=group["weight_decay"], + foreach=group["foreach"], + maximize=group["maximize"], + differentiable=group["differentiable"], + capturable=group["capturable"], + has_complex=has_complex, + ) + + return loss + + +ASGD.__doc__ = rf"""Implements Averaged Stochastic Gradient Descent. + + It has been proposed in `Acceleration of stochastic approximation by + averaging`_. + + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-2) + lambd (float, optional): decay term (default: 1e-4) + alpha (float, optional): power for eta update (default: 0.75) + t0 (float, optional): point at which to start averaging (default: 1e6) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + {_foreach_doc} + {_maximize_doc} + {_differentiable_doc} + {_capturable_doc} + + .. _Acceleration of stochastic approximation by averaging: + https://meyn.ece.ufl.edu/wp-content/uploads/sites/77/archive/spm_files/Courses/ECE555-2011/555media/poljud92.pdf + + """ + + +def _single_tensor_asgd( + params: list[Tensor], + grads: list[Tensor], + axs: list[Tensor], + mus: list[Tensor], + etas: list[Tensor], + state_steps: list[Tensor], + *, + lambd: float, + lr: float, + t0: float, + alpha: float, + weight_decay: float, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for i, param in enumerate(params): + grad = grads[i] + grad = grad if not maximize else -grad + mu = mus[i] + ax = axs[i] + eta = etas[i] + step_t = state_steps[i] + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type + == mu.device.type + == eta.device.type + == step_t.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params, mus, etas, and state_steps must be " + f"on supported devices: {capturable_supported_devices}." + ) + + if torch.is_complex(param): + grad = torch.view_as_real(grad) + param = torch.view_as_real(param) + ax = torch.view_as_real(ax) + + # update step + step_t += 1 + + if weight_decay != 0: + grad = grad.add(param, alpha=weight_decay) + + if capturable: + param.mul_(1 - lambd * eta) + param.addcmul_(grad, eta, value=-1) # update parameter + else: + eta_value = _get_value(eta) + param.mul_(1 - lambd * eta_value) # decay term + param.add_(grad, alpha=-eta_value) # update parameter + + # averaging + if capturable or mu.item() != 1: + ax.add_(param.sub(ax).mul_(mu)) + else: + ax.copy_(param) + + if capturable: + # pyrefly: ignore [unsupported-operation] + eta.copy_(lr / ((1 + lambd * lr * step_t) ** alpha)) + mu.copy_(1 / torch.maximum(step_t - t0, torch.ones_like(step_t))) + else: + step = _get_value(step_t) + new_eta = torch.as_tensor(lr / ((1 + lambd * lr * step) ** alpha)) + eta.copy_(new_eta) + new_mu = torch.as_tensor(1 / max(1, step - t0)) + mu.copy_(new_mu) + + +def _multi_tensor_asgd( + params: list[Tensor], + grads: list[Tensor], + axs: list[Tensor], + mus: list[Tensor], + etas: list[Tensor], + state_steps: list[Tensor], + *, + lambd: float, + lr: float, + t0: float, + alpha: float, + weight_decay: float, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if len(params) == 0: + return + + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == mu.device.type == eta.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, mu, eta, step in zip(params, mus, etas, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params, mus, etas, and state_steps must be on " + f"supported devices: {capturable_supported_devices}." + ) + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, axs, mus, etas, state_steps] # type: ignore[list-item] + ) + for (device, _), ( + ( + grouped_params_, + grouped_grads_, + grouped_axs_, + grouped_mus_, + grouped_etas_, + grouped_state_steps_, + ), + _, + ) in grouped_tensors.items(): + grouped_params = cast(list[Tensor], grouped_params_) + grouped_grads = cast(list[Tensor], grouped_grads_) + grouped_axs = cast(list[Tensor], grouped_axs_) + grouped_mus = cast(list[Tensor], grouped_mus_) + grouped_etas = cast(list[Tensor], grouped_etas_) + grouped_state_steps = cast(list[Tensor], grouped_state_steps_) + + if has_complex: + _view_as_real(grouped_params, grouped_grads, grouped_axs) + + if maximize: + grouped_grads = torch._foreach_neg(grouped_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and grouped_state_steps[0].is_cpu: + torch._foreach_add_( + grouped_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(grouped_state_steps, 1) + + # intermediate = grad + param * lambd + intermediate: tuple[Tensor, ...] | list[Tensor] + if weight_decay != 0: + if maximize: + torch._foreach_add_(grouped_grads, grouped_params, alpha=weight_decay) + intermediate = grouped_grads + else: + intermediate = torch._foreach_add( + grouped_grads, grouped_params, alpha=weight_decay + ) + + torch._foreach_add_(intermediate, grouped_params, alpha=lambd) + else: + intermediate = torch._foreach_add( + grouped_grads, grouped_params, alpha=lambd + ) + + # update param + # param * (1 - lambd * eta) - eta * grad + # => param - param * lambd * eta - eta * grad + # => param - eta * intermediate + torch._foreach_addcmul_(grouped_params, intermediate, grouped_etas, value=-1) + del intermediate + + # update grouped_axs + # averaging: ax = ax + mu * (param - ax) + # Note (mlazos): We can't use lerp here since it requires weight to be float64 + # and our grouping code requires dtypes to match for all tensors in a group (and it should, since + # we use the mus in other places) + # all dtypes need to match, so we could introduce a cast in a loop + # but since this only adds one additional kernel launch, this looks like the cleaner + # and faster solution + intermediate = torch._foreach_sub(grouped_params, grouped_axs) + torch._foreach_addcmul_(grouped_axs, intermediate, grouped_mus) + del intermediate + + new_etas: tuple[Tensor, ...] | list[Tensor] + new_mus: tuple[Tensor, ...] | list[Tensor] + if capturable: + # update grouped_mus + new_mus = torch._foreach_sub(grouped_state_steps, t0) + torch._foreach_maximum_(new_mus, 1.0) + torch._foreach_reciprocal_(new_mus) + torch._foreach_copy_(grouped_mus, new_mus) + del new_mus + + # update eta = lr / ((1 + lambd * lr * step)^alpha) + new_etas = torch._foreach_mul(grouped_state_steps, lambd) + torch._foreach_mul_(new_etas, lr) + torch._foreach_add_(new_etas, 1) + torch._foreach_pow_(new_etas, alpha) + torch._foreach_reciprocal_(new_etas) + torch._foreach_mul_(new_etas, lr) + torch._foreach_copy_(grouped_etas, new_etas) + else: + new_etas = [ + torch.as_tensor(lr / ((1 + lambd * lr * step) ** alpha), device=device) + for step in grouped_state_steps + ] + new_mus = [ + torch.as_tensor(1 / max(1, _get_value(step) - t0), device=device) + for step in grouped_state_steps + ] + torch._foreach_copy_(grouped_etas, new_etas) + torch._foreach_copy_(grouped_mus, new_mus) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_asgd) +def asgd( + params: list[Tensor], + grads: list[Tensor], + axs: list[Tensor], + mus: list[Tensor], + etas: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + maximize: bool = False, + differentiable: bool = False, + capturable: bool = False, + has_complex: bool = False, + *, + lambd: float, + lr: float, + t0: float, + alpha: float, + weight_decay: float, +) -> None: + r"""Functional API that performs asgd algorithm computation. + + See :class:`~torch.optim.ASGD` for details. + """ + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_asgd + else: + func = _single_tensor_asgd + + func( + params, + grads, + axs, + mus, + etas, + state_steps, + lambd=lambd, + lr=lr, + t0=t0, + alpha=alpha, + weight_decay=weight_decay, + maximize=maximize, + differentiable=differentiable, + capturable=capturable, + has_complex=has_complex, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/lbfgs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/lbfgs.py new file mode 100644 index 0000000000000000000000000000000000000000..ed4cf1a8b2e882c0d2217607a03f97333a1123e8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/lbfgs.py @@ -0,0 +1,537 @@ +# mypy: allow-untyped-defs + +import torch +from torch import Tensor + +from .optimizer import _to_scalar, Optimizer, ParamsT + + +__all__ = ["LBFGS"] + + +def _cubic_interpolate(x1, f1, g1, x2, f2, g2, bounds=None): + # ported from https://github.com/torch/optim/blob/master/polyinterp.lua + # Compute bounds of interpolation area + if bounds is not None: + xmin_bound, xmax_bound = bounds + else: + xmin_bound, xmax_bound = (x1, x2) if x1 <= x2 else (x2, x1) + + # Code for most common case: cubic interpolation of 2 points + # w/ function and derivative values for both + # Solution in this case (where x2 is the farthest point): + # d1 = g1 + g2 - 3*(f1-f2)/(x1-x2); + # d2 = sqrt(d1^2 - g1*g2); + # min_pos = x2 - (x2 - x1)*((g2 + d2 - d1)/(g2 - g1 + 2*d2)); + # t_new = min(max(min_pos,xmin_bound),xmax_bound); + d1 = g1 + g2 - 3 * (f1 - f2) / (x1 - x2) + d2_square = d1**2 - g1 * g2 + if d2_square >= 0: + d2 = d2_square.sqrt() + if x1 <= x2: + min_pos = x2 - (x2 - x1) * ((g2 + d2 - d1) / (g2 - g1 + 2 * d2)) + else: + min_pos = x1 - (x1 - x2) * ((g1 + d2 - d1) / (g1 - g2 + 2 * d2)) + return min(max(min_pos, xmin_bound), xmax_bound) + else: + return (xmin_bound + xmax_bound) / 2.0 + + +def _strong_wolfe( + obj_func, x, t, d, f, g, gtd, c1=1e-4, c2=0.9, tolerance_change=1e-9, max_ls=25 +): + # ported from https://github.com/torch/optim/blob/master/lswolfe.lua + d_norm = d.abs().max() + g = g.clone(memory_format=torch.contiguous_format) + # evaluate objective and gradient using initial step + f_new, g_new = obj_func(x, t, d) + ls_func_evals = 1 + gtd_new = g_new.dot(d) + + # bracket an interval containing a point satisfying the Wolfe criteria + t_prev, f_prev, g_prev, gtd_prev = 0, f, g, gtd + done = False + ls_iter = 0 + while ls_iter < max_ls: + # check conditions + if f_new > (f + c1 * t * gtd) or (ls_iter > 1 and f_new >= f_prev): + bracket = [t_prev, t] + bracket_f = [f_prev, f_new] + bracket_g = [g_prev, g_new.clone(memory_format=torch.contiguous_format)] + bracket_gtd = [gtd_prev, gtd_new] + break + + if abs(gtd_new) <= -c2 * gtd: + bracket = [t] + bracket_f = [f_new] + bracket_g = [g_new] + done = True + break + + if gtd_new >= 0: + bracket = [t_prev, t] + bracket_f = [f_prev, f_new] + bracket_g = [g_prev, g_new.clone(memory_format=torch.contiguous_format)] + bracket_gtd = [gtd_prev, gtd_new] + break + + # interpolate + min_step = t + 0.01 * (t - t_prev) + max_step = t * 10 + tmp = t + t = _cubic_interpolate( + t_prev, f_prev, gtd_prev, t, f_new, gtd_new, bounds=(min_step, max_step) + ) + + # next step + t_prev = tmp + f_prev = f_new + g_prev = g_new.clone(memory_format=torch.contiguous_format) + gtd_prev = gtd_new + f_new, g_new = obj_func(x, t, d) + ls_func_evals += 1 + gtd_new = g_new.dot(d) + ls_iter += 1 + + # reached max number of iterations? + if ls_iter == max_ls: + bracket = [0, t] + bracket_f = [f, f_new] + bracket_g = [g, g_new] + + # zoom phase: we now have a point satisfying the criteria, or + # a bracket around it. We refine the bracket until we find the + # exact point satisfying the criteria + insuf_progress = False + # find high and low points in bracket + low_pos, high_pos = (0, 1) if bracket_f[0] <= bracket_f[-1] else (1, 0) # type: ignore[possibly-undefined] + while not done and ls_iter < max_ls: + # line-search bracket is so small + if abs(bracket[1] - bracket[0]) * d_norm < tolerance_change: # type: ignore[possibly-undefined] + break + + # compute new trial value + t = _cubic_interpolate( + # pyrefly: ignore [index-error] + # pyrefly: ignore [unbound-name] + bracket[0], + # pyrefly: ignore [unbound-name] + bracket_f[0], + bracket_gtd[0], # type: ignore[possibly-undefined] + # pyrefly: ignore [index-error] + # pyrefly: ignore [unbound-name] + bracket[1], + # pyrefly: ignore [unbound-name] + bracket_f[1], + # pyrefly: ignore [unbound-name] + bracket_gtd[1], + ) + + # test that we are making sufficient progress: + # in case `t` is so close to boundary, we mark that we are making + # insufficient progress, and if + # + we have made insufficient progress in the last step, or + # + `t` is at one of the boundary, + # we will move `t` to a position which is `0.1 * len(bracket)` + # away from the nearest boundary point. + # pyrefly: ignore [unbound-name] + eps = 0.1 * (max(bracket) - min(bracket)) + # pyrefly: ignore [unbound-name] + if min(max(bracket) - t, t - min(bracket)) < eps: + # interpolation close to boundary + # pyrefly: ignore [unbound-name] + if insuf_progress or t >= max(bracket) or t <= min(bracket): + # evaluate at 0.1 away from boundary + # pyrefly: ignore [unbound-name] + if abs(t - max(bracket)) < abs(t - min(bracket)): + # pyrefly: ignore [unbound-name] + t = max(bracket) - eps + else: + # pyrefly: ignore [unbound-name] + t = min(bracket) + eps + insuf_progress = False + else: + insuf_progress = True + else: + insuf_progress = False + + # Evaluate new point + f_new, g_new = obj_func(x, t, d) + ls_func_evals += 1 + gtd_new = g_new.dot(d) + ls_iter += 1 + + # pyrefly: ignore [unbound-name] + if f_new > (f + c1 * t * gtd) or f_new >= bracket_f[low_pos]: + # Armijo condition not satisfied or not lower than lowest point + # pyrefly: ignore [unsupported-operation] + # pyrefly: ignore [unbound-name] + bracket[high_pos] = t + # pyrefly: ignore [unbound-name] + bracket_f[high_pos] = f_new + bracket_g[high_pos] = g_new.clone(memory_format=torch.contiguous_format) # type: ignore[possibly-undefined] + # pyrefly: ignore [unbound-name] + bracket_gtd[high_pos] = gtd_new + # pyrefly: ignore [unbound-name] + low_pos, high_pos = (0, 1) if bracket_f[0] <= bracket_f[1] else (1, 0) + else: + if abs(gtd_new) <= -c2 * gtd: + # Wolfe conditions satisfied + done = True + # pyrefly: ignore [index-error] + # pyrefly: ignore [unbound-name] + elif gtd_new * (bracket[high_pos] - bracket[low_pos]) >= 0: + # old high becomes new low + # pyrefly: ignore [unsupported-operation] + # pyrefly: ignore [unbound-name] + bracket[high_pos] = bracket[low_pos] + # pyrefly: ignore [unbound-name] + bracket_f[high_pos] = bracket_f[low_pos] + bracket_g[high_pos] = bracket_g[low_pos] # type: ignore[possibly-undefined] + # pyrefly: ignore [unbound-name] + bracket_gtd[high_pos] = bracket_gtd[low_pos] + + # new point becomes new low + # pyrefly: ignore [unsupported-operation] + # pyrefly: ignore [unbound-name] + bracket[low_pos] = t + # pyrefly: ignore [unbound-name] + bracket_f[low_pos] = f_new + bracket_g[low_pos] = g_new.clone(memory_format=torch.contiguous_format) # type: ignore[possibly-undefined] + # pyrefly: ignore [unbound-name] + bracket_gtd[low_pos] = gtd_new + + # return stuff + t = bracket[low_pos] # type: ignore[possibly-undefined] + # pyrefly: ignore [unbound-name] + f_new = bracket_f[low_pos] + g_new = bracket_g[low_pos] # type: ignore[possibly-undefined] + return f_new, g_new, t, ls_func_evals + + +class LBFGS(Optimizer): + """Implements L-BFGS algorithm. + + Heavily inspired by `minFunc + `_. + + .. warning:: + This optimizer doesn't support per-parameter options and parameter + groups (there can be only one). + + .. warning:: + Right now all parameters have to be on a single device. This will be + improved in the future. + + .. note:: + This is a very memory intensive optimizer (it requires additional + ``param_bytes * (history_size + 1)`` bytes). If it doesn't fit in memory + try reducing the history size, or use a different algorithm. + + Args: + params (iterable): iterable of parameters to optimize. Parameters must be real. + lr (float, optional): learning rate (default: 1) + max_iter (int, optional): maximal number of iterations per optimization step + (default: 20) + max_eval (int, optional): maximal number of function evaluations per optimization + step (default: max_iter * 1.25). + tolerance_grad (float, optional): termination tolerance on first order optimality + (default: 1e-7). + tolerance_change (float, optional): termination tolerance on function + value/parameter changes (default: 1e-9). + history_size (int, optional): update history size (default: 100). + line_search_fn (str, optional): either 'strong_wolfe' or None (default: None). + """ + + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1, + max_iter: int = 20, + max_eval: int | None = None, + tolerance_grad: float = 1e-7, + tolerance_change: float = 1e-9, + history_size: int = 100, + line_search_fn: str | None = None, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if max_eval is None: + max_eval = max_iter * 5 // 4 + defaults = { + "lr": lr, + "max_iter": max_iter, + "max_eval": max_eval, + "tolerance_grad": tolerance_grad, + "tolerance_change": tolerance_change, + "history_size": history_size, + "line_search_fn": line_search_fn, + } + super().__init__(params, defaults) + + if len(self.param_groups) != 1: + raise ValueError( + "LBFGS doesn't support per-parameter options (parameter groups)" + ) + + self._params = self.param_groups[0]["params"] + self._numel_cache = None + + def _numel(self): + if self._numel_cache is None: + # pyrefly: ignore [bad-assignment] + self._numel_cache = sum( + 2 * p.numel() if torch.is_complex(p) else p.numel() + for p in self._params + ) + + return self._numel_cache + + def _gather_flat_grad(self): + views = [] + for p in self._params: + if p.grad is None: + view = p.new(p.numel()).zero_() + elif p.grad.is_sparse: + view = p.grad.to_dense().view(-1) + else: + view = p.grad.view(-1) + if torch.is_complex(view): + view = torch.view_as_real(view).view(-1) + views.append(view) + return torch.cat(views, 0) + + def _add_grad(self, step_size, update) -> None: + offset = 0 + for p in self._params: + if torch.is_complex(p): + p = torch.view_as_real(p) + numel = p.numel() + # view as to avoid deprecated pointwise semantics + p.add_(update[offset : offset + numel].view_as(p), alpha=step_size) + offset += numel + if offset != self._numel(): + raise AssertionError(f"Expected offset {offset} to equal {self._numel()}") + + def _clone_param(self): + return [p.clone(memory_format=torch.contiguous_format) for p in self._params] + + def _set_param(self, params_data) -> None: + for p, pdata in zip(self._params, params_data, strict=True): + p.copy_(pdata) + + def _directional_evaluate(self, closure, x, t, d): + self._add_grad(t, d) + loss = float(closure()) + flat_grad = self._gather_flat_grad() + self._set_param(x) + return loss, flat_grad + + @torch.no_grad() + def step(self, closure): # type: ignore[override] + """Perform a single optimization step. + + Args: + closure (Callable): A closure that reevaluates the model + and returns the loss. + """ + if len(self.param_groups) != 1: + raise AssertionError( + f"Expected exactly one param_group, but got {len(self.param_groups)}" + ) + + # Make sure the closure is always called with grad enabled + closure = torch.enable_grad()(closure) + + group = self.param_groups[0] + lr = _to_scalar(group["lr"]) + max_iter = group["max_iter"] + max_eval = group["max_eval"] + tolerance_grad = group["tolerance_grad"] + tolerance_change = group["tolerance_change"] + line_search_fn = group["line_search_fn"] + history_size = group["history_size"] + + # NOTE: LBFGS has only global state, but we register it as state for + # the first param, because this helps with casting in load_state_dict + state = self.state[self._params[0]] + state.setdefault("func_evals", 0) + state.setdefault("n_iter", 0) + + # evaluate initial f(x) and df/dx + orig_loss = closure() + loss = float(orig_loss) + current_evals = 1 + state["func_evals"] += 1 + + flat_grad = self._gather_flat_grad() + opt_cond = flat_grad.abs().max() <= tolerance_grad + + # optimal condition + if opt_cond: + return orig_loss + + # tensors cached in state (for tracing) + d = state.get("d") + t = state.get("t") + old_dirs = state.get("old_dirs") + old_stps = state.get("old_stps") + ro = state.get("ro") + H_diag = state.get("H_diag") + prev_flat_grad = state.get("prev_flat_grad") + prev_loss = state.get("prev_loss") + + n_iter = 0 + # optimize for a max of max_iter iterations + while n_iter < max_iter: + # keep track of nb of iterations + n_iter += 1 + state["n_iter"] += 1 + + ############################################################ + # compute gradient descent direction + ############################################################ + if state["n_iter"] == 1: + d = flat_grad.neg() + old_dirs = [] + old_stps = [] + ro = [] + H_diag = 1 + else: + # do lbfgs update (update memory) + y = flat_grad.sub(prev_flat_grad) + s = d.mul(t) + ys = y.dot(s) # y*s + if ys > 1e-10: + # updating memory + if len(old_dirs) == history_size: + # shift history by one (limited-memory) + old_dirs.pop(0) + old_stps.pop(0) + ro.pop(0) + + # store new direction/step + old_dirs.append(y) + old_stps.append(s) + ro.append(1.0 / ys) + + # update scale of initial Hessian approximation + H_diag = ys / y.dot(y) # (y*y) + + # compute the approximate (L-BFGS) inverse Hessian + # multiplied by the gradient + num_old = len(old_dirs) + + if "al" not in state: + state["al"] = [None] * history_size + al = state["al"] + + # iteration in L-BFGS loop collapsed to use just one buffer + q = flat_grad.neg() + for i in range(num_old - 1, -1, -1): + al[i] = old_stps[i].dot(q) * ro[i] + q.add_(old_dirs[i], alpha=-al[i]) + + # multiply by initial Hessian + # r/d is the final direction + d = r = torch.mul(q, H_diag) + for i in range(num_old): + be_i = old_dirs[i].dot(r) * ro[i] + r.add_(old_stps[i], alpha=al[i] - be_i) + + if prev_flat_grad is None: + prev_flat_grad = flat_grad.clone(memory_format=torch.contiguous_format) + else: + prev_flat_grad.copy_(flat_grad) + prev_loss = loss + + ############################################################ + # compute step length + ############################################################ + # reset initial guess for step size + if state["n_iter"] == 1: + t = min(1.0, 1.0 / flat_grad.abs().sum()) * lr + else: + t = lr + + # directional derivative + gtd = flat_grad.dot(d) # g * d + + # directional derivative is below tolerance + if gtd > -tolerance_change: + break + + # optional line search: user function + ls_func_evals = 0 + if line_search_fn is not None: + # perform line search, using user function + if line_search_fn != "strong_wolfe": + raise RuntimeError("only 'strong_wolfe' is supported") + else: + x_init = self._clone_param() + + def obj_func(x, t, d): + return self._directional_evaluate(closure, x, t, d) + + loss, flat_grad, t, ls_func_evals = _strong_wolfe( + obj_func, + x_init, + t, + d, + loss, + flat_grad, + gtd, + max_ls=max_eval - current_evals, + ) + self._add_grad(t, d) + opt_cond = flat_grad.abs().max() <= tolerance_grad + else: + # no line search, simply move with fixed-step + self._add_grad(t, d) + if n_iter != max_iter: + # re-evaluate function only if not in last iteration + # the reason we do this: in a stochastic setting, + # no use to re-evaluate that function here + with torch.enable_grad(): + loss = closure() + loss = float(loss) + flat_grad = self._gather_flat_grad() + opt_cond = flat_grad.abs().max() <= tolerance_grad + ls_func_evals = 1 + + # update func eval + current_evals += ls_func_evals + state["func_evals"] += ls_func_evals + + ############################################################ + # check conditions + ############################################################ + if n_iter == max_iter: + break + + if current_evals >= max_eval: + break + + # optimal condition + if opt_cond: + break + + # lack of progress + if d.mul(t).abs().max() <= tolerance_change: + break + + if abs(loss - prev_loss) < tolerance_change: + break + + state["d"] = d + state["t"] = t + state["old_dirs"] = old_dirs + state["old_stps"] = old_stps + state["ro"] = ro + state["H_diag"] = H_diag + state["prev_flat_grad"] = prev_flat_grad + state["prev_loss"] = prev_loss + + return orig_loss diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/lr_scheduler.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/lr_scheduler.py new file mode 100644 index 0000000000000000000000000000000000000000..208a182bb1770a72b2427b1cea81160e59bfbe59 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/lr_scheduler.py @@ -0,0 +1,2602 @@ +# mypy: allow-untyped-defs +r"""Learning Rate Scheduler.""" + +from __future__ import annotations + +import math +import types +import warnings +from bisect import bisect_right +from collections import Counter +from functools import partial, wraps +from typing import Any, cast, Literal, SupportsFloat, TYPE_CHECKING, TypedDict +from typing_extensions import override, Self +from weakref import ref + +from torch import inf, Tensor + +from .optimizer import _to_scalar, Optimizer + + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Sequence + + +__all__ = [ + "LambdaLR", + "MultiplicativeLR", + "StepLR", + "MultiStepLR", + "ConstantLR", + "LinearLR", + "ExponentialLR", + "SequentialLR", + "CosineAnnealingLR", + "ChainedScheduler", + "ReduceLROnPlateau", + "CyclicLR", + "CosineAnnealingWarmRestarts", + "OneCycleLR", + "PolynomialLR", + "LRScheduler", +] + +EPOCH_DEPRECATION_WARNING = ( + "The epoch parameter in `scheduler.step()` was not necessary and is being " + "deprecated where possible. Please use `scheduler.step()` to step the " + "scheduler. During the deprecation, if epoch is different from None, the " + "closed form is used instead of the new chainable form, where available. " + "Please open an issue if you are unable to replicate your use case: " + "https://github.com/pytorch/pytorch/issues/new/choose." +) + + +def _format_param(name: str, optimizer: Optimizer, param): + """Return correctly formatted lr/momentum for each param group.""" + + def _copy(_param): + return _param.clone() if isinstance(_param, Tensor) else _param + + if isinstance(param, (list, tuple)): + if len(param) != len(optimizer.param_groups): + raise ValueError( + f"{name} must have the same length as optimizer.param_groups. " + f"{name} has {len(param)} values, param_groups has {len(optimizer.param_groups)}." + ) + else: + param = [param] * len(optimizer.param_groups) + + return list(map(_copy, param)) + + +def _param_groups_val_list(optimizer: Optimizer, key: str) -> list[Any]: + """Create a list containing group[key] for each optimizer param_group. + Prevents aliasing when group[key] could be a Tensor. + Raises a KeyError when group[key] does not exist. + """ + return [ + group[key].clone() if isinstance(group[key], Tensor) else group[key] + for group in optimizer.param_groups + ] + + +def _update_param_group_val( + param_group: dict[str, Any], key: str, val: float | Tensor +) -> None: + """Set param_group[key] to val without aliasing or assignment when they're + both tensors. Raises a KeyError if param_group[key] does not exist. + """ + if isinstance(param_group[key], Tensor): + param_group[key].fill_(_to_scalar(val)) + else: + param_group[key] = val + + +class LRScheduler: + r"""Base class for all learning rate schedulers. + + Subclasses implement :meth:`get_lr` and optionally override :meth:`step` to + define scheduling behavior. + + Args: + optimizer (Optimizer): The optimizer this scheduler will adjust the + learning rates of. + last_epoch (int): Index of the last epoch seen by the scheduler. Use + ``-1`` (default) to initialize the scheduler. Only use a non-default + value when restoring this scheduler from a saved checkpoint. + + .. warning:: + Initializing a scheduler overwrites its optimizer's + ``param_group["lr"]``\s. When restoring a checkpoint, initialize the + scheduler **before** calling your optimizer's + :meth:`~torch.optim.Optimizer.load_state_dict` to avoid overwriting the + loaded learning rates. + """ + + _get_lr_called_within_step: bool = False + _is_initial: bool = False + + def __init__( + self, + optimizer: Optimizer, + last_epoch: int = -1, + ) -> None: # noqa: D107 + # Attach optimizer + if not isinstance(optimizer, Optimizer): + raise TypeError(f"{type(optimizer).__name__} is not an Optimizer") + self.optimizer = optimizer + + # Initialize epoch and base learning rates + if last_epoch == -1: + for group in optimizer.param_groups: + initial_lr = group["lr"] + if isinstance(initial_lr, Tensor): + initial_lr = initial_lr.clone() + group.setdefault("initial_lr", initial_lr) + else: + for i, group in enumerate(optimizer.param_groups): + if "initial_lr" not in group: + raise KeyError( + f"param 'initial_lr' is not specified in param_groups[{i}] when resuming scheduler with last_epoch >= 0.\n" + "This typically happens when:\n" + "1. You're trying to resume training from a checkpoint but haven't properly loaded the optimizer state\n" + "2. You're using last_epoch >= 0 for a fresh training run (not recommended)" + ) + self.base_lrs: list[float | Tensor] = _param_groups_val_list( + optimizer, "initial_lr" + ) + self.last_epoch = last_epoch + + # Following https://github.com/pytorch/pytorch/issues/20124 + # We would like to ensure that `lr_scheduler.step()` is called after + # `optimizer.step()` + def patch_track_step_called(opt: Optimizer): + if hasattr(opt.step, "_wrapped_by_lr_sched"): + # we've already patched + return opt.step + + def wrap_step(step_fn): + opt_ref = ref(self.optimizer) + func = step_fn.__func__ + + @wraps(func) + def wrapper(*args, **kwargs): + opt = opt_ref() + opt._opt_called = True # type: ignore[union-attr] + return func.__get__(opt, opt.__class__)(*args, **kwargs) + + wrapper._wrapped_by_lr_sched = True # type: ignore[attr-defined] + return wrapper + + opt.step = wrap_step(opt.step) # type: ignore[method-assign] + + patch_track_step_called(self.optimizer) + self._initial_step() + + def _initial_step(self) -> None: + """Initialize step counts and perform a step.""" + self._step_count = 0 + with _initial_mode(self): + self.step() + + def state_dict(self) -> dict[str, Any]: + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in ``self.__dict__`` which + is not the optimizer. + """ + return { + key: value for key, value in self.__dict__.items() if key != "optimizer" + } + + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state. + + Args: + state_dict (dict): scheduler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + self.__dict__.update(state_dict) + + def get_last_lr(self) -> list[float | Tensor]: + r"""Get the most recent learning rates computed by this scheduler. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates with entries + for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`, with the same types as + their ``group["lr"]``\s. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + # We always update self._last_lr with _param_groups_val_list, so it's a + # .clone() of the group["lr"]s. If we didn't do this, the user could + # corrupt their learning rates by modifying the outputs in place. + return self._last_lr + + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + raise NotImplementedError + + def step(self, epoch: int | None = None) -> None: + """Step the scheduler. + + Args: + epoch (int, optional): + .. deprecated:: 1.4 + If provided, sets :attr:`last_epoch` to ``epoch`` and uses + :meth:`_get_closed_form_lr` if it is available. This is not + universally supported. Use :meth:`step` without arguments + instead. + + .. note:: + Call this method after calling the optimizer's + :meth:`~torch.optim.Optimizer.step`. + """ + # Raise a warning if old pattern is detected + # https://github.com/pytorch/pytorch/issues/20124 + if self._step_count == 1: + if not hasattr(self.optimizer.step, "_wrapped_by_lr_sched"): + warnings.warn( + "Seems like `optimizer.step()` has been overridden after learning rate scheduler " + "initialization. Please, make sure to call `optimizer.step()` before " + "`lr_scheduler.step()`. See more details at " + "https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate", + UserWarning, + stacklevel=2, + ) + + # Just check if there were two first lr_scheduler.step() calls before optimizer.step() + elif not getattr(self.optimizer, "_opt_called", False): + warnings.warn( + "Detected call of `lr_scheduler.step()` before `optimizer.step()`. " + "In PyTorch 1.1.0 and later, you should call them in the opposite order: " + "`optimizer.step()` before `lr_scheduler.step()`. Failure to do this " + "will result in PyTorch skipping the first value of the learning rate schedule. " + "See more details at " + "https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate", + UserWarning, + stacklevel=2, + ) + + self._step_count += 1 + if epoch is not None: + warnings.warn(EPOCH_DEPRECATION_WARNING, UserWarning, stacklevel=2) + self._update_lr(epoch) + + def _update_lr(self, epoch: int | None = None) -> None: + with _enable_get_lr_call(self): + if epoch is None: + self.last_epoch += 1 + values = self.get_lr() + else: + self.last_epoch = epoch + if hasattr(self, "_get_closed_form_lr"): + values = cast(list[float | Tensor], self._get_closed_form_lr()) + else: + values = self.get_lr() + + for param_group, lr in zip(self.optimizer.param_groups, values, strict=True): + _update_param_group_val(param_group, "lr", lr) + + self._last_lr: list[float | Tensor] = _param_groups_val_list( + self.optimizer, "lr" + ) + + +def _warn_get_lr_called_within_step(lr_scheduler: LRScheduler) -> None: + if not lr_scheduler._get_lr_called_within_step: + warnings.warn( + "To get the last learning rate computed by the scheduler, " + "please use `get_last_lr()`.", + UserWarning, + stacklevel=2, + ) + + +# Including _LRScheduler for backwards compatibility +# Subclass instead of assign because we want __name__ of _LRScheduler to be _LRScheduler (assigning would make it LRScheduler). +class _LRScheduler(LRScheduler): + pass + + +class _enable_get_lr_call: + def __init__(self, o: LRScheduler) -> None: + self.o = o + + def __enter__(self) -> Self: + self.o._get_lr_called_within_step = True + return self + + def __exit__(self, type, value, traceback) -> None: + self.o._get_lr_called_within_step = False + + +class _initial_mode: + def __init__(self, o: LRScheduler) -> None: + self.o = o + + def __enter__(self): + self.o._is_initial = True + + def __exit__(self, type, value, traceback): + self.o._is_initial = False + + +class LambdaLR(LRScheduler): + """Sets the initial learning rate. + + The learning rate of each parameter group is set to the initial lr + times a given function. When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + lr_lambda (function or list): A function which computes a multiplicative + factor given an integer parameter epoch, or a list of such + functions, one for each group in optimizer.param_groups. + last_epoch (int): The index of last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer has two groups. + >>> num_epochs = 100 + >>> lambda1 = lambda epoch: epoch // 30 + >>> lambda2 = lambda epoch: 0.95**epoch + >>> scheduler = LambdaLR(optimizer, lr_lambda=[lambda1, lambda2]) + >>> for epoch in range(num_epochs): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + >>> + >>> # Alternatively, you can use a single lambda function for all groups. + >>> scheduler = LambdaLR(opt, lr_lambda=lambda epoch: epoch // 30) + >>> for epoch in range(num_epochs): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/LambdaLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + lr_lambda: Callable[[int], float] | list[Callable[[int], float]], + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.optimizer = optimizer + + self.lr_lambdas: list[Callable[[int], float]] + if not isinstance(lr_lambda, list) and not isinstance(lr_lambda, tuple): + self.lr_lambdas = [lr_lambda] * len(optimizer.param_groups) + else: + if len(lr_lambda) != len(optimizer.param_groups): + raise ValueError( + f"Expected {len(optimizer.param_groups)} lr_lambdas, but got {len(lr_lambda)}" + ) + self.lr_lambdas = list(lr_lambda) + super().__init__(optimizer, last_epoch) + + @override + def state_dict(self) -> dict[str, Any]: + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in ``self.__dict__`` which is not the optimizer. + The learning rate lambda functions will only be saved if they are callable objects + and not if they are functions or lambdas. + + When saving or loading the scheduler, please make sure to also save or load the state of the optimizer. + """ + state_dict = { + key: value + for key, value in self.__dict__.items() + if key not in ("optimizer", "lr_lambdas") + } + state_dict["lr_lambdas"] = [None] * len(self.lr_lambdas) + + for idx, fn in enumerate(self.lr_lambdas): + if not isinstance(fn, types.FunctionType): + # pyrefly: ignore [unsupported-operation] + state_dict["lr_lambdas"][idx] = fn.__dict__.copy() + + return state_dict + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state. + + When saving or loading the scheduler, please make sure to also save or load the state of the optimizer. + + Args: + state_dict (dict): scheduler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + lr_lambdas = state_dict.pop("lr_lambdas") + self.__dict__.update(state_dict) + # Restore state_dict keys in order to prevent side effects + # https://github.com/pytorch/pytorch/issues/32756 + state_dict["lr_lambdas"] = lr_lambdas + + for idx, fn in enumerate(lr_lambdas): + if fn is not None: + self.lr_lambdas[idx].__dict__.update(fn) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Scales the :attr:`base_lrs` by the outputs of the :attr:`lr_lambdas` at + :attr:`last_epoch`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + return [ + base_lr * lmbda(self.last_epoch) + for lmbda, base_lr in zip(self.lr_lambdas, self.base_lrs, strict=True) + ] + + +class MultiplicativeLR(LRScheduler): + """Multiply the learning rate of each parameter group by the factor given in the specified function. + + When last_epoch=-1, set initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + lr_lambda (function or list): A function which computes a multiplicative + factor given an integer parameter epoch, or a list of such + functions, one for each group in optimizer.param_groups. + last_epoch (int): The index of last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> lmbda = lambda epoch: 0.95 + >>> scheduler = MultiplicativeLR(optimizer, lr_lambda=lmbda) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/MultiplicativeLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + lr_lambda: Callable[[int], float] | list[Callable[[int], float]], + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.optimizer = optimizer + + self.lr_lambdas: list[Callable[[int], float]] + if not isinstance(lr_lambda, list) and not isinstance(lr_lambda, tuple): + self.lr_lambdas = [lr_lambda] * len(optimizer.param_groups) + else: + if len(lr_lambda) != len(optimizer.param_groups): + raise ValueError( + f"Expected {len(optimizer.param_groups)} lr_lambdas, but got {len(lr_lambda)}" + ) + self.lr_lambdas = list(lr_lambda) + for lr_lambda in self.lr_lambdas: + if not callable(lr_lambda): + raise TypeError( + f"lr_lambda should be a function, but got {type(lr_lambda).__name__}" + ) + super().__init__(optimizer, last_epoch) + + @override + def state_dict(self) -> dict[str, Any]: + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in ``self.__dict__`` which + is not the optimizer. + The learning rate lambda functions will only be saved if they are callable objects + and not if they are functions or lambdas. + """ + state_dict = { + key: value + for key, value in self.__dict__.items() + if key not in ("optimizer", "lr_lambdas") + } + state_dict["lr_lambdas"] = [None] * len(self.lr_lambdas) + + for idx, fn in enumerate(self.lr_lambdas): + if not isinstance(fn, types.FunctionType): + # pyrefly: ignore [unsupported-operation] + state_dict["lr_lambdas"][idx] = fn.__dict__.copy() + + return state_dict + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state. + + Args: + state_dict (dict): scheduler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + lr_lambdas = state_dict.pop("lr_lambdas") + self.__dict__.update(state_dict) + # Restore state_dict keys in order to prevent side effects + # https://github.com/pytorch/pytorch/issues/32756 + state_dict["lr_lambdas"] = lr_lambdas + + for idx, fn in enumerate(lr_lambdas): + if fn is not None: + self.lr_lambdas[idx].__dict__.update(fn) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Scales the current ``group["lr"]``\s in each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` by the outputs of the + :attr:`lr_lambdas` at :attr:`last_epoch`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + if not self._is_initial: + return [ + group["lr"] * lmbda(self.last_epoch) + for lmbda, group in zip( + self.lr_lambdas, self.optimizer.param_groups, strict=True + ) + ] + else: + return _param_groups_val_list(self.optimizer, "lr") + + +class StepLR(LRScheduler): + """Decays the learning rate of each parameter group by gamma every step_size epochs. + + Notice that such decay can happen simultaneously with other changes to the learning rate + from outside this scheduler. When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + step_size (int): Period of learning rate decay. + gamma (float): Multiplicative factor of learning rate decay. + Default: 0.1. + last_epoch (int): The index of last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.05 if epoch < 30 + >>> # lr = 0.005 if 30 <= epoch < 60 + >>> # lr = 0.0005 if 60 <= epoch < 90 + >>> # ... + >>> scheduler = StepLR(optimizer, step_size=30, gamma=0.1) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/StepLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + step_size: int, + gamma: float = 0.1, + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.step_size = step_size + self.gamma = gamma + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + If the current epoch is a non-zero multiple of :attr:`step_size`, we + scale the current ``group["lr"]``\s in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` by :attr:`gamma`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + if (self.last_epoch == 0) or (self.last_epoch % self.step_size != 0): + return _param_groups_val_list(self.optimizer, "lr") + return [group["lr"] * self.gamma for group in self.optimizer.param_groups] + + def _get_closed_form_lr(self) -> list[float | Tensor]: + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + return [ + base_lr * self.gamma ** (self.last_epoch // self.step_size) + for base_lr in self.base_lrs + ] + + +class MultiStepLR(LRScheduler): + """Decays the learning rate of each parameter group by gamma once the number of epoch reaches one of the milestones. + + Notice that such decay can happen simultaneously with other changes to the learning rate + from outside this scheduler. When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + milestones (list): List of epoch indices. Must be increasing. + gamma (float): Multiplicative factor of learning rate decay. + Default: 0.1. + last_epoch (int): The index of last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.05 if epoch < 30 + >>> # lr = 0.005 if 30 <= epoch < 80 + >>> # lr = 0.0005 if epoch >= 80 + >>> scheduler = MultiStepLR(optimizer, milestones=[30, 80], gamma=0.1) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/MultiStepLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + milestones: Iterable[int], + gamma: float = 0.1, + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.milestones = Counter(milestones) + self.gamma = gamma + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + If the current epoch is in :attr:`milestones`, decays the + ``group["lr"]``\s in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` by :attr:`gamma`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + + .. note:: + If the current epoch appears in :attr:`milestones` ``n`` times, we + scale by :attr:`gamma` to the power of ``n`` + """ + _warn_get_lr_called_within_step(self) + + if self.last_epoch not in self.milestones: + return _param_groups_val_list(self.optimizer, "lr") + return [ + group["lr"] * self.gamma ** self.milestones[self.last_epoch] + for group in self.optimizer.param_groups + ] + + def _get_closed_form_lr(self): + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + milestones = sorted(self.milestones.elements()) + return [ + base_lr * self.gamma ** bisect_right(milestones, self.last_epoch) + for base_lr in self.base_lrs + ] + + +class ConstantLR(LRScheduler): + """Multiply the learning rate of each parameter group by a small constant factor. + + The multiplication is done until the number of epoch reaches a pre-defined milestone: total_iters. + Notice that such multiplication of the small constant factor can + happen simultaneously with other changes to the learning rate from outside this scheduler. + When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + factor (float): The number we multiply learning rate until the milestone. Default: 1./3. + total_iters (int): The number of steps that the scheduler multiplies the learning rate by the factor. + Default: 5. + last_epoch (int): The index of the last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.025 if epoch == 0 + >>> # lr = 0.025 if epoch == 1 + >>> # lr = 0.025 if epoch == 2 + >>> # lr = 0.025 if epoch == 3 + >>> # ... + >>> # lr = 0.05 if epoch >= 40 + >>> scheduler = ConstantLR(optimizer, factor=0.5, total_iters=40) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/ConstantLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + factor: float = 1.0 / 3, + total_iters: int = 5, + last_epoch: int = -1, + ) -> None: # noqa: D107 + if factor > 1.0 or factor < 0: + raise ValueError( + "Constant multiplicative factor expected to be between 0 and 1." + ) + + self.factor = factor + self.total_iters = total_iters + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + When :attr:`last_epoch` is 0, this method scales the ``group["lr"]``\s + in each of the optimizer's :attr:`~torch.optim.Optimizer.param_groups` + by :attr:`factor`. Once :attr:`total_iters` is reached, it undoes this, + scaling by ``1 / factor``. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + if self.last_epoch == 0: + return [group["lr"] * self.factor for group in self.optimizer.param_groups] + + if self.last_epoch != self.total_iters: + return _param_groups_val_list(self.optimizer, "lr") + + return [ + group["lr"] * (1.0 / self.factor) for group in self.optimizer.param_groups + ] + + def _get_closed_form_lr(self): + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + return [ + base_lr + * (self.factor + (self.last_epoch >= self.total_iters) * (1 - self.factor)) + for base_lr in self.base_lrs + ] + + +class LinearLR(LRScheduler): + """Decays the learning rate of each parameter group by linearly changing small multiplicative factor. + + The multiplication is done until the number of epoch reaches a pre-defined milestone: total_iters. + Notice that such decay can happen simultaneously with other changes to the learning rate + from outside this scheduler. When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + start_factor (float): The number we multiply learning rate in the first epoch. + The multiplication factor changes towards end_factor in the following epochs. + Default: 1./3. + end_factor (float): The number we multiply learning rate at the end of linear changing + process. Default: 1.0. + total_iters (int): The number of iterations that multiplicative factor reaches to 1. + Default: 5. + last_epoch (int): The index of the last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.003687 if epoch == 0 + >>> # lr = 0.004875 if epoch == 1 + >>> # lr = 0.006062 if epoch == 2 + >>> # lr = 0.00725 if epoch == 3 + >>> # ... + >>> # lr = 0.05 if epoch >= 40 + >>> scheduler = LinearLR(optimizer, start_factor=0.05, total_iters=40) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/LinearLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + start_factor: float = 1.0 / 3, + end_factor: float = 1.0, + total_iters: int = 5, + last_epoch: int = -1, + ) -> None: # noqa: D107 + if start_factor > 1.0 or start_factor <= 0: + raise ValueError( + "Starting multiplicative factor expected to be greater than 0 and less or equal to 1." + ) + + if end_factor > 1.0 or end_factor < 0: + raise ValueError( + "Ending multiplicative factor expected to be between 0 and 1." + ) + + self.start_factor = start_factor + self.end_factor = end_factor + self.total_iters = total_iters + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Scales the ``group["lr"]``\s in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` such that successive steps + interpolate linearly from :attr:`start_factor` up to :attr:`end_factor` + across :attr:`total_iters` steps. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + if self.last_epoch == 0: + return [ + group["lr"] * self.start_factor for group in self.optimizer.param_groups + ] + + if self._is_initial or self.last_epoch > self.total_iters: + return _param_groups_val_list(self.optimizer, "lr") + + return [ + group["lr"] + * ( + 1.0 + + (self.end_factor - self.start_factor) + / ( + self.total_iters * self.start_factor + + (self.last_epoch - 1) * (self.end_factor - self.start_factor) + ) + ) + for group in self.optimizer.param_groups + ] + + def _get_closed_form_lr(self): + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + return [ + base_lr + * ( + self.start_factor + + (self.end_factor - self.start_factor) + * min(self.total_iters, self.last_epoch) + / self.total_iters + ) + for base_lr in self.base_lrs + ] + + +class ExponentialLR(LRScheduler): + """Decays the learning rate of each parameter group by gamma every epoch. + + When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + gamma (float): Multiplicative factor of learning rate decay. + last_epoch (int): The index of last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> scheduler = ExponentialLR(optimizer, gamma=0.95) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/ExponentialLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + gamma: float, + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.gamma = gamma + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Multiplies the current ``group["lr"]``\s in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` by :attr:`gamma`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + # when loading from a checkpoint, we don't want _initial_step (called from the constructor) + # to update the lr one more step ahead of itself. + if self._is_initial: + return _param_groups_val_list(self.optimizer, "lr") + return [group["lr"] * self.gamma for group in self.optimizer.param_groups] + + def _get_closed_form_lr(self): + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + return [base_lr * self.gamma**self.last_epoch for base_lr in self.base_lrs] + + +class SequentialLR(LRScheduler): + """Contains a list of schedulers expected to be called sequentially during the optimization process. + + Specifically, the schedulers will be called according to the milestone points, which should provide exact + intervals by which each scheduler should be called at a given epoch. + + Args: + optimizer (Optimizer): Wrapped optimizer. + schedulers (list): List of chained schedulers. + milestones (list): List of integers that reflects milestone points. + last_epoch (int): The index of last epoch. Default: -1. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.005 if epoch == 0 + >>> # lr = 0.005 if epoch == 1 + >>> # lr = 0.005 if epoch == 2 + >>> # ... + >>> # lr = 0.05 if epoch == 20 + >>> # lr = 0.045 if epoch == 21 + >>> # lr = 0.0405 if epoch == 22 + >>> scheduler1 = ConstantLR(optimizer, factor=0.1, total_iters=20) + >>> scheduler2 = ExponentialLR(optimizer, gamma=0.9) + >>> scheduler = SequentialLR( + ... optimizer, + ... schedulers=[scheduler1, scheduler2], + ... milestones=[20], + ... ) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/SequentialLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + schedulers: list[LRScheduler], + milestones: list[int], + last_epoch: int = -1, + ) -> None: # noqa: D107 + if len(schedulers) < 1: + raise ValueError( + f"{self.__class__.__name__} expects at least one scheduler, but got no scheduler." + ) + + for scheduler_idx, scheduler in enumerate(schedulers): + if not hasattr(scheduler, "optimizer"): + raise TypeError( + f"{self.__class__.__name__} at index {scheduler_idx} should have `optimizer` as its attribute." + ) + if isinstance(scheduler, ReduceLROnPlateau): + raise ValueError( + f"{self.__class__.__name__} does not support `ReduceLROnPlateau` scheduler as it " + "requires additional kwargs to be specified when calling `step`, " + f"but got one at index {scheduler_idx} in the given schedulers sequence." + ) + if optimizer != scheduler.optimizer: + raise ValueError( + f"{self.__class__.__name__} expects all schedulers to belong to the same optimizer, but " + f"got scheduler {scheduler.__class__.__name__} at index {scheduler_idx} has {scheduler.optimizer}, " + f"which is different from {optimizer.__class__.__name__}." + ) + + if len(milestones) != len(schedulers) - 1: + raise ValueError( + "Sequential Schedulers expects number of schedulers provided to be one more " + f"than the number of milestone points, but got number of schedulers {len(schedulers)} and the " + f"number of milestones to be equal to {len(milestones)}" + ) + self._schedulers = schedulers + self._milestones = milestones + self.last_epoch = last_epoch + 1 + self.optimizer = optimizer + + # Reset learning rates back to initial values + for group in self.optimizer.param_groups: + _update_param_group_val(group, "lr", group["initial_lr"]) + + # "Undo" the step performed by other schedulers + self.recursive_undo() + + # Perform the initial step for only the first scheduler + self._schedulers[0]._initial_step() + + self._last_lr = schedulers[0].get_last_lr() + + def recursive_undo(self, sched=None) -> None: + """ + Recursively undo any step performed by the initialisation of + schedulers. + """ + scheds = self if sched is None else sched + + if hasattr(scheds, "_schedulers"): + for s in scheds._schedulers: + self.recursive_undo(s) + elif hasattr(scheds, "last_epoch"): + scheds.last_epoch -= 1 + + def step(self) -> None: # type: ignore[override] + """Perform a step.""" + self.last_epoch += 1 + idx = bisect_right(self._milestones, self.last_epoch) + scheduler = self._schedulers[idx] + if idx > 0 and self._milestones[idx - 1] == self.last_epoch: + scheduler._update_lr(0) + else: + scheduler.step() + + self._last_lr = scheduler.get_last_lr() + + @override + def state_dict(self) -> dict[str, Any]: + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in ``self.__dict__`` which + is not the optimizer. + The wrapped scheduler states will also be saved. + """ + state_dict = { + key: value + for key, value in self.__dict__.items() + if key not in ("optimizer", "_schedulers") + } + state_dict["_schedulers"] = [None] * len(self._schedulers) + + for idx, s in enumerate(self._schedulers): + # pyrefly: ignore [unsupported-operation] + state_dict["_schedulers"][idx] = s.state_dict() + + return state_dict + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state. + + Args: + state_dict (dict): scheduler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + _schedulers = state_dict.pop("_schedulers") + self.__dict__.update(state_dict) + # Restore state_dict keys in order to prevent side effects + # https://github.com/pytorch/pytorch/issues/32756 + state_dict["_schedulers"] = _schedulers + + for idx, s in enumerate(_schedulers): + self._schedulers[idx].load_state_dict(s) + + +class PolynomialLR(LRScheduler): + """Decays the learning rate of each parameter group using a polynomial function in the given total_iters. + + When last_epoch=-1, sets initial lr as lr. + + Args: + optimizer (Optimizer): Wrapped optimizer. + total_iters (int): The number of steps that the scheduler decays the learning rate. Default: 5. + power (float): The power of the polynomial. Default: 1.0. + + Example: + >>> # xdoctest: +SKIP("undefined vars") + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.0490 if epoch == 0 + >>> # lr = 0.0481 if epoch == 1 + >>> # lr = 0.0472 if epoch == 2 + >>> # ... + >>> # lr = 0.0 if epoch >= 50 + >>> scheduler = PolynomialLR(optimizer, total_iters=50, power=0.9) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/PolynomialLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + total_iters: int = 5, + power: float = 1.0, + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.total_iters = total_iters + self.power = power + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Scales the ``group["lr"]``\s in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` such that the learning rates + follow + + .. math:: + \texttt{base\_lr} \cdot \left(1 - \frac{\texttt{last\_epoch}} + {\texttt{total\_iters}} \right)^\texttt{power} + + Returns the current learning rates unchanged after :attr:`total_iters` + is reached. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + if self._is_initial or self.last_epoch > self.total_iters: + return _param_groups_val_list(self.optimizer, "lr") + + decay_factor = ( + (1.0 - self.last_epoch / self.total_iters) + / (1.0 - (self.last_epoch - 1) / self.total_iters) + ) ** self.power + return [group["lr"] * decay_factor for group in self.optimizer.param_groups] + + def _get_closed_form_lr(self) -> list[float | Tensor]: + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + return [ + ( + base_lr + * (1.0 - min(self.total_iters, self.last_epoch) / self.total_iters) + ** self.power + ) + for base_lr in self.base_lrs + ] + + +class CosineAnnealingLR(LRScheduler): + r""" + Set the learning rate of each parameter group using a cosine annealing schedule. + + The learning rate is updated recursively using: + + .. math:: + \eta_{t+1} = \eta_{\min} + (\eta_t - \eta_{\min}) \cdot + \frac{1 + \cos\left(\frac{(T_{cur}+1) \pi}{T_{max}}\right)} + {1 + \cos\left(\frac{T_{cur} \pi}{T_{max}}\right)} + + This implements a recursive approximation of the closed-form schedule proposed in + `SGDR: Stochastic Gradient Descent with Warm Restarts`_: + + .. math:: + \eta_t = \eta_{\min} + \frac{1}{2}(\eta_{\max} - \eta_{\min}) \left( + 1 + \cos\left(\frac{T_{cur} \pi}{T_{max}}\right) \right) + + where: + + - :math:`\eta_t` is the learning rate at step :math:`t` + - :math:`T_{cur}` is the number of epochs since the last restart + - :math:`T_{max}` is the maximum number of epochs in a cycle + + Note: + Although SGDR includes periodic restarts, this implementation performs cosine annealing + **without restarts**, so :math:`T_{cur} = t` and increases monotonically with each call + to :meth:`step`. + + Args: + optimizer (Optimizer): Wrapped optimizer. + T_max (int): Maximum number of iterations. + eta_min (float): Minimum learning rate. Default: 0. + last_epoch (int): The index of the last epoch. Default: -1. + + .. _SGDR\: Stochastic Gradient Descent with Warm Restarts: + https://arxiv.org/abs/1608.03983 + + Example: + >>> # xdoctest: +SKIP + >>> num_epochs = 100 + >>> scheduler = CosineAnnealingLR(optimizer, T_max=num_epochs) + >>> for epoch in range(num_epochs): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/CosineAnnealingLR.png + """ + + def __init__( + self, + optimizer: Optimizer, + T_max: int, + eta_min: float = 0.0, + last_epoch: int = -1, + ) -> None: # noqa: D107 + self.T_max = T_max + self.eta_min = eta_min + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Scales the ``group["lr"]``\s in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` such that their learning + rates approximate + + .. math:: + \texttt{eta\_min} + \frac{1}{2} (\texttt{base\_lr} - + \texttt{eta\_min}) \left(1 + \cos\left(\pi \cdot + \frac{\texttt{last\_epoch}}{\texttt{T\_max}}\right) \right) + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + if self._is_initial: + return _param_groups_val_list(self.optimizer, "lr") + elif self._step_count == 1 and self.last_epoch > 0: + return [ + self.eta_min + + (base_lr - self.eta_min) + * (1 + math.cos((self.last_epoch) * math.pi / self.T_max)) + / 2 + for base_lr, group in zip( + self.base_lrs, self.optimizer.param_groups, strict=True + ) + ] + elif (self.last_epoch - 1 - self.T_max) % (2 * self.T_max) == 0: + return [ + group["lr"] + + (base_lr - self.eta_min) * (1 - math.cos(math.pi / self.T_max)) / 2 + for base_lr, group in zip( + self.base_lrs, self.optimizer.param_groups, strict=True + ) + ] + return [ + (1 + math.cos(math.pi * self.last_epoch / self.T_max)) + / (1 + math.cos(math.pi * (self.last_epoch - 1) / self.T_max)) + * (group["lr"] - self.eta_min) + + self.eta_min + for group in self.optimizer.param_groups + ] + + def _get_closed_form_lr(self) -> list[float | Tensor]: + r"""Compute learning rates for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` at :attr:`last_epoch` using + a closed-form formula. + + Uses :attr:`base_lrs` to compute learning rates. This method is called + when an epoch is passed to :meth:`step`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + """ + return [ + self.eta_min + + (base_lr - self.eta_min) + * (1 + math.cos(math.pi * self.last_epoch / self.T_max)) + / 2 + for base_lr in self.base_lrs + ] + + +class ChainedScheduler(LRScheduler): + """Chains a list of learning rate schedulers. + + Takes in a sequence of chainable learning rate schedulers and calls their + step() functions consecutively in just one call to step(). + + Args: + schedulers (sequence): sequence of chained schedulers. + optimizer (Optimizer, optional): Wrapped optimizer. Default: None. + + Example: + >>> # xdoctest: +SKIP + >>> # Assuming optimizer uses lr = 0.05 for all groups + >>> # lr = 0.05 if epoch == 0 + >>> # lr = 0.0450 if epoch == 1 + >>> # lr = 0.0405 if epoch == 2 + >>> # ... + >>> # lr = 0.00675 if epoch == 19 + >>> # lr = 0.06078 if epoch == 20 + >>> # lr = 0.05470 if epoch == 21 + >>> scheduler1 = ConstantLR(optimizer, factor=0.1, total_iters=20) + >>> scheduler2 = ExponentialLR(optimizer, gamma=0.9) + >>> scheduler = ChainedScheduler([scheduler1, scheduler2], optimizer=optimizer) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/ChainedScheduler.png + """ + + def __init__( + self, schedulers: Sequence[LRScheduler], optimizer: Optimizer | None = None + ) -> None: # noqa: D107 + if len(schedulers) < 1: + raise ValueError( + f"{self.__class__.__name__} expects at least one scheduler to be chained, but got no scheduler." + ) + + optimizer = optimizer or schedulers[0].optimizer + for scheduler_idx, scheduler in enumerate(schedulers): + if not hasattr(scheduler, "optimizer"): + raise TypeError( + f"{self.__class__.__name__} at index {scheduler_idx} should have `optimizer` as its attribute." + ) + if isinstance(scheduler, ReduceLROnPlateau): + raise ValueError( + f"{self.__class__.__name__} does not support `ReduceLROnPlateau` scheduler as it " + "requires additional kwargs to be specified when calling `step`, " + f"but got one at index {scheduler_idx} in the given schedulers sequence." + ) + if optimizer != scheduler.optimizer: + raise ValueError( + f"{self.__class__.__name__} expects all schedulers to belong to the same optimizer, but " + f"got scheduler {scheduler.__class__.__name__} at index {scheduler_idx} has {scheduler.optimizer}, " + f"which is different from {optimizer.__class__.__name__}." + ) + self._schedulers = schedulers + self.optimizer = optimizer + self._last_lr = _param_groups_val_list(self._schedulers[-1].optimizer, "lr") + + def step(self) -> None: # type: ignore[override] + """Perform a step.""" + for scheduler in self._schedulers: + scheduler.step() + self._last_lr = _param_groups_val_list(self._schedulers[-1].optimizer, "lr") + + @override + def state_dict(self) -> dict[str, Any]: + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in ``self.__dict__`` which + is not the optimizer. + The wrapped scheduler states will also be saved. + """ + state_dict = { + key: value + for key, value in self.__dict__.items() + if key not in ("optimizer", "_schedulers") + } + state_dict["_schedulers"] = [None] * len(self._schedulers) + + for idx, s in enumerate(self._schedulers): + # pyrefly: ignore [unsupported-operation] + state_dict["_schedulers"][idx] = s.state_dict() + + return state_dict + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state. + + Args: + state_dict (dict): scheduler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + _schedulers = state_dict.pop("_schedulers") + self.__dict__.update(state_dict) + # Restore state_dict keys in order to prevent side effects + # https://github.com/pytorch/pytorch/issues/32756 + state_dict["_schedulers"] = _schedulers + + for idx, s in enumerate(_schedulers): + self._schedulers[idx].load_state_dict(s) + + +class ReduceLROnPlateau(LRScheduler): + """Reduce learning rate when a metric has stopped improving. + + Models often benefit from reducing the learning rate by a factor + of 2-10 once learning stagnates. This scheduler reads a metrics + quantity and if no improvement is seen for a 'patience' number + of epochs, the learning rate is reduced. + + Args: + optimizer (Optimizer): Wrapped optimizer. + mode (str): One of `min`, `max`. In `min` mode, lr will + be reduced when the quantity monitored has stopped + decreasing; in `max` mode it will be reduced when the + quantity monitored has stopped increasing. Default: 'min'. + factor (float): Factor by which the learning rate will be + reduced. new_lr = lr * factor. Default: 0.1. + patience (int): The number of allowed epochs with no improvement after + which the learning rate will be reduced. + For example, consider the case of having no patience (`patience = 0`). + In the first epoch, a baseline is established and is always considered good as there's no previous baseline. + In the second epoch, if the performance is worse than the baseline, + we have what is considered an intolerable epoch. + Since the count of intolerable epochs (1) is greater than the patience level (0), + the learning rate is reduced at the end of this epoch. + From the third epoch onwards, the learning rate continues to be reduced at the end of each epoch + if the performance is worse than the baseline. If the performance improves or remains the same, + the learning rate is not adjusted. + Default: 10. + threshold (float): Threshold for measuring the new optimum, + to only focus on significant changes. Default: 1e-4. + threshold_mode (str): One of `rel`, `abs`. In `rel` mode, + dynamic_threshold = best * ( 1 + threshold ) in 'max' + mode or best * ( 1 - threshold ) in `min` mode. + In `abs` mode, dynamic_threshold = best + threshold in + `max` mode or best - threshold in `min` mode. Default: 'rel'. + cooldown (int): Number of epochs to wait before resuming + normal operation after lr has been reduced. Default: 0. + min_lr (float or list): A scalar or a list of scalars. A + lower bound on the learning rate of all param groups + or each group respectively. Default: 0. + eps (float): Minimal decay applied to lr. If the difference + between new and old lr is smaller than eps, the update is + ignored. Default: 1e-8. + + Example: + >>> # xdoctest: +SKIP + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> scheduler = ReduceLROnPlateau(optimizer, "min") + >>> for epoch in range(10): + >>> train(...) + >>> val_loss = validate(...) + >>> # Note that step should be called after validate() + >>> scheduler.step(val_loss) + + .. image:: ../scripts/lr_scheduler_images/ReduceLROnPlateau.png + """ + + def __init__( + self, + optimizer: Optimizer, + mode: Literal["min", "max"] = "min", + factor: float = 0.1, + patience: int = 10, + threshold: float = 1e-4, + threshold_mode: Literal["rel", "abs"] = "rel", + cooldown: int = 0, + min_lr: list[float] | float = 0, + eps: float = 1e-8, + ) -> None: # noqa: D107 + if factor >= 1.0: + raise ValueError("Factor should be < 1.0.") + self.factor = factor + + # Attach optimizer + if not isinstance(optimizer, Optimizer): + raise TypeError(f"{type(optimizer).__name__} is not an Optimizer") + self.optimizer = optimizer + + if isinstance(min_lr, (list, tuple)): + if len(min_lr) != len(optimizer.param_groups): + raise ValueError( + f"expected {len(optimizer.param_groups)} min_lrs, got {len(min_lr)}" + ) + self.default_min_lr = None + self.min_lrs = list(min_lr) + else: + # pyrefly: ignore [bad-assignment] + self.default_min_lr = min_lr + self.min_lrs = [min_lr] * len(optimizer.param_groups) + + self.patience = patience + self.cooldown = cooldown + self.eps = eps + self.last_epoch = 0 + self._last_lr = _param_groups_val_list(self.optimizer, "lr") + self._init_is_better( + mode=mode, threshold=threshold, threshold_mode=threshold_mode + ) + self._reset() + + def _reset(self) -> None: + """Reset num_bad_epochs counter and cooldown counter.""" + self.best = self.mode_worse + self.cooldown_counter = 0 + self.num_bad_epochs = 0 + + def step(self, metrics: SupportsFloat, epoch=None) -> None: # type: ignore[override] + """Perform a step.""" + # convert `metrics` to float, in case it's a zero-dim Tensor + current = float(metrics) + if epoch is None: + epoch = self.last_epoch + 1 + else: + warnings.warn(EPOCH_DEPRECATION_WARNING, UserWarning, stacklevel=2) + self.last_epoch = epoch + + if self._is_better(current, self.best): + self.best = current + self.num_bad_epochs = 0 + else: + self.num_bad_epochs += 1 + + if self.in_cooldown: + self.cooldown_counter -= 1 + self.num_bad_epochs = 0 # ignore any bad epochs in cooldown + + if self.num_bad_epochs > self.patience: + self._reduce_lr(epoch) + self.cooldown_counter = self.cooldown + self.num_bad_epochs = 0 + + self._last_lr = _param_groups_val_list(self.optimizer, "lr") + + def _reduce_lr(self, epoch) -> None: + if len(self.optimizer.param_groups) != len(self.min_lrs): + if self.default_min_lr is None: + raise RuntimeError( + "The number of param groups in the `optimizer` " + f"({len(self.optimizer.param_groups)}) differs " + f"from when `ReduceLROnPlateau` was initialized " + f"({len(self.min_lrs)}), usually due to a new " + "param group being added to the optimizer. Please " + "modify the `min_lrs` field to match the length " + "of the `optimizer` param groups." + ) + else: + # pyrefly: ignore [bad-assignment] + self.min_lrs = [self.default_min_lr] * len(self.optimizer.param_groups) + + for i, param_group in enumerate(self.optimizer.param_groups): + old_lr = float(param_group["lr"]) + new_lr = max(old_lr * self.factor, self.min_lrs[i]) + if old_lr - new_lr > self.eps: + _update_param_group_val(param_group, "lr", new_lr) + + @property + def in_cooldown(self): # noqa: D102 + return self.cooldown_counter > 0 + + def _is_better(self, a, best): # noqa: D102 + if self.mode == "min" and self.threshold_mode == "rel": + rel_epsilon = 1.0 - self.threshold + return a < best * rel_epsilon + + elif self.mode == "min" and self.threshold_mode == "abs": + return a < best - self.threshold + + elif self.mode == "max" and self.threshold_mode == "rel": + rel_epsilon = self.threshold + 1.0 + return a > best * rel_epsilon + + else: # mode == 'max' and epsilon_mode == 'abs': + return a > best + self.threshold + + def _init_is_better(self, mode, threshold, threshold_mode) -> None: + if mode not in {"min", "max"}: + raise ValueError("mode " + mode + " is unknown!") + if threshold_mode not in {"rel", "abs"}: + raise ValueError("threshold mode " + threshold_mode + " is unknown!") + + # the worse value for the chosen mode + if mode == "min": + self.mode_worse = inf + else: # mode == 'max': + self.mode_worse = -inf + + self.mode = mode + self.threshold = threshold + self.threshold_mode = threshold_mode + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state.""" + self.__dict__.update(state_dict) + self._init_is_better( + mode=self.mode, threshold=self.threshold, threshold_mode=self.threshold_mode + ) + + +class CyclicLR(LRScheduler): + r"""Sets the learning rate of each parameter group according to cyclical learning rate policy (CLR). + + The policy cycles the learning rate between two boundaries with a constant frequency, + as detailed in the paper `Cyclical Learning Rates for Training Neural Networks`_. + The distance between the two boundaries can be scaled on a per-iteration + or per-cycle basis. + + Cyclical learning rate policy changes the learning rate after every batch. + `step` should be called after a batch has been used for training. + + This class has three built-in policies, as put forth in the paper: + + * "triangular": A basic triangular cycle without amplitude scaling. + * "triangular2": A basic triangular cycle that scales initial amplitude by half each cycle. + * "exp_range": A cycle that scales initial amplitude by :math:`\text{gamma}^{\text{cycle iterations}}` + at each cycle iteration. + + This implementation was adapted from the github repo: `bckenstler/CLR`_ + + Args: + optimizer (Optimizer): Wrapped optimizer. + base_lr (float or list): Initial learning rate which is the + lower boundary in the cycle for each parameter group. + max_lr (float or list): Upper learning rate boundaries in the cycle + for each parameter group. Functionally, + it defines the cycle amplitude (max_lr - base_lr). + The lr at any cycle is the sum of base_lr + and some scaling of the amplitude; therefore + max_lr may not actually be reached depending on + scaling function. + step_size_up (int): Number of training iterations in the + increasing half of a cycle. Default: 2000 + step_size_down (int): Number of training iterations in the + decreasing half of a cycle. If step_size_down is None, + it is set to step_size_up. Default: None + mode (str): One of {triangular, triangular2, exp_range}. + Values correspond to policies detailed above. + If scale_fn is not None, this argument is ignored. + Default: 'triangular' + gamma (float): Constant in 'exp_range' scaling function: + gamma**(cycle iterations) + Default: 1.0 + scale_fn (function): Custom scaling policy defined by a single + argument lambda function, where + 0 <= scale_fn(x) <= 1 for all x >= 0. + If specified, then 'mode' is ignored. + Default: None + scale_mode (str): {'cycle', 'iterations'}. + Defines whether scale_fn is evaluated on + cycle number or cycle iterations (training + iterations since start of cycle). + Default: 'cycle' + cycle_momentum (bool): If ``True``, momentum is cycled inversely + to learning rate between 'base_momentum' and 'max_momentum'. + Default: True + base_momentum (float or list): Lower momentum boundaries in the cycle + for each parameter group. Note that momentum is cycled inversely + to learning rate; at the peak of a cycle, momentum is + 'base_momentum' and learning rate is 'max_lr'. + Default: 0.8 + max_momentum (float or list): Upper momentum boundaries in the cycle + for each parameter group. Functionally, + it defines the cycle amplitude (max_momentum - base_momentum). + The momentum at any cycle is the difference of max_momentum + and some scaling of the amplitude; therefore + base_momentum may not actually be reached depending on + scaling function. Note that momentum is cycled inversely + to learning rate; at the start of a cycle, momentum is 'max_momentum' + and learning rate is 'base_lr' + Default: 0.9 + last_epoch (int): The index of the last batch. This parameter is used when + resuming a training job. Since `step()` should be invoked after each + batch instead of after each epoch, this number represents the total + number of *batches* computed, not the total number of epochs computed. + When last_epoch=-1, the schedule is started from the beginning. + Default: -1 + + Example: + >>> # xdoctest: +SKIP + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> scheduler = torch.optim.lr_scheduler.CyclicLR( + ... optimizer, + ... base_lr=0.01, + ... max_lr=0.1, + ... step_size_up=10, + ... ) + >>> data_loader = torch.utils.data.DataLoader(...) + >>> for epoch in range(10): + >>> for batch in data_loader: + >>> train_batch(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/CyclicLR.png + + .. _Cyclical Learning Rates for Training Neural Networks: https://arxiv.org/abs/1506.01186 + .. _bckenstler/CLR: https://github.com/bckenstler/CLR + """ + + def __init__( + self, + optimizer: Optimizer, + base_lr: float | list[float], + max_lr: float | list[float], + step_size_up: int = 2000, + step_size_down: int | None = None, + mode: Literal["triangular", "triangular2", "exp_range"] = "triangular", + gamma: float = 1.0, + scale_fn: Callable[[float], float] | None = None, + scale_mode: Literal["cycle", "iterations"] = "cycle", + cycle_momentum: bool = True, + base_momentum: float = 0.8, + max_momentum: float = 0.9, + last_epoch: int = -1, + ) -> None: # noqa: D107 + # Attach optimizer + if not isinstance(optimizer, Optimizer): + raise TypeError(f"{type(optimizer).__name__} is not an Optimizer") + self.optimizer = optimizer + + base_lrs = _format_param("base_lr", optimizer, base_lr) + if last_epoch == -1: + for lr, group in zip(base_lrs, optimizer.param_groups, strict=True): + _update_param_group_val(group, "lr", lr) + + self.max_lrs = _format_param("max_lr", optimizer, max_lr) + + # pyrefly: ignore [bad-assignment] + step_size_up = float(step_size_up) + step_size_down = ( + # pyrefly: ignore [bad-assignment] + float(step_size_down) if step_size_down is not None else step_size_up + ) + # pyrefly: ignore [unsupported-operation] + self.total_size = step_size_up + step_size_down + self.step_ratio = step_size_up / self.total_size + + if mode not in ["triangular", "triangular2", "exp_range"] and scale_fn is None: + raise ValueError("mode is invalid and scale_fn is None") + + self.mode = mode + self.gamma = gamma + + self._scale_fn_ref: Callable[[float], float] + self._scale_fn_custom = scale_fn + self.scale_mode = scale_mode + self._init_scale_fn() + + self.cycle_momentum = cycle_momentum + if cycle_momentum: + if ( + "momentum" not in optimizer.defaults + and "betas" not in optimizer.defaults + ): + raise ValueError( + "optimizer must support momentum or beta1 with `cycle_momentum` option enabled" + ) + + self.use_beta1 = "betas" in self.optimizer.defaults + self.base_momentums = _format_param( + "base_momentum", optimizer, base_momentum + ) + self.max_momentums = _format_param("max_momentum", optimizer, max_momentum) + if last_epoch == -1: + for m_momentum, b_momentum, group in zip( + self.max_momentums, + self.base_momentums, + optimizer.param_groups, + strict=True, + ): + if self.use_beta1: + group["betas"] = (m_momentum, *group["betas"][1:]) + else: + group["momentum"] = m_momentum + group["max_momentum"] = m_momentum + group["base_momentum"] = b_momentum + + super().__init__(optimizer, last_epoch) + self.base_lrs = base_lrs + + def _init_scale_fn(self) -> None: + if self._scale_fn_custom is not None: + return + if self.mode == "triangular": + self._scale_fn_ref = self._triangular_scale_fn + self.scale_mode = "cycle" + elif self.mode == "triangular2": + self._scale_fn_ref = self._triangular2_scale_fn + self.scale_mode = "cycle" + elif self.mode == "exp_range": + self._scale_fn_ref = partial(self._exp_range_scale_fn, self.gamma) + self.scale_mode = "iterations" + + def scale_fn(self, x) -> float: + """Get the scaling policy.""" + if self._scale_fn_custom is not None: + return self._scale_fn_custom(x) + else: + return self._scale_fn_ref(x) # static method + + @staticmethod + def _triangular_scale_fn(x: float) -> float: + return 1.0 + + @staticmethod + def _triangular2_scale_fn(x: float) -> float: + return 1 / (2.0 ** (x - 1)) + + @staticmethod + def _exp_range_scale_fn(gamma: float, x: float) -> float: + return gamma**x + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Advances each ``group["lr"]`` in the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` along a cycle between the + group's ``base_lr`` and ``max_lr`` using :meth:`scale_fn`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + + .. note:: + This method treats :attr:`last_epoch` as the index of the previous + batch. + + .. note:: + When :attr:`cycle_momentum` is ``True``, this method has a side + effect of updating the optimizer's momentum. + """ + _warn_get_lr_called_within_step(self) + + cycle = math.floor(1 + self.last_epoch / self.total_size) + x = 1.0 + self.last_epoch / self.total_size - cycle + if x <= self.step_ratio: + scale_factor = x / self.step_ratio + else: + scale_factor = (x - 1) / (self.step_ratio - 1) + + lrs = [] + for base_lr, max_lr in zip(self.base_lrs, self.max_lrs, strict=True): + base_height = (max_lr - base_lr) * scale_factor + if self.scale_mode == "cycle": + lr = base_lr + base_height * self.scale_fn(cycle) + else: + lr = base_lr + base_height * self.scale_fn(self.last_epoch) + lrs.append(lr) + + if self.cycle_momentum: + momentums = [] + for base_momentum, max_momentum in zip( + self.base_momentums, self.max_momentums, strict=True + ): + base_height = (max_momentum - base_momentum) * scale_factor + if self.scale_mode == "cycle": + momentum = max_momentum - base_height * self.scale_fn(cycle) + else: + momentum = max_momentum - base_height * self.scale_fn( + self.last_epoch + ) + momentums.append(momentum) + for param_group, momentum in zip( + self.optimizer.param_groups, momentums, strict=True + ): + if self.use_beta1: + param_group["betas"] = (momentum, *param_group["betas"][1:]) + else: + param_group["momentum"] = momentum + + return lrs + + @override + def state_dict(self) -> dict[str, Any]: # noqa: D102 + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in ``self.__dict__`` which + is not the optimizer. + The learning rate lambda functions will only be saved if they are callable objects + and not if they are functions or lambdas. + + When saving or loading the scheduler, please make sure to also save or load the state of the optimizer. + """ + state = super().state_dict() + # We are dropping the `_scale_fn_ref` attribute because it is a + # `weakref.WeakMethod` and can't be pickled. + state.pop("_scale_fn_ref", None) + fn = state.pop("_scale_fn_custom") + state["_scale_fn_custom"] = None + if fn is not None and not isinstance(fn, types.FunctionType): + # The _scale_fn_custom will only be saved if it is a callable object + # and not if it is a function or lambda. + state["_scale_fn_custom"] = fn.__dict__.copy() + + return state + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state.""" + fn = state_dict.pop("_scale_fn_custom") + super().load_state_dict(state_dict) + if fn is not None: + self._scale_fn_custom.__dict__.update(fn) + self._init_scale_fn() + + +class CosineAnnealingWarmRestarts(LRScheduler): + r"""Set the learning rate of each parameter group using a cosine annealing schedule. + + The :math:`\eta_{max}` is set to the initial lr, :math:`T_{cur}` + is the number of epochs since the last restart and :math:`T_{i}` is the number + of epochs between two warm restarts in SGDR: + + .. math:: + \eta_t = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})\left(1 + + \cos\left(\frac{T_{cur}}{T_{i}}\pi\right)\right) + + When :math:`T_{cur}=T_{i}`, set :math:`\eta_t = \eta_{min}`. + When :math:`T_{cur}=0` after restart, set :math:`\eta_t=\eta_{max}`. + + It has been proposed in + `SGDR: Stochastic Gradient Descent with Warm Restarts`_. + + Args: + optimizer (Optimizer): Wrapped optimizer. + T_0 (int): Number of iterations until the first restart. + T_mult (int, optional): A factor by which :math:`T_{i}` increases after a restart. Default: 1. + eta_min (float, optional): Minimum learning rate. Default: 0. + last_epoch (int, optional): The index of the last epoch. Default: -1. + + .. _SGDR\: Stochastic Gradient Descent with Warm Restarts: + https://arxiv.org/abs/1608.03983 + + Example: + >>> # xdoctest: +SKIP + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.05) + >>> scheduler = torch.optim.lr_scheduler.CosineAnnealingWarmRestarts( + ... optimizer, T_0=20 + ... ) + >>> for epoch in range(100): + >>> train(...) + >>> validate(...) + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/CosineAnnealingWarmRestarts.png + """ + + def __init__( + self, + optimizer: Optimizer, + T_0: int, + T_mult: int = 1, + eta_min: float = 0.0, + last_epoch: int = -1, + ) -> None: # noqa: D107 + if T_0 <= 0 or not isinstance(T_0, int): + raise ValueError(f"Expected positive integer T_0, but got {T_0}") + if T_mult < 1 or not isinstance(T_mult, int): + raise ValueError(f"Expected integer T_mult >= 1, but got {T_mult}") + if not isinstance(eta_min, (float, int)): + raise ValueError( + f"Expected float or int eta_min, but got {eta_min} of type {type(eta_min)}" + ) + self.T_0 = T_0 + self.T_i = T_0 + self.T_mult = T_mult + self.eta_min = eta_min + self.T_cur = last_epoch + super().__init__(optimizer, last_epoch) + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Computes learning rates for the optimizer's + :attr:`~torch.optim.Optimizer.param_groups` following: + + .. math:: + \texttt{eta\_min} + \frac{1}{2}(\texttt{base\_lr} - + \texttt{eta\_min})\left(1 + \cos\left(\pi \cdot + \frac{\texttt{T\_cur}}{\texttt{T\_i}}\right)\right) + + Where :attr:`T_cur` is the number of epochs since the last restart and + :attr:`T_i` is the number of epochs between two restarts. Both + :attr:`T_cur` and :attr:`T_i` are updated in :meth:`step`, and + :attr:`T_i` becomes :attr:`T_mult` times larger after each restart. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + _warn_get_lr_called_within_step(self) + + return [ + self.eta_min + + (base_lr - self.eta_min) + * (1 + math.cos(math.pi * self.T_cur / self.T_i)) + / 2 + for base_lr in self.base_lrs + ] + + @override + def step(self, epoch=None) -> None: + """Step could be called after every batch update. + + Example: + >>> # xdoctest: +SKIP("Undefined vars") + >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult) + >>> iters = len(dataloader) + >>> for epoch in range(20): + >>> for i, sample in enumerate(dataloader): + >>> inputs, labels = sample['inputs'], sample['labels'] + >>> optimizer.zero_grad() + >>> outputs = net(inputs) + >>> loss = criterion(outputs, labels) + >>> loss.backward() + >>> optimizer.step() + >>> scheduler.step(epoch + i / iters) + + This function can be called in an interleaved way. + + Example: + >>> # xdoctest: +SKIP("Undefined vars") + >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult) + >>> for epoch in range(20): + >>> scheduler.step() + >>> scheduler.step(26) + >>> scheduler.step() # scheduler.step(27), instead of scheduler(20) + """ + if epoch is None and self.last_epoch < 0: + epoch = 0 + + if epoch is None: + epoch = self.last_epoch + 1 + self.T_cur = self.T_cur + 1 + if self.T_cur >= self.T_i: + self.T_cur = self.T_cur % self.T_i + self.T_i = self.T_i * self.T_mult + else: + if epoch < 0: + raise ValueError(f"Expected non-negative epoch, but got {epoch}") + if epoch >= self.T_0: + if self.T_mult == 1: + self.T_cur = epoch % self.T_0 + else: + n = int( + math.log( + (epoch / self.T_0 * (self.T_mult - 1) + 1), self.T_mult + ) + ) + self.T_cur = epoch - self.T_0 * (self.T_mult**n - 1) / ( + self.T_mult - 1 + ) + self.T_i = self.T_0 * self.T_mult ** (n) + else: + self.T_i = self.T_0 + self.T_cur = epoch + self.last_epoch = math.floor(epoch) + + with _enable_get_lr_call(self): + for param_group, lr in zip( + self.optimizer.param_groups, self.get_lr(), strict=True + ): + _update_param_group_val(param_group, "lr", lr) + + self._last_lr = _param_groups_val_list(self.optimizer, "lr") + + +class _SchedulePhase(TypedDict): + end_step: float + start_lr: str + end_lr: str + start_momentum: str + end_momentum: str + + +class OneCycleLR(LRScheduler): + r"""Sets the learning rate of each parameter group according to the 1cycle learning rate policy. + + The 1cycle policy anneals the learning rate from an initial learning rate to some maximum + learning rate and then from that maximum learning rate to some minimum learning rate much + lower than the initial learning rate. + This policy was initially described in the paper `Super-Convergence: + Very Fast Training of Neural Networks Using Large Learning Rates`_. + + The 1cycle learning rate policy changes the learning rate after every batch. + `step` should be called after a batch has been used for training. + + This scheduler is not chainable. + + Note also that the total number of steps in the cycle can be determined in one + of two ways (listed in order of precedence): + + #. A value for total_steps is explicitly provided. + #. A number of epochs (epochs) and a number of steps per epoch + (steps_per_epoch) are provided. + In this case, the number of total steps is inferred by + total_steps = epochs * steps_per_epoch + + You must either provide a value for total_steps or provide a value for both + epochs and steps_per_epoch. + + The default behaviour of this scheduler follows the fastai implementation of 1cycle, which + claims that "unpublished work has shown even better results by using only two phases". To + mimic the behaviour of the original paper instead, set ``three_phase=True``. + + Args: + optimizer (Optimizer): Wrapped optimizer. + max_lr (float or list): Upper learning rate boundaries in the cycle + for each parameter group. + total_steps (int): The total number of steps in the cycle. Note that + if a value is not provided here, then it must be inferred by providing + a value for epochs and steps_per_epoch. + Default: None + epochs (int): The number of epochs to train for. This is used along + with steps_per_epoch in order to infer the total number of steps in the cycle + if a value for total_steps is not provided. + Default: None + steps_per_epoch (int): The number of steps per epoch to train for. This is + used along with epochs in order to infer the total number of steps in the + cycle if a value for total_steps is not provided. + Default: None + pct_start (float): The percentage of the cycle (in number of steps) spent + increasing the learning rate. + Default: 0.3 + anneal_strategy (str): {'cos', 'linear'} + Specifies the annealing strategy: "cos" for cosine annealing, "linear" for + linear annealing. + Default: 'cos' + cycle_momentum (bool): If ``True``, momentum is cycled inversely + to learning rate between 'base_momentum' and 'max_momentum'. + Default: True + base_momentum (float or list): Lower momentum boundaries in the cycle + for each parameter group. Note that momentum is cycled inversely + to learning rate; at the peak of a cycle, momentum is + 'base_momentum' and learning rate is 'max_lr'. + Default: 0.85 + max_momentum (float or list): Upper momentum boundaries in the cycle + for each parameter group. Functionally, + it defines the cycle amplitude (max_momentum - base_momentum). + Note that momentum is cycled inversely + to learning rate; at the start of a cycle, momentum is 'max_momentum' + and learning rate is 'base_lr' + Default: 0.95 + div_factor (float): Determines the initial learning rate via + initial_lr = max_lr/div_factor + Default: 25 + final_div_factor (float): Determines the minimum learning rate via + min_lr = initial_lr/final_div_factor + Default: 1e4 + three_phase (bool): If ``True``, use a third phase of the schedule to annihilate the + learning rate according to 'final_div_factor' instead of modifying the second + phase (the first two phases will be symmetrical about the step indicated by + 'pct_start'). + last_epoch (int): The index of the last batch. This parameter is used when + resuming a training job. Since `step()` should be invoked after each + batch instead of after each epoch, this number represents the total + number of *batches* computed, not the total number of epochs computed. + When last_epoch=-1, the schedule is started from the beginning. + Default: -1 + + Example: + >>> # xdoctest: +SKIP + >>> data_loader = torch.utils.data.DataLoader(...) + >>> optimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9) + >>> scheduler = torch.optim.lr_scheduler.OneCycleLR( + ... optimizer, max_lr=0.01, steps_per_epoch=len(data_loader), epochs=10 + ... ) + >>> for epoch in range(10): + >>> for batch in data_loader: + >>> train_batch(...) + >>> optimizer.step() + >>> scheduler.step() + + .. image:: ../scripts/lr_scheduler_images/OneCycleLR.png + + .. _Super-Convergence\: Very Fast Training of Neural Networks Using Large Learning Rates: + https://arxiv.org/abs/1708.07120 + """ + + def __init__( + self, + optimizer: Optimizer, + max_lr: float | list[float], + total_steps: int | None = None, + epochs: int | None = None, + steps_per_epoch: int | None = None, + pct_start: float = 0.3, + anneal_strategy: Literal["cos", "linear"] = "cos", + cycle_momentum: bool = True, + base_momentum: float | list[float] = 0.85, + max_momentum: float | list[float] = 0.95, + div_factor: float = 25.0, + final_div_factor: float = 1e4, + three_phase: bool = False, + last_epoch: int = -1, + ) -> None: # noqa: D107 + # Validate optimizer + if not isinstance(optimizer, Optimizer): + raise TypeError(f"{type(optimizer).__name__} is not an Optimizer") + self.optimizer = optimizer + + # Validate total_steps + if total_steps is not None: + if total_steps <= 0 or not isinstance(total_steps, int): + raise ValueError( + f"Expected positive integer total_steps, but got {total_steps}" + ) + self.total_steps = total_steps + elif epochs is not None and steps_per_epoch is not None: + if not isinstance(epochs, int) or epochs <= 0: + raise ValueError(f"Expected positive integer epochs, but got {epochs}") + if not isinstance(steps_per_epoch, int) or steps_per_epoch <= 0: + raise ValueError( + f"Expected positive integer steps_per_epoch, but got {steps_per_epoch}" + ) + self.total_steps = epochs * steps_per_epoch + else: + raise ValueError( + "You must define either total_steps OR (epochs AND steps_per_epoch)" + ) + + self._schedule_phases: list[_SchedulePhase] + if three_phase: + self._schedule_phases = [ + { + "end_step": float(pct_start * self.total_steps) - 1, + "start_lr": "initial_lr", + "end_lr": "max_lr", + "start_momentum": "max_momentum", + "end_momentum": "base_momentum", + }, + { + "end_step": float(2 * pct_start * self.total_steps) - 2, + "start_lr": "max_lr", + "end_lr": "initial_lr", + "start_momentum": "base_momentum", + "end_momentum": "max_momentum", + }, + { + "end_step": self.total_steps - 1, + "start_lr": "initial_lr", + "end_lr": "min_lr", + "start_momentum": "max_momentum", + "end_momentum": "max_momentum", + }, + ] + else: + self._schedule_phases = [ + { + "end_step": float(pct_start * self.total_steps) - 1, + "start_lr": "initial_lr", + "end_lr": "max_lr", + "start_momentum": "max_momentum", + "end_momentum": "base_momentum", + }, + { + "end_step": self.total_steps - 1, + "start_lr": "max_lr", + "end_lr": "min_lr", + "start_momentum": "base_momentum", + "end_momentum": "max_momentum", + }, + ] + + # Validate pct_start + if pct_start < 0 or pct_start > 1 or not isinstance(pct_start, float): + raise ValueError( + f"Expected float between 0 and 1 pct_start, but got {pct_start}" + ) + + # Validate anneal_strategy + if anneal_strategy not in ["cos", "linear"]: + raise ValueError( + f"anneal_strategy must be one of 'cos' or 'linear', instead got {anneal_strategy}" + ) + else: + self._anneal_func_type = anneal_strategy + + # Initialize learning rate variables + max_lrs = _format_param("max_lr", self.optimizer, max_lr) + if last_epoch == -1: + for idx, group in enumerate(self.optimizer.param_groups): + group["initial_lr"] = max_lrs[idx] / div_factor + group["max_lr"] = max_lrs[idx] + group["min_lr"] = group["initial_lr"] / final_div_factor + + # Initialize momentum variables + self.cycle_momentum = cycle_momentum + if self.cycle_momentum: + if ( + "momentum" not in self.optimizer.defaults + and "betas" not in self.optimizer.defaults + ): + raise ValueError( + "optimizer must support momentum or beta1 with `cycle_momentum` option enabled" + ) + self.use_beta1 = "betas" in self.optimizer.defaults + max_momentums = _format_param("max_momentum", optimizer, max_momentum) + base_momentums = _format_param("base_momentum", optimizer, base_momentum) + if last_epoch == -1: + for m_momentum, b_momentum, group in zip( + max_momentums, base_momentums, optimizer.param_groups, strict=True + ): + if self.use_beta1: + group["betas"] = (m_momentum, *group["betas"][1:]) + else: + group["momentum"] = m_momentum + group["max_momentum"] = m_momentum + group["base_momentum"] = b_momentum + + super().__init__(optimizer, last_epoch) + + def _anneal_func(self, *args, **kwargs): + if hasattr(self, "_anneal_func_type"): + if self._anneal_func_type == "cos": + return self._annealing_cos(*args, **kwargs) + elif self._anneal_func_type == "linear": + return self._annealing_linear(*args, **kwargs) + else: + raise ValueError(f"Unknown _anneal_func_type: {self._anneal_func_type}") + else: + # For BC + return self.anneal_func(*args, **kwargs) # type: ignore[attr-defined] + + @staticmethod + def _annealing_cos(start, end, pct): + """Cosine anneal from `start` to `end` as pct goes from 0.0 to 1.0.""" + cos_out = math.cos(math.pi * pct) + 1 + return end + (start - end) / 2.0 * cos_out + + @staticmethod + def _annealing_linear(start, end, pct): + """Linearly anneal from `start` to `end` as pct goes from 0.0 to 1.0.""" + return (end - start) * pct + start + + @override + def get_lr(self) -> list[float | Tensor]: + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Finds the appropriate :attr:`_schedule_phases` entry for the current + step and interpolates between its ``start_lr`` and ``end_lr`` using + :meth:`_anneal_func`. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + + .. note:: + When :attr:`cycle_momentum` is ``True``, this method has a side + effect of updating the optimizer's momentum. + """ + _warn_get_lr_called_within_step(self) + + lrs = [] + step_num = self.last_epoch + + if step_num > self.total_steps: + raise ValueError( + f"Tried to step {step_num} times. The specified number of total steps is {self.total_steps}" + ) + + for group in self.optimizer.param_groups: + start_step = 0.0 + for i, phase in enumerate(self._schedule_phases): + end_step = phase["end_step"] + if step_num <= end_step or i == len(self._schedule_phases) - 1: + pct = (step_num - start_step) / (end_step - start_step) + computed_lr = self._anneal_func( + group[phase["start_lr"]], group[phase["end_lr"]], pct + ) + if self.cycle_momentum: + computed_momentum = self._anneal_func( + group[phase["start_momentum"]], + group[phase["end_momentum"]], + pct, + ) + break + start_step = phase["end_step"] + + lrs.append(computed_lr) # type: ignore[possibly-undefined] + if self.cycle_momentum: + if self.use_beta1: + group["betas"] = (computed_momentum, *group["betas"][1:]) # type: ignore[possibly-undefined] + else: + group["momentum"] = computed_momentum # type: ignore[possibly-undefined] + + return lrs diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/nadam.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/nadam.py new file mode 100644 index 0000000000000000000000000000000000000000..46a9bd47ddc81ae8cf8cc60b6c6634c411754522 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/nadam.py @@ -0,0 +1,673 @@ +# mypy: allow-untyped-defs +r"""Implementation for the NAdam algorithm.""" + +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _get_value, + _maximize_doc, + _params_doc, + _stack_if_compiling, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["NAdam", "nadam"] + + +class NAdam(Optimizer): # noqa: D101 + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 2e-3, + betas: tuple[float, float] = (0.9, 0.999), + eps: float = 1e-8, + weight_decay: float = 0, + momentum_decay: float = 4e-3, + decoupled_weight_decay: bool = False, + *, + foreach: bool | None = None, + maximize: bool = False, + capturable: bool = False, + differentiable: bool = False, + ) -> None: # noqa: D107 + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= betas[0] < 1.0: + raise ValueError(f"Invalid beta parameter at index 0: {betas[0]}") + if not 0.0 <= betas[1] < 1.0: + raise ValueError(f"Invalid beta parameter at index 1: {betas[1]}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + if not 0.0 <= momentum_decay: + raise ValueError(f"Invalid momentum_decay value: {momentum_decay}") + defaults = { + "lr": lr, + "betas": betas, + "eps": eps, + "weight_decay": weight_decay, + "momentum_decay": momentum_decay, + "decoupled_weight_decay": decoupled_weight_decay, + "maximize": maximize, + "foreach": foreach, + "capturable": capturable, + "differentiable": differentiable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): # noqa: D105 + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("maximize", False) + group.setdefault("foreach", None) + group.setdefault("capturable", False) + group.setdefault("differentiable", False) + group.setdefault("decoupled_weight_decay", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0: + if not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + if not torch.is_tensor(p_state["mu_product"]): + mu_prod_val = p_state["mu_product"] + p_state["mu_product"] = ( + torch.tensor( + mu_prod_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(mu_prod_val, dtype=_get_scalar_dtype()) + ) + + def _init_group( + self, + group, + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + mu_products, + state_steps, + ): + has_complex = False + for p in group["params"]: + if p.grad is not None: + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError("NAdam does not support sparse gradients") + grads.append(p.grad) + + state = self.state[p] + # Lazy state initialization + if len(state) == 0: + # note(crcrpar): [special device hosting for step] + # Deliberately host `step` and `mu_product` on CPU if capturable is False. + # This is because kernel launches are costly on CUDA and XLA. + state["step"] = ( + torch.zeros((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.tensor(0.0, dtype=_get_scalar_dtype()) + ) + state["mu_product"] = ( + torch.ones((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.tensor(1.0, dtype=_get_scalar_dtype()) + ) + # Exponential moving average of gradient values + state["exp_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + # Exponential moving average of squared gradient values + state["exp_avg_sq"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + exp_avgs.append(state["exp_avg"]) + exp_avg_sqs.append(state["exp_avg_sq"]) + mu_products.append(state["mu_product"]) + state_steps.append(state["step"]) + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + exp_avgs: list[Tensor] = [] + exp_avg_sqs: list[Tensor] = [] + mu_products: list[Tensor] = [] + state_steps: list[Tensor] = [] + beta1, beta2 = cast(tuple[float, float], group["betas"]) + + has_complex = self._init_group( + group, + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + mu_products, + state_steps, + ) + + nadam( + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + mu_products, + state_steps, + beta1=beta1, + beta2=beta2, + lr=group["lr"], + weight_decay=group["weight_decay"], + momentum_decay=group["momentum_decay"], + eps=group["eps"], + maximize=group["maximize"], + decoupled_weight_decay=group["decoupled_weight_decay"], + foreach=group["foreach"], + capturable=group["capturable"], + differentiable=group["differentiable"], + has_complex=has_complex, + ) + + return loss + + +NAdam.__doc__ = ( + r"""Implements NAdam algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma_t \text{ (lr)}, \: \beta_1,\beta_2 \text{ (betas)}, + \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)} \\ + &\hspace{13mm} \: \lambda \text{ (weight decay)}, \:\psi \text{ (momentum decay)} \\ + &\hspace{13mm} \: \textit{decoupled\_weight\_decay}, \:\textit{maximize} \\ + &\textbf{initialize} : m_0 \leftarrow 0 \text{ ( first moment)}, + v_0 \leftarrow 0 \text{ ( second moment)} \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\ + &\hspace{10mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm} \theta_t \leftarrow \theta_{t-1} \\ + &\hspace{5mm} \textbf{if} \: \lambda \neq 0 \\ + &\hspace{10mm}\textbf{if} \: \textit{decoupled\_weight\_decay} \\ + &\hspace{15mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1} \\ + &\hspace{10mm}\textbf{else} \\ + &\hspace{15mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm} \mu_t \leftarrow \beta_1 \big(1 - \frac{1}{2} 0.96^{t \psi} \big) \\ + &\hspace{5mm} \mu_{t+1} \leftarrow \beta_1 \big(1 - \frac{1}{2} 0.96^{(t+1)\psi}\big)\\ + &\hspace{5mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ + &\hspace{5mm}v_t \leftarrow \beta_2 v_{t-1} + (1-\beta_2) g^2_t \\ + &\hspace{5mm}\widehat{m_t} \leftarrow \mu_{t+1} m_t/(1-\prod_{i=1}^{t+1}\mu_i)\\[-1.ex] + & \hspace{11mm} + (1-\mu_t) g_t /(1-\prod_{i=1}^{t} \mu_{i}) \\ + &\hspace{5mm}\widehat{v_t} \leftarrow v_t/\big(1-\beta_2^t \big) \\ + &\hspace{5mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/ + \big(\sqrt{\widehat{v_t}} + \epsilon \big) \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `Incorporating Nesterov Momentum into Adam`_. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 2e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + momentum_decay (float, optional): momentum momentum_decay (default: 4e-3) + decoupled_weight_decay (bool, optional): whether to decouple the weight + decay as in AdamW to obtain NAdamW. If True, the algorithm does not + accumulate weight decay in the momentum nor variance. (default: False) + {_foreach_doc} + {_maximize_doc} + {_capturable_doc} + {_differentiable_doc} + + .. _Incorporating Nesterov Momentum into Adam: + https://openreview.net/forum?id=OM0jvwB8jIp57ZJjtNEZ + .. _Decoupled Weight Decay Regularization: + https://arxiv.org/abs/1711.05101 + + """ +) + + +def _single_tensor_nadam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + mu_products: list[Tensor], + state_steps: list[Tensor], + *, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + momentum_decay: float, + eps: float, + decoupled_weight_decay: bool, + maximize: bool, + capturable: bool, + differentiable: bool, + has_complex: bool, +) -> None: + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for i, param in enumerate(params): + grad = grads[i] if not maximize else -grads[i] + exp_avg = exp_avgs[i] + exp_avg_sq = exp_avg_sqs[i] + mu_product = mu_products[i] + step_t = state_steps[i] + + if torch.is_complex(param): + param = torch.view_as_real(param) + grad = torch.view_as_real(grad) + exp_avg = torch.view_as_real(exp_avg) + exp_avg_sq = torch.view_as_real(exp_avg_sq) + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type == mu_product.device.type == step_t.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params, mu_products and state_steps must be " + f"on supported devices: {capturable_supported_devices}." + ) + + # update step + step_t += 1 + + if capturable: + step = step_t + else: + step = _get_value(step_t) + + bias_correction2 = 1 - beta2**step + + if weight_decay != 0: + if decoupled_weight_decay: + # Perform stepweight decay + param.mul_(1 - lr * weight_decay) + else: + grad = grad.add(param, alpha=weight_decay) + + # calculate the momentum cache \mu^{t} and \mu^{t+1} + mu = beta1 * (1.0 - 0.5 * (0.96 ** (step * momentum_decay))) + mu_next = beta1 * (1.0 - 0.5 * (0.96 ** ((step + 1) * momentum_decay))) + + # update mu_product + mu_product *= mu + + # decay the first and second moment running average coefficient + exp_avg.lerp_(grad, 1 - beta1) + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) + denom = exp_avg_sq.div(bias_correction2).sqrt() + + if differentiable or capturable: + denom = denom.add(eps) + # Make autograd track the operations + # by updating the grad and exp_avg directly and not using the + # scalar "value" argument of addcdiv. + mu_product_next = mu_product * mu_next + grad = grad * (-lr * (1.0 - mu) / (1.0 - mu_product)) + exp_avg = exp_avg * (-lr * mu_next / (1.0 - mu_product_next)) + param.addcdiv_(grad, denom) + param.addcdiv_(exp_avg, denom) + else: + mu_product_next = _get_value(mu_product) * mu_next + denom.add_(eps) + param.addcdiv_( + grad, denom, value=(-lr * (1.0 - mu) / (1.0 - _get_value(mu_product))) + ) + param.addcdiv_( + exp_avg, + denom, + value=cast(float, (-lr * mu_next) / (1.0 - mu_product_next)), + ) + + +def _multi_tensor_nadam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + mu_products: list[Tensor], + state_steps: list[Tensor], + *, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + momentum_decay: float, + eps: float, + decoupled_weight_decay: bool, + maximize: bool, + capturable: bool, + differentiable: bool, + has_complex: bool, +) -> None: + if len(params) == 0: + return + + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == mp.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, mp, step in zip(params, mu_products, state_steps, strict=True) + ): + raise AssertionError( + "If capturable=True, " + "params, mu_products, and state_steps must be on supported devices: " + f"{capturable_supported_devices}." + ) + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, exp_avgs, exp_avg_sqs, mu_products, state_steps] # type: ignore[list-item] + ) + for ( + grouped_params_, + grouped_grads_, + grouped_exp_avgs_, + grouped_exp_avg_sqs_, + grouped_mu_products_, + grouped_state_steps_, + ), _ in grouped_tensors.values(): + grouped_params = cast(list[Tensor], grouped_params_) + grouped_grads = cast(list[Tensor], grouped_grads_) + grouped_exp_avgs = cast(list[Tensor], grouped_exp_avgs_) + grouped_exp_avg_sqs = cast(list[Tensor], grouped_exp_avg_sqs_) + grouped_mu_products = cast(list[Tensor], grouped_mu_products_) + grouped_state_steps = cast(list[Tensor], grouped_state_steps_) + + # handle complex + if has_complex: + _view_as_real( + grouped_params, grouped_grads, grouped_exp_avgs, grouped_exp_avg_sqs + ) + + if maximize: + grouped_grads = torch._foreach_neg(grouped_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and grouped_state_steps[0].is_cpu: + torch._foreach_add_( + grouped_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(grouped_state_steps, 1) + + if weight_decay != 0: + if decoupled_weight_decay: + # Perform stepweight decay + torch._foreach_mul_(grouped_params, 1 - lr * weight_decay) + else: + # Reuse the intermediate memory (grouped_grads) already allocated for maximize + if maximize: + torch._foreach_add_( + grouped_grads, grouped_params, alpha=weight_decay + ) + else: + grouped_grads = torch._foreach_add( # type: ignore[assignment] + grouped_grads, grouped_params, alpha=weight_decay + ) + + # Decay the first and second moment running average coefficient + torch._foreach_lerp_(grouped_exp_avgs, grouped_grads, 1 - beta1) + + torch._foreach_mul_(grouped_exp_avg_sqs, beta2) + torch._foreach_addcmul_( + grouped_exp_avg_sqs, grouped_grads, grouped_grads, 1 - beta2 + ) + + exp_avg_sq_sqrt = torch._foreach_sqrt(grouped_exp_avg_sqs) + + bias_correction_sqrt: tuple[Tensor, ...] | list[Tensor] + mus: tuple[Tensor, ...] | list[Tensor] + mu_nexts: tuple[Tensor, ...] | list[Tensor] + if capturable: + # mus will be beta1 * (1 - 0.5 * 0.96 ** (step * momentum_decay)) + exponent = torch._foreach_mul(grouped_state_steps, momentum_decay) + mus = torch._foreach_pow(0.96, exponent) + torch._foreach_mul_(mus, -0.5) + torch._foreach_add_(mus, 1.0) + torch._foreach_mul_(mus, beta1) + + # mu_nexts will be beta1 * (1 - 0.5 * 0.96 ** ((step + 1) * momentum_decay)) + torch._foreach_add_(exponent, momentum_decay) + mu_nexts = torch._foreach_pow(0.96, exponent) + torch._foreach_mul_(mu_nexts, -0.5) + torch._foreach_add_(mu_nexts, 1.0) + torch._foreach_mul_(mu_nexts, beta1) + + # save peak memory as we don't need exponent anymore + del exponent + + bias_correction_sqrt = torch._foreach_pow(beta2, grouped_state_steps) + # foreach_sub doesn't allow a scalar as the first arg + torch._foreach_sub_(bias_correction_sqrt, 1.0) + torch._foreach_neg_(bias_correction_sqrt) + torch._foreach_sqrt_(bias_correction_sqrt) + else: + bias_correction_sqrt = [ + (1 - beta2 ** _get_value(step)) ** 0.5 for step in grouped_state_steps + ] + mus = [ + beta1 * (1.0 - 0.5 * (0.96 ** (_get_value(step) * momentum_decay))) + for step in grouped_state_steps + ] + mu_nexts = [ + beta1 + * (1.0 - 0.5 * (0.96 ** ((_get_value(step) + 1) * momentum_decay))) + for step in grouped_state_steps + ] + + # update mu_products + torch._foreach_mul_(grouped_mu_products, mus) + + torch._foreach_div_(exp_avg_sq_sqrt, bias_correction_sqrt) + torch._foreach_add_(exp_avg_sq_sqrt, eps) + + # explicitly delete bias_correction refs to save memory + del bias_correction_sqrt + + if capturable: + # Build up the step_size multiplier for grad, reusing mus' memory + torch._foreach_sub_(mus, 1.0) + torch._foreach_mul_(mus, lr) + # foreach_sub doesn't allow a scalar as the first arg + denom = torch._foreach_sub(grouped_mu_products, 1.0) + torch._foreach_neg_(denom) + torch._foreach_div_(mus, denom) + # - lr * (1 - mu) / (1 - mu_product) + step_size_grads = mus + # explicitly delete denom to save memory + del denom + + # Build up the step_size multiplier for exp_avg, reusing mu_nexts' memory + denom = torch._foreach_mul(grouped_mu_products, mu_nexts) + torch._foreach_mul_(mu_nexts, lr) + # foreach_sub doesn't allow a scalar as the first arg, but it's okay because + # we need a negative here anyway + torch._foreach_sub_(denom, 1.0) + torch._foreach_div_(mu_nexts, denom) + # - lr * mu_next / (1 - mu_product * mu_next) + step_size_expavg = mu_nexts + # explicitly delete denom to save memory + del denom + + # we cannot inplace into step_size_grads cuz it is a list of ScalarTensors + # and mul'ing with grouped_grads will result in a list of bigger Tensors + numerator = torch._foreach_mul(step_size_grads, grouped_grads) + torch._foreach_addcmul_(numerator, step_size_expavg, grouped_exp_avgs) + + # finally, update params + torch._foreach_addcdiv_(grouped_params, numerator, exp_avg_sq_sqrt) + else: + step_size_grads = _stack_if_compiling( + [ + (_get_value(lr) * (1.0 - mu) / (1.0 - _get_value(mu_product))) * -1 + for mu_product, mu in zip(grouped_mu_products, mus, strict=True) + ] + ) + step_size_expavg = _stack_if_compiling( + [ + ( + _get_value(lr) + * mu_next + / (1.0 - _get_value(mu_product) * mu_next) + ) + * -1 + for mu_product, mu_next in zip( + grouped_mu_products, mu_nexts, strict=True + ) + ] + ) + + torch._foreach_addcdiv_( + grouped_params, + grouped_grads, + exp_avg_sq_sqrt, + step_size_grads, # type: ignore[arg-type] + ) + torch._foreach_addcdiv_( + grouped_params, + grouped_exp_avgs, + exp_avg_sq_sqrt, + step_size_expavg, # type: ignore[arg-type] + ) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_nadam) +def nadam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + mu_products: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + decoupled_weight_decay: bool = False, + foreach: bool | None = None, + capturable: bool = False, + differentiable: bool = False, + has_complex: bool = False, + maximize: bool = False, + *, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + momentum_decay: float, + eps: float, +) -> None: + r"""Functional API that performs NAdam algorithm computation. + + See :class:`~torch.optim.NAdam` for details. + """ + if not all(isinstance(t, torch.Tensor) for t in state_steps): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + if not all(isinstance(t, torch.Tensor) for t in mu_products): + raise RuntimeError( + "API has changed, `mu_products` argument must contain a list of singleton tensors" + ) + + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_nadam + else: + func = _single_tensor_nadam + + func( + params, + grads, + exp_avgs, + exp_avg_sqs, + mu_products, + state_steps, + beta1=beta1, + beta2=beta2, + lr=lr, + weight_decay=weight_decay, + momentum_decay=momentum_decay, + maximize=maximize, + decoupled_weight_decay=decoupled_weight_decay, + eps=eps, + capturable=capturable, + differentiable=differentiable, + has_complex=has_complex, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/optimizer.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..8e691389ea50e9c5a73d98a52de8cfed93928bf1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/optimizer.py @@ -0,0 +1,1185 @@ +# mypy: allow-untyped-defs +"""Base optimizer.""" + +import functools +import warnings +from collections import defaultdict, OrderedDict +from collections.abc import Callable, Hashable, Iterable, Sequence +from copy import deepcopy +from itertools import chain +from typing import Any, cast, overload, TypeAlias, TypeVar +from typing_extensions import ParamSpec, Self + +import torch +import torch.utils.hooks as hooks +from torch.utils._foreach_utils import ( + _get_foreach_kernels_supported_devices, + _get_fused_kernels_supported_devices, + _group_tensors_by_device_and_dtype, + Indices, + TensorListList, +) +from torch.utils.hooks import RemovableHandle + + +_T = TypeVar("_T") +_P = ParamSpec("_P") + +Args: TypeAlias = tuple[Any, ...] +Kwargs: TypeAlias = dict[str, Any] +StateDict: TypeAlias = dict[str, Any] +DeviceDict: TypeAlias = dict[torch.device | None, torch.Tensor] +DeviceDtypeDict: TypeAlias = dict[tuple[torch.device, torch.dtype] | None, torch.Tensor] + +GlobalOptimizerPreHook: TypeAlias = Callable[ + ["Optimizer", Args, Kwargs], tuple[Args, Kwargs] | None +] +GlobalOptimizerPostHook: TypeAlias = Callable[["Optimizer", Args, Kwargs], None] + +__all__ = [ + "Optimizer", + "register_optimizer_step_pre_hook", + "register_optimizer_step_post_hook", +] +_global_optimizer_pre_hooks: dict[int, GlobalOptimizerPreHook] = OrderedDict() +_global_optimizer_post_hooks: dict[int, GlobalOptimizerPostHook] = OrderedDict() +_foreach_supported_types = [torch.Tensor, torch.nn.parameter.Parameter] + + +class _RequiredParameter: + """Singleton class representing a required parameter for an Optimizer.""" + + def __repr__(self) -> str: + return "" + + +required = _RequiredParameter() + + +def _use_grad_for_differentiable(func: Callable[_P, _T]) -> Callable[_P, _T]: + def _use_grad(*args: _P.args, **kwargs: _P.kwargs) -> _T: + import torch._dynamo + + # pyrefly: ignore [unsupported-operation] + self = cast(Optimizer, args[0]) # assume first positional arg is `self` + prev_grad = torch.is_grad_enabled() + try: + # Note on graph break below: + # we need to graph break to ensure that aot respects the no_grad annotation. + # This is important for perf because without this, functionalization will generate an epilogue + # which updates the mutated parameters of the optimizer which is *not* visible to inductor, as a result, + # inductor will allocate for every parameter in the model, which is horrible. + # With this, aot correctly sees that this is an inference graph, and functionalization will generate + # an epilogue which is appended to the graph, which *is* visible to inductor, as a result, inductor sees that + # step is in place and is able to avoid the extra allocation. + # In the future, we will either 1) continue to graph break on backward, so this graph break does not matter + # or 2) have a fully fused forward and backward graph, which will have no_grad by default, and we can remove this + # graph break to allow the fully fused fwd-bwd-optimizer graph to be compiled. + # see https://github.com/pytorch/pytorch/issues/104053 + torch.set_grad_enabled(self.defaults["differentiable"]) + torch._dynamo.graph_break() + ret = func(*args, **kwargs) + finally: + torch._dynamo.graph_break() + torch.set_grad_enabled(prev_grad) + return ret + + functools.update_wrapper(_use_grad, func) + return _use_grad + + +def _get_value(x): + # item is significantly faster than a cpu tensor in eager mode + if not torch.jit.is_scripting() and torch.compiler.is_compiling(): + return x + else: + return x.item() if isinstance(x, torch.Tensor) else x + + +def _stack_if_compiling(x): + if not torch.jit.is_scripting() and torch.compiler.is_compiling(): + return torch.stack(x) + else: + return x + + +def _disable_dynamo_if_unsupported( + single_tensor_fn: Callable[..., object] | None = None, +) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: + # workaround for torchscript BC + # it requires all called functions to be in the + # global environment at the site at which the + # maybe_fallback closure is created + if single_tensor_fn: + globals()[single_tensor_fn.__name__] = single_tensor_fn + + def wrapper(func: Callable[_P, _T]) -> Callable[_P, _T]: + import inspect + + disabled_func = torch._disable_dynamo(func) + ps = inspect.signature(func).parameters + has_state_steps = True + try: + state_steps_ind = list(ps.keys()).index("state_steps") + except ValueError: + has_state_steps = False + + # Today, there are cases where we stack state steps + # and pass them as the value arg of foreach ops. + # Having state steps on cuda as the value arg is not supported in eager, + # but this only occurs in the rare case that the user explicitly deletes + # the capturable flag. If capturable=True, this is not a problem. + @functools.wraps(func) + def maybe_fallback(*args: _P.args, **kwargs: _P.kwargs): + if torch.compiler.is_compiling() and ( + not kwargs.get("capturable", False) + and has_state_steps + # pyrefly: ignore [unsupported-operation] + and (arg := args[state_steps_ind]) + and isinstance(arg, Sequence) + and arg[0].is_cuda + or ( + "state_steps" in kwargs + # pyrefly: ignore [unsupported-operation] + and (kwarg := kwargs["state_steps"]) + and isinstance(kwarg, Sequence) + and kwarg[0].is_cuda + ) + ): + return disabled_func(*args, **kwargs) + else: + return func(*args, **kwargs) + + return maybe_fallback + + return wrapper + + +# For any optimizer with a faster implementation, we attempt to default to the +# fastest + stablest whenever possible. For foreach, the requirements are to have +# native params all on CUDA. For fused, there's currently the additional requirement +# that the tensors' dtypes must be floating point. Neither alternative supports +# torch.jit.script nor differentiable, so we fall back to the single tensor +# implementation in those cases. +def _default_to_fused_or_foreach( + params: list[torch.Tensor], differentiable: bool, use_fused: bool = False +) -> tuple[bool, bool]: + if torch.jit.is_scripting() or differentiable: + return False, False + + fused_supported_devices = _get_fused_kernels_supported_devices() + foreach_supported_devices = _get_foreach_kernels_supported_devices() + fused = use_fused and all( + p is None + or ( + type(p) in _foreach_supported_types + and p.device.type in fused_supported_devices + and torch.is_floating_point(p) + ) + for p in params + ) + foreach = not fused and all( + p is None + or ( + type(p) in _foreach_supported_types + and p.device.type in foreach_supported_devices + ) + for p in params + ) + return fused, foreach + + +def _device_dtype_check_for_fused( + p: torch.Tensor, cuda_unsupported: bool = False +) -> None: + fused_supported_devices = _get_fused_kernels_supported_devices() + if cuda_unsupported: + fused_supported_devices.remove("cuda") + if not (p.device.type in fused_supported_devices and torch.is_floating_point(p)): + raise RuntimeError( + "`fused=True` requires all the params to be floating point Tensors of " + f"supported devices: {fused_supported_devices} but {p.dtype} and {p.device.type}" + ) + + +def _view_as_real(params, *state_and_grads) -> None: + for i, p in enumerate(params): + if torch.is_complex(p): + params[i] = torch.view_as_real(params[i]) + for s in state_and_grads: + s[i] = torch.view_as_real(s[i]) + + +def _get_scalar_dtype(is_fused=None): + if is_fused: + return torch.float32 + return ( + torch.float64 if torch.get_default_dtype() == torch.float64 else torch.float32 + ) + + +def _get_capturable_supported_devices(supports_xla: bool = True) -> list[str]: + r"""Return the device type list that supports capturable optimizer.""" + capturable_supported_devices = ["cuda", "xpu", "hpu"] + if not torch.jit.is_scripting(): + capturable_supported_devices.append(torch._C._get_privateuse1_backend_name()) + if supports_xla: + capturable_supported_devices.append("xla") + return capturable_supported_devices + + +def _to_scalar(x: float | torch.Tensor): + r"""This function converts a hyperparameter to a 0-dimension (scalar) tensor + if it is a nonzero-dimensions 1-element tensor. If it is not a tensor, it is + kept as is. + + Args: + x (float or Tensor): A hyperparameter of the optimizer. + If it is Tensor, it is needed to be 1-element. + + Returns: + float or Tensor: + a scalar tensor if x is Tensor otherwise Python scalar (float) value. + """ + if isinstance(x, torch.Tensor) and x.dim() != 0: + return x.squeeze() + else: + return x + + +# Common doc strings among optimizers +_params_doc = r"""params (iterable): iterable of parameters or named_parameters to optimize + or iterable of dicts defining parameter groups. When using named_parameters, + all parameters in all groups should be named""" + +_foreach_doc = r"""foreach (bool, optional): whether foreach implementation of optimizer + is used. If unspecified by the user (so foreach is None), we will try to use + foreach over the for-loop implementation on CUDA, since it is usually + significantly more performant. Note that the foreach implementation uses + ~ sizeof(params) more peak memory than the for-loop version due to the intermediates + being a tensorlist vs just one tensor. If memory is prohibitive, batch fewer + parameters through the optimizer at a time or switch this flag to False (default: None)""" + +_fused_doc = r"""fused (bool, optional): whether the fused implementation is used. + Currently, `torch.float64`, `torch.float32`, `torch.float16`, and `torch.bfloat16` + are supported. (default: None) + + .. note:: The foreach and fused implementations are typically faster than the for-loop, + single-tensor implementation, with fused being theoretically fastest with both + vertical and horizontal fusion. As such, if the user has not specified either + flag (i.e., when foreach = fused = None), we will attempt defaulting to the foreach + implementation when the tensors are all on CUDA. Why not fused? Since the fused + implementation is relatively new, we want to give it sufficient bake-in time. + To specify fused, pass True for fused. To force running the for-loop + implementation, pass False for either foreach or fused. """ + +_capturable_doc = r"""capturable (bool, optional): whether this instance is safe to + capture in a graph, whether for CUDA graphs or for torch.compile support. + Tensors are only capturable when on supported :ref:`accelerators`. + Passing True can impair ungraphed performance, so if you don't intend to graph + capture this instance, leave it False (default: False)""" + +_differentiable_doc = r"""differentiable (bool, optional): whether autograd should + occur through the optimizer step in training. Otherwise, the step() + function runs in a torch.no_grad() context. Setting to True can impair + performance, so leave it False if you don't intend to run autograd + through this instance (default: False)""" + +_maximize_doc = r"""maximize (bool, optional): maximize the objective with respect to the + params, instead of minimizing (default: False)""" + + +def register_optimizer_step_pre_hook(hook: GlobalOptimizerPreHook) -> RemovableHandle: + r"""Register a pre hook common to all optimizers. + + The hook should have the following signature:: + + hook(optimizer, args, kwargs) -> None or modified args and kwargs + + Args: + hook (Callable): A user defined hook which is registered on all optimizers. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(_global_optimizer_pre_hooks) + _global_optimizer_pre_hooks[handle.id] = hook + return handle + + +def register_optimizer_step_post_hook(hook: GlobalOptimizerPostHook) -> RemovableHandle: + r"""Register a post hook common to all optimizers. + + The hook should have the following signature:: + + hook(optimizer, args, kwargs) -> None + + Args: + hook (Callable): A user defined hook which is registered on all optimizers. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(_global_optimizer_post_hooks) + _global_optimizer_post_hooks[handle.id] = hook + return handle + + +ParamsT: TypeAlias = ( + Iterable[torch.Tensor] + | Iterable[dict[str, Any]] + | Iterable[tuple[str, torch.Tensor]] +) + +R = TypeVar("R") +T = TypeVar("T") + + +class Optimizer: + r"""Base class for all optimizers. + + .. warning:: + Parameters need to be specified as collections that have a deterministic + ordering that is consistent between runs. Examples of objects that don't + satisfy those properties are sets and iterators over values of dictionaries. + + Args: + params (iterable): an iterable of :class:`torch.Tensor` s or + :class:`dict` s. Specifies what Tensors should be optimized. + defaults: (dict): a dict containing default values of optimization + options (used when a parameter group doesn't specify them). + """ + + OptimizerPreHook: TypeAlias = Callable[ + [Self, Args, Kwargs], # type: ignore[misc] + tuple[Args, Kwargs] | None, + ] + OptimizerPostHook: TypeAlias = Callable[[Self, Args, Kwargs], None] # type: ignore[misc] + + _optimizer_step_pre_hooks: dict[int, OptimizerPreHook] + _optimizer_step_post_hooks: dict[int, OptimizerPostHook] + # pyrefly: ignore [not-a-type] + _optimizer_state_dict_pre_hooks: 'OrderedDict[int, Callable[["Optimizer"], None]]' + _optimizer_state_dict_post_hooks: ( + # pyrefly: ignore [not-a-type] + 'OrderedDict[int, Callable[["Optimizer", StateDict], StateDict | None]]' + ) + _optimizer_load_state_dict_pre_hooks: ( + # pyrefly: ignore [not-a-type] + 'OrderedDict[int, Callable[["Optimizer", StateDict], StateDict | None]]' + ) + _optimizer_load_state_dict_post_hooks: ( + # pyrefly: ignore [not-a-type] + 'OrderedDict[int, Callable[["Optimizer"], None]]' + ) + + def __init__(self, params: ParamsT, defaults: dict[str, Any]) -> None: # noqa: D107 + torch._C._log_api_usage_once("python.optimizer") + self.defaults = defaults + self._optimizer_step_pre_hooks = OrderedDict() + self._optimizer_step_post_hooks = OrderedDict() + self._optimizer_state_dict_pre_hooks = OrderedDict() + self._optimizer_state_dict_post_hooks = OrderedDict() + self._optimizer_load_state_dict_pre_hooks = OrderedDict() + self._optimizer_load_state_dict_post_hooks = OrderedDict() + + self._patch_step_function() + + if isinstance(params, torch.Tensor): + raise TypeError( + "params argument given to the optimizer should be " + "an iterable of Tensors or dicts, but got " + torch.typename(params) + ) + + self.state: defaultdict[torch.Tensor, Any] = defaultdict(dict) + self.param_groups: list[dict[str, Any]] = [] + + param_groups = list(params) + if len(param_groups) == 0: + raise ValueError("optimizer got an empty parameter list") + if not isinstance(param_groups[0], dict): + param_groups = [{"params": param_groups}] + + for param_group in param_groups: + self.add_param_group(cast(dict, param_group)) + + # Allows _cuda_graph_capture_health_check to rig a poor man's TORCH_WARN_ONCE in python, + # which I don't think exists + # https://github.com/pytorch/pytorch/issues/72948 + self._warned_capturable_if_run_uncaptured = True + + def __getstate__(self) -> dict[str, Any]: # noqa: D105 + return { + "defaults": self.defaults, + "state": self.state, + "param_groups": self.param_groups, + } + + def __setstate__(self, state: dict[str, Any]) -> None: # noqa: D105 + self.__dict__.update(state) + if "_optimizer_step_pre_hooks" not in self.__dict__: + self._optimizer_step_pre_hooks = OrderedDict() + if "_optimizer_step_post_hooks" not in self.__dict__: + self._optimizer_step_post_hooks = OrderedDict() + if "_optimizer_state_dict_pre_hooks" not in self.__dict__: + self._optimizer_state_dict_pre_hooks = OrderedDict() + if "_optimizer_state_dict_post_hooks" not in self.__dict__: + self._optimizer_state_dict_post_hooks = OrderedDict() + if "_optimizer_load_state_dict_pre_hooks" not in self.__dict__: + self._optimizer_load_state_dict_pre_hooks = OrderedDict() + if "_optimizer_load_state_dict_post_hooks" not in self.__dict__: + self._optimizer_load_state_dict_post_hooks = OrderedDict() + self._patch_step_function() # To support multiprocessing pickle/unpickle + self.defaults.setdefault("differentiable", False) + + def __repr__(self) -> str: # noqa: D105 + format_string = self.__class__.__name__ + " (" + for i, group in enumerate(self.param_groups): + format_string += "\n" + format_string += f"Parameter Group {i}\n" + for key in sorted(group.keys()): + if key != "params": + format_string += f" {key}: {group[key]}\n" + format_string += ")" + return format_string + + # Currently needed by Adam and AdamW + def _cuda_graph_capture_health_check(self) -> None: + # Note [torch.compile x capturable] + # If we are compiling, we try to take the capturable path automatically by + # setting the flag to True during tracing. Due to this, we skip all the checks + # normally required for determining whether we can use CUDA graphs and + # shunt the responsibility to torch.inductor. This saves time during tracing + # since the checks are slow without sacrificing UX since inductor will warn + # later if CUDA graphs cannot be enabled, e.g., + # https://github.com/pytorch/pytorch/blob/d3ba8901d8640eb16f88b2bfef9df7fa383d4b47/torch/_inductor/compile_fx.py#L390. + # Thus, when compiling, inductor will determine if cudagraphs + # can be enabled based on whether there is input mutation or CPU tensors. + if ( + not torch.compiler.is_compiling() + and torch.backends.cuda.is_built() + and torch.cuda.is_available() + ): + capturing = torch.cuda.is_current_stream_capturing() + + if capturing and not all( + group["capturable"] for group in self.param_groups + ): + raise RuntimeError( + "Attempting CUDA graph capture of step() for an instance of " + + self.__class__.__name__ + + " but param_groups' capturable is False." + ) + + if ( + (not getattr(self, "_warned_capturable_if_run_uncaptured", False)) + and all(group["capturable"] for group in self.param_groups) + and (not capturing) + ): + warnings.warn( + "This instance was constructed with capturable=True or some of all the param_groups came with capturable=True, " + "but step() is running without CUDA graph capture. If you never intend to graph-capture this " + "instance, capturable=True can impair performance, and you should set capturable=False.", + stacklevel=2, + ) + self._warned_capturable_if_run_uncaptured = True + + def _optimizer_step_code(self) -> None: + """Entry point for `torch.profile.profiler`. + + When python tracing is enabled the profiler will hook into this + function at the CPython level to inspect the optimizer's parameters and + param groups. It is called it after `step()` since many optimizers + lazily initialize state. + + This is a workaround due to lack of a proper step hook on the optimizer, + and will be removed if it exists. + """ + + @staticmethod + def profile_hook_step(func: Callable[_P, R]) -> Callable[_P, R]: # noqa: D102 + @functools.wraps(func) + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> R: + self, *_ = args + self = cast(Optimizer, self) + profile_name = f"Optimizer.step#{self.__class__.__name__}.step" + with torch.autograd.profiler.record_function(profile_name): + # call optimizer step pre hooks + for pre_hook in chain( + _global_optimizer_pre_hooks.values(), + self._optimizer_step_pre_hooks.values(), + ): + result = pre_hook(self, args, kwargs) + if result is not None: + if isinstance(result, tuple) and len(result) == 2: + args, kwargs = result # type: ignore[assignment] + else: + raise RuntimeError( + f"{func} must return None or a tuple of (new_args, new_kwargs), but got {result}." + ) + + # pyrefly: ignore [invalid-param-spec] + out = func(*args, **kwargs) + self._optimizer_step_code() + + # call optimizer step post hooks + for post_hook in chain( + self._optimizer_step_post_hooks.values(), + _global_optimizer_post_hooks.values(), + ): + post_hook(self, args, kwargs) + + return out + + return wrapper + + @staticmethod + def _group_tensors_by_device_and_dtype( + tensorlistlist: TensorListList, + with_indices: bool = False, + ) -> ( + dict[tuple[None, None], tuple[TensorListList, Indices]] + | dict[tuple[torch.device, torch.dtype], tuple[TensorListList, Indices]] + ): + """Group a list of lists of tensors by device and dtype. + + Skips this step if we are compiling since this will occur during inductor lowering. + """ + if torch.compiler.is_compiling(): + return {(None, None): (tensorlistlist, list(range(len(tensorlistlist[0]))))} + else: + return _group_tensors_by_device_and_dtype(tensorlistlist, with_indices) # type: ignore[return-value, arg-type] + + def _patch_step_function(self) -> None: + self._zero_grad_profile_name = ( + f"Optimizer.zero_grad#{self.__class__.__name__}.zero_grad" + ) + hooked = getattr(self.__class__.step, "hooked", None) + if not hooked: + self.__class__.step = self.profile_hook_step(self.__class__.step) # type: ignore[assignment] + self.__class__.step.hooked = True # type: ignore[attr-defined] + + def register_step_pre_hook(self, hook: OptimizerPreHook) -> RemovableHandle: + r"""Register an optimizer step pre hook which will be called before optimizer step. + + It should have the following signature:: + + hook(optimizer, args, kwargs) -> None or modified args and kwargs + + The ``optimizer`` argument is the optimizer instance being used. If + args and kwargs are modified by the pre-hook, then the transformed + values are returned as a tuple containing the new_args and new_kwargs. + + Args: + hook (Callable): The user defined hook to be registered. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(self._optimizer_step_pre_hooks) + self._optimizer_step_pre_hooks[handle.id] = hook + return handle + + def register_step_post_hook(self, hook: OptimizerPostHook) -> RemovableHandle: + r"""Register an optimizer step post hook which will be called after optimizer step. + + It should have the following signature:: + + hook(optimizer, args, kwargs) -> None + + The ``optimizer`` argument is the optimizer instance being used. + + Args: + hook (Callable): The user defined hook to be registered. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(self._optimizer_step_post_hooks) + self._optimizer_step_post_hooks[handle.id] = hook + return handle + + def register_state_dict_pre_hook( + self, hook: Callable[["Optimizer"], None], prepend: bool = False + ) -> RemovableHandle: # noqa: D101 + r"""Register a state dict pre-hook which will be called before :meth:`~torch.optim.Optimizer.state_dict` is called. + + It should have the following signature:: + + hook(optimizer) -> None + + The ``optimizer`` argument is the optimizer instance being used. + The hook will be called with argument ``self`` before calling ``state_dict`` on ``self``. + The registered hook can be used to perform pre-processing before the ``state_dict`` + call is made. + + Args: + hook (Callable): The user defined hook to be registered. + prepend (bool): If True, the provided pre ``hook`` will be fired before + all the already registered pre-hooks on ``state_dict``. Otherwise, + the provided ``hook`` will be fired after all the already registered + pre-hooks. (default: False) + + Returns: + :class:`torch.utils.hooks.RemoveableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(self._optimizer_state_dict_pre_hooks) + self._optimizer_state_dict_pre_hooks[handle.id] = hook + if prepend: + self._optimizer_state_dict_pre_hooks.move_to_end(handle.id, last=False) + return handle + + def register_state_dict_post_hook( + self, + hook: Callable[["Optimizer", StateDict], StateDict | None], + prepend: bool = False, + ) -> RemovableHandle: + r"""Register a state dict post-hook which will be called after :meth:`~torch.optim.Optimizer.state_dict` is called. + + It should have the following signature:: + + hook(optimizer, state_dict) -> state_dict or None + + The hook will be called with arguments ``self`` and ``state_dict`` after generating + a ``state_dict`` on ``self``. The hook may modify the state_dict inplace or optionally + return a new one. The registered hook can be used to perform post-processing + on the ``state_dict`` before it is returned. + + Args: + hook (Callable): The user defined hook to be registered. + prepend (bool): If True, the provided post ``hook`` will be fired before + all the already registered post-hooks on ``state_dict``. Otherwise, + the provided ``hook`` will be fired after all the already registered + post-hooks. (default: False) + + Returns: + :class:`torch.utils.hooks.RemoveableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(self._optimizer_state_dict_post_hooks) + self._optimizer_state_dict_post_hooks[handle.id] = hook + if prepend: + self._optimizer_state_dict_post_hooks.move_to_end(handle.id, last=False) + return handle + + @torch._disable_dynamo + def state_dict(self) -> StateDict: + r"""Return the state of the optimizer as a :class:`dict`. + + It contains two entries: + + * ``state``: a Dict holding current optimization state. Its content + differs between optimizer classes, but some common characteristics + hold. For example, state is saved per parameter, and the parameter + itself is NOT saved. ``state`` is a Dictionary mapping parameter ids + to a Dict with state corresponding to each parameter. + * ``param_groups``: a List containing all parameter groups where each + parameter group is a Dict. Each parameter group contains metadata + specific to the optimizer, such as learning rate and weight decay, + as well as a List of parameter IDs of the parameters in the group. + If a param group was initialized with ``named_parameters()`` the names + content will also be saved in the state dict. + + NOTE: The parameter IDs may look like indices but they are just IDs + associating state with param_group. When loading from a state_dict, + the optimizer will zip the param_group ``params`` (int IDs) and the + optimizer ``param_groups`` (actual ``nn.Parameter`` s) in order to + match state WITHOUT additional verification. + + A returned state dict might look something like: + + .. code-block:: text + + { + 'state': { + 0: {'momentum_buffer': tensor(...), ...}, + 1: {'momentum_buffer': tensor(...), ...}, + 2: {'momentum_buffer': tensor(...), ...}, + 3: {'momentum_buffer': tensor(...), ...} + }, + 'param_groups': [ + { + 'lr': 0.01, + 'weight_decay': 0, + ... + 'params': [0] + 'param_names' ['param0'] (optional) + }, + { + 'lr': 0.001, + 'weight_decay': 0.5, + ... + 'params': [1, 2, 3] + 'param_names': ['param1', 'layer.weight', 'layer.bias'] (optional) + } + ] + } + + """ + for pre_hook in self._optimizer_state_dict_pre_hooks.values(): + pre_hook(self) + + # Save order indices instead of Tensors + param_mappings: dict[int, int] = {} + start_index = 0 + + def pack_group(group: dict[str, Any]) -> dict[str, Any]: + nonlocal start_index + packed = {k: v for k, v in group.items() if k != "params"} + param_mappings.update( + { + id(p): i + for i, p in enumerate(group["params"], start_index) + if id(p) not in param_mappings + } + ) + packed["params"] = [param_mappings[id(p)] for p in group["params"]] + start_index += len(packed["params"]) + return packed + + param_groups = [pack_group(g) for g in self.param_groups] + # Remap state to use order indices as keys + packed_state = { + (param_mappings[id(k)] if isinstance(k, torch.Tensor) else k): v + for k, v in self.state.items() + } + + state_dict = { + "state": packed_state, + "param_groups": param_groups, + } + + for post_hook in self._optimizer_state_dict_post_hooks.values(): + hook_result = post_hook(self, state_dict) + if hook_result is not None: + state_dict = hook_result + return state_dict + + @staticmethod + def _process_value_according_to_param_policy( + param: torch.Tensor, + value: torch.Tensor, + param_id: int, + param_groups: list[dict[Any, Any]], + key: Hashable = None, + ) -> torch.Tensor: + # Floating-point types are a bit special here. They are the only ones + # that are assumed to always match the type of params. + # Make sure state['step'] is not casted https://github.com/pytorch/pytorch/issues/74424 + # UNLESS fused or capturable, see note [special device hosting for step] + fused = False + capturable = False + if param_groups is None: + raise AssertionError("Expected param_groups to be set") + for pg in param_groups: + if param_id in pg["params"]: + fused = pg.get("fused", False) + capturable = pg.get("capturable", False) + break + if key == "step": + if capturable or fused: + return value.to(dtype=torch.float32, device=param.device) + else: + return value + else: + if param.is_floating_point(): + return value.to(dtype=param.dtype, device=param.device) + else: + return value.to(device=param.device) + + def register_load_state_dict_pre_hook( + self, + hook: Callable[["Optimizer", StateDict], StateDict | None], + prepend: bool = False, + ) -> RemovableHandle: # noqa: D205 D400 + r"""Register a load_state_dict pre-hook which will be called before + :meth:`~torch.optim.Optimizer.load_state_dict` is called. It should have the + following signature:: + + hook(optimizer, state_dict) -> state_dict or None + + The ``optimizer`` argument is the optimizer instance being used and the + ``state_dict`` argument is a shallow copy of the ``state_dict`` the user + passed in to ``load_state_dict``. The hook may modify the state_dict inplace + or optionally return a new one. If a state_dict is returned, it will be used + to be loaded into the optimizer. + + The hook will be called with argument ``self`` and ``state_dict`` before + calling ``load_state_dict`` on ``self``. The registered hook can be used to + perform pre-processing before the ``load_state_dict`` call is made. + + Args: + hook (Callable): The user defined hook to be registered. + prepend (bool): If True, the provided pre ``hook`` will be fired before + all the already registered pre-hooks on ``load_state_dict``. Otherwise, + the provided ``hook`` will be fired after all the already registered + pre-hooks. (default: False) + + Returns: + :class:`torch.utils.hooks.RemoveableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(self._optimizer_load_state_dict_pre_hooks) + self._optimizer_load_state_dict_pre_hooks[handle.id] = hook + if prepend: + self._optimizer_load_state_dict_pre_hooks.move_to_end(handle.id, last=False) + return handle + + def register_load_state_dict_post_hook( + self, hook: Callable[["Optimizer"], None], prepend: bool = False + ) -> RemovableHandle: # noqa: D205 D400 + r"""Register a load_state_dict post-hook which will be called after + :meth:`~torch.optim.Optimizer.load_state_dict` is called. It should have the + following signature:: + + hook(optimizer) -> None + + The ``optimizer`` argument is the optimizer instance being used. + + The hook will be called with argument ``self`` after calling + ``load_state_dict`` on ``self``. The registered hook can be used to + perform post-processing after ``load_state_dict`` has loaded the + ``state_dict``. + + Args: + hook (Callable): The user defined hook to be registered. + prepend (bool): If True, the provided post ``hook`` will be fired before + all the already registered post-hooks on ``load_state_dict``. Otherwise, + the provided ``hook`` will be fired after all the already registered + post-hooks. (default: False) + + Returns: + :class:`torch.utils.hooks.RemoveableHandle`: + a handle that can be used to remove the added hook by calling + ``handle.remove()`` + """ + handle = hooks.RemovableHandle(self._optimizer_load_state_dict_post_hooks) + self._optimizer_load_state_dict_post_hooks[handle.id] = hook + if prepend: + self._optimizer_load_state_dict_post_hooks.move_to_end( + handle.id, last=False + ) # type: ignore[attr-defined] + return handle + + @torch._disable_dynamo + def load_state_dict(self, state_dict: StateDict) -> None: + r"""Load the optimizer state. + + Args: + state_dict (dict): optimizer state. Should be an object returned + from a call to :meth:`state_dict`. + + .. warning:: + Make sure this method is called after initializing :class:`torch.optim.lr_scheduler.LRScheduler`, + as calling it beforehand will overwrite the loaded learning rates. + + .. note:: + The names of the parameters (if they exist under the "param_names" key of each param group + in :meth:`state_dict`) will not affect the loading process. + To use the parameters' names for custom cases (such as when the parameters in the loaded state dict + differ from those initialized in the optimizer), + a custom ``register_load_state_dict_pre_hook`` should be implemented to adapt the loaded dict + accordingly. + If ``param_names`` exist in loaded state dict ``param_groups`` they will be saved and override + the current names, if present, in the optimizer state. If they do not exist in loaded state dict, + the optimizer ``param_names`` will remain unchanged. + + Example: + >>> # xdoctest: +SKIP + >>> model = torch.nn.Linear(10, 10) + >>> optim = torch.optim.SGD(model.parameters(), lr=3e-4) + >>> scheduler1 = torch.optim.lr_scheduler.LinearLR( + ... optim, + ... start_factor=0.1, + ... end_factor=1, + ... total_iters=20, + ... ) + >>> scheduler2 = torch.optim.lr_scheduler.CosineAnnealingLR( + ... optim, + ... T_max=80, + ... eta_min=3e-5, + ... ) + >>> lr = torch.optim.lr_scheduler.SequentialLR( + ... optim, + ... schedulers=[scheduler1, scheduler2], + ... milestones=[20], + ... ) + >>> lr.load_state_dict(torch.load("./save_seq.pt")) + >>> # now load the optimizer checkpoint after loading the LRScheduler + >>> optim.load_state_dict(torch.load("./save_optim.pt")) + + """ + # shallow copy, to be consistent with module API + state_dict = state_dict.copy() + + for pre_hook in self._optimizer_load_state_dict_pre_hooks.values(): + hook_result = pre_hook(self, state_dict) + if hook_result is not None: + state_dict = hook_result + + # Validate the state_dict + groups = self.param_groups + + # Deepcopy as we write into saved_groups later to update state + saved_groups = deepcopy(state_dict["param_groups"]) + + if len(groups) != len(saved_groups): + raise ValueError( + "loaded state dict has a different number of parameter groups" + ) + param_lens = (len(g["params"]) for g in groups) + saved_lens = (len(g["params"]) for g in saved_groups) + if any( + p_len != s_len for p_len, s_len in zip(param_lens, saved_lens, strict=True) + ): + raise ValueError( + "loaded state dict contains a parameter group " + "that doesn't match the size of optimizer's group" + ) + + # Update the state + id_map = dict( + zip( + chain.from_iterable(g["params"] for g in saved_groups), + chain.from_iterable(g["params"] for g in groups), + strict=True, + ) + ) + + def _cast(param, value, param_id=None, param_groups=None, key=None): + r"""Make a deep copy of value, casting all tensors to device of param.""" + if isinstance(value, torch.Tensor): + return Optimizer._process_value_according_to_param_policy( + param, + value, + # pyrefly: ignore [bad-argument-type] + param_id, + # pyrefly: ignore [bad-argument-type] + param_groups, + key, + ) + elif isinstance(value, dict): + return { + k: _cast( + param, v, param_id=param_id, param_groups=param_groups, key=k + ) + for k, v in value.items() + } + elif isinstance(value, Iterable): + return type(value)( + # pyrefly: ignore [bad-argument-count] + _cast(param, v, param_id=param_id, param_groups=param_groups) + for v in value + ) # type: ignore[call-arg] + else: + return value + + # Copy state assigned to params (and cast tensors to appropriate types). + # State that is not assigned to params is copied as is (needed for + # backward compatibility). + state: defaultdict[torch.Tensor, dict[Any, Any]] = defaultdict(dict) + for k, v in state_dict["state"].items(): + if k in id_map: + param = id_map[k] + state[param] = _cast( + param, v, param_id=k, param_groups=state_dict["param_groups"] + ) + else: + state[k] = v + + # Update parameter groups, setting their 'params' value + def update_group( + group: dict[str, Any], new_group: dict[str, Any] + ) -> dict[str, Any]: + new_group["params"] = group["params"] + if "param_names" in group and "param_names" not in new_group: + new_group["param_names"] = group["param_names"] + return new_group + + param_groups = [ + update_group(g, ng) for g, ng in zip(groups, saved_groups, strict=True) + ] + self.__setstate__({"state": state, "param_groups": param_groups}) + + for post_hook in self._optimizer_load_state_dict_post_hooks.values(): + post_hook(self) + + @torch._disable_dynamo + def zero_grad(self, set_to_none: bool = True) -> None: + r"""Reset the gradients of all optimized :class:`torch.Tensor` s. + + Args: + set_to_none (bool, optional): Instead of setting to zero, set the grads to None. Default: ``True`` + + This will in general have lower memory footprint, and can modestly improve performance. + However, it changes certain behaviors. For example: + + 1. When the user tries to access a gradient and perform manual ops on it, + a None attribute or a Tensor full of 0s will behave differently. + 2. If the user requests ``zero_grad(set_to_none=True)`` followed by a backward pass, ``.grad``\ s + are guaranteed to be None for params that did not receive a gradient. + 3. ``torch.optim`` optimizers have a different behavior if the gradient is 0 or None + (in one case it does the step with a gradient of 0 and in the other it skips + the step altogether). + """ + foreach = self.defaults.get("foreach", False) or self.defaults.get( + "fused", False + ) + + if not hasattr(self, "_zero_grad_profile_name"): + self._patch_step_function() + + per_device_and_dtype_grads: ( + defaultdict[torch.device, defaultdict[torch.dtype, list[torch.Tensor]]] + | None + ) + if foreach: + per_device_and_dtype_grads = defaultdict(lambda: defaultdict(list)) + else: + per_device_and_dtype_grads = None + + with torch.autograd.profiler.record_function(self._zero_grad_profile_name): + for group in self.param_groups: + for p in group["params"]: + if p.grad is not None: + if set_to_none: + p.grad = None + else: + if p.grad.grad_fn is not None: + p.grad.detach_() + else: + p.grad.requires_grad_(False) + if not foreach or p.grad.is_sparse: + p.grad.zero_() + else: + if per_device_and_dtype_grads is None: + raise AssertionError( + "Expected per_device_and_dtype_grads to be set" + ) + per_device_and_dtype_grads[p.grad.device][ + p.grad.dtype + ].append(p.grad) + if foreach: + if per_device_and_dtype_grads is None: + raise AssertionError( + "Expected per_device_and_dtype_grads to be set" + ) + for per_dtype_grads in per_device_and_dtype_grads.values(): + for grads in per_dtype_grads.values(): + torch._foreach_zero_(grads) + + @overload + def step(self, closure: None = None) -> None: ... + + @overload + def step(self, closure: Callable[[], float]) -> float: ... + + def step(self, closure: Callable[[], float] | None = None) -> float | None: + r"""Perform a single optimization step to update parameter. + + Args: + closure (Callable): A closure that reevaluates the model and + returns the loss. Optional for most optimizers. + """ + raise NotImplementedError + + @torch._disable_dynamo + def add_param_group(self, param_group: dict[str, Any]) -> None: + r"""Add a param group to the :class:`Optimizer` s `param_groups`. + + This can be useful when fine tuning a pre-trained network as frozen layers can be made + trainable and added to the :class:`Optimizer` as training progresses. + + Args: + param_group (dict): Specifies what Tensors should be optimized along with group + specific optimization options. + """ + if not isinstance(param_group, dict): + raise TypeError(f"param_group must be a dict, but got {type(param_group)}") + + params = param_group["params"] + if isinstance(params, torch.Tensor): + param_group["params"] = [params] + elif isinstance(params, set): + raise TypeError( + "optimizer parameters need to be organized in ordered collections, but " + "the ordering of tensors in sets will change between runs. Please use a list instead." + ) + else: + param_group["params"] = list(params) + + extracted_param_tensors = [] + extracted_param_names = [] + for param in param_group["params"]: + if isinstance(param, tuple): + param_name = param[0] + extracted_param_names.append(param_name) + extracted_param_tensors.append(param[1]) + else: + extracted_param_tensors.append(param) + + param_group["params"] = extracted_param_tensors + if len(extracted_param_names) != 0: + if len(extracted_param_names) == len(extracted_param_tensors): + param_group["param_names"] = extracted_param_names + else: + raise ValueError( + "all optimizer params should be with/without names. Some param names are missing" + ) + + for param in param_group["params"]: + if not isinstance(param, torch.Tensor): + raise TypeError( + "optimizer can only optimize Tensors, " + "but one of the params is " + torch.typename(param) + ) + if not self.defaults.get("differentiable", None) and not ( + param.is_leaf or param.retains_grad + ): + raise ValueError("can't optimize a non-leaf Tensor") + + for name, default in self.defaults.items(): + if default is required and name not in param_group: + raise ValueError( + f"parameter group didn't specify a value of required optimization parameter {name}" + ) + else: + param_group.setdefault(name, default) + + params = param_group["params"] + if len(params) != len(set(params)): + warnings.warn( + "optimizer contains a parameter group with duplicate parameters; " + "in future, this will cause an error; " + "see github.com/pytorch/pytorch/issues/40967 for more information", + stacklevel=3, + ) + + param_set: set[torch.Tensor] = set() + for group in self.param_groups: + param_set.update(set(group["params"])) + if ("param_names" in param_group) != ("param_names" in group): + current_group_txt = ( + "with names" if "param_names" in param_group else "without names" + ) + raise ValueError( + "all optimizer param groups should be with/without names. " + f"cannot add param group {current_group_txt} to the optimizer" + ) + + if not param_set.isdisjoint(set(param_group["params"])): + raise ValueError("some parameters appear in more than one parameter group") + + self.param_groups.append(param_group) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/radam.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/radam.py new file mode 100644 index 0000000000000000000000000000000000000000..c54b2bb83db31948b9503df921a4a18d15ce2114 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/radam.py @@ -0,0 +1,628 @@ +# mypy: allow-untyped-defs +r"""Implementation for the RAdam algorithm.""" + +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _get_value, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["RAdam", "radam"] + + +class RAdam(Optimizer): # noqa: D101 + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-3, + betas: tuple[float, float] = (0.9, 0.999), + eps: float = 1e-8, + weight_decay: float = 0, + decoupled_weight_decay: bool = False, + *, + foreach: bool | None = None, + maximize: bool = False, + capturable: bool = False, + differentiable: bool = False, + ) -> None: # noqa: D107 + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= betas[0] < 1.0: + raise ValueError(f"Invalid beta parameter at index 0: {betas[0]}") + if not 0.0 <= betas[1] < 1.0: + raise ValueError(f"Invalid beta parameter at index 1: {betas[1]}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + + defaults = { + "lr": lr, + "betas": betas, + "eps": eps, + "weight_decay": weight_decay, + "maximize": maximize, + "foreach": foreach, + "capturable": capturable, + "decoupled_weight_decay": decoupled_weight_decay, + "differentiable": differentiable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): # noqa: D105 + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + group.setdefault("decoupled_weight_decay", False) + group.setdefault("capturable", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + + def _init_group( + self, group, params_with_grad, grads, exp_avgs, exp_avg_sqs, state_steps + ): + has_complex = False + for p in group["params"]: + if p.grad is not None: + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + if p.grad.is_sparse: + raise RuntimeError("RAdam does not support sparse gradients") + grads.append(p.grad) + + state = self.state[p] + # Lazy state initialization + if len(state) == 0: + state["step"] = ( + torch.zeros((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.tensor(0.0, dtype=_get_scalar_dtype()) + ) + # Exponential moving average of gradient values + state["exp_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + # Exponential moving average of squared gradient values + state["exp_avg_sq"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + exp_avgs.append(state["exp_avg"]) + exp_avg_sqs.append(state["exp_avg_sq"]) + state_steps.append(state["step"]) + + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + exp_avgs: list[Tensor] = [] + exp_avg_sqs: list[Tensor] = [] + state_steps: list[Tensor] = [] + beta1, beta2 = cast(tuple[float, float], group["betas"]) + + has_complex = self._init_group( + group, params_with_grad, grads, exp_avgs, exp_avg_sqs, state_steps + ) + + radam( + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + state_steps, + beta1=beta1, + beta2=beta2, + lr=group["lr"], + weight_decay=group["weight_decay"], + eps=group["eps"], + maximize=group["maximize"], + foreach=group["foreach"], + capturable=group["capturable"], + differentiable=group["differentiable"], + decoupled_weight_decay=group["decoupled_weight_decay"], + has_complex=has_complex, + ) + + return loss + + +RAdam.__doc__ = ( + r"""Implements RAdam algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)}, \: \beta_1, \beta_2 + \text{ (betas)}, \: \theta_0 \text{ (params)}, \:f(\theta) \text{ (objective)}, \: + \lambda \text{ (weightdecay)}, \:\textit{maximize} \\ + &\hspace{13mm} \epsilon \text{ (epsilon)}, \textit{decoupled\_weight\_decay} \\ + &\textbf{initialize} : m_0 \leftarrow 0 \text{ ( first moment)}, + v_0 \leftarrow 0 \text{ ( second moment)}, \\ + &\hspace{18mm} \rho_{\infty} \leftarrow 2/(1-\beta_2) -1 \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{6mm}\textbf{if} \: \textit{maximize}: \\ + &\hspace{12mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{6mm}\textbf{else} \\ + &\hspace{12mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{6mm} \theta_t \leftarrow \theta_{t-1} \\ + &\hspace{6mm} \textbf{if} \: \lambda \neq 0 \\ + &\hspace{12mm}\textbf{if} \: \textit{decoupled\_weight\_decay} \\ + &\hspace{18mm} \theta_t \leftarrow \theta_{t} - \gamma \lambda \theta_{t} \\ + &\hspace{12mm}\textbf{else} \\ + &\hspace{18mm} g_t \leftarrow g_t + \lambda \theta_{t} \\ + &\hspace{6mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ + &\hspace{6mm}v_t \leftarrow \beta_2 v_{t-1} + (1-\beta_2) g^2_t \\ + &\hspace{6mm}\widehat{m_t} \leftarrow m_t/\big(1-\beta_1^t \big) \\ + &\hspace{6mm}\rho_t \leftarrow \rho_{\infty} - + 2 t \beta^t_2 /\big(1-\beta_2^t \big) \\[0.1.ex] + &\hspace{6mm}\textbf{if} \: \rho_t > 5 \\ + &\hspace{12mm} l_t \leftarrow \frac{\sqrt{ (1-\beta^t_2) }}{ \sqrt{v_t} +\epsilon } \\ + &\hspace{12mm} r_t \leftarrow + \sqrt{\frac{(\rho_t-4)(\rho_t-2)\rho_{\infty}}{(\rho_{\infty}-4)(\rho_{\infty}-2) \rho_t}} \\ + &\hspace{12mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t} r_t l_t \\ + &\hspace{6mm}\textbf{else} \\ + &\hspace{12mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t} \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to `On the variance of the adaptive learning rate and beyond`_. + + This implementation provides an option to use either the original weight_decay implementation as in Adam + (where the weight_decay is applied to the gradient) or the one from AdamW (where weight_decay is applied + to the weight) through the decoupled_weight_decay option. When decoupled_weight_decay is set to False + (default), it uses the original Adam style weight decay, otherwise, it uses the AdamW style which + corresponds more closely to the `author's implementation`_ in the RAdam paper. Further information + about decoupled weight decay can be found in `Decoupled Weight Decay Regularization`_. + + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + decoupled_weight_decay (bool, optional): whether to decouple the weight + decay as in AdamW to obtain RAdamW. If True, the algorithm does not + accumulate weight decay in the momentum nor variance. (default: False) + {_foreach_doc} + {_maximize_doc} + {_capturable_doc} + {_differentiable_doc} + + .. _On the variance of the adaptive learning rate and beyond: + https://arxiv.org/abs/1908.03265 + .. _author's implementation: + https://github.com/LiyuanLucasLiu/RAdam + .. _Decoupled Weight Decay Regularization: + https://arxiv.org/abs/1711.05101 + + """ +) + + +def _single_tensor_radam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + *, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + eps: float, + decoupled_weight_decay: bool, + differentiable: bool, + maximize: bool, + capturable: bool, + has_complex: bool, +) -> None: + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for i, param in enumerate(params): + grad = grads[i] if not maximize else -grads[i] + exp_avg = exp_avgs[i] + exp_avg_sq = exp_avg_sqs[i] + step_t = state_steps[i] + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type == step_t.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + if torch.is_complex(param): + param = torch.view_as_real(param) + grad = torch.view_as_real(grad) + exp_avg = torch.view_as_real(exp_avg) + exp_avg_sq = torch.view_as_real(exp_avg_sq) + + # update step + step_t += 1 + step = step_t if capturable else _get_value(step_t) + + if weight_decay != 0: + if decoupled_weight_decay: + param.mul_(1 - lr * weight_decay) + else: + grad = grad.add(param, alpha=weight_decay) + + # Decay the first and second moment running average coefficient + exp_avg.lerp_(grad, 1 - beta1) + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) + + bias_correction1 = 1 - beta1**step + bias_correction2 = 1 - beta2**step + + # correcting bias for the first moving moment + bias_corrected_exp_avg = exp_avg / bias_correction1 + + # maximum length of the approximated SMA + rho_inf = 2 / (1 - beta2) - 1 + # compute the length of the approximated SMA + rho_t = rho_inf - 2 * step * (beta2**step) / bias_correction2 + + def _compute_rect(): + # pyrefly: ignore [unsupported-operation] + return ( + (rho_t - 4) + * (rho_t - 2) + * rho_inf + / ((rho_inf - 4) * (rho_inf - 2) * rho_t) + ) ** 0.5 + + def _compute_adaptive_lr(): + exp_avg_sq_sqrt = exp_avg_sq.sqrt() + if differentiable: + exp_avg_sq_sqrt = exp_avg_sq_sqrt.add(eps) + else: + exp_avg_sq_sqrt = exp_avg_sq_sqrt.add_(eps) + + # pyrefly: ignore [unsupported-operation] + return (bias_correction2**0.5) / exp_avg_sq_sqrt + + # Compute the variance rectification term and update parameters accordingly + if capturable: + update = torch.where( + rho_t > 5.0, _compute_rect() * _compute_adaptive_lr(), 1.0 + ) + param.add_(bias_corrected_exp_avg * lr * update, alpha=-1.0) + else: + if rho_t > 5.0: + param.add_( + bias_corrected_exp_avg + * lr + * _compute_adaptive_lr() + * _compute_rect(), + alpha=-1.0, + ) + else: + param.add_(bias_corrected_exp_avg * lr, alpha=-1.0) + + +def _multi_tensor_radam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + *, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + eps: float, + decoupled_weight_decay: bool, + differentiable: bool, + maximize: bool, + capturable: bool, + has_complex: bool, +) -> None: + if len(params) == 0: + return + + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices( + supports_xla=False + ) + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, exp_avgs, exp_avg_sqs, state_steps] # type: ignore[list-item] + ) + for ( + grouped_params_, + grouped_grads_, + grouped_exp_avgs_, + grouped_exp_avg_sqs_, + grouped_state_steps_, + ), _ in grouped_tensors.values(): + grouped_params = cast(list[Tensor], grouped_params_) + grouped_grads = cast(list[Tensor], grouped_grads_) + grouped_exp_avgs = cast(list[Tensor], grouped_exp_avgs_) + grouped_exp_avg_sqs = cast(list[Tensor], grouped_exp_avg_sqs_) + grouped_state_steps = cast(list[Tensor], grouped_state_steps_) + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and grouped_state_steps[0].is_cpu: + torch._foreach_add_( + grouped_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(grouped_state_steps, 1) + + if has_complex: + _view_as_real( + grouped_params, grouped_grads, grouped_exp_avgs, grouped_exp_avg_sqs + ) + + if maximize: + grouped_grads = torch._foreach_neg(grouped_grads) # type: ignore[assignment] + + # maximum length of the approximated SMA + rho_inf = 2 / (1 - beta2) - 1 + # compute the length of the approximated SMA + bias_correction1: tuple[Tensor, ...] | list[Tensor] + bias_correction2: tuple[Tensor, ...] | list[Tensor] + rho_t_list: tuple[Tensor, ...] | list[Tensor] + if capturable: + bias_correction1 = torch._foreach_pow(beta2, grouped_state_steps) + torch._foreach_neg_(bias_correction1) + torch._foreach_add_(bias_correction1, 1) + bias_correction2 = torch._foreach_pow(beta2, grouped_state_steps) + torch._foreach_mul_(bias_correction2, grouped_state_steps) + torch._foreach_mul_(bias_correction2, 2) + torch._foreach_div_(bias_correction2, bias_correction1) + torch._foreach_neg_(bias_correction2) + torch._foreach_add_(bias_correction2, rho_inf) + rho_t_list = bias_correction2 + else: + rho_t_list = [ + rho_inf + - 2 + * _get_value(step) + * (beta2 ** _get_value(step)) + / (1 - beta2 ** _get_value(step)) + for step in grouped_state_steps + ] + + if weight_decay != 0: + if decoupled_weight_decay: + torch._foreach_mul_(grouped_params, 1 - lr * weight_decay) + else: + # Reuse the intermediate memory (grouped_grads) already allocated for maximize + if maximize: + torch._foreach_add_( + grouped_grads, grouped_params, alpha=weight_decay + ) + else: + grouped_grads = torch._foreach_add( # type: ignore[assignment] + grouped_grads, grouped_params, alpha=weight_decay + ) + + # Decay the first and second moment running average coefficient + torch._foreach_lerp_(grouped_exp_avgs, grouped_grads, 1 - beta1) + + torch._foreach_mul_(grouped_exp_avg_sqs, beta2) + torch._foreach_addcmul_( + grouped_exp_avg_sqs, grouped_grads, grouped_grads, 1 - beta2 + ) + + # Delete the local intermediate since it won't be used anymore to save on peak memory + del grouped_grads + + if capturable: + num = torch._foreach_sub(rho_t_list, 4) + sub2 = torch._foreach_sub(rho_t_list, 2) + torch._foreach_mul_(num, sub2) + del sub2 + torch._foreach_mul_(num, rho_inf) + rho_inf = (rho_inf - 4) * (rho_inf - 2) + denom = torch._foreach_mul(rho_t_list, rho_inf) + torch._foreach_div_(num, denom) + del denom + torch._foreach_sqrt_(num) + + # TODO(mlazos): we should try and get a foreach_where op https://github.com/pytorch/pytorch/issues/117884 + rect = [ + torch.where(rho_t > 5.0, n, 0.0) + for n, rho_t in zip(num, rho_t_list, strict=True) + ] + del num + del rho_t_list + unrect_step_size = [torch.where(rect > 0, 0.0, 1.0) for rect in rect] + torch._foreach_mul_(unrect_step_size, lr) + + bias_correction1 = torch._foreach_pow(beta1, grouped_state_steps) + torch._foreach_neg_(bias_correction1) + torch._foreach_add_(bias_correction1, 1) + + torch._foreach_div_(unrect_step_size, bias_correction1) + torch._foreach_neg_(unrect_step_size) + + bias_correction2 = torch._foreach_pow(beta2, grouped_state_steps) + torch._foreach_neg_(bias_correction2) + torch._foreach_add_(bias_correction2, 1) + torch._foreach_sqrt_(bias_correction2) + torch._foreach_mul_(bias_correction2, lr) + torch._foreach_mul_(bias_correction2, rect) + del rect + torch._foreach_neg_(bias_correction2) + torch._foreach_div_(bias_correction2, bias_correction1) + del bias_correction1 + else: + rect = [ + ( # type: ignore[misc] + (rho_t - 4) # type: ignore[arg-type] + * (rho_t - 2) + * rho_inf + / ((rho_inf - 4) * (rho_inf - 2) * rho_t) + ) + ** 0.5 + if rho_t > 5 + else 0 + for rho_t in rho_t_list + ] + unrectified = [0 if rect > 0 else 1.0 for rect in rect] + + bias_correction1 = [ + 1 - beta1 ** _get_value(step) for step in grouped_state_steps + ] + unrect_step_size = [ + (lr * rect / bc) * -1 + for rect, bc in zip(unrectified, bias_correction1, strict=True) + ] + bias_correction2 = [ + ((1 - beta2 ** _get_value(step)) ** 0.5) * (lr * rect / bc) * -1 + for step, rect, bc in zip( + grouped_state_steps, rect, bias_correction1, strict=True + ) + ] + + buffer = torch._foreach_sqrt(grouped_exp_avg_sqs) + torch._foreach_add_(buffer, eps) + torch._foreach_div_(buffer, bias_correction2) + torch._foreach_reciprocal_(buffer) + torch._foreach_add_(buffer, unrect_step_size) + + # Here, buffer = sqrt(1 - beta2^t) * rect_step_size / (sqrt(v) + eps) + unrect_step_size + torch._foreach_addcmul_(grouped_params, grouped_exp_avgs, buffer) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_radam) +def radam( + params: list[Tensor], + grads: list[Tensor], + exp_avgs: list[Tensor], + exp_avg_sqs: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + decoupled_weight_decay: bool = False, + foreach: bool | None = None, + differentiable: bool = False, + capturable: bool = False, + has_complex: bool = False, + maximize: bool = False, + *, + beta1: float, + beta2: float, + lr: float, + weight_decay: float, + eps: float, +) -> None: + r"""Functional API that performs RAdam algorithm computation. + + See :class:`~torch.optim.RAdam` for details. + """ + if not all(isinstance(t, torch.Tensor) for t in state_steps): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_radam + else: + func = _single_tensor_radam + + func( + params, + grads, + exp_avgs, + exp_avg_sqs, + state_steps, + beta1=beta1, + beta2=beta2, + lr=lr, + weight_decay=weight_decay, + eps=eps, + maximize=maximize, + decoupled_weight_decay=decoupled_weight_decay, + differentiable=differentiable, + capturable=capturable, + has_complex=has_complex, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/rmsprop.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/rmsprop.py new file mode 100644 index 0000000000000000000000000000000000000000..f8e6da5489d74e0607e629b20a22ecc197f47b78 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/rmsprop.py @@ -0,0 +1,542 @@ +# mypy: allow-untyped-defs +r"""Implementation for the RMSprop algorithm.""" + +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["RMSprop", "rmsprop"] + + +class RMSprop(Optimizer): # noqa: D101 + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-2, + alpha: float = 0.99, + eps: float = 1e-8, + weight_decay: float = 0, + momentum: float = 0, + centered: bool = False, + capturable: bool = False, + foreach: bool | None = None, + maximize: bool = False, + differentiable: bool = False, + ) -> None: # noqa: D107 + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 <= eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= momentum: + raise ValueError(f"Invalid momentum value: {momentum}") + if not 0.0 <= weight_decay: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + if not 0.0 <= alpha: + raise ValueError(f"Invalid alpha value: {alpha}") + + defaults = { + "lr": lr, + "momentum": momentum, + "alpha": alpha, + "eps": eps, + "centered": centered, + "weight_decay": weight_decay, + "capturable": capturable, + "foreach": foreach, + "maximize": maximize, + "differentiable": differentiable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): # noqa: D105 + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("momentum", 0) + group.setdefault("centered", False) + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + group.setdefault("capturable", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + + def _init_group( + self, + group, + params_with_grad, + grads, + square_avgs, + momentum_buffer_list, + grad_avgs, + state_steps, + ): + has_complex = False + for p in group["params"]: + if p.grad is None: + continue + has_complex |= torch.is_complex(p) + params_with_grad.append(p) + + if p.grad.is_sparse: + raise RuntimeError("RMSprop does not support sparse gradients") + grads.append(p.grad) + + state = self.state[p] + + # State initialization + if len(state) == 0: + state["step"] = ( + torch.zeros((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.zeros((), dtype=_get_scalar_dtype()) + ) + state["square_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + if group["momentum"] > 0: + state["momentum_buffer"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + if group["centered"]: + state["grad_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + square_avgs.append(state["square_avg"]) + state_steps.append(state["step"]) + + if group["momentum"] > 0: + momentum_buffer_list.append(state["momentum_buffer"]) + if group["centered"]: + grad_avgs.append(state["grad_avg"]) + + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + square_avgs: list[Tensor] = [] + grad_avgs: list[Tensor] = [] + momentum_buffer_list: list[Tensor] = [] + state_steps: list[Tensor] = [] + + has_complex = self._init_group( + group, + params_with_grad, + grads, + square_avgs, + momentum_buffer_list, + grad_avgs, + state_steps, + ) + + rmsprop( + params_with_grad, + grads, + square_avgs, + grad_avgs, + momentum_buffer_list, + state_steps, + lr=group["lr"], + alpha=group["alpha"], + eps=group["eps"], + weight_decay=group["weight_decay"], + momentum=group["momentum"], + centered=group["centered"], + foreach=group["foreach"], + maximize=group["maximize"], + differentiable=group["differentiable"], + capturable=group["capturable"], + has_complex=has_complex, + ) + + return loss + + +RMSprop.__doc__ = ( + r"""Implements RMSprop algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \alpha \text{ (alpha)}, \: \gamma \text{ (lr)}, + \: \theta_0 \text{ (params)}, \: f(\theta) \text{ (objective)} \\ + &\hspace{13mm} \lambda \text{ (weight decay)},\: \mu \text{ (momentum)}, + \: centered, \: \epsilon \text{ (epsilon)} \\ + &\textbf{initialize} : v_0 \leftarrow 0 \text{ (square average)}, \: + \textbf{b}_0 \leftarrow 0 \text{ (buffer)}, \: g^{ave}_0 \leftarrow 0 \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}if \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm}v_t \leftarrow \alpha v_{t-1} + (1 - \alpha) g^2_t + \hspace{8mm} \\ + &\hspace{5mm} \tilde{v_t} \leftarrow v_t \\ + &\hspace{5mm}if \: centered \\ + &\hspace{10mm} g^{ave}_t \leftarrow g^{ave}_{t-1} \alpha + (1-\alpha) g_t \\ + &\hspace{10mm} \tilde{v_t} \leftarrow \tilde{v_t} - \big(g^{ave}_{t} \big)^2 \\ + &\hspace{5mm}if \: \mu > 0 \\ + &\hspace{10mm} \textbf{b}_t\leftarrow \mu \textbf{b}_{t-1} + + g_t/ \big(\sqrt{\tilde{v_t}} + \epsilon \big) \\ + &\hspace{10mm} \theta_t \leftarrow \theta_{t-1} - \gamma \textbf{b}_t \\ + &\hspace{5mm} else \\ + &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} - + \gamma g_t/ \big(\sqrt{\tilde{v_t}} + \epsilon \big) \hspace{3mm} \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to + `lecture notes `_ by G. Hinton. + and centered version `Generating Sequences + With Recurrent Neural Networks `_. + The implementation here takes the square root of the gradient average before + adding epsilon (note that TensorFlow interchanges these two operations). The effective + learning rate is thus :math:`\gamma/(\sqrt{v} + \epsilon)` where :math:`\gamma` + is the scheduled learning rate and :math:`v` is the weighted moving average + of the squared gradient. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-2) + alpha (float, optional): smoothing constant (default: 0.99) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + momentum (float, optional): momentum factor (default: 0) + centered (bool, optional) : if ``True``, compute the centered RMSProp, + the gradient is normalized by an estimation of its variance + {_capturable_doc} + {_foreach_doc} + {_maximize_doc} + {_differentiable_doc} + + """ +) + + +def _single_tensor_rmsprop( + params: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + grad_avgs: list[Tensor], + momentum_buffer_list: list[Tensor], + state_steps: list[Tensor], + *, + lr: float, + alpha: float, + eps: float, + weight_decay: float, + momentum: float, + centered: bool, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for i, param in enumerate(params): + step = state_steps[i] + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type == step.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + grad = grads[i] + grad = grad if not maximize else -grad + square_avg = square_avgs[i] + + step += 1 + + if weight_decay != 0: + grad = grad.add(param, alpha=weight_decay) + + is_complex_param = torch.is_complex(param) + if is_complex_param: + param = torch.view_as_real(param) + grad = torch.view_as_real(grad) + square_avg = torch.view_as_real(square_avg) + + square_avg.mul_(alpha).addcmul_(grad, grad, value=1 - alpha) + + if centered: + grad_avg = grad_avgs[i] + if is_complex_param: + grad_avg = torch.view_as_real(grad_avg) + grad_avg.lerp_(grad, 1 - alpha) + avg = square_avg.addcmul(grad_avg, grad_avg, value=-1).sqrt_() + else: + avg = square_avg.sqrt() + + if differentiable: + avg = avg.add(eps) + else: + avg = avg.add_(eps) + + if momentum > 0: + buf = momentum_buffer_list[i] + if is_complex_param: + buf = torch.view_as_real(buf) + buf.mul_(momentum).addcdiv_(grad, avg) + param.add_(buf, alpha=-lr) + else: + param.addcdiv_(grad, avg, value=-lr) + + +def _multi_tensor_rmsprop( + params: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + grad_avgs: list[Tensor], + momentum_buffer_list: list[Tensor], + state_steps: list[Tensor], + *, + lr: float, + alpha: float, + eps: float, + weight_decay: float, + momentum: float, + centered: bool, + maximize: bool, + differentiable: bool, + capturable: bool, + has_complex: bool, +) -> None: + if len(params) == 0: + return + + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, square_avgs, grad_avgs, momentum_buffer_list, state_steps] # type: ignore[list-item] + ) + for ( + ( + grouped_params_, + grouped_grads_, + grouped_square_avgs_, + grouped_grad_avgs_, + grouped_momentum_buffer_list_, + grouped_state_steps_, + ) + ), _ in grouped_tensors.values(): + grouped_params = cast(list[Tensor], grouped_params_) + grouped_grads = cast(list[Tensor], grouped_grads_) + grouped_square_avgs = cast(list[Tensor], grouped_square_avgs_) + grouped_state_steps = cast(list[Tensor], grouped_state_steps_) + + if has_complex: + state_and_grads = [grouped_grads, grouped_square_avgs] + if momentum > 0: + grouped_momentum_buffer_list = cast( + list[Tensor], grouped_momentum_buffer_list_ + ) + state_and_grads.append(grouped_momentum_buffer_list) + if centered: + grouped_grad_avgs = cast(list[Tensor], grouped_grad_avgs_) + state_and_grads.append(grouped_grad_avgs) + _view_as_real(grouped_params, *state_and_grads) + + if maximize: + grouped_grads = torch._foreach_neg(grouped_grads) # type: ignore[assignment] + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and grouped_state_steps[0].is_cpu: + torch._foreach_add_( + grouped_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(grouped_state_steps, 1) + + if weight_decay != 0: + # Reuse the intermediate memory (grouped_grads) already allocated for maximize + if maximize: + torch._foreach_add_(grouped_grads, grouped_params, alpha=weight_decay) + else: + grouped_grads = torch._foreach_add( # type: ignore[assignment] + grouped_grads, grouped_params, alpha=weight_decay + ) + + torch._foreach_mul_(grouped_square_avgs, alpha) + torch._foreach_addcmul_( + grouped_square_avgs, grouped_grads, grouped_grads, value=1 - alpha + ) + + if centered: + grouped_grad_avgs = cast(list[Tensor], grouped_grad_avgs_) + torch._foreach_lerp_(grouped_grad_avgs, grouped_grads, 1 - alpha) + avg = torch._foreach_addcmul( + grouped_square_avgs, grouped_grad_avgs, grouped_grad_avgs, value=-1 + ) + torch._foreach_sqrt_(avg) + torch._foreach_add_(avg, eps) + else: + avg = torch._foreach_sqrt(grouped_square_avgs) + torch._foreach_add_(avg, eps) + + if momentum > 0: + grouped_momentum_buffer_list = cast( + list[Tensor], grouped_momentum_buffer_list_ + ) + torch._foreach_mul_(grouped_momentum_buffer_list, momentum) + torch._foreach_addcdiv_(grouped_momentum_buffer_list, grouped_grads, avg) + # If LR is a tensor, the else branch will internally call item() + # which will cause silent incorrectness if we are capturing + if capturable and isinstance(lr, torch.Tensor): + momentum_lr = torch._foreach_mul(grouped_momentum_buffer_list, -lr) + torch._foreach_add_(grouped_params, momentum_lr) + else: + torch._foreach_add_( + grouped_params, grouped_momentum_buffer_list, alpha=-lr + ) + else: + # If LR is a tensor, the else branch will internally call item() + # which will cause silent incorrectness if we are capturing + if capturable and isinstance(lr, torch.Tensor): + torch._foreach_div_(avg, -lr) + torch._foreach_addcdiv_(grouped_params, grouped_grads, avg) + else: + torch._foreach_addcdiv_(grouped_params, grouped_grads, avg, value=-lr) + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_rmsprop) +def rmsprop( + params: list[Tensor], + grads: list[Tensor], + square_avgs: list[Tensor], + grad_avgs: list[Tensor], + momentum_buffer_list: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + maximize: bool = False, + differentiable: bool = False, + capturable: bool = False, + has_complex: bool = False, + *, + lr: float, + alpha: float, + eps: float, + weight_decay: float, + momentum: float, + centered: bool, +) -> None: + r"""Functional API that performs rmsprop algorithm computation. + + See :class:`~torch.optim.RMSProp` for details. + """ + # this check is slow during compilation, so we skip it + # if it's strictly needed we can add this check back in dynamo + if not torch.compiler.is_compiling() and not all( + isinstance(t, torch.Tensor) for t in state_steps + ): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_rmsprop + else: + func = _single_tensor_rmsprop + + func( + params, + grads, + square_avgs, + grad_avgs, + momentum_buffer_list, + state_steps, + lr=lr, + alpha=alpha, + eps=eps, + weight_decay=weight_decay, + momentum=momentum, + centered=centered, + maximize=maximize, + capturable=capturable, + differentiable=differentiable, + has_complex=has_complex, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/rprop.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/rprop.py new file mode 100644 index 0000000000000000000000000000000000000000..dcdc91692b7d37206e2d4048e6404ef6b455c003 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/rprop.py @@ -0,0 +1,472 @@ +# mypy: allow-untyped-defs +r"""Implementation for the Resilient backpropagation.""" + +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _capturable_doc, + _default_to_fused_or_foreach, + _differentiable_doc, + _disable_dynamo_if_unsupported, + _foreach_doc, + _get_capturable_supported_devices, + _get_scalar_dtype, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + _view_as_real, + Optimizer, + ParamsT, +) + + +__all__ = ["Rprop", "rprop"] + + +class Rprop(Optimizer): # noqa: D101 + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-2, + etas: tuple[float, float] = (0.5, 1.2), + step_sizes: tuple[float, float] = (1e-6, 50), + *, + capturable: bool = False, + foreach: bool | None = None, + maximize: bool = False, + differentiable: bool = False, + ) -> None: # noqa: D107 + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 <= lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 < etas[0] < 1.0 < etas[1]: + raise ValueError(f"Invalid eta values: {etas[0]}, {etas[1]}") + + defaults = { + "lr": lr, + "etas": etas, + "step_sizes": step_sizes, + "foreach": foreach, + "maximize": maximize, + "differentiable": differentiable, + "capturable": capturable, + } + super().__init__(params, defaults) + + def __setstate__(self, state): # noqa: D105 + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("foreach", None) + group.setdefault("maximize", False) + group.setdefault("differentiable", False) + group.setdefault("capturable", False) + for p in group["params"]: + p_state = self.state.get(p, []) + if len(p_state) != 0 and not torch.is_tensor(p_state["step"]): + step_val = float(p_state["step"]) + p_state["step"] = ( + torch.tensor( + step_val, dtype=_get_scalar_dtype(), device=p.device + ) + if group["capturable"] + else torch.tensor(step_val, dtype=_get_scalar_dtype()) + ) + + def _init_group(self, group, params, grads, prevs, step_sizes, state_steps): + has_complex = False + for p in group["params"]: + if p.grad is None: + continue + has_complex |= torch.is_complex(p) + params.append(p) + grad = p.grad + if grad.is_sparse: + raise RuntimeError("Rprop does not support sparse gradients") + + grads.append(grad) + state = self.state[p] + + # State initialization + if len(state) == 0: + state["step"] = ( + torch.zeros((), dtype=_get_scalar_dtype(), device=p.device) + if group["capturable"] + else torch.zeros((), dtype=_get_scalar_dtype()) + ) + + state["prev"] = torch.zeros_like(p, memory_format=torch.preserve_format) + if p.dtype.is_complex: + # Complex Number should be as if they are two independent real numbers. + # Hence the step_size shouldn't be zero for imaginary part. + state["step_size"] = torch.full_like( + grad, complex(group["lr"], group["lr"]) + ) + else: + state["step_size"] = torch.full_like(grad, _to_scalar(group["lr"])) + + prevs.append(state["prev"]) + step_sizes.append(state["step_size"]) + state_steps.append(state["step"]) + + return has_complex + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + self._cuda_graph_capture_health_check() + + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params: list[Tensor] = [] + grads: list[Tensor] = [] + prevs: list[Tensor] = [] + step_sizes: list[Tensor] = [] + state_steps: list[Tensor] = [] + + etaminus, etaplus = group["etas"] + step_size_min, step_size_max = group["step_sizes"] + foreach = group["foreach"] + maximize = group["maximize"] + + has_complex = self._init_group( + group, params, grads, prevs, step_sizes, state_steps + ) + + rprop( + params, + grads, + prevs, + step_sizes, + state_steps, + step_size_min=step_size_min, + step_size_max=step_size_max, + etaminus=etaminus, + etaplus=etaplus, + foreach=foreach, + maximize=maximize, + differentiable=group["differentiable"], + capturable=group["capturable"], + has_complex=has_complex, + ) + + return loss + + +Rprop.__doc__ = ( + r"""Implements the resilient backpropagation algorithm. + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \theta_0 \in \mathbf{R}^d \text{ (params)},f(\theta) + \text{ (objective)}, \\ + &\hspace{13mm} \eta_{+/-} \text{ (etaplus, etaminus)}, \Gamma_{max/min} + \text{ (step sizes)} \\ + &\textbf{initialize} : g^0_{prev} \leftarrow 0, + \: \eta_0 \leftarrow \text{lr (learning rate)} \\ + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm} \textbf{for} \text{ } i = 0, 1, \ldots, d-1 \: \mathbf{do} \\ + &\hspace{10mm} \textbf{if} \: g^i_{prev} g^i_t > 0 \\ + &\hspace{15mm} \eta^i_t \leftarrow \mathrm{min}(\eta^i_{t-1} \eta_{+}, + \Gamma_{max}) \\ + &\hspace{10mm} \textbf{else if} \: g^i_{prev} g^i_t < 0 \\ + &\hspace{15mm} \eta^i_t \leftarrow \mathrm{max}(\eta^i_{t-1} \eta_{-}, + \Gamma_{min}) \\ + &\hspace{15mm} g^i_t \leftarrow 0 \\ + &\hspace{10mm} \textbf{else} \: \\ + &\hspace{15mm} \eta^i_t \leftarrow \eta^i_{t-1} \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1}- \eta_t \mathrm{sign}(g_t) \\ + &\hspace{5mm}g_{prev} \leftarrow g_t \\ + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + For further details regarding the algorithm we refer to the paper + `A Direct Adaptive Method for Faster Backpropagation Learning: The RPROP Algorithm + `_.""" # codespell:ignore + + rf""" + + Args: + {_params_doc} + lr (float, optional): learning rate (default: 1e-2) + etas (Tuple[float, float], optional): pair of (etaminus, etaplus), that + are multiplicative increase and decrease factors + (default: (0.5, 1.2)) + step_sizes (Tuple[float, float], optional): a pair of minimal and + maximal allowed step sizes (default: (1e-6, 50)) + {_capturable_doc} + {_foreach_doc} + {_maximize_doc} + {_differentiable_doc} + + """ +) + + +def _single_tensor_rprop( + params: list[Tensor], + grads: list[Tensor], + prevs: list[Tensor], + step_sizes: list[Tensor], + state_steps: list[Tensor], + *, + step_size_min: float, + step_size_max: float, + etaminus: float, + etaplus: float, + maximize: bool, + capturable: bool, + differentiable: bool, + has_complex: bool, +) -> None: + for i, param in enumerate(params): + grad = grads[i] + grad = grad if not maximize else -grad + prev = prevs[i] + step_size = step_sizes[i] + step = state_steps[i] + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not ( + param.device.type == step.device.type + and param.device.type in capturable_supported_devices + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + step += 1 + + if torch.is_complex(param): + grad = torch.view_as_real(grad) + prev = torch.view_as_real(prev) + param = torch.view_as_real(param) + step_size = torch.view_as_real(step_size) + if differentiable: + sign = grad.mul(prev.clone()).sign() + else: + sign = grad.mul(prev).sign() + + if capturable: + sign.copy_(torch.where(sign.gt(0), etaplus, sign)) + sign.copy_(torch.where(sign.lt(0), etaminus, sign)) + sign.copy_(torch.where(sign.eq(0), 1, sign)) + else: + sign[sign.gt(0)] = etaplus + sign[sign.lt(0)] = etaminus + sign[sign.eq(0)] = 1 + + # update stepsizes with step size updates + step_size.mul_(sign).clamp_(step_size_min, step_size_max) + + # for dir<0, dfdx=0 + # for dir>=0 dfdx=dfdx + grad = grad.clone(memory_format=torch.preserve_format) + if capturable: + grad.copy_(torch.where(sign.eq(etaminus), 0, grad)) + else: + grad[sign.eq(etaminus)] = 0 + + # update parameters + param.addcmul_(grad.sign(), step_size, value=-1) + prev.copy_(grad) + + +def _multi_tensor_rprop( + params: list[Tensor], + grads: list[Tensor], + prevs: list[Tensor], + step_sizes: list[Tensor], + state_steps: list[Tensor], + *, + step_size_min: float, + step_size_max: float, + etaminus: float, + etaplus: float, + maximize: bool, + capturable: bool, + differentiable: bool, + has_complex: bool, +) -> None: + if len(params) == 0: + return + + if differentiable: + raise AssertionError("_foreach ops don't support autograd") + + # If compiling, the compiler will handle cudagraph checks, see note [torch.compile x capturable] + if not torch.compiler.is_compiling() and capturable: + capturable_supported_devices = _get_capturable_supported_devices() + if not all( + p.device.type == step.device.type + and p.device.type in capturable_supported_devices + for p, step in zip(params, state_steps, strict=True) + ): + raise AssertionError( + f"If capturable=True, params and state_steps must be on supported devices: {capturable_supported_devices}." + ) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, prevs, step_sizes, state_steps] # type: ignore[list-item] + ) + for ( + grouped_params_, + grouped_grads_, + grouped_prevs_, + grouped_step_sizes_, + grouped_state_steps_, + ), _ in grouped_tensors.values(): + grouped_params = cast(list[Tensor], grouped_params_) + grouped_grads = cast(list[Tensor], grouped_grads_) + grouped_prevs = cast(list[Tensor], grouped_prevs_) + grouped_step_sizes = cast(list[Tensor], grouped_step_sizes_) + grouped_state_steps = cast(list[Tensor], grouped_state_steps_) + + # Update steps + # If steps are on CPU, foreach will fall back to the slow path, which is a for-loop calling t.add(1) over + # and over. 1 will then be wrapped into a Tensor over and over again, which is slower than if we just + # wrapped it once now. The alpha is required to assure we go to the right overload. + if not torch.compiler.is_compiling() and grouped_state_steps[0].is_cpu: + torch._foreach_add_( + grouped_state_steps, torch.tensor(1.0, device="cpu"), alpha=1.0 + ) + else: + torch._foreach_add_(grouped_state_steps, 1) + + # Handle complex params + if has_complex: + _view_as_real( + grouped_params, grouped_grads, grouped_prevs, grouped_step_sizes + ) + + signs = torch._foreach_mul(grouped_grads, grouped_prevs) + if maximize: + torch._foreach_neg_(signs) + + # At the end of the step, grouped_prevs will contain the current grads, so we reuse + # grouped_prevs memory instead of creating a new buffer, but, for clarity, we reassign + # to keep referring to the buffer as grouped_grads. + torch._foreach_copy_(grouped_prevs, grouped_grads) + if maximize: + torch._foreach_neg_(grouped_prevs) + grouped_grads = grouped_prevs + + torch._foreach_sign_(signs) + if capturable: + for sign in signs: + sign.copy_(torch.where(sign.gt(0), etaplus, sign)) + sign.copy_(torch.where(sign.lt(0), etaminus, sign)) + sign.copy_(torch.where(sign.eq(0), 1, sign)) + else: + for sign in signs: + sign[sign.gt(0)] = etaplus + sign[sign.lt(0)] = etaminus + sign[sign.eq(0)] = 1 + + # update stepsizes with step size updates + torch._foreach_mul_(grouped_step_sizes, signs) + for step_size in grouped_step_sizes: + step_size.clamp_(step_size_min, step_size_max) + + # for dir<0, dfdx=0 + # for dir>=0 dfdx=dfdx + grouped_grads = list(grouped_grads) + for i in range(len(grouped_grads)): + grouped_grads[i].copy_( + torch.where(signs[i].eq(etaminus), 0, grouped_grads[i]) + ) + + # explicitly del signs as it's not used after here to save memory + del signs + + # update parameters + grad_signs = [grad.sign() for grad in grouped_grads] + torch._foreach_addcmul_( + grouped_params, grad_signs, grouped_step_sizes, value=-1 + ) + + # Logically, you may expect grouped_prevs to get updated to grouped_grads, but that's + # basically already happened since we've been using grouped_prevs' memory to store + # updated grouped_grads! + + +@_disable_dynamo_if_unsupported(single_tensor_fn=_single_tensor_rprop) +def rprop( + params: list[Tensor], + grads: list[Tensor], + prevs: list[Tensor], + step_sizes: list[Tensor], + state_steps: list[Tensor], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + foreach: bool | None = None, + capturable: bool = False, + maximize: bool = False, + differentiable: bool = False, + has_complex: bool = False, + *, + step_size_min: float, + step_size_max: float, + etaminus: float, + etaplus: float, +) -> None: + r"""Functional API that performs rprop algorithm computation. + + See :class:`~torch.optim.Rprop` for details. + """ + # this check is slow during compilation, so we skip it + # if it's strictly needed we can add this check back in dynamo + if not torch.compiler.is_compiling() and not all( + isinstance(t, torch.Tensor) for t in state_steps + ): + raise RuntimeError( + "API has changed, `state_steps` argument must contain a list of singleton tensors" + ) + + if foreach is None: + _, foreach = _default_to_fused_or_foreach( + params, differentiable, use_fused=False + ) + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_rprop + else: + func = _single_tensor_rprop + + func( + params, + grads, + prevs, + step_sizes, + state_steps, + step_size_min=step_size_min, + step_size_max=step_size_max, + etaminus=etaminus, + etaplus=etaplus, + capturable=capturable, + maximize=maximize, + differentiable=differentiable, + has_complex=has_complex, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/sgd.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/sgd.py new file mode 100644 index 0000000000000000000000000000000000000000..8044d853f0b4e852a5e57bfeb04f97eac07d538b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/sgd.py @@ -0,0 +1,545 @@ +# mypy: allow-untyped-defs +r"""Implementation for Stochastic Gradient Descent optimizer.""" + +from typing import cast + +import torch +from torch import Tensor + +from .optimizer import ( + _default_to_fused_or_foreach, + _device_dtype_check_for_fused, + _differentiable_doc, + _foreach_doc, + _fused_doc, + _maximize_doc, + _params_doc, + _to_scalar, + _use_grad_for_differentiable, + DeviceDict, + Optimizer, + ParamsT, +) + + +__all__ = ["SGD", "sgd"] + + +class SGD(Optimizer): # noqa: D101 + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-3, + momentum: float = 0, + dampening: float = 0, + weight_decay: float | Tensor = 0, + nesterov: bool = False, + *, + maximize: bool = False, + foreach: bool | None = None, + differentiable: bool = False, + fused: bool | None = None, + ) -> None: # noqa: D107 + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if lr < 0.0: + raise ValueError(f"Invalid learning rate: {lr}") + if momentum < 0.0: + raise ValueError(f"Invalid momentum value: {momentum}") + if weight_decay < 0.0: + raise ValueError(f"Invalid weight_decay value: {weight_decay}") + + defaults = { + "lr": lr, + "momentum": momentum, + "dampening": dampening, + "weight_decay": weight_decay, + "nesterov": nesterov, + "maximize": maximize, + "foreach": foreach, + "differentiable": differentiable, + "fused": fused, + } + if nesterov and (momentum <= 0 or dampening != 0): + raise ValueError("Nesterov momentum requires a momentum and zero dampening") + super().__init__(params, defaults) + + if fused: + self._step_supports_amp_scaling = True + self._need_device_dtype_check_for_fused = True + if differentiable: + raise RuntimeError("`fused` does not support `differentiable`") + if foreach: + raise RuntimeError("`fused` and `foreach` cannot be `True` together.") + + def __setstate__(self, state): # noqa: D105 + super().__setstate__(state) + for group in self.param_groups: + group.setdefault("nesterov", False) + group.setdefault("maximize", False) + group.setdefault("foreach", None) + group.setdefault("differentiable", False) + group.setdefault("fused", False) + + def _init_group(self, group, params, grads, momentum_buffer_list): + has_sparse_grad = False + + for p in group["params"]: + if p.grad is not None: + if group["fused"] and getattr( + self, "_need_device_dtype_check_for_fused", True + ): + _device_dtype_check_for_fused(p) + self._need_device_dtype_check_for_fused = False + params.append(p) + grads.append(p.grad) + if p.grad.is_sparse: + has_sparse_grad = True + + if group["momentum"] != 0: + state = self.state[p] + momentum_buffer_list.append(state.get("momentum_buffer")) + + return has_sparse_grad + + @_use_grad_for_differentiable + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params: list[Tensor] = [] + grads: list[Tensor] = [] + momentum_buffer_list: list[Tensor | None] = [] + + has_sparse_grad = self._init_group( + group, params, grads, momentum_buffer_list + ) + + sgd( + params, + grads, + momentum_buffer_list, + weight_decay=group["weight_decay"], + momentum=group["momentum"], + lr=group["lr"], + dampening=group["dampening"], + nesterov=group["nesterov"], + maximize=group["maximize"], + has_sparse_grad=has_sparse_grad, + foreach=group["foreach"], + fused=group["fused"], + grad_scale=getattr(self, "grad_scale", None), + found_inf=getattr(self, "found_inf", None), + ) + + if group["momentum"] != 0: + # update momentum_buffers in state + for p, momentum_buffer in zip( + params, momentum_buffer_list, strict=True + ): + state = self.state[p] + state["momentum_buffer"] = momentum_buffer + + return loss + + +SGD.__doc__ = ( + r"""Implements stochastic gradient descent (optionally with momentum). + + .. math:: + \begin{aligned} + &\rule{110mm}{0.4pt} \\ + &\textbf{input} : \gamma \text{ (lr)}, \: \theta_0 \text{ (params)}, \: f(\theta) + \text{ (objective)}, \: \lambda \text{ (weight decay)}, \\ + &\hspace{13mm} \:\mu \text{ (momentum)}, \:\tau \text{ (dampening)}, + \:\textit{ nesterov,}\:\textit{ maximize} \\[-1.ex] + &\rule{110mm}{0.4pt} \\ + &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ + &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\ + &\hspace{10mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{else} \\ + &\hspace{10mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ + &\hspace{5mm}\textbf{if} \: \lambda \neq 0 \\ + &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ + &\hspace{5mm}\textbf{if} \: \mu \neq 0 \\ + &\hspace{10mm}\textbf{if} \: t > 1 \\ + &\hspace{15mm} \textbf{b}_t \leftarrow \mu \textbf{b}_{t-1} + (1-\tau) g_t \\ + &\hspace{10mm}\textbf{else} \\ + &\hspace{15mm} \textbf{b}_t \leftarrow g_t \\ + &\hspace{10mm}\textbf{if} \: \textit{nesterov} \\ + &\hspace{15mm} g_t \leftarrow g_{t} + \mu \textbf{b}_t \\ + &\hspace{10mm}\textbf{else} \\[-1.ex] + &\hspace{15mm} g_t \leftarrow \textbf{b}_t \\ + &\hspace{5mm}\theta_t \leftarrow \theta_{t-1} - \gamma g_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + &\bf{return} \: \theta_t \\[-1.ex] + &\rule{110mm}{0.4pt} \\[-1.ex] + \end{aligned} + + Nesterov momentum is based on the formula from + `On the importance of initialization and momentum in deep learning`__. + """ + + rf""" + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-3) + momentum (float, optional): momentum factor (default: 0) + dampening (float, optional): dampening for momentum (default: 0) + weight_decay (float, optional): weight decay (L2 penalty) (default: 0) + nesterov (bool, optional): enables Nesterov momentum. Only applicable + when momentum is non-zero. (default: False) + {_maximize_doc} + {_foreach_doc} + {_differentiable_doc} + {_fused_doc} + """ + + r""" + + Example: + >>> # xdoctest: +SKIP + >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9) + >>> optimizer.zero_grad() + >>> loss_fn(model(input), target).backward() + >>> optimizer.step() + + __ http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf + + .. note:: + The implementation of SGD with Momentum/Nesterov subtly differs from + Sutskever et al. and implementations in some other frameworks. + + Considering the specific case of Momentum, the update can be written as + + .. math:: + \begin{aligned} + v_{t+1} & = \mu * v_{t} + g_{t+1}, \\ + p_{t+1} & = p_{t} - \text{lr} * v_{t+1}, + \end{aligned} + + where :math:`p`, :math:`g`, :math:`v` and :math:`\mu` denote the + parameters, gradient, velocity, and momentum respectively. + + This is in contrast to Sutskever et al. and + other frameworks which employ an update of the form + + .. math:: + \begin{aligned} + v_{t+1} & = \mu * v_{t} + \text{lr} * g_{t+1}, \\ + p_{t+1} & = p_{t} - v_{t+1}. + \end{aligned} + + The Nesterov version is analogously modified. + + Moreover, the initial value of the momentum buffer is set to the + gradient value at the first step. This is in contrast to some other + frameworks that initialize it to all zeros. One notable side effect + of this decision is that the first momentum value will not be scaled + by dampening. Dampening will be applied starting at the second step. + + """ +) + + +def sgd( + params: list[Tensor], + d_p_list: list[Tensor], + momentum_buffer_list: list[Tensor | None], + # kwonly args with defaults are not supported by functions compiled with torchscript issue #70627 + # setting this as kwarg for now as functional API is compiled by torch/distributed/optim + has_sparse_grad: bool = False, + foreach: bool | None = None, + fused: bool | None = None, + grad_scale: Tensor | None = None, + found_inf: Tensor | None = None, + *, + weight_decay: float, + momentum: float, + lr: float, + dampening: float, + nesterov: bool, + maximize: bool, +) -> None: + r"""Functional API that performs SGD algorithm computation. + + See :class:`~torch.optim.SGD` for details. + """ + # Respect when the user inputs False/True for foreach or fused. We only want to change + # the default when neither have been user-specified. Note that we default to foreach + # and pass False to use_fused. This is not a mistake--we want to give the fused impl + # bake-in time before making it the default, even if it is typically faster. + if foreach is None and fused is None: + # why must we be explicit about an if statement for torch.jit.is_scripting here? + # because JIT can't handle Optionals nor fancy conditionals when scripting + if not torch.jit.is_scripting(): + fused, foreach = _default_to_fused_or_foreach( + params, differentiable=False, use_fused=False + ) + else: + foreach = False + fused = False + if foreach is None: + foreach = False + if fused is None: + fused = False + + if foreach and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with foreach optimizers") + if fused and torch.jit.is_scripting(): + raise RuntimeError("torch.jit.script not supported with fused optimizers") + + if foreach and not torch.jit.is_scripting(): + func = _multi_tensor_sgd + elif fused and not torch.jit.is_scripting(): + func = _fused_sgd + else: + func = _single_tensor_sgd + + func( + params, + d_p_list, + momentum_buffer_list, + weight_decay=weight_decay, + momentum=momentum, + lr=lr, + dampening=dampening, + nesterov=nesterov, + has_sparse_grad=has_sparse_grad, + maximize=maximize, + grad_scale=grad_scale, + found_inf=found_inf, + ) + + +def _single_tensor_sgd( + params: list[Tensor], + grads: list[Tensor], + momentum_buffer_list: list[Tensor | None], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + weight_decay: float, + momentum: float, + lr: float, + dampening: float, + nesterov: bool, + maximize: bool, + has_sparse_grad: bool, +) -> None: + if grad_scale is not None or found_inf is not None: + raise AssertionError("Expected grad_scale and found_inf to be None") + + if not torch.jit.is_scripting(): + lr = _to_scalar(lr) + + for i, param in enumerate(params): + grad = grads[i] if not maximize else -grads[i] + + if weight_decay != 0: + # Nested if is necessary to bypass jitscript rules + if isinstance(weight_decay, Tensor): + if weight_decay.requires_grad: + # usually this is the differentiable path, which is why the param.clone() is needed + grad = grad.addcmul_(param.clone(), weight_decay) + else: + # pyrefly: ignore [bad-argument-type] + grad = grad.add(param, alpha=weight_decay) + else: + grad = grad.add(param, alpha=weight_decay) + + if momentum != 0: + buf = momentum_buffer_list[i] + + if buf is None: + buf = grad.detach().clone() + momentum_buffer_list[i] = buf + else: + buf.mul_(momentum).add_(grad, alpha=1 - dampening) + + if nesterov: + grad = grad.add(buf, alpha=momentum) + else: + grad = buf + + # Nested if is necessary to bypass jitscript rules + if isinstance(lr, Tensor): + if lr.requires_grad: + param.addcmul_(grad, lr, value=-1) + else: + # pyrefly: ignore [bad-argument-type] + param.add_(grad, alpha=-lr) + else: + param.add_(grad, alpha=-lr) + + +def _multi_tensor_sgd( + params: list[Tensor], + grads: list[Tensor], + momentum_buffer_list: list[Tensor | None], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + weight_decay: float, + momentum: float, + lr: float, + dampening: float, + nesterov: bool, + maximize: bool, + has_sparse_grad: bool, +) -> None: + if grad_scale is not None or found_inf is not None: + raise AssertionError("Expected grad_scale and found_inf to be None") + + if len(params) == 0: + return + + lr = _to_scalar(lr) + + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, momentum_buffer_list], # type: ignore[list-item] + with_indices=True, + ) + for ( + device_params_, + device_grads_, + device_momentum_buffer_list, + ), indices in grouped_tensors.values(): + device_params: list[Tensor] = cast(list[Tensor], device_params_) + device_grads: list[Tensor] = cast(list[Tensor], device_grads_) + + device_has_sparse_grad = has_sparse_grad and any( + grad.is_sparse for grad in device_grads + ) + + if maximize: + device_grads = torch._foreach_neg(device_grads) # type: ignore[assignment] + + if weight_decay != 0: + # Reuse the intermediate memory (device_grads) already allocated for maximize + if maximize: + torch._foreach_add_(device_grads, device_params, alpha=weight_decay) + else: + device_grads = torch._foreach_add( # type: ignore[assignment] + device_grads, device_params, alpha=weight_decay + ) + + if momentum != 0: + bufs: list[Tensor] = [] + + all_states_with_momentum_buffer = True + for i in range(len(device_momentum_buffer_list)): + if device_momentum_buffer_list[i] is None: + all_states_with_momentum_buffer = False + break + else: + bufs.append(cast(Tensor, device_momentum_buffer_list[i])) + + if all_states_with_momentum_buffer: + torch._foreach_mul_(bufs, momentum) + torch._foreach_add_(bufs, device_grads, alpha=1 - dampening) + else: + bufs = [] + + for i in range(len(device_momentum_buffer_list)): + if device_momentum_buffer_list[i] is None: + buf = device_momentum_buffer_list[i] = momentum_buffer_list[ + indices[i] + ] = device_grads[i].detach().clone() + else: + buf = cast(Tensor, device_momentum_buffer_list[i]) + buf.mul_(momentum).add_(device_grads[i], alpha=1 - dampening) + + bufs.append(buf) + + if nesterov: + torch._foreach_add_(device_grads, bufs, alpha=momentum) + else: + device_grads = bufs + + if not device_has_sparse_grad: + # handle internal item() call if lr is a tensor + if isinstance(lr, torch.Tensor) and torch.compiler.is_compiling(): + grads_x_lr = torch._foreach_mul(device_grads, -lr) + torch._foreach_add_(device_params, grads_x_lr) + else: + torch._foreach_add_(device_params, device_grads, alpha=-lr) + else: + # foreach APIs don't support sparse + for i in range(len(device_params)): + device_params[i].add_(device_grads[i], alpha=-lr) + + +def _fused_sgd( + params: list[Tensor], + grads: list[Tensor], + momentum_buffer_list: list[Tensor | None], + grad_scale: Tensor | None, + found_inf: Tensor | None, + *, + weight_decay: float, + momentum: float, + lr: float, + dampening: float, + nesterov: bool, + maximize: bool, + has_sparse_grad: bool, +) -> None: + if not params: + return + if has_sparse_grad: + raise RuntimeError("`_fused_sgd` does not support sparse gradients") + grad_scale_dict: DeviceDict = ( + {grad_scale.device: grad_scale} if grad_scale is not None else {} + ) + found_inf_dict: DeviceDict = ( + {found_inf.device: found_inf} if found_inf is not None else {} + ) + + no_momentum_buffer = momentum == 0 + is_first_step = ( + all(t is None for t in momentum_buffer_list) and not no_momentum_buffer + ) + if is_first_step: + for i, g in enumerate(grads): + momentum_buffer_list[i] = torch.empty_like(g) + grouped_tensors = Optimizer._group_tensors_by_device_and_dtype( + [params, grads, momentum_buffer_list], # type: ignore[list-item] + with_indices=False, + ) + for (device, _), ( + (device_params_, device_grads_, device_momentum_buffer_list), + _, + ) in grouped_tensors.items(): + device_params: list[Tensor] = cast(list[Tensor], device_params_) + device_grads: list[Tensor] = cast(list[Tensor], device_grads_) + device_grad_scale, device_found_inf = None, None + if grad_scale is not None: + device_grad_scale = grad_scale_dict.setdefault( + device, grad_scale.to(device) + ) + if found_inf_dict is not None and found_inf is not None: + device_found_inf = found_inf_dict.setdefault(device, found_inf.to(device)) + torch._fused_sgd_( + device_params, + device_grads, + [] + if no_momentum_buffer + else cast(list[Tensor], device_momentum_buffer_list), + weight_decay=weight_decay, + momentum=momentum, + lr=lr, + dampening=dampening, + nesterov=nesterov, + maximize=maximize, + is_first_step=is_first_step, + grad_scale=device_grad_scale, + found_inf=device_found_inf, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/sparse_adam.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/sparse_adam.py new file mode 100644 index 0000000000000000000000000000000000000000..d6196cb20cd4e3c5ab1af7a16272d17a961bcad8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/sparse_adam.py @@ -0,0 +1,189 @@ +# mypy: allow-untyped-defs + +import torch +from torch import Tensor + +from . import _functional as F +from .optimizer import _maximize_doc, _params_doc, _to_scalar, Optimizer, ParamsT + + +__all__ = ["SparseAdam"] + + +class SparseAdam(Optimizer): + def __init__( + self, + params: ParamsT, + lr: float | Tensor = 1e-3, + betas: tuple[float, float] = (0.9, 0.999), + eps: float = 1e-8, + maximize: bool = False, + ) -> None: + if isinstance(lr, Tensor) and lr.numel() != 1: + raise ValueError("Tensor lr must be 1-element") + if not 0.0 < lr: + raise ValueError(f"Invalid learning rate: {lr}") + if not 0.0 < eps: + raise ValueError(f"Invalid epsilon value: {eps}") + if not 0.0 <= betas[0] < 1.0: + raise ValueError(f"Invalid beta parameter at index 0: {betas[0]}") + if not 0.0 <= betas[1] < 1.0: + raise ValueError(f"Invalid beta parameter at index 1: {betas[1]}") + + defaults = { + "lr": lr, + "betas": betas, + "eps": eps, + "maximize": maximize, + } + super().__init__(params, defaults) + + sparse_params = [] + complex_params = [] + for index, param_group in enumerate(self.param_groups): + if not isinstance(param_group, dict): + raise AssertionError( + f"param_groups must be a list of dicts, but got {type(param_group)}" + ) + # given param group, convert given params to a list first before iterating + for d_index, d_param in enumerate(param_group["params"]): + if d_param.is_sparse: + sparse_params.append([index, d_index]) + if d_param.is_complex(): + complex_params.append([index, d_index]) + if sparse_params: + raise ValueError( + f"Sparse params at indices {sparse_params}: SparseAdam requires dense parameter tensors" + ) + if complex_params: + raise ValueError( + f"Complex params at indices {complex_params}: SparseAdam does not support complex parameters" + ) + + @torch.no_grad() + def step(self, closure=None): + """Perform a single optimization step. + + Args: + closure (Callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + params_with_grad: list[Tensor] = [] + grads: list[Tensor] = [] + exp_avgs: list[Tensor] = [] + exp_avg_sqs: list[Tensor] = [] + state_steps: list[int] = [] + beta1, beta2 = group["betas"] + maximize = group.get("maximize", False) + + for p in group["params"]: + if p.grad is not None: + params_with_grad.append(p) + if not p.grad.is_sparse: + raise RuntimeError( + "SparseAdam does not support dense gradients, please consider Adam instead" + ) + grads.append(p.grad) + + state = self.state[p] + + # State initialization + if len(state) == 0: + state["step"] = 0 + # Exponential moving average of gradient values + state["exp_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + # Exponential moving average of squared gradient values + state["exp_avg_sq"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + exp_avgs.append(state["exp_avg"]) + exp_avg_sqs.append(state["exp_avg_sq"]) + + # update the steps for each param group update + state["step"] += 1 + # record the step after step update + state_steps.append(state["step"]) + + F.sparse_adam( + params_with_grad, + grads, + exp_avgs, + exp_avg_sqs, + state_steps, + eps=group["eps"], + beta1=beta1, + beta2=beta2, + lr=_to_scalar(group["lr"]), + maximize=maximize, + ) + + return loss + + +SparseAdam.__doc__ = rf"""SparseAdam implements a masked version of the Adam algorithm + suitable for sparse gradients. Currently, due to implementation constraints (explained + below), SparseAdam is only intended for a narrow subset of use cases, specifically + parameters of a dense layout with gradients of a sparse layout. This occurs in a + special case where the module backwards produces grads already in a sparse layout. + One example NN module that behaves as such is ``nn.Embedding(sparse=True)``. + + SparseAdam approximates the Adam algorithm by masking out the parameter and moment + updates corresponding to the zero values in the gradients. Whereas the Adam algorithm + will update the first moment, the second moment, and the parameters based on all values + of the gradients, SparseAdam only updates the moments and parameters corresponding + to the non-zero values of the gradients. + + A simplified way of thinking about the `intended` implementation is as such: + + 1. Create a mask of the non-zero values in the sparse gradients. For example, + if your gradient looks like [0, 5, 0, 0, 9], the mask would be [0, 1, 0, 0, 1]. + 2. Apply this mask over the running moments and do computation on only the + non-zero values. + 3. Apply this mask over the parameters and only apply an update on non-zero values. + + In actuality, we use sparse layout Tensors to optimize this approximation, which means the + more gradients that are masked by not being materialized, the more performant the optimization. + Since we rely on using sparse layout tensors, we infer that any materialized value in the + sparse layout is non-zero and we do NOT actually verify that all values are not zero! + It is important to not conflate a semantically sparse tensor (a tensor where many + of its values are zeros) with a sparse layout tensor (a tensor where ``.is_sparse`` + returns ``True``). The SparseAdam approximation is intended for `semantically` sparse + tensors and the sparse layout is only a implementation detail. A clearer implementation + would be to use MaskedTensors, but those are experimental. + + + .. note:: + + If you suspect your gradients are semantically sparse (but do not have sparse + layout), this variant may not be the best for you. Ideally, you want to avoid + materializing anything that is suspected to be sparse in the first place, since + needing to convert all your grads from dense layout to sparse layout may outweigh + the performance gain. Here, using Adam may be the best alternative, unless you + can easily rig up your module to output sparse grads similar to + ``nn.Embedding(sparse=True)``. If you insist on converting your grads, you can do + so by manually overriding your parameters' ``.grad`` fields with their sparse + equivalents before calling ``.step()``. + + + Args: + {_params_doc} + lr (float, Tensor, optional): learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + {_maximize_doc} + + .. _Adam\: A Method for Stochastic Optimization: + https://arxiv.org/abs/1412.6980 + + """ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/swa_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/swa_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..260292d23afc099f031529f4587fa24d08fde079 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/optim/swa_utils.py @@ -0,0 +1,549 @@ +# mypy: allow-untyped-defs +r"""Implementation for Stochastic Weight Averaging implementation.""" + +import itertools +import math +import warnings +from collections.abc import Callable, Iterable +from copy import deepcopy +from typing import Any, cast, Literal, Union +from typing_extensions import override + +import torch +from torch import Tensor +from torch.nn import Module +from torch.optim.lr_scheduler import _format_param, LRScheduler +from torch.utils._foreach_utils import _get_foreach_kernels_supported_devices + +from .optimizer import Optimizer + + +__all__ = [ + "AveragedModel", + "update_bn", + "SWALR", + "get_ema_multi_avg_fn", + "get_swa_multi_avg_fn", + "get_ema_avg_fn", + "get_swa_avg_fn", +] + +from torch.utils._foreach_utils import _group_tensors_by_device_and_dtype + + +PARAM_LIST = Union[tuple[Tensor, ...], list[Tensor]] + + +def get_ema_multi_avg_fn(decay=0.999): + """Get the function applying exponential moving average (EMA) across multiple params.""" + + if decay < 0.0 or decay > 1.0: + raise ValueError( + f"Invalid decay value {decay} provided. Please provide a value in [0,1] range." + ) + + @torch.no_grad() + def ema_update( + ema_param_list: PARAM_LIST, current_param_list: PARAM_LIST, _ + ) -> None: + # foreach lerp only handles float and complex + if torch.is_floating_point(ema_param_list[0]) or torch.is_complex( + ema_param_list[0] + ): + torch._foreach_lerp_(ema_param_list, current_param_list, 1 - decay) + else: + for p_ema, p_model in zip(ema_param_list, current_param_list, strict=True): + p_ema.copy_(p_ema * decay + p_model * (1 - decay)) + + return ema_update + + +def get_swa_multi_avg_fn(): + """Get the function applying stochastic weight average (SWA) across multiple params.""" + + @torch.no_grad() + def swa_update( + averaged_param_list: PARAM_LIST, + current_param_list: PARAM_LIST, + num_averaged: Tensor | int, + ) -> None: + # foreach lerp only handles float and complex + if torch.is_floating_point(averaged_param_list[0]) or torch.is_complex( + averaged_param_list[0] + ): + torch._foreach_lerp_( + averaged_param_list, + current_param_list, + cast(float, 1 / (num_averaged + 1)), + ) + else: + diffs = torch._foreach_sub(current_param_list, averaged_param_list) + if isinstance(num_averaged, Tensor): + torch._foreach_addcdiv_( + averaged_param_list, + diffs, + [num_averaged + 1] * len(averaged_param_list), + ) + else: + torch._foreach_add_( + averaged_param_list, diffs, alpha=1.0 / (num_averaged + 1) + ) + + return swa_update + + +def get_ema_avg_fn(decay=0.999): + """Get the function applying exponential moving average (EMA) across a single param.""" + + if decay < 0.0 or decay > 1.0: + raise ValueError( + f"Invalid decay value {decay} provided. Please provide a value in [0,1] range." + ) + + @torch.no_grad() + def ema_update(ema_param: Tensor, current_param: Tensor, num_averaged): + return decay * ema_param + (1 - decay) * current_param + + return ema_update + + +def get_swa_avg_fn(): + """Get the function applying stochastic weight average (SWA) across a single param.""" + + @torch.no_grad() + def swa_update( + averaged_param: Tensor, current_param: Tensor, num_averaged: Tensor | int + ): + return averaged_param + (current_param - averaged_param) / (num_averaged + 1) + + return swa_update + + +class AveragedModel(Module): + r"""Implements averaged model for Stochastic Weight Averaging (SWA) and Exponential Moving Average (EMA). + + Stochastic Weight Averaging was proposed in `Averaging Weights Leads to + Wider Optima and Better Generalization`_ by Pavel Izmailov, Dmitrii + Podoprikhin, Timur Garipov, Dmitry Vetrov and Andrew Gordon Wilson + (UAI 2018). + + Exponential Moving Average is a variation of `Polyak averaging`_, + but using exponential weights instead of equal weights across iterations. + + AveragedModel class creates a copy of the provided module :attr:`model` + on the device :attr:`device` and allows to compute running averages of the + parameters of the :attr:`model`. + + Args: + model (torch.nn.Module): model to use with SWA/EMA + device (torch.device, optional): if provided, the averaged model will be + stored on the :attr:`device` + avg_fn (function, optional): the averaging function used to update + parameters; the function must take in the current value of the + :class:`AveragedModel` parameter, the current value of :attr:`model` + parameter, and the number of models already averaged; if None, + an equally weighted average is used (default: None) + multi_avg_fn (function, optional): the averaging function used to update + parameters inplace; the function must take in the current values of the + :class:`AveragedModel` parameters as a list, the current values of :attr:`model` + parameters as a list, and the number of models already averaged; if None, + an equally weighted average is used (default: None) + use_buffers (bool): if ``True``, it will compute running averages for + both the parameters and the buffers of the model. (default: ``False``) + + Example: + >>> # xdoctest: +SKIP("undefined variables") + >>> loader, optimizer, model, loss_fn = ... + >>> swa_model = torch.optim.swa_utils.AveragedModel(model) + >>> scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, + >>> T_max=300) + >>> swa_start = 160 + >>> swa_scheduler = SWALR(optimizer, swa_lr=0.05) + >>> for i in range(300): + >>> for input, target in loader: + >>> optimizer.zero_grad() + >>> loss_fn(model(input), target).backward() + >>> optimizer.step() + >>> if i > swa_start: + >>> swa_model.update_parameters(model) + >>> swa_scheduler.step() + >>> else: + >>> scheduler.step() + >>> + >>> # Update bn statistics for the swa_model at the end + >>> torch.optim.swa_utils.update_bn(loader, swa_model) + + You can also use custom averaging functions with the `avg_fn` or `multi_avg_fn` parameters. + If no averaging function is provided, the default is to compute + equally-weighted average of the weights (SWA). + + Example: + >>> # xdoctest: +SKIP("undefined variables") + >>> # Compute exponential moving averages of the weights and buffers + >>> ema_model = torch.optim.swa_utils.AveragedModel(model, + >>> torch.optim.swa_utils.get_ema_multi_avg_fn(0.9), use_buffers=True) + + .. note:: + When using SWA/EMA with models containing Batch Normalization you may + need to update the activation statistics for Batch Normalization. + This can be done either by using the :meth:`torch.optim.swa_utils.update_bn` + or by setting :attr:`use_buffers` to `True`. The first approach updates the + statistics in a post-training step by passing data through the model. The + second does it during the parameter update phase by averaging all buffers. + Empirical evidence has shown that updating the statistics in normalization + layers increases accuracy, but you may wish to empirically test which + approach yields the best results in your problem. + + .. note:: + :attr:`avg_fn` and `multi_avg_fn` are not saved in the :meth:`state_dict` of the model. + + .. note:: + When :meth:`update_parameters` is called for the first time (i.e. + :attr:`n_averaged` is `0`) the parameters of `model` are copied + to the parameters of :class:`AveragedModel`. For every subsequent + call of :meth:`update_parameters` the function `avg_fn` is used + to update the parameters. + + .. _Averaging Weights Leads to Wider Optima and Better Generalization: + https://arxiv.org/abs/1803.05407 + .. _There Are Many Consistent Explanations of Unlabeled Data: Why You Should + Average: + https://arxiv.org/abs/1806.05594 + .. _SWALP: Stochastic Weight Averaging in Low-Precision Training: + https://arxiv.org/abs/1904.11943 + .. _Stochastic Weight Averaging in Parallel: Large-Batch Training That + Generalizes Well: + https://arxiv.org/abs/2001.02312 + .. _Polyak averaging: + https://paperswithcode.com/method/polyak-averaging + """ + + n_averaged: Tensor + + def __init__( + self, + model: Module, + device: int | torch.device | None = None, + avg_fn: Callable[[Tensor, Tensor, Tensor | int], Tensor] | None = None, + multi_avg_fn: Callable[[PARAM_LIST, PARAM_LIST, Tensor | int], None] + | None = None, + use_buffers=False, + ) -> None: # noqa: D107 + super().__init__() + if avg_fn is not None and multi_avg_fn is not None: + raise AssertionError( + "Only one of avg_fn and multi_avg_fn should be provided" + ) + self.module = deepcopy(model) + if device is not None: + self.module = self.module.to(device) + self.register_buffer( + "n_averaged", torch.tensor(0, dtype=torch.long, device=device) + ) + self.avg_fn = avg_fn + self.multi_avg_fn = multi_avg_fn + self.use_buffers = use_buffers + + def forward(self, *args, **kwargs): + """Forward pass.""" + return self.module(*args, **kwargs) + + def update_parameters(self, model: Module) -> None: + """Update model parameters.""" + self_param = ( + # pyrefly: ignore [bad-argument-type] + itertools.chain(self.module.parameters(), self.module.buffers()) + if self.use_buffers + else self.parameters() + ) + model_param = ( + # pyrefly: ignore [bad-argument-type] + itertools.chain(model.parameters(), model.buffers()) + if self.use_buffers + else model.parameters() + ) + self_param_detached: list[Tensor | None] = [] + model_param_detached: list[Tensor | None] = [] + copy_param = bool(self.n_averaged == 0) + for p_averaged, p_model in zip(self_param, model_param, strict=False): + p_model_ = p_model.detach().to(p_averaged.device) + self_param_detached.append(p_averaged.detach()) + model_param_detached.append(p_model_) + if copy_param: + p_averaged.detach().copy_(p_model_) + + if self.n_averaged > 0: + if self.multi_avg_fn is not None or self.avg_fn is None: + grouped_tensors = _group_tensors_by_device_and_dtype( + [self_param_detached, model_param_detached] + ) + for (device, _), ( + [self_params, model_params], + _, + ) in grouped_tensors.items(): + if self.multi_avg_fn: + self.multi_avg_fn( + self_params, # type: ignore[arg-type] + model_params, # type: ignore[arg-type] + self.n_averaged.to(device), + ) + elif ( + device is not None + and device.type in _get_foreach_kernels_supported_devices() + ): + multi_avg_fn = get_swa_multi_avg_fn() + multi_avg_fn( + self_params, model_params, self.n_averaged.to(device) + ) + else: + avg_fn = get_swa_avg_fn() + n_averaged = self.n_averaged.to(device) + for p_averaged, p_model in zip( # type: ignore[assignment] + self_params, model_params, strict=True + ): + # pyrefly: ignore [missing-attribute] + p_averaged.copy_(avg_fn(p_averaged, p_model, n_averaged)) + else: + for p_averaged, p_model in zip( # type: ignore[assignment] + self_param_detached, model_param_detached, strict=True + ): + # pyrefly: ignore [missing-attribute] + n_averaged = self.n_averaged.to(p_averaged.device) + # pyrefly: ignore [missing-attribute] + p_averaged.detach().copy_( + # pyrefly: ignore [missing-attribute, bad-argument-type] + self.avg_fn(p_averaged.detach(), p_model, n_averaged) + ) + + if not self.use_buffers: + # If not apply running averages to the buffers, + # keep the buffers in sync with the source model. + for b_swa, b_model in zip( + self.module.buffers(), model.buffers(), strict=True + ): + b_swa.detach().copy_(b_model.detach().to(b_swa.device)) + self.n_averaged += 1 + + +@torch.no_grad() +def update_bn( + loader: Iterable[Any], + model: Module, + device: int | torch.device | None = None, +) -> None: + r"""Update BatchNorm running_mean, running_var buffers in the model. + + It performs one pass over data in `loader` to estimate the activation + statistics for BatchNorm layers in the model. + + Args: + loader (torch.utils.data.DataLoader): dataset loader to compute the + activation statistics on. Each data batch should be either a + tensor, or a list/tuple whose first element is a tensor + containing data. + model (torch.nn.Module): model for which we seek to update BatchNorm + statistics. + device (torch.device, optional): If set, data will be transferred to + :attr:`device` before being passed into :attr:`model`. + + Example: + >>> # xdoctest: +SKIP("Undefined variables") + >>> loader, model = ... + >>> torch.optim.swa_utils.update_bn(loader, model) + + .. note:: + The `update_bn` utility assumes that each data batch in :attr:`loader` + is either a tensor or a list or tuple of tensors; in the latter case it + is assumed that :meth:`model.forward()` should be called on the first + element of the list or tuple corresponding to the data batch. + """ + momenta = {} + for module in model.modules(): + if isinstance(module, torch.nn.modules.batchnorm._BatchNorm): + module.reset_running_stats() + momenta[module] = module.momentum + + if not momenta: + return + + was_training = model.training + model.train() + for module in momenta: + module.momentum = None + + for input in loader: + if isinstance(input, (list, tuple)): + input = input[0] + if device is not None: + input = input.to(device) + + model(input) + + for bn_module in momenta: + bn_module.momentum = momenta[bn_module] + model.train(was_training) + + +class SWALR(LRScheduler): + r"""Anneals the learning rate in each parameter group to a fixed value. + + This learning rate scheduler is meant to be used with Stochastic Weight + Averaging (SWA) method (see `torch.optim.swa_utils.AveragedModel`). + + Args: + optimizer (torch.optim.Optimizer): wrapped optimizer + swa_lrs (float or list): the learning rate value for all param groups + together or separately for each group. + annealing_epochs (int): number of epochs in the annealing phase + (default: 10) + annealing_strategy (str): "cos" or "linear"; specifies the annealing + strategy: "cos" for cosine annealing, "linear" for linear annealing + (default: "cos") + last_epoch (int): the index of the last epoch (default: -1) + + The :class:`SWALR` scheduler can be used together with other + schedulers to switch to a constant learning rate late in the training + as in the example below. + + Example: + >>> # xdoctest: +SKIP("Undefined variables") + >>> loader, optimizer, model = ... + >>> lr_lambda = lambda epoch: 0.9 + >>> scheduler = torch.optim.lr_scheduler.MultiplicativeLR(optimizer, + >>> lr_lambda=lr_lambda) + >>> swa_scheduler = torch.optim.swa_utils.SWALR(optimizer, + >>> anneal_strategy="linear", anneal_epochs=20, swa_lr=0.05) + >>> swa_start = 160 + >>> for i in range(300): + >>> for input, target in loader: + >>> optimizer.zero_grad() + >>> loss_fn(model(input), target).backward() + >>> optimizer.step() + >>> if i > swa_start: + >>> swa_scheduler.step() + >>> else: + >>> scheduler.step() + + .. _Averaging Weights Leads to Wider Optima and Better Generalization: + https://arxiv.org/abs/1803.05407 + """ + + def __init__( + self, + optimizer: Optimizer, + swa_lr: float, + anneal_epochs=10, + anneal_strategy: Literal["cos", "linear"] = "cos", + last_epoch=-1, + ) -> None: # noqa: D107 + swa_lrs = _format_param("swa_lr", optimizer, swa_lr) + for swa_lr, group in zip(swa_lrs, optimizer.param_groups, strict=True): + group["swa_lr"] = swa_lr + if anneal_strategy not in ["cos", "linear"]: + raise ValueError( + "anneal_strategy must by one of 'cos' or 'linear', " + f"instead got {anneal_strategy}" + ) + self._set_anneal_func(anneal_strategy) + if not isinstance(anneal_epochs, int) or anneal_epochs < 0: + raise ValueError( + f"anneal_epochs must be equal or greater than 0, got {anneal_epochs}" + ) + self.anneal_epochs = anneal_epochs + super().__init__(optimizer, last_epoch) + + @staticmethod + def _linear_anneal(t): + return t + + @staticmethod + def _cosine_anneal(t): + return (1 - math.cos(math.pi * t)) / 2 + + @staticmethod + def _get_initial_lr(lr, swa_lr, alpha): + if alpha == 1: + return swa_lr + return (lr - alpha * swa_lr) / (1 - alpha) + + @override + def get_lr(self): + r"""Compute the next learning rate for each of the optimizer's + :attr:`~torch.optim.Optimizer.param_groups`. + + Uses :attr:`anneal_func` to interpolate between each group's + ``group["lr"]`` and ``group["swa_lr"]`` over :attr:`anneal_epochs` + epochs. Once :attr:`anneal_epochs` is reached, keeps the learning rate + fixed at ``group["swa_lr"]``. + + Returns: + list[float | Tensor]: A :class:`list` of learning rates for each of + the optimizer's :attr:`~torch.optim.Optimizer.param_groups` with the + same types as their current ``group["lr"]``\s. + + .. note:: + If you're trying to inspect the most recent learning rate, use + :meth:`get_last_lr()` instead. + + .. note:: + The returned :class:`~torch.Tensor`\s are copies, and never alias + the optimizer's ``group["lr"]``\s. + """ + # `_get_lr_called_within_step` is only available `_enable_get_lr_call`, + # so we ignore the type error here. See `LRScheduler.step()` for more details. + if not self._get_lr_called_within_step: + warnings.warn( + "To get the last learning rate computed by the scheduler, " + "please use `get_last_lr()`.", + UserWarning, + stacklevel=2, + ) + # Set in `LRScheduler._initial_step()` + step = self._step_count - 1 + if self.anneal_epochs == 0: + step = max(1, step) + # pyrefly: ignore [no-matching-overload] + prev_t = max(0, min(1, (step - 1) / max(1, self.anneal_epochs))) + prev_alpha = self.anneal_func(prev_t) + prev_lrs = [ + self._get_initial_lr(group["lr"], group["swa_lr"], prev_alpha) + for group in self.optimizer.param_groups + ] + # pyrefly: ignore [no-matching-overload] + t = max(0, min(1, step / max(1, self.anneal_epochs))) + alpha = self.anneal_func(t) + return [ + group["swa_lr"] * alpha + lr * (1 - alpha) + for group, lr in zip(self.optimizer.param_groups, prev_lrs, strict=True) + ] + + def _set_anneal_func(self, anneal_strategy: Literal["cos", "linear"]) -> None: + self._anneal_strategy = anneal_strategy + if anneal_strategy == "cos": + self.anneal_func = self._cosine_anneal + else: + self.anneal_func = self._linear_anneal + + @override + def state_dict(self) -> dict[str, Any]: + """Return the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in self.__dict__ which + is not the optimizer or anneal_func. + """ + return { + key: value + for key, value in self.__dict__.items() + if key not in ("optimizer", "anneal_func") + } + + @override + def load_state_dict(self, state_dict: dict[str, Any]) -> None: + """Load the scheduler's state. + + Args: + state_dict (dict): scheduler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + self.__dict__.update(state_dict) + self._set_anneal_func(self._anneal_strategy) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..66cace5931ac17c548becfddbb0e56dbbdac3d38 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/__init__.py @@ -0,0 +1,12 @@ +from .analyze.is_from_package import is_from_package +from .file_structure_representation import Directory +from .glob_group import GlobGroup +from .importer import ( + Importer, + ObjMismatchError, + ObjNotFoundError, + OrderedImporter, + sys_importer, +) +from .package_exporter import EmptyMatchError, PackageExporter, PackagingError +from .package_importer import PackageImporter diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_digraph.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_digraph.py new file mode 100644 index 0000000000000000000000000000000000000000..b98b49b507a3777d9e65877e9f9a1be7d299557e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_digraph.py @@ -0,0 +1,173 @@ +# mypy: allow-untyped-defs +from collections import deque + + +class DiGraph: + """Really simple unweighted directed graph data structure to track dependencies. + + The API is pretty much the same as networkx so if you add something just + copy their API. + """ + + def __init__(self): + # Dict of node -> dict of arbitrary attributes + self._node = {} + # Nested dict of node -> successor node -> nothing. + # (didn't implement edge data) + self._succ = {} + # Nested dict of node -> predecessor node -> nothing. + self._pred = {} + + # Keep track of the order in which nodes are added to + # the graph. + self._node_order = {} + self._insertion_idx = 0 + + def add_node(self, n, **kwargs): + """Add a node to the graph. + + Args: + n: the node. Can we any object that is a valid dict key. + **kwargs: any attributes you want to attach to the node. + """ + if n not in self._node: + self._node[n] = kwargs + self._succ[n] = {} + self._pred[n] = {} + self._node_order[n] = self._insertion_idx + self._insertion_idx += 1 + else: + self._node[n].update(kwargs) + + def add_edge(self, u, v): + """Add an edge to graph between nodes ``u`` and ``v`` + + ``u`` and ``v`` will be created if they do not already exist. + """ + # add nodes + self.add_node(u) + self.add_node(v) + + # add the edge + self._succ[u][v] = True + self._pred[v][u] = True + + def successors(self, n): + """Returns an iterator over successor nodes of n.""" + try: + return iter(self._succ[n]) + except KeyError as e: + raise ValueError(f"The node {n} is not in the digraph.") from e + + def predecessors(self, n): + """Returns an iterator over predecessors nodes of n.""" + try: + return iter(self._pred[n]) + except KeyError as e: + raise ValueError(f"The node {n} is not in the digraph.") from e + + @property + def edges(self): + """Returns an iterator over all edges (u, v) in the graph""" + for n, successors in self._succ.items(): + for succ in successors: + yield n, succ + + @property + def nodes(self): + """Returns a dictionary of all nodes to their attributes.""" + return self._node + + def __iter__(self): + """Iterate over the nodes.""" + return iter(self._node) + + def __contains__(self, n): + """Returns True if ``n`` is a node in the graph, False otherwise.""" + try: + return n in self._node + except TypeError: + return False + + def forward_transitive_closure(self, src: str) -> set[str]: + """Returns a set of nodes that are reachable from src""" + + result = set(src) + working_set = deque(src) + while len(working_set) > 0: + cur = working_set.popleft() + for n in self.successors(cur): + if n not in result: + result.add(n) + working_set.append(n) + return result + + def backward_transitive_closure(self, src: str) -> set[str]: + """Returns a set of nodes that are reachable from src in reverse direction""" + + result = set(src) + working_set = deque(src) + while len(working_set) > 0: + cur = working_set.popleft() + for n in self.predecessors(cur): + if n not in result: + result.add(n) + working_set.append(n) + return result + + def all_paths(self, src: str, dst: str): + """Returns a subgraph rooted at src that shows all the paths to dst.""" + + result_graph = DiGraph() + # First compute forward transitive closure of src (all things reachable from src). + forward_reachable_from_src = self.forward_transitive_closure(src) + + if dst not in forward_reachable_from_src: + return result_graph + + # Second walk the reverse dependencies of dst, adding each node to + # the output graph iff it is also present in forward_reachable_from_src. + # we don't use backward_transitive_closures for optimization purposes + working_set = deque(dst) + while len(working_set) > 0: + cur = working_set.popleft() + for n in self.predecessors(cur): + if n in forward_reachable_from_src: + result_graph.add_edge(n, cur) + # only explore further if its reachable from src + working_set.append(n) + + return result_graph.to_dot() + + def first_path(self, dst: str) -> list[str]: + """Returns a list of nodes that show the first path that resulted in dst being added to the graph.""" + path = [] + + while dst: + path.append(dst) + candidates = self._pred[dst].keys() + dst, min_idx = "", None + for candidate in candidates: + idx = self._node_order.get(candidate, None) + if idx is None: + break + if min_idx is None or idx < min_idx: + min_idx = idx + dst = candidate + + return list(reversed(path)) + + def to_dot(self) -> str: + """Returns the dot representation of the graph. + + Returns: + A dot representation of the graph. + """ + edges = "\n".join(f'"{f}" -> "{t}";' for f, t in self.edges) + return f"""\ +digraph G {{ +rankdir = LR; +node [shape=box]; +{edges} +}} +""" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_directory_reader.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_directory_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..52197fb1c84a99c79b4b587e5cdeba6053809b70 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_directory_reader.py @@ -0,0 +1,66 @@ +# mypy: allow-untyped-defs +import os.path +from glob import glob +from typing import cast + +import torch +from torch.types import Storage + + +__serialization_id_record_name__ = ".data/serialization_id" + + +# because get_storage_from_record returns a tensor!? +class _HasStorage: + def __init__(self, storage): + self._storage = storage + + def storage(self): + return self._storage + + +class DirectoryReader: + """ + Class to allow PackageImporter to operate on unzipped packages. Methods + copy the behavior of the internal PyTorchFileReader class (which is used for + accessing packages in all other cases). + + N.B.: ScriptObjects are not depickleable or accessible via this DirectoryReader + class due to ScriptObjects requiring an actual PyTorchFileReader instance. + """ + + def __init__(self, directory): + self.directory = directory + + def get_record(self, name): + filename = f"{self.directory}/{name}" + with open(filename, "rb") as f: + return f.read() + + def get_storage_from_record(self, name, numel, dtype): + filename = f"{self.directory}/{name}" + nbytes = torch._utils._element_size(dtype) * numel + storage = cast(Storage, torch.UntypedStorage) + return _HasStorage(storage.from_file(filename=filename, nbytes=nbytes)) + + def has_record(self, path): + full_path = os.path.join(self.directory, path) + return os.path.isfile(full_path) + + def get_all_records( + self, + ): + files = [ + filename[len(self.directory) + 1 :] + for filename in glob(f"{self.directory}/**", recursive=True) + if not os.path.isdir(filename) + ] + return files + + def serialization_id( + self, + ): + if self.has_record(__serialization_id_record_name__): + return self.get_record(__serialization_id_record_name__) + else: + return "" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_importlib.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_importlib.py new file mode 100644 index 0000000000000000000000000000000000000000..609efd294c4c9650d890fd36aafc9f521068ce8b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_importlib.py @@ -0,0 +1,95 @@ +# mypy: allow-untyped-defs +import _warnings +import os.path + + +# note: implementations +# copied from cpython's import code + + +# _zip_searchorder defines how we search for a module in the Zip +# archive: we first search for a package __init__, then for +# non-package .pyc, and .py entries. The .pyc entries +# are swapped by initzipimport() if we run in optimized mode. Also, +# '/' is replaced by path_sep there. + +_zip_searchorder = ( + ("/__init__.py", True), + (".py", False), +) + + +# Replace any occurrences of '\r\n?' in the input string with '\n'. +# This converts DOS and Mac line endings to Unix line endings. +def _normalize_line_endings(source): + source = source.replace(b"\r\n", b"\n") + source = source.replace(b"\r", b"\n") + return source + + +def _resolve_name(name, package, level): + """Resolve a relative module name to an absolute one.""" + bits = package.rsplit(".", level - 1) + if len(bits) < level: + raise ValueError("attempted relative import beyond top-level package") + base = bits[0] + return f"{base}.{name}" if name else base + + +def _sanity_check(name, package, level): + """Verify arguments are "sane".""" + if not isinstance(name, str): + raise TypeError(f"module name must be str, not {type(name)}") + if level < 0: + raise ValueError("level must be >= 0") + if level > 0: + if not isinstance(package, str): + raise TypeError("__package__ not set to a string") + elif not package: + raise ImportError("attempted relative import with no known parent package") + if not name and level == 0: + raise ValueError("Empty module name") + + +def _calc___package__(globals): + """Calculate what __package__ should be. + + __package__ is not guaranteed to be defined or could be set to None + to represent that its proper value is unknown. + + """ + package = globals.get("__package__") + spec = globals.get("__spec__") + if package is not None: + if spec is not None and package != spec.parent: + _warnings.warn( # noqa: G010 + f"__package__ != __spec__.parent ({package!r} != {spec.parent!r})", # noqa: G004 + ImportWarning, + stacklevel=3, + ) + return package + elif spec is not None: + return spec.parent + else: + _warnings.warn( # noqa: G010 + "can't resolve package from __spec__ or __package__, " + "falling back on __name__ and __path__", + ImportWarning, + stacklevel=3, + ) + package = globals["__name__"] + if "__path__" not in globals: + package = package.rpartition(".")[0] + return package + + +def _normalize_path(path): + """Normalize a path by ensuring it is a string. + + If the resulting string contains path separators, an exception is raised. + """ + parent, file_name = os.path.split(path) + if parent: + raise ValueError(f"{path!r} must be only a file name") + else: + return file_name diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_mangling.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_mangling.py new file mode 100644 index 0000000000000000000000000000000000000000..08b0560f79322a22dd1c6d2a1563159bcaf7e46c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_mangling.py @@ -0,0 +1,65 @@ +# mypy: allow-untyped-defs +"""Import mangling. +See mangling.md for details. +""" + +import re + + +_mangle_index = 0 + + +class PackageMangler: + """ + Used on import, to ensure that all modules imported have a shared mangle parent. + """ + + def __init__(self) -> None: + global _mangle_index + self._mangle_index = _mangle_index + # Increment the global index + _mangle_index += 1 + # Angle brackets are used so that there is almost no chance of + # confusing this module for a real module. Plus, it is Python's + # preferred way of denoting special modules. + self._mangle_parent = f"" + + def mangle(self, name) -> str: + assert len(name) != 0 + return self._mangle_parent + "." + name + + def demangle(self, mangled: str) -> str: + """ + Note: This only demangles names that were mangled by this specific + PackageMangler. It will pass through names created by a different + PackageMangler instance. + """ + if mangled.startswith(self._mangle_parent + "."): + return mangled.partition(".")[2] + + # wasn't a mangled name + return mangled + + def parent_name(self): + return self._mangle_parent + + +def is_mangled(name: str) -> bool: + return bool(re.match(r"", name)) + + +def demangle(name: str) -> str: + """ + Note: Unlike PackageMangler.demangle, this version works on any + mangled name, irrespective of which PackageMangler created it. + """ + if is_mangled(name): + _first, sep, last = name.partition(".") + # If there is only a base mangle prefix, e.g. '', + # then return an empty string. + return last if len(sep) != 0 else "" + return name + + +def get_mangle_prefix(name: str) -> str: + return name.partition(".")[0] if is_mangled(name) else name diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_mock.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_mock.py new file mode 100644 index 0000000000000000000000000000000000000000..44876b1a1d3fb3ef4a485eaf16f26755d5bb00f2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_mock.py @@ -0,0 +1,123 @@ +# mypy: allow-untyped-defs +_magic_methods = [ + "__subclasscheck__", + "__hex__", + "__rmul__", + "__float__", + "__idiv__", + "__setattr__", + "__div__", + "__invert__", + "__nonzero__", + "__rshift__", + "__eq__", + "__pos__", + "__round__", + "__rand__", + "__or__", + "__complex__", + "__divmod__", + "__len__", + "__reversed__", + "__copy__", + "__reduce__", + "__deepcopy__", + "__rdivmod__", + "__rrshift__", + "__ifloordiv__", + "__hash__", + "__iand__", + "__xor__", + "__isub__", + "__oct__", + "__ceil__", + "__imod__", + "__add__", + "__truediv__", + "__unicode__", + "__le__", + "__delitem__", + "__sizeof__", + "__sub__", + "__ne__", + "__pow__", + "__bytes__", + "__mul__", + "__itruediv__", + "__bool__", + "__iter__", + "__abs__", + "__gt__", + "__iadd__", + "__enter__", + "__floordiv__", + "__call__", + "__neg__", + "__and__", + "__ixor__", + "__getitem__", + "__exit__", + "__cmp__", + "__getstate__", + "__index__", + "__contains__", + "__floor__", + "__lt__", + "__getattr__", + "__mod__", + "__trunc__", + "__delattr__", + "__instancecheck__", + "__setitem__", + "__ipow__", + "__ilshift__", + "__long__", + "__irshift__", + "__imul__", + "__lshift__", + "__dir__", + "__ge__", + "__int__", + "__ior__", +] + + +class MockedObject: + _name: str + + def __new__(cls, *args, **kwargs): + # _suppress_err is set by us in the mocked module impl, so that we can + # construct instances of MockedObject to hand out to people looking up + # module attributes. + + # Any other attempt to construct a MockedObject instance (say, in the + # unpickling process) should give an error. + if not kwargs.get("_suppress_err"): + raise NotImplementedError( + f"Object '{cls._name}' was mocked out during packaging " + f"but it is being used in '__new__'. If this error is " + "happening during 'load_pickle', please ensure that your " + "pickled object doesn't contain any mocked objects." + ) + # Otherwise, this is just a regular object creation + # (e.g. `x = MockedObject("foo")`), so pass it through normally. + return super().__new__(cls) + + def __init__(self, name: str, _suppress_err: bool): + self.__dict__["_name"] = name + + def __repr__(self): + return f"MockedObject({self._name})" + + +def install_method(method_name): + def _not_implemented(self, *args, **kwargs): + raise NotImplementedError( + f"Object '{self._name}' was mocked out during packaging but it is being used in {method_name}" + ) + + setattr(MockedObject, method_name, _not_implemented) + + +for method_name in _magic_methods: + install_method(method_name) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_package_pickler.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_package_pickler.py new file mode 100644 index 0000000000000000000000000000000000000000..a4d8e7f752505bedebf8112d54c2171e8d68ab92 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_package_pickler.py @@ -0,0 +1,143 @@ +# mypy: allow-untyped-defs +# pyrefly: ignore [missing-module-attribute] +import sys +from pickle import ( # type: ignore[attr-defined] + _compat_pickle, + _extension_registry, + _getattribute, + _Pickler, + EXT1, + EXT2, + EXT4, + GLOBAL, + PicklingError, + STACK_GLOBAL, +) +from struct import pack +from types import FunctionType + +from .importer import Importer, ObjMismatchError, ObjNotFoundError, sys_importer + + +class _PyTorchLegacyPickler(_Pickler): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._persistent_id = None + + def persistent_id(self, obj): + if self._persistent_id is None: + return super().persistent_id(obj) + return self._persistent_id(obj) + + +class PackagePickler(_PyTorchLegacyPickler): + """Package-aware pickler. + + This behaves the same as a normal pickler, except it uses an `Importer` + to find objects and modules to save. + """ + + def __init__(self, importer: Importer, *args, **kwargs): + self.importer = importer + super().__init__(*args, **kwargs) + + # Make sure the dispatch table copied from _Pickler is up-to-date. + # Previous issues have been encountered where a library (e.g. dill) + # mutate _Pickler.dispatch, PackagePickler makes a copy when this lib + # is imported, then the offending library removes its dispatch entries, + # leaving PackagePickler with a stale dispatch table that may cause + # unwanted behavior. + self.dispatch = _Pickler.dispatch.copy() # type: ignore[misc] + self.dispatch[FunctionType] = PackagePickler.save_global # type: ignore[assignment] + + def save_global(self, obj, name=None): + # ruff: noqa: F841 + # unfortunately the pickler code is factored in a way that + # forces us to copy/paste this function. The only change is marked + # CHANGED below. + write = self.write # type: ignore[attr-defined] + memo = self.memo # type: ignore[attr-defined] + + # CHANGED: import module from module environment instead of __import__ + try: + module_name, name = self.importer.get_name(obj, name) + except (ObjNotFoundError, ObjMismatchError) as err: + raise PicklingError(f"Can't pickle {obj}: {str(err)}") from err + + module = self.importer.import_module(module_name) + if sys.version_info >= (3, 14): + # pickle._getattribute signature changes in 3.14 + # to take iterable and return just the object (not tuple) + # We need to get the parent object that contains the attribute + name_parts = name.split(".") + if "" in name_parts: + raise PicklingError(f"Can't pickle local object {obj!r}") + if len(name_parts) == 1: + parent = module + else: + parent = _getattribute(module, name_parts[:-1]) + else: + _, parent = _getattribute(module, name) + # END CHANGED + + if self.proto >= 2: # type: ignore[attr-defined] + code = _extension_registry.get((module_name, name)) + if code: + assert code > 0 + if code <= 0xFF: + write(EXT1 + pack("= 3. + if self.proto >= 4: # type: ignore[attr-defined] + self.save(module_name) # type: ignore[attr-defined] + self.save(name) # type: ignore[attr-defined] + write(STACK_GLOBAL) + elif parent is not module: + self.save_reduce(getattr, (parent, lastname)) # type: ignore[attr-defined] + elif self.proto >= 3: # type: ignore[attr-defined] + write( + GLOBAL + + bytes(module_name, "utf-8") + + b"\n" + + bytes(name, "utf-8") + + b"\n" + ) + else: + if self.fix_imports: # type: ignore[attr-defined] + r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING + r_import_mapping = _compat_pickle.REVERSE_IMPORT_MAPPING + if (module_name, name) in r_name_mapping: + module_name, name = r_name_mapping[(module_name, name)] + elif module_name in r_import_mapping: + module_name = r_import_mapping[module_name] + try: + write( + GLOBAL + + bytes(module_name, "ascii") + + b"\n" + + bytes(name, "ascii") + + b"\n" + ) + except UnicodeEncodeError as exc: + raise PicklingError( + f"can't pickle global identifier '{module}.{name}' using " + f"pickle protocol {self.proto:d}" # type: ignore[attr-defined] + ) from exc + + self.memoize(obj) # type: ignore[attr-defined] + + +def create_pickler(data_buf, importer, protocol=4): + if importer is sys_importer: + # if we are using the normal import library system, then + # we can use the C implementation of pickle which is faster + return _PyTorchLegacyPickler(data_buf, protocol=protocol) + else: + return PackagePickler(importer, data_buf, protocol=protocol) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_package_unpickler.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_package_unpickler.py new file mode 100644 index 0000000000000000000000000000000000000000..890e6b4e03ba076e30512712d57c4bf715c4c8bb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_package_unpickler.py @@ -0,0 +1,27 @@ +# mypy: allow-untyped-defs +import _compat_pickle +import pickle + +from .importer import Importer + + +class PackageUnpickler(pickle._Unpickler): # type: ignore[name-defined] + """Package-aware unpickler. + + This behaves the same as a normal unpickler, except it uses `importer` to + find any global names that it encounters while unpickling. + """ + + def __init__(self, importer: Importer, *args, **kwargs): + super().__init__(*args, **kwargs) + self._importer = importer + + def find_class(self, module, name): + # Subclasses may override this. + if self.proto < 3 and self.fix_imports: # type: ignore[attr-defined] + if (module, name) in _compat_pickle.NAME_MAPPING: + module, name = _compat_pickle.NAME_MAPPING[(module, name)] + elif module in _compat_pickle.IMPORT_MAPPING: + module = _compat_pickle.IMPORT_MAPPING[module] + mod = self._importer.import_module(module) + return getattr(mod, name) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_stdlib.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_stdlib.py new file mode 100644 index 0000000000000000000000000000000000000000..e07b20a83cc6d6788d8496f24a165120e1edce98 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/_stdlib.py @@ -0,0 +1,21 @@ +# mypy: allow-untyped-defs +"""List of Python standard library modules. + +Sadly, there is no reliable way to tell whether a module is part of the +standard library except by comparing to a canonical list. + +This is taken from https://github.com/PyCQA/isort/tree/develop/isort/stdlibs, +which itself is sourced from the Python documentation. +""" + +import sys + + +def is_stdlib_module(module: str) -> bool: + base_module = module.partition(".")[0] + return base_module in _get_stdlib_modules() + + +def _get_stdlib_modules(): + assert sys.version_info >= (3, 10) + return sys.stdlib_module_names diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/file_structure_representation.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/file_structure_representation.py new file mode 100644 index 0000000000000000000000000000000000000000..2dae130ed6007f7064ea762dfa9267d344f5db64 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/file_structure_representation.py @@ -0,0 +1,137 @@ +# mypy: allow-untyped-defs + +from .glob_group import GlobGroup, GlobPattern + + +__all__ = ["Directory"] + + +class Directory: + """A file structure representation. Organized as Directory nodes that have lists of + their Directory children. Directories for a package are created by calling + :meth:`PackageImporter.file_structure`.""" + + def __init__(self, name: str, is_dir: bool): + self.name = name + self.is_dir = is_dir + self.children: dict[str, Directory] = {} + + def _get_dir(self, dirs: list[str]) -> "Directory": + """Builds path of Directories if not yet built and returns last directory + in list. + + Args: + dirs (List[str]): List of directory names that are treated like a path. + + Returns: + :class:`Directory`: The last Directory specified in the dirs list. + """ + if len(dirs) == 0: + return self + dir_name = dirs[0] + if dir_name not in self.children: + self.children[dir_name] = Directory(dir_name, True) + return self.children[dir_name]._get_dir(dirs[1:]) + + def _add_file(self, file_path: str): + """Adds a file to a Directory. + + Args: + file_path (str): Path of file to add. Last element is added as a file while + other paths items are added as directories. + """ + *dirs, file = file_path.split("/") + dir = self._get_dir(dirs) + dir.children[file] = Directory(file, False) + + def has_file(self, filename: str) -> bool: + """Checks if a file is present in a :class:`Directory`. + + Args: + filename (str): Path of file to search for. + Returns: + bool: If a :class:`Directory` contains the specified file. + """ + lineage = filename.split("/", maxsplit=1) + child = lineage[0] + grandchildren = lineage[1] if len(lineage) > 1 else None + if child in self.children: + if grandchildren is None: + return True + else: + return self.children[child].has_file(grandchildren) + return False + + def __str__(self): + str_list: list[str] = [] + self._stringify_tree(str_list) + return "".join(str_list) + + def _stringify_tree( + self, + str_list: list[str], + preamble: str = "", + dir_ptr: str = "\u2500\u2500\u2500 ", + ): + """Recursive method to generate print-friendly version of a Directory.""" + space = " " + branch = "\u2502 " + tee = "\u251c\u2500\u2500 " + last = "\u2514\u2500\u2500 " + + # add this directory's representation + str_list.append(f"{preamble}{dir_ptr}{self.name}\n") + + # add directory's children representations + if dir_ptr == tee: + preamble = preamble + branch + else: + preamble = preamble + space + + file_keys: list[str] = [] + dir_keys: list[str] = [] + for key, val in self.children.items(): + if val.is_dir: + dir_keys.append(key) + else: + file_keys.append(key) + + for index, key in enumerate(sorted(dir_keys)): + if (index == len(dir_keys) - 1) and len(file_keys) == 0: + self.children[key]._stringify_tree(str_list, preamble, last) + else: + self.children[key]._stringify_tree(str_list, preamble, tee) + for index, file in enumerate(sorted(file_keys)): + pointer = last if (index == len(file_keys) - 1) else tee + str_list.append(f"{preamble}{pointer}{file}\n") + + +def _create_directory_from_file_list( + filename: str, + file_list: list[str], + include: "GlobPattern" = "**", + exclude: "GlobPattern" = (), +) -> Directory: + """Return a :class:`Directory` file structure representation created from a list of files. + + Args: + filename (str): The name given to the top-level directory that will be the + relative root for all file paths found in the file_list. + + file_list (List[str]): List of files to add to the top-level directory. + + include (Union[List[str], str]): An optional pattern that limits what is included from the file_list to + files whose name matches the pattern. + + exclude (Union[List[str], str]): An optional pattern that excludes files whose name match the pattern. + + Returns: + :class:`Directory`: a :class:`Directory` file structure representation created from a list of files. + """ + glob_pattern = GlobGroup(include, exclude=exclude, separator="/") + + top_dir = Directory(filename, True) + for file in file_list: + if glob_pattern.matches(file): + top_dir._add_file(file) + return top_dir diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/find_file_dependencies.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/find_file_dependencies.py new file mode 100644 index 0000000000000000000000000000000000000000..f8579db07811d31a58bb9ec81515685a9d97f78d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/find_file_dependencies.py @@ -0,0 +1,95 @@ +# mypy: allow-untyped-defs +import ast + +from ._importlib import _resolve_name + + +class _ExtractModuleReferences(ast.NodeVisitor): + """ + Extract the list of global variables a block of code will read and write + """ + + @classmethod + def run(cls, src: str, package: str) -> list[tuple[str, str | None]]: + visitor = cls(package) + tree = ast.parse(src) + visitor.visit(tree) + return list(visitor.references.keys()) + + def __init__(self, package): + super().__init__() + self.package = package + self.references = {} + + def _absmodule(self, module_name: str, level: int) -> str: + if level > 0: + return _resolve_name(module_name, self.package, level) + return module_name + + def visit_Import(self, node): + for alias in node.names: + self.references[(alias.name, None)] = True + + def visit_ImportFrom(self, node): + name = self._absmodule(node.module, 0 if node.level is None else node.level) + for alias in node.names: + # from my_package import foo + # foo may be a module, so we have to add it to the list of + # potential references, if import of it fails, we will ignore it + if alias.name != "*": + self.references[(name, alias.name)] = True + else: + self.references[(name, None)] = True + + def _grab_node_int(self, node): + return node.value + + def _grab_node_str(self, node): + return node.value + + def visit_Call(self, node): + # __import__ calls aren't routed to the visit_Import/From nodes + if hasattr(node.func, "id") and node.func.id == "__import__": + try: + name = self._grab_node_str(node.args[0]) + fromlist: list[str] = [] + level = 0 + if len(node.args) > 3: + fromlist.extend(self._grab_node_str(v) for v in node.args[3].elts) + elif hasattr(node, "keywords"): + for keyword in node.keywords: + if keyword.arg == "fromlist": + fromlist.extend( + self._grab_node_str(v) for v in keyword.value.elts + ) + if len(node.args) > 4: + level = self._grab_node_int(node.args[4]) + elif hasattr(node, "keywords"): + for keyword in node.keywords: + if keyword.arg == "level": + level = self._grab_node_int(keyword.value) + if fromlist == []: + # the top-level package (the name up till the first dot) is returned + # when the fromlist argument is empty in normal import system, + # we need to include top level package to match this behavior and last + # level package to capture the intended dependency of user + self.references[(name, None)] = True + top_name = name.rsplit(".", maxsplit=1)[0] + if top_name != name: + top_name = self._absmodule(top_name, level) + self.references[(top_name, None)] = True + else: + name = self._absmodule(name, level) + for alias in fromlist: + # fromlist args may be submodules, so we have to add the fromlist args + # to the list of potential references. If import of an arg fails we + # will ignore it, similar to visit_ImportFrom + if alias != "*": + self.references[(name, alias)] = True + else: + self.references[(name, None)] = True + except Exception: + return + + +find_files_source_depends_on = _ExtractModuleReferences.run diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/glob_group.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/glob_group.py new file mode 100644 index 0000000000000000000000000000000000000000..986938cd256ecb97085f7886c2fa672858119f53 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/glob_group.py @@ -0,0 +1,85 @@ +# mypy: allow-untyped-defs +import re +from collections.abc import Iterable +from typing import Union + + +GlobPattern = Union[str, Iterable[str]] + + +class GlobGroup: + """A set of patterns that candidate strings will be matched against. + + A candidate is composed of a list of segments separated by ``separator``, e.g. "foo.bar.baz". + + A pattern contains one or more segments. Segments can be: + - A literal string (e.g. "foo"), which matches exactly. + - A string containing a wildcard (e.g. "torch*", or "foo*baz*"). The wildcard matches + any string, including the empty string. + - A double wildcard ("**"). This matches against zero or more complete segments. + + Examples: + ``torch.**``: matches ``torch`` and all its submodules, e.g. ``torch.nn`` and ``torch.nn.functional``. + ``torch.*``: matches ``torch.nn`` or ``torch.functional``, but not ``torch.nn.functional``. + ``torch*.**``: matches ``torch``, ``torchvision``, and all their submodules. + + A candidates will match the ``GlobGroup`` if it matches any of the ``include`` patterns and + none of the ``exclude`` patterns. + + Args: + include (Union[str, Iterable[str]]): A string or list of strings, + each representing a pattern to be matched against. A candidate + will match if it matches *any* include pattern + exclude (Union[str, Iterable[str]]): A string or list of strings, + each representing a pattern to be matched against. A candidate + will be excluded from matching if it matches *any* exclude pattern. + separator (str): A string that delimits segments in candidates and + patterns. By default this is "." which corresponds to how modules are + named in Python. Another common value for this is "/", which is + the Unix path separator. + """ + + def __init__( + self, include: GlobPattern, *, exclude: GlobPattern = (), separator: str = "." + ): + self._dbg = f"GlobGroup(include={include}, exclude={exclude})" + self.include = GlobGroup._glob_list(include, separator) + self.exclude = GlobGroup._glob_list(exclude, separator) + self.separator = separator + + def __str__(self): + return self._dbg + + def __repr__(self): + return self._dbg + + def matches(self, candidate: str) -> bool: + candidate = self.separator + candidate + return any(p.fullmatch(candidate) for p in self.include) and all( + not p.fullmatch(candidate) for p in self.exclude + ) + + @staticmethod + def _glob_list(elems: GlobPattern, separator: str = "."): + if isinstance(elems, str): + return [GlobGroup._glob_to_re(elems, separator)] + else: + return [GlobGroup._glob_to_re(e, separator) for e in elems] + + @staticmethod + def _glob_to_re(pattern: str, separator: str = "."): + # to avoid corner cases for the first component, we prefix the candidate string + # with '.' so `import torch` will regex against `.torch`, assuming '.' is the separator + def component_to_re(component): + if "**" in component: + if component == "**": + return "(" + re.escape(separator) + "[^" + separator + "]+)*" + else: + raise ValueError("** can only appear as an entire path segment") + else: + return re.escape(separator) + ("[^" + separator + "]*").join( + re.escape(x) for x in component.split("*") + ) + + result = "".join(component_to_re(c) for c in pattern.split(separator)) + return re.compile(result) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/importer.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/importer.py new file mode 100644 index 0000000000000000000000000000000000000000..c59b13154c0d2023a711f041fcfb6490184549d3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/importer.py @@ -0,0 +1,264 @@ +# mypy: allow-untyped-defs +import importlib +import logging +import sys +from abc import ABC, abstractmethod + +# pyrefly: ignore [missing-module-attribute] +from pickle import ( # type: ignore[attr-defined] + _getattribute, + _Pickler, + whichmodule as _pickle_whichmodule, # pyrefly: ignore # missing-module-attribute +) +from types import ModuleType +from typing import Any + +from ._mangling import demangle, get_mangle_prefix, is_mangled + + +__all__ = ["ObjNotFoundError", "ObjMismatchError", "Importer", "OrderedImporter"] +log = logging.getLogger(__name__) + + +class ObjNotFoundError(Exception): + """Raised when an importer cannot find an object by searching for its name.""" + + +class ObjMismatchError(Exception): + """Raised when an importer found a different object with the same name as the user-provided one.""" + + +class Importer(ABC): + """Represents an environment to import modules from. + + By default, you can figure out what module an object belongs by checking + __module__ and importing the result using __import__ or importlib.import_module. + + torch.package introduces module importers other than the default one. + Each PackageImporter introduces a new namespace. Potentially a single + name (e.g. 'foo.bar') is present in multiple namespaces. + + It supports two main operations: + import_module: module_name -> module object + get_name: object -> (parent module name, name of obj within module) + + The guarantee is that following round-trip will succeed or throw an ObjNotFoundError/ObjMisMatchError. + module_name, obj_name = env.get_name(obj) + module = env.import_module(module_name) + obj2 = getattr(module, obj_name) + assert obj1 is obj2 + """ + + modules: dict[str, ModuleType] + + @abstractmethod + def import_module(self, module_name: str) -> ModuleType: + """Import `module_name` from this environment. + + The contract is the same as for importlib.import_module. + """ + + def get_name(self, obj: Any, name: str | None = None) -> tuple[str, str]: + """Given an object, return a name that can be used to retrieve the + object from this environment. + + Args: + obj: An object to get the module-environment-relative name for. + name: If set, use this name instead of looking up __name__ or __qualname__ on `obj`. + This is only here to match how Pickler handles __reduce__ functions that return a string, + don't use otherwise. + Returns: + A tuple (parent_module_name, attr_name) that can be used to retrieve `obj` from this environment. + Use it like: + mod = importer.import_module(parent_module_name) + obj = getattr(mod, attr_name) + + Raises: + ObjNotFoundError: we couldn't retrieve `obj by name. + ObjMisMatchError: we found a different object with the same name as `obj`. + """ + if name is None and obj and _Pickler.dispatch.get(type(obj)) is None: + # Honor the string return variant of __reduce__, which will give us + # a global name to search for in this environment. + # TODO: I guess we should do copyreg too? + reduce = getattr(obj, "__reduce__", None) + if reduce is not None: + try: + rv = reduce() + if isinstance(rv, str): + name = rv + except Exception: + pass + if name is None: + name = getattr(obj, "__qualname__", None) + if name is None: + name = obj.__name__ + + orig_module_name = self.whichmodule(obj, name) + # Demangle the module name before importing. If this obj came out of a + # PackageImporter, `__module__` will be mangled. See mangling.md for + # details. + module_name = demangle(orig_module_name) + + # Check that this name will indeed return the correct object + try: + module = self.import_module(module_name) + if sys.version_info >= (3, 14): + # pickle._getatribute signature changes in 3.14 + # to take iterable and return just one object + obj2 = _getattribute(module, name.split(".")) + else: + obj2, _ = _getattribute(module, name) + except (ImportError, KeyError, AttributeError): + raise ObjNotFoundError( + f"{obj} was not found as {module_name}.{name}" + ) from None + + if obj is obj2: + return module_name, name + + def get_obj_info(obj): + assert name is not None + module_name = self.whichmodule(obj, name) + is_mangled_ = is_mangled(module_name) + location = ( + get_mangle_prefix(module_name) + if is_mangled_ + else "the current Python environment" + ) + importer_name = ( + f"the importer for {get_mangle_prefix(module_name)}" + if is_mangled_ + else "'sys_importer'" + ) + return module_name, location, importer_name + + obj_module_name, obj_location, obj_importer_name = get_obj_info(obj) + obj2_module_name, obj2_location, obj2_importer_name = get_obj_info(obj2) + msg = ( + f"\n\nThe object provided is from '{obj_module_name}', " + f"which is coming from {obj_location}." + f"\nHowever, when we import '{obj2_module_name}', it's coming from {obj2_location}." + "\nTo fix this, make sure this 'PackageExporter's importer lists " + f"{obj_importer_name} before {obj2_importer_name}." + ) + raise ObjMismatchError(msg) + + def whichmodule(self, obj: Any, name: str) -> str: + """Find the module name an object belongs to. + + This should be considered internal for end-users, but developers of + an importer can override it to customize the behavior. + + Taken from pickle.py, but modified to exclude the search into sys.modules + """ + module_name = getattr(obj, "__module__", None) + if module_name is not None: + return module_name + + # Protect the iteration by using a list copy of self.modules against dynamic + # modules that trigger imports of other modules upon calls to getattr. + for module_name, module in self.modules.copy().items(): + if ( + module_name == "__main__" + or module_name == "__mp_main__" # bpo-42406 + or module is None + ): + continue + try: + if _getattribute(module, name)[0] is obj: + return module_name + except AttributeError: + pass + + return "__main__" + + +class _SysImporter(Importer): + """An importer that implements the default behavior of Python.""" + + def import_module(self, module_name: str): + return importlib.import_module(module_name) + + def whichmodule(self, obj: Any, name: str) -> str: + # In Python 3.14+, pickle.whichmodule tries to import the module, + # which fails for mangled package names like ''. + # Check __module__ first before calling pickle.whichmodule. + module_name = getattr(obj, "__module__", None) + if module_name is not None: + return module_name + return _pickle_whichmodule(obj, name) + + +sys_importer = _SysImporter() + + +class OrderedImporter(Importer): + """A compound importer that takes a list of importers and tries them one at a time. + + The first importer in the list that returns a result "wins". + """ + + def __init__(self, *args): + self._importers: list[Importer] = list(args) + + def _is_torchpackage_dummy(self, module): + """Returns true iff this module is an empty PackageNode in a torch.package. + + If you intern `a.b` but never use `a` in your code, then `a` will be an + empty module with no source. This can break cases where we are trying to + re-package an object after adding a real dependency on `a`, since + OrderedImportere will resolve `a` to the dummy package and stop there. + + See: https://github.com/pytorch/pytorch/pull/71520#issuecomment-1029603769 + """ + if not getattr(module, "__torch_package__", False): + return False + if not hasattr(module, "__path__"): + return False + if not hasattr(module, "__file__"): + return True + return module.__file__ is None + + def get_name(self, obj: Any, name: str | None = None) -> tuple[str, str]: + for importer in self._importers: + try: + return importer.get_name(obj, name) + except (ObjNotFoundError, ObjMismatchError) as e: + warning_message = ( + f"Tried to call get_name with obj {obj}, " + f"and name {name} on {importer} and got {e}" + ) + log.warning(warning_message) + raise ObjNotFoundError( + f"Could not find obj {obj} and name {name} in any of the importers {self._importers}" + ) + + def import_module(self, module_name: str) -> ModuleType: + last_err = None + for importer in self._importers: + if not isinstance(importer, Importer): + raise TypeError( + f"{importer} is not a Importer. " + "All importers in OrderedImporter must inherit from Importer." + ) + try: + module = importer.import_module(module_name) + if self._is_torchpackage_dummy(module): + continue + return module + except ModuleNotFoundError as err: + last_err = err + + if last_err is not None: + raise last_err + else: + raise ModuleNotFoundError(module_name) + + def whichmodule(self, obj: Any, name: str) -> str: + for importer in self._importers: + module_name = importer.whichmodule(obj, name) + if module_name != "__main__": + return module_name + + return "__main__" diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/package_exporter.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/package_exporter.py new file mode 100644 index 0000000000000000000000000000000000000000..bd6f49c1b034049aeabcd9b67d9eef2124e6f842 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/package_exporter.py @@ -0,0 +1,1190 @@ +# mypy: allow-untyped-defs +import collections +import importlib.machinery +import io +import linecache +import os +import pickletools +import platform +import types +from collections import defaultdict, OrderedDict +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from enum import Enum +from importlib.machinery import SourceFileLoader +from pathlib import Path +from typing import Any, cast, IO + +import torch +from torch.serialization import location_tag, normalize_storage_type +from torch.types import FileLike, Storage +from torch.utils.hooks import RemovableHandle + +from ._digraph import DiGraph +from ._importlib import _normalize_path +from ._mangling import demangle, is_mangled +from ._package_pickler import create_pickler +from ._stdlib import is_stdlib_module +from .find_file_dependencies import find_files_source_depends_on +from .glob_group import GlobGroup, GlobPattern +from .importer import Importer, OrderedImporter, sys_importer + + +__all__ = [ + "PackagingErrorReason", + "EmptyMatchError", + "PackagingError", + "PackageExporter", +] + +_gate_torchscript_serialization = True + +ActionHook = Callable[["PackageExporter", str], None] + + +class _ModuleProviderAction(Enum): + """Represents one of the actions that :class:`PackageExporter` can take on a module. + + See :meth:`PackageExporter.extern` and friends for a description of what the actions do. + """ + + INTERN = 1 + EXTERN = 2 + MOCK = 3 + DENY = 4 + # Special case: when a module is mocked, PackageExporter writes out a + # `_mock` module that implements our mocking stubs. If we re-package code, + # we may encounter a `_mock` module from the original package. If we do, + # just ignore it and write a `_mock` module once. + REPACKAGED_MOCK_MODULE = 5 + # Special case: PackageImporter adds a fake module + # (`torch_package_importer`) that allows packaged code to access it. Don't + # re-export this. + SKIP = 6 + + +class PackagingErrorReason(Enum): + """Listing of different reasons a dependency may fail to package. + + This enum is used to provide good error messages when + :class:`PackagingError` is raised. + """ + + def __repr__(self): + return f"<{self.__class__.__name__}.{self.name}>" + + IS_EXTENSION_MODULE = ( + "Module is a C extension module. torch.package supports Python modules only." + ) + NO_DUNDER_FILE = "Module had no __file__ defined." + SOURCE_FILE_NOT_FOUND = ( + "Module had a __file__, but we could not find it in your filesystem." + ) + DEPENDENCY_RESOLUTION_FAILED = "Dependency resolution failed." + NO_ACTION = ( + "Module did not match against any action pattern. Extern, mock, or intern it." + ) + DENIED = "Module was denied by a pattern." + MOCKED_BUT_STILL_USED = ( + "Module was mocked out, but is still being used in the package. " + "Please intern or extern the mocked modules if objects are supposed to be in " + "the package." + ) + + +@dataclass +class _PatternInfo: + """Holds :class:`PackageExporter`-specific info about how to execute matches against""" + + # What action to take on a module that matches this pattern. + action: _ModuleProviderAction + # The value of `allow_empty` the user gave when specifying the pattern. + allow_empty: bool + # Whether this pattern has been matched during packaging. + was_matched: bool + + def __init__(self, action, allow_empty): + self.action = action + self.allow_empty = allow_empty + self.was_matched = False + + +class EmptyMatchError(Exception): + """This is an exception that is thrown when a mock or extern is marked as + ``allow_empty=False``, and is not matched with any module during packaging. + """ + + +class PackagingError(Exception): + """This exception is raised when there is an issue with exporting a package. + ``PackageExporter`` will attempt to gather up all the errors and present + them to you at once. + """ + + def __init__(self, dependency_graph: DiGraph, debug=False): + # Group errors by reason. + broken: dict[PackagingErrorReason, list[str]] = defaultdict(list) + for module_name, attrs in dependency_graph.nodes.items(): + error = attrs.get("error") + if error is None: + continue + if error == PackagingErrorReason.NO_ACTION: + assert "action" not in attrs + broken[error].append(module_name) + + message = io.StringIO() + message.write("\n") + + for reason, module_names in broken.items(): + message.write(f"* {reason.value}\n") + for module_name in module_names: + message.write(f" {module_name}\n") + + # Print additional context if it's provided. + error_context = dependency_graph.nodes[module_name].get("error_context") + if error_context is not None: + message.write(f" Context: {error_context}\n") + if module_name in _DISALLOWED_MODULES: + message.write( + " Note: While we usually use modules in the python standard library " + f"from the local environment, `{module_name}` has a lot of system " + "level access and therefore can pose a security risk. We heavily " + f"recommend removing `{module_name}` from your packaged code. However, if that " + "is not possible, add it to the extern list by calling " + f'PackageExporter.extern("`{module_name}`")\n' + ) + if debug: + module_path = dependency_graph.first_path(module_name) + message.write( + f" A path to {module_name}: {' -> '.join(module_path)}\n" + ) + if not debug: + message.write("\n") + message.write( + "Set debug=True when invoking PackageExporter for a visualization of where " + "broken modules are coming from!\n" + ) + # Save the dependency graph so that tooling can get at it. + self.dependency_graph = dependency_graph + super().__init__(message.getvalue()) + + +class PackageExporter: + """Exporters allow you to write packages of code, pickled Python data, and + arbitrary binary and text resources into a self-contained package. + + Imports can load this code in a hermetic way, such that code is loaded + from the package rather than the normal Python import system. This allows + for the packaging of PyTorch model code and data so that it can be run + on a server or used in the future for transfer learning. + + The code contained in packages is copied file-by-file from the original + source when it is created, and the file format is a specially organized + zip file. Future users of the package can unzip the package, and edit the code + in order to perform custom modifications to it. + + The importer for packages ensures that code in the module can only be loaded from + within the package, except for modules explicitly listed as external using :meth:`extern`. + The file ``extern_modules`` in the zip archive lists all the modules that a package externally depends on. + This prevents "implicit" dependencies where the package runs locally because it is importing + a locally-installed package, but then fails when the package is copied to another machine. + + When source code is added to the package, the exporter can optionally scan it + for further code dependencies (``dependencies=True``). It looks for import statements, + resolves relative references to qualified module names, and performs an action specified by the user + (See: :meth:`extern`, :meth:`mock`, and :meth:`intern`). + """ + + """A importer that will be searched in order to find the modules referenced by other modules or by + pickled objects. The default module environment just uses sys_importer, which searches the Python environment. + """ + importer: Importer + + def __init__( + self, + f: FileLike, + importer: Importer | Sequence[Importer] = sys_importer, + debug: bool = False, + ) -> None: + """ + Create an exporter. + + Args: + f: The location to export to. Can be a ``string``/``Path`` object containing a filename + or a binary I/O object. + importer: If a single Importer is passed, use that to search for modules. + If a sequence of importers are passed, an ``OrderedImporter`` will be constructed out of them. + debug: If set to True, add path of broken modules to PackagingErrors. + """ + torch._C._log_api_usage_once("torch.package.PackageExporter") + self.debug = debug + if isinstance(f, (str, os.PathLike)): + f = os.fspath(f) + self.buffer: IO[bytes] | None = None + else: # is a byte buffer + self.buffer = f + + self.zip_file = torch._C.PyTorchFileWriter(f) + self.zip_file.set_min_version(6) + self._written_files: set[str] = set() + + self.serialized_reduces: dict[int, Any] = {} + + # A graph tracking all the modules and pickle objects added to this + # package and the dependencies between them. + # - Each node is a module name (or a pickle name that looks like '') + # - Each directed edge (u, v) means u depends on v. + # - Nodes may contain metadata that describe how to write the thing to the zipfile. + self.dependency_graph = DiGraph() + self.script_module_serializer = torch._C.ScriptModuleSerializer(self.zip_file) + self.storage_context = self.script_module_serializer.storage_context() + + # These are OrderedDicts for compatibility with RemovableHandle. + # Generic OrderedDict type annotations are not present until 3.7. + # The real type signature is OrderedDict[int, Callable[[PackageExporter, str], None]] + self._extern_hooks: OrderedDict = OrderedDict() + self._mock_hooks: OrderedDict = OrderedDict() + self._intern_hooks: OrderedDict = OrderedDict() + + if isinstance(importer, Importer): + self.importer = importer + else: + if not isinstance(importer, collections.abc.Sequence): + raise TypeError( + "importer arg should be an Importer or a sequence of Importers, " + f"got {type(importer)} instead." + ) + self.importer = OrderedImporter(*importer) + + self.patterns: dict[GlobGroup, _PatternInfo] = {} + self._unique_id = 0 + + def save_source_file( + self, module_name: str, file_or_directory: str, dependencies=True + ): + """Adds the local file system ``file_or_directory`` to the source package to provide the code + for ``module_name``. + + Args: + module_name (str): e.g. ``"my_package.my_subpackage"``, code will be saved to provide code for this package. + file_or_directory (str): the path to a file or directory of code. When a directory, all python files in the directory + are recursively copied using :meth:`save_source_file`. If a file is named ``"/__init__.py"`` the code is treated + as a package. + dependencies (bool, optional): If ``True``, we scan the source for dependencies. + """ + path = Path(file_or_directory) + if path.is_dir(): + to_save = [] # list of tuples with arguments to save_source_string + module_path = module_name.replace(".", "/") + for filename in path.glob("**/*.py"): + relative_path = filename.relative_to(path).as_posix() + archivename = module_path + "/" + relative_path + submodule_name = None + if filename.name == "__init__.py": + submodule_name = archivename[: -len("/__init__.py")].replace( + "/", "." + ) + is_package = True + else: + submodule_name = archivename[: -len(".py")].replace("/", ".") + is_package = False + + # we delay the call to save_source_string so that we record all the source files + # being provided by this directory structure _before_ attempting to resolve the dependencies + # on the source. This makes sure we don't try to copy over modules that will just get + # overwritten by this directory blob + to_save.append( + ( + submodule_name, + _read_file(str(filename)), + is_package, + dependencies, + ) + ) + + for item in to_save: + self.save_source_string(*item) + else: + is_package = path.name == "__init__.py" + self.save_source_string( + module_name, + _read_file(file_or_directory), + is_package, + dependencies, + ) + + def get_unique_id(self) -> str: + """Get an id. This id is guaranteed to only be handed out once for this package.""" + ret = str(self._unique_id) + self._unique_id += 1 + return ret + + def _get_dependencies( + self, src: str, module_name: str, is_package: bool + ) -> list[str]: + """Return all modules that this source code depends on. + + Dependencies are found by scanning the source code for import-like statements. + + Arguments: + src: The Python source code to analyze for dependencies. + module_name: The name of the module that ``src`` corresponds to. + is_package: Whether this module should be treated as a package. + See :py:meth:`save_source_string` for more info. + + Returns: + A list containing modules detected as direct dependencies in + ``src``. The items in the list are guaranteed to be unique. + """ + package_name = ( + module_name if is_package else module_name.rsplit(".", maxsplit=1)[0] + ) + try: + dep_pairs = find_files_source_depends_on(src, package_name) + except Exception as e: + self.dependency_graph.add_node( + module_name, + error=PackagingErrorReason.DEPENDENCY_RESOLUTION_FAILED, + error_context=str(e), + ) + return [] + + # Use a dict to get uniquing but also deterministic order + dependencies = {} + for dep_module_name, dep_module_obj in dep_pairs: + # handle the case where someone did something like `from pack import sub` + # where `sub` is a submodule. In this case we don't have to save pack, just sub. + # this ensures we don't pick up additional dependencies on pack. + # However, in the case where `sub` is not a submodule but an object, then we do have + # to save pack. + if dep_module_obj is not None: + possible_submodule = f"{dep_module_name}.{dep_module_obj}" + if self._module_exists(possible_submodule): + dependencies[possible_submodule] = True + # we don't need to save `pack` + continue + if self._module_exists(dep_module_name): + dependencies[dep_module_name] = True + + return list(dependencies.keys()) + + def save_source_string( + self, + module_name: str, + src: str, + is_package: bool = False, + dependencies: bool = True, + ): + """Adds ``src`` as the source code for ``module_name`` in the exported package. + + Args: + module_name (str): e.g. ``my_package.my_subpackage``, code will be saved to provide code for this package. + src (str): The Python source code to save for this package. + is_package (bool, optional): If ``True``, this module is treated as a package. Packages are allowed to have submodules + (e.g. ``my_package.my_subpackage.my_subsubpackage``), and resources can be saved inside them. Defaults to ``False``. + dependencies (bool, optional): If ``True``, we scan the source for dependencies. + """ + self.dependency_graph.add_node( + module_name, + source=src, + is_package=is_package, + provided=True, + action=_ModuleProviderAction.INTERN, + ) + + if dependencies: + deps = self._get_dependencies(src, module_name, is_package) + + for dep in deps: + self.dependency_graph.add_edge(module_name, dep) + self.add_dependency(dep) + + def _write_source_string( + self, + module_name: str, + src: str, + is_package: bool = False, + ): + """Write ``src`` as the source code for ``module_name`` in the zip archive. + + Arguments are otherwise the same as for :meth:`save_source_string`. + """ + extension = "/__init__.py" if is_package else ".py" + filename = module_name.replace(".", "/") + extension + + self._write(filename, src) + + def _import_module(self, module_name: str): + try: + return self.importer.import_module(module_name) + except ModuleNotFoundError: + if not is_mangled(module_name): + raise + msg = ( + f"Module not found: '{module_name}'. Make sure the PackageImporter that " + "created this module is present in `self.importer`" + ) + raise ModuleNotFoundError(msg) from None + + def _module_exists(self, module_name: str) -> bool: + try: + self._import_module(module_name) + return True + except Exception: + return False + + def _get_source_of_module(self, module: types.ModuleType) -> str | None: + filename = None + spec = getattr(module, "__spec__", None) + if spec is not None: + loader = getattr(spec, "loader", None) + if loader is not None and isinstance(loader, SourceFileLoader): + try: + filename = loader.get_filename(module.__name__) + except ImportError: + pass + if filename is None: + filename = getattr(module, "__file__", None) + if isinstance(filename, str) and filename.endswith(".py"): + return "".join(linecache.getlines(filename, module.__dict__)) + return None + + def add_dependency(self, module_name: str, dependencies=True): + """Given a module, add it to the dependency graph according to patterns + specified by the user. + """ + if ( + module_name in self.dependency_graph + and self.dependency_graph.nodes[module_name].get("provided") is True + ): + return + + # Special case: PackageImporter provides a special module called + # `torch_package_importer` that allows packaged modules to reference + # their PackageImporter. We don't want to re-export this. + if module_name == "torch_package_importer": + self.dependency_graph.add_node( + module_name, + action=_ModuleProviderAction.SKIP, + provided=True, + ) + return + + if module_name == "_mock": + self.dependency_graph.add_node( + module_name, + action=_ModuleProviderAction.REPACKAGED_MOCK_MODULE, + provided=True, + ) + return + + if self._can_implicitly_extern(module_name): + self.dependency_graph.add_node( + module_name, action=_ModuleProviderAction.EXTERN, provided=True + ) + return + + for pattern, pattern_info in self.patterns.items(): + if pattern.matches(module_name): + pattern_info.was_matched = True + self.dependency_graph.add_node( + module_name, action=pattern_info.action, provided=True + ) + + if pattern_info.action == _ModuleProviderAction.DENY: + # Requiring a denied module just adds an error to the graph. + self.dependency_graph.add_node( + module_name, error=PackagingErrorReason.DENIED + ) + + # If we are interning this module, we need to retrieve its + # dependencies and package those as well. + if pattern_info.action == _ModuleProviderAction.INTERN: + self._intern_module(module_name, dependencies) + return + + # No patterns have matched. Explicitly add this as an error. + self.dependency_graph.add_node( + module_name, error=PackagingErrorReason.NO_ACTION + ) + + def save_module(self, module_name: str, dependencies=True): + """Save the code for ``module`` into the package. Code for the module is resolved using the ``importers`` path to find the + module object, and then using its ``__file__`` attribute to find the source code. + + Args: + module_name (str): e.g. ``my_package.my_subpackage``, code will be saved to provide code + for this package. + dependencies (bool, optional): If ``True``, we scan the source for dependencies. + """ + if not isinstance(module_name, str): + raise TypeError( + "save_module() expects a string input, did you perhaps mean to pass `__name__`?" + ) + + self._intern_module(module_name, dependencies) + + def _intern_module( + self, + module_name: str, + dependencies: bool, + ): + """Adds the module to the dependency graph as an interned module, + along with any metadata needed to write it out to the zipfile at serialization time. + """ + module_obj = self._import_module(module_name) + # Subtle: if the import above succeeded, either: + # 1. The module name is not mangled, and this was just a regular import, or + # 2. The module name is mangled, but one of the importers was able to + # recognize the mangling and import it. + # Either way, it is now safe to demangle this name so that we don't + # serialize the mangled version to the package. + module_name = demangle(module_name) + + # Find dependencies of this module and require them as well. + is_package = hasattr(module_obj, "__path__") + source = self._get_source_of_module(module_obj) + if source is None: + # Couldn't find a source! Add it to our dependency graph as broken + # and continue. + filename = getattr(module_obj, "__file__", None) + error_context = None + if filename is None: + packaging_error = PackagingErrorReason.NO_DUNDER_FILE + elif filename.endswith(tuple(importlib.machinery.EXTENSION_SUFFIXES)): + packaging_error = PackagingErrorReason.IS_EXTENSION_MODULE + else: + packaging_error = PackagingErrorReason.SOURCE_FILE_NOT_FOUND + error_context = f"filename: {filename}" + self.dependency_graph.add_node( + module_name, + action=_ModuleProviderAction.INTERN, + is_package=is_package, + error=packaging_error, + error_context=error_context, + provided=True, + ) + return + + self.dependency_graph.add_node( + module_name, + action=_ModuleProviderAction.INTERN, + is_package=is_package, + source=source, + provided=True, + ) + + if dependencies: + deps = self._get_dependencies(source, module_name, is_package) + for dep in deps: + self.dependency_graph.add_edge(module_name, dep) + self.add_dependency(dep) + + def save_pickle( + self, + package: str, + resource: str, + obj: Any, + dependencies: bool = True, + pickle_protocol: int = 3, + ): + """Save a python object to the archive using pickle. Equivalent to :func:`torch.save` but saving into + the archive rather than a stand-alone file. Standard pickle does not save the code, only the objects. + If ``dependencies`` is true, this method will also scan the pickled objects for which modules are required + to reconstruct them and save the relevant code. + + To be able to save an object where ``type(obj).__name__`` is ``my_module.MyObject``, + ``my_module.MyObject`` must resolve to the class of the object according to the ``importer`` order. When saving objects that + have previously been packaged, the importer's ``import_module`` method will need to be present in the ``importer`` list + for this to work. + + Args: + package (str): The name of module package this resource should go in (e.g. ``"my_package.my_subpackage"``). + resource (str): A unique name for the resource, used to identify it to load. + obj (Any): The object to save, must be picklable. + dependencies (bool, optional): If ``True``, we scan the source for dependencies. + """ + + assert (pickle_protocol == 4) or (pickle_protocol == 3), ( + "torch.package only supports pickle protocols 3 and 4" + ) + + filename = self._filename(package, resource) + # Write the pickle data for `obj` + data_buf = io.BytesIO() + pickler = create_pickler(data_buf, self.importer, protocol=pickle_protocol) + pickler.persistent_id = self._persistent_id + pickler.dump(obj) + data_value = data_buf.getvalue() + mocked_modules = defaultdict(list) + name_in_dependency_graph = f"<{package}.{resource}>" + self.dependency_graph.add_node( + name_in_dependency_graph, + action=_ModuleProviderAction.INTERN, + provided=True, + is_pickle=True, + ) + + def _check_mocked_error(module: str | None, field: str | None): + """ + checks if an object (field) comes from a mocked module and then adds + the pair to mocked_modules which contains mocked modules paired with their + list of mocked objects present in the pickle. + + We also hold the invariant that the first user defined rule that applies + to the module is the one we use. + """ + + assert isinstance(module, str) + assert isinstance(field, str) + if self._can_implicitly_extern(module): + return + for pattern, pattern_info in self.patterns.items(): + if pattern.matches(module): + if pattern_info.action == _ModuleProviderAction.MOCK: + mocked_modules[module].append(field) + return + + if dependencies: + all_dependencies = [] + module = None + field = None + memo: defaultdict[int, str] = defaultdict(None) + memo_count = 0 + # pickletools.dis(data_value) + # pyrefly: ignore [bad-assignment] + for opcode, arg, _pos in pickletools.genops(data_value): + if pickle_protocol == 4: + if ( + opcode.name == "SHORT_BINUNICODE" + or opcode.name == "BINUNICODE" + or opcode.name == "BINUNICODE8" + ): + assert isinstance(arg, str) + module = field + field = arg + memo[memo_count] = arg + elif ( + opcode.name == "LONG_BINGET" + or opcode.name == "BINGET" + or opcode.name == "GET" + ): + assert isinstance(arg, int) + module = field + field = memo.get(arg, None) + elif opcode.name == "MEMOIZE": + memo_count += 1 + elif opcode.name == "STACK_GLOBAL": + if module is None: + # If not module was passed on in the entries preceding this one, continue. + continue + assert isinstance(module, str) + if module not in all_dependencies: + all_dependencies.append(module) + _check_mocked_error(module, field) + elif ( + pickle_protocol == 3 and opcode.name == "GLOBAL" + ): # a global reference + assert isinstance(arg, str) + module, field = arg.split(" ") + if module not in all_dependencies: + all_dependencies.append(module) + _check_mocked_error(module, field) + for module_name in all_dependencies: + self.dependency_graph.add_edge(name_in_dependency_graph, module_name) + + """ If an object happens to come from a mocked module, then we collect these errors and spit them + out with the other errors found by package exporter. + """ + if module_name in mocked_modules: + assert isinstance(module_name, str) + fields = mocked_modules[module_name] + self.dependency_graph.add_node( + module_name, + action=_ModuleProviderAction.MOCK, + error=PackagingErrorReason.MOCKED_BUT_STILL_USED, + error_context=f"Object(s) '{fields}' from module `{module_name}` was mocked out during packaging " + f"but is being used in resource - `{resource}` in package `{package}`. ", + provided=True, + ) + else: + self.add_dependency(module_name) + + self._write(filename, data_value) + + def save_text(self, package: str, resource: str, text: str): + """Save text data to the package. + + Args: + package (str): The name of module package this resource should go it (e.g. ``"my_package.my_subpackage"``). + resource (str): A unique name for the resource, used to identify it to load. + text (str): The contents to save. + """ + return self.save_binary(package, resource, text.encode("utf-8")) + + def save_binary(self, package, resource, binary: bytes): + """Save raw bytes to the package. + + Args: + package (str): The name of module package this resource should go it (e.g. ``"my_package.my_subpackage"``). + resource (str): A unique name for the resource, used to identify it to load. + binary (str): The data to save. + """ + filename = self._filename(package, resource) + self._write(filename, binary) + + def register_extern_hook(self, hook: ActionHook) -> RemovableHandle: + """Registers an extern hook on the exporter. + + The hook will be called each time a module matches against an :meth:`extern` pattern. + It should have the following signature:: + + hook(exporter: PackageExporter, module_name: str) -> None + + Hooks will be called in order of registration. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + A handle that can be used to remove the added hook by calling + ``handle.remove()``. + """ + handle = RemovableHandle(self._extern_hooks) + self._extern_hooks[handle.id] = hook + return handle + + def register_mock_hook(self, hook: ActionHook) -> RemovableHandle: + """Registers a mock hook on the exporter. + + The hook will be called each time a module matches against a :meth:`mock` pattern. + It should have the following signature:: + + hook(exporter: PackageExporter, module_name: str) -> None + + Hooks will be called in order of registration. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + A handle that can be used to remove the added hook by calling + ``handle.remove()``. + """ + handle = RemovableHandle(self._mock_hooks) + self._mock_hooks[handle.id] = hook + return handle + + def register_intern_hook(self, hook: ActionHook) -> RemovableHandle: + """Registers an intern hook on the exporter. + + The hook will be called each time a module matches against an :meth:`intern` pattern. + It should have the following signature:: + + hook(exporter: PackageExporter, module_name: str) -> None + + Hooks will be called in order of registration. + + Returns: + :class:`torch.utils.hooks.RemovableHandle`: + A handle that can be used to remove the added hook by calling + ``handle.remove()``. + """ + handle = RemovableHandle(self._intern_hooks) + self._intern_hooks[handle.id] = hook + return handle + + def intern( + self, + include: "GlobPattern", + *, + exclude: "GlobPattern" = (), + allow_empty: bool = True, + ): + """Specify modules that should be packaged. A module must match some ``intern`` pattern in order to be + included in the package and have its dependencies processed recursively. + + Args: + include (Union[List[str], str]): A string e.g. "my_package.my_subpackage", or list of strings + for the names of the modules to be externed. This can also be a glob-style pattern, as described in :meth:`mock`. + + exclude (Union[List[str], str]): An optional pattern that excludes some patterns that match the include string. + + allow_empty (bool): An optional flag that specifies whether the intern modules specified by this call + to the ``intern`` method must be matched to some module during packaging. If an ``intern`` module glob + pattern is added with ``allow_empty=False``, and :meth:`close` is called (either explicitly or via ``__exit__``) + before any modules match that pattern, an exception is thrown. If ``allow_empty=True``, no such exception is thrown. + + """ + self.patterns[GlobGroup(include, exclude=exclude)] = _PatternInfo( + _ModuleProviderAction.INTERN, allow_empty + ) + + def mock( + self, + include: "GlobPattern", + *, + exclude: "GlobPattern" = (), + allow_empty: bool = True, + ): + """Replace some required modules with a mock implementation. Mocked modules will return a fake + object for any attribute accessed from it. Because we copy file-by-file, the dependency resolution will sometimes + find files that are imported by model files but whose functionality is never used + (e.g. custom serialization code or training helpers). + Use this function to mock this functionality out without having to modify the original code. + + Args: + include (Union[List[str], str]): A string e.g. ``"my_package.my_subpackage"``, or list of strings + for the names of the modules to be mocked out. Strings can also be a glob-style pattern + string that may match multiple modules. Any required dependencies that match this pattern + string will be mocked out automatically. + + Examples : + ``'torch.**'`` -- matches ``torch`` and all submodules of torch, e.g. ``'torch.nn'`` + and ``'torch.nn.functional'`` + + ``'torch.*'`` -- matches ``'torch.nn'`` or ``'torch.functional'``, but not + ``'torch.nn.functional'`` + + exclude (Union[List[str], str]): An optional pattern that excludes some patterns that match the include string. + e.g. ``include='torch.**', exclude='torch.foo'`` will mock all torch packages except ``'torch.foo'``, + Default: is ``[]``. + + allow_empty (bool): An optional flag that specifies whether the mock implementation(s) specified by this call + to the :meth:`mock` method must be matched to some module during packaging. If a mock is added with + ``allow_empty=False``, and :meth:`close` is called (either explicitly or via ``__exit__``) and the mock has + not been matched to a module used by the package being exported, an exception is thrown. + If ``allow_empty=True``, no such exception is thrown. + + """ + self.patterns[GlobGroup(include, exclude=exclude)] = _PatternInfo( + _ModuleProviderAction.MOCK, allow_empty + ) + + def extern( + self, + include: "GlobPattern", + *, + exclude: "GlobPattern" = (), + allow_empty: bool = True, + ): + """Include ``module`` in the list of external modules the package can import. + This will prevent dependency discovery from saving + it in the package. The importer will load an external module directly from the standard import system. + Code for extern modules must also exist in the process loading the package. + + Args: + include (Union[List[str], str]): A string e.g. ``"my_package.my_subpackage"``, or list of strings + for the names of the modules to be externed. This can also be a glob-style pattern, as + described in :meth:`mock`. + + exclude (Union[List[str], str]): An optional pattern that excludes some patterns that match the + include string. + + allow_empty (bool): An optional flag that specifies whether the extern modules specified by this call + to the ``extern`` method must be matched to some module during packaging. If an extern module glob + pattern is added with ``allow_empty=False``, and :meth:`close` is called (either explicitly or via + ``__exit__``) before any modules match that pattern, an exception is thrown. If ``allow_empty=True``, + no such exception is thrown. + + """ + self.patterns[GlobGroup(include, exclude=exclude)] = _PatternInfo( + _ModuleProviderAction.EXTERN, allow_empty + ) + + def deny(self, include: "GlobPattern", *, exclude: "GlobPattern" = ()): + """Blocklist modules who names match the given glob patterns from the list of modules the package can import. + If a dependency on any matching packages is found, a :class:`PackagingError` is raised. + + Args: + include (Union[List[str], str]): A string e.g. ``"my_package.my_subpackage"``, or list of strings + for the names of the modules to be externed. This can also be a glob-style pattern, as described in :meth:`mock`. + + exclude (Union[List[str], str]): An optional pattern that excludes some patterns that match the include string. + """ + self.patterns[GlobGroup(include, exclude=exclude)] = _PatternInfo( + _ModuleProviderAction.DENY, allow_empty=True + ) + + def _persistent_id(self, obj): + if torch.is_storage(obj) or isinstance(obj, torch.storage.TypedStorage): + storage: Storage + if isinstance(obj, torch.storage.TypedStorage): + # TODO: Once we decide to break serialization FC, we can + # remove this case + untyped_storage = obj._untyped_storage + storage_type_str = obj.pickle_storage_type() + storage_type = getattr(torch, storage_type_str) + storage = cast(Storage, untyped_storage) + storage_numel = obj.size() + + elif isinstance(obj, torch.UntypedStorage): + untyped_storage = obj + storage = cast(Storage, untyped_storage) + storage_type = normalize_storage_type(type(storage)) + storage_numel = storage.nbytes() + else: + raise RuntimeError(f"storage type not recognized: {type(obj)}") + + location = location_tag(storage) + + # serialize storage if not already written + storage_present = self.storage_context.has_storage(storage) + storage_id = self.storage_context.get_or_add_storage(storage) + if not storage_present: + if storage.device.type != "cpu": + storage = storage.cpu() + num_bytes = storage.nbytes() + self.zip_file.write_record( + f".data/{storage_id}.storage", storage, num_bytes + ) + return ("storage", storage_type, storage_id, location, storage_numel) + + if hasattr(obj, "__reduce_package__"): + if _gate_torchscript_serialization and isinstance( + obj, torch.jit.RecursiveScriptModule + ): + raise Exception( # noqa: TRY002 + "Serializing ScriptModules directly into a package is a beta feature. " + "To use, set global " + "`torch.package.package_exporter._gate_torchscript_serialization` to `False`." + ) + if self.serialized_reduces.get(id(obj)) is None: + self.serialized_reduces[id(obj)] = ( + "reduce_package", + id(obj), + *obj.__reduce_package__(self), + ) + + return self.serialized_reduces[id(obj)] + + return None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + # If __exit__ was called because an exception was raised, we do not + # attempt to finalize the package. Instead, control is returned to the + # caller to continue raising the exception. + if exc_type is not None: + # Do the bare minimum to leave the open buffer in a valid state. + self._finalize_zip() + return + + self.close() + + def _write(self, filename, str_or_bytes): + if filename in self._written_files: + raise AssertionError( + f"Tried to write file '{filename}', but it already exists in this archive. " + "Please file a bug." + ) + self._written_files.add(filename) + + if is_mangled(filename): + raise AssertionError( + f"Tried to save a torch.package'd module as '{filename}'. " + "Directly saving torch.package'd modules is not allowed." + ) + if isinstance(str_or_bytes, str): + str_or_bytes = str_or_bytes.encode("utf-8") + self.zip_file.write_record(filename, str_or_bytes, len(str_or_bytes)) + + def _validate_dependency_graph(self): + # 1. Check the graph for any errors inserted during dependency analysis. + for attrs in self.dependency_graph.nodes.values(): + if "error" in attrs: + raise PackagingError(self.dependency_graph, debug=self.debug) + + # 2. Check that all patterns for which allow_empty=False have been matched at least once. + for pattern, pattern_info in self.patterns.items(): + if not pattern_info.allow_empty and not pattern_info.was_matched: + raise EmptyMatchError( + f"Exporter did not match any modules to {pattern}, which was marked as allow_empty=False" + ) + + def _write_mock_file(self): + if "_mock.py" not in self._written_files: + mock_file = str(Path(__file__).parent / "_mock.py") + self._write_source_string("_mock", _read_file(mock_file), is_package=False) + + def _execute_dependency_graph(self): + """Takes a finalized dependency graph describing how to package all + modules and executes it, writing to the ZIP archive. + """ + self._validate_dependency_graph() + + extern_modules = [] + for module_name, attrs in self.dependency_graph.nodes.items(): + action = attrs["action"] + + if action == _ModuleProviderAction.EXTERN: + for hook in self._extern_hooks.values(): + hook(self, module_name) + + extern_modules.append(module_name) + + elif action == _ModuleProviderAction.MOCK: + for hook in self._mock_hooks.values(): + hook(self, module_name) + + self._write_mock_file() + + is_package = hasattr(self._import_module(module_name), "__path__") + self._write_source_string(module_name, _MOCK_IMPL, is_package) + + elif action == _ModuleProviderAction.INTERN: + for hook in self._intern_hooks.values(): + hook(self, module_name) + + # The node in the dependency graph contains metadata that tells us + # how to intern the module. + if "provided" not in attrs: + raise AssertionError( + f"Module was marked `intern` but not provided: {module_name}" + ) + + if attrs.get("is_pickle") is True: + # This node came from save_pickle, we don't need to write any source for it. + continue + + is_package = attrs["is_package"] + source = attrs["source"] + self._write_source_string(module_name, source, is_package) + + elif action == _ModuleProviderAction.REPACKAGED_MOCK_MODULE: + self._write_mock_file() + elif action == _ModuleProviderAction.SKIP: + continue + else: + raise AssertionError( + f"Invalid action: {module_name}, {action}. Please report a bug to PyTorch." + ) + + extern_file_contents = "\n".join(extern_modules) + "\n" + self._write(".data/extern_modules", extern_file_contents) + + def _write_python_version(self): + """Writes the python version that the package was created with to .data/python_version""" + self._write(".data/python_version", platform.python_version()) + + def close(self): + """Write the package to the filesystem. Any calls after :meth:`close` are now invalid. + It is preferable to use resource guard syntax instead:: + + with PackageExporter("file.zip") as e: + ... + """ + self._execute_dependency_graph() + self._write_python_version() + + self.script_module_serializer.write_files() + self._finalize_zip() + + def _finalize_zip(self): + """Called at the very end of packaging to leave the zipfile in a closed but valid state.""" + del self.zip_file + if self.buffer: + self.buffer.flush() + + def _filename(self, package, resource): + package_path = package.replace(".", "/") + resource = _normalize_path(resource) + return f"{package_path}/{resource}" + + def _can_implicitly_extern(self, module_name: str): + top_level_package_name = module_name.partition(".")[0] + return top_level_package_name == "torch" or ( + top_level_package_name not in _DISALLOWED_MODULES + and is_stdlib_module(top_level_package_name) + ) + + def dependency_graph_string(self) -> str: + """Returns digraph string representation of dependencies in package. + + Returns: + A string representation of dependencies in package. + """ + return self.dependency_graph.to_dot() + + def _nodes_with_action_type( + self, action: _ModuleProviderAction | None + ) -> list[str]: + result = [] + for name, node_dict in self.dependency_graph.nodes.items(): + node_action = node_dict.get("action", None) + if node_action == action and "is_pickle" not in node_dict: + result.append(name) + result.sort() + return result + + def externed_modules(self) -> list[str]: + """Return all modules that are currently externed. + + Returns: + A list containing the names of modules which will be + externed in this package. + """ + return self._nodes_with_action_type(_ModuleProviderAction.EXTERN) + + def interned_modules(self) -> list[str]: + """Return all modules that are currently interned. + + Returns: + A list containing the names of modules which will be + interned in this package. + """ + return self._nodes_with_action_type(_ModuleProviderAction.INTERN) + + def mocked_modules(self) -> list[str]: + """Return all modules that are currently mocked. + + Returns: + A list containing the names of modules which will be + mocked in this package. + """ + return self._nodes_with_action_type(_ModuleProviderAction.MOCK) + + def denied_modules(self) -> list[str]: + """Return all modules that are currently denied. + + Returns: + A list containing the names of modules which will be + denied in this package. + """ + return self._nodes_with_action_type(_ModuleProviderAction.DENY) + + def get_rdeps(self, module_name: str) -> list[str]: + """Return a list of all modules which depend on the module ``module_name``. + + Returns: + A list containing the names of modules which depend on ``module_name``. + """ + if module_name in self.dependency_graph._pred: + return list(self.dependency_graph._pred[module_name].keys()) + else: + return [] + + def all_paths(self, src: str, dst: str) -> str: + """Return a dot representation of the subgraph + that has all paths from src to dst. + + Returns: + A dot representation containing all paths from src to dst. + (https://graphviz.org/doc/info/lang.html) + """ + return self.dependency_graph.all_paths(src, dst) + + +# even though these are in the standard library, we do not allow them to be +# automatically externed since they offer a lot of system level access +_DISALLOWED_MODULES = ["sys", "io"] + +_MOCK_IMPL = """\ +from _mock import MockedObject +def __getattr__(attr: str): + return MockedObject(__name__ + '.' + attr, _suppress_err=True) +""" + + +def _read_file(filename: str) -> str: + with open(filename, "rb") as f: + b = f.read() + return b.decode("utf-8") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/package_importer.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/package_importer.py new file mode 100644 index 0000000000000000000000000000000000000000..e173398f197f04b5865b2ed17913d96c732185ec --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/package/package_importer.py @@ -0,0 +1,789 @@ +# mypy: allow-untyped-defs +import builtins +import importlib +import importlib.machinery +import inspect +import io +import linecache +import os +import sys +import types +from collections.abc import Callable, Iterable +from contextlib import contextmanager +from typing import Any, cast, TYPE_CHECKING +from weakref import WeakValueDictionary + +import torch +from torch.serialization import _get_restore_location, _maybe_decode_ascii +from torch.types import FileLike + +from ._directory_reader import DirectoryReader +from ._importlib import ( + _calc___package__, + _normalize_line_endings, + _normalize_path, + _resolve_name, + _sanity_check, +) +from ._mangling import demangle, PackageMangler +from ._package_unpickler import PackageUnpickler +from .file_structure_representation import _create_directory_from_file_list, Directory +from .importer import Importer + + +if TYPE_CHECKING: + from .glob_group import GlobPattern + +__all__ = ["PackageImporter"] + + +# This is a list of imports that are implicitly allowed even if they haven't +# been marked as extern. This is to work around the fact that Torch implicitly +# depends on numpy and package can't track it. +# https://github.com/pytorch/multipy/issues/46 # codespell:ignore multipy +IMPLICIT_IMPORT_ALLOWLIST: Iterable[str] = [ + "numpy", + "numpy.core", + "numpy.core._multiarray_umath", + # FX GraphModule might depend on builtins module and users usually + # don't extern builtins. Here we import it here by default. + "builtins", +] + + +# Compatibility name mapping to facilitate upgrade of external modules. +# The primary motivation is to enable Numpy upgrade that many modules +# depend on. The latest release of Numpy removed `numpy.str` and +# `numpy.bool` breaking unpickling for many modules. +EXTERN_IMPORT_COMPAT_NAME_MAPPING: dict[str, dict[str, Any]] = { + "numpy": { + "str": str, + "bool": bool, + }, +} + + +class PackageImporter(Importer): + """Importers allow you to load code written to packages by :class:`PackageExporter`. + Code is loaded in a hermetic way, using files from the package + rather than the normal python import system. This allows + for the packaging of PyTorch model code and data so that it can be run + on a server or used in the future for transfer learning. + + The importer for packages ensures that code in the module can only be loaded from + within the package, except for modules explicitly listed as external during export. + The file ``extern_modules`` in the zip archive lists all the modules that a package externally depends on. + This prevents "implicit" dependencies where the package runs locally because it is importing + a locally-installed package, but then fails when the package is copied to another machine. + """ + + """The dictionary of already loaded modules from this package, equivalent to ``sys.modules`` but + local to this importer. + """ + + modules: dict[str, types.ModuleType] + + def __init__( + self, + file_or_buffer: FileLike | torch._C.PyTorchFileReader, + module_allowed: Callable[[str], bool] = lambda module_name: True, + ): + """Open ``file_or_buffer`` for importing. This checks that the imported package only requires modules + allowed by ``module_allowed`` + + Args: + file_or_buffer: a file-like object (has to implement :meth:`read`, :meth:`readline`, :meth:`tell`, and :meth:`seek`), + a string, or an ``os.PathLike`` object containing a filename. + module_allowed (Callable[[str], bool], optional): A method to determine if a externally provided module + should be allowed. Can be used to ensure packages loaded do not depend on modules that the server + does not support. Defaults to allowing anything. + + Raises: + ImportError: If the package will use a disallowed module. + """ + torch._C._log_api_usage_once("torch.package.PackageImporter") + + self.zip_reader: Any + if isinstance(file_or_buffer, torch._C.PyTorchFileReader): + self.filename = "" + self.zip_reader = file_or_buffer + elif isinstance(file_or_buffer, (os.PathLike, str)): + self.filename = os.fspath(file_or_buffer) + if not os.path.isdir(self.filename): + self.zip_reader = torch._C.PyTorchFileReader(self.filename) + else: + self.zip_reader = DirectoryReader(self.filename) + else: + self.filename = "" + self.zip_reader = torch._C.PyTorchFileReader(file_or_buffer) + + torch._C._log_api_usage_metadata( + "torch.package.PackageImporter.metadata", + { + "serialization_id": self.zip_reader.serialization_id(), + "file_name": self.filename, + }, + ) + + self.root = _PackageNode(None) + self.modules = {} + self.extern_modules = self._read_extern() + + for extern_module in self.extern_modules: + if not module_allowed(extern_module): + raise ImportError( + f"package '{file_or_buffer}' needs the external module '{extern_module}' " + f"but that module has been disallowed" + ) + self._add_extern(extern_module) + + for fname in self.zip_reader.get_all_records(): + self._add_file(fname) + + self.patched_builtins = builtins.__dict__.copy() + self.patched_builtins["__import__"] = self.__import__ + # Allow packaged modules to reference their PackageImporter + self.modules["torch_package_importer"] = self # type: ignore[assignment] + + self._mangler = PackageMangler() + + # used for reduce deserializaiton + self.storage_context: Any = None + self.last_map_location = None + + # used for torch.serialization._load + self.Unpickler = lambda *args, **kwargs: PackageUnpickler(self, *args, **kwargs) + + def import_module(self, name: str, package=None): + """Load a module from the package if it hasn't already been loaded, and then return + the module. Modules are loaded locally + to the importer and will appear in ``self.modules`` rather than ``sys.modules``. + + Args: + name (str): Fully qualified name of the module to load. + package ([type], optional): Unused, but present to match the signature of importlib.import_module. Defaults to ``None``. + + Returns: + types.ModuleType: The (possibly already) loaded module. + """ + # We should always be able to support importing modules from this package. + # This is to support something like: + # obj = importer.load_pickle(...) + # importer.import_module(obj.__module__) <- this string will be mangled + # + # Note that _mangler.demangle will not demangle any module names + # produced by a different PackageImporter instance. + name = self._mangler.demangle(name) + + return self._gcd_import(name) + + def load_binary(self, package: str, resource: str) -> bytes: + """Load raw bytes. + + Args: + package (str): The name of module package (e.g. ``"my_package.my_subpackage"``). + resource (str): The unique name for the resource. + + Returns: + bytes: The loaded data. + """ + + path = self._zipfile_path(package, resource) + return self.zip_reader.get_record(path) + + def load_text( + self, + package: str, + resource: str, + encoding: str = "utf-8", + errors: str = "strict", + ) -> str: + """Load a string. + + Args: + package (str): The name of module package (e.g. ``"my_package.my_subpackage"``). + resource (str): The unique name for the resource. + encoding (str, optional): Passed to ``decode``. Defaults to ``'utf-8'``. + errors (str, optional): Passed to ``decode``. Defaults to ``'strict'``. + + Returns: + str: The loaded text. + """ + data = self.load_binary(package, resource) + return data.decode(encoding, errors) + + def load_pickle(self, package: str, resource: str, map_location=None) -> Any: + """Unpickles the resource from the package, loading any modules that are needed to construct the objects + using :meth:`import_module`. + + Args: + package (str): The name of module package (e.g. ``"my_package.my_subpackage"``). + resource (str): The unique name for the resource. + map_location: Passed to `torch.load` to determine how tensors are mapped to devices. Defaults to ``None``. + + Returns: + Any: The unpickled object. + """ + pickle_file = self._zipfile_path(package, resource) + restore_location = _get_restore_location(map_location) + loaded_storages = {} + loaded_reduces = {} + storage_context = torch._C.DeserializationStorageContext() + + def load_tensor(dtype, size, key, location, restore_location): + name = f"{key}.storage" + + if storage_context.has_storage(name): + storage = storage_context.get_storage(name, dtype)._typed_storage() + else: + tensor = self.zip_reader.get_storage_from_record( + ".data/" + name, size, dtype + ) + if isinstance(self.zip_reader, torch._C.PyTorchFileReader): + storage_context.add_storage(name, tensor) + storage = tensor._typed_storage() + loaded_storages[key] = restore_location(storage, location) + + def persistent_load(saved_id): + assert isinstance(saved_id, tuple) + typename = _maybe_decode_ascii(saved_id[0]) + data = saved_id[1:] + + if typename == "storage": + storage_type, key, location, size = data + if storage_type is torch.UntypedStorage: + dtype = torch.uint8 + else: + dtype = storage_type.dtype + + if key not in loaded_storages: + load_tensor( + dtype, + size, + key, + _maybe_decode_ascii(location), + restore_location, + ) + storage = loaded_storages[key] + # TODO: Once we decide to break serialization FC, we can + # stop wrapping with TypedStorage + return torch.storage.TypedStorage( + wrap_storage=storage._untyped_storage, dtype=dtype, _internal=True + ) + elif typename == "reduce_package": + # to fix BC breaking change, objects on this load path + # will be loaded multiple times erroneously + if len(data) == 2: + func, args = data + return func(self, *args) + reduce_id, func, args = data + if reduce_id not in loaded_reduces: + loaded_reduces[reduce_id] = func(self, *args) + return loaded_reduces[reduce_id] + else: + f"Unknown typename for persistent_load, expected 'storage' or 'reduce_package' but got '{typename}'" + + # Load the data (which may in turn use `persistent_load` to load tensors) + data_file = io.BytesIO(self.zip_reader.get_record(pickle_file)) + unpickler = self.Unpickler(data_file) + unpickler.persistent_load = persistent_load # type: ignore[assignment] + + @contextmanager + def set_deserialization_context(): + # to let reduce_package access deserializaiton context + self.storage_context = storage_context + self.last_map_location = map_location + try: + yield + finally: + self.storage_context = None + self.last_map_location = None + + with set_deserialization_context(): + result = unpickler.load() + + # TODO from zdevito: + # This stateful weird function will need to be removed in our efforts + # to unify the format. It has a race condition if multiple python + # threads try to read independent files + torch._utils._validate_loaded_sparse_tensors() + + return result + + def id(self): + """ + Returns internal identifier that torch.package uses to distinguish :class:`PackageImporter` instances. + Looks like:: + + + """ + return self._mangler.parent_name() + + def file_structure( + self, *, include: "GlobPattern" = "**", exclude: "GlobPattern" = () + ) -> Directory: + """Returns a file structure representation of package's zipfile. + + Args: + include (Union[List[str], str]): An optional string e.g. ``"my_package.my_subpackage"``, or optional list of strings + for the names of the files to be included in the zipfile representation. This can also be + a glob-style pattern, as described in :meth:`PackageExporter.mock` + + exclude (Union[List[str], str]): An optional pattern that excludes files whose name match the pattern. + + Returns: + :class:`Directory` + """ + return _create_directory_from_file_list( + self.filename, self.zip_reader.get_all_records(), include, exclude + ) + + def python_version(self): + """Returns the version of python that was used to create this package. + + Note: this function is experimental and not Forward Compatible. The plan is to move this into a lock + file later on. + + Returns: + :class:`Optional[str]` a python version e.g. 3.8.9 or None if no version was stored with this package + """ + python_version_path = ".data/python_version" + return ( + self.zip_reader.get_record(python_version_path).decode("utf-8").strip() + if self.zip_reader.has_record(python_version_path) + else None + ) + + def _read_extern(self): + return ( + self.zip_reader.get_record(".data/extern_modules") + .decode("utf-8") + .splitlines(keepends=False) + ) + + def _make_module( + self, name: str, filename: str | None, is_package: bool, parent: str + ): + mangled_filename = self._mangler.mangle(filename) if filename else None + spec = importlib.machinery.ModuleSpec( + name, + self, # type: ignore[arg-type] + origin="", + is_package=is_package, + ) + module = importlib.util.module_from_spec(spec) + self.modules[name] = module + module.__name__ = self._mangler.mangle(name) + ns = module.__dict__ + ns["__spec__"] = spec + ns["__loader__"] = self + ns["__file__"] = mangled_filename + ns["__cached__"] = None + ns["__builtins__"] = self.patched_builtins + ns["__torch_package__"] = True + + # Add this module to our private global registry. It should be unique due to mangling. + assert module.__name__ not in _package_imported_modules + _package_imported_modules[module.__name__] = module + + # preemptively install on the parent to prevent IMPORT_FROM from trying to + # access sys.modules + self._install_on_parent(parent, name, module) + + if filename is not None: + assert mangled_filename is not None + # preemptively install the source in `linecache` so that stack traces, + # `inspect`, etc. work. + assert filename not in linecache.cache # type: ignore[attr-defined] + linecache.lazycache(mangled_filename, ns) + + code = self._compile_source(filename, mangled_filename) + exec(code, ns) + + return module + + def _load_module(self, name: str, parent: str): + cur: _PathNode = self.root + for atom in name.split("."): + if not isinstance(cur, _PackageNode) or atom not in cur.children: + if name in IMPLICIT_IMPORT_ALLOWLIST: + module = self.modules[name] = importlib.import_module(name) + return module + raise ModuleNotFoundError( + f'No module named "{name}" in self-contained archive "{self.filename}"' + f" and the module is also not in the list of allowed external modules: {self.extern_modules}", + name=name, + ) + cur = cur.children[atom] + if isinstance(cur, _ExternNode): + module = self.modules[name] = importlib.import_module(name) + + if compat_mapping := EXTERN_IMPORT_COMPAT_NAME_MAPPING.get(name): + for old_name, new_name in compat_mapping.items(): + module.__dict__.setdefault(old_name, new_name) + + return module + return self._make_module( + name, + cur.source_file, # type: ignore[attr-defined] + isinstance(cur, _PackageNode), + parent, + ) + + def _compile_source(self, fullpath: str, mangled_filename: str): + source = self.zip_reader.get_record(fullpath) + source = _normalize_line_endings(source) + return compile(source, mangled_filename, "exec", dont_inherit=True) + + # note: named `get_source` so that linecache can find the source + # when this is the __loader__ of a module. + def get_source(self, module_name) -> str: + # linecache calls `get_source` with the `module.__name__` as the argument, so we must demangle it here. + module = self.import_module(demangle(module_name)) + return self.zip_reader.get_record(demangle(module.__file__)).decode("utf-8") + + # note: named `get_resource_reader` so that importlib.resources can find it. + # This is otherwise considered an internal method. + def get_resource_reader(self, fullname): + try: + package = self._get_package(fullname) + except ImportError: + return None + if package.__loader__ is not self: + return None + return _PackageResourceReader(self, fullname) + + def _install_on_parent(self, parent: str, name: str, module: types.ModuleType): + if not parent: + return + # Set the module as an attribute on its parent. + parent_module = self.modules[parent] + if parent_module.__loader__ is self: + setattr(parent_module, name.rpartition(".")[2], module) + + # note: copied from cpython's import code, with call to create module replaced with _make_module + def _do_find_and_load(self, name): + parent = name.rpartition(".")[0] + module_name_no_parent = name.rpartition(".")[-1] + if parent: + if parent not in self.modules: + self._gcd_import(parent) + # Crazy side-effects! + if name in self.modules: + return self.modules[name] + parent_module = self.modules[parent] + + try: + parent_module.__path__ # type: ignore[attr-defined] + + except AttributeError: + # when we attempt to import a package only containing pybinded files, + # the parent directory isn't always a package as defined by python, + # so we search if the package is actually there or not before calling the error. + if isinstance( + parent_module.__loader__, + importlib.machinery.ExtensionFileLoader, + ): + if name not in self.extern_modules: + msg = ( + _ERR_MSG + + "; {!r} is a c extension module which was not externed. C extension modules \ + need to be externed by the PackageExporter in order to be used as we do not support interning them.}." + ).format(name, name) + raise ModuleNotFoundError(msg, name=name) from None + if not isinstance( + parent_module.__dict__.get(module_name_no_parent), + types.ModuleType, + ): + msg = ( + _ERR_MSG + + "; {!r} is a c extension package which does not contain {!r}." + ).format(name, parent, name) + raise ModuleNotFoundError(msg, name=name) from None + else: + msg = (_ERR_MSG + "; {!r} is not a package").format(name, parent) + raise ModuleNotFoundError(msg, name=name) from None + + module = self._load_module(name, parent) + + self._install_on_parent(parent, name, module) + + return module + + # note: copied from cpython's import code + def _find_and_load(self, name): + module = self.modules.get(name, _NEEDS_LOADING) + if module is _NEEDS_LOADING: + return self._do_find_and_load(name) + + if module is None: + message = f"import of {name} halted; None in sys.modules" + raise ModuleNotFoundError(message, name=name) + + # To handle https://github.com/pytorch/pytorch/issues/57490, where std's + # creation of fake submodules via the hacking of sys.modules is not import + # friendly + if name == "os": + self.modules["os.path"] = cast(Any, module).path + elif name == "typing": + if sys.version_info < (3, 13): + self.modules["typing.io"] = cast(Any, module).io + self.modules["typing.re"] = cast(Any, module).re + + return module + + def _gcd_import(self, name, package=None, level=0): + """Import and return the module based on its name, the package the call is + being made from, and the level adjustment. + + This function represents the greatest common denominator of functionality + between import_module and __import__. This includes setting __package__ if + the loader did not. + + """ + _sanity_check(name, package, level) + if level > 0: + name = _resolve_name(name, package, level) + + return self._find_and_load(name) + + # note: copied from cpython's import code + def _handle_fromlist(self, module, fromlist, *, recursive=False): + """Figure out what __import__ should return. + + The import_ parameter is a callable which takes the name of module to + import. It is required to decouple the function from assuming importlib's + import implementation is desired. + + """ + module_name = demangle(module.__name__) + # The hell that is fromlist ... + # If a package was imported, try to import stuff from fromlist. + if hasattr(module, "__path__"): + for x in fromlist: + if not isinstance(x, str): + if recursive: + where = module_name + ".__all__" + else: + where = "``from list''" + raise TypeError( + f"Item in {where} must be str, not {type(x).__name__}" + ) + elif x == "*": + if not recursive and hasattr(module, "__all__"): + self._handle_fromlist(module, module.__all__, recursive=True) + elif not hasattr(module, x): + from_name = f"{module_name}.{x}" + try: + self._gcd_import(from_name) + except ModuleNotFoundError as exc: + # Backwards-compatibility dictates we ignore failed + # imports triggered by fromlist for modules that don't + # exist. + if ( + exc.name == from_name + and self.modules.get(from_name, _NEEDS_LOADING) is not None + ): + continue + raise + return module + + def __import__(self, name, globals=None, locals=None, fromlist=(), level=0): + if level == 0: + module = self._gcd_import(name) + else: + globals_ = globals if globals is not None else {} + package = _calc___package__(globals_) + module = self._gcd_import(name, package, level) + if not fromlist: + # Return up to the first dot in 'name'. This is complicated by the fact + # that 'name' may be relative. + if level == 0: + return self._gcd_import(name.partition(".")[0]) + elif not name: + return module + else: + # Figure out where to slice the module's name up to the first dot + # in 'name'. + cut_off = len(name) - len(name.partition(".")[0]) + # Slice end needs to be positive to alleviate need to special-case + # when ``'.' not in name``. + module_name = demangle(module.__name__) + return self.modules[module_name[: len(module_name) - cut_off]] + else: + return self._handle_fromlist(module, fromlist) + + def _get_package(self, package): + """Take a package name or module object and return the module. + + If a name, the module is imported. If the passed or imported module + object is not a package, raise an exception. + """ + if hasattr(package, "__spec__"): + if package.__spec__.submodule_search_locations is None: + raise TypeError(f"{package.__spec__.name!r} is not a package") + else: + return package + else: + module = self.import_module(package) + if module.__spec__.submodule_search_locations is None: + raise TypeError(f"{package!r} is not a package") + else: + return module + + def _zipfile_path(self, package, resource=None): + package = self._get_package(package) + assert package.__loader__ is self + name = demangle(package.__name__) + if resource is not None: + resource = _normalize_path(resource) + return f"{name.replace('.', '/')}/{resource}" + else: + return f"{name.replace('.', '/')}" + + def _get_or_create_package(self, atoms: list[str]) -> "_PackageNode | _ExternNode": + cur = self.root + for i, atom in enumerate(atoms): + node = cur.children.get(atom, None) + if node is None: + node = cur.children[atom] = _PackageNode(None) + if isinstance(node, _ExternNode): + return node + if isinstance(node, _ModuleNode): + name = ".".join(atoms[:i]) + raise ImportError( + f"inconsistent module structure. module {name} is not a package, but has submodules" + ) + assert isinstance(node, _PackageNode) + cur = node + return cur + + def _add_file(self, filename: str): + """Assembles a Python module out of the given file. Will ignore files in the .data directory. + + Args: + filename (str): the name of the file inside of the package archive to be added + """ + *prefix, last = filename.split("/") + if len(prefix) > 1 and prefix[0] == ".data": + return + package = self._get_or_create_package(prefix) + if isinstance(package, _ExternNode): + raise ImportError( + f"inconsistent module structure. package contains a module file {filename}" + f" that is a subpackage of a module marked external." + ) + if last == "__init__.py": + package.source_file = filename + elif last.endswith(".py"): + package_name = last[: -len(".py")] + package.children[package_name] = _ModuleNode(filename) + + def _add_extern(self, extern_name: str): + *prefix, last = extern_name.split(".") + package = self._get_or_create_package(prefix) + if isinstance(package, _ExternNode): + return # the shorter extern covers this extern case + package.children[last] = _ExternNode() + + +_NEEDS_LOADING = object() +_ERR_MSG_PREFIX = "No module named " +_ERR_MSG = _ERR_MSG_PREFIX + "{!r}" + + +class _PathNode: + __slots__ = [] + + +class _PackageNode(_PathNode): + def __init__(self, source_file: str | None): + self.source_file = source_file + self.children: dict[str, _PathNode] = {} + + +class _ModuleNode(_PathNode): + __slots__ = ["source_file"] + + def __init__(self, source_file: str): + self.source_file = source_file + + +class _ExternNode(_PathNode): + pass + + +# A private global registry of all modules that have been package-imported. +_package_imported_modules: WeakValueDictionary = WeakValueDictionary() + +# `inspect` by default only looks in `sys.modules` to find source files for classes. +# Patch it to check our private registry of package-imported modules as well. +_orig_getfile = inspect.getfile + + +def _patched_getfile(object): + if inspect.isclass(object): + if object.__module__ in _package_imported_modules: + return _package_imported_modules[object.__module__].__file__ + return _orig_getfile(object) + + +inspect.getfile = _patched_getfile + + +class _PackageResourceReader: + """Private class used to support PackageImporter.get_resource_reader(). + + Confirms to the importlib.abc.ResourceReader interface. Allowed to access + the innards of PackageImporter. + """ + + def __init__(self, importer, fullname): + self.importer = importer + self.fullname = fullname + + def open_resource(self, resource): + from io import BytesIO + + return BytesIO(self.importer.load_binary(self.fullname, resource)) + + def resource_path(self, resource): + # The contract for resource_path is that it either returns a concrete + # file system path or raises FileNotFoundError. + if isinstance( + self.importer.zip_reader, DirectoryReader + ) and self.importer.zip_reader.has_record( + os.path.join(self.fullname, resource) + ): + return os.path.join( + self.importer.zip_reader.directory, self.fullname, resource + ) + raise FileNotFoundError + + def is_resource(self, name): + path = self.importer._zipfile_path(self.fullname, name) + return self.importer.zip_reader.has_record(path) + + def contents(self): + from pathlib import Path + + filename = self.fullname.replace(".", "/") + + fullname_path = Path(self.importer._zipfile_path(self.fullname)) + files = self.importer.zip_reader.get_all_records() + subdirs_seen = set() + for filename in files: + try: + relative = Path(filename).relative_to(fullname_path) + except ValueError: + continue + # If the path of the file (which is relative to the top of the zip + # namespace), relative to the package given when the resource + # reader was created, has a parent, then it's a name in a + # subdirectory and thus we skip it. + parent_name = relative.parent.name + if len(parent_name) == 0: + yield relative.name + elif parent_name not in subdirs_seen: + subdirs_seen.add(parent_name) + yield parent_name diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..153d4560e264198ddb8f04649aef22456afbb452 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/__init__.py @@ -0,0 +1,60 @@ +r""" +PyTorch Profiler is a tool that allows the collection of performance metrics during training and inference. +Profiler's context manager API can be used to better understand what model operators are the most expensive, +examine their input shapes and stack traces, study device kernel activity and visualize the execution trace. + +.. note:: + An earlier version of the API in :mod:`torch.autograd` module is considered legacy and will be deprecated. + +""" + +import os +from typing import Any +from typing_extensions import TypeVarTuple, Unpack + +from torch._C._autograd import _supported_activities, DeviceType, kineto_available +from torch._C._profiler import _ExperimentalConfig, ProfilerActivity, RecordScope +from torch._environment import is_fbcode +from torch.autograd.profiler import KinetoStepTracker, record_function +from torch.optim.optimizer import Optimizer, register_optimizer_step_post_hook + +from .profiler import ( + _KinetoProfile, + ExecutionTraceObserver, + profile, + ProfilerAction, + schedule, + supported_activities, + tensorboard_trace_handler, +) + + +__all__ = [ + "profile", + "schedule", + "supported_activities", + "tensorboard_trace_handler", + "ProfilerAction", + "ProfilerActivity", + "kineto_available", + "DeviceType", + "record_function", + "ExecutionTraceObserver", +] + +from . import itt + + +_Ts = TypeVarTuple("_Ts") + + +def _optimizer_post_hook( + optimizer: Optimizer, args: tuple[Unpack[_Ts]], kwargs: dict[str, Any] +) -> None: + KinetoStepTracker.increment_step("Optimizer") + + +if os.environ.get("KINETO_USE_DAEMON", "") or ( + is_fbcode() and os.environ.get("KINETO_FORCE_OPTIMIZER_HOOK", "") +): + _ = register_optimizer_step_post_hook(_optimizer_post_hook) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/_memory_profiler.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/_memory_profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..0064b9fbd09277662e1bcb918a046b979e20a288 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/_memory_profiler.py @@ -0,0 +1,1206 @@ +# mypy: allow-untyped-defs +import collections +import dataclasses +import enum +import itertools as it +import logging +from collections.abc import Iterator +from typing import Any, cast, Literal, Optional + +import torch +from torch._C import FunctionSchema +from torch._C._autograd import _ProfilerResult +from torch._C._profiler import ( + _EventType, + _ExtraFields_Allocation, + _ExtraFields_TorchOp, + _ProfilerEvent, + _TensorMetadata, + RecordScope, +) +from torch._utils import _element_size +from torch.profiler import _utils + + +KeyAndID = tuple["Key", int] +TensorAndID = tuple["TensorKey", int] + +log = logging.getLogger(__name__) + + +class Category(enum.Enum): + INPUT = enum.auto() + TEMPORARY = enum.auto() + ACTIVATION = enum.auto() + GRADIENT = enum.auto() + AUTOGRAD_DETAIL = enum.auto() + PARAMETER = enum.auto() + OPTIMIZER_STATE = enum.auto() + + +_CATEGORY_TO_COLORS = { + Category.PARAMETER: "darkgreen", + Category.OPTIMIZER_STATE: "goldenrod", + Category.INPUT: "black", + Category.TEMPORARY: "mediumpurple", + Category.ACTIVATION: "red", + Category.GRADIENT: "mediumblue", + Category.AUTOGRAD_DETAIL: "royalblue", + None: "grey", +} + +_CATEGORY_TO_INDEX = {c: i for i, c in enumerate(_CATEGORY_TO_COLORS)} + + +class Action(enum.Enum): + PREEXISTING = enum.auto() + CREATE = enum.auto() + INCREMENT_VERSION = enum.auto() + DESTROY = enum.auto() + + +_ACTION_TO_INDEX = {i: i.value for i in Action} + + +@dataclasses.dataclass(eq=True, unsafe_hash=False, frozen=True) +class Key: + device: torch.device + + +@dataclasses.dataclass +class _Storage: + """Bundle storage pointer and id. + + All profiling logic should use `allocation_id`, however it is useful to + print storage pointers for debugging and unit tests sometimes look up + values using the storage data pointer of a live Tensor.""" + + ptr: int + allocation_id: int + + def __repr__(self) -> str: + return f"{hex(self.ptr):>18} ({self.allocation_id})" + + def __eq__(self, other: object) -> bool: + return isinstance(other, _Storage) and self.allocation_id == other.allocation_id + + def __hash__(self) -> int: + return hash(self.allocation_id) + + +@dataclasses.dataclass(eq=True, unsafe_hash=True, frozen=True) +class TensorKey(Key): + """Hashable identifier for a storage which has been assigned an ID. + + A detailed description of Tensor IDs and why they are needed is given in + `torch/csrc/profiler/collection.h` when `TensorID` is declared. To + summarize, multiple Storage buffers can map to the same logical Tensor. + This dataclass is used to refer to a concrete in-memory StorageImpl of + a Tensor. + """ + + id: int + storage: _Storage + + def __repr__(self) -> str: + return f"id={self.id}: {repr(self.storage):<24} ({self.device})" + + def __lt__(self, other: "TensorKey") -> bool: + return self._as_sortable < other._as_sortable + + @staticmethod + def _make( + tensor_id: int | None, + storage_ptr: int | None, + allocation_id: int | None, + device: torch.device, + ) -> Optional["TensorKey"]: + if ( + tensor_id is not None + and storage_ptr is not None + and allocation_id is not None + ): + return TensorKey(device, tensor_id, _Storage(storage_ptr, allocation_id)) + return None + + @classmethod + def from_allocation(cls, alloc: _ExtraFields_Allocation) -> Optional["TensorKey"]: + return cls._make(alloc.id, alloc.ptr, alloc.allocation_id, alloc.device) + + @classmethod + def from_tensor(cls, t: _TensorMetadata | None) -> Optional["TensorKey"]: + if t is not None: + return cls._make(t.id, t.storage_data_ptr, t.allocation_id, t.device) + return None + + @property + def _as_sortable(self) -> tuple[int, int, str, int]: + return self.id, self.storage.allocation_id, self.device.type, self.device.index + + +def _extract_parameters_and_gradients( + node: _ProfilerEvent, +) -> Iterator[tuple[TensorKey | None, TensorKey | None]]: + children = node.children + + # AccumulateGrad is used in the Autograd engine to handle gradient updates. + # There are two possible cases: + # 1) This is a newly created gradient Tensor. In that case there is nothing + # to accumulate, so autograd simply detaches the Tensor. + # + # 2) There is a preexisting gradient Tensor and we need to add the newly + # computed update. This is done with an in-place add (aten::add_) op. + # (The underscore suffix denotes "in-place".) + if ( + node.typed[0] == _EventType.TorchOp + and node.typed[1].scope == RecordScope.BACKWARD_FUNCTION + # TODO(robieta): Move away from load bearing names + and node.name == "torch::autograd::AccumulateGrad" + and children + and children[0].typed[0] == _EventType.TorchOp + and children[0].name in ("aten::detach", "aten::add_") + and children[0].typed[1].inputs + and isinstance(children[0].typed[1].inputs[0], _TensorMetadata) + ): + yield None, TensorKey.from_tensor(children[0].typed[1].inputs[0]) + + # We directly instrument `torch.nn.Module` and `torch.optim.Optimizer` + # NOTE: The values captured by the python tracer are cached; they can be + # used to build up labels but do not imply that a Tensor was live at + # a particular time. + elif node.typed[0] == _EventType.PyCall: + typed_fields = node.typed[1] + assert typed_fields.module is None or typed_fields.optimizer is None + if typed_fields.module is not None: + for _, p, p_grad in typed_fields.module.parameters: + yield TensorKey.from_tensor(p), TensorKey.from_tensor(p_grad) + + if typed_fields.optimizer is not None: + for p, p_grad, _ in typed_fields.optimizer.parameters: + yield TensorKey.from_tensor(p), TensorKey.from_tensor(p_grad) + + +def extract_parameters(node: _ProfilerEvent) -> Iterator[TensorKey]: + for p, _p_grad in _extract_parameters_and_gradients(node): + if p is not None: + yield p + + +def extract_gradients( + node: _ProfilerEvent, +) -> Iterator[tuple[TensorKey | None, TensorKey]]: + for p, p_grad in _extract_parameters_and_gradients(node): + if p_grad is not None: + yield p, p_grad + + +def get_scopes(event: _ProfilerEvent | None) -> tuple[RecordScope, ...]: + scopes = [] + while event: + if event.typed[0] == _EventType.TorchOp: + scopes.append(event.typed[1].scope) + event = event.parent + return tuple(scopes) + + +class SchemaMatcher: + """Lookup operator schema based on profiled name. + + When profiling we record the operator's name but not the schema. However + some analysis requires that information. Fortunately we can look up + registered schema from the recorded name. We do not, however, record the + overload and so we must compare the profiled arguments with all overloads + to determine viable matches. + + Note: Once https://github.com/pytorch/pytorch/issues/78871 is completed + this code will be obsolete. + """ + + @classmethod + def inputs_are_mutable(cls, t: _ExtraFields_TorchOp) -> tuple[bool | None, ...]: + """Determine which inputs may have mutated based on function schema. + + Note that we don't need to resolve down to a single schema to perform + this analysis. An input is mutable if it is mutable in any overload. In + practice, however, it is overwhelmingly common to match a single + overload. If we cannot find any valid schema then we must be + conservative and assume all inputs are mutable. + """ + mutable: list[bool] | None = None + for schema in cls.match_schemas(t): + mutable = mutable or [False for _ in schema.arguments] + for i, arg in enumerate(schema.arguments): + # pyrefly: ignore [unsupported-operation] + mutable[i] |= getattr(arg.alias_info, "is_write", False) + + return tuple(mutable or (None for _ in t.inputs)) + + @classmethod + def match_schemas(cls, t: _ExtraFields_TorchOp) -> tuple[FunctionSchema, ...]: + signature = tuple( + # Tensor + TensorKey.from_tensor(i) + if isinstance(i, _TensorMetadata) + # + # TensorList + else [TensorKey.from_tensor(j) for j in i] + if isinstance(i, list) + # + # Scalar and uncaptured inputs. + else i + for i in t.inputs + ) + + def matches(schema) -> bool: + return len(schema.arguments) == len(signature) and all( + cls._types_match(observed, schema_arg.type) + for observed, schema_arg in zip( + signature, schema.arguments, strict=True + ) + ) + + return tuple(s for s in cls.lookup_schemas(t.name) or () if matches(s)) + + @classmethod + def _types_match(cls, observed, schema_type) -> bool: + if isinstance(schema_type, torch._C.OptionalType): + schema_type = schema_type.getElementType() + return observed is None or cls._types_match(observed, schema_type) + + if isinstance(schema_type, torch._C.AnyType): + return True + + if schema_type.isSubtypeOf(torch._C.ListType.ofTensors()): + return isinstance(observed, list) and all( + isinstance(i, TensorKey) for i in observed + ) + + type_map: tuple[tuple[Any, type | tuple[type, ...]], ...] = ( + (torch._C.TensorType, TensorKey), + (torch._C.NoneType, type(None)), + (torch._C.BoolType, bool), + (torch._C.IntType, int), + (torch._C.FloatType, float), + (torch._C.ComplexType, complex), + (torch._C.NumberType, (bool, int, float, complex)), + ) + + for jit_type, py_types in type_map: + if isinstance(schema_type, jit_type): + return isinstance(observed, py_types) + + # Profiler only records a subset of possible argument types. If we + # reach this point then the schema must call for a type that profiler + # does not record. Thus, the schema can only be a match if `observed` + # is also None. + return observed is None + + @staticmethod + def lookup_schemas(name: str) -> tuple[FunctionSchema, ...] | None: + # TODO(robieta): + # _jit_get_schemas_for_operator is quite expensive. (~100us / call) + # Consider adding `functools.lru_cache` if that becomes an issue. + + try: + # Schema lookup will throw if `name` is malformed. (For example, + # schemas must be namespaced and schema lookup will fail if name + # does not include "::".) We simply catch the exception and return + # `None` to denote that `name` cannot be an operator name. + # + # Note that record_function annotations also go through this path, + # so it is expected that some names will not correspond to PyTorch + # operators. + if "::" not in name: + return None + return tuple(torch._C._jit_get_schemas_for_operator(name)) + except RuntimeError: + return None + + +class OpTree: + def __init__(self, result: _ProfilerResult) -> None: + self._root_nodes = result.experimental_event_tree() + self._sorted_nodes = tuple(sorted(self.dfs(), key=lambda x: x.start_time_ns)) + + def dfs(self, *args, **kwargs) -> Iterator[_ProfilerEvent]: + yield from _utils.traverse_dfs(self._root_nodes, *args, **kwargs) + + @property + def sorted_nodes(self) -> tuple[_ProfilerEvent, ...]: + return self._sorted_nodes + + +class SizeMap: + def __init__(self, op_tree: OpTree) -> None: + self._values: dict[TensorKey, int] = {} + + for node in op_tree.sorted_nodes: + if node.typed[0] == _EventType.TorchOp: + for t in self._flat_tensor_inputs(node.typed[1]): + self._update_values(t) + + elif node.typed[0] == _EventType.PyCall: + typed_fields = node.typed[1] + assert typed_fields.module is None or typed_fields.optimizer is None + if typed_fields.module is not None: + for _, p, p_grad in typed_fields.module.parameters: + self._update_values(p) + self._update_values(p_grad) + + if typed_fields.optimizer is not None: + for p, p_grad, state in typed_fields.optimizer.parameters: + self._update_values(p) + self._update_values(p_grad) + for _, t in state: + self._update_values(t) + + allocations: dict[TensorKey, int] = {} + for node in op_tree.sorted_nodes: + if node.typed[0] == _EventType.Allocation: + alloc_fields = node.typed[1] + key = TensorKey.from_allocation(alloc_fields) + if key: + new_size = abs(alloc_fields.alloc_size) + prior_size = allocations.setdefault(key, new_size) + + # It is possible to resize Storage in PyTorch, however we + # key on data pointer so most resizes will be treated as a + # change in storage. The one corner case that cannot be + # handled is `realloc` which successfully resizes the + # storage. At time of writing this is not done anywhere in + # the core PyTorch codebase. + if prior_size != new_size: + delta = f"{prior_size} vs. {new_size}" + log.warning("Mismatch between allocation and free: %s", delta) + + self._values.update(allocations) + + def _update_values(self, t: _TensorMetadata | None) -> None: + key = TensorKey.from_tensor(t) + if key is not None and t is not None and t.layout == torch.strided: + # Scalars are represented as zero dim Tensors + n = max( + i[0] * i[1] for i in zip(t.sizes or [1], t.strides or [1], strict=True) + ) + + num_bytes = n * _element_size(t.dtype) + assert num_bytes >= 0, f"{num_bytes}" + self._values[key] = max(self._values.get(key, 0), num_bytes) + + @staticmethod + def _flat_tensor_inputs(op: _ExtraFields_TorchOp) -> Iterator[_TensorMetadata]: + for i in op.inputs: + if isinstance(i, _TensorMetadata): + yield i + elif isinstance(i, list): + yield from i + + def __getitem__(self, key: TensorKey): + return self._values[key] + + +@dataclasses.dataclass() +class DataFlowEdge: + input_version: int | None = None + mutated: bool | None = False + + @property + def is_allocation(self) -> bool: + return self.input_version is None + + @property + def is_deletion(self) -> bool: + return self.mutated is None + + +class DataFlowNode: + def __init__(self, event: _ProfilerEvent, graph: "DataFlowGraph") -> None: + self._event = event + self._graph = graph + self._edges: dict[TensorKey, DataFlowEdge] = self._determine_edges() + + for key, edge in self._edges.items(): + if edge.mutated and not edge.is_allocation: + self._graph.bump(key) + + # Make sure the version bumping behavior matches what we expect. + versions = {k: (v, self._graph.lookup(k)) for k, v in self.outputs.items()} + assert all(i == j for i, j in versions.values()), f"{versions}, {self._edges}" + + def _determine_edges(self) -> dict[TensorKey, DataFlowEdge]: + subtree = tuple(_utils.traverse_dfs([self._event])) + + # Start by populating edges from op inputs and outputs. + mutable_by_key: dict[TensorKey | None, set[bool | None]] = {} + for op in (i.typed[1] for i in subtree if i.typed[0] == _EventType.TorchOp): + for op_input, mutable in zip( + op.inputs, SchemaMatcher.inputs_are_mutable(op), strict=True + ): + # Tensor + if isinstance(op_input, _TensorMetadata): + key = TensorKey.from_tensor(op_input) + mutable_by_key.setdefault(key, set()).add(mutable) + + # TensorList + elif isinstance(op_input, list): + for op_input_i in op_input: + key = TensorKey.from_tensor(op_input_i) + mutable_by_key.setdefault(key, set()).add(mutable) + + edges: collections.defaultdict[TensorKey | None, DataFlowEdge] + edges = collections.defaultdict(DataFlowEdge) + for key, mutable_set in mutable_by_key.items(): + if key is not None: + edges[key].input_version = self._graph.lookup(key) if key else -1 + + # We consider an op to be mutated if we encounter a schema where it + # is a mutable argument OR if it is ambiguous. (We never explicitly + # see it in any schema.) + mutated = (True in mutable_set) or (tuple(mutable_set) == (None,)) + edges[key].mutated = mutated + + # Then handle deletions. Note that deleting a Tensor implicitly adds + # it as an input edge. + for i in subtree: + if i.typed[0] == _EventType.Allocation and i.typed[1].alloc_size < 0: + key = TensorKey.from_allocation(i.typed[1]) + edge = edges[key] + assert key is None or edge.mutated is not None, f"Double delete: {key}" + edge.mutated = None + edge.input_version = self._graph.lookup(key) if key else -1 + + # And finally handle allocations. This step must be last, because the + # previous two steps optimistically add input edges. + for i in subtree: + if i.typed[0] == _EventType.Allocation and i.typed[1].alloc_size > 0: + edges[TensorKey.from_allocation(i.typed[1])].input_version = None + + # We don't need to sort the inputs, but it makes debugging and unit tests nicer. + return dict(sorted((k, v) for k, v in edges.items() if k is not None)) + + @property + def inputs(self) -> dict[TensorKey, tuple[bool, int]]: + return { + # MyPy can't see through `is_allocation` to know that + # `v.input_version` is not None. + k: (bool(v.mutated), cast(int, v.input_version)) + for k, v in self._edges.items() + if not v.is_allocation + } + + @property + def outputs(self) -> dict[TensorKey, int]: + return { + k: 0 if v.input_version is None else v.input_version + 1 + for k, v in self._edges.items() + if (v.is_allocation and not v.is_deletion) or v.mutated + } + + @property + def intermediates(self) -> tuple[TensorKey, ...]: + return tuple( + k for k, v in self._edges.items() if v.is_allocation and v.is_deletion + ) + + @property + def start_time(self) -> int: + return self._event.start_time_ns + + +class DataFlowGraph: + def __init__(self, op_tree: OpTree) -> None: + self._op_tree = op_tree + self._leaf_events = self._extract_leaf_events(op_tree) + self._active_version: dict[TensorKey, int | None] = {} + self._flow_nodes = [DataFlowNode(e, self) for e in self.leaf_events] + self._flow_nodes.sort(key=lambda x: x.start_time) + self.validate() + + @property + def flow_nodes(self) -> tuple[DataFlowNode, ...]: + return tuple(self._flow_nodes) + + def validate(self) -> None: + # Check that each (Tensor, version) pair has a unique creation node + outputs: set[tuple[TensorKey, int]] = set() + for node in self.flow_nodes: + node_outputs = set(node.outputs.items()) + duplicates = outputs & node_outputs + assert not duplicates, f"{node._event.name} {node._edges} {duplicates}" + outputs |= node_outputs + + # And check that `self._nodes` forms a valid topologically sorted DAG. + tensor_versions: dict[TensorKey, int] = {} + for node in self.flow_nodes: + for key, (_, version) in node.inputs.items(): + expected = tensor_versions.get(key, 0) + assert expected == version, (expected, version) + + for key, version in node.outputs.items(): + prior_version = tensor_versions.get(key, version) + assert version >= prior_version, (version, prior_version) + tensor_versions[key] = version + + @property + def leaf_events(self) -> tuple[_ProfilerEvent, ...]: + return self._leaf_events + + @staticmethod + def _extract_leaf_events(op_tree: OpTree) -> tuple[_ProfilerEvent, ...]: + """Partially traverse the op tree and extract top level ops. + + Consider the following code: + ``` + with record_function("My annotation"): + x.zero_() + y.zero_() + ``` + + The op tree (assuming no Autograd) will look like: + + TorchOp: "My annotation" + TorchOp: zero_ + TorchOp: fill_ + TorchOp: zero_ + TorchOp: fill_ + + The recursive structure of operator calls makes data flow unwieldy. + In order to simplify analysis we would like to select the highest level + ops to represent in the graph. In this case those are the `zero_` ops; + the fact that `fill_` is called is an implementation detail. We also + do not want to group everything under "My annotation" as this could + create overly coarse bundles and lose critical semantics. + + To address this issue we walk over the graph and select the topmost + torch ops ** which match at least one operator schema **. These form + the leaves of the first pass through the op tree. (As well as any + allocations or frees which do are not part of a kernel.) These events + form the logical nodes in our data flow graph. + """ + + leaf_events: list[_ProfilerEvent] = [] + + def leaf_op(e: _ProfilerEvent) -> bool: + return e.typed[0] == _EventType.TorchOp and ( + e.typed[1].scope == RecordScope.BACKWARD_FUNCTION + or bool(SchemaMatcher.match_schemas(e.typed[1])) + ) + + def children_fn(e: _ProfilerEvent): + if leaf_op(e) or e.tag == _EventType.Allocation: + leaf_events.append(e) + return [] + + return e.children + + for _ in op_tree.dfs(children_fn=children_fn): + pass + + return tuple(sorted(leaf_events, key=lambda x: x.start_time_ns)) + + def lookup(self, key: TensorKey) -> int: + version = self._active_version.setdefault(key, 0) + assert version is not None + return version + + def bump(self, key: TensorKey) -> None: + prior_version = self._active_version.get(key, None) + assert prior_version is not None + self._active_version[key] = prior_version + 1 + + def delete(self, key: TensorKey) -> None: + assert self._active_version.setdefault(key, 0) is not None + self._active_version[key] = None + + +@dataclasses.dataclass +class CategoryElement: + by_id: Category | None = None + by_key: dict[TensorKey, Category] = dataclasses.field(default_factory=dict) + by_version: dict[TensorAndID, Category] = dataclasses.field(default_factory=dict) + + # Used by unit tests to check internals. (And consequently by + # MemoryProfile.lookup) This should not be used in any other capacity. + _by_id_keyset: set[TensorKey] = dataclasses.field(default_factory=set) + + +@dataclasses.dataclass +class CategoryDict: + _values: collections.defaultdict[int, CategoryElement] = dataclasses.field( + default_factory=lambda: collections.defaultdict(CategoryElement) + ) + + def set_by_id(self, key: TensorKey, category: Category) -> None: + self._values[key.id].by_id = category + self._values[key.id]._by_id_keyset.add(key) + + def set_by_key(self, key: TensorKey, category: Category) -> None: + self._values[key.id].by_key[key] = category + + def set_by_version(self, key: TensorKey, version: int, category: Category) -> None: + self._values[key.id].by_version[(key, version)] = category + + def setdefault_by_version( + self, key: TensorKey, version: int, category: Category + ) -> None: + self._values[key.id].by_version.setdefault((key, version), category) + + def get(self, key: Key, version: int) -> Category | None: + if isinstance(key, Key) and not isinstance(key, TensorKey): + return None + element = self._values[key.id] + return ( + element.by_id + or element.by_key.get(key, None) + or element.by_version.get((key, version), None) + ) + + +class MemoryProfile: + def __init__(self, result: _ProfilerResult) -> None: + self._op_tree = OpTree(result) + self._data_flow_graph = DataFlowGraph(self._op_tree) + self._size_map = SizeMap(self._op_tree) + self._categories = CategoryDict() + + self._set_gradients_and_temporaries() + self._set_parameters_using_python_tracer() + self._set_inputs() + self._set_parameters_using_data_flow() + self._set_activations() + self._set_optimizer_state() + self._set_autograd_detail() + + @property + def timeline(self) -> tuple[tuple[int, Action, KeyAndID, int], ...]: + output: list[tuple[int, Action, KeyAndID, int]] = [] + allocation_times: dict[tuple[TensorKey, bool], int] = {} + live_unknown: dict[tuple[int, torch.device], Literal[True]] = {} + + for event in self._op_tree.dfs(): + if event.typed[0] == _EventType.Allocation: + alloc_fields = event.typed[1] + alloc_size = alloc_fields.alloc_size + is_allocation = alloc_size > 0 + t = event.start_time_ns + + tkey = TensorKey.from_allocation(alloc_fields) + if tkey is not None: + allocation_times[(tkey, is_allocation)] = t + + else: + key = Key(alloc_fields.device) + ptr_and_device = (alloc_fields.ptr, key.device) + if is_allocation: + if ptr_and_device in live_unknown: + output.append( + (t, Action.INCREMENT_VERSION, (key, 0), alloc_size) + ) + else: + live_unknown[ptr_and_device] = True + output.append((t, Action.CREATE, (key, 0), alloc_size)) + else: + output.append((t, Action.DESTROY, (key, 0), -alloc_size)) + if not live_unknown.pop(ptr_and_device, False): + output.append( + (-1, Action.PREEXISTING, (key, 0), -alloc_size) + ) + + snapshot = self._category_snapshot() + last_version = dict(sorted(snapshot.keys())) + + events: list[tuple[int, Action, TensorAndID]] = [ + (-1, Action.PREEXISTING, (key, version)) + for key, version in snapshot + if (key, True) not in allocation_times and version == 0 + ] + + for node in self._data_flow_graph.flow_nodes: + for key, edge in node._edges.items(): + if edge.is_allocation: + t = allocation_times[(key, True)] + events.append((t, Action.CREATE, (key, 0))) + + elif edge.mutated: + t = node._event.start_time_ns + version = edge.input_version + assert version is not None + events.append((t, Action.INCREMENT_VERSION, (key, version))) + + if edge.is_deletion: + t = allocation_times[(key, False)] + events.append((t, Action.DESTROY, (key, last_version[key]))) + + output.extend( + (time, action, (key, version), self._size_map[key]) + for time, action, (key, version) in events + ) + + output.sort(key=lambda x: (x[0], x[1].value)) + return tuple(output) + + def _is_gradient(self, *args, **kwargs) -> bool: + return self._categories.get(*args, **kwargs) == Category.GRADIENT + + def _category_snapshot(self) -> dict[TensorAndID, Category | None]: + all_tensor_versions: set[TensorAndID] = set() + + for node in self._data_flow_graph.flow_nodes: + all_tensor_versions.update(((k, v) for k, (_, v) in node.inputs.items())) + all_tensor_versions.update((key, 0) for key in node.intermediates) + all_tensor_versions.update(node.outputs.items()) + + for i in self._categories._values.values(): + all_tensor_versions.update((key, 0) for key in i._by_id_keyset) + + return { + (key, version): self._categories.get(key, version) + for key, version in sorted(all_tensor_versions) + } + + def _any_version_depends_on_gradient(self) -> set[int]: + """Extract IDs of Tensors which depend or will depend on a gradient. + + Note that this weakened definition of "depends" requires us to loop + over the data flow graph multiple times because it allows dependency + information to flow backward through edges and removes the guarantee + that nodes are topologically sorted. (Or indeed, even that a valid + topological order exists.) Put another way, we have converted an + acyclic data flow graph into a cyclic graph and we are attempting to + partition cycles involving a gradient from the rest of the graph. + """ + depends_on_gradient: set[int] = set() + while True: + start_size = len(depends_on_gradient) + for node in self._data_flow_graph.flow_nodes: + ids = tuple( + key.id + for key, (_, version) in node.inputs.items() + if self._categories.get(key, version) + in (Category.GRADIENT, Category.PARAMETER) + or key.id in depends_on_gradient + ) + + if ids: + depends_on_gradient.update(ids) + + depends_on_gradient.update(key.id for key in node.outputs) + + # We are guaranteed to exit because there is a finite set of + # TensorAndID pairs. In practice we do not expect to loop more than + # three times: once to identify the core parameter update loop, + # once to fold the first step into that loop, and a third time + # where no new elements are added. + if len(depends_on_gradient) == start_size: + return depends_on_gradient + + def _set_gradients_and_temporaries(self) -> None: + """Mark Tensors which are unambiguous and simple to reason about.""" + + # Gradients are straightforward to detect. We directly check the + # `.grad` property in the Python tracer, and we can detect any new + # gradient Tensors from `AccumulateGrad` ops. + for event in self._op_tree.dfs(): + for _, p_grad in extract_gradients(event): + self._categories.set_by_id(p_grad, Category.GRADIENT) + + # Similarly, temporary Tensors are easy to identify and are useful to + # flag since they can make memory use "spikier" than one would + # otherwise expect. + for node in self._data_flow_graph.flow_nodes: + for i in node.intermediates: + self._categories.set_by_key(i, Category.TEMPORARY) + + def _set_parameters_using_python_tracer(self) -> None: + for event in self._op_tree.dfs(): + for p in extract_parameters(event): + if p is not None: + self._categories.set_by_id(p, Category.PARAMETER) + + def _set_inputs(self) -> None: + """Mark inputs based on which Tensors are updated using gradients. + + The process for differentiating between inputs and activations is more + involved. Most Tensors in a training loop depend on at least one + gradient: parameters depend on them through updates, and activations + and optimizer state depend on them transitively through parameters. + Critically, we do not need to know which Tensors are parameters to + apply this method; we can simply walk the data flow graph to build the + set of all values which depend on a gradient and then obtain the set + of inputs from the conjugate set. + + There is, however, one hiccup. The first time we see a parameter is + generally on the forward pass of the first step. We know from + inspection of the data flow graph that v1 of that Tensor depends on + a gradient (provided we profile an optimizer step), but not v0. To + address this problem we weaken the definition of "depends on a + gradient" to "any version of this Tensor depends on a gradient", + which in turn strengthens the criteria for the input set enough to + filter the activations in the forward pass of the first step.""" + + # All of this analysis is predicated on using at least one training + # step (or parameters from the python tracer) to partition the graph. + # Absent that we cannot determine which Tensors are inputs and which + # ones are part of the model. + depends_on_gradient = self._any_version_depends_on_gradient() + + # We only want to annotate Tensors which actually contribute to the + # model calculation. + produces_gradient: set[TensorAndID] = set() + for node in reversed(self._data_flow_graph.flow_nodes): + tensors = {(key, version) for key, (_, version) in node.inputs.items()} + tensors |= node.outputs.items() + if any( + self._categories.get(*i) in (Category.GRADIENT, Category.PARAMETER) + or i in produces_gradient + for i in tensors + ): + produces_gradient |= tensors + + # Don't include Tensors created in the backward pass, as these are + # generally Autograd implementation details rather than proper inputs. + input_candidates = produces_gradient.copy() + for node in self._data_flow_graph.flow_nodes: + if RecordScope.BACKWARD_FUNCTION in get_scopes(node._event): + input_candidates -= set(node.outputs.items()) + + for key, version in input_candidates: + if key.id not in depends_on_gradient: + self._categories.setdefault_by_version(key, version, Category.INPUT) + + def _set_parameters_using_data_flow(self) -> None: + """Deduce which Tensors are parameters. + + Consider the following code for the step of SGD with momentum + (nesterov=False), where `d_p` is the gradient of `param` and `buf` is + the momentum buffer. + ``` + buf.mul_(momentum).add_(d_p, alpha=1 - dampening) + d_p = buf + param.add_(d_p, alpha=-lr) + ``` + Both `param` and `buf` take a gradient and perform an in-place update. + + The python tracer will inspect calls to `nn.Module.forward` and + `optim.Optimizer.step` to extract parameter and optimizer state + respectively (including parameters), so this is generally a non-issue. + + However as a fallback we can also exploit several properties of + parameters to distinguish them from other model state. + + First, they are directly used in the forward pass. (At this point we + haven't established which parts of the graph correspond to the forward + pass but we can deduce enough to suffice.) Some mutable state such as + batch norm moving averages also contribute to the forward pass, but + optimizer state does not. + + Second, a parameter is by definition used to compute at least one + gradient and depends on at least one gradient. + """ + snapshot = self._category_snapshot() + + # Determine which Tensors might be parameters based on forward pass + # data flow. Note this these are only candidates; we filter nodes that + # we know are part of the backward pass but that doesn't guarantee that + # they are part of the forward pass. + candidate_parameters: set[TensorAndID] = set() + candidate_fwd_tensors: set[TensorAndID] = { + i for i, category in snapshot.items() if category == Category.INPUT + } + + for node in self._data_flow_graph.flow_nodes: + inputs = {(key, value) for key, (_, value) in node.inputs.items()} + if ( + # Don't check nodes in the backward pass. + RecordScope.BACKWARD_FUNCTION not in get_scopes(node._event) + and not any(self._is_gradient(*i) for i in inputs) + and not any(self._is_gradient(*i) for i in node.outputs.items()) + # + # and only check nodes which depend on an input. + and candidate_fwd_tensors.intersection(inputs) + ): + candidate_fwd_tensors |= node.outputs.items() + candidate_parameters |= inputs.difference(candidate_fwd_tensors) + + # Require that each parameter eventually contributes to the value of a gradient + used_for_gradient: set[TensorAndID] = set() + for node in reversed(self._data_flow_graph.flow_nodes): + if any( + self._is_gradient(*i) or i in used_for_gradient + for i in node.outputs.items() + ): + used_for_gradient.update( + (key, version) for key, (_, version) in node.inputs.items() + ) + candidate_parameters.intersection_update(used_for_gradient) + + # and depends on a gradient. + parameter_keys = {key.id for key, _ in candidate_parameters} + parameter_keys &= self._any_version_depends_on_gradient() + + for key, _ in snapshot: + if key.id in parameter_keys: + self._categories.set_by_id(key, Category.PARAMETER) + + def _set_activations(self) -> None: + """Flood the graph to identify activations.""" + + required = {Category.INPUT, Category.ACTIVATION} + also_allowed = {Category.PARAMETER, Category.TEMPORARY} + for node in self._data_flow_graph.flow_nodes: + inputs = {(key, value) for key, (_, value) in node.inputs.items()} + input_categories = {self._categories.get(*i) for i in inputs} + + if ( + (input_categories & required) + and not (input_categories - (required | also_allowed)) + # + # Stop filling when we reach the backward pass. + and RecordScope.BACKWARD_FUNCTION not in get_scopes(node._event) + ): + for i in node.outputs.items(): + self._categories.setdefault_by_version(*i, Category.ACTIVATION) + + def _set_optimizer_state(self) -> None: + for event in self._op_tree.dfs(): + if event.typed[0] == _EventType.PyCall and event.typed[1].optimizer: + parameters = event.typed[1].optimizer.parameters + for _, t in it.chain.from_iterable( + (state for _, _, state in parameters) + ): + key = TensorKey.from_tensor(t) + if key is not None: + self._categories.set_by_id(key, Category.OPTIMIZER_STATE) + + def _set_autograd_detail(self) -> None: + prior = {None, Category.AUTOGRAD_DETAIL} + for node in self._data_flow_graph.flow_nodes: + if RecordScope.BACKWARD_FUNCTION in get_scopes(node._event): + for key, version in node.outputs.items(): + if version == 0 or self._categories.get(key, version - 1) in prior: + self._categories.setdefault_by_version( + key, version, Category.AUTOGRAD_DETAIL + ) + + +class MemoryProfileTimeline: + def __init__(self, memory_profile) -> None: + """The minimum representation of the memory profile timeline + includes the memory timeline and categories. The timeline + consists of [timestamp, action, (TensorKey, version), numbytes] + elements, to denote any actions (pre-existing, create, destroy, + or increment_version) that occurred to a specific Tensor for a + chunk of memory. The categories help map each (TensorKey, + version) pair into a category.""" + self.timeline = memory_profile.timeline + self.categories = memory_profile._categories + + def _coalesce_timeline(self, device_str): + """Convert the memory timeline and categories into a memory plot + consisting of timestamps and their respective sizes by category + for a given device. + + Input: device + Output: [timestamps, sizes by category] + """ + device = torch.device(device_str) + times: list[int] = [] + sizes: list[list[int]] = [] + + def update(key, version, delta) -> None: + category = ( + self.categories.get(key, version) + if isinstance(key, TensorKey) + else None + ) + index = _CATEGORY_TO_INDEX[category] + 1 + sizes[-1][index] += int(delta) + + t_min = -1 + for t, action, (key, version), numbytes in self.timeline: + if key.device != device: + continue + + # Convert timestamps from ns to us, to match trace events. + if t != -1: + t = int(t / 1000) + + # Save the smallest timestamp to populate pre-existing allocs. + if t_min == -1 or (t < t_min and t > 0): + t_min = t + + # Handle timestep + if len(times) == 0: + times.append(t) + sizes.append([0] + [0 for _ in _CATEGORY_TO_INDEX]) + + elif t != times[-1]: + times.append(t) + sizes.append(sizes[-1].copy()) + + # Handle memory and categories + if action in (Action.PREEXISTING, Action.CREATE): + update(key, version, numbytes) + + elif action == Action.INCREMENT_VERSION: + update(key, version, -numbytes) + update(key, version + 1, numbytes) + + elif action == Action.DESTROY: + update(key, version, -numbytes) + + else: + raise ValueError(f"Unknown action: {action}") + + times = [t_min if t < 0 else t for t in times] + return times, sizes + + def export_memory_timeline(self, path, device_str) -> None: + """Saves the memory timeline as [times, sizes by category] + as a JSON formatted file to the given path for the given + device.""" + times, sizes = self._coalesce_timeline(device_str) + # TODO: Write a faster serialize (orjson not available in CI) + import json + + with open(path, "w") as f: + json.dump([times, sizes], f) + + def export_memory_timeline_raw(self, path, device_str) -> None: + """Saves the memory timeline as raw memory event tuples in the + form of (timestamp, action, numbytes, category) + as a JSON formatted file to the given path for the given + device.""" + device = torch.device(device_str) + raw_events: list[tuple[int, int, int, int]] = [] + + def get_category_index(key, version): + category = ( + self.categories.get(key, version) + if isinstance(key, TensorKey) + else None + ) + return _CATEGORY_TO_INDEX[category] + + for t, action, (key, version), numbytes in self.timeline: + if key.device != device: + continue + + if action in (Action.PREEXISTING, Action.CREATE): + raw_events.append( + # pyrefly: ignore [bad-argument-type] + ( + t, + _ACTION_TO_INDEX[action], + numbytes, + get_category_index(key, version), + ) + ) + + elif action == Action.INCREMENT_VERSION: + raw_events.append( + # pyrefly: ignore [bad-argument-type] + ( + t, + _ACTION_TO_INDEX[action], + -numbytes, + get_category_index(key, version), + ) + ) + raw_events.append( + # pyrefly: ignore [bad-argument-type] + ( + t, + _ACTION_TO_INDEX[action], + numbytes, + get_category_index(key, version + 1), + ) + ) + + elif action == Action.DESTROY: + raw_events.append( + # pyrefly: ignore [bad-argument-type] + ( + t, + _ACTION_TO_INDEX[action], + -numbytes, + get_category_index(key, version), + ) + ) + + else: + raise ValueError(f"Unknown action: {action}") + + import json + + with open(path, "w") as f: + json.dump(raw_events, f) + + def export_memory_timeline_html( + self, path, device_str, figsize=(20, 12), title=None + ) -> None: + """Exports the memory timeline as an HTML file which contains + the memory timeline plot embedded as a PNG file.""" + # Check if user has matplotlib installed, return gracefully if not. + import importlib.util + + matplotlib_spec = importlib.util.find_spec("matplotlib") + if matplotlib_spec is None: + print( + "export_memory_timeline_html failed because matplotlib was not found." + ) + return + + from base64 import b64encode + from tempfile import NamedTemporaryFile + + import matplotlib.pyplot as plt + import numpy as np + + mt = self._coalesce_timeline(device_str) + times, sizes = np.array(mt[0]), np.array(mt[1]) + # For this timeline, start at 0 to match Chrome traces. + t_min = min(times) + times -= t_min + stacked = np.cumsum(sizes, axis=1) / 1024**3 + device = torch.device(device_str) + max_memory_allocated = torch.cuda.max_memory_allocated(device) + max_memory_reserved = torch.cuda.max_memory_reserved(device) + + # Plot memory timeline as stacked data + fig = plt.figure(figsize=figsize, dpi=80) + axes = fig.gca() + for category, color in _CATEGORY_TO_COLORS.items(): + i = _CATEGORY_TO_INDEX[category] + axes.fill_between( + times / 1e3, stacked[:, i], stacked[:, i + 1], color=color, alpha=0.7 + ) + fig.legend(["Unknown" if i is None else i.name for i in _CATEGORY_TO_COLORS]) + # Usually training steps are in magnitude of ms. + axes.set_xlabel("Time (ms)") + axes.set_ylabel("Memory (GB)") + title = "\n\n".join( + ([title] if title else []) + + [ + f"Max memory allocated: {max_memory_allocated / (1024**3):.2f} GiB \n" + f"Max memory reserved: {max_memory_reserved / (1024**3):.2f} GiB" + ] + ) + axes.set_title(title) + + # Embed the memory timeline image into the HTML file + with NamedTemporaryFile("wb", suffix=".png") as tmpfile: + fig.savefig(tmpfile, format="png") + + tmpfile.seek(0, 0) + encoded = b64encode(tmpfile.read()).decode("utf-8") + assert encoded + html = f""" +GPU Memory Timeline HTML + + + +""" + + with open(path, "w", encoding="utf-8") as f: + f.write(html) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/_pattern_matcher.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/_pattern_matcher.py new file mode 100644 index 0000000000000000000000000000000000000000..35ab2119684d600f6fbdc72bbbc75ee2aa874279 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/_pattern_matcher.py @@ -0,0 +1,666 @@ +# mypy: allow-untyped-defs +import json +import math +import os +import re + +import torch +import torch.utils.benchmark as benchmark +from torch._C._profiler import ( + _EventType, + _ExtraFields_PyCall, + _ExtraFields_PyCCall, + _ExtraFields_TorchOp, + _ProfilerEvent, +) +from torch.profiler import profile +from torch.profiler._utils import index_of_first_match, traverse_bfs, traverse_dfs + + +class Pattern: + """ + Base class for all patterns, subclass this class and implement match() + to define custom patterns. + + In subclass, define description and skip property. + """ + + def __init__(self, prof: profile, should_benchmark: bool = False) -> None: + self.prof = prof + self.should_benchmark = should_benchmark + self.name = "Please specify a name for pattern" + self.description = "Please specify a description for pattern" + self.url = "" + assert prof.profiler is not None and prof.profiler.kineto_results is not None + self.event_tree = prof.profiler.kineto_results.experimental_event_tree() + self.tid_root: dict[int, list[_ProfilerEvent]] = {} + for event in self.event_tree: + self.tid_root.setdefault(event.start_tid, []).append(event) + + @property + def skip(self) -> bool: + return False + + def report(self, event: _ProfilerEvent): + msg = ( + f"{self.description}\n[Source Code Location] {source_code_location(event)}" + ) + return msg + + def eventTreeTraversal(self): + """ + Traverse the event tree and yield all events. + Override this method in subclass to customize the traversal. + """ + yield from traverse_dfs(self.event_tree) + + def summary(self, events: list[_ProfilerEvent]): + default_summary = f"{self.name}: {len(events)} events matched." + if self.should_benchmark: + # If benchmark summary is not empty, use it. + return ( + self.benchmark_summary(events) + if hasattr(self, "benchmark") # type: ignore[attr-defined] + else default_summary + ) + return default_summary + + def benchmark_summary(self, events: list[_ProfilerEvent]) -> str: + def format_time(time_ns: int) -> str: + unit_lst = ["ns", "us", "ms"] + for unit in unit_lst: + if time_ns < 1000: + return f"{time_ns:.2f} {unit}" + time_ns //= 1000 + return f"{time_ns:.2f} s" + + assert hasattr(self, "benchmark"), "Please implement benchmark()" + shapes_factor_map = self.benchmark(events) # type: ignore[attr-defined] + original_time = sum(event.duration_time_ns for event in events) + new_time = sum( + shapes_factor_map[input_shapes(event)] * event.duration_time_ns + for event in events + ) + return ( + f"{self.name}: {len(events)} events matched. " + f"Total Estimated Speedup: {format_time(original_time - new_time)} ({round(original_time / new_time, 2)}X)" + ) + + def match(self, event: _ProfilerEvent): + """ + Return True if the event matches the pattern. + This method should be overridden in subclass. + """ + raise NotImplementedError + + def matched_events(self): + if self.skip: + return [] + matched_events = [ + event for event in self.eventTreeTraversal() if self.match(event) + ] + return matched_events + + def root_of(self, event: _ProfilerEvent): + while event.parent: + event = event.parent + return event + + def siblings_of(self, event: _ProfilerEvent): + if event.parent: + children = event.parent.children + else: + children = self.tid_root[event.start_tid] + index = children.index(event) + return children[:index], children[index + 1 :] + + def next_of(self, event: _ProfilerEvent): + _, next_events = self.siblings_of(event) + return next_events[0] if next_events else None + + def prev_of(self, event: _ProfilerEvent): + prev_events, _ = self.siblings_of(event) + return prev_events[-1] if prev_events else None + + def go_up_until(self, event: _ProfilerEvent, predicate): + if not event: + return None + while event.parent and not predicate(event): + event = event.parent + return event + + +# Patterns + + +class NamePattern(Pattern): + def __init__( + self, prof: profile, name: str, should_benchmark: bool = False + ) -> None: + super().__init__(prof, should_benchmark) + self.description = f"Matched Name Event: {name}" + self.name = name + + def match(self, event: _ProfilerEvent): + return re.search(self.name, event.name) is not None + + +class ExtraCUDACopyPattern(Pattern): + """ + This pattern identifies if we creates a constant tensor on CPU and immediately moves it to GPU. + example: torch.zeros((100, 100)).to("cuda") + + Pattern: + built-in method |built-in method + ... | aten::to + aten::fill_/aten::zero_ | aten::_to_copy + + Algorithm: + We start at node aten::to, go parent events' previous events, + and check if we have a aten::fill_/aten::zero_ as we keep going down the tree. + We always select the last child in the children list when we go down the tree. + If at any step we failed, it is not a match. + """ + + def __init__(self, prof: profile, should_benchmark: bool = False) -> None: + super().__init__(prof, should_benchmark) + self.name = "Extra CUDA Copy Pattern" + self.description = "Filled a CPU tensor and immediately moved it to GPU. Please initialize it on GPU." + self.url = "https://pytorch.org/tutorials/recipes/recipes/tuning_guide.html#create-tensors-directly-on-the-target-device" + self.init_ops = { + "aten::fill_", + "aten::zero_", + "aten::normal_", + "aten::uniform_", + } + + @property + def skip(self) -> bool: + return not self.prof.with_stack or not self.prof.record_shapes + + def match(self, event): + # TODO: We should also check tensor identities + if event.name != "aten::to": + return False + to_event = event + if not event.children: + return False + event = event.children[-1] + if event.name != "aten::_to_copy": + return False + if not event.children: + return False + event = event.children[-1] + if event.name != "aten::copy_": + return False + # aten::copy_ should have the first 2 args dtype the same + dtypes = input_dtypes(event) + if len(dtypes) < 2: + return False + if dtypes[0] is None or dtypes[0] != dtypes[1]: + return False + event = to_event + # Up one level + event = event.parent + if event is None: + return False + # Check if we have a aten::fill_ in previous leaf + event = self.prev_of(event) + if event is None: + return False + while event.children: + event = event.children[-1] + # aten::zero_ is a special optimization case where fill_ is not called + if event.name in self.init_ops: + return True + return event.name in self.init_ops + # TODO: Check if tensor is reused + + def benchmark(self, events: list[_ProfilerEvent]): + shapes_factor_map = {input_shapes(event): 0.0 for event in events} + for shape in shapes_factor_map: + size = shape[0] + to_timer = benchmark.Timer( + stmt='torch.ones(size).to("cuda")', globals={"size": size} + ) + de_timer = benchmark.Timer( + stmt='torch.ones(size, device="cuda")', globals={"size": size} + ) + to_time = to_timer.timeit(10).mean + de_time = de_timer.timeit(10).mean + shapes_factor_map[shape] = de_time / to_time + return shapes_factor_map + + +class ForLoopIndexingPattern(Pattern): + """ + This pattern identifies if we use a for loop to index a tensor that + can be vectorized. + example: + tensor = torch.empty((100, 100)) + for i in range(100): + tensor[i] = i + + Pattern: + aten::select | ... | aten::select | ... (Repeat) + + Algorithm: + We start at node aten::select, and we check if we can find this alternating patterns. + We also keep a dictionary to avoid duplicate match in the for loop. + """ + + def __init__(self, prof: profile, should_benchmark: bool = False) -> None: + super().__init__(prof, should_benchmark) + self.name = "For Loop Indexing Pattern" + self.description = "For loop indexing detected. Vectorization recommended." + self.visited: set[int] = set() + + def eventTreeTraversal(self): + """ + We need to use BFS traversal order to avoid duplicate match. + """ + yield from traverse_bfs(self.event_tree) + + def match(self, event: _ProfilerEvent): + if event.name != "aten::select": + return False + if event.id in self.visited: + return False + repeat_count = 1 + _, next = self.siblings_of(event) + if len(next) <= 1: + return False + + # Custom event list matching + def same_ops(list1, list2) -> bool: + if len(list1) != len(list2): + return False + for op1, op2 in zip(list1, list2, strict=True): + if op1.name != op2.name: + return False + return True + + # Record the ops between two aten::select + next_select_idx = index_of_first_match(next, lambda e: e.name == "aten::select") + if next_select_idx is None: + return False + indexing_ops = [event] + next[:next_select_idx] + next = next[len(indexing_ops) - 1 :] + for i in range(0, len(next), len(indexing_ops)): + if same_ops(indexing_ops, next[i : i + len(indexing_ops)]): + repeat_count += 1 + self.visited.add(next[i].id) + else: + break + return repeat_count >= 10 + + +class FP32MatMulPattern(Pattern): + def __init__(self, prof: profile, should_benchmark: bool = False) -> None: + super().__init__(prof, should_benchmark) + self.name = "FP32 MatMul Pattern" + self.description = ( + "You are currently using GPU that supports TF32. " + "Please enable TF32 by setting 'torch.backends.cuda.matmul.allow_tf32 = True'" + ) + self.url = "https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + + @property + def skip(self): + if torch.version.hip is not None: + has_tf32 = False + else: + # Anything less than sm_80 is not Ampere which doesn't support TF32 + has_tf32 = all( + int(re.sub("sm_|compute_", "", arch)) >= 80 + for arch in torch.cuda.get_arch_list() + ) + return has_tf32 is False or super().skip or not self.prof.record_shapes + + def match(self, event: _ProfilerEvent) -> bool: + # If we saw this pattern once, we don't need to match it again + if event.tag != _EventType.TorchOp: + return False + assert isinstance(event.extra_fields, _ExtraFields_TorchOp) + if event.name == "aten::mm": + if event.extra_fields.allow_tf32_cublas is False: + return True + return False + + def report(self, event: _ProfilerEvent): + return self.description + + def benchmark(self, events: list[_ProfilerEvent]): + shapes_factor_map = {input_shapes(event): 0.0 for event in events} + for shape in shapes_factor_map: + matrixA = torch.randn(shape[0], device="cuda", dtype=torch.float32) + matrixB = torch.randn(shape[1], device="cuda", dtype=torch.float32) + fp32_timer = benchmark.Timer( + stmt="torch.mm(matrixA, matrixB)", + globals={"matrixA": matrixA, "matrixB": matrixB}, + ) + tf32_timer = benchmark.Timer( + stmt="torch.mm(matrixA, matrixB)", + setup="torch.backends.cuda.matmul.allow_tf32 = True", + globals={"matrixA": matrixA, "matrixB": matrixB}, + ) + torch.backends.cuda.matmul.allow_tf32 = False + fp32_time = fp32_timer.timeit(10).mean + tf32_time = tf32_timer.timeit(10).mean + shapes_factor_map[shape] = tf32_time / fp32_time + return shapes_factor_map + + +class OptimizerSingleTensorPattern(Pattern): + """ + This pattern identifies if we are using the single-tensor version of an optimizer. + example: + optimizer = torch.optim.SGD(model.parameters(), lr=0.1) + By adding foreach=True to enable multi-tensor optimizer, we can gain speedup when + the kernels are relatively small. + + Pattern: + XXXXX: _single_tenser_ + + Algorithm: + String match + """ + + def __init__(self, prof: profile, should_benchmark: bool = False) -> None: + super().__init__(prof, should_benchmark) + self.name = "Optimizer Single Tensor Pattern" + self.optimizers_with_foreach = ["adam", "sgd", "adamw"] + self.description = ( + "Detected optimizer running with single tensor implementation. " + "Please enable multi tensor implementation by passing 'foreach=True' into optimizer." + ) + self.url = "" + + def match(self, event: _ProfilerEvent) -> bool: + for optimizer in self.optimizers_with_foreach: + if event.name.endswith(f"_single_tensor_{optimizer}"): + return True + return False + + +class SynchronizedDataLoaderPattern(Pattern): + """ + This pattern identifies if we are using num_workers=0 in DataLoader. + example: + torch.utils.data.DataLoader(dataset, batch_size=batch_size) + Add num_workers=N to the arguments. N depends on system configuration. + + Pattern: + dataloader.py(...): __iter__ + dataloader.py(...): _get_iterator + NOT dataloader.py(...): check_worker_number_rationality + + Algorithm: + If we don't see check_worker_number_rationality call in the dataloader __iter__, + It is not an asynchronous dataloader. + + """ + + def __init__(self, prof: profile, should_benchmark: bool = False) -> None: + super().__init__(prof, should_benchmark) + self.name = "Synchronized DataLoader Pattern" + self.description = ( + "Detected DataLoader running with synchronized implementation. " + "Please enable asynchronous dataloading by setting num_workers > 0 when initializing DataLoader." + ) + self.url = ( + "https://pytorch.org/tutorials/recipes/recipes/tuning_guide.html" + "#enable-async-data-loading-and-augmentation" + ) + + def match(self, event: _ProfilerEvent) -> bool: + def is_dataloader_function(name: str, function_name: str): + return name.startswith( + os.path.join("torch", "utils", "data", "dataloader.py") + ) and name.endswith(function_name) + + # TODO: fixme! Due to lifetime issues of the function name, this field might + # actually point to an already freed string when the even is a PyCall. + # Just silently skip this to unblock testing. + try: + event.name + except UnicodeDecodeError: + return False + + if not is_dataloader_function(event.name, "__iter__"): + return False + if not event.children: + return False + event = event.children[0] + if not is_dataloader_function(event.name, "_get_iterator"): + return False + if not event.children: + return False + event = event.children[0] + return not is_dataloader_function(event.name, "check_worker_number_rationality") + # TODO: We should also check if the loader is bottleneck. + + +class GradNotSetToNonePattern(Pattern): + """ + This pattern identifies if we are not setting grad to None in zero_grad. + example: + optimizer.zero_grad() + By setting set_to_none=True, we can gain speedup + + Pattern: + XXXXX: _zero_grad + NOT aten::zeros + aten::zero_ + + aten::zero_ is called on each parameter in the model. + We also want to make sure it is not called by aten::zeros. + + Algorithm: + String match + """ + + def __init__(self, prof: profile, should_benchmark: bool = False) -> None: + super().__init__(prof, should_benchmark) + self.name = "Gradient Set To Zero Instead of None Pattern" + self.description = ( + "Detected gradient set to zero instead of None. " + "Please add 'set_to_none=True' when calling zero_grad()." + ) + self.url = ( + "https://pytorch.org/tutorials/recipes/recipes/tuning_guide.html" + "#disable-gradient-calculation-for-validation-or-inference" + ) + + def match(self, event: _ProfilerEvent) -> bool: + if not event.name.endswith(": zero_grad"): + return False + if not event.children: + return False + + for sub_event in traverse_dfs(event.children): + if ( + sub_event.name == "aten::zero_" + and sub_event.parent.name != "aten::zeros" + ): + return True + # TODO: We should also check if the optimizer's numerical behavior will change. + return False + + +class Conv2dBiasFollowedByBatchNorm2dPattern(Pattern): + """ + This pattern identifies if we are enabling bias in Conv2d which is followed by BatchNorm2d. + Bias doesn't do anything when followed by batchnorm. + Pattern: + nn.Module: Conv2d | nn.Module: BatchNorm2d + ... + aten::conv2d AND dtype of third argument is not null + The third argument is the bias + Algorithm: + String match + """ + + def __init__(self, prof: profile, should_benchmark: bool = False) -> None: + super().__init__(prof, should_benchmark) + self.name = "Enabling Bias in Conv2d Followed By BatchNorm Pattern" + self.description = "Detected bias enabled in Conv2d that is followed by BatchNorm2d. Please set 'bias=False' in Conv2d." + self.url = ( + "https://pytorch.org/tutorials/recipes/recipes/tuning_guide.html" + "#disable-bias-for-convolutions-directly-followed-by-a-batch-norm" + ) + + @property + def skip(self): + return self.prof.record_shapes is False or super().skip + + def match(self, event: _ProfilerEvent): + if event.name != "aten::conv2d": + return False + if len(input_dtypes(event)) < 3 or input_dtypes(event)[2] is None: + return False + # This means bias=True + event = self.go_up_until( + event, lambda e: e.name.startswith("nn.Module: Conv2d") + ) + if not event: + return False + event = self.next_of(event) + if not event: + return False + return event.name.startswith("nn.Module: BatchNorm2d") + + +class MatMulDimInFP16Pattern(Pattern): + def __init__(self, prof: profile, should_benchmark: bool = False) -> None: + super().__init__(prof, should_benchmark) + self.name = "Matrix Multiplication Dimension Not Aligned Pattern" + self.description = "Detected matmul with dimension not aligned. Please use matmul with aligned dimension." + self.url = "https://pytorch.org/tutorials/recipes/recipes/tuning_guide.html#use-mixed-precision-and-amp" + + @property + def skip(self) -> bool: + return not self.prof.with_stack or not self.prof.record_shapes + + def match(self, event: _ProfilerEvent) -> bool: + def mutiple_of(shapes, multiple): + return all(dim % multiple == 0 for shape in shapes for dim in shape[-2:]) + + if event.name not in ("aten::mm", "aten::bmm", "aten::addmm"): + return False + if not input_dtypes(event): + return False + arg_dtype = input_dtypes(event)[0] + if arg_dtype in (torch.bfloat16, torch.half) and not mutiple_of( + input_shapes(event), 8 + ): + return True + return False + + def benchmark(self, events: list[_ProfilerEvent]): + def closest_multiple(shapes, multiple): + return [multiple * math.ceil(shape / multiple) for shape in shapes] + + shapes_factor_map = {input_shapes(event): 0.0 for event in events} + for shape in shapes_factor_map: + matrixA = torch.randn(shape[0], device="cuda", dtype=torch.float16) + matrixB = torch.randn(shape[1], device="cuda", dtype=torch.float16) + not_aligned_dim_timer = benchmark.Timer( + stmt="torch.mm(matrixA, matrixB)", + globals={"matrixA": matrixA, "matrixB": matrixB}, + ) + matrixA = torch.randn( + closest_multiple(shape[0], 8), device="cuda", dtype=torch.float16 + ) + matrixB = torch.randn( + closest_multiple(shape[1], 8), device="cuda", dtype=torch.float16 + ) + aligned_dim_timer = benchmark.Timer( + stmt="torch.mm(matrixA, matrixB)", + globals={"matrixA": matrixA, "matrixB": matrixB}, + ) + not_aligned_dim_time = not_aligned_dim_timer.timeit(10).mean + aligned_dim_time = aligned_dim_timer.timeit(10).mean + shapes_factor_map[shape] = aligned_dim_time / not_aligned_dim_time + return shapes_factor_map + + +def source_code_location(event: _ProfilerEvent | None) -> str: + while event: + if event.tag == _EventType.PyCall or event.tag == _EventType.PyCCall: + assert isinstance( + event.extra_fields, (_ExtraFields_PyCall, _ExtraFields_PyCCall) + ) + if not event.extra_fields.caller.file_name.startswith("torch" + os.sep): + return f"{event.extra_fields.caller.file_name}:{event.extra_fields.caller.line_number}" + event = event.parent + return "No source code location found" + + +def input_shapes(event: _ProfilerEvent): + assert isinstance(event.extra_fields, _ExtraFields_TorchOp) + return tuple(tuple(getattr(i, "sizes", ())) for i in event.extra_fields.inputs) + + +def input_dtypes(event: _ProfilerEvent): + assert isinstance(event.extra_fields, _ExtraFields_TorchOp) + return tuple(getattr(i, "dtype", None) for i in event.extra_fields.inputs) + + +def report_all_anti_patterns( + prof, + should_benchmark: bool = False, + print_enable: bool = True, + json_report_dir: str | None = None, +) -> None: + report_dict: dict = {} + anti_patterns = [ + ExtraCUDACopyPattern(prof, should_benchmark), + # ForLoopIndexingPattern(prof, should_benchmark), + FP32MatMulPattern(prof, should_benchmark), + OptimizerSingleTensorPattern(prof, should_benchmark), + SynchronizedDataLoaderPattern(prof, should_benchmark), + GradNotSetToNonePattern(prof, should_benchmark), + Conv2dBiasFollowedByBatchNorm2dPattern(prof, should_benchmark), + MatMulDimInFP16Pattern(prof, should_benchmark), + ] + reported = set() + summaries = [] + message_list = [f"{'-' * 40}TorchTidy Report{'-' * 40}"] + message_list.append("Matched Events:") + + for anti_pattern in anti_patterns: + matched_events = anti_pattern.matched_events() + if not matched_events: + continue + summaries.append(anti_pattern.summary(matched_events)) + for event in matched_events: + report_msg = anti_pattern.report(event) + if report_msg not in reported: + message_list.append(report_msg) + reported.add(report_msg) + src_location, line_no = source_code_location(event).split(":") + report_dict.setdefault(src_location, []).append( + { + "line_number": int(line_no), + "name": anti_pattern.name, + "url": anti_pattern.url, + "message": anti_pattern.description, + } + ) + + if json_report_dir is not None: + json_report_path = os.path.join(json_report_dir, "torchtidy_report.json") + if os.path.exists(json_report_path): + with open(json_report_path) as f: + exisiting_report = json.load(f) + exisiting_report.update(report_dict) + report_dict = exisiting_report + with open(json_report_path, "w") as f: + json.dump(report_dict, f, indent=4) + + message_list.append("Summary:") + message_list += summaries + message_list.append(f"{'-' * 40}TorchTidy Report{'-' * 40}") + if print_enable: + print("\n".join(message_list)) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8a7647afb7adc69d80a8046c616aca0de867eb9a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/_utils.py @@ -0,0 +1,569 @@ +# mypy: allow-untyped-defs +import functools +import operator +import re +from collections import deque +from dataclasses import dataclass +from typing import Any, Literal, TYPE_CHECKING + +from torch.autograd.profiler import profile +from torch.profiler import DeviceType + + +if TYPE_CHECKING: + from torch.autograd import _KinetoEvent + + +def _traverse(tree, next_fn, children_fn=lambda x: x.children, reverse: bool = False): + order = reversed if reverse else lambda x: x + remaining = deque(order(tree)) + while remaining: + curr_event = next_fn(remaining) + yield curr_event + for child_event in order(children_fn(curr_event)): + remaining.append(child_event) + + +traverse_dfs = functools.partial(_traverse, next_fn=lambda x: x.pop(), reverse=True) +traverse_bfs = functools.partial( + _traverse, next_fn=lambda x: x.popleft(), reverse=False +) + + +@dataclass +class EventMetrics: + duration_time_ns: int = 0 + self_time_ns: int = 0 + idle_time_ns: int = 0 + queue_depth: int = 0 + + @property + def fraction_idle_time(self): + if self.duration_time_ns == 0: + return 0.0 + return self.idle_time_ns / self.duration_time_ns + + +@dataclass +class Interval: + start: int + end: int + queue_depth: int = 0 + + +class EventKey: + def __init__(self, event) -> None: + self.event = event + + def __hash__(self): + return hash(self.event.id) + + def __eq__(self, other): + return self.event.id == other.event.id + + def __repr__(self) -> str: + return f"{self.event.name}" + + def intervals_overlap(self, intervals: list[Interval]): + overlap_time = 0 + intervals = sorted(intervals, key=lambda x: x.start) + + if intervals: + overlap_start = max(self.event.start_time_ns, intervals[0].start) + overlap_end = min(self.event.end_time_ns, intervals[0].end) + + if overlap_start < overlap_end: + overlap_time += overlap_end - overlap_start + + i, j = 0, 1 + while j < len(intervals): + prev_interval = intervals[i] + curr_interval = intervals[j] + j += 1 + if prev_interval.end > curr_interval.start: + # Completely subsumed by previous interval + if prev_interval.end > curr_interval.end: + j += 1 + continue + else: + curr_interval.start = prev_interval.end + i = j + + overlap_start = max(self.event.start_time_ns, curr_interval.start) + overlap_end = min(self.event.end_time_ns, curr_interval.end) + if overlap_start < overlap_end: + overlap_time += overlap_end - overlap_start + + return overlap_time + + +class BasicEvaluation: + def __init__(self, prof: profile) -> None: + self.profile = prof + self.metrics: dict[EventKey, EventMetrics] = {} + self.compute_self_time() + self.event_keys = sorted( + self.metrics.keys(), key=lambda x: x.event.start_time_ns + ) + self.events = [e.event for e in self.event_keys] + self.cuda_events: list[_KinetoEvent] = [] + self.queue_depth_list = self.compute_queue_depth() + self.compute_idle_time() + + def compute_self_time(self) -> None: + """ + Computes event's self time(total time - time in child ops). + """ + assert self.profile.kineto_results is not None + stack = deque(self.profile.kineto_results.experimental_event_tree()) + + # standard iterating dfs + while stack: + curr_event = stack.pop() + self_time = curr_event.duration_time_ns + for child_event in curr_event.children: + self_time -= child_event.duration_time_ns + stack.append(child_event) + assert EventKey(curr_event) not in self.metrics, ( + f"Duplicate id: {curr_event.id}, {curr_event.name}" + ) + self.metrics[EventKey(curr_event)] = EventMetrics(self_time_ns=self_time) + self.metrics[ + EventKey(curr_event) + ].duration_time_ns = curr_event.duration_time_ns + + def compute_queue_depth(self): + """ + Computes queue_depth at each event. This will calculate the queue depth data for + All the events in the tree. + This will return a list of Interval of queue depth data of cuda launch and kernels. + """ + assert self.profile.kineto_results is not None + cuda_event_list = self.profile.kineto_results.events() + + def is_cuda_launch_kernel(e): + """Check if the event is a CUDA launch kernel.""" + launch_patterns = { + "cudaLaunchKernel", # Standard CUDA + "cudaLaunchKernelExC", # Extended C + "__cudaLaunchKernel", # Internal + "cudaLaunchCooperativeKernel", # Collaborative (single-device) + "cudaLaunchCooperativeKernelMultiDevice", # Collaborative (multi-devices) + } + name = str(getattr(e, "name", e)) + return any(name.startswith(pattern) for pattern in launch_patterns) + + def is_cuda_kernel(e): + """Check if the event is a CUDA runtime kernel.""" + # Check if the kernel is CUDA + if e.device_type() != DeviceType.CUDA: + return False + + name = str(getattr(e, "name", e)).lower() + + # Exclude memory operations + exclude_patterns = {"mem", "cpy", "alloc", "free"} + + return not any(pattern in name for pattern in exclude_patterns) + + cuda_launch_events = sorted( + (e for e in cuda_event_list if is_cuda_launch_kernel(e)), + key=lambda x: x.start_ns(), + ) + cuda_kernel_events = sorted( + (e for e in cuda_event_list if is_cuda_kernel(e)), + key=lambda x: x.start_ns(), + ) + + self.cuda_events = sorted( + cuda_launch_events + cuda_kernel_events, key=lambda x: x.start_ns() + ) + + kernel_mapping: dict[_KinetoEvent, int] = {} + last_mapped_kernel = 0 + for cuda_launch_event in cuda_launch_events: + index = index_of_first_match( + cuda_kernel_events, + lambda x: x.linked_correlation_id() + == cuda_launch_event.linked_correlation_id(), + start=last_mapped_kernel, + ) + kernel_mapping[cuda_launch_event] = index + last_mapped_kernel = index if index is not None else last_mapped_kernel + + current_kernel_index = 0 + spawned_kernel_index = -1 + + all_events = cuda_launch_events + cuda_kernel_events + self.events + + def new_old_event_comparator(event): + if hasattr(event, "start_us"): + return event.start_us() * 1000 + if hasattr(event, "start_ns"): + return event.start_ns() + if hasattr(event, "start_time_ns"): + return event.start_time_ns + raise Exception("Unknown Event Type") # noqa: TRY002 + + queue_depth_list: list[Interval] = [] + all_events.sort(key=new_old_event_comparator) + for event in all_events: + # Find latest cuda kernel event + if hasattr(event, "start_us"): + start_time = event.start_us() * 1000 + # pyrefly: ignore [missing-attribute] + end_time = (event.start_us() + event.duration_us()) * 1000 + # Find current spawned cuda kernel event + if event in kernel_mapping and kernel_mapping[event] is not None: + spawned_kernel_index = kernel_mapping[event] + if hasattr(event, "start_ns"): + start_time = event.start_ns() + end_time = event.start_ns() + event.duration_ns() + # Find current spawned cuda kernel event + if event in kernel_mapping and kernel_mapping[event] is not None: + spawned_kernel_index = kernel_mapping[event] + elif hasattr(event, "start_time_ns"): + start_time = event.start_time_ns # type: ignore[attr-defined] + end_time = event.end_time_ns # type: ignore[attr-defined] + + while ( + current_kernel_index < len(cuda_kernel_events) + and (cuda_kernel_events[current_kernel_index].start_ns()) <= start_time # type: ignore[possibly-undefined] + ): + current_kernel_index += 1 + current_queue_depth = spawned_kernel_index - current_kernel_index + 1 + current_queue_depth = max(current_queue_depth, 0) + + if hasattr(event, "start_us") or hasattr(event, "start_ns"): + queue_depth_list.append( + Interval(start_time, end_time, current_queue_depth) # type: ignore[possibly-undefined] + ) + elif hasattr(event, "start_time_ns"): + self.metrics[EventKey(event)].queue_depth = current_queue_depth + + return queue_depth_list + + def compute_idle_time(self) -> None: + """ + Computes idle time of the profile. + """ + # Based on queue_depth_list, we can calculate idle time for all the events + idle = False + idle_start = 0 + idle_intervals: list[Interval] = [] + if self.queue_depth_list and self.events: + idle_intervals += [ + Interval(self.events[0].start_time_ns, self.queue_depth_list[0].start), + Interval(self.queue_depth_list[-1].end, self.events[-1].end_time_ns), + ] + + for data_point in self.queue_depth_list: + if data_point.queue_depth == 0 and not idle: + idle_start = data_point.end + idle = True + if data_point.queue_depth > 0 and idle: + idle_intervals.append(Interval(idle_start, data_point.start)) + idle = False + + event_list = [e.event for e in self.metrics] + for event in event_list: + self.metrics[EventKey(event)].idle_time_ns = EventKey( + event + ).intervals_overlap(idle_intervals) + + def rank_events(self, length): + """ + Filter and Rank the events based on some heuristics: + 1) Events that are in the falling phase of the queue depth. + 2) Events that have a high idle_time, self_time difference. + + Parameters: + length: The number of events to return. + """ + + # Find the interval when qd is falling to 0 + import torch + + queue_depth_list = list(reversed(self.queue_depth_list)) + qd_values = [e.queue_depth for e in queue_depth_list] + + bottom_threashold = 0 + top_threashold = 4 + decrease_interval = [] + i = 0 + while i < len(qd_values): + if qd_values[i] > bottom_threashold: + i += 1 + continue + for j in range(i + 1, len(qd_values)): + # Find next zero and if the max value between them exceeds + # the threshold, then we have a falling interval + next_minimum_idx = index_of_first_match( + qd_values, lambda x: x <= bottom_threashold, start=j + ) + peak_idx = argmax(qd_values, start=j, end=next_minimum_idx) + + # if is a valid peak, we add to list and continue + if peak_idx is not None and qd_values[peak_idx] >= top_threashold: + decrease_interval.append( + Interval( + queue_depth_list[peak_idx].start, queue_depth_list[i].start + ) + ) + i = next_minimum_idx if next_minimum_idx is not None else i + break + i += 1 + # Filter out events that are not in the decrease interval + event_list = [ + event + for event in self.metrics + if event.intervals_overlap(decrease_interval) + ] + if event_list: + self_time = torch.tensor( + [self.metrics[event].self_time_ns for event in event_list], + dtype=torch.float32, + ) + idle_time = torch.tensor( + [self.metrics[event].fraction_idle_time for event in event_list], + dtype=torch.float32, + ) + normalized_gain = (idle_time - torch.mean(idle_time)) / torch.std(idle_time) + normalized_self = (self_time - torch.mean(self_time)) / torch.std(self_time) + heuristic_score_list = normalized_gain + 0.6 * normalized_self + + # Sort events by heuristic + event_list = [ + event + for _, event in sorted( + zip(heuristic_score_list, event_list, strict=True), + key=operator.itemgetter(0), + reverse=True, + ) + ] + event_list = event_list[:length] + return event_list + + def get_optimizable_events(self, length: int = 1, print_enable: bool = True): + event_list = self.rank_events(length) + if not print_enable: + return event_list + output = "Optimizable events:\n" if event_list else "No events to optimize\n" + + output += "\n".join( + [ + f"""{"-" * 80} +Event: {event} +Source code location: {source_code_location(event.event)} +Percentage idle time: {self.metrics[event].fraction_idle_time * 100:.2f}% +{"-" * 80}""" + for event in event_list + ] + ) + if print_enable: + print(output) + return event_list + + +def index_of_first_match(seq, predicate, start=0, end=None): + if end is None or end >= len(seq): + end = len(seq) + for i in range(start, end): + if predicate(seq[i]): + return i + return None + + +def argmax(seq, key=lambda x: x, start=0, end=None): + seq = seq[start:end] + if len(seq) == 0: + return None + return seq.index(max(seq, key=key)) + start + + +def source_code_location(event): + while event is not None: + match = re.search(r"\.py\(.*\)", event.name) + if match is None: + event = event.parent + continue + return event.name + return "No source code location found" + + +# Provide an OSS workaround for cudagraphs + CUPTI issue +# https://github.com/pytorch/pytorch/issues/75504 +# TODO(dberard) - deprecate / remove workaround for CUDA >= 12, when +# we stop supporting older CUDA versions. +def _init_for_cuda_graphs() -> None: + from torch.autograd.profiler import profile + + with profile(): + pass + + +@dataclass +class TimelineEvent: + """Represents an event in the profiler timeline.""" + + timestamp: int + event_type: Literal["start", "end", "regular"] + marker_type: Literal["filename", "node"] | None + identifier: str | int | None + event: dict[str, Any] + + +@dataclass +class ContextStackEntry: + """Represents a context (filename or node) in the stack.""" + + context_type: Literal["filename", "node"] + identifier: str | int + metadata: dict | None + tid: int | None = None # Thread ID associated with this context + + +def map_recorded_events_to_aten_ops_with_stack_trace(traced_data): + """ + Maps recorded profiler events to their corresponding fx nodes and adds stack traces. + + Builds a timeline of all events (regular ops and FX markers for filenames/nodes), + sorts by timestamp, then processes chronologically while maintaining a context stack of active + filename/node scopes. Regular events are augmented with stack traces and node names from the + innermost active context. Runtime is O(n log n) for n events. + + Args: + traced_data: Json of profiler events from Chrome trace + + Returns: + Dict mapping recorded event names to their aten operations with added stack traces + """ + from torch.fx.traceback import _FX_METADATA_REGISTRY + + trace_events = traced_data.get("traceEvents", []) + + # Create event timeline + event_timeline: list[TimelineEvent] = [] + + def is_fx_marker_event(event): + return ( + event.get("cat") == "cpu_op" + and event.get("name", "").startswith("## ") + and event.get("name", "").endswith(" ##") + ) + + def append_fx_marker_event(event_type, identifier, event): + start_ts = event["ts"] + end_ts = start_ts + event["dur"] + event_timeline.append( + TimelineEvent(start_ts, "start", event_type, identifier, event) + ) + event_timeline.append( + TimelineEvent(end_ts, "end", event_type, identifier, event) + ) + + for event in trace_events: + if "ts" not in event or "dur" not in event: + continue + + if is_fx_marker_event(event): + content = event["name"][3:-3] + + if content.endswith(".py"): + append_fx_marker_event("filename", content, event) + else: + try: + node_index = int(content) + except ValueError: + pass + append_fx_marker_event("node", node_index, event) # type: ignore[possibly-undefined] + + else: + # Regular event that needs augmentation + start_ts = event["ts"] + event_timeline.append(TimelineEvent(start_ts, "regular", None, None, event)) + + # Sort by timestamp + event_timeline.sort(key=lambda x: x.timestamp) + + # Process events in chronological order with a stack + context_stack: list[ContextStackEntry] = [] + + # Invariant: all start event has a corresponding end event + for timeline_event in event_timeline: + match timeline_event.event_type: + case "start": + assert timeline_event.identifier is not None + + if timeline_event.marker_type == "filename": + assert isinstance(timeline_event.identifier, str) + # Push filename context - query metadata registry on-demand + metadata = _FX_METADATA_REGISTRY.get(timeline_event.identifier) + tid = timeline_event.event.get("tid") + context_stack.append( + ContextStackEntry( + "filename", timeline_event.identifier, metadata, tid + ) + ) + elif timeline_event.marker_type == "node": + # Find the current filename from stack + current_file_metadata = None + tid = timeline_event.event.get("tid") + for ctx_entry in reversed(context_stack): + if ( + ctx_entry.context_type == "filename" + and ctx_entry.tid == tid + ): + current_file_metadata = ctx_entry.metadata + break + + if current_file_metadata: + node_metadata = current_file_metadata.get("node_metadata", {}) + if timeline_event.identifier in node_metadata: + node_meta: dict | None = node_metadata[ + timeline_event.identifier + ] + context_stack.append( + ContextStackEntry( + "node", timeline_event.identifier, node_meta, tid + ) + ) + + case "end": + # Pop from stack - search backwards to find matching context + for i in range(len(context_stack) - 1, -1, -1): + ctx_entry = context_stack[i] + if ( + timeline_event.marker_type == ctx_entry.context_type + and timeline_event.identifier == ctx_entry.identifier + ): + context_stack.pop(i) + break + + case "regular": + # Apply metadata from current context stack + # Find the most specific context (node takes precedence over filename) + # Only augment events with the same tid as the file/node event matched + current_stack_trace = None + current_node_name = None + event_tid = timeline_event.event.get("tid") + + for ctx_entry in reversed(context_stack): + # Only apply metadata from contexts with matching tid + if ctx_entry.tid == event_tid: + if ctx_entry.context_type == "node" and ctx_entry.metadata: + current_stack_trace = ctx_entry.metadata.get( + "stack_trace", "No model stack trace available" + ) + current_node_name = ctx_entry.metadata.get("name", "") + # Do we want to only attach the stack trace of the lowest node or stack trace of all nodes + # if nodes are nested, e.g. in nested graph modules + break + + # Augment the event + if current_stack_trace or current_node_name: + args = timeline_event.event.setdefault("args", {}) + if current_stack_trace: + args["stack_trace"] = current_stack_trace + if current_node_name: + args["node_name"] = current_node_name diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/itt.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/itt.py new file mode 100644 index 0000000000000000000000000000000000000000..7b1a6eac0f0bc8d69988fe59f7acae26302572c6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/itt.py @@ -0,0 +1,81 @@ +# mypy: allow-untyped-defs +from contextlib import contextmanager +from typing import NoReturn + + +try: + from torch._C import _itt +except ImportError: + + class _ITTStub: + @staticmethod + def _fail(*args, **kwargs) -> NoReturn: + raise RuntimeError( + "ITT functions not installed. Are you sure you have a ITT build?" + ) + + @staticmethod + def is_available() -> bool: + return False + + rangePush = _fail + rangePop = _fail + mark = _fail + + _itt = _ITTStub() # type: ignore[assignment] + + +__all__ = ["is_available", "range_push", "range_pop", "mark", "range"] + + +def is_available(): + """ + Check if ITT feature is available or not + """ + return _itt.is_available() + + +def range_push(msg): + """ + Pushes a range onto a stack of nested range span. Returns zero-based + depth of the range that is started. + + Arguments: + msg (str): ASCII message to associate with range + """ + return _itt.rangePush(msg) + + +def range_pop(): + """ + Pops a range off of a stack of nested range spans. Returns the + zero-based depth of the range that is ended. + """ + return _itt.rangePop() + + +def mark(msg): + """ + Describe an instantaneous event that occurred at some point. + + Arguments: + msg (str): ASCII message to associate with the event. + """ + return _itt.mark(msg) + + +@contextmanager +def range(msg, *args, **kwargs): + """ + Context manager / decorator that pushes an ITT range at the beginning + of its scope, and pops it at the end. If extra arguments are given, + they are passed as arguments to msg.format(). + + Args: + msg (str): message to associate with the range + """ + range_push(msg.format(*args, **kwargs)) + try: + yield + finally: + range_pop() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/profiler.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..20e2c421efb4d337a218078d7c6121edc6575b3c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/profiler.py @@ -0,0 +1,1178 @@ +# mypy: allow-untyped-defs +import gzip +import json +import os +import shutil +import tempfile +from abc import ABC, abstractmethod +from collections.abc import Callable, Iterable +from enum import Enum +from functools import partial +from typing import Any, Optional +from typing_extensions import deprecated, Self +from warnings import warn + +import torch +import torch.autograd.profiler as prof +from torch._C import _get_privateuse1_backend_name +from torch._C._profiler import ( + _add_execution_trace_observer, + _disable_execution_trace_observer, + _enable_execution_trace_observer, + _ExperimentalConfig, + _remove_execution_trace_observer, +) +from torch._environment import is_fbcode +from torch._utils_internal import profiler_allow_cudagraph_cupti_lazy_reinit_cuda12 +from torch.autograd import kineto_available, ProfilerActivity +from torch.profiler._memory_profiler import MemoryProfile, MemoryProfileTimeline + + +__all__ = [ + "supported_activities", + "ProfilerAction", + "schedule", + "tensorboard_trace_handler", + "profile", + "ExecutionTraceObserver", +] +PROFILER_STEP_NAME = "ProfilerStep" + +_WARNINGS_SHOWN = set() + + +def _warn_once(msg, category=UserWarning, stacklevel=2): + if msg not in _WARNINGS_SHOWN: + _WARNINGS_SHOWN.add(msg) + warn(msg, category=category, stacklevel=stacklevel) + + +class _NumpyEncoder(json.JSONEncoder): + """ + Json encoder for numpy types (np.int, np.float, np.array etc.) + Returns default encoder if numpy is not available + """ + + def default(self, obj): + """Encode NumPy types to JSON""" + try: + import numpy as np + except ImportError: + return json.JSONEncoder.default(self, obj) + if isinstance(obj, np.integer): + return int(obj) + elif isinstance(obj, np.floating): + return float(obj) + elif isinstance(obj, np.ndarray): + return obj.tolist() + else: + return json.JSONEncoder.default(self, obj) + + +def supported_activities(): + """ + Returns a set of supported profiler tracing activities. + + Note: profiler uses CUPTI library to trace on-device CUDA kernels. + In case when CUDA is enabled but CUPTI is not available, passing + ``ProfilerActivity.CUDA`` to profiler results in using the legacy CUDA + profiling code (same as in the legacy ``torch.autograd.profiler``). + This, in turn, results in including CUDA time in the profiler table output, + but not in the JSON trace. + """ + return torch.autograd._supported_activities() + + +class _ITraceObserver(ABC): + """Abstract interface for a Trace observer. + This satisfies 3 methods: start, stop and cleanup""" + + @abstractmethod + def start(self): + pass + + @abstractmethod + def stop(self): + pass + + @abstractmethod + def cleanup(self): + pass + + +class _KinetoProfile: + """Low-level profiler wrap the autograd profile + + Args: + activities (iterable): list of activity groups (CPU, CUDA) to use in profiling, supported values: + ``torch.profiler.ProfilerActivity.CPU``, ``torch.profiler.ProfilerActivity.CUDA``, + ``torch.profiler.ProfilerActivity.XPU``. + Default value: ProfilerActivity.CPU and (when available) ProfilerActivity.CUDA + or (when available) ProfilerActivity.XPU. + record_shapes (bool): save information about operator's input shapes. + profile_memory (bool): track tensor memory allocation/deallocation (see ``export_memory_timeline`` + for more details). + with_stack (bool): record source information (file and line number) for the ops. + with_flops (bool): use formula to estimate the FLOPS of specific operators + (matrix multiplication and 2D convolution). + with_modules (bool): record module hierarchy (including function names) + corresponding to the callstack of the op. e.g. If module A's forward call's + module B's forward which contains an aten::add op, + then aten::add's module hierarchy is A.B + Note that this support exist, at the moment, only for TorchScript models + and not eager mode models. + experimental_config (_ExperimentalConfig) : A set of experimental options + used by profiler libraries like Kineto. Note, backward compatibility is not guaranteed. + execution_trace_observer (ExecutionTraceObserver) : A PyTorch Execution Trace Observer object. + `PyTorch Execution Traces `__ offer a graph based + representation of AI/ML workloads and enable replay benchmarks, simulators, and emulators. + When this argument is included the observer start() and stop() will be called for the + same time window as PyTorch profiler. + acc_events (bool): Enable the accumulation of FunctionEvents across multiple profiling cycles + + + .. note:: + This API is experimental and subject to change in the future. + + Enabling shape and stack tracing results in additional overhead. + When record_shapes=True is specified, profiler will temporarily hold references to the tensors; + that may further prevent certain optimizations that depend on the reference count and introduce + extra tensor copies. + """ + + def __init__( + self, + *, + activities: Iterable[ProfilerActivity] | None = None, + record_shapes: bool = False, + profile_memory: bool = False, + with_stack: bool = False, + with_flops: bool = False, + with_modules: bool = False, + experimental_config: _ExperimentalConfig | None = None, + execution_trace_observer: _ITraceObserver | None = None, + acc_events: bool = False, + custom_trace_id_callback: Callable[[], str] | None = None, + ) -> None: + self.activities = set(activities) if activities else supported_activities() + self.record_shapes = record_shapes + self.with_flops = with_flops + self.profile_memory = profile_memory + self.with_stack = with_stack + self.with_modules = with_modules + self.experimental_config = experimental_config + self.execution_trace_observer = execution_trace_observer + self.acc_events = acc_events + self.custom_trace_id_callback = custom_trace_id_callback + self.profiler: prof.profile | None = None + self.has_cudagraphs = False + self.mem_tl: MemoryProfileTimeline | None = None + self.use_device = None + if ProfilerActivity.CUDA in self.activities: + # pyrefly: ignore [bad-assignment] + self.use_device = "cuda" + elif ProfilerActivity.XPU in self.activities: + # pyrefly: ignore [bad-assignment] + self.use_device = "xpu" + elif ProfilerActivity.MTIA in self.activities: + # pyrefly: ignore [bad-assignment] + self.use_device = "mtia" + elif ProfilerActivity.HPU in self.activities: + # pyrefly: ignore [bad-assignment] + self.use_device = "hpu" + elif ProfilerActivity.PrivateUse1 in self.activities: + # pyrefly: ignore [bad-assignment] + self.use_device = _get_privateuse1_backend_name() + + # user-defined metadata to be amended to the trace + self.preset_metadata: dict[str, str] = {} + + def start(self) -> None: + self.prepare_trace() + self.start_trace() + + def stop(self) -> None: + self.stop_trace() + + def prepare_trace(self) -> None: + if hasattr(torch, "_inductor"): + import torch._inductor.config as inductor_config + + self.has_cudagraphs = inductor_config.triton.cudagraphs + if (self.profiler is None) or (not self.acc_events): + self.profiler = prof.profile( + use_cpu=(ProfilerActivity.CPU in self.activities), + use_device=self.use_device, + record_shapes=self.record_shapes, + with_flops=self.with_flops, + profile_memory=self.profile_memory, + with_stack=self.with_stack, + with_modules=self.with_modules, + use_kineto=True, + experimental_config=self.experimental_config, + acc_events=self.acc_events, + custom_trace_id_callback=self.custom_trace_id_callback, + ) + if (self.profiler is not None) and (not self.acc_events): + _warn_once( + "Warning: Profiler clears events at the end of each cycle." + "Only events from the current cycle will be reported." + "To keep events across cycles, set acc_events=True." + ) + self.profiler._prepare_trace() + + def start_trace(self) -> None: + if self.execution_trace_observer: + self.execution_trace_observer.start() + if self.profiler is None: + raise AssertionError("Profiler must be initialized before starting trace") + self.profiler._start_trace() + + if self.profile_memory: + self.add_metadata_json("profile_memory", "1") + if self.with_stack: + self.add_metadata_json("with_stack", "1") + if self.record_shapes: + self.add_metadata_json("record_shapes", "1") + if self.with_modules: + self.add_metadata_json("with_modules", "1") + if self.with_flops: + self.add_metadata_json("with_flops", "1") + + if kineto_available(): + dist_info = self._get_distributed_info() + if dist_info: + self.add_metadata_json( + "distributedInfo", json.dumps(dist_info, cls=_NumpyEncoder) + ) + + cuda_version = None + if hasattr(torch, "version"): + from torch.torch_version import TorchVersion + + cuda_version = TorchVersion(getattr(torch.version, "cuda", "0.0")) + + if self.has_cudagraphs and ( + (cuda_version and cuda_version < "12.6") + or not profiler_allow_cudagraph_cupti_lazy_reinit_cuda12() + ): + os.environ["DISABLE_CUPTI_LAZY_REINIT"] = "1" + self.add_metadata_json("DISABLE_CUPTI_LAZY_REINIT", "1") + # FIXME: CUDA Graph does not work well with CUPTI teardown. + # 1) crashes on 1st lazy CUPTI re-init after teardown (CUDA 11) + # 2) crashes on 2nd non-lazy CUPTI re-init after teardown (CUDA 12) + # Workaround: turn off CUPTI teardown when using CUDA Graphs. + os.environ["TEARDOWN_CUPTI"] = "0" + + # Insert the preset user metadata to the trace + for k, v in self.preset_metadata.items(): + self.add_metadata_json(k, v) + + def stop_trace(self) -> None: + if self.execution_trace_observer: + self.execution_trace_observer.stop() + if self.profiler is None: + raise AssertionError("Profiler must be initialized before stopping trace") + self.profiler.__exit__(None, None, None) + + def export_chrome_trace(self, path: str): + """ + Exports the collected trace in Chrome JSON format. If kineto is enabled, only + last cycle in schedule is exported. + """ + if self.profiler is None: + raise AssertionError( + "Profiler must be initialized before exporting chrome trace" + ) + if path.endswith(".gz"): + with tempfile.NamedTemporaryFile("w+b", suffix=".json") as fp: + retvalue = self.profiler.export_chrome_trace(fp.name) + with open(fp.name, "rb") as fin, gzip.open(path, "wb") as fout: + fout.writelines(fin) + return retvalue + else: + return self.profiler.export_chrome_trace(path) + + def export_stacks(self, path: str, metric: str = "self_cpu_time_total"): + """Save stack traces to a file + + Args: + path (str): save stacks file to this location; + metric (str): metric to use: "self_cpu_time_total" or "self_cuda_time_total" + """ + if self.profiler is None: + raise AssertionError("Profiler must be initialized before exporting stacks") + return self.profiler.export_stacks(path, metric) + + def toggle_collection_dynamic( + self, enable: bool, activities: Iterable[ProfilerActivity] + ) -> None: + """Toggle collection of activities on/off at any point of collection. Currently supports toggling Torch Ops + (CPU) and CUDA activity supported in Kineto + + Args: + activities (iterable): list of activity groups to use in profiling, supported values: + ``torch.profiler.ProfilerActivity.CPU``, ``torch.profiler.ProfilerActivity.CUDA`` + Examples: + + .. code-block:: python + + with torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ] + ) as p: + code_to_profile_0() + // turn off collection of all CUDA activity + p.toggle_collection_dynamic(False, [torch.profiler.ProfilerActivity.CUDA]) + code_to_profile_1() + // turn on collection of all CUDA activity + p.toggle_collection_dynamic(True, [torch.profiler.ProfilerActivity.CUDA]) + code_to_profile_2() + print(p.key_averages().table( + sort_by="self_cuda_time_total", row_limit=-1)) + """ + if self.profiler is None: + return + self.profiler.toggle_collection_dynamic(enable, activities) + + def key_averages( + self, + group_by_input_shape: bool = False, + group_by_stack_n: int = 0, + group_by_overload_name: bool = False, + ): + """Averages events, grouping them by operator name and (optionally) input shapes, stack + and overload name. + + .. note:: + To use shape/stack functionality make sure to set record_shapes/with_stack + when creating profiler context manager. + """ + if self.profiler is None: + raise AssertionError( + "Profiler must be initialized before getting key averages" + ) + return self.profiler.key_averages( + group_by_input_shape, group_by_stack_n, group_by_overload_name + ) + + def events(self): + """ + Returns the list of unaggregated profiler events, + to be used in the trace callback or after the profiling is finished + """ + if self.profiler is None: + raise AssertionError("Profiler must be initialized before accessing events") + return self.profiler.function_events + + def add_metadata(self, key: str, value: str) -> None: + """ + Adds a user defined metadata with a string key and a string value + into the trace file + """ + wrapped_value = '"' + value.replace('"', '\\"') + '"' + torch.autograd._add_metadata_json(key, wrapped_value) + + def add_metadata_json(self, key: str, value: str) -> None: + """ + Adds a user defined metadata with a string key and a valid json value + into the trace file + """ + torch.autograd._add_metadata_json(key, value) + + def preset_metadata_json(self, key: str, value: str) -> None: + """ + Preset a user defined metadata when the profiler is not started + and added into the trace file later. + Metadata is in the format of a string key and a valid json value + """ + self.preset_metadata[key] = value + + def _get_distributed_info(self): + import torch.distributed as dist + + if not dist.is_available() or not dist.is_initialized(): + return None + + backend = dist.get_backend() + dist_info = { + "backend": backend, + "rank": dist.get_rank(), + "world_size": dist.get_world_size(), + "pg_count": dist.get_pg_count(), + "pg_config": dist.distributed_c10d._get_all_pg_configs(), + } + if backend == "nccl": + nccl_version = torch.cuda.nccl.version() + # pyrefly: ignore [unsupported-operation] + dist_info["nccl_version"] = ".".join(str(v) for v in nccl_version) + return dist_info + + def _memory_profile(self) -> MemoryProfile: + required = ("record_shapes", "profile_memory", "with_stack") + missing = [f"{i}=True" for i in required if not getattr(self, i)] + if missing: + raise ValueError(f"{', '.join(missing)} required for memory profiling.") + + if self.profiler is None or self.profiler.kineto_results is None: + raise AssertionError( + "Profiler and kineto_results must be initialized for memory profiling" + ) + return MemoryProfile(self.profiler.kineto_results) + + @deprecated( + "`export_memory_timeline` is deprecated and will be removed in a future version. " + "Please use `torch.cuda.memory._record_memory_history` and `torch.cuda.memory._export_memory_snapshot` instead.", + category=FutureWarning, + ) + def export_memory_timeline(self, path: str, device: str | None = None) -> None: + """Export memory event information from the profiler collected + tree for a given device, and export a timeline plot. There are 3 + exportable files using ``export_memory_timeline``, each controlled by the + ``path``'s suffix. + + - For an HTML compatible plot, use the suffix ``.html``, and a memory timeline + plot will be embedded as a PNG file in the HTML file. + + - For plot points consisting of ``[times, [sizes by category]]``, where + ``times`` are timestamps and ``sizes`` are memory usage for each category. + The memory timeline plot will be saved a JSON (``.json``) or gzipped JSON + (``.json.gz``) depending on the suffix. + + - For raw memory points, use the suffix ``.raw.json.gz``. Each raw memory + event will consist of ``(timestamp, action, numbytes, category)``, where + ``action`` is one of ``[PREEXISTING, CREATE, INCREMENT_VERSION, DESTROY]``, + and ``category`` is one of the enums from + ``torch.profiler._memory_profiler.Category``. + + Output: Memory timeline written as gzipped JSON, JSON, or HTML. + + .. deprecated:: + ``export_memory_timeline`` is deprecated and will be removed in a future version. + Please use ``torch.cuda.memory._record_memory_history`` and + ``torch.cuda.memory._export_memory_snapshot`` instead. + """ + # Default to device 0, if unset. Fallback on cpu. + if device is None: + if self.use_device and self.use_device != "cuda": + device = self.use_device + ":0" + else: + device = "cuda:0" if torch.cuda.is_available() else "cpu" + + # Construct the memory timeline plot data + self.mem_tl = MemoryProfileTimeline(self._memory_profile()) + + # Depending on the file suffix, save the data as json.gz or json. + # For html, we can embed the image into an HTML file. + if path.endswith(".html"): + self.mem_tl.export_memory_timeline_html(path, device) + elif path.endswith(".gz"): + with tempfile.NamedTemporaryFile("w+t", suffix=".json") as fp: + if path.endswith("raw.json.gz"): + self.mem_tl.export_memory_timeline_raw(fp.name, device) + else: + self.mem_tl.export_memory_timeline(fp.name, device) + with open(fp.name) as fin, gzip.open(path, "wt") as fout: + fout.writelines(fin) + else: + self.mem_tl.export_memory_timeline(path, device) + + +class ProfilerAction(Enum): + """ + Profiler actions that can be taken at the specified intervals + """ + + NONE = 0 + WARMUP = 1 + RECORD = 2 + RECORD_AND_SAVE = 3 + + +def schedule( + *, + wait: int, + warmup: int, + active: int, + repeat: int = 0, + skip_first: int = 0, + skip_first_wait: int = 0, +) -> Callable: + """ + Returns a callable that can be used as profiler ``schedule`` argument. The profiler will skip + the first ``skip_first`` steps, then wait for ``wait`` steps, then do the warmup for the next ``warmup`` steps, + then do the active recording for the next ``active`` steps and then repeat the cycle starting with ``wait`` steps. + The optional number of cycles is specified with the ``repeat`` parameter, the zero value means that + the cycles will continue until the profiling is finished. + + The ``skip_first_wait`` parameter controls whether the first ``wait`` stage should be skipped. + This can be useful if a user wants to wait longer than ``skip_first`` between cycles, but not + for the first profile. For example, if ``skip_first`` is 10 and ``wait`` is 20, the first cycle will + wait 10 + 20 = 30 steps before warmup if ``skip_first_wait`` is zero, but will wait only 10 + steps if ``skip_first_wait`` is non-zero. All subsequent cycles will then wait 20 steps between the + last active and warmup. + """ + + def schedule_fn(step: int) -> ProfilerAction: + if step < 0: + raise AssertionError(f"Step must be non-negative. Got {step}.") + if step < skip_first: + return ProfilerAction.NONE + else: + step -= skip_first + # If wait >> skip_first and we want to grab profiling early, shift left by wait if skip_first_wait is True + if skip_first_wait != 0: + step += wait + num_steps = wait + warmup + active + if repeat > 0 and step / num_steps >= repeat: + return ProfilerAction.NONE + mod_step = step % num_steps + if mod_step < wait: + return ProfilerAction.NONE + elif mod_step < wait + warmup: + return ProfilerAction.WARMUP + else: + return ( + ProfilerAction.RECORD + if mod_step < num_steps - 1 + else ProfilerAction.RECORD_AND_SAVE + ) + + if wait < 0 or warmup < 0 or active <= 0 or repeat < 0 or skip_first < 0: + raise AssertionError( + f"Invalid profiler schedule arguments. Got wait={wait} (need >= 0), warmup={warmup} (need >= 0), " + f"active={active} (need > 0), repeat={repeat} (need >= 0), skip_first={skip_first} (need >= 0)." + ) + if warmup == 0: + warn( + "Profiler won't be using warmup, this can skew profiler results", + stacklevel=2, + ) + return schedule_fn + + +def _default_schedule_fn(_: int) -> ProfilerAction: + """ + Default profiler behavior - immediately starts recording the events, + keeps doing it on every profiler step. + """ + return ProfilerAction.RECORD + + +def tensorboard_trace_handler( + dir_name: str, worker_name: str | None = None, use_gzip: bool = False +): + """ + Outputs tracing files to directory of ``dir_name``, then that directory can be + directly delivered to tensorboard as logdir. + ``worker_name`` should be unique for each worker in distributed scenario, + it will be set to '[hostname]_[pid]' by default. + """ + import socket + import time + + def handler_fn(prof) -> None: + nonlocal worker_name + if not os.path.isdir(dir_name): + try: + os.makedirs(dir_name, exist_ok=True) + except Exception as e: + raise RuntimeError("Can't create directory: " + dir_name) from e + if not worker_name: + worker_name = f"{socket.gethostname()}_{os.getpid()}" + # Use nanosecond here to avoid naming clash when exporting the trace + file_name = f"{worker_name}.{time.time_ns()}.pt.trace.json" + if use_gzip: + file_name = file_name + ".gz" + prof.export_chrome_trace(os.path.join(dir_name, file_name)) + + return handler_fn + + +class profile(_KinetoProfile): + """Profiler context manager. + + Args: + activities (iterable): list of activity groups (CPU, CUDA) to use in profiling, supported values: + ``torch.profiler.ProfilerActivity.CPU``, ``torch.profiler.ProfilerActivity.CUDA``, + ``torch.profiler.ProfilerActivity.XPU``. + Default value: ProfilerActivity.CPU and (when available) ProfilerActivity.CUDA + or (when available) ProfilerActivity.XPU. + schedule (Callable): callable that takes step (int) as a single parameter and returns + ``ProfilerAction`` value that specifies the profiler action to perform at each step. + on_trace_ready (Callable): callable that is called at each step when ``schedule`` + returns ``ProfilerAction.RECORD_AND_SAVE`` during the profiling. + record_shapes (bool): save information about operator's input shapes. + profile_memory (bool): track tensor memory allocation/deallocation. + with_stack (bool): record source information (file and line number) for the ops. + with_flops (bool): use formula to estimate the FLOPs (floating point operations) of specific operators + (matrix multiplication and 2D convolution). + with_modules (bool): record module hierarchy (including function names) + corresponding to the callstack of the op. e.g. If module A's forward call's + module B's forward which contains an aten::add op, + then aten::add's module hierarchy is A.B + Note that this support exist, at the moment, only for TorchScript models + and not eager mode models. + experimental_config (_ExperimentalConfig) : A set of experimental options + used for Kineto library features. Note, backward compatibility is not guaranteed. + execution_trace_observer (ExecutionTraceObserver) : A PyTorch Execution Trace Observer object. + `PyTorch Execution Traces `__ offer a graph based + representation of AI/ML workloads and enable replay benchmarks, simulators, and emulators. + When this argument is included the observer start() and stop() will be called for the + same time window as PyTorch profiler. See the examples section below for a code sample. + acc_events (bool): Enable the accumulation of FunctionEvents across multiple profiling cycles + use_cuda (bool): + .. deprecated:: 1.8.1 + use ``activities`` instead. + + .. note:: + Use :func:`~torch.profiler.schedule` to generate the callable schedule. + Non-default schedules are useful when profiling long training jobs + and allow the user to obtain multiple traces at the different iterations + of the training process. + The default schedule simply records all the events continuously for the + duration of the context manager. + + .. note:: + Use :func:`~torch.profiler.tensorboard_trace_handler` to generate result files for TensorBoard: + + ``on_trace_ready=torch.profiler.tensorboard_trace_handler(dir_name)`` + + After profiling, result files can be found in the specified directory. Use the command: + + ``tensorboard --logdir dir_name`` + + to see the results in TensorBoard. + For more information, see + `PyTorch Profiler TensorBoard Plugin `__ + + .. note:: + Enabling shape and stack tracing results in additional overhead. + When record_shapes=True is specified, profiler will temporarily hold references to the tensors; + that may further prevent certain optimizations that depend on the reference count and introduce + extra tensor copies. + + + Examples: + + .. code-block:: python + + with torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ] + ) as p: + code_to_profile() + print(p.key_averages().table(sort_by="self_cuda_time_total", row_limit=-1)) + + Using the profiler's ``schedule``, ``on_trace_ready`` and ``step`` functions: + + .. code-block:: python + + # Non-default profiler schedule allows user to turn profiler on and off + # on different iterations of the training loop; + # trace_handler is called every time a new trace becomes available + def trace_handler(prof): + print( + prof.key_averages().table(sort_by="self_cuda_time_total", row_limit=-1) + ) + # prof.export_chrome_trace("/tmp/test_trace_" + str(prof.step_num) + ".json") + + + with torch.profiler.profile( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + # In this example with wait=1, warmup=1, active=2, repeat=1, + # profiler will skip the first step/iteration, + # start warming up on the second, record + # the third and the forth iterations, + # after which the trace will become available + # and on_trace_ready (when set) is called; + # the cycle repeats starting with the next step + schedule=torch.profiler.schedule(wait=1, warmup=1, active=2, repeat=1), + on_trace_ready=trace_handler, + # on_trace_ready=torch.profiler.tensorboard_trace_handler('./log') + # used when outputting for tensorboard + ) as p: + for iter in range(N): + code_iteration_to_profile(iter) + # send a signal to the profiler that the next iteration has started + p.step() + + The following sample shows how to setup up an Execution Trace Observer (`execution_trace_observer`) + + .. code-block:: python + + with torch.profiler.profile( + ... + execution_trace_observer=( + ExecutionTraceObserver().register_callback("./execution_trace.json") + ), + ) as p: + for iter in range(N): + code_iteration_to_profile(iter) + p.step() + + You can also refer to test_execution_trace_with_kineto() in tests/profiler/test_profiler.py. + Note: One can also pass any object satisfying the _ITraceObserver interface. + """ + + def __init__( + self, + *, + activities: Iterable[ProfilerActivity] | None = None, + schedule: Callable[[int], ProfilerAction] | None = None, + on_trace_ready: Callable[..., Any] | None = None, + record_shapes: bool = False, + profile_memory: bool = False, + with_stack: bool = False, + with_flops: bool = False, + with_modules: bool = False, + experimental_config: _ExperimentalConfig | None = None, + execution_trace_observer: _ITraceObserver | None = None, + acc_events: bool = False, + # deprecated: + use_cuda: bool | None = None, + custom_trace_id_callback: Callable[[], str] | None = None, + ) -> None: + activities_set = set(activities) if activities else supported_activities() + if use_cuda is not None: + warn( + "`use_cuda` is deprecated, use `activities` argument instead", + FutureWarning, + stacklevel=2, + ) + if use_cuda: + activities_set.add(ProfilerActivity.CUDA) + elif ProfilerActivity.CUDA in activities_set: + activities_set.remove(ProfilerActivity.CUDA) + if len(activities_set) == 0: + raise AssertionError("No valid profiler activities found") + + super().__init__( + activities=activities, + record_shapes=record_shapes, + profile_memory=profile_memory, + with_stack=with_stack, + with_flops=with_flops, + with_modules=with_modules, + experimental_config=experimental_config, + execution_trace_observer=execution_trace_observer + if execution_trace_observer + else ExecutionTraceObserver.build_execution_trace_obs_from_env(), + acc_events=acc_events, + custom_trace_id_callback=custom_trace_id_callback, + ) + + if schedule: + self.schedule = schedule + # add step markers into the trace and table view + self.record_steps = True + else: + self.schedule = _default_schedule_fn + self.record_steps = False + self.on_trace_ready = on_trace_ready + self.step_num = 0 + self.current_action = self.schedule(self.step_num) + self.step_rec_fn: prof.record_function | None = None + + self.action_map: dict[ + tuple[ProfilerAction, ProfilerAction | None], list[Any] + ] = { + # key is (prev_action, current_action), value is action list corresponding to the state pair. + (ProfilerAction.NONE, ProfilerAction.NONE): [], + (ProfilerAction.NONE, ProfilerAction.WARMUP): [self.prepare_trace], + (ProfilerAction.NONE, ProfilerAction.RECORD): [ + self.prepare_trace, + self.start_trace, + ], + (ProfilerAction.NONE, ProfilerAction.RECORD_AND_SAVE): [ + self.prepare_trace, + self.start_trace, + ], + (ProfilerAction.WARMUP, ProfilerAction.NONE): [ + partial(warn, "Incorrect schedule: WARMUP followed by NONE"), + self.start_trace, + self.stop_trace, + ], + (ProfilerAction.WARMUP, ProfilerAction.WARMUP): [], + (ProfilerAction.WARMUP, ProfilerAction.RECORD): [self.start_trace], + (ProfilerAction.WARMUP, ProfilerAction.RECORD_AND_SAVE): [self.start_trace], + (ProfilerAction.RECORD, ProfilerAction.NONE): [ + partial(warn, "Incorrect schedule: RECORD followed by NONE"), + self.stop_trace, + ], + (ProfilerAction.RECORD, ProfilerAction.WARMUP): [ + partial(warn, "Incorrect schedule: RECORD followed by WARMUP"), + self.stop_trace, + ], + (ProfilerAction.RECORD, ProfilerAction.RECORD): [], + (ProfilerAction.RECORD, ProfilerAction.RECORD_AND_SAVE): [], + (ProfilerAction.RECORD_AND_SAVE, ProfilerAction.NONE): [ + self.stop_trace, + self._trace_ready, + ], + (ProfilerAction.RECORD_AND_SAVE, ProfilerAction.WARMUP): [ + self.stop_trace, + self._trace_ready, + self.prepare_trace, + ], + (ProfilerAction.RECORD_AND_SAVE, ProfilerAction.RECORD): [ + self.stop_trace, + self._trace_ready, + self.prepare_trace, + self.start_trace, + ], + (ProfilerAction.RECORD_AND_SAVE, ProfilerAction.RECORD_AND_SAVE): [ + self.stop_trace, + self._trace_ready, + self.prepare_trace, + self.start_trace, + ], + # used for exit action + (ProfilerAction.WARMUP, None): [self.start_trace, self.stop_trace], + (ProfilerAction.RECORD, None): [self.stop_trace, self._trace_ready], + (ProfilerAction.RECORD_AND_SAVE, None): [ + self.stop_trace, + self._trace_ready, + ], + } + # Start tracking increments to profiler step, this will be used + # by Kineto + prof.KinetoStepTracker.init_step_count(PROFILER_STEP_NAME) + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop() + prof.KinetoStepTracker.erase_step_count(PROFILER_STEP_NAME) + if self.execution_trace_observer: + self.execution_trace_observer.cleanup() + + def start(self) -> None: + self._transit_action(ProfilerAction.NONE, self.current_action) + if self.record_steps: + self.step_rec_fn = prof.record_function( + "ProfilerStep#" + str(self.step_num) + ) + self.step_rec_fn.__enter__() + + def stop(self) -> None: + if self.record_steps and self.step_rec_fn: + self.step_rec_fn.__exit__(None, None, None) + self._transit_action(self.current_action, None) + + def step(self) -> None: + """ + Signals the profiler that the next profiling step has started. + """ + if self.record_steps and self.step_rec_fn: + self.step_rec_fn.__exit__(None, None, None) + prev_action = self.current_action + self.step_num += 1 + self.current_action = self.schedule(self.step_num) + + self._transit_action(prev_action, self.current_action) + if os.environ.get("KINETO_USE_DAEMON", "") or ( + is_fbcode() and os.environ.get("KINETO_FORCE_STEP_HOOK", "") + ): + prof.KinetoStepTracker.increment_step(PROFILER_STEP_NAME) + + if self.record_steps: + self.step_rec_fn = prof.record_function( + "ProfilerStep#" + str(self.step_num) + ) + self.step_rec_fn.__enter__() + + def set_custom_trace_id_callback(self, callback) -> None: + """ + Sets a callback to be called when a new trace ID is generated. + """ + self.custom_trace_id_callback = callback + + def get_trace_id(self): + """ + Returns the current trace ID. + """ + if self.profiler is None: + return None + return self.profiler.trace_id + + def _trace_ready(self) -> None: + if self.on_trace_ready: + self.on_trace_ready(self) + + def _transit_action(self, prev_action, current_action) -> None: + action_list = self.action_map.get((prev_action, current_action)) + if action_list: + for action in action_list: + action() + + def _stats(self) -> prof._ProfilerStats | None: + if self.profiler is None: + return None + return self.profiler._stats + + +class ExecutionTraceObserver(_ITraceObserver): + """Execution Trace Observer + + Each process can have a single ExecutionTraceObserver instance. The observer + can be added to record function callbacks via calling register_callback() + explicitly. Without calling unregister_callback(), repeated calls to + register_callback() will not add additional observers to record function + callbacks. Once an ExecutionTraceObserver is created, the start() and stop() + methods control when the event data is recorded. + + Deleting or calling unregister_callback() will remove the observer from the + record function callbacks, finalize the output file, and will stop + incurring any overheads. + """ + + def __init__(self) -> None: + """ + Initializes the default states. + """ + self._registered = False + self._execution_trace_running = False + self.extra_resources_collection = False + self.resources_dir: str = "" + self.output_file_path: str = "" + self.output_file_path_observer: str = "" + + def __del__(self) -> None: + """ + Calls unregister_callback() to make sure to finalize outputs. + """ + self.unregister_callback() + + @staticmethod + def build_execution_trace_obs_from_env() -> Optional["ExecutionTraceObserver"]: + """ + Returns an ExecutionTraceObserver instance if the environment variable + ENABLE_PYTORCH_EXECUTION_TRACE is set to 1, otherwise returns None. + + Configures the observer to also collect extra resources if the environment variable + ``ENABLE_PYTORCH_EXECUTION_TRACE_EXTRAS=1``. These are resources such as generated kernels, + index tensor data etc. that are required to make the Execution Trace replayable. + """ + if os.environ.get("ENABLE_PYTORCH_EXECUTION_TRACE", "0") == "1": + try: + with tempfile.NamedTemporaryFile( + "w+t", suffix=".et.json", delete=False + ) as fp: + filename = fp.name + except Exception as e: + warn( + f"Execution trace will not be recorded. Exception on creating default temporary file: {e}", + stacklevel=2, + ) + return None + et = ExecutionTraceObserver() + et.register_callback(filename) + # additionally, check if the env requires us to collect extra resources + if os.environ.get("ENABLE_PYTORCH_EXECUTION_TRACE_EXTRAS", "0") == "1": + et.set_extra_resource_collection(True) + else: + et.set_extra_resource_collection(False) + return et + return None + + def set_extra_resource_collection(self, val) -> None: + """ + Collects extra resources such as generated kernels, index tensor data, and any other + metadata that is required to complete the Execution Trace content. + + The caller should call this method with val=True after calling register_callback() if they want + to collect the extra resources. + """ + self.extra_resources_collection = val + if self.extra_resources_collection: + self.get_resources_dir(can_create=True) + return + + def register_callback(self, output_file_path: str) -> Self: + """ + Adds ET observer to record function callbacks. The data will be + written to output_file_path. + """ + + def get_temp_uncompressed_file() -> str: + with tempfile.NamedTemporaryFile("w+b", suffix=".json", delete=False) as fp: + return fp.name + + if not self._registered: + self.output_file_path = output_file_path + if output_file_path.endswith(".gz"): + output_file_path = get_temp_uncompressed_file() + self.output_file_path_observer = output_file_path + self._registered = _add_execution_trace_observer(output_file_path) + return self + + def get_resources_dir(self, can_create=False) -> str | None: + """ + Generates the resources directory for the generated kernels, + or index tensor data or any other metadata that is required + to complete the Execution Trace content. + + The directory is created right where the ET file is being output. + + Only works if the observer has called set_extra_resource_collection(val=True). + + Returns None if the observer is not configured with extra resource collection. + """ + if not self.extra_resources_collection: + return None + if self.resources_dir: + # already created + return self.resources_dir + generated_path = ExecutionTraceObserver.get_resources_dir_for_et_path( + self.output_file_path, create_dir=can_create + ) + if not generated_path: + # could not find of create the resources dir + return None + self.resources_dir = generated_path + return self.resources_dir + + @staticmethod + def get_resources_dir_for_et_path( + trace_path, create_dir: bool = False + ) -> str | None: + work_dir, file_name = os.path.split(trace_path) + resource_dir = os.path.join( + work_dir, os.path.splitext(file_name)[0] + "_resources" + ) + if not os.path.exists(resource_dir): + if create_dir: + try: + os.mkdir(resource_dir) + except Exception: + warn( + f"Execution trace exception when creating {resource_dir}", + stacklevel=2, + ) + return None + else: + return None + return resource_dir + + def unregister_callback(self) -> None: + """ + Removes ET observer from record function callbacks. + """ + + def _save_triton_kernels() -> None: + try: + resource_dir = self.get_resources_dir() + except Exception as e: + warn( + f"Execution trace exception when generating resource directory: {e}", + stacklevel=2, + ) + return + if not resource_dir: + return + + # Save the kernel paths for the generated kernels + from torch._inductor.codecache import PyCodeCache + + kernel_files = [ + v.__file__ + for v in PyCodeCache.modules + if getattr(v, "__file__", None) is not None + ] + + for kernel_file in kernel_files: + if kernel_file is None: + continue + name = os.path.basename(kernel_file) + dst = os.path.join(resource_dir, name) + shutil.copyfile(kernel_file, dst) + + def _save_gz_file(uncompressed_file: str, output_file: str) -> None: + print(f"Execution Trace: compressing {uncompressed_file} to {output_file}") + with open(uncompressed_file, "rb") as fin: + with gzip.open(output_file, "wb") as fout: + fout.writelines(fin) + os.remove(uncompressed_file) + + if self._registered: + self.stop() + + try: + _save_triton_kernels() + except Exception as e: + warn(f"Execution trace failed to save kernels: {e}", stacklevel=2) + + _remove_execution_trace_observer() + if self.output_file_path.endswith("gz"): + _save_gz_file(self.output_file_path_observer, self.output_file_path) + + self._registered = False + + @property + def is_registered(self): + """ + Returns True if the execution trace observer is registered, otherwise False. + """ + return self._registered + + def is_running(self): + """ + Returns True if the observer is running, otherwise False. + """ + return self._execution_trace_running + + def start(self) -> None: + """ + Starts to capture. + """ + if self._registered and not self._execution_trace_running: + _enable_execution_trace_observer() + self._execution_trace_running = True + self._record_pg_config() + + def stop(self) -> None: + """ + Stops to capture. + """ + if self._execution_trace_running: + _disable_execution_trace_observer() + self._execution_trace_running = False + + def cleanup(self) -> None: + """ + Calls unregister_callback() to make sure to finalize outputs. + """ + self.unregister_callback() + + def get_output_file_path(self) -> str | None: + """ + Returns the output file name or None. + """ + if self.output_file_path: + return self.output_file_path + else: + return None + + def _record_pg_config(self) -> None: + # Records the PG config info to the trace as node: + # ## process_group:init ## + if ( + self.is_registered + and torch.distributed.is_available() + and torch.distributed.is_initialized() + ): + pg_config_info = torch.distributed.distributed_c10d._world.pg_config_info + torch.autograd._record_function_with_args_enter( + "## process_group:init ##", + json.dumps(pg_config_info, cls=_NumpyEncoder), + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/python_tracer.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/python_tracer.py new file mode 100644 index 0000000000000000000000000000000000000000..aff0fbc32ff3a8870ce81f569daa2587f598394f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/profiler/python_tracer.py @@ -0,0 +1,19 @@ +import os +import site +import sys + +import torch + + +def _prefix_regex() -> list[str]: + raw_paths = ( + site.getsitepackages() + + sys.path + + [site.getuserbase()] + + [site.getusersitepackages()] + + [os.path.dirname(os.path.dirname(torch.__file__))] + ) + + path_prefixes = sorted({os.path.abspath(i) for i in raw_paths}, reverse=True) + assert all(isinstance(i, str) for i in path_prefixes) + return [i + os.sep for i in path_prefixes] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..11114de4313869c02c7df7a7b67a5df0e17adff9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/__init__.py @@ -0,0 +1,86 @@ +# mypy: allow-untyped-defs +from .fake_quantize import * # noqa: F403 +from .fuse_modules import fuse_modules +from .fuser_method_mappings import * # noqa: F403 +from .observer import * # noqa: F403 +from .qconfig import * # noqa: F403 +from .quant_type import * # noqa: F403 +from .quantization_mappings import * # noqa: F403 +from .quantize import * # noqa: F403 +from .quantize_jit import * # noqa: F403 +from .stubs import * # noqa: F403 + + +def default_eval_fn(model, calib_data): + r""" + Default evaluation function takes a torch.utils.data.Dataset or a list of + input Tensors and run the model on the dataset + """ + for data, _target in calib_data: + model(data) + + +__all__ = [ + "QuantWrapper", + "QuantStub", + "DeQuantStub", + # Top level API for eager mode quantization + "quantize", + "quantize_dynamic", + "quantize_qat", + "prepare", + "convert", + "prepare_qat", + # Top level API for graph mode quantization on TorchScript + "quantize_jit", + "quantize_dynamic_jit", + "_prepare_ondevice_dynamic_jit", + "_convert_ondevice_dynamic_jit", + "_quantize_ondevice_dynamic_jit", + # Top level API for graph mode quantization on GraphModule(torch.fx) + # 'fuse_fx', 'quantize_fx', # TODO: add quantize_dynamic_fx + # 'prepare_fx', 'prepare_dynamic_fx', 'convert_fx', + "QuantType", # quantization type + # custom module APIs + "get_default_static_quant_module_mappings", + "get_static_quant_module_class", + "get_default_dynamic_quant_module_mappings", + "get_default_qat_module_mappings", + "get_default_qconfig_propagation_list", + "get_default_compare_output_module_list", + "get_quantized_operator", + "get_fuser_method", + # Sub functions for `prepare` and `swap_module` + "propagate_qconfig_", + "add_quant_dequant", + "swap_module", + "default_eval_fn", + # Observers + "ObserverBase", + "WeightObserver", + "HistogramObserver", + "observer", + "default_observer", + "default_weight_observer", + "default_placeholder_observer", + "default_per_channel_weight_observer", + # FakeQuantize (for qat) + "default_fake_quant", + "default_weight_fake_quant", + "default_fixed_qparams_range_neg1to1_fake_quant", + "default_fixed_qparams_range_0to1_fake_quant", + "default_per_channel_weight_fake_quant", + "default_histogram_fake_quant", + # QConfig + "QConfig", + "default_qconfig", + "default_dynamic_qconfig", + "float16_dynamic_qconfig", + "float_qparams_weight_only_qconfig", + # QAT utilities + "default_qat_qconfig", + "prepare_qat", + "quantize_qat", + # module transformations + "fuse_modules", +] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/_numeric_suite.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/_numeric_suite.py new file mode 100644 index 0000000000000000000000000000000000000000..49ccc8e69523f7dbee2335b788a2cb3a7db618a2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/_numeric_suite.py @@ -0,0 +1,28 @@ +# flake8: noqa: F401 +r""" +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/ns/_numeric_suite.py`, while adding an import statement +here. +""" + +from torch.ao.ns._numeric_suite import ( + _convert_tuple_to_list, + _dequantize_tensor_list, + _find_match, + _get_logger_dict_helper, + _is_identical_module_type, + compare_model_outputs, + compare_model_stub, + compare_weights, + get_logger_dict, + get_matching_activations, + Logger, + NON_LEAF_MODULE_TO_ADD_OBSERVER_ALLOW_LIST, + OutputLogger, + prepare_model_outputs, + prepare_model_with_stubs, + Shadow, + ShadowLogger, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/_numeric_suite_fx.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/_numeric_suite_fx.py new file mode 100644 index 0000000000000000000000000000000000000000..55cd7085740d0ce8de79491acbfc4888ebba21f8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/_numeric_suite_fx.py @@ -0,0 +1,26 @@ +# flake8: noqa: F401 +r""" +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/ns/_numeric_suite_fx.py`, while adding an import statement +here. +""" + +from torch.ao.ns._numeric_suite_fx import ( + _add_loggers_impl, + _add_loggers_one_model, + _add_shadow_loggers_impl, + _extract_logger_info_one_model, + _extract_weights_impl, + _extract_weights_one_model, + add_loggers, + add_shadow_loggers, + extend_logger_results_with_comparison, + extract_logger_info, + extract_shadow_logger_info, + extract_weights, + NSTracer, + OutputLogger, + RNNReturnType, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/_quantized_conversions.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/_quantized_conversions.py new file mode 100644 index 0000000000000000000000000000000000000000..0fcb1004f70475d9f6038793716fcdfcd229d625 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/_quantized_conversions.py @@ -0,0 +1,134 @@ +# mypy: allow-untyped-defs +import torch + + +# Pack pairs of int4 values into int8, in row major order; first int4 +# value goes into lower order bits, and second int4 value into higher +# order bits of resulting int8 value. +def pack_int4_to_int8(weight): + assert weight.dim() == 2 + assert weight.shape[1] % 2 == 0 + assert weight.dtype == torch.int8 + return ((weight[:, 1::2] & 0xF) << 4) | (weight[:, 0::2] & 0xF) + + +# Unpack quandruples of bits in int8 values into int4 values, in row +# major order; lower 4 bits go into first int4 value goes, and upper 4 +# bits go into second int4 value. +def unpack_int8_to_int4(weight): + assert weight.dim() == 2 + assert weight.dtype == torch.int8 + return torch.stack((weight & 0xF, (weight >> 4) & 0xF), dim=2).view( + weight.shape[0], 2 * weight.shape[1] + ) + + +# Transpose the weight matrix, and then reorder its elements according +# to underlying requirements of CUTLASS library, so that it could be +# used for CUTLASS-based mixed datatypes linear operation. +def quantized_weight_reorder_for_mixed_dtypes_linear_cutlass( + weight, dtypeq, transpose=False +): + assert weight.dim() == 2 + assert weight.dtype == torch.int8 + assert dtypeq == torch.int8 or dtypeq == torch.quint4x2 + assert weight.device.type == "cuda" + + device = weight.device + + # subbyte_transpose + if not transpose: + if dtypeq == torch.int8: + outp = weight.T + elif dtypeq == torch.quint4x2: + outp = pack_int4_to_int8(unpack_int8_to_int4(weight.view(torch.int8)).T) + else: + outp = weight + + ncols, nrows = outp.shape # type: ignore[possibly-undefined] + assert nrows % (32 if dtypeq == torch.quint4x2 else 64) == 0 + assert ncols % 64 == 0 + + # permute_B_rows_for_mixed_gemm + # (permute cols actually, as transpose is applied first here) + if dtypeq == torch.quint4x2: + cols_permuted = ( + torch.tensor( + [0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15], + device=device, + ) + + (torch.arange(0, nrows // 16, device=device).reshape(-1, 1) * 16).expand( + nrows // 16, 16 + ) + ).view(-1) + else: + cols_permuted = ( + torch.tensor( + [0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15], + device=device, + ) + + (torch.arange(0, nrows // 16, device=device).reshape(-1, 1) * 16).expand( + nrows // 16, 16 + ) + ).view(-1) + # pyrefly: ignore [unbound-name] + outp = outp.index_copy(1, cols_permuted, outp) + + # interleave_column_major_tensor + magic0 = 4 if dtypeq == torch.quint4x2 else 2 + magic1 = 32 // magic0 + + tmp0 = ( + (torch.arange(0, ncols // magic0, device=device) * (nrows // 4 * magic0)) + .view(-1, 1) + .repeat(1, nrows // 4 * magic0) + .view(-1) + ) + tmp1 = ( + (torch.arange(0, nrows // 4 // magic1, device=device) * (magic0 * magic1)) + .view(-1, 1) + .repeat(1, magic1) + .view(-1) + .repeat(ncols) + ) + tmp2 = ( + (torch.arange(0, magic0, device=device) * magic1) + .view(-1, 1) + .repeat(1, nrows // 4) + .view(-1) + .repeat(ncols // magic0) + ) + tmp3 = torch.arange(0, magic1, device=device).repeat(nrows // 4 * ncols // magic1) + + outp_offsets = tmp0 + tmp1 + tmp2 + tmp3 + + tmp = outp.view(-1).view(torch.int32) + outp = torch.zeros_like(tmp) + outp.scatter_(0, outp_offsets, tmp) + outp = outp.view(weight.dtype) + + # add_bias_and_interleave_quantized_tensor_inplace + tmp = outp.view(-1) + + outp = torch.empty_like(tmp) + if dtypeq == torch.int8: + tmp = (tmp.to(torch.int) + 128).to(tmp.dtype) + outp[0::4] = tmp[0::4] + outp[1::4] = tmp[2::4] + outp[2::4] = tmp[1::4] + outp[3::4] = tmp[3::4] + elif dtypeq == torch.quint4x2: + tmp0 = ((tmp & 0xF) + 8) & 0xF + tmp0 = (tmp0[1::2] << 4) | tmp0[0::2] + tmp1 = (((tmp >> 4) & 0xF) + 8) & 0xF + tmp1 = (tmp1[1::2] << 4) | tmp1[0::2] + outp[0::4] = tmp0[0::2] + outp[1::4] = tmp0[1::2] + outp[2::4] = tmp1[0::2] + outp[3::4] = tmp1[1::2] + + if dtypeq == torch.quint4x2: + nrows *= 2 + ncols //= 2 + + return outp.view(nrows, ncols).view(torch.uint8) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/fake_quantize.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/fake_quantize.py new file mode 100644 index 0000000000000000000000000000000000000000..69a5d730bfb68e89e24beb04ad13fd3fa5881ae9 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/fake_quantize.py @@ -0,0 +1,32 @@ +# flake8: noqa: F401 +r""" +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/quantization/fake_quantize.py`, while adding an import statement +here. +""" + +from torch.ao.quantization.fake_quantize import ( + _is_fake_quant_script_module, + _is_per_channel, + _is_per_tensor, + _is_symmetric_quant, + default_fake_quant, + default_fixed_qparams_range_0to1_fake_quant, + default_fixed_qparams_range_neg1to1_fake_quant, + default_fused_act_fake_quant, + default_fused_per_channel_wt_fake_quant, + default_fused_wt_fake_quant, + default_histogram_fake_quant, + default_per_channel_weight_fake_quant, + default_weight_fake_quant, + disable_fake_quant, + disable_observer, + enable_fake_quant, + enable_observer, + FakeQuantize, + FakeQuantizeBase, + FixedQParamsFakeQuantize, + FusedMovingAvgObsFakeQuantize, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/fuse_modules.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/fuse_modules.py new file mode 100644 index 0000000000000000000000000000000000000000..bce403549d68584ec22089c22b14f17010d6252d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/fuse_modules.py @@ -0,0 +1,22 @@ +# flake8: noqa: F401 +r""" +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/quantization/fuse_modules.py`, while adding an import statement +here. +""" + +# TODO: These functions are not used outside the `fuse_modules.py` +# Keeping here for now, need to remove them later. +from torch.ao.quantization.fuse_modules import ( + _fuse_modules, + _get_module, + _set_module, + fuse_known_modules, + fuse_modules, + get_fuser_method, +) + +# for backward compatibility +from torch.ao.quantization.fuser_method_mappings import fuse_conv_bn, fuse_conv_bn_relu diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/fuser_method_mappings.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/fuser_method_mappings.py new file mode 100644 index 0000000000000000000000000000000000000000..5a68fbf02015ff162ebbd4e26bf85a94328322c8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/fuser_method_mappings.py @@ -0,0 +1,16 @@ +# flake8: noqa: F401 +r""" +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/quantization/fuser_method_mappings.py`, while adding an import statement +here. +""" + +from torch.ao.quantization.fuser_method_mappings import ( + _DEFAULT_OP_LIST_TO_FUSER_METHOD, + fuse_conv_bn, + fuse_conv_bn_relu, + fuse_linear_bn, + get_fuser_method, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/observer.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/observer.py new file mode 100644 index 0000000000000000000000000000000000000000..2163e2717b0697d34fe23e05dbb69c3a555da4b3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/observer.py @@ -0,0 +1,37 @@ +# flake8: noqa: F401 +r""" +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/quantization/observer.py`, while adding an import statement +here. +""" + +from torch.ao.quantization.observer import ( + _is_activation_post_process, + _is_per_channel_script_obs_instance, + _ObserverBase, + _PartialWrapper, + _with_args, + _with_callable_args, + ABC, + default_debug_observer, + default_dynamic_quant_observer, + default_float_qparams_observer, + default_histogram_observer, + default_observer, + default_per_channel_weight_observer, + default_placeholder_observer, + default_weight_observer, + get_observer_state_dict, + HistogramObserver, + load_observer_state_dict, + MinMaxObserver, + MovingAverageMinMaxObserver, + MovingAveragePerChannelMinMaxObserver, + NoopObserver, + ObserverBase, + PerChannelMinMaxObserver, + PlaceholderObserver, + RecordingObserver, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/qconfig.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/qconfig.py new file mode 100644 index 0000000000000000000000000000000000000000..75398d3343f93a813346e04d3709885eaaeae5cb --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/qconfig.py @@ -0,0 +1,31 @@ +# flake8: noqa: F401 +r""" +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/quantization/qconfig.py`, while adding an import statement +here. +""" + +from torch.ao.quantization.qconfig import ( + _add_module_to_qconfig_obs_ctr, + _assert_valid_qconfig, + default_activation_only_qconfig, + default_debug_qconfig, + default_dynamic_qconfig, + default_per_channel_qconfig, + default_qat_qconfig, + default_qat_qconfig_v2, + default_qconfig, + default_weight_only_qconfig, + float16_dynamic_qconfig, + float16_static_qconfig, + float_qparams_weight_only_qconfig, + get_default_qat_qconfig, + get_default_qconfig, + per_channel_dynamic_qconfig, + QConfig, + qconfig_equals, + QConfigAny, + QConfigDynamic, # pyrefly: ignore # deprecated +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quant_type.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quant_type.py new file mode 100644 index 0000000000000000000000000000000000000000..8555f03792661f39c85c8facf3f911786cc25d0f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quant_type.py @@ -0,0 +1,10 @@ +# flake8: noqa: F401 +r""" +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/quantization/quant_type.py`, while adding an import statement +here. +""" + +from torch.ao.quantization.quant_type import _get_quant_type_to_str, QuantType diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quantization_mappings.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quantization_mappings.py new file mode 100644 index 0000000000000000000000000000000000000000..faa24d391d31ad65cb54d580a7dc6e8f1ff36f83 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quantization_mappings.py @@ -0,0 +1,30 @@ +# flake8: noqa: F401 +r""" +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/quantization/quantization_mappings.py`, while adding an import statement +here. +""" + +from torch.ao.quantization.quantization_mappings import ( + _get_special_act_post_process, + _has_special_act_post_process, + _INCLUDE_QCONFIG_PROPAGATE_LIST, + DEFAULT_DYNAMIC_QUANT_MODULE_MAPPINGS, + DEFAULT_FLOAT_TO_QUANTIZED_OPERATOR_MAPPINGS, + DEFAULT_MODULE_TO_ACT_POST_PROCESS, + DEFAULT_QAT_MODULE_MAPPINGS, + DEFAULT_REFERENCE_STATIC_QUANT_MODULE_MAPPINGS, + DEFAULT_STATIC_QUANT_MODULE_MAPPINGS, + get_default_compare_output_module_list, + get_default_dynamic_quant_module_mappings, + get_default_float_to_quantized_operator_mappings, + get_default_qat_module_mappings, + get_default_qconfig_propagation_list, + get_default_static_quant_module_mappings, + get_dynamic_quant_module_class, + get_quantized_operator, + get_static_quant_module_class, + no_observer_set, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quantize.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quantize.py new file mode 100644 index 0000000000000000000000000000000000000000..600d3a46fed0346e3ae8909872cd5bf3c733860c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quantize.py @@ -0,0 +1,30 @@ +# flake8: noqa: F401 +r""" +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/quantization/quantize.py`, while adding an import statement +here. +""" + +from torch.ao.quantization.quantize import ( + _add_observer_, + _convert, + _get_observer_dict, + _get_unique_devices_, + _is_activation_post_process, + _observer_forward_hook, + _propagate_qconfig_helper, + _register_activation_post_process_hook, + _remove_activation_post_process, + _remove_qconfig, + add_quant_dequant, + convert, + prepare, + prepare_qat, + propagate_qconfig_, + quantize, + quantize_dynamic, + quantize_qat, + swap_module, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quantize_fx.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quantize_fx.py new file mode 100644 index 0000000000000000000000000000000000000000..649142c7a7eee9885d96b37f70e582f3ea9a9f8d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quantize_fx.py @@ -0,0 +1,26 @@ +# flake8: noqa: F401 +r""" +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/quantization/quantize_fx.py`, while adding an import statement +here. +""" + +from torch.ao.quantization.fx.graph_module import ObservedGraphModule +from torch.ao.quantization.quantize_fx import ( + _check_is_graph_module, + _convert_fx, + _convert_standalone_module_fx, + _fuse_fx, + _prepare_fx, + _prepare_standalone_module_fx, + _swap_ff_with_fxff, + convert_fx, + fuse_fx, + prepare_fx, + prepare_qat_fx, + QuantizationTracer, + Scope, + ScopeContextManager, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quantize_jit.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quantize_jit.py new file mode 100644 index 0000000000000000000000000000000000000000..aa627dc7bb51ef7ea1fde7e2e5da283c9f6c8900 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/quantize_jit.py @@ -0,0 +1,26 @@ +# flake8: noqa: F401 +r""" +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/quantization/quantize_jit.py`, while adding an import statement +here. +""" + +from torch.ao.quantization.quantize_jit import ( + _check_forward_method, + _check_is_script_module, + _convert_jit, + _prepare_jit, + _prepare_ondevice_dynamic_jit, + _quantize_jit, + convert_dynamic_jit, + convert_jit, + fuse_conv_bn_jit, + prepare_dynamic_jit, + prepare_jit, + quantize_dynamic_jit, + quantize_jit, + script_qconfig, + script_qconfig_dict, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/stubs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/stubs.py new file mode 100644 index 0000000000000000000000000000000000000000..d3fd5c63683dc572c35cabc202ee4ddb2b0053c6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/stubs.py @@ -0,0 +1,10 @@ +# flake8: noqa: F401 +r""" +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/quantization/stubs.py`, while adding an import statement +here. +""" + +from torch.ao.quantization.stubs import DeQuantStub, QuantStub, QuantWrapper diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7d51d58f38d7462713f84ab62427852c1dd8e52c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/quantization/utils.py @@ -0,0 +1,29 @@ +# flake8: noqa: F401 +r""" +Utils shared by different modes of quantization (eager/graph) + +This file is in the process of migration to `torch/ao/quantization`, and +is kept here for compatibility while the migration process is ongoing. +If you are adding a new entry/functionality, please, add it to the +`torch/ao/quantization/utils.py`, while adding an import statement +here. +""" + +from torch.ao.quantization.utils import ( + activation_dtype, + activation_is_int8_quantized, + activation_is_statically_quantized, + calculate_qmin_qmax, + check_min_max_valid, + get_combined_dict, + get_qconfig_dtypes, + get_qparam_dict, + get_quant_type, + get_swapped_custom_module_class, + getattr_from_fqn, + is_per_channel, + is_per_tensor, + weight_dtype, + weight_is_quantized, + weight_is_statically_quantized, +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/signal/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/signal/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..74a23ba6fae924b8af8a2b4f0f798ca78b543545 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/signal/__init__.py @@ -0,0 +1,4 @@ +from . import windows + + +__all__ = ["windows"] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ac286c39aa723deec95c6c01aa21fa258a11d37b --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/__init__.py @@ -0,0 +1,713 @@ +# mypy: allow-untyped-defs +# The Tensor classes are added to this module by python_tensor.cpp +# A workaround to support both TorchScript and MyPy: +from typing import Any, Optional, TYPE_CHECKING, Union + +import torch +from torch import Tensor +from torch._C import _add_docstr, _sparse # type: ignore[attr-defined] + +# Semi structured sparsity support +from .semi_structured import ( + SparseSemiStructuredTensor, + SparseSemiStructuredTensorCUSPARSELT, + SparseSemiStructuredTensorCUTLASS, + to_sparse_semi_structured, +) + + +if TYPE_CHECKING: + from torch.types import _dtype as DType + + DimOrDims = Optional[int | tuple[int, ...] | list[int]] +else: + # The JIT doesn't understand Union, nor torch.dtype here + DType = int + DimOrDims = Optional[tuple[int]] + + +__all__ = [ + "addmm", + "check_sparse_tensor_invariants", + "mm", + "sum", + "softmax", + "solve", + "log_softmax", + "SparseSemiStructuredTensor", + "SparseSemiStructuredTensorCUTLASS", + "SparseSemiStructuredTensorCUSPARSELT", + "to_sparse_semi_structured", + "as_sparse_gradcheck", +] + +addmm = _add_docstr( + _sparse._sparse_addmm, + r""" +sparse.addmm(mat, mat1, mat2, *, beta=1., alpha=1.) -> Tensor + +This function does exact same thing as :func:`torch.addmm` in the forward, +except that it supports backward for sparse COO matrix :attr:`mat1`. +When :attr:`mat1` is a COO tensor it must have `sparse_dim = 2`. +When inputs are COO tensors, this function also supports backward for both inputs. + +Supports both CSR and COO storage formats. + +.. note:: + This function doesn't support computing derivatives with respect to CSR matrices. + +Args: + mat (Tensor): a dense matrix to be added + mat1 (Tensor): a sparse matrix to be multiplied + mat2 (Tensor): a dense matrix to be multiplied + beta (Number, optional): multiplier for :attr:`mat` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat1 @ mat2` (:math:`\alpha`) +""", +) + + +mm = _add_docstr( + _sparse._sparse_mm, + r""" + Performs a matrix multiplication of the sparse matrix :attr:`mat1` + and the (sparse or strided) matrix :attr:`mat2`. Similar to :func:`torch.mm`, if :attr:`mat1` is a + :math:`(n \times m)` tensor, :attr:`mat2` is a :math:`(m \times p)` tensor, out will be a + :math:`(n \times p)` tensor. + When :attr:`mat1` is a COO tensor it must have `sparse_dim = 2`. + When inputs are COO tensors, this function also supports backward for both inputs. + + Supports both CSR and COO storage formats. + +.. note:: + This function doesn't support computing derivatives with respect to CSR matrices. + + This function also additionally accepts an optional :attr:`reduce` argument that allows + specification of an optional reduction operation, mathematically performs the following operation: + +.. math:: + + z_{ij} = \bigoplus_{k = 0}^{K - 1} x_{ik} y_{kj} + +where :math:`\bigoplus` defines the reduce operator. :attr:`reduce` is implemented only for +CSR storage format on CPU device. + +Args: + mat1 (Tensor): the first sparse matrix to be multiplied + mat2 (Tensor): the second matrix to be multiplied, which could be sparse or dense + reduce (str, optional): the reduction operation to apply for non-unique indices + (:obj:`"sum"`, :obj:`"mean"`, :obj:`"amax"`, :obj:`"amin"`). Default :obj:`"sum"`. + +Shape: + The format of the output tensor of this function follows: + - sparse x sparse -> sparse + - sparse x dense -> dense + +Example:: + + >>> a = torch.tensor([[1., 0, 2], [0, 3, 0]]).to_sparse().requires_grad_() + >>> a + tensor(indices=tensor([[0, 0, 1], + [0, 2, 1]]), + values=tensor([1., 2., 3.]), + size=(2, 3), nnz=3, layout=torch.sparse_coo, requires_grad=True) + >>> b = torch.tensor([[0, 1.], [2, 0], [0, 0]], requires_grad=True) + >>> b + tensor([[0., 1.], + [2., 0.], + [0., 0.]], requires_grad=True) + >>> y = torch.sparse.mm(a, b) + >>> y + tensor([[0., 1.], + [6., 0.]], grad_fn=) + >>> y.sum().backward() + >>> a.grad + tensor(indices=tensor([[0, 0, 1], + [0, 2, 1]]), + values=tensor([1., 0., 2.]), + size=(2, 3), nnz=3, layout=torch.sparse_coo) + >>> c = a.detach().to_sparse_csr() + >>> c + tensor(crow_indices=tensor([0, 2, 3]), + col_indices=tensor([0, 2, 1]), + values=tensor([1., 2., 3.]), size=(2, 3), nnz=3, + layout=torch.sparse_csr) + >>> y1 = torch.sparse.mm(c, b, 'sum') + >>> y1 + tensor([[0., 1.], + [6., 0.]], grad_fn=) + >>> y2 = torch.sparse.mm(c, b, 'max') + >>> y2 + tensor([[0., 1.], + [6., 0.]], grad_fn=) +""", +) + + +sampled_addmm = _add_docstr( + _sparse.sparse_sampled_addmm, + r""" +sparse.sampled_addmm(input, mat1, mat2, *, beta=1., alpha=1., out=None) -> Tensor + +Performs a matrix multiplication of the dense matrices :attr:`mat1` and :attr:`mat2` at the locations +specified by the sparsity pattern of :attr:`input`. The matrix :attr:`input` is added to the final result. + +Mathematically this performs the following operation: + +.. math:: + + \text{out} = \alpha\ (\text{mat1} \mathbin{@} \text{mat2})*\text{spy}(\text{input}) + \beta\ \text{input} + +where :math:`\text{spy}(\text{input})` is the sparsity pattern matrix of :attr:`input`, :attr:`alpha` +and :attr:`beta` are the scaling factors. +:math:`\text{spy}(\text{input})` has value 1 at the positions where :attr:`input` has non-zero values, and 0 elsewhere. + +.. note:: + :attr:`input` must be a sparse CSR tensor. :attr:`mat1` and :attr:`mat2` must be dense tensors. + +Args: + input (Tensor): a sparse CSR matrix of shape `(m, n)` to be added and used to compute + the sampled matrix multiplication + mat1 (Tensor): a dense matrix of shape `(m, k)` to be multiplied + mat2 (Tensor): a dense matrix of shape `(k, n)` to be multiplied + +Keyword args: + beta (Number, optional): multiplier for :attr:`input` (:math:`\beta`) + alpha (Number, optional): multiplier for :math:`mat1 @ mat2` (:math:`\alpha`) + out (Tensor, optional): output tensor. Ignored if `None`. Default: `None`. + +Examples:: + + >>> input = torch.eye(3, device='cuda').to_sparse_csr() + >>> mat1 = torch.randn(3, 5, device='cuda') + >>> mat2 = torch.randn(5, 3, device='cuda') + >>> torch.sparse.sampled_addmm(input, mat1, mat2) + tensor(crow_indices=tensor([0, 1, 2, 3]), + col_indices=tensor([0, 1, 2]), + values=tensor([ 0.2847, -0.7805, -0.1900]), device='cuda:0', + size=(3, 3), nnz=3, layout=torch.sparse_csr) + >>> torch.sparse.sampled_addmm(input, mat1, mat2).to_dense() + tensor([[ 0.2847, 0.0000, 0.0000], + [ 0.0000, -0.7805, 0.0000], + [ 0.0000, 0.0000, -0.1900]], device='cuda:0') + >>> torch.sparse.sampled_addmm(input, mat1, mat2, beta=0.5, alpha=0.5) + tensor(crow_indices=tensor([0, 1, 2, 3]), + col_indices=tensor([0, 1, 2]), + values=tensor([ 0.1423, -0.3903, -0.0950]), device='cuda:0', + size=(3, 3), nnz=3, layout=torch.sparse_csr) +""", +) + + +def sum(input: Tensor, dim: DimOrDims = None, dtype: DType | None = None) -> Tensor: + r"""Return the sum of each row of the given sparse tensor. + + Returns the sum of each row of the sparse tensor :attr:`input` in the given + dimensions :attr:`dim`. If :attr:`dim` is a list of dimensions, + reduce over all of them. When sum over all ``sparse_dim``, this method + returns a dense tensor instead of a sparse tensor. + + All summed :attr:`dim` are squeezed (see :func:`torch.squeeze`), resulting an output + tensor having :attr:`dim` fewer dimensions than :attr:`input`. + + During backward, only gradients at ``nnz`` locations of :attr:`input` + will propagate back. Note that the gradients of :attr:`input` is coalesced. + + Args: + input (Tensor): the input sparse tensor + dim (int or tuple of ints): a dimension or a list of dimensions to reduce. Default: reduce + over all dims. + dtype (:class:`torch.dtype`, optional): the desired data type of returned Tensor. + Default: dtype of :attr:`input`. + + Example:: + + >>> nnz = 3 + >>> dims = [5, 5, 2, 3] + >>> I = torch.cat([torch.randint(0, dims[0], size=(nnz,)), + torch.randint(0, dims[1], size=(nnz,))], 0).reshape(2, nnz) + >>> V = torch.randn(nnz, dims[2], dims[3]) + >>> size = torch.Size(dims) + >>> # xdoctest: +IGNORE_WANT("non-deterministic") + >>> S = torch.sparse_coo_tensor(I, V, size) + >>> S + tensor(indices=tensor([[2, 0, 3], + [2, 4, 1]]), + values=tensor([[[-0.6438, -1.6467, 1.4004], + [ 0.3411, 0.0918, -0.2312]], + + [[ 0.5348, 0.0634, -2.0494], + [-0.7125, -1.0646, 2.1844]], + + [[ 0.1276, 0.1874, -0.6334], + [-1.9682, -0.5340, 0.7483]]]), + size=(5, 5, 2, 3), nnz=3, layout=torch.sparse_coo) + + # when sum over only part of sparse_dims, return a sparse tensor + >>> torch.sparse.sum(S, [1, 3]) + tensor(indices=tensor([[0, 2, 3]]), + values=tensor([[-1.4512, 0.4073], + [-0.8901, 0.2017], + [-0.3183, -1.7539]]), + size=(5, 2), nnz=3, layout=torch.sparse_coo) + + # when sum over all sparse dim, return a dense tensor + # with summed dims squeezed + >>> torch.sparse.sum(S, [0, 1, 3]) + tensor([-2.6596, -1.1450]) + """ + if dtype is None: + if dim is not None: + return torch._sparse_sum(input, dim) + else: + return torch._sparse_sum(input) + else: + if dim is not None: + return torch._sparse_sum(input, dim, dtype=dtype) + else: + return torch._sparse_sum(input, dtype=dtype) + + +softmax = _add_docstr( + _sparse._sparse_softmax, + r""" +sparse.softmax(input, dim, *, dtype=None) -> Tensor + +Applies a softmax function. + +Softmax is defined as: + +:math:`\text{Softmax}(x_{i}) = \frac{exp(x_i)}{\sum_j exp(x_j)}` + +where :math:`i, j` run over sparse tensor indices and unspecified +entries are ignores. This is equivalent to defining unspecified +entries as negative infinity so that :math:`exp(x_k) = 0` when the +entry with index :math:`k` has not specified. + +It is applied to all slices along `dim`, and will re-scale them so +that the elements lie in the range `[0, 1]` and sum to 1. + +Args: + input (Tensor): input + dim (int): A dimension along which softmax will be computed. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. This is useful for preventing data type + overflows. Default: None +""", +) + + +spsolve = _add_docstr( + _sparse._spsolve, + r""" +sparse.spsolve(input, other, *, left=True) -> Tensor + +Computes the solution of a square system of linear equations with +a unique solution. Its purpose is similar to :func:`torch.linalg.solve`, +except that the system is defined by a sparse CSR matrix with layout +`sparse_csr`. + +Args: + input (Tensor): a sparse CSR matrix of shape `(n, n)` representing the + coefficients of the linear system. + other (Tensor): a dense matrix of shape `(n, )` representing the right-hand + side of the linear system. + left (bool, optional): whether to solve the system for `input @ out = other` + (default) or `out @ input = other`. Only `left=True` is supported. +""", +) + +log_softmax = _add_docstr( + _sparse._sparse_log_softmax, + r""" +sparse.log_softmax(input, dim, *, dtype=None) -> Tensor + +Applies a softmax function followed by logarithm. + +See :class:`~torch.sparse.softmax` for more details. + +Args: + input (Tensor): input + dim (int): A dimension along which softmax will be computed. + dtype (:class:`torch.dtype`, optional): the desired data type + of returned tensor. If specified, the input tensor is + casted to :attr:`dtype` before the operation is + performed. This is useful for preventing data type + overflows. Default: None +""", +) + + +spdiags = _add_docstr( + _sparse._spdiags, + r""" +sparse.spdiags(diagonals, offsets, shape, layout=None) -> Tensor + +Creates a sparse 2D tensor by placing the values from rows of +:attr:`diagonals` along specified diagonals of the output + +The :attr:`offsets` tensor controls which diagonals are set. + +- If :attr:`offsets[i]` = 0, it is the main diagonal +- If :attr:`offsets[i]` < 0, it is below the main diagonal +- If :attr:`offsets[i]` > 0, it is above the main diagonal + +The number of rows in :attr:`diagonals` must match the length of :attr:`offsets`, +and an offset may not be repeated. + +Args: + diagonals (Tensor): Matrix storing diagonals row-wise + offsets (Tensor): The diagonals to be set, stored as a vector + shape (2-tuple of ints): The desired shape of the result +Keyword args: + layout (:class:`torch.layout`, optional): The desired layout of the + returned tensor. ``torch.sparse_coo``, ``torch.sparse_csc`` and ``torch.sparse_csr`` + are supported. Default: ``torch.sparse_coo`` + +Examples: + +Set the main and first two lower diagonals of a matrix:: + + >>> diags = torch.arange(9).reshape(3, 3) + >>> diags + tensor([[0, 1, 2], + [3, 4, 5], + [6, 7, 8]]) + >>> s = torch.sparse.spdiags(diags, torch.tensor([0, -1, -2]), (3, 3)) + >>> s + tensor(indices=tensor([[0, 1, 2, 1, 2, 2], + [0, 1, 2, 0, 1, 0]]), + values=tensor([0, 1, 2, 3, 4, 6]), + size=(3, 3), nnz=6, layout=torch.sparse_coo) + >>> s.to_dense() + tensor([[0, 0, 0], + [3, 1, 0], + [6, 4, 2]]) + + +Change the output layout:: + + >>> diags = torch.arange(9).reshape(3, 3) + >>> diags + tensor([[0, 1, 2],[3, 4, 5], [6, 7, 8]) + >>> s = torch.sparse.spdiags(diags, torch.tensor([0, -1, -2]), (3, 3), layout=torch.sparse_csr) + >>> s + tensor(crow_indices=tensor([0, 1, 3, 6]), + col_indices=tensor([0, 0, 1, 0, 1, 2]), + values=tensor([0, 3, 1, 6, 4, 2]), size=(3, 3), nnz=6, + layout=torch.sparse_csr) + >>> s.to_dense() + tensor([[0, 0, 0], + [3, 1, 0], + [6, 4, 2]]) + +Set partial diagonals of a large output:: + + >>> diags = torch.tensor([[1, 2], [3, 4]]) + >>> offsets = torch.tensor([0, -1]) + >>> torch.sparse.spdiags(diags, offsets, (5, 5)).to_dense() + tensor([[1, 0, 0, 0, 0], + [3, 2, 0, 0, 0], + [0, 4, 0, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0]]) + +.. note:: + + When setting the values along a given diagonal the index into the diagonal + and the index into the row of :attr:`diagonals` is taken as the + column index in the output. This has the effect that when setting a diagonal + with a positive offset `k` the first value along that diagonal will be + the value in position `k` of the row of :attr:`diagonals` + +Specifying a positive offset:: + + >>> diags = torch.tensor([[1, 2, 3], [1, 2, 3], [1, 2, 3]]) + >>> torch.sparse.spdiags(diags, torch.tensor([0, 1, 2]), (5, 5)).to_dense() + tensor([[1, 2, 3, 0, 0], + [0, 2, 3, 0, 0], + [0, 0, 3, 0, 0], + [0, 0, 0, 0, 0], + [0, 0, 0, 0, 0]]) +""", +) + + +class check_sparse_tensor_invariants: + """A tool to control checking sparse tensor invariants. + + The following options exists to manage sparsr tensor invariants + checking in sparse tensor construction: + + 1. Using a context manager: + + .. code:: python + + with torch.sparse.check_sparse_tensor_invariants(): + run_my_model() + + 2. Using a procedural approach: + + .. code:: python + + prev_checks_enabled = torch.sparse.check_sparse_tensor_invariants.is_enabled() + torch.sparse.check_sparse_tensor_invariants.enable() + + run_my_model() + + if not prev_checks_enabled: + torch.sparse.check_sparse_tensor_invariants.disable() + + 3. Using function decoration: + + .. code:: python + + @torch.sparse.check_sparse_tensor_invariants() + def run_my_model(): + ... + + run_my_model() + + 4. Using ``check_invariants`` keyword argument in sparse tensor constructor call. + For example: + + >>> torch.sparse_csr_tensor([0, 1, 3], [0, 1], [1, 2], check_invariants=True) + Traceback (most recent call last): + File "", line 1, in + RuntimeError: `crow_indices[..., -1] == nnz` is not satisfied. + """ + + @staticmethod + def is_enabled(): + r"""Return True if the sparse tensor invariants checking is enabled. + + .. note:: + + Use :func:`torch.sparse.check_sparse_tensor_invariants.enable` or + :func:`torch.sparse.check_sparse_tensor_invariants.disable` to + manage the state of the sparse tensor invariants checks. + """ + return torch._C._check_sparse_tensor_invariants() + + @staticmethod + def enable(): + r"""Enable sparse tensor invariants checking in sparse tensor constructors. + + .. note:: + + By default, the sparse tensor invariants checks are disabled. Use + :func:`torch.sparse.check_sparse_tensor_invariants.is_enabled` to + retrieve the current state of sparse tensor invariants checking. + + .. note:: + + The sparse tensor invariants check flag is effective to all sparse + tensor constructors, both in Python and ATen. + + The flag can be locally overridden by the ``check_invariants`` + optional argument of the sparse tensor constructor functions. + """ + torch._C._set_check_sparse_tensor_invariants(True) + + @staticmethod + def disable(): + r"""Disable sparse tensor invariants checking in sparse tensor constructors. + + See :func:`torch.sparse.check_sparse_tensor_invariants.enable` for more information. + """ + torch._C._set_check_sparse_tensor_invariants(False) + + # context manager support + def __init__(self, enable=True): + self.state = enable + self.saved_state: bool | None = None + + def __enter__(self): + if self.saved_state is not None: + raise RuntimeError( + "This context manager instance is already activated." + " Use a different context manager instance for context nesting." + ) + self.saved_state = self.is_enabled() + torch._C._set_check_sparse_tensor_invariants(self.state) + + def __exit__(self, type, value, traceback): + assert self.saved_state is not None + torch._C._set_check_sparse_tensor_invariants(self.saved_state) + self.saved_state = None + + # decorator support + def __call__(self, mth): + def test_mth(*args, **kwargs): + with type(self)(self.state): + return mth(*args, **kwargs) + + return test_mth + + +def as_sparse_gradcheck(gradcheck): + """Decorate function, to extend gradcheck for sparse tensors. + + Decorator for torch.autograd.gradcheck or its functools.partial + variants that extends the gradcheck function with support to input + functions that operate on or/and return sparse tensors. + + The specified gradcheck function itself is guaranteed to operate + on strided tensors only. + + For example: + + >>> gradcheck = torch.sparse.as_sparse_gradcheck(torch.autograd.gradcheck) + >>> x = ( + ... torch.tensor([[0, 1], [2, 3]], dtype=torch.float64) + ... .to_sparse_coo() + ... .requires_grad_(True) + ... ) + >>> gradcheck(lambda x: x.to_sparse_csr(), x) + True + """ + + def gradcheck_with_sparse_support(func, inputs, **kwargs): + """ + Create gradcheck with support for sparse tensors. + + Same as :func:`torch.autograd.gradcheck` but with sparse tensors inputs and outputs support. + """ + masked = kwargs.pop("masked", False) + sparse_layouts = { + torch.sparse_coo, + torch.sparse_csr, + torch.sparse_csc, + torch.sparse_bsr, + torch.sparse_bsc, + } + sparse_compressed_layouts = { + torch.sparse_csr, + torch.sparse_csc, + torch.sparse_bsr, + torch.sparse_bsc, + } + sparse_block_layouts = {torch.sparse_bsr, torch.sparse_bsc} + STRIDED_REPRESENTATION = "__STRIDED_REPRESENTATION__" + + def convert_to_strided_representation(args): + """Convert differentiable non-strided tensors to a representation containing differentiable strided tensors.""" + if not isinstance(args, (list, tuple)): + args = (args,) + new_args: list[Any] = [] + for obj in args: + if ( + isinstance(obj, torch.Tensor) + and obj.requires_grad + and obj.layout in sparse_layouts + ): + d = { + "layout": obj.layout, + "shape": obj.shape, + } + if not masked: + # Materialize unspecified elements with zero values + batch_dim = obj.ndim - obj.dense_dim() - obj.sparse_dim() + blocksize = ( + obj.values().shape[batch_dim + 1 : batch_dim + 3] + if obj.layout in sparse_block_layouts + else None + ) + full_mask = torch.ones( + obj.shape, device=obj.device, dtype=torch.bool + ).to_sparse( + layout=obj.layout, + blocksize=blocksize, + dense_dim=obj.dense_dim(), + ) + obj = obj.to_dense().sparse_mask(full_mask) + if obj.layout is torch.sparse_coo: + # pyrefly: ignore [no-matching-overload] + d.update( + indices=obj._indices(), is_coalesced=obj.is_coalesced() + ) + values = obj._values() + elif obj.layout in {torch.sparse_csr, torch.sparse_bsr}: + # pyrefly: ignore [no-matching-overload] + d.update( + compressed_indices=obj.crow_indices(), + plain_indices=obj.col_indices(), + ) + values = obj.values() + else: + # pyrefly: ignore [no-matching-overload] + d.update( + compressed_indices=obj.ccol_indices(), + plain_indices=obj.row_indices(), + ) + values = obj.values() + new_args.extend( + (STRIDED_REPRESENTATION, d, values.requires_grad_(True)) + ) + else: + new_args.append(obj) + return tuple(new_args) + + def restore_from_strided_representation(args): + """Restore non-strided differentiable tensosr from their strided representations.""" + new_args = [] + args = list(args) + while args: + a = args.pop(0) + if a == STRIDED_REPRESENTATION: + d, values = args.pop(0), args.pop(0) + if d["layout"] is torch.sparse_coo: + a = torch.sparse_coo_tensor( + d["indices"], + values, + size=d["shape"], + is_coalesced=d["is_coalesced"], + ) + elif d["layout"] in sparse_compressed_layouts: + a = torch.sparse_compressed_tensor( + d["compressed_indices"], + d["plain_indices"], + values, + size=d["shape"], + layout=d["layout"], + ) + else: + raise NotImplementedError( + f"conversion of {d['layout']} strided representation to tensor" + ) + new_args.append(a) + return tuple(new_args) + + def func_wrapper(*args, **kwargs): + restored_args = restore_from_strided_representation(args) + + # convert differentiable output sparse tensors to strided + # tensors: + outputs = func(*restored_args, **kwargs) + + strided_outputs = ( + tuple(outputs) if isinstance(outputs, (list, tuple)) else (outputs,) + ) + strided_outputs = tuple( + ( + o.to_dense(masked_grad=masked) + if isinstance(o, torch.Tensor) + and o.requires_grad + and o.layout in sparse_layouts + else o + ) + for o in strided_outputs + ) + + return ( + strided_outputs + if isinstance(outputs, (list, tuple)) + else strided_outputs[0] + ) + + args = (func_wrapper, convert_to_strided_representation(inputs)) + + return gradcheck(*args, **kwargs) + + return gradcheck_with_sparse_support diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/_semi_structured_conversions.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/_semi_structured_conversions.py new file mode 100644 index 0000000000000000000000000000000000000000..354acdee16a26eeef851e344f9956ac11523cb92 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/_semi_structured_conversions.py @@ -0,0 +1,358 @@ +# mypy: allow-untyped-defs +import torch + + +def _calculate_meta_reordering_scatter_offsets(m, meta_ncols, meta_dtype, device): + """ + This is PyTorch implementation of main part of reorder_meta() + function, from tools/util/include/cutlass/util/host_reorder.h file + of CUTLASS source tree. Furthermore, CUTLASS template for sparse + GEMM decides upon layout of this matrix, and at the moment for the + sparse GEMM executed on tensor cores, this is layout described by + ColumnMajorInterleaved<2> data structure, in + include/cutlass/layout/matrix.h of CUTLASS source tree. The + reordering of meta matrix into meta_reordered matrix calculated + according to these segments of CUTLASS code is re-implemented here. + Note that this calculation produces offsets for scattering metadata + matrix elements into reordered metadata matrix elements (or, + equivalently, for gathering reordered metadata matrix element back + into metadata matrix elements). + """ + dst_rows = torch.arange(0, m, device=device)[:, None].repeat(1, meta_ncols) + dst_cols = torch.arange(0, meta_ncols, device=device).repeat(m, 1) + + # Reorder the rows, then swizzle the 2x2 blocks. + group = 32 if meta_dtype.itemsize == 2 else 16 + interweave = 4 if meta_dtype.itemsize == 2 else 2 + dst_rows = ( + dst_rows // group * group + + (dst_rows % 8) * interweave + + (dst_rows % group) // 8 + ) + + topright = ((dst_rows % 2 == 0) & (dst_cols % 2 == 1)).to(torch.int8) + bottomleft = ((dst_rows % 2 == 1) & (dst_cols % 2 == 0)).to(torch.int8) + dst_rows += topright - bottomleft + dst_cols -= topright - bottomleft + + # Assumed that meta tensor is to be stored in CUTLASS + # InterleavedColumnMajor layout, and reverse engineered + # corresponding code to store values into this tensor. + interleave = 2 + cols_maj = dst_cols // interleave + cols_min = dst_cols % interleave + return (cols_maj * m * interleave + dst_rows * interleave + cols_min).view(-1) + + +def sparse_semi_structured_from_dense_cutlass(dense): + """ + This function converts dense matrix into sparse semi-structured + representation, producing "compressed" matrix, in the layout used by + CUTLASS backend, and corresponding metadata matrix. + """ + if dense.dim() != 2: + raise RuntimeError( + f"Expected 2-dimensional dense tensor, got {dense.dim()}-dimensional tensor" + ) + + m, k = dense.shape + device = dense.device + + meta_dtype = torch.int8 + if dense.dtype == torch.int8: + meta_dtype = torch.int32 + elif dense.dtype in [torch.half, torch.bfloat16, torch.float]: + meta_dtype = torch.int16 + else: + raise RuntimeError(f"Invalid datatype {dense.dtype} of dense matrix") + quadbits_per_meta_elem = meta_dtype.itemsize * 8 // 4 + if quadbits_per_meta_elem not in (4, 8): + raise RuntimeError("Invalid number of elements per meta element calculated") + + if meta_dtype == torch.int32: + if m % 16 != 0: + raise RuntimeError( + f"Number of rows of dense matrix {m} must be divisible by 16" + ) + else: + if m % 32 != 0: + raise RuntimeError( + f"Number of rows of dense matrix {m} must be divisible by 32" + ) + if k % (4 * quadbits_per_meta_elem) != 0: + raise RuntimeError( + f"Number of columns of dense matrix {k} must be divisible by {4 * quadbits_per_meta_elem}" + ) + + if dense.dtype != torch.float: + ksparse = 4 + dense_4 = dense.view(-1, k // ksparse, ksparse) + m0, m1, _m2, m3 = (dense_4 != 0).unbind(-1) + else: + ksparse = 2 + dense_2 = dense.view(-1, k // ksparse, ksparse) + m0, _m2 = m1, m3 = (dense_2 != 0).unbind(-1) + meta_ncols = k // (ksparse * quadbits_per_meta_elem) + + # Encoding quadruples of True/False values as follows: + # [True, True, False, False] -> 0b0100 + # [True, False, True, False] -> 0b1000 + # [False, True, True, False] -> 0b1001 + # [True, False, False, True ] -> 0b1100 + # [False, True, False, True ] -> 0b1101 + # [False, False, True, True ] -> 0b1110 + # Thus, lower two bits in the encoding are index of the True value + # at the lowest index in the quadruple, and the higher two bits in + # the encoding are index of the other True value in the quadruple. + # In case there are less than two True values, than False value or + # values at some index or indices are considered True for the + # encoding. In case there are more than two True values, then the + # excess True value(s) at some indices are considered False for + # the encoding. The exact encodings used for these cases are as + # follows: + # [False, False, False, False] -> 0b1110 + # [False, False, False, True ] -> 0b1110 + # [False, False, True, False] -> 0b1110 + # [False, True, False, False] -> 0b1001 + # [False, True, True, True ] -> 0b1101 + # [True, False, False, False] -> 0b1000 + # [True, False, True, True ] -> 0b1100 + # [True, True, False, True ] -> 0b0100 + # [True, True, True, False] -> 0b0100 + # [True, True, True, True ] -> 0b0100 + # These particular encodings are chosen, with the help of Espresso + # logic minimizer software, for the purpose of minimization of + # corresponding Boolean functions, that translate non-zero flags + # into encoding bits. Note also possible choices for the first + # and last of these encodings were limited only to (0b0100, + # 0b1110), in order to produce valid encodings for 1:2 sparsity + # case. + + expr0 = m0 & m1 + expr1 = ~m0 & m1 + expr2 = ~m0 & ~m1 + bit0 = expr1 + bit1 = expr2 + bit2 = expr0 | expr2 | m3 + bit3 = expr1 | ~m1 + idxs0 = bit0 | (bit1.to(torch.int64) << 1) + idxs1 = bit2 | (bit3.to(torch.int64) << 1) + + if dense.dtype != torch.float: + sparse0 = dense_4.gather(-1, idxs0.unsqueeze(-1)) # type: ignore[possibly-undefined] + # pyrefly: ignore [unbound-name] + sparse1 = dense_4.gather(-1, idxs1.unsqueeze(-1)) + sparse = torch.stack((sparse0, sparse1), dim=-1).view(m, k // 2) + else: + sparse = dense_2.gather(-1, idxs0.unsqueeze(-1) // 2).view(m, k // 2) # type: ignore[possibly-undefined] + + meta_4 = idxs0 | (idxs1 << 2) + meta_n = meta_4.view((-1, meta_ncols, quadbits_per_meta_elem)).to(meta_dtype) + + if quadbits_per_meta_elem == 4: + meta = ( + meta_n[:, :, 0] + | (meta_n[:, :, 1] << 4) + | (meta_n[:, :, 2] << 8) + | (meta_n[:, :, 3] << 12) + ) + elif quadbits_per_meta_elem == 8: + meta = ( + meta_n[:, :, 0] + | (meta_n[:, :, 1] << 4) + | (meta_n[:, :, 2] << 8) + | (meta_n[:, :, 3] << 12) + | (meta_n[:, :, 4] << 16) + | (meta_n[:, :, 5] << 20) + | (meta_n[:, :, 6] << 24) + | (meta_n[:, :, 7] << 28) + ) + + # Reorder meta tensor elements. + meta_reordered = meta.new_empty((m * meta_ncols,)) # type: ignore[possibly-undefined] + meta_offsets = _calculate_meta_reordering_scatter_offsets( + m, meta_ncols, meta_dtype, device + ) + # pyrefly: ignore [unbound-name] + meta_reordered.scatter_(0, meta_offsets, meta.view(-1)) + + return (sparse, meta_reordered.view(m, meta_ncols)) + + +def sparse_semi_structured_to_dense_cutlass(sparse, meta_reordered): + """ + This function performs reverse of the function above - it + reconstructs dense matrix from a pair of "compressed" matrix, given + in the layout used by CUTLASS backend, and accompanying metadata + matrix. + """ + if sparse.dim() != 2: + raise RuntimeError( + f"Expected 2-dimensional sparse tensor, got {sparse.dim()}-dimensional tensor" + ) + + m, k = sparse.shape + device = sparse.device + + if meta_reordered.dim() != 2: + raise RuntimeError( + f"Expected 2-dimensional meta tensor, got {meta_reordered.dim()}-dimensional tensor" + ) + if meta_reordered.device != device: + raise RuntimeError( + f"Expected meta matrix to be on {device} device, got matrix on {meta_reordered.device} device" + ) + + meta_dtype = meta_reordered.dtype + if meta_dtype not in (torch.int16, torch.int32): + raise RuntimeError(f"Invalid datatype {meta_dtype} of meta matrix") + quadbits_per_meta_elem = meta_dtype.itemsize * 8 // 4 + + if sparse.dtype != torch.float: + ksparse = 4 + else: + ksparse = 2 + + meta_nrows, meta_ncols = meta_reordered.shape + if meta_nrows != m: + raise RuntimeError( + f"Number of rows of meta matrix {meta_nrows} must be equal to number of columns of spase matrix {m}" + ) + if meta_ncols * ksparse * quadbits_per_meta_elem != 2 * k: + raise RuntimeError( + f"Number of columns of sparse matrix {k} different from the {meta_ncols * ksparse * quadbits_per_meta_elem // 2}, " + "expected according to the number of columns of meta matrix" + ) + + # Undo meta tensor elements reordering. + meta_offsets = _calculate_meta_reordering_scatter_offsets( + m, meta_ncols, meta_dtype, device + ) + meta = torch.gather(meta_reordered.view(-1), 0, meta_offsets).view(m, meta_ncols) + + # Unpack sparse tensor back to original dense tensor, using + # information provided by meta tensor. Note that torch.float + # datatype is handled pretty much the same as + # torch.half/torch.bfloat16, as metadata for a pair of torch.float + # value is encoded as if underlying 8 bytes contain four + # torch.half/torch.bfloat16 values, where either first two or last + # two are zeros. + meta_2 = torch.empty( + (m, meta_ncols, 2 * quadbits_per_meta_elem), + dtype=meta_dtype, + device=device, + ) + if quadbits_per_meta_elem == 4: + meta_2[:, :, 0] = meta & 0b11 + meta_2[:, :, 1] = (meta >> 2) & 0b11 + meta_2[:, :, 2] = (meta >> 4) & 0b11 + meta_2[:, :, 3] = (meta >> 6) & 0b11 + meta_2[:, :, 4] = (meta >> 8) & 0b11 + meta_2[:, :, 5] = (meta >> 10) & 0b11 + meta_2[:, :, 6] = (meta >> 12) & 0b11 + meta_2[:, :, 7] = (meta >> 14) & 0b11 + elif quadbits_per_meta_elem == 8: + meta_2[:, :, 0] = meta & 0b11 + meta_2[:, :, 1] = (meta >> 2) & 0b11 + meta_2[:, :, 2] = (meta >> 4) & 0b11 + meta_2[:, :, 3] = (meta >> 6) & 0b11 + meta_2[:, :, 4] = (meta >> 8) & 0b11 + meta_2[:, :, 5] = (meta >> 10) & 0b11 + meta_2[:, :, 6] = (meta >> 12) & 0b11 + meta_2[:, :, 7] = (meta >> 14) & 0b11 + meta_2[:, :, 8] = (meta >> 16) & 0b11 + meta_2[:, :, 9] = (meta >> 18) & 0b11 + meta_2[:, :, 10] = (meta >> 20) & 0b11 + meta_2[:, :, 11] = (meta >> 22) & 0b11 + meta_2[:, :, 12] = (meta >> 24) & 0b11 + meta_2[:, :, 13] = (meta >> 26) & 0b11 + meta_2[:, :, 14] = (meta >> 28) & 0b11 + meta_2[:, :, 15] = (meta >> 30) & 0b11 + + dense_offsets = meta_2.view(-1) + ( + torch.arange(0, 2 * m * k // ksparse, device=device) * 4 + ).view(-1, 1).repeat(1, 2).view(-1) + + dense = torch.zeros((m * 2 * k,), dtype=sparse.dtype, device=device) + if sparse.dtype != torch.float: + dense.scatter_(0, dense_offsets, sparse.view(-1)) + else: + dense.view(torch.half).scatter_( + 0, dense_offsets, sparse.view(torch.half).view(-1) + ) + + return dense.view(m, 2 * k) + + +def _sparse_semi_structured_tile(dense): + """ + This function computes a 2:4 sparse tile by greedily taking the largest values. + + Since we take the largest values greedily, how the sorting algorithm handles duplicates affects + the ultimate sparsity pattern. + + Note that this function does not have the same sorting semantics as our CUDA backend, + which is exposed via `torch._sparse_semi_structured_tile` and thus returns a different pattern. + """ + + def greedy_prune_tile(tile): + num_kept_row = [0, 0, 0, 0] + num_kept_col = [0, 0, 0, 0] + + for x in tile.flatten().sort(descending=True, stable=True).indices: + r, c = x // 4, x % 4 + if num_kept_row[r] < 2 and num_kept_col[c] < 2: + num_kept_row[r] += 1 + num_kept_col[c] += 1 + else: + tile[r, c] = 0 + + for batch in dense.unfold(0, 4, 4).unfold(1, 4, 4): + for tile in batch: + greedy_prune_tile(tile) + + return dense + + +def _compute_compressed_swizzled_bitmask(dense): + """ + Calculates the compressed swizzled bitmask from a dense tensor + """ + + # first we need to convert the dense tensor to a bitmask + int_bitmask = dense.bool().to(torch.uint8) + + # Each thread is responsible for an 8x8 tile, which contains 4 4x4 tiles: + # A, B, C and D, as displayed in the following schema: + # +---+---+ + # | A | B | + # +---+---+ + # | C | D | + # +---+---+ + + # we first need to split into the 8x8 tiles + bitmask_8x8_chunks = int_bitmask.unfold(0, 8, 8).unfold(1, 8, 8) + + # then we unfold again to get our individual 4x4 tiles + bitmask_4x4_chunks = bitmask_8x8_chunks.unfold(2, 4, 4).unfold(3, 4, 4) + + # Each 4x4 bitmask defines two 8-bit integers, which encode the sparsity pattern + # of that tile. Note that the least significant bit is stored first. + # [1 1 0 0] + # [1 1 0 0] -> 0011 0011 -> 51 + # [0 0 1 1] 1100 1100 204 + # [0 0 1 1] + + # reshape tensor to expand tiles into 8-bit vectors + bitmask_binary_representation = bitmask_4x4_chunks.reshape( + *bitmask_4x4_chunks.shape[:2], 4, 2, 8 + ) + + # to convert from binary representation, we can do a matmul with powers of two + powers_of_two = 2 ** torch.arange(8, dtype=torch.float, device="cuda") + # To run on GPU: cast to float to do matmul and then cast back + compressed_swizzled_bitmask = ( + bitmask_binary_representation.to(torch.float) @ powers_of_two + ).to(torch.uint8) + + return compressed_swizzled_bitmask diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/_semi_structured_ops.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/_semi_structured_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..8870dce5041906b3a04b2741aa94a4de51eaaf8d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/_semi_structured_ops.py @@ -0,0 +1,198 @@ +# mypy: allow-untyped-defs +import contextlib + +import torch + + +__all__ = [ + "fallback_dispatcher", + "semi_sparse_values", + "semi_sparse_indices", + "semi_sparse_t", + "semi_sparse_view", + "semi_sparse_detach", + "semi_sparse_mm", + "semi_sparse_addmm", + "semi_sparse_linear", + "semi_sparse_scaled_mm", +] + + +@contextlib.contextmanager +def no_dispatch(): + guard = torch._C._DisableTorchDispatch() + try: + yield + finally: + del guard + + +def fallback_dispatcher(func, types, args, kwargs): + with no_dispatch(): + return func(*args) + + +def semi_sparse_values(func, types, args=(), kwargs=None) -> torch.Tensor: + assert len(args) == 1 + A = args[0] + assert isinstance(A, torch.sparse.SparseSemiStructuredTensor) + assert A.packed is not None + if A.meta is None: + m, k = A.shape + num_kept_elements = m * k // 2 + return A.packed.ravel()[:num_kept_elements:].view(m, -1) + else: + return A.packed.detach() + + +def semi_sparse_indices(func, types, args=(), kwargs=None) -> torch.Tensor: + assert len(args) == 1 + A = args[0] + assert isinstance(A, torch.sparse.SparseSemiStructuredTensor) + assert A.packed is not None + if A.meta is None: + m, k = A.shape + num_kept_elements = m * k // 2 + metadata = A.packed.ravel()[num_kept_elements:].view(m, -1) + return metadata.view(torch.int32 if A.dtype == torch.int32 else torch.int16) + else: + return A.meta + + +def semi_sparse_t(func, types, args=(), kwargs=None) -> torch.Tensor: + assert len(args) == 1 + self = args[0] + assert isinstance(self, torch.sparse.SparseSemiStructuredTensor) + assert len(self.shape) == 2 + # Because we cannot go from the compressed representation back to the dense representation currently, + # we just keep track of how many times we have been transposed. Depending on whether the sparse matrix + # is the first or second argument, we expect an even / odd number of calls to transpose respectively. + # pyrefly: ignore [no-matching-overload] + return self.__class__( + torch.Size([self.shape[-1], self.shape[0]]), + packed=self.packed_t, + meta=self.meta_t, + packed_t=self.packed, + meta_t=self.meta, + compressed_swizzled_bitmask=( + self.compressed_swizzled_bitmask.transpose(0, 1) + if self.compressed_swizzled_bitmask is not None + else None + ), + fuse_transpose_cusparselt=args[0].fuse_transpose_cusparselt, + alg_id_cusparselt=args[0].alg_id_cusparselt, + ) + + +def semi_sparse_view(func, types, args=(), kwargs=None) -> torch.Tensor: + assert len(args) == 2 + self, shape = args + if tuple(shape) != self.shape: + raise NotImplementedError( + f"`view` is not implemented for SparseSemiStructuredTensor, except for the dummy case (shape={shape})" + ) + return self + + +def semi_sparse_detach(func, types, args, kwargs) -> torch.Tensor: + assert len(args) == 1 + self = args[0] + return self.__class__( + shape=self.shape, + packed=self.packed, + meta=self.meta, + packed_t=self.packed_t, + meta_t=self.meta_t, + compressed_swizzled_bitmask=self.compressed_swizzled_bitmask, + fuse_transpose_cusparselt=self.fuse_transpose_cusparselt, + alg_id_cusparselt=self.alg_id_cusparselt, + requires_grad=False, + ) + + +def semi_sparse_mm(func, types, args=(), kwargs=None) -> torch.Tensor: + assert len(args) == 2 + A, B = args + if A.ndim != 2 or B.ndim != 2: + raise NotImplementedError( + "`SparseSemiStructuredTensor` matmul: Broadcasting is not implemented" + ) + if isinstance(A, torch.sparse.SparseSemiStructuredTensor): + row, col = B.shape + B_padded = A._pad_dense_input(B) + res = A._mm(B_padded) + return res[:, :col] + else: + B_t = B.t() + assert isinstance(B_t, torch.sparse.SparseSemiStructuredTensor) + row, col = A.shape + A_padded = B._pad_dense_input(A) + res = B_t._mm(A_padded.t()).t() + return res[:row, :] + + +def semi_sparse_addmm(func, types, args=(), kwargs=None) -> torch.Tensor: + assert len(args) == 3 + bias, A, B = args + if A.ndim != 2 or B.ndim != 2: + raise NotImplementedError( + "`SparseSemiStructuredTensor` matmul: Broadcasting is not implemented" + ) + if bias.ndim != 1: + raise NotImplementedError( + f"`SparseSemiStructuredTensor` matmul: only bias dim=1 supported. Shape={bias.shape}" + ) + if isinstance(A, torch.sparse.SparseSemiStructuredTensor): + raise NotImplementedError( + "`SparseSemiStructuredTensor` matmul: only operand B of `addmm` can be sparse" + ) + B_t = B.t() + assert isinstance(B_t, torch.sparse.SparseSemiStructuredTensor) + row, _col = A.shape + A_padded = B_t._pad_dense_input(A) + result = B_t._mm(A_padded.t(), bias=bias).t() + return result[:row, :] + + +def semi_sparse_linear(func, types, args=(), kwargs=None) -> torch.Tensor: + assert len(args) in [2, 3] + A, B = args[:2] + bias = args[2] if len(args) == 3 else None + + shape = A.shape + A_2d = A.view(-1, shape[-1]) + + if bias is None: + res = A_2d @ B.t() + else: + res = semi_sparse_addmm( + func=None, + types=None, + args=[bias, A_2d, B.t()], + ) + + return res.view(*shape[:-1], -1) + + +def semi_sparse_scaled_mm(func, types, args=(), kwargs=None) -> torch.Tensor: + # pull all args, excluding use_fast_accum flag if set. + A, B, A_scale, B_scale, bias, scale_result, out_dtype = args[:7] + + assert A.dtype == torch.float8_e4m3fn + assert B.dtype == torch.float8_e4m3fn + # only cuSPARSELt supports float8_e4m3fn currently + assert isinstance(A, torch.sparse.SparseSemiStructuredTensorCUSPARSELT) + assert A.packed is not None + # Currently we only support per-tensor scaling, with float32 scales + assert A_scale.numel() == 1 and B_scale.numel() == 1 + assert A_scale.dtype == torch.float32 and B_scale.dtype == torch.float32 + + # cuSPARSELt lacks the A and B operand scaling support, so instead we use alpha to scale the result. + # Note that this limits us to per-tensor scalig only. + sparse_result = torch._cslt_sparse_mm( + A.packed, + B, + alpha=A_scale * B_scale, + out_dtype=out_dtype, + ) + return sparse_result diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/_triton_ops.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/_triton_ops.py new file mode 100644 index 0000000000000000000000000000000000000000..cd3d660d2e9c93818d7db3a4fd4da3e396a2f868 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/_triton_ops.py @@ -0,0 +1,2561 @@ +# mypy: allow-untyped-decorators +# mypy: allow-untyped-defs +import math +import os +import weakref +from functools import lru_cache + +import torch +from torch._dynamo.utils import warn_once +from torch.utils._triton import has_triton + +from ._triton_ops_meta import get_meta + + +TORCH_SPARSE_BSR_SCATTER_MM_LRU_CACHE_SIZE = int( + os.getenv("TORCH_SPARSE_BSR_SCATTER_MM_LRU_CACHE_SIZE", 2) +) + + +def check(cond, msg): + if not cond: + raise ValueError(msg) + + +def check_bsr_layout(f_name, t): + check( + t.layout == torch.sparse_bsr, + f"{f_name}(): only BSR sparse format is supported for the sparse argument.", + ) + + +def check_device(f_name, t, device): + check( + t.device == device and t.device.type == "cuda", + f"{f_name}(): all inputs are expected to be on the same GPU device.", + ) + + +def check_mm_compatible_shapes(f_name, lhs, rhs): + check( + lhs.dim() >= 2 and rhs.dim() >= 2, + f"{f_name}(): all inputs involved in the matrix product are expected to be at least 2D, " + f"but got lhs.dim() == {lhs.dim()} and rhs.dim() == {rhs.dim()}.", + ) + + _m, kl = lhs.shape[-2:] + kr, _n = rhs.shape[-2:] + + check( + kl == kr, + f"{f_name}(): arguments' sizes involved in the matrix product are not compatible for matrix multiplication, " + f"got lhs.shape[-1] == {kl} which is not equal to rhs.shape[-2] == {kr}.", + ) + + +def check_dtype(f_name, t, dtype, *additional_dtypes): + check( + t.dtype == dtype + and t.dtype + in ((torch.half, torch.bfloat16, torch.float) + tuple(*additional_dtypes)), + f"{f_name}(): all inputs are expected to be of the same dtype " + f"and one of (half, bfloat16, float32) or {additional_dtypes}, " + f"but got dtype == {t.dtype}.", + ) + + +def check_blocksize(f_name, blocksize): + assert len(blocksize) == 2 + + def is_power_of_two(v): + return not (v & (v - 1)) + + def is_compatible_blocksize(b): + res = True + for blocksize in b: + # Triton loads only blocks which are at least 16 and powers of 2. + res = (blocksize >= 16 and is_power_of_two(blocksize)) and res + return res + + check( + is_compatible_blocksize(blocksize), + f"{f_name}(): sparse inputs' blocksize ({blocksize[0]}, {blocksize[1]}) " + "should be at least 16 and a power of 2 in each dimension.", + ) + + +def make_triton_contiguous(t): + """Return input as a triton-contiguous tensor. + + A triton-contiguous tensor is defined as a tensor that has strides + with minimal value smaller than or equal to 1. + + While triton kernels support triton-non-contiguous tensors (all + strides being greater than 1) arguments, a considerable slow-down + occurs because tensor data is copied element-wise rather than + chunk-wise. Zero strides is assumed to not have this defect. + """ + if min(t.stride()) > 1: + # TODO: investigate if contiguity along other axes than the + # last one can be beneficial for performance + return t.contiguous() + else: + return t + + +def broadcast_batch_dims(f_name, *tensors): + try: + return torch.broadcast_shapes(*(t.shape[:-2] for t in tensors)) + except Exception: + check(False, f"{f_name}(): inputs' batch dimensions are not broadcastable!") + + +def slicer(dim, slice_range, *tensors): + for t in tensors: + slices = [slice(None)] * t.dim() + slices[dim] = slice_range + yield t[slices] + + +def multidim_slicer(dims, slices, *tensors): + for t in tensors: + s = [slice(None)] * t.dim() + for d, d_slice in zip(dims, slices, strict=False): + if d is not None: + s[d] = d_slice + yield t[tuple(s)] + + +def ptr_stride_extractor(*tensors): + for t in tensors: + yield t + yield from t.stride() + + +def grid_partitioner(full_grid, grid_blocks, tensor_dims_map): + assert 0 <= len(full_grid) <= 3 + assert 0 <= len(grid_blocks) <= 3 + + import itertools + + def generate_grid_points(): + for fg, mg in zip(full_grid, grid_blocks, strict=False): + yield range(0, fg, mg) + + def generate_sliced_tensors(slices): + for t, t_dims in tensor_dims_map.items(): + yield next(multidim_slicer(t_dims, slices, t)) + + for grid_point in itertools.product(*generate_grid_points()): + grid = [ + min(fg - gp, mg) + for fg, gp, mg in zip(full_grid, grid_point, grid_blocks, strict=False) + ] + slices = [slice(gp, gp + g) for gp, g in zip(grid_point, grid, strict=False)] + # grid_points are iterated in a "contiguous" order, i.e. + # left dimensions traversed slower than right dimensions. + # This order is reversed for CUDA grids. + yield grid[::-1], *generate_sliced_tensors(slices) + + +def launch_kernel(kernel, tensor_dims_map, full_grid, grid_blocks=None): + # cuda_max_grid = (2 ** 31 - 1, 2 ** 16 - 1, 2 ** 16 - 1) + cuda_max_grid = (2147483647, 65535, 65535)[::-1] + if grid_blocks is None: + grid_blocks = cuda_max_grid + else: + + def valid_grid_dim(g, mg): + if g is None: + return mg + else: + # grid must be at least 1 and no greater than mg + return max(1, min(g, mg)) + + grid_blocks = tuple( + valid_grid_dim(g, mg) + for g, mg in zip(grid_blocks, cuda_max_grid, strict=False) + ) # type: ignore[assignment] + + for grid, *sliced_tensors in grid_partitioner( + full_grid, grid_blocks, tensor_dims_map + ): + kernel(grid, *sliced_tensors) + + +def prepare_inputs(bsr, *dense_tensors): + # Introduce fake batch dimension if not present for convenience. + crow_indices = bsr.crow_indices().unsqueeze(0) + col_indices = bsr.col_indices().unsqueeze(0) + values = make_triton_contiguous(bsr.values().unsqueeze(0)) + tensors = [make_triton_contiguous(t.unsqueeze(0)) for t in dense_tensors] + + # Compute broadcasted batch dimension + batch_dims_broadcasted = torch.broadcast_shapes( + values.shape[:-3], *(t.shape[:-2] for t in tensors) + ) + + # Broadcast batch dimensions and squash. + # The result can be either a view or a copy. + def batch_broadcast_and_squash(t, batch_dims, invariant_dims): + return t.broadcast_to(batch_dims + invariant_dims).flatten( + 0, len(batch_dims) - 1 + ) + + crow_indices = batch_broadcast_and_squash( + crow_indices, batch_dims_broadcasted, (-1,) + ) + + col_indices = batch_broadcast_and_squash(col_indices, batch_dims_broadcasted, (-1,)) + values = batch_broadcast_and_squash( + values, batch_dims_broadcasted, values.shape[-3:] + ) + tensors = [ + batch_broadcast_and_squash(t, batch_dims_broadcasted, t.shape[-2:]) + for t in tensors + ] + + return crow_indices, col_indices, values, *tensors + + +def broadcast_batch_dims_bsr(f_name, bsr, *tensors): + batch_shape = broadcast_batch_dims(f_name, bsr, *tensors) + + crow_indices = bsr.crow_indices().broadcast_to(batch_shape + (-1,)) + col_indices = bsr.col_indices().broadcast_to(batch_shape + (-1,)) + values = bsr.values().broadcast_to(batch_shape + bsr.values().shape[-3:]) + size = batch_shape + bsr.shape[-2:] + return torch.sparse_compressed_tensor( + crow_indices, col_indices, values, size=size, layout=bsr.layout + ) + + +# NOTE: this function will ALWAYS create a view +def tile_to_blocksize(t, blocksize): + *rest, m, n = t.shape + new_shape = rest + [ + m // blocksize[0], + blocksize[0], + n // blocksize[1], + blocksize[1], + ] + # using .view instead of .reshape to ensure that the result is + # indeed a view: + return t.view(new_shape).transpose(-3, -2) + + +def as1Dbatch(tensor): + """Return tensor as 3D tensor by either prepending new dimensions to + the tensor shape (when ``tensor.ndim < 3``), or by collapsing + starting dimensions into the first dimension (when ``tensor.ndim > + 3``). + """ + while tensor.ndim < 3: + tensor = tensor.unsqueeze(0) + if tensor.ndim > 3: + tensor = tensor.flatten(0, tensor.ndim - 3) + assert tensor.ndim == 3, tensor.shape + return tensor + + +def scatter_mm(blocks, others, indices_data, *, accumulators=None): + """Scattered matrix multiplication of tensors. + + A scattered matrix multiplication is defined as a series of matrix + multiplications applied to input tensors according to the input + and output mappings specified by indices data. + + The following indices data formats are supported for defining a + scattered matrix multiplication operation (:attr:`indices_data[0]` + holds the name of the indices data format as specified below): + + - ``"scatter_mm"`` - matrix multiplications scattered in batches + of tensors. + + If :attr:`blocks` is a :math:`(* \times M \times K) tensor, + :attr:`others` is a :math:`(* \times K \times N)` tensor, + :attr:`accumulators` is a :math:`(* \times M \times N)` tensor, + and :attr:`indices = indices_data['indices']` is a :math:`(* + \times 3)` tensor, then the operation is equivalent to the + following code:: + + c_offsets, pq = indices_data[1:] + for r in range(len(c_offsets) - 1): + for g in range(c_offsets[r], c_offsets[r + 1]): + p, q = pq[g] + accumulators[r] += blocks[p] @ others[q] + + - ``"bsr_strided_mm"`` - matrix multiplications scattered in + batches of tensors and a tensor. + + If :attr:`blocks` is a :math:`(Ms \times Ks) tensor, + :attr:`others` is a :math:`(* \times K \times N)` tensor, + :attr:`accumulators` is a :math:`(* \times M \times N)` tensor, then + the operation is equivalent to the following code:: + + c_indices, r_offsets, p_offsets, q_offsets, meta = indices_data[1:] + for b in range(nbatches): + for i, r in enumerate(r_offsets): + r0, r1 = divmod(r, N) + acc = accumulators[b, r0 : r0 + Ms, r1 : r1 + Ns] + for g in range(c_indices[i], c_indices[i + 1]): + p = p_offsets[g] + q0, q1 = divmod(q_offsets[g], N) + acc += blocks[p] @ others[b, q0 : q0 + Ks, q1 : q1 + Ns] + + where ``Ns = N // meta['SPLIT_N']``, and ``M`` and ``K`` are + integer multiples of ``Ms`` and ``Ks``, respectively. + + - ``"bsr_strided_mm_compressed"`` - matrix multiplications + scattered in batches of tensors and a tensor. A memory and + processor efficient version of ``"bsr_strided_mm"`` format. If + :attr:`blocks` is a :math:`(Ms \times Ks) tensor, :attr:`others` + is a :math:`(* \times K \times N)` tensor, :attr:`accumulators` + is a :math:`(* \times M \times N)` tensor, then the operation is + equivalent to the following code:: + + c_indices, r_offsets, q_offsets, meta = indices_data[1:] + for b in range(nbatches): + for r in r_offsets: + m = (r // N) // Ms + n = (r % N) // Ns + r0, r1 = divmod(r, N) + c0, c1 = c_indices[m], c_indices[m + 1] + acc = accumulators[b, r0 : r0 + Ms, r1 : r1 + Ns] + for i, p in enumerate(range(c0, c1)): + q = q_offsets[n * c1 + (SPLIT_N - n) * c0 + i] + q0, q1 = divmod(q, N) + acc += blocks[p] @ others[b, q0 : q0 + Ks, q1 : q1 + Ns] + + where ``Ns = N // meta['SPLIT_N']``, and ``M`` and ``K`` are + integer multiples of ``Ms`` and ``Ks``, respectively. + + Notice that the order of ``r_offsets`` items can be arbitrary; + this property enables defining swizzle operators via + rearrangements of ``r_offsets`` items.. + + Auxiliary functions are provided for pre-computing + :attr:`indices_data`. For example, + :func:`bsr_scatter_mm_indices_data` is used to define indices data + for matrix multiplication of BSR and strided tensors. + + Parameters + ---------- + blocks (Tensor): a 3-D tensor of first matrices to be multiplied + + others (Tensor): a tensor of second matrices to be multiplied. If + ``indices_data[0]=="scatter_mm"``, the tensor is a 1-D batch + tensor of second input matrices to be multiplied. Otherwise, the + second input matrices are slices of the :attr:`others` tensor. + indices_data (tuple): a format data that defines the inputs and + outputs of scattered matrix multiplications. + + Keyword arguments + ----------------- + + accumulators (Tensor, optional): a tensor of matrix product + accumulators. If ``indices_data[0]=="scatter_mm"``, the tensor + is a 1-D batch tensor of output matrices. Otherwise, output + matrices are slices of the :attr:`accumulators` tensor. + """ + indices_format = indices_data[0] + + assert blocks.ndim == 3 + _P, Ms, Ks = blocks.shape + + if indices_format == "scatter_mm": + c_offsets, pq = indices_data[1:] + + assert others.ndim == 3 + _Q, Ks_, Ns = others.shape + assert Ks == Ks_ + + if accumulators is None: + R = c_offsets.shape[0] - 1 + accumulators = torch.zeros( + (R, Ms, Ns), dtype=blocks.dtype, device=blocks.device + ) + else: + R, Ms_, Ns_ = accumulators.shape + assert Ms_ == Ms + assert Ns_ == Ns + + if Ms % 16 or Ks % 16 or Ns % 16 or _scatter_mm2 is None: + for r in range(c_offsets.shape[0] - 1): + g0 = c_offsets[r] + g1 = c_offsets[r + 1] + for g in range(g0, g1): + p, q = pq[g] + + accumulators[r] += blocks[p] @ others[q] + else: + _scatter_mm2(blocks, others, c_offsets, pq, accumulators) + return accumulators + + elif indices_format == "bsr_strided_mm": + others_shape = others.shape + others = as1Dbatch(others) + + B, K, N = others.shape + assert K % Ks == 0 + + c_indices, r_offsets, p_offsets, q_offsets, meta = indices_data[1:] + SPLIT_N = meta["SPLIT_N"] + + if accumulators is None: + M = Ms + (r_offsets.max().item() + 1) // N + accumulators = torch.zeros( + (*others_shape[:-2], M, N), dtype=blocks.dtype, device=blocks.device + ) + else: + M, N_ = accumulators.shape[-2:] + assert N_ == N + + accumulators_shape = accumulators.shape + accumulators = as1Dbatch(accumulators) + + Ns = N // SPLIT_N + + if Ms % 16 or Ks % 16 or Ns % 16 or _scatter_mm6 is None: + accumulators.zero_() + for b in range(B): + for r in range(r_offsets.shape[0]): + r_ = r_offsets[r].item() + g0 = c_indices[r].item() + g1 = c_indices[r + 1].item() + r0, r1 = divmod(r_, N) + acc = accumulators[b, r0 : r0 + Ms, r1 : r1 + Ns] + for g in range(g0, g1): + p, q = p_offsets[g], q_offsets[g] + q0, q1 = divmod(q.item(), N) + acc += blocks[p] @ others[b, q0 : q0 + Ks, q1 : q1 + Ns] + else: + _scatter_mm6( + blocks, + others, + c_indices, + r_offsets, + p_offsets, + q_offsets, + meta, + accumulators, + ) + return accumulators.view(accumulators_shape) + + elif indices_format == "bsr_strided_mm_compressed": + others_shape = others.shape + others = as1Dbatch(others) + + B, K, N = others.shape + assert K % Ks == 0 + + c_indices, r_offsets, q_offsets, meta = indices_data[1:] + SPLIT_N = meta["SPLIT_N"] + + if accumulators is None: + M = Ms + (r_offsets.max().item() + 1) // N + accumulators = torch.zeros( + (*others_shape[:-2], M, N), dtype=blocks.dtype, device=blocks.device + ) + else: + M, N_ = accumulators.shape[-2:] + assert N_ == N + + accumulators_shape = accumulators.shape + accumulators = as1Dbatch(accumulators) + + Ns = N // SPLIT_N + + if Ms % 16 or Ks % 16 or Ns % 16 or _scatter_mm6 is None: + for b in range(B): + for j in range(len(r_offsets)): + r0, r1 = divmod(r_offsets[j].item(), N) + m = r0 // Ms + n = r1 // Ns + c0 = c_indices[m].item() + c1 = c_indices[m + 1].item() + acc = accumulators[b, r0 : r0 + Ms, r1 : r1 + Ns] + for i, p in enumerate(range(c0, c1)): + q = q_offsets[n * c1 + (SPLIT_N - n) * c0 + i].item() + q0, q1 = divmod(q, N) + acc += blocks[p] @ others[b, q0 : q0 + Ks, q1 : q1 + Ns] + else: + p_offsets = torch.empty( + (0,), dtype=q_offsets.dtype, device=q_offsets.device + ) + _scatter_mm6( + blocks, + others, + c_indices, + r_offsets, + p_offsets, + q_offsets, + meta, + accumulators, + ) + return accumulators.view(accumulators_shape) + + else: + raise NotImplementedError(indices_format) + + +def scatter_mm_meta( + M, + K, + N, + Ms, + Ks, + GROUP_SIZE=None, + TILE_M=None, + TILE_N=None, + SPLIT_N=None, + num_warps=None, + num_stages=None, + **extra, +): + if {TILE_M, TILE_N, SPLIT_N, num_warps, num_stages, GROUP_SIZE} == {None}: + device_name = torch.cuda.get_device_name() + meta = get_meta( + "scatter_mm", + (M, K, N, Ms, Ks), + device_name, + version=(0, torch.float16, 0.5), + ) + if meta is not None: + meta.update(**extra) + return meta + # The following parameters are optimized for the performance + # equilibrium points of bsr-dense and dense-dense matrix + # multiplications when using GPU card NVIDIA GeForce RTX 2060 + # SUPER. For points far from the performance equilibrium + # points as well as for other GPU cards, the optimal + # parameters are likely different from what specified below. + if (M, K, N) == (256,) * 3: + if (Ms, Ks) == (16, 16): + SPLIT_N = 1 + TILE_M = 16 + TILE_N = 16 + GROUP_SIZE = 4 + num_stages = 1 + num_warps = 4 # noqa: E225,E231,E702 + elif (Ms, Ks) == (32, 32): + SPLIT_N = 2 + TILE_M = 32 + TILE_N = 16 + GROUP_SIZE = 4 + num_stages = 1 + num_warps = 4 # noqa: E225,E231,E702 + elif (Ms, Ks) == (64, 64): + SPLIT_N = 1 + TILE_M = 32 + TILE_N = 32 + GROUP_SIZE = 4 + num_stages = 1 + num_warps = 4 # noqa: E225,E231,E702 + elif (Ms, Ks) == (128, 128): + SPLIT_N = 1 + TILE_M = 32 + TILE_N = 32 + GROUP_SIZE = 2 + num_stages = 1 + num_warps = 4 # noqa: E225,E231,E702 + elif (M, K, N) == (512,) * 3: + if (Ms, Ks) == (16, 16): + SPLIT_N = 8 + TILE_M = 16 + TILE_N = 64 + GROUP_SIZE = 2 + num_stages = 1 + num_warps = 2 # noqa: E225,E231,E702 + elif (Ms, Ks) == (32, 32): + SPLIT_N = 8 + TILE_M = 32 + TILE_N = 64 + GROUP_SIZE = 4 + num_stages = 1 + num_warps = 2 # noqa: E225,E231,E702 + elif (Ms, Ks) == (64, 64): + SPLIT_N = 4 + TILE_M = 32 + TILE_N = 128 + GROUP_SIZE = 4 + num_stages = 1 + num_warps = 4 # noqa: E225,E231,E702 + elif (Ms, Ks) == (128, 128): + SPLIT_N = 8 + TILE_M = 64 + TILE_N = 64 + GROUP_SIZE = 4 + num_stages = 1 + num_warps = 4 # noqa: E225,E231,E702 + elif (M, K, N) == (1024,) * 3: + if (Ms, Ks) == (16, 16): + SPLIT_N = 4 + TILE_M = 16 + TILE_N = 128 + GROUP_SIZE = 2 + num_stages = 1 + num_warps = 1 # noqa: E225,E231,E702 + elif (Ms, Ks) == (32, 32): + SPLIT_N = 8 + TILE_M = 32 + TILE_N = 64 + GROUP_SIZE = 2 + num_stages = 1 + num_warps = 1 # noqa: E225,E231,E702 + elif (Ms, Ks) == (64, 64): + SPLIT_N = 16 + TILE_M = 64 + TILE_N = 64 + GROUP_SIZE = 4 + num_stages = 1 + num_warps = 2 # noqa: E225,E231,E702 + elif (Ms, Ks) == (128, 128): + SPLIT_N = 16 + TILE_M = 64 + TILE_N = 64 + GROUP_SIZE = 4 + num_stages = 1 + num_warps = 4 # noqa: E225,E231,E702 + elif (Ms, Ks) == (256, 256): + SPLIT_N = 16 + TILE_M = 64 + TILE_N = 64 + GROUP_SIZE = 2 + num_stages = 1 + num_warps = 4 # noqa: E225,E231,E702 + elif (M, K, N) == (2048,) * 3: + if (Ms, Ks) == (16, 16): + SPLIT_N = 4 + TILE_M = 16 + TILE_N = 128 + GROUP_SIZE = 8 + num_stages = 1 + num_warps = 1 # noqa: E225,E231,E702 + elif (Ms, Ks) == (32, 32): + SPLIT_N = 4 + TILE_M = 32 + TILE_N = 64 + GROUP_SIZE = 4 + num_stages = 1 + num_warps = 1 # noqa: E225,E231,E702 + elif (Ms, Ks) == (64, 64): + SPLIT_N = 4 + TILE_M = 64 + TILE_N = 128 + GROUP_SIZE = 4 + num_stages = 1 + num_warps = 4 # noqa: E225,E231,E702 + elif (Ms, Ks) == (128, 128): + SPLIT_N = 8 + TILE_M = 64 + TILE_N = 64 + GROUP_SIZE = 4 + num_stages = 1 + num_warps = 4 # noqa: E225,E231,E702 + elif (Ms, Ks) == (256, 256): + SPLIT_N = 4 + TILE_M = 64 + TILE_N = 64 + GROUP_SIZE = 2 + num_stages = 1 + num_warps = 4 # noqa: E225,E231,E702 + elif (M, K, N) == (4096,) * 3: + if (Ms, Ks) == (16, 16): + SPLIT_N = 2 + TILE_M = 16 + TILE_N = 256 + GROUP_SIZE = 2 + num_stages = 1 + num_warps = 2 # noqa: E225,E231,E702 + elif (Ms, Ks) == (32, 32): + SPLIT_N = 2 + TILE_M = 32 + TILE_N = 64 + GROUP_SIZE = 2 + num_stages = 1 + num_warps = 1 # noqa: E225,E231,E702 + elif (Ms, Ks) == (64, 64): + SPLIT_N = 2 + TILE_M = 64 + TILE_N = 128 + GROUP_SIZE = 2 + num_stages = 1 + num_warps = 4 # noqa: E225,E231,E702 + + if SPLIT_N is None: + # Assume NVIDIA GeForce RTX 2060 SUPER: + # With the probality of 92% (99.9% when N > 512), the + # performance will not be worse more than 2% from the + # performance when using an optimal value. Otherwise, when N + # <= 512, using the following heuristics may give upto 15% + # lower performance. + SPLIT_N = { + 16: 1, + 32: 2, + 64: 4, + 128: 8, + 256: 16, + 512: 8, + 1024: 16, + 4096: 32, + 8192: 64, + }.get(N, 16) + if Ms >= 512 and N >= 2048: + SPLIT_N = 1 + Ns = N // SPLIT_N + if TILE_M is None: + TILE_M = min(64 if Ns < 512 else 32, Ms) + if TILE_N is None: + TILE_N = min(64 if Ns < 512 else 32, Ns) + num_stages = num_stages or 1 + if num_warps is None: + if min(M, N) > 1024: + num_warps = {16: 1, 32: 1, 64: 2}.get(Ms, 4) + elif min(M, N) == 1024: + num_warps = {16: 1, 32: 1, 64: 2}.get(Ms, 4) + elif min(M, N) == 256: + num_warps = {16: 1, 32: 4}.get(Ms, 4) + else: + num_warps = {16: 1, 32: 2}.get(Ms, 4) + GROUP_SIZE = GROUP_SIZE or 4 + + assert TILE_M <= Ms, dict(TILE_M=TILE_M, Ms=Ms) + assert TILE_N <= Ns, dict(TILE_N=TILE_N, Ns=Ns) + assert Ms <= M, dict(M=M, Ms=Ms) + assert Ns <= N, dict(N=N, Ns=Ns) + assert Ks <= K, dict(K=K, Ks=Ks) + + return dict( + TILE_M=TILE_M, + TILE_N=TILE_N, + GROUP_SIZE=GROUP_SIZE, + num_stages=num_stages, + num_warps=num_warps, + SPLIT_N=SPLIT_N, + **extra, + ) + + +def bsr_dense_addmm_meta( + M, + K, + N, + Ms, + Ks, + beta, + alpha, + SPLIT_N=None, + GROUP_SIZE_ROW=None, + num_warps=None, + num_stages=None, + sparsity=None, + dtype=None, + out_dtype=None, + _version=0, + **extra, +): + # Specifying _version is useful for situations when one wants to + # discard existing triton kernel tuning results, say, in testing + # bsr_dense_addmm_meta functionality. + if dtype is None: + dtype = torch.float16 + if out_dtype is None: + out_dtype = dtype + if sparsity is None: + sparsity = 0.5 + if {SPLIT_N, num_warps, num_stages, GROUP_SIZE_ROW} == {None}: + device_name = torch.cuda.get_device_name() + key = (M, K, N, Ms, Ks, beta == 0, beta == 1, alpha == 1) + if dtype is out_dtype: + version_dtype = dtype + else: + version_dtype = dtype, out_dtype + meta = get_meta( + "bsr_dense_addmm", + key, + device_name, + version=(_version, version_dtype, sparsity), + ) + if meta is None and sparsity != 0.5: + meta = get_meta( + "bsr_dense_addmm", + key, + device_name, + version=(_version, version_dtype, 0.5), + ) + if meta is None and dtype is not out_dtype: + meta = get_meta( + "bsr_dense_addmm", key, device_name, version=(_version, dtype, 0.5) + ) + if meta is None: + # find approximate meta such that N % SPLIT_N == 0. + matching_meta = get_meta( + "bsr_dense_addmm", + (*key[:2], "*", *key[3:]), + device_name, + version=(_version, version_dtype, 0.5), + ) + if matching_meta is None and dtype is not out_dtype: + matching_meta = get_meta( + "bsr_dense_addmm", + (*key[:2], "*", *key[3:]), + device_name, + version=(_version, dtype, 0.5), + ) + for mkey in sorted(matching_meta or {}): + meta_ = matching_meta[mkey] + n = mkey[2] + split_n = meta_["SPLIT_N"] + c = n // split_n + if N % c == 0 and n <= N: + meta = dict(meta_) + meta["SPLIT_N"] = N // c + if meta is not None: + meta.update(**extra) + return meta + else: + # see [Computing optimal kernel parameters] in + # _triton_ops_meta.py for ways to avoid this warning + # message + warn_once( + "bsr_dense_addmm uses non-optimal triton kernel parameters" + f" for {M=} {K=} {N=} {Ms=}, {Ks=} {beta=} {alpha=} {dtype=} {out_dtype=}" + ) + + SPLIT_N = SPLIT_N or max(N // Ms, 1) + GROUP_SIZE_ROW = GROUP_SIZE_ROW or 4 + num_stages = num_stages or 1 + num_warps = num_warps or 4 + return dict( + SPLIT_N=SPLIT_N, + GROUP_SIZE_ROW=GROUP_SIZE_ROW, + num_stages=num_stages, + num_warps=num_warps, + **extra, + ) + + +class TensorAsKey: + """A light-weight wrapper of a tensor that enables storing tensors as + keys with efficient memory reference based comparison as an + approximation to data equality based keys. + + Motivation: the hash value of a torch tensor is tensor instance + based that does not use data equality and makes the usage of + tensors as keys less useful. For instance, the result of + ``len({a.crow_indices(), a.crow_indices()})`` is `2`, although, + the tensor results from `crow_indices` method call are equal, in + fact, these share the same data storage. + On the other hand, for efficient caching of tensors we want to + avoid calling torch.equal that compares tensors item-wise. + + TensorAsKey offers a compromise in that it guarantees key equality + of tensors that references data in the same storage in the same + manner and without accessing underlying data. However, this + approach does not always guarantee correctness. For instance, for + a complex tensor ``x``, we have ``TensorAsKey(x) == + TensorAsKey(x.conj())`` while ``torch.equal(x, x.conj())`` would + return False. + """ + + def __init__(self, obj): + def get_tensor_key(obj): + # Warning: TensorAsKey does not track negative nor + # conjugate bits of its input object because in the use + # case of wrapping compressed/plain indices of compressed + # sparse tensors (that are always integer tensors with + # non-negative items) these bits are never set. However, + # when extending the use of TensorAsKey to float or + # complex tensors, the values of these bits (see is_neg + # and is_conj methods) must be included in the key as + # well. + assert not (obj.dtype.is_floating_point or obj.dtype.is_complex), obj.dtype + return ( + obj.data_ptr(), + obj.storage_offset(), + obj.shape, + obj.stride(), + obj.dtype, + ) + + self._obj_ref = weakref.ref(obj) + if obj.layout is torch.strided: + self.key = get_tensor_key(obj) + elif obj.layout in {torch.sparse_csr, torch.sparse_bsr}: + self.key = ( + get_tensor_key(obj.crow_indices()), + get_tensor_key(obj.col_indices()), + ) + elif obj.layout in {torch.sparse_csc, torch.sparse_bsc}: + self.key = ( + get_tensor_key(obj.ccol_indices()), + get_tensor_key(obj.row_indices()), + ) + else: + raise NotImplementedError(obj.layout) + self._hash = hash(self.key) + + def __hash__(self): + return self._hash + + def __eq__(self, other): + if not isinstance(other, TensorAsKey): + return False + if self.obj is None or other.obj is None: + # dead objects always compare unequal unless these are + # same objects + return self is other + return self.key == other.key + + @property + def obj(self): + """Return object if alive, otherwise None.""" + return self._obj_ref() + + +@lru_cache(maxsize=TORCH_SPARSE_BSR_SCATTER_MM_LRU_CACHE_SIZE) +def _bsr_scatter_mm_indices_data( + indices_format, M, K, N, Ms, Ks, nbatches, SPLIT_N, compressed_sparse_tensor_as_key +): + bsr = compressed_sparse_tensor_as_key.obj + assert bsr is not None + crow_indices, col_indices = bsr.crow_indices(), bsr.col_indices() + device = crow_indices.device + indices_dtype = torch.int32 + + if indices_format == "bsr_strided_mm_compressed": + Ns = N // SPLIT_N + q_offsets_lst = [] + b = torch.arange(SPLIT_N, dtype=indices_dtype, device=device) * Ns + for m in range(M // Ms): + r0 = crow_indices[m].item() + r1 = crow_indices[m + 1].item() + if r1 == r0: + continue + q_offsets_lst.append( + (col_indices[r0:r1] * (Ks * N)).repeat(SPLIT_N) + + b.repeat_interleave(r1 - r0) + ) + q_offsets = torch.cat(q_offsets_lst) + crow_indices_diff = crow_indices.diff() + non_zero_row_indices = crow_indices_diff.nonzero() + a = non_zero_row_indices * (Ms * N) + r_offsets = (a + b).view(-1) + c_indices = crow_indices + # swizzle operation: mm elements with longer sums are computed first: + nnz_per_row = crow_indices_diff[non_zero_row_indices].repeat_interleave(SPLIT_N) + nnz_per_row, indices = nnz_per_row.sort(descending=True, stable=True) + r_offsets = r_offsets[indices] + return (indices_format, c_indices, r_offsets, q_offsets) + + elif indices_format == "bsr_strided_mm": + Ns = N // SPLIT_N + p_offsets_lst = [] + q_offsets_lst = [] + b = torch.arange(SPLIT_N, dtype=indices_dtype, device=device) * Ns + for m in range(M // Ms): + r0 = crow_indices[m].item() + r1 = crow_indices[m + 1].item() + if r1 == r0: + continue + p_offsets_lst.append( + torch.arange(r0, r1, dtype=indices_dtype, device=device).repeat(SPLIT_N) + ) + q_offsets_lst.append( + (col_indices[r0:r1] * (Ks * N)).repeat(SPLIT_N) + + b.repeat_interleave(r1 - r0) + ) + q_offsets = torch.cat(q_offsets_lst) + crow_indices_diff = crow_indices.diff() + non_zero_row_indices = crow_indices_diff.nonzero() + a = non_zero_row_indices * (Ms * N) + r_offsets = (a + b).view(-1) + c_indices = torch.cat( + ( + crow_indices[:1], + torch.cumsum( + crow_indices_diff[non_zero_row_indices].repeat_interleave(SPLIT_N), + 0, + ), + ) + ) + p_offsets = torch.cat(p_offsets_lst) + return (indices_format, c_indices, r_offsets, p_offsets, q_offsets) + + elif indices_format == "scatter_mm": + Ns = Ms + c_indices = [0] + pq_offsets = [] + # todo: eliminate inner for-loops for efficiency + for b in range(nbatches): + for m in range(M // Ms): + r0 = crow_indices[m].item() + r1 = crow_indices[m + 1].item() + for n in range(N // Ns): + c_indices.append(c_indices[-1] + r1 - r0) + for t in range(r1 - r0): + p = r0 + t + q = (col_indices[p].item() + b * (K // Ks)) * (N // Ns) + n + pq_offsets.append([p, q]) + + return ( + indices_format, + torch.tensor(c_indices, dtype=indices_dtype, device=device), + torch.tensor(pq_offsets, dtype=indices_dtype, device=device), + ) + + else: + raise ValueError( + f"Invalid {indices_format=}. Expected bsr_strided_mm_compressed|bsr_strided_mm|scatter_mm" + ) + + +def bsr_scatter_mm_indices_data( + bsr, other, indices_format="bsr_strided_mm_compressed", **meta_input +): + """Computes indices data for :func:`scatter_mm` used in BSR and + strided tensor matrix multiplication. + """ + assert bsr.dense_dim() == 0 + assert bsr.ndim == 2 # no batch dims + blocksize = bsr.values().shape[-2:] + M, K = bsr.shape + Ms, Ks = blocksize + K_, N = other.shape[-2:] + assert K_ == K + nbatches = other.shape[:-2].numel() + + meta = scatter_mm_meta(M, K, N, Ms, Ks, **meta_input) + if "allow_tf32" not in meta_input: + meta.update(allow_tf32=bsr.dtype in {torch.float16, torch.bfloat16}) + SPLIT_N = meta["SPLIT_N"] + indices_data = _bsr_scatter_mm_indices_data( + indices_format, M, K, N, Ms, Ks, nbatches, SPLIT_N, TensorAsKey(bsr) + ) + + if indices_format == "bsr_strided_mm_compressed": + meta.update(is_compressed=True) + return indices_data + (meta,) + elif indices_format == "bsr_strided_mm": + meta.update(is_compressed=False) + return indices_data + (meta,) + else: + return indices_data + + +def bsr_scatter_mm(bsr, other, indices_data=None, out=None): + """BSR @ strided -> strided""" + + assert bsr.ndim == 2 + assert other.ndim >= 2 + + Ms, Ks, Ns = bsr.shape[-2], bsr.shape[-1], other.shape[-1] + blocksize = bsr.values().shape[-2:] + + if indices_data is None: + indices_data = bsr_scatter_mm_indices_data( + bsr, other, indices_format="bsr_strided_mm_compressed" + ) + + indices_format = indices_data[0] + + if out is None: + out = torch.empty( + (*other.shape[:-2], Ms, Ns), dtype=bsr.dtype, device=bsr.device + ) + out_shape = out.shape + out = as1Dbatch(out) + + if bsr._nnz() == 0: + out.zero_() + elif indices_format in {"bsr_strided_mm_compressed", "bsr_strided_mm"}: + out.zero_() + scatter_mm(bsr.values(), other, indices_data, accumulators=out) + elif indices_format == "scatter_mm": + nbatches = other.shape[:-2].numel() + accumulators = torch.zeros( + ( + nbatches * Ms // blocksize[0] * Ns // blocksize[0], + blocksize[0], + blocksize[0], + ), + dtype=bsr.dtype, + device=bsr.device, + ) + others = ( + as1Dbatch(other) + .transpose(-2, -1) + .view( + nbatches, + Ns // blocksize[0], + blocksize[0], + Ks // blocksize[1], + blocksize[1], + ) + .movedim( + (3, 1, 4, 2), (1, 2, 3, 4) + ) # equivalent to .transpose(-3, -2).transpose(-2, -1).transpose(-4, -3) + .flatten(0, 2) + ) + scatter_mm(bsr.values(), others, indices_data, accumulators=accumulators) + out.copy_( + accumulators.unflatten( + 0, (nbatches, Ms // blocksize[0], Ns // blocksize[0]) + ) + .movedim( + (1, 2, 3, 4), (3, 1, 4, 2) + ) # equivalent to .transpose(-4, -3).transpose(-2, -1).transpose(-3, -2) + .reshape(nbatches, Ns, Ms) + .transpose(-2, -1) + ) + else: + raise NotImplementedError(indices_format) + + return out.view(out_shape) + + +def _int_bsr_dense_addmm( + input: torch.Tensor, + bsr: torch.Tensor, + dense: torch.Tensor, + *, + beta=1, + alpha=1, + left_alpha: torch.Tensor | None = None, + right_alpha: torch.Tensor | None = None, + out: torch.Tensor | None = None, + skip_checks: bool = False, + max_grid: tuple[int | None, int | None, int | None] | None = None, + meta: dict | None = None, +): + if out is None and dense.dtype is torch.int8: + f_name = "_int_bsr_dense_addmm" + crow_indices = bsr.crow_indices() + batch_ndim = crow_indices.dim() - 1 + M = bsr.shape[batch_ndim] + N = dense.shape[-1] + original_batch_dims_broadcasted = broadcast_batch_dims(f_name, bsr, dense) + out = torch.empty( + original_batch_dims_broadcasted + (M, N), + dtype=torch.int32, + device=dense.device, + ) + return bsr_dense_addmm( + input, + bsr, + dense, + beta=beta, + alpha=alpha, + left_alpha=left_alpha, + right_alpha=right_alpha, + out=out, + skip_checks=skip_checks, + max_grid=max_grid, + meta=meta, + ) + + +def bsr_dense_addmm( + input: torch.Tensor, + bsr: torch.Tensor, + dense: torch.Tensor, + *, + beta=1, + alpha=1, + left_alpha: torch.Tensor | None = None, + right_alpha: torch.Tensor | None = None, + out: torch.Tensor | None = None, + skip_checks: bool = False, + max_grid: tuple[int | None, int | None, int | None] | None = None, + meta: dict | None = None, +): + """Compute + + out = beta * input + left_alpha.reshape(-1, 1) * (alpha * (bsr @ dense)) * right_alpha.reshape(1, -1) + + where left_alpha, right_alpha are (* + 1)-D tensors when + specified, otherwise, these are treated as tensors filled with + ones. + """ + f_name = "bsr_dense_addmm" + values = bsr.values() + crow_indices = bsr.crow_indices() + col_indices = bsr.col_indices() + batch_ndim = crow_indices.dim() - 1 + M, K = bsr.shape[batch_ndim : batch_ndim + 2] + blocksize = values.shape[batch_ndim + 1 : batch_ndim + 3] + N = dense.shape[-1] + + # todo: implement checks + + original_batch_dims_broadcasted = broadcast_batch_dims(f_name, bsr, dense) + if out is None: + out = dense.new_empty(original_batch_dims_broadcasted + (M, N)) + + if bsr._nnz() == 0 or alpha == 0 or N == 0 or M == 0 or K == 0: + if beta == 0: + out.zero_() + else: + out.copy_(input) + if beta != 1: + out.mul_(beta) + return out + + left_alpha_is_one = False + right_alpha_is_one = False + if left_alpha is None: + left_alpha_is_one = True + left_alpha = dense.new_empty(()).expand( + *original_batch_dims_broadcasted, M, N + ) # not referenced + else: + left_alpha = left_alpha.view(*original_batch_dims_broadcasted, M, 1).expand( + *original_batch_dims_broadcasted, M, N + ) + + if right_alpha is None: + right_alpha_is_one = True + right_alpha = dense.new_empty(()).expand( + *original_batch_dims_broadcasted, M, N + ) # not referenced + else: + right_alpha = right_alpha.view(*original_batch_dims_broadcasted, 1, N).expand( + *original_batch_dims_broadcasted, M, N + ) + assert left_alpha.stride()[-1] == 0 + assert right_alpha.stride()[-2] == 0 + + if meta is None: + sparsity = round(1 - bsr._nnz() * blocksize[0] * blocksize[1] / (M * K), 2) + meta = bsr_dense_addmm_meta( + M, + K, + N, + blocksize[0], + blocksize[1], + beta, + alpha, + sparsity=sparsity, + dtype=dense.dtype, + out_dtype=out.dtype, + ) + out_backup = out + + ( + crow_indices, + col_indices, + values, + input, + dense, + left_alpha, + right_alpha, + out, + ) = prepare_inputs(bsr, input, dense, left_alpha, right_alpha, out) + + BM, BK = blocksize + SPLIT_N = meta.get("SPLIT_N", N // BM) + BN = N // SPLIT_N + + out_untiled = out + out = tile_to_blocksize(out, (BM, BN)) + dense = tile_to_blocksize(dense, (BK, BN)) + input = tile_to_blocksize(input, (BM, BN)) + left_alpha = tile_to_blocksize(left_alpha, (BM, BN)) + right_alpha = tile_to_blocksize(right_alpha, (BM, BN)) + + # tl.dot supports float16, float32, int32 as accumulator types. + dot_out_dtype = { + torch.float16: tl.float32, + torch.bfloat16: tl.float32, + torch.float32: tl.float64, + torch.float64: tl.float64, + torch.int8: tl.int32, + torch.int32: tl.int32, + }[out.dtype] + + n_batches = dense.size(0) + n_block_rows = crow_indices.size(-1) - 1 + n_block_cols = dense.size(-3) + + full_grid = (n_batches, n_block_cols, n_block_rows) + if max_grid is not None: + grid_blocks = tuple(max_grid[:3][::-1]) + (None,) * (3 - len(max_grid[:3])) + else: + grid_blocks = None + + tensor_dims_map = { + values: (0, None, None), + crow_indices: (0, None, -1), + col_indices: (0, None, None), + input: (0, -3, -4), + dense: (0, -3, None), + left_alpha: (0, -3, -4), + right_alpha: (0, -3, -4), + out: (0, -3, -4), + } + + assert alpha != 0 + + def kernel(grid, *sliced_tensors): + # pyrefly: ignore [unsupported-operation] + _bsr_strided_addmm_kernel[grid]( + *ptr_stride_extractor(*sliced_tensors), + # pyrefly: ignore # bad-argument-count + beta, + alpha, + # pyrefly: ignore # bad-keyword-argument, bad-argument-type + beta_is_one=beta == 1, + # pyrefly: ignore # bad-keyword-argument, bad-argument-type + beta_is_nonzero=beta != 0, + # pyrefly: ignore # bad-keyword-argument, bad-argument-type + alpha_is_one=alpha == 1, + # pyrefly: ignore # bad-keyword-argument, bad-argument-type + left_alpha_is_one=left_alpha_is_one, + # pyrefly: ignore # bad-keyword-argument, bad-argument-type + right_alpha_is_one=right_alpha_is_one, + # pyrefly: ignore # bad-keyword-argument, bad-argument-type + BLOCKSIZE_ROW=BM, + # pyrefly: ignore # bad-keyword-argument, bad-argument-type + BLOCKSIZE_INNER=BK, + # pyrefly: ignore # bad-keyword-argument + BLOCKSIZE_COL=BN, + # pyrefly: ignore # bad-keyword-argument + allow_tf32=dot_out_dtype == tl.float32, + # pyrefly: ignore # bad-keyword-argument, bad-argument-type + acc_dtype=dot_out_dtype, + **meta, + ) + + launch_kernel(kernel, tensor_dims_map, full_grid, grid_blocks) + + if out.data_ptr() != out_backup.data_ptr(): + # prepare_inputs has made a copy of out, copy its content back + # to out_backup: + out_backup.copy_(out_untiled.view(out_backup.shape)) + + return out_backup + + +if has_triton(): + import triton + import triton.language as tl + + @triton.jit + def _sampled_addmm_kernel( + alpha, + beta, + IS_BETA_ZERO: tl.constexpr, + BLOCKSIZE_ROW: tl.constexpr, + BLOCKSIZE_COL: tl.constexpr, + k, + TILE_K: tl.constexpr, + values_ptr, + values_batch_stride, + values_nnz_stride, + values_row_block_stride, + values_col_block_stride, + crow_indices_ptr, + crow_indices_batch_stride, + crow_indices_stride, + col_indices_ptr, + col_indices_batch_stride, + col_indices_stride, + mat1_ptr, + mat1_batch_stride, + mat1_tiled_row_stride, + mat1_tiled_col_stride, + mat1_row_block_stride, + mat1_col_block_stride, + mat2_ptr, + mat2_batch_stride, + mat2_tiled_row_stride, + mat2_tiled_col_stride, + mat2_row_block_stride, + mat2_col_block_stride, + acc_dtype: tl.constexpr, + allow_tf32: tl.constexpr, + ): + batch_pid = tl.program_id(axis=1) + row_block_pid = tl.program_id(axis=0) + + crow_indices_offset_ptr = ( + crow_indices_ptr + + crow_indices_batch_stride * batch_pid + + crow_indices_stride * row_block_pid + ) + nnz_offset = tl.load(crow_indices_offset_ptr) + nnz_offset_next = tl.load(crow_indices_offset_ptr + crow_indices_stride) + + # Compute nnz for the row with number row_block_pid. + # If it is zero, skip the row. + row_nnz = nnz_offset_next - nnz_offset + if row_nnz == 0: + return + + row_block_arange = tl.arange(0, BLOCKSIZE_ROW) + col_block_arange = tl.arange(0, BLOCKSIZE_COL) + + # Pointers are set to the first block of the current row. + values_block_ptrs = ( + values_ptr + + values_batch_stride * batch_pid + + values_nnz_stride * nnz_offset + + values_row_block_stride * row_block_arange[:, None] + + values_col_block_stride * col_block_arange[None, :] + ) + + col_index_nnz_ptr = ( + col_indices_ptr + + col_indices_batch_stride * batch_pid + + col_indices_stride * nnz_offset + ) + + # Advance mat1 to the current tiled row, ignore columns. + mat1_block_ptrs = ( + mat1_ptr + + mat1_batch_stride * batch_pid + + mat1_tiled_row_stride * row_block_pid + + mat1_row_block_stride * row_block_arange[:, None] + ) + + # Advance mat2 in batch and block col dimension. + mat2_block_ptrs = ( + mat2_ptr + + mat2_batch_stride * batch_pid + + mat2_col_block_stride * col_block_arange[None, :] + ) + + k_tile_arange = tl.arange(0, TILE_K) + for _ in range(row_nnz): + acc_block = tl.zeros((BLOCKSIZE_ROW, BLOCKSIZE_COL), dtype=acc_dtype) + + # find column block index + col_block = tl.load(col_index_nnz_ptr) + + for k_tile in range(0, k, TILE_K): + k_offsets = k_tile + k_tile_arange + mask_k = k_offsets < k + + mat1_block = tl.load( + mat1_block_ptrs + mat1_col_block_stride * k_offsets[None, :], + # pyrefly: ignore [index-error] + mask=mask_k[None, :], + other=0.0, + ) + + mat2_block = tl.load( + mat2_block_ptrs + + mat2_tiled_col_stride * col_block + + mat2_row_block_stride * k_offsets[:, None], + # pyrefly: ignore [index-error] + mask=mask_k[:, None], + other=0.0, + ) + + acc_block += tl.dot( + mat1_block, mat2_block, allow_tf32=allow_tf32, out_dtype=acc_dtype + ) + + if IS_BETA_ZERO: + acc_block *= alpha + else: + acc_block = alpha * acc_block + beta * tl.load(values_block_ptrs) + + # write result + tl.store(values_block_ptrs, acc_block.to(values_ptr.dtype.element_ty)) + + # advance val/col_index ptrs to the next block in the row. + values_block_ptrs += values_nnz_stride + col_index_nnz_ptr += col_indices_stride + + @triton.jit + def _bsr_strided_dense_rowspace_kernel( + # values prologue + values_ptr, + values_batch_stride, + values_nnz_stride, + values_row_block_stride, + values_col_block_stride, + # values epilogue + # crow_indices prologue + crow_indices_ptr, + crow_indices_batch_stride, + crow_indices_stride, + # crow_indices epilogue + # col_indices prologue + col_indices_ptr, + col_indices_batch_stride, + col_indices_stride, + # col_indices epilogue + # dense prologue + dense_ptr, + dense_batch_stride, + dense_tiled_row_stride, + dense_tiled_col_stride, + dense_row_block_stride, + dense_col_block_stride, + # dense epilogue + # output prologue + output_ptr, + output_batch_stride, + output_tiled_row_stride, + output_tiled_col_stride, + output_row_block_stride, + output_col_block_stride, + # output epilogue + # + # gh-113754: Always keep all constexpr arguments at the end of + # triton kernel arguments list because with triton 2.1 or + # earlier non-contiguous outputs will corrupt CUDA state due + # to a triton bug (fixed in openai/triton#2262). + BLOCKSIZE_ROW: tl.constexpr, + BLOCKSIZE_COL: tl.constexpr, + acc_dtype: tl.constexpr, + allow_tf32: tl.constexpr, + GROUP_SIZE_ROW: tl.constexpr, + ): + batch_pid = tl.program_id(axis=2) + row_block_pid = tl.program_id(axis=0) + col_block_pid = tl.program_id(axis=1) + n_block_rows = tl.num_programs(axis=0) + n_block_cols = tl.num_programs(axis=1) + + row_block_pid, col_block_pid = tl.swizzle2d( + row_block_pid, col_block_pid, n_block_rows, n_block_cols, GROUP_SIZE_ROW + ) + + crow_indices_offset_ptr = ( + crow_indices_ptr + + crow_indices_batch_stride * batch_pid + + crow_indices_stride * row_block_pid + ) + nnz_offset = tl.load(crow_indices_offset_ptr) + nnz_offset_next = tl.load(crow_indices_offset_ptr + crow_indices_stride) + + # Compute nnz for the row with number row_block_pid. + # If it is zero, skip the row. + row_nnz = nnz_offset_next - nnz_offset + if row_nnz == 0: + return + + row_block_arange = tl.arange(0, BLOCKSIZE_ROW) + col_block_arange = tl.arange(0, BLOCKSIZE_COL) + + # Pointers are set to the first block of the current row. + values_block_ptrs = ( + values_ptr + + values_batch_stride * batch_pid + + values_nnz_stride * nnz_offset + + values_row_block_stride * row_block_arange[:, None] + + values_col_block_stride * col_block_arange[None, :] + ) + + # NOTE: dense is advanced into all dimensions but the tiled row one. + # That will be advanced in the loop according to values in col_indices. + dense_block_ptrs = ( + dense_ptr + + dense_batch_stride * batch_pid + + dense_tiled_col_stride * col_block_pid + + dense_row_block_stride * col_block_arange[:, None] + + dense_col_block_stride * row_block_arange[None, :] + ) + + # Pointers are set to exact write-to locations + output_ptrs = ( + output_ptr + + output_batch_stride * batch_pid + + output_tiled_row_stride * row_block_pid + + output_tiled_col_stride * col_block_pid + + output_row_block_stride * row_block_arange[:, None] + + output_col_block_stride * row_block_arange[None, :] + ) + + # Set pointer to the first nonzero element in the current row + col_index_nnz_ptr = ( + col_indices_ptr + + col_indices_batch_stride * batch_pid + + col_indices_stride * nnz_offset + ) + + output_acc_block = tl.zeros((BLOCKSIZE_ROW, BLOCKSIZE_COL), dtype=acc_dtype) + for _ in range(row_nnz): + values_block = tl.load(values_block_ptrs) + + # find which row of dense needs to get loaded + # for multiplication with values_block. + dense_row_idx = tl.load(col_index_nnz_ptr) + dense_block = tl.load( + dense_block_ptrs + dense_tiled_row_stride * dense_row_idx + ) + + # do block mm + output_acc_block += tl.dot( + values_block, dense_block, allow_tf32=allow_tf32, out_dtype=acc_dtype + ) + + # move val/col_index ptrs to the next block in the row + values_block_ptrs += values_nnz_stride + col_index_nnz_ptr += col_indices_stride + + # write back the result + tl.store(output_ptrs, output_acc_block.to(output_ptr.dtype.element_ty)) + + def _run_sampled_addmm_kernel( + alpha, + beta, + is_beta_zero, + blocksize, + k, + tile_k, + values, + crow_indices, + col_indices, + mat1, + mat2, + max_grid, + ): + n_batches = values.size(0) + n_block_rows = crow_indices.size(-1) - 1 + + full_grid = (n_batches, n_block_rows) + if max_grid is not None: + grid_blocks = tuple(max_grid[:2][::-1]) + (None,) * (2 - len(max_grid[:2])) + else: + grid_blocks = None + tensor_dims_map = { + values: (0, None), + crow_indices: (0, -1), + col_indices: (0, None), + mat1: (0, -4), + mat2: (0, None), + } + if values.dtype in (torch.half, torch.bfloat16): + acc_dtype = tl.float32 + allow_tf32 = True + else: + acc_dtype = tl.float64 + allow_tf32 = False + + def kernel(grid, *sliced_tensors): + _sampled_addmm_kernel[grid]( + alpha, + beta, + is_beta_zero, + *blocksize, + # pyrefly: ignore # bad-argument-count + k, + tile_k, + *ptr_stride_extractor(*sliced_tensors), + # pyrefly: ignore # bad-keyword-argument, bad-argument-type + acc_dtype=acc_dtype, + # pyrefly: ignore # bad-keyword-argument, bad-argument-type + allow_tf32=allow_tf32, + # pyrefly: ignore # unexpected-keyword + num_stages=1, + # pyrefly: ignore # unexpected-keyword + num_warps=4, + ) + + launch_kernel(kernel, tensor_dims_map, full_grid, grid_blocks) + + def sampled_addmm( + input: torch.Tensor, + mat1: torch.Tensor, + mat2: torch.Tensor, + *, + beta=1.0, + alpha=1.0, + out: torch.Tensor | None = None, + skip_checks: bool = False, + max_grid: tuple[int | None, int | None, int | None] | None = None, + ): + f_name = "sampled_addmm" + + check_bsr_layout(f_name, input) + input_broadcasted = broadcast_batch_dims_bsr(f_name, input, mat1, mat2) + + if not skip_checks: + check_device(f_name, mat1, input.device) + check_device(f_name, mat2, input.device) + if beta != 0.0 and input.dtype is torch.bool: + check( + False, + f"{f_name}(): having beta == {beta} not equal to 0.0 with boolean mask is not allowed.", + ) + if input.dtype is not torch.bool: + check_dtype(f_name, mat1, input.dtype) + check_dtype(f_name, mat2, input.dtype) + else: + check_dtype(f_name, mat1, mat2.dtype) + check_mm_compatible_shapes(f_name, mat1, mat2) + if out is not None: + check_bsr_layout(f_name, out) + check_device(f_name, out, mat1.device) + check_dtype(f_name, out, input.dtype) + check( + out.shape == input_broadcasted.shape and out._nnz() == input._nnz(), + f"{f_name}(): Expects `out` to be of shape {input_broadcasted.shape} " + f"and with nnz equal to {input_broadcasted._nnz()} " + f"but got out.shape = {out.shape} and out.nnz = {out._nnz()}", + ) + + if out is None: + out = input_broadcasted.to(mat1.dtype, copy=True) + else: + out.copy_(input_broadcasted) + + if out.numel() == 0 or out._nnz() == 0: + return out + + blocksize = out.values().shape[-2:] + k = mat1.size(-1) + + # NOTE: (m, 0) @ (0, n) == zeros(m, n) + if alpha == 0.0 or k == 0: + out.values().mul_(beta) + return out + + # prepare inputs by reshaping them to be kernel-compatible + out_backup = out + crow_indices, col_indices, values, mat1, mat2 = prepare_inputs(out, mat1, mat2) + + mat1 = tile_to_blocksize(mat1, (blocksize[0], k)) + mat2 = tile_to_blocksize(mat2, (k, blocksize[1])) + tile_k = max(*blocksize) + + _run_sampled_addmm_kernel( + alpha, + beta, + beta == 0.0, + blocksize, + k, + tile_k, + values, + crow_indices, + col_indices, + mat1, + mat2, + max_grid, + ) + + # If nnz x block strides are not the same in out_backup.values and values, + # it means that out_backup.values and values are not the views of each other, + # so we have to copy. + if out_backup.values().stride()[-3:] != values.stride()[-3:]: + out_backup.values().copy_(values.reshape(out_backup.values().shape)) + return out_backup + + def bsr_dense_mm( + bsr: torch.Tensor, + dense: torch.Tensor, + *, + out: torch.Tensor | None = None, + skip_checks: bool = False, + max_grid: tuple[int | None, int | None, int | None] | None = None, + meta: dict | None = None, + ): + f_name = "bsr_dense_mm" + m, _kl = bsr.shape[-2:] + if not skip_checks: + check_bsr_layout(f_name, bsr) + check_device(f_name, bsr, dense.device) + check_dtype(f_name, bsr, dense.dtype, (torch.int8,)) + check_mm_compatible_shapes(f_name, bsr, dense) + + n = dense.size(-1) + row_block, col_block = bsr.values().shape[-2:] + check_blocksize(f_name, (row_block, col_block)) + check( + not n % 16, + f"{f_name}(): dense.size(-1) == {n} should be divisible by 16", + ) + else: + _kr, n = dense.shape[-2:] + + original_batch_dims_broadcasted = broadcast_batch_dims(f_name, bsr, dense) + + if out is not None and not skip_checks: + expected_out_shape = original_batch_dims_broadcasted + (m, n) + check( + out.shape == expected_out_shape, + "bsr_dense_mm(): `out` argument has wrong shape, " + f"expected {expected_out_shape}, but got {out.shape}.", + ) + check( + out.is_contiguous() or out.transpose(-2, -1).is_contiguous(), + "bsr_dense_mm(): only row-major/col-major `out` arguments are supported, " + "i.e. (out.is_contiguous() or out.transpose(-2, -1).is_contiguous()) " + "should be True.", + ) + + # Allocate out + if out is None: + out = dense.new_empty(original_batch_dims_broadcasted + (m, n)) + + # Short circuit if lhs is zero + if bsr._nnz() == 0: + return out.zero_() + + # with beta==0, addmm ignores input content, so we can use out + # as a placeholder for input because their shapes match: + return bsr_dense_addmm(out, bsr, dense, alpha=1, beta=0, out=out) + + @triton.jit + def _bsr_softmax_kernel( + crow_indices_ptr, + crow_indices_batch_stride, + crow_indices_stride, + values_ptr, + values_batch_stride, + values_row_block_stride, + values_nnz_col_block_stride, + row_block, + col_block, + MAX_ROW_NNZ: tl.constexpr, + TILE: tl.constexpr, + ): + batch_pid = tl.program_id(axis=2) + row_block_offset_pid = tl.program_id(axis=1) + row_block_pid = tl.program_id(axis=0) + + crow_indices_offset_ptr = ( + crow_indices_ptr + + crow_indices_batch_stride * batch_pid + + crow_indices_stride * row_block_pid + ) + nnz_offset = tl.load(crow_indices_offset_ptr) + nnz_offset_next = tl.load(crow_indices_offset_ptr + crow_indices_stride) + + # Compute nnz for the row with number row_block_pid. + # If it is zero, skip the row. + row_nnz = nnz_offset_next - nnz_offset + if row_nnz == 0: + return + + row_arange = tl.arange(0, TILE) + mask = row_arange < row_nnz * col_block + + curr_row_values_ptrs = ( + values_ptr + + values_batch_stride * batch_pid + + values_row_block_stride * row_block_offset_pid + + nnz_offset * col_block + ) + + # find max in the row + row_tile = tl.load( + curr_row_values_ptrs + row_arange, mask=mask, other=-float("inf") + ).to(tl.float32) + max_row_value = tl.max(row_tile, axis=0) + for _ in range(TILE, MAX_ROW_NNZ, TILE): + row_arange += TILE + mask = row_arange < row_nnz * col_block + row_tile = tl.load( + curr_row_values_ptrs + row_arange, mask=mask, other=-float("inf") + ).to(tl.float32) + curr_max_row_value = tl.max(row_tile, axis=0) + max_row_value = tl.where( + max_row_value > curr_max_row_value, max_row_value, curr_max_row_value + ) + + # find denominator for stable softmax + num = tl.exp(row_tile - max_row_value) + denom = tl.sum(num, axis=0) + for _ in range(TILE, MAX_ROW_NNZ, TILE): + row_arange -= TILE + mask = row_arange < row_nnz * col_block + row_tile = tl.load( + curr_row_values_ptrs + row_arange, mask=mask, other=-float("inf") + ).to(tl.float32) + num = tl.exp(row_tile - max_row_value) + denom += tl.sum(num, axis=0) + + # populate output + tl.store( + curr_row_values_ptrs + row_arange, + (num / denom).to(values_ptr.dtype.element_ty), + mask=mask, + ) + for _ in range(TILE, MAX_ROW_NNZ, TILE): + row_arange += TILE + mask = row_arange < row_nnz * col_block + row_tile = tl.load( + curr_row_values_ptrs + row_arange, mask=mask, other=-float("inf") + ).to(tl.float32) + num = tl.exp(row_tile - max_row_value) + tl.store( + curr_row_values_ptrs + row_arange, + (num / denom).to(values_ptr.dtype.element_ty), + mask=mask, + ) + + def bsr_softmax(input, max_row_nnz=None): + f_name = "bsr_softmax" + + check_bsr_layout(f_name, input) + check_dtype(f_name, input, input.dtype) + + if input._nnz() == 0 or input.numel() == 0: + return input.clone() + + m, n = input.shape[-2:] + nnz = input._nnz() + row_block, col_block = input.values().shape[-2:] + + if max_row_nnz is None: + max_row_nnz = triton.next_power_of_2(n) + else: + max_row_nnz = triton.next_power_of_2(max_row_nnz) + + crow_indices = input.crow_indices().unsqueeze(0).flatten(0, -2) + # reshape values from + # (b1, ..., bn, nnz, row_block, col_block) to + # (b1 * ... * bn, row_block, nnz * col_block). + # This simplifies batch dim manipulation and unlocks + # the possibility to access all nnzs in any given row. + if input.values().transpose(-3, -2).is_contiguous(): + # Need to clone to avoid `contiguous` returning a view. + values = input.values().clone() + else: + values = input.values() + values = ( + values.transpose(-3, -2) + .contiguous() + .unsqueeze(0) + .flatten(0, -4) + .reshape(-1, row_block, nnz * col_block) + ) + full_grid = (values.shape[0], row_block, m // row_block) + grid_blocks = None + tensor_dims_map = { + # We span nnz number of blocks, not nnz + 1, + # hence crow_indices[..., :-1] + crow_indices[..., :-1]: (0, None, -1), + values: (0, None, None), + } + + def kernel(grid, *sliced_tensors): + _bsr_softmax_kernel[grid]( + *ptr_stride_extractor(*sliced_tensors), + # pyrefly: ignore # bad-argument-count + row_block, + col_block, + max_row_nnz, + # Triton's max numel is bounded by 2 ** 17. + min(2**17, max_row_nnz), + ) + + launch_kernel(kernel, tensor_dims_map, full_grid, grid_blocks) + + values = ( + values.reshape(-1, row_block, nnz, col_block) + .transpose(-3, -2) + .reshape(*input.values().shape) + ) + + return torch.sparse_compressed_tensor( + input.crow_indices().clone(), + input.col_indices().clone(), + values, + size=input.shape, + layout=input.layout, + ) + + def _scaled_dot_product_attention( + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_mask: torch.Tensor | None, + dropout_p: float = 0.0, + is_causal: bool = False, + scale: float | None = None, + ): + f_name = "_scaled_dot_product_attention" + check(not is_causal, f"{f_name}(): is_causal == True is not supported.") + check(attn_mask is not None, f"{f_name}(): attn_mask == None is not supported.") + assert attn_mask is not None + + check( + attn_mask.layout == torch.sparse_bsr, + f"{f_name}(): " + f"attn_mask.layout must be {torch.sparse_bsr}, but got " + f"attn_mask.layout == {attn_mask.layout}.", + ) + + check_device(f_name, key, query.device) + check_device(f_name, value, query.device) + check_device(f_name, attn_mask, query.device) + + check_dtype(f_name, key, query.dtype) + check_dtype(f_name, value, query.dtype) + if attn_mask.dtype is not torch.bool: + check_dtype(f_name, attn_mask, query.dtype) + + # pyrefly: ignore [not-callable] + sdpa = sampled_addmm( + attn_mask, query, key.transpose(-2, -1), beta=0.0, skip_checks=False + ) + if scale is None and query.size(-1) == 0 or scale == 0.0: + check( + False, + f"{f_name}(): current value of scale == {scale} " + "results in division by zero.", + ) + scale_factor = 1 / math.sqrt(query.size(-1)) if scale is None else scale + sdpa.values().mul_(scale_factor) + # pyrefly: ignore [not-callable] + sdpa = bsr_softmax(sdpa) + torch.nn.functional.dropout(sdpa.values(), p=dropout_p, inplace=True) + # pyrefly: ignore [not-callable] + sdpa = bsr_dense_mm(sdpa, value) + return sdpa + + @triton.jit + def _scatter_mm2_kernel( + M: tl.constexpr, + K: tl.constexpr, + N: tl.constexpr, + blocks_ptr, + blocks_stride_P, + blocks_stride_M, + blocks_stride_K, + others_ptr, + others_stride_Q, + others_stride_K, + others_stride_N, + accumulators_ptr, + accumulators_stride_R, + accumulators_stride_M, + accumulators_stride_N, + pq_offsets_ptr, + pq_offsets_stride, + pq_ptr, + pq_stride_T, + pq_stride_1, + dot_out_dtype: tl.constexpr, + TILE_M: tl.constexpr, + TILE_N: tl.constexpr, + allow_tf32: tl.constexpr, + ): + Ms = M // TILE_M + + pid_t = tl.program_id(axis=0) + + pid = tl.program_id(axis=1) + pid_m = pid // Ms + pid_n = pid % Ms + + rm = pid_m * TILE_M + tl.arange(0, TILE_M) + rn = pid_n * TILE_N + tl.arange(0, TILE_N) + rk = tl.arange(0, K) + + A_ptr = blocks_ptr + ( + rm[:, None] * blocks_stride_M + rk[None, :] * blocks_stride_K + ) + B_ptr = others_ptr + ( + rk[:, None] * others_stride_K + rn[None, :] * others_stride_N + ) + + g0 = tl.load(pq_offsets_ptr + pid_t * pq_offsets_stride) + g1 = tl.load(pq_offsets_ptr + (pid_t + 1) * pq_offsets_stride) + + if g0 == g1: + return + + acc_block = tl.zeros((TILE_M, TILE_N), dtype=dot_out_dtype) + + for i in range(g0, g1): + p = tl.load(pq_ptr + i * pq_stride_T) + q = tl.load(pq_ptr + i * pq_stride_T + pq_stride_1) + A = tl.load(A_ptr + p * blocks_stride_P) + B = tl.load(B_ptr + q * others_stride_Q) + acc_block += tl.dot(A, B, out_dtype=dot_out_dtype, allow_tf32=allow_tf32) + + C_ptr = ( + accumulators_ptr + + pid_t * accumulators_stride_R + + ( + rm[:, None] * accumulators_stride_M + + rn[None, :] * accumulators_stride_N + ) + ) + tl.store(C_ptr, acc_block.to(accumulators_ptr.dtype.element_ty)) + + def _scatter_mm2( + blocks: torch.Tensor, + others: torch.Tensor, + pq_offsets: torch.Tensor, + pq_indices: torch.Tensor, + accumulators: torch.Tensor, + ): + _P, M, K = blocks.shape + _Q, _, N = others.shape + + meta = dict( + TILE_M=max(16, M // 4), TILE_N=max(16, N // 4), num_stages=1, num_warps=2 + ) + + def grid(META): + return ( + pq_offsets.shape[0] - 1, + triton.cdiv(M, META["TILE_M"]) * triton.cdiv(N, META["TILE_N"]), + 1, + ) + + dot_out_dtype = { + torch.float16: tl.float32, + torch.bfloat16: tl.float32, + torch.float32: tl.float64, + torch.float64: tl.float64, + }[accumulators.dtype] + if "allow_tf32" not in meta: + meta.update(allow_tf32=dot_out_dtype == tl.float32) + _scatter_mm2_kernel[grid]( + # pyrefly: ignore # bad-argument-type + M, + # pyrefly: ignore # bad-argument-type + K, + # pyrefly: ignore # bad-argument-type + N, + blocks, + blocks.stride(0), + blocks.stride(1), + blocks.stride(2), + others, + others.stride(0), + others.stride(1), + others.stride(2), + accumulators, + accumulators.stride(0), + accumulators.stride(1), + accumulators.stride(2), + pq_offsets, + pq_offsets.stride(0), + pq_indices, + pq_indices.stride(0), + pq_indices.stride(1), + # pyrefly: ignore # bad-argument-type + dot_out_dtype=dot_out_dtype, + # pyrefly: ignore # bad-argument-type + **meta, + ) + + @triton.jit + def _scatter_mm6_kernel( + nbatches, + Ms, + Ks: tl.constexpr, + N, + blocks_ptr, + blocks_stride_P, + blocks_stride_M, + blocks_stride_K, + others_ptr, + others_stride_B, + others_stride_K, + others_stride_N, + accumulators_ptr, + accumulators_stride_B, + accumulators_stride_M, + accumulators_stride_N, + c_indices_ptr, + r_offsets_ptr, + p_offsets_ptr, + q_offsets_ptr, + is_compressed: tl.constexpr, + dot_out_dtype: tl.constexpr, + SPLIT_N: tl.constexpr, + TILE_M: tl.constexpr, + TILE_N: tl.constexpr, + GROUP_SIZE: tl.constexpr, + allow_tf32: tl.constexpr, + ): + Ns = N // SPLIT_N + BLOCKS_M = Ms // TILE_M + BLOCKS_N = Ns // TILE_N + + pid_t_ = tl.program_id(axis=0) + pid = tl.program_id(axis=1) + pid_b = pid_t_ % nbatches + pid_t = pid_t_ // nbatches + + num_pid_in_group = GROUP_SIZE * BLOCKS_N + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE + group_size_m = min(BLOCKS_M - first_pid_m, GROUP_SIZE) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + rm = pid_m * TILE_M + tl.arange(0, TILE_M) + rn = pid_n * TILE_N + tl.arange(0, TILE_N) + rk = tl.arange(0, Ks) + A_ptr = blocks_ptr + ( + rm[:, None] * blocks_stride_M + rk[None, :] * blocks_stride_K + ) + B_ptr = ( + others_ptr + + pid_b * others_stride_B + + (rk[:, None] * others_stride_K + rn[None, :] * others_stride_N) + ) + + # When is_compressed is True, r is the only variable that + # depends on pid_t. This property allows sorting r values + # before calling the kernel. The sorting of r is equivalent to + # defining swizzle operator outside of the kernel. + r = tl.load(r_offsets_ptr + pid_t) + + if is_compressed: + m = (r // N) // Ms + n = (r % N) // Ns + r0 = tl.load(c_indices_ptr + m) + r1 = tl.load(c_indices_ptr + m + 1) + g0 = n * r1 + (SPLIT_N - n) * r0 + nnz = r1 - r0 + else: + g0 = tl.load(c_indices_ptr + pid_t) + g1 = tl.load(c_indices_ptr + pid_t + 1) + nnz = g1 - g0 + + q_ptr = q_offsets_ptr + g0 + acc_block = tl.zeros((TILE_M, TILE_N), dtype=dot_out_dtype) + + if is_compressed: + A_ptr += r0 * blocks_stride_P # type: ignore[possibly-undefined] + for _ in range(nnz): + q = tl.load(q_ptr) + B = tl.load(B_ptr + q) + A = tl.load(A_ptr) + acc_block += tl.dot( + A, B, out_dtype=dot_out_dtype, allow_tf32=allow_tf32 + ) + A_ptr += blocks_stride_P + q_ptr += 1 + else: + p_ptr = p_offsets_ptr + g0 + for _ in range(nnz): + q = tl.load(q_ptr) + B = tl.load(B_ptr + q) + p = tl.load(p_ptr) + A = tl.load(A_ptr + p * blocks_stride_P) + p_ptr += 1 + q_ptr += 1 + acc_block += tl.dot( + A, B, out_dtype=dot_out_dtype, allow_tf32=allow_tf32 + ) + + C_ptr = ( + accumulators_ptr + + r + + pid_b * accumulators_stride_B + + ( + rm[:, None] * accumulators_stride_M + + rn[None, :] * accumulators_stride_N + ) + ) + tl.store(C_ptr, acc_block.to(accumulators_ptr.dtype.element_ty)) + + def _scatter_mm6( + blocks: torch.Tensor, + others: torch.Tensor, + c_indices: torch.Tensor, + r_offsets: torch.Tensor, + p_offsets: torch.Tensor, + q_offsets: torch.Tensor, + meta: dict, + accumulators: torch.Tensor, + force_contiguous: bool = True, + ): + SPLIT_N = meta["SPLIT_N"] + _P, Ms, Ks = blocks.shape + B, _K, N = others.shape + B_, _M, N_ = accumulators.shape + assert N_ == N + Ns = N // SPLIT_N + assert B_ == B + + def grid(META): + return ( + r_offsets.shape[0] * B, + triton.cdiv(Ms, META["TILE_M"]) * triton.cdiv(Ns, META["TILE_N"]), + ) + + dot_out_dtype = { + torch.float16: tl.float32, + torch.bfloat16: tl.float32, + torch.float32: tl.float64, + torch.float64: tl.float64, + }[accumulators.dtype] + if "allow_tf32" not in meta: + meta.update(allow_tf32=dot_out_dtype == tl.float32) + + assert c_indices.stride(0) == 1 + assert r_offsets.stride(0) == 1 + assert p_offsets.stride(0) == 1 + assert q_offsets.stride(0) == 1 + + # Re non-contiguous tensor arguments. Sometimes triton kernel + # launches may fail with + # + # RuntimeError: Triton Error [CUDA]: an illegal memory access was encountered + # + # that appears to be case when the size of a non-contiguous + # tensor argument is larger than a certain threshold. Could + # this be related to shared memory or L1 cache size of a GPU + # card? In anycase, ensuring that tensor arguments are + # contiguous seems to avoid the above exception. So, in the + # following we'll always convert tensor arguments to + # C-contiguous tensors. + + if force_contiguous: + blocks = blocks.contiguous() + others = others.contiguous() + if not accumulators.is_contiguous(): + accumulators_ = accumulators.contiguous() + else: + accumulators_ = accumulators + else: + accumulators_ = accumulators + + _scatter_mm6_kernel[grid]( + B, + Ms, + # pyrefly: ignore # bad-argument-type + Ks, + N, + blocks, + blocks.stride(0), + blocks.stride(1), + blocks.stride(2), + others, + others.stride(0), + others.stride(1), + others.stride(2), + accumulators_, + accumulators_.stride(0), + accumulators_.stride(1), + accumulators_.stride(2), + c_indices, + r_offsets, + p_offsets, + q_offsets, + # pyrefly: ignore # bad-argument-type + dot_out_dtype=dot_out_dtype, + **meta, + ) + + if force_contiguous and not accumulators.is_contiguous(): + accumulators.copy_(accumulators_) + + @triton.jit + def _bsr_strided_addmm_kernel( + # values prologue + values_ptr, + values_batch_stride, + values_nnz_stride, + values_row_block_stride, + values_col_block_stride, + # values epilogue + # crow_indices prologue + crow_indices_ptr, + crow_indices_batch_stride, + crow_indices_stride, + # crow_indices epilogue + # col_indices prologue + col_indices_ptr, + col_indices_batch_stride, + col_indices_stride, + # col_indices epilogue + # input prologue + input_ptr, + input_batch_stride, + input_tiled_row_stride, + input_tiled_col_stride, + input_row_block_stride, + input_col_block_stride, + # input epilogue + # dense prologue + dense_ptr, + dense_batch_stride, + dense_tiled_row_stride, + dense_tiled_col_stride, + dense_row_block_stride, + dense_col_block_stride, + # dense epilogue + # left_alpha prologue + left_alpha_ptr, + left_alpha_batch_stride, + left_alpha_tiled_row_stride, + left_alpha_tiled_col_stride: tl.constexpr, + left_alpha_row_block_stride, + left_alpha_col_block_stride: tl.constexpr, + # left_alpha epilogue + # right_alpha prologue + right_alpha_ptr, + right_alpha_batch_stride, + right_alpha_tiled_row_stride: tl.constexpr, + right_alpha_tiled_col_stride, + right_alpha_row_block_stride: tl.constexpr, + right_alpha_col_block_stride, + # right_alpha epilogue + # output prologue + output_ptr, + output_batch_stride, + output_tiled_row_stride, + output_tiled_col_stride, + output_row_block_stride, + output_col_block_stride, + # output epilogue + beta, + alpha, + beta_is_one: tl.constexpr, + beta_is_nonzero: tl.constexpr, + alpha_is_one: tl.constexpr, + left_alpha_is_one: tl.constexpr, + right_alpha_is_one: tl.constexpr, + BLOCKSIZE_ROW: tl.constexpr, + BLOCKSIZE_COL: tl.constexpr, + BLOCKSIZE_INNER: tl.constexpr, + acc_dtype: tl.constexpr, + allow_tf32: tl.constexpr, + GROUP_SIZE_ROW: tl.constexpr, + SPLIT_N: tl.constexpr, + ): + # left/right_alpha tensors are originally (* + 1)-dimensional + assert left_alpha_tiled_col_stride == 0 + assert left_alpha_col_block_stride == 0 + assert right_alpha_tiled_row_stride == 0 + assert right_alpha_row_block_stride == 0 + + batch_pid = tl.program_id(axis=2) + row_block_pid = tl.program_id(axis=0) + col_block_pid = tl.program_id(axis=1) + n_block_rows = tl.num_programs(axis=0) + n_block_cols = tl.num_programs(axis=1) + + row_block_pid, col_block_pid = tl.swizzle2d( + row_block_pid, col_block_pid, n_block_rows, n_block_cols, GROUP_SIZE_ROW + ) + + crow_indices_offset_ptr = ( + crow_indices_ptr + + crow_indices_batch_stride * batch_pid + + crow_indices_stride * row_block_pid + ) + nnz_offset = tl.load(crow_indices_offset_ptr) + nnz_offset_next = tl.load(crow_indices_offset_ptr + crow_indices_stride) + + # Compute nnz for the row with number row_block_pid. + row_nnz = nnz_offset_next - nnz_offset + + row_block_arange = tl.arange(0, BLOCKSIZE_ROW) + inner_block_arange = tl.arange(0, BLOCKSIZE_INNER) + col_block_arange = tl.arange(0, BLOCKSIZE_COL) + + # Pointers are set to the first block of the current row. + values_block_ptrs = ( + values_ptr + + values_batch_stride * batch_pid + + values_nnz_stride * nnz_offset + + values_row_block_stride * row_block_arange[:, None] + + values_col_block_stride * inner_block_arange[None, :] + ) + + # NOTE: dense is advanced into all dimensions but the tiled row one. + # That will be advanced in the loop according to values in col_indices. + dense_block_ptrs = ( + dense_ptr + + dense_batch_stride * batch_pid + + dense_tiled_col_stride * col_block_pid + + dense_row_block_stride * inner_block_arange[:, None] + + dense_col_block_stride * col_block_arange[None, :] + ) + + # Pointers are set to exact write-to locations + output_ptrs = ( + output_ptr + + output_batch_stride * batch_pid + + output_tiled_row_stride * row_block_pid + + output_tiled_col_stride * col_block_pid + + output_row_block_stride * row_block_arange[:, None] + + output_col_block_stride * col_block_arange[None, :] + ) + + # Set pointer to the first nonzero element in the current row + col_index_nnz_ptr = ( + col_indices_ptr + + col_indices_batch_stride * batch_pid + + col_indices_stride * nnz_offset + ) + + output_acc_block = tl.zeros((BLOCKSIZE_ROW, BLOCKSIZE_COL), dtype=acc_dtype) + + for _ in range(row_nnz): + values_block = tl.load(values_block_ptrs) + + # find which row of dense needs to get loaded + # for multiplication with values_block. + dense_row_idx = tl.load(col_index_nnz_ptr) + dense_block = tl.load( + dense_block_ptrs + dense_tiled_row_stride * dense_row_idx + ) + + # do block mm + output_acc_block += tl.dot( + values_block, dense_block, allow_tf32=allow_tf32, out_dtype=acc_dtype + ) + + # move val/col_index ptrs to the next block in the row + values_block_ptrs += values_nnz_stride + col_index_nnz_ptr += col_indices_stride + + if not alpha_is_one: + output_acc_block *= alpha + + if not left_alpha_is_one: + left_alpha_ptrs = ( + left_alpha_ptr + + left_alpha_batch_stride * batch_pid + + left_alpha_tiled_row_stride * row_block_pid + + left_alpha_tiled_col_stride * col_block_pid + + left_alpha_row_block_stride * row_block_arange[:, None] + + left_alpha_col_block_stride * col_block_arange[None, :] + ) + output_acc_block *= tl.load(left_alpha_ptrs) + + if not right_alpha_is_one: + right_alpha_ptrs = ( + right_alpha_ptr + + right_alpha_batch_stride * batch_pid + + right_alpha_tiled_row_stride * row_block_pid + + right_alpha_tiled_col_stride * col_block_pid + + right_alpha_row_block_stride * row_block_arange[:, None] + + right_alpha_col_block_stride * col_block_arange[None, :] + ) + output_acc_block *= tl.load(right_alpha_ptrs) + + if beta_is_nonzero: + input_ptrs = ( + input_ptr + + input_batch_stride * batch_pid + + input_tiled_row_stride * row_block_pid + + input_tiled_col_stride * col_block_pid + + input_row_block_stride * row_block_arange[:, None] + + input_col_block_stride * col_block_arange[None, :] + ) + if beta_is_one: + output_acc_block += tl.load(input_ptrs) + else: + output_acc_block += beta * tl.load(input_ptrs) + + # write back the result + tl.store(output_ptrs, output_acc_block.to(output_ptr.dtype.element_ty)) + +else: + bsr_softmax = None # type: ignore[assignment] + bsr_dense_mm = None # type: ignore[assignment] + sampled_addmm = None # type: ignore[assignment] + _scaled_dot_product_attention = None # type: ignore[assignment] + _scatter_mm2 = None # type: ignore[assignment] + _scatter_mm6 = None # type: ignore[assignment] + _bsr_strided_addmm_kernel = None # type: ignore[assignment] diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/_triton_ops_meta.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/_triton_ops_meta.py new file mode 100644 index 0000000000000000000000000000000000000000..ae8e5f4066e27573f084bf34eeffb2a2e09cd6b8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/_triton_ops_meta.py @@ -0,0 +1,7768 @@ +# mypy: allow-untyped-defs +"""Provides optimal triton kernel parameters. + +Aim +--- + +The usage of optimal triton kernel parameters may increase the +performance of operations several times. For example, for large tensor +shapes, the usage of a bsr tensor as mat1 argument in addmm-based +operations typically outperforms the corresponding operation with +strided-only inputs when the blocked representation of a tensor +provides a better alignment with memory access than what the strided +representation would provide. + +Pre-computed kernel parameters +------------------------------ + +This script finds and stores the optimal triton kernel parameters for +a specific set of shape configurations. For instance, the set of shape +configurations of the bsr_dense_addmm kernel is defined as + + input, out: M x N strided tensor + mat1: M x K bsr tensor with blocksize (BM, BK) and given sparsity + mat2: M x N strided tensor + dtype = float16, bfloat16, float32 + sparsity = 0.5 + M = 256, 512, ..., 16384 + K = M + N = 256, 512, ..., 131072 + BM = 16, 32, ..., 128 + BK = BM + alpha = 1 + beta = 0, 1 + GPUs: NVIDIA A100-SXM4-80GB + +Approximations +-------------- + +It is practically infeasible to pre-compute optimal kernel parameter +for all possible shape configurations as well as for all existing +GPUs. Therefore, we'll assume that the pre-computed optimal parameters +are good enough approximations when +1) the used GPU is any of NVIDIA A100 Tensor Core GPUs, +2) the actual sparsity of mat1 is different from sparsity value 0.5. + +If a particular shape configuration does not fall in the set of +pre-computed kernel parameters, or it does not match with the listed +approximations above, or the used GPU device is not a NVIDIA A100 GPU, +then a reference set of triton kernel parameters will be used when +executing operations. The reference kernel parameters are defined in +torch/sparse/_triton_ops.py, see bsr_dense_addmm_meta function, for +instance. + +Computing optimal kernel parameters +----------------------------------- + +If the approximations listed above are unacceptable, e.g. when one +seeks a maximal performance possible, the optimal kernel parameters +for a particular GPU can be computed by simply running this script in +the pytorch development tree:: + + cd /path/to/pytorch + python -m pip install --no-build-isolation -v -e . + python torch/sparse/_triton_ops_meta.py + +This will compute the optimal kernel parameters for the GPU device +available in the host system for all shape configurations listed in +"Pre-computed kernel parameters" above. The results will be stored in +the database of kernel parameters. Currently, this database is defined +as this module (see "BEGIN GENERATED DATA" comment below) that will be +modified when the script is run. Create a pytorch PR with the +corresponding modifications in this file to make the computed optimal +kernel parameters available for other users as pre-computed kernel +parameters. + +Moreover, one can compute the optimal kernel parameters for a specific +set of shape configurations and specific sparsity patterns. For that, +use tuning functions provided by this module: + + tune_bsr_dense_addmm(input, mat1, mat2, beta=1, alpha=1, out=None, verbose=False, store=False) -> meta + +The tuning functions return a dictionary of optimal kernel parameters +that can be passed to the corresponding operation, e.g. + + bsr_dense_addmm(..., meta=meta) + +Or, when store==True, the optimal kernel parameters will be stored in +the database of pre-computed kernel parameters in runtime so that all +addmm-based operations such as torch.addmm, torch.mm, +torch.nn.functional.linear will benefit from using the computed +optimal set of kernel parameters. + +Note that running tune_bsr_dense_addmm can take several minutes. So, +use it wisely, e.g. by implementing persistent storage of optimized +kernel parameters. See the source code of get_meta and +tune_bsr_dense_addmm to learn how to register a custom set of optimal +kernel parameters for addmm-based operations. + +""" + +__all__ = ["get_meta", "tune_bsr_dense_addmm", "tune__int_bsr_dense_addmm"] + +import inspect +import itertools +import re +import warnings +from typing import Any + +import torch +from torch.hub import tqdm +from torch.testing import make_tensor + + +def get_meta(op, key, device_name=None, version=(0, torch.float16, 0.5), exact=False): + """Return triton kernel meta parameters of the specified op and its inputs key. + + Parameters + ---------- + op (str): The name of an operation that implementation uses meta parameters. + key (tuple): A tuple of op input parameters, e.g. shapes, etc. + device_name (optional, str): The name of a device for which op + parameters are provided. + version (optional, hashable): Specifies the version of parameters. + exact (optional, bool): When True, the returned data (if + available) corresponds exactly to the specified device_name and + version information. Otherwise, if the corresponding data is not + available but there exists a data set that is computed for a + similar GPU device, then this data set will be returned. + + Returns + ------- + result (dict): The requested mapping of parameter names and + values, or None when no data is available. If the input `key` + contains `"*"`, the result will be a dictionary of keys and + mappings that match with the given `key`. + """ + if device_name is None: + device_name = torch.cuda.get_device_name() + + op_data = _operation_device_version_data.get((op, device_name, version)) + if op_data is None and not exact: + # A lack of op data could be due to using a (slightly) + # different GPU model compared to a model for which optimal + # meta parameters have been computed. In the following we'll + # assume that there is a set of GPU models that all have + # a similar set of optimal meta parameters. + if re.match(r"NVIDIA A100[^\d]", device_name) is not None: + device_name = "NVIDIA A100-SXM4-80GB" + else: + return + op_data = _operation_device_version_data.get((op, device_name, version)) + if op_data is None: + return + + matching_data = {} + if "*" in key: + for op_key in op_data: + if [ + None + for k1, k2 in zip(op_key, key, strict=True) + if k2 != "*" and k1 != k2 + ]: + continue + matching_data[op_key] = op_data[op_key] + else: + values = op_data.get(key) + if values is not None: + matching_data[key] = values + matching_meta = {} + for op_key, values in matching_data.items(): + if op == "scatter_mm": + names = ( + "GROUP_SIZE", + "SPLIT_N", + "TILE_M", + "TILE_N", + "num_stages", + "num_warps", + ) + meta = dict(zip(names, values, strict=True)) + elif op in {"bsr_dense_addmm", "_int_bsr_dense_addmm"}: + meta = dict( + zip( + ("GROUP_SIZE_ROW", "SPLIT_N", "num_stages", "num_warps"), + values, + strict=True, + ) + ) + else: + raise NotImplementedError(f"names for {op=}") + if "*" not in key: + return meta + + matching_meta[op_key] = meta + + if "*" in key: + return matching_meta + + +def update(op, device_name, version, key, value): + """Update the db of op parameters.""" + # skip storing possible optimization failures: + if not value: + warnings.warn( + f"skipping empty value for {op}: {device_name=} {version=} {key=}", + stacklevel=2, + ) + return + if (op, device_name, version) in _operation_device_version_data: + if _operation_device_version_data[op, device_name, version].get(key) == value: + return + _operation_device_version_data[op, device_name, version][key] = value + else: + _operation_device_version_data[op, device_name, version] = {key: value} + + +def dump(): + """Store the current runtime db state to the module file.""" + current_file = inspect.getfile(dump) + with open(current_file) as f: + current_content = f.read() + begin_data_str = "# BEGIN GENERATED DATA\n" + begin_data_index = current_content.find(begin_data_str) + end_data_index = current_content.find(" # END GENERATED DATA\n") + if begin_data_index == -1 or end_data_index == -1: + warnings.warn( + f"{current_file} cannot be updated:" + " BEGIN/END GENERATED DATA comment blocks appear to be corrupted", + stacklevel=2, + ) + return + + def sort_key(key): + op, device_name, version = key + version = tuple( + (str(item) if isinstance(item, torch.dtype) else item) for item in version + ) + return (op, device_name, version) + + part1 = current_content[: begin_data_index + len(begin_data_str)] + part2 = current_content[end_data_index:] + data_part = [] + for op_key in sorted(_operation_device_version_data, key=sort_key): + # pyrefly: ignore [bad-argument-type] + data_part.append(" " + repr(op_key).replace("'", '"') + ": {") + op_data = _operation_device_version_data[op_key] + # pyrefly: ignore [bad-argument-type] + data_part.extend(f" {key}: {op_data[key]}," for key in sorted(op_data)) + data_part.append(" },") + new_content = part1 + "\n".join(data_part) + "\n" + part2 + if current_content != new_content: + with open(current_file, "w") as f: + f.write(new_content) + + +def minimize( + target_func, + initial_parameters, + reference_parameters, + step_func, + max_step=2, + verbose=False, + all_values=None, +): + """Find a dict of parameters that minimizes the target function using + the initial dict of parameters and a step function that progresses + a specified parameter in a dict of parameters. + + Parameters + ---------- + target_func (callable): a functional with the signature + ``target_func(parameters: dict) -> float`` + initial_parameters (dict): a set of parameters used as an initial + value to the minimization process. + reference_parameters (dict): a set of parameters used as an + reference value with respect to which the speed up is computed. + step_func (callable): a functional with the signature + ``step_func(parameter_name:str, parameter_value:int, direction:int, parameters:dict) -> int`` + that increments or decrements (when ``direction`` is positive or + negative, respectively) the parameter with given name and value. + When return value is equal to ``parameter_value``, it means that + no step along the given direction can be made. + + Returns + ------- + parameters (dict): a set of parameters that minimizes the target + function. + speedup_incr (float): a speedup change given in percentage. + timing (float): the value of the target function at the parameters. + sensitivity_message (str): a message containing sensitivity. + information of parameters around the target function minimizer. + """ + + def to_key(parameters): + return tuple(parameters[k] for k in sorted(parameters)) + + def from_key(key, parameters): + return dict(zip(sorted(parameters), key, strict=True)) + + if all_values is None: + all_values = {} + + directions = list(range(-max_step, max_step + 1)) + names = sorted(initial_parameters) + all_directions = [] + for d_tuple in itertools.product(*((directions,) * len(names))): + dist = sum(map(abs, d_tuple)) + if dist > 0 and dist <= max_step: + all_directions.append((dist, d_tuple)) + all_directions.sort() + + try: + reference_target = target_func(reference_parameters) + except Exception as msg: + if verbose and "out of resource" not in str(msg): + print(f"{reference_parameters=} lead to failure: {msg}.") + reference_target = None + + if reference_target is not None: + all_values[to_key(reference_parameters)] = reference_target + + parameters = initial_parameters + try: + initial_target = target_func(parameters) + except Exception as msg: + if reference_target is None: + if verbose: + print( + f"{initial_parameters=} lead to failure: {msg}. Optimization failed!" + ) + return {}, -1, -1, f"{msg}" + if verbose and "out of resource" not in str(msg): + print( + f"{initial_parameters=} lead to failure: {msg}. Using reference parameters instead of initial parameters." + ) + parameters = reference_parameters + initial_target = reference_target + + if reference_target is None: + if verbose: + print("Using initial parameters instead of reference parameters.") + reference_target = initial_target + + initial_key = to_key(parameters) + minimal_target = all_values[initial_key] = initial_target + pbar = tqdm( + total=len(all_directions), + desc="Tuning...", + disable=not verbose, + ncols=75, + ) + while True: + for i, (_, d_tuple) in enumerate(all_directions): + pbar.update(1) + next_parameters = parameters.copy() + for name, direction in zip(names, d_tuple, strict=True): + value = next_parameters[name] + if direction == 0: + continue + next_value = step_func(name, value, direction, parameters) + if next_value == value: + break + next_parameters[name] = next_value + else: + next_key = to_key(next_parameters) + if next_key in all_values: + continue + try: + next_target = target_func(next_parameters) + except Exception as msg: + all_values[next_key] = str(msg) + if verbose and "out of resource" not in str(msg): + print(f"{next_parameters=} lead to failure: {msg}. Skipping.") + continue + all_values[next_key] = next_target + + if next_target < minimal_target: + minimal_target = next_target + parameters = next_parameters + # pyrefly: ignore [unsupported-operation] + pbar.total += i + 1 + break + else: + # ensure stable minimizer: + minimizer_keys = { + k + for k, v in all_values.items() + if isinstance(v, float) and abs(1 - v / minimal_target) < 0.001 + } + minimizer_key = ( + initial_key if initial_key in minimizer_keys else min(minimizer_keys) + ) + parameters = from_key(minimizer_key, parameters) + speedup_incr = (1 - minimal_target / reference_target) * 100 + if speedup_incr < 0: + if verbose: + print( + f"{speedup_incr=} is negative. Rerunning minimize with reference parameters as initial parameters." + ) + return minimize( + target_func, + reference_parameters, + reference_parameters, + step_func, + max_step=max_step, + verbose=verbose, + all_values=all_values, + ) + sensitivity = [] + for name in parameters: + value = parameters[name] + rel_diffs = [] + for direction in range(-max_step, max_step + 1): + if direction == 0: + continue + next_value = step_func(name, value, direction, parameters) + if next_value == value: + rel_diffs.append(0) + continue + next_parameters = parameters.copy() + next_parameters[name] = next_value + next_key = to_key(next_parameters) + next_target = all_values.get(next_key) + if next_target is None or isinstance(next_target, str): + rel_diffs.append(0) + continue + rel_diff = (next_target / minimal_target - 1) * 100 + rel_diffs.append(rel_diff) + sensitivity.append((max(rel_diffs), rel_diffs, name)) + + sensitivity_message = [f"timing0={initial_target:.3f}"] + for _, rel_diffs, name in sorted(sensitivity, reverse=True): + left_diffs = "|".join( + [f"{rel_diff:.1f}" for rel_diff in rel_diffs[:max_step]] + ) + right_diffs = "|".join( + [f"{rel_diff:.1f}" for rel_diff in rel_diffs[max_step:]] + ) + sensitivity_message.append( + f"{name}={parameters[name]} ({left_diffs}...{right_diffs} %)" + ) + sensitivity_message = ", ".join(sensitivity_message) + return parameters, speedup_incr, minimal_target, sensitivity_message + + +def create_blocked_tensor(B, M, N, blocksize, sparsity, dtype, device): + assert sparsity <= 1.0 and sparsity >= 0.0, ( + "sparsity should be a value between 0 and 1" + ) + assert M % blocksize[0] == 0 + assert N % blocksize[1] == 0 + shape = (B, M // blocksize[0], N // blocksize[1])[int(B == 0) :] + A = torch.bernoulli( + torch.full(shape, 1 - sparsity, dtype=torch.float32, device=device) + ).to(dtype) + expected_nnz = int((1 - sparsity) * M * N / (blocksize[0] * blocksize[1])) + nonzero_indices = A.flatten().nonzero() + actual_nnz = nonzero_indices.shape[0] + if actual_nnz > expected_nnz: + selected_nonzeros = torch.randperm(actual_nnz)[: actual_nnz - expected_nnz] + A.flatten()[nonzero_indices[selected_nonzeros]] = 0 + elif actual_nnz < expected_nnz: + zero_indices = (A == 0).flatten().nonzero() + selected_zeros = torch.randperm(zero_indices.shape[0])[ + : expected_nnz - actual_nnz + ] + A.flatten()[zero_indices[selected_zeros]] = 1 + A = torch.repeat_interleave(A, blocksize[0], dim=-2) + A = torch.repeat_interleave(A, blocksize[1], dim=-1) + return A + + +def optimize_scatter_mm( + m, k, n, bm, bk, dtype=torch.float16, device="cuda", sparsity=0.5, force=False +): + import triton + + from torch.sparse._triton_ops import bsr_scatter_mm, bsr_scatter_mm_indices_data + + key = (m, k, n, bm, bk) + + version = (0, dtype, sparsity) + device_name = torch.cuda.get_device_name() + + reference_meta = dict( + GROUP_SIZE=1, + TILE_M=16, + TILE_N=16, + SPLIT_N=n // 16, + num_stages=1, + num_warps=1, + ) + + initial_meta = get_meta( + "scatter_mm", key, device_name=device_name, version=version, exact=True + ) + if initial_meta is None: + initial_meta = get_meta( + "bsr_dense_addmm", + key, + device_name=device_name, + version=(0, dtype, 0.5), + exact=True, + ) + if initial_meta is None: + initial_meta = reference_meta + elif not force: + return + + torch.manual_seed(0) + bsr = create_blocked_tensor( + 0, m, k, (bm, bk), sparsity, dtype, device + ).to_sparse_bsr((bm, bk)) + dense = make_tensor(k, n, dtype=dtype, device=device) + + def bench(meta, bsr=bsr, dense=dense): + indices_data = bsr_scatter_mm_indices_data( + bsr, dense, indices_format="bsr_strided_mm_compressed", **meta + ) + + def test_func(): + return bsr_scatter_mm(bsr, dense, indices_data=indices_data) + + ms_min = triton.testing.do_bench(test_func, warmup=500, rep=100) + + return ms_min + + def step_meta_parameter(name, value, direction, meta, m=m, n=n, k=k, bm=bm, bk=bk): + # return next value in positive or negative direction, or + # input value if the step will result an invalid + # value. The input value is assumed to be valid. + + is_log = name in {"SPLIT_N", "TILE_M", "TILE_N", "num_warps"} + min_value = dict( + SPLIT_N=1, TILE_M=16, TILE_N=16, num_warps=1, num_stages=1, GROUP_SIZE=1 + )[name] + max_value = dict( + SPLIT_N=n // meta["TILE_N"], TILE_M=bm, TILE_N=n // meta["SPLIT_N"] + ).get(name) + value_step = dict( + SPLIT_N=2, TILE_M=2, TILE_N=2, num_warps=2, num_stages=1, GROUP_SIZE=1 + )[name] + if is_log: + next_value = ( + value * value_step**direction + if direction > 0 + else value // (value_step ** abs(direction)) + ) + else: + next_value = value + value_step * direction + if min_value is not None: + next_value = max(next_value, min_value) + if max_value is not None: + next_value = min(next_value, max_value) + if name == "SPLIT_N" and n % next_value != 0: + return value + # Hard-skip parameter combinations that break CUDA state for pytorch: + if (dtype, name, next_value, m, n, k, bm, bk) in { + (torch.float32, "num_warps", 32, 256, 256, 256, 16, 16), + (torch.float32, "num_warps", 16, 256, 256, 256, 32, 32), + (torch.float32, "num_warps", 16, 256, 256, 256, 64, 64), + (torch.float32, "num_warps", 16, 256, 256, 256, 128, 128), + (torch.float32, "num_warps", 16, 512, 512, 256, 128, 128), + } and re.match(r"NVIDIA A100[^\d]", device_name) is not None: + return value + return next_value + + meta, speedup, timing, _sensitivity_message = minimize( + bench, initial_meta, reference_meta, step_meta_parameter + ) + if initial_meta is not reference_meta and initial_meta == meta and not force: + return + print(f"{meta=} {speedup=:.1f} % {timing=:.3f} ms") + if speedup < 0: + return + device_name = torch.cuda.get_device_name() + + update( + "scatter_mm", device_name, version, key, tuple(meta[k] for k in sorted(meta)) + ) + + +def tune__int_bsr_dense_addmm( + input, + bsr, + dense, + *, + beta=1, + alpha=1, + out=None, + store=False, + verbose=False, + force=False, +): + return tune_bsr_dense_addmm( + input, + bsr, + dense, + beta=beta, + alpha=alpha, + out=out, + store=store, + verbose=verbose, + force=force, + opname="_int_bsr_dense_addmm", + ) + + +def tune_bsr_dense_addmm( + input, + bsr, + dense, + *, + beta=1, + alpha=1, + left_alpha=None, + right_alpha=None, + out=None, + store=False, + verbose=False, + force=False, + opname=None, +): + """Tune bsr_dense_addmm kernel parameters against the given inputs. + + When store is True, the tuning results will be stored in the + database of kernel parameters. + """ + import triton + + if opname is None: + opname = "bsr_dense_addmm" + + if opname == "_int_bsr_dense_addmm": + from torch.sparse._triton_ops import _int_bsr_dense_addmm as bsr_dense_addmm + else: + from torch.sparse._triton_ops import bsr_dense_addmm + + N = dense.shape[-1] + values = bsr.values() + crow_indices = bsr.crow_indices() + batch_ndim = crow_indices.dim() - 1 + M, K = bsr.shape[batch_ndim : batch_ndim + 2] + BM, BK = values.shape[batch_ndim + 1 : batch_ndim + 3] + + # Reference parameters is a set of parameters that leads to a + # successful kernel call and the corresponding timing is used as a + # reference for computing speedups. Avoid changing the reference + # parameters when possible. + reference_meta = dict( + GROUP_SIZE_ROW=1, num_stages=1, num_warps=4, SPLIT_N=max(N // BM, 1) + ) + + # Compute the key of parameters: + sparsity = round(1 - bsr._nnz() * BM * BK / (M * K), 2) + dtype = bsr.dtype + if out is None: + out_dtype = dtype + else: + out_dtype = out.dtype + if out_dtype is dtype: + version_dtype = dtype + else: + version_dtype = (dtype, out_dtype) + version = (0, version_dtype, sparsity) + key = (M, K, N, BM, BK, beta == 0, beta == 1, alpha == 1) + + # For tuning, for an initial state, use parameters from the + # database if available, otherwise, use the reference parameters. + initial_meta = get_meta(opname, key, version=version, exact=True) + if initial_meta is None: + may_skip_update = False + initial_meta = get_meta(opname, key, version=(0, dtype, 0.5), exact=True) + if initial_meta is None: + initial_meta = reference_meta + elif not force: + return initial_meta + else: + may_skip_update = True + + # The target function that is minimized in the tuning process: + def bench(meta, input=input, bsr=bsr, dense=dense, alpha=alpha, out=out): + def test_func(): + return bsr_dense_addmm( + input, + bsr, + dense, + beta=beta, + alpha=alpha, + left_alpha=left_alpha, + right_alpha=right_alpha, + meta=meta, + out=out, + ) + + return triton.testing.do_bench(test_func, warmup=500, rep=100) + + # The step function that increments a specified meta parameter: + def step_meta_parameter(name, value, direction, meta, M=M, N=N, K=K, BM=BM, BK=BK): + # return next value in positive or negative direction, or + # input value if the step will result an invalid + # value. The input value is assumed to be valid. + is_log = name in {"SPLIT_N", "num_warps"} + min_value = dict(SPLIT_N=1, num_warps=1, num_stages=1, GROUP_SIZE_ROW=1)[name] + max_value = dict(SPLIT_N=max(N // BM, 1)).get(name) + value_step = dict(SPLIT_N=2, num_warps=2, num_stages=1, GROUP_SIZE_ROW=1)[name] + if is_log: + next_value = ( + value * value_step**direction + if direction > 0 + else value // (value_step ** abs(direction)) + ) + else: + next_value = value + value_step * direction + if min_value is not None: + next_value = max(next_value, min_value) + if max_value is not None: + next_value = min(next_value, max_value) + if name == "SPLIT_N" and N % next_value != 0: + return value + return next_value + + # Tune: + meta, speedup, timing, sensitivity_message = minimize( + bench, + initial_meta, + reference_meta, + step_meta_parameter, + max_step=2, + verbose=verbose, + ) + if verbose: + print(f"-> {sensitivity_message}, {speedup=:.1f} %, {timing=:.3f} ms") + + if store and not ( + may_skip_update and meta == initial_meta and initial_meta is not reference_meta + ): + device_name = torch.cuda.get_device_name() + update( + opname, + device_name, + version, + key, + tuple(meta[k] for k in sorted(meta)), + ) + + return meta + + +def optimize_bsr_dense_addmm( + m, + k, + n, + bm, + bk, + beta=1, + alpha=1, + use_left_alpha=False, + use_right_alpha=False, + dtype=torch.float16, + out_dtype=None, + device="cuda", + sparsity=0.5, + force=False, + verbose=False, + opname=None, +): + torch.manual_seed(0) + bsr = create_blocked_tensor( + 0, m, k, (bm, bk), sparsity, dtype, device + ).to_sparse_bsr((bm, bk)) + dense = make_tensor(k, n, dtype=dtype, device=device) + input = make_tensor(m, n, dtype=dtype, device=device) + left_alpha = make_tensor(m, dtype=dtype, device=device) if use_left_alpha else None + right_alpha = ( + make_tensor(n, dtype=dtype, device=device) if use_right_alpha else None + ) + if out_dtype is not None: + out = dense.new_empty((m, n), dtype=out_dtype) + else: + out = None + tune_bsr_dense_addmm( + input, + bsr, + dense, + beta=beta, + alpha=alpha, + left_alpha=left_alpha, + right_alpha=right_alpha, + out=out, + store=True, + force=force, + verbose=verbose, + opname=opname, + ) + + +def main(op="scatter_mm", force=False, dtype=torch.float16, verbose=True): + import itertools + + sizes_lst = [ + 256, + 512, + 1024, + 2048, + 4096, + 8192, + 16384, + 32768, + 65536, + 131072, + 50432, + 65792, + ] + sizes3_lst = [3 * sz for sz in [64, 128] + sizes_lst if sz <= 2048] + shapes_lst = [(sz, sz) for sz in sizes_lst[:-4] + sizes3_lst] + shapes_lst.extend([(3072, 768), (768, 3072)]) + shapes_lst.extend([(5120, 1280), (1280, 5120)]) + if dtype is torch.int8: + # triton does not support smaller blocks than 32 + blocksize_lst = [(32, 32), (64, 64), (128, 128), (256, 256)] + else: + blocksize_lst = [(16, 16), (32, 32), (64, 64), (128, 128)] + sparsity_lst = [0.5, 0.7, 0.3][:1] + for sparsity in sparsity_lst: + print(f"{op, dtype, sparsity=}") + try: + for (M, K), N, (BM, BK) in itertools.product( + shapes_lst, sizes_lst, blocksize_lst + ): + if not (BM <= M and BK <= K and M % BM == 0 and K % BK == 0): + continue + if op == "scatter_mm": + optimize_scatter_mm( + M, K, N, BM, BK, force=force, sparsity=sparsity, dtype=dtype + ) + elif op in {"bsr_dense_addmm", "_int_bsr_dense_addmm"}: + if M == K and N == 50432: + continue + print(f"{M, K, N, (BM, BK)=}") + for alpha, beta in [(1, 1), (1, 0)]: + optimize_bsr_dense_addmm( + M, + K, + N, + BM, + BK, + beta=beta, + alpha=alpha, + force=force, + sparsity=sparsity, + dtype=dtype, + verbose=verbose, + opname=op, + ) + else: + raise NotImplementedError(op) + except KeyboardInterrupt: + break + except Exception: + dump() + raise + dump() + + if 0: + # Check performance dependence on sparsity and apply + # adjustments when differences are noticeable (more than 10%). + # + # When using NVIDIA A100 GPU, the performance dependence on + # sparsity is insignificant (0 % ... 10 %) for majority of + # shapes/blocksizes combinations. However, for a very few + # specific size combinations, the effect of sparsity on + # performance can be up to 20 %. + for (M, K), N, (BM, BK) in itertools.product( + shapes_lst, sizes_lst, blocksize_lst + ): + meta_lst: list = [] + key = (M, K, N, BM, BK) + for sparsity1 in sparsity_lst: + torch.manual_seed(0) + bsr = create_blocked_tensor( + 0, M, K, (BM, BK), sparsity1, dtype, device="cuda" + ).to_sparse_bsr((BM, BK)) + dense = make_tensor(K, N, dtype=dtype, device="cuda") + meta_lst = [] + for sparsity in sparsity_lst: + meta = get_meta(op, key, version=(0, dtype, sparsity), exact=True) + if meta is None: + continue + + def bench(meta, bsr=bsr, dense=dense): + import triton + + if op == "scatter_mm": + from torch.sparse._triton_ops import ( + bsr_scatter_mm, + bsr_scatter_mm_indices_data, + ) + + indices_data = bsr_scatter_mm_indices_data( + bsr, + dense, + indices_format="bsr_strided_mm_compressed", + **meta, + ) + + def test_func(): + return bsr_scatter_mm( + bsr, dense, indices_data=indices_data + ) + + else: + raise NotImplementedError(op) + + ms_min = triton.testing.do_bench(test_func, warmup=500, rep=100) + + return ms_min + + meta_lst.append( + (bench(meta), sparsity, tuple(meta[k] for k in sorted(meta))) + ) + if not meta_lst: + continue + meta_lst = sorted(meta_lst) + index = next( + i for i, item in enumerate(meta_lst) if item[1] == sparsity1 + ) + if meta_lst[0][2] == meta_lst[index][2]: + continue + speeddiff = (1 - meta_lst[index][0] / meta_lst[0][0]) * 100 + if abs(speeddiff) < 10: + continue + + print(sparsity1, index, key, meta_lst, speeddiff) + + if index > 0: + device_name = torch.cuda.get_device_name() + meta = get_meta( + op, key, version=(0, dtype, meta_lst[0][1]), exact=True + ) + update( + op, + device_name, + (0, dtype, sparsity1), + key, + tuple(meta[k] for k in sorted(meta)), + ) + print("update") + dump() + + +_operation_device_version_data: dict[Any, dict] = { + # Warning: the data in between the BEGIN/END DATA comment lines + # below is generated. It can be updated either manually or via + # calling dump function defined above. + # + # Legend [op: key -> data]: + # scatter_mm : M, K, N, Ms, Ks -> GROUP_SIZE, SPLIT_N, TILE_M, TILE_N, num_stages, num_warps + # bsr_dense_addmm : M, K, N, Ms, Ks, beta==0, beta==1, alpha==1 -> GROUP_SIZE_ROW, SPLIT_N, num_stages, num_warps + # + # BEGIN GENERATED DATA + ("_int_bsr_dense_addmm", "NVIDIA A100-SXM4-80GB", (0, torch.int8, 0.5)): { + (192, 192, 256, 32, 32, False, True, True): (2, 8, 1, 4), + (192, 192, 256, 32, 32, True, False, True): (2, 8, 5, 4), + (192, 192, 512, 32, 32, False, True, True): (1, 16, 1, 4), + (192, 192, 512, 32, 32, True, False, True): (1, 16, 5, 4), + (192, 192, 1024, 32, 32, False, True, True): (1, 32, 1, 4), + (192, 192, 1024, 32, 32, True, False, True): (4, 32, 4, 4), + (192, 192, 2048, 32, 32, False, True, True): (2, 64, 1, 4), + (192, 192, 2048, 32, 32, True, False, True): (3, 16, 5, 4), + (192, 192, 4096, 32, 32, False, True, True): (1, 128, 1, 4), + (192, 192, 4096, 32, 32, True, False, True): (1, 128, 1, 4), + (192, 192, 8192, 32, 32, False, True, True): (1, 256, 1, 4), + (192, 192, 8192, 32, 32, True, False, True): (1, 64, 3, 4), + (192, 192, 16384, 32, 32, False, True, True): (2, 512, 1, 4), + (192, 192, 16384, 32, 32, True, False, True): (5, 128, 1, 4), + (192, 192, 32768, 32, 32, False, True, True): (1, 1024, 1, 4), + (192, 192, 32768, 32, 32, True, False, True): (1, 256, 1, 4), + (192, 192, 65536, 32, 32, False, True, True): (1, 1024, 1, 8), + (192, 192, 65536, 32, 32, True, False, True): (1, 512, 1, 4), + (192, 192, 131072, 32, 32, False, True, True): (1, 2048, 1, 8), + (192, 192, 131072, 32, 32, True, False, True): (2, 512, 1, 4), + (256, 256, 256, 32, 32, False, True, True): (4, 8, 1, 4), + (256, 256, 256, 32, 32, True, False, True): (1, 8, 6, 4), + (256, 256, 256, 64, 64, False, True, True): (1, 4, 1, 16), + (256, 256, 256, 64, 64, True, False, True): (1, 4, 4, 4), + (256, 256, 256, 128, 128, False, True, True): (3, 2, 1, 16), + (256, 256, 256, 128, 128, True, False, True): (1, 2, 1, 4), + (256, 256, 512, 32, 32, False, True, True): (2, 16, 1, 4), + (256, 256, 512, 32, 32, True, False, True): (2, 16, 4, 4), + (256, 256, 512, 64, 64, False, True, True): (7, 8, 1, 16), + (256, 256, 512, 64, 64, True, False, True): (3, 8, 3, 4), + (256, 256, 512, 128, 128, False, True, True): (1, 4, 1, 32), + (256, 256, 512, 128, 128, True, False, True): (1, 4, 1, 4), + (256, 256, 1024, 32, 32, False, True, True): (1, 32, 1, 4), + (256, 256, 1024, 32, 32, True, False, True): (1, 8, 6, 4), + (256, 256, 1024, 64, 64, False, True, True): (2, 16, 1, 16), + (256, 256, 1024, 64, 64, True, False, True): (1, 16, 5, 4), + (256, 256, 1024, 128, 128, False, True, True): (4, 8, 1, 32), + (256, 256, 1024, 128, 128, True, False, True): (1, 8, 2, 4), + (256, 256, 2048, 32, 32, False, True, True): (1, 64, 1, 4), + (256, 256, 2048, 32, 32, True, False, True): (2, 32, 3, 2), + (256, 256, 2048, 64, 64, False, True, True): (2, 32, 1, 16), + (256, 256, 2048, 64, 64, True, False, True): (1, 16, 3, 4), + (256, 256, 2048, 128, 128, False, True, True): (1, 16, 1, 32), + (256, 256, 2048, 128, 128, True, False, True): (1, 16, 2, 4), + (256, 256, 4096, 32, 32, False, True, True): (2, 128, 1, 4), + (256, 256, 4096, 32, 32, True, False, True): (1, 32, 3, 2), + (256, 256, 4096, 64, 64, False, True, True): (2, 64, 1, 8), + (256, 256, 4096, 64, 64, True, False, True): (1, 64, 3, 2), + (256, 256, 4096, 128, 128, False, True, True): (2, 32, 1, 32), + (256, 256, 4096, 128, 128, True, False, True): (3, 32, 2, 8), + (256, 256, 8192, 32, 32, False, True, True): (1, 256, 1, 4), + (256, 256, 8192, 32, 32, True, False, True): (1, 64, 1, 4), + (256, 256, 8192, 64, 64, False, True, True): (1, 128, 1, 8), + (256, 256, 8192, 64, 64, True, False, True): (2, 128, 1, 4), + (256, 256, 8192, 128, 128, False, True, True): (4, 64, 1, 32), + (256, 256, 8192, 128, 128, True, False, True): (3, 64, 1, 4), + (256, 256, 16384, 32, 32, False, True, True): (1, 256, 1, 8), + (256, 256, 16384, 32, 32, True, False, True): (3, 128, 1, 4), + (256, 256, 16384, 64, 64, False, True, True): (2, 256, 1, 8), + (256, 256, 16384, 64, 64, True, False, True): (2, 256, 1, 4), + (256, 256, 16384, 128, 128, False, True, True): (2, 128, 1, 32), + (256, 256, 16384, 128, 128, True, False, True): (4, 128, 2, 4), + (256, 256, 32768, 32, 32, False, True, True): (2, 512, 1, 8), + (256, 256, 32768, 32, 32, True, False, True): (1, 256, 1, 4), + (256, 256, 32768, 64, 64, False, True, True): (1, 512, 1, 8), + (256, 256, 32768, 64, 64, True, False, True): (1, 512, 1, 4), + (256, 256, 32768, 128, 128, False, True, True): (2, 256, 1, 32), + (256, 256, 32768, 128, 128, True, False, True): (1, 256, 2, 4), + (256, 256, 65536, 32, 32, False, True, True): (1, 1024, 1, 8), + (256, 256, 65536, 32, 32, True, False, True): (1, 512, 1, 4), + (256, 256, 65536, 64, 64, False, True, True): (1, 1024, 1, 8), + (256, 256, 65536, 64, 64, True, False, True): (1, 512, 1, 4), + (256, 256, 65536, 128, 128, False, True, True): (2, 512, 1, 16), + (256, 256, 65536, 128, 128, True, False, True): (1, 512, 1, 4), + (256, 256, 65792, 32, 32, False, True, True): (1, 1028, 1, 8), + (256, 256, 65792, 32, 32, True, False, True): (1, 514, 1, 4), + (256, 256, 65792, 64, 64, False, True, True): (1, 1028, 1, 8), + (256, 256, 65792, 64, 64, True, False, True): (4, 257, 1, 4), + (256, 256, 65792, 128, 128, False, True, True): (2, 514, 1, 16), + (256, 256, 65792, 128, 128, True, False, True): (3, 514, 1, 4), + (256, 256, 131072, 32, 32, False, True, True): (1, 2048, 1, 8), + (256, 256, 131072, 32, 32, True, False, True): (2, 1024, 1, 4), + (256, 256, 131072, 64, 64, False, True, True): (1, 2048, 1, 8), + (256, 256, 131072, 64, 64, True, False, True): (2, 512, 1, 4), + (256, 256, 131072, 128, 128, False, True, True): (2, 1024, 1, 16), + (256, 256, 131072, 128, 128, True, False, True): (4, 1024, 1, 4), + (384, 384, 256, 32, 32, False, True, True): (1, 8, 1, 4), + (384, 384, 256, 32, 32, True, False, True): (5, 8, 5, 4), + (384, 384, 256, 64, 64, False, True, True): (2, 4, 1, 16), + (384, 384, 256, 64, 64, True, False, True): (1, 4, 5, 4), + (384, 384, 512, 32, 32, False, True, True): (2, 16, 1, 4), + (384, 384, 512, 32, 32, True, False, True): (1, 16, 4, 4), + (384, 384, 512, 64, 64, False, True, True): (3, 8, 1, 16), + (384, 384, 512, 64, 64, True, False, True): (3, 8, 3, 4), + (384, 384, 1024, 32, 32, False, True, True): (2, 32, 1, 4), + (384, 384, 1024, 32, 32, True, False, True): (1, 8, 6, 4), + (384, 384, 1024, 64, 64, False, True, True): (2, 16, 1, 16), + (384, 384, 1024, 64, 64, True, False, True): (1, 16, 5, 4), + (384, 384, 2048, 32, 32, False, True, True): (1, 64, 1, 4), + (384, 384, 2048, 32, 32, True, False, True): (3, 16, 4, 4), + (384, 384, 2048, 64, 64, False, True, True): (2, 32, 1, 16), + (384, 384, 2048, 64, 64, True, False, True): (1, 16, 4, 4), + (384, 384, 4096, 32, 32, False, True, True): (4, 64, 1, 8), + (384, 384, 4096, 32, 32, True, False, True): (4, 32, 1, 4), + (384, 384, 4096, 64, 64, False, True, True): (1, 64, 1, 8), + (384, 384, 4096, 64, 64, True, False, True): (1, 64, 1, 4), + (384, 384, 8192, 32, 32, False, True, True): (1, 128, 1, 8), + (384, 384, 8192, 32, 32, True, False, True): (3, 64, 1, 1), + (384, 384, 8192, 64, 64, False, True, True): (2, 128, 1, 8), + (384, 384, 8192, 64, 64, True, False, True): (1, 64, 2, 2), + (384, 384, 16384, 32, 32, False, True, True): (1, 256, 1, 8), + (384, 384, 16384, 32, 32, True, False, True): (1, 128, 1, 4), + (384, 384, 16384, 64, 64, False, True, True): (2, 256, 1, 8), + (384, 384, 16384, 64, 64, True, False, True): (2, 128, 1, 4), + (384, 384, 32768, 32, 32, False, True, True): (1, 512, 1, 8), + (384, 384, 32768, 32, 32, True, False, True): (1, 256, 1, 4), + (384, 384, 32768, 64, 64, False, True, True): (1, 512, 1, 8), + (384, 384, 32768, 64, 64, True, False, True): (1, 256, 3, 2), + (384, 384, 65536, 32, 32, False, True, True): (1, 1024, 1, 8), + (384, 384, 65536, 32, 32, True, False, True): (1, 512, 3, 2), + (384, 384, 65536, 64, 64, False, True, True): (2, 1024, 1, 8), + (384, 384, 65536, 64, 64, True, False, True): (3, 256, 3, 4), + (384, 384, 131072, 32, 32, False, True, True): (1, 2048, 1, 8), + (384, 384, 131072, 32, 32, True, False, True): (1, 1024, 3, 2), + (384, 384, 131072, 64, 64, False, True, True): (1, 2048, 1, 8), + (384, 384, 131072, 64, 64, True, False, True): (2, 512, 3, 4), + (512, 512, 256, 32, 32, False, True, True): (1, 8, 1, 4), + (512, 512, 256, 32, 32, True, False, True): (4, 8, 4, 4), + (512, 512, 256, 64, 64, False, True, True): (3, 4, 1, 16), + (512, 512, 256, 64, 64, True, False, True): (2, 4, 5, 4), + (512, 512, 256, 128, 128, False, True, True): (4, 2, 1, 16), + (512, 512, 256, 128, 128, True, False, True): (1, 2, 3, 4), + (512, 512, 256, 256, 256, False, True, True): (1, 1, 1, 32), + (512, 512, 256, 256, 256, True, False, True): (2, 1, 1, 32), + (512, 512, 512, 32, 32, False, True, True): (3, 16, 1, 4), + (512, 512, 512, 32, 32, True, False, True): (1, 8, 4, 2), + (512, 512, 512, 64, 64, False, True, True): (2, 8, 1, 16), + (512, 512, 512, 64, 64, True, False, True): (2, 8, 5, 4), + (512, 512, 512, 128, 128, False, True, True): (3, 4, 1, 16), + (512, 512, 512, 128, 128, True, False, True): (1, 4, 3, 4), + (512, 512, 512, 256, 256, False, True, True): (1, 2, 1, 32), + (512, 512, 512, 256, 256, True, False, True): (2, 2, 1, 32), + (512, 512, 1024, 32, 32, False, True, True): (2, 32, 1, 4), + (512, 512, 1024, 32, 32, True, False, True): (4, 16, 3, 2), + (512, 512, 1024, 64, 64, False, True, True): (4, 16, 1, 16), + (512, 512, 1024, 64, 64, True, False, True): (1, 8, 4, 4), + (512, 512, 1024, 128, 128, False, True, True): (1, 8, 1, 32), + (512, 512, 1024, 128, 128, True, False, True): (1, 8, 3, 4), + (512, 512, 1024, 256, 256, False, True, True): (4, 4, 1, 32), + (512, 512, 1024, 256, 256, True, False, True): (2, 4, 1, 32), + (512, 512, 2048, 32, 32, False, True, True): (3, 32, 1, 8), + (512, 512, 2048, 32, 32, True, False, True): (1, 16, 3, 4), + (512, 512, 2048, 64, 64, False, True, True): (1, 32, 1, 8), + (512, 512, 2048, 64, 64, True, False, True): (1, 32, 3, 2), + (512, 512, 2048, 128, 128, False, True, True): (4, 16, 1, 32), + (512, 512, 2048, 128, 128, True, False, True): (1, 16, 3, 4), + (512, 512, 2048, 256, 256, False, True, True): (1, 8, 1, 32), + (512, 512, 2048, 256, 256, True, False, True): (3, 8, 1, 32), + (512, 512, 4096, 32, 32, False, True, True): (1, 64, 1, 8), + (512, 512, 4096, 32, 32, True, False, True): (5, 32, 1, 4), + (512, 512, 4096, 64, 64, False, True, True): (1, 64, 1, 8), + (512, 512, 4096, 64, 64, True, False, True): (1, 64, 1, 4), + (512, 512, 4096, 128, 128, False, True, True): (5, 32, 1, 32), + (512, 512, 4096, 128, 128, True, False, True): (2, 32, 3, 4), + (512, 512, 4096, 256, 256, False, True, True): (1, 16, 1, 32), + (512, 512, 4096, 256, 256, True, False, True): (3, 16, 1, 32), + (512, 512, 8192, 32, 32, False, True, True): (3, 128, 1, 8), + (512, 512, 8192, 32, 32, True, False, True): (3, 64, 1, 4), + (512, 512, 8192, 64, 64, False, True, True): (4, 128, 1, 8), + (512, 512, 8192, 64, 64, True, False, True): (1, 64, 3, 2), + (512, 512, 8192, 128, 128, False, True, True): (5, 64, 1, 32), + (512, 512, 8192, 128, 128, True, False, True): (1, 64, 2, 4), + (512, 512, 8192, 256, 256, False, True, True): (1, 32, 1, 32), + (512, 512, 8192, 256, 256, True, False, True): (1, 32, 1, 32), + (512, 512, 16384, 32, 32, False, True, True): (1, 256, 1, 8), + (512, 512, 16384, 32, 32, True, False, True): (2, 128, 1, 4), + (512, 512, 16384, 64, 64, False, True, True): (2, 256, 1, 8), + (512, 512, 16384, 64, 64, True, False, True): (1, 128, 3, 2), + (512, 512, 16384, 128, 128, False, True, True): (4, 128, 1, 16), + (512, 512, 16384, 128, 128, True, False, True): (2, 128, 1, 4), + (512, 512, 16384, 256, 256, False, True, True): (1, 64, 1, 32), + (512, 512, 16384, 256, 256, True, False, True): (2, 64, 1, 32), + (512, 512, 32768, 32, 32, False, True, True): (1, 512, 1, 8), + (512, 512, 32768, 32, 32, True, False, True): (2, 256, 1, 4), + (512, 512, 32768, 64, 64, False, True, True): (1, 512, 1, 8), + (512, 512, 32768, 64, 64, True, False, True): (1, 256, 3, 2), + (512, 512, 32768, 128, 128, False, True, True): (4, 256, 1, 16), + (512, 512, 32768, 128, 128, True, False, True): (2, 256, 1, 4), + (512, 512, 32768, 256, 256, False, True, True): (1, 128, 1, 32), + (512, 512, 32768, 256, 256, True, False, True): (2, 128, 1, 32), + (512, 512, 65536, 32, 32, False, True, True): (1, 1024, 1, 8), + (512, 512, 65536, 32, 32, True, False, True): (2, 512, 1, 2), + (512, 512, 65536, 64, 64, False, True, True): (1, 1024, 1, 8), + (512, 512, 65536, 64, 64, True, False, True): (1, 512, 3, 2), + (512, 512, 65536, 128, 128, False, True, True): (4, 512, 1, 16), + (512, 512, 65536, 128, 128, True, False, True): (1, 512, 1, 4), + (512, 512, 65536, 256, 256, False, True, True): (1, 256, 1, 32), + (512, 512, 65536, 256, 256, True, False, True): (1, 256, 1, 32), + (512, 512, 65792, 32, 32, False, True, True): (1, 1028, 1, 8), + (512, 512, 65792, 32, 32, True, False, True): (1, 514, 3, 2), + (512, 512, 65792, 64, 64, False, True, True): (1, 1028, 1, 8), + (512, 512, 65792, 64, 64, True, False, True): (2, 257, 3, 4), + (512, 512, 65792, 128, 128, False, True, True): (4, 514, 1, 16), + (512, 512, 65792, 128, 128, True, False, True): (1, 514, 1, 4), + (512, 512, 65792, 256, 256, False, True, True): (1, 257, 1, 32), + (512, 512, 65792, 256, 256, True, False, True): (2, 257, 1, 32), + (512, 512, 131072, 32, 32, False, True, True): (1, 2048, 1, 8), + (512, 512, 131072, 32, 32, True, False, True): (1, 1024, 3, 2), + (512, 512, 131072, 64, 64, False, True, True): (1, 2048, 1, 8), + (512, 512, 131072, 64, 64, True, False, True): (1, 1024, 3, 2), + (512, 512, 131072, 128, 128, False, True, True): (4, 1024, 1, 16), + (512, 512, 131072, 128, 128, True, False, True): (1, 1024, 1, 4), + (512, 512, 131072, 256, 256, False, True, True): (1, 512, 1, 32), + (512, 512, 131072, 256, 256, True, False, True): (2, 512, 1, 32), + (768, 768, 256, 32, 32, False, True, True): (1, 8, 1, 4), + (768, 768, 256, 32, 32, True, False, True): (2, 8, 4, 4), + (768, 768, 256, 64, 64, False, True, True): (3, 4, 1, 16), + (768, 768, 256, 64, 64, True, False, True): (2, 4, 4, 4), + (768, 768, 256, 128, 128, False, True, True): (1, 2, 1, 8), + (768, 768, 256, 128, 128, True, False, True): (1, 2, 3, 4), + (768, 768, 512, 32, 32, False, True, True): (1, 16, 1, 4), + (768, 768, 512, 32, 32, True, False, True): (1, 4, 5, 4), + (768, 768, 512, 64, 64, False, True, True): (1, 8, 3, 32), + (768, 768, 512, 64, 64, True, False, True): (4, 8, 4, 4), + (768, 768, 512, 128, 128, False, True, True): (4, 4, 1, 16), + (768, 768, 512, 128, 128, True, False, True): (4, 4, 3, 4), + (768, 768, 1024, 32, 32, False, True, True): (1, 16, 1, 8), + (768, 768, 1024, 32, 32, True, False, True): (1, 8, 3, 4), + (768, 768, 1024, 64, 64, False, True, True): (3, 16, 1, 16), + (768, 768, 1024, 64, 64, True, False, True): (1, 8, 4, 4), + (768, 768, 1024, 128, 128, False, True, True): (3, 8, 1, 32), + (768, 768, 1024, 128, 128, True, False, True): (1, 8, 3, 4), + (768, 768, 2048, 32, 32, False, True, True): (2, 32, 1, 8), + (768, 768, 2048, 32, 32, True, False, True): (3, 16, 1, 4), + (768, 768, 2048, 64, 64, False, True, True): (1, 32, 1, 8), + (768, 768, 2048, 64, 64, True, False, True): (4, 8, 3, 4), + (768, 768, 2048, 128, 128, False, True, True): (1, 16, 1, 32), + (768, 768, 2048, 128, 128, True, False, True): (1, 16, 3, 4), + (768, 768, 4096, 32, 32, False, True, True): (1, 64, 1, 8), + (768, 768, 4096, 32, 32, True, False, True): (1, 32, 1, 1), + (768, 768, 4096, 64, 64, False, True, True): (2, 64, 1, 8), + (768, 768, 4096, 64, 64, True, False, True): (1, 32, 2, 2), + (768, 768, 4096, 128, 128, False, True, True): (1, 32, 1, 32), + (768, 768, 4096, 128, 128, True, False, True): (6, 32, 1, 4), + (768, 768, 8192, 32, 32, False, True, True): (1, 128, 1, 8), + (768, 768, 8192, 32, 32, True, False, True): (1, 64, 1, 4), + (768, 768, 8192, 64, 64, False, True, True): (1, 128, 1, 8), + (768, 768, 8192, 64, 64, True, False, True): (4, 32, 3, 4), + (768, 768, 8192, 128, 128, False, True, True): (2, 64, 1, 16), + (768, 768, 8192, 128, 128, True, False, True): (2, 64, 3, 4), + (768, 768, 16384, 32, 32, False, True, True): (1, 256, 1, 8), + (768, 768, 16384, 32, 32, True, False, True): (1, 128, 1, 4), + (768, 768, 16384, 64, 64, False, True, True): (1, 256, 1, 8), + (768, 768, 16384, 64, 64, True, False, True): (1, 128, 3, 2), + (768, 768, 16384, 128, 128, False, True, True): (2, 128, 1, 16), + (768, 768, 16384, 128, 128, True, False, True): (2, 128, 1, 4), + (768, 768, 32768, 32, 32, False, True, True): (1, 512, 1, 8), + (768, 768, 32768, 32, 32, True, False, True): (1, 256, 3, 2), + (768, 768, 32768, 64, 64, False, True, True): (2, 512, 1, 8), + (768, 768, 32768, 64, 64, True, False, True): (1, 256, 3, 2), + (768, 768, 32768, 128, 128, False, True, True): (2, 256, 1, 16), + (768, 768, 32768, 128, 128, True, False, True): (3, 256, 1, 4), + (768, 768, 65536, 32, 32, False, True, True): (1, 1024, 1, 8), + (768, 768, 65536, 32, 32, True, False, True): (1, 512, 3, 2), + (768, 768, 65536, 64, 64, False, True, True): (2, 512, 1, 4), + (768, 768, 65536, 64, 64, True, False, True): (1, 512, 3, 2), + (768, 768, 65536, 128, 128, False, True, True): (2, 512, 1, 16), + (768, 768, 65536, 128, 128, True, False, True): (2, 512, 1, 4), + (768, 768, 131072, 32, 32, False, True, True): (1, 2048, 1, 8), + (768, 768, 131072, 32, 32, True, False, True): (1, 1024, 3, 2), + (768, 768, 131072, 64, 64, False, True, True): (2, 1024, 1, 4), + (768, 768, 131072, 64, 64, True, False, True): (2, 1024, 3, 2), + (768, 768, 131072, 128, 128, False, True, True): (2, 1024, 1, 16), + (768, 768, 131072, 128, 128, True, False, True): (2, 1024, 1, 4), + (768, 3072, 256, 32, 32, False, True, True): (3, 8, 4, 8), + (768, 3072, 256, 32, 32, True, False, True): (3, 8, 5, 4), + (768, 3072, 256, 64, 64, False, True, True): (1, 4, 4, 16), + (768, 3072, 256, 64, 64, True, False, True): (1, 4, 4, 4), + (768, 3072, 256, 128, 128, False, True, True): (2, 2, 1, 8), + (768, 3072, 256, 128, 128, True, False, True): (2, 2, 4, 4), + (768, 3072, 256, 256, 256, False, True, True): (1, 1, 1, 32), + (768, 3072, 256, 256, 256, True, False, True): (1, 1, 1, 32), + (768, 3072, 512, 32, 32, False, True, True): (1, 16, 1, 4), + (768, 3072, 512, 32, 32, True, False, True): (2, 4, 4, 4), + (768, 3072, 512, 64, 64, False, True, True): (3, 8, 4, 16), + (768, 3072, 512, 64, 64, True, False, True): (1, 8, 4, 4), + (768, 3072, 512, 128, 128, False, True, True): (2, 4, 1, 8), + (768, 3072, 512, 128, 128, True, False, True): (4, 4, 3, 4), + (768, 3072, 512, 256, 256, False, True, True): (1, 2, 1, 32), + (768, 3072, 512, 256, 256, True, False, True): (1, 2, 1, 32), + (768, 3072, 1024, 32, 32, False, True, True): (1, 16, 1, 8), + (768, 3072, 1024, 32, 32, True, False, True): (3, 8, 3, 4), + (768, 3072, 1024, 64, 64, False, True, True): (2, 16, 1, 16), + (768, 3072, 1024, 64, 64, True, False, True): (1, 8, 3, 4), + (768, 3072, 1024, 128, 128, False, True, True): (1, 8, 1, 8), + (768, 3072, 1024, 128, 128, True, False, True): (3, 8, 4, 4), + (768, 3072, 1024, 256, 256, False, True, True): (1, 4, 1, 32), + (768, 3072, 1024, 256, 256, True, False, True): (4, 4, 1, 32), + (768, 3072, 2048, 32, 32, False, True, True): (3, 32, 1, 8), + (768, 3072, 2048, 32, 32, True, False, True): (4, 8, 3, 4), + (768, 3072, 2048, 64, 64, False, True, True): (5, 16, 1, 16), + (768, 3072, 2048, 64, 64, True, False, True): (6, 8, 3, 4), + (768, 3072, 2048, 128, 128, False, True, True): (2, 16, 1, 16), + (768, 3072, 2048, 128, 128, True, False, True): (1, 16, 4, 4), + (768, 3072, 2048, 256, 256, False, True, True): (1, 8, 1, 32), + (768, 3072, 2048, 256, 256, True, False, True): (1, 8, 1, 32), + (768, 3072, 4096, 32, 32, False, True, True): (1, 64, 1, 8), + (768, 3072, 4096, 32, 32, True, False, True): (1, 32, 3, 4), + (768, 3072, 4096, 64, 64, False, True, True): (1, 64, 1, 8), + (768, 3072, 4096, 64, 64, True, False, True): (2, 16, 3, 4), + (768, 3072, 4096, 128, 128, False, True, True): (1, 32, 1, 8), + (768, 3072, 4096, 128, 128, True, False, True): (2, 32, 2, 4), + (768, 3072, 4096, 256, 256, False, True, True): (1, 16, 1, 32), + (768, 3072, 4096, 256, 256, True, False, True): (1, 16, 1, 32), + (768, 3072, 8192, 32, 32, False, True, True): (1, 128, 1, 8), + (768, 3072, 8192, 32, 32, True, False, True): (1, 64, 1, 4), + (768, 3072, 8192, 64, 64, False, True, True): (1, 128, 1, 8), + (768, 3072, 8192, 64, 64, True, False, True): (2, 32, 3, 4), + (768, 3072, 8192, 128, 128, False, True, True): (2, 64, 1, 16), + (768, 3072, 8192, 128, 128, True, False, True): (2, 64, 3, 4), + (768, 3072, 8192, 256, 256, False, True, True): (1, 32, 1, 32), + (768, 3072, 8192, 256, 256, True, False, True): (1, 32, 1, 32), + (768, 3072, 16384, 32, 32, False, True, True): (1, 256, 1, 8), + (768, 3072, 16384, 32, 32, True, False, True): (1, 128, 1, 4), + (768, 3072, 16384, 64, 64, False, True, True): (1, 256, 1, 8), + (768, 3072, 16384, 64, 64, True, False, True): (2, 64, 3, 4), + (768, 3072, 16384, 128, 128, False, True, True): (2, 128, 1, 16), + (768, 3072, 16384, 128, 128, True, False, True): (2, 128, 3, 4), + (768, 3072, 16384, 256, 256, False, True, True): (1, 64, 1, 32), + (768, 3072, 16384, 256, 256, True, False, True): (1, 64, 1, 32), + (768, 3072, 32768, 32, 32, False, True, True): (1, 512, 1, 8), + (768, 3072, 32768, 32, 32, True, False, True): (1, 256, 3, 2), + (768, 3072, 32768, 64, 64, False, True, True): (1, 512, 1, 8), + (768, 3072, 32768, 64, 64, True, False, True): (3, 128, 3, 4), + (768, 3072, 32768, 128, 128, False, True, True): (2, 256, 1, 16), + (768, 3072, 32768, 128, 128, True, False, True): (2, 256, 3, 4), + (768, 3072, 32768, 256, 256, False, True, True): (1, 128, 1, 32), + (768, 3072, 32768, 256, 256, True, False, True): (1, 128, 1, 32), + (768, 3072, 50432, 32, 32, False, True, True): (1, 788, 1, 8), + (768, 3072, 50432, 32, 32, True, False, True): (1, 394, 3, 2), + (768, 3072, 50432, 64, 64, False, True, True): (1, 788, 1, 8), + (768, 3072, 50432, 64, 64, True, False, True): (2, 197, 3, 4), + (768, 3072, 50432, 128, 128, False, True, True): (2, 394, 1, 16), + (768, 3072, 50432, 128, 128, True, False, True): (2, 394, 3, 4), + (768, 3072, 50432, 256, 256, False, True, True): (1, 197, 1, 32), + (768, 3072, 50432, 256, 256, True, False, True): (1, 197, 1, 32), + (768, 3072, 65536, 32, 32, False, True, True): (1, 1024, 1, 8), + (768, 3072, 65536, 32, 32, True, False, True): (1, 512, 3, 2), + (768, 3072, 65536, 64, 64, False, True, True): (1, 1024, 1, 8), + (768, 3072, 65536, 64, 64, True, False, True): (2, 256, 3, 4), + (768, 3072, 65536, 128, 128, False, True, True): (2, 512, 1, 16), + (768, 3072, 65536, 128, 128, True, False, True): (2, 512, 3, 4), + (768, 3072, 65536, 256, 256, False, True, True): (1, 256, 1, 32), + (768, 3072, 65536, 256, 256, True, False, True): (1, 256, 1, 32), + (768, 3072, 131072, 32, 32, False, True, True): (1, 2048, 1, 8), + (768, 3072, 131072, 32, 32, True, False, True): (1, 1024, 3, 2), + (768, 3072, 131072, 64, 64, False, True, True): (1, 2048, 1, 8), + (768, 3072, 131072, 64, 64, True, False, True): (2, 512, 3, 4), + (768, 3072, 131072, 128, 128, False, True, True): (2, 1024, 1, 16), + (768, 3072, 131072, 128, 128, True, False, True): (1, 1024, 3, 4), + (768, 3072, 131072, 256, 256, False, True, True): (1, 512, 1, 32), + (768, 3072, 131072, 256, 256, True, False, True): (1, 512, 1, 32), + (1024, 1024, 256, 32, 32, False, True, True): (1, 8, 1, 4), + (1024, 1024, 256, 32, 32, True, False, True): (1, 8, 5, 4), + (1024, 1024, 256, 64, 64, False, True, True): (1, 4, 1, 16), + (1024, 1024, 256, 64, 64, True, False, True): (4, 4, 4, 4), + (1024, 1024, 256, 128, 128, False, True, True): (1, 2, 1, 8), + (1024, 1024, 256, 128, 128, True, False, True): (1, 2, 3, 8), + (1024, 1024, 256, 256, 256, False, True, True): (1, 1, 1, 32), + (1024, 1024, 256, 256, 256, True, False, True): (1, 1, 1, 32), + (1024, 1024, 512, 32, 32, False, True, True): (5, 16, 1, 4), + (1024, 1024, 512, 32, 32, True, False, True): (2, 8, 4, 2), + (1024, 1024, 512, 64, 64, False, True, True): (4, 8, 1, 16), + (1024, 1024, 512, 64, 64, True, False, True): (1, 4, 3, 4), + (1024, 1024, 512, 128, 128, False, True, True): (3, 4, 1, 16), + (1024, 1024, 512, 128, 128, True, False, True): (1, 4, 2, 4), + (1024, 1024, 512, 256, 256, False, True, True): (1, 2, 1, 32), + (1024, 1024, 512, 256, 256, True, False, True): (1, 2, 1, 32), + (1024, 1024, 1024, 32, 32, False, True, True): (1, 16, 1, 8), + (1024, 1024, 1024, 32, 32, True, False, True): (1, 8, 3, 4), + (1024, 1024, 1024, 64, 64, False, True, True): (3, 16, 1, 8), + (1024, 1024, 1024, 64, 64, True, False, True): (1, 16, 3, 2), + (1024, 1024, 1024, 128, 128, False, True, True): (1, 8, 1, 16), + (1024, 1024, 1024, 128, 128, True, False, True): (2, 8, 3, 8), + (1024, 1024, 1024, 256, 256, False, True, True): (1, 4, 1, 32), + (1024, 1024, 1024, 256, 256, True, False, True): (2, 4, 1, 32), + (1024, 1024, 2048, 32, 32, False, True, True): (2, 32, 1, 8), + (1024, 1024, 2048, 32, 32, True, False, True): (3, 16, 1, 4), + (1024, 1024, 2048, 64, 64, False, True, True): (1, 32, 1, 8), + (1024, 1024, 2048, 64, 64, True, False, True): (3, 32, 1, 4), + (1024, 1024, 2048, 128, 128, False, True, True): (4, 16, 1, 16), + (1024, 1024, 2048, 128, 128, True, False, True): (1, 16, 3, 4), + (1024, 1024, 2048, 256, 256, False, True, True): (1, 8, 1, 32), + (1024, 1024, 2048, 256, 256, True, False, True): (1, 8, 1, 32), + (1024, 1024, 4096, 32, 32, False, True, True): (4, 64, 1, 8), + (1024, 1024, 4096, 32, 32, True, False, True): (3, 32, 1, 4), + (1024, 1024, 4096, 64, 64, False, True, True): (3, 64, 1, 8), + (1024, 1024, 4096, 64, 64, True, False, True): (1, 32, 3, 2), + (1024, 1024, 4096, 128, 128, False, True, True): (4, 32, 1, 16), + (1024, 1024, 4096, 128, 128, True, False, True): (2, 32, 2, 4), + (1024, 1024, 4096, 256, 256, False, True, True): (1, 16, 1, 32), + (1024, 1024, 4096, 256, 256, True, False, True): (7, 16, 1, 32), + (1024, 1024, 8192, 32, 32, False, True, True): (1, 128, 1, 8), + (1024, 1024, 8192, 32, 32, True, False, True): (4, 64, 1, 4), + (1024, 1024, 8192, 64, 64, False, True, True): (2, 128, 1, 8), + (1024, 1024, 8192, 64, 64, True, False, True): (3, 32, 3, 4), + (1024, 1024, 8192, 128, 128, False, True, True): (4, 64, 1, 16), + (1024, 1024, 8192, 128, 128, True, False, True): (2, 64, 2, 4), + (1024, 1024, 8192, 256, 256, False, True, True): (1, 32, 1, 32), + (1024, 1024, 8192, 256, 256, True, False, True): (1, 32, 1, 32), + (1024, 1024, 16384, 32, 32, False, True, True): (1, 256, 1, 8), + (1024, 1024, 16384, 32, 32, True, False, True): (1, 128, 1, 4), + (1024, 1024, 16384, 64, 64, False, True, True): (1, 256, 1, 8), + (1024, 1024, 16384, 64, 64, True, False, True): (4, 64, 3, 4), + (1024, 1024, 16384, 128, 128, False, True, True): (4, 128, 1, 16), + (1024, 1024, 16384, 128, 128, True, False, True): (1, 128, 3, 4), + (1024, 1024, 16384, 256, 256, False, True, True): (1, 64, 1, 32), + (1024, 1024, 16384, 256, 256, True, False, True): (1, 64, 1, 32), + (1024, 1024, 32768, 32, 32, False, True, True): (1, 512, 1, 8), + (1024, 1024, 32768, 32, 32, True, False, True): (1, 256, 3, 2), + (1024, 1024, 32768, 64, 64, False, True, True): (1, 256, 1, 4), + (1024, 1024, 32768, 64, 64, True, False, True): (4, 128, 3, 4), + (1024, 1024, 32768, 128, 128, False, True, True): (4, 256, 1, 16), + (1024, 1024, 32768, 128, 128, True, False, True): (2, 256, 3, 4), + (1024, 1024, 32768, 256, 256, False, True, True): (1, 128, 1, 32), + (1024, 1024, 32768, 256, 256, True, False, True): (2, 128, 1, 32), + (1024, 1024, 65536, 32, 32, False, True, True): (1, 1024, 1, 8), + (1024, 1024, 65536, 32, 32, True, False, True): (1, 512, 3, 2), + (1024, 1024, 65536, 64, 64, False, True, True): (1, 512, 1, 4), + (1024, 1024, 65536, 64, 64, True, False, True): (2, 256, 3, 4), + (1024, 1024, 65536, 128, 128, False, True, True): (4, 512, 1, 16), + (1024, 1024, 65536, 128, 128, True, False, True): (4, 512, 3, 4), + (1024, 1024, 65536, 256, 256, False, True, True): (1, 256, 1, 32), + (1024, 1024, 65536, 256, 256, True, False, True): (1, 256, 1, 32), + (1024, 1024, 65792, 32, 32, False, True, True): (1, 1028, 1, 8), + (1024, 1024, 65792, 32, 32, True, False, True): (1, 514, 3, 2), + (1024, 1024, 65792, 64, 64, False, True, True): (2, 514, 1, 4), + (1024, 1024, 65792, 64, 64, True, False, True): (4, 257, 3, 4), + (1024, 1024, 65792, 128, 128, False, True, True): (2, 514, 1, 16), + (1024, 1024, 65792, 128, 128, True, False, True): (2, 514, 2, 4), + (1024, 1024, 65792, 256, 256, False, True, True): (1, 257, 1, 32), + (1024, 1024, 65792, 256, 256, True, False, True): (1, 257, 1, 32), + (1024, 1024, 131072, 32, 32, False, True, True): (1, 2048, 1, 8), + (1024, 1024, 131072, 32, 32, True, False, True): (1, 1024, 3, 2), + (1024, 1024, 131072, 64, 64, False, True, True): (2, 1024, 1, 4), + (1024, 1024, 131072, 64, 64, True, False, True): (2, 512, 3, 4), + (1024, 1024, 131072, 128, 128, False, True, True): (4, 1024, 1, 16), + (1024, 1024, 131072, 128, 128, True, False, True): (1, 1024, 3, 4), + (1024, 1024, 131072, 256, 256, False, True, True): (1, 512, 1, 32), + (1024, 1024, 131072, 256, 256, True, False, True): (1, 512, 1, 32), + (1280, 5120, 65792, 32, 32, False, True, True): (1, 1028, 1, 8), + (1280, 5120, 65792, 32, 32, True, False, True): (1, 514, 3, 2), + (1280, 5120, 65792, 64, 64, False, True, True): (1, 1028, 1, 8), + (1280, 5120, 65792, 64, 64, True, False, True): (2, 257, 3, 4), + (1280, 5120, 65792, 128, 128, False, True, True): (2, 514, 1, 16), + (1280, 5120, 65792, 128, 128, True, False, True): (1, 514, 3, 4), + (1280, 5120, 65792, 256, 256, False, True, True): (1, 257, 1, 32), + (1280, 5120, 65792, 256, 256, True, False, True): (1, 257, 1, 32), + (1536, 1536, 256, 32, 32, False, True, True): (1, 8, 1, 4), + (1536, 1536, 256, 32, 32, True, False, True): (2, 8, 1, 8), + (1536, 1536, 256, 64, 64, False, True, True): (4, 4, 1, 16), + (1536, 1536, 256, 64, 64, True, False, True): (1, 4, 4, 4), + (1536, 1536, 256, 128, 128, False, True, True): (2, 2, 1, 16), + (1536, 1536, 256, 128, 128, True, False, True): (2, 2, 3, 4), + (1536, 1536, 256, 256, 256, False, True, True): (1, 1, 1, 32), + (1536, 1536, 256, 256, 256, True, False, True): (1, 1, 1, 32), + (1536, 1536, 512, 32, 32, False, True, True): (1, 8, 1, 8), + (1536, 1536, 512, 32, 32, True, False, True): (3, 4, 4, 4), + (1536, 1536, 512, 64, 64, False, True, True): (3, 8, 1, 16), + (1536, 1536, 512, 64, 64, True, False, True): (1, 4, 3, 4), + (1536, 1536, 512, 128, 128, False, True, True): (1, 4, 1, 16), + (1536, 1536, 512, 128, 128, True, False, True): (2, 4, 4, 4), + (1536, 1536, 512, 256, 256, False, True, True): (1, 2, 1, 32), + (1536, 1536, 512, 256, 256, True, False, True): (1, 2, 1, 32), + (1536, 1536, 1024, 32, 32, False, True, True): (4, 16, 1, 8), + (1536, 1536, 1024, 32, 32, True, False, True): (2, 8, 1, 4), + (1536, 1536, 1024, 64, 64, False, True, True): (2, 16, 1, 16), + (1536, 1536, 1024, 64, 64, True, False, True): (2, 4, 3, 4), + (1536, 1536, 1024, 128, 128, False, True, True): (3, 8, 1, 32), + (1536, 1536, 1024, 128, 128, True, False, True): (4, 8, 3, 4), + (1536, 1536, 1024, 256, 256, False, True, True): (1, 4, 1, 32), + (1536, 1536, 1024, 256, 256, True, False, True): (1, 4, 1, 32), + (1536, 1536, 2048, 32, 32, False, True, True): (1, 32, 1, 8), + (1536, 1536, 2048, 32, 32, True, False, True): (1, 16, 1, 4), + (1536, 1536, 2048, 64, 64, False, True, True): (1, 32, 1, 8), + (1536, 1536, 2048, 64, 64, True, False, True): (1, 16, 2, 2), + (1536, 1536, 2048, 128, 128, False, True, True): (2, 16, 1, 16), + (1536, 1536, 2048, 128, 128, True, False, True): (4, 16, 2, 4), + (1536, 1536, 2048, 256, 256, False, True, True): (1, 8, 1, 32), + (1536, 1536, 2048, 256, 256, True, False, True): (1, 8, 1, 32), + (1536, 1536, 4096, 32, 32, False, True, True): (1, 64, 1, 8), + (1536, 1536, 4096, 32, 32, True, False, True): (1, 32, 1, 4), + (1536, 1536, 4096, 64, 64, False, True, True): (3, 64, 1, 8), + (1536, 1536, 4096, 64, 64, True, False, True): (1, 32, 3, 2), + (1536, 1536, 4096, 128, 128, False, True, True): (1, 32, 1, 8), + (1536, 1536, 4096, 128, 128, True, False, True): (2, 32, 2, 4), + (1536, 1536, 4096, 256, 256, False, True, True): (1, 16, 1, 32), + (1536, 1536, 4096, 256, 256, True, False, True): (2, 16, 1, 32), + (1536, 1536, 8192, 32, 32, False, True, True): (1, 128, 1, 8), + (1536, 1536, 8192, 32, 32, True, False, True): (1, 64, 1, 4), + (1536, 1536, 8192, 64, 64, False, True, True): (3, 128, 1, 8), + (1536, 1536, 8192, 64, 64, True, False, True): (1, 64, 3, 2), + (1536, 1536, 8192, 128, 128, False, True, True): (1, 64, 1, 8), + (1536, 1536, 8192, 128, 128, True, False, True): (1, 64, 2, 4), + (1536, 1536, 8192, 256, 256, False, True, True): (1, 32, 1, 32), + (1536, 1536, 8192, 256, 256, True, False, True): (2, 32, 1, 32), + (1536, 1536, 16384, 32, 32, False, True, True): (1, 256, 1, 8), + (1536, 1536, 16384, 32, 32, True, False, True): (1, 128, 3, 2), + (1536, 1536, 16384, 64, 64, False, True, True): (2, 128, 1, 4), + (1536, 1536, 16384, 64, 64, True, False, True): (2, 64, 3, 4), + (1536, 1536, 16384, 128, 128, False, True, True): (1, 128, 1, 8), + (1536, 1536, 16384, 128, 128, True, False, True): (2, 128, 2, 4), + (1536, 1536, 16384, 256, 256, False, True, True): (1, 64, 1, 32), + (1536, 1536, 16384, 256, 256, True, False, True): (2, 64, 1, 32), + (1536, 1536, 32768, 32, 32, False, True, True): (1, 512, 1, 8), + (1536, 1536, 32768, 32, 32, True, False, True): (1, 256, 3, 2), + (1536, 1536, 32768, 64, 64, False, True, True): (1, 256, 1, 4), + (1536, 1536, 32768, 64, 64, True, False, True): (3, 128, 3, 4), + (1536, 1536, 32768, 128, 128, False, True, True): (1, 256, 1, 8), + (1536, 1536, 32768, 128, 128, True, False, True): (1, 256, 2, 4), + (1536, 1536, 32768, 256, 256, False, True, True): (1, 128, 1, 32), + (1536, 1536, 32768, 256, 256, True, False, True): (2, 128, 1, 32), + (1536, 1536, 65536, 32, 32, False, True, True): (1, 1024, 1, 8), + (1536, 1536, 65536, 32, 32, True, False, True): (1, 512, 3, 2), + (1536, 1536, 65536, 64, 64, False, True, True): (1, 512, 1, 4), + (1536, 1536, 65536, 64, 64, True, False, True): (1, 512, 3, 2), + (1536, 1536, 65536, 128, 128, False, True, True): (1, 512, 1, 8), + (1536, 1536, 65536, 128, 128, True, False, True): (1, 512, 3, 4), + (1536, 1536, 65536, 256, 256, False, True, True): (1, 256, 1, 32), + (1536, 1536, 65536, 256, 256, True, False, True): (2, 256, 1, 32), + (1536, 1536, 131072, 32, 32, False, True, True): (1, 2048, 1, 8), + (1536, 1536, 131072, 32, 32, True, False, True): (1, 1024, 3, 2), + (1536, 1536, 131072, 64, 64, False, True, True): (3, 1024, 1, 4), + (1536, 1536, 131072, 64, 64, True, False, True): (3, 512, 3, 4), + (1536, 1536, 131072, 128, 128, False, True, True): (1, 1024, 1, 8), + (1536, 1536, 131072, 128, 128, True, False, True): (1, 1024, 3, 4), + (1536, 1536, 131072, 256, 256, False, True, True): (1, 512, 1, 32), + (1536, 1536, 131072, 256, 256, True, False, True): (2, 512, 1, 32), + (2048, 2048, 256, 32, 32, False, True, True): (3, 8, 1, 4), + (2048, 2048, 256, 32, 32, True, False, True): (1, 4, 4, 2), + (2048, 2048, 256, 64, 64, False, True, True): (2, 4, 1, 16), + (2048, 2048, 256, 64, 64, True, False, True): (1, 2, 3, 4), + (2048, 2048, 256, 128, 128, False, True, True): (1, 2, 1, 8), + (2048, 2048, 256, 128, 128, True, False, True): (1, 2, 4, 4), + (2048, 2048, 256, 256, 256, False, True, True): (1, 1, 1, 32), + (2048, 2048, 256, 256, 256, True, False, True): (1, 1, 1, 32), + (2048, 2048, 512, 32, 32, False, True, True): (3, 8, 1, 8), + (2048, 2048, 512, 32, 32, True, False, True): (4, 4, 3, 2), + (2048, 2048, 512, 64, 64, False, True, True): (1, 8, 1, 8), + (2048, 2048, 512, 64, 64, True, False, True): (1, 8, 3, 4), + (2048, 2048, 512, 128, 128, False, True, True): (1, 4, 1, 8), + (2048, 2048, 512, 128, 128, True, False, True): (1, 4, 4, 4), + (2048, 2048, 512, 256, 256, False, True, True): (1, 2, 1, 32), + (2048, 2048, 512, 256, 256, True, False, True): (2, 2, 1, 32), + (2048, 2048, 1024, 32, 32, False, True, True): (1, 16, 1, 8), + (2048, 2048, 1024, 32, 32, True, False, True): (3, 8, 1, 4), + (2048, 2048, 1024, 64, 64, False, True, True): (4, 16, 1, 8), + (2048, 2048, 1024, 64, 64, True, False, True): (1, 8, 3, 2), + (2048, 2048, 1024, 128, 128, False, True, True): (4, 8, 1, 16), + (2048, 2048, 1024, 128, 128, True, False, True): (2, 8, 2, 4), + (2048, 2048, 1024, 256, 256, False, True, True): (1, 4, 1, 32), + (2048, 2048, 1024, 256, 256, True, False, True): (3, 4, 1, 32), + (2048, 2048, 2048, 32, 32, False, True, True): (1, 32, 1, 8), + (2048, 2048, 2048, 32, 32, True, False, True): (1, 16, 1, 4), + (2048, 2048, 2048, 64, 64, False, True, True): (1, 32, 1, 8), + (2048, 2048, 2048, 64, 64, True, False, True): (1, 16, 3, 2), + (2048, 2048, 2048, 128, 128, False, True, True): (4, 16, 1, 16), + (2048, 2048, 2048, 128, 128, True, False, True): (2, 16, 2, 4), + (2048, 2048, 2048, 256, 256, False, True, True): (1, 8, 1, 32), + (2048, 2048, 2048, 256, 256, True, False, True): (1, 8, 1, 32), + (2048, 2048, 4096, 32, 32, False, True, True): (1, 64, 1, 8), + (2048, 2048, 4096, 32, 32, True, False, True): (1, 32, 1, 4), + (2048, 2048, 4096, 64, 64, False, True, True): (4, 64, 1, 8), + (2048, 2048, 4096, 64, 64, True, False, True): (2, 16, 3, 4), + (2048, 2048, 4096, 128, 128, False, True, True): (4, 32, 1, 8), + (2048, 2048, 4096, 128, 128, True, False, True): (1, 32, 2, 4), + (2048, 2048, 4096, 256, 256, False, True, True): (1, 16, 1, 32), + (2048, 2048, 4096, 256, 256, True, False, True): (4, 16, 1, 32), + (2048, 2048, 8192, 32, 32, False, True, True): (1, 128, 1, 8), + (2048, 2048, 8192, 32, 32, True, False, True): (1, 64, 1, 4), + (2048, 2048, 8192, 64, 64, False, True, True): (2, 64, 1, 4), + (2048, 2048, 8192, 64, 64, True, False, True): (2, 32, 3, 4), + (2048, 2048, 8192, 128, 128, False, True, True): (4, 64, 1, 8), + (2048, 2048, 8192, 128, 128, True, False, True): (2, 64, 2, 4), + (2048, 2048, 8192, 256, 256, False, True, True): (1, 32, 1, 32), + (2048, 2048, 8192, 256, 256, True, False, True): (4, 32, 1, 32), + (2048, 2048, 16384, 32, 32, False, True, True): (1, 256, 1, 8), + (2048, 2048, 16384, 32, 32, True, False, True): (1, 128, 3, 2), + (2048, 2048, 16384, 64, 64, False, True, True): (2, 128, 1, 4), + (2048, 2048, 16384, 64, 64, True, False, True): (2, 64, 3, 4), + (2048, 2048, 16384, 128, 128, False, True, True): (1, 128, 1, 8), + (2048, 2048, 16384, 128, 128, True, False, True): (2, 128, 2, 4), + (2048, 2048, 16384, 256, 256, False, True, True): (1, 64, 1, 32), + (2048, 2048, 16384, 256, 256, True, False, True): (4, 64, 1, 32), + (2048, 2048, 32768, 32, 32, False, True, True): (1, 512, 1, 8), + (2048, 2048, 32768, 32, 32, True, False, True): (1, 256, 3, 2), + (2048, 2048, 32768, 64, 64, False, True, True): (2, 256, 1, 4), + (2048, 2048, 32768, 64, 64, True, False, True): (2, 128, 3, 4), + (2048, 2048, 32768, 128, 128, False, True, True): (1, 256, 1, 8), + (2048, 2048, 32768, 128, 128, True, False, True): (2, 256, 2, 4), + (2048, 2048, 32768, 256, 256, False, True, True): (1, 128, 1, 32), + (2048, 2048, 32768, 256, 256, True, False, True): (4, 128, 1, 32), + (2048, 2048, 65536, 32, 32, False, True, True): (1, 1024, 1, 8), + (2048, 2048, 65536, 32, 32, True, False, True): (1, 512, 3, 2), + (2048, 2048, 65536, 64, 64, False, True, True): (1, 512, 1, 4), + (2048, 2048, 65536, 64, 64, True, False, True): (2, 256, 3, 4), + (2048, 2048, 65536, 128, 128, False, True, True): (1, 512, 1, 8), + (2048, 2048, 65536, 128, 128, True, False, True): (1, 512, 2, 4), + (2048, 2048, 65536, 256, 256, False, True, True): (1, 256, 1, 32), + (2048, 2048, 65536, 256, 256, True, False, True): (4, 256, 1, 32), + (2048, 2048, 65792, 32, 32, False, True, True): (1, 1028, 1, 8), + (2048, 2048, 65792, 32, 32, True, False, True): (1, 514, 3, 2), + (2048, 2048, 65792, 64, 64, False, True, True): (1, 514, 1, 4), + (2048, 2048, 65792, 64, 64, True, False, True): (2, 257, 3, 4), + (2048, 2048, 65792, 128, 128, False, True, True): (1, 514, 1, 8), + (2048, 2048, 65792, 128, 128, True, False, True): (1, 514, 2, 4), + (2048, 2048, 65792, 256, 256, False, True, True): (1, 257, 1, 32), + (2048, 2048, 65792, 256, 256, True, False, True): (1, 257, 1, 32), + (2048, 2048, 131072, 32, 32, False, True, True): (1, 2048, 1, 8), + (2048, 2048, 131072, 32, 32, True, False, True): (1, 1024, 3, 2), + (2048, 2048, 131072, 64, 64, False, True, True): (1, 1024, 1, 4), + (2048, 2048, 131072, 64, 64, True, False, True): (2, 512, 3, 4), + (2048, 2048, 131072, 128, 128, False, True, True): (1, 1024, 1, 8), + (2048, 2048, 131072, 128, 128, True, False, True): (1, 1024, 3, 4), + (2048, 2048, 131072, 256, 256, False, True, True): (1, 512, 1, 32), + (2048, 2048, 131072, 256, 256, True, False, True): (4, 512, 1, 32), + (3072, 768, 256, 32, 32, False, True, True): (5, 4, 1, 8), + (3072, 768, 256, 32, 32, True, False, True): (2, 2, 4, 4), + (3072, 768, 256, 64, 64, False, True, True): (1, 4, 1, 16), + (3072, 768, 256, 64, 64, True, False, True): (2, 2, 3, 4), + (3072, 768, 256, 128, 128, False, True, True): (5, 2, 1, 16), + (3072, 768, 256, 128, 128, True, False, True): (1, 2, 5, 4), + (3072, 768, 256, 256, 256, False, True, True): (1, 1, 1, 32), + (3072, 768, 256, 256, 256, True, False, True): (1, 1, 1, 32), + (3072, 768, 512, 32, 32, False, True, True): (1, 8, 1, 8), + (3072, 768, 512, 32, 32, True, False, True): (5, 4, 1, 4), + (3072, 768, 512, 64, 64, False, True, True): (1, 8, 1, 8), + (3072, 768, 512, 64, 64, True, False, True): (3, 2, 3, 4), + (3072, 768, 512, 128, 128, False, True, True): (3, 4, 1, 32), + (3072, 768, 512, 128, 128, True, False, True): (2, 4, 3, 4), + (3072, 768, 512, 256, 256, False, True, True): (1, 2, 1, 32), + (3072, 768, 512, 256, 256, True, False, True): (2, 2, 1, 32), + (3072, 768, 1024, 32, 32, False, True, True): (2, 16, 1, 8), + (3072, 768, 1024, 32, 32, True, False, True): (3, 8, 1, 4), + (3072, 768, 1024, 64, 64, False, True, True): (4, 16, 1, 8), + (3072, 768, 1024, 64, 64, True, False, True): (1, 8, 3, 2), + (3072, 768, 1024, 128, 128, False, True, True): (2, 8, 1, 32), + (3072, 768, 1024, 128, 128, True, False, True): (3, 8, 2, 4), + (3072, 768, 1024, 256, 256, False, True, True): (1, 4, 1, 32), + (3072, 768, 1024, 256, 256, True, False, True): (4, 4, 1, 32), + (3072, 768, 2048, 32, 32, False, True, True): (1, 32, 1, 8), + (3072, 768, 2048, 32, 32, True, False, True): (1, 16, 1, 4), + (3072, 768, 2048, 64, 64, False, True, True): (2, 32, 1, 8), + (3072, 768, 2048, 64, 64, True, False, True): (2, 8, 3, 4), + (3072, 768, 2048, 128, 128, False, True, True): (2, 16, 1, 16), + (3072, 768, 2048, 128, 128, True, False, True): (2, 16, 1, 4), + (3072, 768, 2048, 256, 256, False, True, True): (1, 8, 1, 32), + (3072, 768, 2048, 256, 256, True, False, True): (2, 8, 1, 32), + (3072, 768, 4096, 32, 32, False, True, True): (1, 64, 1, 8), + (3072, 768, 4096, 32, 32, True, False, True): (1, 32, 1, 2), + (3072, 768, 4096, 64, 64, False, True, True): (2, 64, 1, 8), + (3072, 768, 4096, 64, 64, True, False, True): (2, 32, 2, 2), + (3072, 768, 4096, 128, 128, False, True, True): (1, 32, 1, 8), + (3072, 768, 4096, 128, 128, True, False, True): (2, 32, 2, 4), + (3072, 768, 4096, 256, 256, False, True, True): (1, 16, 1, 32), + (3072, 768, 4096, 256, 256, True, False, True): (4, 16, 1, 32), + (3072, 768, 8192, 32, 32, False, True, True): (1, 128, 1, 8), + (3072, 768, 8192, 32, 32, True, False, True): (3, 64, 1, 2), + (3072, 768, 8192, 64, 64, False, True, True): (1, 128, 1, 8), + (3072, 768, 8192, 64, 64, True, False, True): (2, 64, 2, 2), + (3072, 768, 8192, 128, 128, False, True, True): (1, 64, 1, 8), + (3072, 768, 8192, 128, 128, True, False, True): (2, 64, 2, 4), + (3072, 768, 8192, 256, 256, False, True, True): (1, 32, 1, 32), + (3072, 768, 8192, 256, 256, True, False, True): (4, 32, 1, 32), + (3072, 768, 16384, 32, 32, False, True, True): (1, 256, 1, 8), + (3072, 768, 16384, 32, 32, True, False, True): (1, 128, 1, 2), + (3072, 768, 16384, 64, 64, False, True, True): (2, 128, 1, 4), + (3072, 768, 16384, 64, 64, True, False, True): (1, 128, 2, 2), + (3072, 768, 16384, 128, 128, False, True, True): (1, 128, 1, 8), + (3072, 768, 16384, 128, 128, True, False, True): (1, 128, 1, 4), + (3072, 768, 16384, 256, 256, False, True, True): (1, 64, 1, 32), + (3072, 768, 16384, 256, 256, True, False, True): (4, 64, 1, 32), + (3072, 768, 32768, 32, 32, False, True, True): (1, 512, 1, 8), + (3072, 768, 32768, 32, 32, True, False, True): (1, 256, 1, 2), + (3072, 768, 32768, 64, 64, False, True, True): (1, 256, 1, 4), + (3072, 768, 32768, 64, 64, True, False, True): (2, 256, 2, 2), + (3072, 768, 32768, 128, 128, False, True, True): (1, 256, 1, 8), + (3072, 768, 32768, 128, 128, True, False, True): (2, 256, 1, 4), + (3072, 768, 32768, 256, 256, False, True, True): (1, 128, 1, 32), + (3072, 768, 32768, 256, 256, True, False, True): (4, 128, 1, 32), + (3072, 768, 50432, 32, 32, False, True, True): (1, 788, 1, 8), + (3072, 768, 50432, 32, 32, True, False, True): (1, 394, 1, 2), + (3072, 768, 50432, 64, 64, False, True, True): (2, 394, 1, 4), + (3072, 768, 50432, 64, 64, True, False, True): (2, 394, 2, 2), + (3072, 768, 50432, 128, 128, False, True, True): (1, 394, 1, 8), + (3072, 768, 50432, 128, 128, True, False, True): (2, 394, 1, 4), + (3072, 768, 50432, 256, 256, False, True, True): (1, 197, 1, 32), + (3072, 768, 50432, 256, 256, True, False, True): (1, 197, 1, 32), + (3072, 768, 65536, 32, 32, False, True, True): (1, 1024, 1, 8), + (3072, 768, 65536, 32, 32, True, False, True): (1, 512, 1, 2), + (3072, 768, 65536, 64, 64, False, True, True): (1, 512, 1, 4), + (3072, 768, 65536, 64, 64, True, False, True): (2, 512, 2, 2), + (3072, 768, 65536, 128, 128, False, True, True): (1, 512, 1, 8), + (3072, 768, 65536, 128, 128, True, False, True): (2, 512, 1, 4), + (3072, 768, 65536, 256, 256, False, True, True): (1, 256, 1, 32), + (3072, 768, 65536, 256, 256, True, False, True): (4, 256, 1, 32), + (3072, 768, 131072, 32, 32, False, True, True): (1, 2048, 1, 8), + (3072, 768, 131072, 32, 32, True, False, True): (1, 1024, 1, 2), + (3072, 768, 131072, 64, 64, False, True, True): (2, 1024, 1, 4), + (3072, 768, 131072, 64, 64, True, False, True): (2, 1024, 2, 2), + (3072, 768, 131072, 128, 128, False, True, True): (1, 1024, 1, 8), + (3072, 768, 131072, 128, 128, True, False, True): (2, 1024, 1, 4), + (3072, 768, 131072, 256, 256, False, True, True): (1, 512, 1, 32), + (3072, 768, 131072, 256, 256, True, False, True): (4, 512, 1, 32), + (3072, 3072, 256, 32, 32, False, True, True): (1, 4, 1, 8), + (3072, 3072, 256, 32, 32, True, False, True): (2, 2, 5, 4), + (3072, 3072, 256, 64, 64, False, True, True): (2, 4, 1, 16), + (3072, 3072, 256, 64, 64, True, False, True): (3, 2, 3, 4), + (3072, 3072, 256, 128, 128, False, True, True): (1, 2, 1, 8), + (3072, 3072, 256, 128, 128, True, False, True): (1, 2, 5, 4), + (3072, 3072, 256, 256, 256, False, True, True): (1, 1, 1, 32), + (3072, 3072, 256, 256, 256, True, False, True): (1, 1, 1, 32), + (3072, 3072, 512, 32, 32, False, True, True): (1, 8, 1, 8), + (3072, 3072, 512, 32, 32, True, False, True): (3, 2, 3, 4), + (3072, 3072, 512, 64, 64, False, True, True): (1, 8, 1, 8), + (3072, 3072, 512, 64, 64, True, False, True): (3, 2, 3, 4), + (3072, 3072, 512, 128, 128, False, True, True): (2, 4, 1, 8), + (3072, 3072, 512, 128, 128, True, False, True): (2, 4, 4, 4), + (3072, 3072, 512, 256, 256, False, True, True): (1, 2, 1, 32), + (3072, 3072, 512, 256, 256, True, False, True): (1, 2, 1, 32), + (3072, 3072, 1024, 32, 32, False, True, True): (1, 16, 1, 8), + (3072, 3072, 1024, 32, 32, True, False, True): (3, 8, 3, 4), + (3072, 3072, 1024, 64, 64, False, True, True): (2, 16, 1, 8), + (3072, 3072, 1024, 64, 64, True, False, True): (2, 4, 3, 4), + (3072, 3072, 1024, 128, 128, False, True, True): (1, 8, 1, 8), + (3072, 3072, 1024, 128, 128, True, False, True): (3, 8, 2, 4), + (3072, 3072, 1024, 256, 256, False, True, True): (1, 4, 1, 32), + (3072, 3072, 1024, 256, 256, True, False, True): (3, 4, 1, 32), + (3072, 3072, 2048, 32, 32, False, True, True): (1, 32, 1, 8), + (3072, 3072, 2048, 32, 32, True, False, True): (1, 16, 1, 4), + (3072, 3072, 2048, 64, 64, False, True, True): (1, 32, 1, 8), + (3072, 3072, 2048, 64, 64, True, False, True): (1, 16, 3, 2), + (3072, 3072, 2048, 128, 128, False, True, True): (1, 16, 1, 8), + (3072, 3072, 2048, 128, 128, True, False, True): (2, 16, 2, 4), + (3072, 3072, 2048, 256, 256, False, True, True): (1, 8, 1, 32), + (3072, 3072, 2048, 256, 256, True, False, True): (3, 8, 1, 32), + (3072, 3072, 4096, 32, 32, False, True, True): (1, 64, 1, 8), + (3072, 3072, 4096, 32, 32, True, False, True): (1, 32, 1, 4), + (3072, 3072, 4096, 64, 64, False, True, True): (1, 64, 1, 8), + (3072, 3072, 4096, 64, 64, True, False, True): (3, 16, 3, 4), + (3072, 3072, 4096, 128, 128, False, True, True): (1, 32, 1, 8), + (3072, 3072, 4096, 128, 128, True, False, True): (2, 32, 2, 4), + (3072, 3072, 4096, 256, 256, False, True, True): (1, 16, 1, 32), + (3072, 3072, 4096, 256, 256, True, False, True): (2, 16, 1, 32), + (3072, 3072, 8192, 32, 32, False, True, True): (1, 128, 1, 8), + (3072, 3072, 8192, 32, 32, True, False, True): (1, 64, 1, 2), + (3072, 3072, 8192, 64, 64, False, True, True): (1, 64, 1, 4), + (3072, 3072, 8192, 64, 64, True, False, True): (1, 64, 3, 2), + (3072, 3072, 8192, 128, 128, False, True, True): (1, 64, 1, 8), + (3072, 3072, 8192, 128, 128, True, False, True): (2, 64, 2, 4), + (3072, 3072, 8192, 256, 256, False, True, True): (1, 32, 1, 32), + (3072, 3072, 8192, 256, 256, True, False, True): (4, 32, 1, 32), + (3072, 3072, 16384, 32, 32, False, True, True): (1, 256, 1, 8), + (3072, 3072, 16384, 32, 32, True, False, True): (1, 128, 3, 2), + (3072, 3072, 16384, 64, 64, False, True, True): (1, 128, 1, 4), + (3072, 3072, 16384, 64, 64, True, False, True): (2, 64, 3, 4), + (3072, 3072, 16384, 128, 128, False, True, True): (1, 128, 1, 8), + (3072, 3072, 16384, 128, 128, True, False, True): (1, 128, 2, 4), + (3072, 3072, 16384, 256, 256, False, True, True): (1, 64, 1, 32), + (3072, 3072, 16384, 256, 256, True, False, True): (4, 64, 1, 32), + (3072, 3072, 32768, 32, 32, False, True, True): (1, 512, 1, 8), + (3072, 3072, 32768, 32, 32, True, False, True): (1, 256, 3, 2), + (3072, 3072, 32768, 64, 64, False, True, True): (1, 256, 1, 4), + (3072, 3072, 32768, 64, 64, True, False, True): (1, 256, 3, 2), + (3072, 3072, 32768, 128, 128, False, True, True): (1, 256, 1, 8), + (3072, 3072, 32768, 128, 128, True, False, True): (1, 256, 2, 4), + (3072, 3072, 32768, 256, 256, False, True, True): (1, 128, 1, 32), + (3072, 3072, 32768, 256, 256, True, False, True): (4, 128, 1, 32), + (3072, 3072, 65536, 32, 32, False, True, True): (1, 1024, 1, 8), + (3072, 3072, 65536, 32, 32, True, False, True): (1, 512, 3, 2), + (3072, 3072, 65536, 64, 64, False, True, True): (1, 512, 1, 4), + (3072, 3072, 65536, 64, 64, True, False, True): (2, 256, 3, 4), + (3072, 3072, 65536, 128, 128, False, True, True): (1, 512, 1, 8), + (3072, 3072, 65536, 128, 128, True, False, True): (1, 512, 3, 4), + (3072, 3072, 65536, 256, 256, False, True, True): (1, 256, 1, 32), + (3072, 3072, 65536, 256, 256, True, False, True): (4, 256, 1, 32), + (3072, 3072, 131072, 32, 32, False, True, True): (1, 2048, 1, 8), + (3072, 3072, 131072, 32, 32, True, False, True): (1, 1024, 3, 2), + (3072, 3072, 131072, 64, 64, False, True, True): (1, 1024, 1, 4), + (3072, 3072, 131072, 64, 64, True, False, True): (1, 1024, 3, 2), + (3072, 3072, 131072, 128, 128, False, True, True): (1, 1024, 1, 8), + (3072, 3072, 131072, 128, 128, True, False, True): (1, 1024, 3, 4), + (3072, 3072, 131072, 256, 256, False, True, True): (1, 512, 1, 32), + (3072, 3072, 131072, 256, 256, True, False, True): (4, 512, 1, 32), + (4096, 4096, 256, 32, 32, False, True, True): (1, 4, 1, 8), + (4096, 4096, 256, 32, 32, True, False, True): (5, 2, 3, 4), + (4096, 4096, 256, 64, 64, False, True, True): (3, 4, 1, 8), + (4096, 4096, 256, 64, 64, True, False, True): (3, 4, 3, 2), + (4096, 4096, 256, 128, 128, False, True, True): (1, 2, 1, 8), + (4096, 4096, 256, 128, 128, True, False, True): (2, 2, 4, 4), + (4096, 4096, 256, 256, 256, False, True, True): (1, 1, 1, 32), + (4096, 4096, 256, 256, 256, True, False, True): (1, 1, 1, 32), + (4096, 4096, 512, 32, 32, False, True, True): (1, 8, 1, 8), + (4096, 4096, 512, 32, 32, True, False, True): (1, 4, 1, 4), + (4096, 4096, 512, 64, 64, False, True, True): (1, 8, 1, 8), + (4096, 4096, 512, 64, 64, True, False, True): (3, 4, 2, 2), + (4096, 4096, 512, 128, 128, False, True, True): (2, 4, 1, 8), + (4096, 4096, 512, 128, 128, True, False, True): (2, 4, 2, 4), + (4096, 4096, 512, 256, 256, False, True, True): (2, 2, 1, 32), + (4096, 4096, 512, 256, 256, True, False, True): (2, 2, 1, 32), + (4096, 4096, 1024, 32, 32, False, True, True): (4, 16, 1, 8), + (4096, 4096, 1024, 32, 32, True, False, True): (1, 8, 1, 4), + (4096, 4096, 1024, 64, 64, False, True, True): (1, 16, 1, 8), + (4096, 4096, 1024, 64, 64, True, False, True): (4, 4, 3, 4), + (4096, 4096, 1024, 128, 128, False, True, True): (2, 8, 1, 8), + (4096, 4096, 1024, 128, 128, True, False, True): (1, 8, 3, 4), + (4096, 4096, 1024, 256, 256, False, True, True): (1, 4, 1, 32), + (4096, 4096, 1024, 256, 256, True, False, True): (6, 4, 1, 32), + (4096, 4096, 2048, 32, 32, False, True, True): (1, 32, 1, 8), + (4096, 4096, 2048, 32, 32, True, False, True): (1, 16, 1, 4), + (4096, 4096, 2048, 64, 64, False, True, True): (4, 32, 1, 8), + (4096, 4096, 2048, 64, 64, True, False, True): (4, 8, 3, 4), + (4096, 4096, 2048, 128, 128, False, True, True): (2, 16, 1, 8), + (4096, 4096, 2048, 128, 128, True, False, True): (1, 16, 3, 4), + (4096, 4096, 2048, 256, 256, False, True, True): (1, 8, 1, 32), + (4096, 4096, 2048, 256, 256, True, False, True): (4, 8, 1, 32), + (4096, 4096, 4096, 32, 32, False, True, True): (1, 64, 1, 8), + (4096, 4096, 4096, 32, 32, True, False, True): (1, 32, 1, 4), + (4096, 4096, 4096, 64, 64, False, True, True): (1, 64, 1, 8), + (4096, 4096, 4096, 64, 64, True, False, True): (1, 32, 3, 2), + (4096, 4096, 4096, 128, 128, False, True, True): (1, 32, 1, 8), + (4096, 4096, 4096, 128, 128, True, False, True): (2, 32, 3, 4), + (4096, 4096, 4096, 256, 256, False, True, True): (1, 16, 1, 32), + (4096, 4096, 4096, 256, 256, True, False, True): (4, 16, 1, 32), + (4096, 4096, 8192, 32, 32, False, True, True): (1, 128, 1, 8), + (4096, 4096, 8192, 32, 32, True, False, True): (1, 64, 1, 4), + (4096, 4096, 8192, 64, 64, False, True, True): (1, 128, 1, 8), + (4096, 4096, 8192, 64, 64, True, False, True): (1, 64, 3, 2), + (4096, 4096, 8192, 128, 128, False, True, True): (1, 64, 1, 8), + (4096, 4096, 8192, 128, 128, True, False, True): (1, 64, 3, 4), + (4096, 4096, 8192, 256, 256, False, True, True): (1, 32, 1, 32), + (4096, 4096, 8192, 256, 256, True, False, True): (4, 32, 1, 32), + (4096, 4096, 16384, 32, 32, False, True, True): (1, 256, 1, 8), + (4096, 4096, 16384, 32, 32, True, False, True): (1, 128, 3, 2), + (4096, 4096, 16384, 64, 64, False, True, True): (1, 128, 1, 4), + (4096, 4096, 16384, 64, 64, True, False, True): (4, 64, 3, 4), + (4096, 4096, 16384, 128, 128, False, True, True): (1, 128, 1, 8), + (4096, 4096, 16384, 128, 128, True, False, True): (1, 128, 3, 4), + (4096, 4096, 16384, 256, 256, False, True, True): (1, 64, 1, 32), + (4096, 4096, 16384, 256, 256, True, False, True): (4, 64, 1, 32), + (4096, 4096, 32768, 32, 32, False, True, True): (1, 512, 1, 8), + (4096, 4096, 32768, 32, 32, True, False, True): (1, 256, 3, 2), + (4096, 4096, 32768, 64, 64, False, True, True): (1, 256, 1, 4), + (4096, 4096, 32768, 64, 64, True, False, True): (1, 256, 3, 2), + (4096, 4096, 32768, 128, 128, False, True, True): (1, 256, 1, 8), + (4096, 4096, 32768, 128, 128, True, False, True): (1, 256, 3, 4), + (4096, 4096, 32768, 256, 256, False, True, True): (1, 128, 1, 32), + (4096, 4096, 32768, 256, 256, True, False, True): (4, 128, 1, 32), + (4096, 4096, 65536, 32, 32, False, True, True): (1, 1024, 1, 8), + (4096, 4096, 65536, 32, 32, True, False, True): (1, 512, 3, 2), + (4096, 4096, 65536, 64, 64, False, True, True): (1, 512, 1, 4), + (4096, 4096, 65536, 64, 64, True, False, True): (4, 256, 3, 4), + (4096, 4096, 65536, 128, 128, False, True, True): (1, 512, 1, 8), + (4096, 4096, 65536, 128, 128, True, False, True): (1, 512, 3, 4), + (4096, 4096, 65536, 256, 256, False, True, True): (1, 256, 1, 32), + (4096, 4096, 65536, 256, 256, True, False, True): (4, 256, 1, 32), + (4096, 4096, 65792, 32, 32, False, True, True): (1, 1028, 1, 8), + (4096, 4096, 65792, 32, 32, True, False, True): (1, 514, 3, 2), + (4096, 4096, 65792, 64, 64, False, True, True): (1, 1028, 1, 8), + (4096, 4096, 65792, 64, 64, True, False, True): (1, 514, 3, 2), + (4096, 4096, 65792, 128, 128, False, True, True): (1, 514, 1, 8), + (4096, 4096, 65792, 128, 128, True, False, True): (1, 514, 2, 4), + (4096, 4096, 65792, 256, 256, False, True, True): (1, 257, 1, 32), + (4096, 4096, 65792, 256, 256, True, False, True): (1, 257, 1, 32), + (4096, 4096, 131072, 32, 32, False, True, True): (1, 2048, 1, 8), + (4096, 4096, 131072, 32, 32, True, False, True): (1, 1024, 3, 2), + (4096, 4096, 131072, 64, 64, False, True, True): (1, 2048, 1, 8), + (4096, 4096, 131072, 64, 64, True, False, True): (1, 1024, 3, 2), + (4096, 4096, 131072, 128, 128, False, True, True): (1, 1024, 1, 8), + (4096, 4096, 131072, 128, 128, True, False, True): (1, 1024, 3, 4), + (4096, 4096, 131072, 256, 256, False, True, True): (1, 512, 1, 32), + (4096, 4096, 131072, 256, 256, True, False, True): (4, 512, 1, 32), + (5120, 1280, 65792, 32, 32, False, True, True): (1, 1028, 1, 8), + (5120, 1280, 65792, 32, 32, True, False, True): (1, 514, 1, 2), + (5120, 1280, 65792, 64, 64, False, True, True): (1, 514, 1, 4), + (5120, 1280, 65792, 64, 64, True, False, True): (1, 514, 2, 2), + (5120, 1280, 65792, 128, 128, False, True, True): (1, 514, 1, 8), + (5120, 1280, 65792, 128, 128, True, False, True): (1, 514, 2, 4), + (5120, 1280, 65792, 256, 256, False, True, True): (1, 257, 1, 32), + (5120, 1280, 65792, 256, 256, True, False, True): (1, 257, 1, 32), + (6144, 6144, 256, 32, 32, False, True, True): (2, 4, 1, 8), + (6144, 6144, 256, 32, 32, True, False, True): (2, 1, 4, 4), + (6144, 6144, 256, 64, 64, False, True, True): (1, 4, 1, 8), + (6144, 6144, 256, 64, 64, True, False, True): (5, 1, 3, 4), + (6144, 6144, 256, 128, 128, False, True, True): (1, 2, 1, 8), + (6144, 6144, 256, 128, 128, True, False, True): (1, 2, 3, 4), + (6144, 6144, 256, 256, 256, False, True, True): (1, 1, 1, 32), + (6144, 6144, 256, 256, 256, True, False, True): (1, 1, 1, 32), + (6144, 6144, 512, 32, 32, False, True, True): (1, 8, 1, 8), + (6144, 6144, 512, 32, 32, True, False, True): (1, 4, 4, 2), + (6144, 6144, 512, 64, 64, False, True, True): (2, 8, 1, 8), + (6144, 6144, 512, 64, 64, True, False, True): (2, 2, 3, 4), + (6144, 6144, 512, 128, 128, False, True, True): (3, 4, 1, 8), + (6144, 6144, 512, 128, 128, True, False, True): (2, 4, 3, 4), + (6144, 6144, 512, 256, 256, False, True, True): (1, 2, 1, 32), + (6144, 6144, 512, 256, 256, True, False, True): (2, 2, 1, 32), + (6144, 6144, 1024, 32, 32, False, True, True): (1, 16, 1, 8), + (6144, 6144, 1024, 32, 32, True, False, True): (1, 8, 1, 4), + (6144, 6144, 1024, 64, 64, False, True, True): (1, 16, 1, 8), + (6144, 6144, 1024, 64, 64, True, False, True): (4, 4, 3, 4), + (6144, 6144, 1024, 128, 128, False, True, True): (1, 8, 1, 8), + (6144, 6144, 1024, 128, 128, True, False, True): (3, 8, 3, 4), + (6144, 6144, 1024, 256, 256, False, True, True): (1, 4, 1, 32), + (6144, 6144, 1024, 256, 256, True, False, True): (1, 4, 1, 32), + (6144, 6144, 2048, 32, 32, False, True, True): (1, 32, 1, 8), + (6144, 6144, 2048, 32, 32, True, False, True): (1, 16, 1, 4), + (6144, 6144, 2048, 64, 64, False, True, True): (1, 32, 1, 8), + (6144, 6144, 2048, 64, 64, True, False, True): (4, 8, 3, 4), + (6144, 6144, 2048, 128, 128, False, True, True): (1, 16, 1, 8), + (6144, 6144, 2048, 128, 128, True, False, True): (3, 16, 3, 4), + (6144, 6144, 2048, 256, 256, False, True, True): (1, 8, 1, 32), + (6144, 6144, 2048, 256, 256, True, False, True): (4, 8, 1, 32), + (6144, 6144, 4096, 32, 32, False, True, True): (1, 64, 1, 8), + (6144, 6144, 4096, 32, 32, True, False, True): (1, 32, 1, 4), + (6144, 6144, 4096, 64, 64, False, True, True): (1, 64, 1, 8), + (6144, 6144, 4096, 64, 64, True, False, True): (4, 16, 3, 4), + (6144, 6144, 4096, 128, 128, False, True, True): (1, 32, 1, 8), + (6144, 6144, 4096, 128, 128, True, False, True): (4, 32, 3, 4), + (6144, 6144, 4096, 256, 256, False, True, True): (1, 16, 1, 32), + (6144, 6144, 4096, 256, 256, True, False, True): (4, 16, 1, 32), + (6144, 6144, 8192, 32, 32, False, True, True): (1, 128, 1, 8), + (6144, 6144, 8192, 32, 32, True, False, True): (1, 64, 1, 4), + (6144, 6144, 8192, 64, 64, False, True, True): (1, 128, 1, 8), + (6144, 6144, 8192, 64, 64, True, False, True): (4, 32, 3, 4), + (6144, 6144, 8192, 128, 128, False, True, True): (1, 64, 1, 8), + (6144, 6144, 8192, 128, 128, True, False, True): (1, 64, 3, 4), + (6144, 6144, 8192, 256, 256, False, True, True): (1, 32, 1, 32), + (6144, 6144, 8192, 256, 256, True, False, True): (4, 32, 1, 32), + (6144, 6144, 16384, 32, 32, False, True, True): (1, 256, 1, 8), + (6144, 6144, 16384, 32, 32, True, False, True): (1, 128, 1, 4), + (6144, 6144, 16384, 64, 64, False, True, True): (1, 256, 1, 8), + (6144, 6144, 16384, 64, 64, True, False, True): (4, 64, 3, 4), + (6144, 6144, 16384, 128, 128, False, True, True): (1, 128, 1, 8), + (6144, 6144, 16384, 128, 128, True, False, True): (4, 128, 3, 4), + (6144, 6144, 16384, 256, 256, False, True, True): (1, 64, 1, 32), + (6144, 6144, 16384, 256, 256, True, False, True): (4, 64, 1, 32), + (6144, 6144, 32768, 32, 32, False, True, True): (1, 512, 1, 8), + (6144, 6144, 32768, 32, 32, True, False, True): (1, 256, 1, 4), + (6144, 6144, 32768, 64, 64, False, True, True): (1, 512, 1, 8), + (6144, 6144, 32768, 64, 64, True, False, True): (4, 128, 3, 4), + (6144, 6144, 32768, 128, 128, False, True, True): (1, 256, 1, 8), + (6144, 6144, 32768, 128, 128, True, False, True): (1, 256, 3, 4), + (6144, 6144, 32768, 256, 256, False, True, True): (1, 128, 1, 32), + (6144, 6144, 32768, 256, 256, True, False, True): (4, 128, 1, 32), + (6144, 6144, 65536, 32, 32, False, True, True): (1, 1024, 1, 8), + (6144, 6144, 65536, 32, 32, True, False, True): (1, 512, 1, 4), + (6144, 6144, 65536, 64, 64, False, True, True): (1, 1024, 1, 8), + (6144, 6144, 65536, 64, 64, True, False, True): (4, 256, 3, 4), + (6144, 6144, 65536, 128, 128, False, True, True): (1, 512, 1, 8), + (6144, 6144, 65536, 128, 128, True, False, True): (1, 512, 3, 4), + (6144, 6144, 65536, 256, 256, False, True, True): (1, 256, 1, 32), + (6144, 6144, 65536, 256, 256, True, False, True): (4, 256, 1, 32), + (6144, 6144, 131072, 32, 32, False, True, True): (1, 2048, 1, 8), + (6144, 6144, 131072, 32, 32, True, False, True): (1, 1024, 1, 4), + (6144, 6144, 131072, 64, 64, False, True, True): (1, 2048, 1, 8), + (6144, 6144, 131072, 64, 64, True, False, True): (4, 512, 3, 4), + (6144, 6144, 131072, 128, 128, False, True, True): (1, 1024, 1, 8), + (6144, 6144, 131072, 128, 128, True, False, True): (1, 1024, 3, 4), + (6144, 6144, 131072, 256, 256, False, True, True): (1, 512, 1, 32), + (6144, 6144, 131072, 256, 256, True, False, True): (4, 512, 1, 32), + (8192, 8192, 256, 32, 32, False, True, True): (1, 4, 1, 8), + (8192, 8192, 256, 32, 32, True, False, True): (3, 2, 3, 4), + (8192, 8192, 256, 64, 64, False, True, True): (1, 4, 1, 4), + (8192, 8192, 256, 64, 64, True, False, True): (1, 4, 1, 4), + (8192, 8192, 256, 128, 128, False, True, True): (1, 2, 1, 8), + (8192, 8192, 256, 128, 128, True, False, True): (2, 2, 3, 4), + (8192, 8192, 256, 256, 256, False, True, True): (1, 1, 1, 32), + (8192, 8192, 256, 256, 256, True, False, True): (1, 1, 1, 32), + (8192, 8192, 512, 32, 32, False, True, True): (4, 8, 1, 8), + (8192, 8192, 512, 32, 32, True, False, True): (2, 4, 4, 2), + (8192, 8192, 512, 64, 64, False, True, True): (4, 4, 1, 4), + (8192, 8192, 512, 64, 64, True, False, True): (3, 2, 3, 4), + (8192, 8192, 512, 128, 128, False, True, True): (1, 4, 1, 8), + (8192, 8192, 512, 128, 128, True, False, True): (1, 4, 3, 4), + (8192, 8192, 512, 256, 256, False, True, True): (1, 2, 1, 32), + (8192, 8192, 512, 256, 256, True, False, True): (1, 2, 1, 32), + (8192, 8192, 1024, 32, 32, False, True, True): (4, 16, 1, 8), + (8192, 8192, 1024, 32, 32, True, False, True): (1, 8, 3, 2), + (8192, 8192, 1024, 64, 64, False, True, True): (4, 8, 1, 4), + (8192, 8192, 1024, 64, 64, True, False, True): (4, 4, 3, 4), + (8192, 8192, 1024, 128, 128, False, True, True): (1, 8, 1, 8), + (8192, 8192, 1024, 128, 128, True, False, True): (1, 8, 3, 4), + (8192, 8192, 1024, 256, 256, False, True, True): (1, 4, 1, 32), + (8192, 8192, 1024, 256, 256, True, False, True): (4, 4, 1, 32), + (8192, 8192, 2048, 32, 32, False, True, True): (4, 32, 1, 8), + (8192, 8192, 2048, 32, 32, True, False, True): (1, 16, 3, 2), + (8192, 8192, 2048, 64, 64, False, True, True): (4, 32, 1, 8), + (8192, 8192, 2048, 64, 64, True, False, True): (4, 8, 3, 4), + (8192, 8192, 2048, 128, 128, False, True, True): (4, 16, 1, 8), + (8192, 8192, 2048, 128, 128, True, False, True): (4, 16, 3, 4), + (8192, 8192, 2048, 256, 256, False, True, True): (1, 8, 1, 32), + (8192, 8192, 2048, 256, 256, True, False, True): (4, 8, 1, 32), + (8192, 8192, 4096, 32, 32, False, True, True): (4, 64, 1, 8), + (8192, 8192, 4096, 32, 32, True, False, True): (2, 32, 3, 2), + (8192, 8192, 4096, 64, 64, False, True, True): (4, 64, 1, 8), + (8192, 8192, 4096, 64, 64, True, False, True): (4, 16, 3, 4), + (8192, 8192, 4096, 128, 128, False, True, True): (4, 32, 1, 8), + (8192, 8192, 4096, 128, 128, True, False, True): (4, 32, 3, 4), + (8192, 8192, 4096, 256, 256, False, True, True): (1, 16, 1, 32), + (8192, 8192, 4096, 256, 256, True, False, True): (2, 16, 1, 32), + (8192, 8192, 8192, 32, 32, False, True, True): (4, 128, 1, 8), + (8192, 8192, 8192, 32, 32, True, False, True): (1, 64, 3, 2), + (8192, 8192, 8192, 64, 64, False, True, True): (4, 64, 1, 4), + (8192, 8192, 8192, 64, 64, True, False, True): (4, 32, 3, 4), + (8192, 8192, 8192, 128, 128, False, True, True): (4, 64, 1, 16), + (8192, 8192, 8192, 128, 128, True, False, True): (4, 64, 3, 4), + (8192, 8192, 8192, 256, 256, False, True, True): (1, 32, 1, 32), + (8192, 8192, 8192, 256, 256, True, False, True): (4, 32, 1, 32), + (8192, 8192, 16384, 32, 32, False, True, True): (4, 256, 1, 8), + (8192, 8192, 16384, 32, 32, True, False, True): (4, 128, 3, 2), + (8192, 8192, 16384, 64, 64, False, True, True): (4, 128, 1, 4), + (8192, 8192, 16384, 64, 64, True, False, True): (4, 64, 3, 4), + (8192, 8192, 16384, 128, 128, False, True, True): (4, 128, 1, 16), + (8192, 8192, 16384, 128, 128, True, False, True): (4, 128, 3, 4), + (8192, 8192, 16384, 256, 256, False, True, True): (1, 64, 1, 32), + (8192, 8192, 16384, 256, 256, True, False, True): (4, 64, 1, 32), + (8192, 8192, 32768, 32, 32, False, True, True): (4, 512, 1, 8), + (8192, 8192, 32768, 32, 32, True, False, True): (2, 256, 3, 2), + (8192, 8192, 32768, 64, 64, False, True, True): (4, 256, 1, 4), + (8192, 8192, 32768, 64, 64, True, False, True): (4, 128, 3, 4), + (8192, 8192, 32768, 128, 128, False, True, True): (4, 256, 1, 16), + (8192, 8192, 32768, 128, 128, True, False, True): (4, 256, 3, 4), + (8192, 8192, 32768, 256, 256, False, True, True): (1, 128, 1, 32), + (8192, 8192, 32768, 256, 256, True, False, True): (4, 128, 1, 32), + (8192, 8192, 65536, 32, 32, False, True, True): (4, 1024, 1, 8), + (8192, 8192, 65536, 32, 32, True, False, True): (4, 512, 3, 2), + (8192, 8192, 65536, 64, 64, False, True, True): (4, 512, 1, 4), + (8192, 8192, 65536, 64, 64, True, False, True): (4, 256, 3, 4), + (8192, 8192, 65536, 128, 128, False, True, True): (4, 512, 1, 16), + (8192, 8192, 65536, 128, 128, True, False, True): (4, 512, 3, 4), + (8192, 8192, 65536, 256, 256, False, True, True): (1, 256, 1, 32), + (8192, 8192, 65536, 256, 256, True, False, True): (4, 256, 1, 32), + (8192, 8192, 65792, 32, 32, False, True, True): (4, 1028, 1, 8), + (8192, 8192, 65792, 32, 32, True, False, True): (1, 514, 3, 2), + (8192, 8192, 65792, 64, 64, False, True, True): (4, 1028, 1, 8), + (8192, 8192, 65792, 64, 64, True, False, True): (2, 257, 3, 4), + (8192, 8192, 65792, 128, 128, False, True, True): (4, 514, 1, 16), + (8192, 8192, 65792, 128, 128, True, False, True): (2, 514, 3, 4), + (8192, 8192, 65792, 256, 256, False, True, True): (1, 257, 1, 32), + (8192, 8192, 65792, 256, 256, True, False, True): (1, 257, 1, 32), + (8192, 8192, 131072, 32, 32, False, True, True): (4, 2048, 1, 8), + (8192, 8192, 131072, 32, 32, True, False, True): (4, 1024, 3, 2), + (8192, 8192, 131072, 64, 64, False, True, True): (4, 1024, 1, 4), + (8192, 8192, 131072, 64, 64, True, False, True): (4, 512, 3, 4), + (8192, 8192, 131072, 128, 128, False, True, True): (4, 1024, 1, 16), + (8192, 8192, 131072, 128, 128, True, False, True): (4, 1024, 3, 4), + (8192, 8192, 131072, 256, 256, False, True, True): (1, 512, 1, 32), + (8192, 8192, 131072, 256, 256, True, False, True): (4, 512, 1, 32), + (16384, 16384, 256, 32, 32, False, True, True): (4, 4, 1, 8), + (16384, 16384, 256, 32, 32, True, False, True): (2, 2, 4, 2), + (16384, 16384, 256, 64, 64, False, True, True): (2, 2, 1, 4), + (16384, 16384, 256, 64, 64, True, False, True): (5, 1, 3, 4), + (16384, 16384, 256, 128, 128, False, True, True): (6, 2, 1, 8), + (16384, 16384, 256, 128, 128, True, False, True): (6, 2, 3, 4), + (16384, 16384, 256, 256, 256, False, True, True): (1, 1, 1, 32), + (16384, 16384, 256, 256, 256, True, False, True): (1, 1, 1, 32), + (16384, 16384, 512, 32, 32, False, True, True): (4, 8, 1, 8), + (16384, 16384, 512, 32, 32, True, False, True): (1, 4, 4, 2), + (16384, 16384, 512, 64, 64, False, True, True): (4, 4, 1, 4), + (16384, 16384, 512, 64, 64, True, False, True): (2, 2, 3, 4), + (16384, 16384, 512, 128, 128, False, True, True): (4, 4, 1, 8), + (16384, 16384, 512, 128, 128, True, False, True): (4, 4, 3, 4), + (16384, 16384, 512, 256, 256, False, True, True): (1, 2, 1, 32), + (16384, 16384, 512, 256, 256, True, False, True): (2, 2, 1, 32), + (16384, 16384, 1024, 32, 32, False, True, True): (4, 16, 1, 8), + (16384, 16384, 1024, 32, 32, True, False, True): (1, 8, 3, 2), + (16384, 16384, 1024, 64, 64, False, True, True): (4, 8, 1, 4), + (16384, 16384, 1024, 64, 64, True, False, True): (4, 4, 3, 4), + (16384, 16384, 1024, 128, 128, False, True, True): (4, 4, 1, 8), + (16384, 16384, 1024, 128, 128, True, False, True): (4, 8, 3, 4), + (16384, 16384, 1024, 256, 256, False, True, True): (1, 4, 1, 32), + (16384, 16384, 1024, 256, 256, True, False, True): (4, 4, 1, 32), + (16384, 16384, 2048, 32, 32, False, True, True): (4, 32, 1, 8), + (16384, 16384, 2048, 32, 32, True, False, True): (2, 16, 3, 2), + (16384, 16384, 2048, 64, 64, False, True, True): (4, 16, 1, 4), + (16384, 16384, 2048, 64, 64, True, False, True): (4, 8, 3, 4), + (16384, 16384, 2048, 128, 128, False, True, True): (4, 16, 1, 8), + (16384, 16384, 2048, 128, 128, True, False, True): (4, 16, 3, 4), + (16384, 16384, 2048, 256, 256, False, True, True): (1, 8, 1, 32), + (16384, 16384, 2048, 256, 256, True, False, True): (4, 8, 1, 32), + (16384, 16384, 4096, 32, 32, False, True, True): (4, 64, 1, 8), + (16384, 16384, 4096, 32, 32, True, False, True): (2, 32, 3, 2), + (16384, 16384, 4096, 64, 64, False, True, True): (2, 32, 1, 4), + (16384, 16384, 4096, 64, 64, True, False, True): (4, 16, 3, 4), + (16384, 16384, 4096, 128, 128, False, True, True): (4, 32, 1, 8), + (16384, 16384, 4096, 128, 128, True, False, True): (4, 32, 3, 4), + (16384, 16384, 4096, 256, 256, False, True, True): (1, 16, 1, 32), + (16384, 16384, 4096, 256, 256, True, False, True): (4, 16, 1, 32), + (16384, 16384, 8192, 32, 32, False, True, True): (4, 128, 1, 8), + (16384, 16384, 8192, 32, 32, True, False, True): (2, 64, 3, 2), + (16384, 16384, 8192, 64, 64, False, True, True): (4, 64, 1, 4), + (16384, 16384, 8192, 64, 64, True, False, True): (4, 32, 3, 4), + (16384, 16384, 8192, 128, 128, False, True, True): (4, 64, 1, 16), + (16384, 16384, 8192, 128, 128, True, False, True): (4, 64, 3, 4), + (16384, 16384, 8192, 256, 256, False, True, True): (1, 32, 1, 32), + (16384, 16384, 8192, 256, 256, True, False, True): (4, 32, 1, 32), + (16384, 16384, 16384, 32, 32, False, True, True): (4, 256, 1, 8), + (16384, 16384, 16384, 32, 32, True, False, True): (2, 128, 3, 2), + (16384, 16384, 16384, 64, 64, False, True, True): (4, 128, 1, 4), + (16384, 16384, 16384, 64, 64, True, False, True): (4, 64, 3, 4), + (16384, 16384, 16384, 128, 128, False, True, True): (1, 64, 1, 8), + (16384, 16384, 16384, 128, 128, True, False, True): (4, 128, 3, 4), + (16384, 16384, 16384, 256, 256, False, True, True): (1, 64, 1, 32), + (16384, 16384, 16384, 256, 256, True, False, True): (4, 64, 1, 32), + (16384, 16384, 32768, 32, 32, False, True, True): (4, 512, 1, 8), + (16384, 16384, 32768, 32, 32, True, False, True): (1, 256, 3, 2), + (16384, 16384, 32768, 64, 64, False, True, True): (4, 256, 1, 4), + (16384, 16384, 32768, 64, 64, True, False, True): (4, 128, 3, 4), + (16384, 16384, 32768, 128, 128, False, True, True): (4, 256, 1, 16), + (16384, 16384, 32768, 128, 128, True, False, True): (4, 256, 3, 4), + (16384, 16384, 32768, 256, 256, False, True, True): (1, 128, 1, 32), + (16384, 16384, 32768, 256, 256, True, False, True): (4, 128, 1, 32), + (16384, 16384, 65536, 32, 32, False, True, True): (4, 1024, 1, 8), + (16384, 16384, 65536, 32, 32, True, False, True): (1, 512, 3, 2), + (16384, 16384, 65536, 64, 64, False, True, True): (2, 512, 1, 4), + (16384, 16384, 65536, 64, 64, True, False, True): (4, 256, 3, 4), + (16384, 16384, 65536, 128, 128, False, True, True): (4, 512, 1, 16), + (16384, 16384, 65536, 128, 128, True, False, True): (4, 512, 3, 4), + (16384, 16384, 65536, 256, 256, False, True, True): (1, 256, 1, 32), + (16384, 16384, 65536, 256, 256, True, False, True): (4, 256, 1, 32), + (16384, 16384, 65792, 32, 32, False, True, True): (4, 1028, 1, 8), + (16384, 16384, 65792, 32, 32, True, False, True): (1, 514, 3, 2), + (16384, 16384, 65792, 64, 64, False, True, True): (2, 514, 1, 4), + (16384, 16384, 65792, 64, 64, True, False, True): (2, 257, 3, 4), + (16384, 16384, 65792, 128, 128, False, True, True): (2, 514, 1, 16), + (16384, 16384, 65792, 128, 128, True, False, True): (2, 514, 3, 4), + (16384, 16384, 65792, 256, 256, False, True, True): (1, 257, 1, 32), + (16384, 16384, 65792, 256, 256, True, False, True): (1, 257, 1, 32), + (16384, 16384, 131072, 32, 32, False, True, True): (4, 1024, 1, 8), + (16384, 16384, 131072, 32, 32, True, False, True): (4, 512, 3, 4), + (16384, 16384, 131072, 64, 64, False, True, True): (4, 1024, 1, 4), + (16384, 16384, 131072, 64, 64, True, False, True): (4, 1024, 3, 2), + (16384, 16384, 131072, 128, 128, False, True, True): (2, 1024, 3, 8), + (16384, 16384, 131072, 128, 128, True, False, True): (4, 1024, 3, 4), + (16384, 16384, 131072, 256, 256, False, True, True): (4, 512, 1, 32), + (16384, 16384, 131072, 256, 256, True, False, True): (4, 512, 1, 32), + (32768, 32768, 256, 32, 32, False, True, True): (4, 4, 1, 8), + (32768, 32768, 256, 32, 32, True, False, True): (1, 2, 4, 2), + (32768, 32768, 256, 64, 64, False, True, True): (2, 2, 1, 4), + (32768, 32768, 256, 64, 64, True, False, True): (2, 1, 3, 4), + (32768, 32768, 256, 128, 128, False, True, True): (4, 2, 1, 8), + (32768, 32768, 256, 128, 128, True, False, True): (4, 2, 3, 4), + (32768, 32768, 256, 256, 256, False, True, True): (1, 1, 1, 32), + (32768, 32768, 256, 256, 256, True, False, True): (1, 1, 1, 32), + (32768, 32768, 512, 32, 32, False, True, True): (4, 8, 1, 8), + (32768, 32768, 512, 32, 32, True, False, True): (1, 4, 3, 2), + (32768, 32768, 512, 64, 64, False, True, True): (4, 4, 1, 4), + (32768, 32768, 512, 64, 64, True, False, True): (4, 2, 3, 4), + (32768, 32768, 512, 128, 128, False, True, True): (1, 2, 1, 8), + (32768, 32768, 512, 128, 128, True, False, True): (4, 4, 3, 4), + (32768, 32768, 512, 256, 256, False, True, True): (1, 2, 1, 32), + (32768, 32768, 512, 256, 256, True, False, True): (2, 2, 1, 32), + (32768, 32768, 1024, 32, 32, False, True, True): (4, 16, 1, 8), + (32768, 32768, 1024, 32, 32, True, False, True): (1, 8, 4, 2), + (32768, 32768, 1024, 64, 64, False, True, True): (4, 8, 1, 4), + (32768, 32768, 1024, 64, 64, True, False, True): (4, 4, 3, 4), + (32768, 32768, 1024, 128, 128, False, True, True): (1, 4, 1, 8), + (32768, 32768, 1024, 128, 128, True, False, True): (4, 8, 3, 4), + (32768, 32768, 1024, 256, 256, False, True, True): (1, 4, 1, 32), + (32768, 32768, 1024, 256, 256, True, False, True): (1, 4, 1, 32), + (32768, 32768, 2048, 32, 32, False, True, True): (2, 32, 1, 8), + (32768, 32768, 2048, 32, 32, True, False, True): (1, 16, 4, 2), + (32768, 32768, 2048, 64, 64, False, True, True): (2, 16, 1, 4), + (32768, 32768, 2048, 64, 64, True, False, True): (4, 8, 3, 4), + (32768, 32768, 2048, 128, 128, False, True, True): (1, 8, 1, 8), + (32768, 32768, 2048, 128, 128, True, False, True): (4, 16, 3, 4), + (32768, 32768, 2048, 256, 256, False, True, True): (1, 8, 1, 32), + (32768, 32768, 2048, 256, 256, True, False, True): (4, 8, 1, 32), + (32768, 32768, 4096, 32, 32, False, True, True): (2, 64, 1, 8), + (32768, 32768, 4096, 32, 32, True, False, True): (2, 32, 3, 2), + (32768, 32768, 4096, 64, 64, False, True, True): (2, 32, 1, 4), + (32768, 32768, 4096, 64, 64, True, False, True): (2, 16, 3, 4), + (32768, 32768, 4096, 128, 128, False, True, True): (1, 16, 1, 8), + (32768, 32768, 4096, 128, 128, True, False, True): (2, 32, 3, 4), + (32768, 32768, 4096, 256, 256, False, True, True): (1, 16, 1, 32), + (32768, 32768, 4096, 256, 256, True, False, True): (4, 16, 1, 32), + (32768, 32768, 8192, 32, 32, False, True, True): (2, 128, 1, 8), + (32768, 32768, 8192, 32, 32, True, False, True): (2, 64, 3, 2), + (32768, 32768, 8192, 64, 64, False, True, True): (2, 64, 1, 4), + (32768, 32768, 8192, 64, 64, True, False, True): (2, 32, 3, 4), + (32768, 32768, 8192, 128, 128, False, True, True): (1, 32, 1, 8), + (32768, 32768, 8192, 128, 128, True, False, True): (4, 64, 3, 4), + (32768, 32768, 8192, 256, 256, False, True, True): (1, 32, 1, 32), + (32768, 32768, 8192, 256, 256, True, False, True): (4, 32, 1, 32), + (32768, 32768, 16384, 32, 32, False, True, True): (2, 256, 1, 8), + (32768, 32768, 16384, 32, 32, True, False, True): (2, 128, 4, 2), + (32768, 32768, 16384, 64, 64, False, True, True): (2, 128, 1, 4), + (32768, 32768, 16384, 64, 64, True, False, True): (4, 64, 3, 4), + (32768, 32768, 16384, 128, 128, False, True, True): (1, 64, 1, 8), + (32768, 32768, 16384, 128, 128, True, False, True): (4, 128, 3, 4), + (32768, 32768, 16384, 256, 256, False, True, True): (1, 64, 1, 32), + (32768, 32768, 16384, 256, 256, True, False, True): (2, 64, 1, 32), + (32768, 32768, 32768, 32, 32, False, True, True): (2, 512, 1, 8), + (32768, 32768, 32768, 32, 32, True, False, True): (4, 256, 3, 2), + (32768, 32768, 32768, 64, 64, False, True, True): (1, 256, 1, 4), + (32768, 32768, 32768, 64, 64, True, False, True): (2, 128, 3, 4), + (32768, 32768, 32768, 128, 128, False, True, True): (1, 128, 1, 8), + (32768, 32768, 32768, 128, 128, True, False, True): (2, 256, 3, 4), + (32768, 32768, 32768, 256, 256, False, True, True): (1, 128, 1, 32), + (32768, 32768, 32768, 256, 256, True, False, True): (1, 128, 1, 32), + (32768, 32768, 65536, 32, 32, False, True, True): (2, 512, 1, 8), + (32768, 32768, 65536, 32, 32, True, False, True): (3, 512, 4, 2), + (32768, 32768, 65536, 64, 64, False, True, True): (1, 512, 1, 4), + (32768, 32768, 65536, 64, 64, True, False, True): (2, 512, 3, 2), + (32768, 32768, 65536, 128, 128, False, True, True): (1, 256, 1, 8), + (32768, 32768, 65536, 128, 128, True, False, True): (2, 512, 3, 4), + (32768, 32768, 65536, 256, 256, False, True, True): (1, 256, 1, 32), + (32768, 32768, 65536, 256, 256, True, False, True): (1, 256, 1, 32), + }, + ("_int_bsr_dense_addmm", "NVIDIA A100-SXM4-80GB", (0, torch.int8, 0.56)): { + (192, 192, 256, 64, 64, False, True, True): (3, 4, 3, 32), + (192, 192, 256, 64, 64, True, False, True): (1, 4, 3, 4), + (192, 192, 512, 64, 64, False, True, True): (1, 8, 1, 16), + (192, 192, 512, 64, 64, True, False, True): (1, 8, 5, 4), + (192, 192, 1024, 64, 64, False, True, True): (4, 16, 1, 16), + (192, 192, 1024, 64, 64, True, False, True): (3, 16, 3, 4), + (192, 192, 2048, 64, 64, False, True, True): (5, 32, 1, 8), + (192, 192, 2048, 64, 64, True, False, True): (2, 32, 4, 4), + (192, 192, 4096, 64, 64, False, True, True): (4, 64, 1, 16), + (192, 192, 4096, 64, 64, True, False, True): (1, 32, 4, 4), + (192, 192, 8192, 64, 64, False, True, True): (2, 128, 1, 8), + (192, 192, 8192, 64, 64, True, False, True): (3, 64, 1, 4), + (192, 192, 16384, 64, 64, False, True, True): (2, 256, 1, 8), + (192, 192, 16384, 64, 64, True, False, True): (1, 128, 3, 2), + (192, 192, 32768, 64, 64, False, True, True): (2, 512, 1, 8), + (192, 192, 32768, 64, 64, True, False, True): (3, 128, 1, 4), + (192, 192, 65536, 64, 64, False, True, True): (3, 1024, 1, 8), + (192, 192, 65536, 64, 64, True, False, True): (1, 512, 3, 4), + (192, 192, 131072, 64, 64, False, True, True): (1, 2048, 1, 8), + (192, 192, 131072, 64, 64, True, False, True): (1, 512, 1, 4), + (384, 384, 256, 128, 128, False, True, True): (4, 2, 1, 16), + (384, 384, 256, 128, 128, True, False, True): (1, 2, 3, 4), + (384, 384, 512, 128, 128, False, True, True): (2, 4, 1, 16), + (384, 384, 512, 128, 128, True, False, True): (2, 4, 3, 4), + (384, 384, 1024, 128, 128, False, True, True): (3, 8, 1, 32), + (384, 384, 1024, 128, 128, True, False, True): (3, 8, 3, 4), + (384, 384, 2048, 128, 128, False, True, True): (3, 16, 1, 32), + (384, 384, 2048, 128, 128, True, False, True): (2, 16, 3, 4), + (384, 384, 4096, 128, 128, False, True, True): (3, 32, 1, 32), + (384, 384, 4096, 128, 128, True, False, True): (3, 32, 3, 4), + (384, 384, 8192, 128, 128, False, True, True): (2, 64, 1, 32), + (384, 384, 8192, 128, 128, True, False, True): (4, 64, 1, 4), + (384, 384, 16384, 128, 128, False, True, True): (2, 128, 1, 32), + (384, 384, 16384, 128, 128, True, False, True): (2, 128, 1, 4), + (384, 384, 32768, 128, 128, False, True, True): (3, 256, 1, 16), + (384, 384, 32768, 128, 128, True, False, True): (1, 256, 1, 4), + (384, 384, 65536, 128, 128, False, True, True): (4, 512, 1, 16), + (384, 384, 65536, 128, 128, True, False, True): (1, 512, 1, 4), + (384, 384, 131072, 128, 128, False, True, True): (4, 1024, 1, 16), + (384, 384, 131072, 128, 128, True, False, True): (1, 1024, 1, 4), + (768, 768, 256, 256, 256, False, True, True): (1, 1, 1, 32), + (768, 768, 256, 256, 256, True, False, True): (3, 1, 1, 32), + (768, 768, 512, 256, 256, False, True, True): (1, 2, 1, 32), + (768, 768, 512, 256, 256, True, False, True): (1, 2, 1, 32), + (768, 768, 1024, 256, 256, False, True, True): (1, 4, 1, 32), + (768, 768, 1024, 256, 256, True, False, True): (2, 4, 1, 32), + (768, 768, 2048, 256, 256, False, True, True): (1, 8, 1, 32), + (768, 768, 2048, 256, 256, True, False, True): (2, 8, 1, 32), + (768, 768, 4096, 256, 256, False, True, True): (1, 16, 1, 32), + (768, 768, 4096, 256, 256, True, False, True): (1, 16, 1, 32), + (768, 768, 8192, 256, 256, False, True, True): (1, 32, 1, 32), + (768, 768, 8192, 256, 256, True, False, True): (2, 32, 1, 32), + (768, 768, 16384, 256, 256, False, True, True): (1, 64, 1, 32), + (768, 768, 16384, 256, 256, True, False, True): (7, 64, 1, 32), + (768, 768, 32768, 256, 256, False, True, True): (1, 128, 1, 32), + (768, 768, 32768, 256, 256, True, False, True): (1, 128, 1, 32), + (768, 768, 65536, 256, 256, False, True, True): (1, 256, 1, 32), + (768, 768, 65536, 256, 256, True, False, True): (1, 256, 1, 32), + (768, 768, 131072, 256, 256, False, True, True): (1, 512, 1, 32), + (768, 768, 131072, 256, 256, True, False, True): (1, 512, 1, 32), + }, + ("_int_bsr_dense_addmm", "NVIDIA A100-SXM4-80GB", (0, torch.int8, 1.0)): { + (256, 256, 256, 256, 256, False, True, True): (2, 1, 1, 4), + (256, 256, 256, 256, 256, True, False, True): (2, 1, 2, 1), + (256, 256, 512, 256, 256, False, True, True): (2, 1, 1, 2), + (256, 256, 512, 256, 256, True, False, True): (2, 2, 2, 8), + (256, 256, 1024, 256, 256, False, True, True): (1, 4, 1, 4), + (256, 256, 1024, 256, 256, True, False, True): (1, 2, 2, 4), + (256, 256, 2048, 256, 256, False, True, True): (1, 4, 1, 2), + (256, 256, 2048, 256, 256, True, False, True): (1, 8, 1, 2), + (256, 256, 4096, 256, 256, False, True, True): (1, 16, 1, 4), + (256, 256, 4096, 256, 256, True, False, True): (1, 16, 1, 2), + (256, 256, 8192, 256, 256, False, True, True): (1, 16, 3, 4), + (256, 256, 8192, 256, 256, True, False, True): (1, 8, 1, 4), + (256, 256, 16384, 256, 256, False, True, True): (2, 16, 1, 8), + (256, 256, 16384, 256, 256, True, False, True): (1, 32, 1, 2), + (256, 256, 32768, 256, 256, False, True, True): (1, 128, 1, 8), + (256, 256, 32768, 256, 256, True, False, True): (1, 128, 1, 4), + (256, 256, 65536, 256, 256, False, True, True): (1, 4, 1, 1), + (256, 256, 65536, 256, 256, True, False, True): (1, 128, 1, 4), + (256, 256, 65792, 256, 256, False, True, True): (1, 128, 2, 16), + (256, 256, 65792, 256, 256, True, False, True): (1, 16, 3, 4), + (256, 256, 131072, 256, 256, False, True, True): (1, 512, 1, 4), + (256, 256, 131072, 256, 256, True, False, True): (1, 512, 1, 2), + }, + ("bsr_dense_addmm", "NVIDIA A100-SXM4-80GB", (0, torch.bfloat16, 0.5)): { + (16, 16, 16, 16, 16, False, False, False): (2, 1, 1, 2), + (16, 16, 16, 16, 16, False, False, True): (1, 1, 1, 4), + (16, 16, 16, 16, 16, False, True, False): (1, 1, 3, 16), + (16, 16, 16, 16, 16, False, True, True): (1, 1, 1, 8), + (16, 16, 16, 16, 16, True, False, False): (2, 1, 1, 8), + (16, 16, 16, 16, 16, True, False, True): (1, 1, 1, 8), + (16, 16, 32, 16, 16, False, False, False): (1, 2, 1, 8), + (16, 16, 32, 16, 16, False, False, True): (1, 2, 2, 4), + (16, 16, 32, 16, 16, False, True, False): (1, 1, 2, 4), + (16, 16, 32, 16, 16, False, True, True): (1, 1, 2, 4), + (16, 16, 32, 16, 16, True, False, False): (1, 1, 2, 4), + (16, 16, 32, 16, 16, True, False, True): (2, 2, 1, 2), + (16, 16, 64, 16, 16, False, False, False): (1, 4, 2, 4), + (16, 16, 64, 16, 16, False, False, True): (1, 2, 1, 2), + (16, 16, 64, 16, 16, False, True, False): (2, 1, 1, 2), + (16, 16, 64, 16, 16, False, True, True): (1, 4, 1, 8), + (16, 16, 64, 16, 16, True, False, False): (1, 4, 1, 1), + (16, 16, 64, 16, 16, True, False, True): (1, 4, 2, 4), + (16, 32, 16, 16, 16, False, False, False): (1, 1, 2, 2), + (16, 32, 16, 16, 16, False, False, True): (1, 1, 1, 4), + (16, 32, 16, 16, 16, False, True, False): (1, 1, 1, 2), + (16, 32, 16, 16, 16, False, True, True): (1, 1, 1, 1), + (16, 32, 16, 16, 16, True, False, False): (1, 1, 1, 2), + (16, 32, 16, 16, 16, True, False, True): (2, 1, 1, 2), + (16, 32, 16, 16, 32, False, False, False): (1, 1, 1, 4), + (16, 32, 16, 16, 32, False, False, True): (1, 1, 1, 8), + (16, 32, 16, 16, 32, False, True, False): (1, 1, 1, 8), + (16, 32, 16, 16, 32, False, True, True): (1, 1, 2, 4), + (16, 32, 16, 16, 32, True, False, False): (1, 1, 1, 2), + (16, 32, 16, 16, 32, True, False, True): (1, 1, 1, 1), + (16, 32, 32, 16, 16, False, False, False): (2, 2, 1, 4), + (16, 32, 32, 16, 16, False, False, True): (2, 2, 1, 2), + (16, 32, 32, 16, 16, False, True, False): (1, 1, 2, 8), + (16, 32, 32, 16, 16, False, True, True): (1, 2, 1, 1), + (16, 32, 32, 16, 16, True, False, False): (1, 1, 1, 8), + (16, 32, 32, 16, 16, True, False, True): (1, 2, 1, 4), + (16, 32, 32, 16, 32, False, False, False): (1, 1, 2, 8), + (16, 32, 32, 16, 32, False, False, True): (2, 1, 1, 8), + (16, 32, 32, 16, 32, False, True, False): (1, 1, 1, 4), + (16, 32, 32, 16, 32, False, True, True): (1, 1, 1, 4), + (16, 32, 32, 16, 32, True, False, False): (1, 2, 1, 8), + (16, 32, 32, 16, 32, True, False, True): (1, 1, 1, 4), + (16, 32, 64, 16, 16, False, False, False): (1, 4, 3, 8), + (16, 32, 64, 16, 16, False, False, True): (1, 4, 1, 4), + (16, 32, 64, 16, 16, False, True, False): (1, 4, 1, 4), + (16, 32, 64, 16, 16, False, True, True): (2, 4, 1, 4), + (16, 32, 64, 16, 16, True, False, False): (1, 2, 1, 4), + (16, 32, 64, 16, 16, True, False, True): (1, 2, 1, 4), + (16, 32, 64, 16, 32, False, False, False): (1, 4, 1, 8), + (16, 32, 64, 16, 32, False, False, True): (1, 4, 1, 4), + (16, 32, 64, 16, 32, False, True, False): (1, 4, 1, 2), + (16, 32, 64, 16, 32, False, True, True): (1, 2, 1, 4), + (16, 32, 64, 16, 32, True, False, False): (1, 2, 1, 4), + (16, 32, 64, 16, 32, True, False, True): (1, 2, 1, 2), + (16, 64, 16, 16, 32, False, False, False): (1, 1, 1, 2), + (16, 64, 16, 16, 32, False, False, True): (1, 1, 2, 2), + (16, 64, 16, 16, 32, False, True, False): (1, 1, 2, 8), + (16, 64, 16, 16, 32, False, True, True): (1, 1, 1, 4), + (16, 64, 16, 16, 32, True, False, False): (1, 1, 1, 8), + (16, 64, 16, 16, 32, True, False, True): (1, 1, 1, 4), + (16, 64, 32, 16, 32, False, False, False): (1, 2, 1, 2), + (16, 64, 32, 16, 32, False, False, True): (1, 2, 1, 4), + (16, 64, 32, 16, 32, False, True, False): (1, 2, 1, 4), + (16, 64, 32, 16, 32, False, True, True): (2, 2, 1, 4), + (16, 64, 32, 16, 32, True, False, False): (1, 2, 1, 4), + (16, 64, 32, 16, 32, True, False, True): (1, 2, 1, 8), + (16, 64, 64, 16, 32, False, False, False): (1, 2, 1, 4), + (16, 64, 64, 16, 32, False, False, True): (1, 4, 2, 2), + (16, 64, 64, 16, 32, False, True, False): (1, 1, 1, 4), + (16, 64, 64, 16, 32, False, True, True): (1, 4, 1, 2), + (16, 64, 64, 16, 32, True, False, False): (1, 2, 1, 4), + (16, 64, 64, 16, 32, True, False, True): (1, 4, 1, 4), + (32, 16, 16, 16, 16, False, False, False): (1, 1, 1, 8), + (32, 16, 16, 16, 16, False, False, True): (1, 1, 2, 4), + (32, 16, 16, 16, 16, False, True, False): (1, 1, 1, 4), + (32, 16, 16, 16, 16, False, True, True): (1, 1, 2, 4), + (32, 16, 16, 16, 16, True, False, False): (1, 1, 1, 2), + (32, 16, 16, 16, 16, True, False, True): (1, 1, 1, 4), + (32, 16, 32, 16, 16, False, False, False): (1, 1, 1, 4), + (32, 16, 32, 16, 16, False, False, True): (2, 2, 1, 4), + (32, 16, 32, 16, 16, False, True, False): (1, 2, 2, 2), + (32, 16, 32, 16, 16, False, True, True): (2, 2, 1, 4), + (32, 16, 32, 16, 16, True, False, False): (1, 2, 2, 8), + (32, 16, 32, 16, 16, True, False, True): (1, 2, 1, 2), + (32, 16, 64, 16, 16, False, False, False): (1, 4, 1, 4), + (32, 16, 64, 16, 16, False, False, True): (1, 4, 2, 4), + (32, 16, 64, 16, 16, False, True, False): (1, 2, 2, 2), + (32, 16, 64, 16, 16, False, True, True): (3, 4, 1, 4), + (32, 16, 64, 16, 16, True, False, False): (1, 2, 1, 2), + (32, 16, 64, 16, 16, True, False, True): (1, 2, 1, 4), + (32, 32, 16, 16, 16, False, False, False): (1, 1, 3, 4), + (32, 32, 16, 16, 16, False, False, True): (1, 1, 1, 4), + (32, 32, 16, 16, 16, False, True, False): (1, 1, 1, 2), + (32, 32, 16, 16, 16, False, True, True): (1, 1, 1, 4), + (32, 32, 16, 16, 16, True, False, False): (1, 1, 1, 4), + (32, 32, 16, 16, 16, True, False, True): (1, 1, 2, 2), + (32, 32, 16, 16, 32, False, False, False): (2, 1, 1, 4), + (32, 32, 16, 16, 32, False, False, True): (1, 1, 1, 4), + (32, 32, 16, 16, 32, False, True, False): (1, 1, 1, 4), + (32, 32, 16, 16, 32, False, True, True): (3, 1, 2, 4), + (32, 32, 16, 16, 32, True, False, False): (1, 1, 1, 4), + (32, 32, 16, 16, 32, True, False, True): (1, 1, 1, 4), + (32, 32, 16, 32, 32, False, False, False): (1, 1, 1, 8), + (32, 32, 16, 32, 32, False, False, True): (1, 1, 1, 4), + (32, 32, 16, 32, 32, False, True, False): (1, 1, 2, 1), + (32, 32, 16, 32, 32, False, True, True): (2, 1, 2, 2), + (32, 32, 16, 32, 32, True, False, False): (1, 1, 1, 8), + (32, 32, 16, 32, 32, True, False, True): (2, 1, 3, 4), + (32, 32, 32, 16, 16, False, False, False): (1, 2, 1, 4), + (32, 32, 32, 16, 16, False, False, True): (2, 2, 1, 4), + (32, 32, 32, 16, 16, False, True, False): (1, 1, 1, 8), + (32, 32, 32, 16, 16, False, True, True): (2, 2, 1, 4), + (32, 32, 32, 16, 16, True, False, False): (1, 1, 1, 4), + (32, 32, 32, 16, 16, True, False, True): (2, 2, 2, 4), + (32, 32, 32, 16, 32, False, False, False): (2, 2, 1, 8), + (32, 32, 32, 16, 32, False, False, True): (1, 2, 1, 2), + (32, 32, 32, 16, 32, False, True, False): (1, 2, 1, 4), + (32, 32, 32, 16, 32, False, True, True): (1, 2, 1, 4), + (32, 32, 32, 16, 32, True, False, False): (1, 2, 1, 4), + (32, 32, 32, 16, 32, True, False, True): (1, 2, 1, 2), + (32, 32, 32, 32, 32, False, False, False): (1, 1, 3, 8), + (32, 32, 32, 32, 32, False, False, True): (1, 1, 1, 8), + (32, 32, 32, 32, 32, False, True, False): (2, 1, 3, 4), + (32, 32, 32, 32, 32, False, True, True): (2, 1, 1, 2), + (32, 32, 32, 32, 32, True, False, False): (1, 1, 1, 2), + (32, 32, 32, 32, 32, True, False, True): (4, 1, 1, 1), + (32, 32, 64, 16, 16, False, False, False): (1, 4, 1, 4), + (32, 32, 64, 16, 16, False, False, True): (1, 4, 1, 4), + (32, 32, 64, 16, 16, False, True, False): (1, 2, 1, 8), + (32, 32, 64, 16, 16, False, True, True): (1, 4, 1, 2), + (32, 32, 64, 16, 16, True, False, False): (2, 4, 1, 2), + (32, 32, 64, 16, 16, True, False, True): (1, 4, 1, 2), + (32, 32, 64, 16, 32, False, False, False): (1, 2, 1, 8), + (32, 32, 64, 16, 32, False, False, True): (1, 4, 2, 2), + (32, 32, 64, 16, 32, False, True, False): (1, 2, 1, 4), + (32, 32, 64, 16, 32, False, True, True): (1, 4, 1, 4), + (32, 32, 64, 16, 32, True, False, False): (1, 4, 2, 2), + (32, 32, 64, 16, 32, True, False, True): (3, 4, 2, 2), + (32, 32, 64, 32, 32, False, False, False): (2, 2, 1, 4), + (32, 32, 64, 32, 32, False, False, True): (1, 2, 1, 4), + (32, 32, 64, 32, 32, False, True, False): (1, 1, 1, 8), + (32, 32, 64, 32, 32, False, True, True): (1, 1, 1, 4), + (32, 32, 64, 32, 32, True, False, False): (1, 2, 1, 2), + (32, 32, 64, 32, 32, True, False, True): (3, 2, 1, 8), + (32, 64, 16, 16, 32, False, False, False): (1, 1, 2, 2), + (32, 64, 16, 16, 32, False, False, True): (1, 1, 1, 4), + (32, 64, 16, 16, 32, False, True, False): (1, 1, 2, 4), + (32, 64, 16, 16, 32, False, True, True): (1, 1, 1, 4), + (32, 64, 16, 16, 32, True, False, False): (1, 1, 1, 2), + (32, 64, 16, 16, 32, True, False, True): (2, 1, 2, 2), + (32, 64, 16, 32, 32, False, False, False): (1, 1, 1, 1), + (32, 64, 16, 32, 32, False, False, True): (2, 1, 1, 4), + (32, 64, 16, 32, 32, False, True, False): (1, 1, 1, 1), + (32, 64, 16, 32, 32, False, True, True): (1, 1, 2, 2), + (32, 64, 16, 32, 32, True, False, False): (1, 1, 2, 4), + (32, 64, 16, 32, 32, True, False, True): (1, 1, 1, 4), + (32, 64, 32, 16, 32, False, False, False): (2, 2, 1, 4), + (32, 64, 32, 16, 32, False, False, True): (1, 2, 1, 4), + (32, 64, 32, 16, 32, False, True, False): (1, 1, 1, 4), + (32, 64, 32, 16, 32, False, True, True): (2, 2, 3, 4), + (32, 64, 32, 16, 32, True, False, False): (1, 1, 1, 2), + (32, 64, 32, 16, 32, True, False, True): (1, 2, 1, 2), + (32, 64, 32, 32, 32, False, False, False): (1, 1, 1, 2), + (32, 64, 32, 32, 32, False, False, True): (2, 1, 1, 4), + (32, 64, 32, 32, 32, False, True, False): (1, 1, 1, 8), + (32, 64, 32, 32, 32, False, True, True): (1, 1, 2, 4), + (32, 64, 32, 32, 32, True, False, False): (2, 1, 1, 4), + (32, 64, 32, 32, 32, True, False, True): (1, 1, 2, 4), + (32, 64, 64, 16, 32, False, False, False): (1, 4, 1, 4), + (32, 64, 64, 16, 32, False, False, True): (1, 4, 2, 4), + (32, 64, 64, 16, 32, False, True, False): (1, 4, 2, 2), + (32, 64, 64, 16, 32, False, True, True): (1, 4, 1, 4), + (32, 64, 64, 16, 32, True, False, False): (1, 4, 1, 8), + (32, 64, 64, 16, 32, True, False, True): (1, 4, 2, 1), + (32, 64, 64, 32, 32, False, False, False): (1, 1, 1, 4), + (32, 64, 64, 32, 32, False, False, True): (2, 2, 1, 4), + (32, 64, 64, 32, 32, False, True, False): (1, 1, 1, 4), + (32, 64, 64, 32, 32, False, True, True): (2, 2, 1, 4), + (32, 64, 64, 32, 32, True, False, False): (1, 2, 2, 4), + (32, 64, 64, 32, 32, True, False, True): (2, 2, 3, 4), + (64, 32, 16, 32, 32, False, False, False): (1, 1, 1, 4), + (64, 32, 16, 32, 32, False, False, True): (1, 1, 1, 4), + (64, 32, 16, 32, 32, False, True, False): (1, 1, 1, 8), + (64, 32, 16, 32, 32, False, True, True): (1, 1, 1, 4), + (64, 32, 16, 32, 32, True, False, False): (1, 1, 1, 16), + (64, 32, 16, 32, 32, True, False, True): (2, 1, 1, 4), + (64, 32, 32, 32, 32, False, False, False): (1, 1, 3, 4), + (64, 32, 32, 32, 32, False, False, True): (2, 1, 1, 4), + (64, 32, 32, 32, 32, False, True, False): (1, 1, 2, 4), + (64, 32, 32, 32, 32, False, True, True): (2, 1, 1, 4), + (64, 32, 32, 32, 32, True, False, False): (2, 1, 1, 16), + (64, 32, 32, 32, 32, True, False, True): (2, 1, 1, 4), + (64, 32, 64, 32, 32, False, False, False): (1, 2, 1, 4), + (64, 32, 64, 32, 32, False, False, True): (2, 2, 1, 4), + (64, 32, 64, 32, 32, False, True, False): (1, 1, 1, 4), + (64, 32, 64, 32, 32, False, True, True): (2, 2, 1, 4), + (64, 32, 64, 32, 32, True, False, False): (1, 2, 1, 8), + (64, 32, 64, 32, 32, True, False, True): (2, 2, 3, 4), + (64, 64, 16, 32, 32, False, False, False): (1, 1, 2, 16), + (64, 64, 16, 32, 32, False, False, True): (1, 1, 3, 4), + (64, 64, 16, 32, 32, False, True, False): (1, 1, 1, 2), + (64, 64, 16, 32, 32, False, True, True): (2, 1, 1, 4), + (64, 64, 16, 32, 32, True, False, False): (2, 1, 3, 2), + (64, 64, 16, 32, 32, True, False, True): (1, 1, 2, 4), + (64, 64, 32, 32, 32, False, False, False): (1, 1, 1, 8), + (64, 64, 32, 32, 32, False, False, True): (2, 1, 2, 4), + (64, 64, 32, 32, 32, False, True, False): (2, 1, 1, 4), + (64, 64, 32, 32, 32, False, True, True): (1, 1, 2, 4), + (64, 64, 32, 32, 32, True, False, False): (2, 1, 1, 4), + (64, 64, 32, 32, 32, True, False, True): (1, 1, 2, 4), + (64, 64, 64, 32, 32, False, False, False): (1, 2, 2, 4), + (64, 64, 64, 32, 32, False, False, True): (1, 2, 2, 2), + (64, 64, 64, 32, 32, False, True, False): (1, 2, 1, 2), + (64, 64, 64, 32, 32, False, True, True): (1, 2, 1, 4), + (64, 64, 64, 32, 32, True, False, False): (1, 2, 1, 4), + (64, 64, 64, 32, 32, True, False, True): (1, 2, 1, 4), + (192, 192, 256, 16, 16, False, True, True): (1, 8, 5, 4), + (192, 192, 256, 16, 16, True, False, True): (2, 8, 5, 2), + (192, 192, 256, 32, 32, False, True, True): (1, 8, 6, 4), + (192, 192, 256, 32, 32, True, False, True): (3, 8, 5, 2), + (192, 192, 512, 16, 16, False, True, True): (1, 16, 5, 2), + (192, 192, 512, 16, 16, True, False, True): (1, 8, 4, 2), + (192, 192, 512, 32, 32, False, True, True): (2, 16, 5, 4), + (192, 192, 512, 32, 32, True, False, True): (2, 8, 5, 2), + (192, 192, 1024, 16, 16, False, True, True): (1, 16, 3, 4), + (192, 192, 1024, 16, 16, True, False, True): (1, 16, 6, 2), + (192, 192, 1024, 32, 32, False, True, True): (1, 32, 3, 4), + (192, 192, 1024, 32, 32, True, False, True): (1, 16, 4, 2), + (192, 192, 2048, 16, 16, False, True, True): (1, 32, 1, 4), + (192, 192, 2048, 16, 16, True, False, True): (4, 32, 4, 2), + (192, 192, 2048, 32, 32, False, True, True): (1, 16, 3, 8), + (192, 192, 2048, 32, 32, True, False, True): (2, 32, 4, 2), + (192, 192, 4096, 16, 16, False, True, True): (2, 64, 1, 4), + (192, 192, 4096, 16, 16, True, False, True): (1, 32, 3, 2), + (192, 192, 4096, 32, 32, False, True, True): (1, 64, 1, 8), + (192, 192, 4096, 32, 32, True, False, True): (2, 32, 4, 4), + (192, 192, 8192, 16, 16, False, True, True): (1, 64, 1, 4), + (192, 192, 8192, 16, 16, True, False, True): (2, 32, 3, 1), + (192, 192, 8192, 32, 32, False, True, True): (3, 128, 1, 4), + (192, 192, 8192, 32, 32, True, False, True): (1, 64, 3, 4), + (192, 192, 16384, 16, 16, False, True, True): (1, 128, 1, 4), + (192, 192, 16384, 16, 16, True, False, True): (4, 64, 3, 1), + (192, 192, 16384, 32, 32, False, True, True): (1, 128, 1, 4), + (192, 192, 16384, 32, 32, True, False, True): (1, 64, 3, 4), + (192, 192, 32768, 16, 16, False, True, True): (2, 256, 1, 2), + (192, 192, 32768, 16, 16, True, False, True): (1, 128, 3, 4), + (192, 192, 32768, 32, 32, False, True, True): (2, 256, 1, 4), + (192, 192, 32768, 32, 32, True, False, True): (4, 128, 3, 4), + (192, 192, 65536, 16, 16, False, True, True): (2, 512, 1, 2), + (192, 192, 65536, 16, 16, True, False, True): (2, 256, 3, 2), + (192, 192, 65536, 32, 32, False, True, True): (2, 512, 1, 4), + (192, 192, 65536, 32, 32, True, False, True): (1, 256, 3, 4), + (192, 192, 131072, 16, 16, False, True, True): (4, 1024, 1, 2), + (192, 192, 131072, 16, 16, True, False, True): (3, 512, 3, 2), + (192, 192, 131072, 32, 32, False, True, True): (1, 1024, 1, 2), + (192, 192, 131072, 32, 32, True, False, True): (1, 512, 3, 4), + (256, 256, 256, 16, 16, False, True, True): (4, 8, 5, 1), + (256, 256, 256, 16, 16, True, False, True): (2, 8, 4, 2), + (256, 256, 256, 32, 32, False, True, True): (2, 8, 5, 2), + (256, 256, 256, 32, 32, True, False, True): (1, 8, 5, 4), + (256, 256, 256, 64, 64, False, True, True): (2, 4, 4, 4), + (256, 256, 256, 64, 64, True, False, True): (1, 4, 3, 4), + (256, 256, 256, 128, 128, False, True, True): (4, 2, 2, 8), + (256, 256, 256, 128, 128, True, False, True): (1, 2, 2, 8), + (256, 256, 512, 16, 16, False, True, True): (1, 16, 5, 1), + (256, 256, 512, 16, 16, True, False, True): (3, 16, 3, 2), + (256, 256, 512, 32, 32, False, True, True): (2, 8, 5, 2), + (256, 256, 512, 32, 32, True, False, True): (1, 16, 4, 4), + (256, 256, 512, 64, 64, False, True, True): (1, 8, 4, 4), + (256, 256, 512, 64, 64, True, False, True): (3, 8, 3, 4), + (256, 256, 512, 128, 128, False, True, True): (1, 4, 2, 8), + (256, 256, 512, 128, 128, True, False, True): (1, 4, 2, 8), + (256, 256, 1024, 16, 16, False, True, True): (1, 16, 5, 4), + (256, 256, 1024, 16, 16, True, False, True): (5, 16, 4, 2), + (256, 256, 1024, 32, 32, False, True, True): (1, 32, 5, 2), + (256, 256, 1024, 32, 32, True, False, True): (2, 16, 5, 2), + (256, 256, 1024, 64, 64, False, True, True): (1, 16, 4, 4), + (256, 256, 1024, 64, 64, True, False, True): (1, 16, 4, 4), + (256, 256, 1024, 128, 128, False, True, True): (1, 8, 2, 8), + (256, 256, 1024, 128, 128, True, False, True): (1, 8, 2, 8), + (256, 256, 2048, 16, 16, False, True, True): (1, 16, 4, 4), + (256, 256, 2048, 16, 16, True, False, True): (2, 32, 5, 1), + (256, 256, 2048, 32, 32, False, True, True): (1, 64, 4, 1), + (256, 256, 2048, 32, 32, True, False, True): (2, 32, 4, 2), + (256, 256, 2048, 64, 64, False, True, True): (8, 16, 5, 4), + (256, 256, 2048, 64, 64, True, False, True): (1, 16, 4, 4), + (256, 256, 2048, 128, 128, False, True, True): (2, 16, 2, 8), + (256, 256, 2048, 128, 128, True, False, True): (1, 16, 2, 8), + (256, 256, 4096, 16, 16, False, True, True): (1, 64, 1, 4), + (256, 256, 4096, 16, 16, True, False, True): (1, 16, 3, 2), + (256, 256, 4096, 32, 32, False, True, True): (6, 32, 3, 2), + (256, 256, 4096, 32, 32, True, False, True): (4, 32, 4, 2), + (256, 256, 4096, 64, 64, False, True, True): (6, 64, 3, 4), + (256, 256, 4096, 64, 64, True, False, True): (2, 64, 3, 4), + (256, 256, 4096, 128, 128, False, True, True): (1, 32, 2, 8), + (256, 256, 4096, 128, 128, True, False, True): (1, 32, 2, 8), + (256, 256, 8192, 16, 16, False, True, True): (2, 32, 3, 4), + (256, 256, 8192, 16, 16, True, False, True): (4, 64, 3, 2), + (256, 256, 8192, 32, 32, False, True, True): (1, 64, 3, 4), + (256, 256, 8192, 32, 32, True, False, True): (3, 128, 1, 2), + (256, 256, 8192, 64, 64, False, True, True): (9, 128, 1, 4), + (256, 256, 8192, 64, 64, True, False, True): (8, 128, 1, 4), + (256, 256, 8192, 128, 128, False, True, True): (7, 64, 1, 4), + (256, 256, 8192, 128, 128, True, False, True): (1, 32, 1, 16), + (256, 256, 16384, 16, 16, False, True, True): (3, 128, 3, 2), + (256, 256, 16384, 16, 16, True, False, True): (5, 64, 3, 2), + (256, 256, 16384, 32, 32, False, True, True): (3, 128, 3, 2), + (256, 256, 16384, 32, 32, True, False, True): (1, 128, 3, 2), + (256, 256, 16384, 64, 64, False, True, True): (3, 128, 1, 4), + (256, 256, 16384, 64, 64, True, False, True): (2, 128, 1, 4), + (256, 256, 16384, 128, 128, False, True, True): (7, 128, 1, 4), + (256, 256, 16384, 128, 128, True, False, True): (1, 128, 2, 8), + (256, 256, 32768, 16, 16, False, True, True): (2, 128, 3, 2), + (256, 256, 32768, 16, 16, True, False, True): (1, 128, 3, 2), + (256, 256, 32768, 32, 32, False, True, True): (1, 256, 3, 4), + (256, 256, 32768, 32, 32, True, False, True): (3, 256, 3, 2), + (256, 256, 32768, 64, 64, False, True, True): (1, 256, 1, 4), + (256, 256, 32768, 64, 64, True, False, True): (3, 256, 1, 4), + (256, 256, 32768, 128, 128, False, True, True): (9, 256, 1, 4), + (256, 256, 32768, 128, 128, True, False, True): (2, 256, 1, 4), + (256, 256, 65536, 16, 16, False, True, True): (1, 256, 3, 2), + (256, 256, 65536, 16, 16, True, False, True): (1, 256, 3, 2), + (256, 256, 65536, 32, 32, False, True, True): (2, 512, 3, 2), + (256, 256, 65536, 32, 32, True, False, True): (2, 512, 3, 2), + (256, 256, 65536, 64, 64, False, True, True): (2, 512, 1, 4), + (256, 256, 65536, 64, 64, True, False, True): (1, 512, 1, 4), + (256, 256, 65536, 128, 128, False, True, True): (7, 512, 1, 4), + (256, 256, 65536, 128, 128, True, False, True): (2, 512, 1, 4), + (256, 256, 131072, 16, 16, False, True, True): (1, 512, 3, 2), + (256, 256, 131072, 16, 16, True, False, True): (1, 512, 3, 2), + (256, 256, 131072, 32, 32, False, True, True): (1, 1024, 3, 2), + (256, 256, 131072, 32, 32, True, False, True): (1, 1024, 3, 2), + (256, 256, 131072, 64, 64, False, True, True): (1, 1024, 1, 4), + (256, 256, 131072, 64, 64, True, False, True): (1, 1024, 1, 4), + (256, 256, 131072, 128, 128, False, True, True): (3, 1024, 1, 4), + (256, 256, 131072, 128, 128, True, False, True): (1, 1024, 1, 4), + (384, 384, 256, 16, 16, False, True, True): (1, 8, 5, 2), + (384, 384, 256, 16, 16, True, False, True): (3, 4, 5, 2), + (384, 384, 256, 32, 32, False, True, True): (2, 8, 4, 4), + (384, 384, 256, 32, 32, True, False, True): (1, 4, 6, 2), + (384, 384, 256, 64, 64, False, True, True): (2, 4, 4, 4), + (384, 384, 256, 64, 64, True, False, True): (2, 4, 4, 4), + (384, 384, 512, 16, 16, False, True, True): (1, 8, 4, 2), + (384, 384, 512, 16, 16, True, False, True): (1, 4, 5, 4), + (384, 384, 512, 32, 32, False, True, True): (1, 8, 4, 4), + (384, 384, 512, 32, 32, True, False, True): (3, 8, 5, 2), + (384, 384, 512, 64, 64, False, True, True): (3, 8, 3, 4), + (384, 384, 512, 64, 64, True, False, True): (5, 8, 5, 4), + (384, 384, 1024, 16, 16, False, True, True): (3, 16, 4, 2), + (384, 384, 1024, 16, 16, True, False, True): (1, 8, 4, 4), + (384, 384, 1024, 32, 32, False, True, True): (6, 32, 3, 2), + (384, 384, 1024, 32, 32, True, False, True): (3, 8, 4, 4), + (384, 384, 1024, 64, 64, False, True, True): (3, 16, 3, 4), + (384, 384, 1024, 64, 64, True, False, True): (2, 16, 4, 4), + (384, 384, 2048, 16, 16, False, True, True): (1, 32, 1, 4), + (384, 384, 2048, 16, 16, True, False, True): (1, 16, 5, 2), + (384, 384, 2048, 32, 32, False, True, True): (1, 32, 1, 8), + (384, 384, 2048, 32, 32, True, False, True): (1, 8, 4, 4), + (384, 384, 2048, 64, 64, False, True, True): (4, 16, 3, 4), + (384, 384, 2048, 64, 64, True, False, True): (1, 16, 3, 8), + (384, 384, 4096, 16, 16, False, True, True): (5, 32, 1, 4), + (384, 384, 4096, 16, 16, True, False, True): (6, 32, 3, 2), + (384, 384, 4096, 32, 32, False, True, True): (1, 32, 1, 8), + (384, 384, 4096, 32, 32, True, False, True): (1, 16, 3, 4), + (384, 384, 4096, 64, 64, False, True, True): (1, 64, 1, 4), + (384, 384, 4096, 64, 64, True, False, True): (2, 32, 3, 4), + (384, 384, 8192, 16, 16, False, True, True): (2, 64, 1, 4), + (384, 384, 8192, 16, 16, True, False, True): (3, 32, 3, 2), + (384, 384, 8192, 32, 32, False, True, True): (5, 64, 1, 8), + (384, 384, 8192, 32, 32, True, False, True): (1, 32, 3, 2), + (384, 384, 8192, 64, 64, False, True, True): (1, 128, 1, 4), + (384, 384, 8192, 64, 64, True, False, True): (3, 64, 3, 4), + (384, 384, 16384, 16, 16, False, True, True): (1, 128, 1, 2), + (384, 384, 16384, 16, 16, True, False, True): (4, 128, 3, 2), + (384, 384, 16384, 32, 32, False, True, True): (3, 128, 1, 4), + (384, 384, 16384, 32, 32, True, False, True): (1, 128, 3, 2), + (384, 384, 16384, 64, 64, False, True, True): (3, 256, 1, 4), + (384, 384, 16384, 64, 64, True, False, True): (2, 128, 3, 4), + (384, 384, 32768, 16, 16, False, True, True): (1, 256, 1, 2), + (384, 384, 32768, 16, 16, True, False, True): (1, 128, 3, 4), + (384, 384, 32768, 32, 32, False, True, True): (1, 256, 1, 2), + (384, 384, 32768, 32, 32, True, False, True): (1, 128, 3, 4), + (384, 384, 32768, 64, 64, False, True, True): (2, 256, 1, 4), + (384, 384, 32768, 64, 64, True, False, True): (1, 256, 3, 4), + (384, 384, 65536, 16, 16, False, True, True): (4, 512, 1, 2), + (384, 384, 65536, 16, 16, True, False, True): (1, 256, 3, 4), + (384, 384, 65536, 32, 32, False, True, True): (1, 512, 1, 2), + (384, 384, 65536, 32, 32, True, False, True): (1, 256, 3, 4), + (384, 384, 65536, 64, 64, False, True, True): (3, 512, 1, 4), + (384, 384, 65536, 64, 64, True, False, True): (3, 256, 3, 4), + (384, 384, 131072, 16, 16, False, True, True): (1, 512, 1, 1), + (384, 384, 131072, 16, 16, True, False, True): (1, 512, 3, 4), + (384, 384, 131072, 32, 32, False, True, True): (1, 512, 1, 4), + (384, 384, 131072, 32, 32, True, False, True): (1, 512, 3, 4), + (384, 384, 131072, 64, 64, False, True, True): (3, 1024, 1, 4), + (384, 384, 131072, 64, 64, True, False, True): (3, 512, 3, 4), + (512, 512, 256, 16, 16, False, True, True): (2, 4, 5, 4), + (512, 512, 256, 16, 16, True, False, True): (3, 4, 5, 4), + (512, 512, 256, 32, 32, False, True, True): (1, 4, 5, 2), + (512, 512, 256, 32, 32, True, False, True): (4, 8, 5, 1), + (512, 512, 256, 64, 64, False, True, True): (4, 4, 5, 4), + (512, 512, 256, 64, 64, True, False, True): (5, 4, 5, 4), + (512, 512, 256, 128, 128, False, True, True): (3, 2, 2, 8), + (512, 512, 256, 128, 128, True, False, True): (2, 2, 2, 8), + (512, 512, 512, 16, 16, False, True, True): (1, 8, 5, 4), + (512, 512, 512, 16, 16, True, False, True): (4, 8, 5, 2), + (512, 512, 512, 32, 32, False, True, True): (1, 16, 4, 1), + (512, 512, 512, 32, 32, True, False, True): (1, 8, 5, 2), + (512, 512, 512, 64, 64, False, True, True): (4, 8, 5, 4), + (512, 512, 512, 64, 64, True, False, True): (2, 8, 5, 4), + (512, 512, 512, 128, 128, False, True, True): (2, 4, 2, 8), + (512, 512, 512, 128, 128, True, False, True): (1, 4, 2, 8), + (512, 512, 1024, 16, 16, False, True, True): (2, 8, 4, 4), + (512, 512, 1024, 16, 16, True, False, True): (1, 8, 4, 4), + (512, 512, 1024, 32, 32, False, True, True): (3, 16, 4, 2), + (512, 512, 1024, 32, 32, True, False, True): (1, 16, 5, 2), + (512, 512, 1024, 64, 64, False, True, True): (2, 8, 3, 4), + (512, 512, 1024, 64, 64, True, False, True): (2, 16, 3, 4), + (512, 512, 1024, 128, 128, False, True, True): (2, 8, 2, 8), + (512, 512, 1024, 128, 128, True, False, True): (3, 8, 2, 8), + (512, 512, 2048, 16, 16, False, True, True): (4, 16, 3, 2), + (512, 512, 2048, 16, 16, True, False, True): (1, 16, 4, 2), + (512, 512, 2048, 32, 32, False, True, True): (3, 32, 3, 2), + (512, 512, 2048, 32, 32, True, False, True): (2, 32, 3, 2), + (512, 512, 2048, 64, 64, False, True, True): (6, 32, 3, 2), + (512, 512, 2048, 64, 64, True, False, True): (1, 32, 3, 2), + (512, 512, 2048, 128, 128, False, True, True): (4, 16, 2, 8), + (512, 512, 2048, 128, 128, True, False, True): (1, 16, 2, 8), + (512, 512, 4096, 16, 16, False, True, True): (1, 16, 3, 2), + (512, 512, 4096, 16, 16, True, False, True): (4, 32, 3, 2), + (512, 512, 4096, 32, 32, False, True, True): (3, 32, 3, 2), + (512, 512, 4096, 32, 32, True, False, True): (2, 32, 3, 2), + (512, 512, 4096, 64, 64, False, True, True): (1, 32, 3, 4), + (512, 512, 4096, 64, 64, True, False, True): (1, 64, 3, 4), + (512, 512, 4096, 128, 128, False, True, True): (4, 32, 1, 4), + (512, 512, 4096, 128, 128, True, False, True): (4, 32, 2, 8), + (512, 512, 8192, 16, 16, False, True, True): (8, 64, 3, 2), + (512, 512, 8192, 16, 16, True, False, True): (4, 64, 3, 2), + (512, 512, 8192, 32, 32, False, True, True): (3, 64, 3, 2), + (512, 512, 8192, 32, 32, True, False, True): (3, 64, 3, 2), + (512, 512, 8192, 64, 64, False, True, True): (1, 64, 3, 4), + (512, 512, 8192, 64, 64, True, False, True): (7, 64, 3, 4), + (512, 512, 8192, 128, 128, False, True, True): (1, 64, 1, 4), + (512, 512, 8192, 128, 128, True, False, True): (4, 64, 2, 8), + (512, 512, 16384, 16, 16, False, True, True): (1, 64, 3, 2), + (512, 512, 16384, 16, 16, True, False, True): (1, 128, 3, 2), + (512, 512, 16384, 32, 32, False, True, True): (3, 128, 3, 2), + (512, 512, 16384, 32, 32, True, False, True): (1, 128, 3, 2), + (512, 512, 16384, 64, 64, False, True, True): (4, 64, 2, 4), + (512, 512, 16384, 64, 64, True, False, True): (2, 64, 2, 4), + (512, 512, 16384, 128, 128, False, True, True): (4, 128, 1, 4), + (512, 512, 16384, 128, 128, True, False, True): (2, 128, 1, 4), + (512, 512, 32768, 16, 16, False, True, True): (1, 128, 3, 2), + (512, 512, 32768, 16, 16, True, False, True): (1, 128, 3, 2), + (512, 512, 32768, 32, 32, False, True, True): (1, 256, 3, 2), + (512, 512, 32768, 32, 32, True, False, True): (1, 256, 3, 2), + (512, 512, 32768, 64, 64, False, True, True): (1, 256, 3, 4), + (512, 512, 32768, 64, 64, True, False, True): (2, 256, 3, 4), + (512, 512, 32768, 128, 128, False, True, True): (5, 256, 1, 4), + (512, 512, 32768, 128, 128, True, False, True): (4, 256, 1, 4), + (512, 512, 65536, 16, 16, False, True, True): (1, 256, 3, 2), + (512, 512, 65536, 16, 16, True, False, True): (1, 256, 3, 1), + (512, 512, 65536, 32, 32, False, True, True): (1, 512, 3, 2), + (512, 512, 65536, 32, 32, True, False, True): (1, 512, 3, 2), + (512, 512, 65536, 64, 64, False, True, True): (4, 256, 2, 4), + (512, 512, 65536, 64, 64, True, False, True): (2, 512, 3, 4), + (512, 512, 65536, 128, 128, False, True, True): (6, 512, 1, 4), + (512, 512, 65536, 128, 128, True, False, True): (4, 512, 1, 4), + (512, 512, 131072, 16, 16, False, True, True): (1, 512, 3, 2), + (512, 512, 131072, 16, 16, True, False, True): (1, 512, 3, 1), + (512, 512, 131072, 32, 32, False, True, True): (1, 1024, 3, 2), + (512, 512, 131072, 32, 32, True, False, True): (1, 1024, 3, 2), + (512, 512, 131072, 64, 64, False, True, True): (4, 512, 2, 4), + (512, 512, 131072, 64, 64, True, False, True): (4, 1024, 3, 4), + (512, 512, 131072, 128, 128, False, True, True): (6, 1024, 1, 4), + (512, 512, 131072, 128, 128, True, False, True): (4, 1024, 1, 4), + (768, 768, 256, 16, 16, False, True, True): (1, 8, 4, 1), + (768, 768, 256, 16, 16, True, False, True): (3, 2, 6, 4), + (768, 768, 256, 32, 32, False, True, True): (3, 8, 3, 4), + (768, 768, 256, 32, 32, True, False, True): (1, 4, 4, 2), + (768, 768, 256, 64, 64, False, True, True): (2, 4, 3, 4), + (768, 768, 256, 64, 64, True, False, True): (1, 4, 4, 4), + (768, 768, 256, 128, 128, False, True, True): (2, 2, 3, 8), + (768, 768, 256, 128, 128, True, False, True): (4, 2, 3, 8), + (768, 768, 512, 16, 16, False, True, True): (4, 8, 4, 2), + (768, 768, 512, 16, 16, True, False, True): (4, 8, 6, 2), + (768, 768, 512, 32, 32, False, True, True): (1, 8, 4, 4), + (768, 768, 512, 32, 32, True, False, True): (3, 8, 4, 2), + (768, 768, 512, 64, 64, False, True, True): (1, 8, 3, 4), + (768, 768, 512, 64, 64, True, False, True): (1, 8, 4, 4), + (768, 768, 512, 128, 128, False, True, True): (1, 4, 3, 8), + (768, 768, 512, 128, 128, True, False, True): (4, 4, 3, 8), + (768, 768, 1024, 16, 16, False, True, True): (3, 16, 1, 4), + (768, 768, 1024, 16, 16, True, False, True): (1, 8, 5, 2), + (768, 768, 1024, 32, 32, False, True, True): (3, 16, 1, 8), + (768, 768, 1024, 32, 32, True, False, True): (1, 16, 3, 2), + (768, 768, 1024, 64, 64, False, True, True): (1, 8, 3, 4), + (768, 768, 1024, 64, 64, True, False, True): (2, 8, 3, 8), + (768, 768, 1024, 128, 128, False, True, True): (1, 8, 3, 8), + (768, 768, 1024, 128, 128, True, False, True): (1, 8, 3, 8), + (768, 768, 2048, 16, 16, False, True, True): (2, 16, 1, 2), + (768, 768, 2048, 16, 16, True, False, True): (1, 16, 3, 2), + (768, 768, 2048, 32, 32, False, True, True): (5, 32, 1, 4), + (768, 768, 2048, 32, 32, True, False, True): (3, 8, 3, 4), + (768, 768, 2048, 64, 64, False, True, True): (1, 16, 1, 8), + (768, 768, 2048, 64, 64, True, False, True): (3, 16, 3, 4), + (768, 768, 2048, 128, 128, False, True, True): (2, 16, 3, 8), + (768, 768, 2048, 128, 128, True, False, True): (1, 16, 3, 8), + (768, 768, 4096, 16, 16, False, True, True): (3, 32, 1, 4), + (768, 768, 4096, 16, 16, True, False, True): (2, 32, 3, 1), + (768, 768, 4096, 32, 32, False, True, True): (2, 64, 1, 4), + (768, 768, 4096, 32, 32, True, False, True): (1, 16, 4, 4), + (768, 768, 4096, 64, 64, False, True, True): (3, 64, 3, 4), + (768, 768, 4096, 64, 64, True, False, True): (2, 16, 3, 4), + (768, 768, 4096, 128, 128, False, True, True): (1, 32, 3, 8), + (768, 768, 4096, 128, 128, True, False, True): (4, 32, 3, 8), + (768, 768, 8192, 16, 16, False, True, True): (1, 64, 1, 2), + (768, 768, 8192, 16, 16, True, False, True): (4, 64, 3, 2), + (768, 768, 8192, 32, 32, False, True, True): (1, 64, 1, 8), + (768, 768, 8192, 32, 32, True, False, True): (2, 32, 3, 4), + (768, 768, 8192, 64, 64, False, True, True): (4, 64, 3, 4), + (768, 768, 8192, 64, 64, True, False, True): (2, 32, 3, 4), + (768, 768, 8192, 128, 128, False, True, True): (2, 64, 3, 8), + (768, 768, 8192, 128, 128, True, False, True): (1, 64, 3, 8), + (768, 768, 16384, 16, 16, False, True, True): (1, 128, 1, 2), + (768, 768, 16384, 16, 16, True, False, True): (1, 64, 4, 4), + (768, 768, 16384, 32, 32, False, True, True): (1, 128, 1, 8), + (768, 768, 16384, 32, 32, True, False, True): (1, 64, 3, 4), + (768, 768, 16384, 64, 64, False, True, True): (4, 128, 3, 4), + (768, 768, 16384, 64, 64, True, False, True): (1, 64, 3, 4), + (768, 768, 16384, 128, 128, False, True, True): (3, 128, 1, 4), + (768, 768, 16384, 128, 128, True, False, True): (3, 128, 2, 4), + (768, 768, 32768, 16, 16, False, True, True): (1, 256, 1, 2), + (768, 768, 32768, 16, 16, True, False, True): (1, 128, 4, 4), + (768, 768, 32768, 32, 32, False, True, True): (1, 128, 1, 2), + (768, 768, 32768, 32, 32, True, False, True): (1, 128, 3, 4), + (768, 768, 32768, 64, 64, False, True, True): (1, 256, 1, 4), + (768, 768, 32768, 64, 64, True, False, True): (2, 128, 3, 4), + (768, 768, 32768, 128, 128, False, True, True): (3, 256, 1, 4), + (768, 768, 32768, 128, 128, True, False, True): (2, 256, 2, 4), + (768, 768, 65536, 16, 16, False, True, True): (4, 512, 1, 2), + (768, 768, 65536, 16, 16, True, False, True): (1, 256, 4, 4), + (768, 768, 65536, 32, 32, False, True, True): (1, 256, 1, 2), + (768, 768, 65536, 32, 32, True, False, True): (1, 256, 3, 4), + (768, 768, 65536, 64, 64, False, True, True): (3, 512, 1, 4), + (768, 768, 65536, 64, 64, True, False, True): (2, 256, 3, 4), + (768, 768, 65536, 128, 128, False, True, True): (3, 512, 1, 4), + (768, 768, 65536, 128, 128, True, False, True): (2, 512, 2, 4), + (768, 768, 131072, 16, 16, False, True, True): (4, 1024, 1, 2), + (768, 768, 131072, 16, 16, True, False, True): (1, 512, 4, 1), + (768, 768, 131072, 32, 32, False, True, True): (1, 512, 1, 2), + (768, 768, 131072, 32, 32, True, False, True): (1, 512, 3, 4), + (768, 768, 131072, 64, 64, False, True, True): (1, 1024, 1, 4), + (768, 768, 131072, 64, 64, True, False, True): (2, 512, 3, 4), + (768, 768, 131072, 128, 128, False, True, True): (3, 1024, 1, 4), + (768, 768, 131072, 128, 128, True, False, True): (1, 1024, 2, 4), + (768, 3072, 256, 16, 16, False, True, True): (3, 8, 6, 1), + (768, 3072, 256, 16, 16, True, False, True): (1, 4, 6, 2), + (768, 3072, 256, 32, 32, False, True, True): (1, 8, 4, 4), + (768, 3072, 256, 32, 32, True, False, True): (3, 4, 6, 4), + (768, 3072, 256, 64, 64, False, True, True): (2, 4, 3, 4), + (768, 3072, 256, 64, 64, True, False, True): (1, 4, 4, 4), + (768, 3072, 256, 128, 128, False, True, True): (2, 2, 3, 8), + (768, 3072, 256, 128, 128, True, False, True): (1, 2, 3, 8), + (768, 3072, 512, 16, 16, False, True, True): (1, 8, 4, 2), + (768, 3072, 512, 16, 16, True, False, True): (1, 8, 5, 2), + (768, 3072, 512, 32, 32, False, True, True): (1, 16, 3, 2), + (768, 3072, 512, 32, 32, True, False, True): (1, 8, 5, 2), + (768, 3072, 512, 64, 64, False, True, True): (1, 8, 3, 4), + (768, 3072, 512, 64, 64, True, False, True): (3, 8, 4, 4), + (768, 3072, 512, 128, 128, False, True, True): (1, 4, 3, 8), + (768, 3072, 512, 128, 128, True, False, True): (2, 4, 3, 8), + (768, 3072, 1024, 16, 16, False, True, True): (1, 16, 1, 4), + (768, 3072, 1024, 16, 16, True, False, True): (5, 4, 4, 4), + (768, 3072, 1024, 32, 32, False, True, True): (3, 8, 3, 4), + (768, 3072, 1024, 32, 32, True, False, True): (1, 8, 4, 4), + (768, 3072, 1024, 64, 64, False, True, True): (2, 16, 3, 4), + (768, 3072, 1024, 64, 64, True, False, True): (2, 16, 4, 4), + (768, 3072, 1024, 128, 128, False, True, True): (1, 8, 3, 8), + (768, 3072, 1024, 128, 128, True, False, True): (5, 8, 3, 8), + (768, 3072, 2048, 16, 16, False, True, True): (3, 16, 1, 2), + (768, 3072, 2048, 16, 16, True, False, True): (1, 8, 3, 4), + (768, 3072, 2048, 32, 32, False, True, True): (4, 16, 1, 8), + (768, 3072, 2048, 32, 32, True, False, True): (3, 8, 3, 4), + (768, 3072, 2048, 64, 64, False, True, True): (2, 16, 3, 4), + (768, 3072, 2048, 64, 64, True, False, True): (2, 16, 3, 4), + (768, 3072, 2048, 128, 128, False, True, True): (3, 16, 3, 8), + (768, 3072, 2048, 128, 128, True, False, True): (4, 16, 3, 8), + (768, 3072, 4096, 16, 16, False, True, True): (1, 32, 1, 4), + (768, 3072, 4096, 16, 16, True, False, True): (1, 16, 3, 1), + (768, 3072, 4096, 32, 32, False, True, True): (3, 32, 1, 8), + (768, 3072, 4096, 32, 32, True, False, True): (3, 16, 4, 4), + (768, 3072, 4096, 64, 64, False, True, True): (2, 32, 3, 4), + (768, 3072, 4096, 64, 64, True, False, True): (2, 16, 3, 4), + (768, 3072, 4096, 128, 128, False, True, True): (5, 32, 1, 4), + (768, 3072, 4096, 128, 128, True, False, True): (9, 32, 3, 8), + (768, 3072, 8192, 16, 16, False, True, True): (1, 32, 1, 4), + (768, 3072, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (768, 3072, 8192, 32, 32, False, True, True): (1, 64, 1, 8), + (768, 3072, 8192, 32, 32, True, False, True): (2, 64, 4, 2), + (768, 3072, 8192, 64, 64, False, True, True): (1, 64, 3, 4), + (768, 3072, 8192, 64, 64, True, False, True): (2, 32, 3, 4), + (768, 3072, 8192, 128, 128, False, True, True): (2, 64, 3, 8), + (768, 3072, 8192, 128, 128, True, False, True): (2, 64, 3, 8), + (768, 3072, 16384, 16, 16, False, True, True): (1, 64, 1, 4), + (768, 3072, 16384, 16, 16, True, False, True): (1, 64, 4, 1), + (768, 3072, 16384, 32, 32, False, True, True): (1, 128, 1, 8), + (768, 3072, 16384, 32, 32, True, False, True): (1, 64, 3, 4), + (768, 3072, 16384, 64, 64, False, True, True): (1, 128, 3, 4), + (768, 3072, 16384, 64, 64, True, False, True): (4, 64, 3, 4), + (768, 3072, 16384, 128, 128, False, True, True): (2, 128, 3, 8), + (768, 3072, 16384, 128, 128, True, False, True): (2, 128, 3, 8), + (768, 3072, 32768, 16, 16, False, True, True): (1, 128, 1, 4), + (768, 3072, 32768, 16, 16, True, False, True): (1, 128, 4, 1), + (768, 3072, 32768, 32, 32, False, True, True): (1, 256, 1, 8), + (768, 3072, 32768, 32, 32, True, False, True): (1, 128, 3, 4), + (768, 3072, 32768, 64, 64, False, True, True): (1, 256, 3, 4), + (768, 3072, 32768, 64, 64, True, False, True): (1, 128, 3, 4), + (768, 3072, 32768, 128, 128, False, True, True): (3, 256, 1, 4), + (768, 3072, 32768, 128, 128, True, False, True): (2, 256, 3, 8), + (768, 3072, 50432, 16, 16, False, True, True): (1, 197, 1, 4), + (768, 3072, 50432, 16, 16, True, False, True): (4, 197, 4, 4), + (768, 3072, 50432, 32, 32, False, True, True): (1, 197, 1, 4), + (768, 3072, 50432, 32, 32, True, False, True): (4, 197, 3, 4), + (768, 3072, 50432, 64, 64, False, True, True): (1, 394, 3, 4), + (768, 3072, 50432, 64, 64, True, False, True): (3, 197, 3, 4), + (768, 3072, 50432, 128, 128, False, True, True): (3, 394, 1, 4), + (768, 3072, 50432, 128, 128, True, False, True): (1, 394, 3, 8), + (768, 3072, 65536, 16, 16, False, True, True): (1, 256, 1, 4), + (768, 3072, 65536, 16, 16, True, False, True): (5, 256, 4, 1), + (768, 3072, 65536, 32, 32, False, True, True): (1, 256, 1, 4), + (768, 3072, 65536, 32, 32, True, False, True): (3, 256, 3, 4), + (768, 3072, 65536, 64, 64, False, True, True): (2, 512, 3, 4), + (768, 3072, 65536, 64, 64, True, False, True): (3, 256, 3, 4), + (768, 3072, 65536, 128, 128, False, True, True): (3, 512, 1, 4), + (768, 3072, 65536, 128, 128, True, False, True): (2, 512, 3, 8), + (768, 3072, 131072, 16, 16, False, True, True): (1, 512, 1, 4), + (768, 3072, 131072, 16, 16, True, False, True): (5, 512, 4, 1), + (768, 3072, 131072, 32, 32, False, True, True): (1, 512, 1, 4), + (768, 3072, 131072, 32, 32, True, False, True): (4, 512, 3, 4), + (768, 3072, 131072, 64, 64, False, True, True): (1, 1024, 3, 4), + (768, 3072, 131072, 64, 64, True, False, True): (1, 512, 3, 4), + (768, 3072, 131072, 128, 128, False, True, True): (3, 1024, 1, 4), + (768, 3072, 131072, 128, 128, True, False, True): (1, 1024, 3, 8), + (1024, 1024, 256, 16, 16, False, True, True): (1, 4, 5, 4), + (1024, 1024, 256, 16, 16, True, False, True): (3, 4, 4, 4), + (1024, 1024, 256, 32, 32, False, True, True): (4, 4, 5, 2), + (1024, 1024, 256, 32, 32, True, False, True): (3, 4, 5, 2), + (1024, 1024, 256, 64, 64, False, True, True): (1, 4, 5, 4), + (1024, 1024, 256, 64, 64, True, False, True): (1, 4, 5, 4), + (1024, 1024, 256, 128, 128, False, True, True): (1, 2, 2, 8), + (1024, 1024, 256, 128, 128, True, False, True): (2, 2, 2, 8), + (1024, 1024, 512, 16, 16, False, True, True): (3, 4, 4, 4), + (1024, 1024, 512, 16, 16, True, False, True): (4, 8, 5, 2), + (1024, 1024, 512, 32, 32, False, True, True): (1, 8, 4, 2), + (1024, 1024, 512, 32, 32, True, False, True): (1, 8, 4, 2), + (1024, 1024, 512, 64, 64, False, True, True): (4, 8, 4, 4), + (1024, 1024, 512, 64, 64, True, False, True): (2, 8, 3, 4), + (1024, 1024, 512, 128, 128, False, True, True): (2, 4, 2, 8), + (1024, 1024, 512, 128, 128, True, False, True): (1, 4, 2, 8), + (1024, 1024, 1024, 16, 16, False, True, True): (3, 8, 4, 4), + (1024, 1024, 1024, 16, 16, True, False, True): (4, 8, 4, 2), + (1024, 1024, 1024, 32, 32, False, True, True): (1, 16, 3, 2), + (1024, 1024, 1024, 32, 32, True, False, True): (1, 16, 3, 2), + (1024, 1024, 1024, 64, 64, False, True, True): (1, 16, 3, 4), + (1024, 1024, 1024, 64, 64, True, False, True): (3, 16, 3, 2), + (1024, 1024, 1024, 128, 128, False, True, True): (1, 8, 2, 8), + (1024, 1024, 1024, 128, 128, True, False, True): (2, 8, 2, 8), + (1024, 1024, 2048, 16, 16, False, True, True): (3, 8, 3, 4), + (1024, 1024, 2048, 16, 16, True, False, True): (3, 8, 3, 2), + (1024, 1024, 2048, 32, 32, False, True, True): (5, 16, 3, 4), + (1024, 1024, 2048, 32, 32, True, False, True): (1, 16, 3, 2), + (1024, 1024, 2048, 64, 64, False, True, True): (6, 16, 4, 4), + (1024, 1024, 2048, 64, 64, True, False, True): (5, 16, 3, 4), + (1024, 1024, 2048, 128, 128, False, True, True): (4, 16, 2, 8), + (1024, 1024, 2048, 128, 128, True, False, True): (4, 16, 2, 8), + (1024, 1024, 4096, 16, 16, False, True, True): (8, 32, 3, 2), + (1024, 1024, 4096, 16, 16, True, False, True): (4, 32, 3, 2), + (1024, 1024, 4096, 32, 32, False, True, True): (2, 32, 3, 4), + (1024, 1024, 4096, 32, 32, True, False, True): (3, 32, 3, 2), + (1024, 1024, 4096, 64, 64, False, True, True): (3, 32, 3, 4), + (1024, 1024, 4096, 64, 64, True, False, True): (1, 32, 3, 4), + (1024, 1024, 4096, 128, 128, False, True, True): (4, 32, 2, 8), + (1024, 1024, 4096, 128, 128, True, False, True): (1, 32, 2, 8), + (1024, 1024, 8192, 16, 16, False, True, True): (4, 64, 3, 2), + (1024, 1024, 8192, 16, 16, True, False, True): (4, 64, 3, 2), + (1024, 1024, 8192, 32, 32, False, True, True): (8, 64, 3, 4), + (1024, 1024, 8192, 32, 32, True, False, True): (4, 32, 3, 4), + (1024, 1024, 8192, 64, 64, False, True, True): (4, 64, 3, 4), + (1024, 1024, 8192, 64, 64, True, False, True): (2, 64, 3, 4), + (1024, 1024, 8192, 128, 128, False, True, True): (4, 64, 2, 8), + (1024, 1024, 8192, 128, 128, True, False, True): (4, 64, 1, 4), + (1024, 1024, 16384, 16, 16, False, True, True): (1, 64, 3, 2), + (1024, 1024, 16384, 16, 16, True, False, True): (1, 64, 3, 2), + (1024, 1024, 16384, 32, 32, False, True, True): (1, 128, 3, 2), + (1024, 1024, 16384, 32, 32, True, False, True): (1, 64, 3, 4), + (1024, 1024, 16384, 64, 64, False, True, True): (1, 128, 3, 4), + (1024, 1024, 16384, 64, 64, True, False, True): (1, 128, 3, 4), + (1024, 1024, 16384, 128, 128, False, True, True): (2, 128, 1, 4), + (1024, 1024, 16384, 128, 128, True, False, True): (4, 128, 1, 4), + (1024, 1024, 32768, 16, 16, False, True, True): (1, 128, 3, 2), + (1024, 1024, 32768, 16, 16, True, False, True): (1, 128, 3, 2), + (1024, 1024, 32768, 32, 32, False, True, True): (1, 256, 3, 2), + (1024, 1024, 32768, 32, 32, True, False, True): (1, 128, 3, 4), + (1024, 1024, 32768, 64, 64, False, True, True): (2, 128, 2, 4), + (1024, 1024, 32768, 64, 64, True, False, True): (1, 256, 3, 4), + (1024, 1024, 32768, 128, 128, False, True, True): (2, 256, 1, 4), + (1024, 1024, 32768, 128, 128, True, False, True): (4, 256, 1, 4), + (1024, 1024, 65536, 16, 16, False, True, True): (1, 256, 3, 4), + (1024, 1024, 65536, 16, 16, True, False, True): (1, 256, 3, 4), + (1024, 1024, 65536, 32, 32, False, True, True): (9, 256, 3, 4), + (1024, 1024, 65536, 32, 32, True, False, True): (7, 256, 3, 4), + (1024, 1024, 65536, 64, 64, False, True, True): (2, 256, 2, 4), + (1024, 1024, 65536, 64, 64, True, False, True): (2, 512, 3, 4), + (1024, 1024, 65536, 128, 128, False, True, True): (2, 512, 1, 4), + (1024, 1024, 65536, 128, 128, True, False, True): (4, 512, 1, 4), + (1024, 1024, 131072, 16, 16, False, True, True): (11, 512, 3, 2), + (1024, 1024, 131072, 16, 16, True, False, True): (11, 512, 3, 2), + (1024, 1024, 131072, 32, 32, False, True, True): (4, 512, 3, 4), + (1024, 1024, 131072, 32, 32, True, False, True): (6, 512, 3, 4), + (1024, 1024, 131072, 64, 64, False, True, True): (2, 512, 2, 4), + (1024, 1024, 131072, 64, 64, True, False, True): (2, 1024, 3, 4), + (1024, 1024, 131072, 128, 128, False, True, True): (4, 1024, 1, 4), + (1024, 1024, 131072, 128, 128, True, False, True): (4, 1024, 1, 4), + (1280, 5120, 65792, 16, 16, False, True, True): (1, 257, 1, 4), + (1280, 5120, 65792, 16, 16, True, False, True): (5, 257, 4, 1), + (1280, 5120, 65792, 32, 32, False, True, True): (1, 514, 1, 8), + (1280, 5120, 65792, 32, 32, True, False, True): (2, 257, 3, 4), + (1280, 5120, 65792, 64, 64, False, True, True): (1, 514, 3, 4), + (1280, 5120, 65792, 64, 64, True, False, True): (1, 257, 3, 4), + (1280, 5120, 65792, 128, 128, False, True, True): (1, 514, 3, 8), + (1280, 5120, 65792, 128, 128, True, False, True): (2, 514, 3, 8), + (1536, 1536, 256, 16, 16, False, True, True): (1, 4, 6, 2), + (1536, 1536, 256, 16, 16, True, False, True): (3, 4, 5, 2), + (1536, 1536, 256, 32, 32, False, True, True): (2, 4, 3, 4), + (1536, 1536, 256, 32, 32, True, False, True): (1, 4, 5, 2), + (1536, 1536, 256, 64, 64, False, True, True): (2, 4, 3, 4), + (1536, 1536, 256, 64, 64, True, False, True): (1, 4, 4, 4), + (1536, 1536, 256, 128, 128, False, True, True): (3, 2, 3, 8), + (1536, 1536, 256, 128, 128, True, False, True): (6, 2, 3, 8), + (1536, 1536, 512, 16, 16, False, True, True): (1, 8, 1, 4), + (1536, 1536, 512, 16, 16, True, False, True): (3, 4, 5, 2), + (1536, 1536, 512, 32, 32, False, True, True): (1, 8, 1, 8), + (1536, 1536, 512, 32, 32, True, False, True): (1, 4, 4, 4), + (1536, 1536, 512, 64, 64, False, True, True): (3, 8, 5, 4), + (1536, 1536, 512, 64, 64, True, False, True): (3, 8, 3, 4), + (1536, 1536, 512, 128, 128, False, True, True): (2, 4, 3, 8), + (1536, 1536, 512, 128, 128, True, False, True): (3, 4, 3, 8), + (1536, 1536, 1024, 16, 16, False, True, True): (1, 8, 1, 2), + (1536, 1536, 1024, 16, 16, True, False, True): (2, 8, 4, 2), + (1536, 1536, 1024, 32, 32, False, True, True): (8, 16, 1, 4), + (1536, 1536, 1024, 32, 32, True, False, True): (3, 8, 4, 2), + (1536, 1536, 1024, 64, 64, False, True, True): (1, 16, 3, 4), + (1536, 1536, 1024, 64, 64, True, False, True): (3, 8, 3, 4), + (1536, 1536, 1024, 128, 128, False, True, True): (3, 8, 3, 8), + (1536, 1536, 1024, 128, 128, True, False, True): (3, 8, 3, 8), + (1536, 1536, 2048, 16, 16, False, True, True): (1, 16, 1, 4), + (1536, 1536, 2048, 16, 16, True, False, True): (1, 8, 3, 1), + (1536, 1536, 2048, 32, 32, False, True, True): (3, 16, 1, 8), + (1536, 1536, 2048, 32, 32, True, False, True): (3, 8, 4, 4), + (1536, 1536, 2048, 64, 64, False, True, True): (1, 16, 3, 4), + (1536, 1536, 2048, 64, 64, True, False, True): (3, 8, 3, 4), + (1536, 1536, 2048, 128, 128, False, True, True): (4, 16, 1, 4), + (1536, 1536, 2048, 128, 128, True, False, True): (6, 16, 3, 8), + (1536, 1536, 4096, 16, 16, False, True, True): (1, 32, 1, 2), + (1536, 1536, 4096, 16, 16, True, False, True): (4, 32, 4, 2), + (1536, 1536, 4096, 32, 32, False, True, True): (1, 32, 1, 8), + (1536, 1536, 4096, 32, 32, True, False, True): (5, 32, 4, 2), + (1536, 1536, 4096, 64, 64, False, True, True): (2, 32, 3, 4), + (1536, 1536, 4096, 64, 64, True, False, True): (2, 16, 3, 4), + (1536, 1536, 4096, 128, 128, False, True, True): (4, 32, 3, 8), + (1536, 1536, 4096, 128, 128, True, False, True): (4, 32, 3, 8), + (1536, 1536, 8192, 16, 16, False, True, True): (1, 64, 1, 2), + (1536, 1536, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (1536, 1536, 8192, 32, 32, False, True, True): (2, 64, 1, 8), + (1536, 1536, 8192, 32, 32, True, False, True): (2, 32, 3, 4), + (1536, 1536, 8192, 64, 64, False, True, True): (1, 64, 3, 4), + (1536, 1536, 8192, 64, 64, True, False, True): (2, 32, 3, 4), + (1536, 1536, 8192, 128, 128, False, True, True): (4, 64, 3, 8), + (1536, 1536, 8192, 128, 128, True, False, True): (1, 64, 3, 8), + (1536, 1536, 16384, 16, 16, False, True, True): (1, 128, 1, 2), + (1536, 1536, 16384, 16, 16, True, False, True): (1, 64, 4, 4), + (1536, 1536, 16384, 32, 32, False, True, True): (1, 64, 1, 2), + (1536, 1536, 16384, 32, 32, True, False, True): (1, 64, 3, 4), + (1536, 1536, 16384, 64, 64, False, True, True): (1, 128, 3, 4), + (1536, 1536, 16384, 64, 64, True, False, True): (1, 64, 3, 4), + (1536, 1536, 16384, 128, 128, False, True, True): (1, 128, 1, 4), + (1536, 1536, 16384, 128, 128, True, False, True): (1, 128, 2, 4), + (1536, 1536, 32768, 16, 16, False, True, True): (1, 256, 1, 2), + (1536, 1536, 32768, 16, 16, True, False, True): (1, 128, 3, 2), + (1536, 1536, 32768, 32, 32, False, True, True): (1, 128, 1, 2), + (1536, 1536, 32768, 32, 32, True, False, True): (1, 128, 3, 4), + (1536, 1536, 32768, 64, 64, False, True, True): (1, 256, 3, 4), + (1536, 1536, 32768, 64, 64, True, False, True): (1, 128, 3, 4), + (1536, 1536, 32768, 128, 128, False, True, True): (1, 256, 1, 4), + (1536, 1536, 32768, 128, 128, True, False, True): (2, 256, 2, 4), + (1536, 1536, 65536, 16, 16, False, True, True): (2, 512, 1, 2), + (1536, 1536, 65536, 16, 16, True, False, True): (1, 256, 4, 4), + (1536, 1536, 65536, 32, 32, False, True, True): (1, 256, 1, 2), + (1536, 1536, 65536, 32, 32, True, False, True): (1, 256, 3, 4), + (1536, 1536, 65536, 64, 64, False, True, True): (1, 512, 3, 4), + (1536, 1536, 65536, 64, 64, True, False, True): (3, 256, 3, 4), + (1536, 1536, 65536, 128, 128, False, True, True): (3, 512, 1, 4), + (1536, 1536, 65536, 128, 128, True, False, True): (4, 512, 2, 4), + (1536, 1536, 131072, 16, 16, False, True, True): (2, 1024, 1, 2), + (1536, 1536, 131072, 16, 16, True, False, True): (9, 512, 4, 4), + (1536, 1536, 131072, 32, 32, False, True, True): (1, 512, 1, 2), + (1536, 1536, 131072, 32, 32, True, False, True): (5, 512, 3, 4), + (1536, 1536, 131072, 64, 64, False, True, True): (1, 1024, 3, 4), + (1536, 1536, 131072, 64, 64, True, False, True): (2, 512, 3, 4), + (1536, 1536, 131072, 128, 128, False, True, True): (3, 1024, 1, 4), + (1536, 1536, 131072, 128, 128, True, False, True): (1, 1024, 2, 4), + (2048, 2048, 256, 16, 16, False, True, True): (1, 4, 5, 2), + (2048, 2048, 256, 16, 16, True, False, True): (4, 4, 5, 2), + (2048, 2048, 256, 32, 32, False, True, True): (3, 4, 6, 2), + (2048, 2048, 256, 32, 32, True, False, True): (2, 4, 5, 2), + (2048, 2048, 256, 64, 64, False, True, True): (2, 4, 4, 4), + (2048, 2048, 256, 64, 64, True, False, True): (2, 4, 3, 4), + (2048, 2048, 256, 128, 128, False, True, True): (3, 2, 2, 8), + (2048, 2048, 256, 128, 128, True, False, True): (3, 2, 2, 8), + (2048, 2048, 512, 16, 16, False, True, True): (3, 4, 4, 4), + (2048, 2048, 512, 16, 16, True, False, True): (1, 4, 4, 4), + (2048, 2048, 512, 32, 32, False, True, True): (1, 4, 3, 4), + (2048, 2048, 512, 32, 32, True, False, True): (1, 4, 4, 2), + (2048, 2048, 512, 64, 64, False, True, True): (1, 8, 3, 4), + (2048, 2048, 512, 64, 64, True, False, True): (1, 8, 3, 4), + (2048, 2048, 512, 128, 128, False, True, True): (3, 4, 2, 8), + (2048, 2048, 512, 128, 128, True, False, True): (2, 4, 2, 8), + (2048, 2048, 1024, 16, 16, False, True, True): (3, 4, 3, 4), + (2048, 2048, 1024, 16, 16, True, False, True): (4, 8, 3, 2), + (2048, 2048, 1024, 32, 32, False, True, True): (3, 8, 3, 4), + (2048, 2048, 1024, 32, 32, True, False, True): (1, 8, 3, 2), + (2048, 2048, 1024, 64, 64, False, True, True): (1, 8, 3, 4), + (2048, 2048, 1024, 64, 64, True, False, True): (1, 8, 3, 4), + (2048, 2048, 1024, 128, 128, False, True, True): (4, 8, 1, 4), + (2048, 2048, 1024, 128, 128, True, False, True): (2, 8, 1, 4), + (2048, 2048, 2048, 16, 16, False, True, True): (4, 16, 3, 2), + (2048, 2048, 2048, 16, 16, True, False, True): (4, 16, 3, 2), + (2048, 2048, 2048, 32, 32, False, True, True): (1, 16, 3, 2), + (2048, 2048, 2048, 32, 32, True, False, True): (1, 16, 3, 2), + (2048, 2048, 2048, 64, 64, False, True, True): (4, 16, 3, 4), + (2048, 2048, 2048, 64, 64, True, False, True): (4, 16, 3, 4), + (2048, 2048, 2048, 128, 128, False, True, True): (6, 16, 2, 8), + (2048, 2048, 2048, 128, 128, True, False, True): (3, 16, 1, 4), + (2048, 2048, 4096, 16, 16, False, True, True): (4, 32, 4, 2), + (2048, 2048, 4096, 16, 16, True, False, True): (4, 32, 3, 2), + (2048, 2048, 4096, 32, 32, False, True, True): (4, 16, 3, 8), + (2048, 2048, 4096, 32, 32, True, False, True): (4, 16, 3, 8), + (2048, 2048, 4096, 64, 64, False, True, True): (1, 32, 3, 4), + (2048, 2048, 4096, 64, 64, True, False, True): (3, 32, 3, 4), + (2048, 2048, 4096, 128, 128, False, True, True): (2, 32, 1, 4), + (2048, 2048, 4096, 128, 128, True, False, True): (2, 32, 1, 4), + (2048, 2048, 8192, 16, 16, False, True, True): (4, 64, 4, 2), + (2048, 2048, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (2048, 2048, 8192, 32, 32, False, True, True): (4, 32, 4, 8), + (2048, 2048, 8192, 32, 32, True, False, True): (4, 32, 3, 8), + (2048, 2048, 8192, 64, 64, False, True, True): (4, 64, 3, 4), + (2048, 2048, 8192, 64, 64, True, False, True): (4, 64, 3, 4), + (2048, 2048, 8192, 128, 128, False, True, True): (2, 64, 1, 4), + (2048, 2048, 8192, 128, 128, True, False, True): (2, 64, 1, 4), + (2048, 2048, 16384, 16, 16, False, True, True): (4, 64, 3, 2), + (2048, 2048, 16384, 16, 16, True, False, True): (1, 64, 3, 2), + (2048, 2048, 16384, 32, 32, False, True, True): (4, 64, 3, 4), + (2048, 2048, 16384, 32, 32, True, False, True): (4, 64, 3, 4), + (2048, 2048, 16384, 64, 64, False, True, True): (4, 128, 3, 4), + (2048, 2048, 16384, 64, 64, True, False, True): (4, 128, 3, 4), + (2048, 2048, 16384, 128, 128, False, True, True): (2, 128, 1, 4), + (2048, 2048, 16384, 128, 128, True, False, True): (2, 128, 1, 4), + (2048, 2048, 32768, 16, 16, False, True, True): (8, 128, 3, 2), + (2048, 2048, 32768, 16, 16, True, False, True): (8, 128, 3, 4), + (2048, 2048, 32768, 32, 32, False, True, True): (8, 128, 3, 4), + (2048, 2048, 32768, 32, 32, True, False, True): (8, 128, 3, 4), + (2048, 2048, 32768, 64, 64, False, True, True): (1, 128, 2, 4), + (2048, 2048, 32768, 64, 64, True, False, True): (8, 256, 3, 4), + (2048, 2048, 32768, 128, 128, False, True, True): (2, 256, 1, 4), + (2048, 2048, 32768, 128, 128, True, False, True): (2, 256, 1, 4), + (2048, 2048, 65536, 16, 16, False, True, True): (9, 256, 4, 4), + (2048, 2048, 65536, 16, 16, True, False, True): (7, 256, 4, 4), + (2048, 2048, 65536, 32, 32, False, True, True): (7, 256, 3, 4), + (2048, 2048, 65536, 32, 32, True, False, True): (3, 256, 3, 4), + (2048, 2048, 65536, 64, 64, False, True, True): (2, 256, 2, 4), + (2048, 2048, 65536, 64, 64, True, False, True): (6, 512, 3, 4), + (2048, 2048, 65536, 128, 128, False, True, True): (2, 512, 1, 4), + (2048, 2048, 65536, 128, 128, True, False, True): (2, 512, 1, 4), + (2048, 2048, 131072, 16, 16, False, True, True): (9, 512, 4, 4), + (2048, 2048, 131072, 16, 16, True, False, True): (9, 512, 4, 4), + (2048, 2048, 131072, 32, 32, False, True, True): (7, 512, 4, 4), + (2048, 2048, 131072, 32, 32, True, False, True): (3, 512, 3, 4), + (2048, 2048, 131072, 64, 64, False, True, True): (2, 512, 2, 4), + (2048, 2048, 131072, 64, 64, True, False, True): (4, 1024, 3, 4), + (2048, 2048, 131072, 128, 128, False, True, True): (1, 1024, 1, 4), + (2048, 2048, 131072, 128, 128, True, False, True): (2, 1024, 1, 4), + (3072, 768, 256, 16, 16, False, True, True): (6, 4, 1, 4), + (3072, 768, 256, 16, 16, True, False, True): (3, 1, 4, 4), + (3072, 768, 256, 32, 32, False, True, True): (6, 8, 1, 2), + (3072, 768, 256, 32, 32, True, False, True): (1, 2, 4, 4), + (3072, 768, 256, 64, 64, False, True, True): (1, 4, 4, 4), + (3072, 768, 256, 64, 64, True, False, True): (4, 2, 4, 4), + (3072, 768, 256, 128, 128, False, True, True): (1, 2, 3, 8), + (3072, 768, 256, 128, 128, True, False, True): (1, 2, 3, 8), + (3072, 768, 512, 16, 16, False, True, True): (2, 4, 1, 4), + (3072, 768, 512, 16, 16, True, False, True): (1, 4, 4, 1), + (3072, 768, 512, 32, 32, False, True, True): (3, 8, 1, 4), + (3072, 768, 512, 32, 32, True, False, True): (1, 2, 3, 4), + (3072, 768, 512, 64, 64, False, True, True): (1, 8, 1, 4), + (3072, 768, 512, 64, 64, True, False, True): (4, 4, 3, 4), + (3072, 768, 512, 128, 128, False, True, True): (1, 4, 3, 8), + (3072, 768, 512, 128, 128, True, False, True): (1, 4, 3, 8), + (3072, 768, 1024, 16, 16, False, True, True): (1, 8, 1, 4), + (3072, 768, 1024, 16, 16, True, False, True): (3, 4, 3, 1), + (3072, 768, 1024, 32, 32, False, True, True): (1, 8, 1, 8), + (3072, 768, 1024, 32, 32, True, False, True): (1, 4, 4, 4), + (3072, 768, 1024, 64, 64, False, True, True): (1, 16, 3, 4), + (3072, 768, 1024, 64, 64, True, False, True): (1, 4, 3, 4), + (3072, 768, 1024, 128, 128, False, True, True): (1, 8, 3, 8), + (3072, 768, 1024, 128, 128, True, False, True): (2, 8, 3, 8), + (3072, 768, 2048, 16, 16, False, True, True): (3, 8, 1, 4), + (3072, 768, 2048, 16, 16, True, False, True): (2, 8, 3, 4), + (3072, 768, 2048, 32, 32, False, True, True): (3, 16, 1, 8), + (3072, 768, 2048, 32, 32, True, False, True): (3, 8, 3, 4), + (3072, 768, 2048, 64, 64, False, True, True): (1, 16, 1, 4), + (3072, 768, 2048, 64, 64, True, False, True): (1, 16, 3, 4), + (3072, 768, 2048, 128, 128, False, True, True): (1, 16, 3, 8), + (3072, 768, 2048, 128, 128, True, False, True): (2, 16, 2, 4), + (3072, 768, 4096, 16, 16, False, True, True): (1, 16, 1, 4), + (3072, 768, 4096, 16, 16, True, False, True): (4, 32, 4, 2), + (3072, 768, 4096, 32, 32, False, True, True): (2, 32, 1, 8), + (3072, 768, 4096, 32, 32, True, False, True): (7, 16, 3, 4), + (3072, 768, 4096, 64, 64, False, True, True): (2, 32, 1, 4), + (3072, 768, 4096, 64, 64, True, False, True): (2, 16, 2, 4), + (3072, 768, 4096, 128, 128, False, True, True): (1, 32, 3, 8), + (3072, 768, 4096, 128, 128, True, False, True): (3, 32, 2, 4), + (3072, 768, 8192, 16, 16, False, True, True): (2, 32, 1, 4), + (3072, 768, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (3072, 768, 8192, 32, 32, False, True, True): (4, 32, 1, 4), + (3072, 768, 8192, 32, 32, True, False, True): (4, 32, 3, 4), + (3072, 768, 8192, 64, 64, False, True, True): (2, 64, 1, 4), + (3072, 768, 8192, 64, 64, True, False, True): (4, 32, 2, 4), + (3072, 768, 8192, 128, 128, False, True, True): (3, 64, 1, 4), + (3072, 768, 8192, 128, 128, True, False, True): (6, 64, 2, 4), + (3072, 768, 16384, 16, 16, False, True, True): (1, 64, 1, 4), + (3072, 768, 16384, 16, 16, True, False, True): (1, 64, 1, 1), + (3072, 768, 16384, 32, 32, False, True, True): (1, 64, 1, 4), + (3072, 768, 16384, 32, 32, True, False, True): (4, 64, 3, 4), + (3072, 768, 16384, 64, 64, False, True, True): (4, 128, 1, 4), + (3072, 768, 16384, 64, 64, True, False, True): (4, 64, 2, 4), + (3072, 768, 16384, 128, 128, False, True, True): (3, 128, 1, 4), + (3072, 768, 16384, 128, 128, True, False, True): (4, 128, 2, 4), + (3072, 768, 32768, 16, 16, False, True, True): (1, 128, 1, 4), + (3072, 768, 32768, 16, 16, True, False, True): (8, 128, 4, 1), + (3072, 768, 32768, 32, 32, False, True, True): (1, 128, 1, 4), + (3072, 768, 32768, 32, 32, True, False, True): (8, 128, 3, 4), + (3072, 768, 32768, 64, 64, False, True, True): (1, 256, 1, 4), + (3072, 768, 32768, 64, 64, True, False, True): (1, 128, 2, 4), + (3072, 768, 32768, 128, 128, False, True, True): (3, 256, 1, 4), + (3072, 768, 32768, 128, 128, True, False, True): (8, 256, 2, 4), + (3072, 768, 50432, 16, 16, False, True, True): (1, 197, 1, 4), + (3072, 768, 50432, 16, 16, True, False, True): (7, 197, 4, 1), + (3072, 768, 50432, 32, 32, False, True, True): (1, 197, 1, 4), + (3072, 768, 50432, 32, 32, True, False, True): (4, 197, 3, 4), + (3072, 768, 50432, 64, 64, False, True, True): (1, 394, 1, 4), + (3072, 768, 50432, 64, 64, True, False, True): (3, 197, 2, 4), + (3072, 768, 50432, 128, 128, False, True, True): (3, 394, 1, 4), + (3072, 768, 50432, 128, 128, True, False, True): (8, 394, 2, 4), + (3072, 768, 65536, 16, 16, False, True, True): (1, 256, 1, 4), + (3072, 768, 65536, 16, 16, True, False, True): (15, 256, 4, 1), + (3072, 768, 65536, 32, 32, False, True, True): (1, 256, 1, 4), + (3072, 768, 65536, 32, 32, True, False, True): (15, 256, 3, 4), + (3072, 768, 65536, 64, 64, False, True, True): (1, 512, 1, 4), + (3072, 768, 65536, 64, 64, True, False, True): (2, 256, 2, 4), + (3072, 768, 65536, 128, 128, False, True, True): (3, 512, 1, 4), + (3072, 768, 65536, 128, 128, True, False, True): (3, 512, 2, 4), + (3072, 768, 131072, 16, 16, False, True, True): (1, 512, 1, 4), + (3072, 768, 131072, 16, 16, True, False, True): (15, 512, 4, 1), + (3072, 768, 131072, 32, 32, False, True, True): (1, 512, 1, 4), + (3072, 768, 131072, 32, 32, True, False, True): (9, 512, 3, 4), + (3072, 768, 131072, 64, 64, False, True, True): (1, 1024, 1, 4), + (3072, 768, 131072, 64, 64, True, False, True): (3, 512, 2, 4), + (3072, 768, 131072, 128, 128, False, True, True): (3, 1024, 1, 4), + (3072, 768, 131072, 128, 128, True, False, True): (1, 1024, 2, 4), + (3072, 3072, 256, 16, 16, False, True, True): (5, 4, 1, 4), + (3072, 3072, 256, 16, 16, True, False, True): (1, 2, 5, 2), + (3072, 3072, 256, 32, 32, False, True, True): (5, 4, 1, 8), + (3072, 3072, 256, 32, 32, True, False, True): (1, 4, 4, 2), + (3072, 3072, 256, 64, 64, False, True, True): (2, 4, 4, 4), + (3072, 3072, 256, 64, 64, True, False, True): (2, 4, 4, 4), + (3072, 3072, 256, 128, 128, False, True, True): (1, 2, 3, 8), + (3072, 3072, 256, 128, 128, True, False, True): (1, 2, 3, 8), + (3072, 3072, 512, 16, 16, False, True, True): (5, 4, 1, 2), + (3072, 3072, 512, 16, 16, True, False, True): (1, 2, 3, 4), + (3072, 3072, 512, 32, 32, False, True, True): (3, 8, 1, 4), + (3072, 3072, 512, 32, 32, True, False, True): (1, 4, 4, 2), + (3072, 3072, 512, 64, 64, False, True, True): (1, 8, 2, 2), + (3072, 3072, 512, 64, 64, True, False, True): (2, 4, 3, 4), + (3072, 3072, 512, 128, 128, False, True, True): (2, 4, 3, 8), + (3072, 3072, 512, 128, 128, True, False, True): (1, 4, 3, 8), + (3072, 3072, 1024, 16, 16, False, True, True): (1, 8, 1, 4), + (3072, 3072, 1024, 16, 16, True, False, True): (2, 8, 3, 1), + (3072, 3072, 1024, 32, 32, False, True, True): (1, 16, 1, 4), + (3072, 3072, 1024, 32, 32, True, False, True): (1, 4, 4, 4), + (3072, 3072, 1024, 64, 64, False, True, True): (1, 8, 3, 4), + (3072, 3072, 1024, 64, 64, True, False, True): (2, 4, 3, 4), + (3072, 3072, 1024, 128, 128, False, True, True): (1, 8, 1, 4), + (3072, 3072, 1024, 128, 128, True, False, True): (2, 8, 3, 8), + (3072, 3072, 2048, 16, 16, False, True, True): (1, 16, 1, 2), + (3072, 3072, 2048, 16, 16, True, False, True): (2, 16, 4, 2), + (3072, 3072, 2048, 32, 32, False, True, True): (1, 16, 1, 8), + (3072, 3072, 2048, 32, 32, True, False, True): (3, 8, 4, 4), + (3072, 3072, 2048, 64, 64, False, True, True): (3, 16, 3, 4), + (3072, 3072, 2048, 64, 64, True, False, True): (3, 8, 3, 4), + (3072, 3072, 2048, 128, 128, False, True, True): (1, 16, 3, 8), + (3072, 3072, 2048, 128, 128, True, False, True): (5, 16, 3, 8), + (3072, 3072, 4096, 16, 16, False, True, True): (1, 32, 1, 2), + (3072, 3072, 4096, 16, 16, True, False, True): (4, 32, 4, 2), + (3072, 3072, 4096, 32, 32, False, True, True): (1, 32, 1, 8), + (3072, 3072, 4096, 32, 32, True, False, True): (3, 16, 3, 4), + (3072, 3072, 4096, 64, 64, False, True, True): (1, 32, 3, 4), + (3072, 3072, 4096, 64, 64, True, False, True): (3, 16, 3, 4), + (3072, 3072, 4096, 128, 128, False, True, True): (3, 32, 3, 8), + (3072, 3072, 4096, 128, 128, True, False, True): (3, 32, 3, 8), + (3072, 3072, 8192, 16, 16, False, True, True): (1, 64, 1, 2), + (3072, 3072, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (3072, 3072, 8192, 32, 32, False, True, True): (1, 64, 1, 8), + (3072, 3072, 8192, 32, 32, True, False, True): (6, 32, 3, 4), + (3072, 3072, 8192, 64, 64, False, True, True): (1, 64, 3, 4), + (3072, 3072, 8192, 64, 64, True, False, True): (2, 32, 3, 4), + (3072, 3072, 8192, 128, 128, False, True, True): (2, 64, 3, 8), + (3072, 3072, 8192, 128, 128, True, False, True): (1, 64, 3, 8), + (3072, 3072, 16384, 16, 16, False, True, True): (1, 128, 1, 2), + (3072, 3072, 16384, 16, 16, True, False, True): (4, 128, 4, 2), + (3072, 3072, 16384, 32, 32, False, True, True): (1, 64, 1, 2), + (3072, 3072, 16384, 32, 32, True, False, True): (4, 64, 3, 4), + (3072, 3072, 16384, 64, 64, False, True, True): (1, 128, 3, 4), + (3072, 3072, 16384, 64, 64, True, False, True): (4, 64, 3, 4), + (3072, 3072, 16384, 128, 128, False, True, True): (1, 128, 1, 4), + (3072, 3072, 16384, 128, 128, True, False, True): (1, 128, 3, 8), + (3072, 3072, 32768, 16, 16, False, True, True): (1, 256, 1, 2), + (3072, 3072, 32768, 16, 16, True, False, True): (8, 128, 4, 4), + (3072, 3072, 32768, 32, 32, False, True, True): (1, 256, 1, 8), + (3072, 3072, 32768, 32, 32, True, False, True): (5, 128, 3, 4), + (3072, 3072, 32768, 64, 64, False, True, True): (1, 256, 3, 4), + (3072, 3072, 32768, 64, 64, True, False, True): (3, 128, 3, 4), + (3072, 3072, 32768, 128, 128, False, True, True): (1, 256, 1, 4), + (3072, 3072, 32768, 128, 128, True, False, True): (3, 256, 2, 4), + (3072, 3072, 65536, 16, 16, False, True, True): (1, 512, 1, 2), + (3072, 3072, 65536, 16, 16, True, False, True): (7, 256, 4, 4), + (3072, 3072, 65536, 32, 32, False, True, True): (1, 256, 1, 2), + (3072, 3072, 65536, 32, 32, True, False, True): (5, 256, 3, 4), + (3072, 3072, 65536, 64, 64, False, True, True): (1, 512, 3, 4), + (3072, 3072, 65536, 64, 64, True, False, True): (3, 256, 3, 4), + (3072, 3072, 65536, 128, 128, False, True, True): (1, 512, 1, 4), + (3072, 3072, 65536, 128, 128, True, False, True): (3, 512, 2, 4), + (3072, 3072, 131072, 16, 16, False, True, True): (1, 1024, 1, 2), + (3072, 3072, 131072, 16, 16, True, False, True): (5, 512, 4, 4), + (3072, 3072, 131072, 32, 32, False, True, True): (1, 512, 1, 2), + (3072, 3072, 131072, 32, 32, True, False, True): (5, 512, 3, 4), + (3072, 3072, 131072, 64, 64, False, True, True): (1, 1024, 3, 4), + (3072, 3072, 131072, 64, 64, True, False, True): (3, 512, 3, 4), + (3072, 3072, 131072, 128, 128, False, True, True): (1, 1024, 1, 4), + (3072, 3072, 131072, 128, 128, True, False, True): (6, 1024, 2, 4), + (4096, 4096, 256, 16, 16, False, True, True): (2, 2, 5, 4), + (4096, 4096, 256, 16, 16, True, False, True): (2, 2, 4, 2), + (4096, 4096, 256, 32, 32, False, True, True): (1, 2, 4, 4), + (4096, 4096, 256, 32, 32, True, False, True): (3, 2, 4, 2), + (4096, 4096, 256, 64, 64, False, True, True): (3, 4, 3, 4), + (4096, 4096, 256, 64, 64, True, False, True): (1, 4, 3, 2), + (4096, 4096, 256, 128, 128, False, True, True): (1, 2, 2, 8), + (4096, 4096, 256, 128, 128, True, False, True): (1, 2, 2, 8), + (4096, 4096, 512, 16, 16, False, True, True): (4, 2, 3, 4), + (4096, 4096, 512, 16, 16, True, False, True): (1, 2, 3, 4), + (4096, 4096, 512, 32, 32, False, True, True): (1, 4, 3, 4), + (4096, 4096, 512, 32, 32, True, False, True): (3, 4, 3, 2), + (4096, 4096, 512, 64, 64, False, True, True): (4, 4, 4, 4), + (4096, 4096, 512, 64, 64, True, False, True): (3, 4, 3, 4), + (4096, 4096, 512, 128, 128, False, True, True): (2, 4, 2, 8), + (4096, 4096, 512, 128, 128, True, False, True): (2, 4, 1, 4), + (4096, 4096, 1024, 16, 16, False, True, True): (2, 8, 3, 2), + (4096, 4096, 1024, 16, 16, True, False, True): (2, 8, 3, 2), + (4096, 4096, 1024, 32, 32, False, True, True): (1, 8, 3, 4), + (4096, 4096, 1024, 32, 32, True, False, True): (1, 8, 3, 2), + (4096, 4096, 1024, 64, 64, False, True, True): (1, 8, 3, 4), + (4096, 4096, 1024, 64, 64, True, False, True): (1, 8, 3, 4), + (4096, 4096, 1024, 128, 128, False, True, True): (4, 8, 1, 4), + (4096, 4096, 1024, 128, 128, True, False, True): (2, 8, 2, 8), + (4096, 4096, 2048, 16, 16, False, True, True): (2, 8, 4, 4), + (4096, 4096, 2048, 16, 16, True, False, True): (2, 8, 4, 4), + (4096, 4096, 2048, 32, 32, False, True, True): (4, 8, 3, 8), + (4096, 4096, 2048, 32, 32, True, False, True): (4, 8, 4, 8), + (4096, 4096, 2048, 64, 64, False, True, True): (4, 16, 3, 4), + (4096, 4096, 2048, 64, 64, True, False, True): (4, 16, 3, 4), + (4096, 4096, 2048, 128, 128, False, True, True): (1, 16, 1, 4), + (4096, 4096, 2048, 128, 128, True, False, True): (4, 16, 1, 4), + (4096, 4096, 4096, 16, 16, False, True, True): (4, 32, 4, 4), + (4096, 4096, 4096, 16, 16, True, False, True): (2, 32, 4, 4), + (4096, 4096, 4096, 32, 32, False, True, True): (4, 16, 4, 8), + (4096, 4096, 4096, 32, 32, True, False, True): (4, 16, 4, 8), + (4096, 4096, 4096, 64, 64, False, True, True): (4, 32, 3, 4), + (4096, 4096, 4096, 64, 64, True, False, True): (2, 32, 3, 4), + (4096, 4096, 4096, 128, 128, False, True, True): (2, 32, 1, 4), + (4096, 4096, 4096, 128, 128, True, False, True): (2, 32, 1, 4), + (4096, 4096, 8192, 16, 16, False, True, True): (4, 64, 4, 2), + (4096, 4096, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (4096, 4096, 8192, 32, 32, False, True, True): (4, 32, 4, 8), + (4096, 4096, 8192, 32, 32, True, False, True): (4, 32, 4, 8), + (4096, 4096, 8192, 64, 64, False, True, True): (4, 64, 3, 4), + (4096, 4096, 8192, 64, 64, True, False, True): (4, 64, 3, 4), + (4096, 4096, 8192, 128, 128, False, True, True): (1, 64, 1, 4), + (4096, 4096, 8192, 128, 128, True, False, True): (1, 64, 1, 4), + (4096, 4096, 16384, 16, 16, False, True, True): (4, 64, 4, 4), + (4096, 4096, 16384, 16, 16, True, False, True): (4, 64, 4, 4), + (4096, 4096, 16384, 32, 32, False, True, True): (4, 64, 4, 8), + (4096, 4096, 16384, 32, 32, True, False, True): (4, 64, 4, 8), + (4096, 4096, 16384, 64, 64, False, True, True): (4, 128, 3, 4), + (4096, 4096, 16384, 64, 64, True, False, True): (4, 128, 3, 4), + (4096, 4096, 16384, 128, 128, False, True, True): (1, 128, 1, 4), + (4096, 4096, 16384, 128, 128, True, False, True): (1, 128, 1, 4), + (4096, 4096, 32768, 16, 16, False, True, True): (8, 128, 4, 4), + (4096, 4096, 32768, 16, 16, True, False, True): (5, 128, 4, 4), + (4096, 4096, 32768, 32, 32, False, True, True): (5, 128, 4, 4), + (4096, 4096, 32768, 32, 32, True, False, True): (3, 128, 4, 8), + (4096, 4096, 32768, 64, 64, False, True, True): (3, 256, 3, 4), + (4096, 4096, 32768, 64, 64, True, False, True): (2, 256, 3, 4), + (4096, 4096, 32768, 128, 128, False, True, True): (1, 256, 1, 4), + (4096, 4096, 32768, 128, 128, True, False, True): (1, 256, 1, 4), + (4096, 4096, 65536, 16, 16, False, True, True): (5, 256, 4, 4), + (4096, 4096, 65536, 16, 16, True, False, True): (5, 256, 4, 4), + (4096, 4096, 65536, 32, 32, False, True, True): (4, 256, 4, 8), + (4096, 4096, 65536, 32, 32, True, False, True): (4, 256, 4, 8), + (4096, 4096, 65536, 64, 64, False, True, True): (1, 512, 3, 4), + (4096, 4096, 65536, 64, 64, True, False, True): (3, 512, 3, 4), + (4096, 4096, 65536, 128, 128, False, True, True): (1, 512, 1, 4), + (4096, 4096, 65536, 128, 128, True, False, True): (1, 512, 1, 4), + (4096, 4096, 131072, 16, 16, False, True, True): (5, 512, 4, 4), + (4096, 4096, 131072, 16, 16, True, False, True): (5, 512, 4, 4), + (4096, 4096, 131072, 32, 32, False, True, True): (4, 512, 4, 4), + (4096, 4096, 131072, 32, 32, True, False, True): (2, 512, 3, 4), + (4096, 4096, 131072, 64, 64, False, True, True): (1, 1024, 3, 4), + (4096, 4096, 131072, 64, 64, True, False, True): (3, 1024, 3, 4), + (4096, 4096, 131072, 128, 128, False, True, True): (1, 1024, 1, 4), + (4096, 4096, 131072, 128, 128, True, False, True): (1, 1024, 1, 4), + (5120, 1280, 65792, 16, 16, False, True, True): (1, 257, 1, 4), + (5120, 1280, 65792, 16, 16, True, False, True): (11, 257, 4, 1), + (5120, 1280, 65792, 32, 32, False, True, True): (1, 257, 1, 4), + (5120, 1280, 65792, 32, 32, True, False, True): (5, 257, 3, 4), + (5120, 1280, 65792, 64, 64, False, True, True): (1, 514, 1, 4), + (5120, 1280, 65792, 64, 64, True, False, True): (5, 257, 2, 4), + (5120, 1280, 65792, 128, 128, False, True, True): (3, 514, 1, 4), + (5120, 1280, 65792, 128, 128, True, False, True): (7, 514, 2, 4), + (6144, 6144, 256, 16, 16, False, True, True): (1, 2, 1, 4), + (6144, 6144, 256, 16, 16, True, False, True): (3, 1, 4, 4), + (6144, 6144, 256, 32, 32, False, True, True): (3, 2, 1, 8), + (6144, 6144, 256, 32, 32, True, False, True): (1, 1, 4, 4), + (6144, 6144, 256, 64, 64, False, True, True): (4, 2, 3, 4), + (6144, 6144, 256, 64, 64, True, False, True): (3, 2, 4, 4), + (6144, 6144, 256, 128, 128, False, True, True): (2, 2, 3, 8), + (6144, 6144, 256, 128, 128, True, False, True): (1, 2, 3, 8), + (6144, 6144, 512, 16, 16, False, True, True): (4, 4, 1, 4), + (6144, 6144, 512, 16, 16, True, False, True): (3, 2, 3, 1), + (6144, 6144, 512, 32, 32, False, True, True): (1, 8, 1, 4), + (6144, 6144, 512, 32, 32, True, False, True): (1, 2, 3, 2), + (6144, 6144, 512, 64, 64, False, True, True): (2, 4, 3, 4), + (6144, 6144, 512, 64, 64, True, False, True): (2, 2, 3, 4), + (6144, 6144, 512, 128, 128, False, True, True): (1, 4, 3, 8), + (6144, 6144, 512, 128, 128, True, False, True): (1, 4, 3, 8), + (6144, 6144, 1024, 16, 16, False, True, True): (1, 8, 1, 2), + (6144, 6144, 1024, 16, 16, True, False, True): (4, 8, 4, 4), + (6144, 6144, 1024, 32, 32, False, True, True): (1, 8, 4, 2), + (6144, 6144, 1024, 32, 32, True, False, True): (1, 8, 4, 2), + (6144, 6144, 1024, 64, 64, False, True, True): (4, 8, 3, 4), + (6144, 6144, 1024, 64, 64, True, False, True): (1, 4, 3, 4), + (6144, 6144, 1024, 128, 128, False, True, True): (2, 8, 3, 8), + (6144, 6144, 1024, 128, 128, True, False, True): (1, 8, 3, 8), + (6144, 6144, 2048, 16, 16, False, True, True): (4, 4, 1, 4), + (6144, 6144, 2048, 16, 16, True, False, True): (2, 8, 4, 4), + (6144, 6144, 2048, 32, 32, False, True, True): (1, 16, 4, 2), + (6144, 6144, 2048, 32, 32, True, False, True): (4, 8, 4, 8), + (6144, 6144, 2048, 64, 64, False, True, True): (4, 16, 3, 4), + (6144, 6144, 2048, 64, 64, True, False, True): (2, 8, 3, 4), + (6144, 6144, 2048, 128, 128, False, True, True): (1, 16, 3, 8), + (6144, 6144, 2048, 128, 128, True, False, True): (4, 16, 3, 8), + (6144, 6144, 4096, 16, 16, False, True, True): (4, 8, 1, 4), + (6144, 6144, 4096, 16, 16, True, False, True): (4, 32, 4, 2), + (6144, 6144, 4096, 32, 32, False, True, True): (4, 16, 1, 2), + (6144, 6144, 4096, 32, 32, True, False, True): (2, 8, 3, 8), + (6144, 6144, 4096, 64, 64, False, True, True): (4, 32, 3, 4), + (6144, 6144, 4096, 64, 64, True, False, True): (4, 16, 3, 4), + (6144, 6144, 4096, 128, 128, False, True, True): (4, 32, 3, 8), + (6144, 6144, 4096, 128, 128, True, False, True): (4, 32, 3, 8), + (6144, 6144, 8192, 16, 16, False, True, True): (2, 16, 1, 2), + (6144, 6144, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (6144, 6144, 8192, 32, 32, False, True, True): (4, 32, 1, 2), + (6144, 6144, 8192, 32, 32, True, False, True): (4, 32, 4, 8), + (6144, 6144, 8192, 64, 64, False, True, True): (4, 64, 3, 4), + (6144, 6144, 8192, 64, 64, True, False, True): (4, 32, 3, 4), + (6144, 6144, 8192, 128, 128, False, True, True): (4, 64, 3, 8), + (6144, 6144, 8192, 128, 128, True, False, True): (4, 64, 3, 8), + (6144, 6144, 16384, 16, 16, False, True, True): (2, 32, 1, 2), + (6144, 6144, 16384, 16, 16, True, False, True): (4, 64, 4, 4), + (6144, 6144, 16384, 32, 32, False, True, True): (4, 64, 1, 2), + (6144, 6144, 16384, 32, 32, True, False, True): (4, 64, 3, 2), + (6144, 6144, 16384, 64, 64, False, True, True): (4, 128, 3, 4), + (6144, 6144, 16384, 64, 64, True, False, True): (2, 32, 3, 8), + (6144, 6144, 16384, 128, 128, False, True, True): (4, 128, 3, 8), + (6144, 6144, 16384, 128, 128, True, False, True): (4, 128, 3, 8), + (6144, 6144, 32768, 16, 16, False, True, True): (2, 64, 1, 2), + (6144, 6144, 32768, 16, 16, True, False, True): (3, 128, 4, 4), + (6144, 6144, 32768, 32, 32, False, True, True): (4, 128, 1, 2), + (6144, 6144, 32768, 32, 32, True, False, True): (3, 128, 3, 4), + (6144, 6144, 32768, 64, 64, False, True, True): (4, 256, 3, 4), + (6144, 6144, 32768, 64, 64, True, False, True): (2, 64, 3, 8), + (6144, 6144, 32768, 128, 128, False, True, True): (4, 256, 3, 8), + (6144, 6144, 32768, 128, 128, True, False, True): (4, 256, 3, 8), + (6144, 6144, 65536, 16, 16, False, True, True): (2, 128, 1, 2), + (6144, 6144, 65536, 16, 16, True, False, True): (4, 256, 4, 4), + (6144, 6144, 65536, 32, 32, False, True, True): (4, 256, 1, 2), + (6144, 6144, 65536, 32, 32, True, False, True): (4, 256, 3, 4), + (6144, 6144, 65536, 64, 64, False, True, True): (4, 512, 3, 4), + (6144, 6144, 65536, 64, 64, True, False, True): (2, 128, 3, 8), + (6144, 6144, 65536, 128, 128, False, True, True): (4, 512, 3, 8), + (6144, 6144, 65536, 128, 128, True, False, True): (4, 512, 3, 8), + (6144, 6144, 131072, 16, 16, False, True, True): (2, 256, 1, 2), + (6144, 6144, 131072, 16, 16, True, False, True): (5, 512, 4, 1), + (6144, 6144, 131072, 32, 32, False, True, True): (4, 512, 1, 2), + (6144, 6144, 131072, 32, 32, True, False, True): (4, 512, 3, 2), + (6144, 6144, 131072, 64, 64, False, True, True): (4, 1024, 3, 4), + (6144, 6144, 131072, 64, 64, True, False, True): (2, 256, 3, 8), + (6144, 6144, 131072, 128, 128, False, True, True): (4, 1024, 3, 8), + (6144, 6144, 131072, 128, 128, True, False, True): (4, 1024, 3, 8), + (8192, 8192, 256, 16, 16, False, True, True): (1, 1, 3, 4), + (8192, 8192, 256, 16, 16, True, False, True): (4, 1, 3, 4), + (8192, 8192, 256, 32, 32, False, True, True): (1, 2, 3, 4), + (8192, 8192, 256, 32, 32, True, False, True): (1, 2, 3, 4), + (8192, 8192, 256, 64, 64, False, True, True): (6, 2, 3, 8), + (8192, 8192, 256, 64, 64, True, False, True): (4, 2, 3, 8), + (8192, 8192, 256, 128, 128, False, True, True): (1, 2, 1, 4), + (8192, 8192, 256, 128, 128, True, False, True): (1, 2, 1, 4), + (8192, 8192, 512, 16, 16, False, True, True): (4, 4, 3, 2), + (8192, 8192, 512, 16, 16, True, False, True): (4, 4, 3, 4), + (8192, 8192, 512, 32, 32, False, True, True): (1, 4, 3, 4), + (8192, 8192, 512, 32, 32, True, False, True): (3, 4, 3, 2), + (8192, 8192, 512, 64, 64, False, True, True): (1, 4, 3, 4), + (8192, 8192, 512, 64, 64, True, False, True): (1, 4, 3, 4), + (8192, 8192, 512, 128, 128, False, True, True): (4, 4, 2, 8), + (8192, 8192, 512, 128, 128, True, False, True): (4, 4, 2, 8), + (8192, 8192, 1024, 16, 16, False, True, True): (4, 8, 4, 4), + (8192, 8192, 1024, 16, 16, True, False, True): (2, 8, 4, 4), + (8192, 8192, 1024, 32, 32, False, True, True): (2, 4, 4, 8), + (8192, 8192, 1024, 32, 32, True, False, True): (1, 4, 3, 4), + (8192, 8192, 1024, 64, 64, False, True, True): (4, 8, 3, 4), + (8192, 8192, 1024, 64, 64, True, False, True): (2, 8, 3, 4), + (8192, 8192, 1024, 128, 128, False, True, True): (4, 8, 1, 4), + (8192, 8192, 1024, 128, 128, True, False, True): (4, 8, 1, 4), + (8192, 8192, 2048, 16, 16, False, True, True): (2, 8, 4, 4), + (8192, 8192, 2048, 16, 16, True, False, True): (2, 8, 4, 4), + (8192, 8192, 2048, 32, 32, False, True, True): (2, 8, 4, 8), + (8192, 8192, 2048, 32, 32, True, False, True): (2, 8, 4, 8), + (8192, 8192, 2048, 64, 64, False, True, True): (4, 8, 2, 4), + (8192, 8192, 2048, 64, 64, True, False, True): (4, 16, 3, 4), + (8192, 8192, 2048, 128, 128, False, True, True): (4, 16, 1, 4), + (8192, 8192, 2048, 128, 128, True, False, True): (4, 16, 1, 4), + (8192, 8192, 4096, 16, 16, False, True, True): (4, 16, 4, 4), + (8192, 8192, 4096, 16, 16, True, False, True): (4, 32, 4, 2), + (8192, 8192, 4096, 32, 32, False, True, True): (2, 16, 4, 8), + (8192, 8192, 4096, 32, 32, True, False, True): (2, 16, 4, 8), + (8192, 8192, 4096, 64, 64, False, True, True): (4, 32, 3, 4), + (8192, 8192, 4096, 64, 64, True, False, True): (4, 16, 2, 4), + (8192, 8192, 4096, 128, 128, False, True, True): (4, 32, 1, 4), + (8192, 8192, 4096, 128, 128, True, False, True): (4, 32, 1, 4), + (8192, 8192, 8192, 16, 16, False, True, True): (4, 64, 4, 2), + (8192, 8192, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (8192, 8192, 8192, 32, 32, False, True, True): (2, 32, 4, 8), + (8192, 8192, 8192, 32, 32, True, False, True): (2, 32, 4, 8), + (8192, 8192, 8192, 64, 64, False, True, True): (4, 32, 3, 8), + (8192, 8192, 8192, 64, 64, True, False, True): (4, 32, 2, 4), + (8192, 8192, 8192, 128, 128, False, True, True): (4, 64, 1, 4), + (8192, 8192, 8192, 128, 128, True, False, True): (4, 64, 1, 4), + (8192, 8192, 16384, 16, 16, False, True, True): (4, 64, 4, 4), + (8192, 8192, 16384, 16, 16, True, False, True): (4, 64, 4, 4), + (8192, 8192, 16384, 32, 32, False, True, True): (4, 64, 3, 4), + (8192, 8192, 16384, 32, 32, True, False, True): (4, 64, 4, 8), + (8192, 8192, 16384, 64, 64, False, True, True): (4, 64, 2, 4), + (8192, 8192, 16384, 64, 64, True, False, True): (4, 64, 2, 4), + (8192, 8192, 16384, 128, 128, False, True, True): (4, 128, 1, 4), + (8192, 8192, 16384, 128, 128, True, False, True): (4, 128, 1, 4), + (8192, 8192, 32768, 16, 16, False, True, True): (3, 128, 4, 4), + (8192, 8192, 32768, 16, 16, True, False, True): (3, 128, 4, 4), + (8192, 8192, 32768, 32, 32, False, True, True): (2, 128, 4, 8), + (8192, 8192, 32768, 32, 32, True, False, True): (2, 128, 4, 8), + (8192, 8192, 32768, 64, 64, False, True, True): (2, 128, 2, 4), + (8192, 8192, 32768, 64, 64, True, False, True): (2, 128, 2, 4), + (8192, 8192, 32768, 128, 128, False, True, True): (4, 256, 1, 4), + (8192, 8192, 32768, 128, 128, True, False, True): (4, 256, 1, 4), + (8192, 8192, 65536, 16, 16, False, True, True): (3, 256, 4, 4), + (8192, 8192, 65536, 16, 16, True, False, True): (3, 256, 4, 4), + (8192, 8192, 65536, 32, 32, False, True, True): (2, 256, 3, 4), + (8192, 8192, 65536, 32, 32, True, False, True): (2, 256, 3, 4), + (8192, 8192, 65536, 64, 64, False, True, True): (2, 256, 2, 4), + (8192, 8192, 65536, 64, 64, True, False, True): (2, 256, 3, 8), + (8192, 8192, 65536, 128, 128, False, True, True): (4, 512, 1, 4), + (8192, 8192, 65536, 128, 128, True, False, True): (4, 512, 1, 4), + (8192, 8192, 131072, 16, 16, False, True, True): (3, 512, 4, 4), + (8192, 8192, 131072, 16, 16, True, False, True): (3, 512, 4, 4), + (8192, 8192, 131072, 32, 32, False, True, True): (2, 512, 4, 4), + (8192, 8192, 131072, 32, 32, True, False, True): (2, 512, 3, 4), + (8192, 8192, 131072, 64, 64, False, True, True): (4, 512, 2, 4), + (8192, 8192, 131072, 64, 64, True, False, True): (2, 512, 2, 4), + (8192, 8192, 131072, 128, 128, False, True, True): (4, 1024, 1, 4), + (8192, 8192, 131072, 128, 128, True, False, True): (4, 1024, 1, 4), + (16384, 16384, 256, 16, 16, False, True, True): (2, 2, 6, 4), + (16384, 16384, 256, 16, 16, True, False, True): (2, 2, 6, 4), + (16384, 16384, 256, 32, 32, False, True, True): (4, 2, 3, 2), + (16384, 16384, 256, 32, 32, True, False, True): (4, 2, 3, 2), + (16384, 16384, 256, 64, 64, False, True, True): (2, 2, 4, 4), + (16384, 16384, 256, 64, 64, True, False, True): (4, 2, 3, 8), + (16384, 16384, 256, 128, 128, False, True, True): (4, 2, 2, 8), + (16384, 16384, 256, 128, 128, True, False, True): (4, 2, 2, 8), + (16384, 16384, 512, 16, 16, False, True, True): (1, 2, 4, 4), + (16384, 16384, 512, 16, 16, True, False, True): (1, 2, 4, 4), + (16384, 16384, 512, 32, 32, False, True, True): (2, 2, 4, 8), + (16384, 16384, 512, 32, 32, True, False, True): (2, 2, 4, 8), + (16384, 16384, 512, 64, 64, False, True, True): (4, 4, 3, 4), + (16384, 16384, 512, 64, 64, True, False, True): (4, 4, 3, 4), + (16384, 16384, 512, 128, 128, False, True, True): (4, 4, 2, 8), + (16384, 16384, 512, 128, 128, True, False, True): (4, 4, 2, 8), + (16384, 16384, 1024, 16, 16, False, True, True): (3, 4, 4, 4), + (16384, 16384, 1024, 16, 16, True, False, True): (2, 8, 4, 4), + (16384, 16384, 1024, 32, 32, False, True, True): (2, 4, 4, 8), + (16384, 16384, 1024, 32, 32, True, False, True): (1, 4, 4, 8), + (16384, 16384, 1024, 64, 64, False, True, True): (2, 8, 3, 4), + (16384, 16384, 1024, 64, 64, True, False, True): (2, 8, 3, 4), + (16384, 16384, 1024, 128, 128, False, True, True): (4, 8, 1, 4), + (16384, 16384, 1024, 128, 128, True, False, True): (4, 8, 1, 4), + (16384, 16384, 2048, 16, 16, False, True, True): (2, 8, 4, 4), + (16384, 16384, 2048, 16, 16, True, False, True): (2, 8, 4, 4), + (16384, 16384, 2048, 32, 32, False, True, True): (1, 8, 4, 8), + (16384, 16384, 2048, 32, 32, True, False, True): (2, 8, 4, 8), + (16384, 16384, 2048, 64, 64, False, True, True): (2, 8, 2, 4), + (16384, 16384, 2048, 64, 64, True, False, True): (2, 8, 2, 4), + (16384, 16384, 2048, 128, 128, False, True, True): (4, 16, 1, 4), + (16384, 16384, 2048, 128, 128, True, False, True): (4, 16, 1, 4), + (16384, 16384, 4096, 16, 16, False, True, True): (2, 16, 4, 4), + (16384, 16384, 4096, 16, 16, True, False, True): (2, 16, 4, 4), + (16384, 16384, 4096, 32, 32, False, True, True): (1, 8, 3, 8), + (16384, 16384, 4096, 32, 32, True, False, True): (2, 16, 3, 4), + (16384, 16384, 4096, 64, 64, False, True, True): (2, 16, 2, 4), + (16384, 16384, 4096, 64, 64, True, False, True): (2, 16, 2, 4), + (16384, 16384, 4096, 128, 128, False, True, True): (4, 32, 1, 4), + (16384, 16384, 4096, 128, 128, True, False, True): (4, 32, 1, 4), + (16384, 16384, 8192, 16, 16, False, True, True): (4, 64, 4, 2), + (16384, 16384, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (16384, 16384, 8192, 32, 32, False, True, True): (2, 32, 4, 8), + (16384, 16384, 8192, 32, 32, True, False, True): (2, 32, 3, 4), + (16384, 16384, 8192, 64, 64, False, True, True): (2, 32, 4, 8), + (16384, 16384, 8192, 64, 64, True, False, True): (2, 32, 3, 8), + (16384, 16384, 8192, 128, 128, False, True, True): (4, 64, 1, 4), + (16384, 16384, 8192, 128, 128, True, False, True): (4, 64, 1, 4), + (16384, 16384, 16384, 16, 16, False, True, True): (1, 64, 4, 4), + (16384, 16384, 16384, 16, 16, True, False, True): (1, 64, 4, 4), + (16384, 16384, 16384, 32, 32, False, True, True): (1, 64, 3, 8), + (16384, 16384, 16384, 32, 32, True, False, True): (1, 64, 3, 4), + (16384, 16384, 16384, 64, 64, False, True, True): (1, 64, 2, 4), + (16384, 16384, 16384, 64, 64, True, False, True): (1, 64, 4, 8), + (16384, 16384, 16384, 128, 128, False, True, True): (4, 128, 1, 4), + (16384, 16384, 16384, 128, 128, True, False, True): (4, 128, 1, 4), + (16384, 16384, 32768, 16, 16, False, True, True): (1, 128, 4, 4), + (16384, 16384, 32768, 16, 16, True, False, True): (1, 128, 4, 4), + (16384, 16384, 32768, 32, 32, False, True, True): (1, 128, 4, 2), + (16384, 16384, 32768, 32, 32, True, False, True): (1, 128, 3, 8), + (16384, 16384, 32768, 64, 64, False, True, True): (2, 128, 2, 4), + (16384, 16384, 32768, 64, 64, True, False, True): (1, 128, 3, 8), + (16384, 16384, 32768, 128, 128, False, True, True): (4, 256, 1, 4), + (16384, 16384, 32768, 128, 128, True, False, True): (4, 256, 1, 4), + (16384, 16384, 65536, 16, 16, False, True, True): (1, 256, 4, 4), + (16384, 16384, 65536, 16, 16, True, False, True): (1, 256, 4, 4), + (16384, 16384, 65536, 32, 32, False, True, True): (1, 256, 3, 4), + (16384, 16384, 65536, 32, 32, True, False, True): (1, 256, 3, 4), + (16384, 16384, 65536, 64, 64, False, True, True): (1, 256, 2, 4), + (16384, 16384, 65536, 64, 64, True, False, True): (2, 256, 2, 4), + (16384, 16384, 65536, 128, 128, False, True, True): (4, 512, 1, 4), + (16384, 16384, 65536, 128, 128, True, False, True): (4, 512, 1, 4), + (16384, 16384, 131072, 16, 16, False, True, True): (2, 512, 4, 4), + (16384, 16384, 131072, 16, 16, True, False, True): (1, 512, 4, 4), + (16384, 16384, 131072, 32, 32, False, True, True): (1, 512, 4, 8), + (16384, 16384, 131072, 32, 32, True, False, True): (1, 512, 3, 4), + (16384, 16384, 131072, 64, 64, False, True, True): (2, 512, 2, 4), + (16384, 16384, 131072, 64, 64, True, False, True): (1, 512, 2, 4), + (16384, 16384, 131072, 128, 128, False, True, True): (4, 1024, 1, 4), + (16384, 16384, 131072, 128, 128, True, False, True): (4, 1024, 1, 4), + }, + ("bsr_dense_addmm", "NVIDIA A100-SXM4-80GB", (0, torch.bfloat16, 0.56)): { + (192, 192, 256, 64, 64, False, True, True): (3, 4, 3, 4), + (192, 192, 256, 64, 64, True, False, True): (1, 4, 4, 4), + (192, 192, 512, 64, 64, False, True, True): (2, 8, 3, 4), + (192, 192, 512, 64, 64, True, False, True): (2, 8, 3, 4), + (192, 192, 1024, 64, 64, False, True, True): (1, 16, 3, 4), + (192, 192, 1024, 64, 64, True, False, True): (1, 16, 5, 4), + (192, 192, 2048, 64, 64, False, True, True): (3, 32, 3, 4), + (192, 192, 2048, 64, 64, True, False, True): (5, 32, 3, 4), + (192, 192, 4096, 64, 64, False, True, True): (1, 64, 4, 4), + (192, 192, 4096, 64, 64, True, False, True): (2, 32, 3, 4), + (192, 192, 8192, 64, 64, False, True, True): (1, 128, 2, 4), + (192, 192, 8192, 64, 64, True, False, True): (1, 64, 3, 4), + (192, 192, 16384, 64, 64, False, True, True): (1, 256, 1, 4), + (192, 192, 16384, 64, 64, True, False, True): (1, 64, 3, 4), + (192, 192, 32768, 64, 64, False, True, True): (2, 512, 1, 2), + (192, 192, 32768, 64, 64, True, False, True): (2, 256, 2, 4), + (192, 192, 65536, 64, 64, False, True, True): (3, 512, 1, 4), + (192, 192, 65536, 64, 64, True, False, True): (1, 512, 2, 4), + (192, 192, 131072, 64, 64, False, True, True): (5, 1024, 1, 4), + (192, 192, 131072, 64, 64, True, False, True): (4, 512, 2, 4), + (384, 384, 256, 128, 128, False, True, True): (3, 2, 3, 8), + (384, 384, 256, 128, 128, True, False, True): (1, 2, 3, 8), + (384, 384, 512, 128, 128, False, True, True): (4, 4, 3, 8), + (384, 384, 512, 128, 128, True, False, True): (3, 4, 3, 8), + (384, 384, 1024, 128, 128, False, True, True): (1, 8, 3, 8), + (384, 384, 1024, 128, 128, True, False, True): (2, 8, 3, 8), + (384, 384, 2048, 128, 128, False, True, True): (5, 16, 3, 8), + (384, 384, 2048, 128, 128, True, False, True): (5, 16, 3, 8), + (384, 384, 4096, 128, 128, False, True, True): (3, 32, 3, 8), + (384, 384, 4096, 128, 128, True, False, True): (6, 32, 3, 8), + (384, 384, 8192, 128, 128, False, True, True): (2, 64, 3, 8), + (384, 384, 8192, 128, 128, True, False, True): (4, 32, 2, 8), + (384, 384, 16384, 128, 128, False, True, True): (2, 128, 3, 8), + (384, 384, 16384, 128, 128, True, False, True): (5, 128, 2, 4), + (384, 384, 32768, 128, 128, False, True, True): (2, 256, 3, 8), + (384, 384, 32768, 128, 128, True, False, True): (3, 256, 2, 4), + (384, 384, 65536, 128, 128, False, True, True): (3, 512, 1, 4), + (384, 384, 65536, 128, 128, True, False, True): (1, 512, 2, 4), + (384, 384, 131072, 128, 128, False, True, True): (3, 1024, 1, 4), + (384, 384, 131072, 128, 128, True, False, True): (1, 1024, 2, 4), + }, + ("bsr_dense_addmm", "NVIDIA A100-SXM4-80GB", (0, torch.float16, 0.5)): { + (16, 16, 16, 16, 16, False, False, False): (1, 1, 1, 1), + (16, 16, 16, 16, 16, False, False, True): (1, 1, 2, 2), + (16, 16, 16, 16, 16, False, True, False): (1, 1, 1, 1), + (16, 16, 16, 16, 16, False, True, True): (1, 1, 1, 8), + (16, 16, 16, 16, 16, True, False, False): (3, 1, 3, 4), + (16, 16, 16, 16, 16, True, False, True): (1, 1, 2, 1), + (16, 16, 32, 16, 16, False, False, False): (1, 2, 1, 8), + (16, 16, 32, 16, 16, False, False, True): (1, 2, 1, 2), + (16, 16, 32, 16, 16, False, True, False): (2, 1, 1, 4), + (16, 16, 32, 16, 16, False, True, True): (1, 2, 1, 4), + (16, 16, 32, 16, 16, True, False, False): (1, 1, 1, 4), + (16, 16, 32, 16, 16, True, False, True): (1, 2, 1, 2), + (16, 16, 64, 16, 16, False, False, False): (1, 4, 1, 1), + (16, 16, 64, 16, 16, False, False, True): (1, 2, 2, 4), + (16, 16, 64, 16, 16, False, True, False): (1, 4, 1, 4), + (16, 16, 64, 16, 16, False, True, True): (1, 2, 1, 4), + (16, 16, 64, 16, 16, True, False, False): (1, 4, 1, 2), + (16, 16, 64, 16, 16, True, False, True): (1, 1, 1, 2), + (16, 32, 16, 16, 16, False, False, False): (1, 1, 2, 4), + (16, 32, 16, 16, 16, False, False, True): (1, 1, 1, 4), + (16, 32, 16, 16, 16, False, True, False): (1, 1, 1, 2), + (16, 32, 16, 16, 16, False, True, True): (1, 1, 1, 2), + (16, 32, 16, 16, 16, True, False, False): (1, 1, 2, 16), + (16, 32, 16, 16, 16, True, False, True): (1, 1, 1, 4), + (16, 32, 16, 16, 32, False, False, False): (2, 1, 1, 8), + (16, 32, 16, 16, 32, False, False, True): (2, 1, 1, 8), + (16, 32, 16, 16, 32, False, True, False): (1, 1, 2, 1), + (16, 32, 16, 16, 32, False, True, True): (1, 1, 1, 4), + (16, 32, 16, 16, 32, True, False, False): (2, 1, 1, 8), + (16, 32, 16, 16, 32, True, False, True): (1, 1, 2, 4), + (16, 32, 32, 16, 16, False, False, False): (1, 1, 1, 16), + (16, 32, 32, 16, 16, False, False, True): (1, 2, 1, 2), + (16, 32, 32, 16, 16, False, True, False): (1, 2, 1, 8), + (16, 32, 32, 16, 16, False, True, True): (3, 2, 1, 4), + (16, 32, 32, 16, 16, True, False, False): (1, 2, 1, 4), + (16, 32, 32, 16, 16, True, False, True): (1, 2, 1, 2), + (16, 32, 32, 16, 32, False, False, False): (1, 2, 1, 2), + (16, 32, 32, 16, 32, False, False, True): (1, 1, 1, 4), + (16, 32, 32, 16, 32, False, True, False): (1, 1, 2, 4), + (16, 32, 32, 16, 32, False, True, True): (1, 2, 1, 2), + (16, 32, 32, 16, 32, True, False, False): (1, 2, 1, 2), + (16, 32, 32, 16, 32, True, False, True): (1, 2, 1, 16), + (16, 32, 64, 16, 16, False, False, False): (1, 4, 1, 4), + (16, 32, 64, 16, 16, False, False, True): (2, 4, 1, 4), + (16, 32, 64, 16, 16, False, True, False): (1, 4, 1, 4), + (16, 32, 64, 16, 16, False, True, True): (1, 4, 1, 4), + (16, 32, 64, 16, 16, True, False, False): (3, 4, 1, 2), + (16, 32, 64, 16, 16, True, False, True): (1, 4, 1, 1), + (16, 32, 64, 16, 32, False, False, False): (1, 4, 1, 16), + (16, 32, 64, 16, 32, False, False, True): (1, 2, 1, 2), + (16, 32, 64, 16, 32, False, True, False): (1, 4, 2, 2), + (16, 32, 64, 16, 32, False, True, True): (1, 4, 1, 8), + (16, 32, 64, 16, 32, True, False, False): (1, 4, 1, 8), + (16, 32, 64, 16, 32, True, False, True): (1, 2, 1, 4), + (16, 64, 16, 16, 32, False, False, False): (1, 1, 1, 2), + (16, 64, 16, 16, 32, False, False, True): (1, 1, 1, 4), + (16, 64, 16, 16, 32, False, True, False): (2, 1, 2, 4), + (16, 64, 16, 16, 32, False, True, True): (1, 1, 1, 4), + (16, 64, 16, 16, 32, True, False, False): (1, 1, 1, 4), + (16, 64, 16, 16, 32, True, False, True): (1, 1, 1, 4), + (16, 64, 32, 16, 32, False, False, False): (1, 2, 1, 2), + (16, 64, 32, 16, 32, False, False, True): (1, 1, 1, 4), + (16, 64, 32, 16, 32, False, True, False): (1, 1, 1, 4), + (16, 64, 32, 16, 32, False, True, True): (1, 2, 3, 2), + (16, 64, 32, 16, 32, True, False, False): (1, 1, 1, 4), + (16, 64, 32, 16, 32, True, False, True): (1, 1, 2, 4), + (16, 64, 64, 16, 32, False, False, False): (1, 4, 1, 8), + (16, 64, 64, 16, 32, False, False, True): (1, 4, 1, 4), + (16, 64, 64, 16, 32, False, True, False): (1, 4, 1, 1), + (16, 64, 64, 16, 32, False, True, True): (2, 4, 1, 4), + (16, 64, 64, 16, 32, True, False, False): (1, 4, 1, 4), + (16, 64, 64, 16, 32, True, False, True): (1, 4, 1, 4), + (32, 16, 16, 16, 16, False, False, False): (2, 1, 2, 4), + (32, 16, 16, 16, 16, False, False, True): (2, 1, 1, 2), + (32, 16, 16, 16, 16, False, True, False): (1, 1, 2, 4), + (32, 16, 16, 16, 16, False, True, True): (1, 1, 1, 2), + (32, 16, 16, 16, 16, True, False, False): (1, 1, 1, 4), + (32, 16, 16, 16, 16, True, False, True): (2, 1, 1, 2), + (32, 16, 32, 16, 16, False, False, False): (1, 1, 1, 4), + (32, 16, 32, 16, 16, False, False, True): (1, 1, 1, 4), + (32, 16, 32, 16, 16, False, True, False): (1, 2, 1, 4), + (32, 16, 32, 16, 16, False, True, True): (2, 2, 1, 4), + (32, 16, 32, 16, 16, True, False, False): (2, 1, 1, 4), + (32, 16, 32, 16, 16, True, False, True): (2, 2, 1, 2), + (32, 16, 64, 16, 16, False, False, False): (1, 4, 1, 2), + (32, 16, 64, 16, 16, False, False, True): (1, 4, 1, 4), + (32, 16, 64, 16, 16, False, True, False): (1, 2, 1, 4), + (32, 16, 64, 16, 16, False, True, True): (1, 4, 1, 2), + (32, 16, 64, 16, 16, True, False, False): (1, 4, 2, 8), + (32, 16, 64, 16, 16, True, False, True): (1, 4, 1, 1), + (32, 32, 16, 16, 16, False, False, False): (1, 1, 1, 4), + (32, 32, 16, 16, 16, False, False, True): (2, 1, 1, 4), + (32, 32, 16, 16, 16, False, True, False): (1, 1, 2, 4), + (32, 32, 16, 16, 16, False, True, True): (1, 1, 2, 2), + (32, 32, 16, 16, 16, True, False, False): (1, 1, 1, 8), + (32, 32, 16, 16, 16, True, False, True): (1, 1, 1, 4), + (32, 32, 16, 16, 32, False, False, False): (1, 1, 3, 2), + (32, 32, 16, 16, 32, False, False, True): (2, 1, 1, 4), + (32, 32, 16, 16, 32, False, True, False): (3, 1, 1, 4), + (32, 32, 16, 16, 32, False, True, True): (1, 1, 1, 4), + (32, 32, 16, 16, 32, True, False, False): (2, 1, 1, 8), + (32, 32, 16, 16, 32, True, False, True): (1, 1, 3, 2), + (32, 32, 16, 32, 32, False, False, False): (1, 1, 1, 2), + (32, 32, 16, 32, 32, False, False, True): (2, 1, 1, 8), + (32, 32, 16, 32, 32, False, True, False): (1, 1, 1, 2), + (32, 32, 16, 32, 32, False, True, True): (1, 1, 1, 8), + (32, 32, 16, 32, 32, True, False, False): (1, 1, 2, 4), + (32, 32, 16, 32, 32, True, False, True): (1, 1, 1, 2), + (32, 32, 32, 16, 16, False, False, False): (1, 1, 1, 4), + (32, 32, 32, 16, 16, False, False, True): (1, 2, 1, 4), + (32, 32, 32, 16, 16, False, True, False): (1, 2, 1, 4), + (32, 32, 32, 16, 16, False, True, True): (1, 2, 1, 2), + (32, 32, 32, 16, 16, True, False, False): (1, 2, 1, 4), + (32, 32, 32, 16, 16, True, False, True): (1, 2, 1, 4), + (32, 32, 32, 16, 32, False, False, False): (1, 2, 1, 4), + (32, 32, 32, 16, 32, False, False, True): (1, 2, 1, 2), + (32, 32, 32, 16, 32, False, True, False): (1, 2, 1, 4), + (32, 32, 32, 16, 32, False, True, True): (1, 2, 1, 2), + (32, 32, 32, 16, 32, True, False, False): (1, 2, 1, 1), + (32, 32, 32, 16, 32, True, False, True): (1, 2, 1, 2), + (32, 32, 32, 32, 32, False, False, False): (1, 1, 1, 4), + (32, 32, 32, 32, 32, False, False, True): (2, 1, 1, 4), + (32, 32, 32, 32, 32, False, True, False): (1, 1, 1, 8), + (32, 32, 32, 32, 32, False, True, True): (1, 1, 1, 8), + (32, 32, 32, 32, 32, True, False, False): (1, 1, 3, 4), + (32, 32, 32, 32, 32, True, False, True): (1, 1, 1, 8), + (32, 32, 64, 16, 16, False, False, False): (1, 4, 1, 4), + (32, 32, 64, 16, 16, False, False, True): (1, 4, 1, 2), + (32, 32, 64, 16, 16, False, True, False): (1, 1, 1, 4), + (32, 32, 64, 16, 16, False, True, True): (1, 4, 1, 4), + (32, 32, 64, 16, 16, True, False, False): (1, 4, 1, 8), + (32, 32, 64, 16, 16, True, False, True): (1, 4, 1, 2), + (32, 32, 64, 16, 32, False, False, False): (1, 1, 1, 4), + (32, 32, 64, 16, 32, False, False, True): (1, 4, 1, 4), + (32, 32, 64, 16, 32, False, True, False): (1, 1, 1, 4), + (32, 32, 64, 16, 32, False, True, True): (1, 4, 1, 4), + (32, 32, 64, 16, 32, True, False, False): (2, 2, 1, 8), + (32, 32, 64, 16, 32, True, False, True): (1, 2, 1, 2), + (32, 32, 64, 32, 32, False, False, False): (1, 2, 1, 4), + (32, 32, 64, 32, 32, False, False, True): (1, 2, 1, 1), + (32, 32, 64, 32, 32, False, True, False): (1, 2, 2, 8), + (32, 32, 64, 32, 32, False, True, True): (1, 1, 1, 4), + (32, 32, 64, 32, 32, True, False, False): (1, 2, 1, 4), + (32, 32, 64, 32, 32, True, False, True): (2, 2, 1, 4), + (32, 64, 16, 16, 32, False, False, False): (1, 1, 1, 8), + (32, 64, 16, 16, 32, False, False, True): (1, 1, 1, 4), + (32, 64, 16, 16, 32, False, True, False): (2, 1, 1, 4), + (32, 64, 16, 16, 32, False, True, True): (1, 1, 1, 4), + (32, 64, 16, 16, 32, True, False, False): (1, 1, 2, 4), + (32, 64, 16, 16, 32, True, False, True): (1, 1, 2, 2), + (32, 64, 16, 32, 32, False, False, False): (1, 1, 1, 8), + (32, 64, 16, 32, 32, False, False, True): (2, 1, 1, 4), + (32, 64, 16, 32, 32, False, True, False): (1, 1, 1, 4), + (32, 64, 16, 32, 32, False, True, True): (1, 1, 2, 2), + (32, 64, 16, 32, 32, True, False, False): (1, 1, 1, 2), + (32, 64, 16, 32, 32, True, False, True): (2, 1, 2, 4), + (32, 64, 32, 16, 32, False, False, False): (1, 1, 1, 4), + (32, 64, 32, 16, 32, False, False, True): (1, 2, 1, 2), + (32, 64, 32, 16, 32, False, True, False): (1, 2, 3, 4), + (32, 64, 32, 16, 32, False, True, True): (2, 2, 1, 4), + (32, 64, 32, 16, 32, True, False, False): (1, 1, 1, 4), + (32, 64, 32, 16, 32, True, False, True): (1, 2, 2, 1), + (32, 64, 32, 32, 32, False, False, False): (1, 1, 1, 8), + (32, 64, 32, 32, 32, False, False, True): (1, 1, 1, 4), + (32, 64, 32, 32, 32, False, True, False): (1, 1, 2, 4), + (32, 64, 32, 32, 32, False, True, True): (1, 1, 1, 4), + (32, 64, 32, 32, 32, True, False, False): (2, 1, 1, 2), + (32, 64, 32, 32, 32, True, False, True): (1, 1, 1, 4), + (32, 64, 64, 16, 32, False, False, False): (1, 4, 2, 1), + (32, 64, 64, 16, 32, False, False, True): (3, 4, 1, 4), + (32, 64, 64, 16, 32, False, True, False): (1, 1, 1, 8), + (32, 64, 64, 16, 32, False, True, True): (1, 4, 1, 4), + (32, 64, 64, 16, 32, True, False, False): (1, 4, 1, 4), + (32, 64, 64, 16, 32, True, False, True): (2, 2, 3, 4), + (32, 64, 64, 32, 32, False, False, False): (1, 2, 1, 4), + (32, 64, 64, 32, 32, False, False, True): (1, 2, 1, 4), + (32, 64, 64, 32, 32, False, True, False): (1, 2, 2, 8), + (32, 64, 64, 32, 32, False, True, True): (1, 2, 1, 4), + (32, 64, 64, 32, 32, True, False, False): (1, 2, 2, 4), + (32, 64, 64, 32, 32, True, False, True): (1, 2, 1, 4), + (64, 32, 16, 32, 32, False, False, False): (1, 1, 1, 1), + (64, 32, 16, 32, 32, False, False, True): (1, 1, 2, 4), + (64, 32, 16, 32, 32, False, True, False): (2, 1, 1, 8), + (64, 32, 16, 32, 32, False, True, True): (1, 1, 1, 4), + (64, 32, 16, 32, 32, True, False, False): (2, 1, 1, 2), + (64, 32, 16, 32, 32, True, False, True): (1, 1, 1, 4), + (64, 32, 32, 32, 32, False, False, False): (3, 1, 1, 4), + (64, 32, 32, 32, 32, False, False, True): (1, 1, 1, 4), + (64, 32, 32, 32, 32, False, True, False): (1, 1, 1, 8), + (64, 32, 32, 32, 32, False, True, True): (1, 1, 1, 2), + (64, 32, 32, 32, 32, True, False, False): (1, 1, 1, 2), + (64, 32, 32, 32, 32, True, False, True): (1, 1, 1, 4), + (64, 32, 64, 32, 32, False, False, False): (1, 2, 1, 2), + (64, 32, 64, 32, 32, False, False, True): (3, 2, 1, 4), + (64, 32, 64, 32, 32, False, True, False): (1, 1, 1, 1), + (64, 32, 64, 32, 32, False, True, True): (1, 2, 1, 4), + (64, 32, 64, 32, 32, True, False, False): (1, 1, 3, 4), + (64, 32, 64, 32, 32, True, False, True): (1, 2, 2, 4), + (64, 64, 16, 32, 32, False, False, False): (1, 1, 2, 2), + (64, 64, 16, 32, 32, False, False, True): (1, 1, 3, 2), + (64, 64, 16, 32, 32, False, True, False): (1, 1, 1, 8), + (64, 64, 16, 32, 32, False, True, True): (1, 1, 2, 4), + (64, 64, 16, 32, 32, True, False, False): (1, 1, 2, 4), + (64, 64, 16, 32, 32, True, False, True): (2, 1, 2, 4), + (64, 64, 32, 32, 32, False, False, False): (1, 1, 2, 8), + (64, 64, 32, 32, 32, False, False, True): (1, 1, 2, 4), + (64, 64, 32, 32, 32, False, True, False): (1, 1, 1, 4), + (64, 64, 32, 32, 32, False, True, True): (1, 1, 1, 4), + (64, 64, 32, 32, 32, True, False, False): (1, 1, 1, 4), + (64, 64, 32, 32, 32, True, False, True): (2, 1, 2, 4), + (64, 64, 64, 32, 32, False, False, False): (1, 2, 1, 4), + (64, 64, 64, 32, 32, False, False, True): (1, 2, 1, 4), + (64, 64, 64, 32, 32, False, True, False): (1, 2, 1, 4), + (64, 64, 64, 32, 32, False, True, True): (3, 2, 1, 4), + (64, 64, 64, 32, 32, True, False, False): (1, 2, 1, 8), + (64, 64, 64, 32, 32, True, False, True): (1, 2, 3, 4), + (192, 192, 256, 16, 16, False, True, True): (1, 8, 4, 2), + (192, 192, 256, 16, 16, True, False, True): (1, 4, 4, 4), + (192, 192, 256, 32, 32, False, True, True): (2, 8, 5, 4), + (192, 192, 256, 32, 32, True, False, True): (2, 8, 5, 1), + (192, 192, 512, 16, 16, False, True, True): (3, 8, 4, 4), + (192, 192, 512, 16, 16, True, False, True): (5, 8, 5, 4), + (192, 192, 512, 32, 32, False, True, True): (1, 16, 5, 4), + (192, 192, 512, 32, 32, True, False, True): (1, 8, 6, 2), + (192, 192, 1024, 16, 16, False, True, True): (1, 16, 4, 4), + (192, 192, 1024, 16, 16, True, False, True): (3, 16, 5, 2), + (192, 192, 1024, 32, 32, False, True, True): (3, 16, 4, 4), + (192, 192, 1024, 32, 32, True, False, True): (1, 16, 5, 4), + (192, 192, 2048, 16, 16, False, True, True): (2, 16, 3, 4), + (192, 192, 2048, 16, 16, True, False, True): (1, 16, 4, 4), + (192, 192, 2048, 32, 32, False, True, True): (1, 32, 3, 4), + (192, 192, 2048, 32, 32, True, False, True): (3, 16, 4, 4), + (192, 192, 4096, 16, 16, False, True, True): (1, 64, 1, 4), + (192, 192, 4096, 16, 16, True, False, True): (1, 16, 3, 4), + (192, 192, 4096, 32, 32, False, True, True): (1, 128, 1, 4), + (192, 192, 4096, 32, 32, True, False, True): (2, 32, 4, 2), + (192, 192, 8192, 16, 16, False, True, True): (1, 64, 1, 4), + (192, 192, 8192, 16, 16, True, False, True): (2, 64, 3, 2), + (192, 192, 8192, 32, 32, False, True, True): (1, 128, 1, 4), + (192, 192, 8192, 32, 32, True, False, True): (4, 32, 3, 4), + (192, 192, 16384, 16, 16, False, True, True): (1, 128, 1, 4), + (192, 192, 16384, 16, 16, True, False, True): (1, 64, 3, 2), + (192, 192, 16384, 32, 32, False, True, True): (1, 128, 1, 4), + (192, 192, 16384, 32, 32, True, False, True): (1, 64, 3, 4), + (192, 192, 32768, 16, 16, False, True, True): (2, 256, 1, 2), + (192, 192, 32768, 16, 16, True, False, True): (1, 128, 3, 2), + (192, 192, 32768, 32, 32, False, True, True): (2, 256, 1, 4), + (192, 192, 32768, 32, 32, True, False, True): (1, 256, 3, 2), + (192, 192, 65536, 16, 16, False, True, True): (2, 512, 1, 2), + (192, 192, 65536, 16, 16, True, False, True): (1, 256, 3, 2), + (192, 192, 65536, 32, 32, False, True, True): (2, 512, 1, 4), + (192, 192, 65536, 32, 32, True, False, True): (2, 256, 3, 4), + (192, 192, 131072, 16, 16, False, True, True): (4, 1024, 1, 2), + (192, 192, 131072, 16, 16, True, False, True): (3, 512, 3, 2), + (192, 192, 131072, 32, 32, False, True, True): (1, 1024, 1, 4), + (192, 192, 131072, 32, 32, True, False, True): (3, 512, 3, 4), + (256, 256, 256, 16, 16, False, True, True): (4, 8, 6, 2), + (256, 256, 256, 16, 16, True, False, True): (5, 16, 5, 1), + (256, 256, 256, 32, 32, False, True, True): (1, 8, 7, 4), + (256, 256, 256, 32, 32, True, False, True): (1, 8, 5, 4), + (256, 256, 256, 64, 64, False, True, True): (1, 4, 5, 4), + (256, 256, 256, 64, 64, True, False, True): (2, 4, 3, 4), + (256, 256, 256, 128, 128, False, True, True): (1, 2, 2, 8), + (256, 256, 256, 128, 128, True, False, True): (1, 2, 2, 8), + (256, 256, 512, 16, 16, False, True, True): (4, 8, 4, 4), + (256, 256, 512, 16, 16, True, False, True): (4, 8, 6, 2), + (256, 256, 512, 32, 32, False, True, True): (3, 8, 5, 4), + (256, 256, 512, 32, 32, True, False, True): (2, 8, 5, 4), + (256, 256, 512, 64, 64, False, True, True): (2, 8, 4, 4), + (256, 256, 512, 64, 64, True, False, True): (1, 8, 7, 4), + (256, 256, 512, 128, 128, False, True, True): (2, 4, 2, 8), + (256, 256, 512, 128, 128, True, False, True): (5, 4, 2, 8), + (256, 256, 1024, 16, 16, False, True, True): (1, 8, 4, 4), + (256, 256, 1024, 16, 16, True, False, True): (1, 16, 4, 2), + (256, 256, 1024, 32, 32, False, True, True): (5, 32, 5, 1), + (256, 256, 1024, 32, 32, True, False, True): (1, 16, 4, 2), + (256, 256, 1024, 64, 64, False, True, True): (1, 16, 4, 4), + (256, 256, 1024, 64, 64, True, False, True): (2, 16, 3, 4), + (256, 256, 1024, 128, 128, False, True, True): (9, 8, 2, 8), + (256, 256, 1024, 128, 128, True, False, True): (1, 8, 2, 8), + (256, 256, 2048, 16, 16, False, True, True): (6, 32, 5, 2), + (256, 256, 2048, 16, 16, True, False, True): (2, 32, 4, 2), + (256, 256, 2048, 32, 32, False, True, True): (1, 32, 3, 2), + (256, 256, 2048, 32, 32, True, False, True): (1, 32, 3, 2), + (256, 256, 2048, 64, 64, False, True, True): (2, 32, 4, 4), + (256, 256, 2048, 64, 64, True, False, True): (2, 16, 4, 4), + (256, 256, 2048, 128, 128, False, True, True): (3, 16, 2, 8), + (256, 256, 2048, 128, 128, True, False, True): (4, 16, 2, 8), + (256, 256, 4096, 16, 16, False, True, True): (1, 32, 3, 4), + (256, 256, 4096, 16, 16, True, False, True): (3, 16, 3, 2), + (256, 256, 4096, 32, 32, False, True, True): (3, 32, 3, 2), + (256, 256, 4096, 32, 32, True, False, True): (1, 32, 3, 2), + (256, 256, 4096, 64, 64, False, True, True): (2, 32, 3, 4), + (256, 256, 4096, 64, 64, True, False, True): (2, 32, 3, 4), + (256, 256, 4096, 128, 128, False, True, True): (5, 32, 2, 8), + (256, 256, 4096, 128, 128, True, False, True): (1, 32, 2, 8), + (256, 256, 8192, 16, 16, False, True, True): (8, 32, 3, 4), + (256, 256, 8192, 16, 16, True, False, True): (1, 32, 3, 2), + (256, 256, 8192, 32, 32, False, True, True): (3, 64, 3, 4), + (256, 256, 8192, 32, 32, True, False, True): (2, 128, 1, 2), + (256, 256, 8192, 64, 64, False, True, True): (7, 128, 1, 4), + (256, 256, 8192, 64, 64, True, False, True): (4, 128, 1, 4), + (256, 256, 8192, 128, 128, False, True, True): (2, 64, 1, 4), + (256, 256, 8192, 128, 128, True, False, True): (4, 64, 1, 4), + (256, 256, 16384, 16, 16, False, True, True): (4, 128, 3, 2), + (256, 256, 16384, 16, 16, True, False, True): (5, 64, 3, 2), + (256, 256, 16384, 32, 32, False, True, True): (5, 128, 3, 2), + (256, 256, 16384, 32, 32, True, False, True): (5, 128, 3, 2), + (256, 256, 16384, 64, 64, False, True, True): (1, 256, 1, 4), + (256, 256, 16384, 64, 64, True, False, True): (5, 128, 3, 4), + (256, 256, 16384, 128, 128, False, True, True): (11, 128, 2, 8), + (256, 256, 16384, 128, 128, True, False, True): (3, 128, 1, 4), + (256, 256, 32768, 16, 16, False, True, True): (1, 128, 3, 4), + (256, 256, 32768, 16, 16, True, False, True): (2, 128, 3, 2), + (256, 256, 32768, 32, 32, False, True, True): (4, 256, 3, 2), + (256, 256, 32768, 32, 32, True, False, True): (1, 256, 3, 2), + (256, 256, 32768, 64, 64, False, True, True): (2, 256, 1, 4), + (256, 256, 32768, 64, 64, True, False, True): (2, 256, 1, 4), + (256, 256, 32768, 128, 128, False, True, True): (3, 256, 1, 4), + (256, 256, 32768, 128, 128, True, False, True): (2, 256, 1, 4), + (256, 256, 50432, 16, 16, False, True, True): (4, 197, 1, 4), + (256, 256, 50432, 16, 16, True, False, True): (4, 197, 3, 2), + (256, 256, 50432, 32, 32, False, True, True): (1, 394, 1, 2), + (256, 256, 50432, 32, 32, True, False, True): (4, 197, 3, 4), + (256, 256, 50432, 64, 64, False, True, True): (6, 394, 1, 4), + (256, 256, 50432, 64, 64, True, False, True): (4, 394, 2, 4), + (256, 256, 50432, 128, 128, False, True, True): (3, 394, 1, 4), + (256, 256, 50432, 128, 128, True, False, True): (1, 394, 2, 4), + (256, 256, 65536, 16, 16, False, True, True): (1, 256, 3, 2), + (256, 256, 65536, 16, 16, True, False, True): (1, 256, 3, 2), + (256, 256, 65536, 32, 32, False, True, True): (1, 512, 3, 2), + (256, 256, 65536, 32, 32, True, False, True): (4, 512, 3, 2), + (256, 256, 65536, 64, 64, False, True, True): (2, 512, 1, 4), + (256, 256, 65536, 64, 64, True, False, True): (5, 512, 1, 4), + (256, 256, 65536, 128, 128, False, True, True): (3, 512, 1, 4), + (256, 256, 65536, 128, 128, True, False, True): (1, 512, 1, 4), + (256, 256, 65792, 16, 16, False, True, True): (2, 257, 1, 4), + (256, 256, 65792, 16, 16, True, False, True): (1, 257, 3, 2), + (256, 256, 65792, 32, 32, False, True, True): (2, 257, 1, 4), + (256, 256, 65792, 32, 32, True, False, True): (1, 257, 3, 4), + (256, 256, 65792, 64, 64, False, True, True): (2, 514, 1, 4), + (256, 256, 65792, 64, 64, True, False, True): (2, 514, 2, 4), + (256, 256, 65792, 128, 128, False, True, True): (3, 514, 1, 4), + (256, 256, 65792, 128, 128, True, False, True): (1, 514, 2, 4), + (256, 256, 131072, 16, 16, False, True, True): (1, 512, 3, 1), + (256, 256, 131072, 16, 16, True, False, True): (1, 512, 3, 2), + (256, 256, 131072, 32, 32, False, True, True): (2, 1024, 3, 2), + (256, 256, 131072, 32, 32, True, False, True): (1, 1024, 3, 2), + (256, 256, 131072, 64, 64, False, True, True): (1, 1024, 1, 4), + (256, 256, 131072, 64, 64, True, False, True): (1, 1024, 1, 4), + (256, 256, 131072, 128, 128, False, True, True): (7, 1024, 1, 4), + (256, 256, 131072, 128, 128, True, False, True): (1, 1024, 1, 4), + (384, 384, 256, 16, 16, False, True, True): (3, 16, 4, 1), + (384, 384, 256, 16, 16, True, False, True): (2, 4, 6, 2), + (384, 384, 256, 32, 32, False, True, True): (1, 8, 4, 4), + (384, 384, 256, 32, 32, True, False, True): (1, 4, 5, 2), + (384, 384, 256, 64, 64, False, True, True): (3, 4, 3, 4), + (384, 384, 256, 64, 64, True, False, True): (4, 4, 5, 4), + (384, 384, 512, 16, 16, False, True, True): (1, 16, 4, 1), + (384, 384, 512, 16, 16, True, False, True): (1, 8, 5, 2), + (384, 384, 512, 32, 32, False, True, True): (4, 16, 4, 2), + (384, 384, 512, 32, 32, True, False, True): (1, 8, 5, 2), + (384, 384, 512, 64, 64, False, True, True): (2, 8, 3, 4), + (384, 384, 512, 64, 64, True, False, True): (1, 8, 4, 4), + (384, 384, 1024, 16, 16, False, True, True): (1, 16, 4, 2), + (384, 384, 1024, 16, 16, True, False, True): (7, 8, 5, 2), + (384, 384, 1024, 32, 32, False, True, True): (2, 16, 3, 4), + (384, 384, 1024, 32, 32, True, False, True): (1, 16, 4, 2), + (384, 384, 1024, 64, 64, False, True, True): (6, 16, 3, 4), + (384, 384, 1024, 64, 64, True, False, True): (4, 16, 4, 4), + (384, 384, 2048, 16, 16, False, True, True): (1, 32, 1, 4), + (384, 384, 2048, 16, 16, True, False, True): (1, 16, 3, 2), + (384, 384, 2048, 32, 32, False, True, True): (1, 32, 1, 8), + (384, 384, 2048, 32, 32, True, False, True): (1, 8, 4, 4), + (384, 384, 2048, 64, 64, False, True, True): (2, 32, 1, 8), + (384, 384, 2048, 64, 64, True, False, True): (3, 16, 3, 4), + (384, 384, 4096, 16, 16, False, True, True): (5, 32, 1, 4), + (384, 384, 4096, 16, 16, True, False, True): (1, 32, 3, 2), + (384, 384, 4096, 32, 32, False, True, True): (1, 32, 1, 8), + (384, 384, 4096, 32, 32, True, False, True): (2, 16, 4, 4), + (384, 384, 4096, 64, 64, False, True, True): (1, 64, 1, 4), + (384, 384, 4096, 64, 64, True, False, True): (2, 32, 3, 4), + (384, 384, 8192, 16, 16, False, True, True): (2, 64, 1, 4), + (384, 384, 8192, 16, 16, True, False, True): (3, 32, 3, 2), + (384, 384, 8192, 32, 32, False, True, True): (4, 128, 1, 4), + (384, 384, 8192, 32, 32, True, False, True): (1, 32, 3, 2), + (384, 384, 8192, 64, 64, False, True, True): (1, 128, 1, 4), + (384, 384, 8192, 64, 64, True, False, True): (1, 64, 3, 4), + (384, 384, 16384, 16, 16, False, True, True): (1, 128, 1, 2), + (384, 384, 16384, 16, 16, True, False, True): (1, 64, 3, 2), + (384, 384, 16384, 32, 32, False, True, True): (1, 128, 1, 4), + (384, 384, 16384, 32, 32, True, False, True): (1, 64, 3, 4), + (384, 384, 16384, 64, 64, False, True, True): (5, 128, 3, 4), + (384, 384, 16384, 64, 64, True, False, True): (1, 128, 3, 4), + (384, 384, 32768, 16, 16, False, True, True): (2, 256, 1, 2), + (384, 384, 32768, 16, 16, True, False, True): (1, 128, 3, 4), + (384, 384, 32768, 32, 32, False, True, True): (1, 256, 1, 2), + (384, 384, 32768, 32, 32, True, False, True): (2, 128, 3, 4), + (384, 384, 32768, 64, 64, False, True, True): (3, 256, 1, 4), + (384, 384, 32768, 64, 64, True, False, True): (2, 256, 3, 4), + (384, 384, 65536, 16, 16, False, True, True): (2, 128, 1, 4), + (384, 384, 65536, 16, 16, True, False, True): (1, 256, 3, 4), + (384, 384, 65536, 32, 32, False, True, True): (1, 512, 1, 2), + (384, 384, 65536, 32, 32, True, False, True): (1, 256, 3, 4), + (384, 384, 65536, 64, 64, False, True, True): (3, 512, 1, 4), + (384, 384, 65536, 64, 64, True, False, True): (3, 256, 3, 4), + (384, 384, 131072, 16, 16, False, True, True): (2, 256, 1, 2), + (384, 384, 131072, 16, 16, True, False, True): (1, 512, 3, 4), + (384, 384, 131072, 32, 32, False, True, True): (1, 512, 1, 2), + (384, 384, 131072, 32, 32, True, False, True): (1, 512, 3, 4), + (384, 384, 131072, 64, 64, False, True, True): (3, 1024, 1, 4), + (384, 384, 131072, 64, 64, True, False, True): (3, 512, 3, 4), + (512, 512, 256, 16, 16, False, True, True): (1, 8, 5, 1), + (512, 512, 256, 16, 16, True, False, True): (2, 16, 5, 1), + (512, 512, 256, 32, 32, False, True, True): (2, 8, 5, 2), + (512, 512, 256, 32, 32, True, False, True): (4, 4, 5, 2), + (512, 512, 256, 64, 64, False, True, True): (1, 4, 5, 4), + (512, 512, 256, 64, 64, True, False, True): (3, 4, 5, 4), + (512, 512, 256, 128, 128, False, True, True): (1, 2, 2, 8), + (512, 512, 256, 128, 128, True, False, True): (1, 2, 2, 8), + (512, 512, 512, 16, 16, False, True, True): (1, 8, 4, 4), + (512, 512, 512, 16, 16, True, False, True): (4, 16, 5, 1), + (512, 512, 512, 32, 32, False, True, True): (4, 8, 5, 2), + (512, 512, 512, 32, 32, True, False, True): (7, 16, 4, 1), + (512, 512, 512, 64, 64, False, True, True): (3, 8, 5, 4), + (512, 512, 512, 64, 64, True, False, True): (1, 8, 4, 4), + (512, 512, 512, 128, 128, False, True, True): (4, 4, 2, 8), + (512, 512, 512, 128, 128, True, False, True): (4, 4, 2, 8), + (512, 512, 1024, 16, 16, False, True, True): (2, 8, 4, 4), + (512, 512, 1024, 16, 16, True, False, True): (2, 16, 4, 2), + (512, 512, 1024, 32, 32, False, True, True): (3, 16, 4, 2), + (512, 512, 1024, 32, 32, True, False, True): (3, 16, 3, 2), + (512, 512, 1024, 64, 64, False, True, True): (5, 8, 5, 4), + (512, 512, 1024, 64, 64, True, False, True): (4, 16, 3, 4), + (512, 512, 1024, 128, 128, False, True, True): (6, 8, 2, 8), + (512, 512, 1024, 128, 128, True, False, True): (4, 8, 2, 8), + (512, 512, 2048, 16, 16, False, True, True): (2, 16, 3, 4), + (512, 512, 2048, 16, 16, True, False, True): (1, 16, 4, 2), + (512, 512, 2048, 32, 32, False, True, True): (2, 32, 3, 2), + (512, 512, 2048, 32, 32, True, False, True): (2, 32, 3, 2), + (512, 512, 2048, 64, 64, False, True, True): (1, 32, 3, 4), + (512, 512, 2048, 64, 64, True, False, True): (1, 32, 3, 2), + (512, 512, 2048, 128, 128, False, True, True): (3, 16, 2, 8), + (512, 512, 2048, 128, 128, True, False, True): (1, 16, 2, 8), + (512, 512, 4096, 16, 16, False, True, True): (4, 32, 3, 2), + (512, 512, 4096, 16, 16, True, False, True): (1, 32, 3, 2), + (512, 512, 4096, 32, 32, False, True, True): (3, 32, 3, 2), + (512, 512, 4096, 32, 32, True, False, True): (3, 32, 3, 2), + (512, 512, 4096, 64, 64, False, True, True): (1, 32, 3, 4), + (512, 512, 4096, 64, 64, True, False, True): (1, 64, 1, 4), + (512, 512, 4096, 128, 128, False, True, True): (7, 32, 2, 8), + (512, 512, 4096, 128, 128, True, False, True): (1, 32, 2, 8), + (512, 512, 8192, 16, 16, False, True, True): (4, 64, 3, 2), + (512, 512, 8192, 16, 16, True, False, True): (1, 64, 3, 2), + (512, 512, 8192, 32, 32, False, True, True): (3, 64, 3, 2), + (512, 512, 8192, 32, 32, True, False, True): (1, 64, 3, 2), + (512, 512, 8192, 64, 64, False, True, True): (1, 64, 3, 4), + (512, 512, 8192, 64, 64, True, False, True): (1, 64, 3, 4), + (512, 512, 8192, 128, 128, False, True, True): (7, 64, 2, 8), + (512, 512, 8192, 128, 128, True, False, True): (1, 64, 1, 4), + (512, 512, 16384, 16, 16, False, True, True): (1, 128, 3, 2), + (512, 512, 16384, 16, 16, True, False, True): (1, 64, 3, 2), + (512, 512, 16384, 32, 32, False, True, True): (1, 128, 3, 2), + (512, 512, 16384, 32, 32, True, False, True): (1, 128, 3, 2), + (512, 512, 16384, 64, 64, False, True, True): (1, 128, 3, 4), + (512, 512, 16384, 64, 64, True, False, True): (4, 128, 3, 4), + (512, 512, 16384, 128, 128, False, True, True): (5, 128, 2, 8), + (512, 512, 16384, 128, 128, True, False, True): (2, 128, 1, 4), + (512, 512, 32768, 16, 16, False, True, True): (1, 128, 3, 4), + (512, 512, 32768, 16, 16, True, False, True): (1, 128, 3, 2), + (512, 512, 32768, 32, 32, False, True, True): (1, 256, 3, 2), + (512, 512, 32768, 32, 32, True, False, True): (1, 256, 3, 2), + (512, 512, 32768, 64, 64, False, True, True): (1, 256, 3, 4), + (512, 512, 32768, 64, 64, True, False, True): (1, 256, 3, 4), + (512, 512, 32768, 128, 128, False, True, True): (5, 256, 1, 4), + (512, 512, 32768, 128, 128, True, False, True): (1, 256, 1, 4), + (512, 512, 50432, 16, 16, False, True, True): (4, 197, 1, 4), + (512, 512, 50432, 16, 16, True, False, True): (4, 197, 3, 2), + (512, 512, 50432, 32, 32, False, True, True): (2, 197, 1, 4), + (512, 512, 50432, 32, 32, True, False, True): (4, 197, 3, 4), + (512, 512, 50432, 64, 64, False, True, True): (2, 394, 1, 4), + (512, 512, 50432, 64, 64, True, False, True): (4, 197, 2, 4), + (512, 512, 50432, 128, 128, False, True, True): (5, 394, 1, 4), + (512, 512, 50432, 128, 128, True, False, True): (6, 394, 2, 4), + (512, 512, 65536, 16, 16, False, True, True): (1, 256, 3, 2), + (512, 512, 65536, 16, 16, True, False, True): (1, 256, 3, 1), + (512, 512, 65536, 32, 32, False, True, True): (1, 512, 3, 2), + (512, 512, 65536, 32, 32, True, False, True): (1, 512, 3, 2), + (512, 512, 65536, 64, 64, False, True, True): (2, 256, 2, 4), + (512, 512, 65536, 64, 64, True, False, True): (1, 512, 3, 4), + (512, 512, 65536, 128, 128, False, True, True): (7, 512, 1, 4), + (512, 512, 65536, 128, 128, True, False, True): (5, 512, 1, 4), + (512, 512, 65792, 16, 16, False, True, True): (2, 257, 1, 4), + (512, 512, 65792, 16, 16, True, False, True): (1, 257, 3, 4), + (512, 512, 65792, 32, 32, False, True, True): (2, 257, 1, 4), + (512, 512, 65792, 32, 32, True, False, True): (1, 257, 3, 4), + (512, 512, 65792, 64, 64, False, True, True): (4, 514, 1, 4), + (512, 512, 65792, 64, 64, True, False, True): (4, 257, 2, 4), + (512, 512, 65792, 128, 128, False, True, True): (5, 514, 1, 4), + (512, 512, 65792, 128, 128, True, False, True): (4, 514, 2, 4), + (512, 512, 131072, 16, 16, False, True, True): (1, 512, 3, 1), + (512, 512, 131072, 16, 16, True, False, True): (1, 512, 3, 1), + (512, 512, 131072, 32, 32, False, True, True): (1, 1024, 3, 2), + (512, 512, 131072, 32, 32, True, False, True): (1, 1024, 3, 2), + (512, 512, 131072, 64, 64, False, True, True): (4, 512, 2, 4), + (512, 512, 131072, 64, 64, True, False, True): (2, 512, 2, 4), + (512, 512, 131072, 128, 128, False, True, True): (5, 1024, 1, 4), + (512, 512, 131072, 128, 128, True, False, True): (4, 1024, 1, 4), + (768, 768, 256, 16, 16, False, True, True): (1, 8, 4, 1), + (768, 768, 256, 16, 16, True, False, True): (3, 2, 5, 2), + (768, 768, 256, 32, 32, False, True, True): (1, 8, 4, 2), + (768, 768, 256, 32, 32, True, False, True): (2, 4, 6, 2), + (768, 768, 256, 64, 64, False, True, True): (3, 4, 3, 4), + (768, 768, 256, 64, 64, True, False, True): (2, 4, 4, 4), + (768, 768, 256, 128, 128, False, True, True): (1, 2, 3, 8), + (768, 768, 256, 128, 128, True, False, True): (2, 2, 3, 8), + (768, 768, 512, 16, 16, False, True, True): (1, 8, 4, 2), + (768, 768, 512, 16, 16, True, False, True): (2, 8, 5, 2), + (768, 768, 512, 32, 32, False, True, True): (1, 16, 1, 4), + (768, 768, 512, 32, 32, True, False, True): (3, 8, 5, 2), + (768, 768, 512, 64, 64, False, True, True): (4, 8, 3, 4), + (768, 768, 512, 64, 64, True, False, True): (2, 8, 4, 4), + (768, 768, 512, 128, 128, False, True, True): (1, 4, 3, 8), + (768, 768, 512, 128, 128, True, False, True): (3, 4, 3, 8), + (768, 768, 1024, 16, 16, False, True, True): (1, 16, 1, 4), + (768, 768, 1024, 16, 16, True, False, True): (1, 8, 5, 2), + (768, 768, 1024, 32, 32, False, True, True): (1, 16, 1, 8), + (768, 768, 1024, 32, 32, True, False, True): (1, 4, 4, 4), + (768, 768, 1024, 64, 64, False, True, True): (2, 16, 1, 8), + (768, 768, 1024, 64, 64, True, False, True): (1, 8, 3, 8), + (768, 768, 1024, 128, 128, False, True, True): (1, 8, 3, 8), + (768, 768, 1024, 128, 128, True, False, True): (3, 8, 3, 8), + (768, 768, 2048, 16, 16, False, True, True): (6, 16, 1, 2), + (768, 768, 2048, 16, 16, True, False, True): (2, 16, 4, 2), + (768, 768, 2048, 32, 32, False, True, True): (3, 32, 1, 4), + (768, 768, 2048, 32, 32, True, False, True): (6, 8, 3, 4), + (768, 768, 2048, 64, 64, False, True, True): (2, 32, 2, 2), + (768, 768, 2048, 64, 64, True, False, True): (1, 16, 4, 4), + (768, 768, 2048, 128, 128, False, True, True): (2, 16, 3, 8), + (768, 768, 2048, 128, 128, True, False, True): (4, 16, 3, 8), + (768, 768, 4096, 16, 16, False, True, True): (1, 32, 1, 4), + (768, 768, 4096, 16, 16, True, False, True): (2, 16, 3, 2), + (768, 768, 4096, 32, 32, False, True, True): (3, 32, 1, 8), + (768, 768, 4096, 32, 32, True, False, True): (1, 16, 4, 4), + (768, 768, 4096, 64, 64, False, True, True): (1, 64, 2, 4), + (768, 768, 4096, 64, 64, True, False, True): (1, 8, 3, 8), + (768, 768, 4096, 128, 128, False, True, True): (1, 32, 3, 8), + (768, 768, 4096, 128, 128, True, False, True): (2, 32, 3, 8), + (768, 768, 8192, 16, 16, False, True, True): (1, 64, 1, 2), + (768, 768, 8192, 16, 16, True, False, True): (2, 64, 3, 2), + (768, 768, 8192, 32, 32, False, True, True): (2, 64, 1, 8), + (768, 768, 8192, 32, 32, True, False, True): (2, 32, 3, 4), + (768, 768, 8192, 64, 64, False, True, True): (4, 64, 3, 4), + (768, 768, 8192, 64, 64, True, False, True): (1, 64, 3, 4), + (768, 768, 8192, 128, 128, False, True, True): (4, 64, 3, 8), + (768, 768, 8192, 128, 128, True, False, True): (2, 64, 3, 8), + (768, 768, 16384, 16, 16, False, True, True): (4, 128, 1, 2), + (768, 768, 16384, 16, 16, True, False, True): (1, 64, 3, 4), + (768, 768, 16384, 32, 32, False, True, True): (1, 128, 1, 8), + (768, 768, 16384, 32, 32, True, False, True): (1, 64, 3, 4), + (768, 768, 16384, 64, 64, False, True, True): (1, 128, 3, 4), + (768, 768, 16384, 64, 64, True, False, True): (1, 128, 3, 4), + (768, 768, 16384, 128, 128, False, True, True): (3, 128, 1, 4), + (768, 768, 16384, 128, 128, True, False, True): (1, 128, 2, 4), + (768, 768, 32768, 16, 16, False, True, True): (2, 256, 1, 2), + (768, 768, 32768, 16, 16, True, False, True): (1, 128, 4, 4), + (768, 768, 32768, 32, 32, False, True, True): (1, 128, 1, 2), + (768, 768, 32768, 32, 32, True, False, True): (1, 128, 3, 4), + (768, 768, 32768, 64, 64, False, True, True): (1, 256, 1, 4), + (768, 768, 32768, 64, 64, True, False, True): (1, 128, 3, 4), + (768, 768, 32768, 128, 128, False, True, True): (3, 256, 1, 4), + (768, 768, 32768, 128, 128, True, False, True): (3, 256, 2, 4), + (768, 768, 65536, 16, 16, False, True, True): (4, 512, 1, 2), + (768, 768, 65536, 16, 16, True, False, True): (1, 256, 4, 4), + (768, 768, 65536, 32, 32, False, True, True): (1, 256, 1, 2), + (768, 768, 65536, 32, 32, True, False, True): (1, 256, 3, 4), + (768, 768, 65536, 64, 64, False, True, True): (1, 512, 1, 4), + (768, 768, 65536, 64, 64, True, False, True): (1, 256, 3, 4), + (768, 768, 65536, 128, 128, False, True, True): (3, 512, 1, 4), + (768, 768, 65536, 128, 128, True, False, True): (2, 512, 2, 4), + (768, 768, 131072, 16, 16, False, True, True): (1, 512, 1, 1), + (768, 768, 131072, 16, 16, True, False, True): (1, 512, 4, 4), + (768, 768, 131072, 32, 32, False, True, True): (1, 512, 1, 2), + (768, 768, 131072, 32, 32, True, False, True): (1, 512, 3, 4), + (768, 768, 131072, 64, 64, False, True, True): (1, 1024, 1, 4), + (768, 768, 131072, 64, 64, True, False, True): (3, 512, 3, 4), + (768, 768, 131072, 128, 128, False, True, True): (3, 1024, 1, 4), + (768, 768, 131072, 128, 128, True, False, True): (1, 1024, 2, 4), + (768, 3072, 256, 16, 16, False, True, True): (1, 8, 5, 2), + (768, 3072, 256, 16, 16, True, False, True): (3, 4, 7, 2), + (768, 3072, 256, 32, 32, False, True, True): (1, 8, 4, 2), + (768, 3072, 256, 32, 32, True, False, True): (1, 4, 5, 4), + (768, 3072, 256, 64, 64, False, True, True): (1, 4, 3, 4), + (768, 3072, 256, 64, 64, True, False, True): (1, 4, 5, 4), + (768, 3072, 256, 128, 128, False, True, True): (2, 2, 3, 8), + (768, 3072, 256, 128, 128, True, False, True): (2, 2, 3, 8), + (768, 3072, 512, 16, 16, False, True, True): (1, 8, 5, 2), + (768, 3072, 512, 16, 16, True, False, True): (1, 8, 5, 2), + (768, 3072, 512, 32, 32, False, True, True): (3, 8, 3, 4), + (768, 3072, 512, 32, 32, True, False, True): (1, 8, 7, 4), + (768, 3072, 512, 64, 64, False, True, True): (3, 8, 3, 4), + (768, 3072, 512, 64, 64, True, False, True): (3, 8, 5, 4), + (768, 3072, 512, 128, 128, False, True, True): (1, 4, 3, 8), + (768, 3072, 512, 128, 128, True, False, True): (1, 4, 3, 8), + (768, 3072, 1024, 16, 16, False, True, True): (4, 16, 1, 4), + (768, 3072, 1024, 16, 16, True, False, True): (2, 8, 5, 2), + (768, 3072, 1024, 32, 32, False, True, True): (1, 16, 6, 2), + (768, 3072, 1024, 32, 32, True, False, True): (1, 8, 4, 4), + (768, 3072, 1024, 64, 64, False, True, True): (2, 16, 4, 4), + (768, 3072, 1024, 64, 64, True, False, True): (2, 16, 4, 4), + (768, 3072, 1024, 128, 128, False, True, True): (1, 8, 3, 8), + (768, 3072, 1024, 128, 128, True, False, True): (3, 8, 3, 8), + (768, 3072, 2048, 16, 16, False, True, True): (1, 16, 1, 2), + (768, 3072, 2048, 16, 16, True, False, True): (1, 16, 5, 2), + (768, 3072, 2048, 32, 32, False, True, True): (4, 16, 1, 8), + (768, 3072, 2048, 32, 32, True, False, True): (2, 8, 3, 4), + (768, 3072, 2048, 64, 64, False, True, True): (2, 16, 3, 4), + (768, 3072, 2048, 64, 64, True, False, True): (2, 16, 3, 4), + (768, 3072, 2048, 128, 128, False, True, True): (3, 16, 3, 8), + (768, 3072, 2048, 128, 128, True, False, True): (1, 16, 3, 8), + (768, 3072, 4096, 16, 16, False, True, True): (1, 32, 1, 4), + (768, 3072, 4096, 16, 16, True, False, True): (1, 16, 3, 1), + (768, 3072, 4096, 32, 32, False, True, True): (3, 32, 1, 8), + (768, 3072, 4096, 32, 32, True, False, True): (2, 16, 3, 8), + (768, 3072, 4096, 64, 64, False, True, True): (2, 32, 3, 4), + (768, 3072, 4096, 64, 64, True, False, True): (2, 16, 3, 4), + (768, 3072, 4096, 128, 128, False, True, True): (5, 32, 1, 4), + (768, 3072, 4096, 128, 128, True, False, True): (4, 32, 3, 8), + (768, 3072, 8192, 16, 16, False, True, True): (1, 32, 1, 4), + (768, 3072, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (768, 3072, 8192, 32, 32, False, True, True): (1, 64, 1, 8), + (768, 3072, 8192, 32, 32, True, False, True): (2, 32, 3, 8), + (768, 3072, 8192, 64, 64, False, True, True): (2, 64, 3, 4), + (768, 3072, 8192, 64, 64, True, False, True): (2, 32, 3, 4), + (768, 3072, 8192, 128, 128, False, True, True): (1, 64, 3, 8), + (768, 3072, 8192, 128, 128, True, False, True): (2, 64, 3, 8), + (768, 3072, 16384, 16, 16, False, True, True): (1, 64, 1, 4), + (768, 3072, 16384, 16, 16, True, False, True): (1, 64, 4, 1), + (768, 3072, 16384, 32, 32, False, True, True): (1, 128, 1, 8), + (768, 3072, 16384, 32, 32, True, False, True): (1, 64, 3, 4), + (768, 3072, 16384, 64, 64, False, True, True): (1, 128, 3, 4), + (768, 3072, 16384, 64, 64, True, False, True): (1, 64, 3, 4), + (768, 3072, 16384, 128, 128, False, True, True): (2, 128, 3, 8), + (768, 3072, 16384, 128, 128, True, False, True): (1, 128, 3, 8), + (768, 3072, 32768, 16, 16, False, True, True): (1, 128, 1, 4), + (768, 3072, 32768, 16, 16, True, False, True): (1, 128, 4, 1), + (768, 3072, 32768, 32, 32, False, True, True): (1, 256, 1, 8), + (768, 3072, 32768, 32, 32, True, False, True): (1, 128, 3, 4), + (768, 3072, 32768, 64, 64, False, True, True): (1, 256, 3, 4), + (768, 3072, 32768, 64, 64, True, False, True): (1, 128, 3, 4), + (768, 3072, 32768, 128, 128, False, True, True): (3, 256, 1, 4), + (768, 3072, 32768, 128, 128, True, False, True): (5, 256, 3, 8), + (768, 3072, 50432, 16, 16, False, True, True): (1, 197, 1, 4), + (768, 3072, 50432, 16, 16, True, False, True): (4, 197, 4, 1), + (768, 3072, 50432, 32, 32, False, True, True): (2, 197, 1, 4), + (768, 3072, 50432, 32, 32, True, False, True): (4, 197, 3, 4), + (768, 3072, 50432, 64, 64, False, True, True): (1, 394, 3, 4), + (768, 3072, 50432, 64, 64, True, False, True): (1, 197, 3, 4), + (768, 3072, 50432, 128, 128, False, True, True): (3, 394, 1, 4), + (768, 3072, 50432, 128, 128, True, False, True): (3, 394, 2, 4), + (768, 3072, 65536, 16, 16, False, True, True): (1, 256, 1, 4), + (768, 3072, 65536, 16, 16, True, False, True): (5, 256, 4, 1), + (768, 3072, 65536, 32, 32, False, True, True): (2, 256, 1, 4), + (768, 3072, 65536, 32, 32, True, False, True): (3, 256, 3, 4), + (768, 3072, 65536, 64, 64, False, True, True): (1, 512, 3, 4), + (768, 3072, 65536, 64, 64, True, False, True): (1, 256, 3, 4), + (768, 3072, 65536, 128, 128, False, True, True): (3, 512, 1, 4), + (768, 3072, 65536, 128, 128, True, False, True): (2, 512, 3, 8), + (768, 3072, 131072, 16, 16, False, True, True): (1, 512, 1, 4), + (768, 3072, 131072, 16, 16, True, False, True): (5, 512, 4, 1), + (768, 3072, 131072, 32, 32, False, True, True): (2, 512, 1, 4), + (768, 3072, 131072, 32, 32, True, False, True): (2, 512, 3, 4), + (768, 3072, 131072, 64, 64, False, True, True): (1, 1024, 3, 4), + (768, 3072, 131072, 64, 64, True, False, True): (2, 512, 3, 4), + (768, 3072, 131072, 128, 128, False, True, True): (3, 1024, 1, 4), + (768, 3072, 131072, 128, 128, True, False, True): (2, 1024, 3, 8), + (1024, 1024, 256, 16, 16, False, True, True): (3, 4, 5, 4), + (1024, 1024, 256, 16, 16, True, False, True): (3, 4, 5, 4), + (1024, 1024, 256, 32, 32, False, True, True): (2, 4, 6, 2), + (1024, 1024, 256, 32, 32, True, False, True): (2, 4, 6, 2), + (1024, 1024, 256, 64, 64, False, True, True): (1, 4, 4, 4), + (1024, 1024, 256, 64, 64, True, False, True): (2, 4, 6, 4), + (1024, 1024, 256, 128, 128, False, True, True): (1, 2, 2, 8), + (1024, 1024, 256, 128, 128, True, False, True): (1, 2, 2, 8), + (1024, 1024, 512, 16, 16, False, True, True): (3, 4, 5, 4), + (1024, 1024, 512, 16, 16, True, False, True): (3, 8, 4, 2), + (1024, 1024, 512, 32, 32, False, True, True): (1, 8, 4, 2), + (1024, 1024, 512, 32, 32, True, False, True): (1, 8, 4, 2), + (1024, 1024, 512, 64, 64, False, True, True): (2, 8, 3, 4), + (1024, 1024, 512, 64, 64, True, False, True): (1, 4, 4, 4), + (1024, 1024, 512, 128, 128, False, True, True): (7, 4, 2, 8), + (1024, 1024, 512, 128, 128, True, False, True): (1, 4, 2, 8), + (1024, 1024, 1024, 16, 16, False, True, True): (4, 8, 4, 2), + (1024, 1024, 1024, 16, 16, True, False, True): (3, 8, 5, 2), + (1024, 1024, 1024, 32, 32, False, True, True): (1, 8, 4, 4), + (1024, 1024, 1024, 32, 32, True, False, True): (1, 8, 4, 2), + (1024, 1024, 1024, 64, 64, False, True, True): (1, 16, 3, 4), + (1024, 1024, 1024, 64, 64, True, False, True): (3, 16, 3, 4), + (1024, 1024, 1024, 128, 128, False, True, True): (6, 8, 2, 8), + (1024, 1024, 1024, 128, 128, True, False, True): (4, 8, 2, 8), + (1024, 1024, 2048, 16, 16, False, True, True): (3, 8, 3, 4), + (1024, 1024, 2048, 16, 16, True, False, True): (3, 8, 3, 4), + (1024, 1024, 2048, 32, 32, False, True, True): (1, 16, 3, 4), + (1024, 1024, 2048, 32, 32, True, False, True): (1, 16, 3, 2), + (1024, 1024, 2048, 64, 64, False, True, True): (5, 16, 3, 4), + (1024, 1024, 2048, 64, 64, True, False, True): (5, 16, 3, 4), + (1024, 1024, 2048, 128, 128, False, True, True): (3, 16, 2, 8), + (1024, 1024, 2048, 128, 128, True, False, True): (4, 16, 2, 16), + (1024, 1024, 4096, 16, 16, False, True, True): (4, 32, 3, 2), + (1024, 1024, 4096, 16, 16, True, False, True): (8, 32, 3, 2), + (1024, 1024, 4096, 32, 32, False, True, True): (9, 32, 3, 2), + (1024, 1024, 4096, 32, 32, True, False, True): (1, 32, 3, 2), + (1024, 1024, 4096, 64, 64, False, True, True): (6, 32, 3, 4), + (1024, 1024, 4096, 64, 64, True, False, True): (1, 32, 3, 4), + (1024, 1024, 4096, 128, 128, False, True, True): (4, 32, 2, 8), + (1024, 1024, 4096, 128, 128, True, False, True): (4, 32, 1, 4), + (1024, 1024, 8192, 16, 16, False, True, True): (4, 64, 3, 2), + (1024, 1024, 8192, 16, 16, True, False, True): (4, 64, 3, 2), + (1024, 1024, 8192, 32, 32, False, True, True): (8, 64, 3, 2), + (1024, 1024, 8192, 32, 32, True, False, True): (6, 64, 3, 2), + (1024, 1024, 8192, 64, 64, False, True, True): (2, 64, 3, 4), + (1024, 1024, 8192, 64, 64, True, False, True): (2, 64, 3, 4), + (1024, 1024, 8192, 128, 128, False, True, True): (3, 64, 1, 4), + (1024, 1024, 8192, 128, 128, True, False, True): (2, 64, 1, 4), + (1024, 1024, 16384, 16, 16, False, True, True): (1, 64, 3, 4), + (1024, 1024, 16384, 16, 16, True, False, True): (1, 64, 3, 2), + (1024, 1024, 16384, 32, 32, False, True, True): (1, 128, 3, 4), + (1024, 1024, 16384, 32, 32, True, False, True): (1, 64, 3, 4), + (1024, 1024, 16384, 64, 64, False, True, True): (1, 128, 3, 4), + (1024, 1024, 16384, 64, 64, True, False, True): (1, 128, 3, 4), + (1024, 1024, 16384, 128, 128, False, True, True): (11, 128, 1, 4), + (1024, 1024, 16384, 128, 128, True, False, True): (4, 128, 1, 4), + (1024, 1024, 32768, 16, 16, False, True, True): (1, 128, 3, 4), + (1024, 1024, 32768, 16, 16, True, False, True): (1, 128, 3, 1), + (1024, 1024, 32768, 32, 32, False, True, True): (1, 256, 3, 2), + (1024, 1024, 32768, 32, 32, True, False, True): (1, 128, 3, 4), + (1024, 1024, 32768, 64, 64, False, True, True): (2, 128, 2, 4), + (1024, 1024, 32768, 64, 64, True, False, True): (1, 256, 3, 4), + (1024, 1024, 32768, 128, 128, False, True, True): (7, 256, 1, 4), + (1024, 1024, 32768, 128, 128, True, False, True): (4, 256, 1, 4), + (1024, 1024, 50432, 16, 16, False, True, True): (1, 197, 1, 4), + (1024, 1024, 50432, 16, 16, True, False, True): (4, 197, 3, 4), + (1024, 1024, 50432, 32, 32, False, True, True): (2, 197, 1, 4), + (1024, 1024, 50432, 32, 32, True, False, True): (1, 197, 3, 4), + (1024, 1024, 50432, 64, 64, False, True, True): (2, 394, 1, 4), + (1024, 1024, 50432, 64, 64, True, False, True): (1, 197, 2, 4), + (1024, 1024, 50432, 128, 128, False, True, True): (3, 394, 1, 4), + (1024, 1024, 50432, 128, 128, True, False, True): (2, 394, 2, 4), + (1024, 1024, 65536, 16, 16, False, True, True): (1, 256, 3, 4), + (1024, 1024, 65536, 16, 16, True, False, True): (1, 256, 3, 1), + (1024, 1024, 65536, 32, 32, False, True, True): (1, 512, 3, 2), + (1024, 1024, 65536, 32, 32, True, False, True): (1, 256, 3, 4), + (1024, 1024, 65536, 64, 64, False, True, True): (2, 256, 2, 4), + (1024, 1024, 65536, 64, 64, True, False, True): (1, 512, 3, 4), + (1024, 1024, 65536, 128, 128, False, True, True): (10, 512, 1, 4), + (1024, 1024, 65536, 128, 128, True, False, True): (4, 512, 1, 4), + (1024, 1024, 65792, 16, 16, False, True, True): (1, 257, 1, 4), + (1024, 1024, 65792, 16, 16, True, False, True): (10, 257, 4, 1), + (1024, 1024, 65792, 32, 32, False, True, True): (2, 257, 1, 4), + (1024, 1024, 65792, 32, 32, True, False, True): (1, 257, 3, 4), + (1024, 1024, 65792, 64, 64, False, True, True): (2, 514, 1, 4), + (1024, 1024, 65792, 64, 64, True, False, True): (2, 257, 2, 4), + (1024, 1024, 65792, 128, 128, False, True, True): (6, 514, 1, 4), + (1024, 1024, 65792, 128, 128, True, False, True): (2, 514, 2, 4), + (1024, 1024, 131072, 16, 16, False, True, True): (11, 512, 3, 2), + (1024, 1024, 131072, 16, 16, True, False, True): (11, 512, 3, 2), + (1024, 1024, 131072, 32, 32, False, True, True): (7, 1024, 3, 2), + (1024, 1024, 131072, 32, 32, True, False, True): (6, 512, 3, 4), + (1024, 1024, 131072, 64, 64, False, True, True): (1, 512, 2, 4), + (1024, 1024, 131072, 64, 64, True, False, True): (4, 1024, 3, 4), + (1024, 1024, 131072, 128, 128, False, True, True): (12, 1024, 1, 4), + (1024, 1024, 131072, 128, 128, True, False, True): (4, 1024, 1, 4), + (1280, 5120, 65792, 16, 16, False, True, True): (1, 257, 1, 4), + (1280, 5120, 65792, 16, 16, True, False, True): (5, 257, 4, 1), + (1280, 5120, 65792, 32, 32, False, True, True): (2, 257, 1, 4), + (1280, 5120, 65792, 32, 32, True, False, True): (2, 257, 3, 4), + (1280, 5120, 65792, 64, 64, False, True, True): (1, 514, 3, 4), + (1280, 5120, 65792, 64, 64, True, False, True): (2, 257, 3, 4), + (1280, 5120, 65792, 128, 128, False, True, True): (1, 514, 3, 8), + (1280, 5120, 65792, 128, 128, True, False, True): (1, 514, 3, 8), + (1536, 1536, 256, 16, 16, False, True, True): (5, 4, 4, 2), + (1536, 1536, 256, 16, 16, True, False, True): (3, 4, 5, 2), + (1536, 1536, 256, 32, 32, False, True, True): (2, 4, 4, 4), + (1536, 1536, 256, 32, 32, True, False, True): (1, 4, 6, 2), + (1536, 1536, 256, 64, 64, False, True, True): (5, 4, 4, 4), + (1536, 1536, 256, 64, 64, True, False, True): (2, 4, 4, 4), + (1536, 1536, 256, 128, 128, False, True, True): (1, 2, 3, 8), + (1536, 1536, 256, 128, 128, True, False, True): (2, 2, 3, 8), + (1536, 1536, 512, 16, 16, False, True, True): (1, 8, 1, 4), + (1536, 1536, 512, 16, 16, True, False, True): (3, 4, 4, 2), + (1536, 1536, 512, 32, 32, False, True, True): (1, 8, 1, 8), + (1536, 1536, 512, 32, 32, True, False, True): (1, 4, 4, 4), + (1536, 1536, 512, 64, 64, False, True, True): (3, 8, 3, 4), + (1536, 1536, 512, 64, 64, True, False, True): (5, 8, 3, 4), + (1536, 1536, 512, 128, 128, False, True, True): (3, 4, 3, 8), + (1536, 1536, 512, 128, 128, True, False, True): (1, 4, 3, 8), + (1536, 1536, 1024, 16, 16, False, True, True): (6, 8, 1, 2), + (1536, 1536, 1024, 16, 16, True, False, True): (2, 8, 5, 2), + (1536, 1536, 1024, 32, 32, False, True, True): (6, 8, 1, 8), + (1536, 1536, 1024, 32, 32, True, False, True): (2, 4, 3, 4), + (1536, 1536, 1024, 64, 64, False, True, True): (1, 16, 3, 4), + (1536, 1536, 1024, 64, 64, True, False, True): (3, 8, 3, 4), + (1536, 1536, 1024, 128, 128, False, True, True): (3, 8, 3, 8), + (1536, 1536, 1024, 128, 128, True, False, True): (3, 8, 3, 8), + (1536, 1536, 2048, 16, 16, False, True, True): (1, 16, 1, 4), + (1536, 1536, 2048, 16, 16, True, False, True): (1, 8, 3, 1), + (1536, 1536, 2048, 32, 32, False, True, True): (1, 16, 1, 8), + (1536, 1536, 2048, 32, 32, True, False, True): (4, 8, 3, 2), + (1536, 1536, 2048, 64, 64, False, True, True): (1, 16, 3, 4), + (1536, 1536, 2048, 64, 64, True, False, True): (3, 8, 3, 4), + (1536, 1536, 2048, 128, 128, False, True, True): (6, 16, 1, 4), + (1536, 1536, 2048, 128, 128, True, False, True): (4, 16, 3, 8), + (1536, 1536, 4096, 16, 16, False, True, True): (1, 32, 1, 2), + (1536, 1536, 4096, 16, 16, True, False, True): (4, 32, 4, 2), + (1536, 1536, 4096, 32, 32, False, True, True): (1, 32, 1, 8), + (1536, 1536, 4096, 32, 32, True, False, True): (3, 16, 3, 4), + (1536, 1536, 4096, 64, 64, False, True, True): (1, 32, 3, 4), + (1536, 1536, 4096, 64, 64, True, False, True): (1, 16, 3, 4), + (1536, 1536, 4096, 128, 128, False, True, True): (4, 32, 3, 8), + (1536, 1536, 4096, 128, 128, True, False, True): (2, 32, 3, 8), + (1536, 1536, 8192, 16, 16, False, True, True): (2, 64, 1, 2), + (1536, 1536, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (1536, 1536, 8192, 32, 32, False, True, True): (1, 64, 1, 8), + (1536, 1536, 8192, 32, 32, True, False, True): (12, 32, 3, 4), + (1536, 1536, 8192, 64, 64, False, True, True): (2, 64, 3, 4), + (1536, 1536, 8192, 64, 64, True, False, True): (2, 32, 3, 4), + (1536, 1536, 8192, 128, 128, False, True, True): (3, 64, 1, 4), + (1536, 1536, 8192, 128, 128, True, False, True): (4, 64, 3, 8), + (1536, 1536, 16384, 16, 16, False, True, True): (1, 128, 1, 2), + (1536, 1536, 16384, 16, 16, True, False, True): (1, 64, 4, 4), + (1536, 1536, 16384, 32, 32, False, True, True): (1, 64, 1, 2), + (1536, 1536, 16384, 32, 32, True, False, True): (1, 64, 3, 4), + (1536, 1536, 16384, 64, 64, False, True, True): (1, 128, 3, 4), + (1536, 1536, 16384, 64, 64, True, False, True): (1, 64, 3, 4), + (1536, 1536, 16384, 128, 128, False, True, True): (3, 128, 1, 4), + (1536, 1536, 16384, 128, 128, True, False, True): (1, 128, 2, 4), + (1536, 1536, 32768, 16, 16, False, True, True): (1, 256, 1, 2), + (1536, 1536, 32768, 16, 16, True, False, True): (1, 128, 3, 2), + (1536, 1536, 32768, 32, 32, False, True, True): (1, 128, 1, 2), + (1536, 1536, 32768, 32, 32, True, False, True): (1, 128, 3, 4), + (1536, 1536, 32768, 64, 64, False, True, True): (3, 256, 3, 4), + (1536, 1536, 32768, 64, 64, True, False, True): (1, 128, 3, 4), + (1536, 1536, 32768, 128, 128, False, True, True): (3, 256, 1, 4), + (1536, 1536, 32768, 128, 128, True, False, True): (1, 256, 2, 4), + (1536, 1536, 65536, 16, 16, False, True, True): (4, 512, 1, 2), + (1536, 1536, 65536, 16, 16, True, False, True): (1, 256, 4, 4), + (1536, 1536, 65536, 32, 32, False, True, True): (1, 256, 1, 2), + (1536, 1536, 65536, 32, 32, True, False, True): (1, 256, 3, 4), + (1536, 1536, 65536, 64, 64, False, True, True): (2, 512, 3, 4), + (1536, 1536, 65536, 64, 64, True, False, True): (1, 256, 3, 4), + (1536, 1536, 65536, 128, 128, False, True, True): (3, 512, 1, 4), + (1536, 1536, 65536, 128, 128, True, False, True): (2, 512, 2, 4), + (1536, 1536, 131072, 16, 16, False, True, True): (2, 1024, 1, 2), + (1536, 1536, 131072, 16, 16, True, False, True): (9, 512, 4, 4), + (1536, 1536, 131072, 32, 32, False, True, True): (1, 512, 1, 2), + (1536, 1536, 131072, 32, 32, True, False, True): (9, 512, 3, 4), + (1536, 1536, 131072, 64, 64, False, True, True): (1, 1024, 3, 4), + (1536, 1536, 131072, 64, 64, True, False, True): (1, 512, 3, 4), + (1536, 1536, 131072, 128, 128, False, True, True): (3, 1024, 1, 4), + (1536, 1536, 131072, 128, 128, True, False, True): (1, 1024, 2, 4), + (2048, 2048, 256, 16, 16, False, True, True): (4, 4, 6, 2), + (2048, 2048, 256, 16, 16, True, False, True): (2, 8, 4, 1), + (2048, 2048, 256, 32, 32, False, True, True): (3, 4, 4, 2), + (2048, 2048, 256, 32, 32, True, False, True): (1, 4, 5, 2), + (2048, 2048, 256, 64, 64, False, True, True): (2, 4, 4, 4), + (2048, 2048, 256, 64, 64, True, False, True): (2, 4, 4, 4), + (2048, 2048, 256, 128, 128, False, True, True): (3, 2, 2, 8), + (2048, 2048, 256, 128, 128, True, False, True): (5, 2, 2, 8), + (2048, 2048, 512, 16, 16, False, True, True): (5, 4, 4, 4), + (2048, 2048, 512, 16, 16, True, False, True): (2, 4, 4, 2), + (2048, 2048, 512, 32, 32, False, True, True): (1, 4, 3, 4), + (2048, 2048, 512, 32, 32, True, False, True): (3, 4, 4, 2), + (2048, 2048, 512, 64, 64, False, True, True): (1, 8, 3, 4), + (2048, 2048, 512, 64, 64, True, False, True): (1, 8, 3, 2), + (2048, 2048, 512, 128, 128, False, True, True): (3, 4, 2, 8), + (2048, 2048, 512, 128, 128, True, False, True): (2, 4, 2, 8), + (2048, 2048, 1024, 16, 16, False, True, True): (3, 4, 3, 4), + (2048, 2048, 1024, 16, 16, True, False, True): (2, 8, 3, 2), + (2048, 2048, 1024, 32, 32, False, True, True): (3, 8, 3, 4), + (2048, 2048, 1024, 32, 32, True, False, True): (1, 8, 3, 2), + (2048, 2048, 1024, 64, 64, False, True, True): (1, 8, 3, 4), + (2048, 2048, 1024, 64, 64, True, False, True): (1, 8, 3, 4), + (2048, 2048, 1024, 128, 128, False, True, True): (4, 8, 2, 8), + (2048, 2048, 1024, 128, 128, True, False, True): (4, 8, 1, 4), + (2048, 2048, 2048, 16, 16, False, True, True): (4, 16, 3, 2), + (2048, 2048, 2048, 16, 16, True, False, True): (2, 16, 3, 2), + (2048, 2048, 2048, 32, 32, False, True, True): (1, 16, 3, 4), + (2048, 2048, 2048, 32, 32, True, False, True): (1, 16, 3, 2), + (2048, 2048, 2048, 64, 64, False, True, True): (1, 16, 3, 4), + (2048, 2048, 2048, 64, 64, True, False, True): (1, 16, 3, 4), + (2048, 2048, 2048, 128, 128, False, True, True): (6, 16, 2, 8), + (2048, 2048, 2048, 128, 128, True, False, True): (5, 16, 1, 4), + (2048, 2048, 4096, 16, 16, False, True, True): (4, 32, 4, 2), + (2048, 2048, 4096, 16, 16, True, False, True): (4, 32, 3, 2), + (2048, 2048, 4096, 32, 32, False, True, True): (4, 16, 3, 8), + (2048, 2048, 4096, 32, 32, True, False, True): (4, 16, 3, 4), + (2048, 2048, 4096, 64, 64, False, True, True): (4, 32, 3, 4), + (2048, 2048, 4096, 64, 64, True, False, True): (4, 32, 3, 4), + (2048, 2048, 4096, 128, 128, False, True, True): (4, 32, 2, 8), + (2048, 2048, 4096, 128, 128, True, False, True): (2, 32, 1, 4), + (2048, 2048, 8192, 16, 16, False, True, True): (4, 64, 4, 2), + (2048, 2048, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (2048, 2048, 8192, 32, 32, False, True, True): (4, 32, 3, 8), + (2048, 2048, 8192, 32, 32, True, False, True): (4, 32, 4, 8), + (2048, 2048, 8192, 64, 64, False, True, True): (2, 64, 3, 4), + (2048, 2048, 8192, 64, 64, True, False, True): (4, 64, 3, 4), + (2048, 2048, 8192, 128, 128, False, True, True): (3, 64, 1, 4), + (2048, 2048, 8192, 128, 128, True, False, True): (2, 64, 1, 4), + (2048, 2048, 16384, 16, 16, False, True, True): (4, 64, 3, 4), + (2048, 2048, 16384, 16, 16, True, False, True): (1, 64, 3, 4), + (2048, 2048, 16384, 32, 32, False, True, True): (4, 64, 3, 4), + (2048, 2048, 16384, 32, 32, True, False, True): (4, 64, 3, 4), + (2048, 2048, 16384, 64, 64, False, True, True): (4, 128, 3, 4), + (2048, 2048, 16384, 64, 64, True, False, True): (4, 128, 3, 4), + (2048, 2048, 16384, 128, 128, False, True, True): (3, 128, 1, 4), + (2048, 2048, 16384, 128, 128, True, False, True): (2, 128, 1, 4), + (2048, 2048, 32768, 16, 16, False, True, True): (8, 128, 3, 2), + (2048, 2048, 32768, 16, 16, True, False, True): (8, 128, 3, 4), + (2048, 2048, 32768, 32, 32, False, True, True): (8, 128, 3, 4), + (2048, 2048, 32768, 32, 32, True, False, True): (8, 128, 3, 4), + (2048, 2048, 32768, 64, 64, False, True, True): (8, 256, 3, 4), + (2048, 2048, 32768, 64, 64, True, False, True): (8, 256, 3, 4), + (2048, 2048, 32768, 128, 128, False, True, True): (3, 256, 1, 4), + (2048, 2048, 32768, 128, 128, True, False, True): (1, 256, 1, 4), + (2048, 2048, 50432, 16, 16, False, True, True): (1, 197, 1, 4), + (2048, 2048, 50432, 16, 16, True, False, True): (4, 197, 4, 1), + (2048, 2048, 50432, 32, 32, False, True, True): (2, 197, 1, 4), + (2048, 2048, 50432, 32, 32, True, False, True): (4, 197, 3, 4), + (2048, 2048, 50432, 64, 64, False, True, True): (2, 394, 3, 4), + (2048, 2048, 50432, 64, 64, True, False, True): (4, 197, 2, 4), + (2048, 2048, 50432, 128, 128, False, True, True): (3, 394, 1, 4), + (2048, 2048, 50432, 128, 128, True, False, True): (4, 394, 2, 4), + (2048, 2048, 65536, 16, 16, False, True, True): (9, 256, 3, 2), + (2048, 2048, 65536, 16, 16, True, False, True): (9, 256, 4, 4), + (2048, 2048, 65536, 32, 32, False, True, True): (7, 256, 3, 4), + (2048, 2048, 65536, 32, 32, True, False, True): (7, 256, 3, 4), + (2048, 2048, 65536, 64, 64, False, True, True): (2, 256, 2, 4), + (2048, 2048, 65536, 64, 64, True, False, True): (9, 512, 3, 4), + (2048, 2048, 65536, 128, 128, False, True, True): (5, 512, 1, 4), + (2048, 2048, 65536, 128, 128, True, False, True): (1, 512, 1, 4), + (2048, 2048, 65792, 16, 16, False, True, True): (1, 257, 1, 4), + (2048, 2048, 65792, 16, 16, True, False, True): (7, 257, 4, 1), + (2048, 2048, 65792, 32, 32, False, True, True): (2, 257, 1, 4), + (2048, 2048, 65792, 32, 32, True, False, True): (7, 257, 3, 4), + (2048, 2048, 65792, 64, 64, False, True, True): (1, 514, 3, 4), + (2048, 2048, 65792, 64, 64, True, False, True): (1, 257, 2, 4), + (2048, 2048, 65792, 128, 128, False, True, True): (3, 514, 1, 4), + (2048, 2048, 65792, 128, 128, True, False, True): (1, 514, 2, 4), + (2048, 2048, 131072, 16, 16, False, True, True): (9, 512, 3, 2), + (2048, 2048, 131072, 16, 16, True, False, True): (9, 512, 4, 4), + (2048, 2048, 131072, 32, 32, False, True, True): (7, 512, 3, 4), + (2048, 2048, 131072, 32, 32, True, False, True): (3, 512, 3, 4), + (2048, 2048, 131072, 64, 64, False, True, True): (1, 512, 2, 4), + (2048, 2048, 131072, 64, 64, True, False, True): (2, 1024, 3, 4), + (2048, 2048, 131072, 128, 128, False, True, True): (3, 1024, 1, 4), + (2048, 2048, 131072, 128, 128, True, False, True): (1, 1024, 1, 4), + (3072, 768, 256, 16, 16, False, True, True): (6, 4, 1, 4), + (3072, 768, 256, 16, 16, True, False, True): (2, 1, 5, 2), + (3072, 768, 256, 32, 32, False, True, True): (1, 4, 1, 8), + (3072, 768, 256, 32, 32, True, False, True): (4, 2, 4, 4), + (3072, 768, 256, 64, 64, False, True, True): (1, 2, 3, 4), + (3072, 768, 256, 64, 64, True, False, True): (3, 4, 3, 4), + (3072, 768, 256, 128, 128, False, True, True): (1, 2, 3, 8), + (3072, 768, 256, 128, 128, True, False, True): (3, 2, 3, 8), + (3072, 768, 512, 16, 16, False, True, True): (1, 4, 1, 4), + (3072, 768, 512, 16, 16, True, False, True): (3, 4, 4, 1), + (3072, 768, 512, 32, 32, False, True, True): (5, 8, 1, 4), + (3072, 768, 512, 32, 32, True, False, True): (3, 4, 4, 2), + (3072, 768, 512, 64, 64, False, True, True): (1, 8, 1, 4), + (3072, 768, 512, 64, 64, True, False, True): (1, 4, 3, 4), + (3072, 768, 512, 128, 128, False, True, True): (3, 4, 3, 8), + (3072, 768, 512, 128, 128, True, False, True): (1, 4, 3, 8), + (3072, 768, 1024, 16, 16, False, True, True): (1, 8, 1, 4), + (3072, 768, 1024, 16, 16, True, False, True): (3, 4, 3, 1), + (3072, 768, 1024, 32, 32, False, True, True): (1, 16, 1, 4), + (3072, 768, 1024, 32, 32, True, False, True): (1, 4, 3, 8), + (3072, 768, 1024, 64, 64, False, True, True): (8, 16, 3, 2), + (3072, 768, 1024, 64, 64, True, False, True): (1, 4, 3, 4), + (3072, 768, 1024, 128, 128, False, True, True): (2, 8, 3, 8), + (3072, 768, 1024, 128, 128, True, False, True): (3, 8, 2, 4), + (3072, 768, 2048, 16, 16, False, True, True): (1, 8, 1, 4), + (3072, 768, 2048, 16, 16, True, False, True): (6, 8, 4, 4), + (3072, 768, 2048, 32, 32, False, True, True): (1, 16, 1, 8), + (3072, 768, 2048, 32, 32, True, False, True): (6, 8, 3, 4), + (3072, 768, 2048, 64, 64, False, True, True): (8, 16, 3, 4), + (3072, 768, 2048, 64, 64, True, False, True): (3, 16, 3, 4), + (3072, 768, 2048, 128, 128, False, True, True): (1, 16, 3, 8), + (3072, 768, 2048, 128, 128, True, False, True): (2, 16, 2, 4), + (3072, 768, 4096, 16, 16, False, True, True): (1, 16, 1, 4), + (3072, 768, 4096, 16, 16, True, False, True): (4, 32, 4, 2), + (3072, 768, 4096, 32, 32, False, True, True): (1, 32, 1, 8), + (3072, 768, 4096, 32, 32, True, False, True): (4, 16, 3, 4), + (3072, 768, 4096, 64, 64, False, True, True): (2, 32, 1, 4), + (3072, 768, 4096, 64, 64, True, False, True): (2, 16, 2, 4), + (3072, 768, 4096, 128, 128, False, True, True): (2, 32, 1, 16), + (3072, 768, 4096, 128, 128, True, False, True): (3, 32, 2, 4), + (3072, 768, 8192, 16, 16, False, True, True): (2, 32, 1, 4), + (3072, 768, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (3072, 768, 8192, 32, 32, False, True, True): (2, 32, 1, 4), + (3072, 768, 8192, 32, 32, True, False, True): (6, 32, 3, 4), + (3072, 768, 8192, 64, 64, False, True, True): (2, 64, 1, 4), + (3072, 768, 8192, 64, 64, True, False, True): (2, 32, 2, 4), + (3072, 768, 8192, 128, 128, False, True, True): (3, 64, 1, 4), + (3072, 768, 8192, 128, 128, True, False, True): (2, 64, 2, 4), + (3072, 768, 16384, 16, 16, False, True, True): (1, 64, 1, 4), + (3072, 768, 16384, 16, 16, True, False, True): (1, 64, 1, 1), + (3072, 768, 16384, 32, 32, False, True, True): (2, 64, 1, 4), + (3072, 768, 16384, 32, 32, True, False, True): (4, 64, 3, 4), + (3072, 768, 16384, 64, 64, False, True, True): (2, 128, 1, 4), + (3072, 768, 16384, 64, 64, True, False, True): (4, 64, 2, 4), + (3072, 768, 16384, 128, 128, False, True, True): (3, 128, 1, 4), + (3072, 768, 16384, 128, 128, True, False, True): (1, 128, 2, 4), + (3072, 768, 32768, 16, 16, False, True, True): (1, 128, 1, 4), + (3072, 768, 32768, 16, 16, True, False, True): (8, 256, 3, 2), + (3072, 768, 32768, 32, 32, False, True, True): (2, 128, 1, 4), + (3072, 768, 32768, 32, 32, True, False, True): (8, 128, 3, 4), + (3072, 768, 32768, 64, 64, False, True, True): (1, 256, 1, 4), + (3072, 768, 32768, 64, 64, True, False, True): (8, 128, 2, 4), + (3072, 768, 32768, 128, 128, False, True, True): (3, 256, 1, 4), + (3072, 768, 32768, 128, 128, True, False, True): (3, 256, 2, 4), + (3072, 768, 50432, 16, 16, False, True, True): (1, 197, 1, 4), + (3072, 768, 50432, 16, 16, True, False, True): (7, 197, 4, 1), + (3072, 768, 50432, 32, 32, False, True, True): (2, 197, 1, 4), + (3072, 768, 50432, 32, 32, True, False, True): (10, 197, 3, 4), + (3072, 768, 50432, 64, 64, False, True, True): (1, 394, 1, 4), + (3072, 768, 50432, 64, 64, True, False, True): (3, 197, 2, 4), + (3072, 768, 50432, 128, 128, False, True, True): (3, 394, 1, 4), + (3072, 768, 50432, 128, 128, True, False, True): (2, 394, 2, 4), + (3072, 768, 65536, 16, 16, False, True, True): (1, 256, 1, 4), + (3072, 768, 65536, 16, 16, True, False, True): (15, 256, 4, 1), + (3072, 768, 65536, 32, 32, False, True, True): (2, 256, 1, 4), + (3072, 768, 65536, 32, 32, True, False, True): (10, 256, 3, 4), + (3072, 768, 65536, 64, 64, False, True, True): (1, 512, 1, 4), + (3072, 768, 65536, 64, 64, True, False, True): (3, 256, 2, 4), + (3072, 768, 65536, 128, 128, False, True, True): (3, 512, 1, 4), + (3072, 768, 65536, 128, 128, True, False, True): (3, 512, 2, 4), + (3072, 768, 131072, 16, 16, False, True, True): (1, 512, 1, 4), + (3072, 768, 131072, 16, 16, True, False, True): (15, 512, 4, 1), + (3072, 768, 131072, 32, 32, False, True, True): (2, 512, 1, 4), + (3072, 768, 131072, 32, 32, True, False, True): (9, 512, 3, 4), + (3072, 768, 131072, 64, 64, False, True, True): (1, 1024, 1, 4), + (3072, 768, 131072, 64, 64, True, False, True): (3, 512, 2, 4), + (3072, 768, 131072, 128, 128, False, True, True): (3, 1024, 1, 4), + (3072, 768, 131072, 128, 128, True, False, True): (3, 1024, 2, 4), + (3072, 3072, 256, 16, 16, False, True, True): (5, 4, 1, 4), + (3072, 3072, 256, 16, 16, True, False, True): (1, 2, 5, 2), + (3072, 3072, 256, 32, 32, False, True, True): (1, 4, 1, 8), + (3072, 3072, 256, 32, 32, True, False, True): (3, 4, 4, 2), + (3072, 3072, 256, 64, 64, False, True, True): (2, 4, 3, 4), + (3072, 3072, 256, 64, 64, True, False, True): (3, 4, 4, 4), + (3072, 3072, 256, 128, 128, False, True, True): (1, 2, 3, 8), + (3072, 3072, 256, 128, 128, True, False, True): (1, 2, 3, 8), + (3072, 3072, 512, 16, 16, False, True, True): (5, 4, 1, 2), + (3072, 3072, 512, 16, 16, True, False, True): (1, 2, 4, 4), + (3072, 3072, 512, 32, 32, False, True, True): (3, 8, 1, 4), + (3072, 3072, 512, 32, 32, True, False, True): (4, 2, 3, 4), + (3072, 3072, 512, 64, 64, False, True, True): (1, 8, 2, 2), + (3072, 3072, 512, 64, 64, True, False, True): (2, 4, 3, 4), + (3072, 3072, 512, 128, 128, False, True, True): (1, 4, 3, 8), + (3072, 3072, 512, 128, 128, True, False, True): (4, 4, 3, 8), + (3072, 3072, 1024, 16, 16, False, True, True): (1, 8, 1, 4), + (3072, 3072, 1024, 16, 16, True, False, True): (4, 8, 5, 2), + (3072, 3072, 1024, 32, 32, False, True, True): (1, 8, 1, 8), + (3072, 3072, 1024, 32, 32, True, False, True): (1, 4, 4, 4), + (3072, 3072, 1024, 64, 64, False, True, True): (3, 8, 3, 4), + (3072, 3072, 1024, 64, 64, True, False, True): (2, 4, 3, 4), + (3072, 3072, 1024, 128, 128, False, True, True): (3, 8, 1, 4), + (3072, 3072, 1024, 128, 128, True, False, True): (1, 8, 3, 8), + (3072, 3072, 2048, 16, 16, False, True, True): (1, 16, 1, 2), + (3072, 3072, 2048, 16, 16, True, False, True): (4, 16, 4, 2), + (3072, 3072, 2048, 32, 32, False, True, True): (1, 16, 1, 8), + (3072, 3072, 2048, 32, 32, True, False, True): (3, 8, 4, 4), + (3072, 3072, 2048, 64, 64, False, True, True): (3, 16, 3, 4), + (3072, 3072, 2048, 64, 64, True, False, True): (3, 8, 3, 4), + (3072, 3072, 2048, 128, 128, False, True, True): (4, 16, 3, 8), + (3072, 3072, 2048, 128, 128, True, False, True): (3, 16, 3, 8), + (3072, 3072, 4096, 16, 16, False, True, True): (1, 32, 1, 2), + (3072, 3072, 4096, 16, 16, True, False, True): (4, 32, 4, 2), + (3072, 3072, 4096, 32, 32, False, True, True): (1, 32, 1, 8), + (3072, 3072, 4096, 32, 32, True, False, True): (3, 16, 3, 4), + (3072, 3072, 4096, 64, 64, False, True, True): (1, 32, 3, 4), + (3072, 3072, 4096, 64, 64, True, False, True): (3, 16, 3, 4), + (3072, 3072, 4096, 128, 128, False, True, True): (1, 32, 3, 8), + (3072, 3072, 4096, 128, 128, True, False, True): (3, 32, 3, 8), + (3072, 3072, 8192, 16, 16, False, True, True): (1, 64, 1, 2), + (3072, 3072, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (3072, 3072, 8192, 32, 32, False, True, True): (1, 64, 1, 8), + (3072, 3072, 8192, 32, 32, True, False, True): (8, 32, 3, 4), + (3072, 3072, 8192, 64, 64, False, True, True): (3, 64, 3, 4), + (3072, 3072, 8192, 64, 64, True, False, True): (2, 32, 3, 4), + (3072, 3072, 8192, 128, 128, False, True, True): (2, 64, 3, 8), + (3072, 3072, 8192, 128, 128, True, False, True): (1, 64, 3, 8), + (3072, 3072, 16384, 16, 16, False, True, True): (1, 128, 1, 2), + (3072, 3072, 16384, 16, 16, True, False, True): (4, 128, 4, 2), + (3072, 3072, 16384, 32, 32, False, True, True): (1, 64, 1, 2), + (3072, 3072, 16384, 32, 32, True, False, True): (4, 64, 3, 4), + (3072, 3072, 16384, 64, 64, False, True, True): (1, 128, 3, 4), + (3072, 3072, 16384, 64, 64, True, False, True): (4, 64, 3, 4), + (3072, 3072, 16384, 128, 128, False, True, True): (3, 128, 1, 4), + (3072, 3072, 16384, 128, 128, True, False, True): (1, 128, 3, 8), + (3072, 3072, 32768, 16, 16, False, True, True): (1, 256, 1, 2), + (3072, 3072, 32768, 16, 16, True, False, True): (8, 128, 4, 4), + (3072, 3072, 32768, 32, 32, False, True, True): (1, 256, 1, 8), + (3072, 3072, 32768, 32, 32, True, False, True): (5, 128, 3, 4), + (3072, 3072, 32768, 64, 64, False, True, True): (1, 256, 3, 4), + (3072, 3072, 32768, 64, 64, True, False, True): (1, 128, 3, 4), + (3072, 3072, 32768, 128, 128, False, True, True): (3, 256, 1, 4), + (3072, 3072, 32768, 128, 128, True, False, True): (3, 256, 2, 4), + (3072, 3072, 65536, 16, 16, False, True, True): (1, 512, 1, 2), + (3072, 3072, 65536, 16, 16, True, False, True): (7, 256, 4, 4), + (3072, 3072, 65536, 32, 32, False, True, True): (1, 256, 1, 2), + (3072, 3072, 65536, 32, 32, True, False, True): (5, 256, 3, 4), + (3072, 3072, 65536, 64, 64, False, True, True): (1, 512, 3, 4), + (3072, 3072, 65536, 64, 64, True, False, True): (3, 256, 3, 4), + (3072, 3072, 65536, 128, 128, False, True, True): (3, 512, 1, 4), + (3072, 3072, 65536, 128, 128, True, False, True): (3, 512, 2, 4), + (3072, 3072, 131072, 16, 16, False, True, True): (1, 1024, 1, 2), + (3072, 3072, 131072, 16, 16, True, False, True): (5, 512, 4, 4), + (3072, 3072, 131072, 32, 32, False, True, True): (1, 512, 1, 2), + (3072, 3072, 131072, 32, 32, True, False, True): (3, 512, 3, 4), + (3072, 3072, 131072, 64, 64, False, True, True): (1, 1024, 3, 4), + (3072, 3072, 131072, 64, 64, True, False, True): (3, 512, 3, 4), + (3072, 3072, 131072, 128, 128, False, True, True): (3, 1024, 1, 4), + (3072, 3072, 131072, 128, 128, True, False, True): (1, 1024, 2, 4), + (4096, 4096, 256, 16, 16, False, True, True): (2, 2, 6, 4), + (4096, 4096, 256, 16, 16, True, False, True): (2, 2, 5, 4), + (4096, 4096, 256, 32, 32, False, True, True): (7, 2, 4, 4), + (4096, 4096, 256, 32, 32, True, False, True): (1, 2, 4, 4), + (4096, 4096, 256, 64, 64, False, True, True): (3, 4, 3, 4), + (4096, 4096, 256, 64, 64, True, False, True): (3, 4, 3, 4), + (4096, 4096, 256, 128, 128, False, True, True): (1, 2, 2, 8), + (4096, 4096, 256, 128, 128, True, False, True): (1, 2, 2, 8), + (4096, 4096, 512, 16, 16, False, True, True): (4, 2, 3, 4), + (4096, 4096, 512, 16, 16, True, False, True): (2, 4, 3, 2), + (4096, 4096, 512, 32, 32, False, True, True): (3, 4, 3, 4), + (4096, 4096, 512, 32, 32, True, False, True): (3, 4, 3, 2), + (4096, 4096, 512, 64, 64, False, True, True): (3, 4, 3, 4), + (4096, 4096, 512, 64, 64, True, False, True): (3, 4, 3, 4), + (4096, 4096, 512, 128, 128, False, True, True): (2, 4, 2, 8), + (4096, 4096, 512, 128, 128, True, False, True): (2, 4, 1, 4), + (4096, 4096, 1024, 16, 16, False, True, True): (2, 8, 3, 2), + (4096, 4096, 1024, 16, 16, True, False, True): (2, 8, 3, 2), + (4096, 4096, 1024, 32, 32, False, True, True): (3, 8, 3, 4), + (4096, 4096, 1024, 32, 32, True, False, True): (1, 8, 3, 2), + (4096, 4096, 1024, 64, 64, False, True, True): (1, 8, 3, 4), + (4096, 4096, 1024, 64, 64, True, False, True): (1, 8, 3, 4), + (4096, 4096, 1024, 128, 128, False, True, True): (2, 8, 2, 8), + (4096, 4096, 1024, 128, 128, True, False, True): (2, 8, 2, 8), + (4096, 4096, 2048, 16, 16, False, True, True): (2, 8, 4, 4), + (4096, 4096, 2048, 16, 16, True, False, True): (2, 8, 4, 4), + (4096, 4096, 2048, 32, 32, False, True, True): (4, 8, 4, 8), + (4096, 4096, 2048, 32, 32, True, False, True): (4, 8, 4, 8), + (4096, 4096, 2048, 64, 64, False, True, True): (1, 16, 3, 4), + (4096, 4096, 2048, 64, 64, True, False, True): (4, 16, 3, 4), + (4096, 4096, 2048, 128, 128, False, True, True): (2, 16, 2, 8), + (4096, 4096, 2048, 128, 128, True, False, True): (4, 16, 1, 4), + (4096, 4096, 4096, 16, 16, False, True, True): (4, 32, 4, 4), + (4096, 4096, 4096, 16, 16, True, False, True): (4, 32, 4, 2), + (4096, 4096, 4096, 32, 32, False, True, True): (4, 16, 4, 8), + (4096, 4096, 4096, 32, 32, True, False, True): (4, 16, 3, 8), + (4096, 4096, 4096, 64, 64, False, True, True): (1, 32, 3, 4), + (4096, 4096, 4096, 64, 64, True, False, True): (1, 32, 3, 4), + (4096, 4096, 4096, 128, 128, False, True, True): (3, 32, 1, 4), + (4096, 4096, 4096, 128, 128, True, False, True): (2, 32, 1, 4), + (4096, 4096, 8192, 16, 16, False, True, True): (4, 64, 4, 2), + (4096, 4096, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (4096, 4096, 8192, 32, 32, False, True, True): (4, 32, 4, 8), + (4096, 4096, 8192, 32, 32, True, False, True): (4, 32, 4, 8), + (4096, 4096, 8192, 64, 64, False, True, True): (2, 64, 3, 4), + (4096, 4096, 8192, 64, 64, True, False, True): (2, 64, 3, 4), + (4096, 4096, 8192, 128, 128, False, True, True): (3, 64, 1, 4), + (4096, 4096, 8192, 128, 128, True, False, True): (1, 64, 1, 4), + (4096, 4096, 16384, 16, 16, False, True, True): (4, 64, 3, 4), + (4096, 4096, 16384, 16, 16, True, False, True): (4, 64, 4, 4), + (4096, 4096, 16384, 32, 32, False, True, True): (4, 64, 4, 8), + (4096, 4096, 16384, 32, 32, True, False, True): (4, 64, 4, 8), + (4096, 4096, 16384, 64, 64, False, True, True): (1, 64, 2, 4), + (4096, 4096, 16384, 64, 64, True, False, True): (1, 64, 3, 8), + (4096, 4096, 16384, 128, 128, False, True, True): (3, 128, 1, 4), + (4096, 4096, 16384, 128, 128, True, False, True): (1, 128, 1, 4), + (4096, 4096, 32768, 16, 16, False, True, True): (8, 128, 3, 2), + (4096, 4096, 32768, 16, 16, True, False, True): (5, 128, 4, 4), + (4096, 4096, 32768, 32, 32, False, True, True): (3, 128, 4, 4), + (4096, 4096, 32768, 32, 32, True, False, True): (3, 128, 4, 8), + (4096, 4096, 32768, 64, 64, False, True, True): (1, 128, 2, 4), + (4096, 4096, 32768, 64, 64, True, False, True): (3, 256, 3, 4), + (4096, 4096, 32768, 128, 128, False, True, True): (3, 256, 1, 4), + (4096, 4096, 32768, 128, 128, True, False, True): (1, 256, 1, 4), + (4096, 4096, 50432, 16, 16, False, True, True): (1, 197, 1, 4), + (4096, 4096, 50432, 16, 16, True, False, True): (4, 197, 4, 1), + (4096, 4096, 50432, 32, 32, False, True, True): (1, 197, 1, 4), + (4096, 4096, 50432, 32, 32, True, False, True): (2, 197, 3, 4), + (4096, 4096, 50432, 64, 64, False, True, True): (1, 394, 3, 4), + (4096, 4096, 50432, 64, 64, True, False, True): (1, 197, 2, 4), + (4096, 4096, 50432, 128, 128, False, True, True): (3, 394, 1, 4), + (4096, 4096, 50432, 128, 128, True, False, True): (1, 394, 2, 4), + (4096, 4096, 65536, 16, 16, False, True, True): (5, 256, 4, 4), + (4096, 4096, 65536, 16, 16, True, False, True): (5, 256, 4, 4), + (4096, 4096, 65536, 32, 32, False, True, True): (4, 256, 4, 8), + (4096, 4096, 65536, 32, 32, True, False, True): (4, 256, 3, 8), + (4096, 4096, 65536, 64, 64, False, True, True): (1, 256, 2, 4), + (4096, 4096, 65536, 64, 64, True, False, True): (1, 512, 3, 4), + (4096, 4096, 65536, 128, 128, False, True, True): (3, 512, 1, 4), + (4096, 4096, 65536, 128, 128, True, False, True): (1, 512, 1, 4), + (4096, 4096, 65792, 16, 16, False, True, True): (1, 257, 1, 4), + (4096, 4096, 65792, 16, 16, True, False, True): (5, 257, 4, 1), + (4096, 4096, 65792, 32, 32, False, True, True): (1, 257, 1, 4), + (4096, 4096, 65792, 32, 32, True, False, True): (1, 257, 3, 4), + (4096, 4096, 65792, 64, 64, False, True, True): (1, 514, 3, 4), + (4096, 4096, 65792, 64, 64, True, False, True): (1, 257, 2, 4), + (4096, 4096, 65792, 128, 128, False, True, True): (3, 514, 1, 4), + (4096, 4096, 65792, 128, 128, True, False, True): (1, 514, 2, 4), + (4096, 4096, 131072, 16, 16, False, True, True): (4, 512, 3, 4), + (4096, 4096, 131072, 16, 16, True, False, True): (5, 512, 4, 4), + (4096, 4096, 131072, 32, 32, False, True, True): (1, 512, 4, 8), + (4096, 4096, 131072, 32, 32, True, False, True): (4, 512, 4, 8), + (4096, 4096, 131072, 64, 64, False, True, True): (1, 512, 2, 4), + (4096, 4096, 131072, 64, 64, True, False, True): (1, 512, 2, 4), + (4096, 4096, 131072, 128, 128, False, True, True): (3, 1024, 1, 4), + (4096, 4096, 131072, 128, 128, True, False, True): (1, 1024, 1, 4), + (5120, 1280, 65792, 16, 16, False, True, True): (1, 257, 1, 4), + (5120, 1280, 65792, 16, 16, True, False, True): (7, 257, 4, 1), + (5120, 1280, 65792, 32, 32, False, True, True): (2, 257, 1, 4), + (5120, 1280, 65792, 32, 32, True, False, True): (5, 257, 3, 4), + (5120, 1280, 65792, 64, 64, False, True, True): (1, 514, 1, 4), + (5120, 1280, 65792, 64, 64, True, False, True): (5, 257, 2, 4), + (5120, 1280, 65792, 128, 128, False, True, True): (3, 514, 1, 4), + (5120, 1280, 65792, 128, 128, True, False, True): (4, 514, 2, 4), + (6144, 6144, 256, 16, 16, False, True, True): (1, 2, 1, 4), + (6144, 6144, 256, 16, 16, True, False, True): (1, 1, 4, 4), + (6144, 6144, 256, 32, 32, False, True, True): (3, 2, 1, 8), + (6144, 6144, 256, 32, 32, True, False, True): (2, 1, 3, 4), + (6144, 6144, 256, 64, 64, False, True, True): (2, 2, 3, 4), + (6144, 6144, 256, 64, 64, True, False, True): (6, 2, 4, 4), + (6144, 6144, 256, 128, 128, False, True, True): (2, 2, 3, 8), + (6144, 6144, 256, 128, 128, True, False, True): (1, 2, 3, 8), + (6144, 6144, 512, 16, 16, False, True, True): (4, 4, 1, 4), + (6144, 6144, 512, 16, 16, True, False, True): (3, 2, 3, 1), + (6144, 6144, 512, 32, 32, False, True, True): (1, 8, 1, 4), + (6144, 6144, 512, 32, 32, True, False, True): (2, 2, 3, 8), + (6144, 6144, 512, 64, 64, False, True, True): (4, 4, 3, 4), + (6144, 6144, 512, 64, 64, True, False, True): (6, 2, 3, 4), + (6144, 6144, 512, 128, 128, False, True, True): (3, 4, 1, 4), + (6144, 6144, 512, 128, 128, True, False, True): (4, 4, 3, 8), + (6144, 6144, 1024, 16, 16, False, True, True): (1, 8, 1, 2), + (6144, 6144, 1024, 16, 16, True, False, True): (4, 8, 4, 2), + (6144, 6144, 1024, 32, 32, False, True, True): (1, 8, 4, 2), + (6144, 6144, 1024, 32, 32, True, False, True): (1, 8, 4, 2), + (6144, 6144, 1024, 64, 64, False, True, True): (4, 8, 3, 4), + (6144, 6144, 1024, 64, 64, True, False, True): (1, 4, 3, 4), + (6144, 6144, 1024, 128, 128, False, True, True): (3, 8, 1, 4), + (6144, 6144, 1024, 128, 128, True, False, True): (1, 8, 3, 8), + (6144, 6144, 2048, 16, 16, False, True, True): (4, 4, 1, 4), + (6144, 6144, 2048, 16, 16, True, False, True): (2, 8, 4, 4), + (6144, 6144, 2048, 32, 32, False, True, True): (4, 8, 3, 4), + (6144, 6144, 2048, 32, 32, True, False, True): (2, 8, 3, 4), + (6144, 6144, 2048, 64, 64, False, True, True): (4, 16, 3, 4), + (6144, 6144, 2048, 64, 64, True, False, True): (2, 8, 3, 4), + (6144, 6144, 2048, 128, 128, False, True, True): (3, 16, 1, 4), + (6144, 6144, 2048, 128, 128, True, False, True): (4, 16, 3, 8), + (6144, 6144, 4096, 16, 16, False, True, True): (4, 8, 1, 4), + (6144, 6144, 4096, 16, 16, True, False, True): (4, 32, 4, 2), + (6144, 6144, 4096, 32, 32, False, True, True): (4, 16, 1, 2), + (6144, 6144, 4096, 32, 32, True, False, True): (2, 8, 3, 8), + (6144, 6144, 4096, 64, 64, False, True, True): (4, 32, 3, 4), + (6144, 6144, 4096, 64, 64, True, False, True): (4, 16, 3, 4), + (6144, 6144, 4096, 128, 128, False, True, True): (6, 32, 1, 4), + (6144, 6144, 4096, 128, 128, True, False, True): (4, 32, 3, 8), + (6144, 6144, 8192, 16, 16, False, True, True): (2, 16, 1, 2), + (6144, 6144, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (6144, 6144, 8192, 32, 32, False, True, True): (4, 32, 1, 2), + (6144, 6144, 8192, 32, 32, True, False, True): (4, 32, 3, 4), + (6144, 6144, 8192, 64, 64, False, True, True): (4, 64, 3, 4), + (6144, 6144, 8192, 64, 64, True, False, True): (4, 32, 3, 4), + (6144, 6144, 8192, 128, 128, False, True, True): (6, 64, 1, 4), + (6144, 6144, 8192, 128, 128, True, False, True): (4, 64, 3, 8), + (6144, 6144, 16384, 16, 16, False, True, True): (2, 32, 1, 2), + (6144, 6144, 16384, 16, 16, True, False, True): (4, 64, 4, 4), + (6144, 6144, 16384, 32, 32, False, True, True): (4, 64, 1, 2), + (6144, 6144, 16384, 32, 32, True, False, True): (4, 64, 3, 4), + (6144, 6144, 16384, 64, 64, False, True, True): (4, 128, 3, 4), + (6144, 6144, 16384, 64, 64, True, False, True): (1, 32, 3, 8), + (6144, 6144, 16384, 128, 128, False, True, True): (4, 128, 1, 4), + (6144, 6144, 16384, 128, 128, True, False, True): (4, 128, 3, 8), + (6144, 6144, 32768, 16, 16, False, True, True): (2, 64, 1, 2), + (6144, 6144, 32768, 16, 16, True, False, True): (5, 128, 4, 1), + (6144, 6144, 32768, 32, 32, False, True, True): (4, 128, 1, 2), + (6144, 6144, 32768, 32, 32, True, False, True): (3, 128, 3, 4), + (6144, 6144, 32768, 64, 64, False, True, True): (4, 256, 3, 4), + (6144, 6144, 32768, 64, 64, True, False, True): (2, 64, 3, 8), + (6144, 6144, 32768, 128, 128, False, True, True): (8, 256, 1, 4), + (6144, 6144, 32768, 128, 128, True, False, True): (4, 256, 3, 8), + (6144, 6144, 65536, 16, 16, False, True, True): (2, 128, 1, 2), + (6144, 6144, 65536, 16, 16, True, False, True): (5, 256, 4, 1), + (6144, 6144, 65536, 32, 32, False, True, True): (4, 256, 1, 2), + (6144, 6144, 65536, 32, 32, True, False, True): (2, 256, 3, 4), + (6144, 6144, 65536, 64, 64, False, True, True): (4, 512, 3, 4), + (6144, 6144, 65536, 64, 64, True, False, True): (1, 128, 3, 8), + (6144, 6144, 65536, 128, 128, False, True, True): (4, 512, 1, 4), + (6144, 6144, 65536, 128, 128, True, False, True): (4, 512, 3, 8), + (6144, 6144, 131072, 16, 16, False, True, True): (2, 256, 1, 2), + (6144, 6144, 131072, 16, 16, True, False, True): (3, 512, 4, 4), + (6144, 6144, 131072, 32, 32, False, True, True): (4, 512, 1, 2), + (6144, 6144, 131072, 32, 32, True, False, True): (4, 512, 3, 4), + (6144, 6144, 131072, 64, 64, False, True, True): (4, 1024, 3, 4), + (6144, 6144, 131072, 64, 64, True, False, True): (2, 256, 3, 8), + (6144, 6144, 131072, 128, 128, False, True, True): (4, 1024, 1, 4), + (6144, 6144, 131072, 128, 128, True, False, True): (4, 1024, 3, 8), + (8192, 8192, 256, 16, 16, False, True, True): (2, 2, 6, 4), + (8192, 8192, 256, 16, 16, True, False, True): (2, 4, 2, 2), + (8192, 8192, 256, 32, 32, False, True, True): (4, 2, 3, 4), + (8192, 8192, 256, 32, 32, True, False, True): (4, 2, 3, 4), + (8192, 8192, 256, 64, 64, False, True, True): (2, 2, 3, 8), + (8192, 8192, 256, 64, 64, True, False, True): (6, 2, 3, 8), + (8192, 8192, 256, 128, 128, False, True, True): (3, 2, 1, 4), + (8192, 8192, 256, 128, 128, True, False, True): (1, 2, 1, 4), + (8192, 8192, 512, 16, 16, False, True, True): (4, 4, 3, 2), + (8192, 8192, 512, 16, 16, True, False, True): (4, 4, 3, 4), + (8192, 8192, 512, 32, 32, False, True, True): (1, 4, 3, 4), + (8192, 8192, 512, 32, 32, True, False, True): (5, 4, 3, 2), + (8192, 8192, 512, 64, 64, False, True, True): (1, 4, 3, 4), + (8192, 8192, 512, 64, 64, True, False, True): (2, 2, 3, 8), + (8192, 8192, 512, 128, 128, False, True, True): (4, 4, 2, 8), + (8192, 8192, 512, 128, 128, True, False, True): (4, 4, 2, 8), + (8192, 8192, 1024, 16, 16, False, True, True): (4, 8, 4, 4), + (8192, 8192, 1024, 16, 16, True, False, True): (4, 8, 4, 4), + (8192, 8192, 1024, 32, 32, False, True, True): (2, 4, 4, 8), + (8192, 8192, 1024, 32, 32, True, False, True): (1, 4, 3, 4), + (8192, 8192, 1024, 64, 64, False, True, True): (4, 8, 3, 4), + (8192, 8192, 1024, 64, 64, True, False, True): (2, 8, 3, 4), + (8192, 8192, 1024, 128, 128, False, True, True): (4, 8, 2, 8), + (8192, 8192, 1024, 128, 128, True, False, True): (4, 8, 1, 4), + (8192, 8192, 2048, 16, 16, False, True, True): (2, 8, 4, 4), + (8192, 8192, 2048, 16, 16, True, False, True): (2, 8, 4, 4), + (8192, 8192, 2048, 32, 32, False, True, True): (2, 8, 4, 8), + (8192, 8192, 2048, 32, 32, True, False, True): (2, 8, 4, 8), + (8192, 8192, 2048, 64, 64, False, True, True): (4, 8, 2, 4), + (8192, 8192, 2048, 64, 64, True, False, True): (4, 16, 3, 4), + (8192, 8192, 2048, 128, 128, False, True, True): (6, 16, 1, 4), + (8192, 8192, 2048, 128, 128, True, False, True): (4, 16, 1, 4), + (8192, 8192, 4096, 16, 16, False, True, True): (4, 32, 4, 2), + (8192, 8192, 4096, 16, 16, True, False, True): (4, 32, 4, 2), + (8192, 8192, 4096, 32, 32, False, True, True): (2, 16, 4, 8), + (8192, 8192, 4096, 32, 32, True, False, True): (4, 16, 4, 8), + (8192, 8192, 4096, 64, 64, False, True, True): (4, 16, 2, 4), + (8192, 8192, 4096, 64, 64, True, False, True): (4, 16, 2, 4), + (8192, 8192, 4096, 128, 128, False, True, True): (6, 32, 1, 4), + (8192, 8192, 4096, 128, 128, True, False, True): (4, 32, 1, 4), + (8192, 8192, 8192, 16, 16, False, True, True): (4, 64, 4, 2), + (8192, 8192, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (8192, 8192, 8192, 32, 32, False, True, True): (2, 32, 4, 8), + (8192, 8192, 8192, 32, 32, True, False, True): (2, 32, 4, 8), + (8192, 8192, 8192, 64, 64, False, True, True): (2, 32, 2, 4), + (8192, 8192, 8192, 64, 64, True, False, True): (4, 32, 2, 4), + (8192, 8192, 8192, 128, 128, False, True, True): (6, 64, 1, 4), + (8192, 8192, 8192, 128, 128, True, False, True): (4, 64, 1, 4), + (8192, 8192, 16384, 16, 16, False, True, True): (4, 64, 3, 4), + (8192, 8192, 16384, 16, 16, True, False, True): (4, 64, 4, 4), + (8192, 8192, 16384, 32, 32, False, True, True): (4, 64, 4, 8), + (8192, 8192, 16384, 32, 32, True, False, True): (4, 64, 4, 8), + (8192, 8192, 16384, 64, 64, False, True, True): (4, 64, 2, 4), + (8192, 8192, 16384, 64, 64, True, False, True): (4, 64, 3, 8), + (8192, 8192, 16384, 128, 128, False, True, True): (6, 128, 1, 4), + (8192, 8192, 16384, 128, 128, True, False, True): (4, 128, 1, 4), + (8192, 8192, 32768, 16, 16, False, True, True): (3, 128, 4, 4), + (8192, 8192, 32768, 16, 16, True, False, True): (3, 128, 4, 4), + (8192, 8192, 32768, 32, 32, False, True, True): (2, 128, 4, 8), + (8192, 8192, 32768, 32, 32, True, False, True): (2, 128, 4, 8), + (8192, 8192, 32768, 64, 64, False, True, True): (2, 128, 2, 4), + (8192, 8192, 32768, 64, 64, True, False, True): (2, 128, 3, 8), + (8192, 8192, 32768, 128, 128, False, True, True): (6, 256, 1, 4), + (8192, 8192, 32768, 128, 128, True, False, True): (4, 256, 1, 4), + (8192, 8192, 50432, 16, 16, False, True, True): (1, 197, 1, 1), + (8192, 8192, 50432, 16, 16, True, False, True): (3, 197, 4, 1), + (8192, 8192, 50432, 32, 32, False, True, True): (2, 197, 1, 4), + (8192, 8192, 50432, 32, 32, True, False, True): (2, 197, 3, 4), + (8192, 8192, 50432, 64, 64, False, True, True): (2, 394, 3, 4), + (8192, 8192, 65536, 16, 16, False, True, True): (3, 256, 4, 4), + (8192, 8192, 65536, 16, 16, True, False, True): (4, 256, 4, 4), + (8192, 8192, 65536, 32, 32, False, True, True): (2, 256, 4, 8), + (8192, 8192, 65536, 32, 32, True, False, True): (2, 256, 3, 8), + (8192, 8192, 65536, 64, 64, False, True, True): (2, 256, 2, 4), + (8192, 8192, 65536, 64, 64, True, False, True): (4, 256, 3, 8), + (8192, 8192, 65536, 128, 128, False, True, True): (6, 512, 1, 4), + (8192, 8192, 65536, 128, 128, True, False, True): (4, 512, 1, 4), + (8192, 8192, 65792, 16, 16, False, True, True): (1, 257, 1, 1), + (8192, 8192, 65792, 16, 16, True, False, True): (3, 257, 4, 1), + (8192, 8192, 65792, 32, 32, False, True, True): (2, 257, 1, 4), + (8192, 8192, 65792, 32, 32, True, False, True): (1, 257, 3, 4), + (8192, 8192, 65792, 64, 64, False, True, True): (2, 514, 3, 4), + (8192, 8192, 65792, 64, 64, True, False, True): (1, 257, 3, 4), + (8192, 8192, 65792, 128, 128, False, True, True): (2, 514, 1, 4), + (8192, 8192, 65792, 128, 128, True, False, True): (2, 514, 3, 8), + (8192, 8192, 131072, 16, 16, False, True, True): (4, 512, 4, 4), + (8192, 8192, 131072, 16, 16, True, False, True): (3, 512, 4, 4), + (8192, 8192, 131072, 32, 32, False, True, True): (2, 512, 4, 8), + (8192, 8192, 131072, 32, 32, True, False, True): (2, 512, 4, 8), + (8192, 8192, 131072, 64, 64, False, True, True): (2, 512, 2, 4), + (8192, 8192, 131072, 64, 64, True, False, True): (2, 512, 2, 4), + (8192, 8192, 131072, 128, 128, False, True, True): (4, 1024, 1, 4), + (8192, 8192, 131072, 128, 128, True, False, True): (4, 1024, 1, 4), + (12288, 12288, 256, 16, 16, False, True, True): (4, 2, 1, 4), + (12288, 12288, 256, 16, 16, True, False, True): (1, 1, 3, 1), + (12288, 12288, 256, 32, 32, False, True, True): (4, 4, 1, 4), + (12288, 12288, 256, 32, 32, True, False, True): (2, 1, 3, 2), + (12288, 12288, 256, 64, 64, False, True, True): (4, 2, 3, 4), + (12288, 12288, 256, 64, 64, True, False, True): (3, 1, 3, 4), + (12288, 12288, 256, 128, 128, False, True, True): (6, 2, 1, 4), + (12288, 12288, 256, 128, 128, True, False, True): (4, 2, 3, 8), + (12288, 12288, 512, 16, 16, False, True, True): (4, 4, 1, 2), + (12288, 12288, 512, 16, 16, True, False, True): (4, 4, 4, 2), + (12288, 12288, 512, 32, 32, False, True, True): (4, 4, 4, 2), + (12288, 12288, 512, 32, 32, True, False, True): (2, 2, 3, 8), + (12288, 12288, 512, 64, 64, False, True, True): (4, 4, 3, 4), + (12288, 12288, 512, 64, 64, True, False, True): (8, 2, 3, 4), + (12288, 12288, 512, 128, 128, False, True, True): (4, 4, 3, 8), + (12288, 12288, 512, 128, 128, True, False, True): (4, 4, 3, 8), + (12288, 12288, 1024, 16, 16, False, True, True): (4, 8, 1, 2), + (12288, 12288, 1024, 16, 16, True, False, True): (2, 4, 4, 4), + (12288, 12288, 1024, 32, 32, False, True, True): (4, 4, 3, 4), + (12288, 12288, 1024, 32, 32, True, False, True): (1, 4, 3, 4), + (12288, 12288, 1024, 64, 64, False, True, True): (4, 8, 3, 4), + (12288, 12288, 1024, 64, 64, True, False, True): (2, 4, 3, 4), + (12288, 12288, 1024, 128, 128, False, True, True): (4, 8, 3, 8), + (12288, 12288, 1024, 128, 128, True, False, True): (4, 8, 3, 8), + (12288, 12288, 2048, 16, 16, False, True, True): (2, 4, 1, 4), + (12288, 12288, 2048, 16, 16, True, False, True): (2, 8, 4, 4), + (12288, 12288, 2048, 32, 32, False, True, True): (4, 8, 1, 2), + (12288, 12288, 2048, 32, 32, True, False, True): (2, 8, 4, 8), + (12288, 12288, 2048, 64, 64, False, True, True): (4, 16, 3, 4), + (12288, 12288, 2048, 64, 64, True, False, True): (2, 8, 3, 4), + (12288, 12288, 2048, 128, 128, False, True, True): (4, 16, 3, 8), + (12288, 12288, 2048, 128, 128, True, False, True): (4, 16, 3, 8), + (12288, 12288, 4096, 16, 16, False, True, True): (2, 8, 1, 4), + (12288, 12288, 4096, 16, 16, True, False, True): (2, 16, 4, 4), + (12288, 12288, 4096, 32, 32, False, True, True): (2, 16, 1, 2), + (12288, 12288, 4096, 32, 32, True, False, True): (2, 16, 3, 4), + (12288, 12288, 4096, 64, 64, False, True, True): (4, 32, 3, 4), + (12288, 12288, 4096, 64, 64, True, False, True): (2, 16, 3, 4), + (12288, 12288, 4096, 128, 128, False, True, True): (4, 32, 1, 4), + (12288, 12288, 4096, 128, 128, True, False, True): (4, 32, 3, 8), + (12288, 12288, 8192, 16, 16, False, True, True): (2, 32, 1, 1), + (12288, 12288, 8192, 16, 16, True, False, True): (4, 64, 4, 2), + (12288, 12288, 8192, 32, 32, False, True, True): (2, 32, 1, 2), + (12288, 12288, 8192, 32, 32, True, False, True): (2, 32, 3, 2), + (12288, 12288, 8192, 64, 64, False, True, True): (4, 64, 3, 4), + (12288, 12288, 8192, 64, 64, True, False, True): (2, 32, 3, 4), + (12288, 12288, 8192, 128, 128, False, True, True): (4, 64, 3, 8), + (12288, 12288, 8192, 128, 128, True, False, True): (2, 64, 3, 8), + (12288, 12288, 16384, 16, 16, False, True, True): (4, 128, 1, 2), + (12288, 12288, 16384, 16, 16, True, False, True): (4, 128, 4, 2), + (12288, 12288, 16384, 32, 32, False, True, True): (2, 64, 1, 2), + (12288, 12288, 16384, 32, 32, True, False, True): (2, 64, 3, 4), + (12288, 12288, 16384, 64, 64, False, True, True): (4, 128, 3, 4), + (12288, 12288, 16384, 64, 64, True, False, True): (2, 64, 3, 4), + (12288, 12288, 16384, 128, 128, False, True, True): (4, 128, 1, 4), + (12288, 12288, 16384, 128, 128, True, False, True): (4, 128, 3, 8), + (12288, 12288, 32768, 16, 16, False, True, True): (2, 128, 1, 1), + (12288, 12288, 32768, 16, 16, True, False, True): (3, 128, 4, 1), + (12288, 12288, 32768, 32, 32, False, True, True): (2, 128, 1, 2), + (12288, 12288, 32768, 32, 32, True, False, True): (2, 128, 3, 2), + (12288, 12288, 32768, 64, 64, False, True, True): (4, 256, 3, 4), + (12288, 12288, 32768, 64, 64, True, False, True): (1, 64, 3, 8), + (12288, 12288, 32768, 128, 128, False, True, True): (4, 256, 3, 8), + (12288, 12288, 32768, 128, 128, True, False, True): (4, 256, 3, 8), + (12288, 12288, 65536, 16, 16, False, True, True): (4, 512, 1, 2), + (12288, 12288, 65536, 16, 16, True, False, True): (3, 256, 4, 1), + (12288, 12288, 65536, 32, 32, False, True, True): (2, 256, 1, 2), + (12288, 12288, 65536, 32, 32, True, False, True): (2, 256, 3, 2), + (12288, 12288, 65536, 64, 64, False, True, True): (4, 512, 3, 4), + (12288, 12288, 65536, 64, 64, True, False, True): (2, 256, 3, 4), + (12288, 12288, 65536, 128, 128, False, True, True): (4, 512, 1, 4), + (12288, 12288, 65536, 128, 128, True, False, True): (4, 512, 3, 8), + (12288, 12288, 131072, 16, 16, False, True, True): (2, 512, 1, 1), + (12288, 12288, 131072, 16, 16, True, False, True): (2, 512, 4, 4), + (12288, 12288, 131072, 32, 32, False, True, True): (2, 512, 1, 2), + (12288, 12288, 131072, 32, 32, True, False, True): (2, 512, 3, 4), + (12288, 12288, 131072, 64, 64, False, True, True): (4, 1024, 3, 4), + (12288, 12288, 131072, 64, 64, True, False, True): (2, 512, 3, 4), + (12288, 12288, 131072, 128, 128, False, True, True): (4, 1024, 3, 8), + (12288, 12288, 131072, 128, 128, True, False, True): (4, 1024, 3, 8), + (16384, 16384, 256, 16, 16, False, True, True): (2, 2, 3, 2), + (16384, 16384, 256, 16, 16, True, False, True): (2, 2, 6, 4), + (16384, 16384, 256, 32, 32, False, True, True): (4, 2, 3, 4), + (16384, 16384, 256, 32, 32, True, False, True): (4, 2, 3, 2), + (16384, 16384, 256, 64, 64, False, True, True): (2, 2, 5, 4), + (16384, 16384, 256, 64, 64, True, False, True): (2, 2, 3, 8), + (16384, 16384, 256, 128, 128, False, True, True): (4, 2, 2, 8), + (16384, 16384, 256, 128, 128, True, False, True): (2, 2, 1, 4), + (16384, 16384, 512, 16, 16, False, True, True): (1, 2, 4, 4), + (16384, 16384, 512, 16, 16, True, False, True): (1, 2, 4, 4), + (16384, 16384, 512, 32, 32, False, True, True): (2, 2, 3, 8), + (16384, 16384, 512, 32, 32, True, False, True): (2, 2, 4, 8), + (16384, 16384, 512, 64, 64, False, True, True): (4, 4, 3, 4), + (16384, 16384, 512, 64, 64, True, False, True): (2, 4, 3, 4), + (16384, 16384, 512, 128, 128, False, True, True): (4, 4, 2, 8), + (16384, 16384, 512, 128, 128, True, False, True): (4, 4, 2, 8), + (16384, 16384, 1024, 16, 16, False, True, True): (4, 8, 4, 4), + (16384, 16384, 1024, 16, 16, True, False, True): (2, 4, 4, 4), + (16384, 16384, 1024, 32, 32, False, True, True): (2, 4, 4, 8), + (16384, 16384, 1024, 32, 32, True, False, True): (2, 4, 4, 8), + (16384, 16384, 1024, 64, 64, False, True, True): (4, 4, 2, 4), + (16384, 16384, 1024, 64, 64, True, False, True): (2, 4, 2, 4), + (16384, 16384, 1024, 128, 128, False, True, True): (6, 8, 1, 4), + (16384, 16384, 1024, 128, 128, True, False, True): (4, 8, 1, 4), + (16384, 16384, 2048, 16, 16, False, True, True): (2, 8, 4, 4), + (16384, 16384, 2048, 16, 16, True, False, True): (2, 8, 4, 4), + (16384, 16384, 2048, 32, 32, False, True, True): (2, 8, 4, 8), + (16384, 16384, 2048, 32, 32, True, False, True): (2, 8, 4, 8), + (16384, 16384, 2048, 64, 64, False, True, True): (2, 8, 2, 4), + (16384, 16384, 2048, 64, 64, True, False, True): (2, 8, 2, 4), + (16384, 16384, 2048, 128, 128, False, True, True): (4, 16, 2, 8), + (16384, 16384, 2048, 128, 128, True, False, True): (4, 16, 1, 4), + (16384, 16384, 4096, 16, 16, False, True, True): (2, 16, 4, 4), + (16384, 16384, 4096, 16, 16, True, False, True): (2, 16, 4, 4), + (16384, 16384, 4096, 32, 32, False, True, True): (1, 16, 4, 8), + (16384, 16384, 4096, 32, 32, True, False, True): (2, 16, 3, 4), + (16384, 16384, 4096, 64, 64, False, True, True): (1, 16, 2, 4), + (16384, 16384, 4096, 64, 64, True, False, True): (2, 16, 2, 4), + (16384, 16384, 4096, 128, 128, False, True, True): (4, 32, 2, 8), + (16384, 16384, 4096, 128, 128, True, False, True): (4, 32, 1, 4), + (16384, 16384, 8192, 16, 16, False, True, True): (2, 64, 4, 2), + (16384, 16384, 8192, 16, 16, True, False, True): (2, 64, 4, 2), + (16384, 16384, 8192, 32, 32, False, True, True): (2, 32, 4, 8), + (16384, 16384, 8192, 32, 32, True, False, True): (2, 32, 4, 8), + (16384, 16384, 8192, 64, 64, False, True, True): (2, 32, 2, 4), + (16384, 16384, 8192, 64, 64, True, False, True): (2, 32, 4, 8), + (16384, 16384, 8192, 128, 128, False, True, True): (4, 64, 2, 8), + (16384, 16384, 8192, 128, 128, True, False, True): (4, 64, 1, 4), + (16384, 16384, 16384, 16, 16, False, True, True): (1, 64, 4, 4), + (16384, 16384, 16384, 16, 16, True, False, True): (1, 64, 4, 4), + (16384, 16384, 16384, 32, 32, False, True, True): (1, 64, 4, 8), + (16384, 16384, 16384, 32, 32, True, False, True): (1, 64, 4, 8), + (16384, 16384, 16384, 64, 64, False, True, True): (1, 64, 2, 4), + (16384, 16384, 16384, 64, 64, True, False, True): (1, 64, 3, 8), + (16384, 16384, 16384, 128, 128, False, True, True): (4, 128, 1, 4), + (16384, 16384, 16384, 128, 128, True, False, True): (4, 128, 1, 4), + (16384, 16384, 32768, 16, 16, False, True, True): (1, 128, 4, 4), + (16384, 16384, 32768, 16, 16, True, False, True): (1, 128, 4, 4), + (16384, 16384, 32768, 32, 32, False, True, True): (1, 128, 3, 4), + (16384, 16384, 32768, 32, 32, True, False, True): (1, 128, 3, 8), + (16384, 16384, 32768, 64, 64, False, True, True): (2, 128, 2, 4), + (16384, 16384, 32768, 64, 64, True, False, True): (1, 128, 4, 8), + (16384, 16384, 32768, 128, 128, False, True, True): (4, 256, 2, 8), + (16384, 16384, 32768, 128, 128, True, False, True): (4, 256, 1, 4), + (16384, 16384, 65536, 16, 16, False, True, True): (1, 256, 3, 4), + (16384, 16384, 65536, 16, 16, True, False, True): (1, 256, 4, 4), + (16384, 16384, 65536, 32, 32, False, True, True): (1, 256, 4, 8), + (16384, 16384, 65536, 32, 32, True, False, True): (1, 256, 3, 4), + (16384, 16384, 65536, 64, 64, False, True, True): (2, 256, 2, 4), + (16384, 16384, 65536, 64, 64, True, False, True): (1, 256, 3, 8), + (16384, 16384, 65536, 128, 128, False, True, True): (4, 512, 2, 8), + (16384, 16384, 65536, 128, 128, True, False, True): (4, 512, 1, 4), + (16384, 16384, 65792, 16, 16, False, True, True): (1, 257, 1, 1), + (16384, 16384, 65792, 16, 16, True, False, True): (1, 257, 4, 1), + (16384, 16384, 65792, 32, 32, False, True, True): (1, 257, 1, 4), + (16384, 16384, 65792, 32, 32, True, False, True): (1, 257, 3, 4), + (16384, 16384, 65792, 64, 64, False, True, True): (2, 514, 3, 4), + (16384, 16384, 65792, 64, 64, True, False, True): (1, 257, 3, 4), + (16384, 16384, 65792, 128, 128, False, True, True): (2, 514, 3, 8), + (16384, 16384, 65792, 128, 128, True, False, True): (2, 514, 3, 8), + (16384, 16384, 131072, 16, 16, False, True, True): (1, 512, 4, 4), + (16384, 16384, 131072, 16, 16, True, False, True): (1, 512, 3, 2), + (16384, 16384, 131072, 32, 32, False, True, True): (1, 512, 4, 8), + (16384, 16384, 131072, 32, 32, True, False, True): (1, 512, 3, 2), + (16384, 16384, 131072, 64, 64, False, True, True): (1, 512, 2, 4), + (16384, 16384, 131072, 64, 64, True, False, True): (1, 512, 2, 4), + (16384, 16384, 131072, 128, 128, False, True, True): (4, 1024, 1, 4), + (16384, 16384, 131072, 128, 128, True, False, True): (4, 1024, 1, 4), + (24576, 24576, 256, 16, 16, False, True, True): (6, 2, 1, 2), + (24576, 24576, 256, 16, 16, True, False, True): (2, 2, 5, 4), + (24576, 24576, 256, 32, 32, False, True, True): (4, 4, 1, 4), + (24576, 24576, 256, 32, 32, True, False, True): (2, 2, 4, 2), + (24576, 24576, 256, 64, 64, False, True, True): (2, 2, 3, 4), + (24576, 24576, 256, 64, 64, True, False, True): (1, 1, 3, 4), + (24576, 24576, 256, 128, 128, False, True, True): (6, 2, 1, 4), + (24576, 24576, 256, 128, 128, True, False, True): (2, 2, 3, 8), + (24576, 24576, 512, 16, 16, False, True, True): (4, 4, 1, 2), + (24576, 24576, 512, 16, 16, True, False, True): (2, 2, 4, 4), + (24576, 24576, 512, 32, 32, False, True, True): (1, 2, 3, 4), + (24576, 24576, 512, 32, 32, True, False, True): (1, 2, 3, 4), + (24576, 24576, 512, 64, 64, False, True, True): (4, 4, 3, 4), + (24576, 24576, 512, 64, 64, True, False, True): (1, 2, 3, 4), + (24576, 24576, 512, 128, 128, False, True, True): (4, 4, 3, 8), + (24576, 24576, 512, 128, 128, True, False, True): (4, 4, 3, 8), + (24576, 24576, 1024, 16, 16, False, True, True): (2, 8, 1, 2), + (24576, 24576, 1024, 16, 16, True, False, True): (2, 4, 4, 4), + (24576, 24576, 1024, 32, 32, False, True, True): (2, 4, 1, 2), + (24576, 24576, 1024, 32, 32, True, False, True): (1, 4, 3, 4), + (24576, 24576, 1024, 64, 64, False, True, True): (4, 8, 3, 4), + (24576, 24576, 1024, 64, 64, True, False, True): (1, 4, 3, 4), + (24576, 24576, 1024, 128, 128, False, True, True): (4, 8, 3, 8), + (24576, 24576, 1024, 128, 128, True, False, True): (4, 8, 3, 8), + (24576, 24576, 2048, 16, 16, False, True, True): (1, 4, 1, 4), + (24576, 24576, 2048, 16, 16, True, False, True): (1, 8, 4, 4), + (24576, 24576, 2048, 32, 32, False, True, True): (2, 8, 1, 2), + (24576, 24576, 2048, 32, 32, True, False, True): (1, 8, 3, 4), + (24576, 24576, 2048, 64, 64, False, True, True): (4, 16, 3, 4), + (24576, 24576, 2048, 64, 64, True, False, True): (1, 4, 3, 8), + (24576, 24576, 2048, 128, 128, False, True, True): (4, 16, 3, 8), + (24576, 24576, 2048, 128, 128, True, False, True): (2, 16, 3, 8), + (24576, 24576, 4096, 16, 16, False, True, True): (2, 32, 1, 2), + (24576, 24576, 4096, 16, 16, True, False, True): (1, 16, 4, 4), + (24576, 24576, 4096, 32, 32, False, True, True): (1, 16, 1, 2), + (24576, 24576, 4096, 32, 32, True, False, True): (1, 16, 3, 4), + (24576, 24576, 4096, 64, 64, False, True, True): (4, 32, 3, 4), + (24576, 24576, 4096, 64, 64, True, False, True): (1, 8, 3, 8), + (24576, 24576, 4096, 128, 128, False, True, True): (4, 32, 3, 8), + (24576, 24576, 4096, 128, 128, True, False, True): (2, 32, 3, 8), + (24576, 24576, 8192, 16, 16, False, True, True): (1, 32, 1, 1), + (24576, 24576, 8192, 16, 16, True, False, True): (2, 64, 4, 2), + (24576, 24576, 8192, 32, 32, False, True, True): (1, 32, 1, 2), + (24576, 24576, 8192, 32, 32, True, False, True): (1, 32, 3, 4), + (24576, 24576, 8192, 64, 64, False, True, True): (4, 64, 3, 4), + (24576, 24576, 8192, 64, 64, True, False, True): (1, 32, 3, 4), + (24576, 24576, 8192, 128, 128, False, True, True): (4, 64, 3, 8), + (24576, 24576, 8192, 128, 128, True, False, True): (4, 64, 3, 8), + (24576, 24576, 16384, 16, 16, False, True, True): (2, 128, 1, 2), + (24576, 24576, 16384, 16, 16, True, False, True): (1, 64, 4, 4), + (24576, 24576, 16384, 32, 32, False, True, True): (1, 64, 1, 2), + (24576, 24576, 16384, 32, 32, True, False, True): (1, 64, 3, 2), + (24576, 24576, 16384, 64, 64, False, True, True): (2, 128, 3, 4), + (24576, 24576, 16384, 64, 64, True, False, True): (1, 32, 3, 8), + (24576, 24576, 16384, 128, 128, False, True, True): (4, 128, 3, 8), + (24576, 24576, 16384, 128, 128, True, False, True): (4, 128, 3, 8), + (24576, 24576, 32768, 16, 16, False, True, True): (1, 128, 1, 1), + (24576, 24576, 32768, 16, 16, True, False, True): (1, 128, 4, 4), + (24576, 24576, 32768, 32, 32, False, True, True): (1, 128, 1, 2), + (24576, 24576, 32768, 32, 32, True, False, True): (1, 128, 3, 4), + (24576, 24576, 32768, 64, 64, False, True, True): (2, 256, 3, 4), + (24576, 24576, 32768, 64, 64, True, False, True): (1, 128, 3, 4), + (24576, 24576, 32768, 128, 128, False, True, True): (4, 256, 3, 8), + (24576, 24576, 32768, 128, 128, True, False, True): (2, 256, 3, 8), + (24576, 24576, 65536, 16, 16, False, True, True): (2, 512, 1, 2), + (24576, 24576, 65536, 16, 16, True, False, True): (1, 256, 4, 4), + (32768, 32768, 256, 16, 16, False, True, True): (4, 2, 1, 2), + (32768, 32768, 256, 16, 16, True, False, True): (2, 2, 5, 4), + (32768, 32768, 256, 32, 32, False, True, True): (4, 2, 4, 2), + (32768, 32768, 256, 32, 32, True, False, True): (1, 1, 4, 8), + (32768, 32768, 256, 64, 64, False, True, True): (2, 2, 3, 4), + (32768, 32768, 256, 64, 64, True, False, True): (1, 1, 3, 8), + (32768, 32768, 256, 128, 128, False, True, True): (2, 2, 3, 8), + (32768, 32768, 256, 128, 128, True, False, True): (2, 2, 3, 8), + (32768, 32768, 512, 16, 16, False, True, True): (2, 2, 1, 4), + (32768, 32768, 512, 16, 16, True, False, True): (2, 2, 4, 2), + (32768, 32768, 512, 32, 32, False, True, True): (1, 2, 3, 4), + (32768, 32768, 512, 32, 32, True, False, True): (1, 2, 4, 8), + (32768, 32768, 512, 64, 64, False, True, True): (4, 4, 3, 4), + (32768, 32768, 512, 64, 64, True, False, True): (1, 2, 3, 4), + (32768, 32768, 512, 128, 128, False, True, True): (4, 4, 3, 8), + (32768, 32768, 512, 128, 128, True, False, True): (4, 4, 3, 8), + (32768, 32768, 1024, 16, 16, False, True, True): (2, 4, 1, 1), + (32768, 32768, 1024, 16, 16, True, False, True): (1, 4, 4, 2), + (32768, 32768, 1024, 32, 32, False, True, True): (2, 4, 1, 4), + (32768, 32768, 1024, 32, 32, True, False, True): (1, 4, 3, 4), + (32768, 32768, 1024, 64, 64, False, True, True): (4, 8, 3, 4), + (32768, 32768, 1024, 64, 64, True, False, True): (1, 4, 3, 4), + (32768, 32768, 1024, 128, 128, False, True, True): (4, 8, 3, 8), + (32768, 32768, 1024, 128, 128, True, False, True): (4, 8, 3, 8), + (32768, 32768, 2048, 16, 16, False, True, True): (1, 8, 1, 4), + (32768, 32768, 2048, 16, 16, True, False, True): (1, 8, 4, 4), + (32768, 32768, 2048, 32, 32, False, True, True): (2, 8, 1, 4), + (32768, 32768, 2048, 32, 32, True, False, True): (1, 8, 3, 4), + (32768, 32768, 2048, 64, 64, False, True, True): (4, 16, 3, 4), + (32768, 32768, 2048, 64, 64, True, False, True): (1, 8, 3, 4), + (32768, 32768, 2048, 128, 128, False, True, True): (4, 16, 3, 8), + (32768, 32768, 2048, 128, 128, True, False, True): (2, 16, 3, 8), + (32768, 32768, 4096, 16, 16, False, True, True): (1, 16, 1, 4), + (32768, 32768, 4096, 16, 16, True, False, True): (1, 16, 4, 4), + (32768, 32768, 4096, 32, 32, False, True, True): (2, 16, 1, 4), + (32768, 32768, 4096, 32, 32, True, False, True): (1, 16, 3, 4), + (32768, 32768, 4096, 64, 64, False, True, True): (2, 32, 3, 4), + (32768, 32768, 4096, 64, 64, True, False, True): (1, 16, 3, 4), + (32768, 32768, 4096, 128, 128, False, True, True): (4, 32, 3, 8), + (32768, 32768, 4096, 128, 128, True, False, True): (4, 32, 3, 8), + (32768, 32768, 8192, 16, 16, False, True, True): (1, 32, 1, 4), + (32768, 32768, 8192, 16, 16, True, False, True): (2, 64, 4, 1), + (32768, 32768, 8192, 32, 32, False, True, True): (2, 32, 1, 4), + (32768, 32768, 8192, 32, 32, True, False, True): (1, 32, 3, 4), + (32768, 32768, 8192, 64, 64, False, True, True): (2, 64, 3, 4), + (32768, 32768, 8192, 64, 64, True, False, True): (1, 32, 3, 4), + (32768, 32768, 8192, 128, 128, False, True, True): (4, 64, 3, 8), + (32768, 32768, 8192, 128, 128, True, False, True): (2, 64, 3, 8), + (32768, 32768, 16384, 16, 16, False, True, True): (1, 64, 1, 4), + (32768, 32768, 16384, 16, 16, True, False, True): (1, 64, 4, 1), + (32768, 32768, 16384, 32, 32, False, True, True): (2, 64, 1, 4), + (32768, 32768, 16384, 32, 32, True, False, True): (1, 64, 3, 4), + (32768, 32768, 16384, 64, 64, False, True, True): (2, 128, 3, 4), + (32768, 32768, 16384, 64, 64, True, False, True): (1, 64, 3, 4), + (32768, 32768, 16384, 128, 128, False, True, True): (4, 128, 3, 8), + (32768, 32768, 16384, 128, 128, True, False, True): (2, 128, 3, 8), + (32768, 32768, 32768, 16, 16, False, True, True): (1, 128, 1, 4), + (32768, 32768, 32768, 16, 16, True, False, True): (1, 128, 4, 1), + (32768, 32768, 32768, 32, 32, False, True, True): (2, 128, 1, 4), + (32768, 32768, 32768, 32, 32, True, False, True): (1, 128, 3, 4), + (32768, 32768, 32768, 64, 64, False, True, True): (2, 256, 3, 4), + (32768, 32768, 32768, 64, 64, True, False, True): (1, 128, 3, 4), + (32768, 32768, 32768, 128, 128, False, True, True): (2, 256, 3, 8), + (32768, 32768, 32768, 128, 128, True, False, True): (4, 256, 3, 8), + (32768, 32768, 65536, 16, 16, False, True, True): (1, 256, 1, 4), + (32768, 32768, 65536, 16, 16, True, False, True): (1, 256, 4, 1), + (32768, 32768, 65536, 32, 32, False, True, True): (1, 256, 3, 4), + (32768, 32768, 65536, 32, 32, True, False, True): (1, 256, 3, 4), + (32768, 32768, 65536, 64, 64, False, True, True): (1, 512, 3, 4), + (32768, 32768, 65536, 64, 64, True, False, True): (1, 256, 3, 4), + (32768, 32768, 65536, 128, 128, False, True, True): (4, 512, 1, 4), + (32768, 32768, 65536, 128, 128, True, False, True): (2, 512, 3, 8), + }, + ("bsr_dense_addmm", "NVIDIA A100-SXM4-80GB", (0, torch.float16, 0.56)): { + (192, 192, 256, 64, 64, False, True, True): (1, 4, 3, 4), + (192, 192, 256, 64, 64, True, False, True): (1, 4, 3, 4), + (192, 192, 512, 64, 64, False, True, True): (1, 8, 5, 4), + (192, 192, 512, 64, 64, True, False, True): (1, 8, 3, 4), + (192, 192, 1024, 64, 64, False, True, True): (1, 16, 3, 2), + (192, 192, 1024, 64, 64, True, False, True): (1, 16, 3, 4), + (192, 192, 2048, 64, 64, False, True, True): (1, 32, 5, 4), + (192, 192, 2048, 64, 64, True, False, True): (4, 32, 5, 4), + (192, 192, 4096, 64, 64, False, True, True): (1, 64, 1, 8), + (192, 192, 4096, 64, 64, True, False, True): (1, 32, 3, 4), + (192, 192, 8192, 64, 64, False, True, True): (4, 128, 1, 4), + (192, 192, 8192, 64, 64, True, False, True): (3, 64, 3, 4), + (192, 192, 16384, 64, 64, False, True, True): (1, 256, 1, 4), + (192, 192, 16384, 64, 64, True, False, True): (3, 64, 2, 4), + (192, 192, 32768, 64, 64, False, True, True): (1, 512, 1, 2), + (192, 192, 32768, 64, 64, True, False, True): (2, 256, 2, 4), + (192, 192, 65536, 64, 64, False, True, True): (1, 512, 1, 4), + (192, 192, 65536, 64, 64, True, False, True): (2, 512, 2, 4), + (192, 192, 131072, 64, 64, False, True, True): (1, 1024, 1, 4), + (192, 192, 131072, 64, 64, True, False, True): (1, 512, 3, 4), + (384, 384, 256, 128, 128, False, True, True): (3, 2, 3, 8), + (384, 384, 256, 128, 128, True, False, True): (5, 2, 3, 8), + (384, 384, 512, 128, 128, False, True, True): (4, 4, 3, 8), + (384, 384, 512, 128, 128, True, False, True): (1, 4, 3, 8), + (384, 384, 1024, 128, 128, False, True, True): (1, 8, 3, 8), + (384, 384, 1024, 128, 128, True, False, True): (1, 8, 2, 8), + (384, 384, 2048, 128, 128, False, True, True): (3, 16, 3, 8), + (384, 384, 2048, 128, 128, True, False, True): (1, 16, 3, 8), + (384, 384, 4096, 128, 128, False, True, True): (3, 32, 3, 8), + (384, 384, 4096, 128, 128, True, False, True): (3, 32, 3, 8), + (384, 384, 8192, 128, 128, False, True, True): (2, 64, 3, 8), + (384, 384, 8192, 128, 128, True, False, True): (2, 64, 2, 4), + (384, 384, 16384, 128, 128, False, True, True): (1, 128, 2, 8), + (384, 384, 16384, 128, 128, True, False, True): (3, 128, 2, 4), + (384, 384, 32768, 128, 128, False, True, True): (2, 256, 3, 8), + (384, 384, 32768, 128, 128, True, False, True): (1, 256, 2, 4), + (384, 384, 65536, 128, 128, False, True, True): (7, 512, 1, 4), + (384, 384, 65536, 128, 128, True, False, True): (3, 512, 2, 4), + (384, 384, 131072, 128, 128, False, True, True): (5, 1024, 1, 4), + (384, 384, 131072, 128, 128, True, False, True): (1, 1024, 2, 4), + }, + ("bsr_dense_addmm", "NVIDIA A100-SXM4-80GB", (0, torch.float32, 0.5)): { + (16, 16, 16, 16, 16, False, False, False): (2, 1, 1, 16), + (16, 16, 16, 16, 16, False, False, True): (1, 1, 2, 4), + (16, 16, 16, 16, 16, False, True, False): (1, 1, 2, 16), + (16, 16, 16, 16, 16, False, True, True): (2, 1, 2, 8), + (16, 16, 16, 16, 16, True, False, False): (1, 1, 1, 2), + (16, 16, 16, 16, 16, True, False, True): (2, 1, 1, 4), + (16, 16, 32, 16, 16, False, False, False): (1, 1, 1, 2), + (16, 16, 32, 16, 16, False, False, True): (1, 1, 2, 8), + (16, 16, 32, 16, 16, False, True, False): (1, 2, 1, 4), + (16, 16, 32, 16, 16, False, True, True): (1, 2, 2, 4), + (16, 16, 32, 16, 16, True, False, False): (1, 1, 2, 4), + (16, 16, 32, 16, 16, True, False, True): (1, 2, 2, 4), + (16, 16, 64, 16, 16, False, False, False): (1, 4, 1, 4), + (16, 16, 64, 16, 16, False, False, True): (2, 2, 1, 4), + (16, 16, 64, 16, 16, False, True, False): (1, 4, 1, 4), + (16, 16, 64, 16, 16, False, True, True): (1, 4, 1, 8), + (16, 16, 64, 16, 16, True, False, False): (1, 2, 1, 4), + (16, 16, 64, 16, 16, True, False, True): (1, 4, 2, 8), + (16, 32, 16, 16, 16, False, False, False): (1, 1, 2, 8), + (16, 32, 16, 16, 16, False, False, True): (2, 1, 1, 4), + (16, 32, 16, 16, 16, False, True, False): (1, 1, 1, 4), + (16, 32, 16, 16, 16, False, True, True): (1, 1, 1, 4), + (16, 32, 16, 16, 16, True, False, False): (1, 1, 1, 4), + (16, 32, 16, 16, 16, True, False, True): (1, 1, 2, 8), + (16, 32, 16, 16, 32, False, False, False): (1, 1, 2, 4), + (16, 32, 16, 16, 32, False, False, True): (2, 1, 2, 2), + (16, 32, 16, 16, 32, False, True, False): (1, 1, 1, 8), + (16, 32, 16, 16, 32, False, True, True): (1, 1, 1, 2), + (16, 32, 16, 16, 32, True, False, False): (3, 1, 1, 4), + (16, 32, 16, 16, 32, True, False, True): (1, 1, 1, 4), + (16, 32, 32, 16, 16, False, False, False): (1, 2, 1, 4), + (16, 32, 32, 16, 16, False, False, True): (2, 2, 1, 4), + (16, 32, 32, 16, 16, False, True, False): (1, 2, 1, 2), + (16, 32, 32, 16, 16, False, True, True): (1, 2, 1, 4), + (16, 32, 32, 16, 16, True, False, False): (1, 2, 1, 4), + (16, 32, 32, 16, 16, True, False, True): (1, 2, 1, 4), + (16, 32, 32, 16, 32, False, False, False): (1, 1, 2, 4), + (16, 32, 32, 16, 32, False, False, True): (1, 2, 1, 4), + (16, 32, 32, 16, 32, False, True, False): (1, 2, 2, 8), + (16, 32, 32, 16, 32, False, True, True): (1, 2, 1, 1), + (16, 32, 32, 16, 32, True, False, False): (1, 2, 1, 2), + (16, 32, 32, 16, 32, True, False, True): (1, 2, 1, 4), + (16, 32, 64, 16, 16, False, False, False): (1, 2, 1, 4), + (16, 32, 64, 16, 16, False, False, True): (2, 4, 1, 4), + (16, 32, 64, 16, 16, False, True, False): (1, 4, 2, 4), + (16, 32, 64, 16, 16, False, True, True): (1, 4, 1, 4), + (16, 32, 64, 16, 16, True, False, False): (1, 2, 2, 8), + (16, 32, 64, 16, 16, True, False, True): (1, 4, 1, 2), + (16, 32, 64, 16, 32, False, False, False): (1, 4, 1, 4), + (16, 32, 64, 16, 32, False, False, True): (1, 4, 3, 4), + (16, 32, 64, 16, 32, False, True, False): (1, 2, 1, 4), + (16, 32, 64, 16, 32, False, True, True): (1, 4, 1, 4), + (16, 32, 64, 16, 32, True, False, False): (1, 2, 1, 8), + (16, 32, 64, 16, 32, True, False, True): (1, 2, 1, 4), + (16, 64, 16, 16, 32, False, False, False): (1, 1, 1, 2), + (16, 64, 16, 16, 32, False, False, True): (1, 1, 1, 8), + (16, 64, 16, 16, 32, False, True, False): (1, 1, 1, 8), + (16, 64, 16, 16, 32, False, True, True): (1, 1, 1, 4), + (16, 64, 16, 16, 32, True, False, False): (1, 1, 1, 8), + (16, 64, 16, 16, 32, True, False, True): (1, 1, 1, 4), + (16, 64, 32, 16, 32, False, False, False): (1, 2, 1, 4), + (16, 64, 32, 16, 32, False, False, True): (1, 1, 1, 4), + (16, 64, 32, 16, 32, False, True, False): (1, 2, 1, 1), + (16, 64, 32, 16, 32, False, True, True): (1, 2, 1, 8), + (16, 64, 32, 16, 32, True, False, False): (2, 2, 1, 4), + (16, 64, 32, 16, 32, True, False, True): (2, 2, 1, 4), + (16, 64, 64, 16, 32, False, False, False): (1, 2, 1, 4), + (16, 64, 64, 16, 32, False, False, True): (1, 4, 1, 4), + (16, 64, 64, 16, 32, False, True, False): (1, 4, 1, 4), + (16, 64, 64, 16, 32, False, True, True): (1, 4, 1, 4), + (16, 64, 64, 16, 32, True, False, False): (1, 4, 1, 2), + (16, 64, 64, 16, 32, True, False, True): (3, 4, 1, 4), + (32, 16, 16, 16, 16, False, False, False): (1, 1, 2, 4), + (32, 16, 16, 16, 16, False, False, True): (1, 1, 1, 2), + (32, 16, 16, 16, 16, False, True, False): (1, 1, 2, 4), + (32, 16, 16, 16, 16, False, True, True): (1, 1, 2, 4), + (32, 16, 16, 16, 16, True, False, False): (1, 1, 3, 8), + (32, 16, 16, 16, 16, True, False, True): (1, 1, 2, 4), + (32, 16, 32, 16, 16, False, False, False): (1, 2, 1, 4), + (32, 16, 32, 16, 16, False, False, True): (1, 2, 3, 4), + (32, 16, 32, 16, 16, False, True, False): (1, 1, 1, 8), + (32, 16, 32, 16, 16, False, True, True): (1, 2, 1, 4), + (32, 16, 32, 16, 16, True, False, False): (1, 1, 1, 2), + (32, 16, 32, 16, 16, True, False, True): (1, 1, 1, 4), + (32, 16, 64, 16, 16, False, False, False): (1, 4, 1, 4), + (32, 16, 64, 16, 16, False, False, True): (3, 4, 1, 4), + (32, 16, 64, 16, 16, False, True, False): (1, 4, 1, 1), + (32, 16, 64, 16, 16, False, True, True): (1, 4, 1, 4), + (32, 16, 64, 16, 16, True, False, False): (1, 4, 1, 4), + (32, 16, 64, 16, 16, True, False, True): (1, 4, 1, 4), + (32, 32, 16, 16, 16, False, False, False): (1, 1, 1, 2), + (32, 32, 16, 16, 16, False, False, True): (2, 1, 1, 4), + (32, 32, 16, 16, 16, False, True, False): (1, 1, 1, 2), + (32, 32, 16, 16, 16, False, True, True): (2, 1, 1, 4), + (32, 32, 16, 16, 16, True, False, False): (3, 1, 2, 4), + (32, 32, 16, 16, 16, True, False, True): (1, 1, 2, 4), + (32, 32, 16, 16, 32, False, False, False): (2, 1, 1, 2), + (32, 32, 16, 16, 32, False, False, True): (1, 1, 1, 4), + (32, 32, 16, 16, 32, False, True, False): (1, 1, 1, 4), + (32, 32, 16, 16, 32, False, True, True): (1, 1, 1, 8), + (32, 32, 16, 16, 32, True, False, False): (1, 1, 1, 8), + (32, 32, 16, 16, 32, True, False, True): (1, 1, 1, 4), + (32, 32, 16, 32, 32, False, False, False): (2, 1, 1, 4), + (32, 32, 16, 32, 32, False, False, True): (1, 1, 2, 4), + (32, 32, 16, 32, 32, False, True, False): (2, 1, 1, 1), + (32, 32, 16, 32, 32, False, True, True): (2, 1, 2, 4), + (32, 32, 16, 32, 32, True, False, False): (1, 1, 1, 8), + (32, 32, 16, 32, 32, True, False, True): (1, 1, 1, 4), + (32, 32, 32, 16, 16, False, False, False): (1, 1, 1, 4), + (32, 32, 32, 16, 16, False, False, True): (1, 2, 1, 2), + (32, 32, 32, 16, 16, False, True, False): (2, 2, 1, 4), + (32, 32, 32, 16, 16, False, True, True): (1, 2, 2, 4), + (32, 32, 32, 16, 16, True, False, False): (1, 2, 1, 4), + (32, 32, 32, 16, 16, True, False, True): (2, 2, 1, 4), + (32, 32, 32, 16, 32, False, False, False): (1, 2, 1, 4), + (32, 32, 32, 16, 32, False, False, True): (1, 2, 1, 4), + (32, 32, 32, 16, 32, False, True, False): (1, 2, 1, 4), + (32, 32, 32, 16, 32, False, True, True): (1, 2, 1, 4), + (32, 32, 32, 16, 32, True, False, False): (2, 1, 1, 2), + (32, 32, 32, 16, 32, True, False, True): (2, 2, 2, 4), + (32, 32, 32, 32, 32, False, False, False): (1, 1, 1, 4), + (32, 32, 32, 32, 32, False, False, True): (1, 1, 1, 2), + (32, 32, 32, 32, 32, False, True, False): (1, 1, 1, 4), + (32, 32, 32, 32, 32, False, True, True): (1, 1, 2, 2), + (32, 32, 32, 32, 32, True, False, False): (1, 1, 1, 2), + (32, 32, 32, 32, 32, True, False, True): (1, 1, 2, 1), + (32, 32, 64, 16, 16, False, False, False): (2, 4, 1, 4), + (32, 32, 64, 16, 16, False, False, True): (1, 4, 2, 4), + (32, 32, 64, 16, 16, False, True, False): (1, 4, 1, 4), + (32, 32, 64, 16, 16, False, True, True): (1, 4, 1, 4), + (32, 32, 64, 16, 16, True, False, False): (1, 2, 1, 4), + (32, 32, 64, 16, 16, True, False, True): (2, 4, 1, 4), + (32, 32, 64, 16, 32, False, False, False): (1, 4, 1, 8), + (32, 32, 64, 16, 32, False, False, True): (1, 4, 1, 4), + (32, 32, 64, 16, 32, False, True, False): (1, 4, 1, 4), + (32, 32, 64, 16, 32, False, True, True): (2, 4, 1, 4), + (32, 32, 64, 16, 32, True, False, False): (1, 2, 2, 4), + (32, 32, 64, 16, 32, True, False, True): (2, 4, 1, 4), + (32, 32, 64, 32, 32, False, False, False): (2, 2, 1, 4), + (32, 32, 64, 32, 32, False, False, True): (1, 1, 1, 4), + (32, 32, 64, 32, 32, False, True, False): (1, 1, 1, 8), + (32, 32, 64, 32, 32, False, True, True): (2, 1, 1, 4), + (32, 32, 64, 32, 32, True, False, False): (1, 1, 1, 4), + (32, 32, 64, 32, 32, True, False, True): (1, 2, 1, 1), + (32, 64, 16, 16, 32, False, False, False): (1, 1, 2, 2), + (32, 64, 16, 16, 32, False, False, True): (2, 1, 1, 4), + (32, 64, 16, 16, 32, False, True, False): (1, 1, 1, 8), + (32, 64, 16, 16, 32, False, True, True): (1, 1, 3, 4), + (32, 64, 16, 16, 32, True, False, False): (1, 1, 1, 2), + (32, 64, 16, 16, 32, True, False, True): (1, 1, 2, 4), + (32, 64, 16, 32, 32, False, False, False): (1, 1, 1, 2), + (32, 64, 16, 32, 32, False, False, True): (1, 1, 3, 4), + (32, 64, 16, 32, 32, False, True, False): (1, 1, 2, 4), + (32, 64, 16, 32, 32, False, True, True): (1, 1, 1, 8), + (32, 64, 16, 32, 32, True, False, False): (1, 1, 2, 4), + (32, 64, 16, 32, 32, True, False, True): (1, 1, 1, 8), + (32, 64, 32, 16, 32, False, False, False): (1, 2, 1, 4), + (32, 64, 32, 16, 32, False, False, True): (1, 2, 3, 4), + (32, 64, 32, 16, 32, False, True, False): (1, 2, 1, 8), + (32, 64, 32, 16, 32, False, True, True): (3, 2, 1, 4), + (32, 64, 32, 16, 32, True, False, False): (1, 1, 1, 8), + (32, 64, 32, 16, 32, True, False, True): (1, 2, 1, 4), + (32, 64, 32, 32, 32, False, False, False): (1, 1, 1, 1), + (32, 64, 32, 32, 32, False, False, True): (1, 1, 1, 4), + (32, 64, 32, 32, 32, False, True, False): (1, 1, 1, 4), + (32, 64, 32, 32, 32, False, True, True): (1, 1, 1, 4), + (32, 64, 32, 32, 32, True, False, False): (1, 1, 1, 4), + (32, 64, 32, 32, 32, True, False, True): (1, 1, 2, 8), + (32, 64, 64, 16, 32, False, False, False): (2, 4, 1, 4), + (32, 64, 64, 16, 32, False, False, True): (1, 4, 1, 4), + (32, 64, 64, 16, 32, False, True, False): (1, 4, 1, 4), + (32, 64, 64, 16, 32, False, True, True): (2, 4, 1, 4), + (32, 64, 64, 16, 32, True, False, False): (1, 4, 1, 4), + (32, 64, 64, 16, 32, True, False, True): (1, 4, 1, 4), + (32, 64, 64, 32, 32, False, False, False): (2, 2, 1, 4), + (32, 64, 64, 32, 32, False, False, True): (1, 2, 1, 8), + (32, 64, 64, 32, 32, False, True, False): (1, 2, 1, 4), + (32, 64, 64, 32, 32, False, True, True): (1, 2, 1, 4), + (32, 64, 64, 32, 32, True, False, False): (2, 2, 1, 4), + (32, 64, 64, 32, 32, True, False, True): (1, 2, 3, 8), + (64, 32, 16, 32, 32, False, False, False): (1, 1, 1, 4), + (64, 32, 16, 32, 32, False, False, True): (3, 1, 2, 4), + (64, 32, 16, 32, 32, False, True, False): (2, 1, 1, 2), + (64, 32, 16, 32, 32, False, True, True): (1, 1, 1, 8), + (64, 32, 16, 32, 32, True, False, False): (1, 1, 1, 2), + (64, 32, 16, 32, 32, True, False, True): (1, 1, 1, 4), + (64, 32, 32, 32, 32, False, False, False): (1, 1, 1, 4), + (64, 32, 32, 32, 32, False, False, True): (1, 1, 2, 8), + (64, 32, 32, 32, 32, False, True, False): (1, 1, 1, 8), + (64, 32, 32, 32, 32, False, True, True): (1, 1, 1, 4), + (64, 32, 32, 32, 32, True, False, False): (1, 1, 2, 4), + (64, 32, 32, 32, 32, True, False, True): (1, 1, 3, 8), + (64, 32, 64, 32, 32, False, False, False): (1, 2, 1, 4), + (64, 32, 64, 32, 32, False, False, True): (2, 2, 1, 4), + (64, 32, 64, 32, 32, False, True, False): (1, 1, 1, 4), + (64, 32, 64, 32, 32, False, True, True): (1, 2, 1, 8), + (64, 32, 64, 32, 32, True, False, False): (2, 2, 1, 4), + (64, 32, 64, 32, 32, True, False, True): (1, 2, 1, 8), + (64, 64, 16, 32, 32, False, False, False): (1, 1, 2, 8), + (64, 64, 16, 32, 32, False, False, True): (2, 1, 2, 4), + (64, 64, 16, 32, 32, False, True, False): (1, 1, 1, 2), + (64, 64, 16, 32, 32, False, True, True): (1, 1, 2, 4), + (64, 64, 16, 32, 32, True, False, False): (1, 1, 1, 2), + (64, 64, 16, 32, 32, True, False, True): (1, 1, 2, 4), + (64, 64, 32, 32, 32, False, False, False): (1, 1, 1, 4), + (64, 64, 32, 32, 32, False, False, True): (2, 1, 1, 4), + (64, 64, 32, 32, 32, False, True, False): (1, 1, 1, 8), + (64, 64, 32, 32, 32, False, True, True): (2, 1, 1, 4), + (64, 64, 32, 32, 32, True, False, False): (1, 1, 1, 4), + (64, 64, 32, 32, 32, True, False, True): (1, 1, 1, 8), + (64, 64, 64, 32, 32, False, False, False): (2, 2, 1, 4), + (64, 64, 64, 32, 32, False, False, True): (1, 2, 1, 4), + (64, 64, 64, 32, 32, False, True, False): (1, 2, 1, 4), + (64, 64, 64, 32, 32, False, True, True): (2, 2, 1, 4), + (64, 64, 64, 32, 32, True, False, False): (1, 1, 1, 8), + (64, 64, 64, 32, 32, True, False, True): (1, 2, 2, 4), + (192, 192, 256, 16, 16, False, True, True): (1, 16, 3, 2), + (192, 192, 256, 16, 16, True, False, True): (1, 8, 5, 4), + (192, 192, 256, 32, 32, False, True, True): (2, 8, 4, 4), + (192, 192, 256, 32, 32, True, False, True): (1, 8, 5, 4), + (192, 192, 512, 16, 16, False, True, True): (2, 16, 3, 4), + (192, 192, 512, 16, 16, True, False, True): (1, 16, 5, 4), + (192, 192, 512, 32, 32, False, True, True): (1, 16, 3, 4), + (192, 192, 512, 32, 32, True, False, True): (2, 16, 3, 4), + (192, 192, 1024, 16, 16, False, True, True): (3, 16, 3, 4), + (192, 192, 1024, 16, 16, True, False, True): (2, 8, 3, 4), + (192, 192, 1024, 32, 32, False, True, True): (3, 32, 1, 4), + (192, 192, 1024, 32, 32, True, False, True): (3, 16, 3, 4), + (192, 192, 2048, 16, 16, False, True, True): (1, 32, 3, 4), + (192, 192, 2048, 16, 16, True, False, True): (2, 16, 3, 4), + (192, 192, 2048, 32, 32, False, True, True): (1, 64, 1, 4), + (192, 192, 2048, 32, 32, True, False, True): (1, 64, 2, 4), + (192, 192, 4096, 16, 16, False, True, True): (1, 64, 2, 4), + (192, 192, 4096, 16, 16, True, False, True): (1, 32, 3, 4), + (192, 192, 4096, 32, 32, False, True, True): (3, 128, 2, 4), + (192, 192, 4096, 32, 32, True, False, True): (1, 128, 2, 4), + (192, 192, 8192, 16, 16, False, True, True): (2, 64, 3, 4), + (192, 192, 8192, 16, 16, True, False, True): (1, 64, 3, 4), + (192, 192, 8192, 32, 32, False, True, True): (3, 128, 3, 4), + (192, 192, 8192, 32, 32, True, False, True): (1, 128, 2, 4), + (192, 192, 16384, 16, 16, False, True, True): (1, 256, 3, 2), + (192, 192, 16384, 16, 16, True, False, True): (1, 256, 3, 2), + (192, 192, 16384, 32, 32, False, True, True): (2, 256, 3, 4), + (192, 192, 16384, 32, 32, True, False, True): (2, 256, 3, 4), + (192, 192, 32768, 16, 16, False, True, True): (2, 512, 3, 2), + (192, 192, 32768, 16, 16, True, False, True): (1, 128, 3, 4), + (192, 192, 32768, 32, 32, False, True, True): (2, 512, 3, 4), + (192, 192, 32768, 32, 32, True, False, True): (2, 512, 3, 4), + (192, 192, 65536, 16, 16, False, True, True): (2, 1024, 3, 2), + (192, 192, 65536, 16, 16, True, False, True): (1, 256, 3, 4), + (192, 192, 65536, 32, 32, False, True, True): (2, 1024, 3, 4), + (192, 192, 65536, 32, 32, True, False, True): (2, 1024, 3, 4), + (192, 192, 131072, 16, 16, False, True, True): (2, 512, 3, 4), + (192, 192, 131072, 16, 16, True, False, True): (1, 512, 3, 4), + (192, 192, 131072, 32, 32, False, True, True): (2, 1024, 3, 4), + (192, 192, 131072, 32, 32, True, False, True): (2, 1024, 3, 4), + (256, 256, 256, 16, 16, False, True, True): (1, 16, 3, 4), + (256, 256, 256, 16, 16, True, False, True): (2, 16, 1, 4), + (256, 256, 256, 32, 32, False, True, True): (1, 8, 4, 8), + (256, 256, 256, 32, 32, True, False, True): (4, 8, 4, 4), + (256, 256, 256, 64, 64, False, True, True): (1, 4, 4, 8), + (256, 256, 256, 64, 64, True, False, True): (1, 4, 3, 8), + (256, 256, 256, 128, 128, False, True, True): (7, 2, 1, 32), + (256, 256, 256, 128, 128, True, False, True): (3, 2, 1, 32), + (256, 256, 512, 16, 16, False, True, True): (1, 16, 5, 4), + (256, 256, 512, 16, 16, True, False, True): (1, 16, 3, 2), + (256, 256, 512, 32, 32, False, True, True): (4, 16, 4, 4), + (256, 256, 512, 32, 32, True, False, True): (4, 16, 3, 4), + (256, 256, 512, 64, 64, False, True, True): (1, 8, 3, 8), + (256, 256, 512, 64, 64, True, False, True): (1, 8, 3, 8), + (256, 256, 512, 128, 128, False, True, True): (1, 4, 1, 32), + (256, 256, 512, 128, 128, True, False, True): (3, 4, 1, 32), + (256, 256, 1024, 16, 16, False, True, True): (3, 32, 5, 2), + (256, 256, 1024, 16, 16, True, False, True): (2, 32, 5, 2), + (256, 256, 1024, 32, 32, False, True, True): (1, 32, 4, 4), + (256, 256, 1024, 32, 32, True, False, True): (1, 32, 5, 4), + (256, 256, 1024, 64, 64, False, True, True): (4, 16, 3, 8), + (256, 256, 1024, 64, 64, True, False, True): (1, 16, 3, 8), + (256, 256, 1024, 128, 128, False, True, True): (1, 8, 1, 32), + (256, 256, 1024, 128, 128, True, False, True): (3, 8, 1, 32), + (256, 256, 2048, 16, 16, False, True, True): (3, 32, 3, 4), + (256, 256, 2048, 16, 16, True, False, True): (1, 64, 3, 2), + (256, 256, 2048, 32, 32, False, True, True): (1, 64, 3, 4), + (256, 256, 2048, 32, 32, True, False, True): (1, 64, 3, 4), + (256, 256, 2048, 64, 64, False, True, True): (2, 32, 1, 8), + (256, 256, 2048, 64, 64, True, False, True): (2, 32, 1, 8), + (256, 256, 2048, 128, 128, False, True, True): (4, 16, 1, 32), + (256, 256, 2048, 128, 128, True, False, True): (4, 16, 1, 32), + (256, 256, 4096, 16, 16, False, True, True): (1, 32, 2, 4), + (256, 256, 4096, 16, 16, True, False, True): (1, 32, 3, 4), + (256, 256, 4096, 32, 32, False, True, True): (1, 128, 2, 4), + (256, 256, 4096, 32, 32, True, False, True): (1, 128, 2, 4), + (256, 256, 4096, 64, 64, False, True, True): (2, 64, 4, 8), + (256, 256, 4096, 64, 64, True, False, True): (3, 64, 2, 8), + (256, 256, 4096, 128, 128, False, True, True): (3, 32, 1, 32), + (256, 256, 4096, 128, 128, True, False, True): (2, 32, 1, 32), + (256, 256, 8192, 16, 16, False, True, True): (1, 64, 3, 4), + (256, 256, 8192, 16, 16, True, False, True): (2, 128, 3, 2), + (256, 256, 8192, 32, 32, False, True, True): (3, 128, 3, 4), + (256, 256, 8192, 32, 32, True, False, True): (1, 128, 3, 4), + (256, 256, 8192, 64, 64, False, True, True): (3, 128, 1, 4), + (256, 256, 8192, 64, 64, True, False, True): (4, 128, 2, 8), + (256, 256, 8192, 128, 128, False, True, True): (6, 64, 1, 32), + (256, 256, 8192, 128, 128, True, False, True): (2, 64, 1, 32), + (256, 256, 16384, 16, 16, False, True, True): (4, 128, 3, 4), + (256, 256, 16384, 16, 16, True, False, True): (3, 128, 3, 4), + (256, 256, 16384, 32, 32, False, True, True): (4, 256, 3, 4), + (256, 256, 16384, 32, 32, True, False, True): (2, 256, 3, 4), + (256, 256, 16384, 64, 64, False, True, True): (3, 256, 1, 4), + (256, 256, 16384, 64, 64, True, False, True): (2, 256, 2, 4), + (256, 256, 16384, 128, 128, False, True, True): (1, 128, 1, 32), + (256, 256, 16384, 128, 128, True, False, True): (3, 128, 1, 32), + (256, 256, 32768, 16, 16, False, True, True): (1, 256, 3, 4), + (256, 256, 32768, 16, 16, True, False, True): (2, 128, 3, 4), + (256, 256, 32768, 32, 32, False, True, True): (2, 512, 3, 4), + (256, 256, 32768, 32, 32, True, False, True): (4, 512, 3, 4), + (256, 256, 32768, 64, 64, False, True, True): (1, 512, 1, 8), + (256, 256, 32768, 64, 64, True, False, True): (1, 512, 2, 4), + (256, 256, 32768, 128, 128, False, True, True): (1, 256, 1, 32), + (256, 256, 32768, 128, 128, True, False, True): (1, 256, 1, 32), + (256, 256, 65536, 16, 16, False, True, True): (2, 512, 3, 4), + (256, 256, 65536, 16, 16, True, False, True): (1, 256, 3, 4), + (256, 256, 65536, 32, 32, False, True, True): (1, 1024, 3, 4), + (256, 256, 65536, 32, 32, True, False, True): (2, 1024, 3, 4), + (256, 256, 65536, 64, 64, False, True, True): (1, 1024, 2, 4), + (256, 256, 65536, 64, 64, True, False, True): (1, 1024, 2, 4), + (256, 256, 65536, 128, 128, False, True, True): (1, 512, 1, 32), + (256, 256, 65536, 128, 128, True, False, True): (2, 512, 1, 32), + (256, 256, 131072, 16, 16, False, True, True): (1, 1024, 3, 4), + (256, 256, 131072, 16, 16, True, False, True): (1, 512, 3, 4), + (256, 256, 131072, 32, 32, False, True, True): (1, 2048, 3, 4), + (256, 256, 131072, 32, 32, True, False, True): (1, 2048, 3, 4), + (256, 256, 131072, 64, 64, False, True, True): (1, 2048, 1, 8), + (256, 256, 131072, 64, 64, True, False, True): (1, 2048, 2, 4), + (256, 256, 131072, 128, 128, False, True, True): (1, 1024, 1, 32), + (256, 256, 131072, 128, 128, True, False, True): (4, 1024, 1, 32), + (384, 384, 256, 16, 16, False, True, True): (1, 8, 3, 4), + (384, 384, 256, 16, 16, True, False, True): (1, 8, 3, 4), + (384, 384, 256, 32, 32, False, True, True): (2, 8, 3, 8), + (384, 384, 256, 32, 32, True, False, True): (1, 8, 3, 4), + (384, 384, 256, 64, 64, False, True, True): (1, 4, 4, 8), + (384, 384, 256, 64, 64, True, False, True): (2, 4, 3, 8), + (384, 384, 512, 16, 16, False, True, True): (3, 16, 3, 2), + (384, 384, 512, 16, 16, True, False, True): (3, 16, 3, 2), + (384, 384, 512, 32, 32, False, True, True): (2, 8, 3, 4), + (384, 384, 512, 32, 32, True, False, True): (1, 8, 3, 4), + (384, 384, 512, 64, 64, False, True, True): (2, 8, 3, 8), + (384, 384, 512, 64, 64, True, False, True): (2, 8, 4, 8), + (384, 384, 1024, 16, 16, False, True, True): (3, 16, 3, 2), + (384, 384, 1024, 16, 16, True, False, True): (4, 32, 3, 2), + (384, 384, 1024, 32, 32, False, True, True): (1, 32, 3, 4), + (384, 384, 1024, 32, 32, True, False, True): (2, 16, 3, 4), + (384, 384, 1024, 64, 64, False, True, True): (2, 16, 3, 8), + (384, 384, 1024, 64, 64, True, False, True): (4, 16, 4, 8), + (384, 384, 2048, 16, 16, False, True, True): (3, 16, 3, 4), + (384, 384, 2048, 16, 16, True, False, True): (1, 32, 3, 4), + (384, 384, 2048, 32, 32, False, True, True): (3, 64, 2, 4), + (384, 384, 2048, 32, 32, True, False, True): (1, 64, 3, 4), + (384, 384, 2048, 64, 64, False, True, True): (4, 32, 4, 8), + (384, 384, 2048, 64, 64, True, False, True): (5, 32, 4, 8), + (384, 384, 4096, 16, 16, False, True, True): (1, 32, 3, 4), + (384, 384, 4096, 16, 16, True, False, True): (3, 32, 3, 4), + (384, 384, 4096, 32, 32, False, True, True): (2, 64, 3, 4), + (384, 384, 4096, 32, 32, True, False, True): (2, 64, 3, 4), + (384, 384, 4096, 64, 64, False, True, True): (2, 64, 3, 8), + (384, 384, 4096, 64, 64, True, False, True): (2, 64, 3, 8), + (384, 384, 8192, 16, 16, False, True, True): (1, 128, 3, 2), + (384, 384, 8192, 16, 16, True, False, True): (1, 128, 3, 2), + (384, 384, 8192, 32, 32, False, True, True): (1, 128, 3, 4), + (384, 384, 8192, 32, 32, True, False, True): (1, 128, 3, 4), + (384, 384, 8192, 64, 64, False, True, True): (3, 128, 3, 4), + (384, 384, 8192, 64, 64, True, False, True): (2, 128, 3, 4), + (384, 384, 16384, 16, 16, False, True, True): (1, 256, 3, 2), + (384, 384, 16384, 16, 16, True, False, True): (1, 64, 3, 4), + (384, 384, 16384, 32, 32, False, True, True): (2, 256, 3, 4), + (384, 384, 16384, 32, 32, True, False, True): (4, 256, 3, 4), + (384, 384, 16384, 64, 64, False, True, True): (2, 256, 3, 4), + (384, 384, 16384, 64, 64, True, False, True): (1, 256, 3, 4), + (384, 384, 32768, 16, 16, False, True, True): (1, 128, 3, 4), + (384, 384, 32768, 16, 16, True, False, True): (1, 128, 3, 4), + (384, 384, 32768, 32, 32, False, True, True): (1, 512, 3, 4), + (384, 384, 32768, 32, 32, True, False, True): (1, 512, 2, 4), + (384, 384, 32768, 64, 64, False, True, True): (1, 512, 3, 4), + (384, 384, 32768, 64, 64, True, False, True): (1, 512, 3, 4), + (384, 384, 65536, 16, 16, False, True, True): (1, 256, 3, 4), + (384, 384, 65536, 16, 16, True, False, True): (1, 256, 3, 4), + (384, 384, 65536, 32, 32, False, True, True): (1, 1024, 3, 4), + (384, 384, 65536, 32, 32, True, False, True): (1, 512, 3, 4), + (384, 384, 65536, 64, 64, False, True, True): (1, 1024, 3, 4), + (384, 384, 65536, 64, 64, True, False, True): (1, 1024, 3, 4), + (384, 384, 131072, 16, 16, False, True, True): (1, 512, 3, 4), + (384, 384, 131072, 16, 16, True, False, True): (1, 512, 3, 4), + (384, 384, 131072, 32, 32, False, True, True): (1, 1024, 3, 4), + (384, 384, 131072, 32, 32, True, False, True): (1, 1024, 3, 4), + (384, 384, 131072, 64, 64, False, True, True): (1, 2048, 3, 4), + (384, 384, 131072, 64, 64, True, False, True): (1, 2048, 3, 4), + (512, 512, 256, 16, 16, False, True, True): (1, 8, 4, 4), + (512, 512, 256, 16, 16, True, False, True): (1, 8, 3, 2), + (512, 512, 256, 32, 32, False, True, True): (4, 8, 3, 4), + (512, 512, 256, 32, 32, True, False, True): (4, 8, 3, 4), + (512, 512, 256, 64, 64, False, True, True): (3, 4, 3, 8), + (512, 512, 256, 64, 64, True, False, True): (5, 4, 3, 8), + (512, 512, 256, 128, 128, False, True, True): (1, 2, 1, 32), + (512, 512, 256, 128, 128, True, False, True): (3, 2, 1, 32), + (512, 512, 512, 16, 16, False, True, True): (2, 16, 3, 2), + (512, 512, 512, 16, 16, True, False, True): (1, 8, 4, 4), + (512, 512, 512, 32, 32, False, True, True): (3, 16, 3, 4), + (512, 512, 512, 32, 32, True, False, True): (5, 16, 2, 4), + (512, 512, 512, 64, 64, False, True, True): (1, 8, 3, 8), + (512, 512, 512, 64, 64, True, False, True): (3, 8, 3, 8), + (512, 512, 512, 128, 128, False, True, True): (1, 4, 1, 32), + (512, 512, 512, 128, 128, True, False, True): (3, 4, 1, 16), + (512, 512, 1024, 16, 16, False, True, True): (1, 16, 3, 4), + (512, 512, 1024, 16, 16, True, False, True): (3, 16, 3, 4), + (512, 512, 1024, 32, 32, False, True, True): (3, 32, 3, 4), + (512, 512, 1024, 32, 32, True, False, True): (3, 32, 2, 4), + (512, 512, 1024, 64, 64, False, True, True): (1, 16, 3, 8), + (512, 512, 1024, 64, 64, True, False, True): (4, 16, 3, 8), + (512, 512, 1024, 128, 128, False, True, True): (4, 8, 1, 32), + (512, 512, 1024, 128, 128, True, False, True): (4, 8, 1, 32), + (512, 512, 2048, 16, 16, False, True, True): (5, 16, 3, 4), + (512, 512, 2048, 16, 16, True, False, True): (5, 16, 3, 4), + (512, 512, 2048, 32, 32, False, True, True): (1, 32, 3, 4), + (512, 512, 2048, 32, 32, True, False, True): (1, 32, 4, 4), + (512, 512, 2048, 64, 64, False, True, True): (4, 32, 3, 8), + (512, 512, 2048, 64, 64, True, False, True): (4, 32, 3, 8), + (512, 512, 2048, 128, 128, False, True, True): (3, 16, 1, 32), + (512, 512, 2048, 128, 128, True, False, True): (3, 16, 1, 32), + (512, 512, 4096, 16, 16, False, True, True): (4, 32, 3, 4), + (512, 512, 4096, 16, 16, True, False, True): (4, 64, 3, 2), + (512, 512, 4096, 32, 32, False, True, True): (3, 64, 3, 4), + (512, 512, 4096, 32, 32, True, False, True): (3, 64, 3, 4), + (512, 512, 4096, 64, 64, False, True, True): (4, 64, 2, 4), + (512, 512, 4096, 64, 64, True, False, True): (1, 64, 2, 4), + (512, 512, 4096, 128, 128, False, True, True): (1, 32, 1, 32), + (512, 512, 4096, 128, 128, True, False, True): (1, 32, 1, 32), + (512, 512, 8192, 16, 16, False, True, True): (1, 64, 3, 4), + (512, 512, 8192, 16, 16, True, False, True): (4, 64, 3, 4), + (512, 512, 8192, 32, 32, False, True, True): (2, 128, 3, 4), + (512, 512, 8192, 32, 32, True, False, True): (3, 128, 3, 4), + (512, 512, 8192, 64, 64, False, True, True): (1, 128, 2, 4), + (512, 512, 8192, 64, 64, True, False, True): (1, 128, 2, 4), + (512, 512, 8192, 128, 128, False, True, True): (6, 64, 1, 32), + (512, 512, 8192, 128, 128, True, False, True): (4, 64, 1, 32), + (512, 512, 16384, 16, 16, False, True, True): (1, 128, 3, 4), + (512, 512, 16384, 16, 16, True, False, True): (1, 64, 3, 4), + (512, 512, 16384, 32, 32, False, True, True): (1, 256, 3, 4), + (512, 512, 16384, 32, 32, True, False, True): (4, 256, 3, 4), + (512, 512, 16384, 64, 64, False, True, True): (1, 256, 2, 4), + (512, 512, 16384, 64, 64, True, False, True): (1, 256, 2, 4), + (512, 512, 16384, 128, 128, False, True, True): (1, 128, 1, 32), + (512, 512, 16384, 128, 128, True, False, True): (2, 128, 1, 32), + (512, 512, 32768, 16, 16, False, True, True): (1, 256, 3, 4), + (512, 512, 32768, 16, 16, True, False, True): (1, 128, 3, 4), + (512, 512, 32768, 32, 32, False, True, True): (1, 512, 3, 4), + (512, 512, 32768, 32, 32, True, False, True): (1, 512, 3, 4), + (512, 512, 32768, 64, 64, False, True, True): (1, 512, 2, 4), + (512, 512, 32768, 64, 64, True, False, True): (2, 512, 2, 4), + (512, 512, 32768, 128, 128, False, True, True): (1, 256, 1, 32), + (512, 512, 32768, 128, 128, True, False, True): (2, 256, 1, 32), + (512, 512, 65536, 16, 16, False, True, True): (1, 512, 3, 4), + (512, 512, 65536, 16, 16, True, False, True): (1, 256, 3, 4), + (512, 512, 65536, 32, 32, False, True, True): (1, 1024, 3, 4), + (512, 512, 65536, 32, 32, True, False, True): (1, 1024, 3, 4), + (512, 512, 65536, 64, 64, False, True, True): (1, 1024, 2, 4), + (512, 512, 65536, 64, 64, True, False, True): (1, 1024, 2, 4), + (512, 512, 65536, 128, 128, False, True, True): (1, 512, 1, 32), + (512, 512, 65536, 128, 128, True, False, True): (4, 512, 1, 32), + (512, 512, 131072, 16, 16, False, True, True): (1, 512, 3, 4), + (512, 512, 131072, 16, 16, True, False, True): (1, 512, 3, 4), + (512, 512, 131072, 32, 32, False, True, True): (1, 2048, 3, 4), + (512, 512, 131072, 32, 32, True, False, True): (1, 2048, 3, 4), + (512, 512, 131072, 64, 64, False, True, True): (1, 2048, 2, 4), + (512, 512, 131072, 64, 64, True, False, True): (1, 2048, 2, 4), + (512, 512, 131072, 128, 128, False, True, True): (1, 1024, 1, 32), + (512, 512, 131072, 128, 128, True, False, True): (2, 1024, 1, 32), + (768, 768, 256, 16, 16, False, True, True): (1, 4, 5, 4), + (768, 768, 256, 16, 16, True, False, True): (3, 8, 3, 2), + (768, 768, 256, 32, 32, False, True, True): (2, 4, 3, 4), + (768, 768, 256, 32, 32, True, False, True): (3, 8, 4, 4), + (768, 768, 256, 64, 64, False, True, True): (1, 4, 4, 8), + (768, 768, 256, 64, 64, True, False, True): (3, 4, 3, 8), + (768, 768, 256, 128, 128, False, True, True): (3, 2, 1, 32), + (768, 768, 256, 128, 128, True, False, True): (2, 2, 2, 32), + (768, 768, 512, 16, 16, False, True, True): (2, 4, 5, 4), + (768, 768, 512, 16, 16, True, False, True): (2, 4, 4, 4), + (768, 768, 512, 32, 32, False, True, True): (1, 8, 3, 4), + (768, 768, 512, 32, 32, True, False, True): (3, 8, 4, 4), + (768, 768, 512, 64, 64, False, True, True): (2, 8, 3, 8), + (768, 768, 512, 64, 64, True, False, True): (5, 8, 3, 8), + (768, 768, 512, 128, 128, False, True, True): (2, 4, 1, 32), + (768, 768, 512, 128, 128, True, False, True): (2, 4, 2, 32), + (768, 768, 1024, 16, 16, False, True, True): (2, 16, 4, 2), + (768, 768, 1024, 16, 16, True, False, True): (4, 32, 3, 1), + (768, 768, 1024, 32, 32, False, True, True): (1, 32, 2, 4), + (768, 768, 1024, 32, 32, True, False, True): (1, 16, 5, 4), + (768, 768, 1024, 64, 64, False, True, True): (2, 16, 3, 8), + (768, 768, 1024, 64, 64, True, False, True): (2, 16, 3, 8), + (768, 768, 1024, 128, 128, False, True, True): (1, 8, 2, 32), + (768, 768, 1024, 128, 128, True, False, True): (1, 8, 1, 32), + (768, 768, 2048, 16, 16, False, True, True): (1, 16, 3, 4), + (768, 768, 2048, 16, 16, True, False, True): (1, 16, 3, 4), + (768, 768, 2048, 32, 32, False, True, True): (1, 32, 3, 4), + (768, 768, 2048, 32, 32, True, False, True): (5, 32, 3, 4), + (768, 768, 2048, 64, 64, False, True, True): (1, 32, 3, 8), + (768, 768, 2048, 64, 64, True, False, True): (1, 32, 3, 4), + (768, 768, 2048, 128, 128, False, True, True): (3, 16, 1, 32), + (768, 768, 2048, 128, 128, True, False, True): (4, 16, 1, 32), + (768, 768, 4096, 16, 16, False, True, True): (1, 64, 3, 2), + (768, 768, 4096, 16, 16, True, False, True): (3, 64, 3, 2), + (768, 768, 4096, 32, 32, False, True, True): (1, 64, 3, 4), + (768, 768, 4096, 32, 32, True, False, True): (1, 64, 3, 4), + (768, 768, 4096, 64, 64, False, True, True): (4, 64, 3, 4), + (768, 768, 4096, 64, 64, True, False, True): (4, 64, 3, 4), + (768, 768, 4096, 128, 128, False, True, True): (1, 32, 1, 32), + (768, 768, 4096, 128, 128, True, False, True): (1, 32, 2, 32), + (768, 768, 8192, 16, 16, False, True, True): (1, 128, 3, 2), + (768, 768, 8192, 16, 16, True, False, True): (2, 32, 3, 4), + (768, 768, 8192, 32, 32, False, True, True): (2, 128, 3, 4), + (768, 768, 8192, 32, 32, True, False, True): (1, 128, 2, 4), + (768, 768, 8192, 64, 64, False, True, True): (1, 128, 3, 4), + (768, 768, 8192, 64, 64, True, False, True): (2, 128, 3, 4), + (768, 768, 8192, 128, 128, False, True, True): (1, 64, 1, 32), + (768, 768, 8192, 128, 128, True, False, True): (2, 64, 1, 32), + (768, 768, 16384, 16, 16, False, True, True): (3, 64, 3, 4), + (768, 768, 16384, 16, 16, True, False, True): (1, 64, 3, 4), + (768, 768, 16384, 32, 32, False, True, True): (2, 256, 3, 4), + (768, 768, 16384, 32, 32, True, False, True): (4, 256, 2, 4), + (768, 768, 16384, 64, 64, False, True, True): (1, 256, 3, 4), + (768, 768, 16384, 64, 64, True, False, True): (1, 256, 3, 4), + (768, 768, 16384, 128, 128, False, True, True): (1, 128, 1, 32), + (768, 768, 16384, 128, 128, True, False, True): (2, 128, 1, 32), + (768, 768, 32768, 16, 16, False, True, True): (1, 128, 3, 4), + (768, 768, 32768, 16, 16, True, False, True): (2, 128, 3, 4), + (768, 768, 32768, 32, 32, False, True, True): (2, 256, 3, 4), + (768, 768, 32768, 32, 32, True, False, True): (1, 256, 3, 4), + (768, 768, 32768, 64, 64, False, True, True): (1, 512, 3, 4), + (768, 768, 32768, 64, 64, True, False, True): (1, 512, 3, 4), + (768, 768, 32768, 128, 128, False, True, True): (1, 256, 1, 32), + (768, 768, 32768, 128, 128, True, False, True): (1, 256, 1, 32), + (768, 768, 50432, 16, 16, False, True, True): (1, 197, 3, 4), + (768, 768, 50432, 32, 32, False, True, True): (1, 394, 3, 4), + (768, 768, 50432, 64, 64, False, True, True): (1, 788, 3, 4), + (768, 768, 50432, 128, 128, False, True, True): (3, 394, 1, 32), + (768, 768, 65536, 16, 16, False, True, True): (1, 256, 3, 4), + (768, 768, 65536, 16, 16, True, False, True): (1, 256, 3, 4), + (768, 768, 65536, 32, 32, False, True, True): (1, 512, 3, 4), + (768, 768, 65536, 32, 32, True, False, True): (1, 512, 3, 4), + (768, 768, 65536, 64, 64, False, True, True): (1, 1024, 3, 4), + (768, 768, 65536, 64, 64, True, False, True): (1, 1024, 3, 4), + (768, 768, 65536, 128, 128, False, True, True): (1, 512, 1, 32), + (768, 768, 65536, 128, 128, True, False, True): (1, 512, 1, 32), + (768, 768, 131072, 16, 16, False, True, True): (1, 512, 3, 4), + (768, 768, 131072, 16, 16, True, False, True): (1, 512, 3, 4), + (768, 768, 131072, 32, 32, False, True, True): (1, 1024, 3, 4), + (768, 768, 131072, 32, 32, True, False, True): (1, 1024, 3, 4), + (768, 768, 131072, 64, 64, False, True, True): (1, 2048, 3, 4), + (768, 768, 131072, 64, 64, True, False, True): (1, 2048, 3, 4), + (768, 768, 131072, 128, 128, False, True, True): (1, 1024, 1, 32), + (768, 768, 131072, 128, 128, True, False, True): (1, 1024, 1, 32), + (768, 3072, 256, 16, 16, False, True, True): (1, 2, 4, 4), + (768, 3072, 256, 16, 16, True, False, True): (1, 4, 3, 4), + (768, 3072, 256, 32, 32, False, True, True): (1, 4, 3, 4), + (768, 3072, 256, 32, 32, True, False, True): (3, 4, 3, 4), + (768, 3072, 256, 64, 64, False, True, True): (1, 4, 3, 8), + (768, 3072, 256, 64, 64, True, False, True): (1, 4, 3, 8), + (768, 3072, 256, 128, 128, False, True, True): (2, 2, 2, 32), + (768, 3072, 256, 128, 128, True, False, True): (2, 2, 1, 32), + (768, 3072, 512, 16, 16, False, True, True): (2, 4, 3, 4), + (768, 3072, 512, 16, 16, True, False, True): (1, 8, 3, 2), + (768, 3072, 512, 32, 32, False, True, True): (3, 8, 4, 4), + (768, 3072, 512, 32, 32, True, False, True): (3, 8, 3, 4), + (768, 3072, 512, 64, 64, False, True, True): (1, 8, 4, 8), + (768, 3072, 512, 64, 64, True, False, True): (1, 8, 3, 8), + (768, 3072, 512, 128, 128, False, True, True): (1, 4, 2, 32), + (768, 3072, 512, 128, 128, True, False, True): (1, 4, 1, 32), + (768, 3072, 1024, 16, 16, False, True, True): (4, 16, 3, 2), + (768, 3072, 1024, 16, 16, True, False, True): (4, 16, 3, 2), + (768, 3072, 1024, 32, 32, False, True, True): (4, 16, 5, 4), + (768, 3072, 1024, 32, 32, True, False, True): (4, 16, 5, 4), + (768, 3072, 1024, 64, 64, False, True, True): (2, 16, 3, 8), + (768, 3072, 1024, 64, 64, True, False, True): (2, 16, 3, 8), + (768, 3072, 1024, 128, 128, False, True, True): (1, 8, 1, 32), + (768, 3072, 1024, 128, 128, True, False, True): (1, 8, 1, 32), + (768, 3072, 2048, 16, 16, False, True, True): (2, 16, 3, 4), + (768, 3072, 2048, 16, 16, True, False, True): (2, 16, 3, 4), + (768, 3072, 2048, 32, 32, False, True, True): (4, 32, 5, 4), + (768, 3072, 2048, 32, 32, True, False, True): (2, 32, 3, 4), + (768, 3072, 2048, 64, 64, False, True, True): (2, 32, 3, 8), + (768, 3072, 2048, 64, 64, True, False, True): (2, 32, 3, 8), + (768, 3072, 2048, 128, 128, False, True, True): (1, 16, 1, 32), + (768, 3072, 2048, 128, 128, True, False, True): (2, 16, 1, 32), + (768, 3072, 4096, 16, 16, False, True, True): (1, 32, 5, 4), + (768, 3072, 4096, 16, 16, True, False, True): (3, 64, 3, 2), + (768, 3072, 4096, 32, 32, False, True, True): (5, 64, 3, 4), + (768, 3072, 4096, 32, 32, True, False, True): (5, 64, 3, 4), + (768, 3072, 4096, 64, 64, False, True, True): (1, 64, 3, 8), + (768, 3072, 4096, 64, 64, True, False, True): (5, 64, 3, 4), + (768, 3072, 4096, 128, 128, False, True, True): (1, 32, 1, 32), + (768, 3072, 4096, 128, 128, True, False, True): (1, 32, 1, 32), + (768, 3072, 8192, 16, 16, False, True, True): (1, 128, 3, 2), + (768, 3072, 8192, 16, 16, True, False, True): (1, 128, 3, 2), + (768, 3072, 8192, 32, 32, False, True, True): (1, 128, 3, 4), + (768, 3072, 8192, 32, 32, True, False, True): (1, 64, 3, 4), + (768, 3072, 8192, 64, 64, False, True, True): (3, 128, 3, 4), + (768, 3072, 8192, 64, 64, True, False, True): (3, 128, 3, 4), + (768, 3072, 8192, 128, 128, False, True, True): (4, 64, 2, 32), + (768, 3072, 8192, 128, 128, True, False, True): (2, 64, 1, 32), + (768, 3072, 16384, 16, 16, False, True, True): (1, 256, 2, 2), + (768, 3072, 16384, 16, 16, True, False, True): (1, 64, 3, 4), + (768, 3072, 16384, 32, 32, False, True, True): (8, 128, 3, 4), + (768, 3072, 16384, 32, 32, True, False, True): (1, 128, 3, 4), + (768, 3072, 16384, 64, 64, False, True, True): (1, 256, 3, 4), + (768, 3072, 16384, 64, 64, True, False, True): (3, 256, 3, 4), + (768, 3072, 16384, 128, 128, False, True, True): (3, 128, 1, 32), + (768, 3072, 16384, 128, 128, True, False, True): (2, 128, 2, 32), + (768, 3072, 32768, 16, 16, False, True, True): (1, 512, 3, 1), + (768, 3072, 32768, 16, 16, True, False, True): (1, 128, 3, 4), + (768, 3072, 32768, 32, 32, False, True, True): (1, 256, 3, 4), + (768, 3072, 32768, 32, 32, True, False, True): (1, 256, 3, 4), + (768, 3072, 32768, 64, 64, False, True, True): (2, 512, 3, 4), + (768, 3072, 32768, 64, 64, True, False, True): (1, 512, 3, 4), + (768, 3072, 32768, 128, 128, False, True, True): (1, 256, 1, 32), + (768, 3072, 32768, 128, 128, True, False, True): (2, 256, 2, 32), + (768, 3072, 50432, 16, 16, False, True, True): (1, 197, 3, 4), + (768, 3072, 50432, 16, 16, True, False, True): (1, 197, 3, 4), + (768, 3072, 50432, 32, 32, False, True, True): (1, 788, 2, 4), + (768, 3072, 50432, 32, 32, True, False, True): (1, 394, 3, 4), + (768, 3072, 50432, 64, 64, False, True, True): (1, 788, 3, 4), + (768, 3072, 50432, 64, 64, True, False, True): (2, 788, 3, 4), + (768, 3072, 50432, 128, 128, False, True, True): (1, 394, 1, 32), + (768, 3072, 50432, 128, 128, True, False, True): (2, 394, 2, 32), + (768, 3072, 65536, 16, 16, False, True, True): (1, 1024, 3, 1), + (768, 3072, 65536, 16, 16, True, False, True): (1, 256, 3, 4), + (768, 3072, 65536, 32, 32, False, True, True): (1, 512, 3, 4), + (768, 3072, 65536, 32, 32, True, False, True): (1, 512, 3, 4), + (768, 3072, 65536, 64, 64, False, True, True): (2, 1024, 3, 4), + (768, 3072, 65536, 64, 64, True, False, True): (5, 1024, 3, 4), + (768, 3072, 65536, 128, 128, False, True, True): (1, 512, 1, 32), + (768, 3072, 65536, 128, 128, True, False, True): (2, 512, 2, 32), + (768, 3072, 131072, 16, 16, False, True, True): (1, 2048, 3, 1), + (768, 3072, 131072, 16, 16, True, False, True): (1, 512, 3, 4), + (768, 3072, 131072, 32, 32, False, True, True): (1, 1024, 3, 4), + (768, 3072, 131072, 32, 32, True, False, True): (1, 1024, 3, 4), + (768, 3072, 131072, 64, 64, False, True, True): (1, 2048, 3, 4), + (768, 3072, 131072, 64, 64, True, False, True): (2, 2048, 3, 4), + (768, 3072, 131072, 128, 128, False, True, True): (1, 1024, 1, 32), + (768, 3072, 131072, 128, 128, True, False, True): (1, 1024, 2, 32), + (1024, 1024, 256, 16, 16, False, True, True): (4, 8, 3, 2), + (1024, 1024, 256, 16, 16, True, False, True): (2, 8, 3, 2), + (1024, 1024, 256, 32, 32, False, True, True): (1, 8, 3, 4), + (1024, 1024, 256, 32, 32, True, False, True): (1, 8, 3, 4), + (1024, 1024, 256, 64, 64, False, True, True): (1, 4, 3, 8), + (1024, 1024, 256, 64, 64, True, False, True): (2, 4, 3, 8), + (1024, 1024, 256, 128, 128, False, True, True): (3, 2, 1, 32), + (1024, 1024, 256, 128, 128, True, False, True): (5, 2, 1, 32), + (1024, 1024, 512, 16, 16, False, True, True): (3, 8, 3, 4), + (1024, 1024, 512, 16, 16, True, False, True): (3, 8, 3, 4), + (1024, 1024, 512, 32, 32, False, True, True): (1, 16, 3, 4), + (1024, 1024, 512, 32, 32, True, False, True): (3, 16, 3, 4), + (1024, 1024, 512, 64, 64, False, True, True): (6, 8, 3, 8), + (1024, 1024, 512, 64, 64, True, False, True): (8, 8, 3, 8), + (1024, 1024, 512, 128, 128, False, True, True): (1, 4, 1, 32), + (1024, 1024, 512, 128, 128, True, False, True): (1, 4, 1, 32), + (1024, 1024, 1024, 16, 16, False, True, True): (4, 8, 3, 4), + (1024, 1024, 1024, 16, 16, True, False, True): (1, 8, 3, 4), + (1024, 1024, 1024, 32, 32, False, True, True): (4, 16, 4, 4), + (1024, 1024, 1024, 32, 32, True, False, True): (5, 16, 3, 4), + (1024, 1024, 1024, 64, 64, False, True, True): (6, 16, 3, 8), + (1024, 1024, 1024, 64, 64, True, False, True): (3, 16, 2, 4), + (1024, 1024, 1024, 128, 128, False, True, True): (1, 8, 1, 32), + (1024, 1024, 1024, 128, 128, True, False, True): (2, 8, 1, 32), + (1024, 1024, 2048, 16, 16, False, True, True): (4, 16, 3, 4), + (1024, 1024, 2048, 16, 16, True, False, True): (1, 16, 3, 4), + (1024, 1024, 2048, 32, 32, False, True, True): (1, 32, 3, 4), + (1024, 1024, 2048, 32, 32, True, False, True): (2, 32, 3, 4), + (1024, 1024, 2048, 64, 64, False, True, True): (4, 32, 2, 4), + (1024, 1024, 2048, 64, 64, True, False, True): (8, 32, 2, 4), + (1024, 1024, 2048, 128, 128, False, True, True): (1, 16, 1, 32), + (1024, 1024, 2048, 128, 128, True, False, True): (1, 16, 1, 32), + (1024, 1024, 4096, 16, 16, False, True, True): (4, 32, 3, 4), + (1024, 1024, 4096, 16, 16, True, False, True): (1, 64, 3, 2), + (1024, 1024, 4096, 32, 32, False, True, True): (1, 64, 3, 4), + (1024, 1024, 4096, 32, 32, True, False, True): (1, 64, 3, 4), + (1024, 1024, 4096, 64, 64, False, True, True): (2, 64, 2, 4), + (1024, 1024, 4096, 64, 64, True, False, True): (2, 64, 2, 4), + (1024, 1024, 4096, 128, 128, False, True, True): (1, 32, 1, 32), + (1024, 1024, 4096, 128, 128, True, False, True): (4, 32, 1, 32), + (1024, 1024, 8192, 16, 16, False, True, True): (1, 128, 3, 1), + (1024, 1024, 8192, 16, 16, True, False, True): (1, 128, 3, 1), + (1024, 1024, 8192, 32, 32, False, True, True): (1, 128, 3, 4), + (1024, 1024, 8192, 32, 32, True, False, True): (1, 128, 3, 4), + (1024, 1024, 8192, 64, 64, False, True, True): (2, 128, 2, 4), + (1024, 1024, 8192, 64, 64, True, False, True): (2, 128, 2, 4), + (1024, 1024, 8192, 128, 128, False, True, True): (1, 64, 1, 32), + (1024, 1024, 8192, 128, 128, True, False, True): (4, 64, 1, 32), + (1024, 1024, 16384, 16, 16, False, True, True): (1, 128, 2, 4), + (1024, 1024, 16384, 16, 16, True, False, True): (4, 256, 3, 1), + (1024, 1024, 16384, 32, 32, False, True, True): (1, 256, 3, 4), + (1024, 1024, 16384, 32, 32, True, False, True): (1, 256, 3, 4), + (1024, 1024, 16384, 64, 64, False, True, True): (1, 256, 2, 4), + (1024, 1024, 16384, 64, 64, True, False, True): (1, 256, 2, 4), + (1024, 1024, 16384, 128, 128, False, True, True): (1, 128, 1, 32), + (1024, 1024, 16384, 128, 128, True, False, True): (4, 128, 1, 32), + (1024, 1024, 32768, 16, 16, False, True, True): (1, 256, 2, 4), + (1024, 1024, 32768, 16, 16, True, False, True): (4, 512, 3, 1), + (1024, 1024, 32768, 32, 32, False, True, True): (1, 512, 3, 4), + (1024, 1024, 32768, 32, 32, True, False, True): (1, 512, 3, 4), + (1024, 1024, 32768, 64, 64, False, True, True): (1, 512, 2, 4), + (1024, 1024, 32768, 64, 64, True, False, True): (1, 512, 2, 4), + (1024, 1024, 32768, 128, 128, False, True, True): (1, 256, 1, 32), + (1024, 1024, 32768, 128, 128, True, False, True): (1, 256, 1, 32), + (1024, 1024, 65536, 16, 16, False, True, True): (1, 512, 2, 4), + (1024, 1024, 65536, 16, 16, True, False, True): (1, 1024, 3, 1), + (1024, 1024, 65536, 32, 32, False, True, True): (1, 1024, 3, 4), + (1024, 1024, 65536, 32, 32, True, False, True): (1, 512, 3, 4), + (1024, 1024, 65536, 64, 64, False, True, True): (1, 1024, 2, 4), + (1024, 1024, 65536, 64, 64, True, False, True): (1, 1024, 2, 4), + (1024, 1024, 65536, 128, 128, False, True, True): (1, 512, 1, 32), + (1024, 1024, 65536, 128, 128, True, False, True): (1, 512, 1, 32), + (1024, 1024, 131072, 16, 16, False, True, True): (4, 2048, 3, 1), + (1024, 1024, 131072, 16, 16, True, False, True): (4, 2048, 3, 1), + (1024, 1024, 131072, 32, 32, False, True, True): (1, 2048, 3, 4), + (1024, 1024, 131072, 32, 32, True, False, True): (1, 1024, 3, 4), + (1024, 1024, 131072, 64, 64, False, True, True): (1, 2048, 2, 4), + (1024, 1024, 131072, 64, 64, True, False, True): (1, 2048, 2, 4), + (1024, 1024, 131072, 128, 128, False, True, True): (1, 1024, 1, 32), + (1024, 1024, 131072, 128, 128, True, False, True): (1, 1024, 1, 32), + (1280, 5120, 65792, 16, 16, False, True, True): (1, 1028, 3, 1), + (1280, 5120, 65792, 16, 16, True, False, True): (1, 257, 3, 4), + (1280, 5120, 65792, 32, 32, False, True, True): (1, 514, 3, 4), + (1280, 5120, 65792, 32, 32, True, False, True): (1, 514, 3, 4), + (1280, 5120, 65792, 64, 64, False, True, True): (2, 1028, 3, 4), + (1280, 5120, 65792, 64, 64, True, False, True): (1, 1028, 3, 4), + (1280, 5120, 65792, 128, 128, False, True, True): (2, 514, 2, 32), + (1280, 5120, 65792, 128, 128, True, False, True): (1, 514, 2, 32), + (1536, 1536, 256, 16, 16, False, True, True): (5, 4, 3, 2), + (1536, 1536, 256, 16, 16, True, False, True): (2, 2, 3, 4), + (1536, 1536, 256, 32, 32, False, True, True): (1, 8, 2, 4), + (1536, 1536, 256, 32, 32, True, False, True): (2, 4, 3, 4), + (1536, 1536, 256, 64, 64, False, True, True): (1, 4, 3, 8), + (1536, 1536, 256, 64, 64, True, False, True): (2, 4, 3, 8), + (1536, 1536, 256, 128, 128, False, True, True): (1, 2, 1, 32), + (1536, 1536, 256, 128, 128, True, False, True): (2, 2, 2, 32), + (1536, 1536, 512, 16, 16, False, True, True): (1, 8, 3, 2), + (1536, 1536, 512, 16, 16, True, False, True): (1, 8, 3, 2), + (1536, 1536, 512, 32, 32, False, True, True): (1, 16, 3, 4), + (1536, 1536, 512, 32, 32, True, False, True): (1, 16, 3, 4), + (1536, 1536, 512, 64, 64, False, True, True): (3, 8, 3, 8), + (1536, 1536, 512, 64, 64, True, False, True): (3, 8, 3, 8), + (1536, 1536, 512, 128, 128, False, True, True): (1, 4, 1, 32), + (1536, 1536, 512, 128, 128, True, False, True): (2, 4, 2, 32), + (1536, 1536, 1024, 16, 16, False, True, True): (2, 8, 3, 4), + (1536, 1536, 1024, 16, 16, True, False, True): (2, 8, 3, 4), + (1536, 1536, 1024, 32, 32, False, True, True): (1, 16, 3, 4), + (1536, 1536, 1024, 32, 32, True, False, True): (1, 16, 3, 4), + (1536, 1536, 1024, 64, 64, False, True, True): (2, 16, 3, 8), + (1536, 1536, 1024, 64, 64, True, False, True): (2, 16, 3, 8), + (1536, 1536, 1024, 128, 128, False, True, True): (3, 8, 1, 32), + (1536, 1536, 1024, 128, 128, True, False, True): (1, 8, 2, 32), + (1536, 1536, 2048, 16, 16, False, True, True): (1, 32, 3, 2), + (1536, 1536, 2048, 16, 16, True, False, True): (1, 32, 3, 2), + (1536, 1536, 2048, 32, 32, False, True, True): (3, 32, 2, 4), + (1536, 1536, 2048, 32, 32, True, False, True): (4, 32, 3, 4), + (1536, 1536, 2048, 64, 64, False, True, True): (1, 32, 3, 4), + (1536, 1536, 2048, 64, 64, True, False, True): (1, 32, 3, 4), + (1536, 1536, 2048, 128, 128, False, True, True): (1, 16, 1, 32), + (1536, 1536, 2048, 128, 128, True, False, True): (2, 16, 1, 32), + (1536, 1536, 4096, 16, 16, False, True, True): (1, 64, 3, 2), + (1536, 1536, 4096, 16, 16, True, False, True): (1, 16, 3, 4), + (1536, 1536, 4096, 32, 32, False, True, True): (1, 64, 2, 4), + (1536, 1536, 4096, 32, 32, True, False, True): (1, 64, 2, 4), + (1536, 1536, 4096, 64, 64, False, True, True): (1, 64, 3, 4), + (1536, 1536, 4096, 64, 64, True, False, True): (1, 64, 3, 4), + (1536, 1536, 4096, 128, 128, False, True, True): (1, 32, 1, 32), + (1536, 1536, 4096, 128, 128, True, False, True): (4, 32, 2, 32), + (1536, 1536, 8192, 16, 16, False, True, True): (1, 32, 3, 4), + (1536, 1536, 8192, 16, 16, True, False, True): (5, 32, 3, 4), + (1536, 1536, 8192, 32, 32, False, True, True): (1, 128, 2, 4), + (1536, 1536, 8192, 32, 32, True, False, True): (1, 128, 2, 4), + (1536, 1536, 8192, 64, 64, False, True, True): (1, 128, 3, 4), + (1536, 1536, 8192, 64, 64, True, False, True): (1, 128, 3, 4), + (1536, 1536, 8192, 128, 128, False, True, True): (1, 64, 1, 32), + (1536, 1536, 8192, 128, 128, True, False, True): (4, 64, 2, 32), + (1536, 1536, 16384, 16, 16, False, True, True): (1, 64, 3, 4), + (1536, 1536, 16384, 16, 16, True, False, True): (1, 64, 3, 4), + (1536, 1536, 16384, 32, 32, False, True, True): (1, 256, 2, 4), + (1536, 1536, 16384, 32, 32, True, False, True): (1, 128, 3, 4), + (1536, 1536, 16384, 64, 64, False, True, True): (1, 256, 3, 4), + (1536, 1536, 16384, 64, 64, True, False, True): (3, 256, 3, 4), + (1536, 1536, 16384, 128, 128, False, True, True): (1, 128, 1, 32), + (1536, 1536, 16384, 128, 128, True, False, True): (4, 128, 2, 32), + (1536, 1536, 32768, 16, 16, False, True, True): (1, 128, 3, 4), + (1536, 1536, 32768, 16, 16, True, False, True): (1, 128, 3, 4), + (1536, 1536, 32768, 32, 32, False, True, True): (1, 256, 3, 4), + (1536, 1536, 32768, 32, 32, True, False, True): (1, 256, 3, 4), + (1536, 1536, 32768, 64, 64, False, True, True): (1, 512, 3, 4), + (1536, 1536, 32768, 64, 64, True, False, True): (1, 512, 3, 4), + (1536, 1536, 32768, 128, 128, False, True, True): (1, 256, 1, 32), + (1536, 1536, 32768, 128, 128, True, False, True): (4, 256, 2, 32), + (1536, 1536, 65536, 16, 16, False, True, True): (5, 256, 3, 4), + (1536, 1536, 65536, 16, 16, True, False, True): (1, 256, 3, 4), + (1536, 1536, 65536, 32, 32, False, True, True): (1, 512, 3, 4), + (1536, 1536, 65536, 32, 32, True, False, True): (1, 512, 3, 4), + (1536, 1536, 65536, 64, 64, False, True, True): (1, 1024, 3, 4), + (1536, 1536, 65536, 64, 64, True, False, True): (1, 1024, 3, 4), + (1536, 1536, 65536, 128, 128, False, True, True): (1, 512, 1, 32), + (1536, 1536, 65536, 128, 128, True, False, True): (4, 512, 2, 32), + (1536, 1536, 131072, 16, 16, False, True, True): (3, 512, 3, 4), + (1536, 1536, 131072, 16, 16, True, False, True): (1, 512, 3, 4), + (1536, 1536, 131072, 32, 32, False, True, True): (1, 1024, 3, 4), + (1536, 1536, 131072, 32, 32, True, False, True): (1, 1024, 3, 4), + (1536, 1536, 131072, 64, 64, False, True, True): (1, 2048, 3, 4), + (1536, 1536, 131072, 64, 64, True, False, True): (1, 2048, 3, 4), + (1536, 1536, 131072, 128, 128, False, True, True): (1, 1024, 1, 32), + (1536, 1536, 131072, 128, 128, True, False, True): (4, 1024, 2, 32), + (2048, 2048, 256, 16, 16, False, True, True): (1, 4, 3, 4), + (2048, 2048, 256, 16, 16, True, False, True): (1, 4, 3, 4), + (2048, 2048, 256, 32, 32, False, True, True): (3, 8, 3, 4), + (2048, 2048, 256, 32, 32, True, False, True): (3, 8, 3, 4), + (2048, 2048, 256, 64, 64, False, True, True): (4, 4, 4, 8), + (2048, 2048, 256, 64, 64, True, False, True): (8, 4, 4, 8), + (2048, 2048, 256, 128, 128, False, True, True): (3, 2, 1, 32), + (2048, 2048, 256, 128, 128, True, False, True): (3, 2, 1, 32), + (2048, 2048, 512, 16, 16, False, True, True): (4, 8, 3, 2), + (2048, 2048, 512, 16, 16, True, False, True): (4, 8, 3, 2), + (2048, 2048, 512, 32, 32, False, True, True): (3, 8, 3, 4), + (2048, 2048, 512, 32, 32, True, False, True): (1, 16, 2, 4), + (2048, 2048, 512, 64, 64, False, True, True): (4, 8, 2, 4), + (2048, 2048, 512, 64, 64, True, False, True): (4, 8, 2, 4), + (2048, 2048, 512, 128, 128, False, True, True): (1, 4, 1, 32), + (2048, 2048, 512, 128, 128, True, False, True): (4, 4, 1, 32), + (2048, 2048, 1024, 16, 16, False, True, True): (4, 8, 3, 4), + (2048, 2048, 1024, 16, 16, True, False, True): (4, 8, 3, 4), + (2048, 2048, 1024, 32, 32, False, True, True): (4, 16, 3, 4), + (2048, 2048, 1024, 32, 32, True, False, True): (1, 16, 3, 4), + (2048, 2048, 1024, 64, 64, False, True, True): (2, 16, 2, 4), + (2048, 2048, 1024, 64, 64, True, False, True): (2, 16, 2, 4), + (2048, 2048, 1024, 128, 128, False, True, True): (8, 8, 1, 32), + (2048, 2048, 1024, 128, 128, True, False, True): (4, 8, 1, 32), + (2048, 2048, 2048, 16, 16, False, True, True): (4, 32, 3, 1), + (2048, 2048, 2048, 16, 16, True, False, True): (3, 32, 3, 2), + (2048, 2048, 2048, 32, 32, False, True, True): (1, 32, 3, 4), + (2048, 2048, 2048, 32, 32, True, False, True): (1, 32, 3, 4), + (2048, 2048, 2048, 64, 64, False, True, True): (2, 32, 2, 4), + (2048, 2048, 2048, 64, 64, True, False, True): (2, 32, 2, 4), + (2048, 2048, 2048, 128, 128, False, True, True): (6, 16, 1, 32), + (2048, 2048, 2048, 128, 128, True, False, True): (4, 16, 1, 32), + (2048, 2048, 4096, 16, 16, False, True, True): (4, 64, 3, 1), + (2048, 2048, 4096, 16, 16, True, False, True): (1, 64, 3, 1), + (2048, 2048, 4096, 32, 32, False, True, True): (1, 64, 3, 4), + (2048, 2048, 4096, 32, 32, True, False, True): (4, 64, 3, 4), + (2048, 2048, 4096, 64, 64, False, True, True): (2, 64, 2, 4), + (2048, 2048, 4096, 64, 64, True, False, True): (2, 64, 2, 4), + (2048, 2048, 4096, 128, 128, False, True, True): (4, 32, 1, 32), + (2048, 2048, 4096, 128, 128, True, False, True): (4, 32, 1, 32), + (2048, 2048, 8192, 16, 16, False, True, True): (4, 128, 3, 1), + (2048, 2048, 8192, 16, 16, True, False, True): (1, 128, 3, 1), + (2048, 2048, 8192, 32, 32, False, True, True): (4, 128, 3, 4), + (2048, 2048, 8192, 32, 32, True, False, True): (4, 64, 3, 4), + (2048, 2048, 8192, 64, 64, False, True, True): (1, 128, 2, 4), + (2048, 2048, 8192, 64, 64, True, False, True): (2, 128, 2, 4), + (2048, 2048, 8192, 128, 128, False, True, True): (1, 64, 1, 32), + (2048, 2048, 8192, 128, 128, True, False, True): (4, 64, 1, 32), + (2048, 2048, 16384, 16, 16, False, True, True): (4, 256, 3, 1), + (2048, 2048, 16384, 16, 16, True, False, True): (1, 256, 3, 1), + (2048, 2048, 16384, 32, 32, False, True, True): (1, 256, 3, 4), + (2048, 2048, 16384, 32, 32, True, False, True): (1, 128, 3, 4), + (2048, 2048, 16384, 64, 64, False, True, True): (1, 256, 2, 4), + (2048, 2048, 16384, 64, 64, True, False, True): (1, 256, 2, 4), + (2048, 2048, 16384, 128, 128, False, True, True): (1, 128, 1, 32), + (2048, 2048, 16384, 128, 128, True, False, True): (4, 128, 1, 32), + (2048, 2048, 32768, 16, 16, False, True, True): (8, 512, 3, 1), + (2048, 2048, 32768, 16, 16, True, False, True): (1, 512, 3, 1), + (2048, 2048, 32768, 32, 32, False, True, True): (1, 512, 3, 4), + (2048, 2048, 32768, 32, 32, True, False, True): (1, 256, 3, 4), + (2048, 2048, 32768, 64, 64, False, True, True): (1, 512, 2, 4), + (2048, 2048, 32768, 64, 64, True, False, True): (1, 512, 2, 4), + (2048, 2048, 32768, 128, 128, False, True, True): (1, 256, 1, 32), + (2048, 2048, 32768, 128, 128, True, False, True): (4, 256, 1, 32), + (2048, 2048, 65536, 16, 16, False, True, True): (4, 1024, 3, 1), + (2048, 2048, 65536, 16, 16, True, False, True): (1, 1024, 3, 1), + (2048, 2048, 65536, 32, 32, False, True, True): (1, 1024, 3, 4), + (2048, 2048, 65536, 32, 32, True, False, True): (1, 512, 3, 4), + (2048, 2048, 65536, 64, 64, False, True, True): (1, 1024, 2, 4), + (2048, 2048, 65536, 64, 64, True, False, True): (1, 1024, 2, 4), + (2048, 2048, 65536, 128, 128, False, True, True): (1, 512, 1, 32), + (2048, 2048, 65536, 128, 128, True, False, True): (4, 512, 1, 32), + (2048, 2048, 131072, 16, 16, False, True, True): (4, 2048, 3, 1), + (2048, 2048, 131072, 16, 16, True, False, True): (1, 2048, 3, 1), + (2048, 2048, 131072, 32, 32, False, True, True): (1, 2048, 3, 4), + (2048, 2048, 131072, 32, 32, True, False, True): (1, 1024, 3, 4), + (2048, 2048, 131072, 64, 64, False, True, True): (1, 2048, 2, 4), + (2048, 2048, 131072, 64, 64, True, False, True): (1, 2048, 2, 4), + (2048, 2048, 131072, 128, 128, False, True, True): (1, 1024, 1, 32), + (2048, 2048, 131072, 128, 128, True, False, True): (4, 1024, 1, 32), + (3072, 768, 256, 16, 16, False, True, True): (4, 4, 3, 2), + (3072, 768, 256, 16, 16, True, False, True): (1, 2, 6, 4), + (3072, 768, 256, 32, 32, False, True, True): (1, 4, 6, 4), + (3072, 768, 256, 32, 32, True, False, True): (5, 4, 3, 4), + (3072, 768, 256, 64, 64, False, True, True): (4, 4, 3, 8), + (3072, 768, 256, 64, 64, True, False, True): (4, 4, 3, 8), + (3072, 768, 256, 128, 128, False, True, True): (1, 2, 1, 32), + (3072, 768, 256, 128, 128, True, False, True): (5, 2, 1, 32), + (3072, 768, 512, 16, 16, False, True, True): (4, 4, 3, 4), + (3072, 768, 512, 16, 16, True, False, True): (1, 4, 3, 4), + (3072, 768, 512, 32, 32, False, True, True): (3, 8, 3, 4), + (3072, 768, 512, 32, 32, True, False, True): (3, 8, 3, 4), + (3072, 768, 512, 64, 64, False, True, True): (2, 8, 3, 8), + (3072, 768, 512, 64, 64, True, False, True): (2, 8, 3, 8), + (3072, 768, 512, 128, 128, False, True, True): (1, 4, 2, 32), + (3072, 768, 512, 128, 128, True, False, True): (1, 4, 1, 32), + (3072, 768, 1024, 16, 16, False, True, True): (1, 16, 3, 2), + (3072, 768, 1024, 16, 16, True, False, True): (3, 16, 3, 2), + (3072, 768, 1024, 32, 32, False, True, True): (1, 16, 3, 4), + (3072, 768, 1024, 32, 32, True, False, True): (3, 16, 3, 4), + (3072, 768, 1024, 64, 64, False, True, True): (4, 16, 3, 8), + (3072, 768, 1024, 64, 64, True, False, True): (4, 16, 3, 4), + (3072, 768, 1024, 128, 128, False, True, True): (5, 8, 1, 32), + (3072, 768, 1024, 128, 128, True, False, True): (5, 8, 1, 32), + (3072, 768, 2048, 16, 16, False, True, True): (4, 32, 3, 2), + (3072, 768, 2048, 16, 16, True, False, True): (1, 32, 3, 2), + (3072, 768, 2048, 32, 32, False, True, True): (1, 32, 3, 4), + (3072, 768, 2048, 32, 32, True, False, True): (1, 32, 2, 4), + (3072, 768, 2048, 64, 64, False, True, True): (2, 32, 3, 4), + (3072, 768, 2048, 64, 64, True, False, True): (4, 32, 3, 4), + (3072, 768, 2048, 128, 128, False, True, True): (1, 16, 1, 32), + (3072, 768, 2048, 128, 128, True, False, True): (1, 16, 1, 32), + (3072, 768, 4096, 16, 16, False, True, True): (3, 64, 3, 2), + (3072, 768, 4096, 16, 16, True, False, True): (1, 64, 3, 2), + (3072, 768, 4096, 32, 32, False, True, True): (1, 64, 3, 4), + (3072, 768, 4096, 32, 32, True, False, True): (1, 32, 3, 4), + (3072, 768, 4096, 64, 64, False, True, True): (2, 64, 3, 4), + (3072, 768, 4096, 64, 64, True, False, True): (2, 64, 3, 4), + (3072, 768, 4096, 128, 128, False, True, True): (1, 32, 1, 32), + (3072, 768, 4096, 128, 128, True, False, True): (1, 32, 1, 32), + (3072, 768, 8192, 16, 16, False, True, True): (4, 128, 3, 1), + (3072, 768, 8192, 16, 16, True, False, True): (1, 32, 3, 4), + (3072, 768, 8192, 32, 32, False, True, True): (1, 64, 3, 4), + (3072, 768, 8192, 32, 32, True, False, True): (1, 64, 3, 4), + (3072, 768, 8192, 64, 64, False, True, True): (2, 128, 3, 4), + (3072, 768, 8192, 64, 64, True, False, True): (2, 128, 3, 4), + (3072, 768, 8192, 128, 128, False, True, True): (1, 64, 1, 32), + (3072, 768, 8192, 128, 128, True, False, True): (1, 64, 1, 32), + (3072, 768, 16384, 16, 16, False, True, True): (4, 256, 3, 1), + (3072, 768, 16384, 16, 16, True, False, True): (1, 64, 3, 4), + (3072, 768, 16384, 32, 32, False, True, True): (1, 128, 3, 4), + (3072, 768, 16384, 32, 32, True, False, True): (1, 128, 3, 4), + (3072, 768, 16384, 64, 64, False, True, True): (2, 256, 3, 4), + (3072, 768, 16384, 64, 64, True, False, True): (2, 256, 3, 4), + (3072, 768, 16384, 128, 128, False, True, True): (1, 128, 1, 32), + (3072, 768, 16384, 128, 128, True, False, True): (1, 128, 1, 32), + (3072, 768, 32768, 16, 16, False, True, True): (4, 512, 3, 1), + (3072, 768, 32768, 16, 16, True, False, True): (1, 128, 3, 4), + (3072, 768, 32768, 32, 32, False, True, True): (1, 256, 3, 4), + (3072, 768, 32768, 32, 32, True, False, True): (1, 256, 3, 4), + (3072, 768, 32768, 64, 64, False, True, True): (2, 512, 3, 4), + (3072, 768, 32768, 64, 64, True, False, True): (2, 512, 3, 4), + (3072, 768, 32768, 128, 128, False, True, True): (1, 256, 1, 32), + (3072, 768, 32768, 128, 128, True, False, True): (1, 256, 1, 32), + (3072, 768, 50432, 16, 16, False, True, True): (4, 788, 3, 1), + (3072, 768, 50432, 16, 16, True, False, True): (1, 197, 3, 4), + (3072, 768, 50432, 32, 32, False, True, True): (1, 394, 3, 4), + (3072, 768, 50432, 32, 32, True, False, True): (1, 394, 3, 4), + (3072, 768, 50432, 64, 64, False, True, True): (1, 788, 3, 4), + (3072, 768, 50432, 64, 64, True, False, True): (2, 788, 3, 4), + (3072, 768, 50432, 128, 128, False, True, True): (1, 394, 1, 32), + (3072, 768, 50432, 128, 128, True, False, True): (1, 394, 1, 32), + (3072, 768, 65536, 16, 16, False, True, True): (4, 1024, 3, 1), + (3072, 768, 65536, 16, 16, True, False, True): (1, 256, 3, 4), + (3072, 768, 65536, 32, 32, False, True, True): (1, 512, 3, 4), + (3072, 768, 65536, 32, 32, True, False, True): (1, 512, 3, 4), + (3072, 768, 65536, 64, 64, False, True, True): (2, 1024, 3, 4), + (3072, 768, 65536, 64, 64, True, False, True): (2, 1024, 3, 4), + (3072, 768, 65536, 128, 128, False, True, True): (1, 512, 1, 32), + (3072, 768, 65536, 128, 128, True, False, True): (1, 512, 1, 32), + (3072, 768, 131072, 16, 16, False, True, True): (4, 2048, 3, 1), + (3072, 768, 131072, 16, 16, True, False, True): (1, 512, 3, 4), + (3072, 768, 131072, 32, 32, False, True, True): (1, 1024, 3, 4), + (3072, 768, 131072, 32, 32, True, False, True): (1, 1024, 3, 4), + (3072, 768, 131072, 64, 64, False, True, True): (2, 2048, 3, 4), + (3072, 768, 131072, 64, 64, True, False, True): (2, 2048, 3, 4), + (3072, 768, 131072, 128, 128, False, True, True): (1, 1024, 1, 32), + (3072, 768, 131072, 128, 128, True, False, True): (1, 1024, 1, 32), + (3072, 3072, 256, 16, 16, False, True, True): (1, 4, 5, 2), + (3072, 3072, 256, 16, 16, True, False, True): (1, 4, 3, 2), + (3072, 3072, 256, 32, 32, False, True, True): (1, 4, 4, 4), + (3072, 3072, 256, 32, 32, True, False, True): (1, 4, 3, 4), + (3072, 3072, 256, 64, 64, False, True, True): (2, 4, 3, 8), + (3072, 3072, 256, 64, 64, True, False, True): (2, 4, 3, 8), + (3072, 3072, 256, 128, 128, False, True, True): (6, 2, 1, 32), + (3072, 3072, 256, 128, 128, True, False, True): (8, 2, 2, 32), + (3072, 3072, 512, 16, 16, False, True, True): (2, 4, 3, 4), + (3072, 3072, 512, 16, 16, True, False, True): (2, 4, 3, 4), + (3072, 3072, 512, 32, 32, False, True, True): (2, 8, 3, 4), + (3072, 3072, 512, 32, 32, True, False, True): (2, 8, 3, 4), + (3072, 3072, 512, 64, 64, False, True, True): (2, 8, 3, 8), + (3072, 3072, 512, 64, 64, True, False, True): (2, 8, 3, 8), + (3072, 3072, 512, 128, 128, False, True, True): (5, 4, 1, 32), + (3072, 3072, 512, 128, 128, True, False, True): (5, 4, 2, 32), + (3072, 3072, 1024, 16, 16, False, True, True): (1, 16, 3, 2), + (3072, 3072, 1024, 16, 16, True, False, True): (1, 16, 3, 2), + (3072, 3072, 1024, 32, 32, False, True, True): (2, 16, 3, 4), + (3072, 3072, 1024, 32, 32, True, False, True): (1, 16, 3, 4), + (3072, 3072, 1024, 64, 64, False, True, True): (1, 16, 3, 4), + (3072, 3072, 1024, 64, 64, True, False, True): (1, 16, 3, 4), + (3072, 3072, 1024, 128, 128, False, True, True): (1, 8, 1, 32), + (3072, 3072, 1024, 128, 128, True, False, True): (3, 8, 2, 32), + (3072, 3072, 2048, 16, 16, False, True, True): (1, 32, 3, 2), + (3072, 3072, 2048, 16, 16, True, False, True): (1, 16, 2, 4), + (3072, 3072, 2048, 32, 32, False, True, True): (1, 32, 2, 4), + (3072, 3072, 2048, 32, 32, True, False, True): (1, 32, 3, 4), + (3072, 3072, 2048, 64, 64, False, True, True): (1, 32, 3, 4), + (3072, 3072, 2048, 64, 64, True, False, True): (1, 32, 3, 4), + (3072, 3072, 2048, 128, 128, False, True, True): (1, 16, 1, 32), + (3072, 3072, 2048, 128, 128, True, False, True): (4, 16, 2, 32), + (3072, 3072, 4096, 16, 16, False, True, True): (2, 16, 3, 4), + (3072, 3072, 4096, 16, 16, True, False, True): (2, 16, 3, 4), + (3072, 3072, 4096, 32, 32, False, True, True): (1, 64, 2, 4), + (3072, 3072, 4096, 32, 32, True, False, True): (1, 32, 3, 4), + (3072, 3072, 4096, 64, 64, False, True, True): (1, 64, 3, 4), + (3072, 3072, 4096, 64, 64, True, False, True): (1, 64, 3, 4), + (3072, 3072, 4096, 128, 128, False, True, True): (1, 32, 1, 32), + (3072, 3072, 4096, 128, 128, True, False, True): (2, 32, 2, 32), + (3072, 3072, 8192, 16, 16, False, True, True): (2, 32, 3, 4), + (3072, 3072, 8192, 16, 16, True, False, True): (2, 32, 3, 4), + (3072, 3072, 8192, 32, 32, False, True, True): (1, 64, 3, 4), + (3072, 3072, 8192, 32, 32, True, False, True): (1, 64, 3, 4), + (3072, 3072, 8192, 64, 64, False, True, True): (1, 128, 3, 4), + (3072, 3072, 8192, 64, 64, True, False, True): (1, 128, 3, 4), + (3072, 3072, 8192, 128, 128, False, True, True): (1, 64, 1, 32), + (3072, 3072, 8192, 128, 128, True, False, True): (4, 64, 2, 32), + (3072, 3072, 16384, 16, 16, False, True, True): (2, 64, 3, 4), + (3072, 3072, 16384, 16, 16, True, False, True): (1, 64, 3, 4), + (3072, 3072, 16384, 32, 32, False, True, True): (1, 128, 3, 4), + (3072, 3072, 16384, 32, 32, True, False, True): (1, 128, 3, 4), + (3072, 3072, 16384, 64, 64, False, True, True): (1, 256, 3, 4), + (3072, 3072, 16384, 64, 64, True, False, True): (1, 256, 3, 4), + (3072, 3072, 16384, 128, 128, False, True, True): (1, 128, 1, 32), + (3072, 3072, 16384, 128, 128, True, False, True): (4, 128, 2, 32), + (3072, 3072, 32768, 16, 16, False, True, True): (3, 128, 3, 4), + (3072, 3072, 32768, 16, 16, True, False, True): (1, 128, 3, 4), + (3072, 3072, 32768, 32, 32, False, True, True): (1, 256, 3, 4), + (3072, 3072, 32768, 32, 32, True, False, True): (1, 256, 3, 4), + (3072, 3072, 32768, 64, 64, False, True, True): (1, 512, 3, 4), + (3072, 3072, 32768, 64, 64, True, False, True): (1, 512, 3, 4), + (3072, 3072, 32768, 128, 128, False, True, True): (1, 256, 1, 32), + (3072, 3072, 32768, 128, 128, True, False, True): (4, 256, 2, 32), + (3072, 3072, 65536, 16, 16, False, True, True): (5, 256, 3, 4), + (3072, 3072, 65536, 16, 16, True, False, True): (2, 256, 3, 4), + (3072, 3072, 65536, 32, 32, False, True, True): (1, 512, 3, 4), + (3072, 3072, 65536, 32, 32, True, False, True): (1, 512, 3, 4), + (3072, 3072, 65536, 64, 64, False, True, True): (1, 1024, 3, 4), + (3072, 3072, 65536, 64, 64, True, False, True): (1, 1024, 3, 4), + (3072, 3072, 65536, 128, 128, False, True, True): (1, 512, 1, 32), + (3072, 3072, 65536, 128, 128, True, False, True): (4, 512, 2, 32), + (3072, 3072, 131072, 16, 16, False, True, True): (5, 512, 3, 4), + (3072, 3072, 131072, 16, 16, True, False, True): (1, 512, 3, 4), + (3072, 3072, 131072, 32, 32, False, True, True): (1, 1024, 3, 4), + (3072, 3072, 131072, 32, 32, True, False, True): (1, 1024, 3, 4), + (3072, 3072, 131072, 64, 64, False, True, True): (1, 2048, 3, 4), + (3072, 3072, 131072, 64, 64, True, False, True): (1, 2048, 3, 4), + (3072, 3072, 131072, 128, 128, False, True, True): (1, 1024, 1, 32), + (3072, 3072, 131072, 128, 128, True, False, True): (4, 1024, 2, 32), + (4096, 4096, 256, 16, 16, False, True, True): (1, 4, 3, 2), + (4096, 4096, 256, 16, 16, True, False, True): (1, 2, 3, 4), + (4096, 4096, 256, 32, 32, False, True, True): (4, 4, 4, 4), + (4096, 4096, 256, 32, 32, True, False, True): (4, 4, 4, 4), + (4096, 4096, 256, 64, 64, False, True, True): (1, 4, 3, 8), + (4096, 4096, 256, 64, 64, True, False, True): (4, 4, 2, 4), + (4096, 4096, 256, 128, 128, False, True, True): (1, 2, 1, 32), + (4096, 4096, 256, 128, 128, True, False, True): (3, 2, 1, 32), + (4096, 4096, 512, 16, 16, False, True, True): (1, 4, 3, 4), + (4096, 4096, 512, 16, 16, True, False, True): (5, 8, 3, 2), + (4096, 4096, 512, 32, 32, False, True, True): (4, 8, 3, 4), + (4096, 4096, 512, 32, 32, True, False, True): (4, 8, 3, 4), + (4096, 4096, 512, 64, 64, False, True, True): (1, 8, 2, 4), + (4096, 4096, 512, 64, 64, True, False, True): (1, 8, 2, 4), + (4096, 4096, 512, 128, 128, False, True, True): (4, 4, 1, 32), + (4096, 4096, 512, 128, 128, True, False, True): (4, 4, 1, 32), + (4096, 4096, 1024, 16, 16, False, True, True): (1, 8, 3, 4), + (4096, 4096, 1024, 16, 16, True, False, True): (1, 8, 3, 4), + (4096, 4096, 1024, 32, 32, False, True, True): (1, 16, 3, 4), + (4096, 4096, 1024, 32, 32, True, False, True): (1, 16, 3, 4), + (4096, 4096, 1024, 64, 64, False, True, True): (4, 16, 2, 4), + (4096, 4096, 1024, 64, 64, True, False, True): (4, 16, 2, 4), + (4096, 4096, 1024, 128, 128, False, True, True): (4, 8, 1, 32), + (4096, 4096, 1024, 128, 128, True, False, True): (4, 8, 1, 32), + (4096, 4096, 2048, 16, 16, False, True, True): (1, 32, 3, 1), + (4096, 4096, 2048, 16, 16, True, False, True): (6, 8, 3, 4), + (4096, 4096, 2048, 32, 32, False, True, True): (1, 32, 3, 4), + (4096, 4096, 2048, 32, 32, True, False, True): (1, 32, 3, 4), + (4096, 4096, 2048, 64, 64, False, True, True): (4, 32, 2, 4), + (4096, 4096, 2048, 64, 64, True, False, True): (4, 32, 2, 4), + (4096, 4096, 2048, 128, 128, False, True, True): (4, 16, 1, 32), + (4096, 4096, 2048, 128, 128, True, False, True): (4, 16, 1, 32), + (4096, 4096, 4096, 16, 16, False, True, True): (1, 16, 3, 4), + (4096, 4096, 4096, 16, 16, True, False, True): (1, 64, 3, 1), + (4096, 4096, 4096, 32, 32, False, True, True): (1, 64, 3, 4), + (4096, 4096, 4096, 32, 32, True, False, True): (1, 32, 3, 4), + (4096, 4096, 4096, 64, 64, False, True, True): (4, 64, 2, 4), + (4096, 4096, 4096, 64, 64, True, False, True): (4, 64, 2, 4), + (4096, 4096, 4096, 128, 128, False, True, True): (4, 32, 1, 32), + (4096, 4096, 4096, 128, 128, True, False, True): (4, 32, 1, 32), + (4096, 4096, 8192, 16, 16, False, True, True): (4, 128, 3, 1), + (4096, 4096, 8192, 16, 16, True, False, True): (1, 128, 3, 1), + (4096, 4096, 8192, 32, 32, False, True, True): (1, 128, 3, 4), + (4096, 4096, 8192, 32, 32, True, False, True): (1, 64, 3, 4), + (4096, 4096, 8192, 64, 64, False, True, True): (4, 128, 2, 4), + (4096, 4096, 8192, 64, 64, True, False, True): (4, 128, 2, 4), + (4096, 4096, 8192, 128, 128, False, True, True): (4, 64, 1, 32), + (4096, 4096, 8192, 128, 128, True, False, True): (4, 64, 1, 32), + (4096, 4096, 16384, 16, 16, False, True, True): (1, 64, 3, 4), + (4096, 4096, 16384, 16, 16, True, False, True): (1, 256, 3, 1), + (4096, 4096, 16384, 32, 32, False, True, True): (1, 256, 3, 4), + (4096, 4096, 16384, 32, 32, True, False, True): (1, 128, 3, 4), + (4096, 4096, 16384, 64, 64, False, True, True): (4, 256, 2, 4), + (4096, 4096, 16384, 64, 64, True, False, True): (4, 256, 2, 4), + (4096, 4096, 16384, 128, 128, False, True, True): (4, 128, 1, 32), + (4096, 4096, 16384, 128, 128, True, False, True): (4, 128, 1, 32), + (4096, 4096, 32768, 16, 16, False, True, True): (1, 128, 3, 4), + (4096, 4096, 32768, 16, 16, True, False, True): (1, 512, 3, 1), + (4096, 4096, 32768, 32, 32, False, True, True): (1, 512, 3, 4), + (4096, 4096, 32768, 32, 32, True, False, True): (1, 256, 3, 4), + (4096, 4096, 32768, 64, 64, False, True, True): (4, 512, 2, 4), + (4096, 4096, 32768, 64, 64, True, False, True): (4, 512, 2, 4), + (4096, 4096, 32768, 128, 128, False, True, True): (4, 256, 1, 32), + (4096, 4096, 32768, 128, 128, True, False, True): (4, 256, 1, 32), + (4096, 4096, 65536, 16, 16, False, True, True): (1, 256, 3, 4), + (4096, 4096, 65536, 16, 16, True, False, True): (1, 1024, 3, 1), + (4096, 4096, 65536, 32, 32, False, True, True): (1, 1024, 3, 4), + (4096, 4096, 65536, 32, 32, True, False, True): (1, 512, 3, 4), + (4096, 4096, 65536, 64, 64, False, True, True): (4, 1024, 2, 4), + (4096, 4096, 65536, 64, 64, True, False, True): (2, 1024, 2, 4), + (4096, 4096, 65536, 128, 128, False, True, True): (4, 512, 1, 32), + (4096, 4096, 65536, 128, 128, True, False, True): (4, 512, 1, 32), + (4096, 4096, 131072, 16, 16, False, True, True): (2, 2048, 3, 1), + (4096, 4096, 131072, 16, 16, True, False, True): (1, 2048, 3, 1), + (4096, 4096, 131072, 32, 32, False, True, True): (2, 2048, 3, 4), + (4096, 4096, 131072, 32, 32, True, False, True): (1, 1024, 3, 4), + (4096, 4096, 131072, 64, 64, False, True, True): (2, 2048, 2, 4), + (4096, 4096, 131072, 64, 64, True, False, True): (2, 2048, 2, 4), + (4096, 4096, 131072, 128, 128, False, True, True): (4, 1024, 1, 32), + (4096, 4096, 131072, 128, 128, True, False, True): (4, 1024, 1, 32), + (5120, 1280, 65792, 16, 16, False, True, True): (2, 1028, 3, 1), + (5120, 1280, 65792, 16, 16, True, False, True): (1, 257, 3, 4), + (5120, 1280, 65792, 32, 32, False, True, True): (1, 514, 3, 4), + (5120, 1280, 65792, 32, 32, True, False, True): (1, 514, 3, 4), + (5120, 1280, 65792, 64, 64, False, True, True): (1, 1028, 3, 4), + (5120, 1280, 65792, 64, 64, True, False, True): (5, 1028, 3, 4), + (5120, 1280, 65792, 128, 128, False, True, True): (1, 514, 1, 32), + (5120, 1280, 65792, 128, 128, True, False, True): (4, 514, 2, 32), + (6144, 6144, 256, 16, 16, False, True, True): (2, 2, 3, 4), + (6144, 6144, 256, 16, 16, True, False, True): (2, 2, 3, 4), + (6144, 6144, 256, 32, 32, False, True, True): (2, 4, 3, 4), + (6144, 6144, 256, 32, 32, True, False, True): (2, 4, 3, 4), + (6144, 6144, 256, 64, 64, False, True, True): (1, 4, 3, 4), + (6144, 6144, 256, 64, 64, True, False, True): (1, 4, 3, 4), + (6144, 6144, 256, 128, 128, False, True, True): (1, 2, 1, 32), + (6144, 6144, 256, 128, 128, True, False, True): (5, 2, 2, 32), + (6144, 6144, 512, 16, 16, False, True, True): (4, 8, 3, 2), + (6144, 6144, 512, 16, 16, True, False, True): (4, 8, 3, 2), + (6144, 6144, 512, 32, 32, False, True, True): (2, 8, 3, 4), + (6144, 6144, 512, 32, 32, True, False, True): (2, 8, 3, 4), + (6144, 6144, 512, 64, 64, False, True, True): (1, 8, 3, 4), + (6144, 6144, 512, 64, 64, True, False, True): (1, 8, 3, 4), + (6144, 6144, 512, 128, 128, False, True, True): (1, 4, 1, 32), + (6144, 6144, 512, 128, 128, True, False, True): (4, 4, 2, 32), + (6144, 6144, 1024, 16, 16, False, True, True): (4, 16, 3, 2), + (6144, 6144, 1024, 16, 16, True, False, True): (4, 4, 3, 4), + (6144, 6144, 1024, 32, 32, False, True, True): (1, 16, 3, 4), + (6144, 6144, 1024, 32, 32, True, False, True): (1, 16, 3, 4), + (6144, 6144, 1024, 64, 64, False, True, True): (1, 16, 3, 4), + (6144, 6144, 1024, 64, 64, True, False, True): (1, 16, 3, 4), + (6144, 6144, 1024, 128, 128, False, True, True): (1, 8, 1, 32), + (6144, 6144, 1024, 128, 128, True, False, True): (4, 8, 2, 32), + (6144, 6144, 2048, 16, 16, False, True, True): (1, 8, 3, 4), + (6144, 6144, 2048, 16, 16, True, False, True): (4, 8, 3, 4), + (6144, 6144, 2048, 32, 32, False, True, True): (1, 16, 3, 4), + (6144, 6144, 2048, 32, 32, True, False, True): (1, 16, 3, 4), + (6144, 6144, 2048, 64, 64, False, True, True): (1, 32, 3, 4), + (6144, 6144, 2048, 64, 64, True, False, True): (3, 32, 3, 4), + (6144, 6144, 2048, 128, 128, False, True, True): (1, 16, 1, 32), + (6144, 6144, 2048, 128, 128, True, False, True): (1, 16, 2, 32), + (6144, 6144, 4096, 16, 16, False, True, True): (3, 16, 3, 4), + (6144, 6144, 4096, 16, 16, True, False, True): (4, 16, 3, 4), + (6144, 6144, 4096, 32, 32, False, True, True): (1, 32, 3, 4), + (6144, 6144, 4096, 32, 32, True, False, True): (1, 32, 3, 4), + (6144, 6144, 4096, 64, 64, False, True, True): (1, 64, 3, 4), + (6144, 6144, 4096, 64, 64, True, False, True): (1, 64, 3, 4), + (6144, 6144, 4096, 128, 128, False, True, True): (1, 32, 1, 32), + (6144, 6144, 4096, 128, 128, True, False, True): (4, 32, 2, 32), + (6144, 6144, 8192, 16, 16, False, True, True): (1, 32, 3, 4), + (6144, 6144, 8192, 16, 16, True, False, True): (4, 32, 3, 4), + (6144, 6144, 8192, 32, 32, False, True, True): (1, 64, 3, 4), + (6144, 6144, 8192, 32, 32, True, False, True): (1, 64, 3, 4), + (6144, 6144, 8192, 64, 64, False, True, True): (1, 128, 3, 4), + (6144, 6144, 8192, 64, 64, True, False, True): (1, 128, 3, 4), + (6144, 6144, 8192, 128, 128, False, True, True): (1, 64, 1, 32), + (6144, 6144, 8192, 128, 128, True, False, True): (4, 64, 2, 32), + (6144, 6144, 16384, 16, 16, False, True, True): (1, 64, 3, 4), + (6144, 6144, 16384, 16, 16, True, False, True): (4, 64, 3, 4), + (6144, 6144, 16384, 32, 32, False, True, True): (1, 128, 3, 4), + (6144, 6144, 16384, 32, 32, True, False, True): (1, 128, 3, 4), + (6144, 6144, 16384, 64, 64, False, True, True): (1, 256, 3, 4), + (6144, 6144, 16384, 64, 64, True, False, True): (1, 256, 3, 4), + (6144, 6144, 16384, 128, 128, False, True, True): (1, 128, 1, 32), + (6144, 6144, 16384, 128, 128, True, False, True): (4, 128, 2, 32), + (6144, 6144, 32768, 16, 16, False, True, True): (1, 128, 3, 4), + (6144, 6144, 32768, 16, 16, True, False, True): (4, 128, 3, 4), + (6144, 6144, 32768, 32, 32, False, True, True): (1, 256, 3, 4), + (6144, 6144, 32768, 32, 32, True, False, True): (1, 256, 3, 4), + (6144, 6144, 32768, 64, 64, False, True, True): (1, 512, 3, 4), + (6144, 6144, 32768, 64, 64, True, False, True): (1, 512, 3, 4), + (6144, 6144, 32768, 128, 128, False, True, True): (1, 256, 1, 32), + (6144, 6144, 32768, 128, 128, True, False, True): (4, 256, 2, 32), + (6144, 6144, 65536, 16, 16, False, True, True): (1, 256, 3, 4), + (6144, 6144, 65536, 16, 16, True, False, True): (2, 256, 3, 4), + (6144, 6144, 65536, 32, 32, False, True, True): (1, 512, 3, 4), + (6144, 6144, 65536, 32, 32, True, False, True): (1, 512, 3, 4), + (6144, 6144, 65536, 64, 64, False, True, True): (1, 1024, 3, 4), + (6144, 6144, 65536, 64, 64, True, False, True): (1, 1024, 3, 4), + (6144, 6144, 65536, 128, 128, False, True, True): (1, 512, 1, 32), + (6144, 6144, 65536, 128, 128, True, False, True): (4, 512, 2, 32), + (6144, 6144, 131072, 16, 16, False, True, True): (1, 512, 3, 4), + (6144, 6144, 131072, 16, 16, True, False, True): (2, 512, 3, 4), + (6144, 6144, 131072, 32, 32, False, True, True): (1, 1024, 3, 4), + (6144, 6144, 131072, 32, 32, True, False, True): (1, 1024, 3, 4), + (6144, 6144, 131072, 64, 64, False, True, True): (1, 2048, 3, 4), + (6144, 6144, 131072, 64, 64, True, False, True): (1, 2048, 3, 4), + (6144, 6144, 131072, 128, 128, False, True, True): (1, 1024, 1, 32), + (6144, 6144, 131072, 128, 128, True, False, True): (4, 1024, 2, 32), + (8192, 8192, 256, 16, 16, False, True, True): (2, 2, 4, 4), + (8192, 8192, 256, 16, 16, True, False, True): (1, 1, 3, 4), + (8192, 8192, 256, 32, 32, False, True, True): (2, 4, 3, 4), + (8192, 8192, 256, 32, 32, True, False, True): (2, 4, 3, 4), + (8192, 8192, 256, 64, 64, False, True, True): (4, 4, 2, 4), + (8192, 8192, 256, 64, 64, True, False, True): (4, 4, 2, 4), + (8192, 8192, 256, 128, 128, False, True, True): (1, 2, 1, 32), + (8192, 8192, 256, 128, 128, True, False, True): (4, 2, 1, 32), + (8192, 8192, 512, 16, 16, False, True, True): (1, 4, 3, 4), + (8192, 8192, 512, 16, 16, True, False, True): (3, 4, 3, 4), + (8192, 8192, 512, 32, 32, False, True, True): (1, 8, 3, 4), + (8192, 8192, 512, 32, 32, True, False, True): (6, 8, 3, 4), + (8192, 8192, 512, 64, 64, False, True, True): (4, 8, 2, 4), + (8192, 8192, 512, 64, 64, True, False, True): (4, 8, 2, 4), + (8192, 8192, 512, 128, 128, False, True, True): (4, 4, 1, 32), + (8192, 8192, 512, 128, 128, True, False, True): (4, 4, 1, 32), + (8192, 8192, 1024, 16, 16, False, True, True): (1, 4, 3, 4), + (8192, 8192, 1024, 16, 16, True, False, True): (1, 32, 3, 1), + (8192, 8192, 1024, 32, 32, False, True, True): (1, 16, 3, 4), + (8192, 8192, 1024, 32, 32, True, False, True): (1, 16, 3, 4), + (8192, 8192, 1024, 64, 64, False, True, True): (4, 16, 2, 4), + (8192, 8192, 1024, 64, 64, True, False, True): (4, 16, 2, 4), + (8192, 8192, 1024, 128, 128, False, True, True): (4, 8, 1, 32), + (8192, 8192, 1024, 128, 128, True, False, True): (4, 8, 1, 32), + (8192, 8192, 2048, 16, 16, False, True, True): (4, 8, 3, 4), + (8192, 8192, 2048, 16, 16, True, False, True): (1, 32, 3, 1), + (8192, 8192, 2048, 32, 32, False, True, True): (1, 32, 3, 4), + (8192, 8192, 2048, 32, 32, True, False, True): (1, 16, 4, 4), + (8192, 8192, 2048, 64, 64, False, True, True): (4, 32, 2, 4), + (8192, 8192, 2048, 64, 64, True, False, True): (4, 32, 2, 4), + (8192, 8192, 2048, 128, 128, False, True, True): (4, 16, 1, 32), + (8192, 8192, 2048, 128, 128, True, False, True): (4, 16, 1, 32), + (8192, 8192, 4096, 16, 16, False, True, True): (3, 16, 3, 4), + (8192, 8192, 4096, 16, 16, True, False, True): (2, 64, 3, 1), + (8192, 8192, 4096, 32, 32, False, True, True): (1, 64, 3, 4), + (8192, 8192, 4096, 32, 32, True, False, True): (1, 32, 3, 4), + (8192, 8192, 4096, 64, 64, False, True, True): (4, 64, 2, 4), + (8192, 8192, 4096, 64, 64, True, False, True): (2, 64, 2, 4), + (8192, 8192, 4096, 128, 128, False, True, True): (4, 32, 1, 32), + (8192, 8192, 4096, 128, 128, True, False, True): (4, 32, 1, 32), + (8192, 8192, 8192, 16, 16, False, True, True): (2, 128, 3, 1), + (8192, 8192, 8192, 16, 16, True, False, True): (2, 128, 3, 1), + (8192, 8192, 8192, 32, 32, False, True, True): (1, 128, 3, 4), + (8192, 8192, 8192, 32, 32, True, False, True): (1, 64, 3, 4), + (8192, 8192, 8192, 64, 64, False, True, True): (4, 128, 2, 4), + (8192, 8192, 8192, 64, 64, True, False, True): (2, 128, 2, 4), + (8192, 8192, 8192, 128, 128, False, True, True): (4, 64, 1, 32), + (8192, 8192, 8192, 128, 128, True, False, True): (4, 64, 1, 32), + (8192, 8192, 16384, 16, 16, False, True, True): (1, 64, 3, 4), + (8192, 8192, 16384, 16, 16, True, False, True): (1, 256, 3, 1), + (8192, 8192, 16384, 32, 32, False, True, True): (1, 256, 3, 4), + (8192, 8192, 16384, 32, 32, True, False, True): (1, 128, 3, 4), + (8192, 8192, 16384, 64, 64, False, True, True): (2, 256, 2, 4), + (8192, 8192, 16384, 64, 64, True, False, True): (2, 256, 2, 4), + (8192, 8192, 16384, 128, 128, False, True, True): (4, 128, 1, 32), + (8192, 8192, 16384, 128, 128, True, False, True): (4, 128, 1, 32), + (8192, 8192, 32768, 16, 16, False, True, True): (1, 512, 3, 1), + (8192, 8192, 32768, 16, 16, True, False, True): (1, 512, 3, 1), + (8192, 8192, 32768, 32, 32, False, True, True): (1, 512, 3, 4), + (8192, 8192, 32768, 32, 32, True, False, True): (1, 256, 3, 4), + (8192, 8192, 32768, 64, 64, False, True, True): (2, 512, 2, 4), + (8192, 8192, 32768, 64, 64, True, False, True): (2, 512, 2, 4), + (8192, 8192, 32768, 128, 128, False, True, True): (4, 256, 1, 32), + (8192, 8192, 32768, 128, 128, True, False, True): (4, 256, 1, 32), + (8192, 8192, 65536, 16, 16, False, True, True): (1, 256, 3, 4), + (8192, 8192, 65536, 16, 16, True, False, True): (1, 1024, 3, 1), + (8192, 8192, 65536, 32, 32, False, True, True): (1, 1024, 3, 4), + (8192, 8192, 65536, 32, 32, True, False, True): (1, 512, 3, 4), + (8192, 8192, 65536, 64, 64, False, True, True): (4, 1024, 2, 4), + (8192, 8192, 65536, 64, 64, True, False, True): (2, 1024, 2, 4), + (8192, 8192, 65536, 128, 128, False, True, True): (4, 512, 1, 32), + (8192, 8192, 65536, 128, 128, True, False, True): (4, 512, 1, 32), + (8192, 8192, 131072, 16, 16, False, True, True): (1, 2048, 3, 1), + (8192, 8192, 131072, 16, 16, True, False, True): (2, 2048, 3, 1), + (8192, 8192, 131072, 32, 32, False, True, True): (4, 2048, 3, 4), + (8192, 8192, 131072, 32, 32, True, False, True): (1, 1024, 3, 4), + (8192, 8192, 131072, 64, 64, False, True, True): (2, 2048, 2, 4), + (8192, 8192, 131072, 64, 64, True, False, True): (2, 2048, 2, 4), + (8192, 8192, 131072, 128, 128, False, True, True): (4, 1024, 1, 32), + (8192, 8192, 131072, 128, 128, True, False, True): (4, 1024, 1, 32), + (16384, 16384, 256, 16, 16, False, True, True): (1, 2, 3, 4), + (16384, 16384, 256, 16, 16, True, False, True): (1, 2, 3, 4), + (16384, 16384, 256, 32, 32, False, True, True): (1, 4, 3, 4), + (16384, 16384, 256, 32, 32, True, False, True): (1, 4, 3, 4), + (16384, 16384, 256, 64, 64, False, True, True): (2, 4, 2, 4), + (16384, 16384, 256, 64, 64, True, False, True): (2, 4, 2, 4), + (16384, 16384, 256, 128, 128, False, True, True): (2, 2, 1, 32), + (16384, 16384, 256, 128, 128, True, False, True): (2, 2, 1, 32), + (16384, 16384, 512, 16, 16, False, True, True): (1, 2, 3, 4), + (16384, 16384, 512, 16, 16, True, False, True): (5, 2, 3, 4), + (16384, 16384, 512, 32, 32, False, True, True): (1, 8, 3, 4), + (16384, 16384, 512, 32, 32, True, False, True): (1, 4, 3, 4), + (16384, 16384, 512, 64, 64, False, True, True): (4, 8, 2, 4), + (16384, 16384, 512, 64, 64, True, False, True): (4, 8, 2, 4), + (16384, 16384, 512, 128, 128, False, True, True): (4, 4, 1, 32), + (16384, 16384, 512, 128, 128, True, False, True): (4, 4, 1, 32), + (16384, 16384, 1024, 16, 16, False, True, True): (1, 4, 3, 4), + (16384, 16384, 1024, 16, 16, True, False, True): (2, 16, 3, 1), + (16384, 16384, 1024, 32, 32, False, True, True): (1, 16, 3, 4), + (16384, 16384, 1024, 32, 32, True, False, True): (1, 8, 3, 4), + (16384, 16384, 1024, 64, 64, False, True, True): (4, 16, 2, 4), + (16384, 16384, 1024, 64, 64, True, False, True): (4, 16, 2, 4), + (16384, 16384, 1024, 128, 128, False, True, True): (4, 8, 1, 32), + (16384, 16384, 1024, 128, 128, True, False, True): (4, 8, 1, 32), + (16384, 16384, 2048, 16, 16, False, True, True): (1, 8, 3, 4), + (16384, 16384, 2048, 16, 16, True, False, True): (2, 32, 3, 1), + (16384, 16384, 2048, 32, 32, False, True, True): (1, 32, 3, 4), + (16384, 16384, 2048, 32, 32, True, False, True): (1, 16, 3, 4), + (16384, 16384, 2048, 64, 64, False, True, True): (4, 32, 2, 4), + (16384, 16384, 2048, 64, 64, True, False, True): (2, 32, 2, 4), + (16384, 16384, 2048, 128, 128, False, True, True): (4, 16, 1, 32), + (16384, 16384, 2048, 128, 128, True, False, True): (4, 16, 1, 32), + (16384, 16384, 4096, 16, 16, False, True, True): (1, 16, 3, 4), + (16384, 16384, 4096, 16, 16, True, False, True): (2, 64, 3, 1), + (16384, 16384, 4096, 32, 32, False, True, True): (1, 64, 3, 4), + (16384, 16384, 4096, 32, 32, True, False, True): (1, 32, 3, 4), + (16384, 16384, 4096, 64, 64, False, True, True): (4, 64, 2, 4), + (16384, 16384, 4096, 64, 64, True, False, True): (2, 64, 2, 4), + (16384, 16384, 4096, 128, 128, False, True, True): (4, 32, 1, 32), + (16384, 16384, 4096, 128, 128, True, False, True): (4, 32, 1, 32), + (16384, 16384, 8192, 16, 16, False, True, True): (1, 128, 3, 1), + (16384, 16384, 8192, 16, 16, True, False, True): (2, 128, 3, 1), + (16384, 16384, 8192, 32, 32, False, True, True): (1, 128, 3, 4), + (16384, 16384, 8192, 32, 32, True, False, True): (1, 64, 3, 4), + (16384, 16384, 8192, 64, 64, False, True, True): (2, 128, 2, 4), + (16384, 16384, 8192, 64, 64, True, False, True): (2, 128, 2, 4), + (16384, 16384, 8192, 128, 128, False, True, True): (4, 64, 1, 32), + (16384, 16384, 8192, 128, 128, True, False, True): (4, 64, 1, 32), + (16384, 16384, 16384, 16, 16, False, True, True): (1, 64, 3, 4), + (16384, 16384, 16384, 16, 16, True, False, True): (2, 256, 3, 1), + (16384, 16384, 16384, 32, 32, False, True, True): (1, 256, 3, 4), + (16384, 16384, 16384, 32, 32, True, False, True): (1, 128, 3, 4), + (16384, 16384, 16384, 64, 64, False, True, True): (2, 256, 2, 4), + (16384, 16384, 16384, 64, 64, True, False, True): (2, 256, 2, 4), + (16384, 16384, 16384, 128, 128, False, True, True): (4, 128, 1, 32), + (16384, 16384, 16384, 128, 128, True, False, True): (4, 128, 1, 32), + (16384, 16384, 32768, 16, 16, False, True, True): (1, 512, 3, 1), + (16384, 16384, 32768, 16, 16, True, False, True): (1, 128, 3, 4), + (16384, 16384, 32768, 32, 32, False, True, True): (2, 512, 3, 4), + (16384, 16384, 32768, 32, 32, True, False, True): (1, 256, 4, 4), + (16384, 16384, 32768, 64, 64, False, True, True): (2, 512, 2, 4), + (16384, 16384, 32768, 64, 64, True, False, True): (2, 512, 2, 4), + (16384, 16384, 32768, 128, 128, False, True, True): (4, 256, 1, 32), + (16384, 16384, 32768, 128, 128, True, False, True): (4, 256, 1, 32), + (16384, 16384, 65536, 16, 16, False, True, True): (1, 256, 3, 4), + (16384, 16384, 65536, 16, 16, True, False, True): (1, 1024, 3, 1), + (16384, 16384, 65536, 32, 32, False, True, True): (1, 1024, 3, 4), + (16384, 16384, 65536, 32, 32, True, False, True): (1, 512, 4, 4), + (16384, 16384, 65536, 64, 64, False, True, True): (2, 1024, 2, 4), + (16384, 16384, 65536, 64, 64, True, False, True): (2, 1024, 2, 4), + (16384, 16384, 65536, 128, 128, False, True, True): (4, 512, 1, 32), + (16384, 16384, 65536, 128, 128, True, False, True): (4, 512, 1, 32), + (16384, 16384, 131072, 16, 16, False, True, True): (1, 1024, 4, 4), + (16384, 16384, 131072, 16, 16, True, False, True): (2, 2048, 3, 1), + (16384, 16384, 131072, 32, 32, False, True, True): (1, 1024, 2, 4), + (16384, 16384, 131072, 32, 32, True, False, True): (1, 1024, 2, 4), + (16384, 16384, 131072, 64, 64, False, True, True): (4, 2048, 2, 4), + (16384, 16384, 131072, 64, 64, True, False, True): (2, 2048, 2, 4), + (16384, 16384, 131072, 128, 128, False, True, True): (4, 1024, 1, 32), + (16384, 16384, 131072, 128, 128, True, False, True): (4, 1024, 1, 32), + }, + ("bsr_dense_addmm", "NVIDIA A100-SXM4-80GB", (0, torch.float32, 0.56)): { + (192, 192, 256, 64, 64, False, True, True): (1, 4, 3, 8), + (192, 192, 256, 64, 64, True, False, True): (1, 4, 3, 8), + (192, 192, 512, 64, 64, False, True, True): (2, 8, 3, 8), + (192, 192, 512, 64, 64, True, False, True): (5, 8, 3, 8), + (192, 192, 1024, 64, 64, False, True, True): (2, 16, 4, 8), + (192, 192, 1024, 64, 64, True, False, True): (1, 16, 3, 8), + (192, 192, 2048, 64, 64, False, True, True): (3, 32, 3, 8), + (192, 192, 2048, 64, 64, True, False, True): (5, 32, 5, 8), + (192, 192, 4096, 64, 64, False, True, True): (3, 64, 2, 8), + (192, 192, 4096, 64, 64, True, False, True): (1, 64, 3, 8), + (192, 192, 8192, 64, 64, False, True, True): (3, 128, 3, 8), + (192, 192, 8192, 64, 64, True, False, True): (6, 128, 3, 4), + (192, 192, 16384, 64, 64, False, True, True): (1, 256, 1, 8), + (192, 192, 16384, 64, 64, True, False, True): (1, 256, 3, 4), + (192, 192, 32768, 64, 64, False, True, True): (1, 512, 1, 8), + (192, 192, 32768, 64, 64, True, False, True): (1, 512, 3, 4), + (192, 192, 65536, 64, 64, False, True, True): (1, 1024, 1, 8), + (192, 192, 65536, 64, 64, True, False, True): (1, 1024, 3, 4), + (192, 192, 131072, 64, 64, False, True, True): (1, 2048, 1, 8), + (192, 192, 131072, 64, 64, True, False, True): (3, 2048, 1, 4), + (384, 384, 256, 128, 128, False, True, True): (1, 2, 1, 32), + (384, 384, 256, 128, 128, True, False, True): (1, 2, 1, 32), + (384, 384, 512, 128, 128, False, True, True): (1, 4, 1, 32), + (384, 384, 512, 128, 128, True, False, True): (2, 4, 1, 32), + (384, 384, 1024, 128, 128, False, True, True): (1, 8, 1, 32), + (384, 384, 1024, 128, 128, True, False, True): (4, 8, 1, 32), + (384, 384, 2048, 128, 128, False, True, True): (1, 16, 1, 32), + (384, 384, 2048, 128, 128, True, False, True): (1, 16, 1, 32), + (384, 384, 4096, 128, 128, False, True, True): (1, 32, 1, 32), + (384, 384, 4096, 128, 128, True, False, True): (2, 32, 2, 32), + (384, 384, 8192, 128, 128, False, True, True): (1, 64, 1, 32), + (384, 384, 8192, 128, 128, True, False, True): (1, 64, 2, 32), + (384, 384, 16384, 128, 128, False, True, True): (1, 128, 1, 32), + (384, 384, 16384, 128, 128, True, False, True): (4, 128, 1, 32), + (384, 384, 32768, 128, 128, False, True, True): (3, 256, 1, 32), + (384, 384, 32768, 128, 128, True, False, True): (3, 256, 1, 32), + (384, 384, 65536, 128, 128, False, True, True): (3, 512, 1, 32), + (384, 384, 65536, 128, 128, True, False, True): (3, 512, 1, 32), + (384, 384, 131072, 128, 128, False, True, True): (1, 1024, 1, 32), + (384, 384, 131072, 128, 128, True, False, True): (3, 1024, 1, 32), + }, + ("bsr_dense_addmm", "NVIDIA A100-SXM4-80GB", (0, torch.int8, 0.5)): { + (1280, 5120, 65792, 32, 32, False, True, True): (1, 1028, 1, 8), + (1280, 5120, 65792, 32, 32, True, False, True): (1, 514, 3, 2), + (1280, 5120, 65792, 64, 64, False, True, True): (2, 514, 1, 4), + (1280, 5120, 65792, 64, 64, True, False, True): (1, 514, 3, 2), + (1280, 5120, 65792, 128, 128, False, True, True): (2, 514, 1, 8), + (1280, 5120, 65792, 128, 128, True, False, True): (1, 514, 2, 4), + (1280, 5120, 65792, 256, 256, False, True, True): (1, 257, 1, 32), + (1280, 5120, 65792, 256, 256, True, False, True): (1, 257, 1, 32), + (5120, 1280, 65792, 32, 32, False, True, True): (3, 1028, 1, 8), + (5120, 1280, 65792, 32, 32, True, False, True): (1, 514, 1, 2), + (5120, 1280, 65792, 64, 64, False, True, True): (1, 514, 1, 4), + (5120, 1280, 65792, 64, 64, True, False, True): (2, 514, 2, 2), + (5120, 1280, 65792, 128, 128, False, True, True): (2, 514, 1, 8), + (5120, 1280, 65792, 128, 128, True, False, True): (2, 514, 2, 4), + (5120, 1280, 65792, 256, 256, False, True, True): (1, 257, 1, 32), + (5120, 1280, 65792, 256, 256, True, False, True): (1, 257, 1, 32), + }, + ("scatter_mm", "NVIDIA A100-SXM4-80GB", (0, torch.bfloat16, 0.5)): { + (256, 256, 256, 16, 16): (1, 1, 16, 16, 1, 2), + (256, 256, 256, 32, 32): (1, 1, 16, 16, 1, 4), + (256, 256, 256, 64, 64): (1, 1, 16, 16, 1, 1), + (256, 256, 256, 128, 128): (2, 4, 16, 64, 1, 4), + (256, 256, 512, 16, 16): (1, 1, 16, 16, 1, 4), + (256, 256, 512, 32, 32): (1, 1, 16, 32, 1, 4), + (256, 256, 512, 64, 64): (1, 1, 16, 32, 1, 1), + (256, 256, 512, 128, 128): (1, 1, 32, 32, 1, 4), + (256, 256, 1024, 16, 16): (1, 1, 16, 16, 1, 4), + (256, 256, 1024, 32, 32): (1, 2, 16, 32, 1, 1), + (256, 256, 1024, 64, 64): (1, 1, 32, 32, 1, 2), + (256, 256, 1024, 128, 128): (1, 1, 32, 64, 1, 4), + (256, 256, 2048, 16, 16): (1, 1, 16, 64, 1, 8), + (256, 256, 2048, 32, 32): (2, 1, 32, 64, 1, 2), + (256, 256, 2048, 64, 64): (1, 1, 32, 32, 1, 1), + (256, 256, 2048, 128, 128): (1, 1, 64, 64, 1, 4), + (256, 256, 4096, 16, 16): (1, 1, 16, 64, 1, 1), + (256, 256, 4096, 32, 32): (2, 2, 32, 64, 1, 2), + (256, 256, 4096, 64, 64): (1, 1, 32, 128, 1, 4), + (256, 256, 4096, 128, 128): (1, 1, 64, 64, 1, 4), + (256, 256, 8192, 16, 16): (1, 2, 16, 64, 1, 2), + (256, 256, 8192, 32, 32): (1, 1, 32, 64, 1, 2), + (256, 256, 8192, 64, 64): (1, 1, 32, 64, 1, 2), + (256, 256, 8192, 128, 128): (1, 1, 64, 64, 1, 4), + (256, 256, 16384, 16, 16): (1, 1, 16, 64, 1, 2), + (256, 256, 16384, 32, 32): (1, 1, 32, 64, 1, 2), + (256, 256, 16384, 64, 64): (1, 1, 64, 64, 1, 2), + (256, 256, 16384, 128, 128): (2, 16, 64, 64, 1, 4), + (256, 256, 32768, 16, 16): (1, 1, 16, 128, 1, 2), + (256, 256, 32768, 32, 32): (1, 1, 32, 64, 1, 2), + (256, 256, 32768, 64, 64): (1, 1, 64, 64, 1, 2), + (256, 256, 32768, 128, 128): (2, 32, 64, 64, 1, 4), + (256, 256, 65536, 16, 16): (1, 1, 16, 64, 1, 1), + (256, 256, 65536, 32, 32): (1, 1, 32, 64, 1, 2), + (256, 256, 65536, 64, 64): (1, 1, 64, 32, 1, 1), + (256, 256, 65536, 128, 128): (2, 32, 64, 64, 1, 4), + (256, 256, 131072, 16, 16): (1, 1, 16, 64, 1, 1), + (256, 256, 131072, 32, 32): (1, 1, 32, 64, 1, 2), + (256, 256, 131072, 64, 64): (4, 1, 64, 32, 1, 1), + (256, 256, 131072, 128, 128): (2, 64, 64, 64, 1, 4), + (512, 512, 256, 16, 16): (1, 1, 16, 16, 1, 2), + (512, 512, 256, 32, 32): (1, 1, 16, 32, 1, 1), + (512, 512, 256, 64, 64): (1, 2, 16, 32, 1, 1), + (512, 512, 256, 128, 128): (2, 16, 64, 16, 2, 4), + (512, 512, 512, 16, 16): (1, 1, 16, 16, 1, 4), + (512, 512, 512, 32, 32): (1, 1, 16, 32, 1, 1), + (512, 512, 512, 64, 64): (1, 1, 32, 32, 1, 2), + (512, 512, 512, 128, 128): (2, 8, 32, 64, 1, 4), + (512, 512, 1024, 16, 16): (1, 1, 16, 64, 1, 8), + (512, 512, 1024, 32, 32): (1, 1, 32, 32, 3, 1), + (512, 512, 1024, 64, 64): (1, 4, 32, 64, 1, 2), + (512, 512, 1024, 128, 128): (1, 4, 64, 64, 1, 4), + (512, 512, 2048, 16, 16): (1, 1, 16, 64, 1, 2), + (512, 512, 2048, 32, 32): (1, 1, 32, 64, 1, 2), + (512, 512, 2048, 64, 64): (1, 1, 64, 64, 3, 4), + (512, 512, 2048, 128, 128): (1, 1, 64, 64, 1, 4), + (512, 512, 4096, 16, 16): (1, 1, 16, 64, 1, 2), + (512, 512, 4096, 32, 32): (2, 64, 32, 64, 1, 2), + (512, 512, 4096, 64, 64): (1, 1, 64, 64, 3, 4), + (512, 512, 4096, 128, 128): (1, 1, 64, 64, 1, 4), + (512, 512, 8192, 16, 16): (1, 2, 16, 128, 1, 2), + (512, 512, 8192, 32, 32): (1, 1, 32, 64, 1, 2), + (512, 512, 8192, 64, 64): (1, 1, 64, 64, 1, 2), + (512, 512, 8192, 128, 128): (1, 1, 64, 64, 1, 4), + (512, 512, 16384, 16, 16): (1, 2, 16, 128, 1, 2), + (512, 512, 16384, 32, 32): (1, 1, 32, 64, 1, 2), + (512, 512, 16384, 64, 64): (1, 1, 64, 64, 3, 2), + (512, 512, 16384, 128, 128): (2, 1, 64, 64, 1, 4), + (512, 512, 32768, 16, 16): (1, 2, 16, 128, 1, 2), + (512, 512, 32768, 32, 32): (1, 1, 32, 64, 1, 2), + (512, 512, 32768, 64, 64): (1, 1, 64, 64, 3, 4), + (512, 512, 32768, 128, 128): (2, 1, 64, 64, 1, 4), + (512, 512, 65536, 16, 16): (1, 2, 16, 128, 1, 2), + (512, 512, 65536, 32, 32): (1, 1, 32, 64, 1, 2), + (512, 512, 65536, 64, 64): (1, 1, 64, 64, 3, 4), + (512, 512, 65536, 128, 128): (2, 1, 64, 64, 1, 4), + (512, 512, 131072, 16, 16): (1, 1, 16, 64, 1, 1), + (512, 512, 131072, 32, 32): (1, 1, 32, 64, 1, 2), + (512, 512, 131072, 64, 64): (1, 1, 64, 64, 3, 4), + (512, 512, 131072, 128, 128): (2, 4, 64, 64, 1, 4), + (1024, 1024, 256, 16, 16): (1, 1, 16, 16, 1, 4), + (1024, 1024, 256, 32, 32): (2, 16, 32, 16, 3, 4), + (1024, 1024, 256, 64, 64): (1, 4, 32, 32, 1, 2), + (1024, 1024, 256, 128, 128): (1, 4, 128, 16, 3, 16), + (1024, 1024, 512, 16, 16): (1, 1, 16, 64, 1, 2), + (1024, 1024, 512, 32, 32): (2, 2, 32, 64, 1, 2), + (1024, 1024, 512, 64, 64): (2, 8, 64, 64, 3, 4), + (1024, 1024, 512, 128, 128): (1, 4, 64, 64, 1, 8), + (1024, 1024, 1024, 16, 16): (1, 1, 16, 64, 1, 2), + (1024, 1024, 1024, 32, 32): (1, 1, 32, 64, 1, 2), + (1024, 1024, 1024, 64, 64): (1, 8, 64, 64, 3, 4), + (1024, 1024, 1024, 128, 128): (1, 8, 64, 64, 1, 4), + (1024, 1024, 2048, 16, 16): (1, 2, 16, 64, 1, 2), + (1024, 1024, 2048, 32, 32): (1, 1, 32, 64, 1, 2), + (1024, 1024, 2048, 64, 64): (2, 16, 64, 64, 2, 2), + (1024, 1024, 2048, 128, 128): (2, 32, 64, 64, 1, 4), + (1024, 1024, 4096, 16, 16): (2, 16, 16, 128, 1, 2), + (1024, 1024, 4096, 32, 32): (1, 16, 32, 64, 3, 2), + (1024, 1024, 4096, 64, 64): (1, 1, 64, 64, 3, 4), + (1024, 1024, 4096, 128, 128): (2, 64, 128, 64, 1, 4), + (1024, 1024, 8192, 16, 16): (2, 16, 16, 128, 1, 2), + (1024, 1024, 8192, 32, 32): (1, 16, 32, 64, 3, 2), + (1024, 1024, 8192, 64, 64): (1, 1, 64, 64, 3, 4), + (1024, 1024, 8192, 128, 128): (2, 1, 64, 64, 1, 4), + (1024, 1024, 16384, 16, 16): (1, 2, 16, 128, 1, 2), + (1024, 1024, 16384, 32, 32): (1, 16, 32, 64, 3, 2), + (1024, 1024, 16384, 64, 64): (1, 1, 64, 64, 3, 4), + (1024, 1024, 16384, 128, 128): (2, 16, 128, 64, 1, 4), + (1024, 1024, 32768, 16, 16): (1, 1, 16, 128, 1, 2), + (1024, 1024, 32768, 32, 32): (1, 1, 32, 128, 1, 2), + (1024, 1024, 32768, 64, 64): (1, 32, 64, 32, 2, 1), + (1024, 1024, 32768, 128, 128): (2, 8, 128, 64, 1, 4), + (1024, 1024, 65536, 16, 16): (3, 2, 16, 128, 1, 2), + (1024, 1024, 65536, 32, 32): (1, 1, 32, 128, 1, 2), + (1024, 1024, 65536, 64, 64): (2, 4, 64, 32, 2, 1), + (1024, 1024, 65536, 128, 128): (2, 8, 128, 64, 1, 4), + (1024, 1024, 131072, 16, 16): (2, 1, 16, 128, 1, 2), + (1024, 1024, 131072, 32, 32): (1, 1, 32, 128, 1, 2), + (1024, 1024, 131072, 64, 64): (1, 4, 64, 32, 2, 1), + (1024, 1024, 131072, 128, 128): (4, 1, 128, 64, 1, 4), + (2048, 2048, 256, 16, 16): (1, 1, 16, 64, 1, 8), + (2048, 2048, 256, 32, 32): (1, 1, 32, 32, 3, 1), + (2048, 2048, 256, 64, 64): (1, 1, 32, 32, 2, 1), + (2048, 2048, 256, 128, 128): (1, 4, 64, 64, 1, 8), + (2048, 2048, 512, 16, 16): (1, 2, 16, 64, 1, 2), + (2048, 2048, 512, 32, 32): (1, 2, 32, 64, 1, 4), + (2048, 2048, 512, 64, 64): (1, 4, 64, 64, 1, 8), + (2048, 2048, 512, 128, 128): (1, 4, 64, 64, 1, 4), + (2048, 2048, 1024, 16, 16): (1, 2, 16, 128, 1, 2), + (2048, 2048, 1024, 32, 32): (1, 1, 32, 64, 1, 2), + (2048, 2048, 1024, 64, 64): (1, 8, 64, 64, 1, 4), + (2048, 2048, 1024, 128, 128): (1, 8, 128, 64, 1, 4), + (2048, 2048, 2048, 16, 16): (3, 4, 16, 128, 1, 2), + (2048, 2048, 2048, 32, 32): (1, 16, 32, 64, 5, 2), + (2048, 2048, 2048, 64, 64): (1, 1, 64, 64, 3, 4), + (2048, 2048, 2048, 128, 128): (1, 8, 128, 64, 1, 4), + (2048, 2048, 4096, 16, 16): (1, 2, 16, 128, 1, 2), + (2048, 2048, 4096, 32, 32): (1, 8, 32, 64, 3, 2), + (2048, 2048, 4096, 64, 64): (1, 1, 64, 64, 3, 4), + (2048, 2048, 4096, 128, 128): (1, 8, 128, 64, 1, 4), + (2048, 2048, 8192, 16, 16): (2, 4, 16, 128, 1, 2), + (2048, 2048, 8192, 32, 32): (1, 4, 32, 128, 3, 2), + (2048, 2048, 8192, 64, 64): (1, 8, 64, 64, 3, 2), + (2048, 2048, 8192, 128, 128): (1, 8, 128, 64, 1, 4), + (2048, 2048, 16384, 16, 16): (1, 2, 16, 128, 1, 2), + (2048, 2048, 16384, 32, 32): (1, 4, 32, 128, 3, 2), + (2048, 2048, 16384, 64, 64): (1, 8, 64, 64, 3, 2), + (2048, 2048, 16384, 128, 128): (1, 4, 128, 64, 1, 4), + (2048, 2048, 32768, 16, 16): (3, 2, 16, 128, 1, 2), + (2048, 2048, 32768, 32, 32): (1, 1, 32, 128, 3, 2), + (2048, 2048, 32768, 64, 64): (1, 1, 64, 64, 3, 2), + (2048, 2048, 32768, 128, 128): (1, 4, 128, 64, 1, 4), + (2048, 2048, 65536, 16, 16): (1, 2, 16, 128, 1, 2), + (2048, 2048, 65536, 32, 32): (1, 4, 32, 128, 1, 2), + (2048, 2048, 65536, 64, 64): (1, 1, 64, 64, 3, 2), + (2048, 2048, 65536, 128, 128): (1, 2, 128, 64, 1, 4), + (2048, 2048, 131072, 16, 16): (4, 2, 16, 128, 1, 2), + (2048, 2048, 131072, 32, 32): (1, 1, 32, 128, 3, 2), + (2048, 2048, 131072, 64, 64): (1, 1, 64, 64, 3, 2), + (2048, 2048, 131072, 128, 128): (1, 2, 128, 64, 1, 4), + (4096, 4096, 256, 16, 16): (1, 1, 16, 64, 1, 2), + (4096, 4096, 256, 32, 32): (1, 1, 32, 64, 3, 4), + (4096, 4096, 256, 64, 64): (1, 1, 64, 64, 3, 4), + (4096, 4096, 256, 128, 128): (3, 4, 128, 32, 1, 4), + (4096, 4096, 512, 16, 16): (1, 2, 16, 128, 1, 2), + (4096, 4096, 512, 32, 32): (1, 2, 32, 64, 3, 2), + (4096, 4096, 512, 64, 64): (1, 4, 64, 64, 1, 4), + (4096, 4096, 512, 128, 128): (1, 4, 128, 64, 1, 4), + (4096, 4096, 1024, 16, 16): (1, 2, 16, 128, 1, 2), + (4096, 4096, 1024, 32, 32): (1, 8, 32, 64, 3, 2), + (4096, 4096, 1024, 64, 64): (1, 4, 64, 64, 1, 4), + (4096, 4096, 1024, 128, 128): (2, 4, 128, 64, 1, 4), + (4096, 4096, 2048, 16, 16): (1, 1, 16, 128, 1, 2), + (4096, 4096, 2048, 32, 32): (1, 4, 32, 128, 1, 4), + (4096, 4096, 2048, 64, 64): (1, 1, 64, 64, 3, 4), + (4096, 4096, 2048, 128, 128): (1, 16, 128, 64, 1, 4), + (4096, 4096, 4096, 16, 16): (1, 1, 16, 64, 3, 1), + (4096, 4096, 4096, 32, 32): (1, 4, 32, 64, 3, 2), + (4096, 4096, 4096, 64, 64): (1, 1, 64, 64, 3, 4), + (4096, 4096, 4096, 128, 128): (5, 1, 128, 64, 1, 4), + (4096, 4096, 8192, 16, 16): (1, 1, 16, 128, 1, 2), + (4096, 4096, 8192, 32, 32): (1, 1, 32, 128, 3, 2), + (4096, 4096, 8192, 64, 64): (1, 1, 64, 64, 3, 4), + (4096, 4096, 8192, 128, 128): (2, 1, 128, 64, 1, 4), + (4096, 4096, 16384, 16, 16): (1, 1, 16, 128, 1, 2), + (4096, 4096, 16384, 32, 32): (1, 1, 32, 128, 3, 2), + (4096, 4096, 16384, 64, 64): (1, 1, 64, 64, 4, 4), + (4096, 4096, 16384, 128, 128): (2, 1, 128, 64, 1, 4), + (4096, 4096, 32768, 16, 16): (3, 1, 16, 128, 1, 2), + (4096, 4096, 32768, 32, 32): (1, 1, 32, 128, 3, 2), + (4096, 4096, 32768, 64, 64): (1, 1, 64, 64, 3, 4), + (4096, 4096, 32768, 128, 128): (2, 1, 128, 64, 1, 4), + (4096, 4096, 65536, 16, 16): (2, 2, 16, 128, 1, 2), + (4096, 4096, 65536, 32, 32): (1, 1, 32, 128, 4, 2), + (4096, 4096, 65536, 64, 64): (1, 1, 64, 64, 4, 4), + (4096, 4096, 65536, 128, 128): (2, 1, 128, 64, 1, 4), + (4096, 4096, 131072, 16, 16): (2, 1, 16, 128, 1, 2), + (4096, 4096, 131072, 32, 32): (1, 1, 32, 128, 3, 2), + (4096, 4096, 131072, 64, 64): (1, 1, 64, 64, 3, 4), + (4096, 4096, 131072, 128, 128): (2, 1, 128, 64, 1, 4), + (8192, 8192, 256, 16, 16): (1, 2, 16, 64, 1, 2), + (8192, 8192, 256, 32, 32): (1, 1, 32, 64, 1, 2), + (8192, 8192, 256, 64, 64): (1, 2, 64, 64, 1, 4), + (8192, 8192, 256, 128, 128): (3, 16, 128, 16, 1, 2), + (8192, 8192, 512, 16, 16): (1, 2, 16, 128, 1, 2), + (8192, 8192, 512, 32, 32): (1, 4, 32, 64, 3, 2), + (8192, 8192, 512, 64, 64): (2, 8, 64, 64, 4, 4), + (8192, 8192, 512, 128, 128): (1, 8, 128, 64, 1, 4), + (8192, 8192, 1024, 16, 16): (4, 2, 16, 128, 1, 2), + (8192, 8192, 1024, 32, 32): (1, 8, 32, 128, 1, 2), + (8192, 8192, 1024, 64, 64): (1, 16, 64, 64, 3, 2), + (8192, 8192, 1024, 128, 128): (2, 16, 128, 64, 2, 4), + (8192, 8192, 2048, 16, 16): (2, 1, 16, 64, 4, 1), + (8192, 8192, 2048, 32, 32): (1, 16, 32, 64, 5, 2), + (8192, 8192, 2048, 64, 64): (1, 16, 64, 64, 3, 2), + (8192, 8192, 2048, 128, 128): (2, 16, 128, 64, 2, 4), + (8192, 8192, 4096, 16, 16): (1, 1, 16, 64, 4, 1), + (8192, 8192, 4096, 32, 32): (1, 16, 32, 64, 5, 2), + (8192, 8192, 4096, 64, 64): (1, 16, 64, 64, 3, 2), + (8192, 8192, 4096, 128, 128): (2, 64, 128, 64, 2, 4), + (8192, 8192, 8192, 16, 16): (1, 1, 16, 64, 4, 1), + (8192, 8192, 8192, 32, 32): (1, 8, 32, 128, 5, 4), + (8192, 8192, 8192, 64, 64): (1, 8, 64, 64, 3, 2), + (8192, 8192, 8192, 128, 128): (2, 8, 128, 64, 1, 4), + (8192, 8192, 16384, 16, 16): (1, 1, 16, 64, 4, 1), + (8192, 8192, 16384, 32, 32): (1, 8, 32, 64, 5, 2), + (8192, 8192, 16384, 64, 64): (1, 8, 64, 64, 3, 2), + (8192, 8192, 16384, 128, 128): (1, 8, 128, 64, 1, 4), + (8192, 8192, 32768, 16, 16): (1, 1, 16, 64, 4, 1), + (8192, 8192, 32768, 32, 32): (1, 8, 32, 64, 5, 2), + (8192, 8192, 32768, 64, 64): (3, 8, 64, 64, 3, 2), + (8192, 8192, 32768, 128, 128): (2, 8, 128, 64, 1, 4), + (8192, 8192, 65536, 16, 16): (1, 1, 16, 64, 4, 1), + (8192, 8192, 65536, 32, 32): (5, 4, 32, 64, 3, 2), + (8192, 8192, 65536, 64, 64): (1, 8, 64, 64, 3, 2), + (8192, 8192, 65536, 128, 128): (2, 8, 128, 64, 1, 4), + (8192, 8192, 131072, 16, 16): (2, 1, 16, 64, 4, 1), + (8192, 8192, 131072, 32, 32): (1, 4, 32, 64, 5, 2), + (8192, 8192, 131072, 64, 64): (1, 4, 64, 128, 3, 4), + (8192, 8192, 131072, 128, 128): (2, 8, 128, 64, 1, 4), + (16384, 16384, 256, 16, 16): (1, 2, 16, 128, 1, 2), + (16384, 16384, 256, 32, 32): (1, 4, 32, 64, 3, 2), + (16384, 16384, 256, 64, 64): (2, 4, 64, 64, 4, 4), + (16384, 16384, 256, 128, 128): (1, 4, 128, 64, 1, 16), + (16384, 16384, 512, 16, 16): (1, 2, 16, 128, 3, 2), + (16384, 16384, 512, 32, 32): (1, 4, 32, 128, 5, 4), + (16384, 16384, 512, 64, 64): (1, 8, 64, 64, 3, 2), + (16384, 16384, 512, 128, 128): (2, 8, 128, 64, 1, 4), + (16384, 16384, 1024, 16, 16): (1, 2, 16, 128, 1, 2), + (16384, 16384, 1024, 32, 32): (1, 8, 32, 64, 5, 2), + (16384, 16384, 1024, 64, 64): (1, 16, 64, 64, 3, 2), + (16384, 16384, 1024, 128, 128): (5, 16, 128, 64, 2, 4), + (16384, 16384, 2048, 16, 16): (1, 2, 16, 128, 1, 2), + (16384, 16384, 2048, 32, 32): (1, 8, 32, 64, 5, 2), + (16384, 16384, 2048, 64, 64): (1, 16, 64, 64, 3, 2), + (16384, 16384, 2048, 128, 128): (4, 32, 128, 64, 2, 4), + (16384, 16384, 4096, 16, 16): (3, 2, 16, 128, 1, 2), + (16384, 16384, 4096, 32, 32): (1, 4, 32, 64, 5, 2), + (16384, 16384, 4096, 64, 64): (2, 16, 64, 64, 3, 2), + (16384, 16384, 4096, 128, 128): (3, 32, 128, 64, 2, 4), + (16384, 16384, 8192, 16, 16): (1, 2, 16, 128, 1, 2), + (16384, 16384, 8192, 32, 32): (1, 4, 32, 64, 5, 2), + (16384, 16384, 8192, 64, 64): (4, 8, 64, 64, 3, 2), + (16384, 16384, 8192, 128, 128): (5, 8, 128, 64, 1, 4), + (16384, 16384, 16384, 16, 16): (1, 2, 16, 128, 1, 2), + (16384, 16384, 16384, 32, 32): (1, 4, 32, 64, 5, 2), + (16384, 16384, 16384, 64, 64): (2, 4, 64, 128, 3, 4), + (16384, 16384, 16384, 128, 128): (4, 8, 128, 64, 1, 4), + (16384, 16384, 32768, 16, 16): (4, 2, 16, 128, 1, 2), + (16384, 16384, 32768, 32, 32): (1, 4, 32, 64, 5, 2), + (16384, 16384, 32768, 64, 64): (1, 8, 64, 64, 3, 2), + (16384, 16384, 32768, 128, 128): (2, 512, 128, 64, 2, 4), + (16384, 16384, 65536, 16, 16): (3, 2, 16, 128, 1, 2), + (16384, 16384, 65536, 32, 32): (1, 4, 32, 64, 5, 2), + (16384, 16384, 65536, 64, 64): (1, 4, 64, 128, 3, 4), + (16384, 16384, 65536, 128, 128): (2, 1024, 128, 64, 2, 4), + (16384, 16384, 131072, 16, 16): (1, 2, 16, 128, 1, 2), + (16384, 16384, 131072, 32, 32): (1, 4, 32, 64, 5, 2), + (16384, 16384, 131072, 64, 64): (3, 4, 64, 128, 3, 4), + (16384, 16384, 131072, 128, 128): (4, 2048, 128, 64, 2, 4), + }, + ("scatter_mm", "NVIDIA A100-SXM4-80GB", (0, torch.float16, 0.5)): { + (256, 256, 256, 16, 16): (5, 4, 16, 16, 1, 4), + (256, 256, 256, 32, 32): (5, 2, 32, 16, 1, 4), + (256, 256, 256, 64, 64): (4, 1, 32, 32, 1, 8), + (256, 256, 256, 128, 128): (2, 1, 32, 32, 1, 4), + (256, 256, 512, 16, 16): (2, 2, 16, 32, 1, 4), + (256, 256, 512, 32, 32): (4, 8, 32, 32, 1, 8), + (256, 256, 512, 64, 64): (4, 8, 32, 64, 1, 4), + (256, 256, 512, 128, 128): (4, 8, 32, 64, 1, 4), + (256, 256, 1024, 16, 16): (4, 2, 16, 64, 1, 2), + (256, 256, 1024, 32, 32): (4, 16, 32, 64, 1, 2), + (256, 256, 1024, 64, 64): (4, 16, 32, 64, 1, 4), + (256, 256, 1024, 128, 128): (4, 16, 64, 64, 1, 8), + (256, 256, 2048, 16, 16): (2, 16, 16, 64, 1, 8), + (256, 256, 2048, 32, 32): (4, 16, 32, 64, 1, 2), + (256, 256, 2048, 64, 64): (4, 16, 32, 64, 1, 4), + (256, 256, 2048, 128, 128): (4, 16, 64, 64, 1, 4), + (256, 256, 4096, 16, 16): (4, 32, 16, 64, 1, 1), + (256, 256, 4096, 32, 32): (2, 64, 32, 64, 1, 2), + (256, 256, 4096, 64, 64): (4, 64, 64, 64, 1, 4), + (256, 256, 4096, 128, 128): (4, 32, 64, 64, 1, 4), + (256, 256, 8192, 16, 16): (4, 64, 16, 64, 1, 1), + (256, 256, 8192, 32, 32): (4, 128, 32, 64, 1, 2), + (256, 256, 8192, 64, 64): (4, 64, 64, 64, 1, 4), + (256, 256, 8192, 128, 128): (4, 64, 64, 64, 1, 4), + (256, 256, 16384, 16, 16): (4, 128, 16, 64, 1, 1), + (256, 256, 16384, 32, 32): (2, 128, 32, 64, 1, 2), + (256, 256, 16384, 64, 64): (4, 32, 32, 128, 1, 4), + (256, 256, 16384, 128, 128): (4, 16, 64, 64, 1, 4), + (256, 256, 32768, 16, 16): (4, 64, 16, 64, 1, 1), + (256, 256, 32768, 32, 32): (2, 256, 32, 64, 1, 2), + (256, 256, 32768, 64, 64): (4, 32, 32, 128, 1, 4), + (256, 256, 32768, 128, 128): (4, 32, 64, 64, 1, 4), + (256, 256, 65536, 16, 16): (4, 128, 16, 64, 1, 1), + (256, 256, 65536, 32, 32): (4, 1, 32, 64, 1, 2), + (256, 256, 65536, 64, 64): (2, 1, 64, 64, 1, 2), + (256, 256, 65536, 128, 128): (4, 32, 64, 64, 1, 4), + (256, 256, 131072, 16, 16): (4, 64, 16, 64, 1, 1), + (256, 256, 131072, 32, 32): (2, 1, 32, 64, 1, 2), + (256, 256, 131072, 64, 64): (4, 32, 32, 128, 1, 4), + (256, 256, 131072, 128, 128): (4, 32, 64, 64, 1, 4), + (512, 512, 256, 16, 16): (4, 16, 16, 16, 1, 4), + (512, 512, 256, 32, 32): (2, 4, 32, 16, 1, 4), + (512, 512, 256, 64, 64): (2, 16, 64, 16, 3, 8), + (512, 512, 256, 128, 128): (4, 16, 64, 16, 1, 4), + (512, 512, 512, 16, 16): (1, 1, 16, 64, 1, 8), + (512, 512, 512, 32, 32): (2, 4, 16, 32, 1, 1), + (512, 512, 512, 64, 64): (2, 1, 32, 32, 1, 2), + (512, 512, 512, 128, 128): (4, 8, 32, 64, 1, 4), + (512, 512, 1024, 16, 16): (2, 8, 16, 64, 1, 8), + (512, 512, 1024, 32, 32): (4, 16, 32, 64, 1, 2), + (512, 512, 1024, 64, 64): (4, 16, 64, 64, 1, 4), + (512, 512, 1024, 128, 128): (2, 8, 64, 64, 1, 4), + (512, 512, 2048, 16, 16): (4, 16, 16, 64, 1, 4), + (512, 512, 2048, 32, 32): (4, 16, 32, 64, 1, 2), + (512, 512, 2048, 64, 64): (4, 16, 64, 64, 1, 8), + (512, 512, 2048, 128, 128): (4, 16, 64, 64, 1, 4), + (512, 512, 4096, 16, 16): (4, 32, 16, 128, 1, 2), + (512, 512, 4096, 32, 32): (4, 32, 32, 64, 1, 2), + (512, 512, 4096, 64, 64): (4, 32, 64, 64, 1, 4), + (512, 512, 4096, 128, 128): (4, 32, 64, 64, 1, 4), + (512, 512, 8192, 16, 16): (2, 32, 16, 128, 1, 2), + (512, 512, 8192, 32, 32): (4, 64, 32, 64, 1, 2), + (512, 512, 8192, 64, 64): (4, 128, 64, 64, 1, 2), + (512, 512, 8192, 128, 128): (4, 64, 64, 64, 1, 4), + (512, 512, 16384, 16, 16): (4, 32, 16, 64, 1, 1), + (512, 512, 16384, 32, 32): (4, 64, 32, 64, 1, 2), + (512, 512, 16384, 64, 64): (4, 16, 64, 64, 1, 4), + (512, 512, 16384, 128, 128): (4, 32, 64, 64, 1, 4), + (512, 512, 32768, 16, 16): (7, 16, 16, 128, 1, 2), + (512, 512, 32768, 32, 32): (4, 64, 32, 64, 1, 2), + (512, 512, 32768, 64, 64): (2, 32, 64, 64, 3, 2), + (512, 512, 32768, 128, 128): (2, 32, 64, 64, 1, 4), + (512, 512, 65536, 16, 16): (2, 32, 16, 64, 1, 1), + (512, 512, 65536, 32, 32): (4, 64, 32, 64, 1, 2), + (512, 512, 65536, 64, 64): (3, 32, 64, 64, 3, 2), + (512, 512, 65536, 128, 128): (4, 16, 64, 64, 1, 4), + (512, 512, 131072, 16, 16): (3, 32, 16, 128, 1, 2), + (512, 512, 131072, 32, 32): (4, 64, 32, 64, 1, 2), + (512, 512, 131072, 64, 64): (2, 32, 64, 64, 3, 2), + (512, 512, 131072, 128, 128): (3, 1, 64, 64, 1, 4), + (1024, 1024, 256, 16, 16): (4, 16, 16, 16, 1, 4), + (1024, 1024, 256, 32, 32): (4, 16, 32, 16, 1, 4), + (1024, 1024, 256, 64, 64): (4, 4, 64, 32, 1, 16), + (1024, 1024, 256, 128, 128): (4, 16, 64, 16, 1, 8), + (1024, 1024, 512, 16, 16): (2, 8, 16, 64, 1, 8), + (1024, 1024, 512, 32, 32): (3, 2, 32, 64, 1, 2), + (1024, 1024, 512, 64, 64): (4, 8, 32, 64, 1, 8), + (1024, 1024, 512, 128, 128): (4, 8, 64, 64, 1, 8), + (1024, 1024, 1024, 16, 16): (2, 2, 16, 64, 1, 2), + (1024, 1024, 1024, 32, 32): (2, 8, 32, 64, 1, 2), + (1024, 1024, 1024, 64, 64): (2, 8, 32, 128, 1, 4), + (1024, 1024, 1024, 128, 128): (2, 8, 64, 64, 1, 4), + (1024, 1024, 2048, 16, 16): (2, 16, 16, 128, 3, 2), + (1024, 1024, 2048, 32, 32): (4, 32, 32, 64, 1, 2), + (1024, 1024, 2048, 64, 64): (4, 16, 64, 64, 1, 4), + (1024, 1024, 2048, 128, 128): (4, 32, 64, 64, 1, 4), + (1024, 1024, 4096, 16, 16): (4, 16, 16, 128, 1, 2), + (1024, 1024, 4096, 32, 32): (3, 32, 32, 64, 1, 2), + (1024, 1024, 4096, 64, 64): (4, 32, 64, 64, 1, 4), + (1024, 1024, 4096, 128, 128): (4, 32, 64, 64, 1, 4), + (1024, 1024, 8192, 16, 16): (5, 16, 16, 128, 1, 2), + (1024, 1024, 8192, 32, 32): (2, 32, 32, 64, 3, 2), + (1024, 1024, 8192, 64, 64): (1, 16, 64, 64, 3, 2), + (1024, 1024, 8192, 128, 128): (4, 32, 64, 64, 1, 4), + (1024, 1024, 16384, 16, 16): (4, 16, 16, 128, 1, 2), + (1024, 1024, 16384, 32, 32): (1, 32, 32, 64, 3, 2), + (1024, 1024, 16384, 64, 64): (4, 16, 64, 64, 3, 2), + (1024, 1024, 16384, 128, 128): (4, 32, 128, 64, 1, 4), + (1024, 1024, 32768, 16, 16): (3, 16, 16, 128, 1, 2), + (1024, 1024, 32768, 32, 32): (1, 8, 32, 64, 3, 2), + (1024, 1024, 32768, 64, 64): (4, 16, 64, 64, 3, 2), + (1024, 1024, 32768, 128, 128): (4, 8, 128, 64, 2, 4), + (1024, 1024, 65536, 16, 16): (1, 2, 16, 128, 1, 2), + (1024, 1024, 65536, 32, 32): (2, 4, 32, 64, 3, 2), + (1024, 1024, 65536, 64, 64): (5, 16, 64, 64, 3, 2), + (1024, 1024, 65536, 128, 128): (5, 8, 128, 64, 2, 4), + (1024, 1024, 131072, 16, 16): (5, 2, 16, 128, 1, 2), + (1024, 1024, 131072, 32, 32): (1, 2, 32, 64, 3, 2), + (1024, 1024, 131072, 64, 64): (5, 16, 64, 64, 3, 2), + (1024, 1024, 131072, 128, 128): (2, 1, 128, 64, 2, 4), + (2048, 2048, 256, 16, 16): (4, 4, 16, 64, 1, 8), + (2048, 2048, 256, 32, 32): (4, 8, 32, 32, 1, 8), + (2048, 2048, 256, 64, 64): (4, 16, 64, 16, 1, 8), + (2048, 2048, 256, 128, 128): (4, 4, 128, 32, 3, 8), + (2048, 2048, 512, 16, 16): (2, 2, 16, 64, 1, 2), + (2048, 2048, 512, 32, 32): (2, 4, 32, 64, 3, 2), + (2048, 2048, 512, 64, 64): (4, 4, 64, 64, 1, 8), + (2048, 2048, 512, 128, 128): (4, 8, 64, 64, 1, 4), + (2048, 2048, 1024, 16, 16): (1, 8, 16, 64, 1, 2), + (2048, 2048, 1024, 32, 32): (2, 16, 32, 64, 3, 2), + (2048, 2048, 1024, 64, 64): (4, 8, 64, 64, 1, 4), + (2048, 2048, 1024, 128, 128): (4, 8, 128, 64, 1, 4), + (2048, 2048, 2048, 16, 16): (5, 4, 16, 128, 1, 2), + (2048, 2048, 2048, 32, 32): (1, 16, 32, 64, 3, 2), + (2048, 2048, 2048, 64, 64): (2, 8, 64, 64, 1, 4), + (2048, 2048, 2048, 128, 128): (2, 8, 128, 64, 1, 4), + (2048, 2048, 4096, 16, 16): (4, 2, 16, 128, 1, 2), + (2048, 2048, 4096, 32, 32): (2, 16, 32, 64, 3, 2), + (2048, 2048, 4096, 64, 64): (2, 8, 64, 64, 3, 2), + (2048, 2048, 4096, 128, 128): (4, 8, 128, 64, 1, 4), + (2048, 2048, 8192, 16, 16): (5, 4, 16, 128, 1, 2), + (2048, 2048, 8192, 32, 32): (2, 8, 32, 64, 3, 2), + (2048, 2048, 8192, 64, 64): (4, 8, 64, 64, 3, 2), + (2048, 2048, 8192, 128, 128): (4, 8, 128, 64, 1, 4), + (2048, 2048, 16384, 16, 16): (3, 2, 16, 128, 1, 2), + (2048, 2048, 16384, 32, 32): (2, 4, 32, 128, 3, 2), + (2048, 2048, 16384, 64, 64): (4, 8, 64, 64, 3, 2), + (2048, 2048, 16384, 128, 128): (4, 4, 128, 64, 1, 4), + (2048, 2048, 32768, 16, 16): (3, 2, 16, 128, 1, 2), + (2048, 2048, 32768, 32, 32): (3, 4, 32, 128, 3, 2), + (2048, 2048, 32768, 64, 64): (6, 4, 64, 64, 3, 2), + (2048, 2048, 32768, 128, 128): (3, 4, 128, 64, 1, 4), + (2048, 2048, 65536, 16, 16): (6, 2, 16, 128, 1, 2), + (2048, 2048, 65536, 32, 32): (1, 2, 32, 128, 1, 2), + (2048, 2048, 65536, 64, 64): (5, 4, 64, 64, 3, 2), + (2048, 2048, 65536, 128, 128): (5, 1, 128, 64, 2, 4), + (2048, 2048, 131072, 16, 16): (3, 2, 16, 128, 1, 2), + (2048, 2048, 131072, 32, 32): (2, 1, 32, 128, 3, 2), + (2048, 2048, 131072, 64, 64): (4, 1, 64, 64, 3, 2), + (2048, 2048, 131072, 128, 128): (3, 1, 128, 64, 2, 4), + (4096, 4096, 256, 16, 16): (5, 8, 16, 32, 1, 4), + (4096, 4096, 256, 32, 32): (4, 16, 32, 16, 2, 4), + (4096, 4096, 256, 64, 64): (2, 1, 64, 64, 3, 4), + (4096, 4096, 256, 128, 128): (4, 4, 128, 32, 1, 4), + (4096, 4096, 512, 16, 16): (4, 2, 16, 128, 1, 2), + (4096, 4096, 512, 32, 32): (4, 8, 32, 64, 1, 2), + (4096, 4096, 512, 64, 64): (4, 4, 64, 64, 1, 4), + (4096, 4096, 512, 128, 128): (4, 8, 128, 64, 2, 4), + (4096, 4096, 1024, 16, 16): (1, 2, 16, 128, 1, 2), + (4096, 4096, 1024, 32, 32): (6, 8, 32, 64, 3, 2), + (4096, 4096, 1024, 64, 64): (2, 16, 64, 64, 4, 4), + (4096, 4096, 1024, 128, 128): (2, 4, 128, 64, 2, 4), + (4096, 4096, 2048, 16, 16): (3, 1, 16, 128, 1, 2), + (4096, 4096, 2048, 32, 32): (1, 4, 32, 64, 5, 2), + (4096, 4096, 2048, 64, 64): (3, 16, 64, 64, 3, 2), + (4096, 4096, 2048, 128, 128): (4, 32, 128, 64, 2, 4), + (4096, 4096, 4096, 16, 16): (1, 2, 16, 128, 1, 2), + (4096, 4096, 4096, 32, 32): (1, 4, 32, 64, 3, 2), + (4096, 4096, 4096, 64, 64): (1, 1, 64, 64, 4, 4), + (4096, 4096, 4096, 128, 128): (2, 1, 128, 128, 1, 8), + (4096, 4096, 8192, 16, 16): (3, 1, 16, 128, 1, 2), + (4096, 4096, 8192, 32, 32): (2, 2, 32, 64, 5, 2), + (4096, 4096, 8192, 64, 64): (4, 16, 64, 64, 3, 2), + (4096, 4096, 8192, 128, 128): (4, 16, 128, 64, 2, 4), + (4096, 4096, 16384, 16, 16): (1, 2, 16, 128, 1, 2), + (4096, 4096, 16384, 32, 32): (4, 2, 32, 64, 5, 2), + (4096, 4096, 16384, 64, 64): (4, 16, 64, 64, 3, 2), + (4096, 4096, 16384, 128, 128): (4, 16, 128, 64, 2, 4), + (4096, 4096, 32768, 16, 16): (3, 1, 16, 128, 1, 2), + (4096, 4096, 32768, 32, 32): (3, 1, 32, 128, 1, 4), + (4096, 4096, 32768, 64, 64): (3, 1, 64, 64, 3, 4), + (4096, 4096, 32768, 128, 128): (5, 16, 128, 64, 2, 4), + (4096, 4096, 65536, 16, 16): (5, 1, 16, 128, 1, 2), + (4096, 4096, 65536, 32, 32): (5, 1, 32, 128, 1, 4), + (4096, 4096, 65536, 64, 64): (1, 1, 64, 64, 3, 4), + (4096, 4096, 65536, 128, 128): (3, 16, 128, 64, 2, 4), + (4096, 4096, 131072, 16, 16): (3, 1, 16, 128, 1, 2), + (4096, 4096, 131072, 32, 32): (3, 1, 32, 128, 3, 2), + (4096, 4096, 131072, 64, 64): (2, 1, 64, 64, 3, 4), + (4096, 4096, 131072, 128, 128): (1, 1, 128, 64, 1, 4), + (8192, 8192, 256, 16, 16): (4, 16, 16, 16, 1, 4), + (8192, 8192, 256, 32, 32): (1, 16, 32, 16, 4, 4), + (8192, 8192, 256, 64, 64): (4, 16, 64, 16, 3, 8), + (8192, 8192, 256, 128, 128): (4, 16, 128, 16, 1, 2), + (8192, 8192, 512, 16, 16): (2, 8, 16, 64, 1, 4), + (8192, 8192, 512, 32, 32): (4, 8, 32, 64, 3, 2), + (8192, 8192, 512, 64, 64): (2, 8, 64, 64, 4, 4), + (8192, 8192, 512, 128, 128): (4, 8, 128, 64, 2, 4), + (8192, 8192, 1024, 16, 16): (4, 16, 16, 64, 1, 8), + (8192, 8192, 1024, 32, 32): (2, 8, 32, 64, 5, 2), + (8192, 8192, 1024, 64, 64): (1, 16, 64, 64, 3, 2), + (8192, 8192, 1024, 128, 128): (5, 16, 128, 64, 2, 4), + (8192, 8192, 2048, 16, 16): (7, 2, 16, 128, 1, 2), + (8192, 8192, 2048, 32, 32): (1, 16, 32, 64, 5, 2), + (8192, 8192, 2048, 64, 64): (4, 16, 64, 64, 3, 2), + (8192, 8192, 2048, 128, 128): (6, 16, 128, 64, 2, 4), + (8192, 8192, 4096, 16, 16): (4, 2, 16, 128, 1, 2), + (8192, 8192, 4096, 32, 32): (2, 8, 32, 64, 5, 2), + (8192, 8192, 4096, 64, 64): (3, 16, 64, 64, 3, 2), + (8192, 8192, 4096, 128, 128): (3, 64, 128, 64, 2, 4), + (8192, 8192, 8192, 16, 16): (4, 2, 16, 128, 1, 2), + (8192, 8192, 8192, 32, 32): (1, 4, 32, 128, 5, 4), + (8192, 8192, 8192, 64, 64): (4, 4, 64, 64, 1, 4), + (8192, 8192, 8192, 128, 128): (2, 2, 128, 128, 3, 8), + (8192, 8192, 16384, 16, 16): (1, 2, 16, 128, 1, 2), + (8192, 8192, 16384, 32, 32): (4, 8, 32, 64, 5, 2), + (8192, 8192, 16384, 64, 64): (5, 8, 64, 64, 3, 2), + (8192, 8192, 16384, 128, 128): (3, 16, 128, 64, 2, 4), + (8192, 8192, 32768, 16, 16): (7, 2, 16, 128, 1, 2), + (8192, 8192, 32768, 32, 32): (3, 4, 32, 64, 3, 2), + (8192, 8192, 32768, 64, 64): (2, 8, 64, 64, 3, 2), + (8192, 8192, 32768, 128, 128): (6, 16, 128, 64, 2, 4), + (8192, 8192, 65536, 16, 16): (9, 2, 16, 128, 1, 2), + (8192, 8192, 65536, 32, 32): (7, 4, 32, 64, 5, 2), + (8192, 8192, 65536, 64, 64): (4, 8, 64, 64, 3, 2), + (8192, 8192, 65536, 128, 128): (3, 16, 128, 64, 2, 4), + (8192, 8192, 131072, 16, 16): (9, 2, 16, 128, 1, 2), + (8192, 8192, 131072, 32, 32): (1, 8, 32, 64, 5, 2), + (8192, 8192, 131072, 64, 64): (1, 8, 64, 64, 3, 2), + (8192, 8192, 131072, 128, 128): (4, 16, 128, 64, 2, 4), + (16384, 16384, 256, 16, 16): (5, 16, 16, 16, 1, 4), + (16384, 16384, 256, 32, 32): (4, 16, 32, 16, 4, 4), + (16384, 16384, 256, 64, 64): (4, 16, 64, 16, 3, 8), + (16384, 16384, 256, 128, 128): (4, 16, 128, 16, 1, 2), + (16384, 16384, 512, 16, 16): (2, 8, 16, 64, 1, 4), + (16384, 16384, 512, 32, 32): (1, 4, 32, 64, 5, 2), + (16384, 16384, 512, 64, 64): (4, 8, 64, 64, 1, 4), + (16384, 16384, 512, 128, 128): (3, 8, 128, 64, 2, 4), + (16384, 16384, 1024, 16, 16): (4, 2, 16, 128, 1, 2), + (16384, 16384, 1024, 32, 32): (4, 8, 32, 64, 5, 2), + (16384, 16384, 1024, 64, 64): (6, 16, 64, 64, 3, 2), + (16384, 16384, 1024, 128, 128): (3, 16, 128, 64, 2, 4), + (16384, 16384, 2048, 16, 16): (3, 2, 16, 128, 1, 2), + (16384, 16384, 2048, 32, 32): (1, 8, 32, 64, 5, 2), + (16384, 16384, 2048, 64, 64): (5, 16, 64, 64, 3, 2), + (16384, 16384, 2048, 128, 128): (2, 32, 128, 64, 2, 4), + (16384, 16384, 4096, 16, 16): (2, 2, 16, 128, 1, 2), + (16384, 16384, 4096, 32, 32): (1, 4, 32, 64, 3, 2), + (16384, 16384, 4096, 64, 64): (2, 8, 64, 64, 3, 2), + (16384, 16384, 4096, 128, 128): (3, 16, 128, 64, 2, 4), + (16384, 16384, 8192, 16, 16): (3, 2, 16, 128, 1, 2), + (16384, 16384, 8192, 32, 32): (2, 4, 32, 64, 5, 2), + (16384, 16384, 8192, 64, 64): (4, 8, 64, 64, 3, 2), + (16384, 16384, 8192, 128, 128): (8, 32, 128, 64, 2, 4), + (16384, 16384, 16384, 16, 16): (1, 2, 16, 256, 1, 4), + (16384, 16384, 16384, 32, 32): (1, 4, 32, 128, 3, 4), + (16384, 16384, 16384, 64, 64): (5, 4, 64, 64, 1, 4), + (16384, 16384, 16384, 128, 128): (4, 8, 128, 64, 2, 4), + (16384, 16384, 32768, 16, 16): (2, 2, 16, 128, 1, 2), + (16384, 16384, 32768, 32, 32): (1, 4, 32, 64, 3, 2), + (16384, 16384, 32768, 64, 64): (5, 4, 64, 64, 1, 4), + (16384, 16384, 32768, 128, 128): (5, 8, 128, 64, 2, 4), + (16384, 16384, 65536, 16, 16): (8, 2, 16, 128, 1, 2), + (16384, 16384, 65536, 32, 32): (6, 4, 32, 64, 5, 2), + (16384, 16384, 65536, 64, 64): (2, 4, 64, 64, 1, 4), + (16384, 16384, 65536, 128, 128): (4, 8, 128, 64, 2, 4), + (16384, 16384, 131072, 16, 16): (3, 1, 16, 128, 1, 2), + (16384, 16384, 131072, 32, 32): (1, 4, 32, 64, 3, 2), + (16384, 16384, 131072, 64, 64): (4, 4, 64, 64, 1, 4), + (16384, 16384, 131072, 128, 128): (1, 8, 128, 64, 2, 4), + (32768, 32768, 256, 16, 16): (4, 16, 16, 16, 1, 4), + (32768, 32768, 512, 16, 16): (4, 2, 16, 128, 1, 2), + (32768, 32768, 1024, 16, 16): (3, 2, 16, 128, 1, 2), + (32768, 32768, 2048, 16, 16): (4, 2, 16, 128, 1, 2), + (32768, 32768, 4096, 16, 16): (5, 4, 16, 64, 1, 1), + (32768, 32768, 8192, 16, 16): (4, 4, 16, 64, 1, 1), + (32768, 32768, 16384, 16, 16): (4, 4, 16, 64, 1, 1), + (32768, 32768, 32768, 16, 16): (5, 4, 16, 64, 1, 1), + }, + ("scatter_mm", "NVIDIA A100-SXM4-80GB", (0, torch.float32, 0.5)): { + (256, 256, 256, 16, 16): (1, 1, 16, 16, 1, 8), + (256, 256, 256, 32, 32): (1, 1, 16, 16, 1, 4), + (256, 256, 256, 64, 64): (1, 1, 16, 16, 1, 4), + (256, 256, 256, 128, 128): (1, 1, 16, 16, 1, 1), + (256, 256, 512, 16, 16): (1, 1, 16, 16, 1, 4), + (256, 256, 512, 32, 32): (1, 16, 16, 16, 1, 1), + (256, 256, 512, 64, 64): (1, 1, 16, 16, 1, 1), + (256, 256, 512, 128, 128): (1, 1, 32, 32, 1, 4), + (256, 256, 1024, 16, 16): (1, 1, 16, 32, 1, 2), + (256, 256, 1024, 32, 32): (1, 4, 16, 16, 1, 1), + (256, 256, 1024, 64, 64): (1, 1, 32, 32, 1, 4), + (256, 256, 1024, 128, 128): (1, 1, 32, 32, 1, 4), + (256, 256, 2048, 16, 16): (1, 2, 16, 32, 1, 2), + (256, 256, 2048, 32, 32): (1, 1, 16, 32, 1, 2), + (256, 256, 2048, 64, 64): (2, 1, 16, 32, 1, 2), + (256, 256, 2048, 128, 128): (1, 1, 16, 16, 1, 1), + (256, 256, 4096, 16, 16): (1, 1, 16, 32, 1, 2), + (256, 256, 4096, 32, 32): (1, 1, 16, 32, 1, 2), + (256, 256, 4096, 64, 64): (1, 1, 32, 32, 1, 4), + (256, 256, 4096, 128, 128): (3, 1, 32, 64, 1, 4), + (256, 256, 8192, 16, 16): (1, 32, 16, 64, 1, 2), + (256, 256, 8192, 32, 32): (1, 1, 32, 64, 1, 4), + (256, 256, 8192, 64, 64): (1, 1, 32, 64, 1, 4), + (256, 256, 8192, 128, 128): (2, 1, 64, 32, 1, 4), + (256, 256, 16384, 16, 16): (1, 1, 16, 64, 1, 2), + (256, 256, 16384, 32, 32): (1, 1, 32, 64, 1, 4), + (256, 256, 16384, 64, 64): (1, 128, 64, 64, 1, 4), + (256, 256, 16384, 128, 128): (2, 1, 64, 32, 1, 4), + (256, 256, 32768, 16, 16): (2, 128, 16, 64, 1, 1), + (256, 256, 32768, 32, 32): (1, 1, 32, 64, 1, 4), + (256, 256, 32768, 64, 64): (1, 128, 64, 64, 1, 4), + (256, 256, 32768, 128, 128): (2, 1, 64, 64, 1, 4), + (256, 256, 65536, 16, 16): (1, 1, 16, 64, 1, 2), + (256, 256, 65536, 32, 32): (1, 1, 32, 64, 1, 4), + (256, 256, 65536, 64, 64): (2, 1, 64, 64, 1, 4), + (256, 256, 65536, 128, 128): (1, 1, 128, 32, 1, 4), + (256, 256, 131072, 16, 16): (3, 128, 16, 64, 1, 1), + (256, 256, 131072, 32, 32): (1, 1, 32, 64, 1, 4), + (256, 256, 131072, 64, 64): (2, 1, 64, 64, 1, 4), + (256, 256, 131072, 128, 128): (1, 8192, 64, 16, 1, 4), + (512, 512, 256, 16, 16): (1, 2, 16, 16, 1, 1), + (512, 512, 256, 32, 32): (1, 4, 16, 16, 1, 1), + (512, 512, 256, 64, 64): (1, 16, 16, 16, 1, 1), + (512, 512, 256, 128, 128): (1, 1, 16, 32, 1, 4), + (512, 512, 512, 16, 16): (1, 8, 16, 32, 1, 2), + (512, 512, 512, 32, 32): (1, 8, 16, 32, 1, 2), + (512, 512, 512, 64, 64): (1, 2, 16, 32, 1, 2), + (512, 512, 512, 128, 128): (1, 1, 32, 32, 1, 4), + (512, 512, 1024, 16, 16): (1, 1, 16, 32, 1, 2), + (512, 512, 1024, 32, 32): (1, 1, 16, 32, 1, 2), + (512, 512, 1024, 64, 64): (1, 1, 16, 32, 1, 2), + (512, 512, 1024, 128, 128): (1, 1, 64, 32, 1, 4), + (512, 512, 2048, 16, 16): (1, 16, 16, 64, 1, 2), + (512, 512, 2048, 32, 32): (1, 1, 32, 32, 1, 4), + (512, 512, 2048, 64, 64): (1, 1, 32, 32, 1, 4), + (512, 512, 2048, 128, 128): (2, 1, 32, 32, 1, 4), + (512, 512, 4096, 16, 16): (2, 64, 16, 64, 1, 1), + (512, 512, 4096, 32, 32): (1, 64, 32, 64, 1, 4), + (512, 512, 4096, 64, 64): (1, 1, 32, 32, 1, 4), + (512, 512, 4096, 128, 128): (1, 1, 64, 32, 1, 4), + (512, 512, 8192, 16, 16): (2, 64, 16, 64, 1, 1), + (512, 512, 8192, 32, 32): (1, 256, 32, 32, 1, 1), + (512, 512, 8192, 64, 64): (1, 64, 64, 64, 1, 4), + (512, 512, 8192, 128, 128): (2, 1, 64, 32, 1, 8), + (512, 512, 16384, 16, 16): (2, 64, 16, 64, 1, 1), + (512, 512, 16384, 32, 32): (1, 128, 32, 32, 1, 1), + (512, 512, 16384, 64, 64): (1, 64, 64, 64, 1, 4), + (512, 512, 16384, 128, 128): (3, 1, 64, 32, 1, 8), + (512, 512, 32768, 16, 16): (2, 64, 16, 64, 1, 1), + (512, 512, 32768, 32, 32): (1, 128, 32, 32, 1, 1), + (512, 512, 32768, 64, 64): (1, 64, 64, 64, 1, 4), + (512, 512, 32768, 128, 128): (2, 1, 64, 32, 1, 8), + (512, 512, 65536, 16, 16): (2, 32, 16, 64, 1, 1), + (512, 512, 65536, 32, 32): (1, 128, 32, 32, 1, 1), + (512, 512, 65536, 64, 64): (1, 64, 64, 64, 1, 4), + (512, 512, 65536, 128, 128): (2, 1, 64, 32, 1, 8), + (512, 512, 131072, 16, 16): (2, 32, 16, 64, 1, 1), + (512, 512, 131072, 32, 32): (1, 128, 32, 32, 1, 1), + (512, 512, 131072, 64, 64): (3, 64, 64, 64, 1, 4), + (512, 512, 131072, 128, 128): (1, 8192, 64, 16, 1, 4), + (1024, 1024, 256, 16, 16): (1, 4, 16, 32, 1, 2), + (1024, 1024, 256, 32, 32): (1, 4, 16, 32, 1, 2), + (1024, 1024, 256, 64, 64): (1, 1, 16, 32, 1, 2), + (1024, 1024, 256, 128, 128): (1, 1, 16, 16, 1, 1), + (1024, 1024, 512, 16, 16): (1, 8, 16, 32, 1, 2), + (1024, 1024, 512, 32, 32): (1, 8, 16, 32, 1, 1), + (1024, 1024, 512, 64, 64): (1, 8, 32, 32, 1, 4), + (1024, 1024, 512, 128, 128): (2, 1, 32, 32, 1, 4), + (1024, 1024, 1024, 16, 16): (1, 16, 16, 32, 1, 2), + (1024, 1024, 1024, 32, 32): (1, 16, 32, 64, 1, 4), + (1024, 1024, 1024, 64, 64): (1, 16, 32, 64, 1, 4), + (1024, 1024, 1024, 128, 128): (1, 1, 32, 32, 1, 4), + (1024, 1024, 2048, 16, 16): (2, 32, 16, 64, 1, 1), + (1024, 1024, 2048, 32, 32): (1, 32, 32, 64, 1, 4), + (1024, 1024, 2048, 64, 64): (1, 32, 64, 64, 1, 4), + (1024, 1024, 2048, 128, 128): (1, 1, 32, 64, 1, 4), + (1024, 1024, 4096, 16, 16): (2, 16, 16, 64, 1, 1), + (1024, 1024, 4096, 32, 32): (1, 64, 32, 32, 1, 1), + (1024, 1024, 4096, 64, 64): (1, 64, 64, 64, 1, 4), + (1024, 1024, 4096, 128, 128): (2, 64, 64, 32, 1, 8), + (1024, 1024, 8192, 16, 16): (2, 16, 16, 64, 1, 1), + (1024, 1024, 8192, 32, 32): (1, 64, 32, 32, 1, 1), + (1024, 1024, 8192, 64, 64): (1, 64, 64, 64, 1, 4), + (1024, 1024, 8192, 128, 128): (4, 1, 32, 64, 1, 4), + (1024, 1024, 16384, 16, 16): (2, 16, 16, 64, 1, 1), + (1024, 1024, 16384, 32, 32): (1, 64, 32, 32, 1, 1), + (1024, 1024, 16384, 64, 64): (1, 32, 64, 64, 1, 4), + (1024, 1024, 16384, 128, 128): (2, 64, 64, 32, 1, 4), + (1024, 1024, 32768, 16, 16): (2, 16, 16, 64, 1, 1), + (1024, 1024, 32768, 32, 32): (1, 64, 32, 32, 1, 1), + (1024, 1024, 32768, 64, 64): (1, 32, 64, 64, 1, 4), + (1024, 1024, 32768, 128, 128): (4, 1, 32, 64, 1, 4), + (1024, 1024, 65536, 16, 16): (2, 16, 16, 64, 1, 1), + (1024, 1024, 65536, 32, 32): (1, 32, 32, 32, 1, 1), + (1024, 1024, 65536, 64, 64): (2, 32, 64, 64, 1, 4), + (1024, 1024, 65536, 128, 128): (4, 1, 64, 32, 1, 4), + (1024, 1024, 131072, 16, 16): (2, 16, 16, 64, 1, 1), + (1024, 1024, 131072, 32, 32): (1, 32, 32, 32, 1, 1), + (1024, 1024, 131072, 64, 64): (1, 16, 64, 64, 1, 4), + (1024, 1024, 131072, 128, 128): (1, 8192, 64, 16, 1, 4), + (2048, 2048, 256, 16, 16): (1, 4, 16, 32, 1, 2), + (2048, 2048, 256, 32, 32): (1, 8, 16, 32, 1, 1), + (2048, 2048, 256, 64, 64): (1, 8, 32, 32, 1, 4), + (2048, 2048, 256, 128, 128): (1, 4, 64, 64, 1, 8), + (2048, 2048, 512, 16, 16): (2, 8, 16, 32, 1, 2), + (2048, 2048, 512, 32, 32): (2, 8, 32, 64, 1, 4), + (2048, 2048, 512, 64, 64): (2, 4, 64, 64, 1, 4), + (2048, 2048, 512, 128, 128): (1, 8, 32, 64, 1, 4), + (2048, 2048, 1024, 16, 16): (2, 16, 16, 64, 3, 1), + (2048, 2048, 1024, 32, 32): (1, 32, 32, 32, 1, 1), + (2048, 2048, 1024, 64, 64): (1, 16, 64, 64, 1, 4), + (2048, 2048, 1024, 128, 128): (2, 4, 64, 64, 1, 8), + (2048, 2048, 2048, 16, 16): (2, 16, 16, 64, 1, 1), + (2048, 2048, 2048, 32, 32): (1, 32, 32, 32, 1, 1), + (2048, 2048, 2048, 64, 64): (1, 16, 64, 64, 1, 4), + (2048, 2048, 2048, 128, 128): (2, 32, 32, 64, 1, 4), + (2048, 2048, 4096, 16, 16): (3, 2, 16, 64, 1, 1), + (2048, 2048, 4096, 32, 32): (3, 4, 32, 32, 1, 1), + (2048, 2048, 4096, 64, 64): (1, 16, 64, 64, 1, 4), + (2048, 2048, 4096, 128, 128): (2, 32, 64, 32, 1, 4), + (2048, 2048, 8192, 16, 16): (3, 4, 16, 64, 1, 1), + (2048, 2048, 8192, 32, 32): (2, 4, 32, 32, 1, 1), + (2048, 2048, 8192, 64, 64): (2, 32, 64, 32, 1, 2), + (2048, 2048, 8192, 128, 128): (4, 1, 32, 64, 1, 4), + (2048, 2048, 16384, 16, 16): (3, 4, 16, 64, 1, 1), + (2048, 2048, 16384, 32, 32): (1, 4, 32, 32, 1, 1), + (2048, 2048, 16384, 64, 64): (2, 8, 64, 32, 1, 2), + (2048, 2048, 16384, 128, 128): (2, 8, 64, 32, 1, 4), + (2048, 2048, 32768, 16, 16): (2, 4, 16, 64, 1, 1), + (2048, 2048, 32768, 32, 32): (2, 8, 32, 32, 1, 1), + (2048, 2048, 32768, 64, 64): (1, 16, 64, 32, 1, 2), + (2048, 2048, 32768, 128, 128): (4, 1, 32, 64, 1, 4), + (2048, 2048, 65536, 16, 16): (3, 4, 16, 64, 1, 1), + (2048, 2048, 65536, 32, 32): (1, 8, 32, 32, 1, 1), + (2048, 2048, 65536, 64, 64): (1, 8, 64, 32, 1, 2), + (2048, 2048, 65536, 128, 128): (4, 1, 64, 32, 1, 4), + (2048, 2048, 131072, 16, 16): (2, 4, 16, 64, 1, 1), + (2048, 2048, 131072, 32, 32): (1, 8, 32, 32, 1, 1), + (2048, 2048, 131072, 64, 64): (3, 1, 64, 32, 1, 2), + (2048, 2048, 131072, 128, 128): (1, 8192, 128, 16, 1, 8), + (4096, 4096, 256, 16, 16): (2, 4, 16, 32, 1, 2), + (4096, 4096, 256, 32, 32): (1, 4, 32, 64, 1, 4), + (4096, 4096, 256, 64, 64): (1, 4, 64, 64, 1, 4), + (4096, 4096, 256, 128, 128): (1, 4, 32, 64, 1, 4), + (4096, 4096, 512, 16, 16): (2, 8, 16, 64, 3, 1), + (4096, 4096, 512, 32, 32): (2, 16, 32, 32, 1, 1), + (4096, 4096, 512, 64, 64): (1, 8, 64, 64, 1, 4), + (4096, 4096, 512, 128, 128): (1, 8, 32, 64, 1, 4), + (4096, 4096, 1024, 16, 16): (1, 8, 16, 64, 3, 1), + (4096, 4096, 1024, 32, 32): (1, 16, 32, 32, 1, 1), + (4096, 4096, 1024, 64, 64): (1, 16, 64, 32, 1, 2), + (4096, 4096, 1024, 128, 128): (1, 16, 32, 64, 1, 4), + (4096, 4096, 2048, 16, 16): (1, 16, 16, 64, 3, 1), + (4096, 4096, 2048, 32, 32): (1, 16, 32, 32, 1, 1), + (4096, 4096, 2048, 64, 64): (3, 16, 64, 32, 1, 2), + (4096, 4096, 2048, 128, 128): (4, 8, 32, 64, 1, 4), + (4096, 4096, 4096, 16, 16): (1, 8, 16, 64, 3, 1), + (4096, 4096, 4096, 32, 32): (1, 1, 32, 32, 1, 1), + (4096, 4096, 4096, 64, 64): (2, 16, 64, 32, 1, 2), + (4096, 4096, 4096, 128, 128): (4, 8, 32, 64, 1, 4), + (4096, 4096, 8192, 16, 16): (1, 8, 16, 64, 3, 1), + (4096, 4096, 8192, 32, 32): (2, 1, 32, 32, 1, 1), + (4096, 4096, 8192, 64, 64): (1, 16, 64, 32, 1, 2), + (4096, 4096, 8192, 128, 128): (2, 1, 32, 64, 1, 4), + (4096, 4096, 16384, 16, 16): (1, 8, 16, 64, 3, 1), + (4096, 4096, 16384, 32, 32): (1, 1, 32, 32, 1, 1), + (4096, 4096, 16384, 64, 64): (2, 8, 64, 32, 1, 2), + (4096, 4096, 16384, 128, 128): (2, 1, 32, 64, 1, 4), + (4096, 4096, 32768, 16, 16): (1, 8, 16, 64, 3, 1), + (4096, 4096, 32768, 32, 32): (1, 1, 32, 32, 1, 1), + (4096, 4096, 32768, 64, 64): (1, 8, 64, 32, 1, 2), + (4096, 4096, 32768, 128, 128): (2, 1, 32, 64, 1, 4), + (4096, 4096, 65536, 16, 16): (1, 8, 16, 64, 3, 1), + (4096, 4096, 65536, 32, 32): (3, 1, 32, 32, 1, 1), + (4096, 4096, 65536, 64, 64): (3, 4, 64, 32, 1, 2), + (4096, 4096, 65536, 128, 128): (2, 1, 32, 64, 1, 4), + (4096, 4096, 131072, 16, 16): (1, 8, 16, 64, 3, 1), + (4096, 4096, 131072, 32, 32): (1, 1, 32, 32, 1, 1), + (4096, 4096, 131072, 64, 64): (2, 8, 64, 32, 1, 2), + (4096, 4096, 131072, 128, 128): (1, 8192, 128, 16, 1, 8), + (8192, 8192, 256, 16, 16): (2, 4, 16, 64, 3, 1), + (8192, 8192, 256, 32, 32): (1, 8, 32, 32, 1, 1), + (8192, 8192, 256, 64, 64): (1, 4, 64, 64, 1, 4), + (8192, 8192, 256, 128, 128): (1, 4, 32, 64, 1, 4), + (8192, 8192, 512, 16, 16): (1, 4, 16, 64, 3, 1), + (8192, 8192, 512, 32, 32): (1, 16, 32, 32, 1, 1), + (8192, 8192, 512, 64, 64): (2, 4, 64, 64, 1, 4), + (8192, 8192, 512, 128, 128): (2, 1, 32, 64, 1, 4), + (8192, 8192, 1024, 16, 16): (3, 8, 16, 64, 3, 1), + (8192, 8192, 1024, 32, 32): (1, 16, 32, 32, 1, 1), + (8192, 8192, 1024, 64, 64): (1, 8, 64, 32, 1, 2), + (8192, 8192, 1024, 128, 128): (2, 4, 32, 64, 1, 4), + (8192, 8192, 2048, 16, 16): (1, 8, 16, 64, 3, 1), + (8192, 8192, 2048, 32, 32): (1, 16, 32, 32, 1, 1), + (8192, 8192, 2048, 64, 64): (2, 8, 64, 32, 1, 2), + (8192, 8192, 2048, 128, 128): (4, 1, 32, 64, 1, 4), + (8192, 8192, 4096, 16, 16): (1, 8, 16, 64, 3, 1), + (8192, 8192, 4096, 32, 32): (1, 16, 32, 32, 1, 1), + (8192, 8192, 4096, 64, 64): (1, 4, 64, 32, 1, 2), + (8192, 8192, 4096, 128, 128): (3, 1, 32, 64, 1, 4), + (8192, 8192, 8192, 16, 16): (1, 8, 16, 64, 3, 1), + (8192, 8192, 8192, 32, 32): (1, 8, 32, 32, 1, 1), + (8192, 8192, 8192, 64, 64): (1, 8, 64, 32, 1, 2), + (8192, 8192, 8192, 128, 128): (4, 1, 32, 64, 1, 4), + (8192, 8192, 16384, 16, 16): (3, 4, 16, 64, 3, 1), + (8192, 8192, 16384, 32, 32): (1, 8, 32, 32, 1, 1), + (8192, 8192, 16384, 64, 64): (2, 2, 64, 32, 1, 2), + (8192, 8192, 16384, 128, 128): (7, 1, 32, 64, 1, 4), + (8192, 8192, 32768, 16, 16): (1, 4, 16, 64, 3, 1), + (8192, 8192, 32768, 32, 32): (1, 8, 32, 32, 1, 1), + (8192, 8192, 32768, 64, 64): (3, 2, 64, 32, 1, 2), + (8192, 8192, 32768, 128, 128): (6, 1, 32, 64, 1, 4), + (8192, 8192, 65536, 16, 16): (1, 4, 16, 64, 3, 1), + (8192, 8192, 65536, 32, 32): (4, 8, 32, 32, 1, 1), + (8192, 8192, 65536, 64, 64): (1, 2, 64, 32, 1, 2), + (8192, 8192, 65536, 128, 128): (4, 1, 32, 64, 1, 4), + (8192, 8192, 131072, 16, 16): (1, 4, 16, 64, 3, 1), + (8192, 8192, 131072, 32, 32): (1, 8, 32, 32, 1, 1), + (8192, 8192, 131072, 64, 64): (5, 4, 64, 32, 1, 2), + (8192, 8192, 131072, 128, 128): (1, 4096, 128, 16, 1, 8), + (16384, 16384, 256, 16, 16): (1, 4, 16, 64, 3, 1), + (16384, 16384, 256, 32, 32): (1, 8, 32, 32, 1, 1), + (16384, 16384, 256, 64, 64): (1, 4, 64, 32, 1, 2), + (16384, 16384, 256, 128, 128): (1, 4, 32, 64, 1, 4), + (16384, 16384, 512, 16, 16): (1, 8, 16, 64, 3, 1), + (16384, 16384, 512, 32, 32): (1, 16, 32, 32, 1, 1), + (16384, 16384, 512, 64, 64): (1, 4, 64, 32, 1, 2), + (16384, 16384, 512, 128, 128): (3, 1, 32, 64, 1, 4), + (16384, 16384, 1024, 16, 16): (1, 8, 16, 64, 3, 1), + (16384, 16384, 1024, 32, 32): (1, 16, 32, 32, 1, 1), + (16384, 16384, 1024, 64, 64): (2, 4, 64, 32, 1, 2), + (16384, 16384, 1024, 128, 128): (1, 2, 32, 64, 1, 4), + (16384, 16384, 2048, 16, 16): (1, 4, 16, 64, 3, 1), + (16384, 16384, 2048, 32, 32): (1, 16, 32, 32, 1, 1), + (16384, 16384, 2048, 64, 64): (3, 4, 64, 32, 1, 2), + (16384, 16384, 2048, 128, 128): (2, 1, 32, 64, 1, 4), + (16384, 16384, 4096, 16, 16): (4, 8, 16, 64, 3, 1), + (16384, 16384, 4096, 32, 32): (5, 16, 32, 32, 1, 1), + (16384, 16384, 4096, 64, 64): (3, 2, 64, 32, 1, 2), + (16384, 16384, 4096, 128, 128): (2, 1, 32, 64, 1, 4), + (16384, 16384, 8192, 16, 16): (1, 4, 16, 64, 3, 1), + (16384, 16384, 8192, 32, 32): (1, 4, 32, 32, 1, 1), + (16384, 16384, 8192, 64, 64): (1, 2, 64, 32, 1, 2), + (16384, 16384, 8192, 128, 128): (2, 1, 32, 64, 1, 4), + (16384, 16384, 16384, 16, 16): (1, 8, 16, 64, 3, 1), + (16384, 16384, 16384, 32, 32): (1, 4, 32, 32, 1, 1), + (16384, 16384, 16384, 64, 64): (1, 2, 64, 32, 1, 2), + (16384, 16384, 16384, 128, 128): (3, 1, 32, 64, 1, 4), + (16384, 16384, 32768, 16, 16): (1, 4, 16, 64, 3, 1), + (16384, 16384, 32768, 32, 32): (1, 2, 32, 32, 1, 1), + (16384, 16384, 32768, 64, 64): (3, 2, 64, 32, 1, 2), + (16384, 16384, 32768, 128, 128): (3, 1, 32, 64, 1, 4), + (16384, 16384, 65536, 16, 16): (1, 8, 16, 64, 3, 1), + (16384, 16384, 65536, 32, 32): (1, 4, 32, 32, 1, 1), + (16384, 16384, 65536, 64, 64): (4, 4, 64, 32, 1, 2), + (16384, 16384, 65536, 128, 128): (5, 1, 32, 64, 1, 4), + (16384, 16384, 131072, 16, 16): (1, 2, 16, 64, 3, 1), + (16384, 16384, 131072, 32, 32): (1, 4, 32, 32, 1, 1), + (16384, 16384, 131072, 64, 64): (1, 2, 64, 32, 1, 2), + (16384, 16384, 131072, 128, 128): (1, 4096, 128, 16, 1, 8), + }, + # END GENERATED DATA +} + +if __name__ == "__main__": + for dtype in [torch.int8]: + for op in ["_int_bsr_dense_addmm"]: + main(op=op, force=False, dtype=dtype) + for dtype in [torch.float16, torch.bfloat16, torch.float32, torch.int8]: + for op in ["bsr_dense_addmm"]: + main(op=op, force=False, dtype=dtype) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/semi_structured.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/semi_structured.py new file mode 100644 index 0000000000000000000000000000000000000000..a50c74083c01cbfb44a04243959b2f851862b992 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/sparse/semi_structured.py @@ -0,0 +1,687 @@ +# mypy: allow-untyped-defs +import warnings +from collections import namedtuple +from collections.abc import Callable +from typing import Any + +import torch +from torch.sparse._semi_structured_conversions import ( + sparse_semi_structured_from_dense_cutlass, + sparse_semi_structured_to_dense_cutlass, +) +from torch.sparse._semi_structured_ops import ( + fallback_dispatcher, + semi_sparse_addmm, + semi_sparse_detach, + semi_sparse_indices, + semi_sparse_linear, + semi_sparse_mm, + semi_sparse_scaled_mm, + semi_sparse_t, + semi_sparse_values, + semi_sparse_view, +) + + +__all__ = [ + "SparseSemiStructuredTensor", + "SparseSemiStructuredTensorCUTLASS", + "SparseSemiStructuredTensorCUSPARSELT", + "to_sparse_semi_structured", +] + +_SEMI_STRUCTURED_SPARSE_CONFIG = namedtuple( + "_SEMI_STRUCTURED_SPARSE_CONFIG", + "sparse_min_rows sparse_min_cols dense_min_rows dense_min_cols", +) + + +class SparseSemiStructuredTensor(torch.Tensor): + """ + This class implements semi-structured sparsity as a Tensor subclass. + + Semi-structured sparsity describes a sparsity pattern where n in every 2n elements are sparse, + depending on the datatype. It is also referred to as 2:4 sparsity or fine-grained + structured sparsity. + + There are two backends available for semi_structred sparsity, either cuSPARSELt or CUTLASS. + This class is meant to serve as a base class for both implementations. SparseSemiStructuredCUTLASS + and SparseSemiStructuredCUSPARSELT both inherit from this class and define three backend-specific items. + Note that as such, this class cannot be instantiated directly. + + -`_DTYPE_SHAPE_CONSTRAINTS` - A dictionary holding backend specific dense/sparse min shape constraints + - `def from_dense()` - backend specific compression routines + - `def _mm()` - backend specific mm op (either torch._cslt_sparse_mm or torch._sparse_semi_structured_(mm|addmm)) + """ + + _DEFAULT_ALG_ID: int = 0 + _DTYPE_SHAPE_CONSTRAINTS: dict[torch.dtype, _SEMI_STRUCTURED_SPARSE_CONFIG] + _FORCE_CUTLASS: bool = False + _FUSE_TRANSPOSE: bool = False + _PROTOTYPE_WARNING_SHOWN: bool = False + + BACKEND: str + SPARSE_DISPATCH: dict[Callable, Callable] + + packed: torch.Tensor | None + meta: torch.Tensor | None + packed_t: torch.Tensor | None + meta_t: torch.Tensor | None + compressed_swizzled_bitmask: torch.Tensor | None + fuse_transpose_cusparselt: bool + alg_id_cusparselt: int + + __slots__ = ["packed", "meta", "packed_t", "meta_t", "compressed_swizzled_bitmask"] + + @staticmethod + def __new__( # noqa: PYI034 + cls, + shape: torch.Size, + packed: torch.Tensor | None, + meta: torch.Tensor | None, + packed_t: torch.Tensor | None, + meta_t: torch.Tensor | None, + compressed_swizzled_bitmask: torch.Tensor | None, + fuse_transpose_cusparselt: bool = False, + alg_id_cusparselt: int = 0, + requires_grad: bool = False, + ): + """ + Create a new instance of the tensor subclass from the compressed sparse representation. + + We have the option to create the subclass with the compressed representations of both X and X', for training. + For inference, we only need a single representation (either X or X'), while the corresponding other set will be None. + + Depending on the backend selected, certain fields will be set to None. (CUSPARSELT vs CUTLASS) + + Args: + shape: The shape of the original dense tensor + packed: The compressed representation of the original dense tensor + meta: The metadata of the original dense tensor, if it is stored separately + packed_t: The compressed representation of the transposed original dense tensor + meta_t: The metadata of the transposed original dense tensor, if it is stored separately + compressed_swizzled_bitmask: The masks used by the CUTLASS backend to determine which threads should + participate in the computation. Used for pointwise ops. + fuse_transpose_cusparselt: When running with cuSPARSELt, we have the option to fuse a transposition + with a matmul, which is useful in the case of 2:4 sparse training. + alg_id_cusparselt: The algorithm id to use when using cuSPARSELT, will have effect on performance + + Returns: + torch.Tensor: A torch.Tensor wrapper subclass. + + Raises: + ValueError: If all of the tensor arguments are None. + """ + if not cls._PROTOTYPE_WARNING_SHOWN: + warnings.warn( + ( + "The PyTorch API of SparseSemiStructuredTensor is in prototype stage " + "and will change in the near future. Please open a Github issue " + "for features requests and see our documentation on the torch.sparse " + "module for further information about the project." + ), + UserWarning, + stacklevel=2, + ) + cls._PROTOTYPE_WARNING_SHOWN = True + + # Because this only runs once, we also load the dispatch table here as well. + # We can't define the dispatch table explicitly because of torch.ops import errors, so we do this instead + # But this is useful since it allows users to overload the dispatch table for debugging / testing. + cls._load_dispatch_table() + + # we can also register the classes with dynamo when the warning is shown. + torch._dynamo.allow_in_graph(cls) + + if packed is not None: + previous_tensor = packed + elif packed_t is not None: + previous_tensor = packed_t + else: + raise ValueError("At least one of packed or packed_t must be provided") + + tensor = torch.Tensor._make_wrapper_subclass( + cls, + shape, + device=previous_tensor.device, + dtype=previous_tensor.dtype, + layout=previous_tensor.layout, + requires_grad=requires_grad, + ) + + tensor.packed = packed + tensor.meta = meta + tensor.packed_t = packed_t + tensor.meta_t = meta_t + tensor.compressed_swizzled_bitmask = compressed_swizzled_bitmask + tensor.fuse_transpose_cusparselt = fuse_transpose_cusparselt + tensor.alg_id_cusparselt = alg_id_cusparselt + return tensor + + def __repr__(self) -> str: # type: ignore[override] + assert hasattr(self, "shape") + return f"{self.__class__.__name__}(shape={self.shape})" + + def __tensor_flatten__( + self, + ) -> tuple[list[str], tuple[torch.Size, bool, int, bool]]: + inner_tensors = list( + filter(lambda x: getattr(self, x) is not None, self.__slots__) + ) + tensor_meta = ( + self.shape, + self.fuse_transpose_cusparselt, + self.alg_id_cusparselt, + self.requires_grad, + ) + return inner_tensors, tensor_meta + + @classmethod + def __tensor_unflatten__( + cls, + inner_tensors, + tensor_meta: tuple[torch.Size, bool, int, bool], + outer_size, + outer_stride, + ) -> torch.Tensor: + shape, fuse_transpose_cusparselt, alg_id_cusparselt, requires_grad = tensor_meta + # pyrefly: ignore [no-matching-overload] + return cls( + shape=shape, + packed=inner_tensors.get("packed", None), + meta=inner_tensors.get("meta", None), + packed_t=inner_tensors.get("packed_t", None), + meta_t=inner_tensors.get("meta_t", None), + compressed_swizzled_bitmask=inner_tensors.get( + "compressed_swizzled_bitmask", None + ), + fuse_transpose_cusparselt=fuse_transpose_cusparselt, + alg_id_cusparselt=alg_id_cusparselt, + requires_grad=requires_grad, + ) + + __torch_function__ = torch._C._disabled_torch_function_impl # type: ignore[assignment] + + @classmethod + def __torch_dispatch__(cls, func, types, args, kwargs) -> Any: # type: ignore[override] + if func._overloadpacket not in cls.SPARSE_DISPATCH: + raise NotImplementedError( + f"{cls.__name__} only supports a specific set of operations, " + f"can't perform requested op ({func.__name__})" + ) + return cls.SPARSE_DISPATCH[func._overloadpacket](func, types, args, kwargs) + + @classmethod + def _load_dispatch_table(cls, custom_dispatch_table=None) -> None: + """ + Loads the op overload sparse dispatch table for the current class. + """ + if getattr(cls, "SPARSE_DISPATCH", None) is None: + cls.SPARSE_DISPATCH = { + torch.ops.aten.values: semi_sparse_values, + torch.ops.aten.indices: semi_sparse_indices, + torch.ops.aten.is_same_size: fallback_dispatcher, + torch.ops.aten.detach_: fallback_dispatcher, + torch.ops.aten.detach: semi_sparse_detach, + torch.ops.aten.t: semi_sparse_t, + torch.ops.aten.view: semi_sparse_view, + torch.ops.aten.mm: semi_sparse_mm, + torch.ops.aten.matmul: semi_sparse_mm, + torch.ops.aten.addmm: semi_sparse_addmm, + torch.ops.aten.linear: semi_sparse_linear, + torch.ops.aten._to_copy: fallback_dispatcher, + torch.ops.aten._scaled_mm: semi_sparse_scaled_mm, + } + if custom_dispatch_table is not None: + cls.SPARSE_DISPATCH.update(custom_dispatch_table) + + @classmethod + def _validate_device_dim_dtype_shape(cls, original_tensor: torch.Tensor) -> None: + """ + Assert that the given tensor is valid for semi-structured sparse compression. + """ + # check device + if not original_tensor.is_cuda: + raise RuntimeError( + f"Error original_tensor.device= {original_tensor.device} is not supported! " + "Only CUDA tensors are currently supported." + ) + + # check dim + if original_tensor.dim() != 2: + raise RuntimeError( + f"Error original_tensor.dim = {original_tensor.dim()} is not supported! " + "Only 2d tensors are currently supported." + ) + + # check contiguous + if not original_tensor.is_contiguous(): + raise RuntimeError( + "Error original_tensor is not contiguous!" + "Only contiguous tensors are currently supported." + ) + + # check dtype + if original_tensor.dtype not in cls._DTYPE_SHAPE_CONSTRAINTS: + raise RuntimeError( + f"Error original_tensor.dtype {original_tensor.dtype} is not a supported dtype for {cls}!" + ) + + # check shape + m, n = original_tensor.shape + min_rows = cls._DTYPE_SHAPE_CONSTRAINTS[original_tensor.dtype].sparse_min_rows + min_cols = cls._DTYPE_SHAPE_CONSTRAINTS[original_tensor.dtype].sparse_min_cols + if m < min_rows or m % min_rows or n < min_cols or n % min_cols: + # TODO in the future we can add in padding to support sparse dimensions that aren't perfect multiples + raise RuntimeError( + f"Error original_tensor.shape {original_tensor.shape} is not supported! " + f"Both dimensions must be larger or equal than and a multiple of ({min_rows}, {min_cols})" + ) + + @classmethod + def _pad_dense_input(cls, dense_input: torch.Tensor) -> torch.Tensor: + """ + Calculates padding for dense tensor and pads tensor if necessary. + If padding is not required, this function returns the original tensor. + """ + # only 2d matmul + assert dense_input.dim() == 2 + + # check shape + m, n = dense_input.shape + min_rows = cls._DTYPE_SHAPE_CONSTRAINTS[dense_input.dtype].dense_min_rows + min_cols = cls._DTYPE_SHAPE_CONSTRAINTS[dense_input.dtype].dense_min_cols + + # calculate padding + to_pad_m = -m % min_rows if m < min_rows or m % min_rows else 0 + to_pad_n = -n % min_cols if n < min_cols or n % min_rows else 0 + if to_pad_m or to_pad_n: + return torch.nn.functional.pad(dense_input, (0, to_pad_n, 0, to_pad_m)) + else: + return dense_input + + def to_dense(self): # type:ignore[override] + col = self.shape[-1] + return torch.mm(self, torch.eye(col, dtype=self.dtype, device=self.device)) + + @classmethod + def from_dense(cls, original_tensor: torch.Tensor) -> "SparseSemiStructuredTensor": + raise NotImplementedError + + def _mm( + self, + B: torch.Tensor, + *, + bias: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor: + raise NotImplementedError + + +def to_sparse_semi_structured( + original_tensor: torch.Tensor, + transposed: bool = False, +) -> SparseSemiStructuredTensor: + """ + This function converts a dense tensor into a sparse semi-structured tensor. + It will return a SparseSemiStructuredTensor, a subclass of torch.Tensor. + + This function will check to ensure the dense tensor has the right dtype, size, dims, and device. + We currently only support semi-structured sparse tensors for 2d CUDA tensors. + Additionally, your tensor must be a positive multiple of the minimum sparse block size, given in + `_DTYPE_TO_SHAPE_CONSTRAINTS` for each dtype (float32, float16, bfloat16, int8). + + Args: + original_tensor (Tensor): the dense tensor to convert + transposed (bool, optional): deprecated arg to be removed in another release. Do not use. + Returns: + SparseSemiStructuredTensor: A sparse semi-structured tensor created from the given original_tensor + Raises: + None + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> A = torch.Tensor([0, 0, 1, 1]).tile((128, 32)).half().cuda() + tensor([[0., 0., 1., ..., 0., 1., 1.], + [0., 0., 1., ..., 0., 1., 1.], + [0., 0., 1., ..., 0., 1., 1.], + ..., + [0., 0., 1., ..., 0., 1., 1.], + [0., 0., 1., ..., 0., 1., 1.], + [0., 0., 1., ..., 0., 1., 1.]], device='cuda:0', dtype=torch.float16) + >>> A_sparse = to_sparse_semi_structured(A) + SparseSemiStructuredTensor(shape=torch.Size([128, 128])) + >>> A_sparse.values() + tensor([[1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + ..., + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.], + [1., 1., 1., ..., 1., 1., 1.]], device='cuda:0', dtype=torch.float16), + >>> A_sparse.indices() + tensor([[-4370, -4370, -4370, ..., -4370, -4370, -4370], + [-4370, -4370, -4370, ..., -4370, -4370, -4370], + [-4370, -4370, -4370, ..., -4370, -4370, -4370], + ..., + [-4370, -4370, -4370, ..., -4370, -4370, -4370], + [-4370, -4370, -4370, ..., -4370, -4370, -4370], + [-4370, -4370, -4370, ..., -4370, -4370, -4370]], device='cuda:0', dtype=torch.int16)) + """ + if transposed: + warnings.warn( + "Setting transpose from `to_sparse_semi_structured` is deprecated " + "and will be removed in a future release. " + "`SparseSemiStructuredTensor` only support contiguous input tensors.", + FutureWarning, + stacklevel=2, + ) + + # set from _FORCE_CUTLASS flag + SPARSE_SUBCLASS = ( + torch.sparse.SparseSemiStructuredTensorCUTLASS + if SparseSemiStructuredTensor._FORCE_CUTLASS + else torch.sparse.SparseSemiStructuredTensorCUSPARSELT + ) + + return SPARSE_SUBCLASS.from_dense(original_tensor) + + +class SparseSemiStructuredTensorCUTLASS(SparseSemiStructuredTensor): + """ + This class implements semi-structured sparsity for the CUTLASS backend. + + + In this implementation, the specified elements and metadata are stored separately, + in packed and meta respectively. + + When _FORCE_CUTLASS is set, or when cuSPARSELt is not available, this subclass calls into _sparse_semi_structured_(mm|addmm) and + sparse_semi_structured_from_dense for conversion to the compressed format. + """ + + BACKEND = "cutlass" + _DTYPE_SHAPE_CONSTRAINTS = { + torch.int8: _SEMI_STRUCTURED_SPARSE_CONFIG(16, 128, 16, 16), + torch.float16: _SEMI_STRUCTURED_SPARSE_CONFIG(32, 64, 8, 8), + torch.bfloat16: _SEMI_STRUCTURED_SPARSE_CONFIG(32, 64, 8, 8), + torch.float32: _SEMI_STRUCTURED_SPARSE_CONFIG(32, 32, 4, 4), + } + + @classmethod + def from_dense( + cls, original_tensor: torch.Tensor + ) -> "SparseSemiStructuredTensorCUTLASS": + cls._validate_device_dim_dtype_shape(original_tensor) + ( + sparse_tensor_cutlass, + meta_tensor_cutlass, + ) = sparse_semi_structured_from_dense_cutlass(original_tensor) + # pyrefly: ignore [no-matching-overload] + return cls( + original_tensor.shape, + packed=sparse_tensor_cutlass, + meta=meta_tensor_cutlass, + packed_t=None, + meta_t=None, + compressed_swizzled_bitmask=None, + requires_grad=original_tensor.requires_grad, + ) + + def to_dense(self): # type: ignore[override] + assert self.meta is not None and self.packed is not None + return ( + sparse_semi_structured_to_dense_cutlass( + self.packed, + self.meta, + ) + if self.meta.ndim == 2 + else super().to_dense() + ) + + @classmethod + def prune_dense_static_sort( + cls, original_tensor: torch.Tensor, algorithm="" + ) -> "SparseSemiStructuredTensor": + """ + This function takes in a unpruned dense tensor and runs a (branchless) static sort across a 4x4 tile. + + It greedily picks the largest values in the tile, upholding the 2:4 sparsity constraint across both rows and columns. + The algorithm used to prune the matrix is implemented in `_sparse_semi_structured_tile`. + + Then it creates the packed and meta tensors for the compressed sparse representation of the pruned dense tensor. + It also calculates the packed_t and meta_t tensors for the compressed sparse representation of the transposed + pruned dense tensor. + Since we cannot transpose the compressed representations, we store both for the fw/bw pass respectively. + + Finally, this function also computes a compressed swizzled bitmask that encodes the sparsity pattern + This can be used in the backward pass to mask the gradients. + + [9 1 7 4] [9 0 7 0] + [1 2 3 0] [0 2 0 0] + [8 3 5 4] -> prune 4x4 tile -> [8 0 0 4] -> pack to CUTLASS semi-structured -> packed + [1 2 6 2] [0 0 6 2] -> metadata + + -> pack to transposed CUTLASS -> packed_t + semi-structured representation -> metadata_t + + -> compute swizzled bitmask -> compressed_swizzled_bitmask + + + The equivalent PyTorch code to create the same five outputs from the dense tensor can be found below: + ``` + from torch.sparse import SparseSemiStructuredTensorCUTLASS + from torch.sparse._semi_structured_conversions import ( + _sparse_semi_structured_tile, + _compute_compressed_swizzled_bitmask, + ) + + pruned = _sparse_semi_structured_tile(dense) + packed_cutlass, meta_cutlass = sparse_semi_structured_from_dense_cutlass(pruned) + packed_t_cutlass, meta_t_cutlass = sparse_semi_structured_from_dense_cutlass( + pruned.t().contiguous() + ) + bitmask = _compute_compressed_swizzled_bitmask(pruned) + + SparseSemiStructuredTensorCUTLASS( + dense.shape, + packed_cutlass, + meta_cutlass, + packed_t_cutlass, + meta_t_cutlass, + bitmask, + ) + ``` + """ + # We can either pack to the CUTLASS or cuSPARSELt representation, depending on the use_cutlass flag. + ( + packed, + meta, + packed_t, + meta_t, + compressed_swizzled_bitmask, + ) = torch._sparse_semi_structured_tile( + original_tensor, algorithm=algorithm, use_cutlass=True + ) + + # pyrefly: ignore [no-matching-overload] + return cls( + original_tensor.shape, + packed=packed, + meta=meta, + packed_t=packed_t, + meta_t=meta_t, + compressed_swizzled_bitmask=compressed_swizzled_bitmask, + requires_grad=False, + ) + + def _mm( + self, B: torch.Tensor, *, bias: torch.Tensor | None = None, **kwargs + ) -> torch.Tensor: + if isinstance(B, SparseSemiStructuredTensor): + raise ValueError( + "`SparseSemiStructuredTensor @ SparseSemiStructuredTensor` is not supported by the hardware" + ) + cls_name = self.__class__.__name__ + if self.ndim != 2 or B.ndim != 2: + raise NotImplementedError( + f"`{cls_name}` matmul: Broadcasting is not implemented" + ) + if self.packed is None or self.meta is None: + raise NotImplementedError( + f"`{cls_name}` matmul: operation is not supported" + ) + else: + if bias is None: + res = torch._sparse_semi_structured_mm(self.packed, self.meta, B) + else: + res = torch._sparse_semi_structured_addmm( + bias, self.packed, self.meta, B + ) + return res[: self.shape[0]] + + +class SparseSemiStructuredTensorCUSPARSELT(SparseSemiStructuredTensor): + """ + The cuSPARSELt backend expects the specified elements and the metadata to be stored in a single tensor: + packed = [ specified elements of original tensor | metadata ] + For an original tensor of size (m, k) we expect the first m * k // 2 elements to be the kept elements + The rest of the tensor is metadata. Since there is only one tensor, we only use the packed and packed_t + attributes respectively. + + cuSPARSELt also supports transposition fusion, which is necessary for performant 2:4 sparse training, as well + as specifying alg_id, a config that affects the performance of the matmul depending on matmul sizes. + """ + + BACKEND = "cusparselt" + _DTYPE_SHAPE_CONSTRAINTS = { + torch.float8_e4m3fn: _SEMI_STRUCTURED_SPARSE_CONFIG(32, 32, 16, 16), + torch.int8: _SEMI_STRUCTURED_SPARSE_CONFIG(32, 32, 16, 16), + torch.float16: _SEMI_STRUCTURED_SPARSE_CONFIG(16, 16, 8, 8), + torch.bfloat16: _SEMI_STRUCTURED_SPARSE_CONFIG(16, 16, 8, 8), + } + + @classmethod + def from_dense( + cls, original_tensor: torch.Tensor + ) -> "SparseSemiStructuredTensorCUSPARSELT": + cls._validate_device_dim_dtype_shape(original_tensor) + # pyrefly: ignore [no-matching-overload] + return cls( + shape=original_tensor.shape, + packed=torch._cslt_compress(original_tensor), + meta=None, + packed_t=None, + meta_t=None, + compressed_swizzled_bitmask=None, + fuse_transpose_cusparselt=SparseSemiStructuredTensor._FUSE_TRANSPOSE, + alg_id_cusparselt=SparseSemiStructuredTensor._DEFAULT_ALG_ID, + requires_grad=original_tensor.requires_grad, + ) + + @classmethod + def prune_dense_static_sort( + cls, original_tensor: torch.Tensor, algorithm="" + ) -> "SparseSemiStructuredTensor": + """ + This function does the same thing as described in SparseSemiStructuredCUTLASS, but uses the cuSPARSELt metadata + layout and sparse matmul. + + The only functional difference is that cuSPARSELt stores `metadata` and `packed` together into a single tensor. + + [9 1 7 4] [9 0 7 0] + [1 2 3 0] [0 2 0 0] + [8 3 5 4] -> prune 4x4 tile -> [8 0 0 4] -> pack to cuSPARSELT semi-structured -> packed + [1 2 6 2] [0 0 6 2] + + -> pack to transposed cuSPARSELt -> packed_t + semi-structured representation + + -> compute swizzled bitmask -> compressed_swizzled_bitmask + + + The equivalent PyTorch code to create the same three outputs from the dense tensor can be found below: + ``` + from torch.sparse import SparseSemiStructuredTensorCUSPARSELT + from torch.sparse._semi_structured_conversions import ( + _sparse_semi_structured_tile, + _compute_compressed_swizzled_bitmask, + ) + + pruned = _sparse_semi_structured_tile(dense) + packed_cusparselt = torch._cslt_compress(pruned) + packed_t_cusparselt = torch._cslt_compress(pruned.t().contiguous()) + bitmask = _compute_compressed_swizzled_bitmask(pruned) + + SparseSemiStructuredTensorCUSPARSELT( + dense.shape, packed_cutlass, None, packed_t_cutlass, None, bitmask + ) + ``` + """ + ( + packed, + meta, + packed_t, + meta_t, + compressed_swizzled_bitmask, + ) = torch._sparse_semi_structured_tile( + original_tensor, algorithm=algorithm, use_cutlass=False + ) + + # Map this two 2-dim view of packed data. + # TODO: is this proper cuSPARSELt metadata? + packed = packed.view(original_tensor.shape[0], -1) + packed_t = packed_t.view(original_tensor.shape[1], -1) + + # pyrefly: ignore [no-matching-overload] + return cls( + original_tensor.shape, + packed=packed, + meta=meta, + packed_t=packed_t, + meta_t=meta_t, + compressed_swizzled_bitmask=compressed_swizzled_bitmask, + requires_grad=False, + ) + + def _mm( + self, B: torch.Tensor, *, bias: torch.Tensor | None = None, **kwargs + ) -> torch.Tensor: + if isinstance(B, SparseSemiStructuredTensor): + raise ValueError( + "`SparseSemiStructuredTensor @ SparseSemiStructuredTensor` is not supported by the hardware" + ) + if self.ndim != 2 or B.ndim != 2: + raise NotImplementedError( + f"`{self.__class__.__name__}` matmul: Broadcasting is not implemented" + ) + if B.dtype != self.dtype: + raise NotImplementedError( + f"`{self.__class__.__name__}` matmul: trying to do `A={tuple(self.shape)} @ B={tuple(B.shape)}`, " + f"with A.dtype={self.dtype} and B.dtype={B.dtype}. " + "This operation is only supported when A and B have the same data type." + ) + if bias is not None and bias.dtype != self.dtype: + raise NotImplementedError( + f"`{self.__class__.__name__}` matmul: trying to do `A={tuple(self.shape)} @ B={tuple(B.shape)} + C`, " + f"with A.dtype=B.dtype={self.dtype} and C.dtype={B.dtype}. " + "This operation is only supported when A, B and C have the same data type." + ) + # Force fp8 mm to error to be consistent with torch + if self.dtype == torch.float8_e4m3fn: + raise NotImplementedError( + f"`{self.__class__.__name__}` matmul: trying to do `A={tuple(self.shape)} @ B={tuple(B.shape)}`, " + f"with A.dtype=B.dtype={self.dtype}. " + "mm is not supported for float8_e4m3fn, please use `torch._scaled_mm` instead." + ) + if self.packed is None: + raise NotImplementedError( + f"`{self.__class__.__name__}` matmul: operation is not supported" + ) + else: + res = torch._cslt_sparse_mm( + self.packed, + B, + bias=bias, + transpose_result=self.fuse_transpose_cusparselt, + alg_id=self.alg_id_cusparselt, + ) + return res.t() if self.fuse_transpose_cusparselt else res diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/special/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/special/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dbc9314ad20875b6d37732e201a86f7bc66488dd --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/special/__init__.py @@ -0,0 +1,1459 @@ +import torch +from torch._C import _add_docstr, _special # type: ignore[attr-defined] +from torch._torch_docs import common_args, multi_dim_common + + +__all__ = [ + "airy_ai", + "bessel_j0", + "bessel_j1", + "bessel_y0", + "bessel_y1", + "chebyshev_polynomial_t", + "chebyshev_polynomial_u", + "chebyshev_polynomial_v", + "chebyshev_polynomial_w", + "digamma", + "entr", + "erf", + "erfc", + "erfcx", + "erfinv", + "exp2", + "expit", + "expm1", + "gammainc", + "gammaincc", + "gammaln", + "hermite_polynomial_h", + "hermite_polynomial_he", + "i0", + "i0e", + "i1", + "i1e", + "laguerre_polynomial_l", + "legendre_polynomial_p", + "log1p", + "log_ndtr", + "log_softmax", + "logit", + "logsumexp", + "modified_bessel_i0", + "modified_bessel_i1", + "modified_bessel_k0", + "modified_bessel_k1", + "multigammaln", + "ndtr", + "ndtri", + "polygamma", + "psi", + "round", + "shifted_chebyshev_polynomial_t", + "shifted_chebyshev_polynomial_u", + "shifted_chebyshev_polynomial_v", + "shifted_chebyshev_polynomial_w", + "scaled_modified_bessel_k0", + "scaled_modified_bessel_k1", + "sinc", + "softmax", + "spherical_bessel_j0", + "xlog1py", + "xlogy", + "zeta", +] + +Tensor = torch.Tensor + +entr = _add_docstr( + _special.special_entr, + r""" +entr(input, *, out=None) -> Tensor +Computes the entropy on :attr:`input` (as defined below), elementwise. + +.. math:: + \begin{align} + \text{entr(x)} = \begin{cases} + -x * \ln(x) & x > 0 \\ + 0 & x = 0.0 \\ + -\infty & x < 0 + \end{cases} + \end{align} +""" + + """ + +Args: + input (Tensor): the input tensor. + +Keyword args: + out (Tensor, optional): the output tensor. + +Example:: + + >>> a = torch.arange(-0.5, 1, 0.5) + >>> a + tensor([-0.5000, 0.0000, 0.5000]) + >>> torch.special.entr(a) + tensor([ -inf, 0.0000, 0.3466]) +""", +) + +psi = _add_docstr( + _special.special_psi, + r""" +psi(input, *, out=None) -> Tensor + +Alias for :func:`torch.special.digamma`. +""", +) + +digamma = _add_docstr( + _special.special_digamma, + r""" +digamma(input, *, out=None) -> Tensor + +Computes the logarithmic derivative of the gamma function on `input`. + +.. math:: + \digamma(x) = \frac{d}{dx} \ln\left(\Gamma\left(x\right)\right) = \frac{\Gamma'(x)}{\Gamma(x)} +""" + + r""" +Args: + input (Tensor): the tensor to compute the digamma function on + +Keyword args: + {out} + +.. note:: This function is similar to SciPy's `scipy.special.digamma`. + +.. note:: From PyTorch 1.8 onwards, the digamma function returns `-Inf` for `0`. + Previously it returned `NaN` for `0`. + +Example:: + + >>> a = torch.tensor([1, 0.5]) + >>> torch.special.digamma(a) + tensor([-0.5772, -1.9635]) + +""".format(**common_args), +) + +gammaln = _add_docstr( + _special.special_gammaln, + r""" +gammaln(input, *, out=None) -> Tensor + +Computes the natural logarithm of the absolute value of the gamma function on :attr:`input`. + +.. math:: + \text{out}_{i} = \ln \Gamma(|\text{input}_{i}|) +""" + + """ +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> a = torch.arange(0.5, 2, 0.5) + >>> torch.special.gammaln(a) + tensor([ 0.5724, 0.0000, -0.1208]) + +""".format(**common_args), +) + +polygamma = _add_docstr( + _special.special_polygamma, + r""" +polygamma(n, input, *, out=None) -> Tensor + +Computes the :math:`n^{th}` derivative of the digamma function on :attr:`input`. +:math:`n \geq 0` is called the order of the polygamma function. + +.. math:: + \psi^{(n)}(x) = \frac{d^{(n)}}{dx^{(n)}} \psi(x) + +.. note:: + This function is implemented only for nonnegative integers :math:`n \geq 0`. +""" + + """ +Args: + n (int): the order of the polygamma function + {input} + +Keyword args: + {out} + +Example:: + + >>> a = torch.tensor([1, 0.5]) + >>> torch.special.polygamma(1, a) + tensor([1.64493, 4.9348]) + >>> torch.special.polygamma(2, a) + tensor([ -2.4041, -16.8288]) + >>> torch.special.polygamma(3, a) + tensor([ 6.4939, 97.4091]) + >>> torch.special.polygamma(4, a) + tensor([ -24.8863, -771.4742]) +""".format(**common_args), +) + +erf = _add_docstr( + _special.special_erf, + r""" +erf(input, *, out=None) -> Tensor + +Computes the error function of :attr:`input`. The error function is defined as follows: + +.. math:: + \mathrm{erf}(x) = \frac{2}{\sqrt{\pi}} \int_{0}^{x} e^{-t^2} dt +""" + + r""" +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> torch.special.erf(torch.tensor([0, -1., 10.])) + tensor([ 0.0000, -0.8427, 1.0000]) +""".format(**common_args), +) + +erfc = _add_docstr( + _special.special_erfc, + r""" +erfc(input, *, out=None) -> Tensor + +Computes the complementary error function of :attr:`input`. +The complementary error function is defined as follows: + +.. math:: + \mathrm{erfc}(x) = 1 - \frac{2}{\sqrt{\pi}} \int_{0}^{x} e^{-t^2} dt +""" + + r""" +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> torch.special.erfc(torch.tensor([0, -1., 10.])) + tensor([ 1.0000, 1.8427, 0.0000]) +""".format(**common_args), +) + +erfcx = _add_docstr( + _special.special_erfcx, + r""" +erfcx(input, *, out=None) -> Tensor + +Computes the scaled complementary error function for each element of :attr:`input`. +The scaled complementary error function is defined as follows: + +.. math:: + \mathrm{erfcx}(x) = e^{x^2} \mathrm{erfc}(x) +""" + + r""" + +""" + + r""" +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> torch.special.erfcx(torch.tensor([0, -1., 10.])) + tensor([ 1.0000, 5.0090, 0.0561]) +""".format(**common_args), +) + +erfinv = _add_docstr( + _special.special_erfinv, + r""" +erfinv(input, *, out=None) -> Tensor + +Computes the inverse error function of :attr:`input`. +The inverse error function is defined in the range :math:`(-1, 1)` as: + +.. math:: + \mathrm{erfinv}(\mathrm{erf}(x)) = x +""" + + r""" + +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> torch.special.erfinv(torch.tensor([0, 0.5, -1.])) + tensor([ 0.0000, 0.4769, -inf]) +""".format(**common_args), +) + +logit = _add_docstr( + _special.special_logit, + r""" +logit(input, eps=None, *, out=None) -> Tensor + +Returns a new tensor with the logit of the elements of :attr:`input`. +:attr:`input` is clamped to [eps, 1 - eps] when eps is not None. +When eps is None and :attr:`input` < 0 or :attr:`input` > 1, the function will yields NaN. + +.. math:: + \begin{align} + y_{i} &= \ln(\frac{z_{i}}{1 - z_{i}}) \\ + z_{i} &= \begin{cases} + x_{i} & \text{if eps is None} \\ + \text{eps} & \text{if } x_{i} < \text{eps} \\ + x_{i} & \text{if } \text{eps} \leq x_{i} \leq 1 - \text{eps} \\ + 1 - \text{eps} & \text{if } x_{i} > 1 - \text{eps} + \end{cases} + \end{align} +""" + + r""" +Args: + {input} + eps (float, optional): the epsilon for input clamp bound. Default: ``None`` + +Keyword args: + {out} + +Example:: + + >>> a = torch.rand(5) + >>> a + tensor([0.2796, 0.9331, 0.6486, 0.1523, 0.6516]) + >>> torch.special.logit(a, eps=1e-6) + tensor([-0.9466, 2.6352, 0.6131, -1.7169, 0.6261]) +""".format(**common_args), +) + +logsumexp = _add_docstr( + _special.special_logsumexp, + r""" +logsumexp(input, dim, keepdim=False, *, out=None) + +Alias for :func:`torch.logsumexp`. +""".format(**multi_dim_common), +) + +expit = _add_docstr( + _special.special_expit, + r""" +expit(input, *, out=None) -> Tensor + +Computes the expit (also known as the logistic sigmoid function) of the elements of :attr:`input`. + +.. math:: + \text{out}_{i} = \frac{1}{1 + e^{-\text{input}_{i}}} +""" + + r""" +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> t = torch.randn(4) + >>> t + tensor([ 0.9213, 1.0887, -0.8858, -1.7683]) + >>> torch.special.expit(t) + tensor([ 0.7153, 0.7481, 0.2920, 0.1458]) +""".format(**common_args), +) + +exp2 = _add_docstr( + _special.special_exp2, + r""" +exp2(input, *, out=None) -> Tensor + +Computes the base two exponential function of :attr:`input`. + +.. math:: + y_{i} = 2^{x_{i}} + +""" + + r""" +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> torch.special.exp2(torch.tensor([0, math.log2(2.), 3, 4])) + tensor([ 1., 2., 8., 16.]) +""".format(**common_args), +) + +expm1 = _add_docstr( + _special.special_expm1, + r""" +expm1(input, *, out=None) -> Tensor + +Computes the exponential of the elements minus 1 +of :attr:`input`. + +.. math:: + y_{i} = e^{x_{i}} - 1 + +.. note:: This function provides greater precision than exp(x) - 1 for small values of x. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> torch.special.expm1(torch.tensor([0, math.log(2.)])) + tensor([ 0., 1.]) +""".format(**common_args), +) + +xlog1py = _add_docstr( + _special.special_xlog1py, + r""" +xlog1py(input, other, *, out=None) -> Tensor + +Computes ``input * log1p(other)`` with the following cases. + +.. math:: + \text{out}_{i} = \begin{cases} + \text{NaN} & \text{if } \text{other}_{i} = \text{NaN} \\ + 0 & \text{if } \text{input}_{i} = 0.0 \text{ and } \text{other}_{i} != \text{NaN} \\ + \text{input}_{i} * \text{log1p}(\text{other}_{i})& \text{otherwise} + \end{cases} + +Similar to SciPy's `scipy.special.xlog1py`. + +""" + + r""" + +Args: + input (Number or Tensor) : Multiplier + other (Number or Tensor) : Argument + +.. note:: At least one of :attr:`input` or :attr:`other` must be a tensor. + +Keyword args: + {out} + +Example:: + + >>> x = torch.zeros(5,) + >>> y = torch.tensor([-1, 0, 1, float('inf'), float('nan')]) + >>> torch.special.xlog1py(x, y) + tensor([0., 0., 0., 0., nan]) + >>> x = torch.tensor([1, 2, 3]) + >>> y = torch.tensor([3, 2, 1]) + >>> torch.special.xlog1py(x, y) + tensor([1.3863, 2.1972, 2.0794]) + >>> torch.special.xlog1py(x, 4) + tensor([1.6094, 3.2189, 4.8283]) + >>> torch.special.xlog1py(2, y) + tensor([2.7726, 2.1972, 1.3863]) +""".format(**common_args), +) + +xlogy = _add_docstr( + _special.special_xlogy, + r""" +xlogy(input, other, *, out=None) -> Tensor + +Computes ``input * log(other)`` with the following cases. + +.. math:: + \text{out}_{i} = \begin{cases} + \text{NaN} & \text{if } \text{other}_{i} = \text{NaN} \\ + 0 & \text{if } \text{input}_{i} = 0.0 \\ + \text{input}_{i} * \log{(\text{other}_{i})} & \text{otherwise} + \end{cases} + +Similar to SciPy's `scipy.special.xlogy`. + +""" + + r""" + +Args: + input (Number or Tensor) : Multiplier + other (Number or Tensor) : Argument + +.. note:: At least one of :attr:`input` or :attr:`other` must be a tensor. + +Keyword args: + {out} + +Example:: + + >>> x = torch.zeros(5,) + >>> y = torch.tensor([-1, 0, 1, float('inf'), float('nan')]) + >>> torch.special.xlogy(x, y) + tensor([0., 0., 0., 0., nan]) + >>> x = torch.tensor([1, 2, 3]) + >>> y = torch.tensor([3, 2, 1]) + >>> torch.special.xlogy(x, y) + tensor([1.0986, 1.3863, 0.0000]) + >>> torch.special.xlogy(x, 4) + tensor([1.3863, 2.7726, 4.1589]) + >>> torch.special.xlogy(2, y) + tensor([2.1972, 1.3863, 0.0000]) +""".format(**common_args), +) + +i0 = _add_docstr( + _special.special_i0, + r""" +i0(input, *, out=None) -> Tensor + +Computes the zeroth order modified Bessel function of the first kind for each element of :attr:`input`. + +.. math:: + \text{out}_{i} = I_0(\text{input}_{i}) = \sum_{k=0}^{\infty} \frac{(\text{input}_{i}^2/4)^k}{(k!)^2} + +""" + + r""" +Args: + input (Tensor): the input tensor + +Keyword args: + {out} + +Example:: + + >>> torch.i0(torch.arange(5, dtype=torch.float32)) + tensor([ 1.0000, 1.2661, 2.2796, 4.8808, 11.3019]) + +""".format(**common_args), +) + +i0e = _add_docstr( + _special.special_i0e, + r""" +i0e(input, *, out=None) -> Tensor +Computes the exponentially scaled zeroth order modified Bessel function of the first kind (as defined below) +for each element of :attr:`input`. + +.. math:: + \text{out}_{i} = \exp(-|x|) * i0(x) = \exp(-|x|) * \sum_{k=0}^{\infty} \frac{(\text{input}_{i}^2/4)^k}{(k!)^2} + +""" + + r""" +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> torch.special.i0e(torch.arange(5, dtype=torch.float32)) + tensor([1.0000, 0.4658, 0.3085, 0.2430, 0.2070]) +""".format(**common_args), +) + +i1 = _add_docstr( + _special.special_i1, + r""" +i1(input, *, out=None) -> Tensor +Computes the first order modified Bessel function of the first kind (as defined below) +for each element of :attr:`input`. + +.. math:: + \text{out}_{i} = \frac{(\text{input}_{i})}{2} * \sum_{k=0}^{\infty} \frac{(\text{input}_{i}^2/4)^k}{(k!) * (k+1)!} + +""" + + r""" +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> torch.special.i1(torch.arange(5, dtype=torch.float32)) + tensor([0.0000, 0.5652, 1.5906, 3.9534, 9.7595]) +""".format(**common_args), +) + +i1e = _add_docstr( + _special.special_i1e, + r""" +i1e(input, *, out=None) -> Tensor +Computes the exponentially scaled first order modified Bessel function of the first kind (as defined below) +for each element of :attr:`input`. + +.. math:: + \text{out}_{i} = \exp(-|x|) * i1(x) = + \exp(-|x|) * \frac{(\text{input}_{i})}{2} * \sum_{k=0}^{\infty} \frac{(\text{input}_{i}^2/4)^k}{(k!) * (k+1)!} + +""" + + r""" +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> torch.special.i1e(torch.arange(5, dtype=torch.float32)) + tensor([0.0000, 0.2079, 0.2153, 0.1968, 0.1788]) +""".format(**common_args), +) + +ndtr = _add_docstr( + _special.special_ndtr, + r""" +ndtr(input, *, out=None) -> Tensor +Computes the area under the standard Gaussian probability density function, +integrated from minus infinity to :attr:`input`, elementwise. + +.. math:: + \text{ndtr}(x) = \frac{1}{\sqrt{2 \pi}}\int_{-\infty}^{x} e^{-\frac{1}{2}t^2} dt + +""" + + r""" +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> torch.special.ndtr(torch.tensor([-3., -2, -1, 0, 1, 2, 3])) + tensor([0.0013, 0.0228, 0.1587, 0.5000, 0.8413, 0.9772, 0.9987]) +""".format(**common_args), +) + +ndtri = _add_docstr( + _special.special_ndtri, + r""" +ndtri(input, *, out=None) -> Tensor +Computes the argument, x, for which the area under the Gaussian probability density function +(integrated from minus infinity to x) is equal to :attr:`input`, elementwise. + +.. math:: + \text{ndtri}(p) = \sqrt{2}\text{erf}^{-1}(2p - 1) + +.. note:: + Also known as quantile function for Normal Distribution. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> torch.special.ndtri(torch.tensor([0, 0.25, 0.5, 0.75, 1])) + tensor([ -inf, -0.6745, 0.0000, 0.6745, inf]) +""".format(**common_args), +) + +log_ndtr = _add_docstr( + _special.special_log_ndtr, + r""" +log_ndtr(input, *, out=None) -> Tensor +Computes the log of the area under the standard Gaussian probability density function, +integrated from minus infinity to :attr:`input`, elementwise. + +.. math:: + \text{log\_ndtr}(x) = \log\left(\frac{1}{\sqrt{2 \pi}}\int_{-\infty}^{x} e^{-\frac{1}{2}t^2} dt \right) + +""" + + r""" +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> torch.special.log_ndtr(torch.tensor([-3., -2, -1, 0, 1, 2, 3])) + tensor([-6.6077 -3.7832 -1.841 -0.6931 -0.1728 -0.023 -0.0014]) +""".format(**common_args), +) + +log1p = _add_docstr( + _special.special_log1p, + r""" +log1p(input, *, out=None) -> Tensor + +Alias for :func:`torch.log1p`. +""", +) + +sinc = _add_docstr( + _special.special_sinc, + r""" +sinc(input, *, out=None) -> Tensor + +Computes the normalized sinc of :attr:`input.` + +.. math:: + \text{out}_{i} = + \begin{cases} + 1, & \text{if}\ \text{input}_{i}=0 \\ + \sin(\pi \text{input}_{i}) / (\pi \text{input}_{i}), & \text{otherwise} + \end{cases} +""" + + r""" + +Args: + {input} + +Keyword args: + {out} + +Example:: + + >>> t = torch.randn(4) + >>> t + tensor([ 0.2252, -0.2948, 1.0267, -1.1566]) + >>> torch.special.sinc(t) + tensor([ 0.9186, 0.8631, -0.0259, -0.1300]) +""".format(**common_args), +) + +round = _add_docstr( + _special.special_round, + r""" +round(input, *, out=None) -> Tensor + +Alias for :func:`torch.round`. +""", +) + +softmax = _add_docstr( + _special.special_softmax, + r""" +softmax(input, dim, *, dtype=None) -> Tensor + +Computes the softmax function. + +Softmax is defined as: + +:math:`\text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)}` + +It is applied to all slices along dim, and will re-scale them so that the elements +lie in the range `[0, 1]` and sum to 1. + +Args: + input (Tensor): input + dim (int): A dimension along which softmax will be computed. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is cast to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + +Examples:: + >>> t = torch.ones(2, 2) + >>> torch.special.softmax(t, 0) + tensor([[0.5000, 0.5000], + [0.5000, 0.5000]]) + +""", +) + +log_softmax = _add_docstr( + _special.special_log_softmax, + r""" +log_softmax(input, dim, *, dtype=None) -> Tensor + +Computes softmax followed by a logarithm. + +While mathematically equivalent to log(softmax(x)), doing these two +operations separately is slower and numerically unstable. This function +is computed as: + +.. math:: + \text{log\_softmax}(x_{i}) = \log\left(\frac{\exp(x_i) }{ \sum_j \exp(x_j)} \right) +""" + + r""" + +Args: + input (Tensor): input + dim (int): A dimension along which log_softmax will be computed. + dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. + If specified, the input tensor is cast to :attr:`dtype` before the operation + is performed. This is useful for preventing data type overflows. Default: None. + +Example:: + + >>> t = torch.ones(2, 2) + >>> torch.special.log_softmax(t, 0) + tensor([[-0.6931, -0.6931], + [-0.6931, -0.6931]]) +""", +) + +zeta = _add_docstr( + _special.special_zeta, + r""" +zeta(input, other, *, out=None) -> Tensor + +Computes the Hurwitz zeta function, elementwise. + +.. math:: + \zeta(x, q) = \sum_{k=0}^{\infty} \frac{1}{(k + q)^x} + +""" + + r""" +Args: + input (Tensor): the input tensor corresponding to `x`. + other (Tensor): the input tensor corresponding to `q`. + +.. note:: + The Riemann zeta function corresponds to the case when `q = 1` + +Keyword args: + {out} + +Example:: + + >>> x = torch.tensor([2., 4.]) + >>> torch.special.zeta(x, 1) + tensor([1.6449, 1.0823]) + >>> torch.special.zeta(x, torch.tensor([1., 2.])) + tensor([1.6449, 0.0823]) + >>> torch.special.zeta(2, torch.tensor([1., 2.])) + tensor([1.6449, 0.6449]) +""".format(**common_args), +) + +multigammaln = _add_docstr( + _special.special_multigammaln, + r""" +multigammaln(input, p, *, out=None) -> Tensor + +Computes the `multivariate log-gamma function +`_ with dimension +:math:`p` element-wise, given by + +.. math:: + \log(\Gamma_{p}(a)) = C + \displaystyle \sum_{i=1}^{p} \log\left(\Gamma\left(a - \frac{i - 1}{2}\right)\right) + +where :math:`C = \log(\pi) \cdot \frac{p (p - 1)}{4}` and :math:`\Gamma(-)` is the Gamma function. + +All elements must be greater than :math:`\frac{p - 1}{2}`, otherwise the behavior is undefined. +""" + + """ + +Args: + input (Tensor): the tensor to compute the multivariate log-gamma function + p (int): the number of dimensions + +Keyword args: + {out} + +Example:: + + >>> a = torch.empty(2, 3).uniform_(1, 2) + >>> a + tensor([[1.6835, 1.8474, 1.1929], + [1.0475, 1.7162, 1.4180]]) + >>> torch.special.multigammaln(a, 2) + tensor([[0.3928, 0.4007, 0.7586], + [1.0311, 0.3901, 0.5049]]) +""".format(**common_args), +) + +gammainc = _add_docstr( + _special.special_gammainc, + r""" +gammainc(input, other, *, out=None) -> Tensor + +Computes the regularized lower incomplete gamma function: + +.. math:: + \text{out}_{i} = \frac{1}{\Gamma(\text{input}_i)} \int_0^{\text{other}_i} t^{\text{input}_i-1} e^{-t} dt + +where both :math:`\text{input}_i` and :math:`\text{other}_i` are weakly positive +and at least one is strictly positive. +If both are zero or either is negative then :math:`\text{out}_i=\text{nan}`. +:math:`\Gamma(\cdot)` in the equation above is the gamma function, + +.. math:: + \Gamma(\text{input}_i) = \int_0^\infty t^{(\text{input}_i-1)} e^{-t} dt. + +See :func:`torch.special.gammaincc` and :func:`torch.special.gammaln` for related functions. + +Supports :ref:`broadcasting to a common shape ` +and float inputs. + +.. note:: + The backward pass with respect to :attr:`input` is not yet supported. + Please open an issue on PyTorch's Github to request it. + +""" + + r""" +Args: + input (Tensor): the first non-negative input tensor + other (Tensor): the second non-negative input tensor + +Keyword args: + {out} + +Example:: + + >>> a1 = torch.tensor([4.0]) + >>> a2 = torch.tensor([3.0, 4.0, 5.0]) + >>> a = torch.special.gammaincc(a1, a2) + tensor([0.3528, 0.5665, 0.7350]) + tensor([0.3528, 0.5665, 0.7350]) + >>> b = torch.special.gammainc(a1, a2) + torch.special.gammaincc(a1, a2) + tensor([1., 1., 1.]) + +""".format(**common_args), +) + +gammaincc = _add_docstr( + _special.special_gammaincc, + r""" +gammaincc(input, other, *, out=None) -> Tensor + +Computes the regularized upper incomplete gamma function: + +.. math:: + \text{out}_{i} = \frac{1}{\Gamma(\text{input}_i)} \int_{\text{other}_i}^{\infty} t^{\text{input}_i-1} e^{-t} dt + +where both :math:`\text{input}_i` and :math:`\text{other}_i` are weakly positive +and at least one is strictly positive. +If both are zero or either is negative then :math:`\text{out}_i=\text{nan}`. +:math:`\Gamma(\cdot)` in the equation above is the gamma function, + +.. math:: + \Gamma(\text{input}_i) = \int_0^\infty t^{(\text{input}_i-1)} e^{-t} dt. + +See :func:`torch.special.gammainc` and :func:`torch.special.gammaln` for related functions. + +Supports :ref:`broadcasting to a common shape ` +and float inputs. + +.. note:: + The backward pass with respect to :attr:`input` is not yet supported. + Please open an issue on PyTorch's Github to request it. + +""" + + r""" +Args: + input (Tensor): the first non-negative input tensor + other (Tensor): the second non-negative input tensor + +Keyword args: + {out} + +Example:: + + >>> a1 = torch.tensor([4.0]) + >>> a2 = torch.tensor([3.0, 4.0, 5.0]) + >>> a = torch.special.gammaincc(a1, a2) + tensor([0.6472, 0.4335, 0.2650]) + >>> b = torch.special.gammainc(a1, a2) + torch.special.gammaincc(a1, a2) + tensor([1., 1., 1.]) + +""".format(**common_args), +) + +airy_ai = _add_docstr( + _special.special_airy_ai, + r""" +airy_ai(input, *, out=None) -> Tensor + +Airy function :math:`\text{Ai}\left(\text{input}\right)`. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} +""".format(**common_args), +) + +bessel_j0 = _add_docstr( + _special.special_bessel_j0, + r""" +bessel_j0(input, *, out=None) -> Tensor + +Bessel function of the first kind of order :math:`0`. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} +""".format(**common_args), +) + +bessel_j1 = _add_docstr( + _special.special_bessel_j1, + r""" +bessel_j1(input, *, out=None) -> Tensor + +Bessel function of the first kind of order :math:`1`. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} +""".format(**common_args), +) + +bessel_y0 = _add_docstr( + _special.special_bessel_y0, + r""" +bessel_y0(input, *, out=None) -> Tensor + +Bessel function of the second kind of order :math:`0`. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} +""".format(**common_args), +) + +bessel_y1 = _add_docstr( + _special.special_bessel_y1, + r""" +bessel_y1(input, *, out=None) -> Tensor + +Bessel function of the second kind of order :math:`1`. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} +""".format(**common_args), +) + +chebyshev_polynomial_t = _add_docstr( + _special.special_chebyshev_polynomial_t, + r""" +chebyshev_polynomial_t(input, n, *, out=None) -> Tensor + +Chebyshev polynomial of the first kind :math:`T_{n}(\text{input})`. + +If :math:`n = 0`, :math:`1` is returned. If :math:`n = 1`, :math:`\text{input}` +is returned. If :math:`n < 6` or :math:`|\text{input}| > 1` the recursion: + +.. math:: + T_{n + 1}(\text{input}) = 2 \times \text{input} \times T_{n}(\text{input}) - T_{n - 1}(\text{input}) + +is evaluated. Otherwise, the explicit trigonometric formula: + +.. math:: + T_{n}(\text{input}) = \text{cos}(n \times \text{arccos}(x)) + +is evaluated. + +""" + + r""" +Args: + {input} + n (Tensor): Degree of the polynomial. + +Keyword args: + {out} +""".format(**common_args), +) + +chebyshev_polynomial_u = _add_docstr( + _special.special_chebyshev_polynomial_u, + r""" +chebyshev_polynomial_u(input, n, *, out=None) -> Tensor + +Chebyshev polynomial of the second kind :math:`U_{n}(\text{input})`. + +If :math:`n = 0`, :math:`1` is returned. If :math:`n = 1`, +:math:`2 \times \text{input}` is returned. If :math:`n < 6` or +:math:`|\text{input}| > 1`, the recursion: + +.. math:: + U_{n + 1}(\text{input}) = 2 \times \text{input} \times U_{n}(\text{input}) - U_{n - 1}(\text{input}) + +is evaluated. Otherwise, the explicit trigonometric formula: + +.. math:: + \frac{\text{sin}((n + 1) \times \text{arccos}(\text{input}))}{\text{sin}(\text{arccos}(\text{input}))} + +is evaluated. + +""" + + r""" +Args: + {input} + n (Tensor): Degree of the polynomial. + +Keyword args: + {out} +""".format(**common_args), +) + +chebyshev_polynomial_v = _add_docstr( + _special.special_chebyshev_polynomial_v, + r""" +chebyshev_polynomial_v(input, n, *, out=None) -> Tensor + +Chebyshev polynomial of the third kind :math:`V_{n}^{\ast}(\text{input})`. + +""" + + r""" +Args: + {input} + n (Tensor): Degree of the polynomial. + +Keyword args: + {out} +""".format(**common_args), +) + +chebyshev_polynomial_w = _add_docstr( + _special.special_chebyshev_polynomial_w, + r""" +chebyshev_polynomial_w(input, n, *, out=None) -> Tensor + +Chebyshev polynomial of the fourth kind :math:`W_{n}^{\ast}(\text{input})`. + +""" + + r""" +Args: + {input} + n (Tensor): Degree of the polynomial. + +Keyword args: + {out} +""".format(**common_args), +) + +hermite_polynomial_h = _add_docstr( + _special.special_hermite_polynomial_h, + r""" +hermite_polynomial_h(input, n, *, out=None) -> Tensor + +Physicist's Hermite polynomial :math:`H_{n}(\text{input})`. + +If :math:`n = 0`, :math:`1` is returned. If :math:`n = 1`, :math:`\text{input}` +is returned. Otherwise, the recursion: + +.. math:: + H_{n + 1}(\text{input}) = 2 \times \text{input} \times H_{n}(\text{input}) - H_{n - 1}(\text{input}) + +is evaluated. + +""" + + r""" +Args: + {input} + n (Tensor): Degree of the polynomial. + +Keyword args: + {out} +""".format(**common_args), +) + +hermite_polynomial_he = _add_docstr( + _special.special_hermite_polynomial_he, + r""" +hermite_polynomial_he(input, n, *, out=None) -> Tensor + +Probabilist's Hermite polynomial :math:`He_{n}(\text{input})`. + +If :math:`n = 0`, :math:`1` is returned. If :math:`n = 1`, :math:`\text{input}` +is returned. Otherwise, the recursion: + +.. math:: + He_{n + 1}(\text{input}) = 2 \times \text{input} \times He_{n}(\text{input}) - He_{n - 1}(\text{input}) + +is evaluated. + +""" + + r""" +Args: + {input} + n (Tensor): Degree of the polynomial. + +Keyword args: + {out} +""".format(**common_args), +) + +laguerre_polynomial_l = _add_docstr( + _special.special_laguerre_polynomial_l, + r""" +laguerre_polynomial_l(input, n, *, out=None) -> Tensor + +Laguerre polynomial :math:`L_{n}(\text{input})`. + +If :math:`n = 0`, :math:`1` is returned. If :math:`n = 1`, :math:`\text{input}` +is returned. Otherwise, the recursion: + +.. math:: + L_{n + 1}(\text{input}) = 2 \times \text{input} \times L_{n}(\text{input}) - L_{n - 1}(\text{input}) + +is evaluated. + +""" + + r""" +Args: + {input} + n (Tensor): Degree of the polynomial. + +Keyword args: + {out} +""".format(**common_args), +) + +legendre_polynomial_p = _add_docstr( + _special.special_legendre_polynomial_p, + r""" +legendre_polynomial_p(input, n, *, out=None) -> Tensor + +Legendre polynomial :math:`P_{n}(\text{input})`. + +If :math:`n = 0`, :math:`1` is returned. If :math:`n = 1`, :math:`\text{input}` +is returned. Otherwise, the recursion: + +.. math:: + P_{n + 1}(\text{input}) = 2 \times \text{input} \times P_{n}(\text{input}) - P_{n - 1}(\text{input}) + +is evaluated. + +""" + + r""" +Args: + {input} + n (Tensor): Degree of the polynomial. + +Keyword args: + {out} +""".format(**common_args), +) + +modified_bessel_i0 = _add_docstr( + _special.special_modified_bessel_i0, + r""" +modified_bessel_i0(input, *, out=None) -> Tensor + +Modified Bessel function of the first kind of order :math:`0`. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} +""".format(**common_args), +) + +modified_bessel_i1 = _add_docstr( + _special.special_modified_bessel_i1, + r""" +modified_bessel_i1(input, *, out=None) -> Tensor + +Modified Bessel function of the first kind of order :math:`1`. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} +""".format(**common_args), +) + +modified_bessel_k0 = _add_docstr( + _special.special_modified_bessel_k0, + r""" +modified_bessel_k0(input, *, out=None) -> Tensor + +Modified Bessel function of the second kind of order :math:`0`. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} +""".format(**common_args), +) + +modified_bessel_k1 = _add_docstr( + _special.special_modified_bessel_k1, + r""" +modified_bessel_k1(input, *, out=None) -> Tensor + +Modified Bessel function of the second kind of order :math:`1`. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} +""".format(**common_args), +) + +scaled_modified_bessel_k0 = _add_docstr( + _special.special_scaled_modified_bessel_k0, + r""" +scaled_modified_bessel_k0(input, *, out=None) -> Tensor + +Scaled modified Bessel function of the second kind of order :math:`0`. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} +""".format(**common_args), +) + +scaled_modified_bessel_k1 = _add_docstr( + _special.special_scaled_modified_bessel_k1, + r""" +scaled_modified_bessel_k1(input, *, out=None) -> Tensor + +Scaled modified Bessel function of the second kind of order :math:`1`. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} +""".format(**common_args), +) + +shifted_chebyshev_polynomial_t = _add_docstr( + _special.special_shifted_chebyshev_polynomial_t, + r""" +shifted_chebyshev_polynomial_t(input, n, *, out=None) -> Tensor + +Chebyshev polynomial of the first kind :math:`T_{n}^{\ast}(\text{input})`. + +""" + + r""" +Args: + {input} + n (Tensor): Degree of the polynomial. + +Keyword args: + {out} +""".format(**common_args), +) + +shifted_chebyshev_polynomial_u = _add_docstr( + _special.special_shifted_chebyshev_polynomial_u, + r""" +shifted_chebyshev_polynomial_u(input, n, *, out=None) -> Tensor + +Chebyshev polynomial of the second kind :math:`U_{n}^{\ast}(\text{input})`. + +""" + + r""" +Args: + {input} + n (Tensor): Degree of the polynomial. + +Keyword args: + {out} +""".format(**common_args), +) + +shifted_chebyshev_polynomial_v = _add_docstr( + _special.special_shifted_chebyshev_polynomial_v, + r""" +shifted_chebyshev_polynomial_v(input, n, *, out=None) -> Tensor + +Chebyshev polynomial of the third kind :math:`V_{n}^{\ast}(\text{input})`. + +""" + + r""" +Args: + {input} + n (Tensor): Degree of the polynomial. + +Keyword args: + {out} +""".format(**common_args), +) + +shifted_chebyshev_polynomial_w = _add_docstr( + _special.special_shifted_chebyshev_polynomial_w, + r""" +shifted_chebyshev_polynomial_w(input, n, *, out=None) -> Tensor + +Chebyshev polynomial of the fourth kind :math:`W_{n}^{\ast}(\text{input})`. + +""" + + r""" +Args: + {input} + n (Tensor): Degree of the polynomial. + +Keyword args: + {out} +""".format(**common_args), +) + +spherical_bessel_j0 = _add_docstr( + _special.special_spherical_bessel_j0, + r""" +spherical_bessel_j0(input, *, out=None) -> Tensor + +Spherical Bessel function of the first kind of order :math:`0`. + +""" + + r""" +Args: + {input} + +Keyword args: + {out} +""".format(**common_args), +) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/testing/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/testing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6724bd3d523b09a582cef5ed256f4c14b281fa72 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/testing/__init__.py @@ -0,0 +1,7 @@ +from torch._C import FileCheck as FileCheck + +from . import _utils + +# pyrefly: ignore [deprecated] +from ._comparison import assert_allclose, assert_close as assert_close +from ._creation import make_tensor as make_tensor diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/testing/_comparison.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/testing/_comparison.py new file mode 100644 index 0000000000000000000000000000000000000000..0f3f7f3e02aa7bbea69fed08dde01aff965f1c94 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/testing/_comparison.py @@ -0,0 +1,1650 @@ +# mypy: allow-untyped-defs +import abc +import cmath +import collections.abc +import contextlib +from collections.abc import Callable, Collection, Sequence +from typing import Any, NoReturn, Optional, Union +from typing_extensions import deprecated + +import torch + + +try: + import numpy as np + + HAS_NUMPY = True +except ModuleNotFoundError: + HAS_NUMPY = False + np = None # type: ignore[assignment] + + +class ErrorMeta(Exception): + """Internal testing exception that makes that carries error metadata.""" + + def __init__( + self, type: type[Exception], msg: str, *, id: tuple[Any, ...] = () + ) -> None: + super().__init__( + "If you are a user and see this message during normal operation " + "please file an issue at https://github.com/pytorch/pytorch/issues. " + "If you are a developer and working on the comparison functions, please `raise ErrorMeta.to_error()` " + "for user facing errors." + ) + self.type = type + self.msg = msg + self.id = id + + def to_error( + self, msg: Optional[Union[str, Callable[[str], str]]] = None + ) -> Exception: + if not isinstance(msg, str): + generated_msg = self.msg + if self.id: + generated_msg += f"\n\nThe failure occurred for item {''.join(str([item]) for item in self.id)}" + + msg = msg(generated_msg) if callable(msg) else generated_msg + + return self.type(msg) + + +# Some analysis of tolerance by logging tests from test_torch.py can be found in +# https://github.com/pytorch/pytorch/pull/32538. +# {dtype: (rtol, atol)} +_DTYPE_PRECISIONS = { + torch.float16: (0.001, 1e-5), + torch.bfloat16: (0.016, 1e-5), + torch.float32: (1.3e-6, 1e-5), + torch.float64: (1e-7, 1e-7), + torch.complex32: (0.001, 1e-5), + torch.complex64: (1.3e-6, 1e-5), + torch.complex128: (1e-7, 1e-7), +} +# The default tolerances of torch.float32 are used for quantized dtypes, because quantized tensors are compared in +# their dequantized and floating point representation. For more details see `TensorLikePair._compare_quantized_values` +_DTYPE_PRECISIONS.update( + dict.fromkeys( + (torch.quint8, torch.quint2x4, torch.quint4x2, torch.qint8, torch.qint32), + _DTYPE_PRECISIONS[torch.float32], + ) +) + + +def default_tolerances( + *inputs: Union[torch.Tensor, torch.dtype], + dtype_precisions: Optional[dict[torch.dtype, tuple[float, float]]] = None, +) -> tuple[float, float]: + """Returns the default absolute and relative testing tolerances for a set of inputs based on the dtype. + + See :func:`assert_close` for a table of the default tolerance for each dtype. + + Returns: + (Tuple[float, float]): Loosest tolerances of all input dtypes. + """ + dtypes = [] + for input in inputs: + if isinstance(input, torch.Tensor): + dtypes.append(input.dtype) + elif isinstance(input, torch.dtype): + dtypes.append(input) + else: + raise TypeError( + f"Expected a torch.Tensor or a torch.dtype, but got {type(input)} instead." + ) + dtype_precisions = dtype_precisions or _DTYPE_PRECISIONS + rtols, atols = zip( + *[dtype_precisions.get(dtype, (0.0, 0.0)) for dtype in dtypes], strict=True + ) + return max(rtols), max(atols) + + +def get_tolerances( + *inputs: Union[torch.Tensor, torch.dtype], + rtol: Optional[float], + atol: Optional[float], + id: tuple[Any, ...] = (), +) -> tuple[float, float]: + """Gets absolute and relative to be used for numeric comparisons. + + If both ``rtol`` and ``atol`` are specified, this is a no-op. If both are not specified, the return value of + :func:`default_tolerances` is used. + + Raises: + ErrorMeta: With :class:`ValueError`, if only ``rtol`` or ``atol`` is specified. + + Returns: + (Tuple[float, float]): Valid absolute and relative tolerances. + """ + if (rtol is None) ^ (atol is None): + # We require both tolerance to be omitted or specified, because specifying only one might lead to surprising + # results. Imagine setting atol=0.0 and the tensors still match because rtol>0.0. + raise ErrorMeta( + ValueError, + f"Both 'rtol' and 'atol' must be either specified or omitted, " + f"but got no {'rtol' if rtol is None else 'atol'}.", + id=id, + ) + elif rtol is not None and atol is not None: + return rtol, atol + else: + return default_tolerances(*inputs) + + +def _make_bitwise_mismatch_msg( + *, + default_identifier: str, + identifier: Optional[Union[str, Callable[[str], str]]] = None, + extra: Optional[str] = None, + first_mismatch_idx: Optional[tuple[int, ...]] = None, +): + """Makes a mismatch error message for bitwise values. + + Args: + default_identifier (str): Default description of the compared values, e.g. "Tensor-likes". + identifier (Optional[Union[str, Callable[[str], str]]]): Optional identifier that overrides + ``default_identifier``. Can be passed as callable in which case it will be called with + ``default_identifier`` to create the description at runtime. + extra (Optional[str]): Extra information to be placed after the message header and the mismatch statistics. + first_mismatch_idx (Optional[tuple[int, ...]]): the index of the first mismatch, for each dimension. + """ + if identifier is None: + identifier = default_identifier + elif callable(identifier): + identifier = identifier(default_identifier) + + msg = f"{identifier} are not 'equal'!\n\n" + + if extra: + msg += f"{extra.strip()}\n" + if first_mismatch_idx is not None: + msg += f"The first mismatched element is at index {first_mismatch_idx}.\n" + return msg.strip() + + +def _make_mismatch_msg( + *, + default_identifier: str, + identifier: Optional[Union[str, Callable[[str], str]]] = None, + extra: Optional[str] = None, + abs_diff: float, + abs_diff_idx: Optional[Union[int, tuple[int, ...]]] = None, + atol: float, + rel_diff: float, + rel_diff_idx: Optional[Union[int, tuple[int, ...]]] = None, + rtol: float, +) -> str: + """Makes a mismatch error message for numeric values. + + Args: + default_identifier (str): Default description of the compared values, e.g. "Tensor-likes". + identifier (Optional[Union[str, Callable[[str], str]]]): Optional identifier that overrides + ``default_identifier``. Can be passed as callable in which case it will be called with + ``default_identifier`` to create the description at runtime. + extra (Optional[str]): Extra information to be placed after the message header and the mismatch statistics. + abs_diff (float): Absolute difference. + abs_diff_idx (Optional[Union[int, Tuple[int, ...]]]): Optional index of the absolute difference. + atol (float): Allowed absolute tolerance. Will only be added to mismatch statistics if it or ``rtol`` are + ``> 0``. + rel_diff (float): Relative difference. + rel_diff_idx (Optional[Union[int, Tuple[int, ...]]]): Optional index of the relative difference. + rtol (float): Allowed relative tolerance. Will only be added to mismatch statistics if it or ``atol`` are + ``> 0``. + """ + equality = rtol == 0 and atol == 0 + + def make_diff_msg( + *, + type: str, + diff: float, + idx: Optional[Union[int, tuple[int, ...]]], + tol: float, + ) -> str: + if idx is None: + msg = f"{type.title()} difference: {diff}" + else: + msg = f"Greatest {type} difference: {diff} at index {idx}" + if not equality: + msg += f" (up to {tol} allowed)" + return msg + "\n" + + if identifier is None: + identifier = default_identifier + elif callable(identifier): + identifier = identifier(default_identifier) + + msg = f"{identifier} are not {'equal' if equality else 'close'}!\n\n" + + if extra: + msg += f"{extra.strip()}\n" + + msg += make_diff_msg(type="absolute", diff=abs_diff, idx=abs_diff_idx, tol=atol) + msg += make_diff_msg(type="relative", diff=rel_diff, idx=rel_diff_idx, tol=rtol) + + return msg.strip() + + +def make_scalar_mismatch_msg( + actual: Union[bool, int, float, complex], + expected: Union[bool, int, float, complex], + *, + rtol: float, + atol: float, + identifier: Optional[Union[str, Callable[[str], str]]] = None, +) -> str: + """Makes a mismatch error message for scalars. + + Args: + actual (Union[bool, int, float, complex]): Actual scalar. + expected (Union[bool, int, float, complex]): Expected scalar. + rtol (float): Relative tolerance. + atol (float): Absolute tolerance. + identifier (Optional[Union[str, Callable[[str], str]]]): Optional description for the scalars. Can be passed + as callable in which case it will be called by the default value to create the description at runtime. + Defaults to "Scalars". + """ + abs_diff = abs(actual - expected) + # pyrefly: ignore [bad-argument-type] + rel_diff = float("inf") if expected == 0 else abs_diff / abs(expected) + return _make_mismatch_msg( + default_identifier="Scalars", + identifier=identifier, + extra=f"Expected {expected} but got {actual}.", + abs_diff=abs_diff, + atol=atol, + rel_diff=rel_diff, + rtol=rtol, + ) + + +def make_tensor_mismatch_msg( + actual: torch.Tensor, + expected: torch.Tensor, + matches: torch.Tensor, + *, + rtol: float, + atol: float, + identifier: Optional[Union[str, Callable[[str], str]]] = None, +): + """Makes a mismatch error message for tensors. + + Args: + actual (torch.Tensor): Actual tensor. + expected (torch.Tensor): Expected tensor. + matches (torch.Tensor): Boolean mask of the same shape as ``actual`` and ``expected`` that indicates the + location of matches. + rtol (float): Relative tolerance. + atol (float): Absolute tolerance. + identifier (Optional[Union[str, Callable[[str], str]]]): Optional description for the tensors. Can be passed + as callable in which case it will be called by the default value to create the description at runtime. + Defaults to "Tensor-likes". + """ + + def unravel_flat_index(flat_index: int) -> tuple[int, ...]: + if not matches.shape: + return () + + inverse_index = [] + for size in matches.shape[::-1]: + div, mod = divmod(flat_index, size) + flat_index = div + inverse_index.append(mod) + + return tuple(inverse_index[::-1]) + + number_of_elements = matches.numel() + total_mismatches = number_of_elements - int(torch.sum(matches)) + extra = ( + f"Mismatched elements: {total_mismatches} / {number_of_elements} " + f"({total_mismatches / number_of_elements:.1%})" + ) + if actual.dtype.is_floating_point and actual.dtype.itemsize == 1: + # skip checking for max_abs_diff and max_rel_diff for float8-like values + first_mismatch_idx = tuple(torch.nonzero(~matches, as_tuple=False)[0].tolist()) + return _make_bitwise_mismatch_msg( + default_identifier="Tensor-likes", + identifier=identifier, + extra=extra, + first_mismatch_idx=first_mismatch_idx, + ) + + actual_flat = actual.flatten() + expected_flat = expected.flatten() + matches_flat = matches.flatten() + + if not actual.dtype.is_floating_point and not actual.dtype.is_complex: + # TODO: Instead of always upcasting to int64, it would be sufficient to cast to the next higher dtype to avoid + # overflow + actual_flat = actual_flat.to(torch.int64) + expected_flat = expected_flat.to(torch.int64) + + abs_diff = torch.abs(actual_flat - expected_flat) + # Ensure that only mismatches are used for the max_abs_diff computation + abs_diff[matches_flat] = 0 + max_abs_diff, max_abs_diff_flat_idx = torch.max(abs_diff, 0) + + rel_diff = abs_diff / torch.abs(expected_flat) + # Ensure that only mismatches are used for the max_rel_diff computation + rel_diff[matches_flat] = 0 + max_rel_diff, max_rel_diff_flat_idx = torch.max(rel_diff, 0) + return _make_mismatch_msg( + default_identifier="Tensor-likes", + identifier=identifier, + extra=extra, + abs_diff=max_abs_diff.item(), + abs_diff_idx=unravel_flat_index(int(max_abs_diff_flat_idx)), + atol=atol, + rel_diff=max_rel_diff.item(), + rel_diff_idx=unravel_flat_index(int(max_rel_diff_flat_idx)), + rtol=rtol, + ) + + +class UnsupportedInputs(Exception): # noqa: B903 + """Exception to be raised during the construction of a :class:`Pair` in case it doesn't support the inputs.""" + + +class Pair(abc.ABC): + """ABC for all comparison pairs to be used in conjunction with :func:`assert_equal`. + + Each subclass needs to overwrite :meth:`Pair.compare` that performs the actual comparison. + + Each pair receives **all** options, so select the ones applicable for the subclass and forward the rest to the + super class. Raising an :class:`UnsupportedInputs` during constructions indicates that the pair is not able to + handle the inputs and the next pair type will be tried. + + All other errors should be raised as :class:`ErrorMeta`. After the instantiation, :meth:`Pair._make_error_meta` can + be used to automatically handle overwriting the message with a user supplied one and id handling. + """ + + def __init__( + self, + actual: Any, + expected: Any, + *, + id: tuple[Any, ...] = (), + **unknown_parameters: Any, + ) -> None: + self.actual = actual + self.expected = expected + self.id = id + self._unknown_parameters = unknown_parameters + + @staticmethod + def _inputs_not_supported() -> NoReturn: + raise UnsupportedInputs + + @staticmethod + def _check_inputs_isinstance(*inputs: Any, cls: Union[type, tuple[type, ...]]): + """Checks if all inputs are instances of a given class and raise :class:`UnsupportedInputs` otherwise.""" + if not all(isinstance(input, cls) for input in inputs): + Pair._inputs_not_supported() + + def _fail( + self, type: type[Exception], msg: str, *, id: tuple[Any, ...] = () + ) -> NoReturn: + """Raises an :class:`ErrorMeta` from a given exception type and message and the stored id. + + .. warning:: + + If you use this before the ``super().__init__(...)`` call in the constructor, you have to pass the ``id`` + explicitly. + """ + raise ErrorMeta(type, msg, id=self.id if not id and hasattr(self, "id") else id) + + @abc.abstractmethod + def compare(self) -> None: + """Compares the inputs and raises an :class`ErrorMeta` in case they mismatch.""" + + def extra_repr(self) -> Sequence[Union[str, tuple[str, Any]]]: + """Returns extra information that will be included in the representation. + + Should be overwritten by all subclasses that use additional options. The representation of the object will only + be surfaced in case we encounter an unexpected error and thus should help debug the issue. Can be a sequence of + key-value-pairs or attribute names. + """ + return [] + + def __repr__(self) -> str: + head = f"{type(self).__name__}(" + tail = ")" + body = [ + f" {name}={value!s}," + for name, value in [ + ("id", self.id), + ("actual", self.actual), + ("expected", self.expected), + *[ + (extra, getattr(self, extra)) if isinstance(extra, str) else extra + for extra in self.extra_repr() + ], + ] + ] + return "\n".join((head, *body, *tail)) + + +class ObjectPair(Pair): + """Pair for any type of inputs that will be compared with the `==` operator. + + .. note:: + + Since this will instantiate for any kind of inputs, it should only be used as fallback after all other pairs + couldn't handle the inputs. + + """ + + def compare(self) -> None: + try: + equal = self.actual == self.expected + except Exception as error: + # We are not using `self._raise_error_meta` here since we need the exception chaining + raise ErrorMeta( + ValueError, + f"{self.actual} == {self.expected} failed with:\n{error}.", + id=self.id, + ) from error + + if not equal: + self._fail(AssertionError, f"{self.actual} != {self.expected}") + + +class NonePair(Pair): + """Pair for ``None`` inputs.""" + + def __init__(self, actual: Any, expected: Any, **other_parameters: Any) -> None: + if not (actual is None or expected is None): + self._inputs_not_supported() + + super().__init__(actual, expected, **other_parameters) + + def compare(self) -> None: + if not (self.actual is None and self.expected is None): + self._fail( + AssertionError, f"None mismatch: {self.actual} is not {self.expected}" + ) + + +class BooleanPair(Pair): + """Pair for :class:`bool` inputs. + + .. note:: + + If ``numpy`` is available, also handles :class:`numpy.bool_` inputs. + + """ + + def __init__( + self, + actual: Any, + expected: Any, + *, + id: tuple[Any, ...], + **other_parameters: Any, + ) -> None: + actual, expected = self._process_inputs(actual, expected, id=id) + super().__init__(actual, expected, **other_parameters) + + @property + def _supported_types(self) -> tuple[type, ...]: + cls: list[type] = [bool] + if HAS_NUMPY: + # pyrefly: ignore [missing-attribute] + cls.append(np.bool_) + return tuple(cls) + + def _process_inputs( + self, actual: Any, expected: Any, *, id: tuple[Any, ...] + ) -> tuple[bool, bool]: + self._check_inputs_isinstance(actual, expected, cls=self._supported_types) + actual, expected = ( + self._to_bool(bool_like, id=id) for bool_like in (actual, expected) + ) + return actual, expected + + def _to_bool(self, bool_like: Any, *, id: tuple[Any, ...]) -> bool: + if isinstance(bool_like, bool): + return bool_like + # pyrefly: ignore [missing-attribute] + elif isinstance(bool_like, np.bool_): + return bool_like.item() + else: + raise ErrorMeta( + TypeError, f"Unknown boolean type {type(bool_like)}.", id=id + ) + + def compare(self) -> None: + if self.actual is not self.expected: + self._fail( + AssertionError, + f"Booleans mismatch: {self.actual} is not {self.expected}", + ) + + +class NumberPair(Pair): + """Pair for Python number (:class:`int`, :class:`float`, and :class:`complex`) inputs. + + .. note:: + + If ``numpy`` is available, also handles :class:`numpy.number` inputs. + + Kwargs: + rtol (Optional[float]): Relative tolerance. If specified ``atol`` must also be specified. If omitted, default + values based on the type are selected with the below table. + atol (Optional[float]): Absolute tolerance. If specified ``rtol`` must also be specified. If omitted, default + values based on the type are selected with the below table. + equal_nan (bool): If ``True``, two ``NaN`` values are considered equal. Defaults to ``False``. + check_dtype (bool): If ``True``, the type of the inputs will be checked for equality. Defaults to ``False``. + + The following table displays correspondence between Python number type and the ``torch.dtype``'s. See + :func:`assert_close` for the corresponding tolerances. + + +------------------+-------------------------------+ + | ``type`` | corresponding ``torch.dtype`` | + +==================+===============================+ + | :class:`int` | :attr:`~torch.int64` | + +------------------+-------------------------------+ + | :class:`float` | :attr:`~torch.float64` | + +------------------+-------------------------------+ + | :class:`complex` | :attr:`~torch.complex64` | + +------------------+-------------------------------+ + """ + + _TYPE_TO_DTYPE = { + int: torch.int64, + float: torch.float64, + complex: torch.complex128, + } + _NUMBER_TYPES = tuple(_TYPE_TO_DTYPE.keys()) + + def __init__( + self, + actual: Any, + expected: Any, + *, + id: tuple[Any, ...] = (), + rtol: Optional[float] = None, + atol: Optional[float] = None, + equal_nan: bool = False, + check_dtype: bool = False, + **other_parameters: Any, + ) -> None: + actual, expected = self._process_inputs(actual, expected, id=id) + super().__init__(actual, expected, id=id, **other_parameters) + + self.rtol, self.atol = get_tolerances( + *[self._TYPE_TO_DTYPE[type(input)] for input in (actual, expected)], + rtol=rtol, + atol=atol, + id=id, + ) + self.equal_nan = equal_nan + self.check_dtype = check_dtype + + @property + def _supported_types(self) -> tuple[type, ...]: + cls = list(self._NUMBER_TYPES) + if HAS_NUMPY: + # pyrefly: ignore [missing-attribute] + cls.append(np.number) + return tuple(cls) + + def _process_inputs( + self, actual: Any, expected: Any, *, id: tuple[Any, ...] + ) -> tuple[Union[int, float, complex], Union[int, float, complex]]: + self._check_inputs_isinstance(actual, expected, cls=self._supported_types) + actual, expected = ( + self._to_number(number_like, id=id) for number_like in (actual, expected) + ) + return actual, expected + + def _to_number( + self, number_like: Any, *, id: tuple[Any, ...] + ) -> Union[int, float, complex]: + # pyrefly: ignore [missing-attribute] + if HAS_NUMPY and isinstance(number_like, np.number): + return number_like.item() + elif isinstance(number_like, self._NUMBER_TYPES): + return number_like # type: ignore[return-value] + else: + raise ErrorMeta( + TypeError, f"Unknown number type {type(number_like)}.", id=id + ) + + def compare(self) -> None: + if self.check_dtype and type(self.actual) is not type(self.expected): + self._fail( + AssertionError, + f"The (d)types do not match: {type(self.actual)} != {type(self.expected)}.", + ) + + if self.actual == self.expected: + return + + if self.equal_nan and cmath.isnan(self.actual) and cmath.isnan(self.expected): + return + + abs_diff = abs(self.actual - self.expected) + tolerance = self.atol + self.rtol * abs(self.expected) + + if cmath.isfinite(abs_diff) and abs_diff <= tolerance: + return + + self._fail( + AssertionError, + make_scalar_mismatch_msg( + self.actual, self.expected, rtol=self.rtol, atol=self.atol + ), + ) + + def extra_repr(self) -> Sequence[str]: + return ( + "rtol", + "atol", + "equal_nan", + "check_dtype", + ) + + +class TensorLikePair(Pair): + """Pair for :class:`torch.Tensor`-like inputs. + + Kwargs: + allow_subclasses (bool): + rtol (Optional[float]): Relative tolerance. If specified ``atol`` must also be specified. If omitted, default + values based on the type are selected. See :func:assert_close: for details. + atol (Optional[float]): Absolute tolerance. If specified ``rtol`` must also be specified. If omitted, default + values based on the type are selected. See :func:assert_close: for details. + equal_nan (bool): If ``True``, two ``NaN`` values are considered equal. Defaults to ``False``. + check_device (bool): If ``True`` (default), asserts that corresponding tensors are on the same + :attr:`~torch.Tensor.device`. If this check is disabled, tensors on different + :attr:`~torch.Tensor.device`'s are moved to the CPU before being compared. + check_dtype (bool): If ``True`` (default), asserts that corresponding tensors have the same ``dtype``. If this + check is disabled, tensors with different ``dtype``'s are promoted to a common ``dtype`` (according to + :func:`torch.promote_types`) before being compared. + check_layout (bool): If ``True`` (default), asserts that corresponding tensors have the same ``layout``. If this + check is disabled, tensors with different ``layout``'s are converted to strided tensors before being + compared. + check_stride (bool): If ``True`` and corresponding tensors are strided, asserts that they have the same stride. + """ + + def __init__( + self, + actual: Any, + expected: Any, + *, + id: tuple[Any, ...] = (), + allow_subclasses: bool = True, + rtol: Optional[float] = None, + atol: Optional[float] = None, + equal_nan: bool = False, + check_device: bool = True, + check_dtype: bool = True, + check_layout: bool = True, + check_stride: bool = False, + **other_parameters: Any, + ): + actual, expected = self._process_inputs( + actual, expected, id=id, allow_subclasses=allow_subclasses + ) + super().__init__(actual, expected, id=id, **other_parameters) + + self.rtol, self.atol = get_tolerances( + actual, expected, rtol=rtol, atol=atol, id=self.id + ) + self.equal_nan = equal_nan + self.check_device = check_device + self.check_dtype = check_dtype + self.check_layout = check_layout + self.check_stride = check_stride + + def _process_inputs( + self, actual: Any, expected: Any, *, id: tuple[Any, ...], allow_subclasses: bool + ) -> tuple[torch.Tensor, torch.Tensor]: + directly_related = isinstance(actual, type(expected)) or isinstance( + expected, type(actual) + ) + if not directly_related: + self._inputs_not_supported() + + if not allow_subclasses and type(actual) is not type(expected): + self._inputs_not_supported() + + actual, expected = (self._to_tensor(input) for input in (actual, expected)) + for tensor in (actual, expected): + self._check_supported(tensor, id=id) + return actual, expected + + def _to_tensor(self, tensor_like: Any) -> torch.Tensor: + if isinstance(tensor_like, torch.Tensor): + return tensor_like + + try: + return torch.as_tensor(tensor_like) + except Exception: + self._inputs_not_supported() + + def _check_supported(self, tensor: torch.Tensor, *, id: tuple[Any, ...]) -> None: + if tensor.layout not in { + torch.strided, + torch.jagged, + torch.sparse_coo, + torch.sparse_csr, + torch.sparse_csc, + torch.sparse_bsr, + torch.sparse_bsc, + }: + raise ErrorMeta( + ValueError, f"Unsupported tensor layout {tensor.layout}", id=id + ) + + def compare(self) -> None: + actual, expected = self.actual, self.expected + + self._compare_attributes(actual, expected) + if any(input.device.type == "meta" for input in (actual, expected)): + return + + actual, expected = self._equalize_attributes(actual, expected) + self._compare_values(actual, expected) + + def _compare_attributes( + self, + actual: torch.Tensor, + expected: torch.Tensor, + ) -> None: + """Checks if the attributes of two tensors match. + + Always checks + + - the :attr:`~torch.Tensor.shape`, + - whether both inputs are quantized or not, + - and if they use the same quantization scheme. + + Checks for + + - :attr:`~torch.Tensor.layout`, + - :meth:`~torch.Tensor.stride`, + - :attr:`~torch.Tensor.device`, and + - :attr:`~torch.Tensor.dtype` + + are optional and can be disabled through the corresponding ``check_*`` flag during construction of the pair. + """ + + def raise_mismatch_error( + attribute_name: str, actual_value: Any, expected_value: Any + ) -> NoReturn: + self._fail( + AssertionError, + f"The values for attribute '{attribute_name}' do not match: {actual_value} != {expected_value}.", + ) + + if actual.shape != expected.shape: + raise_mismatch_error("shape", actual.shape, expected.shape) + + if actual.is_quantized != expected.is_quantized: + raise_mismatch_error( + "is_quantized", actual.is_quantized, expected.is_quantized + ) + elif actual.is_quantized and actual.qscheme() != expected.qscheme(): + raise_mismatch_error("qscheme()", actual.qscheme(), expected.qscheme()) + + if actual.layout != expected.layout: + if self.check_layout: + raise_mismatch_error("layout", actual.layout, expected.layout) + elif ( + actual.layout == torch.strided + and self.check_stride + and actual.stride() != expected.stride() + ): + raise_mismatch_error("stride()", actual.stride(), expected.stride()) + + if self.check_device and actual.device != expected.device: + raise_mismatch_error("device", actual.device, expected.device) + + if self.check_dtype and actual.dtype != expected.dtype: + raise_mismatch_error("dtype", actual.dtype, expected.dtype) + + def _equalize_attributes( + self, actual: torch.Tensor, expected: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + """Equalizes some attributes of two tensors for value comparison. + + If ``actual`` and ``expected`` are ... + + - ... not on the same :attr:`~torch.Tensor.device`, they are moved CPU memory. + - ... not of the same ``dtype``, they are promoted to a common ``dtype`` (according to + :func:`torch.promote_types`). + - ... not of the same ``layout``, they are converted to strided tensors. + + Args: + actual (Tensor): Actual tensor. + expected (Tensor): Expected tensor. + + Returns: + (Tuple[Tensor, Tensor]): Equalized tensors. + """ + # The comparison logic uses operators currently not supported by the MPS backends. + # See https://github.com/pytorch/pytorch/issues/77144 for details. + # TODO: Remove this conversion as soon as all operations are supported natively by the MPS backend + if actual.is_mps or expected.is_mps: # type: ignore[attr-defined] + actual = actual.cpu() + expected = expected.cpu() + + if actual.device != expected.device: + actual = actual.cpu() + expected = expected.cpu() + + if actual.dtype != expected.dtype: + actual_dtype = actual.dtype + expected_dtype = expected.dtype + # For uint64, this is not sound in general, which is why promote_types doesn't + # allow it, but for easy testing, we're unlikely to get confused + # by large uint64 overflowing into negative int64 + if actual_dtype in [torch.uint64, torch.uint32, torch.uint16]: + actual_dtype = torch.int64 + if expected_dtype in [torch.uint64, torch.uint32, torch.uint16]: + expected_dtype = torch.int64 + dtype = torch.promote_types(actual_dtype, expected_dtype) + actual = actual.to(dtype) + expected = expected.to(dtype) + + if actual.layout != expected.layout: + # These checks are needed, since Tensor.to_dense() fails on tensors that are already strided + actual = actual.to_dense() if actual.layout != torch.strided else actual + expected = ( + expected.to_dense() if expected.layout != torch.strided else expected + ) + + return actual, expected + + def _compare_values(self, actual: torch.Tensor, expected: torch.Tensor) -> None: + if actual.is_quantized: + compare_fn = self._compare_quantized_values + elif actual.is_sparse: + compare_fn = self._compare_sparse_coo_values + elif actual.layout in { + torch.sparse_csr, + torch.sparse_csc, + torch.sparse_bsr, + torch.sparse_bsc, + }: + compare_fn = self._compare_sparse_compressed_values + elif actual.layout == torch.jagged: + actual, expected = actual.values(), expected.values() + compare_fn = self._compare_regular_values_close + elif actual.dtype.is_floating_point and actual.dtype.itemsize == 1: + + def bitwise_comp( + actual: torch.Tensor, + expected: torch.Tensor, + *, + rtol: float, + atol: float, + equal_nan: bool, + identifier: Optional[Union[str, Callable[[str], str]]] = None, + ) -> None: + if rtol != 0.0 or atol != 0.0: + raise ErrorMeta( + AssertionError, + f"Rtol={rtol} and atol={atol} are not supported for bitwise comparison of low" + " dimensional floats. Please use rtol=0.0 and atol=0.0.", + ) + + return self._compare_regular_values_close( + actual, + expected, + rtol=rtol, + atol=atol, + equal_nan=equal_nan, + identifier=identifier, + ) + + compare_fn = bitwise_comp + else: + compare_fn = self._compare_regular_values_close + + compare_fn( + actual, expected, rtol=self.rtol, atol=self.atol, equal_nan=self.equal_nan + ) + + def _compare_quantized_values( + self, + actual: torch.Tensor, + expected: torch.Tensor, + *, + rtol: float, + atol: float, + equal_nan: bool, + ) -> None: + """Compares quantized tensors by comparing the :meth:`~torch.Tensor.dequantize`'d variants for closeness. + + .. note:: + + A detailed discussion about why only the dequantized variant is checked for closeness rather than checking + the individual quantization parameters for closeness and the integer representation for equality can be + found in https://github.com/pytorch/pytorch/issues/68548. + """ + return self._compare_regular_values_close( + actual.dequantize(), + expected.dequantize(), + rtol=rtol, + atol=atol, + equal_nan=equal_nan, + identifier=lambda default_identifier: f"Quantized {default_identifier.lower()}", + ) + + def _compare_sparse_coo_values( + self, + actual: torch.Tensor, + expected: torch.Tensor, + *, + rtol: float, + atol: float, + equal_nan: bool, + ) -> None: + """Compares sparse COO tensors by comparing + + - the number of sparse dimensions, + - the number of non-zero elements (nnz) for equality, + - the indices for equality, and + - the values for closeness. + """ + if actual.sparse_dim() != expected.sparse_dim(): + self._fail( + AssertionError, + ( + f"The number of sparse dimensions in sparse COO tensors does not match: " + f"{actual.sparse_dim()} != {expected.sparse_dim()}" + ), + ) + + if actual._nnz() != expected._nnz(): + self._fail( + AssertionError, + ( + f"The number of specified values in sparse COO tensors does not match: " + f"{actual._nnz()} != {expected._nnz()}" + ), + ) + + self._compare_regular_values_equal( + actual._indices(), + expected._indices(), + identifier="Sparse COO indices", + ) + self._compare_regular_values_close( + actual._values(), + expected._values(), + rtol=rtol, + atol=atol, + equal_nan=equal_nan, + identifier="Sparse COO values", + ) + + def _compare_sparse_compressed_values( + self, + actual: torch.Tensor, + expected: torch.Tensor, + *, + rtol: float, + atol: float, + equal_nan: bool, + ) -> None: + """Compares sparse compressed tensors by comparing + + - the number of non-zero elements (nnz) for equality, + - the plain indices for equality, + - the compressed indices for equality, and + - the values for closeness. + """ + format_name, compressed_indices_method, plain_indices_method = { + torch.sparse_csr: ( + "CSR", + torch.Tensor.crow_indices, + torch.Tensor.col_indices, + ), + torch.sparse_csc: ( + "CSC", + torch.Tensor.ccol_indices, + torch.Tensor.row_indices, + ), + torch.sparse_bsr: ( + "BSR", + torch.Tensor.crow_indices, + torch.Tensor.col_indices, + ), + torch.sparse_bsc: ( + "BSC", + torch.Tensor.ccol_indices, + torch.Tensor.row_indices, + ), + }[actual.layout] + + if actual._nnz() != expected._nnz(): + self._fail( + AssertionError, + ( + f"The number of specified values in sparse {format_name} tensors does not match: " + f"{actual._nnz()} != {expected._nnz()}" + ), + ) + + # Compressed and plain indices in the CSR / CSC / BSR / BSC sparse formats can be `torch.int32` _or_ + # `torch.int64`. While the same dtype is enforced for the compressed and plain indices of a single tensor, it + # can be different between two tensors. Thus, we need to convert them to the same dtype, or the comparison will + # fail. + actual_compressed_indices = compressed_indices_method(actual) + expected_compressed_indices = compressed_indices_method(expected) + indices_dtype = torch.promote_types( + actual_compressed_indices.dtype, expected_compressed_indices.dtype + ) + + self._compare_regular_values_equal( + actual_compressed_indices.to(indices_dtype), + expected_compressed_indices.to(indices_dtype), + identifier=f"Sparse {format_name} {compressed_indices_method.__name__}", + ) + self._compare_regular_values_equal( + plain_indices_method(actual).to(indices_dtype), + plain_indices_method(expected).to(indices_dtype), + identifier=f"Sparse {format_name} {plain_indices_method.__name__}", + ) + self._compare_regular_values_close( + actual.values(), + expected.values(), + rtol=rtol, + atol=atol, + equal_nan=equal_nan, + identifier=f"Sparse {format_name} values", + ) + + def _compare_regular_values_equal( + self, + actual: torch.Tensor, + expected: torch.Tensor, + *, + equal_nan: bool = False, + identifier: Optional[Union[str, Callable[[str], str]]] = None, + ) -> None: + """Checks if the values of two tensors are equal.""" + self._compare_regular_values_close( + actual, expected, rtol=0, atol=0, equal_nan=equal_nan, identifier=identifier + ) + + def _compare_regular_values_close( + self, + actual: torch.Tensor, + expected: torch.Tensor, + *, + rtol: float, + atol: float, + equal_nan: bool, + identifier: Optional[Union[str, Callable[[str], str]]] = None, + ) -> None: + """Checks if the values of two tensors are close up to a desired tolerance.""" + matches = torch.isclose( + actual, expected, rtol=rtol, atol=atol, equal_nan=equal_nan + ) + if torch.all(matches): + return + + if actual.shape == torch.Size([]): + msg = make_scalar_mismatch_msg( + actual.item(), + expected.item(), + rtol=rtol, + atol=atol, + identifier=identifier, + ) + else: + msg = make_tensor_mismatch_msg( + actual, expected, matches, rtol=rtol, atol=atol, identifier=identifier + ) + self._fail(AssertionError, msg) + + def extra_repr(self) -> Sequence[str]: + return ( + "rtol", + "atol", + "equal_nan", + "check_device", + "check_dtype", + "check_layout", + "check_stride", + ) + + +def originate_pairs( + actual: Any, + expected: Any, + *, + pair_types: Sequence[type[Pair]], + sequence_types: tuple[type, ...] = (collections.abc.Sequence,), + mapping_types: tuple[type, ...] = (collections.abc.Mapping,), + id: tuple[Any, ...] = (), + **options: Any, + # pyrefly: ignore [bad-return] +) -> list[Pair]: + """Originates pairs from the individual inputs. + + ``actual`` and ``expected`` can be possibly nested :class:`~collections.abc.Sequence`'s or + :class:`~collections.abc.Mapping`'s. In this case the pairs are originated by recursing through them. + + Args: + actual (Any): Actual input. + expected (Any): Expected input. + pair_types (Sequence[Type[Pair]]): Sequence of pair types that will be tried to construct with the inputs. + First successful pair will be used. + sequence_types (Tuple[Type, ...]): Optional types treated as sequences that will be checked elementwise. + mapping_types (Tuple[Type, ...]): Optional types treated as mappings that will be checked elementwise. + id (Tuple[Any, ...]): Optional id of a pair that will be included in an error message. + **options (Any): Options passed to each pair during construction. + + Raises: + ErrorMeta: With :class`AssertionError`, if the inputs are :class:`~collections.abc.Sequence`'s, but their + length does not match. + ErrorMeta: With :class`AssertionError`, if the inputs are :class:`~collections.abc.Mapping`'s, but their set of + keys do not match. + ErrorMeta: With :class`TypeError`, if no pair is able to handle the inputs. + ErrorMeta: With any expected exception that happens during the construction of a pair. + + Returns: + (List[Pair]): Originated pairs. + """ + # We explicitly exclude str's here since they are self-referential and would cause an infinite recursion loop: + # "a" == "a"[0][0]... + if ( + isinstance(actual, sequence_types) + and not isinstance(actual, str) + and isinstance(expected, sequence_types) + and not isinstance(expected, str) + ): + actual_len = len(actual) # type: ignore[arg-type] + expected_len = len(expected) # type: ignore[arg-type] + if actual_len != expected_len: + raise ErrorMeta( + AssertionError, + f"The length of the sequences mismatch: {actual_len} != {expected_len}", + id=id, + ) + + pairs = [] + for idx in range(actual_len): + pairs.extend( + originate_pairs( + actual[idx], # type: ignore[index] + expected[idx], # type: ignore[index] + pair_types=pair_types, + sequence_types=sequence_types, + mapping_types=mapping_types, + id=(*id, idx), + **options, + ) + ) + return pairs + + elif isinstance(actual, mapping_types) and isinstance(expected, mapping_types): + actual_keys = set(actual.keys()) # type: ignore[attr-defined] + expected_keys = set(expected.keys()) # type: ignore[attr-defined] + if actual_keys != expected_keys: + missing_keys = expected_keys - actual_keys + additional_keys = actual_keys - expected_keys + raise ErrorMeta( + AssertionError, + ( + f"The keys of the mappings do not match:\n" + f"Missing keys in the actual mapping: {sorted(missing_keys)}\n" + f"Additional keys in the actual mapping: {sorted(additional_keys)}" + ), + id=id, + ) + + keys: Collection = actual_keys + # Since the origination aborts after the first failure, we try to be deterministic + with contextlib.suppress(Exception): + keys = sorted(keys) + + pairs = [] + for key in keys: + pairs.extend( + originate_pairs( + actual[key], # type: ignore[index] + expected[key], # type: ignore[index] + pair_types=pair_types, + sequence_types=sequence_types, + mapping_types=mapping_types, + id=(*id, key), + **options, + ) + ) + return pairs + + else: + for pair_type in pair_types: + try: + # pyrefly: ignore [bad-instantiation] + return [pair_type(actual, expected, id=id, **options)] + # Raising an `UnsupportedInputs` during origination indicates that the pair type is not able to handle the + # inputs. Thus, we try the next pair type. + except UnsupportedInputs: + continue + # Raising an `ErrorMeta` during origination is the orderly way to abort and so we simply re-raise it. This + # is only in a separate branch, because the one below would also except it. + except ErrorMeta: + raise + # Raising any other exception during origination is unexpected and will give some extra information about + # what happened. If applicable, the exception should be expected in the future. + except Exception as error: + raise RuntimeError( + f"Originating a {pair_type.__name__}() at item {''.join(str([item]) for item in id)} with\n\n" + f"{type(actual).__name__}(): {actual}\n\n" + f"and\n\n" + f"{type(expected).__name__}(): {expected}\n\n" + f"resulted in the unexpected exception above. " + f"If you are a user and see this message during normal operation " + "please file an issue at https://github.com/pytorch/pytorch/issues. " + "If you are a developer and working on the comparison functions, " + "please except the previous error and raise an expressive `ErrorMeta` instead." + ) from error + else: + raise ErrorMeta( + TypeError, + f"No comparison pair was able to handle inputs of type {type(actual)} and {type(expected)}.", + id=id, + ) + + +def not_close_error_metas( + actual: Any, + expected: Any, + *, + pair_types: Sequence[type[Pair]] = (ObjectPair,), + sequence_types: tuple[type, ...] = (collections.abc.Sequence,), + mapping_types: tuple[type, ...] = (collections.abc.Mapping,), + **options: Any, +) -> list[ErrorMeta]: + """Asserts that inputs are equal. + + ``actual`` and ``expected`` can be possibly nested :class:`~collections.abc.Sequence`'s or + :class:`~collections.abc.Mapping`'s. In this case the comparison happens elementwise by recursing through them. + + Args: + actual (Any): Actual input. + expected (Any): Expected input. + pair_types (Sequence[Type[Pair]]): Sequence of :class:`Pair` types that will be tried to construct with the + inputs. First successful pair will be used. Defaults to only using :class:`ObjectPair`. + sequence_types (Tuple[Type, ...]): Optional types treated as sequences that will be checked elementwise. + mapping_types (Tuple[Type, ...]): Optional types treated as mappings that will be checked elementwise. + **options (Any): Options passed to each pair during construction. + """ + # Hide this function from `pytest`'s traceback + __tracebackhide__ = True + + try: + pairs = originate_pairs( + actual, + expected, + pair_types=pair_types, + sequence_types=sequence_types, + mapping_types=mapping_types, + **options, + ) + except ErrorMeta as error_meta: + # Explicitly raising from None to hide the internal traceback + raise error_meta.to_error() from None # noqa: RSE102 + + error_metas: list[ErrorMeta] = [] + for pair in pairs: + try: + pair.compare() + except ErrorMeta as error_meta: + error_metas.append(error_meta) + # Raising any exception besides `ErrorMeta` while comparing is unexpected and will give some extra information + # about what happened. If applicable, the exception should be expected in the future. + except Exception as error: + raise RuntimeError( + f"Comparing\n\n" + f"{pair}\n\n" + f"resulted in the unexpected exception above. " + f"If you are a user and see this message during normal operation " + "please file an issue at https://github.com/pytorch/pytorch/issues. " + "If you are a developer and working on the comparison functions, " + "please except the previous error and raise an expressive `ErrorMeta` instead." + ) from error + + # [ErrorMeta Cycles] + # ErrorMeta objects in this list capture + # tracebacks that refer to the frame of this function. + # The local variable `error_metas` refers to the error meta + # objects, creating a reference cycle. Frames in the traceback + # would not get freed until cycle collection, leaking cuda memory in tests. + # We break the cycle by removing the reference to the error_meta objects + # from this frame as it returns. + # pyrefly: ignore [bad-assignment] + error_metas = [error_metas] + # pyrefly: ignore [bad-return] + return error_metas.pop() + + +def assert_close( + actual: Any, + expected: Any, + *, + allow_subclasses: bool = True, + rtol: Optional[float] = None, + atol: Optional[float] = None, + equal_nan: bool = False, + check_device: bool = True, + check_dtype: bool = True, + check_layout: bool = True, + check_stride: bool = False, + msg: Optional[Union[str, Callable[[str], str]]] = None, +): + r"""Asserts that ``actual`` and ``expected`` are close. + + If ``actual`` and ``expected`` are strided, non-quantized, real-valued, and finite, they are considered close if + + .. math:: + + \lvert \text{actual} - \text{expected} \rvert \le \texttt{atol} + \texttt{rtol} \cdot \lvert \text{expected} \rvert + + Non-finite values (``-inf`` and ``inf``) are only considered close if and only if they are equal. ``NaN``'s are + only considered equal to each other if ``equal_nan`` is ``True``. + + In addition, they are only considered close if they have the same + + - :attr:`~torch.Tensor.device` (if ``check_device`` is ``True``), + - ``dtype`` (if ``check_dtype`` is ``True``), + - ``layout`` (if ``check_layout`` is ``True``), and + - stride (if ``check_stride`` is ``True``). + + If either ``actual`` or ``expected`` is a meta tensor, only the attribute checks will be performed. + + If ``actual`` and ``expected`` are sparse (either having COO, CSR, CSC, BSR, or BSC layout), their strided members are + checked individually. Indices, namely ``indices`` for COO, ``crow_indices`` and ``col_indices`` for CSR and BSR, + or ``ccol_indices`` and ``row_indices`` for CSC and BSC layouts, respectively, + are always checked for equality whereas the values are checked for closeness according to the definition above. + + If ``actual`` and ``expected`` are quantized, they are considered close if they have the same + :meth:`~torch.Tensor.qscheme` and the result of :meth:`~torch.Tensor.dequantize` is close according to the + definition above. + + ``actual`` and ``expected`` can be :class:`~torch.Tensor`'s or any tensor-or-scalar-likes from which + :class:`torch.Tensor`'s can be constructed with :func:`torch.as_tensor`. Except for Python scalars the input types + have to be directly related. In addition, ``actual`` and ``expected`` can be :class:`~collections.abc.Sequence`'s + or :class:`~collections.abc.Mapping`'s in which case they are considered close if their structure matches and all + their elements are considered close according to the above definition. + + .. note:: + + Python scalars are an exception to the type relation requirement, because their :func:`type`, i.e. + :class:`int`, :class:`float`, and :class:`complex`, is equivalent to the ``dtype`` of a tensor-like. Thus, + Python scalars of different types can be checked, but require ``check_dtype=False``. + + Args: + actual (Any): Actual input. + expected (Any): Expected input. + allow_subclasses (bool): If ``True`` (default) and except for Python scalars, inputs of directly related types + are allowed. Otherwise type equality is required. + rtol (Optional[float]): Relative tolerance. If specified ``atol`` must also be specified. If omitted, default + values based on the :attr:`~torch.Tensor.dtype` are selected with the below table. + atol (Optional[float]): Absolute tolerance. If specified ``rtol`` must also be specified. If omitted, default + values based on the :attr:`~torch.Tensor.dtype` are selected with the below table. + equal_nan (Union[bool, str]): If ``True``, two ``NaN`` values will be considered equal. + check_device (bool): If ``True`` (default), asserts that corresponding tensors are on the same + :attr:`~torch.Tensor.device`. If this check is disabled, tensors on different + :attr:`~torch.Tensor.device`'s are moved to the CPU before being compared. + check_dtype (bool): If ``True`` (default), asserts that corresponding tensors have the same ``dtype``. If this + check is disabled, tensors with different ``dtype``'s are promoted to a common ``dtype`` (according to + :func:`torch.promote_types`) before being compared. + check_layout (bool): If ``True`` (default), asserts that corresponding tensors have the same ``layout``. If this + check is disabled, tensors with different ``layout``'s are converted to strided tensors before being + compared. + check_stride (bool): If ``True`` and corresponding tensors are strided, asserts that they have the same stride. + msg (Optional[Union[str, Callable[[str], str]]]): Optional error message to use in case a failure occurs during + the comparison. Can also passed as callable in which case it will be called with the generated message and + should return the new message. + + Raises: + ValueError: If no :class:`torch.Tensor` can be constructed from an input. + ValueError: If only ``rtol`` or ``atol`` is specified. + AssertionError: If corresponding inputs are not Python scalars and are not directly related. + AssertionError: If ``allow_subclasses`` is ``False``, but corresponding inputs are not Python scalars and have + different types. + AssertionError: If the inputs are :class:`~collections.abc.Sequence`'s, but their length does not match. + AssertionError: If the inputs are :class:`~collections.abc.Mapping`'s, but their set of keys do not match. + AssertionError: If corresponding tensors do not have the same :attr:`~torch.Tensor.shape`. + AssertionError: If ``check_layout`` is ``True``, but corresponding tensors do not have the same + :attr:`~torch.Tensor.layout`. + AssertionError: If only one of corresponding tensors is quantized. + AssertionError: If corresponding tensors are quantized, but have different :meth:`~torch.Tensor.qscheme`'s. + AssertionError: If ``check_device`` is ``True``, but corresponding tensors are not on the same + :attr:`~torch.Tensor.device`. + AssertionError: If ``check_dtype`` is ``True``, but corresponding tensors do not have the same ``dtype``. + AssertionError: If ``check_stride`` is ``True``, but corresponding strided tensors do not have the same stride. + AssertionError: If the values of corresponding tensors are not close according to the definition above. + + The following table displays the default ``rtol`` and ``atol`` for different ``dtype``'s. In case of mismatching + ``dtype``'s, the maximum of both tolerances is used. + + +---------------------------+------------+----------+ + | ``dtype`` | ``rtol`` | ``atol`` | + +===========================+============+==========+ + | :attr:`~torch.float16` | ``1e-3`` | ``1e-5`` | + +---------------------------+------------+----------+ + | :attr:`~torch.bfloat16` | ``1.6e-2`` | ``1e-5`` | + +---------------------------+------------+----------+ + | :attr:`~torch.float32` | ``1.3e-6`` | ``1e-5`` | + +---------------------------+------------+----------+ + | :attr:`~torch.float64` | ``1e-7`` | ``1e-7`` | + +---------------------------+------------+----------+ + | :attr:`~torch.complex32` | ``1e-3`` | ``1e-5`` | + +---------------------------+------------+----------+ + | :attr:`~torch.complex64` | ``1.3e-6`` | ``1e-5`` | + +---------------------------+------------+----------+ + | :attr:`~torch.complex128` | ``1e-7`` | ``1e-7`` | + +---------------------------+------------+----------+ + | :attr:`~torch.quint8` | ``1.3e-6`` | ``1e-5`` | + +---------------------------+------------+----------+ + | :attr:`~torch.quint2x4` | ``1.3e-6`` | ``1e-5`` | + +---------------------------+------------+----------+ + | :attr:`~torch.quint4x2` | ``1.3e-6`` | ``1e-5`` | + +---------------------------+------------+----------+ + | :attr:`~torch.qint8` | ``1.3e-6`` | ``1e-5`` | + +---------------------------+------------+----------+ + | :attr:`~torch.qint32` | ``1.3e-6`` | ``1e-5`` | + +---------------------------+------------+----------+ + | other | ``0.0`` | ``0.0`` | + +---------------------------+------------+----------+ + + .. note:: + + :func:`~torch.testing.assert_close` is highly configurable with strict default settings. Users are encouraged + to :func:`~functools.partial` it to fit their use case. For example, if an equality check is needed, one might + define an ``assert_equal`` that uses zero tolerances for every ``dtype`` by default: + + >>> import functools + >>> assert_equal = functools.partial(torch.testing.assert_close, rtol=0, atol=0) + >>> assert_equal(1e-9, 1e-10) + Traceback (most recent call last): + ... + AssertionError: Scalars are not equal! + + Expected 1e-10 but got 1e-09. + Absolute difference: 9.000000000000001e-10 + Relative difference: 9.0 + + Examples: + >>> # tensor to tensor comparison + >>> expected = torch.tensor([1e0, 1e-1, 1e-2]) + >>> actual = torch.acos(torch.cos(expected)) + >>> torch.testing.assert_close(actual, expected) + + >>> # scalar to scalar comparison + >>> import math + >>> expected = math.sqrt(2.0) + >>> actual = 2.0 / math.sqrt(2.0) + >>> torch.testing.assert_close(actual, expected) + + >>> # numpy array to numpy array comparison + >>> import numpy as np + >>> expected = np.array([1e0, 1e-1, 1e-2]) + >>> actual = np.arccos(np.cos(expected)) + >>> torch.testing.assert_close(actual, expected) + + >>> # sequence to sequence comparison + >>> import numpy as np + >>> # The types of the sequences do not have to match. They only have to have the same + >>> # length and their elements have to match. + >>> expected = [torch.tensor([1.0]), 2.0, np.array(3.0)] + >>> actual = tuple(expected) + >>> torch.testing.assert_close(actual, expected) + + >>> # mapping to mapping comparison + >>> from collections import OrderedDict + >>> import numpy as np + >>> foo = torch.tensor(1.0) + >>> bar = 2.0 + >>> baz = np.array(3.0) + >>> # The types and a possible ordering of mappings do not have to match. They only + >>> # have to have the same set of keys and their elements have to match. + >>> expected = OrderedDict([("foo", foo), ("bar", bar), ("baz", baz)]) + >>> actual = {"baz": baz, "bar": bar, "foo": foo} + >>> torch.testing.assert_close(actual, expected) + + >>> expected = torch.tensor([1.0, 2.0, 3.0]) + >>> actual = expected.clone() + >>> # By default, directly related instances can be compared + >>> torch.testing.assert_close(torch.nn.Parameter(actual), expected) + >>> # This check can be made more strict with allow_subclasses=False + >>> torch.testing.assert_close( + ... torch.nn.Parameter(actual), expected, allow_subclasses=False + ... ) + Traceback (most recent call last): + ... + TypeError: No comparison pair was able to handle inputs of type + and . + >>> # If the inputs are not directly related, they are never considered close + >>> torch.testing.assert_close(actual.numpy(), expected) + Traceback (most recent call last): + ... + TypeError: No comparison pair was able to handle inputs of type + and . + >>> # Exceptions to these rules are Python scalars. They can be checked regardless of + >>> # their type if check_dtype=False. + >>> torch.testing.assert_close(1.0, 1, check_dtype=False) + + >>> # NaN != NaN by default. + >>> expected = torch.tensor(float("Nan")) + >>> actual = expected.clone() + >>> torch.testing.assert_close(actual, expected) + Traceback (most recent call last): + ... + AssertionError: Scalars are not close! + + Expected nan but got nan. + Absolute difference: nan (up to 1e-05 allowed) + Relative difference: nan (up to 1.3e-06 allowed) + >>> torch.testing.assert_close(actual, expected, equal_nan=True) + + >>> expected = torch.tensor([1.0, 2.0, 3.0]) + >>> actual = torch.tensor([1.0, 4.0, 5.0]) + >>> # The default error message can be overwritten. + >>> torch.testing.assert_close( + ... actual, expected, msg="Argh, the tensors are not close!" + ... ) + Traceback (most recent call last): + ... + AssertionError: Argh, the tensors are not close! + >>> # If msg is a callable, it can be used to augment the generated message with + >>> # extra information + >>> torch.testing.assert_close( + ... actual, expected, msg=lambda msg: f"Header\n\n{msg}\n\nFooter" + ... ) + Traceback (most recent call last): + ... + AssertionError: Header + + Tensor-likes are not close! + + Mismatched elements: 2 / 3 (66.7%) + Greatest absolute difference: 2.0 at index (1,) (up to 1e-05 allowed) + Greatest relative difference: 1.0 at index (1,) (up to 1.3e-06 allowed) + + Footer + """ + # Hide this function from `pytest`'s traceback + __tracebackhide__ = True + + error_metas = not_close_error_metas( + actual, + expected, + pair_types=( + NonePair, + BooleanPair, + NumberPair, + TensorLikePair, + ), + allow_subclasses=allow_subclasses, + rtol=rtol, + atol=atol, + equal_nan=equal_nan, + check_device=check_device, + check_dtype=check_dtype, + check_layout=check_layout, + check_stride=check_stride, + msg=msg, + ) + + if error_metas: + # TODO: compose all metas into one AssertionError + raise error_metas[0].to_error(msg) + + +@deprecated( + "`torch.testing.assert_allclose()` is deprecated since 1.12 and will be removed in a future release. " + "Please use `torch.testing.assert_close()` instead. " + "You can find detailed upgrade instructions in https://github.com/pytorch/pytorch/issues/61844.", + category=FutureWarning, +) +def assert_allclose( + actual: Any, + expected: Any, + rtol: Optional[float] = None, + atol: Optional[float] = None, + equal_nan: bool = True, + msg: str = "", +) -> None: + """ + .. warning:: + + :func:`torch.testing.assert_allclose` is deprecated since ``1.12`` and will be removed in a future release. + Please use :func:`torch.testing.assert_close` instead. You can find detailed upgrade instructions + `here `_. + """ + if not isinstance(actual, torch.Tensor): + actual = torch.tensor(actual) + if not isinstance(expected, torch.Tensor): + expected = torch.tensor(expected, dtype=actual.dtype) + + if rtol is None and atol is None: + rtol, atol = default_tolerances( + actual, + expected, + dtype_precisions={ + torch.float16: (1e-3, 1e-3), + torch.float32: (1e-4, 1e-5), + torch.float64: (1e-5, 1e-8), + }, + ) + + torch.testing.assert_close( + actual, + expected, + rtol=rtol, + atol=atol, + equal_nan=equal_nan, + check_device=True, + check_dtype=False, + check_stride=False, + msg=msg or None, + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/testing/_creation.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/testing/_creation.py new file mode 100644 index 0000000000000000000000000000000000000000..23d80d6ceae4fdbeceef2beb39c7aab8087d267c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/testing/_creation.py @@ -0,0 +1,276 @@ +""" +This module contains tensor creation utilities. +""" + +import collections.abc +import functools +import math +import warnings +from typing import cast, Optional, Union + +import torch + + +_INTEGRAL_TYPES = [ + torch.uint8, + torch.int8, + torch.int16, + torch.int32, + torch.int64, + torch.uint16, + torch.uint32, + torch.uint64, +] +_FLOATING_TYPES = [torch.float16, torch.bfloat16, torch.float32, torch.float64] +_FLOATING_8BIT_TYPES = [ + torch.float8_e4m3fn, + torch.float8_e5m2, + torch.float8_e4m3fnuz, + torch.float8_e5m2fnuz, +] +_COMPLEX_TYPES = [torch.complex32, torch.complex64, torch.complex128] +_BOOLEAN_OR_INTEGRAL_TYPES = [torch.bool, *_INTEGRAL_TYPES] +_FLOATING_OR_COMPLEX_TYPES = [*_FLOATING_TYPES, *_COMPLEX_TYPES] + + +def _uniform_random_(t: torch.Tensor, low: float, high: float) -> torch.Tensor: + # uniform_ requires to-from <= std::numeric_limits::max() + # Work around this by scaling the range before and after the PRNG + if high - low >= torch.finfo(t.dtype).max: + return t.uniform_(low / 2, high / 2).mul_(2) + else: + return t.uniform_(low, high) + + +def make_tensor( + *shape: Union[int, torch.Size, list[int], tuple[int, ...]], + dtype: torch.dtype, + device: Union[str, torch.device], + low: Optional[float] = None, + high: Optional[float] = None, + requires_grad: bool = False, + noncontiguous: bool = False, + exclude_zero: bool = False, + memory_format: Optional[torch.memory_format] = None, +) -> torch.Tensor: + r"""Creates a tensor with the given :attr:`shape`, :attr:`device`, and :attr:`dtype`, and filled with + values uniformly drawn from ``[low, high)``. + + If :attr:`low` or :attr:`high` are specified and are outside the range of the :attr:`dtype`'s representable + finite values then they are clamped to the lowest or highest representable finite value, respectively. + If ``None``, then the following table describes the default values for :attr:`low` and :attr:`high`, + which depend on :attr:`dtype`. + + +---------------------------+------------+----------+ + | ``dtype`` | ``low`` | ``high`` | + +===========================+============+==========+ + | boolean type | ``0`` | ``2`` | + +---------------------------+------------+----------+ + | unsigned integral type | ``0`` | ``10`` | + +---------------------------+------------+----------+ + | signed integral types | ``-9`` | ``10`` | + +---------------------------+------------+----------+ + | floating types | ``-9`` | ``9`` | + +---------------------------+------------+----------+ + | complex types | ``-9`` | ``9`` | + +---------------------------+------------+----------+ + + Args: + shape (Tuple[int, ...]): Single integer or a sequence of integers defining the shape of the output tensor. + dtype (:class:`torch.dtype`): The data type of the returned tensor. + device (Union[str, torch.device]): The device of the returned tensor. + low (Optional[Number]): Sets the lower limit (inclusive) of the given range. If a number is provided it is + clamped to the least representable finite value of the given dtype. When ``None`` (default), + this value is determined based on the :attr:`dtype` (see the table above). Default: ``None``. + high (Optional[Number]): Sets the upper limit (exclusive) of the given range. If a number is provided it is + clamped to the greatest representable finite value of the given dtype. When ``None`` (default) this value + is determined based on the :attr:`dtype` (see the table above). Default: ``None``. + + .. deprecated:: 2.1 + + Passing ``low==high`` to :func:`~torch.testing.make_tensor` for floating or complex types is deprecated + since 2.1 and will be removed in 2.3. Use :func:`torch.full` instead. + + requires_grad (Optional[bool]): If autograd should record operations on the returned tensor. Default: ``False``. + noncontiguous (Optional[bool]): If `True`, the returned tensor will be noncontiguous. This argument is + ignored if the constructed tensor has fewer than two elements. Mutually exclusive with ``memory_format``. + exclude_zero (Optional[bool]): If ``True`` then zeros are replaced with the dtype's small positive value + depending on the :attr:`dtype`. For bool and integer types zero is replaced with one. For floating + point types it is replaced with the dtype's smallest positive normal number (the "tiny" value of the + :attr:`dtype`'s :func:`~torch.finfo` object), and for complex types it is replaced with a complex number + whose real and imaginary parts are both the smallest positive normal number representable by the complex + type. Default ``False``. + memory_format (Optional[torch.memory_format]): The memory format of the returned tensor. Mutually exclusive + with ``noncontiguous``. + + Raises: + ValueError: If ``requires_grad=True`` is passed for integral `dtype` + ValueError: If ``low >= high``. + ValueError: If either :attr:`low` or :attr:`high` is ``nan``. + ValueError: If both :attr:`noncontiguous` and :attr:`memory_format` are passed. + TypeError: If :attr:`dtype` isn't supported by this function. + + Examples: + >>> # xdoctest: +SKIP + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) + >>> from torch.testing import make_tensor + >>> # Creates a float tensor with values in [-1, 1) + >>> make_tensor((3,), device="cpu", dtype=torch.float32, low=-1, high=1) + >>> # xdoctest: +SKIP + tensor([ 0.1205, 0.2282, -0.6380]) + >>> # Creates a bool tensor on CUDA + >>> make_tensor((2, 2), device="cuda", dtype=torch.bool) + tensor([[False, False], + [False, True]], device='cuda:0') + """ + + def modify_low_high( + low: Optional[float], + high: Optional[float], + *, + lowest_inclusive: float, + highest_exclusive: float, + default_low: float, + default_high: float, + ) -> tuple[float, float]: + """ + Modifies (and raises ValueError when appropriate) low and high values given by the user (input_low, input_high) + if required. + """ + + def clamp(a: float, l: float, h: float) -> float: + return min(max(a, l), h) + + low = low if low is not None else default_low + high = high if high is not None else default_high + + if any(isinstance(value, float) and math.isnan(value) for value in [low, high]): + raise ValueError( + f"`low` and `high` cannot be NaN, but got {low=} and {high=}" + ) + elif low == high and dtype in _FLOATING_OR_COMPLEX_TYPES: + warnings.warn( + "Passing `low==high` to `torch.testing.make_tensor` for floating or complex types " + "is deprecated since 2.1 and will be removed in 2.3. " + "Use `torch.full(...)` instead.", + FutureWarning, + stacklevel=3, + ) + elif low >= high: + raise ValueError(f"`low` must be less than `high`, but got {low} >= {high}") + elif high < lowest_inclusive or low >= highest_exclusive: + raise ValueError( + f"The value interval specified by `low` and `high` is [{low}, {high}), " + f"but {dtype} only supports [{lowest_inclusive}, {highest_exclusive})" + ) + + low = clamp(low, lowest_inclusive, highest_exclusive) + high = clamp(high, lowest_inclusive, highest_exclusive) + + if dtype in _BOOLEAN_OR_INTEGRAL_TYPES: + # 1. `low` is ceiled to avoid creating values smaller than `low` and thus outside the specified interval + # 2. Following the same reasoning as for 1., `high` should be floored. However, the higher bound of + # `torch.randint` is exclusive, and thus we need to ceil here as well. + return math.ceil(low), math.ceil(high) + + return low, high + + if len(shape) == 1 and isinstance(shape[0], collections.abc.Sequence): + shape = shape[0] # type: ignore[assignment] + shape = cast(tuple[int, ...], tuple(shape)) + + if noncontiguous and memory_format is not None: + raise ValueError( + f"The parameters `noncontiguous` and `memory_format` are mutually exclusive, " + f"but got {noncontiguous=} and {memory_format=}" + ) + + if requires_grad and dtype in _BOOLEAN_OR_INTEGRAL_TYPES: + raise ValueError( + f"`requires_grad=True` is not supported for boolean and integral dtypes, but got {dtype=}" + ) + + noncontiguous = noncontiguous and functools.reduce(lambda x, y: x * y, shape, 1) > 1 + if noncontiguous: + # Double the size of the shape in the last dimension, so that we have + # non-identical values when we make the non-contiguous operation. + shape = cast(tuple[int, ...], (*shape[:-1], 2 * shape[-1])) + + if dtype is torch.bool: + low, high = cast( + tuple[int, int], + modify_low_high( + low, + high, + lowest_inclusive=0, + highest_exclusive=2, + default_low=0, + default_high=2, + ), + ) + result = torch.randint(low, high, shape, device=device, dtype=dtype) + elif dtype in _BOOLEAN_OR_INTEGRAL_TYPES: + low, high = cast( + tuple[int, int], + modify_low_high( + low, + high, + lowest_inclusive=torch.iinfo(dtype).min, + highest_exclusive=torch.iinfo(dtype).max + # In theory, `highest_exclusive` should always be the maximum value + 1. However, `torch.randint` + # internally converts the bounds to an int64 and would overflow. In other words: `torch.randint` cannot + # sample 2**63 - 1, i.e. the maximum value of `torch.int64` and we need to account for that here. + + (1 if dtype is not torch.int64 else 0), + # This is incorrect for `torch.uint8`, but since we clamp to `lowest`, i.e. 0 for `torch.uint8`, + # _after_ we use the default value, we don't need to special case it here + default_low=-9, + default_high=10, + ), + ) + result = torch.randint(low, high, shape, device=device, dtype=dtype) + elif dtype in _FLOATING_OR_COMPLEX_TYPES: + low, high = modify_low_high( + low, + high, + lowest_inclusive=torch.finfo(dtype).min, + highest_exclusive=torch.finfo(dtype).max, + default_low=-9, + default_high=9, + ) + result = torch.empty(shape, device=device, dtype=dtype) + _uniform_random_( + torch.view_as_real(result) if dtype in _COMPLEX_TYPES else result, low, high + ) + elif dtype in _FLOATING_8BIT_TYPES: + low, high = modify_low_high( + low, + high, + lowest_inclusive=torch.finfo(dtype).min, + highest_exclusive=torch.finfo(dtype).max, + default_low=-9, + default_high=9, + ) + result = torch.empty(shape, device=device, dtype=torch.float32) + _uniform_random_(result, low, high) + result = result.to(dtype) + else: + raise TypeError( + f"The requested dtype '{dtype}' is not supported by torch.testing.make_tensor()." + " To request support, file an issue at: https://github.com/pytorch/pytorch/issues" + ) + + if noncontiguous: + # Offset by 1 to also catch offsetting issues + result = result[..., 1::2] + elif memory_format is not None: + result = result.clone(memory_format=memory_format) + + if exclude_zero: + result[result == 0] = ( + 1 if dtype in _BOOLEAN_OR_INTEGRAL_TYPES else torch.finfo(dtype).tiny + ) + + if dtype in _FLOATING_OR_COMPLEX_TYPES: + result.requires_grad = requires_grad + + return result diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/testing/_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/testing/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b86edfdd67f3c60b960ef7d239a9559ada7685f4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/testing/_utils.py @@ -0,0 +1,52 @@ +# mypy: allow-untyped-defs +import contextlib + +import torch + + +# Common testing utilities for use in public testing APIs. +# NB: these should all be importable without optional dependencies +# (like numpy and expecttest). + + +def wrapper_set_seed(op, *args, **kwargs): + """Wrapper to set seed manually for some functions like dropout + See: https://github.com/pytorch/pytorch/pull/62315#issuecomment-896143189 for more details. + """ + with freeze_rng_state(): + torch.manual_seed(42) + output = op(*args, **kwargs) + + if isinstance(output, torch.Tensor) and output.device.type == "lazy": + # We need to call mark step inside freeze_rng_state so that numerics + # match eager execution + torch._lazy.mark_step() # type: ignore[attr-defined] + + return output + + +@contextlib.contextmanager +def freeze_rng_state(): + # no_dispatch needed for test_composite_compliance + # Some OpInfos use freeze_rng_state for rng determinism, but + # test_composite_compliance overrides dispatch for all torch functions + # which we need to disable to get and set rng state + with torch.utils._mode_utils.no_dispatch(), torch._C._DisableFuncTorch(): + rng_state = torch.get_rng_state() + if torch.cuda.is_available(): + cuda_rng_state = torch.cuda.get_rng_state() + try: + yield + finally: + # Modes are not happy with torch.cuda.set_rng_state + # because it clones the state (which could produce a Tensor Subclass) + # and then grabs the new tensor's data pointer in generator.set_state. + # + # In the long run torch.cuda.set_rng_state should probably be + # an operator. + # + # NB: Mode disable is to avoid running cross-ref tests on this seeding + with torch.utils._mode_utils.no_dispatch(), torch._C._DisableFuncTorch(): + if torch.cuda.is_available(): + torch.cuda.set_rng_state(cuda_rng_state) # type: ignore[possibly-undefined] + torch.set_rng_state(rng_state) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/__init__.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1c3ec1579006399a6025e8475470effeccf7cd22 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/__init__.py @@ -0,0 +1,107 @@ +# mypy: allow-untyped-defs + +import copyreg +import os.path as _osp +import weakref + +import torch +from torch.utils import ( + backcompat as backcompat, + collect_env as collect_env, + data as data, + deterministic as deterministic, + hooks as hooks, +) +from torch.utils.backend_registration import ( + generate_methods_for_privateuse1_backend, + rename_privateuse1_backend, +) +from torch.utils.cpp_backtrace import get_cpp_backtrace +from torch.utils.throughput_benchmark import ThroughputBenchmark + + +def set_module(obj, mod): + """ + Set the module attribute on a python object for a given object for nicer printing + """ + if not isinstance(mod, str): + raise TypeError("The mod argument should be a string") + obj.__module__ = mod + + +cmake_prefix_path = _osp.join(_osp.dirname(_osp.dirname(__file__)), "share", "cmake") + + +def swap_tensors(t1, t2): + """ + This function swaps the content of the two Tensor objects. + At a high level, this will make t1 have the content of t2 while preserving + its identity. + + This will not work if t1 and t2 have different slots. + """ + # Ensure there are no weakrefs + if weakref.getweakrefs(t1): + raise RuntimeError("Cannot swap t1 because it has weakref associated with it") + if weakref.getweakrefs(t2): + raise RuntimeError("Cannot swap t2 because it has weakref associated with it") + t1_slots = set(copyreg._slotnames(t1.__class__)) # type: ignore[attr-defined] + t2_slots = set(copyreg._slotnames(t2.__class__)) # type: ignore[attr-defined] + if t1_slots != t2_slots: + raise RuntimeError("Cannot swap t1 and t2 if they have different slots") + + def swap_attr(name): + tmp = getattr(t1, name) + setattr(t1, name, (getattr(t2, name))) + setattr(t2, name, tmp) + + def error_pre_hook(grad_outputs): + raise RuntimeError( + "Trying to execute AccumulateGrad node that was poisoned by swap_tensors " + "this can happen when you try to run backward on a tensor that was swapped. " + "For a module m with `torch.__future__.set_swap_module_params_on_conversion(True)` " + "you should not change the device or dtype of the module (e.g. `m.cpu()` or `m.half()`) " + "between running forward and backward. To resolve this, please only change the " + "device/dtype before running forward (or after both forward and backward)." + ) + + def check_use_count(t, name="t1"): + use_count = t._use_count() + error_str = ( + f"Expected use_count of {name} to be 1 or 2 with an AccumulateGrad node but got {use_count} " + f"make sure you are not holding references to the tensor in other places." + ) + if use_count > 1: + if use_count == 2 and t.is_leaf: + accum_grad_node = torch.autograd.graph.get_gradient_edge(t).node + # Make sure that the accumulate_grad node was not lazy_init-ed by get_gradient_edge + if t._use_count() == 2: + accum_grad_node.register_prehook(error_pre_hook) + else: + raise RuntimeError(error_str) + else: + raise RuntimeError(error_str) + + check_use_count(t1, "t1") + check_use_count(t2, "t2") + + # Swap the types + # Note that this will fail if there are mismatched slots + swap_attr("__class__") + + # Swap the dynamic attributes + swap_attr("__dict__") + + # Swap the slots + for slot in t1_slots: + if hasattr(t1, slot) and hasattr(t2, slot): + swap_attr(slot) + elif hasattr(t1, slot): + setattr(t2, slot, (getattr(t1, slot))) + delattr(t1, slot) + elif hasattr(t2, slot): + setattr(t1, slot, (getattr(t2, slot))) + delattr(t2, slot) + + # Swap the at::Tensor they point to + torch._C._swap_tensor_impl(t1, t2) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_appending_byte_serializer.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_appending_byte_serializer.py new file mode 100644 index 0000000000000000000000000000000000000000..82cced0b3dc826484e2d44788e4814b9dbe32693 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_appending_byte_serializer.py @@ -0,0 +1,135 @@ +import base64 +import zlib +from collections.abc import Callable, Iterable +from typing import Generic, TypeVar + + +T = TypeVar("T") + +_ENCODING_VERSION: int = 1 + +__all__ = ["AppendingByteSerializer"] + + +####################################### +# Helper classes +####################################### + +CHECKSUM_DIGEST_SIZE = 4 + + +class BytesWriter: + def __init__(self) -> None: + # Reserve CHECKSUM_DIGEST_SIZE bytes for checksum + self._data = bytearray(CHECKSUM_DIGEST_SIZE) + + def write_uint64(self, i: int) -> None: + self._data.extend(i.to_bytes(8, byteorder="big", signed=False)) + + def write_str(self, s: str) -> None: + payload = base64.b64encode(s.encode("utf-8")) + self.write_bytes(payload) + + def write_bytes(self, b: bytes) -> None: + self.write_uint64(len(b)) + self._data.extend(b) + + def to_bytes(self) -> bytes: + digest = zlib.crc32(self._data[CHECKSUM_DIGEST_SIZE:]).to_bytes( + 4, byteorder="big", signed=False + ) + if len(digest) != CHECKSUM_DIGEST_SIZE: + raise AssertionError("Computed checksum digest has unexpected size") + self._data[0:CHECKSUM_DIGEST_SIZE] = digest + return bytes(self._data) + + +class BytesReader: + def __init__(self, data: bytes) -> None: + # Check for data corruption + if len(data) < CHECKSUM_DIGEST_SIZE: + raise AssertionError("Input data is too short to contain checksum") + digest = zlib.crc32(data[CHECKSUM_DIGEST_SIZE:]).to_bytes( + 4, byteorder="big", signed=False + ) + if len(digest) != CHECKSUM_DIGEST_SIZE: + raise AssertionError("Computed checksum digest has unexpected size") + if data[0:CHECKSUM_DIGEST_SIZE] != digest: + raise RuntimeError( + "Bytes object is corrupted, checksum does not match. " + f"Expected: {data[0:CHECKSUM_DIGEST_SIZE]!r}, Got: {digest!r}" + ) + + self._data = data + self._i = CHECKSUM_DIGEST_SIZE + + def is_finished(self) -> bool: + return len(self._data) == self._i + + def read_uint64(self) -> int: + result = int.from_bytes( + self._data[self._i : self._i + 8], byteorder="big", signed=False + ) + self._i += 8 + return result + + def read_str(self) -> str: + return base64.b64decode(self.read_bytes()).decode("utf-8") + + def read_bytes(self) -> bytes: + size = self.read_uint64() + result = self._data[self._i : self._i + size] + self._i += size + return result + + +####################################### +# AppendingByteSerializer +####################################### + + +class AppendingByteSerializer(Generic[T]): + """ + Provides efficient serialization and deserialization of list of bytes + Note that this does not provide any guarantees around byte order + """ + + _serialize_fn: Callable[[BytesWriter, T], None] + _writer: BytesWriter + + def __init__( + self, + *, + serialize_fn: Callable[[BytesWriter, T], None], + ) -> None: + self._serialize_fn = serialize_fn + self.clear() + + def clear(self) -> None: + self._writer = BytesWriter() + # First 8-bytes are for version + self._writer.write_uint64(_ENCODING_VERSION) + + def append(self, data: T) -> None: + self._serialize_fn(self._writer, data) + + def extend(self, elems: Iterable[T]) -> None: + for elem in elems: + self.append(elem) + + def to_bytes(self) -> bytes: + return self._writer.to_bytes() + + @staticmethod + def to_list(data: bytes, *, deserialize_fn: Callable[[BytesReader], T]) -> list[T]: + reader = BytesReader(data) + if reader.read_uint64() != _ENCODING_VERSION: + raise AssertionError( + f"Encoding version mismatch in AppendingByteSerializer.to_list, \ + got {reader.read_uint64()}" + ) + + result: list[T] = [] + while not reader.is_finished(): + result.append(deserialize_fn(reader)) + return result diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_config_module.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_config_module.py new file mode 100644 index 0000000000000000000000000000000000000000..16fbad73a3097267fa38df03e0d933f0f78cd13f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_config_module.py @@ -0,0 +1,825 @@ +import contextlib +import copy +import hashlib +import importlib +import inspect +import io +import os +import pickle +import tokenize +import unittest +from collections.abc import Callable +from dataclasses import dataclass +from types import FunctionType, ModuleType +from typing import Any, Generic, NoReturn, Optional, TYPE_CHECKING, TypeVar +from typing_extensions import deprecated +from unittest import mock + +from torch._utils_internal import justknobs_check + + +# Types saved/loaded in configs +CONFIG_TYPES = (int, float, bool, type(None), str, list, set, tuple, dict) + + +# Duplicated, because mypy needs these types statically +T = TypeVar("T", bound=int | float | bool | None | str | list | set | tuple | dict) + + +_UNSET_SENTINEL = object() + + +@dataclass(kw_only=True) +class _Config(Generic[T]): + """Represents a config with richer behaviour than just a default value. + :: + i.e. + foo = Config(justknob="//foo:bar", default=False) + install_config_module(...) + + This configs must be installed with install_config_module to be used + + Precedence Order: + alias: If set, the directly use the value of the alias. + env_name_force: If set, this environment variable has precedence over + everything after this. + If multiple env variables are given, the precedence order is from + left to right. + user_override: If a user sets a value (i.e. foo.bar=True), that + has precedence over everything after this. + env_name_default: If set, this environment variable will override everything + after this. + If multiple env variables are given, the precedence order is from + left to right. + justknob: If this pytorch installation supports justknobs, that will + override defaults, but will not override the user_override precedence. + default: This value is the lowest precedence, and will be used if nothing is + set. + + Environment Variables: + These are interpreted to be either "0" or "1" to represent true and false. + + Arguments: + justknob: the name of the feature / JK. In OSS this is unused. + default: is the value to default this knob to in OSS. + alias: The alias config to read instead. + env_name_force: The environment variable, or list of, to read that is a FORCE + environment variable. I.e. it overrides everything except for alias. + env_name_default: The environment variable, or list of, to read that changes the + default behaviour. I.e. user overrides take preference. + """ + + default: T | object + justknob: str | None = None + env_name_default: list[str] | None = None + env_name_force: list[str] | None = None + value_type: type | None = None + alias: str | None = None + + def __post_init__(self) -> None: + self.env_name_default = _Config.string_or_list_of_string_to_list( + self.env_name_default + ) + self.env_name_force = _Config.string_or_list_of_string_to_list( + self.env_name_force + ) + + if self.alias is not None: + if ( + self.default is not _UNSET_SENTINEL + or self.justknob is not None + or self.env_name_default is not None + or self.env_name_force is not None + ): + raise AssertionError( + "if alias is set, none of {default, justknob, \ + env_name_default and env_name_force} can be set" + ) + + @staticmethod + def string_or_list_of_string_to_list( + val: str | list[str] | None, + ) -> list[str] | None: + if val is None: + return None + if isinstance(val, str): + return [val] + if not isinstance(val, list): + raise AssertionError(f"val is not a list, got {type(val)}") + return val + + +# In runtime, we unbox the Config[T] to a T, but typechecker cannot see this, +# so in order to allow for this dynamic behavior to work correctly with +# typechecking we are going to lie to the typechecker that Config[T] returns +# a T. +if TYPE_CHECKING: + + def Config( + default: T | object = _UNSET_SENTINEL, + justknob: str | None = None, + env_name_default: str | list[str] | None = None, + env_name_force: str | list[str] | None = None, + value_type: type | None = None, + alias: str | None = None, + ) -> T: ... + +else: + + def Config( + default: T | object = _UNSET_SENTINEL, + justknob: str | None = None, + env_name_default: str | list[str] | None = None, + env_name_force: str | list[str] | None = None, + value_type: type | None = None, + alias: str | None = None, + ) -> _Config[T]: + return _Config( + default=default, + justknob=justknob, + env_name_default=env_name_default, + env_name_force=env_name_force, + value_type=value_type, + alias=alias, + ) + + +def _read_env_variable(name: str) -> bool | str | None: + value = os.environ.get(name) + if value == "1": + return True + if value == "0": + return False + return value + + +def install_config_module(module: ModuleType) -> None: + """ + Converts a module-level config into a `ConfigModule()`. + + See _config_typing.pyi for instructions on how to get the converted module to typecheck. + """ + + class ConfigModuleInstance(ConfigModule): + # __annotations__ is written to by Sphinx autodoc + _bypass_keys = set({"_is_dirty", "_hash_digest", "__annotations__"}) + + def visit( + source: ModuleType | type, + dest: ModuleType | SubConfigProxy, + prefix: str, + ) -> None: + """Walk the module structure and move everything to module._config""" + type_hints = inspect.get_annotations(source) + for key, value in list(source.__dict__.items()): + if ( + key.startswith("__") + or isinstance(value, (ModuleType, FunctionType)) + or ( + hasattr(value, "__module__") + and ( + value.__module__ == "typing" + or value.__module__.startswith("collections.abc") + ) + ) + # Handle from torch.utils._config_module import Config + or (isinstance(value, type) and issubclass(value, _Config)) + ): + continue + + name = f"{prefix}{key}" + annotated_type = type_hints.get(key, None) + if isinstance(value, CONFIG_TYPES): + config[name] = _ConfigEntry( + _Config(default=value, value_type=annotated_type) + ) + if dest is module: + delattr(module, key) + elif isinstance(value, _Config): + if annotated_type is not None and value.value_type is None: + value.value_type = annotated_type + + config[name] = _ConfigEntry(value) + + if dest is module: + delattr(module, key) + elif isinstance(value, type): + if value.__module__ != module.__name__: + raise AssertionError( + f"subconfig class {value} must be defined in module {module.__name__}" + ) + # a subconfig with `class Blah:` syntax + proxy = SubConfigProxy(module, f"{name}.") + visit(value, proxy, f"{name}.") + if dest is module: + setattr(dest, key, proxy) + else: + dest.__dict__[key] = proxy + else: + raise AssertionError(f"Unhandled config {key}={value} ({type(value)})") + + config: dict[str, _ConfigEntry] = {} + + compile_ignored_keys = get_assignments_with_compile_ignored_comments(module) + + visit(module, module, "") + module._config = config # type: ignore[attr-defined] + module._compile_ignored_keys = compile_ignored_keys # type: ignore[attr-defined] + module.__class__ = ConfigModuleInstance + module._is_dirty = True # type: ignore[attr-defined] + module._hash_digest = None # type: ignore[attr-defined] + + +COMPILE_IGNORED_MARKER = "@compile_ignored" + + +# Gets all the keys (i.e. assignments) with a @compile_ignored comment +def get_assignments_with_compile_ignored_comments(module: ModuleType) -> set[str]: + source_code = inspect.getsource(module) + assignments = set() + + # Tokenize the source code to retrieve comments + tokens = tokenize.tokenize(io.BytesIO(source_code.encode("utf-8")).readline) + current_comment = "", -1 + prev_name = "" + + for token in tokens: + if token.type == tokenize.COMMENT: + prev_name = "" + maybe_current = token.string.strip() + if COMPILE_IGNORED_MARKER in maybe_current: + if current_comment != ("", -1): + raise AssertionError(f"unconsumed {COMPILE_IGNORED_MARKER}") + current_comment = maybe_current, token.start[0] + elif token.type == tokenize.NAME: + # Only accept the first name token, to handle if you have + # something like foo: Bar = ... + if not prev_name: + prev_name = token.string + elif token.type == tokenize.OP and token.string == "=": + # Check if the current assignment follows a comment + # with COMPILE_IGNORED_MARKER + if ( + COMPILE_IGNORED_MARKER in current_comment[0] + and current_comment[1] == token.start[0] - 1 + ): + assignments.add(prev_name) + current_comment = "", -1 # reset + prev_name = "" + if current_comment != ("", -1): + raise AssertionError(f"unconsumed {COMPILE_IGNORED_MARKER}") + return assignments + + +@dataclass +class _ConfigEntry: + # The default value specified in the configuration + default: Any + # The type of the configuration value + value_type: type + # The value specified by the user when they overrode the configuration + # _UNSET_SENTINEL indicates the value is not set. + user_override: Any = _UNSET_SENTINEL + # The justknob to check for this config + justknob: str | None = None + # environment variables are read at install time + env_value_force: Any = _UNSET_SENTINEL + env_value_default: Any = _UNSET_SENTINEL + # Used to work arounds bad assumptions in unittest.mock.patch + # The code to blame is + # https://github.com/python/cpython/blob/94a7a4e22fb8f567090514785c69e65298acca42/Lib/unittest/mock.py#L1637 + # Essentially, mock.patch requires, that if __dict__ isn't accessible + # (which it isn't), that after delattr is called on the object, the + # object must throw when hasattr is called. Otherwise, it doesn't call + # setattr again. + # Technically we'll have an intermediate state of hiding the config while + # mock.patch is unpatching itself, but it calls setattr after the delete + # call so the final state is correct. It's just very unintuitive. + # upstream bug - python/cpython#126886 + hide: bool = False + alias: str | None = None + + def __init__(self, config: _Config) -> None: + self.default = config.default + self.value_type = ( + config.value_type if config.value_type is not None else type(self.default) + ) + self.justknob = config.justknob + self.alias = config.alias + if config.env_name_default is not None: + for val in config.env_name_default: + if (env_value := _read_env_variable(val)) is not None: + self.env_value_default = env_value + break + if config.env_name_force is not None: + for val in config.env_name_force: + if (env_value := _read_env_variable(val)) is not None: + self.env_value_force = env_value + break + + # Ensure justknobs and envvars are allowlisted types + if self.justknob is not None and self.default is not None: + if not isinstance(self.default, bool): + raise AssertionError( + f"justknobs only support booleans, {self.default} is not a boolean" + ) + if self.value_type is not None and ( + config.env_name_default is not None or config.env_name_force is not None + ): + if self.value_type not in ( + bool, + str, + Optional[bool], + Optional[str], + ): + raise AssertionError( + f"envvar configs only support (optional) booleans or strings, {self.value_type} is neither" + ) + + +class ConfigModule(ModuleType): + # NOTE: This should be kept in sync with _config_typing.pyi. + + # The actual configuration settings. E.g., torch._dynamo.config.debug + # would live as "debug" in the key, and torch._inductor.config.triton.cudagraphs + # maps as "triton.cudagraphs". See discussion on the class for meaning of various sub items + _config: dict[str, _ConfigEntry] + _bypass_keys: set[str] + _compile_ignored_keys: set[str] + _is_dirty: bool + _hash_digest: bytes | None + + def __init__(self) -> None: + raise NotImplementedError( + f"use {__name__}.install_config_module(sys.modules[__name__])" + ) + + def __setattr__(self, name: str, value: object) -> None: + if name in self._bypass_keys: + super().__setattr__(name, value) + elif name not in self._config: + raise AttributeError(f"{self.__name__}.{name} does not exist") + elif self._config[name].alias is not None: + self._set_alias_val(self._config[name], value) + else: + self._config[name].user_override = value + self._is_dirty = True + self._config[name].hide = False + + def __getattr__(self, name: str) -> Any: + try: + config = self._config[name] + + if config.hide: + raise AttributeError(f"{self.__name__}.{name} does not exist") + + alias_val = self._get_alias_val(config) + if alias_val is not _UNSET_SENTINEL: + return alias_val + + if config.env_value_force is not _UNSET_SENTINEL: + return config.env_value_force + + if config.user_override is not _UNSET_SENTINEL: + return config.user_override + + if config.env_value_default is not _UNSET_SENTINEL: + return config.env_value_default + + if config.justknob is not None: + # JK only supports bools and ints + return justknobs_check(name=config.justknob, default=config.default) + + # Note that reference types can still be modified, so we + # copy them to user_overrides in case the user overrides + # them + if isinstance(config.default, (list, set, dict)): + config.user_override = copy.deepcopy(config.default) + return config.user_override + return config.default + + except KeyError as e: + # make hasattr() work properly + raise AttributeError(f"{self.__name__}.{name} does not exist") from e + + def __delattr__(self, name: str) -> None: + self._is_dirty = True + # must support delete because unittest.mock.patch deletes + # then recreate things + self._config[name].user_override = _UNSET_SENTINEL + self._config[name].hide = True + + def _get_alias_module_and_name( + self, entry: _ConfigEntry + ) -> tuple[ModuleType, str] | None: + alias = entry.alias + if alias is None: + return None + module_name, constant_name = alias.rsplit(".", 1) + try: + module = importlib.import_module(module_name) + except ImportError as e: + raise AttributeError(f"config alias {alias} does not exist") from e + return module, constant_name + + def _get_alias_val(self, entry: _ConfigEntry) -> Any: + data = self._get_alias_module_and_name(entry) + if data is None: + return _UNSET_SENTINEL + module, constant_name = data + constant_value = getattr(module, constant_name) + return constant_value + + def _set_alias_val(self, entry: _ConfigEntry, val: Any) -> None: + data = self._get_alias_module_and_name(entry) + if data is None: + raise AssertionError( + "alias data should not be None when setting alias value" + ) + module, constant_name = data + setattr(module, constant_name, val) + + def _is_default(self, name: str) -> bool: + """ + Returns true if the config is at its default value. + configs overridden by the env are not considered default. + """ + config_val = self._config[name] + # The config is not overridden by the user, and the env_value_default + # is different from the default value (meaning user has set the env to + # change the default value). + not_set_env_default = ( + config_val.env_value_default is _UNSET_SENTINEL + or config_val.env_value_default == config_val.default + ) + not_set_env_force = ( + config_val.env_value_force is _UNSET_SENTINEL + or config_val.env_value_force == config_val.default + ) + + unset = config_val.user_override is _UNSET_SENTINEL + # Handle reference types specially to avoid spammy warnings + if isinstance(config_val.default, (list, set, dict)): + unset = unset or config_val.user_override == config_val.default + return unset and not_set_env_default and not_set_env_force + + def _get_dict( + self, + ignored_keys: list[str] | None = None, + ignored_prefixes: list[str] | None = None, + skip_default: bool = False, + ) -> dict[str, Any]: + """Export a dictionary of current configuration keys and values. + + This function is design to provide a single point which handles + accessing config options and exporting them into a dictionary. + This is used by a number of different user facing export methods + which all have slightly different semantics re: how and what to + skip. + If a config is aliased, it skips this config. + + Arguments: + ignored_keys are keys that should not be exported. + ignored_prefixes are prefixes that if a key matches should + not be exported + skip_default does two things. One if a key has not been modified + it skips it. + """ + config: dict[str, Any] = {} + for key in self._config: + if ignored_keys and key in ignored_keys: + continue + if ignored_prefixes: + if any(key.startswith(prefix) for prefix in ignored_prefixes): + continue + if skip_default and self._is_default(key): + continue + if self._config[key].alias is not None: + continue + config[key] = copy.deepcopy(getattr(self, key)) + + return config + + def get_type(self, config_name: str) -> type: + return self._config[config_name].value_type + + def save_config(self) -> bytes: + """Convert config to a pickled blob""" + ignored_keys = getattr(self, "_save_config_ignore", []) + return pickle.dumps( + self._get_dict(ignored_keys=ignored_keys), + protocol=2, + ) + + def save_config_portable( + self, *, ignore_private_configs: bool = True + ) -> dict[str, Any]: + """Convert config to portable format""" + prefixes = [] + if ignore_private_configs: + prefixes.append("_") + prefixes.extend(getattr(self, "_cache_config_ignore_prefix", [])) + return self._get_dict(ignored_prefixes=prefixes) + + def codegen_config(self) -> str: + """Convert config to Python statements that replicate current config. + This does NOT include config settings that are at default values. + """ + + # additional imports required + imports = set() + + def get_module_name(func: Callable, add_dot: bool) -> str: + module_name = func.__module__ + if module_name == "builtins": + module_name = "" + if add_dot and module_name != "": + module_name += "." + return module_name + + def add_import(func: Callable) -> None: + module_name = get_module_name(func, False) + if module_name: + imports.add(module_name) + + def list_of_callables_to_string(v: list | set) -> list[str]: + return [f"{get_module_name(item, True)}{item.__name__}" for item in v] + + def importable_callable(v: Any) -> bool: + # functools.partial has no attributes below but is a callable + return callable(v) and hasattr(v, "__module__") and hasattr(v, "__name__") + + def get_config_line(mod, k, v) -> str: # type: ignore[no-untyped-def] + """ + Return a string version of the config line. + Handle v when v is a callable, or a list/dict of callables. Add import statements for callables if necessary. + We assume that the value of a single config won't be a mix of callables and non-callables. + + Example output: + import logging + import _warnings + torch._dynamo.config.reorderable_logging_functions = { _warnings.warn, logging.warn, print } + """ + if importable_callable(v): + add_import(v) + return f"{mod}.{k} = {get_module_name(v, True)}{v.__name__}" + elif isinstance(v, (list, set)) and all( + importable_callable(item) for item in v + ): + for item in v: + add_import(item) + v_list = list_of_callables_to_string(v) + if isinstance(v, list): + return f"{mod}.{k} = {v_list}" + else: + return f"{mod}.{k} = {{ {', '.join(v_list)} }}" + else: + return f"{mod}.{k} = {v!r}" + + lines = [] + mod = self.__name__ + for k, v in self._get_dict( + ignored_keys=getattr(self, "_save_config_ignore", []), skip_default=True + ).items(): + lines.append(get_config_line(mod, k, v)) + for import_name in imports: + lines.insert(0, f"import {import_name}") + return "\n".join(lines) + + def get_hash(self) -> bytes: + """Hashes the configs that are not compile_ignored""" + if self._is_dirty or self._hash_digest is None: + dict_to_hash = self._get_dict(ignored_keys=list(self._compile_ignored_keys)) + string_to_hash = repr(sorted(dict_to_hash.items())) + self._hash_digest = hashlib.md5( + string_to_hash.encode("utf-8"), usedforsecurity=False + ).digest() + self._is_dirty = False + return self._hash_digest + + @deprecated( + "`config.to_dict()` has been deprecated. It no longer changes the underlying config." + " use `config.get_config_copy()` instead if you just want a copy of the config, or " + "config.load_config if you need mutable access", + category=FutureWarning, + ) + def to_dict(self) -> dict[str, Any]: + return self.get_config_copy() + + @deprecated( + "`config.shallow_copy_dict()` has been deprecated. It no longer changes the underlying config." + " use `config.get_config_copy()` instead if you just want a copy of the config, or " + "config.load_config if you need mutable access", + category=FutureWarning, + ) + def shallow_copy_dict(self) -> dict[str, Any]: + return self.get_config_copy() + + def load_config(self, maybe_pickled_config: bytes | dict[str, Any]) -> None: + """Restore from a prior call to save_config() or shallow_copy_dict()""" + if not isinstance(maybe_pickled_config, dict): + config = pickle.loads(maybe_pickled_config) + else: + config = maybe_pickled_config + for k, v in config.items(): + if k in self._config: + setattr(self, k, v) + else: + from torch._dynamo.utils import warn_once + + warn_once(f"key {k} with value {v} is not understood by this config") + + def get_config_copy(self) -> dict[str, Any]: + return self._get_dict() + + def get_serializable_config_copy(self) -> dict[str, Any]: + return self._get_dict(ignored_keys=getattr(self, "_save_config_ignore", [])) + + def patch( + self, + arg1: str | dict[str, Any] | None = None, + arg2: Any = None, + **kwargs: dict[str, Any], + ) -> "ContextDecorator": + """ + Decorator and/or context manager to make temporary changes to a config. + + As a decorator: + + @config.patch("name", val) + @config.patch(name1=val1, name2=val2) + @config.patch({"name1": val1, "name2", val2}) + def foo(...): + ... + + As a context manager: + + with config.patch("name", val): + ... + """ + changes: dict[str, Any] + if arg1 is not None: + if arg2 is not None: + if not isinstance(arg1, str): + raise AssertionError( + "first argument must be a string when passing 2 positional args to patch" + ) + # patch("key", True) syntax + changes = {arg1: arg2} + else: + if not isinstance(arg1, dict): + raise AssertionError( + "first argument must be a dict when passing a single positional arg to patch" + ) + # patch({"key": True}) syntax + changes = arg1 + if kwargs: + raise AssertionError( + "cannot pass both positional and keyword arguments to patch" + ) + else: + # patch(key=True) syntax + changes = kwargs + if arg2 is not None: + raise AssertionError( + "second positional argument is only valid when first argument is a key string" + ) + if not isinstance(changes, dict): + raise AssertionError(f"expected `dict` got {type(changes)}") + prior: dict[str, Any] = {} + config = self + + class ConfigPatch(ContextDecorator): + def __init__(self) -> None: + self.changes = changes + + def __enter__(self) -> None: + if prior: + raise AssertionError( + "prior should be empty when entering ConfigPatch" + ) + for key in self.changes: + # KeyError on invalid entry + prior[key] = config.__getattr__(key) + for k, v in self.changes.items(): + config.__setattr__(k, v) + + def __exit__(self, exc_type, exc_val, exc_tb): # type: ignore[no-untyped-def] + for k, v in prior.items(): + config.__setattr__(k, v) + prior.clear() + + return ConfigPatch() + + def _make_closure_patcher(self, **changes: dict[str, Any]) -> Any: + """ + A lower-overhead version of patch() for things on the critical path. + + Usage: + + # do this off the critical path + change_fn = config.make_closure_patcher(foo=True) + + ... + + revert = change_fn() + try: + ... + finally: + revert() + + """ + config = self._config + + def change() -> Callable[[], None]: + prior = {k: config[k].user_override for k in changes} + for k, v in changes.items(): + self._config[k].user_override = v + + def revert() -> None: + for k, v in prior.items(): + self._config[k].user_override = v + + return revert + + return change + + +class ContextDecorator(contextlib.ContextDecorator): + """ + Same as contextlib.ContextDecorator, but with support for + `unittest.TestCase` + """ + + def __enter__(self) -> None: + raise NotImplementedError("NYI") + + def __exit__(self, exc_type, exc_val, exc_tb) -> NoReturn: # type: ignore[no-untyped-def] + raise NotImplementedError("NYI") + + def __call__(self, func: Callable[[Any], Any]) -> Any: + if isinstance(func, type) and issubclass(func, unittest.TestCase): + + class _TestCase(func): # type: ignore[valid-type, misc] + @classmethod + def setUpClass(cls) -> None: + self.__enter__() + try: + super().setUpClass() + except Exception: + self.__exit__(None, None, None) + raise + + @classmethod + def tearDownClass(cls) -> None: + try: + super().tearDownClass() + finally: + self.__exit__(None, None, None) + + _TestCase.__name__ = func.__name__ + _TestCase.__qualname__ = func.__qualname__ + _TestCase.__module__ = func.__module__ + + return _TestCase + + return super().__call__(func) + + +class SubConfigProxy: + """ + Shim to redirect to main config. + `config.triton.cudagraphs` maps to _config["triton.cudagraphs"] + """ + + def __init__(self, config: object, prefix: str) -> None: + # `super().__setattr__` to bypass custom `__setattr__` + super().__setattr__("_config", config) + super().__setattr__("_prefix", prefix) + + def __setattr__(self, name: str, value: object) -> None: + return self._config.__setattr__(self._prefix + name, value) + + def __getattr__(self, name: str) -> Any: + return self._config.__getattr__(self._prefix + name) + + def __delattr__(self, name: str) -> None: + return self._config.__delattr__(self._prefix + name) + + +def patch_object(obj: object, name: str, value: object) -> object: + """ + Workaround `mock.patch.object` issue with ConfigModule + """ + if isinstance(obj, ConfigModule): + return obj.patch(name, value) + return mock.patch.object(obj, name, value) + + +def get_tristate_env(name: str, default: Any = None) -> bool | None: + value = os.environ.get(name) + if value == "1": + return True + if value == "0": + return False + return default diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_config_typing.pyi b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_config_typing.pyi new file mode 100644 index 0000000000000000000000000000000000000000..9cae7368cfa5ee74392cde78942ae1634f5b868e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_config_typing.pyi @@ -0,0 +1,36 @@ +# mypy: allow-untyped-defs +from typing import Any, TYPE_CHECKING + +""" +This was semi-automatically generated by running + + stubgen torch.utils._config_module.py + +And then manually extracting the methods of ConfigModule and converting them into top-level functions. + +This file should be imported into any file that uses install_config_module like so: + + if TYPE_CHECKING: + from torch.utils._config_typing import * # noqa: F401, F403 + + from torch.utils._config_module import install_config_module + + # adds patch, save_config, etc + install_config_module(sys.modules[__name__]) + +Note that the import should happen before the call to install_config_module(), otherwise runtime errors may occur. +""" + +if not TYPE_CHECKING: # noqa: PYI002 + raise AssertionError("Do not use at runtime") # noqa: W291 + +def save_config() -> bytes: ... +def save_config_portable(*, ignore_private_configs: bool = True) -> dict[str, Any]: ... +def codegen_config() -> str: ... +def get_hash() -> bytes: ... +def to_dict() -> dict[str, Any]: ... +def shallow_copy_dict() -> dict[str, Any]: ... +def load_config(config: bytes | dict[str, Any]) -> None: ... +def get_config_copy() -> dict[str, Any]: ... +def get_serializable_config_copy() -> dict[str, Any]: ... +def patch(arg1: str | dict[str, Any] | None = None, arg2: Any = None, **kwargs): ... diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_content_store.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_content_store.py new file mode 100644 index 0000000000000000000000000000000000000000..234355210057ab703b9f0aea3729757787f89eb6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_content_store.py @@ -0,0 +1,241 @@ +# mypy: allow-untyped-defs +# This module provides a FAST (on GPU) content addressable store for storages +# (and tensors on top of them) with VERY WEAK portability guarantees (e.g., +# don't expect CPU/CUDA to address to the same hash, don't expect it to be +# portable across devices) that is NOT cryptographically secure. In return, +# we are able to hash 40G of tensor data on GPU in less than a second, +# compared to running SHA-1 in CPU which would a minute or so. The primary +# use case is for efficiently snapshotting intermediate tensor data for +# offline debugging, but it's been put in this module in case you think of +# another use case for it. The hash function could be replaced with a +# straight reimplementation of SHA-1, which would give us much stronger +# portability guarantees. +# +# WARNING: THERE IS NO BC/FC GUARANTEE FOR THIS FORMAT! If you need to format +# shift the result, consider packing it into a single torch.save object +# with traditional view sharing. +# +# Because of the weak portability guarantees, you can only write to the +# content store from a single process; we don't provide any capability +# of "reopening" a content store to add more things to it. But we don't +# assume that you can keep all of the tensors you want to add to the store +# in memory at once, because you probably can't! Nor do we assume that +# you know a priori whether or not two storages can be deduplicated or not. +# +# Note: only storages are content-addressed; tensors are name addressed +# +# Note: our padding strategy means that [1, 0] and [1] int16 tensors would +# map to the same (padded) storage. We think this will be immaterial for most +# users. + +import ctypes +import functools +import hashlib +import os.path +import struct +from collections import defaultdict + +import torch +import torch._prims as prims +import torch._utils +import torch.nn.functional as F +from torch.multiprocessing.reductions import StorageWeakRef + + +def lazy_compile(**compile_kwargs): + """Lazily wrap a function with torch.compile on the first call + + This avoids eagerly importing dynamo. + """ + + def decorate_fn(fn): + @functools.wraps(fn) + def compile_hook(*args, **kwargs): + compiled_fn = torch.compile(fn, **compile_kwargs) + globals()[fn.__name__] = functools.wraps(fn)(compiled_fn) + return compiled_fn(*args, **kwargs) + + return compile_hook + + return decorate_fn + + +# Use of torch.compile is mandatory for (1) good memory usage +# and (2) xor_sum implementation. This is our first instance of +# using PT2 to implement a kernel in PyTorch; if we get AOT capabilities +# it would be good to apply it here. +@lazy_compile(dynamic=True) +def hash_storage_kernel(x): + # The randint calls are carefully written to hit things we + # have lowerings for in inductor. Lack of unsigned 32-bit integer + # is a pain. + a = torch.randint( + -(2**31), 2**31, x.shape, device=x.device, dtype=torch.int32 + ).abs() + a = ((a % (2**31 - 1)) + 1).long() + b = ( + torch.randint(-(2**31), 2**31, x.shape, device=x.device, dtype=torch.int32) + .abs() + .long() + ) + # This is a standard shift-multiply universal hash family + # plus xor sum hash, using Philox to generate random numbers. + # Our Philox RNG is not deterministic across devices so + # don't use this for stable hashing. + # + # This assumes fixed length so you're also obligated to bucket + # by the length of tensor as well + return prims.xor_sum((a * x + b).int(), [0]) + + +# Returns a hex digest of the data in the storage. Guaranteed to be +# SHA-1 if stable_hash=True, otherwise it will consistent for a single +# process run but not necessarily across processes. +def hash_storage(storage: torch.UntypedStorage, *, stable_hash: bool = False) -> str: + import torch._dynamo + from torch._dynamo.utils import is_compile_supported + + device_type = storage.device.type + if stable_hash or not is_compile_supported(device_type): + cpu_storage = storage.cpu() + # TODO: make storage support buffer protocol so this isn't + # necessary + buf = (ctypes.c_byte * cpu_storage.nbytes()).from_address( + cpu_storage.data_ptr() + ) + sha1 = hashlib.sha1(usedforsecurity=False) + sha1.update(buf) + return sha1.hexdigest() + + # TODO: factor this into a random utility + if device_type == "cpu": + generator = torch._C.default_generator + elif device_type == "cuda": + generator = torch.cuda.default_generators[storage.device.index] + elif device_type == "mps": + generator = torch.mps._get_default_mps_generator() + elif device_type == "xpu": + generator = torch.xpu.default_generators[storage.device.index] + else: + raise AssertionError(f"unhandled device type {device_type}") + state = generator.get_state() + try: + generator.manual_seed(0) + x = torch.empty(0, dtype=torch.uint8, device=storage.device).set_(storage) # type: ignore[call-overload] + # The dtype-casting view cannot be compiled, and so the + # padding/reshaping also needs to be done externally even + # though it could be profitably fused + pad = -x.numel() % 4 + if pad > 0: + x = F.pad(x, (0, pad), "constant", 0) + x = x.view(torch.int32) + # We run the 32-bit hash five times with differing parameters to + # reduce chance of collision + ITER = 5 + cs = [hash_storage_kernel(x).item() for _ in range(ITER)] + return struct.pack(">" + "i" * ITER, *cs).hex() + finally: + generator.set_state(state) + + +class ContentStoreWriter: + # Structure: + # storages/ + # 00/ + # 0000..00 + # tensors/ + # name + def __init__(self, loc: str, stable_hash: bool = False) -> None: + self.loc: str = loc + self.seen_storage_hashes: set[str] = set() + self.stable_hash = stable_hash + + # TODO: offer some sort of non-blocking API to speed things up + def write_storage(self, storage: torch.UntypedStorage) -> str: + h = hash_storage(storage, stable_hash=self.stable_hash) + if h in self.seen_storage_hashes: + return h + # TODO: consider not using torch.save for this; we don't actually + # need any metadata for the storage + subfolder = os.path.join(self.loc, "storages") + os.makedirs(subfolder, exist_ok=True) + target = os.path.join(subfolder, h) + if os.path.exists(target): + return h + torch.save(storage, target) + self.seen_storage_hashes.add(h) + return h + + def compute_tensor_metadata(self, t: torch.Tensor, h=None): + if h is None: + h = hash_storage(t.untyped_storage(), stable_hash=self.stable_hash) + return ( + t.dtype, + h, + t.storage_offset(), + tuple(t.shape), + t.stride(), + torch._utils.get_tensor_metadata(t), + ) + + def write_tensor(self, name: str, t: torch.Tensor) -> None: + storage = t.untyped_storage() + h = self.write_storage(storage) + # TODO: Support more advanced snapshotting of requires_grad/grad/etc + d, f = os.path.split(name) + payload = self.compute_tensor_metadata(t, h=h) + subfolder = os.path.join(self.loc, "tensors", d) + os.makedirs(subfolder, exist_ok=True) + torch.save(payload, os.path.join(subfolder, f)) + + +class ContentStoreReader: + def __init__(self, loc: str, *, cache=True) -> None: + self.loc = loc + self.storage_cache: ( + dict[torch.device | None, dict[str, StorageWeakRef]] | None + ) = None + if cache: + self.storage_cache = defaultdict(dict) + + def read_storage(self, h: str, *, device=None) -> torch.UntypedStorage: + if device is not None: + device = torch.device(device) + ws = ( + self.storage_cache[device].get(h) + if self.storage_cache is not None + else None + ) + s: torch.UntypedStorage | None + if ws is not None: + s = torch.UntypedStorage._new_with_weak_ptr(ws.cdata) + if s is not None: + return s + s = torch.load( + os.path.join(self.loc, "storages", h), + weights_only=True, + map_location=device, + )._untyped_storage + if s is None: + raise AssertionError( + f"expected storage for hash {h} in {os.path.join(self.loc, 'storages')}, got None" + ) + if self.storage_cache is not None: + self.storage_cache[device][h] = StorageWeakRef(s) + return s + + def read_tensor_metadata(self, name: str): + fn = os.path.join(self.loc, "tensors", name) + if not os.path.exists(fn): + raise FileNotFoundError(fn) + return torch.load(fn, weights_only=True) + + def read_tensor(self, name: str, *, device=None) -> torch.Tensor: + dtype, h, storage_offset, size, stride, metadata = self.read_tensor_metadata( + name + ) + storage = self.read_storage(h, device=device) + t = torch.tensor([], dtype=dtype, device=storage.device) + t.set_(storage, storage_offset, size, stride) + torch._utils.set_tensor_metadata(t, metadata) + return t diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_contextlib.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_contextlib.py new file mode 100644 index 0000000000000000000000000000000000000000..408cdfe7d7b778c776ac88a050c3fe8c6a6edbb5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_contextlib.py @@ -0,0 +1,170 @@ +# mypy: allow-untyped-defs +# Extra utilities for working with context managers that should have been +# in the standard library but are not + +import functools +import inspect +import sys +import warnings +from collections.abc import Callable +from typing import Any, cast, overload, TypeVar +from typing_extensions import Self + + +# Used for annotating the decorator usage of _DecoratorContextManager (e.g., +# 'no_grad' and 'enable_grad'). +# See https://mypy.readthedocs.io/en/latest/generics.html#declaring-decorators +FuncType = Callable[..., Any] +F = TypeVar("F", bound=FuncType) + + +def _wrap_generator(ctx_factory, func): + """ + Wrap each generator invocation with the context manager factory. + + The input should be a function that returns a context manager, + not a context manager itself, to handle one-shot context managers. + """ + + @functools.wraps(func) + def generator_context(*args, **kwargs): + gen = func(*args, **kwargs) + + # Generators are suspended and unsuspended at `yield`, hence we + # make sure the grad mode is properly set every time the execution + # flow returns into the wrapped generator and restored when it + # returns through our `yield` to our caller (see PR #49017). + try: + # Issuing `None` to a generator fires it up + with ctx_factory(): + response = gen.send(None) + + while True: + try: + # Forward the response to our caller and get its next request + request = yield response + + except GeneratorExit: + # Inform the still active generator about its imminent closure + with ctx_factory(): + gen.close() + raise + + except BaseException: # noqa: B036 + # Propagate the exception thrown at us by the caller + with ctx_factory(): + response = gen.throw(*sys.exc_info()) + + else: + # Pass the last request to the generator and get its response + with ctx_factory(): + response = gen.send(request) + + # We let the exceptions raised above by the generator's `.throw` or + # `.send` methods bubble up to our caller, except for StopIteration + except StopIteration as e: + # The generator informed us that it is done: take whatever its + # returned value (if any) was and indicate that we're done too + # by returning it (see docs for python's return-statement). + return e.value + + return generator_context + + +def context_decorator(ctx, func): + """ + Like contextlib.ContextDecorator. + + But with the following differences: + 1. Is done by wrapping, rather than inheritance, so it works with context + managers that are implemented from C and thus cannot easily inherit from + Python classes + 2. Wraps generators in the intuitive way (c.f. https://bugs.python.org/issue37743) + 3. Errors out if you try to wrap a class, because it is ambiguous whether + or not you intended to wrap only the constructor + + The input argument can either be a context manager (in which case it must + be a multi-shot context manager that can be directly invoked multiple times) + or a callable that produces a context manager. + """ + if callable(ctx) and hasattr(ctx, "__enter__"): + raise AssertionError( + f"Passed in {ctx} is both callable and also a valid context manager " + "(has __enter__), making it ambiguous which interface to use. If you " + "intended to pass a context manager factory, rewrite your call as " + "context_decorator(lambda: ctx()); if you intended to pass a context " + "manager directly, rewrite your call as context_decorator(lambda: ctx)" + ) + + if not callable(ctx): + + def ctx_factory(): + return ctx + + else: + ctx_factory = ctx + + if inspect.isclass(func): + raise RuntimeError( + "Cannot decorate classes; it is ambiguous whether or not only the " + "constructor or all methods should have the context manager applied; " + "additionally, decorating a class at definition-site will prevent " + "use of the identifier as a conventional type. " + "To specify which methods to decorate, decorate each of them " + "individually." + ) + + if inspect.isgeneratorfunction(func): + return _wrap_generator(ctx_factory, func) + + @functools.wraps(func) + def decorate_context(*args, **kwargs): + # pyrefly: ignore [bad-context-manager] + with ctx_factory(): + return func(*args, **kwargs) + + return decorate_context + + +class _DecoratorContextManager: + """Allow a context manager to be used as a decorator.""" + + def __call__(self, orig_func: F) -> F: + if inspect.isclass(orig_func): + warnings.warn( + "Decorating classes is deprecated and will be disabled in " + "future versions. You should only decorate functions or methods. " + "To preserve the current behavior of class decoration, you can " + "directly decorate the `__init__` method and nothing else.", + FutureWarning, + stacklevel=2, + ) + func = cast(F, lambda *args, **kwargs: orig_func(*args, **kwargs)) + else: + func = orig_func + + return cast(F, context_decorator(self.clone, func)) + + def __enter__(self) -> None: + raise NotImplementedError + + def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None: + raise NotImplementedError + + def clone(self): + # override this method if your children class takes __init__ parameters + return self.__class__() + + +class _NoParamDecoratorContextManager(_DecoratorContextManager): + """Allow a context manager to be used as a decorator without parentheses.""" + + @overload + def __new__(cls, orig_func: F) -> F: ... # type: ignore[misc] + @overload + def __new__(cls, orig_func: None = None) -> Self: ... + + def __new__(cls, orig_func: F | None = None) -> Self | F: # type: ignore[misc] + if orig_func is None: + return super().__new__(cls) + return cls()(orig_func) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_cpp_embed_headers.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_cpp_embed_headers.py new file mode 100644 index 0000000000000000000000000000000000000000..88ab41aadffe75dd1bdbe0b3482b414f38b7e093 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_cpp_embed_headers.py @@ -0,0 +1,57 @@ +from collections.abc import Sequence +from pathlib import Path +from re import match as _match + + +def read_file(fname: Path | str) -> list[str]: + with open(fname, encoding="utf-8") as f: + return f.readlines() + + +def _embed_headers( + content: list[str], include_dirs: list[Path], processed_files: set[str] +) -> str: + for line_idx, cur_line in enumerate(content): + # Eliminate warning: `#pragma once in main file` + if cur_line.startswith("#pragma once"): + content[line_idx] = "" + continue + m = _match('^\\s*#include\\s*[<"]([^>"]+)[>"]', cur_line) + if m is None: + continue + for include_dir in include_dirs: + path = include_dir / m[1] + if not path.exists(): + continue + if str(path) in processed_files: + content[line_idx] = "" + continue + processed_files.add(str(path)) + content[line_idx] = _embed_headers( + read_file(path), include_dirs, processed_files + ) + break + return "".join(content) + + +def embed_headers( + fname: str, include_dirs: Sequence[str] | Sequence[Path] | str | None = None +) -> str: + if include_dirs is None: + base_dir = Path(__file__).parent.parent.parent + include_dirs = [base_dir, base_dir / "aten" / "src"] + elif isinstance(include_dirs, str): + include_dirs = [Path(include_dirs)] + else: + include_dirs = [Path(x) for x in include_dirs] + + return _embed_headers(read_file(fname), include_dirs, {fname}) + + +if __name__ == "__main__": + import sys + + if len(sys.argv) < 2: + print(f"Usage:\n {sys.argv[0]} filename") + sys.exit(1) + print(embed_headers(sys.argv[1])) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_cpp_extension_versioner.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_cpp_extension_versioner.py new file mode 100644 index 0000000000000000000000000000000000000000..d1391dd9aaab0e920276c7046282ea1002346a15 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_cpp_extension_versioner.py @@ -0,0 +1,63 @@ +# mypy: allow-untyped-defs +import collections + + +Entry = collections.namedtuple("Entry", "version, hash") + + +def update_hash(seed, value): + # Good old boost::hash_combine + # https://www.boost.org/doc/libs/1_35_0/doc/html/boost/hash_combine_id241013.html + return seed ^ (hash(value) + 0x9E3779B9 + (seed << 6) + (seed >> 2)) + + +def hash_source_files(hash_value, source_files): + for filename in source_files: + with open(filename, "rb") as file: + hash_value = update_hash(hash_value, file.read()) + return hash_value + + +def hash_build_arguments(hash_value, build_arguments): + for group in build_arguments: + if group: + for argument in group: + hash_value = update_hash(hash_value, argument) + return hash_value + + +class ExtensionVersioner: + def __init__(self) -> None: + self.entries = {} + + def get_version(self, name): + entry = self.entries.get(name) + return None if entry is None else entry.version + + def bump_version_if_changed( + self, + name, + source_files, + build_arguments, + build_directory, + with_cuda, + with_sycl, + is_python_module, + is_standalone, + ): + hash_value = 0 + hash_value = hash_source_files(hash_value, source_files) + hash_value = hash_build_arguments(hash_value, build_arguments) + hash_value = update_hash(hash_value, build_directory) + hash_value = update_hash(hash_value, with_cuda) + hash_value = update_hash(hash_value, with_sycl) + hash_value = update_hash(hash_value, is_python_module) + hash_value = update_hash(hash_value, is_standalone) + + entry = self.entries.get(name) + if entry is None: + self.entries[name] = entry = Entry(0, hash_value) + elif hash_value != entry.hash: + self.entries[name] = entry = Entry(entry.version + 1, hash_value) + + return entry.version diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_cxx_pytree.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_cxx_pytree.py new file mode 100644 index 0000000000000000000000000000000000000000..3c6f79bfe2243be8563f16787aec860366896088 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_cxx_pytree.py @@ -0,0 +1,1140 @@ +""" +Contains utility functions for working with nested python data structures. + +A *pytree* is Python nested data structure. It is a tree in the sense that +nodes are Python collections (e.g., list, tuple, dict) and the leaves are +Python values. Furthermore, a pytree should not contain reference cycles. + +pytrees are useful for working with nested collections of Tensors. For example, +one can use `tree_map` to map a function over all Tensors inside some nested +collection of Tensors and `tree_leaves` to get a flat list of all Tensors +inside some nested collection. pytrees are helpful for implementing nested +collection support for PyTorch APIs. +""" + +import functools +import sys +import types +from collections.abc import Callable, Iterable, Mapping +from typing import Any, overload, TypeAlias, TypeVar +from typing_extensions import deprecated, Self, TypeIs + +import torch.utils._pytree as python_pytree +from torch.torch_version import TorchVersion as _TorchVersion +from torch.utils._pytree import ( + is_namedtuple, + is_namedtuple_class, + is_namedtuple_instance, + is_structseq, + is_structseq_class, + is_structseq_instance, + KeyEntry, +) + + +# Do not try to import `optree` package if the static version check already fails. +if not python_pytree._cxx_pytree_dynamo_traceable: + raise ImportError( + f"{__name__} depends on `optree>={python_pytree._optree_minimum_version}`, " + "which is an optional dependency of PyTorch. " + "To use it, please upgrade your optree package via " + "`python3 -m pip install --upgrade optree`" + ) + + +import optree +from optree import PyTreeSpec # direct import for type annotations + + +__all__ = [ + "PyTree", + "Context", + "FlattenFunc", + "UnflattenFunc", + "DumpableContext", + "ToDumpableContextFn", + "FromDumpableContextFn", + "PyTreeSpec", + "TreeSpec", + "LeafSpec", + "keystr", + "key_get", + "register_pytree_node", + "tree_is_leaf", + "tree_flatten", + "tree_flatten_with_path", + "tree_unflatten", + "tree_iter", + "tree_leaves", + "tree_leaves_with_path", + "tree_structure", + "tree_map", + "tree_map_with_path", + "tree_map_", + "tree_map_only", + "tree_map_only_", + "tree_all", + "tree_any", + "tree_all_only", + "tree_any_only", + "treespec_dumps", + "treespec_loads", + "treespec_pprint", + "is_namedtuple", + "is_namedtuple_class", + "is_namedtuple_instance", + "is_structseq", + "is_structseq_class", + "is_structseq_instance", +] + + +# In-tree installation may have VCS-based versioning. Update the previous static version. +python_pytree._optree_version = _TorchVersion(optree.__version__) # type: ignore[attr-defined] + +__TORCH_DICT_SESSION = optree.dict_insertion_ordered(True, namespace="torch") +__TORCH_DICT_SESSION.__enter__() # enable globally and permanently + + +T = TypeVar("T") +S = TypeVar("S") +U = TypeVar("U") +R = TypeVar("R") + + +TreeSpec: TypeAlias = PyTreeSpec + +Context = Any +PyTree = Any +FlattenFunc = Callable[[PyTree], tuple[list[Any], Context]] +UnflattenFunc = Callable[[Iterable[Any], Context], PyTree] +OpTreeUnflattenFunc = Callable[[Context, Iterable[Any]], PyTree] +DumpableContext = Any # Any json dumpable text +ToDumpableContextFn = Callable[[Context], DumpableContext] +FromDumpableContextFn = Callable[[DumpableContext], Context] +KeyPath = tuple[KeyEntry, ...] +FlattenWithKeysFunc = Callable[[PyTree], tuple[list[tuple[KeyEntry, Any]], Any]] + + +def _reverse_args(func: UnflattenFunc) -> OpTreeUnflattenFunc: + @functools.wraps(func) + def wrapped(*args: Any, **kwargs: Any) -> Any: + return func(*reversed(args), **kwargs) + + return wrapped + + +def register_pytree_node( + cls: type[Any], + flatten_fn: FlattenFunc, + unflatten_fn: UnflattenFunc, + *, + serialized_type_name: str | None = None, + to_dumpable_context: ToDumpableContextFn | None = None, + from_dumpable_context: FromDumpableContextFn | None = None, + flatten_with_keys_fn: FlattenWithKeysFunc | None = None, +) -> None: + """Register a container-like type as pytree node. + + Args: + cls (type): A Python type to treat as an internal pytree node. + flatten_fn (callable): A function to be used during flattening, taking an instance of + ``cls`` and returning a pair, with (1) an iterable for the children to be flattened + recursively, and (2) some hashable auxiliary data to be stored in the treespec and to be + passed to the ``unflatten_fn``. + unflatten_fn (callable): A function taking two arguments: the auxiliary data that was + returned by ``flatten_fn`` and stored in the treespec, and the unflattened children. + The function should return an instance of ``cls``. + serialized_type_name (str, optional): A keyword argument used to specify the fully + qualified name used when serializing the tree spec. + to_dumpable_context (callable, optional): An optional keyword argument to custom specify how + to convert the context of the pytree to a custom json dumpable representation. This is + used for json serialization, which is being used in :mod:`torch.export` right now. + from_dumpable_context (callable, optional): An optional keyword argument to custom specify + how to convert the custom json dumpable representation of the context back to the + original context. This is used for json deserialization, which is being used in + :mod:`torch.export` right now. + + Example:: + + >>> # xdoctest: +SKIP + >>> # Registry a Python type with lambda functions + >>> register_pytree_node( + ... set, + ... lambda s: (sorted(s), None, None), + ... lambda children, _: set(children), + ... ) + """ + if flatten_with_keys_fn is not None: + raise NotImplementedError("KeyPaths are not yet supported in cxx_pytree.") + + _private_register_pytree_node( + cls, + flatten_fn, + unflatten_fn, + serialized_type_name=serialized_type_name, + to_dumpable_context=to_dumpable_context, + from_dumpable_context=from_dumpable_context, + ) + + python_pytree._private_register_pytree_node( + cls, + flatten_fn, + unflatten_fn, + serialized_type_name=serialized_type_name, + to_dumpable_context=to_dumpable_context, + from_dumpable_context=from_dumpable_context, + ) + + +@deprecated( + "`torch.utils._cxx_pytree._register_pytree_node` is deprecated. " + "Please use `torch.utils._cxx_pytree.register_pytree_node` instead.", + category=FutureWarning, +) +def _register_pytree_node( + cls: type[Any], + flatten_fn: FlattenFunc, + unflatten_fn: UnflattenFunc, + *, + serialized_type_name: str | None = None, + to_dumpable_context: ToDumpableContextFn | None = None, + from_dumpable_context: FromDumpableContextFn | None = None, +) -> None: + """Register a container-like type as pytree node for the C++ pytree only. + + The ``namespace`` argument is used to avoid collisions that occur when different libraries + register the same Python type with different behaviors. It is recommended to add a unique prefix + to the namespace to avoid conflicts with other libraries. Namespaces can also be used to specify + the same class in different namespaces for different use cases. + + .. warning:: + For safety reasons, a ``namespace`` must be specified while registering a custom type. It is + used to isolate the behavior of flattening and unflattening a pytree node type. This is to + prevent accidental collisions between different libraries that may register the same type. + + Args: + cls (type): A Python type to treat as an internal pytree node. + flatten_fn (callable): A function to be used during flattening, taking an instance of + ``cls`` and returning a pair, with (1) an iterable for the children to be flattened + recursively, and (2) some hashable auxiliary data to be stored in the treespec and to be + passed to the ``unflatten_fn``. + unflatten_fn (callable): A function taking two arguments: the auxiliary data that was + returned by ``flatten_fn`` and stored in the treespec, and the unflattened children. + The function should return an instance of ``cls``. + serialized_type_name (str, optional): A keyword argument used to specify the fully + qualified name used when serializing the tree spec. + to_dumpable_context (callable, optional): An optional keyword argument to custom specify how + to convert the context of the pytree to a custom json dumpable representation. This is + used for json serialization, which is being used in :mod:`torch.export` right now. + from_dumpable_context (callable, optional): An optional keyword argument to custom specify + how to convert the custom json dumpable representation of the context back to the + original context. This is used for json deserialization, which is being used in + :mod:`torch.export` right now. + """ + + _private_register_pytree_node( + cls, + flatten_fn, + unflatten_fn, + serialized_type_name=serialized_type_name, + to_dumpable_context=to_dumpable_context, + from_dumpable_context=from_dumpable_context, + ) + + +def _private_register_pytree_node( + cls: type[Any], + flatten_fn: FlattenFunc, + unflatten_fn: UnflattenFunc, + *, + serialized_type_name: str | None = None, + to_dumpable_context: ToDumpableContextFn | None = None, + from_dumpable_context: FromDumpableContextFn | None = None, +) -> None: + """This is an internal function that is used to register a pytree node type + for the C++ pytree only. End-users should use :func:`register_pytree_node` + instead. + """ + # TODO(XuehaiPan): remove this condition when we make Python pytree out-of-box support + # PyStructSequence types + if not optree.is_structseq_class(cls): + optree.register_pytree_node( + cls, + flatten_fn, + _reverse_args(unflatten_fn), + namespace="torch", + ) + + +def _is_pytreespec_instance( + obj: Any, + /, +) -> TypeIs[TreeSpec | python_pytree.PyTreeSpec]: + if isinstance(obj, (TreeSpec, python_pytree.PyTreeSpec)): + return True + if "torch._dynamo.polyfills.pytree" in sys.modules: + # The PyTorch Dynamo pytree module is not always available, so we check if it is loaded. + # If the PyTorch Dynamo pytree module is loaded, we can check if the treespec + # is an instance of the PyTorch Dynamo TreeSpec class. + import torch._dynamo.polyfills.pytree as dynamo_pytree + + return isinstance(obj, dynamo_pytree.PyTreeSpec) + return False + + +def treespec_leaf() -> TreeSpec: + """Make a treespec representing a leaf node.""" + return optree.treespec_leaf(none_is_leaf=True, namespace="torch") + + +def treespec_tuple(iterable: Iterable[TreeSpec] = (), /) -> TreeSpec: + """Make a tuple treespec from an iterable of child treespecs.""" + return optree.treespec_tuple(iterable, none_is_leaf=True, namespace="torch") + + +def treespec_dict( + mapping: Mapping[Any, TreeSpec] | Iterable[tuple[Any, TreeSpec]] = (), + /, + **kwargs: TreeSpec, +) -> TreeSpec: + """Make a dict treespec from a dict of child treespecs.""" + return optree.treespec_dict( + mapping, + **kwargs, + none_is_leaf=True, + namespace="torch", + ) + + +def tree_is_leaf( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: + """Check if a pytree is a leaf. + + >>> tree_is_leaf(1) + True + >>> tree_is_leaf(None) + True + >>> tree_is_leaf([1, 2, 3]) + False + >>> tree_is_leaf((1, 2, 3), is_leaf=lambda x: isinstance(x, tuple)) + True + >>> tree_is_leaf({"a": 1, "b": 2, "c": 3}) + False + >>> tree_is_leaf({"a": 1, "b": 2, "c": None}) + False + + Args: + tree (pytree): A pytree to check if it is a leaf node. + is_leaf (callable, optional): An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + + Returns: + A boolean indicating if the pytree is a leaf node. + """ + return optree.tree_is_leaf( + tree, + is_leaf=is_leaf, + none_is_leaf=True, + namespace="torch", + ) + + +def tree_flatten( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> tuple[list[Any], TreeSpec]: + """Flatten a pytree. + + See also :func:`tree_unflatten`. + + The flattening order (i.e., the order of elements in the output list) is deterministic, + corresponding to a left-to-right depth-first tree traversal. + + >>> tree = {"b": (2, [3, 4]), "a": 1, "c": None, "d": 5} + >>> tree_flatten(tree) + ([2, 3, 4, 1, None, 5], PyTreeSpec({'b': (*, [*, *]), 'a': *, 'c': *, 'd': *}, NoneIsLeaf, namespace='torch')) + >>> tree_flatten(1) + ([1], PyTreeSpec(*, NoneIsLeaf, namespace='torch')) + >>> tree_flatten(None) + ([None], PyTreeSpec(*, NoneIsLeaf, namespace='torch')) + >>> from collections import OrderedDict + >>> tree = OrderedDict([("b", (2, [3, 4])), ("a", 1), ("c", None), ("d", 5)]) + >>> tree_flatten(tree) + ([2, 3, 4, 1, None, 5], PyTreeSpec(OrderedDict({'b': (*, [*, *]), 'a': *, 'c': *, 'd': *}), NoneIsLeaf, namespace='torch')) + + Args: + tree (pytree): A pytree to flatten. + is_leaf (callable, optional): An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + + Returns: + A pair ``(leaves, treespec)`` where the first element is a list of leaf values and the + second element is a treespec representing the structure of the pytree. + """ + return optree.tree_flatten( # type: ignore[return-value] + tree, + is_leaf=is_leaf, + none_is_leaf=True, + namespace="torch", + ) + + +def tree_unflatten(leaves: Iterable[Any], treespec: TreeSpec) -> PyTree: + """Reconstruct a pytree from the treespec and the leaves. + + The inverse of :func:`tree_flatten`. + + >>> tree = {"b": (2, [3, 4]), "a": 1, "c": None, "d": 5} + >>> leaves, treespec = tree_flatten(tree) + >>> tree == tree_unflatten(leaves, treespec) + True + + Args: + leaves (iterable): The list of leaves to use for reconstruction. The list must match the + number of leaves of the treespec. + treespec (TreeSpec): The treespec to reconstruct. + + Returns: + The reconstructed pytree, containing the ``leaves`` placed in the structure described by + ``treespec``. + """ + if not _is_pytreespec_instance(treespec): + if not _is_pytreespec_instance(leaves): + raise TypeError( + f"Expected `treespec` to be an instance of " + f"PyTreeSpec but got item of type {type(treespec)}." + ) + # Allow passing the PyTreeSpec instance as the first argument + leaves, treespec = treespec, leaves + return treespec.unflatten(leaves) + + +def tree_iter( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> Iterable[Any]: + """Get an iterator over the leaves of a pytree. + + See also :func:`tree_flatten`. + + >>> tree = {"b": (2, [3, 4]), "a": 1, "c": None, "d": 5} + >>> list(tree_iter(tree)) + [2, 3, 4, 1, None, 5] + >>> list(tree_iter(1)) + [1] + >>> list(tree_iter(None)) + [None] + + Args: + tree (pytree): A pytree to flatten. + is_leaf (callable, optional): An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + + Returns: + An iterator over the leaf values. + """ + return optree.tree_iter( + tree, + is_leaf=is_leaf, + none_is_leaf=True, + namespace="torch", + ) + + +def tree_leaves( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> list[Any]: + """Get the leaves of a pytree. + + See also :func:`tree_flatten`. + + >>> tree = {"b": (2, [3, 4]), "a": 1, "c": None, "d": 5} + >>> tree_leaves(tree) + [2, 3, 4, 1, None, 5] + >>> tree_leaves(1) + [1] + >>> tree_leaves(None) + [None] + + Args: + tree (pytree): A pytree to flatten. + is_leaf (callable, optional): An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + + Returns: + A list of leaf values. + """ + return optree.tree_leaves( + tree, + is_leaf=is_leaf, + none_is_leaf=True, + namespace="torch", + ) + + +def tree_structure( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> TreeSpec: + """Get the treespec for a pytree. + + See also :func:`tree_flatten`. + + >>> tree = {"b": (2, [3, 4]), "a": 1, "c": None, "d": 5} + >>> tree_structure(tree) + PyTreeSpec({'b': (*, [*, *]), 'a': *, 'c': *, 'd': *}, NoneIsLeaf, namespace='torch') + >>> tree_structure(1) + PyTreeSpec(*, NoneIsLeaf, namespace='torch') + >>> tree_structure(None) + PyTreeSpec(*, NoneIsLeaf, namespace='torch') + + Args: + tree (pytree): A pytree to flatten. + is_leaf (callable, optional): An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + + Returns: + A treespec object representing the structure of the pytree. + """ + return optree.tree_structure( # type: ignore[return-value] + tree, + is_leaf=is_leaf, + none_is_leaf=True, + namespace="torch", + ) + + +def tree_map( + func: Callable[..., Any], + tree: PyTree, + *rests: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: + """Map a multi-input function over pytree args to produce a new pytree. + + See also :func:`tree_map_`. + + >>> tree_map(lambda x: x + 1, {"x": 7, "y": (42, 64)}) + {'x': 8, 'y': (43, 65)} + >>> tree_map(lambda x: x is None, {"x": 7, "y": (42, 64), "z": None}) + {'x': False, 'y': (False, False), 'z': True} + + If multiple inputs are given, the structure of the tree is taken from the first input; + subsequent inputs need only have ``tree`` as a prefix: + + >>> tree_map(lambda x, y: [x] + y, [5, 6], [[7, 9], [1, 2]]) + [[5, 7, 9], [6, 1, 2]] + + Args: + func (callable): A function that takes ``1 + len(rests)`` arguments, to be applied at the + corresponding leaves of the pytrees. + tree (pytree): A pytree to be mapped over, with each leaf providing the first positional + argument to function ``func``. + rests (tuple of pytree): A tuple of pytrees, each of which has the same structure as + ``tree`` or has ``tree`` as a prefix. + is_leaf (callable, optional): An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + + Returns: + A new pytree with the same structure as ``tree`` but with the value at each leaf given by + ``func(x, *xs)`` where ``x`` is the value at the corresponding leaf in ``tree`` and ``xs`` + is the tuple of values at corresponding nodes in ``rests``. + """ + return optree.tree_map( + func, + tree, + *rests, + is_leaf=is_leaf, + none_is_leaf=True, + namespace="torch", + ) + + +def tree_map_( + func: Callable[..., Any], + tree: PyTree, + *rests: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: + """Like :func:`tree_map`, but do an inplace call on each leaf and return the original tree. + + See also :func:`tree_map`. + + Args: + func (callable): A function that takes ``1 + len(rests)`` arguments, to be applied at the + corresponding leaves of the pytrees. + tree (pytree): A pytree to be mapped over, with each leaf providing the first positional + argument to function ``func``. + rests (tuple of pytree): A tuple of pytrees, each of which has the same structure as + ``tree`` or has ``tree`` as a prefix. + is_leaf (callable, optional): An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + + Returns: + The original ``tree`` with the value at each leaf is given by the side-effect of function + ``func(x, *xs)`` (not the return value) where ``x`` is the value at the corresponding leaf + in ``tree`` and ``xs`` is the tuple of values at values at corresponding nodes in ``rests``. + """ + return optree.tree_map_( + func, + tree, + *rests, + is_leaf=is_leaf, + none_is_leaf=True, + namespace="torch", + ) + + +Type2 = tuple[type[T], type[S]] +Type3 = tuple[type[T], type[S], type[U]] +TypeAny = type[Any] | tuple[type[Any], ...] | types.UnionType + +Fn2 = Callable[[T | S], R] +Fn3 = Callable[[T | S | U], R] +Fn = Callable[[T], R] +FnAny = Callable[[Any], R] + +MapOnlyFn = Callable[[T], Callable[[Any], Any]] + + +# These specializations help with type inference on the lambda passed to this +# function +@overload +def map_only(type_or_types_or_pred: type[T], /) -> MapOnlyFn[Fn[T, Any]]: ... + + +@overload +def map_only(type_or_types_or_pred: Type2[T, S], /) -> MapOnlyFn[Fn2[T, S, Any]]: ... + + +@overload +def map_only( + type_or_types_or_pred: Type3[T, S, U], / +) -> MapOnlyFn[Fn3[T, S, U, Any]]: ... + + +# This specialization is needed for the implementations below that call +@overload +def map_only(type_or_types_or_pred: TypeAny, /) -> MapOnlyFn[FnAny[Any]]: ... + + +@overload +def map_only( + type_or_types_or_pred: Callable[[Any], bool], / +) -> MapOnlyFn[FnAny[Any]]: ... + + +def map_only( + type_or_types_or_pred: TypeAny | Callable[[Any], bool], / +) -> MapOnlyFn[FnAny[Any]]: + """ + Suppose you are writing a tree_map over tensors, leaving everything + else unchanged. Ordinarily you would have to write: + + def go(t): + if isinstance(t, Tensor): + return ... + else: + return t + + With this function, you only need to write: + + @map_only(Tensor) + def go(t): + return ... + + You can also directly use 'tree_map_only' + """ + if isinstance(type_or_types_or_pred, (type, tuple, types.UnionType)): + + def pred(x: Any) -> bool: + return isinstance(x, type_or_types_or_pred) # type: ignore[arg-type] + + elif callable(type_or_types_or_pred): + pred = type_or_types_or_pred # type: ignore[assignment] + else: + raise TypeError("Argument must be a type, a tuple of types, or a callable.") + + def wrapper(func: Callable[[T], Any]) -> Callable[[Any], Any]: + @functools.wraps(func) + def wrapped(x: T) -> Any: + if pred(x): + return func(x) + return x + + return wrapped + + return wrapper + + +@overload +def tree_map_only( + type_or_types_or_pred: type[T], + /, + func: Fn[T, Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only( + type_or_types_or_pred: Type2[T, S], + /, + func: Fn2[T, S, Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only( + type_or_types_or_pred: Type3[T, S, U], + /, + func: Fn3[T, S, U, Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only( + type_or_types_or_pred: TypeAny, + /, + func: FnAny[Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only( + type_or_types_or_pred: Callable[[Any], bool], + /, + func: FnAny[Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +def tree_map_only( + type_or_types_or_pred: TypeAny | Callable[[Any], bool], + /, + func: FnAny[Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: + return tree_map(map_only(type_or_types_or_pred)(func), tree, is_leaf=is_leaf) + + +@overload +def tree_map_only_( + type_or_types_or_pred: type[T], + /, + func: Fn[T, Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only_( + type_or_types_or_pred: Type2[T, S], + /, + func: Fn2[T, S, Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only_( + type_or_types_or_pred: Type3[T, S, U], + /, + func: Fn3[T, S, U, Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only_( + type_or_types_or_pred: TypeAny, + /, + func: FnAny[Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only_( + type_or_types_or_pred: Callable[[Any], bool], + /, + func: FnAny[Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +def tree_map_only_( + type_or_types_or_pred: TypeAny | Callable[[Any], bool], + /, + func: FnAny[Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: + return tree_map_(map_only(type_or_types_or_pred)(func), tree, is_leaf=is_leaf) + + +def tree_all( + pred: Callable[[Any], bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: + flat_args = tree_iter(tree, is_leaf=is_leaf) + return all(map(pred, flat_args)) + + +def tree_any( + pred: Callable[[Any], bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: + flat_args = tree_iter(tree, is_leaf=is_leaf) + return any(map(pred, flat_args)) + + +@overload +def tree_all_only( + type_or_types: type[T], + /, + pred: Fn[T, bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: ... + + +@overload +def tree_all_only( + type_or_types: Type2[T, S], + /, + pred: Fn2[T, S, bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: ... + + +@overload +def tree_all_only( + type_or_types: Type3[T, S, U], + /, + pred: Fn3[T, S, U, bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: ... + + +def tree_all_only( + type_or_types: TypeAny, + /, + pred: FnAny[bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: + flat_args = tree_iter(tree, is_leaf=is_leaf) + return all(pred(x) for x in flat_args if isinstance(x, type_or_types)) + + +@overload +def tree_any_only( + type_or_types: type[T], + /, + pred: Fn[T, bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: ... + + +@overload +def tree_any_only( + type_or_types: Type2[T, S], + /, + pred: Fn2[T, S, bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: ... + + +@overload +def tree_any_only( + type_or_types: Type3[T, S, U], + /, + pred: Fn3[T, S, U, bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: ... + + +def tree_any_only( + type_or_types: TypeAny, + /, + pred: FnAny[bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: + flat_args = tree_iter(tree, is_leaf=is_leaf) + return any(pred(x) for x in flat_args if isinstance(x, type_or_types)) + + +def broadcast_prefix( + prefix_tree: PyTree, + full_tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> list[Any]: + """Return a list of broadcasted leaves in ``prefix_tree`` to match the number of leaves in ``full_tree``. + + If a ``prefix_tree`` is a prefix of a ``full_tree``, this means the ``full_tree`` can be + constructed by replacing the leaves of ``prefix_tree`` with appropriate **subtrees**. + + This function returns a list of leaves with the same size as ``full_tree``. The leaves are + replicated from ``prefix_tree``. The number of replicas is determined by the corresponding + subtree in ``full_tree``. + + >>> broadcast_prefix(1, [1, 2, 3]) + [1, 1, 1] + >>> broadcast_prefix([1, 2, 3], [1, 2, 3]) + [1, 2, 3] + >>> broadcast_prefix([1, 2, 3], [1, 2, 3, 4]) + Traceback (most recent call last): + ... + ValueError: list arity mismatch; expected: 3, got: 4; list: [1, 2, 3, 4]. + >>> broadcast_prefix([1, 2, 3], [1, 2, (3, 4)]) + [1, 2, 3, 3] + >>> broadcast_prefix([1, 2, 3], [1, 2, {"a": 3, "b": 4, "c": (None, 5)}]) + [1, 2, 3, 3, 3, 3] + + Args: + prefix_tree (pytree): A pytree with the same structure as a prefix of ``full_tree``. + full_tree (pytree): A pytree with the same structure as a suffix of ``prefix_tree``. + is_leaf (callable, optional): An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + + Returns: + A list of leaves in ``prefix_tree`` broadcasted to match the number of leaves in ``full_tree``. + """ + result: list[Any] = [] + + def add_leaves(x: Any, subtree: PyTree) -> None: + subtreespec = tree_structure(subtree, is_leaf=is_leaf) + result.extend([x] * subtreespec.num_leaves) + + tree_map_( + add_leaves, + prefix_tree, + full_tree, + is_leaf=is_leaf, + ) + return result + + +# Broadcasts a pytree to the provided TreeSpec and returns the flattened +# values. If this is not possible, then this function returns None. +# +# For example, given pytree=0 and spec=TreeSpec(list, None, [LeafSpec(), LeafSpec()]), +# would return [0, 0]. This is useful for part of the vmap implementation: +# a user can pass in vmap(fn, in_dims)(*inputs). `in_dims` should be +# broadcastable to the tree structure of `inputs` and we use +# _broadcast_to_and_flatten to check this. +def _broadcast_to_and_flatten( + tree: PyTree, + treespec: TreeSpec, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> list[Any] | None: + if not _is_pytreespec_instance(treespec): + raise TypeError( + f"Expected `treespec` to be an instance of " + f"PyTreeSpec but got item of type {type(treespec)}." + ) + full_tree = tree_unflatten([0] * treespec.num_leaves, treespec) + try: + return broadcast_prefix(tree, full_tree, is_leaf=is_leaf) + except ValueError: + return None + + +def treespec_dumps(treespec: TreeSpec, protocol: int | None = None) -> str: + """Serialize a treespec to a JSON string.""" + if not _is_pytreespec_instance(treespec): + raise TypeError( + f"Expected `treespec` to be an instance of " + f"PyTreeSpec but got item of type {type(treespec)}." + ) + + dummy_tree = tree_unflatten([0] * treespec.num_leaves, treespec) + orig_treespec = python_pytree.tree_structure(dummy_tree) + return python_pytree.treespec_dumps(orig_treespec, protocol=protocol) + + +@functools.lru_cache +def treespec_loads(serialized: str) -> TreeSpec: + """Deserialize a treespec from a JSON string.""" + orig_treespec = python_pytree.treespec_loads(serialized) + dummy_tree = python_pytree.tree_unflatten( + [0] * orig_treespec.num_leaves, + orig_treespec, + ) + treespec = tree_structure(dummy_tree) + return treespec + + +class _DummyLeaf: + def __repr__(self) -> str: + return "*" + + +def treespec_pprint(treespec: TreeSpec) -> str: + dummy_tree = tree_unflatten( + [_DummyLeaf() for _ in range(treespec.num_leaves)], + treespec, + ) + return repr(dummy_tree) + + +class LeafSpecMeta(type(TreeSpec)): # type: ignore[misc] + def __instancecheck__(self, instance: object) -> bool: + return _is_pytreespec_instance(instance) and instance.is_leaf() + + +@deprecated( + "`isinstance(treespec, LeafSpec)` is deprecated, " + "use `isinstance(treespec, TreeSpec)` and `treespec.is_leaf()` instead.", + category=FutureWarning, +) +class LeafSpec(TreeSpec, metaclass=LeafSpecMeta): # type: ignore[misc,final] + def __new__(cls) -> Self: + return treespec_leaf() # type: ignore[return-value] + + +def tree_flatten_with_path( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> tuple[list[tuple[KeyPath, Any]], TreeSpec]: + """Flattens a pytree like :func:`tree_flatten`, but also returns each leaf's key path. + + Args: + tree: a pytree to flatten. If it contains a custom type, that type must be + registered with an appropriate `tree_flatten_with_path_fn` when registered + with :func:`register_pytree_node`. + is_leaf: An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + Returns: + A tuple where the first element is a list of (key path, leaf) pairs, and the + second element is a :class:`TreeSpec` representing the structure of the flattened + tree. + """ + raise NotImplementedError("KeyPaths are not yet supported in cxx_pytree.") + + +def tree_leaves_with_path( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> list[tuple[KeyPath, Any]]: + """Gets the leaves of a pytree like ``tree_leaves`` and returns each leaf's key path. + + Args: + tree: a pytree. If it contains a custom type, that type must be + registered with an appropriate `tree_flatten_with_path_fn` when registered + with :func:`register_pytree_node`. + is_leaf: An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + Returns: + A list of (key path, leaf) pairs. + """ + raise NotImplementedError("KeyPaths are not yet supported in cxx_pytree.") + + +def tree_map_with_path( + func: Callable[..., Any], + tree: PyTree, + *rests: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: + """Like :func:`tree_map`, but the provided callable takes an additional key path argument. + + Args: + func: A function that takes ``2 + len(rests)`` arguments, to be applied at the + corresponding leaves of the pytrees. The first positional argument + to ``func`` is the key path of the leaf in question. The second + positional argument is the value of the leaf. + tree: A pytree to be mapped over, with each leaf providing the first positional + argument to function ``func``. + rests: A tuple of pytrees, each of which has the same structure as + ``tree`` or has ``tree`` as a prefix. + is_leaf: An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + + Returns + A new pytree with the same structure as ``tree`` but with the value at each leaf given by + ``func(keypath, x, *xs)`` where ``keypath`` is the key path at the + corresponding leaf in ``tree``, ``x`` is the value at that leaf, and + ``xs`` is the tuple of values at corresponding nodes in ``rests``. + """ + raise NotImplementedError("KeyPaths are not yet supported in cxx_pytree.") + + +def keystr(kp: KeyPath) -> str: + """Given a key path, return a pretty-printed representation.""" + raise NotImplementedError("KeyPaths are not yet supported in cxx_pytree.") + + +def key_get(obj: Any, kp: KeyPath) -> Any: + """Given an object and a key path, return the value at the key path.""" + raise NotImplementedError("KeyPaths are not yet supported in cxx_pytree.") + + +with python_pytree._NODE_REGISTRY_LOCK: + # pyrefly: ignore [bad-assignment] + python_pytree._cxx_pytree_imported = True + args, kwargs = (), {} # type: ignore[var-annotated] + for args, kwargs in python_pytree._cxx_pytree_pending_imports: + _private_register_pytree_node(*args, **kwargs) + python_pytree._cxx_pytree_pending_imports.clear() + del args, kwargs diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_debug_mode.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_debug_mode.py new file mode 100644 index 0000000000000000000000000000000000000000..ef8ccd0cc6bac71fd3a0611ed33b6bc8dc61ed82 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_debug_mode.py @@ -0,0 +1,1395 @@ +# mypy: allow-untyped-defs +""" +DebugMode is a debugging TorchDispatchMode that intercepts and logs runtime calls +to a hierarchical string dump. It logs real tensor, DTensor, and optionally FakeTensor +operations, with some additional handling for DTensor internals. + +An example dump from an eager mode DTensor matmul: + + torch.mm(dt$0: f32[8, 8]| S(0), dt$1: f32[8, 32]| S(0)) -> dt$6: f32[8, 32]| S(0) + aten::mm(dt$0: f32[8, 8]| S(0), dt$1: f32[8, 32]| S(0)) + redistribute_input(1, S(0) -> R) + redistribute_input(t$2: f32[1, 32], trace: S(0)->R) + _c10d_functional::all_gather_into_tensor(t$2: f32[1, 32], 8, 0) -> t$3: f32[8, 32] + _c10d_functional::wait_tensor(t$3: f32[8, 32]) -> t$3: f32[8, 32] + aten::mm(t$4: f32[1, 8], t$3: f32[8, 32]) -> t$5: f32[1, 32] + +This mode runs "under" compile, which means it hides itself during compilation, and is re-enabled +at runtime, and DebugMode-related operations won't show up in the compiled region. +DebugMode also provides some visibility into non-torch-dispatch calls (e.g. DTensor redistribute calls, +inductor-generated triton kernels), but requires special handling for these, since dispatch modes +can't intercept them by default. + +The mode also provides some extensions for custom debugging (e.g. adding custom dispatch call hooks +via dispatch_hooks), or numerics debugging (e.g. tensor hashing for bitwise equivalence/closeness, +via log_tensor_hashes). These decorators allow annotating string dumps with additional per-call information, +for any region of runtime code. + +Usage:: + + with DebugMode() as debug_mode: + result = some_pytorch_operation(tensor_input) + print(debug_mode.debug_string()) +""" + +import contextlib +import functools +import inspect +import logging +import os +import traceback +import weakref +from collections.abc import Callable +from typing import Any, TYPE_CHECKING + +import torch +from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode +from torch.fx.graph import _parse_stack_trace +from torch.utils._dtype_abbrs import dtype_abbrs +from torch.utils._python_dispatch import ( + _get_current_dispatch_mode, + _get_current_dispatch_mode_stack, + TorchDispatchMode, +) +from torch.utils._pytree import keystr, tree_all, tree_map, tree_map_with_path +from torch.utils._traceback import CapturedTraceback +from torch.utils.weak import WeakIdRef + + +if TYPE_CHECKING: + from torch._dynamo.device_interface import DeviceInterface + from torch.distributed._tools.mod_tracker import ModTracker + + +log = logging.getLogger(__name__) + +__all__ = ["DebugMode", "get_active_debug_mode"] + + +REDISTRIBUTE_FUNC = "redistribute_input" +# registered dispatch call hooks +_DISPATCH_RECORD_HOOKS: list[Callable] = [] +_DISPATCH_LOG_HOOKS: list[Callable] = [] +_DISPATCH_PRE_LOG_HOOKS: list[Callable] = [] +# Tracks if we're in inductor benchmarking, and temporarily disables logging +# (for ignoring autotuning kernel launches which don't affect the user-facing result) +_IN_INDUCTOR_BENCHMARK = False +# For record_outputs, log_tensor_hashes hooks for triton kernels. +# Stores kernel outputs in call.record["output"] +_RECORD_TRITON_OUTPUTS = False +# Annotates kernel output hashes, and stores them in call.post_hashes +_TRITON_OUTPUT_HASH_FN = None +# Annotates kernel input hashes, and stores them in call.pre_hashes +_TRITON_INPUT_HASH_FN = None + + +def _stringify_shape(shape) -> str: + return f"[{', '.join([str(x) for x in shape])}]" + + +def _stringify_device_mesh(mesh) -> str: + return f"DM({', '.join([str(s) for s in mesh.shape])})" + + +def _stringify_placement(placement) -> str: + return f"[{', '.join([str(p) for p in placement])}]" + + +def _stringify_attributes(tensor, attributes) -> str: + pairs = {} + for attr in attributes: + if hasattr(tensor, attr): + pairs[attr] = getattr(tensor, attr) + if len(pairs) == 0: + return "" + return f"{{{', '.join([f'{k}={v}' for k, v in pairs.items()])}}}" + + +def _stringify_dtensor_spec(spec) -> str: + from torch.distributed.tensor._dtensor_spec import DTensorSpec + + return DTensorSpec.format_shard_order_str(spec.placements, spec.shard_order) + + +class TensorIdTracker: + def __init__(self) -> None: + self.tensor_memo: dict[WeakIdRef, int] = {} + self.next_tensor_id = 0 + + def _id(self, tensor) -> int: + with torch._C._DisablePythonDispatcher(): + o = WeakIdRef(tensor) + + def del_memo() -> None: + self.tensor_memo.pop(o, None) + + weakref.finalize(tensor, del_memo) + if o not in self.tensor_memo: + self.tensor_memo[o] = self.next_tensor_id + self.next_tensor_id += 1 + return self.tensor_memo[o] + + +def _tensor_debug_string(tensor, attributes, tensor_memo=None) -> str: + """Convert tensor to debug string representation.""" + + if isinstance(tensor, torch.Tensor): + tensor_debug_str = f"{dtype_abbrs[tensor.dtype]}{_stringify_shape(tensor.shape)}{_stringify_attributes(tensor, attributes)}" + id_str = f"${tensor_memo._id(tensor)}" if tensor_memo is not None else "" + if isinstance(tensor, torch.distributed.tensor.DTensor): + # omitted device mesh + return f"dt{id_str}: {tensor_debug_str}| {_stringify_dtensor_spec(tensor._spec)}" + elif isinstance(tensor, FakeTensor): + return f"ft{id_str}: {tensor_debug_str}" + else: + return f"t{id_str}: {tensor_debug_str}" + else: + raise RuntimeError(f"Unsupported tensor type: {type(tensor)}") + + +def _arg_to_str(arg, attributes, tensor_memo=None) -> str: + from torch.distributed.tensor._dtensor_spec import DTensorSpec + + def to_str(x): + if isinstance(x, torch.Tensor): + return _tensor_debug_string(x, attributes, tensor_memo) + elif isinstance(x, DTensorSpec): + return _stringify_dtensor_spec(x) + return x + + arg = tree_map(to_str, arg) + return str(arg) + + +def norm_hash_fn(t: torch.Tensor, use_scalar: bool = False) -> torch.Tensor | float: + """ + from Observer. Computes a hash for a tensor by converting it to float (if needed), making it contiguous, + replacing NaN/inf values with fixed numbers, and then computing the L1 norm in float64 or complex128. + This is used to generate a deterministic summary value for tensor comparison. + """ + with torch._C._DisablePythonDispatcher(): + if not (t.is_floating_point() or t.is_complex()): + t = t.float() + t = t.contiguous() + + if t.is_complex(): + t_float = t.to(dtype=torch.complex128) + else: + t_float = t.to(dtype=torch.float64) + + out = t_float.norm(p=1) + if use_scalar: + return out.item() + return out + + +def _compute_rel_diff(hash1, hash2): + # Relative difference: |hash1 - hash2| / max(|hash1|, |hash2|, eps) + numerator = abs(hash1 - hash2) + denominator = max(abs(hash1), abs(hash2), 1e-10) + return numerator / denominator + + +def hash_tensor_fn(t: torch.Tensor, use_scalar: bool = False) -> torch.Tensor | int: + """ + wrapper over torch.hash_tensor + """ + if isinstance(t, torch.distributed.tensor.DTensor): + t = t.to_local() + + if t.is_floating_point(): + t_clean = t.to(dtype=torch.float64) + elif t.is_complex(): + t_clean = t.to(dtype=torch.complex128).view(torch.float64) + else: + t_clean = t.to(dtype=torch.int64) + + if t.numel() > 0: + out = torch.hash_tensor(t_clean) + else: + out = torch.zeros((), device=t_clean.device, dtype=torch.uint64) + + if use_scalar: + return out.item() # type: ignore[attribute] + return out + + +def _get_stack_trace() -> str: + from torch.fx.experimental.symbolic_shapes import uninteresting_files + + summary = CapturedTraceback.extract().summary() + summary = summary[:-4] # filter out DebugMode frames + summary = [ + frame for frame in summary if frame.filename not in uninteresting_files() + ] + summary = traceback.StackSummary.from_list(summary) + return "".join(summary.format()) + + +def _get_user_stack_trace(stack_trace_str: str) -> str | None: + # Extract user code stack trace, filtering out torch internals. + torch_dir = os.path.dirname(inspect.getfile(torch)) + filter_fn = lambda file, name, code: not file.startswith(torch_dir + os.path.sep) # noqa: E731 + trace = _parse_stack_trace(stack_trace_str, filter_fn=filter_fn) + if trace: + return f"File: {trace.file}:{trace.lineno} in {trace.name}, code: {trace.code}" + return None + + +def _maybe_get_autograd_trace() -> str | None: + if torch._C._current_autograd_node() is not None: + tb = torch._C._current_autograd_node().metadata.get("traceback_") # type: ignore[attr-defined] + if tb: + return "".join(tb) + return None + + +def _get_op_name(op) -> str: + if isinstance(op, torch._ops.OpOverload): + op_name = op.__qualname__ + elif hasattr(op, "__module__") and hasattr(op, "__name__"): + op_name = f"{op.__module__}.{op.__name__}" + else: + op_name = str(op) + return op_name + + +_annotate_decorated = False + + +def _ensure_annotate_decorated(): + """ + Lazily apply dont_skip_tracing decorator to DebugMode._annotate, to avoid circular import/initialization issues. + """ + global _annotate_decorated + if not _annotate_decorated: + DebugMode._annotate = torch._dynamo.dont_skip_tracing(DebugMode._annotate) # type: ignore[has-type] + + # Mark annotate as side-effectful so aot_eager doesn't DCE it. + from torch.fx.node import _side_effectful_functions + + _side_effectful_functions.add(torch.ops.debug_mode_ops.annotate.default) + + # Register no-op lowering for inductor backend + from torch._inductor.lowering import register_lowering + from torch._logging import warning_once + + @register_lowering(torch.ops.debug_mode_ops.annotate) + def _annotate_lowering(tag: str) -> None: + warning_once(log, 'DebugMode._annotate() is a no-op for backend="inductor"') + return None + + _annotate_decorated = True + + +class _DebugCall: + """Base class for tracking operator calls in DebugMode""" + + def __init__( + self, + call_depth: int, + record: dict[str, Any] | None = None, + log: dict[str, Any] | None = None, + stack: bool = False, + ) -> None: + self.call_depth = call_depth + if stack: + self.stack_trace = _get_stack_trace() + self.fwd_stack_trace = _maybe_get_autograd_trace() + + # results from dispatch hooks + self.record = record + self.log = log + self.output_str: str | None = None + + def stringify_args( + self, attributes: list[str], tensor_memo: TensorIdTracker | None = None + ) -> None: + """ + To reduce memory consumption, this method stringifies args/kwargs, stores the result, and deletes original args/kwargs. + """ + raise NotImplementedError( + "Subclasses must implement stringify_args(), even if no-op" + ) + + def stringify_output( + self, + output: Any, + attributes: list[str], + tensor_memo: TensorIdTracker | None = None, + ) -> None: + """Store stringified version of call output in self.output_str""" + if tree_all(lambda x: x is None, output): + return + output_str = tree_map(lambda x: _arg_to_str(x, attributes, tensor_memo), output) + self.output_str = f" -> {str(output_str)}" + + def render(self, attributes: list[str]) -> str: + raise NotImplementedError("Subclasses must implement string render()") + + def __repr__(self) -> str: + return self.render([]) + + +class _OpCall(_DebugCall): + """Normal operator call""" + + def __init__( + self, + op, + args: tuple, + kwargs: dict, + call_depth: int, + stack: bool = False, + ) -> None: + super().__init__(call_depth, stack=stack) + self.op = op + self.args = args + self.kwargs = kwargs + + self.args_str: str | None = None + self.kwargs_str: str | None = None + + def stringify_args( + self, attributes: list[str], tensor_memo: TensorIdTracker | None = None + ) -> None: + self.args_str = ", ".join( + _arg_to_str(arg, attributes, tensor_memo) for arg in self.args + ) + if self.kwargs: + self.kwargs_str = ", " + ", ".join( + f"{k}={_arg_to_str(v, attributes, tensor_memo)}" + for k, v in self.kwargs.items() + ) + else: + self.kwargs_str = "" + del self.args + del self.kwargs + + def render(self, attributes: list[str]) -> str: + if self.args_str is not None: + args_str = self.args_str + else: + args_str = ", ".join(_arg_to_str(arg, attributes) for arg in self.args) + + if self.kwargs_str is not None: + kwargs_str = self.kwargs_str + else: + if self.kwargs: + kwargs_str = ", " + ", ".join( + f"{k}={_arg_to_str(v, attributes)}" for k, v in self.kwargs.items() + ) + else: + kwargs_str = "" + + if isinstance(self.op, torch._ops.OpOverload): + op_name = self.op.__qualname__ + elif hasattr(self.op, "__module__") and hasattr(self.op, "__name__"): + op_name = f"{self.op.__module__}.{self.op.__name__}" + else: + op_name = str(self.op) + + base_str = f"{op_name}({args_str}{kwargs_str})" + + if self.output_str: + base_str += self.output_str + if self.log: + base_str += f" # {self.log}" + return base_str + + def __iter__(self): + # for BC; tuple(self) returns (op, args, kwargs, call_depth) + if self.args_str is not None: + yield from [self.op, self.args_str, self.kwargs_str, self.call_depth] + else: + yield from [self.op, self.args, self.kwargs, self.call_depth] + + +class _RedistributeCall(_DebugCall): + """Redistribute call from DTensor dispatch""" + + def __init__( + self, + arg, + src_placement, + dst_placement, + transform_info_str, + call_depth, + stack=False, + ) -> None: + super().__init__(call_depth, stack=stack) + self.arg = arg + self.src_placement = src_placement + self.dst_placement = dst_placement + self.transform_info_str = transform_info_str + + self.arg_str: str | None = None + + def stringify_args( + self, attributes: list[str], tensor_memo: TensorIdTracker | None = None + ) -> None: + self.arg_str = f"{_arg_to_str(self.arg, attributes, tensor_memo)}" + del self.arg + + def render(self, attributes: list[str]) -> str: + if self.arg_str is not None: + arg_str = self.arg_str + else: + arg_str = f"{_arg_to_str(self.arg, attributes)}" + + if self.transform_info_str is not None: # prioritize over src/dst placements + placement_str = f"trace: {self.transform_info_str}" + else: + src_placement_str = _arg_to_str(self.src_placement, attributes) + dst_placement_str = _arg_to_str(self.dst_placement, attributes) + placement_str = f"{src_placement_str} -> {dst_placement_str}" + + base_str = f"{REDISTRIBUTE_FUNC}({arg_str}, {placement_str})" + if self.output_str: + base_str += self.output_str + return base_str + + def __iter__(self): + # for BC; tuple(self) returns (op, placement info, kwargs, call_depth) + if self.arg_str is not None: + arg = self.arg_str + else: + arg = self.arg + + yield REDISTRIBUTE_FUNC + if self.transform_info_str: + yield [arg, self.transform_info_str] + else: + yield [arg, self.src_placement, self.dst_placement] + yield {} + yield self.call_depth + + +class _NNModuleCall(_DebugCall): + """Designates entering an nn.Module's forward method""" + + def __init__(self, module_name: str, call_depth: int, stack: bool = False) -> None: + super().__init__(call_depth, stack=stack) + self.module_name = module_name + + def stringify_args( + self, attributes: list[str], tensor_memo: TensorIdTracker | None = None + ) -> None: + pass # nothing to stringify + + def render(self, attributes: list[str]) -> str: + return f"[nn.Mod] {self.module_name}" + + def __iter__(self): + yield from [ + f"[nn.Mod] {self.module_name}", + (), + {}, + self.call_depth, + ] + + +class _TritonKernelCall(_DebugCall): + """Triton kernel call from Inductor""" + + def __init__( + self, + kernel_name: str, + kwargs: dict[str, Any], + call_depth: int, + ): + super().__init__(call_depth) + self.kernel_name = kernel_name + self.kwargs = kwargs + self.kwargs_str: str | None = None + + self.pre_hashes: dict[str, Any] | None = None + self.post_hashes: dict[str, Any] | None = None + + def stringify_args( + self, attributes: list[str], tensor_memo: TensorIdTracker | None = None + ) -> None: + # Optionally hash kernel inputs before launch + global _TRITON_INPUT_HASH_FN + if hash_fn := _TRITON_INPUT_HASH_FN: + self.pre_hashes = { + k: hash_fn(v) + for k, v in self.kwargs.items() + if isinstance(v, torch.Tensor) + } + + if self.kwargs: + self.kwargs_str = ", ".join( + f"{k}={_arg_to_str(v, attributes, tensor_memo)}" + for k, v in self.kwargs.items() + ) + else: + self.kwargs_str = "" + + def render(self, attributes: list[str]) -> str: + base_str = f"[triton] {self.kernel_name}({self.kwargs_str})" + if self.pre_hashes: + pre_hashes_str = ", ".join(f"{k}: {v}" for k, v in self.pre_hashes.items()) + pre_hashes_str = ( + "\n " + + " " * self.call_depth + + f"# pre-kernel hashes: {{{pre_hashes_str}}}" + ) + else: + pre_hashes_str = "" + if self.post_hashes: + post_hashes_str = ", ".join( + f"{k}: {v}" for k, v in self.post_hashes.items() + ) + post_hashes_str = ( + "\n " + + " " * self.call_depth + + f"# post-kernel hashes: {{{post_hashes_str}}}" + ) + else: + post_hashes_str = "" + return f"{base_str}{pre_hashes_str}{post_hashes_str}\n" + + def finalize(self, device_interface: "DeviceInterface"): + # synchronize -> hash/store kernel results + global _RECORD_TRITON_OUTPUTS, _TRITON_OUTPUT_HASH_FN + device_interface.synchronize(device_interface.current_device()) + if _RECORD_TRITON_OUTPUTS: + self.record = { + "output": { + k: v.clone() if isinstance(v, torch.Tensor) else v + for k, v in self.kwargs.items() + } + } + if hash_fn := _TRITON_OUTPUT_HASH_FN: + self.post_hashes = { + k: hash_fn(v) + for k, v in self.kwargs.items() + if isinstance(v, torch.Tensor) + } + + # don't store tensors + del self.kwargs + + def __iter__(self): + yield from [self.kernel_name, (), self.kwargs_str, self.call_depth] + + +class _AnnotateCall(_DebugCall): + """Custom annotation call""" + + def __init__( + self, tag: Any, is_profiler_record: bool, call_depth: int, stack: bool = False + ) -> None: + super().__init__(call_depth, stack=stack) + self.tag = tag + self.is_profiler_record = is_profiler_record + + def render(self, attributes: list[str]) -> str: + if self.is_profiler_record: + return f"[record function] {self.tag}" + else: + return f"[annotate] {self.tag}" + + def __iter__(self): + yield from [ + ( + f"[record function] {self.tag}" + if self.is_profiler_record + else f"[annotate] {self.tag}" + ), + (), + {}, + self.call_depth, + ] + + +def _run_hook(hook, *args): + out = hook(*args) + assert out is None or isinstance(out, dict) + return out + + +def _run_dispatch_pre_log_hooks(call: _DebugCall, func, types, args, kwargs) -> None: + global _DISPATCH_PRE_LOG_HOOKS + if _DISPATCH_PRE_LOG_HOOKS: + for hook in _DISPATCH_PRE_LOG_HOOKS: + hook_out = _run_hook(hook, func, types, args, kwargs, call) + if hook_out is not None: + # Store pre-hook results in call.log + if call.log is None: + call.log = {} + call.log.update(hook_out) + + +def _run_dispatch_hooks(call: _DebugCall, func, types, args, kwargs, result) -> None: + global _DISPATCH_RECORD_HOOKS, _DISPATCH_LOG_HOOKS + if _DISPATCH_RECORD_HOOKS: + record = {} + for hook in _DISPATCH_RECORD_HOOKS: + hook_out = _run_hook(hook, func, types, args, kwargs, result) + if hook_out is not None: + record.update(hook_out) + if record: + call.record = record + + if _DISPATCH_LOG_HOOKS: + # Preserve existing log from pre-hooks (e.g., input_hash) + if call.log is None: + call.log = {} + for hook in _DISPATCH_LOG_HOOKS: + hook_out = _run_hook(hook, func, types, args, kwargs, result) + if hook_out is not None: + call.log.update(hook_out) + + +def _get_call_name(call: _DebugCall) -> str: + """String identifying _DebugCall (e.g. func, kernel, module name)""" + if isinstance(call, _OpCall): + return _get_op_name(call.op) + elif isinstance(call, _TritonKernelCall): + return call.kernel_name + elif isinstance(call, _NNModuleCall): + return call.module_name + elif isinstance(call, _RedistributeCall): + return REDISTRIBUTE_FUNC + else: + return str(call) + + +@torch.library.custom_op("debug_mode_ops::annotate", mutates_args=()) +def _annotate(tag: str) -> None: + # This is special-cased in DebugMode.__torch_dispatch__ + return None + + +@_annotate.register_fake +def _annotate_fake(tag: str) -> None: + return None + + +class DebugMode(TorchDispatchMode): + def __init__( + self, + *, + record_torchfunction=False, + record_faketensor=False, + record_realtensor=True, + record_tensor_attributes=None, + record_nn_module=False, + store_original_args=False, + record_stack_trace=False, + record_output=True, + record_ids=False, + record_profiler_context=True, + ) -> None: + super().__init__() + import torch.distributed.tensor # noqa: F401 + + _ensure_annotate_decorated() + self.supports_higher_order_operators = True + + # Pushes DebugMode onto the torchfunction stack, and records __torch_function__ calls as well. + # WARNING: currently incompatible with torch.compile due to dynamo guard failures. + self.record_torchfunction = record_torchfunction + + # Records __torch_dispatch__ calls on FakeTensors. + self.record_faketensor = record_faketensor + + # Records __torch_dispatch__ calls on real tensors. + self.record_realtensor = record_realtensor + + # Optional list[str] of tensor attributes, to be annotated in the string dump. + self.record_tensor_attributes = record_tensor_attributes or [] + + # Uses ModTracker to record nn.Module entrances, as _NNModuleCall entries. + # This flag currently has no effect on torch.compiled-regions. + self.record_nn_module = record_nn_module + + self.module_tracker: ModTracker | None = None + if self.record_nn_module: + self.module_tracker_setup() + + # If True, stores call args/kwargs in logs, without immediately stringifying. + # Defaults to False for memory concerns. + self.store_original_args = store_original_args + + # For stack trace recording, stores log call stack traces in .stack_trace. + # For backward graph nodes, will also store the corresponding forward stack traces in .fwd_stack_trace. + # NOTE: this is only available if autograd tracebacks are being set during the forward pass, + # e.g. via DebugMode(record_stack_trace=True), or torch.autograd.set_detect_anomaly(). + self.record_stack_trace = record_stack_trace + + # Records call outputs in logs (e.g. for __torch_dispatch__, __torch_function__, redistribute_input) + self.record_output: bool = record_output + + # Annotates string dumps with graph-style tensor ids, e.g. op($1, $2) -> $3. + self.record_ids: bool = record_ids + + # Annotates string dumps with profiler.record_function contexts from runtime code. + # Currently does not preserve contexts inside torch.compile-d regions. + self.record_profiler_context: bool = record_profiler_context + + self.reset() + + def reset(self) -> None: + self.operators = [] + self.call_depth = 0 + self._tensor_memo = TensorIdTracker() + self._output_info: dict[int, object] = {} + self.ignored_record_functions = 0 + + def _track_op_output(self, op_index, result) -> None: + """Assign IDs to output tensors and store in output_info""" + # self._track_tensor_ids(result) + self._output_info[op_index] = result + + # Without this override, running torch.compile under DebugMode + # will force torch.compile to always use the “eager” backend + # With this, DebugMode will not take effect on torch.compile + @classmethod + def ignore_compile_internals(cls) -> bool: + return True + + def _record_call(self, call) -> None: + global _IN_INDUCTOR_BENCHMARK + if _IN_INDUCTOR_BENCHMARK: + return + + if str(call).startswith("profiler::_record_function"): + return + + if not self.store_original_args: + call.stringify_args( + self.record_tensor_attributes, + self._tensor_memo if self.record_ids else None, + ) + self.operators.append(call) + + def _record_call_output(self, call, output) -> None: + if not self.record_output: + return + call.stringify_output( + output, + self.record_tensor_attributes, + self._tensor_memo if self.record_ids else None, + ) + + def __torch_function__(self, func, types, args=(), kwargs=None): + if kwargs is None: + kwargs = {} + + call = _OpCall( + func, args, kwargs, self.call_depth, stack=self.record_stack_trace + ) + self._record_call(call) + + try: + self.call_depth += 1 + result = func(*args, **kwargs) + self._record_call_output(call, result) + return result + finally: + self.call_depth -= 1 + + def _maybe_record_function(self, tag): + # filter out tags that appear noisy, or aren't runtime-related + if any( + tag.startswith(prefix) + for prefix in [ + # assuming these are from benchmarking, not the actual runtime call + "CachingAutotuner.", + "InductorBenchmarker.", + # inductor compilation + "compile_fx..", + ] + ): + self.ignored_record_functions += 1 + return + + call = _AnnotateCall(tag, True, self.call_depth, stack=self.record_stack_trace) + self.operators.append(call) + self.call_depth += 1 + + def _maybe_exit_record_function(self): + assert self.ignored_record_functions >= 0 + if self.ignored_record_functions > 0: + self.ignored_record_functions -= 1 + else: + self.call_depth -= 1 + + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + if kwargs is None: + kwargs = {} + + # Handle record_function entries + if self.record_profiler_context: + if func == torch.ops.profiler._record_function_enter_new.default: + assert len(args) == 1 + self._maybe_record_function(args[0]) + elif func == torch.ops.profiler._record_function_exit._RecordFunction: + self._maybe_exit_record_function() + + # Handle DebugMode._annotate() + if func is torch.ops.debug_mode_ops.annotate.default: + assert len(args) == 1 + self._handle_annotate(args[0]) + return + + # Record the operation with its call depth + call = None + if torch.distributed.tensor.DTensor in types: + call = _OpCall( + func, args, kwargs, self.call_depth, stack=self.record_stack_trace + ) + self._record_call(call) + return NotImplemented + elif FakeTensor in types or isinstance( + _get_current_dispatch_mode(), FakeTensorMode + ): + if self.record_faketensor: + if func != torch.ops.prim.device.default: + call = _OpCall( + func, + args, + kwargs, + self.call_depth + 1, + stack=self.record_stack_trace, + ) + self._record_call(call) + elif len(types) == 0: + if self.record_realtensor: + call = _OpCall( + func, + args, + kwargs, + self.call_depth + 1, + stack=self.record_stack_trace, + ) + self._record_call(call) + + # Run pre-hooks before executing the operation to hash inputs + # We have to run becore the func() call in case there's any + # in-place mutation + if call: + _run_dispatch_pre_log_hooks(call, func, types, args, kwargs) + + result = func(*args, **kwargs) + if call: + self._record_call_output(call, result) + _run_dispatch_hooks(call, func, types, args, kwargs, result) + + return result + + def __enter__(self): + self.reset() + + if self.record_torchfunction: + torch._C._push_on_torch_function_stack(self) + + super().__enter__() + if self.record_nn_module: + self.module_tracker.__enter__() # type: ignore[attribute, union-attr] + + if self.record_stack_trace: + self.anomaly_for_traces = torch.autograd.set_detect_anomaly( + True, check_nan=False + ) + self.anomaly_for_traces.__enter__() + return self + + # pyrefly: ignore [bad-override] + def __exit__(self, *args): + super().__exit__(*args) + if self.record_nn_module: + self.module_tracker.__exit__() # type: ignore[attribute, union-attr] + if self.record_torchfunction: + torch._C._pop_torch_function_stack() + if self.record_stack_trace: + self.anomaly_for_traces.__exit__(*args) + + def module_tracker_setup(self) -> None: + from torch.distributed._tools.mod_tracker import ModTracker + + self.module_tracker = ModTracker() + + # module pre-fw hook: record module call + def pre_fw_hook(module, input) -> None: + fqn = self.module_tracker._get_mod_name(module) # type: ignore[attribute, union-attr] + self.operators.append(_NNModuleCall(fqn, self.call_depth)) + self.call_depth += 1 + + # module post-fw hook: decrement call depth + def post_fw_hook(module, input, output) -> None: + self.call_depth -= 1 + + self.module_tracker.register_user_hooks(pre_fw_hook, post_fw_hook) + + @contextlib.contextmanager + def record_redistribute_calls( + self, + arg, + src_placement, + dst_placement, + transform_info_str: str | None = None, + ): + try: + self._record_call( + _RedistributeCall( + arg, + src_placement=src_placement, + dst_placement=dst_placement, + transform_info_str=transform_info_str, + call_depth=self.call_depth + 1, + stack=self.record_stack_trace, + ) + ) + self.call_depth += 1 + yield + finally: + self.call_depth -= 1 + + def record_triton_kernel( + self, kernel_name: str, kwargs: dict[str, Any] + ) -> _TritonKernelCall: + call = _TritonKernelCall(kernel_name, kwargs, self.call_depth + 1) + call.stringify_args(self.record_tensor_attributes) + self.operators.append(call) + return call + + def debug_string(self, show_stack_trace: bool | None = None) -> str: + """ + show_stack_trace: option to display one-line stack trace summaries above groups + of operations (similar to gm.print_readable() style). + Requires record_stack_trace=True. + if None, uses self.record_stack_trace, otherwise overrides it. + """ + show_stack_trace = ( + self.record_stack_trace if show_stack_trace is None else show_stack_trace + ) + + with torch._C.DisableTorchFunction(): + if not show_stack_trace: + result = "\n".join( + " " + + " " * op.call_depth + + op.render(self.record_tensor_attributes) + for op in self.operators + ) + return result + + # Group operations by stack trace + lines = [] + prev_stack_summary = None + + for op in self.operators: + # Get the stack trace: prefer fwd_stack_trace, fallback to stack_trace + stack_trace = None + if hasattr(op, "fwd_stack_trace") and op.fwd_stack_trace: + stack_trace = op.fwd_stack_trace + elif hasattr(op, "stack_trace") and op.stack_trace: + stack_trace = op.stack_trace + + stack_summary = None + if stack_trace: + stack_summary = _get_user_stack_trace(stack_trace) + + if stack_summary and stack_summary != prev_stack_summary: + # add blank line before stack trace comment for readability + if lines: # don't add blank line at the very start + lines.append("") + indent = " " * (op.call_depth + 1) + lines.append(indent + "# " + stack_summary) + prev_stack_summary = stack_summary + + # Add the operation line + line = ( + " " + + " " * op.call_depth + + op.render(self.record_tensor_attributes) + ) + lines.append(line) + + return "\n".join(lines) + + @staticmethod + @contextlib.contextmanager + def dispatch_hooks( + record_hook: Callable | None = None, + log_hook: Callable | None = None, + pre_log_hook: Callable | None = None, + ): + """ + Allows installing post-hooks on arguments to intercepted __torch_dispatch__ calls; + hook signatures are expected as (func, types, args, kwargs, result), + i.e. __torch_dispatch__ args + return value. + + Logging hook outputs are stored in call.log and annotate calls in debug_string(), + while recording hook outputs are just stored in call.record. + For now hooks are expected to return dictionaries. + + pre_log_hook signature is (func, types, args, kwargs, call) and is executed before + the operation. It allows capturing state before in-place mutations. + """ + global _DISPATCH_RECORD_HOOKS, _DISPATCH_LOG_HOOKS, _DISPATCH_PRE_LOG_HOOKS + + if record_hook: + _DISPATCH_RECORD_HOOKS.append(record_hook) + if log_hook: + _DISPATCH_LOG_HOOKS.append(log_hook) + if pre_log_hook: + _DISPATCH_PRE_LOG_HOOKS.append(pre_log_hook) + try: + yield + finally: + if record_hook: + _DISPATCH_RECORD_HOOKS.pop() + if log_hook: + _DISPATCH_LOG_HOOKS.pop() + if pre_log_hook: + _DISPATCH_PRE_LOG_HOOKS.pop() + + @staticmethod + @contextlib.contextmanager + def record_outputs(): + """ + Hook for storing cloned output tensors in .record["output"]. + """ + + def dispatch_hook(func, types, args, kwargs, result): + out = tree_map( + lambda x: x.clone() if isinstance(x, torch.Tensor) else x, result + ) + return {"output": out} + + global _RECORD_TRITON_OUTPUTS + try: + _old_record_triton = _RECORD_TRITON_OUTPUTS + _RECORD_TRITON_OUTPUTS = True + with DebugMode.dispatch_hooks(record_hook=dispatch_hook): + yield + finally: + _RECORD_TRITON_OUTPUTS = _old_record_triton + + @staticmethod + @contextlib.contextmanager + def log_tensor_hashes( + hash_fn: Callable | str | list[str] = "norm", hash_inputs: bool = False + ): + """ + Installs hook for tensor hash logging. + + hash_fn: One of: + - Custom-defined hash function + - String: one of ("norm", "hash_tensor") + - "norm": uses norm_hash_fn; basically tensor's L1 norm + - "hash_tensor": uses torch.hash_tensor (XOR sum reduction) + - List of strings: returns tuple of hashes from above options + hash_inputs: if True, also hashes tensors in (args, kwargs), storing them in "input_hash". + Input hashes are captured before the operation executes, so they reflect the state before + any in-place mutations. + """ + + def hash_fn_option(hash_type): + assert isinstance(hash_type, str) and hash_type in ["norm", "hash_tensor"] + return functools.partial( + norm_hash_fn if hash_type == "norm" else hash_tensor_fn, use_scalar=True + ) + + if callable(hash_fn): + fn = hash_fn + elif isinstance(hash_fn, str): + fn = hash_fn_option(hash_fn) + elif isinstance(hash_fn, list): + fns = [hash_fn_option(fn) for fn in hash_fn] + fn = lambda x: tuple(fn(x) for fn in fns) # noqa: E731 + else: + raise NotImplementedError( + f"log_tensor_hashes() expected hash_fn to be callable, str, or list[str], but found {type(hash_fn)}" + ) + + def _tree_hash(obj): + return tree_map( + lambda x: fn(x) if isinstance(x, torch.Tensor) else None, obj + ) + + def _dispatch_pre_log_hook(func, types, args, kwargs, call): + """Pre-hook to capture input hashes before operation executes""" + if "empty" in str(func) or "profiler" in str(func): + return None + + if hash_inputs: + # Capture input hashes before the operation + input_hash = _tree_hash((args, kwargs)) + if not tree_all(lambda x: x is None, input_hash): + return {"input_hash": input_hash} + return None + + def _dispatch_post_hook(func, types, args, kwargs, result): + """Post-hook to capture output hashes after operation executes""" + if "empty" in str(func) or "profiler" in str(func): + return None + + out = {} + out["hash"] = _tree_hash(result) + + if tree_all(lambda x: x is None, out.values()): + return None + return out + + global _TRITON_INPUT_HASH_FN, _TRITON_OUTPUT_HASH_FN + try: + if hash_inputs: + _old_input_hfn = _TRITON_INPUT_HASH_FN + _TRITON_INPUT_HASH_FN = fn + _old_output_hfn = _TRITON_OUTPUT_HASH_FN + _TRITON_OUTPUT_HASH_FN = fn + with DebugMode.dispatch_hooks( + log_hook=_dispatch_post_hook, + pre_log_hook=_dispatch_pre_log_hook if hash_inputs else None, + ): + yield + finally: + if hash_inputs: + _TRITON_INPUT_HASH_FN = _old_input_hfn # type: ignore[assignment] + _TRITON_OUTPUT_HASH_FN = _old_output_hfn + + @staticmethod + @contextlib.contextmanager + def _benchmarking_inductor(): + """ + Context manager for disabling logging during inductor benchmarking, + so logs don't contain all kernels launched from autotuning. + """ + global _IN_INDUCTOR_BENCHMARK + try: + _IN_INDUCTOR_BENCHMARK = True + yield + finally: + _IN_INDUCTOR_BENCHMARK = False + + @property + def logs(self): + return list(self.operators) + + def _handle_annotate(self, tag): + """Handles DebugMode._annotate()""" + call = _AnnotateCall(tag, False, self.call_depth, self.record_stack_trace) + self.operators.append(call) + + @staticmethod + def _annotate(tag: Any) -> None: + """ + If an active DebugMode exists, adds an "[annotate] " entry to the logs. Useful for contextualizing logs. + Implemented with a custom op. + """ + torch.ops.debug_mode_ops.annotate(tag) + + @staticmethod + def check_hash_mismatches( + logs1: list, logs2: list, compare_inputs: bool = False + ) -> list[dict]: + """ + Compares tensor hashes between two DebugMode runs, for checking run-to-run numerical divergence. + + This first validates the two log sequences have identical structure (same operations, input shapes/dtypes, etc.), + then compares tensor hash values, and returns a list of call outputs where mismatches were found. + Expects input logs to have been run with log_tensor_hashes, and looks for hashes in .log["hash"] & .log["input_hash"] + (or .post_hashes & .pre_hashes for triton kernels). + + note: skips checking log pairs where hashes aren't present, but will raise if present in one & not the other. + + Args: + logs1: logs from the first DebugMode run (from debug_mode.logs) + logs2: logs from the second DebugMode run + compare_inputs: If True, also compare input tensor hashes (default: only output checking) + + Returns: + List of dictionaries describing hash mismatches. Each dict contains: + - call_type: "torch op" or "triton kernel" + - call: Operator/kernel name + - arg_name: For triton kernels, the argument name; None for torch ops + - pytree_path: For torch ops, the pytree path to the differing tensor; None for kernels + - hash1: Hash value from the first run + - hash2: Hash value from the second run + - rel_diff: Relative difference between hash values + - is_input_hash: True if this is an input hash, False for output hash + + Raises: + ValueError: If logs have different lengths, call types, operator names, or call depths + + Usage:: + + # Run model first time + with DebugMode() as debug_mode, DebugMode.log_tensor_hashes(): + model(x) + logs1 = debug_mode.logs + + # Run again, in exactly the same way + with DebugMode() as debug_mode, DebugMode.log_tensor_hashes(): + model(x) + logs2 = debug_mode.logs + + mismatches = DebugMode.check_hash_mismatches(logs1, logs2) + for m in mismatches: + print(f"{m['call']}: hash diff {m['rel_diff']:.2e}") + """ + if len(logs1) != len(logs2): + raise ValueError(f"Log lengths don't match: {len(logs1)} vs {len(logs2)}") + + difference_info = [] + for i, (log1, log2) in enumerate(zip(logs1, logs2)): + # check call type + call1_type = type(log1).__name__ + call2_type = type(log2).__name__ + if call1_type != call2_type: + raise ValueError( + f"Call types don't match at index {i}: {call1_type} vs {call2_type}" + ) + call_type = call1_type + + # check call name + op1_name, op2_name = _get_call_name(log1), _get_call_name(log2) + if op1_name != op2_name: + raise ValueError( + f"Operators don't match at index {i}: {call_type}[{op1_name}] vs {call_type}[{op2_name}]" + ) + op_name = op1_name + + # check call depth + if log1.call_depth != log2.call_depth: + raise ValueError( + f"Call depths for {call_type}[{op_name}] don't match at index {i}: {log1.call_depth} vs {log2.call_depth}" + ) + + # Redistribute: call args should be the same + if isinstance(log1, _RedistributeCall): + if tuple(log1) != tuple(log2): + raise ValueError( + f"Redistribute calls don't match at index {i}: {log1} vs {log2}" + ) + + # Triton kernel: same arg names, arg types + elif isinstance(log1, _TritonKernelCall): + if log1.kwargs_str != log2.kwargs_str: + raise ValueError( + f"Triton kernel call args don't match for {log1.kernel_name} at index {i}:" + f"\n\nlog1: {log1.kwargs_str}\n\nlog2: {log2.kwargs_str}" + ) + + def compare_triton_hashes(hashes1, hashes2, is_input): + assert set(hashes1.keys()) == set(hashes2.keys()) # type: ignore[union-attr] + for key in hashes1: + if hashes1[key] != hashes2[key]: + difference_info.append( + { + "call_type": "triton kernel", + "call": op_name, + "arg_name": key, + "pytree_path": None, + "hash1": hashes1[key], + "hash2": hashes2[key], + "rel_diff": _compute_rel_diff( + hashes1[key], hashes2[key] + ), + "is_input_hash": is_input, + } + ) + + # check output hashes + has_post_1, has_post_2 = ( + log1.post_hashes is not None, + log2.post_hashes is not None, + ) + if has_post_1 != has_post_2: + raise ValueError( + f"Triton kernel post-hash presence inconsistent for {log1.kernel_name} " + f"at index {i}: log1 has post_hashes={has_post_1}, log2 has post_hashes={has_post_2}" + ) + + if has_post_1: + compare_triton_hashes( + log1.post_hashes, log2.post_hashes, is_input=False + ) + + # maybe check input hashes + if compare_inputs: + has_pre_1, has_pre_2 = ( + log1.pre_hashes is not None, + log2.pre_hashes is not None, + ) + if has_pre_1 != has_pre_2: + raise ValueError( + f"Triton kernel pre-hash presence inconsistent for {log1.kernel_name} " + f"at index {i}: log1 has pre_hashes={has_pre_1}, log2 has pre_hashes={has_pre_2}" + ) + + if has_pre_1: + compare_triton_hashes( + log1.pre_hashes, log2.pre_hashes, is_input=True + ) + + # regular log calls + elif isinstance(log1, _OpCall): + + def compare_op_hashes(hashes1, hashes2, is_input): + def _helper(keypath, hash1, hash2): + if hash1 != hash2: + difference_info.append( + { + "call_type": "torch op", + "call": op_name, + "arg_name": None, + "pytree_path": keystr(keypath), + "hash1": hash1, + "hash2": hash2, + "rel_diff": _compute_rel_diff(hash1, hash2), + "is_input_hash": is_input, + } + ) + + tree_map_with_path(_helper, hashes1, hashes2) + + # check output hashes + has_hash1 = log1.log is not None and "hash" in log1.log + has_hash2 = log2.log is not None and "hash" in log2.log + if has_hash1 != has_hash2: + raise ValueError( + f"Output hash presence inconsistent for triton kernel {call_type}[{op_name}] " + f"at index {i}: log1 has hash={has_hash1}, log2 has hash={has_hash2}" + ) + + if has_hash1: + compare_op_hashes( + log1.log["hash"], # type: ignore[union-attr] + log2.log["hash"], + is_input=False, + ) + + # maybe check input hashes + if compare_inputs: + has_hash1 = log1.log is not None and "input_hash" in log1.log + has_hash2 = log2.log is not None and "input_hash" in log2.log + if has_hash1 != has_hash2: + raise ValueError( + f"Input hash presence inconsistent for triton kernel {call_type}[{op_name}] " + f"at index {i}: log1 has input_hash={has_hash1}, log2 has input_hash={has_hash2}" + ) + + if has_hash1: + compare_op_hashes( + log1.log["input_hash"], # type: ignore[union-attr] + log2.log["input_hash"], + is_input=True, + ) + + return difference_info + + +def get_active_debug_mode() -> DebugMode | None: + debug_mode = None + for mode in _get_current_dispatch_mode_stack(): + if isinstance(mode, DebugMode): + debug_mode = mode + break + return debug_mode diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_device.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_device.py new file mode 100644 index 0000000000000000000000000000000000000000..aafa336415ec6c70a7871959bfb1bc9efd829486 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_device.py @@ -0,0 +1,124 @@ +# mypy: allow-untyped-defs +import functools + +import torch +from torch._C import _len_torch_function_stack +from torch.overrides import _pop_mode, _push_mode, TorchFunctionMode +from torch.utils._contextlib import context_decorator + + +CURRENT_DEVICE: torch.device | None = None + + +@functools.lru_cache(1) +def _device_constructors(): + return { + # standard ones + torch.empty, + torch.empty_permuted, + torch.empty_strided, + torch.empty_quantized, + torch.ones, + torch.arange, + torch.bartlett_window, + torch.blackman_window, + torch.eye, + torch.fft.fftfreq, + torch.fft.rfftfreq, + torch.full, + torch.hamming_window, + torch.hann_window, + torch.kaiser_window, + torch.linspace, + torch.logspace, + torch.nested.nested_tensor, + # This function doesn't actually take a device argument + # torch.normal, + torch.rand, + torch.randn, + torch.randint, + torch.randperm, + torch.range, + torch.sparse_coo_tensor, + torch.sparse_compressed_tensor, + torch.sparse_csr_tensor, + torch.sparse_csc_tensor, + torch.sparse_bsr_tensor, + torch.sparse_bsc_tensor, + torch.tril_indices, + torch.triu_indices, + torch.zeros, + torch.asarray, + # weird ones + torch.tensor, + torch.as_tensor, + torch.scalar_tensor, + } + + +# NB: This is directly called from C++ in torch/csrc/Device.cpp +class DeviceContext(TorchFunctionMode): + def __init__(self, device) -> None: + # pyrefly: ignore [read-only] + self.device = torch.device(device) + + def __enter__(self): + global CURRENT_DEVICE + self.old_device = CURRENT_DEVICE + CURRENT_DEVICE = self.device + # We need to put the device at the bottom of the stack + # If we set default device within a function mode context + # exiting that context mode will pop the device function mode off + # of the stack incorrectly + cur_stack = [_pop_mode() for _ in range(_len_torch_function_stack())] + + _push_mode(self) + + for mode in reversed(cur_stack): + _push_mode(mode) + + def __exit__(self, exc_type, exc_val, exc_tb): + global CURRENT_DEVICE + CURRENT_DEVICE = self.old_device + cur_stack = [] + # Invariant: there should only be one DeviceContext on the stack at any time + # (At the bottom), pop all modes until we hit the bottom, assert it's a DeviceContext + # or else someone else has popped it! + for _ in range(_len_torch_function_stack() - 1): + mode = _pop_mode() + if isinstance(mode, DeviceContext): + raise AssertionError( + "Found nested DeviceContext on the mode stack where none expected" + ) + cur_stack.append(mode) + + if _len_torch_function_stack() > 0: + mode = _pop_mode() + if not isinstance(mode, DeviceContext): + raise AssertionError( + "Expected a DeviceContext at the bottom of the mode stack" + ) + + for mode in reversed(cur_stack): + _push_mode(mode) + + def __torch_function__(self, func, types, args=(), kwargs=None): + kwargs = kwargs or {} + if func in _device_constructors() and kwargs.get("device") is None: + kwargs["device"] = self.device + return func(*args, **kwargs) + + +# NB: This is directly called from C++ in torch/csrc/Device.cpp +def device_decorator(device, func): + return context_decorator(lambda: device, func) + + +def set_device(device): + """ + Set the default device inside of the wrapped function by decorating it with this function. + + If you would like to use this as a context manager, use device as a + context manager directly, e.g., ``with torch.device(device)``. + """ + return lambda func: device_decorator(torch.device(device), func) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_dtype_abbrs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_dtype_abbrs.py new file mode 100644 index 0000000000000000000000000000000000000000..c4eb9c56671dba774aa09d27887330fc350311fd --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_dtype_abbrs.py @@ -0,0 +1,30 @@ +import torch + + +# Used for testing and logging +dtype_abbrs = { + torch.bfloat16: "bf16", + torch.float64: "f64", + torch.float32: "f32", + torch.float16: "f16", + torch.float8_e4m3fn: "f8e4m3fn", + torch.float8_e5m2: "f8e5m2", + torch.float8_e4m3fnuz: "f8e4m3fnuz", + torch.float8_e5m2fnuz: "f8e5m2fnuz", + torch.float8_e8m0fnu: "f8e8m0fnu", + torch.float4_e2m1fn_x2: "f4e2m1fnx2", + torch.complex32: "c32", + torch.complex64: "c64", + torch.complex128: "c128", + torch.int8: "i8", + torch.int16: "i16", + torch.int32: "i32", + torch.int64: "i64", + torch.bool: "b8", + torch.uint8: "u8", + torch.uint16: "u16", + torch.uint32: "u32", + torch.uint64: "u64", + torch.bits16: "b16", + torch.bits1x8: "b1x8", +} diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_exposed_in.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_exposed_in.py new file mode 100644 index 0000000000000000000000000000000000000000..2cca4ce240ad99afd44b8347135300b8c48ae66f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_exposed_in.py @@ -0,0 +1,21 @@ +from collections.abc import Callable +from typing import TypeVar + + +F = TypeVar("F") + + +# Allows one to expose an API in a private submodule publicly as per the definition +# in PyTorch's public api policy. +# +# It is a temporary solution while we figure out if it should be the long-term solution +# or if we should amend PyTorch's public api policy. The concern is that this approach +# may not be very robust because it's not clear what __module__ is used for. +# However, both numpy and jax overwrite the __module__ attribute of their APIs +# without problem, so it seems fine. +def exposed_in(module: str) -> Callable[[F], F]: + def wrapper(fn: F) -> F: + fn.__module__ = module + return fn + + return wrapper diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_filelock.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_filelock.py new file mode 100644 index 0000000000000000000000000000000000000000..a291f59b4ba7fc1d9c1ecca703a6ae1fdf8e4cf1 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_filelock.py @@ -0,0 +1,41 @@ +from types import TracebackType +from typing_extensions import Self + +from filelock import FileLock as base_FileLock + +from torch.monitor import _WaitCounter + + +class FileLock(base_FileLock): + """ + This behaves like a normal file lock. + + However, it adds waitcounters for acquiring and releasing the filelock + as well as for the critical region within it. + + pytorch.filelock.enter - While we're acquiring the filelock. + pytorch.filelock.region - While we're holding the filelock and doing work. + pytorch.filelock.exit - While we're releasing the filelock. + """ + + def __enter__(self) -> Self: + self.region_counter = _WaitCounter("pytorch.filelock.region").guard() + with _WaitCounter("pytorch.filelock.enter").guard(): + result = super().__enter__() + self.region_counter.__enter__() + return result + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self.region_counter.__exit__() + with _WaitCounter("pytorch.filelock.exit").guard(): + # Returns nothing per + # https://github.com/tox-dev/filelock/blob/57f488ff8fdc2193572efe102408fb63cfefe4e4/src/filelock/_api.py#L379 + super().__exit__(exc_type, exc_value, traceback) + # Returns nothing per + # https://github.com/pytorch/pytorch/blob/0f6bfc58a2cfb7a5c052bea618ab62becaf5c912/torch/csrc/monitor/python_init.cpp#L315 + return None diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_foreach_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_foreach_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e88720a93ce3fad081a9ab05c6f7042b603f0e9a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_foreach_utils.py @@ -0,0 +1,60 @@ +from typing import TypeAlias + +import torch +from torch import Tensor +from torch.autograd.grad_mode import no_grad + + +def _get_foreach_kernels_supported_devices() -> list[str]: + r"""Return the device type list that supports foreach kernels.""" + return ["cuda", "xpu", "mtia", torch._C._get_privateuse1_backend_name()] + + +def _get_fused_kernels_supported_devices() -> list[str]: + r"""Return the device type list that supports fused kernels in optimizer.""" + return [ + "mps", + "cuda", + "xpu", + "hpu", + "cpu", + "mtia", + torch._C._get_privateuse1_backend_name(), + ] + + +TensorListList: TypeAlias = list[list[Tensor | None]] +Indices: TypeAlias = list[int] +_foreach_supported_types = [torch.Tensor] + + +# This util function splits tensors into groups by device and dtype, which is useful before sending +# tensors off to a foreach implementation, which requires tensors to be on one device and dtype. +# If tensorlistlist contains more than one tensorlist, the following assumptions are made BUT NOT verified: +# - tensorlists CAN be None +# - all tensors in the first specified list cannot be None +# - given an index i, all specified tensorlist[i]s match in dtype and device +# with_indices (bool, optional): whether to track previous indices as the last list per dictionary entry. +# It comes in handy if there are Nones or literals in the tensorlists that are getting scattered out. +# Whereas mutating a tensor in the resulting split-up tensorlists WILL propagate changes back to the +# original input tensorlists, changing up Nones/literals WILL NOT propagate, and manual propagation +# may be necessary. Check out torch/optim/sgd.py for an example. +@no_grad() +def _group_tensors_by_device_and_dtype( + tensorlistlist: TensorListList, + with_indices: bool = False, +) -> dict[tuple[torch.device, torch.dtype], tuple[TensorListList, Indices]]: + return torch._C._group_tensors_by_device_and_dtype(tensorlistlist, with_indices) + + +def _device_has_foreach_support(device: torch.device) -> bool: + return ( + device.type in (_get_foreach_kernels_supported_devices() + ["cpu"]) + and not torch.jit.is_scripting() + ) + + +def _has_foreach_support(tensors: list[Tensor], device: torch.device) -> bool: + return _device_has_foreach_support(device) and all( + t is None or type(t) in _foreach_supported_types for t in tensors + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_functools.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_functools.py new file mode 100644 index 0000000000000000000000000000000000000000..dd3a460efb1ccc6556a58b6d236c17b7cb7122ab --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_functools.py @@ -0,0 +1,48 @@ +import functools +from collections.abc import Callable +from typing import Concatenate, TypeVar +from typing_extensions import ParamSpec + + +_P = ParamSpec("_P") +_T = TypeVar("_T") +_C = TypeVar("_C") + +# Sentinel used to indicate that cache lookup failed. +_cache_sentinel = object() + + +def cache_method( + f: Callable[Concatenate[_C, _P], _T], +) -> Callable[Concatenate[_C, _P], _T]: + """ + Like `@functools.cache` but for methods. + + `@functools.cache` (and similarly `@functools.lru_cache`) shouldn't be used + on methods because it caches `self`, keeping it alive + forever. `@cache_method` ignores `self` so won't keep `self` alive (assuming + no cycles with `self` in the parameters). + + Footgun warning: This decorator completely ignores self's properties so only + use it when you know that self is frozen or won't change in a meaningful + way (such as the wrapped function being pure). + """ + cache_name = "_cache_method_" + f.__name__ + + @functools.wraps(f) + def wrap(self: _C, *args: _P.args, **kwargs: _P.kwargs) -> _T: + if kwargs: + raise AssertionError("cache_method does not accept keyword arguments") + if not (cache := getattr(self, cache_name, None)): + cache = {} + setattr(self, cache_name, cache) + # pyrefly: ignore [unbound-name] + cached_value = cache.get(args, _cache_sentinel) + if cached_value is not _cache_sentinel: + return cached_value + value = f(self, *args, **kwargs) + # pyrefly: ignore [unbound-name] + cache[args] = value + return value + + return wrap diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_get_clean_triton.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_get_clean_triton.py new file mode 100644 index 0000000000000000000000000000000000000000..fbbabc3f50e6248e33633af61885c8cf5d15d5d5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_get_clean_triton.py @@ -0,0 +1,200 @@ +# mypy: allow-untyped-defs +import argparse +import os +import re +import subprocess +import sys +from pathlib import Path + + +def remove_triton_function_declaration(source_code: str) -> str: + remove_head = re.sub(r"(\n.+\s\'\'\'\n)", "\n", source_code) + remove_tail = re.sub(r"(\'\'\'\,.+)", "\n", remove_head) + return remove_tail + + +def remove_async_compile(source_code: str) -> str: + remove_top_level = str.replace(source_code, "async_compile = AsyncCompile()", "") + remove_compile = str.replace(remove_top_level, "async_compile.wait(globals())", "") + remove_del = str.replace(remove_compile, "del async_compile", "") + return remove_del + + +def rename_kernels(source_code: str) -> str: + pattern = r"(\w+)\s*=\s*async_compile\.triton\('triton_',\s" + triton_kernel_decl = "def triton_" + matches = [ + (match.end(), match.group(1)) + for match in re.finditer(pattern, source_code, re.DOTALL) + ] + + # Starting from the last match to avoid issues with shifting indices after replacements + for end_index, captured_string in reversed(matches): + # Find the index of the next "B" after the current match + index_of_B = source_code.find(triton_kernel_decl, end_index) + if index_of_B != -1: + # Replace the triton_kernel_decl with the captured string + source_code = ( + source_code[:index_of_B] + + f"def {captured_string}" + + source_code[index_of_B + len(triton_kernel_decl) :] + ) + else: + # If triton_kernel_decl is not found after the current match, continue to the next + continue + + return source_code + + +def merge_params(original_params: list[str], new_params: list[str]) -> list[str]: + for idx in range(len(new_params)): + if new_params[idx] == "T": + new_params[idx] = original_params[idx] + return new_params + + +def add_launch_params( + original: str, kernel_to_params: dict[str, tuple[str, str]] +) -> str: + # Regex to match the function call in the original string + pattern = r"(\w+)\.run\((.*)\)" + + def replace(match) -> str: + # Extract parts from the regex match + func_name = match.group(1) + params = match.group(2) + new_params, grid = kernel_to_params[func_name] + new_params = merge_params(params.split(", "), new_params.split(", ")) + + # Format the new function call + new_string = f"{func_name}[{grid}]({', '.join(new_params)})" + return new_string + + transformed = re.sub(pattern, replace, original) + + remove_inductor_wrappers = re.sub( + r"@triton_heuristics[^@]*@triton.jit", + r"@triton.jit", + transformed, + flags=re.DOTALL, + ) + + return remove_inductor_wrappers + + +def process_file( + input_filename: str, output_filename: str, auto_generate_params: bool = True +) -> str: + with open(input_filename) as file: + source_code = file.read() + + transformed_code = source_code + if "def triton_(" in source_code: + raise RuntimeError( + "Need to run original Pytorch code generating kernels with TORCHINDUCTOR_UNIQUE_KERNEL_NAMES=1" + ) + # transformed_code = rename_kernels(transformed_code) + transformed_code = remove_triton_function_declaration(transformed_code) + transformed_code = remove_async_compile(transformed_code) + + launch_params_filename = f"{input_filename}.launch_params" + + # Auto-generate launch_params if they don't exist and auto_generate_params is True + if not os.path.exists(launch_params_filename) and auto_generate_params: + print(f"Launch params file {launch_params_filename} not found. Generating...") + try: + # Set environment variable and run the input file + env = os.environ.copy() + env["TORCHINDUCTOR_DUMP_LAUNCH_PARAMS"] = "1" + + result = subprocess.run( + [sys.executable, input_filename], + env=env, + capture_output=True, + text=True, + cwd=os.path.dirname(input_filename) or ".", + ) + + if result.returncode != 0: + print(f"Error running {input_filename}:") + print(f"stdout: {result.stdout}") + print(f"stderr: {result.stderr}") + raise RuntimeError( + f"Failed to generate launch params. Command failed with return code {result.returncode}" + ) + + print(f"Successfully generated {launch_params_filename}") + + except Exception as e: + raise RuntimeError( + f"Failed to generate launch params by running {input_filename}: {str(e)}" + ) from e + + if not os.path.exists(launch_params_filename): + raise RuntimeError( + f"Missing {launch_params_filename}. Run `TORCHINDUCTOR_DUMP_LAUNCH_PARAMS=1 python {input_filename}` first." + ) + + with open(launch_params_filename) as f: + launch_params_meta = f.readlines() + + split_params = [i.split("|") for i in launch_params_meta] + kernel_args_grid = {a.strip(): (b.strip(), c.strip()) for a, b, c in split_params} + transformed_code = add_launch_params(transformed_code, kernel_args_grid) + + with open(output_filename, "w") as file: + file.write(transformed_code) + print(f"Successfully generated {output_filename}") + return transformed_code + + +def get_clean_triton( + input_path: Path, + output_path: Path = Path("triton_only_repro.py"), + auto_generate_params: bool = True, +): + """Run experiments and output results to file + + Args: + input_path (Optional[Path]): Path to inductor generated output codede + output_path (Optional[Path]): Path to write out the new python file + auto_generate_params (bool): Whether to automatically generate launch_params if missing + """ + return process_file(str(input_path), str(output_path), auto_generate_params) + + +if __name__ == "__main__": + """Sample usage: + # Running sweep + python _get_clean_triton.py output_code.py + + # To disable auto-generation of launch params: + python _get_clean_triton.py output_code.py --no-auto-generate + """ + parser = argparse.ArgumentParser( + description="Clean Inductor generated code to remove Inductor dependencies" + ) + + # Add the arguments + parser.add_argument( + "input_path", type=Path, help="Path to inductor generated output code" + ) + parser.add_argument( + "--output_path", + type=Path, + default=Path("triton_only_repro.py"), + help="Path to write out the clean triton output", + ) + parser.add_argument( + "--no-auto-generate", + action="store_true", + help="Disable automatic generation of launch_params file", + ) + + # Parse the arguments + args = parser.parse_args() + + # Call the function with parsed arguments + result = get_clean_triton( + args.input_path, args.output_path, not args.no_auto_generate + ) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_helion.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_helion.py new file mode 100644 index 0000000000000000000000000000000000000000..6d30832cf3f74158267cd82c21f31e5744022161 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_helion.py @@ -0,0 +1,17 @@ +import functools + +from torch.utils._triton import has_triton + + +@functools.cache +def has_helion_package() -> bool: + try: + import helion # type: ignore[import-untyped, import-not-found] # noqa: F401 + except ImportError: + return False + return True + + +@functools.cache +def has_helion() -> bool: + return has_helion_package() and has_triton() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_import_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_import_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..47e48fb7144e5d60f82f746b0dc862677f322552 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_import_utils.py @@ -0,0 +1,37 @@ +import functools +import importlib.util +from types import ModuleType + + +def _check_module_exists(name: str) -> bool: + r"""Returns if a top-level module with :attr:`name` exists *without** + importing it. This is generally safer than try-catch block around a + `import X`. It avoids third party libraries breaking assumptions of some of + our tests, e.g., setting multiprocessing start method when imported + (see librosa/#747, torchvision/#544). + """ + try: + spec = importlib.util.find_spec(name) + return spec is not None + except ImportError: + return False + + +@functools.lru_cache +def dill_available() -> bool: + return _check_module_exists("dill") + + +@functools.lru_cache +def import_dill() -> ModuleType | None: + if not dill_available(): + return None + + import dill + + # XXX: By default, dill writes the Pickler dispatch table to inject its + # own logic there. This globally affects the behavior of the standard library + # pickler for any user who transitively depends on this module! + # Undo this extension to avoid altering the behavior of the pickler globally. + dill.extend(use_dill=False) + return dill diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_mode_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_mode_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b79b52b13449e829b6168bceeb07f254cb6c6180 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_mode_utils.py @@ -0,0 +1,15 @@ +# mypy: allow-untyped-defs +from typing import TypeVar + +import torch + + +T = TypeVar("T") + + +# returns if all are the same mode +def all_same_mode(modes): + return all(tuple(mode == modes[0] for mode in modes)) + + +no_dispatch = torch._C._DisableTorchDispatch diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_ordered_set.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_ordered_set.py new file mode 100644 index 0000000000000000000000000000000000000000..f00b4ac31ef7497858b12f20bcd1e3ef75cd381a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_ordered_set.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +from collections.abc import ( + Hashable, + Iterable, + Iterator, + MutableSet, + Reversible, + Set as AbstractSet, +) +from typing import Any, cast, TypeVar + + +T = TypeVar("T", bound=Hashable) +T_co = TypeVar("T_co", bound=Hashable, covariant=True) + +__all__ = ["OrderedSet"] + + +class OrderedSet(MutableSet[T], Reversible[T]): + """ + Insertion ordered set, similar to OrderedDict. + """ + + __slots__ = ("_dict",) + + def __init__(self, iterable: Iterable[T] | None = None) -> None: + self._dict = dict.fromkeys(iterable, None) if iterable is not None else {} + + @staticmethod + def _from_dict(dict_inp: dict[T, None]) -> OrderedSet[T]: + s: OrderedSet[T] = OrderedSet() + s._dict = dict_inp + return s + + # + # Required overridden abstract methods + # + def __contains__(self, elem: object) -> bool: + return elem in self._dict + + def __iter__(self) -> Iterator[T]: + return iter(self._dict) + + def __len__(self) -> int: + return len(self._dict) + + def __reversed__(self) -> Iterator[T]: + return reversed(self._dict) + + def add(self, elem: T) -> None: + self._dict[elem] = None + + def discard(self, elem: T) -> None: + self._dict.pop(elem, None) + + def clear(self) -> None: + # overridden because MutableSet impl is slow + self._dict.clear() + + # Unimplemented set() methods in _collections_abc.MutableSet + + @classmethod + def _wrap_iter_in_set(cls, other: Any) -> Any: + """ + Wrap non-Set Iterables in OrderedSets + + Some of the magic methods are more strict on input types than + the public apis, so we need to wrap inputs in sets. + """ + + if not isinstance(other, AbstractSet) and isinstance(other, Iterable): + return cls(other) + else: + return other + + def pop(self) -> T: + if not self: + raise KeyError("pop from an empty set") + # pyrefly: ignore [bad-return] + return self._dict.popitem()[0] + + def copy(self) -> OrderedSet[T]: + return OrderedSet._from_dict(self._dict.copy()) + + def difference(self, *others: Iterable[T]) -> OrderedSet[T]: + res = self.copy() + res.difference_update(*others) + return res + + def difference_update(self, *others: Iterable[T]) -> None: + for other in others: + self -= other # type: ignore[arg-type] + + def update(self, *others: Iterable[T]) -> None: + for other in others: + self |= other + + def intersection(self, *others: Iterable[T]) -> OrderedSet[T]: + res = self.copy() + for other in others: + if other is not self: + res &= other # type: ignore[arg-type] + return res + + def intersection_update(self, *others: Iterable[T]) -> None: + for other in others: + self &= other # type: ignore[arg-type] + + def issubset(self, other: Iterable[T]) -> bool: + return self <= self._wrap_iter_in_set(other) + + def issuperset(self, other: Iterable[T]) -> bool: + return self >= self._wrap_iter_in_set(other) + + def symmetric_difference(self, other: Iterable[T]) -> OrderedSet[T]: + return self ^ other # type: ignore[operator] + + def symmetric_difference_update(self, other: Iterable[T]) -> None: + self ^= other # type: ignore[arg-type] + + def union(self, *others: Iterable[T]) -> OrderedSet[T]: + res = self.copy() + for other in others: + if other is self: + continue + res |= other + return res + + # Specify here for correct type inference, otherwise would + # return AbstractSet[T] + def __sub__(self, other: AbstractSet[T_co]) -> OrderedSet[T]: + # following cpython set impl optimization + if isinstance(other, OrderedSet) and (len(self) * 4) > len(other): + out = self.copy() + out -= other + return out + return cast(OrderedSet[T], super().__sub__(other)) + + def __ior__(self, other: Iterable[T]) -> OrderedSet[T]: # type: ignore[misc, override] # noqa: PYI034 + if isinstance(other, OrderedSet): + self._dict.update(other._dict) + return self + return super().__ior__(other) # type: ignore[arg-type] + + def __eq__(self, other: object) -> bool: + if isinstance(other, OrderedSet): + return self._dict == other._dict + return super().__eq__(other) + + def __ne__(self, other: object) -> bool: + if isinstance(other, OrderedSet): + return self._dict != other._dict + return super().__ne__(other) + + def __or__(self, other: AbstractSet[T_co]) -> OrderedSet[T]: + return cast(OrderedSet[T], super().__or__(other)) + + def __and__(self, other: AbstractSet[T_co]) -> OrderedSet[T]: + # MutableSet impl will iterate over other, iter over smaller of two sets + if isinstance(other, OrderedSet) and len(self) < len(other): + # pyrefly: ignore [unsupported-operation, bad-return] + return other & self + return cast(OrderedSet[T], super().__and__(other)) + + def __xor__(self, other: AbstractSet[T_co]) -> OrderedSet[T]: + return cast(OrderedSet[T], super().__xor__(other)) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}({list(self)})" + + def __getstate__(self) -> list[T]: + return list(self._dict.keys()) + + def __setstate__(self, state: list[T]) -> None: + self._dict = dict.fromkeys(state, None) + + def __reduce__(self) -> tuple[type[OrderedSet[T]], tuple[list[T]]]: + return (OrderedSet, (list(self),)) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_pallas.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_pallas.py new file mode 100644 index 0000000000000000000000000000000000000000..86359e4e556a75653a9380d5ff132d75529321c4 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_pallas.py @@ -0,0 +1,103 @@ +import functools + +import torch + + +@functools.cache +def has_jax_package() -> bool: + """Check if JAX is installed.""" + try: + import jax # noqa: F401 # type: ignore[import-not-found] + + return True + except ImportError: + return False + + +@functools.cache +def has_pallas_package() -> bool: + """Check if Pallas (JAX experimental) is available.""" + if not has_jax_package(): + return False + try: + from jax.experimental import ( # noqa: F401 # type: ignore[import-not-found] + pallas as pl, + ) + + return True + except ImportError: + return False + + +@functools.cache +def get_jax_version(fallback: tuple[int, int, int] = (0, 0, 0)) -> tuple[int, int, int]: + """Get JAX version as (major, minor, patch) tuple.""" + try: + import jax # type: ignore[import-not-found] + + version_parts = jax.__version__.split(".") + major, minor, patch = (int(v) for v in version_parts[:3]) + return (major, minor, patch) + except (ImportError, ValueError, AttributeError): + return fallback + + +@functools.cache +def has_jax_cuda_backend() -> bool: + """Check if JAX has CUDA backend support.""" + if not has_jax_package(): + return False + try: + import jax # type: ignore[import-not-found] + + # Check if CUDA backend is available + devices = jax.devices("gpu") + return len(devices) > 0 + except Exception: + return False + + +@functools.cache +def has_jax_tpu_backend() -> bool: + """Check if JAX has TPU backend support.""" + if not has_jax_package(): + return False + try: + import jax # type: ignore[import-not-found] + + # Check if TPU backend is available + devices = jax.devices("tpu") + return len(devices) > 0 + except Exception: + return False + + +@functools.cache +def has_cpu_pallas() -> bool: + """Checks for a full Pallas-on-CPU environment.""" + return has_pallas_package() + + +@functools.cache +def has_cuda_pallas() -> bool: + """Checks for a full Pallas-on-CUDA environment.""" + return has_pallas_package() and torch.cuda.is_available() and has_jax_cuda_backend() + + +@functools.cache +def has_tpu_pallas() -> bool: + """Checks for a full Pallas-on-TPU environment.""" + return has_pallas_package() and has_jax_tpu_backend() + + +@functools.cache +def has_pallas() -> bool: + """ + Check if Pallas backend is fully available for use. + + Requirements: + - JAX package installed + - Pallas (jax.experimental.pallas) available + - A compatible backend (CUDA or TPU) is available in both PyTorch and JAX. + """ + return has_cpu_pallas() or has_cuda_pallas() or has_tpu_pallas() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_python_dispatch.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_python_dispatch.py new file mode 100644 index 0000000000000000000000000000000000000000..7116d584892ea37cddde5b793140a73bdae19d61 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_python_dispatch.py @@ -0,0 +1,911 @@ +# mypy: allow-untyped-defs +from __future__ import annotations + +import contextlib +import functools +import warnings +from collections import deque +from dataclasses import dataclass +from typing import cast, overload, Protocol, TYPE_CHECKING +from typing_extensions import TypeIs + +import torch +import torchgen +import torchgen.model +from torch._C import ( + _get_dispatch_stack_at, + _len_torch_dispatch_stack, + _pop_torch_dispatch_stack, + _push_on_torch_dispatch_stack, + DispatchKey, +) +from torch._C._dynamo.guards import set_is_in_mode_without_ignore_compile_internals + + +if TYPE_CHECKING: + from collections.abc import Sequence + + +# TODO: Limitations and things about enable_torch_dispatch_mode we should fix before exposing it: +# - We need a better user-facing api for _DisableTorchDispatch that +# is able to selectively disable __torch_dispatch__ of a particular class. +# - It doesn't work with the tensor constructors (torch.tensor, torch.Tensor) +# - Better name (see https://github.com/pytorch/pytorch/pull/63496#discussion_r694091694) + +_is_in_torch_dispatch_mode = False +_is_in_non_infra_torch_dispatch_mode = False +# If inside any mode that has ignore_compile_internals() = False +_is_in_any_mode_without_ignore_compile_internals = False + + +def is_in_torch_dispatch_mode(include_infra_modes: bool = True) -> bool: + return ( + _is_in_torch_dispatch_mode + if include_infra_modes + else _is_in_non_infra_torch_dispatch_mode + ) + + +def is_in_any_mode_without_ignore_compile_internals() -> bool: + return _is_in_any_mode_without_ignore_compile_internals + + +class TorchDispatchMode: + """ + A ``TorchDispatchMode`` allows you to override the meaning of all + ``__torch_dispatch__`` overrideable functions within a dynamic scope, + without having to actually create a tensor subclass or manually + monkey-patch functions in the PyTorch API. Some common situations + where you should use a mode: + + * You want to override the meaning of factory functions, or other + functions that do not otherwise take a tensor as an argument + (these cannot be overridden with tensor subclasses). + + * You want to override the behavior of all functions without needing + to wrap your inputs in tensor subclasses; e.g., if you are just + interested in logging intermediate computations. + + * You want to control the order of execution of various tensor + subclasses explicitly, rather than implicitly via the return of + ``NotImplemented``. + + Independent subclasses of :class:`TorchDispatchMode` are compositional: + modes can be pushed onto a stack using ``with MyMode():``. + When you call functions in the PyTorch API inside your + ``__torch_dispatch__`` implementation, by default, they will forward on to + the next mode on the mode stack. If you want recursively call back into + your current ``__torch_dispatch__`` implementation, either explicitly + invoke ``self.__torch_dispatch__(...)``, or use the context manager + ``self`` to make PyTorch + API self-referential (beware of infinite loops, in this case!) + """ + + # - When False, custom torch dispatch mode will error out explicitly when a hop + # is called under the mode. + # - When True, custom torch dispatch mode's __torch_dispatch__ will be triggered. + # Mode authors can implement how the mode interacts with higher order operators. + supports_higher_order_operators = False + + def __init__(self, _dispatch_key=None) -> None: + if _dispatch_key is not None: + if not isinstance(_dispatch_key, torch._C.DispatchKey): + raise AssertionError("_dispatch_key must be a torch._C.DispatchKey") + self.__dict__["_dispatch_key"] = _dispatch_key + + self.old_dispatch_mode_flags: deque[bool] = deque() + self.old_non_infra_dispatch_mode_flags: deque[bool] = deque() + self.old_without_ignore_compile_internals_dispatch_mode_flags: deque[bool] = ( + deque() + ) + + def _lazy_init_old_dispatch_mode_flags(self) -> None: + if not hasattr(self, "old_dispatch_mode_flags"): + self.old_dispatch_mode_flags: deque[bool] = deque() # type: ignore[no-redef] + + if not hasattr(self, "old_non_infra_dispatch_mode_flags"): + self.old_non_infra_dispatch_mode_flags: deque[bool] = deque() # type: ignore[no-redef] + + if not hasattr( + self, "old_without_ignore_compile_internals_dispatch_mode_flags" + ): + self.old_without_ignore_compile_internals_dispatch_mode_flags: deque[ # type: ignore[no-redef] + bool + ] = deque() + + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + raise NotImplementedError + + def __enter__(self): + global _is_in_torch_dispatch_mode + global _is_in_non_infra_torch_dispatch_mode + global _is_in_any_mode_without_ignore_compile_internals + + # Previously, there wasn't any state in this class' constructor + # super calls were added to existing modes, but for any new modes + # this will replicate the previous behavior of not strictly needing + # to call super().__init__() + self._lazy_init_old_dispatch_mode_flags() + self.old_dispatch_mode_flags.append(_is_in_torch_dispatch_mode) + _is_in_torch_dispatch_mode = True + self.old_non_infra_dispatch_mode_flags.append( + _is_in_non_infra_torch_dispatch_mode + ) + _is_in_non_infra_torch_dispatch_mode = ( + _is_in_non_infra_torch_dispatch_mode or not self.is_infra_mode() + ) + self.old_without_ignore_compile_internals_dispatch_mode_flags.append( + _is_in_any_mode_without_ignore_compile_internals + ) + _is_in_any_mode_without_ignore_compile_internals = ( + _is_in_any_mode_without_ignore_compile_internals + or not self.ignore_compile_internals() + ) + set_is_in_mode_without_ignore_compile_internals( + _is_in_any_mode_without_ignore_compile_internals + ) + _push_mode(self) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + mb_dk_or_mode_key = self.__dict__.get("_dispatch_key", None) + if mb_dk_or_mode_key is None: + # Today, mode keys are not used at all in the per-dispatch-key-mode logic (for pre-dispatch) + # We should probably revisit this. + mb_dk_or_mode_key = self.__dict__.get("_mode_key", None) + global _is_in_torch_dispatch_mode + _is_in_torch_dispatch_mode = self.old_dispatch_mode_flags.pop() + global _is_in_non_infra_torch_dispatch_mode + _is_in_non_infra_torch_dispatch_mode = ( + self.old_non_infra_dispatch_mode_flags.pop() + ) + global _is_in_any_mode_without_ignore_compile_internals + _is_in_any_mode_without_ignore_compile_internals = ( + self.old_without_ignore_compile_internals_dispatch_mode_flags.pop() + ) + set_is_in_mode_without_ignore_compile_internals( + _is_in_any_mode_without_ignore_compile_internals + ) + _pop_mode(mb_dk_or_mode_key) + + @classmethod + def push(cls, *args, **kwargs): + warnings.warn( + "`Mode.push()` is no longer necessary and can be replaced with just `with Mode()`", + stacklevel=2, + ) + instance = cls(*args, **kwargs) + return instance + + @classmethod + def is_infra_mode(cls) -> bool: + return False + + @classmethod + def ignore_compile_internals(cls) -> bool: + """Ignore operators that are compiled via torch.compile. + + If ``True``, then this TorchDispatchMode ignores operators that + are optimized by :func:`torch.compile`. Mechanically, this involves + turning off the TorchDispatchMode throughout the whole compilation process, + and turning it back on for the runtime of the compiled artifact(s). + For example, + + @torch.compile + def f(x): + return x.sin().cos() + + with LoggingMode(): + f(x) + + The above example will not log anything if + ``LoggingMode.ignore_compile_internals()`` is True. + torch.compile will fuse sin() and cos() into a single operation + and this TorchDispatchMode will not be passed sin and cos. + + If ``False`` (default), :func:`torch.compile` will respect + the eager semantics of passing this TorchDispatchMode all + operators that would have run during eager execution. + The way this will usually happen is that :func:`torch.compile` + will just fallback to eager-mode PyTorch. + """ + if cls.is_infra_mode(): + return True + return False + + +def _get_current_dispatch_mode() -> TorchDispatchMode | None: + """ + Return the top user mode on the stack (the next one that would be + executed) if there are any. + """ + stack_len = _len_torch_dispatch_stack() + if stack_len > 0: + return _get_dispatch_stack_at(stack_len - 1) + return None + + +def _detect_infra_mode(key): + if key not in ( + torch._C._TorchDispatchModeKey.FUNCTIONAL, + torch._C._TorchDispatchModeKey.PROXY, + ): + raise AssertionError( + f"key must be either FUNCTIONAL ({torch._C._TorchDispatchModeKey.FUNCTIONAL}) \ + or PROXY ({torch._C._TorchDispatchModeKey.PROXY}) _TorchDispatchModeKey, \ + got {key}" + ) + from torch._ops import _get_dispatch_mode_pre_dispatch + + pre_dispatch_mode = _get_dispatch_mode_pre_dispatch(key) + post_dispatch_mode = torch._C._get_dispatch_mode(key) + + if pre_dispatch_mode is not None and post_dispatch_mode is not None: + raise AssertionError( + "At most one of pre_dispatch_mode and post_dispatch_mode may be active" + ) + + if pre_dispatch_mode is None: + return post_dispatch_mode + + return pre_dispatch_mode + + +def _unset_infra_mode(key): + from torch._ops import _get_dispatch_mode_pre_dispatch, unset_mode_pre_dispatch + + pre_dispatch_mode = _get_dispatch_mode_pre_dispatch(key) + post_dispatch_mode = torch._C._get_dispatch_mode(key) + if pre_dispatch_mode and post_dispatch_mode: + raise AssertionError( + "Can't have active infra mode on both pre and post dispatch mode stack" + ) + + if pre_dispatch_mode: + mode = unset_mode_pre_dispatch(key) + return mode + if post_dispatch_mode: + return torch._C._unset_dispatch_mode(key) + + +def _disable_infra_mode(key): + if key not in ( + torch._C._TorchDispatchModeKey.FUNCTIONAL, + torch._C._TorchDispatchModeKey.PROXY, + ): + raise AssertionError( + "key must be either FUNCTIONAL or PROXY _TorchDispatchModeKey" + ) + mode_unset = _unset_infra_mode(key) + try: + yield mode_unset + finally: + if mode_unset is not None: + _push_mode(mode_unset) + + +def _get_current_dispatch_mode_stack() -> list[TorchDispatchMode]: + """ + Returns the current stack of dispatch modes, with the most recent + (i.e., the one that will be processed first) at the end of the + list (standard stack convention). + """ + stack_len = _len_torch_dispatch_stack() + return [_get_dispatch_stack_at(i) for i in range(stack_len)] + + +def _push_mode(mode: TorchDispatchMode) -> None: + k = mode._dispatch_key if hasattr(mode, "_dispatch_key") else None + if k is not None and k != torch._C.DispatchKey.PreDispatch: + raise AssertionError( + "mode._dispatch_key must be None or DispatchKey.PreDispatch" + ) + if k is None: + _push_on_torch_dispatch_stack(mode) + return + + from torch._ops import _set_mode_pre_dispatch, get_cached_ops + + # See Note [Not Caching Per-Dispatch-Key Mode Handlers] + # Clear the cache of every op that has been used so far, for this particular key. + ks = torch._C._functionality_to_backend_keys(k) + for op in get_cached_ops(): + for key in ks: + op._uncache_dispatch(key) + _set_mode_pre_dispatch(mode) + + +def _pop_mode(k: DispatchKey | torch._C._TorchDispatchModeKey | None = None): + if k == torch._C.DispatchKey.PreDispatch: # type: ignore[attr-defined] + from torch._ops import _pop_mode_from_pre_dispatch + + return _pop_mode_from_pre_dispatch() + + if k is None or isinstance(k, torch._C._TorchDispatchModeKey): + return _pop_torch_dispatch_stack(k) + + +@contextlib.contextmanager +def _pop_mode_temporarily(k: DispatchKey | None = None): + old = _pop_mode(k) + try: + yield old + finally: + _push_mode(old) + + +@contextlib.contextmanager +def _disable_current_modes(): + from torch._ops import ( + _len_torch_dispatch_stack_pre_dispatch, + _pop_mode_from_pre_dispatch, + ) + from torch._subclasses.functional_tensor import FunctionalTensorMode + from torch._subclasses.schema_check_mode import SchemaCheckMode + from torch.fx.experimental.proxy_tensor import ProxyTorchDispatchMode + + mode_len_pre_dispatch = _len_torch_dispatch_stack_pre_dispatch() + old_pre_dispatch_modes = [ + _pop_mode_from_pre_dispatch() for _ in range(mode_len_pre_dispatch) + ] + + has_proxy_mode_in_pre_dispatch = False + has_functional_mode_in_pre_dispatch = False + has_schema_check_mode_in_pre_dispatch = False + + for i in old_pre_dispatch_modes: + if isinstance(i, ProxyTorchDispatchMode): + has_proxy_mode_in_pre_dispatch = True + if isinstance(i, FunctionalTensorMode): + has_functional_mode_in_pre_dispatch = True + if isinstance(i, SchemaCheckMode): + has_schema_check_mode_in_pre_dispatch = True + + mode_len = _len_torch_dispatch_stack() + old_modes = [_pop_mode() for _ in range(mode_len)] + + for old in old_modes: + if ( + isinstance(old, FunctionalTensorMode) + and has_functional_mode_in_pre_dispatch + ): + raise AssertionError( + "Can't have FunctionalMode available both in PreDispatch and Python Key" + ) + if isinstance(old, ProxyTorchDispatchMode) and has_proxy_mode_in_pre_dispatch: + raise AssertionError( + "Can't have ProxyTorchDispatchMode available both in PreDispatch and Python Key" + ) + if isinstance(old, SchemaCheckMode) and has_schema_check_mode_in_pre_dispatch: + raise AssertionError( + "Can't have SchemaCheckMode available both in PreDispatch and Python Key" + ) + + # Manually disable proxy and fake modes, if any are active + try: + yield old_pre_dispatch_modes + old_modes + finally: + for mode in reversed(old_modes): + _push_mode(mode) + for mode in reversed(old_pre_dispatch_modes): + _push_mode(mode) + + +class BaseTorchDispatchMode(TorchDispatchMode): + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + if kwargs is None: + kwargs = {} + return func(*args, **kwargs) + + +# Subtypes which have __tensor_flatten__ and __tensor_unflatten__. +class TensorWithFlatten(Protocol): + def __tensor_flatten__(self) -> tuple[Sequence[str], object]: ... + + @staticmethod + def __tensor_unflatten__( + inner_tensors: int, flatten_spec: int, outer_size: int, outer_stride: int + ) -> torch.Tensor: ... + + # It would be really nice to be able to say that the return of + # is_traceable_wrapper_subclass() is Intersection[torch.Tensor, + # TensorWithFlatten] - but that doesn't exist. + + shape: torch._C.Size + + @overload + def stride(self, dim: None = None) -> tuple[int, ...]: ... + + @overload + def stride(self, dim: int) -> int: ... + + @overload + def size(self, dim: None = None) -> tuple[int, ...]: ... + + @overload + def size(self, dim: int) -> int: ... + + def storage_offset(self) -> int: ... + + def dim(self) -> int: ... + + @overload + def to( + self, + dtype: torch.types._dtype, + non_blocking: bool = False, + copy: bool = False, + *, + memory_format: torch.memory_format | None = None, + ) -> torch.Tensor: ... + + @overload + def to( + self, + device: torch._prims_common.DeviceLikeType | None = None, + dtype: torch.types._dtype | None = None, + non_blocking: bool = False, + copy: bool = False, + *, + memory_format: torch.memory_format | None = None, + ) -> torch.Tensor: ... + + @overload + def to( + self, + other: torch.Tensor, + non_blocking: bool = False, + copy: bool = False, + *, + memory_format: torch.memory_format | None = None, + ) -> torch.Tensor: ... + + +def is_traceable_wrapper_subclass(t: object) -> TypeIs[TensorWithFlatten]: + """ + Returns whether or not a tensor subclass that implements __torch_dispatch__ + is 'traceable' with torch.compile. + In order for a tensor subclass to support TorchDispatchMode-style tracing in PT2, + It must implement two magic methods: __tensor_flatten__ and __tensor_unflatten__. + It is also expected to obey some restrictions around traceability and aliasing: + * The subclass's __torch_dispatch__() implementation should desugar into pytorch + dispatcher operations that can be traced into a graph. + * The subclass should use return_and_correct_aliasing(). This is needed today to make + sure that torch.compile does the right thing in a few cases around input mutation + and output aliasing. + + Expected magic method signatures: + attrs, ctx = t.__tensor_flatten__() + attrs: list of attribute name strings for inner tensors + ctx: dict containing any other subclass-specific metadata needed for unflattening + + t = MySubClass.__tensor_unflatten__(inner_tensors, ctx, outer_size, outer_stride) + inner_tensors: dict mapping attribute name -> tensor for each inner tensor + ctx: dict with subclass metadata in the form that __tensor_flatten__() produces + outer_size: expected (possibly symbolic) size that the returned subclass + instance should have. Note that this arg is useful for certain subclasses + that require the shape info to be constructed. In most cases, this arg can be + safely ignored. + outer_stride: expected (possibly symbolic) stride that the returned subclass + instance should have. Note that this arg is useful for certain subclasses + that require the stride info to be constructed. In most cases, this arg can be + safely ignored. + """ + is_subclass = isinstance(t, torch.Tensor) and type(t) is not torch.Tensor + return ( + is_subclass + and hasattr(t, "__tensor_flatten__") + and hasattr(t, "__tensor_unflatten__") + ) + + +def is_traceable_wrapper_subclass_type(t: type) -> TypeIs[type[TensorWithFlatten]]: + """Same as above, but takes a type argument instead of an instance.""" + return ( + issubclass(t, torch.Tensor) + and t is not torch.Tensor + and hasattr(t, "__tensor_flatten__") + and hasattr(t, "__tensor_unflatten__") + ) + + +def transform_subclass(t, callback, outer_size=None, outer_stride=None): + """ + Given a traceable, wrapper tensor subclass ``t`` that implements + ``__torch_dispatch__`` and holds some inner tensors, + and a callback of type ``Callable[[str, torch.Tensor], torch.Tensor]``, + `transform_subclass` will construct a fresh instance of the wrapper tensor subclass. + It will do so by grabbing each inner tensor attribute from the wrapper, + passing them into ``callback`` to get a transformed tensor, + and putting each transformed tensor into the fresh tensor subclass instance. + + Note: this function will not handle ensuring that the fresh subclass + gets the same (autograd, and aliasing) metadata as the original tensor. + This is generally handled in other subsystems like AOTAutograd. + """ + outer_size = outer_size if outer_size is not None else t.size() + outer_stride = outer_stride if outer_stride is not None else t.stride() + + attrs, ctx = t.__tensor_flatten__() + transformed_tensors_dict = {} + for attr in attrs: + transformed_tensors_dict[attr] = callback(attr, getattr(t, attr)) + sub = type(t).__tensor_unflatten__( + transformed_tensors_dict, ctx, outer_size, outer_stride + ) + + # NB: Purposefully guard here to simplify the inner / outer symbols. + # Using sym_eq() for symbolic comparison can result in an expression that's too + # difficult to guard on, so we use == here. + if sub.shape != outer_size: + raise AssertionError( + f"Expected return value from {type(t)}__tensor_unflatten__() to have " + f"shape equal to {outer_size}, but got: {sub.shape}" + ) + if sub.stride() != outer_stride: + raise AssertionError( + f"Expected return value from {type(t)}__tensor_unflatten__() to have " + f"stride equal to {outer_stride}, but got: {sub.stride()}" + ) + + return sub + + +def _correct_storage_aliasing(func, schema_info, args, outs) -> None: + """ + Given: an OpOverload, a SchemaInfo (cached information from torchgen about schema), + and the inputs/outputs to the OpOverload, + this function checks to see if func is a view operator + (by checking if any of the outputs in the op's schema + are immutable aliases of inputs). + If so, this function manually aliases the storage of the output tensor + with its corresponding input tensor alias. + It does this by unsafely overwriting the storage field of the output tensor + to be the same storage as the input. + """ + if not isinstance(func, torch._ops.OpOverload): + raise AssertionError(f"func must be an OpOverload, got {type(args)}") + if not isinstance(args, tuple): + raise AssertionError(f"args must be a tuple, got {type(args)}") + if not isinstance(outs, (list, tuple)): + raise AssertionError(f"outs must be a list or tuple, got {type(args)}") + + def alias_non_inplace_storage(arg, ret) -> None: + # This is hopefully a reasonable assert: + # subclasses that rely on this API for output aliasing + # should always return wrapper tensor subclasses for us to manually alias. + # in theory if a subclass that needs this API wants to sometimes return + # plain tensors, we could remove the assert and just not perform the aliasing, + # but it seems safer to learn more about this case first. + # + # Performance note: This is all just to assert that the argument and result + # types match, checking that is cheaper than is_traceable_wrapper_subclass_type, + # and multiple returns are relatively unlikely, so just check up front! + arg_type = type(arg) + ret_type = type(ret) + if arg_type is not ret_type and ( + is_traceable_wrapper_subclass_type(arg_type) + or is_traceable_wrapper_subclass_type(ret_type) + ): + ret_list = ret if isinstance(ret, list) else [ret] + for r in ret_list: + if type(arg) is not type(r): + raise AssertionError( + f"Called {str(func)} with input of type {type(arg)}\n" + f"and output of type {type(ret)}. But expected types to match." + ) + # Need to call a non-dispatcher helper, because we explicitly do **not** + # want our subclass to intercept the set_() call. + # instead, our subclass should directly have its storage swapped out. + # we **explicitly** don't want to reset the sizes on ret, if the storage implies a size change. + # Why? + # The purpose of this API is *not* to change the size/strides of our output- we assume it's already correct. + # We just want to "fix up" the storage aliasing, without modifying or output's metadata. + # Example: out = inp.expand(inp.shape[0], inp.shape[0]) + # This requires swapping the storage of out to be the same as inp, + # but we do *not* want it to change the sizes/strides that were compute for out. + + if isinstance(ret, list): + for r in ret: + torch._functionalize_unsafe_set(r, arg) + else: + if not isinstance(ret, torch.Tensor): + raise AssertionError(f"expected torch.Tensor, got {type(ret)}") + torch._functionalize_unsafe_set(ret, arg) + + for arg_idx, return_idx in schema_info.read_only_alias_match_indexes: + alias_non_inplace_storage(args[arg_idx], outs[return_idx]) + + +def _get_write_alias(x) -> str | None: + alias_set = x.alias_set + if not alias_set or not x.is_write: + return None + # torchscript allows for complicated alias sets, but our dispatcher ops only really involve simple aliasing + if len(alias_set) != 1: + raise AssertionError("Expected alias_set to contain exactly one element") + # timeit says next(iter(alias_set)) is faster than list(alias_set)[0] even for + # set of size 1 on Python 3.13. + return next(iter(alias_set)) + + +# This abstracts over the fact that in return_and_correct_aliasing, +# we sometimes use torchgen schema parsing (for aten ops, since torchscript's schema parsing is sometimes buggy), +# and sometimes use torchscript schema parsing (for custom ops, for which torchgen parsing is untested). +@dataclass +class AliasInfo: + alias_set: set[str] + is_write: bool + name: str | None + + +@dataclass +class SchemaInfo: + args: list[AliasInfo] + outs: list[AliasInfo] + + is_inplace_view_op: bool + + # [_get_write_alias(x) for x in outs]. Guaranteed to contain no Nones; we coerce + # all-Nones result to empty list instead, and we don't support + # some-but-not-all-Nones. + outs_write_aliases: list[str] | None + + # List of (arg_idx, return_idx) where args[arg_idx].alias_set & + # outs[out_idx].alias_set is not empty, and not args[arg_idx].is_write. + read_only_alias_match_indexes: list[tuple[int, int]] + + +# Given an OpOverload, returns schema information on it. +# This is cached for efficiency, since it can involve running torchgen +@functools.cache +def get_alias_info(func) -> SchemaInfo: + # For ATen ops: use torchgen (since torchscript parser doesn't handle alias annotations + # properly for some ops that output tensorlists) + if func.namespace == "aten": + torchgen_schema_str = str(func._schema) + if not torchgen_schema_str.startswith("aten::"): + raise AssertionError( + "Expected torchgen schema string to start with 'aten::'" + ) + # remove the aten:: namespace, which is added by the torchscript parser, + # and torchgen doesn't know how to handle + torchgen_schema_str = torchgen_schema_str[6:] + import re + + # the torchscript parser ends up converting int[2]=1 into int[2]=[1, 1], + # which torchgen chokes on. + torchgen_schema_str = re.sub(r"=\[[0, ]+\]", "=0", torchgen_schema_str) + torchgen_schema_str = re.sub(r"=\[[1, ]+\]", "=1", torchgen_schema_str) + # for aten::rot90 / aten:fft_* + torchgen_schema_str = re.sub( + r"=\[(-?[0-9]+), (-?[0-9]+)\]", r"=[\1,\2]", torchgen_schema_str + ) + torchgen_schema = torchgen.model.FunctionSchema.parse(torchgen_schema_str) + arg_schemas = [ + AliasInfo( + alias_set=( + set() if a.annotation is None else set(a.annotation.alias_set) + ), + is_write=a.annotation is not None and a.annotation.is_write, + name=a.name, + ) + for a in torchgen_schema.arguments.flat_all + ] + out_schemas = [ + AliasInfo( + alias_set=( + set() if a.annotation is None else set(a.annotation.alias_set) + ), + is_write=a.annotation is not None and a.annotation.is_write, + name=a.name, + ) + for a in torchgen_schema.returns + ] + else: + # For non-aten ops, torchgen is untested so we rely on torchscript schema parsing + arg_schemas = [ + AliasInfo( + alias_set=( + set() if a.alias_info is None else set(a.alias_info.before_set) + ), + is_write=a.alias_info is not None and a.alias_info.is_write, + name=a.name, + ) + for a in func._schema.arguments + ] + out_schemas = [ + AliasInfo( + alias_set=( + set() if a.alias_info is None else set(a.alias_info.before_set) + ), + is_write=a.alias_info is not None and a.alias_info.is_write, + name=a.name, + ) + for a in func._schema.returns + ] + read_only_alias_match_indexes = [] + for arg_idx, schema_arg in enumerate(arg_schemas): + for return_idx, schema_out in enumerate(out_schemas): + is_read_only_alias_match = ( + schema_arg.alias_set & schema_out.alias_set + ) and not schema_arg.is_write + if is_read_only_alias_match: + read_only_alias_match_indexes.append((arg_idx, return_idx)) + + outs_write_aliases_list: list[str | None] = [ + _get_write_alias(r) for r in out_schemas + ] + non_nones = sum(x is not None for x in outs_write_aliases_list) + if non_nones == 0: + outs_write_aliases: list[str] | None = None + elif non_nones != len(outs_write_aliases_list): + # simplifying assumption: we don't have **any** ops with return types like "-> (Tensor(a!), Tensor)" + raise RuntimeError("Unsupported schema: " + str(func._schema)) + else: + outs_write_aliases = cast(list[str], outs_write_aliases_list) + + schema_info = SchemaInfo( + args=arg_schemas, + outs=out_schemas, + # This check is surprisingly expensive because pybind11 enum_s are + # inefficient. Just cache it. + is_inplace_view_op=torch.Tag.inplace_view in func.tags, + outs_write_aliases=outs_write_aliases, + read_only_alias_match_indexes=read_only_alias_match_indexes, + ) + return schema_info + + +def autograd_would_have_decomposed( + func: torch._ops.OpOverload, flat_args: Sequence[torch.Tensor | object] +) -> bool: + """ + Suppose that an operator has CompositeImplicitAutograd decomp registered. + Would autograd have used this decomposition? It will only use it if there + isn't an explicit backend registration for the device as well. This function + will tell if this would have occurred. + + Why do we need to apply these decompositions later? When inference mode is + on, the autograd key is bypassed entirely, so a lower level mode cannot rely + on the decomposition have been applied. It's easy to accidentally never apply + the decomposition, resulting in an operator showing up in a graph that + is unexpected. + + Why do we need to AVOID applying the decomposition when autograd wouldn't + have decomposed? If autograd doesn't decompose, this means in eager mode + we would have run the fused kernel. It must be possible to trace this + fused kernel directly into the graph for fidelity with eager (NB: a user + has the option of then further decomposing at proxy tensor mode via + decomposition table, but we must preserve it to proxy mode to have the + choice.) + + Why does functionalization need to also perform the test here? This is + because some CompositeImplicitAutograd decompositions are not functional. + If we are eventually going to decompose, we need to do this while we can + still turn functionalization back on, so those decompositions get functionalized. + So an early decomposition in functionalization may still be necessary. Note that + if proxy tensor decomposition process could turn functionalization back on, this + wouldn't be necessary, and maybe that is a useful thing to do anyway because + the decomposition table is user specified and a user could violate the functional + decomp requirement with a bad decomp. If this happened, then you could always + pass through functionalization. + """ + has_backend_registration = False + for a in flat_args: + if isinstance(a, torch.Tensor): + backend_key = torch._C._parse_dispatch_key( + torch._C._dispatch_key_for_device(a.device.type) + ) + assert backend_key is not None + # TODO: use func.has_kernel_for_dispatch_key(backend_key) + # but this one checks py_impl and CompositeImplicitAutograd + # incorrectly shows up as has backend reg here + has_backend_registration = torch._C._dispatch_has_kernel_for_dispatch_key( + func.name(), backend_key + ) + + # in theory we should take all backend keys and take the highest priority one + # to properly mimic the dispatcher, + # this just grabs the first tensor and takes its device key + break + return not has_backend_registration + + +def return_and_correct_aliasing(func, args, kwargs, out): + """ + This function should be used by wrapper tensor ``__torch_dispatch__`` subclasses + that would like to work with torch.compile. It ensures that the subclass + properly implements the aliasing behavior of every op, + which is needed for correctness in AOTAutograd. + This function will handle: + + * When we see a view op, we will alias the storages of any + input and output tensor subclasses + + * When we see an inplace or out= op, we will directly + return the corresponding input tensor, instead of returning + a (potentially) fresh output tensor. + """ + + # Caching here because torchgen parsing is definitely not fast, and this function is called + # once for every op in the graph during functionalization. + schema_info = get_alias_info(func) + + def get_arg_from_alias(output_alias, schema_info, args, kwargs): + new_args, new_kwargs = torch.fx.operator_schemas.normalize_function( # type: ignore[misc] + func, args=args, kwargs=kwargs + ) + + arg_indices = [ + i for i, a in enumerate(schema_info.args) if output_alias in a.alias_set + ] + # For any dispatcher op with an output alias, we expect it to map to exactly one alias in the schema's input arguments. + if len(arg_indices) != 1: + raise AssertionError( + "Expected exactly one argument index for the given output alias" + ) + idx = arg_indices[0] + arg_info = schema_info.args[idx] + if arg_info.name is not None and arg_info.name in new_kwargs: + return new_kwargs[arg_info.name] + return new_args[idx] + + # Fix up the storages of any outs so that they point to the same storage as the input, + # if func is a view op. + _correct_storage_aliasing( + func, schema_info, args, (out,) if not isinstance(out, tuple) else out + ) + + # For inplace_view ops in particular, we'll try hard to make sure that the wrapper subclass's + # metadata is set correctly. + if schema_info.is_inplace_view_op: + # no_dispatch() to make sure that we secretly change the metadata on the wrapper, + # but don't end up dispatching the op anywhere else. + mutated_args = [ + x + for i, x in enumerate(args) + if _get_write_alias(schema_info.args[i]) is not None + ] + # Assumption: we have a very small number of inplace_view ops that follow a strict schema: + # there is only a single argument that gets its metadata mutated. + if len(mutated_args) != 1: + raise AssertionError( + "expected exactly one mutated arg for inplace_view ops" + ) + # This check exists because we generally *do* want to update the metadata of any wrapper subclasses, + # but FunctionalTensor is special: it overrides all size/stride calls to plumb to the inner tensor. + # so we don't actually need to update the metadata (and attempting to do so causes errors) + from torch._subclasses.functional_tensor import FunctionalTensor + + if not isinstance(mutated_args[0], FunctionalTensor): + with torch.utils._mode_utils.no_dispatch(): + # See Note: [Fake Tensor Dispatch Keys] + # we're borrowing the way it modifies dispatch key TLS. + meta_in_tls = torch._C._meta_in_tls_dispatch_include() + torch._C._set_meta_in_tls_dispatch_include(True) + try: + func(*args, **kwargs) + finally: + torch._C._set_meta_in_tls_dispatch_include(meta_in_tls) + + # Next: we need to make sure to return inputs directly, if the output is a mutable alias (e.g. add_()). + + schema_info_outs_write_aliases = schema_info.outs_write_aliases + # simple case: none of our outputs have mutable aliases, so we can return the output as-is + if schema_info_outs_write_aliases is None: + return out + + if len(schema_info_outs_write_aliases) == 1: + return get_arg_from_alias( + schema_info_outs_write_aliases[0], schema_info, args, kwargs + ) + + # In the multi-return case, all aten ops return a tuple / list, so cast accordingly. + outs_to_return = type(out)( + [ + (get_arg_from_alias(write_alias, schema_info, args, kwargs)) + for write_alias in schema_info_outs_write_aliases + ] + ) + return outs_to_return diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_pytree.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_pytree.py new file mode 100644 index 0000000000000000000000000000000000000000..57b0de2caa10d67e97b4051b0051b0472de1e9dd --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_pytree.py @@ -0,0 +1,2216 @@ +""" +Contains utility functions for working with nested python data structures. + +A *pytree* is Python nested data structure. It is a tree in the sense that +nodes are Python collections (e.g., list, tuple, dict) and the leaves are +Python values. Furthermore, a pytree should not contain reference cycles. + +pytrees are useful for working with nested collections of Tensors. For example, +one can use `tree_map` to map a function over all Tensors inside some nested +collection of Tensors and `tree_leaves` to get a flat list of all Tensors +inside some nested collection. pytrees are helpful for implementing nested +collection support for PyTorch APIs. + +This pytree implementation is not very performant due to Python overhead +To improve the performance we can move parts of the implementation to C++. +""" + +import dataclasses +import functools +import importlib +import importlib.metadata +import json +import sys +import threading +import types +import warnings +from collections import defaultdict, deque, namedtuple, OrderedDict +from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence +from enum import Enum +from typing import ( + Any, + cast, + ClassVar, + Final, + Generic, + NoReturn, + overload, + Protocol, + TYPE_CHECKING, + TypeAlias, + TypeVar, + Union, +) +from typing_extensions import deprecated, NamedTuple, Self, TypeIs + +from torch.torch_version import TorchVersion as _TorchVersion + + +if TYPE_CHECKING: + import torch.utils._cxx_pytree as cxx_pytree + + +__all__ = [ + "PyTree", + "Context", + "FlattenFunc", + "UnflattenFunc", + "DumpableContext", + "ToDumpableContextFn", + "FromDumpableContextFn", + "PyTreeSpec", + "TreeSpec", + "LeafSpec", + "keystr", + "key_get", + "register_pytree_node", + "tree_is_leaf", + "tree_flatten", + "tree_flatten_with_path", + "tree_unflatten", + "tree_iter", + "tree_leaves", + "tree_leaves_with_path", + "tree_structure", + "tree_map", + "tree_map_with_path", + "tree_map_", + "tree_map_only", + "tree_map_only_", + "tree_all", + "tree_any", + "tree_all_only", + "tree_any_only", + "treespec_dumps", + "treespec_loads", + "treespec_pprint", + "is_namedtuple", + "is_namedtuple_class", + "is_namedtuple_instance", + "is_structseq", + "is_structseq_class", + "is_structseq_instance", +] + + +T = TypeVar("T") +S = TypeVar("S") +U = TypeVar("U") +R = TypeVar("R") + + +DEFAULT_TREESPEC_SERIALIZATION_PROTOCOL = 1 +NO_SERIALIZED_TYPE_NAME_FOUND = "NO_SERIALIZED_TYPE_NAME_FOUND" + + +class KeyEntry(Protocol): + def __hash__(self) -> int: ... + + def __eq__(self, other: object) -> bool: ... + + def __str__(self) -> str: ... + + def get(self, parent: Any) -> Any: ... + + +class EnumEncoder(json.JSONEncoder): + def default(self, obj: object) -> str | dict[str, Any]: + if isinstance(obj, Enum): + return { + "__enum__": True, + "fqn": f"{obj.__class__.__module__}:{obj.__class__.__qualname__}", + "name": obj.name, + } + return cast(str, super().default(obj)) + + +Context = Any +PyTree = Any +FlattenFunc = Callable[[PyTree], tuple[list[Any], Context]] +UnflattenFunc = Callable[[Iterable[Any], Context], PyTree] +DumpableContext = Any # Any json dumpable text +ToDumpableContextFn = Callable[[Context], DumpableContext] +FromDumpableContextFn = Callable[[DumpableContext], Context] +ToStrFunc = Callable[["TreeSpec", list[str]], str] +MaybeFromStrFunc = Callable[[str], tuple[Any, Context, str] | None] +KeyPath = tuple[KeyEntry, ...] +FlattenWithKeysFunc = Callable[[PyTree], tuple[list[tuple[KeyEntry, Any]], Any]] + + +# A NodeDef holds two callables: +# - flatten_fn should take the collection and return a flat list of values. +# It can also return some context that is used in reconstructing the +# collection. +# - unflatten_fn should take a flat list of values and some context +# (returned by flatten_fn). It returns the collection by reconstructing +# it from the list and the context. +# - flatten_with_keys_fn, which is a callable that takes a +# pytree and returns a list of (keypath, value) pairs and a context. +class NodeDef(NamedTuple): + type: type[Any] + flatten_fn: FlattenFunc + unflatten_fn: UnflattenFunc + flatten_with_keys_fn: FlattenWithKeysFunc | None + + +_NODE_REGISTRY_LOCK = threading.RLock() +SUPPORTED_NODES: dict[type[Any], NodeDef] = {} + + +# _SerializeNodeDef holds the following: +# - typ: the type of the node (e.g., "Dict", "List", etc) +# - serialized_type_name: the fully qualified name of the type, e.g. "collections.OrderedDict" +# - to_dumpable_context takes a TreeSpec, and returns a serialized string format of the +# context, and the version number +# - from_dumpable_context takes in a string representation of the context, and the +# version, and returns the deserialized context +class _SerializeNodeDef(NamedTuple): + typ: type[Any] + serialized_type_name: str + to_dumpable_context: ToDumpableContextFn | None + from_dumpable_context: FromDumpableContextFn | None + + +SUPPORTED_SERIALIZED_TYPES: dict[type[Any], _SerializeNodeDef] = {} +SERIALIZED_TYPE_TO_PYTHON_TYPE: dict[str, type[Any]] = {} + +# NB: we try really hard to not import _cxx_pytree (which depends on optree) +# as much as possible. This is for isolation: a user who is not using C++ pytree +# shouldn't pay for it, and it helps makes things like cpython upgrades easier. +_optree_minimum_version = _TorchVersion("0.13.0") +try: + _optree_version = importlib.metadata.version("optree") +except importlib.metadata.PackageNotFoundError: + # No optree package found + _cxx_pytree_dynamo_traceable = _cxx_pytree_exists = False + _optree_version = _TorchVersion("0.0.0a0") +else: + _optree_version = _TorchVersion(_optree_version) + if _optree_version < _optree_minimum_version: + # optree package less than our required minimum version. + # Pretend the optree package doesn't exist. + # NB: We will raise ImportError if the user directly tries to + # `import torch.utils._cxx_pytree` (look in that file for the check). + _cxx_pytree_dynamo_traceable = _cxx_pytree_exists = False + else: + _cxx_pytree_dynamo_traceable = _cxx_pytree_exists = True + +_cxx_pytree_imported = False +_cxx_pytree_pending_imports: list[Any] = [] + + +def register_pytree_node( + cls: type[Any], + flatten_fn: FlattenFunc, + unflatten_fn: UnflattenFunc, + *, + serialized_type_name: str | None = None, + to_dumpable_context: ToDumpableContextFn | None = None, + from_dumpable_context: FromDumpableContextFn | None = None, + flatten_with_keys_fn: FlattenWithKeysFunc | None = None, +) -> None: + """Register a container-like type as pytree node. + + Note: + :func:`register_dataclass` is a simpler way of registering a container-like + type as a pytree node. + + Args: + cls: the type to register + flatten_fn: A callable that takes a pytree and returns a flattened + representation of the pytree and additional context to represent the + flattened pytree. + unflatten_fn: A callable that takes a flattened version of the pytree, + additional context, and returns an unflattened pytree. + serialized_type_name: A keyword argument used to specify the fully qualified + name used when serializing the tree spec. + to_dumpable_context: An optional keyword argument to custom specify how + to convert the context of the pytree to a custom json dumpable + representation. This is used for json serialization, which is being + used in torch.export right now. + from_dumpable_context: An optional keyword argument to custom specify how + to convert the custom json dumpable representation of the context + back to the original context. This is used for json deserialization, + which is being used in torch.export right now. + flatten_with_keys_fn: An optional keyword argument to specify how to + access each pytree leaf's keypath when flattening and tree-mapping. + Like ``flatten_fn``, but in place of a List[leaf], it should return + a List[(keypath, leaf)]. + """ + with _NODE_REGISTRY_LOCK: + if cls in SUPPORTED_NODES: + raise ValueError(f"{cls} is already registered as pytree node.") + + _private_register_pytree_node( + cls, + flatten_fn, + unflatten_fn, + serialized_type_name=serialized_type_name, + to_dumpable_context=to_dumpable_context, + from_dumpable_context=from_dumpable_context, + flatten_with_keys_fn=flatten_with_keys_fn, + ) + + if not _cxx_pytree_exists: + return + + if _cxx_pytree_imported: + import torch.utils._cxx_pytree as cxx_pytree + + cxx_pytree._private_register_pytree_node( + cls, + flatten_fn, + unflatten_fn, + serialized_type_name=serialized_type_name, + to_dumpable_context=to_dumpable_context, + from_dumpable_context=from_dumpable_context, + ) + else: + args = (cls, flatten_fn, unflatten_fn) + kwargs = { + "serialized_type_name": serialized_type_name, + "to_dumpable_context": to_dumpable_context, + "from_dumpable_context": from_dumpable_context, + } + _cxx_pytree_pending_imports.append((args, kwargs)) + + +def register_dataclass( + cls: type[Any], + *, + field_names: list[str] | None = None, + drop_field_names: list[str] | None = None, + serialized_type_name: str | None = None, +) -> None: + """ + Registers a type that has the semantics of a ``dataclasses.dataclass`` type + as a pytree node. + + This is a simpler API than :func:`register_pytree_node` for registering + a dataclass or a custom class with the semantics of a dataclass. + + Args: + cls: The python type to register. The class must have the semantics of a + dataclass; in particular, it must be constructed by passing the fields + in. + field_names (Optional[List[str]]): A list of field names that correspond + to the **non-constant data** in this class. This list must contain + all the fields that are used to initialize the class. This argument + is optional if ``cls`` is a dataclass, in which case the fields will + be taken from ``dataclasses.fields()``. + drop_field_names (Optional[List[str]]): A list of field names that + should not be included in the pytree. + serialized_type_name: A keyword argument used to specify the fully + qualified name used when serializing the tree spec. This is only + needed for serializing the treespec in torch.export. + + Example: + + >>> from torch import Tensor + >>> from dataclasses import dataclass + >>> import torch.utils._pytree as pytree + >>> + >>> @dataclass + >>> class Point: + >>> x: Tensor + >>> y: Tensor + >>> + >>> pytree.register_dataclass(Point) + >>> + >>> point = Point(torch.tensor(0), torch.tensor(1)) + >>> point = pytree.tree_map(lambda x: x + 1, point) + >>> assert torch.allclose(point.x, torch.tensor(1)) + >>> assert torch.allclose(point.y, torch.tensor(2)) + + """ + drop_field_names = drop_field_names or [] + + if not dataclasses.is_dataclass(cls): + if field_names is None: + raise ValueError( + "field_names must be specified with a list of all fields used to " + f"initialize {cls}, as it is not a dataclass." + ) + elif field_names is None: + field_names = [f.name for f in dataclasses.fields(cls) if f.init] + else: + dataclass_init_fields = {f.name for f in dataclasses.fields(cls) if f.init} + dataclass_init_fields.difference_update(drop_field_names) + + if dataclass_init_fields != set(field_names): + error_msg = "field_names does not include all dataclass fields.\n" + + if missing := dataclass_init_fields - set(field_names): + error_msg += ( + f"Missing fields in `field_names`: {missing}. If you want " + "to include these fields in the pytree, please add them " + "to `field_names`, otherwise please add them to " + "`drop_field_names`.\n" + ) + + if unexpected := set(field_names) - dataclass_init_fields: + error_msg += ( + f"Unexpected fields in `field_names`: {unexpected}. " + "Please remove these fields, or add them to `drop_field_names`.\n" + ) + + raise ValueError(error_msg) + + def _flatten_fn(obj: Any) -> tuple[list[Any], Context]: + flattened = [] + flat_names = [] + none_names = [] + for name in field_names: + val = getattr(obj, name) + if val is not None: + flattened.append(val) + flat_names.append(name) + else: + none_names.append(name) + return flattened, [flat_names, none_names] + + def _unflatten_fn(values: Iterable[Any], context: Context) -> Any: + flat_names, none_names = context + return cls( + **dict(zip(flat_names, values, strict=True)), **dict.fromkeys(none_names) + ) + + def _flatten_fn_with_keys(obj: Any) -> tuple[list[Any], Context]: + flattened, (flat_names, _none_names) = _flatten_fn(obj) # type: ignore[misc] + return [ + (GetAttrKey(k), v) for k, v in zip(flat_names, flattened, strict=True) + ], flat_names + + _private_register_pytree_node( + cls, + _flatten_fn, + _unflatten_fn, + serialized_type_name=serialized_type_name, + flatten_with_keys_fn=_flatten_fn_with_keys, + ) + + +CONSTANT_NODES: set[type] = set() + + +def register_constant(cls: type[Any]) -> None: + """Registers a type as a pytree node with no leaves. + + In a :func:`torch.compile` region, if instances of these types get passed to + :func:`torch._dynamo.nonstrict_trace`-ed function, they treated as a + constant (sometimes referred to as "static"): + + 1. if the instance object existed before the :func:`torch.compile` region, + we _assume_ no mutation will happen to it inside the :func:`torch.compile` + region, require that it has non-default `__eq__` and `__hash__` methods, and + we guard on the instance based on its `__eq__` method, i.e., if a new + instance fails to match any instances from the previous compilations, + :func:`torch.compile` will recompile the function using the new instance. + + 2. else if the instance object is created inside the :func:`torch.compile` + region, we currently don't support using it in a + :func:`torch._dynamo.nonstrict_trace`-ed function. + + In general, if your class holds Tensors or dynamic int/float/bool (values that + may change from run-to-run of a function being compiled), then you probably + do not want to register it as a constant. + + Otherwise if you want to pass instance of a class to a + :func:`torch._dynamo.nonstrict_trace`-ed function, but you either can't use + :func:`register_pytree_node` on the class, or the class is "constant" enough + that you don't want to bother using :func:`register_pytree_node`, you should + consider using this function. + + Args: + cls: the type to register as a constant. This type must be hashable. + + Example: + + >>> from dataclasses import dataclass + >>> import torch.utils._pytree as pytree + >>> + >>> @dataclass(frozen=True) + >>> class Config: + >>> norm: str + >>> + >>> pytree.register_constant(Config) + >>> + >>> config = Config("l2") + >>> values, spec = pytree.tree_flatten(config) + >>> assert len(values) == 0 + + """ + if cls.__eq__ is object.__eq__: # type: ignore[comparison-overlap] + raise TypeError( + "register_constant(cls) expects `cls` to have a non-default `__eq__` implementation." + ) + + # Class with a custom `__eq__` without `__hash__` won't inherit the default + # `__hash__` from object; see https://stackoverflow.com/a/1608907. + if cls.__hash__ is None: # type: ignore[comparison-overlap] + raise TypeError( + "register_constant(cls) expects `cls` to have a non-default `__hash__` implementation." + ) + + def _flatten(x): # type: ignore[no-untyped-def] + return [], ConstantNode(x) + + def _unflatten(_, context): # type: ignore[no-untyped-def] + return context.value + + def _flatten_with_keys(x): # type: ignore[no-untyped-def] + return [], ConstantNode(x) + + with _NODE_REGISTRY_LOCK: + _private_register_pytree_node( + cls, + _flatten, + _unflatten, + flatten_with_keys_fn=_flatten_with_keys, + ) + CONSTANT_NODES.add(cls) + + +def is_constant_class(cls: type[Any]) -> bool: + return isinstance(cls, type) and cls in CONSTANT_NODES + + +@dataclasses.dataclass(frozen=True) +class ConstantNode: + value: Any + + +def _is_constant_holder(spec: "TreeSpec") -> bool: + """Checks if the spec is from a pytree registered with register_constant""" + return isinstance(spec._context, ConstantNode) + + +def _retrieve_constant(spec: "TreeSpec") -> Any: + """Given a spec from a pytree registered with register_constant, retrieves the constant""" + if not _is_constant_holder(spec): + raise AssertionError("spec does not correspond to a registered constant pytree") + return tree_unflatten([], spec) + + +def _register_namedtuple( + cls: type[Any], + *, + serialized_type_name: str, +) -> None: + """ + Registers a namedtuple as a valid pytree node. By default namedtuples are + valid pytree nodes, but they are not serializable. This API provides the + argument `serialized_type_name` which allows these namedtuples to be + serialized. + + Args: + cls: the dataclass type to register + serialized_type_name: The serialized name for the dataclass. This is + required if you want to serialize the pytree TreeSpec containing this + namedtuple. + """ + _private_register_pytree_node( + cls, + _namedtuple_flatten, + _namedtuple_unflatten, + serialized_type_name=serialized_type_name, + to_dumpable_context=_namedtuple_serialize, + from_dumpable_context=_namedtuple_deserialize, + flatten_with_keys_fn=_namedtuple_flatten_with_keys, + ) + + +@deprecated( + "`torch.utils._pytree._register_pytree_node` is deprecated. " + "Please use `torch.utils._pytree.register_pytree_node` instead.", + category=FutureWarning, +) +def _register_pytree_node( + cls: type[Any], + flatten_fn: FlattenFunc, + unflatten_fn: UnflattenFunc, + to_str_fn: ToStrFunc | None = None, # deprecated + maybe_from_str_fn: MaybeFromStrFunc | None = None, # deprecated + *, + serialized_type_name: str | None = None, + to_dumpable_context: ToDumpableContextFn | None = None, + from_dumpable_context: FromDumpableContextFn | None = None, + flatten_with_keys_fn: FlattenWithKeysFunc | None = None, +) -> None: + """Register a container-like type as pytree node for the Python pytree only. + + Args: + cls: the type to register + flatten_fn: A callable that takes a pytree and returns a flattened + representation of the pytree and additional context to represent the + flattened pytree. + unflatten_fn: A callable that takes a flattened version of the pytree, + additional context, and returns an unflattened pytree. + serialized_type_name: A keyword argument used to specify the fully qualified + name used when serializing the tree spec. + to_dumpable_context: An optional keyword argument to custom specify how + to convert the context of the pytree to a custom json dumpable + representation. This is used for json serialization, which is being + used in torch.export right now. + from_dumpable_context: An optional keyword argument to custom specify how + to convert the custom json dumpable representation of the context + back to the original context. This is used for json deserialization, + which is being used in torch.export right now. + flatten_with_keys_fn: An optional keyword argument to specify how to + access each pytree leaf's keypath when flattening and tree-mapping. + Like ``flatten_fn``, but in place of a List[leaf], it should return + a List[(keypath, leaf)]. + """ + if to_str_fn is not None or maybe_from_str_fn is not None: + warnings.warn( + "`to_str_fn` and `maybe_from_str_fn` is deprecated. " + "Please use `to_dumpable_context` and `from_dumpable_context` instead.", + FutureWarning, + stacklevel=2, + ) + + _private_register_pytree_node( + cls, + flatten_fn, + unflatten_fn, + serialized_type_name=serialized_type_name, + to_dumpable_context=to_dumpable_context, + from_dumpable_context=from_dumpable_context, + flatten_with_keys_fn=flatten_with_keys_fn, + ) + + +def _deregister_pytree_node( + cls: type[Any], +) -> None: + """This is an internal function that is used to deregister a pytree node type + for the Python pytree only. This should be only used inside PyTorch. + """ + with _NODE_REGISTRY_LOCK: + del SUPPORTED_NODES[cls] + node_def = SUPPORTED_SERIALIZED_TYPES[cls] + del SERIALIZED_TYPE_TO_PYTHON_TYPE[node_def.serialized_type_name] + del SUPPORTED_SERIALIZED_TYPES[cls] + CONSTANT_NODES.discard(cls) + + +def _private_register_pytree_node( + cls: type[Any], + flatten_fn: FlattenFunc, + unflatten_fn: UnflattenFunc, + *, + serialized_type_name: str | None = None, + to_dumpable_context: ToDumpableContextFn | None = None, + from_dumpable_context: FromDumpableContextFn | None = None, + flatten_with_keys_fn: FlattenWithKeysFunc | None = None, +) -> None: + """This is an internal function that is used to register a pytree node type + for the Python pytree only. End-users should use :func:`register_pytree_node` + instead. + """ + from torch._library.opaque_object import is_opaque_type + + if is_opaque_type(cls): + raise ValueError( + f"{cls} cannot be registered as a pytree as it has been " + "registered as an opaque object. Opaque objects must be pytree leaves." + ) + + with _NODE_REGISTRY_LOCK: + if cls in SUPPORTED_NODES: + # TODO: change this warning to an error after OSS/internal stabilize + warnings.warn( + f"{cls} is already registered as pytree node. " + "Overwriting the previous registration.", + stacklevel=2, + ) + + node_def = NodeDef(cls, flatten_fn, unflatten_fn, flatten_with_keys_fn) + SUPPORTED_NODES[cls] = node_def + + if (to_dumpable_context is None) ^ (from_dumpable_context is None): + raise ValueError( + f"Both to_dumpable_context and from_dumpable_context for {cls} must " + "be None or registered." + ) + + if serialized_type_name is None: + serialized_type_name = NO_SERIALIZED_TYPE_NAME_FOUND + + serialize_node_def = _SerializeNodeDef( + cls, + serialized_type_name, + to_dumpable_context, + from_dumpable_context, + ) + SUPPORTED_SERIALIZED_TYPES[cls] = serialize_node_def + SERIALIZED_TYPE_TO_PYTHON_TYPE[serialized_type_name] = cls + + +@dataclasses.dataclass(frozen=True) +class SequenceKey(Generic[T]): + idx: int + + def __str__(self) -> str: + return f"[{self.idx!r}]" + + def get(self, sequence: Sequence[T]) -> T: + return sequence[self.idx] + + +K = TypeVar("K", bound=Hashable) + + +@dataclasses.dataclass(frozen=True) +class MappingKey(Generic[K, T]): + key: K + + def __str__(self) -> str: + return f"[{self.key!r}]" + + def get(self, mapping: Mapping[K, T]) -> T: + return mapping[self.key] + + +@dataclasses.dataclass(frozen=True) +class GetAttrKey: + name: str + + def __str__(self) -> str: + return f".{self.name}" + + def get(self, obj: Any) -> Any: + return getattr(obj, self.name) + + +# Reference: https://github.com/metaopt/optree/blob/main/optree/typing.py +def is_namedtuple(obj: object | type) -> bool: + """Return whether the object is an instance of namedtuple or a subclass of namedtuple.""" + cls = obj if isinstance(obj, type) else type(obj) + return is_namedtuple_class(cls) + + +# Reference: https://github.com/metaopt/optree/blob/main/optree/typing.py +def is_namedtuple_class(cls: type) -> bool: + """Return whether the class is a subclass of namedtuple.""" + return ( + isinstance(cls, type) + and issubclass(cls, tuple) + and isinstance(getattr(cls, "_fields", None), tuple) + and all(type(field) is str for field in cls._fields) # type: ignore[attr-defined] + and callable(getattr(cls, "_make", None)) + and callable(getattr(cls, "_asdict", None)) + ) + + +# Reference: https://github.com/metaopt/optree/blob/main/optree/typing.py +def is_namedtuple_instance(obj: object) -> bool: + """Return whether the object is an instance of namedtuple.""" + return is_namedtuple_class(type(obj)) + + +_T_co = TypeVar("_T_co", covariant=True) + + +# Reference: https://github.com/metaopt/optree/blob/main/optree/typing.py +class structseq(tuple[_T_co, ...]): + """A generic type stub for CPython's ``PyStructSequence`` type.""" + + __slots__: ClassVar[tuple[()]] = () + + n_fields: Final[int] # type: ignore[misc] + n_sequence_fields: Final[int] # type: ignore[misc] + n_unnamed_fields: Final[int] # type: ignore[misc] + + def __init_subclass__(cls) -> NoReturn: + """Prohibit subclassing.""" + raise TypeError("type 'structseq' is not an acceptable base type") + + def __new__( + cls: type[Self], + sequence: Iterable[_T_co], + # pyrefly: ignore [bad-function-definition] + dict: dict[str, Any] = ..., + ) -> Self: + raise NotImplementedError + + +# Reference: https://github.com/metaopt/optree/blob/main/optree/typing.py +def is_structseq(obj: object | type) -> bool: + """Return whether the object is an instance of PyStructSequence or a class of PyStructSequence.""" + cls = obj if isinstance(obj, type) else type(obj) + return is_structseq_class(cls) + + +# Set if the type allows subclassing (see CPython's Include/object.h) +Py_TPFLAGS_BASETYPE: int = 1 << 10 + + +# Reference: https://github.com/metaopt/optree/blob/main/optree/typing.py +def is_structseq_class(cls: type) -> bool: + """Return whether the class is a class of PyStructSequence.""" + return ( + isinstance(cls, type) + # Check direct inheritance from `tuple` rather than `issubclass(cls, tuple)` + and cls.__bases__ == (tuple,) + # Check PyStructSequence members + and isinstance(getattr(cls, "n_fields", None), int) + and isinstance(getattr(cls, "n_sequence_fields", None), int) + and isinstance(getattr(cls, "n_unnamed_fields", None), int) + # Check the type does not allow subclassing + and not bool(cls.__flags__ & Py_TPFLAGS_BASETYPE) # only works for CPython + ) + + +# Reference: https://github.com/metaopt/optree/blob/main/optree/typing.py +def is_structseq_instance(obj: object) -> bool: + """Return whether the object is an instance of PyStructSequence.""" + return is_structseq_class(type(obj)) + + +def _tuple_flatten(d: tuple[T, ...]) -> tuple[list[T], Context]: + return list(d), None + + +def _tuple_flatten_with_keys( + d: tuple[T, ...], +) -> tuple[list[tuple[KeyEntry, T]], Context]: + values, context = _tuple_flatten(d) + # pyrefly: ignore [bad-return] + return [(SequenceKey(i), v) for i, v in enumerate(values)], context + + +def _tuple_unflatten(values: Iterable[T], context: Context) -> tuple[T, ...]: + return tuple(values) + + +def _list_flatten(d: list[T]) -> tuple[list[T], Context]: + return d, None + + +def _list_flatten_with_keys(d: list[T]) -> tuple[list[tuple[KeyEntry, T]], Context]: + values, context = _list_flatten(d) + # pyrefly: ignore [bad-return] + return [(SequenceKey(i), v) for i, v in enumerate(values)], context + + +def _list_unflatten(values: Iterable[T], context: Context) -> list[T]: + return list(values) + + +def _dict_flatten(d: dict[Any, T]) -> tuple[list[T], Context]: + return list(d.values()), list(d.keys()) + + +def _dict_flatten_with_keys( + d: dict[Any, T], +) -> tuple[list[tuple[KeyEntry, T]], Context]: + values, context = _dict_flatten(d) + # pyrefly: ignore [bad-return] + return [(MappingKey(k), v) for k, v in zip(context, values, strict=True)], context + + +def _dict_unflatten(values: Iterable[T], context: Context) -> dict[Any, T]: + return dict(zip(context, values, strict=True)) + + +def _namedtuple_flatten(d: NamedTuple) -> tuple[list[Any], Context]: + return list(d), type(d) + + +def _namedtuple_flatten_with_keys( + d: NamedTuple, +) -> tuple[list[tuple[KeyEntry, Any]], Context]: + values, context = _namedtuple_flatten(d) + # pyrefly: ignore [bad-return] + return ( + [ + (GetAttrKey(field), v) + for field, v in zip(context._fields, values, strict=True) + ], + context, + ) + + +def _namedtuple_unflatten(values: Iterable[T], context: Context) -> NamedTuple: + return cast(NamedTuple, context(*values)) + + +def _namedtuple_serialize(context: Context) -> DumpableContext: + if context not in SUPPORTED_SERIALIZED_TYPES: + raise NotImplementedError( + f"Can't serialize TreeSpec of namedtuple class {context} because we " + "didn't register a serializated_type_name. Please register using " + "`_register_namedtuple`." + ) + + serialize_node_def = SUPPORTED_SERIALIZED_TYPES[context] + serialized_type_name = serialize_node_def.serialized_type_name + + if serialized_type_name == NO_SERIALIZED_TYPE_NAME_FOUND: + raise NotImplementedError( + f"Can't serialize TreeSpec of namedtuple class {context} because we " + "couldn't find a serializated_type_name. Please register using " + "`_register_namedtuple`." + ) + return serialized_type_name + + +def _namedtuple_deserialize(dumpable_context: DumpableContext) -> Context: + if dumpable_context not in SERIALIZED_TYPE_TO_PYTHON_TYPE: + raise NotImplementedError( + f"Can't deserialize TreeSpec of namedtuple class {dumpable_context} " + "because we couldn't find a serializated name." + ) + + typ = SERIALIZED_TYPE_TO_PYTHON_TYPE[dumpable_context] + return typ + + +def _ordereddict_flatten(d: OrderedDict[Any, T]) -> tuple[list[T], Context]: + return list(d.values()), list(d.keys()) + + +def _ordereddict_flatten_with_keys( + d: OrderedDict[Any, T], +) -> tuple[list[tuple[KeyEntry, T]], Context]: + values, context = _ordereddict_flatten(d) + # pyrefly: ignore [bad-return] + return [(MappingKey(k), v) for k, v in zip(context, values, strict=True)], context + + +def _ordereddict_unflatten( + values: Iterable[T], + context: Context, +) -> OrderedDict[Any, T]: + return OrderedDict((key, value) for key, value in zip(context, values, strict=True)) + + +_odict_flatten = _ordereddict_flatten +_odict_unflatten = _ordereddict_unflatten + + +def _defaultdict_flatten(d: defaultdict[Any, T]) -> tuple[list[T], Context]: + values, dict_context = _dict_flatten(d) + return values, [d.default_factory, dict_context] + + +def _defaultdict_flatten_with_keys( + d: defaultdict[Any, T], +) -> tuple[list[tuple[KeyEntry, T]], Context]: + values, context = _defaultdict_flatten(d) + _, dict_context = context + # pyrefly: ignore [bad-return] + return [ + (MappingKey(k), v) for k, v in zip(dict_context, values, strict=True) + ], context + + +def _defaultdict_unflatten( + values: Iterable[T], + context: Context, +) -> defaultdict[Any, T]: + default_factory, dict_context = context + return defaultdict(default_factory, _dict_unflatten(values, dict_context)) + + +def _defaultdict_serialize(context: Context) -> DumpableContext: + default_factory, dict_context = context + json_defaultdict = { + "default_factory_module": default_factory.__module__, + "default_factory_name": default_factory.__qualname__, + "dict_context": dict_context, + } + return json_defaultdict + + +def _defaultdict_deserialize(dumpable_context: DumpableContext) -> Context: + if not isinstance(dumpable_context, dict): + raise AssertionError("dumpable_context must be a dict") + + expected_keys = { + "default_factory_module", + "default_factory_name", + "dict_context", + } + if set(dumpable_context) != expected_keys: + raise AssertionError( + f"dumpable_context keys must be {expected_keys}, got {set(dumpable_context)}" + ) + + default_factory_module = dumpable_context["default_factory_module"] + default_factory_name = dumpable_context["default_factory_name"] + if not isinstance(default_factory_module, str): + raise AssertionError("default_factory_module must be a string") + if not isinstance(default_factory_name, str): + raise AssertionError("default_factory_name must be a string") + module = importlib.import_module(default_factory_module) + default_factory = getattr(module, default_factory_name) + + dict_context = dumpable_context["dict_context"] + return [default_factory, dict_context] + + +def _deque_flatten(d: deque[T]) -> tuple[list[T], Context]: + return list(d), d.maxlen + + +def _deque_flatten_with_keys( + d: deque[T], +) -> tuple[list[tuple[KeyEntry, T]], Context]: + values, context = _deque_flatten(d) + # pyrefly: ignore [bad-return] + return [(SequenceKey(i), v) for i, v in enumerate(values)], context + + +def _deque_unflatten(values: Iterable[T], context: Context) -> deque[T]: + return deque(values, maxlen=context) + + +_private_register_pytree_node( + tuple, + _tuple_flatten, + _tuple_unflatten, + serialized_type_name="builtins.tuple", + flatten_with_keys_fn=_tuple_flatten_with_keys, +) +_private_register_pytree_node( + list, + _list_flatten, + _list_unflatten, + serialized_type_name="builtins.list", + flatten_with_keys_fn=_list_flatten_with_keys, +) +_private_register_pytree_node( + dict, + _dict_flatten, + _dict_unflatten, + serialized_type_name="builtins.dict", + flatten_with_keys_fn=_dict_flatten_with_keys, +) +_private_register_pytree_node( + namedtuple, # type: ignore[arg-type] + _namedtuple_flatten, + _namedtuple_unflatten, + serialized_type_name="collections.namedtuple", + to_dumpable_context=_namedtuple_serialize, + from_dumpable_context=_namedtuple_deserialize, + flatten_with_keys_fn=_namedtuple_flatten_with_keys, +) +_private_register_pytree_node( + OrderedDict, + _ordereddict_flatten, + _ordereddict_unflatten, + serialized_type_name="collections.OrderedDict", + flatten_with_keys_fn=_ordereddict_flatten_with_keys, +) +_private_register_pytree_node( + defaultdict, + _defaultdict_flatten, + _defaultdict_unflatten, + serialized_type_name="collections.defaultdict", + to_dumpable_context=_defaultdict_serialize, + from_dumpable_context=_defaultdict_deserialize, + flatten_with_keys_fn=_defaultdict_flatten_with_keys, +) +_private_register_pytree_node( + deque, + _deque_flatten, + _deque_unflatten, + serialized_type_name="collections.deque", + flatten_with_keys_fn=_deque_flatten_with_keys, +) + + +STANDARD_DICT_TYPES: frozenset[type] = frozenset({dict, OrderedDict, defaultdict}) +BUILTIN_TYPES: frozenset[type] = frozenset( + { + tuple, + list, + dict, + namedtuple, # type: ignore[arg-type] + OrderedDict, + defaultdict, + deque, + }, +) + + +@deprecated( + "torch.utils._pytree._is_namedtuple_instance is private and will be removed in a future release. " + "Please use torch.utils._pytree.is_namedtuple_instance instead.", + category=FutureWarning, +) +def _is_namedtuple_instance(tree: Any) -> bool: + return is_namedtuple_instance(tree) + + +def _get_node_type(tree: Any) -> Any: + node_type = type(tree) + # All namedtuple types are implicitly registered as pytree nodes. + # XXX: Other parts of the codebase expect namedtuple types always return + # `namedtuple` instead of the actual namedtuple type. Even if the type + # is explicitly registered. + if is_namedtuple_class(node_type): + return namedtuple + return node_type + + +# A leaf is defined as anything that is not a Node. +def tree_is_leaf( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: + """Check if a pytree is a leaf. + + >>> tree_is_leaf(1) + True + >>> tree_is_leaf(None) + True + >>> tree_is_leaf([1, 2, 3]) + False + >>> tree_is_leaf((1, 2, 3), is_leaf=lambda x: isinstance(x, tuple)) + True + >>> tree_is_leaf({"a": 1, "b": 2, "c": 3}) + False + >>> tree_is_leaf({"a": 1, "b": 2, "c": None}) + False + """ + if is_leaf is not None and is_leaf(tree): + return True + return _get_node_type(tree) not in SUPPORTED_NODES + + +@deprecated( + "torch.utils._pytree._is_leaf is private and will be removed in a future release. " + "Please use torch.utils._pytree.tree_is_leaf instead.", + category=FutureWarning, +) +def _is_leaf(tree: PyTree, is_leaf: Callable[[PyTree], bool] | None = None) -> bool: + return tree_is_leaf(tree, is_leaf=is_leaf) + + +# A TreeSpec represents the structure of a pytree. It holds: +# "type": the type of root Node of the pytree +# context: some context that is useful in unflattening the pytree +# children(): specs for each child of the root Node +# num_nodes: the total number of nodes +# num_leaves: the number of leaves +# num_children: the number of children of the root Node (i.e., len(children())) +# is_leaf(): whether the root Node is a leaf +@dataclasses.dataclass(init=False, frozen=True, eq=True, repr=False) +class TreeSpec: + type: Any + _context: Context + _children: list[Self] + + num_nodes: int = dataclasses.field(init=False) + num_leaves: int = dataclasses.field(init=False) + num_children: int = dataclasses.field(init=False) + + def __init__( + self, + type: Any, + context: Context, # keep for backward compatibility + children_specs: list[Self], # keep for backward compatibility + ) -> None: + object.__setattr__(self, "type", type) + object.__setattr__(self, "_context", context) + object.__setattr__(self, "_children", children_specs) + self.__post_init__() + + def __post_init__(self) -> None: + if self.type is None: + assert self._context is None + assert len(self._children) == 0 + num_nodes = 1 + num_leaves = 1 + num_children = 0 + else: + num_nodes = 1 + num_leaves = 0 + for child in self._children: + num_nodes += child.num_nodes + num_leaves += child.num_leaves + num_children = len(self._children) + object.__setattr__(self, "num_nodes", num_nodes) + object.__setattr__(self, "num_leaves", num_leaves) + object.__setattr__(self, "num_children", num_children) + + def __repr__(self, indent: int = 0) -> str: + repr_prefix: str = f"TreeSpec({self.type.__name__}, {self._context}, [" + children_specs_str: str = "" + if self.num_children > 0: + indent += 2 + children_specs_str += self._children[0].__repr__(indent) + children_specs_str += "," if self.num_children > 1 else "" + children_specs_str += ",".join( + [ + "\n" + " " * indent + child.__repr__(indent) + for child in self._children[1:] + ] + ) + repr_suffix: str = f"{children_specs_str}])" + return repr_prefix + repr_suffix + + def __eq__(self, other: PyTree) -> bool: + if self is other: + return True + elif other.__class__ is self.__class__: + if str(self.type) != str(other.type): + return False + if self._context != other._context: + return False + elif self._children != other._children: + return False + return True + return NotImplemented + + @property + def context(self) -> Context: + return self._context + + @property + @deprecated( + "`treespec.children_specs` is deprecated. " + "Use `treespec.child(index)` to access a single child, " + "or `treespec.children()` to get all children.", + category=FutureWarning, + ) + def children_specs(self) -> list[Self]: + return self._children + + def is_leaf(self) -> bool: + return self.num_nodes == 1 and self.num_leaves == 1 + + def children(self) -> list[Self]: + return self._children.copy() + + def child(self, index: int) -> Self: + return self._children[index] + + def flatten_up_to(self, tree: PyTree) -> list[PyTree]: + def helper(treespec: TreeSpec, node: PyTree, subtrees: list[PyTree]) -> None: + if treespec.is_leaf(): + subtrees.append(node) + return + + node_type = _get_node_type(node) + if treespec.type not in BUILTIN_TYPES: + # Always require custom node types to match exactly + if node_type != treespec.type: + raise ValueError( + f"Type mismatch; " + f"expected {treespec.type!r}, but got {node_type!r}.", + ) + flatten_fn = SUPPORTED_NODES[node_type].flatten_fn + children, context = flatten_fn(node) + if len(children) != treespec.num_children: + raise ValueError( + f"Node arity mismatch; " + f"expected {treespec.num_children}, but got {len(children)}.", + ) + if context != treespec._context: + raise ValueError( + f"Node context mismatch for custom node type {treespec.type!r}.", + ) + else: + # For builtin dictionary types, we allow some flexibility + # Otherwise, we require exact matches + both_standard_dict = ( + treespec.type in STANDARD_DICT_TYPES + and node_type in STANDARD_DICT_TYPES + ) + if not both_standard_dict and node_type != treespec.type: + raise ValueError( + f"Node type mismatch; " + f"expected {treespec.type!r}, but got {node_type!r}.", + ) + if len(node) != treespec.num_children: + raise ValueError( + f"Node arity mismatch; " + f"expected {treespec.num_children}, but got {len(node)}.", + ) + + if both_standard_dict: + # dictionary types are compatible with each other + dict_context = ( + treespec._context + if treespec.type is not defaultdict + # ignore mismatch of `default_factory` for defaultdict + else treespec._context[1] + ) + expected_keys = dict_context + got_key_set = set(node) + expected_key_set = set(expected_keys) + if got_key_set != expected_key_set: + missing_keys = expected_key_set.difference(got_key_set) + extra_keys = got_key_set.difference(expected_key_set) + message = "" + if missing_keys: + message += f"; missing key(s): {missing_keys}" + if extra_keys: + message += f"; extra key(s): {extra_keys}" + raise ValueError(f"Node keys mismatch{message}.") + children = [node[key] for key in expected_keys] + else: + # node_type is treespec.type + flatten_fn = SUPPORTED_NODES[node_type].flatten_fn + children, context = flatten_fn(node) + if ( + node_type is not deque # ignore mismatch of `maxlen` for deque + ) and context != treespec._context: + raise ValueError( + f"Node context mismatch for node type {treespec.type!r}; " + f"expected {treespec._context!r}, but got {context!r}.", # namedtuple type mismatch + ) + + for subtree, subspec in zip(children, treespec._children, strict=True): + helper(subspec, subtree, subtrees) + + subtrees: list[PyTree] = [] + helper(self, tree, subtrees) + return subtrees + + def unflatten(self, leaves: Iterable[Any]) -> PyTree: + if not isinstance(leaves, (list, tuple)): + leaves = list(leaves) + if len(leaves) != self.num_leaves: + raise ValueError( + f"treespec.unflatten(leaves): `leaves` has length {len(leaves)} " + f"but the spec refers to a pytree that holds {self.num_leaves} " + f"items ({self}).", + ) + if self.is_leaf(): + return leaves[0] + + unflatten_fn = SUPPORTED_NODES[self.type].unflatten_fn + + # Recursively unflatten the children + start = 0 + end = 0 + child_pytrees = [] + for child_spec in self._children: + end += child_spec.num_leaves + child_pytrees.append(child_spec.unflatten(leaves[start:end])) + start = end + + return unflatten_fn(child_pytrees, self._context) + + def __hash__(self) -> int: + node_type = self.type + if node_type is defaultdict: + default_factory, dict_context = self._context + hashable_context = (default_factory, tuple(dict_context)) + elif node_type in (dict, OrderedDict): + hashable_context = tuple(self._context) + elif node_type is None or node_type in BUILTIN_TYPES: + hashable_context = self._context + elif isinstance(self._context, ConstantNode): + hashable_context = self._context.value + else: + # The context for user-defined node types might not be hashable. + # Ignore it for hashing. + # This does not break the correctness that equal objects imply the + # same hash. This might increase the hash collision rate, but we + # don't care about that. + hashable_context = None + return hash((node_type, hashable_context, tuple(self._children))) + + +PyTreeSpec: TypeAlias = TreeSpec + + +# NOTE: subclassing a dataclass is subtle. In order to enable reasoning about +# this class with `dataclasses.fields`, etc., while having a simplified +# constructor that takes no argument, we wrap with `dataclass(init=True, ...)` +# again, with fields that have `init=False`. +@deprecated( + "`isinstance(treespec, LeafSpec)` is deprecated, " + "use `isinstance(treespec, TreeSpec) and treespec.is_leaf()` instead.", + category=FutureWarning, +) +@dataclasses.dataclass(init=True, frozen=True, eq=False, repr=False) +class LeafSpec(TreeSpec): + type: Any = dataclasses.field(default=None, init=False) + _context: Context = dataclasses.field(default=None, init=False) + _children: list[Self] = dataclasses.field(default_factory=list, init=False) + + def __post_init__(self) -> None: + # Override `__post_init__` for `num_leaves` derivation. + object.__setattr__(self, "num_nodes", 1) + object.__setattr__(self, "num_leaves", 1) + object.__setattr__(self, "num_children", 0) + + def __repr__(self, indent: int = 0) -> str: + return "*" + + +# All leaves are equivalent, so represent with a single object to save on +# object construction time +with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", category=FutureWarning, module=__name__, append=False + ) + _LEAF_SPEC = LeafSpec() + + +def treespec_leaf() -> LeafSpec: + """Make a treespec representing a leaf node.""" + return _LEAF_SPEC + + +def treespec_tuple(iterable: Iterable[TreeSpec] = (), /) -> TreeSpec: + """Make a tuple treespec from an iterable of child treespecs.""" + children = list(iterable) + if any(not isinstance(child, TreeSpec) for child in children): + raise ValueError(f"Expected a tuple of TreeSpec values, got: {children!r}.") + return TreeSpec(tuple, None, children) + + +def treespec_dict( + mapping: Mapping[Any, TreeSpec] | Iterable[tuple[Any, TreeSpec]] = (), + /, + **kwargs: TreeSpec, +) -> TreeSpec: + """Make a dict treespec from a dict of child treespecs.""" + dct = dict(mapping, **kwargs) + if any(not isinstance(child, TreeSpec) for child in dct.values()): + raise ValueError(f"Expected a dictionary of TreeSpec values, got: {dct!r}.") + return TreeSpec(dict, list(dct.keys()), list(dct.values())) + + +def _is_pytreespec_instance( + obj: Any, +) -> TypeIs[Union[TreeSpec, "cxx_pytree.PyTreeSpec"]]: + if isinstance(obj, TreeSpec): + return True + if "torch.utils._cxx_pytree" in sys.modules: + # The C++ pytree module is not always available, so we check if it is loaded. + # If the C++ pytree module is loaded, we can check if the treespec + # is an instance of the C++ TreeSpec class. + import torch.utils._cxx_pytree as cxx_pytree + + if isinstance(obj, cxx_pytree.PyTreeSpec): + return True + if "torch._dynamo.polyfills.pytree" in sys.modules: + # The PyTorch Dynamo pytree module is not always available, so we check if it is loaded. + # If the PyTorch Dynamo pytree module is loaded, we can check if the treespec + # is an instance of the PyTorch Dynamo TreeSpec class. + import torch._dynamo.polyfills.pytree as dynamo_pytree + + return isinstance(obj, dynamo_pytree.PyTreeSpec) + return False + + +def _ensure_python_treespec_instance( + treespec: Union[TreeSpec, "cxx_pytree.PyTreeSpec"], +) -> TreeSpec: + if isinstance(treespec, TreeSpec): + return treespec + + if not _is_pytreespec_instance(treespec): + raise TypeError( + f"Expected `treespec` to be an instance of " + f"PyTreeSpec but got item of type {type(treespec)}." + ) + dummy_tree = treespec.unflatten([0] * treespec.num_leaves) + return tree_structure(dummy_tree) + + +def tree_flatten( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> tuple[list[Any], TreeSpec]: + """Flattens a pytree into a list of values and a TreeSpec that can be used + to reconstruct the pytree. + """ + + def helper(node: PyTree, leaves: list[Any]) -> TreeSpec: + if tree_is_leaf(node, is_leaf=is_leaf): + leaves.append(node) + return _LEAF_SPEC + + node_type = _get_node_type(node) + flatten_fn = SUPPORTED_NODES[node_type].flatten_fn + children, context = flatten_fn(node) + + # Recursively flatten the children + subspecs = [helper(child, leaves) for child in children] + return TreeSpec(node_type, context, subspecs) + + leaves: list[Any] = [] + treespec = helper(tree, leaves) + return leaves, treespec + + +def tree_unflatten(leaves: Iterable[Any], treespec: TreeSpec) -> PyTree: + """Given a list of values and a TreeSpec, builds a pytree. + This is the inverse operation of `tree_flatten`. + """ + if not _is_pytreespec_instance(treespec): + if not _is_pytreespec_instance(leaves): + raise TypeError( + f"Expected `treespec` to be an instance of " + f"PyTreeSpec but got item of type {type(treespec)}." + ) + # Allow passing the PyTreeSpec instance as the first argument + leaves, treespec = treespec, leaves + return treespec.unflatten(leaves) + + +def tree_iter( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> Iterable[Any]: + """Get an iterator over the leaves of a pytree.""" + if tree_is_leaf(tree, is_leaf=is_leaf): + yield tree + else: + node_type = _get_node_type(tree) + flatten_fn = SUPPORTED_NODES[node_type].flatten_fn + child_pytrees, _ = flatten_fn(tree) + + # Recursively flatten the children + for child in child_pytrees: + yield from tree_iter(child, is_leaf=is_leaf) + + +def tree_leaves( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> list[Any]: + """Get a list of leaves of a pytree.""" + return list(tree_iter(tree, is_leaf=is_leaf)) + + +def tree_structure( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> TreeSpec: + """Get the TreeSpec for a pytree.""" + return tree_flatten(tree, is_leaf=is_leaf)[1] + + +def tree_map( + func: Callable[..., Any], + tree: PyTree, + *rests: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: + """Map a multi-input function over pytree args to produce a new pytree. + + See also :func:`tree_map_`. + + >>> tree_map(lambda x: x + 1, {"x": 7, "y": (42, 64)}) + {'x': 8, 'y': (43, 65)} + >>> tree_map(lambda x: x is None, {"x": 7, "y": (42, 64), "z": None}) + {'x': False, 'y': (False, False), 'z': True} + + If multiple inputs are given, the structure of the tree is taken from the first input; + subsequent inputs need only have ``tree`` as a prefix: + + >>> tree_map(lambda x, y: [x] + y, [5, 6], [[7, 9], [1, 2]]) + [[5, 7, 9], [6, 1, 2]] + + Args: + func (callable): A function that takes ``1 + len(rests)`` arguments, to be applied at the + corresponding leaves of the pytrees. + tree (pytree): A pytree to be mapped over, with each leaf providing the first positional + argument to function ``func``. + rests (tuple of pytree): A tuple of pytrees, each of which has the same structure as + ``tree`` or has ``tree`` as a prefix. + is_leaf (callable, optional): An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + + Returns: + A new pytree with the same structure as ``tree`` but with the value at each leaf given by + ``func(x, *xs)`` where ``x`` is the value at the corresponding leaf in ``tree`` and ``xs`` + is the tuple of values at corresponding nodes in ``rests``. + """ + leaves, treespec = tree_flatten(tree, is_leaf=is_leaf) + flat_args = [leaves] + [treespec.flatten_up_to(r) for r in rests] + return treespec.unflatten(map(func, *flat_args)) + + +def tree_map_( + func: Callable[..., Any], + tree: PyTree, + *rests: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: + """Like :func:`tree_map`, but do an inplace call on each leaf and return the original tree. + + See also :func:`tree_map`. + + Args: + func (callable): A function that takes ``1 + len(rests)`` arguments, to be applied at the + corresponding leaves of the pytrees. + tree (pytree): A pytree to be mapped over, with each leaf providing the first positional + argument to function ``func``. + rests (tuple of pytree): A tuple of pytrees, each of which has the same structure as + ``tree`` or has ``tree`` as a prefix. + is_leaf (callable, optional): An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + + Returns: + The original ``tree`` with the value at each leaf is given by the side-effect of function + ``func(x, *xs)`` (not the return value) where ``x`` is the value at the corresponding leaf + in ``tree`` and ``xs`` is the tuple of values at values at corresponding nodes in ``rests``. + """ + leaves, treespec = tree_flatten(tree, is_leaf=is_leaf) + flat_args = [leaves] + [treespec.flatten_up_to(r) for r in rests] + deque(map(func, *flat_args), maxlen=0) # consume and exhaust the iterable + return tree + + +Type2 = tuple[type[T], type[S]] +Type3 = tuple[type[T], type[S], type[U]] +TypeAny = type[Any] | tuple[type[Any], ...] | types.UnionType + +Fn2 = Callable[[T | S], R] +Fn3 = Callable[[T | S | U], R] +Fn = Callable[[T], R] +FnAny = Callable[[Any], R] + +MapOnlyFn = Callable[[T], Callable[[Any], Any]] + + +# These specializations help with type inference on the lambda passed to this +# function +@overload +def map_only(type_or_types_or_pred: type[T], /) -> MapOnlyFn[Fn[T, Any]]: ... + + +@overload +def map_only(type_or_types_or_pred: Type2[T, S], /) -> MapOnlyFn[Fn2[T, S, Any]]: ... + + +@overload +def map_only( + type_or_types_or_pred: Type3[T, S, U], / +) -> MapOnlyFn[Fn3[T, S, U, Any]]: ... + + +# This specialization is needed for the implementations below that call +@overload +def map_only(type_or_types_or_pred: TypeAny, /) -> MapOnlyFn[FnAny[Any]]: ... + + +@overload +def map_only( + type_or_types_or_pred: Callable[[Any], bool], / +) -> MapOnlyFn[FnAny[Any]]: ... + + +def map_only( + type_or_types_or_pred: TypeAny | Callable[[Any], bool], / +) -> MapOnlyFn[FnAny[Any]]: + """ + Suppose you are writing a tree_map over tensors, leaving everything + else unchanged. Ordinarily you would have to write: + + def go(t): + if isinstance(t, Tensor): + return ... + else: + return t + + With this function, you only need to write: + + @map_only(Tensor) + def go(t): + return ... + + You can also directly use 'tree_map_only' + """ + if isinstance(type_or_types_or_pred, (type, tuple, types.UnionType)): + + def pred(x: Any) -> bool: + return isinstance(x, type_or_types_or_pred) # type: ignore[arg-type] + + elif callable(type_or_types_or_pred): + pred = type_or_types_or_pred # type: ignore[assignment] + else: + raise TypeError("Argument must be a type, a tuple of types, or a callable.") + + def wrapper(func: Callable[[T], Any]) -> Callable[[Any], Any]: + @functools.wraps(func) + def wrapped(x: T) -> Any: + if pred(x): + return func(x) + return x + + return wrapped + + return wrapper + + +@overload +def tree_map_only( + type_or_types_or_pred: type[T], + /, + func: Fn[T, Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only( + type_or_types_or_pred: Type2[T, S], + /, + func: Fn2[T, S, Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only( + type_or_types_or_pred: Type3[T, S, U], + /, + func: Fn3[T, S, U, Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only( + type_or_types_or_pred: TypeAny, + /, + func: FnAny[Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only( + type_or_types_or_pred: Callable[[Any], bool], + /, + func: FnAny[Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +def tree_map_only( + type_or_types_or_pred: TypeAny | Callable[[Any], bool], + /, + func: FnAny[Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: + return tree_map(map_only(type_or_types_or_pred)(func), tree, is_leaf=is_leaf) + + +@overload +def tree_map_only_( + type_or_types_or_pred: type[T], + /, + func: Fn[T, Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only_( + type_or_types_or_pred: Type2[T, S], + /, + func: Fn2[T, S, Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only_( + type_or_types_or_pred: Type3[T, S, U], + /, + func: Fn3[T, S, U, Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only_( + type_or_types_or_pred: TypeAny, + /, + func: FnAny[Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +@overload +def tree_map_only_( + type_or_types_or_pred: Callable[[Any], bool], + /, + func: FnAny[Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: ... + + +def tree_map_only_( + type_or_types_or_pred: TypeAny | Callable[[Any], bool], + /, + func: FnAny[Any], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: + return tree_map_(map_only(type_or_types_or_pred)(func), tree, is_leaf=is_leaf) + + +def tree_all( + pred: Callable[[Any], bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: + flat_args = tree_iter(tree, is_leaf=is_leaf) + return all(map(pred, flat_args)) + + +def tree_any( + pred: Callable[[Any], bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: + flat_args = tree_iter(tree, is_leaf=is_leaf) + return any(map(pred, flat_args)) + + +@overload +def tree_all_only( + type_or_types: type[T], + /, + pred: Fn[T, bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: ... + + +@overload +def tree_all_only( + type_or_types: Type2[T, S], + /, + pred: Fn2[T, S, bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: ... + + +@overload +def tree_all_only( + type_or_types: Type3[T, S, U], + /, + pred: Fn3[T, S, U, bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: ... + + +def tree_all_only( + type_or_types: TypeAny, + /, + pred: FnAny[bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: + flat_args = tree_iter(tree, is_leaf=is_leaf) + return all(pred(x) for x in flat_args if isinstance(x, type_or_types)) + + +@overload +def tree_any_only( + type_or_types: type[T], + /, + pred: Fn[T, bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: ... + + +@overload +def tree_any_only( + type_or_types: Type2[T, S], + /, + pred: Fn2[T, S, bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: ... + + +@overload +def tree_any_only( + type_or_types: Type3[T, S, U], + /, + pred: Fn3[T, S, U, bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: ... + + +def tree_any_only( + type_or_types: TypeAny, + /, + pred: FnAny[bool], + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> bool: + flat_args = tree_iter(tree, is_leaf=is_leaf) + return any(pred(x) for x in flat_args if isinstance(x, type_or_types)) + + +# Broadcasts a pytree to the provided TreeSpec and returns the flattened +# values. If this is not possible, then this function returns None. +# +# For example, given pytree=0 and spec=TreeSpec(list, None, [LeafSpec(), LeafSpec()]), +# would return [0, 0]. This is useful for part of the vmap implementation: +# a user can pass in vmap(fn, in_dims)(*inputs). `in_dims` should be +# broadcastable to the tree structure of `inputs` and we use +# _broadcast_to_and_flatten to check this. +def _broadcast_to_and_flatten( + tree: PyTree, + treespec: TreeSpec, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> list[Any] | None: + def broadcast_prefix( + prefix_tree: PyTree, + full_tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, + ) -> list[Any]: + result: list[Any] = [] + + def add_leaves(x: Any, subtree: PyTree) -> None: + subtreespec = tree_structure(subtree, is_leaf=is_leaf) + result.extend([x] * subtreespec.num_leaves) + + tree_map_( + add_leaves, + prefix_tree, + full_tree, + is_leaf=is_leaf, + ) + return result + + full_tree = tree_unflatten([0] * treespec.num_leaves, treespec) + try: + return broadcast_prefix(tree, full_tree, is_leaf=is_leaf) + except ValueError: + return None + + +@dataclasses.dataclass +class _TreeSpecSchema: + """ + _TreeSpecSchema is the schema used to serialize the TreeSpec + It contains the following fields: + - type: A string name of the type. null for the case of a LeafSpec. + - context: Any format which is json dumpable + - children_spec: A list of children serialized specs. + """ + + type: str | None + context: DumpableContext + children_spec: list["_TreeSpecSchema"] + + +class _ProtocolFn(NamedTuple): + treespec_to_json: Callable[[TreeSpec], DumpableContext] + json_to_treespec: Callable[[DumpableContext], TreeSpec] + + +_SUPPORTED_PROTOCOLS: dict[int, _ProtocolFn] = {} + + +def _treespec_to_json(treespec: TreeSpec) -> _TreeSpecSchema: + if treespec.is_leaf(): + return _TreeSpecSchema(None, None, []) + + if treespec.type not in SUPPORTED_SERIALIZED_TYPES: + raise NotImplementedError( + f"Serializing {treespec.type} in pytree is not registered.", + ) + + serialize_node_def = SUPPORTED_SERIALIZED_TYPES[treespec.type] + + serialized_type_name = serialize_node_def.serialized_type_name + + if serialized_type_name == NO_SERIALIZED_TYPE_NAME_FOUND: + raise NotImplementedError( + f"No registered serialization name for {treespec.type} found. " + "Please update your _register_pytree_node call with a `serialized_type_name` kwarg." + ) + + if serialize_node_def.to_dumpable_context is None: + try: + serialized_context = json.dumps(treespec._context, cls=EnumEncoder) + except TypeError as e: + raise TypeError( + "Unable to serialize context. " + "Please make the context json dump-able, or register a " + "custom serializer using _register_pytree_node." + ) from e + else: + serialized_context = serialize_node_def.to_dumpable_context(treespec._context) + + child_schemas = [_treespec_to_json(child) for child in treespec._children] + + return _TreeSpecSchema(serialized_type_name, serialized_context, child_schemas) + + +def enum_object_hook(obj: dict[str, Any]) -> Enum | dict[str, Any]: + if "__enum__" in obj: + modname, _, classname = obj["fqn"].partition(":") + mod = importlib.import_module(modname) + enum_cls = mod + for attr in classname.split("."): + enum_cls = getattr(enum_cls, attr) + enum_cls = cast(type[Enum], enum_cls) + # pyrefly: ignore [unsupported-operation] + return enum_cls[obj["name"]] + return obj + + +def _json_to_treespec(json_schema: DumpableContext) -> TreeSpec: + if ( + json_schema["type"] is None + and json_schema["context"] is None + and len(json_schema["children_spec"]) == 0 + ): + return _LEAF_SPEC + + if json_schema["type"] not in SERIALIZED_TYPE_TO_PYTHON_TYPE: + raise NotImplementedError( + f"Deserializing {json_schema['type']} in pytree is not registered.", + ) + + typ = SERIALIZED_TYPE_TO_PYTHON_TYPE[json_schema["type"]] + serialize_node_def = SUPPORTED_SERIALIZED_TYPES[typ] + + if serialize_node_def.from_dumpable_context is None: + try: + context = json.loads(json_schema["context"], object_hook=enum_object_hook) + except TypeError as ex: + raise TypeError( + "Unable to deserialize context. " + "Please make the context json load-able, or register a " + "custom serializer using _register_pytree_node.", + ) from ex + else: + context = serialize_node_def.from_dumpable_context(json_schema["context"]) + + children_specs = [ + _json_to_treespec(child_string) for child_string in json_schema["children_spec"] + ] + + return TreeSpec(typ, context, children_specs) + + +_SUPPORTED_PROTOCOLS[1] = _ProtocolFn(_treespec_to_json, _json_to_treespec) + + +def treespec_dumps(treespec: TreeSpec, protocol: int | None = None) -> str: + treespec = _ensure_python_treespec_instance(treespec) + + if protocol is None: + protocol = DEFAULT_TREESPEC_SERIALIZATION_PROTOCOL + + if protocol in _SUPPORTED_PROTOCOLS: + json_spec = _SUPPORTED_PROTOCOLS[protocol].treespec_to_json(treespec) + else: + raise ValueError( + f"Unknown protocol {protocol}. " + f"Available protocols: {list(_SUPPORTED_PROTOCOLS.keys())}", + ) + + str_spec = json.dumps((protocol, dataclasses.asdict(json_spec)), cls=EnumEncoder) + return str_spec + + +@functools.lru_cache +def treespec_loads(serialized: str) -> TreeSpec: + protocol, json_schema = json.loads(serialized) + + if protocol in _SUPPORTED_PROTOCOLS: + return _SUPPORTED_PROTOCOLS[protocol].json_to_treespec(json_schema) + raise ValueError( + f"Unknown protocol {protocol}. " + f"Available protocols: {list(_SUPPORTED_PROTOCOLS.keys())}", + ) + + +class _DummyLeaf: + def __repr__(self) -> str: + return "*" + + +def treespec_pprint(treespec: TreeSpec) -> str: + dummy_tree = tree_unflatten( + [_DummyLeaf() for _ in range(treespec.num_leaves)], + treespec, + ) + return repr(dummy_tree) + + +# TODO(angelayi): remove this function after OSS/internal stabilize +@deprecated( + "`pytree_to_str` is deprecated. Please use `treespec_dumps` instead.", + category=FutureWarning, +) +def pytree_to_str(treespec: TreeSpec) -> str: + return treespec_dumps(treespec) + + +# TODO(angelayi): remove this function after OSS/internal stabilize +@deprecated( + "`str_to_pytree` is deprecated. Please use `treespec_loads` instead.", + category=FutureWarning, +) +def str_to_pytree(json: str) -> TreeSpec: + return treespec_loads(json) + + +def arg_tree_leaves(*args: PyTree, **kwargs: PyTree) -> list[Any]: + """Get a flat list of arguments to this function + + A slightly faster version of tree_leaves((args, kwargs)) + """ + leaves: list[Any] = [] + for a in args: + leaves.extend(tree_iter(a)) + for a in kwargs.values(): + leaves.extend(tree_iter(a)) + return leaves + + +def tree_flatten_with_path( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> tuple[list[tuple[KeyPath, Any]], TreeSpec]: + """Flattens a pytree like :func:`tree_flatten`, but also returns each leaf's key path. + + Args: + tree: a pytree to flatten. If it contains a custom type, that type must be + registered with an appropriate `tree_flatten_with_path_fn` when registered + with :func:`register_pytree_node`. + is_leaf: An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + Returns: + A tuple where the first element is a list of (key path, leaf) pairs, and the + second element is a :class:`TreeSpec` representing the structure of the flattened + tree. + """ + _, treespec = tree_flatten(tree, is_leaf) + return list(_generate_key_paths((), tree, is_leaf)), treespec + + +def tree_leaves_with_path( + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> list[tuple[KeyPath, Any]]: + """Gets the leaves of a pytree like ``tree_leaves`` and returns each leaf's key path. + + Args: + tree: a pytree. If it contains a custom type, that type must be + registered with an appropriate `tree_flatten_with_path_fn` when registered + with :func:`register_pytree_node`. + is_leaf: An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + Returns: + A list of (key path, leaf) pairs. + """ + return list(_generate_key_paths((), tree, is_leaf)) + + +def _generate_key_paths( + key_path: KeyPath, + tree: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> Iterable[tuple[KeyPath, Any]]: + if is_leaf and is_leaf(tree): + yield key_path, tree + return + + node_type = _get_node_type(tree) + handler = SUPPORTED_NODES.get(node_type) + if not handler: + # This is a leaf + yield key_path, tree + return + + flatten_with_keys = handler.flatten_with_keys_fn + if flatten_with_keys: + key_children, _ = flatten_with_keys(tree) + for k, c in key_children: + yield from _generate_key_paths((*key_path, k), c, is_leaf) + else: + # We registered this pytree but didn't add a flatten_with_keys_fn, complain. + raise ValueError( + f"Did not find a flatten_with_keys_fn for type: {node_type}. " + "Please pass a flatten_with_keys_fn argument to register_pytree_node." + ) + + +def tree_map_with_path( + func: Callable[..., Any], + tree: PyTree, + *rests: PyTree, + is_leaf: Callable[[PyTree], bool] | None = None, +) -> PyTree: + """Like :func:`tree_map`, but the provided callable takes an additional key path argument. + + Args: + func: A function that takes ``2 + len(rests)`` arguments, to be applied at the + corresponding leaves of the pytrees. The first positional argument + to ``func`` is the key path of the leaf in question. The second + positional argument is the value of the leaf. + tree: A pytree to be mapped over, with each leaf providing the first positional + argument to function ``func``. + rests: A tuple of pytrees, each of which has the same structure as + ``tree`` or has ``tree`` as a prefix. + is_leaf: An extra leaf predicate function that will be called at each + flattening step. The function should have a single argument with signature + ``is_leaf(node) -> bool``. If it returns :data:`True`, the whole subtree being treated + as a leaf. Otherwise, the default pytree registry will be used to determine a node is a + leaf or not. If the function is not specified, the default pytree registry will be used. + + Returns + A new pytree with the same structure as ``tree`` but with the value at each leaf given by + ``func(keypath, x, *xs)`` where ``keypath`` is the key path at the + corresponding leaf in ``tree``, ``x`` is the value at that leaf, and + ``xs`` is the tuple of values at corresponding nodes in ``rests``. + """ + keypath_leaves, treespec = tree_flatten_with_path(tree, is_leaf) + keypath_leaves = list(zip(*keypath_leaves, strict=True)) + all_keypath_leaves = keypath_leaves + [treespec.flatten_up_to(r) for r in rests] + return treespec.unflatten(func(*xs) for xs in zip(*all_keypath_leaves, strict=True)) + + +def keystr(kp: KeyPath) -> str: + """Given a key path, return a pretty-printed representation.""" + return "".join([str(k) for k in kp]) + + +def key_get(obj: Any, kp: KeyPath) -> Any: + """Given an object and a key path, return the value at the key path.""" + for k in kp: + obj = k.get(obj) + return obj diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_runtime_estimation.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_runtime_estimation.py new file mode 100644 index 0000000000000000000000000000000000000000..fcda7cceaee4873fb76b73e73a2a19ebfd5e139a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_runtime_estimation.py @@ -0,0 +1,151 @@ +import math +import os + +import torch +from torch._inductor.utils import get_device_tflops, get_gpu_dram_gbps +from torch.utils._ordered_set import OrderedSet + +from .flop_counter import flop_registry + + +aten = torch.ops.aten + +_FLOAT_TYPES = OrderedSet( + [ + torch.float16, + torch.bfloat16, + torch.float32, + torch.float64, + ] +) + +# This value is hard-coded here: +# https://github.com/pytorch/pytorch/blob/5fba5d83f0703ff8077ab65448a998e9ad6598fd/c10/cuda/CUDACachingAllocator.cpp#L117 +_PYTORCH_MIN_ALLOCATE = ( + 2**9 if int(os.environ.get("PYTORCH_NO_CUDA_MEMORY_CACHING", 0)) == 0 else 1 +) + +# No fall-back kernel needed/exists for view ops +_VIEW_OPS = OrderedSet( + [ + aten.lift_fresh, + aten.t, + aten.transpose, + aten.view, + aten.detach, + aten._unsafe_view, + aten.split, + aten.adjoint, + aten.as_strided, + aten.diagonal, + aten.expand, + aten.expand_as, + aten.movedim, + aten.permute, + aten.select, + aten.squeeze, + aten.mT, + aten.mH, + aten.real, + aten.imag, + aten.view_as, + aten.unflatten, + aten.unfold, + aten.unbind, + aten.unsqueeze, + aten.vsplit, + aten.hsplit, + aten.split_with_sizes, + aten.swapaxes, + aten.swapdims, + aten.chunk, + ] +) +# We can ignore benchmarking tensor create ops +_CREATE_OPS = OrderedSet( + [ + aten.randint, + aten.randn, + aten.rand, + aten.randn_like, + aten.rand_like, + aten.randint_like, + aten.arange, + aten.ones_like, + aten.zeros_like, + ] +) + +_IGNORE_OPS = _VIEW_OPS | _CREATE_OPS + + +def get_compute_time(func_packet, args, kwargs, out, out_dtypes) -> float: # type: ignore[no-untyped-def] + """ + Estimates the compute time of an aten operator. + + Args: + func_packet: The operator overload packet. + args: The arguments to the operator. + kwargs: The keyword arguments to the operator. + out: The output of the operator. + out_dtypes: The output data types. + + Returns: + float: The estimated compute time in nanoseconds. + """ + if func_packet in flop_registry: + assert len(out_dtypes) == 1, ( + f"Only support single out dtype got {out_dtypes} for {func_packet}" + ) + dtype = out_dtypes.pop() + # This actually gives peta-FLOPs/s hence multiply by 1e15 to get the FLOPs/s + peak_gpu_flops = get_device_tflops(dtype) * 1e15 + # We can expect to achieve 75% of theoretical peak flops + factor = 0.75 + peak_empirical_flops = factor * peak_gpu_flops + flop_count_func = flop_registry[func_packet] + # We divide by a factor of 2 to get the MACs (multiply and accumulate) + flop_count = flop_count_func(*args, **kwargs, out_val=out) / 2 + # We multiply by 1e9 to get the time in nano seconds + compute_time = (flop_count / peak_empirical_flops) * 1e9 + return compute_time + return 0.0 + + +def get_num_bytes(t: torch.Tensor) -> int: + """ + Calculates the memory consumption of a tensor. + + Args: + t (torch.Tensor): The input tensor. + + Returns: + int: The memory consumption of the tensor in bytes. + """ + num_bytes = t.untyped_storage().nbytes() + mem_consumed = math.ceil(num_bytes / _PYTORCH_MIN_ALLOCATE) * _PYTORCH_MIN_ALLOCATE + return mem_consumed + + +def get_transfer_time(flat_args_kwargs, flat_outs) -> float: # type: ignore[no-untyped-def] + """ + Estimates the memory transfer time of input and output tensors. + + Args: + flat_args_kwargs (List[torch.Tensor]): The flat list of arguments and keyword arguments. + flat_outs (List[torch.Tensor]): The flat list of outputs. + + Returns: + float: The estimated memory transfer time in nanoseconds. + """ + gpu_memory_bandwidth = get_gpu_dram_gbps() + read_bytes = sum( + get_num_bytes(t) for t in flat_args_kwargs if isinstance(t, torch.Tensor) + ) + write_bytes = sum( + get_num_bytes(t) for t in flat_outs if isinstance(t, torch.Tensor) + ) + counted_bytes = read_bytes + write_bytes + # The GPU memory bandwidth is in GB/s so the transfer time is in nanoseconds + transfer_time = counted_bytes / gpu_memory_bandwidth + return transfer_time diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_stats.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_stats.py new file mode 100644 index 0000000000000000000000000000000000000000..b8a2978c3ea7066c56070382d0d4866faf58ab8d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_stats.py @@ -0,0 +1,31 @@ +# NOTE! PLEASE KEEP THIS FILE *FREE* OF TORCH DEPS! IT SHOULD BE IMPORTABLE ANYWHERE. +# IF YOU FEEL AN OVERWHELMING URGE TO ADD A TORCH DEP, MAKE A TRAMPOLINE FILE A LA torch._dynamo.utils +# AND SCRUB AWAY TORCH NOTIONS THERE. +import collections +import functools +from collections import OrderedDict +from collections.abc import Callable +from typing import TypeVar +from typing_extensions import ParamSpec + + +simple_call_counter: OrderedDict[str, int] = collections.OrderedDict() + +_P = ParamSpec("_P") +_R = TypeVar("_R") + + +def count_label(label: str) -> None: + prev = simple_call_counter.setdefault(label, 0) + simple_call_counter[label] = prev + 1 + + +def count(fn: Callable[_P, _R]) -> Callable[_P, _R]: + @functools.wraps(fn) + def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R: + if fn.__qualname__ not in simple_call_counter: + simple_call_counter[fn.__qualname__] = 0 + simple_call_counter[fn.__qualname__] = simple_call_counter[fn.__qualname__] + 1 + return fn(*args, **kwargs) + + return wrapper diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_thunk.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_thunk.py new file mode 100644 index 0000000000000000000000000000000000000000..b5ab598077f4e8d3d9de9169a1352918771f07f6 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_thunk.py @@ -0,0 +1,29 @@ +from collections.abc import Callable +from typing import Generic, TypeVar + + +R = TypeVar("R") + + +class Thunk(Generic[R]): + """ + A simple lazy evaluation implementation that lets you delay + execution of a function. It properly handles releasing the + function once it is forced. + """ + + f: Callable[[], R] | None + r: R | None + + __slots__ = ["f", "r"] + + def __init__(self, f: Callable[[], R]) -> None: + self.f = f + self.r = None + + def force(self) -> R: + if self.f is None: + return self.r # type: ignore[return-value] + self.r = self.f() + self.f = None + return self.r diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_traceback.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_traceback.py new file mode 100644 index 0000000000000000000000000000000000000000..f5415002092a23350f7d7d3436388d34b1eb9501 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_traceback.py @@ -0,0 +1,260 @@ +# mypy: allow-untyped-defs +import contextlib +import inspect +import os.path +import tempfile +import traceback +from types import TracebackType + + +# This file contains utilities for ensuring dynamically compile()'d +# code fragments display their line numbers in backtraces. +# +# The constraints: +# +# - We don't have control over the user exception printer (in particular, +# we cannot assume the linecache trick will work, c.f. +# https://stackoverflow.com/q/50515651/23845 ) +# +# - We don't want to create temporary files every time we compile() +# some code; file creation should happen lazily only at exception +# time. Arguably, you *should* be willing to write out your +# generated Python code to file system, but in some situations +# (esp. library code) it would violate user expectation to write +# to the file system, so we try to avoid it. In particular, we'd +# like to keep the files around, so users can open up the files +# mentioned in the trace; if the file is invisible, we want to +# avoid clogging up the filesystem. +# +# If this is not a constraint for you, there is a substantially simpler +# way to implement the functionality in this PR: instead of using +# eval/exec directly, just always write a Python file to filesystem +# and compile that. +# +# - You have control over a context where the compiled code will get +# executed, so that we can interpose while the stack is unwinding +# (otherwise, we have no way to interpose on the exception printing +# process.) +# +# There are two things you have to do to make use of the utilities here: +# +# - When you compile your source code, you must save its string source +# in its f_globals under the magic name "__compile_source__" +# +# - Before running the compiled code, enter the +# report_compile_source_on_error() context manager. + + +@contextlib.contextmanager +def report_compile_source_on_error(): + try: + yield + except Exception as exc: + tb = exc.__traceback__ + + # Walk the traceback, looking for frames that have + # source attached + stack = [] + while tb is not None: + filename = tb.tb_frame.f_code.co_filename + source = tb.tb_frame.f_globals.get("__compile_source__") + + if filename == "" and source is not None: + # What black magic are we doing here? Intuitively, what + # we would like to do is overwrite the co_filename on any + # frames that were generated from exec/eval so that they + # point to a temporary file that has the actual line + # information, so Python's default error printer can print + # useful line information on it. + # + # Writing out the temporary file is easy. But overwriting + # co_filename is not! You can't modify the code object + # associated with a frame. You can, however, reconstruct + # a traceback with entirely new frames from scratch, so that's + # what we do. But there's another problem, which is how to + # make the frame? + # + # The black magic is we make a frankenstein frame and code + # object which resembles the original frame/code enough so + # that it will print properly under traceback and the default + # error printer, but IT IS NOT THE ORIGINAL FRAME (you + # couldn't, e.g., execute its code with different variables + # and expect it to work.) + + # Don't delete the temporary file so the user can inspect it + # TODO: This creates a temporary file for every frame, but we + # technically only need one per distinct __compile_source__ + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".py" + ) as f: + f.write(source) + # Create a frame. Python doesn't let you construct + # FrameType directly, so just make one with compile + frame = tb.tb_frame + code = compile("__inspect_currentframe()", f.name, "eval") + code = code.replace(co_name=frame.f_code.co_name) + # Python 3.11 only + if hasattr(frame.f_code, "co_linetable"): + # We can't copy ALL of the metadata over, because you + # can cause Python to segfault this way. What exactly + # do we need? We need enough information for + # traceback to be able to print the exception + # correctly. Code reading Lib/traceback.py reveals + # that traceback calls code.co_positions() in order to + # get the augmented line/col numbers. Objects/codeobject.c, + # specifically _PyCode_InitAddressRange, reveals that + # this iterator is initialized from co_linetable and + # co_firstfileno. So copy these we must! + code = code.replace( # type: ignore[call-arg] + co_linetable=frame.f_code.co_linetable, # type: ignore[attr-defined] + co_firstlineno=frame.f_code.co_firstlineno, # type: ignore[attr-defined] + ) + fake_frame = eval( + code, + frame.f_globals, + {**frame.f_locals, "__inspect_currentframe": inspect.currentframe}, + ) + fake_tb = TracebackType(None, fake_frame, tb.tb_lasti, tb.tb_lineno) + stack.append(fake_tb) + else: + stack.append(tb) + + tb = tb.tb_next + + # Reconstruct the linked list + tb_next = None + for tb in reversed(stack): + tb.tb_next = tb_next + tb_next = tb + + raise exc.with_traceback(tb_next) # noqa: B904 + + +def shorten_filename(fn, *, base=None): + """Shorten a source filepath, with the assumption that torch/ subdirectories don't need to be shown to user.""" + if base is None: + base = os.path.dirname(os.path.dirname(__file__)) + # Truncate torch/foo.py to foo.py + try: + prefix = os.path.commonpath([fn, base]) + except ValueError: + return fn + else: + return fn[len(prefix) + 1 :] + + +def format_frame(frame, *, base=None, line=False) -> str: + """ + Format a FrameSummary in a short way, without printing full absolute path or code. + + The idea is the result fits on a single line. + """ + extra_line = "" + if line: + extra_line = f"{frame.line} # " + return f"{extra_line}{shorten_filename(frame.filename, base=base)}:{frame.lineno} in {frame.name}" + + +def format_traceback_short(tb): + """Format a TracebackType in a short way, printing only the inner-most frame.""" + return format_frame(traceback.extract_tb(tb)[-1]) + + +class CapturedTraceback: + __slots__ = ["tb", "skip"] + + def __init__(self, tb, skip=0) -> None: + self.tb = tb + self.skip = skip + + def cleanup(self) -> None: + self.tb = None + + def summary(self): + import torch._C._profiler + + if self.tb is None: + # TODO: Maybe indicate that the traceback was elided? + return traceback.StackSummary() + + return _extract_symbolized_tb( + torch._C._profiler.symbolize_tracebacks([self.tb])[0], self.skip + ) + + def __getstate__(self): + return ( + None, + { + "tb": None, # TB is not pickleable + "skip": self.skip, + }, + ) + + @staticmethod + def extract(*, script=False, cpp=False, skip=0): + """ + Like traceback.extract_stack(), but faster (approximately 20x faster); it + is fast enough that you can unconditionally log stacks this way as part of + normal execution. It returns a torch._C._profiler.CapturedTraceback + object that must be formatted specially with format_captured_tb. + + By default, this only reports Python backtraces (like extract_stack). You + can set the script/cpp kwargs to also turn on TorchScript/C++ trace + reporting. + """ + import torch._C._profiler + + if script or cpp: + if skip != 0: + raise AssertionError("skip with script/cpp NYI") + + return CapturedTraceback( + torch._C._profiler.gather_traceback(python=True, script=script, cpp=cpp), + # Elide extract() frame if we don't have script/cpp frames. If + # we do have those frames, it doesn't work so force zero. + 0 if script or cpp else skip + 1, + ) + + def format(self): + """ + Formats a single torch._C._profiler.CapturedTraceback into a list of + strings equivalent to the output of traceback.format_list. Note that if + pass it CapturedTraceback with C++ traces, it is better not to use this + function and use the batch formatting API format_captured_tbs to amortize + the cost of symbolization + """ + return traceback.format_list(self.summary()) + + @staticmethod + def format_all(tbs): + """ + Bulk version of CapturedTraceback.format. Returns a list of list of strings. + """ + import torch._C._profiler + + # Directly populate tracebacks that already have cached summaries + rs: list[list[str] | None] = [] + delayed_idxs = [] + for i, tb in enumerate(tbs): + if tb.tb is None: + rs.append([]) + else: + rs.append(None) + delayed_idxs.append(i) + + torch._C._profiler.symbolize_tracebacks([tbs[i].tb for i in delayed_idxs]) + for i in delayed_idxs: + rs[i] = traceback.format_list(tbs[i].summary()) + + return rs + + +def _extract_symbolized_tb(tb, skip): + """ + Given a symbolized traceback from symbolize_tracebacks, return a StackSummary object of + pre-processed stack trace entries. + """ + stack = traceback.StackSummary() + for f in reversed(tb[skip:]): + stack.append(traceback.FrameSummary(f["filename"], f["line"], f["name"])) + return stack diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_triton.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_triton.py new file mode 100644 index 0000000000000000000000000000000000000000..98de7bbcc5868b3850a5633a4165d4d965ee262e --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_triton.py @@ -0,0 +1,204 @@ +import functools +import hashlib +from typing import Any + + +@functools.cache +def has_triton_package() -> bool: + try: + import triton # noqa: F401 + + return True + except ImportError: + return False + + +@functools.cache +def get_triton_version(fallback: tuple[int, int] = (0, 0)) -> tuple[int, int]: + try: + import triton + + major, minor = tuple(int(v) for v in triton.__version__.split(".")[:2]) + return (major, minor) + except ImportError: + return fallback + + +@functools.cache +def _device_supports_tma() -> bool: + import torch + + return ( + torch.cuda.is_available() + and torch.cuda.get_device_capability() >= (9, 0) + and not torch.version.hip + ) + + +@functools.cache +def has_triton_experimental_host_tma() -> bool: + if has_triton_package(): + if _device_supports_tma(): + try: + from triton.tools.experimental_descriptor import ( # noqa: F401 + create_1d_tma_descriptor, + create_2d_tma_descriptor, + ) + + try: + from triton.tools.experimental_descriptor import enable_in_pytorch + + return enable_in_pytorch() + except ImportError: + return True + except ImportError: + pass + + return False + + +@functools.cache +def has_triton_tensor_descriptor_host_tma() -> bool: + if has_triton_package(): + if _device_supports_tma(): + try: + from triton.tools.tensor_descriptor import ( # noqa: F401 + TensorDescriptor, + ) + + return True + except ImportError: + pass + + return False + + +@functools.cache +def has_triton_tma() -> bool: + return has_triton_tensor_descriptor_host_tma() or has_triton_experimental_host_tma() + + +@functools.cache +def has_triton_tma_device() -> bool: + if has_triton_package(): + import torch + + if ( + torch.cuda.is_available() + and torch.cuda.get_device_capability() >= (9, 0) + and not torch.version.hip + ) or torch.xpu.is_available(): + # old API + try: + from triton.language.extra.cuda import ( # noqa: F401 + experimental_device_tensormap_create1d, + experimental_device_tensormap_create2d, + ) + + return True + except ImportError: + pass + + # new API + try: + from triton.language import make_tensor_descriptor # noqa: F401 + + return True + except ImportError: + pass + + return False + + +@functools.cache +def has_datacenter_blackwell_tma_device() -> bool: + import torch + + if ( + torch.cuda.is_available() + and torch.cuda.get_device_capability() >= (10, 0) + and torch.cuda.get_device_capability() < (11, 0) + and not torch.version.hip + ): + return has_triton_tma_device() and has_triton_tensor_descriptor_host_tma() + + return False + + +@functools.lru_cache(None) +def has_triton_stable_tma_api() -> bool: + if has_triton_package(): + import torch + + if ( + torch.cuda.is_available() + and torch.cuda.get_device_capability() >= (9, 0) + and not torch.version.hip + ) or torch.xpu.is_available(): + try: + from triton.language import make_tensor_descriptor # noqa: F401 + + return True + except ImportError: + pass + return False + + +@functools.cache +def has_triton() -> bool: + if not has_triton_package(): + return False + + from torch._inductor.config import triton_disable_device_detection + + if triton_disable_device_detection: + return False + + from torch._dynamo.device_interface import get_interface_for_device + + def cuda_extra_check(device_interface: Any) -> bool: + return device_interface.Worker.get_device_properties().major >= 7 + + def cpu_extra_check(device_interface: Any) -> bool: + import triton.backends + + return "cpu" in triton.backends.backends + + def _return_true(device_interface: Any) -> bool: + return True + + triton_supported_devices = { + "cuda": cuda_extra_check, + "xpu": _return_true, + "cpu": cpu_extra_check, + "mtia": _return_true, + } + + def is_device_compatible_with_triton() -> bool: + for device, extra_check in triton_supported_devices.items(): + device_interface = get_interface_for_device(device) + if device_interface.is_available() and extra_check(device_interface): + return True + return False + + return is_device_compatible_with_triton() + + +@functools.cache +def triton_backend() -> Any: + from triton.compiler.compiler import make_backend + from triton.runtime.driver import driver + + target = driver.active.get_current_target() + return make_backend(target) + + +@functools.cache +def triton_hash_with_backend() -> str: + from torch._inductor.runtime.triton_compat import triton_key + + backend = triton_backend() + key = f"{triton_key()}-{backend.hash()}" + + # Hash is upper case so that it can't contain any Python keywords. + return hashlib.sha256(key.encode("utf-8")).hexdigest().upper() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_typing_utils.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_typing_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f28c9f94100b76cfb75f6d300c2a5ecaae325fa8 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_typing_utils.py @@ -0,0 +1,14 @@ +"""Miscellaneous utilities to aid with typing.""" + +from typing import TypeVar + + +# Helper to turn Optional[T] into T when we know None either isn't +# possible or should trigger an exception. +T = TypeVar("T") + + +def not_none(obj: T | None) -> T: + if obj is None: + raise TypeError("Invariant encountered: value was None when it should not be") + return obj diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_zip.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_zip.py new file mode 100644 index 0000000000000000000000000000000000000000..c4bfbcb0b9b637f82fba7c9722cd6fbc5690555c --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/_zip.py @@ -0,0 +1,86 @@ +# mypy: allow-untyped-defs +import argparse +import glob +import os +from pathlib import Path +from zipfile import ZipFile + + +# Exclude some standard library modules to: +# 1. Slim down the final zipped file size +# 2. Remove functionality we don't want to support. +DENY_LIST = [ + # Interface to unix databases + "dbm", + # ncurses bindings (terminal interfaces) + "curses", + # Tcl/Tk GUI + "tkinter", + "tkinter", + # Tests for the standard library + "test", + "tests", + "idle_test", + "__phello__.foo.py", + # importlib frozen modules. These are already baked into CPython. + "_bootstrap.py", + "_bootstrap_external.py", +] + +strip_file_dir = "" + + +def remove_prefix(text, prefix): + if text.startswith(prefix): + return text[len(prefix) :] + return text + + +def write_to_zip(file_path, strip_file_path, zf, prepend_str="") -> None: + stripped_file_path = prepend_str + remove_prefix(file_path, strip_file_dir + "/") + path = Path(stripped_file_path) + if path.name in DENY_LIST: + return + zf.write(file_path, stripped_file_path) + + +def main() -> None: + global strip_file_dir + parser = argparse.ArgumentParser(description="Zip py source") + parser.add_argument("paths", nargs="*", help="Paths to zip.") + parser.add_argument( + "--install-dir", "--install_dir", help="Root directory for all output files" + ) + parser.add_argument( + "--strip-dir", + "--strip_dir", + help="The absolute directory we want to remove from zip", + ) + parser.add_argument( + "--prepend-str", + "--prepend_str", + help="A string to prepend onto all paths of a file in the zip", + default="", + ) + parser.add_argument("--zip-name", "--zip_name", help="Output zip name") + + args = parser.parse_args() + + zip_file_name = args.install_dir + "/" + args.zip_name + strip_file_dir = args.strip_dir + prepend_str = args.prepend_str + with ZipFile(zip_file_name, mode="w") as zf: + for p in sorted(args.paths): + if os.path.isdir(p): + files = glob.glob(p + "/**/*.py", recursive=True) + for file_path in sorted(files): + # strip the absolute path + write_to_zip( + file_path, strip_file_dir + "/", zf, prepend_str=prepend_str + ) + else: + write_to_zip(p, strip_file_dir + "/", zf, prepend_str=prepend_str) + + +if __name__ == "__main__": + main() # pragma: no cover diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/backend_registration.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/backend_registration.py new file mode 100644 index 0000000000000000000000000000000000000000..2300306d22d2d5f246aecbc85e5cd25f85b609cf --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/backend_registration.py @@ -0,0 +1,521 @@ +# mypy: allow-untyped-defs + +import torch +from torch._C import _get_privateuse1_backend_name, _rename_privateuse1_backend +from torch.overrides import handle_torch_function, has_torch_function_unary + + +__all__ = [ + "rename_privateuse1_backend", + "generate_methods_for_privateuse1_backend", +] + +# TODO: Should use `torch._C._get_privateuse1_backend_name()` to get +# renamed-backend name for `privateuse1`, but the func will cause an +# error with torch.jit.script, so we use the global variable named +# `_privateuse1_backend_name`. +_privateuse1_backend_name = "privateuseone" + + +def rename_privateuse1_backend(backend_name: str) -> None: + r""" + Rename the privateuse1 backend device to make it more convenient to use as a device name within PyTorch APIs. + + The steps are: + + (1) (In C++) implement kernels for various torch operations, and register them + to the PrivateUse1 dispatch key. + (2) (In python) call torch.utils.rename_privateuse1_backend("foo") + + You can now use "foo" as an ordinary device string in python. + + Note: this API can only be called once per process. Attempting to change + the external backend after it's already been set will result in an error. + + Note(AMP): If you want to support AMP on your device, you can register a custom backend module. + The backend must register a custom backend module with ``torch._register_device_module("foo", BackendModule)``. + BackendModule needs to have the following API's: + + (1) ``get_amp_supported_dtype() -> List[torch.dtype]`` + get the supported dtypes on your "foo" device in AMP, maybe the "foo" device supports one more dtype. + + Note(random): If you want to support to set seed for your device, BackendModule needs to have the following API's: + + (1) ``_is_in_bad_fork() -> bool`` + Return ``True`` if now it is in bad_fork, else return ``False``. + + (2) ``manual_seed_all(seed int) -> None`` + Sets the seed for generating random numbers for your devices. + + (3) ``device_count() -> int`` + Returns the number of "foo"s available. + + (4) ``get_rng_state(device: Union[int, str, torch.device] = 'foo') -> Tensor`` + Returns a list of ByteTensor representing the random number states of all devices. + + (5) ``set_rng_state(new_state: Tensor, device: Union[int, str, torch.device] = 'foo') -> None`` + Sets the random number generator state of the specified "foo" device. + + And there are some common funcs: + + (1) ``is_available() -> bool`` + Returns a bool indicating if "foo" is currently available. + + (2) ``current_device() -> int`` + Returns the index of a currently selected device. + + For more details, see https://pytorch.org/tutorials/advanced/extend_dispatcher.html#get-a-dispatch-key-for-your-backend + For an existing example, see https://github.com/bdhirsh/pytorch_open_registration_example + + Example:: + + >>> # xdoctest: +SKIP("failing") + >>> torch.utils.rename_privateuse1_backend("foo") + # This will work, assuming that you've implemented the right C++ kernels + # to implement torch.ones. + >>> a = torch.ones(2, device="foo") + + """ + _rename_privateuse1_backend(backend_name) + global _privateuse1_backend_name + _privateuse1_backend_name = backend_name + + +def _check_register_once(module, attr) -> None: + if hasattr(module, attr): + raise RuntimeError( + f"The custom device module of {module} has already been registered with {attr}" + ) + + +def _normalization_device( + custom_backend_name: str, device: int | str | torch.device | None = None +) -> int: + def _get_current_device_index(): + _get_device_index = "current_device" + if hasattr(torch, custom_backend_name) and hasattr( + getattr(torch, custom_backend_name), _get_device_index + ): + return getattr(getattr(torch, custom_backend_name), _get_device_index)() + else: + # The default device index is 0. + return 0 + + if device is None: + return _get_current_device_index() + # if isinstance(device, str), this means that the parameter passed in is in the string format "foo:0" + # convert str object to torch.device object, and then process it uniformly + elif isinstance(device, str): + device = torch.device(device) + + # variable device can only be torch.device type or int type + if isinstance(device, torch.device): + if device.type != custom_backend_name: + raise RuntimeError(f"Invalid device, must be {custom_backend_name} device") + elif device.index is None: + device_idx = _get_current_device_index() + else: + device_idx = device.index + # if isinstance(device, int), we can take the index number directly + else: + device_idx = device + return device_idx + + +def _generate_tensor_methods_for_privateuse1_backend(custom_backend_name: str) -> None: + @property # type: ignore[misc] + def wrap_tensor_backend(self: torch.Tensor) -> bool: + if has_torch_function_unary(self): + # TODO mypy doesn't support @property, see: https://github.com/python/mypy/issues/6185 + return handle_torch_function(wrap_tensor_backend.__get__, (self,), self) # type: ignore[attr-defined] + return self.device.type == custom_backend_name + + _check_register_once(torch.Tensor, f"is_{custom_backend_name}") + wrap_tensor_backend.fget.__name__ = f"is_{custom_backend_name}" # type: ignore[attr-defined] + setattr(torch.Tensor, f"is_{custom_backend_name}", wrap_tensor_backend) + + def wrap_tensor_to( + self: torch.Tensor, + device: int | torch.device | None = None, + non_blocking=False, + **kwargs, + ) -> torch.Tensor: + r"""Perform Tensor device conversion. Call the to operator implementation. + + .. note:: + If the ``self`` Tensor already + has the correct :class:`torch.device`, then ``self`` is returned. + Otherwise, the returned tensor is a copy of ``self`` with the desired :class:`torch.device`. + + Args: + device (int, optional): if specified, all parameters will be copied to that device + non_blocking (bool): If ``True`` and the source is in pinned memory, + the copy will be asynchronous with respect to the host. Otherwise, + the argument has no effect. + **kwargs (dict): For compatibility, may contain the key ``memory_format`` argument. + """ + if has_torch_function_unary(self): + return handle_torch_function( + wrap_tensor_to, + (self,), + self, + device=device, + non_blocking=False, + **kwargs, + ) + device_idx = _normalization_device(custom_backend_name, device) + return self.to( + device=torch.device(f"{custom_backend_name}:{device_idx}"), + non_blocking=non_blocking, + **kwargs, + ) + + _check_register_once(torch.Tensor, custom_backend_name) + wrap_tensor_to.__name__ = custom_backend_name + setattr(torch.Tensor, custom_backend_name, wrap_tensor_to) + + +def _generate_module_methods_for_privateuse1_backend(custom_backend_name: str) -> None: + # Generate Module attributes and methods depends on Tensor methods, + # so we need to check whether Tensor methods is already registered. + if not hasattr(torch.Tensor, custom_backend_name): + raise RuntimeError( + f"Can not automatically generate {custom_backend_name}() method for torch.nn.Module." + f"Because torch.Tensor doesn't has the method {custom_backend_name}()." + f"For this error, you can try setting for_tensor=True." + ) + + def wrap_module_to( + self: torch.nn.modules.module.T, + device: int | torch.device | None = None, + ) -> torch.nn.modules.module.T: + r"""Move all model parameters and buffers to the custom device. + + This also makes associated parameters and buffers different objects. So + it should be called before constructing optimizer if the module will + live on device while being optimized. + + .. note:: + This method modifies the module in-place. + + Args: + device (int, optional): if specified, all parameters will be copied to that device + """ + # pyrefly: ignore [missing-attribute] + return self._apply(lambda t: getattr(t, custom_backend_name)(device)) + + _check_register_once(torch.nn.Module, custom_backend_name) + setattr(torch.nn.Module, custom_backend_name, wrap_module_to) + + +def _generate_packed_sequence_methods_for_privateuse1_backend( + custom_backend_name: str, +) -> None: + # Generate PackedSequence Module attributes and methods depends on Tensor methods, + # so we need to check whether Tensor methods is already registered. + if not hasattr(torch.Tensor, f"is_{custom_backend_name}") or not hasattr( + torch.Tensor, custom_backend_name + ): + raise RuntimeError( + f"Can not automatically generate is_{custom_backend_name}() or " + f"{custom_backend_name}() method for torch.nn.utils.rnn.PackedSequence." + f"Because torch.Tensor doesn't has the method is_{custom_backend_name}()" + f"or {custom_backend_name}()." + f"For this error, you can try setting for_tensor=True." + ) + + @property # type: ignore[misc] + def wrap_tensor_backend(self: torch.nn.utils.rnn.PackedSequence) -> bool: + return self.data.device.type == custom_backend_name + + _check_register_once(torch.nn.utils.rnn.PackedSequence, f"is_{custom_backend_name}") + setattr( + torch.nn.utils.rnn.PackedSequence, + f"is_{custom_backend_name}", + wrap_tensor_backend, + ) + + def wrap_module_to( + self: torch.nn.utils.rnn.PackedSequence, *args, **kwargs + ) -> torch.nn.utils.rnn.PackedSequence: + r"""Move all model parameters and buffers to the custom device. + + This also makes associated parameters and buffers different objects. So + it should be called before constructing optimizer if the module will + live on device while being optimized. + + .. note:: + This method modifies the module in-place. + + Args: + device (int, optional): if specified, all parameters will be copied to that device + """ + ex = torch.tensor((), dtype=self.data.dtype, device=self.data.device).to( + # pyrefly: ignore [not-iterable] + *args, + **kwargs, + ) + if ex.device.type == custom_backend_name: + # pyrefly: ignore [not-iterable] + return self.to(*args, **kwargs) + kwargs.update({"device": custom_backend_name}) + # pyrefly: ignore [not-iterable] + return self.to(*args, **kwargs) + + _check_register_once(torch.nn.utils.rnn.PackedSequence, custom_backend_name) + setattr(torch.nn.utils.rnn.PackedSequence, custom_backend_name, wrap_module_to) + + +def _generate_storage_methods_for_privateuse1_backend( + custom_backend_name: str, unsupported_dtype: list[torch.dtype] | None = None +) -> None: + # Attribute is registered in the _StorageBase class + # and UntypedStorage obtains through inheritance. + @property # type: ignore[misc] + def wrap_storage_backend(self: torch.storage._StorageBase) -> bool: + r"""Return the internal :class:`torch.UntypedStorage`.""" + return self.device.type == custom_backend_name + + _check_register_once(torch.storage._StorageBase, f"is_{custom_backend_name}") + setattr( + torch.storage._StorageBase, f"is_{custom_backend_name}", wrap_storage_backend + ) + + def wrap_storage_to(self, device=None, non_blocking=False): + r"""Return a copy of this object in custom device memory. + + If this object is already in device memory and on the correct device, then + no copy is performed and the original object is returned. + + Args: + device (int): The destination device id. Defaults to the current device. + non_blocking (bool): If ``True`` and the source is in pinned memory, + the copy will be asynchronous with respect to the host. Otherwise, + the argument has no effect. + """ + # There should be a judgment related to storage device and a judgment related to storage type, + # but it depends on the extended function, so this part is temporarily omitted in the automatic generation. + device_idx = _normalization_device(custom_backend_name, device) + + if getattr(self, f"is_{custom_backend_name}"): + # storage has already on expected device. + if self.get_device() == device_idx: + return self + # For sparse storage, custom need to extend the implementation by themselves. + if self.is_sparse: + raise RuntimeError( + f"Can not support a sparse storage move to {custom_backend_name} backend" + ) + # create untyped_storage and copy data + untyped_storage = torch.UntypedStorage( + self.size(), device=torch.device(f"{custom_backend_name}:{device_idx}") + ) + untyped_storage.copy_(self, non_blocking) + return untyped_storage + + _check_register_once(torch.storage._StorageBase, custom_backend_name) + setattr(torch.storage._StorageBase, custom_backend_name, wrap_storage_to) + + # Register the corresponding attribute for the TypedStorage class. + # When the TypedStorage class is removed, the registration is also removed. + + @property # type: ignore[misc] + def wrap_typed_storage_backend(self: torch.storage.TypedStorage) -> bool: + torch.storage._warn_typed_storage_removal() + return self._untyped_storage.device.type == custom_backend_name + + _check_register_once(torch.TypedStorage, f"is_{custom_backend_name}") + setattr( + torch.storage.TypedStorage, + f"is_{custom_backend_name}", + wrap_typed_storage_backend, + ) + + def wrap_typed_storage_to( + self: torch.storage.TypedStorage, device=None, non_blocking=False, **kwargs + ) -> torch.storage.TypedStorage: + torch.storage._warn_typed_storage_removal() + if unsupported_dtype and self.dtype in unsupported_dtype: + raise RuntimeError( + f"Cannot create {custom_backend_name} storage " + f"as {self.dtype} dtype is not supported by this backend" + ) + custom_backend_storage: torch.UntypedStorage = getattr( + self._untyped_storage, custom_backend_name + )(device, non_blocking, **kwargs) + return self._new_wrapped_storage(custom_backend_storage) + + _check_register_once(torch.TypedStorage, custom_backend_name) + setattr(torch.TypedStorage, custom_backend_name, wrap_typed_storage_to) + + +def generate_methods_for_privateuse1_backend( + for_tensor: bool = True, + for_module: bool = True, + for_packed_sequence: bool = True, + for_storage: bool = False, + unsupported_dtype: list[torch.dtype] | None = None, +) -> None: + r""" + Automatically generate attributes and methods for the custom backend after rename privateuse1 backend. + + In the default scenario, storage-related methods will not be generated automatically. + + When you implement kernels for various torch operations, and register them to the PrivateUse1 dispatch key. + And call the function torch.rename_privateuse1_backend("foo") to rename your backend name. + At this point, you can easily register specific methods and attributes by calling this function. + Just like torch.Tensor.foo(), torch.Tensor.is_foo, torch.Storage.foo(), torch.Storage.is_foo. + + Note: We recommend you use generic functions (check devices are equal or to(device=)). + We provide these methods for convenience only and they will be "monkey patched" onto the objects + and so will not be properly typed. For Storage methods generate, if you need to support sparse data storage, + you need to extend the implementation yourself. + + Args: + for_tensor (bool): whether register related methods for torch.Tensor class. + for_module (bool): whether register related methods for torch.nn.Module class. + for_storage (bool): whether register related methods for torch.Storage class. + unsupported_dtype (List[torch.dtype]): takes effect only when the storage method needs to be generated, + indicating that the storage does not support the torch.dtype type. + + Example:: + + >>> # xdoctest: +SKIP("failing") + >>> torch.utils.rename_privateuse1_backend("foo") + >>> torch.utils.generate_methods_for_privateuse1_backend() + # Then automatically generate backend-related attributes and methods. + >>> a = torch.tensor(2).foo() + >>> a.is_foo + >>> hasattr(torch.nn.Module, 'foo') + """ + custom_backend_name = _get_privateuse1_backend_name() + + if for_tensor: + _generate_tensor_methods_for_privateuse1_backend(custom_backend_name) + + if for_module: + _generate_module_methods_for_privateuse1_backend(custom_backend_name) + + if for_storage: + _generate_storage_methods_for_privateuse1_backend( + custom_backend_name, unsupported_dtype + ) + + if for_packed_sequence: + _generate_packed_sequence_methods_for_privateuse1_backend(custom_backend_name) + + +def _get_custom_mod_func(func_name: str): + r""" + Return the func named `func_name` defined in custom device module. If not defined, + return `None`. And the func is registered with `torch.utils.rename_privateuse1_backend('foo')` + and `torch._register_device_module('foo', BackendModule)`. + If the custom device module or the func is not defined, it will give warning or error message. + Args: + func_name (str): return the callable func named func_name defined in custom device module. + Example:: + class DummyfooModule: + @staticmethod + def is_available(): + return True + @staticmethod + def func_name(*args, **kwargs): + .... + torch.utils.rename_privateuse1_backend("foo") + torch._register_device_module("foo", DummyfooModule) + foo_is_available_func = torch.utils.backend_registration._get_custom_mod_func("is_available") + if foo_is_available_func: + foo_is_available = foo_is_available_func() + func_ = torch.utils.backend_registration._get_custom_mod_func("func_name") + if func_: + result = func_(*args, **kwargs) + Attention: This function is not meant to be used directly by users, which is why + it is marked as private. It is a convenience function for backend implementers to + more easily call the hooks into their backend extensions. + """ + if not isinstance(func_name, str): + raise AssertionError(f"func_name must be `str`, but got `{type(func_name)}`.") + backend_name = _get_privateuse1_backend_name() + custom_device_mod = getattr(torch, backend_name, None) + function = getattr(custom_device_mod, func_name, None) + if custom_device_mod is None or function is None: + message = f"Try to call torch.{backend_name}.{func_name}. The backend must register a custom backend " + message += f"module with `torch._register_device_module('{backend_name}', BackendModule)`. And " + message += f"BackendModule needs to have the following API's:\n `{func_name}(*args, **kwargs)`. \n" + raise RuntimeError(message) + return function + + +class _DummyBackendModule: + def is_initialized(self) -> bool: + return True + + def is_available(self) -> bool: + return True + + def current_device(self) -> int: + return 0 + + def _is_in_bad_fork(self) -> bool: + return False + + def manual_seed_all(self, seed: int) -> None: + pass + + def device_count(self) -> int: + return 1 + + +class _DummyPrivateUse1Hook(torch._C._acc.PrivateUse1Hooks): + def is_available(self) -> bool: + return True + + def has_primary_context(self, dev_id) -> bool: + return True + + def is_built(self) -> bool: + return True + + +class _DummyDeviceGuard(torch._C._acc.DeviceGuard): + def type_(self): + return torch._C._autograd.DeviceType.PrivateUse1 + + +def _setup_privateuseone_for_python_backend( + rename=None, backend_module=None, hook=None, device_guard=None +) -> None: + """This function will prepare the PrivateUse1 dispatch key to be used as a python backend. + + WARNING: this API is experimental and might change without notice. + + Formally, this registers things that Pytorch expects a registered backend + in C++ to have: including device guards, hooks, and backend modules and what not. + + after this call, one can use `torch.library` to write Ops for this dispatch key + and expect it to behave like a backend registered in C++. + + See the unit test at test/test_privateuseone_python_backend.py for more details. + + Args: + rename: str | None, if passed in, we will rename privateuseone backend to + the name given. + backend_module: object | None, if passed in None, we will use DummyBackendModule + hook: object | None, if passed in None, we will use DummyPrivateUse1Hook + device_guard: object | None, if passed in None, we will use DummyDeviceGuard + """ + # NOTE: the ordering of which these functions are called is important. + if rename is not None: + torch.utils.rename_privateuse1_backend(rename) + else: + rename = "privateuseone" + torch.utils.generate_methods_for_privateuse1_backend() + if backend_module is None: + backend_module = _DummyBackendModule() + if hook is None: + hook = _DummyPrivateUse1Hook() + if device_guard is None: + device_guard = _DummyDeviceGuard() + torch._register_device_module(rename, backend_module) + torch._C._acc.register_python_privateuseone_hook(hook) + torch._C._acc.register_python_privateuseone_device_guard(device_guard) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/bundled_inputs.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/bundled_inputs.py new file mode 100644 index 0000000000000000000000000000000000000000..e91129a03864b4ef6547702a349ca79410fe2b4a --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/bundled_inputs.py @@ -0,0 +1,472 @@ +#!/usr/bin/env python3 +# mypy: allow-untyped-defs +from typing import Any, TypeVar, NamedTuple +from collections.abc import Callable, Sequence +import textwrap +import torch +from torch._C import TupleType, ListType +from torch.jit._recursive import wrap_cpp_module + + +T = TypeVar("T") + +MAX_RAW_TENSOR_SIZE = 16 + +class InflatableArg(NamedTuple): + """Helper type for bundled inputs. + + 'value' is the compressed/deflated input that is stored in the model. Value + must be of the same type as the argument to the function that it is a deflated + input for. + + 'fmt' is a formatable code string that is executed to inflate the compressed data into + the appropriate input. It can use 'value' as an input to the format str. It must result + in a value of the same type as 'value'. + + 'fmt_fn' is a formatable function code string that is executed to inflate the compressed + data into the appropriate input. It must result in a value of the same type as 'value'. + The function name should be the formatable part of the string. + + Note: Only top level InflatableArgs can be inflated. i.e. you cannot place + an inflatable arg inside of some other structure. You should instead create + an inflatable arg such that the fmt code string returns the full structure + of your input. + """ + + value: Any + fmt: str = "{}" + fmt_fn: str = "" + + +def bundle_inputs( + model: torch.jit.ScriptModule, + inputs: Sequence[tuple[Any, ...]] | None | dict[Callable, Sequence[tuple[Any, ...]] | None], + info: list[str] | dict[Callable, list[str]] | None = None, + *, + _receive_inflate_expr: list[str] | None = None, +) -> torch.jit.ScriptModule: + """Create and return a copy of the specified model with inputs attached. + + The original model is not mutated or changed in any way. + + Models with bundled inputs can be invoked in a uniform manner by + benchmarking and code coverage tools. + + If inputs is passed in as a list then the inputs will be bundled for 'forward'. + If inputs is instead passed in as a map then all the methods specified in the map + will have their corresponding inputs bundled. Info should match watchever type is + chosen for the inputs. + + The returned model will support the following methods: + + `get_all_bundled_inputs_for_() -> List[Tuple[Any, ...]]` + Returns a list of tuples suitable for passing to the model like + `for inp in model.get_all_bundled_inputs_for_foo(): model.foo(*inp)` + + `get_bundled_inputs_functions_and_info() -> Dict[str, Dict[str: List[str]]]` + Returns a dictionary mapping function names to a metadata dictionary. + This nested dictionary maps preset strings like: + 'get_inputs_function_name' -> the name of a function attribute in this model that can be + run to get back a list of inputs corresponding to that function. + 'info' -> the user provided extra information about the bundled inputs + + If forward has bundled inputs then these following functions will also be defined on the returned module: + + `get_all_bundled_inputs() -> List[Tuple[Any, ...]]` + Returns a list of tuples suitable for passing to the model like + `for inp in model.get_all_bundled_inputs(): model(*inp)` + + `get_num_bundled_inputs() -> int` + Equivalent to `len(model.get_all_bundled_inputs())`, + but slightly easier to call from C++. + + Inputs can be specified in one of two ways: + + - The model can define `_generate_bundled_inputs_for_`. + If the user chooses this method inputs[] should map to None + + - The `inputs` argument to this function can be a dictionary mapping functions to a + list of inputs, of the same form that will be returned by get_all_bundled_inputs_for_. + Alternatively if only bundling inputs for forward the map can be omitted and a singular list of inputs + can be provided instead. + + The type of the inputs is List[Tuple[Any, ...]]. The outer list corresponds with a + list of inputs, the inner tuple is the list of args that together make up one input. + For inputs of functions that take one arg, this will be a tuple of length one. The Any, ... + is the actual data that makes up the args, e.g. a tensor. + + Info is an optional parameter that maps functions to a list of strings providing extra information about that + function's bundled inputs. Alternatively if only bundling inputs for forward the map can be omitted and + a singular list of information can be provided instead. This could be descriptions, expected outputs, etc. + - Ex: info={model.forward : ['man eating icecream', 'an airplane', 'a dog']} + + This function will attempt to optimize arguments so that (e.g.) + arguments like `torch.zeros(1000)` will be represented compactly. + Only top-level arguments will be optimized. + Tensors in lists or tuples will not. + """ + if not isinstance(model, torch.jit.ScriptModule): + raise Exception("Only ScriptModule is supported.") # noqa: TRY002 + + ignored_methods, ignored_attrs = _get_bundled_inputs_attributes_and_methods(model) + clone = torch._C._hack_do_not_use_clone_module_with_class( # type: ignore[attr-defined] + model._c, + ignored_methods, + ignored_attrs, + ) + + # The above cloning function returns a torch._C.scriptmodule and we need a torch.jit.scriptmodule. + # Fortunately there is a function in _recursive that does exactly that conversion. + cloned_module = wrap_cpp_module(clone) + if isinstance(inputs, dict): + if not isinstance(info, dict) and info is not None: + raise AssertionError("If inputs is a dict, info must be a dict or None") + augment_many_model_functions_with_bundled_inputs(cloned_module, inputs, _receive_inflate_expr, info) + else: + if not isinstance(info, list) and info is not None: + raise AssertionError("If inputs is a list, info must be a list or None") + augment_model_with_bundled_inputs(cloned_module, inputs, _receive_inflate_expr, info) + return cloned_module + +def augment_model_with_bundled_inputs( + model: torch.jit.ScriptModule, + inputs: Sequence[tuple[Any, ...]] | None = None, + _receive_inflate_expr: list[str] | None = None, # For debugging. + info: list[str] | None = None, # Optional argument to provide info about forward or its inputs + skip_size_check=False, +) -> None: + """Add bundled sample inputs to a model for the forward function. + + Models with bundled inputs can be invoked in a uniform manner by + benchmarking and code coverage tools. + + Augmented models will support the following methods: + + `get_all_bundled_inputs() -> List[Tuple[Any, ...]]` + Returns a list of tuples suitable for passing to the model like + `for inp in model.get_all_bundled_inputs(): model(*inp)` + + `get_num_bundled_inputs() -> int` + Equivalent to `len(model.get_all_bundled_inputs())`, + but slightly easier to call from C++. + + `get_bundled_inputs_functions_and_info() -> Dict[str, Dict[str: List[str]]]` + Returns a dictionary mapping function names to a metadata dictionary. + This nested dictionary maps preset strings like: + 'get_inputs_function_name' -> the name of a function attribute in this model that can be + run to get back a list of inputs corresponding to that function. + 'info' -> the user provided extra information about the bundled inputs + + Inputs can be specified in one of two ways: + + - The model can define `_generate_bundled_inputs_for_forward`. + If the user chooses this method inputs should be None + + - `inputs` is a list of inputs of form List[Tuple[Any, ...]]. A list of tuples where the elements + of each tuple are the args that make up one input. + """ + if not isinstance(model, torch.jit.ScriptModule): + raise Exception("Only ScriptModule is supported.") # noqa: TRY002 + + forward: Callable = model.forward + + # Sometimes forward won't have a name attached so just in case + if not hasattr(forward, "__name__"): + forward.__name__ = 'forward' + augment_many_model_functions_with_bundled_inputs( + model, + inputs={forward : inputs}, + _receive_inflate_expr=_receive_inflate_expr, + info={forward : info} if info else None, + skip_size_check=skip_size_check, + ) + + +def augment_many_model_functions_with_bundled_inputs( + model: torch.jit.ScriptModule, + inputs: dict[Callable, Sequence[tuple[Any, ...]] | None], + _receive_inflate_expr: list[str] | None = None, # For debugging. + info: dict[Callable, list[str]] | None = None, # Optional argument to provide info about the function or its inputs + skip_size_check=False, +) -> None: + """Add bundled sample inputs to a model for an arbitrary list of public functions. + + Models with bundled inputs can be invoked in a uniform manner by + benchmarking and code coverage tools. + + Augmented models will support the following methods: + + `get_all_bundled_inputs_for_() -> List[Tuple[Any, ...]]` + Returns a list of tuples suitable for passing to the model like + `for inp in model.get_all_bundled_inputs_for_foo(): model.foo(*inp)` + + `get_bundled_inputs_functions_and_info() -> Dict[str, Dict[str: List[str]]]` + Returns a dictionary mapping function names to a metadata dictionary. + This nested dictionary maps preset strings like: + 'get_inputs_function_name' -> the name of a function attribute in this model that can be + run to get back a list of inputs corresponding to that function. + 'info' -> the user provided extra information about the bundled inputs + + If forward has bundled inputs then these following functions are also defined: + + `get_all_bundled_inputs() -> List[Tuple[Any, ...]]` + Returns a list of tuples suitable for passing to the model like + `for inp in model.get_all_bundled_inputs(): model(*inp)` + + `get_num_bundled_inputs() -> int` + Equivalent to `len(model.get_all_bundled_inputs())`, + but slightly easier to call from C++. + + Inputs can be specified in one of two ways: + + - The model can define `_generate_bundled_inputs_for_`. + If the user chooses this method inputs[] should map to None + + - The `inputs` argument to this function can be a dictionary mapping functions to a + list of inputs, of the same form that will be returned by get_all_bundled_inputs_for_. + The type of the inputs is List[Tuple[Any, ...]]. The outer list corresponds with a + list of inputs, the inner tuple is the list of args that together make up one input. + For inputs of functions that take one arg, this will be a tuple of length one. The Any, ... + is the actual data that makes up the args, e.g. a tensor. + + Info is an optional parameter that maps functions to a list of strings providing extra information about that + function's bundled inputs. This could be descriptions, expected outputs, etc. + - Ex: info={model.forward : ['man eating icecream', 'an airplane', 'a dog']} + + This function will attempt to optimize arguments so that (e.g.) + arguments like `torch.zeros(1000)` will be represented compactly. + Only top-level arguments will be optimized. + Tensors in lists or tuples will not. + """ + if not isinstance(model, torch.jit.ScriptModule): + raise Exception("Only ScriptModule is supported.") # noqa: TRY002 + + if not inputs: + raise Exception("Please provide inputs for at least 1 function") # noqa: TRY002 + + if hasattr(model, "get_all_bundled_inputs") or hasattr(model, "get_bundled_inputs_functions_and_info"): + raise Exception( # noqa: TRY002 + "Models can only be augmented with bundled inputs once. " + "This Model seems to have already been augmented with " + "bundled inputs. Please start afresh with one that " + "doesn't have bundled inputs.", + ) + + get_bundled_inputs_functions_and_info_template = "" + + for function, input_list in inputs.items(): + if hasattr(function, "__name__"): + function_name = function.__name__ + else: + if hasattr(function, "name"): + function_name = function.name # type: ignore[attr-defined] + else: + raise Exception( # noqa: TRY002 + 'At least one of your functions has no attribute name please ensure all have one. m.foo.name = "foo"') + + + if input_list is not None and not isinstance(input_list, Sequence): + raise TypeError(f"Error inputs for function {function_name} is not a Sequence") + + function_arg_types = [arg.type for arg in function.schema.arguments[1:]] # type: ignore[attr-defined] + deflated_inputs_type: ListType = ListType(TupleType(function_arg_types)) + model._c._register_attribute(f"_bundled_inputs_deflated_{function_name}", deflated_inputs_type, []) + + if hasattr(model, "_generate_bundled_inputs_for_" + function_name): + if input_list is not None: + raise Exception( # noqa: TRY002 + f"inputs[{function_name}] is not None, but _generate_bundled_inputs_for_{function_name} is already defined" + ) + # Model author already defined _generate_bundled_inputs_for_. + elif input_list is None or len(input_list) == 0: + raise Exception( # noqa: TRY002 + f"inputs for {function_name} must be specified if " + f"_generate_bundled_inputs_for_{function_name} is not already defined" + ) + else: + # Iterate over the inputs and args in each input. + # Accumulate `deflated_inputs` as (possibly) compressed values + # and `parts` to be joined into the expression that unpacks them. + deflated_inputs = [] + parts = [] + for inp_idx, args in enumerate(input_list): + if not isinstance(args, tuple) and not isinstance(args, list): # type: ignore[arg-type] + raise TypeError( + f"Error bundled input for function {function_name} idx: {inp_idx} is not a Tuple or a List" + ) + deflated_args = [] + parts.append("(") + for arg_idx, arg in enumerate(args): + inflate_helper_fn_name = _get_inflate_helper_fn_name(arg_idx, inp_idx, function_name) + deflated, inflater, helper_definition = _inflate_expr( + arg, + f"deflated[{inp_idx}][{arg_idx}]", + inflate_helper_fn_name, + skip_size_check=skip_size_check, + ) + deflated_args.append(deflated) + parts.append(f" {inflater},") + if helper_definition: + model.define(textwrap.dedent(helper_definition)) + deflated_inputs.append(tuple(deflated_args)) + parts.append("),") + parts.append("") + expr = "\n".join(parts) + + # Back-channel return this expr for debugging. + if _receive_inflate_expr is not None: + _receive_inflate_expr.append(expr) + setattr(model, f"_bundled_inputs_deflated_{function_name}", deflated_inputs) + definition = textwrap.dedent(""" + def _generate_bundled_inputs_for_{name}(self): + deflated = self._bundled_inputs_deflated_{name} + return [ + {expr} + ] + """).format(expr=expr, name=function_name) + model.define(definition) + + # Define get_all_bundled_inputs_for_ that caches the generated inputs. + model.define(textwrap.dedent(""" + def get_all_bundled_inputs_for_{name}(self): + all_inputs = self._generate_bundled_inputs_for_{name}() + assert all_inputs is not None + return all_inputs + """).format(name=function_name)) + + # Add to the high level helper methods + inputs_info = repr(info[function]) if info and function in info else '[]' + get_bundled_inputs_functions_and_info_template += f""" + temp_dict : Dict[str,List[str]] = {{}} + info: List[str] = {inputs_info} + + temp_dict['info'] = info + temp_dict['get_inputs_function_name'] = ['get_all_bundled_inputs_for_{function_name}'] + all_inputs['{function_name}'] = temp_dict + """ + + # To ensure backwards compatibility and a streamlined api for forward these wrappers are provided + if function_name == 'forward': + model.define(textwrap.dedent(""" + def get_all_bundled_inputs(self): + return self.get_all_bundled_inputs_for_forward() + """)) + model.define(textwrap.dedent(""" + def get_num_bundled_inputs(self): + return len(self.get_all_bundled_inputs_for_forward()) + """)) + + # Define some high level helper methods that act on all bundled inputs + model.define(textwrap.dedent(f""" + def get_bundled_inputs_functions_and_info(self): + all_inputs : Dict[str, Dict[str,List[str]]] = {{}} + {get_bundled_inputs_functions_and_info_template} + return all_inputs + """)) + +def _inflate_expr( + arg: T, ref: str, inflate_helper_fn_name: str, skip_size_check: bool = False +) -> tuple[T | torch.Tensor, str, str | None]: + # Allow custom inflation expressions any object. + # For example, calling custom image-decoding ops. + # Or just use "{}" as the format string to ignore size limits. + if isinstance(arg, InflatableArg): + if arg.fmt_fn: + if arg.fmt not in ["{}", ""]: + raise Exception( # noqa: TRY002 + f"Bundled input argument at position '{ref}' has " + f"both arg.fmt_fn => \n{arg.fmt_fn} " + f"\n and arg.fmt => {arg.fmt}. " + "Please choose `arg.fmt` if the deflater is straightforward or " + "`arg.fmt_fn` if you need a function." + ) + + helper_definition = arg.fmt_fn.format(inflate_helper_fn_name) + expr = f"self.{inflate_helper_fn_name}({ref})" + + return arg.value, expr, helper_definition + else: + return arg.value, arg.fmt.format(ref), None + + if isinstance(arg, torch.Tensor): + # Small-storage tensors can just be saved directly. + if arg._typed_storage().size() <= MAX_RAW_TENSOR_SIZE or skip_size_check: + return arg, ref, None + # Small contiguous tensors can be cloned to have small storage. + # TODO: Should we do this even for non-contiguous tensors? + if arg.is_contiguous() and arg.numel() <= MAX_RAW_TENSOR_SIZE: + return arg.clone(), ref, None + # Example inputs commonly come from torch.zeros, torch.ones, or torch.full. + # These can be represented compactly. + for fmt in [torch.contiguous_format, torch.channels_last]: + if arg.is_contiguous(memory_format=fmt) and (arg == arg.flatten()[0]).all().item(): + return (arg.flatten()[0].clone().expand(*arg.size()), + f"{ref}.contiguous(memory_format={fmt})", None) + # Prevent big tensors from being bundled by default. + # TODO: Provide more useful diagnostics. + raise Exception( # noqa: TRY002 + f"Bundled input argument at position '{ref}' is " + f"a tensor with storage size {arg._typed_storage().size()}. " + f"You probably don't want to bundle this as an input. " + ) + else: + return arg, ref, None + +def _get_bundled_inputs_attributes_and_methods(script_module: torch.jit.ScriptModule) -> tuple[list[str], list[str]]: + methods: list[str] = [] + attributes: list[str] = [] + + # Has bundled inputs for forward + if hasattr(script_module, 'get_all_bundled_inputs'): + methods.append('get_all_bundled_inputs') + methods.append('get_num_bundled_inputs') + methods.append('run_on_bundled_input') + + if hasattr(script_module, 'get_bundled_inputs_functions_and_info'): + methods.append('get_bundled_inputs_functions_and_info') + all_info = script_module.get_bundled_inputs_functions_and_info() + for function_name in all_info: + methods.append("get_all_bundled_inputs_for_" + function_name) + methods.append("_generate_bundled_inputs_for_" + function_name) + attributes.append("_bundled_inputs_deflated_" + function_name) + + bundled_inputs_fn = getattr( + script_module, + f"get_all_bundled_inputs_for_{function_name}" + ) + num_bundled_inputs: int = len(bundled_inputs_fn()) + + # Check inflate helper functions for each function, argument and bundled input + func = getattr(script_module, function_name) + for arg_idx in range(len(func.schema.arguments) - 1): + for input_idx in range(num_bundled_inputs): + helper_fn_name = _get_inflate_helper_fn_name( + arg_idx=arg_idx, + input_idx=input_idx, + function_name=function_name + ) + # if the arg has an InflatableArg with fmt_fn, add the helper function name + if hasattr(script_module, helper_fn_name): + methods.append(helper_fn_name) + + return (methods, attributes) + + +def _get_inflate_helper_fn_name( + arg_idx: int, + input_idx: int, + function_name: str, +) -> str: + return f"_inflate_helper_for_{function_name}_input_{input_idx}_arg_{arg_idx}" + + + +def bundle_randn(*size, dtype=None): + """Generate a tensor that will be inflated with torch.randn.""" + stub = torch.zeros(1, dtype=dtype).expand(*size) + return InflatableArg(value=stub, fmt="torch.randn_like({})") + + +def bundle_large_tensor(t): + """Wrap a tensor to allow bundling regardless of size.""" + return InflatableArg(value=t, fmt="{}") diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/checkpoint.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/checkpoint.py new file mode 100644 index 0000000000000000000000000000000000000000..da74334025111c7edb932f99e983aaefbb1c0344 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/checkpoint.py @@ -0,0 +1,1669 @@ +# mypy: allow-untyped-defs +import contextlib +import platform +import uuid +import warnings +import weakref +from collections import defaultdict +from typing import * # noqa: F403 +import enum +from weakref import ReferenceType + +import torch +import torch.fx.traceback as fx_traceback +from torch.utils._pytree import tree_map +from torch.testing._internal.logging_tensor import capture_logs, LoggingTensorMode +from torch.utils._python_dispatch import TorchDispatchMode +from typing import NoReturn + +__all__ = [ + "checkpoint", + "checkpoint_sequential", + "CheckpointError", + "CheckpointFunction", + "check_backward_validity", + "detach_variable", + "get_device_states", + "set_device_states", + "noop_context_fn", + "set_checkpoint_early_stop", + "DefaultDeviceType", + "set_checkpoint_debug_enabled", + "CheckpointPolicy", + "SelectiveCheckpointContext", + "create_selective_checkpoint_contexts", + "SAC_IGNORED_OPS", + "GraphExecGroup", +] + +_DEFAULT_DETERMINISM_MODE = "default" + +_checkpoint_debug_enabled: Optional[bool] = None + + +@contextlib.contextmanager +def set_checkpoint_debug_enabled(enabled: Optional[bool]): + """ + Context manager that sets whether checkpoint should print additional debug + information when running. See the ``debug`` flag for + :func:`~torch.utils.checkpoint.checkpoint` for more information. Note that + when set, this context manager overrides the value of ``debug`` passed to + checkpoint. To defer to the local setting, pass ``None`` to this context. + + Args: + enabled (bool): Whether checkpoint should print debug information. + Default is 'None'. + """ + global _checkpoint_debug_enabled + try: + prev = _checkpoint_debug_enabled + _checkpoint_debug_enabled = enabled + yield + finally: + _checkpoint_debug_enabled = prev + + +def detach_variable(inputs: Tuple[Any, ...]) -> Tuple[torch.Tensor, ...]: + if isinstance(inputs, tuple): + out = [] + for inp in inputs: + if not isinstance(inp, torch.Tensor): + out.append(inp) + continue + + x = inp.detach() + x.requires_grad = inp.requires_grad + out.append(x) + return tuple(out) + else: + raise RuntimeError( + "Only tuple of tensors is supported. Got Unsupported input type: ", + type(inputs).__name__, + ) + + +def check_backward_validity(inputs: Iterable[Any]) -> None: + if not any(inp.requires_grad for inp in inputs if isinstance(inp, torch.Tensor)): + warnings.warn( + "None of the inputs have requires_grad=True. Gradients will be None", stacklevel=2 + ) + + +def _get_device_module(device="cuda"): + if device == "meta": + return torch.device("meta") + device_module = getattr(torch, device) + return device_module + + +class DefaultDeviceType: + r""" + A class that manages the default device type for checkpointing. + + If no non-CPU tensors are present, the default device type will + be used. The default value is 'cuda'. The device type is used in + the checkpointing process when determining which device states + to save and restore for recomputation. + """ + + _default_device_type: Optional[str] = None + + @staticmethod + def set_device_type(device: str = "cuda") -> None: + """ + Set the default device type for checkpointing. + + Args: + device (str): The device type to be set as default. Default is 'cuda'. + """ + DefaultDeviceType._default_device_type = device + + @staticmethod + def get_device_type() -> str: + """ + Get the current default device type for checkpointing. + + Returns: + str: The current default device type. + """ + if not DefaultDeviceType._default_device_type: + DefaultDeviceType._default_device_type = acc.type if (acc := torch.accelerator.current_accelerator(True)) else "cpu" + + return DefaultDeviceType._default_device_type + + +def _infer_device_type(*args): + device_types = [] + + def add_device_types(arg) -> None: + nonlocal device_types + if isinstance(arg, torch.Tensor) and arg.device.type != "cpu": + device_types.append(arg.device.type) + tree_map(add_device_types, args) + + device_types_set = set(device_types) + if len(device_types_set) > 1: + warnings.warn( + "Tensor arguments, excluding CPU tensors, are detected on at least two types of devices. " + "Device state will only be saved for devices of a single device type, and the remaining " + "devices will be ignored. Consequently, if any checkpointed functions involve randomness, " + "this may result in incorrect gradients. (Note that if CUDA devices are among the devices " + "detected, it will be prioritized; otherwise, the first device encountered will be selected.)" + f"\nDevice types: {sorted(device_types_set)} first device type: {device_types[0]}", stacklevel=2 + ) + if len(device_types) == 0: + return DefaultDeviceType.get_device_type() + elif "cuda" in device_types_set: + return "cuda" + else: + return device_types[0] + + +# We can't know if the run_fn will internally move some args to different devices, +# which would require logic to preserve rng states for those devices as well. +# We could paranoically stash and restore ALL the rng states for all visible devices, +# but that seems very wasteful for most cases. Compromise: Stash the RNG state for +# the device of all Tensor args. +# +# To consider: maybe get_device_states and set_device_states should reside in torch/random.py? +def get_device_states(*args) -> Tuple[List[int], List[torch.Tensor]]: + # This will not error out if "arg" is a CPU tensor or a non-tensor type because + # the conditionals short-circuit. + fwd_device_ids = [] + + def add_device_ids(arg) -> None: + nonlocal fwd_device_ids + if isinstance(arg, torch.Tensor) and arg.device.type not in {"cpu", "meta"}: + fwd_device_ids.append(arg.get_device()) + tree_map(add_device_ids, args) + + fwd_device_states = [] + device_module = _get_device_module(_infer_device_type(*args)) + for device_id in fwd_device_ids: + with device_module.device(device_id): + fwd_device_states.append(device_module.get_rng_state()) + + return fwd_device_ids, fwd_device_states + + +def set_device_states(devices, states, *, device_type=None) -> None: + """Sets random number generator states for the specified devices. + + Args: + devices: Device ids to set states for. + states: States to set. + device_type: ``device_type`` of the devices to set states for. Default + is the device returned by a call to ``DefaultDeviceType.get_device_type()``, + which is ``cuda`` if not changed by calling ``DefaultDeviceType::set_device_type()``. + """ + if device_type is None: + device_type = DefaultDeviceType.get_device_type() + if device_type == "meta": + return + device_module = _get_device_module(device_type) + for device, state in zip(devices, states, strict=False): + with device_module.device(device): + device_module.set_rng_state(state) + + +def _get_autocast_kwargs(device_type="cuda"): + if torch.amp.is_autocast_available(device_type): + device_autocast_kwargs = { + "enabled": torch.is_autocast_enabled(device_type), + "dtype": torch.get_autocast_dtype(device_type), + "cache_enabled": torch.is_autocast_cache_enabled(), + } + else: + device_autocast_kwargs = None + + cpu_autocast_kwargs = { + "enabled": torch.is_autocast_enabled('cpu'), + "dtype": torch.get_autocast_dtype('cpu'), + "cache_enabled": torch.is_autocast_cache_enabled(), + } + + return device_autocast_kwargs, cpu_autocast_kwargs + + +class CheckpointFunction(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(ctx, run_function, preserve_rng_state, *args): + check_backward_validity(args) + ctx.run_function = run_function + ctx.preserve_rng_state = preserve_rng_state + # Accommodates the (remote) possibility that autocast is enabled for cpu AND gpu. + ctx.device_type = _infer_device_type(*args) + ctx.device_autocast_kwargs, ctx.cpu_autocast_kwargs = _get_autocast_kwargs( + ctx.device_type + ) + if preserve_rng_state: + ctx.fwd_cpu_state = torch.get_rng_state() + # Don't eagerly initialize the cuda context by accident. + # (If the user intends that the context is initialized later, within their + # run_function, we SHOULD actually stash the cuda state here. Unfortunately, + # we have no way to anticipate this will happen before we run the function.) + ctx.had_device_in_fwd = False + device_module = _get_device_module(ctx.device_type) + if getattr(device_module, "_initialized", False): + ctx.had_device_in_fwd = True + ctx.fwd_devices, ctx.fwd_device_states = get_device_states(*args) + + # Save non-tensor inputs in ctx, keep a placeholder None for tensors + # to be filled out during the backward. + ctx.inputs = [] + ctx.tensor_indices = [] + tensor_inputs = [] + for i, arg in enumerate(args): + if torch.is_tensor(arg): + tensor_inputs.append(arg) + ctx.tensor_indices.append(i) + ctx.inputs.append(None) + else: + ctx.inputs.append(arg) + + ctx.save_for_backward(*tensor_inputs) + + with torch.no_grad(): + outputs = run_function(*args) + return outputs + + @staticmethod + def backward(ctx, *args): + if not torch.autograd._is_checkpoint_valid(): + raise RuntimeError( + "When use_reentrant=True, torch.utils.checkpoint is incompatible" + " with .grad() or passing an `inputs` parameter to .backward()." + " To resolve this error, you can either set use_reentrant=False," + " or call .backward() without passing the `inputs` argument." + ) + # Copy the list to avoid modifying original list. + inputs = list(ctx.inputs) + tensor_indices = ctx.tensor_indices + tensors = ctx.saved_tensors + + # Fill in inputs with appropriate saved tensors. + for i, idx in enumerate(tensor_indices): + inputs[idx] = tensors[i] + + # Stash the surrounding rng state, and mimic the state that was + # present at this time during forward. Restore the surrounding state + # when we're done. + rng_devices = [] + if ctx.preserve_rng_state and ctx.had_device_in_fwd: + rng_devices = ctx.fwd_devices + with torch.random.fork_rng( + devices=rng_devices, enabled=ctx.preserve_rng_state, device_type=ctx.device_type + ): + if ctx.preserve_rng_state: + torch.set_rng_state(ctx.fwd_cpu_state) + if ctx.had_device_in_fwd: + set_device_states(ctx.fwd_devices, ctx.fwd_device_states, device_type=ctx.device_type) + detached_inputs = detach_variable(tuple(inputs)) + + device_autocast_ctx = torch.amp.autocast( + device_type=ctx.device_type, **ctx.device_autocast_kwargs + ) if torch.amp.is_autocast_available(ctx.device_type) else contextlib.nullcontext() + with torch.enable_grad(), device_autocast_ctx, torch.amp.autocast("cpu", **ctx.cpu_autocast_kwargs): # type: ignore[attr-defined] + outputs = ctx.run_function(*detached_inputs) + + if isinstance(outputs, torch.Tensor): + outputs = (outputs,) + + # run backward() with only tensor that requires grad + outputs_with_grad = [] + args_with_grad = [] + for i in range(len(outputs)): + if torch.is_tensor(outputs[i]) and outputs[i].requires_grad: + outputs_with_grad.append(outputs[i]) + args_with_grad.append(args[i]) + if len(outputs_with_grad) == 0: + raise RuntimeError( + "none of output has requires_grad=True," + " this checkpoint() is not necessary" + ) + torch.autograd.backward(outputs_with_grad, args_with_grad) + grads = tuple( + inp.grad if isinstance(inp, torch.Tensor) else None + for inp in detached_inputs + ) + + return (None, None) + grads + + +def noop_context_fn(): + return contextlib.nullcontext(), contextlib.nullcontext() + +# Note: [torch.compile and checkpoint] +# TorchDynamo does not step inside utils.checkpoint function. The flow +# looks likes this +# 1) TorchDynamo tries to wrap utils.checkpoint in a HigherOrderOp by +# speculatively checking if the forward function is safe to trace. +# 2) If yes, then Dynamo-generated Fx graph has the wrapped higher +# order op. As a result, TorchDynamo does not look inside utils.checkpoint. +# 3) If not, then TorchDynamo falls back to eager by performing a graph +# break. And here, the following disable wrapper ensures that +# TorchDynamo does not trigger again on the frames created by +# utils.checkpoint innards. +@torch._disable_dynamo +def checkpoint( + function, + *args, + use_reentrant: Optional[bool] = None, + context_fn: Callable[[], Tuple[ContextManager, ContextManager]] = noop_context_fn, + determinism_check: str = _DEFAULT_DETERMINISM_MODE, + debug: bool = False, + early_stop: bool = True, + **kwargs +): + r"""Checkpoint a model or part of the model. + + Activation checkpointing is a technique that trades compute for memory. + By default, tensors computed during the forward pass are kept alive until + they are used in gradient computations in the backward pass. To reduce this + memory usage, tensors produced in the passed :attr:`function` are not kept + alive until the backward pass. Instead, any passed tensors in :attr:`args` + are kept alive, and the unsaved tensors are recomputed by re-invoking + :attr:`function` in the backward pass as needed for gradient computation. + Activation checkpointing can be applied to any part of a model -- this is + sometimes described as "checkpointing" that part of the model. + + There are currently two checkpointing implementations available, determined + by the :attr:`use_reentrant` parameter. It is recommended that you use + ``use_reentrant=False``. Please refer the note below for a discussion of + their differences. + + .. warning:: + + If the :attr:`function` invocation during the backward pass differs + from the forward pass, e.g., due to a global variable, the checkpointed + version may not be equivalent, potentially causing an + error being raised or leading to silently incorrect gradients. + + .. warning:: + + The ``use_reentrant`` parameter should be passed explicitly. In version + 2.9 we will raise an exception if ``use_reentrant`` is not passed. + If you are using the ``use_reentrant=True`` variant, please refer to the + note below for important considerations and potential limitations. + + .. note:: + + The reentrant variant of checkpoint (``use_reentrant=True``) and + the non-reentrant variant of checkpoint (``use_reentrant=False``) + differ in the following ways: + + * Non-reentrant checkpoint stops recomputation as soon as all needed + intermediate activations have been recomputed. This feature is enabled + by default, but can be disabled with :func:`set_checkpoint_early_stop`. + Reentrant checkpoint always recomputes :attr:`function` in its + entirety during the backward pass. + + * The reentrant variant does not record the autograd graph during the + forward pass, as it runs with the forward pass under + :func:`torch.no_grad`. The non-reentrant version does record the + autograd graph, allowing one to perform backward on the graph within + checkpointed regions. + + * The reentrant checkpoint only supports the + :func:`torch.autograd.backward` API for the backward pass without its + `inputs` argument, while the non-reentrant version supports all ways + of performing the backward pass. + + * At least one input and output must have ``requires_grad=True`` for the + reentrant variant. If this condition is unmet, the checkpointed part + of the model will not have gradients. The non-reentrant version does + not have this requirement. + + * The reentrant version does not consider tensors in nested structures + (e.g., custom objects, lists, dicts, etc) as participating in + autograd, while the non-reentrant version does. + + * The reentrant checkpoint does not support checkpointed regions with + detached tensors from the computational graph, whereas the + non-reentrant version does. For the reentrant variant, if the + checkpointed segment contains tensors detached using ``detach()`` or + with :func:`torch.no_grad`, the backward pass will raise an error. + This is because ``checkpoint`` makes all the outputs require gradients + and this causes issues when a tensor is defined to have no gradient in + the model. To avoid this, detach the tensors outside of the + ``checkpoint`` function. + + Args: + function: describes what to run in the forward pass of the model or + part of the model. It should also know how to handle the inputs + passed as the tuple. For example, in LSTM, if user passes + ``(activation, hidden)``, :attr:`function` should correctly use the + first input as ``activation`` and the second input as ``hidden`` + args: tuple containing inputs to the :attr:`function` + + Keyword args: + preserve_rng_state(bool, optional): Omit stashing and restoring + the RNG state during each checkpoint. Note that under torch.compile, + this flag doesn't take effect and we always preserve RNG state. + Default: ``True`` + use_reentrant(bool): + specify whether to use the activation checkpoint variant that + requires reentrant autograd. This parameter should be passed + explicitly. In version 2.9 we will raise an exception if + ``use_reentrant`` is not passed. If ``use_reentrant=False``, + ``checkpoint`` will use an implementation that does not require + reentrant autograd. This allows ``checkpoint`` to support additional + functionality, such as working as expected with + ``torch.autograd.grad`` and support for keyword arguments input into + the checkpointed function. + context_fn(Callable, optional): A callable returning a tuple of two + context managers. The function and its recomputation will be run + under the first and second context managers respectively. + This argument is only supported if ``use_reentrant=False``. + determinism_check(str, optional): A string specifying the determinism + check to perform. By default it is set to ``"default"`` which + compares the shapes, dtypes, and devices of the recomputed tensors + against those the saved tensors. To turn off this check, specify + ``"none"``. Currently these are the only two supported values. + Please open an issue if you would like to see more determinism + checks. This argument is only supported if ``use_reentrant=False``, + if ``use_reentrant=True``, the determinism check is always disabled. + debug(bool, optional): If ``True``, error messages will also include + a trace of the operators ran during the original forward computation + as well as the recomputation. This argument is only supported if + ``use_reentrant=False``. + early_stop(bool, optional): If ``True``, non-reentrant checkpoint stops + recomputation as soon as it has computed all needed Tensors. This + argument is ignored if ``use_reentrant=True``. Can be overridden + globally using :func:`set_checkpoint_early_stop` context manager. + Default: ``True``. + + Returns: + Output of running :attr:`function` on :attr:`*args` + """ + if use_reentrant is None: + warnings.warn( + "torch.utils.checkpoint: the use_reentrant parameter should be " + "passed explicitly. Starting in PyTorch 2.9, calling checkpoint " + "without use_reentrant will raise an exception. use_reentrant=False is " + "recommended, but if you need to preserve the current default " + "behavior, you can pass use_reentrant=True. Refer to docs for more " + "details on the differences between the two variants.", + stacklevel=2 + ) + use_reentrant = True + + # Hack to mix *args with **kwargs in a python 2.7-compliant way + preserve = kwargs.pop("preserve_rng_state", True) + if kwargs and use_reentrant: + raise ValueError( + "Unexpected keyword arguments: " + ",".join(arg for arg in kwargs) + ) + + if use_reentrant: + if context_fn is not noop_context_fn or debug is not False: + raise ValueError( + "Passing `context_fn` or `debug` is only supported when " + "use_reentrant=False." + ) + return CheckpointFunction.apply(function, preserve, *args) + else: + gen = _checkpoint_without_reentrant_generator( + function, preserve, context_fn, determinism_check, debug, early_stop, *args, **kwargs + ) + # Runs pre-forward logic + next(gen) + ret = function(*args, **kwargs) + # Runs post-forward logic + try: + next(gen) + except StopIteration: + return ret + + +def checkpoint_sequential(functions, segments, input, use_reentrant=None, **kwargs): + r"""Checkpoint a sequential model to save memory. + + Sequential models execute a list of modules/functions in order + (sequentially). Therefore, we can divide such a model in various segments + and checkpoint each segment. All segments except the last will not store + the intermediate activations. The inputs of each checkpointed segment will + be saved for re-running the segment in the backward pass. + + .. warning:: + The ``use_reentrant`` parameter should be passed explicitly. In version + 2.9 we will raise an exception if ``use_reentrant`` is not passed. + If you are using the ``use_reentrant=True` variant, please see + :func:`~torch.utils.checkpoint.checkpoint` for + the important considerations and limitations of this variant. It is + recommended that you use ``use_reentrant=False``. + + .. warning: + Since PyTorch 1.4, it allows only one Tensor as the input and + intermediate outputs, just like :class:`torch.nn.Sequential`. + + Args: + functions: A :class:`torch.nn.Sequential` or the list of modules or + functions (comprising the model) to run sequentially. + segments: Number of chunks to create in the model + input: A Tensor that is input to :attr:`functions` + preserve_rng_state(bool, optional): Omit stashing and restoring + the RNG state during each checkpoint. + Default: ``True`` + use_reentrant(bool): + specify whether to use the activation checkpoint variant that + requires reentrant autograd. This parameter should be passed + explicitly. In version 2.5 we will raise an exception if + ``use_reentrant`` is not passed. If ``use_reentrant=False``, + ``checkpoint`` will use an implementation that does not require + reentrant autograd. This allows ``checkpoint`` to support additional + functionality, such as working as expected with + ``torch.autograd.grad`` and support for keyword arguments input into + the checkpointed function. + + Returns: + Output of running :attr:`functions` sequentially on :attr:`*inputs` + + Example: + >>> # xdoctest: +SKIP("stub") + >>> model = nn.Sequential(...) + >>> input_var = checkpoint_sequential(model, chunks, input_var) + """ + if use_reentrant is None: + warnings.warn( + "torch.utils.checkpoint.checkpoint_sequential: the use_reentrant " + "parameter should be passed explicitly. " + "In version 2.9 we will raise an exception if use_reentrant " + "is not passed. use_reentrant=False is " + "recommended, but if you need to preserve the current default " + "behavior, you can pass use_reentrant=True. Refer to docs for more " + "details on the differences between the two variants.", stacklevel=2 + ) + use_reentrant = True + + # Hack for keyword-only parameter in a python 2.7-compliant way + preserve = kwargs.pop("preserve_rng_state", True) + if kwargs: + raise ValueError( + "Unexpected keyword arguments: " + ",".join(arg for arg in kwargs) + ) + + def run_function(start, end, functions): + def forward(input): + for j in range(start, end + 1): + input = functions[j](input) + return input + + return forward + + if isinstance(functions, torch.nn.Sequential): + functions = list(functions.children()) + + segment_size = len(functions) // segments + # the last chunk has to be non-volatile + end = -1 + for start in range(0, segment_size * (segments - 1), segment_size): + end = start + segment_size - 1 + input = checkpoint( + run_function(start, end, functions), + input, + use_reentrant=use_reentrant, + preserve_rng_state=preserve, + ) + return run_function(end + 1, len(functions) - 1, functions)(input) + + +def _internal_assert(cond) -> None: + if not cond: + raise AssertionError( + "Something went unexpectedly wrong in activation checkpoint. " + "Please report this bug by filing an issue to PyTorch." + ) + + +# NOTE [ Nestable Checkpoint ] +# +# The semantics of nested checkpoint can be defined by two basic rules. +# Following the two rules leads to an important implication that is central +# to motivating the design. +# +# Rule 1. Saved tensors are managed by inner-most checkpoint only and hidden +# from any outer layers of checkpoint. +# +# Rule 2. The inputs of inner checkpoints are treated as tensors saved to its +# parent checkpoint. +# +# Implication: To recompute any given saved tensor, we need to recompute all of +# the checkpoints wrapping it. +# +# Why is this implied? To unpack a saved tensor X during backward we need to +# recompute the inner-most checkpoint (#1), and in order to recompute that +# checkpoint I need to have its inputs, which are managed by that checkpoint's +# parent (#2), which thus also needs to be recomputed first. Continue this line +# of reasoning and we realize that in order to unpack X, all checkpoints that +# were active at the time X was saved need to be recomputed. (unless we have +# already done so in that backward for some other saved tensor). +# +# In practice, we use a noop autograd Function to save inputs as saved tensors. +# During unpack calling ctx.saved_tensor triggers the parent checkpoint to +# recompute. +# +# Rule 3. We should start recomputation as if there are no checkpoints currently +# active. Checkpoints encountered during recomputation are still +# respected. +# +# When we start recomputation, we push the saved variable hook meant for +# recomputation on the stack. See examples in Rule 6 for more context. +# +# * * * * +# +# Beyond the basic semantics specific to nested checkpoint, we impose several +# more constraints that may apply to checkpointing in general. +# +# Rule 4. Lifetime of recomputed tensors +# +# Recomputed tensors are considered specific to particular invocations +# of backward and are always cleared immediately as they are unpacked +# Particularly, we require this to happen even if retain_graph=True. +# +# [ Implementation details of Rule 4 ] +# +# If we were okay with recomputed tensors staying alive after backward is run +# with retain_graph=True, we would store recomputed variables as the values of a +# WeakKeyDictionary and pack strong references to the keys, so that as we +# backward, those packed keys would be cleared as long as retain_graph=False. +# Clearing the packed key clears the corresponding entry in the WKD. +# +# If we wish recomputed variables to be immediately cleared as we unpack them in +# the retain_graph=True case, we cannot rely on the packed keys to be cleared by +# backward automatically. Instead of packing the strong reference to the key +# directly, we pack a container object, which we manually clear as we unpack. +# +# An important detail is that if a second backward happens, the second +# recomputation needs to reset the container with a newly created key. +# +# Rule 5. Stop recomputation as soon as we've recomputed the saved tensors we +# know we need. +# +# [ Implementation details of Rule 5 ] +# +# During recomputation, raise an exception if the number of recomputed tensors +# matches the number of tensors that we expected to recompute. We wrap the +# recomputation call with a try-catch to catch this specific exception. See +# Rule #6 below for some examples. +# +# Rule 6. We support doing backward inside checkpoint context +# +# [ retain_graph is True] +# +# def fn(x): +# y = x.sin() +# z = y.cos() +# gx, = torch.autograd.grad(z, x, retains_grad=True) +# return gx, z +# +# out = checkpoint(fn)(inp) +# out.backward() +# +# Because z is saved by cos while checkpoint is enabled, it would not be +# actually saved, and so the .grad() call inside must trigger a recomputation. +# +# During recomputation the "inner pack hook" has two responsibilities: +# +# 1) As usual, populating the WeakKeyDictionary storing recomputed tensors +# 2) Pack the actual tensor (detached) so that one may perform backward on the +# recomputed graph. The tensors saved to this graph will live until the end +# of recomputation, or die earlier if someone performs backward with +# retain_graph=False. +# +# More generally performing backward on the recomputed graph occurs in the +# following cases: +# - If backward is performed inside forward, +# - During the original forward IF early-stop is disabled +# - During the original backward +# - If there are multiple .grad()/.backward() calls, we would perform backward +# on the recomputed graph even if early-stop is enabled (see the example below) +# +# [ retain_graph is False ] +# +# The example below shows what happens if during recomputation we find that some +# of the tensors we are trying to recompute have already been cleared. +# +# Spoiler: we don't do anything special, we just skip over them! +# +# def fn(x): +# y = x.sin() # (1) +# z = y.cos() # (2) +# gx, = torch.autograd.grad(z, x) # (3) +# return x.cos() * gx # (4) +# +# out = checkpoint(fn)(inp) +# out.backward() # (5) +# +# 1, 2. Don't save x and y since we are inside a checkpoint. +# 3. Trigger a recompute of fn since x and y weren't saved. +# And depending on whether early stop is enabled, either stop at (2) or +# continue running the function. +# Because we are running backward with retain_graph=False, we clear x and y's +# holders. +# 4. Don't save x since we are inside a checkpoint. +# 5. Calling backward triggers another recompute of fn. During recompute, we see +# that x and y have already been cleared in the original graph as indicated +# by holder=None. We skip over them. We still save x at (4) (since its holder +# is still alive.) + +_enable_checkpoint_early_stop: Optional[bool] = None + + +@contextlib.contextmanager +def set_checkpoint_early_stop(enable: bool): + """Context manager that sets whether checkpoint should stop recomputation early. + + By default, non-reentrant checkpoint stops recomputation as soon as it + has computed all needed Tensors. This context manager can be used to disable + that feature if it is problematic for your specific application. + + This context manager only needs to be active when forward is run. It does + not need to be active during backward. + + Example:: + + >>> # xdoctest: +SKIP(failing) + >>> message = "saved tensors default hooks are disabled" + >>> with set_checkpoint_early_stop(False): + ... # Any checkpoint under this context manager will respect this + ... # context manager, even if its backward is performed outside. + ... out = checkpoint(fn, inputs) + ... + >>> out.backward() + """ + global _enable_checkpoint_early_stop + try: + prev = _enable_checkpoint_early_stop + _enable_checkpoint_early_stop = enable + yield + finally: + _enable_checkpoint_early_stop = prev + + +class _Handle: + pass + + +class _Holder: + def __init__(self) -> None: + self.handles: Dict[int, Optional[_Handle]] = {} + + +class _NoopSaveInputs(torch.autograd.Function): + @staticmethod + # pyrefly: ignore [bad-override] + def forward(*args): + return torch.empty((0,)) + + @staticmethod + def setup_context(ctx: Any, inputs: Tuple[Any, ...], output: Any) -> None: + # Only tensors can be saved with ctx.save_for_backward, everything else + # is captured by get_args, which is saved directly on ctx + tensor_indices, tensors = zip( + *[(i, o) for i, o in enumerate(inputs) if isinstance(o, torch.Tensor)], strict=False + ) + idx2saved_idx = {b: a for a, b in enumerate(tensor_indices)} + # args but with tensors replaced with None as placeholders + args = [None if isinstance(o, torch.Tensor) else o for o in inputs] + + def get_args(saved_tensors): + # restore the placeholders with the original tensors grabbed from + # ctx.saved_tensors (which may be saved on a parent checkpoint if + # this checkpoint is nested, and that would trigger a recursive + # unpack!) + ret = [ + saved_tensors[idx2saved_idx[i]] if i in tensor_indices else o + for i, o in enumerate(args) + ] + # grab the tail since we also saved the dummy to avoid having to explicitly + # handle the case where there are no tensor inputs + return ret[1:] + + ctx.get_args = get_args + ctx.save_for_backward(*tensors) + + @staticmethod + def backward(ctx, *grad_outputs) -> NoReturn: + raise AssertionError("Did not expect to backward on this graph") + + +class _CheckpointFrame: + def __init__(self, recompute_fn, early_stop, unpack_error_cb, metadata_fn) -> None: + self.recompute_fn = recompute_fn + self.input_saver = None + self.weak_holders: List[ReferenceType] = [] + # We store this as a weakkeydictionary so that in the case of a partial + # backward, the entries in the dict are cleared alongside the Holder + # which will be removed when the SavedVariable is cleared. + self.recomputed: DefaultDict[ + int, weakref.WeakKeyDictionary[_Handle, torch.Tensor] + ] = defaultdict(weakref.WeakKeyDictionary) + # We need both recomp_counter and recomputed since they can diverge + # https://github.com/pytorch/pytorch/pull/90105#discussion_r1135889885 + self.recomp_counter: DefaultDict[int, int] = defaultdict(int) + self.is_recomputed: DefaultDict[int, bool] = defaultdict(bool) + + # See Rule 5 + self.early_stop = early_stop + + # Debugging + self.metadata_fn = metadata_fn + self.unpack_error_cb = unpack_error_cb + self.x_metadatas = [] + self.forward_completed = False + self.ignore_saved_mismatch = False + + def check_recomputed_tensors_match(self, gid) -> None: + if self.ignore_saved_mismatch: + # TODO: we can probably make this check stricter by checking that + # the metadata of the first tensors still match. + return + # NOTE [ Error handling for checkpoint ] + # + # At a high level, we need to check that the tensors saved + # during original forward matches tensors saved during recompute + # This means handling 3 cases: + # + # 1. During recompute, more tensors were saved. + # + # Usually this is hidden due to the StopRecomputationError + # but if early stop is not enabled, or we would have errored + # anyway because there aren't enough weak_holders. But we + # do want to have a nice error. See the _recomputation_hook + # for details. + if not len(self.weak_holders) == self.recomp_counter[gid]: + # 2. During recompute, fewer tensors were saved + # + # We know that every time we save something do original forward + # we append to weak_holder, and every time we save a tensor + # during recompute we increment recompute_counter. + raise CheckpointError( + "torch.utils.checkpoint: A different number of tensors was saved " + "during the original forward and recomputation.\n" + f"Number of tensors saved during forward: {len(self.weak_holders)}\n" + f"Number of tensors saved during recomputation: {self.recomp_counter[gid]}.\n" + f"{_debug_tip_msg}" + ) + + # 3. During recompute, the same tensors were saved, but they + # have different metadata + nb_meta_different = [] + for idx, weak_holder in enumerate(self.weak_holders): + holder = weak_holder() + if holder is None: + continue + # We've seen all holders since we iterate over them in order + # For every holder that is still alive now, it must've been + # alive when we saw it during recompute, therefore, the + # gid must be set. + _internal_assert(gid in holder.handles) + # We know this is the first unpack, so it couldn't have been set + # to None yet. + _internal_assert(holder.handles[gid] is not None) + # We always set these together in the recomputation hook + _internal_assert(holder.handles[gid] in self.recomputed[gid]) + # see pack hook, x_metadata is 1:1 with weak_holders. + x_meta = self.x_metadatas[idx] + recomputed_x = self.recomputed[gid][holder.handles[gid]] + if x_meta != self.metadata_fn(recomputed_x): + nb_meta_different.append((idx, x_meta, self.metadata_fn(recomputed_x))) + + if len(nb_meta_different) > 0: + mismatched_tensors = "" + for idx, x_meta, recomputed_meta in nb_meta_different: + mismatched_tensors += ( + f"tensor at position {idx}:\n" + f"saved metadata: {x_meta}\n" + f"recomputed metadata: {recomputed_meta}\n" + ) + raise CheckpointError( + "torch.utils.checkpoint: Recomputed values for the following tensors " + "have different metadata than during the forward pass.\n" + f"{mismatched_tensors}.\n" + f"{_debug_tip_msg}" + ) + + +_debug_tip_msg = """ +Tip: To see a more detailed error message, either pass `debug=True` to +`torch.utils.checkpoint.checkpoint(...)` or wrap the code block +with `with torch.utils.checkpoint.set_checkpoint_debug_enabled(True):` to +enable checkpoint‑debug mode globally. +""" + + +_checkpoint_error_template = """ \ +An error happened while unpacking tensors; dumping logs of latest computation +because you passed `debug=True` to `torch.utils.checkpoint.checkpoint()`. +Scroll all the way down for guidance on how to navigate these logs. + ++~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+ +| 1. Stack traces of the operators that ran in the original forward | ++------------------------------------------------------------------------------+ + +{forward_traces} ++~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+ +| 2. Stack traces of the operators that ran during recomputation | ++------------------------------------------------------------------------------+ + +{recompute_traces} ++~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+ +| 3. Log of operators in the original forward and recomputation | ++------------------------------------------------------------------------------+ +(Scroll up to correlate stack traces with each operation listed below. This + helps identify their source in the code.) + +IMPORTANT: Differences in "detach" calls between the original forward and the + recomputation are expected. They are introduced by the checkpointing + mechanism and can be ignored. + +Operations executed during the original forward: + +{forward_ops} + +Operations executed during recomputation: + +{recompute_ops} + ++------------------------------------------------------------------------------+ + ERROR: Detected non-determinism while running activation checkpointing + + You are seeing this error because you passed `debug=True` to checkpoint and + tensors to be saved during the original forward and differ between those saved + during recomputation. This can happen if different operators were ran in the + original forward and in the recomputation. + + To identify where the mismatch may be coming from, you can do the following: + + 1) Compare the operators ran during original forward and recomputation to + see where they differ. These operators are printed above in the order they + were executed. + + 2) Review the stack trace for each operator to locate its invocation source. + Each operator's stack trace is printed in their execution order. + + Note that the logs can be quite long. Here's how they are structured: + (Tip: you can Ctrl-f for these headers) + + 1. Stack traces of the operators that ran in the original forward + 2. Stack traces of the operators that ran during recomputation + 3. Log of operators in the original forward and recomputation + 4. Error message <--- You are here +-------------------------------------------------------------------------------- +""" + +class CheckpointError(RuntimeError): + pass + + +def _get_debug_context_and_cb() -> Tuple[Callable[[], Any], Callable[[CheckpointError], None]]: + # This function returns the context_fn and error_cb to be used by the + # checkpointing mechanism. error_cb is invoked when an error is detected + # during unpack. + + # record_context_cpp is not support on non-linux non-x86_64 platforms + cpp_tb = platform.machine() == 'x86_64' and platform.system() == 'Linux' + + class CaptureLogs: + def __init__(self) -> None: + self.logs = None + self.tbs = None + + def get_context_manager(self): + @contextlib.contextmanager + def logging_mode(): + with LoggingTensorMode(), \ + capture_logs(True, python_tb=True, script_tb=True, cpp_tb=cpp_tb) as logs_and_tb: + # pyrefly: ignore [bad-assignment] + self.logs, self.tbs = logs_and_tb + yield logs_and_tb + return logging_mode() + + capture_logs_fwd = CaptureLogs() + capture_logs_recompute = CaptureLogs() + + def unpack_error_cb(e: CheckpointError) -> NoReturn: + def get_str_tb(label, capture_logs): + out = "" + total_len = len(capture_logs.logs) + for i, (log, tb) in enumerate(zip(capture_logs.logs, capture_logs.tbs, strict=False)): + out += f"{log} ({i + 1} of {total_len} in {label})\n\n" + found_torch_dispatch = False + for line in tb: + # Start printing stack trace only after __torch_dispatch__ is found + is_torch_dispatch = line['name'] == '__torch_dispatch__' + if not found_torch_dispatch and not is_torch_dispatch: + continue + elif is_torch_dispatch: + found_torch_dispatch = True + continue + out += f"{line['filename']}:{line['line']}:{line['name']}\n" + out += "\n\n" + return out + if capture_logs_fwd.logs is None: + raise AssertionError("capture_logs_fwd.logs is None") + if capture_logs_recompute.logs is None: + raise AssertionError("capture_logs_recompute.logs is None") + raise CheckpointError( + _checkpoint_error_template.format( + forward_traces=get_str_tb("original", capture_logs_fwd), + recompute_traces=get_str_tb("recompute", capture_logs_recompute), + forward_ops="\n".join(capture_logs_fwd.logs), + recompute_ops="\n".join(capture_logs_recompute.logs) + ) + ) from e + + def context_fn(): + return capture_logs_fwd.get_context_manager(), capture_logs_recompute.get_context_manager() + + return context_fn, unpack_error_cb + +def _default_meta_extractor(x: torch.Tensor) -> Dict[str, Any]: + # These properties are fast to check, easy to understand + return { + "shape": x.shape, + "dtype": x.dtype, + "device": x.device + } + +_allowed_determinism_checks_to_fns: Dict[str, Callable[[torch.Tensor], Any]] = { + _DEFAULT_DETERMINISM_MODE: _default_meta_extractor, + "none": lambda _: None, +} + +# See Rule 5 +class _StopRecomputationError(Exception): + pass + + +class _recomputation_hook(torch.autograd.graph.saved_tensors_hooks): + def __init__(self, target_frame_ref: ReferenceType, gid: Union["GraphExecGroup", int]) -> None: + def pack_hook(x): + x = x.detach() if x.requires_grad else x + target_frame = target_frame_ref() + if target_frame is None: + raise AssertionError("Internal error: target_frame reference is None") + recomp_idx = target_frame.recomp_counter[gid] + target_frame.recomp_counter[gid] += 1 + + if recomp_idx >= len(target_frame.weak_holders): + if target_frame.early_stop: + raise AssertionError("Unexpected state: target_frame.early_stop is set") + if not target_frame.forward_completed: + # We run into this case when early stop is not enabled and do + # grad within checkpoint. + # We need to set this flag, so we don't error out later when + # we check if the number of tensors saved during forward and + # recomputation match. + target_frame.ignore_saved_mismatch = True + return x + raise CheckpointError( + "torch.utils.checkpoint: trying to save more tensors during " + "recomputation than during the original forward pass.\n" + f"{_debug_tip_msg}" + ) + + holder = target_frame.weak_holders[recomp_idx]() + + # This holder may have been cleared because someone may have called + # backward within forward. If so, we don't need to save. + if holder is not None: + _internal_assert(holder.handles.get(gid, None) is None) + holder.handles[gid] = _Handle() + target_frame.recomputed[gid][holder.handles[gid]] = x + + if target_frame.early_stop and target_frame.recomp_counter[gid] == len( + target_frame.weak_holders + ): + raise _StopRecomputationError + # See Rule 6: [ retain_graph is True ] above + return x + + def unpack_hook(x): + # See Rule 6: [ retain_graph is True ] above for an example of when + # the graph created during recomputation could be backwarded. + return x + + super().__init__(pack_hook, unpack_hook) + + +# torch._disable_dynamo creates a reference cycle with decorated function +# This function is used to ensure that the decorated function does not have +# a closure, so that other objects aren't also kept alive. +# https://github.com/pytorch/pytorch/issues/154642 +# Note: does not work when fn is compiled +@torch._disable_dynamo +def _run_fn_with_dynamo_disabled(fn, *args, **kwargs): + return fn(*args, **kwargs) + + +class _checkpoint_hook(torch.autograd.graph.saved_tensors_hooks): + def __init__(self, frame) -> None: + def pack_hook(x): + # See Rule 4 above + holder = _Holder() + frame.weak_holders.append(weakref.ref(holder)) + # Save metadata to detect non-determinism + if frame.metadata_fn is not None: + with torch.no_grad(): + frame.x_metadatas.append(frame.metadata_fn(x)) + return holder + + def unpack_hook(holder): + # First check if we're inside a GraphExecGroup context + gid: Union[GraphExecGroup, None, int] = GraphExecGroup._get_current_group() + if gid is None: + # Fallback to using the current graph task id + gid = torch._C._current_graph_task_id() + if gid == -1: + # generate a temporary id if we trigger unpack outside of a backward call + gid = int(uuid.uuid4()) + + if not frame.is_recomputed[gid]: + ctx = frame.input_saver.grad_fn + args = ctx.get_args(ctx.saved_tensors) + + try: + with _recomputation_hook( + weakref.ref(frame), gid + ), torch.autograd.enable_grad(): + # See Note: [compiled autograd and checkpoint unpack hook] + _run_fn_with_dynamo_disabled(frame.recompute_fn, *args) + except _StopRecomputationError: + pass + frame.is_recomputed[gid] = True + frame.check_recomputed_tensors_match(gid) + + _internal_assert(gid in holder.handles) + + if holder.handles[gid] is None: + extra = "" + if torch._C._get_graph_exec_group() is not None: + extra = ( + "Performing two backward calls that overlap (i.e. require the same " + "saved activation in order to compute gradients) is not allowed while " + "under the torch.utils.checkpoint.GraphExecGroup context. " + ) + raise CheckpointError( + "torch.utils.checkpoint: Unpack is being triggered for a tensor that was already " + f"unpacked once. {extra}If you are calling ctx.saved_tensors in backward, make sure " + "to do so only once. Otherwise please open an issue with details on your use case." + ) + _internal_assert(holder.handles[gid] in frame.recomputed[gid]) + ret = frame.recomputed[gid][holder.handles[gid]] + holder.handles[gid] = None + return ret + + if frame.unpack_error_cb is not None: + def unpack_hook_with_error_cb(holder): + try: + return unpack_hook(holder) + except CheckpointError as e: + frame.unpack_error_cb(e) + super().__init__(pack_hook, unpack_hook_with_error_cb) + else: + super().__init__(pack_hook, unpack_hook) + + +def _is_compiling(func, args, kwargs): + # Check if we are under AOTAutograd tracing + # Checking that a functional mode is active should always do what we want + return torch._C._get_dispatch_mode(torch._C._TorchDispatchModeKey.PROXY) is not None + + +class _VersionWrapper: + # Check that cached tensors are not mutated. + def __init__(self, val) -> None: + self.val: Union[torch.Tensor, Any] = val + self.version: Optional[int] = val._version if isinstance(val, torch.Tensor) else None + + def get_val(self, allow_cache_entry_mutation): + if self.version is not None and not allow_cache_entry_mutation: + if self.val._version != self.version: + # Can we give user a stack trace of where the mutation happened? + raise RuntimeError( + "Tensor cached during selective activation checkpoint has been mutated" + ) + return self.val + + +def _maybe_detach(x, any_ret_has_alias_info): + # We detach for two separate reasons: + # - For view ops, we need to ensure that when the tensor is returned from + # CachedDispatchMode, as_view sees that the AutogradMeta is nullptr + # - Avoid reference cycles + # For case 1, it is not enough to check whether x has differentiable dtype + # because non-differentiable dtype can have non-nullptr AutogradMeta, e.g. + # when the tensor is a view. + if isinstance(x, torch.Tensor) and (x.is_floating_point() or x.is_complex() or any_ret_has_alias_info): + with torch._C._SetExcludeDispatchKeyGuard(torch._C.DispatchKey.ADInplaceOrView, False): + # Ensure that view performed beneath autograd properly propagates + # version counter. TODO: Use reentrant_dispatch instead of + # manually manipulating dispatch keys. Using reentrant_dispatch + # would respect inference_mode, though that is not relevant for + # this case. + x = x.detach() + return x + + +class SelectiveCheckpointContext: + """ + Context passed to policy function during selective checkpointing. + + This class is used to pass relevant metadata to the policy function during + selective checkpointing. The metadata includes whether the current invocation + of the policy function is during recomputation or not. + + Example: + >>> # xdoctest: +SKIP(stub) + >>> + >>> def policy_fn(ctx, op, *args, **kwargs): + >>> print(ctx.is_recompute) + >>> + >>> context_fn = functools.partial(create_selective_checkpoint_contexts, policy_fn) + >>> + >>> out = torch.utils.checkpoint.checkpoint( + >>> fn, x, y, + >>> use_reentrant=False, + >>> context_fn=context_fn, + >>> ) + """ + def __init__(self, *, is_recompute) -> None: + self.is_recompute = is_recompute + + +class CheckpointPolicy(enum.Enum): + """ + Enum for specifying the policy for checkpointing during backpropagation. + + The following policies are supported: + + - ``{MUST,PREFER}_SAVE``: The operation's output will be saved during the forward + pass and will not be recomputed during the backward pass + - ``{MUST,PREFER}_RECOMPUTE``: The operation's output will not be saved during the + forward pass and will be recomputed during the backward pass + + Use ``MUST_*`` over ``PREFER_*`` to indicate that the policy should not be overridden + by other subsystems like `torch.compile`. + + .. note:: + A policy function that always returns ``PREFER_RECOMPUTE`` is + equivalent to vanilla checkpointing. + + A policy function that returns ``PREFER_SAVE`` every op is + NOT equivalent to not using checkpointing. Using such a policy would + save additional tensors not limited to ones that are actually needed for + gradient computation. + """ + MUST_SAVE = 0 + PREFER_SAVE = 1 + MUST_RECOMPUTE = 2 + PREFER_RECOMPUTE = 3 + + +def _policy_from_bool(b): + # For backward compatibility + return CheckpointPolicy.MUST_SAVE if b else CheckpointPolicy.PREFER_RECOMPUTE + + +SAC_IGNORED_OPS = { + # AC inserts different number of detach during forward and recompute. + torch.ops.aten.detach.default, + # AC's determinism check invokes additional metadata ops during forward. + # With subclasses involved, these metadata ops become dispatchable, this + # can result in incorrectness if these ops are selected cached. + torch.ops.prim.device.default, +} | set(torch._subclasses.functional_tensor.FunctionalTensor.metadata_fns) # type: ignore[has-type] + + +class _CachingTorchDispatchMode(TorchDispatchMode): + @classmethod + def ignore_compile_internals(cls): + return True + + # Used together with _CachedTorchDispatchMode to implement SAC. + def __init__(self, policy_fn, storage) -> None: + self.policy_fn = policy_fn + self.storage = storage + + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + if func in SAC_IGNORED_OPS: + return func(*args, **kwargs) + + kwargs = {} if kwargs is None else kwargs + policy = self.policy_fn(SelectiveCheckpointContext(is_recompute=False), + func, *args, **kwargs) + if isinstance(policy, bool): + policy = _policy_from_bool(policy) + + is_compiling = _is_compiling(func, args, kwargs) + + if is_compiling: + # Overwrite each node's "recompute" tag to add in the user annotation. + fx_traceback.current_meta["recompute"] = policy + + out = func(*args, **kwargs) + + # HOPs don't support func._schema + # HOPs don't alias -> this is always true today and will be always true for a long time + # TODO HOPs don't mutate -> this is always true today but will not be true forever + if isinstance(func, torch._ops.HigherOrderOperator): + any_ret_has_alias_info = False + else: + any_ret_has_alias_info = any(ret.alias_info is not None for ret in func._schema.returns) + + if policy in (CheckpointPolicy.MUST_SAVE, CheckpointPolicy.PREFER_SAVE) or is_compiling: + self.storage[func].append(tree_map(lambda x: _VersionWrapper(_maybe_detach(x, any_ret_has_alias_info)), out)) + return out + +class _CachedTorchDispatchMode(TorchDispatchMode): + @classmethod + def ignore_compile_internals(cls): + return True + + # Used together with _CachedTorchDispatchMode to implement SAC. + def __init__(self, policy_fn, storage, allow_cache_entry_mutation) -> None: + self.policy_fn = policy_fn + self.storage = storage + self.allow_cache_entry_mutation = allow_cache_entry_mutation + + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + if func in SAC_IGNORED_OPS: + return func(*args, **kwargs) + + kwargs = {} if kwargs is None else kwargs + policy = self.policy_fn(SelectiveCheckpointContext(is_recompute=True), + func, *args, **kwargs) + if isinstance(policy, bool): + policy = _policy_from_bool(policy) + + is_compiling = _is_compiling(func, args, kwargs) + + if policy in (CheckpointPolicy.MUST_SAVE, CheckpointPolicy.PREFER_SAVE) or is_compiling: + storage = self.storage.get(func) + if storage is None: + raise RuntimeError(f"{func} encountered during backward, but not found in storage") + if len(storage) == 0: + raise RuntimeError( + "Trying to backward an extra time. You are only allowed to backward once " + "on any region computed under selective activation checkpoint." + ) + out = tree_map(lambda x: x.get_val(self.allow_cache_entry_mutation), storage.pop(0)) + else: + out = func(*args, **kwargs) + return out + + +def create_selective_checkpoint_contexts(policy_fn_or_list, allow_cache_entry_mutation=False): + """ + Helper to avoid recomputing certain ops during activation checkpointing. + + Use this with `torch.utils.checkpoint.checkpoint` to control which + operations are recomputed during the backward pass. + + Args: + policy_fn_or_list (Callable or List): + - If a policy function is provided, it should accept a + :class:`SelectiveCheckpointContext`, the :class:`OpOverload`, args and + kwargs to the op, and return a :class:`CheckpointPolicy` enum value + indicating whether the execution of the op should be recomputed or not. + - If a list of operations is provided, it is equivalent to a policy + returning `CheckpointPolicy.MUST_SAVE` for the specified + operations and `CheckpointPolicy.PREFER_RECOMPUTE` for all other + operations. + allow_cache_entry_mutation (bool, optional): By default, an error is + raised if any tensors cached by selective activation checkpoint are + mutated in order to ensure correctness. If set to `True`, this check + is disabled. + Returns: + A tuple of two context managers. + + Example: + >>> # xdoctest: +REQUIRES(LINUX) + >>> import functools + >>> + >>> x = torch.rand(10, 10, requires_grad=True) + >>> y = torch.rand(10, 10, requires_grad=True) + >>> + >>> ops_to_save = [ + >>> torch.ops.aten.mm.default, + >>> ] + >>> + >>> def policy_fn(ctx, op, *args, **kwargs): + >>> if op in ops_to_save: + >>> return CheckpointPolicy.MUST_SAVE + >>> else: + >>> return CheckpointPolicy.PREFER_RECOMPUTE + >>> + >>> context_fn = functools.partial(create_selective_checkpoint_contexts, policy_fn) + >>> + >>> # or equivalently + >>> context_fn = functools.partial(create_selective_checkpoint_contexts, ops_to_save) + >>> + >>> def fn(x, y): + >>> return torch.sigmoid(torch.matmul(torch.matmul(x, y), y)) * y + >>> + >>> out = torch.utils.checkpoint.checkpoint( + >>> fn, x, y, + >>> use_reentrant=False, + >>> context_fn=context_fn, + >>> ) + """ + # NB: If grad_mode is disabled, checkpoint would not run forward under + # context_fn anyway, so proceed as usual. + if isinstance(policy_fn_or_list, list): + for op in policy_fn_or_list: + if not isinstance(op, (torch._ops.OpOverload, torch._ops.HigherOrderOperator)): + _extra_msg = ( + "Please update the OpOverloadPacket to a specific OpOverload." + "For example, if you have `torch.ops.aten.mm`, change it to `torch.ops.aten.mm.default`." + ) if isinstance(op, torch._ops.OpOverloadPacket) else "" + raise ValueError( + f"Expected op in `op_list` to be an OpOverload but got: {op} " + f"of type {type(op)}. {_extra_msg}" + ) + + def policy_fn(ctx, op, *args, **kwargs): + if op in policy_fn_or_list: + return CheckpointPolicy.MUST_SAVE + else: + return CheckpointPolicy.PREFER_RECOMPUTE + elif callable(policy_fn_or_list): + policy_fn = policy_fn_or_list + else: + raise TypeError("policy_fn_or_list must be either a function or a list of ops.") + + storage: Dict[Any, List[Any]] = defaultdict(list) + return ( + _CachingTorchDispatchMode(policy_fn, storage), + _CachedTorchDispatchMode(policy_fn, storage, allow_cache_entry_mutation), + ) + +# NB: this helper wraps fn before calling checkpoint_impl. kwargs and +# saving/restoring of global state is handled here. + +def _checkpoint_without_reentrant_generator( + fn, + preserve_rng_state=True, + context_fn: Callable[[], Tuple[ContextManager, ContextManager]] = noop_context_fn, + determinism_check: str = _DEFAULT_DETERMINISM_MODE, + debug: bool = False, + early_stop: bool = True, + *args, + **kwargs +): + """Checkpointing without reentrant autograd. + + Args: + fn: describes what to run in the forward pass of the model or + part of the model. It should also know how to handle the inputs + passed as the tuple. For example, in LSTM, if user passes + ``(activation, hidden)``, :attr:`function` should correctly use the + first input as ``activation`` and the second input as ``hidden`` + preserve_rng_state(bool, optional): Omit stashing and restoring + the RNG state during each checkpoint. + Default: ``True`` + context_fn(Callable, optional): A callable returning a tuple of two + context managers. The function and its recomputation will be run + under the first and second context managers respectively. + determinism_check(str, optional): A string specifying the determinism + check to perform. By default it is set to ``"default"`` which + compares the shapes, dtypes, and devices of the recomputed tensors + against those the saved tensors. To turn off this check, specify + ``"none"``. Currently these are the only two supported values. + Please open an issue if you would like to see more determinism + checks. + debug(bool, optional): If ``True``, error messages will also include + a trace of the operators ran during the original forward computation + as well as the recomputation. + early_stop(bool, optional): If ``True``, non-reentrant checkpoint stops + recomputation as soon as it has computed all needed Tensors. Can be + overridden globally using :func:`set_checkpoint_early_stop` context + manager. Default: ``True``. + *args: Arguments to pass in to the given ``function``. + **kwargs: Keyword arguments to pass into the given ``function``. + """ + unpack_error_cb = None + + if _checkpoint_debug_enabled if _checkpoint_debug_enabled is not None else debug: + if context_fn is not noop_context_fn: + raise ValueError( + "debug=True is incompatible with non-default context_fn" + ) + context_fn, unpack_error_cb = _get_debug_context_and_cb() + + if determinism_check in _allowed_determinism_checks_to_fns: + metadata_fn = _allowed_determinism_checks_to_fns[determinism_check] + else: + raise ValueError( + f"determinism_check should be one of {list(_allowed_determinism_checks_to_fns.keys())}, " + f"but got {determinism_check}" + ) + + device_type = _infer_device_type(*args) + device_module = _get_device_module(device_type) + forward_context, recompute_context = context_fn() + if _is_compiling(fn, args, kwargs) and context_fn is not noop_context_fn: + if ( + not isinstance(forward_context, TorchDispatchMode) + or not isinstance(recompute_context, TorchDispatchMode) + ): + raise AssertionError( + "In torch.compile mode, `context_fn` arg passed to `torch.utils.checkpoint` " + "must generate a tuple of two `TorchDispatchMode`s." + ) + # Accommodates the (remote) possibility that autocast is enabled for cpu AND gpu. + device_autocast_kwargs, cpu_autocast_kwargs = _get_autocast_kwargs(device_type=device_type) + + if preserve_rng_state: + fwd_cpu_state = torch.get_rng_state() + # Don't eagerly initialize the cuda context by accident. + # (If the user intends that the context is initialized later, within their + # run_function, we SHOULD actually stash the cuda state here. Unfortunately, + # we have no way to anticipate this will happen before we run the function. + # If they do so, we raise an error.) + had_device_in_fwd = False + if getattr(device_module, "_initialized", False): + had_device_in_fwd = True + fwd_devices, fwd_device_states = get_device_states(*args) + + def recompute_fn(*inputs) -> None: + kwargs, *args = inputs + # This will be called later during recomputation. This wrapping enables + # the necessary global state to be captured. + rng_devices = [] + if preserve_rng_state and had_device_in_fwd: + rng_devices = fwd_devices + with torch.random.fork_rng( + devices=rng_devices, enabled=preserve_rng_state, device_type=device_type + ): + if preserve_rng_state: + torch.set_rng_state(fwd_cpu_state) + if had_device_in_fwd: + set_device_states(fwd_devices, fwd_device_states, device_type=device_type) + + device_autocast_ctx = torch.amp.autocast( + device_type=device_type, **device_autocast_kwargs + ) if torch.amp.is_autocast_available(device_type) else contextlib.nullcontext() + with device_autocast_ctx, torch.amp.autocast("cpu", **cpu_autocast_kwargs), recompute_context: # type: ignore[attr-defined] + fn(*args, **kwargs) + + new_frame = _CheckpointFrame( + recompute_fn, + _enable_checkpoint_early_stop if _enable_checkpoint_early_stop is not None else early_stop, + unpack_error_cb, + metadata_fn + ) + dummy = torch.empty((0,), requires_grad=True) + new_frame.input_saver = _NoopSaveInputs.apply(dummy, kwargs, *args) + + # When ambient grad_mode is False + if new_frame.input_saver.grad_fn is None: + yield + return + + with _checkpoint_hook(new_frame), forward_context: + yield + new_frame.forward_completed = True + + if getattr(device_module, "_initialized", False) and \ + preserve_rng_state and not had_device_in_fwd: # type: ignore[possibly-undefined] + # Device was not initialized before running the forward, so we didn't + # stash the device state. + raise RuntimeError( + "PyTorch's device state was initialized in the forward pass " + "of a Checkpoint, which is not allowed. Please open an issue " + "if you need this feature." + ) + + return + + +class GraphExecGroup: + """Any checkpointed regions encountered by backward under the same instance + of this context manager will trigger recompute at most once, even if + there are multiple calls to backward. + + Backward calls under the same instance of this context manager must execute + over non-overlapping regions of the backward graph even if retain_graph=True. + In particular, any two backward call cannot use the same saved activation for + gradient computation. + + .. note:: + This context manager only affects checkpoint with use_reentrant=False, and + is a no-op otherwise. + """ + + def __enter__(self) -> "GraphExecGroup": + if torch._C._get_graph_exec_group() is not None: + raise RuntimeError( + "GraphExecGroup contexts cannot be nested. " + f"Already inside group {torch._C._get_graph_exec_group()}" + ) + torch._C._set_graph_exec_group(self) + return self + + def __exit__(self, *args: object) -> None: + torch._C._set_graph_exec_group(None) + + @classmethod + def _get_current_group(cls) -> Optional["GraphExecGroup"]: + # Private API to be used by utils like AC + return torch._C._get_graph_exec_group() + + +# Note: [compiled autograd and checkpoint unpack hook] +# When tracing via compiled autograd, this hook will be visible to the +# compiler if the forward of this checkpointed region ran in eager. +# If the forward had ran under compile, it would have been wrapped in a +# higher order op. See Note: [torch.compile and checkpoint]. +# +# Since we run the recomputation hook under a enable_grad context, +# AOTDispatch will trace a joint graph for this hook, and may +# save different activations than in eager. This conflicts with the +# strict activation count checks in `frame.check_recomputed_tensors_match`. +# So, we disable this hook to force it to recompute eager checkpointed regions +# in eager. This could be removed if we can disable the partitioner for this +# graph segment. diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/collect_env.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/collect_env.py new file mode 100644 index 0000000000000000000000000000000000000000..b566cd94e5d0314afc50150b24e2af7cb984a390 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/collect_env.py @@ -0,0 +1,944 @@ +# mypy: allow-untyped-defs + +# Unlike the rest of the PyTorch this file must be python2 compliant. +# This script outputs relevant system environment info +# Run it with `python collect_env.py` or `python -m torch.utils.collect_env` +import datetime +import json +import locale +import os +import re +import subprocess +import sys +from collections import namedtuple +from typing import cast as _cast, Dict as _Dict + + +try: + import torch + + TORCH_AVAILABLE = True +except (ImportError, NameError, AttributeError, OSError): + TORCH_AVAILABLE = False + +# System Environment Information +SystemEnv = namedtuple( + "SystemEnv", + [ + "torch_version", + "is_debug_build", + "cuda_compiled_version", + "gcc_version", + "clang_version", + "cmake_version", + "os", + "libc_version", + "python_version", + "python_platform", + "is_cuda_available", + "cuda_runtime_version", + "cuda_module_loading", + "nvidia_driver_version", + "nvidia_gpu_models", + "cudnn_version", + "is_xpu_available", + "pip_version", # 'pip' or 'pip3' + "pip_packages", + "conda_packages", + "hip_compiled_version", + "hip_runtime_version", + "miopen_runtime_version", + "caching_allocator_config", + "is_xnnpack_available", + "cpu_info", + ], +) + +COMMON_PATTERNS = [ + "torch", + "numpy", + "triton", + "optree", +] + +NVIDIA_PATTERNS = [ + "cuda-cudart", + "cuda-cupti", + "cuda-libraries", + "cuda-opencl", + "cuda-nvrtc", + "cuda-runtime", + "cublas", + "cudnn", + "cufft", + "curand", + "cusolver", + "cusparse", + "nccl", + "nvjitlink", + "nvtx", +] + +ONEAPI_PATTERNS = [ + "dpcpp-cpp-rt", + "intel-cmplr-lib-rt", + "intel-cmplr-lib-ur", + "intel-cmplr-lic-rt", + "intel-opencl-rt", + "intel-sycl-rt", + "mkl", + "onemkl-sycl-blas", + "onemkl-sycl-dft", + "onemkl-sycl-lapack", + "onemkl-sycl-rng", + "onemkl-sycl-sparse", + "intel-openmp", + "tbb", + "impi-rt", + "impi-devel", + "oneccl", + "oneccl-devel", + "intel-pti", + "umf", + "tcmlib", +] + +CONDA_PATTERNS = [ + "cudatoolkit", + "soumith", + "mkl", + "magma", +] + +PIP_PATTERNS = [ + "mypy", + "flake8", + "onnx", +] + + +def run(command): + """Return (return-code, stdout, stderr).""" + shell = type(command) is str + p = subprocess.Popen( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell + ) + raw_output, raw_err = p.communicate() + rc = p.returncode + if get_platform() == "win32": + enc = "oem" + else: + enc = locale.getpreferredencoding() + output = raw_output.decode(enc) + err = raw_err.decode(enc) + return rc, output.strip(), err.strip() + + +def run_and_read_all(run_lambda, command): + """Run command using run_lambda; reads and returns entire output if rc is 0.""" + rc, out, _ = run_lambda(command) + if rc != 0: + return None + return out + + +def run_and_parse_first_match(run_lambda, command, regex): + """Run command using run_lambda, returns the first regex match if it exists.""" + rc, out, _ = run_lambda(command) + if rc != 0: + return None + match = re.search(regex, out) + if match is None: + return None + return match.group(1) + + +def run_and_return_first_line(run_lambda, command): + """Run command using run_lambda and returns first line if output is not empty.""" + rc, out, _ = run_lambda(command) + if rc != 0: + return None + return out.split("\n")[0] + + +def get_conda_packages(run_lambda, patterns=None): + if patterns is None: + patterns = CONDA_PATTERNS + COMMON_PATTERNS + NVIDIA_PATTERNS + ONEAPI_PATTERNS + conda = os.environ.get("CONDA_EXE", "conda") + out = run_and_read_all(run_lambda, "{} list".format(conda)) + if out is None: + return out + + return "\n".join( + line + for line in out.splitlines() + if not line.startswith("#") and any(name in line for name in patterns) + ) + + +def get_gcc_version(run_lambda): + return run_and_parse_first_match(run_lambda, "gcc --version", r"gcc (.*)") + + +def get_clang_version(run_lambda): + return run_and_parse_first_match( + run_lambda, "clang --version", r"clang version (.*)" + ) + + +def get_cmake_version(run_lambda): + return run_and_parse_first_match(run_lambda, "cmake --version", r"cmake (.*)") + + +def get_nvidia_driver_version(run_lambda): + if get_platform() == "darwin": + cmd = "kextstat | grep -i cuda" + return run_and_parse_first_match( + run_lambda, cmd, r"com[.]nvidia[.]CUDA [(](.*?)[)]" + ) + smi = get_nvidia_smi() + return run_and_parse_first_match(run_lambda, smi, r"Driver Version: (.*?) ") + + +def get_gpu_info(run_lambda): + if get_platform() == "darwin" or ( + TORCH_AVAILABLE + and hasattr(torch.version, "hip") + and torch.version.hip is not None + ): + if TORCH_AVAILABLE and torch.cuda.is_available(): + if torch.version.hip is not None: + prop = torch.cuda.get_device_properties(0) + if hasattr(prop, "gcnArchName"): + gcnArch = " ({})".format(prop.gcnArchName) + else: + gcnArch = "NoGCNArchNameOnOldPyTorch" + else: + gcnArch = "" + return torch.cuda.get_device_name(None) + gcnArch + return None + smi = get_nvidia_smi() + uuid_regex = re.compile(r" \(UUID: .+?\)") + rc, out, _ = run_lambda(smi + " -L") + if rc != 0: + return None + # Anonymize GPUs by removing their UUID + return re.sub(uuid_regex, "", out) + + +def get_running_cuda_version(run_lambda): + return run_and_parse_first_match(run_lambda, "nvcc --version", r"release .+ V(.*)") + + +def get_cudnn_version(run_lambda): + """Return a list of libcudnn.so; it's hard to tell which one is being used.""" + if get_platform() == "win32": + system_root = os.environ.get("SYSTEMROOT", "C:\\Windows") + cuda_path = os.environ.get("CUDA_PATH", "%CUDA_PATH%") + where_cmd = os.path.join(system_root, "System32", "where") + cudnn_cmd = '{} /R "{}\\bin" cudnn*.dll'.format(where_cmd, cuda_path) + elif get_platform() == "darwin": + # CUDA libraries and drivers can be found in /usr/local/cuda/. See + # https://docs.nvidia.com/cuda/archive/9.0/cuda-installation-guide-mac-os-x/index.html#installation + # https://docs.nvidia.com/deeplearning/cudnn/installation/latest/ + # Use CUDNN_LIBRARY when cudnn library is installed elsewhere. + cudnn_cmd = "ls /usr/local/cuda/lib/libcudnn*" + else: + cudnn_cmd = 'ldconfig -p | grep libcudnn | rev | cut -d" " -f1 | rev' + rc, out, _ = run_lambda(cudnn_cmd) + # find will return 1 if there are permission errors or if not found + if len(out) == 0 or (rc != 1 and rc != 0): + l = os.environ.get("CUDNN_LIBRARY") + if l is not None and os.path.isfile(l): + return os.path.realpath(l) + return None + files_set = set() + for fn in out.split("\n"): + fn = os.path.realpath(fn) # eliminate symbolic links + if os.path.isfile(fn): + files_set.add(fn) + if not files_set: + return None + # Alphabetize the result because the order is non-deterministic otherwise + files = sorted(files_set) + if len(files) == 1: + return files[0] + result = "\n".join(files) + return "Probably one of the following:\n{}".format(result) + + +def get_nvidia_smi(): + # Note: nvidia-smi is currently available only on Windows and Linux + smi = "nvidia-smi" + if get_platform() == "win32": + system_root = os.environ.get("SYSTEMROOT", "C:\\Windows") + program_files_root = os.environ.get("PROGRAMFILES", "C:\\Program Files") + legacy_path = os.path.join( + program_files_root, "NVIDIA Corporation", "NVSMI", smi + ) + new_path = os.path.join(system_root, "System32", smi) + smis = [new_path, legacy_path] + for candidate_smi in smis: + if os.path.exists(candidate_smi): + smi = '"{}"'.format(candidate_smi) + break + return smi + + +def _detect_linux_pkg_manager(): + if get_platform() != "linux": + return "N/A" + for mgr_name in ["dpkg", "dnf", "yum", "zypper"]: + rc, _, _ = run(f"which {mgr_name}") + if rc == 0: + return mgr_name + return "N/A" + + +def get_linux_pkg_version(run_lambda, pkg_name): + pkg_mgr = _detect_linux_pkg_manager() + if pkg_mgr == "N/A": + return "N/A" + + grep_version = { + "dpkg": { + "field_index": 2, + "command": "dpkg -l | grep {}", + }, + "dnf": { + "field_index": 1, + "command": "dnf list | grep {}", + }, + "yum": { + "field_index": 1, + "command": "yum list | grep {}", + }, + "zypper": { + "field_index": 2, + "command": "zypper info {} | grep Version", + }, + } + + field_index: int = int(_cast(int, grep_version[pkg_mgr]["field_index"])) + cmd: str = str(grep_version[pkg_mgr]["command"]) + cmd = cmd.format(pkg_name) + ret = run_and_read_all(run_lambda, cmd) + if ret is None or ret == "": + return "N/A" + lst = re.sub(" +", " ", ret).split(" ") + if len(lst) <= field_index: + return "N/A" + return lst[field_index] + + +def get_intel_gpu_driver_version(run_lambda): + lst = [] + platform = get_platform() + if platform == "linux": + pkgs = { # type: ignore[var-annotated] + "dpkg": { + "intel-opencl-icd", + "libze1", + "level-zero", + }, + "dnf": { + "intel-opencl", + "level-zero", + }, + "yum": { + "intel-opencl", + "level-zero", + }, + "zypper": { + "intel-opencl", + "level-zero", + }, + }.get(_detect_linux_pkg_manager(), {}) + for pkg in pkgs: + ver = get_linux_pkg_version(run_lambda, pkg) + if ver != "N/A": + lst.append(f"* {pkg}:\t{ver}") + if platform in ["win32", "cygwin"]: + txt = run_and_read_all( + run_lambda, + 'powershell.exe "gwmi -Class Win32_PnpSignedDriver | where{$_.DeviceClass -eq \\"DISPLAY\\"\ + -and $_.Manufacturer -match \\"Intel\\"} | Select-Object -Property DeviceName,DriverVersion,DriverDate\ + | ConvertTo-Json"', + ) + try: + obj = json.loads(txt) + if type(obj) is list: + for o in obj: + lst.append( + f'* {o["DeviceName"]}: {o["DriverVersion"]} ({o["DriverDate"]})' + ) + else: + lst.append(f'* {obj["DriverVersion"]} ({obj["DriverDate"]})') + except ValueError as e: + lst.append(txt) + lst.append(str(e)) + return "\n".join(lst) + + +def get_intel_gpu_onboard(run_lambda): + lst: list[str] = [] + platform = get_platform() + if platform == "linux": + txt = run_and_read_all(run_lambda, "xpu-smi discovery -j") + if txt: + try: + obj = json.loads(txt) + device_list = obj.get("device_list", []) + if isinstance(device_list, list) and device_list: + lst.extend(f'* {device["device_name"]}' for device in device_list) + else: + lst.append("N/A") + except (ValueError, TypeError) as e: + lst.append(txt) + lst.append(str(e)) + else: + lst.append("N/A") + if platform in ["win32", "cygwin"]: + txt = run_and_read_all( + run_lambda, + 'powershell.exe "gwmi -Class Win32_PnpSignedDriver | where{$_.DeviceClass -eq \\"DISPLAY\\"\ + -and $_.Manufacturer -match \\"Intel\\"} | Select-Object -Property DeviceName | ConvertTo-Json"', + ) + if txt: + try: + obj = json.loads(txt) + if isinstance(obj, list) and obj: + lst.extend(f'* {device["DeviceName"]}' for device in obj) + else: + lst.append(f'* {obj.get("DeviceName", "N/A")}') + except ValueError as e: + lst.append(txt) + lst.append(str(e)) + else: + lst.append("N/A") + return "\n".join(lst) + + +def get_intel_gpu_detected(run_lambda): + if not TORCH_AVAILABLE or not hasattr(torch, "xpu"): + return "N/A" + + device_count = torch.xpu.device_count() + if device_count == 0: + return "N/A" + + devices = [ + f"* [{i}] {torch.xpu.get_device_properties(i)}" for i in range(device_count) + ] + return "\n".join(devices) + + +# example outputs of CPU infos +# * linux +# Architecture: x86_64 +# CPU op-mode(s): 32-bit, 64-bit +# Address sizes: 46 bits physical, 48 bits virtual +# Byte Order: Little Endian +# CPU(s): 128 +# On-line CPU(s) list: 0-127 +# Vendor ID: GenuineIntel +# Model name: Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz +# CPU family: 6 +# Model: 106 +# Thread(s) per core: 2 +# Core(s) per socket: 32 +# Socket(s): 2 +# Stepping: 6 +# BogoMIPS: 5799.78 +# Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr +# sse sse2 ss ht syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl +# xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq monitor ssse3 fma cx16 +# pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand +# hypervisor lahf_lm abm 3dnowprefetch invpcid_single ssbd ibrs ibpb stibp ibrs_enhanced +# fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid avx512f avx512dq rdseed adx smap +# avx512ifma clflushopt clwb avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 +# xsaves wbnoinvd ida arat avx512vbmi pku ospke avx512_vbmi2 gfni vaes vpclmulqdq +# avx512_vnni avx512_bitalg tme avx512_vpopcntdq rdpid md_clear flush_l1d arch_capabilities +# Virtualization features: +# Hypervisor vendor: KVM +# Virtualization type: full +# Caches (sum of all): +# L1d: 3 MiB (64 instances) +# L1i: 2 MiB (64 instances) +# L2: 80 MiB (64 instances) +# L3: 108 MiB (2 instances) +# NUMA: +# NUMA node(s): 2 +# NUMA node0 CPU(s): 0-31,64-95 +# NUMA node1 CPU(s): 32-63,96-127 +# Vulnerabilities: +# Itlb multihit: Not affected +# L1tf: Not affected +# Mds: Not affected +# Meltdown: Not affected +# Mmio stale data: Vulnerable: Clear CPU buffers attempted, no microcode; SMT Host state unknown +# Retbleed: Not affected +# Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl and seccomp +# Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization +# Spectre v2: Mitigation; Enhanced IBRS, IBPB conditional, RSB filling, PBRSB-eIBRS SW sequence +# Srbds: Not affected +# Tsx async abort: Not affected +# * win32 +# Architecture=9 +# CurrentClockSpeed=2900 +# DeviceID=CPU0 +# Family=179 +# L2CacheSize=40960 +# L2CacheSpeed= +# Manufacturer=GenuineIntel +# MaxClockSpeed=2900 +# Name=Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz +# ProcessorType=3 +# Revision=27142 +# +# Architecture=9 +# CurrentClockSpeed=2900 +# DeviceID=CPU1 +# Family=179 +# L2CacheSize=40960 +# L2CacheSpeed= +# Manufacturer=GenuineIntel +# MaxClockSpeed=2900 +# Name=Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz +# ProcessorType=3 +# Revision=27142 + + +def get_cpu_info(run_lambda): + rc, out, err = 0, "", "" + if get_platform() == "linux": + rc, out, err = run_lambda("lscpu") + elif get_platform() == "win32": + rc, out, err = run_lambda( + 'powershell.exe "gwmi -Class Win32_Processor | Select-Object -Property Name,Manufacturer,Family,\ + Architecture,ProcessorType,DeviceID,CurrentClockSpeed,MaxClockSpeed,L2CacheSize,L2CacheSpeed,Revision\ + | ConvertTo-Json"' + ) + if rc == 0: + lst = [] + try: + obj = json.loads(out) + if type(obj) is list: + for o in obj: + lst.append("----------------------") + lst.extend([f"{k}: {v}" for (k, v) in o.items()]) + else: + lst.extend([f"{k}: {v}" for (k, v) in obj.items()]) + except ValueError as e: + lst.append(out) + lst.append(str(e)) + out = "\n".join(lst) + elif get_platform() == "darwin": + rc, out, err = run_lambda("sysctl -n machdep.cpu.brand_string") + cpu_info = "None" + if rc == 0: + cpu_info = out + else: + cpu_info = err + return cpu_info + + +def get_platform(): + if sys.platform.startswith("linux"): + return "linux" + elif sys.platform.startswith("win32"): + return "win32" + elif sys.platform.startswith("cygwin"): + return "cygwin" + elif sys.platform.startswith("darwin"): + return "darwin" + else: + return sys.platform + + +def get_mac_version(run_lambda): + return run_and_parse_first_match(run_lambda, "sw_vers -productVersion", r"(.*)") + + +def get_windows_version(run_lambda): + ret = run_and_read_all( + run_lambda, + 'powershell.exe "gwmi -Class Win32_OperatingSystem | Select-Object -Property Caption,\ + OSArchitecture,Version | ConvertTo-Json"', + ) + try: + obj = json.loads(ret) + ret = f'{obj["Caption"]} ({obj["Version"]} {obj["OSArchitecture"]})' + except ValueError as e: + ret += f"\n{str(e)}" + return ret + + +def get_lsb_version(run_lambda): + return run_and_parse_first_match( + run_lambda, "lsb_release -a", r"Description:\t(.*)" + ) + + +def check_release_file(run_lambda): + return run_and_parse_first_match( + run_lambda, "cat /etc/*-release", r'PRETTY_NAME="(.*)"' + ) + + +def get_os(run_lambda): + from platform import machine + + platform = get_platform() + + if platform in ["win32", "cygwin"]: + return get_windows_version(run_lambda) + + if platform == "darwin": + version = get_mac_version(run_lambda) + if version is None: + return None + return "macOS {} ({})".format(version, machine()) + + if platform == "linux": + # Ubuntu/Debian based + desc = get_lsb_version(run_lambda) + if desc is not None: + return "{} ({})".format(desc, machine()) + + # Try reading /etc/*-release + desc = check_release_file(run_lambda) + if desc is not None: + return "{} ({})".format(desc, machine()) + + return "{} ({})".format(platform, machine()) + + # Unknown platform + return platform + + +def get_python_platform(): + import platform + + return platform.platform() + + +def get_libc_version(): + import platform + + if get_platform() != "linux": + return "N/A" + return "-".join(platform.libc_ver()) + + +def get_pip_packages(run_lambda, patterns=None): + """Return `pip list` output. Note: will also find conda-installed pytorch and numpy packages.""" + if patterns is None: + patterns = PIP_PATTERNS + COMMON_PATTERNS + NVIDIA_PATTERNS + ONEAPI_PATTERNS + + pip_version = "pip3" if sys.version_info.major == 3 else "pip" + + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + # People generally have pip as `pip` or `pip3` + # But here it is invoked as `python -mpip` + out = run_and_read_all( + run_lambda, [sys.executable, "-mpip", "list", "--format=freeze"] + ) + if out is None: + return pip_version, out + + filtered_out = "\n".join( + line for line in out.splitlines() if any(name in line for name in patterns) + ) + + return pip_version, filtered_out + + +def get_cachingallocator_config() -> _Dict[str, str]: + """Return the caching allocator configuration from environment variables. + """ + # pyrefly: ignore [bad-return] + return { + var: os.environ.get(var) + for var in ( + "PYTORCH_CUDA_ALLOC_CONF", + "PYTORCH_HIP_ALLOC_CONF", + "PYTORCH_ALLOC_CONF", + ) + if os.environ.get(var) + } + + +def get_cuda_module_loading_config(): + if TORCH_AVAILABLE and torch.cuda.is_available(): + torch.cuda.init() + config = os.environ.get("CUDA_MODULE_LOADING", "") + return config + else: + return "N/A" + + +def is_xnnpack_available(): + if TORCH_AVAILABLE: + import torch.backends.xnnpack + + return str(torch.backends.xnnpack.enabled) # type: ignore[attr-defined] + else: + return "N/A" + + +def get_env_info(): + """ + Collects environment information to aid in debugging. + + The returned environment information contains details on torch version, is debug build + or not, cuda compiled version, gcc version, clang version, cmake version, operating + system, libc version, python version, python platform, CUDA availability, CUDA + runtime version, CUDA module loading config, GPU model and configuration, Nvidia + driver version, cuDNN version, pip version and versions of relevant pip and + conda packages, HIP runtime version, MIOpen runtime version, + Caching allocator config, XNNPACK availability and CPU information. + + Returns: + SystemEnv (namedtuple): A tuple containing various environment details + and system information. + """ + run_lambda = run + pip_version, pip_list_output = get_pip_packages(run_lambda) + + if TORCH_AVAILABLE: + version_str = torch.__version__ + debug_mode_str = str(torch.version.debug) + cuda_available_str = str(torch.cuda.is_available()) + cuda_version_str = torch.version.cuda + xpu_available_str = str(torch.xpu.is_available()) + if torch.xpu.is_available(): + xpu_available_str = ( + f"{xpu_available_str}\n" + + f"XPU used to build PyTorch: {torch.version.xpu}\n" + + f"Intel GPU driver version:\n{get_intel_gpu_driver_version(run_lambda)}\n" + + f"Intel GPU models onboard:\n{get_intel_gpu_onboard(run_lambda)}\n" + + f"Intel GPU models detected:\n{get_intel_gpu_detected(run_lambda)}" + ) + if ( + not hasattr(torch.version, "hip") or torch.version.hip is None + ): # cuda version + hip_compiled_version = hip_runtime_version = miopen_runtime_version = "N/A" + else: # HIP version + + def get_version_or_na(cfg, prefix): + _lst = [s.rsplit(None, 1)[-1] for s in cfg if prefix in s] + return _lst[0] if _lst else "N/A" + + cfg = torch._C._show_config().split("\n") + hip_runtime_version = get_version_or_na(cfg, "HIP Runtime") + miopen_runtime_version = get_version_or_na(cfg, "MIOpen") + cuda_version_str = "N/A" + hip_compiled_version = torch.version.hip + else: + version_str = debug_mode_str = cuda_available_str = cuda_version_str = xpu_available_str = "N/A" # type: ignore[assignment] + hip_compiled_version = hip_runtime_version = miopen_runtime_version = "N/A" + + sys_version = sys.version.replace("\n", " ") + + conda_packages = get_conda_packages(run_lambda) + + return SystemEnv( + torch_version=version_str, + is_debug_build=debug_mode_str, + python_version="{} ({}-bit runtime)".format( + sys_version, sys.maxsize.bit_length() + 1 + ), + python_platform=get_python_platform(), + is_cuda_available=cuda_available_str, + cuda_compiled_version=cuda_version_str, + cuda_runtime_version=get_running_cuda_version(run_lambda), + cuda_module_loading=get_cuda_module_loading_config(), + nvidia_gpu_models=get_gpu_info(run_lambda), + nvidia_driver_version=get_nvidia_driver_version(run_lambda), + cudnn_version=get_cudnn_version(run_lambda), + is_xpu_available=xpu_available_str, + hip_compiled_version=hip_compiled_version, + hip_runtime_version=hip_runtime_version, + miopen_runtime_version=miopen_runtime_version, + pip_version=pip_version, + pip_packages=pip_list_output, + conda_packages=conda_packages, + os=get_os(run_lambda), + libc_version=get_libc_version(), + gcc_version=get_gcc_version(run_lambda), + clang_version=get_clang_version(run_lambda), + cmake_version=get_cmake_version(run_lambda), + caching_allocator_config=get_cachingallocator_config(), + is_xnnpack_available=is_xnnpack_available(), + cpu_info=get_cpu_info(run_lambda), + ) + + +env_info_fmt = """ +PyTorch version: {torch_version} +Is debug build: {is_debug_build} +CUDA used to build PyTorch: {cuda_compiled_version} +ROCM used to build PyTorch: {hip_compiled_version} + +OS: {os} +GCC version: {gcc_version} +Clang version: {clang_version} +CMake version: {cmake_version} +Libc version: {libc_version} + +Python version: {python_version} +Python platform: {python_platform} +Is CUDA available: {is_cuda_available} +CUDA runtime version: {cuda_runtime_version} +CUDA_MODULE_LOADING set to: {cuda_module_loading} +GPU models and configuration: {nvidia_gpu_models} +Nvidia driver version: {nvidia_driver_version} +cuDNN version: {cudnn_version} +Is XPU available: {is_xpu_available} +HIP runtime version: {hip_runtime_version} +MIOpen runtime version: {miopen_runtime_version} +Is XNNPACK available: {is_xnnpack_available} +Caching allocator config: {caching_allocator_config} + +CPU: +{cpu_info} + +Versions of relevant libraries: +{pip_packages} +{conda_packages} +""".strip() + + +def pretty_str(envinfo): + def replace_nones(dct, replacement="Could not collect"): + for key in dct: + if dct[key] is not None: + continue + dct[key] = replacement + return dct + + def replace_bools(dct, true="Yes", false="No"): + for key in dct: + if dct[key] is True: + dct[key] = true + elif dct[key] is False: + dct[key] = false + return dct + + def prepend(text, tag="[prepend]"): + lines = text.split("\n") + updated_lines = [tag + line for line in lines] + return "\n".join(updated_lines) + + def replace_if_empty(text, replacement="No relevant packages"): + if text is not None and len(text) == 0: + return replacement + return text + + def maybe_start_on_next_line(string): + # If `string` is multiline, prepend a \n to it. + if string is not None and len(string.split("\n")) > 1: + return "\n{}\n".format(string) + return string + + mutable_dict = envinfo._asdict() + + # If nvidia_gpu_models is multiline, start on the next line + mutable_dict["nvidia_gpu_models"] = maybe_start_on_next_line( + envinfo.nvidia_gpu_models + ) + + # If the machine doesn't have CUDA, report some fields as 'No CUDA' + dynamic_cuda_fields = [ + "cuda_runtime_version", + "nvidia_gpu_models", + "nvidia_driver_version", + ] + all_cuda_fields = dynamic_cuda_fields + ["cudnn_version"] + all_dynamic_cuda_fields_missing = all( + mutable_dict[field] is None for field in dynamic_cuda_fields + ) + if ( + TORCH_AVAILABLE + and not torch.cuda.is_available() + and all_dynamic_cuda_fields_missing + ): + for field in all_cuda_fields: + mutable_dict[field] = "No CUDA" + if envinfo.cuda_compiled_version is None: + mutable_dict["cuda_compiled_version"] = "None" + + # Replace True with Yes, False with No + mutable_dict = replace_bools(mutable_dict) + + # Replace all None objects with 'Could not collect' + mutable_dict = replace_nones(mutable_dict) + + # If either of these are '', replace with 'No relevant packages' + mutable_dict["pip_packages"] = replace_if_empty(mutable_dict["pip_packages"]) + mutable_dict["conda_packages"] = replace_if_empty(mutable_dict["conda_packages"]) + + # Tag conda and pip packages with a prefix + # If they were previously None, they'll show up as ie '[conda] Could not collect' + if mutable_dict["pip_packages"]: + mutable_dict["pip_packages"] = prepend( + mutable_dict["pip_packages"], "[{}] ".format(envinfo.pip_version) + ) + if mutable_dict["conda_packages"]: + mutable_dict["conda_packages"] = prepend( + mutable_dict["conda_packages"], "[conda] " + ) + mutable_dict["cpu_info"] = envinfo.cpu_info + mutable_dict["caching_allocator_config"] = envinfo.caching_allocator_config + if not envinfo.caching_allocator_config: + mutable_dict["caching_allocator_config"] = "N/A" + return env_info_fmt.format(**mutable_dict) + + +def get_pretty_env_info(): + """ + Returns a pretty string of environment information. + + This function retrieves environment information by calling the `get_env_info` function + and then formats the information into a human-readable string. The retrieved environment + information is listed in the document of `get_env_info`. + This function is used in `python collect_env.py` that should be executed when reporting a bug. + + Returns: + str: A pretty string of the environment information. + """ + return pretty_str(get_env_info()) + + +def main() -> None: + print("Collecting environment information...") + output = get_pretty_env_info() + print(output) + + if ( + TORCH_AVAILABLE + and hasattr(torch, "utils") + and hasattr(torch.utils, "_crash_handler") + ): + minidump_dir = torch.utils._crash_handler.DEFAULT_MINIDUMP_DIR + if sys.platform == "linux" and os.path.exists(minidump_dir): + dumps = [ + os.path.join(minidump_dir, dump) for dump in os.listdir(minidump_dir) + ] + latest = max(dumps, key=os.path.getctime) + ctime = os.path.getctime(latest) + creation_time = datetime.datetime.fromtimestamp(ctime).strftime( + "%Y-%m-%d %H:%M:%S" + ) + msg = ( + "\n*** Detected a minidump at {} created on {}, ".format( + latest, creation_time + ) + + "if this is related to your bug please include it when you file a report ***" + ) + print(msg, file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/cpp_backtrace.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/cpp_backtrace.py new file mode 100644 index 0000000000000000000000000000000000000000..af4a7fcb63e263038255359c946a6a0d4a21dbd0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/cpp_backtrace.py @@ -0,0 +1,12 @@ +# mypy: allow-untyped-defs +from torch._C import _get_cpp_backtrace + +def get_cpp_backtrace(frames_to_skip=0, maximum_number_of_frames=64) -> str: + r""" + Return a string containing the C++ stack trace of the current thread. + + Args: + frames_to_skip (int): the number of frames to skip from the top of the stack + maximum_number_of_frames (int): the maximum number of frames to return + """ + return _get_cpp_backtrace(frames_to_skip, maximum_number_of_frames) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/cpp_extension.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/cpp_extension.py new file mode 100644 index 0000000000000000000000000000000000000000..f29c382f0e3f30dc2472043f08273229c1a726c2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/cpp_extension.py @@ -0,0 +1,3115 @@ +# mypy: allow-untyped-defs +import copy +import glob +import importlib +import importlib.abc +import os +import re +import shlex +import shutil +import setuptools +import subprocess +import sys +import sysconfig +import types +import collections +from pathlib import Path +import errno +import logging + +logger = logging.getLogger(__name__) + +import torch +import torch._appdirs +from .file_baton import FileBaton +from ._cpp_extension_versioner import ExtensionVersioner +from typing_extensions import deprecated +from torch.torch_version import TorchVersion, Version + + +from setuptools.command.build_ext import build_ext + +IS_WINDOWS = sys.platform == 'win32' +IS_MACOS = sys.platform.startswith('darwin') +IS_LINUX = sys.platform.startswith('linux') +LIB_EXT = '.pyd' if IS_WINDOWS else '.so' +EXEC_EXT = '.exe' if IS_WINDOWS else '' +CLIB_PREFIX = '' if IS_WINDOWS else 'lib' +CLIB_EXT = '.dll' if IS_WINDOWS else '.so' +SHARED_FLAG = '/DLL' if IS_WINDOWS else '-shared' + +_HERE = os.path.abspath(__file__) +_TORCH_PATH = os.path.dirname(os.path.dirname(_HERE)) +TORCH_LIB_PATH = os.path.join(_TORCH_PATH, 'lib') + + +SUBPROCESS_DECODE_ARGS = ('oem',) if IS_WINDOWS else () +MINIMUM_GCC_VERSION = (5, 0, 0) +MINIMUM_MSVC_VERSION = (19, 0, 24215) + +VersionRange = tuple[tuple[int, ...], tuple[int, ...]] +VersionMap = dict[str, VersionRange] +# The following values were taken from the following GitHub gist that +# summarizes the minimum valid major versions of g++/clang++ for each supported +# CUDA version: https://gist.github.com/ax3l/9489132 +# Or from include/crt/host_config.h in the CUDA SDK +# The second value is the exclusive(!) upper bound, i.e. min <= version < max +CUDA_GCC_VERSIONS: VersionMap = { + '11.0': (MINIMUM_GCC_VERSION, (10, 0)), + '11.1': (MINIMUM_GCC_VERSION, (11, 0)), + '11.2': (MINIMUM_GCC_VERSION, (11, 0)), + '11.3': (MINIMUM_GCC_VERSION, (11, 0)), + '11.4': ((6, 0, 0), (12, 0)), + '11.5': ((6, 0, 0), (12, 0)), + '11.6': ((6, 0, 0), (12, 0)), + '11.7': ((6, 0, 0), (12, 0)), +} + +MINIMUM_CLANG_VERSION = (3, 3, 0) +CUDA_CLANG_VERSIONS: VersionMap = { + '11.1': (MINIMUM_CLANG_VERSION, (11, 0)), + '11.2': (MINIMUM_CLANG_VERSION, (12, 0)), + '11.3': (MINIMUM_CLANG_VERSION, (12, 0)), + '11.4': (MINIMUM_CLANG_VERSION, (13, 0)), + '11.5': (MINIMUM_CLANG_VERSION, (13, 0)), + '11.6': (MINIMUM_CLANG_VERSION, (14, 0)), + '11.7': (MINIMUM_CLANG_VERSION, (14, 0)), +} + +__all__ = ["get_default_build_root", "check_compiler_ok_for_platform", "get_compiler_abi_compatibility_and_version", "BuildExtension", + "CppExtension", "CUDAExtension", "SyclExtension", "include_paths", "library_paths", "load", "load_inline", "is_ninja_available", + "verify_ninja_availability", "remove_extension_h_precompiler_headers", "get_cxx_compiler", "check_compiler_is_gcc"] +# Taken directly from python stdlib < 3.9 +# See https://github.com/pytorch/pytorch/issues/48617 +def _nt_quote_args(args: list[str] | None) -> list[str]: + """Quote command-line arguments for DOS/Windows conventions. + + Just wraps every argument which contains blanks in double quotes, and + returns a new argument list. + """ + # Cover None-type + if not args: + return [] + return [f'"{arg}"' if ' ' in arg else arg for arg in args] + +def _find_cuda_home() -> str | None: + """Find the CUDA install path.""" + # Guess #1 + cuda_home = os.environ.get('CUDA_HOME') or os.environ.get('CUDA_PATH') + if cuda_home is None: + # Guess #2 + nvcc_path = shutil.which("nvcc") + if nvcc_path is not None: + cuda_home = os.path.dirname(os.path.dirname(nvcc_path)) + else: + # Guess #3 + if IS_WINDOWS: + cuda_homes = glob.glob( + 'C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v*.*') + if len(cuda_homes) == 0: + cuda_home = '' + else: + cuda_home = cuda_homes[0] + else: + cuda_home = '/usr/local/cuda' + if not os.path.exists(cuda_home): + cuda_home = None + if cuda_home and not torch.cuda.is_available(): + logger.warning("No CUDA runtime is found, using CUDA_HOME='%s'", cuda_home) + return cuda_home + +def _find_rocm_home() -> str | None: + """Find the ROCm install path.""" + # Guess #1 + rocm_home = os.environ.get('ROCM_HOME') or os.environ.get('ROCM_PATH') + if rocm_home is None: + # Guess #2 + hipcc_path = shutil.which('hipcc') + if hipcc_path is not None: + rocm_home = os.path.dirname(os.path.dirname( + os.path.realpath(hipcc_path))) + # can be either /hip/bin/hipcc or /bin/hipcc + if os.path.basename(rocm_home) == 'hip': + rocm_home = os.path.dirname(rocm_home) + else: + # Guess #3 + fallback_path = '/opt/rocm' + if os.path.exists(fallback_path): + rocm_home = fallback_path + if rocm_home and torch.version.hip is None: + logger.warning("No ROCm runtime is found, using ROCM_HOME='%s'", rocm_home) + return rocm_home + +def _find_sycl_home() -> str | None: + sycl_home = None + icpx_path = shutil.which('icpx') + # Guess 1: for source code build developer/user, we'll have icpx in PATH, + # which will tell us the SYCL_HOME location. + if icpx_path is not None: + sycl_home = os.path.dirname(os.path.dirname( + os.path.realpath(icpx_path))) + + # Guess 2: for users install Pytorch with XPU support, the sycl runtime is + # inside intel-sycl-rt, which is automatically installed via pip dependency. + else: + try: + files = importlib.metadata.files('intel-sycl-rt') or [] + for f in files: + if f.name == "libsycl.so": + sycl_home = os.path.dirname(Path(f.locate()).parent.resolve()) + break + except importlib.metadata.PackageNotFoundError: + logger.warning("Trying to find SYCL_HOME from intel-sycl-rt package, but it is not installed.") + return sycl_home + +def _join_rocm_home(*paths) -> str: + """ + Join paths with ROCM_HOME, or raises an error if it ROCM_HOME is not set. + + This is basically a lazy way of raising an error for missing $ROCM_HOME + only once we need to get any ROCm-specific path. + """ + if ROCM_HOME is None: + raise OSError('ROCM_HOME environment variable is not set. ' + 'Please set it to your ROCm install root.') + return os.path.join(ROCM_HOME, *paths) + +def _join_sycl_home(*paths) -> str: + """ + Join paths with SYCL_HOME, or raises an error if it SYCL_HOME is not found. + + This is basically a lazy way of raising an error for missing SYCL_HOME + only once we need to get any SYCL-specific path. + """ + if SYCL_HOME is None: + raise OSError('SYCL runtime is not dected. Please setup the pytorch ' + 'prerequisites for Intel GPU following the instruction in ' + 'https://github.com/pytorch/pytorch?tab=readme-ov-file#intel-gpu-support ' + 'or install intel-sycl-rt via pip.') + + return os.path.join(SYCL_HOME, *paths) + + + +ABI_INCOMPATIBILITY_WARNING = ( + " !! WARNING !!" + "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + "Your compiler (%s) may be ABI-incompatible with PyTorch!" + "Please use a compiler that is ABI-compatible with GCC 5.0 and above." + "See https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html." + "See https://gist.github.com/goldsborough/d466f43e8ffc948ff92de7486c5216d6" + "for instructions on how to install GCC 5 or higher." + "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + " !! WARNING !!" +) +WRONG_COMPILER_WARNING = ( + " !! WARNING !!" + "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + "Your compiler (%s) is not compatible with the compiler Pytorch was" + "built with for this platform, which is %s on %s. Please" + "use %s to compile your extension. Alternatively, you may" + "compile PyTorch from source using %s, and then you can also use" + "%s to compile your extension." + "See https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md for help" + "with compiling PyTorch from source." + "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + " !! WARNING !!" +) +CUDA_MISMATCH_MESSAGE = ( + "The detected CUDA version (%s) mismatches the version that was used to compile" + "PyTorch (%s). Please make sure to use the same CUDA versions." +) +CUDA_MISMATCH_WARN = ( + "The detected CUDA version (%s) has a minor version mismatch with the version that was used to compile PyTorch (%s). Most likely this shouldn't be a problem." +) +CUDA_NOT_FOUND_MESSAGE = ( + "CUDA was not found on the system, please set the CUDA_HOME or the CUDA_PATH" + "environment variable or add NVCC to your system PATH. The extension compilation will fail." +) +ROCM_HOME = _find_rocm_home() if (torch.cuda._is_compiled() and torch.version.hip) else None +HIP_HOME = _join_rocm_home('hip') if ROCM_HOME else None +IS_HIP_EXTENSION = bool(ROCM_HOME is not None and torch.version.hip is not None) +ROCM_VERSION = None +if torch.version.hip is not None: + ROCM_VERSION = tuple(int(v) for v in torch.version.hip.split('.')[:2]) + +CUDA_HOME = _find_cuda_home() if (torch.cuda._is_compiled() and torch.version.cuda) else None +CUDNN_HOME = os.environ.get('CUDNN_HOME') or os.environ.get('CUDNN_PATH') +SYCL_HOME = _find_sycl_home() if torch.xpu._is_compiled() else None +WINDOWS_CUDA_HOME = os.environ.get('WINDOWS_CUDA_HOME') # used for AOTI cross-compilation + +# PyTorch releases have the version pattern major.minor.patch, whereas when +# PyTorch is built from source, we append the git commit hash, which gives +# it the below pattern. +BUILT_FROM_SOURCE_VERSION_PATTERN = re.compile(r'\d+\.\d+\.\d+\w+\+\w+') + +COMMON_MSVC_FLAGS = ['/MD', '/wd4819', '/wd4251', '/wd4244', '/wd4267', '/wd4275', '/wd4018', '/wd4190', '/wd4624', '/wd4067', '/wd4068', '/EHsc'] + +MSVC_IGNORE_CUDAFE_WARNINGS = [ + 'base_class_has_different_dll_interface', + 'field_without_dll_interface', + 'dll_interface_conflict_none_assumed', + 'dll_interface_conflict_dllexport_assumed' +] + +COMMON_NVCC_FLAGS = [ + '-D__CUDA_NO_HALF_OPERATORS__', + '-D__CUDA_NO_HALF_CONVERSIONS__', + '-D__CUDA_NO_BFLOAT16_CONVERSIONS__', + '-D__CUDA_NO_HALF2_OPERATORS__', + '--expt-relaxed-constexpr' +] + +COMMON_HIP_FLAGS = [ + '-D__HIP_PLATFORM_AMD__=1', + '-DUSE_ROCM=1', + '-DHIPBLAS_V2', +] + +if not IS_WINDOWS: + COMMON_HIP_FLAGS.append('-fPIC') + +COMMON_HIPCC_FLAGS = [ + '-DCUDA_HAS_FP16=1', + '-D__HIP_NO_HALF_OPERATORS__=1', + '-D__HIP_NO_HALF_CONVERSIONS__=1', + '-DHIP_ENABLE_WARP_SYNC_BUILTINS=1' +] + +if IS_WINDOWS: + # Compatibility flags, similar to those set in cmake/Dependencies.cmake. + COMMON_HIPCC_FLAGS.append('-fms-extensions') + # Suppress warnings about dllexport. + COMMON_HIPCC_FLAGS.append('-Wno-ignored-attributes') + + +def _get_icpx_version() -> str: + icpx = 'icx' if IS_WINDOWS else 'icpx' + compiler_info = subprocess.check_output([icpx, '--version']) + match = re.search(r'(\d+)\.(\d+)\.(\d+)', compiler_info.decode().strip()) + version = ['0', '0', '0'] if match is None else list(match.groups()) + version = list(map(int, version)) + if len(version) != 3: + raise AssertionError("Failed to parse DPC++ compiler version") + # Aligning version format with what torch.version.xpu() returns + return f"{version[0]}{version[1]:02}{version[2]:02}" + + +def _get_sycl_arch_list(): + if 'TORCH_XPU_ARCH_LIST' in os.environ: + return os.environ.get('TORCH_XPU_ARCH_LIST') + arch_list = torch.xpu.get_arch_list() + # Dropping dg2* archs since they lack hardware support for fp64 and require + # special consideration from the user. If needed these platforms can + # be requested thru TORCH_XPU_ARCH_LIST environment variable. + arch_list = [x for x in arch_list if not x.startswith('dg2')] + return ','.join(arch_list) + + +# If arch list returned by _get_sycl_arch_list() is empty, then sycl kernels will be compiled +# for default spir64 target and avoid device specific compilations entirely. Further, kernels +# will be JIT compiled at runtime. +def _append_sycl_targets_if_missing(cflags) -> None: + if any(flag.startswith('-fsycl-targets=') for flag in cflags): + # do nothing: user has manually specified sycl targets + return + if _get_sycl_arch_list() != '': + # AOT (spir64_gen) + JIT (spir64) + cflags.append('-fsycl-targets=spir64_gen,spir64') + else: + # JIT (spir64) + cflags.append('-fsycl-targets=spir64') + +def _get_sycl_device_flags(cflags): + # We need last occurrence of -fsycl-targets as it will be the one taking effect. + # So searching in reversed list. + flags = [f for f in reversed(cflags) if f.startswith('-fsycl-targets=')] + if not flags: + raise AssertionError("bug: -fsycl-targets should have been amended to cflags") + + arch_list = _get_sycl_arch_list() + if arch_list != '': + flags += [f'-Xs "-device {arch_list}"'] + return flags + +_COMMON_SYCL_FLAGS = [ + '-fsycl', +] + +_SYCL_DLINK_FLAGS = [ + *_COMMON_SYCL_FLAGS, + '-fsycl-link', + '--offload-compress', +] + +JIT_EXTENSION_VERSIONER = ExtensionVersioner() + +PLAT_TO_VCVARS = { + 'win32' : 'x86', + 'win-amd64' : 'x86_amd64', +} + +min_supported_cpython = "0x030A0000" # Python 3.10 hexcode + +def get_cxx_compiler(): + if IS_WINDOWS: + compiler = os.environ.get('CXX', 'cl') + else: + compiler = os.environ.get('CXX', 'c++') + return compiler + +def _is_binary_build() -> bool: + return not BUILT_FROM_SOURCE_VERSION_PATTERN.match(torch.version.__version__) + + +def _accepted_compilers_for_platform() -> list[str]: + # gnu-c++ and gnu-cc are the conda gcc compilers + return ['clang++', 'clang'] if IS_MACOS else ['g++', 'gcc', 'gnu-c++', 'gnu-cc', 'clang++', 'clang'] + +def _maybe_write(filename, new_content) -> None: + r''' + Equivalent to writing the content into the file but will not touch the file + if it already had the right content (to avoid triggering recompile). + ''' + if os.path.exists(filename): + with open(filename) as f: + content = f.read() + + if content == new_content: + # The file already contains the right thing! + return + + with open(filename, 'w') as source_file: + source_file.write(new_content) + +def get_default_build_root() -> str: + """ + Return the path to the root folder under which extensions will built. + + For each extension module built, there will be one folder underneath the + folder returned by this function. For example, if ``p`` is the path + returned by this function and ``ext`` the name of an extension, the build + folder for the extension will be ``p/ext``. + + This directory is **user-specific** so that multiple users on the same + machine won't meet permission issues. + """ + return os.path.realpath(torch._appdirs.user_cache_dir(appname='torch_extensions')) + + +def check_compiler_ok_for_platform(compiler: str) -> bool: + """ + Verify that the compiler is the expected one for the current platform. + + Args: + compiler (str): The compiler executable to check. + + Returns: + True if the compiler is gcc/g++ on Linux or clang/clang++ on macOS, + and always True for Windows. + """ + if IS_WINDOWS: + return True + compiler_path = shutil.which(compiler) + if compiler_path is None: + return False + # Use os.path.realpath to resolve any symlinks, in particular from 'c++' to e.g. 'g++'. + compiler_path = os.path.realpath(compiler_path) + # Check the compiler name + if any(name in compiler_path for name in _accepted_compilers_for_platform()): + return True + # If compiler wrapper is used try to infer the actual compiler by invoking it with -v flag + env = os.environ.copy() + env['LC_ALL'] = 'C' # Don't localize output + try: + version_string = subprocess.check_output([compiler, '-v'], stderr=subprocess.STDOUT, env=env).decode(*SUBPROCESS_DECODE_ARGS) + except subprocess.CalledProcessError: + # If '-v' fails, try '--version' + version_string = subprocess.check_output([compiler, '--version'], stderr=subprocess.STDOUT, env=env).decode(*SUBPROCESS_DECODE_ARGS) + if IS_LINUX: + # Check for 'gcc' or 'g++' for sccache wrapper + pattern = re.compile("^COLLECT_GCC=(.*)$", re.MULTILINE) + results = re.findall(pattern, version_string) + if len(results) != 1: + # Clang is also a supported compiler on Linux + # Though on Ubuntu it's sometimes called "Ubuntu clang version" + return 'clang version' in version_string + compiler_path = os.path.realpath(results[0].strip()) + # On RHEL/CentOS c++ is a gcc compiler wrapper + if os.path.basename(compiler_path) == 'c++' and 'gcc version' in version_string: + return True + return any(name in compiler_path for name in _accepted_compilers_for_platform()) + if IS_MACOS: + # Check for 'clang' or 'clang++' + return version_string.startswith("Apple clang") + return False + + +def get_compiler_abi_compatibility_and_version(compiler) -> tuple[bool, TorchVersion]: + """ + Determine if the given compiler is ABI-compatible with PyTorch alongside its version. + + Args: + compiler (str): The compiler executable name to check (e.g. ``g++``). + Must be executable in a shell process. + + Returns: + A tuple that contains a boolean that defines if the compiler is (likely) ABI-incompatible with PyTorch, + followed by a `TorchVersion` string that contains the compiler version separated by dots. + """ + if not _is_binary_build(): + return (True, TorchVersion('0.0.0')) + if os.environ.get('TORCH_DONT_CHECK_COMPILER_ABI') in ['ON', '1', 'YES', 'TRUE', 'Y']: + return (True, TorchVersion('0.0.0')) + + # First check if the compiler is one of the expected ones for the particular platform. + if not check_compiler_ok_for_platform(compiler): + logger.warning(WRONG_COMPILER_WARNING, compiler, _accepted_compilers_for_platform()[0], sys.platform, _accepted_compilers_for_platform()[0]) + return (False, TorchVersion('0.0.0')) + + if IS_MACOS: + # There is no particular minimum version we need for clang, so we're good here. + return (True, TorchVersion('0.0.0')) + try: + if IS_LINUX: + minimum_required_version = MINIMUM_GCC_VERSION + compiler_info = subprocess.check_output([compiler, '-dumpfullversion', '-dumpversion']) + else: + minimum_required_version = MINIMUM_MSVC_VERSION + compiler_info = subprocess.check_output(compiler, stderr=subprocess.STDOUT) + match = re.search(r'(\d+)\.(\d+)\.(\d+)', compiler_info.decode(*SUBPROCESS_DECODE_ARGS).strip()) + version = ['0', '0', '0'] if match is None else list(match.groups()) + except Exception: + _, error, _ = sys.exc_info() + logger.warning('Error checking compiler version for %s: %s', compiler, error) + return (False, TorchVersion('0.0.0')) + + # convert alphanumeric string to numeric string + # amdclang++ returns str like 0.0.0git, others return 0.0.0 + numeric_version = [re.sub(r'\D', '', v) for v in version] + + if tuple(map(int, numeric_version)) >= minimum_required_version: + return (True, TorchVersion('.'.join(numeric_version))) + + compiler = f'{compiler} {".".join(numeric_version)}' + logger.warning(ABI_INCOMPATIBILITY_WARNING, compiler) + + return (False, TorchVersion('.'.join(numeric_version))) + + +def _check_cuda_version(compiler_name: str, compiler_version: TorchVersion) -> None: + if not CUDA_HOME: + raise RuntimeError(CUDA_NOT_FOUND_MESSAGE) + + nvcc = os.path.join(CUDA_HOME, 'bin', 'nvcc.exe' if IS_WINDOWS else 'nvcc') + if not os.path.exists(nvcc): + raise FileNotFoundError(f"nvcc not found at '{nvcc}'. Ensure CUDA path '{CUDA_HOME}' is correct.") + + cuda_version_str = subprocess.check_output([nvcc, '--version']).strip().decode(*SUBPROCESS_DECODE_ARGS) + cuda_version = re.search(r'release (\d+[.]\d+)', cuda_version_str) + if cuda_version is None: + return + + cuda_str_version = cuda_version.group(1) + cuda_ver = Version(cuda_str_version) + if torch.version.cuda is None: + return + + torch_cuda_version = Version(torch.version.cuda) + if cuda_ver != torch_cuda_version: + # major/minor attributes are only available in setuptools>=49.4.0 + if getattr(cuda_ver, "major", None) is None: + raise ValueError("setuptools>=49.4.0 is required") + if cuda_ver.major != torch_cuda_version.major: + raise RuntimeError(CUDA_MISMATCH_MESSAGE, cuda_str_version, torch.version.cuda) + logger.warning(CUDA_MISMATCH_WARN, cuda_str_version, torch.version.cuda) + + if not (sys.platform.startswith('linux') and + os.environ.get('TORCH_DONT_CHECK_COMPILER_ABI') not in ['ON', '1', 'YES', 'TRUE', 'Y'] and + _is_binary_build()): + return + + cuda_compiler_bounds: VersionMap = CUDA_CLANG_VERSIONS if compiler_name.startswith('clang') else CUDA_GCC_VERSIONS + + if cuda_str_version not in cuda_compiler_bounds: + logger.warning('There are no %s version bounds defined for CUDA version %s', compiler_name, cuda_str_version) + else: + min_compiler_version, max_excl_compiler_version = cuda_compiler_bounds[cuda_str_version] + # Special case for 11.4.0, which has lower compiler bounds than 11.4.1 + if "V11.4.48" in cuda_version_str and cuda_compiler_bounds == CUDA_GCC_VERSIONS: + max_excl_compiler_version = (11, 0) + min_compiler_version_str = '.'.join(map(str, min_compiler_version)) + max_excl_compiler_version_str = '.'.join(map(str, max_excl_compiler_version)) + + version_bound_str = f'>={min_compiler_version_str}, <{max_excl_compiler_version_str}' + + if compiler_version < TorchVersion(min_compiler_version_str): + raise RuntimeError( + f'The current installed version of {compiler_name} ({compiler_version}) is less ' + f'than the minimum required version by CUDA {cuda_str_version} ({min_compiler_version_str}). ' + f'Please make sure to use an adequate version of {compiler_name} ({version_bound_str}).' + ) + if compiler_version >= TorchVersion(max_excl_compiler_version_str): + raise RuntimeError( + f'The current installed version of {compiler_name} ({compiler_version}) is greater ' + f'than the maximum required version by CUDA {cuda_str_version}. ' + f'Please make sure to use an adequate version of {compiler_name} ({version_bound_str}).' + ) + + +# Specify Visual Studio C runtime library for hipcc +def _set_hipcc_runtime_lib(is_standalone, debug) -> None: + if is_standalone: + if debug: + COMMON_HIP_FLAGS.append('-fms-runtime-lib=static_dbg') + else: + COMMON_HIP_FLAGS.append('-fms-runtime-lib=static') + else: + if debug: + COMMON_HIP_FLAGS.append('-fms-runtime-lib=dll_dbg') + else: + COMMON_HIP_FLAGS.append('-fms-runtime-lib=dll') + +def _append_sycl_std_if_no_std_present(cflags) -> None: + if not any(flag.startswith('-sycl-std=') for flag in cflags): + cflags.append('-sycl-std=2020') + + +def _wrap_sycl_host_flags(cflags): + host_cflags = [] + host_cxx = get_cxx_compiler() + if IS_WINDOWS: + for flag in cflags: + if flag.startswith("-I"): + flag = flag.replace("\\", "\\\\").replace("-I", "/I") + else: + flag = flag.replace("-D", "/D") + flag = flag.replace('"', '\\"') + host_cflags.append(flag) + joined_host_cflags = ' '.join(host_cflags) + + external_include = _join_sycl_home("include").replace("\\", "\\\\") + + # Some versions of DPC++ compiler pass paths to SYCL headers as user include paths (`-I`) rather + # than system paths (`-isystem`). This makes host compiler to report warnings encountered in the + # SYCL headers, such as deprecated warnings, even if warmed API is not actually used in the program. + # We expect that this issue will be addressed in the later version of DPC++ compiler. To workaround the + # issue now we wrap paths to SYCL headers in `/external:I`. Warning free compilation is especially important + # for Windows build as `/sdl` compilation flag assumes that and we will fail compilation otherwise. + wrapped_host_cflags = [ + f"-fsycl-host-compiler={host_cxx}", + f'-fsycl-host-compiler-options="\\"/external:I{external_include}\\" /external:W0 {joined_host_cflags}"', + ] + else: + joined_host_cflags = ' '.join(cflags) + wrapped_host_cflags = [ + f"-fsycl-host-compiler={host_cxx}", + shlex.quote(f"-fsycl-host-compiler-options={joined_host_cflags}"), + ] + return wrapped_host_cflags + + +class BuildExtension(build_ext): + """ + A custom :mod:`setuptools` build extension . + + This :class:`setuptools.build_ext` subclass takes care of passing the + minimum required compiler flags (e.g. ``-std=c++17``) as well as mixed + C++/CUDA/SYCL compilation (and support for CUDA/SYCL files in general). + + When using :class:`BuildExtension`, it is allowed to supply a dictionary + for ``extra_compile_args`` (rather than the usual list) that maps from + languages/compilers (the only expected values are ``cxx``, ``nvcc`` or + ``sycl``) to a list of additional compiler flags to supply to the compiler. + This makes it possible to supply different flags to the C++, CUDA and SYCL + compiler during mixed compilation. + + ``use_ninja`` (bool): If ``use_ninja`` is ``True`` (default), then we + attempt to build using the Ninja backend. Ninja greatly speeds up + compilation compared to the standard ``setuptools.build_ext``. + Fallbacks to the standard distutils backend if Ninja is not available. + + .. note:: + By default, the Ninja backend uses #CPUS + 2 workers to build the + extension. This may use up too many resources on some systems. One + can control the number of workers by setting the `MAX_JOBS` environment + variable to a non-negative number. + """ + + @classmethod + def with_options(cls, **options): + """Return a subclass with alternative constructor that extends any original keyword arguments to the original constructor with the given options.""" + class cls_with_options(cls): # type: ignore[misc, valid-type] + def __init__(self, *args, **kwargs) -> None: + kwargs.update(options) + super().__init__(*args, **kwargs) + + return cls_with_options + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.no_python_abi_suffix = kwargs.get("no_python_abi_suffix", False) + + self.use_ninja = kwargs.get('use_ninja', True) + if self.use_ninja: + # Test if we can use ninja. Fallback otherwise. + msg = ('Attempted to use ninja as the BuildExtension backend but ' + '%s. Falling back to using the slow distutils backend.') + if not is_ninja_available(): + logger.warning(msg, 'we could not find ninja.') + self.use_ninja = False + + def finalize_options(self) -> None: + super().finalize_options() + if self.use_ninja: + self.force = True + + def build_extensions(self) -> None: + compiler_name, compiler_version = self._check_abi() + + cuda_ext = False + sycl_ext = False + extension_iter = iter(self.extensions) + extension = next(extension_iter, None) + while not (cuda_ext and sycl_ext) and extension: + for source in extension.sources: + _, ext = os.path.splitext(source) + if ext == '.cu': + cuda_ext = True + elif ext == '.sycl': + sycl_ext = True + + # This check accounts on a case when cuda and sycl sources + # are mixed in the same extension. We can stop checking + # sources if both are found or there is no more sources. + if cuda_ext and sycl_ext: + break + + extension = next(extension_iter, None) + + if sycl_ext: + if not self.use_ninja: + raise AssertionError("ninja is required to build sycl extensions.") + + if cuda_ext and not IS_HIP_EXTENSION: + _check_cuda_version(compiler_name, compiler_version) + + for extension in self.extensions: + # Ensure at least an empty list of flags for 'cxx', 'nvcc' and 'sycl' when + # extra_compile_args is a dict. Otherwise, default torch flags do + # not get passed. Necessary when only one of 'cxx', 'nvcc' or 'sycl' is + # passed to extra_compile_args in CUDAExtension or SyclExtension, i.e. + # CUDAExtension(..., extra_compile_args={'cxx': [...]}) + # or + # CUDAExtension(..., extra_compile_args={'nvcc': [...]}) + if isinstance(extension.extra_compile_args, dict): + for ext in ['cxx', 'nvcc', 'sycl']: + if ext not in extension.extra_compile_args: + extension.extra_compile_args[ext] = [] + + self._add_compile_flag(extension, '-DTORCH_API_INCLUDE_EXTENSION_H') + + if IS_HIP_EXTENSION: + self._hipify_compile_flags(extension) + + if extension.py_limited_api: + # compile any extension that has passed in py_limited_api to the + # Extension constructor with the Py_LIMITED_API flag set to our + # min supported CPython version. + # See https://docs.python.org/3/c-api/stable.html#c.Py_LIMITED_API + self._add_compile_flag(extension, f'-DPy_LIMITED_API={min_supported_cpython}') + self._define_torch_extension_name(extension) + + if 'nvcc_dlink' in extension.extra_compile_args: + if not self.use_ninja: + raise AssertionError( + f"With dlink=True, ninja is required to build cuda extension {extension.name}." + ) + + # Register .cu, .cuh, .hip, .mm and .sycl as valid source extensions. + # NOTE: At the moment .sycl is not a standard extension for SYCL supported + # by compiler. Here we introduce a torch level convention that SYCL sources + # should have .sycl file extension. + self.compiler.src_extensions += ['.cu', '.cuh', '.hip', '.sycl'] + if torch.backends.mps.is_built(): + self.compiler.src_extensions += ['.mm'] + # Save the original _compile method for later. + if self.compiler.compiler_type == 'msvc': + self.compiler._cpp_extensions += ['.cu', '.cuh'] + original_compile = self.compiler.compile + original_spawn = self.compiler.spawn + else: + original_compile = self.compiler._compile + + def append_std17_if_no_std_present(cflags) -> None: + # NVCC does not allow multiple -std to be passed, so we avoid + # overriding the option if the user explicitly passed it. + cpp_format_prefix = '/{}:' if self.compiler.compiler_type == 'msvc' else '-{}=' + cpp_flag_prefix = cpp_format_prefix.format('std') + cpp_flag = cpp_flag_prefix + 'c++17' + if not any(flag.startswith(cpp_flag_prefix) for flag in cflags): + cflags.append(cpp_flag) + + def unix_cuda_flags(cflags): + cflags = (COMMON_NVCC_FLAGS + + ['--compiler-options', "'-fPIC'"] + + cflags + _get_cuda_arch_flags(cflags)) + + # NVCC does not allow multiple -ccbin/--compiler-bindir to be passed, so we avoid + # overriding the option if the user explicitly passed it. + _ccbin = os.getenv("CC") + if ( + _ccbin is not None + and not any(flag.startswith(('-ccbin', '--compiler-bindir')) for flag in cflags) + ): + cflags.extend(['-ccbin', _ccbin]) + + return cflags + + def convert_to_absolute_paths_inplace(paths) -> None: + # Helper function. See Note [Absolute include_dirs] + if paths is not None: + for i in range(len(paths)): + if not os.path.isabs(paths[i]): + paths[i] = os.path.abspath(paths[i]) + + def unix_wrap_single_compile(obj, src, ext, cc_args, extra_postargs, pp_opts) -> None: + # Copy before we make any modifications. + cflags = copy.deepcopy(extra_postargs) + try: + original_compiler = self.compiler.compiler_so + if _is_cuda_file(src): + nvcc = [_join_rocm_home('bin', 'hipcc') if IS_HIP_EXTENSION else _join_cuda_home('bin', 'nvcc')] + self.compiler.set_executable('compiler_so', nvcc) + if isinstance(cflags, dict): + cflags = cflags['nvcc'] + if IS_HIP_EXTENSION: + cflags = COMMON_HIPCC_FLAGS + cflags + _get_rocm_arch_flags(cflags) + else: + cflags = unix_cuda_flags(cflags) + elif isinstance(cflags, dict): + cflags = cflags['cxx'] + if IS_HIP_EXTENSION: + cflags = COMMON_HIP_FLAGS + cflags + append_std17_if_no_std_present(cflags) + + original_compile(obj, src, ext, cc_args, cflags, pp_opts) + finally: + # Put the original compiler back in place. + self.compiler.set_executable('compiler_so', original_compiler) + + def unix_wrap_ninja_compile(sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None): + r"""Compiles sources by outputting a ninja file and running it.""" + # NB: I copied some lines from self.compiler (which is an instance + # of distutils.UnixCCompiler). See the following link. + # https://github.com/python/cpython/blob/f03a8f8d5001963ad5b5b28dbd95497e9cc15596/Lib/distutils/ccompiler.py#L564-L567 # codespell:ignore + # This can be fragile, but a lot of other repos also do this + # (see https://github.com/search?q=_setup_compile&type=Code) + # so it is probably OK; we'll also get CI signal if/when + # we update our python version (which is when distutils can be + # upgraded) + + # Use absolute path for output_dir so that the object file paths + # (`objects`) get generated with absolute paths. + # pyrefly: ignore [no-matching-overload] + output_dir = os.path.abspath(output_dir) + + # See Note [Absolute include_dirs] + convert_to_absolute_paths_inplace(self.compiler.include_dirs) + + _, objects, extra_postargs, pp_opts, _ = \ + self.compiler._setup_compile(output_dir, macros, + include_dirs, sources, + depends, extra_postargs) + common_cflags = self.compiler._get_cc_args(pp_opts, debug, extra_preargs) + extra_cc_cflags = self.compiler.compiler_so[1:] + with_cuda = any(map(_is_cuda_file, sources)) + with_sycl = any(map(_is_sycl_file, sources)) + assert not (with_sycl and with_cuda) + + # extra_postargs can be either: + # - a dict mapping cxx/nvcc/sycl to extra flags + # - a list of extra flags. + if isinstance(extra_postargs, dict): + post_cflags = extra_postargs['cxx'] + else: + post_cflags = list(extra_postargs) + if IS_HIP_EXTENSION: + post_cflags = COMMON_HIP_FLAGS + post_cflags + append_std17_if_no_std_present(post_cflags) + + cuda_post_cflags = None + cuda_cflags = None + if with_cuda: + cuda_cflags = common_cflags + if isinstance(extra_postargs, dict): + cuda_post_cflags = extra_postargs['nvcc'] + else: + cuda_post_cflags = list(extra_postargs) + if IS_HIP_EXTENSION: + cuda_post_cflags = cuda_post_cflags + _get_rocm_arch_flags(cuda_post_cflags) + cuda_post_cflags = COMMON_HIP_FLAGS + COMMON_HIPCC_FLAGS + cuda_post_cflags + else: + cuda_post_cflags = unix_cuda_flags(cuda_post_cflags) + append_std17_if_no_std_present(cuda_post_cflags) + cuda_cflags = [shlex.quote(f) for f in cuda_cflags] + cuda_post_cflags = [shlex.quote(f) for f in cuda_post_cflags] + + if isinstance(extra_postargs, dict) and 'nvcc_dlink' in extra_postargs: + cuda_dlink_post_cflags = unix_cuda_flags(extra_postargs['nvcc_dlink']) + cuda_dlink_post_cflags = [shlex.quote(f) for f in cuda_dlink_post_cflags] + else: + cuda_dlink_post_cflags = None + + sycl_post_cflags = None + sycl_cflags = None + sycl_dlink_post_cflags = None + if with_sycl: + sycl_cflags = extra_cc_cflags + common_cflags + _COMMON_SYCL_FLAGS + if isinstance(extra_postargs, dict): + sycl_post_cflags = extra_postargs['sycl'] + else: + sycl_post_cflags = list(extra_postargs) + _append_sycl_targets_if_missing(sycl_post_cflags) + append_std17_if_no_std_present(sycl_cflags) + _append_sycl_std_if_no_std_present(sycl_cflags) + host_cflags = extra_cc_cflags + common_cflags + post_cflags + append_std17_if_no_std_present(host_cflags) + # escaping quoted arguments to pass them thru SYCL compiler + icpx_version = _get_icpx_version() + if int(icpx_version) >= 20250200: + host_cflags = [item.replace('"', '\\"') for item in host_cflags] + else: + host_cflags = [item.replace('"', '\\\\"') for item in host_cflags] + # Note the order: shlex.quote sycl_flags first, _wrap_sycl_host_flags + # second. Reason is that sycl host flags are quoted, space containing + # strings passed to SYCL compiler. + sycl_cflags = [shlex.quote(f) for f in sycl_cflags] + sycl_cflags += _wrap_sycl_host_flags(host_cflags) + sycl_dlink_post_cflags = _SYCL_DLINK_FLAGS.copy() + sycl_dlink_post_cflags += _get_sycl_device_flags(sycl_post_cflags) + sycl_post_cflags = [shlex.quote(f) for f in sycl_post_cflags] + + _write_ninja_file_and_compile_objects( + sources=sources, + objects=objects, + cflags=[shlex.quote(f) for f in extra_cc_cflags + common_cflags], + post_cflags=[shlex.quote(f) for f in post_cflags], + cuda_cflags=cuda_cflags, + cuda_post_cflags=cuda_post_cflags, + cuda_dlink_post_cflags=cuda_dlink_post_cflags, + sycl_cflags=sycl_cflags, + sycl_post_cflags=sycl_post_cflags, + sycl_dlink_post_cflags=sycl_dlink_post_cflags, + build_directory=output_dir, + verbose=True, + with_cuda=with_cuda, + with_sycl=with_sycl) + + # Return *all* object filenames, not just the ones we just built. + return objects + + def win_cuda_flags(cflags): + return (COMMON_NVCC_FLAGS + + cflags + _get_cuda_arch_flags(cflags)) + + def win_hip_flags(cflags): + return (COMMON_HIPCC_FLAGS + COMMON_HIP_FLAGS + cflags + _get_rocm_arch_flags(cflags)) + + def win_wrap_single_compile(sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None): + + self.cflags = copy.deepcopy(extra_postargs) + extra_postargs = None + + def spawn(cmd): + # Using regex to match src, obj and include files + src_regex = re.compile('/T(p|c)(.*)') + src_list = [ + m.group(2) for m in (src_regex.match(elem) for elem in cmd) + if m + ] + + obj_regex = re.compile('/Fo(.*)') # codespell:ignore + obj_list = [ + m.group(1) for m in (obj_regex.match(elem) for elem in cmd) + if m + ] + + include_regex = re.compile(r'((\-|\/)I.*)') + include_list = [ + m.group(1) + for m in (include_regex.match(elem) for elem in cmd) if m + ] + + if len(src_list) >= 1 and len(obj_list) >= 1: + src = src_list[0] + obj = obj_list[0] + if _is_cuda_file(src): + if IS_HIP_EXTENSION: + nvcc = _get_hipcc_path() + else: + nvcc = _join_cuda_home('bin', 'nvcc') + if isinstance(self.cflags, dict): + cflags = self.cflags['nvcc'] + elif isinstance(self.cflags, list): + cflags = self.cflags + else: + cflags = [] + + if IS_HIP_EXTENSION: + cflags = win_hip_flags(cflags) + else: + cflags = win_cuda_flags(cflags) + ['-std=c++17', '--use-local-env'] + for ignore_warning in MSVC_IGNORE_CUDAFE_WARNINGS: + cflags = ['-Xcudafe', '--diag_suppress=' + ignore_warning] + cflags + for flag in COMMON_MSVC_FLAGS: + cflags = ['-Xcompiler', flag] + cflags + cmd = [nvcc, '-c', src, '-o', obj] + include_list + cflags + elif isinstance(self.cflags, dict): + cflags = COMMON_MSVC_FLAGS + self.cflags['cxx'] + append_std17_if_no_std_present(cflags) + cmd += cflags + elif isinstance(self.cflags, list): + cflags = COMMON_MSVC_FLAGS + self.cflags + append_std17_if_no_std_present(cflags) + cmd += cflags + + return original_spawn(cmd) + + try: + self.compiler.spawn = spawn + return original_compile(sources, output_dir, macros, + include_dirs, debug, extra_preargs, + extra_postargs, depends) + finally: + self.compiler.spawn = original_spawn + + def win_wrap_ninja_compile(sources, + output_dir=None, + macros=None, + include_dirs=None, + debug=0, + extra_preargs=None, + extra_postargs=None, + depends=None, + is_standalone=False): + if not self.compiler.initialized: + self.compiler.initialize() + # pyrefly: ignore [no-matching-overload] + output_dir = os.path.abspath(output_dir) + + # Note [Absolute include_dirs] + # Convert relative path in self.compiler.include_dirs to absolute path if any. + # For ninja build, the build location is not local, but instead, the build happens + # in a script-created build folder. Thus, relative paths lose their correctness. + # To be consistent with jit extension, we allow user to enter relative include_dirs + # in setuptools.setup, and we convert the relative path to absolute path here. + convert_to_absolute_paths_inplace(self.compiler.include_dirs) + + _, objects, extra_postargs, pp_opts, _ = \ + self.compiler._setup_compile(output_dir, macros, + include_dirs, sources, + depends, extra_postargs) + # Replace space with \ when using hipcc (hipcc passes includes to clang without ""s so clang sees space in include paths as new argument) + if IS_HIP_EXTENSION: + pp_opts = ["-I{}".format(s[2:].replace(" ", "\\")) if s.startswith('-I') else s for s in pp_opts] + common_cflags = extra_preargs or [] + cflags = [] + if debug: + cflags.extend(self.compiler.compile_options_debug) + else: + cflags.extend(self.compiler.compile_options) + cflags = cflags + common_cflags + pp_opts + COMMON_MSVC_FLAGS + if IS_HIP_EXTENSION: + _set_hipcc_runtime_lib(is_standalone, debug) + common_cflags.extend(COMMON_HIP_FLAGS) + else: + common_cflags.extend(COMMON_MSVC_FLAGS) + with_cuda = any(map(_is_cuda_file, sources)) + with_sycl = any(map(_is_sycl_file, sources)) + assert not (with_sycl and with_cuda) + + # extra_postargs can be either: + # - a dict mapping cxx/nvcc to extra flags + # - a list of extra flags. + if isinstance(extra_postargs, dict): + post_cflags = extra_postargs['cxx'] + else: + post_cflags = list(extra_postargs) + if IS_HIP_EXTENSION: + post_cflags = COMMON_HIP_FLAGS + post_cflags + append_std17_if_no_std_present(post_cflags) + + cuda_post_cflags = None + cuda_cflags = None + if with_cuda: + cuda_cflags = ['-std=c++17'] + for common_cflag in common_cflags: + cuda_cflags.append('-Xcompiler') + cuda_cflags.append(common_cflag) + if not IS_HIP_EXTENSION: + cuda_cflags.append('--use-local-env') + for ignore_warning in MSVC_IGNORE_CUDAFE_WARNINGS: + cuda_cflags.append('-Xcudafe') + cuda_cflags.append('--diag_suppress=' + ignore_warning) + cuda_cflags.extend(pp_opts) + if isinstance(extra_postargs, dict): + cuda_post_cflags = extra_postargs['nvcc'] + else: + cuda_post_cflags = list(extra_postargs) + if IS_HIP_EXTENSION: + cuda_post_cflags = win_hip_flags(cuda_post_cflags) + else: + cuda_post_cflags = win_cuda_flags(cuda_post_cflags) + cflags = _nt_quote_args(cflags) + post_cflags = _nt_quote_args(post_cflags) + if with_cuda: + cuda_cflags = _nt_quote_args(cuda_cflags) + cuda_post_cflags = _nt_quote_args(cuda_post_cflags) + if isinstance(extra_postargs, dict) and 'nvcc_dlink' in extra_postargs: + cuda_dlink_post_cflags = win_cuda_flags(extra_postargs['nvcc_dlink']) + else: + cuda_dlink_post_cflags = None + + sycl_cflags = None + sycl_post_cflags = None + sycl_dlink_post_cflags = None + if with_sycl: + sycl_cflags = common_cflags + pp_opts + _COMMON_SYCL_FLAGS + if isinstance(extra_postargs, dict): + sycl_post_cflags = extra_postargs['sycl'] + else: + sycl_post_cflags = list(extra_postargs) + _append_sycl_targets_if_missing(sycl_post_cflags) + append_std17_if_no_std_present(sycl_cflags) + _append_sycl_std_if_no_std_present(sycl_cflags) + host_cflags = common_cflags + pp_opts + post_cflags + append_std17_if_no_std_present(host_cflags) + + sycl_cflags = _nt_quote_args(sycl_cflags) + host_cflags = _nt_quote_args(host_cflags) + + sycl_cflags += _wrap_sycl_host_flags(host_cflags) + sycl_dlink_post_cflags = _SYCL_DLINK_FLAGS.copy() + sycl_dlink_post_cflags += _get_sycl_device_flags(sycl_post_cflags) + sycl_post_cflags = _nt_quote_args(sycl_post_cflags) + + + _write_ninja_file_and_compile_objects( + sources=sources, + objects=objects, + cflags=cflags, + post_cflags=post_cflags, + cuda_cflags=cuda_cflags, + cuda_post_cflags=cuda_post_cflags, + cuda_dlink_post_cflags=cuda_dlink_post_cflags, + sycl_cflags=sycl_cflags, + sycl_post_cflags=sycl_post_cflags, + sycl_dlink_post_cflags=sycl_dlink_post_cflags, + build_directory=output_dir, + verbose=True, + with_cuda=with_cuda, + with_sycl=with_sycl) + + # Return *all* object filenames, not just the ones we just built. + return objects + # Monkey-patch the _compile or compile method. + # https://github.com/python/cpython/blob/dc0284ee8f7a270b6005467f26d8e5773d76e959/Lib/distutils/ccompiler.py#L511 # codespell:ignore + if self.compiler.compiler_type == 'msvc': + if self.use_ninja: + self.compiler.compile = win_wrap_ninja_compile + else: + self.compiler.compile = win_wrap_single_compile + else: + if self.use_ninja: + self.compiler.compile = unix_wrap_ninja_compile + else: + self.compiler._compile = unix_wrap_single_compile + + build_ext.build_extensions(self) + + def get_ext_filename(self, ext_name): + # Get the original shared library name. For Python 3, this name will be + # suffixed with ".so", where will be something like + # cpython-37m-x86_64-linux-gnu. + ext_filename = super().get_ext_filename(ext_name) + # If `no_python_abi_suffix` is `True`, we omit the Python 3 ABI + # component. This makes building shared libraries with setuptools that + # aren't Python modules nicer. + if self.no_python_abi_suffix: + # The parts will be e.g. ["my_extension", "cpython-37m-x86_64-linux-gnu", "so"]. + ext_filename_parts = ext_filename.split('.') + # Omit the second to last element. + without_abi = ext_filename_parts[:-2] + ext_filename_parts[-1:] + ext_filename = '.'.join(without_abi) + return ext_filename + + def _check_abi(self) -> tuple[str, TorchVersion]: + # On some platforms, like Windows, compiler_cxx is not available. + if hasattr(self.compiler, 'compiler_cxx'): + compiler = self.compiler.compiler_cxx[0] + else: + compiler = get_cxx_compiler() + _, version = get_compiler_abi_compatibility_and_version(compiler) + # Warn user if VC env is activated but `DISTUILS_USE_SDK` is not set. + if IS_WINDOWS and 'VSCMD_ARG_TGT_ARCH' in os.environ and 'DISTUTILS_USE_SDK' not in os.environ: + msg = ('It seems that the VC environment is activated but DISTUTILS_USE_SDK is not set.' + 'This may lead to multiple activations of the VC env.' + 'Please set `DISTUTILS_USE_SDK=1` and try again.') + raise UserWarning(msg) + return compiler, version + + def _add_compile_flag(self, extension, flag) -> None: + extension.extra_compile_args = copy.deepcopy(extension.extra_compile_args) + if isinstance(extension.extra_compile_args, dict): + for args in extension.extra_compile_args.values(): + args.append(flag) + else: + extension.extra_compile_args.append(flag) + + # Simple hipify, replace the first occurrence of CUDA with HIP + # in flags starting with "-" and containing "CUDA", but exclude -I flags + def _hipify_compile_flags(self, extension) -> None: + if isinstance(extension.extra_compile_args, dict) and 'nvcc' in extension.extra_compile_args: + modified_flags = [] + for flag in extension.extra_compile_args['nvcc']: + if flag.startswith("-") and "CUDA" in flag and not flag.startswith("-I"): + # check/split flag into flag and value + parts = flag.split("=", 1) + if len(parts) == 2: + flag_part, value_part = parts + # replace fist instance of "CUDA" with "HIP" only in the flag and not flag value + modified_flag_part = flag_part.replace("CUDA", "HIP", 1) + modified_flag = f"{modified_flag_part}={value_part}" + else: + # replace fist instance of "CUDA" with "HIP" in flag + modified_flag = flag.replace("CUDA", "HIP", 1) + modified_flags.append(modified_flag) + logger.info('Modified flag: %s -> %s', flag, modified_flag) + else: + modified_flags.append(flag) + extension.extra_compile_args['nvcc'] = modified_flags + + def _define_torch_extension_name(self, extension) -> None: + # pybind11 doesn't support dots in the names + # so in order to support extensions in the packages + # like torch._C, we take the last part of the string + # as the library name + names = extension.name.split('.') + name = names[-1] + define = f'-DTORCH_EXTENSION_NAME={name}' + self._add_compile_flag(extension, define) + + +def CppExtension(name, sources, *args, **kwargs): + """ + Create a :class:`setuptools.Extension` for C++. + + Convenience method that creates a :class:`setuptools.Extension` with the + bare minimum (but often sufficient) arguments to build a C++ extension. + + All arguments are forwarded to the :class:`setuptools.Extension` + constructor. Full list arguments can be found at + https://setuptools.pypa.io/en/latest/userguide/ext_modules.html#extension-api-reference + + .. warning:: + The PyTorch python API (as provided in libtorch_python) cannot be built + with the flag ``py_limited_api=True``. When this flag is passed, it is + the user's responsibility in their library to not use APIs from + libtorch_python (in particular pytorch/python bindings) and to only use + APIs from libtorch (aten objects, operators and the dispatcher). For + example, to give access to custom ops from python, the library should + register the ops through the dispatcher. + + Contrary to CPython setuptools, who does not define -DPy_LIMITED_API + as a compile flag when py_limited_api is specified as an option for + the "bdist_wheel" command in ``setup``, PyTorch does! We will specify + -DPy_LIMITED_API=min_supported_cpython to best enforce consistency, + safety, and sanity in order to encourage best practices. To target a + different version, set min_supported_cpython to the hexcode of the + CPython version of choice. + + Example: + >>> # xdoctest: +SKIP + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CPP_EXT) + >>> from setuptools import setup + >>> from torch.utils.cpp_extension import BuildExtension, CppExtension + >>> setup( + ... name='extension', + ... ext_modules=[ + ... CppExtension( + ... name='extension', + ... sources=['extension.cpp'], + ... extra_compile_args=['-g'], + ... extra_link_args=['-Wl,--no-as-needed', '-lm']) + ... ], + ... cmdclass={ + ... 'build_ext': BuildExtension + ... }) + """ + include_dirs = kwargs.get('include_dirs', []) + include_dirs += include_paths() + kwargs['include_dirs'] = include_dirs + + library_dirs = kwargs.get('library_dirs', []) + library_dirs += library_paths() + kwargs['library_dirs'] = library_dirs + + libraries = kwargs.get('libraries', []) + libraries.append('c10') + libraries.append('torch') + libraries.append('torch_cpu') + if not kwargs.get('py_limited_api', False): + # torch_python uses more than the python limited api + libraries.append('torch_python') + if IS_WINDOWS: + libraries.append("sleef") + + kwargs['libraries'] = libraries + + kwargs['language'] = 'c++' + return setuptools.Extension(name, sources, *args, **kwargs) + + +def CUDAExtension(name, sources, *args, **kwargs): + """ + Create a :class:`setuptools.Extension` for CUDA/C++. + + Convenience method that creates a :class:`setuptools.Extension` with the + bare minimum (but often sufficient) arguments to build a CUDA/C++ + extension. This includes the CUDA include path, library path and runtime + library. + + All arguments are forwarded to the :class:`setuptools.Extension` + constructor. Full list arguments can be found at + https://setuptools.pypa.io/en/latest/userguide/ext_modules.html#extension-api-reference + + .. warning:: + The PyTorch python API (as provided in libtorch_python) cannot be built + with the flag ``py_limited_api=True``. When this flag is passed, it is + the user's responsibility in their library to not use APIs from + libtorch_python (in particular pytorch/python bindings) and to only use + APIs from libtorch (aten objects, operators and the dispatcher). For + example, to give access to custom ops from python, the library should + register the ops through the dispatcher. + + Contrary to CPython setuptools, who does not define -DPy_LIMITED_API + as a compile flag when py_limited_api is specified as an option for + the "bdist_wheel" command in ``setup``, PyTorch does! We will specify + -DPy_LIMITED_API=min_supported_cpython to best enforce consistency, + safety, and sanity in order to encourage best practices. To target a + different version, set min_supported_cpython to the hexcode of the + CPython version of choice. + + Example: + >>> # xdoctest: +SKIP + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CPP_EXT) + >>> from setuptools import setup + >>> from torch.utils.cpp_extension import BuildExtension, CUDAExtension + >>> setup( + ... name='cuda_extension', + ... ext_modules=[ + ... CUDAExtension( + ... name='cuda_extension', + ... sources=['extension.cpp', 'extension_kernel.cu'], + ... extra_compile_args={'cxx': ['-g'], + ... 'nvcc': ['-O2']}, + ... extra_link_args=['-Wl,--no-as-needed', '-lcuda']) + ... ], + ... cmdclass={ + ... 'build_ext': BuildExtension + ... }) + + Compute capabilities: + + By default the extension will be compiled to run on all archs of the cards visible during the + building process of the extension, plus PTX. If down the road a new card is installed the + extension may need to be recompiled. If a visible card has a compute capability (CC) that's + newer than the newest version for which your nvcc can build fully-compiled binaries, PyTorch + will make nvcc fall back to building kernels with the newest version of PTX your nvcc does + support (see below for details on PTX). + + You can override the default behavior using `TORCH_CUDA_ARCH_LIST` to explicitly specify which + CCs you want the extension to support: + + ``TORCH_CUDA_ARCH_LIST="6.1 8.6" python build_my_extension.py`` + ``TORCH_CUDA_ARCH_LIST="5.2 6.0 6.1 7.0 7.5 8.0 8.6+PTX" python build_my_extension.py`` + + The +PTX option causes extension kernel binaries to include PTX instructions for the specified + CC. PTX is an intermediate representation that allows kernels to runtime-compile for any CC >= + the specified CC (for example, 8.6+PTX generates PTX that can runtime-compile for any GPU with + CC >= 8.6). This improves your binary's forward compatibility. However, relying on older PTX to + provide forward compat by runtime-compiling for newer CCs can modestly reduce performance on + those newer CCs. If you know exact CC(s) of the GPUs you want to target, you're always better + off specifying them individually. For example, if you want your extension to run on 8.0 and 8.6, + "8.0+PTX" would work functionally because it includes PTX that can runtime-compile for 8.6, but + "8.0 8.6" would be better. + + Note that while it's possible to include all supported archs, the more archs get included the + slower the building process will be, as it will build a separate kernel image for each arch. + + Note that CUDA-11.5 nvcc will hit internal compiler error while parsing torch/extension.h on Windows. + To workaround the issue, move python binding logic to pure C++ file. + + Example use: + #include + at::Tensor SigmoidAlphaBlendForwardCuda(....) + + Instead of: + #include + torch::Tensor SigmoidAlphaBlendForwardCuda(...) + + Currently open issue for nvcc bug: https://github.com/pytorch/pytorch/issues/69460 + Complete workaround code example: https://github.com/facebookresearch/pytorch3d/commit/cb170ac024a949f1f9614ffe6af1c38d972f7d48 + + Relocatable device code linking: + + If you want to reference device symbols across compilation units (across object files), + the object files need to be built with `relocatable device code` (-rdc=true or -dc). + An exception to this rule is "dynamic parallelism" (nested kernel launches) which is not used a lot anymore. + `Relocatable device code` is less optimized so it needs to be used only on object files that need it. + Using `-dlto` (Device Link Time Optimization) at the device code compilation step and `dlink` step + helps reduce the protentional perf degradation of `-rdc`. + Note that it needs to be used at both steps to be useful. + + If you have `rdc` objects you need to have an extra `-dlink` (device linking) step before the CPU symbol linking step. + There is also a case where `-dlink` is used without `-rdc`: + when an extension is linked against a static lib containing rdc-compiled objects + like the [NVSHMEM library](https://developer.nvidia.com/nvshmem). + + Note: Ninja is required to build a CUDA Extension with RDC linking. + + Example: + >>> # xdoctest: +SKIP + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CPP_EXT) + >>> CUDAExtension( + ... name='cuda_extension', + ... sources=['extension.cpp', 'extension_kernel.cu'], + ... dlink=True, + ... dlink_libraries=["dlink_lib"], + ... extra_compile_args={'cxx': ['-g'], + ... 'nvcc': ['-O2', '-rdc=true']}) + """ + library_dirs = kwargs.get('library_dirs', []) + library_dirs += library_paths(device_type="cuda") + kwargs['library_dirs'] = library_dirs + + libraries = kwargs.get('libraries', []) + libraries.append('c10') + libraries.append('torch') + libraries.append('torch_cpu') + if not kwargs.get('py_limited_api', False): + # torch_python uses more than the python limited api + libraries.append('torch_python') + if IS_HIP_EXTENSION: + libraries.append('amdhip64') + libraries.append('c10_hip') + libraries.append('torch_hip') + else: + libraries.append('cudart') + libraries.append('c10_cuda') + libraries.append('torch_cuda') + kwargs['libraries'] = libraries + + include_dirs = kwargs.get('include_dirs', []) + + if IS_HIP_EXTENSION: + from .hipify import hipify_python + build_dir = os.getcwd() + hipify_result = hipify_python.hipify( + project_directory=build_dir, + output_directory=build_dir, + header_include_dirs=include_dirs, + includes=[os.path.join(build_dir, '*')], # limit scope to build_dir only + extra_files=[os.path.abspath(s) for s in sources], + show_detailed=True, + is_pytorch_extension=True, + hipify_extra_files_only=True, # don't hipify everything in includes path + ) + + hipified_sources = set() + for source in sources: + s_abs = os.path.abspath(source) + hipified_s_abs = (hipify_result[s_abs].hipified_path if (s_abs in hipify_result and + hipify_result[s_abs].hipified_path is not None) else s_abs) + # setup() arguments must *always* be /-separated paths relative to the setup.py directory, + # *never* absolute paths + hipified_sources.add(os.path.relpath(hipified_s_abs, build_dir)) + + sources = list(hipified_sources) + + include_dirs += include_paths(device_type="cuda") + kwargs['include_dirs'] = include_dirs + + kwargs['language'] = 'c++' + + dlink_libraries = kwargs.get('dlink_libraries', []) + dlink = kwargs.get('dlink', False) or dlink_libraries + if dlink: + extra_compile_args = kwargs.get('extra_compile_args', {}) + + extra_compile_args_dlink = extra_compile_args.get('nvcc_dlink', []) + extra_compile_args_dlink += ['-dlink'] + extra_compile_args_dlink += [f'-L{x}' for x in library_dirs] + extra_compile_args_dlink += [f'-l{x}' for x in dlink_libraries] + + if (torch.version.cuda is not None) and TorchVersion(torch.version.cuda) >= '11.2': + extra_compile_args_dlink += ['-dlto'] # Device Link Time Optimization started from cuda 11.2 + + extra_compile_args['nvcc_dlink'] = extra_compile_args_dlink + + kwargs['extra_compile_args'] = extra_compile_args + + return setuptools.Extension(name, sources, *args, **kwargs) + + +def SyclExtension(name, sources, *args, **kwargs): + r""" + Creates a :class:`setuptools.Extension` for SYCL/C++. + + Convenience method that creates a :class:`setuptools.Extension` with the + bare minimum (but often sufficient) arguments to build a SYCL/C++ + extension. + + All arguments are forwarded to the :class:`setuptools.Extension` + constructor. + + .. warning:: + The PyTorch python API (as provided in libtorch_python) cannot be built + with the flag ``py_limited_api=True``. When this flag is passed, it is + the user's responsibility in their library to not use APIs from + libtorch_python (in particular pytorch/python bindings) and to only use + APIs from libtorch (aten objects, operators and the dispatcher). For + example, to give access to custom ops from python, the library should + register the ops through the dispatcher. + + Contrary to CPython setuptools, who does not define -DPy_LIMITED_API + as a compile flag when py_limited_api is specified as an option for + the "bdist_wheel" command in ``setup``, PyTorch does! We will specify + -DPy_LIMITED_API=min_supported_cpython to best enforce consistency, + safety, and sanity in order to encourage best practices. To target a + different version, set min_supported_cpython to the hexcode of the + CPython version of choice. + + Example: + >>> # xdoctest: +SKIP + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CPP_EXT) + >>> from torch.utils.cpp_extension import BuildExtension, SyclExtension + >>> setup( + ... name='xpu_extension', + ... ext_modules=[ + ... SyclExtension( + ... name='xpu_extension', + ... sources=['extension.cpp', 'extension_kernel.cpp'], + ... extra_compile_args={'cxx': ['-g', '-std=c++20', '-fPIC']}) + ... ], + ... cmdclass={ + ... 'build_ext': BuildExtension + ... }) + + By default the extension will be compiled to run on all archs of the cards visible during the + building process of the extension. If down the road a new card is installed the + extension may need to be recompiled. You can override the default behavior using + `TORCH_XPU_ARCH_LIST` to explicitly specify which device architectures you want the extension + to support: + + ``TORCH_XPU_ARCH_LIST="pvc,xe-lpg" python build_my_extension.py`` + + Note that while it's possible to include all supported archs, the more archs get included the + slower the building process will be, as it will build a separate kernel image for each arch. + + Note: Ninja is required to build SyclExtension. + """ + library_dirs = kwargs.get("library_dirs", []) + library_dirs += library_paths() + kwargs["library_dirs"] = library_dirs + + libraries = kwargs.get("libraries", []) + libraries.append("c10") + libraries.append("c10_xpu") + libraries.append("torch") + libraries.append("torch_cpu") + libraries.append("sycl") + if not kwargs.get('py_limited_api', False): + # torch_python uses more than the python limited api + libraries.append("torch_python") + libraries.append("torch_xpu") + kwargs["libraries"] = libraries + + include_dirs = kwargs.get("include_dirs", []) + include_dirs += include_paths(device_type="xpu") + kwargs["include_dirs"] = include_dirs + + kwargs["language"] = "c++" + + return setuptools.Extension(name, sources, *args, **kwargs) + +def include_paths(device_type: str = "cpu", torch_include_dirs=True) -> list[str]: + """ + Get the include paths required to build a C++ or CUDA or SYCL extension. + + Args: + device_type: Defaults to "cpu". + Returns: + A list of include path strings. + """ + paths = [] + lib_include = os.path.join(_TORCH_PATH, 'include') + if torch_include_dirs: + paths.extend([ + lib_include, + # Remove this once torch/torch.h is officially no longer supported for C++ extensions. + os.path.join(lib_include, 'torch', 'csrc', 'api', 'include'), + ]) + if device_type == "cuda" and IS_HIP_EXTENSION: + paths.append(os.path.join(lib_include, 'THH')) + paths.append(_join_rocm_home('include')) + elif device_type == "cuda": + cuda_home_include = _join_cuda_home('include') + # if we have the Debian/Ubuntu packages for cuda, we get /usr as cuda home. + # but gcc doesn't like having /usr/include passed explicitly + if cuda_home_include != '/usr/include': + paths.append(cuda_home_include) + + # Support CUDA_INC_PATH env variable supported by CMake files + if (cuda_inc_path := os.environ.get("CUDA_INC_PATH", None)) and \ + cuda_inc_path != '/usr/include': + + paths.append(cuda_inc_path) + if CUDNN_HOME is not None: + paths.append(os.path.join(CUDNN_HOME, 'include')) + elif device_type == "xpu": + paths.append(_join_sycl_home('include')) + paths.append(_join_sycl_home('include', 'sycl')) + return paths + + +def library_paths(device_type: str = "cpu", torch_include_dirs: bool = True, cross_target_platform: str | None = None) -> list[str]: + """ + Get the library paths required to build a C++ or CUDA extension. + + Args: + device_type: Defaults to "cpu". + + Returns: + A list of library path strings. + """ + + paths = [] + + if torch_include_dirs: + # We need to link against libtorch.so + paths.extend([TORCH_LIB_PATH]) + + if device_type == "cuda" and IS_HIP_EXTENSION: + lib_dir = 'lib' + paths.append(_join_rocm_home(lib_dir)) + if HIP_HOME is not None: + paths.append(os.path.join(HIP_HOME, 'lib')) + elif device_type == "cuda": + if cross_target_platform == "windows": + lib_dir = os.path.join('lib', 'x64') + if WINDOWS_CUDA_HOME is None: + raise RuntimeError("Need to set WINDOWS_CUDA_HOME for windows cross-compilation") + paths.append(os.path.join(WINDOWS_CUDA_HOME, lib_dir)) + else: + if IS_WINDOWS: + lib_dir = os.path.join('lib', 'x64') + else: + lib_dir = 'lib64' + if (not os.path.exists(_join_cuda_home(lib_dir)) and + os.path.exists(_join_cuda_home('lib'))): + # 64-bit CUDA may be installed in 'lib' (see e.g. gh-16955) + # Note that it's also possible both don't exist (see + # _find_cuda_home) - in that case we stay with 'lib64'. + lib_dir = 'lib' + + paths.append(_join_cuda_home(lib_dir)) + if CUDNN_HOME is not None: + paths.append(os.path.join(CUDNN_HOME, lib_dir)) + elif device_type == "xpu": + if IS_WINDOWS: + lib_dir = os.path.join('lib', 'x64') + else: + lib_dir = 'lib64' + if (not os.path.exists(_join_sycl_home(lib_dir)) and + os.path.exists(_join_sycl_home('lib'))): + lib_dir = 'lib' + + paths.append(_join_sycl_home(lib_dir)) + + return paths + + +def load(name, + sources: str | list[str], + extra_cflags=None, + extra_cuda_cflags=None, + extra_sycl_cflags=None, + extra_ldflags=None, + extra_include_paths=None, + build_directory=None, + verbose=False, + with_cuda: bool | None = None, + with_sycl: bool | None = None, + is_python_module=True, + is_standalone=False, + keep_intermediates=True): + """ + Load a PyTorch C++ extension just-in-time (JIT). + + To load an extension, a Ninja build file is emitted, which is used to + compile the given sources into a dynamic library. This library is + subsequently loaded into the current Python process as a module and + returned from this function, ready for use. + + By default, the directory to which the build file is emitted and the + resulting library compiled to is ``/torch_extensions/``, where + ```` is the temporary folder on the current platform and ```` + the name of the extension. This location can be overridden in two ways. + First, if the ``TORCH_EXTENSIONS_DIR`` environment variable is set, it + replaces ``/torch_extensions`` and all extensions will be compiled + into subfolders of this directory. Second, if the ``build_directory`` + argument to this function is supplied, it overrides the entire path, i.e. + the library will be compiled into that folder directly. + + To compile the sources, the default system compiler (``c++``) is used, + which can be overridden by setting the ``CXX`` environment variable. To pass + additional arguments to the compilation process, ``extra_cflags`` or + ``extra_ldflags`` can be provided. For example, to compile your extension + with optimizations, pass ``extra_cflags=['-O3']``. You can also use + ``extra_cflags`` to pass further include directories. + + CUDA support with mixed compilation is provided. Simply pass CUDA source + files (``.cu`` or ``.cuh``) along with other sources. Such files will be + detected and compiled with nvcc rather than the C++ compiler. This includes + passing the CUDA lib64 directory as a library directory, and linking + ``cudart``. You can pass additional flags to nvcc via + ``extra_cuda_cflags``, just like with ``extra_cflags`` for C++. Various + heuristics for finding the CUDA install directory are used, which usually + work fine. If not, setting the ``CUDA_HOME`` environment variable is the + safest option. + + SYCL support with mixed compilation is provided. Simply pass SYCL source + files (``.sycl``) along with other sources. Such files will be detected + and compiled with SYCL compiler (such as Intel DPC++ Compiler) rather + than the C++ compiler. You can pass additional flags to SYCL compiler + via ``extra_sycl_cflags``, just like with ``extra_cflags`` for C++. + SYCL compiler is expected to be found via system PATH environment + variable. + + Args: + name: The name of the extension to build. This MUST be the same as the + name of the pybind11 module! + sources: A list of relative or absolute paths to C++ source files. + extra_cflags: optional list of compiler flags to forward to the build. + extra_cuda_cflags: optional list of compiler flags to forward to nvcc + when building CUDA sources. + extra_sycl_cflags: optional list of compiler flags to forward to SYCL + compiler when building SYCL sources. + extra_ldflags: optional list of linker flags to forward to the build. + extra_include_paths: optional list of include directories to forward + to the build. + build_directory: optional path to use as build workspace. + verbose: If ``True``, turns on verbose logging of load steps. + with_cuda: Determines whether CUDA headers and libraries are added to + the build. If set to ``None`` (default), this value is + automatically determined based on the existence of ``.cu`` or + ``.cuh`` in ``sources``. Set it to `True`` to force CUDA headers + and libraries to be included. + with_sycl: Determines whether SYCL headers and libraries are added to + the build. If set to ``None`` (default), this value is + automatically determined based on the existence of ``.sycl`` in + ``sources``. Set it to `True`` to force SYCL headers and + libraries to be included. + is_python_module: If ``True`` (default), imports the produced shared + library as a Python module. If ``False``, behavior depends on + ``is_standalone``. + is_standalone: If ``False`` (default) loads the constructed extension + into the process as a plain dynamic library. If ``True``, build a + standalone executable. + + Returns: + If ``is_python_module`` is ``True``: + Returns the loaded PyTorch extension as a Python module. + + If ``is_python_module`` is ``False`` and ``is_standalone`` is ``False``: + Returns nothing. (The shared library is loaded into the process as + a side effect.) + + If ``is_standalone`` is ``True``. + Return the path to the executable. (On Windows, TORCH_LIB_PATH is + added to the PATH environment variable as a side effect.) + + Example: + >>> # xdoctest: +SKIP + >>> from torch.utils.cpp_extension import load + >>> module = load( + ... name='extension', + ... sources=['extension.cpp', 'extension_kernel.cu'], + ... extra_cflags=['-O2'], + ... verbose=True) + """ + return _jit_compile( + name, + [sources] if isinstance(sources, str) else sources, + extra_cflags, + extra_cuda_cflags, + extra_sycl_cflags, + extra_ldflags, + extra_include_paths, + build_directory or _get_build_directory(name, verbose), + verbose, + with_cuda, + with_sycl, + is_python_module, + is_standalone, + keep_intermediates=keep_intermediates) + +@deprecated("PyBind11 ABI handling is internal to PyBind11; this will be removed after PyTorch 2.9.0") +def _get_pybind11_abi_build_flags() -> list[str]: + return [] + +def check_compiler_is_gcc(compiler) -> bool: + if not IS_LINUX: + return False + + env = os.environ.copy() + env['LC_ALL'] = 'C' # Don't localize output + try: + version_string = subprocess.check_output([compiler, '-v'], stderr=subprocess.STDOUT, env=env).decode(*SUBPROCESS_DECODE_ARGS) + except Exception: + try: + version_string = subprocess.check_output([compiler, '--version'], stderr=subprocess.STDOUT, env=env).decode(*SUBPROCESS_DECODE_ARGS) + except Exception: + return False + # Check for GCC by verifying both COLLECT_GCC and gcc version string are present + # This works for c++, g++, gcc, and versioned variants like g++-13 + pattern = re.compile("^COLLECT_GCC=(.*)$", re.MULTILINE) + has_collect_gcc = pattern.search(version_string) is not None + if has_collect_gcc and 'gcc version' in version_string: + return True + return False + +def _check_and_build_extension_h_precompiler_headers( + extra_cflags, + extra_include_paths, + is_standalone=False) -> None: + r''' + Precompiled Headers(PCH) can pre-build the same headers and reduce build time for pytorch load_inline modules. + GCC official manual: https://gcc.gnu.org/onlinedocs/gcc-4.0.4/gcc/Precompiled-Headers.html + PCH only works when built pch file(header.h.gch) and build target have the same build parameters. So, We need + add a signature file to record PCH file parameters. If the build parameters(signature) changed, it should rebuild + PCH file. + + Note: + 1. Windows and MacOS have different PCH mechanism. We only support Linux currently. + 2. It only works on GCC/G++. + ''' + if not IS_LINUX: + return + + compiler = get_cxx_compiler() + + b_is_gcc = check_compiler_is_gcc(compiler) + if b_is_gcc is False: + return + + head_file = os.path.join(_TORCH_PATH, 'include', 'torch', 'extension.h') + head_file_pch = os.path.join(_TORCH_PATH, 'include', 'torch', 'extension.h.gch') + head_file_signature = os.path.join(_TORCH_PATH, 'include', 'torch', 'extension.h.sign') + + def listToString(s): + # initialize an empty string + string = "" + if s is None: + return string + + # traverse in the string + for element in s: + string += (element + ' ') + # return string + return string + + def format_precompiler_header_cmd(compiler, head_file, head_file_pch, common_cflags, torch_include_dirs, extra_cflags, extra_include_paths): + return re.sub( + r"[ \n]+", + " ", + f""" + {compiler} -x c++-header {head_file} -o {head_file_pch} {torch_include_dirs} {extra_include_paths} {extra_cflags} {common_cflags} + """, + ).strip() + + def command_to_signature(cmd): + signature = cmd.replace(' ', '_') + return signature + + def check_pch_signature_in_file(file_path, signature): + b_exist = os.path.isfile(file_path) + if b_exist is False: + return False + + with open(file_path) as file: + # read all content of a file + content = file.read() + # check if string present in a file + return signature == content + + def _create_if_not_exist(path_dir) -> None: + if not os.path.exists(path_dir): + try: + Path(path_dir).mkdir(parents=True, exist_ok=True) + except OSError as exc: # Guard against race condition + if exc.errno != errno.EEXIST: + raise RuntimeError(f"Fail to create path {path_dir}") from exc + + def write_pch_signature_to_file(file_path, pch_sign) -> None: + _create_if_not_exist(os.path.dirname(file_path)) + with open(file_path, "w") as f: + f.write(pch_sign) + f.close() + + def build_precompile_header(pch_cmd) -> None: + try: + subprocess.check_output(shlex.split(pch_cmd), stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Compile PreCompile Header fail, command: {pch_cmd}") from e + + extra_cflags_str = listToString(extra_cflags) + extra_include_paths_str = " ".join( + [f"-I{include}" for include in extra_include_paths] if extra_include_paths else [] + ) + + lib_include = os.path.join(_TORCH_PATH, 'include') + torch_include_dirs = [ + f"-I {lib_include}", + # Python.h + "-I {}".format(sysconfig.get_path("include")), + # torch/all.h + "-I {}".format(os.path.join(lib_include, 'torch', 'csrc', 'api', 'include')), + ] + + torch_include_dirs_str = listToString(torch_include_dirs) + + common_cflags = [] + if not is_standalone: + common_cflags += ['-DTORCH_API_INCLUDE_EXTENSION_H'] + + common_cflags += ['-std=c++17', '-fPIC'] + common_cflags_str = listToString(common_cflags) + + pch_cmd = format_precompiler_header_cmd(compiler, head_file, head_file_pch, common_cflags_str, torch_include_dirs_str, extra_cflags_str, extra_include_paths_str) + pch_sign = command_to_signature(pch_cmd) + + if os.path.isfile(head_file_pch) is not True: + build_precompile_header(pch_cmd) + write_pch_signature_to_file(head_file_signature, pch_sign) + else: + b_same_sign = check_pch_signature_in_file(head_file_signature, pch_sign) + if b_same_sign is False: + build_precompile_header(pch_cmd) + write_pch_signature_to_file(head_file_signature, pch_sign) + +def remove_extension_h_precompiler_headers() -> None: + def _remove_if_file_exists(path_file) -> None: + if os.path.exists(path_file): + os.remove(path_file) + + head_file_pch = os.path.join(_TORCH_PATH, 'include', 'torch', 'extension.h.gch') + head_file_signature = os.path.join(_TORCH_PATH, 'include', 'torch', 'extension.h.sign') + + _remove_if_file_exists(head_file_pch) + _remove_if_file_exists(head_file_signature) + +def load_inline(name, + cpp_sources, + cuda_sources=None, + sycl_sources=None, + functions=None, + extra_cflags=None, + extra_cuda_cflags=None, + extra_sycl_cflags=None, + extra_ldflags=None, + extra_include_paths=None, + build_directory=None, + verbose=False, + with_cuda=None, + with_sycl=None, + is_python_module=True, + with_pytorch_error_handling=True, + keep_intermediates=True, + use_pch=False, + no_implicit_headers=False): + r''' + Load a PyTorch C++ extension just-in-time (JIT) from string sources. + + This function behaves exactly like :func:`load`, but takes its sources as + strings rather than filenames. These strings are stored to files in the + build directory, after which the behavior of :func:`load_inline` is + identical to :func:`load`. + + See `the + tests `_ + for good examples of using this function. + + Sources may omit two required parts of a typical non-inline C++ extension: + the necessary header includes, as well as the (pybind11) binding code. More + precisely, strings passed to ``cpp_sources`` are first concatenated into a + single ``.cpp`` file. This file is then prepended with ``#include + `` + + Furthermore, if the ``functions`` argument is supplied, bindings will be + automatically generated for each function specified. ``functions`` can + either be a list of function names, or a dictionary mapping from function + names to docstrings. If a list is given, the name of each function is used + as its docstring. + + The sources in ``cuda_sources`` are concatenated into a separate ``.cu`` + file and prepended with ``torch/types.h``, ``cuda.h`` and + ``cuda_runtime.h`` includes. The ``.cpp`` and ``.cu`` files are compiled + separately, but ultimately linked into a single library. Note that no + bindings are generated for functions in ``cuda_sources`` per se. To bind + to a CUDA kernel, you must create a C++ function that calls it, and either + declare or define this C++ function in one of the ``cpp_sources`` (and + include its name in ``functions``). + + The sources in ``sycl_sources`` are concatenated into a separate ``.sycl`` + file and prepended with ``torch/types.h``, ``sycl/sycl.hpp`` includes. + The ``.cpp`` and ``.sycl`` files are compiled separately, but ultimately + linked into a single library. Note that no bindings are generated for + functions in ``sycl_sources`` per se. To bind to a SYCL kernel, you must + create a C++ function that calls it, and either declare or define this + C++ function in one of the ``cpp_sources`` (and include its name + in ``functions``). + + + + See :func:`load` for a description of arguments omitted below. + + Args: + cpp_sources: A string, or list of strings, containing C++ source code. + cuda_sources: A string, or list of strings, containing CUDA source code. + sycl_sources: A string, or list of strings, containing SYCL source code. + functions: A list of function names for which to generate function + bindings. If a dictionary is given, it should map function names to + docstrings (which are otherwise just the function names). + with_cuda: Determines whether CUDA headers and libraries are added to + the build. If set to ``None`` (default), this value is + automatically determined based on whether ``cuda_sources`` is + provided. Set it to ``True`` to force CUDA headers + and libraries to be included. + with_sycl: Determines whether SYCL headers and libraries are added to + the build. If set to ``None`` (default), this value is + automatically determined based on whether ``sycl_sources`` is + provided. Set it to ``True`` to force SYCL headers + and libraries to be included. + with_pytorch_error_handling: Determines whether pytorch error and + warning macros are handled by pytorch instead of pybind. To do + this, each function ``foo`` is called via an intermediary ``_safe_foo`` + function. This redirection might cause issues in obscure cases + of cpp. This flag should be set to ``False`` when this redirect + causes issues. + no_implicit_headers: If ``True``, skips automatically adding headers, most notably + ``#include `` and ``#include `` lines. + Use this option to improve cold start times when you + already include the necessary headers in your source code. Default: ``False``. + + Example: + >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CPP_EXT) + >>> from torch.utils.cpp_extension import load_inline + >>> source = """ + at::Tensor sin_add(at::Tensor x, at::Tensor y) { + return x.sin() + y.sin(); + } + """ + >>> module = load_inline(name='inline_extension', + ... cpp_sources=[source], + ... functions=['sin_add']) + + .. note:: + Since load_inline will just-in-time compile the source code, please ensure + that you have the right toolchains installed in the runtime. For example, + when loading C++, make sure a C++ compiler is available. If you're loading + a CUDA extension, you will need to additionally install the corresponding CUDA + toolkit (nvcc and any other dependencies your code has). Compiling toolchains + are not included when you install torch and must be additionally installed. + + During compiling, by default, the Ninja backend uses #CPUS + 2 workers to build + the extension. This may use up too many resources on some systems. One + can control the number of workers by setting the `MAX_JOBS` environment + variable to a non-negative number. + ''' + build_directory = build_directory or _get_build_directory(name, verbose) + + if isinstance(cpp_sources, str): + cpp_sources = [cpp_sources] + cuda_sources = cuda_sources or [] + if isinstance(cuda_sources, str): + cuda_sources = [cuda_sources] + sycl_sources = sycl_sources or [] + if isinstance(sycl_sources, str): + sycl_sources = [sycl_sources] + + if not no_implicit_headers: + cpp_sources.insert(0, '#include ') + + if use_pch is True: + # Using PreCompile Header('torch/extension.h') to reduce compile time. + _check_and_build_extension_h_precompiler_headers(extra_cflags, extra_include_paths) + else: + remove_extension_h_precompiler_headers() + + # If `functions` is supplied, we create the pybind11 bindings for the user. + # Here, `functions` is (or becomes, after some processing) a map from + # function names to function docstrings. + if functions is not None: + module_def = [] + module_def.append('PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {') + if isinstance(functions, str): + functions = [functions] + if isinstance(functions, list): + # Make the function docstring the same as the function name. + functions = {f: f for f in functions} + elif not isinstance(functions, dict): + raise ValueError(f"Expected 'functions' to be a list or dict, but was {type(functions)}") + for function_name, docstring in functions.items(): + if with_pytorch_error_handling: + module_def.append(f'm.def("{function_name}", torch::wrap_pybind_function({function_name}), "{docstring}");') + else: + module_def.append(f'm.def("{function_name}", {function_name}, "{docstring}");') + module_def.append('}') + cpp_sources += module_def + + cpp_source_path = os.path.join(build_directory, 'main.cpp') + _maybe_write(cpp_source_path, "\n".join(cpp_sources)) + + sources = [cpp_source_path] + + if cuda_sources: + if not no_implicit_headers: + cuda_sources.insert(0, '#include ') + cuda_sources.insert(1, '#include ') + cuda_sources.insert(2, '#include ') + + cuda_source_path = os.path.join(build_directory, 'cuda.cu') + _maybe_write(cuda_source_path, "\n".join(cuda_sources)) + + sources.append(cuda_source_path) + + if sycl_sources: + if not no_implicit_headers: + sycl_sources.insert(0, '#include ') + sycl_sources.insert(1, '#include ') + + sycl_source_path = os.path.join(build_directory, 'sycl.sycl') + _maybe_write(sycl_source_path, "\n".join(sycl_sources)) + + sources.append(sycl_source_path) + + return _jit_compile( + name, + sources, + extra_cflags, + extra_cuda_cflags, + extra_sycl_cflags, + extra_ldflags, + extra_include_paths, + build_directory, + verbose, + with_cuda, + with_sycl, + is_python_module, + is_standalone=False, + keep_intermediates=keep_intermediates) + + +def _jit_compile(name, + sources, + extra_cflags, + extra_cuda_cflags, + extra_sycl_cflags, + extra_ldflags, + extra_include_paths, + build_directory: str, + verbose: bool, + with_cuda: bool | None, + with_sycl: bool | None, + is_python_module, + is_standalone, + keep_intermediates=True) -> types.ModuleType | str: + if is_python_module and is_standalone: + raise ValueError("`is_python_module` and `is_standalone` are mutually exclusive.") + + if with_cuda is None: + with_cuda = any(map(_is_cuda_file, sources)) + with_cudnn = any('cudnn' in f for f in extra_ldflags or []) + if with_sycl is None: + with_sycl = any(map(_is_sycl_file, sources)) + assert not (with_sycl and with_cuda) + old_version = JIT_EXTENSION_VERSIONER.get_version(name) + version = JIT_EXTENSION_VERSIONER.bump_version_if_changed( + name, + sources, + build_arguments=[extra_cflags, extra_cuda_cflags, extra_ldflags, extra_include_paths], + build_directory=build_directory, + with_cuda=with_cuda, + with_sycl=with_sycl, + is_python_module=is_python_module, + is_standalone=is_standalone, + ) + if version > 0: + if version != old_version and verbose: + logger.info('The input conditions for extension module %s have changed.', name) + logger.info('Bumping to version %s and re-building as %s_v%s...', version, name, version) + name = f'{name}_v{version}' + + baton = FileBaton(os.path.join(build_directory, 'lock')) + if baton.try_acquire(): + try: + if version != old_version: + from .hipify import hipify_python + from .hipify.hipify_python import GeneratedFileCleaner + with GeneratedFileCleaner(keep_intermediates=keep_intermediates) as clean_ctx: + if IS_HIP_EXTENSION and (with_cuda or with_cudnn): + hipify_result = hipify_python.hipify( + project_directory=build_directory, + output_directory=build_directory, + header_include_dirs=(extra_include_paths if extra_include_paths is not None else []), + extra_files=[os.path.abspath(s) for s in sources], + ignores=[_join_rocm_home('*'), os.path.join(_TORCH_PATH, '*')], # no need to hipify ROCm or PyTorch headers + show_detailed=verbose, + show_progress=verbose, + is_pytorch_extension=True, + clean_ctx=clean_ctx + ) + + hipified_sources = set() + for source in sources: + s_abs = os.path.abspath(source) + hipified_sources.add(hipify_result[s_abs].hipified_path if s_abs in hipify_result else s_abs) + + sources = list(hipified_sources) + + _write_ninja_file_and_build_library( + name=name, + sources=sources, + extra_cflags=extra_cflags or [], + extra_cuda_cflags=extra_cuda_cflags or [], + extra_sycl_cflags=extra_sycl_cflags or [], + extra_ldflags=extra_ldflags or [], + extra_include_paths=extra_include_paths or [], + build_directory=build_directory, + verbose=verbose, + with_cuda=with_cuda, + with_sycl=with_sycl, + is_standalone=is_standalone) + elif verbose: + logger.debug('No modifications detected for re-loaded extension module %s, skipping build step...', name) + finally: + baton.release() + else: + baton.wait() + + if verbose: + logger.info('Loading extension module %s...', name) + + if is_standalone: + return _get_exec_path(name, build_directory) + + return _import_module_from_library(name, build_directory, is_python_module) + +def _get_hipcc_path(): + if IS_WINDOWS: + # mypy thinks ROCM_VERSION is None but it will never be None here + hipcc_exe = 'hipcc.exe' if ROCM_VERSION >= (6, 4) else 'hipcc.bat' # type: ignore[operator] + return _join_rocm_home('bin', hipcc_exe) + else: + return _join_rocm_home('bin', 'hipcc') + +def _write_ninja_file_and_compile_objects( + sources: list[str], + objects, + cflags, + post_cflags, + cuda_cflags, + cuda_post_cflags, + cuda_dlink_post_cflags, + sycl_cflags, + sycl_post_cflags, + sycl_dlink_post_cflags, + build_directory: str, + verbose: bool, + with_cuda: bool | None, + with_sycl: bool | None) -> None: + verify_ninja_availability() + + compiler = get_cxx_compiler() + + get_compiler_abi_compatibility_and_version(compiler) + if with_cuda is None: + with_cuda = any(map(_is_cuda_file, sources)) + if with_sycl is None: + with_sycl = any(map(_is_sycl_file, sources)) + assert not (with_sycl and with_cuda) + build_file_path = os.path.join(build_directory, 'build.ninja') + if verbose: + logger.debug('Emitting ninja build file %s...', build_file_path) + + # Create build_directory if it does not exist + if not os.path.exists(build_directory): + if verbose: + logger.debug('Creating directory %s...', build_directory) + # This is like mkdir -p, i.e. will also create parent directories. + os.makedirs(build_directory, exist_ok=True) + + _write_ninja_file( + path=build_file_path, + cflags=cflags, + post_cflags=post_cflags, + cuda_cflags=cuda_cflags, + cuda_post_cflags=cuda_post_cflags, + cuda_dlink_post_cflags=cuda_dlink_post_cflags, + sycl_cflags=sycl_cflags, + sycl_post_cflags=sycl_post_cflags, + sycl_dlink_post_cflags=sycl_dlink_post_cflags, + sources=sources, + objects=objects, + ldflags=None, + library_target=None, + with_cuda=with_cuda, + with_sycl=with_sycl) + if verbose: + logger.info('Compiling objects...') + _run_ninja_build( + build_directory, + verbose, + # It would be better if we could tell users the name of the extension + # that failed to build but there isn't a good way to get it here. + error_prefix='Error compiling objects for extension') + + +def _write_ninja_file_and_build_library( + name, + sources: list[str], + extra_cflags, + extra_cuda_cflags, + extra_sycl_cflags, + extra_ldflags, + extra_include_paths, + build_directory: str, + verbose: bool, + with_cuda: bool | None, + with_sycl: bool | None, + is_standalone: bool = False) -> None: + verify_ninja_availability() + + compiler = get_cxx_compiler() + + get_compiler_abi_compatibility_and_version(compiler) + if with_cuda is None: + with_cuda = any(map(_is_cuda_file, sources)) + if with_sycl is None: + with_sycl = any(map(_is_sycl_file, sources)) + assert not (with_sycl and with_cuda) + extra_ldflags = _prepare_ldflags( + extra_ldflags or [], + with_cuda, + with_sycl, + verbose, + is_standalone) + build_file_path = os.path.join(build_directory, 'build.ninja') + if verbose: + logger.debug('Emitting ninja build file %s...', build_file_path) + + # Create build_directory if it does not exist + if not os.path.exists(build_directory): + if verbose: + logger.debug('Creating directory %s...', build_directory) + # This is like mkdir -p, i.e. will also create parent directories. + os.makedirs(build_directory, exist_ok=True) + + # NOTE: Emitting a new ninja build file does not cause re-compilation if + # the sources did not change, so it's ok to re-emit (and it's fast). + _write_ninja_file_to_build_library( + path=build_file_path, + name=name, + sources=sources, + extra_cflags=extra_cflags or [], + extra_cuda_cflags=extra_cuda_cflags or [], + extra_sycl_cflags=extra_sycl_cflags or [], + extra_ldflags=extra_ldflags or [], + extra_include_paths=extra_include_paths or [], + with_cuda=with_cuda, + with_sycl=with_sycl, + is_standalone=is_standalone) + + if verbose: + logger.info('Building extension module %s...', name) + _run_ninja_build( + build_directory, + verbose, + error_prefix=f"Error building extension '{name}'") + + +def is_ninja_available() -> bool: + """Return ``True`` if the `ninja `_ build system is available on the system, ``False`` otherwise.""" + try: + subprocess.check_output(['ninja', '--version']) + except Exception: + return False + else: + return True + + +def verify_ninja_availability() -> None: + """Raise ``RuntimeError`` if `ninja `_ build system is not available on the system, does nothing otherwise.""" + if not is_ninja_available(): + raise RuntimeError("Ninja is required to load C++ extensions (pip install ninja to get it)") + + +def _prepare_ldflags(extra_ldflags, with_cuda, with_sycl, verbose, is_standalone): + if IS_WINDOWS: + python_lib_path = os.path.join(sys.base_exec_prefix, 'libs') + + extra_ldflags.append('c10.lib') + if with_cuda: + extra_ldflags.append('c10_hip.lib' if IS_HIP_EXTENSION else 'c10_cuda.lib') + if with_sycl: + extra_ldflags.append('c10_xpu.lib') + extra_ldflags.append('torch_cpu.lib') + if with_cuda: + extra_ldflags.append('torch_hip.lib' if IS_HIP_EXTENSION else 'torch_cuda.lib') + # /INCLUDE is used to ensure torch_cuda is linked against in a project that relies on it. + # Related issue: https://github.com/pytorch/pytorch/issues/31611 + extra_ldflags.append('-INCLUDE:?warp_size@cuda@at@@YAHXZ') + if with_sycl: + extra_ldflags.append('torch_xpu.lib') + extra_ldflags.append('torch.lib') + extra_ldflags.append(f'/LIBPATH:{TORCH_LIB_PATH}') + if not is_standalone: + extra_ldflags.append('torch_python.lib') + extra_ldflags.append(f'/LIBPATH:{python_lib_path}') + + else: + extra_ldflags.append(f'-L{TORCH_LIB_PATH}') + extra_ldflags.append('-lc10') + if with_cuda: + extra_ldflags.append('-lc10_hip' if IS_HIP_EXTENSION else '-lc10_cuda') + if with_sycl: + extra_ldflags.append('-lc10_xpu') + extra_ldflags.append('-ltorch_cpu') + if with_cuda: + extra_ldflags.append('-ltorch_hip' if IS_HIP_EXTENSION else '-ltorch_cuda') + if with_sycl: + extra_ldflags.append('-ltorch_xpu') + extra_ldflags.append('-ltorch') + if not is_standalone: + extra_ldflags.append('-ltorch_python') + + if is_standalone: + extra_ldflags.append(f"-Wl,-rpath,{TORCH_LIB_PATH}") + + if with_cuda: + if verbose: + logger.info('Detected CUDA files, patching ldflags') + if IS_WINDOWS and not IS_HIP_EXTENSION: + extra_ldflags.append(f'/LIBPATH:{_join_cuda_home("lib", "x64")}') + extra_ldflags.append('cudart.lib') + if CUDNN_HOME is not None: + extra_ldflags.append(f'/LIBPATH:{os.path.join(CUDNN_HOME, "lib", "x64")}') + elif not IS_HIP_EXTENSION: + extra_lib_dir = "lib64" + if (not os.path.exists(_join_cuda_home(extra_lib_dir)) and + os.path.exists(_join_cuda_home("lib"))): + # 64-bit CUDA may be installed in "lib" + # Note that it's also possible both don't exist (see _find_cuda_home) - in that case we stay with "lib64" + extra_lib_dir = "lib" + extra_ldflags.append(f'-L{_join_cuda_home(extra_lib_dir)}') + extra_ldflags.append('-lcudart') + if CUDNN_HOME is not None: + extra_ldflags.append(f'-L{os.path.join(CUDNN_HOME, "lib64")}') + elif IS_HIP_EXTENSION: + if IS_WINDOWS: + extra_ldflags.append(f'/LIBPATH:{_join_rocm_home("lib")}') + extra_ldflags.append('amdhip64.lib') + else: + extra_ldflags.append(f'-L{_join_rocm_home("lib")}') + extra_ldflags.append('-lamdhip64') + if with_sycl: + if IS_WINDOWS: + extra_ldflags.append(f'/LIBPATH:{_join_sycl_home("lib")}') + extra_ldflags.append('sycl.lib') + else: + extra_ldflags.append(f'-L{_join_sycl_home("lib")}') + extra_ldflags.append('-lsycl') + return extra_ldflags + + +def _get_cuda_arch_flags(cflags: list[str] | None = None) -> list[str]: + """ + Determine CUDA arch flags to use. + + For an arch, say "6.1", the added compile flag will be + ``-gencode=arch=compute_61,code=sm_61``. + For an added "+PTX", an additional + ``-gencode=arch=compute_xx,code=compute_xx`` is added. + + See select_compute_arch.cmake for corresponding named and supported arches + when building with CMake. + """ + # If cflags is given, there may already be user-provided arch flags in it + # (from `extra_compile_args`) + if cflags is not None: + for flag in cflags: + if 'TORCH_EXTENSION_NAME' in flag: + continue + if 'arch' in flag: + return [] + + # Note: keep combined names ("arch1+arch2") above single names, otherwise + # string replacement may not do the right thing + named_arches = collections.OrderedDict([ + ('Kepler+Tesla', '3.7'), + ('Kepler', '3.5+PTX'), + ('Maxwell+Tegra', '5.3'), + ('Maxwell', '5.0;5.2+PTX'), + ('Pascal', '6.0;6.1+PTX'), + ('Volta+Tegra', '7.2'), + ('Volta', '7.0+PTX'), + ('Turing', '7.5+PTX'), + ('Ampere+Tegra', '8.7'), + ('Ampere', '8.0;8.6+PTX'), + ('Ada', '8.9+PTX'), + ('Hopper', '9.0+PTX'), + ('Blackwell+Tegra', '11.0'), + ('Blackwell', '10.0;10.3;12.0;12.1+PTX'), + ]) + + supported_arches = ['3.5', '3.7', '5.0', '5.2', '5.3', '6.0', '6.1', '6.2', + '7.0', '7.2', '7.5', '8.0', '8.6', '8.7', '8.9', '9.0', '9.0a', + '10.0', '10.0a', '11.0', '11.0a', '10.3', '10.3a', '12.0', + '12.0a', '12.1', '12.1a'] + valid_arch_strings = supported_arches + [s + "+PTX" for s in supported_arches] + + # The default is sm_30 for CUDA 9.x and 10.x + # First check for an env var (same as used by the main setup.py) + # Can be one or more architectures, e.g. "6.1" or "3.5;5.2;6.0;6.1;7.0+PTX" + # See cmake/Modules_CUDA_fix/upstream/FindCUDA/select_compute_arch.cmake + _arch_list = os.environ.get('TORCH_CUDA_ARCH_LIST', None) + + # If not given or set as native, determine what's best for the GPU / CUDA version that can be found + if not _arch_list or _arch_list == "native": + arch_list = [] + # the assumption is that the extension should run on any of the currently visible cards, + # which could be of different types - therefore all archs for visible cards should be included + for i in range(torch.cuda.device_count()): + capability = torch.cuda.get_device_capability(i) + supported_sm = [int("".join(re.findall(r"\d+", arch.split('_')[1]))) + for arch in torch.cuda.get_arch_list() if 'sm_' in arch] + max_supported_sm = max((sm // 10, sm % 10) for sm in supported_sm) + # Capability of the device may be higher than what's supported by the user's + # NVCC, causing compilation error. User's NVCC is expected to match the one + # used to build pytorch, so we use the maximum supported capability of pytorch + # to clamp the capability. + capability = min(max_supported_sm, capability) + arch = f'{capability[0]}.{capability[1]}' + if arch not in arch_list: + arch_list.append(arch) + arch_list = sorted(arch_list) + arch_list[-1] += '+PTX' + + if not _arch_list: + # Only log on rank 0 in distributed settings to avoid spam + if not torch.distributed.is_available() or not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0: + arch_list_str = ';'.join(arch_list) + logger.debug( + "TORCH_CUDA_ARCH_LIST is not set, using TORCH_CUDA_ARCH_LIST='%s' " + "for visible GPU architectures. Set os.environ['TORCH_CUDA_ARCH_LIST'] to override.", + arch_list_str) + else: + # Deal with lists that are ' ' separated (only deal with ';' after) + _arch_list = _arch_list.replace(' ', ';') + # Expand named arches + for named_arch, archival in named_arches.items(): + _arch_list = _arch_list.replace(named_arch, archival) + + arch_list = _arch_list.split(';') + + flags = [] + for arch in arch_list: + if arch not in valid_arch_strings: + raise ValueError(f"Unknown CUDA arch ({arch}) or GPU not supported") + else: + # Handle both single and double-digit architecture versions + version = arch.split('+')[0] # Remove "+PTX" if present + major, minor = version.split('.') + num = f"{major}{minor}" + flags.append(f'-gencode=arch=compute_{num},code=sm_{num}') + if arch.endswith('+PTX'): + flags.append(f'-gencode=arch=compute_{num},code=compute_{num}') + + return sorted(set(flags)) + + +def _get_rocm_arch_flags(cflags: list[str] | None = None) -> list[str]: + # If cflags is given, there may already be user-provided arch flags in it + # (from `extra_compile_args`). If user also specified -fgpu-rdc or -fno-gpu-rdc, we + # assume they know what they're doing. Otherwise, we force -fno-gpu-rdc default. + has_gpu_rdc_flag = False + if cflags is not None: + has_custom_flags = False + for flag in cflags: + if 'amdgpu-target' in flag or 'offload-arch' in flag: + has_custom_flags = True + elif 'gpu-rdc' in flag: + has_gpu_rdc_flag = True + if has_custom_flags: + return [] if has_gpu_rdc_flag else ['-fno-gpu-rdc'] + # Use same defaults as used for building PyTorch + # Allow env var to override, just like during initial cmake build. + _archs = os.environ.get('PYTORCH_ROCM_ARCH', None) + if not _archs: + archFlags = torch._C._cuda_getArchFlags() + if archFlags: + archs = archFlags.split() + else: + archs = [] + else: + archs = _archs.replace(' ', ';').split(';') + flags = [f'--offload-arch={arch}' for arch in archs] + flags += [] if has_gpu_rdc_flag else ['-fno-gpu-rdc'] + return flags + +def _get_build_directory(name: str, verbose: bool) -> str: + """ + Get the build directory for an extension. + + Args: + name: The name of the extension + verbose: Whether to print verbose information + + Returns: + The path to the build directory + """ + root_extensions_directory = os.environ.get('TORCH_EXTENSIONS_DIR') + if root_extensions_directory is None: + root_extensions_directory = get_default_build_root() + cu_str = ('cpu' if torch.version.cuda is None else + f'cu{torch.version.cuda.replace(".", "")}') + python_version = f'py{sys.version_info.major}{sys.version_info.minor}{getattr(sys, "abiflags", "")}' + build_folder = f'{python_version}_{cu_str}' + + root_extensions_directory = os.path.join( + root_extensions_directory, build_folder) + + if verbose: + logger.info('Using %s as PyTorch extensions root...', root_extensions_directory) + + build_directory = os.path.join(root_extensions_directory, name) + if not os.path.exists(build_directory): + if verbose: + logger.debug('Creating extension directory %s...', build_directory) + # This is like mkdir -p, i.e. will also create parent directories. + os.makedirs(build_directory, exist_ok=True) + + return build_directory + + +def _get_num_workers(verbose: bool) -> int | None: + max_jobs = os.environ.get('MAX_JOBS') + if max_jobs is not None and max_jobs.isdigit(): + if verbose: + logger.debug('Using envvar MAX_JOBS (%s) as the number of workers...', max_jobs) + return int(max_jobs) + if verbose: + logger.info( + 'Allowing ninja to set a default number of workers... ' + '(overridable by setting the environment variable MAX_JOBS=N)' + ) + return None + + +def _get_vc_env(vc_arch: str) -> dict[str, str]: + try: + from setuptools import distutils # type: ignore[attr-defined] + # pyrefly: ignore [missing-attribute] + return distutils._msvccompiler._get_vc_env(vc_arch) + except AttributeError: + try: + from setuptools._distutils import _msvccompiler + return _msvccompiler._get_vc_env(vc_arch) # type: ignore[attr-defined] + except AttributeError: + from setuptools._distutils.compilers.C import msvc + return msvc._get_vc_env(vc_arch) # type: ignore[attr-defined] + +def _run_ninja_build(build_directory: str, verbose: bool, error_prefix: str) -> None: + command = ['ninja', '-v'] + num_workers = _get_num_workers(verbose) + if num_workers is not None: + command.extend(['-j', str(num_workers)]) + env = os.environ.copy() + # Try to activate the vc env for the users + if IS_WINDOWS and 'VSCMD_ARG_TGT_ARCH' not in env: + from setuptools import distutils # type: ignore[attr-defined] + + plat_name = distutils.util.get_platform() + plat_spec = PLAT_TO_VCVARS[plat_name] + vc_env = {k.upper(): v for k, v in _get_vc_env(plat_spec).items()} + for k, v in env.items(): + uk = k.upper() + if uk not in vc_env: + vc_env[uk] = v + env = vc_env + try: + sys.stdout.flush() + sys.stderr.flush() + # Warning: don't pass stdout=None to subprocess.run to get output. + # subprocess.run assumes that sys.__stdout__ has not been modified and + # attempts to write to it by default. However, when we call _run_ninja_build + # from ahead-of-time cpp extensions, the following happens: + # 1) If the stdout encoding is not utf-8, setuptools detaches __stdout__. + # https://github.com/pypa/setuptools/blob/7e97def47723303fafabe48b22168bbc11bb4821/setuptools/dist.py#L1110 + # (it probably shouldn't do this) + # 2) subprocess.run (on POSIX, with no stdout override) relies on + # __stdout__ not being detached: + # https://github.com/python/cpython/blob/c352e6c7446c894b13643f538db312092b351789/Lib/subprocess.py#L1214 + # To work around this, we pass in the fileno directly and hope that + # it is valid. + stdout_fileno = 1 + subprocess.run( + command, + shell=IS_WINDOWS and IS_HIP_EXTENSION, + stdout=stdout_fileno if verbose else subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd=build_directory, + check=True, + env=env) + except subprocess.CalledProcessError as e: + # Python 2 and 3 compatible way of getting the error object. + _, error, _ = sys.exc_info() + # error.output contains the stdout and stderr of the build attempt. + message = error_prefix + # `error` is a CalledProcessError (which has an `output`) attribute, but + # mypy thinks it's Optional[BaseException] and doesn't narrow + if hasattr(error, 'output') and error.output: # type: ignore[union-attr] + message += f": {error.output.decode(*SUBPROCESS_DECODE_ARGS)}" # type: ignore[union-attr] + raise RuntimeError(message) from e + + +def _get_exec_path(module_name, path): + if IS_WINDOWS and TORCH_LIB_PATH not in os.getenv('PATH', '').split(';'): + torch_lib_in_path = any( + os.path.exists(p) and os.path.samefile(p, TORCH_LIB_PATH) + for p in os.getenv('PATH', '').split(';') + ) + if not torch_lib_in_path: + os.environ['PATH'] = f"{TORCH_LIB_PATH};{os.getenv('PATH', '')}" + return os.path.join(path, f'{module_name}{EXEC_EXT}') + + +def _import_module_from_library(module_name, path, is_python_module): + filepath = os.path.join(path, f"{module_name}{LIB_EXT}") + if is_python_module: + # https://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path + spec = importlib.util.spec_from_file_location(module_name, filepath) + if spec is None: + raise AssertionError(f"Failed to create spec for module {module_name} at {filepath}") + module = importlib.util.module_from_spec(spec) + if not isinstance(spec.loader, importlib.abc.Loader): + raise AssertionError("spec.loader is not a valid importlib Loader") + spec.loader.exec_module(module) + return module + else: + torch.ops.load_library(filepath) + return filepath + + +def _write_ninja_file_to_build_library(path, + name, + sources, + extra_cflags, + extra_cuda_cflags, + extra_sycl_cflags, + extra_ldflags, + extra_include_paths, + with_cuda, + with_sycl, + is_standalone) -> None: + extra_cflags = [flag.strip() for flag in extra_cflags] + extra_cuda_cflags = [flag.strip() for flag in extra_cuda_cflags] + extra_sycl_cflags = [flag.strip() for flag in extra_sycl_cflags] + extra_ldflags = [flag.strip() for flag in extra_ldflags] + extra_include_paths = [flag.strip() for flag in extra_include_paths] + + # Turn into absolute paths so we can emit them into the ninja build + # file wherever it is. + user_includes = [os.path.abspath(file) for file in extra_include_paths] + + # include_paths() gives us the location of torch/extension.h + # TODO generalize with_cuda as specific device type. + if with_cuda: + system_includes = include_paths("cuda") + elif with_sycl: + system_includes = include_paths("xpu") + else: + system_includes = include_paths("cpu") + # sysconfig.get_path('include') gives us the location of Python.h + # Explicitly specify 'posix_prefix' scheme on non-Windows platforms to workaround error on some MacOS + # installations where default `get_path` points to non-existing `/Library/Python/M.m/include` folder + python_include_path = sysconfig.get_path('include', scheme='nt' if IS_WINDOWS else 'posix_prefix') + if python_include_path is not None: + system_includes.append(python_include_path) + + common_cflags = [] + if not is_standalone: + common_cflags.append(f'-DTORCH_EXTENSION_NAME={name}') + common_cflags.append('-DTORCH_API_INCLUDE_EXTENSION_H') + + # Windows does not understand `-isystem` and quotes flags later. + if IS_WINDOWS: + common_cflags += [f'-I{include}' for include in user_includes + system_includes] + else: + common_cflags += [f'-I{shlex.quote(include)}' for include in user_includes] + common_cflags += [f'-isystem {shlex.quote(include)}' for include in system_includes] + + if IS_WINDOWS: + COMMON_HIP_FLAGS.extend(['-fms-runtime-lib=dll']) + cflags = common_cflags + ['/std:c++17'] + extra_cflags + cflags += COMMON_MSVC_FLAGS + (COMMON_HIP_FLAGS if IS_HIP_EXTENSION else []) + cflags = _nt_quote_args(cflags) + else: + cflags = common_cflags + ['-fPIC', '-std=c++17'] + extra_cflags + + if with_cuda and IS_HIP_EXTENSION: + cuda_flags = ['-DWITH_HIP'] + common_cflags + extra_cflags + COMMON_HIP_FLAGS + COMMON_HIPCC_FLAGS + cuda_flags = cuda_flags + ['-std=c++17'] + cuda_flags += _get_rocm_arch_flags(cuda_flags) + cuda_flags += extra_cuda_cflags + if IS_WINDOWS: + cuda_flags = _nt_quote_args(cuda_flags) + elif with_cuda: + cuda_flags = common_cflags + COMMON_NVCC_FLAGS + _get_cuda_arch_flags(extra_cuda_cflags) + if IS_WINDOWS: + for flag in COMMON_MSVC_FLAGS: + cuda_flags = ['-Xcompiler', flag] + cuda_flags + for ignore_warning in MSVC_IGNORE_CUDAFE_WARNINGS: + cuda_flags = ['-Xcudafe', '--diag_suppress=' + ignore_warning] + cuda_flags + cuda_flags = cuda_flags + ['-std=c++17'] + cuda_flags = _nt_quote_args(cuda_flags) + cuda_flags += _nt_quote_args(extra_cuda_cflags) + else: + cuda_flags += ['--compiler-options', "'-fPIC'"] + cuda_flags += extra_cuda_cflags + if not any(flag.startswith('-std=') for flag in cuda_flags): + cuda_flags.append('-std=c++17') + cc_env = os.getenv("CC") + if cc_env is not None: + cuda_flags = ['-ccbin', cc_env] + cuda_flags + else: + cuda_flags = None + + if with_sycl: + sycl_cflags = cflags + _COMMON_SYCL_FLAGS + sycl_cflags += extra_sycl_cflags + _append_sycl_targets_if_missing(sycl_cflags) + _append_sycl_std_if_no_std_present(sycl_cflags) + host_cflags = cflags + # escaping quoted arguments to pass them thru SYCL compiler + icpx_version = _get_icpx_version() + if int(icpx_version) < 20250200: + host_cflags = [item.replace('\\"', '\\\\"') for item in host_cflags] + + sycl_cflags += _wrap_sycl_host_flags(host_cflags) + sycl_dlink_post_cflags = _SYCL_DLINK_FLAGS.copy() + sycl_dlink_post_cflags += _get_sycl_device_flags(sycl_cflags) + else: + sycl_cflags = None + sycl_dlink_post_cflags = None + + def object_file_path(source_file: str) -> str: + # '/path/to/file.cpp' -> 'file' + file_name = os.path.splitext(os.path.basename(source_file))[0] + if _is_cuda_file(source_file) and with_cuda: + # Use a different object filename in case a C++ and CUDA file have + # the same filename but different extension (.cpp vs. .cu). + target = f'{file_name}.cuda.o' + elif _is_sycl_file(source_file) and with_sycl: + target = f'{file_name}.sycl.o' + else: + target = f'{file_name}.o' + return target + + objects = [object_file_path(src) for src in sources] + ldflags = ([] if is_standalone else [SHARED_FLAG]) + extra_ldflags + + # The darwin linker needs explicit consent to ignore unresolved symbols. + if IS_MACOS: + ldflags.append('-undefined dynamic_lookup') + elif IS_WINDOWS: + ldflags = _nt_quote_args(ldflags) + + ext = EXEC_EXT if is_standalone else LIB_EXT + library_target = f'{name}{ext}' + + _write_ninja_file( + path=path, + cflags=cflags, + post_cflags=None, + cuda_cflags=cuda_flags, + cuda_post_cflags=None, + cuda_dlink_post_cflags=None, + sycl_cflags=sycl_cflags, + sycl_post_cflags=[], + sycl_dlink_post_cflags=sycl_dlink_post_cflags, + sources=sources, + objects=objects, + ldflags=ldflags, + library_target=library_target, + with_cuda=with_cuda, + with_sycl=with_sycl) + + +def _write_ninja_file(path, + cflags, + post_cflags, + cuda_cflags, + cuda_post_cflags, + cuda_dlink_post_cflags, + sycl_cflags, + sycl_post_cflags, + sycl_dlink_post_cflags, + sources, + objects, + ldflags, + library_target, + with_cuda, + with_sycl) -> None: + r"""Write a ninja file that does the desired compiling and linking. + + `path`: Where to write this file + `cflags`: list of flags to pass to $cxx. Can be None. + `post_cflags`: list of flags to append to the $cxx invocation. Can be None. + `cuda_cflags`: list of flags to pass to $nvcc. Can be None. + `cuda_post_cflags`: list of flags to append to the $nvcc invocation. Can be None. + `cuda_dlink_post_cflags`: list of flags to append to the $nvcc device code link invocation. Can be None. + `sycl_cflags`: list of flags to pass to SYCL compiler. Can be None. + `sycl_post_cflags`: list of flags to append to the SYCL compiler invocation. Can be None. + `sycl_dlink_post_cflags`: list of flags to append to the SYCL compiler device code link invocation. Can be None. +e. + `sources`: list of paths to source files + `objects`: list of desired paths to objects, one per source. + `ldflags`: list of flags to pass to linker. Can be None. + `library_target`: Name of the output library. Can be None; in that case, + we do no linking. + `with_cuda`: If we should be compiling with CUDA. + """ + def sanitize_flags(flags): + if flags is None: + return [] + else: + return [flag.strip() for flag in flags] + + cflags = sanitize_flags(cflags) + post_cflags = sanitize_flags(post_cflags) + cuda_cflags = sanitize_flags(cuda_cflags) + cuda_post_cflags = sanitize_flags(cuda_post_cflags) + cuda_dlink_post_cflags = sanitize_flags(cuda_dlink_post_cflags) + sycl_cflags = sanitize_flags(sycl_cflags) + sycl_post_cflags = sanitize_flags(sycl_post_cflags) + sycl_dlink_post_cflags = sanitize_flags(sycl_dlink_post_cflags) + ldflags = sanitize_flags(ldflags) + + # Sanity checks... + if len(sources) != len(objects): + raise AssertionError("sources and objects lists must be the same length") + if len(sources) == 0: + raise AssertionError("At least one source is required to build a library") + + compiler = get_cxx_compiler() + + # Version 1.3 is required for the `deps` directive. + config = ['ninja_required_version = 1.3'] + config.append(f'cxx = {compiler}') + if with_cuda or cuda_dlink_post_cflags: + if "PYTORCH_NVCC" in os.environ: + nvcc = os.getenv("PYTORCH_NVCC") # user can set nvcc compiler with ccache using the environment variable here + else: + if IS_HIP_EXTENSION: + nvcc = _get_hipcc_path() + else: + nvcc = _join_cuda_home('bin', 'nvcc') + config.append(f'nvcc = {nvcc}') + if with_sycl or sycl_dlink_post_cflags: + sycl = 'icx' if IS_WINDOWS else 'icpx' + config.append(f'sycl = {sycl}') + + if IS_HIP_EXTENSION: + post_cflags = COMMON_HIP_FLAGS + post_cflags + flags = [f'cflags = {" ".join(cflags)}'] + flags.append(f'post_cflags = {" ".join(post_cflags)}') + if with_cuda: + flags.append(f'cuda_cflags = {" ".join(cuda_cflags)}') + flags.append(f'cuda_post_cflags = {" ".join(cuda_post_cflags)}') + flags.append(f'cuda_dlink_post_cflags = {" ".join(cuda_dlink_post_cflags)}') + if with_sycl: + flags.append(f'sycl_cflags = {" ".join(sycl_cflags)}') + flags.append(f'sycl_post_cflags = {" ".join(sycl_post_cflags)}') + flags.append(f'sycl_dlink_post_cflags = {" ".join(sycl_dlink_post_cflags)}') + flags.append(f'ldflags = {" ".join(ldflags)}') + + # Turn into absolute paths so we can emit them into the ninja build + # file wherever it is. + sources = [os.path.abspath(file) for file in sources] + + # See https://ninja-build.org/build.ninja.html for reference. + compile_rule = ['rule compile'] + if IS_WINDOWS: + compiler_name = "$cxx" if IS_HIP_EXTENSION else "cl" + compile_rule.append( + f' command = {compiler_name} ' + '/showIncludes $cflags -c $in /Fo$out $post_cflags' # codespell:ignore + ) + if not IS_HIP_EXTENSION: + compile_rule.append(' deps = msvc') + else: + compile_rule.append( + ' command = $cxx -MMD -MF $out.d $cflags -c $in -o $out $post_cflags') + compile_rule.append(' depfile = $out.d') + compile_rule.append(' deps = gcc') + + if with_cuda: + cuda_compile_rule = ['rule cuda_compile'] + nvcc_gendeps = '' + # --generate-dependencies-with-compile is not supported by ROCm + # Nvcc flag `--generate-dependencies-with-compile` is not supported by sccache, which may increase build time. + if torch.version.cuda is not None and os.getenv('TORCH_EXTENSION_SKIP_NVCC_GEN_DEPENDENCIES', '0') != '1': + cuda_compile_rule.append(' depfile = $out.d') + cuda_compile_rule.append(' deps = gcc') + # Note: non-system deps with nvcc are only supported + # on Linux so use --generate-dependencies-with-compile + # to make this work on Windows too. + nvcc_gendeps = '--generate-dependencies-with-compile --dependency-output $out.d' + cuda_compile_rule.append( + f' command = $nvcc {nvcc_gendeps} $cuda_cflags -c $in -o $out $cuda_post_cflags') + + if with_sycl: + sycl_compile_rule = ['rule sycl_compile'] + # SYCL compiler does not recognize .sycl extension automatically, + # so we pass '-x c++' explicitly notifying compiler of file format + sycl_compile_rule.append( + ' command = $sycl $sycl_cflags -c -x c++ $in -o $out $sycl_post_cflags') + + + # Emit one build rule per source to enable incremental build. + build = [] + for source_file, object_file in zip(sources, objects, strict=True): + is_cuda_source = _is_cuda_file(source_file) and with_cuda + is_sycl_source = _is_sycl_file(source_file) and with_sycl + if is_cuda_source: + rule = 'cuda_compile' + elif is_sycl_source: + rule = 'sycl_compile' + else: + rule = 'compile' + if IS_WINDOWS: + source_file = source_file.replace(':', '$:') + object_file = object_file.replace(':', '$:') + source_file = source_file.replace(" ", "$ ") + object_file = object_file.replace(" ", "$ ") + build.append(f'build {object_file}: {rule} {source_file}') + + if cuda_dlink_post_cflags: + cuda_devlink_out = os.path.join(os.path.dirname(objects[0]), 'dlink.o') + cuda_devlink_rule = ['rule cuda_devlink'] + cuda_devlink_rule.append(' command = $nvcc $in -o $out $cuda_dlink_post_cflags') + cuda_devlink = [f'build {cuda_devlink_out}: cuda_devlink {" ".join(objects)}'] + objects += [cuda_devlink_out] + else: + cuda_devlink_rule, cuda_devlink = [], [] + + if sycl_dlink_post_cflags: + sycl_devlink_out = os.path.join(os.path.dirname(objects[0]), "sycl_dlink.o") + if IS_WINDOWS: + sycl_devlink_objects = [obj.replace(":", "$:") for obj in objects] + objects += [sycl_devlink_out] + sycl_devlink_out = sycl_devlink_out.replace(":", "$:") + else: + sycl_devlink_objects = list(objects) + objects += [sycl_devlink_out] + sycl_devlink_rule = ["rule sycl_devlink"] + sycl_devlink_rule.append( + " command = $sycl $in -o $out $sycl_dlink_post_cflags" + ) + sycl_devlink = [ + f"build {sycl_devlink_out}: sycl_devlink {' '.join(sycl_devlink_objects)}" + ] + else: + sycl_devlink_rule, sycl_devlink = [], [] + + if library_target is not None: + link_rule = ['rule link'] + if IS_WINDOWS: + cl_paths = subprocess.check_output(['where', + 'cl']).decode(*SUBPROCESS_DECODE_ARGS).split('\r\n') + if len(cl_paths) >= 1: + cl_path = os.path.dirname(cl_paths[0]).replace(':', '$:') + else: + raise RuntimeError("MSVC is required to load C++ extensions") + link_rule.append(f' command = "{cl_path}/link.exe" $in /nologo $ldflags /out:$out') + else: + link_rule.append(' command = $cxx $in $ldflags -o $out') + + link = [f'build {library_target}: link {" ".join(objects)}'] + + default = [f'default {library_target}'] + else: + link_rule, link, default = [], [], [] + + # 'Blocks' should be separated by newlines, for visual benefit. + blocks = [config, flags, compile_rule] + if with_cuda: + blocks.append(cuda_compile_rule) # type: ignore[possibly-undefined] + if with_sycl: + blocks.append(sycl_compile_rule) # type: ignore[possibly-undefined] + blocks += [cuda_devlink_rule, sycl_devlink_rule, link_rule, build, cuda_devlink, sycl_devlink, link, default] + content = "\n\n".join("\n".join(b) for b in blocks) + # Ninja requires a new lines at the end of the .ninja file + content += "\n" + _maybe_write(path, content) + +def _join_cuda_home(*paths) -> str: + """ + Join paths with CUDA_HOME, or raises an error if it CUDA_HOME is not set. + + This is basically a lazy way of raising an error for missing $CUDA_HOME + only once we need to get any CUDA-specific path. + """ + if CUDA_HOME is None: + raise OSError('CUDA_HOME environment variable is not set. ' + 'Please set it to your CUDA install root.') + return os.path.join(CUDA_HOME, *paths) + + +def _is_cuda_file(path: str) -> bool: + valid_ext = ['.cu', '.cuh'] + if IS_HIP_EXTENSION: + valid_ext.append('.hip') + return os.path.splitext(path)[1] in valid_ext + +def _is_sycl_file(path: str) -> bool: + valid_ext = ['.sycl'] + return os.path.splitext(path)[1] in valid_ext diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/deterministic.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/deterministic.py new file mode 100644 index 0000000000000000000000000000000000000000..a055c43be531a5c65c4f29f6c8165104e98e5ca0 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/deterministic.py @@ -0,0 +1,22 @@ +# mypy: allow-untyped-defs +import sys +import types + +import torch + + +class _Deterministic(types.ModuleType): + @property + def fill_uninitialized_memory(self): + """ + Whether to fill uninitialized memory with a known value when + :meth:`torch.use_deterministic_algorithms()` is set to ``True``. + """ + return torch._C._get_deterministic_fill_uninitialized_memory() + + @fill_uninitialized_memory.setter + def fill_uninitialized_memory(self, mode): + return torch._C._set_deterministic_fill_uninitialized_memory(mode) + + +sys.modules[__name__].__class__ = _Deterministic diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/dlpack.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/dlpack.py new file mode 100644 index 0000000000000000000000000000000000000000..aef32100ee7105d364f0e144dc1cf2e0368f7767 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/dlpack.py @@ -0,0 +1,231 @@ +from typing import Any + +import torch +import enum + +from torch._C import _to_dlpack as to_dlpack +from torch.types import Device as _Device + +__all__ = [ + "DLDeviceType", + "from_dlpack", +] + +class DLDeviceType(enum.IntEnum): + # Enums as in DLPack specification (aten/src/ATen/dlpack.h) + kDLCPU = 1, + kDLCUDA = 2, + kDLCUDAHost = 3, + kDLOpenCL = 4, + kDLVulkan = 7, + kDLMetal = 8, + kDLVPI = 9, + kDLROCM = 10, + kDLROCMHost = 11, + kDLExtDev = 12, + kDLCUDAManaged = 13, + kDLOneAPI = 14, + kDLWebGPU = 15, + kDLHexagon = 16, + kDLMAIA = 17, + + +torch._C._add_docstr(to_dlpack, r"""to_dlpack(tensor) -> PyCapsule + +Returns an opaque object (a "DLPack capsule") representing the tensor. + +.. note:: + ``to_dlpack`` is a legacy DLPack interface. The capsule it returns + cannot be used for anything in Python other than use it as input to + ``from_dlpack``. The more idiomatic use of DLPack is to call + ``from_dlpack`` directly on the tensor object - this works when that + object has a ``__dlpack__`` method, which PyTorch and most other + libraries indeed have now. + +.. warning:: + Only call ``from_dlpack`` once per capsule produced with ``to_dlpack``. + Behavior when a capsule is consumed multiple times is undefined. + +Args: + tensor: a tensor to be exported + +The DLPack capsule shares the tensor's memory. +""") + + +# TODO: add a typing.Protocol to be able to tell Mypy that only objects with +# __dlpack__ and __dlpack_device__ methods are accepted. +def from_dlpack( + ext_tensor: Any, + *, + device: _Device | None = None, + copy: bool | None = None +) -> 'torch.Tensor': + """from_dlpack(ext_tensor) -> Tensor + + Converts a tensor from an external library into a ``torch.Tensor``. + + The returned PyTorch tensor will share the memory with the input tensor + (which may have come from another library). Note that in-place operations + will therefore also affect the data of the input tensor. This may lead to + unexpected issues (e.g., other libraries may have read-only flags or + immutable data structures), so the user should only do this if they know + for sure that this is fine. + + Args: + ext_tensor (object with ``__dlpack__`` attribute, or a DLPack capsule): + The tensor or DLPack capsule to convert. + + If ``ext_tensor`` is a tensor (or ndarray) object, it must support + the ``__dlpack__`` protocol (i.e., have a ``ext_tensor.__dlpack__`` + method). Otherwise ``ext_tensor`` may be a DLPack capsule, which is + an opaque ``PyCapsule`` instance, typically produced by a + ``to_dlpack`` function or method. + + device (torch.device or str or None): An optional PyTorch device + specifying where to place the new tensor. If None (default), the + new tensor will be on the same device as ``ext_tensor``. + + copy (bool or None): An optional boolean indicating whether or not to copy + ``self``. If None, PyTorch will copy only if necessary. + + Examples:: + + >>> import torch.utils.dlpack + >>> t = torch.arange(4) + + # Convert a tensor directly (supported in PyTorch >= 1.10) + >>> t2 = torch.from_dlpack(t) + >>> t2[:2] = -1 # show that memory is shared + >>> t2 + tensor([-1, -1, 2, 3]) + >>> t + tensor([-1, -1, 2, 3]) + + # The old-style DLPack usage, with an intermediate capsule object + >>> capsule = torch.utils.dlpack.to_dlpack(t) + >>> capsule + + >>> t3 = torch.from_dlpack(capsule) + >>> t3 + tensor([-1, -1, 2, 3]) + >>> t3[0] = -9 # now we're sharing memory between 3 tensors + >>> t3 + tensor([-9, -1, 2, 3]) + >>> t2 + tensor([-9, -1, 2, 3]) + >>> t + tensor([-9, -1, 2, 3]) + + """ + + if hasattr(ext_tensor, '__dlpack__'): + # Only populate kwargs if any of the optional arguments are, in fact, not None. Otherwise, + # leave them out, since we might end up falling back to no-extra-kwargs __dlpack__ call. + kwargs: dict[str, Any] = {} + kwargs["max_version"] = (1, 0) + + # Track copy request for potential manual handling + requested_copy = copy + producer_handled_copy = True + cross_device_transfer = False # Will be set to True if device transfer is needed + + if copy is not None: + kwargs["copy"] = copy + + # Parse the device parameter. + # At this moment, it can either be a torch.device or a str representing + # a torch.device, e.g. "cpu", "cuda", etc. + # Get source device first (we need it to detect cross-device transfers) + ext_device = ext_tensor.__dlpack_device__() + + if device is not None: + if isinstance(device, str): + device = torch.device(device) + if not isinstance(device, torch.device): + raise AssertionError(f"from_dlpack: unsupported device type: {type(device)}") + + # Convert target device to DLPack format + target_dl_device = torch._C._torchDeviceToDLDevice(device) + + # Detect cross-device transfer by comparing source and target devices + # E.g. CPU->CUDA, cuda:0->cuda:1, etc. + cross_device_transfer = (ext_device != target_dl_device) + + # Only pass dl_device to producer if NOT cross-device transfer + if not cross_device_transfer: + kwargs["dl_device"] = target_dl_device + + # Cross-device transfer always requires a copy + if cross_device_transfer and copy is False: + raise ValueError( + f"cannot move DLPack tensor from device {ext_device} to {target_dl_device} " + "without copying. Set copy=None or copy=True." + ) + + # ext_device is either CUDA or ROCm, we need to pass the current + # stream + if ext_device[0] in (DLDeviceType.kDLCUDA, DLDeviceType.kDLROCM): + stream = torch.cuda.current_stream(f'cuda:{ext_device[1]}') + # cuda_stream is the pointer to the stream and it is a public + # attribute, but it is not documented + # The array API specify that the default legacy stream must be passed + # with a value of 1 for CUDA + # https://data-apis.org/array-api/latest/API_specification/array_object.html?dlpack-self-stream-none#dlpack-self-stream-none + is_cuda = ext_device[0] == DLDeviceType.kDLCUDA + # Since pytorch is not using PTDS by default, lets directly pass + # the legacy stream + stream_ptr = 1 if is_cuda and stream.cuda_stream == 0 else stream.cuda_stream + kwargs["stream"] = stream_ptr + + # Try different parameter combinations until one works + dlpack = None + + # Attempt 1: Try with all the parameters + try: + dlpack = ext_tensor.__dlpack__(**kwargs) + except TypeError: + pass + + # Attempt 2: Remove max_version + if dlpack is None: + kwargs.pop("max_version", None) + try: + dlpack = ext_tensor.__dlpack__(**kwargs) + except TypeError: + pass + + # Attempt 3: Remove copy + if dlpack is None: + kwargs.pop("copy", None) + producer_handled_copy = False + try: + dlpack = ext_tensor.__dlpack__(**kwargs) + except TypeError: + pass + + # Attempt 4: Remove dl_device + if dlpack is None: + kwargs.pop("dl_device", None) + dlpack = ext_tensor.__dlpack__(**kwargs) + + tensor = torch._C._from_dlpack(dlpack) + + # Manual copy if producer didn't handle it (cross-device already copies via .to()) + if requested_copy is True and not producer_handled_copy and not cross_device_transfer: + tensor = tensor.clone() + + # Handle cross-device transfer by moving tensor to target device + if cross_device_transfer: + tensor = tensor.to(device) + + return tensor + + else: + if device is not None or copy is not None: + raise AssertionError( + "device and copy kwargs not supported when ext_tensor is already a DLPack capsule." + ) + # Old versions just call the converter + dlpack = ext_tensor + return torch._C._from_dlpack(dlpack) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/file_baton.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/file_baton.py new file mode 100644 index 0000000000000000000000000000000000000000..5b4f55d8c88dd6ab20fc17000f9d4b2c2a42b88d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/file_baton.py @@ -0,0 +1,64 @@ +# mypy: allow-untyped-defs +import os +import time +import warnings + + +class FileBaton: + """A primitive, file-based synchronization utility.""" + + def __init__(self, lock_file_path, wait_seconds=0.1, warn_after_seconds=None) -> None: + """ + Create a new :class:`FileBaton`. + + Args: + lock_file_path: The path to the file used for locking. + wait_seconds: The seconds to periodically sleep (spin) when + calling ``wait()``. + warn_after_seconds: The seconds to wait before showing + lock file path to warn existing lock file. + """ + self.lock_file_path = lock_file_path + self.wait_seconds = wait_seconds + self.fd = None + self.warn_after_seconds = warn_after_seconds + + def try_acquire(self) -> bool | None: + """ + Try to atomically create a file under exclusive access. + + Returns: + True if the file could be created, else False. + """ + try: + # pyrefly: ignore [bad-assignment] + self.fd = os.open(self.lock_file_path, os.O_CREAT | os.O_EXCL) + return True + except FileExistsError: + return False + + def wait(self) -> None: + """ + Periodically sleeps for a certain amount until the baton is released. + + The amount of time slept depends on the ``wait_seconds`` parameter + passed to the constructor. + """ + has_warned = False + + start_time = time.time() + while os.path.exists(self.lock_file_path): + time.sleep(self.wait_seconds) + + if self.warn_after_seconds is not None: + if time.time() - start_time > self.warn_after_seconds and not has_warned: + warnings.warn(f'Waited on lock file "{self.lock_file_path}" for ' + f'{self.warn_after_seconds} seconds.', stacklevel=2) + has_warned = True + + def release(self) -> None: + """Release the baton and removes its file.""" + if self.fd is not None: + os.close(self.fd) + + os.remove(self.lock_file_path) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/flop_counter.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/flop_counter.py new file mode 100644 index 0000000000000000000000000000000000000000..7d08a14158300326931cb4026e5ef912dd8219c3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/flop_counter.py @@ -0,0 +1,896 @@ +# mypy: allow-untyped-defs +import torch +from torch.utils._pytree import tree_map, tree_flatten, tree_unflatten +from .module_tracker import ModuleTracker +from typing import Any, TypeVar +from collections.abc import Callable +from collections.abc import Iterator +from typing_extensions import ParamSpec +from collections import defaultdict +from torch.utils._python_dispatch import TorchDispatchMode +from math import prod +from functools import wraps +import warnings + +__all__ = ["FlopCounterMode", "register_flop_formula"] + +_T = TypeVar("_T") +_P = ParamSpec("_P") + +aten = torch.ops.aten + +def get_shape(i): + if isinstance(i, torch.Tensor): + return i.shape + return i + +flop_registry: dict[Any, Any] = {} + +def shape_wrapper(f): + @wraps(f) + def nf(*args, out_val=None, **kwargs): + args, kwargs, out_shape = tree_map(get_shape, (args, kwargs, out_val)) + return f(*args, out_shape=out_shape, **kwargs) + return nf + +def register_flop_formula(targets, get_raw=False) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: + def register_fun(flop_formula: Callable[_P, _T]) -> Callable[_P, _T]: + if not get_raw: + flop_formula = shape_wrapper(flop_formula) + + def register(target) -> None: + if not isinstance(target, torch._ops.OpOverloadPacket): + raise ValueError( + f"register_flop_formula(targets): expected each target to be " + f"OpOverloadPacket (i.e. torch.ops.mylib.foo), got " + f"{target} which is of type {type(target)}") + if target in flop_registry: + raise RuntimeError(f"duplicate registrations for {target}") + flop_registry[target] = flop_formula + + # To handle allowing multiple aten_ops at once + torch.utils._pytree.tree_map_(register, targets) + + return flop_formula + + return register_fun + +@register_flop_formula(aten.mm) +def mm_flop(a_shape, b_shape, *args, out_shape=None, **kwargs) -> int: + """Count flops for matmul.""" + # Inputs should be a list of length 2. + # Inputs contains the shapes of two matrices. + m, k = a_shape + k2, n = b_shape + if k != k2: + raise AssertionError(f"matmul: inner dimensions must match (k == k2), got {k} and {k2}") + # NB(chilli): Should be 2 * k - 1 technically for FLOPs. + return m * n * 2 * k + +@register_flop_formula(aten.addmm) +def addmm_flop(self_shape, a_shape, b_shape, out_shape=None, **kwargs) -> int: + """Count flops for addmm.""" + return mm_flop(a_shape, b_shape) + +@register_flop_formula(aten.bmm) +def bmm_flop(a_shape, b_shape, out_shape=None, **kwargs) -> int: + """Count flops for the bmm operation.""" + # Inputs should be a list of length 2. + # Inputs contains the shapes of two tensor. + b, m, k = a_shape + b2, k2, n = b_shape + if b != b2: + raise AssertionError(f"bmm: batch dimensions must match (b == b2), got {b} and {b2}") + if k != k2: + raise AssertionError(f"bmm: inner dimensions must match (k == k2), got {k} and {k2}") + # NB(chilli): Should be 2 * k - 1 technically for FLOPs. + flop = b * m * n * 2 * k + return flop + +@register_flop_formula(aten.baddbmm) +def baddbmm_flop(self_shape, a_shape, b_shape, out_shape=None, **kwargs) -> int: + """Count flops for the baddbmm operation.""" + # Inputs should be a list of length 3. + # Inputs contains the shapes of three tensors. + return bmm_flop(a_shape, b_shape) + +@register_flop_formula(aten._scaled_mm) +def _scaled_mm_flop( + a_shape, + b_shape, + scale_a_shape, + scale_b_shape, + bias_shape=None, + scale_result_shape=None, + out_dtype=None, + use_fast_accum=False, + out_shape=None, + **kwargs, +) -> int: + """Count flops for _scaled_mm.""" + return mm_flop(a_shape, b_shape) + + +def conv_flop_count( + x_shape: list[int], + w_shape: list[int], + out_shape: list[int], + transposed: bool = False, +) -> int: + """Count flops for convolution. + + Note only multiplication is + counted. Computation for bias are ignored. + Flops for a transposed convolution are calculated as + flops = (x_shape[2:] * prod(w_shape) * batch_size). + Args: + x_shape (list(int)): The input shape before convolution. + w_shape (list(int)): The filter shape. + out_shape (list(int)): The output shape after convolution. + transposed (bool): is the convolution transposed + Returns: + int: the number of flops + """ + batch_size = x_shape[0] + conv_shape = (x_shape if transposed else out_shape)[2:] + c_out, c_in, *filter_size = w_shape + + """ + General idea here is that for a regular conv, for each point in the output + spatial dimension we convolve the filter with something (hence + `prod(conv_shape) * prod(filter_size)` ops). Then, this gets multiplied by + 1. batch_size, 2. the cross product of input and weight channels. + + For the transpose, it's not each point in the *output* spatial dimension but + each point in the *input* spatial dimension. + """ + # NB(chilli): I don't think this properly accounts for padding :think: + # NB(chilli): Should be 2 * c_in - 1 technically for FLOPs. + flop = prod(conv_shape) * prod(filter_size) * batch_size * c_out * c_in * 2 + return flop + +@register_flop_formula([aten.convolution, + aten._convolution, + aten.cudnn_convolution, + aten._slow_conv2d_forward, + aten.convolution_overrideable]) +def conv_flop(x_shape, w_shape, _bias, _stride, _padding, _dilation, transposed, *args, out_shape=None, **kwargs) -> int: + """Count flops for convolution.""" + # pyrefly: ignore [bad-argument-type] + return conv_flop_count(x_shape, w_shape, out_shape, transposed=transposed) + + +@register_flop_formula(aten.convolution_backward) +def conv_backward_flop( + grad_out_shape, + x_shape, + w_shape, + _bias, + _stride, + _padding, + _dilation, + transposed, + _output_padding, + _groups, + output_mask, + out_shape) -> int: + + def t(shape): + return [shape[1], shape[0]] + list(shape[2:]) + flop_count = 0 + + """ + Let's say we have a regular 1D conv + {A, B, C} [inp] + {i, j} [weight] + => (conv) + {Ai + Bj, Bi + Cj} [out] + + And as a reminder, the transposed conv of the above is + => {Ai, Aj + Bi, Bj + Ci, Cj} [transposed conv out] + + For the backwards of conv, we now have + {D, E} [grad_out] + {A, B, C} [inp] + {i, j} [weight] + + # grad_inp as conv_transpose(grad_out, weight) + Let's first compute grad_inp. To do so, we can simply look at all the + multiplications that each element of inp is involved in. For example, A is + only involved in the first element of the output (and thus only depends upon + D in grad_out), and C is only involved in the last element of the output + (and thus only depends upon E in grad_out) + + {Di, Dj + Ei, Ej} [grad_inp] + + Note that this corresponds to the below conv_transpose. This gives us the + output_mask[0] branch, which is grad_inp. + + {D, E} [inp (grad_out)] + {i, j} [weight] + => (conv_transpose) + {Di, Dj + Ei, Ej} [out (grad_inp)] + + I leave the fact that grad_inp for a transposed conv is just conv(grad_out, + weight) as an exercise for the reader. + + # grad_weight as conv(inp, grad_out) + To compute grad_weight, we again look at the terms in the output, which as + a reminder is: + => {Ai + Bj, Bi + Cj} [out] + => {D, E} [grad_out] + If we manually compute the gradient for the weights, we see it's + {AD + BE, BD + CE} [grad_weight] + + This corresponds to the below conv + {A, B, C} [inp] + {D, E} [weight (grad_out)] + => (conv) + {AD + BE, BD + CE} [out (grad_weight)] + + # grad_weight of transposed conv as conv(grad_out, inp) + As a reminder, the terms of the output of a transposed conv are: + => {Ai, Aj + Bi, Bj + Ci, Cj} [transposed conv out] + => {D, E, F, G} [grad_out] + + Manually computing the gradient for the weights, we see it's + {AD + BE + CF, AE + BF + CG} [grad_weight] + + This corresponds to the below conv + {D, E, F, G} [inp (grad_out)] + {A, B, C} [weight (inp)] + => (conv) + {AD + BE + CF, AE + BF + CG} [out (grad_weight)] + + For the full backwards formula, there are also some details involving + transpose of the batch/channel dimensions and groups, but I skip those for + the sake of brevity (and they're pretty similar to matmul backwards) + + Check [conv backwards decomposition as conv forwards] + """ + # grad_inp as conv_transpose(grad_out, weight) + if output_mask[0]: + grad_input_shape = get_shape(out_shape[0]) + flop_count += conv_flop_count(grad_out_shape, w_shape, grad_input_shape, not transposed) + + if output_mask[1]: + grad_weight_shape = get_shape(out_shape[1]) + if transposed: + # grad_weight of transposed conv as conv(grad_out, inp) + flop_count += conv_flop_count(t(grad_out_shape), t(x_shape), t(grad_weight_shape), transposed=False) + else: + # grad_weight as conv(inp, grad_out) + flop_count += conv_flop_count(t(x_shape), t(grad_out_shape), t(grad_weight_shape), transposed=False) + + return flop_count + +def sdpa_flop_count(query_shape, key_shape, value_shape): + """ + Count flops for self-attention. + + NB: We can assume that value_shape == key_shape + """ + b, h, s_q, d_q = query_shape + _b2, _h2, s_k, _d2 = key_shape + _b3, _h3, _s3, d_v = value_shape + if not b == _b2 == _b3 or not h == _h2 == _h3 or not d_q == _d2 or not s_k == _s3 or not d_q == _d2: + raise AssertionError("sdpa_flop_count: query/key/value shapes are incompatible") + total_flops = 0 + # q: [b, h, s_q, d_q] @ k: [b, h, d_q, s_k] -> scores: [b, h, s_q, s_k] + total_flops += bmm_flop((b * h, s_q, d_q), (b * h, d_q, s_k)) + # scores: [b, h, s_q, s_k] @ v: [b, h, s_k, d_v] -> out: [b, h, s_q, d_v] + total_flops += bmm_flop((b * h, s_q, s_k), (b * h, s_k, d_v)) + return total_flops + + +@register_flop_formula([aten._scaled_dot_product_efficient_attention, + aten._scaled_dot_product_flash_attention, + aten._scaled_dot_product_cudnn_attention]) +def sdpa_flop(query_shape, key_shape, value_shape, *args, out_shape=None, **kwargs) -> int: + """Count flops for self-attention.""" + # NB: We aren't accounting for causal attention here + return sdpa_flop_count(query_shape, key_shape, value_shape) + + +def _offsets_to_lengths(offsets, max_len): + """ + If the offsets tensor is fake, then we don't know the actual lengths. + In that case, we can just assume the worst case; each batch has max length. + """ + from torch._subclasses.fake_tensor import FakeTensor + from torch._subclasses.functional_tensor import FunctionalTensor + if not isinstance(offsets, (FakeTensor, FunctionalTensor)) and offsets.device.type != "meta": + return offsets.diff().tolist() + return [max_len] * (offsets.size(0) - 1) + + +def _unpack_flash_attention_nested_shapes( + *, + query, + key, + value, + grad_out=None, + cum_seq_q, + cum_seq_k, + max_q, + max_k, +) -> Iterator[tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...] | None]]: + """ + Given inputs to a flash_attention_(forward|backward) kernel, this will handle behavior for + NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for + each batch element. + + In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. + """ + if cum_seq_q is not None: + # This means we should be dealing with a Nested Jagged Tensor query. + # The inputs will have shape (sum(sequence len), heads, dimension) + # In comparison, non-Nested inputs have shape (batch, heads, sequence len, dimension) + # To deal with this, we convert to a shape of (batch, heads, max_seq_len, dimension) + # So the flops calculation in this case is an overestimate of the actual flops. + if len(key.shape) != 3: + raise AssertionError("sdpa_flop_count: expected key.shape to be 3-dimensional") + if len(value.shape) != 3: + raise AssertionError("sdpa_flop_count: expected value.shape to be 3-dimensional") + if grad_out is not None and grad_out.shape != query.shape: + raise AssertionError("sdpa_flop_count: grad_out.shape must match query.shape when provided") + _, h_q, d_q = query.shape + _, h_k, d_k = key.shape + _, h_v, d_v = value.shape + if cum_seq_q is None: + raise AssertionError("sdpa_flop_count: cum_seq_q must not be None") + if cum_seq_k is None: + raise AssertionError("sdpa_flop_count: cum_seq_k must not be None") + if cum_seq_q.shape != cum_seq_k.shape: + raise AssertionError("sdpa_flop_count: cum_seq_q and cum_seq_k must have the same shape") + seq_q_lengths = _offsets_to_lengths(cum_seq_q, max_q) + seq_k_lengths = _offsets_to_lengths(cum_seq_k, max_k) + for (seq_q_len, seq_k_len) in zip(seq_q_lengths, seq_k_lengths, strict=True): + new_query_shape = (1, h_q, seq_q_len, d_q) + new_key_shape = (1, h_k, seq_k_len, d_k) + new_value_shape = (1, h_v, seq_k_len, d_v) + new_grad_out_shape = new_query_shape if grad_out is not None else None + yield new_query_shape, new_key_shape, new_value_shape, new_grad_out_shape + return + + yield query.shape, key.shape, value.shape, grad_out.shape if grad_out is not None else None + + +def _unpack_efficient_attention_nested_shapes( + *, + query, + key, + value, + grad_out=None, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, +) -> Iterator[tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...], tuple[int, ...] | None]]: + """ + Given inputs to a efficient_attention_(forward|backward) kernel, this will handle behavior for + NestedTensor inputs by effectively unbinding the NestedTensor and yielding the shapes for + each batch element. + + In the case that this isn't a NestedTensor kernel, then it just yields the original shapes. + """ + if cu_seqlens_q is not None: + # Unlike flash_attention_forward, we get a 4D tensor instead of a 3D tensor for efficient attention. + # + # This means we should be dealing with a Nested Jagged Tensor query. + # The inputs will have shape (sum(sequence len), heads, dimension) + # In comparison, non-Nested inputs have shape (batch, heads, sequence len, dimension) + # To deal with this, we convert to a shape of (batch, heads, max_seq_len, dimension) + # So the flops calculation in this case is an overestimate of the actual flops. + if len(key.shape) != 4: + raise AssertionError("_unpack_efficient_attention_nested_shapes: expected key.shape to be 4-dimensional") + if len(value.shape) != 4: + raise AssertionError("_unpack_efficient_attention_nested_shapes: expected value.shape to be 4-dimensional") + if grad_out is not None and grad_out.shape != query.shape: + raise AssertionError("_unpack_efficient_attention_nested_shapes: grad_out.shape must match query.shape when provided") + _, _, h_q, d_q = query.shape + _, _, h_k, d_k = key.shape + _, _, h_v, d_v = value.shape + if cu_seqlens_q is None: + raise AssertionError("_unpack_efficient_attention_nested_shapes: cu_seqlens_q must not be None") + if cu_seqlens_k is None: + raise AssertionError("_unpack_efficient_attention_nested_shapes: cu_seqlens_k must not be None") + if cu_seqlens_q.shape != cu_seqlens_k.shape: + raise AssertionError("_unpack_efficient_attention_nested_shapes: " + "cu_seqlens_q and cu_seqlens_k must have the same shape") + seqlens_q = _offsets_to_lengths(cu_seqlens_q, max_seqlen_q) + seqlens_k = _offsets_to_lengths(cu_seqlens_k, max_seqlen_k) + for len_q, len_k in zip(seqlens_q, seqlens_k, strict=True): + new_query_shape = (1, h_q, len_q, d_q) + new_key_shape = (1, h_k, len_k, d_k) + new_value_shape = (1, h_v, len_k, d_v) + new_grad_out_shape = new_query_shape if grad_out is not None else None + yield new_query_shape, new_key_shape, new_value_shape, new_grad_out_shape + return + + yield query.shape, key.shape, value.shape, grad_out.shape if grad_out is not None else None + + +@register_flop_formula(aten._flash_attention_forward, get_raw=True) +def _flash_attention_forward_flop( + query, + key, + value, + cum_seq_q, + cum_seq_k, + max_q, + max_k, + *args, + out_shape=None, + **kwargs +) -> int: + """Count flops for self-attention.""" + # NB: We aren't accounting for causal attention here + # in case this is a nested tensor, we unpack the individual batch elements + # and then sum the flops per batch element + sizes = _unpack_flash_attention_nested_shapes( + query=query, + key=key, + value=value, + cum_seq_q=cum_seq_q, + cum_seq_k=cum_seq_k, + max_q=max_q, + max_k=max_k, + ) + return sum( + sdpa_flop_count(query_shape, key_shape, value_shape) + for query_shape, key_shape, value_shape, _ in sizes + ) + + +@register_flop_formula(aten._efficient_attention_forward, get_raw=True) +def _efficient_attention_forward_flop( + query, + key, + value, + bias, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + *args, + **kwargs +) -> int: + """Count flops for self-attention.""" + # NB: We aren't accounting for causal attention here + # in case this is a nested tensor, we unpack the individual batch elements + # and then sum the flops per batch element + sizes = _unpack_efficient_attention_nested_shapes( + query=query, + key=key, + value=value, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + ) + return sum( + sdpa_flop_count(query_shape, key_shape, value_shape) + for query_shape, key_shape, value_shape, _ in sizes + ) + + +def sdpa_backward_flop_count(grad_out_shape, query_shape, key_shape, value_shape): + total_flops = 0 + b, h, s_q, d_q = query_shape + _b2, _h2, s_k, _d2 = key_shape + _b3, _h3, _s3, d_v = value_shape + _b4, _h4, _s4, _d4 = grad_out_shape + if not b == _b2 == _b3 == _b4 or not h == _h2 == _h3 == _h4 or not d_q == _d2: + raise AssertionError("sdpa_backward_flop_count: batch/heads/dimension mismatch among tensors") + if not d_v == _d4 or not s_k == _s3 or not s_q == _s4: + raise AssertionError("sdpa_backward_flop_count: grad_out/value/key/query shapes are incompatible") + total_flops = 0 + # Step 1: We recompute the scores matrix. + # q: [b, h, s_q, d_q] @ k: [b, h, d_q, s_k] -> scores: [b, h, s_q, s_k] + total_flops += bmm_flop((b * h, s_q, d_q), (b * h, d_q, s_k)) + + # Step 2: We propagate the gradients through the score @ v operation. + # gradOut: [b, h, s_q, d_v] @ v: [b, h, d_v, s_k] -> gradScores: [b, h, s_q, s_k] + total_flops += bmm_flop((b * h, s_q, d_v), (b * h, d_v, s_k)) + # scores: [b, h, s_k, s_q] @ gradOut: [b, h, s_q, d_v] -> gradV: [b, h, s_k, d_v] + total_flops += bmm_flop((b * h, s_k, s_q), (b * h, s_q, d_v)) + + # Step 3: We propagate th gradients through the k @ v operation + # gradScores: [b, h, s_q, s_k] @ k: [b, h, s_k, d_q] -> gradQ: [b, h, s_q, d_q] + total_flops += bmm_flop((b * h, s_q, s_k), (b * h, s_k, d_q)) + # q: [b, h, d_q, s_q] @ gradScores: [b, h, s_q, s_k] -> gradK: [b, h, d_q, s_k] + total_flops += bmm_flop((b * h, d_q, s_q), (b * h, s_q, s_k)) + return total_flops + + +@register_flop_formula([aten._scaled_dot_product_efficient_attention_backward, + aten._scaled_dot_product_flash_attention_backward, + aten._scaled_dot_product_cudnn_attention_backward]) +def sdpa_backward_flop(grad_out_shape, query_shape, key_shape, value_shape, *args, out_shape=None, **kwargs) -> int: + """Count flops for self-attention backward.""" + return sdpa_backward_flop_count(grad_out_shape, query_shape, key_shape, value_shape) + +@register_flop_formula(aten._flash_attention_backward, get_raw=True) +def _flash_attention_backward_flop( + grad_out, + query, + key, + value, + out, # named _out_shape to avoid kwarg collision with out_shape created in wrapper + logsumexp, + cum_seq_q, + cum_seq_k, + max_q, + max_k, + *args, + **kwargs, +) -> int: + # in case this is a nested tensor, we unpack the individual batch elements + # and then sum the flops per batch element + shapes = _unpack_flash_attention_nested_shapes( + query=query, + key=key, + value=value, + grad_out=grad_out, + cum_seq_q=cum_seq_q, + cum_seq_k=cum_seq_k, + max_q=max_q, + max_k=max_k, + ) + return sum( + sdpa_backward_flop_count(grad_out_shape, query_shape, key_shape, value_shape) + for query_shape, key_shape, value_shape, grad_out_shape in shapes + ) + + +@register_flop_formula(aten._efficient_attention_backward, get_raw=True) +def _efficient_attention_backward_flop( + grad_out, + query, + key, + value, + bias, + out, # named _out to avoid kwarg collision with out created in wrapper + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + *args, + **kwargs, +) -> int: + # in case this is a nested tensor, we unpack the individual batch elements + # and then sum the flops per batch element + shapes = _unpack_efficient_attention_nested_shapes( + query=query, + key=key, + value=value, + grad_out=grad_out, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + ) + return sum( + sdpa_backward_flop_count(grad_out_shape, query_shape, key_shape, value_shape) + for query_shape, key_shape, value_shape, grad_out_shape in shapes + ) + + +flop_registry = { + aten.mm: mm_flop, + aten.addmm: addmm_flop, + aten.bmm: bmm_flop, + aten.baddbmm: baddbmm_flop, + aten._scaled_mm: _scaled_mm_flop, + aten.convolution: conv_flop, + aten._convolution: conv_flop, + aten.cudnn_convolution: conv_flop, + aten.convolution_overrideable: conv_flop, + aten._slow_conv2d_forward: conv_flop, + aten.convolution_backward: conv_backward_flop, + aten._scaled_dot_product_efficient_attention: sdpa_flop, + aten._scaled_dot_product_flash_attention: sdpa_flop, + aten._scaled_dot_product_cudnn_attention: sdpa_flop, + aten._scaled_dot_product_efficient_attention_backward: sdpa_backward_flop, + aten._scaled_dot_product_flash_attention_backward: sdpa_backward_flop, + aten._scaled_dot_product_cudnn_attention_backward: sdpa_backward_flop, + aten._flash_attention_forward: _flash_attention_forward_flop, + aten._efficient_attention_forward: _efficient_attention_forward_flop, + aten._flash_attention_backward: _flash_attention_backward_flop, + aten._efficient_attention_backward: _efficient_attention_backward_flop, +} + +def normalize_tuple(x): + if not isinstance(x, tuple): + return (x,) + return x + + +# Define the suffixes for different orders of magnitude +suffixes = ["", "K", "M", "B", "T"] +# Thanks BingChat! +def get_suffix_str(number): + # Find the index of the appropriate suffix based on the number of digits + # with some additional overflow. + # i.e. 1.01B should be displayed as 1001M, not 1.001B + index = max(0, min(len(suffixes) - 1, (len(str(number)) - 2) // 3)) + return suffixes[index] + +def convert_num_with_suffix(number, suffix): + index = suffixes.index(suffix) + # Divide the number by 1000^index and format it to two decimal places + value = f"{number / 1000 ** index:.3f}" + # Return the value and the suffix as a string + return value + suffixes[index] + +def convert_to_percent_str(num, denom) -> str: + if denom == 0: + return "0%" + return f"{num / denom:.2%}" + +def _pytreeify_preserve_structure(f): + @wraps(f) + def nf(args): + flat_args, spec = tree_flatten(args) + out = f(*flat_args) + return tree_unflatten(out, spec) + + return nf + + +class FlopCounterMode: + """ + ``FlopCounterMode`` is a context manager that counts the number of flops within its context. + + It does this using a ``TorchDispatchMode``. + + It also supports hierarchical output by passing a module (or list of + modules) to FlopCounterMode on construction. If you do not need hierarchical + output, you do not need to use it with a module. + + Example usage + + .. code-block:: python + + mod = ... + with FlopCounterMode(mod) as flop_counter: + mod.sum().backward() + + """ + + def __init__( + self, + mods: torch.nn.Module | list[torch.nn.Module] | None = None, + depth: int = 2, + display: bool = True, + custom_mapping: dict[Any, Any] | None = None) -> None: + super().__init__() + self.flop_counts: dict[str, dict[Any, int]] = defaultdict(lambda: defaultdict(int)) + self.depth = depth + self.display = display + self.mode: _FlopCounterMode | None = None + if custom_mapping is None: + custom_mapping = {} + if mods is not None: + warnings.warn("mods argument is not needed anymore, you can stop passing it", stacklevel=2) + self.flop_registry = { + **flop_registry, + **{k: v if getattr(v, "_get_raw", False) else shape_wrapper(v) for k, v in custom_mapping.items()} + } + self.mod_tracker = ModuleTracker() + + def get_total_flops(self) -> int: + return sum(self.flop_counts['Global'].values()) + + def get_flop_counts(self) -> dict[str, dict[Any, int]]: + """Return the flop counts as a dictionary of dictionaries. + + The outer + dictionary is keyed by module name, and the inner dictionary is keyed by + operation name. + + Returns: + Dict[str, Dict[Any, int]]: The flop counts as a dictionary. + """ + return {k: dict(v) for k, v in self.flop_counts.items()} + + def get_table(self, depth=None): + if depth is None: + depth = self.depth + if depth is None: + depth = 999999 + + + import tabulate + + tabulate.PRESERVE_WHITESPACE = True + header = ["Module", "FLOP", "% Total"] + values = [] + global_flops = self.get_total_flops() + global_suffix = get_suffix_str(global_flops) + is_global_subsumed = False + + def process_mod(mod_name, depth): + nonlocal is_global_subsumed + + total_flops = sum(self.flop_counts[mod_name].values()) + + is_global_subsumed |= total_flops >= global_flops + + padding = " " * depth + values = [] + values.append([ + padding + mod_name, + convert_num_with_suffix(total_flops, global_suffix), + convert_to_percent_str(total_flops, global_flops) + ]) + for k, v in self.flop_counts[mod_name].items(): + values.append([ + padding + " - " + str(k), + convert_num_with_suffix(v, global_suffix), + convert_to_percent_str(v, global_flops) + ]) + return values + + for mod in sorted(self.flop_counts.keys()): + if mod == 'Global': + continue + mod_depth = mod.count(".") + 1 + if mod_depth > depth: + continue + + cur_values = process_mod(mod, mod_depth - 1) + values.extend(cur_values) + + # We do a bit of messing around here to only output the "Global" value + # if there are any FLOPs in there that aren't already fully contained by + # a module. + if 'Global' in self.flop_counts and not is_global_subsumed: + for value in values: + value[0] = " " + value[0] + + values = process_mod('Global', 0) + values + + if len(values) == 0: + values = [["Global", "0", "0%"]] + + return tabulate.tabulate(values, headers=header, colalign=("left", "right", "right")) + + # NB: This context manager is NOT reentrant + def __enter__(self): + self.flop_counts.clear() + self.mod_tracker.__enter__() + self.mode = _FlopCounterMode(self) + self.mode.__enter__() + return self + + def __exit__(self, *args): + if self.mode is None: + raise AssertionError("Internal error: FlopCounter.__exit__ called but mode is None") + b = self.mode.__exit__(*args) + self.mode = None # break cycles + self.mod_tracker.__exit__() + if self.display: + print(self.get_table(self.depth)) + return b + + def _count_flops(self, func_packet, out, args, kwargs): + if func_packet in self.flop_registry: + flop_count_func = self.flop_registry[func_packet] + flop_count = flop_count_func(*args, **kwargs, out_val=out) # type: ignore[operator] + for par in set(self.mod_tracker.parents): + self.flop_counts[par][func_packet] += flop_count + + return out + +class _FlopCounterMode(TorchDispatchMode): + supports_higher_order_operators = True + + def __init__(self, counter: FlopCounterMode) -> None: + self.counter = counter + + def _execute_with_isolated_flop_counting(self, branch_fn, operands): + """Execute a branch function and capture its FLOP counts without + affecting self.counter.flop_counts + + Args: + branch_fn: The branch function to execute + operands: Arguments to pass to the branch function + + Returns: + Tuple of (result, flop_counts) where result is the branch output + and flop_counts is a copy of the FLOP counts after execution + """ + import copy + checkpointed_flop_counts = copy.copy(self.counter.flop_counts) + with self: + result = branch_fn(*operands) + flop_counts = copy.copy(self.counter.flop_counts) + self.counter.flop_counts = checkpointed_flop_counts + return result, flop_counts + + def _handle_higher_order_ops(self, func, types, args, kwargs): + if func is not torch.ops.higher_order.cond: + return NotImplemented + + # The flop counter for cond counts the upper bound of flops. + # For example, if a matmul is executed 2 times in true branch + # but only 1 time in the false branch, the flop counter will + # record the larger number of flops, i.e. 2 times. + if func is torch.ops.higher_order.cond: + + pred, true_branch, false_branch, operands = args + # Step 1: Count flops for true branch and false branch separately + true_out, true_flop_counts = self._execute_with_isolated_flop_counting( + true_branch, operands + ) + if true_out is NotImplemented: + return NotImplemented + + false_out, false_flop_counts = self._execute_with_isolated_flop_counting( + false_branch, operands + ) + if false_out is NotImplemented: + return NotImplemented + + # Step 2: merge flop counts + all_mod_keys = set(true_flop_counts.keys()) | set(false_flop_counts.keys()) + merged_flop_counts = {} + for outer_key in all_mod_keys: + true_func_counts = true_flop_counts[outer_key] + false_func_counts = false_flop_counts[outer_key] + + merged_func_counts = {} + all_func_keys = set(true_func_counts.keys()) | set(false_func_counts.keys()) + + for func_key in all_func_keys: + true_val = true_func_counts.get(func_key, 0) + false_val = false_func_counts.get(func_key, 0) + merged_func_counts[func_key] = max(true_val, false_val) + + merged_flop_counts[outer_key] = merged_func_counts + + # Step 3: update the counter with merged counts + for outer_key, inner_dict in merged_flop_counts.items(): + self.counter.flop_counts[outer_key].update(inner_dict) + + # It doesn't matter which one we return since true_fn and false_fn return + # output with the same structure. + return true_out + + def __torch_dispatch__(self, func, types, args=(), kwargs=None): + kwargs = kwargs if kwargs else {} + + # Skip ops from non-standard dispatch_sizes_strides_policy such as NJT + if func in {torch.ops.aten.sym_is_contiguous.default, + torch.ops.aten.is_contiguous.default, + torch.ops.aten.is_contiguous.memory_format, + torch.ops.aten.is_strides_like_format.default, + torch.ops.aten.is_non_overlapping_and_dense.default, + torch.ops.aten.size.default, + torch.ops.aten.sym_size.default, + torch.ops.aten.stride.default, + torch.ops.aten.sym_stride.default, + torch.ops.aten.storage_offset.default, + torch.ops.aten.sym_storage_offset.default, + torch.ops.aten.numel.default, + torch.ops.aten.sym_numel.default, + torch.ops.aten.dim.default, + torch.ops.prim.layout.default}: + + return NotImplemented + + if isinstance(func, torch._ops.HigherOrderOperator): + return self._handle_higher_order_ops(func, types, args, kwargs) + + # If we don't have func in flop_registry, see if it can decompose + if func not in self.counter.flop_registry and func is not torch.ops.prim.device.default: + with self: + r = func.decompose(*args, **kwargs) + if r is not NotImplemented: + return r + + # no further decomposition; execute & count flops + out = func(*args, **kwargs) + return self.counter._count_flops(func._overloadpacket, out, args, kwargs) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/hooks.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/hooks.py new file mode 100644 index 0000000000000000000000000000000000000000..8e89d3ec9b3a089e7a129c11e0a54309e35bfa18 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/hooks.py @@ -0,0 +1,258 @@ +# mypy: allow-untyped-defs +import torch +from collections import OrderedDict +import weakref +import warnings +from typing import Any + +__all__ = ["RemovableHandle", "unserializable_hook", "warn_if_has_hooks", "BackwardHook"] + +class RemovableHandle: + r""" + A handle which provides the capability to remove a hook. + + Args: + hooks_dict (dict): A dictionary of hooks, indexed by hook ``id``. + extra_dict (Union[dict, List[dict]]): An additional dictionary or list of + dictionaries whose keys will be deleted when the same keys are + removed from ``hooks_dict``. + """ + + id: int + next_id: int = 0 + + def __init__(self, hooks_dict: Any, *, extra_dict: Any = None) -> None: + self.hooks_dict_ref = weakref.ref(hooks_dict) + self.id = RemovableHandle.next_id + RemovableHandle.next_id += 1 + + self.extra_dict_ref: tuple = () + if isinstance(extra_dict, dict): + self.extra_dict_ref = (weakref.ref(extra_dict),) + elif isinstance(extra_dict, list): + self.extra_dict_ref = tuple(weakref.ref(d) for d in extra_dict) + + def remove(self) -> None: + hooks_dict = self.hooks_dict_ref() + if hooks_dict is not None and self.id in hooks_dict: + del hooks_dict[self.id] + + for ref in self.extra_dict_ref: + extra_dict = ref() + if extra_dict is not None and self.id in extra_dict: + del extra_dict[self.id] + + def __getstate__(self): + if self.extra_dict_ref is None: + return (self.hooks_dict_ref(), self.id) + else: + return (self.hooks_dict_ref(), self.id, tuple(ref() for ref in self.extra_dict_ref)) + + def __setstate__(self, state) -> None: + if state[0] is None: + # create a dead reference + self.hooks_dict_ref = weakref.ref(OrderedDict()) + else: + self.hooks_dict_ref = weakref.ref(state[0]) + self.id = state[1] + RemovableHandle.next_id = max(RemovableHandle.next_id, self.id + 1) + + if len(state) < 3 or state[2] is None: + self.extra_dict_ref = () + else: + self.extra_dict_ref = tuple(weakref.ref(d) for d in state[2]) + + def __enter__(self) -> "RemovableHandle": + return self + + def __exit__(self, type: Any, value: Any, tb: Any) -> None: + self.remove() + + +def unserializable_hook(f): + """ + Mark a function as an unserializable hook with this decorator. + + This suppresses warnings that would otherwise arise if you attempt + to serialize a tensor that has a hook. + """ + f.__torch_unserializable__ = True + return f + + +def warn_if_has_hooks(tensor) -> None: + if tensor._backward_hooks: + for k in tensor._backward_hooks: + hook = tensor._backward_hooks[k] + if not hasattr(hook, "__torch_unserializable__"): + warnings.warn(f"backward hook {repr(hook)} on tensor will not be " + "serialized. If this is expected, you can " + "decorate the function with @torch.utils.hooks.unserializable_hook " + "to suppress this warning", stacklevel=2) + +class BackwardHook: + """ + A wrapper class to implement nn.Module backward hooks. + + It handles: + - Ignoring non-Tensor inputs and replacing them by None before calling the user hook + - Generating the proper Node to capture a set of Tensor's gradients + - Linking the gradients captures for the outputs with the gradients captured for the input + - Calling the user hook once both output and input gradients are available + """ + + def __init__(self, module, user_hooks, user_pre_hooks) -> None: + self.user_hooks = user_hooks + self.user_pre_hooks = user_pre_hooks + self.module = module + + self.grad_outputs = None + self.n_outputs = -1 + self.output_tensors_index = None + self.n_inputs = -1 + self.input_tensors_index = None + + def _pack_with_none(self, indices, values, size): + res = [None] * size + for idx, val in zip(indices, values, strict=True): + res[idx] = val + + return tuple(res) + + def _unpack_none(self, indices, values): + res = [values[idx] for idx in indices] + + return tuple(res) + + def _set_user_hook(self, grad_fn) -> None: + def hook(grad_input, _): + if self.grad_outputs is None: + # This happens because the gradient in your nn.Module flows to + # the Module's input without " passing through the Module's + # output, e.g. when you're doing double backward. + return + res = self._pack_with_none(self.input_tensors_index, grad_input, self.n_inputs) + + for hook in self.user_hooks: + out = hook(self.module, res, self.grad_outputs) + + if out is None: + continue + + if len(out) != len(res): + raise RuntimeError("Backward hook returned an invalid number of grad_input, " + f"got {len(out)}, but expected {len(res)}") + + res = out + + # pyrefly: ignore [bad-assignment] + self.grad_outputs = None + + return self._unpack_none(self.input_tensors_index, res) + + grad_fn.register_hook(hook) + + def _apply_on_tensors(self, fn, args): + # Can be used to apply the given function to the tensors contained in the + # args. Will return updated args and the tensors indices + tensors_idx = [] + tensors = [] + + requires_grad = False + for i, arg in enumerate(args): + if isinstance(arg, torch.Tensor): + tensors_idx.append(i) + tensors.append(arg) + requires_grad |= arg.requires_grad + + if not (requires_grad and torch.is_grad_enabled()): + return args, None + + new_tensors = torch.nn.modules._functions.BackwardHookFunction.apply(*tensors) + if len(new_tensors) == 0: + raise RuntimeError("Cannot set Module backward hook for a Module with no input Tensors.") + + grad_fns = [t.grad_fn for t in new_tensors if t.grad_fn is not None and t.grad_fn.name() == "BackwardHookFunctionBackward"] + if len(grad_fns) == 0: + raise RuntimeError("Error while setting up backward hooks. Please open " + "an issue with a code sample to reproduce this.") + + fn(grad_fns[0]) + + arg_list = list(args) + for idx, val in zip(tensors_idx, new_tensors, strict=True): + arg_list[idx] = val + + if type(args) is tuple: + out = tuple(arg_list) + else: + out = type(args)(*arg_list) + return out, tensors_idx + + def setup_input_hook(self, args): + def fn(grad_fn) -> None: + self._set_user_hook(grad_fn) + + res, input_idx = self._apply_on_tensors(fn, args) + self.n_inputs = len(args) + self.input_tensors_index = input_idx + return res + + def setup_output_hook(self, args): + def fn(grad_fn) -> None: + def hook(_, grad_output): + self.grad_outputs = self._pack_with_none(self.output_tensors_index, + grad_output, + self.n_outputs) + + if self.user_pre_hooks: + expected_len = len(self.grad_outputs) + for user_pre_hook in self.user_pre_hooks: + hook_grad_outputs = user_pre_hook(self.module, self.grad_outputs) + if hook_grad_outputs is None: + continue + + actual_len = len(hook_grad_outputs) + if actual_len != expected_len: + raise RuntimeError("Backward pre hook returned an invalid number of grad_output, " + f"got {actual_len}, but expected {expected_len}") + self.grad_outputs = hook_grad_outputs + + # We need to be able to clear self.grad_outputs but also return it + local_grad_outputs = self.grad_outputs + + # Special case if no input required gradients, this hook should call the user + # hook directly + if self.input_tensors_index is None: + warnings.warn("Full backward hook is firing when gradients are computed " + "with respect to module outputs since no inputs require gradients. See " + "https://docs.pytorch.org/docs/main/generated/torch.nn.Module.html#torch.nn.Module.register_full_backward_hook " # noqa: B950 + "for more details.", + stacklevel=5) + grad_inputs = self._pack_with_none([], [], self.n_inputs) + for user_hook in self.user_hooks: + res = user_hook(self.module, grad_inputs, self.grad_outputs) + if res is not None and not (isinstance(res, tuple) and all(el is None for el in res)): + raise RuntimeError("Backward hook for Modules where no input requires " + "gradient should always return None or None for all gradients.") + self.grad_outputs = None + + if local_grad_outputs is not None: + if self.output_tensors_index is None: + raise AssertionError("output_tensors_index should not be None when grad_outputs is not None") + return tuple(local_grad_outputs[i] for i in self.output_tensors_index) + + grad_fn.register_hook(hook) + + is_tuple = True + if not isinstance(args, tuple): + args = (args,) + is_tuple = False + + res, output_idx = self._apply_on_tensors(fn, args) + self.n_outputs = len(args) + self.output_tensors_index = output_idx + + if not is_tuple: + res = res[0] + return res diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/mkldnn.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/mkldnn.py new file mode 100644 index 0000000000000000000000000000000000000000..11bb4e442b2960a601c3c6c66c5e326ac3c9c166 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/mkldnn.py @@ -0,0 +1,238 @@ +# mypy: allow-untyped-defs +import torch + + +class MkldnnLinear(torch.jit.ScriptModule): + def __init__(self, dense_module, dtype) -> None: + super().__init__() + self.register_buffer('weight', dense_module.weight.to_mkldnn(dtype)) + if dense_module.bias is not None: + # Bias can be fp32 or bf16 for OneDNN bf16 path, but for good accuracy, + # we use fp32 dtype. + self.register_buffer('bias', dense_module.bias.to_mkldnn()) + else: + # TODO: Remove this once ScriptModule supports registering None buffer + self.register_buffer( + 'bias', + torch.zeros([dense_module.weight.size(0)], dtype=torch.float).to_mkldnn()) + + @torch.jit.script_method + def __getstate__(self): + return (self.weight.to_dense(), self.bias.to_dense(), self.training) + + @torch.jit.script_method + def __setstate__(self, state): + self.weight = state[0].to_mkldnn() + self.bias = state[1].to_mkldnn() + self.training = state[2] + + @torch.jit.script_method + def forward(self, x): + x_mkldnn = x if x.is_mkldnn else x.to_mkldnn() + y_mkldnn = torch._C._nn.mkldnn_linear(x_mkldnn, self.weight, self.bias) + y = y_mkldnn if x.is_mkldnn else y_mkldnn.to_dense() + return y + + +class _MkldnnConvNd(torch.jit.ScriptModule): + """Common base of MkldnnConv1d and MkldnnConv2d.""" + + __constants__ = ['stride', 'padding', 'dilation', 'groups'] + + def __init__(self, dense_module) -> None: + super().__init__() + + self.stride = dense_module.stride + self.padding = dense_module.padding + self.dilation = dense_module.dilation + self.groups = dense_module.groups + + if dense_module.bias is not None: + self.register_buffer('bias', dense_module.bias.to_mkldnn()) + else: + # Bias can be fp32 or bf16 for OneDNN bf16 path, but for good accuracy, + # we use fp32 dtype. + # TODO: Remove this once ScriptModule supports registering None buffer + self.register_buffer( + 'bias', + torch.zeros([dense_module.weight.size(0)], dtype=torch.float).to_mkldnn()) + + @torch.jit.script_method + def __getstate__(self): + return (self.weight.to_dense(), self.bias.to_dense(), self.training) + + @torch.jit.script_method + def forward(self, x): + return torch.mkldnn_convolution( + x, + self.weight, + self.bias, + self.padding, + self.stride, + self.dilation, + self.groups) + + +class MkldnnConv1d(_MkldnnConvNd): + def __init__(self, dense_module, dtype) -> None: + super().__init__(dense_module) + + self.register_buffer('weight', dense_module.weight.to_mkldnn(dtype)) + + @torch.jit.script_method + def __setstate__(self, state): + self.weight = state[0].to_mkldnn() + self.bias = state[1].to_mkldnn() + self.training = state[2] + + +class MkldnnConv2d(_MkldnnConvNd): + def __init__(self, dense_module, dtype) -> None: + super().__init__(dense_module) + + self.register_buffer('weight', torch._C._nn.mkldnn_reorder_conv2d_weight( + dense_module.weight.to_mkldnn(dtype), + self.padding, + self.stride, + self.dilation, + self.groups)) + + @torch.jit.script_method + def __setstate__(self, state): + self.weight = torch._C._nn.mkldnn_reorder_conv2d_weight( + state[0].to_mkldnn(), + self.padding, + self.stride, + self.dilation, + self.groups) + self.bias = state[1].to_mkldnn() + self.training = state[2] + +class MkldnnConv3d(_MkldnnConvNd): + def __init__(self, dense_module, dtype) -> None: + super().__init__(dense_module) + + self.register_buffer('weight', torch._C._nn.mkldnn_reorder_conv3d_weight( + dense_module.weight.to_mkldnn(dtype), + self.padding, + self.stride, + self.dilation, + self.groups)) + + @torch.jit.script_method + def __setstate__(self, state): + self.weight = torch._C._nn.mkldnn_reorder_conv3d_weight( + state[0].to_mkldnn(), + self.padding, + self.stride, + self.dilation, + self.groups) + self.bias = state[1].to_mkldnn() + self.training = state[2] + + +class MkldnnBatchNorm(torch.jit.ScriptModule): + __constants__ = ['exponential_average_factor', 'eps'] + + def __init__(self, dense_module) -> None: + super().__init__() + + if dense_module.training: + raise AssertionError("Only support eval mode batchnorm for mkldnn path now") + if not dense_module.track_running_stats: + raise AssertionError("Only support track_running_stats=True for mkldnn path now") + if not dense_module.affine: + raise AssertionError("Only support affine=True for mkldnn path now") + + if dense_module.momentum is None: + self.exponential_average_factor = 0.0 + else: + self.exponential_average_factor = dense_module.momentum + self.eps = dense_module.eps + + self.register_buffer('weight', dense_module.weight.to_mkldnn()) + self.register_buffer('bias', dense_module.bias.to_mkldnn()) + self.register_buffer('running_mean', dense_module.running_mean.to_mkldnn()) + self.register_buffer('running_var', dense_module.running_var.to_mkldnn()) + + @torch.jit.script_method + def __getstate__(self): + weight = self.weight.to_dense() + bias = self.bias.to_dense() + running_mean = self.running_mean.to_dense() + running_var = self.running_var.to_dense() + return (weight, bias, running_mean, running_var, self.training) + + @torch.jit.script_method + def __setstate__(self, state): + self.weight = state[0].to_mkldnn() + self.bias = state[1].to_mkldnn() + self.running_mean = state[2].to_mkldnn() + self.running_var = state[3].to_mkldnn() + self.training = state[4] + + @torch.jit.script_method + def forward(self, x): + return torch.batch_norm( + x, + self.weight, + self.bias, + self.running_mean, + self.running_var, + False, # training + self.exponential_average_factor, + self.eps, + False, # cuda_enabled + ) + +class MkldnnPrelu(torch.jit.ScriptModule): + def __init__(self, dense_module, dtype) -> None: + super().__init__() + self.register_buffer('weight', dense_module.weight.to_mkldnn(dtype)) + + @torch.jit.script_method + def __getstate__(self): + return (self.weight.to_dense(), self.training) + + @torch.jit.script_method + def __setstate__(self, state): + self.weight = state[0].to_mkldnn() + self.training = state[1] + + @torch.jit.script_method + def forward(self, x): + x_mkldnn = x if x.is_mkldnn else x.to_mkldnn() + y_mkldnn = torch.prelu(x_mkldnn, self.weight) + y = y_mkldnn if x.is_mkldnn else y_mkldnn.to_dense() + return y + +def to_mkldnn(module, dtype=torch.float): + if dtype not in (torch.float, torch.bfloat16, torch.half): + raise AssertionError("MKLDNN only support float, bfloat16, and half path now") + + + def m_fn(m, d): + if isinstance(m, torch.nn.Linear): + return MkldnnLinear(m, d) + elif isinstance(m, torch.nn.Conv1d): + return MkldnnConv1d(m, d) + elif isinstance(m, torch.nn.Conv2d): + return MkldnnConv2d(m, d) + elif isinstance(m, torch.nn.Conv3d): + return MkldnnConv3d(m, d) + elif isinstance(m, (torch.nn.BatchNorm2d, torch.nn.BatchNorm3d)): + # For batchnorm bf16 path, OneDNN requires weight and bias need fp32 dtype. + # so it doesn't need dtype argument. + return MkldnnBatchNorm(m) + elif isinstance(m, torch.nn.PReLU): + return MkldnnPrelu(m, d) + else: + return m + + def m_fn_rec(m, d): + new_m = m_fn(m, d) + for name, sub_m in m.named_children(): + setattr(new_m, name, m_fn_rec(sub_m, d)) + return new_m + + return m_fn_rec(module, dtype) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/mobile_optimizer.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/mobile_optimizer.py new file mode 100644 index 0000000000000000000000000000000000000000..1ad0a65204a4733323e7ed29a51403aa47556bbd --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/mobile_optimizer.py @@ -0,0 +1,135 @@ +# mypy: allow-untyped-defs +"""This module contains utility method for mobile model optimization and lint.""" + +import torch +from enum import Enum +from torch._C import _MobileOptimizerType as MobileOptimizerType +from typing import AnyStr + +class LintCode(Enum): + BUNDLED_INPUT = 1 + REQUIRES_GRAD = 2 + DROPOUT = 3 + BATCHNORM = 4 + +def optimize_for_mobile( + script_module: torch.jit.ScriptModule, + optimization_blocklist: set[MobileOptimizerType] | None = None, + preserved_methods: list[AnyStr] | None = None, + backend: str = 'CPU') -> torch.jit.RecursiveScriptModule: + """ + Optimize a torch script module for mobile deployment. + + Args: + script_module: An instance of torch script module with type of ScriptModule. + optimization_blocklist: A set with type of MobileOptimizerType. When set is not passed, + optimization method will run all the optimizer pass; otherwise, optimizer + method will run the optimization pass that is not included inside optimization_blocklist. + preserved_methods: A list of methods that needed to be preserved when freeze_module pass is invoked + backend: Device type to use for running the result model ('CPU'(default), 'Vulkan' or 'Metal'). + Returns: + A new optimized torch script module + """ + if not isinstance(script_module, torch.jit.ScriptModule): + raise TypeError( + f'Got {type(script_module)}, but ScriptModule is expected.') + + if optimization_blocklist is None: + optimization_blocklist = set() + + if preserved_methods is None: + preserved_methods = [] + + # Convert potential byte arrays into strings (if there is any) to pass type checking + # Here we use a new name as assigning it back to preserved_methods will invoke + # mypy errors (i.e. List[AnyStr] = List[str]) + preserved_methods_str: list[str] = [str(method) for method in preserved_methods] + + bundled_inputs_attributes = _get_bundled_inputs_preserved_attributes(script_module, preserved_methods_str) + if all(hasattr(script_module, method) for method in bundled_inputs_attributes): + preserved_methods_str = list(set(preserved_methods_str + bundled_inputs_attributes)) + + non_exist_methods = [method for method in preserved_methods_str if not hasattr(script_module, method)] + if non_exist_methods: + raise AttributeError( + f"The following methods to preserve do not exist in script_module: {', '.join(non_exist_methods)}") + + backend = backend.lower() + if backend == 'cpu': + optimized_cpp_module = torch._C._jit_pass_optimize_for_mobile( + script_module._c, + optimization_blocklist, + preserved_methods_str) + elif backend == 'vulkan': + optimized_cpp_module = torch._C._jit_pass_vulkan_optimize_for_mobile( + script_module._c, + optimization_blocklist, + preserved_methods_str) + elif backend == 'metal': + optimized_cpp_module = torch._C._jit_pass_metal_optimize_for_mobile(script_module._c, preserved_methods_str) + else: + raise TypeError("Unknown backend, must be one of 'CPU', 'Vulkan' or 'Metal'") + + return torch.jit._recursive.wrap_cpp_module(optimized_cpp_module) + + +def generate_mobile_module_lints(script_module: torch.jit.ScriptModule): + """ + Generate a list of lints for a given torch script module. + + Args: + script_module: An instance of torch script module with type of ScriptModule. + + Returns: + lint_map: A list of dictionary that contains modules lints + """ + if not isinstance(script_module, torch.jit.ScriptModule): + raise TypeError( + f'Got {type(script_module)}, but ScriptModule is expected.') + + lint_list = [] + + if not hasattr(script_module, "_generate_bundled_inputs_for_forward"): + lint_list.append({"name": LintCode.BUNDLED_INPUT.name, "message": "No bundled input for forward, please add bundled inputs " + "before saving the module using torch.utils.bundled_inputs.augment_model_with_bundled_inputs."}) + + for name, param in script_module.named_parameters(): + if param.requires_grad: + lint_list.append({"name": LintCode.REQUIRES_GRAD.name, "message": f"Param {name} requires grad, " + "please set torch.no_grad() to reduce memory usage and improve computation speed during " + "inference phase."}) + + op_names = torch.jit.export_opnames(script_module) + for op_name in op_names: + if "dropout" in op_name: + lint_list.append({"name": LintCode.DROPOUT.name, + "message": f"Operator {op_name} exists, remember to call eval() before " + "saving the module.and call torch.utils.mobile_optimizer.optimize_for_mobile to drop dropout " + "operator."}) + if "batch_norm" in op_name: + lint_list.append({"name": LintCode.BATCHNORM.name, + "message": f"Operator {op_name} exists, remember to call eval() before " + "saving the module and call torch.utils.mobile_optimizer.optimize_for_mobile to drop batch_norm " + "operator."}) + + return lint_list + +def _get_bundled_inputs_preserved_attributes(script_module: torch.jit.ScriptModule, preserved_methods: list[str]) -> list[str]: + + bundled_inputs_attributes = [] + # Has bundled inputs for forward + if hasattr(script_module, 'get_all_bundled_inputs'): + bundled_inputs_attributes.append('get_all_bundled_inputs') + bundled_inputs_attributes.append('get_num_bundled_inputs') + + # Bundled inputs in module after the change that introduced bundled inputs for multiple functions + if hasattr(script_module, 'get_bundled_inputs_functions_and_info'): + bundled_inputs_attributes.append('get_bundled_inputs_functions_and_info') + all_info = script_module.get_bundled_inputs_functions_and_info() + for function_name in all_info: + if function_name not in preserved_methods: + bundled_inputs_attributes.append(function_name) + bundled_inputs_attributes.append("get_all_bundled_inputs_for_" + function_name) + bundled_inputs_attributes.append("_bundled_inputs_deflated_" + function_name) + + return bundled_inputs_attributes diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/model_zoo.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/model_zoo.py new file mode 100644 index 0000000000000000000000000000000000000000..e0c6004e23ea806a2c83e12cd2998e0279e0b16f --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/model_zoo.py @@ -0,0 +1,2 @@ +# torchvision imports tqdm from here. +from torch.hub import tqdm, load_state_dict_from_url as load_url # noqa: F401 diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/module_tracker.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/module_tracker.py new file mode 100644 index 0000000000000000000000000000000000000000..7b5a8aad4dda9880c860850e42071a55ee7442e5 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/module_tracker.py @@ -0,0 +1,160 @@ +# mypy: allow-untyped-defs +import logging +import weakref +from typing import TYPE_CHECKING + +import torch +from torch.autograd.graph import register_multi_grad_hook +from torch.nn.modules.module import ( + register_module_forward_hook, + register_module_forward_pre_hook, +) +from torch.utils._pytree import tree_flatten + + +if TYPE_CHECKING: + from torch.utils.hooks import RemovableHandle + + +logger = logging.getLogger(__name__) + + +__all__ = ["ModuleTracker"] + + +class ModuleTracker: + """ + ``ModuleTracker`` is a context manager that tracks the nn.Module hierarchy during execution + so that other system can query which Module is currently being executed (or its backward is being + executed). + + You can access the ``parents`` attribute on this context manager to get the set of all the + Modules currently being executed via their fqn (fully qualified name, also used as the key within + the state_dict). + You can access the ``is_bw`` attribute to know if you are currently running in backward or not. + + Note that ``parents`` is never empty and always contains the "Global" key. The ``is_bw`` flag + will remain ``True`` after the forward until another Module is executed. If you need it to be + more accurate, please submit an issue requesting this. Adding a map from fqn to the module instance + is possible but not done yet, please submit an issue requesting this if you need it. + + Example usage + + .. code-block:: python + + mod = torch.nn.Linear(2, 2) + + with ModuleTracker() as tracker: + # Access anything during the forward pass + def my_linear(m1, m2, bias): + print(f"Current modules: {tracker.parents}") + return torch.mm(m1, m2.t()) + bias + + torch.nn.functional.linear = my_linear + + mod(torch.rand(2, 2)) + + """ + + parents: set[str] + """ + A Set containing the fqn for each module currently running their forward + """ + + def __init__(self) -> None: + self.parents = {"Global"} + self._known_modules: weakref.WeakKeyDictionary = weakref.WeakKeyDictionary() + self._seen_modules: weakref.WeakSet = weakref.WeakSet() + self._has_callback = False + self._hooks: list[RemovableHandle] = [] + + def _maybe_set_engine_callback(self) -> None: + # This assumes no concurrent calls to backward + if self._has_callback: + return + + def callback() -> None: + self.parents = {"Global"} + self._has_callback = False + + torch.autograd.Variable._execution_engine.queue_callback(callback) + self._has_callback = True + + @property + def is_bw(self): + """ + A boolean marking if this is currently running during the backward pass or not + """ + return torch._C._current_graph_task_id() != -1 + + def _get_mod_name(self, mod): + if mod not in self._known_modules: + self._known_modules[mod] = type(mod).__name__ + mod_name = self._known_modules[mod] + if mod not in self._seen_modules: + for name, submod in mod.named_children(): + self._known_modules[submod] = f"{mod_name}.{name}" + self._get_mod_name(submod) + self._seen_modules.add(mod) + return mod_name + + def _get_append_fn(self, name, is_bw): + def fn(*args) -> None: + if is_bw: + self._maybe_set_engine_callback() + if name in self.parents: + logger.info( + "The module hierarchy tracking seems to be broken as this Module was already entered. %s during %s", + name, + "backward" if is_bw else "forward", + ) + self.parents.add(name) + + return fn + + def _get_pop_fn(self, name, is_bw): + def fn(*args) -> None: + if name in self.parents: + self.parents.remove(name) + else: + logger.info( + "The Module hierarchy tracking is confused as we're exiting a Module that was never entered. %s during %s", + name, + "backward" if is_bw else "forward", + ) + + return fn + + def _fw_pre_hook(self, mod, input) -> None: + name = self._get_mod_name(mod) + self._get_append_fn(name, False)() + + args, _ = tree_flatten(input) + tensors = [a for a in args if isinstance(a, torch.Tensor) and a.requires_grad] + if tensors: + self._hooks.append( + register_multi_grad_hook(tensors, self._get_pop_fn(name, True)) + ) + + def _fw_post_hook(self, mod, input, output) -> None: + name = self._get_mod_name(mod) + self._get_pop_fn(name, False)() + + args, _ = tree_flatten(output) + tensors = [a for a in args if isinstance(a, torch.Tensor) and a.requires_grad] + if tensors: + self._hooks.append( + register_multi_grad_hook(tensors, self._get_append_fn(name, True)) + ) + + def __enter__(self): + self._fw_pre_handle = register_module_forward_pre_hook(self._fw_pre_hook) + self._fw_post_handle = register_module_forward_hook(self._fw_post_hook) + return self + + def __exit__(self, *args): + self._fw_pre_handle.remove() + self._fw_post_handle.remove() + for hook in self._hooks: + hook.remove() + self._hooks.clear() diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/show_pickle.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/show_pickle.py new file mode 100644 index 0000000000000000000000000000000000000000..269ba3fbda4230c71ff14fe0e97872f6a8c57e6d --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/show_pickle.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# mypy: allow-untyped-defs +import sys +import pickle +import struct +import pprint +import zipfile +import fnmatch +from typing import Any, IO + +__all__ = ["FakeObject", "FakeClass", "DumpUnpickler", "main"] + +class FakeObject: + def __init__(self, module, name, args) -> None: + self.module = module + self.name = name + self.args = args + # NOTE: We don't distinguish between state never set and state set to None. + self.state = None + + def __repr__(self) -> str: + state_str = "" if self.state is None else f"(state={self.state!r})" + return f"{self.module}.{self.name}{self.args!r}{state_str}" + + def __setstate__(self, state): + self.state = state + + @staticmethod + def pp_format(printer, obj, stream, indent, allowance, context, level) -> None: + if not obj.args and obj.state is None: + stream.write(repr(obj)) + return + if obj.state is None: + stream.write(f"{obj.module}.{obj.name}") + printer._format(obj.args, stream, indent + 1, allowance + 1, context, level) + return + if not obj.args: + stream.write(f"{obj.module}.{obj.name}()(state=\n") + indent += printer._indent_per_level + stream.write(" " * indent) + printer._format(obj.state, stream, indent, allowance + 1, context, level + 1) + stream.write(")") + return + raise Exception("Need to implement") # noqa: TRY002 + + +class FakeClass: + def __init__(self, module, name) -> None: + self.module = module + self.name = name + self.__new__ = self.fake_new # type: ignore[assignment] + + def __repr__(self) -> str: + return f"{self.module}.{self.name}" + + def __call__(self, *args): + return FakeObject(self.module, self.name, args) + + def fake_new(self, *args): + return FakeObject(self.module, self.name, args[1:]) + + +class DumpUnpickler(pickle._Unpickler): # type: ignore[name-defined] + def __init__( + self, + file, + *, + catch_invalid_utf8=False, + **kwargs) -> None: + super().__init__(file, **kwargs) + self.catch_invalid_utf8 = catch_invalid_utf8 + + def find_class(self, module, name): + return FakeClass(module, name) + + def persistent_load(self, pid): + return FakeObject("pers", "obj", (pid,)) + + dispatch = dict(pickle._Unpickler.dispatch) # type: ignore[attr-defined] + + # Custom objects in TorchScript are able to return invalid UTF-8 strings + # from their pickle (__getstate__) functions. Install a custom loader + # for strings that catches the decode exception and replaces it with + # a sentinel object. + def load_binunicode(self) -> None: + strlen, = struct.unpack(" sys.maxsize: + raise Exception("String too long.") # noqa: TRY002 + str_bytes = self.read(strlen) # type: ignore[attr-defined] + obj: Any + try: + obj = str(str_bytes, "utf-8", "surrogatepass") + except UnicodeDecodeError as exn: + if not self.catch_invalid_utf8: + raise + obj = FakeObject("builtin", "UnicodeDecodeError", (str(exn),)) + self.append(obj) # type: ignore[attr-defined] + dispatch[pickle.BINUNICODE[0]] = load_binunicode # type: ignore[assignment] + + @classmethod + def dump(cls, in_stream, out_stream): + value = cls(in_stream).load() + pprint.pprint(value, stream=out_stream) + return value + + +def main(argv, output_stream=None) -> int | None: + if len(argv) != 2: + # Don't spam stderr if not using stdout. + if output_stream is not None: + raise Exception("Pass argv of length 2.") # noqa: TRY002 + sys.stderr.write("usage: show_pickle PICKLE_FILE\n") + sys.stderr.write(" PICKLE_FILE can be any of:\n") + sys.stderr.write(" path to a pickle file\n") + sys.stderr.write(" file.zip@member.pkl\n") + sys.stderr.write(" file.zip@*/pattern.*\n") + sys.stderr.write(" (shell glob pattern for members)\n") + sys.stderr.write(" (only first match will be shown)\n") + return 2 + + fname = argv[1] + handle: IO[bytes] + if "@" not in fname: + with open(fname, "rb") as handle: + DumpUnpickler.dump(handle, output_stream) + else: + zfname, mname = fname.split("@", 1) + with zipfile.ZipFile(zfname) as zf: + if "*" not in mname: + with zf.open(mname) as handle: + DumpUnpickler.dump(handle, output_stream) + else: + found = False + for info in zf.infolist(): + if fnmatch.fnmatch(info.filename, mname): + with zf.open(info) as handle: + DumpUnpickler.dump(handle, output_stream) + found = True + break + if not found: + raise Exception(f"Could not find member matching {mname} in {zfname}") # noqa: TRY002 + + +if __name__ == "__main__": + # This hack works on every version of Python I've tested. + # I've tested on the following versions: + # 3.7.4 + if True: + pprint.PrettyPrinter._dispatch[FakeObject.__repr__] = FakeObject.pp_format # type: ignore[attr-defined] + + sys.exit(main(sys.argv)) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/throughput_benchmark.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/throughput_benchmark.py new file mode 100644 index 0000000000000000000000000000000000000000..d4b94e0b13a39fbe192f89e791c663b2ecf45ea2 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/throughput_benchmark.py @@ -0,0 +1,161 @@ +# mypy: allow-untyped-defs + +import torch._C + + +def format_time(time_us=None, time_ms=None, time_s=None) -> str: + """Define time formatting.""" + if sum([time_us is not None, time_ms is not None, time_s is not None]) != 1: + raise AssertionError("Expected only one of time_us, time_ms, time_s is given.") + + US_IN_SECOND = 1e6 + US_IN_MS = 1e3 + + if time_us is None: + if time_ms is not None: + time_us = time_ms * US_IN_MS + elif time_s is not None: + time_us = time_s * US_IN_SECOND + else: + raise AssertionError("Shouldn't reach here :)") + + if time_us >= US_IN_SECOND: + return f'{time_us / US_IN_SECOND:.3f}s' + if time_us >= US_IN_MS: + return f'{time_us / US_IN_MS:.3f}ms' + return f'{time_us:.3f}us' + + +class ExecutionStats: + def __init__(self, c_stats, benchmark_config) -> None: + self._c_stats = c_stats + self.benchmark_config = benchmark_config + + @property + def latency_avg_ms(self): + return self._c_stats.latency_avg_ms + + @property + def num_iters(self): + return self._c_stats.num_iters + + @property + def iters_per_second(self): + """Return total number of iterations per second across all calling threads.""" + return self.num_iters / self.total_time_seconds + + @property + def total_time_seconds(self): + return self.num_iters * ( + self.latency_avg_ms / 1000.0) / self.benchmark_config.num_calling_threads + + def __str__(self) -> str: + return '\n'.join([ + "Average latency per example: " + format_time(time_ms=self.latency_avg_ms), + f"Total number of iterations: {self.num_iters}", + f"Total number of iterations per second (across all threads): {self.iters_per_second:.2f}", + "Total time: " + format_time(time_s=self.total_time_seconds) + ]) + + +class ThroughputBenchmark: + """ + This class is a wrapper around a c++ component throughput_benchmark::ThroughputBenchmark. + + This wrapper on the throughput_benchmark::ThroughputBenchmark component is responsible + for executing a PyTorch module (nn.Module or ScriptModule) under an inference + server like load. It can emulate multiple calling threads to a single module + provided. In the future we plan to enhance this component to support inter and + intra-op parallelism as well as multiple models running in a single process. + + Please note that even though nn.Module is supported, it might incur an overhead + from the need to hold GIL every time we execute Python code or pass around + inputs as Python objects. As soon as you have a ScriptModule version of your + model for inference deployment it is better to switch to using it in this + benchmark. + + Example:: + + >>> # xdoctest: +SKIP("undefined vars") + >>> from torch.utils import ThroughputBenchmark + >>> bench = ThroughputBenchmark(my_module) + >>> # Pre-populate benchmark's data set with the inputs + >>> for input in inputs: + ... # Both args and kwargs work, same as any PyTorch Module / ScriptModule + ... bench.add_input(input[0], x2=input[1]) + >>> # Inputs supplied above are randomly used during the execution + >>> stats = bench.benchmark( + ... num_calling_threads=4, + ... num_warmup_iters = 100, + ... num_iters = 1000, + ... ) + >>> print("Avg latency (ms): {}".format(stats.latency_avg_ms)) + >>> print("Number of iterations: {}".format(stats.num_iters)) + """ + + def __init__(self, module) -> None: + if isinstance(module, torch.jit.ScriptModule): + self._benchmark = torch._C.ThroughputBenchmark(module._c) + else: + self._benchmark = torch._C.ThroughputBenchmark(module) + + def run_once(self, *args, **kwargs): + """ + Given input id (input_idx) run benchmark once and return prediction. + + This is useful for testing that benchmark actually runs the module you + want it to run. input_idx here is an index into inputs array populated + by calling add_input() method. + """ + return self._benchmark.run_once(*args, **kwargs) + + def add_input(self, *args, **kwargs) -> None: + """ + Store a single input to a module into the benchmark memory and keep it there. + + During the benchmark execution every thread is going to pick up a + random input from the all the inputs ever supplied to the benchmark via + this function. + """ + self._benchmark.add_input(*args, **kwargs) + + def benchmark( + self, + num_calling_threads=1, + num_warmup_iters=10, + num_iters=100, + profiler_output_path=""): + """ + Run a benchmark on the module. + + Args: + num_warmup_iters (int): Warmup iters are used to make sure we run a module + a few times before actually measuring things. This way we avoid cold + caches and any other similar problems. This is the number of warmup + iterations for each of the thread in separate + + num_iters (int): Number of iterations the benchmark should run with. + This number is separate from the warmup iterations. Also the number is + shared across all the threads. Once the num_iters iterations across all + the threads is reached, we will stop execution. Though total number of + iterations might be slightly larger. Which is reported as + stats.num_iters where stats is the result of this function + + profiler_output_path (str): Location to save Autograd Profiler trace. + If not empty, Autograd Profiler will be enabled for the main benchmark + execution (but not the warmup phase). The full trace will be saved + into the file path provided by this argument + + + This function returns BenchmarkExecutionStats object which is defined via pybind11. + It currently has two fields: + - num_iters - number of actual iterations the benchmark have made + - avg_latency_ms - average time it took to infer on one input example in milliseconds + """ + config = torch._C.BenchmarkConfig() + config.num_calling_threads = num_calling_threads + config.num_warmup_iters = num_warmup_iters + config.num_iters = num_iters + config.profiler_output_path = profiler_output_path + c_stats = self._benchmark.benchmark(config) + return ExecutionStats(c_stats, config) diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/weak.py b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/weak.py new file mode 100644 index 0000000000000000000000000000000000000000..f71912b59f53aaf70058a498dffcbe22b3d695e3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/torch/utils/weak.py @@ -0,0 +1,367 @@ +# mypy: allow-untyped-defs +from __future__ import annotations + +import collections.abc as _collections_abc +import weakref +from collections.abc import Mapping, MutableMapping +from weakref import ref + +from torch import Tensor + + +WeakRef = ref + + +__all__ = [ + "TensorWeakRef", + "WeakIdRef", + "WeakIdKeyDictionary", + "WeakTensorKeyDictionary", +] + + +# TODO: make weakref properly thread safe following +# https://github.com/python/cpython/pull/125325 +class _IterationGuard: + # This context manager registers itself in the current iterators of the + # weak container, such as to delay all removals until the context manager + # exits. + # This technique should be relatively thread-safe (since sets are). + + def __init__(self, weakcontainer) -> None: + # Don't create cycles + self.weakcontainer = ref(weakcontainer) + + def __enter__(self): + w = self.weakcontainer() + if w is not None: + w._iterating.add(self) + return self + + def __exit__(self, e, t, b): + w = self.weakcontainer() + if w is not None: + s = w._iterating + s.remove(self) + if not s: + w._commit_removals() + + +# This file defines a variant of WeakKeyDictionary that overrides the hashing +# behavior of the key to use object identity, rather than the builtin +# __eq__/__hash__ functions. This is useful for Tensor weak keys, as their +# __eq__ implementation return a Tensor (elementwise equality), which means +# you can't use them directly with the WeakKeyDictionary in standard library. +# +# Our implementation strategy is to create a wrapper weak key object, which we +# use as a key in a stock Python dictionary. This is similar to how weakref +# implements WeakKeyDictionary, but instead of using weakref.ref as the +# wrapper, we use a custom wrapper that has different __eq__ and __hash__ +# behavior. Note that we subsequently store this weak key directly in an +# ORDINARY dictionary, since the newly constructed WeakIdKey's only use would +# be a dictionary so it would have no strong references. Ensuring that +# only live WeakIdKeys are in the map is handled by putting finalizers on the +# original key object. + + +# It is simpler to implement this with composition, but if we want to +# directly reuse the callback mechanism on weakref, we need the weakref +# and the key to be exactly the same object. Reusing the callback mechanism +# minimizes the divergence between our implementation and Lib/weakref.py +# +# NB: Prefer using this when working with weakrefs of Tensors; e.g., do +# WeakIdRef(tensor) rather than weakref.ref(tensor); it handles a number of +# easy to get wrong cases transparently for you. +class WeakIdRef(weakref.ref): + __slots__ = ["_id"] + + def __init__(self, key, callback=None) -> None: + # Unlike stock weakref, which preserves hash semantics of the + # original object but lazily defers hash calls until the first + # time the user attempts to hash the weakref, we can eagerly + # cache the id of the key as we know this is definitely the hash + # method + self._id = id(key) + super().__init__(key, callback) # type: ignore[call-arg] + + def __call__(self): + r = super().__call__() + # Special logic for Tensor PyObject resurrection + if hasattr(r, "_fix_weakref"): + r._fix_weakref() # type: ignore[union-attr] + return r + + def __hash__(self): + return self._id + + def __eq__(self, other): + # An attractive but wrong alternate implementation is to only test if + # the stored _ids match. This can lead to an ABA problem if you have: + # + # a1 = A() + # w1 = WeakIdRef(a1) + # del a1 + # a2 = A() # suppose it gets the same ID as a1 + # w2 = WeakIdRef(a2) + # print(w1 == w2) + # + # This should be False, as a1 and a2 are unrelated (and a1 is + # dead anyway) + a = self() + b = other() + if a is not None and b is not None: + return a is b + return self is other + + +# This is the same as WeakIdRef but equality is checked using hash() rather than id. +# This will be equivalent to the one above except for classes where hash is not their id. +class _WeakHashRef(weakref.ref): + __slots__ = ["_id"] + + def __init__(self, key, callback=None) -> None: + # Unlike stock weakref, which preserves hash semantics of the + # original object but lazily defers hash calls until the first + # time the user attempts to hash the weakref, we can eagerly + # cache the id of the key as we know this is definitely the hash + # method + self._id = hash(key) + super().__init__(key, callback) # type: ignore[call-arg] + + def __call__(self): + r = super().__call__() + # Special logic for Tensor PyObject resurrection + if hasattr(r, "_fix_weakref"): + r._fix_weakref() # type: ignore[union-attr] + return r + + def __hash__(self): + return self._id + + def __eq__(self, other): + # Use hash equality to determine ref equality. + # ScriptObject implements __hash__ to return the wrapped IValue's id, so + # this is equivalent to doing an identity comparison. + a = self() + b = other() + if a is not None and b is not None: + return hash(a) == hash(b) + return self is other + + +# This is directly adapted from cpython/Lib/weakref.py +class WeakIdKeyDictionary(MutableMapping): + def __init__(self, dict=None, ref_type=WeakIdRef) -> None: # CHANGED + self.data = {} + + self.ref_type = ref_type # CHANGED + + def remove(k, selfref=ref(self)) -> None: + self = selfref() + if self is not None: + if self._iterating: + self._pending_removals.append(k) + else: + try: + del self.data[k] + except KeyError: + pass + + self._remove = remove + # A list of dead weakrefs (keys to be removed) + self._pending_removals = [] + self._iterating = set() + self._dirty_len = False + if dict is not None: + self.update(dict) + + def _commit_removals(self) -> None: + # NOTE: We don't need to call this method before mutating the dict, + # because a dead weakref never compares equal to a live weakref, + # even if they happened to refer to equal objects. + # However, it means keys may already have been removed. + pop = self._pending_removals.pop + d = self.data + while True: + try: + key = pop() + except IndexError: + return + + try: + del d[key] + except KeyError: + pass + + def _scrub_removals(self) -> None: + d = self.data + self._pending_removals = [k for k in self._pending_removals if k in d] + self._dirty_len = False + + def __delitem__(self, key) -> None: + self._dirty_len = True + del self.data[self.ref_type(key)] # CHANGED + + def __getitem__(self, key): + return self.data[self.ref_type(key)] # CHANGED + + def __len__(self) -> int: + if self._dirty_len and self._pending_removals: + # self._pending_removals may still contain keys which were + # explicitly removed, we have to scrub them (see issue #21173). + self._scrub_removals() + return len(self.data) - len(self._pending_removals) + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} at {id(self):#x}>" + + def __setitem__(self, key, value) -> None: + self.data[self.ref_type(key, self._remove)] = value # CHANGED + + def copy(self): + new = WeakIdKeyDictionary() + with _IterationGuard(self): + for key, value in self.data.items(): + o = key() + if o is not None: + new[o] = value + return new + + __copy__ = copy + + def __deepcopy__(self, memo): + from copy import deepcopy + + new = self.__class__() + with _IterationGuard(self): + for key, value in self.data.items(): + o = key() + if o is not None: + new[o] = deepcopy(value, memo) + return new + + def get(self, key, default=None): + return self.data.get(self.ref_type(key), default) # CHANGED + + def __contains__(self, key) -> bool: + try: + wr = self.ref_type(key) # CHANGED + except TypeError: + return False + return wr in self.data + + def items(self): + with _IterationGuard(self): + for wr, value in self.data.items(): + key = wr() + if key is not None: + yield key, value + + def keys(self): + with _IterationGuard(self): + for wr in self.data: + obj = wr() + if obj is not None: + yield obj + + __iter__ = keys + + def values(self): + with _IterationGuard(self): + for wr, value in self.data.items(): + if wr() is not None: + yield value + + def keyrefs(self): + """Return a list of weak references to the keys. + + The references are not guaranteed to be 'live' at the time + they are used, so the result of calling the references needs + to be checked before being used. This can be used to avoid + creating references that will cause the garbage collector to + keep the keys around longer than needed. + + """ + return list(self.data) + + def popitem(self): + self._dirty_len = True + while True: + key, value = self.data.popitem() + o = key() + if o is not None: + return o, value + + # pyrefly: ignore [bad-override] + def pop(self, key, *args): + self._dirty_len = True + # pyrefly: ignore [not-iterable] + return self.data.pop(self.ref_type(key), *args) # CHANGED + + def setdefault(self, key, default=None): + return self.data.setdefault( + self.ref_type(key, self._remove), default + ) # CHANGED + + def update(self, dict=None, **kwargs) -> None: # type: ignore[override] + d = self.data + if dict is not None: + if not hasattr(dict, "items"): + dict = type({})(dict) + for key, value in dict.items(): + d[self.ref_type(key, self._remove)] = value # CHANGED + if kwargs: + self.update(kwargs) + + def __ior__(self, other): + self.update(other) + return self + + def __or__(self, other): + if isinstance(other, _collections_abc.Mapping): + c = self.copy() + c.update(other) + return c + return NotImplemented + + def __ror__(self, other): + if isinstance(other, _collections_abc.Mapping): + c = self.__class__() + c.update(other) + c.update(self) + return c + return NotImplemented + + # Default Mapping equality will tests keys for equality, but + # we want to test ids for equality + def __eq__(self, other): + if not isinstance(other, Mapping): + return NotImplemented + return {id(k): v for k, v in self.items()} == { + id(k): v for k, v in other.items() + } + + +# Convenience alias +WeakTensorKeyDictionary = WeakIdKeyDictionary + + +class TensorWeakRef: + """Wrapper around a weak ref of a Tensor that handles the _fix_weakref() call required when unwrapping a Tensor weakref.""" + + ref: WeakRef[Tensor] + + def __init__(self, tensor: Tensor) -> None: + if not isinstance(tensor, Tensor): + raise AssertionError(f"expected torch.Tensor, got {type(tensor)}.") + self.ref = weakref.ref(tensor) + + def __call__(self): + out = self.ref() + if out is None: + return out + if not isinstance(out, Tensor): + raise AssertionError(f"expected torch.Tensor, got {type(out)}.") + # TODO, add _fix_weakref type binding + out._fix_weakref() # type: ignore[attr-defined] + return out diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/transformers-4.57.6.dist-info/licenses/LICENSE b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/transformers-4.57.6.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..68b7d66c97d66c58de883ed0c451af2b3183e6f3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/transformers-4.57.6.dist-info/licenses/LICENSE @@ -0,0 +1,203 @@ +Copyright 2018- The Hugging Face team. All rights reserved. + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/__init__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7324b3df72d805098a1210a1805fd1769a243077 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/__init__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/__main__.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/__main__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a346c53d67147d2542dc28213bfd27f07e03ece Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/__main__.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/_completion_classes.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/_completion_classes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d21e61ed1702af57886275d61837de116e00e7a7 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/_completion_classes.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/_completion_shared.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/_completion_shared.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dfbc9aa8f63ca9525f1fee7240b73072d32f6851 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/_completion_shared.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/_types.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/_types.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3992c0fc72c87b5ea1f07b2f95b46909ea9f5045 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/_types.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/_typing.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/_typing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f765f34134a66ab57a83ce1a9545d03b6cfc33fd Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/_typing.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/cli.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/cli.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..80d9b2cb0eae4739219b87461717eeb2c95cab93 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/cli.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/colors.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/colors.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..936776098cec7a7e8288c94fdd1c91e2cc87896b Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/colors.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/completion.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/completion.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb8fa059f72dc715f96a37b7278f25a7dfd18896 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/completion.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/core.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/core.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..22419dc798a078f51bb3047337559aa8df5460a2 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/core.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/main.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/main.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..554b368f4317070d53720bbc0678d74c1a1e656f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/main.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/models.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/models.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..699c480aa541c112e3ba1ce93543d59c8fb21fe1 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/models.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/params.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/params.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5593fc71e99c81042310fbca949b449de052adca Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/params.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/rich_utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/rich_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9a0f51d840de18314dfb16b8d5f3da5b88bb009f Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/rich_utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/testing.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/testing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a5f8d258f26c1d61c6753fdde7886afb0e25b98 Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/testing.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/utils.cpython-312.pyc b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..60b20d732943587b7193c86a61210fd437a9766d Binary files /dev/null and b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer/__pycache__/utils.cpython-312.pyc differ diff --git a/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer_slim-0.21.2.dist-info/licenses/LICENSE b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer_slim-0.21.2.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..a7694736cf37716aafec14b24aa8d6316ebe07a3 --- /dev/null +++ b/Prism/LLaDA/LLaDA_Prism/.venv/lib/python3.12/site-packages/typer_slim-0.21.2.dist-info/licenses/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2019 Sebastián Ramírez + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE.